From f1be516223f1c5ae5554f1b842d90cf429611617 Mon Sep 17 00:00:00 2001
From: Sterling-Augustine
 <56981066+Sterling-Augustine@users.noreply.github.com>
Date: Wed, 23 Oct 2024 10:26:18 -0700
Subject: [PATCH 001/693] [SandboxVectorizer] New class to actually collect and
 manage seeds (#113386)

This relands d91318b643188bb855f115b02af9532f87c787b7, with test-only
changes to make gcc-10 happy.
---
 .../SandboxVectorizer/SeedCollector.h         |  27 +++
 .../Vectorize/SandboxVectorizer/VecUtils.h    |  30 +++
 .../SandboxVectorizer/SeedCollector.cpp       |  67 +++++++
 .../SandboxVectorizer/SeedCollectorTest.cpp   | 184 ++++++++++++++++++
 4 files changed, 308 insertions(+)
 create mode 100644 llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h

diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
index a4512862136a..ed1cb8488c29 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
@@ -284,6 +284,33 @@ public:
 #endif // NDEBUG
 };
 
+class SeedCollector {
+  SeedContainer StoreSeeds;
+  SeedContainer LoadSeeds;
+  Context &Ctx;
+
+  /// \Returns the number of SeedBundle groups for all seed types.
+  /// This is to be used for limiting compilation time.
+  unsigned totalNumSeedGroups() const {
+    return StoreSeeds.size() + LoadSeeds.size();
+  }
+
+public:
+  SeedCollector(BasicBlock *BB, ScalarEvolution &SE);
+  ~SeedCollector();
+
+  iterator_range<SeedContainer::iterator> getStoreSeeds() {
+    return {StoreSeeds.begin(), StoreSeeds.end()};
+  }
+  iterator_range<SeedContainer::iterator> getLoadSeeds() {
+    return {LoadSeeds.begin(), LoadSeeds.end()};
+  }
+#ifndef NDEBUG
+  void print(raw_ostream &OS) const;
+  LLVM_DUMP_METHOD void dump() const;
+#endif
+};
+
 } // namespace llvm::sandboxir
 
 #endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_SEEDCOLLECTOR_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
new file mode 100644
index 000000000000..64f57edb3848
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
@@ -0,0 +1,30 @@
+//===- VecUtils.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Collector for SandboxVectorizer related convenience functions that don't
+// belong in other classes.
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
+
+class Utils {
+public:
+  /// \Returns the number of elements in \p Ty. That is the number of lanes if a
+  /// fixed vector or 1 if scalar. ScalableVectors have unknown size and
+  /// therefore are unsupported.
+  static int getNumElements(Type *Ty) {
+    assert(!isa<ScalableVectorType>(Ty));
+    return Ty->isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() : 1;
+  }
+  /// Returns \p Ty if scalar or its element type if vector.
+  static Type *getElementType(Type *Ty) {
+    return Ty->isVectorTy() ? cast<FixedVectorType>(Ty)->getElementType() : Ty;
+  }
+}
+
+#endif LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
index 66fac080a7b7..0d928af19020 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
@@ -22,6 +22,16 @@ namespace llvm::sandboxir {
 cl::opt<unsigned> SeedBundleSizeLimit(
     "sbvec-seed-bundle-size-limit", cl::init(32), cl::Hidden,
     cl::desc("Limit the size of the seed bundle to cap compilation time."));
+#define LoadSeedsDef "loads"
+#define StoreSeedsDef "stores"
+cl::opt<std::string> CollectSeeds(
+    "sbvec-collect-seeds", cl::init(LoadSeedsDef "," StoreSeedsDef), cl::Hidden,
+    cl::desc("Collect these seeds. Use empty for none or a comma-separated "
+             "list of '" LoadSeedsDef "' and '" StoreSeedsDef "'."));
+cl::opt<unsigned> SeedGroupsLimit(
+    "sbvec-seed-groups-limit", cl::init(256), cl::Hidden,
+    cl::desc("Limit the number of collected seeds groups in a BB to "
+             "cap compilation time."));
 
 MutableArrayRef<Instruction *> SeedBundle::getSlice(unsigned StartIdx,
                                                     unsigned MaxVecRegBits,
@@ -131,4 +141,61 @@ void SeedContainer::print(raw_ostream &OS) const {
 LLVM_DUMP_METHOD void SeedContainer::dump() const { print(dbgs()); }
 #endif // NDEBUG
 
+template <typename LoadOrStoreT> static bool isValidMemSeed(LoadOrStoreT *LSI) {
+  if (LSI->isSimple())
+    return true;
+  auto *Ty = Utils::getExpectedType(LSI);
+  // Omit types that are architecturally unvectorizable
+  if (Ty->isX86_FP80Ty() || Ty->isPPC_FP128Ty())
+    return false;
+  // Omit vector types without compile-time-known lane counts
+  if (isa<ScalableVectorType>(Ty))
+    return false;
+  if (auto *VTy = dyn_cast<FixedVectorType>(Ty))
+    return VectorType::isValidElementType(VTy->getElementType());
+  return VectorType::isValidElementType(Ty);
+}
+
+template bool isValidMemSeed<LoadInst>(LoadInst *LSI);
+template bool isValidMemSeed<StoreInst>(StoreInst *LSI);
+
+SeedCollector::SeedCollector(BasicBlock *BB, ScalarEvolution &SE)
+    : StoreSeeds(SE), LoadSeeds(SE), Ctx(BB->getContext()) {
+  // TODO: Register a callback for updating the Collector data structures upon
+  // instr removal
+
+  bool CollectStores = CollectSeeds.find(StoreSeedsDef) != std::string::npos;
+  bool CollectLoads = CollectSeeds.find(LoadSeedsDef) != std::string::npos;
+  if (!CollectStores && !CollectLoads)
+    return;
+  // Actually collect the seeds.
+  for (auto &I : *BB) {
+    if (StoreInst *SI = dyn_cast<StoreInst>(&I))
+      if (CollectStores && isValidMemSeed(SI))
+        StoreSeeds.insert(SI);
+    if (LoadInst *LI = dyn_cast<LoadInst>(&I))
+      if (CollectLoads && isValidMemSeed(LI))
+        LoadSeeds.insert(LI);
+    // Cap compilation time.
+    if (totalNumSeedGroups() > SeedGroupsLimit)
+      break;
+  }
+}
+
+SeedCollector::~SeedCollector() {
+  // TODO: Unregister the callback for updating the seed datastructures upon
+  // instr removal
+}
+
+#ifndef NDEBUG
+void SeedCollector::print(raw_ostream &OS) const {
+  OS << "=== StoreSeeds ===\n";
+  StoreSeeds.print(OS);
+  OS << "=== LoadSeeds ===\n";
+  LoadSeeds.print(OS);
+}
+
+void SeedCollector::dump() const { print(dbgs()); }
+#endif
+
 } // namespace llvm::sandboxir
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp
index 82b230d50c4e..b1b0156a4fe3 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp
@@ -22,6 +22,22 @@
 
 using namespace llvm;
 
+// TODO: gcc-10 has a bug that causes the below line not to compile due to some
+// macro-magic in gunit in combination with a class with pure-virtual
+// function. Once gcc-10 is no longer supported, replace this function with
+// something like the following:
+//
+// EXPECT_THAT(SB, testing::ElementsAre(St0, St1, St2, St3));
+static void
+ExpectThatElementsAre(sandboxir::SeedBundle &SR,
+                      llvm::ArrayRef<sandboxir::Instruction *> Contents) {
+  EXPECT_EQ(range_size(SR), Contents.size());
+  auto CI = Contents.begin();
+  if (range_size(SR) == Contents.size())
+    for (auto &S : SR)
+      EXPECT_EQ(S, *CI++);
+}
+
 struct SeedBundleTest : public testing::Test {
   LLVMContext C;
   std::unique_ptr<Module> M;
@@ -268,3 +284,171 @@ bb:
   }
   EXPECT_EQ(Cnt, 0u);
 }
+
+TEST_F(SeedBundleTest, ConsecutiveStores) {
+  // Where "Consecutive" means the stores address consecutive locations in
+  // memory, but not in program order. Check to see that the collector puts them
+  // in the proper order for vectorization.
+  parseIR(C, R"IR(
+define void @foo(ptr noalias %ptr, float %val) {
+bb:
+  %ptr0 = getelementptr float, ptr %ptr, i32 0
+  %ptr1 = getelementptr float, ptr %ptr, i32 1
+  %ptr2 = getelementptr float, ptr %ptr, i32 2
+  %ptr3 = getelementptr float, ptr %ptr, i32 3
+  store float %val, ptr %ptr0
+  store float %val, ptr %ptr2
+  store float %val, ptr %ptr1
+  store float %val, ptr %ptr3
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  DominatorTree DT(LLVMF);
+  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfo TLI(TLII);
+  DataLayout DL(M->getDataLayout());
+  LoopInfo LI(DT);
+  AssumptionCache AC(LLVMF);
+  ScalarEvolution SE(LLVMF, TLI, AC, DT, LI);
+
+  sandboxir::Context Ctx(C);
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto BB = F.begin();
+  sandboxir::SeedCollector SC(&*BB, SE);
+
+  // Find the stores
+  auto It = std::next(BB->begin(), 4);
+  // StX with X as the order by offset in memory
+  auto *St0 = &*It++;
+  auto *St2 = &*It++;
+  auto *St1 = &*It++;
+  auto *St3 = &*It++;
+
+  auto StoreSeedsRange = SC.getStoreSeeds();
+  auto &SB = *StoreSeedsRange.begin();
+  //  Expect just one vector of store seeds
+  EXPECT_EQ(range_size(StoreSeedsRange), 1u);
+  ExpectThatElementsAre(SB, {St0, St1, St2, St3});
+}
+
+TEST_F(SeedBundleTest, StoresWithGaps) {
+  parseIR(C, R"IR(
+define void @foo(ptr noalias %ptr, float %val) {
+bb:
+  %ptr0 = getelementptr float, ptr %ptr, i32 0
+  %ptr1 = getelementptr float, ptr %ptr, i32 3
+  %ptr2 = getelementptr float, ptr %ptr, i32 5
+  %ptr3 = getelementptr float, ptr %ptr, i32 7
+  store float %val, ptr %ptr0
+  store float %val, ptr %ptr2
+  store float %val, ptr %ptr1
+  store float %val, ptr %ptr3
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  DominatorTree DT(LLVMF);
+  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfo TLI(TLII);
+  DataLayout DL(M->getDataLayout());
+  LoopInfo LI(DT);
+  AssumptionCache AC(LLVMF);
+  ScalarEvolution SE(LLVMF, TLI, AC, DT, LI);
+
+  sandboxir::Context Ctx(C);
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto BB = F.begin();
+  sandboxir::SeedCollector SC(&*BB, SE);
+
+  // Find the stores
+  auto It = std::next(BB->begin(), 4);
+  // StX with X as the order by offset in memory
+  auto *St0 = &*It++;
+  auto *St2 = &*It++;
+  auto *St1 = &*It++;
+  auto *St3 = &*It++;
+
+  auto StoreSeedsRange = SC.getStoreSeeds();
+  auto &SB = *StoreSeedsRange.begin();
+  // Expect just one vector of store seeds
+  EXPECT_EQ(range_size(StoreSeedsRange), 1u);
+  ExpectThatElementsAre(SB, {St0, St1, St2, St3});
+}
+
+TEST_F(SeedBundleTest, VectorStores) {
+  parseIR(C, R"IR(
+define void @foo(ptr noalias %ptr, <2 x float> %val) {
+bb:
+  %ptr0 = getelementptr float, ptr %ptr, i32 0
+  %ptr1 = getelementptr float, ptr %ptr, i32 1
+  store <2 x float> %val, ptr %ptr1
+  store <2 x float> %val, ptr %ptr0
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  DominatorTree DT(LLVMF);
+  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfo TLI(TLII);
+  DataLayout DL(M->getDataLayout());
+  LoopInfo LI(DT);
+  AssumptionCache AC(LLVMF);
+  ScalarEvolution SE(LLVMF, TLI, AC, DT, LI);
+
+  sandboxir::Context Ctx(C);
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto BB = F.begin();
+  sandboxir::SeedCollector SC(&*BB, SE);
+
+  // Find the stores
+  auto It = std::next(BB->begin(), 2);
+  // StX with X as the order by offset in memory
+  auto *St1 = &*It++;
+  auto *St0 = &*It++;
+
+  auto StoreSeedsRange = SC.getStoreSeeds();
+  EXPECT_EQ(range_size(StoreSeedsRange), 1u);
+  auto &SB = *StoreSeedsRange.begin();
+  ExpectThatElementsAre(SB, {St0, St1});
+}
+
+TEST_F(SeedBundleTest, MixedScalarVectors) {
+  parseIR(C, R"IR(
+define void @foo(ptr noalias %ptr, float %v, <2 x float> %val) {
+bb:
+  %ptr0 = getelementptr float, ptr %ptr, i32 0
+  %ptr1 = getelementptr float, ptr %ptr, i32 1
+  %ptr3 = getelementptr float, ptr %ptr, i32 3
+  store float %v, ptr %ptr0
+  store float %v, ptr %ptr3
+  store <2 x float> %val, ptr %ptr1
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  DominatorTree DT(LLVMF);
+  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfo TLI(TLII);
+  DataLayout DL(M->getDataLayout());
+  LoopInfo LI(DT);
+  AssumptionCache AC(LLVMF);
+  ScalarEvolution SE(LLVMF, TLI, AC, DT, LI);
+
+  sandboxir::Context Ctx(C);
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto BB = F.begin();
+  sandboxir::SeedCollector SC(&*BB, SE);
+
+  // Find the stores
+  auto It = std::next(BB->begin(), 3);
+  // StX with X as the order by offset in memory
+  auto *St0 = &*It++;
+  auto *St3 = &*It++;
+  auto *St1 = &*It++;
+
+  auto StoreSeedsRange = SC.getStoreSeeds();
+  EXPECT_EQ(range_size(StoreSeedsRange), 1u);
+  auto &SB = *StoreSeedsRange.begin();
+  ExpectThatElementsAre(SB, {St0, St1, St3});
+}
-- 
GitLab


From 141574bacb2c10b795490d0fa5ea31acbc5d8c6e Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 23 Oct 2024 10:44:09 -0700
Subject: [PATCH 002/693] [llvm] Remove redundant calls to
 std::unique_ptr<T>::get (NFC) (#113415)

---
 llvm/lib/CGData/CodeGenData.cpp                |  2 +-
 llvm/lib/CodeGen/MachineOutliner.cpp           |  2 +-
 llvm/lib/Support/TimeProfiler.cpp              |  2 +-
 llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp      |  2 +-
 llvm/unittests/IR/DroppedVariableStatsTest.cpp | 14 +++++++-------
 5 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/CGData/CodeGenData.cpp b/llvm/lib/CGData/CodeGenData.cpp
index c56a8b77a523..2a3a74c8bc37 100644
--- a/llvm/lib/CGData/CodeGenData.cpp
+++ b/llvm/lib/CGData/CodeGenData.cpp
@@ -165,7 +165,7 @@ CodeGenData &CodeGenData::getInstance() {
         Instance->publishOutlinedHashTree(Reader->releaseOutlinedHashTree());
     }
   });
-  return *(Instance.get());
+  return *Instance;
 }
 
 namespace IndexedCGData {
diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp
index 6006ea7be806..2ae6b4f2c64c 100644
--- a/llvm/lib/CodeGen/MachineOutliner.cpp
+++ b/llvm/lib/CodeGen/MachineOutliner.cpp
@@ -1370,7 +1370,7 @@ void MachineOutliner::emitOutlinedHashTree(Module &M) {
 
     Triple TT(M.getTargetTriple());
     embedBufferInModule(
-        M, *Buffer.get(),
+        M, *Buffer,
         getCodeGenDataSectionName(CG_outline, TT.getObjectFormat()));
   }
 }
diff --git a/llvm/lib/Support/TimeProfiler.cpp b/llvm/lib/Support/TimeProfiler.cpp
index 29c1c97ddf3c..865a663115e1 100644
--- a/llvm/lib/Support/TimeProfiler.cpp
+++ b/llvm/lib/Support/TimeProfiler.cpp
@@ -160,7 +160,7 @@ struct llvm::TimeTraceProfiler {
     if (Stack.empty())
       return;
 
-    Stack.back().get()->InstantEvents.emplace_back(TimeTraceProfilerEntry(
+    Stack.back()->InstantEvents.emplace_back(TimeTraceProfilerEntry(
         ClockType::now(), TimePointType(), std::move(Name), Detail(),
         TimeTraceEventType::InstantEvent));
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index a367db706277..e4e1412d7def 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -191,7 +191,7 @@ public:
   bool allowedByRules(const SUnit *SU,
                       SmallVectorImpl<SchedGroup> &SyncPipe) const {
     for (auto &Rule : Rules) {
-      if (!Rule.get()->apply(SU, Collection, SyncPipe))
+      if (!Rule->apply(SU, Collection, SyncPipe))
         return false;
     }
     return true;
diff --git a/llvm/unittests/IR/DroppedVariableStatsTest.cpp b/llvm/unittests/IR/DroppedVariableStatsTest.cpp
index 63b7b4de1937..61f3a87bb355 100644
--- a/llvm/unittests/IR/DroppedVariableStatsTest.cpp
+++ b/llvm/unittests/IR/DroppedVariableStatsTest.cpp
@@ -84,7 +84,7 @@ TEST(DroppedVariableStats, BothDeleted) {
                       llvm::Any(const_cast<const llvm::Module *>(M.get())));
 
   // This loop simulates an IR pass that drops debug information.
-  for (auto &F : *M.get()) {
+  for (auto &F : *M) {
     for (auto &I : instructions(&F)) {
       I.dropDbgRecords();
       I.eraseFromParent();
@@ -141,7 +141,7 @@ TEST(DroppedVariableStats, DbgValLost) {
                       llvm::Any(const_cast<const llvm::Module *>(M.get())));
 
   // This loop simulates an IR pass that drops debug information.
-  for (auto &F : *M.get()) {
+  for (auto &F : *M) {
     for (auto &I : instructions(&F)) {
       I.dropDbgRecords();
       break;
@@ -198,7 +198,7 @@ TEST(DroppedVariableStats, UnrelatedScopes) {
                       llvm::Any(const_cast<const llvm::Module *>(M.get())));
 
   // This loop simulates an IR pass that drops debug information.
-  for (auto &F : *M.get()) {
+  for (auto &F : *M) {
     for (auto &I : instructions(&F)) {
       I.dropDbgRecords();
       break;
@@ -255,7 +255,7 @@ TEST(DroppedVariableStats, ChildScopes) {
                       llvm::Any(const_cast<const llvm::Module *>(M.get())));
 
   // This loop simulates an IR pass that drops debug information.
-  for (auto &F : *M.get()) {
+  for (auto &F : *M) {
     for (auto &I : instructions(&F)) {
       I.dropDbgRecords();
       break;
@@ -313,7 +313,7 @@ TEST(DroppedVariableStats, InlinedAt) {
                       llvm::Any(const_cast<const llvm::Module *>(M.get())));
 
   // This loop simulates an IR pass that drops debug information.
-  for (auto &F : *M.get()) {
+  for (auto &F : *M) {
     for (auto &I : instructions(&F)) {
       I.dropDbgRecords();
       break;
@@ -371,7 +371,7 @@ TEST(DroppedVariableStats, InlinedAtShared) {
                       llvm::Any(const_cast<const llvm::Module *>(M.get())));
 
   // This loop simulates an IR pass that drops debug information.
-  for (auto &F : *M.get()) {
+  for (auto &F : *M) {
     for (auto &I : instructions(&F)) {
       I.dropDbgRecords();
       break;
@@ -430,7 +430,7 @@ TEST(DroppedVariableStats, InlinedAtChild) {
                       llvm::Any(const_cast<const llvm::Module *>(M.get())));
 
   // This loop simulates an IR pass that drops debug information.
-  for (auto &F : *M.get()) {
+  for (auto &F : *M) {
     for (auto &I : instructions(&F)) {
       I.dropDbgRecords();
       break;
-- 
GitLab


From 7a1036935f8122d9120df5ecf5afb5c927a9ea8b Mon Sep 17 00:00:00 2001
From: Helena Kotas <hekotas@microsoft.com>
Date: Wed, 23 Oct 2024 10:44:48 -0700
Subject: [PATCH 003/693] [HLSL] Change StructuredBuffer resource class to SRV
 (#113397)

Change `StructuredBuffer` resource class to SRV (read-only) to match DXC.

Part of llvm/llvm-project#112775
---
 clang/lib/Sema/HLSLExternalSemaSource.cpp     |  2 +-
 clang/test/AST/HLSL/StructuredBuffer-AST.hlsl |  4 +-
 .../StructuredBuffer-annotations.hlsl         | 10 ++--
 .../StructuredBuffer-constructor.hlsl         | 12 ++---
 .../StructuredBuffer-elementtype.hlsl         | 52 +++++++++----------
 .../builtins/StructuredBuffer-subscript.hlsl  |  4 ++
 .../CodeGenHLSL/builtins/hlsl_resource_t.hlsl |  2 +-
 7 files changed, 45 insertions(+), 41 deletions(-)

diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp
index 5f51047b4d7b..50bb307982d3 100644
--- a/clang/lib/Sema/HLSLExternalSemaSource.cpp
+++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp
@@ -513,7 +513,7 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
              .addSimpleTemplateParams(*SemaPtr, {"element_type"})
              .Record;
   onCompletion(Decl, [this](CXXRecordDecl *Decl) {
-    setupBufferType(Decl, *SemaPtr, ResourceClass::UAV,
+    setupBufferType(Decl, *SemaPtr, ResourceClass::SRV,
                     ResourceKind::TypedBuffer, /*IsROV=*/false,
                     /*RawBuffer=*/true)
         .addArraySubscriptOperators()
diff --git a/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl b/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl
index 030fcfc31691..2c12e993270c 100644
--- a/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl
+++ b/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl
@@ -31,7 +31,7 @@ StructuredBuffer<float> Buffer;
 
 // CHECK: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
 // CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit h '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
 // CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-NEXT: HLSLResourceAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit TypedBuffer
@@ -58,7 +58,7 @@ StructuredBuffer<float> Buffer;
 // CHECK-NEXT: BuiltinType 0x{{[0-9A-Fa-f]+}} 'float'
 // CHECK-NEXT: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
 // CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit h '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
 // CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(float)]]
 // CHECK-NEXT: HLSLResourceAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit TypedBuffer
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-annotations.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-annotations.hlsl
index 4d3d4908c396..a88ea774f332 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-annotations.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-annotations.hlsl
@@ -3,17 +3,17 @@
 StructuredBuffer<float> Buffer1;
 StructuredBuffer<vector<float, 4> > BufferArray[4];
 
-StructuredBuffer<float> Buffer2 : register(u3);
-StructuredBuffer<vector<float, 4> > BufferArray2[4] : register(u4);
+StructuredBuffer<float> Buffer2 : register(t3);
+StructuredBuffer<vector<float, 4> > BufferArray2[4] : register(t4);
 
-StructuredBuffer<float> Buffer3 : register(u3, space1);
-StructuredBuffer<vector<float, 4> > BufferArray3[4] : register(u4, space1);
+StructuredBuffer<float> Buffer3 : register(t3, space1);
+StructuredBuffer<vector<float, 4> > BufferArray3[4] : register(t4, space1);
 
 [numthreads(1,1,1)]
 void main() {
 }
 
-// CHECK: !hlsl.uavs = !{![[Single:[0-9]+]], ![[Array:[0-9]+]], ![[SingleAllocated:[0-9]+]], ![[ArrayAllocated:[0-9]+]], ![[SingleSpace:[0-9]+]], ![[ArraySpace:[0-9]+]]}
+// CHECK: !hlsl.srvs = !{![[Single:[0-9]+]], ![[Array:[0-9]+]], ![[SingleAllocated:[0-9]+]], ![[ArrayAllocated:[0-9]+]], ![[SingleSpace:[0-9]+]], ![[ArraySpace:[0-9]+]]}
 // CHECK-DAG: ![[Single]] = !{ptr @Buffer1, i32 10, i32 9, i1 false, i32 -1, i32 0}
 // CHECK-DAG: ![[Array]] = !{ptr @BufferArray, i32 10, i32 9, i1 false, i32 -1, i32 0}
 // CHECK-DAG: ![[SingleAllocated]] = !{ptr @Buffer2, i32 10, i32 9, i1 false, i32 3, i32 0}
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-constructor.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-constructor.hlsl
index 4dbca9bc0a4d..8189aed89d74 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-constructor.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-constructor.hlsl
@@ -3,9 +3,9 @@
 
 // NOTE: SPIRV codegen for resource types is not yet implemented
 
-StructuredBuffer<float> Buf : register(u10);
+StructuredBuffer<float> Buf : register(t10);
 
-// CHECK: %"class.hlsl::StructuredBuffer" = type { target("dx.RawBuffer", float, 1, 0), float }
+// CHECK: %"class.hlsl::StructuredBuffer" = type { target("dx.RawBuffer", float, 0, 0), float }
 // CHECK: @Buf = global %"class.hlsl::StructuredBuffer" zeroinitializer, align 4
 
 // CHECK: define linkonce_odr void @_ZN4hlsl16StructuredBufferIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(8) %this)
@@ -18,7 +18,7 @@ StructuredBuffer<float> Buf : register(u10);
 
 // CHECK: define internal void @_init_resource_bindings() {
 // CHECK-NEXT: entry:
-// CHECK-DXIL-NEXT: %Buf_h = call target("dx.RawBuffer", float, 1, 0) @llvm.dx.handle.fromBinding.tdx.RawBuffer_f32_1_0t(i32 0, i32 10, i32 1, i32 0, i1 false)
-// CHECK-DXIL-NEXT: store target("dx.RawBuffer", float, 1, 0) %Buf_h, ptr @Buf, align 4
-// CHECK-SPIRV-NEXT: %Buf_h = call target("dx.RawBuffer", float, 1, 0) @llvm.spv.handle.fromBinding.tdx.RawBuffer_f32_1_0t(i32 0, i32 10, i32 1, i32 0, i1 false)
-// CHECK-SPIRV-NEXT: store target("dx.RawBuffer", float, 1, 0) %Buf_h, ptr @Buf", align 4
+// CHECK-DXIL-NEXT: %Buf_h = call target("dx.RawBuffer", float, 0, 0) @llvm.dx.handle.fromBinding.tdx.RawBuffer_f32_0_0t(i32 0, i32 10, i32 1, i32 0, i1 false)
+// CHECK-DXIL-NEXT: store target("dx.RawBuffer", float, 0, 0) %Buf_h, ptr @Buf, align 4
+// CHECK-SPIRV-NEXT: %Buf_h = call target("dx.RawBuffer", float, 0, 0) @llvm.spv.handle.fromBinding.tdx.RawBuffer_f32_0_0t(i32 0, i32 10, i32 1, i32 0, i1 false)
+// CHECK-SPIRV-NEXT: store target("dx.RawBuffer", float, 0, 0) %Buf_h, ptr @Buf", align 4
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
index a99c7f98a1af..d22958f902b5 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
@@ -4,19 +4,19 @@
 // likely change once subscript operators are properly implemented (llvm/llvm-project#95956) 
 // and theinterim field of the contained type is removed.
 
-// CHECK: %"class.hlsl::StructuredBuffer" = type <{ target("dx.RawBuffer", i16, 1, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.0" = type <{ target("dx.RawBuffer", i16, 1, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.2" = type { target("dx.RawBuffer", i32, 1, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.3" = type { target("dx.RawBuffer", i32, 1, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.4" = type { target("dx.RawBuffer", i64, 1, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.5" = type { target("dx.RawBuffer", i64, 1, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.6" = type <{ target("dx.RawBuffer", half, 1, 0) 
-// CHECK: %"class.hlsl::StructuredBuffer.8" = type { target("dx.RawBuffer", float, 1, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.9" = type { target("dx.RawBuffer", double, 1, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.10" = type { target("dx.RawBuffer", <4 x i16>, 1, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.11" = type { target("dx.RawBuffer", <3 x i32>, 1, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.12" = type { target("dx.RawBuffer", <2 x half>, 1, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.13" = type { target("dx.RawBuffer", <3 x float>, 1, 0)
+// CHECK: %"class.hlsl::StructuredBuffer" = type <{ target("dx.RawBuffer", i16, 0, 0)
+// CHECK: %"class.hlsl::StructuredBuffer.0" = type <{ target("dx.RawBuffer", i16, 0, 0)
+// CHECK: %"class.hlsl::StructuredBuffer.2" = type { target("dx.RawBuffer", i32, 0, 0)
+// CHECK: %"class.hlsl::StructuredBuffer.3" = type { target("dx.RawBuffer", i32, 0, 0)
+// CHECK: %"class.hlsl::StructuredBuffer.4" = type { target("dx.RawBuffer", i64, 0, 0)
+// CHECK: %"class.hlsl::StructuredBuffer.5" = type { target("dx.RawBuffer", i64, 0, 0)
+// CHECK: %"class.hlsl::StructuredBuffer.6" = type <{ target("dx.RawBuffer", half, 0, 0) 
+// CHECK: %"class.hlsl::StructuredBuffer.8" = type { target("dx.RawBuffer", float, 0, 0)
+// CHECK: %"class.hlsl::StructuredBuffer.9" = type { target("dx.RawBuffer", double, 0, 0)
+// CHECK: %"class.hlsl::StructuredBuffer.10" = type { target("dx.RawBuffer", <4 x i16>, 0, 0)
+// CHECK: %"class.hlsl::StructuredBuffer.11" = type { target("dx.RawBuffer", <3 x i32>, 0, 0)
+// CHECK: %"class.hlsl::StructuredBuffer.12" = type { target("dx.RawBuffer", <2 x half>, 0, 0)
+// CHECK: %"class.hlsl::StructuredBuffer.13" = type { target("dx.RawBuffer", <3 x float>, 0, 0)
 
 StructuredBuffer<int16_t> BufI16;
 StructuredBuffer<uint16_t> BufU16;
@@ -40,19 +40,19 @@ StructuredBuffer<float3> BufF32x3;
 
 [numthreads(1,1,1)]
 void main(int GI : SV_GroupIndex) {
-  BufI16[GI] = 0;
-  BufU16[GI] = 0;
-  BufI32[GI] = 0;
-  BufU32[GI] = 0;
-  BufI64[GI] = 0;
-  BufU64[GI] = 0;
-  BufF16[GI] = 0;
-  BufF32[GI] = 0;
-  BufF64[GI] = 0;
-  BufI16x4[GI] = 0;
-  BufU32x3[GI] = 0;
-  BufF16x2[GI] = 0;
-  BufF32x3[GI] = 0;
+  int16_t v1 = BufI16[GI];
+  uint16_t v2 = BufU16[GI];
+  int v3 = BufI32[GI];
+  uint v4 = BufU32[GI];
+  int64_t v5 = BufI64[GI];
+  uint64_t v6 = BufU64[GI];
+  half v7 = BufF16[GI];
+  float v8 = BufF32[GI];
+  double v9 = BufF64[GI];
+  vector<int16_t,4> v10 = BufI16x4[GI];
+  vector<int, 3> v11 = BufU32x3[GI];
+  half2 v12 = BufF16x2[GI];
+  float3 v13 = BufF32x3[GI];
 }
 
 // CHECK: !{{[0-9]+}} = !{ptr @BufI16, i32 10, i32 2,
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-subscript.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-subscript.hlsl
index 155749ec4f94..db75d534f854 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-subscript.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-subscript.hlsl
@@ -1,5 +1,9 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -emit-llvm -o - -O0 %s | FileCheck %s
 
+// FIXME: This tests will fail once StructuredBuffer subscript operator is properly
+// implemented (llvm-project/llvm#112977). StructuredBuffer is a read-only resource
+// and the result of a subscript operator cannot be assigned to.
+
 StructuredBuffer<int> In;
 StructuredBuffer<int> Out;
 
diff --git a/clang/test/CodeGenHLSL/builtins/hlsl_resource_t.hlsl b/clang/test/CodeGenHLSL/builtins/hlsl_resource_t.hlsl
index 6751cf2703ce..c206874a4ca9 100644
--- a/clang/test/CodeGenHLSL/builtins/hlsl_resource_t.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/hlsl_resource_t.hlsl
@@ -3,7 +3,7 @@
 using handle_float_t = __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float)]];
 
 // CHECK: %"class.hlsl::RWBuffer" = type { target("dx.TypedBuffer", <4 x float>, 1, 0, 0)
-// CHECK: %"class.hlsl::StructuredBuffer" = type { target("dx.RawBuffer", %struct.MyStruct = type { <4 x float>, <2 x i32>, [8 x i8] }, 1, 0)
+// CHECK: %"class.hlsl::StructuredBuffer" = type { target("dx.RawBuffer", %struct.MyStruct = type { <4 x float>, <2 x i32>, [8 x i8] }, 0, 0)
 
 // CHECK: define void @_Z2faU9_Res_u_CTfu17__hlsl_resource_t(target("dx.TypedBuffer", float, 1, 0, 0) %a)
 // CHECK: call void @_Z4foo1U9_Res_u_CTfu17__hlsl_resource_t(target("dx.TypedBuffer", float, 1, 0, 0) %0)
-- 
GitLab


From a3508e0246741851f06cb9ba86203be8d89febed Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Wed, 23 Oct 2024 10:25:41 -0700
Subject: [PATCH 004/693] [SLP]Small buidlvector only graph should contains
 scalars from same block

If the graph is small and has single buildvector node, all scalars
instructions must be from the same basic block to prevent compiler
crash.

Fixes #113451
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  3 +-
 .../small-graph-diff-block-instructions.ll    | 33 +++++++++++++++++++
 2 files changed, 35 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/small-graph-diff-block-instructions.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 756b25ac9856..d824c40c401d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -11588,7 +11588,8 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const {
 
   if (VectorizableTree.back()->isGather() &&
       VectorizableTree.back()->isAltShuffle() &&
-      VectorizableTree.back()->getVectorFactor() > 2)
+      VectorizableTree.back()->getVectorFactor() > 2 &&
+      allSameBlock(VectorizableTree.back()->Scalars))
     return false;
 
   assert(VectorizableTree.empty()
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/small-graph-diff-block-instructions.ll b/llvm/test/Transforms/SLPVectorizer/X86/small-graph-diff-block-instructions.ll
new file mode 100644
index 000000000000..82c8b1d707cf
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/small-graph-diff-block-instructions.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512dq -slp-threshold=-99999 < %s | FileCheck %s
+
+define i32 @test(i32 %arg, i32 %arg1) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: i32 [[ARG:%.*]], i32 [[ARG1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[BB:.*:]]
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[ARG1]] to i64
+; CHECK-NEXT:    [[ZEXT2:%.*]] = zext i32 [[ARG]] to i64
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[ARG]] to i64
+; CHECK-NEXT:    br label %[[BB3:.*]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[ZEXT]]
+; CHECK-NEXT:    [[GETELEMENTPTR4:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[ZEXT2]]
+; CHECK-NEXT:    [[GETELEMENTPTR5:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[SEXT]]
+; CHECK-NEXT:    [[ZEXT6:%.*]] = zext i32 0 to i64
+; CHECK-NEXT:    [[GETELEMENTPTR7:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[ZEXT6]]
+; CHECK-NEXT:    ret i32 0
+;
+bb:
+  %zext = zext i32 %arg1 to i64
+  %zext2 = zext i32 %arg to i64
+  %sext = sext i32 %arg to i64
+  br label %bb3
+
+bb3:
+  %getelementptr = getelementptr i64, ptr addrspace(1) null, i64 %zext
+  %getelementptr4 = getelementptr i64, ptr addrspace(1) null, i64 %zext2
+  %getelementptr5 = getelementptr i64, ptr addrspace(1) null, i64 %sext
+  %zext6 = zext i32 0 to i64
+  %getelementptr7 = getelementptr i64, ptr addrspace(1) null, i64 %zext6
+  ret i32 0
+}
-- 
GitLab


From 60105ac6bab130c2694fc7f5b7b6a5fddaaab752 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Wed, 23 Oct 2024 11:25:58 -0700
Subject: [PATCH 005/693] [flang][cuda] Fix kernel registration (#113372)

The registration needs the fct pointer and the name. This patch updates
the entry point with an extra arg and the translation as well.
---
 flang/include/flang/Runtime/CUDA/registration.h    |  3 ++-
 .../Dialect/CUF/CUFToLLVMIRTranslation.cpp         | 14 ++++++++------
 flang/runtime/CUDA/registration.cpp                |  7 ++++---
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/flang/include/flang/Runtime/CUDA/registration.h b/flang/include/flang/Runtime/CUDA/registration.h
index cbe202c4d23e..009715613e29 100644
--- a/flang/include/flang/Runtime/CUDA/registration.h
+++ b/flang/include/flang/Runtime/CUDA/registration.h
@@ -20,7 +20,8 @@ extern "C" {
 void *RTDECL(CUFRegisterModule)(void *data);
 
 /// Register a device function.
-void RTDECL(CUFRegisterFunction)(void **module, const char *fct);
+void RTDECL(CUFRegisterFunction)(
+    void **module, const char *fctSym, char *fctName);
 
 } // extern "C"
 
diff --git a/flang/lib/Optimizer/Dialect/CUF/CUFToLLVMIRTranslation.cpp b/flang/lib/Optimizer/Dialect/CUF/CUFToLLVMIRTranslation.cpp
index c6c9f96b8113..63eac46a9977 100644
--- a/flang/lib/Optimizer/Dialect/CUF/CUFToLLVMIRTranslation.cpp
+++ b/flang/lib/Optimizer/Dialect/CUF/CUFToLLVMIRTranslation.cpp
@@ -63,13 +63,15 @@ LogicalResult registerKernel(cuf::RegisterKernelOp op,
   llvm::Type *ptrTy = builder.getPtrTy(0);
   llvm::FunctionCallee fct = module->getOrInsertFunction(
       RTNAME_STRING(CUFRegisterFunction),
-      llvm::FunctionType::get(ptrTy, ArrayRef<llvm::Type *>({ptrTy, ptrTy}),
-                              false));
+      llvm::FunctionType::get(
+          ptrTy, ArrayRef<llvm::Type *>({ptrTy, ptrTy, ptrTy}), false));
   llvm::Value *modulePtr = moduleTranslation.lookupValue(op.getModulePtr());
-  builder.CreateCall(
-      fct, {modulePtr, getOrCreateFunctionName(module, builder,
-                                               op.getKernelModuleName().str(),
-                                               op.getKernelName().str())});
+  llvm::Function *fctSym =
+      moduleTranslation.lookupFunction(op.getKernelName().str());
+  builder.CreateCall(fct, {modulePtr, fctSym,
+                           getOrCreateFunctionName(
+                               module, builder, op.getKernelModuleName().str(),
+                               op.getKernelName().str())});
   return mlir::success();
 }
 
diff --git a/flang/runtime/CUDA/registration.cpp b/flang/runtime/CUDA/registration.cpp
index e5d9503e95fd..22d43a7dc57a 100644
--- a/flang/runtime/CUDA/registration.cpp
+++ b/flang/runtime/CUDA/registration.cpp
@@ -26,9 +26,10 @@ void *RTDECL(CUFRegisterModule)(void *data) {
   return fatHandle;
 }
 
-void RTDEF(CUFRegisterFunction)(void **module, const char *fct) {
-  __cudaRegisterFunction(module, fct, const_cast<char *>(fct), fct, -1,
-      (uint3 *)0, (uint3 *)0, (dim3 *)0, (dim3 *)0, (int *)0);
+void RTDEF(CUFRegisterFunction)(
+    void **module, const char *fctSym, char *fctName) {
+  __cudaRegisterFunction(module, fctSym, fctName, fctName, -1, (uint3 *)0,
+      (uint3 *)0, (dim3 *)0, (dim3 *)0, (int *)0);
 }
 }
 } // namespace Fortran::runtime::cuda
-- 
GitLab


From 743f839a886ef0b5ca17b580bfb37fc762ef5cec Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Wed, 23 Oct 2024 11:32:05 -0700
Subject: [PATCH 006/693] [NFC][LLVM][TableGen] Change `RecordKeeper::getClass`
 to return const pointer (#112261)

Change `RecordKeeper::getClass` to return const record pointer. This is
a part of effort to have better const correctness in TableGen backends:


https://discourse.llvm.org/t/psa-planned-changes-to-tablegen-getallderiveddefinitions-api-potential-downstream-breakages/81089
---
 llvm/include/llvm/TableGen/Record.h           |  8 +++---
 llvm/lib/TableGen/Record.cpp                  |  6 ++---
 llvm/lib/TableGen/TGParser.cpp                | 27 ++++++++++---------
 llvm/lib/TableGen/TGParser.h                  | 10 +++----
 .../TableGen/Common/CodeGenRegisters.cpp      |  2 +-
 mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp      |  2 +-
 6 files changed, 28 insertions(+), 27 deletions(-)

diff --git a/llvm/include/llvm/TableGen/Record.h b/llvm/include/llvm/TableGen/Record.h
index 78b44cfc649a..e64b78c3c1e3 100644
--- a/llvm/include/llvm/TableGen/Record.h
+++ b/llvm/include/llvm/TableGen/Record.h
@@ -1355,11 +1355,11 @@ class VarDefInit final
       public FoldingSetNode,
       public TrailingObjects<VarDefInit, const ArgumentInit *> {
   SMLoc Loc;
-  Record *Class;
+  const Record *Class;
   const DefInit *Def = nullptr; // after instantiation
   unsigned NumArgs;
 
-  explicit VarDefInit(SMLoc Loc, Record *Class, unsigned N);
+  explicit VarDefInit(SMLoc Loc, const Record *Class, unsigned N);
 
   const DefInit *instantiate();
 
@@ -1373,7 +1373,7 @@ public:
   static bool classof(const Init *I) {
     return I->getKind() == IK_VarDefInit;
   }
-  static const VarDefInit *get(SMLoc Loc, Record *Class,
+  static const VarDefInit *get(SMLoc Loc, const Record *Class,
                                ArrayRef<const ArgumentInit *> Args);
 
   void Profile(FoldingSetNodeID &ID) const;
@@ -2000,7 +2000,7 @@ public:
   const GlobalMap &getGlobals() const { return ExtraGlobals; }
 
   /// Get the class with the specified name.
-  Record *getClass(StringRef Name) const {
+  const Record *getClass(StringRef Name) const {
     auto I = Classes.find(Name);
     return I == Classes.end() ? nullptr : I->second.get();
   }
diff --git a/llvm/lib/TableGen/Record.cpp b/llvm/lib/TableGen/Record.cpp
index 9241fb3d8e72..1d71482b020b 100644
--- a/llvm/lib/TableGen/Record.cpp
+++ b/llvm/lib/TableGen/Record.cpp
@@ -2294,7 +2294,7 @@ const RecTy *DefInit::getFieldType(const StringInit *FieldName) const {
 
 std::string DefInit::getAsString() const { return std::string(Def->getName()); }
 
-static void ProfileVarDefInit(FoldingSetNodeID &ID, Record *Class,
+static void ProfileVarDefInit(FoldingSetNodeID &ID, const Record *Class,
                               ArrayRef<const ArgumentInit *> Args) {
   ID.AddInteger(Args.size());
   ID.AddPointer(Class);
@@ -2303,11 +2303,11 @@ static void ProfileVarDefInit(FoldingSetNodeID &ID, Record *Class,
     ID.AddPointer(I);
 }
 
-VarDefInit::VarDefInit(SMLoc Loc, Record *Class, unsigned N)
+VarDefInit::VarDefInit(SMLoc Loc, const Record *Class, unsigned N)
     : TypedInit(IK_VarDefInit, RecordRecTy::get(Class)), Loc(Loc), Class(Class),
       NumArgs(N) {}
 
-const VarDefInit *VarDefInit::get(SMLoc Loc, Record *Class,
+const VarDefInit *VarDefInit::get(SMLoc Loc, const Record *Class,
                                   ArrayRef<const ArgumentInit *> Args) {
   FoldingSetNodeID ID;
   ProfileVarDefInit(ID, Class, Args);
diff --git a/llvm/lib/TableGen/TGParser.cpp b/llvm/lib/TableGen/TGParser.cpp
index f315557f38aa..e01342ffcd3c 100644
--- a/llvm/lib/TableGen/TGParser.cpp
+++ b/llvm/lib/TableGen/TGParser.cpp
@@ -34,7 +34,7 @@ namespace llvm {
 
 struct SubClassReference {
   SMRange RefRange;
-  Record *Rec = nullptr;
+  const Record *Rec = nullptr;
   SmallVector<const ArgumentInit *, 4> TemplateArgs;
 
   SubClassReference() = default;
@@ -110,7 +110,7 @@ static void checkConcrete(Record &R) {
 
 /// Return an Init with a qualifier prefix referring
 /// to CurRec's name.
-static const Init *QualifyName(Record &CurRec, const Init *Name) {
+static const Init *QualifyName(const Record &CurRec, const Init *Name) {
   RecordKeeper &RK = CurRec.getRecords();
   const Init *NewName = BinOpInit::getStrConcat(
       CurRec.getNameInit(),
@@ -127,7 +127,7 @@ static const Init *QualifyName(MultiClass *MC, const Init *Name) {
 }
 
 /// Return the qualified version of the implicit 'NAME' template argument.
-static const Init *QualifiedNameOfImplicitName(Record &Rec) {
+static const Init *QualifiedNameOfImplicitName(const Record &Rec) {
   return QualifyName(Rec, StringInit::get(Rec.getRecords(), "NAME"));
 }
 
@@ -298,7 +298,7 @@ bool TGParser::SetValue(Record *CurRec, SMLoc Loc, const Init *ValName,
 /// AddSubClass - Add SubClass as a subclass to CurRec, resolving its template
 /// args as SubClass's template arguments.
 bool TGParser::AddSubClass(Record *CurRec, SubClassReference &SubClass) {
-  Record *SC = SubClass.Rec;
+  const Record *SC = SubClass.Rec;
   MapResolver R(CurRec);
 
   // Loop over all the subclass record's fields. Add regular fields to the new
@@ -588,7 +588,7 @@ bool TGParser::addDefOne(std::unique_ptr<Record> Rec) {
   return false;
 }
 
-bool TGParser::resolveArguments(Record *Rec,
+bool TGParser::resolveArguments(const Record *Rec,
                                 ArrayRef<const ArgumentInit *> ArgValues,
                                 SMLoc Loc, ArgValueHandler ArgValueHandler) {
   ArrayRef<const Init *> ArgNames = Rec->getTemplateArgs();
@@ -632,7 +632,7 @@ bool TGParser::resolveArguments(Record *Rec,
 
 /// Resolve the arguments of class and set them to MapResolver.
 /// Returns true if failed.
-bool TGParser::resolveArgumentsOfClass(MapResolver &R, Record *Rec,
+bool TGParser::resolveArgumentsOfClass(MapResolver &R, const Record *Rec,
                                        ArrayRef<const ArgumentInit *> ArgValues,
                                        SMLoc Loc) {
   return resolveArguments(
@@ -710,13 +710,13 @@ const Init *TGParser::ParseObjectName(MultiClass *CurMultiClass) {
 ///
 ///    ClassID ::= ID
 ///
-Record *TGParser::ParseClassID() {
+const Record *TGParser::ParseClassID() {
   if (Lex.getCode() != tgtok::Id) {
     TokError("expected name for ClassID");
     return nullptr;
   }
 
-  Record *Result = Records.getClass(Lex.getCurStrVal());
+  const Record *Result = Records.getClass(Lex.getCurStrVal());
   if (!Result) {
     std::string Msg("Couldn't find class '" + Lex.getCurStrVal() + "'");
     if (MultiClasses[Lex.getCurStrVal()].get())
@@ -2708,7 +2708,7 @@ const Init *TGParser::ParseSimpleValue(Record *CurRec, const RecTy *ItemType,
     // Value ::= CLASSID '<' ArgValueList '>' (CLASSID has been consumed)
     // This is supposed to synthesize a new anonymous definition, deriving
     // from the class with the template arguments, but no body.
-    Record *Class = Records.getClass(Name->getValue());
+    const Record *Class = Records.getClass(Name->getValue());
     if (!Class) {
       Error(NameLoc.Start,
             "Expected a class name, got '" + Name->getValue() + "'");
@@ -3196,7 +3196,7 @@ void TGParser::ParseValueList(SmallVectorImpl<const Init *> &Result,
 //   NamedArgValueList ::= [NameValue '=' Value {',' NameValue '=' Value}*]
 bool TGParser::ParseTemplateArgValueList(
     SmallVectorImpl<const ArgumentInit *> &Result, Record *CurRec,
-    Record *ArgsRec) {
+    const Record *ArgsRec) {
   assert(Result.empty() && "Result vector is not empty");
   ArrayRef<const Init *> TArgs = ArgsRec->getTemplateArgs();
 
@@ -3990,7 +3990,7 @@ bool TGParser::ParseClass() {
     return TokError("expected class name after 'class' keyword");
 
   const std::string &Name = Lex.getCurStrVal();
-  Record *CurRec = Records.getClass(Name);
+  Record *CurRec = const_cast<Record *>(Records.getClass(Name));
   if (CurRec) {
     // If the body was previously defined, this is an error.
     if (!CurRec->getValues().empty() ||
@@ -4411,7 +4411,8 @@ bool TGParser::ParseFile() {
 // If necessary, replace an argument with a cast to the required type.
 // The argument count has already been checked.
 bool TGParser::CheckTemplateArgValues(
-    SmallVectorImpl<const ArgumentInit *> &Values, SMLoc Loc, Record *ArgsRec) {
+    SmallVectorImpl<const ArgumentInit *> &Values, SMLoc Loc,
+    const Record *ArgsRec) {
   ArrayRef<const Init *> TArgs = ArgsRec->getTemplateArgs();
 
   for (const ArgumentInit *&Value : Values) {
@@ -4421,7 +4422,7 @@ bool TGParser::CheckTemplateArgValues(
     if (Value->isNamed())
       ArgName = Value->getName();
 
-    RecordVal *Arg = ArgsRec->getValue(ArgName);
+    const RecordVal *Arg = ArgsRec->getValue(ArgName);
     const RecTy *ArgType = Arg->getType();
 
     if (const auto *ArgValue = dyn_cast<TypedInit>(Value->getValue())) {
diff --git a/llvm/lib/TableGen/TGParser.h b/llvm/lib/TableGen/TGParser.h
index a1f1db6622ac..cac1ba827f11 100644
--- a/llvm/lib/TableGen/TGParser.h
+++ b/llvm/lib/TableGen/TGParser.h
@@ -248,9 +248,9 @@ private: // Semantic analysis methods.
 
   using ArgValueHandler = std::function<void(const Init *, const Init *)>;
   bool resolveArguments(
-      Record *Rec, ArrayRef<const ArgumentInit *> ArgValues, SMLoc Loc,
+      const Record *Rec, ArrayRef<const ArgumentInit *> ArgValues, SMLoc Loc,
       ArgValueHandler ArgValueHandler = [](const Init *, const Init *) {});
-  bool resolveArgumentsOfClass(MapResolver &R, Record *Rec,
+  bool resolveArgumentsOfClass(MapResolver &R, const Record *Rec,
                                ArrayRef<const ArgumentInit *> ArgValues,
                                SMLoc Loc);
   bool resolveArgumentsOfMultiClass(SubstStack &Substs, MultiClass *MC,
@@ -296,7 +296,7 @@ private:  // Parser methods.
   void ParseValueList(SmallVectorImpl<const Init *> &Result, Record *CurRec,
                       const RecTy *ItemType = nullptr);
   bool ParseTemplateArgValueList(SmallVectorImpl<const ArgumentInit *> &Result,
-                                 Record *CurRec, Record *ArgsRec);
+                                 Record *CurRec, const Record *ArgsRec);
   void ParseDagArgList(
       SmallVectorImpl<std::pair<const Init *, const StringInit *>> &Result,
       Record *CurRec);
@@ -316,12 +316,12 @@ private:  // Parser methods.
   const Init *ParseOperationCond(Record *CurRec, const RecTy *ItemType);
   const RecTy *ParseOperatorType();
   const Init *ParseObjectName(MultiClass *CurMultiClass);
-  Record *ParseClassID();
+  const Record *ParseClassID();
   MultiClass *ParseMultiClassID();
   bool ApplyLetStack(Record *CurRec);
   bool ApplyLetStack(RecordsEntry &Entry);
   bool CheckTemplateArgValues(SmallVectorImpl<const ArgumentInit *> &Values,
-                              SMLoc Loc, Record *ArgsRec);
+                              SMLoc Loc, const Record *ArgsRec);
 };
 
 } // end namespace llvm
diff --git a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
index 9e1ebf32c464..78f6dcdf305f 100644
--- a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
@@ -649,7 +649,7 @@ struct TupleExpander : SetTheory::Expander {
       return;
 
     // Precompute some types.
-    Record *RegisterCl = Def->getRecords().getClass("Register");
+    const Record *RegisterCl = Def->getRecords().getClass("Register");
     const RecTy *RegisterRecTy = RecordRecTy::get(RegisterCl);
     std::vector<StringRef> RegNames =
         Def->getValueAsListOfStrings("RegAsmNames");
diff --git a/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp b/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
index 3480c81ff7d4..75286231f590 100644
--- a/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
+++ b/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
@@ -1184,7 +1184,7 @@ static bool emitSerializationFns(const RecordKeeper &records, raw_ostream &os) {
       utilsString;
   raw_string_ostream dSerFn(dSerFnString), dDesFn(dDesFnString),
       serFn(serFnString), deserFn(deserFnString);
-  Record *attrClass = records.getClass("Attr");
+  const Record *attrClass = records.getClass("Attr");
 
   // Emit the serialization and deserialization functions simultaneously.
   StringRef opVar("op");
-- 
GitLab


From c99f3950f4697666124160fbc1e3e712b470d9e4 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Wed, 23 Oct 2024 13:33:36 -0500
Subject: [PATCH 007/693] [flang][OpenMP] Order clause AST nodes
 alphabetically, NFC (#113469)

This makes it easier to navigate the parse-tree.h file.
---
 flang/include/flang/Parser/parse-tree.h | 308 ++++++++++++------------
 1 file changed, 157 insertions(+), 151 deletions(-)

diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index e923e2199bff..f727525a759e 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3424,7 +3424,20 @@ struct AssignedGotoStmt {
 
 WRAPPER_CLASS(PauseStmt, std::optional<StopCode>);
 
-// Parse tree nodes for OpenMP 5.2 directives and clauses
+// Parse tree nodes for OpenMP directives and clauses
+
+// --- Common definitions
+
+// 2.1 Directives or clauses may accept a list or extended-list.
+//     A list item is a variable, array section or common block name (enclosed
+//     in slashes). An extended list item is a list item or a procedure Name.
+// variable-name | / common-block / | array-sections
+struct OmpObject {
+  UNION_CLASS_BOILERPLATE(OmpObject);
+  std::variant<Designator, /*common block*/ Name> u;
+};
+
+WRAPPER_CLASS(OmpObjectList, std::list<OmpObject>);
 
 // [5.0] 2.1.6 iterator-specifier -> type-declaration-stmt = subscript-triple
 //             iterator-modifier -> iterator-specifier-list
@@ -3436,50 +3449,54 @@ struct OmpIteratorSpecifier {
 
 WRAPPER_CLASS(OmpIteratorModifier, std::list<OmpIteratorSpecifier>);
 
-// 2.5 proc-bind-clause -> PROC_BIND (MASTER | CLOSE | SPREAD)
-struct OmpProcBindClause {
-  ENUM_CLASS(Type, Close, Master, Spread, Primary)
-  WRAPPER_CLASS_BOILERPLATE(OmpProcBindClause, Type);
+// 2.15.3.6 reduction-identifier -> + | - | * | .AND. | .OR. | .EQV. | .NEQV. |
+//                         MAX | MIN | IAND | IOR | IEOR
+struct OmpReductionOperator {
+  UNION_CLASS_BOILERPLATE(OmpReductionOperator);
+  std::variant<DefinedOperator, ProcedureDesignator> u;
 };
 
-// 2.15.3.1 default-clause -> DEFAULT (PRIVATE | FIRSTPRIVATE | SHARED | NONE)
-struct OmpDefaultClause {
-  ENUM_CLASS(Type, Private, Firstprivate, Shared, None)
-  WRAPPER_CLASS_BOILERPLATE(OmpDefaultClause, Type);
-};
+// --- Clauses
 
-// 2.1 Directives or clauses may accept a list or extended-list.
-//     A list item is a variable, array section or common block name (enclosed
-//     in slashes). An extended list item is a list item or a procedure Name.
-// variable-name | / common-block / | array-sections
-struct OmpObject {
-  UNION_CLASS_BOILERPLATE(OmpObject);
-  std::variant<Designator, /*common block*/ Name> u;
+// 2.8.1 aligned-clause -> ALIGNED (variable-name-list[ : scalar-constant])
+struct OmpAlignedClause {
+  TUPLE_CLASS_BOILERPLATE(OmpAlignedClause);
+  CharBlock source;
+  std::tuple<OmpObjectList, std::optional<ScalarIntConstantExpr>> t;
 };
 
-WRAPPER_CLASS(OmpObjectList, std::list<OmpObject>);
+// OMP 5.0 2.11.4 allocate-clause -> ALLOCATE ([allocator:] variable-name-list)
+// OMP 5.2 2.13.4 allocate-clause -> ALLOCATE ([allocate-modifier [,
+//  	                               allocate-modifier] :]
+//                                   variable-name-list)
+//                allocate-modifier -> allocator | align
+struct OmpAllocateClause {
+  struct AllocateModifier {
+    WRAPPER_CLASS(Allocator, ScalarIntExpr);
+    WRAPPER_CLASS(Align, ScalarIntExpr);
+    struct ComplexModifier {
+      TUPLE_CLASS_BOILERPLATE(ComplexModifier);
+      std::tuple<Allocator, Align> t;
+    };
+    UNION_CLASS_BOILERPLATE(AllocateModifier);
+    std::variant<Allocator, ComplexModifier, Align> u;
+  };
+  TUPLE_CLASS_BOILERPLATE(OmpAllocateClause);
+  std::tuple<std::optional<AllocateModifier>, OmpObjectList> t;
+};
 
-// 2.15.5.1 map ->
-//    MAP ([[map-type-modifier-list [,]] [iterator-modifier [,]] map-type : ]
-//         variable-name-list)
-// map-type-modifier-list -> map-type-modifier [,] [...]
-// map-type-modifier -> ALWAYS | CLOSE | PRESENT | OMPX_HOLD
-// map-type -> TO | FROM | TOFROM | ALLOC | RELEASE | DELETE
-struct OmpMapClause {
-  ENUM_CLASS(TypeModifier, Always, Close, Present, Ompx_Hold);
-  ENUM_CLASS(Type, To, From, Tofrom, Alloc, Release, Delete)
-  TUPLE_CLASS_BOILERPLATE(OmpMapClause);
+// OMP 5.0 2.4 atomic-default-mem-order-clause ->
+//                 ATOMIC_DEFAULT_MEM_ORDER (SEQ_CST | ACQ_REL |
+//                                           RELAXED)
+struct OmpAtomicDefaultMemOrderClause {
+  WRAPPER_CLASS_BOILERPLATE(
+      OmpAtomicDefaultMemOrderClause, common::OmpAtomicDefaultMemOrderType);
+};
 
-  // All modifiers are parsed into optional lists, even if they are unique.
-  // The checks for satisfying those constraints are deferred to semantics.
-  // In OpenMP 5.2 the non-comma syntax has been deprecated: keep the
-  // information about separator presence to emit a diagnostic if needed.
-  std::tuple<std::optional<std::list<TypeModifier>>,
-      std::optional<std::list<OmpIteratorModifier>>, // unique
-      std::optional<std::list<Type>>, // unique
-      OmpObjectList,
-      bool> // were the modifiers comma-separated?
-      t;
+// 2.15.3.1 default-clause -> DEFAULT (PRIVATE | FIRSTPRIVATE | SHARED | NONE)
+struct OmpDefaultClause {
+  ENUM_CLASS(Type, Private, Firstprivate, Shared, None)
+  WRAPPER_CLASS_BOILERPLATE(OmpDefaultClause, Type);
 };
 
 // 2.15.5.2 defaultmap -> DEFAULTMAP (implicit-behavior[:variable-category])
@@ -3491,29 +3508,6 @@ struct OmpDefaultmapClause {
   std::tuple<ImplicitBehavior, std::optional<VariableCategory>> t;
 };
 
-// 2.7.1 sched-modifier -> MONOTONIC | NONMONOTONIC | SIMD
-struct OmpScheduleModifierType {
-  ENUM_CLASS(ModType, Monotonic, Nonmonotonic, Simd)
-  WRAPPER_CLASS_BOILERPLATE(OmpScheduleModifierType, ModType);
-};
-
-struct OmpScheduleModifier {
-  TUPLE_CLASS_BOILERPLATE(OmpScheduleModifier);
-  WRAPPER_CLASS(Modifier1, OmpScheduleModifierType);
-  WRAPPER_CLASS(Modifier2, OmpScheduleModifierType);
-  std::tuple<Modifier1, std::optional<Modifier2>> t;
-};
-
-// 2.7.1 schedule-clause -> SCHEDULE ([sched-modifier1] [, sched-modifier2]:]
-//                                    kind[, chunk_size])
-struct OmpScheduleClause {
-  TUPLE_CLASS_BOILERPLATE(OmpScheduleClause);
-  ENUM_CLASS(ScheduleType, Static, Dynamic, Guided, Auto, Runtime)
-  std::tuple<std::optional<OmpScheduleModifier>, ScheduleType,
-      std::optional<ScalarIntExpr>>
-      t;
-};
-
 // device([ device-modifier :] scalar-integer-expression)
 struct OmpDeviceClause {
   TUPLE_CLASS_BOILERPLATE(OmpDeviceClause);
@@ -3527,6 +3521,37 @@ struct OmpDeviceTypeClause {
   WRAPPER_CLASS_BOILERPLATE(OmpDeviceTypeClause, Type);
 };
 
+// 2.13.9 depend-vec-length -> +/- non-negative-constant
+struct OmpDependSinkVecLength {
+  TUPLE_CLASS_BOILERPLATE(OmpDependSinkVecLength);
+  std::tuple<DefinedOperator, ScalarIntConstantExpr> t;
+};
+
+// 2.13.9 depend-vec -> iterator [+/- depend-vec-length],...,iterator[...]
+struct OmpDependSinkVec {
+  TUPLE_CLASS_BOILERPLATE(OmpDependSinkVec);
+  std::tuple<Name, std::optional<OmpDependSinkVecLength>> t;
+};
+
+// 2.13.9 depend-type -> IN | OUT | INOUT | SOURCE | SINK
+struct OmpDependenceType {
+  ENUM_CLASS(Type, In, Out, Inout, Source, Sink)
+  WRAPPER_CLASS_BOILERPLATE(OmpDependenceType, Type);
+};
+
+// 2.13.9 depend-clause -> DEPEND (((IN | OUT | INOUT) : variable-name-list) |
+//                                 SOURCE | SINK : depend-vec)
+struct OmpDependClause {
+  UNION_CLASS_BOILERPLATE(OmpDependClause);
+  EMPTY_CLASS(Source);
+  WRAPPER_CLASS(Sink, std::list<OmpDependSinkVec>);
+  struct InOut {
+    TUPLE_CLASS_BOILERPLATE(InOut);
+    std::tuple<OmpDependenceType, std::list<Designator>> t;
+  };
+  std::variant<Source, Sink, InOut> u;
+};
+
 // 2.12 if-clause -> IF ([ directive-name-modifier :] scalar-logical-expr)
 struct OmpIfClause {
   TUPLE_CLASS_BOILERPLATE(OmpIfClause);
@@ -3535,24 +3560,20 @@ struct OmpIfClause {
   std::tuple<std::optional<DirectiveNameModifier>, ScalarLogicalExpr> t;
 };
 
-// 2.8.1 aligned-clause -> ALIGNED (variable-name-list[ : scalar-constant])
-struct OmpAlignedClause {
-  TUPLE_CLASS_BOILERPLATE(OmpAlignedClause);
-  CharBlock source;
-  std::tuple<OmpObjectList, std::optional<ScalarIntConstantExpr>> t;
-};
-
-// 2.9.5 order-clause -> ORDER ([order-modifier :]concurrent)
-struct OmpOrderModifier {
-  UNION_CLASS_BOILERPLATE(OmpOrderModifier);
-  ENUM_CLASS(Kind, Reproducible, Unconstrained)
-  std::variant<Kind> u;
+// OMP 5.0 2.19.5.6 in_reduction-clause -> IN_REDUCTION (reduction-identifier:
+//                                         variable-name-list)
+struct OmpInReductionClause {
+  TUPLE_CLASS_BOILERPLATE(OmpInReductionClause);
+  std::tuple<OmpReductionOperator, OmpObjectList> t;
 };
 
-struct OmpOrderClause {
-  TUPLE_CLASS_BOILERPLATE(OmpOrderClause);
-  ENUM_CLASS(Type, Concurrent)
-  std::tuple<std::optional<OmpOrderModifier>, Type> t;
+// OMP 5.0 2.19.4.5 lastprivate-clause ->
+//                    LASTPRIVATE ([lastprivate-modifier :] list)
+//                  lastprivate-modifier -> CONDITIONAL
+struct OmpLastprivateClause {
+  TUPLE_CLASS_BOILERPLATE(OmpLastprivateClause);
+  ENUM_CLASS(LastprivateModifier, Conditional);
+  std::tuple<std::optional<LastprivateModifier>, OmpObjectList> t;
 };
 
 // 2.15.3.7 linear-modifier -> REF | VAL | UVAL
@@ -3585,96 +3606,79 @@ struct OmpLinearClause {
   std::variant<WithModifier, WithoutModifier> u;
 };
 
-// 2.15.3.6 reduction-identifier -> + | - | * | .AND. | .OR. | .EQV. | .NEQV. |
-//                         MAX | MIN | IAND | IOR | IEOR
-struct OmpReductionOperator {
-  UNION_CLASS_BOILERPLATE(OmpReductionOperator);
-  std::variant<DefinedOperator, ProcedureDesignator> u;
-};
+// 2.15.5.1 map ->
+//    MAP ([[map-type-modifier-list [,]] [iterator-modifier [,]] map-type : ]
+//         variable-name-list)
+// map-type-modifier-list -> map-type-modifier [,] [...]
+// map-type-modifier -> ALWAYS | CLOSE | PRESENT | OMPX_HOLD
+// map-type -> TO | FROM | TOFROM | ALLOC | RELEASE | DELETE
+struct OmpMapClause {
+  ENUM_CLASS(TypeModifier, Always, Close, Present, Ompx_Hold);
+  ENUM_CLASS(Type, To, From, Tofrom, Alloc, Release, Delete)
+  TUPLE_CLASS_BOILERPLATE(OmpMapClause);
 
-// 2.15.3.6 reduction-clause -> REDUCTION (reduction-identifier:
-//                                         variable-name-list)
-struct OmpReductionClause {
-  TUPLE_CLASS_BOILERPLATE(OmpReductionClause);
-  ENUM_CLASS(ReductionModifier, Inscan, Task, Default)
-  std::tuple<std::optional<ReductionModifier>, OmpReductionOperator,
-      OmpObjectList>
+  // All modifiers are parsed into optional lists, even if they are unique.
+  // The checks for satisfying those constraints are deferred to semantics.
+  // In OpenMP 5.2 the non-comma syntax has been deprecated: keep the
+  // information about separator presence to emit a diagnostic if needed.
+  std::tuple<std::optional<std::list<TypeModifier>>,
+      std::optional<std::list<OmpIteratorModifier>>, // unique
+      std::optional<std::list<Type>>, // unique
+      OmpObjectList,
+      bool> // were the modifiers comma-separated?
       t;
 };
 
-// OMP 5.0 2.19.5.6 in_reduction-clause -> IN_REDUCTION (reduction-identifier:
-//                                         variable-name-list)
-struct OmpInReductionClause {
-  TUPLE_CLASS_BOILERPLATE(OmpInReductionClause);
-  std::tuple<OmpReductionOperator, OmpObjectList> t;
-};
-
-// OMP 5.0 2.11.4 allocate-clause -> ALLOCATE ([allocator:] variable-name-list)
-// OMP 5.2 2.13.4 allocate-clause -> ALLOCATE ([allocate-modifier [,
-//  	                               allocate-modifier] :]
-//                                   variable-name-list)
-//                allocate-modifier -> allocator | align
-struct OmpAllocateClause {
-  struct AllocateModifier {
-    WRAPPER_CLASS(Allocator, ScalarIntExpr);
-    WRAPPER_CLASS(Align, ScalarIntExpr);
-    struct ComplexModifier {
-      TUPLE_CLASS_BOILERPLATE(ComplexModifier);
-      std::tuple<Allocator, Align> t;
-    };
-    UNION_CLASS_BOILERPLATE(AllocateModifier);
-    std::variant<Allocator, ComplexModifier, Align> u;
-  };
-  TUPLE_CLASS_BOILERPLATE(OmpAllocateClause);
-  std::tuple<std::optional<AllocateModifier>, OmpObjectList> t;
+// 2.9.5 order-clause -> ORDER ([order-modifier :]concurrent)
+struct OmpOrderModifier {
+  UNION_CLASS_BOILERPLATE(OmpOrderModifier);
+  ENUM_CLASS(Kind, Reproducible, Unconstrained)
+  std::variant<Kind> u;
 };
 
-// 2.13.9 depend-vec-length -> +/- non-negative-constant
-struct OmpDependSinkVecLength {
-  TUPLE_CLASS_BOILERPLATE(OmpDependSinkVecLength);
-  std::tuple<DefinedOperator, ScalarIntConstantExpr> t;
+struct OmpOrderClause {
+  TUPLE_CLASS_BOILERPLATE(OmpOrderClause);
+  ENUM_CLASS(Type, Concurrent)
+  std::tuple<std::optional<OmpOrderModifier>, Type> t;
 };
 
-// 2.13.9 depend-vec -> iterator [+/- depend-vec-length],...,iterator[...]
-struct OmpDependSinkVec {
-  TUPLE_CLASS_BOILERPLATE(OmpDependSinkVec);
-  std::tuple<Name, std::optional<OmpDependSinkVecLength>> t;
+// 2.5 proc-bind-clause -> PROC_BIND (MASTER | CLOSE | SPREAD)
+struct OmpProcBindClause {
+  ENUM_CLASS(Type, Close, Master, Spread, Primary)
+  WRAPPER_CLASS_BOILERPLATE(OmpProcBindClause, Type);
 };
 
-// 2.13.9 depend-type -> IN | OUT | INOUT | SOURCE | SINK
-struct OmpDependenceType {
-  ENUM_CLASS(Type, In, Out, Inout, Source, Sink)
-  WRAPPER_CLASS_BOILERPLATE(OmpDependenceType, Type);
+// 2.15.3.6 reduction-clause -> REDUCTION (reduction-identifier:
+//                                         variable-name-list)
+struct OmpReductionClause {
+  TUPLE_CLASS_BOILERPLATE(OmpReductionClause);
+  ENUM_CLASS(ReductionModifier, Inscan, Task, Default)
+  std::tuple<std::optional<ReductionModifier>, OmpReductionOperator,
+      OmpObjectList>
+      t;
 };
 
-// 2.13.9 depend-clause -> DEPEND (((IN | OUT | INOUT) : variable-name-list) |
-//                                 SOURCE | SINK : depend-vec)
-struct OmpDependClause {
-  UNION_CLASS_BOILERPLATE(OmpDependClause);
-  EMPTY_CLASS(Source);
-  WRAPPER_CLASS(Sink, std::list<OmpDependSinkVec>);
-  struct InOut {
-    TUPLE_CLASS_BOILERPLATE(InOut);
-    std::tuple<OmpDependenceType, std::list<Designator>> t;
-  };
-  std::variant<Source, Sink, InOut> u;
+// 2.7.1 sched-modifier -> MONOTONIC | NONMONOTONIC | SIMD
+struct OmpScheduleModifierType {
+  ENUM_CLASS(ModType, Monotonic, Nonmonotonic, Simd)
+  WRAPPER_CLASS_BOILERPLATE(OmpScheduleModifierType, ModType);
 };
 
-// OMP 5.0 2.4 atomic-default-mem-order-clause ->
-//                 ATOMIC_DEFAULT_MEM_ORDER (SEQ_CST | ACQ_REL |
-//                                           RELAXED)
-struct OmpAtomicDefaultMemOrderClause {
-  WRAPPER_CLASS_BOILERPLATE(
-      OmpAtomicDefaultMemOrderClause, common::OmpAtomicDefaultMemOrderType);
+struct OmpScheduleModifier {
+  TUPLE_CLASS_BOILERPLATE(OmpScheduleModifier);
+  WRAPPER_CLASS(Modifier1, OmpScheduleModifierType);
+  WRAPPER_CLASS(Modifier2, OmpScheduleModifierType);
+  std::tuple<Modifier1, std::optional<Modifier2>> t;
 };
 
-// OMP 5.0 2.19.4.5 lastprivate-clause ->
-//                    LASTPRIVATE ([lastprivate-modifier :] list)
-//                  lastprivate-modifier -> CONDITIONAL
-struct OmpLastprivateClause {
-  TUPLE_CLASS_BOILERPLATE(OmpLastprivateClause);
-  ENUM_CLASS(LastprivateModifier, Conditional);
-  std::tuple<std::optional<LastprivateModifier>, OmpObjectList> t;
+// 2.7.1 schedule-clause -> SCHEDULE ([sched-modifier1] [, sched-modifier2]:]
+//                                    kind[, chunk_size])
+struct OmpScheduleClause {
+  TUPLE_CLASS_BOILERPLATE(OmpScheduleClause);
+  ENUM_CLASS(ScheduleType, Static, Dynamic, Guided, Auto, Runtime)
+  std::tuple<std::optional<OmpScheduleModifier>, ScheduleType,
+      std::optional<ScalarIntExpr>>
+      t;
 };
 
 // OpenMP Clauses
@@ -3699,6 +3703,8 @@ struct OmpClauseList {
   CharBlock source;
 };
 
+// --- Directives and constructs
+
 // 2.7.2 SECTIONS
 // 2.11.2 PARALLEL SECTIONS
 struct OmpSectionsDirective {
-- 
GitLab


From c2293b33ddc6c722178087d85b2b4312ea830d12 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Wed, 23 Oct 2024 13:39:58 -0500
Subject: [PATCH 008/693] [WebAssembly] Implement the wide-arithmetic proposal
 (#111598)

This commit implements the [wide-arithmetic] proposal which has recently
reached phase 2 in the WebAssembly proposals process. The goal here is
to implement support in LLVM for emitting these instructions which are
gated behind a new feature flag by default. A new `wide-arithmetic`
feature flag is introduced which gates these four new instructions from
being emitted.

Emission of each instruction itself is relatively simple given LLVM's
preexisting lowering rules and infrastructure. The main gotcha is that
due to the multi-result nature of all of these instructions it needed
the lowerings to be implemented in C++ rather than in TableGen.

[wide-arithmetic]: https://github.com/WebAssembly/wide-arithmetic
---
 clang/include/clang/Driver/Options.td         |   2 +
 clang/lib/Basic/Targets/WebAssembly.cpp       |  12 ++
 clang/lib/Basic/Targets/WebAssembly.h         |   1 +
 clang/test/Driver/wasm-features.c             |   6 +
 .../test/Preprocessor/wasm-target-features.c  |  12 ++
 llvm/lib/Target/WebAssembly/WebAssembly.td    |   4 +
 .../lib/Target/WebAssembly/WebAssemblyISD.def |   4 +
 .../WebAssembly/WebAssemblyISelLowering.cpp   |  71 ++++++++++
 .../WebAssembly/WebAssemblyISelLowering.h     |   2 +
 .../WebAssembly/WebAssemblyInstrInfo.td       |   4 +
 .../WebAssembly/WebAssemblyInstrInteger.td    |  45 ++++++
 .../Target/WebAssembly/WebAssemblySubtarget.h |   2 +
 .../CodeGen/WebAssembly/wide-arithmetic.ll    | 132 ++++++++++++++++++
 13 files changed, 297 insertions(+)
 create mode 100644 llvm/test/CodeGen/WebAssembly/wide-arithmetic.ll

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 0cb7c0c0ae04..2ddb2f531214 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5106,6 +5106,8 @@ def msimd128 : Flag<["-"], "msimd128">, Group<m_wasm_Features_Group>;
 def mno_simd128 : Flag<["-"], "mno-simd128">, Group<m_wasm_Features_Group>;
 def mtail_call : Flag<["-"], "mtail-call">, Group<m_wasm_Features_Group>;
 def mno_tail_call : Flag<["-"], "mno-tail-call">, Group<m_wasm_Features_Group>;
+def mwide_arithmetic : Flag<["-"], "mwide-arithmetic">, Group<m_wasm_Features_Group>;
+def mno_wide_arithmetic : Flag<["-"], "mno-wide-arithmetic">, Group<m_wasm_Features_Group>;
 def mexec_model_EQ : Joined<["-"], "mexec-model=">, Group<m_wasm_Features_Driver_Group>,
                      Values<"command,reactor">,
                      HelpText<"Execution model (WebAssembly only)">,
diff --git a/clang/lib/Basic/Targets/WebAssembly.cpp b/clang/lib/Basic/Targets/WebAssembly.cpp
index 5ac9421663ad..4c9df6007b78 100644
--- a/clang/lib/Basic/Targets/WebAssembly.cpp
+++ b/clang/lib/Basic/Targets/WebAssembly.cpp
@@ -59,6 +59,7 @@ bool WebAssemblyTargetInfo::hasFeature(StringRef Feature) const {
       .Case("sign-ext", HasSignExt)
       .Case("simd128", SIMDLevel >= SIMD128)
       .Case("tail-call", HasTailCall)
+      .Case("wide-arithmetic", HasWideArithmetic)
       .Default(false);
 }
 
@@ -102,6 +103,8 @@ void WebAssemblyTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__wasm_simd128__");
   if (HasTailCall)
     Builder.defineMacro("__wasm_tail_call__");
+  if (HasWideArithmetic)
+    Builder.defineMacro("__wasm_wide_arithmetic__");
 
   Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1");
   Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2");
@@ -166,6 +169,7 @@ bool WebAssemblyTargetInfo::initFeatureMap(
     Features["multimemory"] = true;
     Features["nontrapping-fptoint"] = true;
     Features["tail-call"] = true;
+    Features["wide-arithmetic"] = true;
     setSIMDLevel(Features, RelaxedSIMD, true);
   };
   if (CPU == "generic") {
@@ -293,6 +297,14 @@ bool WebAssemblyTargetInfo::handleTargetFeatures(
       HasTailCall = false;
       continue;
     }
+    if (Feature == "+wide-arithmetic") {
+      HasWideArithmetic = true;
+      continue;
+    }
+    if (Feature == "-wide-arithmetic") {
+      HasWideArithmetic = false;
+      continue;
+    }
 
     Diags.Report(diag::err_opt_not_valid_with_opt)
         << Feature << "-target-feature";
diff --git a/clang/lib/Basic/Targets/WebAssembly.h b/clang/lib/Basic/Targets/WebAssembly.h
index 213ec42ca84b..c8e93c79bb65 100644
--- a/clang/lib/Basic/Targets/WebAssembly.h
+++ b/clang/lib/Basic/Targets/WebAssembly.h
@@ -65,6 +65,7 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyTargetInfo : public TargetInfo {
   bool HasReferenceTypes = false;
   bool HasSignExt = false;
   bool HasTailCall = false;
+  bool HasWideArithmetic = false;
 
   std::string ABI;
 
diff --git a/clang/test/Driver/wasm-features.c b/clang/test/Driver/wasm-features.c
index 57f0fc4ef36b..746bd7b8dfcb 100644
--- a/clang/test/Driver/wasm-features.c
+++ b/clang/test/Driver/wasm-features.c
@@ -94,3 +94,9 @@
 
 // TAIL-CALL: "-target-feature" "+tail-call"
 // NO-TAIL-CALL: "-target-feature" "-tail-call"
+
+// RUN: %clang --target=wasm32-unknown-unknown -### %s -mwide-arithmetic 2>&1 | FileCheck %s -check-prefix=WIDE-ARITH
+// RUN: %clang --target=wasm32-unknown-unknown -### %s -mno-wide-arithmetic 2>&1 | FileCheck %s -check-prefix=NO-WIDE-ARITH
+
+// WIDE-ARITH: "-target-feature" "+wide-arithmetic"
+// NO-WIDE-ARITH: "-target-feature" "-wide-arithmetic"
diff --git a/clang/test/Preprocessor/wasm-target-features.c b/clang/test/Preprocessor/wasm-target-features.c
index c64d3a0aa228..14d2fbf4423d 100644
--- a/clang/test/Preprocessor/wasm-target-features.c
+++ b/clang/test/Preprocessor/wasm-target-features.c
@@ -154,6 +154,7 @@
 // MVP-NOT: #define __wasm_sign_ext__ 1{{$}}
 // MVP-NOT: #define __wasm_simd128__ 1{{$}}
 // MVP-NOT: #define __wasm_tail_call__ 1{{$}}
+// MVP-NOT: #define __wasm_wide_arithmetic__ 1{{$}}
 
 // RUN: %clang -E -dM %s -o - 2>&1 \
 // RUN:     -target wasm32-unknown-unknown -mcpu=generic \
@@ -184,6 +185,7 @@
 // GENERIC-NOT: #define __wasm_relaxed_simd__ 1{{$}}
 // GENERIC-NOT: #define __wasm_simd128__ 1{{$}}
 // GENERIC-NOT: #define __wasm_tail_call__ 1{{$}}
+// GENERIC-NOT: #define __wasm_wide_arithmetic__ 1{{$}}
 
 // RUN: %clang -E -dM %s -o - 2>&1 \
 // RUN:     -target wasm32-unknown-unknown -mcpu=bleeding-edge \
@@ -206,6 +208,7 @@
 // BLEEDING-EDGE-INCLUDE-DAG: #define __wasm_sign_ext__ 1{{$}}
 // BLEEDING-EDGE-INCLUDE-DAG: #define __wasm_simd128__ 1{{$}}
 // BLEEDING-EDGE-INCLUDE-DAG: #define __wasm_tail_call__ 1{{$}}
+// BLEEDING-EDGE-INCLUDE-DAG: #define __wasm_wide_arithmetic__ 1{{$}}
 
 // RUN: %clang -E -dM %s -o - 2>&1 \
 // RUN:     -target wasm32-unknown-unknown -mcpu=bleeding-edge -mno-simd128 \
@@ -215,3 +218,12 @@
 // RUN:   | FileCheck %s -check-prefix=BLEEDING-EDGE-NO-SIMD128
 //
 // BLEEDING-EDGE-NO-SIMD128-NOT: #define __wasm_simd128__ 1{{$}}
+
+// RUN: %clang -E -dM %s -o - 2>&1 \
+// RUN:     -target wasm32-unknown-unknown -mwide-arithmetic \
+// RUN:   | FileCheck %s -check-prefix=WIDE-ARITHMETIC
+// RUN: %clang -E -dM %s -o - 2>&1 \
+// RUN:     -target wasm64-unknown-unknown -mwide-arithmetic \
+// RUN:   | FileCheck %s -check-prefix=WIDE-ARITHMETIC
+//
+// WIDE-ARITHMETIC: #define __wasm_wide_arithmetic__ 1{{$}}
diff --git a/llvm/lib/Target/WebAssembly/WebAssembly.td b/llvm/lib/Target/WebAssembly/WebAssembly.td
index c632d4a74355..37d99690c25b 100644
--- a/llvm/lib/Target/WebAssembly/WebAssembly.td
+++ b/llvm/lib/Target/WebAssembly/WebAssembly.td
@@ -78,6 +78,10 @@ def FeatureTailCall :
       SubtargetFeature<"tail-call", "HasTailCall", "true",
                        "Enable tail call instructions">;
 
+def FeatureWideArithmetic :
+      SubtargetFeature<"wide-arithmetic", "HasWideArithmetic", "true",
+                       "Enable wide-arithmetic instructions">;
+
 //===----------------------------------------------------------------------===//
 // Architectures.
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
index b8954f4693f0..f1ef58ef1f34 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
@@ -44,6 +44,10 @@ HANDLE_NODETYPE(TRUNC_SAT_ZERO_U)
 HANDLE_NODETYPE(DEMOTE_ZERO)
 HANDLE_NODETYPE(MEMORY_COPY)
 HANDLE_NODETYPE(MEMORY_FILL)
+HANDLE_NODETYPE(I64_ADD128)
+HANDLE_NODETYPE(I64_SUB128)
+HANDLE_NODETYPE(I64_MUL_WIDE_S)
+HANDLE_NODETYPE(I64_MUL_WIDE_U)
 
 // Memory intrinsics
 HANDLE_MEM_NODETYPE(GLOBAL_GET)
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 5f76d666823e..8611c05be89a 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -167,6 +167,13 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
         setOperationAction(Op, T, Expand);
   }
 
+  if (Subtarget->hasWideArithmetic()) {
+    setOperationAction(ISD::ADD, MVT::i128, Custom);
+    setOperationAction(ISD::SUB, MVT::i128, Custom);
+    setOperationAction(ISD::SMUL_LOHI, MVT::i64, Custom);
+    setOperationAction(ISD::UMUL_LOHI, MVT::i64, Custom);
+  }
+
   if (Subtarget->hasNontrappingFPToInt())
     for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT})
       for (auto T : {MVT::i32, MVT::i64})
@@ -1419,6 +1426,10 @@ void WebAssemblyTargetLowering::ReplaceNodeResults(
     // Do not add any results, signifying that N should not be custom lowered.
     // EXTEND_VECTOR_INREG is implemented for some vectors, but not all.
     break;
+  case ISD::ADD:
+  case ISD::SUB:
+    Results.push_back(Replace128Op(N, DAG));
+    break;
   default:
     llvm_unreachable(
         "ReplaceNodeResults not implemented for this op for WebAssembly!");
@@ -1495,6 +1506,9 @@ SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
     return DAG.UnrollVectorOp(Op.getNode());
   case ISD::CLEAR_CACHE:
     report_fatal_error("llvm.clear_cache is not supported on wasm");
+  case ISD::SMUL_LOHI:
+  case ISD::UMUL_LOHI:
+    return LowerMUL_LOHI(Op, DAG);
   }
 }
 
@@ -1593,6 +1607,63 @@ SDValue WebAssemblyTargetLowering::LowerLoad(SDValue Op,
   return Op;
 }
 
+SDValue WebAssemblyTargetLowering::LowerMUL_LOHI(SDValue Op,
+                                                 SelectionDAG &DAG) const {
+  assert(Subtarget->hasWideArithmetic());
+  assert(Op.getValueType() == MVT::i64);
+  SDLoc DL(Op);
+  unsigned Opcode;
+  switch (Op.getOpcode()) {
+  case ISD::UMUL_LOHI:
+    Opcode = WebAssemblyISD::I64_MUL_WIDE_U;
+    break;
+  case ISD::SMUL_LOHI:
+    Opcode = WebAssemblyISD::I64_MUL_WIDE_S;
+    break;
+  default:
+    llvm_unreachable("unexpected opcode");
+  }
+  SDValue LHS = Op.getOperand(0);
+  SDValue RHS = Op.getOperand(1);
+  SDValue Hi =
+      DAG.getNode(Opcode, DL, DAG.getVTList(MVT::i64, MVT::i64), LHS, RHS);
+  SDValue Lo(Hi.getNode(), 1);
+  SDValue Ops[] = {Hi, Lo};
+  return DAG.getMergeValues(Ops, DL);
+}
+
+SDValue WebAssemblyTargetLowering::Replace128Op(SDNode *N,
+                                                SelectionDAG &DAG) const {
+  assert(Subtarget->hasWideArithmetic());
+  auto ValTy = N->getValueType(0);
+  assert(ValTy == MVT::i128);
+  SDLoc DL(N);
+  unsigned Opcode;
+  switch (N->getOpcode()) {
+  case ISD::ADD:
+    Opcode = WebAssemblyISD::I64_ADD128;
+    break;
+  case ISD::SUB:
+    Opcode = WebAssemblyISD::I64_SUB128;
+    break;
+  default:
+    llvm_unreachable("unexpected opcode");
+  }
+  SDValue LHS = N->getOperand(0);
+  SDValue RHS = N->getOperand(1);
+
+  SDValue C0 = DAG.getConstant(0, DL, MVT::i64);
+  SDValue C1 = DAG.getConstant(1, DL, MVT::i64);
+  SDValue LHS_0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, LHS, C0);
+  SDValue LHS_1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, LHS, C1);
+  SDValue RHS_0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, RHS, C0);
+  SDValue RHS_1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, RHS, C1);
+  SDValue Result_LO = DAG.getNode(Opcode, DL, DAG.getVTList(MVT::i64, MVT::i64),
+                                  LHS_0, LHS_1, RHS_0, RHS_1);
+  SDValue Result_HI(Result_LO.getNode(), 1);
+  return DAG.getNode(ISD::BUILD_PAIR, DL, N->getVTList(), Result_LO, Result_HI);
+}
+
 SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
                                                   SelectionDAG &DAG) const {
   SDValue Src = Op.getOperand(2);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
index 139b064aa042..82b33b6d1933 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
@@ -136,6 +136,8 @@ private:
   SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerLoad(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerStore(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
+  SDValue Replace128Op(SDNode *N, SelectionDAG &DAG) const;
 
   // Custom DAG combine hooks
   SDValue
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
index 767ac86f1351..b3ea499c4f91 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
@@ -84,6 +84,10 @@ def HasTailCall :
     Predicate<"Subtarget->hasTailCall()">,
     AssemblerPredicate<(all_of FeatureTailCall), "tail-call">;
 
+def HasWideArithmetic :
+    Predicate<"Subtarget->hasWideArithmetic()">,
+    AssemblerPredicate<(all_of FeatureWideArithmetic), "wide-arithmetic">;
+
 //===----------------------------------------------------------------------===//
 // WebAssembly-specific DAG Node Types.
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInteger.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInteger.td
index 7a0c524d63b0..d4c8f92c883e 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInteger.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInteger.td
@@ -129,3 +129,48 @@ def : Pat<(select (i32 (seteq I32:$cond, 0)), I32:$lhs, I32:$rhs),
           (SELECT_I32 I32:$rhs, I32:$lhs, I32:$cond)>;
 def : Pat<(select (i32 (seteq I32:$cond, 0)), I64:$lhs, I64:$rhs),
           (SELECT_I64 I64:$rhs, I64:$lhs, I32:$cond)>;
+
+let Predicates = [HasWideArithmetic] in {
+defm I64_ADD128 : I<(outs I64:$lo, I64:$hi), (ins I64:$lhs_lo, I64:$lhs_hi, I64:$rhs_lo, I64:$rhs_hi),
+                    (outs), (ins),
+                    [],
+                     "i64.add128\t$lo, $hi, $lhs_lo, $lhs_hi, $rhs_lo, $rhs_hi",
+                     "i64.add128",
+                     0xfc13>;
+defm I64_SUB128 : I<(outs I64:$lo, I64:$hi), (ins I64:$lhs_lo, I64:$lhs_hi, I64:$rhs_lo, I64:$rhs_hi),
+                    (outs), (ins),
+                    [],
+                     "i64.sub128\t$lo, $hi, $lhs_lo, $lhs_hi, $rhs_lo, $rhs_hi",
+                     "i64.sub128",
+                     0xfc14>;
+defm I64_MUL_WIDE_S : I<(outs I64:$lo, I64:$hi), (ins I64:$lhs, I64:$rhs),
+                        (outs), (ins),
+                        [],
+                         "i64.mul_wide_s\t$lo, $hi, $lhs, $rhs",
+                         "i64.mul_wide_s",
+                         0xfc15>;
+defm I64_MUL_WIDE_U : I<(outs I64:$lo, I64:$hi), (ins I64:$lhs, I64:$rhs),
+                        (outs), (ins),
+                        [],
+                        "i64.mul_wide_u\t$lo, $hi, $lhs, $rhs",
+                        "i64.mul_wide_u",
+                        0xfc16>;
+} // Predicates = [HasWideArithmetic]
+
+def wasm_binop128_t : SDTypeProfile<2, 4, []>;
+def wasm_add128 : SDNode<"WebAssemblyISD::I64_ADD128", wasm_binop128_t>;
+def wasm_sub128 : SDNode<"WebAssemblyISD::I64_SUB128", wasm_binop128_t>;
+
+def : Pat<(wasm_add128 I64:$a, I64:$b, I64:$c, I64:$d),
+          (I64_ADD128 $a, $b, $c, $d)>;
+def : Pat<(wasm_sub128 I64:$a, I64:$b, I64:$c, I64:$d),
+          (I64_SUB128 $a, $b, $c, $d)>;
+
+def wasm_mul_wide_t : SDTypeProfile<2, 2, []>;
+def wasm_mul_wide_s : SDNode<"WebAssemblyISD::I64_MUL_WIDE_S", wasm_mul_wide_t>;
+def wasm_mul_wide_u : SDNode<"WebAssemblyISD::I64_MUL_WIDE_U", wasm_mul_wide_t>;
+
+def : Pat<(wasm_mul_wide_s I64:$x, I64:$y),
+          (I64_MUL_WIDE_S $x, $y)>;
+def : Pat<(wasm_mul_wide_u I64:$x, I64:$y),
+          (I64_MUL_WIDE_U $x, $y)>;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.h b/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.h
index f990120775d1..f2bf2902f775 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.h
@@ -51,6 +51,7 @@ class WebAssemblySubtarget final : public WebAssemblyGenSubtargetInfo {
   bool HasReferenceTypes = false;
   bool HasSignExt = false;
   bool HasTailCall = false;
+  bool HasWideArithmetic = false;
 
   /// What processor and OS we're targeting.
   Triple TargetTriple;
@@ -106,6 +107,7 @@ public:
   bool hasSignExt() const { return HasSignExt; }
   bool hasSIMD128() const { return SIMDLevel >= SIMD128; }
   bool hasTailCall() const { return HasTailCall; }
+  bool hasWideArithmetic() const { return HasWideArithmetic; }
 
   /// Parses features string setting specified subtarget options. Definition of
   /// function is auto generated by tblgen.
diff --git a/llvm/test/CodeGen/WebAssembly/wide-arithmetic.ll b/llvm/test/CodeGen/WebAssembly/wide-arithmetic.ll
new file mode 100644
index 000000000000..deff551d0eab
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/wide-arithmetic.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+wide-arithmetic < %s | FileCheck %s
+
+target triple = "wasm32-unknown-unknown"
+
+define i128 @add_i128(i128 %a, i128 %b) {
+; CHECK-LABEL: add_i128:
+; CHECK:         .functype add_i128 (i32, i64, i64, i64, i64) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    local.get 3
+; CHECK-NEXT:    local.get 4
+; CHECK-NEXT:    i64.add128
+; CHECK-NEXT:    local.set 3
+; CHECK-NEXT:    local.set 4
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    local.get 3
+; CHECK-NEXT:    i64.store 8
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    local.get 4
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %c = add i128 %a, %b
+  ret i128 %c
+}
+
+define i128 @sub_i128(i128 %a, i128 %b) {
+; CHECK-LABEL: sub_i128:
+; CHECK:         .functype sub_i128 (i32, i64, i64, i64, i64) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    local.get 3
+; CHECK-NEXT:    local.get 4
+; CHECK-NEXT:    i64.sub128
+; CHECK-NEXT:    local.set 3
+; CHECK-NEXT:    local.set 4
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    local.get 3
+; CHECK-NEXT:    i64.store 8
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    local.get 4
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %c = sub i128 %a, %b
+  ret i128 %c
+}
+
+define i128 @mul_i128(i128 %a, i128 %b) {
+; CHECK-LABEL: mul_i128:
+; CHECK:         .functype mul_i128 (i32, i64, i64, i64, i64) -> ()
+; CHECK-NEXT:    .local i64
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 3
+; CHECK-NEXT:    i64.mul_wide_u
+; CHECK-NEXT:    local.set 5
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    local.get 5
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 4
+; CHECK-NEXT:    i64.mul
+; CHECK-NEXT:    i64.add
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    local.get 3
+; CHECK-NEXT:    i64.mul
+; CHECK-NEXT:    i64.add
+; CHECK-NEXT:    i64.store 8
+; CHECK-NEXT:    # fallthrough-return
+  %c = mul i128 %a, %b
+  ret i128 %c
+}
+
+define i128 @i64_mul_wide_s(i64 %a, i64 %b) {
+; CHECK-LABEL: i64_mul_wide_s:
+; CHECK:         .functype i64_mul_wide_s (i32, i64, i64) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i64.mul_wide_s
+; CHECK-NEXT:    local.set 1
+; CHECK-NEXT:    local.set 2
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i64.store 8
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %a128 = sext i64 %a to i128
+  %b128 = sext i64 %b to i128
+  %c = mul i128 %a128, %b128
+  ret i128 %c
+}
+
+define i128 @i64_mul_wide_u(i64 %a, i64 %b) {
+; CHECK-LABEL: i64_mul_wide_u:
+; CHECK:         .functype i64_mul_wide_u (i32, i64, i64) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i64.mul_wide_u
+; CHECK-NEXT:    local.set 1
+; CHECK-NEXT:    local.set 2
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i64.store 8
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %a128 = zext i64 %a to i128
+  %b128 = zext i64 %b to i128
+  %c = mul i128 %a128, %b128
+  ret i128 %c
+}
+
+define i64 @mul_i128_only_lo(i128 %a, i128 %b) {
+; CHECK-LABEL: mul_i128_only_lo:
+; CHECK:         .functype mul_i128_only_lo (i64, i64, i64, i64) -> (i64)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i64.mul
+; CHECK-NEXT:    # fallthrough-return
+  %c = mul i128 %a, %b
+  %d = trunc i128 %c to i64
+  ret i64 %d
+}
-- 
GitLab


From a5d919b4b2f24c582838659fe7f30073e7285524 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990@gmail.com>
Date: Wed, 23 Oct 2024 19:42:15 +0100
Subject: [PATCH 009/693] [libc++] Disallow std::prev(non_cpp17_bidi_iterator)
 (#112102)

The std::prev function appeared to work on non Cpp17BidirectionalIterators, however it
behaved strangely by being the identity function. That was extremely misleading and
potentially dangerous, since several recent iterators that are C++20 bidirectional_iterators
don't satisfy the Cpp17BidirectionalIterator named requirement. For example:

    auto zv = std::views::zip(vec1, vec2);
    auto it = zv.begin() + 5;
    auto it2 = std::prev(it); // "it2" will be the same as "it",  instead of the previous one

Here, zip_view::iterator is a c++20 random_access_iterator, but it only satisfies the
Cpp17InputIterator named requirement because its reference type is a proxy type. Hence
`std::prev` would silently accept that iterator but it would do a no-op instead of going
to the previous position.

This patch changes `std::prev(it)` to produce an error at compile-time when instantiated
with a type that is not a Cpp17BidirectionalIterator.

Fixes #109456
---
 libcxx/include/__iterator/prev.h              | 18 +++++++++++++++++-
 .../iterator.operations/prev.verify.cpp       | 19 +++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)
 create mode 100644 libcxx/test/libcxx/iterators/iterator.primitives/iterator.operations/prev.verify.cpp

diff --git a/libcxx/include/__iterator/prev.h b/libcxx/include/__iterator/prev.h
index 7e97203836eb..bffd5527dc95 100644
--- a/libcxx/include/__iterator/prev.h
+++ b/libcxx/include/__iterator/prev.h
@@ -17,16 +17,20 @@
 #include <__iterator/incrementable_traits.h>
 #include <__iterator/iterator_traits.h>
 #include <__type_traits/enable_if.h>
+#include <__utility/move.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> = 0>
 [[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _InputIter
-prev(_InputIter __x, typename iterator_traits<_InputIter>::difference_type __n = 1) {
+prev(_InputIter __x, typename iterator_traits<_InputIter>::difference_type __n) {
   // Calling `advance` with a negative value on a non-bidirectional iterator is a no-op in the current implementation.
   // Note that this check duplicates the similar check in `std::advance`.
   _LIBCPP_ASSERT_PEDANTIC(__n <= 0 || __has_bidirectional_iterator_category<_InputIter>::value,
@@ -35,6 +39,16 @@ prev(_InputIter __x, typename iterator_traits<_InputIter>::difference_type __n =
   return __x;
 }
 
+// LWG 3197
+// It is unclear what the implications of "BidirectionalIterator" in the standard are.
+// However, calling std::prev(non-bidi-iterator) is obviously an error and we should catch it at compile time.
+template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> = 0>
+[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _InputIter prev(_InputIter __it) {
+  static_assert(__has_bidirectional_iterator_category<_InputIter>::value,
+                "Attempt to prev(it) with a non-bidirectional iterator");
+  return std::prev(std::move(__it), 1);
+}
+
 #if _LIBCPP_STD_VER >= 20
 
 // [range.iter.op.prev]
@@ -70,4 +84,6 @@ inline constexpr auto prev = __prev{};
 
 _LIBCPP_END_NAMESPACE_STD
 
+_LIBCPP_POP_MACROS
+
 #endif // _LIBCPP___ITERATOR_PREV_H
diff --git a/libcxx/test/libcxx/iterators/iterator.primitives/iterator.operations/prev.verify.cpp b/libcxx/test/libcxx/iterators/iterator.primitives/iterator.operations/prev.verify.cpp
new file mode 100644
index 000000000000..da0a336815a5
--- /dev/null
+++ b/libcxx/test/libcxx/iterators/iterator.primitives/iterator.operations/prev.verify.cpp
@@ -0,0 +1,19 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// std::prev
+
+#include <iterator>
+#include "test_iterators.h"
+
+void test() {
+  int arr[] = {1, 2};
+  cpp17_input_iterator<int*> it(&arr[0]);
+  it = std::prev(it);
+  // expected-error-re@*:* {{static assertion failed due to requirement {{.*}}: Attempt to prev(it) with a non-bidirectional iterator}}
+}
-- 
GitLab


From 3605d9a456185f4af78c01a2684b822b57bca9b0 Mon Sep 17 00:00:00 2001
From: Nicolas van Kempen <nvankemp@gmail.com>
Date: Wed, 23 Oct 2024 11:57:09 -0700
Subject: [PATCH 010/693] [clang-tidy][readability-container-contains] Fix
 matching of non-binaryOperator cases (#110386)

Fix #79437.
---
 .../readability/ContainerContainsCheck.cpp    | 40 +++++++++----------
 clang-tools-extra/docs/ReleaseNotes.rst       |  3 +-
 .../readability/container-contains.cpp        | 11 +++--
 3 files changed, 30 insertions(+), 24 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/readability/ContainerContainsCheck.cpp b/clang-tools-extra/clang-tidy/readability/ContainerContainsCheck.cpp
index 05d4c87bc73c..fb68c7d334b7 100644
--- a/clang-tools-extra/clang-tidy/readability/ContainerContainsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/ContainerContainsCheck.cpp
@@ -62,44 +62,44 @@ void ContainerContainsCheck::registerMatchers(MatchFinder *Finder) {
                          .bind("positiveComparison"),
                      this);
   AddSimpleMatcher(
-      binaryOperator(hasOperatorName("!="), hasOperands(CountCall, Literal0))
+      binaryOperation(hasOperatorName("!="), hasOperands(CountCall, Literal0))
           .bind("positiveComparison"));
   AddSimpleMatcher(
-      binaryOperator(hasLHS(CountCall), hasOperatorName(">"), hasRHS(Literal0))
+      binaryOperation(hasLHS(CountCall), hasOperatorName(">"), hasRHS(Literal0))
           .bind("positiveComparison"));
   AddSimpleMatcher(
-      binaryOperator(hasLHS(Literal0), hasOperatorName("<"), hasRHS(CountCall))
-          .bind("positiveComparison"));
-  AddSimpleMatcher(
-      binaryOperator(hasLHS(CountCall), hasOperatorName(">="), hasRHS(Literal1))
-          .bind("positiveComparison"));
-  AddSimpleMatcher(
-      binaryOperator(hasLHS(Literal1), hasOperatorName("<="), hasRHS(CountCall))
+      binaryOperation(hasLHS(Literal0), hasOperatorName("<"), hasRHS(CountCall))
           .bind("positiveComparison"));
+  AddSimpleMatcher(binaryOperation(hasLHS(CountCall), hasOperatorName(">="),
+                                   hasRHS(Literal1))
+                       .bind("positiveComparison"));
+  AddSimpleMatcher(binaryOperation(hasLHS(Literal1), hasOperatorName("<="),
+                                   hasRHS(CountCall))
+                       .bind("positiveComparison"));
 
   // Find inverted membership tests which use `count()`.
   AddSimpleMatcher(
-      binaryOperator(hasOperatorName("=="), hasOperands(CountCall, Literal0))
-          .bind("negativeComparison"));
-  AddSimpleMatcher(
-      binaryOperator(hasLHS(CountCall), hasOperatorName("<="), hasRHS(Literal0))
-          .bind("negativeComparison"));
-  AddSimpleMatcher(
-      binaryOperator(hasLHS(Literal0), hasOperatorName(">="), hasRHS(CountCall))
+      binaryOperation(hasOperatorName("=="), hasOperands(CountCall, Literal0))
           .bind("negativeComparison"));
+  AddSimpleMatcher(binaryOperation(hasLHS(CountCall), hasOperatorName("<="),
+                                   hasRHS(Literal0))
+                       .bind("negativeComparison"));
+  AddSimpleMatcher(binaryOperation(hasLHS(Literal0), hasOperatorName(">="),
+                                   hasRHS(CountCall))
+                       .bind("negativeComparison"));
   AddSimpleMatcher(
-      binaryOperator(hasLHS(CountCall), hasOperatorName("<"), hasRHS(Literal1))
+      binaryOperation(hasLHS(CountCall), hasOperatorName("<"), hasRHS(Literal1))
           .bind("negativeComparison"));
   AddSimpleMatcher(
-      binaryOperator(hasLHS(Literal1), hasOperatorName(">"), hasRHS(CountCall))
+      binaryOperation(hasLHS(Literal1), hasOperatorName(">"), hasRHS(CountCall))
           .bind("negativeComparison"));
 
   // Find membership tests based on `find() == end()`.
   AddSimpleMatcher(
-      binaryOperator(hasOperatorName("!="), hasOperands(FindCall, EndCall))
+      binaryOperation(hasOperatorName("!="), hasOperands(FindCall, EndCall))
           .bind("positiveComparison"));
   AddSimpleMatcher(
-      binaryOperator(hasOperatorName("=="), hasOperands(FindCall, EndCall))
+      binaryOperation(hasOperatorName("=="), hasOperands(FindCall, EndCall))
           .bind("negativeComparison"));
 }
 
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 9afe497fb3d8..876689c40fcd 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -238,7 +238,8 @@ Changes in existing checks
 
 - Improved :doc:`readability-container-contains
   <clang-tidy/checks/readability/container-contains>` check to let it work on
-  any class that has a ``contains`` method.
+  any class that has a ``contains`` method. Fix some false negatives in the
+  ``find()`` case.
 
 - Improved :doc:`readability-enum-initial-value
   <clang-tidy/checks/readability/enum-initial-value>` check by only issuing
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/container-contains.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/container-contains.cpp
index 9a9b233e0722..f345b3e7768a 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/container-contains.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/container-contains.cpp
@@ -1,14 +1,19 @@
-// RUN: %check_clang_tidy -std=c++20-or-later %s readability-container-contains %t
+// RUN: %check_clang_tidy -std=c++11-or-later %s readability-container-contains %t
 
 // Some *very* simplified versions of `map` etc.
 namespace std {
 
 template <class Key, class T>
 struct map {
+  struct iterator {
+    bool operator==(const iterator &Other) const;
+    bool operator!=(const iterator &Other) const;
+  };
+
   unsigned count(const Key &K) const;
   bool contains(const Key &K) const;
-  void *find(const Key &K);
-  void *end();
+  iterator find(const Key &K);
+  iterator end();
 };
 
 template <class Key>
-- 
GitLab


From b65b2b4ab60763515694c740935989f908a03312 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Wed, 23 Oct 2024 12:03:38 -0700
Subject: [PATCH 011/693] [SLP]Expand vector to the whole register size in
 extracts adjustment

Need to expand the number of elements to the whole register to correctly
process estimation and avoid compiler crash.

Fixes #113462
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 11 ++-
 .../SLPVectorizer/partial-register-extract.ll | 99 +++++++++++++++++++
 2 files changed, 105 insertions(+), 5 deletions(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/partial-register-extract.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d824c40c401d..889c4d94ba5e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9840,13 +9840,14 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
             ::getShuffleCost(TTI, *RegShuffleKind,
                              getWidenedType(ScalarTy, EltsPerVector), SubMask);
       }
+      const unsigned BaseVF = getFullVectorNumberOfElements(
+          *R.TTI, VL.front()->getType(), alignTo(NumElts, EltsPerVector));
       for (unsigned Idx : Indices) {
-        assert((Idx + EltsPerVector) <= alignTo(NumElts, EltsPerVector) &&
+        assert((Idx + EltsPerVector) <= BaseVF &&
                "SK_ExtractSubvector index out of range");
-        Cost += ::getShuffleCost(
-            TTI, TTI::SK_ExtractSubvector,
-            getWidenedType(ScalarTy, alignTo(NumElts, EltsPerVector)), {},
-            CostKind, Idx, getWidenedType(ScalarTy, EltsPerVector));
+        Cost += ::getShuffleCost(TTI, TTI::SK_ExtractSubvector,
+                                 getWidenedType(ScalarTy, BaseVF), {}, CostKind,
+                                 Idx, getWidenedType(ScalarTy, EltsPerVector));
       }
       // Second attempt to check, if just a permute is better estimated than
       // subvector extract.
diff --git a/llvm/test/Transforms/SLPVectorizer/partial-register-extract.ll b/llvm/test/Transforms/SLPVectorizer/partial-register-extract.ll
new file mode 100644
index 000000000000..43ce36337f4d
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/partial-register-extract.ll
@@ -0,0 +1,99 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer < %s | FileCheck %s
+
+define i32 @test(i32 %v, ptr %p) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: i32 [[V:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[LD:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    br i1 false, label %[[INC:.*]], label %[[PH:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[LD]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <4 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 0, i32 0>, i32 [[V]], i32 13
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <16 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i1> [[TMP2]], <4 x i1> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i1> [[TMP5]], <16 x i1> [[TMP4]], <4 x i32> <i32 0, i32 31, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i1> [[TMP6]], <4 x i1> <i1 poison, i1 poison, i1 false, i1 false>, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> zeroinitializer, <4 x i64> zeroinitializer
+; CHECK-NEXT:    [[I8_I_I:%.*]] = select i1 false, i64 0, i64 0
+; CHECK-NEXT:    [[I9_I_I:%.*]] = select i1 false, i64 0, i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP8]])
+; CHECK-NEXT:    [[OP_RDX1:%.*]] = or i64 [[TMP9]], [[I8_I_I]]
+; CHECK-NEXT:    [[OP_RDX2:%.*]] = or i64 [[OP_RDX1]], [[I9_I_I]]
+; CHECK-NEXT:    [[TMP10:%.*]] = freeze <16 x i1> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP10]])
+; CHECK-NEXT:    [[TMP12:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
+; CHECK-NEXT:    [[OP_RDX:%.*]] = select i1 [[TMP11]], i1 true, i1 [[TMP13]]
+; CHECK-NEXT:    [[AND252_US_I_24_I_I:%.*]] = select i1 [[OP_RDX]], i32 0, i32 0
+; CHECK-NEXT:    br label %[[INC]]
+; CHECK:       [[INC]]:
+; CHECK-NEXT:    [[P1:%.*]] = phi i32 [ [[AND252_US_I_24_I_I]], %[[PH]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[P2:%.*]] = phi i64 [ [[OP_RDX2]], %[[PH]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %ld = load i32, ptr %p, align 4
+  br i1 false, label %inc, label %ph
+
+ph:
+  %bi.i.not = icmp eq i32 %ld, 0
+  %b1.i.i = icmp eq i32 %ld, 0
+  %b3.i.i = icmp eq i32 %ld, 0
+  %0 = or i1 %b3.i.i, %b1.i.i
+  %b4.i.i = icmp eq i32 %ld, 0
+  %i4.i.i = select i1 %b4.i.i, i64 0, i64 0
+  %b5.i.i = icmp eq i32 0, 0
+  %i5.i.i = select i1 %b5.i.i, i64 0, i64 0
+  %inc34.5.i.i = or i64 %i4.i.i, %i5.i.i
+  %1 = or i1 %b5.i.i, %b4.i.i
+  %i6.i.i = select i1 false, i64 0, i64 0
+  %inc34.6.i.i = or i64 %inc34.5.i.i, %i6.i.i
+  %b7.i.i = icmp eq i32 0, 0
+  %i7.i.i = select i1 false, i64 0, i64 0
+  %inc34.7.i.i = or i64 %inc34.6.i.i, %i7.i.i
+  %i8.i.i = select i1 false, i64 0, i64 0
+  %inc34.8.i.i = or i64 %inc34.7.i.i, %i8.i.i
+  %i9.i.i = select i1 false, i64 0, i64 0
+  %inc34.9.i.i = or i64 %inc34.8.i.i, %i9.i.i
+  %b10.i.i = icmp eq i32 0, 0
+  %b11.i.i = icmp eq i32 0, 0
+  %2 = or i1 %b11.i.i, %b10.i.i
+  %b12.i.i = icmp eq i32 %v, 0
+  %3 = or i1 %b12.i.i, %2
+  %b13.i.i = icmp eq i32 0, 0
+  %b14.i.i = icmp eq i32 0, 0
+  %4 = or i1 %b14.i.i, %b13.i.i
+  %b16.i.i = icmp eq i32 0, 0
+  %b17.i.i = icmp eq i32 0, 0
+  %5 = or i1 %b17.i.i, %b16.i.i
+  %b18.i.i = icmp eq i32 0, 0
+  %6 = or i1 %b18.i.i, %5
+  %b19.i.i = icmp eq i32 0, 0
+  %b20.i.i = icmp eq i32 0, 0
+  %7 = or i1 %b20.i.i, %b19.i.i
+  %b21.i.i = icmp eq i32 0, 0
+  %8 = or i1 %b21.i.i, %7
+  %b22.i.i = icmp eq i32 0, 0
+  %b23.i.i = icmp eq i32 0, 0
+  %9 = or i1 %b23.i.i, %b22.i.i
+  %b24.i.i = icmp eq i32 0, 0
+  %10 = or i1 %b24.i.i, %9
+  %11 = select i1 %10, i1 true, i1 %8
+  %12 = select i1 %11, i1 true, i1 %6
+  %13 = select i1 %12, i1 true, i1 %4
+  %14 = select i1 %13, i1 true, i1 %3
+  %15 = select i1 %14, i1 true, i1 %b7.i.i
+  %16 = select i1 %15, i1 true, i1 %1
+  %17 = or i1 %0, %bi.i.not
+  %18 = select i1 %16, i1 true, i1 %17
+  %and252.us.i.24.i.i = select i1 %18, i32 0, i32 0
+  br label %inc
+
+inc:
+  %p1 = phi i32 [ %and252.us.i.24.i.i, %ph ], [ 0, %entry ]
+  %p2 = phi i64 [ %inc34.9.i.i, %ph ], [ 0, %entry ]
+  ret i32 0
+}
-- 
GitLab


From 4583dbc7b85d25bdda652ad1afe5ec3e95183a16 Mon Sep 17 00:00:00 2001
From: Daniel Paoliello <danpao@microsoft.com>
Date: Wed, 23 Oct 2024 12:18:02 -0700
Subject: [PATCH 012/693] [lit][aix] Always use internal lit shell on AIX
 (#113355)

Diff on AIX doesn't have all the required features used in tests (see
<https://github.com/llvm/llvm-project/pull/108871> and
<https://github.com/llvm/llvm-project/pull/112997#issuecomment-2429656192>),
so always use the internal shell.
---
 llvm/utils/lit/lit/llvm/config.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py
index 5f762ec7f351..1ef5796cd32e 100644
--- a/llvm/utils/lit/lit/llvm/config.py
+++ b/llvm/utils/lit/lit/llvm/config.py
@@ -57,6 +57,13 @@ class LLVMConfig(object):
                 self.lit_config.note("using lit tools: {}".format(path))
                 lit_path_displayed = True
 
+        if platform.system() == "AIX":
+            # Diff on AIX doesn't have all the required features (see
+            # https://github.com/llvm/llvm-project/pull/108871 and
+            # https://github.com/llvm/llvm-project/pull/112997#issuecomment-2429656192)
+            # so always use the internal shell.
+            self.use_lit_shell = True
+
         if platform.system() == "OS/390":
             self.with_environment("_BPXK_AUTOCVT", "ON")
             self.with_environment("_TAG_REDIR_IN", "TXT")
-- 
GitLab


From 04aa0c63f09e82bcbafe23981b36263056019538 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Wed, 23 Oct 2024 22:39:32 +0300
Subject: [PATCH 013/693] [gitattributes] Mark some llvm-rc inputs as requiring
 LF newlines (#113222)

These text files get embedded verbatim in
llvm/test/tools/llvm-rc/tag-html.test, so their newlines form needs to
match the expected form exactly.

This reverts commit 2fdf49db7562eadbe01b18f0d01a955cd41b94ea, as that
change no longer should be necessary. (At the time, buildbots had
checkouts with all files having CRLF, and those don't get updated even
if the toplevel .gitattributes were reverted in
e669bbbb7265a7d4d59bac2d3889194efa167ea8. Now those buildbots have
gotten fresh checkouts with all files having the right line endings.)
---
 llvm/.gitattributes                   | 1 +
 llvm/test/tools/llvm-rc/tag-html.test | 4 +---
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/llvm/.gitattributes b/llvm/.gitattributes
index 5f0d9990b6b5..fc3afb28a8d5 100644
--- a/llvm/.gitattributes
+++ b/llvm/.gitattributes
@@ -25,6 +25,7 @@ test/tools/llvm-mca/X86/directives-handle-crlf.s text eol=crlf
 
 # These files must have LF line endings because the test requires exact matching
 test/Object/Inputs/*.tbd eol=lf
+test/tools/llvm-rc/Inputs/webpage*.html text eol=lf
 test/tools/llvm-strings/radix.test text eol=lf
 test/tools/llvm-tapi-diff/Inputs/*.tbd text eol=lf
 test/tools/split-file/basic.test text eol=lf
diff --git a/llvm/test/tools/llvm-rc/tag-html.test b/llvm/test/tools/llvm-rc/tag-html.test
index e0fe2e659dd0..3620fe954ba1 100644
--- a/llvm/test/tools/llvm-rc/tag-html.test
+++ b/llvm/test/tools/llvm-rc/tag-html.test
@@ -1,7 +1,5 @@
 ; RUN: rm -rf %t && mkdir %t && cd %t
-; Remove `\r` in case Git on Windows decided to checkout with Windows newlines.
-; RUN: tr -d '\r' <%p/Inputs/webpage1.html > ./webpage1.html
-; RUN: tr -d '\r' <%p/Inputs/webpage2.html > ./webpage2.html
+; RUN: cp %p/Inputs/webpage*.html .
 ; RUN: llvm-rc -no-preprocess /FO %t/tag-html.res -- %p/Inputs/tag-html.rc
 ; RUN: llvm-readobj %t/tag-html.res | FileCheck %s --check-prefix HTML
 
-- 
GitLab


From 2002533802dbe74c82476e30d093baf6d4cdee50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Fri, 18 Oct 2024 09:21:03 +0000
Subject: [PATCH 014/693] [compiler-rt] [test] Mark a couple files as requiring
 LF newlines

Some of the profile test files fail if they have CRLF newlines;
add a .gitattributes file that forces them to be checked out
with LF newlines, regarless of the user Git configuration.
---
 compiler-rt/test/profile/Inputs/.gitattributes | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 compiler-rt/test/profile/Inputs/.gitattributes

diff --git a/compiler-rt/test/profile/Inputs/.gitattributes b/compiler-rt/test/profile/Inputs/.gitattributes
new file mode 100644
index 000000000000..c363807c4e95
--- /dev/null
+++ b/compiler-rt/test/profile/Inputs/.gitattributes
@@ -0,0 +1,3 @@
+instrprof-gcov-*.c text eol=lf
+instrprof-gcov-*.cpp text eol=lf
+instrprof-shared-*.c text eol=lf
-- 
GitLab


From 2dfb1c664c0a0afc512b900c45989e247406e523 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 23 Oct 2024 13:12:03 -0700
Subject: [PATCH 015/693] [VPlan] Try to hoist Previous (and operands), if
 sinking fails for FORs. (#108945)

In some cases, Previous (and its operands) can be hoisted. This allows
supporting additional cases where sinking of all users of to FOR fails,
e.g. due having to sink recipes with side-effects.

This fixes a crash where we fail to create a scalar VPlan for a
first-order recurrence, but can create a vector VPlan, because the trunc
instruction of an IV which generates the previous value of the
recurrence has been optimized to a truncated induction recipe, thus
hoisting it to the beginning.

Fixes https://github.com/llvm/llvm-project/issues/106523.

PR: https://github.com/llvm/llvm-project/pull/108945
---
 .../Transforms/Vectorize/VPlanTransforms.cpp  | 102 ++++++++++++++++-
 .../Transforms/Vectorize/VPlanTransforms.h    |  10 +-
 .../X86/fixed-order-recurrence.ll             |  76 +++++++++++++
 .../first-order-recurrence-chains-vplan.ll    | 104 ++++++++++++++++--
 ...t-order-recurrence-multiply-recurrences.ll |  69 +++++++++---
 5 files changed, 331 insertions(+), 30 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d50f3c0c3f3e..c6e09c4f2e6e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -772,6 +772,105 @@ sinkRecurrenceUsersAfterPrevious(VPFirstOrderRecurrencePHIRecipe *FOR,
   return true;
 }
 
+/// Try to hoist \p Previous and its operands before all users of \p FOR.
+static bool hoistPreviousBeforeFORUsers(VPFirstOrderRecurrencePHIRecipe *FOR,
+                                        VPRecipeBase *Previous,
+                                        VPDominatorTree &VPDT) {
+  if (Previous->mayHaveSideEffects() || Previous->mayReadFromMemory())
+    return false;
+
+  // Collect recipes that need hoisting.
+  SmallVector<VPRecipeBase *> HoistCandidates;
+  SmallPtrSet<VPRecipeBase *, 8> Visited;
+  VPRecipeBase *HoistPoint = nullptr;
+  // Find the closest hoist point by looking at all users of FOR and selecting
+  // the recipe dominating all other users.
+  for (VPUser *U : FOR->users()) {
+    auto *R = dyn_cast<VPRecipeBase>(U);
+    if (!R)
+      continue;
+    if (!HoistPoint || VPDT.properlyDominates(R, HoistPoint))
+      HoistPoint = R;
+  }
+  assert(all_of(FOR->users(),
+                [&VPDT, HoistPoint](VPUser *U) {
+                  auto *R = dyn_cast<VPRecipeBase>(U);
+                  return !R || HoistPoint == R ||
+                         VPDT.properlyDominates(HoistPoint, R);
+                }) &&
+         "HoistPoint must dominate all users of FOR");
+
+  auto NeedsHoisting = [HoistPoint, &VPDT,
+                        &Visited](VPValue *HoistCandidateV) -> VPRecipeBase * {
+    VPRecipeBase *HoistCandidate = HoistCandidateV->getDefiningRecipe();
+    if (!HoistCandidate)
+      return nullptr;
+    VPRegionBlock *EnclosingLoopRegion =
+        HoistCandidate->getParent()->getEnclosingLoopRegion();
+    assert((!HoistCandidate->getParent()->getParent() ||
+            HoistCandidate->getParent()->getParent() == EnclosingLoopRegion) &&
+           "CFG in VPlan should still be flat, without replicate regions");
+    // Hoist candidate was already visited, no need to hoist.
+    if (!Visited.insert(HoistCandidate).second)
+      return nullptr;
+
+    // Candidate is outside loop region or a header phi, dominates FOR users w/o
+    // hoisting.
+    if (!EnclosingLoopRegion || isa<VPHeaderPHIRecipe>(HoistCandidate))
+      return nullptr;
+
+    // If we reached a recipe that dominates HoistPoint, we don't need to
+    // hoist the recipe.
+    if (VPDT.properlyDominates(HoistCandidate, HoistPoint))
+      return nullptr;
+    return HoistCandidate;
+  };
+  auto CanHoist = [&](VPRecipeBase *HoistCandidate) {
+    // Avoid hoisting candidates with side-effects, as we do not yet analyze
+    // associated dependencies.
+    return !HoistCandidate->mayHaveSideEffects();
+  };
+
+  if (!NeedsHoisting(Previous->getVPSingleValue()))
+    return true;
+
+  // Recursively try to hoist Previous and its operands before all users of FOR.
+  HoistCandidates.push_back(Previous);
+
+  for (unsigned I = 0; I != HoistCandidates.size(); ++I) {
+    VPRecipeBase *Current = HoistCandidates[I];
+    assert(Current->getNumDefinedValues() == 1 &&
+           "only recipes with a single defined value expected");
+    if (!CanHoist(Current))
+      return false;
+
+    for (VPValue *Op : Current->operands()) {
+      // If we reach FOR, it means the original Previous depends on some other
+      // recurrence that in turn depends on FOR. If that is the case, we would
+      // also need to hoist recipes involving the other FOR, which may break
+      // dependencies.
+      if (Op == FOR)
+        return false;
+
+      if (auto *R = NeedsHoisting(Op))
+        HoistCandidates.push_back(R);
+    }
+  }
+
+  // Order recipes to hoist by dominance so earlier instructions are processed
+  // first.
+  sort(HoistCandidates, [&VPDT](const VPRecipeBase *A, const VPRecipeBase *B) {
+    return VPDT.properlyDominates(A, B);
+  });
+
+  for (VPRecipeBase *HoistCandidate : HoistCandidates) {
+    HoistCandidate->moveBefore(*HoistPoint->getParent(),
+                               HoistPoint->getIterator());
+  }
+
+  return true;
+}
+
 bool VPlanTransforms::adjustFixedOrderRecurrences(VPlan &Plan,
                                                   VPBuilder &LoopBuilder) {
   VPDominatorTree VPDT;
@@ -795,7 +894,8 @@ bool VPlanTransforms::adjustFixedOrderRecurrences(VPlan &Plan,
       Previous = PrevPhi->getBackedgeValue()->getDefiningRecipe();
     }
 
-    if (!sinkRecurrenceUsersAfterPrevious(FOR, Previous, VPDT))
+    if (!sinkRecurrenceUsersAfterPrevious(FOR, Previous, VPDT) &&
+        !hoistPreviousBeforeFORUsers(FOR, Previous, VPDT))
       return false;
 
     // Introduce a recipe to combine the incoming and previous values of a
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 60a44bfb0dca..11e094db6294 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -36,11 +36,11 @@ struct VPlanTransforms {
                                 GetIntOrFpInductionDescriptor,
                             ScalarEvolution &SE, const TargetLibraryInfo &TLI);
 
-  /// Sink users of fixed-order recurrences after the recipe defining their
-  /// previous value. Then introduce FirstOrderRecurrenceSplice VPInstructions
-  /// to combine the value from the recurrence phis and previous values. The
-  /// current implementation assumes all users can be sunk after the previous
-  /// value, which is enforced by earlier legality checks.
+  /// Try to have all users of fixed-order recurrences appear after the recipe
+  /// defining their previous value, by either sinking users or hoisting recipes
+  /// defining their previous value (and its operands). Then introduce
+  /// FirstOrderRecurrenceSplice VPInstructions to combine the value from the
+  /// recurrence phis and previous values.
   /// \returns true if all users of fixed-order recurrences could be re-arranged
   /// as needed or false if it is not possible. In the latter case, \p Plan is
   /// not valid.
diff --git a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
index dcdc723b4b9b..ca38d2747760 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
@@ -278,3 +278,79 @@ exit:
   store double %.lcssa, ptr %C
   ret i64 %.in.lcssa
 }
+
+; Test for https://github.com/llvm/llvm-project/issues/106523.
+; %for.2 requires no code motion, as its previous (%or) precedes its (first)
+; user (store). Furthermore, its user cannot sink, being a store.
+;
+; %for.1 requires code motion, as its previous (%trunc) follows its (first)
+; user (%or). Sinking %or past %trunc seems possible, as %or has no uses
+; (except for feeding %for.2; worth strengthening VPlan's dce?). However, %or
+; is both the user of %for.1 and the previous of %for.2, and we refrain from
+; sinking instructions that act as previous because they (may) serve points to
+; sink after.
+
+; Instead, %for.1 can be reconciled by hoisting its previous above its user
+; %or, as this user %trunc depends only on %iv.
+define void @for_iv_trunc_optimized(ptr %dst) {
+; CHECK-LABEL: @for_iv_trunc_optimized(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 1>, [[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[STEP_ADD]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[VEC_IND]], <4 x i32> [[STEP_ADD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i32> [[TMP0]], <i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT:    [[TMP3]] = or <4 x i32> [[TMP1]], <i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP5]], i32 3
+; CHECK-NEXT:    store i32 [[TMP6]], ptr [[DST:%.*]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], <i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 336
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[STEP_ADD]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 337, [[MIDDLE_BLOCK]] ], [ 1, [[BB:%.*]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 1, [[BB]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT4:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT3]], [[MIDDLE_BLOCK]] ], [ 0, [[BB]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TRUNC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[FOR_2:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT4]], [[SCALAR_PH]] ], [ [[OR:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OR]] = or i32 [[FOR_1]], 3
+; CHECK-NEXT:    [[ADD]] = add i64 [[IV]], 1
+; CHECK-NEXT:    store i32 [[FOR_2]], ptr [[DST]], align 4
+; CHECK-NEXT:    [[ICMP:%.*]] = icmp ult i64 [[IV]], 337
+; CHECK-NEXT:    [[TRUNC]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    br i1 [[ICMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+bb:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1, %bb ], [ %add, %loop ]
+  %for.1 = phi i32 [ 1, %bb ], [ %trunc, %loop ]
+  %for.2 = phi i32 [ 0, %bb ], [ %or, %loop ]
+  %or = or i32 %for.1, 3
+  %add = add i64 %iv, 1
+  store i32 %for.2, ptr %dst, align 4
+  %icmp = icmp ult i64 %iv, 337
+  %trunc = trunc i64 %iv to i32
+  br i1 %icmp, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
index 9de675b28530..fe48008792ff 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
@@ -147,14 +147,57 @@ exit:
 }
 
 ; This test has two FORs (for.x and for.y) where incoming value from the previous
-; iteration (for.x.prev) of one FOR (for.y) depends on another FOR (for.x). Due to
-; this dependency all uses of the former FOR (for.y) should be sunk after
-; incoming value from the previous iteration (for.x.prev) of te latter FOR (for.y).
-; That means side-effecting user (store i64 %for.y.i64, ptr %gep) of the latter
-; FOR (for.y) should be moved which is not currently supported.
+; iteration (for.x.prev) of one FOR (for.y) depends on another FOR (for.x).
+; Sinking would require moving a recipe with side effects (store). Instead,
+; for.x.next can be hoisted.
 define i32 @test_chained_first_order_recurrences_4(ptr %base, i64 %x) {
 ; CHECK-LABEL: 'test_chained_first_order_recurrences_4'
-; CHECK: No VPlans built.
+; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<4098> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT:   WIDEN ir<%for.x.next> = mul ir<%x>, ir<2>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT:   vector.body:
+; CHECK-NEXT:     EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
+; CHECK-NEXT:     FIRST-ORDER-RECURRENCE-PHI ir<%for.x> = phi ir<0>, ir<%for.x.next>
+; CHECK-NEXT:     FIRST-ORDER-RECURRENCE-PHI ir<%for.y> = phi ir<0>, ir<%for.x.prev>
+; CHECK-NEXT:     vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
+; CHECK-NEXT:     CLONE ir<%gep> = getelementptr ir<%base>, vp<[[SCALAR_STEPS]]>
+; CHECK-NEXT:     EMIT vp<[[SPLICE_X:%.]]> = first-order splice ir<%for.x>, ir<%for.x.next>
+; CHECK-NEXT:     WIDEN-CAST ir<%for.x.prev> = trunc  vp<[[SPLICE_X]]> to i32
+; CHECK-NEXT:     EMIT vp<[[SPLICE_Y:%.+]]> = first-order splice ir<%for.y>, ir<%for.x.prev>
+; CHECK-NEXT:     WIDEN-CAST ir<%for.y.i64> = sext  vp<[[SPLICE_Y]]> to i64
+; CHECK-NEXT:     vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
+; CHECK-NEXT:     WIDEN store vp<[[VEC_PTR]]>, ir<%for.y.i64>
+; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:     EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
+; CHECK-NEXT:   No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT:   EMIT vp<[[EXT_X:%.+]]> = extract-from-end ir<%for.x.next>, ir<1>
+; CHECK-NEXT:   EMIT vp<[[EXT_Y:%.+]]>.1 = extract-from-end ir<%for.x.prev>, ir<1>
+; CHECK-NEXT:   EMIT vp<[[MIDDLE_C:%.+]]> = icmp eq ir<4098>, vp<[[VTC]]>
+; CHECK-NEXT:   EMIT branch-on-cond vp<[[MIDDLE_C]]>
+; CHECK-NEXT: Successor(s): ir-bb<ret>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<ret>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT:   EMIT vp<[[RESUME_X:%.+]]> = resume-phi vp<[[EXT_X]]>, ir<0>
+; CHECK-NEXT:   EMIT vp<[[RESUME_Y:%.+]]>.1 = resume-phi vp<[[EXT_Y]]>.1, ir<0>
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: Live-out i64 %for.x = vp<[[RESUME_X]]>
+; CHECK-NEXT: Live-out i32 %for.y = vp<[[RESUME_Y]]>.1
+; CHECK-NEXT: }
 ;
 entry:
   br label %loop
@@ -178,7 +221,54 @@ ret:
 
 define i32 @test_chained_first_order_recurrences_5_hoist_to_load(ptr %base) {
 ; CHECK-LABEL: 'test_chained_first_order_recurrences_5_hoist_to_load'
-; CHECK: No VPlans built.
+; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<4098> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT:   vector.body:
+; CHECK-NEXT:     EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
+; CHECK-NEXT:     FIRST-ORDER-RECURRENCE-PHI ir<%for.x> = phi ir<0>, ir<%for.x.next>
+; CHECK-NEXT:     FIRST-ORDER-RECURRENCE-PHI ir<%for.y> = phi ir<0>, ir<%for.x.prev>
+; CHECK-NEXT:     vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
+; CHECK-NEXT:     CLONE ir<%gep> = getelementptr ir<%base>, vp<[[SCALAR_STEPS]]>
+; CHECK-NEXT:     vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
+; CHECK-NEXT:     WIDEN ir<%l> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT:     WIDEN ir<%for.x.next> = mul ir<%l>, ir<2>
+; CHECK-NEXT:     EMIT vp<[[SPLICE_X:%.]]> = first-order splice ir<%for.x>, ir<%for.x.next>
+; CHECK-NEXT:     WIDEN-CAST ir<%for.x.prev> = trunc  vp<[[SPLICE_X]]> to i32
+; CHECK-NEXT:     EMIT vp<[[SPLICE_Y:%.+]]> = first-order splice ir<%for.y>, ir<%for.x.prev>
+; CHECK-NEXT:     WIDEN-CAST ir<%for.y.i64> = sext  vp<[[SPLICE_Y]]> to i64
+; CHECK-NEXT:     vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
+; CHECK-NEXT:     WIDEN store vp<[[VEC_PTR]]>, ir<%for.y.i64>
+; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:     EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
+; CHECK-NEXT:   No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT:   EMIT vp<[[EXT_X:%.+]]> = extract-from-end ir<%for.x.next>, ir<1>
+; CHECK-NEXT:   EMIT vp<[[EXT_Y:%.+]]>.1 = extract-from-end ir<%for.x.prev>, ir<1>
+; CHECK-NEXT:   EMIT vp<[[MIDDLE_C:%.+]]> = icmp eq ir<4098>, vp<[[VTC]]>
+; CHECK-NEXT:   EMIT branch-on-cond vp<[[MIDDLE_C]]>
+; CHECK-NEXT: Successor(s): ir-bb<ret>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<ret>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT:   EMIT vp<[[RESUME_X:%.+]]> = resume-phi vp<[[EXT_X]]>, ir<0>
+; CHECK-NEXT:   EMIT vp<[[RESUME_Y:%.+]]>.1 = resume-phi vp<[[EXT_Y]]>.1, ir<0>
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: Live-out i64 %for.x = vp<[[RESUME_X]]>
+; CHECK-NEXT: Live-out i32 %for.y = vp<[[RESUME_Y]]>.1
+; CHECK-NEXT: }
 ;
 entry:
   br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll
index dbe373b46cce..20acc1053532 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll
@@ -123,9 +123,9 @@ define void @test_pr54223_sink_after_insertion_order(ptr noalias %a, ptr noalias
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi float [ [[TMP2]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT5:%.*]] = phi float [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT5:%.*]] = phi float [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -246,9 +246,9 @@ define void @test_pr54233_for_depend_on_each_other(ptr noalias %a, ptr noalias %
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP4]], i32 3
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT3:%.*]] = phi i32 [ [[TMP1]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT3:%.*]] = phi i32 [ [[TMP1]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -382,22 +382,57 @@ exit:
   ret void
 }
 
-define void @hoist_previous_value_and_operands(ptr %dst, i64 %mask) {
-; CHECK-LABEL: @hoist_previous_value_and_operands(
+; Similar to the test cases for https://github.com/llvm/llvm-project/issues/106523.
+; The previous truncation (%trunc) gets vectorized (rather than folded into an
+; IV) and hoisted along with its AND operand above the user 'or'.
+define void @hoist_previous_value_and_operand(ptr %dst, i64 %mask) {
+; CHECK-LABEL: @hoist_previous_value_and_operand(
 ; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[MASK:%.*]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 1>, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP4]] = trunc <4 x i64> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP6]] = or <4 x i32> [[TMP5]], <i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR1]], <4 x i32> [[TMP6]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 336
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP4]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT2:%.*]] = extractelement <4 x i32> [[TMP6]], i32 3
+; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 337, [[MIDDLE_BLOCK]] ], [ 1, [[BB:%.*]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 1, [[BB]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT3:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT2]], [[MIDDLE_BLOCK]] ], [ 0, [[BB]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[ADD:%.*]], [[LOOP]] ], [ 1, [[BB:%.*]] ]
-; CHECK-NEXT:    [[FOR_1:%.*]] = phi i32 [ [[TRUNC:%.*]], [[LOOP]] ], [ 1, [[BB]] ]
-; CHECK-NEXT:    [[FOR_2:%.*]] = phi i32 [ [[OR:%.*]], [[LOOP]] ], [ 0, [[BB]] ]
-; CHECK-NEXT:    [[OR]] = or i32 [[FOR_1]], 0
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TRUNC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[FOR_2:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT3]], [[SCALAR_PH]] ], [ [[OR:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OR]] = or i32 [[FOR_1]], 3
 ; CHECK-NEXT:    [[ADD]] = add i64 [[IV]], 1
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[IV]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
 ; CHECK-NEXT:    store i32 [[FOR_2]], ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[ICMP:%.*]] = icmp ult i64 [[IV]], 337
-; CHECK-NEXT:    [[A:%.*]] = and i64 [[IV]], [[MASK:%.*]]
+; CHECK-NEXT:    [[A:%.*]] = and i64 [[IV]], [[MASK]]
 ; CHECK-NEXT:    [[TRUNC]] = trunc i64 [[A]] to i32
-; CHECK-NEXT:    br i1 [[ICMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK-NEXT:    br i1 [[ICMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -405,10 +440,10 @@ bb:
   br label %loop
 
 loop:
-  %iv = phi i64 [ %add, %loop ], [ 1, %bb ]
-  %for.1 = phi i32 [ %trunc, %loop ], [ 1, %bb ]
-  %for.2 = phi i32 [ %or, %loop ], [ 0, %bb ]
-  %or = or i32 %for.1, 0
+  %iv = phi i64 [ 1, %bb ], [ %add, %loop ]
+  %for.1 = phi i32 [ 1, %bb ], [ %trunc, %loop ]
+  %for.2 = phi i32 [ 0, %bb ], [ %or, %loop ]
+  %or = or i32 %for.1, 3
   %add = add i64 %iv, 1
   %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
   store i32 %for.2, ptr %gep, align 4
-- 
GitLab


From e2766b2bceb75dcb946a67e9115e1548296c133f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Wed, 23 Oct 2024 13:44:02 -0700
Subject: [PATCH 016/693] [flang][cuda] Add entry point to launch cuda fortran
 kernel (#113490)

---
 flang/include/flang/Runtime/CUDA/common.h |  2 +-
 flang/include/flang/Runtime/CUDA/kernel.h | 27 +++++++++++++++++++
 flang/runtime/CUDA/CMakeLists.txt         |  1 +
 flang/runtime/CUDA/kernel.cpp             | 33 +++++++++++++++++++++++
 4 files changed, 62 insertions(+), 1 deletion(-)
 create mode 100644 flang/include/flang/Runtime/CUDA/kernel.h
 create mode 100644 flang/runtime/CUDA/kernel.cpp

diff --git a/flang/include/flang/Runtime/CUDA/common.h b/flang/include/flang/Runtime/CUDA/common.h
index b73bc390ea8c..e9f61932230e 100644
--- a/flang/include/flang/Runtime/CUDA/common.h
+++ b/flang/include/flang/Runtime/CUDA/common.h
@@ -30,7 +30,7 @@ static constexpr unsigned kDeviceToDevice = 2;
     const char *name = cudaGetErrorName(err); \
     if (!name) \
       name = "<unknown>"; \
-    Terminator terminator{__FILE__, __LINE__}; \
+    Fortran::runtime::Terminator terminator{__FILE__, __LINE__}; \
     terminator.Crash("'%s' failed with '%s'", #expr, name); \
   }(expr)
 
diff --git a/flang/include/flang/Runtime/CUDA/kernel.h b/flang/include/flang/Runtime/CUDA/kernel.h
new file mode 100644
index 000000000000..cf07d874a082
--- /dev/null
+++ b/flang/include/flang/Runtime/CUDA/kernel.h
@@ -0,0 +1,27 @@
+//===-- include/flang/Runtime/CUDA/kernel.h ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_RUNTIME_CUDA_KERNEL_H_
+#define FORTRAN_RUNTIME_CUDA_KERNEL_H_
+
+#include "flang/Runtime/entry-names.h"
+#include <cstddef>
+#include <stdint.h>
+
+extern "C" {
+
+// This function uses intptr_t instead of CUDA's unsigned int to match
+// the type of MLIR's index type. This avoids the need for casts in the
+// generated MLIR code.
+void RTDEF(CUFLaunchKernel)(const void *kernelName, intptr_t gridX,
+    intptr_t gridY, intptr_t gridZ, intptr_t blockX, intptr_t blockY,
+    intptr_t blockZ, int32_t smem, void **params, void **extra);
+
+} // extern "C"
+
+#endif // FORTRAN_RUNTIME_CUDA_KERNEL_H_
diff --git a/flang/runtime/CUDA/CMakeLists.txt b/flang/runtime/CUDA/CMakeLists.txt
index 86523b419f87..ce87f3efdc36 100644
--- a/flang/runtime/CUDA/CMakeLists.txt
+++ b/flang/runtime/CUDA/CMakeLists.txt
@@ -17,6 +17,7 @@ add_flang_library(${CUFRT_LIBNAME}
   allocator.cpp
   allocatable.cpp
   descriptor.cpp
+  kernel.cpp
   memory.cpp
   registration.cpp
 )
diff --git a/flang/runtime/CUDA/kernel.cpp b/flang/runtime/CUDA/kernel.cpp
new file mode 100644
index 000000000000..f81153a1af4b
--- /dev/null
+++ b/flang/runtime/CUDA/kernel.cpp
@@ -0,0 +1,33 @@
+//===-- runtime/CUDA/kernel.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Runtime/CUDA/kernel.h"
+#include "../terminator.h"
+#include "flang/Runtime/CUDA/common.h"
+
+#include "cuda_runtime.h"
+
+extern "C" {
+
+void RTDEF(CUFLaunchKernel)(const void *kernel, intptr_t gridX, intptr_t gridY,
+    intptr_t gridZ, intptr_t blockX, intptr_t blockY, intptr_t blockZ,
+    int32_t smem, void **params, void **extra) {
+  dim3 gridDim;
+  gridDim.x = gridX;
+  gridDim.y = gridY;
+  gridDim.z = gridZ;
+  dim3 blockDim;
+  blockDim.x = blockX;
+  blockDim.y = blockY;
+  blockDim.z = blockZ;
+  cudaStream_t stream = 0;
+  CUDA_REPORT_IF_ERROR(
+      cudaLaunchKernel(kernel, gridDim, blockDim, params, smem, stream));
+}
+
+} // extern "C"
-- 
GitLab


From d269ec321dfdfcd817544da0a94dc42c109694a2 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Wed, 23 Oct 2024 17:07:15 -0400
Subject: [PATCH 017/693] [libc++] Add a escape hatch for making
 __libcpp_verbose_abort non-noexcept again (#113310)

This allows a slightly smoother transition for people after #109151, as
requested on that PR.
---
 libcxx/docs/ReleaseNotes/20.rst |  7 ++++++-
 libcxx/include/__verbose_abort  | 10 ++++++++--
 libcxx/src/verbose_abort.cpp    |  2 +-
 3 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/libcxx/docs/ReleaseNotes/20.rst b/libcxx/docs/ReleaseNotes/20.rst
index 44912d2ddafa..39546493ae8d 100644
--- a/libcxx/docs/ReleaseNotes/20.rst
+++ b/libcxx/docs/ReleaseNotes/20.rst
@@ -83,7 +83,9 @@ Deprecations and Removals
   ambiguity in name lookup. Code that expects such ambiguity will possibly not compile in LLVM 20.
 
 - The function ``__libcpp_verbose_abort()`` is now ``noexcept``, to match ``std::terminate()``. (The combination of
-  ``noexcept`` and ``[[noreturn]]`` has special significance for function effects analysis.)
+  ``noexcept`` and ``[[noreturn]]`` has special significance for function effects analysis.) For backwards compatibility,
+  the ``_LIBCPP_VERBOSE_ABORT_NOT_NOEXCEPT`` macro can be defined to make the function non-``noexcept``. That macro
+  will be removed in LLVM 21.
 
 Upcoming Deprecations and Removals
 ----------------------------------
@@ -106,6 +108,9 @@ LLVM 21
   If you are using C++03 in your project, you should consider moving to a newer version of the Standard to get the most
   out of libc++.
 
+- The ``_LIBCPP_VERBOSE_ABORT_NOT_NOEXCEPT`` macro will be removed in LLVM 21, making ``std::__libcpp_verbose_abort``
+  unconditionally ``noexcept``.
+
 
 ABI Affecting Changes
 ---------------------
diff --git a/libcxx/include/__verbose_abort b/libcxx/include/__verbose_abort
index 73295cae4261..2d45cd0eb7f5 100644
--- a/libcxx/include/__verbose_abort
+++ b/libcxx/include/__verbose_abort
@@ -18,10 +18,16 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+#if defined(_LIBCPP_VERBOSE_ABORT_NOT_NOEXCEPT)
+#  define _LIBCPP_VERBOSE_ABORT_NOEXCEPT
+#else
+#  define _LIBCPP_VERBOSE_ABORT_NOEXCEPT _NOEXCEPT
+#endif
+
 // This function should never be called directly from the code -- it should only be called through
 // the _LIBCPP_VERBOSE_ABORT macro.
-[[__noreturn__]] _LIBCPP_AVAILABILITY_VERBOSE_ABORT _LIBCPP_OVERRIDABLE_FUNC_VIS
-_LIBCPP_ATTRIBUTE_FORMAT(__printf__, 1, 2) void __libcpp_verbose_abort(const char* __format, ...) _NOEXCEPT;
+[[__noreturn__]] _LIBCPP_AVAILABILITY_VERBOSE_ABORT _LIBCPP_OVERRIDABLE_FUNC_VIS _LIBCPP_ATTRIBUTE_FORMAT(
+    __printf__, 1, 2) void __libcpp_verbose_abort(const char* __format, ...) _LIBCPP_VERBOSE_ABORT_NOEXCEPT;
 
 // _LIBCPP_VERBOSE_ABORT(format, args...)
 //
diff --git a/libcxx/src/verbose_abort.cpp b/libcxx/src/verbose_abort.cpp
index 0019063405a8..6704709d247c 100644
--- a/libcxx/src/verbose_abort.cpp
+++ b/libcxx/src/verbose_abort.cpp
@@ -28,7 +28,7 @@ extern "C" void android_set_abort_message(const char* msg);
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-_LIBCPP_WEAK void __libcpp_verbose_abort(char const* format, ...) noexcept {
+_LIBCPP_WEAK void __libcpp_verbose_abort(char const* format, ...) _LIBCPP_VERBOSE_ABORT_NOEXCEPT {
   // Write message to stderr. We do this before formatting into a
   // buffer so that we still get some information out if that fails.
   {
-- 
GitLab


From c9199700b8fdde50a920857207e4387cfe69da45 Mon Sep 17 00:00:00 2001
From: Shubham Sandeep Rastogi <srastogi22@apple.com>
Date: Wed, 23 Oct 2024 14:31:15 -0700
Subject: [PATCH 018/693] Revert "[lldb] Fix crash missing MSInheritanceAttr on
 CXXRecordDecl with DWARF on Windows" (#113498)

Reverts llvm/llvm-project#112928

This is because it broke greendragon:
SymbolFile/DWARF/x86/member-pointers.cpp
---
 .../TypeSystem/Clang/TypeSystemClang.cpp      |  3 ---
 .../SymbolFile/DWARF/x86/member-pointers.cpp  | 24 -------------------
 2 files changed, 27 deletions(-)
 delete mode 100644 lldb/test/Shell/SymbolFile/DWARF/x86/member-pointers.cpp

diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index f5063175d6e0..a57cd8efce8a 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -2771,9 +2771,6 @@ static bool GetCompleteQualType(clang::ASTContext *ast,
         ast, llvm::cast<clang::AttributedType>(qual_type)->getModifiedType(),
         allow_completion);
 
-  case clang::Type::MemberPointer:
-    return !qual_type.getTypePtr()->isIncompleteType();
-
   default:
     break;
   }
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/member-pointers.cpp b/lldb/test/Shell/SymbolFile/DWARF/x86/member-pointers.cpp
deleted file mode 100644
index 817833d4fa75..000000000000
--- a/lldb/test/Shell/SymbolFile/DWARF/x86/member-pointers.cpp
+++ /dev/null
@@ -1,24 +0,0 @@
-// REQUIRES: lld
-
-// Itanium ABI:
-// RUN: %clang --target=x86_64-pc-linux -gdwarf -c -o %t_linux.o %s
-// RUN: %lldb -f %t_linux.o -b -o "target variable mp" | FileCheck %s
-//
-// CHECK: (char SI::*) mp = 0x0000000000000000
-
-// Microsoft ABI:
-// RUN: %clang_cl --target=x86_64-windows-msvc -c -gdwarf %s -o %t_win.obj
-// RUN: lld-link /out:%t_win.exe %t_win.obj /nodefaultlib /entry:main /debug
-// RUN: %lldb -f %t_win.exe -b -o "target variable mp" | FileCheck --check-prefix=CHECK-MSVC %s
-//
-// DWARF has no representation of MSInheritanceAttr, so we cannot determine the size
-// of member-pointers yet. For the moment, make sure we don't crash on such variables.
-// CHECK-MSVC: error: Unable to determine byte size.
-
-struct SI {
-  char si;
-};
-
-char SI::*mp = &SI::si;
-
-int main() { return 0; }
-- 
GitLab


From e78f53d1e8d622ee4b12dbc2ac8252b4805d5719 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Thu, 24 Oct 2024 00:17:37 +0200
Subject: [PATCH 019/693] Reapply "[libc++][C++03] Copy the LLVM 19 headers
 (#108999)" (#112127)

This reverts commit 68c04b0ae62d8431d72d8b47fc13008002ee4387.

This disables the IWYU mapping that caused the failure, since
the headers aren't reachable for now.

This is the first part of the "Freezing C++03 headers" proposal
explained in
https://discourse.llvm.org/t/rfc-freezing-c-03-headers-in-libc/77319/58.

This patch mechanically copies the headers as of the LLVM 19.1 release
into a subdirectory of libc++ so that we can start using these headers
when building in C++03 mode. We are going to be backporting important
changes to that copy of the headers until the LLVM 21 release. After the
LLVM 21 release, only critical bugfixes will be fixed in the C++03 copy
of the headers.

This patch only performs a copy of the headers -- these headers are
still unused by the rest of the codebase.
---
 libcxx/include/__cxx03/CMakeLists.txt         | 1092 +++
 .../__cxx03/__algorithm/adjacent_find.h       |   58 +
 libcxx/include/__cxx03/__algorithm/all_of.h   |   32 +
 libcxx/include/__cxx03/__algorithm/any_of.h   |   32 +
 .../__cxx03/__algorithm/binary_search.h       |   39 +
 libcxx/include/__cxx03/__algorithm/clamp.h    |   44 +
 libcxx/include/__cxx03/__algorithm/comp.h     |   49 +
 .../__cxx03/__algorithm/comp_ref_type.h       |   67 +
 libcxx/include/__cxx03/__algorithm/copy.h     |  123 +
 .../__cxx03/__algorithm/copy_backward.h       |  137 +
 libcxx/include/__cxx03/__algorithm/copy_if.h  |   34 +
 .../__cxx03/__algorithm/copy_move_common.h    |  114 +
 libcxx/include/__cxx03/__algorithm/copy_n.h   |   60 +
 libcxx/include/__cxx03/__algorithm/count.h    |   92 +
 libcxx/include/__cxx03/__algorithm/count_if.h |   35 +
 libcxx/include/__cxx03/__algorithm/equal.h    |  133 +
 .../include/__cxx03/__algorithm/equal_range.h |   85 +
 libcxx/include/__cxx03/__algorithm/fill.h     |   45 +
 libcxx/include/__cxx03/__algorithm/fill_n.h   |   98 +
 libcxx/include/__cxx03/__algorithm/find.h     |  181 +
 libcxx/include/__cxx03/__algorithm/find_end.h |  225 +
 .../__cxx03/__algorithm/find_first_of.h       |   55 +
 libcxx/include/__cxx03/__algorithm/find_if.h  |   32 +
 .../include/__cxx03/__algorithm/find_if_not.h |   32 +
 .../__cxx03/__algorithm/find_segment_if.h     |   62 +
 libcxx/include/__cxx03/__algorithm/fold.h     |  128 +
 libcxx/include/__cxx03/__algorithm/for_each.h |   57 +
 .../include/__cxx03/__algorithm/for_each_n.h  |   41 +
 .../__cxx03/__algorithm/for_each_segment.h    |   53 +
 libcxx/include/__cxx03/__algorithm/generate.h |   29 +
 .../include/__cxx03/__algorithm/generate_n.h  |   33 +
 .../__cxx03/__algorithm/half_positive.h       |   37 +
 .../__cxx03/__algorithm/in_found_result.h     |   54 +
 .../__cxx03/__algorithm/in_fun_result.h       |   54 +
 .../__cxx03/__algorithm/in_in_out_result.h    |   59 +
 .../__cxx03/__algorithm/in_in_result.h        |   56 +
 .../__cxx03/__algorithm/in_out_out_result.h   |   57 +
 .../__cxx03/__algorithm/in_out_result.h       |   56 +
 libcxx/include/__cxx03/__algorithm/includes.h |   79 +
 .../__cxx03/__algorithm/inplace_merge.h       |  240 +
 libcxx/include/__cxx03/__algorithm/is_heap.h  |   38 +
 .../__cxx03/__algorithm/is_heap_until.h       |   62 +
 .../__cxx03/__algorithm/is_partitioned.h      |   37 +
 .../__cxx03/__algorithm/is_permutation.h      |  308 +
 .../include/__cxx03/__algorithm/is_sorted.h   |   38 +
 .../__cxx03/__algorithm/is_sorted_until.h     |   51 +
 .../include/__cxx03/__algorithm/iter_swap.h   |   31 +
 .../__cxx03/__algorithm/iterator_operations.h |  223 +
 .../__algorithm/lexicographical_compare.h     |   57 +
 .../lexicographical_compare_three_way.h       |  125 +
 .../include/__cxx03/__algorithm/lower_bound.h |  109 +
 .../include/__cxx03/__algorithm/make_heap.h   |   60 +
 .../__cxx03/__algorithm/make_projected.h      |  106 +
 libcxx/include/__cxx03/__algorithm/max.h      |   58 +
 .../include/__cxx03/__algorithm/max_element.h |   51 +
 libcxx/include/__cxx03/__algorithm/merge.h    |   69 +
 libcxx/include/__cxx03/__algorithm/min.h      |   58 +
 .../include/__cxx03/__algorithm/min_element.h |   71 +
 .../__cxx03/__algorithm/min_max_result.h      |   56 +
 libcxx/include/__cxx03/__algorithm/minmax.h   |   59 +
 .../__cxx03/__algorithm/minmax_element.h      |  100 +
 libcxx/include/__cxx03/__algorithm/mismatch.h |  217 +
 libcxx/include/__cxx03/__algorithm/move.h     |  127 +
 .../__cxx03/__algorithm/move_backward.h       |  137 +
 .../__cxx03/__algorithm/next_permutation.h    |   75 +
 libcxx/include/__cxx03/__algorithm/none_of.h  |   32 +
 .../include/__cxx03/__algorithm/nth_element.h |  261 +
 .../__cxx03/__algorithm/partial_sort.h        |   91 +
 .../__cxx03/__algorithm/partial_sort_copy.h   |  106 +
 .../include/__cxx03/__algorithm/partition.h   |   90 +
 .../__cxx03/__algorithm/partition_copy.h      |   43 +
 .../__cxx03/__algorithm/partition_point.h     |   44 +
 libcxx/include/__cxx03/__algorithm/pop_heap.h |   81 +
 .../__cxx03/__algorithm/prev_permutation.h    |   75 +
 libcxx/include/__cxx03/__algorithm/pstl.h     |  663 ++
 .../include/__cxx03/__algorithm/push_heap.h   |   84 +
 .../__algorithm/ranges_adjacent_find.h        |   83 +
 .../__cxx03/__algorithm/ranges_all_of.h       |   74 +
 .../__cxx03/__algorithm/ranges_any_of.h       |   74 +
 .../__algorithm/ranges_binary_search.h        |   73 +
 .../__cxx03/__algorithm/ranges_clamp.h        |   66 +
 .../__cxx03/__algorithm/ranges_contains.h     |   66 +
 .../__algorithm/ranges_contains_subrange.h    |   97 +
 .../include/__cxx03/__algorithm/ranges_copy.h |   71 +
 .../__algorithm/ranges_copy_backward.h        |   69 +
 .../__cxx03/__algorithm/ranges_copy_if.h      |   87 +
 .../__cxx03/__algorithm/ranges_copy_n.h       |   81 +
 .../__cxx03/__algorithm/ranges_count.h        |   66 +
 .../__cxx03/__algorithm/ranges_count_if.h     |   79 +
 .../__cxx03/__algorithm/ranges_ends_with.h    |  201 +
 .../__cxx03/__algorithm/ranges_equal.h        |  109 +
 .../__cxx03/__algorithm/ranges_equal_range.h  |   80 +
 .../include/__cxx03/__algorithm/ranges_fill.h |   62 +
 .../__cxx03/__algorithm/ranges_fill_n.h       |   53 +
 .../include/__cxx03/__algorithm/ranges_find.h |   80 +
 .../__cxx03/__algorithm/ranges_find_end.h     |  103 +
 .../__algorithm/ranges_find_first_of.h        |  106 +
 .../__cxx03/__algorithm/ranges_find_if.h      |   75 +
 .../__cxx03/__algorithm/ranges_find_if_not.h  |   69 +
 .../__cxx03/__algorithm/ranges_find_last.h    |  175 +
 .../__cxx03/__algorithm/ranges_for_each.h     |   81 +
 .../__cxx03/__algorithm/ranges_for_each_n.h   |   64 +
 .../__cxx03/__algorithm/ranges_generate.h     |   73 +
 .../__cxx03/__algorithm/ranges_generate_n.h   |   65 +
 .../__cxx03/__algorithm/ranges_includes.h     |   98 +
 .../__algorithm/ranges_inplace_merge.h        |   84 +
 .../__cxx03/__algorithm/ranges_is_heap.h      |   81 +
 .../__algorithm/ranges_is_heap_until.h        |   81 +
 .../__algorithm/ranges_is_partitioned.h       |   86 +
 .../__algorithm/ranges_is_permutation.h       |  107 +
 .../__cxx03/__algorithm/ranges_is_sorted.h    |   67 +
 .../__algorithm/ranges_is_sorted_until.h      |   82 +
 .../__algorithm/ranges_iterator_concept.h     |   56 +
 .../ranges_lexicographical_compare.h          |  106 +
 .../__cxx03/__algorithm/ranges_lower_bound.h  |   73 +
 .../__cxx03/__algorithm/ranges_make_heap.h    |   85 +
 .../include/__cxx03/__algorithm/ranges_max.h  |  103 +
 .../__cxx03/__algorithm/ranges_max_element.h  |   69 +
 .../__cxx03/__algorithm/ranges_merge.h        |  138 +
 .../include/__cxx03/__algorithm/ranges_min.h  |   95 +
 .../__cxx03/__algorithm/ranges_min_element.h  |   81 +
 .../__cxx03/__algorithm/ranges_minmax.h       |  175 +
 .../__algorithm/ranges_minmax_element.h       |   78 +
 .../__cxx03/__algorithm/ranges_mismatch.h     |  100 +
 .../include/__cxx03/__algorithm/ranges_move.h |   74 +
 .../__algorithm/ranges_move_backward.h        |   76 +
 .../__algorithm/ranges_next_permutation.h     |   78 +
 .../__cxx03/__algorithm/ranges_none_of.h      |   75 +
 .../__cxx03/__algorithm/ranges_nth_element.h  |   84 +
 .../__cxx03/__algorithm/ranges_partial_sort.h |   82 +
 .../__algorithm/ranges_partial_sort_copy.h    |  114 +
 .../__cxx03/__algorithm/ranges_partition.h    |   88 +
 .../__algorithm/ranges_partition_copy.h       |  110 +
 .../__algorithm/ranges_partition_point.h      |   93 +
 .../__cxx03/__algorithm/ranges_pop_heap.h     |   86 +
 .../__algorithm/ranges_prev_permutation.h     |   78 +
 .../__cxx03/__algorithm/ranges_push_heap.h    |   85 +
 .../__cxx03/__algorithm/ranges_remove.h       |   68 +
 .../__cxx03/__algorithm/ranges_remove_copy.h  |   81 +
 .../__algorithm/ranges_remove_copy_if.h       |   95 +
 .../__cxx03/__algorithm/ranges_remove_if.h    |   89 +
 .../__cxx03/__algorithm/ranges_replace.h      |   68 +
 .../__cxx03/__algorithm/ranges_replace_copy.h |   93 +
 .../__algorithm/ranges_replace_copy_if.h      |   98 +
 .../__cxx03/__algorithm/ranges_replace_if.h   |   81 +
 .../__cxx03/__algorithm/ranges_reverse.h      |   79 +
 .../__cxx03/__algorithm/ranges_reverse_copy.h |   70 +
 .../__cxx03/__algorithm/ranges_rotate.h       |   71 +
 .../__cxx03/__algorithm/ranges_rotate_copy.h  |   68 +
 .../__cxx03/__algorithm/ranges_sample.h       |   74 +
 .../__cxx03/__algorithm/ranges_search.h       |  134 +
 .../__cxx03/__algorithm/ranges_search_n.h     |  116 +
 .../__algorithm/ranges_set_difference.h       |  109 +
 .../__algorithm/ranges_set_intersection.h     |  114 +
 .../ranges_set_symmetric_difference.h         |  114 +
 .../__cxx03/__algorithm/ranges_set_union.h    |  115 +
 .../__cxx03/__algorithm/ranges_shuffle.h      |   72 +
 .../include/__cxx03/__algorithm/ranges_sort.h |   84 +
 .../__cxx03/__algorithm/ranges_sort_heap.h    |   85 +
 .../__algorithm/ranges_stable_partition.h     |   92 +
 .../__cxx03/__algorithm/ranges_stable_sort.h  |   82 +
 .../__cxx03/__algorithm/ranges_starts_with.h  |   95 +
 .../__cxx03/__algorithm/ranges_swap_ranges.h  |   70 +
 .../__cxx03/__algorithm/ranges_transform.h    |  177 +
 .../__cxx03/__algorithm/ranges_unique.h       |   82 +
 .../__cxx03/__algorithm/ranges_unique_copy.h  |  120 +
 .../__cxx03/__algorithm/ranges_upper_bound.h  |   74 +
 libcxx/include/__cxx03/__algorithm/remove.h   |   46 +
 .../include/__cxx03/__algorithm/remove_copy.h |   34 +
 .../__cxx03/__algorithm/remove_copy_if.h      |   34 +
 .../include/__cxx03/__algorithm/remove_if.h   |   45 +
 libcxx/include/__cxx03/__algorithm/replace.h  |   30 +
 .../__cxx03/__algorithm/replace_copy.h        |   37 +
 .../__cxx03/__algorithm/replace_copy_if.h     |   37 +
 .../include/__cxx03/__algorithm/replace_if.h  |   30 +
 libcxx/include/__cxx03/__algorithm/reverse.h  |   62 +
 .../__cxx03/__algorithm/reverse_copy.h        |   30 +
 libcxx/include/__cxx03/__algorithm/rotate.h   |  198 +
 .../include/__cxx03/__algorithm/rotate_copy.h |   29 +
 libcxx/include/__cxx03/__algorithm/sample.h   |  123 +
 libcxx/include/__cxx03/__algorithm/search.h   |  192 +
 libcxx/include/__cxx03/__algorithm/search_n.h |  155 +
 .../__cxx03/__algorithm/set_difference.h      |   79 +
 .../__cxx03/__algorithm/set_intersection.h    |  217 +
 .../__algorithm/set_symmetric_difference.h    |  109 +
 .../include/__cxx03/__algorithm/set_union.h   |  105 +
 .../include/__cxx03/__algorithm/shift_left.h  |   59 +
 .../include/__cxx03/__algorithm/shift_right.h |  105 +
 libcxx/include/__cxx03/__algorithm/shuffle.h  |  167 +
 .../include/__cxx03/__algorithm/sift_down.h   |  118 +
 .../include/__cxx03/__algorithm/simd_utils.h  |  164 +
 libcxx/include/__cxx03/__algorithm/sort.h     | 1016 +++
 .../include/__cxx03/__algorithm/sort_heap.h   |   63 +
 .../__cxx03/__algorithm/stable_partition.h    |  307 +
 .../include/__cxx03/__algorithm/stable_sort.h |  273 +
 .../include/__cxx03/__algorithm/swap_ranges.h |   62 +
 .../__algorithm/three_way_comp_ref_type.h     |   74 +
 .../include/__cxx03/__algorithm/transform.h   |   42 +
 .../uniform_random_bit_generator_adaptor.h    |   58 +
 libcxx/include/__cxx03/__algorithm/unique.h   |   64 +
 .../include/__cxx03/__algorithm/unique_copy.h |  127 +
 .../include/__cxx03/__algorithm/unwrap_iter.h |   85 +
 .../__cxx03/__algorithm/unwrap_range.h        |   99 +
 .../include/__cxx03/__algorithm/upper_bound.h |   68 +
 libcxx/include/__cxx03/__assert               |  118 +
 libcxx/include/__cxx03/__atomic/aliases.h     |  110 +
 libcxx/include/__cxx03/__atomic/atomic.h      |  622 ++
 libcxx/include/__cxx03/__atomic/atomic_base.h |  221 +
 libcxx/include/__cxx03/__atomic/atomic_flag.h |  189 +
 libcxx/include/__cxx03/__atomic/atomic_init.h |   25 +
 .../__cxx03/__atomic/atomic_lock_free.h       |   48 +
 libcxx/include/__cxx03/__atomic/atomic_ref.h  |  378 ++
 libcxx/include/__cxx03/__atomic/atomic_sync.h |  205 +
 .../__cxx03/__atomic/check_memory_order.h     |   34 +
 .../include/__cxx03/__atomic/contention_t.h   |   32 +
 .../__cxx03/__atomic/cxx_atomic_impl.h        |  510 ++
 libcxx/include/__cxx03/__atomic/fence.h       |   28 +
 .../__cxx03/__atomic/is_always_lock_free.h    |   28 +
 .../__cxx03/__atomic/kill_dependency.h        |   27 +
 .../include/__cxx03/__atomic/memory_order.h   |   65 +
 .../include/__cxx03/__atomic/to_gcc_order.h   |   54 +
 libcxx/include/__cxx03/__bit/bit_cast.h       |   44 +
 libcxx/include/__cxx03/__bit/bit_ceil.h       |   54 +
 libcxx/include/__cxx03/__bit/bit_floor.h      |   34 +
 libcxx/include/__cxx03/__bit/bit_log2.h       |   34 +
 libcxx/include/__cxx03/__bit/bit_width.h      |   33 +
 libcxx/include/__cxx03/__bit/blsr.h           |   34 +
 libcxx/include/__cxx03/__bit/byteswap.h       |   53 +
 libcxx/include/__cxx03/__bit/countl.h         |  113 +
 libcxx/include/__cxx03/__bit/countr.h         |   84 +
 libcxx/include/__cxx03/__bit/endian.h         |   38 +
 libcxx/include/__cxx03/__bit/has_single_bit.h |   37 +
 libcxx/include/__cxx03/__bit/invert_if.h      |   30 +
 libcxx/include/__cxx03/__bit/popcount.h       |   71 +
 libcxx/include/__cxx03/__bit/rotate.h         |   72 +
 libcxx/include/__cxx03/__bit_reference        | 1024 +++
 .../include/__cxx03/__charconv/chars_format.h |   61 +
 .../__cxx03/__charconv/from_chars_integral.h  |  240 +
 .../__cxx03/__charconv/from_chars_result.h    |   39 +
 libcxx/include/__cxx03/__charconv/tables.h    |  163 +
 libcxx/include/__cxx03/__charconv/to_chars.h  |   25 +
 .../__cxx03/__charconv/to_chars_base_10.h     |  188 +
 .../__charconv/to_chars_floating_point.h      |   55 +
 .../__cxx03/__charconv/to_chars_integral.h    |  327 +
 .../__cxx03/__charconv/to_chars_result.h      |   39 +
 libcxx/include/__cxx03/__charconv/traits.h    |  200 +
 libcxx/include/__cxx03/__chrono/calendar.h    |   44 +
 libcxx/include/__cxx03/__chrono/concepts.h    |   36 +
 .../__cxx03/__chrono/convert_to_timespec.h    |   51 +
 .../include/__cxx03/__chrono/convert_to_tm.h  |  202 +
 libcxx/include/__cxx03/__chrono/day.h         |   99 +
 libcxx/include/__cxx03/__chrono/duration.h    |  550 ++
 libcxx/include/__cxx03/__chrono/exception.h   |  135 +
 libcxx/include/__cxx03/__chrono/file_clock.h  |   80 +
 libcxx/include/__cxx03/__chrono/formatter.h   |  990 +++
 libcxx/include/__cxx03/__chrono/hh_mm_ss.h    |  112 +
 .../__cxx03/__chrono/high_resolution_clock.h  |   35 +
 libcxx/include/__cxx03/__chrono/leap_second.h |  126 +
 libcxx/include/__cxx03/__chrono/literals.h    |   45 +
 libcxx/include/__cxx03/__chrono/local_info.h  |   50 +
 libcxx/include/__cxx03/__chrono/month.h       |  115 +
 .../include/__cxx03/__chrono/month_weekday.h  |  105 +
 libcxx/include/__cxx03/__chrono/monthday.h    |  133 +
 libcxx/include/__cxx03/__chrono/ostream.h     |  322 +
 .../__cxx03/__chrono/parser_std_format_spec.h |  416 ++
 .../__cxx03/__chrono/statically_widen.h       |   52 +
 .../include/__cxx03/__chrono/steady_clock.h   |   42 +
 libcxx/include/__cxx03/__chrono/sys_info.h    |   51 +
 .../include/__cxx03/__chrono/system_clock.h   |   52 +
 libcxx/include/__cxx03/__chrono/time_point.h  |  220 +
 libcxx/include/__cxx03/__chrono/time_zone.h   |  182 +
 .../include/__cxx03/__chrono/time_zone_link.h |   79 +
 libcxx/include/__cxx03/__chrono/tzdb.h        |   94 +
 libcxx/include/__cxx03/__chrono/tzdb_list.h   |  108 +
 libcxx/include/__cxx03/__chrono/weekday.h     |  186 +
 libcxx/include/__cxx03/__chrono/year.h        |  118 +
 libcxx/include/__cxx03/__chrono/year_month.h  |  123 +
 .../include/__cxx03/__chrono/year_month_day.h |  337 +
 .../__cxx03/__chrono/year_month_weekday.h     |  287 +
 libcxx/include/__cxx03/__chrono/zoned_time.h  |  227 +
 .../__compare/common_comparison_category.h    |   86 +
 .../compare_partial_order_fallback.h          |   76 +
 .../__compare/compare_strong_order_fallback.h |   73 +
 .../__cxx03/__compare/compare_three_way.h     |   40 +
 .../__compare/compare_three_way_result.h      |   45 +
 .../__compare/compare_weak_order_fallback.h   |   73 +
 libcxx/include/__cxx03/__compare/is_eq.h      |   34 +
 libcxx/include/__cxx03/__compare/ordering.h   |  278 +
 .../include/__cxx03/__compare/partial_order.h |   77 +
 .../include/__cxx03/__compare/strong_order.h  |  143 +
 .../__cxx03/__compare/synth_three_way.h       |   52 +
 .../__cxx03/__compare/three_way_comparable.h  |   55 +
 libcxx/include/__cxx03/__compare/weak_order.h |  105 +
 .../include/__cxx03/__concepts/arithmetic.h   |   56 +
 .../include/__cxx03/__concepts/assignable.h   |   41 +
 .../__cxx03/__concepts/boolean_testable.h     |   38 +
 .../__cxx03/__concepts/class_or_enum.h        |   35 +
 .../__concepts/common_reference_with.h        |   36 +
 .../include/__cxx03/__concepts/common_with.h  |   52 +
 .../__cxx03/__concepts/constructible.h        |   55 +
 .../__cxx03/__concepts/convertible_to.h       |   33 +
 libcxx/include/__cxx03/__concepts/copyable.h  |   41 +
 .../include/__cxx03/__concepts/derived_from.h |   33 +
 .../include/__cxx03/__concepts/destructible.h |   32 +
 .../__cxx03/__concepts/different_from.h       |   31 +
 .../__cxx03/__concepts/equality_comparable.h  |   56 +
 libcxx/include/__cxx03/__concepts/invocable.h |   40 +
 libcxx/include/__cxx03/__concepts/movable.h   |   35 +
 libcxx/include/__cxx03/__concepts/predicate.h |   34 +
 libcxx/include/__cxx03/__concepts/regular.h   |   33 +
 libcxx/include/__cxx03/__concepts/relation.h  |   43 +
 libcxx/include/__cxx03/__concepts/same_as.h   |   35 +
 .../include/__cxx03/__concepts/semiregular.h  |   33 +
 libcxx/include/__cxx03/__concepts/swappable.h |  123 +
 .../__cxx03/__concepts/totally_ordered.h      |   59 +
 .../__condition_variable/condition_variable.h |  244 +
 libcxx/include/__cxx03/__config               | 1228 ++++
 libcxx/include/__cxx03/__config_site.in       |   56 +
 libcxx/include/__cxx03/__configuration/abi.h  |  172 +
 .../__cxx03/__configuration/availability.h    |  400 ++
 .../__cxx03/__configuration/compiler.h        |   51 +
 .../__cxx03/__configuration/language.h        |   46 +
 .../__cxx03/__configuration/platform.h        |   54 +
 .../__cxx03/__coroutine/coroutine_handle.h    |  176 +
 .../__cxx03/__coroutine/coroutine_traits.h    |   48 +
 .../__coroutine/noop_coroutine_handle.h       |   99 +
 .../__cxx03/__coroutine/trivial_awaitables.h  |   40 +
 .../__cxx03/__debug_utils/randomize_range.h   |   42 +
 .../__cxx03/__debug_utils/sanitizers.h        |  104 +
 .../strict_weak_ordering_check.h              |   77 +
 .../include/__cxx03/__exception/exception.h   |   94 +
 .../__cxx03/__exception/exception_ptr.h       |  177 +
 .../__cxx03/__exception/nested_exception.h    |   99 +
 .../include/__cxx03/__exception/operations.h  |   41 +
 .../include/__cxx03/__exception/terminate.h   |   22 +
 .../__cxx03/__expected/bad_expected_access.h  |   73 +
 libcxx/include/__cxx03/__expected/expected.h  | 1874 ++++++
 libcxx/include/__cxx03/__expected/unexpect.h  |   32 +
 .../include/__cxx03/__expected/unexpected.h   |  127 +
 .../__cxx03/__filesystem/copy_options.h       |   69 +
 .../__cxx03/__filesystem/directory_entry.h    |  435 ++
 .../__cxx03/__filesystem/directory_iterator.h |  151 +
 .../__cxx03/__filesystem/directory_options.h  |   57 +
 .../__cxx03/__filesystem/file_status.h        |   67 +
 .../__cxx03/__filesystem/file_time_type.h     |   31 +
 .../include/__cxx03/__filesystem/file_type.h  |   42 +
 .../__cxx03/__filesystem/filesystem_error.h   |   88 +
 .../include/__cxx03/__filesystem/operations.h |  310 +
 libcxx/include/__cxx03/__filesystem/path.h    |  931 +++
 .../__cxx03/__filesystem/path_iterator.h      |  115 +
 .../__cxx03/__filesystem/perm_options.h       |   57 +
 libcxx/include/__cxx03/__filesystem/perms.h   |   80 +
 .../recursive_directory_iterator.h            |  164 +
 .../include/__cxx03/__filesystem/space_info.h |   38 +
 libcxx/include/__cxx03/__filesystem/u8path.h  |  100 +
 libcxx/include/__cxx03/__format/buffer.h      |  655 ++
 libcxx/include/__cxx03/__format/concepts.h    |   83 +
 .../__cxx03/__format/container_adaptor.h      |   73 +
 .../__cxx03/__format/enable_insertable.h      |   35 +
 .../__cxx03/__format/escaped_output_table.h   |  863 +++
 .../extended_grapheme_cluster_table.h         | 1663 +++++
 libcxx/include/__cxx03/__format/format_arg.h  |  401 ++
 .../__cxx03/__format/format_arg_store.h       |  266 +
 libcxx/include/__cxx03/__format/format_args.h |   78 +
 .../include/__cxx03/__format/format_context.h |  220 +
 .../include/__cxx03/__format/format_error.h   |   50 +
 .../__cxx03/__format/format_functions.h       |  680 ++
 .../__cxx03/__format/format_parse_context.h   |  105 +
 .../include/__cxx03/__format/format_string.h  |  160 +
 .../__cxx03/__format/format_to_n_result.h     |   35 +
 libcxx/include/__cxx03/__format/formatter.h   |   53 +
 .../include/__cxx03/__format/formatter_bool.h |   76 +
 .../include/__cxx03/__format/formatter_char.h |   93 +
 .../__format/formatter_floating_point.h       |  783 +++
 .../__cxx03/__format/formatter_integer.h      |   95 +
 .../__cxx03/__format/formatter_integral.h     |  445 ++
 .../__cxx03/__format/formatter_output.h       |  335 +
 .../__cxx03/__format/formatter_pointer.h      |   72 +
 .../__cxx03/__format/formatter_string.h       |  150 +
 .../__cxx03/__format/formatter_tuple.h        |  150 +
 .../__format/indic_conjunct_break_table.h     |  350 +
 .../__cxx03/__format/parser_std_format_spec.h | 1172 ++++
 .../__format/range_default_formatter.h        |  214 +
 .../__cxx03/__format/range_formatter.h        |  264 +
 libcxx/include/__cxx03/__format/unicode.h     |  602 ++
 .../__cxx03/__format/width_estimation_table.h |  270 +
 .../include/__cxx03/__format/write_escaped.h  |  242 +
 .../__cxx03/__functional/binary_function.h    |   54 +
 .../__cxx03/__functional/binary_negate.h      |   51 +
 libcxx/include/__cxx03/__functional/bind.h    |  296 +
 .../include/__cxx03/__functional/bind_back.h  |   83 +
 .../include/__cxx03/__functional/bind_front.h |   54 +
 .../include/__cxx03/__functional/binder1st.h  |   54 +
 .../include/__cxx03/__functional/binder2nd.h  |   54 +
 .../__functional/boyer_moore_searcher.h       |  306 +
 libcxx/include/__cxx03/__functional/compose.h |   53 +
 .../__cxx03/__functional/default_searcher.h   |   54 +
 .../include/__cxx03/__functional/function.h   | 1048 +++
 libcxx/include/__cxx03/__functional/hash.h    |  542 ++
 .../include/__cxx03/__functional/identity.h   |   65 +
 libcxx/include/__cxx03/__functional/invoke.h  |   54 +
 .../__cxx03/__functional/is_transparent.h     |   34 +
 libcxx/include/__cxx03/__functional/mem_fn.h  |   54 +
 .../__cxx03/__functional/mem_fun_ref.h        |  146 +
 libcxx/include/__cxx03/__functional/not_fn.h  |   53 +
 .../include/__cxx03/__functional/operations.h |  541 ++
 .../__cxx03/__functional/perfect_forward.h    |  104 +
 .../__functional/pointer_to_binary_function.h |   44 +
 .../__functional/pointer_to_unary_function.h  |   44 +
 .../__cxx03/__functional/ranges_operations.h  |  109 +
 .../__cxx03/__functional/reference_wrapper.h  |  154 +
 .../__cxx03/__functional/unary_function.h     |   51 +
 .../__cxx03/__functional/unary_negate.h       |   48 +
 .../__cxx03/__functional/weak_result_type.h   |  231 +
 libcxx/include/__cxx03/__fwd/array.h          |   46 +
 libcxx/include/__cxx03/__fwd/bit_reference.h  |   25 +
 libcxx/include/__cxx03/__fwd/complex.h        |   42 +
 libcxx/include/__cxx03/__fwd/deque.h          |   26 +
 libcxx/include/__cxx03/__fwd/format.h         |   38 +
 libcxx/include/__cxx03/__fwd/fstream.h        |   53 +
 libcxx/include/__cxx03/__fwd/functional.h     |   28 +
 libcxx/include/__cxx03/__fwd/ios.h            |   43 +
 libcxx/include/__cxx03/__fwd/istream.h        |   43 +
 libcxx/include/__cxx03/__fwd/mdspan.h         |   57 +
 libcxx/include/__cxx03/__fwd/memory.h         |   25 +
 .../include/__cxx03/__fwd/memory_resource.h   |   27 +
 libcxx/include/__cxx03/__fwd/ostream.h        |   35 +
 libcxx/include/__cxx03/__fwd/pair.h           |   45 +
 libcxx/include/__cxx03/__fwd/queue.h          |   31 +
 libcxx/include/__cxx03/__fwd/span.h           |   38 +
 libcxx/include/__cxx03/__fwd/sstream.h        |   58 +
 libcxx/include/__cxx03/__fwd/stack.h          |   26 +
 libcxx/include/__cxx03/__fwd/streambuf.h      |   35 +
 libcxx/include/__cxx03/__fwd/string.h         |  107 +
 libcxx/include/__cxx03/__fwd/string_view.h    |   50 +
 libcxx/include/__cxx03/__fwd/subrange.h       |   49 +
 libcxx/include/__cxx03/__fwd/tuple.h          |   52 +
 libcxx/include/__cxx03/__fwd/vector.h         |   26 +
 libcxx/include/__cxx03/__hash_table           | 2044 ++++++
 libcxx/include/__cxx03/__ios/fpos.h           |   76 +
 libcxx/include/__cxx03/__iterator/access.h    |   95 +
 libcxx/include/__cxx03/__iterator/advance.h   |  205 +
 .../__cxx03/__iterator/aliasing_iterator.h    |  127 +
 .../__cxx03/__iterator/back_insert_iterator.h |   85 +
 .../include/__cxx03/__iterator/bounded_iter.h |  283 +
 .../__cxx03/__iterator/common_iterator.h      |  299 +
 libcxx/include/__cxx03/__iterator/concepts.h  |  257 +
 .../__cxx03/__iterator/counted_iterator.h     |  289 +
 .../__iterator/cpp17_iterator_concepts.h      |  190 +
 libcxx/include/__cxx03/__iterator/data.h      |   49 +
 .../__cxx03/__iterator/default_sentinel.h     |   30 +
 libcxx/include/__cxx03/__iterator/distance.h  |   99 +
 libcxx/include/__cxx03/__iterator/empty.h     |   45 +
 .../__cxx03/__iterator/erase_if_container.h   |   43 +
 .../__iterator/front_insert_iterator.h        |   83 +
 .../__cxx03/__iterator/incrementable_traits.h |   79 +
 .../__iterator/indirectly_comparable.h        |   33 +
 .../__cxx03/__iterator/insert_iterator.h      |   95 +
 .../__cxx03/__iterator/istream_iterator.h     |  101 +
 .../__cxx03/__iterator/istreambuf_iterator.h  |  109 +
 libcxx/include/__cxx03/__iterator/iter_move.h |  103 +
 libcxx/include/__cxx03/__iterator/iter_swap.h |  108 +
 libcxx/include/__cxx03/__iterator/iterator.h  |   33 +
 .../__cxx03/__iterator/iterator_traits.h      |  528 ++
 .../__cxx03/__iterator/iterator_with_data.h   |  105 +
 libcxx/include/__cxx03/__iterator/mergeable.h |   42 +
 .../__cxx03/__iterator/move_iterator.h        |  347 +
 .../__cxx03/__iterator/move_sentinel.h        |   61 +
 libcxx/include/__cxx03/__iterator/next.h      |   83 +
 .../__cxx03/__iterator/ostream_iterator.h     |   75 +
 .../__cxx03/__iterator/ostreambuf_iterator.h  |   72 +
 .../include/__cxx03/__iterator/permutable.h   |   34 +
 libcxx/include/__cxx03/__iterator/prev.h      |   76 +
 libcxx/include/__cxx03/__iterator/projected.h |   53 +
 .../__iterator/ranges_iterator_traits.h       |   40 +
 .../__cxx03/__iterator/readable_traits.h      |   81 +
 .../__cxx03/__iterator/reverse_access.h       |   80 +
 .../__cxx03/__iterator/reverse_iterator.h     |  346 +
 .../__cxx03/__iterator/segmented_iterator.h   |   79 +
 libcxx/include/__cxx03/__iterator/size.h      |   59 +
 libcxx/include/__cxx03/__iterator/sortable.h  |   35 +
 .../__cxx03/__iterator/unreachable_sentinel.h |   37 +
 libcxx/include/__cxx03/__iterator/wrap_iter.h |  244 +
 libcxx/include/__cxx03/__locale               | 1513 +++++
 .../__cxx03/__locale_dir/locale_base_api.h    |   98 +
 .../__locale_dir/locale_base_api/android.h    |   50 +
 .../locale_base_api/bsd_locale_defaults.h     |   36 +
 .../locale_base_api/bsd_locale_fallbacks.h    |  126 +
 .../__locale_dir/locale_base_api/fuchsia.h    |   18 +
 .../__locale_dir/locale_base_api/ibm.h        |  108 +
 .../locale_base_api/locale_guard.h            |   78 +
 .../__locale_dir/locale_base_api/musl.h       |   31 +
 .../__locale_dir/locale_base_api/newlib.h     |   12 +
 .../__locale_dir/locale_base_api/openbsd.h    |   19 +
 .../__locale_dir/locale_base_api/win32.h      |  235 +
 libcxx/include/__cxx03/__math/abs.h           |   46 +
 libcxx/include/__cxx03/__math/copysign.h      |   45 +
 .../include/__cxx03/__math/error_functions.h  |   60 +
 .../__cxx03/__math/exponential_functions.h    |  171 +
 libcxx/include/__cxx03/__math/fdim.h          |   48 +
 libcxx/include/__cxx03/__math/fma.h           |   55 +
 libcxx/include/__cxx03/__math/gamma.h         |   62 +
 .../__cxx03/__math/hyperbolic_functions.h     |   76 +
 libcxx/include/__cxx03/__math/hypot.h         |  109 +
 .../__math/inverse_hyperbolic_functions.h     |   76 +
 .../__math/inverse_trigonometric_functions.h  |   99 +
 libcxx/include/__cxx03/__math/logarithms.h    |  124 +
 libcxx/include/__cxx03/__math/min_max.h       |   74 +
 libcxx/include/__cxx03/__math/modulo.h        |   63 +
 libcxx/include/__cxx03/__math/remainder.h     |   73 +
 libcxx/include/__cxx03/__math/roots.h         |   62 +
 .../__cxx03/__math/rounding_functions.h       |  245 +
 .../__cxx03/__math/special_functions.h        |   84 +
 libcxx/include/__cxx03/__math/traits.h        |  188 +
 .../__cxx03/__math/trigonometric_functions.h  |   76 +
 libcxx/include/__cxx03/__mbstate_t.h          |   54 +
 .../__cxx03/__mdspan/default_accessor.h       |   66 +
 libcxx/include/__cxx03/__mdspan/extents.h     |  532 ++
 libcxx/include/__cxx03/__mdspan/layout_left.h |  204 +
 .../include/__cxx03/__mdspan/layout_right.h   |  201 +
 .../include/__cxx03/__mdspan/layout_stride.h  |  366 ++
 libcxx/include/__cxx03/__mdspan/mdspan.h      |  319 +
 libcxx/include/__cxx03/__memory/addressof.h   |   61 +
 libcxx/include/__cxx03/__memory/align.h       |   25 +
 .../include/__cxx03/__memory/aligned_alloc.h  |   64 +
 .../__cxx03/__memory/allocate_at_least.h      |   48 +
 .../__cxx03/__memory/allocation_guard.h       |  108 +
 libcxx/include/__cxx03/__memory/allocator.h   |  268 +
 .../__cxx03/__memory/allocator_arg_t.h        |   75 +
 .../__cxx03/__memory/allocator_destructor.h   |   40 +
 .../__cxx03/__memory/allocator_traits.h       |  424 ++
 .../include/__cxx03/__memory/assume_aligned.h |   50 +
 libcxx/include/__cxx03/__memory/auto_ptr.h    |   92 +
 .../__cxx03/__memory/builtin_new_allocator.h  |   67 +
 .../__cxx03/__memory/compressed_pair.h        |  171 +
 libcxx/include/__cxx03/__memory/concepts.h    |   63 +
 .../include/__cxx03/__memory/construct_at.h   |  128 +
 libcxx/include/__cxx03/__memory/destruct_n.h  |   63 +
 libcxx/include/__cxx03/__memory/inout_ptr.h   |  109 +
 libcxx/include/__cxx03/__memory/out_ptr.h     |  101 +
 .../include/__cxx03/__memory/pointer_traits.h |  307 +
 .../__cxx03/__memory/ranges_construct_at.h    |  124 +
 .../ranges_uninitialized_algorithms.h         |  325 +
 .../__cxx03/__memory/raw_storage_iterator.h   |   87 +
 libcxx/include/__cxx03/__memory/shared_ptr.h  | 1694 +++++
 .../include/__cxx03/__memory/swap_allocator.h |   54 +
 libcxx/include/__cxx03/__memory/temp_value.h  |   58 +
 .../__cxx03/__memory/temporary_buffer.h       |   75 +
 .../__memory/uninitialized_algorithms.h       |  653 ++
 libcxx/include/__cxx03/__memory/unique_ptr.h  |  693 ++
 .../include/__cxx03/__memory/uses_allocator.h |   52 +
 .../__memory/uses_allocator_construction.h    |  247 +
 libcxx/include/__cxx03/__memory/voidify.h     |   30 +
 .../__memory_resource/memory_resource.h       |   88 +
 .../monotonic_buffer_resource.h               |  122 +
 .../__memory_resource/polymorphic_allocator.h |  231 +
 .../__cxx03/__memory_resource/pool_options.h  |   38 +
 .../synchronized_pool_resource.h              |   92 +
 .../unsynchronized_pool_resource.h            |  106 +
 libcxx/include/__cxx03/__mutex/lock_guard.h   |   48 +
 libcxx/include/__cxx03/__mutex/mutex.h        |   53 +
 libcxx/include/__cxx03/__mutex/once_flag.h    |  159 +
 libcxx/include/__cxx03/__mutex/tag_types.h    |   44 +
 libcxx/include/__cxx03/__mutex/unique_lock.h  |  177 +
 libcxx/include/__cxx03/__node_handle          |  209 +
 libcxx/include/__cxx03/__numeric/accumulate.h |   53 +
 .../__cxx03/__numeric/adjacent_difference.h   |   68 +
 .../__cxx03/__numeric/exclusive_scan.h        |   58 +
 libcxx/include/__cxx03/__numeric/gcd_lcm.h    |  132 +
 .../__cxx03/__numeric/inclusive_scan.h        |   59 +
 .../include/__cxx03/__numeric/inner_product.h |   58 +
 libcxx/include/__cxx03/__numeric/iota.h       |   30 +
 libcxx/include/__cxx03/__numeric/midpoint.h   |   88 +
 .../include/__cxx03/__numeric/partial_sum.h   |   66 +
 libcxx/include/__cxx03/__numeric/pstl.h       |  174 +
 libcxx/include/__cxx03/__numeric/reduce.h     |   53 +
 .../__cxx03/__numeric/saturation_arithmetic.h |  145 +
 .../__numeric/transform_exclusive_scan.h      |   42 +
 .../__numeric/transform_inclusive_scan.h      |   52 +
 .../__cxx03/__numeric/transform_reduce.h      |   59 +
 .../include/__cxx03/__ostream/basic_ostream.h |  860 +++
 libcxx/include/__cxx03/__ostream/print.h      |  179 +
 libcxx/include/__cxx03/__pstl/backend.h       |   35 +
 libcxx/include/__cxx03/__pstl/backend_fwd.h   |  301 +
 .../include/__cxx03/__pstl/backends/default.h |  503 ++
 .../__cxx03/__pstl/backends/libdispatch.h     |  397 ++
 .../include/__cxx03/__pstl/backends/serial.h  |  181 +
 .../__cxx03/__pstl/backends/std_thread.h      |  136 +
 .../include/__cxx03/__pstl/cpu_algos/any_of.h |   99 +
 .../__cxx03/__pstl/cpu_algos/cpu_traits.h     |   86 +
 .../include/__cxx03/__pstl/cpu_algos/fill.h   |   66 +
 .../__cxx03/__pstl/cpu_algos/find_if.h        |  137 +
 .../__cxx03/__pstl/cpu_algos/for_each.h       |   66 +
 .../include/__cxx03/__pstl/cpu_algos/merge.h  |   85 +
 .../__cxx03/__pstl/cpu_algos/stable_sort.h    |   47 +
 .../__cxx03/__pstl/cpu_algos/transform.h      |  153 +
 .../__pstl/cpu_algos/transform_reduce.h       |  216 +
 libcxx/include/__cxx03/__pstl/dispatch.h      |   66 +
 .../include/__cxx03/__pstl/handle_exception.h |   57 +
 .../__cxx03/__random/bernoulli_distribution.h |  123 +
 .../__cxx03/__random/binomial_distribution.h  |  198 +
 .../__cxx03/__random/cauchy_distribution.h    |  139 +
 .../__random/chi_squared_distribution.h       |  125 +
 .../__cxx03/__random/clamp_to_integral.h      |   59 +
 .../__cxx03/__random/default_random_engine.h  |   25 +
 .../__cxx03/__random/discard_block_engine.h   |  172 +
 .../__cxx03/__random/discrete_distribution.h  |  212 +
 .../__random/exponential_distribution.h       |  132 +
 .../__random/extreme_value_distribution.h     |  139 +
 .../__cxx03/__random/fisher_f_distribution.h  |  138 +
 .../__cxx03/__random/gamma_distribution.h     |  181 +
 .../__cxx03/__random/generate_canonical.h     |   51 +
 .../__cxx03/__random/geometric_distribution.h |  120 +
 .../__random/independent_bits_engine.h        |  205 +
 .../__cxx03/__random/is_seed_sequence.h       |   31 +
 libcxx/include/__cxx03/__random/is_valid.h    |   94 +
 libcxx/include/__cxx03/__random/knuth_b.h     |   26 +
 .../__random/linear_congruential_engine.h     |  387 ++
 libcxx/include/__cxx03/__random/log2.h        |   66 +
 .../__cxx03/__random/lognormal_distribution.h |  126 +
 .../__random/mersenne_twister_engine.h        |  914 +++
 .../__random/negative_binomial_distribution.h |  157 +
 .../__cxx03/__random/normal_distribution.h    |  176 +
 .../piecewise_constant_distribution.h         |  302 +
 .../__random/piecewise_linear_distribution.h  |  315 +
 .../__cxx03/__random/poisson_distribution.h   |  241 +
 .../include/__cxx03/__random/random_device.h  |   81 +
 libcxx/include/__cxx03/__random/ranlux.h      |   31 +
 libcxx/include/__cxx03/__random/seed_seq.h    |  166 +
 .../__cxx03/__random/shuffle_order_engine.h   |  230 +
 .../__cxx03/__random/student_t_distribution.h |  132 +
 .../__random/subtract_with_carry_engine.h     |  276 +
 .../__random/uniform_int_distribution.h       |  264 +
 .../__random/uniform_random_bit_generator.h   |   44 +
 .../__random/uniform_real_distribution.h      |  138 +
 .../__cxx03/__random/weibull_distribution.h   |  132 +
 libcxx/include/__cxx03/__ranges/access.h      |  210 +
 libcxx/include/__cxx03/__ranges/all.h         |   78 +
 .../include/__cxx03/__ranges/as_rvalue_view.h |  142 +
 .../include/__cxx03/__ranges/chunk_by_view.h  |  235 +
 libcxx/include/__cxx03/__ranges/common_view.h |  142 +
 libcxx/include/__cxx03/__ranges/concepts.h    |  142 +
 .../__ranges/container_compatible_range.h     |   33 +
 libcxx/include/__cxx03/__ranges/counted.h     |   89 +
 libcxx/include/__cxx03/__ranges/dangling.h    |   42 +
 libcxx/include/__cxx03/__ranges/data.h        |  102 +
 libcxx/include/__cxx03/__ranges/drop_view.h   |  329 +
 .../__cxx03/__ranges/drop_while_view.h        |  136 +
 .../include/__cxx03/__ranges/elements_view.h  |  418 ++
 libcxx/include/__cxx03/__ranges/empty.h       |   71 +
 libcxx/include/__cxx03/__ranges/empty_view.h  |   54 +
 .../__cxx03/__ranges/enable_borrowed_range.h  |   40 +
 libcxx/include/__cxx03/__ranges/enable_view.h |   51 +
 libcxx/include/__cxx03/__ranges/filter_view.h |  260 +
 libcxx/include/__cxx03/__ranges/from_range.h  |   33 +
 libcxx/include/__cxx03/__ranges/iota_view.h   |  404 ++
 .../include/__cxx03/__ranges/istream_view.h   |  141 +
 libcxx/include/__cxx03/__ranges/join_view.h   |  423 ++
 .../__cxx03/__ranges/lazy_split_view.h        |  441 ++
 libcxx/include/__cxx03/__ranges/movable_box.h |  247 +
 .../__cxx03/__ranges/non_propagating_cache.h  |  103 +
 libcxx/include/__cxx03/__ranges/owning_view.h |  116 +
 .../include/__cxx03/__ranges/range_adaptor.h  |  101 +
 libcxx/include/__cxx03/__ranges/rbegin.h      |  120 +
 libcxx/include/__cxx03/__ranges/ref_view.h    |   89 +
 libcxx/include/__cxx03/__ranges/rend.h        |  123 +
 libcxx/include/__cxx03/__ranges/repeat_view.h |  266 +
 .../include/__cxx03/__ranges/reverse_view.h   |  203 +
 libcxx/include/__cxx03/__ranges/single_view.h |  111 +
 libcxx/include/__cxx03/__ranges/size.h        |  137 +
 libcxx/include/__cxx03/__ranges/split_view.h  |  232 +
 libcxx/include/__cxx03/__ranges/subrange.h    |  273 +
 libcxx/include/__cxx03/__ranges/take_view.h   |  369 ++
 .../__cxx03/__ranges/take_while_view.h        |  170 +
 libcxx/include/__cxx03/__ranges/to.h          |  245 +
 .../include/__cxx03/__ranges/transform_view.h |  417 ++
 .../include/__cxx03/__ranges/view_interface.h |  170 +
 libcxx/include/__cxx03/__ranges/views.h       |   35 +
 libcxx/include/__cxx03/__ranges/zip_view.h    |  516 ++
 libcxx/include/__cxx03/__split_buffer         |  585 ++
 libcxx/include/__cxx03/__std_clang_module     |  215 +
 libcxx/include/__cxx03/__std_mbstate_t.h      |   29 +
 .../__cxx03/__stop_token/atomic_unique_lock.h |  140 +
 .../__stop_token/intrusive_list_view.h        |   85 +
 .../__stop_token/intrusive_shared_ptr.h       |  134 +
 .../__cxx03/__stop_token/stop_callback.h      |  102 +
 .../__cxx03/__stop_token/stop_source.h        |   91 +
 .../include/__cxx03/__stop_token/stop_state.h |  236 +
 .../include/__cxx03/__stop_token/stop_token.h |   63 +
 libcxx/include/__cxx03/__string/char_traits.h |  543 ++
 .../__cxx03/__string/constexpr_c_functions.h  |  234 +
 .../__cxx03/__string/extern_template_lists.h  |  133 +
 .../__cxx03/__support/ibm/gettod_zos.h        |   52 +
 .../__cxx03/__support/ibm/locale_mgmt_zos.h   |   53 +
 .../include/__cxx03/__support/ibm/nanosleep.h |   55 +
 .../__support/xlocale/__nop_locale_mgmt.h     |   35 +
 .../__support/xlocale/__posix_l_fallback.h    |  107 +
 .../__support/xlocale/__strtonum_fallback.h   |   45 +
 libcxx/include/__cxx03/__system_error/errc.h  |  266 +
 .../__cxx03/__system_error/error_category.h   |   75 +
 .../__cxx03/__system_error/error_code.h       |  143 +
 .../__cxx03/__system_error/error_condition.h  |  130 +
 .../__cxx03/__system_error/system_error.h     |   54 +
 libcxx/include/__cxx03/__thread/formatter.h   |   80 +
 libcxx/include/__cxx03/__thread/id.h          |  121 +
 libcxx/include/__cxx03/__thread/jthread.h     |  134 +
 .../__cxx03/__thread/poll_with_backoff.h      |   67 +
 libcxx/include/__cxx03/__thread/support.h     |  123 +
 libcxx/include/__cxx03/__thread/support/c11.h |  191 +
 .../__cxx03/__thread/support/external.h       |   21 +
 .../__cxx03/__thread/support/pthread.h        |  221 +
 .../__cxx03/__thread/support/windows.h        |  133 +
 libcxx/include/__cxx03/__thread/this_thread.h |   74 +
 libcxx/include/__cxx03/__thread/thread.h      |  258 +
 .../__cxx03/__thread/timed_backoff_policy.h   |   44 +
 libcxx/include/__cxx03/__tree                 | 2299 +++++++
 libcxx/include/__cxx03/__tuple/find_index.h   |   62 +
 libcxx/include/__cxx03/__tuple/ignore.h       |   39 +
 .../__cxx03/__tuple/make_tuple_types.h        |   80 +
 .../include/__cxx03/__tuple/sfinae_helpers.h  |  142 +
 .../include/__cxx03/__tuple/tuple_element.h   |   58 +
 .../include/__cxx03/__tuple/tuple_indices.h   |   37 +
 libcxx/include/__cxx03/__tuple/tuple_like.h   |   42 +
 .../include/__cxx03/__tuple/tuple_like_ext.h  |   52 +
 .../__cxx03/__tuple/tuple_like_no_subrange.h  |   61 +
 libcxx/include/__cxx03/__tuple/tuple_size.h   |   75 +
 libcxx/include/__cxx03/__tuple/tuple_types.h  |   25 +
 .../include/__cxx03/__type_traits/add_const.h |   32 +
 libcxx/include/__cxx03/__type_traits/add_cv.h |   32 +
 .../__type_traits/add_lvalue_reference.h      |   54 +
 .../__cxx03/__type_traits/add_pointer.h       |   55 +
 .../__type_traits/add_rvalue_reference.h      |   54 +
 .../__cxx03/__type_traits/add_volatile.h      |   32 +
 .../__cxx03/__type_traits/aligned_storage.h   |  138 +
 .../__cxx03/__type_traits/aligned_union.h     |   51 +
 .../__cxx03/__type_traits/alignment_of.h      |   32 +
 .../__cxx03/__type_traits/can_extract_key.h   |   53 +
 .../__cxx03/__type_traits/common_reference.h  |  193 +
 .../__cxx03/__type_traits/common_type.h       |  102 +
 .../__cxx03/__type_traits/conditional.h       |   58 +
 .../__cxx03/__type_traits/conjunction.h       |   65 +
 .../include/__cxx03/__type_traits/copy_cv.h   |   51 +
 .../__cxx03/__type_traits/copy_cvref.h        |   43 +
 .../__cxx03/__type_traits/datasizeof.h        |   65 +
 libcxx/include/__cxx03/__type_traits/decay.h  |   73 +
 .../__cxx03/__type_traits/dependent_type.h    |   25 +
 .../__cxx03/__type_traits/desugars_to.h       |   40 +
 .../__cxx03/__type_traits/disjunction.h       |   58 +
 .../include/__cxx03/__type_traits/enable_if.h |   37 +
 libcxx/include/__cxx03/__type_traits/extent.h |   54 +
 .../has_unique_object_representation.h        |   40 +
 .../__type_traits/has_virtual_destructor.h    |   31 +
 .../__cxx03/__type_traits/integral_constant.h |   47 +
 libcxx/include/__cxx03/__type_traits/invoke.h |  270 +
 .../__cxx03/__type_traits/is_abstract.h       |   31 +
 .../__cxx03/__type_traits/is_aggregate.h      |   33 +
 .../__cxx03/__type_traits/is_allocator.h      |   34 +
 .../__type_traits/is_always_bitcastable.h     |   85 +
 .../__cxx03/__type_traits/is_arithmetic.h     |   34 +
 .../include/__cxx03/__type_traits/is_array.h  |   51 +
 .../__cxx03/__type_traits/is_assignable.h     |   52 +
 .../__cxx03/__type_traits/is_base_of.h        |   31 +
 .../__cxx03/__type_traits/is_bounded_array.h  |   41 +
 .../__cxx03/__type_traits/is_callable.h       |   32 +
 .../__cxx03/__type_traits/is_char_like_type.h |   28 +
 .../include/__cxx03/__type_traits/is_class.h  |   31 +
 .../__cxx03/__type_traits/is_compound.h       |   46 +
 .../include/__cxx03/__type_traits/is_const.h  |   47 +
 .../__type_traits/is_constant_evaluated.h     |   32 +
 .../__cxx03/__type_traits/is_constructible.h  |   59 +
 .../__cxx03/__type_traits/is_convertible.h    |   31 +
 .../__type_traits/is_core_convertible.h       |   35 +
 .../__cxx03/__type_traits/is_destructible.h   |   97 +
 .../include/__cxx03/__type_traits/is_empty.h  |   31 +
 .../include/__cxx03/__type_traits/is_enum.h   |   41 +
 .../__type_traits/is_equality_comparable.h    |   88 +
 .../__type_traits/is_execution_policy.h       |   59 +
 .../include/__cxx03/__type_traits/is_final.h  |   36 +
 .../__cxx03/__type_traits/is_floating_point.h |   39 +
 .../__cxx03/__type_traits/is_function.h       |   31 +
 .../__cxx03/__type_traits/is_fundamental.h    |   48 +
 .../is_implicitly_default_constructible.h     |   46 +
 .../__cxx03/__type_traits/is_integral.h       |   74 +
 .../__cxx03/__type_traits/is_literal_type.h   |   34 +
 .../__cxx03/__type_traits/is_member_pointer.h |   43 +
 .../__type_traits/is_nothrow_assignable.h     |   56 +
 .../__type_traits/is_nothrow_constructible.h  |   61 +
 .../__type_traits/is_nothrow_convertible.h    |   62 +
 .../__type_traits/is_nothrow_destructible.h   |   63 +
 .../__cxx03/__type_traits/is_null_pointer.h   |   37 +
 .../include/__cxx03/__type_traits/is_object.h |   31 +
 libcxx/include/__cxx03/__type_traits/is_pod.h |   31 +
 .../__cxx03/__type_traits/is_pointer.h        |   65 +
 .../__cxx03/__type_traits/is_polymorphic.h    |   31 +
 .../__type_traits/is_primary_template.h       |   31 +
 .../__cxx03/__type_traits/is_reference.h      |   68 +
 .../__type_traits/is_reference_wrapper.h      |   32 +
 .../__cxx03/__type_traits/is_referenceable.h  |   41 +
 .../include/__cxx03/__type_traits/is_same.h   |   44 +
 .../include/__cxx03/__type_traits/is_scalar.h |   69 +
 .../include/__cxx03/__type_traits/is_signed.h |   59 +
 .../__cxx03/__type_traits/is_signed_integer.h |   35 +
 .../__cxx03/__type_traits/is_specialization.h |   45 +
 .../__type_traits/is_standard_layout.h        |   31 +
 .../__cxx03/__type_traits/is_swappable.h      |  104 +
 .../__cxx03/__type_traits/is_trivial.h        |   31 +
 .../__type_traits/is_trivially_assignable.h   |   56 +
 .../is_trivially_constructible.h              |   61 +
 .../__type_traits/is_trivially_copyable.h     |   37 +
 .../__type_traits/is_trivially_destructible.h |   47 +
 ...s_trivially_lexicographically_comparable.h |   53 +
 .../__type_traits/is_trivially_relocatable.h  |   42 +
 .../__type_traits/is_unbounded_array.h        |   40 +
 .../include/__cxx03/__type_traits/is_union.h  |   31 +
 .../__cxx03/__type_traits/is_unsigned.h       |   59 +
 .../__type_traits/is_unsigned_integer.h       |   35 +
 .../__type_traits/is_valid_expansion.h        |   31 +
 .../include/__cxx03/__type_traits/is_void.h   |   31 +
 .../__cxx03/__type_traits/is_volatile.h       |   47 +
 libcxx/include/__cxx03/__type_traits/lazy.h   |   25 +
 .../__type_traits/make_32_64_or_128_bit.h     |   49 +
 .../__type_traits/make_const_lvalue_ref.h     |   26 +
 .../__cxx03/__type_traits/make_signed.h       |   89 +
 .../__cxx03/__type_traits/make_unsigned.h     |  101 +
 .../__cxx03/__type_traits/maybe_const.h       |   26 +
 libcxx/include/__cxx03/__type_traits/nat.h    |   31 +
 .../include/__cxx03/__type_traits/negation.h  |   33 +
 .../noexcept_move_assign_container.h          |   37 +
 .../include/__cxx03/__type_traits/promote.h   |  128 +
 libcxx/include/__cxx03/__type_traits/rank.h   |   46 +
 .../__type_traits/remove_all_extents.h        |   54 +
 .../__cxx03/__type_traits/remove_const.h      |   49 +
 .../__cxx03/__type_traits/remove_const_ref.h  |   27 +
 .../include/__cxx03/__type_traits/remove_cv.h |   47 +
 .../__cxx03/__type_traits/remove_cvref.h      |   46 +
 .../__cxx03/__type_traits/remove_extent.h     |   54 +
 .../__cxx03/__type_traits/remove_pointer.h    |   53 +
 .../__cxx03/__type_traits/remove_reference.h  |   47 +
 .../__cxx03/__type_traits/remove_volatile.h   |   49 +
 .../include/__cxx03/__type_traits/result_of.h |   38 +
 .../__cxx03/__type_traits/strip_signature.h   |   81 +
 .../__cxx03/__type_traits/type_identity.h     |   39 +
 .../include/__cxx03/__type_traits/type_list.h |   42 +
 .../__cxx03/__type_traits/underlying_type.h   |   42 +
 .../__cxx03/__type_traits/unwrap_ref.h        |   58 +
 libcxx/include/__cxx03/__type_traits/void_t.h |   30 +
 libcxx/include/__cxx03/__undef_macros         |   28 +
 libcxx/include/__cxx03/__utility/as_const.h   |   35 +
 libcxx/include/__cxx03/__utility/as_lvalue.h  |   37 +
 libcxx/include/__cxx03/__utility/auto_cast.h  |   22 +
 libcxx/include/__cxx03/__utility/cmp.h        |   81 +
 .../__cxx03/__utility/convert_to_integral.h   |   68 +
 libcxx/include/__cxx03/__utility/declval.h    |   38 +
 libcxx/include/__cxx03/__utility/empty.h      |   24 +
 .../__cxx03/__utility/exception_guard.h       |  144 +
 libcxx/include/__cxx03/__utility/exchange.h   |   41 +
 libcxx/include/__cxx03/__utility/forward.h    |   38 +
 .../include/__cxx03/__utility/forward_like.h  |   46 +
 libcxx/include/__cxx03/__utility/in_place.h   |   63 +
 .../__cxx03/__utility/integer_sequence.h      |   90 +
 .../__cxx03/__utility/is_pointer_in_range.h   |   62 +
 .../__cxx03/__utility/is_valid_range.h        |   37 +
 libcxx/include/__cxx03/__utility/move.h       |   49 +
 libcxx/include/__cxx03/__utility/no_destroy.h |   54 +
 libcxx/include/__cxx03/__utility/pair.h       |  718 ++
 .../__cxx03/__utility/piecewise_construct.h   |   32 +
 .../include/__cxx03/__utility/priority_tag.h  |   28 +
 .../__utility/private_constructor_tag.h       |   28 +
 libcxx/include/__cxx03/__utility/rel_ops.h    |   46 +
 .../include/__cxx03/__utility/small_buffer.h  |   99 +
 libcxx/include/__cxx03/__utility/swap.h       |   59 +
 .../include/__cxx03/__utility/to_underlying.h |   38 +
 .../include/__cxx03/__utility/unreachable.h   |   34 +
 libcxx/include/__cxx03/__variant/monostate.h  |   64 +
 libcxx/include/__cxx03/__verbose_abort        |   52 +
 libcxx/include/__cxx03/algorithm              | 2049 ++++++
 libcxx/include/__cxx03/any                    |  614 ++
 libcxx/include/__cxx03/array                  |  514 ++
 libcxx/include/__cxx03/atomic                 |  632 ++
 libcxx/include/__cxx03/barrier                |  308 +
 libcxx/include/__cxx03/bit                    |  101 +
 libcxx/include/__cxx03/bitset                 |  969 +++
 libcxx/include/__cxx03/cassert                |   31 +
 libcxx/include/__cxx03/ccomplex               |   26 +
 libcxx/include/__cxx03/cctype                 |  127 +
 libcxx/include/__cxx03/cerrno                 |   48 +
 libcxx/include/__cxx03/cfenv                  |   90 +
 libcxx/include/__cxx03/cfloat                 |   88 +
 libcxx/include/__cxx03/charconv               |  115 +
 libcxx/include/__cxx03/chrono                 | 1022 +++
 libcxx/include/__cxx03/cinttypes              |  270 +
 libcxx/include/__cxx03/ciso646                |   24 +
 libcxx/include/__cxx03/climits                |   48 +
 libcxx/include/__cxx03/clocale                |   61 +
 libcxx/include/__cxx03/cmath                  |  646 ++
 libcxx/include/__cxx03/codecvt                |  597 ++
 libcxx/include/__cxx03/compare                |  178 +
 libcxx/include/__cxx03/complex                | 1480 +++++
 libcxx/include/__cxx03/complex.h              |   32 +
 libcxx/include/__cxx03/concepts               |  173 +
 libcxx/include/__cxx03/condition_variable     |  369 ++
 libcxx/include/__cxx03/coroutine              |   67 +
 libcxx/include/__cxx03/csetjmp                |   54 +
 libcxx/include/__cxx03/csignal                |   64 +
 libcxx/include/__cxx03/cstdarg                |   54 +
 libcxx/include/__cxx03/cstdbool               |   31 +
 libcxx/include/__cxx03/cstddef                |  133 +
 libcxx/include/__cxx03/cstdint                |  199 +
 libcxx/include/__cxx03/cstdio                 |  174 +
 libcxx/include/__cxx03/cstdlib                |  154 +
 libcxx/include/__cxx03/cstring                |  104 +
 libcxx/include/__cxx03/ctgmath                |   28 +
 libcxx/include/__cxx03/ctime                  |   86 +
 libcxx/include/__cxx03/ctype.h                |   61 +
 libcxx/include/__cxx03/cuchar                 |   75 +
 libcxx/include/__cxx03/cwchar                 |  261 +
 libcxx/include/__cxx03/cwctype                |   97 +
 libcxx/include/__cxx03/deque                  | 2621 ++++++++
 libcxx/include/__cxx03/errno.h                |  399 ++
 libcxx/include/__cxx03/exception              |   96 +
 libcxx/include/__cxx03/execution              |  149 +
 libcxx/include/__cxx03/expected               |   62 +
 libcxx/include/__cxx03/experimental/__config  |   45 +
 .../__cxx03/experimental/__simd/aligned_tag.h |   75 +
 .../__cxx03/experimental/__simd/declaration.h |   81 +
 .../__cxx03/experimental/__simd/reference.h   |  105 +
 .../__cxx03/experimental/__simd/scalar.h      |   87 +
 .../__cxx03/experimental/__simd/simd.h        |  102 +
 .../__cxx03/experimental/__simd/simd_mask.h   |   90 +
 .../__cxx03/experimental/__simd/traits.h      |   75 +
 .../__cxx03/experimental/__simd/utility.h     |  103 +
 .../__cxx03/experimental/__simd/vec_ext.h     |  119 +
 libcxx/include/__cxx03/experimental/iterator  |  127 +
 libcxx/include/__cxx03/experimental/memory    |  198 +
 .../__cxx03/experimental/propagate_const      |  490 ++
 libcxx/include/__cxx03/experimental/simd      |   88 +
 .../include/__cxx03/experimental/type_traits  |  153 +
 libcxx/include/__cxx03/experimental/utility   |   46 +
 libcxx/include/__cxx03/ext/__hash             |   85 +
 libcxx/include/__cxx03/ext/hash_map           |  872 +++
 libcxx/include/__cxx03/ext/hash_set           |  584 ++
 libcxx/include/__cxx03/fenv.h                 |  114 +
 libcxx/include/__cxx03/filesystem             |  577 ++
 libcxx/include/__cxx03/float.h                |   95 +
 libcxx/include/__cxx03/format                 |  257 +
 libcxx/include/__cxx03/forward_list           | 1574 +++++
 libcxx/include/__cxx03/fstream                | 1572 +++++
 libcxx/include/__cxx03/functional             |  598 ++
 libcxx/include/__cxx03/future                 | 2062 ++++++
 libcxx/include/__cxx03/initializer_list       |   98 +
 libcxx/include/__cxx03/inttypes.h             |  264 +
 libcxx/include/__cxx03/iomanip                |  542 ++
 libcxx/include/__cxx03/ios                    |  889 +++
 libcxx/include/__cxx03/iosfwd                 |  186 +
 libcxx/include/__cxx03/iostream               |   67 +
 libcxx/include/__cxx03/istream                | 1373 ++++
 libcxx/include/__cxx03/iterator               |  754 +++
 libcxx/include/__cxx03/latch                  |  129 +
 libcxx/include/__cxx03/limits                 |  584 ++
 libcxx/include/__cxx03/list                   | 1742 +++++
 libcxx/include/__cxx03/locale                 | 3767 +++++++++++
 libcxx/include/__cxx03/locale.h               |   46 +
 libcxx/include/__cxx03/map                    | 2181 ++++++
 libcxx/include/__cxx03/math.h                 |  520 ++
 libcxx/include/__cxx03/mdspan                 |  438 ++
 libcxx/include/__cxx03/memory                 |  998 +++
 libcxx/include/__cxx03/memory_resource        |   83 +
 libcxx/include/__cxx03/module.modulemap       | 2136 ++++++
 libcxx/include/__cxx03/mutex                  |  516 ++
 libcxx/include/__cxx03/new                    |  362 +
 libcxx/include/__cxx03/numbers                |  164 +
 libcxx/include/__cxx03/numeric                |  207 +
 libcxx/include/__cxx03/optional               | 1304 ++++
 libcxx/include/__cxx03/ostream                |  202 +
 libcxx/include/__cxx03/print                  |  402 ++
 libcxx/include/__cxx03/queue                  |  956 +++
 libcxx/include/__cxx03/random                 | 1742 +++++
 libcxx/include/__cxx03/ranges                 |  463 ++
 libcxx/include/__cxx03/ratio                  |  517 ++
 libcxx/include/__cxx03/regex                  | 5836 +++++++++++++++++
 libcxx/include/__cxx03/scoped_allocator       |  573 ++
 libcxx/include/__cxx03/semaphore              |  188 +
 libcxx/include/__cxx03/set                    | 1500 +++++
 libcxx/include/__cxx03/shared_mutex           |  464 ++
 libcxx/include/__cxx03/source_location        |   85 +
 libcxx/include/__cxx03/span                   |  636 ++
 libcxx/include/__cxx03/sstream                | 1276 ++++
 libcxx/include/__cxx03/stack                  |  379 ++
 libcxx/include/__cxx03/stdatomic.h            |  235 +
 libcxx/include/__cxx03/stdbool.h              |   40 +
 libcxx/include/__cxx03/stddef.h               |   44 +
 libcxx/include/__cxx03/stdexcept              |  286 +
 libcxx/include/__cxx03/stdint.h               |  127 +
 libcxx/include/__cxx03/stdio.h                |  123 +
 libcxx/include/__cxx03/stdlib.h               |  151 +
 libcxx/include/__cxx03/stop_token             |   56 +
 libcxx/include/__cxx03/streambuf              |  445 ++
 libcxx/include/__cxx03/string                 | 4352 ++++++++++++
 libcxx/include/__cxx03/string.h               |  110 +
 libcxx/include/__cxx03/string_view            |  948 +++
 libcxx/include/__cxx03/strstream              |  351 +
 libcxx/include/__cxx03/syncstream             |  512 ++
 libcxx/include/__cxx03/system_error           |  171 +
 libcxx/include/__cxx03/tgmath.h               |   34 +
 libcxx/include/__cxx03/thread                 |  130 +
 libcxx/include/__cxx03/tuple                  | 1419 ++++
 libcxx/include/__cxx03/type_traits            |  523 ++
 libcxx/include/__cxx03/typeindex              |  106 +
 libcxx/include/__cxx03/typeinfo               |  389 ++
 libcxx/include/__cxx03/uchar.h                |   56 +
 libcxx/include/__cxx03/unordered_map          | 2541 +++++++
 libcxx/include/__cxx03/unordered_set          | 1811 +++++
 libcxx/include/__cxx03/utility                |  309 +
 libcxx/include/__cxx03/valarray               | 3364 ++++++++++
 libcxx/include/__cxx03/variant                | 1650 +++++
 libcxx/include/__cxx03/vector                 | 3030 +++++++++
 libcxx/include/__cxx03/version                |  566 ++
 libcxx/include/__cxx03/wchar.h                |  211 +
 libcxx/include/__cxx03/wctype.h               |   95 +
 libcxx/utils/generate_iwyu_mapping.py         |    1 +
 1020 files changed, 199790 insertions(+)
 create mode 100644 libcxx/include/__cxx03/CMakeLists.txt
 create mode 100644 libcxx/include/__cxx03/__algorithm/adjacent_find.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/all_of.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/any_of.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/binary_search.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/clamp.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/comp.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/comp_ref_type.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/copy_backward.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/copy_if.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/copy_move_common.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/copy_n.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/count.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/count_if.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/equal.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/equal_range.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/fill.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/fill_n.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/find.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/find_end.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/find_first_of.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/find_if.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/find_if_not.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/find_segment_if.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/fold.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/for_each.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/for_each_n.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/for_each_segment.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/generate.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/generate_n.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/half_positive.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/in_found_result.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/in_fun_result.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/in_in_out_result.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/in_in_result.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/in_out_out_result.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/in_out_result.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/includes.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/inplace_merge.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/is_heap.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/is_heap_until.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/is_partitioned.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/is_permutation.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/is_sorted.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/is_sorted_until.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/iter_swap.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/iterator_operations.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/lexicographical_compare.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/lexicographical_compare_three_way.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/lower_bound.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/make_heap.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/make_projected.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/max.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/max_element.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/merge.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/min.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/min_element.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/min_max_result.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/minmax.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/minmax_element.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/mismatch.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/move.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/move_backward.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/next_permutation.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/none_of.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/nth_element.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/partial_sort.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/partial_sort_copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/partition.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/partition_copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/partition_point.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/pop_heap.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/prev_permutation.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/pstl.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/push_heap.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_adjacent_find.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_all_of.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_any_of.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_binary_search.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_clamp.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_contains.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_contains_subrange.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_copy_backward.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_copy_if.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_copy_n.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_count.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_count_if.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_ends_with.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_equal.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_equal_range.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_fill.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_fill_n.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_find.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_find_end.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_find_first_of.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_find_if.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_find_if_not.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_find_last.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_for_each.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_for_each_n.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_generate.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_generate_n.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_includes.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_inplace_merge.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_is_heap.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_is_heap_until.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_is_partitioned.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_is_permutation.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_is_sorted.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_is_sorted_until.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_iterator_concept.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_lexicographical_compare.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_lower_bound.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_make_heap.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_max.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_max_element.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_merge.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_min.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_min_element.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_minmax.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_minmax_element.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_mismatch.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_move.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_move_backward.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_next_permutation.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_none_of.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_nth_element.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_partial_sort.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_partial_sort_copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_partition.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_partition_copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_partition_point.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_pop_heap.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_prev_permutation.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_push_heap.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_remove.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_remove_copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_remove_copy_if.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_remove_if.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_replace.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_replace_copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_replace_copy_if.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_replace_if.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_reverse.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_reverse_copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_rotate.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_rotate_copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_sample.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_search.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_search_n.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_set_difference.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_set_intersection.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_set_symmetric_difference.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_set_union.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_shuffle.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_sort.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_sort_heap.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_stable_partition.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_stable_sort.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_starts_with.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_swap_ranges.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_transform.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_unique.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_unique_copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/ranges_upper_bound.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/remove.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/remove_copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/remove_copy_if.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/remove_if.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/replace.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/replace_copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/replace_copy_if.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/replace_if.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/reverse.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/reverse_copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/rotate.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/rotate_copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/sample.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/search.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/search_n.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/set_difference.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/set_intersection.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/set_symmetric_difference.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/set_union.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/shift_left.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/shift_right.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/shuffle.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/sift_down.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/simd_utils.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/sort.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/sort_heap.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/stable_partition.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/stable_sort.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/swap_ranges.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/three_way_comp_ref_type.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/transform.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/uniform_random_bit_generator_adaptor.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/unique.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/unique_copy.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/unwrap_iter.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/unwrap_range.h
 create mode 100644 libcxx/include/__cxx03/__algorithm/upper_bound.h
 create mode 100644 libcxx/include/__cxx03/__assert
 create mode 100644 libcxx/include/__cxx03/__atomic/aliases.h
 create mode 100644 libcxx/include/__cxx03/__atomic/atomic.h
 create mode 100644 libcxx/include/__cxx03/__atomic/atomic_base.h
 create mode 100644 libcxx/include/__cxx03/__atomic/atomic_flag.h
 create mode 100644 libcxx/include/__cxx03/__atomic/atomic_init.h
 create mode 100644 libcxx/include/__cxx03/__atomic/atomic_lock_free.h
 create mode 100644 libcxx/include/__cxx03/__atomic/atomic_ref.h
 create mode 100644 libcxx/include/__cxx03/__atomic/atomic_sync.h
 create mode 100644 libcxx/include/__cxx03/__atomic/check_memory_order.h
 create mode 100644 libcxx/include/__cxx03/__atomic/contention_t.h
 create mode 100644 libcxx/include/__cxx03/__atomic/cxx_atomic_impl.h
 create mode 100644 libcxx/include/__cxx03/__atomic/fence.h
 create mode 100644 libcxx/include/__cxx03/__atomic/is_always_lock_free.h
 create mode 100644 libcxx/include/__cxx03/__atomic/kill_dependency.h
 create mode 100644 libcxx/include/__cxx03/__atomic/memory_order.h
 create mode 100644 libcxx/include/__cxx03/__atomic/to_gcc_order.h
 create mode 100644 libcxx/include/__cxx03/__bit/bit_cast.h
 create mode 100644 libcxx/include/__cxx03/__bit/bit_ceil.h
 create mode 100644 libcxx/include/__cxx03/__bit/bit_floor.h
 create mode 100644 libcxx/include/__cxx03/__bit/bit_log2.h
 create mode 100644 libcxx/include/__cxx03/__bit/bit_width.h
 create mode 100644 libcxx/include/__cxx03/__bit/blsr.h
 create mode 100644 libcxx/include/__cxx03/__bit/byteswap.h
 create mode 100644 libcxx/include/__cxx03/__bit/countl.h
 create mode 100644 libcxx/include/__cxx03/__bit/countr.h
 create mode 100644 libcxx/include/__cxx03/__bit/endian.h
 create mode 100644 libcxx/include/__cxx03/__bit/has_single_bit.h
 create mode 100644 libcxx/include/__cxx03/__bit/invert_if.h
 create mode 100644 libcxx/include/__cxx03/__bit/popcount.h
 create mode 100644 libcxx/include/__cxx03/__bit/rotate.h
 create mode 100644 libcxx/include/__cxx03/__bit_reference
 create mode 100644 libcxx/include/__cxx03/__charconv/chars_format.h
 create mode 100644 libcxx/include/__cxx03/__charconv/from_chars_integral.h
 create mode 100644 libcxx/include/__cxx03/__charconv/from_chars_result.h
 create mode 100644 libcxx/include/__cxx03/__charconv/tables.h
 create mode 100644 libcxx/include/__cxx03/__charconv/to_chars.h
 create mode 100644 libcxx/include/__cxx03/__charconv/to_chars_base_10.h
 create mode 100644 libcxx/include/__cxx03/__charconv/to_chars_floating_point.h
 create mode 100644 libcxx/include/__cxx03/__charconv/to_chars_integral.h
 create mode 100644 libcxx/include/__cxx03/__charconv/to_chars_result.h
 create mode 100644 libcxx/include/__cxx03/__charconv/traits.h
 create mode 100644 libcxx/include/__cxx03/__chrono/calendar.h
 create mode 100644 libcxx/include/__cxx03/__chrono/concepts.h
 create mode 100644 libcxx/include/__cxx03/__chrono/convert_to_timespec.h
 create mode 100644 libcxx/include/__cxx03/__chrono/convert_to_tm.h
 create mode 100644 libcxx/include/__cxx03/__chrono/day.h
 create mode 100644 libcxx/include/__cxx03/__chrono/duration.h
 create mode 100644 libcxx/include/__cxx03/__chrono/exception.h
 create mode 100644 libcxx/include/__cxx03/__chrono/file_clock.h
 create mode 100644 libcxx/include/__cxx03/__chrono/formatter.h
 create mode 100644 libcxx/include/__cxx03/__chrono/hh_mm_ss.h
 create mode 100644 libcxx/include/__cxx03/__chrono/high_resolution_clock.h
 create mode 100644 libcxx/include/__cxx03/__chrono/leap_second.h
 create mode 100644 libcxx/include/__cxx03/__chrono/literals.h
 create mode 100644 libcxx/include/__cxx03/__chrono/local_info.h
 create mode 100644 libcxx/include/__cxx03/__chrono/month.h
 create mode 100644 libcxx/include/__cxx03/__chrono/month_weekday.h
 create mode 100644 libcxx/include/__cxx03/__chrono/monthday.h
 create mode 100644 libcxx/include/__cxx03/__chrono/ostream.h
 create mode 100644 libcxx/include/__cxx03/__chrono/parser_std_format_spec.h
 create mode 100644 libcxx/include/__cxx03/__chrono/statically_widen.h
 create mode 100644 libcxx/include/__cxx03/__chrono/steady_clock.h
 create mode 100644 libcxx/include/__cxx03/__chrono/sys_info.h
 create mode 100644 libcxx/include/__cxx03/__chrono/system_clock.h
 create mode 100644 libcxx/include/__cxx03/__chrono/time_point.h
 create mode 100644 libcxx/include/__cxx03/__chrono/time_zone.h
 create mode 100644 libcxx/include/__cxx03/__chrono/time_zone_link.h
 create mode 100644 libcxx/include/__cxx03/__chrono/tzdb.h
 create mode 100644 libcxx/include/__cxx03/__chrono/tzdb_list.h
 create mode 100644 libcxx/include/__cxx03/__chrono/weekday.h
 create mode 100644 libcxx/include/__cxx03/__chrono/year.h
 create mode 100644 libcxx/include/__cxx03/__chrono/year_month.h
 create mode 100644 libcxx/include/__cxx03/__chrono/year_month_day.h
 create mode 100644 libcxx/include/__cxx03/__chrono/year_month_weekday.h
 create mode 100644 libcxx/include/__cxx03/__chrono/zoned_time.h
 create mode 100644 libcxx/include/__cxx03/__compare/common_comparison_category.h
 create mode 100644 libcxx/include/__cxx03/__compare/compare_partial_order_fallback.h
 create mode 100644 libcxx/include/__cxx03/__compare/compare_strong_order_fallback.h
 create mode 100644 libcxx/include/__cxx03/__compare/compare_three_way.h
 create mode 100644 libcxx/include/__cxx03/__compare/compare_three_way_result.h
 create mode 100644 libcxx/include/__cxx03/__compare/compare_weak_order_fallback.h
 create mode 100644 libcxx/include/__cxx03/__compare/is_eq.h
 create mode 100644 libcxx/include/__cxx03/__compare/ordering.h
 create mode 100644 libcxx/include/__cxx03/__compare/partial_order.h
 create mode 100644 libcxx/include/__cxx03/__compare/strong_order.h
 create mode 100644 libcxx/include/__cxx03/__compare/synth_three_way.h
 create mode 100644 libcxx/include/__cxx03/__compare/three_way_comparable.h
 create mode 100644 libcxx/include/__cxx03/__compare/weak_order.h
 create mode 100644 libcxx/include/__cxx03/__concepts/arithmetic.h
 create mode 100644 libcxx/include/__cxx03/__concepts/assignable.h
 create mode 100644 libcxx/include/__cxx03/__concepts/boolean_testable.h
 create mode 100644 libcxx/include/__cxx03/__concepts/class_or_enum.h
 create mode 100644 libcxx/include/__cxx03/__concepts/common_reference_with.h
 create mode 100644 libcxx/include/__cxx03/__concepts/common_with.h
 create mode 100644 libcxx/include/__cxx03/__concepts/constructible.h
 create mode 100644 libcxx/include/__cxx03/__concepts/convertible_to.h
 create mode 100644 libcxx/include/__cxx03/__concepts/copyable.h
 create mode 100644 libcxx/include/__cxx03/__concepts/derived_from.h
 create mode 100644 libcxx/include/__cxx03/__concepts/destructible.h
 create mode 100644 libcxx/include/__cxx03/__concepts/different_from.h
 create mode 100644 libcxx/include/__cxx03/__concepts/equality_comparable.h
 create mode 100644 libcxx/include/__cxx03/__concepts/invocable.h
 create mode 100644 libcxx/include/__cxx03/__concepts/movable.h
 create mode 100644 libcxx/include/__cxx03/__concepts/predicate.h
 create mode 100644 libcxx/include/__cxx03/__concepts/regular.h
 create mode 100644 libcxx/include/__cxx03/__concepts/relation.h
 create mode 100644 libcxx/include/__cxx03/__concepts/same_as.h
 create mode 100644 libcxx/include/__cxx03/__concepts/semiregular.h
 create mode 100644 libcxx/include/__cxx03/__concepts/swappable.h
 create mode 100644 libcxx/include/__cxx03/__concepts/totally_ordered.h
 create mode 100644 libcxx/include/__cxx03/__condition_variable/condition_variable.h
 create mode 100644 libcxx/include/__cxx03/__config
 create mode 100644 libcxx/include/__cxx03/__config_site.in
 create mode 100644 libcxx/include/__cxx03/__configuration/abi.h
 create mode 100644 libcxx/include/__cxx03/__configuration/availability.h
 create mode 100644 libcxx/include/__cxx03/__configuration/compiler.h
 create mode 100644 libcxx/include/__cxx03/__configuration/language.h
 create mode 100644 libcxx/include/__cxx03/__configuration/platform.h
 create mode 100644 libcxx/include/__cxx03/__coroutine/coroutine_handle.h
 create mode 100644 libcxx/include/__cxx03/__coroutine/coroutine_traits.h
 create mode 100644 libcxx/include/__cxx03/__coroutine/noop_coroutine_handle.h
 create mode 100644 libcxx/include/__cxx03/__coroutine/trivial_awaitables.h
 create mode 100644 libcxx/include/__cxx03/__debug_utils/randomize_range.h
 create mode 100644 libcxx/include/__cxx03/__debug_utils/sanitizers.h
 create mode 100644 libcxx/include/__cxx03/__debug_utils/strict_weak_ordering_check.h
 create mode 100644 libcxx/include/__cxx03/__exception/exception.h
 create mode 100644 libcxx/include/__cxx03/__exception/exception_ptr.h
 create mode 100644 libcxx/include/__cxx03/__exception/nested_exception.h
 create mode 100644 libcxx/include/__cxx03/__exception/operations.h
 create mode 100644 libcxx/include/__cxx03/__exception/terminate.h
 create mode 100644 libcxx/include/__cxx03/__expected/bad_expected_access.h
 create mode 100644 libcxx/include/__cxx03/__expected/expected.h
 create mode 100644 libcxx/include/__cxx03/__expected/unexpect.h
 create mode 100644 libcxx/include/__cxx03/__expected/unexpected.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/copy_options.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/directory_entry.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/directory_iterator.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/directory_options.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/file_status.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/file_time_type.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/file_type.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/filesystem_error.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/operations.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/path.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/path_iterator.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/perm_options.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/perms.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/recursive_directory_iterator.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/space_info.h
 create mode 100644 libcxx/include/__cxx03/__filesystem/u8path.h
 create mode 100644 libcxx/include/__cxx03/__format/buffer.h
 create mode 100644 libcxx/include/__cxx03/__format/concepts.h
 create mode 100644 libcxx/include/__cxx03/__format/container_adaptor.h
 create mode 100644 libcxx/include/__cxx03/__format/enable_insertable.h
 create mode 100644 libcxx/include/__cxx03/__format/escaped_output_table.h
 create mode 100644 libcxx/include/__cxx03/__format/extended_grapheme_cluster_table.h
 create mode 100644 libcxx/include/__cxx03/__format/format_arg.h
 create mode 100644 libcxx/include/__cxx03/__format/format_arg_store.h
 create mode 100644 libcxx/include/__cxx03/__format/format_args.h
 create mode 100644 libcxx/include/__cxx03/__format/format_context.h
 create mode 100644 libcxx/include/__cxx03/__format/format_error.h
 create mode 100644 libcxx/include/__cxx03/__format/format_functions.h
 create mode 100644 libcxx/include/__cxx03/__format/format_parse_context.h
 create mode 100644 libcxx/include/__cxx03/__format/format_string.h
 create mode 100644 libcxx/include/__cxx03/__format/format_to_n_result.h
 create mode 100644 libcxx/include/__cxx03/__format/formatter.h
 create mode 100644 libcxx/include/__cxx03/__format/formatter_bool.h
 create mode 100644 libcxx/include/__cxx03/__format/formatter_char.h
 create mode 100644 libcxx/include/__cxx03/__format/formatter_floating_point.h
 create mode 100644 libcxx/include/__cxx03/__format/formatter_integer.h
 create mode 100644 libcxx/include/__cxx03/__format/formatter_integral.h
 create mode 100644 libcxx/include/__cxx03/__format/formatter_output.h
 create mode 100644 libcxx/include/__cxx03/__format/formatter_pointer.h
 create mode 100644 libcxx/include/__cxx03/__format/formatter_string.h
 create mode 100644 libcxx/include/__cxx03/__format/formatter_tuple.h
 create mode 100644 libcxx/include/__cxx03/__format/indic_conjunct_break_table.h
 create mode 100644 libcxx/include/__cxx03/__format/parser_std_format_spec.h
 create mode 100644 libcxx/include/__cxx03/__format/range_default_formatter.h
 create mode 100644 libcxx/include/__cxx03/__format/range_formatter.h
 create mode 100644 libcxx/include/__cxx03/__format/unicode.h
 create mode 100644 libcxx/include/__cxx03/__format/width_estimation_table.h
 create mode 100644 libcxx/include/__cxx03/__format/write_escaped.h
 create mode 100644 libcxx/include/__cxx03/__functional/binary_function.h
 create mode 100644 libcxx/include/__cxx03/__functional/binary_negate.h
 create mode 100644 libcxx/include/__cxx03/__functional/bind.h
 create mode 100644 libcxx/include/__cxx03/__functional/bind_back.h
 create mode 100644 libcxx/include/__cxx03/__functional/bind_front.h
 create mode 100644 libcxx/include/__cxx03/__functional/binder1st.h
 create mode 100644 libcxx/include/__cxx03/__functional/binder2nd.h
 create mode 100644 libcxx/include/__cxx03/__functional/boyer_moore_searcher.h
 create mode 100644 libcxx/include/__cxx03/__functional/compose.h
 create mode 100644 libcxx/include/__cxx03/__functional/default_searcher.h
 create mode 100644 libcxx/include/__cxx03/__functional/function.h
 create mode 100644 libcxx/include/__cxx03/__functional/hash.h
 create mode 100644 libcxx/include/__cxx03/__functional/identity.h
 create mode 100644 libcxx/include/__cxx03/__functional/invoke.h
 create mode 100644 libcxx/include/__cxx03/__functional/is_transparent.h
 create mode 100644 libcxx/include/__cxx03/__functional/mem_fn.h
 create mode 100644 libcxx/include/__cxx03/__functional/mem_fun_ref.h
 create mode 100644 libcxx/include/__cxx03/__functional/not_fn.h
 create mode 100644 libcxx/include/__cxx03/__functional/operations.h
 create mode 100644 libcxx/include/__cxx03/__functional/perfect_forward.h
 create mode 100644 libcxx/include/__cxx03/__functional/pointer_to_binary_function.h
 create mode 100644 libcxx/include/__cxx03/__functional/pointer_to_unary_function.h
 create mode 100644 libcxx/include/__cxx03/__functional/ranges_operations.h
 create mode 100644 libcxx/include/__cxx03/__functional/reference_wrapper.h
 create mode 100644 libcxx/include/__cxx03/__functional/unary_function.h
 create mode 100644 libcxx/include/__cxx03/__functional/unary_negate.h
 create mode 100644 libcxx/include/__cxx03/__functional/weak_result_type.h
 create mode 100644 libcxx/include/__cxx03/__fwd/array.h
 create mode 100644 libcxx/include/__cxx03/__fwd/bit_reference.h
 create mode 100644 libcxx/include/__cxx03/__fwd/complex.h
 create mode 100644 libcxx/include/__cxx03/__fwd/deque.h
 create mode 100644 libcxx/include/__cxx03/__fwd/format.h
 create mode 100644 libcxx/include/__cxx03/__fwd/fstream.h
 create mode 100644 libcxx/include/__cxx03/__fwd/functional.h
 create mode 100644 libcxx/include/__cxx03/__fwd/ios.h
 create mode 100644 libcxx/include/__cxx03/__fwd/istream.h
 create mode 100644 libcxx/include/__cxx03/__fwd/mdspan.h
 create mode 100644 libcxx/include/__cxx03/__fwd/memory.h
 create mode 100644 libcxx/include/__cxx03/__fwd/memory_resource.h
 create mode 100644 libcxx/include/__cxx03/__fwd/ostream.h
 create mode 100644 libcxx/include/__cxx03/__fwd/pair.h
 create mode 100644 libcxx/include/__cxx03/__fwd/queue.h
 create mode 100644 libcxx/include/__cxx03/__fwd/span.h
 create mode 100644 libcxx/include/__cxx03/__fwd/sstream.h
 create mode 100644 libcxx/include/__cxx03/__fwd/stack.h
 create mode 100644 libcxx/include/__cxx03/__fwd/streambuf.h
 create mode 100644 libcxx/include/__cxx03/__fwd/string.h
 create mode 100644 libcxx/include/__cxx03/__fwd/string_view.h
 create mode 100644 libcxx/include/__cxx03/__fwd/subrange.h
 create mode 100644 libcxx/include/__cxx03/__fwd/tuple.h
 create mode 100644 libcxx/include/__cxx03/__fwd/vector.h
 create mode 100644 libcxx/include/__cxx03/__hash_table
 create mode 100644 libcxx/include/__cxx03/__ios/fpos.h
 create mode 100644 libcxx/include/__cxx03/__iterator/access.h
 create mode 100644 libcxx/include/__cxx03/__iterator/advance.h
 create mode 100644 libcxx/include/__cxx03/__iterator/aliasing_iterator.h
 create mode 100644 libcxx/include/__cxx03/__iterator/back_insert_iterator.h
 create mode 100644 libcxx/include/__cxx03/__iterator/bounded_iter.h
 create mode 100644 libcxx/include/__cxx03/__iterator/common_iterator.h
 create mode 100644 libcxx/include/__cxx03/__iterator/concepts.h
 create mode 100644 libcxx/include/__cxx03/__iterator/counted_iterator.h
 create mode 100644 libcxx/include/__cxx03/__iterator/cpp17_iterator_concepts.h
 create mode 100644 libcxx/include/__cxx03/__iterator/data.h
 create mode 100644 libcxx/include/__cxx03/__iterator/default_sentinel.h
 create mode 100644 libcxx/include/__cxx03/__iterator/distance.h
 create mode 100644 libcxx/include/__cxx03/__iterator/empty.h
 create mode 100644 libcxx/include/__cxx03/__iterator/erase_if_container.h
 create mode 100644 libcxx/include/__cxx03/__iterator/front_insert_iterator.h
 create mode 100644 libcxx/include/__cxx03/__iterator/incrementable_traits.h
 create mode 100644 libcxx/include/__cxx03/__iterator/indirectly_comparable.h
 create mode 100644 libcxx/include/__cxx03/__iterator/insert_iterator.h
 create mode 100644 libcxx/include/__cxx03/__iterator/istream_iterator.h
 create mode 100644 libcxx/include/__cxx03/__iterator/istreambuf_iterator.h
 create mode 100644 libcxx/include/__cxx03/__iterator/iter_move.h
 create mode 100644 libcxx/include/__cxx03/__iterator/iter_swap.h
 create mode 100644 libcxx/include/__cxx03/__iterator/iterator.h
 create mode 100644 libcxx/include/__cxx03/__iterator/iterator_traits.h
 create mode 100644 libcxx/include/__cxx03/__iterator/iterator_with_data.h
 create mode 100644 libcxx/include/__cxx03/__iterator/mergeable.h
 create mode 100644 libcxx/include/__cxx03/__iterator/move_iterator.h
 create mode 100644 libcxx/include/__cxx03/__iterator/move_sentinel.h
 create mode 100644 libcxx/include/__cxx03/__iterator/next.h
 create mode 100644 libcxx/include/__cxx03/__iterator/ostream_iterator.h
 create mode 100644 libcxx/include/__cxx03/__iterator/ostreambuf_iterator.h
 create mode 100644 libcxx/include/__cxx03/__iterator/permutable.h
 create mode 100644 libcxx/include/__cxx03/__iterator/prev.h
 create mode 100644 libcxx/include/__cxx03/__iterator/projected.h
 create mode 100644 libcxx/include/__cxx03/__iterator/ranges_iterator_traits.h
 create mode 100644 libcxx/include/__cxx03/__iterator/readable_traits.h
 create mode 100644 libcxx/include/__cxx03/__iterator/reverse_access.h
 create mode 100644 libcxx/include/__cxx03/__iterator/reverse_iterator.h
 create mode 100644 libcxx/include/__cxx03/__iterator/segmented_iterator.h
 create mode 100644 libcxx/include/__cxx03/__iterator/size.h
 create mode 100644 libcxx/include/__cxx03/__iterator/sortable.h
 create mode 100644 libcxx/include/__cxx03/__iterator/unreachable_sentinel.h
 create mode 100644 libcxx/include/__cxx03/__iterator/wrap_iter.h
 create mode 100644 libcxx/include/__cxx03/__locale
 create mode 100644 libcxx/include/__cxx03/__locale_dir/locale_base_api.h
 create mode 100644 libcxx/include/__cxx03/__locale_dir/locale_base_api/android.h
 create mode 100644 libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_defaults.h
 create mode 100644 libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_fallbacks.h
 create mode 100644 libcxx/include/__cxx03/__locale_dir/locale_base_api/fuchsia.h
 create mode 100644 libcxx/include/__cxx03/__locale_dir/locale_base_api/ibm.h
 create mode 100644 libcxx/include/__cxx03/__locale_dir/locale_base_api/locale_guard.h
 create mode 100644 libcxx/include/__cxx03/__locale_dir/locale_base_api/musl.h
 create mode 100644 libcxx/include/__cxx03/__locale_dir/locale_base_api/newlib.h
 create mode 100644 libcxx/include/__cxx03/__locale_dir/locale_base_api/openbsd.h
 create mode 100644 libcxx/include/__cxx03/__locale_dir/locale_base_api/win32.h
 create mode 100644 libcxx/include/__cxx03/__math/abs.h
 create mode 100644 libcxx/include/__cxx03/__math/copysign.h
 create mode 100644 libcxx/include/__cxx03/__math/error_functions.h
 create mode 100644 libcxx/include/__cxx03/__math/exponential_functions.h
 create mode 100644 libcxx/include/__cxx03/__math/fdim.h
 create mode 100644 libcxx/include/__cxx03/__math/fma.h
 create mode 100644 libcxx/include/__cxx03/__math/gamma.h
 create mode 100644 libcxx/include/__cxx03/__math/hyperbolic_functions.h
 create mode 100644 libcxx/include/__cxx03/__math/hypot.h
 create mode 100644 libcxx/include/__cxx03/__math/inverse_hyperbolic_functions.h
 create mode 100644 libcxx/include/__cxx03/__math/inverse_trigonometric_functions.h
 create mode 100644 libcxx/include/__cxx03/__math/logarithms.h
 create mode 100644 libcxx/include/__cxx03/__math/min_max.h
 create mode 100644 libcxx/include/__cxx03/__math/modulo.h
 create mode 100644 libcxx/include/__cxx03/__math/remainder.h
 create mode 100644 libcxx/include/__cxx03/__math/roots.h
 create mode 100644 libcxx/include/__cxx03/__math/rounding_functions.h
 create mode 100644 libcxx/include/__cxx03/__math/special_functions.h
 create mode 100644 libcxx/include/__cxx03/__math/traits.h
 create mode 100644 libcxx/include/__cxx03/__math/trigonometric_functions.h
 create mode 100644 libcxx/include/__cxx03/__mbstate_t.h
 create mode 100644 libcxx/include/__cxx03/__mdspan/default_accessor.h
 create mode 100644 libcxx/include/__cxx03/__mdspan/extents.h
 create mode 100644 libcxx/include/__cxx03/__mdspan/layout_left.h
 create mode 100644 libcxx/include/__cxx03/__mdspan/layout_right.h
 create mode 100644 libcxx/include/__cxx03/__mdspan/layout_stride.h
 create mode 100644 libcxx/include/__cxx03/__mdspan/mdspan.h
 create mode 100644 libcxx/include/__cxx03/__memory/addressof.h
 create mode 100644 libcxx/include/__cxx03/__memory/align.h
 create mode 100644 libcxx/include/__cxx03/__memory/aligned_alloc.h
 create mode 100644 libcxx/include/__cxx03/__memory/allocate_at_least.h
 create mode 100644 libcxx/include/__cxx03/__memory/allocation_guard.h
 create mode 100644 libcxx/include/__cxx03/__memory/allocator.h
 create mode 100644 libcxx/include/__cxx03/__memory/allocator_arg_t.h
 create mode 100644 libcxx/include/__cxx03/__memory/allocator_destructor.h
 create mode 100644 libcxx/include/__cxx03/__memory/allocator_traits.h
 create mode 100644 libcxx/include/__cxx03/__memory/assume_aligned.h
 create mode 100644 libcxx/include/__cxx03/__memory/auto_ptr.h
 create mode 100644 libcxx/include/__cxx03/__memory/builtin_new_allocator.h
 create mode 100644 libcxx/include/__cxx03/__memory/compressed_pair.h
 create mode 100644 libcxx/include/__cxx03/__memory/concepts.h
 create mode 100644 libcxx/include/__cxx03/__memory/construct_at.h
 create mode 100644 libcxx/include/__cxx03/__memory/destruct_n.h
 create mode 100644 libcxx/include/__cxx03/__memory/inout_ptr.h
 create mode 100644 libcxx/include/__cxx03/__memory/out_ptr.h
 create mode 100644 libcxx/include/__cxx03/__memory/pointer_traits.h
 create mode 100644 libcxx/include/__cxx03/__memory/ranges_construct_at.h
 create mode 100644 libcxx/include/__cxx03/__memory/ranges_uninitialized_algorithms.h
 create mode 100644 libcxx/include/__cxx03/__memory/raw_storage_iterator.h
 create mode 100644 libcxx/include/__cxx03/__memory/shared_ptr.h
 create mode 100644 libcxx/include/__cxx03/__memory/swap_allocator.h
 create mode 100644 libcxx/include/__cxx03/__memory/temp_value.h
 create mode 100644 libcxx/include/__cxx03/__memory/temporary_buffer.h
 create mode 100644 libcxx/include/__cxx03/__memory/uninitialized_algorithms.h
 create mode 100644 libcxx/include/__cxx03/__memory/unique_ptr.h
 create mode 100644 libcxx/include/__cxx03/__memory/uses_allocator.h
 create mode 100644 libcxx/include/__cxx03/__memory/uses_allocator_construction.h
 create mode 100644 libcxx/include/__cxx03/__memory/voidify.h
 create mode 100644 libcxx/include/__cxx03/__memory_resource/memory_resource.h
 create mode 100644 libcxx/include/__cxx03/__memory_resource/monotonic_buffer_resource.h
 create mode 100644 libcxx/include/__cxx03/__memory_resource/polymorphic_allocator.h
 create mode 100644 libcxx/include/__cxx03/__memory_resource/pool_options.h
 create mode 100644 libcxx/include/__cxx03/__memory_resource/synchronized_pool_resource.h
 create mode 100644 libcxx/include/__cxx03/__memory_resource/unsynchronized_pool_resource.h
 create mode 100644 libcxx/include/__cxx03/__mutex/lock_guard.h
 create mode 100644 libcxx/include/__cxx03/__mutex/mutex.h
 create mode 100644 libcxx/include/__cxx03/__mutex/once_flag.h
 create mode 100644 libcxx/include/__cxx03/__mutex/tag_types.h
 create mode 100644 libcxx/include/__cxx03/__mutex/unique_lock.h
 create mode 100644 libcxx/include/__cxx03/__node_handle
 create mode 100644 libcxx/include/__cxx03/__numeric/accumulate.h
 create mode 100644 libcxx/include/__cxx03/__numeric/adjacent_difference.h
 create mode 100644 libcxx/include/__cxx03/__numeric/exclusive_scan.h
 create mode 100644 libcxx/include/__cxx03/__numeric/gcd_lcm.h
 create mode 100644 libcxx/include/__cxx03/__numeric/inclusive_scan.h
 create mode 100644 libcxx/include/__cxx03/__numeric/inner_product.h
 create mode 100644 libcxx/include/__cxx03/__numeric/iota.h
 create mode 100644 libcxx/include/__cxx03/__numeric/midpoint.h
 create mode 100644 libcxx/include/__cxx03/__numeric/partial_sum.h
 create mode 100644 libcxx/include/__cxx03/__numeric/pstl.h
 create mode 100644 libcxx/include/__cxx03/__numeric/reduce.h
 create mode 100644 libcxx/include/__cxx03/__numeric/saturation_arithmetic.h
 create mode 100644 libcxx/include/__cxx03/__numeric/transform_exclusive_scan.h
 create mode 100644 libcxx/include/__cxx03/__numeric/transform_inclusive_scan.h
 create mode 100644 libcxx/include/__cxx03/__numeric/transform_reduce.h
 create mode 100644 libcxx/include/__cxx03/__ostream/basic_ostream.h
 create mode 100644 libcxx/include/__cxx03/__ostream/print.h
 create mode 100644 libcxx/include/__cxx03/__pstl/backend.h
 create mode 100644 libcxx/include/__cxx03/__pstl/backend_fwd.h
 create mode 100644 libcxx/include/__cxx03/__pstl/backends/default.h
 create mode 100644 libcxx/include/__cxx03/__pstl/backends/libdispatch.h
 create mode 100644 libcxx/include/__cxx03/__pstl/backends/serial.h
 create mode 100644 libcxx/include/__cxx03/__pstl/backends/std_thread.h
 create mode 100644 libcxx/include/__cxx03/__pstl/cpu_algos/any_of.h
 create mode 100644 libcxx/include/__cxx03/__pstl/cpu_algos/cpu_traits.h
 create mode 100644 libcxx/include/__cxx03/__pstl/cpu_algos/fill.h
 create mode 100644 libcxx/include/__cxx03/__pstl/cpu_algos/find_if.h
 create mode 100644 libcxx/include/__cxx03/__pstl/cpu_algos/for_each.h
 create mode 100644 libcxx/include/__cxx03/__pstl/cpu_algos/merge.h
 create mode 100644 libcxx/include/__cxx03/__pstl/cpu_algos/stable_sort.h
 create mode 100644 libcxx/include/__cxx03/__pstl/cpu_algos/transform.h
 create mode 100644 libcxx/include/__cxx03/__pstl/cpu_algos/transform_reduce.h
 create mode 100644 libcxx/include/__cxx03/__pstl/dispatch.h
 create mode 100644 libcxx/include/__cxx03/__pstl/handle_exception.h
 create mode 100644 libcxx/include/__cxx03/__random/bernoulli_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/binomial_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/cauchy_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/chi_squared_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/clamp_to_integral.h
 create mode 100644 libcxx/include/__cxx03/__random/default_random_engine.h
 create mode 100644 libcxx/include/__cxx03/__random/discard_block_engine.h
 create mode 100644 libcxx/include/__cxx03/__random/discrete_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/exponential_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/extreme_value_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/fisher_f_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/gamma_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/generate_canonical.h
 create mode 100644 libcxx/include/__cxx03/__random/geometric_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/independent_bits_engine.h
 create mode 100644 libcxx/include/__cxx03/__random/is_seed_sequence.h
 create mode 100644 libcxx/include/__cxx03/__random/is_valid.h
 create mode 100644 libcxx/include/__cxx03/__random/knuth_b.h
 create mode 100644 libcxx/include/__cxx03/__random/linear_congruential_engine.h
 create mode 100644 libcxx/include/__cxx03/__random/log2.h
 create mode 100644 libcxx/include/__cxx03/__random/lognormal_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/mersenne_twister_engine.h
 create mode 100644 libcxx/include/__cxx03/__random/negative_binomial_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/normal_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/piecewise_constant_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/piecewise_linear_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/poisson_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/random_device.h
 create mode 100644 libcxx/include/__cxx03/__random/ranlux.h
 create mode 100644 libcxx/include/__cxx03/__random/seed_seq.h
 create mode 100644 libcxx/include/__cxx03/__random/shuffle_order_engine.h
 create mode 100644 libcxx/include/__cxx03/__random/student_t_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/subtract_with_carry_engine.h
 create mode 100644 libcxx/include/__cxx03/__random/uniform_int_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/uniform_random_bit_generator.h
 create mode 100644 libcxx/include/__cxx03/__random/uniform_real_distribution.h
 create mode 100644 libcxx/include/__cxx03/__random/weibull_distribution.h
 create mode 100644 libcxx/include/__cxx03/__ranges/access.h
 create mode 100644 libcxx/include/__cxx03/__ranges/all.h
 create mode 100644 libcxx/include/__cxx03/__ranges/as_rvalue_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/chunk_by_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/common_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/concepts.h
 create mode 100644 libcxx/include/__cxx03/__ranges/container_compatible_range.h
 create mode 100644 libcxx/include/__cxx03/__ranges/counted.h
 create mode 100644 libcxx/include/__cxx03/__ranges/dangling.h
 create mode 100644 libcxx/include/__cxx03/__ranges/data.h
 create mode 100644 libcxx/include/__cxx03/__ranges/drop_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/drop_while_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/elements_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/empty.h
 create mode 100644 libcxx/include/__cxx03/__ranges/empty_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/enable_borrowed_range.h
 create mode 100644 libcxx/include/__cxx03/__ranges/enable_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/filter_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/from_range.h
 create mode 100644 libcxx/include/__cxx03/__ranges/iota_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/istream_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/join_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/lazy_split_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/movable_box.h
 create mode 100644 libcxx/include/__cxx03/__ranges/non_propagating_cache.h
 create mode 100644 libcxx/include/__cxx03/__ranges/owning_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/range_adaptor.h
 create mode 100644 libcxx/include/__cxx03/__ranges/rbegin.h
 create mode 100644 libcxx/include/__cxx03/__ranges/ref_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/rend.h
 create mode 100644 libcxx/include/__cxx03/__ranges/repeat_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/reverse_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/single_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/size.h
 create mode 100644 libcxx/include/__cxx03/__ranges/split_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/subrange.h
 create mode 100644 libcxx/include/__cxx03/__ranges/take_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/take_while_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/to.h
 create mode 100644 libcxx/include/__cxx03/__ranges/transform_view.h
 create mode 100644 libcxx/include/__cxx03/__ranges/view_interface.h
 create mode 100644 libcxx/include/__cxx03/__ranges/views.h
 create mode 100644 libcxx/include/__cxx03/__ranges/zip_view.h
 create mode 100644 libcxx/include/__cxx03/__split_buffer
 create mode 100644 libcxx/include/__cxx03/__std_clang_module
 create mode 100644 libcxx/include/__cxx03/__std_mbstate_t.h
 create mode 100644 libcxx/include/__cxx03/__stop_token/atomic_unique_lock.h
 create mode 100644 libcxx/include/__cxx03/__stop_token/intrusive_list_view.h
 create mode 100644 libcxx/include/__cxx03/__stop_token/intrusive_shared_ptr.h
 create mode 100644 libcxx/include/__cxx03/__stop_token/stop_callback.h
 create mode 100644 libcxx/include/__cxx03/__stop_token/stop_source.h
 create mode 100644 libcxx/include/__cxx03/__stop_token/stop_state.h
 create mode 100644 libcxx/include/__cxx03/__stop_token/stop_token.h
 create mode 100644 libcxx/include/__cxx03/__string/char_traits.h
 create mode 100644 libcxx/include/__cxx03/__string/constexpr_c_functions.h
 create mode 100644 libcxx/include/__cxx03/__string/extern_template_lists.h
 create mode 100644 libcxx/include/__cxx03/__support/ibm/gettod_zos.h
 create mode 100644 libcxx/include/__cxx03/__support/ibm/locale_mgmt_zos.h
 create mode 100644 libcxx/include/__cxx03/__support/ibm/nanosleep.h
 create mode 100644 libcxx/include/__cxx03/__support/xlocale/__nop_locale_mgmt.h
 create mode 100644 libcxx/include/__cxx03/__support/xlocale/__posix_l_fallback.h
 create mode 100644 libcxx/include/__cxx03/__support/xlocale/__strtonum_fallback.h
 create mode 100644 libcxx/include/__cxx03/__system_error/errc.h
 create mode 100644 libcxx/include/__cxx03/__system_error/error_category.h
 create mode 100644 libcxx/include/__cxx03/__system_error/error_code.h
 create mode 100644 libcxx/include/__cxx03/__system_error/error_condition.h
 create mode 100644 libcxx/include/__cxx03/__system_error/system_error.h
 create mode 100644 libcxx/include/__cxx03/__thread/formatter.h
 create mode 100644 libcxx/include/__cxx03/__thread/id.h
 create mode 100644 libcxx/include/__cxx03/__thread/jthread.h
 create mode 100644 libcxx/include/__cxx03/__thread/poll_with_backoff.h
 create mode 100644 libcxx/include/__cxx03/__thread/support.h
 create mode 100644 libcxx/include/__cxx03/__thread/support/c11.h
 create mode 100644 libcxx/include/__cxx03/__thread/support/external.h
 create mode 100644 libcxx/include/__cxx03/__thread/support/pthread.h
 create mode 100644 libcxx/include/__cxx03/__thread/support/windows.h
 create mode 100644 libcxx/include/__cxx03/__thread/this_thread.h
 create mode 100644 libcxx/include/__cxx03/__thread/thread.h
 create mode 100644 libcxx/include/__cxx03/__thread/timed_backoff_policy.h
 create mode 100644 libcxx/include/__cxx03/__tree
 create mode 100644 libcxx/include/__cxx03/__tuple/find_index.h
 create mode 100644 libcxx/include/__cxx03/__tuple/ignore.h
 create mode 100644 libcxx/include/__cxx03/__tuple/make_tuple_types.h
 create mode 100644 libcxx/include/__cxx03/__tuple/sfinae_helpers.h
 create mode 100644 libcxx/include/__cxx03/__tuple/tuple_element.h
 create mode 100644 libcxx/include/__cxx03/__tuple/tuple_indices.h
 create mode 100644 libcxx/include/__cxx03/__tuple/tuple_like.h
 create mode 100644 libcxx/include/__cxx03/__tuple/tuple_like_ext.h
 create mode 100644 libcxx/include/__cxx03/__tuple/tuple_like_no_subrange.h
 create mode 100644 libcxx/include/__cxx03/__tuple/tuple_size.h
 create mode 100644 libcxx/include/__cxx03/__tuple/tuple_types.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/add_const.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/add_cv.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/add_lvalue_reference.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/add_pointer.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/add_rvalue_reference.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/add_volatile.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/aligned_storage.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/aligned_union.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/alignment_of.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/can_extract_key.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/common_reference.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/common_type.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/conditional.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/conjunction.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/copy_cv.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/copy_cvref.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/datasizeof.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/decay.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/dependent_type.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/desugars_to.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/disjunction.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/enable_if.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/extent.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/has_unique_object_representation.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/has_virtual_destructor.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/integral_constant.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/invoke.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_abstract.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_aggregate.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_allocator.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_always_bitcastable.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_arithmetic.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_array.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_assignable.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_base_of.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_bounded_array.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_callable.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_char_like_type.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_class.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_compound.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_const.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_constant_evaluated.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_constructible.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_convertible.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_core_convertible.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_destructible.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_empty.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_enum.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_equality_comparable.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_execution_policy.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_final.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_floating_point.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_function.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_fundamental.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_implicitly_default_constructible.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_integral.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_literal_type.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_member_pointer.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_nothrow_assignable.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_nothrow_constructible.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_nothrow_convertible.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_nothrow_destructible.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_null_pointer.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_object.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_pod.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_pointer.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_polymorphic.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_primary_template.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_reference.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_reference_wrapper.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_referenceable.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_same.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_scalar.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_signed.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_signed_integer.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_specialization.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_standard_layout.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_swappable.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_trivial.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_trivially_assignable.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_trivially_constructible.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_trivially_copyable.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_trivially_destructible.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_trivially_lexicographically_comparable.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_trivially_relocatable.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_unbounded_array.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_union.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_unsigned.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_unsigned_integer.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_valid_expansion.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_void.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/is_volatile.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/lazy.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/make_32_64_or_128_bit.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/make_const_lvalue_ref.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/make_signed.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/make_unsigned.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/maybe_const.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/nat.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/negation.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/noexcept_move_assign_container.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/promote.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/rank.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/remove_all_extents.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/remove_const.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/remove_const_ref.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/remove_cv.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/remove_cvref.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/remove_extent.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/remove_pointer.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/remove_reference.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/remove_volatile.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/result_of.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/strip_signature.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/type_identity.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/type_list.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/underlying_type.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/unwrap_ref.h
 create mode 100644 libcxx/include/__cxx03/__type_traits/void_t.h
 create mode 100644 libcxx/include/__cxx03/__undef_macros
 create mode 100644 libcxx/include/__cxx03/__utility/as_const.h
 create mode 100644 libcxx/include/__cxx03/__utility/as_lvalue.h
 create mode 100644 libcxx/include/__cxx03/__utility/auto_cast.h
 create mode 100644 libcxx/include/__cxx03/__utility/cmp.h
 create mode 100644 libcxx/include/__cxx03/__utility/convert_to_integral.h
 create mode 100644 libcxx/include/__cxx03/__utility/declval.h
 create mode 100644 libcxx/include/__cxx03/__utility/empty.h
 create mode 100644 libcxx/include/__cxx03/__utility/exception_guard.h
 create mode 100644 libcxx/include/__cxx03/__utility/exchange.h
 create mode 100644 libcxx/include/__cxx03/__utility/forward.h
 create mode 100644 libcxx/include/__cxx03/__utility/forward_like.h
 create mode 100644 libcxx/include/__cxx03/__utility/in_place.h
 create mode 100644 libcxx/include/__cxx03/__utility/integer_sequence.h
 create mode 100644 libcxx/include/__cxx03/__utility/is_pointer_in_range.h
 create mode 100644 libcxx/include/__cxx03/__utility/is_valid_range.h
 create mode 100644 libcxx/include/__cxx03/__utility/move.h
 create mode 100644 libcxx/include/__cxx03/__utility/no_destroy.h
 create mode 100644 libcxx/include/__cxx03/__utility/pair.h
 create mode 100644 libcxx/include/__cxx03/__utility/piecewise_construct.h
 create mode 100644 libcxx/include/__cxx03/__utility/priority_tag.h
 create mode 100644 libcxx/include/__cxx03/__utility/private_constructor_tag.h
 create mode 100644 libcxx/include/__cxx03/__utility/rel_ops.h
 create mode 100644 libcxx/include/__cxx03/__utility/small_buffer.h
 create mode 100644 libcxx/include/__cxx03/__utility/swap.h
 create mode 100644 libcxx/include/__cxx03/__utility/to_underlying.h
 create mode 100644 libcxx/include/__cxx03/__utility/unreachable.h
 create mode 100644 libcxx/include/__cxx03/__variant/monostate.h
 create mode 100644 libcxx/include/__cxx03/__verbose_abort
 create mode 100644 libcxx/include/__cxx03/algorithm
 create mode 100644 libcxx/include/__cxx03/any
 create mode 100644 libcxx/include/__cxx03/array
 create mode 100644 libcxx/include/__cxx03/atomic
 create mode 100644 libcxx/include/__cxx03/barrier
 create mode 100644 libcxx/include/__cxx03/bit
 create mode 100644 libcxx/include/__cxx03/bitset
 create mode 100644 libcxx/include/__cxx03/cassert
 create mode 100644 libcxx/include/__cxx03/ccomplex
 create mode 100644 libcxx/include/__cxx03/cctype
 create mode 100644 libcxx/include/__cxx03/cerrno
 create mode 100644 libcxx/include/__cxx03/cfenv
 create mode 100644 libcxx/include/__cxx03/cfloat
 create mode 100644 libcxx/include/__cxx03/charconv
 create mode 100644 libcxx/include/__cxx03/chrono
 create mode 100644 libcxx/include/__cxx03/cinttypes
 create mode 100644 libcxx/include/__cxx03/ciso646
 create mode 100644 libcxx/include/__cxx03/climits
 create mode 100644 libcxx/include/__cxx03/clocale
 create mode 100644 libcxx/include/__cxx03/cmath
 create mode 100644 libcxx/include/__cxx03/codecvt
 create mode 100644 libcxx/include/__cxx03/compare
 create mode 100644 libcxx/include/__cxx03/complex
 create mode 100644 libcxx/include/__cxx03/complex.h
 create mode 100644 libcxx/include/__cxx03/concepts
 create mode 100644 libcxx/include/__cxx03/condition_variable
 create mode 100644 libcxx/include/__cxx03/coroutine
 create mode 100644 libcxx/include/__cxx03/csetjmp
 create mode 100644 libcxx/include/__cxx03/csignal
 create mode 100644 libcxx/include/__cxx03/cstdarg
 create mode 100644 libcxx/include/__cxx03/cstdbool
 create mode 100644 libcxx/include/__cxx03/cstddef
 create mode 100644 libcxx/include/__cxx03/cstdint
 create mode 100644 libcxx/include/__cxx03/cstdio
 create mode 100644 libcxx/include/__cxx03/cstdlib
 create mode 100644 libcxx/include/__cxx03/cstring
 create mode 100644 libcxx/include/__cxx03/ctgmath
 create mode 100644 libcxx/include/__cxx03/ctime
 create mode 100644 libcxx/include/__cxx03/ctype.h
 create mode 100644 libcxx/include/__cxx03/cuchar
 create mode 100644 libcxx/include/__cxx03/cwchar
 create mode 100644 libcxx/include/__cxx03/cwctype
 create mode 100644 libcxx/include/__cxx03/deque
 create mode 100644 libcxx/include/__cxx03/errno.h
 create mode 100644 libcxx/include/__cxx03/exception
 create mode 100644 libcxx/include/__cxx03/execution
 create mode 100644 libcxx/include/__cxx03/expected
 create mode 100644 libcxx/include/__cxx03/experimental/__config
 create mode 100644 libcxx/include/__cxx03/experimental/__simd/aligned_tag.h
 create mode 100644 libcxx/include/__cxx03/experimental/__simd/declaration.h
 create mode 100644 libcxx/include/__cxx03/experimental/__simd/reference.h
 create mode 100644 libcxx/include/__cxx03/experimental/__simd/scalar.h
 create mode 100644 libcxx/include/__cxx03/experimental/__simd/simd.h
 create mode 100644 libcxx/include/__cxx03/experimental/__simd/simd_mask.h
 create mode 100644 libcxx/include/__cxx03/experimental/__simd/traits.h
 create mode 100644 libcxx/include/__cxx03/experimental/__simd/utility.h
 create mode 100644 libcxx/include/__cxx03/experimental/__simd/vec_ext.h
 create mode 100644 libcxx/include/__cxx03/experimental/iterator
 create mode 100644 libcxx/include/__cxx03/experimental/memory
 create mode 100644 libcxx/include/__cxx03/experimental/propagate_const
 create mode 100644 libcxx/include/__cxx03/experimental/simd
 create mode 100644 libcxx/include/__cxx03/experimental/type_traits
 create mode 100644 libcxx/include/__cxx03/experimental/utility
 create mode 100644 libcxx/include/__cxx03/ext/__hash
 create mode 100644 libcxx/include/__cxx03/ext/hash_map
 create mode 100644 libcxx/include/__cxx03/ext/hash_set
 create mode 100644 libcxx/include/__cxx03/fenv.h
 create mode 100644 libcxx/include/__cxx03/filesystem
 create mode 100644 libcxx/include/__cxx03/float.h
 create mode 100644 libcxx/include/__cxx03/format
 create mode 100644 libcxx/include/__cxx03/forward_list
 create mode 100644 libcxx/include/__cxx03/fstream
 create mode 100644 libcxx/include/__cxx03/functional
 create mode 100644 libcxx/include/__cxx03/future
 create mode 100644 libcxx/include/__cxx03/initializer_list
 create mode 100644 libcxx/include/__cxx03/inttypes.h
 create mode 100644 libcxx/include/__cxx03/iomanip
 create mode 100644 libcxx/include/__cxx03/ios
 create mode 100644 libcxx/include/__cxx03/iosfwd
 create mode 100644 libcxx/include/__cxx03/iostream
 create mode 100644 libcxx/include/__cxx03/istream
 create mode 100644 libcxx/include/__cxx03/iterator
 create mode 100644 libcxx/include/__cxx03/latch
 create mode 100644 libcxx/include/__cxx03/limits
 create mode 100644 libcxx/include/__cxx03/list
 create mode 100644 libcxx/include/__cxx03/locale
 create mode 100644 libcxx/include/__cxx03/locale.h
 create mode 100644 libcxx/include/__cxx03/map
 create mode 100644 libcxx/include/__cxx03/math.h
 create mode 100644 libcxx/include/__cxx03/mdspan
 create mode 100644 libcxx/include/__cxx03/memory
 create mode 100644 libcxx/include/__cxx03/memory_resource
 create mode 100644 libcxx/include/__cxx03/module.modulemap
 create mode 100644 libcxx/include/__cxx03/mutex
 create mode 100644 libcxx/include/__cxx03/new
 create mode 100644 libcxx/include/__cxx03/numbers
 create mode 100644 libcxx/include/__cxx03/numeric
 create mode 100644 libcxx/include/__cxx03/optional
 create mode 100644 libcxx/include/__cxx03/ostream
 create mode 100644 libcxx/include/__cxx03/print
 create mode 100644 libcxx/include/__cxx03/queue
 create mode 100644 libcxx/include/__cxx03/random
 create mode 100644 libcxx/include/__cxx03/ranges
 create mode 100644 libcxx/include/__cxx03/ratio
 create mode 100644 libcxx/include/__cxx03/regex
 create mode 100644 libcxx/include/__cxx03/scoped_allocator
 create mode 100644 libcxx/include/__cxx03/semaphore
 create mode 100644 libcxx/include/__cxx03/set
 create mode 100644 libcxx/include/__cxx03/shared_mutex
 create mode 100644 libcxx/include/__cxx03/source_location
 create mode 100644 libcxx/include/__cxx03/span
 create mode 100644 libcxx/include/__cxx03/sstream
 create mode 100644 libcxx/include/__cxx03/stack
 create mode 100644 libcxx/include/__cxx03/stdatomic.h
 create mode 100644 libcxx/include/__cxx03/stdbool.h
 create mode 100644 libcxx/include/__cxx03/stddef.h
 create mode 100644 libcxx/include/__cxx03/stdexcept
 create mode 100644 libcxx/include/__cxx03/stdint.h
 create mode 100644 libcxx/include/__cxx03/stdio.h
 create mode 100644 libcxx/include/__cxx03/stdlib.h
 create mode 100644 libcxx/include/__cxx03/stop_token
 create mode 100644 libcxx/include/__cxx03/streambuf
 create mode 100644 libcxx/include/__cxx03/string
 create mode 100644 libcxx/include/__cxx03/string.h
 create mode 100644 libcxx/include/__cxx03/string_view
 create mode 100644 libcxx/include/__cxx03/strstream
 create mode 100644 libcxx/include/__cxx03/syncstream
 create mode 100644 libcxx/include/__cxx03/system_error
 create mode 100644 libcxx/include/__cxx03/tgmath.h
 create mode 100644 libcxx/include/__cxx03/thread
 create mode 100644 libcxx/include/__cxx03/tuple
 create mode 100644 libcxx/include/__cxx03/type_traits
 create mode 100644 libcxx/include/__cxx03/typeindex
 create mode 100644 libcxx/include/__cxx03/typeinfo
 create mode 100644 libcxx/include/__cxx03/uchar.h
 create mode 100644 libcxx/include/__cxx03/unordered_map
 create mode 100644 libcxx/include/__cxx03/unordered_set
 create mode 100644 libcxx/include/__cxx03/utility
 create mode 100644 libcxx/include/__cxx03/valarray
 create mode 100644 libcxx/include/__cxx03/variant
 create mode 100644 libcxx/include/__cxx03/vector
 create mode 100644 libcxx/include/__cxx03/version
 create mode 100644 libcxx/include/__cxx03/wchar.h
 create mode 100644 libcxx/include/__cxx03/wctype.h

diff --git a/libcxx/include/__cxx03/CMakeLists.txt b/libcxx/include/__cxx03/CMakeLists.txt
new file mode 100644
index 000000000000..32579272858a
--- /dev/null
+++ b/libcxx/include/__cxx03/CMakeLists.txt
@@ -0,0 +1,1092 @@
+set(files
+  __algorithm/adjacent_find.h
+  __algorithm/all_of.h
+  __algorithm/any_of.h
+  __algorithm/binary_search.h
+  __algorithm/clamp.h
+  __algorithm/comp.h
+  __algorithm/comp_ref_type.h
+  __algorithm/copy.h
+  __algorithm/copy_backward.h
+  __algorithm/copy_if.h
+  __algorithm/copy_move_common.h
+  __algorithm/copy_n.h
+  __algorithm/count.h
+  __algorithm/count_if.h
+  __algorithm/equal.h
+  __algorithm/equal_range.h
+  __algorithm/fill.h
+  __algorithm/fill_n.h
+  __algorithm/find.h
+  __algorithm/find_end.h
+  __algorithm/find_first_of.h
+  __algorithm/find_if.h
+  __algorithm/find_if_not.h
+  __algorithm/find_segment_if.h
+  __algorithm/fold.h
+  __algorithm/for_each.h
+  __algorithm/for_each_n.h
+  __algorithm/for_each_segment.h
+  __algorithm/generate.h
+  __algorithm/generate_n.h
+  __algorithm/half_positive.h
+  __algorithm/in_found_result.h
+  __algorithm/in_fun_result.h
+  __algorithm/in_in_out_result.h
+  __algorithm/in_in_result.h
+  __algorithm/in_out_out_result.h
+  __algorithm/in_out_result.h
+  __algorithm/includes.h
+  __algorithm/inplace_merge.h
+  __algorithm/is_heap.h
+  __algorithm/is_heap_until.h
+  __algorithm/is_partitioned.h
+  __algorithm/is_permutation.h
+  __algorithm/is_sorted.h
+  __algorithm/is_sorted_until.h
+  __algorithm/iter_swap.h
+  __algorithm/iterator_operations.h
+  __algorithm/lexicographical_compare.h
+  __algorithm/lexicographical_compare_three_way.h
+  __algorithm/lower_bound.h
+  __algorithm/make_heap.h
+  __algorithm/make_projected.h
+  __algorithm/max.h
+  __algorithm/max_element.h
+  __algorithm/merge.h
+  __algorithm/min.h
+  __algorithm/min_element.h
+  __algorithm/min_max_result.h
+  __algorithm/minmax.h
+  __algorithm/minmax_element.h
+  __algorithm/mismatch.h
+  __algorithm/move.h
+  __algorithm/move_backward.h
+  __algorithm/next_permutation.h
+  __algorithm/none_of.h
+  __algorithm/nth_element.h
+  __algorithm/partial_sort.h
+  __algorithm/partial_sort_copy.h
+  __algorithm/partition.h
+  __algorithm/partition_copy.h
+  __algorithm/partition_point.h
+  __algorithm/pop_heap.h
+  __algorithm/prev_permutation.h
+  __algorithm/pstl.h
+  __algorithm/push_heap.h
+  __algorithm/ranges_adjacent_find.h
+  __algorithm/ranges_all_of.h
+  __algorithm/ranges_any_of.h
+  __algorithm/ranges_binary_search.h
+  __algorithm/ranges_clamp.h
+  __algorithm/ranges_contains.h
+  __algorithm/ranges_contains_subrange.h
+  __algorithm/ranges_copy.h
+  __algorithm/ranges_copy_backward.h
+  __algorithm/ranges_copy_if.h
+  __algorithm/ranges_copy_n.h
+  __algorithm/ranges_count.h
+  __algorithm/ranges_count_if.h
+  __algorithm/ranges_ends_with.h
+  __algorithm/ranges_equal.h
+  __algorithm/ranges_equal_range.h
+  __algorithm/ranges_fill.h
+  __algorithm/ranges_fill_n.h
+  __algorithm/ranges_find.h
+  __algorithm/ranges_find_end.h
+  __algorithm/ranges_find_first_of.h
+  __algorithm/ranges_find_if.h
+  __algorithm/ranges_find_if_not.h
+  __algorithm/ranges_find_last.h
+  __algorithm/ranges_for_each.h
+  __algorithm/ranges_for_each_n.h
+  __algorithm/ranges_generate.h
+  __algorithm/ranges_generate_n.h
+  __algorithm/ranges_includes.h
+  __algorithm/ranges_inplace_merge.h
+  __algorithm/ranges_is_heap.h
+  __algorithm/ranges_is_heap_until.h
+  __algorithm/ranges_is_partitioned.h
+  __algorithm/ranges_is_permutation.h
+  __algorithm/ranges_is_sorted.h
+  __algorithm/ranges_is_sorted_until.h
+  __algorithm/ranges_iterator_concept.h
+  __algorithm/ranges_lexicographical_compare.h
+  __algorithm/ranges_lower_bound.h
+  __algorithm/ranges_make_heap.h
+  __algorithm/ranges_max.h
+  __algorithm/ranges_max_element.h
+  __algorithm/ranges_merge.h
+  __algorithm/ranges_min.h
+  __algorithm/ranges_min_element.h
+  __algorithm/ranges_minmax.h
+  __algorithm/ranges_minmax_element.h
+  __algorithm/ranges_mismatch.h
+  __algorithm/ranges_move.h
+  __algorithm/ranges_move_backward.h
+  __algorithm/ranges_next_permutation.h
+  __algorithm/ranges_none_of.h
+  __algorithm/ranges_nth_element.h
+  __algorithm/ranges_partial_sort.h
+  __algorithm/ranges_partial_sort_copy.h
+  __algorithm/ranges_partition.h
+  __algorithm/ranges_partition_copy.h
+  __algorithm/ranges_partition_point.h
+  __algorithm/ranges_pop_heap.h
+  __algorithm/ranges_prev_permutation.h
+  __algorithm/ranges_push_heap.h
+  __algorithm/ranges_remove.h
+  __algorithm/ranges_remove_copy.h
+  __algorithm/ranges_remove_copy_if.h
+  __algorithm/ranges_remove_if.h
+  __algorithm/ranges_replace.h
+  __algorithm/ranges_replace_copy.h
+  __algorithm/ranges_replace_copy_if.h
+  __algorithm/ranges_replace_if.h
+  __algorithm/ranges_reverse.h
+  __algorithm/ranges_reverse_copy.h
+  __algorithm/ranges_rotate.h
+  __algorithm/ranges_rotate_copy.h
+  __algorithm/ranges_sample.h
+  __algorithm/ranges_search.h
+  __algorithm/ranges_search_n.h
+  __algorithm/ranges_set_difference.h
+  __algorithm/ranges_set_intersection.h
+  __algorithm/ranges_set_symmetric_difference.h
+  __algorithm/ranges_set_union.h
+  __algorithm/ranges_shuffle.h
+  __algorithm/ranges_sort.h
+  __algorithm/ranges_sort_heap.h
+  __algorithm/ranges_stable_partition.h
+  __algorithm/ranges_stable_sort.h
+  __algorithm/ranges_starts_with.h
+  __algorithm/ranges_swap_ranges.h
+  __algorithm/ranges_transform.h
+  __algorithm/ranges_unique.h
+  __algorithm/ranges_unique_copy.h
+  __algorithm/ranges_upper_bound.h
+  __algorithm/remove.h
+  __algorithm/remove_copy.h
+  __algorithm/remove_copy_if.h
+  __algorithm/remove_if.h
+  __algorithm/replace.h
+  __algorithm/replace_copy.h
+  __algorithm/replace_copy_if.h
+  __algorithm/replace_if.h
+  __algorithm/reverse.h
+  __algorithm/reverse_copy.h
+  __algorithm/rotate.h
+  __algorithm/rotate_copy.h
+  __algorithm/sample.h
+  __algorithm/search.h
+  __algorithm/search_n.h
+  __algorithm/set_difference.h
+  __algorithm/set_intersection.h
+  __algorithm/set_symmetric_difference.h
+  __algorithm/set_union.h
+  __algorithm/shift_left.h
+  __algorithm/shift_right.h
+  __algorithm/shuffle.h
+  __algorithm/sift_down.h
+  __algorithm/simd_utils.h
+  __algorithm/sort.h
+  __algorithm/sort_heap.h
+  __algorithm/stable_partition.h
+  __algorithm/stable_sort.h
+  __algorithm/swap_ranges.h
+  __algorithm/three_way_comp_ref_type.h
+  __algorithm/transform.h
+  __algorithm/uniform_random_bit_generator_adaptor.h
+  __algorithm/unique.h
+  __algorithm/unique_copy.h
+  __algorithm/unwrap_iter.h
+  __algorithm/unwrap_range.h
+  __algorithm/upper_bound.h
+  __assert
+  __atomic/aliases.h
+  __atomic/atomic.h
+  __atomic/atomic_base.h
+  __atomic/atomic_flag.h
+  __atomic/atomic_init.h
+  __atomic/atomic_lock_free.h
+  __atomic/atomic_ref.h
+  __atomic/atomic_sync.h
+  __atomic/check_memory_order.h
+  __atomic/contention_t.h
+  __atomic/cxx_atomic_impl.h
+  __atomic/fence.h
+  __atomic/is_always_lock_free.h
+  __atomic/kill_dependency.h
+  __atomic/memory_order.h
+  __atomic/to_gcc_order.h
+  __bit/bit_cast.h
+  __bit/bit_ceil.h
+  __bit/bit_floor.h
+  __bit/bit_log2.h
+  __bit/bit_width.h
+  __bit/blsr.h
+  __bit/byteswap.h
+  __bit/countl.h
+  __bit/countr.h
+  __bit/endian.h
+  __bit/has_single_bit.h
+  __bit/invert_if.h
+  __bit/popcount.h
+  __bit/rotate.h
+  __bit_reference
+  __charconv/chars_format.h
+  __charconv/from_chars_integral.h
+  __charconv/from_chars_result.h
+  __charconv/tables.h
+  __charconv/to_chars.h
+  __charconv/to_chars_base_10.h
+  __charconv/to_chars_floating_point.h
+  __charconv/to_chars_integral.h
+  __charconv/to_chars_result.h
+  __charconv/traits.h
+  __chrono/calendar.h
+  __chrono/concepts.h
+  __chrono/convert_to_timespec.h
+  __chrono/convert_to_tm.h
+  __chrono/day.h
+  __chrono/duration.h
+  __chrono/exception.h
+  __chrono/file_clock.h
+  __chrono/formatter.h
+  __chrono/hh_mm_ss.h
+  __chrono/high_resolution_clock.h
+  __chrono/leap_second.h
+  __chrono/literals.h
+  __chrono/local_info.h
+  __chrono/month.h
+  __chrono/month_weekday.h
+  __chrono/monthday.h
+  __chrono/ostream.h
+  __chrono/parser_std_format_spec.h
+  __chrono/statically_widen.h
+  __chrono/steady_clock.h
+  __chrono/sys_info.h
+  __chrono/system_clock.h
+  __chrono/time_point.h
+  __chrono/time_zone.h
+  __chrono/time_zone_link.h
+  __chrono/tzdb.h
+  __chrono/tzdb_list.h
+  __chrono/weekday.h
+  __chrono/year.h
+  __chrono/year_month.h
+  __chrono/year_month_day.h
+  __chrono/year_month_weekday.h
+  __chrono/zoned_time.h
+  __compare/common_comparison_category.h
+  __compare/compare_partial_order_fallback.h
+  __compare/compare_strong_order_fallback.h
+  __compare/compare_three_way.h
+  __compare/compare_three_way_result.h
+  __compare/compare_weak_order_fallback.h
+  __compare/is_eq.h
+  __compare/ordering.h
+  __compare/partial_order.h
+  __compare/strong_order.h
+  __compare/synth_three_way.h
+  __compare/three_way_comparable.h
+  __compare/weak_order.h
+  __concepts/arithmetic.h
+  __concepts/assignable.h
+  __concepts/boolean_testable.h
+  __concepts/class_or_enum.h
+  __concepts/common_reference_with.h
+  __concepts/common_with.h
+  __concepts/constructible.h
+  __concepts/convertible_to.h
+  __concepts/copyable.h
+  __concepts/derived_from.h
+  __concepts/destructible.h
+  __concepts/different_from.h
+  __concepts/equality_comparable.h
+  __concepts/invocable.h
+  __concepts/movable.h
+  __concepts/predicate.h
+  __concepts/regular.h
+  __concepts/relation.h
+  __concepts/same_as.h
+  __concepts/semiregular.h
+  __concepts/swappable.h
+  __concepts/totally_ordered.h
+  __condition_variable/condition_variable.h
+  __config
+  __configuration/abi.h
+  __configuration/availability.h
+  __configuration/compiler.h
+  __configuration/language.h
+  __configuration/platform.h
+  __coroutine/coroutine_handle.h
+  __coroutine/coroutine_traits.h
+  __coroutine/noop_coroutine_handle.h
+  __coroutine/trivial_awaitables.h
+  __debug_utils/randomize_range.h
+  __debug_utils/sanitizers.h
+  __debug_utils/strict_weak_ordering_check.h
+  __exception/exception.h
+  __exception/exception_ptr.h
+  __exception/nested_exception.h
+  __exception/operations.h
+  __exception/terminate.h
+  __expected/bad_expected_access.h
+  __expected/expected.h
+  __expected/unexpect.h
+  __expected/unexpected.h
+  __filesystem/copy_options.h
+  __filesystem/directory_entry.h
+  __filesystem/directory_iterator.h
+  __filesystem/directory_options.h
+  __filesystem/file_status.h
+  __filesystem/file_time_type.h
+  __filesystem/file_type.h
+  __filesystem/filesystem_error.h
+  __filesystem/operations.h
+  __filesystem/path.h
+  __filesystem/path_iterator.h
+  __filesystem/perm_options.h
+  __filesystem/perms.h
+  __filesystem/recursive_directory_iterator.h
+  __filesystem/space_info.h
+  __filesystem/u8path.h
+  __format/buffer.h
+  __format/concepts.h
+  __format/container_adaptor.h
+  __format/enable_insertable.h
+  __format/escaped_output_table.h
+  __format/extended_grapheme_cluster_table.h
+  __format/format_arg.h
+  __format/format_arg_store.h
+  __format/format_args.h
+  __format/format_context.h
+  __format/format_error.h
+  __format/format_functions.h
+  __format/format_parse_context.h
+  __format/format_string.h
+  __format/format_to_n_result.h
+  __format/formatter.h
+  __format/formatter_bool.h
+  __format/formatter_char.h
+  __format/formatter_floating_point.h
+  __format/formatter_integer.h
+  __format/formatter_integral.h
+  __format/formatter_output.h
+  __format/formatter_pointer.h
+  __format/formatter_string.h
+  __format/formatter_tuple.h
+  __format/indic_conjunct_break_table.h
+  __format/parser_std_format_spec.h
+  __format/range_default_formatter.h
+  __format/range_formatter.h
+  __format/unicode.h
+  __format/width_estimation_table.h
+  __format/write_escaped.h
+  __functional/binary_function.h
+  __functional/binary_negate.h
+  __functional/bind.h
+  __functional/bind_back.h
+  __functional/bind_front.h
+  __functional/binder1st.h
+  __functional/binder2nd.h
+  __functional/boyer_moore_searcher.h
+  __functional/compose.h
+  __functional/default_searcher.h
+  __functional/function.h
+  __functional/hash.h
+  __functional/identity.h
+  __functional/invoke.h
+  __functional/is_transparent.h
+  __functional/mem_fn.h
+  __functional/mem_fun_ref.h
+  __functional/not_fn.h
+  __functional/operations.h
+  __functional/perfect_forward.h
+  __functional/pointer_to_binary_function.h
+  __functional/pointer_to_unary_function.h
+  __functional/ranges_operations.h
+  __functional/reference_wrapper.h
+  __functional/unary_function.h
+  __functional/unary_negate.h
+  __functional/weak_result_type.h
+  __fwd/array.h
+  __fwd/bit_reference.h
+  __fwd/complex.h
+  __fwd/deque.h
+  __fwd/format.h
+  __fwd/fstream.h
+  __fwd/functional.h
+  __fwd/ios.h
+  __fwd/istream.h
+  __fwd/mdspan.h
+  __fwd/memory.h
+  __fwd/memory_resource.h
+  __fwd/ostream.h
+  __fwd/pair.h
+  __fwd/queue.h
+  __fwd/span.h
+  __fwd/sstream.h
+  __fwd/stack.h
+  __fwd/streambuf.h
+  __fwd/string.h
+  __fwd/string_view.h
+  __fwd/subrange.h
+  __fwd/tuple.h
+  __fwd/vector.h
+  __hash_table
+  __ios/fpos.h
+  __iterator/access.h
+  __iterator/advance.h
+  __iterator/aliasing_iterator.h
+  __iterator/back_insert_iterator.h
+  __iterator/bounded_iter.h
+  __iterator/common_iterator.h
+  __iterator/concepts.h
+  __iterator/counted_iterator.h
+  __iterator/cpp17_iterator_concepts.h
+  __iterator/data.h
+  __iterator/default_sentinel.h
+  __iterator/distance.h
+  __iterator/empty.h
+  __iterator/erase_if_container.h
+  __iterator/front_insert_iterator.h
+  __iterator/incrementable_traits.h
+  __iterator/indirectly_comparable.h
+  __iterator/insert_iterator.h
+  __iterator/istream_iterator.h
+  __iterator/istreambuf_iterator.h
+  __iterator/iter_move.h
+  __iterator/iter_swap.h
+  __iterator/iterator.h
+  __iterator/iterator_traits.h
+  __iterator/iterator_with_data.h
+  __iterator/mergeable.h
+  __iterator/move_iterator.h
+  __iterator/move_sentinel.h
+  __iterator/next.h
+  __iterator/ostream_iterator.h
+  __iterator/ostreambuf_iterator.h
+  __iterator/permutable.h
+  __iterator/prev.h
+  __iterator/projected.h
+  __iterator/ranges_iterator_traits.h
+  __iterator/readable_traits.h
+  __iterator/reverse_access.h
+  __iterator/reverse_iterator.h
+  __iterator/segmented_iterator.h
+  __iterator/size.h
+  __iterator/sortable.h
+  __iterator/unreachable_sentinel.h
+  __iterator/wrap_iter.h
+  __locale
+  __locale_dir/locale_base_api.h
+  __locale_dir/locale_base_api/android.h
+  __locale_dir/locale_base_api/bsd_locale_defaults.h
+  __locale_dir/locale_base_api/bsd_locale_fallbacks.h
+  __locale_dir/locale_base_api/fuchsia.h
+  __locale_dir/locale_base_api/ibm.h
+  __locale_dir/locale_base_api/locale_guard.h
+  __locale_dir/locale_base_api/musl.h
+  __locale_dir/locale_base_api/newlib.h
+  __locale_dir/locale_base_api/openbsd.h
+  __locale_dir/locale_base_api/win32.h
+  __math/abs.h
+  __math/copysign.h
+  __math/error_functions.h
+  __math/exponential_functions.h
+  __math/fdim.h
+  __math/fma.h
+  __math/gamma.h
+  __math/hyperbolic_functions.h
+  __math/hypot.h
+  __math/inverse_hyperbolic_functions.h
+  __math/inverse_trigonometric_functions.h
+  __math/logarithms.h
+  __math/min_max.h
+  __math/modulo.h
+  __math/remainder.h
+  __math/roots.h
+  __math/rounding_functions.h
+  __math/special_functions.h
+  __math/traits.h
+  __math/trigonometric_functions.h
+  __mbstate_t.h
+  __mdspan/default_accessor.h
+  __mdspan/extents.h
+  __mdspan/layout_left.h
+  __mdspan/layout_right.h
+  __mdspan/layout_stride.h
+  __mdspan/mdspan.h
+  __memory/addressof.h
+  __memory/align.h
+  __memory/aligned_alloc.h
+  __memory/allocate_at_least.h
+  __memory/allocation_guard.h
+  __memory/allocator.h
+  __memory/allocator_arg_t.h
+  __memory/allocator_destructor.h
+  __memory/allocator_traits.h
+  __memory/assume_aligned.h
+  __memory/auto_ptr.h
+  __memory/builtin_new_allocator.h
+  __memory/compressed_pair.h
+  __memory/concepts.h
+  __memory/construct_at.h
+  __memory/destruct_n.h
+  __memory/inout_ptr.h
+  __memory/out_ptr.h
+  __memory/pointer_traits.h
+  __memory/ranges_construct_at.h
+  __memory/ranges_uninitialized_algorithms.h
+  __memory/raw_storage_iterator.h
+  __memory/shared_ptr.h
+  __memory/swap_allocator.h
+  __memory/temp_value.h
+  __memory/temporary_buffer.h
+  __memory/uninitialized_algorithms.h
+  __memory/unique_ptr.h
+  __memory/uses_allocator.h
+  __memory/uses_allocator_construction.h
+  __memory/voidify.h
+  __memory_resource/memory_resource.h
+  __memory_resource/monotonic_buffer_resource.h
+  __memory_resource/polymorphic_allocator.h
+  __memory_resource/pool_options.h
+  __memory_resource/synchronized_pool_resource.h
+  __memory_resource/unsynchronized_pool_resource.h
+  __mutex/lock_guard.h
+  __mutex/mutex.h
+  __mutex/once_flag.h
+  __mutex/tag_types.h
+  __mutex/unique_lock.h
+  __node_handle
+  __numeric/accumulate.h
+  __numeric/adjacent_difference.h
+  __numeric/exclusive_scan.h
+  __numeric/gcd_lcm.h
+  __numeric/inclusive_scan.h
+  __numeric/inner_product.h
+  __numeric/iota.h
+  __numeric/midpoint.h
+  __numeric/partial_sum.h
+  __numeric/pstl.h
+  __numeric/reduce.h
+  __numeric/saturation_arithmetic.h
+  __numeric/transform_exclusive_scan.h
+  __numeric/transform_inclusive_scan.h
+  __numeric/transform_reduce.h
+  __ostream/basic_ostream.h
+  __ostream/print.h
+  __pstl/backend.h
+  __pstl/backend_fwd.h
+  __pstl/backends/default.h
+  __pstl/backends/libdispatch.h
+  __pstl/backends/serial.h
+  __pstl/backends/std_thread.h
+  __pstl/cpu_algos/any_of.h
+  __pstl/cpu_algos/cpu_traits.h
+  __pstl/cpu_algos/fill.h
+  __pstl/cpu_algos/find_if.h
+  __pstl/cpu_algos/for_each.h
+  __pstl/cpu_algos/merge.h
+  __pstl/cpu_algos/stable_sort.h
+  __pstl/cpu_algos/transform.h
+  __pstl/cpu_algos/transform_reduce.h
+  __pstl/dispatch.h
+  __pstl/handle_exception.h
+  __random/bernoulli_distribution.h
+  __random/binomial_distribution.h
+  __random/cauchy_distribution.h
+  __random/chi_squared_distribution.h
+  __random/clamp_to_integral.h
+  __random/default_random_engine.h
+  __random/discard_block_engine.h
+  __random/discrete_distribution.h
+  __random/exponential_distribution.h
+  __random/extreme_value_distribution.h
+  __random/fisher_f_distribution.h
+  __random/gamma_distribution.h
+  __random/generate_canonical.h
+  __random/geometric_distribution.h
+  __random/independent_bits_engine.h
+  __random/is_seed_sequence.h
+  __random/is_valid.h
+  __random/knuth_b.h
+  __random/linear_congruential_engine.h
+  __random/log2.h
+  __random/lognormal_distribution.h
+  __random/mersenne_twister_engine.h
+  __random/negative_binomial_distribution.h
+  __random/normal_distribution.h
+  __random/piecewise_constant_distribution.h
+  __random/piecewise_linear_distribution.h
+  __random/poisson_distribution.h
+  __random/random_device.h
+  __random/ranlux.h
+  __random/seed_seq.h
+  __random/shuffle_order_engine.h
+  __random/student_t_distribution.h
+  __random/subtract_with_carry_engine.h
+  __random/uniform_int_distribution.h
+  __random/uniform_random_bit_generator.h
+  __random/uniform_real_distribution.h
+  __random/weibull_distribution.h
+  __ranges/access.h
+  __ranges/all.h
+  __ranges/as_rvalue_view.h
+  __ranges/chunk_by_view.h
+  __ranges/common_view.h
+  __ranges/concepts.h
+  __ranges/container_compatible_range.h
+  __ranges/counted.h
+  __ranges/dangling.h
+  __ranges/data.h
+  __ranges/drop_view.h
+  __ranges/drop_while_view.h
+  __ranges/elements_view.h
+  __ranges/empty.h
+  __ranges/empty_view.h
+  __ranges/enable_borrowed_range.h
+  __ranges/enable_view.h
+  __ranges/filter_view.h
+  __ranges/from_range.h
+  __ranges/iota_view.h
+  __ranges/istream_view.h
+  __ranges/join_view.h
+  __ranges/lazy_split_view.h
+  __ranges/movable_box.h
+  __ranges/non_propagating_cache.h
+  __ranges/owning_view.h
+  __ranges/range_adaptor.h
+  __ranges/rbegin.h
+  __ranges/ref_view.h
+  __ranges/rend.h
+  __ranges/repeat_view.h
+  __ranges/reverse_view.h
+  __ranges/single_view.h
+  __ranges/size.h
+  __ranges/split_view.h
+  __ranges/subrange.h
+  __ranges/take_view.h
+  __ranges/take_while_view.h
+  __ranges/to.h
+  __ranges/transform_view.h
+  __ranges/view_interface.h
+  __ranges/views.h
+  __ranges/zip_view.h
+  __split_buffer
+  __std_clang_module
+  __std_mbstate_t.h
+  __stop_token/atomic_unique_lock.h
+  __stop_token/intrusive_list_view.h
+  __stop_token/intrusive_shared_ptr.h
+  __stop_token/stop_callback.h
+  __stop_token/stop_source.h
+  __stop_token/stop_state.h
+  __stop_token/stop_token.h
+  __string/char_traits.h
+  __string/constexpr_c_functions.h
+  __string/extern_template_lists.h
+  __support/ibm/gettod_zos.h
+  __support/ibm/locale_mgmt_zos.h
+  __support/ibm/nanosleep.h
+  __support/xlocale/__nop_locale_mgmt.h
+  __support/xlocale/__posix_l_fallback.h
+  __support/xlocale/__strtonum_fallback.h
+  __system_error/errc.h
+  __system_error/error_category.h
+  __system_error/error_code.h
+  __system_error/error_condition.h
+  __system_error/system_error.h
+  __thread/formatter.h
+  __thread/id.h
+  __thread/jthread.h
+  __thread/poll_with_backoff.h
+  __thread/support.h
+  __thread/support/c11.h
+  __thread/support/external.h
+  __thread/support/pthread.h
+  __thread/support/windows.h
+  __thread/this_thread.h
+  __thread/thread.h
+  __thread/timed_backoff_policy.h
+  __tree
+  __tuple/find_index.h
+  __tuple/ignore.h
+  __tuple/make_tuple_types.h
+  __tuple/sfinae_helpers.h
+  __tuple/tuple_element.h
+  __tuple/tuple_indices.h
+  __tuple/tuple_like.h
+  __tuple/tuple_like_ext.h
+  __tuple/tuple_like_no_subrange.h
+  __tuple/tuple_size.h
+  __tuple/tuple_types.h
+  __type_traits/add_const.h
+  __type_traits/add_cv.h
+  __type_traits/add_lvalue_reference.h
+  __type_traits/add_pointer.h
+  __type_traits/add_rvalue_reference.h
+  __type_traits/add_volatile.h
+  __type_traits/aligned_storage.h
+  __type_traits/aligned_union.h
+  __type_traits/alignment_of.h
+  __type_traits/can_extract_key.h
+  __type_traits/common_reference.h
+  __type_traits/common_type.h
+  __type_traits/conditional.h
+  __type_traits/conjunction.h
+  __type_traits/copy_cv.h
+  __type_traits/copy_cvref.h
+  __type_traits/datasizeof.h
+  __type_traits/decay.h
+  __type_traits/dependent_type.h
+  __type_traits/desugars_to.h
+  __type_traits/disjunction.h
+  __type_traits/enable_if.h
+  __type_traits/extent.h
+  __type_traits/has_unique_object_representation.h
+  __type_traits/has_virtual_destructor.h
+  __type_traits/integral_constant.h
+  __type_traits/invoke.h
+  __type_traits/is_abstract.h
+  __type_traits/is_aggregate.h
+  __type_traits/is_allocator.h
+  __type_traits/is_always_bitcastable.h
+  __type_traits/is_arithmetic.h
+  __type_traits/is_array.h
+  __type_traits/is_assignable.h
+  __type_traits/is_base_of.h
+  __type_traits/is_bounded_array.h
+  __type_traits/is_callable.h
+  __type_traits/is_char_like_type.h
+  __type_traits/is_class.h
+  __type_traits/is_compound.h
+  __type_traits/is_const.h
+  __type_traits/is_constant_evaluated.h
+  __type_traits/is_constructible.h
+  __type_traits/is_convertible.h
+  __type_traits/is_core_convertible.h
+  __type_traits/is_destructible.h
+  __type_traits/is_empty.h
+  __type_traits/is_enum.h
+  __type_traits/is_equality_comparable.h
+  __type_traits/is_execution_policy.h
+  __type_traits/is_final.h
+  __type_traits/is_floating_point.h
+  __type_traits/is_function.h
+  __type_traits/is_fundamental.h
+  __type_traits/is_implicitly_default_constructible.h
+  __type_traits/is_integral.h
+  __type_traits/is_literal_type.h
+  __type_traits/is_member_pointer.h
+  __type_traits/is_nothrow_assignable.h
+  __type_traits/is_nothrow_constructible.h
+  __type_traits/is_nothrow_convertible.h
+  __type_traits/is_nothrow_destructible.h
+  __type_traits/is_null_pointer.h
+  __type_traits/is_object.h
+  __type_traits/is_pod.h
+  __type_traits/is_pointer.h
+  __type_traits/is_polymorphic.h
+  __type_traits/is_primary_template.h
+  __type_traits/is_reference.h
+  __type_traits/is_reference_wrapper.h
+  __type_traits/is_referenceable.h
+  __type_traits/is_same.h
+  __type_traits/is_scalar.h
+  __type_traits/is_signed.h
+  __type_traits/is_signed_integer.h
+  __type_traits/is_specialization.h
+  __type_traits/is_standard_layout.h
+  __type_traits/is_swappable.h
+  __type_traits/is_trivial.h
+  __type_traits/is_trivially_assignable.h
+  __type_traits/is_trivially_constructible.h
+  __type_traits/is_trivially_copyable.h
+  __type_traits/is_trivially_destructible.h
+  __type_traits/is_trivially_lexicographically_comparable.h
+  __type_traits/is_trivially_relocatable.h
+  __type_traits/is_unbounded_array.h
+  __type_traits/is_union.h
+  __type_traits/is_unsigned.h
+  __type_traits/is_unsigned_integer.h
+  __type_traits/is_valid_expansion.h
+  __type_traits/is_void.h
+  __type_traits/is_volatile.h
+  __type_traits/lazy.h
+  __type_traits/make_32_64_or_128_bit.h
+  __type_traits/make_const_lvalue_ref.h
+  __type_traits/make_signed.h
+  __type_traits/make_unsigned.h
+  __type_traits/maybe_const.h
+  __type_traits/nat.h
+  __type_traits/negation.h
+  __type_traits/noexcept_move_assign_container.h
+  __type_traits/promote.h
+  __type_traits/rank.h
+  __type_traits/remove_all_extents.h
+  __type_traits/remove_const.h
+  __type_traits/remove_const_ref.h
+  __type_traits/remove_cv.h
+  __type_traits/remove_cvref.h
+  __type_traits/remove_extent.h
+  __type_traits/remove_pointer.h
+  __type_traits/remove_reference.h
+  __type_traits/remove_volatile.h
+  __type_traits/result_of.h
+  __type_traits/strip_signature.h
+  __type_traits/type_identity.h
+  __type_traits/type_list.h
+  __type_traits/underlying_type.h
+  __type_traits/unwrap_ref.h
+  __type_traits/void_t.h
+  __undef_macros
+  __utility/as_const.h
+  __utility/as_lvalue.h
+  __utility/auto_cast.h
+  __utility/cmp.h
+  __utility/convert_to_integral.h
+  __utility/declval.h
+  __utility/empty.h
+  __utility/exception_guard.h
+  __utility/exchange.h
+  __utility/forward.h
+  __utility/forward_like.h
+  __utility/in_place.h
+  __utility/integer_sequence.h
+  __utility/is_pointer_in_range.h
+  __utility/is_valid_range.h
+  __utility/move.h
+  __utility/no_destroy.h
+  __utility/pair.h
+  __utility/piecewise_construct.h
+  __utility/priority_tag.h
+  __utility/private_constructor_tag.h
+  __utility/rel_ops.h
+  __utility/small_buffer.h
+  __utility/swap.h
+  __utility/to_underlying.h
+  __utility/unreachable.h
+  __variant/monostate.h
+  __verbose_abort
+  algorithm
+  any
+  array
+  atomic
+  barrier
+  bit
+  bitset
+  cassert
+  ccomplex
+  cctype
+  cerrno
+  cfenv
+  cfloat
+  charconv
+  chrono
+  cinttypes
+  ciso646
+  climits
+  clocale
+  cmath
+  codecvt
+  compare
+  complex
+  complex.h
+  concepts
+  condition_variable
+  coroutine
+  csetjmp
+  csignal
+  cstdarg
+  cstdbool
+  cstddef
+  cstdint
+  cstdio
+  cstdlib
+  cstring
+  ctgmath
+  ctime
+  ctype.h
+  cuchar
+  cwchar
+  cwctype
+  deque
+  errno.h
+  exception
+  execution
+  expected
+  experimental/__config
+  experimental/__simd/aligned_tag.h
+  experimental/__simd/declaration.h
+  experimental/__simd/reference.h
+  experimental/__simd/scalar.h
+  experimental/__simd/simd.h
+  experimental/__simd/simd_mask.h
+  experimental/__simd/traits.h
+  experimental/__simd/utility.h
+  experimental/__simd/vec_ext.h
+  experimental/iterator
+  experimental/memory
+  experimental/propagate_const
+  experimental/simd
+  experimental/type_traits
+  experimental/utility
+  ext/__hash
+  ext/hash_map
+  ext/hash_set
+  fenv.h
+  filesystem
+  float.h
+  format
+  forward_list
+  fstream
+  functional
+  future
+  initializer_list
+  inttypes.h
+  iomanip
+  ios
+  iosfwd
+  iostream
+  istream
+  iterator
+  latch
+  limits
+  list
+  locale
+  locale.h
+  map
+  math.h
+  mdspan
+  memory
+  memory_resource
+  module.modulemap
+  mutex
+  new
+  numbers
+  numeric
+  optional
+  ostream
+  print
+  queue
+  random
+  ranges
+  ratio
+  regex
+  scoped_allocator
+  semaphore
+  set
+  shared_mutex
+  source_location
+  span
+  sstream
+  stack
+  stdatomic.h
+  stdbool.h
+  stddef.h
+  stdexcept
+  stdint.h
+  stdio.h
+  stdlib.h
+  stop_token
+  streambuf
+  string
+  string.h
+  string_view
+  strstream
+  syncstream
+  system_error
+  tgmath.h
+  thread
+  tuple
+  type_traits
+  typeindex
+  typeinfo
+  uchar.h
+  unordered_map
+  unordered_set
+  utility
+  valarray
+  variant
+  vector
+  version
+  wchar.h
+  wctype.h
+  )
+
+configure_file("__config_site.in" "${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}/__config_site" @ONLY)
+configure_file("${LIBCXX_ASSERTION_HANDLER_FILE}" "${LIBCXX_GENERATED_INCLUDE_DIR}/__assertion_handler" COPYONLY)
+
+set(_all_includes "${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}/__config_site"
+                  "${LIBCXX_GENERATED_INCLUDE_DIR}/__assertion_handler")
+foreach(f ${files})
+  set(src "${CMAKE_CURRENT_SOURCE_DIR}/${f}")
+  set(dst "${LIBCXX_GENERATED_INCLUDE_DIR}/${f}")
+  add_custom_command(OUTPUT ${dst}
+    DEPENDS ${src}
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different ${src} ${dst}
+    COMMENT "Copying CXX header ${f}")
+  list(APPEND _all_includes "${dst}")
+endforeach()
+
+# Generate the IWYU mapping. This depends on all header files but it's also considered as an
+# "include" for dependency tracking.
+add_custom_command(OUTPUT "${LIBCXX_GENERATED_INCLUDE_DIR}/libcxx.imp"
+  COMMAND "${Python3_EXECUTABLE}" "${LIBCXX_SOURCE_DIR}/utils/generate_iwyu_mapping.py" "-o" "${LIBCXX_GENERATED_INCLUDE_DIR}/libcxx.imp"
+  DEPENDS ${_all_includes}
+  COMMENT "Generate the mapping file for include-what-you-use"
+)
+list(APPEND _all_includes "${LIBCXX_GENERATED_INCLUDE_DIR}/libcxx.imp")
+
+add_custom_target(generate-cxx-headers ALL DEPENDS ${_all_includes})
+
+add_library(cxx-headers INTERFACE)
+target_link_libraries(cxx-headers INTERFACE libcxx-libc-headers libcxx-abi-headers)
+add_dependencies(cxx-headers generate-cxx-headers)
+# It's important that the arch directory be included first so that its header files
+# which interpose on the default include dir be included instead of the default ones.
+target_include_directories(cxx-headers INTERFACE ${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}
+                                                 ${LIBCXX_GENERATED_INCLUDE_DIR})
+
+if (LIBCXX_INSTALL_HEADERS)
+  foreach(file ${files})
+    get_filename_component(dir ${file} DIRECTORY)
+    install(FILES ${file}
+      DESTINATION "${LIBCXX_INSTALL_INCLUDE_DIR}/${dir}"
+      COMPONENT cxx-headers
+      PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
+    )
+  endforeach()
+
+  # Install the generated __config_site file to the per-target include dir.
+  install(FILES "${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}/__config_site"
+    DESTINATION "${LIBCXX_INSTALL_INCLUDE_TARGET_DIR}"
+    PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
+    COMPONENT cxx-headers)
+
+  # Install the generated __assertion_handler file to the generic include dir.
+  install(FILES "${LIBCXX_GENERATED_INCLUDE_DIR}/__assertion_handler"
+    DESTINATION "${LIBCXX_INSTALL_INCLUDE_DIR}"
+    PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
+    COMPONENT cxx-headers)
+
+  # Install the generated IWYU file to the generic include dir.
+  install(FILES "${LIBCXX_GENERATED_INCLUDE_DIR}/libcxx.imp"
+    DESTINATION "${LIBCXX_INSTALL_INCLUDE_DIR}"
+    PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
+    COMPONENT cxx-headers)
+
+  if (NOT CMAKE_CONFIGURATION_TYPES)
+    add_custom_target(install-cxx-headers
+                      DEPENDS cxx-headers
+                      COMMAND "${CMAKE_COMMAND}"
+                              -DCMAKE_INSTALL_COMPONENT=cxx-headers
+                              -P "${CMAKE_BINARY_DIR}/cmake_install.cmake")
+    # Stripping is a no-op for headers
+    add_custom_target(install-cxx-headers-stripped DEPENDS install-cxx-headers)
+  endif()
+endif()
diff --git a/libcxx/include/__cxx03/__algorithm/adjacent_find.h b/libcxx/include/__cxx03/__algorithm/adjacent_find.h
new file mode 100644
index 000000000000..6f15456e3a4d
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/adjacent_find.h
@@ -0,0 +1,58 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_ADJACENT_FIND_H
+#define _LIBCPP___ALGORITHM_ADJACENT_FIND_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Iter, class _Sent, class _BinaryPredicate>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Iter
+__adjacent_find(_Iter __first, _Sent __last, _BinaryPredicate&& __pred) {
+  if (__first == __last)
+    return __first;
+  _Iter __i = __first;
+  while (++__i != __last) {
+    if (__pred(*__first, *__i))
+      return __first;
+    __first = __i;
+  }
+  return __i;
+}
+
+template <class _ForwardIterator, class _BinaryPredicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+adjacent_find(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred) {
+  return std::__adjacent_find(std::move(__first), std::move(__last), __pred);
+}
+
+template <class _ForwardIterator>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+adjacent_find(_ForwardIterator __first, _ForwardIterator __last) {
+  return std::adjacent_find(std::move(__first), std::move(__last), __equal_to());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_ADJACENT_FIND_H
diff --git a/libcxx/include/__cxx03/__algorithm/all_of.h b/libcxx/include/__cxx03/__algorithm/all_of.h
new file mode 100644
index 000000000000..ec84eea75929
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/all_of.h
@@ -0,0 +1,32 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_ALL_OF_H
+#define _LIBCPP___ALGORITHM_ALL_OF_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _Predicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+all_of(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
+  for (; __first != __last; ++__first)
+    if (!__pred(*__first))
+      return false;
+  return true;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_ALL_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/any_of.h b/libcxx/include/__cxx03/__algorithm/any_of.h
new file mode 100644
index 000000000000..b5ff778c4171
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/any_of.h
@@ -0,0 +1,32 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_ANY_OF_H
+#define _LIBCPP___ALGORITHM_ANY_OF_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _Predicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+any_of(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
+  for (; __first != __last; ++__first)
+    if (__pred(*__first))
+      return true;
+  return false;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_ANY_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/binary_search.h b/libcxx/include/__cxx03/__algorithm/binary_search.h
new file mode 100644
index 000000000000..6065fc37274d
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/binary_search.h
@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_BINARY_SEARCH_H
+#define _LIBCPP___ALGORITHM_BINARY_SEARCH_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/lower_bound.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ForwardIterator, class _Tp, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+binary_search(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, _Compare __comp) {
+  __first = std::lower_bound<_ForwardIterator, _Tp, __comp_ref_type<_Compare> >(__first, __last, __value, __comp);
+  return __first != __last && !__comp(__value, *__first);
+}
+
+template <class _ForwardIterator, class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+binary_search(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
+  return std::binary_search(__first, __last, __value, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_BINARY_SEARCH_H
diff --git a/libcxx/include/__cxx03/__algorithm/clamp.h b/libcxx/include/__cxx03/__algorithm/clamp.h
new file mode 100644
index 000000000000..1a5a3d0744be
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/clamp.h
@@ -0,0 +1,44 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_CLAMP_H
+#define _LIBCPP___ALGORITHM_CLAMP_H
+
+#include <__algorithm/comp.h>
+#include <__assert>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp, class _Compare>
+[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI constexpr const _Tp&
+clamp(_LIBCPP_LIFETIMEBOUND const _Tp& __v,
+      _LIBCPP_LIFETIMEBOUND const _Tp& __lo,
+      _LIBCPP_LIFETIMEBOUND const _Tp& __hi,
+      _Compare __comp) {
+  _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(!__comp(__hi, __lo), "Bad bounds passed to std::clamp");
+  return __comp(__v, __lo) ? __lo : __comp(__hi, __v) ? __hi : __v;
+}
+
+template <class _Tp>
+[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI constexpr const _Tp&
+clamp(_LIBCPP_LIFETIMEBOUND const _Tp& __v,
+      _LIBCPP_LIFETIMEBOUND const _Tp& __lo,
+      _LIBCPP_LIFETIMEBOUND const _Tp& __hi) {
+  return std::clamp(__v, __lo, __hi, __less<>());
+}
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_CLAMP_H
diff --git a/libcxx/include/__cxx03/__algorithm/comp.h b/libcxx/include/__cxx03/__algorithm/comp.h
new file mode 100644
index 000000000000..a0fa88d6d2ac
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/comp.h
@@ -0,0 +1,49 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_COMP_H
+#define _LIBCPP___ALGORITHM_COMP_H
+
+#include <__config>
+#include <__type_traits/desugars_to.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct __equal_to {
+  template <class _T1, class _T2>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator()(const _T1& __x, const _T2& __y) const {
+    return __x == __y;
+  }
+};
+
+template <class _Tp, class _Up>
+inline const bool __desugars_to_v<__equal_tag, __equal_to, _Tp, _Up> = true;
+
+// The definition is required because __less is part of the ABI, but it's empty
+// because all comparisons should be transparent.
+template <class _T1 = void, class _T2 = _T1>
+struct __less {};
+
+template <>
+struct __less<void, void> {
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator()(const _Tp& __lhs, const _Up& __rhs) const {
+    return __lhs < __rhs;
+  }
+};
+
+template <class _Tp>
+inline const bool __desugars_to_v<__less_tag, __less<>, _Tp, _Tp> = true;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_COMP_H
diff --git a/libcxx/include/__cxx03/__algorithm/comp_ref_type.h b/libcxx/include/__cxx03/__algorithm/comp_ref_type.h
new file mode 100644
index 000000000000..c367fbb91ac2
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/comp_ref_type.h
@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_COMP_REF_TYPE_H
+#define _LIBCPP___ALGORITHM_COMP_REF_TYPE_H
+
+#include <__assert>
+#include <__config>
+#include <__utility/declval.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Compare>
+struct __debug_less {
+  _Compare& __comp_;
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI __debug_less(_Compare& __c) : __comp_(__c) {}
+
+  template <class _Tp, class _Up>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Up& __y) {
+    bool __r = __comp_(__x, __y);
+    if (__r)
+      __do_compare_assert(0, __y, __x);
+    return __r;
+  }
+
+  template <class _Tp, class _Up>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(_Tp& __x, _Up& __y) {
+    bool __r = __comp_(__x, __y);
+    if (__r)
+      __do_compare_assert(0, __y, __x);
+    return __r;
+  }
+
+  template <class _LHS, class _RHS>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 inline
+      _LIBCPP_HIDE_FROM_ABI decltype((void)std::declval<_Compare&>()(std::declval<_LHS&>(), std::declval<_RHS&>()))
+      __do_compare_assert(int, _LHS& __l, _RHS& __r) {
+    _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(!__comp_(__l, __r), "Comparator does not induce a strict weak ordering");
+    (void)__l;
+    (void)__r;
+  }
+
+  template <class _LHS, class _RHS>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 inline _LIBCPP_HIDE_FROM_ABI void __do_compare_assert(long, _LHS&, _RHS&) {}
+};
+
+// Pass the comparator by lvalue reference. Or in the debug mode, using a debugging wrapper that stores a reference.
+#if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG
+template <class _Comp>
+using __comp_ref_type = __debug_less<_Comp>;
+#else
+template <class _Comp>
+using __comp_ref_type = _Comp&;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_COMP_REF_TYPE_H
diff --git a/libcxx/include/__cxx03/__algorithm/copy.h b/libcxx/include/__cxx03/__algorithm/copy.h
new file mode 100644
index 000000000000..0890b895f540
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/copy.h
@@ -0,0 +1,123 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_COPY_H
+#define _LIBCPP___ALGORITHM_COPY_H
+
+#include <__algorithm/copy_move_common.h>
+#include <__algorithm/for_each_segment.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/min.h>
+#include <__config>
+#include <__iterator/segmented_iterator.h>
+#include <__type_traits/common_type.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class, class _InIter, class _Sent, class _OutIter>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter> __copy(_InIter, _Sent, _OutIter);
+
+template <class _AlgPolicy>
+struct __copy_impl {
+  template <class _InIter, class _Sent, class _OutIter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result) const {
+    while (__first != __last) {
+      *__result = *__first;
+      ++__first;
+      ++__result;
+    }
+
+    return std::make_pair(std::move(__first), std::move(__result));
+  }
+
+  template <class _InIter, class _OutIter>
+  struct _CopySegment {
+    using _Traits = __segmented_iterator_traits<_InIter>;
+
+    _OutIter& __result_;
+
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit _CopySegment(_OutIter& __result)
+        : __result_(__result) {}
+
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+    operator()(typename _Traits::__local_iterator __lfirst, typename _Traits::__local_iterator __llast) {
+      __result_ = std::__copy<_AlgPolicy>(__lfirst, __llast, std::move(__result_)).second;
+    }
+  };
+
+  template <class _InIter, class _OutIter, __enable_if_t<__is_segmented_iterator<_InIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
+  operator()(_InIter __first, _InIter __last, _OutIter __result) const {
+    std::__for_each_segment(__first, __last, _CopySegment<_InIter, _OutIter>(__result));
+    return std::make_pair(__last, std::move(__result));
+  }
+
+  template <class _InIter,
+            class _OutIter,
+            __enable_if_t<__has_random_access_iterator_category<_InIter>::value &&
+                              !__is_segmented_iterator<_InIter>::value && __is_segmented_iterator<_OutIter>::value,
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
+  operator()(_InIter __first, _InIter __last, _OutIter __result) const {
+    using _Traits = __segmented_iterator_traits<_OutIter>;
+    using _DiffT  = typename common_type<__iter_diff_t<_InIter>, __iter_diff_t<_OutIter> >::type;
+
+    if (__first == __last)
+      return std::make_pair(std::move(__first), std::move(__result));
+
+    auto __local_first      = _Traits::__local(__result);
+    auto __segment_iterator = _Traits::__segment(__result);
+    while (true) {
+      auto __local_last = _Traits::__end(__segment_iterator);
+      auto __size       = std::min<_DiffT>(__local_last - __local_first, __last - __first);
+      auto __iters      = std::__copy<_AlgPolicy>(__first, __first + __size, __local_first);
+      __first           = std::move(__iters.first);
+
+      if (__first == __last)
+        return std::make_pair(std::move(__first), _Traits::__compose(__segment_iterator, std::move(__iters.second)));
+
+      __local_first = _Traits::__begin(++__segment_iterator);
+    }
+  }
+
+  // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer.
+  template <class _In, class _Out, __enable_if_t<__can_lower_copy_assignment_to_memmove<_In, _Out>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*>
+  operator()(_In* __first, _In* __last, _Out* __result) const {
+    return std::__copy_trivial_impl(__first, __last, __result);
+  }
+};
+
+template <class _AlgPolicy, class _InIter, class _Sent, class _OutIter>
+pair<_InIter, _OutIter> inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+__copy(_InIter __first, _Sent __last, _OutIter __result) {
+  return std::__copy_move_unwrap_iters<__copy_impl<_AlgPolicy> >(
+      std::move(__first), std::move(__last), std::move(__result));
+}
+
+template <class _InputIterator, class _OutputIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result) {
+  return std::__copy<_ClassicAlgPolicy>(__first, __last, __result).second;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/copy_backward.h b/libcxx/include/__cxx03/__algorithm/copy_backward.h
new file mode 100644
index 000000000000..73dc846a975a
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/copy_backward.h
@@ -0,0 +1,137 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_COPY_BACKWARD_H
+#define _LIBCPP___ALGORITHM_COPY_BACKWARD_H
+
+#include <__algorithm/copy_move_common.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/min.h>
+#include <__config>
+#include <__iterator/segmented_iterator.h>
+#include <__type_traits/common_type.h>
+#include <__type_traits/is_constructible.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _InIter, class _Sent, class _OutIter>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InIter, _OutIter>
+__copy_backward(_InIter __first, _Sent __last, _OutIter __result);
+
+template <class _AlgPolicy>
+struct __copy_backward_impl {
+  template <class _InIter, class _Sent, class _OutIter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result) const {
+    auto __last_iter          = _IterOps<_AlgPolicy>::next(__first, __last);
+    auto __original_last_iter = __last_iter;
+
+    while (__first != __last_iter) {
+      *--__result = *--__last_iter;
+    }
+
+    return std::make_pair(std::move(__original_last_iter), std::move(__result));
+  }
+
+  template <class _InIter, class _OutIter, __enable_if_t<__is_segmented_iterator<_InIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
+  operator()(_InIter __first, _InIter __last, _OutIter __result) const {
+    using _Traits = __segmented_iterator_traits<_InIter>;
+    auto __sfirst = _Traits::__segment(__first);
+    auto __slast  = _Traits::__segment(__last);
+    if (__sfirst == __slast) {
+      auto __iters =
+          std::__copy_backward<_AlgPolicy>(_Traits::__local(__first), _Traits::__local(__last), std::move(__result));
+      return std::make_pair(__last, __iters.second);
+    }
+
+    __result =
+        std::__copy_backward<_AlgPolicy>(_Traits::__begin(__slast), _Traits::__local(__last), std::move(__result))
+            .second;
+    --__slast;
+    while (__sfirst != __slast) {
+      __result =
+          std::__copy_backward<_AlgPolicy>(_Traits::__begin(__slast), _Traits::__end(__slast), std::move(__result))
+              .second;
+      --__slast;
+    }
+    __result = std::__copy_backward<_AlgPolicy>(_Traits::__local(__first), _Traits::__end(__slast), std::move(__result))
+                   .second;
+    return std::make_pair(__last, std::move(__result));
+  }
+
+  template <class _InIter,
+            class _OutIter,
+            __enable_if_t<__has_random_access_iterator_category<_InIter>::value &&
+                              !__is_segmented_iterator<_InIter>::value && __is_segmented_iterator<_OutIter>::value,
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
+  operator()(_InIter __first, _InIter __last, _OutIter __result) const {
+    using _Traits           = __segmented_iterator_traits<_OutIter>;
+    auto __orig_last        = __last;
+    auto __segment_iterator = _Traits::__segment(__result);
+
+    // When the range contains no elements, __result might not be a valid iterator
+    if (__first == __last)
+      return std::make_pair(__first, __result);
+
+    auto __local_last = _Traits::__local(__result);
+    while (true) {
+      using _DiffT = typename common_type<__iter_diff_t<_InIter>, __iter_diff_t<_OutIter> >::type;
+
+      auto __local_first = _Traits::__begin(__segment_iterator);
+      auto __size        = std::min<_DiffT>(__local_last - __local_first, __last - __first);
+      auto __iter        = std::__copy_backward<_AlgPolicy>(__last - __size, __last, __local_last).second;
+      __last -= __size;
+
+      if (__first == __last)
+        return std::make_pair(std::move(__orig_last), _Traits::__compose(__segment_iterator, std::move(__iter)));
+      --__segment_iterator;
+      __local_last = _Traits::__end(__segment_iterator);
+    }
+  }
+
+  // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer.
+  template <class _In, class _Out, __enable_if_t<__can_lower_copy_assignment_to_memmove<_In, _Out>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*>
+  operator()(_In* __first, _In* __last, _Out* __result) const {
+    return std::__copy_backward_trivial_impl(__first, __last, __result);
+  }
+};
+
+template <class _AlgPolicy, class _BidirectionalIterator1, class _Sentinel, class _BidirectionalIterator2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_BidirectionalIterator1, _BidirectionalIterator2>
+__copy_backward(_BidirectionalIterator1 __first, _Sentinel __last, _BidirectionalIterator2 __result) {
+  return std::__copy_move_unwrap_iters<__copy_backward_impl<_AlgPolicy> >(
+      std::move(__first), std::move(__last), std::move(__result));
+}
+
+template <class _BidirectionalIterator1, class _BidirectionalIterator2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _BidirectionalIterator2
+copy_backward(_BidirectionalIterator1 __first, _BidirectionalIterator1 __last, _BidirectionalIterator2 __result) {
+  static_assert(std::is_copy_constructible<_BidirectionalIterator1>::value &&
+                    std::is_copy_constructible<_BidirectionalIterator1>::value,
+                "Iterators must be copy constructible.");
+
+  return std::__copy_backward<_ClassicAlgPolicy>(std::move(__first), std::move(__last), std::move(__result)).second;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_COPY_BACKWARD_H
diff --git a/libcxx/include/__cxx03/__algorithm/copy_if.h b/libcxx/include/__cxx03/__algorithm/copy_if.h
new file mode 100644
index 000000000000..228e4d22323e
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/copy_if.h
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_COPY_IF_H
+#define _LIBCPP___ALGORITHM_COPY_IF_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _OutputIterator, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+copy_if(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _Predicate __pred) {
+  for (; __first != __last; ++__first) {
+    if (__pred(*__first)) {
+      *__result = *__first;
+      ++__result;
+    }
+  }
+  return __result;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_COPY_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/copy_move_common.h b/libcxx/include/__cxx03/__algorithm/copy_move_common.h
new file mode 100644
index 000000000000..8a98451a8f96
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/copy_move_common.h
@@ -0,0 +1,114 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_COPY_MOVE_COMMON_H
+#define _LIBCPP___ALGORITHM_COPY_MOVE_COMMON_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/unwrap_iter.h>
+#include <__algorithm/unwrap_range.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__memory/pointer_traits.h>
+#include <__string/constexpr_c_functions.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_always_bitcastable.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_trivially_assignable.h>
+#include <__type_traits/is_volatile.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Type traits.
+
+template <class _From, class _To>
+struct __can_lower_copy_assignment_to_memmove {
+  static const bool value =
+      // If the types are always bitcastable, it's valid to do a bitwise copy between them.
+      __is_always_bitcastable<_From, _To>::value &&
+      // Reject conversions that wouldn't be performed by the regular built-in assignment (e.g. between arrays).
+      is_trivially_assignable<_To&, const _From&>::value &&
+      // `memmove` doesn't accept `volatile` pointers, make sure the optimization SFINAEs away in that case.
+      !is_volatile<_From>::value && !is_volatile<_To>::value;
+};
+
+template <class _From, class _To>
+struct __can_lower_move_assignment_to_memmove {
+  static const bool value =
+      __is_always_bitcastable<_From, _To>::value && is_trivially_assignable<_To&, _From&&>::value &&
+      !is_volatile<_From>::value && !is_volatile<_To>::value;
+};
+
+// `memmove` algorithms implementation.
+
+template <class _In, class _Out>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*>
+__copy_trivial_impl(_In* __first, _In* __last, _Out* __result) {
+  const size_t __n = static_cast<size_t>(__last - __first);
+
+  std::__constexpr_memmove(__result, __first, __element_count(__n));
+
+  return std::make_pair(__last, __result + __n);
+}
+
+template <class _In, class _Out>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*>
+__copy_backward_trivial_impl(_In* __first, _In* __last, _Out* __result) {
+  const size_t __n = static_cast<size_t>(__last - __first);
+  __result -= __n;
+
+  std::__constexpr_memmove(__result, __first, __element_count(__n));
+
+  return std::make_pair(__last, __result);
+}
+
+// Iterator unwrapping and dispatching to the correct overload.
+
+template <class _InIter, class _OutIter>
+struct __can_rewrap
+    : integral_constant<bool, is_copy_constructible<_InIter>::value && is_copy_constructible<_OutIter>::value> {};
+
+template <class _Algorithm,
+          class _InIter,
+          class _Sent,
+          class _OutIter,
+          __enable_if_t<__can_rewrap<_InIter, _OutIter>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 pair<_InIter, _OutIter>
+__copy_move_unwrap_iters(_InIter __first, _Sent __last, _OutIter __out_first) {
+  auto __range  = std::__unwrap_range(__first, std::move(__last));
+  auto __result = _Algorithm()(std::move(__range.first), std::move(__range.second), std::__unwrap_iter(__out_first));
+  return std::make_pair(std::__rewrap_range<_Sent>(std::move(__first), std::move(__result.first)),
+                        std::__rewrap_iter(std::move(__out_first), std::move(__result.second)));
+}
+
+template <class _Algorithm,
+          class _InIter,
+          class _Sent,
+          class _OutIter,
+          __enable_if_t<!__can_rewrap<_InIter, _OutIter>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 pair<_InIter, _OutIter>
+__copy_move_unwrap_iters(_InIter __first, _Sent __last, _OutIter __out_first) {
+  return _Algorithm()(std::move(__first), std::move(__last), std::move(__out_first));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_COPY_MOVE_COMMON_H
diff --git a/libcxx/include/__cxx03/__algorithm/copy_n.h b/libcxx/include/__cxx03/__algorithm/copy_n.h
new file mode 100644
index 000000000000..f93f39203a7e
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/copy_n.h
@@ -0,0 +1,60 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_COPY_N_H
+#define _LIBCPP___ALGORITHM_COPY_N_H
+
+#include <__algorithm/copy.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/enable_if.h>
+#include <__utility/convert_to_integral.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator,
+          class _Size,
+          class _OutputIterator,
+          __enable_if_t<__has_input_iterator_category<_InputIterator>::value &&
+                            !__has_random_access_iterator_category<_InputIterator>::value,
+                        int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+copy_n(_InputIterator __first, _Size __orig_n, _OutputIterator __result) {
+  typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize;
+  _IntegralSize __n = __orig_n;
+  if (__n > 0) {
+    *__result = *__first;
+    ++__result;
+    for (--__n; __n > 0; --__n) {
+      ++__first;
+      *__result = *__first;
+      ++__result;
+    }
+  }
+  return __result;
+}
+
+template <class _InputIterator,
+          class _Size,
+          class _OutputIterator,
+          __enable_if_t<__has_random_access_iterator_category<_InputIterator>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+copy_n(_InputIterator __first, _Size __orig_n, _OutputIterator __result) {
+  typedef typename iterator_traits<_InputIterator>::difference_type difference_type;
+  typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize;
+  _IntegralSize __n = __orig_n;
+  return std::copy(__first, __first + difference_type(__n), __result);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_COPY_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/count.h b/libcxx/include/__cxx03/__algorithm/count.h
new file mode 100644
index 000000000000..1cfe7f631ac1
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/count.h
@@ -0,0 +1,92 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_COUNT_H
+#define _LIBCPP___ALGORITHM_COUNT_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/min.h>
+#include <__bit/invert_if.h>
+#include <__bit/popcount.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__fwd/bit_reference.h>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// generic implementation
+template <class _AlgPolicy, class _Iter, class _Sent, class _Tp, class _Proj>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename _IterOps<_AlgPolicy>::template __difference_type<_Iter>
+__count(_Iter __first, _Sent __last, const _Tp& __value, _Proj& __proj) {
+  typename _IterOps<_AlgPolicy>::template __difference_type<_Iter> __r(0);
+  for (; __first != __last; ++__first)
+    if (std::__invoke(__proj, *__first) == __value)
+      ++__r;
+  return __r;
+}
+
+// __bit_iterator implementation
+template <bool _ToCount, class _Cp, bool _IsConst>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __bit_iterator<_Cp, _IsConst>::difference_type
+__count_bool(__bit_iterator<_Cp, _IsConst> __first, typename _Cp::size_type __n) {
+  using _It             = __bit_iterator<_Cp, _IsConst>;
+  using __storage_type  = typename _It::__storage_type;
+  using difference_type = typename _It::difference_type;
+
+  const int __bits_per_word = _It::__bits_per_word;
+  difference_type __r       = 0;
+  // do first partial word
+  if (__first.__ctz_ != 0) {
+    __storage_type __clz_f = static_cast<__storage_type>(__bits_per_word - __first.__ctz_);
+    __storage_type __dn    = std::min(__clz_f, __n);
+    __storage_type __m     = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn));
+    __r                    = std::__libcpp_popcount(std::__invert_if<!_ToCount>(*__first.__seg_) & __m);
+    __n -= __dn;
+    ++__first.__seg_;
+  }
+  // do middle whole words
+  for (; __n >= __bits_per_word; ++__first.__seg_, __n -= __bits_per_word)
+    __r += std::__libcpp_popcount(std::__invert_if<!_ToCount>(*__first.__seg_));
+  // do last partial word
+  if (__n > 0) {
+    __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+    __r += std::__libcpp_popcount(std::__invert_if<!_ToCount>(*__first.__seg_) & __m);
+  }
+  return __r;
+}
+
+template <class, class _Cp, bool _IsConst, class _Tp, class _Proj, __enable_if_t<__is_identity<_Proj>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __iter_diff_t<__bit_iterator<_Cp, _IsConst> >
+__count(__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, const _Tp& __value, _Proj&) {
+  if (__value)
+    return std::__count_bool<true>(__first, static_cast<typename _Cp::size_type>(__last - __first));
+  return std::__count_bool<false>(__first, static_cast<typename _Cp::size_type>(__last - __first));
+}
+
+template <class _InputIterator, class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __iter_diff_t<_InputIterator>
+count(_InputIterator __first, _InputIterator __last, const _Tp& __value) {
+  __identity __proj;
+  return std::__count<_ClassicAlgPolicy>(__first, __last, __value, __proj);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_COUNT_H
diff --git a/libcxx/include/__cxx03/__algorithm/count_if.h b/libcxx/include/__cxx03/__algorithm/count_if.h
new file mode 100644
index 000000000000..25782069d032
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/count_if.h
@@ -0,0 +1,35 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_COUNT_IF_H
+#define _LIBCPP___ALGORITHM_COUNT_IF_H
+
+#include <__config>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _Predicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+typename iterator_traits<_InputIterator>::difference_type
+count_if(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
+  typename iterator_traits<_InputIterator>::difference_type __r(0);
+  for (; __first != __last; ++__first)
+    if (__pred(*__first))
+      ++__r;
+  return __r;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_COUNT_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/equal.h b/libcxx/include/__cxx03/__algorithm/equal.h
new file mode 100644
index 000000000000..bfc8f72f6eb1
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/equal.h
@@ -0,0 +1,133 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_EQUAL_H
+#define _LIBCPP___ALGORITHM_EQUAL_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/unwrap_iter.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__string/constexpr_c_functions.h>
+#include <__type_traits/desugars_to.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_equality_comparable.h>
+#include <__type_traits/is_volatile.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __equal_iter_impl(
+    _InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _BinaryPredicate& __pred) {
+  for (; __first1 != __last1; ++__first1, (void)++__first2)
+    if (!__pred(*__first1, *__first2))
+      return false;
+  return true;
+}
+
+template <class _Tp,
+          class _Up,
+          class _BinaryPredicate,
+          __enable_if_t<__desugars_to_v<__equal_tag, _BinaryPredicate, _Tp, _Up> && !is_volatile<_Tp>::value &&
+                            !is_volatile<_Up>::value && __libcpp_is_trivially_equality_comparable<_Tp, _Up>::value,
+                        int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+__equal_iter_impl(_Tp* __first1, _Tp* __last1, _Up* __first2, _BinaryPredicate&) {
+  return std::__constexpr_memcmp_equal(__first1, __first2, __element_count(__last1 - __first1));
+}
+
+template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+equal(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _BinaryPredicate __pred) {
+  return std::__equal_iter_impl(
+      std::__unwrap_iter(__first1), std::__unwrap_iter(__last1), std::__unwrap_iter(__first2), __pred);
+}
+
+template <class _InputIterator1, class _InputIterator2>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+equal(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2) {
+  return std::equal(__first1, __last1, __first2, __equal_to());
+}
+
+#if _LIBCPP_STD_VER >= 14
+
+template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class _Pred, class _Proj1, class _Proj2>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __equal_impl(
+    _Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, _Pred& __comp, _Proj1& __proj1, _Proj2& __proj2) {
+  while (__first1 != __last1 && __first2 != __last2) {
+    if (!std::__invoke(__comp, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2)))
+      return false;
+    ++__first1;
+    ++__first2;
+  }
+  return __first1 == __last1 && __first2 == __last2;
+}
+
+template <class _Tp,
+          class _Up,
+          class _Pred,
+          class _Proj1,
+          class _Proj2,
+          __enable_if_t<__desugars_to_v<__equal_tag, _Pred, _Tp, _Up> && __is_identity<_Proj1>::value &&
+                            __is_identity<_Proj2>::value && !is_volatile<_Tp>::value && !is_volatile<_Up>::value &&
+                            __libcpp_is_trivially_equality_comparable<_Tp, _Up>::value,
+                        int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+__equal_impl(_Tp* __first1, _Tp* __last1, _Up* __first2, _Up*, _Pred&, _Proj1&, _Proj2&) {
+  return std::__constexpr_memcmp_equal(__first1, __first2, __element_count(__last1 - __first1));
+}
+
+template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+equal(_InputIterator1 __first1,
+      _InputIterator1 __last1,
+      _InputIterator2 __first2,
+      _InputIterator2 __last2,
+      _BinaryPredicate __pred) {
+  if constexpr (__has_random_access_iterator_category<_InputIterator1>::value &&
+                __has_random_access_iterator_category<_InputIterator2>::value) {
+    if (std::distance(__first1, __last1) != std::distance(__first2, __last2))
+      return false;
+  }
+  __identity __proj;
+  return std::__equal_impl(
+      std::__unwrap_iter(__first1),
+      std::__unwrap_iter(__last1),
+      std::__unwrap_iter(__first2),
+      std::__unwrap_iter(__last2),
+      __pred,
+      __proj,
+      __proj);
+}
+
+template <class _InputIterator1, class _InputIterator2>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+equal(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2) {
+  return std::equal(__first1, __last1, __first2, __last2, __equal_to());
+}
+
+#endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_EQUAL_H
diff --git a/libcxx/include/__cxx03/__algorithm/equal_range.h b/libcxx/include/__cxx03/__algorithm/equal_range.h
new file mode 100644
index 000000000000..09bbf8f00602
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/equal_range.h
@@ -0,0 +1,85 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_EQUAL_RANGE_H
+#define _LIBCPP___ALGORITHM_EQUAL_RANGE_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/half_positive.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/lower_bound.h>
+#include <__algorithm/upper_bound.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/advance.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__type_traits/is_callable.h>
+#include <__type_traits/is_constructible.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Compare, class _Iter, class _Sent, class _Tp, class _Proj>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter, _Iter>
+__equal_range(_Iter __first, _Sent __last, const _Tp& __value, _Compare&& __comp, _Proj&& __proj) {
+  auto __len  = _IterOps<_AlgPolicy>::distance(__first, __last);
+  _Iter __end = _IterOps<_AlgPolicy>::next(__first, __last);
+  while (__len != 0) {
+    auto __half_len = std::__half_positive(__len);
+    _Iter __mid     = _IterOps<_AlgPolicy>::next(__first, __half_len);
+    if (std::__invoke(__comp, std::__invoke(__proj, *__mid), __value)) {
+      __first = ++__mid;
+      __len -= __half_len + 1;
+    } else if (std::__invoke(__comp, __value, std::__invoke(__proj, *__mid))) {
+      __end = __mid;
+      __len = __half_len;
+    } else {
+      _Iter __mp1 = __mid;
+      return pair<_Iter, _Iter>(std::__lower_bound<_AlgPolicy>(__first, __mid, __value, __comp, __proj),
+                                std::__upper_bound<_AlgPolicy>(++__mp1, __end, __value, __comp, __proj));
+    }
+  }
+  return pair<_Iter, _Iter>(__first, __first);
+}
+
+template <class _ForwardIterator, class _Tp, class _Compare>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_ForwardIterator, _ForwardIterator>
+equal_range(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, _Compare __comp) {
+  static_assert(__is_callable<_Compare, decltype(*__first), const _Tp&>::value, "The comparator has to be callable");
+  static_assert(is_copy_constructible<_ForwardIterator>::value, "Iterator has to be copy constructible");
+  return std::__equal_range<_ClassicAlgPolicy>(
+      std::move(__first),
+      std::move(__last),
+      __value,
+      static_cast<__comp_ref_type<_Compare> >(__comp),
+      std::__identity());
+}
+
+template <class _ForwardIterator, class _Tp>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_ForwardIterator, _ForwardIterator>
+equal_range(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
+  return std::equal_range(std::move(__first), std::move(__last), __value, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_EQUAL_RANGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/fill.h b/libcxx/include/__cxx03/__algorithm/fill.h
new file mode 100644
index 000000000000..1ce3eadb013d
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/fill.h
@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_FILL_H
+#define _LIBCPP___ALGORITHM_FILL_H
+
+#include <__algorithm/fill_n.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// fill isn't specialized for std::memset, because the compiler already optimizes the loop to a call to std::memset.
+
+template <class _ForwardIterator, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+__fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, forward_iterator_tag) {
+  for (; __first != __last; ++__first)
+    *__first = __value;
+}
+
+template <class _RandomAccessIterator, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+__fill(_RandomAccessIterator __first, _RandomAccessIterator __last, const _Tp& __value, random_access_iterator_tag) {
+  std::fill_n(__first, __last - __first, __value);
+}
+
+template <class _ForwardIterator, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
+  std::__fill(__first, __last, __value, typename iterator_traits<_ForwardIterator>::iterator_category());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_FILL_H
diff --git a/libcxx/include/__cxx03/__algorithm/fill_n.h b/libcxx/include/__cxx03/__algorithm/fill_n.h
new file mode 100644
index 000000000000..f29633f88087
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/fill_n.h
@@ -0,0 +1,98 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_FILL_N_H
+#define _LIBCPP___ALGORITHM_FILL_N_H
+
+#include <__algorithm/min.h>
+#include <__config>
+#include <__fwd/bit_reference.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/pointer_traits.h>
+#include <__utility/convert_to_integral.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// fill_n isn't specialized for std::memset, because the compiler already optimizes the loop to a call to std::memset.
+
+template <class _OutputIterator, class _Size, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+__fill_n(_OutputIterator __first, _Size __n, const _Tp& __value);
+
+template <bool _FillVal, class _Cp>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+__fill_n_bool(__bit_iterator<_Cp, false> __first, typename _Cp::size_type __n) {
+  using _It            = __bit_iterator<_Cp, false>;
+  using __storage_type = typename _It::__storage_type;
+
+  const int __bits_per_word = _It::__bits_per_word;
+  // do first partial word
+  if (__first.__ctz_ != 0) {
+    __storage_type __clz_f = static_cast<__storage_type>(__bits_per_word - __first.__ctz_);
+    __storage_type __dn    = std::min(__clz_f, __n);
+    __storage_type __m     = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn));
+    if (_FillVal)
+      *__first.__seg_ |= __m;
+    else
+      *__first.__seg_ &= ~__m;
+    __n -= __dn;
+    ++__first.__seg_;
+  }
+  // do middle whole words
+  __storage_type __nw = __n / __bits_per_word;
+  std::__fill_n(std::__to_address(__first.__seg_), __nw, _FillVal ? static_cast<__storage_type>(-1) : 0);
+  __n -= __nw * __bits_per_word;
+  // do last partial word
+  if (__n > 0) {
+    __first.__seg_ += __nw;
+    __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+    if (_FillVal)
+      *__first.__seg_ |= __m;
+    else
+      *__first.__seg_ &= ~__m;
+  }
+}
+
+template <class _Cp, class _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, false>
+__fill_n(__bit_iterator<_Cp, false> __first, _Size __n, const bool& __value) {
+  if (__n > 0) {
+    if (__value)
+      std::__fill_n_bool<true>(__first, __n);
+    else
+      std::__fill_n_bool<false>(__first, __n);
+  }
+  return __first + __n;
+}
+
+template <class _OutputIterator, class _Size, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+__fill_n(_OutputIterator __first, _Size __n, const _Tp& __value) {
+  for (; __n > 0; ++__first, (void)--__n)
+    *__first = __value;
+  return __first;
+}
+
+template <class _OutputIterator, class _Size, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+fill_n(_OutputIterator __first, _Size __n, const _Tp& __value) {
+  return std::__fill_n(__first, std::__convert_to_integral(__n), __value);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_FILL_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/find.h b/libcxx/include/__cxx03/__algorithm/find.h
new file mode 100644
index 000000000000..7f58dbb13a57
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/find.h
@@ -0,0 +1,181 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_FIND_H
+#define _LIBCPP___ALGORITHM_FIND_H
+
+#include <__algorithm/find_segment_if.h>
+#include <__algorithm/min.h>
+#include <__algorithm/unwrap_iter.h>
+#include <__bit/countr.h>
+#include <__bit/invert_if.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__fwd/bit_reference.h>
+#include <__iterator/segmented_iterator.h>
+#include <__string/constexpr_c_functions.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_signed.h>
+#include <__utility/move.h>
+#include <limits>
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+#  include <cwchar>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// generic implementation
+template <class _Iter, class _Sent, class _Tp, class _Proj>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter
+__find(_Iter __first, _Sent __last, const _Tp& __value, _Proj& __proj) {
+  for (; __first != __last; ++__first)
+    if (std::__invoke(__proj, *__first) == __value)
+      break;
+  return __first;
+}
+
+// trivially equality comparable implementations
+template <class _Tp,
+          class _Up,
+          class _Proj,
+          __enable_if_t<__is_identity<_Proj>::value && __libcpp_is_trivially_equality_comparable<_Tp, _Up>::value &&
+                            sizeof(_Tp) == 1,
+                        int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __find(_Tp* __first, _Tp* __last, const _Up& __value, _Proj&) {
+  if (auto __ret = std::__constexpr_memchr(__first, __value, __last - __first))
+    return __ret;
+  return __last;
+}
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class _Tp,
+          class _Up,
+          class _Proj,
+          __enable_if_t<__is_identity<_Proj>::value && __libcpp_is_trivially_equality_comparable<_Tp, _Up>::value &&
+                            sizeof(_Tp) == sizeof(wchar_t) && _LIBCPP_ALIGNOF(_Tp) >= _LIBCPP_ALIGNOF(wchar_t),
+                        int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __find(_Tp* __first, _Tp* __last, const _Up& __value, _Proj&) {
+  if (auto __ret = std::__constexpr_wmemchr(__first, __value, __last - __first))
+    return __ret;
+  return __last;
+}
+#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+// TODO: This should also be possible to get right with different signedness
+// cast integral types to allow vectorization
+template <class _Tp,
+          class _Up,
+          class _Proj,
+          __enable_if_t<__is_identity<_Proj>::value && !__libcpp_is_trivially_equality_comparable<_Tp, _Up>::value &&
+                            is_integral<_Tp>::value && is_integral<_Up>::value &&
+                            is_signed<_Tp>::value == is_signed<_Up>::value,
+                        int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp*
+__find(_Tp* __first, _Tp* __last, const _Up& __value, _Proj& __proj) {
+  if (__value < numeric_limits<_Tp>::min() || __value > numeric_limits<_Tp>::max())
+    return __last;
+  return std::__find(__first, __last, _Tp(__value), __proj);
+}
+
+// __bit_iterator implementation
+template <bool _ToFind, class _Cp, bool _IsConst>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, _IsConst>
+__find_bool(__bit_iterator<_Cp, _IsConst> __first, typename _Cp::size_type __n) {
+  using _It            = __bit_iterator<_Cp, _IsConst>;
+  using __storage_type = typename _It::__storage_type;
+
+  const int __bits_per_word = _It::__bits_per_word;
+  // do first partial word
+  if (__first.__ctz_ != 0) {
+    __storage_type __clz_f = static_cast<__storage_type>(__bits_per_word - __first.__ctz_);
+    __storage_type __dn    = std::min(__clz_f, __n);
+    __storage_type __m     = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn));
+    __storage_type __b     = std::__invert_if<!_ToFind>(*__first.__seg_) & __m;
+    if (__b)
+      return _It(__first.__seg_, static_cast<unsigned>(std::__libcpp_ctz(__b)));
+    if (__n == __dn)
+      return __first + __n;
+    __n -= __dn;
+    ++__first.__seg_;
+  }
+  // do middle whole words
+  for (; __n >= __bits_per_word; ++__first.__seg_, __n -= __bits_per_word) {
+    __storage_type __b = std::__invert_if<!_ToFind>(*__first.__seg_);
+    if (__b)
+      return _It(__first.__seg_, static_cast<unsigned>(std::__libcpp_ctz(__b)));
+  }
+  // do last partial word
+  if (__n > 0) {
+    __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+    __storage_type __b = std::__invert_if<!_ToFind>(*__first.__seg_) & __m;
+    if (__b)
+      return _It(__first.__seg_, static_cast<unsigned>(std::__libcpp_ctz(__b)));
+  }
+  return _It(__first.__seg_, static_cast<unsigned>(__n));
+}
+
+template <class _Cp, bool _IsConst, class _Tp, class _Proj, __enable_if_t<__is_identity<_Proj>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, _IsConst>
+__find(__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, const _Tp& __value, _Proj&) {
+  if (static_cast<bool>(__value))
+    return std::__find_bool<true>(__first, static_cast<typename _Cp::size_type>(__last - __first));
+  return std::__find_bool<false>(__first, static_cast<typename _Cp::size_type>(__last - __first));
+}
+
+// segmented iterator implementation
+
+template <class>
+struct __find_segment;
+
+template <class _SegmentedIterator,
+          class _Tp,
+          class _Proj,
+          __enable_if_t<__is_segmented_iterator<_SegmentedIterator>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _SegmentedIterator
+__find(_SegmentedIterator __first, _SegmentedIterator __last, const _Tp& __value, _Proj& __proj) {
+  return std::__find_segment_if(std::move(__first), std::move(__last), __find_segment<_Tp>(__value), __proj);
+}
+
+template <class _Tp>
+struct __find_segment {
+  const _Tp& __value_;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __find_segment(const _Tp& __value) : __value_(__value) {}
+
+  template <class _InputIterator, class _Proj>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _InputIterator
+  operator()(_InputIterator __first, _InputIterator __last, _Proj& __proj) const {
+    return std::__find(__first, __last, __value_, __proj);
+  }
+};
+
+// public API
+template <class _InputIterator, class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
+find(_InputIterator __first, _InputIterator __last, const _Tp& __value) {
+  __identity __proj;
+  return std::__rewrap_iter(
+      __first, std::__find(std::__unwrap_iter(__first), std::__unwrap_iter(__last), __value, __proj));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_FIND_H
diff --git a/libcxx/include/__cxx03/__algorithm/find_end.h b/libcxx/include/__cxx03/__algorithm/find_end.h
new file mode 100644
index 000000000000..7e08e7953534
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/find_end.h
@@ -0,0 +1,225 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_FIND_END_OF_H
+#define _LIBCPP___ALGORITHM_FIND_END_OF_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/search.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/advance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/reverse_iterator.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template < class _AlgPolicy,
+           class _Iter1,
+           class _Sent1,
+           class _Iter2,
+           class _Sent2,
+           class _Pred,
+           class _Proj1,
+           class _Proj2>
+_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1> __find_end_impl(
+    _Iter1 __first1,
+    _Sent1 __last1,
+    _Iter2 __first2,
+    _Sent2 __last2,
+    _Pred& __pred,
+    _Proj1& __proj1,
+    _Proj2& __proj2,
+    forward_iterator_tag,
+    forward_iterator_tag) {
+  // modeled after search algorithm
+  _Iter1 __match_first = _IterOps<_AlgPolicy>::next(__first1, __last1); // __last1 is the "default" answer
+  _Iter1 __match_last  = __match_first;
+  if (__first2 == __last2)
+    return pair<_Iter1, _Iter1>(__match_last, __match_last);
+  while (true) {
+    while (true) {
+      if (__first1 == __last1) // if source exhausted return last correct answer (or __last1 if never found)
+        return pair<_Iter1, _Iter1>(__match_first, __match_last);
+      if (std::__invoke(__pred, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2)))
+        break;
+      ++__first1;
+    }
+    // *__first1 matches *__first2, now match elements after here
+    _Iter1 __m1 = __first1;
+    _Iter2 __m2 = __first2;
+    while (true) {
+      if (++__m2 == __last2) { // Pattern exhaused, record answer and search for another one
+        __match_first = __first1;
+        __match_last  = ++__m1;
+        ++__first1;
+        break;
+      }
+      if (++__m1 == __last1) // Source exhausted, return last answer
+        return pair<_Iter1, _Iter1>(__match_first, __match_last);
+      // mismatch, restart with a new __first
+      if (!std::__invoke(__pred, std::__invoke(__proj1, *__m1), std::__invoke(__proj2, *__m2))) {
+        ++__first1;
+        break;
+      } // else there is a match, check next elements
+    }
+  }
+}
+
+template < class _IterOps,
+           class _Pred,
+           class _Iter1,
+           class _Sent1,
+           class _Iter2,
+           class _Sent2,
+           class _Proj1,
+           class _Proj2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Iter1 __find_end(
+    _Iter1 __first1,
+    _Sent1 __sent1,
+    _Iter2 __first2,
+    _Sent2 __sent2,
+    _Pred& __pred,
+    _Proj1& __proj1,
+    _Proj2& __proj2,
+    bidirectional_iterator_tag,
+    bidirectional_iterator_tag) {
+  auto __last1 = _IterOps::next(__first1, __sent1);
+  auto __last2 = _IterOps::next(__first2, __sent2);
+  // modeled after search algorithm (in reverse)
+  if (__first2 == __last2)
+    return __last1; // Everything matches an empty sequence
+  _Iter1 __l1 = __last1;
+  _Iter2 __l2 = __last2;
+  --__l2;
+  while (true) {
+    // Find last element in sequence 1 that matchs *(__last2-1), with a mininum of loop checks
+    while (true) {
+      if (__first1 == __l1) // return __last1 if no element matches *__first2
+        return __last1;
+      if (std::__invoke(__pred, std::__invoke(__proj1, *--__l1), std::__invoke(__proj2, *__l2)))
+        break;
+    }
+    // *__l1 matches *__l2, now match elements before here
+    _Iter1 __m1 = __l1;
+    _Iter2 __m2 = __l2;
+    while (true) {
+      if (__m2 == __first2) // If pattern exhausted, __m1 is the answer (works for 1 element pattern)
+        return __m1;
+      if (__m1 == __first1) // Otherwise if source exhaused, pattern not found
+        return __last1;
+
+      // if there is a mismatch, restart with a new __l1
+      if (!std::__invoke(__pred, std::__invoke(__proj1, *--__m1), std::__invoke(__proj2, *--__m2))) {
+        break;
+      } // else there is a match, check next elements
+    }
+  }
+}
+
+template < class _AlgPolicy,
+           class _Pred,
+           class _Iter1,
+           class _Sent1,
+           class _Iter2,
+           class _Sent2,
+           class _Proj1,
+           class _Proj2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter1 __find_end(
+    _Iter1 __first1,
+    _Sent1 __sent1,
+    _Iter2 __first2,
+    _Sent2 __sent2,
+    _Pred& __pred,
+    _Proj1& __proj1,
+    _Proj2& __proj2,
+    random_access_iterator_tag,
+    random_access_iterator_tag) {
+  typedef typename iterator_traits<_Iter1>::difference_type _D1;
+  auto __last1 = _IterOps<_AlgPolicy>::next(__first1, __sent1);
+  auto __last2 = _IterOps<_AlgPolicy>::next(__first2, __sent2);
+  // Take advantage of knowing source and pattern lengths.  Stop short when source is smaller than pattern
+  auto __len2 = __last2 - __first2;
+  if (__len2 == 0)
+    return __last1;
+  auto __len1 = __last1 - __first1;
+  if (__len1 < __len2)
+    return __last1;
+  const _Iter1 __s = __first1 + _D1(__len2 - 1); // End of pattern match can't go before here
+  _Iter1 __l1      = __last1;
+  _Iter2 __l2      = __last2;
+  --__l2;
+  while (true) {
+    while (true) {
+      if (__s == __l1)
+        return __last1;
+      if (std::__invoke(__pred, std::__invoke(__proj1, *--__l1), std::__invoke(__proj2, *__l2)))
+        break;
+    }
+    _Iter1 __m1 = __l1;
+    _Iter2 __m2 = __l2;
+    while (true) {
+      if (__m2 == __first2)
+        return __m1;
+      // no need to check range on __m1 because __s guarantees we have enough source
+      if (!std::__invoke(__pred, std::__invoke(__proj1, *--__m1), std::__invoke(*--__m2))) {
+        break;
+      }
+    }
+  }
+}
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator1 __find_end_classic(
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _BinaryPredicate& __pred) {
+  auto __proj = __identity();
+  return std::__find_end_impl<_ClassicAlgPolicy>(
+             __first1,
+             __last1,
+             __first2,
+             __last2,
+             __pred,
+             __proj,
+             __proj,
+             typename iterator_traits<_ForwardIterator1>::iterator_category(),
+             typename iterator_traits<_ForwardIterator2>::iterator_category())
+      .first;
+}
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator1 find_end(
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _BinaryPredicate __pred) {
+  return std::__find_end_classic(__first1, __last1, __first2, __last2, __pred);
+}
+
+template <class _ForwardIterator1, class _ForwardIterator2>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator1
+find_end(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2) {
+  return std::find_end(__first1, __last1, __first2, __last2, __equal_to());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_FIND_END_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/find_first_of.h b/libcxx/include/__cxx03/__algorithm/find_first_of.h
new file mode 100644
index 000000000000..6b99f562f880
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/find_first_of.h
@@ -0,0 +1,55 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_FIND_FIRST_OF_H
+#define _LIBCPP___ALGORITHM_FIND_FIRST_OF_H
+
+#include <__algorithm/comp.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator1 __find_first_of_ce(
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _BinaryPredicate&& __pred) {
+  for (; __first1 != __last1; ++__first1)
+    for (_ForwardIterator2 __j = __first2; __j != __last2; ++__j)
+      if (__pred(*__first1, *__j))
+        return __first1;
+  return __last1;
+}
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator1 find_first_of(
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _BinaryPredicate __pred) {
+  return std::__find_first_of_ce(__first1, __last1, __first2, __last2, __pred);
+}
+
+template <class _ForwardIterator1, class _ForwardIterator2>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator1 find_first_of(
+    _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2) {
+  return std::__find_first_of_ce(__first1, __last1, __first2, __last2, __equal_to());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_FIND_FIRST_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/find_if.h b/libcxx/include/__cxx03/__algorithm/find_if.h
new file mode 100644
index 000000000000..22092d352b06
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/find_if.h
@@ -0,0 +1,32 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_FIND_IF_H
+#define _LIBCPP___ALGORITHM_FIND_IF_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _Predicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
+find_if(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
+  for (; __first != __last; ++__first)
+    if (__pred(*__first))
+      break;
+  return __first;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_FIND_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/find_if_not.h b/libcxx/include/__cxx03/__algorithm/find_if_not.h
new file mode 100644
index 000000000000..cc2001967f0c
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/find_if_not.h
@@ -0,0 +1,32 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_FIND_IF_NOT_H
+#define _LIBCPP___ALGORITHM_FIND_IF_NOT_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _Predicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
+find_if_not(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
+  for (; __first != __last; ++__first)
+    if (!__pred(*__first))
+      break;
+  return __first;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_FIND_IF_NOT_H
diff --git a/libcxx/include/__cxx03/__algorithm/find_segment_if.h b/libcxx/include/__cxx03/__algorithm/find_segment_if.h
new file mode 100644
index 000000000000..9d6064f3e283
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/find_segment_if.h
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_FIND_SEGMENT_IF_H
+#define _LIBCPP___ALGORITHM_FIND_SEGMENT_IF_H
+
+#include <__config>
+#include <__iterator/segmented_iterator.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// __find_segment_if is a utility function for optimizing iteration over segmented iterators linearly.
+// [__first, __last) has to be a segmented range. __pred is expected to take a range of local iterators and the __proj.
+// It returns an iterator to the first element that satisfies the predicate, or a one-past-the-end iterator if there was
+// no match. __proj may be anything that should be passed to __pred, but is expected to be a projection to support
+// ranges algorithms, or __identity for classic algorithms.
+
+template <class _SegmentedIterator, class _Pred, class _Proj>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _SegmentedIterator
+__find_segment_if(_SegmentedIterator __first, _SegmentedIterator __last, _Pred __pred, _Proj& __proj) {
+  using _Traits = __segmented_iterator_traits<_SegmentedIterator>;
+
+  auto __sfirst = _Traits::__segment(__first);
+  auto __slast  = _Traits::__segment(__last);
+
+  // We are in a single segment, so we might not be at the beginning or end
+  if (__sfirst == __slast)
+    return _Traits::__compose(__sfirst, __pred(_Traits::__local(__first), _Traits::__local(__last), __proj));
+
+  { // We have more than one segment. Iterate over the first segment, since we might not start at the beginning
+    auto __llast = _Traits::__end(__sfirst);
+    auto __liter = __pred(_Traits::__local(__first), __llast, __proj);
+    if (__liter != __llast)
+      return _Traits::__compose(__sfirst, __liter);
+  }
+  ++__sfirst;
+
+  // Iterate over the segments which are guaranteed to be completely in the range
+  while (__sfirst != __slast) {
+    auto __llast = _Traits::__end(__sfirst);
+    auto __liter = __pred(_Traits::__begin(__sfirst), _Traits::__end(__sfirst), __proj);
+    if (__liter != __llast)
+      return _Traits::__compose(__sfirst, __liter);
+    ++__sfirst;
+  }
+
+  // Iterate over the last segment
+  return _Traits::__compose(__sfirst, __pred(_Traits::__begin(__sfirst), _Traits::__local(__last), __proj));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_FIND_SEGMENT_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/fold.h b/libcxx/include/__cxx03/__algorithm/fold.h
new file mode 100644
index 000000000000..255658f52324
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/fold.h
@@ -0,0 +1,128 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_FOLD_H
+#define _LIBCPP___ALGORITHM_FOLD_H
+
+#include <__concepts/assignable.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/invocable.h>
+#include <__concepts/movable.h>
+#include <__config>
+#include <__functional/invoke.h>
+#include <__functional/reference_wrapper.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/invoke.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+namespace ranges {
+template <class _Ip, class _Tp>
+struct in_value_result {
+  _LIBCPP_NO_UNIQUE_ADDRESS _Ip in;
+  _LIBCPP_NO_UNIQUE_ADDRESS _Tp value;
+
+  template <class _I2, class _T2>
+    requires convertible_to<const _Ip&, _I2> && convertible_to<const _Tp&, _T2>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator in_value_result<_I2, _T2>() const& {
+    return {in, value};
+  }
+
+  template <class _I2, class _T2>
+    requires convertible_to<_Ip, _I2> && convertible_to<_Tp, _T2>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator in_value_result<_I2, _T2>() && {
+    return {std::move(in), std::move(value)};
+  }
+};
+
+template <class _Ip, class _Tp>
+using fold_left_with_iter_result = in_value_result<_Ip, _Tp>;
+
+template <class _Fp, class _Tp, class _Ip, class _Rp, class _Up = decay_t<_Rp>>
+concept __indirectly_binary_left_foldable_impl =
+    convertible_to<_Rp, _Up> &&                    //
+    movable<_Tp> &&                                //
+    movable<_Up> &&                                //
+    convertible_to<_Tp, _Up> &&                    //
+    invocable<_Fp&, _Up, iter_reference_t<_Ip>> && //
+    assignable_from<_Up&, invoke_result_t<_Fp&, _Up, iter_reference_t<_Ip>>>;
+
+template <class _Fp, class _Tp, class _Ip>
+concept __indirectly_binary_left_foldable =
+    copy_constructible<_Fp> &&                     //
+    invocable<_Fp&, _Tp, iter_reference_t<_Ip>> && //
+    __indirectly_binary_left_foldable_impl<_Fp, _Tp, _Ip, invoke_result_t<_Fp&, _Tp, iter_reference_t<_Ip>>>;
+
+struct __fold_left_with_iter {
+  template <input_iterator _Ip, sentinel_for<_Ip> _Sp, class _Tp, __indirectly_binary_left_foldable<_Tp, _Ip> _Fp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()(_Ip __first, _Sp __last, _Tp __init, _Fp __f) {
+    using _Up = decay_t<invoke_result_t<_Fp&, _Tp, iter_reference_t<_Ip>>>;
+
+    if (__first == __last) {
+      return fold_left_with_iter_result<_Ip, _Up>{std::move(__first), _Up(std::move(__init))};
+    }
+
+    _Up __result = std::invoke(__f, std::move(__init), *__first);
+    for (++__first; __first != __last; ++__first) {
+      __result = std::invoke(__f, std::move(__result), *__first);
+    }
+
+    return fold_left_with_iter_result<_Ip, _Up>{std::move(__first), std::move(__result)};
+  }
+
+  template <input_range _Rp, class _Tp, __indirectly_binary_left_foldable<_Tp, iterator_t<_Rp>> _Fp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()(_Rp&& __r, _Tp __init, _Fp __f) {
+    auto __result = operator()(ranges::begin(__r), ranges::end(__r), std::move(__init), std::ref(__f));
+
+    using _Up = decay_t<invoke_result_t<_Fp&, _Tp, range_reference_t<_Rp>>>;
+    return fold_left_with_iter_result<borrowed_iterator_t<_Rp>, _Up>{std::move(__result.in), std::move(__result.value)};
+  }
+};
+
+inline constexpr auto fold_left_with_iter = __fold_left_with_iter();
+
+struct __fold_left {
+  template <input_iterator _Ip, sentinel_for<_Ip> _Sp, class _Tp, __indirectly_binary_left_foldable<_Tp, _Ip> _Fp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()(_Ip __first, _Sp __last, _Tp __init, _Fp __f) {
+    return fold_left_with_iter(std::move(__first), std::move(__last), std::move(__init), std::ref(__f)).value;
+  }
+
+  template <input_range _Rp, class _Tp, __indirectly_binary_left_foldable<_Tp, iterator_t<_Rp>> _Fp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()(_Rp&& __r, _Tp __init, _Fp __f) {
+    return fold_left_with_iter(ranges::begin(__r), ranges::end(__r), std::move(__init), std::ref(__f)).value;
+  }
+};
+
+inline constexpr auto fold_left = __fold_left();
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_FOLD_H
diff --git a/libcxx/include/__cxx03/__algorithm/for_each.h b/libcxx/include/__cxx03/__algorithm/for_each.h
new file mode 100644
index 000000000000..259e527f87f9
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/for_each.h
@@ -0,0 +1,57 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_FOR_EACH_H
+#define _LIBCPP___ALGORITHM_FOR_EACH_H
+
+#include <__algorithm/for_each_segment.h>
+#include <__config>
+#include <__iterator/segmented_iterator.h>
+#include <__ranges/movable_box.h>
+#include <__type_traits/enable_if.h>
+#include <__utility/in_place.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _Function>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Function
+for_each(_InputIterator __first, _InputIterator __last, _Function __f) {
+  for (; __first != __last; ++__first)
+    __f(*__first);
+  return __f;
+}
+
+// __movable_box is available in C++20, but is actually a copyable-box, so optimization is only correct in C++23
+#if _LIBCPP_STD_VER >= 23
+template <class _SegmentedIterator, class _Function>
+  requires __is_segmented_iterator<_SegmentedIterator>::value
+_LIBCPP_HIDE_FROM_ABI constexpr _Function
+for_each(_SegmentedIterator __first, _SegmentedIterator __last, _Function __func) {
+  ranges::__movable_box<_Function> __wrapped_func(in_place, std::move(__func));
+  std::__for_each_segment(__first, __last, [&](auto __lfirst, auto __llast) {
+    __wrapped_func =
+        ranges::__movable_box<_Function>(in_place, std::for_each(__lfirst, __llast, std::move(*__wrapped_func)));
+  });
+  return std::move(*__wrapped_func);
+}
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_FOR_EACH_H
diff --git a/libcxx/include/__cxx03/__algorithm/for_each_n.h b/libcxx/include/__cxx03/__algorithm/for_each_n.h
new file mode 100644
index 000000000000..fce380b49df3
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/for_each_n.h
@@ -0,0 +1,41 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_FOR_EACH_N_H
+#define _LIBCPP___ALGORITHM_FOR_EACH_N_H
+
+#include <__config>
+#include <__utility/convert_to_integral.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _InputIterator, class _Size, class _Function>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
+for_each_n(_InputIterator __first, _Size __orig_n, _Function __f) {
+  typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize;
+  _IntegralSize __n = __orig_n;
+  while (__n > 0) {
+    __f(*__first);
+    ++__first;
+    --__n;
+  }
+  return __first;
+}
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_FOR_EACH_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/for_each_segment.h b/libcxx/include/__cxx03/__algorithm/for_each_segment.h
new file mode 100644
index 000000000000..93aa8259b2f7
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/for_each_segment.h
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_FOR_EACH_SEGMENT_H
+#define _LIBCPP___ALGORITHM_FOR_EACH_SEGMENT_H
+
+#include <__config>
+#include <__iterator/segmented_iterator.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// __for_each_segment is a utility function for optimizing iterating over segmented iterators linearly.
+// __first and __last are expected to be a segmented range. __func is expected to take a range of local iterators.
+// Anything that is returned from __func is ignored.
+
+template <class _SegmentedIterator, class _Functor>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+__for_each_segment(_SegmentedIterator __first, _SegmentedIterator __last, _Functor __func) {
+  using _Traits = __segmented_iterator_traits<_SegmentedIterator>;
+
+  auto __sfirst = _Traits::__segment(__first);
+  auto __slast  = _Traits::__segment(__last);
+
+  // We are in a single segment, so we might not be at the beginning or end
+  if (__sfirst == __slast) {
+    __func(_Traits::__local(__first), _Traits::__local(__last));
+    return;
+  }
+
+  // We have more than one segment. Iterate over the first segment, since we might not start at the beginning
+  __func(_Traits::__local(__first), _Traits::__end(__sfirst));
+  ++__sfirst;
+  // iterate over the segments which are guaranteed to be completely in the range
+  while (__sfirst != __slast) {
+    __func(_Traits::__begin(__sfirst), _Traits::__end(__sfirst));
+    ++__sfirst;
+  }
+  // iterate over the last segment
+  __func(_Traits::__begin(__sfirst), _Traits::__local(__last));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_FOR_EACH_SEGMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/generate.h b/libcxx/include/__cxx03/__algorithm/generate.h
new file mode 100644
index 000000000000..c95b527402f5
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/generate.h
@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_GENERATE_H
+#define _LIBCPP___ALGORITHM_GENERATE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ForwardIterator, class _Generator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+generate(_ForwardIterator __first, _ForwardIterator __last, _Generator __gen) {
+  for (; __first != __last; ++__first)
+    *__first = __gen();
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_GENERATE_H
diff --git a/libcxx/include/__cxx03/__algorithm/generate_n.h b/libcxx/include/__cxx03/__algorithm/generate_n.h
new file mode 100644
index 000000000000..f36403fd0f94
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/generate_n.h
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_GENERATE_N_H
+#define _LIBCPP___ALGORITHM_GENERATE_N_H
+
+#include <__config>
+#include <__utility/convert_to_integral.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _OutputIterator, class _Size, class _Generator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+generate_n(_OutputIterator __first, _Size __orig_n, _Generator __gen) {
+  typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize;
+  _IntegralSize __n = __orig_n;
+  for (; __n > 0; ++__first, (void)--__n)
+    *__first = __gen();
+  return __first;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_GENERATE_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/half_positive.h b/libcxx/include/__cxx03/__algorithm/half_positive.h
new file mode 100644
index 000000000000..ebda0da37236
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/half_positive.h
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_HALF_POSITIVE_H
+#define _LIBCPP___ALGORITHM_HALF_POSITIVE_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/make_unsigned.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Perform division by two quickly for positive integers (llvm.org/PR39129)
+
+template <typename _Integral, __enable_if_t<is_integral<_Integral>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Integral __half_positive(_Integral __value) {
+  return static_cast<_Integral>(static_cast<__make_unsigned_t<_Integral> >(__value) / 2);
+}
+
+template <typename _Tp, __enable_if_t<!is_integral<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp __half_positive(_Tp __value) {
+  return __value / 2;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_HALF_POSITIVE_H
diff --git a/libcxx/include/__cxx03/__algorithm/in_found_result.h b/libcxx/include/__cxx03/__algorithm/in_found_result.h
new file mode 100644
index 000000000000..a67ae387974c
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/in_found_result.h
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_IN_FOUND_RESULT_H
+#define _LIBCPP___ALGORITHM_IN_FOUND_RESULT_H
+
+#include <__concepts/convertible_to.h>
+#include <__config>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+template <class _InIter1>
+struct in_found_result {
+  _LIBCPP_NO_UNIQUE_ADDRESS _InIter1 in;
+  bool found;
+
+  template <class _InIter2>
+    requires convertible_to<const _InIter1&, _InIter2>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator in_found_result<_InIter2>() const& {
+    return {in, found};
+  }
+
+  template <class _InIter2>
+    requires convertible_to<_InIter1, _InIter2>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator in_found_result<_InIter2>() && {
+    return {std::move(in), found};
+  }
+};
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_IN_FOUND_RESULT_H
diff --git a/libcxx/include/__cxx03/__algorithm/in_fun_result.h b/libcxx/include/__cxx03/__algorithm/in_fun_result.h
new file mode 100644
index 000000000000..a22069a9a8dd
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/in_fun_result.h
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_IN_FUN_RESULT_H
+#define _LIBCPP___ALGORITHM_IN_FUN_RESULT_H
+
+#include <__concepts/convertible_to.h>
+#include <__config>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+template <class _InIter1, class _Func1>
+struct in_fun_result {
+  _LIBCPP_NO_UNIQUE_ADDRESS _InIter1 in;
+  _LIBCPP_NO_UNIQUE_ADDRESS _Func1 fun;
+
+  template <class _InIter2, class _Func2>
+    requires convertible_to<const _InIter1&, _InIter2> && convertible_to<const _Func1&, _Func2>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator in_fun_result<_InIter2, _Func2>() const& {
+    return {in, fun};
+  }
+
+  template <class _InIter2, class _Func2>
+    requires convertible_to<_InIter1, _InIter2> && convertible_to<_Func1, _Func2>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator in_fun_result<_InIter2, _Func2>() && {
+    return {std::move(in), std::move(fun)};
+  }
+};
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_IN_FUN_RESULT_H
diff --git a/libcxx/include/__cxx03/__algorithm/in_in_out_result.h b/libcxx/include/__cxx03/__algorithm/in_in_out_result.h
new file mode 100644
index 000000000000..ba0380b5c681
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/in_in_out_result.h
@@ -0,0 +1,59 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_IN_IN_OUT_RESULT_H
+#define _LIBCPP___ALGORITHM_IN_IN_OUT_RESULT_H
+
+#include <__concepts/convertible_to.h>
+#include <__config>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+template <class _InIter1, class _InIter2, class _OutIter1>
+struct in_in_out_result {
+  _LIBCPP_NO_UNIQUE_ADDRESS _InIter1 in1;
+  _LIBCPP_NO_UNIQUE_ADDRESS _InIter2 in2;
+  _LIBCPP_NO_UNIQUE_ADDRESS _OutIter1 out;
+
+  template <class _InIter3, class _InIter4, class _OutIter2>
+    requires convertible_to<const _InIter1&, _InIter3> && convertible_to<const _InIter2&, _InIter4> &&
+             convertible_to<const _OutIter1&, _OutIter2>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator in_in_out_result<_InIter3, _InIter4, _OutIter2>() const& {
+    return {in1, in2, out};
+  }
+
+  template <class _InIter3, class _InIter4, class _OutIter2>
+    requires convertible_to<_InIter1, _InIter3> && convertible_to<_InIter2, _InIter4> &&
+             convertible_to<_OutIter1, _OutIter2>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator in_in_out_result<_InIter3, _InIter4, _OutIter2>() && {
+    return {std::move(in1), std::move(in2), std::move(out)};
+  }
+};
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_IN_IN_OUT_RESULT_H
diff --git a/libcxx/include/__cxx03/__algorithm/in_in_result.h b/libcxx/include/__cxx03/__algorithm/in_in_result.h
new file mode 100644
index 000000000000..994573fc70fd
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/in_in_result.h
@@ -0,0 +1,56 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_IN_IN_RESULT_H
+#define _LIBCPP___ALGORITHM_IN_IN_RESULT_H
+
+#include <__concepts/convertible_to.h>
+#include <__config>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+template <class _InIter1, class _InIter2>
+struct in_in_result {
+  _LIBCPP_NO_UNIQUE_ADDRESS _InIter1 in1;
+  _LIBCPP_NO_UNIQUE_ADDRESS _InIter2 in2;
+
+  template <class _InIter3, class _InIter4>
+    requires convertible_to<const _InIter1&, _InIter3> && convertible_to<const _InIter2&, _InIter4>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator in_in_result<_InIter3, _InIter4>() const& {
+    return {in1, in2};
+  }
+
+  template <class _InIter3, class _InIter4>
+    requires convertible_to<_InIter1, _InIter3> && convertible_to<_InIter2, _InIter4>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator in_in_result<_InIter3, _InIter4>() && {
+    return {std::move(in1), std::move(in2)};
+  }
+};
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_IN_IN_RESULT_H
diff --git a/libcxx/include/__cxx03/__algorithm/in_out_out_result.h b/libcxx/include/__cxx03/__algorithm/in_out_out_result.h
new file mode 100644
index 000000000000..8ceb452841a4
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/in_out_out_result.h
@@ -0,0 +1,57 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_IN_OUT_OUT_RESULT_H
+#define _LIBCPP___ALGORITHM_IN_OUT_OUT_RESULT_H
+
+#include <__concepts/convertible_to.h>
+#include <__config>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+template <class _InIter1, class _OutIter1, class _OutIter2>
+struct in_out_out_result {
+  _LIBCPP_NO_UNIQUE_ADDRESS _InIter1 in;
+  _LIBCPP_NO_UNIQUE_ADDRESS _OutIter1 out1;
+  _LIBCPP_NO_UNIQUE_ADDRESS _OutIter2 out2;
+
+  template <class _InIter2, class _OutIter3, class _OutIter4>
+    requires convertible_to<const _InIter1&, _InIter2> && convertible_to<const _OutIter1&, _OutIter3> &&
+             convertible_to<const _OutIter2&, _OutIter4>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator in_out_out_result<_InIter2, _OutIter3, _OutIter4>() const& {
+    return {in, out1, out2};
+  }
+
+  template <class _InIter2, class _OutIter3, class _OutIter4>
+    requires convertible_to<_InIter1, _InIter2> && convertible_to<_OutIter1, _OutIter3> &&
+             convertible_to<_OutIter2, _OutIter4>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator in_out_out_result<_InIter2, _OutIter3, _OutIter4>() && {
+    return {std::move(in), std::move(out1), std::move(out2)};
+  }
+};
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_IN_OUT_OUT_RESULT_H
diff --git a/libcxx/include/__cxx03/__algorithm/in_out_result.h b/libcxx/include/__cxx03/__algorithm/in_out_result.h
new file mode 100644
index 000000000000..a7a986cf8e6c
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/in_out_result.h
@@ -0,0 +1,56 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_IN_OUT_RESULT_H
+#define _LIBCPP___ALGORITHM_IN_OUT_RESULT_H
+
+#include <__concepts/convertible_to.h>
+#include <__config>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+template <class _InIter1, class _OutIter1>
+struct in_out_result {
+  _LIBCPP_NO_UNIQUE_ADDRESS _InIter1 in;
+  _LIBCPP_NO_UNIQUE_ADDRESS _OutIter1 out;
+
+  template <class _InIter2, class _OutIter2>
+    requires convertible_to<const _InIter1&, _InIter2> && convertible_to<const _OutIter1&, _OutIter2>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator in_out_result<_InIter2, _OutIter2>() const& {
+    return {in, out};
+  }
+
+  template <class _InIter2, class _OutIter2>
+    requires convertible_to<_InIter1, _InIter2> && convertible_to<_OutIter1, _OutIter2>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator in_out_result<_InIter2, _OutIter2>() && {
+    return {std::move(in), std::move(out)};
+  }
+};
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_IN_OUT_RESULT_H
diff --git a/libcxx/include/__cxx03/__algorithm/includes.h b/libcxx/include/__cxx03/__algorithm/includes.h
new file mode 100644
index 000000000000..62af03c37426
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/includes.h
@@ -0,0 +1,79 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_INCLUDES_H
+#define _LIBCPP___ALGORITHM_INCLUDES_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/is_callable.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class _Comp, class _Proj1, class _Proj2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __includes(
+    _Iter1 __first1,
+    _Sent1 __last1,
+    _Iter2 __first2,
+    _Sent2 __last2,
+    _Comp&& __comp,
+    _Proj1&& __proj1,
+    _Proj2&& __proj2) {
+  for (; __first2 != __last2; ++__first1) {
+    if (__first1 == __last1 ||
+        std::__invoke(__comp, std::__invoke(__proj2, *__first2), std::__invoke(__proj1, *__first1)))
+      return false;
+    if (!std::__invoke(__comp, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2)))
+      ++__first2;
+  }
+  return true;
+}
+
+template <class _InputIterator1, class _InputIterator2, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+includes(_InputIterator1 __first1,
+         _InputIterator1 __last1,
+         _InputIterator2 __first2,
+         _InputIterator2 __last2,
+         _Compare __comp) {
+  static_assert(
+      __is_callable<_Compare, decltype(*__first1), decltype(*__first2)>::value, "Comparator has to be callable");
+
+  return std::__includes(
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__last2),
+      static_cast<__comp_ref_type<_Compare> >(__comp),
+      __identity(),
+      __identity());
+}
+
+template <class _InputIterator1, class _InputIterator2>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+includes(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2) {
+  return std::includes(std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_INCLUDES_H
diff --git a/libcxx/include/__cxx03/__algorithm/inplace_merge.h b/libcxx/include/__cxx03/__algorithm/inplace_merge.h
new file mode 100644
index 000000000000..a6bcc66a2fa4
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/inplace_merge.h
@@ -0,0 +1,240 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_INPLACE_MERGE_H
+#define _LIBCPP___ALGORITHM_INPLACE_MERGE_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/lower_bound.h>
+#include <__algorithm/min.h>
+#include <__algorithm/move.h>
+#include <__algorithm/rotate.h>
+#include <__algorithm/upper_bound.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__iterator/advance.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/reverse_iterator.h>
+#include <__memory/destruct_n.h>
+#include <__memory/temporary_buffer.h>
+#include <__memory/unique_ptr.h>
+#include <__utility/pair.h>
+#include <new>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Predicate>
+class __invert // invert the sense of a comparison
+{
+private:
+  _Predicate __p_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __invert() {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit __invert(_Predicate __p) : __p_(__p) {}
+
+  template <class _T1>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _T1& __x) {
+    return !__p_(__x);
+  }
+
+  template <class _T1, class _T2>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _T1& __x, const _T2& __y) {
+    return __p_(__y, __x);
+  }
+};
+
+template <class _AlgPolicy,
+          class _Compare,
+          class _InputIterator1,
+          class _Sent1,
+          class _InputIterator2,
+          class _Sent2,
+          class _OutputIterator>
+_LIBCPP_HIDE_FROM_ABI void __half_inplace_merge(
+    _InputIterator1 __first1,
+    _Sent1 __last1,
+    _InputIterator2 __first2,
+    _Sent2 __last2,
+    _OutputIterator __result,
+    _Compare&& __comp) {
+  for (; __first1 != __last1; ++__result) {
+    if (__first2 == __last2) {
+      std::__move<_AlgPolicy>(__first1, __last1, __result);
+      return;
+    }
+
+    if (__comp(*__first2, *__first1)) {
+      *__result = _IterOps<_AlgPolicy>::__iter_move(__first2);
+      ++__first2;
+    } else {
+      *__result = _IterOps<_AlgPolicy>::__iter_move(__first1);
+      ++__first1;
+    }
+  }
+  // __first2 through __last2 are already in the right spot.
+}
+
+template <class _AlgPolicy, class _Compare, class _BidirectionalIterator>
+_LIBCPP_HIDE_FROM_ABI void __buffered_inplace_merge(
+    _BidirectionalIterator __first,
+    _BidirectionalIterator __middle,
+    _BidirectionalIterator __last,
+    _Compare&& __comp,
+    typename iterator_traits<_BidirectionalIterator>::difference_type __len1,
+    typename iterator_traits<_BidirectionalIterator>::difference_type __len2,
+    typename iterator_traits<_BidirectionalIterator>::value_type* __buff) {
+  typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type;
+  __destruct_n __d(0);
+  unique_ptr<value_type, __destruct_n&> __h2(__buff, __d);
+  if (__len1 <= __len2) {
+    value_type* __p = __buff;
+    for (_BidirectionalIterator __i = __first; __i != __middle;
+         __d.template __incr<value_type>(), (void)++__i, (void)++__p)
+      ::new ((void*)__p) value_type(_IterOps<_AlgPolicy>::__iter_move(__i));
+    std::__half_inplace_merge<_AlgPolicy>(__buff, __p, __middle, __last, __first, __comp);
+  } else {
+    value_type* __p = __buff;
+    for (_BidirectionalIterator __i = __middle; __i != __last;
+         __d.template __incr<value_type>(), (void)++__i, (void)++__p)
+      ::new ((void*)__p) value_type(_IterOps<_AlgPolicy>::__iter_move(__i));
+    typedef reverse_iterator<_BidirectionalIterator> _RBi;
+    typedef reverse_iterator<value_type*> _Rv;
+    typedef __invert<_Compare> _Inverted;
+    std::__half_inplace_merge<_AlgPolicy>(
+        _Rv(__p), _Rv(__buff), _RBi(__middle), _RBi(__first), _RBi(__last), _Inverted(__comp));
+  }
+}
+
+template <class _AlgPolicy, class _Compare, class _BidirectionalIterator>
+void __inplace_merge(
+    _BidirectionalIterator __first,
+    _BidirectionalIterator __middle,
+    _BidirectionalIterator __last,
+    _Compare&& __comp,
+    typename iterator_traits<_BidirectionalIterator>::difference_type __len1,
+    typename iterator_traits<_BidirectionalIterator>::difference_type __len2,
+    typename iterator_traits<_BidirectionalIterator>::value_type* __buff,
+    ptrdiff_t __buff_size) {
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  typedef typename iterator_traits<_BidirectionalIterator>::difference_type difference_type;
+  while (true) {
+    // if __middle == __last, we're done
+    if (__len2 == 0)
+      return;
+    if (__len1 <= __buff_size || __len2 <= __buff_size)
+      return std::__buffered_inplace_merge<_AlgPolicy>(__first, __middle, __last, __comp, __len1, __len2, __buff);
+    // shrink [__first, __middle) as much as possible (with no moves), returning if it shrinks to 0
+    for (; true; ++__first, (void)--__len1) {
+      if (__len1 == 0)
+        return;
+      if (__comp(*__middle, *__first))
+        break;
+    }
+    // __first < __middle < __last
+    // *__first > *__middle
+    // partition [__first, __m1) [__m1, __middle) [__middle, __m2) [__m2, __last) such that
+    //     all elements in:
+    //         [__first, __m1)  <= [__middle, __m2)
+    //         [__middle, __m2) <  [__m1, __middle)
+    //         [__m1, __middle) <= [__m2, __last)
+    //     and __m1 or __m2 is in the middle of its range
+    _BidirectionalIterator __m1; // "median" of [__first, __middle)
+    _BidirectionalIterator __m2; // "median" of [__middle, __last)
+    difference_type __len11;     // distance(__first, __m1)
+    difference_type __len21;     // distance(__middle, __m2)
+    // binary search smaller range
+    if (__len1 < __len2) { // __len >= 1, __len2 >= 2
+      __len21 = __len2 / 2;
+      __m2    = __middle;
+      _Ops::advance(__m2, __len21);
+      __m1    = std::__upper_bound<_AlgPolicy>(__first, __middle, *__m2, __comp, std::__identity());
+      __len11 = _Ops::distance(__first, __m1);
+    } else {
+      if (__len1 == 1) { // __len1 >= __len2 && __len2 > 0, therefore __len2 == 1
+                         // It is known *__first > *__middle
+        _Ops::iter_swap(__first, __middle);
+        return;
+      }
+      // __len1 >= 2, __len2 >= 1
+      __len11 = __len1 / 2;
+      __m1    = __first;
+      _Ops::advance(__m1, __len11);
+      __m2    = std::lower_bound(__middle, __last, *__m1, __comp);
+      __len21 = _Ops::distance(__middle, __m2);
+    }
+    difference_type __len12 = __len1 - __len11; // distance(__m1, __middle)
+    difference_type __len22 = __len2 - __len21; // distance(__m2, __last)
+    // [__first, __m1) [__m1, __middle) [__middle, __m2) [__m2, __last)
+    // swap middle two partitions
+    __middle = std::__rotate<_AlgPolicy>(__m1, __middle, __m2).first;
+    // __len12 and __len21 now have swapped meanings
+    // merge smaller range with recursive call and larger with tail recursion elimination
+    if (__len11 + __len21 < __len12 + __len22) {
+      std::__inplace_merge<_AlgPolicy>(__first, __m1, __middle, __comp, __len11, __len21, __buff, __buff_size);
+      __first  = __middle;
+      __middle = __m2;
+      __len1   = __len12;
+      __len2   = __len22;
+    } else {
+      std::__inplace_merge<_AlgPolicy>(__middle, __m2, __last, __comp, __len12, __len22, __buff, __buff_size);
+      __last   = __middle;
+      __middle = __m1;
+      __len1   = __len11;
+      __len2   = __len21;
+    }
+  }
+}
+
+template <class _AlgPolicy, class _BidirectionalIterator, class _Compare>
+_LIBCPP_HIDE_FROM_ABI void __inplace_merge(
+    _BidirectionalIterator __first, _BidirectionalIterator __middle, _BidirectionalIterator __last, _Compare&& __comp) {
+  typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type;
+  typedef typename iterator_traits<_BidirectionalIterator>::difference_type difference_type;
+  difference_type __len1     = _IterOps<_AlgPolicy>::distance(__first, __middle);
+  difference_type __len2     = _IterOps<_AlgPolicy>::distance(__middle, __last);
+  difference_type __buf_size = std::min(__len1, __len2);
+  // TODO: Remove the use of std::get_temporary_buffer
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  pair<value_type*, ptrdiff_t> __buf = std::get_temporary_buffer<value_type>(__buf_size);
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  unique_ptr<value_type, __return_temporary_buffer> __h(__buf.first);
+  return std::__inplace_merge<_AlgPolicy>(
+      std::move(__first), std::move(__middle), std::move(__last), __comp, __len1, __len2, __buf.first, __buf.second);
+}
+
+template <class _BidirectionalIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI void inplace_merge(
+    _BidirectionalIterator __first, _BidirectionalIterator __middle, _BidirectionalIterator __last, _Compare __comp) {
+  std::__inplace_merge<_ClassicAlgPolicy>(
+      std::move(__first), std::move(__middle), std::move(__last), static_cast<__comp_ref_type<_Compare> >(__comp));
+}
+
+template <class _BidirectionalIterator>
+inline _LIBCPP_HIDE_FROM_ABI void
+inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator __middle, _BidirectionalIterator __last) {
+  std::inplace_merge(std::move(__first), std::move(__middle), std::move(__last), __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_INPLACE_MERGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/is_heap.h b/libcxx/include/__cxx03/__algorithm/is_heap.h
new file mode 100644
index 000000000000..c589b804a5dc
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/is_heap.h
@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_IS_HEAP_H
+#define _LIBCPP___ALGORITHM_IS_HEAP_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/is_heap_until.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _RandomAccessIterator, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+is_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+  return std::__is_heap_until(__first, __last, static_cast<__comp_ref_type<_Compare> >(__comp)) == __last;
+}
+
+template <class _RandomAccessIterator>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+is_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) {
+  return std::is_heap(__first, __last, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_IS_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/is_heap_until.h b/libcxx/include/__cxx03/__algorithm/is_heap_until.h
new file mode 100644
index 000000000000..a174f2453cfc
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/is_heap_until.h
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_IS_HEAP_UNTIL_H
+#define _LIBCPP___ALGORITHM_IS_HEAP_UNTIL_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Compare, class _RandomAccessIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator
+__is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare&& __comp) {
+  typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  difference_type __len      = __last - __first;
+  difference_type __p        = 0;
+  difference_type __c        = 1;
+  _RandomAccessIterator __pp = __first;
+  while (__c < __len) {
+    _RandomAccessIterator __cp = __first + __c;
+    if (__comp(*__pp, *__cp))
+      return __cp;
+    ++__c;
+    ++__cp;
+    if (__c == __len)
+      return __last;
+    if (__comp(*__pp, *__cp))
+      return __cp;
+    ++__p;
+    ++__pp;
+    __c = 2 * __p + 1;
+  }
+  return __last;
+}
+
+template <class _RandomAccessIterator, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator
+is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+  return std::__is_heap_until(__first, __last, static_cast<__comp_ref_type<_Compare> >(__comp));
+}
+
+template <class _RandomAccessIterator>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator
+is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last) {
+  return std::__is_heap_until(__first, __last, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_IS_HEAP_UNTIL_H
diff --git a/libcxx/include/__cxx03/__algorithm/is_partitioned.h b/libcxx/include/__cxx03/__algorithm/is_partitioned.h
new file mode 100644
index 000000000000..1f7c8b0b267e
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/is_partitioned.h
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_IS_PARTITIONED_H
+#define _LIBCPP___ALGORITHM_IS_PARTITIONED_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _Predicate>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+is_partitioned(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
+  for (; __first != __last; ++__first)
+    if (!__pred(*__first))
+      break;
+  if (__first == __last)
+    return true;
+  ++__first;
+  for (; __first != __last; ++__first)
+    if (__pred(*__first))
+      return false;
+  return true;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_IS_PARTITIONED_H
diff --git a/libcxx/include/__cxx03/__algorithm/is_permutation.h b/libcxx/include/__cxx03/__algorithm/is_permutation.h
new file mode 100644
index 000000000000..2ddfb32a212b
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/is_permutation.h
@@ -0,0 +1,308 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_IS_PERMUTATION_H
+#define _LIBCPP___ALGORITHM_IS_PERMUTATION_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__type_traits/is_callable.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class = void>
+struct _ConstTimeDistance : false_type {};
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Iter1, class _Sent1, class _Iter2, class _Sent2>
+struct _ConstTimeDistance<_Iter1,
+                          _Sent1,
+                          _Iter2,
+                          _Sent2,
+                          __enable_if_t< sized_sentinel_for<_Sent1, _Iter1> && sized_sentinel_for<_Sent2, _Iter2> >>
+    : true_type {};
+
+#else
+
+template <class _Iter1, class _Iter2>
+struct _ConstTimeDistance<
+    _Iter1,
+    _Iter1,
+    _Iter2,
+    _Iter2,
+    __enable_if_t< is_same<typename iterator_traits<_Iter1>::iterator_category, random_access_iterator_tag>::value &&
+                   is_same<typename iterator_traits<_Iter2>::iterator_category, random_access_iterator_tag>::value > >
+    : true_type {};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+// Internal functions
+
+// For each element in [f1, l1) see if there are the same number of equal elements in [f2, l2)
+template <class _AlgPolicy,
+          class _Iter1,
+          class _Sent1,
+          class _Iter2,
+          class _Sent2,
+          class _Proj1,
+          class _Proj2,
+          class _Pred>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __is_permutation_impl(
+    _Iter1 __first1,
+    _Sent1 __last1,
+    _Iter2 __first2,
+    _Sent2 __last2,
+    _Pred&& __pred,
+    _Proj1&& __proj1,
+    _Proj2&& __proj2) {
+  using _D1 = __iter_diff_t<_Iter1>;
+
+  for (auto __i = __first1; __i != __last1; ++__i) {
+    //  Have we already counted the number of *__i in [f1, l1)?
+    auto __match = __first1;
+    for (; __match != __i; ++__match) {
+      if (std::__invoke(__pred, std::__invoke(__proj1, *__match), std::__invoke(__proj1, *__i)))
+        break;
+    }
+
+    if (__match == __i) {
+      // Count number of *__i in [f2, l2)
+      _D1 __c2 = 0;
+      for (auto __j = __first2; __j != __last2; ++__j) {
+        if (std::__invoke(__pred, std::__invoke(__proj1, *__i), std::__invoke(__proj2, *__j)))
+          ++__c2;
+      }
+      if (__c2 == 0)
+        return false;
+
+      // Count number of *__i in [__i, l1) (we can start with 1)
+      _D1 __c1 = 1;
+      for (auto __j = _IterOps<_AlgPolicy>::next(__i); __j != __last1; ++__j) {
+        if (std::__invoke(__pred, std::__invoke(__proj1, *__i), std::__invoke(__proj1, *__j)))
+          ++__c1;
+      }
+      if (__c1 != __c2)
+        return false;
+    }
+  }
+
+  return true;
+}
+
+// 2+1 iterators, predicate. Not used by range algorithms.
+template <class _AlgPolicy, class _ForwardIterator1, class _Sentinel1, class _ForwardIterator2, class _BinaryPredicate>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __is_permutation(
+    _ForwardIterator1 __first1, _Sentinel1 __last1, _ForwardIterator2 __first2, _BinaryPredicate&& __pred) {
+  // Shorten sequences as much as possible by lopping of any equal prefix.
+  for (; __first1 != __last1; ++__first1, (void)++__first2) {
+    if (!__pred(*__first1, *__first2))
+      break;
+  }
+
+  if (__first1 == __last1)
+    return true;
+
+  //  __first1 != __last1 && *__first1 != *__first2
+  using _D1 = __iter_diff_t<_ForwardIterator1>;
+  _D1 __l1  = _IterOps<_AlgPolicy>::distance(__first1, __last1);
+  if (__l1 == _D1(1))
+    return false;
+  auto __last2 = _IterOps<_AlgPolicy>::next(__first2, __l1);
+
+  return std::__is_permutation_impl<_AlgPolicy>(
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__last2),
+      __pred,
+      __identity(),
+      __identity());
+}
+
+// 2+2 iterators, predicate, non-constant time `distance`.
+template <class _AlgPolicy,
+          class _Iter1,
+          class _Sent1,
+          class _Iter2,
+          class _Sent2,
+          class _Proj1,
+          class _Proj2,
+          class _Pred>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __is_permutation(
+    _Iter1 __first1,
+    _Sent1 __last1,
+    _Iter2 __first2,
+    _Sent2 __last2,
+    _Pred&& __pred,
+    _Proj1&& __proj1,
+    _Proj2&& __proj2,
+    /*_ConstTimeDistance=*/false_type) {
+  // Shorten sequences as much as possible by lopping of any equal prefix.
+  while (__first1 != __last1 && __first2 != __last2) {
+    if (!std::__invoke(__pred, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2)))
+      break;
+    ++__first1;
+    ++__first2;
+  }
+
+  if (__first1 == __last1)
+    return __first2 == __last2;
+  if (__first2 == __last2) // Second range is shorter
+    return false;
+
+  using _D1 = __iter_diff_t<_Iter1>;
+  _D1 __l1  = _IterOps<_AlgPolicy>::distance(__first1, __last1);
+
+  using _D2 = __iter_diff_t<_Iter2>;
+  _D2 __l2  = _IterOps<_AlgPolicy>::distance(__first2, __last2);
+  if (__l1 != __l2)
+    return false;
+
+  return std::__is_permutation_impl<_AlgPolicy>(
+      std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __pred, __proj1, __proj2);
+}
+
+// 2+2 iterators, predicate, specialization for constant-time `distance` call.
+template <class _AlgPolicy,
+          class _Iter1,
+          class _Sent1,
+          class _Iter2,
+          class _Sent2,
+          class _Proj1,
+          class _Proj2,
+          class _Pred>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __is_permutation(
+    _Iter1 __first1,
+    _Sent1 __last1,
+    _Iter2 __first2,
+    _Sent2 __last2,
+    _Pred&& __pred,
+    _Proj1&& __proj1,
+    _Proj2&& __proj2,
+    /*_ConstTimeDistance=*/true_type) {
+  if (std::distance(__first1, __last1) != std::distance(__first2, __last2))
+    return false;
+  return std::__is_permutation<_AlgPolicy>(
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__last2),
+      __pred,
+      __proj1,
+      __proj2,
+      /*_ConstTimeDistance=*/false_type());
+}
+
+// 2+2 iterators, predicate
+template <class _AlgPolicy,
+          class _Iter1,
+          class _Sent1,
+          class _Iter2,
+          class _Sent2,
+          class _Proj1,
+          class _Proj2,
+          class _Pred>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __is_permutation(
+    _Iter1 __first1,
+    _Sent1 __last1,
+    _Iter2 __first2,
+    _Sent2 __last2,
+    _Pred&& __pred,
+    _Proj1&& __proj1,
+    _Proj2&& __proj2) {
+  return std::__is_permutation<_AlgPolicy>(
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__last2),
+      __pred,
+      __proj1,
+      __proj2,
+      _ConstTimeDistance<_Iter1, _Sent1, _Iter2, _Sent2>());
+}
+
+// Public interface
+
+// 2+1 iterators, predicate
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool is_permutation(
+    _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _BinaryPredicate __pred) {
+  static_assert(__is_callable<_BinaryPredicate, decltype(*__first1), decltype(*__first2)>::value,
+                "The predicate has to be callable");
+
+  return std::__is_permutation<_ClassicAlgPolicy>(std::move(__first1), std::move(__last1), std::move(__first2), __pred);
+}
+
+// 2+1 iterators
+template <class _ForwardIterator1, class _ForwardIterator2>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+is_permutation(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2) {
+  return std::is_permutation(__first1, __last1, __first2, __equal_to());
+}
+
+#if _LIBCPP_STD_VER >= 14
+
+// 2+2 iterators
+template <class _ForwardIterator1, class _ForwardIterator2>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool is_permutation(
+    _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2) {
+  return std::__is_permutation<_ClassicAlgPolicy>(
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__last2),
+      __equal_to(),
+      __identity(),
+      __identity());
+}
+
+// 2+2 iterators, predicate
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool is_permutation(
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _BinaryPredicate __pred) {
+  static_assert(__is_callable<_BinaryPredicate, decltype(*__first1), decltype(*__first2)>::value,
+                "The predicate has to be callable");
+
+  return std::__is_permutation<_ClassicAlgPolicy>(
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__last2),
+      __pred,
+      __identity(),
+      __identity());
+}
+
+#endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_IS_PERMUTATION_H
diff --git a/libcxx/include/__cxx03/__algorithm/is_sorted.h b/libcxx/include/__cxx03/__algorithm/is_sorted.h
new file mode 100644
index 000000000000..3befb1ac9c26
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/is_sorted.h
@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_IS_SORTED_H
+#define _LIBCPP___ALGORITHM_IS_SORTED_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/is_sorted_until.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ForwardIterator, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+is_sorted(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) {
+  return std::__is_sorted_until<__comp_ref_type<_Compare> >(__first, __last, __comp) == __last;
+}
+
+template <class _ForwardIterator>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+is_sorted(_ForwardIterator __first, _ForwardIterator __last) {
+  return std::is_sorted(__first, __last, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_IS_SORTED_H
diff --git a/libcxx/include/__cxx03/__algorithm/is_sorted_until.h b/libcxx/include/__cxx03/__algorithm/is_sorted_until.h
new file mode 100644
index 000000000000..53a49f00de31
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/is_sorted_until.h
@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_IS_SORTED_UNTIL_H
+#define _LIBCPP___ALGORITHM_IS_SORTED_UNTIL_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Compare, class _ForwardIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+__is_sorted_until(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) {
+  if (__first != __last) {
+    _ForwardIterator __i = __first;
+    while (++__i != __last) {
+      if (__comp(*__i, *__first))
+        return __i;
+      __first = __i;
+    }
+  }
+  return __last;
+}
+
+template <class _ForwardIterator, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+is_sorted_until(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) {
+  return std::__is_sorted_until<__comp_ref_type<_Compare> >(__first, __last, __comp);
+}
+
+template <class _ForwardIterator>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+is_sorted_until(_ForwardIterator __first, _ForwardIterator __last) {
+  return std::is_sorted_until(__first, __last, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_IS_SORTED_UNTIL_H
diff --git a/libcxx/include/__cxx03/__algorithm/iter_swap.h b/libcxx/include/__cxx03/__algorithm/iter_swap.h
new file mode 100644
index 000000000000..a1412e5d8720
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/iter_swap.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_ITER_SWAP_H
+#define _LIBCPP___ALGORITHM_ITER_SWAP_H
+
+#include <__config>
+#include <__utility/declval.h>
+#include <__utility/swap.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ForwardIterator1, class _ForwardIterator2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void iter_swap(_ForwardIterator1 __a, _ForwardIterator2 __b)
+    //                                  _NOEXCEPT_(_NOEXCEPT_(swap(*__a, *__b)))
+    _NOEXCEPT_(_NOEXCEPT_(swap(*std::declval<_ForwardIterator1>(), *std::declval<_ForwardIterator2>()))) {
+  swap(*__a, *__b);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_ITER_SWAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/iterator_operations.h b/libcxx/include/__cxx03/__algorithm/iterator_operations.h
new file mode 100644
index 000000000000..8ced989233bc
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/iterator_operations.h
@@ -0,0 +1,223 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_ITERATOR_OPERATIONS_H
+#define _LIBCPP___ALGORITHM_ITERATOR_OPERATIONS_H
+
+#include <__algorithm/iter_swap.h>
+#include <__algorithm/ranges_iterator_concept.h>
+#include <__assert>
+#include <__config>
+#include <__iterator/advance.h>
+#include <__iterator/distance.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/iter_swap.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/prev.h>
+#include <__iterator/readable_traits.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy>
+struct _IterOps;
+
+#if _LIBCPP_STD_VER >= 20
+struct _RangeAlgPolicy {};
+
+template <>
+struct _IterOps<_RangeAlgPolicy> {
+  template <class _Iter>
+  using __value_type = iter_value_t<_Iter>;
+
+  template <class _Iter>
+  using __iterator_category = ranges::__iterator_concept<_Iter>;
+
+  template <class _Iter>
+  using __difference_type = iter_difference_t<_Iter>;
+
+  static constexpr auto advance      = ranges::advance;
+  static constexpr auto distance     = ranges::distance;
+  static constexpr auto __iter_move  = ranges::iter_move;
+  static constexpr auto iter_swap    = ranges::iter_swap;
+  static constexpr auto next         = ranges::next;
+  static constexpr auto prev         = ranges::prev;
+  static constexpr auto __advance_to = ranges::advance;
+};
+
+#endif
+
+struct _ClassicAlgPolicy {};
+
+template <>
+struct _IterOps<_ClassicAlgPolicy> {
+  template <class _Iter>
+  using __value_type = typename iterator_traits<_Iter>::value_type;
+
+  template <class _Iter>
+  using __iterator_category = typename iterator_traits<_Iter>::iterator_category;
+
+  template <class _Iter>
+  using __difference_type = typename iterator_traits<_Iter>::difference_type;
+
+  // advance
+  template <class _Iter, class _Distance>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static void advance(_Iter& __iter, _Distance __count) {
+    std::advance(__iter, __count);
+  }
+
+  // distance
+  template <class _Iter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static typename iterator_traits<_Iter>::difference_type
+  distance(_Iter __first, _Iter __last) {
+    return std::distance(__first, __last);
+  }
+
+  template <class _Iter>
+  using __deref_t = decltype(*std::declval<_Iter&>());
+
+  template <class _Iter>
+  using __move_t = decltype(std::move(*std::declval<_Iter&>()));
+
+  template <class _Iter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static void __validate_iter_reference() {
+    static_assert(
+        is_same<__deref_t<_Iter>, typename iterator_traits<__remove_cvref_t<_Iter> >::reference>::value,
+        "It looks like your iterator's `iterator_traits<It>::reference` does not match the return type of "
+        "dereferencing the iterator, i.e., calling `*it`. This is undefined behavior according to [input.iterators] "
+        "and can lead to dangling reference issues at runtime, so we are flagging this.");
+  }
+
+  // iter_move
+  template <class _Iter, __enable_if_t<is_reference<__deref_t<_Iter> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static
+      // If the result of dereferencing `_Iter` is a reference type, deduce the result of calling `std::move` on it.
+      // Note that the C++03 mode doesn't support `decltype(auto)` as the return type.
+      __move_t<_Iter>
+      __iter_move(_Iter&& __i) {
+    __validate_iter_reference<_Iter>();
+
+    return std::move(*std::forward<_Iter>(__i));
+  }
+
+  template <class _Iter, __enable_if_t<!is_reference<__deref_t<_Iter> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static
+      // If the result of dereferencing `_Iter` is a value type, deduce the return value of this function to also be a
+      // value -- otherwise, after `operator*` returns a temporary, this function would return a dangling reference to
+      // that temporary. Note that the C++03 mode doesn't support `auto` as the return type.
+      __deref_t<_Iter>
+      __iter_move(_Iter&& __i) {
+    __validate_iter_reference<_Iter>();
+
+    return *std::forward<_Iter>(__i);
+  }
+
+  // iter_swap
+  template <class _Iter1, class _Iter2>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static void iter_swap(_Iter1&& __a, _Iter2&& __b) {
+    std::iter_swap(std::forward<_Iter1>(__a), std::forward<_Iter2>(__b));
+  }
+
+  // next
+  template <class _Iterator>
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iterator next(_Iterator, _Iterator __last) {
+    return __last;
+  }
+
+  template <class _Iter>
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 __remove_cvref_t<_Iter>
+  next(_Iter&& __it, typename iterator_traits<__remove_cvref_t<_Iter> >::difference_type __n = 1) {
+    return std::next(std::forward<_Iter>(__it), __n);
+  }
+
+  // prev
+  template <class _Iter>
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 __remove_cvref_t<_Iter>
+  prev(_Iter&& __iter, typename iterator_traits<__remove_cvref_t<_Iter> >::difference_type __n = 1) {
+    return std::prev(std::forward<_Iter>(__iter), __n);
+  }
+
+  template <class _Iter>
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 void __advance_to(_Iter& __first, _Iter __last) {
+    __first = __last;
+  }
+
+  // advance with sentinel, a la std::ranges::advance
+  template <class _Iter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static __difference_type<_Iter>
+  __advance_to(_Iter& __iter, __difference_type<_Iter> __count, const _Iter& __sentinel) {
+    return _IterOps::__advance_to(__iter, __count, __sentinel, typename iterator_traits<_Iter>::iterator_category());
+  }
+
+private:
+  // advance with sentinel, a la std::ranges::advance -- InputIterator specialization
+  template <class _InputIter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static __difference_type<_InputIter> __advance_to(
+      _InputIter& __iter, __difference_type<_InputIter> __count, const _InputIter& __sentinel, input_iterator_tag) {
+    __difference_type<_InputIter> __dist = 0;
+    for (; __dist < __count && __iter != __sentinel; ++__dist)
+      ++__iter;
+    return __count - __dist;
+  }
+
+  // advance with sentinel, a la std::ranges::advance -- BidirectionalIterator specialization
+  template <class _BiDirIter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static __difference_type<_BiDirIter>
+  __advance_to(_BiDirIter& __iter,
+               __difference_type<_BiDirIter> __count,
+               const _BiDirIter& __sentinel,
+               bidirectional_iterator_tag) {
+    __difference_type<_BiDirIter> __dist = 0;
+    if (__count >= 0)
+      for (; __dist < __count && __iter != __sentinel; ++__dist)
+        ++__iter;
+    else
+      for (__count = -__count; __dist < __count && __iter != __sentinel; ++__dist)
+        --__iter;
+    return __count - __dist;
+  }
+
+  // advance with sentinel, a la std::ranges::advance -- RandomIterator specialization
+  template <class _RandIter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static __difference_type<_RandIter>
+  __advance_to(_RandIter& __iter,
+               __difference_type<_RandIter> __count,
+               const _RandIter& __sentinel,
+               random_access_iterator_tag) {
+    auto __dist = _IterOps::distance(__iter, __sentinel);
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(
+        __count == 0 || (__dist < 0) == (__count < 0), "__sentinel must precede __iter when __count < 0");
+    if (__count < 0)
+      __dist = __dist > __count ? __dist : __count;
+    else
+      __dist = __dist < __count ? __dist : __count;
+    __iter += __dist;
+    return __count - __dist;
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_ITERATOR_OPERATIONS_H
diff --git a/libcxx/include/__cxx03/__algorithm/lexicographical_compare.h b/libcxx/include/__cxx03/__algorithm/lexicographical_compare.h
new file mode 100644
index 000000000000..edc29e269c88
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/lexicographical_compare.h
@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_LEXICOGRAPHICAL_COMPARE_H
+#define _LIBCPP___ALGORITHM_LEXICOGRAPHICAL_COMPARE_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Compare, class _InputIterator1, class _InputIterator2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __lexicographical_compare(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _InputIterator2 __last2,
+    _Compare __comp) {
+  for (; __first2 != __last2; ++__first1, (void)++__first2) {
+    if (__first1 == __last1 || __comp(*__first1, *__first2))
+      return true;
+    if (__comp(*__first2, *__first1))
+      return false;
+  }
+  return false;
+}
+
+template <class _InputIterator1, class _InputIterator2, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool lexicographical_compare(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _InputIterator2 __last2,
+    _Compare __comp) {
+  return std::__lexicographical_compare<__comp_ref_type<_Compare> >(__first1, __last1, __first2, __last2, __comp);
+}
+
+template <class _InputIterator1, class _InputIterator2>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool lexicographical_compare(
+    _InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2) {
+  return std::lexicographical_compare(__first1, __last1, __first2, __last2, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_LEXICOGRAPHICAL_COMPARE_H
diff --git a/libcxx/include/__cxx03/__algorithm/lexicographical_compare_three_way.h b/libcxx/include/__cxx03/__algorithm/lexicographical_compare_three_way.h
new file mode 100644
index 000000000000..a5872e90cf8d
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/lexicographical_compare_three_way.h
@@ -0,0 +1,125 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_LEXICOGRAPHICAL_COMPARE_THREE_WAY_H
+#define _LIBCPP___ALGORITHM_LEXICOGRAPHICAL_COMPARE_THREE_WAY_H
+
+#include <__algorithm/min.h>
+#include <__algorithm/three_way_comp_ref_type.h>
+#include <__compare/compare_three_way.h>
+#include <__compare/ordering.h>
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/common_type.h>
+#include <__type_traits/is_constructible.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// Fast path for random access iterators which computes the number of loop iterations up-front and
+// then skips the iterator comparisons inside the loop.
+template <class _InputIterator1, class _InputIterator2, class _Cmp>
+_LIBCPP_HIDE_FROM_ABI constexpr auto __lexicographical_compare_three_way_fast_path(
+    _InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2, _Cmp& __comp)
+    -> decltype(__comp(*__first1, *__first2)) {
+  static_assert(
+      signed_integral<__iter_diff_t<_InputIterator1>>, "Using a non-integral difference_type is undefined behavior.");
+  static_assert(
+      signed_integral<__iter_diff_t<_InputIterator2>>, "Using a non-integral difference_type is undefined behavior.");
+
+  using _Len1   = __iter_diff_t<_InputIterator1>;
+  using _Len2   = __iter_diff_t<_InputIterator2>;
+  using _Common = common_type_t<_Len1, _Len2>;
+
+  _Len1 __len1      = __last1 - __first1;
+  _Len2 __len2      = __last2 - __first2;
+  _Common __min_len = std::min<_Common>(__len1, __len2);
+
+  for (_Common __i = 0; __i < __min_len; ++__i) {
+    auto __c = __comp(*__first1, *__first2);
+    if (__c != 0) {
+      return __c;
+    }
+    ++__first1;
+    ++__first2;
+  }
+
+  return __len1 <=> __len2;
+}
+
+// Unoptimized implementation which compares the iterators against the end in every loop iteration
+template <class _InputIterator1, class _InputIterator2, class _Cmp>
+_LIBCPP_HIDE_FROM_ABI constexpr auto __lexicographical_compare_three_way_slow_path(
+    _InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2, _Cmp& __comp)
+    -> decltype(__comp(*__first1, *__first2)) {
+  while (true) {
+    bool __exhausted1 = __first1 == __last1;
+    bool __exhausted2 = __first2 == __last2;
+
+    if (__exhausted1 || __exhausted2) {
+      if (!__exhausted1)
+        return strong_ordering::greater;
+      if (!__exhausted2)
+        return strong_ordering::less;
+      return strong_ordering::equal;
+    }
+
+    auto __c = __comp(*__first1, *__first2);
+    if (__c != 0) {
+      return __c;
+    }
+
+    ++__first1;
+    ++__first2;
+  }
+}
+
+template <class _InputIterator1, class _InputIterator2, class _Cmp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto lexicographical_compare_three_way(
+    _InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2, _Cmp __comp)
+    -> decltype(__comp(*__first1, *__first2)) {
+  static_assert(__comparison_category<decltype(__comp(*__first1, *__first2))>,
+                "The comparator passed to lexicographical_compare_three_way must return a comparison category type.");
+  static_assert(std::is_copy_constructible_v<_InputIterator1>, "Iterators must be copy constructible.");
+  static_assert(std::is_copy_constructible_v<_InputIterator2>, "Iterators must be copy constructible.");
+  __three_way_comp_ref_type<_Cmp> __wrapped_comp_ref(__comp);
+  if constexpr (__has_random_access_iterator_category<_InputIterator1>::value &&
+                __has_random_access_iterator_category<_InputIterator2>::value) {
+    return std::__lexicographical_compare_three_way_fast_path(
+        std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __wrapped_comp_ref);
+  } else {
+    // Unoptimized implementation which compares the iterators against the end in every loop iteration
+    return std::__lexicographical_compare_three_way_slow_path(
+        std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __wrapped_comp_ref);
+  }
+}
+
+template <class _InputIterator1, class _InputIterator2>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto lexicographical_compare_three_way(
+    _InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2) {
+  return std::lexicographical_compare_three_way(
+      std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), std::compare_three_way());
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_LEXICOGRAPHICAL_COMPARE_THREE_WAY_H
diff --git a/libcxx/include/__cxx03/__algorithm/lower_bound.h b/libcxx/include/__cxx03/__algorithm/lower_bound.h
new file mode 100644
index 000000000000..c417d8483549
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/lower_bound.h
@@ -0,0 +1,109 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_LOWER_BOUND_H
+#define _LIBCPP___ALGORITHM_LOWER_BOUND_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/half_positive.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/advance.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/is_callable.h>
+#include <__type_traits/remove_reference.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Iter, class _Type, class _Proj, class _Comp>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Iter __lower_bound_bisecting(
+    _Iter __first,
+    const _Type& __value,
+    typename iterator_traits<_Iter>::difference_type __len,
+    _Comp& __comp,
+    _Proj& __proj) {
+  while (__len != 0) {
+    auto __l2 = std::__half_positive(__len);
+    _Iter __m = __first;
+    _IterOps<_AlgPolicy>::advance(__m, __l2);
+    if (std::__invoke(__comp, std::__invoke(__proj, *__m), __value)) {
+      __first = ++__m;
+      __len -= __l2 + 1;
+    } else {
+      __len = __l2;
+    }
+  }
+  return __first;
+}
+
+// One-sided binary search, aka meta binary search, has been in the public domain for decades, and has the general
+// advantage of being \Omega(1) rather than the classic algorithm's \Omega(log(n)), with the downside of executing at
+// most 2*log(n) comparisons vs the classic algorithm's exact log(n). There are two scenarios in which it really shines:
+// the first one is when operating over non-random-access iterators, because the classic algorithm requires knowing the
+// container's size upfront, which adds \Omega(n) iterator increments to the complexity. The second one is when you're
+// traversing the container in order, trying to fast-forward to the next value: in that case, the classic algorithm
+// would yield \Omega(n*log(n)) comparisons and, for non-random-access iterators, \Omega(n^2) iterator increments,
+// whereas the one-sided version will yield O(n) operations on both counts, with a \Omega(log(n)) bound on the number of
+// comparisons.
+template <class _AlgPolicy, class _ForwardIterator, class _Sent, class _Type, class _Proj, class _Comp>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+__lower_bound_onesided(_ForwardIterator __first, _Sent __last, const _Type& __value, _Comp& __comp, _Proj& __proj) {
+  // step = 0, ensuring we can always short-circuit when distance is 1 later on
+  if (__first == __last || !std::__invoke(__comp, std::__invoke(__proj, *__first), __value))
+    return __first;
+
+  using _Distance = typename iterator_traits<_ForwardIterator>::difference_type;
+  for (_Distance __step = 1; __first != __last; __step <<= 1) {
+    auto __it   = __first;
+    auto __dist = __step - _IterOps<_AlgPolicy>::__advance_to(__it, __step, __last);
+    // once we reach the last range where needle can be we must start
+    // looking inwards, bisecting that range
+    if (__it == __last || !std::__invoke(__comp, std::__invoke(__proj, *__it), __value)) {
+      // we've already checked the previous value and it was less, we can save
+      // one comparison by skipping bisection
+      if (__dist == 1)
+        return __it;
+      return std::__lower_bound_bisecting<_AlgPolicy>(__first, __value, __dist, __comp, __proj);
+    }
+    // range not found, move forward!
+    __first = __it;
+  }
+  return __first;
+}
+
+template <class _AlgPolicy, class _ForwardIterator, class _Sent, class _Type, class _Proj, class _Comp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+__lower_bound(_ForwardIterator __first, _Sent __last, const _Type& __value, _Comp& __comp, _Proj& __proj) {
+  const auto __dist = _IterOps<_AlgPolicy>::distance(__first, __last);
+  return std::__lower_bound_bisecting<_AlgPolicy>(__first, __value, __dist, __comp, __proj);
+}
+
+template <class _ForwardIterator, class _Tp, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+lower_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, _Compare __comp) {
+  static_assert(__is_callable<_Compare, decltype(*__first), const _Tp&>::value, "The comparator has to be callable");
+  auto __proj = std::__identity();
+  return std::__lower_bound<_ClassicAlgPolicy>(__first, __last, __value, __comp, __proj);
+}
+
+template <class _ForwardIterator, class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+lower_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
+  return std::lower_bound(__first, __last, __value, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_LOWER_BOUND_H
diff --git a/libcxx/include/__cxx03/__algorithm/make_heap.h b/libcxx/include/__cxx03/__algorithm/make_heap.h
new file mode 100644
index 000000000000..e8f0cdb27333
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/make_heap.h
@@ -0,0 +1,60 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_MAKE_HEAP_H
+#define _LIBCPP___ALGORITHM_MAKE_HEAP_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/sift_down.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+__make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare&& __comp) {
+  __comp_ref_type<_Compare> __comp_ref = __comp;
+
+  using difference_type = typename iterator_traits<_RandomAccessIterator>::difference_type;
+  difference_type __n   = __last - __first;
+  if (__n > 1) {
+    // start from the first parent, there is no need to consider children
+    for (difference_type __start = (__n - 2) / 2; __start >= 0; --__start) {
+      std::__sift_down<_AlgPolicy>(__first, __comp_ref, __n, __first + __start);
+    }
+  }
+}
+
+template <class _RandomAccessIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+  std::__make_heap<_ClassicAlgPolicy>(std::move(__first), std::move(__last), __comp);
+}
+
+template <class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) {
+  std::make_heap(std::move(__first), std::move(__last), __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_MAKE_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/make_projected.h b/libcxx/include/__cxx03/__algorithm/make_projected.h
new file mode 100644
index 000000000000..5245e523f3df
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/make_projected.h
@@ -0,0 +1,106 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_MAKE_PROJECTED_H
+#define _LIBCPP___ALGORITHM_MAKE_PROJECTED_H
+
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_member_pointer.h>
+#include <__type_traits/is_same.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Pred, class _Proj>
+struct _ProjectedPred {
+  _Pred& __pred; // Can be a unary or a binary predicate.
+  _Proj& __proj;
+
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI _ProjectedPred(_Pred& __pred_arg, _Proj& __proj_arg)
+      : __pred(__pred_arg), __proj(__proj_arg) {}
+
+  template <class _Tp>
+  typename __invoke_of<_Pred&, decltype(std::__invoke(std::declval<_Proj&>(), std::declval<_Tp>()))>::type
+      _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI
+      operator()(_Tp&& __v) const {
+    return std::__invoke(__pred, std::__invoke(__proj, std::forward<_Tp>(__v)));
+  }
+
+  template <class _T1, class _T2>
+  typename __invoke_of<_Pred&,
+                       decltype(std::__invoke(std::declval<_Proj&>(), std::declval<_T1>())),
+                       decltype(std::__invoke(std::declval<_Proj&>(), std::declval<_T2>()))>::type _LIBCPP_CONSTEXPR
+  _LIBCPP_HIDE_FROM_ABI
+  operator()(_T1&& __lhs, _T2&& __rhs) const {
+    return std::__invoke(
+        __pred, std::__invoke(__proj, std::forward<_T1>(__lhs)), std::__invoke(__proj, std::forward<_T2>(__rhs)));
+  }
+};
+
+template <
+    class _Pred,
+    class _Proj,
+    __enable_if_t<!(!is_member_pointer<__decay_t<_Pred> >::value && __is_identity<__decay_t<_Proj> >::value), int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ProjectedPred<_Pred, _Proj> __make_projected(_Pred& __pred, _Proj& __proj) {
+  return _ProjectedPred<_Pred, _Proj>(__pred, __proj);
+}
+
+// Avoid creating the functor and just use the pristine comparator -- for certain algorithms, this would enable
+// optimizations that rely on the type of the comparator. Additionally, this results in less layers of indirection in
+// the call stack when the comparator is invoked, even in an unoptimized build.
+template <
+    class _Pred,
+    class _Proj,
+    __enable_if_t<!is_member_pointer<__decay_t<_Pred> >::value && __is_identity<__decay_t<_Proj> >::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Pred& __make_projected(_Pred& __pred, _Proj&) {
+  return __pred;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _Comp, class _Proj1, class _Proj2>
+_LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) __make_projected_comp(_Comp& __comp, _Proj1& __proj1, _Proj2& __proj2) {
+  if constexpr (__is_identity<decay_t<_Proj1>>::value && __is_identity<decay_t<_Proj2>>::value &&
+                !is_member_pointer_v<decay_t<_Comp>>) {
+    // Avoid creating the lambda and just use the pristine comparator -- for certain algorithms, this would enable
+    // optimizations that rely on the type of the comparator.
+    return __comp;
+
+  } else {
+    return [&](auto&& __lhs, auto&& __rhs) -> bool {
+      return std::invoke(__comp,
+                         std::invoke(__proj1, std::forward<decltype(__lhs)>(__lhs)),
+                         std::invoke(__proj2, std::forward<decltype(__rhs)>(__rhs)));
+    };
+  }
+}
+
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___ALGORITHM_MAKE_PROJECTED_H
diff --git a/libcxx/include/__cxx03/__algorithm/max.h b/libcxx/include/__cxx03/__algorithm/max.h
new file mode 100644
index 000000000000..d4c99f6f3643
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/max.h
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_MAX_H
+#define _LIBCPP___ALGORITHM_MAX_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/max_element.h>
+#include <__config>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp&
+max(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b, _Compare __comp) {
+  return __comp(__a, __b) ? __b : __a;
+}
+
+template <class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp&
+max(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b) {
+  return std::max(__a, __b, __less<>());
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp
+max(initializer_list<_Tp> __t, _Compare __comp) {
+  return *std::__max_element<__comp_ref_type<_Compare> >(__t.begin(), __t.end(), __comp);
+}
+
+template <class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp max(initializer_list<_Tp> __t) {
+  return *std::max_element(__t.begin(), __t.end(), __less<>());
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_MAX_H
diff --git a/libcxx/include/__cxx03/__algorithm/max_element.h b/libcxx/include/__cxx03/__algorithm/max_element.h
new file mode 100644
index 000000000000..c036726cbccd
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/max_element.h
@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_MAX_ELEMENT_H
+#define _LIBCPP___ALGORITHM_MAX_ELEMENT_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Compare, class _ForwardIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator
+__max_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) {
+  static_assert(
+      __has_forward_iterator_category<_ForwardIterator>::value, "std::max_element requires a ForwardIterator");
+  if (__first != __last) {
+    _ForwardIterator __i = __first;
+    while (++__i != __last)
+      if (__comp(*__first, *__i))
+        __first = __i;
+  }
+  return __first;
+}
+
+template <class _ForwardIterator, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator
+max_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) {
+  return std::__max_element<__comp_ref_type<_Compare> >(__first, __last, __comp);
+}
+
+template <class _ForwardIterator>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator
+max_element(_ForwardIterator __first, _ForwardIterator __last) {
+  return std::max_element(__first, __last, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_MAX_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/merge.h b/libcxx/include/__cxx03/__algorithm/merge.h
new file mode 100644
index 000000000000..bad663c4b9f1
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/merge.h
@@ -0,0 +1,69 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_MERGE_H
+#define _LIBCPP___ALGORITHM_MERGE_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/copy.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Compare, class _InputIterator1, class _InputIterator2, class _OutputIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator __merge(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _InputIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp) {
+  for (; __first1 != __last1; ++__result) {
+    if (__first2 == __last2)
+      return std::copy(__first1, __last1, __result);
+    if (__comp(*__first2, *__first1)) {
+      *__result = *__first2;
+      ++__first2;
+    } else {
+      *__result = *__first1;
+      ++__first1;
+    }
+  }
+  return std::copy(__first2, __last2, __result);
+}
+
+template <class _InputIterator1, class _InputIterator2, class _OutputIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+merge(_InputIterator1 __first1,
+      _InputIterator1 __last1,
+      _InputIterator2 __first2,
+      _InputIterator2 __last2,
+      _OutputIterator __result,
+      _Compare __comp) {
+  return std::__merge<__comp_ref_type<_Compare> >(__first1, __last1, __first2, __last2, __result, __comp);
+}
+
+template <class _InputIterator1, class _InputIterator2, class _OutputIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+merge(_InputIterator1 __first1,
+      _InputIterator1 __last1,
+      _InputIterator2 __first2,
+      _InputIterator2 __last2,
+      _OutputIterator __result) {
+  return std::merge(__first1, __last1, __first2, __last2, __result, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_MERGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/min.h b/libcxx/include/__cxx03/__algorithm/min.h
new file mode 100644
index 000000000000..1bafad8a461e
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/min.h
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_MIN_H
+#define _LIBCPP___ALGORITHM_MIN_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/min_element.h>
+#include <__config>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp&
+min(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b, _Compare __comp) {
+  return __comp(__b, __a) ? __b : __a;
+}
+
+template <class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp&
+min(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b) {
+  return std::min(__a, __b, __less<>());
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp
+min(initializer_list<_Tp> __t, _Compare __comp) {
+  return *std::__min_element<__comp_ref_type<_Compare> >(__t.begin(), __t.end(), __comp);
+}
+
+template <class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp min(initializer_list<_Tp> __t) {
+  return *std::min_element(__t.begin(), __t.end(), __less<>());
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_MIN_H
diff --git a/libcxx/include/__cxx03/__algorithm/min_element.h b/libcxx/include/__cxx03/__algorithm/min_element.h
new file mode 100644
index 000000000000..65f3594d630c
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/min_element.h
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_MIN_ELEMENT_H
+#define _LIBCPP___ALGORITHM_MIN_ELEMENT_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/is_callable.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Comp, class _Iter, class _Sent, class _Proj>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter
+__min_element(_Iter __first, _Sent __last, _Comp __comp, _Proj& __proj) {
+  if (__first == __last)
+    return __first;
+
+  _Iter __i = __first;
+  while (++__i != __last)
+    if (std::__invoke(__comp, std::__invoke(__proj, *__i), std::__invoke(__proj, *__first)))
+      __first = __i;
+
+  return __first;
+}
+
+template <class _Comp, class _Iter, class _Sent>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter __min_element(_Iter __first, _Sent __last, _Comp __comp) {
+  auto __proj = __identity();
+  return std::__min_element<_Comp>(std::move(__first), std::move(__last), __comp, __proj);
+}
+
+template <class _ForwardIterator, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator
+min_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) {
+  static_assert(
+      __has_forward_iterator_category<_ForwardIterator>::value, "std::min_element requires a ForwardIterator");
+  static_assert(
+      __is_callable<_Compare, decltype(*__first), decltype(*__first)>::value, "The comparator has to be callable");
+
+  return std::__min_element<__comp_ref_type<_Compare> >(std::move(__first), std::move(__last), __comp);
+}
+
+template <class _ForwardIterator>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator
+min_element(_ForwardIterator __first, _ForwardIterator __last) {
+  return std::min_element(__first, __last, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_MIN_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/min_max_result.h b/libcxx/include/__cxx03/__algorithm/min_max_result.h
new file mode 100644
index 000000000000..e988df7c114e
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/min_max_result.h
@@ -0,0 +1,56 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_MIN_MAX_RESULT_H
+#define _LIBCPP___ALGORITHM_MIN_MAX_RESULT_H
+
+#include <__concepts/convertible_to.h>
+#include <__config>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+template <class _T1>
+struct min_max_result {
+  _LIBCPP_NO_UNIQUE_ADDRESS _T1 min;
+  _LIBCPP_NO_UNIQUE_ADDRESS _T1 max;
+
+  template <class _T2>
+    requires convertible_to<const _T1&, _T2>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator min_max_result<_T2>() const& {
+    return {min, max};
+  }
+
+  template <class _T2>
+    requires convertible_to<_T1, _T2>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator min_max_result<_T2>() && {
+    return {std::move(min), std::move(max)};
+  }
+};
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_MIN_MAX_RESULT_H
diff --git a/libcxx/include/__cxx03/__algorithm/minmax.h b/libcxx/include/__cxx03/__algorithm/minmax.h
new file mode 100644
index 000000000000..9feda2b4c0da
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/minmax.h
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_MINMAX_H
+#define _LIBCPP___ALGORITHM_MINMAX_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/minmax_element.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__type_traits/is_callable.h>
+#include <__utility/pair.h>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<const _Tp&, const _Tp&>
+minmax(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b, _Compare __comp) {
+  return __comp(__b, __a) ? pair<const _Tp&, const _Tp&>(__b, __a) : pair<const _Tp&, const _Tp&>(__a, __b);
+}
+
+template <class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<const _Tp&, const _Tp&>
+minmax(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b) {
+  return std::minmax(__a, __b, __less<>());
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Tp, _Tp>
+minmax(initializer_list<_Tp> __t, _Compare __comp) {
+  static_assert(__is_callable<_Compare, _Tp, _Tp>::value, "The comparator has to be callable");
+  __identity __proj;
+  auto __ret = std::__minmax_element_impl(__t.begin(), __t.end(), __comp, __proj);
+  return pair<_Tp, _Tp>(*__ret.first, *__ret.second);
+}
+
+template <class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Tp, _Tp>
+minmax(initializer_list<_Tp> __t) {
+  return std::minmax(__t, __less<>());
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_MINMAX_H
diff --git a/libcxx/include/__cxx03/__algorithm/minmax_element.h b/libcxx/include/__cxx03/__algorithm/minmax_element.h
new file mode 100644
index 000000000000..43cb23347c34
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/minmax_element.h
@@ -0,0 +1,100 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_MINMAX_ELEMENT_H
+#define _LIBCPP___ALGORITHM_MINMAX_ELEMENT_H
+
+#include <__algorithm/comp.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/is_callable.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Comp, class _Proj>
+class _MinmaxElementLessFunc {
+  _Comp& __comp_;
+  _Proj& __proj_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _MinmaxElementLessFunc(_Comp& __comp, _Proj& __proj)
+      : __comp_(__comp), __proj_(__proj) {}
+
+  template <class _Iter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator()(_Iter& __it1, _Iter& __it2) {
+    return std::__invoke(__comp_, std::__invoke(__proj_, *__it1), std::__invoke(__proj_, *__it2));
+  }
+};
+
+template <class _Iter, class _Sent, class _Proj, class _Comp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter, _Iter>
+__minmax_element_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+  auto __less = _MinmaxElementLessFunc<_Comp, _Proj>(__comp, __proj);
+
+  pair<_Iter, _Iter> __result(__first, __first);
+  if (__first == __last || ++__first == __last)
+    return __result;
+
+  if (__less(__first, __result.first))
+    __result.first = __first;
+  else
+    __result.second = __first;
+
+  while (++__first != __last) {
+    _Iter __i = __first;
+    if (++__first == __last) {
+      if (__less(__i, __result.first))
+        __result.first = __i;
+      else if (!__less(__i, __result.second))
+        __result.second = __i;
+      return __result;
+    }
+
+    if (__less(__first, __i)) {
+      if (__less(__first, __result.first))
+        __result.first = __first;
+      if (!__less(__i, __result.second))
+        __result.second = __i;
+    } else {
+      if (__less(__i, __result.first))
+        __result.first = __i;
+      if (!__less(__first, __result.second))
+        __result.second = __first;
+    }
+  }
+
+  return __result;
+}
+
+template <class _ForwardIterator, class _Compare>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_ForwardIterator, _ForwardIterator>
+minmax_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) {
+  static_assert(
+      __has_forward_iterator_category<_ForwardIterator>::value, "std::minmax_element requires a ForwardIterator");
+  static_assert(
+      __is_callable<_Compare, decltype(*__first), decltype(*__first)>::value, "The comparator has to be callable");
+  auto __proj = __identity();
+  return std::__minmax_element_impl(__first, __last, __comp, __proj);
+}
+
+template <class _ForwardIterator>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_ForwardIterator, _ForwardIterator>
+minmax_element(_ForwardIterator __first, _ForwardIterator __last) {
+  return std::minmax_element(__first, __last, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_MINMAX_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/mismatch.h b/libcxx/include/__cxx03/__algorithm/mismatch.h
new file mode 100644
index 000000000000..632bec02406a
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/mismatch.h
@@ -0,0 +1,217 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_MISMATCH_H
+#define _LIBCPP___ALGORITHM_MISMATCH_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/min.h>
+#include <__algorithm/simd_utils.h>
+#include <__algorithm/unwrap_iter.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__iterator/aliasing_iterator.h>
+#include <__type_traits/desugars_to.h>
+#include <__type_traits/invoke.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_equality_comparable.h>
+#include <__type_traits/is_integral.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <__utility/unreachable.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Iter1, class _Sent1, class _Iter2, class _Pred, class _Proj1, class _Proj2>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter1, _Iter2>
+__mismatch_loop(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+  while (__first1 != __last1) {
+    if (!std::__invoke(__pred, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2)))
+      break;
+    ++__first1;
+    ++__first2;
+  }
+  return std::make_pair(std::move(__first1), std::move(__first2));
+}
+
+template <class _Iter1, class _Sent1, class _Iter2, class _Pred, class _Proj1, class _Proj2>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter1, _Iter2>
+__mismatch(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+  return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj1, __proj2);
+}
+
+#if _LIBCPP_VECTORIZE_ALGORITHMS
+
+template <class _Iter>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter, _Iter>
+__mismatch_vectorized(_Iter __first1, _Iter __last1, _Iter __first2) {
+  using __value_type              = __iter_value_type<_Iter>;
+  constexpr size_t __unroll_count = 4;
+  constexpr size_t __vec_size     = __native_vector_size<__value_type>;
+  using __vec                     = __simd_vector<__value_type, __vec_size>;
+
+  if (!__libcpp_is_constant_evaluated()) {
+    auto __orig_first1 = __first1;
+    auto __last2       = __first2 + (__last1 - __first1);
+    while (static_cast<size_t>(__last1 - __first1) >= __unroll_count * __vec_size) [[__unlikely__]] {
+      __vec __lhs[__unroll_count];
+      __vec __rhs[__unroll_count];
+
+      for (size_t __i = 0; __i != __unroll_count; ++__i) {
+        __lhs[__i] = std::__load_vector<__vec>(__first1 + __i * __vec_size);
+        __rhs[__i] = std::__load_vector<__vec>(__first2 + __i * __vec_size);
+      }
+
+      for (size_t __i = 0; __i != __unroll_count; ++__i) {
+        if (auto __cmp_res = __lhs[__i] == __rhs[__i]; !std::__all_of(__cmp_res)) {
+          auto __offset = __i * __vec_size + std::__find_first_not_set(__cmp_res);
+          return {__first1 + __offset, __first2 + __offset};
+        }
+      }
+
+      __first1 += __unroll_count * __vec_size;
+      __first2 += __unroll_count * __vec_size;
+    }
+
+    // check the remaining 0-3 vectors
+    while (static_cast<size_t>(__last1 - __first1) >= __vec_size) {
+      if (auto __cmp_res = std::__load_vector<__vec>(__first1) == std::__load_vector<__vec>(__first2);
+          !std::__all_of(__cmp_res)) {
+        auto __offset = std::__find_first_not_set(__cmp_res);
+        return {__first1 + __offset, __first2 + __offset};
+      }
+      __first1 += __vec_size;
+      __first2 += __vec_size;
+    }
+
+    if (__last1 - __first1 == 0)
+      return {__first1, __first2};
+
+    // Check if we can load elements in front of the current pointer. If that's the case load a vector at
+    // (last - vector_size) to check the remaining elements
+    if (static_cast<size_t>(__first1 - __orig_first1) >= __vec_size) {
+      __first1 = __last1 - __vec_size;
+      __first2 = __last2 - __vec_size;
+      auto __offset =
+          std::__find_first_not_set(std::__load_vector<__vec>(__first1) == std::__load_vector<__vec>(__first2));
+      return {__first1 + __offset, __first2 + __offset};
+    } // else loop over the elements individually
+  }
+
+  __equal_to __pred;
+  __identity __proj;
+  return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj, __proj);
+}
+
+template <class _Tp,
+          class _Pred,
+          class _Proj1,
+          class _Proj2,
+          __enable_if_t<is_integral<_Tp>::value && __desugars_to_v<__equal_tag, _Pred, _Tp, _Tp> &&
+                            __is_identity<_Proj1>::value && __is_identity<_Proj2>::value,
+                        int> = 0>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Tp*, _Tp*>
+__mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Pred&, _Proj1&, _Proj2&) {
+  return std::__mismatch_vectorized(__first1, __last1, __first2);
+}
+
+template <class _Tp,
+          class _Pred,
+          class _Proj1,
+          class _Proj2,
+          __enable_if_t<!is_integral<_Tp>::value && __desugars_to_v<__equal_tag, _Pred, _Tp, _Tp> &&
+                            __is_identity<_Proj1>::value && __is_identity<_Proj2>::value &&
+                            __can_map_to_integer_v<_Tp> && __libcpp_is_trivially_equality_comparable<_Tp, _Tp>::value,
+                        int> = 0>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Tp*, _Tp*>
+__mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+  if (__libcpp_is_constant_evaluated()) {
+    return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj1, __proj2);
+  } else {
+    using _Iter = __aliasing_iterator<_Tp*, __get_as_integer_type_t<_Tp>>;
+    auto __ret  = std::__mismatch_vectorized(_Iter(__first1), _Iter(__last1), _Iter(__first2));
+    return {__ret.first.__base(), __ret.second.__base()};
+  }
+}
+#endif // _LIBCPP_VECTORIZE_ALGORITHMS
+
+template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2>
+mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _BinaryPredicate __pred) {
+  __identity __proj;
+  auto __res = std::__mismatch(
+      std::__unwrap_iter(__first1), std::__unwrap_iter(__last1), std::__unwrap_iter(__first2), __pred, __proj, __proj);
+  return std::make_pair(std::__rewrap_iter(__first1, __res.first), std::__rewrap_iter(__first2, __res.second));
+}
+
+template <class _InputIterator1, class _InputIterator2>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2>
+mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2) {
+  return std::mismatch(__first1, __last1, __first2, __equal_to());
+}
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class _Pred, class _Proj1, class _Proj2>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter1, _Iter2> __mismatch(
+    _Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+  while (__first1 != __last1 && __first2 != __last2) {
+    if (!std::__invoke(__pred, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2)))
+      break;
+    ++__first1;
+    ++__first2;
+  }
+  return {std::move(__first1), std::move(__first2)};
+}
+
+template <class _Tp, class _Pred, class _Proj1, class _Proj2>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Tp*, _Tp*>
+__mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Tp* __last2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+  auto __len = std::min(__last1 - __first1, __last2 - __first2);
+  return std::__mismatch(__first1, __first1 + __len, __first2, __pred, __proj1, __proj2);
+}
+
+template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2>
+mismatch(_InputIterator1 __first1,
+         _InputIterator1 __last1,
+         _InputIterator2 __first2,
+         _InputIterator2 __last2,
+         _BinaryPredicate __pred) {
+  __identity __proj;
+  auto __res = std::__mismatch(
+      std::__unwrap_iter(__first1),
+      std::__unwrap_iter(__last1),
+      std::__unwrap_iter(__first2),
+      std::__unwrap_iter(__last2),
+      __pred,
+      __proj,
+      __proj);
+  return {std::__rewrap_iter(__first1, __res.first), std::__rewrap_iter(__first2, __res.second)};
+}
+
+template <class _InputIterator1, class _InputIterator2>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2>
+mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2) {
+  return std::mismatch(__first1, __last1, __first2, __last2, __equal_to());
+}
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_MISMATCH_H
diff --git a/libcxx/include/__cxx03/__algorithm/move.h b/libcxx/include/__cxx03/__algorithm/move.h
new file mode 100644
index 000000000000..1716d43e2a61
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/move.h
@@ -0,0 +1,127 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_MOVE_H
+#define _LIBCPP___ALGORITHM_MOVE_H
+
+#include <__algorithm/copy_move_common.h>
+#include <__algorithm/for_each_segment.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/min.h>
+#include <__config>
+#include <__iterator/segmented_iterator.h>
+#include <__type_traits/common_type.h>
+#include <__type_traits/is_constructible.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _InIter, class _Sent, class _OutIter>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
+__move(_InIter __first, _Sent __last, _OutIter __result);
+
+template <class _AlgPolicy>
+struct __move_impl {
+  template <class _InIter, class _Sent, class _OutIter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result) const {
+    while (__first != __last) {
+      *__result = _IterOps<_AlgPolicy>::__iter_move(__first);
+      ++__first;
+      ++__result;
+    }
+    return std::make_pair(std::move(__first), std::move(__result));
+  }
+
+  template <class _InIter, class _OutIter>
+  struct _MoveSegment {
+    using _Traits = __segmented_iterator_traits<_InIter>;
+
+    _OutIter& __result_;
+
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit _MoveSegment(_OutIter& __result)
+        : __result_(__result) {}
+
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+    operator()(typename _Traits::__local_iterator __lfirst, typename _Traits::__local_iterator __llast) {
+      __result_ = std::__move<_AlgPolicy>(__lfirst, __llast, std::move(__result_)).second;
+    }
+  };
+
+  template <class _InIter, class _OutIter, __enable_if_t<__is_segmented_iterator<_InIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
+  operator()(_InIter __first, _InIter __last, _OutIter __result) const {
+    std::__for_each_segment(__first, __last, _MoveSegment<_InIter, _OutIter>(__result));
+    return std::make_pair(__last, std::move(__result));
+  }
+
+  template <class _InIter,
+            class _OutIter,
+            __enable_if_t<__has_random_access_iterator_category<_InIter>::value &&
+                              !__is_segmented_iterator<_InIter>::value && __is_segmented_iterator<_OutIter>::value,
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
+  operator()(_InIter __first, _InIter __last, _OutIter __result) const {
+    using _Traits = __segmented_iterator_traits<_OutIter>;
+    using _DiffT  = typename common_type<__iter_diff_t<_InIter>, __iter_diff_t<_OutIter> >::type;
+
+    if (__first == __last)
+      return std::make_pair(std::move(__first), std::move(__result));
+
+    auto __local_first      = _Traits::__local(__result);
+    auto __segment_iterator = _Traits::__segment(__result);
+    while (true) {
+      auto __local_last = _Traits::__end(__segment_iterator);
+      auto __size       = std::min<_DiffT>(__local_last - __local_first, __last - __first);
+      auto __iters      = std::__move<_AlgPolicy>(__first, __first + __size, __local_first);
+      __first           = std::move(__iters.first);
+
+      if (__first == __last)
+        return std::make_pair(std::move(__first), _Traits::__compose(__segment_iterator, std::move(__iters.second)));
+
+      __local_first = _Traits::__begin(++__segment_iterator);
+    }
+  }
+
+  // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer.
+  template <class _In, class _Out, __enable_if_t<__can_lower_move_assignment_to_memmove<_In, _Out>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*>
+  operator()(_In* __first, _In* __last, _Out* __result) const {
+    return std::__copy_trivial_impl(__first, __last, __result);
+  }
+};
+
+template <class _AlgPolicy, class _InIter, class _Sent, class _OutIter>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
+__move(_InIter __first, _Sent __last, _OutIter __result) {
+  return std::__copy_move_unwrap_iters<__move_impl<_AlgPolicy> >(
+      std::move(__first), std::move(__last), std::move(__result));
+}
+
+template <class _InputIterator, class _OutputIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+move(_InputIterator __first, _InputIterator __last, _OutputIterator __result) {
+  static_assert(is_copy_constructible<_InputIterator>::value, "Iterators has to be copy constructible.");
+  static_assert(is_copy_constructible<_OutputIterator>::value, "The output iterator has to be copy constructible.");
+
+  return std::__move<_ClassicAlgPolicy>(std::move(__first), std::move(__last), std::move(__result)).second;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_MOVE_H
diff --git a/libcxx/include/__cxx03/__algorithm/move_backward.h b/libcxx/include/__cxx03/__algorithm/move_backward.h
new file mode 100644
index 000000000000..4beb7bdbaac0
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/move_backward.h
@@ -0,0 +1,137 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_MOVE_BACKWARD_H
+#define _LIBCPP___ALGORITHM_MOVE_BACKWARD_H
+
+#include <__algorithm/copy_move_common.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/min.h>
+#include <__config>
+#include <__iterator/segmented_iterator.h>
+#include <__type_traits/common_type.h>
+#include <__type_traits/is_constructible.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _BidirectionalIterator1, class _Sentinel, class _BidirectionalIterator2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_BidirectionalIterator1, _BidirectionalIterator2>
+__move_backward(_BidirectionalIterator1 __first, _Sentinel __last, _BidirectionalIterator2 __result);
+
+template <class _AlgPolicy>
+struct __move_backward_impl {
+  template <class _InIter, class _Sent, class _OutIter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result) const {
+    auto __last_iter          = _IterOps<_AlgPolicy>::next(__first, __last);
+    auto __original_last_iter = __last_iter;
+
+    while (__first != __last_iter) {
+      *--__result = _IterOps<_AlgPolicy>::__iter_move(--__last_iter);
+    }
+
+    return std::make_pair(std::move(__original_last_iter), std::move(__result));
+  }
+
+  template <class _InIter, class _OutIter, __enable_if_t<__is_segmented_iterator<_InIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
+  operator()(_InIter __first, _InIter __last, _OutIter __result) const {
+    using _Traits = __segmented_iterator_traits<_InIter>;
+    auto __sfirst = _Traits::__segment(__first);
+    auto __slast  = _Traits::__segment(__last);
+    if (__sfirst == __slast) {
+      auto __iters =
+          std::__move_backward<_AlgPolicy>(_Traits::__local(__first), _Traits::__local(__last), std::move(__result));
+      return std::make_pair(__last, __iters.second);
+    }
+
+    __result =
+        std::__move_backward<_AlgPolicy>(_Traits::__begin(__slast), _Traits::__local(__last), std::move(__result))
+            .second;
+    --__slast;
+    while (__sfirst != __slast) {
+      __result =
+          std::__move_backward<_AlgPolicy>(_Traits::__begin(__slast), _Traits::__end(__slast), std::move(__result))
+              .second;
+      --__slast;
+    }
+    __result = std::__move_backward<_AlgPolicy>(_Traits::__local(__first), _Traits::__end(__slast), std::move(__result))
+                   .second;
+    return std::make_pair(__last, std::move(__result));
+  }
+
+  template <class _InIter,
+            class _OutIter,
+            __enable_if_t<__has_random_access_iterator_category<_InIter>::value &&
+                              !__is_segmented_iterator<_InIter>::value && __is_segmented_iterator<_OutIter>::value,
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
+  operator()(_InIter __first, _InIter __last, _OutIter __result) const {
+    using _Traits = __segmented_iterator_traits<_OutIter>;
+    using _DiffT  = typename common_type<__iter_diff_t<_InIter>, __iter_diff_t<_OutIter> >::type;
+
+    // When the range contains no elements, __result might not be a valid iterator
+    if (__first == __last)
+      return std::make_pair(__first, __result);
+
+    auto __orig_last = __last;
+
+    auto __local_last       = _Traits::__local(__result);
+    auto __segment_iterator = _Traits::__segment(__result);
+    while (true) {
+      auto __local_first = _Traits::__begin(__segment_iterator);
+      auto __size        = std::min<_DiffT>(__local_last - __local_first, __last - __first);
+      auto __iter        = std::__move_backward<_AlgPolicy>(__last - __size, __last, __local_last).second;
+      __last -= __size;
+
+      if (__first == __last)
+        return std::make_pair(std::move(__orig_last), _Traits::__compose(__segment_iterator, std::move(__iter)));
+
+      __local_last = _Traits::__end(--__segment_iterator);
+    }
+  }
+
+  // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer.
+  template <class _In, class _Out, __enable_if_t<__can_lower_move_assignment_to_memmove<_In, _Out>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*>
+  operator()(_In* __first, _In* __last, _Out* __result) const {
+    return std::__copy_backward_trivial_impl(__first, __last, __result);
+  }
+};
+
+template <class _AlgPolicy, class _BidirectionalIterator1, class _Sentinel, class _BidirectionalIterator2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_BidirectionalIterator1, _BidirectionalIterator2>
+__move_backward(_BidirectionalIterator1 __first, _Sentinel __last, _BidirectionalIterator2 __result) {
+  static_assert(std::is_copy_constructible<_BidirectionalIterator1>::value &&
+                    std::is_copy_constructible<_BidirectionalIterator1>::value,
+                "Iterators must be copy constructible.");
+
+  return std::__copy_move_unwrap_iters<__move_backward_impl<_AlgPolicy> >(
+      std::move(__first), std::move(__last), std::move(__result));
+}
+
+template <class _BidirectionalIterator1, class _BidirectionalIterator2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _BidirectionalIterator2
+move_backward(_BidirectionalIterator1 __first, _BidirectionalIterator1 __last, _BidirectionalIterator2 __result) {
+  return std::__move_backward<_ClassicAlgPolicy>(std::move(__first), std::move(__last), std::move(__result)).second;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_MOVE_BACKWARD_H
diff --git a/libcxx/include/__cxx03/__algorithm/next_permutation.h b/libcxx/include/__cxx03/__algorithm/next_permutation.h
new file mode 100644
index 000000000000..011ee028cc2f
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/next_permutation.h
@@ -0,0 +1,75 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_NEXT_PERMUTATION_H
+#define _LIBCPP___ALGORITHM_NEXT_PERMUTATION_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/reverse.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Compare, class _BidirectionalIterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_BidirectionalIterator, bool>
+__next_permutation(_BidirectionalIterator __first, _Sentinel __last, _Compare&& __comp) {
+  using _Result = pair<_BidirectionalIterator, bool>;
+
+  _BidirectionalIterator __last_iter = _IterOps<_AlgPolicy>::next(__first, __last);
+  _BidirectionalIterator __i         = __last_iter;
+  if (__first == __last || __first == --__i)
+    return _Result(std::move(__last_iter), false);
+
+  while (true) {
+    _BidirectionalIterator __ip1 = __i;
+    if (__comp(*--__i, *__ip1)) {
+      _BidirectionalIterator __j = __last_iter;
+      while (!__comp(*__i, *--__j))
+        ;
+      _IterOps<_AlgPolicy>::iter_swap(__i, __j);
+      std::__reverse<_AlgPolicy>(__ip1, __last_iter);
+      return _Result(std::move(__last_iter), true);
+    }
+    if (__i == __first) {
+      std::__reverse<_AlgPolicy>(__first, __last_iter);
+      return _Result(std::move(__last_iter), false);
+    }
+  }
+}
+
+template <class _BidirectionalIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+next_permutation(_BidirectionalIterator __first, _BidirectionalIterator __last, _Compare __comp) {
+  return std::__next_permutation<_ClassicAlgPolicy>(
+             std::move(__first), std::move(__last), static_cast<__comp_ref_type<_Compare> >(__comp))
+      .second;
+}
+
+template <class _BidirectionalIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+next_permutation(_BidirectionalIterator __first, _BidirectionalIterator __last) {
+  return std::next_permutation(__first, __last, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_NEXT_PERMUTATION_H
diff --git a/libcxx/include/__cxx03/__algorithm/none_of.h b/libcxx/include/__cxx03/__algorithm/none_of.h
new file mode 100644
index 000000000000..50841ba17cc6
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/none_of.h
@@ -0,0 +1,32 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_NONE_OF_H
+#define _LIBCPP___ALGORITHM_NONE_OF_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _Predicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+none_of(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
+  for (; __first != __last; ++__first)
+    if (__pred(*__first))
+      return false;
+  return true;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_NONE_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/nth_element.h b/libcxx/include/__cxx03/__algorithm/nth_element.h
new file mode 100644
index 000000000000..da748d7255ab
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/nth_element.h
@@ -0,0 +1,261 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_NTH_ELEMENT_H
+#define _LIBCPP___ALGORITHM_NTH_ELEMENT_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/sort.h>
+#include <__assert>
+#include <__config>
+#include <__debug_utils/randomize_range.h>
+#include <__iterator/iterator_traits.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Compare, class _RandomAccessIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool __nth_element_find_guard(
+    _RandomAccessIterator& __i, _RandomAccessIterator& __j, _RandomAccessIterator __m, _Compare __comp) {
+  // manually guard downward moving __j against __i
+  while (true) {
+    if (__i == --__j) {
+      return false;
+    }
+    if (__comp(*__j, *__m)) {
+      return true; // found guard for downward moving __j, now use unguarded partition
+    }
+  }
+}
+
+template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+// NOLINTNEXTLINE(readability-function-cognitive-complexity)
+__nth_element(
+    _RandomAccessIterator __first, _RandomAccessIterator __nth, _RandomAccessIterator __last, _Compare __comp) {
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  // _Compare is known to be a reference type
+  typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  const difference_type __limit = 7;
+  while (true) {
+    if (__nth == __last)
+      return;
+    difference_type __len = __last - __first;
+    switch (__len) {
+    case 0:
+    case 1:
+      return;
+    case 2:
+      if (__comp(*--__last, *__first))
+        _Ops::iter_swap(__first, __last);
+      return;
+    case 3: {
+      _RandomAccessIterator __m = __first;
+      std::__sort3<_AlgPolicy, _Compare>(__first, ++__m, --__last, __comp);
+      return;
+    }
+    }
+    if (__len <= __limit) {
+      std::__selection_sort<_AlgPolicy, _Compare>(__first, __last, __comp);
+      return;
+    }
+    // __len > __limit >= 3
+    _RandomAccessIterator __m   = __first + __len / 2;
+    _RandomAccessIterator __lm1 = __last;
+    unsigned __n_swaps          = std::__sort3<_AlgPolicy, _Compare>(__first, __m, --__lm1, __comp);
+    // *__m is median
+    // partition [__first, __m) < *__m and *__m <= [__m, __last)
+    // (this inhibits tossing elements equivalent to __m around unnecessarily)
+    _RandomAccessIterator __i = __first;
+    _RandomAccessIterator __j = __lm1;
+    // j points beyond range to be tested, *__lm1 is known to be <= *__m
+    // The search going up is known to be guarded but the search coming down isn't.
+    // Prime the downward search with a guard.
+    if (!__comp(*__i, *__m)) // if *__first == *__m
+    {
+      // *__first == *__m, *__first doesn't go in first part
+      if (std::__nth_element_find_guard<_Compare>(__i, __j, __m, __comp)) {
+        _Ops::iter_swap(__i, __j);
+        ++__n_swaps;
+      } else {
+        // *__first == *__m, *__m <= all other elements
+        // Partition instead into [__first, __i) == *__first and *__first < [__i, __last)
+        ++__i; // __first + 1
+        __j = __last;
+        if (!__comp(*__first, *--__j)) { // we need a guard if *__first == *(__last-1)
+          while (true) {
+            if (__i == __j) {
+              return; // [__first, __last) all equivalent elements
+            } else if (__comp(*__first, *__i)) {
+              _Ops::iter_swap(__i, __j);
+              ++__n_swaps;
+              ++__i;
+              break;
+            }
+            ++__i;
+          }
+        }
+        // [__first, __i) == *__first and *__first < [__j, __last) and __j == __last - 1
+        if (__i == __j) {
+          return;
+        }
+        while (true) {
+          while (!__comp(*__first, *__i)) {
+            ++__i;
+            _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+                __i != __last,
+                "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+          }
+          do {
+            _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+                __j != __first,
+                "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+            --__j;
+          } while (__comp(*__first, *__j));
+          if (__i >= __j)
+            break;
+          _Ops::iter_swap(__i, __j);
+          ++__n_swaps;
+          ++__i;
+        }
+        // [__first, __i) == *__first and *__first < [__i, __last)
+        // The first part is sorted,
+        if (__nth < __i) {
+          return;
+        }
+        // __nth_element the second part
+        // std::__nth_element<_Compare>(__i, __nth, __last, __comp);
+        __first = __i;
+        continue;
+      }
+    }
+    ++__i;
+    // j points beyond range to be tested, *__lm1 is known to be <= *__m
+    // if not yet partitioned...
+    if (__i < __j) {
+      // known that *(__i - 1) < *__m
+      while (true) {
+        // __m still guards upward moving __i
+        while (__comp(*__i, *__m)) {
+          ++__i;
+          _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+              __i != __last,
+              "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+        }
+        // It is now known that a guard exists for downward moving __j
+        do {
+          _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+              __j != __first,
+              "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+          --__j;
+        } while (!__comp(*__j, *__m));
+        if (__i >= __j)
+          break;
+        _Ops::iter_swap(__i, __j);
+        ++__n_swaps;
+        // It is known that __m != __j
+        // If __m just moved, follow it
+        if (__m == __i)
+          __m = __j;
+        ++__i;
+      }
+    }
+    // [__first, __i) < *__m and *__m <= [__i, __last)
+    if (__i != __m && __comp(*__m, *__i)) {
+      _Ops::iter_swap(__i, __m);
+      ++__n_swaps;
+    }
+    // [__first, __i) < *__i and *__i <= [__i+1, __last)
+    if (__nth == __i)
+      return;
+    if (__n_swaps == 0) {
+      // We were given a perfectly partitioned sequence.  Coincidence?
+      if (__nth < __i) {
+        // Check for [__first, __i) already sorted
+        __j = __m = __first;
+        while (true) {
+          if (++__j == __i) {
+            // [__first, __i) sorted
+            return;
+          }
+          if (__comp(*__j, *__m)) {
+            // not yet sorted, so sort
+            break;
+          }
+          __m = __j;
+        }
+      } else {
+        // Check for [__i, __last) already sorted
+        __j = __m = __i;
+        while (true) {
+          if (++__j == __last) {
+            // [__i, __last) sorted
+            return;
+          }
+          if (__comp(*__j, *__m)) {
+            // not yet sorted, so sort
+            break;
+          }
+          __m = __j;
+        }
+      }
+    }
+    // __nth_element on range containing __nth
+    if (__nth < __i) {
+      // std::__nth_element<_Compare>(__first, __nth, __i, __comp);
+      __last = __i;
+    } else {
+      // std::__nth_element<_Compare>(__i+1, __nth, __last, __comp);
+      __first = ++__i;
+    }
+  }
+}
+
+template <class _AlgPolicy, class _RandomAccessIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __nth_element_impl(
+    _RandomAccessIterator __first, _RandomAccessIterator __nth, _RandomAccessIterator __last, _Compare& __comp) {
+  if (__nth == __last)
+    return;
+
+  std::__debug_randomize_range<_AlgPolicy>(__first, __last);
+
+  std::__nth_element<_AlgPolicy, __comp_ref_type<_Compare> >(__first, __nth, __last, __comp);
+
+  std::__debug_randomize_range<_AlgPolicy>(__first, __nth);
+  if (__nth != __last) {
+    std::__debug_randomize_range<_AlgPolicy>(++__nth, __last);
+  }
+}
+
+template <class _RandomAccessIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth, _RandomAccessIterator __last, _Compare __comp) {
+  std::__nth_element_impl<_ClassicAlgPolicy>(std::move(__first), std::move(__nth), std::move(__last), __comp);
+}
+
+template <class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth, _RandomAccessIterator __last) {
+  std::nth_element(std::move(__first), std::move(__nth), std::move(__last), __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_NTH_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/partial_sort.h b/libcxx/include/__cxx03/__algorithm/partial_sort.h
new file mode 100644
index 000000000000..7f8d0c49147e
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/partial_sort.h
@@ -0,0 +1,91 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_PARTIAL_SORT_H
+#define _LIBCPP___ALGORITHM_PARTIAL_SORT_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_heap.h>
+#include <__algorithm/sift_down.h>
+#include <__algorithm/sort_heap.h>
+#include <__config>
+#include <__debug_utils/randomize_range.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_constructible.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Compare, class _RandomAccessIterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator __partial_sort_impl(
+    _RandomAccessIterator __first, _RandomAccessIterator __middle, _Sentinel __last, _Compare&& __comp) {
+  if (__first == __middle) {
+    return _IterOps<_AlgPolicy>::next(__middle, __last);
+  }
+
+  std::__make_heap<_AlgPolicy>(__first, __middle, __comp);
+
+  typename iterator_traits<_RandomAccessIterator>::difference_type __len = __middle - __first;
+  _RandomAccessIterator __i                                              = __middle;
+  for (; __i != __last; ++__i) {
+    if (__comp(*__i, *__first)) {
+      _IterOps<_AlgPolicy>::iter_swap(__i, __first);
+      std::__sift_down<_AlgPolicy>(__first, __comp, __len, __first);
+    }
+  }
+  std::__sort_heap<_AlgPolicy>(std::move(__first), std::move(__middle), __comp);
+
+  return __i;
+}
+
+template <class _AlgPolicy, class _Compare, class _RandomAccessIterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator
+__partial_sort(_RandomAccessIterator __first, _RandomAccessIterator __middle, _Sentinel __last, _Compare& __comp) {
+  if (__first == __middle)
+    return _IterOps<_AlgPolicy>::next(__middle, __last);
+
+  std::__debug_randomize_range<_AlgPolicy>(__first, __last);
+
+  auto __last_iter =
+      std::__partial_sort_impl<_AlgPolicy>(__first, __middle, __last, static_cast<__comp_ref_type<_Compare> >(__comp));
+
+  std::__debug_randomize_range<_AlgPolicy>(__middle, __last);
+
+  return __last_iter;
+}
+
+template <class _RandomAccessIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void partial_sort(
+    _RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last, _Compare __comp) {
+  static_assert(std::is_copy_constructible<_RandomAccessIterator>::value, "Iterators must be copy constructible.");
+  static_assert(std::is_copy_assignable<_RandomAccessIterator>::value, "Iterators must be copy assignable.");
+
+  (void)std::__partial_sort<_ClassicAlgPolicy>(std::move(__first), std::move(__middle), std::move(__last), __comp);
+}
+
+template <class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+partial_sort(_RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last) {
+  std::partial_sort(__first, __middle, __last, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_PARTIAL_SORT_H
diff --git a/libcxx/include/__cxx03/__algorithm/partial_sort_copy.h b/libcxx/include/__cxx03/__algorithm/partial_sort_copy.h
new file mode 100644
index 000000000000..ef7c9d34d949
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/partial_sort_copy.h
@@ -0,0 +1,106 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_PARTIAL_SORT_COPY_H
+#define _LIBCPP___ALGORITHM_PARTIAL_SORT_COPY_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_heap.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/sift_down.h>
+#include <__algorithm/sort_heap.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/is_callable.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy,
+          class _Compare,
+          class _InputIterator,
+          class _Sentinel1,
+          class _RandomAccessIterator,
+          class _Sentinel2,
+          class _Proj1,
+          class _Proj2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator, _RandomAccessIterator> __partial_sort_copy(
+    _InputIterator __first,
+    _Sentinel1 __last,
+    _RandomAccessIterator __result_first,
+    _Sentinel2 __result_last,
+    _Compare&& __comp,
+    _Proj1&& __proj1,
+    _Proj2&& __proj2) {
+  _RandomAccessIterator __r = __result_first;
+  auto&& __projected_comp   = std::__make_projected(__comp, __proj2);
+
+  if (__r != __result_last) {
+    for (; __first != __last && __r != __result_last; ++__first, (void)++__r)
+      *__r = *__first;
+    std::__make_heap<_AlgPolicy>(__result_first, __r, __projected_comp);
+    typename iterator_traits<_RandomAccessIterator>::difference_type __len = __r - __result_first;
+    for (; __first != __last; ++__first)
+      if (std::__invoke(__comp, std::__invoke(__proj1, *__first), std::__invoke(__proj2, *__result_first))) {
+        *__result_first = *__first;
+        std::__sift_down<_AlgPolicy>(__result_first, __projected_comp, __len, __result_first);
+      }
+    std::__sort_heap<_AlgPolicy>(__result_first, __r, __projected_comp);
+  }
+
+  return pair<_InputIterator, _RandomAccessIterator>(
+      _IterOps<_AlgPolicy>::next(std::move(__first), std::move(__last)), std::move(__r));
+}
+
+template <class _InputIterator, class _RandomAccessIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator partial_sort_copy(
+    _InputIterator __first,
+    _InputIterator __last,
+    _RandomAccessIterator __result_first,
+    _RandomAccessIterator __result_last,
+    _Compare __comp) {
+  static_assert(
+      __is_callable<_Compare, decltype(*__first), decltype(*__result_first)>::value, "Comparator has to be callable");
+
+  auto __result = std::__partial_sort_copy<_ClassicAlgPolicy>(
+      __first,
+      __last,
+      __result_first,
+      __result_last,
+      static_cast<__comp_ref_type<_Compare> >(__comp),
+      __identity(),
+      __identity());
+  return __result.second;
+}
+
+template <class _InputIterator, class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator partial_sort_copy(
+    _InputIterator __first,
+    _InputIterator __last,
+    _RandomAccessIterator __result_first,
+    _RandomAccessIterator __result_last) {
+  return std::partial_sort_copy(__first, __last, __result_first, __result_last, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_PARTIAL_SORT_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/partition.h b/libcxx/include/__cxx03/__algorithm/partition.h
new file mode 100644
index 000000000000..824e49b9ec21
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/partition.h
@@ -0,0 +1,90 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_PARTITION_H
+#define _LIBCPP___ALGORITHM_PARTITION_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Predicate, class _AlgPolicy, class _ForwardIterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_ForwardIterator, _ForwardIterator>
+__partition_impl(_ForwardIterator __first, _Sentinel __last, _Predicate __pred, forward_iterator_tag) {
+  while (true) {
+    if (__first == __last)
+      return std::make_pair(std::move(__first), std::move(__first));
+    if (!__pred(*__first))
+      break;
+    ++__first;
+  }
+
+  _ForwardIterator __p = __first;
+  while (++__p != __last) {
+    if (__pred(*__p)) {
+      _IterOps<_AlgPolicy>::iter_swap(__first, __p);
+      ++__first;
+    }
+  }
+  return std::make_pair(std::move(__first), std::move(__p));
+}
+
+template <class _Predicate, class _AlgPolicy, class _BidirectionalIterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_BidirectionalIterator, _BidirectionalIterator>
+__partition_impl(_BidirectionalIterator __first, _Sentinel __sentinel, _Predicate __pred, bidirectional_iterator_tag) {
+  _BidirectionalIterator __original_last = _IterOps<_AlgPolicy>::next(__first, __sentinel);
+  _BidirectionalIterator __last          = __original_last;
+
+  while (true) {
+    while (true) {
+      if (__first == __last)
+        return std::make_pair(std::move(__first), std::move(__original_last));
+      if (!__pred(*__first))
+        break;
+      ++__first;
+    }
+    do {
+      if (__first == --__last)
+        return std::make_pair(std::move(__first), std::move(__original_last));
+    } while (!__pred(*__last));
+    _IterOps<_AlgPolicy>::iter_swap(__first, __last);
+    ++__first;
+  }
+}
+
+template <class _AlgPolicy, class _ForwardIterator, class _Sentinel, class _Predicate, class _IterCategory>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_ForwardIterator, _ForwardIterator>
+__partition(_ForwardIterator __first, _Sentinel __last, _Predicate&& __pred, _IterCategory __iter_category) {
+  return std::__partition_impl<__remove_cvref_t<_Predicate>&, _AlgPolicy>(
+      std::move(__first), std::move(__last), __pred, __iter_category);
+}
+
+template <class _ForwardIterator, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+partition(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
+  using _IterCategory = typename iterator_traits<_ForwardIterator>::iterator_category;
+  auto __result = std::__partition<_ClassicAlgPolicy>(std::move(__first), std::move(__last), __pred, _IterCategory());
+  return __result.first;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_PARTITION_H
diff --git a/libcxx/include/__cxx03/__algorithm/partition_copy.h b/libcxx/include/__cxx03/__algorithm/partition_copy.h
new file mode 100644
index 000000000000..147b45c7882a
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/partition_copy.h
@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_PARTITION_COPY_H
+#define _LIBCPP___ALGORITHM_PARTITION_COPY_H
+
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _OutputIterator1, class _OutputIterator2, class _Predicate>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_OutputIterator1, _OutputIterator2> partition_copy(
+    _InputIterator __first,
+    _InputIterator __last,
+    _OutputIterator1 __out_true,
+    _OutputIterator2 __out_false,
+    _Predicate __pred) {
+  for (; __first != __last; ++__first) {
+    if (__pred(*__first)) {
+      *__out_true = *__first;
+      ++__out_true;
+    } else {
+      *__out_false = *__first;
+      ++__out_false;
+    }
+  }
+  return pair<_OutputIterator1, _OutputIterator2>(__out_true, __out_false);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_PARTITION_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/partition_point.h b/libcxx/include/__cxx03/__algorithm/partition_point.h
new file mode 100644
index 000000000000..504dbf1d1a05
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/partition_point.h
@@ -0,0 +1,44 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_PARTITION_POINT_H
+#define _LIBCPP___ALGORITHM_PARTITION_POINT_H
+
+#include <__algorithm/half_positive.h>
+#include <__config>
+#include <__iterator/advance.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ForwardIterator, class _Predicate>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+partition_point(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
+  typedef typename iterator_traits<_ForwardIterator>::difference_type difference_type;
+  difference_type __len = std::distance(__first, __last);
+  while (__len != 0) {
+    difference_type __l2 = std::__half_positive(__len);
+    _ForwardIterator __m = __first;
+    std::advance(__m, __l2);
+    if (__pred(*__m)) {
+      __first = ++__m;
+      __len -= __l2 + 1;
+    } else
+      __len = __l2;
+  }
+  return __first;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_PARTITION_POINT_H
diff --git a/libcxx/include/__cxx03/__algorithm/pop_heap.h b/libcxx/include/__cxx03/__algorithm/pop_heap.h
new file mode 100644
index 000000000000..6d23830097ff
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/pop_heap.h
@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_POP_HEAP_H
+#define _LIBCPP___ALGORITHM_POP_HEAP_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/push_heap.h>
+#include <__algorithm/sift_down.h>
+#include <__assert>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_constructible.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+__pop_heap(_RandomAccessIterator __first,
+           _RandomAccessIterator __last,
+           _Compare& __comp,
+           typename iterator_traits<_RandomAccessIterator>::difference_type __len) {
+  // Calling `pop_heap` on an empty range is undefined behavior, but in practice it will be a no-op.
+  _LIBCPP_ASSERT_PEDANTIC(__len > 0, "The heap given to pop_heap must be non-empty");
+
+  __comp_ref_type<_Compare> __comp_ref = __comp;
+
+  using value_type = typename iterator_traits<_RandomAccessIterator>::value_type;
+  if (__len > 1) {
+    value_type __top             = _IterOps<_AlgPolicy>::__iter_move(__first); // create a hole at __first
+    _RandomAccessIterator __hole = std::__floyd_sift_down<_AlgPolicy>(__first, __comp_ref, __len);
+    --__last;
+
+    if (__hole == __last) {
+      *__hole = std::move(__top);
+    } else {
+      *__hole = _IterOps<_AlgPolicy>::__iter_move(__last);
+      ++__hole;
+      *__last = std::move(__top);
+      std::__sift_up<_AlgPolicy>(__first, __hole, __comp_ref, __hole - __first);
+    }
+  }
+}
+
+template <class _RandomAccessIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+pop_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+  static_assert(std::is_copy_constructible<_RandomAccessIterator>::value, "Iterators must be copy constructible.");
+  static_assert(std::is_copy_assignable<_RandomAccessIterator>::value, "Iterators must be copy assignable.");
+
+  typename iterator_traits<_RandomAccessIterator>::difference_type __len = __last - __first;
+  std::__pop_heap<_ClassicAlgPolicy>(std::move(__first), std::move(__last), __comp, __len);
+}
+
+template <class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+pop_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) {
+  std::pop_heap(std::move(__first), std::move(__last), __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_POP_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/prev_permutation.h b/libcxx/include/__cxx03/__algorithm/prev_permutation.h
new file mode 100644
index 000000000000..8d15b6806401
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/prev_permutation.h
@@ -0,0 +1,75 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_PREV_PERMUTATION_H
+#define _LIBCPP___ALGORITHM_PREV_PERMUTATION_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/reverse.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Compare, class _BidirectionalIterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_BidirectionalIterator, bool>
+__prev_permutation(_BidirectionalIterator __first, _Sentinel __last, _Compare&& __comp) {
+  using _Result = pair<_BidirectionalIterator, bool>;
+
+  _BidirectionalIterator __last_iter = _IterOps<_AlgPolicy>::next(__first, __last);
+  _BidirectionalIterator __i         = __last_iter;
+  if (__first == __last || __first == --__i)
+    return _Result(std::move(__last_iter), false);
+
+  while (true) {
+    _BidirectionalIterator __ip1 = __i;
+    if (__comp(*__ip1, *--__i)) {
+      _BidirectionalIterator __j = __last_iter;
+      while (!__comp(*--__j, *__i))
+        ;
+      _IterOps<_AlgPolicy>::iter_swap(__i, __j);
+      std::__reverse<_AlgPolicy>(__ip1, __last_iter);
+      return _Result(std::move(__last_iter), true);
+    }
+    if (__i == __first) {
+      std::__reverse<_AlgPolicy>(__first, __last_iter);
+      return _Result(std::move(__last_iter), false);
+    }
+  }
+}
+
+template <class _BidirectionalIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+prev_permutation(_BidirectionalIterator __first, _BidirectionalIterator __last, _Compare __comp) {
+  return std::__prev_permutation<_ClassicAlgPolicy>(
+             std::move(__first), std::move(__last), static_cast<__comp_ref_type<_Compare> >(__comp))
+      .second;
+}
+
+template <class _BidirectionalIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+prev_permutation(_BidirectionalIterator __first, _BidirectionalIterator __last) {
+  return std::prev_permutation(__first, __last, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_PREV_PERMUTATION_H
diff --git a/libcxx/include/__cxx03/__algorithm/pstl.h b/libcxx/include/__cxx03/__algorithm/pstl.h
new file mode 100644
index 000000000000..0bb052b3f97c
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/pstl.h
@@ -0,0 +1,663 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_PSTL_H
+#define _LIBCPP___ALGORITHM_PSTL_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
+
+#  include <__functional/operations.h>
+#  include <__iterator/cpp17_iterator_concepts.h>
+#  include <__iterator/iterator_traits.h>
+#  include <__pstl/backend.h>
+#  include <__pstl/dispatch.h>
+#  include <__pstl/handle_exception.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/is_execution_policy.h>
+#  include <__type_traits/remove_cvref.h>
+#  include <__utility/forward.h>
+#  include <__utility/move.h>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Predicate,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool
+any_of(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "any_of requires a ForwardIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__any_of, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Pred,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool
+all_of(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "all_of requires a ForwardIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__all_of, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Pred,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool
+none_of(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "none_of requires a ForwardIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__none_of, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _ForwardOutIterator,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator
+copy(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _ForwardOutIterator __result) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(
+      _ForwardIterator, "copy(first, last, result) requires [first, last) to be ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(
+      _ForwardOutIterator, "copy(first, last, result) requires result to be a ForwardIterator");
+  _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(
+      _ForwardOutIterator, decltype(*__first), "copy(first, last, result) requires result to be an OutputIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__copy, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__result));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _ForwardOutIterator,
+          class _Size,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator
+copy_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _Size __n, _ForwardOutIterator __result) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(
+      _ForwardIterator, "copy_n(first, n, result) requires first to be a ForwardIterator");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(
+      _ForwardOutIterator, "copy_n(first, n, result) requires result to be a ForwardIterator");
+  _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(
+      _ForwardOutIterator, decltype(*__first), "copy_n(first, n, result) requires result to be an OutputIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__copy_n, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__n), std::move(__result));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Predicate,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI __iter_diff_t<_ForwardIterator>
+count_if(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(
+      _ForwardIterator, "count_if(first, last, pred) requires [first, last) to be ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__count_if, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Tp,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI __iter_diff_t<_ForwardIterator>
+count(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(
+      _ForwardIterator, "count(first, last, val) requires [first, last) to be ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__count, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), __value);
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _Pred,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI bool
+equal(_ExecutionPolicy&& __policy,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _Pred __pred) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "equal requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "equal requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__equal_3leg, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__pred));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI bool
+equal(_ExecutionPolicy&& __policy, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "equal requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "equal requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__equal_3leg, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      equal_to{});
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _Pred,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI bool
+equal(_ExecutionPolicy&& __policy,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _ForwardIterator2 __last2,
+      _Pred __pred) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "equal requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "equal requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__equal, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__last2),
+      std::move(__pred));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI bool
+equal(_ExecutionPolicy&& __policy,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _ForwardIterator2 __last2) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "equal requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "equal requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__equal, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__last2),
+      equal_to{});
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Tp,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+fill(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "fill requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__fill, __pstl::__current_configuration, _RawPolicy>;
+  __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), __value);
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Size,
+          class _Tp,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+fill_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _Size __n, const _Tp& __value) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "fill_n requires a ForwardIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__fill_n, __pstl::__current_configuration, _RawPolicy>;
+  __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__n), __value);
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Predicate,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator
+find_if(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "find_if requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__find_if, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Predicate,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator
+find_if_not(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "find_if_not requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__find_if_not, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Tp,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator
+find(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "find requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__find, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), __value);
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Function,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+for_each(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Function __func) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "for_each requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__for_each, __pstl::__current_configuration, _RawPolicy>;
+  __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__func));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Size,
+          class _Function,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+for_each_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _Size __size, _Function __func) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "for_each_n requires a ForwardIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__for_each_n, __pstl::__current_configuration, _RawPolicy>;
+  __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__size), std::move(__func));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Generator,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+generate(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Generator __gen) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "generate requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__generate, __pstl::__current_configuration, _RawPolicy>;
+  __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__gen));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Size,
+          class _Generator,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+generate_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _Size __n, _Generator __gen) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "generate_n requires a ForwardIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__generate_n, __pstl::__current_configuration, _RawPolicy>;
+  __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__n), std::move(__gen));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Predicate,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool
+is_partitioned(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "is_partitioned requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__is_partitioned, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardOutIterator,
+          class _Comp,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator
+merge(_ExecutionPolicy&& __policy,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _ForwardIterator2 __last2,
+      _ForwardOutIterator __result,
+      _Comp __comp) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "merge requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "merge requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(*__first1), "merge requires an OutputIterator");
+  _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(*__first2), "merge requires an OutputIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__merge, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__last2),
+      std::move(__result),
+      std::move(__comp));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardOutIterator,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator
+merge(_ExecutionPolicy&& __policy,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _ForwardIterator2 __last2,
+      _ForwardOutIterator __result) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "merge requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "merge requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(*__first1), "merge requires an OutputIterator");
+  _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(*__first2), "merge requires an OutputIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__merge, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__last2),
+      std::move(__result),
+      less{});
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _ForwardOutIterator,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator
+move(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _ForwardOutIterator __result) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "move requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "move requires an OutputIterator");
+  _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(
+      _ForwardOutIterator, decltype(std::move(*__first)), "move requires an OutputIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__move, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__result));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Pred,
+          class _Tp,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+replace_if(_ExecutionPolicy&& __policy,
+           _ForwardIterator __first,
+           _ForwardIterator __last,
+           _Pred __pred,
+           const _Tp& __new_value) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "replace_if requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__replace_if, __pstl::__current_configuration, _RawPolicy>;
+  __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred), __new_value);
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Tp,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+replace(_ExecutionPolicy&& __policy,
+        _ForwardIterator __first,
+        _ForwardIterator __last,
+        const _Tp& __old_value,
+        const _Tp& __new_value) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "replace requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__replace, __pstl::__current_configuration, _RawPolicy>;
+  __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), __old_value, __new_value);
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _ForwardOutIterator,
+          class _Pred,
+          class _Tp,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void replace_copy_if(
+    _ExecutionPolicy&& __policy,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _ForwardOutIterator __result,
+    _Pred __pred,
+    const _Tp& __new_value) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "replace_copy_if requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "replace_copy_if requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(
+      _ForwardOutIterator, decltype(*__first), "replace_copy_if requires an OutputIterator");
+  _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, const _Tp&, "replace_copy requires an OutputIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__replace_copy_if, __pstl::__current_configuration, _RawPolicy>;
+  __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first),
+      std::move(__last),
+      std::move(__result),
+      std::move(__pred),
+      __new_value);
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _ForwardOutIterator,
+          class _Tp,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void replace_copy(
+    _ExecutionPolicy&& __policy,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _ForwardOutIterator __result,
+    const _Tp& __old_value,
+    const _Tp& __new_value) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "replace_copy requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "replace_copy requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(
+      _ForwardOutIterator, decltype(*__first), "replace_copy requires an OutputIterator");
+  _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, const _Tp&, "replace_copy requires an OutputIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__replace_copy, __pstl::__current_configuration, _RawPolicy>;
+  __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first),
+      std::move(__last),
+      std::move(__result),
+      __old_value,
+      __new_value);
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _ForwardOutIterator,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator rotate_copy(
+    _ExecutionPolicy&& __policy,
+    _ForwardIterator __first,
+    _ForwardIterator __middle,
+    _ForwardIterator __last,
+    _ForwardOutIterator __result) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "rotate_copy requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "rotate_copy requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(
+      _ForwardOutIterator, decltype(*__first), "rotate_copy requires an OutputIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__rotate_copy, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first),
+      std::move(__middle),
+      std::move(__last),
+      std::move(__result));
+}
+
+template <class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _Comp,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+sort(_ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) {
+  _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "sort requires RandomAccessIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__sort, __pstl::__current_configuration, _RawPolicy>;
+  __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__comp));
+}
+
+template <class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+sort(_ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last) {
+  _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "sort requires RandomAccessIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__sort, __pstl::__current_configuration, _RawPolicy>;
+  __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), less{});
+}
+
+template <class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _Comp,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+stable_sort(_ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) {
+  _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "stable_sort requires RandomAccessIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__stable_sort, __pstl::__current_configuration, _RawPolicy>;
+  __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__comp));
+}
+
+template <class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+stable_sort(_ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last) {
+  _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "stable_sort requires RandomAccessIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__stable_sort, __pstl::__current_configuration, _RawPolicy>;
+  __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), less{});
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _ForwardOutIterator,
+          class _UnaryOperation,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator transform(
+    _ExecutionPolicy&& __policy,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _ForwardOutIterator __result,
+    _UnaryOperation __op) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "transform requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "transform requires an OutputIterator");
+  _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(
+      _ForwardOutIterator, decltype(__op(*__first)), "transform requires an OutputIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__transform, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first),
+      std::move(__last),
+      std::move(__result),
+      std::move(__op));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardOutIterator,
+          class _BinaryOperation,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator transform(
+    _ExecutionPolicy&& __policy,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardOutIterator __result,
+    _BinaryOperation __op) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "transform requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "transform requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "transform requires an OutputIterator");
+  _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(
+      _ForwardOutIterator, decltype(__op(*__first1, *__first2)), "transform requires an OutputIterator");
+  using _Implementation = __pstl::__dispatch<__pstl::__transform_binary, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__result),
+      std::move(__op));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_PSTL_H
diff --git a/libcxx/include/__cxx03/__algorithm/push_heap.h b/libcxx/include/__cxx03/__algorithm/push_heap.h
new file mode 100644
index 000000000000..ec0b445f2b70
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/push_heap.h
@@ -0,0 +1,84 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_PUSH_HEAP_H
+#define _LIBCPP___ALGORITHM_PUSH_HEAP_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_constructible.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+__sift_up(_RandomAccessIterator __first,
+          _RandomAccessIterator __last,
+          _Compare&& __comp,
+          typename iterator_traits<_RandomAccessIterator>::difference_type __len) {
+  using value_type = typename iterator_traits<_RandomAccessIterator>::value_type;
+
+  if (__len > 1) {
+    __len                       = (__len - 2) / 2;
+    _RandomAccessIterator __ptr = __first + __len;
+
+    if (__comp(*__ptr, *--__last)) {
+      value_type __t(_IterOps<_AlgPolicy>::__iter_move(__last));
+      do {
+        *__last = _IterOps<_AlgPolicy>::__iter_move(__ptr);
+        __last  = __ptr;
+        if (__len == 0)
+          break;
+        __len = (__len - 1) / 2;
+        __ptr = __first + __len;
+      } while (__comp(*__ptr, __t));
+
+      *__last = std::move(__t);
+    }
+  }
+}
+
+template <class _AlgPolicy, class _RandomAccessIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+__push_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare& __comp) {
+  typename iterator_traits<_RandomAccessIterator>::difference_type __len = __last - __first;
+  std::__sift_up<_AlgPolicy, __comp_ref_type<_Compare> >(std::move(__first), std::move(__last), __comp, __len);
+}
+
+template <class _RandomAccessIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+push_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+  static_assert(std::is_copy_constructible<_RandomAccessIterator>::value, "Iterators must be copy constructible.");
+  static_assert(std::is_copy_assignable<_RandomAccessIterator>::value, "Iterators must be copy assignable.");
+
+  std::__push_heap<_ClassicAlgPolicy>(std::move(__first), std::move(__last), __comp);
+}
+
+template <class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+push_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) {
+  std::push_heap(std::move(__first), std::move(__last), __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_PUSH_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_adjacent_find.h b/libcxx/include/__cxx03/__algorithm/ranges_adjacent_find.h
new file mode 100644
index 000000000000..3c54f723310a
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_adjacent_find.h
@@ -0,0 +1,83 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_ADJACENT_FIND_H
+#define _LIBCPP___ALGORITHM_RANGES_ADJACENT_FIND_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __adjacent_find {
+struct __fn {
+  template <class _Iter, class _Sent, class _Proj, class _Pred>
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __adjacent_find_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
+    if (__first == __last)
+      return __first;
+
+    auto __i = __first;
+    while (++__i != __last) {
+      if (std::invoke(__pred, std::invoke(__proj, *__first), std::invoke(__proj, *__i)))
+        return __first;
+      __first = __i;
+    }
+    return __i;
+  }
+
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj                                                                       = identity,
+            indirect_binary_predicate<projected<_Iter, _Proj>, projected<_Iter, _Proj>> _Pred = ranges::equal_to>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Pred __pred = {}, _Proj __proj = {}) const {
+    return __adjacent_find_impl(std::move(__first), std::move(__last), __pred, __proj);
+  }
+
+  template <forward_range _Range,
+            class _Proj = identity,
+            indirect_binary_predicate<projected<iterator_t<_Range>, _Proj>, projected<iterator_t<_Range>, _Proj>>
+                _Pred = ranges::equal_to>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __range, _Pred __pred = {}, _Proj __proj = {}) const {
+    return __adjacent_find_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
+  }
+};
+} // namespace __adjacent_find
+
+inline namespace __cpo {
+inline constexpr auto adjacent_find = __adjacent_find::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_ADJACENT_FIND_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_all_of.h b/libcxx/include/__cxx03/__algorithm/ranges_all_of.h
new file mode 100644
index 000000000000..2f603b32f32d
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_all_of.h
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_ALL_OF_H
+#define _LIBCPP___ALGORITHM_RANGES_ALL_OF_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __all_of {
+struct __fn {
+  template <class _Iter, class _Sent, class _Proj, class _Pred>
+  _LIBCPP_HIDE_FROM_ABI constexpr static bool __all_of_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
+    for (; __first != __last; ++__first) {
+      if (!std::invoke(__pred, std::invoke(__proj, *__first)))
+        return false;
+    }
+    return true;
+  }
+
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
+    return __all_of_impl(std::move(__first), std::move(__last), __pred, __proj);
+  }
+
+  template <input_range _Range,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+    return __all_of_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
+  }
+};
+} // namespace __all_of
+
+inline namespace __cpo {
+inline constexpr auto all_of = __all_of::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_ALL_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_any_of.h b/libcxx/include/__cxx03/__algorithm/ranges_any_of.h
new file mode 100644
index 000000000000..205fcecc086e
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_any_of.h
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_ANY_OF_H
+#define _LIBCPP___ALGORITHM_RANGES_ANY_OF_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __any_of {
+struct __fn {
+  template <class _Iter, class _Sent, class _Proj, class _Pred>
+  _LIBCPP_HIDE_FROM_ABI constexpr static bool __any_of_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
+    for (; __first != __last; ++__first) {
+      if (std::invoke(__pred, std::invoke(__proj, *__first)))
+        return true;
+    }
+    return false;
+  }
+
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Iter __first, _Sent __last, _Pred __pred = {}, _Proj __proj = {}) const {
+    return __any_of_impl(std::move(__first), std::move(__last), __pred, __proj);
+  }
+
+  template <input_range _Range,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+    return __any_of_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
+  }
+};
+} // namespace __any_of
+
+inline namespace __cpo {
+inline constexpr auto any_of = __any_of::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_ANY_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_binary_search.h b/libcxx/include/__cxx03/__algorithm/ranges_binary_search.h
new file mode 100644
index 000000000000..1ef2bd62b599
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_binary_search.h
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_BINARY_SEARCH_H
+#define _LIBCPP___ALGORITHM_RANGES_BINARY_SEARCH_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/lower_bound.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __binary_search {
+struct __fn {
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Type,
+            class _Proj                                                             = identity,
+            indirect_strict_weak_order<const _Type*, projected<_Iter, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Iter __first, _Sent __last, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __ret = std::__lower_bound<_RangeAlgPolicy>(__first, __last, __value, __comp, __proj);
+    return __ret != __last && !std::invoke(__comp, __value, std::invoke(__proj, *__ret));
+  }
+
+  template <forward_range _Range,
+            class _Type,
+            class _Proj                                                                          = identity,
+            indirect_strict_weak_order<const _Type*, projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Range&& __r, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __first = ranges::begin(__r);
+    auto __last  = ranges::end(__r);
+    auto __ret   = std::__lower_bound<_RangeAlgPolicy>(__first, __last, __value, __comp, __proj);
+    return __ret != __last && !std::invoke(__comp, __value, std::invoke(__proj, *__ret));
+  }
+};
+} // namespace __binary_search
+
+inline namespace __cpo {
+inline constexpr auto binary_search = __binary_search::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_BINARY_SEARCH_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_clamp.h b/libcxx/include/__cxx03/__algorithm/ranges_clamp.h
new file mode 100644
index 000000000000..e6181ef9435e
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_clamp.h
@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_CLAMP_H
+#define _LIBCPP___ALGORITHM_RANGES_CLAMP_H
+
+#include <__assert>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __clamp {
+struct __fn {
+  template <class _Type,
+            class _Proj                                                      = identity,
+            indirect_strict_weak_order<projected<const _Type*, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Type& operator()(
+      const _Type& __value, const _Type& __low, const _Type& __high, _Comp __comp = {}, _Proj __proj = {}) const {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        !bool(std::invoke(__comp, std::invoke(__proj, __high), std::invoke(__proj, __low))),
+        "Bad bounds passed to std::ranges::clamp");
+
+    auto&& __projected = std::invoke(__proj, __value);
+    if (std::invoke(__comp, std::forward<decltype(__projected)>(__projected), std::invoke(__proj, __low)))
+      return __low;
+    else if (std::invoke(__comp, std::invoke(__proj, __high), std::forward<decltype(__projected)>(__projected)))
+      return __high;
+    else
+      return __value;
+  }
+};
+} // namespace __clamp
+
+inline namespace __cpo {
+inline constexpr auto clamp = __clamp::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_CLAMP_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_contains.h b/libcxx/include/__cxx03/__algorithm/ranges_contains.h
new file mode 100644
index 000000000000..4836c3baed17
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_contains.h
@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_CONTAINS_H
+#define _LIBCPP___ALGORITHM_RANGES_CONTAINS_H
+
+#include <__algorithm/ranges_find.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__functional/reference_wrapper.h>
+#include <__iterator/concepts.h>
+#include <__iterator/indirectly_comparable.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __contains {
+struct __fn {
+  template <input_iterator _Iter, sentinel_for<_Iter> _Sent, class _Type, class _Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<_Iter, _Proj>, const _Type*>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool static
+  operator()(_Iter __first, _Sent __last, const _Type& __value, _Proj __proj = {}) {
+    return ranges::find(std::move(__first), __last, __value, std::ref(__proj)) != __last;
+  }
+
+  template <input_range _Range, class _Type, class _Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _Type*>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool static
+  operator()(_Range&& __range, const _Type& __value, _Proj __proj = {}) {
+    return ranges::find(ranges::begin(__range), ranges::end(__range), __value, std::ref(__proj)) !=
+           ranges::end(__range);
+  }
+};
+} // namespace __contains
+
+inline namespace __cpo {
+inline constexpr auto contains = __contains::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_CONTAINS_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_contains_subrange.h b/libcxx/include/__cxx03/__algorithm/ranges_contains_subrange.h
new file mode 100644
index 000000000000..4398c457fd05
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_contains_subrange.h
@@ -0,0 +1,97 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_CONTAINS_SUBRANGE_H
+#define _LIBCPP___ALGORITHM_RANGES_CONTAINS_SUBRANGE_H
+
+#include <__algorithm/ranges_search.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__functional/reference_wrapper.h>
+#include <__iterator/concepts.h>
+#include <__iterator/indirectly_comparable.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/size.h>
+#include <__ranges/subrange.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __contains_subrange {
+struct __fn {
+  template <forward_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            forward_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_comparable<_Iter1, _Iter2, _Pred, _Proj1, _Proj2>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool static operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred __pred   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) {
+    if (__first2 == __last2)
+      return true;
+
+    auto __ret = ranges::search(
+        std::move(__first1), __last1, std::move(__first2), __last2, __pred, std::ref(__proj1), std::ref(__proj2));
+    return __ret.empty() == false;
+  }
+
+  template <forward_range _Range1,
+            forward_range _Range2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_comparable<iterator_t<_Range1>, iterator_t<_Range2>, _Pred, _Proj1, _Proj2>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool static
+  operator()(_Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) {
+    if constexpr (sized_range<_Range2>) {
+      if (ranges::size(__range2) == 0)
+        return true;
+    } else {
+      if (ranges::begin(__range2) == ranges::end(__range2))
+        return true;
+    }
+
+    auto __ret = ranges::search(__range1, __range2, __pred, std::ref(__proj1), std::ref(__proj2));
+    return __ret.empty() == false;
+  }
+};
+} // namespace __contains_subrange
+
+inline namespace __cpo {
+inline constexpr auto contains_subrange = __contains_subrange::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_CONTAINS_SUBRANGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_copy.h
new file mode 100644
index 000000000000..e1d6d32f05f7
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_copy.h
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_COPY_H
+#define _LIBCPP___ALGORITHM_RANGES_COPY_H
+
+#include <__algorithm/copy.h>
+#include <__algorithm/in_out_result.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__iterator/concepts.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter, class _OutIter>
+using copy_result = in_out_result<_InIter, _OutIter>;
+
+namespace __copy {
+struct __fn {
+  template <input_iterator _InIter, sentinel_for<_InIter> _Sent, weakly_incrementable _OutIter>
+    requires indirectly_copyable<_InIter, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr copy_result<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result) const {
+    auto __ret = std::__copy<_RangeAlgPolicy>(std::move(__first), std::move(__last), std::move(__result));
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+
+  template <input_range _Range, weakly_incrementable _OutIter>
+    requires indirectly_copyable<iterator_t<_Range>, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr copy_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __r, _OutIter __result) const {
+    auto __ret = std::__copy<_RangeAlgPolicy>(ranges::begin(__r), ranges::end(__r), std::move(__result));
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+};
+} // namespace __copy
+
+inline namespace __cpo {
+inline constexpr auto copy = __copy::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_copy_backward.h b/libcxx/include/__cxx03/__algorithm/ranges_copy_backward.h
new file mode 100644
index 000000000000..93e326042503
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_copy_backward.h
@@ -0,0 +1,69 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_COPY_BACKWARD_H
+#define _LIBCPP___ALGORITHM_RANGES_COPY_BACKWARD_H
+
+#include <__algorithm/copy_backward.h>
+#include <__algorithm/in_out_result.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _Ip, class _Op>
+using copy_backward_result = in_out_result<_Ip, _Op>;
+
+namespace __copy_backward {
+struct __fn {
+  template <bidirectional_iterator _InIter1, sentinel_for<_InIter1> _Sent1, bidirectional_iterator _InIter2>
+    requires indirectly_copyable<_InIter1, _InIter2>
+  _LIBCPP_HIDE_FROM_ABI constexpr copy_backward_result<_InIter1, _InIter2>
+  operator()(_InIter1 __first, _Sent1 __last, _InIter2 __result) const {
+    auto __ret = std::__copy_backward<_RangeAlgPolicy>(std::move(__first), std::move(__last), std::move(__result));
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+
+  template <bidirectional_range _Range, bidirectional_iterator _Iter>
+    requires indirectly_copyable<iterator_t<_Range>, _Iter>
+  _LIBCPP_HIDE_FROM_ABI constexpr copy_backward_result<borrowed_iterator_t<_Range>, _Iter>
+  operator()(_Range&& __r, _Iter __result) const {
+    auto __ret = std::__copy_backward<_RangeAlgPolicy>(ranges::begin(__r), ranges::end(__r), std::move(__result));
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+};
+} // namespace __copy_backward
+
+inline namespace __cpo {
+inline constexpr auto copy_backward = __copy_backward::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_COPY_BACKWARD_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_copy_if.h b/libcxx/include/__cxx03/__algorithm/ranges_copy_if.h
new file mode 100644
index 000000000000..4b41d2154e7f
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_copy_if.h
@@ -0,0 +1,87 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_COPY_IF_H
+#define _LIBCPP___ALGORITHM_RANGES_COPY_IF_H
+
+#include <__algorithm/in_out_result.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _Ip, class _Op>
+using copy_if_result = in_out_result<_Ip, _Op>;
+
+namespace __copy_if {
+struct __fn {
+  template <class _InIter, class _Sent, class _OutIter, class _Proj, class _Pred>
+  _LIBCPP_HIDE_FROM_ABI static constexpr copy_if_result<_InIter, _OutIter>
+  __copy_if_impl(_InIter __first, _Sent __last, _OutIter __result, _Pred& __pred, _Proj& __proj) {
+    for (; __first != __last; ++__first) {
+      if (std::invoke(__pred, std::invoke(__proj, *__first))) {
+        *__result = *__first;
+        ++__result;
+      }
+    }
+    return {std::move(__first), std::move(__result)};
+  }
+
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            weakly_incrementable _OutIter,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
+    requires indirectly_copyable<_Iter, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr copy_if_result<_Iter, _OutIter>
+  operator()(_Iter __first, _Sent __last, _OutIter __result, _Pred __pred, _Proj __proj = {}) const {
+    return __copy_if_impl(std::move(__first), std::move(__last), std::move(__result), __pred, __proj);
+  }
+
+  template <input_range _Range,
+            weakly_incrementable _OutIter,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+    requires indirectly_copyable<iterator_t<_Range>, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr copy_if_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __r, _OutIter __result, _Pred __pred, _Proj __proj = {}) const {
+    return __copy_if_impl(ranges::begin(__r), ranges::end(__r), std::move(__result), __pred, __proj);
+  }
+};
+} // namespace __copy_if
+
+inline namespace __cpo {
+inline constexpr auto copy_if = __copy_if::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_COPY_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_copy_n.h b/libcxx/include/__cxx03/__algorithm/ranges_copy_n.h
new file mode 100644
index 000000000000..4353fa99278c
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_copy_n.h
@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_COPY_N_H
+#define _LIBCPP___ALGORITHM_RANGES_COPY_N_H
+
+#include <__algorithm/copy.h>
+#include <__algorithm/in_out_result.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/ranges_copy.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/unreachable_sentinel.h>
+#include <__iterator/wrap_iter.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+template <class _Ip, class _Op>
+using copy_n_result = in_out_result<_Ip, _Op>;
+
+namespace __copy_n {
+struct __fn {
+  template <class _InIter, class _DiffType, class _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr static copy_n_result<_InIter, _OutIter>
+  __go(_InIter __first, _DiffType __n, _OutIter __result) {
+    while (__n != 0) {
+      *__result = *__first;
+      ++__first;
+      ++__result;
+      --__n;
+    }
+    return {std::move(__first), std::move(__result)};
+  }
+
+  template <random_access_iterator _InIter, class _DiffType, random_access_iterator _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr static copy_n_result<_InIter, _OutIter>
+  __go(_InIter __first, _DiffType __n, _OutIter __result) {
+    auto __ret = std::__copy<_RangeAlgPolicy>(__first, __first + __n, __result);
+    return {__ret.first, __ret.second};
+  }
+
+  template <input_iterator _Ip, weakly_incrementable _Op>
+    requires indirectly_copyable<_Ip, _Op>
+  _LIBCPP_HIDE_FROM_ABI constexpr copy_n_result<_Ip, _Op>
+  operator()(_Ip __first, iter_difference_t<_Ip> __n, _Op __result) const {
+    return __go(std::move(__first), __n, std::move(__result));
+  }
+};
+} // namespace __copy_n
+
+inline namespace __cpo {
+inline constexpr auto copy_n = __copy_n::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_COPY_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_count.h b/libcxx/include/__cxx03/__algorithm/ranges_count.h
new file mode 100644
index 000000000000..4f3511743870
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_count.h
@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_COUNT_H
+#define _LIBCPP___ALGORITHM_RANGES_COUNT_H
+
+#include <__algorithm/count.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __count {
+struct __fn {
+  template <input_iterator _Iter, sentinel_for<_Iter> _Sent, class _Type, class _Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<_Iter, _Proj>, const _Type*>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Iter>
+  operator()(_Iter __first, _Sent __last, const _Type& __value, _Proj __proj = {}) const {
+    return std::__count<_RangeAlgPolicy>(std::move(__first), std::move(__last), __value, __proj);
+  }
+
+  template <input_range _Range, class _Type, class _Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _Type*>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr range_difference_t<_Range>
+  operator()(_Range&& __r, const _Type& __value, _Proj __proj = {}) const {
+    return std::__count<_RangeAlgPolicy>(ranges::begin(__r), ranges::end(__r), __value, __proj);
+  }
+};
+} // namespace __count
+
+inline namespace __cpo {
+inline constexpr auto count = __count::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_COUNT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_count_if.h b/libcxx/include/__cxx03/__algorithm/ranges_count_if.h
new file mode 100644
index 000000000000..5f2396ff7d53
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_count_if.h
@@ -0,0 +1,79 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_COUNT_IF_H
+#define _LIBCPP___ALGORITHM_RANGES_COUNT_IF_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+template <class _Iter, class _Sent, class _Proj, class _Pred>
+_LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Iter>
+__count_if_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
+  iter_difference_t<_Iter> __counter(0);
+  for (; __first != __last; ++__first) {
+    if (std::invoke(__pred, std::invoke(__proj, *__first)))
+      ++__counter;
+  }
+  return __counter;
+}
+
+namespace __count_if {
+struct __fn {
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Iter, _Proj>> _Predicate>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Iter>
+  operator()(_Iter __first, _Sent __last, _Predicate __pred, _Proj __proj = {}) const {
+    return ranges::__count_if_impl(std::move(__first), std::move(__last), __pred, __proj);
+  }
+
+  template <input_range _Range,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Predicate>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr range_difference_t<_Range>
+  operator()(_Range&& __r, _Predicate __pred, _Proj __proj = {}) const {
+    return ranges::__count_if_impl(ranges::begin(__r), ranges::end(__r), __pred, __proj);
+  }
+};
+} // namespace __count_if
+
+inline namespace __cpo {
+inline constexpr auto count_if = __count_if::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_COUNT_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_ends_with.h b/libcxx/include/__cxx03/__algorithm/ranges_ends_with.h
new file mode 100644
index 000000000000..06efdef36b7c
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_ends_with.h
@@ -0,0 +1,201 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_ENDS_WITH_H
+#define _LIBCPP___ALGORITHM_RANGES_ENDS_WITH_H
+
+#include <__algorithm/ranges_equal.h>
+#include <__algorithm/ranges_starts_with.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__functional/reference_wrapper.h>
+#include <__iterator/advance.h>
+#include <__iterator/concepts.h>
+#include <__iterator/distance.h>
+#include <__iterator/indirectly_comparable.h>
+#include <__iterator/reverse_iterator.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __ends_with {
+struct __fn {
+  template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class _Pred, class _Proj1, class _Proj2>
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __ends_with_fn_impl_bidirectional(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred& __pred,
+      _Proj1& __proj1,
+      _Proj2& __proj2) {
+    auto __rbegin1 = std::make_reverse_iterator(__last1);
+    auto __rend1   = std::make_reverse_iterator(__first1);
+    auto __rbegin2 = std::make_reverse_iterator(__last2);
+    auto __rend2   = std::make_reverse_iterator(__first2);
+    return ranges::starts_with(
+        __rbegin1, __rend1, __rbegin2, __rend2, std::ref(__pred), std::ref(__proj1), std::ref(__proj2));
+  }
+
+  template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class _Pred, class _Proj1, class _Proj2>
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __ends_with_fn_impl(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred& __pred,
+      _Proj1& __proj1,
+      _Proj2& __proj2) {
+    if constexpr (std::bidirectional_iterator<_Sent1> && std::bidirectional_iterator<_Sent2> &&
+                  (!std::random_access_iterator<_Sent1>) && (!std::random_access_iterator<_Sent2>)) {
+      return __ends_with_fn_impl_bidirectional(__first1, __last1, __first2, __last2, __pred, __proj1, __proj2);
+
+    } else {
+      auto __n1 = ranges::distance(__first1, __last1);
+      auto __n2 = ranges::distance(__first2, __last2);
+      if (__n2 == 0)
+        return true;
+      if (__n2 > __n1)
+        return false;
+
+      return __ends_with_fn_impl_with_offset(
+          std::move(__first1),
+          std::move(__last1),
+          std::move(__first2),
+          std::move(__last2),
+          __pred,
+          __proj1,
+          __proj2,
+          __n1 - __n2);
+    }
+  }
+
+  template <class _Iter1,
+            class _Sent1,
+            class _Iter2,
+            class _Sent2,
+            class _Pred,
+            class _Proj1,
+            class _Proj2,
+            class _Offset>
+  static _LIBCPP_HIDE_FROM_ABI constexpr bool __ends_with_fn_impl_with_offset(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred& __pred,
+      _Proj1& __proj1,
+      _Proj2& __proj2,
+      _Offset __offset) {
+    if constexpr (std::bidirectional_iterator<_Sent1> && std::bidirectional_iterator<_Sent2> &&
+                  !std::random_access_iterator<_Sent1> && !std::random_access_iterator<_Sent2>) {
+      return __ends_with_fn_impl_bidirectional(
+          std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __pred, __proj1, __proj2);
+
+    } else {
+      ranges::advance(__first1, __offset);
+      return ranges::equal(
+          std::move(__first1),
+          std::move(__last1),
+          std::move(__first2),
+          std::move(__last2),
+          std::ref(__pred),
+          std::ref(__proj1),
+          std::ref(__proj2));
+    }
+  }
+
+  template <input_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            input_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires(forward_iterator<_Iter1> || sized_sentinel_for<_Sent1, _Iter1>) &&
+            (forward_iterator<_Iter2> || sized_sentinel_for<_Sent2, _Iter2>) &&
+            indirectly_comparable<_Iter1, _Iter2, _Pred, _Proj1, _Proj2>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred __pred   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    return __ends_with_fn_impl(
+        std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __pred, __proj1, __proj2);
+  }
+
+  template <input_range _Range1,
+            input_range _Range2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires(forward_range<_Range1> || sized_range<_Range1>) && (forward_range<_Range2> || sized_range<_Range2>) &&
+            indirectly_comparable<iterator_t<_Range1>, iterator_t<_Range2>, _Pred, _Proj1, _Proj2>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+    if constexpr (sized_range<_Range1> && sized_range<_Range2>) {
+      auto __n1 = ranges::size(__range1);
+      auto __n2 = ranges::size(__range2);
+      if (__n2 == 0)
+        return true;
+      if (__n2 > __n1)
+        return false;
+      auto __offset = __n1 - __n2;
+
+      return __ends_with_fn_impl_with_offset(
+          ranges::begin(__range1),
+          ranges::end(__range1),
+          ranges::begin(__range2),
+          ranges::end(__range2),
+          __pred,
+          __proj1,
+          __proj2,
+          __offset);
+
+    } else {
+      return __ends_with_fn_impl(
+          ranges::begin(__range1),
+          ranges::end(__range1),
+          ranges::begin(__range2),
+          ranges::end(__range2),
+          __pred,
+          __proj1,
+          __proj2);
+    }
+  }
+};
+} // namespace __ends_with
+
+inline namespace __cpo {
+inline constexpr auto ends_with = __ends_with::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_ENDS_WITH_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_equal.h b/libcxx/include/__cxx03/__algorithm/ranges_equal.h
new file mode 100644
index 000000000000..edbd0e3641c1
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_equal.h
@@ -0,0 +1,109 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_EQUAL_H
+#define _LIBCPP___ALGORITHM_RANGES_EQUAL_H
+
+#include <__algorithm/equal.h>
+#include <__algorithm/unwrap_range.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/distance.h>
+#include <__iterator/indirectly_comparable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __equal {
+struct __fn {
+  template <input_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            input_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_comparable<_Iter1, _Iter2, _Pred, _Proj1, _Proj2>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred __pred   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    if constexpr (sized_sentinel_for<_Sent1, _Iter1> && sized_sentinel_for<_Sent2, _Iter2>) {
+      if (__last1 - __first1 != __last2 - __first2)
+        return false;
+    }
+    auto __unwrapped1 = std::__unwrap_range(std::move(__first1), std::move(__last1));
+    auto __unwrapped2 = std::__unwrap_range(std::move(__first2), std::move(__last2));
+    return std::__equal_impl(
+        std::move(__unwrapped1.first),
+        std::move(__unwrapped1.second),
+        std::move(__unwrapped2.first),
+        std::move(__unwrapped2.second),
+        __pred,
+        __proj1,
+        __proj2);
+  }
+
+  template <input_range _Range1,
+            input_range _Range2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_comparable<iterator_t<_Range1>, iterator_t<_Range2>, _Pred, _Proj1, _Proj2>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+    if constexpr (sized_range<_Range1> && sized_range<_Range2>) {
+      if (ranges::distance(__range1) != ranges::distance(__range2))
+        return false;
+    }
+    auto __unwrapped1 = std::__unwrap_range(ranges::begin(__range1), ranges::end(__range1));
+    auto __unwrapped2 = std::__unwrap_range(ranges::begin(__range2), ranges::end(__range2));
+    return std::__equal_impl(
+        std::move(__unwrapped1.first),
+        std::move(__unwrapped1.second),
+        std::move(__unwrapped2.first),
+        std::move(__unwrapped2.second),
+        __pred,
+        __proj1,
+        __proj2);
+    return false;
+  }
+};
+} // namespace __equal
+
+inline namespace __cpo {
+inline constexpr auto equal = __equal::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_EQUAL_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_equal_range.h b/libcxx/include/__cxx03/__algorithm/ranges_equal_range.h
new file mode 100644
index 000000000000..4a308e016b54
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_equal_range.h
@@ -0,0 +1,80 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_EQUAL_RANGE_H
+#define _LIBCPP___ALGORITHM_RANGES_EQUAL_RANGE_H
+
+#include <__algorithm/equal_range.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__ranges/subrange.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __equal_range {
+
+struct __fn {
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Tp,
+            class _Proj                                                           = identity,
+            indirect_strict_weak_order<const _Tp*, projected<_Iter, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
+  operator()(_Iter __first, _Sent __last, const _Tp& __value, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __ret = std::__equal_range<_RangeAlgPolicy>(std::move(__first), std::move(__last), __value, __comp, __proj);
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+
+  template <forward_range _Range,
+            class _Tp,
+            class _Proj                                                                        = identity,
+            indirect_strict_weak_order<const _Tp*, projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, const _Tp& __value, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __ret =
+        std::__equal_range<_RangeAlgPolicy>(ranges::begin(__range), ranges::end(__range), __value, __comp, __proj);
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+};
+
+} // namespace __equal_range
+
+inline namespace __cpo {
+inline constexpr auto equal_range = __equal_range::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_EQUAL_RANGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_fill.h b/libcxx/include/__cxx03/__algorithm/ranges_fill.h
new file mode 100644
index 000000000000..7a177d85e9f0
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_fill.h
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_FILL_H
+#define _LIBCPP___ALGORITHM_RANGES_FILL_H
+
+#include <__algorithm/ranges_fill_n.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __fill {
+struct __fn {
+  template <class _Type, output_iterator<const _Type&> _Iter, sentinel_for<_Iter> _Sent>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter operator()(_Iter __first, _Sent __last, const _Type& __value) const {
+    if constexpr (random_access_iterator<_Iter> && sized_sentinel_for<_Sent, _Iter>) {
+      return ranges::fill_n(__first, __last - __first, __value);
+    } else {
+      for (; __first != __last; ++__first)
+        *__first = __value;
+      return __first;
+    }
+  }
+
+  template <class _Type, output_range<const _Type&> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range> operator()(_Range&& __range, const _Type& __value) const {
+    return (*this)(ranges::begin(__range), ranges::end(__range), __value);
+  }
+};
+} // namespace __fill
+
+inline namespace __cpo {
+inline constexpr auto fill = __fill::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_FILL_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_fill_n.h b/libcxx/include/__cxx03/__algorithm/ranges_fill_n.h
new file mode 100644
index 000000000000..a6e988c0089c
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_fill_n.h
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_FILL_N_H
+#define _LIBCPP___ALGORITHM_RANGES_FILL_N_H
+
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __fill_n {
+struct __fn {
+  template <class _Type, output_iterator<const _Type&> _Iter>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, iter_difference_t<_Iter> __n, const _Type& __value) const {
+    for (; __n != 0; --__n) {
+      *__first = __value;
+      ++__first;
+    }
+    return __first;
+  }
+};
+} // namespace __fill_n
+
+inline namespace __cpo {
+inline constexpr auto fill_n = __fill_n::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_FILL_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_find.h b/libcxx/include/__cxx03/__algorithm/ranges_find.h
new file mode 100644
index 000000000000..6b0d5efe37ab
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_find.h
@@ -0,0 +1,80 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_FIND_H
+#define _LIBCPP___ALGORITHM_RANGES_FIND_H
+
+#include <__algorithm/find.h>
+#include <__algorithm/ranges_find_if.h>
+#include <__algorithm/unwrap_range.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __find {
+struct __fn {
+  template <class _Iter, class _Sent, class _Tp, class _Proj>
+  _LIBCPP_HIDE_FROM_ABI static constexpr _Iter
+  __find_unwrap(_Iter __first, _Sent __last, const _Tp& __value, _Proj& __proj) {
+    if constexpr (forward_iterator<_Iter>) {
+      auto [__first_un, __last_un] = std::__unwrap_range(__first, std::move(__last));
+      return std::__rewrap_range<_Sent>(
+          std::move(__first), std::__find(std::move(__first_un), std::move(__last_un), __value, __proj));
+    } else {
+      return std::__find(std::move(__first), std::move(__last), __value, __proj);
+    }
+  }
+
+  template <input_iterator _Ip, sentinel_for<_Ip> _Sp, class _Tp, class _Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<_Ip, _Proj>, const _Tp*>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Ip
+  operator()(_Ip __first, _Sp __last, const _Tp& __value, _Proj __proj = {}) const {
+    return __find_unwrap(std::move(__first), std::move(__last), __value, __proj);
+  }
+
+  template <input_range _Rp, class _Tp, class _Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Rp>, _Proj>, const _Tp*>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Rp>
+  operator()(_Rp&& __r, const _Tp& __value, _Proj __proj = {}) const {
+    return __find_unwrap(ranges::begin(__r), ranges::end(__r), __value, __proj);
+  }
+};
+} // namespace __find
+
+inline namespace __cpo {
+inline constexpr auto find = __find::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_FIND_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_find_end.h b/libcxx/include/__cxx03/__algorithm/ranges_find_end.h
new file mode 100644
index 000000000000..e49e66dd4ac0
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_find_end.h
@@ -0,0 +1,103 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_FIND_END_H
+#define _LIBCPP___ALGORITHM_RANGES_FIND_END_H
+
+#include <__algorithm/find_end.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/ranges_iterator_concept.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/indirectly_comparable.h>
+#include <__iterator/iterator_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/subrange.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __find_end {
+struct __fn {
+  template <forward_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            forward_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_comparable<_Iter1, _Iter2, _Pred, _Proj1, _Proj2>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter1> operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred __pred   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    auto __ret = std::__find_end_impl<_RangeAlgPolicy>(
+        __first1,
+        __last1,
+        __first2,
+        __last2,
+        __pred,
+        __proj1,
+        __proj2,
+        __iterator_concept<_Iter1>(),
+        __iterator_concept<_Iter2>());
+    return {__ret.first, __ret.second};
+  }
+
+  template <forward_range _Range1,
+            forward_range _Range2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_comparable<iterator_t<_Range1>, iterator_t<_Range2>, _Pred, _Proj1, _Proj2>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range1> operator()(
+      _Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+    auto __ret = std::__find_end_impl<_RangeAlgPolicy>(
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        __pred,
+        __proj1,
+        __proj2,
+        __iterator_concept<iterator_t<_Range1>>(),
+        __iterator_concept<iterator_t<_Range2>>());
+    return {__ret.first, __ret.second};
+  }
+};
+} // namespace __find_end
+
+inline namespace __cpo {
+inline constexpr auto find_end = __find_end::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_FIND_END_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_find_first_of.h b/libcxx/include/__cxx03/__algorithm/ranges_find_first_of.h
new file mode 100644
index 000000000000..d92d9686bc44
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_find_first_of.h
@@ -0,0 +1,106 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_FIND_FIRST_OF_H
+#define _LIBCPP___ALGORITHM_RANGES_FIND_FIRST_OF_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/indirectly_comparable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __find_first_of {
+struct __fn {
+  template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class _Pred, class _Proj1, class _Proj2>
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter1 __find_first_of_impl(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred& __pred,
+      _Proj1& __proj1,
+      _Proj2& __proj2) {
+    for (; __first1 != __last1; ++__first1) {
+      for (auto __j = __first2; __j != __last2; ++__j) {
+        if (std::invoke(__pred, std::invoke(__proj1, *__first1), std::invoke(__proj2, *__j)))
+          return __first1;
+      }
+    }
+    return __first1;
+  }
+
+  template <input_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            forward_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_comparable<_Iter1, _Iter2, _Pred, _Proj1, _Proj2>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Iter1 operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred __pred   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    return __find_first_of_impl(
+        std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __pred, __proj1, __proj2);
+  }
+
+  template <input_range _Range1,
+            forward_range _Range2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_comparable<iterator_t<_Range1>, iterator_t<_Range2>, _Pred, _Proj1, _Proj2>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range1> operator()(
+      _Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+    return __find_first_of_impl(
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        __pred,
+        __proj1,
+        __proj2);
+  }
+};
+} // namespace __find_first_of
+
+inline namespace __cpo {
+inline constexpr auto find_first_of = __find_first_of::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_FIND_FIRST_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_find_if.h b/libcxx/include/__cxx03/__algorithm/ranges_find_if.h
new file mode 100644
index 000000000000..888f9ec3cb2d
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_find_if.h
@@ -0,0 +1,75 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_FIND_IF_H
+#define _LIBCPP___ALGORITHM_RANGES_FIND_IF_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _Ip, class _Sp, class _Pred, class _Proj>
+_LIBCPP_HIDE_FROM_ABI constexpr _Ip __find_if_impl(_Ip __first, _Sp __last, _Pred& __pred, _Proj& __proj) {
+  for (; __first != __last; ++__first) {
+    if (std::invoke(__pred, std::invoke(__proj, *__first)))
+      break;
+  }
+  return __first;
+}
+
+namespace __find_if {
+struct __fn {
+  template <input_iterator _Ip,
+            sentinel_for<_Ip> _Sp,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Ip, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Ip
+  operator()(_Ip __first, _Sp __last, _Pred __pred, _Proj __proj = {}) const {
+    return ranges::__find_if_impl(std::move(__first), std::move(__last), __pred, __proj);
+  }
+
+  template <input_range _Rp, class _Proj = identity, indirect_unary_predicate<projected<iterator_t<_Rp>, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Rp>
+  operator()(_Rp&& __r, _Pred __pred, _Proj __proj = {}) const {
+    return ranges::__find_if_impl(ranges::begin(__r), ranges::end(__r), __pred, __proj);
+  }
+};
+} // namespace __find_if
+
+inline namespace __cpo {
+inline constexpr auto find_if = __find_if::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_FIND_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_find_if_not.h b/libcxx/include/__cxx03/__algorithm/ranges_find_if_not.h
new file mode 100644
index 000000000000..ec19545b5a1b
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_find_if_not.h
@@ -0,0 +1,69 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_FIND_IF_NOT_H
+#define _LIBCPP___ALGORITHM_RANGES_FIND_IF_NOT_H
+
+#include <__algorithm/ranges_find_if.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __find_if_not {
+struct __fn {
+  template <input_iterator _Ip,
+            sentinel_for<_Ip> _Sp,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Ip, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Ip
+  operator()(_Ip __first, _Sp __last, _Pred __pred, _Proj __proj = {}) const {
+    auto __pred2 = [&](auto&& __e) -> bool { return !std::invoke(__pred, std::forward<decltype(__e)>(__e)); };
+    return ranges::__find_if_impl(std::move(__first), std::move(__last), __pred2, __proj);
+  }
+
+  template <input_range _Rp, class _Proj = identity, indirect_unary_predicate<projected<iterator_t<_Rp>, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Rp>
+  operator()(_Rp&& __r, _Pred __pred, _Proj __proj = {}) const {
+    auto __pred2 = [&](auto&& __e) -> bool { return !std::invoke(__pred, std::forward<decltype(__e)>(__e)); };
+    return ranges::__find_if_impl(ranges::begin(__r), ranges::end(__r), __pred2, __proj);
+  }
+};
+} // namespace __find_if_not
+
+inline namespace __cpo {
+inline constexpr auto find_if_not = __find_if_not::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_FIND_IF_NOT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_find_last.h b/libcxx/include/__cxx03/__algorithm/ranges_find_last.h
new file mode 100644
index 000000000000..95f7e77b8ccb
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_find_last.h
@@ -0,0 +1,175 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_FIND_LAST_H
+#define _LIBCPP___ALGORITHM_RANGES_FIND_LAST_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/indirectly_comparable.h>
+#include <__iterator/next.h>
+#include <__iterator/prev.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/subrange.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _Iter, class _Sent, class _Pred, class _Proj>
+_LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
+__find_last_impl(_Iter __first, _Sent __last, _Pred __pred, _Proj& __proj) {
+  if (__first == __last) {
+    return subrange<_Iter>(__first, __first);
+  }
+
+  if constexpr (bidirectional_iterator<_Iter>) {
+    auto __last_it = ranges::next(__first, __last);
+    for (auto __it = ranges::prev(__last_it); __it != __first; --__it) {
+      if (__pred(std::invoke(__proj, *__it))) {
+        return subrange<_Iter>(std::move(__it), std::move(__last_it));
+      }
+    }
+    if (__pred(std::invoke(__proj, *__first))) {
+      return subrange<_Iter>(std::move(__first), std::move(__last_it));
+    }
+    return subrange<_Iter>(__last_it, __last_it);
+  } else {
+    bool __found = false;
+    _Iter __found_it;
+    for (; __first != __last; ++__first) {
+      if (__pred(std::invoke(__proj, *__first))) {
+        __found    = true;
+        __found_it = __first;
+      }
+    }
+
+    if (__found) {
+      return subrange<_Iter>(std::move(__found_it), std::move(__first));
+    } else {
+      return subrange<_Iter>(__first, __first);
+    }
+  }
+}
+
+namespace __find_last {
+struct __fn {
+  template <class _Type>
+  struct __op {
+    const _Type& __value;
+    template <class _Elem>
+    _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator()(_Elem&& __elem) const {
+      return std::forward<_Elem>(__elem) == __value;
+    }
+  };
+
+  template <forward_iterator _Iter, sentinel_for<_Iter> _Sent, class _Type, class _Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<_Iter, _Proj>, const _Type*>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr static subrange<_Iter>
+  operator()(_Iter __first, _Sent __last, const _Type& __value, _Proj __proj = {}) {
+    return ranges::__find_last_impl(std::move(__first), std::move(__last), __op<_Type>{__value}, __proj);
+  }
+
+  template <forward_range _Range, class _Type, class _Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _Type*>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr static borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, const _Type& __value, _Proj __proj = {}) {
+    return ranges::__find_last_impl(ranges::begin(__range), ranges::end(__range), __op<_Type>{__value}, __proj);
+  }
+};
+} // namespace __find_last
+
+namespace __find_last_if {
+struct __fn {
+  template <class _Pred>
+  struct __op {
+    _Pred& __pred;
+    template <class _Elem>
+    _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator()(_Elem&& __elem) const {
+      return std::invoke(__pred, std::forward<_Elem>(__elem));
+    }
+  };
+
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr static subrange<_Iter>
+  operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) {
+    return ranges::__find_last_impl(std::move(__first), std::move(__last), __op<_Pred>{__pred}, __proj);
+  }
+
+  template <forward_range _Range,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr static borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) {
+    return ranges::__find_last_impl(ranges::begin(__range), ranges::end(__range), __op<_Pred>{__pred}, __proj);
+  }
+};
+} // namespace __find_last_if
+
+namespace __find_last_if_not {
+struct __fn {
+  template <class _Pred>
+  struct __op {
+    _Pred& __pred;
+    template <class _Elem>
+    _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator()(_Elem&& __elem) const {
+      return !std::invoke(__pred, std::forward<_Elem>(__elem));
+    }
+  };
+
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr static subrange<_Iter>
+  operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) {
+    return ranges::__find_last_impl(std::move(__first), std::move(__last), __op<_Pred>{__pred}, __proj);
+  }
+
+  template <forward_range _Range,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr static borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) {
+    return ranges::__find_last_impl(ranges::begin(__range), ranges::end(__range), __op<_Pred>{__pred}, __proj);
+  }
+};
+} // namespace __find_last_if_not
+
+inline namespace __cpo {
+inline constexpr auto find_last        = __find_last::__fn{};
+inline constexpr auto find_last_if     = __find_last_if::__fn{};
+inline constexpr auto find_last_if_not = __find_last_if_not::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_FIND_LAST_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_for_each.h b/libcxx/include/__cxx03/__algorithm/ranges_for_each.h
new file mode 100644
index 000000000000..225dc774c876
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_for_each.h
@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_FOR_EACH_H
+#define _LIBCPP___ALGORITHM_RANGES_FOR_EACH_H
+
+#include <__algorithm/in_fun_result.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _Iter, class _Func>
+using for_each_result = in_fun_result<_Iter, _Func>;
+
+namespace __for_each {
+struct __fn {
+private:
+  template <class _Iter, class _Sent, class _Proj, class _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr static for_each_result<_Iter, _Func>
+  __for_each_impl(_Iter __first, _Sent __last, _Func& __func, _Proj& __proj) {
+    for (; __first != __last; ++__first)
+      std::invoke(__func, std::invoke(__proj, *__first));
+    return {std::move(__first), std::move(__func)};
+  }
+
+public:
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
+            indirectly_unary_invocable<projected<_Iter, _Proj>> _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr for_each_result<_Iter, _Func>
+  operator()(_Iter __first, _Sent __last, _Func __func, _Proj __proj = {}) const {
+    return __for_each_impl(std::move(__first), std::move(__last), __func, __proj);
+  }
+
+  template <input_range _Range,
+            class _Proj = identity,
+            indirectly_unary_invocable<projected<iterator_t<_Range>, _Proj>> _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr for_each_result<borrowed_iterator_t<_Range>, _Func>
+  operator()(_Range&& __range, _Func __func, _Proj __proj = {}) const {
+    return __for_each_impl(ranges::begin(__range), ranges::end(__range), __func, __proj);
+  }
+};
+} // namespace __for_each
+
+inline namespace __cpo {
+inline constexpr auto for_each = __for_each::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_FOR_EACH_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_for_each_n.h b/libcxx/include/__cxx03/__algorithm/ranges_for_each_n.h
new file mode 100644
index 000000000000..d1fdca34cc5a
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_for_each_n.h
@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_FOR_EACH_N_H
+#define _LIBCPP___ALGORITHM_RANGES_FOR_EACH_N_H
+
+#include <__algorithm/in_fun_result.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/projected.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _Iter, class _Func>
+using for_each_n_result = in_fun_result<_Iter, _Func>;
+
+namespace __for_each_n {
+struct __fn {
+  template <input_iterator _Iter, class _Proj = identity, indirectly_unary_invocable<projected<_Iter, _Proj>> _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr for_each_n_result<_Iter, _Func>
+  operator()(_Iter __first, iter_difference_t<_Iter> __count, _Func __func, _Proj __proj = {}) const {
+    while (__count-- > 0) {
+      std::invoke(__func, std::invoke(__proj, *__first));
+      ++__first;
+    }
+    return {std::move(__first), std::move(__func)};
+  }
+};
+} // namespace __for_each_n
+
+inline namespace __cpo {
+inline constexpr auto for_each_n = __for_each_n::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_FOR_EACH_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_generate.h b/libcxx/include/__cxx03/__algorithm/ranges_generate.h
new file mode 100644
index 000000000000..e6467198e6ba
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_generate.h
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_GENERATE_H
+#define _LIBCPP___ALGORITHM_RANGES_GENERATE_H
+
+#include <__concepts/constructible.h>
+#include <__concepts/invocable.h>
+#include <__config>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __generate {
+
+struct __fn {
+  template <class _OutIter, class _Sent, class _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr static _OutIter __generate_fn_impl(_OutIter __first, _Sent __last, _Func& __gen) {
+    for (; __first != __last; ++__first) {
+      *__first = __gen();
+    }
+
+    return __first;
+  }
+
+  template <input_or_output_iterator _OutIter, sentinel_for<_OutIter> _Sent, copy_constructible _Func>
+    requires invocable<_Func&> && indirectly_writable<_OutIter, invoke_result_t<_Func&>>
+  _LIBCPP_HIDE_FROM_ABI constexpr _OutIter operator()(_OutIter __first, _Sent __last, _Func __gen) const {
+    return __generate_fn_impl(std::move(__first), std::move(__last), __gen);
+  }
+
+  template <class _Range, copy_constructible _Func>
+    requires invocable<_Func&> && output_range<_Range, invoke_result_t<_Func&>>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range> operator()(_Range&& __range, _Func __gen) const {
+    return __generate_fn_impl(ranges::begin(__range), ranges::end(__range), __gen);
+  }
+};
+
+} // namespace __generate
+
+inline namespace __cpo {
+inline constexpr auto generate = __generate::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_GENERATE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_generate_n.h b/libcxx/include/__cxx03/__algorithm/ranges_generate_n.h
new file mode 100644
index 000000000000..cd5fd7483ab2
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_generate_n.h
@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_GENERATE_N_H
+#define _LIBCPP___ALGORITHM_RANGES_GENERATE_N_H
+
+#include <__concepts/constructible.h>
+#include <__concepts/invocable.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __generate_n {
+
+struct __fn {
+  template <input_or_output_iterator _OutIter, copy_constructible _Func>
+    requires invocable<_Func&> && indirectly_writable<_OutIter, invoke_result_t<_Func&>>
+  _LIBCPP_HIDE_FROM_ABI constexpr _OutIter
+  operator()(_OutIter __first, iter_difference_t<_OutIter> __n, _Func __gen) const {
+    for (; __n > 0; --__n) {
+      *__first = __gen();
+      ++__first;
+    }
+
+    return __first;
+  }
+};
+
+} // namespace __generate_n
+
+inline namespace __cpo {
+inline constexpr auto generate_n = __generate_n::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_GENERATE_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_includes.h b/libcxx/include/__cxx03/__algorithm/ranges_includes.h
new file mode 100644
index 000000000000..c4c3b8ed088d
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_includes.h
@@ -0,0 +1,98 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_INCLUDES_H
+#define _LIBCPP___ALGORITHM_RANGES_INCLUDES_H
+
+#include <__algorithm/includes.h>
+#include <__algorithm/make_projected.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __includes {
+
+struct __fn {
+  template <input_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            input_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Proj1                                                                           = identity,
+            class _Proj2                                                                           = identity,
+            indirect_strict_weak_order<projected<_Iter1, _Proj1>, projected<_Iter2, _Proj2>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Comp __comp   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    return std::__includes(
+        std::move(__first1),
+        std::move(__last1),
+        std::move(__first2),
+        std::move(__last2),
+        std::move(__comp),
+        std::move(__proj1),
+        std::move(__proj2));
+  }
+
+  template <input_range _Range1,
+            input_range _Range2,
+            class _Proj1 = identity,
+            class _Proj2 = identity,
+            indirect_strict_weak_order<projected<iterator_t<_Range1>, _Proj1>, projected<iterator_t<_Range2>, _Proj2>>
+                _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Range1&& __range1, _Range2&& __range2, _Comp __comp = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+    return std::__includes(
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        std::move(__comp),
+        std::move(__proj1),
+        std::move(__proj2));
+  }
+};
+
+} // namespace __includes
+
+inline namespace __cpo {
+inline constexpr auto includes = __includes::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_INCLUDES_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_inplace_merge.h b/libcxx/include/__cxx03/__algorithm/ranges_inplace_merge.h
new file mode 100644
index 000000000000..d94c0ad46567
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_inplace_merge.h
@@ -0,0 +1,84 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_INPLACE_MERGE_H
+#define _LIBCPP___ALGORITHM_RANGES_INPLACE_MERGE_H
+
+#include <__algorithm/inplace_merge.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/projected.h>
+#include <__iterator/sortable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __inplace_merge {
+
+struct __fn {
+  template <class _Iter, class _Sent, class _Comp, class _Proj>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto
+  __inplace_merge_impl(_Iter __first, _Iter __middle, _Sent __last, _Comp&& __comp, _Proj&& __proj) {
+    auto __last_iter = ranges::next(__middle, __last);
+    std::__inplace_merge<_RangeAlgPolicy>(
+        std::move(__first), std::move(__middle), __last_iter, std::__make_projected(__comp, __proj));
+    return __last_iter;
+  }
+
+  template <bidirectional_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<_Iter, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI _Iter
+  operator()(_Iter __first, _Iter __middle, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __inplace_merge_impl(
+        std::move(__first), std::move(__middle), std::move(__last), std::move(__comp), std::move(__proj));
+  }
+
+  template <bidirectional_range _Range, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<iterator_t<_Range>, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI borrowed_iterator_t<_Range>
+  operator()(_Range&& __range, iterator_t<_Range> __middle, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __inplace_merge_impl(
+        ranges::begin(__range), std::move(__middle), ranges::end(__range), std::move(__comp), std::move(__proj));
+  }
+};
+
+} // namespace __inplace_merge
+
+inline namespace __cpo {
+inline constexpr auto inplace_merge = __inplace_merge::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_INPLACE_MERGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_is_heap.h b/libcxx/include/__cxx03/__algorithm/ranges_is_heap.h
new file mode 100644
index 000000000000..3d9e18ce1d90
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_is_heap.h
@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_IS_HEAP_H
+#define _LIBCPP___ALGORITHM_RANGES_IS_HEAP_H
+
+#include <__algorithm/is_heap_until.h>
+#include <__algorithm/make_projected.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __is_heap {
+
+struct __fn {
+  template <class _Iter, class _Sent, class _Proj, class _Comp>
+  _LIBCPP_HIDE_FROM_ABI constexpr static bool
+  __is_heap_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+    auto __last_iter        = ranges::next(__first, __last);
+    auto&& __projected_comp = std::__make_projected(__comp, __proj);
+
+    auto __result = std::__is_heap_until(std::move(__first), std::move(__last_iter), __projected_comp);
+    return __result == __last;
+  }
+
+  template <random_access_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj                                               = identity,
+            indirect_strict_weak_order<projected<_Iter, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __is_heap_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
+  }
+
+  template <random_access_range _Range,
+            class _Proj                                                            = identity,
+            indirect_strict_weak_order<projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __is_heap_fn_impl(ranges::begin(__range), ranges::end(__range), __comp, __proj);
+  }
+};
+
+} // namespace __is_heap
+
+inline namespace __cpo {
+inline constexpr auto is_heap = __is_heap::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_IS_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_is_heap_until.h b/libcxx/include/__cxx03/__algorithm/ranges_is_heap_until.h
new file mode 100644
index 000000000000..7a2e1fc7705b
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_is_heap_until.h
@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_IS_HEAP_UNTIL_H
+#define _LIBCPP___ALGORITHM_RANGES_IS_HEAP_UNTIL_H
+
+#include <__algorithm/is_heap_until.h>
+#include <__algorithm/make_projected.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __is_heap_until {
+
+struct __fn {
+  template <class _Iter, class _Sent, class _Proj, class _Comp>
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __is_heap_until_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+    auto __last_iter        = ranges::next(__first, __last);
+    auto&& __projected_comp = std::__make_projected(__comp, __proj);
+
+    return std::__is_heap_until(std::move(__first), std::move(__last_iter), __projected_comp);
+  }
+
+  template <random_access_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj                                               = identity,
+            indirect_strict_weak_order<projected<_Iter, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __is_heap_until_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
+  }
+
+  template <random_access_range _Range,
+            class _Proj                                                            = identity,
+            indirect_strict_weak_order<projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __is_heap_until_fn_impl(ranges::begin(__range), ranges::end(__range), __comp, __proj);
+  }
+};
+
+} // namespace __is_heap_until
+
+inline namespace __cpo {
+inline constexpr auto is_heap_until = __is_heap_until::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_IS_HEAP_UNTIL_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_is_partitioned.h b/libcxx/include/__cxx03/__algorithm/ranges_is_partitioned.h
new file mode 100644
index 000000000000..5be6fba46fd9
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_is_partitioned.h
@@ -0,0 +1,86 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_IS_PARTITIONED_H
+#define _LIBCPP___ALGORITHM_RANGES_IS_PARTITIONED_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/indirectly_comparable.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __is_partitioned {
+struct __fn {
+  template <class _Iter, class _Sent, class _Proj, class _Pred>
+  _LIBCPP_HIDE_FROM_ABI constexpr static bool
+  __is_partitioned_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
+    for (; __first != __last; ++__first) {
+      if (!std::invoke(__pred, std::invoke(__proj, *__first)))
+        break;
+    }
+
+    if (__first == __last)
+      return true;
+    ++__first;
+
+    for (; __first != __last; ++__first) {
+      if (std::invoke(__pred, std::invoke(__proj, *__first)))
+        return false;
+    }
+
+    return true;
+  }
+
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
+    return __is_partitioned_impl(std::move(__first), std::move(__last), __pred, __proj);
+  }
+
+  template <input_range _Range,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+    return __is_partitioned_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
+  }
+};
+} // namespace __is_partitioned
+
+inline namespace __cpo {
+inline constexpr auto is_partitioned = __is_partitioned::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_IS_PARTITIONED_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_is_permutation.h b/libcxx/include/__cxx03/__algorithm/ranges_is_permutation.h
new file mode 100644
index 000000000000..1f8d67007a57
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_is_permutation.h
@@ -0,0 +1,107 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_IS_PERMUTATION_H
+#define _LIBCPP___ALGORITHM_RANGES_IS_PERMUTATION_H
+
+#include <__algorithm/is_permutation.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/distance.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __is_permutation {
+struct __fn {
+  template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class _Proj1, class _Proj2, class _Pred>
+  _LIBCPP_HIDE_FROM_ABI constexpr static bool __is_permutation_func_impl(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred& __pred,
+      _Proj1& __proj1,
+      _Proj2& __proj2) {
+    return std::__is_permutation<_RangeAlgPolicy>(
+        std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __pred, __proj1, __proj2);
+  }
+
+  template <
+      forward_iterator _Iter1,
+      sentinel_for<_Iter1> _Sent1,
+      forward_iterator _Iter2,
+      sentinel_for<_Iter2> _Sent2,
+      class _Proj1                                                                              = identity,
+      class _Proj2                                                                              = identity,
+      indirect_equivalence_relation<projected<_Iter1, _Proj1>, projected<_Iter2, _Proj2>> _Pred = ranges::equal_to>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred __pred   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    return __is_permutation_func_impl(
+        std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __pred, __proj1, __proj2);
+  }
+
+  template <forward_range _Range1,
+            forward_range _Range2,
+            class _Proj1                                                                = identity,
+            class _Proj2                                                                = identity,
+            indirect_equivalence_relation<projected<iterator_t<_Range1>, _Proj1>,
+                                          projected<iterator_t<_Range2>, _Proj2>> _Pred = ranges::equal_to>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+    if constexpr (sized_range<_Range1> && sized_range<_Range2>) {
+      if (ranges::distance(__range1) != ranges::distance(__range2))
+        return false;
+    }
+
+    return __is_permutation_func_impl(
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        __pred,
+        __proj1,
+        __proj2);
+  }
+};
+} // namespace __is_permutation
+
+inline namespace __cpo {
+inline constexpr auto is_permutation = __is_permutation::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_IS_PERMUTATION_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_is_sorted.h b/libcxx/include/__cxx03/__algorithm/ranges_is_sorted.h
new file mode 100644
index 000000000000..5b88d422b4b0
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_is_sorted.h
@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP__ALGORITHM_RANGES_IS_SORTED_H
+#define _LIBCPP__ALGORITHM_RANGES_IS_SORTED_H
+
+#include <__algorithm/ranges_is_sorted_until.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __is_sorted {
+struct __fn {
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj                                               = identity,
+            indirect_strict_weak_order<projected<_Iter, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    return ranges::__is_sorted_until_impl(std::move(__first), __last, __comp, __proj) == __last;
+  }
+
+  template <forward_range _Range,
+            class _Proj                                                            = identity,
+            indirect_strict_weak_order<projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __last = ranges::end(__range);
+    return ranges::__is_sorted_until_impl(ranges::begin(__range), __last, __comp, __proj) == __last;
+  }
+};
+} // namespace __is_sorted
+
+inline namespace __cpo {
+inline constexpr auto is_sorted = __is_sorted::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP__ALGORITHM_RANGES_IS_SORTED_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_is_sorted_until.h b/libcxx/include/__cxx03/__algorithm/ranges_is_sorted_until.h
new file mode 100644
index 000000000000..54de530c8b2f
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_is_sorted_until.h
@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP__ALGORITHM_RANGES_IS_SORTED_UNTIL_H
+#define _LIBCPP__ALGORITHM_RANGES_IS_SORTED_UNTIL_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _Iter, class _Sent, class _Proj, class _Comp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Iter
+__is_sorted_until_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+  if (__first == __last)
+    return __first;
+  auto __i = __first;
+  while (++__i != __last) {
+    if (std::invoke(__comp, std::invoke(__proj, *__i), std::invoke(__proj, *__first)))
+      return __i;
+    __first = __i;
+  }
+  return __i;
+}
+
+namespace __is_sorted_until {
+struct __fn {
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj                                               = identity,
+            indirect_strict_weak_order<projected<_Iter, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    return ranges::__is_sorted_until_impl(std::move(__first), std::move(__last), __comp, __proj);
+  }
+
+  template <forward_range _Range,
+            class _Proj                                                            = identity,
+            indirect_strict_weak_order<projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
+    return ranges::__is_sorted_until_impl(ranges::begin(__range), ranges::end(__range), __comp, __proj);
+  }
+};
+} // namespace __is_sorted_until
+
+inline namespace __cpo {
+inline constexpr auto is_sorted_until = __is_sorted_until::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP__ALGORITHM_RANGES_IS_SORTED_UNTIL_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_iterator_concept.h b/libcxx/include/__cxx03/__algorithm/ranges_iterator_concept.h
new file mode 100644
index 000000000000..2af891d3af00
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_iterator_concept.h
@@ -0,0 +1,56 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_ITERATOR_CONCEPT_H
+#define _LIBCPP___ALGORITHM_RANGES_ITERATOR_CONCEPT_H
+
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/remove_cvref.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _IterMaybeQualified>
+consteval auto __get_iterator_concept() {
+  using _Iter = __remove_cvref_t<_IterMaybeQualified>;
+
+  if constexpr (contiguous_iterator<_Iter>)
+    return contiguous_iterator_tag();
+  else if constexpr (random_access_iterator<_Iter>)
+    return random_access_iterator_tag();
+  else if constexpr (bidirectional_iterator<_Iter>)
+    return bidirectional_iterator_tag();
+  else if constexpr (forward_iterator<_Iter>)
+    return forward_iterator_tag();
+  else if constexpr (input_iterator<_Iter>)
+    return input_iterator_tag();
+}
+
+template <class _Iter>
+using __iterator_concept = decltype(__get_iterator_concept<_Iter>());
+
+} // namespace ranges
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_ITERATOR_CONCEPT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_lexicographical_compare.h b/libcxx/include/__cxx03/__algorithm/ranges_lexicographical_compare.h
new file mode 100644
index 000000000000..6d82017e302a
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_lexicographical_compare.h
@@ -0,0 +1,106 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_LEXICOGRAPHICAL_COMPARE_H
+#define _LIBCPP___ALGORITHM_RANGES_LEXICOGRAPHICAL_COMPARE_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __lexicographical_compare {
+struct __fn {
+  template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class _Proj1, class _Proj2, class _Comp>
+  _LIBCPP_HIDE_FROM_ABI constexpr static bool __lexicographical_compare_impl(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Comp& __comp,
+      _Proj1& __proj1,
+      _Proj2& __proj2) {
+    while (__first2 != __last2) {
+      if (__first1 == __last1 || std::invoke(__comp, std::invoke(__proj1, *__first1), std::invoke(__proj2, *__first2)))
+        return true;
+      if (std::invoke(__comp, std::invoke(__proj2, *__first2), std::invoke(__proj1, *__first1)))
+        return false;
+      ++__first1;
+      ++__first2;
+    }
+    return false;
+  }
+
+  template <input_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            input_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Proj1                                                                           = identity,
+            class _Proj2                                                                           = identity,
+            indirect_strict_weak_order<projected<_Iter1, _Proj1>, projected<_Iter2, _Proj2>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Comp __comp   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    return __lexicographical_compare_impl(
+        std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __comp, __proj1, __proj2);
+  }
+
+  template <input_range _Range1,
+            input_range _Range2,
+            class _Proj1 = identity,
+            class _Proj2 = identity,
+            indirect_strict_weak_order<projected<iterator_t<_Range1>, _Proj1>, projected<iterator_t<_Range2>, _Proj2>>
+                _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Range1&& __range1, _Range2&& __range2, _Comp __comp = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+    return __lexicographical_compare_impl(
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        __comp,
+        __proj1,
+        __proj2);
+  }
+};
+} // namespace __lexicographical_compare
+
+inline namespace __cpo {
+inline constexpr auto lexicographical_compare = __lexicographical_compare::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_LEXICOGRAPHICAL_COMPARE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_lower_bound.h b/libcxx/include/__cxx03/__algorithm/ranges_lower_bound.h
new file mode 100644
index 000000000000..0651147e0424
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_lower_bound.h
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_LOWER_BOUND_H
+#define _LIBCPP___ALGORITHM_RANGES_LOWER_BOUND_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/lower_bound.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/advance.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+namespace __lower_bound {
+struct __fn {
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Type,
+            class _Proj                                                             = identity,
+            indirect_strict_weak_order<const _Type*, projected<_Iter, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
+    return std::__lower_bound<_RangeAlgPolicy>(__first, __last, __value, __comp, __proj);
+  }
+
+  template <forward_range _Range,
+            class _Type,
+            class _Proj                                                                          = identity,
+            indirect_strict_weak_order<const _Type*, projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
+    return std::__lower_bound<_RangeAlgPolicy>(ranges::begin(__r), ranges::end(__r), __value, __comp, __proj);
+  }
+};
+} // namespace __lower_bound
+
+inline namespace __cpo {
+inline constexpr auto lower_bound = __lower_bound::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_LOWER_BOUND_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_make_heap.h b/libcxx/include/__cxx03/__algorithm/ranges_make_heap.h
new file mode 100644
index 000000000000..fe9c024fbf8a
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_make_heap.h
@@ -0,0 +1,85 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_MAKE_HEAP_H
+#define _LIBCPP___ALGORITHM_RANGES_MAKE_HEAP_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_heap.h>
+#include <__algorithm/make_projected.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/projected.h>
+#include <__iterator/sortable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __make_heap {
+
+struct __fn {
+  template <class _Iter, class _Sent, class _Comp, class _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __make_heap_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+    auto __last_iter = ranges::next(__first, __last);
+
+    auto&& __projected_comp = std::__make_projected(__comp, __proj);
+    std::__make_heap<_RangeAlgPolicy>(std::move(__first), __last_iter, __projected_comp);
+
+    return __last_iter;
+  }
+
+  template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<_Iter, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __make_heap_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
+  }
+
+  template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<iterator_t<_Range>, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __make_heap_fn_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
+  }
+};
+
+} // namespace __make_heap
+
+inline namespace __cpo {
+inline constexpr auto make_heap = __make_heap::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_MAKE_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_max.h b/libcxx/include/__cxx03/__algorithm/ranges_max.h
new file mode 100644
index 000000000000..d0ee6f314b0c
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_max.h
@@ -0,0 +1,103 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_MAX_H
+#define _LIBCPP___ALGORITHM_RANGES_MAX_H
+
+#include <__algorithm/ranges_min_element.h>
+#include <__assert>
+#include <__concepts/copyable.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__type_traits/is_trivially_copyable.h>
+#include <__utility/move.h>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __max {
+struct __fn {
+  template <class _Tp,
+            class _Proj                                                    = identity,
+            indirect_strict_weak_order<projected<const _Tp*, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Tp&
+  operator()(_LIBCPP_LIFETIMEBOUND const _Tp& __a,
+             _LIBCPP_LIFETIMEBOUND const _Tp& __b,
+             _Comp __comp = {},
+             _Proj __proj = {}) const {
+    return std::invoke(__comp, std::invoke(__proj, __a), std::invoke(__proj, __b)) ? __b : __a;
+  }
+
+  template <copyable _Tp,
+            class _Proj                                                    = identity,
+            indirect_strict_weak_order<projected<const _Tp*, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp
+  operator()(initializer_list<_Tp> __il, _Comp __comp = {}, _Proj __proj = {}) const {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __il.begin() != __il.end(), "initializer_list must contain at least one element");
+
+    auto __comp_lhs_rhs_swapped = [&](auto&& __lhs, auto&& __rhs) -> bool { return std::invoke(__comp, __rhs, __lhs); };
+    return *ranges::__min_element_impl(__il.begin(), __il.end(), __comp_lhs_rhs_swapped, __proj);
+  }
+
+  template <input_range _Rp,
+            class _Proj                                                         = identity,
+            indirect_strict_weak_order<projected<iterator_t<_Rp>, _Proj>> _Comp = ranges::less>
+    requires indirectly_copyable_storable<iterator_t<_Rp>, range_value_t<_Rp>*>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr range_value_t<_Rp>
+  operator()(_Rp&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __first = ranges::begin(__r);
+    auto __last  = ranges::end(__r);
+
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__first != __last, "range must contain at least one element");
+
+    if constexpr (forward_range<_Rp> && !__is_cheap_to_copy<range_value_t<_Rp>>) {
+      auto __comp_lhs_rhs_swapped = [&](auto&& __lhs, auto&& __rhs) -> bool {
+        return std::invoke(__comp, __rhs, __lhs);
+      };
+      return *ranges::__min_element_impl(std::move(__first), std::move(__last), __comp_lhs_rhs_swapped, __proj);
+    } else {
+      range_value_t<_Rp> __result = *__first;
+      while (++__first != __last) {
+        if (std::invoke(__comp, std::invoke(__proj, __result), std::invoke(__proj, *__first)))
+          __result = *__first;
+      }
+      return __result;
+    }
+  }
+};
+} // namespace __max
+
+inline namespace __cpo {
+inline constexpr auto max = __max::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___ALGORITHM_RANGES_MAX_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_max_element.h b/libcxx/include/__cxx03/__algorithm/ranges_max_element.h
new file mode 100644
index 000000000000..c57730927116
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_max_element.h
@@ -0,0 +1,69 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_MAX_ELEMENT_H
+#define _LIBCPP___ALGORITHM_RANGES_MAX_ELEMENT_H
+
+#include <__algorithm/ranges_min_element.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __max_element {
+struct __fn {
+  template <forward_iterator _Ip,
+            sentinel_for<_Ip> _Sp,
+            class _Proj                                             = identity,
+            indirect_strict_weak_order<projected<_Ip, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Ip
+  operator()(_Ip __first, _Sp __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __comp_lhs_rhs_swapped = [&](auto&& __lhs, auto&& __rhs) -> bool { return std::invoke(__comp, __rhs, __lhs); };
+    return ranges::__min_element_impl(__first, __last, __comp_lhs_rhs_swapped, __proj);
+  }
+
+  template <forward_range _Rp,
+            class _Proj                                                         = identity,
+            indirect_strict_weak_order<projected<iterator_t<_Rp>, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Rp>
+  operator()(_Rp&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __comp_lhs_rhs_swapped = [&](auto&& __lhs, auto&& __rhs) -> bool { return std::invoke(__comp, __rhs, __lhs); };
+    return ranges::__min_element_impl(ranges::begin(__r), ranges::end(__r), __comp_lhs_rhs_swapped, __proj);
+  }
+};
+} // namespace __max_element
+
+inline namespace __cpo {
+inline constexpr auto max_element = __max_element::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_MAX_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_merge.h b/libcxx/include/__cxx03/__algorithm/ranges_merge.h
new file mode 100644
index 000000000000..bdf9a62d90bd
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_merge.h
@@ -0,0 +1,138 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_MERGE_H
+#define _LIBCPP___ALGORITHM_RANGES_MERGE_H
+
+#include <__algorithm/in_in_out_result.h>
+#include <__algorithm/ranges_copy.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/mergeable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter1, class _InIter2, class _OutIter>
+using merge_result = in_in_out_result<_InIter1, _InIter2, _OutIter>;
+
+namespace __merge {
+
+template < class _InIter1,
+           class _Sent1,
+           class _InIter2,
+           class _Sent2,
+           class _OutIter,
+           class _Comp,
+           class _Proj1,
+           class _Proj2>
+_LIBCPP_HIDE_FROM_ABI constexpr merge_result<__remove_cvref_t<_InIter1>,
+                                             __remove_cvref_t<_InIter2>,
+                                             __remove_cvref_t<_OutIter>>
+__merge_impl(_InIter1&& __first1,
+             _Sent1&& __last1,
+             _InIter2&& __first2,
+             _Sent2&& __last2,
+             _OutIter&& __result,
+             _Comp&& __comp,
+             _Proj1&& __proj1,
+             _Proj2&& __proj2) {
+  for (; __first1 != __last1 && __first2 != __last2; ++__result) {
+    if (std::invoke(__comp, std::invoke(__proj2, *__first2), std::invoke(__proj1, *__first1))) {
+      *__result = *__first2;
+      ++__first2;
+    } else {
+      *__result = *__first1;
+      ++__first1;
+    }
+  }
+  auto __ret1 = ranges::copy(std::move(__first1), std::move(__last1), std::move(__result));
+  auto __ret2 = ranges::copy(std::move(__first2), std::move(__last2), std::move(__ret1.out));
+  return {std::move(__ret1.in), std::move(__ret2.in), std::move(__ret2.out)};
+}
+
+struct __fn {
+  template <input_iterator _InIter1,
+            sentinel_for<_InIter1> _Sent1,
+            input_iterator _InIter2,
+            sentinel_for<_InIter2> _Sent2,
+            weakly_incrementable _OutIter,
+            class _Comp  = less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires mergeable<_InIter1, _InIter2, _OutIter, _Comp, _Proj1, _Proj2>
+  _LIBCPP_HIDE_FROM_ABI constexpr merge_result<_InIter1, _InIter2, _OutIter> operator()(
+      _InIter1 __first1,
+      _Sent1 __last1,
+      _InIter2 __first2,
+      _Sent2 __last2,
+      _OutIter __result,
+      _Comp __comp   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    return __merge::__merge_impl(__first1, __last1, __first2, __last2, __result, __comp, __proj1, __proj2);
+  }
+
+  template <input_range _Range1,
+            input_range _Range2,
+            weakly_incrementable _OutIter,
+            class _Comp  = less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires mergeable<iterator_t<_Range1>, iterator_t<_Range2>, _OutIter, _Comp, _Proj1, _Proj2>
+  _LIBCPP_HIDE_FROM_ABI constexpr merge_result<borrowed_iterator_t<_Range1>, borrowed_iterator_t<_Range2>, _OutIter>
+  operator()(_Range1&& __range1,
+             _Range2&& __range2,
+             _OutIter __result,
+             _Comp __comp   = {},
+             _Proj1 __proj1 = {},
+             _Proj2 __proj2 = {}) const {
+    return __merge::__merge_impl(
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        __result,
+        __comp,
+        __proj1,
+        __proj2);
+  }
+};
+
+} // namespace __merge
+
+inline namespace __cpo {
+inline constexpr auto merge = __merge::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_MERGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_min.h b/libcxx/include/__cxx03/__algorithm/ranges_min.h
new file mode 100644
index 000000000000..cc569d2a060c
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_min.h
@@ -0,0 +1,95 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_MIN_H
+#define _LIBCPP___ALGORITHM_RANGES_MIN_H
+
+#include <__algorithm/ranges_min_element.h>
+#include <__assert>
+#include <__concepts/copyable.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__type_traits/is_trivially_copyable.h>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __min {
+struct __fn {
+  template <class _Tp,
+            class _Proj                                                    = identity,
+            indirect_strict_weak_order<projected<const _Tp*, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Tp&
+  operator()(_LIBCPP_LIFETIMEBOUND const _Tp& __a,
+             _LIBCPP_LIFETIMEBOUND const _Tp& __b,
+             _Comp __comp = {},
+             _Proj __proj = {}) const {
+    return std::invoke(__comp, std::invoke(__proj, __b), std::invoke(__proj, __a)) ? __b : __a;
+  }
+
+  template <copyable _Tp,
+            class _Proj                                                    = identity,
+            indirect_strict_weak_order<projected<const _Tp*, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp
+  operator()(initializer_list<_Tp> __il, _Comp __comp = {}, _Proj __proj = {}) const {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __il.begin() != __il.end(), "initializer_list must contain at least one element");
+    return *ranges::__min_element_impl(__il.begin(), __il.end(), __comp, __proj);
+  }
+
+  template <input_range _Rp,
+            class _Proj                                                         = identity,
+            indirect_strict_weak_order<projected<iterator_t<_Rp>, _Proj>> _Comp = ranges::less>
+    requires indirectly_copyable_storable<iterator_t<_Rp>, range_value_t<_Rp>*>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr range_value_t<_Rp>
+  operator()(_Rp&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __first = ranges::begin(__r);
+    auto __last  = ranges::end(__r);
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__first != __last, "range must contain at least one element");
+    if constexpr (forward_range<_Rp> && !__is_cheap_to_copy<range_value_t<_Rp>>) {
+      return *ranges::__min_element_impl(__first, __last, __comp, __proj);
+    } else {
+      range_value_t<_Rp> __result = *__first;
+      while (++__first != __last) {
+        if (std::invoke(__comp, std::invoke(__proj, *__first), std::invoke(__proj, __result)))
+          __result = *__first;
+      }
+      return __result;
+    }
+  }
+};
+} // namespace __min
+
+inline namespace __cpo {
+inline constexpr auto min = __min::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___ALGORITHM_RANGES_MIN_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_min_element.h b/libcxx/include/__cxx03/__algorithm/ranges_min_element.h
new file mode 100644
index 000000000000..588ef258e26f
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_min_element.h
@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_MIN_ELEMENT_H
+#define _LIBCPP___ALGORITHM_RANGES_MIN_ELEMENT_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+// TODO(ranges): `ranges::min_element` can now simply delegate to `std::__min_element`.
+template <class _Ip, class _Sp, class _Proj, class _Comp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Ip __min_element_impl(_Ip __first, _Sp __last, _Comp& __comp, _Proj& __proj) {
+  if (__first == __last)
+    return __first;
+
+  _Ip __i = __first;
+  while (++__i != __last)
+    if (std::invoke(__comp, std::invoke(__proj, *__i), std::invoke(__proj, *__first)))
+      __first = __i;
+  return __first;
+}
+
+namespace __min_element {
+struct __fn {
+  template <forward_iterator _Ip,
+            sentinel_for<_Ip> _Sp,
+            class _Proj                                             = identity,
+            indirect_strict_weak_order<projected<_Ip, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Ip
+  operator()(_Ip __first, _Sp __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    return ranges::__min_element_impl(__first, __last, __comp, __proj);
+  }
+
+  template <forward_range _Rp,
+            class _Proj                                                         = identity,
+            indirect_strict_weak_order<projected<iterator_t<_Rp>, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Rp>
+  operator()(_Rp&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+    return ranges::__min_element_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
+  }
+};
+} // namespace __min_element
+
+inline namespace __cpo {
+inline constexpr auto min_element = __min_element::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_MIN_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_minmax.h b/libcxx/include/__cxx03/__algorithm/ranges_minmax.h
new file mode 100644
index 000000000000..09cbefd91a8c
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_minmax.h
@@ -0,0 +1,175 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_MINMAX_H
+#define _LIBCPP___ALGORITHM_RANGES_MINMAX_H
+
+#include <__algorithm/min_max_result.h>
+#include <__algorithm/minmax_element.h>
+#include <__assert>
+#include <__concepts/copyable.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/next.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__type_traits/desugars_to.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/is_trivially_copyable.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+template <class _T1>
+using minmax_result = min_max_result<_T1>;
+
+namespace __minmax {
+struct __fn {
+  template <class _Type,
+            class _Proj                                                      = identity,
+            indirect_strict_weak_order<projected<const _Type*, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr ranges::minmax_result<const _Type&>
+  operator()(_LIBCPP_LIFETIMEBOUND const _Type& __a,
+             _LIBCPP_LIFETIMEBOUND const _Type& __b,
+             _Comp __comp = {},
+             _Proj __proj = {}) const {
+    if (std::invoke(__comp, std::invoke(__proj, __b), std::invoke(__proj, __a)))
+      return {__b, __a};
+    return {__a, __b};
+  }
+
+  template <copyable _Type,
+            class _Proj                                                      = identity,
+            indirect_strict_weak_order<projected<const _Type*, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr ranges::minmax_result<_Type>
+  operator()(initializer_list<_Type> __il, _Comp __comp = {}, _Proj __proj = {}) const {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __il.begin() != __il.end(), "initializer_list has to contain at least one element");
+    auto __iters = std::__minmax_element_impl(__il.begin(), __il.end(), __comp, __proj);
+    return ranges::minmax_result<_Type>{*__iters.first, *__iters.second};
+  }
+
+  template <input_range _Range,
+            class _Proj                                                            = identity,
+            indirect_strict_weak_order<projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
+    requires indirectly_copyable_storable<iterator_t<_Range>, range_value_t<_Range>*>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr ranges::minmax_result<range_value_t<_Range>>
+  operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __first  = ranges::begin(__r);
+    auto __last   = ranges::end(__r);
+    using _ValueT = range_value_t<_Range>;
+
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__first != __last, "range has to contain at least one element");
+
+    // This optimiation is not in minmax_element because clang doesn't see through the pointers and as a result doesn't
+    // vectorize the code.
+    if constexpr (contiguous_range<_Range> && is_integral_v<_ValueT> &&
+                  __is_cheap_to_copy<_ValueT> & __is_identity<_Proj>::value &&
+                  __desugars_to_v<__less_tag, _Comp, _ValueT, _ValueT>) {
+      minmax_result<_ValueT> __result = {__r[0], __r[0]};
+      for (auto __e : __r) {
+        if (__e < __result.min)
+          __result.min = __e;
+        if (__result.max < __e)
+          __result.max = __e;
+      }
+      return __result;
+    } else if constexpr (forward_range<_Range>) {
+      // Special-case the one element case. Avoid repeatedly initializing objects from the result of an iterator
+      // dereference when doing so might not be idempotent. The `if constexpr` avoids the extra branch in cases where
+      // it's not needed.
+      if constexpr (!same_as<remove_cvref_t<range_reference_t<_Range>>, _ValueT> ||
+                    is_rvalue_reference_v<range_reference_t<_Range>>) {
+        if (ranges::next(__first) == __last) {
+          // During initialization, members are allowed to refer to already initialized members
+          // (see http://eel.is/c++draft/dcl.init.aggr#6)
+          minmax_result<_ValueT> __result = {*__first, __result.min};
+          return __result;
+        }
+      }
+      auto __result = std::__minmax_element_impl(__first, __last, __comp, __proj);
+      return {*__result.first, *__result.second};
+    } else {
+      // input_iterators can't be copied, so the implementation for input_iterators has to store
+      // the values instead of a pointer to the correct values
+      auto __less = [&](auto&& __a, auto&& __b) -> bool {
+        return std::invoke(__comp,
+                           std::invoke(__proj, std::forward<decltype(__a)>(__a)),
+                           std::invoke(__proj, std::forward<decltype(__b)>(__b)));
+      };
+
+      // During initialization, members are allowed to refer to already initialized members
+      // (see http://eel.is/c++draft/dcl.init.aggr#6)
+      ranges::minmax_result<_ValueT> __result = {*__first, __result.min};
+      if (__first == __last || ++__first == __last)
+        return __result;
+
+      if (__less(*__first, __result.min))
+        __result.min = *__first;
+      else
+        __result.max = *__first;
+
+      while (++__first != __last) {
+        _ValueT __i = *__first;
+        if (++__first == __last) {
+          if (__less(__i, __result.min))
+            __result.min = __i;
+          else if (!__less(__i, __result.max))
+            __result.max = __i;
+          return __result;
+        }
+
+        if (__less(*__first, __i)) {
+          if (__less(*__first, __result.min))
+            __result.min = *__first;
+          if (!__less(__i, __result.max))
+            __result.max = std::move(__i);
+        } else {
+          if (__less(__i, __result.min))
+            __result.min = std::move(__i);
+          if (!__less(*__first, __result.max))
+            __result.max = *__first;
+        }
+      }
+      return __result;
+    }
+  }
+};
+} // namespace __minmax
+
+inline namespace __cpo {
+inline constexpr auto minmax = __minmax::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___ALGORITHM_RANGES_MINMAX_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_minmax_element.h b/libcxx/include/__cxx03/__algorithm/ranges_minmax_element.h
new file mode 100644
index 000000000000..4bf6d2404e46
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_minmax_element.h
@@ -0,0 +1,78 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_MINMAX_ELEMENT_H
+#define _LIBCPP___ALGORITHM_RANGES_MINMAX_ELEMENT_H
+
+#include <__algorithm/min_max_result.h>
+#include <__algorithm/minmax_element.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _T1>
+using minmax_element_result = min_max_result<_T1>;
+
+namespace __minmax_element {
+struct __fn {
+  template <forward_iterator _Ip,
+            sentinel_for<_Ip> _Sp,
+            class _Proj                                             = identity,
+            indirect_strict_weak_order<projected<_Ip, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr ranges::minmax_element_result<_Ip>
+  operator()(_Ip __first, _Sp __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __ret = std::__minmax_element_impl(std::move(__first), std::move(__last), __comp, __proj);
+    return {__ret.first, __ret.second};
+  }
+
+  template <forward_range _Rp,
+            class _Proj                                                         = identity,
+            indirect_strict_weak_order<projected<iterator_t<_Rp>, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr ranges::minmax_element_result<borrowed_iterator_t<_Rp>>
+  operator()(_Rp&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __ret = std::__minmax_element_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
+    return {__ret.first, __ret.second};
+  }
+};
+} // namespace __minmax_element
+
+inline namespace __cpo {
+inline constexpr auto minmax_element = __minmax_element::__fn{};
+} // namespace __cpo
+
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_MINMAX_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_mismatch.h b/libcxx/include/__cxx03/__algorithm/ranges_mismatch.h
new file mode 100644
index 000000000000..c4bf0022a9bc
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_mismatch.h
@@ -0,0 +1,100 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_MISMATCH_H
+#define _LIBCPP___ALGORITHM_RANGES_MISMATCH_H
+
+#include <__algorithm/in_in_result.h>
+#include <__algorithm/mismatch.h>
+#include <__algorithm/unwrap_range.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/indirectly_comparable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+template <class _I1, class _I2>
+using mismatch_result = in_in_result<_I1, _I2>;
+
+namespace __mismatch {
+struct __fn {
+  template <class _I1, class _S1, class _I2, class _S2, class _Pred, class _Proj1, class _Proj2>
+  static _LIBCPP_HIDE_FROM_ABI constexpr mismatch_result<_I1, _I2>
+  __go(_I1 __first1, _S1 __last1, _I2 __first2, _S2 __last2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+    if constexpr (forward_iterator<_I1> && forward_iterator<_I2>) {
+      auto __range1 = std::__unwrap_range(__first1, __last1);
+      auto __range2 = std::__unwrap_range(__first2, __last2);
+      auto __res =
+          std::__mismatch(__range1.first, __range1.second, __range2.first, __range2.second, __pred, __proj1, __proj2);
+      return {std::__rewrap_range<_S1>(__first1, __res.first), std::__rewrap_range<_S2>(__first2, __res.second)};
+    } else {
+      auto __res = std::__mismatch(
+          std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __pred, __proj1, __proj2);
+      return {std::move(__res.first), std::move(__res.second)};
+    }
+  }
+
+  template <input_iterator _I1,
+            sentinel_for<_I1> _S1,
+            input_iterator _I2,
+            sentinel_for<_I2> _S2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_comparable<_I1, _I2, _Pred, _Proj1, _Proj2>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr mismatch_result<_I1, _I2> operator()(
+      _I1 __first1, _S1 __last1, _I2 __first2, _S2 __last2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {})
+      const {
+    return __go(std::move(__first1), __last1, std::move(__first2), __last2, __pred, __proj1, __proj2);
+  }
+
+  template <input_range _R1,
+            input_range _R2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_comparable<iterator_t<_R1>, iterator_t<_R2>, _Pred, _Proj1, _Proj2>
+  [[nodiscard]]
+  _LIBCPP_HIDE_FROM_ABI constexpr mismatch_result<borrowed_iterator_t<_R1>, borrowed_iterator_t<_R2>>
+  operator()(_R1&& __r1, _R2&& __r2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+    return __go(
+        ranges::begin(__r1), ranges::end(__r1), ranges::begin(__r2), ranges::end(__r2), __pred, __proj1, __proj2);
+  }
+};
+} // namespace __mismatch
+
+inline namespace __cpo {
+constexpr inline auto mismatch = __mismatch::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_MISMATCH_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_move.h b/libcxx/include/__cxx03/__algorithm/ranges_move.h
new file mode 100644
index 000000000000..be869f36c973
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_move.h
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_MOVE_H
+#define _LIBCPP___ALGORITHM_RANGES_MOVE_H
+
+#include <__algorithm/in_out_result.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/move.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter, class _OutIter>
+using move_result = in_out_result<_InIter, _OutIter>;
+
+namespace __move {
+struct __fn {
+  template <class _InIter, class _Sent, class _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr static move_result<_InIter, _OutIter>
+  __move_impl(_InIter __first, _Sent __last, _OutIter __result) {
+    auto __ret = std::__move<_RangeAlgPolicy>(std::move(__first), std::move(__last), std::move(__result));
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+
+  template <input_iterator _InIter, sentinel_for<_InIter> _Sent, weakly_incrementable _OutIter>
+    requires indirectly_movable<_InIter, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr move_result<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result) const {
+    return __move_impl(std::move(__first), std::move(__last), std::move(__result));
+  }
+
+  template <input_range _Range, weakly_incrementable _OutIter>
+    requires indirectly_movable<iterator_t<_Range>, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr move_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __range, _OutIter __result) const {
+    return __move_impl(ranges::begin(__range), ranges::end(__range), std::move(__result));
+  }
+};
+} // namespace __move
+
+inline namespace __cpo {
+inline constexpr auto move = __move::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_MOVE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_move_backward.h b/libcxx/include/__cxx03/__algorithm/ranges_move_backward.h
new file mode 100644
index 000000000000..6d4071a33b81
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_move_backward.h
@@ -0,0 +1,76 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_MOVE_BACKWARD_H
+#define _LIBCPP___ALGORITHM_RANGES_MOVE_BACKWARD_H
+
+#include <__algorithm/in_out_result.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/move_backward.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/next.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter, class _OutIter>
+using move_backward_result = in_out_result<_InIter, _OutIter>;
+
+namespace __move_backward {
+struct __fn {
+  template <class _InIter, class _Sent, class _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr static move_backward_result<_InIter, _OutIter>
+  __move_backward_impl(_InIter __first, _Sent __last, _OutIter __result) {
+    auto __ret = std::__move_backward<_RangeAlgPolicy>(std::move(__first), std::move(__last), std::move(__result));
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+
+  template <bidirectional_iterator _InIter, sentinel_for<_InIter> _Sent, bidirectional_iterator _OutIter>
+    requires indirectly_movable<_InIter, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr move_backward_result<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result) const {
+    return __move_backward_impl(std::move(__first), std::move(__last), std::move(__result));
+  }
+
+  template <bidirectional_range _Range, bidirectional_iterator _Iter>
+    requires indirectly_movable<iterator_t<_Range>, _Iter>
+  _LIBCPP_HIDE_FROM_ABI constexpr move_backward_result<borrowed_iterator_t<_Range>, _Iter>
+  operator()(_Range&& __range, _Iter __result) const {
+    return __move_backward_impl(ranges::begin(__range), ranges::end(__range), std::move(__result));
+  }
+};
+} // namespace __move_backward
+
+inline namespace __cpo {
+inline constexpr auto move_backward = __move_backward::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_MOVE_BACKWARD_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_next_permutation.h b/libcxx/include/__cxx03/__algorithm/ranges_next_permutation.h
new file mode 100644
index 000000000000..18535e0a6254
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_next_permutation.h
@@ -0,0 +1,78 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_NEXT_PERMUTATION_H
+#define _LIBCPP___ALGORITHM_RANGES_NEXT_PERMUTATION_H
+
+#include <__algorithm/in_found_result.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/next_permutation.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/sortable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter>
+using next_permutation_result = in_found_result<_InIter>;
+
+namespace __next_permutation {
+
+struct __fn {
+  template <bidirectional_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<_Iter, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr next_permutation_result<_Iter>
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __result = std::__next_permutation<_RangeAlgPolicy>(
+        std::move(__first), std::move(__last), std::__make_projected(__comp, __proj));
+    return {std::move(__result.first), std::move(__result.second)};
+  }
+
+  template <bidirectional_range _Range, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<iterator_t<_Range>, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr next_permutation_result<borrowed_iterator_t<_Range>>
+  operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __result = std::__next_permutation<_RangeAlgPolicy>(
+        ranges::begin(__range), ranges::end(__range), std::__make_projected(__comp, __proj));
+    return {std::move(__result.first), std::move(__result.second)};
+  }
+};
+
+} // namespace __next_permutation
+
+inline namespace __cpo {
+constexpr inline auto next_permutation = __next_permutation::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_NEXT_PERMUTATION_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_none_of.h b/libcxx/include/__cxx03/__algorithm/ranges_none_of.h
new file mode 100644
index 000000000000..7df3c1829fcf
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_none_of.h
@@ -0,0 +1,75 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_NONE_OF_H
+#define _LIBCPP___ALGORITHM_RANGES_NONE_OF_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __none_of {
+struct __fn {
+  template <class _Iter, class _Sent, class _Proj, class _Pred>
+  _LIBCPP_HIDE_FROM_ABI constexpr static bool
+  __none_of_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
+    for (; __first != __last; ++__first) {
+      if (std::invoke(__pred, std::invoke(__proj, *__first)))
+        return false;
+    }
+    return true;
+  }
+
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Iter __first, _Sent __last, _Pred __pred = {}, _Proj __proj = {}) const {
+    return __none_of_impl(std::move(__first), std::move(__last), __pred, __proj);
+  }
+
+  template <input_range _Range,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+    return __none_of_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
+  }
+};
+} // namespace __none_of
+
+inline namespace __cpo {
+inline constexpr auto none_of = __none_of::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_NONE_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_nth_element.h b/libcxx/include/__cxx03/__algorithm/ranges_nth_element.h
new file mode 100644
index 000000000000..90ade9efe10d
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_nth_element.h
@@ -0,0 +1,84 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_NTH_ELEMENT_H
+#define _LIBCPP___ALGORITHM_RANGES_NTH_ELEMENT_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/nth_element.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/projected.h>
+#include <__iterator/sortable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __nth_element {
+
+struct __fn {
+  template <class _Iter, class _Sent, class _Comp, class _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __nth_element_fn_impl(_Iter __first, _Iter __nth, _Sent __last, _Comp& __comp, _Proj& __proj) {
+    auto __last_iter = ranges::next(__first, __last);
+
+    auto&& __projected_comp = std::__make_projected(__comp, __proj);
+    std::__nth_element_impl<_RangeAlgPolicy>(std::move(__first), std::move(__nth), __last_iter, __projected_comp);
+
+    return __last_iter;
+  }
+
+  template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<_Iter, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Iter __nth, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __nth_element_fn_impl(std::move(__first), std::move(__nth), std::move(__last), __comp, __proj);
+  }
+
+  template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<iterator_t<_Range>, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, iterator_t<_Range> __nth, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __nth_element_fn_impl(ranges::begin(__r), std::move(__nth), ranges::end(__r), __comp, __proj);
+  }
+};
+
+} // namespace __nth_element
+
+inline namespace __cpo {
+inline constexpr auto nth_element = __nth_element::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_NTH_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_partial_sort.h b/libcxx/include/__cxx03/__algorithm/ranges_partial_sort.h
new file mode 100644
index 000000000000..c67247d2e0a7
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_partial_sort.h
@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_PARTIAL_SORT_H
+#define _LIBCPP___ALGORITHM_RANGES_PARTIAL_SORT_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/partial_sort.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/projected.h>
+#include <__iterator/sortable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __partial_sort {
+
+struct __fn {
+  template <class _Iter, class _Sent, class _Comp, class _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __partial_sort_fn_impl(_Iter __first, _Iter __middle, _Sent __last, _Comp& __comp, _Proj& __proj) {
+    auto&& __projected_comp = std::__make_projected(__comp, __proj);
+    return std::__partial_sort<_RangeAlgPolicy>(std::move(__first), std::move(__middle), __last, __projected_comp);
+  }
+
+  template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<_Iter, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Iter __middle, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __partial_sort_fn_impl(std::move(__first), std::move(__middle), std::move(__last), __comp, __proj);
+  }
+
+  template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<iterator_t<_Range>, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, iterator_t<_Range> __middle, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __partial_sort_fn_impl(ranges::begin(__r), std::move(__middle), ranges::end(__r), __comp, __proj);
+  }
+};
+
+} // namespace __partial_sort
+
+inline namespace __cpo {
+inline constexpr auto partial_sort = __partial_sort::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_PARTIAL_SORT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_partial_sort_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_partial_sort_copy.h
new file mode 100644
index 000000000000..b3bdeb78fb6f
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_partial_sort_copy.h
@@ -0,0 +1,114 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_PARTIAL_SORT_COPY_H
+#define _LIBCPP___ALGORITHM_RANGES_PARTIAL_SORT_COPY_H
+
+#include <__algorithm/in_out_result.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/partial_sort_copy.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/projected.h>
+#include <__iterator/sortable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter, class _OutIter>
+using partial_sort_copy_result = in_out_result<_InIter, _OutIter>;
+
+namespace __partial_sort_copy {
+
+struct __fn {
+  template <input_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            random_access_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Comp  = ranges::less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_copyable<_Iter1, _Iter2> && sortable<_Iter2, _Comp, _Proj2> &&
+             indirect_strict_weak_order<_Comp, projected<_Iter1, _Proj1>, projected<_Iter2, _Proj2>>
+  _LIBCPP_HIDE_FROM_ABI constexpr partial_sort_copy_result<_Iter1, _Iter2> operator()(
+      _Iter1 __first,
+      _Sent1 __last,
+      _Iter2 __result_first,
+      _Sent2 __result_last,
+      _Comp __comp   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    auto __result = std::__partial_sort_copy<_RangeAlgPolicy>(
+        std::move(__first),
+        std::move(__last),
+        std::move(__result_first),
+        std::move(__result_last),
+        __comp,
+        __proj1,
+        __proj2);
+    return {std::move(__result.first), std::move(__result.second)};
+  }
+
+  template <input_range _Range1,
+            random_access_range _Range2,
+            class _Comp  = ranges::less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_copyable<iterator_t<_Range1>, iterator_t<_Range2>> &&
+             sortable<iterator_t<_Range2>, _Comp, _Proj2> &&
+             indirect_strict_weak_order<_Comp,
+                                        projected<iterator_t<_Range1>, _Proj1>,
+                                        projected<iterator_t<_Range2>, _Proj2>>
+  _LIBCPP_HIDE_FROM_ABI constexpr partial_sort_copy_result<borrowed_iterator_t<_Range1>, borrowed_iterator_t<_Range2>>
+  operator()(
+      _Range1&& __range, _Range2&& __result_range, _Comp __comp = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+    auto __result = std::__partial_sort_copy<_RangeAlgPolicy>(
+        ranges::begin(__range),
+        ranges::end(__range),
+        ranges::begin(__result_range),
+        ranges::end(__result_range),
+        __comp,
+        __proj1,
+        __proj2);
+    return {std::move(__result.first), std::move(__result.second)};
+  }
+};
+
+} // namespace __partial_sort_copy
+
+inline namespace __cpo {
+inline constexpr auto partial_sort_copy = __partial_sort_copy::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_PARTIAL_SORT_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_partition.h b/libcxx/include/__cxx03/__algorithm/ranges_partition.h
new file mode 100644
index 000000000000..a67ac4c96757
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_partition.h
@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_PARTITION_H
+#define _LIBCPP___ALGORITHM_RANGES_PARTITION_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/partition.h>
+#include <__algorithm/ranges_iterator_concept.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/permutable.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/subrange.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __partition {
+
+struct __fn {
+  template <class _Iter, class _Sent, class _Proj, class _Pred>
+  _LIBCPP_HIDE_FROM_ABI static constexpr subrange<__remove_cvref_t<_Iter>>
+  __partition_fn_impl(_Iter&& __first, _Sent&& __last, _Pred&& __pred, _Proj&& __proj) {
+    auto&& __projected_pred = std::__make_projected(__pred, __proj);
+    auto __result           = std::__partition<_RangeAlgPolicy>(
+        std::move(__first), std::move(__last), __projected_pred, __iterator_concept<_Iter>());
+
+    return {std::move(__result.first), std::move(__result.second)};
+  }
+
+  template <permutable _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
+  _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
+  operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
+    return __partition_fn_impl(__first, __last, __pred, __proj);
+  }
+
+  template <forward_range _Range,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+    requires permutable<iterator_t<_Range>>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+    return __partition_fn_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
+  }
+};
+
+} // namespace __partition
+
+inline namespace __cpo {
+inline constexpr auto partition = __partition::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_PARTITION_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_partition_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_partition_copy.h
new file mode 100644
index 000000000000..d60c865dd2a8
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_partition_copy.h
@@ -0,0 +1,110 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_PARTITION_COPY_H
+#define _LIBCPP___ALGORITHM_RANGES_PARTITION_COPY_H
+
+#include <__algorithm/in_out_out_result.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter, class _OutIter1, class _OutIter2>
+using partition_copy_result = in_out_out_result<_InIter, _OutIter1, _OutIter2>;
+
+namespace __partition_copy {
+
+struct __fn {
+  // TODO(ranges): delegate to the classic algorithm.
+  template <class _InIter, class _Sent, class _OutIter1, class _OutIter2, class _Proj, class _Pred>
+  _LIBCPP_HIDE_FROM_ABI constexpr static partition_copy_result<__remove_cvref_t<_InIter>,
+                                                               __remove_cvref_t<_OutIter1>,
+                                                               __remove_cvref_t<_OutIter2> >
+  __partition_copy_fn_impl(
+      _InIter&& __first,
+      _Sent&& __last,
+      _OutIter1&& __out_true,
+      _OutIter2&& __out_false,
+      _Pred& __pred,
+      _Proj& __proj) {
+    for (; __first != __last; ++__first) {
+      if (std::invoke(__pred, std::invoke(__proj, *__first))) {
+        *__out_true = *__first;
+        ++__out_true;
+
+      } else {
+        *__out_false = *__first;
+        ++__out_false;
+      }
+    }
+
+    return {std::move(__first), std::move(__out_true), std::move(__out_false)};
+  }
+
+  template <input_iterator _InIter,
+            sentinel_for<_InIter> _Sent,
+            weakly_incrementable _OutIter1,
+            weakly_incrementable _OutIter2,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_InIter, _Proj>> _Pred>
+    requires indirectly_copyable<_InIter, _OutIter1> && indirectly_copyable<_InIter, _OutIter2>
+  _LIBCPP_HIDE_FROM_ABI constexpr partition_copy_result<_InIter, _OutIter1, _OutIter2> operator()(
+      _InIter __first, _Sent __last, _OutIter1 __out_true, _OutIter2 __out_false, _Pred __pred, _Proj __proj = {})
+      const {
+    return __partition_copy_fn_impl(
+        std::move(__first), std::move(__last), std::move(__out_true), std::move(__out_false), __pred, __proj);
+  }
+
+  template <input_range _Range,
+            weakly_incrementable _OutIter1,
+            weakly_incrementable _OutIter2,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+    requires indirectly_copyable<iterator_t<_Range>, _OutIter1> && indirectly_copyable<iterator_t<_Range>, _OutIter2>
+  _LIBCPP_HIDE_FROM_ABI constexpr partition_copy_result<borrowed_iterator_t<_Range>, _OutIter1, _OutIter2>
+  operator()(_Range&& __range, _OutIter1 __out_true, _OutIter2 __out_false, _Pred __pred, _Proj __proj = {}) const {
+    return __partition_copy_fn_impl(
+        ranges::begin(__range), ranges::end(__range), std::move(__out_true), std::move(__out_false), __pred, __proj);
+  }
+};
+
+} // namespace __partition_copy
+
+inline namespace __cpo {
+inline constexpr auto partition_copy = __partition_copy::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_PARTITION_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_partition_point.h b/libcxx/include/__cxx03/__algorithm/ranges_partition_point.h
new file mode 100644
index 000000000000..c5b11b5fed19
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_partition_point.h
@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_PARTITION_POINT_H
+#define _LIBCPP___ALGORITHM_RANGES_PARTITION_POINT_H
+
+#include <__algorithm/half_positive.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __partition_point {
+
+struct __fn {
+  // TODO(ranges): delegate to the classic algorithm.
+  template <class _Iter, class _Sent, class _Proj, class _Pred>
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __partition_point_fn_impl(_Iter&& __first, _Sent&& __last, _Pred& __pred, _Proj& __proj) {
+    auto __len = ranges::distance(__first, __last);
+
+    while (__len != 0) {
+      auto __half_len = std::__half_positive(__len);
+      auto __mid      = ranges::next(__first, __half_len);
+
+      if (std::invoke(__pred, std::invoke(__proj, *__mid))) {
+        __first = ++__mid;
+        __len -= __half_len + 1;
+
+      } else {
+        __len = __half_len;
+      }
+    }
+
+    return __first;
+  }
+
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
+    return __partition_point_fn_impl(std::move(__first), std::move(__last), __pred, __proj);
+  }
+
+  template <forward_range _Range,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+    return __partition_point_fn_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
+  }
+};
+
+} // namespace __partition_point
+
+inline namespace __cpo {
+inline constexpr auto partition_point = __partition_point::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_PARTITION_POINT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_pop_heap.h b/libcxx/include/__cxx03/__algorithm/ranges_pop_heap.h
new file mode 100644
index 000000000000..01f92c0f2288
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_pop_heap.h
@@ -0,0 +1,86 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_POP_HEAP_H
+#define _LIBCPP___ALGORITHM_RANGES_POP_HEAP_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/pop_heap.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/projected.h>
+#include <__iterator/sortable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __pop_heap {
+
+struct __fn {
+  template <class _Iter, class _Sent, class _Comp, class _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __pop_heap_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+    auto __last_iter = ranges::next(__first, __last);
+    auto __len       = __last_iter - __first;
+
+    auto&& __projected_comp = std::__make_projected(__comp, __proj);
+    std::__pop_heap<_RangeAlgPolicy>(std::move(__first), __last_iter, __projected_comp, __len);
+
+    return __last_iter;
+  }
+
+  template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<_Iter, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __pop_heap_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
+  }
+
+  template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<iterator_t<_Range>, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __pop_heap_fn_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
+  }
+};
+
+} // namespace __pop_heap
+
+inline namespace __cpo {
+inline constexpr auto pop_heap = __pop_heap::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_POP_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_prev_permutation.h b/libcxx/include/__cxx03/__algorithm/ranges_prev_permutation.h
new file mode 100644
index 000000000000..225cee9b75ec
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_prev_permutation.h
@@ -0,0 +1,78 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_PREV_PERMUTATION_H
+#define _LIBCPP___ALGORITHM_RANGES_PREV_PERMUTATION_H
+
+#include <__algorithm/in_found_result.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/prev_permutation.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/sortable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter>
+using prev_permutation_result = in_found_result<_InIter>;
+
+namespace __prev_permutation {
+
+struct __fn {
+  template <bidirectional_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<_Iter, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr prev_permutation_result<_Iter>
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __result = std::__prev_permutation<_RangeAlgPolicy>(
+        std::move(__first), std::move(__last), std::__make_projected(__comp, __proj));
+    return {std::move(__result.first), std::move(__result.second)};
+  }
+
+  template <bidirectional_range _Range, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<iterator_t<_Range>, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr prev_permutation_result<borrowed_iterator_t<_Range>>
+  operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __result = std::__prev_permutation<_RangeAlgPolicy>(
+        ranges::begin(__range), ranges::end(__range), std::__make_projected(__comp, __proj));
+    return {std::move(__result.first), std::move(__result.second)};
+  }
+};
+
+} // namespace __prev_permutation
+
+inline namespace __cpo {
+constexpr inline auto prev_permutation = __prev_permutation::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_PREV_PERMUTATION_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_push_heap.h b/libcxx/include/__cxx03/__algorithm/ranges_push_heap.h
new file mode 100644
index 000000000000..9d187af38c53
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_push_heap.h
@@ -0,0 +1,85 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_PUSH_HEAP_H
+#define _LIBCPP___ALGORITHM_RANGES_PUSH_HEAP_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/push_heap.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/projected.h>
+#include <__iterator/sortable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __push_heap {
+
+struct __fn {
+  template <class _Iter, class _Sent, class _Comp, class _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __push_heap_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+    auto __last_iter = ranges::next(__first, __last);
+
+    auto&& __projected_comp = std::__make_projected(__comp, __proj);
+    std::__push_heap<_RangeAlgPolicy>(std::move(__first), __last_iter, __projected_comp);
+
+    return __last_iter;
+  }
+
+  template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<_Iter, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __push_heap_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
+  }
+
+  template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<iterator_t<_Range>, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __push_heap_fn_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
+  }
+};
+
+} // namespace __push_heap
+
+inline namespace __cpo {
+inline constexpr auto push_heap = __push_heap::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_PUSH_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_remove.h b/libcxx/include/__cxx03/__algorithm/ranges_remove.h
new file mode 100644
index 000000000000..17c3a2c5cd06
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_remove.h
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_REMOVE_H
+#define _LIBCPP___ALGORITHM_RANGES_REMOVE_H
+#include <__config>
+
+#include <__algorithm/ranges_remove_if.h>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/permutable.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/subrange.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __remove {
+struct __fn {
+  template <permutable _Iter, sentinel_for<_Iter> _Sent, class _Type, class _Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<_Iter, _Proj>, const _Type*>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
+  operator()(_Iter __first, _Sent __last, const _Type& __value, _Proj __proj = {}) const {
+    auto __pred = [&](auto&& __other) -> bool { return __value == __other; };
+    return ranges::__remove_if_impl(std::move(__first), std::move(__last), __pred, __proj);
+  }
+
+  template <forward_range _Range, class _Type, class _Proj = identity>
+    requires permutable<iterator_t<_Range>> &&
+             indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _Type*>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, const _Type& __value, _Proj __proj = {}) const {
+    auto __pred = [&](auto&& __other) -> bool { return __value == __other; };
+    return ranges::__remove_if_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
+  }
+};
+} // namespace __remove
+
+inline namespace __cpo {
+inline constexpr auto remove = __remove::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_REMOVE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_remove_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_remove_copy.h
new file mode 100644
index 000000000000..84529eceac68
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_remove_copy.h
@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_REMOVE_COPY_H
+#define _LIBCPP___ALGORITHM_RANGES_REMOVE_COPY_H
+
+#include <__algorithm/in_out_result.h>
+#include <__algorithm/ranges_remove_copy_if.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter, class _OutIter>
+using remove_copy_result = in_out_result<_InIter, _OutIter>;
+
+namespace __remove_copy {
+
+struct __fn {
+  template <input_iterator _InIter,
+            sentinel_for<_InIter> _Sent,
+            weakly_incrementable _OutIter,
+            class _Type,
+            class _Proj = identity>
+    requires indirectly_copyable<_InIter, _OutIter> &&
+             indirect_binary_predicate<ranges::equal_to, projected<_InIter, _Proj>, const _Type*>
+  _LIBCPP_HIDE_FROM_ABI constexpr remove_copy_result<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result, const _Type& __value, _Proj __proj = {}) const {
+    auto __pred = [&](auto&& __val) -> bool { return __value == __val; };
+    return ranges::__remove_copy_if_impl(std::move(__first), std::move(__last), std::move(__result), __pred, __proj);
+  }
+
+  template <input_range _Range, weakly_incrementable _OutIter, class _Type, class _Proj = identity>
+    requires indirectly_copyable<iterator_t<_Range>, _OutIter> &&
+             indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _Type*>
+  _LIBCPP_HIDE_FROM_ABI constexpr remove_copy_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __range, _OutIter __result, const _Type& __value, _Proj __proj = {}) const {
+    auto __pred = [&](auto&& __val) -> bool { return __value == __val; };
+    return ranges::__remove_copy_if_impl(
+        ranges::begin(__range), ranges::end(__range), std::move(__result), __pred, __proj);
+  }
+};
+
+} // namespace __remove_copy
+
+inline namespace __cpo {
+inline constexpr auto remove_copy = __remove_copy::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_REMOVE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_remove_copy_if.h b/libcxx/include/__cxx03/__algorithm/ranges_remove_copy_if.h
new file mode 100644
index 000000000000..56fe01753312
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_remove_copy_if.h
@@ -0,0 +1,95 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_REMOVE_COPY_IF_H
+#define _LIBCPP___ALGORITHM_RANGES_REMOVE_COPY_IF_H
+
+#include <__algorithm/in_out_result.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/remove_copy_if.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter, class _OutIter>
+using remove_copy_if_result = in_out_result<_InIter, _OutIter>;
+
+template <class _InIter, class _Sent, class _OutIter, class _Proj, class _Pred>
+_LIBCPP_HIDE_FROM_ABI constexpr in_out_result<_InIter, _OutIter>
+__remove_copy_if_impl(_InIter __first, _Sent __last, _OutIter __result, _Pred& __pred, _Proj& __proj) {
+  for (; __first != __last; ++__first) {
+    if (!std::invoke(__pred, std::invoke(__proj, *__first))) {
+      *__result = *__first;
+      ++__result;
+    }
+  }
+  return {std::move(__first), std::move(__result)};
+}
+
+namespace __remove_copy_if {
+
+struct __fn {
+  template <input_iterator _InIter,
+            sentinel_for<_InIter> _Sent,
+            weakly_incrementable _OutIter,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_InIter, _Proj>> _Pred>
+    requires indirectly_copyable<_InIter, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr remove_copy_if_result<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result, _Pred __pred, _Proj __proj = {}) const {
+    return ranges::__remove_copy_if_impl(std::move(__first), std::move(__last), std::move(__result), __pred, __proj);
+  }
+
+  template <input_range _Range,
+            weakly_incrementable _OutIter,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+    requires indirectly_copyable<iterator_t<_Range>, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr remove_copy_if_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __range, _OutIter __result, _Pred __pred, _Proj __proj = {}) const {
+    return ranges::__remove_copy_if_impl(
+        ranges::begin(__range), ranges::end(__range), std::move(__result), __pred, __proj);
+  }
+};
+
+} // namespace __remove_copy_if
+
+inline namespace __cpo {
+inline constexpr auto remove_copy_if = __remove_copy_if::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_REMOVE_COPY_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_remove_if.h b/libcxx/include/__cxx03/__algorithm/ranges_remove_if.h
new file mode 100644
index 000000000000..0ea5d9a01b88
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_remove_if.h
@@ -0,0 +1,89 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_REMOVE_IF_H
+#define _LIBCPP___ALGORITHM_RANGES_REMOVE_IF_H
+#include <__config>
+
+#include <__algorithm/ranges_find_if.h>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/permutable.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/subrange.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _Iter, class _Sent, class _Proj, class _Pred>
+_LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
+__remove_if_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
+  auto __new_end = ranges::__find_if_impl(__first, __last, __pred, __proj);
+  if (__new_end == __last)
+    return {__new_end, __new_end};
+
+  _Iter __i = __new_end;
+  while (++__i != __last) {
+    if (!std::invoke(__pred, std::invoke(__proj, *__i))) {
+      *__new_end = ranges::iter_move(__i);
+      ++__new_end;
+    }
+  }
+  return {__new_end, __i};
+}
+
+namespace __remove_if {
+struct __fn {
+  template <permutable _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
+  operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
+    return ranges::__remove_if_impl(std::move(__first), std::move(__last), __pred, __proj);
+  }
+
+  template <forward_range _Range,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+    requires permutable<iterator_t<_Range>>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+    return ranges::__remove_if_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
+  }
+};
+} // namespace __remove_if
+
+inline namespace __cpo {
+inline constexpr auto remove_if = __remove_if::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_REMOVE_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_replace.h b/libcxx/include/__cxx03/__algorithm/ranges_replace.h
new file mode 100644
index 000000000000..2b88dc032972
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_replace.h
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_REPLACE_H
+#define _LIBCPP___ALGORITHM_RANGES_REPLACE_H
+
+#include <__algorithm/ranges_replace_if.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __replace {
+struct __fn {
+  template <input_iterator _Iter, sentinel_for<_Iter> _Sent, class _Type1, class _Type2, class _Proj = identity>
+    requires indirectly_writable<_Iter, const _Type2&> &&
+             indirect_binary_predicate<ranges::equal_to, projected<_Iter, _Proj>, const _Type1*>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter operator()(
+      _Iter __first, _Sent __last, const _Type1& __old_value, const _Type2& __new_value, _Proj __proj = {}) const {
+    auto __pred = [&](const auto& __val) -> bool { return __val == __old_value; };
+    return ranges::__replace_if_impl(std::move(__first), std::move(__last), __pred, __new_value, __proj);
+  }
+
+  template <input_range _Range, class _Type1, class _Type2, class _Proj = identity>
+    requires indirectly_writable<iterator_t<_Range>, const _Type2&> &&
+             indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _Type1*>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __range, const _Type1& __old_value, const _Type2& __new_value, _Proj __proj = {}) const {
+    auto __pred = [&](auto&& __val) -> bool { return __val == __old_value; };
+    return ranges::__replace_if_impl(ranges::begin(__range), ranges::end(__range), __pred, __new_value, __proj);
+  }
+};
+} // namespace __replace
+
+inline namespace __cpo {
+inline constexpr auto replace = __replace::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_REPLACE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_replace_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_replace_copy.h
new file mode 100644
index 000000000000..633f993e5c94
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_replace_copy.h
@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_REPLACE_COPY_H
+#define _LIBCPP___ALGORITHM_RANGES_REPLACE_COPY_H
+
+#include <__algorithm/in_out_result.h>
+#include <__algorithm/ranges_replace_copy_if.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter, class _OutIter>
+using replace_copy_result = in_out_result<_InIter, _OutIter>;
+
+namespace __replace_copy {
+
+struct __fn {
+  template <input_iterator _InIter,
+            sentinel_for<_InIter> _Sent,
+            class _OldType,
+            class _NewType,
+            output_iterator<const _NewType&> _OutIter,
+            class _Proj = identity>
+    requires indirectly_copyable<_InIter, _OutIter> &&
+             indirect_binary_predicate<ranges::equal_to, projected<_InIter, _Proj>, const _OldType*>
+  _LIBCPP_HIDE_FROM_ABI constexpr replace_copy_result<_InIter, _OutIter>
+  operator()(_InIter __first,
+             _Sent __last,
+             _OutIter __result,
+             const _OldType& __old_value,
+             const _NewType& __new_value,
+             _Proj __proj = {}) const {
+    auto __pred = [&](const auto& __value) -> bool { return __value == __old_value; };
+    return ranges::__replace_copy_if_impl(
+        std::move(__first), std::move(__last), std::move(__result), __pred, __new_value, __proj);
+  }
+
+  template <input_range _Range,
+            class _OldType,
+            class _NewType,
+            output_iterator<const _NewType&> _OutIter,
+            class _Proj = identity>
+    requires indirectly_copyable<iterator_t<_Range>, _OutIter> &&
+             indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _OldType*>
+  _LIBCPP_HIDE_FROM_ABI constexpr replace_copy_result<borrowed_iterator_t<_Range>, _OutIter> operator()(
+      _Range&& __range, _OutIter __result, const _OldType& __old_value, const _NewType& __new_value, _Proj __proj = {})
+      const {
+    auto __pred = [&](const auto& __value) -> bool { return __value == __old_value; };
+    return ranges::__replace_copy_if_impl(
+        ranges::begin(__range), ranges::end(__range), std::move(__result), __pred, __new_value, __proj);
+  }
+};
+
+} // namespace __replace_copy
+
+inline namespace __cpo {
+inline constexpr auto replace_copy = __replace_copy::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_REPLACE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_replace_copy_if.h b/libcxx/include/__cxx03/__algorithm/ranges_replace_copy_if.h
new file mode 100644
index 000000000000..e065c3ac0acc
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_replace_copy_if.h
@@ -0,0 +1,98 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_REPLACE_COPY_IF_H
+#define _LIBCPP___ALGORITHM_RANGES_REPLACE_COPY_IF_H
+
+#include <__algorithm/in_out_result.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter, class _OutIter>
+using replace_copy_if_result = in_out_result<_InIter, _OutIter>;
+
+template <class _InIter, class _Sent, class _OutIter, class _Pred, class _Type, class _Proj>
+_LIBCPP_HIDE_FROM_ABI constexpr replace_copy_if_result<_InIter, _OutIter> __replace_copy_if_impl(
+    _InIter __first, _Sent __last, _OutIter __result, _Pred& __pred, const _Type& __new_value, _Proj& __proj) {
+  while (__first != __last) {
+    if (std::invoke(__pred, std::invoke(__proj, *__first)))
+      *__result = __new_value;
+    else
+      *__result = *__first;
+
+    ++__first;
+    ++__result;
+  }
+
+  return {std::move(__first), std::move(__result)};
+}
+
+namespace __replace_copy_if {
+
+struct __fn {
+  template <input_iterator _InIter,
+            sentinel_for<_InIter> _Sent,
+            class _Type,
+            output_iterator<const _Type&> _OutIter,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_InIter, _Proj>> _Pred>
+    requires indirectly_copyable<_InIter, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr replace_copy_if_result<_InIter, _OutIter> operator()(
+      _InIter __first, _Sent __last, _OutIter __result, _Pred __pred, const _Type& __new_value, _Proj __proj = {})
+      const {
+    return ranges::__replace_copy_if_impl(
+        std::move(__first), std::move(__last), std::move(__result), __pred, __new_value, __proj);
+  }
+
+  template <input_range _Range,
+            class _Type,
+            output_iterator<const _Type&> _OutIter,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+    requires indirectly_copyable<iterator_t<_Range>, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr replace_copy_if_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __range, _OutIter __result, _Pred __pred, const _Type& __new_value, _Proj __proj = {}) const {
+    return ranges::__replace_copy_if_impl(
+        ranges::begin(__range), ranges::end(__range), std::move(__result), __pred, __new_value, __proj);
+  }
+};
+
+} // namespace __replace_copy_if
+
+inline namespace __cpo {
+inline constexpr auto replace_copy_if = __replace_copy_if::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_REPLACE_COPY_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_replace_if.h b/libcxx/include/__cxx03/__algorithm/ranges_replace_if.h
new file mode 100644
index 000000000000..6445f42aea19
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_replace_if.h
@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_REPLACE_IF_H
+#define _LIBCPP___ALGORITHM_RANGES_REPLACE_IF_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _Iter, class _Sent, class _Type, class _Proj, class _Pred>
+_LIBCPP_HIDE_FROM_ABI constexpr _Iter
+__replace_if_impl(_Iter __first, _Sent __last, _Pred& __pred, const _Type& __new_value, _Proj& __proj) {
+  for (; __first != __last; ++__first) {
+    if (std::invoke(__pred, std::invoke(__proj, *__first)))
+      *__first = __new_value;
+  }
+  return __first;
+}
+
+namespace __replace_if {
+struct __fn {
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Type,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
+    requires indirectly_writable<_Iter, const _Type&>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Pred __pred, const _Type& __new_value, _Proj __proj = {}) const {
+    return ranges::__replace_if_impl(std::move(__first), std::move(__last), __pred, __new_value, __proj);
+  }
+
+  template <input_range _Range,
+            class _Type,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+    requires indirectly_writable<iterator_t<_Range>, const _Type&>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __range, _Pred __pred, const _Type& __new_value, _Proj __proj = {}) const {
+    return ranges::__replace_if_impl(ranges::begin(__range), ranges::end(__range), __pred, __new_value, __proj);
+  }
+};
+} // namespace __replace_if
+
+inline namespace __cpo {
+inline constexpr auto replace_if = __replace_if::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_REPLACE_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_reverse.h b/libcxx/include/__cxx03/__algorithm/ranges_reverse.h
new file mode 100644
index 000000000000..9ec865995b4a
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_reverse.h
@@ -0,0 +1,79 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_REVERSE_H
+#define _LIBCPP___ALGORITHM_RANGES_REVERSE_H
+
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iter_swap.h>
+#include <__iterator/next.h>
+#include <__iterator/permutable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __reverse {
+struct __fn {
+  template <bidirectional_iterator _Iter, sentinel_for<_Iter> _Sent>
+    requires permutable<_Iter>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter operator()(_Iter __first, _Sent __last) const {
+    if constexpr (random_access_iterator<_Iter>) {
+      if (__first == __last)
+        return __first;
+
+      auto __end = ranges::next(__first, __last);
+      auto __ret = __end;
+
+      while (__first < --__end) {
+        ranges::iter_swap(__first, __end);
+        ++__first;
+      }
+      return __ret;
+    } else {
+      auto __end = ranges::next(__first, __last);
+      auto __ret = __end;
+
+      while (__first != __end) {
+        if (__first == --__end)
+          break;
+
+        ranges::iter_swap(__first, __end);
+        ++__first;
+      }
+      return __ret;
+    }
+  }
+
+  template <bidirectional_range _Range>
+    requires permutable<iterator_t<_Range>>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range> operator()(_Range&& __range) const {
+    return (*this)(ranges::begin(__range), ranges::end(__range));
+  }
+};
+} // namespace __reverse
+
+inline namespace __cpo {
+inline constexpr auto reverse = __reverse::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___ALGORITHM_RANGES_REVERSE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_reverse_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_reverse_copy.h
new file mode 100644
index 000000000000..60043787a717
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_reverse_copy.h
@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_REVERSE_COPY_H
+#define _LIBCPP___ALGORITHM_RANGES_REVERSE_COPY_H
+
+#include <__algorithm/in_out_result.h>
+#include <__algorithm/ranges_copy.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/next.h>
+#include <__iterator/reverse_iterator.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__ranges/subrange.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter, class _OutIter>
+using reverse_copy_result = in_out_result<_InIter, _OutIter>;
+
+namespace __reverse_copy {
+struct __fn {
+  template <bidirectional_iterator _InIter, sentinel_for<_InIter> _Sent, weakly_incrementable _OutIter>
+    requires indirectly_copyable<_InIter, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr reverse_copy_result<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result) const {
+    return (*this)(subrange(std::move(__first), std::move(__last)), std::move(__result));
+  }
+
+  template <bidirectional_range _Range, weakly_incrementable _OutIter>
+    requires indirectly_copyable<iterator_t<_Range>, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr reverse_copy_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __range, _OutIter __result) const {
+    auto __ret = ranges::copy(std::__reverse_range(__range), std::move(__result));
+    return {ranges::next(ranges::begin(__range), ranges::end(__range)), std::move(__ret.out)};
+  }
+};
+} // namespace __reverse_copy
+
+inline namespace __cpo {
+inline constexpr auto reverse_copy = __reverse_copy::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_REVERSE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_rotate.h b/libcxx/include/__cxx03/__algorithm/ranges_rotate.h
new file mode 100644
index 000000000000..8d33a6f0799b
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_rotate.h
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_ROTATE_H
+#define _LIBCPP___ALGORITHM_RANGES_ROTATE_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/ranges_iterator_concept.h>
+#include <__algorithm/rotate.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/permutable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/subrange.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __rotate {
+
+struct __fn {
+  template <class _Iter, class _Sent>
+  _LIBCPP_HIDE_FROM_ABI constexpr static subrange<_Iter> __rotate_fn_impl(_Iter __first, _Iter __middle, _Sent __last) {
+    auto __ret = std::__rotate<_RangeAlgPolicy>(std::move(__first), std::move(__middle), std::move(__last));
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+
+  template <permutable _Iter, sentinel_for<_Iter> _Sent>
+  _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter> operator()(_Iter __first, _Iter __middle, _Sent __last) const {
+    return __rotate_fn_impl(std::move(__first), std::move(__middle), std::move(__last));
+  }
+
+  template <forward_range _Range>
+    requires permutable<iterator_t<_Range>>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, iterator_t<_Range> __middle) const {
+    return __rotate_fn_impl(ranges::begin(__range), std::move(__middle), ranges::end(__range));
+  }
+};
+
+} // namespace __rotate
+
+inline namespace __cpo {
+inline constexpr auto rotate = __rotate::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_ROTATE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_rotate_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_rotate_copy.h
new file mode 100644
index 000000000000..26fe110b5389
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_rotate_copy.h
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_ROTATE_COPY_H
+#define _LIBCPP___ALGORITHM_RANGES_ROTATE_COPY_H
+
+#include <__algorithm/in_out_result.h>
+#include <__algorithm/ranges_copy.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter, class _OutIter>
+using rotate_copy_result = in_out_result<_InIter, _OutIter>;
+
+namespace __rotate_copy {
+struct __fn {
+  template <forward_iterator _InIter, sentinel_for<_InIter> _Sent, weakly_incrementable _OutIter>
+    requires indirectly_copyable<_InIter, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr rotate_copy_result<_InIter, _OutIter>
+  operator()(_InIter __first, _InIter __middle, _Sent __last, _OutIter __result) const {
+    auto __res1 = ranges::copy(__middle, __last, std::move(__result));
+    auto __res2 = ranges::copy(__first, __middle, std::move(__res1.out));
+    return {std::move(__res1.in), std::move(__res2.out)};
+  }
+
+  template <forward_range _Range, weakly_incrementable _OutIter>
+    requires indirectly_copyable<iterator_t<_Range>, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr rotate_copy_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __range, iterator_t<_Range> __middle, _OutIter __result) const {
+    return (*this)(ranges::begin(__range), std::move(__middle), ranges::end(__range), std::move(__result));
+  }
+};
+} // namespace __rotate_copy
+
+inline namespace __cpo {
+inline constexpr auto rotate_copy = __rotate_copy::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_ROTATE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_sample.h b/libcxx/include/__cxx03/__algorithm/ranges_sample.h
new file mode 100644
index 000000000000..e4f60a7b66be
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_sample.h
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_SAMPLE_H
+#define _LIBCPP___ALGORITHM_RANGES_SAMPLE_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/sample.h>
+#include <__algorithm/uniform_random_bit_generator_adaptor.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__random/uniform_random_bit_generator.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __sample {
+
+struct __fn {
+  template <input_iterator _Iter, sentinel_for<_Iter> _Sent, weakly_incrementable _OutIter, class _Gen>
+    requires(forward_iterator<_Iter> || random_access_iterator<_OutIter>) && indirectly_copyable<_Iter, _OutIter> &&
+            uniform_random_bit_generator<remove_reference_t<_Gen>>
+  _LIBCPP_HIDE_FROM_ABI _OutIter
+  operator()(_Iter __first, _Sent __last, _OutIter __out_first, iter_difference_t<_Iter> __n, _Gen&& __gen) const {
+    _ClassicGenAdaptor<_Gen> __adapted_gen(__gen);
+    return std::__sample<_RangeAlgPolicy>(
+        std::move(__first), std::move(__last), std::move(__out_first), __n, __adapted_gen);
+  }
+
+  template <input_range _Range, weakly_incrementable _OutIter, class _Gen>
+    requires(forward_range<_Range> || random_access_iterator<_OutIter>) &&
+            indirectly_copyable<iterator_t<_Range>, _OutIter> && uniform_random_bit_generator<remove_reference_t<_Gen>>
+  _LIBCPP_HIDE_FROM_ABI _OutIter
+  operator()(_Range&& __range, _OutIter __out_first, range_difference_t<_Range> __n, _Gen&& __gen) const {
+    return (*this)(
+        ranges::begin(__range), ranges::end(__range), std::move(__out_first), __n, std::forward<_Gen>(__gen));
+  }
+};
+
+} // namespace __sample
+
+inline namespace __cpo {
+inline constexpr auto sample = __sample::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_SAMPLE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_search.h b/libcxx/include/__cxx03/__algorithm/ranges_search.h
new file mode 100644
index 000000000000..55294c60631b
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_search.h
@@ -0,0 +1,134 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_SEARCH_H
+#define _LIBCPP___ALGORITHM_RANGES_SEARCH_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/search.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/advance.h>
+#include <__iterator/concepts.h>
+#include <__iterator/distance.h>
+#include <__iterator/indirectly_comparable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/size.h>
+#include <__ranges/subrange.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __search {
+struct __fn {
+  template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class _Pred, class _Proj1, class _Proj2>
+  _LIBCPP_HIDE_FROM_ABI static constexpr subrange<_Iter1> __ranges_search_impl(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred& __pred,
+      _Proj1& __proj1,
+      _Proj2& __proj2) {
+    if constexpr (sized_sentinel_for<_Sent2, _Iter2>) {
+      auto __size2 = ranges::distance(__first2, __last2);
+      if (__size2 == 0)
+        return {__first1, __first1};
+
+      if constexpr (sized_sentinel_for<_Sent1, _Iter1>) {
+        auto __size1 = ranges::distance(__first1, __last1);
+        if (__size1 < __size2) {
+          ranges::advance(__first1, __last1);
+          return {__first1, __first1};
+        }
+
+        if constexpr (random_access_iterator<_Iter1> && random_access_iterator<_Iter2>) {
+          auto __ret = std::__search_random_access_impl<_RangeAlgPolicy>(
+              __first1, __last1, __first2, __last2, __pred, __proj1, __proj2, __size1, __size2);
+          return {__ret.first, __ret.second};
+        }
+      }
+    }
+
+    auto __ret =
+        std::__search_forward_impl<_RangeAlgPolicy>(__first1, __last1, __first2, __last2, __pred, __proj1, __proj2);
+    return {__ret.first, __ret.second};
+  }
+
+  template <forward_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            forward_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_comparable<_Iter1, _Iter2, _Pred, _Proj1, _Proj2>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter1> operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred __pred   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    return __ranges_search_impl(__first1, __last1, __first2, __last2, __pred, __proj1, __proj2);
+  }
+
+  template <forward_range _Range1,
+            forward_range _Range2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_comparable<iterator_t<_Range1>, iterator_t<_Range2>, _Pred, _Proj1, _Proj2>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range1> operator()(
+      _Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+    auto __first1 = ranges::begin(__range1);
+    if constexpr (sized_range<_Range2>) {
+      auto __size2 = ranges::size(__range2);
+      if (__size2 == 0)
+        return {__first1, __first1};
+      if constexpr (sized_range<_Range1>) {
+        auto __size1 = ranges::size(__range1);
+        if (__size1 < __size2) {
+          ranges::advance(__first1, ranges::end(__range1));
+          return {__first1, __first1};
+        }
+      }
+    }
+
+    return __ranges_search_impl(
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        __pred,
+        __proj1,
+        __proj2);
+  }
+};
+} // namespace __search
+
+inline namespace __cpo {
+inline constexpr auto search = __search::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___ALGORITHM_RANGES_SEARCH_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_search_n.h b/libcxx/include/__cxx03/__algorithm/ranges_search_n.h
new file mode 100644
index 000000000000..56e12755b9bf
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_search_n.h
@@ -0,0 +1,116 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_SEARCH_N_H
+#define _LIBCPP___ALGORITHM_RANGES_SEARCH_N_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/search_n.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/advance.h>
+#include <__iterator/concepts.h>
+#include <__iterator/distance.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/indirectly_comparable.h>
+#include <__iterator/iterator_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/size.h>
+#include <__ranges/subrange.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __search_n {
+struct __fn {
+  template <class _Iter1, class _Sent1, class _SizeT, class _Type, class _Pred, class _Proj>
+  _LIBCPP_HIDE_FROM_ABI static constexpr subrange<_Iter1> __ranges_search_n_impl(
+      _Iter1 __first, _Sent1 __last, _SizeT __count, const _Type& __value, _Pred& __pred, _Proj& __proj) {
+    if (__count == 0)
+      return {__first, __first};
+
+    if constexpr (sized_sentinel_for<_Sent1, _Iter1>) {
+      auto __size = ranges::distance(__first, __last);
+      if (__size < __count) {
+        ranges::advance(__first, __last);
+        return {__first, __first};
+      }
+
+      if constexpr (random_access_iterator<_Iter1>) {
+        auto __ret = std::__search_n_random_access_impl<_RangeAlgPolicy>(
+            __first, __last, __count, __value, __pred, __proj, __size);
+        return {std::move(__ret.first), std::move(__ret.second)};
+      }
+    }
+
+    auto __ret = std::__search_n_forward_impl<_RangeAlgPolicy>(__first, __last, __count, __value, __pred, __proj);
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Type,
+            class _Pred = ranges::equal_to,
+            class _Proj = identity>
+    requires indirectly_comparable<_Iter, const _Type*, _Pred, _Proj>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
+  operator()(_Iter __first,
+             _Sent __last,
+             iter_difference_t<_Iter> __count,
+             const _Type& __value,
+             _Pred __pred = {},
+             _Proj __proj = _Proj{}) const {
+    return __ranges_search_n_impl(__first, __last, __count, __value, __pred, __proj);
+  }
+
+  template <forward_range _Range, class _Type, class _Pred = ranges::equal_to, class _Proj = identity>
+    requires indirectly_comparable<iterator_t<_Range>, const _Type*, _Pred, _Proj>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range> operator()(
+      _Range&& __range, range_difference_t<_Range> __count, const _Type& __value, _Pred __pred = {}, _Proj __proj = {})
+      const {
+    auto __first = ranges::begin(__range);
+    if (__count <= 0)
+      return {__first, __first};
+    if constexpr (sized_range<_Range>) {
+      auto __size1 = ranges::size(__range);
+      if (__size1 < static_cast<range_size_t<_Range>>(__count)) {
+        ranges::advance(__first, ranges::end(__range));
+        return {__first, __first};
+      }
+    }
+
+    return __ranges_search_n_impl(ranges::begin(__range), ranges::end(__range), __count, __value, __pred, __proj);
+  }
+};
+} // namespace __search_n
+
+inline namespace __cpo {
+inline constexpr auto search_n = __search_n::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_SEARCH_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_set_difference.h b/libcxx/include/__cxx03/__algorithm/ranges_set_difference.h
new file mode 100644
index 000000000000..0841fb4ffd0c
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_set_difference.h
@@ -0,0 +1,109 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_SET_DIFFERENCE_H
+#define _LIBCPP___ALGORITHM_RANGES_SET_DIFFERENCE_H
+
+#include <__algorithm/in_out_result.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/set_difference.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/mergeable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__type_traits/decay.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter, class _OutIter>
+using set_difference_result = in_out_result<_InIter, _OutIter>;
+
+namespace __set_difference {
+
+struct __fn {
+  template <input_iterator _InIter1,
+            sentinel_for<_InIter1> _Sent1,
+            input_iterator _InIter2,
+            sentinel_for<_InIter2> _Sent2,
+            weakly_incrementable _OutIter,
+            class _Comp  = less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires mergeable<_InIter1, _InIter2, _OutIter, _Comp, _Proj1, _Proj2>
+  _LIBCPP_HIDE_FROM_ABI constexpr set_difference_result<_InIter1, _OutIter> operator()(
+      _InIter1 __first1,
+      _Sent1 __last1,
+      _InIter2 __first2,
+      _Sent2 __last2,
+      _OutIter __result,
+      _Comp __comp   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    auto __ret = std::__set_difference<_RangeAlgPolicy>(
+        __first1, __last1, __first2, __last2, __result, ranges::__make_projected_comp(__comp, __proj1, __proj2));
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+
+  template <input_range _Range1,
+            input_range _Range2,
+            weakly_incrementable _OutIter,
+            class _Comp  = less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires mergeable<iterator_t<_Range1>, iterator_t<_Range2>, _OutIter, _Comp, _Proj1, _Proj2>
+  _LIBCPP_HIDE_FROM_ABI constexpr set_difference_result<borrowed_iterator_t<_Range1>, _OutIter>
+  operator()(_Range1&& __range1,
+             _Range2&& __range2,
+             _OutIter __result,
+             _Comp __comp   = {},
+             _Proj1 __proj1 = {},
+             _Proj2 __proj2 = {}) const {
+    auto __ret = std::__set_difference<_RangeAlgPolicy>(
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        __result,
+        ranges::__make_projected_comp(__comp, __proj1, __proj2));
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+};
+
+} // namespace __set_difference
+
+inline namespace __cpo {
+inline constexpr auto set_difference = __set_difference::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_SET_DIFFERENCE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_set_intersection.h b/libcxx/include/__cxx03/__algorithm/ranges_set_intersection.h
new file mode 100644
index 000000000000..9427379745b6
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_set_intersection.h
@@ -0,0 +1,114 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_SET_INTERSECTION_H
+#define _LIBCPP___ALGORITHM_RANGES_SET_INTERSECTION_H
+
+#include <__algorithm/in_in_out_result.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/set_intersection.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/mergeable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter1, class _InIter2, class _OutIter>
+using set_intersection_result = in_in_out_result<_InIter1, _InIter2, _OutIter>;
+
+namespace __set_intersection {
+
+struct __fn {
+  template <input_iterator _InIter1,
+            sentinel_for<_InIter1> _Sent1,
+            input_iterator _InIter2,
+            sentinel_for<_InIter2> _Sent2,
+            weakly_incrementable _OutIter,
+            class _Comp  = less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires mergeable<_InIter1, _InIter2, _OutIter, _Comp, _Proj1, _Proj2>
+  _LIBCPP_HIDE_FROM_ABI constexpr set_intersection_result<_InIter1, _InIter2, _OutIter> operator()(
+      _InIter1 __first1,
+      _Sent1 __last1,
+      _InIter2 __first2,
+      _Sent2 __last2,
+      _OutIter __result,
+      _Comp __comp   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    auto __ret = std::__set_intersection<_RangeAlgPolicy>(
+        std::move(__first1),
+        std::move(__last1),
+        std::move(__first2),
+        std::move(__last2),
+        std::move(__result),
+        ranges::__make_projected_comp(__comp, __proj1, __proj2));
+    return {std::move(__ret.__in1_), std::move(__ret.__in2_), std::move(__ret.__out_)};
+  }
+
+  template <input_range _Range1,
+            input_range _Range2,
+            weakly_incrementable _OutIter,
+            class _Comp  = less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires mergeable<iterator_t<_Range1>, iterator_t<_Range2>, _OutIter, _Comp, _Proj1, _Proj2>
+  _LIBCPP_HIDE_FROM_ABI constexpr set_intersection_result<borrowed_iterator_t<_Range1>,
+                                                          borrowed_iterator_t<_Range2>,
+                                                          _OutIter>
+  operator()(_Range1&& __range1,
+             _Range2&& __range2,
+             _OutIter __result,
+             _Comp __comp   = {},
+             _Proj1 __proj1 = {},
+             _Proj2 __proj2 = {}) const {
+    auto __ret = std::__set_intersection<_RangeAlgPolicy>(
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        std::move(__result),
+        ranges::__make_projected_comp(__comp, __proj1, __proj2));
+    return {std::move(__ret.__in1_), std::move(__ret.__in2_), std::move(__ret.__out_)};
+  }
+};
+
+} // namespace __set_intersection
+
+inline namespace __cpo {
+inline constexpr auto set_intersection = __set_intersection::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_SET_INTERSECTION_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_set_symmetric_difference.h b/libcxx/include/__cxx03/__algorithm/ranges_set_symmetric_difference.h
new file mode 100644
index 000000000000..995eb0999d94
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_set_symmetric_difference.h
@@ -0,0 +1,114 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_SET_SYMMETRIC_DIFFERENCE_H
+#define _LIBCPP___ALGORITHM_RANGES_SET_SYMMETRIC_DIFFERENCE_H
+
+#include <__algorithm/in_in_out_result.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/set_symmetric_difference.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/mergeable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter1, class _InIter2, class _OutIter>
+using set_symmetric_difference_result = in_in_out_result<_InIter1, _InIter2, _OutIter>;
+
+namespace __set_symmetric_difference {
+
+struct __fn {
+  template <input_iterator _InIter1,
+            sentinel_for<_InIter1> _Sent1,
+            input_iterator _InIter2,
+            sentinel_for<_InIter2> _Sent2,
+            weakly_incrementable _OutIter,
+            class _Comp  = ranges::less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires mergeable<_InIter1, _InIter2, _OutIter, _Comp, _Proj1, _Proj2>
+  _LIBCPP_HIDE_FROM_ABI constexpr set_symmetric_difference_result<_InIter1, _InIter2, _OutIter> operator()(
+      _InIter1 __first1,
+      _Sent1 __last1,
+      _InIter2 __first2,
+      _Sent2 __last2,
+      _OutIter __result,
+      _Comp __comp   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    auto __ret = std::__set_symmetric_difference<_RangeAlgPolicy>(
+        std::move(__first1),
+        std::move(__last1),
+        std::move(__first2),
+        std::move(__last2),
+        std::move(__result),
+        ranges::__make_projected_comp(__comp, __proj1, __proj2));
+    return {std::move(__ret.__in1_), std::move(__ret.__in2_), std::move(__ret.__out_)};
+  }
+
+  template <input_range _Range1,
+            input_range _Range2,
+            weakly_incrementable _OutIter,
+            class _Comp  = ranges::less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires mergeable<iterator_t<_Range1>, iterator_t<_Range2>, _OutIter, _Comp, _Proj1, _Proj2>
+  _LIBCPP_HIDE_FROM_ABI constexpr set_symmetric_difference_result<borrowed_iterator_t<_Range1>,
+                                                                  borrowed_iterator_t<_Range2>,
+                                                                  _OutIter>
+  operator()(_Range1&& __range1,
+             _Range2&& __range2,
+             _OutIter __result,
+             _Comp __comp   = {},
+             _Proj1 __proj1 = {},
+             _Proj2 __proj2 = {}) const {
+    auto __ret = std::__set_symmetric_difference<_RangeAlgPolicy>(
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        std::move(__result),
+        ranges::__make_projected_comp(__comp, __proj1, __proj2));
+    return {std::move(__ret.__in1_), std::move(__ret.__in2_), std::move(__ret.__out_)};
+  }
+};
+
+} // namespace __set_symmetric_difference
+
+inline namespace __cpo {
+inline constexpr auto set_symmetric_difference = __set_symmetric_difference::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_SET_SYMMETRIC_DIFFERENCE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_set_union.h b/libcxx/include/__cxx03/__algorithm/ranges_set_union.h
new file mode 100644
index 000000000000..e870e390cc66
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_set_union.h
@@ -0,0 +1,115 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_SET_UNION_H
+#define _LIBCPP___ALGORITHM_RANGES_SET_UNION_H
+
+#include <__algorithm/in_in_out_result.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/set_union.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/mergeable.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter1, class _InIter2, class _OutIter>
+using set_union_result = in_in_out_result<_InIter1, _InIter2, _OutIter>;
+
+namespace __set_union {
+
+struct __fn {
+  template <input_iterator _InIter1,
+            sentinel_for<_InIter1> _Sent1,
+            input_iterator _InIter2,
+            sentinel_for<_InIter2> _Sent2,
+            weakly_incrementable _OutIter,
+            class _Comp  = ranges::less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires mergeable<_InIter1, _InIter2, _OutIter, _Comp, _Proj1, _Proj2>
+  _LIBCPP_HIDE_FROM_ABI constexpr set_union_result<_InIter1, _InIter2, _OutIter> operator()(
+      _InIter1 __first1,
+      _Sent1 __last1,
+      _InIter2 __first2,
+      _Sent2 __last2,
+      _OutIter __result,
+      _Comp __comp   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    auto __ret = std::__set_union<_RangeAlgPolicy>(
+        std::move(__first1),
+        std::move(__last1),
+        std::move(__first2),
+        std::move(__last2),
+        std::move(__result),
+        ranges::__make_projected_comp(__comp, __proj1, __proj2));
+    return {std::move(__ret.__in1_), std::move(__ret.__in2_), std::move(__ret.__out_)};
+  }
+
+  template <input_range _Range1,
+            input_range _Range2,
+            weakly_incrementable _OutIter,
+            class _Comp  = ranges::less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires mergeable<iterator_t<_Range1>, iterator_t<_Range2>, _OutIter, _Comp, _Proj1, _Proj2>
+  _LIBCPP_HIDE_FROM_ABI constexpr set_union_result<borrowed_iterator_t<_Range1>, borrowed_iterator_t<_Range2>, _OutIter>
+  operator()(_Range1&& __range1,
+             _Range2&& __range2,
+             _OutIter __result,
+             _Comp __comp   = {},
+             _Proj1 __proj1 = {},
+             _Proj2 __proj2 = {}) const {
+    auto __ret = std::__set_union<_RangeAlgPolicy>(
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        std::move(__result),
+        ranges::__make_projected_comp(__comp, __proj1, __proj2));
+    return {std::move(__ret.__in1_), std::move(__ret.__in2_), std::move(__ret.__out_)};
+  }
+};
+
+} // namespace __set_union
+
+inline namespace __cpo {
+inline constexpr auto set_union = __set_union::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_SET_UNION_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_shuffle.h b/libcxx/include/__cxx03/__algorithm/ranges_shuffle.h
new file mode 100644
index 000000000000..ab98ea22caab
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_shuffle.h
@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_SHUFFLE_H
+#define _LIBCPP___ALGORITHM_RANGES_SHUFFLE_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/shuffle.h>
+#include <__algorithm/uniform_random_bit_generator_adaptor.h>
+#include <__config>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/permutable.h>
+#include <__random/uniform_random_bit_generator.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __shuffle {
+
+struct __fn {
+  template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Gen>
+    requires permutable<_Iter> && uniform_random_bit_generator<remove_reference_t<_Gen>>
+  _LIBCPP_HIDE_FROM_ABI _Iter operator()(_Iter __first, _Sent __last, _Gen&& __gen) const {
+    _ClassicGenAdaptor<_Gen> __adapted_gen(__gen);
+    return std::__shuffle<_RangeAlgPolicy>(std::move(__first), std::move(__last), __adapted_gen);
+  }
+
+  template <random_access_range _Range, class _Gen>
+    requires permutable<iterator_t<_Range>> && uniform_random_bit_generator<remove_reference_t<_Gen>>
+  _LIBCPP_HIDE_FROM_ABI borrowed_iterator_t<_Range> operator()(_Range&& __range, _Gen&& __gen) const {
+    return (*this)(ranges::begin(__range), ranges::end(__range), std::forward<_Gen>(__gen));
+  }
+};
+
+} // namespace __shuffle
+
+inline namespace __cpo {
+inline constexpr auto shuffle = __shuffle::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_SHUFFLE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_sort.h b/libcxx/include/__cxx03/__algorithm/ranges_sort.h
new file mode 100644
index 000000000000..0296c146b3ed
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_sort.h
@@ -0,0 +1,84 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_SORT_H
+#define _LIBCPP___ALGORITHM_RANGES_SORT_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/sort.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/projected.h>
+#include <__iterator/sortable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __sort {
+
+struct __fn {
+  template <class _Iter, class _Sent, class _Comp, class _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __sort_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+    auto __last_iter = ranges::next(__first, __last);
+
+    auto&& __projected_comp = std::__make_projected(__comp, __proj);
+    std::__sort_impl<_RangeAlgPolicy>(std::move(__first), __last_iter, __projected_comp);
+
+    return __last_iter;
+  }
+
+  template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<_Iter, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __sort_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
+  }
+
+  template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<iterator_t<_Range>, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __sort_fn_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
+  }
+};
+
+} // namespace __sort
+
+inline namespace __cpo {
+inline constexpr auto sort = __sort::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_SORT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_sort_heap.h b/libcxx/include/__cxx03/__algorithm/ranges_sort_heap.h
new file mode 100644
index 000000000000..bab30df1708c
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_sort_heap.h
@@ -0,0 +1,85 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_SORT_HEAP_H
+#define _LIBCPP___ALGORITHM_RANGES_SORT_HEAP_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/sort_heap.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/projected.h>
+#include <__iterator/sortable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __sort_heap {
+
+struct __fn {
+  template <class _Iter, class _Sent, class _Comp, class _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __sort_heap_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+    auto __last_iter = ranges::next(__first, __last);
+
+    auto&& __projected_comp = std::__make_projected(__comp, __proj);
+    std::__sort_heap<_RangeAlgPolicy>(std::move(__first), __last_iter, __projected_comp);
+
+    return __last_iter;
+  }
+
+  template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<_Iter, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __sort_heap_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
+  }
+
+  template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<iterator_t<_Range>, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __sort_heap_fn_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
+  }
+};
+
+} // namespace __sort_heap
+
+inline namespace __cpo {
+inline constexpr auto sort_heap = __sort_heap::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_SORT_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_stable_partition.h b/libcxx/include/__cxx03/__algorithm/ranges_stable_partition.h
new file mode 100644
index 000000000000..f34027ff772c
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_stable_partition.h
@@ -0,0 +1,92 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_STABLE_PARTITION_H
+#define _LIBCPP___ALGORITHM_RANGES_STABLE_PARTITION_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/ranges_iterator_concept.h>
+#include <__algorithm/stable_partition.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/permutable.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__ranges/subrange.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __stable_partition {
+
+struct __fn {
+  template <class _Iter, class _Sent, class _Proj, class _Pred>
+  _LIBCPP_HIDE_FROM_ABI static subrange<__remove_cvref_t<_Iter>>
+  __stable_partition_fn_impl(_Iter&& __first, _Sent&& __last, _Pred&& __pred, _Proj&& __proj) {
+    auto __last_iter = ranges::next(__first, __last);
+
+    auto&& __projected_pred = std::__make_projected(__pred, __proj);
+    auto __result           = std::__stable_partition<_RangeAlgPolicy>(
+        std::move(__first), __last_iter, __projected_pred, __iterator_concept<_Iter>());
+
+    return {std::move(__result), std::move(__last_iter)};
+  }
+
+  template <bidirectional_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
+    requires permutable<_Iter>
+  _LIBCPP_HIDE_FROM_ABI subrange<_Iter> operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
+    return __stable_partition_fn_impl(__first, __last, __pred, __proj);
+  }
+
+  template <bidirectional_range _Range,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+    requires permutable<iterator_t<_Range>>
+  _LIBCPP_HIDE_FROM_ABI borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+    return __stable_partition_fn_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
+  }
+};
+
+} // namespace __stable_partition
+
+inline namespace __cpo {
+inline constexpr auto stable_partition = __stable_partition::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_STABLE_PARTITION_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_stable_sort.h b/libcxx/include/__cxx03/__algorithm/ranges_stable_sort.h
new file mode 100644
index 000000000000..93909e253cc0
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_stable_sort.h
@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_STABLE_SORT_H
+#define _LIBCPP___ALGORITHM_RANGES_STABLE_SORT_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/stable_sort.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/projected.h>
+#include <__iterator/sortable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __stable_sort {
+
+struct __fn {
+  template <class _Iter, class _Sent, class _Comp, class _Proj>
+  _LIBCPP_HIDE_FROM_ABI static _Iter __stable_sort_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+    auto __last_iter = ranges::next(__first, __last);
+
+    auto&& __projected_comp = std::__make_projected(__comp, __proj);
+    std::__stable_sort_impl<_RangeAlgPolicy>(std::move(__first), __last_iter, __projected_comp);
+
+    return __last_iter;
+  }
+
+  template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<_Iter, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI _Iter operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __stable_sort_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
+  }
+
+  template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<iterator_t<_Range>, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __stable_sort_fn_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
+  }
+};
+
+} // namespace __stable_sort
+
+inline namespace __cpo {
+inline constexpr auto stable_sort = __stable_sort::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_STABLE_SORT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_starts_with.h b/libcxx/include/__cxx03/__algorithm/ranges_starts_with.h
new file mode 100644
index 000000000000..17084e4f2433
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_starts_with.h
@@ -0,0 +1,95 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_STARTS_WITH_H
+#define _LIBCPP___ALGORITHM_RANGES_STARTS_WITH_H
+
+#include <__algorithm/in_in_result.h>
+#include <__algorithm/ranges_mismatch.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/indirectly_comparable.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __starts_with {
+struct __fn {
+  template <input_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            input_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_comparable<_Iter1, _Iter2, _Pred, _Proj1, _Proj2>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr bool operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred __pred   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) {
+    return __mismatch::__fn::__go(
+               std::move(__first1),
+               std::move(__last1),
+               std::move(__first2),
+               std::move(__last2),
+               __pred,
+               __proj1,
+               __proj2)
+               .in2 == __last2;
+  }
+
+  template <input_range _Range1,
+            input_range _Range2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_comparable<iterator_t<_Range1>, iterator_t<_Range2>, _Pred, _Proj1, _Proj2>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr bool
+  operator()(_Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) {
+    return __mismatch::__fn::__go(
+               ranges::begin(__range1),
+               ranges::end(__range1),
+               ranges::begin(__range2),
+               ranges::end(__range2),
+               __pred,
+               __proj1,
+               __proj2)
+               .in2 == ranges::end(__range2);
+  }
+};
+} // namespace __starts_with
+inline namespace __cpo {
+inline constexpr auto starts_with = __starts_with::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_STARTS_WITH_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_swap_ranges.h b/libcxx/include/__cxx03/__algorithm/ranges_swap_ranges.h
new file mode 100644
index 000000000000..b6d9f618395a
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_swap_ranges.h
@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_SWAP_RANGES_H
+#define _LIBCPP___ALGORITHM_RANGES_SWAP_RANGES_H
+
+#include <__algorithm/in_in_result.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/swap_ranges.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iter_swap.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _I1, class _I2>
+using swap_ranges_result = in_in_result<_I1, _I2>;
+
+namespace __swap_ranges {
+struct __fn {
+  template <input_iterator _I1, sentinel_for<_I1> _S1, input_iterator _I2, sentinel_for<_I2> _S2>
+    requires indirectly_swappable<_I1, _I2>
+  _LIBCPP_HIDE_FROM_ABI constexpr swap_ranges_result<_I1, _I2>
+  operator()(_I1 __first1, _S1 __last1, _I2 __first2, _S2 __last2) const {
+    auto __ret = std::__swap_ranges<_RangeAlgPolicy>(
+        std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2));
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+
+  template <input_range _R1, input_range _R2>
+    requires indirectly_swappable<iterator_t<_R1>, iterator_t<_R2>>
+  _LIBCPP_HIDE_FROM_ABI constexpr swap_ranges_result<borrowed_iterator_t<_R1>, borrowed_iterator_t<_R2>>
+  operator()(_R1&& __r1, _R2&& __r2) const {
+    return operator()(ranges::begin(__r1), ranges::end(__r1), ranges::begin(__r2), ranges::end(__r2));
+  }
+};
+} // namespace __swap_ranges
+
+inline namespace __cpo {
+inline constexpr auto swap_ranges = __swap_ranges::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_SWAP_RANGES_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_transform.h b/libcxx/include/__cxx03/__algorithm/ranges_transform.h
new file mode 100644
index 000000000000..7850ec4f8465
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_transform.h
@@ -0,0 +1,177 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_TRANSFORM_H
+#define _LIBCPP___ALGORITHM_RANGES_TRANSFORM_H
+
+#include <__algorithm/in_in_out_result.h>
+#include <__algorithm/in_out_result.h>
+#include <__concepts/constructible.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _Ip, class _Op>
+using unary_transform_result = in_out_result<_Ip, _Op>;
+
+template <class _I1, class _I2, class _O1>
+using binary_transform_result = in_in_out_result<_I1, _I2, _O1>;
+
+namespace __transform {
+struct __fn {
+private:
+  template <class _InIter, class _Sent, class _OutIter, class _Func, class _Proj>
+  _LIBCPP_HIDE_FROM_ABI static constexpr unary_transform_result<_InIter, _OutIter>
+  __unary(_InIter __first, _Sent __last, _OutIter __result, _Func& __operation, _Proj& __projection) {
+    while (__first != __last) {
+      *__result = std::invoke(__operation, std::invoke(__projection, *__first));
+      ++__first;
+      ++__result;
+    }
+
+    return {std::move(__first), std::move(__result)};
+  }
+
+  template <class _InIter1,
+            class _Sent1,
+            class _InIter2,
+            class _Sent2,
+            class _OutIter,
+            class _Func,
+            class _Proj1,
+            class _Proj2>
+  _LIBCPP_HIDE_FROM_ABI static constexpr binary_transform_result<_InIter1, _InIter2, _OutIter>
+  __binary(_InIter1 __first1,
+           _Sent1 __last1,
+           _InIter2 __first2,
+           _Sent2 __last2,
+           _OutIter __result,
+           _Func& __binary_operation,
+           _Proj1& __projection1,
+           _Proj2& __projection2) {
+    while (__first1 != __last1 && __first2 != __last2) {
+      *__result =
+          std::invoke(__binary_operation, std::invoke(__projection1, *__first1), std::invoke(__projection2, *__first2));
+      ++__first1;
+      ++__first2;
+      ++__result;
+    }
+    return {std::move(__first1), std::move(__first2), std::move(__result)};
+  }
+
+public:
+  template <input_iterator _InIter,
+            sentinel_for<_InIter> _Sent,
+            weakly_incrementable _OutIter,
+            copy_constructible _Func,
+            class _Proj = identity>
+    requires indirectly_writable<_OutIter, indirect_result_t<_Func&, projected<_InIter, _Proj>>>
+  _LIBCPP_HIDE_FROM_ABI constexpr unary_transform_result<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result, _Func __operation, _Proj __proj = {}) const {
+    return __unary(std::move(__first), std::move(__last), std::move(__result), __operation, __proj);
+  }
+
+  template <input_range _Range, weakly_incrementable _OutIter, copy_constructible _Func, class _Proj = identity>
+    requires indirectly_writable<_OutIter, indirect_result_t<_Func, projected<iterator_t<_Range>, _Proj>>>
+  _LIBCPP_HIDE_FROM_ABI constexpr unary_transform_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __range, _OutIter __result, _Func __operation, _Proj __projection = {}) const {
+    return __unary(ranges::begin(__range), ranges::end(__range), std::move(__result), __operation, __projection);
+  }
+
+  template <input_iterator _InIter1,
+            sentinel_for<_InIter1> _Sent1,
+            input_iterator _InIter2,
+            sentinel_for<_InIter2> _Sent2,
+            weakly_incrementable _OutIter,
+            copy_constructible _Func,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_writable<_OutIter,
+                                 indirect_result_t<_Func&, projected<_InIter1, _Proj1>, projected<_InIter2, _Proj2>>>
+  _LIBCPP_HIDE_FROM_ABI constexpr binary_transform_result<_InIter1, _InIter2, _OutIter> operator()(
+      _InIter1 __first1,
+      _Sent1 __last1,
+      _InIter2 __first2,
+      _Sent2 __last2,
+      _OutIter __result,
+      _Func __binary_operation,
+      _Proj1 __projection1 = {},
+      _Proj2 __projection2 = {}) const {
+    return __binary(
+        std::move(__first1),
+        std::move(__last1),
+        std::move(__first2),
+        std::move(__last2),
+        std::move(__result),
+        __binary_operation,
+        __projection1,
+        __projection2);
+  }
+
+  template <input_range _Range1,
+            input_range _Range2,
+            weakly_incrementable _OutIter,
+            copy_constructible _Func,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_writable<
+        _OutIter,
+        indirect_result_t<_Func&, projected<iterator_t<_Range1>, _Proj1>, projected<iterator_t<_Range2>, _Proj2>>>
+  _LIBCPP_HIDE_FROM_ABI constexpr binary_transform_result<borrowed_iterator_t<_Range1>,
+                                                          borrowed_iterator_t<_Range2>,
+                                                          _OutIter>
+  operator()(_Range1&& __range1,
+             _Range2&& __range2,
+             _OutIter __result,
+             _Func __binary_operation,
+             _Proj1 __projection1 = {},
+             _Proj2 __projection2 = {}) const {
+    return __binary(
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        std::move(__result),
+        __binary_operation,
+        __projection1,
+        __projection2);
+  }
+};
+} // namespace __transform
+
+inline namespace __cpo {
+inline constexpr auto transform = __transform::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_TRANSFORM_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_unique.h b/libcxx/include/__cxx03/__algorithm/ranges_unique.h
new file mode 100644
index 000000000000..7a9b78432187
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_unique.h
@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_UNIQUE_H
+#define _LIBCPP___ALGORITHM_RANGES_UNIQUE_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/unique.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/permutable.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__ranges/subrange.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __unique {
+
+struct __fn {
+  template <permutable _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj                                                  = identity,
+            indirect_equivalence_relation<projected<_Iter, _Proj>> _Comp = ranges::equal_to>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __ret =
+        std::__unique<_RangeAlgPolicy>(std::move(__first), std::move(__last), std::__make_projected(__comp, __proj));
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+
+  template <forward_range _Range,
+            class _Proj                                                               = identity,
+            indirect_equivalence_relation<projected<iterator_t<_Range>, _Proj>> _Comp = ranges::equal_to>
+    requires permutable<iterator_t<_Range>>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __ret = std::__unique<_RangeAlgPolicy>(
+        ranges::begin(__range), ranges::end(__range), std::__make_projected(__comp, __proj));
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+};
+
+} // namespace __unique
+
+inline namespace __cpo {
+inline constexpr auto unique = __unique::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_UNIQUE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_unique_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_unique_copy.h
new file mode 100644
index 000000000000..61133885ae80
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_unique_copy.h
@@ -0,0 +1,120 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_UNIQUE_COPY_H
+#define _LIBCPP___ALGORITHM_RANGES_UNIQUE_COPY_H
+
+#include <__algorithm/in_out_result.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/make_projected.h>
+#include <__algorithm/unique_copy.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _InIter, class _OutIter>
+using unique_copy_result = in_out_result<_InIter, _OutIter>;
+
+namespace __unique_copy {
+
+template <class _InIter, class _OutIter>
+concept __can_reread_from_output = (input_iterator<_OutIter> && same_as<iter_value_t<_InIter>, iter_value_t<_OutIter>>);
+
+struct __fn {
+  template <class _InIter, class _OutIter>
+  static consteval auto __get_algo_tag() {
+    if constexpr (forward_iterator<_InIter>) {
+      return __unique_copy_tags::__reread_from_input_tag{};
+    } else if constexpr (__can_reread_from_output<_InIter, _OutIter>) {
+      return __unique_copy_tags::__reread_from_output_tag{};
+    } else if constexpr (indirectly_copyable_storable<_InIter, _OutIter>) {
+      return __unique_copy_tags::__read_from_tmp_value_tag{};
+    }
+  }
+
+  template <class _InIter, class _OutIter>
+  using __algo_tag_t = decltype(__get_algo_tag<_InIter, _OutIter>());
+
+  template <input_iterator _InIter,
+            sentinel_for<_InIter> _Sent,
+            weakly_incrementable _OutIter,
+            class _Proj                                                    = identity,
+            indirect_equivalence_relation<projected<_InIter, _Proj>> _Comp = ranges::equal_to>
+    requires indirectly_copyable<_InIter, _OutIter> &&
+             (forward_iterator<_InIter> ||
+              (input_iterator<_OutIter> && same_as<iter_value_t<_InIter>, iter_value_t<_OutIter>>) ||
+              indirectly_copyable_storable<_InIter, _OutIter>)
+  _LIBCPP_HIDE_FROM_ABI constexpr unique_copy_result<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __ret = std::__unique_copy<_RangeAlgPolicy>(
+        std::move(__first),
+        std::move(__last),
+        std::move(__result),
+        std::__make_projected(__comp, __proj),
+        __algo_tag_t<_InIter, _OutIter>());
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+
+  template <input_range _Range,
+            weakly_incrementable _OutIter,
+            class _Proj                                                               = identity,
+            indirect_equivalence_relation<projected<iterator_t<_Range>, _Proj>> _Comp = ranges::equal_to>
+    requires indirectly_copyable<iterator_t<_Range>, _OutIter> &&
+             (forward_iterator<iterator_t<_Range>> ||
+              (input_iterator<_OutIter> && same_as<range_value_t<_Range>, iter_value_t<_OutIter>>) ||
+              indirectly_copyable_storable<iterator_t<_Range>, _OutIter>)
+  _LIBCPP_HIDE_FROM_ABI constexpr unique_copy_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __range, _OutIter __result, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __ret = std::__unique_copy<_RangeAlgPolicy>(
+        ranges::begin(__range),
+        ranges::end(__range),
+        std::move(__result),
+        std::__make_projected(__comp, __proj),
+        __algo_tag_t<iterator_t<_Range>, _OutIter>());
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+};
+
+} // namespace __unique_copy
+
+inline namespace __cpo {
+inline constexpr auto unique_copy = __unique_copy::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_RANGES_UNIQUE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_upper_bound.h b/libcxx/include/__cxx03/__algorithm/ranges_upper_bound.h
new file mode 100644
index 000000000000..fa6fa7f70ed5
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/ranges_upper_bound.h
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_UPPER_BOUND_H
+#define _LIBCPP___ALGORITHM_RANGES_UPPER_BOUND_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/lower_bound.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+namespace __upper_bound {
+struct __fn {
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Type,
+            class _Proj                                                             = identity,
+            indirect_strict_weak_order<const _Type*, projected<_Iter, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __comp_lhs_rhs_swapped = [&](const auto& __lhs, const auto& __rhs) -> bool {
+      return !std::invoke(__comp, __rhs, __lhs);
+    };
+
+    return std::__lower_bound<_RangeAlgPolicy>(__first, __last, __value, __comp_lhs_rhs_swapped, __proj);
+  }
+
+  template <forward_range _Range,
+            class _Type,
+            class _Proj                                                                          = identity,
+            indirect_strict_weak_order<const _Type*, projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __comp_lhs_rhs_swapped = [&](const auto& __lhs, const auto& __rhs) -> bool {
+      return !std::invoke(__comp, __rhs, __lhs);
+    };
+
+    return std::__lower_bound<_RangeAlgPolicy>(
+        ranges::begin(__r), ranges::end(__r), __value, __comp_lhs_rhs_swapped, __proj);
+  }
+};
+} // namespace __upper_bound
+
+inline namespace __cpo {
+inline constexpr auto upper_bound = __upper_bound::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___ALGORITHM_RANGES_UPPER_BOUND_H
diff --git a/libcxx/include/__cxx03/__algorithm/remove.h b/libcxx/include/__cxx03/__algorithm/remove.h
new file mode 100644
index 000000000000..fd01c23cb670
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/remove.h
@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_REMOVE_H
+#define _LIBCPP___ALGORITHM_REMOVE_H
+
+#include <__algorithm/find.h>
+#include <__algorithm/find_if.h>
+#include <__config>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ForwardIterator, class _Tp>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+remove(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
+  __first = std::find(__first, __last, __value);
+  if (__first != __last) {
+    _ForwardIterator __i = __first;
+    while (++__i != __last) {
+      if (!(*__i == __value)) {
+        *__first = std::move(*__i);
+        ++__first;
+      }
+    }
+  }
+  return __first;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_REMOVE_H
diff --git a/libcxx/include/__cxx03/__algorithm/remove_copy.h b/libcxx/include/__cxx03/__algorithm/remove_copy.h
new file mode 100644
index 000000000000..7be4c166ce3d
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/remove_copy.h
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_REMOVE_COPY_H
+#define _LIBCPP___ALGORITHM_REMOVE_COPY_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _OutputIterator, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+remove_copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result, const _Tp& __value) {
+  for (; __first != __last; ++__first) {
+    if (!(*__first == __value)) {
+      *__result = *__first;
+      ++__result;
+    }
+  }
+  return __result;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_REMOVE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/remove_copy_if.h b/libcxx/include/__cxx03/__algorithm/remove_copy_if.h
new file mode 100644
index 000000000000..dcafed169157
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/remove_copy_if.h
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_REMOVE_COPY_IF_H
+#define _LIBCPP___ALGORITHM_REMOVE_COPY_IF_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _OutputIterator, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+remove_copy_if(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _Predicate __pred) {
+  for (; __first != __last; ++__first) {
+    if (!__pred(*__first)) {
+      *__result = *__first;
+      ++__result;
+    }
+  }
+  return __result;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_REMOVE_COPY_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/remove_if.h b/libcxx/include/__cxx03/__algorithm/remove_if.h
new file mode 100644
index 000000000000..b14f3c0efa7e
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/remove_if.h
@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_REMOVE_IF_H
+#define _LIBCPP___ALGORITHM_REMOVE_IF_H
+
+#include <__algorithm/find_if.h>
+#include <__config>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ForwardIterator, class _Predicate>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+remove_if(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
+  __first = std::find_if<_ForwardIterator, _Predicate&>(__first, __last, __pred);
+  if (__first != __last) {
+    _ForwardIterator __i = __first;
+    while (++__i != __last) {
+      if (!__pred(*__i)) {
+        *__first = std::move(*__i);
+        ++__first;
+      }
+    }
+  }
+  return __first;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_REMOVE_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/replace.h b/libcxx/include/__cxx03/__algorithm/replace.h
new file mode 100644
index 000000000000..8057c78686e1
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/replace.h
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_REPLACE_H
+#define _LIBCPP___ALGORITHM_REPLACE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ForwardIterator, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+replace(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __old_value, const _Tp& __new_value) {
+  for (; __first != __last; ++__first)
+    if (*__first == __old_value)
+      *__first = __new_value;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_REPLACE_H
diff --git a/libcxx/include/__cxx03/__algorithm/replace_copy.h b/libcxx/include/__cxx03/__algorithm/replace_copy.h
new file mode 100644
index 000000000000..9a2258d9f58e
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/replace_copy.h
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_REPLACE_COPY_H
+#define _LIBCPP___ALGORITHM_REPLACE_COPY_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _OutputIterator, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator replace_copy(
+    _InputIterator __first,
+    _InputIterator __last,
+    _OutputIterator __result,
+    const _Tp& __old_value,
+    const _Tp& __new_value) {
+  for (; __first != __last; ++__first, (void)++__result)
+    if (*__first == __old_value)
+      *__result = __new_value;
+    else
+      *__result = *__first;
+  return __result;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_REPLACE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/replace_copy_if.h b/libcxx/include/__cxx03/__algorithm/replace_copy_if.h
new file mode 100644
index 000000000000..c2ed30f08d59
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/replace_copy_if.h
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_REPLACE_COPY_IF_H
+#define _LIBCPP___ALGORITHM_REPLACE_COPY_IF_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _OutputIterator, class _Predicate, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator replace_copy_if(
+    _InputIterator __first,
+    _InputIterator __last,
+    _OutputIterator __result,
+    _Predicate __pred,
+    const _Tp& __new_value) {
+  for (; __first != __last; ++__first, (void)++__result)
+    if (__pred(*__first))
+      *__result = __new_value;
+    else
+      *__result = *__first;
+  return __result;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_REPLACE_COPY_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/replace_if.h b/libcxx/include/__cxx03/__algorithm/replace_if.h
new file mode 100644
index 000000000000..78487e3deed7
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/replace_if.h
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_REPLACE_IF_H
+#define _LIBCPP___ALGORITHM_REPLACE_IF_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ForwardIterator, class _Predicate, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+replace_if(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred, const _Tp& __new_value) {
+  for (; __first != __last; ++__first)
+    if (__pred(*__first))
+      *__first = __new_value;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_REPLACE_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/reverse.h b/libcxx/include/__cxx03/__algorithm/reverse.h
new file mode 100644
index 000000000000..4167c9116d96
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/reverse.h
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_REVERSE_H
+#define _LIBCPP___ALGORITHM_REVERSE_H
+
+#include <__algorithm/iter_swap.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _BidirectionalIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+__reverse_impl(_BidirectionalIterator __first, _BidirectionalIterator __last, bidirectional_iterator_tag) {
+  while (__first != __last) {
+    if (__first == --__last)
+      break;
+    _IterOps<_AlgPolicy>::iter_swap(__first, __last);
+    ++__first;
+  }
+}
+
+template <class _AlgPolicy, class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+__reverse_impl(_RandomAccessIterator __first, _RandomAccessIterator __last, random_access_iterator_tag) {
+  if (__first != __last)
+    for (; __first < --__last; ++__first)
+      _IterOps<_AlgPolicy>::iter_swap(__first, __last);
+}
+
+template <class _AlgPolicy, class _BidirectionalIterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __reverse(_BidirectionalIterator __first, _Sentinel __last) {
+  using _IterCategory = typename _IterOps<_AlgPolicy>::template __iterator_category<_BidirectionalIterator>;
+  std::__reverse_impl<_AlgPolicy>(std::move(__first), std::move(__last), _IterCategory());
+}
+
+template <class _BidirectionalIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+reverse(_BidirectionalIterator __first, _BidirectionalIterator __last) {
+  std::__reverse<_ClassicAlgPolicy>(std::move(__first), std::move(__last));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_REVERSE_H
diff --git a/libcxx/include/__cxx03/__algorithm/reverse_copy.h b/libcxx/include/__cxx03/__algorithm/reverse_copy.h
new file mode 100644
index 000000000000..0fcecc392326
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/reverse_copy.h
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_REVERSE_COPY_H
+#define _LIBCPP___ALGORITHM_REVERSE_COPY_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _BidirectionalIterator, class _OutputIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+reverse_copy(_BidirectionalIterator __first, _BidirectionalIterator __last, _OutputIterator __result) {
+  for (; __first != __last; ++__result)
+    *__result = *--__last;
+  return __result;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_REVERSE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/rotate.h b/libcxx/include/__cxx03/__algorithm/rotate.h
new file mode 100644
index 000000000000..df4ca95aac95
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/rotate.h
@@ -0,0 +1,198 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_ROTATE_H
+#define _LIBCPP___ALGORITHM_ROTATE_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/move.h>
+#include <__algorithm/move_backward.h>
+#include <__algorithm/swap_ranges.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/is_trivially_assignable.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _ForwardIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator
+__rotate_left(_ForwardIterator __first, _ForwardIterator __last) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type value_type;
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  value_type __tmp       = _Ops::__iter_move(__first);
+  _ForwardIterator __lm1 = std::__move<_AlgPolicy>(_Ops::next(__first), __last, __first).second;
+  *__lm1                 = std::move(__tmp);
+  return __lm1;
+}
+
+template <class _AlgPolicy, class _BidirectionalIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _BidirectionalIterator
+__rotate_right(_BidirectionalIterator __first, _BidirectionalIterator __last) {
+  typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type;
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  _BidirectionalIterator __lm1 = _Ops::prev(__last);
+  value_type __tmp             = _Ops::__iter_move(__lm1);
+  _BidirectionalIterator __fp1 = std::__move_backward<_AlgPolicy>(__first, __lm1, std::move(__last)).second;
+  *__first                     = std::move(__tmp);
+  return __fp1;
+}
+
+template <class _AlgPolicy, class _ForwardIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _ForwardIterator
+__rotate_forward(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last) {
+  _ForwardIterator __i = __middle;
+  while (true) {
+    _IterOps<_AlgPolicy>::iter_swap(__first, __i);
+    ++__first;
+    if (++__i == __last)
+      break;
+    if (__first == __middle)
+      __middle = __i;
+  }
+  _ForwardIterator __r = __first;
+  if (__first != __middle) {
+    __i = __middle;
+    while (true) {
+      _IterOps<_AlgPolicy>::iter_swap(__first, __i);
+      ++__first;
+      if (++__i == __last) {
+        if (__first == __middle)
+          break;
+        __i = __middle;
+      } else if (__first == __middle)
+        __middle = __i;
+    }
+  }
+  return __r;
+}
+
+template <typename _Integral>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _Integral __algo_gcd(_Integral __x, _Integral __y) {
+  do {
+    _Integral __t = __x % __y;
+    __x           = __y;
+    __y           = __t;
+  } while (__y);
+  return __x;
+}
+
+template <class _AlgPolicy, typename _RandomAccessIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _RandomAccessIterator
+__rotate_gcd(_RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last) {
+  typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type;
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  const difference_type __m1 = __middle - __first;
+  const difference_type __m2 = _Ops::distance(__middle, __last);
+  if (__m1 == __m2) {
+    std::__swap_ranges<_AlgPolicy>(__first, __middle, __middle, __last);
+    return __middle;
+  }
+  const difference_type __g = std::__algo_gcd(__m1, __m2);
+  for (_RandomAccessIterator __p = __first + __g; __p != __first;) {
+    value_type __t(_Ops::__iter_move(--__p));
+    _RandomAccessIterator __p1 = __p;
+    _RandomAccessIterator __p2 = __p1 + __m1;
+    do {
+      *__p1                     = _Ops::__iter_move(__p2);
+      __p1                      = __p2;
+      const difference_type __d = _Ops::distance(__p2, __last);
+      if (__m1 < __d)
+        __p2 += __m1;
+      else
+        __p2 = __first + (__m1 - __d);
+    } while (__p2 != __p);
+    *__p1 = std::move(__t);
+  }
+  return __first + __m2;
+}
+
+template <class _AlgPolicy, class _ForwardIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator
+__rotate_impl(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last, std::forward_iterator_tag) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type value_type;
+  if (is_trivially_move_assignable<value_type>::value) {
+    if (_IterOps<_AlgPolicy>::next(__first) == __middle)
+      return std::__rotate_left<_AlgPolicy>(__first, __last);
+  }
+  return std::__rotate_forward<_AlgPolicy>(__first, __middle, __last);
+}
+
+template <class _AlgPolicy, class _BidirectionalIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _BidirectionalIterator __rotate_impl(
+    _BidirectionalIterator __first,
+    _BidirectionalIterator __middle,
+    _BidirectionalIterator __last,
+    bidirectional_iterator_tag) {
+  typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type;
+  if (is_trivially_move_assignable<value_type>::value) {
+    if (_IterOps<_AlgPolicy>::next(__first) == __middle)
+      return std::__rotate_left<_AlgPolicy>(__first, __last);
+    if (_IterOps<_AlgPolicy>::next(__middle) == __last)
+      return std::__rotate_right<_AlgPolicy>(__first, __last);
+  }
+  return std::__rotate_forward<_AlgPolicy>(__first, __middle, __last);
+}
+
+template <class _AlgPolicy, class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _RandomAccessIterator __rotate_impl(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __middle,
+    _RandomAccessIterator __last,
+    random_access_iterator_tag) {
+  typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type;
+  if (is_trivially_move_assignable<value_type>::value) {
+    if (_IterOps<_AlgPolicy>::next(__first) == __middle)
+      return std::__rotate_left<_AlgPolicy>(__first, __last);
+    if (_IterOps<_AlgPolicy>::next(__middle) == __last)
+      return std::__rotate_right<_AlgPolicy>(__first, __last);
+    return std::__rotate_gcd<_AlgPolicy>(__first, __middle, __last);
+  }
+  return std::__rotate_forward<_AlgPolicy>(__first, __middle, __last);
+}
+
+template <class _AlgPolicy, class _Iterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iterator, _Iterator>
+__rotate(_Iterator __first, _Iterator __middle, _Sentinel __last) {
+  using _Ret            = pair<_Iterator, _Iterator>;
+  _Iterator __last_iter = _IterOps<_AlgPolicy>::next(__middle, __last);
+
+  if (__first == __middle)
+    return _Ret(__last_iter, __last_iter);
+  if (__middle == __last)
+    return _Ret(std::move(__first), std::move(__last_iter));
+
+  using _IterCategory = typename _IterOps<_AlgPolicy>::template __iterator_category<_Iterator>;
+  auto __result = std::__rotate_impl<_AlgPolicy>(std::move(__first), std::move(__middle), __last_iter, _IterCategory());
+
+  return _Ret(std::move(__result), std::move(__last_iter));
+}
+
+template <class _ForwardIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+rotate(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last) {
+  return std::__rotate<_ClassicAlgPolicy>(std::move(__first), std::move(__middle), std::move(__last)).first;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_ROTATE_H
diff --git a/libcxx/include/__cxx03/__algorithm/rotate_copy.h b/libcxx/include/__cxx03/__algorithm/rotate_copy.h
new file mode 100644
index 000000000000..cddcadd237d9
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/rotate_copy.h
@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_ROTATE_COPY_H
+#define _LIBCPP___ALGORITHM_ROTATE_COPY_H
+
+#include <__algorithm/copy.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ForwardIterator, class _OutputIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+rotate_copy(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last, _OutputIterator __result) {
+  return std::copy(__first, __middle, std::copy(__middle, __last, __result));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_ROTATE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/sample.h b/libcxx/include/__cxx03/__algorithm/sample.h
new file mode 100644
index 000000000000..ebe5180b7eec
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/sample.h
@@ -0,0 +1,123 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SAMPLE_H
+#define _LIBCPP___ALGORITHM_SAMPLE_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/min.h>
+#include <__assert>
+#include <__config>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__random/uniform_int_distribution.h>
+#include <__type_traits/common_type.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy,
+          class _PopulationIterator,
+          class _PopulationSentinel,
+          class _SampleIterator,
+          class _Distance,
+          class _UniformRandomNumberGenerator>
+_LIBCPP_HIDE_FROM_ABI _SampleIterator __sample(
+    _PopulationIterator __first,
+    _PopulationSentinel __last,
+    _SampleIterator __output_iter,
+    _Distance __n,
+    _UniformRandomNumberGenerator& __g,
+    input_iterator_tag) {
+  _Distance __k = 0;
+  for (; __first != __last && __k < __n; ++__first, (void)++__k)
+    __output_iter[__k] = *__first;
+  _Distance __sz = __k;
+  for (; __first != __last; ++__first, (void)++__k) {
+    _Distance __r = uniform_int_distribution<_Distance>(0, __k)(__g);
+    if (__r < __sz)
+      __output_iter[__r] = *__first;
+  }
+  return __output_iter + std::min(__n, __k);
+}
+
+template <class _AlgPolicy,
+          class _PopulationIterator,
+          class _PopulationSentinel,
+          class _SampleIterator,
+          class _Distance,
+          class _UniformRandomNumberGenerator>
+_LIBCPP_HIDE_FROM_ABI _SampleIterator __sample(
+    _PopulationIterator __first,
+    _PopulationSentinel __last,
+    _SampleIterator __output_iter,
+    _Distance __n,
+    _UniformRandomNumberGenerator& __g,
+    forward_iterator_tag) {
+  _Distance __unsampled_sz = _IterOps<_AlgPolicy>::distance(__first, __last);
+  for (__n = std::min(__n, __unsampled_sz); __n != 0; ++__first) {
+    _Distance __r = uniform_int_distribution<_Distance>(0, --__unsampled_sz)(__g);
+    if (__r < __n) {
+      *__output_iter++ = *__first;
+      --__n;
+    }
+  }
+  return __output_iter;
+}
+
+template <class _AlgPolicy,
+          class _PopulationIterator,
+          class _PopulationSentinel,
+          class _SampleIterator,
+          class _Distance,
+          class _UniformRandomNumberGenerator>
+_LIBCPP_HIDE_FROM_ABI _SampleIterator __sample(
+    _PopulationIterator __first,
+    _PopulationSentinel __last,
+    _SampleIterator __output_iter,
+    _Distance __n,
+    _UniformRandomNumberGenerator& __g) {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n >= 0, "N must be a positive number.");
+
+  using _PopIterCategory = typename _IterOps<_AlgPolicy>::template __iterator_category<_PopulationIterator>;
+  using _Difference      = typename _IterOps<_AlgPolicy>::template __difference_type<_PopulationIterator>;
+  using _CommonType      = typename common_type<_Distance, _Difference>::type;
+
+  return std::__sample<_AlgPolicy>(
+      std::move(__first), std::move(__last), std::move(__output_iter), _CommonType(__n), __g, _PopIterCategory());
+}
+
+#if _LIBCPP_STD_VER >= 17
+template <class _PopulationIterator, class _SampleIterator, class _Distance, class _UniformRandomNumberGenerator>
+inline _LIBCPP_HIDE_FROM_ABI _SampleIterator
+sample(_PopulationIterator __first,
+       _PopulationIterator __last,
+       _SampleIterator __output_iter,
+       _Distance __n,
+       _UniformRandomNumberGenerator&& __g) {
+  static_assert(__has_forward_iterator_category<_PopulationIterator>::value ||
+                    __has_random_access_iterator_category<_SampleIterator>::value,
+                "SampleIterator must meet the requirements of RandomAccessIterator");
+
+  return std::__sample<_ClassicAlgPolicy>(std::move(__first), std::move(__last), std::move(__output_iter), __n, __g);
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_SAMPLE_H
diff --git a/libcxx/include/__cxx03/__algorithm/search.h b/libcxx/include/__cxx03/__algorithm/search.h
new file mode 100644
index 000000000000..b82ca7809535
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/search.h
@@ -0,0 +1,192 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SEARCH_H
+#define _LIBCPP___ALGORITHM_SEARCH_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/advance.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_callable.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy,
+          class _Iter1,
+          class _Sent1,
+          class _Iter2,
+          class _Sent2,
+          class _Pred,
+          class _Proj1,
+          class _Proj2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1> __search_forward_impl(
+    _Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+  if (__first2 == __last2)
+    return std::make_pair(__first1, __first1); // Everything matches an empty sequence
+  while (true) {
+    // Find first element in sequence 1 that matchs *__first2, with a mininum of loop checks
+    while (true) {
+      if (__first1 == __last1) { // return __last1 if no element matches *__first2
+        _IterOps<_AlgPolicy>::__advance_to(__first1, __last1);
+        return std::make_pair(__first1, __first1);
+      }
+      if (std::__invoke(__pred, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2)))
+        break;
+      ++__first1;
+    }
+    // *__first1 matches *__first2, now match elements after here
+    _Iter1 __m1 = __first1;
+    _Iter2 __m2 = __first2;
+    while (true) {
+      if (++__m2 == __last2) // If pattern exhausted, __first1 is the answer (works for 1 element pattern)
+        return std::make_pair(__first1, ++__m1);
+      if (++__m1 == __last1) { // Otherwise if source exhaused, pattern not found
+        return std::make_pair(__m1, __m1);
+      }
+
+      // if there is a mismatch, restart with a new __first1
+      if (!std::__invoke(__pred, std::__invoke(__proj1, *__m1), std::__invoke(__proj2, *__m2))) {
+        ++__first1;
+        break;
+      } // else there is a match, check next elements
+    }
+  }
+}
+
+template <class _AlgPolicy,
+          class _Iter1,
+          class _Sent1,
+          class _Iter2,
+          class _Sent2,
+          class _Pred,
+          class _Proj1,
+          class _Proj2,
+          class _DiffT1,
+          class _DiffT2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1> __search_random_access_impl(
+    _Iter1 __first1,
+    _Sent1 __last1,
+    _Iter2 __first2,
+    _Sent2 __last2,
+    _Pred& __pred,
+    _Proj1& __proj1,
+    _Proj2& __proj2,
+    _DiffT1 __size1,
+    _DiffT2 __size2) {
+  const _Iter1 __s = __first1 + __size1 - _DiffT1(__size2 - 1); // Start of pattern match can't go beyond here
+
+  while (true) {
+    while (true) {
+      if (__first1 == __s) {
+        _IterOps<_AlgPolicy>::__advance_to(__first1, __last1);
+        return std::make_pair(__first1, __first1);
+      }
+      if (std::__invoke(__pred, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2)))
+        break;
+      ++__first1;
+    }
+
+    _Iter1 __m1 = __first1;
+    _Iter2 __m2 = __first2;
+    while (true) {
+      if (++__m2 == __last2)
+        return std::make_pair(__first1, __first1 + _DiffT1(__size2));
+      ++__m1; // no need to check range on __m1 because __s guarantees we have enough source
+      if (!std::__invoke(__pred, std::__invoke(__proj1, *__m1), std::__invoke(__proj2, *__m2))) {
+        ++__first1;
+        break;
+      }
+    }
+  }
+}
+
+template <class _Iter1,
+          class _Sent1,
+          class _Iter2,
+          class _Sent2,
+          class _Pred,
+          class _Proj1,
+          class _Proj2,
+          __enable_if_t<__has_random_access_iterator_category<_Iter1>::value &&
+                            __has_random_access_iterator_category<_Iter2>::value,
+                        int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1> __search_impl(
+    _Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+  auto __size2 = __last2 - __first2;
+  if (__size2 == 0)
+    return std::make_pair(__first1, __first1);
+
+  auto __size1 = __last1 - __first1;
+  if (__size1 < __size2) {
+    return std::make_pair(__last1, __last1);
+  }
+
+  return std::__search_random_access_impl<_ClassicAlgPolicy>(
+      __first1, __last1, __first2, __last2, __pred, __proj1, __proj2, __size1, __size2);
+}
+
+template <
+    class _Iter1,
+    class _Sent1,
+    class _Iter2,
+    class _Sent2,
+    class _Pred,
+    class _Proj1,
+    class _Proj2,
+    __enable_if_t<__has_forward_iterator_category<_Iter1>::value && __has_forward_iterator_category<_Iter2>::value &&
+                      !(__has_random_access_iterator_category<_Iter1>::value &&
+                        __has_random_access_iterator_category<_Iter2>::value),
+                  int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1> __search_impl(
+    _Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+  return std::__search_forward_impl<_ClassicAlgPolicy>(__first1, __last1, __first2, __last2, __pred, __proj1, __proj2);
+}
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator1
+search(_ForwardIterator1 __first1,
+       _ForwardIterator1 __last1,
+       _ForwardIterator2 __first2,
+       _ForwardIterator2 __last2,
+       _BinaryPredicate __pred) {
+  static_assert(__is_callable<_BinaryPredicate, decltype(*__first1), decltype(*__first2)>::value,
+                "BinaryPredicate has to be callable");
+  auto __proj = __identity();
+  return std::__search_impl(__first1, __last1, __first2, __last2, __pred, __proj, __proj).first;
+}
+
+template <class _ForwardIterator1, class _ForwardIterator2>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator1
+search(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2) {
+  return std::search(__first1, __last1, __first2, __last2, __equal_to());
+}
+
+#if _LIBCPP_STD_VER >= 17
+template <class _ForwardIterator, class _Searcher>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+search(_ForwardIterator __f, _ForwardIterator __l, const _Searcher& __s) {
+  return __s(__f, __l).first;
+}
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_SEARCH_H
diff --git a/libcxx/include/__cxx03/__algorithm/search_n.h b/libcxx/include/__cxx03/__algorithm/search_n.h
new file mode 100644
index 000000000000..771647d3168a
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/search_n.h
@@ -0,0 +1,155 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SEARCH_N_H
+#define _LIBCPP___ALGORITHM_SEARCH_N_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/advance.h>
+#include <__iterator/concepts.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__ranges/concepts.h>
+#include <__type_traits/is_callable.h>
+#include <__utility/convert_to_integral.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Pred, class _Iter, class _Sent, class _SizeT, class _Type, class _Proj>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter, _Iter> __search_n_forward_impl(
+    _Iter __first, _Sent __last, _SizeT __count, const _Type& __value, _Pred& __pred, _Proj& __proj) {
+  if (__count <= 0)
+    return std::make_pair(__first, __first);
+  while (true) {
+    // Find first element in sequence that matchs __value, with a mininum of loop checks
+    while (true) {
+      if (__first == __last) { // return __last if no element matches __value
+        _IterOps<_AlgPolicy>::__advance_to(__first, __last);
+        return std::make_pair(__first, __first);
+      }
+      if (std::__invoke(__pred, std::__invoke(__proj, *__first), __value))
+        break;
+      ++__first;
+    }
+    // *__first matches __value, now match elements after here
+    _Iter __m = __first;
+    _SizeT __c(0);
+    while (true) {
+      if (++__c == __count) // If pattern exhausted, __first is the answer (works for 1 element pattern)
+        return std::make_pair(__first, ++__m);
+      if (++__m == __last) { // Otherwise if source exhaused, pattern not found
+        _IterOps<_AlgPolicy>::__advance_to(__first, __last);
+        return std::make_pair(__first, __first);
+      }
+
+      // if there is a mismatch, restart with a new __first
+      if (!std::__invoke(__pred, std::__invoke(__proj, *__m), __value)) {
+        __first = __m;
+        ++__first;
+        break;
+      } // else there is a match, check next elements
+    }
+  }
+}
+
+template <class _AlgPolicy, class _Pred, class _Iter, class _Sent, class _SizeT, class _Type, class _Proj, class _DiffT>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 std::pair<_Iter, _Iter> __search_n_random_access_impl(
+    _Iter __first, _Sent __last, _SizeT __count, const _Type& __value, _Pred& __pred, _Proj& __proj, _DiffT __size1) {
+  using difference_type = typename iterator_traits<_Iter>::difference_type;
+  if (__count == 0)
+    return std::make_pair(__first, __first);
+  if (__size1 < static_cast<_DiffT>(__count)) {
+    _IterOps<_AlgPolicy>::__advance_to(__first, __last);
+    return std::make_pair(__first, __first);
+  }
+
+  const auto __s = __first + __size1 - difference_type(__count - 1); // Start of pattern match can't go beyond here
+  while (true) {
+    // Find first element in sequence that matchs __value, with a mininum of loop checks
+    while (true) {
+      if (__first >= __s) { // return __last if no element matches __value
+        _IterOps<_AlgPolicy>::__advance_to(__first, __last);
+        return std::make_pair(__first, __first);
+      }
+      if (std::__invoke(__pred, std::__invoke(__proj, *__first), __value))
+        break;
+      ++__first;
+    }
+    // *__first matches __value_, now match elements after here
+    auto __m = __first;
+    _SizeT __c(0);
+    while (true) {
+      if (++__c == __count) // If pattern exhausted, __first is the answer (works for 1 element pattern)
+        return std::make_pair(__first, __first + _DiffT(__count));
+      ++__m; // no need to check range on __m because __s guarantees we have enough source
+
+      // if there is a mismatch, restart with a new __first
+      if (!std::__invoke(__pred, std::__invoke(__proj, *__m), __value)) {
+        __first = __m;
+        ++__first;
+        break;
+      } // else there is a match, check next elements
+    }
+  }
+}
+
+template <class _Iter,
+          class _Sent,
+          class _DiffT,
+          class _Type,
+          class _Pred,
+          class _Proj,
+          __enable_if_t<__has_random_access_iterator_category<_Iter>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter, _Iter>
+__search_n_impl(_Iter __first, _Sent __last, _DiffT __count, const _Type& __value, _Pred& __pred, _Proj& __proj) {
+  return std::__search_n_random_access_impl<_ClassicAlgPolicy>(
+      __first, __last, __count, __value, __pred, __proj, __last - __first);
+}
+
+template <class _Iter1,
+          class _Sent1,
+          class _DiffT,
+          class _Type,
+          class _Pred,
+          class _Proj,
+          __enable_if_t<__has_forward_iterator_category<_Iter1>::value &&
+                            !__has_random_access_iterator_category<_Iter1>::value,
+                        int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1>
+__search_n_impl(_Iter1 __first, _Sent1 __last, _DiffT __count, const _Type& __value, _Pred& __pred, _Proj& __proj) {
+  return std::__search_n_forward_impl<_ClassicAlgPolicy>(__first, __last, __count, __value, __pred, __proj);
+}
+
+template <class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator search_n(
+    _ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value, _BinaryPredicate __pred) {
+  static_assert(
+      __is_callable<_BinaryPredicate, decltype(*__first), const _Tp&>::value, "BinaryPredicate has to be callable");
+  auto __proj = __identity();
+  return std::__search_n_impl(__first, __last, std::__convert_to_integral(__count), __value, __pred, __proj).first;
+}
+
+template <class _ForwardIterator, class _Size, class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+search_n(_ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value) {
+  return std::search_n(__first, __last, std::__convert_to_integral(__count), __value, __equal_to());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_SEARCH_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/set_difference.h b/libcxx/include/__cxx03/__algorithm/set_difference.h
new file mode 100644
index 000000000000..f414bcecb50d
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/set_difference.h
@@ -0,0 +1,79 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SET_DIFFERENCE_H
+#define _LIBCPP___ALGORITHM_SET_DIFFERENCE_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/copy.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Comp, class _InIter1, class _Sent1, class _InIter2, class _Sent2, class _OutIter>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<__remove_cvref_t<_InIter1>, __remove_cvref_t<_OutIter> >
+__set_difference(
+    _InIter1&& __first1, _Sent1&& __last1, _InIter2&& __first2, _Sent2&& __last2, _OutIter&& __result, _Comp&& __comp) {
+  while (__first1 != __last1 && __first2 != __last2) {
+    if (__comp(*__first1, *__first2)) {
+      *__result = *__first1;
+      ++__first1;
+      ++__result;
+    } else if (__comp(*__first2, *__first1)) {
+      ++__first2;
+    } else {
+      ++__first1;
+      ++__first2;
+    }
+  }
+  return std::__copy<_AlgPolicy>(std::move(__first1), std::move(__last1), std::move(__result));
+}
+
+template <class _InputIterator1, class _InputIterator2, class _OutputIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator set_difference(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _InputIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp) {
+  return std::__set_difference<_ClassicAlgPolicy, __comp_ref_type<_Compare> >(
+             __first1, __last1, __first2, __last2, __result, __comp)
+      .second;
+}
+
+template <class _InputIterator1, class _InputIterator2, class _OutputIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator set_difference(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _InputIterator2 __last2,
+    _OutputIterator __result) {
+  return std::__set_difference<_ClassicAlgPolicy>(__first1, __last1, __first2, __last2, __result, __less<>()).second;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_SET_DIFFERENCE_H
diff --git a/libcxx/include/__cxx03/__algorithm/set_intersection.h b/libcxx/include/__cxx03/__algorithm/set_intersection.h
new file mode 100644
index 000000000000..bb0d86cd0f58
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/set_intersection.h
@@ -0,0 +1,217 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SET_INTERSECTION_H
+#define _LIBCPP___ALGORITHM_SET_INTERSECTION_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/lower_bound.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__type_traits/is_same.h>
+#include <__utility/exchange.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InIter1, class _InIter2, class _OutIter>
+struct __set_intersection_result {
+  _InIter1 __in1_;
+  _InIter2 __in2_;
+  _OutIter __out_;
+
+  // need a constructor as C++03 aggregate init is hard
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  __set_intersection_result(_InIter1&& __in_iter1, _InIter2&& __in_iter2, _OutIter&& __out_iter)
+      : __in1_(std::move(__in_iter1)), __in2_(std::move(__in_iter2)), __out_(std::move(__out_iter)) {}
+};
+
+// Helper for __set_intersection() with one-sided binary search: populate result and advance input iterators if they
+// are found to potentially contain the same value in two consecutive calls. This function is very intimately related to
+// the way it is used and doesn't attempt to abstract that, it's not appropriate for general usage outside of its
+// context.
+template <class _InForwardIter1, class _InForwardIter2, class _OutIter>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __set_intersection_add_output_if_equal(
+    bool __may_be_equal,
+    _InForwardIter1& __first1,
+    _InForwardIter2& __first2,
+    _OutIter& __result,
+    bool& __prev_may_be_equal) {
+  if (__may_be_equal && __prev_may_be_equal) {
+    *__result = *__first1;
+    ++__result;
+    ++__first1;
+    ++__first2;
+    __prev_may_be_equal = false;
+  } else {
+    __prev_may_be_equal = __may_be_equal;
+  }
+}
+
+// With forward iterators we can make multiple passes over the data, allowing the use of one-sided binary search to
+// reduce best-case complexity to log(N). Understanding how we can use binary search and still respect complexity
+// guarantees is _not_ straightforward: the guarantee is "at most 2*(N+M)-1 comparisons", and one-sided binary search
+// will necessarily overshoot depending on the position of the needle in the haystack -- for instance, if we're
+// searching for 3 in (1, 2, 3, 4), we'll check if 3<1, then 3<2, then 3<4, and, finally, 3<3, for a total of 4
+// comparisons, when linear search would have yielded 3. However, because we won't need to perform the intervening
+// reciprocal comparisons (ie 1<3, 2<3, 4<3), that extra comparison doesn't run afoul of the guarantee. Additionally,
+// this type of scenario can only happen for match distances of up to 5 elements, because 2*log2(8) is 6, and we'll
+// still be worse-off at position 5 of an 8-element set. From then onwards these scenarios can't happen. TL;DR: we'll be
+// 1 comparison worse-off compared to the classic linear-searching algorithm if matching position 3 of a set with 4
+// elements, or position 5 if the set has 7 or 8 elements, but we'll never exceed the complexity guarantees from the
+// standard.
+template <class _AlgPolicy,
+          class _Compare,
+          class _InForwardIter1,
+          class _Sent1,
+          class _InForwardIter2,
+          class _Sent2,
+          class _OutIter>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI
+_LIBCPP_CONSTEXPR_SINCE_CXX20 __set_intersection_result<_InForwardIter1, _InForwardIter2, _OutIter>
+__set_intersection(
+    _InForwardIter1 __first1,
+    _Sent1 __last1,
+    _InForwardIter2 __first2,
+    _Sent2 __last2,
+    _OutIter __result,
+    _Compare&& __comp,
+    std::forward_iterator_tag,
+    std::forward_iterator_tag) {
+  _LIBCPP_CONSTEXPR std::__identity __proj;
+  bool __prev_may_be_equal = false;
+
+  while (__first2 != __last2) {
+    _InForwardIter1 __first1_next =
+        std::__lower_bound_onesided<_AlgPolicy>(__first1, __last1, *__first2, __comp, __proj);
+    std::swap(__first1_next, __first1);
+    // keeping in mind that a==b iff !(a<b) && !(b<a):
+    // if we can't advance __first1, that means !(*__first1 < *_first2), therefore __may_be_equal==true
+    std::__set_intersection_add_output_if_equal(
+        __first1 == __first1_next, __first1, __first2, __result, __prev_may_be_equal);
+    if (__first1 == __last1)
+      break;
+
+    _InForwardIter2 __first2_next =
+        std::__lower_bound_onesided<_AlgPolicy>(__first2, __last2, *__first1, __comp, __proj);
+    std::swap(__first2_next, __first2);
+    std::__set_intersection_add_output_if_equal(
+        __first2 == __first2_next, __first1, __first2, __result, __prev_may_be_equal);
+  }
+  return __set_intersection_result<_InForwardIter1, _InForwardIter2, _OutIter>(
+      _IterOps<_AlgPolicy>::next(std::move(__first1), std::move(__last1)),
+      _IterOps<_AlgPolicy>::next(std::move(__first2), std::move(__last2)),
+      std::move(__result));
+}
+
+// input iterators are not suitable for multipass algorithms, so we stick to the classic single-pass version
+template <class _AlgPolicy,
+          class _Compare,
+          class _InInputIter1,
+          class _Sent1,
+          class _InInputIter2,
+          class _Sent2,
+          class _OutIter>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI
+_LIBCPP_CONSTEXPR_SINCE_CXX20 __set_intersection_result<_InInputIter1, _InInputIter2, _OutIter>
+__set_intersection(
+    _InInputIter1 __first1,
+    _Sent1 __last1,
+    _InInputIter2 __first2,
+    _Sent2 __last2,
+    _OutIter __result,
+    _Compare&& __comp,
+    std::input_iterator_tag,
+    std::input_iterator_tag) {
+  while (__first1 != __last1 && __first2 != __last2) {
+    if (__comp(*__first1, *__first2))
+      ++__first1;
+    else {
+      if (!__comp(*__first2, *__first1)) {
+        *__result = *__first1;
+        ++__result;
+        ++__first1;
+      }
+      ++__first2;
+    }
+  }
+
+  return __set_intersection_result<_InInputIter1, _InInputIter2, _OutIter>(
+      _IterOps<_AlgPolicy>::next(std::move(__first1), std::move(__last1)),
+      _IterOps<_AlgPolicy>::next(std::move(__first2), std::move(__last2)),
+      std::move(__result));
+}
+
+template <class _AlgPolicy, class _Compare, class _InIter1, class _Sent1, class _InIter2, class _Sent2, class _OutIter>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI
+_LIBCPP_CONSTEXPR_SINCE_CXX20 __set_intersection_result<_InIter1, _InIter2, _OutIter>
+__set_intersection(
+    _InIter1 __first1, _Sent1 __last1, _InIter2 __first2, _Sent2 __last2, _OutIter __result, _Compare&& __comp) {
+  return std::__set_intersection<_AlgPolicy>(
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__last2),
+      std::move(__result),
+      std::forward<_Compare>(__comp),
+      typename std::_IterOps<_AlgPolicy>::template __iterator_category<_InIter1>(),
+      typename std::_IterOps<_AlgPolicy>::template __iterator_category<_InIter2>());
+}
+
+template <class _InputIterator1, class _InputIterator2, class _OutputIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator set_intersection(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _InputIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp) {
+  return std::__set_intersection<_ClassicAlgPolicy, __comp_ref_type<_Compare> >(
+             std::move(__first1),
+             std::move(__last1),
+             std::move(__first2),
+             std::move(__last2),
+             std::move(__result),
+             __comp)
+      .__out_;
+}
+
+template <class _InputIterator1, class _InputIterator2, class _OutputIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator set_intersection(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _InputIterator2 __last2,
+    _OutputIterator __result) {
+  return std::__set_intersection<_ClassicAlgPolicy>(
+             std::move(__first1),
+             std::move(__last1),
+             std::move(__first2),
+             std::move(__last2),
+             std::move(__result),
+             __less<>())
+      .__out_;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_SET_INTERSECTION_H
diff --git a/libcxx/include/__cxx03/__algorithm/set_symmetric_difference.h b/libcxx/include/__cxx03/__algorithm/set_symmetric_difference.h
new file mode 100644
index 000000000000..db36665a6136
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/set_symmetric_difference.h
@@ -0,0 +1,109 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SET_SYMMETRIC_DIFFERENCE_H
+#define _LIBCPP___ALGORITHM_SET_SYMMETRIC_DIFFERENCE_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/copy.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InIter1, class _InIter2, class _OutIter>
+struct __set_symmetric_difference_result {
+  _InIter1 __in1_;
+  _InIter2 __in2_;
+  _OutIter __out_;
+
+  // need a constructor as C++03 aggregate init is hard
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  __set_symmetric_difference_result(_InIter1&& __in_iter1, _InIter2&& __in_iter2, _OutIter&& __out_iter)
+      : __in1_(std::move(__in_iter1)), __in2_(std::move(__in_iter2)), __out_(std::move(__out_iter)) {}
+};
+
+template <class _AlgPolicy, class _Compare, class _InIter1, class _Sent1, class _InIter2, class _Sent2, class _OutIter>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __set_symmetric_difference_result<_InIter1, _InIter2, _OutIter>
+__set_symmetric_difference(
+    _InIter1 __first1, _Sent1 __last1, _InIter2 __first2, _Sent2 __last2, _OutIter __result, _Compare&& __comp) {
+  while (__first1 != __last1) {
+    if (__first2 == __last2) {
+      auto __ret1 = std::__copy<_AlgPolicy>(std::move(__first1), std::move(__last1), std::move(__result));
+      return __set_symmetric_difference_result<_InIter1, _InIter2, _OutIter>(
+          std::move(__ret1.first), std::move(__first2), std::move((__ret1.second)));
+    }
+    if (__comp(*__first1, *__first2)) {
+      *__result = *__first1;
+      ++__result;
+      ++__first1;
+    } else {
+      if (__comp(*__first2, *__first1)) {
+        *__result = *__first2;
+        ++__result;
+      } else {
+        ++__first1;
+      }
+      ++__first2;
+    }
+  }
+  auto __ret2 = std::__copy<_AlgPolicy>(std::move(__first2), std::move(__last2), std::move(__result));
+  return __set_symmetric_difference_result<_InIter1, _InIter2, _OutIter>(
+      std::move(__first1), std::move(__ret2.first), std::move((__ret2.second)));
+}
+
+template <class _InputIterator1, class _InputIterator2, class _OutputIterator, class _Compare>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator set_symmetric_difference(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _InputIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp) {
+  return std::__set_symmetric_difference<_ClassicAlgPolicy, __comp_ref_type<_Compare> >(
+             std::move(__first1),
+             std::move(__last1),
+             std::move(__first2),
+             std::move(__last2),
+             std::move(__result),
+             __comp)
+      .__out_;
+}
+
+template <class _InputIterator1, class _InputIterator2, class _OutputIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator set_symmetric_difference(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _InputIterator2 __last2,
+    _OutputIterator __result) {
+  return std::set_symmetric_difference(
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__last2),
+      std::move(__result),
+      __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_SET_SYMMETRIC_DIFFERENCE_H
diff --git a/libcxx/include/__cxx03/__algorithm/set_union.h b/libcxx/include/__cxx03/__algorithm/set_union.h
new file mode 100644
index 000000000000..a79c50fd3cf2
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/set_union.h
@@ -0,0 +1,105 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SET_UNION_H
+#define _LIBCPP___ALGORITHM_SET_UNION_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/copy.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InIter1, class _InIter2, class _OutIter>
+struct __set_union_result {
+  _InIter1 __in1_;
+  _InIter2 __in2_;
+  _OutIter __out_;
+
+  // need a constructor as C++03 aggregate init is hard
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  __set_union_result(_InIter1&& __in_iter1, _InIter2&& __in_iter2, _OutIter&& __out_iter)
+      : __in1_(std::move(__in_iter1)), __in2_(std::move(__in_iter2)), __out_(std::move(__out_iter)) {}
+};
+
+template <class _AlgPolicy, class _Compare, class _InIter1, class _Sent1, class _InIter2, class _Sent2, class _OutIter>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __set_union_result<_InIter1, _InIter2, _OutIter> __set_union(
+    _InIter1 __first1, _Sent1 __last1, _InIter2 __first2, _Sent2 __last2, _OutIter __result, _Compare&& __comp) {
+  for (; __first1 != __last1; ++__result) {
+    if (__first2 == __last2) {
+      auto __ret1 = std::__copy<_AlgPolicy>(std::move(__first1), std::move(__last1), std::move(__result));
+      return __set_union_result<_InIter1, _InIter2, _OutIter>(
+          std::move(__ret1.first), std::move(__first2), std::move((__ret1.second)));
+    }
+    if (__comp(*__first2, *__first1)) {
+      *__result = *__first2;
+      ++__first2;
+    } else {
+      if (!__comp(*__first1, *__first2)) {
+        ++__first2;
+      }
+      *__result = *__first1;
+      ++__first1;
+    }
+  }
+  auto __ret2 = std::__copy<_AlgPolicy>(std::move(__first2), std::move(__last2), std::move(__result));
+  return __set_union_result<_InIter1, _InIter2, _OutIter>(
+      std::move(__first1), std::move(__ret2.first), std::move((__ret2.second)));
+}
+
+template <class _InputIterator1, class _InputIterator2, class _OutputIterator, class _Compare>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator set_union(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _InputIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp) {
+  return std::__set_union<_ClassicAlgPolicy, __comp_ref_type<_Compare> >(
+             std::move(__first1),
+             std::move(__last1),
+             std::move(__first2),
+             std::move(__last2),
+             std::move(__result),
+             __comp)
+      .__out_;
+}
+
+template <class _InputIterator1, class _InputIterator2, class _OutputIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator set_union(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _InputIterator2 __last2,
+    _OutputIterator __result) {
+  return std::set_union(
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__last2),
+      std::move(__result),
+      __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_SET_UNION_H
diff --git a/libcxx/include/__cxx03/__algorithm/shift_left.h b/libcxx/include/__cxx03/__algorithm/shift_left.h
new file mode 100644
index 000000000000..06cd7c5f8764
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/shift_left.h
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SHIFT_LEFT_H
+#define _LIBCPP___ALGORITHM_SHIFT_LEFT_H
+
+#include <__algorithm/move.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _ForwardIterator>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _ForwardIterator
+shift_left(_ForwardIterator __first,
+           _ForwardIterator __last,
+           typename iterator_traits<_ForwardIterator>::difference_type __n) {
+  if (__n == 0) {
+    return __last;
+  }
+
+  _ForwardIterator __m = __first;
+  if constexpr (__has_random_access_iterator_category<_ForwardIterator>::value) {
+    if (__n >= __last - __first) {
+      return __first;
+    }
+    __m += __n;
+  } else {
+    for (; __n > 0; --__n) {
+      if (__m == __last) {
+        return __first;
+      }
+      ++__m;
+    }
+  }
+  return std::move(__m, __last, __first);
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_SHIFT_LEFT_H
diff --git a/libcxx/include/__cxx03/__algorithm/shift_right.h b/libcxx/include/__cxx03/__algorithm/shift_right.h
new file mode 100644
index 000000000000..01853057fc47
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/shift_right.h
@@ -0,0 +1,105 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SHIFT_RIGHT_H
+#define _LIBCPP___ALGORITHM_SHIFT_RIGHT_H
+
+#include <__algorithm/move.h>
+#include <__algorithm/move_backward.h>
+#include <__algorithm/swap_ranges.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__utility/swap.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _ForwardIterator>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _ForwardIterator
+shift_right(_ForwardIterator __first,
+            _ForwardIterator __last,
+            typename iterator_traits<_ForwardIterator>::difference_type __n) {
+  if (__n == 0) {
+    return __first;
+  }
+
+  if constexpr (__has_random_access_iterator_category<_ForwardIterator>::value) {
+    decltype(__n) __d = __last - __first;
+    if (__n >= __d) {
+      return __last;
+    }
+    _ForwardIterator __m = __first + (__d - __n);
+    return std::move_backward(__first, __m, __last);
+  } else if constexpr (__has_bidirectional_iterator_category<_ForwardIterator>::value) {
+    _ForwardIterator __m = __last;
+    for (; __n > 0; --__n) {
+      if (__m == __first) {
+        return __last;
+      }
+      --__m;
+    }
+    return std::move_backward(__first, __m, __last);
+  } else {
+    _ForwardIterator __ret = __first;
+    for (; __n > 0; --__n) {
+      if (__ret == __last) {
+        return __last;
+      }
+      ++__ret;
+    }
+
+    // We have an __n-element scratch space from __first to __ret.
+    // Slide an __n-element window [__trail, __lead) from left to right.
+    // We're essentially doing swap_ranges(__first, __ret, __trail, __lead)
+    // over and over; but once __lead reaches __last we needn't bother
+    // to save the values of elements [__trail, __last).
+
+    auto __trail = __first;
+    auto __lead  = __ret;
+    while (__trail != __ret) {
+      if (__lead == __last) {
+        std::move(__first, __trail, __ret);
+        return __ret;
+      }
+      ++__trail;
+      ++__lead;
+    }
+
+    _ForwardIterator __mid = __first;
+    while (true) {
+      if (__lead == __last) {
+        __trail = std::move(__mid, __ret, __trail);
+        std::move(__first, __mid, __trail);
+        return __ret;
+      }
+      swap(*__mid, *__trail);
+      ++__mid;
+      ++__trail;
+      ++__lead;
+      if (__mid == __ret) {
+        __mid = __first;
+      }
+    }
+  }
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_SHIFT_RIGHT_H
diff --git a/libcxx/include/__cxx03/__algorithm/shuffle.h b/libcxx/include/__cxx03/__algorithm/shuffle.h
new file mode 100644
index 000000000000..c9c56ce8c2c0
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/shuffle.h
@@ -0,0 +1,167 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SHUFFLE_H
+#define _LIBCPP___ALGORITHM_SHUFFLE_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__random/uniform_int_distribution.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+#include <cstddef>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+class _LIBCPP_EXPORTED_FROM_ABI __libcpp_debug_randomizer {
+public:
+  _LIBCPP_HIDE_FROM_ABI __libcpp_debug_randomizer() {
+    __state_ = __seed();
+    __inc_   = __state_ + 0xda3e39cb94b95bdbULL;
+    __inc_   = (__inc_ << 1) | 1;
+  }
+  typedef uint_fast32_t result_type;
+
+  static const result_type _Min = 0;
+  static const result_type _Max = 0xFFFFFFFF;
+
+  _LIBCPP_HIDE_FROM_ABI result_type operator()() {
+    uint_fast64_t __oldstate = __state_;
+    __state_                 = __oldstate * 6364136223846793005ULL + __inc_;
+    return __oldstate >> 32;
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR result_type min() { return _Min; }
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR result_type max() { return _Max; }
+
+private:
+  uint_fast64_t __state_;
+  uint_fast64_t __inc_;
+  _LIBCPP_HIDE_FROM_ABI static uint_fast64_t __seed() {
+#ifdef _LIBCPP_DEBUG_RANDOMIZE_UNSPECIFIED_STABILITY_SEED
+    return _LIBCPP_DEBUG_RANDOMIZE_UNSPECIFIED_STABILITY_SEED;
+#else
+    static char __x;
+    return reinterpret_cast<uintptr_t>(&__x);
+#endif
+  }
+};
+
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_RANDOM_SHUFFLE) || defined(_LIBCPP_BUILDING_LIBRARY)
+class _LIBCPP_EXPORTED_FROM_ABI __rs_default;
+
+_LIBCPP_EXPORTED_FROM_ABI __rs_default __rs_get();
+
+class _LIBCPP_EXPORTED_FROM_ABI __rs_default {
+  static unsigned __c_;
+
+  __rs_default();
+
+public:
+  typedef uint_fast32_t result_type;
+
+  static const result_type _Min = 0;
+  static const result_type _Max = 0xFFFFFFFF;
+
+  __rs_default(const __rs_default&);
+  ~__rs_default();
+
+  result_type operator()();
+
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR result_type min() { return _Min; }
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR result_type max() { return _Max; }
+
+  friend _LIBCPP_EXPORTED_FROM_ABI __rs_default __rs_get();
+};
+
+_LIBCPP_EXPORTED_FROM_ABI __rs_default __rs_get();
+
+template <class _RandomAccessIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_IN_CXX14 void
+random_shuffle(_RandomAccessIterator __first, _RandomAccessIterator __last) {
+  typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  typedef uniform_int_distribution<ptrdiff_t> _Dp;
+  typedef typename _Dp::param_type _Pp;
+  difference_type __d = __last - __first;
+  if (__d > 1) {
+    _Dp __uid;
+    __rs_default __g = __rs_get();
+    for (--__last, (void)--__d; __first < __last; ++__first, (void)--__d) {
+      difference_type __i = __uid(__g, _Pp(0, __d));
+      if (__i != difference_type(0))
+        swap(*__first, *(__first + __i));
+    }
+  }
+}
+
+template <class _RandomAccessIterator, class _RandomNumberGenerator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_IN_CXX14 void
+random_shuffle(_RandomAccessIterator __first,
+               _RandomAccessIterator __last,
+#  ifndef _LIBCPP_CXX03_LANG
+               _RandomNumberGenerator&& __rand)
+#  else
+               _RandomNumberGenerator& __rand)
+#  endif
+{
+  typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  difference_type __d = __last - __first;
+  if (__d > 1) {
+    for (--__last; __first < __last; ++__first, (void)--__d) {
+      difference_type __i = __rand(__d);
+      if (__i != difference_type(0))
+        swap(*__first, *(__first + __i));
+    }
+  }
+}
+#endif
+
+template <class _AlgPolicy, class _RandomAccessIterator, class _Sentinel, class _UniformRandomNumberGenerator>
+_LIBCPP_HIDE_FROM_ABI _RandomAccessIterator
+__shuffle(_RandomAccessIterator __first, _Sentinel __last_sentinel, _UniformRandomNumberGenerator&& __g) {
+  typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  typedef uniform_int_distribution<ptrdiff_t> _Dp;
+  typedef typename _Dp::param_type _Pp;
+
+  auto __original_last = _IterOps<_AlgPolicy>::next(__first, __last_sentinel);
+  auto __last          = __original_last;
+  difference_type __d  = __last - __first;
+  if (__d > 1) {
+    _Dp __uid;
+    for (--__last, (void)--__d; __first < __last; ++__first, (void)--__d) {
+      difference_type __i = __uid(__g, _Pp(0, __d));
+      if (__i != difference_type(0))
+        _IterOps<_AlgPolicy>::iter_swap(__first, __first + __i);
+    }
+  }
+
+  return __original_last;
+}
+
+template <class _RandomAccessIterator, class _UniformRandomNumberGenerator>
+_LIBCPP_HIDE_FROM_ABI void
+shuffle(_RandomAccessIterator __first, _RandomAccessIterator __last, _UniformRandomNumberGenerator&& __g) {
+  (void)std::__shuffle<_ClassicAlgPolicy>(
+      std::move(__first), std::move(__last), std::forward<_UniformRandomNumberGenerator>(__g));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_SHUFFLE_H
diff --git a/libcxx/include/__cxx03/__algorithm/sift_down.h b/libcxx/include/__cxx03/__algorithm/sift_down.h
new file mode 100644
index 000000000000..42803e30631f
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/sift_down.h
@@ -0,0 +1,118 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SIFT_DOWN_H
+#define _LIBCPP___ALGORITHM_SIFT_DOWN_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__assert>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+__sift_down(_RandomAccessIterator __first,
+            _Compare&& __comp,
+            typename iterator_traits<_RandomAccessIterator>::difference_type __len,
+            _RandomAccessIterator __start) {
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type;
+  // left-child of __start is at 2 * __start + 1
+  // right-child of __start is at 2 * __start + 2
+  difference_type __child = __start - __first;
+
+  if (__len < 2 || (__len - 2) / 2 < __child)
+    return;
+
+  __child                         = 2 * __child + 1;
+  _RandomAccessIterator __child_i = __first + __child;
+
+  if ((__child + 1) < __len && __comp(*__child_i, *(__child_i + difference_type(1)))) {
+    // right-child exists and is greater than left-child
+    ++__child_i;
+    ++__child;
+  }
+
+  // check if we are in heap-order
+  if (__comp(*__child_i, *__start))
+    // we are, __start is larger than its largest child
+    return;
+
+  value_type __top(_Ops::__iter_move(__start));
+  do {
+    // we are not in heap-order, swap the parent with its largest child
+    *__start = _Ops::__iter_move(__child_i);
+    __start  = __child_i;
+
+    if ((__len - 2) / 2 < __child)
+      break;
+
+    // recompute the child based off of the updated parent
+    __child   = 2 * __child + 1;
+    __child_i = __first + __child;
+
+    if ((__child + 1) < __len && __comp(*__child_i, *(__child_i + difference_type(1)))) {
+      // right-child exists and is greater than left-child
+      ++__child_i;
+      ++__child;
+    }
+
+    // check if we are in heap-order
+  } while (!__comp(*__child_i, __top));
+  *__start = std::move(__top);
+}
+
+template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _RandomAccessIterator __floyd_sift_down(
+    _RandomAccessIterator __first,
+    _Compare&& __comp,
+    typename iterator_traits<_RandomAccessIterator>::difference_type __len) {
+  using difference_type = typename iterator_traits<_RandomAccessIterator>::difference_type;
+  _LIBCPP_ASSERT_INTERNAL(__len >= 2, "shouldn't be called unless __len >= 2");
+
+  _RandomAccessIterator __hole    = __first;
+  _RandomAccessIterator __child_i = __first;
+  difference_type __child         = 0;
+
+  while (true) {
+    __child_i += difference_type(__child + 1);
+    __child = 2 * __child + 1;
+
+    if ((__child + 1) < __len && __comp(*__child_i, *(__child_i + difference_type(1)))) {
+      // right-child exists and is greater than left-child
+      ++__child_i;
+      ++__child;
+    }
+
+    // swap __hole with its largest child
+    *__hole = _IterOps<_AlgPolicy>::__iter_move(__child_i);
+    __hole  = __child_i;
+
+    // if __hole is now a leaf, we're done
+    if (__child > (__len - 2) / 2)
+      return __hole;
+  }
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_SIFT_DOWN_H
diff --git a/libcxx/include/__cxx03/__algorithm/simd_utils.h b/libcxx/include/__cxx03/__algorithm/simd_utils.h
new file mode 100644
index 000000000000..549197be8018
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/simd_utils.h
@@ -0,0 +1,164 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SIMD_UTILS_H
+#define _LIBCPP___ALGORITHM_SIMD_UTILS_H
+
+#include <__algorithm/min.h>
+#include <__bit/bit_cast.h>
+#include <__bit/countl.h>
+#include <__bit/countr.h>
+#include <__config>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_same.h>
+#include <__utility/integer_sequence.h>
+#include <cstddef>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+// TODO: Find out how altivec changes things and allow vectorizations there too.
+#if _LIBCPP_STD_VER >= 14 && defined(_LIBCPP_CLANG_VER) && !defined(__ALTIVEC__)
+#  define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 1
+#else
+#  define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 0
+#endif
+
+#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS && !defined(__OPTIMIZE_SIZE__)
+#  define _LIBCPP_VECTORIZE_ALGORITHMS 1
+#else
+#  define _LIBCPP_VECTORIZE_ALGORITHMS 0
+#endif
+
+#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+inline constexpr bool __can_map_to_integer_v =
+    sizeof(_Tp) == alignof(_Tp) && (sizeof(_Tp) == 1 || sizeof(_Tp) == 2 || sizeof(_Tp) == 4 || sizeof(_Tp) == 8);
+
+template <size_t _TypeSize>
+struct __get_as_integer_type_impl;
+
+template <>
+struct __get_as_integer_type_impl<1> {
+  using type = uint8_t;
+};
+
+template <>
+struct __get_as_integer_type_impl<2> {
+  using type = uint16_t;
+};
+template <>
+struct __get_as_integer_type_impl<4> {
+  using type = uint32_t;
+};
+template <>
+struct __get_as_integer_type_impl<8> {
+  using type = uint64_t;
+};
+
+template <class _Tp>
+using __get_as_integer_type_t = typename __get_as_integer_type_impl<sizeof(_Tp)>::type;
+
+// This isn't specialized for 64 byte vectors on purpose. They have the potential to significantly reduce performance
+// in mixed simd/non-simd workloads and don't provide any performance improvement for currently vectorized algorithms
+// as far as benchmarks are concerned.
+#  if defined(__AVX__) || defined(__MVS__)
+template <class _Tp>
+inline constexpr size_t __native_vector_size = 32 / sizeof(_Tp);
+#  elif defined(__SSE__) || defined(__ARM_NEON__)
+template <class _Tp>
+inline constexpr size_t __native_vector_size = 16 / sizeof(_Tp);
+#  elif defined(__MMX__)
+template <class _Tp>
+inline constexpr size_t __native_vector_size = 8 / sizeof(_Tp);
+#  else
+template <class _Tp>
+inline constexpr size_t __native_vector_size = 1;
+#  endif
+
+template <class _ArithmeticT, size_t _Np>
+using __simd_vector __attribute__((__ext_vector_type__(_Np))) = _ArithmeticT;
+
+template <class _VecT>
+inline constexpr size_t __simd_vector_size_v = []<bool _False = false>() -> size_t {
+  static_assert(_False, "Not a vector!");
+}();
+
+template <class _Tp, size_t _Np>
+inline constexpr size_t __simd_vector_size_v<__simd_vector<_Tp, _Np>> = _Np;
+
+template <class _Tp, size_t _Np>
+_LIBCPP_HIDE_FROM_ABI _Tp __simd_vector_underlying_type_impl(__simd_vector<_Tp, _Np>) {
+  return _Tp{};
+}
+
+template <class _VecT>
+using __simd_vector_underlying_type_t = decltype(std::__simd_vector_underlying_type_impl(_VecT{}));
+
+// This isn't inlined without always_inline when loading chars.
+template <class _VecT, class _Iter>
+_LIBCPP_NODISCARD _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT __load_vector(_Iter __iter) noexcept {
+  return [=]<size_t... _Indices>(index_sequence<_Indices...>) _LIBCPP_ALWAYS_INLINE noexcept {
+    return _VecT{__iter[_Indices]...};
+  }(make_index_sequence<__simd_vector_size_v<_VecT>>{});
+}
+
+template <class _Tp, size_t _Np>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool __all_of(__simd_vector<_Tp, _Np> __vec) noexcept {
+  return __builtin_reduce_and(__builtin_convertvector(__vec, __simd_vector<bool, _Np>));
+}
+
+template <class _Tp, size_t _Np>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_set(__simd_vector<_Tp, _Np> __vec) noexcept {
+  using __mask_vec = __simd_vector<bool, _Np>;
+
+  // This has MSan disabled du to https://github.com/llvm/llvm-project/issues/85876
+  auto __impl = [&]<class _MaskT>(_MaskT) _LIBCPP_NO_SANITIZE("memory") noexcept {
+#  if defined(_LIBCPP_BIG_ENDIAN)
+    return std::min<size_t>(
+        _Np, std::__countl_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec))));
+#  else
+    return std::min<size_t>(
+        _Np, std::__countr_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec))));
+#  endif
+  };
+
+  if constexpr (sizeof(__mask_vec) == sizeof(uint8_t)) {
+    return __impl(uint8_t{});
+  } else if constexpr (sizeof(__mask_vec) == sizeof(uint16_t)) {
+    return __impl(uint16_t{});
+  } else if constexpr (sizeof(__mask_vec) == sizeof(uint32_t)) {
+    return __impl(uint32_t{});
+  } else if constexpr (sizeof(__mask_vec) == sizeof(uint64_t)) {
+    return __impl(uint64_t{});
+  } else {
+    static_assert(sizeof(__mask_vec) == 0, "unexpected required size for mask integer type");
+    return 0;
+  }
+}
+
+template <class _Tp, size_t _Np>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_not_set(__simd_vector<_Tp, _Np> __vec) noexcept {
+  return std::__find_first_set(~__vec);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_SIMD_UTILS_H
diff --git a/libcxx/include/__cxx03/__algorithm/sort.h b/libcxx/include/__cxx03/__algorithm/sort.h
new file mode 100644
index 000000000000..07b5814639e9
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/sort.h
@@ -0,0 +1,1016 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SORT_H
+#define _LIBCPP___ALGORITHM_SORT_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/iter_swap.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/min_element.h>
+#include <__algorithm/partial_sort.h>
+#include <__algorithm/unwrap_iter.h>
+#include <__assert>
+#include <__bit/blsr.h>
+#include <__bit/countl.h>
+#include <__bit/countr.h>
+#include <__config>
+#include <__debug_utils/randomize_range.h>
+#include <__debug_utils/strict_weak_ordering_check.h>
+#include <__functional/operations.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/disjunction.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <climits>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// stable, 2-3 compares, 0-2 swaps
+
+template <class _AlgPolicy, class _Compare, class _ForwardIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 unsigned
+__sort3(_ForwardIterator __x, _ForwardIterator __y, _ForwardIterator __z, _Compare __c) {
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  unsigned __r = 0;
+  if (!__c(*__y, *__x)) // if x <= y
+  {
+    if (!__c(*__z, *__y))      // if y <= z
+      return __r;              // x <= y && y <= z
+                               // x <= y && y > z
+    _Ops::iter_swap(__y, __z); // x <= z && y < z
+    __r = 1;
+    if (__c(*__y, *__x)) // if x > y
+    {
+      _Ops::iter_swap(__x, __y); // x < y && y <= z
+      __r = 2;
+    }
+    return __r; // x <= y && y < z
+  }
+  if (__c(*__z, *__y)) // x > y, if y > z
+  {
+    _Ops::iter_swap(__x, __z); // x < y && y < z
+    __r = 1;
+    return __r;
+  }
+  _Ops::iter_swap(__x, __y); // x > y && y <= z
+  __r = 1;                   // x < y && x <= z
+  if (__c(*__z, *__y))       // if y > z
+  {
+    _Ops::iter_swap(__y, __z); // x <= y && y < z
+    __r = 2;
+  }
+  return __r;
+} // x <= y && y <= z
+
+// stable, 3-6 compares, 0-5 swaps
+
+template <class _AlgPolicy, class _Compare, class _ForwardIterator>
+_LIBCPP_HIDE_FROM_ABI void
+__sort4(_ForwardIterator __x1, _ForwardIterator __x2, _ForwardIterator __x3, _ForwardIterator __x4, _Compare __c) {
+  using _Ops = _IterOps<_AlgPolicy>;
+  std::__sort3<_AlgPolicy, _Compare>(__x1, __x2, __x3, __c);
+  if (__c(*__x4, *__x3)) {
+    _Ops::iter_swap(__x3, __x4);
+    if (__c(*__x3, *__x2)) {
+      _Ops::iter_swap(__x2, __x3);
+      if (__c(*__x2, *__x1)) {
+        _Ops::iter_swap(__x1, __x2);
+      }
+    }
+  }
+}
+
+// stable, 4-10 compares, 0-9 swaps
+
+template <class _AlgPolicy, class _Comp, class _ForwardIterator>
+_LIBCPP_HIDE_FROM_ABI void
+__sort5(_ForwardIterator __x1,
+        _ForwardIterator __x2,
+        _ForwardIterator __x3,
+        _ForwardIterator __x4,
+        _ForwardIterator __x5,
+        _Comp __comp) {
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  std::__sort4<_AlgPolicy, _Comp>(__x1, __x2, __x3, __x4, __comp);
+  if (__comp(*__x5, *__x4)) {
+    _Ops::iter_swap(__x4, __x5);
+    if (__comp(*__x4, *__x3)) {
+      _Ops::iter_swap(__x3, __x4);
+      if (__comp(*__x3, *__x2)) {
+        _Ops::iter_swap(__x2, __x3);
+        if (__comp(*__x2, *__x1)) {
+          _Ops::iter_swap(__x1, __x2);
+        }
+      }
+    }
+  }
+}
+
+// The comparator being simple is a prerequisite for using the branchless optimization.
+template <class _Tp>
+struct __is_simple_comparator : false_type {};
+template <>
+struct __is_simple_comparator<__less<>&> : true_type {};
+template <class _Tp>
+struct __is_simple_comparator<less<_Tp>&> : true_type {};
+template <class _Tp>
+struct __is_simple_comparator<greater<_Tp>&> : true_type {};
+#if _LIBCPP_STD_VER >= 20
+template <>
+struct __is_simple_comparator<ranges::less&> : true_type {};
+template <>
+struct __is_simple_comparator<ranges::greater&> : true_type {};
+#endif
+
+template <class _Compare, class _Iter, class _Tp = typename iterator_traits<_Iter>::value_type>
+using __use_branchless_sort =
+    integral_constant<bool,
+                      __libcpp_is_contiguous_iterator<_Iter>::value && sizeof(_Tp) <= sizeof(void*) &&
+                          is_arithmetic<_Tp>::value && __is_simple_comparator<_Compare>::value>;
+
+namespace __detail {
+
+// Size in bits for the bitset in use.
+enum { __block_size = sizeof(uint64_t) * 8 };
+
+} // namespace __detail
+
+// Ensures that __c(*__x, *__y) is true by swapping *__x and *__y if necessary.
+template <class _Compare, class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI void __cond_swap(_RandomAccessIterator __x, _RandomAccessIterator __y, _Compare __c) {
+  // Note: this function behaves correctly even with proxy iterators (because it relies on `value_type`).
+  using value_type = typename iterator_traits<_RandomAccessIterator>::value_type;
+  bool __r         = __c(*__x, *__y);
+  value_type __tmp = __r ? *__x : *__y;
+  *__y             = __r ? *__y : *__x;
+  *__x             = __tmp;
+}
+
+// Ensures that *__x, *__y and *__z are ordered according to the comparator __c,
+// under the assumption that *__y and *__z are already ordered.
+template <class _Compare, class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI void
+__partially_sorted_swap(_RandomAccessIterator __x, _RandomAccessIterator __y, _RandomAccessIterator __z, _Compare __c) {
+  // Note: this function behaves correctly even with proxy iterators (because it relies on `value_type`).
+  using value_type = typename iterator_traits<_RandomAccessIterator>::value_type;
+  bool __r         = __c(*__z, *__x);
+  value_type __tmp = __r ? *__z : *__x;
+  *__z             = __r ? *__x : *__z;
+  __r              = __c(__tmp, *__y);
+  *__x             = __r ? *__x : *__y;
+  *__y             = __r ? *__y : __tmp;
+}
+
+template <class,
+          class _Compare,
+          class _RandomAccessIterator,
+          __enable_if_t<__use_branchless_sort<_Compare, _RandomAccessIterator>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI void __sort3_maybe_branchless(
+    _RandomAccessIterator __x1, _RandomAccessIterator __x2, _RandomAccessIterator __x3, _Compare __c) {
+  std::__cond_swap<_Compare>(__x2, __x3, __c);
+  std::__partially_sorted_swap<_Compare>(__x1, __x2, __x3, __c);
+}
+
+template <class _AlgPolicy,
+          class _Compare,
+          class _RandomAccessIterator,
+          __enable_if_t<!__use_branchless_sort<_Compare, _RandomAccessIterator>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI void __sort3_maybe_branchless(
+    _RandomAccessIterator __x1, _RandomAccessIterator __x2, _RandomAccessIterator __x3, _Compare __c) {
+  std::__sort3<_AlgPolicy, _Compare>(__x1, __x2, __x3, __c);
+}
+
+template <class,
+          class _Compare,
+          class _RandomAccessIterator,
+          __enable_if_t<__use_branchless_sort<_Compare, _RandomAccessIterator>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI void __sort4_maybe_branchless(
+    _RandomAccessIterator __x1,
+    _RandomAccessIterator __x2,
+    _RandomAccessIterator __x3,
+    _RandomAccessIterator __x4,
+    _Compare __c) {
+  std::__cond_swap<_Compare>(__x1, __x3, __c);
+  std::__cond_swap<_Compare>(__x2, __x4, __c);
+  std::__cond_swap<_Compare>(__x1, __x2, __c);
+  std::__cond_swap<_Compare>(__x3, __x4, __c);
+  std::__cond_swap<_Compare>(__x2, __x3, __c);
+}
+
+template <class _AlgPolicy,
+          class _Compare,
+          class _RandomAccessIterator,
+          __enable_if_t<!__use_branchless_sort<_Compare, _RandomAccessIterator>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI void __sort4_maybe_branchless(
+    _RandomAccessIterator __x1,
+    _RandomAccessIterator __x2,
+    _RandomAccessIterator __x3,
+    _RandomAccessIterator __x4,
+    _Compare __c) {
+  std::__sort4<_AlgPolicy, _Compare>(__x1, __x2, __x3, __x4, __c);
+}
+
+template <class _AlgPolicy,
+          class _Compare,
+          class _RandomAccessIterator,
+          __enable_if_t<__use_branchless_sort<_Compare, _RandomAccessIterator>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI void __sort5_maybe_branchless(
+    _RandomAccessIterator __x1,
+    _RandomAccessIterator __x2,
+    _RandomAccessIterator __x3,
+    _RandomAccessIterator __x4,
+    _RandomAccessIterator __x5,
+    _Compare __c) {
+  std::__cond_swap<_Compare>(__x1, __x2, __c);
+  std::__cond_swap<_Compare>(__x4, __x5, __c);
+  std::__partially_sorted_swap<_Compare>(__x3, __x4, __x5, __c);
+  std::__cond_swap<_Compare>(__x2, __x5, __c);
+  std::__partially_sorted_swap<_Compare>(__x1, __x3, __x4, __c);
+  std::__partially_sorted_swap<_Compare>(__x2, __x3, __x4, __c);
+}
+
+template <class _AlgPolicy,
+          class _Compare,
+          class _RandomAccessIterator,
+          __enable_if_t<!__use_branchless_sort<_Compare, _RandomAccessIterator>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI void __sort5_maybe_branchless(
+    _RandomAccessIterator __x1,
+    _RandomAccessIterator __x2,
+    _RandomAccessIterator __x3,
+    _RandomAccessIterator __x4,
+    _RandomAccessIterator __x5,
+    _Compare __c) {
+  std::__sort5<_AlgPolicy, _Compare, _RandomAccessIterator>(
+      std::move(__x1), std::move(__x2), std::move(__x3), std::move(__x4), std::move(__x5), __c);
+}
+
+// Assumes size > 0
+template <class _AlgPolicy, class _Compare, class _BidirectionalIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+__selection_sort(_BidirectionalIterator __first, _BidirectionalIterator __last, _Compare __comp) {
+  _BidirectionalIterator __lm1 = __last;
+  for (--__lm1; __first != __lm1; ++__first) {
+    _BidirectionalIterator __i = std::__min_element<_Compare>(__first, __last, __comp);
+    if (__i != __first)
+      _IterOps<_AlgPolicy>::iter_swap(__first, __i);
+  }
+}
+
+// Sort the iterator range [__first, __last) using the comparator __comp using
+// the insertion sort algorithm.
+template <class _AlgPolicy, class _Compare, class _BidirectionalIterator>
+_LIBCPP_HIDE_FROM_ABI void
+__insertion_sort(_BidirectionalIterator __first, _BidirectionalIterator __last, _Compare __comp) {
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type;
+  if (__first == __last)
+    return;
+  _BidirectionalIterator __i = __first;
+  for (++__i; __i != __last; ++__i) {
+    _BidirectionalIterator __j = __i;
+    --__j;
+    if (__comp(*__i, *__j)) {
+      value_type __t(_Ops::__iter_move(__i));
+      _BidirectionalIterator __k = __j;
+      __j                        = __i;
+      do {
+        *__j = _Ops::__iter_move(__k);
+        __j  = __k;
+      } while (__j != __first && __comp(__t, *--__k));
+      *__j = std::move(__t);
+    }
+  }
+}
+
+// Sort the iterator range [__first, __last) using the comparator __comp using
+// the insertion sort algorithm.  Insertion sort has two loops, outer and inner.
+// The implementation below has no bounds check (unguarded) for the inner loop.
+// Assumes that there is an element in the position (__first - 1) and that each
+// element in the input range is greater or equal to the element at __first - 1.
+template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
+_LIBCPP_HIDE_FROM_ABI void
+__insertion_sort_unguarded(_RandomAccessIterator const __first, _RandomAccessIterator __last, _Compare __comp) {
+  using _Ops = _IterOps<_AlgPolicy>;
+  typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type;
+  if (__first == __last)
+    return;
+  const _RandomAccessIterator __leftmost = __first - difference_type(1);
+  (void)__leftmost; // can be unused when assertions are disabled
+  for (_RandomAccessIterator __i = __first + difference_type(1); __i != __last; ++__i) {
+    _RandomAccessIterator __j = __i - difference_type(1);
+    if (__comp(*__i, *__j)) {
+      value_type __t(_Ops::__iter_move(__i));
+      _RandomAccessIterator __k = __j;
+      __j                       = __i;
+      do {
+        *__j = _Ops::__iter_move(__k);
+        __j  = __k;
+        _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+            __k != __leftmost,
+            "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+      } while (__comp(__t, *--__k)); // No need for bounds check due to the assumption stated above.
+      *__j = std::move(__t);
+    }
+  }
+}
+
+template <class _AlgPolicy, class _Comp, class _RandomAccessIterator>
+_LIBCPP_HIDE_FROM_ABI bool
+__insertion_sort_incomplete(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) {
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  switch (__last - __first) {
+  case 0:
+  case 1:
+    return true;
+  case 2:
+    if (__comp(*--__last, *__first))
+      _Ops::iter_swap(__first, __last);
+    return true;
+  case 3:
+    std::__sort3_maybe_branchless<_AlgPolicy, _Comp>(__first, __first + difference_type(1), --__last, __comp);
+    return true;
+  case 4:
+    std::__sort4_maybe_branchless<_AlgPolicy, _Comp>(
+        __first, __first + difference_type(1), __first + difference_type(2), --__last, __comp);
+    return true;
+  case 5:
+    std::__sort5_maybe_branchless<_AlgPolicy, _Comp>(
+        __first,
+        __first + difference_type(1),
+        __first + difference_type(2),
+        __first + difference_type(3),
+        --__last,
+        __comp);
+    return true;
+  }
+  typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type;
+  _RandomAccessIterator __j = __first + difference_type(2);
+  std::__sort3_maybe_branchless<_AlgPolicy, _Comp>(__first, __first + difference_type(1), __j, __comp);
+  const unsigned __limit = 8;
+  unsigned __count       = 0;
+  for (_RandomAccessIterator __i = __j + difference_type(1); __i != __last; ++__i) {
+    if (__comp(*__i, *__j)) {
+      value_type __t(_Ops::__iter_move(__i));
+      _RandomAccessIterator __k = __j;
+      __j                       = __i;
+      do {
+        *__j = _Ops::__iter_move(__k);
+        __j  = __k;
+      } while (__j != __first && __comp(__t, *--__k));
+      *__j = std::move(__t);
+      if (++__count == __limit)
+        return ++__i == __last;
+    }
+    __j = __i;
+  }
+  return true;
+}
+
+template <class _AlgPolicy, class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI void __swap_bitmap_pos(
+    _RandomAccessIterator __first, _RandomAccessIterator __last, uint64_t& __left_bitset, uint64_t& __right_bitset) {
+  using _Ops = _IterOps<_AlgPolicy>;
+  typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  // Swap one pair on each iteration as long as both bitsets have at least one
+  // element for swapping.
+  while (__left_bitset != 0 && __right_bitset != 0) {
+    difference_type __tz_left  = __libcpp_ctz(__left_bitset);
+    __left_bitset              = __libcpp_blsr(__left_bitset);
+    difference_type __tz_right = __libcpp_ctz(__right_bitset);
+    __right_bitset             = __libcpp_blsr(__right_bitset);
+    _Ops::iter_swap(__first + __tz_left, __last - __tz_right);
+  }
+}
+
+template <class _Compare,
+          class _RandomAccessIterator,
+          class _ValueType = typename iterator_traits<_RandomAccessIterator>::value_type>
+inline _LIBCPP_HIDE_FROM_ABI void
+__populate_left_bitset(_RandomAccessIterator __first, _Compare __comp, _ValueType& __pivot, uint64_t& __left_bitset) {
+  // Possible vectorization. With a proper "-march" flag, the following loop
+  // will be compiled into a set of SIMD instructions.
+  _RandomAccessIterator __iter = __first;
+  for (int __j = 0; __j < __detail::__block_size;) {
+    bool __comp_result = !__comp(*__iter, __pivot);
+    __left_bitset |= (static_cast<uint64_t>(__comp_result) << __j);
+    __j++;
+    ++__iter;
+  }
+}
+
+template <class _Compare,
+          class _RandomAccessIterator,
+          class _ValueType = typename iterator_traits<_RandomAccessIterator>::value_type>
+inline _LIBCPP_HIDE_FROM_ABI void
+__populate_right_bitset(_RandomAccessIterator __lm1, _Compare __comp, _ValueType& __pivot, uint64_t& __right_bitset) {
+  // Possible vectorization. With a proper "-march" flag, the following loop
+  // will be compiled into a set of SIMD instructions.
+  _RandomAccessIterator __iter = __lm1;
+  for (int __j = 0; __j < __detail::__block_size;) {
+    bool __comp_result = __comp(*__iter, __pivot);
+    __right_bitset |= (static_cast<uint64_t>(__comp_result) << __j);
+    __j++;
+    --__iter;
+  }
+}
+
+template <class _AlgPolicy,
+          class _Compare,
+          class _RandomAccessIterator,
+          class _ValueType = typename iterator_traits<_RandomAccessIterator>::value_type>
+inline _LIBCPP_HIDE_FROM_ABI void __bitset_partition_partial_blocks(
+    _RandomAccessIterator& __first,
+    _RandomAccessIterator& __lm1,
+    _Compare __comp,
+    _ValueType& __pivot,
+    uint64_t& __left_bitset,
+    uint64_t& __right_bitset) {
+  typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  difference_type __remaining_len = __lm1 - __first + 1;
+  difference_type __l_size;
+  difference_type __r_size;
+  if (__left_bitset == 0 && __right_bitset == 0) {
+    __l_size = __remaining_len / 2;
+    __r_size = __remaining_len - __l_size;
+  } else if (__left_bitset == 0) {
+    // We know at least one side is a full block.
+    __l_size = __remaining_len - __detail::__block_size;
+    __r_size = __detail::__block_size;
+  } else { // if (__right_bitset == 0)
+    __l_size = __detail::__block_size;
+    __r_size = __remaining_len - __detail::__block_size;
+  }
+  // Record the comparison outcomes for the elements currently on the left side.
+  if (__left_bitset == 0) {
+    _RandomAccessIterator __iter = __first;
+    for (int __j = 0; __j < __l_size; __j++) {
+      bool __comp_result = !__comp(*__iter, __pivot);
+      __left_bitset |= (static_cast<uint64_t>(__comp_result) << __j);
+      ++__iter;
+    }
+  }
+  // Record the comparison outcomes for the elements currently on the right
+  // side.
+  if (__right_bitset == 0) {
+    _RandomAccessIterator __iter = __lm1;
+    for (int __j = 0; __j < __r_size; __j++) {
+      bool __comp_result = __comp(*__iter, __pivot);
+      __right_bitset |= (static_cast<uint64_t>(__comp_result) << __j);
+      --__iter;
+    }
+  }
+  std::__swap_bitmap_pos<_AlgPolicy, _RandomAccessIterator>(__first, __lm1, __left_bitset, __right_bitset);
+  __first += (__left_bitset == 0) ? __l_size : 0;
+  __lm1 -= (__right_bitset == 0) ? __r_size : 0;
+}
+
+template <class _AlgPolicy, class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI void __swap_bitmap_pos_within(
+    _RandomAccessIterator& __first, _RandomAccessIterator& __lm1, uint64_t& __left_bitset, uint64_t& __right_bitset) {
+  using _Ops = _IterOps<_AlgPolicy>;
+  typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  if (__left_bitset) {
+    // Swap within the left side.  Need to find set positions in the reverse
+    // order.
+    while (__left_bitset != 0) {
+      difference_type __tz_left = __detail::__block_size - 1 - __libcpp_clz(__left_bitset);
+      __left_bitset &= (static_cast<uint64_t>(1) << __tz_left) - 1;
+      _RandomAccessIterator __it = __first + __tz_left;
+      if (__it != __lm1) {
+        _Ops::iter_swap(__it, __lm1);
+      }
+      --__lm1;
+    }
+    __first = __lm1 + difference_type(1);
+  } else if (__right_bitset) {
+    // Swap within the right side.  Need to find set positions in the reverse
+    // order.
+    while (__right_bitset != 0) {
+      difference_type __tz_right = __detail::__block_size - 1 - __libcpp_clz(__right_bitset);
+      __right_bitset &= (static_cast<uint64_t>(1) << __tz_right) - 1;
+      _RandomAccessIterator __it = __lm1 - __tz_right;
+      if (__it != __first) {
+        _Ops::iter_swap(__it, __first);
+      }
+      ++__first;
+    }
+  }
+}
+
+// Partition [__first, __last) using the comparator __comp.  *__first has the
+// chosen pivot.  Elements that are equivalent are kept to the left of the
+// pivot.  Returns the iterator for the pivot and a bool value which is true if
+// the provided range is already sorted, false otherwise.  We assume that the
+// length of the range is at least three elements.
+//
+// __bitset_partition uses bitsets for storing outcomes of the comparisons
+// between the pivot and other elements.
+template <class _AlgPolicy, class _RandomAccessIterator, class _Compare>
+_LIBCPP_HIDE_FROM_ABI std::pair<_RandomAccessIterator, bool>
+__bitset_partition(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+  using _Ops = _IterOps<_AlgPolicy>;
+  typedef typename std::iterator_traits<_RandomAccessIterator>::value_type value_type;
+  typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  _LIBCPP_ASSERT_INTERNAL(__last - __first >= difference_type(3), "");
+  const _RandomAccessIterator __begin = __first; // used for bounds checking, those are not moved around
+  const _RandomAccessIterator __end   = __last;
+  (void)__end; //
+
+  value_type __pivot(_Ops::__iter_move(__first));
+  // Find the first element greater than the pivot.
+  if (__comp(__pivot, *(__last - difference_type(1)))) {
+    // Not guarded since we know the last element is greater than the pivot.
+    do {
+      ++__first;
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          __first != __end,
+          "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+    } while (!__comp(__pivot, *__first));
+  } else {
+    while (++__first < __last && !__comp(__pivot, *__first)) {
+    }
+  }
+  // Find the last element less than or equal to the pivot.
+  if (__first < __last) {
+    // It will be always guarded because __introsort will do the median-of-three
+    // before calling this.
+    do {
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          __last != __begin,
+          "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+      --__last;
+    } while (__comp(__pivot, *__last));
+  }
+  // If the first element greater than the pivot is at or after the
+  // last element less than or equal to the pivot, then we have covered the
+  // entire range without swapping elements.  This implies the range is already
+  // partitioned.
+  bool __already_partitioned = __first >= __last;
+  if (!__already_partitioned) {
+    _Ops::iter_swap(__first, __last);
+    ++__first;
+  }
+
+  // In [__first, __last) __last is not inclusive. From now on, it uses last
+  // minus one to be inclusive on both sides.
+  _RandomAccessIterator __lm1 = __last - difference_type(1);
+  uint64_t __left_bitset      = 0;
+  uint64_t __right_bitset     = 0;
+
+  // Reminder: length = __lm1 - __first + 1.
+  while (__lm1 - __first >= 2 * __detail::__block_size - 1) {
+    // Record the comparison outcomes for the elements currently on the left
+    // side.
+    if (__left_bitset == 0)
+      std::__populate_left_bitset<_Compare>(__first, __comp, __pivot, __left_bitset);
+    // Record the comparison outcomes for the elements currently on the right
+    // side.
+    if (__right_bitset == 0)
+      std::__populate_right_bitset<_Compare>(__lm1, __comp, __pivot, __right_bitset);
+    // Swap the elements recorded to be the candidates for swapping in the
+    // bitsets.
+    std::__swap_bitmap_pos<_AlgPolicy, _RandomAccessIterator>(__first, __lm1, __left_bitset, __right_bitset);
+    // Only advance the iterator if all the elements that need to be moved to
+    // other side were moved.
+    __first += (__left_bitset == 0) ? difference_type(__detail::__block_size) : difference_type(0);
+    __lm1 -= (__right_bitset == 0) ? difference_type(__detail::__block_size) : difference_type(0);
+  }
+  // Now, we have a less-than a block worth of elements on at least one of the
+  // sides.
+  std::__bitset_partition_partial_blocks<_AlgPolicy, _Compare>(
+      __first, __lm1, __comp, __pivot, __left_bitset, __right_bitset);
+  // At least one the bitsets would be empty.  For the non-empty one, we need to
+  // properly partition the elements that appear within that bitset.
+  std::__swap_bitmap_pos_within<_AlgPolicy>(__first, __lm1, __left_bitset, __right_bitset);
+
+  // Move the pivot to its correct position.
+  _RandomAccessIterator __pivot_pos = __first - difference_type(1);
+  if (__begin != __pivot_pos) {
+    *__begin = _Ops::__iter_move(__pivot_pos);
+  }
+  *__pivot_pos = std::move(__pivot);
+  return std::make_pair(__pivot_pos, __already_partitioned);
+}
+
+// Partition [__first, __last) using the comparator __comp.  *__first has the
+// chosen pivot.  Elements that are equivalent are kept to the right of the
+// pivot.  Returns the iterator for the pivot and a bool value which is true if
+// the provided range is already sorted, false otherwise.  We assume that the
+// length of the range is at least three elements.
+template <class _AlgPolicy, class _RandomAccessIterator, class _Compare>
+_LIBCPP_HIDE_FROM_ABI std::pair<_RandomAccessIterator, bool>
+__partition_with_equals_on_right(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+  using _Ops = _IterOps<_AlgPolicy>;
+  typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  typedef typename std::iterator_traits<_RandomAccessIterator>::value_type value_type;
+  _LIBCPP_ASSERT_INTERNAL(__last - __first >= difference_type(3), "");
+  const _RandomAccessIterator __begin = __first; // used for bounds checking, those are not moved around
+  const _RandomAccessIterator __end   = __last;
+  (void)__end; //
+  value_type __pivot(_Ops::__iter_move(__first));
+  // Find the first element greater or equal to the pivot.  It will be always
+  // guarded because __introsort will do the median-of-three before calling
+  // this.
+  do {
+    ++__first;
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __first != __end,
+        "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+  } while (__comp(*__first, __pivot));
+
+  // Find the last element less than the pivot.
+  if (__begin == __first - difference_type(1)) {
+    while (__first < __last && !__comp(*--__last, __pivot))
+      ;
+  } else {
+    // Guarded.
+    do {
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          __last != __begin,
+          "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+      --__last;
+    } while (!__comp(*__last, __pivot));
+  }
+
+  // If the first element greater than or equal to the pivot is at or after the
+  // last element less than the pivot, then we have covered the entire range
+  // without swapping elements.  This implies the range is already partitioned.
+  bool __already_partitioned = __first >= __last;
+  // Go through the remaining elements.  Swap pairs of elements (one to the
+  // right of the pivot and the other to left of the pivot) that are not on the
+  // correct side of the pivot.
+  while (__first < __last) {
+    _Ops::iter_swap(__first, __last);
+    do {
+      ++__first;
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          __first != __end,
+          "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+    } while (__comp(*__first, __pivot));
+    do {
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          __last != __begin,
+          "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+      --__last;
+    } while (!__comp(*__last, __pivot));
+  }
+  // Move the pivot to its correct position.
+  _RandomAccessIterator __pivot_pos = __first - difference_type(1);
+  if (__begin != __pivot_pos) {
+    *__begin = _Ops::__iter_move(__pivot_pos);
+  }
+  *__pivot_pos = std::move(__pivot);
+  return std::make_pair(__pivot_pos, __already_partitioned);
+}
+
+// Similar to the above function.  Elements equivalent to the pivot are put to
+// the left of the pivot.  Returns the iterator to the pivot element.
+template <class _AlgPolicy, class _RandomAccessIterator, class _Compare>
+_LIBCPP_HIDE_FROM_ABI _RandomAccessIterator
+__partition_with_equals_on_left(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+  using _Ops = _IterOps<_AlgPolicy>;
+  typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  typedef typename std::iterator_traits<_RandomAccessIterator>::value_type value_type;
+  const _RandomAccessIterator __begin = __first; // used for bounds checking, those are not moved around
+  const _RandomAccessIterator __end   = __last;
+  (void)__end; //
+  value_type __pivot(_Ops::__iter_move(__first));
+  if (__comp(__pivot, *(__last - difference_type(1)))) {
+    // Guarded.
+    do {
+      ++__first;
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          __first != __end,
+          "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+    } while (!__comp(__pivot, *__first));
+  } else {
+    while (++__first < __last && !__comp(__pivot, *__first)) {
+    }
+  }
+
+  if (__first < __last) {
+    // It will be always guarded because __introsort will do the
+    // median-of-three before calling this.
+    do {
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          __last != __begin,
+          "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+      --__last;
+    } while (__comp(__pivot, *__last));
+  }
+  while (__first < __last) {
+    _Ops::iter_swap(__first, __last);
+    do {
+      ++__first;
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          __first != __end,
+          "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+    } while (!__comp(__pivot, *__first));
+    do {
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          __last != __begin,
+          "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+      --__last;
+    } while (__comp(__pivot, *__last));
+  }
+  _RandomAccessIterator __pivot_pos = __first - difference_type(1);
+  if (__begin != __pivot_pos) {
+    *__begin = _Ops::__iter_move(__pivot_pos);
+  }
+  *__pivot_pos = std::move(__pivot);
+  return __first;
+}
+
+// The main sorting function.  Implements introsort combined with other ideas:
+//  - option of using block quick sort for partitioning,
+//  - guarded and unguarded insertion sort for small lengths,
+//  - Tuckey's ninther technique for computing the pivot,
+//  - check on whether partition was not required.
+// The implementation is partly based on Orson Peters' pattern-defeating
+// quicksort, published at: <https://github.com/orlp/pdqsort>.
+template <class _AlgPolicy, class _Compare, class _RandomAccessIterator, bool _UseBitSetPartition>
+void __introsort(_RandomAccessIterator __first,
+                 _RandomAccessIterator __last,
+                 _Compare __comp,
+                 typename iterator_traits<_RandomAccessIterator>::difference_type __depth,
+                 bool __leftmost = true) {
+  using _Ops = _IterOps<_AlgPolicy>;
+  typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  using _Comp_ref = __comp_ref_type<_Compare>;
+  // Upper bound for using insertion sort for sorting.
+  _LIBCPP_CONSTEXPR difference_type __limit = 24;
+  // Lower bound for using Tuckey's ninther technique for median computation.
+  _LIBCPP_CONSTEXPR difference_type __ninther_threshold = 128;
+  while (true) {
+    difference_type __len = __last - __first;
+    switch (__len) {
+    case 0:
+    case 1:
+      return;
+    case 2:
+      if (__comp(*--__last, *__first))
+        _Ops::iter_swap(__first, __last);
+      return;
+    case 3:
+      std::__sort3_maybe_branchless<_AlgPolicy, _Compare>(__first, __first + difference_type(1), --__last, __comp);
+      return;
+    case 4:
+      std::__sort4_maybe_branchless<_AlgPolicy, _Compare>(
+          __first, __first + difference_type(1), __first + difference_type(2), --__last, __comp);
+      return;
+    case 5:
+      std::__sort5_maybe_branchless<_AlgPolicy, _Compare>(
+          __first,
+          __first + difference_type(1),
+          __first + difference_type(2),
+          __first + difference_type(3),
+          --__last,
+          __comp);
+      return;
+    }
+    // Use insertion sort if the length of the range is below the specified limit.
+    if (__len < __limit) {
+      if (__leftmost) {
+        std::__insertion_sort<_AlgPolicy, _Compare>(__first, __last, __comp);
+      } else {
+        std::__insertion_sort_unguarded<_AlgPolicy, _Compare>(__first, __last, __comp);
+      }
+      return;
+    }
+    if (__depth == 0) {
+      // Fallback to heap sort as Introsort suggests.
+      std::__partial_sort<_AlgPolicy, _Compare>(__first, __last, __last, __comp);
+      return;
+    }
+    --__depth;
+    {
+      difference_type __half_len = __len / 2;
+      // Use Tuckey's ninther technique or median of 3 for pivot selection
+      // depending on the length of the range being sorted.
+      if (__len > __ninther_threshold) {
+        std::__sort3<_AlgPolicy, _Compare>(__first, __first + __half_len, __last - difference_type(1), __comp);
+        std::__sort3<_AlgPolicy, _Compare>(
+            __first + difference_type(1), __first + (__half_len - 1), __last - difference_type(2), __comp);
+        std::__sort3<_AlgPolicy, _Compare>(
+            __first + difference_type(2), __first + (__half_len + 1), __last - difference_type(3), __comp);
+        std::__sort3<_AlgPolicy, _Compare>(
+            __first + (__half_len - 1), __first + __half_len, __first + (__half_len + 1), __comp);
+        _Ops::iter_swap(__first, __first + __half_len);
+      } else {
+        std::__sort3<_AlgPolicy, _Compare>(__first + __half_len, __first, __last - difference_type(1), __comp);
+      }
+    }
+    // The elements to the left of the current iterator range are already
+    // sorted.  If the current iterator range to be sorted is not the
+    // leftmost part of the entire iterator range and the pivot is same as
+    // the highest element in the range to the left, then we know that all
+    // the elements in the range [first, pivot] would be equal to the pivot,
+    // assuming the equal elements are put on the left side when
+    // partitioned.  This also means that we do not need to sort the left
+    // side of the partition.
+    if (!__leftmost && !__comp(*(__first - difference_type(1)), *__first)) {
+      __first = std::__partition_with_equals_on_left<_AlgPolicy, _RandomAccessIterator, _Comp_ref>(
+          __first, __last, _Comp_ref(__comp));
+      continue;
+    }
+    // Use bitset partition only if asked for.
+    auto __ret                = _UseBitSetPartition
+                                  ? std::__bitset_partition<_AlgPolicy, _RandomAccessIterator, _Compare>(__first, __last, __comp)
+                                  : std::__partition_with_equals_on_right<_AlgPolicy, _RandomAccessIterator, _Compare>(
+                         __first, __last, __comp);
+    _RandomAccessIterator __i = __ret.first;
+    // [__first, __i) < *__i and *__i <= [__i+1, __last)
+    // If we were given a perfect partition, see if insertion sort is quick...
+    if (__ret.second) {
+      bool __fs = std::__insertion_sort_incomplete<_AlgPolicy, _Compare>(__first, __i, __comp);
+      if (std::__insertion_sort_incomplete<_AlgPolicy, _Compare>(__i + difference_type(1), __last, __comp)) {
+        if (__fs)
+          return;
+        __last = __i;
+        continue;
+      } else {
+        if (__fs) {
+          __first = ++__i;
+          continue;
+        }
+      }
+    }
+    // Sort the left partiton recursively and the right partition with tail recursion elimination.
+    std::__introsort<_AlgPolicy, _Compare, _RandomAccessIterator, _UseBitSetPartition>(
+        __first, __i, __comp, __depth, __leftmost);
+    __leftmost = false;
+    __first    = ++__i;
+  }
+}
+
+template <typename _Number>
+inline _LIBCPP_HIDE_FROM_ABI _Number __log2i(_Number __n) {
+  if (__n == 0)
+    return 0;
+  if (sizeof(__n) <= sizeof(unsigned))
+    return sizeof(unsigned) * CHAR_BIT - 1 - __libcpp_clz(static_cast<unsigned>(__n));
+  if (sizeof(__n) <= sizeof(unsigned long))
+    return sizeof(unsigned long) * CHAR_BIT - 1 - __libcpp_clz(static_cast<unsigned long>(__n));
+  if (sizeof(__n) <= sizeof(unsigned long long))
+    return sizeof(unsigned long long) * CHAR_BIT - 1 - __libcpp_clz(static_cast<unsigned long long>(__n));
+
+  _Number __log2 = 0;
+  while (__n > 1) {
+    __log2++;
+    __n >>= 1;
+  }
+  return __log2;
+}
+
+template <class _Comp, class _RandomAccessIterator>
+void __sort(_RandomAccessIterator, _RandomAccessIterator, _Comp);
+
+extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less<char>&, char*>(char*, char*, __less<char>&);
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less<wchar_t>&, wchar_t*>(wchar_t*, wchar_t*, __less<wchar_t>&);
+#endif
+extern template _LIBCPP_EXPORTED_FROM_ABI void
+__sort<__less<signed char>&, signed char*>(signed char*, signed char*, __less<signed char>&);
+extern template _LIBCPP_EXPORTED_FROM_ABI void
+__sort<__less<unsigned char>&, unsigned char*>(unsigned char*, unsigned char*, __less<unsigned char>&);
+extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less<short>&, short*>(short*, short*, __less<short>&);
+extern template _LIBCPP_EXPORTED_FROM_ABI void
+__sort<__less<unsigned short>&, unsigned short*>(unsigned short*, unsigned short*, __less<unsigned short>&);
+extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less<int>&, int*>(int*, int*, __less<int>&);
+extern template _LIBCPP_EXPORTED_FROM_ABI void
+__sort<__less<unsigned>&, unsigned*>(unsigned*, unsigned*, __less<unsigned>&);
+extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less<long>&, long*>(long*, long*, __less<long>&);
+extern template _LIBCPP_EXPORTED_FROM_ABI void
+__sort<__less<unsigned long>&, unsigned long*>(unsigned long*, unsigned long*, __less<unsigned long>&);
+extern template _LIBCPP_EXPORTED_FROM_ABI void
+__sort<__less<long long>&, long long*>(long long*, long long*, __less<long long>&);
+extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less<unsigned long long>&, unsigned long long*>(
+    unsigned long long*, unsigned long long*, __less<unsigned long long>&);
+extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less<float>&, float*>(float*, float*, __less<float>&);
+extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less<double>&, double*>(double*, double*, __less<double>&);
+extern template _LIBCPP_EXPORTED_FROM_ABI void
+__sort<__less<long double>&, long double*>(long double*, long double*, __less<long double>&);
+
+template <class _AlgPolicy, class _RandomAccessIterator, class _Comp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+__sort_dispatch(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp& __comp) {
+  typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  difference_type __depth_limit = 2 * std::__log2i(__last - __first);
+
+  // Only use bitset partitioning for arithmetic types.  We should also check
+  // that the default comparator is in use so that we are sure that there are no
+  // branches in the comparator.
+  std::__introsort<_AlgPolicy,
+                   _Comp&,
+                   _RandomAccessIterator,
+                   __use_branchless_sort<_Comp, _RandomAccessIterator>::value>(__first, __last, __comp, __depth_limit);
+}
+
+template <class _Type, class... _Options>
+using __is_any_of = _Or<is_same<_Type, _Options>...>;
+
+template <class _Type>
+using __sort_is_specialized_in_library = __is_any_of<
+    _Type,
+    char,
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+    wchar_t,
+#endif
+    signed char,
+    unsigned char,
+    short,
+    unsigned short,
+    int,
+    unsigned int,
+    long,
+    unsigned long,
+    long long,
+    unsigned long long,
+    float,
+    double,
+    long double>;
+
+template <class _AlgPolicy, class _Type, __enable_if_t<__sort_is_specialized_in_library<_Type>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void __sort_dispatch(_Type* __first, _Type* __last, __less<>&) {
+  __less<_Type> __comp;
+  std::__sort<__less<_Type>&, _Type*>(__first, __last, __comp);
+}
+
+template <class _AlgPolicy, class _Type, __enable_if_t<__sort_is_specialized_in_library<_Type>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void __sort_dispatch(_Type* __first, _Type* __last, less<_Type>&) {
+  __less<_Type> __comp;
+  std::__sort<__less<_Type>&, _Type*>(__first, __last, __comp);
+}
+
+#if _LIBCPP_STD_VER >= 14
+template <class _AlgPolicy, class _Type, __enable_if_t<__sort_is_specialized_in_library<_Type>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void __sort_dispatch(_Type* __first, _Type* __last, less<>&) {
+  __less<_Type> __comp;
+  std::__sort<__less<_Type>&, _Type*>(__first, __last, __comp);
+}
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+template <class _AlgPolicy, class _Type, __enable_if_t<__sort_is_specialized_in_library<_Type>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void __sort_dispatch(_Type* __first, _Type* __last, ranges::less&) {
+  __less<_Type> __comp;
+  std::__sort<__less<_Type>&, _Type*>(__first, __last, __comp);
+}
+#endif
+
+template <class _AlgPolicy, class _RandomAccessIterator, class _Comp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+__sort_impl(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp& __comp) {
+  std::__debug_randomize_range<_AlgPolicy>(__first, __last);
+
+  if (__libcpp_is_constant_evaluated()) {
+    std::__partial_sort<_AlgPolicy>(
+        std::__unwrap_iter(__first), std::__unwrap_iter(__last), std::__unwrap_iter(__last), __comp);
+  } else {
+    std::__sort_dispatch<_AlgPolicy>(std::__unwrap_iter(__first), std::__unwrap_iter(__last), __comp);
+  }
+  std::__check_strict_weak_ordering_sorted(std::__unwrap_iter(__first), std::__unwrap_iter(__last), __comp);
+}
+
+template <class _RandomAccessIterator, class _Comp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) {
+  std::__sort_impl<_ClassicAlgPolicy>(std::move(__first), std::move(__last), __comp);
+}
+
+template <class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+sort(_RandomAccessIterator __first, _RandomAccessIterator __last) {
+  std::sort(__first, __last, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_SORT_H
diff --git a/libcxx/include/__cxx03/__algorithm/sort_heap.h b/libcxx/include/__cxx03/__algorithm/sort_heap.h
new file mode 100644
index 000000000000..f20b110c7fd1
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/sort_heap.h
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SORT_HEAP_H
+#define _LIBCPP___ALGORITHM_SORT_HEAP_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/pop_heap.h>
+#include <__config>
+#include <__debug_utils/strict_weak_ordering_check.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_constructible.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+__sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare&& __comp) {
+  _RandomAccessIterator __saved_last   = __last;
+  __comp_ref_type<_Compare> __comp_ref = __comp;
+
+  using difference_type = typename iterator_traits<_RandomAccessIterator>::difference_type;
+  for (difference_type __n = __last - __first; __n > 1; --__last, (void)--__n)
+    std::__pop_heap<_AlgPolicy>(__first, __last, __comp_ref, __n);
+  std::__check_strict_weak_ordering_sorted(__first, __saved_last, __comp_ref);
+}
+
+template <class _RandomAccessIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+  static_assert(std::is_copy_constructible<_RandomAccessIterator>::value, "Iterators must be copy constructible.");
+  static_assert(std::is_copy_assignable<_RandomAccessIterator>::value, "Iterators must be copy assignable.");
+
+  std::__sort_heap<_ClassicAlgPolicy>(std::move(__first), std::move(__last), __comp);
+}
+
+template <class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) {
+  std::sort_heap(std::move(__first), std::move(__last), __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_SORT_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/stable_partition.h b/libcxx/include/__cxx03/__algorithm/stable_partition.h
new file mode 100644
index 000000000000..8bb1eaf2d224
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/stable_partition.h
@@ -0,0 +1,307 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_STABLE_PARTITION_H
+#define _LIBCPP___ALGORITHM_STABLE_PARTITION_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/rotate.h>
+#include <__config>
+#include <__iterator/advance.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/destruct_n.h>
+#include <__memory/temporary_buffer.h>
+#include <__memory/unique_ptr.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <new>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Predicate, class _ForwardIterator, class _Distance, class _Pair>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator __stable_partition_impl(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _Predicate __pred,
+    _Distance __len,
+    _Pair __p,
+    forward_iterator_tag __fit) {
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  // *__first is known to be false
+  // __len >= 1
+  if (__len == 1)
+    return __first;
+  if (__len == 2) {
+    _ForwardIterator __m = __first;
+    if (__pred(*++__m)) {
+      _Ops::iter_swap(__first, __m);
+      return __m;
+    }
+    return __first;
+  }
+  if (__len <= __p.second) { // The buffer is big enough to use
+    typedef typename iterator_traits<_ForwardIterator>::value_type value_type;
+    __destruct_n __d(0);
+    unique_ptr<value_type, __destruct_n&> __h(__p.first, __d);
+    // Move the falses into the temporary buffer, and the trues to the front of the line
+    // Update __first to always point to the end of the trues
+    value_type* __t = __p.first;
+    ::new ((void*)__t) value_type(_Ops::__iter_move(__first));
+    __d.template __incr<value_type>();
+    ++__t;
+    _ForwardIterator __i = __first;
+    while (++__i != __last) {
+      if (__pred(*__i)) {
+        *__first = _Ops::__iter_move(__i);
+        ++__first;
+      } else {
+        ::new ((void*)__t) value_type(_Ops::__iter_move(__i));
+        __d.template __incr<value_type>();
+        ++__t;
+      }
+    }
+    // All trues now at start of range, all falses in buffer
+    // Move falses back into range, but don't mess up __first which points to first false
+    __i = __first;
+    for (value_type* __t2 = __p.first; __t2 < __t; ++__t2, (void)++__i)
+      *__i = _Ops::__iter_move(__t2);
+    // __h destructs moved-from values out of the temp buffer, but doesn't deallocate buffer
+    return __first;
+  }
+  // Else not enough buffer, do in place
+  // __len >= 3
+  _ForwardIterator __m = __first;
+  _Distance __len2     = __len / 2; // __len2 >= 2
+  _Ops::advance(__m, __len2);
+  // recurse on [__first, __m), *__first know to be false
+  // F?????????????????
+  // f       m         l
+  _ForwardIterator __first_false =
+      std::__stable_partition_impl<_AlgPolicy, _Predicate&>(__first, __m, __pred, __len2, __p, __fit);
+  // TTTFFFFF??????????
+  // f  ff   m         l
+  // recurse on [__m, __last], except increase __m until *(__m) is false, *__last know to be true
+  _ForwardIterator __m1           = __m;
+  _ForwardIterator __second_false = __last;
+  _Distance __len_half            = __len - __len2;
+  while (__pred(*__m1)) {
+    if (++__m1 == __last)
+      goto __second_half_done;
+    --__len_half;
+  }
+  // TTTFFFFFTTTF??????
+  // f  ff   m  m1     l
+  __second_false = std::__stable_partition_impl<_AlgPolicy, _Predicate&>(__m1, __last, __pred, __len_half, __p, __fit);
+__second_half_done:
+  // TTTFFFFFTTTTTFFFFF
+  // f  ff   m    sf   l
+  return std::__rotate<_AlgPolicy>(__first_false, __m, __second_false).first;
+  // TTTTTTTTFFFFFFFFFF
+  //         |
+}
+
+template <class _AlgPolicy, class _Predicate, class _ForwardIterator>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator
+__stable_partition_impl(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred, forward_iterator_tag) {
+  typedef typename iterator_traits<_ForwardIterator>::difference_type difference_type;
+  typedef typename iterator_traits<_ForwardIterator>::value_type value_type;
+
+  const difference_type __alloc_limit = 3; // might want to make this a function of trivial assignment
+  // Either prove all true and return __first or point to first false
+  while (true) {
+    if (__first == __last)
+      return __first;
+    if (!__pred(*__first))
+      break;
+    ++__first;
+  }
+  // We now have a reduced range [__first, __last)
+  // *__first is known to be false
+  difference_type __len = _IterOps<_AlgPolicy>::distance(__first, __last);
+  pair<value_type*, ptrdiff_t> __p(0, 0);
+  unique_ptr<value_type, __return_temporary_buffer> __h;
+  if (__len >= __alloc_limit) {
+    // TODO: Remove the use of std::get_temporary_buffer
+    _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+    __p = std::get_temporary_buffer<value_type>(__len);
+    _LIBCPP_SUPPRESS_DEPRECATED_POP
+    __h.reset(__p.first);
+  }
+  return std::__stable_partition_impl<_AlgPolicy, _Predicate&>(
+      std::move(__first), std::move(__last), __pred, __len, __p, forward_iterator_tag());
+}
+
+template <class _AlgPolicy, class _Predicate, class _BidirectionalIterator, class _Distance, class _Pair>
+_BidirectionalIterator __stable_partition_impl(
+    _BidirectionalIterator __first,
+    _BidirectionalIterator __last,
+    _Predicate __pred,
+    _Distance __len,
+    _Pair __p,
+    bidirectional_iterator_tag __bit) {
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  // *__first is known to be false
+  // *__last is known to be true
+  // __len >= 2
+  if (__len == 2) {
+    _Ops::iter_swap(__first, __last);
+    return __last;
+  }
+  if (__len == 3) {
+    _BidirectionalIterator __m = __first;
+    if (__pred(*++__m)) {
+      _Ops::iter_swap(__first, __m);
+      _Ops::iter_swap(__m, __last);
+      return __last;
+    }
+    _Ops::iter_swap(__m, __last);
+    _Ops::iter_swap(__first, __m);
+    return __m;
+  }
+  if (__len <= __p.second) { // The buffer is big enough to use
+    typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type;
+    __destruct_n __d(0);
+    unique_ptr<value_type, __destruct_n&> __h(__p.first, __d);
+    // Move the falses into the temporary buffer, and the trues to the front of the line
+    // Update __first to always point to the end of the trues
+    value_type* __t = __p.first;
+    ::new ((void*)__t) value_type(_Ops::__iter_move(__first));
+    __d.template __incr<value_type>();
+    ++__t;
+    _BidirectionalIterator __i = __first;
+    while (++__i != __last) {
+      if (__pred(*__i)) {
+        *__first = _Ops::__iter_move(__i);
+        ++__first;
+      } else {
+        ::new ((void*)__t) value_type(_Ops::__iter_move(__i));
+        __d.template __incr<value_type>();
+        ++__t;
+      }
+    }
+    // move *__last, known to be true
+    *__first = _Ops::__iter_move(__i);
+    __i      = ++__first;
+    // All trues now at start of range, all falses in buffer
+    // Move falses back into range, but don't mess up __first which points to first false
+    for (value_type* __t2 = __p.first; __t2 < __t; ++__t2, (void)++__i)
+      *__i = _Ops::__iter_move(__t2);
+    // __h destructs moved-from values out of the temp buffer, but doesn't deallocate buffer
+    return __first;
+  }
+  // Else not enough buffer, do in place
+  // __len >= 4
+  _BidirectionalIterator __m = __first;
+  _Distance __len2           = __len / 2; // __len2 >= 2
+  _Ops::advance(__m, __len2);
+  // recurse on [__first, __m-1], except reduce __m-1 until *(__m-1) is true, *__first know to be false
+  // F????????????????T
+  // f       m        l
+  _BidirectionalIterator __m1          = __m;
+  _BidirectionalIterator __first_false = __first;
+  _Distance __len_half                 = __len2;
+  while (!__pred(*--__m1)) {
+    if (__m1 == __first)
+      goto __first_half_done;
+    --__len_half;
+  }
+  // F???TFFF?????????T
+  // f   m1  m        l
+  __first_false = std::__stable_partition_impl<_AlgPolicy, _Predicate&>(__first, __m1, __pred, __len_half, __p, __bit);
+__first_half_done:
+  // TTTFFFFF?????????T
+  // f  ff   m        l
+  // recurse on [__m, __last], except increase __m until *(__m) is false, *__last know to be true
+  __m1                                  = __m;
+  _BidirectionalIterator __second_false = __last;
+  ++__second_false;
+  __len_half = __len - __len2;
+  while (__pred(*__m1)) {
+    if (++__m1 == __last)
+      goto __second_half_done;
+    --__len_half;
+  }
+  // TTTFFFFFTTTF?????T
+  // f  ff   m  m1    l
+  __second_false = std::__stable_partition_impl<_AlgPolicy, _Predicate&>(__m1, __last, __pred, __len_half, __p, __bit);
+__second_half_done:
+  // TTTFFFFFTTTTTFFFFF
+  // f  ff   m    sf  l
+  return std::__rotate<_AlgPolicy>(__first_false, __m, __second_false).first;
+  // TTTTTTTTFFFFFFFFFF
+  //         |
+}
+
+template <class _AlgPolicy, class _Predicate, class _BidirectionalIterator>
+_LIBCPP_HIDE_FROM_ABI _BidirectionalIterator __stable_partition_impl(
+    _BidirectionalIterator __first, _BidirectionalIterator __last, _Predicate __pred, bidirectional_iterator_tag) {
+  typedef typename iterator_traits<_BidirectionalIterator>::difference_type difference_type;
+  typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type;
+  const difference_type __alloc_limit = 4; // might want to make this a function of trivial assignment
+  // Either prove all true and return __first or point to first false
+  while (true) {
+    if (__first == __last)
+      return __first;
+    if (!__pred(*__first))
+      break;
+    ++__first;
+  }
+  // __first points to first false, everything prior to __first is already set.
+  // Either prove [__first, __last) is all false and return __first, or point __last to last true
+  do {
+    if (__first == --__last)
+      return __first;
+  } while (!__pred(*__last));
+  // We now have a reduced range [__first, __last]
+  // *__first is known to be false
+  // *__last is known to be true
+  // __len >= 2
+  difference_type __len = _IterOps<_AlgPolicy>::distance(__first, __last) + 1;
+  pair<value_type*, ptrdiff_t> __p(0, 0);
+  unique_ptr<value_type, __return_temporary_buffer> __h;
+  if (__len >= __alloc_limit) {
+    // TODO: Remove the use of std::get_temporary_buffer
+    _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+    __p = std::get_temporary_buffer<value_type>(__len);
+    _LIBCPP_SUPPRESS_DEPRECATED_POP
+    __h.reset(__p.first);
+  }
+  return std::__stable_partition_impl<_AlgPolicy, _Predicate&>(
+      std::move(__first), std::move(__last), __pred, __len, __p, bidirectional_iterator_tag());
+}
+
+template <class _AlgPolicy, class _Predicate, class _ForwardIterator, class _IterCategory>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator __stable_partition(
+    _ForwardIterator __first, _ForwardIterator __last, _Predicate&& __pred, _IterCategory __iter_category) {
+  return std::__stable_partition_impl<_AlgPolicy, __remove_cvref_t<_Predicate>&>(
+      std::move(__first), std::move(__last), __pred, __iter_category);
+}
+
+template <class _ForwardIterator, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
+stable_partition(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
+  using _IterCategory = typename iterator_traits<_ForwardIterator>::iterator_category;
+  return std::__stable_partition<_ClassicAlgPolicy, _Predicate&>(
+      std::move(__first), std::move(__last), __pred, _IterCategory());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_STABLE_PARTITION_H
diff --git a/libcxx/include/__cxx03/__algorithm/stable_sort.h b/libcxx/include/__cxx03/__algorithm/stable_sort.h
new file mode 100644
index 000000000000..726e7e16b356
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/stable_sort.h
@@ -0,0 +1,273 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_STABLE_SORT_H
+#define _LIBCPP___ALGORITHM_STABLE_SORT_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/inplace_merge.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/sort.h>
+#include <__config>
+#include <__debug_utils/strict_weak_ordering_check.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/destruct_n.h>
+#include <__memory/temporary_buffer.h>
+#include <__memory/unique_ptr.h>
+#include <__type_traits/is_trivially_assignable.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <new>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Compare, class _BidirectionalIterator>
+_LIBCPP_HIDE_FROM_ABI void __insertion_sort_move(
+    _BidirectionalIterator __first1,
+    _BidirectionalIterator __last1,
+    typename iterator_traits<_BidirectionalIterator>::value_type* __first2,
+    _Compare __comp) {
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type;
+  if (__first1 != __last1) {
+    __destruct_n __d(0);
+    unique_ptr<value_type, __destruct_n&> __h(__first2, __d);
+    value_type* __last2 = __first2;
+    ::new ((void*)__last2) value_type(_Ops::__iter_move(__first1));
+    __d.template __incr<value_type>();
+    for (++__last2; ++__first1 != __last1; ++__last2) {
+      value_type* __j2 = __last2;
+      value_type* __i2 = __j2;
+      if (__comp(*__first1, *--__i2)) {
+        ::new ((void*)__j2) value_type(std::move(*__i2));
+        __d.template __incr<value_type>();
+        for (--__j2; __i2 != __first2 && __comp(*__first1, *--__i2); --__j2)
+          *__j2 = std::move(*__i2);
+        *__j2 = _Ops::__iter_move(__first1);
+      } else {
+        ::new ((void*)__j2) value_type(_Ops::__iter_move(__first1));
+        __d.template __incr<value_type>();
+      }
+    }
+    __h.release();
+  }
+}
+
+template <class _AlgPolicy, class _Compare, class _InputIterator1, class _InputIterator2>
+_LIBCPP_HIDE_FROM_ABI void __merge_move_construct(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _InputIterator2 __last2,
+    typename iterator_traits<_InputIterator1>::value_type* __result,
+    _Compare __comp) {
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  typedef typename iterator_traits<_InputIterator1>::value_type value_type;
+  __destruct_n __d(0);
+  unique_ptr<value_type, __destruct_n&> __h(__result, __d);
+  for (; true; ++__result) {
+    if (__first1 == __last1) {
+      for (; __first2 != __last2; ++__first2, (void)++__result, __d.template __incr<value_type>())
+        ::new ((void*)__result) value_type(_Ops::__iter_move(__first2));
+      __h.release();
+      return;
+    }
+    if (__first2 == __last2) {
+      for (; __first1 != __last1; ++__first1, (void)++__result, __d.template __incr<value_type>())
+        ::new ((void*)__result) value_type(_Ops::__iter_move(__first1));
+      __h.release();
+      return;
+    }
+    if (__comp(*__first2, *__first1)) {
+      ::new ((void*)__result) value_type(_Ops::__iter_move(__first2));
+      __d.template __incr<value_type>();
+      ++__first2;
+    } else {
+      ::new ((void*)__result) value_type(_Ops::__iter_move(__first1));
+      __d.template __incr<value_type>();
+      ++__first1;
+    }
+  }
+}
+
+template <class _AlgPolicy, class _Compare, class _InputIterator1, class _InputIterator2, class _OutputIterator>
+_LIBCPP_HIDE_FROM_ABI void __merge_move_assign(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _InputIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp) {
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  for (; __first1 != __last1; ++__result) {
+    if (__first2 == __last2) {
+      for (; __first1 != __last1; ++__first1, (void)++__result)
+        *__result = _Ops::__iter_move(__first1);
+      return;
+    }
+    if (__comp(*__first2, *__first1)) {
+      *__result = _Ops::__iter_move(__first2);
+      ++__first2;
+    } else {
+      *__result = _Ops::__iter_move(__first1);
+      ++__first1;
+    }
+  }
+  for (; __first2 != __last2; ++__first2, (void)++__result)
+    *__result = _Ops::__iter_move(__first2);
+}
+
+template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
+void __stable_sort(_RandomAccessIterator __first,
+                   _RandomAccessIterator __last,
+                   _Compare __comp,
+                   typename iterator_traits<_RandomAccessIterator>::difference_type __len,
+                   typename iterator_traits<_RandomAccessIterator>::value_type* __buff,
+                   ptrdiff_t __buff_size);
+
+template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
+void __stable_sort_move(_RandomAccessIterator __first1,
+                        _RandomAccessIterator __last1,
+                        _Compare __comp,
+                        typename iterator_traits<_RandomAccessIterator>::difference_type __len,
+                        typename iterator_traits<_RandomAccessIterator>::value_type* __first2) {
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type;
+  switch (__len) {
+  case 0:
+    return;
+  case 1:
+    ::new ((void*)__first2) value_type(_Ops::__iter_move(__first1));
+    return;
+  case 2:
+    __destruct_n __d(0);
+    unique_ptr<value_type, __destruct_n&> __h2(__first2, __d);
+    if (__comp(*--__last1, *__first1)) {
+      ::new ((void*)__first2) value_type(_Ops::__iter_move(__last1));
+      __d.template __incr<value_type>();
+      ++__first2;
+      ::new ((void*)__first2) value_type(_Ops::__iter_move(__first1));
+    } else {
+      ::new ((void*)__first2) value_type(_Ops::__iter_move(__first1));
+      __d.template __incr<value_type>();
+      ++__first2;
+      ::new ((void*)__first2) value_type(_Ops::__iter_move(__last1));
+    }
+    __h2.release();
+    return;
+  }
+  if (__len <= 8) {
+    std::__insertion_sort_move<_AlgPolicy, _Compare>(__first1, __last1, __first2, __comp);
+    return;
+  }
+  typename iterator_traits<_RandomAccessIterator>::difference_type __l2 = __len / 2;
+  _RandomAccessIterator __m                                             = __first1 + __l2;
+  std::__stable_sort<_AlgPolicy, _Compare>(__first1, __m, __comp, __l2, __first2, __l2);
+  std::__stable_sort<_AlgPolicy, _Compare>(__m, __last1, __comp, __len - __l2, __first2 + __l2, __len - __l2);
+  std::__merge_move_construct<_AlgPolicy, _Compare>(__first1, __m, __m, __last1, __first2, __comp);
+}
+
+template <class _Tp>
+struct __stable_sort_switch {
+  static const unsigned value = 128 * is_trivially_copy_assignable<_Tp>::value;
+};
+
+template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
+void __stable_sort(_RandomAccessIterator __first,
+                   _RandomAccessIterator __last,
+                   _Compare __comp,
+                   typename iterator_traits<_RandomAccessIterator>::difference_type __len,
+                   typename iterator_traits<_RandomAccessIterator>::value_type* __buff,
+                   ptrdiff_t __buff_size) {
+  typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type;
+  typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
+  switch (__len) {
+  case 0:
+  case 1:
+    return;
+  case 2:
+    if (__comp(*--__last, *__first))
+      _IterOps<_AlgPolicy>::iter_swap(__first, __last);
+    return;
+  }
+  if (__len <= static_cast<difference_type>(__stable_sort_switch<value_type>::value)) {
+    std::__insertion_sort<_AlgPolicy, _Compare>(__first, __last, __comp);
+    return;
+  }
+  typename iterator_traits<_RandomAccessIterator>::difference_type __l2 = __len / 2;
+  _RandomAccessIterator __m                                             = __first + __l2;
+  if (__len <= __buff_size) {
+    __destruct_n __d(0);
+    unique_ptr<value_type, __destruct_n&> __h2(__buff, __d);
+    std::__stable_sort_move<_AlgPolicy, _Compare>(__first, __m, __comp, __l2, __buff);
+    __d.__set(__l2, (value_type*)nullptr);
+    std::__stable_sort_move<_AlgPolicy, _Compare>(__m, __last, __comp, __len - __l2, __buff + __l2);
+    __d.__set(__len, (value_type*)nullptr);
+    std::__merge_move_assign<_AlgPolicy, _Compare>(
+        __buff, __buff + __l2, __buff + __l2, __buff + __len, __first, __comp);
+    //         std::__merge<_Compare>(move_iterator<value_type*>(__buff),
+    //                                  move_iterator<value_type*>(__buff + __l2),
+    //                                  move_iterator<_RandomAccessIterator>(__buff + __l2),
+    //                                  move_iterator<_RandomAccessIterator>(__buff + __len),
+    //                                  __first, __comp);
+    return;
+  }
+  std::__stable_sort<_AlgPolicy, _Compare>(__first, __m, __comp, __l2, __buff, __buff_size);
+  std::__stable_sort<_AlgPolicy, _Compare>(__m, __last, __comp, __len - __l2, __buff, __buff_size);
+  std::__inplace_merge<_AlgPolicy>(__first, __m, __last, __comp, __l2, __len - __l2, __buff, __buff_size);
+}
+
+template <class _AlgPolicy, class _RandomAccessIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI void
+__stable_sort_impl(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare& __comp) {
+  using value_type      = typename iterator_traits<_RandomAccessIterator>::value_type;
+  using difference_type = typename iterator_traits<_RandomAccessIterator>::difference_type;
+
+  difference_type __len = __last - __first;
+  pair<value_type*, ptrdiff_t> __buf(0, 0);
+  unique_ptr<value_type, __return_temporary_buffer> __h;
+  if (__len > static_cast<difference_type>(__stable_sort_switch<value_type>::value)) {
+    // TODO: Remove the use of std::get_temporary_buffer
+    _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+    __buf = std::get_temporary_buffer<value_type>(__len);
+    _LIBCPP_SUPPRESS_DEPRECATED_POP
+    __h.reset(__buf.first);
+  }
+
+  std::__stable_sort<_AlgPolicy, __comp_ref_type<_Compare> >(__first, __last, __comp, __len, __buf.first, __buf.second);
+  std::__check_strict_weak_ordering_sorted(__first, __last, __comp);
+}
+
+template <class _RandomAccessIterator, class _Compare>
+inline _LIBCPP_HIDE_FROM_ABI void
+stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+  std::__stable_sort_impl<_ClassicAlgPolicy>(std::move(__first), std::move(__last), __comp);
+}
+
+template <class _RandomAccessIterator>
+inline _LIBCPP_HIDE_FROM_ABI void stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last) {
+  std::stable_sort(__first, __last, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_STABLE_SORT_H
diff --git a/libcxx/include/__cxx03/__algorithm/swap_ranges.h b/libcxx/include/__cxx03/__algorithm/swap_ranges.h
new file mode 100644
index 000000000000..54b453b72360
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/swap_ranges.h
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SWAP_RANGES_H
+#define _LIBCPP___ALGORITHM_SWAP_RANGES_H
+
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// 2+2 iterators: the shorter size will be used.
+template <class _AlgPolicy, class _ForwardIterator1, class _Sentinel1, class _ForwardIterator2, class _Sentinel2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_ForwardIterator1, _ForwardIterator2>
+__swap_ranges(_ForwardIterator1 __first1, _Sentinel1 __last1, _ForwardIterator2 __first2, _Sentinel2 __last2) {
+  while (__first1 != __last1 && __first2 != __last2) {
+    _IterOps<_AlgPolicy>::iter_swap(__first1, __first2);
+    ++__first1;
+    ++__first2;
+  }
+
+  return pair<_ForwardIterator1, _ForwardIterator2>(std::move(__first1), std::move(__first2));
+}
+
+// 2+1 iterators: size2 >= size1.
+template <class _AlgPolicy, class _ForwardIterator1, class _Sentinel1, class _ForwardIterator2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_ForwardIterator1, _ForwardIterator2>
+__swap_ranges(_ForwardIterator1 __first1, _Sentinel1 __last1, _ForwardIterator2 __first2) {
+  while (__first1 != __last1) {
+    _IterOps<_AlgPolicy>::iter_swap(__first1, __first2);
+    ++__first1;
+    ++__first2;
+  }
+
+  return pair<_ForwardIterator1, _ForwardIterator2>(std::move(__first1), std::move(__first2));
+}
+
+template <class _ForwardIterator1, class _ForwardIterator2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator2
+swap_ranges(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2) {
+  return std::__swap_ranges<_ClassicAlgPolicy>(std::move(__first1), std::move(__last1), std::move(__first2)).second;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_SWAP_RANGES_H
diff --git a/libcxx/include/__cxx03/__algorithm/three_way_comp_ref_type.h b/libcxx/include/__cxx03/__algorithm/three_way_comp_ref_type.h
new file mode 100644
index 000000000000..5702a1fee082
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/three_way_comp_ref_type.h
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_THREE_WAY_COMP_REF_TYPE_H
+#define _LIBCPP___ALGORITHM_THREE_WAY_COMP_REF_TYPE_H
+
+#include <__assert>
+#include <__compare/ordering.h>
+#include <__config>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Comp>
+struct __debug_three_way_comp {
+  _Comp& __comp_;
+  _LIBCPP_HIDE_FROM_ABI constexpr __debug_three_way_comp(_Comp& __c) : __comp_(__c) {}
+
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(const _Tp& __x, const _Up& __y) {
+    auto __r = __comp_(__x, __y);
+    if constexpr (__comparison_category<decltype(__comp_(__x, __y))>)
+      __do_compare_assert(__y, __x, __r);
+    return __r;
+  }
+
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp& __x, _Up& __y) {
+    auto __r = __comp_(__x, __y);
+    if constexpr (__comparison_category<decltype(__comp_(__x, __y))>)
+      __do_compare_assert(__y, __x, __r);
+    return __r;
+  }
+
+  template <class _LHS, class _RHS, class _Order>
+  _LIBCPP_HIDE_FROM_ABI constexpr void __do_compare_assert(_LHS& __l, _RHS& __r, _Order __o) {
+    _Order __expected = __o;
+    if (__o == _Order::less)
+      __expected = _Order::greater;
+    if (__o == _Order::greater)
+      __expected = _Order::less;
+    _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
+        __comp_(__l, __r) == __expected, "Comparator does not induce a strict weak ordering");
+    (void)__l;
+    (void)__r;
+  }
+};
+
+// Pass the comparator by lvalue reference. Or in the debug mode, using a debugging wrapper that stores a reference.
+#  if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG
+template <class _Comp>
+using __three_way_comp_ref_type = __debug_three_way_comp<_Comp>;
+#  else
+template <class _Comp>
+using __three_way_comp_ref_type = _Comp&;
+#  endif
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_THREE_WAY_COMP_REF_TYPE_H
diff --git a/libcxx/include/__cxx03/__algorithm/transform.h b/libcxx/include/__cxx03/__algorithm/transform.h
new file mode 100644
index 000000000000..1b424409591c
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/transform.h
@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_TRANSFORM_H
+#define _LIBCPP___ALGORITHM_TRANSFORM_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _OutputIterator, class _UnaryOperation>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+transform(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _UnaryOperation __op) {
+  for (; __first != __last; ++__first, (void)++__result)
+    *__result = __op(*__first);
+  return __result;
+}
+
+template <class _InputIterator1, class _InputIterator2, class _OutputIterator, class _BinaryOperation>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator transform(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _OutputIterator __result,
+    _BinaryOperation __binary_op) {
+  for (; __first1 != __last1; ++__first1, (void)++__first2, ++__result)
+    *__result = __binary_op(*__first1, *__first2);
+  return __result;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_TRANSFORM_H
diff --git a/libcxx/include/__cxx03/__algorithm/uniform_random_bit_generator_adaptor.h b/libcxx/include/__cxx03/__algorithm/uniform_random_bit_generator_adaptor.h
new file mode 100644
index 000000000000..aef0fbfb7c28
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/uniform_random_bit_generator_adaptor.h
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_RANGES_UNIFORM_RANDOM_BIT_GENERATOR_ADAPTOR_H
+#define _LIBCPP___ALGORITHM_RANGES_UNIFORM_RANDOM_BIT_GENERATOR_ADAPTOR_H
+
+#include <__config>
+#include <__functional/invoke.h>
+#include <__type_traits/remove_cvref.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Range versions of random algorithms (e.g. `std::shuffle`) are less constrained than their classic counterparts.
+// Range algorithms only require the given generator to satisfy the `std::uniform_random_bit_generator` concept.
+// Classic algorithms require the given generator to meet the uniform random bit generator requirements; these
+// requirements include satisfying `std::uniform_random_bit_generator` and add a requirement for the generator to
+// provide a nested `result_type` typedef (see `[rand.req.urng]`).
+//
+// To be able to reuse classic implementations, make the given generator meet the classic requirements by wrapping
+// it into an adaptor type that forwards all of its interface and adds the required typedef.
+template <class _Gen>
+class _ClassicGenAdaptor {
+private:
+  // The generator is not required to be copyable or movable, so it has to be stored as a reference.
+  _Gen& __gen_;
+
+public:
+  using result_type = invoke_result_t<_Gen&>;
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto min() { return __remove_cvref_t<_Gen>::min(); }
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto max() { return __remove_cvref_t<_Gen>::max(); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit _ClassicGenAdaptor(_Gen& __g) : __gen_(__g) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()() const { return __gen_(); }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___ALGORITHM_RANGES_UNIFORM_RANDOM_BIT_GENERATOR_ADAPTOR_H
diff --git a/libcxx/include/__cxx03/__algorithm/unique.h b/libcxx/include/__cxx03/__algorithm/unique.h
new file mode 100644
index 000000000000..d597014596f2
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/unique.h
@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_UNIQUE_H
+#define _LIBCPP___ALGORITHM_UNIQUE_H
+
+#include <__algorithm/adjacent_find.h>
+#include <__algorithm/comp.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// unique
+
+template <class _AlgPolicy, class _Iter, class _Sent, class _BinaryPredicate>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 std::pair<_Iter, _Iter>
+__unique(_Iter __first, _Sent __last, _BinaryPredicate&& __pred) {
+  __first = std::__adjacent_find(__first, __last, __pred);
+  if (__first != __last) {
+    // ...  a  a  ?  ...
+    //      f     i
+    _Iter __i = __first;
+    for (++__i; ++__i != __last;)
+      if (!__pred(*__first, *__i))
+        *++__first = _IterOps<_AlgPolicy>::__iter_move(__i);
+    ++__first;
+    return std::pair<_Iter, _Iter>(std::move(__first), std::move(__i));
+  }
+  return std::pair<_Iter, _Iter>(__first, __first);
+}
+
+template <class _ForwardIterator, class _BinaryPredicate>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+unique(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred) {
+  return std::__unique<_ClassicAlgPolicy>(std::move(__first), std::move(__last), __pred).first;
+}
+
+template <class _ForwardIterator>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+unique(_ForwardIterator __first, _ForwardIterator __last) {
+  return std::unique(__first, __last, __equal_to());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_UNIQUE_H
diff --git a/libcxx/include/__cxx03/__algorithm/unique_copy.h b/libcxx/include/__cxx03/__algorithm/unique_copy.h
new file mode 100644
index 000000000000..16ce80cab32f
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/unique_copy.h
@@ -0,0 +1,127 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_UNIQUE_COPY_H
+#define _LIBCPP___ALGORITHM_UNIQUE_COPY_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/is_base_of.h>
+#include <__type_traits/is_same.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __unique_copy_tags {
+
+struct __reread_from_input_tag {};
+struct __reread_from_output_tag {};
+struct __read_from_tmp_value_tag {};
+
+} // namespace __unique_copy_tags
+
+template <class _AlgPolicy, class _BinaryPredicate, class _InputIterator, class _Sent, class _OutputIterator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _OutputIterator>
+__unique_copy(_InputIterator __first,
+              _Sent __last,
+              _OutputIterator __result,
+              _BinaryPredicate&& __pred,
+              __unique_copy_tags::__read_from_tmp_value_tag) {
+  if (__first != __last) {
+    typename _IterOps<_AlgPolicy>::template __value_type<_InputIterator> __t(*__first);
+    *__result = __t;
+    ++__result;
+    while (++__first != __last) {
+      if (!__pred(__t, *__first)) {
+        __t       = *__first;
+        *__result = __t;
+        ++__result;
+      }
+    }
+  }
+  return pair<_InputIterator, _OutputIterator>(std::move(__first), std::move(__result));
+}
+
+template <class _AlgPolicy, class _BinaryPredicate, class _ForwardIterator, class _Sent, class _OutputIterator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pair<_ForwardIterator, _OutputIterator>
+__unique_copy(_ForwardIterator __first,
+              _Sent __last,
+              _OutputIterator __result,
+              _BinaryPredicate&& __pred,
+              __unique_copy_tags::__reread_from_input_tag) {
+  if (__first != __last) {
+    _ForwardIterator __i = __first;
+    *__result            = *__i;
+    ++__result;
+    while (++__first != __last) {
+      if (!__pred(*__i, *__first)) {
+        *__result = *__first;
+        ++__result;
+        __i = __first;
+      }
+    }
+  }
+  return pair<_ForwardIterator, _OutputIterator>(std::move(__first), std::move(__result));
+}
+
+template <class _AlgPolicy, class _BinaryPredicate, class _InputIterator, class _Sent, class _InputAndOutputIterator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _InputAndOutputIterator>
+__unique_copy(_InputIterator __first,
+              _Sent __last,
+              _InputAndOutputIterator __result,
+              _BinaryPredicate&& __pred,
+              __unique_copy_tags::__reread_from_output_tag) {
+  if (__first != __last) {
+    *__result = *__first;
+    while (++__first != __last)
+      if (!__pred(*__result, *__first))
+        *++__result = *__first;
+    ++__result;
+  }
+  return pair<_InputIterator, _InputAndOutputIterator>(std::move(__first), std::move(__result));
+}
+
+template <class _InputIterator, class _OutputIterator, class _BinaryPredicate>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+unique_copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _BinaryPredicate __pred) {
+  using __algo_tag = __conditional_t<
+      is_base_of<forward_iterator_tag, typename iterator_traits<_InputIterator>::iterator_category>::value,
+      __unique_copy_tags::__reread_from_input_tag,
+      __conditional_t<
+          is_base_of<forward_iterator_tag, typename iterator_traits<_OutputIterator>::iterator_category>::value &&
+              is_same< typename iterator_traits<_InputIterator>::value_type,
+                       typename iterator_traits<_OutputIterator>::value_type>::value,
+          __unique_copy_tags::__reread_from_output_tag,
+          __unique_copy_tags::__read_from_tmp_value_tag> >;
+  return std::__unique_copy<_ClassicAlgPolicy>(
+             std::move(__first), std::move(__last), std::move(__result), __pred, __algo_tag())
+      .second;
+}
+
+template <class _InputIterator, class _OutputIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+unique_copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result) {
+  return std::unique_copy(std::move(__first), std::move(__last), std::move(__result), __equal_to());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_UNIQUE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/unwrap_iter.h b/libcxx/include/__cxx03/__algorithm/unwrap_iter.h
new file mode 100644
index 000000000000..8cc0d22d4fc2
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/unwrap_iter.h
@@ -0,0 +1,85 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_UNWRAP_ITER_H
+#define _LIBCPP___ALGORITHM_UNWRAP_ITER_H
+
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__memory/pointer_traits.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_constructible.h>
+#include <__utility/declval.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// TODO: Change the name of __unwrap_iter_impl to something more appropriate
+// The job of __unwrap_iter is to remove iterator wrappers (like reverse_iterator or __wrap_iter),
+// to reduce the number of template instantiations and to enable pointer-based optimizations e.g. in std::copy.
+//
+// Some algorithms (e.g. std::copy, but not std::sort) need to convert an
+// "unwrapped" result back into the original iterator type. Doing that is the job of __rewrap_iter.
+
+// Default case - we can't unwrap anything
+template <class _Iter, bool = __libcpp_is_contiguous_iterator<_Iter>::value>
+struct __unwrap_iter_impl {
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Iter __rewrap(_Iter, _Iter __iter) { return __iter; }
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Iter __unwrap(_Iter __i) _NOEXCEPT { return __i; }
+};
+
+// TODO(hardening): make sure that the following unwrapping doesn't unexpectedly turn hardened iterators into raw
+// pointers.
+
+// It's a contiguous iterator, so we can use a raw pointer instead
+template <class _Iter>
+struct __unwrap_iter_impl<_Iter, true> {
+  using _ToAddressT = decltype(std::__to_address(std::declval<_Iter>()));
+
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Iter __rewrap(_Iter __orig_iter, _ToAddressT __unwrapped_iter) {
+    return __orig_iter + (__unwrapped_iter - std::__to_address(__orig_iter));
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToAddressT __unwrap(_Iter __i) _NOEXCEPT {
+    return std::__to_address(__i);
+  }
+};
+
+template <class _Iter,
+          class _Impl                                             = __unwrap_iter_impl<_Iter>,
+          __enable_if_t<is_copy_constructible<_Iter>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 decltype(_Impl::__unwrap(std::declval<_Iter>()))
+__unwrap_iter(_Iter __i) _NOEXCEPT {
+  return _Impl::__unwrap(__i);
+}
+
+// Allow input_iterators to be passed to __unwrap_iter (but not __rewrap_iter)
+#if _LIBCPP_STD_VER >= 20
+template <class _Iter, __enable_if_t<!is_copy_constructible<_Iter>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _Iter __unwrap_iter(_Iter __i) noexcept {
+  return __i;
+}
+#endif
+
+template <class _OrigIter, class _Iter, class _Impl = __unwrap_iter_impl<_OrigIter> >
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _OrigIter __rewrap_iter(_OrigIter __orig_iter, _Iter __iter) _NOEXCEPT {
+  return _Impl::__rewrap(std::move(__orig_iter), std::move(__iter));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_UNWRAP_ITER_H
diff --git a/libcxx/include/__cxx03/__algorithm/unwrap_range.h b/libcxx/include/__cxx03/__algorithm/unwrap_range.h
new file mode 100644
index 000000000000..2d4b9bb5545a
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/unwrap_range.h
@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_UNWRAP_RANGE_H
+#define _LIBCPP___ALGORITHM_UNWRAP_RANGE_H
+
+#include <__algorithm/unwrap_iter.h>
+#include <__concepts/constructible.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/next.h>
+#include <__utility/declval.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// __unwrap_range and __rewrap_range are used to unwrap ranges which may have different iterator and sentinel types.
+// __unwrap_iter and __rewrap_iter don't work for this, because they assume that the iterator and sentinel have
+// the same type. __unwrap_range tries to get two iterators and then forward to __unwrap_iter.
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Iter, class _Sent>
+struct __unwrap_range_impl {
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __unwrap(_Iter __first, _Sent __sent)
+    requires random_access_iterator<_Iter> && sized_sentinel_for<_Sent, _Iter>
+  {
+    auto __last = ranges::next(__first, __sent);
+    return pair{std::__unwrap_iter(std::move(__first)), std::__unwrap_iter(std::move(__last))};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __unwrap(_Iter __first, _Sent __last) {
+    return pair{std::move(__first), std::move(__last)};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto
+  __rewrap(_Iter __orig_iter, decltype(std::__unwrap_iter(std::move(__orig_iter))) __iter)
+    requires random_access_iterator<_Iter> && sized_sentinel_for<_Sent, _Iter>
+  {
+    return std::__rewrap_iter(std::move(__orig_iter), std::move(__iter));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __rewrap(const _Iter&, _Iter __iter)
+    requires(!(random_access_iterator<_Iter> && sized_sentinel_for<_Sent, _Iter>))
+  {
+    return __iter;
+  }
+};
+
+template <class _Iter>
+struct __unwrap_range_impl<_Iter, _Iter> {
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __unwrap(_Iter __first, _Iter __last) {
+    return pair{std::__unwrap_iter(std::move(__first)), std::__unwrap_iter(std::move(__last))};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto
+  __rewrap(_Iter __orig_iter, decltype(std::__unwrap_iter(__orig_iter)) __iter) {
+    return std::__rewrap_iter(std::move(__orig_iter), std::move(__iter));
+  }
+};
+
+template <class _Iter, class _Sent>
+_LIBCPP_HIDE_FROM_ABI constexpr auto __unwrap_range(_Iter __first, _Sent __last) {
+  return __unwrap_range_impl<_Iter, _Sent>::__unwrap(std::move(__first), std::move(__last));
+}
+
+template < class _Sent, class _Iter, class _Unwrapped>
+_LIBCPP_HIDE_FROM_ABI constexpr _Iter __rewrap_range(_Iter __orig_iter, _Unwrapped __iter) {
+  return __unwrap_range_impl<_Iter, _Sent>::__rewrap(std::move(__orig_iter), std::move(__iter));
+}
+#else  // _LIBCPP_STD_VER >= 20
+template <class _Iter, class _Unwrapped = decltype(std::__unwrap_iter(std::declval<_Iter>()))>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR pair<_Unwrapped, _Unwrapped> __unwrap_range(_Iter __first, _Iter __last) {
+  return std::make_pair(std::__unwrap_iter(std::move(__first)), std::__unwrap_iter(std::move(__last)));
+}
+
+template <class _Iter, class _Unwrapped>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Iter __rewrap_range(_Iter __orig_iter, _Unwrapped __iter) {
+  return std::__rewrap_iter(std::move(__orig_iter), std::move(__iter));
+}
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_UNWRAP_RANGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/upper_bound.h b/libcxx/include/__cxx03/__algorithm/upper_bound.h
new file mode 100644
index 000000000000..c39dec2e8969
--- /dev/null
+++ b/libcxx/include/__cxx03/__algorithm/upper_bound.h
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_UPPER_BOUND_H
+#define _LIBCPP___ALGORITHM_UPPER_BOUND_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/half_positive.h>
+#include <__algorithm/iterator_operations.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/invoke.h>
+#include <__iterator/advance.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/is_constructible.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Compare, class _Iter, class _Sent, class _Tp, class _Proj>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Iter
+__upper_bound(_Iter __first, _Sent __last, const _Tp& __value, _Compare&& __comp, _Proj&& __proj) {
+  auto __len = _IterOps<_AlgPolicy>::distance(__first, __last);
+  while (__len != 0) {
+    auto __half_len = std::__half_positive(__len);
+    auto __mid      = _IterOps<_AlgPolicy>::next(__first, __half_len);
+    if (std::__invoke(__comp, __value, std::__invoke(__proj, *__mid)))
+      __len = __half_len;
+    else {
+      __first = ++__mid;
+      __len -= __half_len + 1;
+    }
+  }
+  return __first;
+}
+
+template <class _ForwardIterator, class _Tp, class _Compare>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+upper_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, _Compare __comp) {
+  static_assert(is_copy_constructible<_ForwardIterator>::value, "Iterator has to be copy constructible");
+  return std::__upper_bound<_ClassicAlgPolicy>(
+      std::move(__first), std::move(__last), __value, std::move(__comp), std::__identity());
+}
+
+template <class _ForwardIterator, class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+upper_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
+  return std::upper_bound(std::move(__first), std::move(__last), __value, __less<>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ALGORITHM_UPPER_BOUND_H
diff --git a/libcxx/include/__cxx03/__assert b/libcxx/include/__cxx03/__assert
new file mode 100644
index 000000000000..49769fb4d449
--- /dev/null
+++ b/libcxx/include/__cxx03/__assert
@@ -0,0 +1,118 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ASSERT
+#define _LIBCPP___ASSERT
+
+#include <__assertion_handler> // Note: this include is generated by CMake and is potentially vendor-provided.
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#define _LIBCPP_ASSERT(expression, message)                                                                            \
+  (__builtin_expect(static_cast<bool>(expression), 1)                                                                  \
+       ? (void)0                                                                                                       \
+       : _LIBCPP_ASSERTION_HANDLER(__FILE__ ":" _LIBCPP_TOSTRING(__LINE__) ": assertion " _LIBCPP_TOSTRING(            \
+             expression) " failed: " message "\n"))
+
+// TODO: __builtin_assume can currently inhibit optimizations. Until this has been fixed and we can add
+//       assumptions without a clear optimization intent, disable that to avoid worsening the code generation.
+//       See https://discourse.llvm.org/t/llvm-assume-blocks-optimization/71609 for a discussion.
+#if 0 && __has_builtin(__builtin_assume)
+#  define _LIBCPP_ASSUME(expression)                                                                                   \
+    (_LIBCPP_DIAGNOSTIC_PUSH _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wassume")                                              \
+         __builtin_assume(static_cast<bool>(expression)) _LIBCPP_DIAGNOSTIC_POP)
+#else
+#  define _LIBCPP_ASSUME(expression) ((void)0)
+#endif
+
+// clang-format off
+// Fast hardening mode checks.
+
+#if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_FAST
+
+// Enabled checks.
+#  define _LIBCPP_ASSERT_VALID_INPUT_RANGE(expression, message)       _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(expression, message)    _LIBCPP_ASSERT(expression, message)
+// Disabled checks.
+// On most modern platforms, dereferencing a null pointer does not lead to an actual memory access.
+#  define _LIBCPP_ASSERT_NON_NULL(expression, message)                _LIBCPP_ASSUME(expression)
+// Overlapping ranges will make algorithms produce incorrect results but don't directly lead to a security
+// vulnerability.
+#  define _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(expression, message)  _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_VALID_DEALLOCATION(expression, message)      _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(expression, message) _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(expression, message)    _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(expression, message)  _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_PEDANTIC(expression, message)                _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(expression, message)    _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_INTERNAL(expression, message)                _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_UNCATEGORIZED(expression, message)           _LIBCPP_ASSUME(expression)
+
+// Extensive hardening mode checks.
+
+#elif _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_EXTENSIVE
+
+// Enabled checks.
+#  define _LIBCPP_ASSERT_VALID_INPUT_RANGE(expression, message)       _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(expression, message)    _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_NON_NULL(expression, message)                _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(expression, message)  _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_VALID_DEALLOCATION(expression, message)      _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(expression, message) _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(expression, message)    _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(expression, message)  _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_PEDANTIC(expression, message)                _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_UNCATEGORIZED(expression, message)           _LIBCPP_ASSERT(expression, message)
+// Disabled checks.
+#  define _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(expression, message)    _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_INTERNAL(expression, message)                _LIBCPP_ASSUME(expression)
+
+// Debug hardening mode checks.
+
+#elif _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG
+
+// All checks enabled.
+#  define _LIBCPP_ASSERT_VALID_INPUT_RANGE(expression, message)       _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(expression, message)    _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_NON_NULL(expression, message)                _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(expression, message)  _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_VALID_DEALLOCATION(expression, message)      _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(expression, message) _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(expression, message)    _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(expression, message)  _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_PEDANTIC(expression, message)                _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(expression, message)    _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_INTERNAL(expression, message)                _LIBCPP_ASSERT(expression, message)
+#  define _LIBCPP_ASSERT_UNCATEGORIZED(expression, message)           _LIBCPP_ASSERT(expression, message)
+
+// Disable all checks if hardening is not enabled.
+
+#else
+
+// All checks disabled.
+#  define _LIBCPP_ASSERT_VALID_INPUT_RANGE(expression, message)       _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(expression, message)    _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_NON_NULL(expression, message)                _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(expression, message)  _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_VALID_DEALLOCATION(expression, message)      _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(expression, message) _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(expression, message)    _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(expression, message)  _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_PEDANTIC(expression, message)                _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(expression, message)    _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_INTERNAL(expression, message)                _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_UNCATEGORIZED(expression, message)           _LIBCPP_ASSUME(expression)
+
+#endif // _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_FAST
+// clang-format on
+
+#endif // _LIBCPP___ASSERT
diff --git a/libcxx/include/__cxx03/__atomic/aliases.h b/libcxx/include/__cxx03/__atomic/aliases.h
new file mode 100644
index 000000000000..e27e09af6b77
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/aliases.h
@@ -0,0 +1,110 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_ALIASES_H
+#define _LIBCPP___ATOMIC_ALIASES_H
+
+#include <__atomic/atomic.h>
+#include <__atomic/atomic_lock_free.h>
+#include <__atomic/contention_t.h>
+#include <__atomic/is_always_lock_free.h>
+#include <__config>
+#include <__type_traits/conditional.h>
+#include <__type_traits/make_unsigned.h>
+#include <cstddef>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using atomic_bool   = atomic<bool>;
+using atomic_char   = atomic<char>;
+using atomic_schar  = atomic<signed char>;
+using atomic_uchar  = atomic<unsigned char>;
+using atomic_short  = atomic<short>;
+using atomic_ushort = atomic<unsigned short>;
+using atomic_int    = atomic<int>;
+using atomic_uint   = atomic<unsigned int>;
+using atomic_long   = atomic<long>;
+using atomic_ulong  = atomic<unsigned long>;
+using atomic_llong  = atomic<long long>;
+using atomic_ullong = atomic<unsigned long long>;
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+using atomic_char8_t = atomic<char8_t>;
+#endif
+using atomic_char16_t = atomic<char16_t>;
+using atomic_char32_t = atomic<char32_t>;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using atomic_wchar_t = atomic<wchar_t>;
+#endif
+
+using atomic_int_least8_t   = atomic<int_least8_t>;
+using atomic_uint_least8_t  = atomic<uint_least8_t>;
+using atomic_int_least16_t  = atomic<int_least16_t>;
+using atomic_uint_least16_t = atomic<uint_least16_t>;
+using atomic_int_least32_t  = atomic<int_least32_t>;
+using atomic_uint_least32_t = atomic<uint_least32_t>;
+using atomic_int_least64_t  = atomic<int_least64_t>;
+using atomic_uint_least64_t = atomic<uint_least64_t>;
+
+using atomic_int_fast8_t   = atomic<int_fast8_t>;
+using atomic_uint_fast8_t  = atomic<uint_fast8_t>;
+using atomic_int_fast16_t  = atomic<int_fast16_t>;
+using atomic_uint_fast16_t = atomic<uint_fast16_t>;
+using atomic_int_fast32_t  = atomic<int_fast32_t>;
+using atomic_uint_fast32_t = atomic<uint_fast32_t>;
+using atomic_int_fast64_t  = atomic<int_fast64_t>;
+using atomic_uint_fast64_t = atomic<uint_fast64_t>;
+
+using atomic_int8_t   = atomic< int8_t>;
+using atomic_uint8_t  = atomic<uint8_t>;
+using atomic_int16_t  = atomic< int16_t>;
+using atomic_uint16_t = atomic<uint16_t>;
+using atomic_int32_t  = atomic< int32_t>;
+using atomic_uint32_t = atomic<uint32_t>;
+using atomic_int64_t  = atomic< int64_t>;
+using atomic_uint64_t = atomic<uint64_t>;
+
+using atomic_intptr_t  = atomic<intptr_t>;
+using atomic_uintptr_t = atomic<uintptr_t>;
+using atomic_size_t    = atomic<size_t>;
+using atomic_ptrdiff_t = atomic<ptrdiff_t>;
+using atomic_intmax_t  = atomic<intmax_t>;
+using atomic_uintmax_t = atomic<uintmax_t>;
+
+// C++20 atomic_{signed,unsigned}_lock_free: prefer the contention type most highly, then the largest lock-free type
+#if _LIBCPP_STD_VER >= 20
+#  if ATOMIC_LLONG_LOCK_FREE == 2
+using __largest_lock_free_type = long long;
+#  elif ATOMIC_INT_LOCK_FREE == 2
+using __largest_lock_free_type = int;
+#  elif ATOMIC_SHORT_LOCK_FREE == 2
+using __largest_lock_free_type = short;
+#  elif ATOMIC_CHAR_LOCK_FREE == 2
+using __largest_lock_free_type = char;
+#  else
+#    define _LIBCPP_NO_LOCK_FREE_TYPES // There are no lockfree types (this can happen on unusual platforms)
+#  endif
+
+#  ifndef _LIBCPP_NO_LOCK_FREE_TYPES
+using __contention_t_or_largest =
+    __conditional_t<__libcpp_is_always_lock_free<__cxx_contention_t>::__value,
+                    __cxx_contention_t,
+                    __largest_lock_free_type>;
+
+using atomic_signed_lock_free   = atomic<__contention_t_or_largest>;
+using atomic_unsigned_lock_free = atomic<make_unsigned_t<__contention_t_or_largest>>;
+#  endif // !_LIBCPP_NO_LOCK_FREE_TYPES
+#endif   // C++20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_ALIASES_H
diff --git a/libcxx/include/__cxx03/__atomic/atomic.h b/libcxx/include/__cxx03/__atomic/atomic.h
new file mode 100644
index 000000000000..bd3f659c22df
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/atomic.h
@@ -0,0 +1,622 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_ATOMIC_H
+#define _LIBCPP___ATOMIC_ATOMIC_H
+
+#include <__atomic/atomic_base.h>
+#include <__atomic/check_memory_order.h>
+#include <__atomic/cxx_atomic_impl.h>
+#include <__atomic/memory_order.h>
+#include <__config>
+#include <__functional/operations.h>
+#include <__memory/addressof.h>
+#include <__type_traits/is_floating_point.h>
+#include <__type_traits/is_function.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_const.h>
+#include <__type_traits/remove_pointer.h>
+#include <__type_traits/remove_volatile.h>
+#include <__utility/forward.h>
+#include <cstddef>
+#include <cstring>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct atomic : public __atomic_base<_Tp> {
+  using __base          = __atomic_base<_Tp>;
+  using value_type      = _Tp;
+  using difference_type = value_type;
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI atomic() = default;
+#else
+  _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default;
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {}
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile _NOEXCEPT {
+    __base::store(__d);
+    return __d;
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) _NOEXCEPT {
+    __base::store(__d);
+    return __d;
+  }
+
+  atomic& operator=(const atomic&)          = delete;
+  atomic& operator=(const atomic&) volatile = delete;
+};
+
+// atomic<T*>
+
+template <class _Tp>
+struct atomic<_Tp*> : public __atomic_base<_Tp*> {
+  using __base          = __atomic_base<_Tp*>;
+  using value_type      = _Tp*;
+  using difference_type = ptrdiff_t;
+
+  _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {}
+
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) volatile _NOEXCEPT {
+    __base::store(__d);
+    return __d;
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) _NOEXCEPT {
+    __base::store(__d);
+    return __d;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
+    // __atomic_fetch_add accepts function pointers, guard against them.
+    static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
+    return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+    // __atomic_fetch_add accepts function pointers, guard against them.
+    static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
+    return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
+    // __atomic_fetch_add accepts function pointers, guard against them.
+    static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
+    return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+    // __atomic_fetch_add accepts function pointers, guard against them.
+    static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
+    return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) volatile _NOEXCEPT { return fetch_add(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) _NOEXCEPT { return fetch_add(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) volatile _NOEXCEPT { return fetch_sub(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) _NOEXCEPT { return fetch_sub(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator++() volatile _NOEXCEPT { return fetch_add(1) + 1; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator++() _NOEXCEPT { return fetch_add(1) + 1; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator--() volatile _NOEXCEPT { return fetch_sub(1) - 1; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator--() _NOEXCEPT { return fetch_sub(1) - 1; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT { return fetch_add(__op) + __op; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT { return fetch_sub(__op) - __op; }
+
+  atomic& operator=(const atomic&)          = delete;
+  atomic& operator=(const atomic&) volatile = delete;
+};
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp>
+  requires is_floating_point_v<_Tp>
+struct atomic<_Tp> : __atomic_base<_Tp> {
+private:
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __is_fp80_long_double() {
+    // Only x87-fp80 long double has 64-bit mantissa
+    return __LDBL_MANT_DIG__ == 64 && std::is_same_v<_Tp, long double>;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __has_rmw_builtin() {
+#  ifndef _LIBCPP_COMPILER_CLANG_BASED
+    return false;
+#  else
+    // The builtin __cxx_atomic_fetch_add errors during compilation for
+    // long double on platforms with fp80 format.
+    // For more details, see
+    // lib/Sema/SemaChecking.cpp function IsAllowedValueType
+    // LLVM Parser does not allow atomicrmw with x86_fp80 type.
+    // if (ValType->isSpecificBuiltinType(BuiltinType::LongDouble) &&
+    //    &Context.getTargetInfo().getLongDoubleFormat() ==
+    //        &llvm::APFloat::x87DoubleExtended())
+    // For more info
+    // https://github.com/llvm/llvm-project/issues/68602
+    // https://reviews.llvm.org/D53965
+    return !__is_fp80_long_double();
+#  endif
+  }
+
+  template <class _This, class _Operation, class _BuiltinOp>
+  _LIBCPP_HIDE_FROM_ABI static _Tp
+  __rmw_op(_This&& __self, _Tp __operand, memory_order __m, _Operation __operation, _BuiltinOp __builtin_op) {
+    if constexpr (__has_rmw_builtin()) {
+      return __builtin_op(std::addressof(std::forward<_This>(__self).__a_), __operand, __m);
+    } else {
+      _Tp __old = __self.load(memory_order_relaxed);
+      _Tp __new = __operation(__old, __operand);
+      while (!__self.compare_exchange_weak(__old, __new, __m, memory_order_relaxed)) {
+#  ifdef _LIBCPP_COMPILER_CLANG_BASED
+        if constexpr (__is_fp80_long_double()) {
+          // https://github.com/llvm/llvm-project/issues/47978
+          // clang bug: __old is not updated on failure for atomic<long double>::compare_exchange_weak
+          // Note __old = __self.load(memory_order_relaxed) will not work
+          std::__cxx_atomic_load_inplace(std::addressof(__self.__a_), &__old, memory_order_relaxed);
+        }
+#  endif
+        __new = __operation(__old, __operand);
+      }
+      return __old;
+    }
+  }
+
+  template <class _This>
+  _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_add(_This&& __self, _Tp __operand, memory_order __m) {
+    auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
+      return std::__cxx_atomic_fetch_add(__a, __builtin_operand, __order);
+    };
+    return __rmw_op(std::forward<_This>(__self), __operand, __m, std::plus<>{}, __builtin_op);
+  }
+
+  template <class _This>
+  _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_sub(_This&& __self, _Tp __operand, memory_order __m) {
+    auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
+      return std::__cxx_atomic_fetch_sub(__a, __builtin_operand, __order);
+    };
+    return __rmw_op(std::forward<_This>(__self), __operand, __m, std::minus<>{}, __builtin_op);
+  }
+
+public:
+  using __base          = __atomic_base<_Tp>;
+  using value_type      = _Tp;
+  using difference_type = value_type;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr atomic() noexcept = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr atomic(_Tp __d) noexcept : __base(__d) {}
+
+  atomic(const atomic&)                     = delete;
+  atomic& operator=(const atomic&)          = delete;
+  atomic& operator=(const atomic&) volatile = delete;
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile noexcept
+    requires __base::is_always_lock_free
+  {
+    __base::store(__d);
+    return __d;
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) noexcept {
+    __base::store(__d);
+    return __d;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
+    requires __base::is_always_lock_free
+  {
+    return __fetch_add(*this, __op, __m);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {
+    return __fetch_add(*this, __op, __m);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
+    requires __base::is_always_lock_free
+  {
+    return __fetch_sub(*this, __op, __m);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {
+    return __fetch_sub(*this, __op, __m);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile noexcept
+    requires __base::is_always_lock_free
+  {
+    return fetch_add(__op) + __op;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) noexcept { return fetch_add(__op) + __op; }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile noexcept
+    requires __base::is_always_lock_free
+  {
+    return fetch_sub(__op) - __op;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) noexcept { return fetch_sub(__op) - __op; }
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+// atomic_is_lock_free
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT {
+  return __o->is_lock_free();
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT {
+  return __o->is_lock_free();
+}
+
+// atomic_init
+
+template <class _Tp>
+_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void
+atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
+  std::__cxx_atomic_init(std::addressof(__o->__a_), __d);
+}
+
+template <class _Tp>
+_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void
+atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
+  std::__cxx_atomic_init(std::addressof(__o->__a_), __d);
+}
+
+// atomic_store
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
+  __o->store(__d);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
+  __o->store(__d);
+}
+
+// atomic_store_explicit
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
+    _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
+  __o->store(__d, __m);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
+    _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
+  __o->store(__d, __m);
+}
+
+// atomic_load
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT {
+  return __o->load();
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const atomic<_Tp>* __o) _NOEXCEPT {
+  return __o->load();
+}
+
+// atomic_load_explicit
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
+    _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
+  return __o->load(__m);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
+    _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
+  return __o->load(__m);
+}
+
+// atomic_exchange
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
+  return __o->exchange(__d);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
+  return __o->exchange(__d);
+}
+
+// atomic_exchange_explicit
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT {
+  return __o->exchange(__d, __m);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT {
+  return __o->exchange(__d, __m);
+}
+
+// atomic_compare_exchange_weak
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak(
+    volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
+  return __o->compare_exchange_weak(*__e, __d);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak(
+    atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
+  return __o->compare_exchange_weak(*__e, __d);
+}
+
+// atomic_compare_exchange_strong
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong(
+    volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
+  return __o->compare_exchange_strong(*__e, __d);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong(
+    atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
+  return __o->compare_exchange_strong(*__e, __d);
+}
+
+// atomic_compare_exchange_weak_explicit
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(
+    volatile atomic<_Tp>* __o,
+    typename atomic<_Tp>::value_type* __e,
+    typename atomic<_Tp>::value_type __d,
+    memory_order __s,
+    memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
+  return __o->compare_exchange_weak(*__e, __d, __s, __f);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(
+    atomic<_Tp>* __o,
+    typename atomic<_Tp>::value_type* __e,
+    typename atomic<_Tp>::value_type __d,
+    memory_order __s,
+    memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
+  return __o->compare_exchange_weak(*__e, __d, __s, __f);
+}
+
+// atomic_compare_exchange_strong_explicit
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(
+    volatile atomic<_Tp>* __o,
+    typename atomic<_Tp>::value_type* __e,
+    typename atomic<_Tp>::value_type __d,
+    memory_order __s,
+    memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
+  return __o->compare_exchange_strong(*__e, __d, __s, __f);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(
+    atomic<_Tp>* __o,
+    typename atomic<_Tp>::value_type* __e,
+    typename atomic<_Tp>::value_type __d,
+    memory_order __s,
+    memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
+  return __o->compare_exchange_strong(*__e, __d, __s, __f);
+}
+
+// atomic_wait
+
+template <class _Tp>
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+atomic_wait(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT {
+  return __o->wait(__v);
+}
+
+template <class _Tp>
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+atomic_wait(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT {
+  return __o->wait(__v);
+}
+
+// atomic_wait_explicit
+
+template <class _Tp>
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+atomic_wait_explicit(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT
+    _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
+  return __o->wait(__v, __m);
+}
+
+template <class _Tp>
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+atomic_wait_explicit(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT
+    _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
+  return __o->wait(__v, __m);
+}
+
+// atomic_notify_one
+
+template <class _Tp>
+_LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT {
+  __o->notify_one();
+}
+template <class _Tp>
+_LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT {
+  __o->notify_one();
+}
+
+// atomic_notify_all
+
+template <class _Tp>
+_LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT {
+  __o->notify_all();
+}
+template <class _Tp>
+_LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT {
+  __o->notify_all();
+}
+
+// atomic_fetch_add
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
+  return __o->fetch_add(__op);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
+  return __o->fetch_add(__op);
+}
+
+// atomic_fetch_add_explicit
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add_explicit(
+    volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
+  return __o->fetch_add(__op, __m);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
+  return __o->fetch_add(__op, __m);
+}
+
+// atomic_fetch_sub
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
+  return __o->fetch_sub(__op);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
+  return __o->fetch_sub(__op);
+}
+
+// atomic_fetch_sub_explicit
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub_explicit(
+    volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
+  return __o->fetch_sub(__op, __m);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
+  return __o->fetch_sub(__op, __m);
+}
+
+// atomic_fetch_and
+
+template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
+  return __o->fetch_and(__op);
+}
+
+template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
+  return __o->fetch_and(__op);
+}
+
+// atomic_fetch_and_explicit
+
+template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and_explicit(
+    volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
+  return __o->fetch_and(__op, __m);
+}
+
+template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp
+atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
+  return __o->fetch_and(__op, __m);
+}
+
+// atomic_fetch_or
+
+template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
+  return __o->fetch_or(__op);
+}
+
+template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
+  return __o->fetch_or(__op);
+}
+
+// atomic_fetch_or_explicit
+
+template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp
+atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
+  return __o->fetch_or(__op, __m);
+}
+
+template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp
+atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
+  return __o->fetch_or(__op, __m);
+}
+
+// atomic_fetch_xor
+
+template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
+  return __o->fetch_xor(__op);
+}
+
+template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
+  return __o->fetch_xor(__op);
+}
+
+// atomic_fetch_xor_explicit
+
+template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor_explicit(
+    volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
+  return __o->fetch_xor(__op, __m);
+}
+
+template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp
+atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
+  return __o->fetch_xor(__op, __m);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_ATOMIC_H
diff --git a/libcxx/include/__cxx03/__atomic/atomic_base.h b/libcxx/include/__cxx03/__atomic/atomic_base.h
new file mode 100644
index 000000000000..7e26434c9c3a
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/atomic_base.h
@@ -0,0 +1,221 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_ATOMIC_BASE_H
+#define _LIBCPP___ATOMIC_ATOMIC_BASE_H
+
+#include <__atomic/atomic_sync.h>
+#include <__atomic/check_memory_order.h>
+#include <__atomic/cxx_atomic_impl.h>
+#include <__atomic/is_always_lock_free.h>
+#include <__atomic/memory_order.h>
+#include <__config>
+#include <__memory/addressof.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_same.h>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value>
+struct __atomic_base // false
+{
+  mutable __cxx_atomic_impl<_Tp> __a_;
+
+#if _LIBCPP_STD_VER >= 17
+  static constexpr bool is_always_lock_free = __libcpp_is_always_lock_free<__cxx_atomic_impl<_Tp> >::__value;
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const volatile _NOEXCEPT {
+    return __cxx_atomic_is_lock_free(sizeof(__cxx_atomic_impl<_Tp>));
+  }
+  _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT {
+    return static_cast<__atomic_base const volatile*>(this)->is_lock_free();
+  }
+  _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
+      _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
+    std::__cxx_atomic_store(std::addressof(__a_), __d, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT
+      _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
+    std::__cxx_atomic_store(std::addressof(__a_), __d, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
+      _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
+    return std::__cxx_atomic_load(std::addressof(__a_), __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
+      _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
+    return std::__cxx_atomic_load(std::addressof(__a_), __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI operator _Tp() const volatile _NOEXCEPT { return load(); }
+  _LIBCPP_HIDE_FROM_ABI operator _Tp() const _NOEXCEPT { return load(); }
+  _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
+    return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+    return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool
+  compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT
+      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
+    return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT
+      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
+    return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool
+  compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT
+      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
+    return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT
+      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
+    return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool
+  compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
+    return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool
+  compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+    return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool
+  compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
+    return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool
+  compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+    return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
+  }
+
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const
+      volatile _NOEXCEPT {
+    std::__atomic_wait(*this, __v, __m);
+  }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+  wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT {
+    std::__atomic_wait(*this, __v, __m);
+  }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT {
+    std::__atomic_notify_one(*this);
+  }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT {
+    std::__atomic_notify_all(*this);
+  }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); }
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI constexpr __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI __atomic_base() _NOEXCEPT = default;
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {}
+
+  __atomic_base(const __atomic_base&) = delete;
+};
+
+// atomic<Integral>
+
+template <class _Tp>
+struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> {
+  using __base = __atomic_base<_Tp, false>;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __atomic_base() _NOEXCEPT = default;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {}
+
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
+    return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+    return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
+    return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+    return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
+    return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+    return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
+    return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+    return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
+    return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+    return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) volatile _NOEXCEPT { return fetch_add(_Tp(1)); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) _NOEXCEPT { return fetch_add(_Tp(1)); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) volatile _NOEXCEPT { return fetch_sub(_Tp(1)); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) _NOEXCEPT { return fetch_sub(_Tp(1)); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator++() volatile _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator++() _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator--() volatile _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator--() _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) _NOEXCEPT { return fetch_add(__op) + __op; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) _NOEXCEPT { return fetch_sub(__op) - __op; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) volatile _NOEXCEPT { return fetch_and(__op) & __op; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) _NOEXCEPT { return fetch_and(__op) & __op; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) volatile _NOEXCEPT { return fetch_or(__op) | __op; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) _NOEXCEPT { return fetch_or(__op) | __op; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) volatile _NOEXCEPT { return fetch_xor(__op) ^ __op; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) _NOEXCEPT { return fetch_xor(__op) ^ __op; }
+};
+
+// Here we need _IsIntegral because the default template argument is not enough
+// e.g  __atomic_base<int> is __atomic_base<int, true>, which inherits from
+// __atomic_base<int, false> and the caller of the wait function is
+// __atomic_base<int, false>. So specializing __atomic_base<_Tp> does not work
+template <class _Tp, bool _IsIntegral>
+struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > {
+  static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_base<_Tp, _IsIntegral>& __a, memory_order __order) {
+    return __a.load(__order);
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI _Tp
+  __atomic_load(const volatile __atomic_base<_Tp, _IsIntegral>& __this, memory_order __order) {
+    return __this.load(__order);
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_Tp>*
+  __atomic_contention_address(const __atomic_base<_Tp, _IsIntegral>& __a) {
+    return std::addressof(__a.__a_);
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_Tp>*
+  __atomic_contention_address(const volatile __atomic_base<_Tp, _IsIntegral>& __this) {
+    return std::addressof(__this.__a_);
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_ATOMIC_BASE_H
diff --git a/libcxx/include/__cxx03/__atomic/atomic_flag.h b/libcxx/include/__cxx03/__atomic/atomic_flag.h
new file mode 100644
index 000000000000..00b157cdff78
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/atomic_flag.h
@@ -0,0 +1,189 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_ATOMIC_FLAG_H
+#define _LIBCPP___ATOMIC_ATOMIC_FLAG_H
+
+#include <__atomic/atomic_sync.h>
+#include <__atomic/contention_t.h>
+#include <__atomic/cxx_atomic_impl.h>
+#include <__atomic/memory_order.h>
+#include <__chrono/duration.h>
+#include <__config>
+#include <__memory/addressof.h>
+#include <__thread/support.h>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct atomic_flag {
+  __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_;
+
+  _LIBCPP_HIDE_FROM_ABI bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT {
+    return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT {
+    return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
+    return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+    return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
+    __cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);
+  }
+  _LIBCPP_HIDE_FROM_ABI void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+    __cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);
+  }
+
+  _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+  wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT {
+    std::__atomic_wait(*this, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);
+  }
+  _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+  wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT {
+    std::__atomic_wait(*this, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);
+  }
+  _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT {
+    std::__atomic_notify_one(*this);
+  }
+  _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT {
+    std::__atomic_notify_one(*this);
+  }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT {
+    std::__atomic_notify_all(*this);
+  }
+  _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT {
+    std::__atomic_notify_all(*this);
+  }
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI constexpr atomic_flag() _NOEXCEPT : __a_(false) {}
+#else
+  atomic_flag() _NOEXCEPT = default;
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION
+
+  atomic_flag(const atomic_flag&)                     = delete;
+  atomic_flag& operator=(const atomic_flag&)          = delete;
+  atomic_flag& operator=(const atomic_flag&) volatile = delete;
+};
+
+template <>
+struct __atomic_waitable_traits<atomic_flag> {
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_ATOMIC_FLAG_TYPE __atomic_load(const atomic_flag& __a, memory_order __order) {
+    return std::__cxx_atomic_load(&__a.__a_, __order);
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_ATOMIC_FLAG_TYPE
+  __atomic_load(const volatile atomic_flag& __a, memory_order __order) {
+    return std::__cxx_atomic_load(&__a.__a_, __order);
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE>*
+  __atomic_contention_address(const atomic_flag& __a) {
+    return std::addressof(__a.__a_);
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE>*
+  __atomic_contention_address(const volatile atomic_flag& __a) {
+    return std::addressof(__a.__a_);
+  }
+};
+
+inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT { return __o->test(); }
+
+inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test(const atomic_flag* __o) _NOEXCEPT { return __o->test(); }
+
+inline _LIBCPP_HIDE_FROM_ABI bool
+atomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT {
+  return __o->test(__m);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT {
+  return __o->test(__m);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT {
+  return __o->test_and_set();
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT { return __o->test_and_set(); }
+
+inline _LIBCPP_HIDE_FROM_ABI bool
+atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT {
+  return __o->test_and_set(__m);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT {
+  return __o->test_and_set(__m);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT { __o->clear(); }
+
+inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear(atomic_flag* __o) _NOEXCEPT { __o->clear(); }
+
+inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT {
+  __o->clear(__m);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT {
+  __o->clear(__m);
+}
+
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT {
+  __o->wait(__v);
+}
+
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT {
+  __o->wait(__v);
+}
+
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+atomic_flag_wait_explicit(const volatile atomic_flag* __o, bool __v, memory_order __m) _NOEXCEPT {
+  __o->wait(__v, __m);
+}
+
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+atomic_flag_wait_explicit(const atomic_flag* __o, bool __v, memory_order __m) _NOEXCEPT {
+  __o->wait(__v, __m);
+}
+
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT {
+  __o->notify_one();
+}
+
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT {
+  __o->notify_one();
+}
+
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT {
+  __o->notify_all();
+}
+
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT {
+  __o->notify_all();
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_ATOMIC_FLAG_H
diff --git a/libcxx/include/__cxx03/__atomic/atomic_init.h b/libcxx/include/__cxx03/__atomic/atomic_init.h
new file mode 100644
index 000000000000..8e86ba31b4ac
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/atomic_init.h
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_ATOMIC_INIT_H
+#define _LIBCPP___ATOMIC_ATOMIC_INIT_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#define ATOMIC_FLAG_INIT {false}
+#define ATOMIC_VAR_INIT(__v) {__v}
+
+#if _LIBCPP_STD_VER >= 20 && defined(_LIBCPP_COMPILER_CLANG_BASED) && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS)
+#  pragma clang deprecated(ATOMIC_VAR_INIT)
+#endif
+
+#endif // _LIBCPP___ATOMIC_ATOMIC_INIT_H
diff --git a/libcxx/include/__cxx03/__atomic/atomic_lock_free.h b/libcxx/include/__cxx03/__atomic/atomic_lock_free.h
new file mode 100644
index 000000000000..0715439db450
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/atomic_lock_free.h
@@ -0,0 +1,48 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_ATOMIC_LOCK_FREE_H
+#define _LIBCPP___ATOMIC_ATOMIC_LOCK_FREE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE)
+#  define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE
+#  define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE
+#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+#    define ATOMIC_CHAR8_T_LOCK_FREE __CLANG_ATOMIC_CHAR8_T_LOCK_FREE
+#  endif
+#  define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE
+#  define ATOMIC_CHAR32_T_LOCK_FREE __CLANG_ATOMIC_CHAR32_T_LOCK_FREE
+#  define ATOMIC_WCHAR_T_LOCK_FREE __CLANG_ATOMIC_WCHAR_T_LOCK_FREE
+#  define ATOMIC_SHORT_LOCK_FREE __CLANG_ATOMIC_SHORT_LOCK_FREE
+#  define ATOMIC_INT_LOCK_FREE __CLANG_ATOMIC_INT_LOCK_FREE
+#  define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE
+#  define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE
+#  define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE
+#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE)
+#  define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE
+#  define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE
+#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+#    define ATOMIC_CHAR8_T_LOCK_FREE __GCC_ATOMIC_CHAR8_T_LOCK_FREE
+#  endif
+#  define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE
+#  define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE
+#  define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE
+#  define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE
+#  define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE
+#  define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE
+#  define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE
+#  define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE
+#endif
+
+#endif // _LIBCPP___ATOMIC_ATOMIC_LOCK_FREE_H
diff --git a/libcxx/include/__cxx03/__atomic/atomic_ref.h b/libcxx/include/__cxx03/__atomic/atomic_ref.h
new file mode 100644
index 000000000000..b0180a37ab50
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/atomic_ref.h
@@ -0,0 +1,378 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//                        Kokkos v. 4.0
+//       Copyright (2022) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_ATOMIC_REF_H
+#define _LIBCPP___ATOMIC_ATOMIC_REF_H
+
+#include <__assert>
+#include <__atomic/atomic_sync.h>
+#include <__atomic/check_memory_order.h>
+#include <__atomic/to_gcc_order.h>
+#include <__concepts/arithmetic.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__memory/addressof.h>
+#include <__type_traits/has_unique_object_representation.h>
+#include <__type_traits/is_trivially_copyable.h>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// These types are required to make __atomic_is_always_lock_free work across GCC and Clang.
+// The purpose of this trick is to make sure that we provide an object with the correct alignment
+// to __atomic_is_always_lock_free, since that answer depends on the alignment.
+template <size_t _Alignment>
+struct __alignment_checker_type {
+  alignas(_Alignment) char __data;
+};
+
+template <size_t _Alignment>
+struct __get_aligner_instance {
+  static constexpr __alignment_checker_type<_Alignment> __instance{};
+};
+
+template <class _Tp>
+struct __atomic_ref_base {
+private:
+  _LIBCPP_HIDE_FROM_ABI static _Tp* __clear_padding(_Tp& __val) noexcept {
+    _Tp* __ptr = std::addressof(__val);
+#  if __has_builtin(__builtin_clear_padding)
+    __builtin_clear_padding(__ptr);
+#  endif
+    return __ptr;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static bool __compare_exchange(
+      _Tp* __ptr, _Tp* __expected, _Tp* __desired, bool __is_weak, int __success, int __failure) noexcept {
+    if constexpr (
+#  if __has_builtin(__builtin_clear_padding)
+        has_unique_object_representations_v<_Tp> || floating_point<_Tp>
+#  else
+        true // NOLINT(readability-simplify-boolean-expr)
+#  endif
+    ) {
+      return __atomic_compare_exchange(__ptr, __expected, __desired, __is_weak, __success, __failure);
+    } else { // _Tp has padding bits and __builtin_clear_padding is available
+      __clear_padding(*__desired);
+      _Tp __copy = *__expected;
+      __clear_padding(__copy);
+      // The algorithm we use here is basically to perform `__atomic_compare_exchange` on the
+      // values until it has either succeeded, or failed because the value representation of the
+      // objects involved was different. This is why we loop around __atomic_compare_exchange:
+      // we basically loop until its failure is caused by the value representation of the objects
+      // being different, not only their object representation.
+      while (true) {
+        _Tp __prev = __copy;
+        if (__atomic_compare_exchange(__ptr, std::addressof(__copy), __desired, __is_weak, __success, __failure)) {
+          return true;
+        }
+        _Tp __curr = __copy;
+        if (std::memcmp(__clear_padding(__prev), __clear_padding(__curr), sizeof(_Tp)) != 0) {
+          // Value representation without padding bits do not compare equal ->
+          // write the current content of *ptr into *expected
+          std::memcpy(__expected, std::addressof(__copy), sizeof(_Tp));
+          return false;
+        }
+      }
+    }
+  }
+
+  friend struct __atomic_waitable_traits<__atomic_ref_base<_Tp>>;
+
+  // require types that are 1, 2, 4, 8, or 16 bytes in length to be aligned to at least their size to be potentially
+  // used lock-free
+  static constexpr size_t __min_alignment = (sizeof(_Tp) & (sizeof(_Tp) - 1)) || (sizeof(_Tp) > 16) ? 0 : sizeof(_Tp);
+
+public:
+  using value_type = _Tp;
+
+  static constexpr size_t required_alignment = alignof(_Tp) > __min_alignment ? alignof(_Tp) : __min_alignment;
+
+  // The __atomic_always_lock_free builtin takes into account the alignment of the pointer if provided,
+  // so we create a fake pointer with a suitable alignment when querying it. Note that we are guaranteed
+  // that the pointer is going to be aligned properly at runtime because that is a (checked) precondition
+  // of atomic_ref's constructor.
+  static constexpr bool is_always_lock_free =
+      __atomic_always_lock_free(sizeof(_Tp), &__get_aligner_instance<required_alignment>::__instance);
+
+  _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return __atomic_is_lock_free(sizeof(_Tp), __ptr_); }
+
+  _LIBCPP_HIDE_FROM_ABI void store(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept
+      _LIBCPP_CHECK_STORE_MEMORY_ORDER(__order) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __order == memory_order::relaxed || __order == memory_order::release || __order == memory_order::seq_cst,
+        "atomic_ref: memory order argument to atomic store operation is invalid");
+    __atomic_store(__ptr_, __clear_padding(__desired), std::__to_gcc_order(__order));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept {
+    store(__desired);
+    return __desired;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __order = memory_order::seq_cst) const noexcept
+      _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__order) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
+            __order == memory_order::seq_cst,
+        "atomic_ref: memory order argument to atomic load operation is invalid");
+    alignas(_Tp) byte __mem[sizeof(_Tp)];
+    auto* __ret = reinterpret_cast<_Tp*>(__mem);
+    __atomic_load(__ptr_, __ret, std::__to_gcc_order(__order));
+    return *__ret;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI operator _Tp() const noexcept { return load(); }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
+    alignas(_Tp) byte __mem[sizeof(_Tp)];
+    auto* __ret = reinterpret_cast<_Tp*>(__mem);
+    __atomic_exchange(__ptr_, __clear_padding(__desired), __ret, std::__to_gcc_order(__order));
+    return *__ret;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool
+  compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
+      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __failure == memory_order::relaxed || __failure == memory_order::consume ||
+            __failure == memory_order::acquire || __failure == memory_order::seq_cst,
+        "atomic_ref: failure memory order argument to weak atomic compare-and-exchange operation is invalid");
+    return __compare_exchange(
+        __ptr_,
+        std::addressof(__expected),
+        std::addressof(__desired),
+        true,
+        std::__to_gcc_order(__success),
+        std::__to_gcc_order(__failure));
+  }
+  _LIBCPP_HIDE_FROM_ABI bool
+  compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
+      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __failure == memory_order::relaxed || __failure == memory_order::consume ||
+            __failure == memory_order::acquire || __failure == memory_order::seq_cst,
+        "atomic_ref: failure memory order argument to strong atomic compare-and-exchange operation is invalid");
+    return __compare_exchange(
+        __ptr_,
+        std::addressof(__expected),
+        std::addressof(__desired),
+        false,
+        std::__to_gcc_order(__success),
+        std::__to_gcc_order(__failure));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool
+  compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
+    return __compare_exchange(
+        __ptr_,
+        std::addressof(__expected),
+        std::addressof(__desired),
+        true,
+        std::__to_gcc_order(__order),
+        std::__to_gcc_failure_order(__order));
+  }
+  _LIBCPP_HIDE_FROM_ABI bool
+  compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
+    return __compare_exchange(
+        __ptr_,
+        std::addressof(__expected),
+        std::addressof(__desired),
+        false,
+        std::__to_gcc_order(__order),
+        std::__to_gcc_failure_order(__order));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void wait(_Tp __old, memory_order __order = memory_order::seq_cst) const noexcept
+      _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__order) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
+            __order == memory_order::seq_cst,
+        "atomic_ref: memory order argument to atomic wait operation is invalid");
+    std::__atomic_wait(*this, __old, __order);
+  }
+  _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept { std::__atomic_notify_one(*this); }
+  _LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept { std::__atomic_notify_all(*this); }
+
+protected:
+  typedef _Tp _Aligned_Tp __attribute__((aligned(required_alignment)));
+  _Aligned_Tp* __ptr_;
+
+  _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __ptr_(std::addressof(__obj)) {}
+};
+
+template <class _Tp>
+struct __atomic_waitable_traits<__atomic_ref_base<_Tp>> {
+  static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_ref_base<_Tp>& __a, memory_order __order) {
+    return __a.load(__order);
+  }
+  static _LIBCPP_HIDE_FROM_ABI const _Tp* __atomic_contention_address(const __atomic_ref_base<_Tp>& __a) {
+    return __a.__ptr_;
+  }
+};
+
+template <class _Tp>
+struct atomic_ref : public __atomic_ref_base<_Tp> {
+  static_assert(is_trivially_copyable_v<_Tp>, "std::atomic_ref<T> requires that 'T' be a trivially copyable type");
+
+  using __base = __atomic_ref_base<_Tp>;
+
+  _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        reinterpret_cast<uintptr_t>(std::addressof(__obj)) % __base::required_alignment == 0,
+        "atomic_ref ctor: referenced object must be aligned to required_alignment");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
+
+  atomic_ref& operator=(const atomic_ref&) = delete;
+};
+
+template <class _Tp>
+  requires(std::integral<_Tp> && !std::same_as<bool, _Tp>)
+struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {
+  using __base = __atomic_ref_base<_Tp>;
+
+  using difference_type = __base::value_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        reinterpret_cast<uintptr_t>(std::addressof(__obj)) % __base::required_alignment == 0,
+        "atomic_ref ctor: referenced object must be aligned to required_alignment");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
+
+  atomic_ref& operator=(const atomic_ref&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __atomic_fetch_add(this->__ptr_, __arg, std::__to_gcc_order(__order));
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __atomic_fetch_sub(this->__ptr_, __arg, std::__to_gcc_order(__order));
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __atomic_fetch_and(this->__ptr_, __arg, std::__to_gcc_order(__order));
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __atomic_fetch_or(this->__ptr_, __arg, std::__to_gcc_order(__order));
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __atomic_fetch_xor(this->__ptr_, __arg, std::__to_gcc_order(__order));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) const noexcept { return fetch_add(_Tp(1)); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) const noexcept { return fetch_sub(_Tp(1)); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator++() const noexcept { return fetch_add(_Tp(1)) + _Tp(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator--() const noexcept { return fetch_sub(_Tp(1)) - _Tp(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __arg) const noexcept { return fetch_and(__arg) & __arg; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __arg) const noexcept { return fetch_or(__arg) | __arg; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __arg) const noexcept { return fetch_xor(__arg) ^ __arg; }
+};
+
+template <class _Tp>
+  requires std::floating_point<_Tp>
+struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {
+  using __base = __atomic_ref_base<_Tp>;
+
+  using difference_type = __base::value_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        reinterpret_cast<uintptr_t>(std::addressof(__obj)) % __base::required_alignment == 0,
+        "atomic_ref ctor: referenced object must be aligned to required_alignment");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
+
+  atomic_ref& operator=(const atomic_ref&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    _Tp __old = this->load(memory_order_relaxed);
+    _Tp __new = __old + __arg;
+    while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {
+      __new = __old + __arg;
+    }
+    return __old;
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    _Tp __old = this->load(memory_order_relaxed);
+    _Tp __new = __old - __arg;
+    while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {
+      __new = __old - __arg;
+    }
+    return __old;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; }
+};
+
+template <class _Tp>
+struct atomic_ref<_Tp*> : public __atomic_ref_base<_Tp*> {
+  using __base = __atomic_ref_base<_Tp*>;
+
+  using difference_type = ptrdiff_t;
+
+  _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp*& __ptr) : __base(__ptr) {}
+
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __desired) const noexcept { return __base::operator=(__desired); }
+
+  atomic_ref& operator=(const atomic_ref&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __atomic_fetch_add(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order));
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __atomic_fetch_sub(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) const noexcept { return fetch_add(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) const noexcept { return fetch_sub(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator++() const noexcept { return fetch_add(1) + 1; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator--() const noexcept { return fetch_sub(1) - 1; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __arg) const noexcept { return fetch_add(__arg) + __arg; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __arg) const noexcept { return fetch_sub(__arg) - __arg; }
+};
+
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(atomic_ref);
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP__ATOMIC_ATOMIC_REF_H
diff --git a/libcxx/include/__cxx03/__atomic/atomic_sync.h b/libcxx/include/__cxx03/__atomic/atomic_sync.h
new file mode 100644
index 000000000000..aaf81f58731a
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/atomic_sync.h
@@ -0,0 +1,205 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_H
+#define _LIBCPP___ATOMIC_ATOMIC_SYNC_H
+
+#include <__atomic/contention_t.h>
+#include <__atomic/cxx_atomic_impl.h>
+#include <__atomic/memory_order.h>
+#include <__atomic/to_gcc_order.h>
+#include <__chrono/duration.h>
+#include <__config>
+#include <__memory/addressof.h>
+#include <__thread/poll_with_backoff.h>
+#include <__thread/support.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/invoke.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+#include <cstring>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// The customisation points to enable the following functions:
+// - __atomic_wait
+// - __atomic_wait_unless
+// - __atomic_notify_one
+// - __atomic_notify_all
+// Note that std::atomic<T>::wait was back-ported to C++03
+// The below implementations look ugly to support C++03
+template <class _Tp, class = void>
+struct __atomic_waitable_traits {
+  template <class _AtomicWaitable>
+  static void __atomic_load(_AtomicWaitable&&, memory_order) = delete;
+
+  template <class _AtomicWaitable>
+  static void __atomic_contention_address(_AtomicWaitable&&) = delete;
+};
+
+template <class _Tp, class = void>
+struct __atomic_waitable : false_type {};
+
+template <class _Tp>
+struct __atomic_waitable< _Tp,
+                          __void_t<decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_load(
+                                       std::declval<const _Tp&>(), std::declval<memory_order>())),
+                                   decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_contention_address(
+                                       std::declval<const _Tp&>()))> > : true_type {};
+
+template <class _AtomicWaitable, class _Poll>
+struct __atomic_wait_poll_impl {
+  const _AtomicWaitable& __a_;
+  _Poll __poll_;
+  memory_order __order_;
+
+  _LIBCPP_HIDE_FROM_ABI bool operator()() const {
+    auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a_, __order_);
+    return __poll_(__current_val);
+  }
+};
+
+#ifndef _LIBCPP_HAS_NO_THREADS
+
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*) _NOEXCEPT;
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*) _NOEXCEPT;
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
+__libcpp_atomic_monitor(void const volatile*) _NOEXCEPT;
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
+__libcpp_atomic_wait(void const volatile*, __cxx_contention_t) _NOEXCEPT;
+
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
+__cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*) _NOEXCEPT;
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
+__cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*) _NOEXCEPT;
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
+__libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*) _NOEXCEPT;
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
+__libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t) _NOEXCEPT;
+
+template <class _AtomicWaitable, class _Poll>
+struct __atomic_wait_backoff_impl {
+  const _AtomicWaitable& __a_;
+  _Poll __poll_;
+  memory_order __order_;
+
+  using __waitable_traits = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
+
+  _LIBCPP_AVAILABILITY_SYNC
+  _LIBCPP_HIDE_FROM_ABI bool
+  __update_monitor_val_and_poll(__cxx_atomic_contention_t const volatile*, __cxx_contention_t& __monitor_val) const {
+    // In case the contention type happens to be __cxx_atomic_contention_t, i.e. __cxx_atomic_impl<int64_t>,
+    // the platform wait is directly monitoring the atomic value itself.
+    // `__poll_` takes the current value of the atomic as an in-out argument
+    // to potentially modify it. After it returns, `__monitor` has a value
+    // which can be safely waited on by `std::__libcpp_atomic_wait` without any
+    // ABA style issues.
+    __monitor_val = __waitable_traits::__atomic_load(__a_, __order_);
+    return __poll_(__monitor_val);
+  }
+
+  _LIBCPP_AVAILABILITY_SYNC
+  _LIBCPP_HIDE_FROM_ABI bool
+  __update_monitor_val_and_poll(void const volatile* __contention_address, __cxx_contention_t& __monitor_val) const {
+    // In case the contention type is anything else, platform wait is monitoring a __cxx_atomic_contention_t
+    // from the global pool, the monitor comes from __libcpp_atomic_monitor
+    __monitor_val      = std::__libcpp_atomic_monitor(__contention_address);
+    auto __current_val = __waitable_traits::__atomic_load(__a_, __order_);
+    return __poll_(__current_val);
+  }
+
+  _LIBCPP_AVAILABILITY_SYNC
+  _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const {
+    if (__elapsed > chrono::microseconds(64)) {
+      auto __contention_address = __waitable_traits::__atomic_contention_address(__a_);
+      __cxx_contention_t __monitor_val;
+      if (__update_monitor_val_and_poll(__contention_address, __monitor_val))
+        return true;
+      std::__libcpp_atomic_wait(__contention_address, __monitor_val);
+    } else if (__elapsed > chrono::microseconds(4))
+      __libcpp_thread_yield();
+    else {
+    } // poll
+    return false;
+  }
+};
+
+// The semantics of this function are similar to `atomic`'s
+// `.wait(T old, std::memory_order order)`, but instead of having a hardcoded
+// predicate (is the loaded value unequal to `old`?), the predicate function is
+// specified as an argument. The loaded value is given as an in-out argument to
+// the predicate. If the predicate function returns `true`,
+// `__atomic_wait_unless` will return. If the predicate function returns
+// `false`, it must set the argument to its current understanding of the atomic
+// value. The predicate function must not return `false` spuriously.
+template <class _AtomicWaitable, class _Poll>
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+__atomic_wait_unless(const _AtomicWaitable& __a, _Poll&& __poll, memory_order __order) {
+  static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
+  __atomic_wait_poll_impl<_AtomicWaitable, __decay_t<_Poll> > __poll_impl     = {__a, __poll, __order};
+  __atomic_wait_backoff_impl<_AtomicWaitable, __decay_t<_Poll> > __backoff_fn = {__a, __poll, __order};
+  std::__libcpp_thread_poll_with_backoff(__poll_impl, __backoff_fn);
+}
+
+template <class _AtomicWaitable>
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) {
+  static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
+  std::__cxx_atomic_notify_one(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
+}
+
+template <class _AtomicWaitable>
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) {
+  static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
+  std::__cxx_atomic_notify_all(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
+}
+
+#else // _LIBCPP_HAS_NO_THREADS
+
+template <class _AtomicWaitable, class _Poll>
+_LIBCPP_HIDE_FROM_ABI void __atomic_wait_unless(const _AtomicWaitable& __a, _Poll&& __poll, memory_order __order) {
+  __atomic_wait_poll_impl<_AtomicWaitable, __decay_t<_Poll> > __poll_fn = {__a, __poll, __order};
+  std::__libcpp_thread_poll_with_backoff(__poll_fn, __spinning_backoff_policy());
+}
+
+template <class _AtomicWaitable>
+_LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable&) {}
+
+template <class _AtomicWaitable>
+_LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable&) {}
+
+#endif // _LIBCPP_HAS_NO_THREADS
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) {
+  return std::memcmp(std::addressof(__lhs), std::addressof(__rhs), sizeof(_Tp)) == 0;
+}
+
+template <class _Tp>
+struct __atomic_compare_unequal_to {
+  _Tp __val_;
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __arg) const {
+    return !std::__cxx_nonatomic_compare_equal(__arg, __val_);
+  }
+};
+
+template <class _AtomicWaitable, class _Up>
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+__atomic_wait(_AtomicWaitable& __a, _Up __val, memory_order __order) {
+  static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
+  __atomic_compare_unequal_to<_Up> __nonatomic_equal = {__val};
+  std::__atomic_wait_unless(__a, __nonatomic_equal, __order);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_ATOMIC_SYNC_H
diff --git a/libcxx/include/__cxx03/__atomic/check_memory_order.h b/libcxx/include/__cxx03/__atomic/check_memory_order.h
new file mode 100644
index 000000000000..536f764a6190
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/check_memory_order.h
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_CHECK_MEMORY_ORDER_H
+#define _LIBCPP___ATOMIC_CHECK_MEMORY_ORDER_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m)                                                                          \
+  _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || __m == memory_order_acquire || __m == memory_order_acq_rel,  \
+                           "memory order argument to atomic operation is invalid")
+
+#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)                                                                           \
+  _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || __m == memory_order_acq_rel,                                 \
+                           "memory order argument to atomic operation is invalid")
+
+#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f)                                                                  \
+  _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || __f == memory_order_acq_rel,                                 \
+                           "memory order argument to atomic operation is invalid")
+
+#define _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__m)                                                                           \
+  _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || __m == memory_order_acq_rel,                                 \
+                           "memory order argument to atomic operation is invalid")
+
+#endif // _LIBCPP___ATOMIC_CHECK_MEMORY_ORDER_H
diff --git a/libcxx/include/__cxx03/__atomic/contention_t.h b/libcxx/include/__cxx03/__atomic/contention_t.h
new file mode 100644
index 000000000000..65890f338ce9
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/contention_t.h
@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_CONTENTION_T_H
+#define _LIBCPP___ATOMIC_CONTENTION_T_H
+
+#include <__atomic/cxx_atomic_impl.h>
+#include <__config>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if defined(__linux__) || (defined(_AIX) && !defined(__64BIT__))
+using __cxx_contention_t = int32_t;
+#else
+using __cxx_contention_t = int64_t;
+#endif // __linux__ || (_AIX && !__64BIT__)
+
+using __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_CONTENTION_T_H
diff --git a/libcxx/include/__cxx03/__atomic/cxx_atomic_impl.h b/libcxx/include/__cxx03/__atomic/cxx_atomic_impl.h
new file mode 100644
index 000000000000..18e88aa97bec
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/cxx_atomic_impl.h
@@ -0,0 +1,510 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H
+#define _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H
+
+#include <__atomic/memory_order.h>
+#include <__atomic/to_gcc_order.h>
+#include <__config>
+#include <__memory/addressof.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_trivially_copyable.h>
+#include <__type_traits/remove_const.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP)
+
+// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because
+// the default operator= in an object is not volatile, a byte-by-byte copy
+// is required.
+template <typename _Tp, typename _Tv, __enable_if_t<is_assignable<_Tp&, _Tv>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) {
+  __a_value = __val;
+}
+template <typename _Tp, typename _Tv, __enable_if_t<is_assignable<_Tp&, _Tv>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) {
+  volatile char* __to         = reinterpret_cast<volatile char*>(std::addressof(__a_value));
+  volatile char* __end        = __to + sizeof(_Tp);
+  volatile const char* __from = reinterpret_cast<volatile const char*>(std::addressof(__val));
+  while (__to != __end)
+    *__to++ = *__from++;
+}
+
+template <typename _Tp>
+struct __cxx_atomic_base_impl {
+  _LIBCPP_HIDE_FROM_ABI
+#  ifndef _LIBCPP_CXX03_LANG
+  __cxx_atomic_base_impl() _NOEXCEPT = default;
+#  else
+  __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
+  }
+#  endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT : __a_value(value) {}
+  _Tp __a_value;
+};
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
+  __cxx_atomic_assign_volatile(__a->__a_value, __val);
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
+  __a->__a_value = __val;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) {
+  __atomic_thread_fence(__to_gcc_order(__order));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) {
+  __atomic_signal_fence(__to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
+  __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
+  __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
+  _Tp __ret;
+  __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
+  return __ret;
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_load_inplace(const volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
+  __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_load_inplace(const __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
+  __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
+  _Tp __ret;
+  __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
+  return __ret;
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
+  _Tp __ret;
+  __atomic_exchange(
+      std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
+  return __ret;
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
+  _Tp __ret;
+  __atomic_exchange(
+      std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
+  return __ret;
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
+    volatile __cxx_atomic_base_impl<_Tp>* __a,
+    _Tp* __expected,
+    _Tp __value,
+    memory_order __success,
+    memory_order __failure) {
+  return __atomic_compare_exchange(
+      std::addressof(__a->__a_value),
+      __expected,
+      std::addressof(__value),
+      false,
+      __to_gcc_order(__success),
+      __to_gcc_failure_order(__failure));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
+    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
+  return __atomic_compare_exchange(
+      std::addressof(__a->__a_value),
+      __expected,
+      std::addressof(__value),
+      false,
+      __to_gcc_order(__success),
+      __to_gcc_failure_order(__failure));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
+    volatile __cxx_atomic_base_impl<_Tp>* __a,
+    _Tp* __expected,
+    _Tp __value,
+    memory_order __success,
+    memory_order __failure) {
+  return __atomic_compare_exchange(
+      std::addressof(__a->__a_value),
+      __expected,
+      std::addressof(__value),
+      true,
+      __to_gcc_order(__success),
+      __to_gcc_failure_order(__failure));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
+    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
+  return __atomic_compare_exchange(
+      std::addressof(__a->__a_value),
+      __expected,
+      std::addressof(__value),
+      true,
+      __to_gcc_order(__success),
+      __to_gcc_failure_order(__failure));
+}
+
+template <typename _Tp>
+struct __skip_amt {
+  enum { value = 1 };
+};
+
+template <typename _Tp>
+struct __skip_amt<_Tp*> {
+  enum { value = sizeof(_Tp) };
+};
+
+// FIXME: Haven't figured out what the spec says about using arrays with
+// atomic_fetch_add. Force a failure rather than creating bad behavior.
+template <typename _Tp>
+struct __skip_amt<_Tp[]> {};
+template <typename _Tp, int n>
+struct __skip_amt<_Tp[n]> {};
+
+template <typename _Tp, typename _Td>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
+  return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
+}
+
+template <typename _Tp, typename _Td>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
+  return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
+}
+
+template <typename _Tp, typename _Td>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
+  return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
+}
+
+template <typename _Tp, typename _Td>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
+  return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+  return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+  return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+  return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+  return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+  return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+  return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+}
+
+#  define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0)
+
+#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP)
+
+template <typename _Tp>
+struct __cxx_atomic_base_impl {
+  _LIBCPP_HIDE_FROM_ABI
+#  ifndef _LIBCPP_CXX03_LANG
+  __cxx_atomic_base_impl() _NOEXCEPT = default;
+#  else
+  __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
+  }
+#  endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {}
+  _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value;
+};
+
+#  define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s)
+
+_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT {
+  __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT {
+  __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT {
+  __c11_atomic_init(std::addressof(__a->__a_value), __val);
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) _NOEXCEPT {
+  __c11_atomic_init(std::addressof(__a->__a_value), __val);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
+  __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
+  __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT {
+  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
+  return __c11_atomic_load(
+      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT {
+  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
+  return __c11_atomic_load(
+      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
+  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
+  *__dst           = __c11_atomic_load(
+      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
+  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
+  *__dst           = __c11_atomic_load(
+      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_exchange(
+      std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_exchange(
+      std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) {
+  // Avoid switch statement to make this a constexpr.
+  return __order == memory_order_release
+           ? memory_order_relaxed
+           : (__order == memory_order_acq_rel ? memory_order_acquire : __order);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
+    __cxx_atomic_base_impl<_Tp> volatile* __a,
+    _Tp* __expected,
+    _Tp __value,
+    memory_order __success,
+    memory_order __failure) _NOEXCEPT {
+  return __c11_atomic_compare_exchange_strong(
+      std::addressof(__a->__a_value),
+      __expected,
+      __value,
+      static_cast<__memory_order_underlying_t>(__success),
+      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
+    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
+    _NOEXCEPT {
+  return __c11_atomic_compare_exchange_strong(
+      std::addressof(__a->__a_value),
+      __expected,
+      __value,
+      static_cast<__memory_order_underlying_t>(__success),
+      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
+    __cxx_atomic_base_impl<_Tp> volatile* __a,
+    _Tp* __expected,
+    _Tp __value,
+    memory_order __success,
+    memory_order __failure) _NOEXCEPT {
+  return __c11_atomic_compare_exchange_weak(
+      std::addressof(__a->__a_value),
+      __expected,
+      __value,
+      static_cast<__memory_order_underlying_t>(__success),
+      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
+    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
+    _NOEXCEPT {
+  return __c11_atomic_compare_exchange_weak(
+      std::addressof(__a->__a_value),
+      __expected,
+      __value,
+      static_cast<__memory_order_underlying_t>(__success),
+      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_add(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_add(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp*
+__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_add(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp*
+__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_add(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_sub(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_sub(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp*
+__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_sub(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp*
+__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_sub(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_and(
+      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_and(
+      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_or(
+      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_or(
+      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_xor(
+      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_xor(
+      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
+}
+
+#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP
+
+template <typename _Tp, typename _Base = __cxx_atomic_base_impl<_Tp> >
+struct __cxx_atomic_impl : public _Base {
+  static_assert(is_trivially_copyable<_Tp>::value, "std::atomic<T> requires that 'T' be a trivially copyable type");
+
+  _LIBCPP_HIDE_FROM_ABI __cxx_atomic_impl() _NOEXCEPT = default;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp __value) _NOEXCEPT : _Base(__value) {}
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H
diff --git a/libcxx/include/__cxx03/__atomic/fence.h b/libcxx/include/__cxx03/__atomic/fence.h
new file mode 100644
index 000000000000..8c27ea54d62d
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/fence.h
@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_FENCE_H
+#define _LIBCPP___ATOMIC_FENCE_H
+
+#include <__atomic/cxx_atomic_impl.h>
+#include <__atomic/memory_order.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+inline _LIBCPP_HIDE_FROM_ABI void atomic_thread_fence(memory_order __m) _NOEXCEPT { __cxx_atomic_thread_fence(__m); }
+
+inline _LIBCPP_HIDE_FROM_ABI void atomic_signal_fence(memory_order __m) _NOEXCEPT { __cxx_atomic_signal_fence(__m); }
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_FENCE_H
diff --git a/libcxx/include/__cxx03/__atomic/is_always_lock_free.h b/libcxx/include/__cxx03/__atomic/is_always_lock_free.h
new file mode 100644
index 000000000000..f928e79f70ce
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/is_always_lock_free.h
@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_IS_ALWAYS_LOCK_FREE_H
+#define _LIBCPP___ATOMIC_IS_ALWAYS_LOCK_FREE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct __libcpp_is_always_lock_free {
+  // __atomic_always_lock_free is available in all Standard modes
+  static const bool __value = __atomic_always_lock_free(sizeof(_Tp), nullptr);
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_IS_ALWAYS_LOCK_FREE_H
diff --git a/libcxx/include/__cxx03/__atomic/kill_dependency.h b/libcxx/include/__cxx03/__atomic/kill_dependency.h
new file mode 100644
index 000000000000..103d52d35787
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/kill_dependency.h
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_KILL_DEPENDENCY_H
+#define _LIBCPP___ATOMIC_KILL_DEPENDENCY_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp kill_dependency(_Tp __y) _NOEXCEPT {
+  return __y;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_KILL_DEPENDENCY_H
diff --git a/libcxx/include/__cxx03/__atomic/memory_order.h b/libcxx/include/__cxx03/__atomic/memory_order.h
new file mode 100644
index 000000000000..294121d1c4e7
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/memory_order.h
@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_MEMORY_ORDER_H
+#define _LIBCPP___ATOMIC_MEMORY_ORDER_H
+
+#include <__config>
+#include <__type_traits/is_same.h>
+#include <__type_traits/underlying_type.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Figure out what the underlying type for `memory_order` would be if it were
+// declared as an unscoped enum (accounting for -fshort-enums). Use this result
+// to pin the underlying type in C++20.
+enum __legacy_memory_order { __mo_relaxed, __mo_consume, __mo_acquire, __mo_release, __mo_acq_rel, __mo_seq_cst };
+
+using __memory_order_underlying_t = underlying_type<__legacy_memory_order>::type;
+
+#if _LIBCPP_STD_VER >= 20
+
+enum class memory_order : __memory_order_underlying_t {
+  relaxed = __mo_relaxed,
+  consume = __mo_consume,
+  acquire = __mo_acquire,
+  release = __mo_release,
+  acq_rel = __mo_acq_rel,
+  seq_cst = __mo_seq_cst
+};
+
+static_assert(is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value,
+              "unexpected underlying type for std::memory_order");
+
+inline constexpr auto memory_order_relaxed = memory_order::relaxed;
+inline constexpr auto memory_order_consume = memory_order::consume;
+inline constexpr auto memory_order_acquire = memory_order::acquire;
+inline constexpr auto memory_order_release = memory_order::release;
+inline constexpr auto memory_order_acq_rel = memory_order::acq_rel;
+inline constexpr auto memory_order_seq_cst = memory_order::seq_cst;
+
+#else
+
+enum memory_order {
+  memory_order_relaxed = __mo_relaxed,
+  memory_order_consume = __mo_consume,
+  memory_order_acquire = __mo_acquire,
+  memory_order_release = __mo_release,
+  memory_order_acq_rel = __mo_acq_rel,
+  memory_order_seq_cst = __mo_seq_cst,
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_MEMORY_ORDER_H
diff --git a/libcxx/include/__cxx03/__atomic/to_gcc_order.h b/libcxx/include/__cxx03/__atomic/to_gcc_order.h
new file mode 100644
index 000000000000..d04c111addd3
--- /dev/null
+++ b/libcxx/include/__cxx03/__atomic/to_gcc_order.h
@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_TO_GCC_ORDER_H
+#define _LIBCPP___ATOMIC_TO_GCC_ORDER_H
+
+#include <__atomic/memory_order.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if defined(__ATOMIC_RELAXED) && defined(__ATOMIC_CONSUME) && defined(__ATOMIC_ACQUIRE) &&                             \
+    defined(__ATOMIC_RELEASE) && defined(__ATOMIC_ACQ_REL) && defined(__ATOMIC_SEQ_CST)
+
+_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) {
+  // Avoid switch statement to make this a constexpr.
+  return __order == memory_order_relaxed
+           ? __ATOMIC_RELAXED
+           : (__order == memory_order_acquire
+                  ? __ATOMIC_ACQUIRE
+                  : (__order == memory_order_release
+                         ? __ATOMIC_RELEASE
+                         : (__order == memory_order_seq_cst
+                                ? __ATOMIC_SEQ_CST
+                                : (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL : __ATOMIC_CONSUME))));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) {
+  // Avoid switch statement to make this a constexpr.
+  return __order == memory_order_relaxed
+           ? __ATOMIC_RELAXED
+           : (__order == memory_order_acquire
+                  ? __ATOMIC_ACQUIRE
+                  : (__order == memory_order_release
+                         ? __ATOMIC_RELAXED
+                         : (__order == memory_order_seq_cst
+                                ? __ATOMIC_SEQ_CST
+                                : (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE : __ATOMIC_CONSUME))));
+}
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_TO_GCC_ORDER_H
diff --git a/libcxx/include/__cxx03/__bit/bit_cast.h b/libcxx/include/__cxx03/__bit/bit_cast.h
new file mode 100644
index 000000000000..cd0456738179
--- /dev/null
+++ b/libcxx/include/__cxx03/__bit/bit_cast.h
@@ -0,0 +1,44 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___BIT_BIT_CAST_H
+#define _LIBCPP___BIT_BIT_CAST_H
+
+#include <__config>
+#include <__type_traits/is_trivially_copyable.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _ToType, class _FromType>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI constexpr _ToType __bit_cast(const _FromType& __from) noexcept {
+  return __builtin_bit_cast(_ToType, __from);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _ToType, class _FromType>
+  requires(sizeof(_ToType) == sizeof(_FromType) && is_trivially_copyable_v<_ToType> &&
+           is_trivially_copyable_v<_FromType>)
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _ToType bit_cast(const _FromType& __from) noexcept {
+  return __builtin_bit_cast(_ToType, __from);
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___BIT_BIT_CAST_H
diff --git a/libcxx/include/__cxx03/__bit/bit_ceil.h b/libcxx/include/__cxx03/__bit/bit_ceil.h
new file mode 100644
index 000000000000..cfd792dc2e2a
--- /dev/null
+++ b/libcxx/include/__cxx03/__bit/bit_ceil.h
@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___BIT_BIT_CEIL_H
+#define _LIBCPP___BIT_BIT_CEIL_H
+
+#include <__assert>
+#include <__bit/countl.h>
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp __bit_ceil(_Tp __t) noexcept {
+  if (__t < 2)
+    return 1;
+  const unsigned __n = numeric_limits<_Tp>::digits - std::__countl_zero((_Tp)(__t - 1u));
+  _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__n != numeric_limits<_Tp>::digits, "Bad input to bit_ceil");
+
+  if constexpr (sizeof(_Tp) >= sizeof(unsigned))
+    return _Tp{1} << __n;
+  else {
+    const unsigned __extra   = numeric_limits<unsigned>::digits - numeric_limits<_Tp>::digits;
+    const unsigned __ret_val = 1u << (__n + __extra);
+    return (_Tp)(__ret_val >> __extra);
+  }
+}
+
+#  if _LIBCPP_STD_VER >= 20
+
+template <__libcpp_unsigned_integer _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp bit_ceil(_Tp __t) noexcept {
+  return std::__bit_ceil(__t);
+}
+
+#  endif // _LIBCPP_STD_VER >= 20
+#endif   // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___BIT_BIT_CEIL_H
diff --git a/libcxx/include/__cxx03/__bit/bit_floor.h b/libcxx/include/__cxx03/__bit/bit_floor.h
new file mode 100644
index 000000000000..133e369504e4
--- /dev/null
+++ b/libcxx/include/__cxx03/__bit/bit_floor.h
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___BIT_BIT_FLOOR_H
+#define _LIBCPP___BIT_BIT_FLOOR_H
+
+#include <__bit/bit_log2.h>
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <__libcpp_unsigned_integer _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp bit_floor(_Tp __t) noexcept {
+  return __t == 0 ? 0 : _Tp{1} << std::__bit_log2(__t);
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___BIT_BIT_FLOOR_H
diff --git a/libcxx/include/__cxx03/__bit/bit_log2.h b/libcxx/include/__cxx03/__bit/bit_log2.h
new file mode 100644
index 000000000000..62936f678686
--- /dev/null
+++ b/libcxx/include/__cxx03/__bit/bit_log2.h
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___BIT_BIT_LOG2_H
+#define _LIBCPP___BIT_BIT_LOG2_H
+
+#include <__bit/countl.h>
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <__libcpp_unsigned_integer _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp __bit_log2(_Tp __t) noexcept {
+  return numeric_limits<_Tp>::digits - 1 - std::countl_zero(__t);
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___BIT_BIT_LOG2_H
diff --git a/libcxx/include/__cxx03/__bit/bit_width.h b/libcxx/include/__cxx03/__bit/bit_width.h
new file mode 100644
index 000000000000..853e481776f7
--- /dev/null
+++ b/libcxx/include/__cxx03/__bit/bit_width.h
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___BIT_BIT_WIDTH_H
+#define _LIBCPP___BIT_BIT_WIDTH_H
+
+#include <__bit/bit_log2.h>
+#include <__concepts/arithmetic.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <__libcpp_unsigned_integer _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr int bit_width(_Tp __t) noexcept {
+  return __t == 0 ? 0 : std::__bit_log2(__t) + 1;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___BIT_BIT_WIDTH_H
diff --git a/libcxx/include/__cxx03/__bit/blsr.h b/libcxx/include/__cxx03/__bit/blsr.h
new file mode 100644
index 000000000000..76bd521f5c30
--- /dev/null
+++ b/libcxx/include/__cxx03/__bit/blsr.h
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___BIT_BLSR_H
+#define _LIBCPP___BIT_BLSR_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR unsigned __libcpp_blsr(unsigned __x) _NOEXCEPT {
+  return __x ^ (__x & -__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR unsigned long __libcpp_blsr(unsigned long __x) _NOEXCEPT {
+  return __x ^ (__x & -__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR unsigned long long __libcpp_blsr(unsigned long long __x) _NOEXCEPT {
+  return __x ^ (__x & -__x);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___BIT_BLSR_H
diff --git a/libcxx/include/__cxx03/__bit/byteswap.h b/libcxx/include/__cxx03/__bit/byteswap.h
new file mode 100644
index 000000000000..6225ecf2f92d
--- /dev/null
+++ b/libcxx/include/__cxx03/__bit/byteswap.h
@@ -0,0 +1,53 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___BIT_BYTESWAP_H
+#define _LIBCPP___BIT_BYTESWAP_H
+
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <integral _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp byteswap(_Tp __val) noexcept {
+  if constexpr (sizeof(_Tp) == 1) {
+    return __val;
+  } else if constexpr (sizeof(_Tp) == 2) {
+    return __builtin_bswap16(__val);
+  } else if constexpr (sizeof(_Tp) == 4) {
+    return __builtin_bswap32(__val);
+  } else if constexpr (sizeof(_Tp) == 8) {
+    return __builtin_bswap64(__val);
+#  ifndef _LIBCPP_HAS_NO_INT128
+  } else if constexpr (sizeof(_Tp) == 16) {
+#    if __has_builtin(__builtin_bswap128)
+    return __builtin_bswap128(__val);
+#    else
+    return static_cast<_Tp>(byteswap(static_cast<uint64_t>(__val))) << 64 |
+           static_cast<_Tp>(byteswap(static_cast<uint64_t>(__val >> 64)));
+#    endif // __has_builtin(__builtin_bswap128)
+#  endif   // _LIBCPP_HAS_NO_INT128
+  } else {
+    static_assert(sizeof(_Tp) == 0, "byteswap is unimplemented for integral types of this size");
+  }
+}
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___BIT_BYTESWAP_H
diff --git a/libcxx/include/__cxx03/__bit/countl.h b/libcxx/include/__cxx03/__bit/countl.h
new file mode 100644
index 000000000000..998a0b44c19d
--- /dev/null
+++ b/libcxx/include/__cxx03/__bit/countl.h
@@ -0,0 +1,113 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// TODO: __builtin_clzg is available since Clang 19 and GCC 14. When support for older versions is dropped, we can
+//  refactor this code to exclusively use __builtin_clzg.
+
+#ifndef _LIBCPP___BIT_COUNTL_H
+#define _LIBCPP___BIT_COUNTL_H
+
+#include <__bit/rotate.h>
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <__type_traits/is_unsigned_integer.h>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_clz(unsigned __x) _NOEXCEPT {
+  return __builtin_clz(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_clz(unsigned long __x) _NOEXCEPT {
+  return __builtin_clzl(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_clz(unsigned long long __x) _NOEXCEPT {
+  return __builtin_clzll(__x);
+}
+
+#ifndef _LIBCPP_HAS_NO_INT128
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_clz(__uint128_t __x) _NOEXCEPT {
+#  if __has_builtin(__builtin_clzg)
+  return __builtin_clzg(__x);
+#  else
+  // The function is written in this form due to C++ constexpr limitations.
+  // The algorithm:
+  // - Test whether any bit in the high 64-bits is set
+  // - No bits set:
+  //   - The high 64-bits contain 64 leading zeros,
+  //   - Add the result of the low 64-bits.
+  // - Any bits set:
+  //   - The number of leading zeros of the input is the number of leading
+  //     zeros in the high 64-bits.
+  return ((__x >> 64) == 0) ? (64 + __builtin_clzll(static_cast<unsigned long long>(__x)))
+                            : __builtin_clzll(static_cast<unsigned long long>(__x >> 64));
+#  endif
+}
+#endif // _LIBCPP_HAS_NO_INT128
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int __countl_zero(_Tp __t) _NOEXCEPT {
+  static_assert(__libcpp_is_unsigned_integer<_Tp>::value, "__countl_zero requires an unsigned integer type");
+#if __has_builtin(__builtin_clzg)
+  return __builtin_clzg(__t, numeric_limits<_Tp>::digits);
+#else  // __has_builtin(__builtin_clzg)
+  if (__t == 0)
+    return numeric_limits<_Tp>::digits;
+
+  if (sizeof(_Tp) <= sizeof(unsigned int))
+    return std::__libcpp_clz(static_cast<unsigned int>(__t)) -
+           (numeric_limits<unsigned int>::digits - numeric_limits<_Tp>::digits);
+  else if (sizeof(_Tp) <= sizeof(unsigned long))
+    return std::__libcpp_clz(static_cast<unsigned long>(__t)) -
+           (numeric_limits<unsigned long>::digits - numeric_limits<_Tp>::digits);
+  else if (sizeof(_Tp) <= sizeof(unsigned long long))
+    return std::__libcpp_clz(static_cast<unsigned long long>(__t)) -
+           (numeric_limits<unsigned long long>::digits - numeric_limits<_Tp>::digits);
+  else {
+    int __ret                      = 0;
+    int __iter                     = 0;
+    const unsigned int __ulldigits = numeric_limits<unsigned long long>::digits;
+    while (true) {
+      __t = std::__rotl(__t, __ulldigits);
+      if ((__iter = std::__countl_zero(static_cast<unsigned long long>(__t))) != __ulldigits)
+        break;
+      __ret += __iter;
+    }
+    return __ret + __iter;
+  }
+#endif // __has_builtin(__builtin_clzg)
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+template <__libcpp_unsigned_integer _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr int countl_zero(_Tp __t) noexcept {
+  return std::__countl_zero(__t);
+}
+
+template <__libcpp_unsigned_integer _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr int countl_one(_Tp __t) noexcept {
+  return __t != numeric_limits<_Tp>::max() ? std::countl_zero(static_cast<_Tp>(~__t)) : numeric_limits<_Tp>::digits;
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___BIT_COUNTL_H
diff --git a/libcxx/include/__cxx03/__bit/countr.h b/libcxx/include/__cxx03/__bit/countr.h
new file mode 100644
index 000000000000..9e92021fba35
--- /dev/null
+++ b/libcxx/include/__cxx03/__bit/countr.h
@@ -0,0 +1,84 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// TODO: __builtin_ctzg is available since Clang 19 and GCC 14. When support for older versions is dropped, we can
+//  refactor this code to exclusively use __builtin_ctzg.
+
+#ifndef _LIBCPP___BIT_COUNTR_H
+#define _LIBCPP___BIT_COUNTR_H
+
+#include <__bit/rotate.h>
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_ctz(unsigned __x) _NOEXCEPT {
+  return __builtin_ctz(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_ctz(unsigned long __x) _NOEXCEPT {
+  return __builtin_ctzl(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_ctz(unsigned long long __x) _NOEXCEPT {
+  return __builtin_ctzll(__x);
+}
+
+template <class _Tp>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int __countr_zero(_Tp __t) _NOEXCEPT {
+#if __has_builtin(__builtin_ctzg)
+  return __builtin_ctzg(__t, numeric_limits<_Tp>::digits);
+#else  // __has_builtin(__builtin_ctzg)
+  if (__t == 0)
+    return numeric_limits<_Tp>::digits;
+  if (sizeof(_Tp) <= sizeof(unsigned int))
+    return std::__libcpp_ctz(static_cast<unsigned int>(__t));
+  else if (sizeof(_Tp) <= sizeof(unsigned long))
+    return std::__libcpp_ctz(static_cast<unsigned long>(__t));
+  else if (sizeof(_Tp) <= sizeof(unsigned long long))
+    return std::__libcpp_ctz(static_cast<unsigned long long>(__t));
+  else {
+    int __ret                      = 0;
+    const unsigned int __ulldigits = numeric_limits<unsigned long long>::digits;
+    while (static_cast<unsigned long long>(__t) == 0uLL) {
+      __ret += __ulldigits;
+      __t >>= __ulldigits;
+    }
+    return __ret + std::__libcpp_ctz(static_cast<unsigned long long>(__t));
+  }
+#endif // __has_builtin(__builtin_ctzg)
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+template <__libcpp_unsigned_integer _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) noexcept {
+  return std::__countr_zero(__t);
+}
+
+template <__libcpp_unsigned_integer _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr int countr_one(_Tp __t) noexcept {
+  return __t != numeric_limits<_Tp>::max() ? std::countr_zero(static_cast<_Tp>(~__t)) : numeric_limits<_Tp>::digits;
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___BIT_COUNTR_H
diff --git a/libcxx/include/__cxx03/__bit/endian.h b/libcxx/include/__cxx03/__bit/endian.h
new file mode 100644
index 000000000000..2d31e5ddff4f
--- /dev/null
+++ b/libcxx/include/__cxx03/__bit/endian.h
@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___BIT_ENDIAN_H
+#define _LIBCPP___BIT_ENDIAN_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+enum class endian {
+  little = 0xDEAD,
+  big    = 0xFACE,
+#  if defined(_LIBCPP_LITTLE_ENDIAN)
+  native = little
+#  elif defined(_LIBCPP_BIG_ENDIAN)
+  native = big
+#  else
+  native = 0xCAFE
+#  endif
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___BIT_ENDIAN_H
diff --git a/libcxx/include/__cxx03/__bit/has_single_bit.h b/libcxx/include/__cxx03/__bit/has_single_bit.h
new file mode 100644
index 000000000000..52f5853a1bc8
--- /dev/null
+++ b/libcxx/include/__cxx03/__bit/has_single_bit.h
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___BIT_HAS_SINGLE_BIT_H
+#define _LIBCPP___BIT_HAS_SINGLE_BIT_H
+
+#include <__concepts/arithmetic.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <__libcpp_unsigned_integer _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool has_single_bit(_Tp __t) noexcept {
+  return __t != 0 && (((__t & (__t - 1)) == 0));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___BIT_HAS_SINGLE_BIT_H
diff --git a/libcxx/include/__cxx03/__bit/invert_if.h b/libcxx/include/__cxx03/__bit/invert_if.h
new file mode 100644
index 000000000000..f7606ede26da
--- /dev/null
+++ b/libcxx/include/__cxx03/__bit/invert_if.h
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___BIT_INVERT_IF_H
+#define _LIBCPP___BIT_INVERT_IF_H
+
+#include <__concepts/arithmetic.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <bool _Invert, class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp __invert_if(_Tp __v) {
+  if (_Invert)
+    return ~__v;
+  return __v;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___BIT_INVERT_IF_H
diff --git a/libcxx/include/__cxx03/__bit/popcount.h b/libcxx/include/__cxx03/__bit/popcount.h
new file mode 100644
index 000000000000..5cf0a01d0733
--- /dev/null
+++ b/libcxx/include/__cxx03/__bit/popcount.h
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// TODO: __builtin_popcountg is available since Clang 19 and GCC 14. When support for older versions is dropped, we can
+//  refactor this code to exclusively use __builtin_popcountg.
+
+#ifndef _LIBCPP___BIT_POPCOUNT_H
+#define _LIBCPP___BIT_POPCOUNT_H
+
+#include <__bit/rotate.h>
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_popcount(unsigned __x) _NOEXCEPT {
+  return __builtin_popcount(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_popcount(unsigned long __x) _NOEXCEPT {
+  return __builtin_popcountl(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_popcount(unsigned long long __x) _NOEXCEPT {
+  return __builtin_popcountll(__x);
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+template <__libcpp_unsigned_integer _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr int popcount(_Tp __t) noexcept {
+#  if __has_builtin(__builtin_popcountg)
+  return __builtin_popcountg(__t);
+#  else  // __has_builtin(__builtin_popcountg)
+  if (sizeof(_Tp) <= sizeof(unsigned int))
+    return std::__libcpp_popcount(static_cast<unsigned int>(__t));
+  else if (sizeof(_Tp) <= sizeof(unsigned long))
+    return std::__libcpp_popcount(static_cast<unsigned long>(__t));
+  else if (sizeof(_Tp) <= sizeof(unsigned long long))
+    return std::__libcpp_popcount(static_cast<unsigned long long>(__t));
+  else {
+    int __ret = 0;
+    while (__t != 0) {
+      __ret += std::__libcpp_popcount(static_cast<unsigned long long>(__t));
+      __t >>= numeric_limits<unsigned long long>::digits;
+    }
+    return __ret;
+  }
+#  endif // __has_builtin(__builtin_popcountg)
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___BIT_POPCOUNT_H
diff --git a/libcxx/include/__cxx03/__bit/rotate.h b/libcxx/include/__cxx03/__bit/rotate.h
new file mode 100644
index 000000000000..90e430e9d042
--- /dev/null
+++ b/libcxx/include/__cxx03/__bit/rotate.h
@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___BIT_ROTATE_H
+#define _LIBCPP___BIT_ROTATE_H
+
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <__type_traits/is_unsigned_integer.h>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Writing two full functions for rotl and rotr makes it easier for the compiler
+// to optimize the code. On x86 this function becomes the ROL instruction and
+// the rotr function becomes the ROR instruction.
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp __rotl(_Tp __x, int __s) _NOEXCEPT {
+  static_assert(__libcpp_is_unsigned_integer<_Tp>::value, "__rotl requires an unsigned integer type");
+  const int __N = numeric_limits<_Tp>::digits;
+  int __r       = __s % __N;
+
+  if (__r == 0)
+    return __x;
+
+  if (__r > 0)
+    return (__x << __r) | (__x >> (__N - __r));
+
+  return (__x >> -__r) | (__x << (__N + __r));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp __rotr(_Tp __x, int __s) _NOEXCEPT {
+  static_assert(__libcpp_is_unsigned_integer<_Tp>::value, "__rotr requires an unsigned integer type");
+  const int __N = numeric_limits<_Tp>::digits;
+  int __r       = __s % __N;
+
+  if (__r == 0)
+    return __x;
+
+  if (__r > 0)
+    return (__x >> __r) | (__x << (__N - __r));
+
+  return (__x << -__r) | (__x >> (__N + __r));
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+template <__libcpp_unsigned_integer _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp rotl(_Tp __t, int __cnt) noexcept {
+  return std::__rotl(__t, __cnt);
+}
+
+template <__libcpp_unsigned_integer _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp rotr(_Tp __t, int __cnt) noexcept {
+  return std::__rotr(__t, __cnt);
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___BIT_ROTATE_H
diff --git a/libcxx/include/__cxx03/__bit_reference b/libcxx/include/__cxx03/__bit_reference
new file mode 100644
index 000000000000..22637d439741
--- /dev/null
+++ b/libcxx/include/__cxx03/__bit_reference
@@ -0,0 +1,1024 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___BIT_REFERENCE
+#define _LIBCPP___BIT_REFERENCE
+
+#include <__algorithm/copy_n.h>
+#include <__algorithm/fill_n.h>
+#include <__algorithm/min.h>
+#include <__bit/countr.h>
+#include <__bit/invert_if.h>
+#include <__bit/popcount.h>
+#include <__compare/ordering.h>
+#include <__config>
+#include <__fwd/bit_reference.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/construct_at.h>
+#include <__memory/pointer_traits.h>
+#include <__type_traits/conditional.h>
+#include <__utility/swap.h>
+#include <cstring>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Cp>
+class __bit_const_reference;
+
+template <class _Tp>
+struct __has_storage_type {
+  static const bool value = false;
+};
+
+template <class _Cp, bool = __has_storage_type<_Cp>::value>
+class __bit_reference {
+  using __storage_type    = typename _Cp::__storage_type;
+  using __storage_pointer = typename _Cp::__storage_pointer;
+
+  __storage_pointer __seg_;
+  __storage_type __mask_;
+
+  friend typename _Cp::__self;
+
+  friend class __bit_const_reference<_Cp>;
+  friend class __bit_iterator<_Cp, false>;
+
+public:
+  using __container = typename _Cp::__self;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_reference(const __bit_reference&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 operator bool() const _NOEXCEPT {
+    return static_cast<bool>(*__seg_ & __mask_);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool operator~() const _NOEXCEPT {
+    return !static_cast<bool>(*this);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_reference& operator=(bool __x) _NOEXCEPT {
+    if (__x)
+      *__seg_ |= __mask_;
+    else
+      *__seg_ &= ~__mask_;
+    return *this;
+  }
+
+#if _LIBCPP_STD_VER >= 23
+  _LIBCPP_HIDE_FROM_ABI constexpr const __bit_reference& operator=(bool __x) const noexcept {
+    if (__x)
+      *__seg_ |= __mask_;
+    else
+      *__seg_ &= ~__mask_;
+    return *this;
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_reference& operator=(const __bit_reference& __x) _NOEXCEPT {
+    return operator=(static_cast<bool>(__x));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void flip() _NOEXCEPT { *__seg_ ^= __mask_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, false> operator&() const _NOEXCEPT {
+    return __bit_iterator<_Cp, false>(__seg_, static_cast<unsigned>(std::__libcpp_ctz(__mask_)));
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit __bit_reference(__storage_pointer __s, __storage_type __m) _NOEXCEPT
+      : __seg_(__s),
+        __mask_(__m) {}
+};
+
+template <class _Cp>
+class __bit_reference<_Cp, false> {};
+
+template <class _Cp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+swap(__bit_reference<_Cp> __x, __bit_reference<_Cp> __y) _NOEXCEPT {
+  bool __t = __x;
+  __x      = __y;
+  __y      = __t;
+}
+
+template <class _Cp, class _Dp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+swap(__bit_reference<_Cp> __x, __bit_reference<_Dp> __y) _NOEXCEPT {
+  bool __t = __x;
+  __x      = __y;
+  __y      = __t;
+}
+
+template <class _Cp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(__bit_reference<_Cp> __x, bool& __y) _NOEXCEPT {
+  bool __t = __x;
+  __x      = __y;
+  __y      = __t;
+}
+
+template <class _Cp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(bool& __x, __bit_reference<_Cp> __y) _NOEXCEPT {
+  bool __t = __x;
+  __x      = __y;
+  __y      = __t;
+}
+
+template <class _Cp>
+class __bit_const_reference {
+  using __storage_type    = typename _Cp::__storage_type;
+  using __storage_pointer = typename _Cp::__const_storage_pointer;
+
+  __storage_pointer __seg_;
+  __storage_type __mask_;
+
+  friend typename _Cp::__self;
+  friend class __bit_iterator<_Cp, true>;
+
+public:
+  using __container = typename _Cp::__self;
+
+  _LIBCPP_HIDE_FROM_ABI __bit_const_reference(const __bit_const_reference&) = default;
+  __bit_const_reference& operator=(const __bit_const_reference&)            = delete;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_const_reference(const __bit_reference<_Cp>& __x) _NOEXCEPT
+      : __seg_(__x.__seg_),
+        __mask_(__x.__mask_) {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR operator bool() const _NOEXCEPT {
+    return static_cast<bool>(*__seg_ & __mask_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, true> operator&() const _NOEXCEPT {
+    return __bit_iterator<_Cp, true>(__seg_, static_cast<unsigned>(std::__libcpp_ctz(__mask_)));
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR explicit __bit_const_reference(__storage_pointer __s, __storage_type __m) _NOEXCEPT
+      : __seg_(__s),
+        __mask_(__m) {}
+};
+
+// copy
+
+template <class _Cp, bool _IsConst>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_aligned(
+    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
+  using _In             = __bit_iterator<_Cp, _IsConst>;
+  using difference_type = typename _In::difference_type;
+  using __storage_type  = typename _In::__storage_type;
+
+  const int __bits_per_word = _In::__bits_per_word;
+  difference_type __n       = __last - __first;
+  if (__n > 0) {
+    // do first word
+    if (__first.__ctz_ != 0) {
+      unsigned __clz       = __bits_per_word - __first.__ctz_;
+      difference_type __dn = std::min(static_cast<difference_type>(__clz), __n);
+      __n -= __dn;
+      __storage_type __m = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz - __dn));
+      __storage_type __b = *__first.__seg_ & __m;
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b;
+      __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
+      __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
+      ++__first.__seg_;
+      // __first.__ctz_ = 0;
+    }
+    // __first.__ctz_ == 0;
+    // do middle words
+    __storage_type __nw = __n / __bits_per_word;
+    std::copy_n(std::__to_address(__first.__seg_), __nw, std::__to_address(__result.__seg_));
+    __n -= __nw * __bits_per_word;
+    __result.__seg_ += __nw;
+    // do last word
+    if (__n > 0) {
+      __first.__seg_ += __nw;
+      __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+      __storage_type __b = *__first.__seg_ & __m;
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b;
+      __result.__ctz_ = static_cast<unsigned>(__n);
+    }
+  }
+  return __result;
+}
+
+template <class _Cp, bool _IsConst>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_unaligned(
+    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
+  using _In             = __bit_iterator<_Cp, _IsConst>;
+  using difference_type = typename _In::difference_type;
+  using __storage_type  = typename _In::__storage_type;
+
+  const int __bits_per_word = _In::__bits_per_word;
+  difference_type __n       = __last - __first;
+  if (__n > 0) {
+    // do first word
+    if (__first.__ctz_ != 0) {
+      unsigned __clz_f     = __bits_per_word - __first.__ctz_;
+      difference_type __dn = std::min(static_cast<difference_type>(__clz_f), __n);
+      __n -= __dn;
+      __storage_type __m   = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn));
+      __storage_type __b   = *__first.__seg_ & __m;
+      unsigned __clz_r     = __bits_per_word - __result.__ctz_;
+      __storage_type __ddn = std::min<__storage_type>(__dn, __clz_r);
+      __m                  = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __ddn));
+      *__result.__seg_ &= ~__m;
+      if (__result.__ctz_ > __first.__ctz_)
+        *__result.__seg_ |= __b << (__result.__ctz_ - __first.__ctz_);
+      else
+        *__result.__seg_ |= __b >> (__first.__ctz_ - __result.__ctz_);
+      __result.__seg_ += (__ddn + __result.__ctz_) / __bits_per_word;
+      __result.__ctz_ = static_cast<unsigned>((__ddn + __result.__ctz_) % __bits_per_word);
+      __dn -= __ddn;
+      if (__dn > 0) {
+        __m = ~__storage_type(0) >> (__bits_per_word - __dn);
+        *__result.__seg_ &= ~__m;
+        *__result.__seg_ |= __b >> (__first.__ctz_ + __ddn);
+        __result.__ctz_ = static_cast<unsigned>(__dn);
+      }
+      ++__first.__seg_;
+      // __first.__ctz_ = 0;
+    }
+    // __first.__ctz_ == 0;
+    // do middle words
+    unsigned __clz_r   = __bits_per_word - __result.__ctz_;
+    __storage_type __m = ~__storage_type(0) << __result.__ctz_;
+    for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first.__seg_) {
+      __storage_type __b = *__first.__seg_;
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b << __result.__ctz_;
+      ++__result.__seg_;
+      *__result.__seg_ &= __m;
+      *__result.__seg_ |= __b >> __clz_r;
+    }
+    // do last word
+    if (__n > 0) {
+      __m                 = ~__storage_type(0) >> (__bits_per_word - __n);
+      __storage_type __b  = *__first.__seg_ & __m;
+      __storage_type __dn = std::min(__n, static_cast<difference_type>(__clz_r));
+      __m                 = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __dn));
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b << __result.__ctz_;
+      __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
+      __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
+      __n -= __dn;
+      if (__n > 0) {
+        __m = ~__storage_type(0) >> (__bits_per_word - __n);
+        *__result.__seg_ &= ~__m;
+        *__result.__seg_ |= __b >> __dn;
+        __result.__ctz_ = static_cast<unsigned>(__n);
+      }
+    }
+  }
+  return __result;
+}
+
+template <class _Cp, bool _IsConst>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, false>
+copy(__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
+  if (__first.__ctz_ == __result.__ctz_)
+    return std::__copy_aligned(__first, __last, __result);
+  return std::__copy_unaligned(__first, __last, __result);
+}
+
+// copy_backward
+
+template <class _Cp, bool _IsConst>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_backward_aligned(
+    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
+  using _In             = __bit_iterator<_Cp, _IsConst>;
+  using difference_type = typename _In::difference_type;
+  using __storage_type  = typename _In::__storage_type;
+
+  const int __bits_per_word = _In::__bits_per_word;
+  difference_type __n       = __last - __first;
+  if (__n > 0) {
+    // do first word
+    if (__last.__ctz_ != 0) {
+      difference_type __dn = std::min(static_cast<difference_type>(__last.__ctz_), __n);
+      __n -= __dn;
+      unsigned __clz     = __bits_per_word - __last.__ctz_;
+      __storage_type __m = (~__storage_type(0) << (__last.__ctz_ - __dn)) & (~__storage_type(0) >> __clz);
+      __storage_type __b = *__last.__seg_ & __m;
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b;
+      __result.__ctz_ = static_cast<unsigned>(((-__dn & (__bits_per_word - 1)) + __result.__ctz_) % __bits_per_word);
+      // __last.__ctz_ = 0
+    }
+    // __last.__ctz_ == 0 || __n == 0
+    // __result.__ctz_ == 0 || __n == 0
+    // do middle words
+    __storage_type __nw = __n / __bits_per_word;
+    __result.__seg_ -= __nw;
+    __last.__seg_ -= __nw;
+    std::copy_n(std::__to_address(__last.__seg_), __nw, std::__to_address(__result.__seg_));
+    __n -= __nw * __bits_per_word;
+    // do last word
+    if (__n > 0) {
+      __storage_type __m = ~__storage_type(0) << (__bits_per_word - __n);
+      __storage_type __b = *--__last.__seg_ & __m;
+      *--__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b;
+      __result.__ctz_ = static_cast<unsigned>(-__n & (__bits_per_word - 1));
+    }
+  }
+  return __result;
+}
+
+template <class _Cp, bool _IsConst>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_backward_unaligned(
+    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
+  using _In             = __bit_iterator<_Cp, _IsConst>;
+  using difference_type = typename _In::difference_type;
+  using __storage_type  = typename _In::__storage_type;
+
+  const int __bits_per_word = _In::__bits_per_word;
+  difference_type __n       = __last - __first;
+  if (__n > 0) {
+    // do first word
+    if (__last.__ctz_ != 0) {
+      difference_type __dn = std::min(static_cast<difference_type>(__last.__ctz_), __n);
+      __n -= __dn;
+      unsigned __clz_l     = __bits_per_word - __last.__ctz_;
+      __storage_type __m   = (~__storage_type(0) << (__last.__ctz_ - __dn)) & (~__storage_type(0) >> __clz_l);
+      __storage_type __b   = *__last.__seg_ & __m;
+      unsigned __clz_r     = __bits_per_word - __result.__ctz_;
+      __storage_type __ddn = std::min(__dn, static_cast<difference_type>(__result.__ctz_));
+      if (__ddn > 0) {
+        __m = (~__storage_type(0) << (__result.__ctz_ - __ddn)) & (~__storage_type(0) >> __clz_r);
+        *__result.__seg_ &= ~__m;
+        if (__result.__ctz_ > __last.__ctz_)
+          *__result.__seg_ |= __b << (__result.__ctz_ - __last.__ctz_);
+        else
+          *__result.__seg_ |= __b >> (__last.__ctz_ - __result.__ctz_);
+        __result.__ctz_ = static_cast<unsigned>(((-__ddn & (__bits_per_word - 1)) + __result.__ctz_) % __bits_per_word);
+        __dn -= __ddn;
+      }
+      if (__dn > 0) {
+        // __result.__ctz_ == 0
+        --__result.__seg_;
+        __result.__ctz_ = static_cast<unsigned>(-__dn & (__bits_per_word - 1));
+        __m             = ~__storage_type(0) << __result.__ctz_;
+        *__result.__seg_ &= ~__m;
+        __last.__ctz_ -= __dn + __ddn;
+        *__result.__seg_ |= __b << (__result.__ctz_ - __last.__ctz_);
+      }
+      // __last.__ctz_ = 0
+    }
+    // __last.__ctz_ == 0 || __n == 0
+    // __result.__ctz_ != 0 || __n == 0
+    // do middle words
+    unsigned __clz_r   = __bits_per_word - __result.__ctz_;
+    __storage_type __m = ~__storage_type(0) >> __clz_r;
+    for (; __n >= __bits_per_word; __n -= __bits_per_word) {
+      __storage_type __b = *--__last.__seg_;
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b >> __clz_r;
+      *--__result.__seg_ &= __m;
+      *__result.__seg_ |= __b << __result.__ctz_;
+    }
+    // do last word
+    if (__n > 0) {
+      __m                 = ~__storage_type(0) << (__bits_per_word - __n);
+      __storage_type __b  = *--__last.__seg_ & __m;
+      __clz_r             = __bits_per_word - __result.__ctz_;
+      __storage_type __dn = std::min(__n, static_cast<difference_type>(__result.__ctz_));
+      __m                 = (~__storage_type(0) << (__result.__ctz_ - __dn)) & (~__storage_type(0) >> __clz_r);
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b >> (__bits_per_word - __result.__ctz_);
+      __result.__ctz_ = static_cast<unsigned>(((-__dn & (__bits_per_word - 1)) + __result.__ctz_) % __bits_per_word);
+      __n -= __dn;
+      if (__n > 0) {
+        // __result.__ctz_ == 0
+        --__result.__seg_;
+        __result.__ctz_ = static_cast<unsigned>(-__n & (__bits_per_word - 1));
+        __m             = ~__storage_type(0) << __result.__ctz_;
+        *__result.__seg_ &= ~__m;
+        *__result.__seg_ |= __b << (__result.__ctz_ - (__bits_per_word - __n - __dn));
+      }
+    }
+  }
+  return __result;
+}
+
+template <class _Cp, bool _IsConst>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, false> copy_backward(
+    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
+  if (__last.__ctz_ == __result.__ctz_)
+    return std::__copy_backward_aligned(__first, __last, __result);
+  return std::__copy_backward_unaligned(__first, __last, __result);
+}
+
+// move
+
+template <class _Cp, bool _IsConst>
+inline _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false>
+move(__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
+  return std::copy(__first, __last, __result);
+}
+
+// move_backward
+
+template <class _Cp, bool _IsConst>
+inline _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> move_backward(
+    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
+  return std::copy_backward(__first, __last, __result);
+}
+
+// swap_ranges
+
+template <class _Cl, class _Cr>
+_LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cr, false> __swap_ranges_aligned(
+    __bit_iterator<_Cl, false> __first, __bit_iterator<_Cl, false> __last, __bit_iterator<_Cr, false> __result) {
+  using _I1             = __bit_iterator<_Cl, false>;
+  using difference_type = typename _I1::difference_type;
+  using __storage_type  = typename _I1::__storage_type;
+
+  const int __bits_per_word = _I1::__bits_per_word;
+  difference_type __n       = __last - __first;
+  if (__n > 0) {
+    // do first word
+    if (__first.__ctz_ != 0) {
+      unsigned __clz       = __bits_per_word - __first.__ctz_;
+      difference_type __dn = std::min(static_cast<difference_type>(__clz), __n);
+      __n -= __dn;
+      __storage_type __m  = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz - __dn));
+      __storage_type __b1 = *__first.__seg_ & __m;
+      *__first.__seg_ &= ~__m;
+      __storage_type __b2 = *__result.__seg_ & __m;
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b1;
+      *__first.__seg_ |= __b2;
+      __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
+      __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
+      ++__first.__seg_;
+      // __first.__ctz_ = 0;
+    }
+    // __first.__ctz_ == 0;
+    // do middle words
+    for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first.__seg_, ++__result.__seg_)
+      swap(*__first.__seg_, *__result.__seg_);
+    // do last word
+    if (__n > 0) {
+      __storage_type __m  = ~__storage_type(0) >> (__bits_per_word - __n);
+      __storage_type __b1 = *__first.__seg_ & __m;
+      *__first.__seg_ &= ~__m;
+      __storage_type __b2 = *__result.__seg_ & __m;
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b1;
+      *__first.__seg_ |= __b2;
+      __result.__ctz_ = static_cast<unsigned>(__n);
+    }
+  }
+  return __result;
+}
+
+template <class _Cl, class _Cr>
+_LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cr, false> __swap_ranges_unaligned(
+    __bit_iterator<_Cl, false> __first, __bit_iterator<_Cl, false> __last, __bit_iterator<_Cr, false> __result) {
+  using _I1             = __bit_iterator<_Cl, false>;
+  using difference_type = typename _I1::difference_type;
+  using __storage_type  = typename _I1::__storage_type;
+
+  const int __bits_per_word = _I1::__bits_per_word;
+  difference_type __n       = __last - __first;
+  if (__n > 0) {
+    // do first word
+    if (__first.__ctz_ != 0) {
+      unsigned __clz_f     = __bits_per_word - __first.__ctz_;
+      difference_type __dn = std::min(static_cast<difference_type>(__clz_f), __n);
+      __n -= __dn;
+      __storage_type __m  = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn));
+      __storage_type __b1 = *__first.__seg_ & __m;
+      *__first.__seg_ &= ~__m;
+      unsigned __clz_r     = __bits_per_word - __result.__ctz_;
+      __storage_type __ddn = std::min<__storage_type>(__dn, __clz_r);
+      __m                  = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __ddn));
+      __storage_type __b2  = *__result.__seg_ & __m;
+      *__result.__seg_ &= ~__m;
+      if (__result.__ctz_ > __first.__ctz_) {
+        unsigned __s = __result.__ctz_ - __first.__ctz_;
+        *__result.__seg_ |= __b1 << __s;
+        *__first.__seg_ |= __b2 >> __s;
+      } else {
+        unsigned __s = __first.__ctz_ - __result.__ctz_;
+        *__result.__seg_ |= __b1 >> __s;
+        *__first.__seg_ |= __b2 << __s;
+      }
+      __result.__seg_ += (__ddn + __result.__ctz_) / __bits_per_word;
+      __result.__ctz_ = static_cast<unsigned>((__ddn + __result.__ctz_) % __bits_per_word);
+      __dn -= __ddn;
+      if (__dn > 0) {
+        __m  = ~__storage_type(0) >> (__bits_per_word - __dn);
+        __b2 = *__result.__seg_ & __m;
+        *__result.__seg_ &= ~__m;
+        unsigned __s = __first.__ctz_ + __ddn;
+        *__result.__seg_ |= __b1 >> __s;
+        *__first.__seg_ |= __b2 << __s;
+        __result.__ctz_ = static_cast<unsigned>(__dn);
+      }
+      ++__first.__seg_;
+      // __first.__ctz_ = 0;
+    }
+    // __first.__ctz_ == 0;
+    // do middle words
+    __storage_type __m = ~__storage_type(0) << __result.__ctz_;
+    unsigned __clz_r   = __bits_per_word - __result.__ctz_;
+    for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first.__seg_) {
+      __storage_type __b1 = *__first.__seg_;
+      __storage_type __b2 = *__result.__seg_ & __m;
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b1 << __result.__ctz_;
+      *__first.__seg_ = __b2 >> __result.__ctz_;
+      ++__result.__seg_;
+      __b2 = *__result.__seg_ & ~__m;
+      *__result.__seg_ &= __m;
+      *__result.__seg_ |= __b1 >> __clz_r;
+      *__first.__seg_ |= __b2 << __clz_r;
+    }
+    // do last word
+    if (__n > 0) {
+      __m                 = ~__storage_type(0) >> (__bits_per_word - __n);
+      __storage_type __b1 = *__first.__seg_ & __m;
+      *__first.__seg_ &= ~__m;
+      __storage_type __dn = std::min<__storage_type>(__n, __clz_r);
+      __m                 = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __dn));
+      __storage_type __b2 = *__result.__seg_ & __m;
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __b1 << __result.__ctz_;
+      *__first.__seg_ |= __b2 >> __result.__ctz_;
+      __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
+      __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
+      __n -= __dn;
+      if (__n > 0) {
+        __m  = ~__storage_type(0) >> (__bits_per_word - __n);
+        __b2 = *__result.__seg_ & __m;
+        *__result.__seg_ &= ~__m;
+        *__result.__seg_ |= __b1 >> __dn;
+        *__first.__seg_ |= __b2 << __dn;
+        __result.__ctz_ = static_cast<unsigned>(__n);
+      }
+    }
+  }
+  return __result;
+}
+
+template <class _Cl, class _Cr>
+inline _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cr, false> swap_ranges(
+    __bit_iterator<_Cl, false> __first1, __bit_iterator<_Cl, false> __last1, __bit_iterator<_Cr, false> __first2) {
+  if (__first1.__ctz_ == __first2.__ctz_)
+    return std::__swap_ranges_aligned(__first1, __last1, __first2);
+  return std::__swap_ranges_unaligned(__first1, __last1, __first2);
+}
+
+// rotate
+
+template <class _Cp>
+struct __bit_array {
+  using difference_type   = typename _Cp::difference_type;
+  using __storage_type    = typename _Cp::__storage_type;
+  using __storage_pointer = typename _Cp::__storage_pointer;
+  using iterator          = typename _Cp::iterator;
+
+  static const unsigned __bits_per_word = _Cp::__bits_per_word;
+  static const unsigned _Np             = 4;
+
+  difference_type __size_;
+  __storage_type __word_[_Np];
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static difference_type capacity() {
+    return static_cast<difference_type>(_Np * __bits_per_word);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit __bit_array(difference_type __s) : __size_(__s) {
+    if (__libcpp_is_constant_evaluated()) {
+      for (size_t __i = 0; __i != __bit_array<_Cp>::_Np; ++__i)
+        std::__construct_at(__word_ + __i, 0);
+    }
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator begin() {
+    return iterator(pointer_traits<__storage_pointer>::pointer_to(__word_[0]), 0);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator end() {
+    return iterator(pointer_traits<__storage_pointer>::pointer_to(__word_[0]) + __size_ / __bits_per_word,
+                    static_cast<unsigned>(__size_ % __bits_per_word));
+  }
+};
+
+template <class _Cp>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false>
+rotate(__bit_iterator<_Cp, false> __first, __bit_iterator<_Cp, false> __middle, __bit_iterator<_Cp, false> __last) {
+  using _I1             = __bit_iterator<_Cp, false>;
+  using difference_type = typename _I1::difference_type;
+
+  difference_type __d1 = __middle - __first;
+  difference_type __d2 = __last - __middle;
+  _I1 __r              = __first + __d2;
+  while (__d1 != 0 && __d2 != 0) {
+    if (__d1 <= __d2) {
+      if (__d1 <= __bit_array<_Cp>::capacity()) {
+        __bit_array<_Cp> __b(__d1);
+        std::copy(__first, __middle, __b.begin());
+        std::copy(__b.begin(), __b.end(), std::copy(__middle, __last, __first));
+        break;
+      } else {
+        __bit_iterator<_Cp, false> __mp = std::swap_ranges(__first, __middle, __middle);
+        __first                         = __middle;
+        __middle                        = __mp;
+        __d2 -= __d1;
+      }
+    } else {
+      if (__d2 <= __bit_array<_Cp>::capacity()) {
+        __bit_array<_Cp> __b(__d2);
+        std::copy(__middle, __last, __b.begin());
+        std::copy_backward(__b.begin(), __b.end(), std::copy_backward(__first, __middle, __last));
+        break;
+      } else {
+        __bit_iterator<_Cp, false> __mp = __first + __d2;
+        std::swap_ranges(__first, __mp, __middle);
+        __first = __mp;
+        __d1 -= __d2;
+      }
+    }
+  }
+  return __r;
+}
+
+// equal
+
+template <class _Cp, bool _IC1, bool _IC2>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __equal_unaligned(
+    __bit_iterator<_Cp, _IC1> __first1, __bit_iterator<_Cp, _IC1> __last1, __bit_iterator<_Cp, _IC2> __first2) {
+  using _It             = __bit_iterator<_Cp, _IC1>;
+  using difference_type = typename _It::difference_type;
+  using __storage_type  = typename _It::__storage_type;
+
+  const int __bits_per_word = _It::__bits_per_word;
+  difference_type __n       = __last1 - __first1;
+  if (__n > 0) {
+    // do first word
+    if (__first1.__ctz_ != 0) {
+      unsigned __clz_f     = __bits_per_word - __first1.__ctz_;
+      difference_type __dn = std::min(static_cast<difference_type>(__clz_f), __n);
+      __n -= __dn;
+      __storage_type __m   = (~__storage_type(0) << __first1.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn));
+      __storage_type __b   = *__first1.__seg_ & __m;
+      unsigned __clz_r     = __bits_per_word - __first2.__ctz_;
+      __storage_type __ddn = std::min<__storage_type>(__dn, __clz_r);
+      __m                  = (~__storage_type(0) << __first2.__ctz_) & (~__storage_type(0) >> (__clz_r - __ddn));
+      if (__first2.__ctz_ > __first1.__ctz_) {
+        if ((*__first2.__seg_ & __m) != (__b << (__first2.__ctz_ - __first1.__ctz_)))
+          return false;
+      } else {
+        if ((*__first2.__seg_ & __m) != (__b >> (__first1.__ctz_ - __first2.__ctz_)))
+          return false;
+      }
+      __first2.__seg_ += (__ddn + __first2.__ctz_) / __bits_per_word;
+      __first2.__ctz_ = static_cast<unsigned>((__ddn + __first2.__ctz_) % __bits_per_word);
+      __dn -= __ddn;
+      if (__dn > 0) {
+        __m = ~__storage_type(0) >> (__bits_per_word - __dn);
+        if ((*__first2.__seg_ & __m) != (__b >> (__first1.__ctz_ + __ddn)))
+          return false;
+        __first2.__ctz_ = static_cast<unsigned>(__dn);
+      }
+      ++__first1.__seg_;
+      // __first1.__ctz_ = 0;
+    }
+    // __first1.__ctz_ == 0;
+    // do middle words
+    unsigned __clz_r   = __bits_per_word - __first2.__ctz_;
+    __storage_type __m = ~__storage_type(0) << __first2.__ctz_;
+    for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first1.__seg_) {
+      __storage_type __b = *__first1.__seg_;
+      if ((*__first2.__seg_ & __m) != (__b << __first2.__ctz_))
+        return false;
+      ++__first2.__seg_;
+      if ((*__first2.__seg_ & ~__m) != (__b >> __clz_r))
+        return false;
+    }
+    // do last word
+    if (__n > 0) {
+      __m                 = ~__storage_type(0) >> (__bits_per_word - __n);
+      __storage_type __b  = *__first1.__seg_ & __m;
+      __storage_type __dn = std::min(__n, static_cast<difference_type>(__clz_r));
+      __m                 = (~__storage_type(0) << __first2.__ctz_) & (~__storage_type(0) >> (__clz_r - __dn));
+      if ((*__first2.__seg_ & __m) != (__b << __first2.__ctz_))
+        return false;
+      __first2.__seg_ += (__dn + __first2.__ctz_) / __bits_per_word;
+      __first2.__ctz_ = static_cast<unsigned>((__dn + __first2.__ctz_) % __bits_per_word);
+      __n -= __dn;
+      if (__n > 0) {
+        __m = ~__storage_type(0) >> (__bits_per_word - __n);
+        if ((*__first2.__seg_ & __m) != (__b >> __dn))
+          return false;
+      }
+    }
+  }
+  return true;
+}
+
+template <class _Cp, bool _IC1, bool _IC2>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __equal_aligned(
+    __bit_iterator<_Cp, _IC1> __first1, __bit_iterator<_Cp, _IC1> __last1, __bit_iterator<_Cp, _IC2> __first2) {
+  using _It             = __bit_iterator<_Cp, _IC1>;
+  using difference_type = typename _It::difference_type;
+  using __storage_type  = typename _It::__storage_type;
+
+  const int __bits_per_word = _It::__bits_per_word;
+  difference_type __n       = __last1 - __first1;
+  if (__n > 0) {
+    // do first word
+    if (__first1.__ctz_ != 0) {
+      unsigned __clz       = __bits_per_word - __first1.__ctz_;
+      difference_type __dn = std::min(static_cast<difference_type>(__clz), __n);
+      __n -= __dn;
+      __storage_type __m = (~__storage_type(0) << __first1.__ctz_) & (~__storage_type(0) >> (__clz - __dn));
+      if ((*__first2.__seg_ & __m) != (*__first1.__seg_ & __m))
+        return false;
+      ++__first2.__seg_;
+      ++__first1.__seg_;
+      // __first1.__ctz_ = 0;
+      // __first2.__ctz_ = 0;
+    }
+    // __first1.__ctz_ == 0;
+    // __first2.__ctz_ == 0;
+    // do middle words
+    for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first1.__seg_, ++__first2.__seg_)
+      if (*__first2.__seg_ != *__first1.__seg_)
+        return false;
+    // do last word
+    if (__n > 0) {
+      __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+      if ((*__first2.__seg_ & __m) != (*__first1.__seg_ & __m))
+        return false;
+    }
+  }
+  return true;
+}
+
+template <class _Cp, bool _IC1, bool _IC2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+equal(__bit_iterator<_Cp, _IC1> __first1, __bit_iterator<_Cp, _IC1> __last1, __bit_iterator<_Cp, _IC2> __first2) {
+  if (__first1.__ctz_ == __first2.__ctz_)
+    return std::__equal_aligned(__first1, __last1, __first2);
+  return std::__equal_unaligned(__first1, __last1, __first2);
+}
+
+template <class _Cp, bool _IsConst, typename _Cp::__storage_type>
+class __bit_iterator {
+public:
+  using difference_type = typename _Cp::difference_type;
+  using value_type      = bool;
+  using pointer         = __bit_iterator;
+#ifndef _LIBCPP_ABI_BITSET_VECTOR_BOOL_CONST_SUBSCRIPT_RETURN_BOOL
+  using reference = __conditional_t<_IsConst, __bit_const_reference<_Cp>, __bit_reference<_Cp> >;
+#else
+  using reference = __conditional_t<_IsConst, bool, __bit_reference<_Cp> >;
+#endif
+  using iterator_category = random_access_iterator_tag;
+
+private:
+  using __storage_type = typename _Cp::__storage_type;
+  using __storage_pointer =
+      __conditional_t<_IsConst, typename _Cp::__const_storage_pointer, typename _Cp::__storage_pointer>;
+
+  static const unsigned __bits_per_word = _Cp::__bits_per_word;
+
+  __storage_pointer __seg_;
+  unsigned __ctz_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator() _NOEXCEPT
+#if _LIBCPP_STD_VER >= 14
+      : __seg_(nullptr),
+        __ctz_(0)
+#endif
+  {
+  }
+
+  // When _IsConst=false, this is the copy constructor.
+  // It is non-trivial. Making it trivial would break ABI.
+  // When _IsConst=true, this is a converting constructor;
+  // the copy and move constructors are implicitly generated
+  // and trivial.
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator(const __bit_iterator<_Cp, false>& __it) _NOEXCEPT
+      : __seg_(__it.__seg_),
+        __ctz_(__it.__ctz_) {}
+
+  // When _IsConst=false, we have a user-provided copy constructor,
+  // so we must also provide a copy assignment operator because
+  // the implicit generation of a defaulted one is deprecated.
+  // When _IsConst=true, the assignment operators are
+  // implicitly generated and trivial.
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator&
+  operator=(const _If<_IsConst, struct __private_nat, __bit_iterator>& __it) {
+    __seg_ = __it.__seg_;
+    __ctz_ = __it.__ctz_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference operator*() const _NOEXCEPT {
+    return __conditional_t<_IsConst, __bit_const_reference<_Cp>, __bit_reference<_Cp> >(
+        __seg_, __storage_type(1) << __ctz_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator& operator++() {
+    if (__ctz_ != __bits_per_word - 1)
+      ++__ctz_;
+    else {
+      __ctz_ = 0;
+      ++__seg_;
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator operator++(int) {
+    __bit_iterator __tmp = *this;
+    ++(*this);
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator& operator--() {
+    if (__ctz_ != 0)
+      --__ctz_;
+    else {
+      __ctz_ = __bits_per_word - 1;
+      --__seg_;
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator operator--(int) {
+    __bit_iterator __tmp = *this;
+    --(*this);
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator& operator+=(difference_type __n) {
+    if (__n >= 0)
+      __seg_ += (__n + __ctz_) / __bits_per_word;
+    else
+      __seg_ += static_cast<difference_type>(__n - __bits_per_word + __ctz_ + 1) /
+                static_cast<difference_type>(__bits_per_word);
+    __n &= (__bits_per_word - 1);
+    __ctz_ = static_cast<unsigned>((__n + __ctz_) % __bits_per_word);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator& operator-=(difference_type __n) {
+    return *this += -__n;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator operator+(difference_type __n) const {
+    __bit_iterator __t(*this);
+    __t += __n;
+    return __t;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator operator-(difference_type __n) const {
+    __bit_iterator __t(*this);
+    __t -= __n;
+    return __t;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator
+  operator+(difference_type __n, const __bit_iterator& __it) {
+    return __it + __n;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 friend difference_type
+  operator-(const __bit_iterator& __x, const __bit_iterator& __y) {
+    return (__x.__seg_ - __y.__seg_) * __bits_per_word + __x.__ctz_ - __y.__ctz_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference operator[](difference_type __n) const {
+    return *(*this + __n);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 friend bool
+  operator==(const __bit_iterator& __x, const __bit_iterator& __y) {
+    return __x.__seg_ == __y.__seg_ && __x.__ctz_ == __y.__ctz_;
+  }
+
+#if _LIBCPP_STD_VER <= 17
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 friend bool
+  operator!=(const __bit_iterator& __x, const __bit_iterator& __y) {
+    return !(__x == __y);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 friend bool
+  operator<(const __bit_iterator& __x, const __bit_iterator& __y) {
+    return __x.__seg_ < __y.__seg_ || (__x.__seg_ == __y.__seg_ && __x.__ctz_ < __y.__ctz_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 friend bool
+  operator>(const __bit_iterator& __x, const __bit_iterator& __y) {
+    return __y < __x;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 friend bool
+  operator<=(const __bit_iterator& __x, const __bit_iterator& __y) {
+    return !(__y < __x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 friend bool
+  operator>=(const __bit_iterator& __x, const __bit_iterator& __y) {
+    return !(__x < __y);
+  }
+#else  // _LIBCPP_STD_VER <= 17
+  _LIBCPP_HIDE_FROM_ABI constexpr friend strong_ordering
+  operator<=>(const __bit_iterator& __x, const __bit_iterator& __y) {
+    if (__x.__seg_ < __y.__seg_)
+      return strong_ordering::less;
+
+    if (__x.__seg_ == __y.__seg_)
+      return __x.__ctz_ <=> __y.__ctz_;
+
+    return strong_ordering::greater;
+  }
+#endif // _LIBCPP_STD_VER <= 17
+
+private:
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit __bit_iterator(__storage_pointer __s, unsigned __ctz) _NOEXCEPT
+      : __seg_(__s),
+        __ctz_(__ctz) {}
+
+  friend typename _Cp::__self;
+
+  friend class __bit_reference<_Cp>;
+  friend class __bit_const_reference<_Cp>;
+  friend class __bit_iterator<_Cp, true>;
+  template <class _Dp>
+  friend struct __bit_array;
+
+  template <bool _FillVal, class _Dp>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend void
+  __fill_n_bool(__bit_iterator<_Dp, false> __first, typename _Dp::size_type __n);
+
+  template <class _Dp, bool _IC>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_aligned(
+      __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
+  template <class _Dp, bool _IC>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_unaligned(
+      __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
+  template <class _Dp, bool _IC>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false>
+  copy(__bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
+  template <class _Dp, bool _IC>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_backward_aligned(
+      __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
+  template <class _Dp, bool _IC>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_backward_unaligned(
+      __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
+  template <class _Dp, bool _IC>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false>
+  copy_backward(__bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
+  template <class _Cl, class _Cr>
+  friend __bit_iterator<_Cr, false>
+      __swap_ranges_aligned(__bit_iterator<_Cl, false>, __bit_iterator<_Cl, false>, __bit_iterator<_Cr, false>);
+  template <class _Cl, class _Cr>
+  friend __bit_iterator<_Cr, false>
+      __swap_ranges_unaligned(__bit_iterator<_Cl, false>, __bit_iterator<_Cl, false>, __bit_iterator<_Cr, false>);
+  template <class _Cl, class _Cr>
+  friend __bit_iterator<_Cr, false>
+      swap_ranges(__bit_iterator<_Cl, false>, __bit_iterator<_Cl, false>, __bit_iterator<_Cr, false>);
+  template <class _Dp>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false>
+      rotate(__bit_iterator<_Dp, false>, __bit_iterator<_Dp, false>, __bit_iterator<_Dp, false>);
+  template <class _Dp, bool _IC1, bool _IC2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend bool
+      __equal_aligned(__bit_iterator<_Dp, _IC1>, __bit_iterator<_Dp, _IC1>, __bit_iterator<_Dp, _IC2>);
+  template <class _Dp, bool _IC1, bool _IC2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend bool
+      __equal_unaligned(__bit_iterator<_Dp, _IC1>, __bit_iterator<_Dp, _IC1>, __bit_iterator<_Dp, _IC2>);
+  template <class _Dp, bool _IC1, bool _IC2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend bool
+      equal(__bit_iterator<_Dp, _IC1>, __bit_iterator<_Dp, _IC1>, __bit_iterator<_Dp, _IC2>);
+  template <bool _ToFind, class _Dp, bool _IC>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, _IC>
+      __find_bool(__bit_iterator<_Dp, _IC>, typename _Dp::size_type);
+  template <bool _ToCount, class _Dp, bool _IC>
+  friend typename __bit_iterator<_Dp, _IC>::difference_type _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 __count_bool(__bit_iterator<_Dp, _IC>, typename _Dp::size_type);
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___BIT_REFERENCE
diff --git a/libcxx/include/__cxx03/__charconv/chars_format.h b/libcxx/include/__cxx03/__charconv/chars_format.h
new file mode 100644
index 000000000000..c76cebd5d184
--- /dev/null
+++ b/libcxx/include/__cxx03/__charconv/chars_format.h
@@ -0,0 +1,61 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHARCONV_CHARS_FORMAT_H
+#define _LIBCPP___CHARCONV_CHARS_FORMAT_H
+
+#include <__config>
+#include <__utility/to_underlying.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+enum class chars_format { scientific = 0x1, fixed = 0x2, hex = 0x4, general = fixed | scientific };
+
+inline _LIBCPP_HIDE_FROM_ABI constexpr chars_format operator~(chars_format __x) {
+  return chars_format(~std::__to_underlying(__x));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI constexpr chars_format operator&(chars_format __x, chars_format __y) {
+  return chars_format(std::__to_underlying(__x) & std::__to_underlying(__y));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI constexpr chars_format operator|(chars_format __x, chars_format __y) {
+  return chars_format(std::__to_underlying(__x) | std::__to_underlying(__y));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI constexpr chars_format operator^(chars_format __x, chars_format __y) {
+  return chars_format(std::__to_underlying(__x) ^ std::__to_underlying(__y));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI constexpr chars_format& operator&=(chars_format& __x, chars_format __y) {
+  __x = __x & __y;
+  return __x;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI constexpr chars_format& operator|=(chars_format& __x, chars_format __y) {
+  __x = __x | __y;
+  return __x;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI constexpr chars_format& operator^=(chars_format& __x, chars_format __y) {
+  __x = __x ^ __y;
+  return __x;
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CHARCONV_CHARS_FORMAT_H
diff --git a/libcxx/include/__cxx03/__charconv/from_chars_integral.h b/libcxx/include/__cxx03/__charconv/from_chars_integral.h
new file mode 100644
index 000000000000..c1f033b37b91
--- /dev/null
+++ b/libcxx/include/__cxx03/__charconv/from_chars_integral.h
@@ -0,0 +1,240 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHARCONV_FROM_CHARS_INTEGRAL_H
+#define _LIBCPP___CHARCONV_FROM_CHARS_INTEGRAL_H
+
+#include <__algorithm/copy_n.h>
+#include <__assert>
+#include <__charconv/from_chars_result.h>
+#include <__charconv/traits.h>
+#include <__config>
+#include <__memory/addressof.h>
+#include <__system_error/errc.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_unsigned.h>
+#include <__type_traits/make_unsigned.h>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+from_chars_result from_chars(const char*, const char*, bool, int = 10) = delete;
+
+template <typename _It, typename _Tp, typename _Fn, typename... _Ts>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI from_chars_result
+__sign_combinator(_It __first, _It __last, _Tp& __value, _Fn __f, _Ts... __args) {
+  using __tl = numeric_limits<_Tp>;
+  decltype(std::__to_unsigned_like(__value)) __x;
+
+  bool __neg = (__first != __last && *__first == '-');
+  auto __r   = __f(__neg ? __first + 1 : __first, __last, __x, __args...);
+  switch (__r.ec) {
+  case errc::invalid_argument:
+    return {__first, __r.ec};
+  case errc::result_out_of_range:
+    return __r;
+  default:
+    break;
+  }
+
+  if (__neg) {
+    if (__x <= std::__complement(std::__to_unsigned_like(__tl::min()))) {
+      __x = std::__complement(__x);
+      std::copy_n(std::addressof(__x), 1, std::addressof(__value));
+      return __r;
+    }
+  } else {
+    if (__x <= std::__to_unsigned_like(__tl::max())) {
+      __value = __x;
+      return __r;
+    }
+  }
+
+  return {__r.ptr, errc::result_out_of_range};
+}
+
+template <typename _Tp>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool __in_pattern(_Tp __c) {
+  return '0' <= __c && __c <= '9';
+}
+
+struct _LIBCPP_HIDDEN __in_pattern_result {
+  bool __ok;
+  int __val;
+
+  explicit _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI operator bool() const { return __ok; }
+};
+
+template <typename _Tp>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI __in_pattern_result __in_pattern(_Tp __c, int __base) {
+  if (__base <= 10)
+    return {'0' <= __c && __c < '0' + __base, __c - '0'};
+  else if (std::__in_pattern(__c))
+    return {true, __c - '0'};
+  else if ('a' <= __c && __c < 'a' + __base - 10)
+    return {true, __c - 'a' + 10};
+  else
+    return {'A' <= __c && __c < 'A' + __base - 10, __c - 'A' + 10};
+}
+
+template <typename _It, typename _Tp, typename _Fn, typename... _Ts>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI from_chars_result
+__subject_seq_combinator(_It __first, _It __last, _Tp& __value, _Fn __f, _Ts... __args) {
+  auto __find_non_zero = [](_It __firstit, _It __lastit) {
+    for (; __firstit != __lastit; ++__firstit)
+      if (*__firstit != '0')
+        break;
+    return __firstit;
+  };
+
+  auto __p = __find_non_zero(__first, __last);
+  if (__p == __last || !std::__in_pattern(*__p, __args...)) {
+    if (__p == __first)
+      return {__first, errc::invalid_argument};
+    else {
+      __value = 0;
+      return {__p, {}};
+    }
+  }
+
+  auto __r = __f(__p, __last, __value, __args...);
+  if (__r.ec == errc::result_out_of_range) {
+    for (; __r.ptr != __last; ++__r.ptr) {
+      if (!std::__in_pattern(*__r.ptr, __args...))
+        break;
+    }
+  }
+
+  return __r;
+}
+
+template <typename _Tp, __enable_if_t<is_unsigned<_Tp>::value, int> = 0>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI from_chars_result
+__from_chars_atoi(const char* __first, const char* __last, _Tp& __value) {
+  using __tx          = __itoa::__traits<_Tp>;
+  using __output_type = typename __tx::type;
+
+  return std::__subject_seq_combinator(
+      __first, __last, __value, [](const char* __f, const char* __l, _Tp& __val) -> from_chars_result {
+        __output_type __a, __b;
+        auto __p = __tx::__read(__f, __l, __a, __b);
+        if (__p == __l || !std::__in_pattern(*__p)) {
+          __output_type __m = numeric_limits<_Tp>::max();
+          if (__m >= __a && __m - __a >= __b) {
+            __val = __a + __b;
+            return {__p, {}};
+          }
+        }
+        return {__p, errc::result_out_of_range};
+      });
+}
+
+template <typename _Tp, __enable_if_t<is_signed<_Tp>::value, int> = 0>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI from_chars_result
+__from_chars_atoi(const char* __first, const char* __last, _Tp& __value) {
+  using __t = decltype(std::__to_unsigned_like(__value));
+  return std::__sign_combinator(__first, __last, __value, __from_chars_atoi<__t>);
+}
+
+/*
+// Code used to generate __from_chars_log2f_lut.
+#include <cmath>
+#include <format>
+#include <iostream>
+
+int main() {
+  for (int i = 2; i <= 36; ++i)
+    std::cout << std::format("{},\n", log2f(i));
+}
+*/
+/// log2f table for bases [2, 36].
+inline constexpr float __from_chars_log2f_lut[35] = {
+    1,         1.5849625, 2,         2.321928, 2.5849626, 2.807355, 3,        3.169925,  3.321928,
+    3.4594316, 3.5849626, 3.7004397, 3.807355, 3.9068906, 4,        4.087463, 4.169925,  4.2479277,
+    4.321928,  4.3923173, 4.4594316, 4.523562, 4.5849624, 4.643856, 4.70044,  4.7548876, 4.807355,
+    4.857981,  4.9068904, 4.9541965, 5,        5.044394,  5.087463, 5.129283, 5.169925};
+
+template <typename _Tp, __enable_if_t<is_unsigned<_Tp>::value, int> = 0>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI from_chars_result
+__from_chars_integral(const char* __first, const char* __last, _Tp& __value, int __base) {
+  if (__base == 10)
+    return std::__from_chars_atoi(__first, __last, __value);
+
+  return std::__subject_seq_combinator(
+      __first,
+      __last,
+      __value,
+      [](const char* __p, const char* __lastp, _Tp& __val, int __b) -> from_chars_result {
+        using __tl = numeric_limits<_Tp>;
+        // __base is always between 2 and 36 inclusive.
+        auto __digits = __tl::digits / __from_chars_log2f_lut[__b - 2];
+        _Tp __x = __in_pattern(*__p++, __b).__val, __y = 0;
+
+        for (int __i = 1; __p != __lastp; ++__i, ++__p) {
+          if (auto __c = __in_pattern(*__p, __b)) {
+            if (__i < __digits - 1)
+              __x = __x * __b + __c.__val;
+            else {
+              if (!__itoa::__mul_overflowed(__x, __b, __x))
+                ++__p;
+              __y = __c.__val;
+              break;
+            }
+          } else
+            break;
+        }
+
+        if (__p == __lastp || !__in_pattern(*__p, __b)) {
+          if (__tl::max() - __x >= __y) {
+            __val = __x + __y;
+            return {__p, {}};
+          }
+        }
+        return {__p, errc::result_out_of_range};
+      },
+      __base);
+}
+
+template <typename _Tp, __enable_if_t<is_signed<_Tp>::value, int> = 0>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI from_chars_result
+__from_chars_integral(const char* __first, const char* __last, _Tp& __value, int __base) {
+  using __t = decltype(std::__to_unsigned_like(__value));
+  return std::__sign_combinator(__first, __last, __value, __from_chars_integral<__t>, __base);
+}
+
+template <typename _Tp, __enable_if_t<is_integral<_Tp>::value, int> = 0>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI from_chars_result
+from_chars(const char* __first, const char* __last, _Tp& __value) {
+  return std::__from_chars_atoi(__first, __last, __value);
+}
+
+template <typename _Tp, __enable_if_t<is_integral<_Tp>::value, int> = 0>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI from_chars_result
+from_chars(const char* __first, const char* __last, _Tp& __value, int __base) {
+  _LIBCPP_ASSERT_UNCATEGORIZED(2 <= __base && __base <= 36, "base not in [2, 36]");
+  return std::__from_chars_integral(__first, __last, __value, __base);
+}
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___CHARCONV_FROM_CHARS_INTEGRAL_H
diff --git a/libcxx/include/__cxx03/__charconv/from_chars_result.h b/libcxx/include/__cxx03/__charconv/from_chars_result.h
new file mode 100644
index 000000000000..a7bfd6530a8a
--- /dev/null
+++ b/libcxx/include/__cxx03/__charconv/from_chars_result.h
@@ -0,0 +1,39 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHARCONV_FROM_CHARS_RESULT_H
+#define _LIBCPP___CHARCONV_FROM_CHARS_RESULT_H
+
+#include <__config>
+#include <__system_error/errc.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+struct _LIBCPP_EXPORTED_FROM_ABI from_chars_result {
+  const char* ptr;
+  errc ec;
+#  if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI friend bool operator==(const from_chars_result&, const from_chars_result&) = default;
+#  endif
+#  if _LIBCPP_STD_VER >= 26
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return ec == errc{}; }
+#  endif
+};
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CHARCONV_FROM_CHARS_RESULT_H
diff --git a/libcxx/include/__cxx03/__charconv/tables.h b/libcxx/include/__cxx03/__charconv/tables.h
new file mode 100644
index 000000000000..6b93536b8c1b
--- /dev/null
+++ b/libcxx/include/__cxx03/__charconv/tables.h
@@ -0,0 +1,163 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHARCONV_TABLES
+#define _LIBCPP___CHARCONV_TABLES
+
+#include <__config>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+namespace __itoa {
+
+inline constexpr char __base_2_lut[64] = {
+    '0', '0', '0', '0', '0', '0', '0', '1', '0', '0', '1', '0', '0', '0', '1', '1', '0', '1', '0', '0', '0', '1',
+    '0', '1', '0', '1', '1', '0', '0', '1', '1', '1', '1', '0', '0', '0', '1', '0', '0', '1', '1', '0', '1', '0',
+    '1', '0', '1', '1', '1', '1', '0', '0', '1', '1', '0', '1', '1', '1', '1', '0', '1', '1', '1', '1'};
+
+inline constexpr char __base_8_lut[128] = {
+    '0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', '0', '6', '0', '7', '1', '0', '1', '1', '1', '2',
+    '1', '3', '1', '4', '1', '5', '1', '6', '1', '7', '2', '0', '2', '1', '2', '2', '2', '3', '2', '4', '2', '5',
+    '2', '6', '2', '7', '3', '0', '3', '1', '3', '2', '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '4', '0',
+    '4', '1', '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '5', '0', '5', '1', '5', '2', '5', '3',
+    '5', '4', '5', '5', '5', '6', '5', '7', '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6',
+    '6', '7', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7'};
+
+inline constexpr char __base_16_lut[512] = {
+    '0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', '0', '6', '0', '7', '0', '8', '0', '9', '0', 'a', '0',
+    'b', '0', 'c', '0', 'd', '0', 'e', '0', 'f', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', '1', '5', '1', '6',
+    '1', '7', '1', '8', '1', '9', '1', 'a', '1', 'b', '1', 'c', '1', 'd', '1', 'e', '1', 'f', '2', '0', '2', '1', '2',
+    '2', '2', '3', '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', '2', 'a', '2', 'b', '2', 'c', '2', 'd',
+    '2', 'e', '2', 'f', '3', '0', '3', '1', '3', '2', '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3',
+    '9', '3', 'a', '3', 'b', '3', 'c', '3', 'd', '3', 'e', '3', 'f', '4', '0', '4', '1', '4', '2', '4', '3', '4', '4',
+    '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '4', 'a', '4', 'b', '4', 'c', '4', 'd', '4', 'e', '4', 'f', '5',
+    '0', '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9', '5', 'a', '5', 'b',
+    '5', 'c', '5', 'd', '5', 'e', '5', 'f', '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6',
+    '7', '6', '8', '6', '9', '6', 'a', '6', 'b', '6', 'c', '6', 'd', '6', 'e', '6', 'f', '7', '0', '7', '1', '7', '2',
+    '7', '3', '7', '4', '7', '5', '7', '6', '7', '7', '7', '8', '7', '9', '7', 'a', '7', 'b', '7', 'c', '7', 'd', '7',
+    'e', '7', 'f', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6', '8', '7', '8', '8', '8', '9',
+    '8', 'a', '8', 'b', '8', 'c', '8', 'd', '8', 'e', '8', 'f', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9',
+    '5', '9', '6', '9', '7', '9', '8', '9', '9', '9', 'a', '9', 'b', '9', 'c', '9', 'd', '9', 'e', '9', 'f', 'a', '0',
+    'a', '1', 'a', '2', 'a', '3', 'a', '4', 'a', '5', 'a', '6', 'a', '7', 'a', '8', 'a', '9', 'a', 'a', 'a', 'b', 'a',
+    'c', 'a', 'd', 'a', 'e', 'a', 'f', 'b', '0', 'b', '1', 'b', '2', 'b', '3', 'b', '4', 'b', '5', 'b', '6', 'b', '7',
+    'b', '8', 'b', '9', 'b', 'a', 'b', 'b', 'b', 'c', 'b', 'd', 'b', 'e', 'b', 'f', 'c', '0', 'c', '1', 'c', '2', 'c',
+    '3', 'c', '4', 'c', '5', 'c', '6', 'c', '7', 'c', '8', 'c', '9', 'c', 'a', 'c', 'b', 'c', 'c', 'c', 'd', 'c', 'e',
+    'c', 'f', 'd', '0', 'd', '1', 'd', '2', 'd', '3', 'd', '4', 'd', '5', 'd', '6', 'd', '7', 'd', '8', 'd', '9', 'd',
+    'a', 'd', 'b', 'd', 'c', 'd', 'd', 'd', 'e', 'd', 'f', 'e', '0', 'e', '1', 'e', '2', 'e', '3', 'e', '4', 'e', '5',
+    'e', '6', 'e', '7', 'e', '8', 'e', '9', 'e', 'a', 'e', 'b', 'e', 'c', 'e', 'd', 'e', 'e', 'e', 'f', 'f', '0', 'f',
+    '1', 'f', '2', 'f', '3', 'f', '4', 'f', '5', 'f', '6', 'f', '7', 'f', '8', 'f', '9', 'f', 'a', 'f', 'b', 'f', 'c',
+    'f', 'd', 'f', 'e', 'f', 'f'};
+
+inline constexpr uint32_t __pow10_32[10] = {
+    UINT32_C(0),
+    UINT32_C(10),
+    UINT32_C(100),
+    UINT32_C(1000),
+    UINT32_C(10000),
+    UINT32_C(100000),
+    UINT32_C(1000000),
+    UINT32_C(10000000),
+    UINT32_C(100000000),
+    UINT32_C(1000000000)};
+
+inline constexpr uint64_t __pow10_64[20] = {
+    UINT64_C(0),
+    UINT64_C(10),
+    UINT64_C(100),
+    UINT64_C(1000),
+    UINT64_C(10000),
+    UINT64_C(100000),
+    UINT64_C(1000000),
+    UINT64_C(10000000),
+    UINT64_C(100000000),
+    UINT64_C(1000000000),
+    UINT64_C(10000000000),
+    UINT64_C(100000000000),
+    UINT64_C(1000000000000),
+    UINT64_C(10000000000000),
+    UINT64_C(100000000000000),
+    UINT64_C(1000000000000000),
+    UINT64_C(10000000000000000),
+    UINT64_C(100000000000000000),
+    UINT64_C(1000000000000000000),
+    UINT64_C(10000000000000000000)};
+
+#  ifndef _LIBCPP_HAS_NO_INT128
+inline constexpr int __pow10_128_offset      = 0;
+inline constexpr __uint128_t __pow10_128[40] = {
+    UINT64_C(0),
+    UINT64_C(10),
+    UINT64_C(100),
+    UINT64_C(1000),
+    UINT64_C(10000),
+    UINT64_C(100000),
+    UINT64_C(1000000),
+    UINT64_C(10000000),
+    UINT64_C(100000000),
+    UINT64_C(1000000000),
+    UINT64_C(10000000000),
+    UINT64_C(100000000000),
+    UINT64_C(1000000000000),
+    UINT64_C(10000000000000),
+    UINT64_C(100000000000000),
+    UINT64_C(1000000000000000),
+    UINT64_C(10000000000000000),
+    UINT64_C(100000000000000000),
+    UINT64_C(1000000000000000000),
+    UINT64_C(10000000000000000000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(10),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(100),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(1000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(10000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(100000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(1000000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(10000000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(100000000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(1000000000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(10000000000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(100000000000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(1000000000000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(10000000000000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(100000000000000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(1000000000000000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(10000000000000000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(100000000000000000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(1000000000000000000),
+    __uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(10000000000000000000),
+    (__uint128_t(UINT64_C(10000000000000000000)) * UINT64_C(10000000000000000000)) * 10};
+#  endif
+
+inline constexpr char __digits_base_10[200] = {
+    // clang-format off
+    '0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', '0', '6', '0', '7', '0', '8', '0', '9',
+    '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', '1', '5', '1', '6', '1', '7', '1', '8', '1', '9',
+    '2', '0', '2', '1', '2', '2', '2', '3', '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9',
+    '3', '0', '3', '1', '3', '2', '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9',
+    '4', '0', '4', '1', '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9',
+    '5', '0', '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9',
+    '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', '6', '9',
+    '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7', '7', '8', '7', '9',
+    '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6', '8', '7', '8', '8', '8', '9',
+    '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', '9', '6', '9', '7', '9', '8', '9', '9'};
+// clang-format on
+
+} // namespace __itoa
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CHARCONV_TABLES
diff --git a/libcxx/include/__cxx03/__charconv/to_chars.h b/libcxx/include/__cxx03/__charconv/to_chars.h
new file mode 100644
index 000000000000..8ef09af73755
--- /dev/null
+++ b/libcxx/include/__cxx03/__charconv/to_chars.h
@@ -0,0 +1,25 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHARCONV_TO_CHARS
+#define _LIBCPP___CHARCONV_TO_CHARS
+
+#include <__charconv/to_chars_floating_point.h>
+#include <__charconv/to_chars_integral.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CHARCONV_TO_CHARS
diff --git a/libcxx/include/__cxx03/__charconv/to_chars_base_10.h b/libcxx/include/__cxx03/__charconv/to_chars_base_10.h
new file mode 100644
index 000000000000..c49f4f6797aa
--- /dev/null
+++ b/libcxx/include/__cxx03/__charconv/to_chars_base_10.h
@@ -0,0 +1,188 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHARCONV_TO_CHARS_BASE_10_H
+#define _LIBCPP___CHARCONV_TO_CHARS_BASE_10_H
+
+#include <__algorithm/copy_n.h>
+#include <__assert>
+#include <__charconv/tables.h>
+#include <__config>
+#include <cstdint>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+namespace __itoa {
+
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI inline char* __append1(char* __first, uint32_t __value) noexcept {
+  *__first = '0' + static_cast<char>(__value);
+  return __first + 1;
+}
+
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI inline char* __append2(char* __first, uint32_t __value) noexcept {
+  return std::copy_n(&__digits_base_10[__value * 2], 2, __first);
+}
+
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI inline char* __append3(char* __first, uint32_t __value) noexcept {
+  return __itoa::__append2(__itoa::__append1(__first, __value / 100), __value % 100);
+}
+
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI inline char* __append4(char* __first, uint32_t __value) noexcept {
+  return __itoa::__append2(__itoa::__append2(__first, __value / 100), __value % 100);
+}
+
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI inline char* __append5(char* __first, uint32_t __value) noexcept {
+  return __itoa::__append4(__itoa::__append1(__first, __value / 10000), __value % 10000);
+}
+
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI inline char* __append6(char* __first, uint32_t __value) noexcept {
+  return __itoa::__append4(__itoa::__append2(__first, __value / 10000), __value % 10000);
+}
+
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI inline char* __append7(char* __first, uint32_t __value) noexcept {
+  return __itoa::__append6(__itoa::__append1(__first, __value / 1000000), __value % 1000000);
+}
+
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI inline char* __append8(char* __first, uint32_t __value) noexcept {
+  return __itoa::__append6(__itoa::__append2(__first, __value / 1000000), __value % 1000000);
+}
+
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI inline char* __append9(char* __first, uint32_t __value) noexcept {
+  return __itoa::__append8(__itoa::__append1(__first, __value / 100000000), __value % 100000000);
+}
+
+template <class _Tp>
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI char* __append10(char* __first, _Tp __value) noexcept {
+  return __itoa::__append8(__itoa::__append2(__first, static_cast<uint32_t>(__value / 100000000)),
+                           static_cast<uint32_t>(__value % 100000000));
+}
+
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI inline char*
+__base_10_u32(char* __first, uint32_t __value) noexcept {
+  if (__value < 1000000) {
+    if (__value < 10000) {
+      if (__value < 100) {
+        // 0 <= __value < 100
+        if (__value < 10)
+          return __itoa::__append1(__first, __value);
+        return __itoa::__append2(__first, __value);
+      }
+      // 100 <= __value < 10'000
+      if (__value < 1000)
+        return __itoa::__append3(__first, __value);
+      return __itoa::__append4(__first, __value);
+    }
+
+    // 10'000 <= __value < 1'000'000
+    if (__value < 100000)
+      return __itoa::__append5(__first, __value);
+    return __itoa::__append6(__first, __value);
+  }
+
+  // __value => 1'000'000
+  if (__value < 100000000) {
+    // 1'000'000 <= __value < 100'000'000
+    if (__value < 10000000)
+      return __itoa::__append7(__first, __value);
+    return __itoa::__append8(__first, __value);
+  }
+
+  // 100'000'000 <= __value < max
+  if (__value < 1000000000)
+    return __itoa::__append9(__first, __value);
+  return __itoa::__append10(__first, __value);
+}
+
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI inline char*
+__base_10_u64(char* __buffer, uint64_t __value) noexcept {
+  if (__value <= UINT32_MAX)
+    return __itoa::__base_10_u32(__buffer, static_cast<uint32_t>(__value));
+
+  // Numbers in the range UINT32_MAX <= val < 10'000'000'000 always contain 10
+  // digits and are outputted after this if statement.
+  if (__value >= 10000000000) {
+    // This function properly deterimines the first non-zero leading digit.
+    __buffer = __itoa::__base_10_u32(__buffer, static_cast<uint32_t>(__value / 10000000000));
+    __value %= 10000000000;
+  }
+  return __itoa::__append10(__buffer, __value);
+}
+
+#  ifndef _LIBCPP_HAS_NO_INT128
+/// \returns 10^\a exp
+///
+/// \pre \a exp [19, 39]
+///
+/// \note The lookup table contains a partial set of exponents limiting the
+/// range that can be used. However the range is sufficient for
+/// \ref __base_10_u128.
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI inline __uint128_t __pow_10(int __exp) noexcept {
+  _LIBCPP_ASSERT_INTERNAL(__exp >= __pow10_128_offset, "Index out of bounds");
+  return __pow10_128[__exp - __pow10_128_offset];
+}
+
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI inline char*
+__base_10_u128(char* __buffer, __uint128_t __value) noexcept {
+  _LIBCPP_ASSERT_INTERNAL(
+      __value > numeric_limits<uint64_t>::max(), "The optimizations for this algorithm fails when this isn't true.");
+
+  // Unlike the 64 to 32 bit case the 128 bit case the "upper half" can't be
+  // stored in the "lower half". Instead we first need to handle the top most
+  // digits separately.
+  //
+  // Maximum unsigned values
+  // 64  bit                             18'446'744'073'709'551'615 (20 digits)
+  // 128 bit    340'282'366'920'938'463'463'374'607'431'768'211'455 (39 digits)
+  // step 1     ^                                                   ([0-1] digits)
+  // step 2      ^^^^^^^^^^^^^^^^^^^^^^^^^                          ([0-19] digits)
+  // step 3                               ^^^^^^^^^^^^^^^^^^^^^^^^^ (19 digits)
+  if (__value >= __itoa::__pow_10(38)) {
+    // step 1
+    __buffer = __itoa::__append1(__buffer, static_cast<uint32_t>(__value / __itoa::__pow_10(38)));
+    __value %= __itoa::__pow_10(38);
+
+    // step 2 always 19 digits.
+    // They are handled here since leading zeros need to be appended to the buffer,
+    __buffer = __itoa::__append9(__buffer, static_cast<uint32_t>(__value / __itoa::__pow_10(29)));
+    __value %= __itoa::__pow_10(29);
+    __buffer = __itoa::__append10(__buffer, static_cast<uint64_t>(__value / __itoa::__pow_10(19)));
+    __value %= __itoa::__pow_10(19);
+  } else {
+    // step 2
+    // This version needs to determine the position of the leading non-zero digit.
+    __buffer = __base_10_u64(__buffer, static_cast<uint64_t>(__value / __itoa::__pow_10(19)));
+    __value %= __itoa::__pow_10(19);
+  }
+
+  // Step 3
+  __buffer = __itoa::__append9(__buffer, static_cast<uint32_t>(__value / 10000000000));
+  __buffer = __itoa::__append10(__buffer, static_cast<uint64_t>(__value % 10000000000));
+
+  return __buffer;
+}
+#  endif
+} // namespace __itoa
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___CHARCONV_TO_CHARS_BASE_10_H
diff --git a/libcxx/include/__cxx03/__charconv/to_chars_floating_point.h b/libcxx/include/__cxx03/__charconv/to_chars_floating_point.h
new file mode 100644
index 000000000000..118f316b21a1
--- /dev/null
+++ b/libcxx/include/__cxx03/__charconv/to_chars_floating_point.h
@@ -0,0 +1,55 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHARCONV_TO_CHARS_FLOATING_POINT_H
+#define _LIBCPP___CHARCONV_TO_CHARS_FLOATING_POINT_H
+
+#include <__charconv/chars_format.h>
+#include <__charconv/to_chars_result.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_EXPORTED_FROM_ABI to_chars_result
+to_chars(char* __first, char* __last, float __value);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_EXPORTED_FROM_ABI to_chars_result
+to_chars(char* __first, char* __last, double __value);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_EXPORTED_FROM_ABI to_chars_result
+to_chars(char* __first, char* __last, long double __value);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_EXPORTED_FROM_ABI to_chars_result
+to_chars(char* __first, char* __last, float __value, chars_format __fmt);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_EXPORTED_FROM_ABI to_chars_result
+to_chars(char* __first, char* __last, double __value, chars_format __fmt);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_EXPORTED_FROM_ABI to_chars_result
+to_chars(char* __first, char* __last, long double __value, chars_format __fmt);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_EXPORTED_FROM_ABI to_chars_result
+to_chars(char* __first, char* __last, float __value, chars_format __fmt, int __precision);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_EXPORTED_FROM_ABI to_chars_result
+to_chars(char* __first, char* __last, double __value, chars_format __fmt, int __precision);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_EXPORTED_FROM_ABI to_chars_result
+to_chars(char* __first, char* __last, long double __value, chars_format __fmt, int __precision);
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CHARCONV_TO_CHARS_FLOATING_POINT_H
diff --git a/libcxx/include/__cxx03/__charconv/to_chars_integral.h b/libcxx/include/__cxx03/__charconv/to_chars_integral.h
new file mode 100644
index 000000000000..0369f4dfb9bd
--- /dev/null
+++ b/libcxx/include/__cxx03/__charconv/to_chars_integral.h
@@ -0,0 +1,327 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHARCONV_TO_CHARS_INTEGRAL_H
+#define _LIBCPP___CHARCONV_TO_CHARS_INTEGRAL_H
+
+#include <__algorithm/copy_n.h>
+#include <__assert>
+#include <__bit/countl.h>
+#include <__charconv/tables.h>
+#include <__charconv/to_chars_base_10.h>
+#include <__charconv/to_chars_result.h>
+#include <__charconv/traits.h>
+#include <__config>
+#include <__system_error/errc.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/make_32_64_or_128_bit.h>
+#include <__type_traits/make_unsigned.h>
+#include <__utility/unreachable.h>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+to_chars_result to_chars(char*, char*, bool, int = 10) = delete;
+
+template <typename _Tp>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI to_chars_result
+__to_chars_itoa(char* __first, char* __last, _Tp __value, false_type);
+
+template <typename _Tp>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI to_chars_result
+__to_chars_itoa(char* __first, char* __last, _Tp __value, true_type) {
+  auto __x = std::__to_unsigned_like(__value);
+  if (__value < 0 && __first != __last) {
+    *__first++ = '-';
+    __x        = std::__complement(__x);
+  }
+
+  return std::__to_chars_itoa(__first, __last, __x, false_type());
+}
+
+template <typename _Tp>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI to_chars_result
+__to_chars_itoa(char* __first, char* __last, _Tp __value, false_type) {
+  using __tx  = __itoa::__traits<_Tp>;
+  auto __diff = __last - __first;
+
+  if (__tx::digits <= __diff || __tx::__width(__value) <= __diff)
+    return {__tx::__convert(__first, __value), errc(0)};
+  else
+    return {__last, errc::value_too_large};
+}
+
+#  ifndef _LIBCPP_HAS_NO_INT128
+template <>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI to_chars_result
+__to_chars_itoa(char* __first, char* __last, __uint128_t __value, false_type) {
+  // When the value fits in 64-bits use the 64-bit code path. This reduces
+  // the number of expensive calculations on 128-bit values.
+  //
+  // NOTE the 128-bit code path requires this optimization.
+  if (__value <= numeric_limits<uint64_t>::max())
+    return __to_chars_itoa(__first, __last, static_cast<uint64_t>(__value), false_type());
+
+  using __tx  = __itoa::__traits<__uint128_t>;
+  auto __diff = __last - __first;
+
+  if (__tx::digits <= __diff || __tx::__width(__value) <= __diff)
+    return {__tx::__convert(__first, __value), errc(0)};
+  else
+    return {__last, errc::value_too_large};
+}
+#  endif
+
+template <class _Tp>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI to_chars_result
+__to_chars_integral(char* __first, char* __last, _Tp __value, int __base, false_type);
+
+template <typename _Tp>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI to_chars_result
+__to_chars_integral(char* __first, char* __last, _Tp __value, int __base, true_type) {
+  auto __x = std::__to_unsigned_like(__value);
+  if (__value < 0 && __first != __last) {
+    *__first++ = '-';
+    __x        = std::__complement(__x);
+  }
+
+  return std::__to_chars_integral(__first, __last, __x, __base, false_type());
+}
+
+namespace __itoa {
+
+template <unsigned _Base>
+struct _LIBCPP_HIDDEN __integral;
+
+template <>
+struct _LIBCPP_HIDDEN __integral<2> {
+  template <typename _Tp>
+  _LIBCPP_HIDE_FROM_ABI static constexpr int __width(_Tp __value) noexcept {
+    // If value == 0 still need one digit. If the value != this has no
+    // effect since the code scans for the most significant bit set. (Note
+    // that __libcpp_clz doesn't work for 0.)
+    return numeric_limits<_Tp>::digits - std::__libcpp_clz(__value | 1);
+  }
+
+  template <typename _Tp>
+  _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI static to_chars_result
+  __to_chars(char* __first, char* __last, _Tp __value) {
+    ptrdiff_t __cap = __last - __first;
+    int __n         = __width(__value);
+    if (__n > __cap)
+      return {__last, errc::value_too_large};
+
+    __last                   = __first + __n;
+    char* __p                = __last;
+    const unsigned __divisor = 16;
+    while (__value > __divisor) {
+      unsigned __c = __value % __divisor;
+      __value /= __divisor;
+      __p -= 4;
+      std::copy_n(&__base_2_lut[4 * __c], 4, __p);
+    }
+    do {
+      unsigned __c = __value % 2;
+      __value /= 2;
+      *--__p = "01"[__c];
+    } while (__value != 0);
+    return {__last, errc(0)};
+  }
+};
+
+template <>
+struct _LIBCPP_HIDDEN __integral<8> {
+  template <typename _Tp>
+  _LIBCPP_HIDE_FROM_ABI static constexpr int __width(_Tp __value) noexcept {
+    // If value == 0 still need one digit. If the value != this has no
+    // effect since the code scans for the most significat bit set. (Note
+    // that __libcpp_clz doesn't work for 0.)
+    return ((numeric_limits<_Tp>::digits - std::__libcpp_clz(__value | 1)) + 2) / 3;
+  }
+
+  template <typename _Tp>
+  _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI static to_chars_result
+  __to_chars(char* __first, char* __last, _Tp __value) {
+    ptrdiff_t __cap = __last - __first;
+    int __n         = __width(__value);
+    if (__n > __cap)
+      return {__last, errc::value_too_large};
+
+    __last             = __first + __n;
+    char* __p          = __last;
+    unsigned __divisor = 64;
+    while (__value > __divisor) {
+      unsigned __c = __value % __divisor;
+      __value /= __divisor;
+      __p -= 2;
+      std::copy_n(&__base_8_lut[2 * __c], 2, __p);
+    }
+    do {
+      unsigned __c = __value % 8;
+      __value /= 8;
+      *--__p = "01234567"[__c];
+    } while (__value != 0);
+    return {__last, errc(0)};
+  }
+};
+
+template <>
+struct _LIBCPP_HIDDEN __integral<16> {
+  template <typename _Tp>
+  _LIBCPP_HIDE_FROM_ABI static constexpr int __width(_Tp __value) noexcept {
+    // If value == 0 still need one digit. If the value != this has no
+    // effect since the code scans for the most significat bit set. (Note
+    // that __libcpp_clz doesn't work for 0.)
+    return (numeric_limits<_Tp>::digits - std::__libcpp_clz(__value | 1) + 3) / 4;
+  }
+
+  template <typename _Tp>
+  _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI static to_chars_result
+  __to_chars(char* __first, char* __last, _Tp __value) {
+    ptrdiff_t __cap = __last - __first;
+    int __n         = __width(__value);
+    if (__n > __cap)
+      return {__last, errc::value_too_large};
+
+    __last             = __first + __n;
+    char* __p          = __last;
+    unsigned __divisor = 256;
+    while (__value > __divisor) {
+      unsigned __c = __value % __divisor;
+      __value /= __divisor;
+      __p -= 2;
+      std::copy_n(&__base_16_lut[2 * __c], 2, __p);
+    }
+    if (__first != __last)
+      do {
+        unsigned __c = __value % 16;
+        __value /= 16;
+        *--__p = "0123456789abcdef"[__c];
+      } while (__value != 0);
+    return {__last, errc(0)};
+  }
+};
+
+} // namespace __itoa
+
+template <unsigned _Base, typename _Tp, __enable_if_t<(sizeof(_Tp) >= sizeof(unsigned)), int> = 0>
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI int __to_chars_integral_width(_Tp __value) {
+  return __itoa::__integral<_Base>::__width(__value);
+}
+
+template <unsigned _Base, typename _Tp, __enable_if_t<(sizeof(_Tp) < sizeof(unsigned)), int> = 0>
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI int __to_chars_integral_width(_Tp __value) {
+  return std::__to_chars_integral_width<_Base>(static_cast<unsigned>(__value));
+}
+
+template <unsigned _Base, typename _Tp, __enable_if_t<(sizeof(_Tp) >= sizeof(unsigned)), int> = 0>
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI to_chars_result
+__to_chars_integral(char* __first, char* __last, _Tp __value) {
+  return __itoa::__integral<_Base>::__to_chars(__first, __last, __value);
+}
+
+template <unsigned _Base, typename _Tp, __enable_if_t<(sizeof(_Tp) < sizeof(unsigned)), int> = 0>
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI to_chars_result
+__to_chars_integral(char* __first, char* __last, _Tp __value) {
+  return std::__to_chars_integral<_Base>(__first, __last, static_cast<unsigned>(__value));
+}
+
+template <typename _Tp>
+_LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI int __to_chars_integral_width(_Tp __value, unsigned __base) {
+  _LIBCPP_ASSERT_INTERNAL(__value >= 0, "The function requires a non-negative value.");
+
+  unsigned __base_2 = __base * __base;
+  unsigned __base_3 = __base_2 * __base;
+  unsigned __base_4 = __base_2 * __base_2;
+
+  int __r = 0;
+  while (true) {
+    if (__value < __base)
+      return __r + 1;
+    if (__value < __base_2)
+      return __r + 2;
+    if (__value < __base_3)
+      return __r + 3;
+    if (__value < __base_4)
+      return __r + 4;
+
+    __value /= __base_4;
+    __r += 4;
+  }
+
+  __libcpp_unreachable();
+}
+
+template <typename _Tp>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI to_chars_result
+__to_chars_integral(char* __first, char* __last, _Tp __value, int __base, false_type) {
+  if (__base == 10) [[likely]]
+    return std::__to_chars_itoa(__first, __last, __value, false_type());
+
+  switch (__base) {
+  case 2:
+    return std::__to_chars_integral<2>(__first, __last, __value);
+  case 8:
+    return std::__to_chars_integral<8>(__first, __last, __value);
+  case 16:
+    return std::__to_chars_integral<16>(__first, __last, __value);
+  }
+
+  ptrdiff_t __cap = __last - __first;
+  int __n         = std::__to_chars_integral_width(__value, __base);
+  if (__n > __cap)
+    return {__last, errc::value_too_large};
+
+  __last    = __first + __n;
+  char* __p = __last;
+  do {
+    unsigned __c = __value % __base;
+    __value /= __base;
+    *--__p = "0123456789abcdefghijklmnopqrstuvwxyz"[__c];
+  } while (__value != 0);
+  return {__last, errc(0)};
+}
+
+template <typename _Tp, __enable_if_t<is_integral<_Tp>::value, int> = 0>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI to_chars_result
+to_chars(char* __first, char* __last, _Tp __value) {
+  using _Type = __make_32_64_or_128_bit_t<_Tp>;
+  static_assert(!is_same<_Type, void>::value, "unsupported integral type used in to_chars");
+  return std::__to_chars_itoa(__first, __last, static_cast<_Type>(__value), is_signed<_Tp>());
+}
+
+template <typename _Tp, __enable_if_t<is_integral<_Tp>::value, int> = 0>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI to_chars_result
+to_chars(char* __first, char* __last, _Tp __value, int __base) {
+  _LIBCPP_ASSERT_UNCATEGORIZED(2 <= __base && __base <= 36, "base not in [2, 36]");
+
+  using _Type = __make_32_64_or_128_bit_t<_Tp>;
+  return std::__to_chars_integral(__first, __last, static_cast<_Type>(__value), __base, is_signed<_Tp>());
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___CHARCONV_TO_CHARS_INTEGRAL_H
diff --git a/libcxx/include/__cxx03/__charconv/to_chars_result.h b/libcxx/include/__cxx03/__charconv/to_chars_result.h
new file mode 100644
index 000000000000..8df0897a49fb
--- /dev/null
+++ b/libcxx/include/__cxx03/__charconv/to_chars_result.h
@@ -0,0 +1,39 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHARCONV_TO_CHARS_RESULT_H
+#define _LIBCPP___CHARCONV_TO_CHARS_RESULT_H
+
+#include <__config>
+#include <__system_error/errc.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+struct _LIBCPP_EXPORTED_FROM_ABI to_chars_result {
+  char* ptr;
+  errc ec;
+#  if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI friend bool operator==(const to_chars_result&, const to_chars_result&) = default;
+#  endif
+#  if _LIBCPP_STD_VER >= 26
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return ec == errc{}; }
+#  endif
+};
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CHARCONV_TO_CHARS_RESULT_H
diff --git a/libcxx/include/__cxx03/__charconv/traits.h b/libcxx/include/__cxx03/__charconv/traits.h
new file mode 100644
index 000000000000..c91c6da32479
--- /dev/null
+++ b/libcxx/include/__cxx03/__charconv/traits.h
@@ -0,0 +1,200 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHARCONV_TRAITS
+#define _LIBCPP___CHARCONV_TRAITS
+
+#include <__assert>
+#include <__bit/countl.h>
+#include <__charconv/tables.h>
+#include <__charconv/to_chars_base_10.h>
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_unsigned.h>
+#include <cstdint>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+namespace __itoa {
+
+template <typename _Tp, typename = void>
+struct _LIBCPP_HIDDEN __traits_base;
+
+template <typename _Tp>
+struct _LIBCPP_HIDDEN __traits_base<_Tp, __enable_if_t<sizeof(_Tp) <= sizeof(uint32_t)>> {
+  using type = uint32_t;
+
+  /// The width estimation using a log10 algorithm.
+  ///
+  /// The algorithm is based on
+  /// http://graphics.stanford.edu/~seander/bithacks.html#IntegerLog10
+  /// Instead of using IntegerLogBase2 it uses __libcpp_clz. Since that
+  /// function requires its input to have at least one bit set the value of
+  /// zero is set to one. This means the first element of the lookup table is
+  /// zero.
+  static _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI int __width(_Tp __v) {
+    auto __t = (32 - std::__libcpp_clz(static_cast<type>(__v | 1))) * 1233 >> 12;
+    return __t - (__v < __itoa::__pow10_32[__t]) + 1;
+  }
+
+  static _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI char* __convert(char* __p, _Tp __v) {
+    return __itoa::__base_10_u32(__p, __v);
+  }
+
+  static _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI decltype(__pow10_32)& __pow() {
+    return __itoa::__pow10_32;
+  }
+};
+
+template <typename _Tp>
+struct _LIBCPP_HIDDEN __traits_base<_Tp, __enable_if_t<sizeof(_Tp) == sizeof(uint64_t)>> {
+  using type = uint64_t;
+
+  /// The width estimation using a log10 algorithm.
+  ///
+  /// The algorithm is based on
+  /// http://graphics.stanford.edu/~seander/bithacks.html#IntegerLog10
+  /// Instead of using IntegerLogBase2 it uses __libcpp_clz. Since that
+  /// function requires its input to have at least one bit set the value of
+  /// zero is set to one. This means the first element of the lookup table is
+  /// zero.
+  static _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI int __width(_Tp __v) {
+    auto __t = (64 - std::__libcpp_clz(static_cast<type>(__v | 1))) * 1233 >> 12;
+    return __t - (__v < __itoa::__pow10_64[__t]) + 1;
+  }
+
+  static _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI char* __convert(char* __p, _Tp __v) {
+    return __itoa::__base_10_u64(__p, __v);
+  }
+
+  static _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI decltype(__pow10_64)& __pow() {
+    return __itoa::__pow10_64;
+  }
+};
+
+#  ifndef _LIBCPP_HAS_NO_INT128
+template <typename _Tp>
+struct _LIBCPP_HIDDEN __traits_base<_Tp, __enable_if_t<sizeof(_Tp) == sizeof(__uint128_t)> > {
+  using type = __uint128_t;
+
+  /// The width estimation using a log10 algorithm.
+  ///
+  /// The algorithm is based on
+  /// http://graphics.stanford.edu/~seander/bithacks.html#IntegerLog10
+  /// Instead of using IntegerLogBase2 it uses __libcpp_clz. Since that
+  /// function requires its input to have at least one bit set the value of
+  /// zero is set to one. This means the first element of the lookup table is
+  /// zero.
+  static _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI int __width(_Tp __v) {
+    _LIBCPP_ASSERT_INTERNAL(
+        __v > numeric_limits<uint64_t>::max(), "The optimizations for this algorithm fail when this isn't true.");
+    // There's always a bit set in the upper 64-bits.
+    auto __t = (128 - std::__libcpp_clz(static_cast<uint64_t>(__v >> 64))) * 1233 >> 12;
+    _LIBCPP_ASSERT_INTERNAL(__t >= __itoa::__pow10_128_offset, "Index out of bounds");
+    // __t is adjusted since the lookup table misses the lower entries.
+    return __t - (__v < __itoa::__pow10_128[__t - __itoa::__pow10_128_offset]) + 1;
+  }
+
+  static _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI char* __convert(char* __p, _Tp __v) {
+    return __itoa::__base_10_u128(__p, __v);
+  }
+
+  // TODO FMT This pow function should get an index.
+  // By moving this to its own header it can be reused by the pow function in to_chars_base_10.
+  static _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI decltype(__pow10_128)& __pow() {
+    return __itoa::__pow10_128;
+  }
+};
+#  endif
+
+template <typename _Tp>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool
+__mul_overflowed(unsigned char __a, _Tp __b, unsigned char& __r) {
+  auto __c = __a * __b;
+  __r      = __c;
+  return __c > numeric_limits<unsigned char>::max();
+}
+
+template <typename _Tp>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool
+__mul_overflowed(unsigned short __a, _Tp __b, unsigned short& __r) {
+  auto __c = __a * __b;
+  __r      = __c;
+  return __c > numeric_limits<unsigned short>::max();
+}
+
+template <typename _Tp>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool __mul_overflowed(_Tp __a, _Tp __b, _Tp& __r) {
+  static_assert(is_unsigned<_Tp>::value, "");
+  return __builtin_mul_overflow(__a, __b, &__r);
+}
+
+template <typename _Tp, typename _Up>
+inline _LIBCPP_HIDE_FROM_ABI bool _LIBCPP_CONSTEXPR_SINCE_CXX23 __mul_overflowed(_Tp __a, _Up __b, _Tp& __r) {
+  return __itoa::__mul_overflowed(__a, static_cast<_Tp>(__b), __r);
+}
+
+template <typename _Tp>
+struct _LIBCPP_HIDDEN __traits : __traits_base<_Tp> {
+  static constexpr int digits = numeric_limits<_Tp>::digits10 + 1;
+  using __traits_base<_Tp>::__pow;
+  using typename __traits_base<_Tp>::type;
+
+  // precondition: at least one non-zero character available
+  static _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI char const*
+  __read(char const* __p, char const* __ep, type& __a, type& __b) {
+    type __cprod[digits];
+    int __j = digits - 1;
+    int __i = digits;
+    do {
+      if (*__p < '0' || *__p > '9')
+        break;
+      __cprod[--__i] = *__p++ - '0';
+    } while (__p != __ep && __i != 0);
+
+    __a = __inner_product(__cprod + __i + 1, __cprod + __j, __pow() + 1, __cprod[__i]);
+    if (__itoa::__mul_overflowed(__cprod[__j], __pow()[__j - __i], __b))
+      --__p;
+    return __p;
+  }
+
+  template <typename _It1, typename _It2, class _Up>
+  static _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI _Up
+  __inner_product(_It1 __first1, _It1 __last1, _It2 __first2, _Up __init) {
+    for (; __first1 < __last1; ++__first1, ++__first2)
+      __init = __init + *__first1 * *__first2;
+    return __init;
+  }
+};
+
+} // namespace __itoa
+
+template <typename _Tp>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI _Tp __complement(_Tp __x) {
+  static_assert(is_unsigned<_Tp>::value, "cast to unsigned first");
+  return _Tp(~__x + 1);
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___CHARCONV_TRAITS
diff --git a/libcxx/include/__cxx03/__chrono/calendar.h b/libcxx/include/__cxx03/__chrono/calendar.h
new file mode 100644
index 000000000000..bb1c5e7ebc8d
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/calendar.h
@@ -0,0 +1,44 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_CALENDAR_H
+#define _LIBCPP___CHRONO_CALENDAR_H
+
+#include <__chrono/duration.h>
+#include <__chrono/time_point.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+struct local_t {};
+template <class _Duration>
+using local_time    = time_point<local_t, _Duration>;
+using local_seconds = local_time<seconds>;
+using local_days    = local_time<days>;
+
+struct last_spec {
+  explicit last_spec() = default;
+};
+inline constexpr last_spec last{};
+
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___CHRONO_CALENDAR_H
diff --git a/libcxx/include/__cxx03/__chrono/concepts.h b/libcxx/include/__cxx03/__chrono/concepts.h
new file mode 100644
index 000000000000..61ec256b23ab
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/concepts.h
@@ -0,0 +1,36 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_CONCEPTS_H
+#define _LIBCPP___CHRONO_CONCEPTS_H
+
+#include <__chrono/hh_mm_ss.h>
+#include <__chrono/time_point.h>
+#include <__config>
+#include <__type_traits/is_specialization.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+concept __is_hh_mm_ss = __is_specialization_v<_Tp, chrono::hh_mm_ss>;
+
+template <class _Tp>
+concept __is_time_point = __is_specialization_v<_Tp, chrono::time_point>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CHRONO_CONCEPTS_H
diff --git a/libcxx/include/__cxx03/__chrono/convert_to_timespec.h b/libcxx/include/__cxx03/__chrono/convert_to_timespec.h
new file mode 100644
index 000000000000..11e0b826d05b
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/convert_to_timespec.h
@@ -0,0 +1,51 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_CONVERT_TO_TIMESPEC_H
+#define _LIBCPP___CHRONO_CONVERT_TO_TIMESPEC_H
+
+#include <__chrono/duration.h>
+#include <__config>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Convert a nanoseconds duration to the given TimeSpec type, which must have
+// the same properties as std::timespec.
+template <class _TimeSpec>
+_LIBCPP_HIDE_FROM_ABI inline _TimeSpec __convert_to_timespec(const chrono::nanoseconds& __ns) {
+  using namespace chrono;
+  seconds __s = duration_cast<seconds>(__ns);
+  _TimeSpec __ts;
+  typedef decltype(__ts.tv_sec) __ts_sec;
+  const __ts_sec __ts_sec_max = numeric_limits<__ts_sec>::max();
+
+  if (__s.count() < __ts_sec_max) {
+    __ts.tv_sec  = static_cast<__ts_sec>(__s.count());
+    __ts.tv_nsec = static_cast<decltype(__ts.tv_nsec)>((__ns - __s).count());
+  } else {
+    __ts.tv_sec  = __ts_sec_max;
+    __ts.tv_nsec = 999999999; // (10^9 - 1)
+  }
+
+  return __ts;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___CHRONO_CONVERT_TO_TIMESPEC_H
diff --git a/libcxx/include/__cxx03/__chrono/convert_to_tm.h b/libcxx/include/__cxx03/__chrono/convert_to_tm.h
new file mode 100644
index 000000000000..3a51019b8078
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/convert_to_tm.h
@@ -0,0 +1,202 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_CONVERT_TO_TM_H
+#define _LIBCPP___CHRONO_CONVERT_TO_TM_H
+
+#include <__chrono/calendar.h>
+#include <__chrono/concepts.h>
+#include <__chrono/day.h>
+#include <__chrono/duration.h>
+#include <__chrono/file_clock.h>
+#include <__chrono/hh_mm_ss.h>
+#include <__chrono/local_info.h>
+#include <__chrono/month.h>
+#include <__chrono/month_weekday.h>
+#include <__chrono/monthday.h>
+#include <__chrono/statically_widen.h>
+#include <__chrono/sys_info.h>
+#include <__chrono/system_clock.h>
+#include <__chrono/time_point.h>
+#include <__chrono/weekday.h>
+#include <__chrono/year.h>
+#include <__chrono/year_month.h>
+#include <__chrono/year_month_day.h>
+#include <__chrono/year_month_weekday.h>
+#include <__chrono/zoned_time.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/format_error.h>
+#include <__memory/addressof.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_specialization.h>
+#include <cstdint>
+#include <ctime>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// Conerts a chrono date and weekday to a given _Tm type.
+//
+// This is an implementation detail for the function
+//   template <class _Tm, class _ChronoT>
+//   _Tm __convert_to_tm(const _ChronoT& __value)
+//
+// This manually converts the two values to the proper type. It is possible to
+// convert from sys_days to time_t and then to _Tm. But this leads to the Y2K
+// bug when time_t is a 32-bit signed integer. Chrono considers years beyond
+// the year 2038 valid, so instead do the transformation manually.
+template <class _Tm, class _Date>
+  requires(same_as<_Date, chrono::year_month_day> || same_as<_Date, chrono::year_month_day_last>)
+_LIBCPP_HIDE_FROM_ABI _Tm __convert_to_tm(const _Date& __date, chrono::weekday __weekday) {
+  _Tm __result = {};
+#  ifdef __GLIBC__
+  __result.tm_zone = "UTC";
+#  endif
+  __result.tm_year = static_cast<int>(__date.year()) - 1900;
+  __result.tm_mon  = static_cast<unsigned>(__date.month()) - 1;
+  __result.tm_mday = static_cast<unsigned>(__date.day());
+  __result.tm_wday = static_cast<unsigned>(__weekday.c_encoding());
+  __result.tm_yday =
+      (static_cast<chrono::sys_days>(__date) -
+       static_cast<chrono::sys_days>(chrono::year_month_day{__date.year(), chrono::January, chrono::day{1}}))
+          .count();
+
+  return __result;
+}
+
+template <class _Tm, class _Duration>
+_LIBCPP_HIDE_FROM_ABI _Tm __convert_to_tm(const chrono::sys_time<_Duration> __tp) {
+  chrono::sys_days __days = chrono::floor<chrono::days>(__tp);
+  chrono::year_month_day __ymd{__days};
+
+  _Tm __result = std::__convert_to_tm<_Tm>(chrono::year_month_day{__ymd}, chrono::weekday{__days});
+
+  uint64_t __sec =
+      chrono::duration_cast<chrono::seconds>(__tp - chrono::time_point_cast<chrono::seconds>(__days)).count();
+  __sec %= 24 * 3600;
+  __result.tm_hour = __sec / 3600;
+  __sec %= 3600;
+  __result.tm_min = __sec / 60;
+  __result.tm_sec = __sec % 60;
+
+  return __result;
+}
+
+// Convert a chrono (calendar) time point, or dururation to the given _Tm type,
+// which must have the same properties as std::tm.
+template <class _Tm, class _ChronoT>
+_LIBCPP_HIDE_FROM_ABI _Tm __convert_to_tm(const _ChronoT& __value) {
+  _Tm __result = {};
+#  ifdef __GLIBC__
+  __result.tm_zone = "UTC";
+#  endif
+
+  if constexpr (__is_time_point<_ChronoT>) {
+    if constexpr (same_as<typename _ChronoT::clock, chrono::system_clock>)
+      return std::__convert_to_tm<_Tm>(__value);
+    else if constexpr (same_as<typename _ChronoT::clock, chrono::file_clock>)
+      return std::__convert_to_tm<_Tm>(_ChronoT::clock::to_sys(__value));
+    else if constexpr (same_as<typename _ChronoT::clock, chrono::local_t>)
+      return std::__convert_to_tm<_Tm>(chrono::sys_time<typename _ChronoT::duration>{__value.time_since_epoch()});
+    else
+      static_assert(sizeof(_ChronoT) == 0, "TODO: Add the missing clock specialization");
+  } else if constexpr (chrono::__is_duration<_ChronoT>::value) {
+    // [time.format]/6
+    //   ...  However, if a flag refers to a "time of day" (e.g. %H, %I, %p,
+    //   etc.), then a specialization of duration is interpreted as the time of
+    //   day elapsed since midnight.
+
+    // Not all values can be converted to hours, it may run into ratio
+    // conversion errors. In that case the conversion to seconds works.
+    if constexpr (is_convertible_v<_ChronoT, chrono::hours>) {
+      auto __hour      = chrono::floor<chrono::hours>(__value);
+      auto __sec       = chrono::duration_cast<chrono::seconds>(__value - __hour);
+      __result.tm_hour = __hour.count() % 24;
+      __result.tm_min  = __sec.count() / 60;
+      __result.tm_sec  = __sec.count() % 60;
+    } else {
+      uint64_t __sec = chrono::duration_cast<chrono::seconds>(__value).count();
+      __sec %= 24 * 3600;
+      __result.tm_hour = __sec / 3600;
+      __sec %= 3600;
+      __result.tm_min = __sec / 60;
+      __result.tm_sec = __sec % 60;
+    }
+  } else if constexpr (same_as<_ChronoT, chrono::day>)
+    __result.tm_mday = static_cast<unsigned>(__value);
+  else if constexpr (same_as<_ChronoT, chrono::month>)
+    __result.tm_mon = static_cast<unsigned>(__value) - 1;
+  else if constexpr (same_as<_ChronoT, chrono::year>)
+    __result.tm_year = static_cast<int>(__value) - 1900;
+  else if constexpr (same_as<_ChronoT, chrono::weekday>)
+    __result.tm_wday = __value.c_encoding();
+  else if constexpr (same_as<_ChronoT, chrono::weekday_indexed> || same_as<_ChronoT, chrono::weekday_last>)
+    __result.tm_wday = __value.weekday().c_encoding();
+  else if constexpr (same_as<_ChronoT, chrono::month_day>) {
+    __result.tm_mday = static_cast<unsigned>(__value.day());
+    __result.tm_mon  = static_cast<unsigned>(__value.month()) - 1;
+  } else if constexpr (same_as<_ChronoT, chrono::month_day_last>) {
+    __result.tm_mon = static_cast<unsigned>(__value.month()) - 1;
+  } else if constexpr (same_as<_ChronoT, chrono::month_weekday> || same_as<_ChronoT, chrono::month_weekday_last>) {
+    __result.tm_wday = __value.weekday_indexed().weekday().c_encoding();
+    __result.tm_mon  = static_cast<unsigned>(__value.month()) - 1;
+  } else if constexpr (same_as<_ChronoT, chrono::year_month>) {
+    __result.tm_year = static_cast<int>(__value.year()) - 1900;
+    __result.tm_mon  = static_cast<unsigned>(__value.month()) - 1;
+  } else if constexpr (same_as<_ChronoT, chrono::year_month_day> || same_as<_ChronoT, chrono::year_month_day_last>) {
+    return std::__convert_to_tm<_Tm>(
+        chrono::year_month_day{__value}, chrono::weekday{static_cast<chrono::sys_days>(__value)});
+  } else if constexpr (same_as<_ChronoT, chrono::year_month_weekday> ||
+                       same_as<_ChronoT, chrono::year_month_weekday_last>) {
+    return std::__convert_to_tm<_Tm>(chrono::year_month_day{static_cast<chrono::sys_days>(__value)}, __value.weekday());
+  } else if constexpr (__is_hh_mm_ss<_ChronoT>) {
+    __result.tm_sec = __value.seconds().count();
+    __result.tm_min = __value.minutes().count();
+    // In libc++ hours is stored as a long. The type in std::tm is an int. So
+    // the overflow can only occur when hour uses more bits than an int
+    // provides.
+    if constexpr (sizeof(std::chrono::hours::rep) > sizeof(__result.tm_hour))
+      if (__value.hours().count() > std::numeric_limits<decltype(__result.tm_hour)>::max())
+        std::__throw_format_error("Formatting hh_mm_ss, encountered an hour overflow");
+    __result.tm_hour = __value.hours().count();
+#  if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+  } else if constexpr (same_as<_ChronoT, chrono::sys_info>) {
+    // Has no time information.
+  } else if constexpr (same_as<_ChronoT, chrono::local_info>) {
+    // Has no time information.
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  } else if constexpr (__is_specialization_v<_ChronoT, chrono::zoned_time>) {
+    return std::__convert_to_tm<_Tm>(
+        chrono::sys_time<typename _ChronoT::duration>{__value.get_local_time().time_since_epoch()});
+#    endif
+#  endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+  } else
+    static_assert(sizeof(_ChronoT) == 0, "Add the missing type specialization");
+
+  return __result;
+}
+
+#endif // if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___CHRONO_CONVERT_TO_TM_H
diff --git a/libcxx/include/__cxx03/__chrono/day.h b/libcxx/include/__cxx03/__chrono/day.h
new file mode 100644
index 000000000000..7342084b08c8
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/day.h
@@ -0,0 +1,99 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_DAY_H
+#define _LIBCPP___CHRONO_DAY_H
+
+#include <__chrono/duration.h>
+#include <__config>
+#include <compare>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+class day {
+private:
+  unsigned char __d_;
+
+public:
+  day() = default;
+  _LIBCPP_HIDE_FROM_ABI explicit inline constexpr day(unsigned __val) noexcept
+      : __d_(static_cast<unsigned char>(__val)) {}
+  _LIBCPP_HIDE_FROM_ABI inline constexpr day& operator++() noexcept {
+    ++__d_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr day operator++(int) noexcept {
+    day __tmp = *this;
+    ++(*this);
+    return __tmp;
+  }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr day& operator--() noexcept {
+    --__d_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr day operator--(int) noexcept {
+    day __tmp = *this;
+    --(*this);
+    return __tmp;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr day& operator+=(const days& __dd) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr day& operator-=(const days& __dd) noexcept;
+  _LIBCPP_HIDE_FROM_ABI explicit inline constexpr operator unsigned() const noexcept { return __d_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __d_ >= 1 && __d_ <= 31; }
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const day& __lhs, const day& __rhs) noexcept {
+  return static_cast<unsigned>(__lhs) == static_cast<unsigned>(__rhs);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering operator<=>(const day& __lhs, const day& __rhs) noexcept {
+  return static_cast<unsigned>(__lhs) <=> static_cast<unsigned>(__rhs);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr day operator+(const day& __lhs, const days& __rhs) noexcept {
+  return day(static_cast<unsigned>(__lhs) + __rhs.count());
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr day operator+(const days& __lhs, const day& __rhs) noexcept {
+  return __rhs + __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr day operator-(const day& __lhs, const days& __rhs) noexcept {
+  return __lhs + -__rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr days operator-(const day& __lhs, const day& __rhs) noexcept {
+  return days(static_cast<int>(static_cast<unsigned>(__lhs)) - static_cast<int>(static_cast<unsigned>(__rhs)));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr day& day::operator+=(const days& __dd) noexcept {
+  *this = *this + __dd;
+  return *this;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr day& day::operator-=(const days& __dd) noexcept {
+  *this = *this - __dd;
+  return *this;
+}
+
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___CHRONO_DAY_H
diff --git a/libcxx/include/__cxx03/__chrono/duration.h b/libcxx/include/__cxx03/__chrono/duration.h
new file mode 100644
index 000000000000..1e36d7342836
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/duration.h
@@ -0,0 +1,550 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_DURATION_H
+#define _LIBCPP___CHRONO_DURATION_H
+
+#include <__compare/ordering.h>
+#include <__compare/three_way_comparable.h>
+#include <__config>
+#include <__type_traits/common_type.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_floating_point.h>
+#include <limits>
+#include <ratio>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+template <class _Rep, class _Period = ratio<1> >
+class _LIBCPP_TEMPLATE_VIS duration;
+
+template <class _Tp>
+struct __is_duration : false_type {};
+
+template <class _Rep, class _Period>
+struct __is_duration<duration<_Rep, _Period> > : true_type {};
+
+template <class _Rep, class _Period>
+struct __is_duration<const duration<_Rep, _Period> > : true_type {};
+
+template <class _Rep, class _Period>
+struct __is_duration<volatile duration<_Rep, _Period> > : true_type {};
+
+template <class _Rep, class _Period>
+struct __is_duration<const volatile duration<_Rep, _Period> > : true_type {};
+
+} // namespace chrono
+
+template <class _Rep1, class _Period1, class _Rep2, class _Period2>
+struct _LIBCPP_TEMPLATE_VIS common_type<chrono::duration<_Rep1, _Period1>, chrono::duration<_Rep2, _Period2> > {
+  typedef chrono::duration<typename common_type<_Rep1, _Rep2>::type, typename __ratio_gcd<_Period1, _Period2>::type>
+      type;
+};
+
+namespace chrono {
+
+// duration_cast
+
+template <class _FromDuration,
+          class _ToDuration,
+          class _Period = typename ratio_divide<typename _FromDuration::period, typename _ToDuration::period>::type,
+          bool          = _Period::num == 1,
+          bool          = _Period::den == 1>
+struct __duration_cast;
+
+template <class _FromDuration, class _ToDuration, class _Period>
+struct __duration_cast<_FromDuration, _ToDuration, _Period, true, true> {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration operator()(const _FromDuration& __fd) const {
+    return _ToDuration(static_cast<typename _ToDuration::rep>(__fd.count()));
+  }
+};
+
+template <class _FromDuration, class _ToDuration, class _Period>
+struct __duration_cast<_FromDuration, _ToDuration, _Period, true, false> {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration operator()(const _FromDuration& __fd) const {
+    typedef typename common_type<typename _ToDuration::rep, typename _FromDuration::rep, intmax_t>::type _Ct;
+    return _ToDuration(
+        static_cast<typename _ToDuration::rep>(static_cast<_Ct>(__fd.count()) / static_cast<_Ct>(_Period::den)));
+  }
+};
+
+template <class _FromDuration, class _ToDuration, class _Period>
+struct __duration_cast<_FromDuration, _ToDuration, _Period, false, true> {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration operator()(const _FromDuration& __fd) const {
+    typedef typename common_type<typename _ToDuration::rep, typename _FromDuration::rep, intmax_t>::type _Ct;
+    return _ToDuration(
+        static_cast<typename _ToDuration::rep>(static_cast<_Ct>(__fd.count()) * static_cast<_Ct>(_Period::num)));
+  }
+};
+
+template <class _FromDuration, class _ToDuration, class _Period>
+struct __duration_cast<_FromDuration, _ToDuration, _Period, false, false> {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration operator()(const _FromDuration& __fd) const {
+    typedef typename common_type<typename _ToDuration::rep, typename _FromDuration::rep, intmax_t>::type _Ct;
+    return _ToDuration(static_cast<typename _ToDuration::rep>(
+        static_cast<_Ct>(__fd.count()) * static_cast<_Ct>(_Period::num) / static_cast<_Ct>(_Period::den)));
+  }
+};
+
+template <class _ToDuration, class _Rep, class _Period, __enable_if_t<__is_duration<_ToDuration>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration duration_cast(const duration<_Rep, _Period>& __fd) {
+  return __duration_cast<duration<_Rep, _Period>, _ToDuration>()(__fd);
+}
+
+template <class _Rep>
+struct _LIBCPP_TEMPLATE_VIS treat_as_floating_point : is_floating_point<_Rep> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Rep>
+inline constexpr bool treat_as_floating_point_v = treat_as_floating_point<_Rep>::value;
+#endif
+
+template <class _Rep>
+struct _LIBCPP_TEMPLATE_VIS duration_values {
+public:
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR _Rep zero() _NOEXCEPT { return _Rep(0); }
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR _Rep max() _NOEXCEPT { return numeric_limits<_Rep>::max(); }
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR _Rep min() _NOEXCEPT { return numeric_limits<_Rep>::lowest(); }
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _ToDuration, class _Rep, class _Period, enable_if_t<__is_duration<_ToDuration>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration floor(const duration<_Rep, _Period>& __d) {
+  _ToDuration __t = chrono::duration_cast<_ToDuration>(__d);
+  if (__t > __d)
+    __t = __t - _ToDuration{1};
+  return __t;
+}
+
+template <class _ToDuration, class _Rep, class _Period, enable_if_t<__is_duration<_ToDuration>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration ceil(const duration<_Rep, _Period>& __d) {
+  _ToDuration __t = chrono::duration_cast<_ToDuration>(__d);
+  if (__t < __d)
+    __t = __t + _ToDuration{1};
+  return __t;
+}
+
+template <class _ToDuration, class _Rep, class _Period, enable_if_t<__is_duration<_ToDuration>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration round(const duration<_Rep, _Period>& __d) {
+  _ToDuration __lower = chrono::floor<_ToDuration>(__d);
+  _ToDuration __upper = __lower + _ToDuration{1};
+  auto __lower_diff   = __d - __lower;
+  auto __upper_diff   = __upper - __d;
+  if (__lower_diff < __upper_diff)
+    return __lower;
+  if (__lower_diff > __upper_diff)
+    return __upper;
+  return __lower.count() & 1 ? __upper : __lower;
+}
+#endif
+
+// duration
+
+template <class _Rep, class _Period>
+class _LIBCPP_TEMPLATE_VIS duration {
+  static_assert(!__is_duration<_Rep>::value, "A duration representation can not be a duration");
+  static_assert(__is_ratio<_Period>::value, "Second template parameter of duration must be a std::ratio");
+  static_assert(_Period::num > 0, "duration period must be positive");
+
+  template <class _R1, class _R2>
+  struct __no_overflow {
+  private:
+    static const intmax_t __gcd_n1_n2 = __static_gcd<_R1::num, _R2::num>::value;
+    static const intmax_t __gcd_d1_d2 = __static_gcd<_R1::den, _R2::den>::value;
+    static const intmax_t __n1        = _R1::num / __gcd_n1_n2;
+    static const intmax_t __d1        = _R1::den / __gcd_d1_d2;
+    static const intmax_t __n2        = _R2::num / __gcd_n1_n2;
+    static const intmax_t __d2        = _R2::den / __gcd_d1_d2;
+    static const intmax_t max         = -((intmax_t(1) << (sizeof(intmax_t) * CHAR_BIT - 1)) + 1);
+
+    template <intmax_t _Xp, intmax_t _Yp, bool __overflow>
+    struct __mul // __overflow == false
+    {
+      static const intmax_t value = _Xp * _Yp;
+    };
+
+    template <intmax_t _Xp, intmax_t _Yp>
+    struct __mul<_Xp, _Yp, true> {
+      static const intmax_t value = 1;
+    };
+
+  public:
+    static const bool value = (__n1 <= max / __d2) && (__n2 <= max / __d1);
+    typedef ratio<__mul<__n1, __d2, !value>::value, __mul<__n2, __d1, !value>::value> type;
+  };
+
+public:
+  typedef _Rep rep;
+  typedef typename _Period::type period;
+
+private:
+  rep __rep_;
+
+public:
+#ifndef _LIBCPP_CXX03_LANG
+  constexpr duration() = default;
+#else
+  _LIBCPP_HIDE_FROM_ABI duration() {}
+#endif
+
+  template <class _Rep2,
+            __enable_if_t<is_convertible<const _Rep2&, rep>::value &&
+                              (treat_as_floating_point<rep>::value || !treat_as_floating_point<_Rep2>::value),
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit duration(const _Rep2& __r) : __rep_(__r) {}
+
+  // conversions
+  template <class _Rep2,
+            class _Period2,
+            __enable_if_t<__no_overflow<_Period2, period>::value && (treat_as_floating_point<rep>::value ||
+                                                                     (__no_overflow<_Period2, period>::type::den == 1 &&
+                                                                      !treat_as_floating_point<_Rep2>::value)),
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR duration(const duration<_Rep2, _Period2>& __d)
+      : __rep_(chrono::duration_cast<duration>(__d).count()) {}
+
+  // observer
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR rep count() const { return __rep_; }
+
+  // arithmetic
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename common_type<duration>::type operator+() const {
+    return typename common_type<duration>::type(*this);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename common_type<duration>::type operator-() const {
+    return typename common_type<duration>::type(-__rep_);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator++() {
+    ++__rep_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration operator++(int) { return duration(__rep_++); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator--() {
+    --__rep_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration operator--(int) { return duration(__rep_--); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator+=(const duration& __d) {
+    __rep_ += __d.count();
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator-=(const duration& __d) {
+    __rep_ -= __d.count();
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator*=(const rep& __rhs) {
+    __rep_ *= __rhs;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator/=(const rep& __rhs) {
+    __rep_ /= __rhs;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator%=(const rep& __rhs) {
+    __rep_ %= __rhs;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator%=(const duration& __rhs) {
+    __rep_ %= __rhs.count();
+    return *this;
+  }
+
+  // special values
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR duration zero() _NOEXCEPT {
+    return duration(duration_values<rep>::zero());
+  }
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR duration min() _NOEXCEPT {
+    return duration(duration_values<rep>::min());
+  }
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR duration max() _NOEXCEPT {
+    return duration(duration_values<rep>::max());
+  }
+};
+
+typedef duration<long long, nano> nanoseconds;
+typedef duration<long long, micro> microseconds;
+typedef duration<long long, milli> milliseconds;
+typedef duration<long long > seconds;
+typedef duration< long, ratio< 60> > minutes;
+typedef duration< long, ratio<3600> > hours;
+#if _LIBCPP_STD_VER >= 20
+typedef duration< int, ratio_multiply<ratio<24>, hours::period>> days;
+typedef duration< int, ratio_multiply<ratio<7>, days::period>> weeks;
+typedef duration< int, ratio_multiply<ratio<146097, 400>, days::period>> years;
+typedef duration< int, ratio_divide<years::period, ratio<12>>> months;
+#endif
+// Duration ==
+
+template <class _LhsDuration, class _RhsDuration>
+struct __duration_eq {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator()(const _LhsDuration& __lhs, const _RhsDuration& __rhs) const {
+    typedef typename common_type<_LhsDuration, _RhsDuration>::type _Ct;
+    return _Ct(__lhs).count() == _Ct(__rhs).count();
+  }
+};
+
+template <class _LhsDuration>
+struct __duration_eq<_LhsDuration, _LhsDuration> {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator()(const _LhsDuration& __lhs, const _LhsDuration& __rhs) const {
+    return __lhs.count() == __rhs.count();
+  }
+};
+
+template <class _Rep1, class _Period1, class _Rep2, class _Period2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator==(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
+  return __duration_eq<duration<_Rep1, _Period1>, duration<_Rep2, _Period2> >()(__lhs, __rhs);
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+// Duration !=
+
+template <class _Rep1, class _Period1, class _Rep2, class _Period2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator!=(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
+  return !(__lhs == __rhs);
+}
+
+#endif // _LIBCPP_STD_VER <= 17
+
+// Duration <
+
+template <class _LhsDuration, class _RhsDuration>
+struct __duration_lt {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator()(const _LhsDuration& __lhs, const _RhsDuration& __rhs) const {
+    typedef typename common_type<_LhsDuration, _RhsDuration>::type _Ct;
+    return _Ct(__lhs).count() < _Ct(__rhs).count();
+  }
+};
+
+template <class _LhsDuration>
+struct __duration_lt<_LhsDuration, _LhsDuration> {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator()(const _LhsDuration& __lhs, const _LhsDuration& __rhs) const {
+    return __lhs.count() < __rhs.count();
+  }
+};
+
+template <class _Rep1, class _Period1, class _Rep2, class _Period2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator<(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
+  return __duration_lt<duration<_Rep1, _Period1>, duration<_Rep2, _Period2> >()(__lhs, __rhs);
+}
+
+// Duration >
+
+template <class _Rep1, class _Period1, class _Rep2, class _Period2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator>(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
+  return __rhs < __lhs;
+}
+
+// Duration <=
+
+template <class _Rep1, class _Period1, class _Rep2, class _Period2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator<=(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
+  return !(__rhs < __lhs);
+}
+
+// Duration >=
+
+template <class _Rep1, class _Period1, class _Rep2, class _Period2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator>=(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
+  return !(__lhs < __rhs);
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Rep1, class _Period1, class _Rep2, class _Period2>
+  requires three_way_comparable<common_type_t<_Rep1, _Rep2>>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+operator<=>(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
+  using _Ct = common_type_t<duration<_Rep1, _Period1>, duration<_Rep2, _Period2>>;
+  return _Ct(__lhs).count() <=> _Ct(__rhs).count();
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+// Duration +
+
+template <class _Rep1, class _Period1, class _Rep2, class _Period2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
+typename common_type<duration<_Rep1, _Period1>, duration<_Rep2, _Period2> >::type
+operator+(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
+  typedef typename common_type<duration<_Rep1, _Period1>, duration<_Rep2, _Period2> >::type _Cd;
+  return _Cd(_Cd(__lhs).count() + _Cd(__rhs).count());
+}
+
+// Duration -
+
+template <class _Rep1, class _Period1, class _Rep2, class _Period2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
+typename common_type<duration<_Rep1, _Period1>, duration<_Rep2, _Period2> >::type
+operator-(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
+  typedef typename common_type<duration<_Rep1, _Period1>, duration<_Rep2, _Period2> >::type _Cd;
+  return _Cd(_Cd(__lhs).count() - _Cd(__rhs).count());
+}
+
+// Duration *
+
+template <class _Rep1,
+          class _Period,
+          class _Rep2,
+          __enable_if_t<is_convertible<const _Rep2&, typename common_type<_Rep1, _Rep2>::type>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR duration<typename common_type<_Rep1, _Rep2>::type, _Period>
+operator*(const duration<_Rep1, _Period>& __d, const _Rep2& __s) {
+  typedef typename common_type<_Rep1, _Rep2>::type _Cr;
+  typedef duration<_Cr, _Period> _Cd;
+  return _Cd(_Cd(__d).count() * static_cast<_Cr>(__s));
+}
+
+template <class _Rep1,
+          class _Period,
+          class _Rep2,
+          __enable_if_t<is_convertible<const _Rep1&, typename common_type<_Rep1, _Rep2>::type>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR duration<typename common_type<_Rep1, _Rep2>::type, _Period>
+operator*(const _Rep1& __s, const duration<_Rep2, _Period>& __d) {
+  return __d * __s;
+}
+
+// Duration /
+
+template <class _Rep1,
+          class _Period,
+          class _Rep2,
+          __enable_if_t<!__is_duration<_Rep2>::value &&
+                            is_convertible<const _Rep2&, typename common_type<_Rep1, _Rep2>::type>::value,
+                        int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR duration<typename common_type<_Rep1, _Rep2>::type, _Period>
+operator/(const duration<_Rep1, _Period>& __d, const _Rep2& __s) {
+  typedef typename common_type<_Rep1, _Rep2>::type _Cr;
+  typedef duration<_Cr, _Period> _Cd;
+  return _Cd(_Cd(__d).count() / static_cast<_Cr>(__s));
+}
+
+template <class _Rep1, class _Period1, class _Rep2, class _Period2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename common_type<_Rep1, _Rep2>::type
+operator/(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
+  typedef typename common_type<duration<_Rep1, _Period1>, duration<_Rep2, _Period2> >::type _Ct;
+  return _Ct(__lhs).count() / _Ct(__rhs).count();
+}
+
+// Duration %
+
+template <class _Rep1,
+          class _Period,
+          class _Rep2,
+          __enable_if_t<!__is_duration<_Rep2>::value &&
+                            is_convertible<const _Rep2&, typename common_type<_Rep1, _Rep2>::type>::value,
+                        int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR duration<typename common_type<_Rep1, _Rep2>::type, _Period>
+operator%(const duration<_Rep1, _Period>& __d, const _Rep2& __s) {
+  typedef typename common_type<_Rep1, _Rep2>::type _Cr;
+  typedef duration<_Cr, _Period> _Cd;
+  return _Cd(_Cd(__d).count() % static_cast<_Cr>(__s));
+}
+
+template <class _Rep1, class _Period1, class _Rep2, class _Period2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
+typename common_type<duration<_Rep1, _Period1>, duration<_Rep2, _Period2> >::type
+operator%(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
+  typedef typename common_type<_Rep1, _Rep2>::type _Cr;
+  typedef typename common_type<duration<_Rep1, _Period1>, duration<_Rep2, _Period2> >::type _Cd;
+  return _Cd(static_cast<_Cr>(_Cd(__lhs).count()) % static_cast<_Cr>(_Cd(__rhs).count()));
+}
+
+} // namespace chrono
+
+#if _LIBCPP_STD_VER >= 14
+// Suffixes for duration literals [time.duration.literals]
+inline namespace literals {
+inline namespace chrono_literals {
+
+_LIBCPP_HIDE_FROM_ABI constexpr chrono::hours operator""h(unsigned long long __h) {
+  return chrono::hours(static_cast<chrono::hours::rep>(__h));
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double, ratio<3600, 1>> operator""h(long double __h) {
+  return chrono::duration<long double, ratio<3600, 1>>(__h);
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr chrono::minutes operator""min(unsigned long long __m) {
+  return chrono::minutes(static_cast<chrono::minutes::rep>(__m));
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double, ratio<60, 1>> operator""min(long double __m) {
+  return chrono::duration<long double, ratio<60, 1>>(__m);
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr chrono::seconds operator""s(unsigned long long __s) {
+  return chrono::seconds(static_cast<chrono::seconds::rep>(__s));
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double> operator""s(long double __s) {
+  return chrono::duration<long double>(__s);
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr chrono::milliseconds operator""ms(unsigned long long __ms) {
+  return chrono::milliseconds(static_cast<chrono::milliseconds::rep>(__ms));
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double, milli> operator""ms(long double __ms) {
+  return chrono::duration<long double, milli>(__ms);
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr chrono::microseconds operator""us(unsigned long long __us) {
+  return chrono::microseconds(static_cast<chrono::microseconds::rep>(__us));
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double, micro> operator""us(long double __us) {
+  return chrono::duration<long double, micro>(__us);
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr chrono::nanoseconds operator""ns(unsigned long long __ns) {
+  return chrono::nanoseconds(static_cast<chrono::nanoseconds::rep>(__ns));
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double, nano> operator""ns(long double __ns) {
+  return chrono::duration<long double, nano>(__ns);
+}
+
+} // namespace chrono_literals
+} // namespace literals
+
+namespace chrono { // hoist the literals into namespace std::chrono
+using namespace literals::chrono_literals;
+} // namespace chrono
+
+#endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP___CHRONO_DURATION_H
diff --git a/libcxx/include/__cxx03/__chrono/exception.h b/libcxx/include/__cxx03/__chrono/exception.h
new file mode 100644
index 000000000000..266f8fac4417
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/exception.h
@@ -0,0 +1,135 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
+
+#ifndef _LIBCPP___CHRONO_EXCEPTION_H
+#define _LIBCPP___CHRONO_EXCEPTION_H
+
+#include <version>
+// Enable the contents of the header only when libc++ was built with experimental features enabled.
+#if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#  include <__chrono/calendar.h>
+#  include <__chrono/local_info.h>
+#  include <__chrono/time_point.h>
+#  include <__config>
+#  include <__configuration/availability.h>
+#  include <__verbose_abort>
+#  include <format>
+#  include <stdexcept>
+#  include <string>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  if _LIBCPP_STD_VER >= 20
+
+namespace chrono {
+
+class nonexistent_local_time : public runtime_error {
+public:
+  template <class _Duration>
+  _LIBCPP_HIDE_FROM_ABI nonexistent_local_time(const local_time<_Duration>& __time, const local_info& __info)
+      : runtime_error{__create_message(__time, __info)} {
+    // [time.zone.exception.nonexist]/2
+    //   Preconditions: i.result == local_info::nonexistent is true.
+    // The value of __info.result is not used.
+    _LIBCPP_ASSERT_PEDANTIC(__info.result == local_info::nonexistent,
+                            "creating an nonexistent_local_time from a local_info that is not non-existent");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI nonexistent_local_time(const nonexistent_local_time&)            = default;
+  _LIBCPP_HIDE_FROM_ABI nonexistent_local_time& operator=(const nonexistent_local_time&) = default;
+
+  _LIBCPP_AVAILABILITY_TZDB _LIBCPP_EXPORTED_FROM_ABI ~nonexistent_local_time() override; // exported as key function
+
+private:
+  template <class _Duration>
+  _LIBCPP_HIDE_FROM_ABI string __create_message(const local_time<_Duration>& __time, const local_info& __info) {
+    return std::format(
+        R"({} is in a gap between
+{} {} and
+{} {} which are both equivalent to
+{} UTC)",
+        __time,
+        local_seconds{__info.first.end.time_since_epoch()} + __info.first.offset,
+        __info.first.abbrev,
+        local_seconds{__info.second.begin.time_since_epoch()} + __info.second.offset,
+        __info.second.abbrev,
+        __info.first.end);
+  }
+};
+
+template <class _Duration>
+_LIBCPP_NORETURN _LIBCPP_AVAILABILITY_TZDB _LIBCPP_HIDE_FROM_ABI void __throw_nonexistent_local_time(
+    [[maybe_unused]] const local_time<_Duration>& __time, [[maybe_unused]] const local_info& __info) {
+#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw nonexistent_local_time(__time, __info);
+#    else
+  _LIBCPP_VERBOSE_ABORT("nonexistent_local_time was thrown in -fno-exceptions mode");
+#    endif
+}
+
+class ambiguous_local_time : public runtime_error {
+public:
+  template <class _Duration>
+  _LIBCPP_HIDE_FROM_ABI ambiguous_local_time(const local_time<_Duration>& __time, const local_info& __info)
+      : runtime_error{__create_message(__time, __info)} {
+    // [time.zone.exception.ambig]/2
+    //   Preconditions: i.result == local_info::ambiguous is true.
+    // The value of __info.result is not used.
+    _LIBCPP_ASSERT_PEDANTIC(__info.result == local_info::ambiguous,
+                            "creating an ambiguous_local_time from a local_info that is not ambiguous");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ambiguous_local_time(const ambiguous_local_time&)            = default;
+  _LIBCPP_HIDE_FROM_ABI ambiguous_local_time& operator=(const ambiguous_local_time&) = default;
+
+  _LIBCPP_AVAILABILITY_TZDB _LIBCPP_EXPORTED_FROM_ABI ~ambiguous_local_time() override; // exported as key function
+
+private:
+  template <class _Duration>
+  _LIBCPP_HIDE_FROM_ABI string __create_message(const local_time<_Duration>& __time, const local_info& __info) {
+    return std::format(
+        // There are two spaces after the full-stop; this has been verified
+        // in the sources of the Standard.
+        R"({0} is ambiguous.  It could be
+{0} {1} == {2} UTC or
+{0} {3} == {4} UTC)",
+        __time,
+        __info.first.abbrev,
+        __time - __info.first.offset,
+        __info.second.abbrev,
+        __time - __info.second.offset);
+  }
+};
+
+template <class _Duration>
+_LIBCPP_NORETURN _LIBCPP_AVAILABILITY_TZDB _LIBCPP_HIDE_FROM_ABI void __throw_ambiguous_local_time(
+    [[maybe_unused]] const local_time<_Duration>& __time, [[maybe_unused]] const local_info& __info) {
+#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw ambiguous_local_time(__time, __info);
+#    else
+  _LIBCPP_VERBOSE_ABORT("ambiguous_local_time was thrown in -fno-exceptions mode");
+#    endif
+}
+
+} // namespace chrono
+
+#  endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#endif // _LIBCPP___CHRONO_EXCEPTION_H
diff --git a/libcxx/include/__cxx03/__chrono/file_clock.h b/libcxx/include/__cxx03/__chrono/file_clock.h
new file mode 100644
index 000000000000..4dd3f88ce5ba
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/file_clock.h
@@ -0,0 +1,80 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_FILE_CLOCK_H
+#define _LIBCPP___CHRONO_FILE_CLOCK_H
+
+#include <__chrono/duration.h>
+#include <__chrono/system_clock.h>
+#include <__chrono/time_point.h>
+#include <__config>
+#include <ratio>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+struct _FilesystemClock;
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+#endif // !_LIBCPP_CXX03_LANG
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+// [time.clock.file], type file_clock
+using file_clock = filesystem::_FilesystemClock;
+
+template <class _Duration>
+using file_time = time_point<file_clock, _Duration>;
+
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#ifndef _LIBCPP_CXX03_LANG
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+struct _FilesystemClock {
+#  if !defined(_LIBCPP_HAS_NO_INT128)
+  typedef __int128_t rep;
+  typedef nano period;
+#  else
+  typedef long long rep;
+  typedef nano period;
+#  endif
+
+  typedef chrono::duration<rep, period> duration;
+  typedef chrono::time_point<_FilesystemClock> time_point;
+
+  _LIBCPP_EXPORTED_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 const bool is_steady = false;
+
+  _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_EXPORTED_FROM_ABI static time_point now() noexcept;
+
+#  if _LIBCPP_STD_VER >= 20
+  template <class _Duration>
+  _LIBCPP_HIDE_FROM_ABI static chrono::sys_time<_Duration> to_sys(const chrono::file_time<_Duration>& __t) {
+    return chrono::sys_time<_Duration>(__t.time_since_epoch());
+  }
+
+  template <class _Duration>
+  _LIBCPP_HIDE_FROM_ABI static chrono::file_time<_Duration> from_sys(const chrono::sys_time<_Duration>& __t) {
+    return chrono::file_time<_Duration>(__t.time_since_epoch());
+  }
+#  endif // _LIBCPP_STD_VER >= 20
+};
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+#endif // !_LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___CHRONO_FILE_CLOCK_H
diff --git a/libcxx/include/__cxx03/__chrono/formatter.h b/libcxx/include/__cxx03/__chrono/formatter.h
new file mode 100644
index 000000000000..449c415e9576
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/formatter.h
@@ -0,0 +1,990 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_FORMATTER_H
+#define _LIBCPP___CHRONO_FORMATTER_H
+
+#include <__algorithm/ranges_copy.h>
+#include <__chrono/calendar.h>
+#include <__chrono/concepts.h>
+#include <__chrono/convert_to_tm.h>
+#include <__chrono/day.h>
+#include <__chrono/duration.h>
+#include <__chrono/file_clock.h>
+#include <__chrono/hh_mm_ss.h>
+#include <__chrono/local_info.h>
+#include <__chrono/month.h>
+#include <__chrono/month_weekday.h>
+#include <__chrono/monthday.h>
+#include <__chrono/ostream.h>
+#include <__chrono/parser_std_format_spec.h>
+#include <__chrono/statically_widen.h>
+#include <__chrono/sys_info.h>
+#include <__chrono/system_clock.h>
+#include <__chrono/time_point.h>
+#include <__chrono/weekday.h>
+#include <__chrono/year.h>
+#include <__chrono/year_month.h>
+#include <__chrono/year_month_day.h>
+#include <__chrono/year_month_weekday.h>
+#include <__chrono/zoned_time.h>
+#include <__concepts/arithmetic.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/concepts.h>
+#include <__format/format_error.h>
+#include <__format/format_functions.h>
+#include <__format/format_parse_context.h>
+#include <__format/formatter.h>
+#include <__format/parser_std_format_spec.h>
+#include <__format/write_escaped.h>
+#include <__memory/addressof.h>
+#include <__type_traits/is_specialization.h>
+#include <cmath>
+#include <ctime>
+#include <limits>
+#include <sstream>
+#include <string_view>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __formatter {
+
+/// Formats a time based on a tm struct.
+///
+/// This formatter passes the formatting to time_put which uses strftime. When
+/// the value is outside the valid range it's unspecified what strftime will
+/// output. For example weekday 8 can print 1 when the day is processed modulo
+/// 7 since that handles the Sunday for 0-based weekday. It can also print 8 if
+/// 7 is handled as a special case.
+///
+/// The Standard doesn't specify what to do in this case so the result depends
+/// on the result of the underlying code.
+///
+/// \pre When the (abbreviated) weekday or month name are used, the caller
+///      validates whether the value is valid. So the caller handles that
+///      requirement of Table 97: Meaning of conversion specifiers
+///      [tab:time.format.spec].
+///
+/// When no chrono-specs are provided it uses the stream formatter.
+
+// For tiny ratios it's not possible to convert a duration to a hh_mm_ss. This
+// fails compile-time due to the limited precision of the ratio (64-bit is too
+// small). Therefore a duration uses its own conversion.
+template <class _CharT, class _Rep, class _Period>
+_LIBCPP_HIDE_FROM_ABI void
+__format_sub_seconds(basic_stringstream<_CharT>& __sstr, const chrono::duration<_Rep, _Period>& __value) {
+  __sstr << std::use_facet<numpunct<_CharT>>(__sstr.getloc()).decimal_point();
+
+  using __duration = chrono::duration<_Rep, _Period>;
+
+  auto __fraction = __value - chrono::duration_cast<chrono::seconds>(__value);
+  // Converts a negative fraction to its positive value.
+  if (__value < chrono::seconds{0} && __fraction != __duration{0})
+    __fraction += chrono::seconds{1};
+  if constexpr (chrono::treat_as_floating_point_v<_Rep>)
+    // When the floating-point value has digits itself they are ignored based
+    // on the wording in [tab:time.format.spec]
+    //   If the precision of the input cannot be exactly represented with
+    //   seconds, then the format is a decimal floating-point number with a
+    //   fixed format and a precision matching that of the precision of the
+    //   input (or to a microseconds precision if the conversion to
+    //   floating-point decimal seconds cannot be made within 18 fractional
+    //   digits).
+    //
+    // This matches the behaviour of MSVC STL, fmtlib interprets this
+    // differently and uses 3 decimals.
+    // https://godbolt.org/z/6dsbnW8ba
+    std::format_to(std::ostreambuf_iterator<_CharT>{__sstr},
+                   _LIBCPP_STATICALLY_WIDEN(_CharT, "{:0{}.0f}"),
+                   chrono::duration_cast<typename chrono::hh_mm_ss<__duration>::precision>(__fraction).count(),
+                   chrono::hh_mm_ss<__duration>::fractional_width);
+  else
+    std::format_to(std::ostreambuf_iterator<_CharT>{__sstr},
+                   _LIBCPP_STATICALLY_WIDEN(_CharT, "{:0{}}"),
+                   chrono::duration_cast<typename chrono::hh_mm_ss<__duration>::precision>(__fraction).count(),
+                   chrono::hh_mm_ss<__duration>::fractional_width);
+}
+
+template <class _CharT, __is_time_point _Tp>
+_LIBCPP_HIDE_FROM_ABI void __format_sub_seconds(basic_stringstream<_CharT>& __sstr, const _Tp& __value) {
+  __formatter::__format_sub_seconds(__sstr, __value.time_since_epoch());
+}
+
+template <class _CharT, class _Duration>
+_LIBCPP_HIDE_FROM_ABI void
+__format_sub_seconds(basic_stringstream<_CharT>& __sstr, const chrono::hh_mm_ss<_Duration>& __value) {
+  __sstr << std::use_facet<numpunct<_CharT>>(__sstr.getloc()).decimal_point();
+  if constexpr (chrono::treat_as_floating_point_v<typename _Duration::rep>)
+    std::format_to(std::ostreambuf_iterator<_CharT>{__sstr},
+                   _LIBCPP_STATICALLY_WIDEN(_CharT, "{:0{}.0f}"),
+                   __value.subseconds().count(),
+                   __value.fractional_width);
+  else
+    std::format_to(std::ostreambuf_iterator<_CharT>{__sstr},
+                   _LIBCPP_STATICALLY_WIDEN(_CharT, "{:0{}}"),
+                   __value.subseconds().count(),
+                   __value.fractional_width);
+}
+
+#  if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB) && !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) &&                     \
+      !defined(_LIBCPP_HAS_NO_FILESYSTEM) && !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+template <class _CharT, class _Duration, class _TimeZonePtr>
+_LIBCPP_HIDE_FROM_ABI void
+__format_sub_seconds(basic_stringstream<_CharT>& __sstr, const chrono::zoned_time<_Duration, _TimeZonePtr>& __value) {
+  __formatter::__format_sub_seconds(__sstr, __value.get_local_time().time_since_epoch());
+}
+#  endif
+
+template <class _Tp>
+consteval bool __use_fraction() {
+  if constexpr (__is_time_point<_Tp>)
+    return chrono::hh_mm_ss<typename _Tp::duration>::fractional_width;
+#  if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB) && !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) &&                     \
+      !defined(_LIBCPP_HAS_NO_FILESYSTEM) && !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  else if constexpr (__is_specialization_v<_Tp, chrono::zoned_time>)
+    return chrono::hh_mm_ss<typename _Tp::duration>::fractional_width;
+#  endif
+  else if constexpr (chrono::__is_duration<_Tp>::value)
+    return chrono::hh_mm_ss<_Tp>::fractional_width;
+  else if constexpr (__is_hh_mm_ss<_Tp>)
+    return _Tp::fractional_width;
+  else
+    return false;
+}
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI void __format_year(basic_stringstream<_CharT>& __sstr, int __year) {
+  if (__year < 0) {
+    __sstr << _CharT('-');
+    __year = -__year;
+  }
+
+  // TODO FMT Write an issue
+  //   If the result has less than four digits it is zero-padded with 0 to two digits.
+  // is less -> has less
+  // left-padded -> zero-padded, otherwise the proper value would be 000-0.
+
+  // Note according to the wording it should be left padded, which is odd.
+  __sstr << std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "{:04}"), __year);
+}
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI void __format_century(basic_stringstream<_CharT>& __sstr, int __year) {
+  // TODO FMT Write an issue
+  // [tab:time.format.spec]
+  //   %C The year divided by 100 using floored division. If the result is a
+  //   single decimal digit, it is prefixed with 0.
+
+  bool __negative = __year < 0;
+  int __century   = (__year - (99 * __negative)) / 100; // floored division
+  __sstr << std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "{:02}"), __century);
+}
+
+// Implements the %z format specifier according to [tab:time.format.spec], where
+// '__modifier' signals %Oz or %Ez were used. (Both modifiers behave the same,
+// so there is no need to distinguish between them.)
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI void
+__format_zone_offset(basic_stringstream<_CharT>& __sstr, chrono::seconds __offset, bool __modifier) {
+  if (__offset < 0s) {
+    __sstr << _CharT('-');
+    __offset = -__offset;
+  } else {
+    __sstr << _CharT('+');
+  }
+
+  chrono::hh_mm_ss __hms{__offset};
+  std::ostreambuf_iterator<_CharT> __out_it{__sstr};
+  // Note HMS does not allow formatting hours > 23, but the offset is not limited to 24H.
+  std::format_to(__out_it, _LIBCPP_STATICALLY_WIDEN(_CharT, "{:02}"), __hms.hours().count());
+  if (__modifier)
+    __sstr << _CharT(':');
+  std::format_to(__out_it, _LIBCPP_STATICALLY_WIDEN(_CharT, "{:02}"), __hms.minutes().count());
+}
+
+// Helper to store the time zone information needed for formatting.
+struct _LIBCPP_HIDE_FROM_ABI __time_zone {
+  // Typically these abbreviations are short and fit in the string's internal
+  // buffer.
+  string __abbrev;
+  chrono::seconds __offset;
+};
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI __time_zone __convert_to_time_zone([[maybe_unused]] const _Tp& __value) {
+#  if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+  if constexpr (same_as<_Tp, chrono::sys_info>)
+    return {__value.abbrev, __value.offset};
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  else if constexpr (__is_specialization_v<_Tp, chrono::zoned_time>)
+    return __formatter::__convert_to_time_zone(__value.get_info());
+#    endif
+  else
+#  endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+    return {"UTC", chrono::seconds{0}};
+}
+
+template <class _CharT, class _Tp>
+_LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs(
+    basic_stringstream<_CharT>& __sstr, const _Tp& __value, basic_string_view<_CharT> __chrono_specs) {
+  tm __t              = std::__convert_to_tm<tm>(__value);
+  __time_zone __z     = __formatter::__convert_to_time_zone(__value);
+  const auto& __facet = std::use_facet<time_put<_CharT>>(__sstr.getloc());
+  for (auto __it = __chrono_specs.begin(); __it != __chrono_specs.end(); ++__it) {
+    if (*__it == _CharT('%')) {
+      auto __s = __it;
+      ++__it;
+      // We only handle the types that can't be directly handled by time_put.
+      // (as an optimization n, t, and % are also handled directly.)
+      switch (*__it) {
+      case _CharT('n'):
+        __sstr << _CharT('\n');
+        break;
+      case _CharT('t'):
+        __sstr << _CharT('\t');
+        break;
+      case _CharT('%'):
+        __sstr << *__it;
+        break;
+
+      case _CharT('C'): {
+        // strftime's output is only defined in the range [00, 99].
+        int __year = __t.tm_year + 1900;
+        if (__year < 1000 || __year > 9999)
+          __formatter::__format_century(__sstr, __year);
+        else
+          __facet.put(
+              {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1));
+      } break;
+
+      case _CharT('j'):
+        if constexpr (chrono::__is_duration<_Tp>::value)
+          // Converting a duration where the period has a small ratio to days
+          // may fail to compile. This due to loss of precision in the
+          // conversion. In order to avoid that issue convert to seconds as
+          // an intemediate step.
+          __sstr << chrono::duration_cast<chrono::days>(chrono::duration_cast<chrono::seconds>(__value)).count();
+        else
+          __facet.put(
+              {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1));
+        break;
+
+      case _CharT('q'):
+        if constexpr (chrono::__is_duration<_Tp>::value) {
+          __sstr << chrono::__units_suffix<_CharT, typename _Tp::period>();
+          break;
+        }
+        __builtin_unreachable();
+
+      case _CharT('Q'):
+        // TODO FMT Determine the proper ideas
+        // - Should it honour the precision?
+        // - Shoult it honour the locale setting for the separators?
+        // The wording for Q doesn't use the word locale and the effect of
+        // precision is unspecified.
+        //
+        // MSVC STL ignores precision but uses separator
+        // FMT honours precision and has a bug for separator
+        // https://godbolt.org/z/78b7sMxns
+        if constexpr (chrono::__is_duration<_Tp>::value) {
+          __sstr << std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "{}"), __value.count());
+          break;
+        }
+        __builtin_unreachable();
+
+      case _CharT('S'):
+      case _CharT('T'):
+        __facet.put(
+            {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1));
+        if constexpr (__use_fraction<_Tp>())
+          __formatter::__format_sub_seconds(__sstr, __value);
+        break;
+
+        // Unlike time_put and strftime the formatting library requires %Y
+        //
+        // [tab:time.format.spec]
+        //   The year as a decimal number. If the result is less than four digits
+        //   it is left-padded with 0 to four digits.
+        //
+        // This means years in the range (-1000, 1000) need manual formatting.
+        // It's unclear whether %EY needs the same treatment. For example the
+        // Japanese EY contains the era name and year. This is zero-padded to 2
+        // digits in time_put (note that older glibc versions didn't do
+        // padding.) However most eras won't reach 100 years, let alone 1000.
+        // So padding to 4 digits seems unwanted for Japanese.
+        //
+        // The same applies to %Ex since that too depends on the era.
+        //
+        // %x the locale's date representation is currently doesn't handle the
+        // zero-padding too.
+        //
+        // The 4 digits can be implemented better at a later time. On POSIX
+        // systems the required information can be extracted by nl_langinfo
+        // https://man7.org/linux/man-pages/man3/nl_langinfo.3.html
+        //
+        // Note since year < -1000 is expected to be rare it uses the more
+        // expensive year routine.
+        //
+        // TODO FMT evaluate the comment above.
+
+#  if defined(__GLIBC__) || defined(_AIX) || defined(_WIN32)
+      case _CharT('y'):
+        // Glibc fails for negative values, AIX for positive values too.
+        __sstr << std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "{:02}"), (std::abs(__t.tm_year + 1900)) % 100);
+        break;
+#  endif // defined(__GLIBC__) || defined(_AIX) || defined(_WIN32)
+
+      case _CharT('Y'):
+        // Depending on the platform's libc the range of supported years is
+        // limited. Intead of of testing all conditions use the internal
+        // implementation unconditionally.
+        __formatter::__format_year(__sstr, __t.tm_year + 1900);
+        break;
+
+      case _CharT('F'):
+        // Depending on the platform's libc the range of supported years is
+        // limited. Instead of testing all conditions use the internal
+        // implementation unconditionally.
+        __formatter::__format_year(__sstr, __t.tm_year + 1900);
+        __sstr << std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "-{:02}-{:02}"), __t.tm_mon + 1, __t.tm_mday);
+        break;
+
+      case _CharT('z'):
+        __formatter::__format_zone_offset(__sstr, __z.__offset, false);
+        break;
+
+      case _CharT('Z'):
+        // __abbrev is always a char so the copy may convert.
+        ranges::copy(__z.__abbrev, std::ostreambuf_iterator<_CharT>{__sstr});
+        break;
+
+      case _CharT('O'):
+        if constexpr (__use_fraction<_Tp>()) {
+          // Handle OS using the normal representation for the non-fractional
+          // part. There seems to be no locale information regarding how the
+          // fractional part should be formatted.
+          if (*(__it + 1) == 'S') {
+            ++__it;
+            __facet.put(
+                {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1));
+            __formatter::__format_sub_seconds(__sstr, __value);
+            break;
+          }
+        }
+
+        // Oz produces the same output as Ez below.
+        [[fallthrough]];
+      case _CharT('E'):
+        ++__it;
+        if (*__it == 'z') {
+          __formatter::__format_zone_offset(__sstr, __z.__offset, true);
+          break;
+        }
+        [[fallthrough]];
+      default:
+        __facet.put(
+            {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1));
+        break;
+      }
+    } else {
+      __sstr << *__it;
+    }
+  }
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool __weekday_ok(const _Tp& __value) {
+  if constexpr (__is_time_point<_Tp>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::day>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::month>)
+    return __value.ok();
+  else if constexpr (same_as<_Tp, chrono::year>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::weekday>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::weekday_indexed>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::weekday_last>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::month_day>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::month_day_last>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::month_weekday>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::month_weekday_last>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::year_month>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::year_month_day>)
+    return __value.ok();
+  else if constexpr (same_as<_Tp, chrono::year_month_day_last>)
+    return __value.ok();
+  else if constexpr (same_as<_Tp, chrono::year_month_weekday>)
+    return __value.weekday().ok();
+  else if constexpr (same_as<_Tp, chrono::year_month_weekday_last>)
+    return __value.weekday().ok();
+  else if constexpr (__is_hh_mm_ss<_Tp>)
+    return true;
+#  if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+  else if constexpr (same_as<_Tp, chrono::sys_info>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::local_info>)
+    return true;
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  else if constexpr (__is_specialization_v<_Tp, chrono::zoned_time>)
+    return true;
+#    endif
+#  endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+  else
+    static_assert(sizeof(_Tp) == 0, "Add the missing type specialization");
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool __weekday_name_ok(const _Tp& __value) {
+  if constexpr (__is_time_point<_Tp>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::day>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::month>)
+    return __value.ok();
+  else if constexpr (same_as<_Tp, chrono::year>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::weekday>)
+    return __value.ok();
+  else if constexpr (same_as<_Tp, chrono::weekday_indexed>)
+    return __value.weekday().ok();
+  else if constexpr (same_as<_Tp, chrono::weekday_last>)
+    return __value.weekday().ok();
+  else if constexpr (same_as<_Tp, chrono::month_day>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::month_day_last>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::month_weekday>)
+    return __value.weekday_indexed().ok();
+  else if constexpr (same_as<_Tp, chrono::month_weekday_last>)
+    return __value.weekday_indexed().ok();
+  else if constexpr (same_as<_Tp, chrono::year_month>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::year_month_day>)
+    return __value.ok();
+  else if constexpr (same_as<_Tp, chrono::year_month_day_last>)
+    return __value.ok();
+  else if constexpr (same_as<_Tp, chrono::year_month_weekday>)
+    return __value.weekday().ok();
+  else if constexpr (same_as<_Tp, chrono::year_month_weekday_last>)
+    return __value.weekday().ok();
+  else if constexpr (__is_hh_mm_ss<_Tp>)
+    return true;
+#  if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+  else if constexpr (same_as<_Tp, chrono::sys_info>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::local_info>)
+    return true;
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  else if constexpr (__is_specialization_v<_Tp, chrono::zoned_time>)
+    return true;
+#    endif
+#  endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+  else
+    static_assert(sizeof(_Tp) == 0, "Add the missing type specialization");
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool __date_ok(const _Tp& __value) {
+  if constexpr (__is_time_point<_Tp>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::day>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::month>)
+    return __value.ok();
+  else if constexpr (same_as<_Tp, chrono::year>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::weekday>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::weekday_indexed>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::weekday_last>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::month_day>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::month_day_last>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::month_weekday>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::month_weekday_last>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::year_month>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::year_month_day>)
+    return __value.ok();
+  else if constexpr (same_as<_Tp, chrono::year_month_day_last>)
+    return __value.ok();
+  else if constexpr (same_as<_Tp, chrono::year_month_weekday>)
+    return __value.ok();
+  else if constexpr (same_as<_Tp, chrono::year_month_weekday_last>)
+    return __value.ok();
+  else if constexpr (__is_hh_mm_ss<_Tp>)
+    return true;
+#  if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+  else if constexpr (same_as<_Tp, chrono::sys_info>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::local_info>)
+    return true;
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  else if constexpr (__is_specialization_v<_Tp, chrono::zoned_time>)
+    return true;
+#    endif
+#  endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+  else
+    static_assert(sizeof(_Tp) == 0, "Add the missing type specialization");
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool __month_name_ok(const _Tp& __value) {
+  if constexpr (__is_time_point<_Tp>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::day>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::month>)
+    return __value.ok();
+  else if constexpr (same_as<_Tp, chrono::year>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::weekday>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::weekday_indexed>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::weekday_last>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::month_day>)
+    return __value.month().ok();
+  else if constexpr (same_as<_Tp, chrono::month_day_last>)
+    return __value.month().ok();
+  else if constexpr (same_as<_Tp, chrono::month_weekday>)
+    return __value.month().ok();
+  else if constexpr (same_as<_Tp, chrono::month_weekday_last>)
+    return __value.month().ok();
+  else if constexpr (same_as<_Tp, chrono::year_month>)
+    return __value.month().ok();
+  else if constexpr (same_as<_Tp, chrono::year_month_day>)
+    return __value.month().ok();
+  else if constexpr (same_as<_Tp, chrono::year_month_day_last>)
+    return __value.month().ok();
+  else if constexpr (same_as<_Tp, chrono::year_month_weekday>)
+    return __value.month().ok();
+  else if constexpr (same_as<_Tp, chrono::year_month_weekday_last>)
+    return __value.month().ok();
+  else if constexpr (__is_hh_mm_ss<_Tp>)
+    return true;
+#  if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+  else if constexpr (same_as<_Tp, chrono::sys_info>)
+    return true;
+  else if constexpr (same_as<_Tp, chrono::local_info>)
+    return true;
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  else if constexpr (__is_specialization_v<_Tp, chrono::zoned_time>)
+    return true;
+#    endif
+#  endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+  else
+    static_assert(sizeof(_Tp) == 0, "Add the missing type specialization");
+}
+
+template <class _CharT, class _Tp, class _FormatContext>
+_LIBCPP_HIDE_FROM_ABI auto
+__format_chrono(const _Tp& __value,
+                _FormatContext& __ctx,
+                __format_spec::__parsed_specifications<_CharT> __specs,
+                basic_string_view<_CharT> __chrono_specs) {
+  basic_stringstream<_CharT> __sstr;
+  // [time.format]/2
+  // 2.1 - the "C" locale if the L option is not present in chrono-format-spec, otherwise
+  // 2.2 - the locale passed to the formatting function if any, otherwise
+  // 2.3 - the global locale.
+  // Note that the __ctx's locale() call does 2.2 and 2.3.
+  if (__specs.__chrono_.__locale_specific_form_)
+    __sstr.imbue(__ctx.locale());
+  else
+    __sstr.imbue(locale::classic());
+
+  if (__chrono_specs.empty())
+    __sstr << __value;
+  else {
+    if constexpr (chrono::__is_duration<_Tp>::value) {
+      // A duration can be a user defined arithmetic type. Users may specialize
+      // numeric_limits, but they may not specialize is_signed.
+      if constexpr (numeric_limits<typename _Tp::rep>::is_signed) {
+        if (__value < __value.zero()) {
+          __sstr << _CharT('-');
+          __formatter::__format_chrono_using_chrono_specs(__sstr, -__value, __chrono_specs);
+        } else
+          __formatter::__format_chrono_using_chrono_specs(__sstr, __value, __chrono_specs);
+      } else
+        __formatter::__format_chrono_using_chrono_specs(__sstr, __value, __chrono_specs);
+      // TODO FMT When keeping the precision it will truncate the string.
+      // Note that the behaviour what the precision does isn't specified.
+      __specs.__precision_ = -1;
+    } else {
+      // Test __weekday_name_ before __weekday_ to give a better error.
+      if (__specs.__chrono_.__weekday_name_ && !__formatter::__weekday_name_ok(__value))
+        std::__throw_format_error("Formatting a weekday name needs a valid weekday");
+
+      if (__specs.__chrono_.__weekday_ && !__formatter::__weekday_ok(__value))
+        std::__throw_format_error("Formatting a weekday needs a valid weekday");
+
+      if (__specs.__chrono_.__day_of_year_ && !__formatter::__date_ok(__value))
+        std::__throw_format_error("Formatting a day of year needs a valid date");
+
+      if (__specs.__chrono_.__week_of_year_ && !__formatter::__date_ok(__value))
+        std::__throw_format_error("Formatting a week of year needs a valid date");
+
+      if (__specs.__chrono_.__month_name_ && !__formatter::__month_name_ok(__value))
+        std::__throw_format_error("Formatting a month name from an invalid month number");
+
+      if constexpr (__is_hh_mm_ss<_Tp>) {
+        // Note this is a pedantic intepretation of the Standard. A hh_mm_ss
+        // is no longer a time_of_day and can store an arbitrary number of
+        // hours. A number of hours in a 12 or 24 hour clock can't represent
+        // 24 hours or more. The functions std::chrono::make12 and
+        // std::chrono::make24 reaffirm this view point.
+        //
+        // Interestingly this will be the only output stream function that
+        // throws.
+        //
+        // TODO FMT The wording probably needs to be adapted to
+        // - The displayed hours is hh_mm_ss.hours() % 24
+        // - It should probably allow %j in the same fashion as duration.
+        // - The stream formatter should change its output when hours >= 24
+        //   - Write it as not valid,
+        //   - or write the number of days.
+        if (__specs.__chrono_.__hour_ && __value.hours().count() > 23)
+          std::__throw_format_error("Formatting a hour needs a valid value");
+
+        if (__value.is_negative())
+          __sstr << _CharT('-');
+      }
+
+      __formatter::__format_chrono_using_chrono_specs(__sstr, __value, __chrono_specs);
+    }
+  }
+
+  return __formatter::__write_string(__sstr.view(), __ctx.out(), __specs);
+}
+
+} // namespace __formatter
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS __formatter_chrono {
+public:
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator
+  __parse(_ParseContext& __ctx, __format_spec::__fields __fields, __format_spec::__flags __flags) {
+    return __parser_.__parse(__ctx, __fields, __flags);
+  }
+
+  template <class _Tp, class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(const _Tp& __value, _FormatContext& __ctx) const {
+    return __formatter::__format_chrono(
+        __value, __ctx, __parser_.__parser_.__get_parsed_chrono_specifications(__ctx), __parser_.__chrono_specs_);
+  }
+
+  __format_spec::__parser_chrono<_CharT> __parser_;
+};
+
+template <class _Duration, __fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::sys_time<_Duration>, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__clock);
+  }
+};
+
+template <class _Duration, __fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::file_time<_Duration>, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__clock);
+  }
+};
+
+template <class _Duration, __fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::local_time<_Duration>, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    // The flags are not __clock since there is no associated time-zone.
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__date_time);
+  }
+};
+
+template <class _Rep, class _Period, __fmt_char_type _CharT>
+struct formatter<chrono::duration<_Rep, _Period>, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    // [time.format]/1
+    // Giving a precision specification in the chrono-format-spec is valid only
+    // for std::chrono::duration types where the representation type Rep is a
+    // floating-point type. For all other Rep types, an exception of type
+    // format_error is thrown if the chrono-format-spec contains a precision
+    // specification.
+    //
+    // Note this doesn't refer to chrono::treat_as_floating_point_v<_Rep>.
+    if constexpr (std::floating_point<_Rep>)
+      return _Base::__parse(__ctx, __format_spec::__fields_chrono_fractional, __format_spec::__flags::__duration);
+    else
+      return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__duration);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::day, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__day);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::month, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__month);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::year, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__year);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::weekday, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__weekday);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::weekday_indexed, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__weekday);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::weekday_last, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__weekday);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::month_day, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__month_day);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::month_day_last, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__month);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::month_weekday, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__month_weekday);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::month_weekday_last, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__month_weekday);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::year_month, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__year_month);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::year_month_day, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__date);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::year_month_day_last, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__date);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::year_month_weekday, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__date);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::year_month_weekday_last, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__date);
+  }
+};
+
+template <class _Duration, __fmt_char_type _CharT>
+struct formatter<chrono::hh_mm_ss<_Duration>, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__time);
+  }
+};
+
+#  if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+template <__fmt_char_type _CharT>
+struct formatter<chrono::sys_info, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__time_zone);
+  }
+};
+
+template <__fmt_char_type _CharT>
+struct formatter<chrono::local_info, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags{});
+  }
+};
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+// Note due to how libc++'s formatters are implemented there is no need to add
+// the exposition only local-time-format-t abstraction.
+template <class _Duration, class _TimeZonePtr, __fmt_char_type _CharT>
+struct formatter<chrono::zoned_time<_Duration, _TimeZonePtr>, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__clock);
+  }
+};
+#    endif // !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&
+           // !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  endif   // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#endif // if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif //  _LIBCPP___CHRONO_FORMATTER_H
diff --git a/libcxx/include/__cxx03/__chrono/hh_mm_ss.h b/libcxx/include/__cxx03/__chrono/hh_mm_ss.h
new file mode 100644
index 000000000000..57d2247fe6a3
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/hh_mm_ss.h
@@ -0,0 +1,112 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_HH_MM_SS_H
+#define _LIBCPP___CHRONO_HH_MM_SS_H
+
+#include <__chrono/duration.h>
+#include <__chrono/time_point.h>
+#include <__config>
+#include <__type_traits/common_type.h>
+#include <ratio>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+template <class _Duration>
+class hh_mm_ss {
+private:
+  static_assert(__is_duration<_Duration>::value, "template parameter of hh_mm_ss must be a std::chrono::duration");
+  using __CommonType = common_type_t<_Duration, chrono::seconds>;
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr uint64_t __pow10(unsigned __exp) {
+    uint64_t __ret = 1;
+    for (unsigned __i = 0; __i < __exp; ++__i)
+      __ret *= 10U;
+    return __ret;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr unsigned __width(uint64_t __n, uint64_t __d = 10, unsigned __w = 0) {
+    if (__n >= 2 && __d != 0 && __w < 19)
+      return 1 + __width(__n, __d % __n * 10, __w + 1);
+    return 0;
+  }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI static unsigned constexpr fractional_width =
+      __width(__CommonType::period::den) < 19 ? __width(__CommonType::period::den) : 6u;
+  using precision = duration<typename __CommonType::rep, ratio<1, __pow10(fractional_width)>>;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr hh_mm_ss() noexcept : hh_mm_ss{_Duration::zero()} {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit hh_mm_ss(_Duration __d) noexcept
+      : __is_neg_(__d < _Duration(0)),
+        __h_(chrono::duration_cast<chrono::hours>(chrono::abs(__d))),
+        __m_(chrono::duration_cast<chrono::minutes>(chrono::abs(__d) - hours())),
+        __s_(chrono::duration_cast<chrono::seconds>(chrono::abs(__d) - hours() - minutes())),
+        __f_(chrono::duration_cast<precision>(chrono::abs(__d) - hours() - minutes() - seconds())) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool is_negative() const noexcept { return __is_neg_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr chrono::hours hours() const noexcept { return __h_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr chrono::minutes minutes() const noexcept { return __m_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr chrono::seconds seconds() const noexcept { return __s_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr precision subseconds() const noexcept { return __f_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr precision to_duration() const noexcept {
+    auto __dur = __h_ + __m_ + __s_ + __f_;
+    return __is_neg_ ? -__dur : __dur;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit operator precision() const noexcept { return to_duration(); }
+
+private:
+  bool __is_neg_;
+  chrono::hours __h_;
+  chrono::minutes __m_;
+  chrono::seconds __s_;
+  precision __f_;
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(hh_mm_ss);
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_am(const hours& __h) noexcept {
+  return __h >= hours(0) && __h < hours(12);
+}
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_pm(const hours& __h) noexcept {
+  return __h >= hours(12) && __h < hours(24);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr hours make12(const hours& __h) noexcept {
+  if (__h == hours(0))
+    return hours(12);
+  else if (__h <= hours(12))
+    return __h;
+  else
+    return __h - hours(12);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr hours make24(const hours& __h, bool __is_pm) noexcept {
+  if (__is_pm)
+    return __h == hours(12) ? __h : __h + hours(12);
+  else
+    return __h == hours(12) ? hours(0) : __h;
+}
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___CHRONO_HH_MM_SS_H
diff --git a/libcxx/include/__cxx03/__chrono/high_resolution_clock.h b/libcxx/include/__cxx03/__chrono/high_resolution_clock.h
new file mode 100644
index 000000000000..0697fd2de9b4
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/high_resolution_clock.h
@@ -0,0 +1,35 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_HIGH_RESOLUTION_CLOCK_H
+#define _LIBCPP___CHRONO_HIGH_RESOLUTION_CLOCK_H
+
+#include <__chrono/steady_clock.h>
+#include <__chrono/system_clock.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+#ifndef _LIBCPP_HAS_NO_MONOTONIC_CLOCK
+typedef steady_clock high_resolution_clock;
+#else
+typedef system_clock high_resolution_clock;
+#endif
+
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CHRONO_HIGH_RESOLUTION_CLOCK_H
diff --git a/libcxx/include/__cxx03/__chrono/leap_second.h b/libcxx/include/__cxx03/__chrono/leap_second.h
new file mode 100644
index 000000000000..1a0e7f3107de
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/leap_second.h
@@ -0,0 +1,126 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
+
+#ifndef _LIBCPP___CHRONO_LEAP_SECOND_H
+#define _LIBCPP___CHRONO_LEAP_SECOND_H
+
+#include <version>
+// Enable the contents of the header only when libc++ was built with experimental features enabled.
+#if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#  include <__chrono/duration.h>
+#  include <__chrono/system_clock.h>
+#  include <__chrono/time_point.h>
+#  include <__compare/ordering.h>
+#  include <__compare/three_way_comparable.h>
+#  include <__config>
+#  include <__utility/private_constructor_tag.h>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  if _LIBCPP_STD_VER >= 20
+
+namespace chrono {
+
+class leap_second {
+public:
+  [[nodiscard]]
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr leap_second(__private_constructor_tag, sys_seconds __date, seconds __value)
+      : __date_(__date), __value_(__value) {}
+
+  _LIBCPP_HIDE_FROM_ABI leap_second(const leap_second&)            = default;
+  _LIBCPP_HIDE_FROM_ABI leap_second& operator=(const leap_second&) = default;
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI constexpr sys_seconds date() const noexcept { return __date_; }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI constexpr seconds value() const noexcept { return __value_; }
+
+private:
+  sys_seconds __date_;
+  seconds __value_;
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const leap_second& __x, const leap_second& __y) {
+  return __x.date() == __y.date();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering operator<=>(const leap_second& __x, const leap_second& __y) {
+  return __x.date() <=> __y.date();
+}
+
+template <class _Duration>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator==(const leap_second& __x, const sys_time<_Duration>& __y) {
+  return __x.date() == __y;
+}
+
+template <class _Duration>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<(const leap_second& __x, const sys_time<_Duration>& __y) {
+  return __x.date() < __y;
+}
+
+template <class _Duration>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<(const sys_time<_Duration>& __x, const leap_second& __y) {
+  return __x < __y.date();
+}
+
+template <class _Duration>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>(const leap_second& __x, const sys_time<_Duration>& __y) {
+  return __y < __x;
+}
+
+template <class _Duration>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>(const sys_time<_Duration>& __x, const leap_second& __y) {
+  return __y < __x;
+}
+
+template <class _Duration>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<=(const leap_second& __x, const sys_time<_Duration>& __y) {
+  return !(__y < __x);
+}
+
+template <class _Duration>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<=(const sys_time<_Duration>& __x, const leap_second& __y) {
+  return !(__y < __x);
+}
+
+template <class _Duration>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>=(const leap_second& __x, const sys_time<_Duration>& __y) {
+  return !(__x < __y);
+}
+
+template <class _Duration>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>=(const sys_time<_Duration>& __x, const leap_second& __y) {
+  return !(__x < __y);
+}
+
+#    ifndef _LIBCPP_COMPILER_GCC
+// This requirement cause a compilation loop in GCC-13 and running out of memory.
+// TODO TZDB Test whether GCC-14 fixes this.
+template <class _Duration>
+  requires three_way_comparable_with<sys_seconds, sys_time<_Duration>>
+_LIBCPP_HIDE_FROM_ABI constexpr auto operator<=>(const leap_second& __x, const sys_time<_Duration>& __y) {
+  return __x.date() <=> __y;
+}
+#    endif
+
+} // namespace chrono
+
+#  endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#endif // _LIBCPP___CHRONO_LEAP_SECOND_H
diff --git a/libcxx/include/__cxx03/__chrono/literals.h b/libcxx/include/__cxx03/__chrono/literals.h
new file mode 100644
index 000000000000..89800440edf4
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/literals.h
@@ -0,0 +1,45 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_LITERALS_H
+#define _LIBCPP___CHRONO_LITERALS_H
+
+#include <__chrono/day.h>
+#include <__chrono/year.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+inline namespace literals {
+inline namespace chrono_literals {
+_LIBCPP_HIDE_FROM_ABI constexpr chrono::day operator""d(unsigned long long __d) noexcept {
+  return chrono::day(static_cast<unsigned>(__d));
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr chrono::year operator""y(unsigned long long __y) noexcept {
+  return chrono::year(static_cast<int>(__y));
+}
+} // namespace chrono_literals
+} // namespace literals
+
+namespace chrono { // hoist the literals into namespace std::chrono
+using namespace literals::chrono_literals;
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___CHRONO_LITERALS_H
diff --git a/libcxx/include/__cxx03/__chrono/local_info.h b/libcxx/include/__cxx03/__chrono/local_info.h
new file mode 100644
index 000000000000..cfe1448904d3
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/local_info.h
@@ -0,0 +1,50 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
+
+#ifndef _LIBCPP___CHRONO_LOCAL_INFO_H
+#define _LIBCPP___CHRONO_LOCAL_INFO_H
+
+#include <version>
+// Enable the contents of the header only when libc++ was built with experimental features enabled.
+#if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#  include <__chrono/sys_info.h>
+#  include <__config>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  if _LIBCPP_STD_VER >= 20
+
+namespace chrono {
+
+struct local_info {
+  static constexpr int unique      = 0;
+  static constexpr int nonexistent = 1;
+  static constexpr int ambiguous   = 2;
+
+  int result;
+  sys_info first;
+  sys_info second;
+};
+
+} // namespace chrono
+
+#  endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#endif // _LIBCPP___CHRONO_LOCAL_INFO_H
diff --git a/libcxx/include/__cxx03/__chrono/month.h b/libcxx/include/__cxx03/__chrono/month.h
new file mode 100644
index 000000000000..ce5cc21aab7d
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/month.h
@@ -0,0 +1,115 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_MONTH_H
+#define _LIBCPP___CHRONO_MONTH_H
+
+#include <__chrono/duration.h>
+#include <__config>
+#include <compare>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+class month {
+private:
+  unsigned char __m_;
+
+public:
+  month() = default;
+  _LIBCPP_HIDE_FROM_ABI explicit inline constexpr month(unsigned __val) noexcept
+      : __m_(static_cast<unsigned char>(__val)) {}
+  _LIBCPP_HIDE_FROM_ABI inline constexpr month& operator++() noexcept {
+    *this += months{1};
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr month operator++(int) noexcept {
+    month __tmp = *this;
+    ++(*this);
+    return __tmp;
+  }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr month& operator--() noexcept {
+    *this -= months{1};
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr month operator--(int) noexcept {
+    month __tmp = *this;
+    --(*this);
+    return __tmp;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr month& operator+=(const months& __m1) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr month& operator-=(const months& __m1) noexcept;
+  _LIBCPP_HIDE_FROM_ABI explicit inline constexpr operator unsigned() const noexcept { return __m_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __m_ >= 1 && __m_ <= 12; }
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const month& __lhs, const month& __rhs) noexcept {
+  return static_cast<unsigned>(__lhs) == static_cast<unsigned>(__rhs);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering operator<=>(const month& __lhs, const month& __rhs) noexcept {
+  return static_cast<unsigned>(__lhs) <=> static_cast<unsigned>(__rhs);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month operator+(const month& __lhs, const months& __rhs) noexcept {
+  auto const __mu = static_cast<long long>(static_cast<unsigned>(__lhs)) + (__rhs.count() - 1);
+  auto const __yr = (__mu >= 0 ? __mu : __mu - 11) / 12;
+  return month{static_cast<unsigned>(__mu - __yr * 12 + 1)};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month operator+(const months& __lhs, const month& __rhs) noexcept {
+  return __rhs + __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month operator-(const month& __lhs, const months& __rhs) noexcept {
+  return __lhs + -__rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr months operator-(const month& __lhs, const month& __rhs) noexcept {
+  auto const __dm = static_cast<unsigned>(__lhs) - static_cast<unsigned>(__rhs);
+  return months(__dm <= 11 ? __dm : __dm + 12);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month& month::operator+=(const months& __dm) noexcept {
+  *this = *this + __dm;
+  return *this;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month& month::operator-=(const months& __dm) noexcept {
+  *this = *this - __dm;
+  return *this;
+}
+
+inline constexpr month January{1};
+inline constexpr month February{2};
+inline constexpr month March{3};
+inline constexpr month April{4};
+inline constexpr month May{5};
+inline constexpr month June{6};
+inline constexpr month July{7};
+inline constexpr month August{8};
+inline constexpr month September{9};
+inline constexpr month October{10};
+inline constexpr month November{11};
+inline constexpr month December{12};
+
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___CHRONO_MONTH_H
diff --git a/libcxx/include/__cxx03/__chrono/month_weekday.h b/libcxx/include/__cxx03/__chrono/month_weekday.h
new file mode 100644
index 000000000000..791987965521
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/month_weekday.h
@@ -0,0 +1,105 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_MONTH_WEEKDAY_H
+#define _LIBCPP___CHRONO_MONTH_WEEKDAY_H
+
+#include <__chrono/month.h>
+#include <__chrono/weekday.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+class month_weekday {
+private:
+  chrono::month __m_;
+  chrono::weekday_indexed __wdi_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr month_weekday(const chrono::month& __mval,
+                                                const chrono::weekday_indexed& __wdival) noexcept
+      : __m_{__mval}, __wdi_{__wdival} {}
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday_indexed weekday_indexed() const noexcept { return __wdi_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __m_.ok() && __wdi_.ok(); }
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool
+operator==(const month_weekday& __lhs, const month_weekday& __rhs) noexcept {
+  return __lhs.month() == __rhs.month() && __lhs.weekday_indexed() == __rhs.weekday_indexed();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday
+operator/(const month& __lhs, const weekday_indexed& __rhs) noexcept {
+  return month_weekday{__lhs, __rhs};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday operator/(int __lhs, const weekday_indexed& __rhs) noexcept {
+  return month_weekday{month(__lhs), __rhs};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday
+operator/(const weekday_indexed& __lhs, const month& __rhs) noexcept {
+  return month_weekday{__rhs, __lhs};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday operator/(const weekday_indexed& __lhs, int __rhs) noexcept {
+  return month_weekday{month(__rhs), __lhs};
+}
+
+class month_weekday_last {
+  chrono::month __m_;
+  chrono::weekday_last __wdl_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr month_weekday_last(const chrono::month& __mval,
+                                                     const chrono::weekday_last& __wdlval) noexcept
+      : __m_{__mval}, __wdl_{__wdlval} {}
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday_last weekday_last() const noexcept { return __wdl_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __m_.ok() && __wdl_.ok(); }
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool
+operator==(const month_weekday_last& __lhs, const month_weekday_last& __rhs) noexcept {
+  return __lhs.month() == __rhs.month() && __lhs.weekday_last() == __rhs.weekday_last();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday_last
+operator/(const month& __lhs, const weekday_last& __rhs) noexcept {
+  return month_weekday_last{__lhs, __rhs};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday_last operator/(int __lhs, const weekday_last& __rhs) noexcept {
+  return month_weekday_last{month(__lhs), __rhs};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday_last
+operator/(const weekday_last& __lhs, const month& __rhs) noexcept {
+  return month_weekday_last{__rhs, __lhs};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday_last operator/(const weekday_last& __lhs, int __rhs) noexcept {
+  return month_weekday_last{month(__rhs), __lhs};
+}
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___CHRONO_MONTH_WEEKDAY_H
diff --git a/libcxx/include/__cxx03/__chrono/monthday.h b/libcxx/include/__cxx03/__chrono/monthday.h
new file mode 100644
index 000000000000..a89d16e51861
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/monthday.h
@@ -0,0 +1,133 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_MONTHDAY_H
+#define _LIBCPP___CHRONO_MONTHDAY_H
+
+#include <__chrono/calendar.h>
+#include <__chrono/day.h>
+#include <__chrono/month.h>
+#include <__config>
+#include <compare>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+class month_day {
+private:
+  chrono::month __m_;
+  chrono::day __d_;
+
+public:
+  month_day() = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr month_day(const chrono::month& __mval, const chrono::day& __dval) noexcept
+      : __m_{__mval}, __d_{__dval} {}
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::day day() const noexcept { return __d_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr bool ok() const noexcept;
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool month_day::ok() const noexcept {
+  if (!__m_.ok())
+    return false;
+  const unsigned __dval = static_cast<unsigned>(__d_);
+  if (__dval < 1 || __dval > 31)
+    return false;
+  if (__dval <= 29)
+    return true;
+  //  Now we've got either 30 or 31
+  const unsigned __mval = static_cast<unsigned>(__m_);
+  if (__mval == 2)
+    return false;
+  if (__mval == 4 || __mval == 6 || __mval == 9 || __mval == 11)
+    return __dval == 30;
+  return true;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const month_day& __lhs, const month_day& __rhs) noexcept {
+  return __lhs.month() == __rhs.month() && __lhs.day() == __rhs.day();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering
+operator<=>(const month_day& __lhs, const month_day& __rhs) noexcept {
+  if (auto __c = __lhs.month() <=> __rhs.month(); __c != 0)
+    return __c;
+  return __lhs.day() <=> __rhs.day();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_day operator/(const month& __lhs, const day& __rhs) noexcept {
+  return month_day{__lhs, __rhs};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_day operator/(const day& __lhs, const month& __rhs) noexcept {
+  return __rhs / __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_day operator/(const month& __lhs, int __rhs) noexcept {
+  return __lhs / day(__rhs);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_day operator/(int __lhs, const day& __rhs) noexcept {
+  return month(__lhs) / __rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_day operator/(const day& __lhs, int __rhs) noexcept {
+  return month(__rhs) / __lhs;
+}
+
+class month_day_last {
+private:
+  chrono::month __m_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr month_day_last(const chrono::month& __val) noexcept : __m_{__val} {}
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __m_.ok(); }
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool
+operator==(const month_day_last& __lhs, const month_day_last& __rhs) noexcept {
+  return __lhs.month() == __rhs.month();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering
+operator<=>(const month_day_last& __lhs, const month_day_last& __rhs) noexcept {
+  return __lhs.month() <=> __rhs.month();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_day_last operator/(const month& __lhs, last_spec) noexcept {
+  return month_day_last{__lhs};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_day_last operator/(last_spec, const month& __rhs) noexcept {
+  return month_day_last{__rhs};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_day_last operator/(int __lhs, last_spec) noexcept {
+  return month_day_last{month(__lhs)};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr month_day_last operator/(last_spec, int __rhs) noexcept {
+  return month_day_last{month(__rhs)};
+}
+
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___CHRONO_MONTHDAY_H
diff --git a/libcxx/include/__cxx03/__chrono/ostream.h b/libcxx/include/__cxx03/__chrono/ostream.h
new file mode 100644
index 000000000000..e6c43254eea1
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/ostream.h
@@ -0,0 +1,322 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_OSTREAM_H
+#define _LIBCPP___CHRONO_OSTREAM_H
+
+#include <__chrono/calendar.h>
+#include <__chrono/day.h>
+#include <__chrono/duration.h>
+#include <__chrono/file_clock.h>
+#include <__chrono/hh_mm_ss.h>
+#include <__chrono/local_info.h>
+#include <__chrono/month.h>
+#include <__chrono/month_weekday.h>
+#include <__chrono/monthday.h>
+#include <__chrono/statically_widen.h>
+#include <__chrono/sys_info.h>
+#include <__chrono/system_clock.h>
+#include <__chrono/weekday.h>
+#include <__chrono/year.h>
+#include <__chrono/year_month.h>
+#include <__chrono/year_month_day.h>
+#include <__chrono/year_month_weekday.h>
+#include <__chrono/zoned_time.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/format_functions.h>
+#include <__fwd/ostream.h>
+#include <ratio>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace chrono {
+
+template <class _CharT, class _Traits, class _Duration>
+  requires(!treat_as_floating_point_v<typename _Duration::rep> && _Duration{1} < days{1})
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const sys_time<_Duration>& __tp) {
+  return __os << std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L%F %T}"), __tp);
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const sys_days& __dp) {
+  return __os << year_month_day{__dp};
+}
+
+template <class _CharT, class _Traits, class _Duration>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const file_time<_Duration> __tp) {
+  return __os << std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L%F %T}"), __tp);
+}
+
+template <class _CharT, class _Traits, class _Duration>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const local_time<_Duration> __tp) {
+  return __os << sys_time<_Duration>{__tp.time_since_epoch()};
+}
+
+// Depending on the type the return is a const _CharT* or a basic_string<_CharT>
+template <class _CharT, class _Period>
+_LIBCPP_HIDE_FROM_ABI auto __units_suffix() {
+  // TODO FMT LWG issue the suffixes are always char and not STATICALLY-WIDEN'ed.
+  if constexpr (same_as<typename _Period::type, atto>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "as");
+  else if constexpr (same_as<typename _Period::type, femto>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "fs");
+  else if constexpr (same_as<typename _Period::type, pico>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "ps");
+  else if constexpr (same_as<typename _Period::type, nano>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "ns");
+  else if constexpr (same_as<typename _Period::type, micro>)
+#  ifndef _LIBCPP_HAS_NO_UNICODE
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "\u00b5s");
+#  else
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "us");
+#  endif
+  else if constexpr (same_as<typename _Period::type, milli>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "ms");
+  else if constexpr (same_as<typename _Period::type, centi>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "cs");
+  else if constexpr (same_as<typename _Period::type, deci>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "ds");
+  else if constexpr (same_as<typename _Period::type, ratio<1>>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "s");
+  else if constexpr (same_as<typename _Period::type, deca>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "das");
+  else if constexpr (same_as<typename _Period::type, hecto>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "hs");
+  else if constexpr (same_as<typename _Period::type, kilo>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "ks");
+  else if constexpr (same_as<typename _Period::type, mega>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "Ms");
+  else if constexpr (same_as<typename _Period::type, giga>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "Gs");
+  else if constexpr (same_as<typename _Period::type, tera>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "Ts");
+  else if constexpr (same_as<typename _Period::type, peta>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "Ps");
+  else if constexpr (same_as<typename _Period::type, exa>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "Es");
+  else if constexpr (same_as<typename _Period::type, ratio<60>>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "min");
+  else if constexpr (same_as<typename _Period::type, ratio<3600>>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "h");
+  else if constexpr (same_as<typename _Period::type, ratio<86400>>)
+    return _LIBCPP_STATICALLY_WIDEN(_CharT, "d");
+  else if constexpr (_Period::den == 1)
+    return std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "[{}]s"), _Period::num);
+  else
+    return std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "[{}/{}]s"), _Period::num, _Period::den);
+}
+
+template <class _CharT, class _Traits, class _Rep, class _Period>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const duration<_Rep, _Period>& __d) {
+  basic_ostringstream<_CharT, _Traits> __s;
+  __s.flags(__os.flags());
+  __s.imbue(__os.getloc());
+  __s.precision(__os.precision());
+  __s << __d.count() << chrono::__units_suffix<_CharT, _Period>();
+  return __os << __s.str();
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>& operator<<(basic_ostream<_CharT, _Traits>& __os, const day& __d) {
+  return __os << (__d.ok() ? std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "{:%d}"), __d)
+                           // Note this error differs from the wording of the Standard. The
+                           // Standard wording doesn't work well on AIX or Windows. There
+                           // the formatted day seems to be either modulo 100 or completely
+                           // omitted. Judging by the wording this is valid.
+                           // TODO FMT Write a paper of file an LWG issue.
+                           : std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "{:02} is not a valid day"),
+                                         static_cast<unsigned>(__d)));
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const month& __m) {
+  return __os << (__m.ok() ? std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L%b}"), __m)
+                           : std::format(__os.getloc(),
+                                         _LIBCPP_STATICALLY_WIDEN(_CharT, "{} is not a valid month"),
+                                         static_cast<unsigned>(__m))); // TODO FMT Standard mandated locale isn't used.
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const year& __y) {
+  return __os << (__y.ok() ? std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "{:%Y}"), __y)
+                           : std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "{:%Y} is not a valid year"), __y));
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const weekday& __wd) {
+  return __os << (__wd.ok() ? std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L%a}"), __wd)
+                            : std::format(__os.getloc(), // TODO FMT Standard mandated locale isn't used.
+                                          _LIBCPP_STATICALLY_WIDEN(_CharT, "{} is not a valid weekday"),
+                                          static_cast<unsigned>(__wd.c_encoding())));
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const weekday_indexed& __wdi) {
+  auto __i = __wdi.index();
+  return __os << (__i >= 1 && __i <= 5
+                      ? std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L}[{}]"), __wdi.weekday(), __i)
+                      : std::format(__os.getloc(),
+                                    _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L}[{} is not a valid index]"),
+                                    __wdi.weekday(),
+                                    __i));
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const weekday_last& __wdl) {
+  return __os << std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L}[last]"), __wdl.weekday());
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const month_day& __md) {
+  // TODO FMT The Standard allows 30th of February to be printed.
+  // It would be nice to show an error message instead.
+  return __os << std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L}/{}"), __md.month(), __md.day());
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const month_day_last& __mdl) {
+  return __os << std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L}/last"), __mdl.month());
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const month_weekday& __mwd) {
+  return __os << std::format(
+             __os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L}/{:L}"), __mwd.month(), __mwd.weekday_indexed());
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const month_weekday_last& __mwdl) {
+  return __os << std::format(
+             __os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L}/{:L}"), __mwdl.month(), __mwdl.weekday_last());
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const year_month& __ym) {
+  return __os << std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{}/{:L}"), __ym.year(), __ym.month());
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const year_month_day& __ymd) {
+  return __os << (__ymd.ok() ? std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "{:%F}"), __ymd)
+                             : std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "{:%F} is not a valid date"), __ymd));
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const year_month_day_last& __ymdl) {
+  return __os << std::format(
+             __os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{}/{:L}"), __ymdl.year(), __ymdl.month_day_last());
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const year_month_weekday& __ymwd) {
+  return __os << std::format(
+             __os.getloc(),
+             _LIBCPP_STATICALLY_WIDEN(_CharT, "{}/{:L}/{:L}"),
+             __ymwd.year(),
+             __ymwd.month(),
+             __ymwd.weekday_indexed());
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const year_month_weekday_last& __ymwdl) {
+  return __os << std::format(
+             __os.getloc(),
+             _LIBCPP_STATICALLY_WIDEN(_CharT, "{}/{:L}/{:L}"),
+             __ymwdl.year(),
+             __ymwdl.month(),
+             __ymwdl.weekday_last());
+}
+
+template <class _CharT, class _Traits, class _Duration>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const hh_mm_ss<_Duration> __hms) {
+  return __os << std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L%T}"), __hms);
+}
+
+#  if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const sys_info& __info) {
+  // __info.abbrev is always std::basic_string<char>.
+  // Since these strings typically are short the conversion should be cheap.
+  std::basic_string<_CharT> __abbrev{__info.abbrev.begin(), __info.abbrev.end()};
+  return __os << std::format(
+             _LIBCPP_STATICALLY_WIDEN(_CharT, "[{:%F %T}, {:%F %T}) {:%T} {:%Q%q} \"{}\""),
+             __info.begin,
+             __info.end,
+             hh_mm_ss{__info.offset},
+             __info.save,
+             __abbrev);
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const local_info& __info) {
+  auto __result = [&]() -> basic_string<_CharT> {
+    switch (__info.result) {
+    case local_info::unique:
+      return _LIBCPP_STATICALLY_WIDEN(_CharT, "unique");
+    case local_info::nonexistent:
+      return _LIBCPP_STATICALLY_WIDEN(_CharT, "non-existent");
+    case local_info::ambiguous:
+      return _LIBCPP_STATICALLY_WIDEN(_CharT, "ambiguous");
+
+    default:
+      return std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "unspecified result ({})"), __info.result);
+    };
+  };
+
+  return __os << std::format(
+             _LIBCPP_STATICALLY_WIDEN(_CharT, "{}: {{{}, {}}}"), __result(), __info.first, __info.second);
+}
+
+#    if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                          \
+        !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+template <class _CharT, class _Traits, class _Duration, class _TimeZonePtr>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const zoned_time<_Duration, _TimeZonePtr>& __tp) {
+  return __os << std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L%F %T %Z}"), __tp);
+}
+#    endif
+#  endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+} // namespace chrono
+
+#endif // if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CHRONO_OSTREAM_H
diff --git a/libcxx/include/__cxx03/__chrono/parser_std_format_spec.h b/libcxx/include/__cxx03/__chrono/parser_std_format_spec.h
new file mode 100644
index 000000000000..785bbae198e4
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/parser_std_format_spec.h
@@ -0,0 +1,416 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_PARSER_STD_FORMAT_SPEC_H
+#define _LIBCPP___CHRONO_PARSER_STD_FORMAT_SPEC_H
+
+#include <__config>
+#include <__format/concepts.h>
+#include <__format/format_error.h>
+#include <__format/format_parse_context.h>
+#include <__format/formatter_string.h>
+#include <__format/parser_std_format_spec.h>
+#include <string_view>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __format_spec {
+
+// By not placing this constant in the formatter class it's not duplicated for char and wchar_t
+inline constexpr __fields __fields_chrono_fractional{
+    .__precision_ = true, .__locale_specific_form_ = true, .__type_ = false};
+inline constexpr __fields __fields_chrono{.__locale_specific_form_ = true, .__type_ = false};
+
+/// Flags available or required in a chrono type.
+///
+/// The caller of the chrono formatter lists the types it has available and the
+/// validation tests whether the requested type spec (e.g. %M) is available in
+/// the formatter.
+/// When the type in the chrono-format-spec isn't present in the data a
+/// \ref format_error is thrown.
+enum class __flags {
+  __second = 0x1,
+  __minute = 0x2,
+  __hour   = 0x4,
+  __time   = __hour | __minute | __second,
+
+  __day   = 0x8,
+  __month = 0x10,
+  __year  = 0x20,
+
+  __weekday = 0x40,
+
+  __month_day     = __day | __month,
+  __month_weekday = __weekday | __month,
+  __year_month    = __month | __year,
+  __date          = __day | __month | __year | __weekday,
+
+  __date_time = __date | __time,
+
+  __duration = 0x80 | __time,
+
+  __time_zone = 0x100,
+
+  __clock = __date_time | __time_zone
+};
+
+_LIBCPP_HIDE_FROM_ABI constexpr __flags operator&(__flags __lhs, __flags __rhs) {
+  return static_cast<__flags>(static_cast<unsigned>(__lhs) & static_cast<unsigned>(__rhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr void __validate_second(__flags __flags) {
+  if ((__flags & __flags::__second) != __flags::__second)
+    std::__throw_format_error("The supplied date time doesn't contain a second");
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr void __validate_minute(__flags __flags) {
+  if ((__flags & __flags::__minute) != __flags::__minute)
+    std::__throw_format_error("The supplied date time doesn't contain a minute");
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr void __validate_hour(__flags __flags) {
+  if ((__flags & __flags::__hour) != __flags::__hour)
+    std::__throw_format_error("The supplied date time doesn't contain an hour");
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr void __validate_time(__flags __flags) {
+  if ((__flags & __flags::__time) != __flags::__time)
+    std::__throw_format_error("The supplied date time doesn't contain a time");
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr void __validate_day(__flags __flags) {
+  if ((__flags & __flags::__day) != __flags::__day)
+    std::__throw_format_error("The supplied date time doesn't contain a day");
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr void __validate_month(__flags __flags) {
+  if ((__flags & __flags::__month) != __flags::__month)
+    std::__throw_format_error("The supplied date time doesn't contain a month");
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr void __validate_year(__flags __flags) {
+  if ((__flags & __flags::__year) != __flags::__year)
+    std::__throw_format_error("The supplied date time doesn't contain a year");
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr void __validate_date(__flags __flags) {
+  if ((__flags & __flags::__date) != __flags::__date)
+    std::__throw_format_error("The supplied date time doesn't contain a date");
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr void __validate_date_or_duration(__flags __flags) {
+  if (((__flags & __flags::__date) != __flags::__date) && ((__flags & __flags::__duration) != __flags::__duration))
+    std::__throw_format_error("The supplied date time doesn't contain a date or duration");
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr void __validate_date_time(__flags __flags) {
+  if ((__flags & __flags::__date_time) != __flags::__date_time)
+    std::__throw_format_error("The supplied date time doesn't contain a date and time");
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr void __validate_weekday(__flags __flags) {
+  if ((__flags & __flags::__weekday) != __flags::__weekday)
+    std::__throw_format_error("The supplied date time doesn't contain a weekday");
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr void __validate_duration(__flags __flags) {
+  if ((__flags & __flags::__duration) != __flags::__duration)
+    std::__throw_format_error("The supplied date time doesn't contain a duration");
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr void __validate_time_zone(__flags __flags) {
+  if ((__flags & __flags::__time_zone) != __flags::__time_zone)
+    std::__throw_format_error("The supplied date time doesn't contain a time zone");
+}
+
+template <class _CharT>
+class _LIBCPP_TEMPLATE_VIS __parser_chrono {
+  using _ConstIterator = typename basic_format_parse_context<_CharT>::const_iterator;
+
+public:
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator
+  __parse(_ParseContext& __ctx, __fields __fields, __flags __flags) {
+    _ConstIterator __begin = __parser_.__parse(__ctx, __fields);
+    _ConstIterator __end   = __ctx.end();
+    if (__begin == __end)
+      return __begin;
+
+    _ConstIterator __last = __parse_chrono_specs(__begin, __end, __flags);
+    __chrono_specs_       = basic_string_view<_CharT>{__begin, __last};
+
+    return __last;
+  }
+
+  __parser<_CharT> __parser_;
+  basic_string_view<_CharT> __chrono_specs_;
+
+private:
+  _LIBCPP_HIDE_FROM_ABI constexpr _ConstIterator
+  __parse_chrono_specs(_ConstIterator __begin, _ConstIterator __end, __flags __flags) {
+    _LIBCPP_ASSERT_INTERNAL(__begin != __end,
+                            "When called with an empty input the function will cause "
+                            "undefined behavior by evaluating data not in the input");
+
+    if (*__begin != _CharT('%') && *__begin != _CharT('}'))
+      std::__throw_format_error("The format specifier expects a '%' or a '}'");
+
+    do {
+      switch (*__begin) {
+      case _CharT('{'):
+        std::__throw_format_error("The chrono specifiers contain a '{'");
+
+      case _CharT('}'):
+        return __begin;
+
+      case _CharT('%'):
+        __parse_conversion_spec(__begin, __end, __flags);
+        [[fallthrough]];
+
+      default:
+        // All other literals
+        ++__begin;
+      }
+
+    } while (__begin != __end && *__begin != _CharT('}'));
+
+    return __begin;
+  }
+
+  /// \pre *__begin == '%'
+  /// \post __begin points at the end parsed conversion-spec
+  _LIBCPP_HIDE_FROM_ABI constexpr void
+  __parse_conversion_spec(_ConstIterator& __begin, _ConstIterator __end, __flags __flags) {
+    ++__begin;
+    if (__begin == __end)
+      std::__throw_format_error("End of input while parsing a conversion specifier");
+
+    switch (*__begin) {
+    case _CharT('n'):
+    case _CharT('t'):
+    case _CharT('%'):
+      break;
+
+    case _CharT('S'):
+      __format_spec::__validate_second(__flags);
+      break;
+
+    case _CharT('M'):
+      __format_spec::__validate_minute(__flags);
+      break;
+
+    case _CharT('p'): // TODO FMT does the formater require an hour or a time?
+    case _CharT('H'):
+    case _CharT('I'):
+      __parser_.__hour_ = true;
+      __validate_hour(__flags);
+      break;
+
+    case _CharT('r'):
+    case _CharT('R'):
+    case _CharT('T'):
+    case _CharT('X'):
+      __parser_.__hour_ = true;
+      __format_spec::__validate_time(__flags);
+      break;
+
+    case _CharT('d'):
+    case _CharT('e'):
+      __format_spec::__validate_day(__flags);
+      break;
+
+    case _CharT('b'):
+    case _CharT('h'):
+    case _CharT('B'):
+      __parser_.__month_name_ = true;
+      [[fallthrough]];
+    case _CharT('m'):
+      __format_spec::__validate_month(__flags);
+      break;
+
+    case _CharT('y'):
+    case _CharT('C'):
+    case _CharT('Y'):
+      __format_spec::__validate_year(__flags);
+      break;
+
+    case _CharT('j'):
+      __parser_.__day_of_year_ = true;
+      __format_spec::__validate_date_or_duration(__flags);
+      break;
+
+    case _CharT('g'):
+    case _CharT('G'):
+    case _CharT('U'):
+    case _CharT('V'):
+    case _CharT('W'):
+      __parser_.__week_of_year_ = true;
+      [[fallthrough]];
+    case _CharT('x'):
+    case _CharT('D'):
+    case _CharT('F'):
+      __format_spec::__validate_date(__flags);
+      break;
+
+    case _CharT('c'):
+      __format_spec::__validate_date_time(__flags);
+      break;
+
+    case _CharT('a'):
+    case _CharT('A'):
+      __parser_.__weekday_name_ = true;
+      [[fallthrough]];
+    case _CharT('u'):
+    case _CharT('w'):
+      __parser_.__weekday_ = true;
+      __validate_weekday(__flags);
+      __format_spec::__validate_weekday(__flags);
+      break;
+
+    case _CharT('q'):
+    case _CharT('Q'):
+      __format_spec::__validate_duration(__flags);
+      break;
+
+    case _CharT('E'):
+      __parse_modifier_E(__begin, __end, __flags);
+      break;
+
+    case _CharT('O'):
+      __parse_modifier_O(__begin, __end, __flags);
+      break;
+
+    case _CharT('z'):
+    case _CharT('Z'):
+      // Currently there's no time zone information. However some clocks have a
+      // hard-coded "time zone", for these clocks the information can be used.
+      // TODO FMT implement time zones.
+      __format_spec::__validate_time_zone(__flags);
+      break;
+
+    default: // unknown type;
+      std::__throw_format_error("The date time type specifier is invalid");
+    }
+  }
+
+  /// \pre *__begin == 'E'
+  /// \post __begin is incremented by one.
+  _LIBCPP_HIDE_FROM_ABI constexpr void
+  __parse_modifier_E(_ConstIterator& __begin, _ConstIterator __end, __flags __flags) {
+    ++__begin;
+    if (__begin == __end)
+      std::__throw_format_error("End of input while parsing the modifier E");
+
+    switch (*__begin) {
+    case _CharT('X'):
+      __parser_.__hour_ = true;
+      __format_spec::__validate_time(__flags);
+      break;
+
+    case _CharT('y'):
+    case _CharT('C'):
+    case _CharT('Y'):
+      __format_spec::__validate_year(__flags);
+      break;
+
+    case _CharT('x'):
+      __format_spec::__validate_date(__flags);
+      break;
+
+    case _CharT('c'):
+      __format_spec::__validate_date_time(__flags);
+      break;
+
+    case _CharT('z'):
+      // Currently there's no time zone information. However some clocks have a
+      // hard-coded "time zone", for these clocks the information can be used.
+      // TODO FMT implement time zones.
+      __format_spec::__validate_time_zone(__flags);
+      break;
+
+    default:
+      std::__throw_format_error("The date time type specifier for modifier E is invalid");
+    }
+  }
+
+  /// \pre *__begin == 'O'
+  /// \post __begin is incremented by one.
+  _LIBCPP_HIDE_FROM_ABI constexpr void
+  __parse_modifier_O(_ConstIterator& __begin, _ConstIterator __end, __flags __flags) {
+    ++__begin;
+    if (__begin == __end)
+      std::__throw_format_error("End of input while parsing the modifier O");
+
+    switch (*__begin) {
+    case _CharT('S'):
+      __format_spec::__validate_second(__flags);
+      break;
+
+    case _CharT('M'):
+      __format_spec::__validate_minute(__flags);
+      break;
+
+    case _CharT('I'):
+    case _CharT('H'):
+      __parser_.__hour_ = true;
+      __format_spec::__validate_hour(__flags);
+      break;
+
+    case _CharT('d'):
+    case _CharT('e'):
+      __format_spec::__validate_day(__flags);
+      break;
+
+    case _CharT('m'):
+      __format_spec::__validate_month(__flags);
+      break;
+
+    case _CharT('y'):
+      __format_spec::__validate_year(__flags);
+      break;
+
+    case _CharT('U'):
+    case _CharT('V'):
+    case _CharT('W'):
+      __parser_.__week_of_year_ = true;
+      __format_spec::__validate_date(__flags);
+      break;
+
+    case _CharT('u'):
+    case _CharT('w'):
+      __parser_.__weekday_ = true;
+      __format_spec::__validate_weekday(__flags);
+      break;
+
+    case _CharT('z'):
+      // Currently there's no time zone information. However some clocks have a
+      // hard-coded "time zone", for these clocks the information can be used.
+      // TODO FMT implement time zones.
+      __format_spec::__validate_time_zone(__flags);
+      break;
+
+    default:
+      std::__throw_format_error("The date time type specifier for modifier O is invalid");
+    }
+  }
+};
+
+} // namespace __format_spec
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CHRONO_PARSER_STD_FORMAT_SPEC_H
diff --git a/libcxx/include/__cxx03/__chrono/statically_widen.h b/libcxx/include/__cxx03/__chrono/statically_widen.h
new file mode 100644
index 000000000000..a18c46f057a8
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/statically_widen.h
@@ -0,0 +1,52 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_STATICALLY_WIDEN_H
+#define _LIBCPP___CHRONO_STATICALLY_WIDEN_H
+
+// Implements the STATICALLY-WIDEN exposition-only function. ([time.general]/2)
+
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/concepts.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <__fmt_char_type _CharT>
+_LIBCPP_HIDE_FROM_ABI constexpr const _CharT* __statically_widen(const char* __str, const wchar_t* __wstr) {
+  if constexpr (same_as<_CharT, char>)
+    return __str;
+  else
+    return __wstr;
+}
+#    define _LIBCPP_STATICALLY_WIDEN(_CharT, __str) ::std::__statically_widen<_CharT>(__str, L##__str)
+#  else // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+// Without this indirection the unit test test/libcxx/modules_include.sh.cpp
+// fails for the CI build "No wide characters". This seems like a bug.
+// TODO FMT investigate why this is needed.
+template <__fmt_char_type _CharT>
+_LIBCPP_HIDE_FROM_ABI constexpr const _CharT* __statically_widen(const char* __str) {
+  return __str;
+}
+#    define _LIBCPP_STATICALLY_WIDEN(_CharT, __str) ::std::__statically_widen<_CharT>(__str)
+#  endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CHRONO_STATICALLY_WIDEN_H
diff --git a/libcxx/include/__cxx03/__chrono/steady_clock.h b/libcxx/include/__cxx03/__chrono/steady_clock.h
new file mode 100644
index 000000000000..612a7f156e63
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/steady_clock.h
@@ -0,0 +1,42 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_STEADY_CLOCK_H
+#define _LIBCPP___CHRONO_STEADY_CLOCK_H
+
+#include <__chrono/duration.h>
+#include <__chrono/time_point.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+#ifndef _LIBCPP_HAS_NO_MONOTONIC_CLOCK
+class _LIBCPP_EXPORTED_FROM_ABI steady_clock {
+public:
+  typedef nanoseconds duration;
+  typedef duration::rep rep;
+  typedef duration::period period;
+  typedef chrono::time_point<steady_clock, duration> time_point;
+  static _LIBCPP_CONSTEXPR_SINCE_CXX14 const bool is_steady = true;
+
+  static time_point now() _NOEXCEPT;
+};
+#endif
+
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CHRONO_STEADY_CLOCK_H
diff --git a/libcxx/include/__cxx03/__chrono/sys_info.h b/libcxx/include/__cxx03/__chrono/sys_info.h
new file mode 100644
index 000000000000..11536cbde3a3
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/sys_info.h
@@ -0,0 +1,51 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
+
+#ifndef _LIBCPP___CHRONO_SYS_INFO_H
+#define _LIBCPP___CHRONO_SYS_INFO_H
+
+#include <version>
+// Enable the contents of the header only when libc++ was built with experimental features enabled.
+#if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#  include <__chrono/duration.h>
+#  include <__chrono/system_clock.h>
+#  include <__chrono/time_point.h>
+#  include <__config>
+#  include <string>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  if _LIBCPP_STD_VER >= 20
+
+namespace chrono {
+
+struct sys_info {
+  sys_seconds begin;
+  sys_seconds end;
+  seconds offset;
+  minutes save;
+  string abbrev;
+};
+
+} // namespace chrono
+
+#  endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#endif // _LIBCPP___CHRONO_SYS_INFO_H
diff --git a/libcxx/include/__cxx03/__chrono/system_clock.h b/libcxx/include/__cxx03/__chrono/system_clock.h
new file mode 100644
index 000000000000..5a9eb65bdae7
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/system_clock.h
@@ -0,0 +1,52 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_SYSTEM_CLOCK_H
+#define _LIBCPP___CHRONO_SYSTEM_CLOCK_H
+
+#include <__chrono/duration.h>
+#include <__chrono/time_point.h>
+#include <__config>
+#include <ctime>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+class _LIBCPP_EXPORTED_FROM_ABI system_clock {
+public:
+  typedef microseconds duration;
+  typedef duration::rep rep;
+  typedef duration::period period;
+  typedef chrono::time_point<system_clock> time_point;
+  static _LIBCPP_CONSTEXPR_SINCE_CXX14 const bool is_steady = false;
+
+  static time_point now() _NOEXCEPT;
+  static time_t to_time_t(const time_point& __t) _NOEXCEPT;
+  static time_point from_time_t(time_t __t) _NOEXCEPT;
+};
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Duration>
+using sys_time    = time_point<system_clock, _Duration>;
+using sys_seconds = sys_time<seconds>;
+using sys_days    = sys_time<days>;
+
+#endif
+
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CHRONO_SYSTEM_CLOCK_H
diff --git a/libcxx/include/__cxx03/__chrono/time_point.h b/libcxx/include/__cxx03/__chrono/time_point.h
new file mode 100644
index 000000000000..aaf0b098f280
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/time_point.h
@@ -0,0 +1,220 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_TIME_POINT_H
+#define _LIBCPP___CHRONO_TIME_POINT_H
+
+#include <__chrono/duration.h>
+#include <__compare/ordering.h>
+#include <__compare/three_way_comparable.h>
+#include <__config>
+#include <__type_traits/common_type.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_convertible.h>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+template <class _Clock, class _Duration = typename _Clock::duration>
+class _LIBCPP_TEMPLATE_VIS time_point {
+  static_assert(__is_duration<_Duration>::value,
+                "Second template parameter of time_point must be a std::chrono::duration");
+
+public:
+  typedef _Clock clock;
+  typedef _Duration duration;
+  typedef typename duration::rep rep;
+  typedef typename duration::period period;
+
+private:
+  duration __d_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 time_point() : __d_(duration::zero()) {}
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit time_point(const duration& __d) : __d_(__d) {}
+
+  // conversions
+  template <class _Duration2, __enable_if_t<is_convertible<_Duration2, duration>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 time_point(const time_point<clock, _Duration2>& __t)
+      : __d_(__t.time_since_epoch()) {}
+
+  // observer
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 duration time_since_epoch() const { return __d_; }
+
+  // arithmetic
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 time_point& operator+=(const duration& __d) {
+    __d_ += __d;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 time_point& operator-=(const duration& __d) {
+    __d_ -= __d;
+    return *this;
+  }
+
+  // special values
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR time_point min() _NOEXCEPT { return time_point(duration::min()); }
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR time_point max() _NOEXCEPT { return time_point(duration::max()); }
+};
+
+} // namespace chrono
+
+template <class _Clock, class _Duration1, class _Duration2>
+struct _LIBCPP_TEMPLATE_VIS
+common_type<chrono::time_point<_Clock, _Duration1>, chrono::time_point<_Clock, _Duration2> > {
+  typedef chrono::time_point<_Clock, typename common_type<_Duration1, _Duration2>::type> type;
+};
+
+namespace chrono {
+
+template <class _ToDuration, class _Clock, class _Duration>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 time_point<_Clock, _ToDuration>
+time_point_cast(const time_point<_Clock, _Duration>& __t) {
+  return time_point<_Clock, _ToDuration>(chrono::duration_cast<_ToDuration>(__t.time_since_epoch()));
+}
+
+#if _LIBCPP_STD_VER >= 17
+template <class _ToDuration, class _Clock, class _Duration, enable_if_t<__is_duration<_ToDuration>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI constexpr time_point<_Clock, _ToDuration> floor(const time_point<_Clock, _Duration>& __t) {
+  return time_point<_Clock, _ToDuration>{chrono::floor<_ToDuration>(__t.time_since_epoch())};
+}
+
+template <class _ToDuration, class _Clock, class _Duration, enable_if_t<__is_duration<_ToDuration>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI constexpr time_point<_Clock, _ToDuration> ceil(const time_point<_Clock, _Duration>& __t) {
+  return time_point<_Clock, _ToDuration>{chrono::ceil<_ToDuration>(__t.time_since_epoch())};
+}
+
+template <class _ToDuration, class _Clock, class _Duration, enable_if_t<__is_duration<_ToDuration>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI constexpr time_point<_Clock, _ToDuration> round(const time_point<_Clock, _Duration>& __t) {
+  return time_point<_Clock, _ToDuration>{chrono::round<_ToDuration>(__t.time_since_epoch())};
+}
+
+template <class _Rep, class _Period, enable_if_t<numeric_limits<_Rep>::is_signed, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI constexpr duration<_Rep, _Period> abs(duration<_Rep, _Period> __d) {
+  return __d >= __d.zero() ? +__d : -__d;
+}
+#endif // _LIBCPP_STD_VER >= 17
+
+// time_point ==
+
+template <class _Clock, class _Duration1, class _Duration2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator==(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) {
+  return __lhs.time_since_epoch() == __rhs.time_since_epoch();
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+// time_point !=
+
+template <class _Clock, class _Duration1, class _Duration2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator!=(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) {
+  return !(__lhs == __rhs);
+}
+
+#endif // _LIBCPP_STD_VER <= 17
+
+// time_point <
+
+template <class _Clock, class _Duration1, class _Duration2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator<(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) {
+  return __lhs.time_since_epoch() < __rhs.time_since_epoch();
+}
+
+// time_point >
+
+template <class _Clock, class _Duration1, class _Duration2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator>(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) {
+  return __rhs < __lhs;
+}
+
+// time_point <=
+
+template <class _Clock, class _Duration1, class _Duration2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator<=(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) {
+  return !(__rhs < __lhs);
+}
+
+// time_point >=
+
+template <class _Clock, class _Duration1, class _Duration2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator>=(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) {
+  return !(__lhs < __rhs);
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Clock, class _Duration1, three_way_comparable_with<_Duration1> _Duration2>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+operator<=>(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) {
+  return __lhs.time_since_epoch() <=> __rhs.time_since_epoch();
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+// time_point operator+(time_point x, duration y);
+
+template <class _Clock, class _Duration1, class _Rep2, class _Period2>
+inline _LIBCPP_HIDE_FROM_ABI
+_LIBCPP_CONSTEXPR_SINCE_CXX14 time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type>
+operator+(const time_point<_Clock, _Duration1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
+  typedef time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type> _Tr;
+  return _Tr(__lhs.time_since_epoch() + __rhs);
+}
+
+// time_point operator+(duration x, time_point y);
+
+template <class _Rep1, class _Period1, class _Clock, class _Duration2>
+inline _LIBCPP_HIDE_FROM_ABI
+_LIBCPP_CONSTEXPR_SINCE_CXX14 time_point<_Clock, typename common_type<duration<_Rep1, _Period1>, _Duration2>::type>
+operator+(const duration<_Rep1, _Period1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) {
+  return __rhs + __lhs;
+}
+
+// time_point operator-(time_point x, duration y);
+
+template <class _Clock, class _Duration1, class _Rep2, class _Period2>
+inline _LIBCPP_HIDE_FROM_ABI
+_LIBCPP_CONSTEXPR_SINCE_CXX14 time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type>
+operator-(const time_point<_Clock, _Duration1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
+  typedef time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type> _Ret;
+  return _Ret(__lhs.time_since_epoch() - __rhs);
+}
+
+// duration operator-(time_point x, time_point y);
+
+template <class _Clock, class _Duration1, class _Duration2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename common_type<_Duration1, _Duration2>::type
+operator-(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) {
+  return __lhs.time_since_epoch() - __rhs.time_since_epoch();
+}
+
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___CHRONO_TIME_POINT_H
diff --git a/libcxx/include/__cxx03/__chrono/time_zone.h b/libcxx/include/__cxx03/__chrono/time_zone.h
new file mode 100644
index 000000000000..de11dac1eef0
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/time_zone.h
@@ -0,0 +1,182 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
+
+#ifndef _LIBCPP___CHRONO_TIME_ZONE_H
+#define _LIBCPP___CHRONO_TIME_ZONE_H
+
+#include <version>
+// Enable the contents of the header only when libc++ was built with experimental features enabled.
+#if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#  include <__chrono/calendar.h>
+#  include <__chrono/duration.h>
+#  include <__chrono/exception.h>
+#  include <__chrono/local_info.h>
+#  include <__chrono/sys_info.h>
+#  include <__chrono/system_clock.h>
+#  include <__compare/strong_order.h>
+#  include <__config>
+#  include <__memory/unique_ptr.h>
+#  include <__type_traits/common_type.h>
+#  include <string_view>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&   \
+      !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+
+namespace chrono {
+
+enum class choose { earliest, latest };
+
+class _LIBCPP_AVAILABILITY_TZDB time_zone {
+  _LIBCPP_HIDE_FROM_ABI time_zone() = default;
+
+public:
+  class __impl; // public so it can be used by make_unique.
+
+  // The "constructor".
+  //
+  // The default constructor is private to avoid the constructor from being
+  // part of the ABI. Instead use an __ugly_named function as an ABI interface,
+  // since that gives us the ability to change it in the future.
+  [[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI static time_zone __create(unique_ptr<__impl>&& __p);
+
+  _LIBCPP_EXPORTED_FROM_ABI ~time_zone();
+
+  _LIBCPP_HIDE_FROM_ABI time_zone(time_zone&&)            = default;
+  _LIBCPP_HIDE_FROM_ABI time_zone& operator=(time_zone&&) = default;
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI string_view name() const noexcept { return __name(); }
+
+  template <class _Duration>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI sys_info get_info(const sys_time<_Duration>& __time) const {
+    return __get_info(chrono::time_point_cast<seconds>(__time));
+  }
+
+  template <class _Duration>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI local_info get_info(const local_time<_Duration>& __time) const {
+    return __get_info(chrono::time_point_cast<seconds>(__time));
+  }
+
+  // We don't apply nodiscard here since this function throws on many inputs,
+  // so it could be used as a validation.
+  template <class _Duration>
+  _LIBCPP_HIDE_FROM_ABI sys_time<common_type_t<_Duration, seconds>> to_sys(const local_time<_Duration>& __time) const {
+    local_info __info = get_info(__time);
+    switch (__info.result) {
+    case local_info::unique:
+      return sys_time<common_type_t<_Duration, seconds>>{__time.time_since_epoch() - __info.first.offset};
+
+    case local_info::nonexistent:
+      chrono::__throw_nonexistent_local_time(__time, __info);
+
+    case local_info::ambiguous:
+      chrono::__throw_ambiguous_local_time(__time, __info);
+    }
+
+    // TODO TZDB The Standard does not specify anything in these cases.
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __info.result != -1, "cannot convert the local time; it would be before the minimum system clock value");
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __info.result != -2, "cannot convert the local time; it would be after the maximum system clock value");
+
+    return {};
+  }
+
+  template <class _Duration>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI sys_time<common_type_t<_Duration, seconds>>
+  to_sys(const local_time<_Duration>& __time, choose __z) const {
+    local_info __info = get_info(__time);
+    switch (__info.result) {
+    case local_info::unique:
+    case local_info::nonexistent: // first and second are the same
+      return sys_time<common_type_t<_Duration, seconds>>{__time.time_since_epoch() - __info.first.offset};
+
+    case local_info::ambiguous:
+      switch (__z) {
+      case choose::earliest:
+        return sys_time<common_type_t<_Duration, seconds>>{__time.time_since_epoch() - __info.first.offset};
+
+      case choose::latest:
+        return sys_time<common_type_t<_Duration, seconds>>{__time.time_since_epoch() - __info.second.offset};
+
+        // Note a value out of bounds is not specified.
+      }
+    }
+
+    // TODO TZDB The standard does not specify anything in these cases.
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __info.result != -1, "cannot convert the local time; it would be before the minimum system clock value");
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __info.result != -2, "cannot convert the local time; it would be after the maximum system clock value");
+
+    return {};
+  }
+
+  template <class _Duration>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI local_time<common_type_t<_Duration, seconds>>
+  to_local(const sys_time<_Duration>& __time) const {
+    using _Dp = common_type_t<_Duration, seconds>;
+
+    sys_info __info = get_info(__time);
+
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __info.offset >= chrono::seconds{0} || __time.time_since_epoch() >= _Dp::min() - __info.offset,
+        "cannot convert the system time; it would be before the minimum local clock value");
+
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __info.offset <= chrono::seconds{0} || __time.time_since_epoch() <= _Dp::max() - __info.offset,
+        "cannot convert the system time; it would be after the maximum local clock value");
+
+    return local_time<_Dp>{__time.time_since_epoch() + __info.offset};
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI const __impl& __implementation() const noexcept { return *__impl_; }
+
+private:
+  [[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI string_view __name() const noexcept;
+
+  [[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_EXPORTED_FROM_ABI sys_info __get_info(sys_seconds __time) const;
+  [[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_EXPORTED_FROM_ABI local_info __get_info(local_seconds __time) const;
+
+  unique_ptr<__impl> __impl_;
+};
+
+[[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_HIDE_FROM_ABI inline bool
+operator==(const time_zone& __x, const time_zone& __y) noexcept {
+  return __x.name() == __y.name();
+}
+
+[[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_HIDE_FROM_ABI inline strong_ordering
+operator<=>(const time_zone& __x, const time_zone& __y) noexcept {
+  return __x.name() <=> __y.name();
+}
+
+} // namespace chrono
+
+#  endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM)
+         // && !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#endif // _LIBCPP___CHRONO_TIME_ZONE_H
diff --git a/libcxx/include/__cxx03/__chrono/time_zone_link.h b/libcxx/include/__cxx03/__chrono/time_zone_link.h
new file mode 100644
index 000000000000..b2d365c5fd08
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/time_zone_link.h
@@ -0,0 +1,79 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
+
+#ifndef _LIBCPP___CHRONO_TIME_ZONE_LINK_H
+#define _LIBCPP___CHRONO_TIME_ZONE_LINK_H
+
+#include <version>
+// Enable the contents of the header only when libc++ was built with experimental features enabled.
+#if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#  include <__compare/strong_order.h>
+#  include <__config>
+#  include <__utility/private_constructor_tag.h>
+#  include <string>
+#  include <string_view>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&   \
+      !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+
+namespace chrono {
+
+class time_zone_link {
+public:
+  [[nodiscard]]
+  _LIBCPP_HIDE_FROM_ABI explicit time_zone_link(__private_constructor_tag, string_view __name, string_view __target)
+      : __name_{__name}, __target_{__target} {}
+
+  _LIBCPP_HIDE_FROM_ABI time_zone_link(time_zone_link&&)            = default;
+  _LIBCPP_HIDE_FROM_ABI time_zone_link& operator=(time_zone_link&&) = default;
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI string_view name() const noexcept { return __name_; }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI string_view target() const noexcept { return __target_; }
+
+private:
+  string __name_;
+  // TODO TZDB instead of the name we can store the pointer to a zone. These
+  // pointers are immutable. This makes it possible to directly return a
+  // pointer in the time_zone in the 'locate_zone' function.
+  string __target_;
+};
+
+[[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_HIDE_FROM_ABI inline bool
+operator==(const time_zone_link& __x, const time_zone_link& __y) noexcept {
+  return __x.name() == __y.name();
+}
+
+[[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_HIDE_FROM_ABI inline strong_ordering
+operator<=>(const time_zone_link& __x, const time_zone_link& __y) noexcept {
+  return __x.name() <=> __y.name();
+}
+
+} // namespace chrono
+
+#  endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#endif // _LIBCPP___CHRONO_TIME_ZONE_LINK_H
diff --git a/libcxx/include/__cxx03/__chrono/tzdb.h b/libcxx/include/__cxx03/__chrono/tzdb.h
new file mode 100644
index 000000000000..f731f8c318be
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/tzdb.h
@@ -0,0 +1,94 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
+
+#ifndef _LIBCPP___CHRONO_TZDB_H
+#define _LIBCPP___CHRONO_TZDB_H
+
+#include <version>
+// Enable the contents of the header only when libc++ was built with experimental features enabled.
+#if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#  include <__algorithm/ranges_lower_bound.h>
+#  include <__chrono/leap_second.h>
+#  include <__chrono/time_zone.h>
+#  include <__chrono/time_zone_link.h>
+#  include <__config>
+#  include <string>
+#  include <vector>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&   \
+      !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+
+namespace chrono {
+
+struct tzdb {
+  string version;
+  vector<time_zone> zones;
+  vector<time_zone_link> links;
+
+  vector<leap_second> leap_seconds;
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI const time_zone* __locate_zone(string_view __name) const {
+    if (const time_zone* __result = __find_in_zone(__name))
+      return __result;
+
+    if (auto __it = ranges::lower_bound(links, __name, {}, &time_zone_link::name);
+        __it != links.end() && __it->name() == __name)
+      if (const time_zone* __result = __find_in_zone(__it->target()))
+        return __result;
+
+    return nullptr;
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI const time_zone* locate_zone(string_view __name) const {
+    if (const time_zone* __result = __locate_zone(__name))
+      return __result;
+
+    std::__throw_runtime_error("tzdb: requested time zone not found");
+  }
+
+  [[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_HIDE_FROM_ABI const time_zone* current_zone() const {
+    return __current_zone();
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI const time_zone* __find_in_zone(string_view __name) const noexcept {
+    if (auto __it = ranges::lower_bound(zones, __name, {}, &time_zone::name);
+        __it != zones.end() && __it->name() == __name)
+      return std::addressof(*__it);
+
+    return nullptr;
+  }
+
+  [[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_EXPORTED_FROM_ABI const time_zone* __current_zone() const;
+};
+
+} // namespace chrono
+
+#  endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM)
+         // && !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#endif // _LIBCPP___CHRONO_TZDB_H
diff --git a/libcxx/include/__cxx03/__chrono/tzdb_list.h b/libcxx/include/__cxx03/__chrono/tzdb_list.h
new file mode 100644
index 000000000000..aeef4fe1aba3
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/tzdb_list.h
@@ -0,0 +1,108 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
+
+#ifndef _LIBCPP___CHRONO_TZDB_LIST_H
+#define _LIBCPP___CHRONO_TZDB_LIST_H
+
+#include <version>
+// Enable the contents of the header only when libc++ was built with experimental features enabled.
+#if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#  include <__chrono/time_zone.h>
+#  include <__chrono/tzdb.h>
+#  include <__config>
+#  include <__fwd/string.h>
+#  include <forward_list>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&   \
+      !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+
+namespace chrono {
+
+// TODO TZDB
+// Libc++ recently switched to only export __ugly_names from the dylib.
+// Since the library is still experimental the functions in this header
+// should be adapted to this new style. The other tzdb headers should be
+// evaluated too.
+
+class _LIBCPP_AVAILABILITY_TZDB tzdb_list {
+public:
+  class __impl; // public to allow construction in dylib
+  _LIBCPP_HIDE_FROM_ABI explicit tzdb_list(__impl* __p) : __impl_(__p) {
+    _LIBCPP_ASSERT_NON_NULL(__impl_ != nullptr, "initialized time_zone without a valid pimpl object");
+  }
+  _LIBCPP_EXPORTED_FROM_ABI ~tzdb_list();
+
+  tzdb_list(const tzdb_list&)            = delete;
+  tzdb_list& operator=(const tzdb_list&) = delete;
+
+  using const_iterator = forward_list<tzdb>::const_iterator;
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI const tzdb& front() const noexcept { return __front(); }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator erase_after(const_iterator __p) { return __erase_after(__p); }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI const_iterator begin() const noexcept { return __begin(); }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI const_iterator end() const noexcept { return __end(); }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const noexcept { return __cbegin(); }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI const_iterator cend() const noexcept { return __cend(); }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI __impl& __implementation() { return *__impl_; }
+
+private:
+  [[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI const tzdb& __front() const noexcept;
+
+  _LIBCPP_EXPORTED_FROM_ABI const_iterator __erase_after(const_iterator __p);
+
+  [[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI const_iterator __begin() const noexcept;
+  [[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI const_iterator __end() const noexcept;
+
+  [[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI const_iterator __cbegin() const noexcept;
+  [[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI const_iterator __cend() const noexcept;
+
+  __impl* __impl_;
+};
+
+[[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_EXPORTED_FROM_ABI tzdb_list& get_tzdb_list();
+
+[[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_HIDE_FROM_ABI inline const tzdb& get_tzdb() {
+  return get_tzdb_list().front();
+}
+
+[[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_HIDE_FROM_ABI inline const time_zone* locate_zone(string_view __name) {
+  return get_tzdb().locate_zone(__name);
+}
+
+[[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_HIDE_FROM_ABI inline const time_zone* current_zone() {
+  return get_tzdb().current_zone();
+}
+
+_LIBCPP_AVAILABILITY_TZDB _LIBCPP_EXPORTED_FROM_ABI const tzdb& reload_tzdb();
+
+[[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_EXPORTED_FROM_ABI string remote_version();
+
+} // namespace chrono
+
+#  endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM)
+         // && !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#endif // _LIBCPP___CHRONO_TZDB_LIST_H
diff --git a/libcxx/include/__cxx03/__chrono/weekday.h b/libcxx/include/__cxx03/__chrono/weekday.h
new file mode 100644
index 000000000000..86c780cc7182
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/weekday.h
@@ -0,0 +1,186 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_WEEKDAY_H
+#define _LIBCPP___CHRONO_WEEKDAY_H
+
+#include <__chrono/calendar.h>
+#include <__chrono/duration.h>
+#include <__chrono/system_clock.h>
+#include <__chrono/time_point.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+class weekday_indexed;
+class weekday_last;
+
+class weekday {
+private:
+  unsigned char __wd_;
+  _LIBCPP_HIDE_FROM_ABI static constexpr unsigned char __weekday_from_days(int __days) noexcept;
+
+public:
+  weekday() = default;
+  _LIBCPP_HIDE_FROM_ABI inline explicit constexpr weekday(unsigned __val) noexcept
+      : __wd_(static_cast<unsigned char>(__val == 7 ? 0 : __val)) {}
+  _LIBCPP_HIDE_FROM_ABI inline constexpr weekday(const sys_days& __sysd) noexcept
+      : __wd_(__weekday_from_days(__sysd.time_since_epoch().count())) {}
+  _LIBCPP_HIDE_FROM_ABI inline explicit constexpr weekday(const local_days& __locd) noexcept
+      : __wd_(__weekday_from_days(__locd.time_since_epoch().count())) {}
+
+  _LIBCPP_HIDE_FROM_ABI inline constexpr weekday& operator++() noexcept {
+    __wd_ = (__wd_ == 6 ? 0 : __wd_ + 1);
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr weekday operator++(int) noexcept {
+    weekday __tmp = *this;
+    ++(*this);
+    return __tmp;
+  }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr weekday& operator--() noexcept {
+    __wd_ = (__wd_ == 0 ? 6 : __wd_ - 1);
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr weekday operator--(int) noexcept {
+    weekday __tmp = *this;
+    --(*this);
+    return __tmp;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr weekday& operator+=(const days& __dd) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr weekday& operator-=(const days& __dd) noexcept;
+  _LIBCPP_HIDE_FROM_ABI inline constexpr unsigned c_encoding() const noexcept { return __wd_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr unsigned iso_encoding() const noexcept { return __wd_ == 0u ? 7 : __wd_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __wd_ <= 6; }
+  _LIBCPP_HIDE_FROM_ABI constexpr weekday_indexed operator[](unsigned __index) const noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr weekday_last operator[](last_spec) const noexcept;
+};
+
+// https://howardhinnant.github.io/date_algorithms.html#weekday_from_days
+_LIBCPP_HIDE_FROM_ABI inline constexpr unsigned char weekday::__weekday_from_days(int __days) noexcept {
+  return static_cast<unsigned char>(static_cast<unsigned>(__days >= -4 ? (__days + 4) % 7 : (__days + 5) % 7 + 6));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const weekday& __lhs, const weekday& __rhs) noexcept {
+  return __lhs.c_encoding() == __rhs.c_encoding();
+}
+
+// TODO(LLVM 20): Remove the escape hatch
+#  ifdef _LIBCPP_ENABLE_REMOVED_WEEKDAY_RELATIONAL_OPERATORS
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator<(const weekday& __lhs, const weekday& __rhs) noexcept {
+  return __lhs.c_encoding() < __rhs.c_encoding();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator>(const weekday& __lhs, const weekday& __rhs) noexcept {
+  return __rhs < __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator<=(const weekday& __lhs, const weekday& __rhs) noexcept {
+  return !(__rhs < __lhs);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator>=(const weekday& __lhs, const weekday& __rhs) noexcept {
+  return !(__lhs < __rhs);
+}
+#  endif // _LIBCPP_ENABLE_REMOVED_WEEKDAY_RELATIONAL_OPERATORS
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr weekday operator+(const weekday& __lhs, const days& __rhs) noexcept {
+  auto const __mu = static_cast<long long>(__lhs.c_encoding()) + __rhs.count();
+  auto const __yr = (__mu >= 0 ? __mu : __mu - 6) / 7;
+  return weekday{static_cast<unsigned>(__mu - __yr * 7)};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr weekday operator+(const days& __lhs, const weekday& __rhs) noexcept {
+  return __rhs + __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr weekday operator-(const weekday& __lhs, const days& __rhs) noexcept {
+  return __lhs + -__rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr days operator-(const weekday& __lhs, const weekday& __rhs) noexcept {
+  const int __wdu = __lhs.c_encoding() - __rhs.c_encoding();
+  const int __wk  = (__wdu >= 0 ? __wdu : __wdu - 6) / 7;
+  return days{__wdu - __wk * 7};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr weekday& weekday::operator+=(const days& __dd) noexcept {
+  *this = *this + __dd;
+  return *this;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr weekday& weekday::operator-=(const days& __dd) noexcept {
+  *this = *this - __dd;
+  return *this;
+}
+
+class weekday_indexed {
+private:
+  chrono::weekday __wd_;
+  unsigned char __idx_;
+
+public:
+  weekday_indexed() = default;
+  _LIBCPP_HIDE_FROM_ABI inline constexpr weekday_indexed(const chrono::weekday& __wdval, unsigned __idxval) noexcept
+      : __wd_{__wdval}, __idx_(__idxval) {}
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday weekday() const noexcept { return __wd_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr unsigned index() const noexcept { return __idx_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __wd_.ok() && __idx_ >= 1 && __idx_ <= 5; }
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool
+operator==(const weekday_indexed& __lhs, const weekday_indexed& __rhs) noexcept {
+  return __lhs.weekday() == __rhs.weekday() && __lhs.index() == __rhs.index();
+}
+
+class weekday_last {
+private:
+  chrono::weekday __wd_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr weekday_last(const chrono::weekday& __val) noexcept : __wd_{__val} {}
+  _LIBCPP_HIDE_FROM_ABI constexpr chrono::weekday weekday() const noexcept { return __wd_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr bool ok() const noexcept { return __wd_.ok(); }
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const weekday_last& __lhs, const weekday_last& __rhs) noexcept {
+  return __lhs.weekday() == __rhs.weekday();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr weekday_indexed weekday::operator[](unsigned __index) const noexcept {
+  return weekday_indexed{*this, __index};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr weekday_last weekday::operator[](last_spec) const noexcept {
+  return weekday_last{*this};
+}
+
+inline constexpr weekday Sunday{0};
+inline constexpr weekday Monday{1};
+inline constexpr weekday Tuesday{2};
+inline constexpr weekday Wednesday{3};
+inline constexpr weekday Thursday{4};
+inline constexpr weekday Friday{5};
+inline constexpr weekday Saturday{6};
+
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___CHRONO_WEEKDAY_H
diff --git a/libcxx/include/__cxx03/__chrono/year.h b/libcxx/include/__cxx03/__chrono/year.h
new file mode 100644
index 000000000000..1899d09f38db
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/year.h
@@ -0,0 +1,118 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_YEAR_H
+#define _LIBCPP___CHRONO_YEAR_H
+
+#include <__chrono/duration.h>
+#include <__config>
+#include <compare>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+class year {
+private:
+  short __y_;
+
+public:
+  year() = default;
+  _LIBCPP_HIDE_FROM_ABI explicit inline constexpr year(int __val) noexcept : __y_(static_cast<short>(__val)) {}
+
+  _LIBCPP_HIDE_FROM_ABI inline constexpr year& operator++() noexcept {
+    ++__y_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr year operator++(int) noexcept {
+    year __tmp = *this;
+    ++(*this);
+    return __tmp;
+  }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr year& operator--() noexcept {
+    --__y_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr year operator--(int) noexcept {
+    year __tmp = *this;
+    --(*this);
+    return __tmp;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr year& operator+=(const years& __dy) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr year& operator-=(const years& __dy) noexcept;
+  _LIBCPP_HIDE_FROM_ABI inline constexpr year operator+() const noexcept { return *this; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr year operator-() const noexcept { return year{-__y_}; }
+
+  _LIBCPP_HIDE_FROM_ABI inline constexpr bool is_leap() const noexcept {
+    return __y_ % 4 == 0 && (__y_ % 100 != 0 || __y_ % 400 == 0);
+  }
+  _LIBCPP_HIDE_FROM_ABI explicit inline constexpr operator int() const noexcept { return __y_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr bool ok() const noexcept;
+  _LIBCPP_HIDE_FROM_ABI static inline constexpr year min() noexcept { return year{-32767}; }
+  _LIBCPP_HIDE_FROM_ABI static inline constexpr year max() noexcept { return year{32767}; }
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const year& __lhs, const year& __rhs) noexcept {
+  return static_cast<int>(__lhs) == static_cast<int>(__rhs);
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr strong_ordering operator<=>(const year& __lhs, const year& __rhs) noexcept {
+  return static_cast<int>(__lhs) <=> static_cast<int>(__rhs);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year operator+(const year& __lhs, const years& __rhs) noexcept {
+  return year(static_cast<int>(__lhs) + __rhs.count());
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year operator+(const years& __lhs, const year& __rhs) noexcept {
+  return __rhs + __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year operator-(const year& __lhs, const years& __rhs) noexcept {
+  return __lhs + -__rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr years operator-(const year& __lhs, const year& __rhs) noexcept {
+  return years{static_cast<int>(__lhs) - static_cast<int>(__rhs)};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year& year::operator+=(const years& __dy) noexcept {
+  *this = *this + __dy;
+  return *this;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year& year::operator-=(const years& __dy) noexcept {
+  *this = *this - __dy;
+  return *this;
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr bool year::ok() const noexcept {
+  static_assert(static_cast<int>(std::numeric_limits<decltype(__y_)>::max()) == static_cast<int>(max()));
+  return static_cast<int>(min()) <= __y_;
+}
+
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___CHRONO_YEAR_H
diff --git a/libcxx/include/__cxx03/__chrono/year_month.h b/libcxx/include/__cxx03/__chrono/year_month.h
new file mode 100644
index 000000000000..369ea38f7560
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/year_month.h
@@ -0,0 +1,123 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_YEAR_MONTH_H
+#define _LIBCPP___CHRONO_YEAR_MONTH_H
+
+#include <__chrono/duration.h>
+#include <__chrono/month.h>
+#include <__chrono/year.h>
+#include <__config>
+#include <compare>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+class year_month {
+  chrono::year __y_;
+  chrono::month __m_;
+
+public:
+  year_month() = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month(const chrono::year& __yval, const chrono::month& __mval) noexcept
+      : __y_{__yval}, __m_{__mval} {}
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::year year() const noexcept { return __y_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr year_month& operator+=(const months& __dm) noexcept;
+  _LIBCPP_HIDE_FROM_ABI inline constexpr year_month& operator-=(const months& __dm) noexcept;
+  _LIBCPP_HIDE_FROM_ABI inline constexpr year_month& operator+=(const years& __dy) noexcept;
+  _LIBCPP_HIDE_FROM_ABI inline constexpr year_month& operator-=(const years& __dy) noexcept;
+  _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __y_.ok() && __m_.ok(); }
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator/(const year& __y, const month& __m) noexcept {
+  return year_month{__y, __m};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator/(const year& __y, int __m) noexcept {
+  return year_month{__y, month(__m)};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const year_month& __lhs, const year_month& __rhs) noexcept {
+  return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering
+operator<=>(const year_month& __lhs, const year_month& __rhs) noexcept {
+  if (auto __c = __lhs.year() <=> __rhs.year(); __c != 0)
+    return __c;
+  return __lhs.month() <=> __rhs.month();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator+(const year_month& __lhs, const months& __rhs) noexcept {
+  int __dmi      = static_cast<int>(static_cast<unsigned>(__lhs.month())) - 1 + __rhs.count();
+  const int __dy = (__dmi >= 0 ? __dmi : __dmi - 11) / 12;
+  __dmi          = __dmi - __dy * 12 + 1;
+  return (__lhs.year() + years(__dy)) / month(static_cast<unsigned>(__dmi));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator+(const months& __lhs, const year_month& __rhs) noexcept {
+  return __rhs + __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator+(const year_month& __lhs, const years& __rhs) noexcept {
+  return (__lhs.year() + __rhs) / __lhs.month();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator+(const years& __lhs, const year_month& __rhs) noexcept {
+  return __rhs + __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr months operator-(const year_month& __lhs, const year_month& __rhs) noexcept {
+  return (__lhs.year() - __rhs.year()) +
+         months(static_cast<unsigned>(__lhs.month()) - static_cast<unsigned>(__rhs.month()));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator-(const year_month& __lhs, const months& __rhs) noexcept {
+  return __lhs + -__rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator-(const year_month& __lhs, const years& __rhs) noexcept {
+  return __lhs + -__rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month& year_month::operator+=(const months& __dm) noexcept {
+  *this = *this + __dm;
+  return *this;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month& year_month::operator-=(const months& __dm) noexcept {
+  *this = *this - __dm;
+  return *this;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month& year_month::operator+=(const years& __dy) noexcept {
+  *this = *this + __dy;
+  return *this;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month& year_month::operator-=(const years& __dy) noexcept {
+  *this = *this - __dy;
+  return *this;
+}
+
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___CHRONO_YEAR_MONTH_H
diff --git a/libcxx/include/__cxx03/__chrono/year_month_day.h b/libcxx/include/__cxx03/__chrono/year_month_day.h
new file mode 100644
index 000000000000..b06c0be03e0d
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/year_month_day.h
@@ -0,0 +1,337 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_YEAR_MONTH_DAY_H
+#define _LIBCPP___CHRONO_YEAR_MONTH_DAY_H
+
+#include <__chrono/calendar.h>
+#include <__chrono/day.h>
+#include <__chrono/duration.h>
+#include <__chrono/month.h>
+#include <__chrono/monthday.h>
+#include <__chrono/system_clock.h>
+#include <__chrono/time_point.h>
+#include <__chrono/year.h>
+#include <__chrono/year_month.h>
+#include <__config>
+#include <compare>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+class year_month_day_last;
+
+class year_month_day {
+private:
+  chrono::year __y_;
+  chrono::month __m_;
+  chrono::day __d_;
+
+public:
+  year_month_day() = default;
+  _LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day(
+      const chrono::year& __yval, const chrono::month& __mval, const chrono::day& __dval) noexcept
+      : __y_{__yval}, __m_{__mval}, __d_{__dval} {}
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_day(const year_month_day_last& __ymdl) noexcept;
+  _LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day(const sys_days& __sysd) noexcept
+      : year_month_day(__from_days(__sysd.time_since_epoch())) {}
+  _LIBCPP_HIDE_FROM_ABI inline explicit constexpr year_month_day(const local_days& __locd) noexcept
+      : year_month_day(__from_days(__locd.time_since_epoch())) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_day& operator+=(const months& __dm) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_day& operator-=(const months& __dm) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_day& operator+=(const years& __dy) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_day& operator-=(const years& __dy) noexcept;
+
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::year year() const noexcept { return __y_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::day day() const noexcept { return __d_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr operator sys_days() const noexcept { return sys_days{__to_days()}; }
+  _LIBCPP_HIDE_FROM_ABI inline explicit constexpr operator local_days() const noexcept {
+    return local_days{__to_days()};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool ok() const noexcept;
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr year_month_day __from_days(days __d) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr days __to_days() const noexcept;
+};
+
+// https://howardhinnant.github.io/date_algorithms.html#civil_from_days
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day year_month_day::__from_days(days __d) noexcept {
+  static_assert(numeric_limits<unsigned>::digits >= 18, "");
+  static_assert(numeric_limits<int>::digits >= 20, "");
+  const int __z        = __d.count() + 719468;
+  const int __era      = (__z >= 0 ? __z : __z - 146096) / 146097;
+  const unsigned __doe = static_cast<unsigned>(__z - __era * 146097);                   // [0, 146096]
+  const unsigned __yoe = (__doe - __doe / 1460 + __doe / 36524 - __doe / 146096) / 365; // [0, 399]
+  const int __yr       = static_cast<int>(__yoe) + __era * 400;
+  const unsigned __doy = __doe - (365 * __yoe + __yoe / 4 - __yoe / 100); // [0, 365]
+  const unsigned __mp  = (5 * __doy + 2) / 153;                           // [0, 11]
+  const unsigned __dy  = __doy - (153 * __mp + 2) / 5 + 1;                // [1, 31]
+  const unsigned __mth = __mp + (__mp < 10 ? 3 : -9);                     // [1, 12]
+  return year_month_day{chrono::year{__yr + (__mth <= 2)}, chrono::month{__mth}, chrono::day{__dy}};
+}
+
+// https://howardhinnant.github.io/date_algorithms.html#days_from_civil
+_LIBCPP_HIDE_FROM_ABI inline constexpr days year_month_day::__to_days() const noexcept {
+  static_assert(numeric_limits<unsigned>::digits >= 18, "");
+  static_assert(numeric_limits<int>::digits >= 20, "");
+
+  const int __yr       = static_cast<int>(__y_) - (__m_ <= February);
+  const unsigned __mth = static_cast<unsigned>(__m_);
+  const unsigned __dy  = static_cast<unsigned>(__d_);
+
+  const int __era      = (__yr >= 0 ? __yr : __yr - 399) / 400;
+  const unsigned __yoe = static_cast<unsigned>(__yr - __era * 400);                 // [0, 399]
+  const unsigned __doy = (153 * (__mth + (__mth > 2 ? -3 : 9)) + 2) / 5 + __dy - 1; // [0, 365]
+  const unsigned __doe = __yoe * 365 + __yoe / 4 - __yoe / 100 + __doy;             // [0, 146096]
+  return days{__era * 146097 + static_cast<int>(__doe) - 719468};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool
+operator==(const year_month_day& __lhs, const year_month_day& __rhs) noexcept {
+  return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month() && __lhs.day() == __rhs.day();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering
+operator<=>(const year_month_day& __lhs, const year_month_day& __rhs) noexcept {
+  if (auto __c = __lhs.year() <=> __rhs.year(); __c != 0)
+    return __c;
+  if (auto __c = __lhs.month() <=> __rhs.month(); __c != 0)
+    return __c;
+  return __lhs.day() <=> __rhs.day();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day operator/(const year_month& __lhs, const day& __rhs) noexcept {
+  return year_month_day{__lhs.year(), __lhs.month(), __rhs};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day operator/(const year_month& __lhs, int __rhs) noexcept {
+  return __lhs / day(__rhs);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day operator/(const year& __lhs, const month_day& __rhs) noexcept {
+  return __lhs / __rhs.month() / __rhs.day();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day operator/(int __lhs, const month_day& __rhs) noexcept {
+  return year(__lhs) / __rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day operator/(const month_day& __lhs, const year& __rhs) noexcept {
+  return __rhs / __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day operator/(const month_day& __lhs, int __rhs) noexcept {
+  return year(__rhs) / __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day
+operator+(const year_month_day& __lhs, const months& __rhs) noexcept {
+  return (__lhs.year() / __lhs.month() + __rhs) / __lhs.day();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day
+operator+(const months& __lhs, const year_month_day& __rhs) noexcept {
+  return __rhs + __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day
+operator-(const year_month_day& __lhs, const months& __rhs) noexcept {
+  return __lhs + -__rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day
+operator+(const year_month_day& __lhs, const years& __rhs) noexcept {
+  return (__lhs.year() + __rhs) / __lhs.month() / __lhs.day();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day
+operator+(const years& __lhs, const year_month_day& __rhs) noexcept {
+  return __rhs + __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day
+operator-(const year_month_day& __lhs, const years& __rhs) noexcept {
+  return __lhs + -__rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day& year_month_day::operator+=(const months& __dm) noexcept {
+  *this = *this + __dm;
+  return *this;
+}
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day& year_month_day::operator-=(const months& __dm) noexcept {
+  *this = *this - __dm;
+  return *this;
+}
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day& year_month_day::operator+=(const years& __dy) noexcept {
+  *this = *this + __dy;
+  return *this;
+}
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day& year_month_day::operator-=(const years& __dy) noexcept {
+  *this = *this - __dy;
+  return *this;
+}
+
+class year_month_day_last {
+private:
+  chrono::year __y_;
+  chrono::month_day_last __mdl_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_day_last(const year& __yval, const month_day_last& __mdlval) noexcept
+      : __y_{__yval}, __mdl_{__mdlval} {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_day_last& operator+=(const months& __m) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_day_last& operator-=(const months& __m) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_day_last& operator+=(const years& __y) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_day_last& operator-=(const years& __y) noexcept;
+
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::year year() const noexcept { return __y_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __mdl_.month(); }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month_day_last month_day_last() const noexcept { return __mdl_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr chrono::day day() const noexcept;
+  _LIBCPP_HIDE_FROM_ABI inline constexpr operator sys_days() const noexcept {
+    return sys_days{year() / month() / day()};
+  }
+  _LIBCPP_HIDE_FROM_ABI inline explicit constexpr operator local_days() const noexcept {
+    return local_days{year() / month() / day()};
+  }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __y_.ok() && __mdl_.ok(); }
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr chrono::day year_month_day_last::day() const noexcept {
+  constexpr chrono::day __d[] = {
+      chrono::day(31),
+      chrono::day(28),
+      chrono::day(31),
+      chrono::day(30),
+      chrono::day(31),
+      chrono::day(30),
+      chrono::day(31),
+      chrono::day(31),
+      chrono::day(30),
+      chrono::day(31),
+      chrono::day(30),
+      chrono::day(31)};
+  return (month() != February || !__y_.is_leap()) && month().ok()
+           ? __d[static_cast<unsigned>(month()) - 1]
+           : chrono::day{29};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool
+operator==(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept {
+  return __lhs.year() == __rhs.year() && __lhs.month_day_last() == __rhs.month_day_last();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering
+operator<=>(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept {
+  if (auto __c = __lhs.year() <=> __rhs.year(); __c != 0)
+    return __c;
+  return __lhs.month_day_last() <=> __rhs.month_day_last();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last operator/(const year_month& __lhs, last_spec) noexcept {
+  return year_month_day_last{__lhs.year(), month_day_last{__lhs.month()}};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last
+operator/(const year& __lhs, const month_day_last& __rhs) noexcept {
+  return year_month_day_last{__lhs, __rhs};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last operator/(int __lhs, const month_day_last& __rhs) noexcept {
+  return year_month_day_last{year{__lhs}, __rhs};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last
+operator/(const month_day_last& __lhs, const year& __rhs) noexcept {
+  return __rhs / __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last operator/(const month_day_last& __lhs, int __rhs) noexcept {
+  return year{__rhs} / __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last
+operator+(const year_month_day_last& __lhs, const months& __rhs) noexcept {
+  return (__lhs.year() / __lhs.month() + __rhs) / last;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last
+operator+(const months& __lhs, const year_month_day_last& __rhs) noexcept {
+  return __rhs + __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last
+operator-(const year_month_day_last& __lhs, const months& __rhs) noexcept {
+  return __lhs + (-__rhs);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last
+operator+(const year_month_day_last& __lhs, const years& __rhs) noexcept {
+  return year_month_day_last{__lhs.year() + __rhs, __lhs.month_day_last()};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last
+operator+(const years& __lhs, const year_month_day_last& __rhs) noexcept {
+  return __rhs + __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last
+operator-(const year_month_day_last& __lhs, const years& __rhs) noexcept {
+  return __lhs + (-__rhs);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last&
+year_month_day_last::operator+=(const months& __dm) noexcept {
+  *this = *this + __dm;
+  return *this;
+}
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last&
+year_month_day_last::operator-=(const months& __dm) noexcept {
+  *this = *this - __dm;
+  return *this;
+}
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last&
+year_month_day_last::operator+=(const years& __dy) noexcept {
+  *this = *this + __dy;
+  return *this;
+}
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last&
+year_month_day_last::operator-=(const years& __dy) noexcept {
+  *this = *this - __dy;
+  return *this;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day::year_month_day(const year_month_day_last& __ymdl) noexcept
+    : __y_{__ymdl.year()}, __m_{__ymdl.month()}, __d_{__ymdl.day()} {}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool year_month_day::ok() const noexcept {
+  if (!__y_.ok() || !__m_.ok())
+    return false;
+  return chrono::day{1} <= __d_ && __d_ <= (__y_ / __m_ / last).day();
+}
+
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___CHRONO_YEAR_MONTH_DAY_H
diff --git a/libcxx/include/__cxx03/__chrono/year_month_weekday.h b/libcxx/include/__cxx03/__chrono/year_month_weekday.h
new file mode 100644
index 000000000000..0c3dd494c878
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/year_month_weekday.h
@@ -0,0 +1,287 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CHRONO_YEAR_MONTH_WEEKDAY_H
+#define _LIBCPP___CHRONO_YEAR_MONTH_WEEKDAY_H
+
+#include <__chrono/calendar.h>
+#include <__chrono/day.h>
+#include <__chrono/duration.h>
+#include <__chrono/month.h>
+#include <__chrono/month_weekday.h>
+#include <__chrono/system_clock.h>
+#include <__chrono/time_point.h>
+#include <__chrono/weekday.h>
+#include <__chrono/year.h>
+#include <__chrono/year_month.h>
+#include <__chrono/year_month_day.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+class year_month_weekday {
+  chrono::year __y_;
+  chrono::month __m_;
+  chrono::weekday_indexed __wdi_;
+
+public:
+  year_month_weekday() = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday(
+      const chrono::year& __yval, const chrono::month& __mval, const chrono::weekday_indexed& __wdival) noexcept
+      : __y_{__yval}, __m_{__mval}, __wdi_{__wdival} {}
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday(const sys_days& __sysd) noexcept
+      : year_month_weekday(__from_days(__sysd.time_since_epoch())) {}
+  _LIBCPP_HIDE_FROM_ABI inline explicit constexpr year_month_weekday(const local_days& __locd) noexcept
+      : year_month_weekday(__from_days(__locd.time_since_epoch())) {}
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday& operator+=(const months&) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday& operator-=(const months&) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday& operator+=(const years&) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday& operator-=(const years&) noexcept;
+
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::year year() const noexcept { return __y_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday weekday() const noexcept { return __wdi_.weekday(); }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr unsigned index() const noexcept { return __wdi_.index(); }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday_indexed weekday_indexed() const noexcept { return __wdi_; }
+
+  _LIBCPP_HIDE_FROM_ABI inline constexpr operator sys_days() const noexcept { return sys_days{__to_days()}; }
+  _LIBCPP_HIDE_FROM_ABI inline explicit constexpr operator local_days() const noexcept {
+    return local_days{__to_days()};
+  }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept {
+    if (!__y_.ok() || !__m_.ok() || !__wdi_.ok())
+      return false;
+    if (__wdi_.index() <= 4)
+      return true;
+    auto __nth_weekday_day =
+        __wdi_.weekday() - chrono::weekday{static_cast<sys_days>(__y_ / __m_ / 1)} + days{(__wdi_.index() - 1) * 7 + 1};
+    return static_cast<unsigned>(__nth_weekday_day.count()) <= static_cast<unsigned>((__y_ / __m_ / last).day());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr year_month_weekday __from_days(days __d) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr days __to_days() const noexcept;
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday year_month_weekday::__from_days(days __d) noexcept {
+  const sys_days __sysd{__d};
+  const chrono::weekday __wd = chrono::weekday(__sysd);
+  const year_month_day __ymd = year_month_day(__sysd);
+  return year_month_weekday{__ymd.year(), __ymd.month(), __wd[(static_cast<unsigned>(__ymd.day()) - 1) / 7 + 1]};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr days year_month_weekday::__to_days() const noexcept {
+  const sys_days __sysd = sys_days(__y_ / __m_ / 1);
+  return (__sysd + (__wdi_.weekday() - chrono::weekday(__sysd) + days{(__wdi_.index() - 1) * 7})).time_since_epoch();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool
+operator==(const year_month_weekday& __lhs, const year_month_weekday& __rhs) noexcept {
+  return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month() &&
+         __lhs.weekday_indexed() == __rhs.weekday_indexed();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday
+operator/(const year_month& __lhs, const weekday_indexed& __rhs) noexcept {
+  return year_month_weekday{__lhs.year(), __lhs.month(), __rhs};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday
+operator/(const year& __lhs, const month_weekday& __rhs) noexcept {
+  return year_month_weekday{__lhs, __rhs.month(), __rhs.weekday_indexed()};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday operator/(int __lhs, const month_weekday& __rhs) noexcept {
+  return year(__lhs) / __rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday
+operator/(const month_weekday& __lhs, const year& __rhs) noexcept {
+  return __rhs / __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday operator/(const month_weekday& __lhs, int __rhs) noexcept {
+  return year(__rhs) / __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday
+operator+(const year_month_weekday& __lhs, const months& __rhs) noexcept {
+  return (__lhs.year() / __lhs.month() + __rhs) / __lhs.weekday_indexed();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday
+operator+(const months& __lhs, const year_month_weekday& __rhs) noexcept {
+  return __rhs + __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday
+operator-(const year_month_weekday& __lhs, const months& __rhs) noexcept {
+  return __lhs + (-__rhs);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday
+operator+(const year_month_weekday& __lhs, const years& __rhs) noexcept {
+  return year_month_weekday{__lhs.year() + __rhs, __lhs.month(), __lhs.weekday_indexed()};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday
+operator+(const years& __lhs, const year_month_weekday& __rhs) noexcept {
+  return __rhs + __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday
+operator-(const year_month_weekday& __lhs, const years& __rhs) noexcept {
+  return __lhs + (-__rhs);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday& year_month_weekday::operator+=(const months& __dm) noexcept {
+  *this = *this + __dm;
+  return *this;
+}
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday& year_month_weekday::operator-=(const months& __dm) noexcept {
+  *this = *this - __dm;
+  return *this;
+}
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday& year_month_weekday::operator+=(const years& __dy) noexcept {
+  *this = *this + __dy;
+  return *this;
+}
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday& year_month_weekday::operator-=(const years& __dy) noexcept {
+  *this = *this - __dy;
+  return *this;
+}
+
+class year_month_weekday_last {
+private:
+  chrono::year __y_;
+  chrono::month __m_;
+  chrono::weekday_last __wdl_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday_last(
+      const chrono::year& __yval, const chrono::month& __mval, const chrono::weekday_last& __wdlval) noexcept
+      : __y_{__yval}, __m_{__mval}, __wdl_{__wdlval} {}
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday_last& operator+=(const months& __dm) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday_last& operator-=(const months& __dm) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday_last& operator+=(const years& __dy) noexcept;
+  _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday_last& operator-=(const years& __dy) noexcept;
+
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::year year() const noexcept { return __y_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday weekday() const noexcept { return __wdl_.weekday(); }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday_last weekday_last() const noexcept { return __wdl_; }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr operator sys_days() const noexcept { return sys_days{__to_days()}; }
+  _LIBCPP_HIDE_FROM_ABI inline explicit constexpr operator local_days() const noexcept {
+    return local_days{__to_days()};
+  }
+  _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __y_.ok() && __m_.ok() && __wdl_.ok(); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr days __to_days() const noexcept;
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr days year_month_weekday_last::__to_days() const noexcept {
+  const sys_days __last = sys_days{__y_ / __m_ / last};
+  return (__last - (chrono::weekday{__last} - __wdl_.weekday())).time_since_epoch();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool
+operator==(const year_month_weekday_last& __lhs, const year_month_weekday_last& __rhs) noexcept {
+  return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month() && __lhs.weekday_last() == __rhs.weekday_last();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last
+operator/(const year_month& __lhs, const weekday_last& __rhs) noexcept {
+  return year_month_weekday_last{__lhs.year(), __lhs.month(), __rhs};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last
+operator/(const year& __lhs, const month_weekday_last& __rhs) noexcept {
+  return year_month_weekday_last{__lhs, __rhs.month(), __rhs.weekday_last()};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last
+operator/(int __lhs, const month_weekday_last& __rhs) noexcept {
+  return year(__lhs) / __rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last
+operator/(const month_weekday_last& __lhs, const year& __rhs) noexcept {
+  return __rhs / __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last
+operator/(const month_weekday_last& __lhs, int __rhs) noexcept {
+  return year(__rhs) / __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last
+operator+(const year_month_weekday_last& __lhs, const months& __rhs) noexcept {
+  return (__lhs.year() / __lhs.month() + __rhs) / __lhs.weekday_last();
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last
+operator+(const months& __lhs, const year_month_weekday_last& __rhs) noexcept {
+  return __rhs + __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last
+operator-(const year_month_weekday_last& __lhs, const months& __rhs) noexcept {
+  return __lhs + (-__rhs);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last
+operator+(const year_month_weekday_last& __lhs, const years& __rhs) noexcept {
+  return year_month_weekday_last{__lhs.year() + __rhs, __lhs.month(), __lhs.weekday_last()};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last
+operator+(const years& __lhs, const year_month_weekday_last& __rhs) noexcept {
+  return __rhs + __lhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last
+operator-(const year_month_weekday_last& __lhs, const years& __rhs) noexcept {
+  return __lhs + (-__rhs);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last&
+year_month_weekday_last::operator+=(const months& __dm) noexcept {
+  *this = *this + __dm;
+  return *this;
+}
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last&
+year_month_weekday_last::operator-=(const months& __dm) noexcept {
+  *this = *this - __dm;
+  return *this;
+}
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last&
+year_month_weekday_last::operator+=(const years& __dy) noexcept {
+  *this = *this + __dy;
+  return *this;
+}
+_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last&
+year_month_weekday_last::operator-=(const years& __dy) noexcept {
+  *this = *this - __dy;
+  return *this;
+}
+
+} // namespace chrono
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___CHRONO_YEAR_MONTH_WEEKDAY_H
diff --git a/libcxx/include/__cxx03/__chrono/zoned_time.h b/libcxx/include/__cxx03/__chrono/zoned_time.h
new file mode 100644
index 000000000000..8cfa2122642c
--- /dev/null
+++ b/libcxx/include/__cxx03/__chrono/zoned_time.h
@@ -0,0 +1,227 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
+
+#ifndef _LIBCPP___CHRONO_ZONED_TIME_H
+#define _LIBCPP___CHRONO_ZONED_TIME_H
+
+#include <version>
+// Enable the contents of the header only when libc++ was built with experimental features enabled.
+#if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#  include <__chrono/calendar.h>
+#  include <__chrono/duration.h>
+#  include <__chrono/sys_info.h>
+#  include <__chrono/system_clock.h>
+#  include <__chrono/time_zone.h>
+#  include <__chrono/tzdb_list.h>
+#  include <__config>
+#  include <__fwd/string_view.h>
+#  include <__type_traits/common_type.h>
+#  include <__type_traits/conditional.h>
+#  include <__type_traits/remove_cvref.h>
+#  include <__utility/move.h>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&   \
+      !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+
+namespace chrono {
+
+template <class>
+struct zoned_traits {};
+
+template <>
+struct zoned_traits<const time_zone*> {
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static const time_zone* default_zone() { return chrono::locate_zone("UTC"); }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static const time_zone* locate_zone(string_view __name) {
+    return chrono::locate_zone(__name);
+  }
+};
+
+template <class _Duration, class _TimeZonePtr = const time_zone*>
+class zoned_time {
+  // [time.zone.zonedtime.ctor]/2
+  static_assert(__is_duration<_Duration>::value,
+                "the program is ill-formed since _Duration is not a specialization of std::chrono::duration");
+
+  // The wording uses the constraints like
+  //   constructible_from<zoned_time, decltype(__traits::locate_zone(string_view{}))>
+  // Using these constraints in the code causes the compiler to give an
+  // error that the constraint depends on itself. To avoid that issue use
+  // the fact it is possible to create this object from a _TimeZonePtr.
+  using __traits = zoned_traits<_TimeZonePtr>;
+
+public:
+  using duration = common_type_t<_Duration, seconds>;
+
+  _LIBCPP_HIDE_FROM_ABI zoned_time()
+    requires requires { __traits::default_zone(); }
+      : __zone_{__traits::default_zone()}, __tp_{} {}
+
+  _LIBCPP_HIDE_FROM_ABI zoned_time(const zoned_time&)            = default;
+  _LIBCPP_HIDE_FROM_ABI zoned_time& operator=(const zoned_time&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI zoned_time(const sys_time<_Duration>& __tp)
+    requires requires { __traits::default_zone(); }
+      : __zone_{__traits::default_zone()}, __tp_{__tp} {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit zoned_time(_TimeZonePtr __zone) : __zone_{std::move(__zone)}, __tp_{} {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit zoned_time(string_view __name)
+    requires(requires { __traits::locate_zone(string_view{}); } &&
+             constructible_from<_TimeZonePtr, decltype(__traits::locate_zone(string_view{}))>)
+      : __zone_{__traits::locate_zone(__name)}, __tp_{} {}
+
+  template <class _Duration2>
+  _LIBCPP_HIDE_FROM_ABI zoned_time(const zoned_time<_Duration2, _TimeZonePtr>& __zt)
+    requires is_convertible_v<sys_time<_Duration2>, sys_time<_Duration>>
+      : __zone_{__zt.get_time_zone()}, __tp_{__zt.get_sys_time()} {}
+
+  _LIBCPP_HIDE_FROM_ABI zoned_time(_TimeZonePtr __zone, const sys_time<_Duration>& __tp)
+      : __zone_{std::move(__zone)}, __tp_{__tp} {}
+
+  _LIBCPP_HIDE_FROM_ABI zoned_time(string_view __name, const sys_time<_Duration>& __tp)
+    requires requires { _TimeZonePtr{__traits::locate_zone(string_view{})}; }
+      : zoned_time{__traits::locate_zone(__name), __tp} {}
+
+  _LIBCPP_HIDE_FROM_ABI zoned_time(_TimeZonePtr __zone, const local_time<_Duration>& __tp)
+    requires(is_convertible_v<decltype(std::declval<_TimeZonePtr&>() -> to_sys(local_time<_Duration>{})),
+                              sys_time<duration>>)
+      : __zone_{std::move(__zone)}, __tp_{__zone_->to_sys(__tp)} {}
+
+  _LIBCPP_HIDE_FROM_ABI zoned_time(string_view __name, const local_time<_Duration>& __tp)
+    requires(requires {
+      _TimeZonePtr{__traits::locate_zone(string_view{})};
+    } && is_convertible_v<decltype(std::declval<_TimeZonePtr&>() -> to_sys(local_time<_Duration>{})),
+                          sys_time<duration>>)
+      : zoned_time{__traits::locate_zone(__name), __tp} {}
+
+  _LIBCPP_HIDE_FROM_ABI zoned_time(_TimeZonePtr __zone, const local_time<_Duration>& __tp, choose __c)
+    requires(is_convertible_v<
+                decltype(std::declval<_TimeZonePtr&>() -> to_sys(local_time<_Duration>{}, choose::earliest)),
+                sys_time<duration>>)
+      : __zone_{std::move(__zone)}, __tp_{__zone_->to_sys(__tp, __c)} {}
+
+  _LIBCPP_HIDE_FROM_ABI zoned_time(string_view __name, const local_time<_Duration>& __tp, choose __c)
+    requires(requires {
+      _TimeZonePtr{__traits::locate_zone(string_view{})};
+    } && is_convertible_v<decltype(std::declval<_TimeZonePtr&>() -> to_sys(local_time<_Duration>{}, choose::earliest)),
+                          sys_time<duration>>)
+      : zoned_time{__traits::locate_zone(__name), __tp, __c} {}
+
+  template <class _Duration2, class _TimeZonePtr2>
+  _LIBCPP_HIDE_FROM_ABI zoned_time(_TimeZonePtr __zone, const zoned_time<_Duration2, _TimeZonePtr2>& __zt)
+    requires is_convertible_v<sys_time<_Duration2>, sys_time<_Duration>>
+      : __zone_{std::move(__zone)}, __tp_{__zt.get_sys_time()} {}
+
+  // per wording choose has no effect
+  template <class _Duration2, class _TimeZonePtr2>
+  _LIBCPP_HIDE_FROM_ABI zoned_time(_TimeZonePtr __zone, const zoned_time<_Duration2, _TimeZonePtr2>& __zt, choose)
+    requires is_convertible_v<sys_time<_Duration2>, sys_time<_Duration>>
+      : __zone_{std::move(__zone)}, __tp_{__zt.get_sys_time()} {}
+
+  template <class _Duration2, class _TimeZonePtr2>
+  _LIBCPP_HIDE_FROM_ABI zoned_time(string_view __name, const zoned_time<_Duration2, _TimeZonePtr2>& __zt)
+    requires(requires {
+      _TimeZonePtr{__traits::locate_zone(string_view{})};
+    } && is_convertible_v<sys_time<_Duration2>, sys_time<_Duration>>)
+      : zoned_time{__traits::locate_zone(__name), __zt} {}
+
+  template <class _Duration2, class _TimeZonePtr2>
+  _LIBCPP_HIDE_FROM_ABI zoned_time(string_view __name, const zoned_time<_Duration2, _TimeZonePtr2>& __zt, choose __c)
+    requires(requires {
+      _TimeZonePtr{__traits::locate_zone(string_view{})};
+    } && is_convertible_v<sys_time<_Duration2>, sys_time<_Duration>>)
+      : zoned_time{__traits::locate_zone(__name), __zt, __c} {}
+
+  _LIBCPP_HIDE_FROM_ABI zoned_time& operator=(const sys_time<_Duration>& __tp) {
+    __tp_ = __tp;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI zoned_time& operator=(const local_time<_Duration>& __tp) {
+    // TODO TZDB This seems wrong.
+    // Assigning a non-existent or ambiguous time will throw and not satisfy
+    // the post condition. This seems quite odd; I constructed an object with
+    // choose::earliest and that choice is not respected.
+    // what did LEWG do with this.
+    // MSVC STL and libstdc++ behave the same
+    __tp_ = __zone_->to_sys(__tp);
+    return *this;
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI operator sys_time<duration>() const { return get_sys_time(); }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI explicit operator local_time<duration>() const { return get_local_time(); }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI _TimeZonePtr get_time_zone() const { return __zone_; }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI local_time<duration> get_local_time() const { return __zone_->to_local(__tp_); }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI sys_time<duration> get_sys_time() const { return __tp_; }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI sys_info get_info() const { return __zone_->get_info(__tp_); }
+
+private:
+  _TimeZonePtr __zone_;
+  sys_time<duration> __tp_;
+};
+
+zoned_time() -> zoned_time<seconds>;
+
+template <class _Duration>
+zoned_time(sys_time<_Duration>) -> zoned_time<common_type_t<_Duration, seconds>>;
+
+template <class _TimeZonePtrOrName>
+using __time_zone_representation =
+    conditional_t<is_convertible_v<_TimeZonePtrOrName, string_view>,
+                  const time_zone*,
+                  remove_cvref_t<_TimeZonePtrOrName>>;
+
+template <class _TimeZonePtrOrName>
+zoned_time(_TimeZonePtrOrName&&) -> zoned_time<seconds, __time_zone_representation<_TimeZonePtrOrName>>;
+
+template <class _TimeZonePtrOrName, class _Duration>
+zoned_time(_TimeZonePtrOrName&&, sys_time<_Duration>)
+    -> zoned_time<common_type_t<_Duration, seconds>, __time_zone_representation<_TimeZonePtrOrName>>;
+
+template <class _TimeZonePtrOrName, class _Duration>
+zoned_time(_TimeZonePtrOrName&&, local_time<_Duration>, choose = choose::earliest)
+    -> zoned_time<common_type_t<_Duration, seconds>, __time_zone_representation<_TimeZonePtrOrName>>;
+
+template <class _Duration, class _TimeZonePtrOrName, class TimeZonePtr2>
+zoned_time(_TimeZonePtrOrName&&, zoned_time<_Duration, TimeZonePtr2>, choose = choose::earliest)
+    -> zoned_time<common_type_t<_Duration, seconds>, __time_zone_representation<_TimeZonePtrOrName>>;
+
+using zoned_seconds = zoned_time<seconds>;
+
+template <class _Duration1, class _Duration2, class _TimeZonePtr>
+_LIBCPP_HIDE_FROM_ABI bool
+operator==(const zoned_time<_Duration1, _TimeZonePtr>& __lhs, const zoned_time<_Duration2, _TimeZonePtr>& __rhs) {
+  return __lhs.get_time_zone() == __rhs.get_time_zone() && __lhs.get_sys_time() == __rhs.get_sys_time();
+}
+
+} // namespace chrono
+
+#  endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM)
+         // && !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#endif // _LIBCPP___CHRONO_ZONED_TIME_H
diff --git a/libcxx/include/__cxx03/__compare/common_comparison_category.h b/libcxx/include/__cxx03/__compare/common_comparison_category.h
new file mode 100644
index 000000000000..7aeb3da03a4f
--- /dev/null
+++ b/libcxx/include/__cxx03/__compare/common_comparison_category.h
@@ -0,0 +1,86 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COMPARE_COMMON_COMPARISON_CATEGORY_H
+#define _LIBCPP___COMPARE_COMMON_COMPARISON_CATEGORY_H
+
+#include <__compare/ordering.h>
+#include <__config>
+#include <__type_traits/is_same.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __comp_detail {
+
+enum _ClassifyCompCategory : unsigned { _None, _PartialOrd, _WeakOrd, _StrongOrd, _CCC_Size };
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _ClassifyCompCategory __type_to_enum() noexcept {
+  if (is_same_v<_Tp, partial_ordering>)
+    return _PartialOrd;
+  if (is_same_v<_Tp, weak_ordering>)
+    return _WeakOrd;
+  if (is_same_v<_Tp, strong_ordering>)
+    return _StrongOrd;
+  return _None;
+}
+
+template <size_t _Size>
+_LIBCPP_HIDE_FROM_ABI constexpr _ClassifyCompCategory
+__compute_comp_type(const _ClassifyCompCategory (&__types)[_Size]) {
+  int __seen[_CCC_Size] = {};
+  for (auto __type : __types)
+    ++__seen[__type];
+  if (__seen[_None])
+    return _None;
+  if (__seen[_PartialOrd])
+    return _PartialOrd;
+  if (__seen[_WeakOrd])
+    return _WeakOrd;
+  return _StrongOrd;
+}
+
+template <class... _Ts, bool _False = false>
+_LIBCPP_HIDE_FROM_ABI constexpr auto __get_comp_type() {
+  using _CCC                    = _ClassifyCompCategory;
+  constexpr _CCC __type_kinds[] = {_StrongOrd, __type_to_enum<_Ts>()...};
+  constexpr _CCC __cat          = __comp_detail::__compute_comp_type(__type_kinds);
+  if constexpr (__cat == _None)
+    return void();
+  else if constexpr (__cat == _PartialOrd)
+    return partial_ordering::equivalent;
+  else if constexpr (__cat == _WeakOrd)
+    return weak_ordering::equivalent;
+  else if constexpr (__cat == _StrongOrd)
+    return strong_ordering::equivalent;
+  else
+    static_assert(_False, "unhandled case");
+}
+} // namespace __comp_detail
+
+// [cmp.common], common comparison category type
+template <class... _Ts>
+struct _LIBCPP_TEMPLATE_VIS common_comparison_category {
+  using type = decltype(__comp_detail::__get_comp_type<_Ts...>());
+};
+
+template <class... _Ts>
+using common_comparison_category_t = typename common_comparison_category<_Ts...>::type;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___COMPARE_COMMON_COMPARISON_CATEGORY_H
diff --git a/libcxx/include/__cxx03/__compare/compare_partial_order_fallback.h b/libcxx/include/__cxx03/__compare/compare_partial_order_fallback.h
new file mode 100644
index 000000000000..e0efa3ccb88d
--- /dev/null
+++ b/libcxx/include/__cxx03/__compare/compare_partial_order_fallback.h
@@ -0,0 +1,76 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COMPARE_COMPARE_PARTIAL_ORDER_FALLBACK
+#define _LIBCPP___COMPARE_COMPARE_PARTIAL_ORDER_FALLBACK
+
+#include <__compare/ordering.h>
+#include <__compare/partial_order.h>
+#include <__config>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_same.h>
+#include <__utility/forward.h>
+#include <__utility/priority_tag.h>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [cmp.alg]
+namespace __compare_partial_order_fallback {
+struct __fn {
+  template <class _Tp, class _Up>
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) noexcept(
+      noexcept(std::partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))))
+      -> decltype(std::partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))) {
+    return std::partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u));
+  }
+
+  template <class _Tp, class _Up>
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept(noexcept(
+      std::forward<_Tp>(__t) == std::forward<_Up>(__u)  ? partial_ordering::equivalent
+      : std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? partial_ordering::less
+      : std::forward<_Up>(__u) < std::forward<_Tp>(__t)
+          ? partial_ordering::greater
+          : partial_ordering::unordered))
+      -> decltype(std::forward<_Tp>(__t) == std::forward<_Up>(__u)  ? partial_ordering::equivalent
+                  : std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? partial_ordering::less
+                  : std::forward<_Up>(__u) < std::forward<_Tp>(__t)
+                      ? partial_ordering::greater
+                      : partial_ordering::unordered) {
+    return std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? partial_ordering::equivalent
+         : std::forward<_Tp>(__t) < std::forward<_Up>(__u)  ? partial_ordering::less
+         : std::forward<_Up>(__u) < std::forward<_Tp>(__t)
+             ? partial_ordering::greater
+             : partial_ordering::unordered;
+  }
+
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const
+      noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>())))
+          -> decltype(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>())) {
+    return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>());
+  }
+};
+} // namespace __compare_partial_order_fallback
+
+inline namespace __cpo {
+inline constexpr auto compare_partial_order_fallback = __compare_partial_order_fallback::__fn{};
+} // namespace __cpo
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___COMPARE_COMPARE_PARTIAL_ORDER_FALLBACK
diff --git a/libcxx/include/__cxx03/__compare/compare_strong_order_fallback.h b/libcxx/include/__cxx03/__compare/compare_strong_order_fallback.h
new file mode 100644
index 000000000000..a94d517ed30f
--- /dev/null
+++ b/libcxx/include/__cxx03/__compare/compare_strong_order_fallback.h
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COMPARE_COMPARE_STRONG_ORDER_FALLBACK
+#define _LIBCPP___COMPARE_COMPARE_STRONG_ORDER_FALLBACK
+
+#include <__compare/ordering.h>
+#include <__compare/strong_order.h>
+#include <__config>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_same.h>
+#include <__utility/forward.h>
+#include <__utility/priority_tag.h>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [cmp.alg]
+namespace __compare_strong_order_fallback {
+struct __fn {
+  template <class _Tp, class _Up>
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) noexcept(
+      noexcept(std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))))
+      -> decltype(std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))) {
+    return std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u));
+  }
+
+  template <class _Tp, class _Up>
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept(noexcept(
+      std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? strong_ordering::equal
+      : std::forward<_Tp>(__t) < std::forward<_Up>(__u)
+          ? strong_ordering::less
+          : strong_ordering::greater))
+      -> decltype(std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? strong_ordering::equal
+                  : std::forward<_Tp>(__t) < std::forward<_Up>(__u)
+                      ? strong_ordering::less
+                      : strong_ordering::greater) {
+    return std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? strong_ordering::equal
+         : std::forward<_Tp>(__t) < std::forward<_Up>(__u)
+             ? strong_ordering::less
+             : strong_ordering::greater;
+  }
+
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const
+      noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>())))
+          -> decltype(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>())) {
+    return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>());
+  }
+};
+} // namespace __compare_strong_order_fallback
+
+inline namespace __cpo {
+inline constexpr auto compare_strong_order_fallback = __compare_strong_order_fallback::__fn{};
+} // namespace __cpo
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___COMPARE_COMPARE_STRONG_ORDER_FALLBACK
diff --git a/libcxx/include/__cxx03/__compare/compare_three_way.h b/libcxx/include/__cxx03/__compare/compare_three_way.h
new file mode 100644
index 000000000000..01c12076c0d7
--- /dev/null
+++ b/libcxx/include/__cxx03/__compare/compare_three_way.h
@@ -0,0 +1,40 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COMPARE_COMPARE_THREE_WAY_H
+#define _LIBCPP___COMPARE_COMPARE_THREE_WAY_H
+
+#include <__compare/three_way_comparable.h>
+#include <__config>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+struct _LIBCPP_TEMPLATE_VIS compare_three_way {
+  template <class _T1, class _T2>
+    requires three_way_comparable_with<_T1, _T2>
+  constexpr _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) <=> std::forward<_T2>(__u))) {
+    return std::forward<_T1>(__t) <=> std::forward<_T2>(__u);
+  }
+
+  using is_transparent = void;
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___COMPARE_COMPARE_THREE_WAY_H
diff --git a/libcxx/include/__cxx03/__compare/compare_three_way_result.h b/libcxx/include/__cxx03/__compare/compare_three_way_result.h
new file mode 100644
index 000000000000..d7508073433a
--- /dev/null
+++ b/libcxx/include/__cxx03/__compare/compare_three_way_result.h
@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COMPARE_COMPARE_THREE_WAY_RESULT_H
+#define _LIBCPP___COMPARE_COMPARE_THREE_WAY_RESULT_H
+
+#include <__config>
+#include <__type_traits/make_const_lvalue_ref.h>
+#include <__utility/declval.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class, class, class>
+struct _LIBCPP_HIDE_FROM_ABI __compare_three_way_result {};
+
+template <class _Tp, class _Up>
+struct _LIBCPP_HIDE_FROM_ABI __compare_three_way_result<
+    _Tp,
+    _Up,
+    decltype(std::declval<__make_const_lvalue_ref<_Tp>>() <=> std::declval<__make_const_lvalue_ref<_Up>>(), void())> {
+  using type = decltype(std::declval<__make_const_lvalue_ref<_Tp>>() <=> std::declval<__make_const_lvalue_ref<_Up>>());
+};
+
+template <class _Tp, class _Up = _Tp>
+struct _LIBCPP_TEMPLATE_VIS compare_three_way_result : __compare_three_way_result<_Tp, _Up, void> {};
+
+template <class _Tp, class _Up = _Tp>
+using compare_three_way_result_t = typename compare_three_way_result<_Tp, _Up>::type;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___COMPARE_COMPARE_THREE_WAY_RESULT_H
diff --git a/libcxx/include/__cxx03/__compare/compare_weak_order_fallback.h b/libcxx/include/__cxx03/__compare/compare_weak_order_fallback.h
new file mode 100644
index 000000000000..062b7b582cd7
--- /dev/null
+++ b/libcxx/include/__cxx03/__compare/compare_weak_order_fallback.h
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COMPARE_COMPARE_WEAK_ORDER_FALLBACK
+#define _LIBCPP___COMPARE_COMPARE_WEAK_ORDER_FALLBACK
+
+#include <__compare/ordering.h>
+#include <__compare/weak_order.h>
+#include <__config>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_same.h>
+#include <__utility/forward.h>
+#include <__utility/priority_tag.h>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [cmp.alg]
+namespace __compare_weak_order_fallback {
+struct __fn {
+  template <class _Tp, class _Up>
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) noexcept(
+      noexcept(std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))))
+      -> decltype(std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))) {
+    return std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u));
+  }
+
+  template <class _Tp, class _Up>
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept(noexcept(
+      std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? weak_ordering::equivalent
+      : std::forward<_Tp>(__t) < std::forward<_Up>(__u)
+          ? weak_ordering::less
+          : weak_ordering::greater))
+      -> decltype(std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? weak_ordering::equivalent
+                  : std::forward<_Tp>(__t) < std::forward<_Up>(__u)
+                      ? weak_ordering::less
+                      : weak_ordering::greater) {
+    return std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? weak_ordering::equivalent
+         : std::forward<_Tp>(__t) < std::forward<_Up>(__u)
+             ? weak_ordering::less
+             : weak_ordering::greater;
+  }
+
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const
+      noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>())))
+          -> decltype(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>())) {
+    return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>());
+  }
+};
+} // namespace __compare_weak_order_fallback
+
+inline namespace __cpo {
+inline constexpr auto compare_weak_order_fallback = __compare_weak_order_fallback::__fn{};
+} // namespace __cpo
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___COMPARE_COMPARE_WEAK_ORDER_FALLBACK
diff --git a/libcxx/include/__cxx03/__compare/is_eq.h b/libcxx/include/__cxx03/__compare/is_eq.h
new file mode 100644
index 000000000000..9a82df1ebe88
--- /dev/null
+++ b/libcxx/include/__cxx03/__compare/is_eq.h
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COMPARE_IS_EQ_H
+#define _LIBCPP___COMPARE_IS_EQ_H
+
+#include <__compare/ordering.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_eq(partial_ordering __c) noexcept { return __c == 0; }
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_neq(partial_ordering __c) noexcept { return __c != 0; }
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_lt(partial_ordering __c) noexcept { return __c < 0; }
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_lteq(partial_ordering __c) noexcept { return __c <= 0; }
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_gt(partial_ordering __c) noexcept { return __c > 0; }
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_gteq(partial_ordering __c) noexcept { return __c >= 0; }
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___COMPARE_IS_EQ_H
diff --git a/libcxx/include/__cxx03/__compare/ordering.h b/libcxx/include/__cxx03/__compare/ordering.h
new file mode 100644
index 000000000000..2995d381304f
--- /dev/null
+++ b/libcxx/include/__cxx03/__compare/ordering.h
@@ -0,0 +1,278 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COMPARE_ORDERING_H
+#define _LIBCPP___COMPARE_ORDERING_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_same.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// exposition only
+enum class _OrdResult : signed char { __less = -1, __equiv = 0, __greater = 1 };
+
+enum class _NCmpResult : signed char { __unordered = -127 };
+
+class partial_ordering;
+class weak_ordering;
+class strong_ordering;
+
+template <class _Tp, class... _Args>
+inline constexpr bool __one_of_v = (is_same_v<_Tp, _Args> || ...);
+
+struct _CmpUnspecifiedParam {
+  _LIBCPP_HIDE_FROM_ABI constexpr _CmpUnspecifiedParam(int _CmpUnspecifiedParam::*) noexcept {}
+
+  template <class _Tp, class = enable_if_t<!__one_of_v<_Tp, int, partial_ordering, weak_ordering, strong_ordering>>>
+  _CmpUnspecifiedParam(_Tp) = delete;
+};
+
+class partial_ordering {
+  using _ValueT = signed char;
+
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr partial_ordering(_OrdResult __v) noexcept : __value_(_ValueT(__v)) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr partial_ordering(_NCmpResult __v) noexcept : __value_(_ValueT(__v)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __is_ordered() const noexcept {
+    return __value_ != _ValueT(_NCmpResult::__unordered);
+  }
+
+public:
+  // valid values
+  static const partial_ordering less;
+  static const partial_ordering equivalent;
+  static const partial_ordering greater;
+  static const partial_ordering unordered;
+
+  // comparisons
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(partial_ordering, partial_ordering) noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(partial_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v.__is_ordered() && __v.__value_ == 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(partial_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v.__is_ordered() && __v.__value_ < 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(partial_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v.__is_ordered() && __v.__value_ <= 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(partial_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v.__is_ordered() && __v.__value_ > 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(partial_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v.__is_ordered() && __v.__value_ >= 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(_CmpUnspecifiedParam, partial_ordering __v) noexcept {
+    return __v.__is_ordered() && 0 < __v.__value_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(_CmpUnspecifiedParam, partial_ordering __v) noexcept {
+    return __v.__is_ordered() && 0 <= __v.__value_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(_CmpUnspecifiedParam, partial_ordering __v) noexcept {
+    return __v.__is_ordered() && 0 > __v.__value_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(_CmpUnspecifiedParam, partial_ordering __v) noexcept {
+    return __v.__is_ordered() && 0 >= __v.__value_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr partial_ordering
+  operator<=>(partial_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr partial_ordering
+  operator<=>(_CmpUnspecifiedParam, partial_ordering __v) noexcept {
+    return __v < 0 ? partial_ordering::greater : (__v > 0 ? partial_ordering::less : __v);
+  }
+
+private:
+  _ValueT __value_;
+};
+
+inline constexpr partial_ordering partial_ordering::less(_OrdResult::__less);
+inline constexpr partial_ordering partial_ordering::equivalent(_OrdResult::__equiv);
+inline constexpr partial_ordering partial_ordering::greater(_OrdResult::__greater);
+inline constexpr partial_ordering partial_ordering::unordered(_NCmpResult ::__unordered);
+
+class weak_ordering {
+  using _ValueT = signed char;
+
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr weak_ordering(_OrdResult __v) noexcept : __value_(_ValueT(__v)) {}
+
+public:
+  static const weak_ordering less;
+  static const weak_ordering equivalent;
+  static const weak_ordering greater;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr operator partial_ordering() const noexcept {
+    return __value_ == 0 ? partial_ordering::equivalent
+                         : (__value_ < 0 ? partial_ordering::less : partial_ordering::greater);
+  }
+
+  // comparisons
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(weak_ordering, weak_ordering) noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(weak_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v.__value_ == 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(weak_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v.__value_ < 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(weak_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v.__value_ <= 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(weak_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v.__value_ > 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(weak_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v.__value_ >= 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(_CmpUnspecifiedParam, weak_ordering __v) noexcept {
+    return 0 < __v.__value_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(_CmpUnspecifiedParam, weak_ordering __v) noexcept {
+    return 0 <= __v.__value_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(_CmpUnspecifiedParam, weak_ordering __v) noexcept {
+    return 0 > __v.__value_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(_CmpUnspecifiedParam, weak_ordering __v) noexcept {
+    return 0 >= __v.__value_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr weak_ordering operator<=>(weak_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr weak_ordering operator<=>(_CmpUnspecifiedParam, weak_ordering __v) noexcept {
+    return __v < 0 ? weak_ordering::greater : (__v > 0 ? weak_ordering::less : __v);
+  }
+
+private:
+  _ValueT __value_;
+};
+
+inline constexpr weak_ordering weak_ordering::less(_OrdResult::__less);
+inline constexpr weak_ordering weak_ordering::equivalent(_OrdResult::__equiv);
+inline constexpr weak_ordering weak_ordering::greater(_OrdResult::__greater);
+
+class strong_ordering {
+  using _ValueT = signed char;
+
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr strong_ordering(_OrdResult __v) noexcept : __value_(_ValueT(__v)) {}
+
+public:
+  static const strong_ordering less;
+  static const strong_ordering equal;
+  static const strong_ordering equivalent;
+  static const strong_ordering greater;
+
+  // conversions
+  _LIBCPP_HIDE_FROM_ABI constexpr operator partial_ordering() const noexcept {
+    return __value_ == 0 ? partial_ordering::equivalent
+                         : (__value_ < 0 ? partial_ordering::less : partial_ordering::greater);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr operator weak_ordering() const noexcept {
+    return __value_ == 0 ? weak_ordering::equivalent : (__value_ < 0 ? weak_ordering::less : weak_ordering::greater);
+  }
+
+  // comparisons
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(strong_ordering, strong_ordering) noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(strong_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v.__value_ == 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(strong_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v.__value_ < 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(strong_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v.__value_ <= 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(strong_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v.__value_ > 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(strong_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v.__value_ >= 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(_CmpUnspecifiedParam, strong_ordering __v) noexcept {
+    return 0 < __v.__value_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(_CmpUnspecifiedParam, strong_ordering __v) noexcept {
+    return 0 <= __v.__value_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(_CmpUnspecifiedParam, strong_ordering __v) noexcept {
+    return 0 > __v.__value_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(_CmpUnspecifiedParam, strong_ordering __v) noexcept {
+    return 0 >= __v.__value_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr strong_ordering
+  operator<=>(strong_ordering __v, _CmpUnspecifiedParam) noexcept {
+    return __v;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr strong_ordering
+  operator<=>(_CmpUnspecifiedParam, strong_ordering __v) noexcept {
+    return __v < 0 ? strong_ordering::greater : (__v > 0 ? strong_ordering::less : __v);
+  }
+
+private:
+  _ValueT __value_;
+};
+
+inline constexpr strong_ordering strong_ordering::less(_OrdResult::__less);
+inline constexpr strong_ordering strong_ordering::equal(_OrdResult::__equiv);
+inline constexpr strong_ordering strong_ordering::equivalent(_OrdResult::__equiv);
+inline constexpr strong_ordering strong_ordering::greater(_OrdResult::__greater);
+
+/// [cmp.categories.pre]/1
+/// The types partial_ordering, weak_ordering, and strong_ordering are
+/// collectively termed the comparison category types.
+template <class _Tp>
+concept __comparison_category = __one_of_v<_Tp, partial_ordering, weak_ordering, strong_ordering>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___COMPARE_ORDERING_H
diff --git a/libcxx/include/__cxx03/__compare/partial_order.h b/libcxx/include/__cxx03/__compare/partial_order.h
new file mode 100644
index 000000000000..1d2fae63e5f2
--- /dev/null
+++ b/libcxx/include/__cxx03/__compare/partial_order.h
@@ -0,0 +1,77 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COMPARE_PARTIAL_ORDER
+#define _LIBCPP___COMPARE_PARTIAL_ORDER
+
+#include <__compare/compare_three_way.h>
+#include <__compare/ordering.h>
+#include <__compare/weak_order.h>
+#include <__config>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_same.h>
+#include <__utility/forward.h>
+#include <__utility/priority_tag.h>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [cmp.alg]
+namespace __partial_order {
+void partial_order() = delete;
+
+struct __fn {
+  // NOLINTBEGIN(libcpp-robust-against-adl) partial_order should use ADL, but only here
+  template <class _Tp, class _Up>
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<2>) noexcept(
+      noexcept(partial_ordering(partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))))
+      -> decltype(partial_ordering(partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) {
+    return partial_ordering(partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)));
+  }
+  // NOLINTEND(libcpp-robust-against-adl)
+
+  template <class _Tp, class _Up>
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) noexcept(
+      noexcept(partial_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u)))))
+      -> decltype(partial_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) {
+    return partial_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u)));
+  }
+
+  template <class _Tp, class _Up>
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept(
+      noexcept(partial_ordering(std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))))
+      -> decltype(partial_ordering(std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) {
+    return partial_ordering(std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)));
+  }
+
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const
+      noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>())))
+          -> decltype(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>())) {
+    return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>());
+  }
+};
+} // namespace __partial_order
+
+inline namespace __cpo {
+inline constexpr auto partial_order = __partial_order::__fn{};
+} // namespace __cpo
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___COMPARE_PARTIAL_ORDER
diff --git a/libcxx/include/__cxx03/__compare/strong_order.h b/libcxx/include/__cxx03/__compare/strong_order.h
new file mode 100644
index 000000000000..8c363b563822
--- /dev/null
+++ b/libcxx/include/__cxx03/__compare/strong_order.h
@@ -0,0 +1,143 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COMPARE_STRONG_ORDER
+#define _LIBCPP___COMPARE_STRONG_ORDER
+
+#include <__bit/bit_cast.h>
+#include <__compare/compare_three_way.h>
+#include <__compare/ordering.h>
+#include <__config>
+#include <__math/exponential_functions.h>
+#include <__math/traits.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_floating_point.h>
+#include <__type_traits/is_same.h>
+#include <__utility/forward.h>
+#include <__utility/priority_tag.h>
+#include <cstdint>
+#include <limits>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [cmp.alg]
+namespace __strong_order {
+void strong_order() = delete;
+
+struct __fn {
+  // NOLINTBEGIN(libcpp-robust-against-adl) strong_order should use ADL, but only here
+  template <class _Tp, class _Up>
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<2>) noexcept(
+      noexcept(strong_ordering(strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))))
+      -> decltype(strong_ordering(strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) {
+    return strong_ordering(strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)));
+  }
+  // NOLINTEND(libcpp-robust-against-adl)
+
+  template <class _Tp, class _Up, class _Dp = decay_t<_Tp>>
+    requires is_same_v<_Dp, decay_t<_Up>> && is_floating_point_v<_Dp>
+  _LIBCPP_HIDE_FROM_ABI static constexpr strong_ordering __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) noexcept {
+    if constexpr (numeric_limits<_Dp>::is_iec559 && sizeof(_Dp) == sizeof(int32_t)) {
+      int32_t __rx = std::bit_cast<int32_t>(__t);
+      int32_t __ry = std::bit_cast<int32_t>(__u);
+      __rx         = (__rx < 0) ? (numeric_limits<int32_t>::min() - __rx - 1) : __rx;
+      __ry         = (__ry < 0) ? (numeric_limits<int32_t>::min() - __ry - 1) : __ry;
+      return (__rx <=> __ry);
+    } else if constexpr (numeric_limits<_Dp>::is_iec559 && sizeof(_Dp) == sizeof(int64_t)) {
+      int64_t __rx = std::bit_cast<int64_t>(__t);
+      int64_t __ry = std::bit_cast<int64_t>(__u);
+      __rx         = (__rx < 0) ? (numeric_limits<int64_t>::min() - __rx - 1) : __rx;
+      __ry         = (__ry < 0) ? (numeric_limits<int64_t>::min() - __ry - 1) : __ry;
+      return (__rx <=> __ry);
+    } else if (__t < __u) {
+      return strong_ordering::less;
+    } else if (__t > __u) {
+      return strong_ordering::greater;
+    } else if (__t == __u) {
+      if constexpr (numeric_limits<_Dp>::radix == 2) {
+        return __math::signbit(__u) <=> __math::signbit(__t);
+      } else {
+        // This is bullet 3 of the IEEE754 algorithm, relevant
+        // only for decimal floating-point;
+        // see https://stackoverflow.com/questions/69068075/
+        if (__t == 0 || __math::isinf(__t)) {
+          return __math::signbit(__u) <=> __math::signbit(__t);
+        } else {
+          int __texp, __uexp;
+          (void)__math::frexp(__t, &__texp);
+          (void)__math::frexp(__u, &__uexp);
+          return (__t < 0) ? (__texp <=> __uexp) : (__uexp <=> __texp);
+        }
+      }
+    } else {
+      // They're unordered, so one of them must be a NAN.
+      // The order is -QNAN, -SNAN, numbers, +SNAN, +QNAN.
+      bool __t_is_nan      = __math::isnan(__t);
+      bool __u_is_nan      = __math::isnan(__u);
+      bool __t_is_negative = __math::signbit(__t);
+      bool __u_is_negative = __math::signbit(__u);
+      using _IntType =
+          conditional_t< sizeof(__t) == sizeof(int32_t),
+                         int32_t,
+                         conditional_t< sizeof(__t) == sizeof(int64_t), int64_t, void> >;
+      if constexpr (is_same_v<_IntType, void>) {
+        static_assert(sizeof(_Dp) == 0, "std::strong_order is unimplemented for this floating-point type");
+      } else if (__t_is_nan && __u_is_nan) {
+        // Order by sign bit, then by "payload bits" (we'll just use bit_cast).
+        if (__t_is_negative != __u_is_negative) {
+          return (__u_is_negative <=> __t_is_negative);
+        } else {
+          return std::bit_cast<_IntType>(__t) <=> std::bit_cast<_IntType>(__u);
+        }
+      } else if (__t_is_nan) {
+        return __t_is_negative ? strong_ordering::less : strong_ordering::greater;
+      } else {
+        return __u_is_negative ? strong_ordering::greater : strong_ordering::less;
+      }
+    }
+  }
+
+  template <class _Tp, class _Up>
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept(
+      noexcept(strong_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u)))))
+      -> decltype(strong_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) {
+    return strong_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u)));
+  }
+
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const
+      noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>())))
+          -> decltype(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>())) {
+    return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>());
+  }
+};
+} // namespace __strong_order
+
+inline namespace __cpo {
+inline constexpr auto strong_order = __strong_order::__fn{};
+} // namespace __cpo
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___COMPARE_STRONG_ORDER
diff --git a/libcxx/include/__cxx03/__compare/synth_three_way.h b/libcxx/include/__cxx03/__compare/synth_three_way.h
new file mode 100644
index 000000000000..e48ce4979983
--- /dev/null
+++ b/libcxx/include/__cxx03/__compare/synth_three_way.h
@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COMPARE_SYNTH_THREE_WAY_H
+#define _LIBCPP___COMPARE_SYNTH_THREE_WAY_H
+
+#include <__compare/ordering.h>
+#include <__compare/three_way_comparable.h>
+#include <__concepts/boolean_testable.h>
+#include <__config>
+#include <__utility/declval.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [expos.only.func]
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr auto __synth_three_way = []<class _Tp, class _Up>(const _Tp& __t, const _Up& __u)
+  requires requires {
+    { __t < __u } -> __boolean_testable;
+    { __u < __t } -> __boolean_testable;
+  }
+{
+  if constexpr (three_way_comparable_with<_Tp, _Up>) {
+    return __t <=> __u;
+  } else {
+    if (__t < __u)
+      return weak_ordering::less;
+    if (__u < __t)
+      return weak_ordering::greater;
+    return weak_ordering::equivalent;
+  }
+};
+
+template <class _Tp, class _Up = _Tp>
+using __synth_three_way_result = decltype(std::__synth_three_way(std::declval<_Tp&>(), std::declval<_Up&>()));
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___COMPARE_SYNTH_THREE_WAY_H
diff --git a/libcxx/include/__cxx03/__compare/three_way_comparable.h b/libcxx/include/__cxx03/__compare/three_way_comparable.h
new file mode 100644
index 000000000000..7a44ea9158a6
--- /dev/null
+++ b/libcxx/include/__cxx03/__compare/three_way_comparable.h
@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COMPARE_THREE_WAY_COMPARABLE_H
+#define _LIBCPP___COMPARE_THREE_WAY_COMPARABLE_H
+
+#include <__compare/common_comparison_category.h>
+#include <__compare/ordering.h>
+#include <__concepts/common_reference_with.h>
+#include <__concepts/equality_comparable.h>
+#include <__concepts/same_as.h>
+#include <__concepts/totally_ordered.h>
+#include <__config>
+#include <__type_traits/common_reference.h>
+#include <__type_traits/make_const_lvalue_ref.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp, class _Cat>
+concept __compares_as = same_as<common_comparison_category_t<_Tp, _Cat>, _Cat>;
+
+template <class _Tp, class _Cat = partial_ordering>
+concept three_way_comparable =
+    __weakly_equality_comparable_with<_Tp, _Tp> && __partially_ordered_with<_Tp, _Tp> &&
+    requires(__make_const_lvalue_ref<_Tp> __a, __make_const_lvalue_ref<_Tp> __b) {
+      { __a <=> __b } -> __compares_as<_Cat>;
+    };
+
+template <class _Tp, class _Up, class _Cat = partial_ordering>
+concept three_way_comparable_with =
+    three_way_comparable<_Tp, _Cat> && three_way_comparable<_Up, _Cat> &&
+    common_reference_with<__make_const_lvalue_ref<_Tp>, __make_const_lvalue_ref<_Up>> &&
+    three_way_comparable<common_reference_t<__make_const_lvalue_ref<_Tp>, __make_const_lvalue_ref<_Up>>, _Cat> &&
+    __weakly_equality_comparable_with<_Tp, _Up> && __partially_ordered_with<_Tp, _Up> &&
+    requires(__make_const_lvalue_ref<_Tp> __t, __make_const_lvalue_ref<_Up> __u) {
+      { __t <=> __u } -> __compares_as<_Cat>;
+      { __u <=> __t } -> __compares_as<_Cat>;
+    };
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___COMPARE_THREE_WAY_COMPARABLE_H
diff --git a/libcxx/include/__cxx03/__compare/weak_order.h b/libcxx/include/__cxx03/__compare/weak_order.h
new file mode 100644
index 000000000000..1a3e85feb233
--- /dev/null
+++ b/libcxx/include/__cxx03/__compare/weak_order.h
@@ -0,0 +1,105 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COMPARE_WEAK_ORDER
+#define _LIBCPP___COMPARE_WEAK_ORDER
+
+#include <__compare/compare_three_way.h>
+#include <__compare/ordering.h>
+#include <__compare/strong_order.h>
+#include <__config>
+#include <__math/traits.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_floating_point.h>
+#include <__type_traits/is_same.h>
+#include <__utility/forward.h>
+#include <__utility/priority_tag.h>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [cmp.alg]
+namespace __weak_order {
+void weak_order() = delete;
+
+struct __fn {
+  // NOLINTBEGIN(libcpp-robust-against-adl) weak_order should use ADL, but only here
+  template <class _Tp, class _Up>
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<3>) noexcept(
+      noexcept(weak_ordering(weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))))
+      -> decltype(weak_ordering(weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) {
+    return weak_ordering(weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)));
+  }
+  // NOLINTEND(libcpp-robust-against-adl)
+
+  template <class _Tp, class _Up, class _Dp = decay_t<_Tp>>
+    requires is_same_v<_Dp, decay_t<_Up>> && is_floating_point_v<_Dp>
+  _LIBCPP_HIDE_FROM_ABI static constexpr weak_ordering __go(_Tp&& __t, _Up&& __u, __priority_tag<2>) noexcept {
+    partial_ordering __po = (__t <=> __u);
+    if (__po == partial_ordering::less) {
+      return weak_ordering::less;
+    } else if (__po == partial_ordering::equivalent) {
+      return weak_ordering::equivalent;
+    } else if (__po == partial_ordering::greater) {
+      return weak_ordering::greater;
+    } else {
+      // Otherwise, at least one of them is a NaN.
+      bool __t_is_nan      = __math::isnan(__t);
+      bool __u_is_nan      = __math::isnan(__u);
+      bool __t_is_negative = __math::signbit(__t);
+      bool __u_is_negative = __math::signbit(__u);
+      if (__t_is_nan && __u_is_nan) {
+        return (__u_is_negative <=> __t_is_negative);
+      } else if (__t_is_nan) {
+        return __t_is_negative ? weak_ordering::less : weak_ordering::greater;
+      } else {
+        return __u_is_negative ? weak_ordering::greater : weak_ordering::less;
+      }
+    }
+  }
+
+  template <class _Tp, class _Up>
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) noexcept(
+      noexcept(weak_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u)))))
+      -> decltype(weak_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) {
+    return weak_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u)));
+  }
+
+  template <class _Tp, class _Up>
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept(
+      noexcept(weak_ordering(std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))))
+      -> decltype(weak_ordering(std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) {
+    return weak_ordering(std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)));
+  }
+
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const
+      noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<3>())))
+          -> decltype(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<3>())) {
+    return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<3>());
+  }
+};
+} // namespace __weak_order
+
+inline namespace __cpo {
+inline constexpr auto weak_order = __weak_order::__fn{};
+} // namespace __cpo
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___COMPARE_WEAK_ORDER
diff --git a/libcxx/include/__cxx03/__concepts/arithmetic.h b/libcxx/include/__cxx03/__concepts/arithmetic.h
new file mode 100644
index 000000000000..0c44f117805f
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/arithmetic.h
@@ -0,0 +1,56 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_ARITHMETIC_H
+#define _LIBCPP___CONCEPTS_ARITHMETIC_H
+
+#include <__config>
+#include <__type_traits/is_floating_point.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_signed.h>
+#include <__type_traits/is_signed_integer.h>
+#include <__type_traits/is_unsigned_integer.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concepts.arithmetic], arithmetic concepts
+
+template <class _Tp>
+concept integral = is_integral_v<_Tp>;
+
+template <class _Tp>
+concept signed_integral = integral<_Tp> && is_signed_v<_Tp>;
+
+template <class _Tp>
+concept unsigned_integral = integral<_Tp> && !signed_integral<_Tp>;
+
+template <class _Tp>
+concept floating_point = is_floating_point_v<_Tp>;
+
+// Concept helpers for the internal type traits for the fundamental types.
+
+template <class _Tp>
+concept __libcpp_unsigned_integer = __libcpp_is_unsigned_integer<_Tp>::value;
+
+template <class _Tp>
+concept __libcpp_signed_integer = __libcpp_is_signed_integer<_Tp>::value;
+
+template <class _Tp>
+concept __libcpp_integer = __libcpp_unsigned_integer<_Tp> || __libcpp_signed_integer<_Tp>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_ARITHMETIC_H
diff --git a/libcxx/include/__cxx03/__concepts/assignable.h b/libcxx/include/__cxx03/__concepts/assignable.h
new file mode 100644
index 000000000000..7423daabba78
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/assignable.h
@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_ASSIGNABLE_H
+#define _LIBCPP___CONCEPTS_ASSIGNABLE_H
+
+#include <__concepts/common_reference_with.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/make_const_lvalue_ref.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.assignable]
+
+template <class _Lhs, class _Rhs>
+concept assignable_from =
+    is_lvalue_reference_v<_Lhs> &&
+    common_reference_with<__make_const_lvalue_ref<_Lhs>, __make_const_lvalue_ref<_Rhs>> &&
+    requires(_Lhs __lhs, _Rhs&& __rhs) {
+      { __lhs = std::forward<_Rhs>(__rhs) } -> same_as<_Lhs>;
+    };
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_ASSIGNABLE_H
diff --git a/libcxx/include/__cxx03/__concepts/boolean_testable.h b/libcxx/include/__cxx03/__concepts/boolean_testable.h
new file mode 100644
index 000000000000..b379fe9c5a88
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/boolean_testable.h
@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_BOOLEAN_TESTABLE_H
+#define _LIBCPP___CONCEPTS_BOOLEAN_TESTABLE_H
+
+#include <__concepts/convertible_to.h>
+#include <__config>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concepts.booleantestable]
+
+template <class _Tp>
+concept __boolean_testable_impl = convertible_to<_Tp, bool>;
+
+template <class _Tp>
+concept __boolean_testable = __boolean_testable_impl<_Tp> && requires(_Tp&& __t) {
+  { !std::forward<_Tp>(__t) } -> __boolean_testable_impl;
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_BOOLEAN_TESTABLE_H
diff --git a/libcxx/include/__cxx03/__concepts/class_or_enum.h b/libcxx/include/__cxx03/__concepts/class_or_enum.h
new file mode 100644
index 000000000000..2739e31e14ba
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/class_or_enum.h
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_CLASS_OR_ENUM_H
+#define _LIBCPP___CONCEPTS_CLASS_OR_ENUM_H
+
+#include <__config>
+#include <__type_traits/is_class.h>
+#include <__type_traits/is_enum.h>
+#include <__type_traits/is_union.h>
+#include <__type_traits/remove_cvref.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// Whether a type is a class type or enumeration type according to the Core wording.
+
+template <class _Tp>
+concept __class_or_enum = is_class_v<_Tp> || is_union_v<_Tp> || is_enum_v<_Tp>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_CLASS_OR_ENUM_H
diff --git a/libcxx/include/__cxx03/__concepts/common_reference_with.h b/libcxx/include/__cxx03/__concepts/common_reference_with.h
new file mode 100644
index 000000000000..4eb687e071bc
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/common_reference_with.h
@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_COMMON_REFERENCE_WITH_H
+#define _LIBCPP___CONCEPTS_COMMON_REFERENCE_WITH_H
+
+#include <__concepts/convertible_to.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__type_traits/common_reference.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.commonref]
+
+template <class _Tp, class _Up>
+concept common_reference_with =
+    same_as<common_reference_t<_Tp, _Up>, common_reference_t<_Up, _Tp>> &&
+    convertible_to<_Tp, common_reference_t<_Tp, _Up>> && convertible_to<_Up, common_reference_t<_Tp, _Up>>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_COMMON_REFERENCE_WITH_H
diff --git a/libcxx/include/__cxx03/__concepts/common_with.h b/libcxx/include/__cxx03/__concepts/common_with.h
new file mode 100644
index 000000000000..85abb05efbc2
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/common_with.h
@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_COMMON_WITH_H
+#define _LIBCPP___CONCEPTS_COMMON_WITH_H
+
+#include <__concepts/common_reference_with.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/common_reference.h>
+#include <__type_traits/common_type.h>
+#include <__utility/declval.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.common]
+
+// clang-format off
+template <class _Tp, class _Up>
+concept common_with =
+    same_as<common_type_t<_Tp, _Up>, common_type_t<_Up, _Tp>> &&
+    requires {
+        static_cast<common_type_t<_Tp, _Up>>(std::declval<_Tp>());
+        static_cast<common_type_t<_Tp, _Up>>(std::declval<_Up>());
+    } &&
+    common_reference_with<
+        add_lvalue_reference_t<const _Tp>,
+        add_lvalue_reference_t<const _Up>> &&
+    common_reference_with<
+        add_lvalue_reference_t<common_type_t<_Tp, _Up>>,
+        common_reference_t<
+            add_lvalue_reference_t<const _Tp>,
+            add_lvalue_reference_t<const _Up>>>;
+// clang-format on
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_COMMON_WITH_H
diff --git a/libcxx/include/__cxx03/__concepts/constructible.h b/libcxx/include/__cxx03/__concepts/constructible.h
new file mode 100644
index 000000000000..835a44429c09
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/constructible.h
@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_CONSTRUCTIBLE_H
+#define _LIBCPP___CONCEPTS_CONSTRUCTIBLE_H
+
+#include <__concepts/convertible_to.h>
+#include <__concepts/destructible.h>
+#include <__config>
+#include <__type_traits/is_constructible.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.constructible]
+template <class _Tp, class... _Args>
+concept constructible_from = destructible<_Tp> && is_constructible_v<_Tp, _Args...>;
+
+// [concept.default.init]
+
+template <class _Tp>
+concept __default_initializable = requires { ::new _Tp; };
+
+template <class _Tp>
+concept default_initializable = constructible_from<_Tp> && requires { _Tp{}; } && __default_initializable<_Tp>;
+
+// [concept.moveconstructible]
+template <class _Tp>
+concept move_constructible = constructible_from<_Tp, _Tp> && convertible_to<_Tp, _Tp>;
+
+// [concept.copyconstructible]
+// clang-format off
+template <class _Tp>
+concept copy_constructible =
+    move_constructible<_Tp> &&
+    constructible_from<_Tp, _Tp&> && convertible_to<_Tp&, _Tp> &&
+    constructible_from<_Tp, const _Tp&> && convertible_to<const _Tp&, _Tp> &&
+    constructible_from<_Tp, const _Tp> && convertible_to<const _Tp, _Tp>;
+// clang-format on
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_CONSTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__concepts/convertible_to.h b/libcxx/include/__cxx03/__concepts/convertible_to.h
new file mode 100644
index 000000000000..6d5b6c1268d5
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/convertible_to.h
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_CONVERTIBLE_TO_H
+#define _LIBCPP___CONCEPTS_CONVERTIBLE_TO_H
+
+#include <__config>
+#include <__type_traits/is_convertible.h>
+#include <__utility/declval.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.convertible]
+
+template <class _From, class _To>
+concept convertible_to = is_convertible_v<_From, _To> && requires { static_cast<_To>(std::declval<_From>()); };
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_CONVERTIBLE_TO_H
diff --git a/libcxx/include/__cxx03/__concepts/copyable.h b/libcxx/include/__cxx03/__concepts/copyable.h
new file mode 100644
index 000000000000..2bf0ad42fc1a
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/copyable.h
@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_COPYABLE_H
+#define _LIBCPP___CONCEPTS_COPYABLE_H
+
+#include <__concepts/assignable.h>
+#include <__concepts/constructible.h>
+#include <__concepts/movable.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concepts.object]
+
+// clang-format off
+template <class _Tp>
+concept copyable =
+    copy_constructible<_Tp> &&
+    movable<_Tp> &&
+    assignable_from<_Tp&, _Tp&> &&
+    assignable_from<_Tp&, const _Tp&> &&
+    assignable_from<_Tp&, const _Tp>;
+// clang-format on
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_COPYABLE_H
diff --git a/libcxx/include/__cxx03/__concepts/derived_from.h b/libcxx/include/__cxx03/__concepts/derived_from.h
new file mode 100644
index 000000000000..9875faee81b9
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/derived_from.h
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_DERIVED_FROM_H
+#define _LIBCPP___CONCEPTS_DERIVED_FROM_H
+
+#include <__config>
+#include <__type_traits/is_base_of.h>
+#include <__type_traits/is_convertible.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.derived]
+
+template <class _Dp, class _Bp>
+concept derived_from = is_base_of_v<_Bp, _Dp> && is_convertible_v<const volatile _Dp*, const volatile _Bp*>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_DERIVED_FROM_H
diff --git a/libcxx/include/__cxx03/__concepts/destructible.h b/libcxx/include/__cxx03/__concepts/destructible.h
new file mode 100644
index 000000000000..28b4b1bc24ec
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/destructible.h
@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_DESTRUCTIBLE_H
+#define _LIBCPP___CONCEPTS_DESTRUCTIBLE_H
+
+#include <__config>
+#include <__type_traits/is_nothrow_destructible.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.destructible]
+
+template <class _Tp>
+concept destructible = is_nothrow_destructible_v<_Tp>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_DESTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__concepts/different_from.h b/libcxx/include/__cxx03/__concepts/different_from.h
new file mode 100644
index 000000000000..fd31f6e25805
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/different_from.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_DIFFERENT_FROM_H
+#define _LIBCPP___CONCEPTS_DIFFERENT_FROM_H
+
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__type_traits/remove_cvref.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp, class _Up>
+concept __different_from = !same_as<remove_cvref_t<_Tp>, remove_cvref_t<_Up>>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_DIFFERENT_FROM_H
diff --git a/libcxx/include/__cxx03/__concepts/equality_comparable.h b/libcxx/include/__cxx03/__concepts/equality_comparable.h
new file mode 100644
index 000000000000..278fc7640928
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/equality_comparable.h
@@ -0,0 +1,56 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_EQUALITY_COMPARABLE_H
+#define _LIBCPP___CONCEPTS_EQUALITY_COMPARABLE_H
+
+#include <__concepts/boolean_testable.h>
+#include <__concepts/common_reference_with.h>
+#include <__config>
+#include <__type_traits/common_reference.h>
+#include <__type_traits/make_const_lvalue_ref.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.equalitycomparable]
+
+template <class _Tp, class _Up>
+concept __weakly_equality_comparable_with =
+    requires(__make_const_lvalue_ref<_Tp> __t, __make_const_lvalue_ref<_Up> __u) {
+      { __t == __u } -> __boolean_testable;
+      { __t != __u } -> __boolean_testable;
+      { __u == __t } -> __boolean_testable;
+      { __u != __t } -> __boolean_testable;
+    };
+
+template <class _Tp>
+concept equality_comparable = __weakly_equality_comparable_with<_Tp, _Tp>;
+
+// clang-format off
+template <class _Tp, class _Up>
+concept equality_comparable_with =
+    equality_comparable<_Tp> && equality_comparable<_Up> &&
+    common_reference_with<__make_const_lvalue_ref<_Tp>, __make_const_lvalue_ref<_Up>> &&
+    equality_comparable<
+        common_reference_t<
+            __make_const_lvalue_ref<_Tp>,
+            __make_const_lvalue_ref<_Up>>> &&
+    __weakly_equality_comparable_with<_Tp, _Up>;
+// clang-format on
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_EQUALITY_COMPARABLE_H
diff --git a/libcxx/include/__cxx03/__concepts/invocable.h b/libcxx/include/__cxx03/__concepts/invocable.h
new file mode 100644
index 000000000000..8a29398b3a29
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/invocable.h
@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_INVOCABLE_H
+#define _LIBCPP___CONCEPTS_INVOCABLE_H
+
+#include <__config>
+#include <__functional/invoke.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.invocable]
+
+template <class _Fn, class... _Args>
+concept invocable = requires(_Fn&& __fn, _Args&&... __args) {
+  std::invoke(std::forward<_Fn>(__fn), std::forward<_Args>(__args)...); // not required to be equality preserving
+};
+
+// [concept.regular.invocable]
+
+template <class _Fn, class... _Args>
+concept regular_invocable = invocable<_Fn, _Args...>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_INVOCABLE_H
diff --git a/libcxx/include/__cxx03/__concepts/movable.h b/libcxx/include/__cxx03/__concepts/movable.h
new file mode 100644
index 000000000000..bc5b9d767c6a
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/movable.h
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_MOVABLE_H
+#define _LIBCPP___CONCEPTS_MOVABLE_H
+
+#include <__concepts/assignable.h>
+#include <__concepts/constructible.h>
+#include <__concepts/swappable.h>
+#include <__config>
+#include <__type_traits/is_object.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concepts.object]
+
+template <class _Tp>
+concept movable = is_object_v<_Tp> && move_constructible<_Tp> && assignable_from<_Tp&, _Tp> && swappable<_Tp>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_MOVABLE_H
diff --git a/libcxx/include/__cxx03/__concepts/predicate.h b/libcxx/include/__cxx03/__concepts/predicate.h
new file mode 100644
index 000000000000..00731efc8fcd
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/predicate.h
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_PREDICATE_H
+#define _LIBCPP___CONCEPTS_PREDICATE_H
+
+#include <__concepts/boolean_testable.h>
+#include <__concepts/invocable.h>
+#include <__config>
+#include <__functional/invoke.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.predicate]
+
+template <class _Fn, class... _Args>
+concept predicate = regular_invocable<_Fn, _Args...> && __boolean_testable<invoke_result_t<_Fn, _Args...>>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_PREDICATE_H
diff --git a/libcxx/include/__cxx03/__concepts/regular.h b/libcxx/include/__cxx03/__concepts/regular.h
new file mode 100644
index 000000000000..9f3d8bf30be3
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/regular.h
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_REGULAR_H
+#define _LIBCPP___CONCEPTS_REGULAR_H
+
+#include <__concepts/equality_comparable.h>
+#include <__concepts/semiregular.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.object]
+
+template <class _Tp>
+concept regular = semiregular<_Tp> && equality_comparable<_Tp>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_REGULAR_H
diff --git a/libcxx/include/__cxx03/__concepts/relation.h b/libcxx/include/__cxx03/__concepts/relation.h
new file mode 100644
index 000000000000..7545a7db93da
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/relation.h
@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_RELATION_H
+#define _LIBCPP___CONCEPTS_RELATION_H
+
+#include <__concepts/predicate.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.relation]
+
+template <class _Rp, class _Tp, class _Up>
+concept relation =
+    predicate<_Rp, _Tp, _Tp> && predicate<_Rp, _Up, _Up> && predicate<_Rp, _Tp, _Up> && predicate<_Rp, _Up, _Tp>;
+
+// [concept.equiv]
+
+template <class _Rp, class _Tp, class _Up>
+concept equivalence_relation = relation<_Rp, _Tp, _Up>;
+
+// [concept.strictweakorder]
+
+template <class _Rp, class _Tp, class _Up>
+concept strict_weak_order = relation<_Rp, _Tp, _Up>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_RELATION_H
diff --git a/libcxx/include/__cxx03/__concepts/same_as.h b/libcxx/include/__cxx03/__concepts/same_as.h
new file mode 100644
index 000000000000..4241131c70c1
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/same_as.h
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_SAME_AS_H
+#define _LIBCPP___CONCEPTS_SAME_AS_H
+
+#include <__config>
+#include <__type_traits/is_same.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.same]
+
+template <class _Tp, class _Up>
+concept __same_as_impl = _IsSame<_Tp, _Up>::value;
+
+template <class _Tp, class _Up>
+concept same_as = __same_as_impl<_Tp, _Up> && __same_as_impl<_Up, _Tp>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_SAME_AS_H
diff --git a/libcxx/include/__cxx03/__concepts/semiregular.h b/libcxx/include/__cxx03/__concepts/semiregular.h
new file mode 100644
index 000000000000..7a159d17dfc1
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/semiregular.h
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_SEMIREGULAR_H
+#define _LIBCPP___CONCEPTS_SEMIREGULAR_H
+
+#include <__concepts/constructible.h>
+#include <__concepts/copyable.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.object]
+
+template <class _Tp>
+concept semiregular = copyable<_Tp> && default_initializable<_Tp>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_SEMIREGULAR_H
diff --git a/libcxx/include/__cxx03/__concepts/swappable.h b/libcxx/include/__cxx03/__concepts/swappable.h
new file mode 100644
index 000000000000..d339488a087a
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/swappable.h
@@ -0,0 +1,123 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_SWAPPABLE_H
+#define _LIBCPP___CONCEPTS_SWAPPABLE_H
+
+#include <__concepts/assignable.h>
+#include <__concepts/class_or_enum.h>
+#include <__concepts/common_reference_with.h>
+#include <__concepts/constructible.h>
+#include <__config>
+#include <__type_traits/extent.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/exchange.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.swappable]
+
+namespace ranges {
+namespace __swap {
+
+template <class _Tp>
+void swap(_Tp&, _Tp&) = delete;
+
+// clang-format off
+template <class _Tp, class _Up>
+concept __unqualified_swappable_with =
+    (__class_or_enum<remove_cvref_t<_Tp>> || __class_or_enum<remove_cvref_t<_Up>>) &&
+    requires(_Tp&& __t, _Up&& __u) {
+        swap(std::forward<_Tp>(__t), std::forward<_Up>(__u));
+    };
+// clang-format on
+
+struct __fn;
+
+// clang-format off
+template <class _Tp, class _Up, size_t _Size>
+concept __swappable_arrays =
+    !__unqualified_swappable_with<_Tp (&)[_Size], _Up (&)[_Size]> &&
+    extent_v<_Tp> == extent_v<_Up> &&
+    requires(_Tp (&__t)[_Size], _Up (&__u)[_Size], const __fn& __swap) {
+        __swap(__t[0], __u[0]);
+    };
+// clang-format on
+
+template <class _Tp>
+concept __exchangeable =
+    !__unqualified_swappable_with<_Tp&, _Tp&> && move_constructible<_Tp> && assignable_from<_Tp&, _Tp>;
+
+struct __fn {
+  // 2.1   `S` is `(void)swap(E1, E2)`* if `E1` or `E2` has class or enumeration type and...
+  // *The name `swap` is used here unqualified.
+  template <class _Tp, class _Up>
+    requires __unqualified_swappable_with<_Tp, _Up>
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Tp&& __t, _Up&& __u) const
+      noexcept(noexcept(swap(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) {
+    swap(std::forward<_Tp>(__t), std::forward<_Up>(__u));
+  }
+
+  // 2.2   Otherwise, if `E1` and `E2` are lvalues of array types with equal extent and...
+  template <class _Tp, class _Up, size_t _Size>
+    requires __swappable_arrays<_Tp, _Up, _Size>
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Tp (&__t)[_Size], _Up (&__u)[_Size]) const
+      noexcept(noexcept((*this)(*__t, *__u))) {
+    // TODO(cjdb): replace with `ranges::swap_ranges`.
+    for (size_t __i = 0; __i < _Size; ++__i) {
+      (*this)(__t[__i], __u[__i]);
+    }
+  }
+
+  // 2.3   Otherwise, if `E1` and `E2` are lvalues of the same type `T` that models...
+  template <__exchangeable _Tp>
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Tp& __x, _Tp& __y) const
+      noexcept(is_nothrow_move_constructible_v<_Tp> && is_nothrow_move_assignable_v<_Tp>) {
+    __y = std::exchange(__x, std::move(__y));
+  }
+};
+} // namespace __swap
+
+inline namespace __cpo {
+inline constexpr auto swap = __swap::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+template <class _Tp>
+concept swappable = requires(_Tp& __a, _Tp& __b) { ranges::swap(__a, __b); };
+
+template <class _Tp, class _Up>
+concept swappable_with = common_reference_with<_Tp, _Up> && requires(_Tp&& __t, _Up&& __u) {
+  ranges::swap(std::forward<_Tp>(__t), std::forward<_Tp>(__t));
+  ranges::swap(std::forward<_Up>(__u), std::forward<_Up>(__u));
+  ranges::swap(std::forward<_Tp>(__t), std::forward<_Up>(__u));
+  ranges::swap(std::forward<_Up>(__u), std::forward<_Tp>(__t));
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___CONCEPTS_SWAPPABLE_H
diff --git a/libcxx/include/__cxx03/__concepts/totally_ordered.h b/libcxx/include/__cxx03/__concepts/totally_ordered.h
new file mode 100644
index 000000000000..186c3b430dd5
--- /dev/null
+++ b/libcxx/include/__cxx03/__concepts/totally_ordered.h
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONCEPTS_TOTALLY_ORDERED_H
+#define _LIBCPP___CONCEPTS_TOTALLY_ORDERED_H
+
+#include <__concepts/boolean_testable.h>
+#include <__concepts/equality_comparable.h>
+#include <__config>
+#include <__type_traits/common_reference.h>
+#include <__type_traits/make_const_lvalue_ref.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [concept.totallyordered]
+
+template <class _Tp, class _Up>
+concept __partially_ordered_with = requires(__make_const_lvalue_ref<_Tp> __t, __make_const_lvalue_ref<_Up> __u) {
+  { __t < __u } -> __boolean_testable;
+  { __t > __u } -> __boolean_testable;
+  { __t <= __u } -> __boolean_testable;
+  { __t >= __u } -> __boolean_testable;
+  { __u < __t } -> __boolean_testable;
+  { __u > __t } -> __boolean_testable;
+  { __u <= __t } -> __boolean_testable;
+  { __u >= __t } -> __boolean_testable;
+};
+
+template <class _Tp>
+concept totally_ordered = equality_comparable<_Tp> && __partially_ordered_with<_Tp, _Tp>;
+
+// clang-format off
+template <class _Tp, class _Up>
+concept totally_ordered_with =
+    totally_ordered<_Tp> && totally_ordered<_Up> &&
+    equality_comparable_with<_Tp, _Up> &&
+    totally_ordered<
+        common_reference_t<
+            __make_const_lvalue_ref<_Tp>,
+            __make_const_lvalue_ref<_Up>>> &&
+    __partially_ordered_with<_Tp, _Up>;
+// clang-format on
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___CONCEPTS_TOTALLY_ORDERED_H
diff --git a/libcxx/include/__cxx03/__condition_variable/condition_variable.h b/libcxx/include/__cxx03/__condition_variable/condition_variable.h
new file mode 100644
index 000000000000..de35aaca1070
--- /dev/null
+++ b/libcxx/include/__cxx03/__condition_variable/condition_variable.h
@@ -0,0 +1,244 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONDITION_VARIABLE_CONDITION_VARIABLE_H
+#define _LIBCPP___CONDITION_VARIABLE_CONDITION_VARIABLE_H
+
+#include <__chrono/duration.h>
+#include <__chrono/steady_clock.h>
+#include <__chrono/system_clock.h>
+#include <__chrono/time_point.h>
+#include <__config>
+#include <__mutex/mutex.h>
+#include <__mutex/unique_lock.h>
+#include <__system_error/system_error.h>
+#include <__thread/support.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_floating_point.h>
+#include <__utility/move.h>
+#include <limits>
+#include <ratio>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifndef _LIBCPP_HAS_NO_THREADS
+
+// enum class cv_status
+_LIBCPP_DECLARE_STRONG_ENUM(cv_status){no_timeout, timeout};
+_LIBCPP_DECLARE_STRONG_ENUM_EPILOG(cv_status)
+
+class _LIBCPP_EXPORTED_FROM_ABI condition_variable {
+  __libcpp_condvar_t __cv_ = _LIBCPP_CONDVAR_INITIALIZER;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR condition_variable() _NOEXCEPT = default;
+
+#  ifdef _LIBCPP_HAS_TRIVIAL_CONDVAR_DESTRUCTION
+  ~condition_variable() = default;
+#  else
+  ~condition_variable();
+#  endif
+
+  condition_variable(const condition_variable&)            = delete;
+  condition_variable& operator=(const condition_variable&) = delete;
+
+  void notify_one() _NOEXCEPT;
+  void notify_all() _NOEXCEPT;
+
+  void wait(unique_lock<mutex>& __lk) _NOEXCEPT;
+  template <class _Predicate>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS void wait(unique_lock<mutex>& __lk, _Predicate __pred);
+
+  template <class _Clock, class _Duration>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS cv_status
+  wait_until(unique_lock<mutex>& __lk, const chrono::time_point<_Clock, _Duration>& __t);
+
+  template <class _Clock, class _Duration, class _Predicate>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS bool
+  wait_until(unique_lock<mutex>& __lk, const chrono::time_point<_Clock, _Duration>& __t, _Predicate __pred);
+
+  template <class _Rep, class _Period>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS cv_status
+  wait_for(unique_lock<mutex>& __lk, const chrono::duration<_Rep, _Period>& __d);
+
+  template <class _Rep, class _Period, class _Predicate>
+  bool _LIBCPP_HIDE_FROM_ABI
+  wait_for(unique_lock<mutex>& __lk, const chrono::duration<_Rep, _Period>& __d, _Predicate __pred);
+
+  typedef __libcpp_condvar_t* native_handle_type;
+  _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__cv_; }
+
+private:
+  void
+  __do_timed_wait(unique_lock<mutex>& __lk, chrono::time_point<chrono::system_clock, chrono::nanoseconds>) _NOEXCEPT;
+#  if defined(_LIBCPP_HAS_COND_CLOCKWAIT)
+  _LIBCPP_HIDE_FROM_ABI void
+  __do_timed_wait(unique_lock<mutex>& __lk, chrono::time_point<chrono::steady_clock, chrono::nanoseconds>) _NOEXCEPT;
+#  endif
+  template <class _Clock>
+  _LIBCPP_HIDE_FROM_ABI void
+  __do_timed_wait(unique_lock<mutex>& __lk, chrono::time_point<_Clock, chrono::nanoseconds>) _NOEXCEPT;
+};
+#endif // !_LIBCPP_HAS_NO_THREADS
+
+template <class _Rep, class _Period, __enable_if_t<is_floating_point<_Rep>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI chrono::nanoseconds __safe_nanosecond_cast(chrono::duration<_Rep, _Period> __d) {
+  using namespace chrono;
+  using __ratio       = ratio_divide<_Period, nano>;
+  using __ns_rep      = nanoseconds::rep;
+  _Rep __result_float = __d.count() * __ratio::num / __ratio::den;
+
+  _Rep __result_max = numeric_limits<__ns_rep>::max();
+  if (__result_float >= __result_max) {
+    return nanoseconds::max();
+  }
+
+  _Rep __result_min = numeric_limits<__ns_rep>::min();
+  if (__result_float <= __result_min) {
+    return nanoseconds::min();
+  }
+
+  return nanoseconds(static_cast<__ns_rep>(__result_float));
+}
+
+template <class _Rep, class _Period, __enable_if_t<!is_floating_point<_Rep>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI chrono::nanoseconds __safe_nanosecond_cast(chrono::duration<_Rep, _Period> __d) {
+  using namespace chrono;
+  if (__d.count() == 0) {
+    return nanoseconds(0);
+  }
+
+  using __ratio         = ratio_divide<_Period, nano>;
+  using __ns_rep        = nanoseconds::rep;
+  __ns_rep __result_max = numeric_limits<__ns_rep>::max();
+  if (__d.count() > 0 && __d.count() > __result_max / __ratio::num) {
+    return nanoseconds::max();
+  }
+
+  __ns_rep __result_min = numeric_limits<__ns_rep>::min();
+  if (__d.count() < 0 && __d.count() < __result_min / __ratio::num) {
+    return nanoseconds::min();
+  }
+
+  __ns_rep __result = __d.count() * __ratio::num / __ratio::den;
+  if (__result == 0) {
+    return nanoseconds(1);
+  }
+
+  return nanoseconds(__result);
+}
+
+#ifndef _LIBCPP_HAS_NO_THREADS
+template <class _Predicate>
+void condition_variable::wait(unique_lock<mutex>& __lk, _Predicate __pred) {
+  while (!__pred())
+    wait(__lk);
+}
+
+template <class _Clock, class _Duration>
+cv_status condition_variable::wait_until(unique_lock<mutex>& __lk, const chrono::time_point<_Clock, _Duration>& __t) {
+  using namespace chrono;
+  using __clock_tp_ns = time_point<_Clock, nanoseconds>;
+
+  typename _Clock::time_point __now = _Clock::now();
+  if (__t <= __now)
+    return cv_status::timeout;
+
+  __clock_tp_ns __t_ns = __clock_tp_ns(std::__safe_nanosecond_cast(__t.time_since_epoch()));
+
+  __do_timed_wait(__lk, __t_ns);
+  return _Clock::now() < __t ? cv_status::no_timeout : cv_status::timeout;
+}
+
+template <class _Clock, class _Duration, class _Predicate>
+bool condition_variable::wait_until(
+    unique_lock<mutex>& __lk, const chrono::time_point<_Clock, _Duration>& __t, _Predicate __pred) {
+  while (!__pred()) {
+    if (wait_until(__lk, __t) == cv_status::timeout)
+      return __pred();
+  }
+  return true;
+}
+
+template <class _Rep, class _Period>
+cv_status condition_variable::wait_for(unique_lock<mutex>& __lk, const chrono::duration<_Rep, _Period>& __d) {
+  using namespace chrono;
+  if (__d <= __d.zero())
+    return cv_status::timeout;
+  using __ns_rep                   = nanoseconds::rep;
+  steady_clock::time_point __c_now = steady_clock::now();
+
+#  if defined(_LIBCPP_HAS_COND_CLOCKWAIT)
+  using __clock_tp_ns     = time_point<steady_clock, nanoseconds>;
+  __ns_rep __now_count_ns = std::__safe_nanosecond_cast(__c_now.time_since_epoch()).count();
+#  else
+  using __clock_tp_ns     = time_point<system_clock, nanoseconds>;
+  __ns_rep __now_count_ns = std::__safe_nanosecond_cast(system_clock::now().time_since_epoch()).count();
+#  endif
+
+  __ns_rep __d_ns_count = std::__safe_nanosecond_cast(__d).count();
+
+  if (__now_count_ns > numeric_limits<__ns_rep>::max() - __d_ns_count) {
+    __do_timed_wait(__lk, __clock_tp_ns::max());
+  } else {
+    __do_timed_wait(__lk, __clock_tp_ns(nanoseconds(__now_count_ns + __d_ns_count)));
+  }
+
+  return steady_clock::now() - __c_now < __d ? cv_status::no_timeout : cv_status::timeout;
+}
+
+template <class _Rep, class _Period, class _Predicate>
+inline bool
+condition_variable::wait_for(unique_lock<mutex>& __lk, const chrono::duration<_Rep, _Period>& __d, _Predicate __pred) {
+  return wait_until(__lk, chrono::steady_clock::now() + __d, std::move(__pred));
+}
+
+#  if defined(_LIBCPP_HAS_COND_CLOCKWAIT)
+inline void condition_variable::__do_timed_wait(
+    unique_lock<mutex>& __lk, chrono::time_point<chrono::steady_clock, chrono::nanoseconds> __tp) _NOEXCEPT {
+  using namespace chrono;
+  if (!__lk.owns_lock())
+    __throw_system_error(EPERM, "condition_variable::timed wait: mutex not locked");
+  nanoseconds __d = __tp.time_since_epoch();
+  timespec __ts;
+  seconds __s                 = duration_cast<seconds>(__d);
+  using __ts_sec              = decltype(__ts.tv_sec);
+  const __ts_sec __ts_sec_max = numeric_limits<__ts_sec>::max();
+  if (__s.count() < __ts_sec_max) {
+    __ts.tv_sec  = static_cast<__ts_sec>(__s.count());
+    __ts.tv_nsec = (__d - __s).count();
+  } else {
+    __ts.tv_sec  = __ts_sec_max;
+    __ts.tv_nsec = giga::num - 1;
+  }
+  int __ec = pthread_cond_clockwait(&__cv_, __lk.mutex()->native_handle(), CLOCK_MONOTONIC, &__ts);
+  if (__ec != 0 && __ec != ETIMEDOUT)
+    __throw_system_error(__ec, "condition_variable timed_wait failed");
+}
+#  endif // _LIBCPP_HAS_COND_CLOCKWAIT
+
+template <class _Clock>
+inline void condition_variable::__do_timed_wait(unique_lock<mutex>& __lk,
+                                                chrono::time_point<_Clock, chrono::nanoseconds> __tp) _NOEXCEPT {
+  wait_for(__lk, __tp - _Clock::now());
+}
+
+#endif // _LIBCPP_HAS_NO_THREADS
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___CONDITION_VARIABLE_CONDITION_VARIABLE_H
diff --git a/libcxx/include/__cxx03/__config b/libcxx/include/__cxx03/__config
new file mode 100644
index 000000000000..661af5be3c22
--- /dev/null
+++ b/libcxx/include/__cxx03/__config
@@ -0,0 +1,1228 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONFIG
+#define _LIBCPP___CONFIG
+
+#include <__config_site>
+#include <__configuration/abi.h>
+#include <__configuration/availability.h>
+#include <__configuration/compiler.h>
+#include <__configuration/platform.h>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+#ifdef __cplusplus
+
+// The attributes supported by clang are documented at https://clang.llvm.org/docs/AttributeReference.html
+
+// _LIBCPP_VERSION represents the version of libc++, which matches the version of LLVM.
+// Given a LLVM release LLVM XX.YY.ZZ (e.g. LLVM 17.0.1 == 17.00.01), _LIBCPP_VERSION is
+// defined to XXYYZZ.
+#  define _LIBCPP_VERSION 190100
+
+#  define _LIBCPP_CONCAT_IMPL(_X, _Y) _X##_Y
+#  define _LIBCPP_CONCAT(_X, _Y) _LIBCPP_CONCAT_IMPL(_X, _Y)
+
+#  if __STDC_HOSTED__ == 0
+#    define _LIBCPP_FREESTANDING
+#  endif
+
+// HARDENING {
+
+// This is for backward compatibility -- make enabling `_LIBCPP_ENABLE_ASSERTIONS` (which predates hardening modes)
+// equivalent to setting the extensive mode. This is deprecated and will be removed in LLVM 20.
+#  ifdef _LIBCPP_ENABLE_ASSERTIONS
+#    warning "_LIBCPP_ENABLE_ASSERTIONS is deprecated, please use _LIBCPP_HARDENING_MODE instead"
+#    if _LIBCPP_ENABLE_ASSERTIONS != 0 && _LIBCPP_ENABLE_ASSERTIONS != 1
+#      error "_LIBCPP_ENABLE_ASSERTIONS must be set to 0 or 1"
+#    endif
+#    if _LIBCPP_ENABLE_ASSERTIONS
+#      define _LIBCPP_HARDENING_MODE _LIBCPP_HARDENING_MODE_EXTENSIVE
+#    endif
+#  endif
+
+// The library provides the macro `_LIBCPP_HARDENING_MODE` which can be set to one of the following values:
+//
+// - `_LIBCPP_HARDENING_MODE_NONE`;
+// - `_LIBCPP_HARDENING_MODE_FAST`;
+// - `_LIBCPP_HARDENING_MODE_EXTENSIVE`;
+// - `_LIBCPP_HARDENING_MODE_DEBUG`.
+//
+// These values have the following effects:
+//
+// - `_LIBCPP_HARDENING_MODE_NONE` -- sets the hardening mode to "none" which disables all runtime hardening checks;
+//
+// - `_LIBCPP_HARDENING_MODE_FAST` -- sets that hardening mode to "fast". The fast mode enables security-critical checks
+//   that can be done with relatively little runtime overhead in constant time;
+//
+// - `_LIBCPP_HARDENING_MODE_EXTENSIVE` -- sets the hardening mode to "extensive". The extensive mode is a superset of
+//   the fast mode that additionally enables checks that are relatively cheap and prevent common types of logic errors
+//   but are not necessarily security-critical;
+//
+// - `_LIBCPP_HARDENING_MODE_DEBUG` -- sets the hardening mode to "debug". The debug mode is a superset of the extensive
+//   mode and enables all checks available in the library, including internal assertions. Checks that are part of the
+//   debug mode can be very expensive and thus the debug mode is intended to be used for testing, not in production.
+
+// Inside the library, assertions are categorized so they can be cherry-picked based on the chosen hardening mode. These
+// macros are only for internal use -- users should only pick one of the high-level hardening modes described above.
+//
+// - `_LIBCPP_ASSERT_VALID_INPUT_RANGE` -- checks that ranges (whether expressed as an iterator pair, an iterator and
+//   a sentinel, an iterator and a count, or a `std::range`) given as input to library functions are valid:
+//   - the sentinel is reachable from the begin iterator;
+//   - TODO(hardening): both iterators refer to the same container.
+//
+// - `_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS` -- checks that any attempts to access a container element, whether through
+//   the container object or through an iterator, are valid and do not attempt to go out of bounds or otherwise access
+//   a non-existent element. For iterator checks to work, bounded iterators must be enabled in the ABI. Types like
+//   `optional` and `function` are considered one-element containers for the purposes of this check.
+//
+// - `_LIBCPP_ASSERT_NON_NULL` -- checks that the pointer being dereferenced is not null. On most modern platforms zero
+//   address does not refer to an actual location in memory, so a null pointer dereference would not compromize the
+//   memory security of a program (however, it is still undefined behavior that can result in strange errors due to
+//   compiler optimizations).
+//
+// - `_LIBCPP_ASSERT_NON_OVERLAPPING_RANGES` -- for functions that take several ranges as arguments, checks that the
+//   given ranges do not overlap.
+//
+// - `_LIBCPP_ASSERT_VALID_DEALLOCATION` -- checks that an attempt to deallocate memory is valid (e.g. the given object
+//   was allocated by the given allocator). Violating this category typically results in a memory leak.
+//
+// - `_LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL` -- checks that a call to an external API doesn't fail in
+//   an unexpected manner. This includes triggering documented cases of undefined behavior in an external library (like
+//   attempting to unlock an unlocked mutex in pthreads). Any API external to the library falls under this category
+//   (from system calls to compiler intrinsics). We generally don't expect these failures to compromize memory safety or
+//   otherwise create an immediate security issue.
+//
+// - `_LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR` -- checks any operations that exchange nodes between containers to make sure
+//   the containers have compatible allocators.
+//
+// - `_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN` -- checks that the given argument is within the domain of valid arguments
+//   for the function. Violating this typically produces an incorrect result (e.g. the clamp algorithm returns the
+//   original value without clamping it due to incorrect functors) or puts an object into an invalid state (e.g.
+//   a string view where only a subset of elements is possible to access). This category is for assertions violating
+//   which doesn't cause any immediate issues in the library -- whatever the consequences are, they will happen in the
+//   user code.
+//
+// - `_LIBCPP_ASSERT_PEDANTIC` -- checks prerequisites which are imposed by the Standard, but violating which happens to
+//   be benign in our implementation.
+//
+// - `_LIBCPP_ASSERT_SEMANTIC_REQUIREMENT` -- checks that the given argument satisfies the semantic requirements imposed
+//   by the Standard. Typically, there is no simple way to completely prove that a semantic requirement is satisfied;
+//   thus, this would often be a heuristic check and it might be quite expensive.
+//
+// - `_LIBCPP_ASSERT_INTERNAL` -- checks that internal invariants of the library hold. These assertions don't depend on
+//   user input.
+//
+// - `_LIBCPP_ASSERT_UNCATEGORIZED` -- for assertions that haven't been properly classified yet.
+
+// clang-format off
+#  define _LIBCPP_HARDENING_MODE_NONE      (1 << 1)
+#  define _LIBCPP_HARDENING_MODE_FAST      (1 << 2)
+#  define _LIBCPP_HARDENING_MODE_EXTENSIVE (1 << 4) // Deliberately not ordered.
+#  define _LIBCPP_HARDENING_MODE_DEBUG     (1 << 3)
+// clang-format on
+
+#  ifndef _LIBCPP_HARDENING_MODE
+
+#    ifndef _LIBCPP_HARDENING_MODE_DEFAULT
+#      error _LIBCPP_HARDENING_MODE_DEFAULT is not defined. This definition should be set at configuration time in the \
+`__config_site` header, please make sure your installation of libc++ is not broken.
+#    endif
+
+#    define _LIBCPP_HARDENING_MODE _LIBCPP_HARDENING_MODE_DEFAULT
+#  endif
+
+#  if _LIBCPP_HARDENING_MODE != _LIBCPP_HARDENING_MODE_NONE &&                                                         \
+      _LIBCPP_HARDENING_MODE != _LIBCPP_HARDENING_MODE_FAST &&                                                         \
+      _LIBCPP_HARDENING_MODE != _LIBCPP_HARDENING_MODE_EXTENSIVE &&                                                    \
+      _LIBCPP_HARDENING_MODE != _LIBCPP_HARDENING_MODE_DEBUG
+#    error _LIBCPP_HARDENING_MODE must be set to one of the following values: \
+_LIBCPP_HARDENING_MODE_NONE, \
+_LIBCPP_HARDENING_MODE_FAST, \
+_LIBCPP_HARDENING_MODE_EXTENSIVE, \
+_LIBCPP_HARDENING_MODE_DEBUG
+#  endif
+
+// } HARDENING
+
+#  define _LIBCPP_TOSTRING2(x) #x
+#  define _LIBCPP_TOSTRING(x) _LIBCPP_TOSTRING2(x)
+
+// NOLINTNEXTLINE(libcpp-cpp-version-check)
+#  if __cplusplus < 201103L
+#    define _LIBCPP_CXX03_LANG
+#  endif
+
+#  ifndef __has_constexpr_builtin
+#    define __has_constexpr_builtin(x) 0
+#  endif
+
+// This checks wheter a Clang module is built
+#  ifndef __building_module
+#    define __building_module(...) 0
+#  endif
+
+// '__is_identifier' returns '0' if '__x' is a reserved identifier provided by
+// the compiler and '1' otherwise.
+#  ifndef __is_identifier
+#    define __is_identifier(__x) 1
+#  endif
+
+#  ifndef __has_declspec_attribute
+#    define __has_declspec_attribute(__x) 0
+#  endif
+
+#  define __has_keyword(__x) !(__is_identifier(__x))
+
+#  ifndef __has_warning
+#    define __has_warning(...) 0
+#  endif
+
+#  if !defined(_LIBCPP_COMPILER_CLANG_BASED) && __cplusplus < 201103L
+#    error "libc++ only supports C++03 with Clang-based compilers. Please enable C++11"
+#  endif
+
+// FIXME: ABI detection should be done via compiler builtin macros. This
+// is just a placeholder until Clang implements such macros. For now assume
+// that Windows compilers pretending to be MSVC++ target the Microsoft ABI,
+// and allow the user to explicitly specify the ABI to handle cases where this
+// heuristic falls short.
+#  if defined(_LIBCPP_ABI_FORCE_ITANIUM) && defined(_LIBCPP_ABI_FORCE_MICROSOFT)
+#    error "Only one of _LIBCPP_ABI_FORCE_ITANIUM and _LIBCPP_ABI_FORCE_MICROSOFT can be defined"
+#  elif defined(_LIBCPP_ABI_FORCE_ITANIUM)
+#    define _LIBCPP_ABI_ITANIUM
+#  elif defined(_LIBCPP_ABI_FORCE_MICROSOFT)
+#    define _LIBCPP_ABI_MICROSOFT
+#  else
+#    if defined(_WIN32) && defined(_MSC_VER)
+#      define _LIBCPP_ABI_MICROSOFT
+#    else
+#      define _LIBCPP_ABI_ITANIUM
+#    endif
+#  endif
+
+#  if defined(_LIBCPP_ABI_MICROSOFT) && !defined(_LIBCPP_NO_VCRUNTIME)
+#    define _LIBCPP_ABI_VCRUNTIME
+#  endif
+
+#  if __has_feature(experimental_library)
+#    ifndef _LIBCPP_ENABLE_EXPERIMENTAL
+#      define _LIBCPP_ENABLE_EXPERIMENTAL
+#    endif
+#  endif
+
+// Incomplete features get their own specific disabling flags. This makes it
+// easier to grep for target specific flags once the feature is complete.
+#  if !defined(_LIBCPP_ENABLE_EXPERIMENTAL) && !defined(_LIBCPP_BUILDING_LIBRARY)
+#    define _LIBCPP_HAS_NO_INCOMPLETE_PSTL
+#    define _LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN
+#    define _LIBCPP_HAS_NO_EXPERIMENTAL_TZDB
+#    define _LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM
+#  endif
+
+#  if defined(__MVS__)
+#    include <features.h> // for __NATIVE_ASCII_F
+#  endif
+
+#  if defined(_WIN32)
+#    define _LIBCPP_WIN32API
+#    define _LIBCPP_SHORT_WCHAR 1
+// Both MinGW and native MSVC provide a "MSVC"-like environment
+#    define _LIBCPP_MSVCRT_LIKE
+// If mingw not explicitly detected, assume using MS C runtime only if
+// a MS compatibility version is specified.
+#    if defined(_MSC_VER) && !defined(__MINGW32__)
+#      define _LIBCPP_MSVCRT // Using Microsoft's C Runtime library
+#    endif
+#    if (defined(_M_AMD64) || defined(__x86_64__)) || (defined(_M_ARM) || defined(__arm__))
+#      define _LIBCPP_HAS_BITSCAN64
+#    endif
+#    define _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  endif // defined(_WIN32)
+
+#  if defined(_AIX) && !defined(__64BIT__)
+// The size of wchar is 2 byte on 32-bit mode on AIX.
+#    define _LIBCPP_SHORT_WCHAR 1
+#  endif
+
+// Libc++ supports various implementations of std::random_device.
+//
+// _LIBCPP_USING_DEV_RANDOM
+//      Read entropy from the given file, by default `/dev/urandom`.
+//      If a token is provided, it is assumed to be the path to a file
+//      to read entropy from. This is the default behavior if nothing
+//      else is specified. This implementation requires storing state
+//      inside `std::random_device`.
+//
+// _LIBCPP_USING_ARC4_RANDOM
+//      Use arc4random(). This allows obtaining random data even when
+//      using sandboxing mechanisms. On some platforms like Apple, this
+//      is the recommended source of entropy for user-space programs.
+//      When this option is used, the token passed to `std::random_device`'s
+//      constructor *must* be "/dev/urandom" -- anything else is an error.
+//
+// _LIBCPP_USING_GETENTROPY
+//      Use getentropy().
+//      When this option is used, the token passed to `std::random_device`'s
+//      constructor *must* be "/dev/urandom" -- anything else is an error.
+//
+// _LIBCPP_USING_FUCHSIA_CPRNG
+//      Use Fuchsia's zx_cprng_draw() system call, which is specified to
+//      deliver high-quality entropy and cannot fail.
+//      When this option is used, the token passed to `std::random_device`'s
+//      constructor *must* be "/dev/urandom" -- anything else is an error.
+//
+// _LIBCPP_USING_NACL_RANDOM
+//      NaCl's sandbox (which PNaCl also runs in) doesn't allow filesystem access,
+//      including accesses to the special files under `/dev`. This implementation
+//      uses the NaCL syscall `nacl_secure_random_init()` to get entropy.
+//      When this option is used, the token passed to `std::random_device`'s
+//      constructor *must* be "/dev/urandom" -- anything else is an error.
+//
+// _LIBCPP_USING_WIN32_RANDOM
+//      Use rand_s(), for use on Windows.
+//      When this option is used, the token passed to `std::random_device`'s
+//      constructor *must* be "/dev/urandom" -- anything else is an error.
+#  if defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) ||                     \
+      defined(__DragonFly__)
+#    define _LIBCPP_USING_ARC4_RANDOM
+#  elif defined(__wasi__) || defined(__EMSCRIPTEN__)
+#    define _LIBCPP_USING_GETENTROPY
+#  elif defined(__Fuchsia__)
+#    define _LIBCPP_USING_FUCHSIA_CPRNG
+#  elif defined(__native_client__)
+#    define _LIBCPP_USING_NACL_RANDOM
+#  elif defined(_LIBCPP_WIN32API)
+#    define _LIBCPP_USING_WIN32_RANDOM
+#  else
+#    define _LIBCPP_USING_DEV_RANDOM
+#  endif
+
+#  ifndef _LIBCPP_CXX03_LANG
+
+#    define _LIBCPP_ALIGNOF(_Tp) alignof(_Tp)
+#    define _ALIGNAS_TYPE(x) alignas(x)
+#    define _ALIGNAS(x) alignas(x)
+#    define _LIBCPP_NORETURN [[noreturn]]
+#    define _NOEXCEPT noexcept
+#    define _NOEXCEPT_(...) noexcept(__VA_ARGS__)
+#    define _LIBCPP_CONSTEXPR constexpr
+
+#  else
+
+#    define _LIBCPP_ALIGNOF(_Tp) _Alignof(_Tp)
+#    define _ALIGNAS_TYPE(x) __attribute__((__aligned__(_LIBCPP_ALIGNOF(x))))
+#    define _ALIGNAS(x) __attribute__((__aligned__(x)))
+#    define _LIBCPP_NORETURN __attribute__((__noreturn__))
+#    define _LIBCPP_HAS_NO_NOEXCEPT
+#    define nullptr __nullptr
+#    define _NOEXCEPT throw()
+#    define _NOEXCEPT_(...)
+#    define static_assert(...) _Static_assert(__VA_ARGS__)
+#    define decltype(...) __decltype(__VA_ARGS__)
+#    define _LIBCPP_CONSTEXPR
+
+typedef __char16_t char16_t;
+typedef __char32_t char32_t;
+
+#  endif
+
+#  define _LIBCPP_PREFERRED_ALIGNOF(_Tp) __alignof(_Tp)
+
+// Objective-C++ features (opt-in)
+#  if __has_feature(objc_arc)
+#    define _LIBCPP_HAS_OBJC_ARC
+#  endif
+
+#  if __has_feature(objc_arc_weak)
+#    define _LIBCPP_HAS_OBJC_ARC_WEAK
+#  endif
+
+#  if __has_extension(blocks)
+#    define _LIBCPP_HAS_EXTENSION_BLOCKS
+#  endif
+
+#  if defined(_LIBCPP_HAS_EXTENSION_BLOCKS) && defined(__APPLE__)
+#    define _LIBCPP_HAS_BLOCKS_RUNTIME
+#  endif
+
+#  if !__has_feature(address_sanitizer)
+#    define _LIBCPP_HAS_NO_ASAN
+#  endif
+
+#  define _LIBCPP_ALWAYS_INLINE __attribute__((__always_inline__))
+
+#  define _LIBCPP_DISABLE_EXTENSION_WARNING __extension__
+
+#  if defined(_LIBCPP_OBJECT_FORMAT_COFF)
+
+#    ifdef _DLL
+#      define _LIBCPP_CRT_FUNC __declspec(dllimport)
+#    else
+#      define _LIBCPP_CRT_FUNC
+#    endif
+
+#    if defined(_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS) || (defined(__MINGW32__) && !defined(_LIBCPP_BUILDING_LIBRARY))
+#      define _LIBCPP_DLL_VIS
+#      define _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS
+#      define _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS
+#      define _LIBCPP_OVERRIDABLE_FUNC_VIS
+#      define _LIBCPP_EXPORTED_FROM_ABI
+#    elif defined(_LIBCPP_BUILDING_LIBRARY)
+#      define _LIBCPP_DLL_VIS __declspec(dllexport)
+#      if defined(__MINGW32__)
+#        define _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS _LIBCPP_DLL_VIS
+#        define _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS
+#      else
+#        define _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS
+#        define _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS _LIBCPP_DLL_VIS
+#      endif
+#      define _LIBCPP_OVERRIDABLE_FUNC_VIS _LIBCPP_DLL_VIS
+#      define _LIBCPP_EXPORTED_FROM_ABI __declspec(dllexport)
+#    else
+#      define _LIBCPP_DLL_VIS __declspec(dllimport)
+#      define _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS _LIBCPP_DLL_VIS
+#      define _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS
+#      define _LIBCPP_OVERRIDABLE_FUNC_VIS
+#      define _LIBCPP_EXPORTED_FROM_ABI __declspec(dllimport)
+#    endif
+
+#    define _LIBCPP_HIDDEN
+#    define _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS
+#    define _LIBCPP_TEMPLATE_VIS
+#    define _LIBCPP_TEMPLATE_DATA_VIS
+#    define _LIBCPP_TYPE_VISIBILITY_DEFAULT
+
+#  else
+
+#    if !defined(_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS)
+#      define _LIBCPP_VISIBILITY(vis) __attribute__((__visibility__(vis)))
+#    else
+#      define _LIBCPP_VISIBILITY(vis)
+#    endif
+
+#    define _LIBCPP_HIDDEN _LIBCPP_VISIBILITY("hidden")
+#    define _LIBCPP_TEMPLATE_DATA_VIS _LIBCPP_VISIBILITY("default")
+#    define _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_VISIBILITY("default")
+#    define _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS _LIBCPP_VISIBILITY("default")
+#    define _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS
+
+// TODO: Make this a proper customization point or remove the option to override it.
+#    ifndef _LIBCPP_OVERRIDABLE_FUNC_VIS
+#      define _LIBCPP_OVERRIDABLE_FUNC_VIS _LIBCPP_VISIBILITY("default")
+#    endif
+
+#    if !defined(_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS)
+// The inline should be removed once PR32114 is resolved
+#      define _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS inline _LIBCPP_HIDDEN
+#    else
+#      define _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS
+#    endif
+
+// GCC doesn't support the type_visibility attribute, so we have to keep the visibility attribute on templates
+#    if !defined(_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS) && !__has_attribute(__type_visibility__)
+#      define _LIBCPP_TEMPLATE_VIS __attribute__((__visibility__("default")))
+#    else
+#      define _LIBCPP_TEMPLATE_VIS
+#    endif
+
+#    if !defined(_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS) && __has_attribute(__type_visibility__)
+#      define _LIBCPP_TYPE_VISIBILITY_DEFAULT __attribute__((__type_visibility__("default")))
+#    else
+#      define _LIBCPP_TYPE_VISIBILITY_DEFAULT
+#    endif
+
+#  endif // defined(_LIBCPP_OBJECT_FORMAT_COFF)
+
+#  if __has_attribute(exclude_from_explicit_instantiation)
+#    define _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION __attribute__((__exclude_from_explicit_instantiation__))
+#  else
+// Try to approximate the effect of exclude_from_explicit_instantiation
+// (which is that entities are not assumed to be provided by explicit
+// template instantiations in the dylib) by always inlining those entities.
+#    define _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION _LIBCPP_ALWAYS_INLINE
+#  endif
+
+#  ifdef _LIBCPP_COMPILER_CLANG_BASED
+#    define _LIBCPP_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push")
+#    define _LIBCPP_DIAGNOSTIC_POP _Pragma("clang diagnostic pop")
+#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED(str) _Pragma(_LIBCPP_TOSTRING(clang diagnostic ignored str))
+#    define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str)
+#  elif defined(_LIBCPP_COMPILER_GCC)
+#    define _LIBCPP_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push")
+#    define _LIBCPP_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop")
+#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED(str)
+#    define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str) _Pragma(_LIBCPP_TOSTRING(GCC diagnostic ignored str))
+#  else
+#    define _LIBCPP_DIAGNOSTIC_PUSH
+#    define _LIBCPP_DIAGNOSTIC_POP
+#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED(str)
+#    define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str)
+#  endif
+
+#  if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_FAST
+#    define _LIBCPP_HARDENING_SIG f
+#  elif _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_EXTENSIVE
+#    define _LIBCPP_HARDENING_SIG s
+#  elif _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG
+#    define _LIBCPP_HARDENING_SIG d
+#  else
+#    define _LIBCPP_HARDENING_SIG n // "none"
+#  endif
+
+#  ifdef _LIBCPP_HAS_NO_EXCEPTIONS
+#    define _LIBCPP_EXCEPTIONS_SIG n
+#  else
+#    define _LIBCPP_EXCEPTIONS_SIG e
+#  endif
+
+#  define _LIBCPP_ODR_SIGNATURE                                                                                        \
+    _LIBCPP_CONCAT(_LIBCPP_CONCAT(_LIBCPP_HARDENING_SIG, _LIBCPP_EXCEPTIONS_SIG), _LIBCPP_VERSION)
+
+// This macro marks a symbol as being hidden from libc++'s ABI. This is achieved
+// on two levels:
+// 1. The symbol is given hidden visibility, which ensures that users won't start exporting
+//    symbols from their dynamic library by means of using the libc++ headers. This ensures
+//    that those symbols stay private to the dynamic library in which it is defined.
+//
+// 2. The symbol is given an ABI tag that encodes the ODR-relevant properties of the library.
+//    This ensures that no ODR violation can arise from mixing two TUs compiled with different
+//    versions or configurations of libc++ (such as exceptions vs no-exceptions). Indeed, if the
+//    program contains two definitions of a function, the ODR requires them to be token-by-token
+//    equivalent, and the linker is allowed to pick either definition and discard the other one.
+//
+//    For example, if a program contains a copy of `vector::at()` compiled with exceptions enabled
+//    *and* a copy of `vector::at()` compiled with exceptions disabled (by means of having two TUs
+//    compiled with different settings), the two definitions are both visible by the linker and they
+//    have the same name, but they have a meaningfully different implementation (one throws an exception
+//    and the other aborts the program). This violates the ODR and makes the program ill-formed, and in
+//    practice what will happen is that the linker will pick one of the definitions at random and will
+//    discard the other one. This can quite clearly lead to incorrect program behavior.
+//
+//    A similar reasoning holds for many other properties that are ODR-affecting. Essentially any
+//    property that causes the code of a function to differ from the code in another configuration
+//    can be considered ODR-affecting. In practice, we don't encode all such properties in the ABI
+//    tag, but we encode the ones that we think are most important: library version, exceptions, and
+//    hardening mode.
+//
+//    Note that historically, solving this problem has been achieved in various ways, including
+//    force-inlining all functions or giving internal linkage to all functions. Both these previous
+//    solutions suffer from drawbacks that lead notably to code bloat.
+//
+// Note that we use _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION to ensure that we don't depend
+// on _LIBCPP_HIDE_FROM_ABI methods of classes explicitly instantiated in the dynamic library.
+//
+// Also note that the _LIBCPP_HIDE_FROM_ABI_VIRTUAL macro should be used on virtual functions
+// instead of _LIBCPP_HIDE_FROM_ABI. That macro does not use an ABI tag. Indeed, the mangled
+// name of a virtual function is part of its ABI, since some architectures like arm64e can sign
+// the virtual function pointer in the vtable based on the mangled name of the function. Since
+// we use an ABI tag that changes with each released version, the mangled name of the virtual
+// function would change, which is incorrect. Note that it doesn't make much sense to change
+// the implementation of a virtual function in an ABI-incompatible way in the first place,
+// since that would be an ABI break anyway. Hence, the lack of ABI tag should not be noticeable.
+//
+// The macro can be applied to record and enum types. When the tagged type is nested in
+// a record this "parent" record needs to have the macro too. Another use case for applying
+// this macro to records and unions is to apply an ABI tag to inline constexpr variables.
+// This can be useful for inline variables that are implementation details which are expected
+// to change in the future.
+//
+// TODO: We provide a escape hatch with _LIBCPP_NO_ABI_TAG for folks who want to avoid increasing
+//       the length of symbols with an ABI tag. In practice, we should remove the escape hatch and
+//       use compression mangling instead, see https://github.com/itanium-cxx-abi/cxx-abi/issues/70.
+#  ifndef _LIBCPP_NO_ABI_TAG
+#    define _LIBCPP_HIDE_FROM_ABI                                                                                      \
+      _LIBCPP_HIDDEN _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION                                                       \
+      __attribute__((__abi_tag__(_LIBCPP_TOSTRING(_LIBCPP_ODR_SIGNATURE))))
+#  else
+#    define _LIBCPP_HIDE_FROM_ABI _LIBCPP_HIDDEN _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION
+#  endif
+#  define _LIBCPP_HIDE_FROM_ABI_VIRTUAL _LIBCPP_HIDDEN _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION
+
+#  ifdef _LIBCPP_BUILDING_LIBRARY
+#    if _LIBCPP_ABI_VERSION > 1
+#      define _LIBCPP_HIDE_FROM_ABI_AFTER_V1 _LIBCPP_HIDE_FROM_ABI
+#    else
+#      define _LIBCPP_HIDE_FROM_ABI_AFTER_V1
+#    endif
+#  else
+#    define _LIBCPP_HIDE_FROM_ABI_AFTER_V1 _LIBCPP_HIDE_FROM_ABI
+#  endif
+
+// TODO: Remove this workaround once we drop support for Clang 16
+#  if __has_warning("-Wc++23-extensions")
+#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++23-extensions")
+#  else
+#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++2b-extensions")
+#  endif
+
+// Clang modules take a significant compile time hit when pushing and popping diagnostics.
+// Since all the headers are marked as system headers in the modulemap, we can simply disable this
+// pushing and popping when building with clang modules.
+#  if !__has_feature(modules)
+#    define _LIBCPP_PUSH_EXTENSION_DIAGNOSTICS                                                                         \
+      _LIBCPP_DIAGNOSTIC_PUSH                                                                                          \
+      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++11-extensions")                                                           \
+      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++14-extensions")                                                           \
+      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++17-extensions")                                                           \
+      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++20-extensions")                                                           \
+      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION                                                                 \
+      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++14-extensions")                                                             \
+      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++17-extensions")                                                             \
+      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++20-extensions")                                                             \
+      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++23-extensions")
+#    define _LIBCPP_POP_EXTENSION_DIAGNOSTICS _LIBCPP_DIAGNOSTIC_POP
+#  else
+#    define _LIBCPP_PUSH_EXTENSION_DIAGNOSTICS
+#    define _LIBCPP_POP_EXTENSION_DIAGNOSTICS
+#  endif
+
+// Inline namespaces are available in Clang/GCC/MSVC regardless of C++ dialect.
+// clang-format off
+#  define _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_PUSH_EXTENSION_DIAGNOSTICS                                               \
+                                      namespace _LIBCPP_TYPE_VISIBILITY_DEFAULT std {                                  \
+                               inline namespace _LIBCPP_ABI_NAMESPACE {
+#  define _LIBCPP_END_NAMESPACE_STD }} _LIBCPP_POP_EXTENSION_DIAGNOSTICS
+
+#ifdef _LIBCPP_ABI_NO_FILESYSTEM_INLINE_NAMESPACE
+#  define _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM _LIBCPP_BEGIN_NAMESPACE_STD namespace filesystem {
+#  define _LIBCPP_END_NAMESPACE_FILESYSTEM } _LIBCPP_END_NAMESPACE_STD
+#else
+#  define _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM _LIBCPP_BEGIN_NAMESPACE_STD                                               \
+                                             inline namespace __fs { namespace filesystem {
+
+#  define _LIBCPP_END_NAMESPACE_FILESYSTEM }} _LIBCPP_END_NAMESPACE_STD
+#endif
+
+// clang-format on
+
+#  if __has_attribute(__enable_if__)
+#    define _LIBCPP_PREFERRED_OVERLOAD __attribute__((__enable_if__(true, "")))
+#  endif
+
+#  if !defined(__SIZEOF_INT128__) || defined(_MSC_VER)
+#    define _LIBCPP_HAS_NO_INT128
+#  endif
+
+#  ifdef _LIBCPP_CXX03_LANG
+#    define _LIBCPP_DECLARE_STRONG_ENUM(x)                                                                             \
+      struct _LIBCPP_EXPORTED_FROM_ABI x {                                                                             \
+        enum __lx
+// clang-format off
+#    define _LIBCPP_DECLARE_STRONG_ENUM_EPILOG(x)                                                                      \
+      __lx __v_;                                                                                                       \
+      _LIBCPP_HIDE_FROM_ABI x(__lx __v) : __v_(__v) {}                                                                 \
+      _LIBCPP_HIDE_FROM_ABI explicit x(int __v) : __v_(static_cast<__lx>(__v)) {}                                      \
+      _LIBCPP_HIDE_FROM_ABI operator int() const { return __v_; }                                                      \
+      };
+// clang-format on
+
+#  else // _LIBCPP_CXX03_LANG
+#    define _LIBCPP_DECLARE_STRONG_ENUM(x) enum class x
+#    define _LIBCPP_DECLARE_STRONG_ENUM_EPILOG(x)
+#  endif // _LIBCPP_CXX03_LANG
+
+#  if defined(__APPLE__) || defined(__FreeBSD__) || defined(_LIBCPP_MSVCRT_LIKE) || defined(__NetBSD__)
+#    define _LIBCPP_LOCALE__L_EXTENSIONS 1
+#  endif
+
+#  ifdef __FreeBSD__
+#    define _DECLARE_C99_LDBL_MATH 1
+#  endif
+
+// If we are getting operator new from the MSVC CRT, then allocation overloads
+// for align_val_t were added in 19.12, aka VS 2017 version 15.3.
+#  if defined(_LIBCPP_MSVCRT) && defined(_MSC_VER) && _MSC_VER < 1912
+#    define _LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+#  elif defined(_LIBCPP_ABI_VCRUNTIME) && !defined(__cpp_aligned_new)
+// We're deferring to Microsoft's STL to provide aligned new et al. We don't
+// have it unless the language feature test macro is defined.
+#    define _LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+#  elif defined(__MVS__)
+#    define _LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+#  endif
+
+#  if defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION) || (!defined(__cpp_aligned_new) || __cpp_aligned_new < 201606)
+#    define _LIBCPP_HAS_NO_ALIGNED_ALLOCATION
+#  endif
+
+// It is not yet possible to use aligned_alloc() on all Apple platforms since
+// 10.15 was the first version to ship an implementation of aligned_alloc().
+#  if defined(__APPLE__)
+#    if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) &&                                                     \
+         __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101500)
+#      define _LIBCPP_HAS_NO_C11_ALIGNED_ALLOC
+#    endif
+#  elif defined(__ANDROID__) && __ANDROID_API__ < 28
+// Android only provides aligned_alloc when targeting API 28 or higher.
+#    define _LIBCPP_HAS_NO_C11_ALIGNED_ALLOC
+#  endif
+
+#  if defined(__APPLE__) || defined(__FreeBSD__)
+#    define _LIBCPP_HAS_DEFAULTRUNELOCALE
+#  endif
+
+#  if defined(__APPLE__) || defined(__FreeBSD__)
+#    define _LIBCPP_WCTYPE_IS_MASK
+#  endif
+
+#  if _LIBCPP_STD_VER <= 17 || !defined(__cpp_char8_t)
+#    define _LIBCPP_HAS_NO_CHAR8_T
+#  endif
+
+// Deprecation macros.
+//
+// Deprecations warnings are always enabled, except when users explicitly opt-out
+// by defining _LIBCPP_DISABLE_DEPRECATION_WARNINGS.
+#  if !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS)
+#    if __has_attribute(__deprecated__)
+#      define _LIBCPP_DEPRECATED __attribute__((__deprecated__))
+#      define _LIBCPP_DEPRECATED_(m) __attribute__((__deprecated__(m)))
+#    elif _LIBCPP_STD_VER >= 14
+#      define _LIBCPP_DEPRECATED [[deprecated]]
+#      define _LIBCPP_DEPRECATED_(m) [[deprecated(m)]]
+#    else
+#      define _LIBCPP_DEPRECATED
+#      define _LIBCPP_DEPRECATED_(m)
+#    endif
+#  else
+#    define _LIBCPP_DEPRECATED
+#    define _LIBCPP_DEPRECATED_(m)
+#  endif
+
+#  if _LIBCPP_STD_VER < 20
+#    define _LIBCPP_DEPRECATED_ATOMIC_SYNC                                                                             \
+      _LIBCPP_DEPRECATED_("The C++20 synchronization library has been deprecated prior to C++20. Please update to "    \
+                          "using -std=c++20 if you need to use these facilities.")
+#  else
+#    define _LIBCPP_DEPRECATED_ATOMIC_SYNC /* nothing */
+#  endif
+
+#  if !defined(_LIBCPP_CXX03_LANG)
+#    define _LIBCPP_DEPRECATED_IN_CXX11 _LIBCPP_DEPRECATED
+#  else
+#    define _LIBCPP_DEPRECATED_IN_CXX11
+#  endif
+
+#  if _LIBCPP_STD_VER >= 14
+#    define _LIBCPP_DEPRECATED_IN_CXX14 _LIBCPP_DEPRECATED
+#  else
+#    define _LIBCPP_DEPRECATED_IN_CXX14
+#  endif
+
+#  if _LIBCPP_STD_VER >= 17
+#    define _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_DEPRECATED
+#  else
+#    define _LIBCPP_DEPRECATED_IN_CXX17
+#  endif
+
+#  if _LIBCPP_STD_VER >= 20
+#    define _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_DEPRECATED
+#  else
+#    define _LIBCPP_DEPRECATED_IN_CXX20
+#  endif
+
+#  if _LIBCPP_STD_VER >= 23
+#    define _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_DEPRECATED
+#  else
+#    define _LIBCPP_DEPRECATED_IN_CXX23
+#  endif
+
+#  if _LIBCPP_STD_VER >= 26
+#    define _LIBCPP_DEPRECATED_IN_CXX26 _LIBCPP_DEPRECATED
+#  else
+#    define _LIBCPP_DEPRECATED_IN_CXX26
+#  endif
+
+#  if !defined(_LIBCPP_HAS_NO_CHAR8_T)
+#    define _LIBCPP_DEPRECATED_WITH_CHAR8_T _LIBCPP_DEPRECATED
+#  else
+#    define _LIBCPP_DEPRECATED_WITH_CHAR8_T
+#  endif
+
+// Macros to enter and leave a state where deprecation warnings are suppressed.
+#  if defined(_LIBCPP_COMPILER_CLANG_BASED) || defined(_LIBCPP_COMPILER_GCC)
+#    define _LIBCPP_SUPPRESS_DEPRECATED_PUSH                                                                           \
+      _Pragma("GCC diagnostic push") _Pragma("GCC diagnostic ignored \"-Wdeprecated\"")                                \
+          _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"")
+#    define _LIBCPP_SUPPRESS_DEPRECATED_POP _Pragma("GCC diagnostic pop")
+#  else
+#    define _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+#    define _LIBCPP_SUPPRESS_DEPRECATED_POP
+#  endif
+
+#  if _LIBCPP_STD_VER <= 11
+#    define _LIBCPP_EXPLICIT_SINCE_CXX14
+#  else
+#    define _LIBCPP_EXPLICIT_SINCE_CXX14 explicit
+#  endif
+
+#  if _LIBCPP_STD_VER >= 23
+#    define _LIBCPP_EXPLICIT_SINCE_CXX23 explicit
+#  else
+#    define _LIBCPP_EXPLICIT_SINCE_CXX23
+#  endif
+
+#  if _LIBCPP_STD_VER >= 14
+#    define _LIBCPP_CONSTEXPR_SINCE_CXX14 constexpr
+#  else
+#    define _LIBCPP_CONSTEXPR_SINCE_CXX14
+#  endif
+
+#  if _LIBCPP_STD_VER >= 17
+#    define _LIBCPP_CONSTEXPR_SINCE_CXX17 constexpr
+#  else
+#    define _LIBCPP_CONSTEXPR_SINCE_CXX17
+#  endif
+
+#  if _LIBCPP_STD_VER >= 20
+#    define _LIBCPP_CONSTEXPR_SINCE_CXX20 constexpr
+#  else
+#    define _LIBCPP_CONSTEXPR_SINCE_CXX20
+#  endif
+
+#  if _LIBCPP_STD_VER >= 23
+#    define _LIBCPP_CONSTEXPR_SINCE_CXX23 constexpr
+#  else
+#    define _LIBCPP_CONSTEXPR_SINCE_CXX23
+#  endif
+
+#  ifndef _LIBCPP_WEAK
+#    define _LIBCPP_WEAK __attribute__((__weak__))
+#  endif
+
+// Thread API
+// clang-format off
+#  if !defined(_LIBCPP_HAS_NO_THREADS) &&                                                                              \
+      !defined(_LIBCPP_HAS_THREAD_API_PTHREAD) &&                                                                      \
+      !defined(_LIBCPP_HAS_THREAD_API_WIN32) &&                                                                        \
+      !defined(_LIBCPP_HAS_THREAD_API_EXTERNAL)
+
+#    if defined(__FreeBSD__) ||                                                                                        \
+        defined(__wasi__) ||                                                                                           \
+        defined(__NetBSD__) ||                                                                                         \
+        defined(__OpenBSD__) ||                                                                                        \
+        defined(__NuttX__) ||                                                                                          \
+        defined(__linux__) ||                                                                                          \
+        defined(__GNU__) ||                                                                                            \
+        defined(__APPLE__) ||                                                                                          \
+        defined(__MVS__) ||                                                                                            \
+        defined(_AIX) ||                                                                                               \
+        defined(__EMSCRIPTEN__)
+// clang-format on
+#      define _LIBCPP_HAS_THREAD_API_PTHREAD
+#    elif defined(__Fuchsia__)
+// TODO(44575): Switch to C11 thread API when possible.
+#      define _LIBCPP_HAS_THREAD_API_PTHREAD
+#    elif defined(_LIBCPP_WIN32API)
+#      define _LIBCPP_HAS_THREAD_API_WIN32
+#    else
+#      error "No thread API"
+#    endif // _LIBCPP_HAS_THREAD_API
+#  endif   // _LIBCPP_HAS_NO_THREADS
+
+#  if defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
+#    if defined(__ANDROID__) && __ANDROID_API__ >= 30
+#      define _LIBCPP_HAS_COND_CLOCKWAIT
+#    elif defined(_LIBCPP_GLIBC_PREREQ)
+#      if _LIBCPP_GLIBC_PREREQ(2, 30)
+#        define _LIBCPP_HAS_COND_CLOCKWAIT
+#      endif
+#    endif
+#  endif
+
+#  if defined(_LIBCPP_HAS_NO_THREADS) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
+#    error _LIBCPP_HAS_THREAD_API_PTHREAD may only be defined when \
+       _LIBCPP_HAS_NO_THREADS is not defined.
+#  endif
+
+#  if defined(_LIBCPP_HAS_NO_THREADS) && defined(_LIBCPP_HAS_THREAD_API_EXTERNAL)
+#    error _LIBCPP_HAS_THREAD_API_EXTERNAL may not be defined when \
+       _LIBCPP_HAS_NO_THREADS is defined.
+#  endif
+
+#  if defined(_LIBCPP_HAS_NO_MONOTONIC_CLOCK) && !defined(_LIBCPP_HAS_NO_THREADS)
+#    error _LIBCPP_HAS_NO_MONOTONIC_CLOCK may only be defined when \
+       _LIBCPP_HAS_NO_THREADS is defined.
+#  endif
+
+#  if !defined(_LIBCPP_HAS_NO_THREADS) && !defined(__STDCPP_THREADS__)
+#    define __STDCPP_THREADS__ 1
+#  endif
+
+// The glibc and Bionic implementation of pthreads implements
+// pthread_mutex_destroy as nop for regular mutexes. Additionally, Win32
+// mutexes have no destroy mechanism.
+//
+// This optimization can't be performed on Apple platforms, where
+// pthread_mutex_destroy can allow the kernel to release resources.
+// See https://llvm.org/D64298 for details.
+//
+// TODO(EricWF): Enable this optimization on Bionic after speaking to their
+//               respective stakeholders.
+// clang-format off
+#  if (defined(_LIBCPP_HAS_THREAD_API_PTHREAD) && defined(__GLIBC__)) ||                                               \
+      (defined(_LIBCPP_HAS_THREAD_API_C11) && defined(__Fuchsia__)) ||                                                 \
+       defined(_LIBCPP_HAS_THREAD_API_WIN32)
+// clang-format on
+#    define _LIBCPP_HAS_TRIVIAL_MUTEX_DESTRUCTION
+#  endif
+
+// Destroying a condvar is a nop on Windows.
+//
+// This optimization can't be performed on Apple platforms, where
+// pthread_cond_destroy can allow the kernel to release resources.
+// See https://llvm.org/D64298 for details.
+//
+// TODO(EricWF): This is potentially true for some pthread implementations
+// as well.
+#  if (defined(_LIBCPP_HAS_THREAD_API_C11) && defined(__Fuchsia__)) || defined(_LIBCPP_HAS_THREAD_API_WIN32)
+#    define _LIBCPP_HAS_TRIVIAL_CONDVAR_DESTRUCTION
+#  endif
+
+#  if defined(__BIONIC__) || defined(__NuttX__) || defined(__Fuchsia__) || defined(__wasi__) ||                        \
+      defined(_LIBCPP_HAS_MUSL_LIBC) || defined(__OpenBSD__)
+#    define _LIBCPP_PROVIDES_DEFAULT_RUNE_TABLE
+#  endif
+
+#  if __has_feature(cxx_atomic) || __has_extension(c_atomic) || __has_keyword(_Atomic)
+#    define _LIBCPP_HAS_C_ATOMIC_IMP
+#  elif defined(_LIBCPP_COMPILER_GCC)
+#    define _LIBCPP_HAS_GCC_ATOMIC_IMP
+#  endif
+
+#  if !defined(_LIBCPP_HAS_C_ATOMIC_IMP) && !defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) &&                                    \
+      !defined(_LIBCPP_HAS_EXTERNAL_ATOMIC_IMP)
+#    define _LIBCPP_HAS_NO_ATOMIC_HEADER
+#  else
+#    ifndef _LIBCPP_ATOMIC_FLAG_TYPE
+#      define _LIBCPP_ATOMIC_FLAG_TYPE bool
+#    endif
+#  endif
+
+#  if defined(__FreeBSD__) && defined(__clang__) && __has_attribute(__no_thread_safety_analysis__)
+#    define _LIBCPP_NO_THREAD_SAFETY_ANALYSIS __attribute__((__no_thread_safety_analysis__))
+#  else
+#    define _LIBCPP_NO_THREAD_SAFETY_ANALYSIS
+#  endif
+
+#  if defined(_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS)
+#    if defined(__clang__) && __has_attribute(acquire_capability)
+// Work around the attribute handling in clang.  When both __declspec and
+// __attribute__ are present, the processing goes awry preventing the definition
+// of the types. In MinGW mode, __declspec evaluates to __attribute__, and thus
+// combining the two does work.
+#      if !defined(_MSC_VER)
+#        define _LIBCPP_HAS_THREAD_SAFETY_ANNOTATIONS
+#      endif
+#    endif
+#  endif
+
+#  ifdef _LIBCPP_HAS_THREAD_SAFETY_ANNOTATIONS
+#    define _LIBCPP_THREAD_SAFETY_ANNOTATION(x) __attribute__((x))
+#  else
+#    define _LIBCPP_THREAD_SAFETY_ANNOTATION(x)
+#  endif
+
+#  if _LIBCPP_STD_VER >= 20
+#    define _LIBCPP_CONSTINIT constinit
+#  elif __has_attribute(__require_constant_initialization__)
+#    define _LIBCPP_CONSTINIT __attribute__((__require_constant_initialization__))
+#  else
+#    define _LIBCPP_CONSTINIT
+#  endif
+
+#  if defined(__CUDACC__) || defined(__CUDA_ARCH__) || defined(__CUDA_LIBDEVICE__)
+// The CUDA SDK contains an unfortunate definition for the __noinline__ macro,
+// which breaks the regular __attribute__((__noinline__)) syntax. Therefore,
+// when compiling for CUDA we use the non-underscored version of the noinline
+// attribute.
+//
+// This is a temporary workaround and we still expect the CUDA SDK team to solve
+// this issue properly in the SDK headers.
+//
+// See https://github.com/llvm/llvm-project/pull/73838 for more details.
+#    define _LIBCPP_NOINLINE __attribute__((noinline))
+#  elif __has_attribute(__noinline__)
+#    define _LIBCPP_NOINLINE __attribute__((__noinline__))
+#  else
+#    define _LIBCPP_NOINLINE
+#  endif
+
+// We often repeat things just for handling wide characters in the library.
+// When wide characters are disabled, it can be useful to have a quick way of
+// disabling it without having to resort to #if-#endif, which has a larger
+// impact on readability.
+#  if defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS)
+#    define _LIBCPP_IF_WIDE_CHARACTERS(...)
+#  else
+#    define _LIBCPP_IF_WIDE_CHARACTERS(...) __VA_ARGS__
+#  endif
+
+// clang-format off
+#  define _LIBCPP_PUSH_MACROS _Pragma("push_macro(\"min\")") _Pragma("push_macro(\"max\")") _Pragma("push_macro(\"refresh\")") _Pragma("push_macro(\"move\")") _Pragma("push_macro(\"erase\")")
+#  define _LIBCPP_POP_MACROS _Pragma("pop_macro(\"min\")") _Pragma("pop_macro(\"max\")") _Pragma("pop_macro(\"refresh\")") _Pragma("pop_macro(\"move\")") _Pragma("pop_macro(\"erase\")")
+// clang-format on
+
+#  ifndef _LIBCPP_NO_AUTO_LINK
+#    if defined(_LIBCPP_ABI_MICROSOFT) && !defined(_LIBCPP_BUILDING_LIBRARY)
+#      if !defined(_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS)
+#        pragma comment(lib, "c++.lib")
+#      else
+#        pragma comment(lib, "libc++.lib")
+#      endif
+#    endif // defined(_LIBCPP_ABI_MICROSOFT) && !defined(_LIBCPP_BUILDING_LIBRARY)
+#  endif   // _LIBCPP_NO_AUTO_LINK
+
+// Configures the fopen close-on-exec mode character, if any. This string will
+// be appended to any mode string used by fstream for fopen/fdopen.
+//
+// Not all platforms support this, but it helps avoid fd-leaks on platforms that
+// do.
+#  if defined(__BIONIC__)
+#    define _LIBCPP_FOPEN_CLOEXEC_MODE "e"
+#  else
+#    define _LIBCPP_FOPEN_CLOEXEC_MODE
+#  endif
+
+#  if __has_cpp_attribute(msvc::no_unique_address)
+// MSVC implements [[no_unique_address]] as a silent no-op currently.
+// (If/when MSVC breaks its C++ ABI, it will be changed to work as intended.)
+// However, MSVC implements [[msvc::no_unique_address]] which does what
+// [[no_unique_address]] is supposed to do, in general.
+
+// Clang-cl does not yet (14.0) implement either [[no_unique_address]] or
+// [[msvc::no_unique_address]] though. If/when it does implement
+// [[msvc::no_unique_address]], this should be preferred though.
+#    define _LIBCPP_NO_UNIQUE_ADDRESS [[msvc::no_unique_address]]
+#  elif __has_cpp_attribute(no_unique_address)
+#    define _LIBCPP_NO_UNIQUE_ADDRESS [[__no_unique_address__]]
+#  else
+#    define _LIBCPP_NO_UNIQUE_ADDRESS /* nothing */
+// Note that this can be replaced by #error as soon as clang-cl
+// implements msvc::no_unique_address, since there should be no C++20
+// compiler that doesn't support one of the two attributes at that point.
+// We generally don't want to use this macro outside of C++20-only code,
+// because using it conditionally in one language version only would make
+// the ABI inconsistent.
+#  endif
+
+// c8rtomb() and mbrtoc8() were added in C++20 and C23. Support for these
+// functions is gradually being added to existing C libraries. The conditions
+// below check for known C library versions and conditions under which these
+// functions are declared by the C library.
+#  define _LIBCPP_HAS_NO_C8RTOMB_MBRTOC8
+// GNU libc 2.36 and newer declare c8rtomb() and mbrtoc8() in C++ modes if
+// __cpp_char8_t is defined or if C2X extensions are enabled. Determining
+// the latter depends on internal GNU libc details that are not appropriate
+// to depend on here, so any declarations present when __cpp_char8_t is not
+// defined are ignored.
+#  if defined(_LIBCPP_GLIBC_PREREQ)
+#    if _LIBCPP_GLIBC_PREREQ(2, 36) && defined(__cpp_char8_t)
+#      undef _LIBCPP_HAS_NO_C8RTOMB_MBRTOC8
+#    endif
+#  endif
+
+// There are a handful of public standard library types that are intended to
+// support CTAD but don't need any explicit deduction guides to do so. This
+// macro is used to mark them as such, which suppresses the
+// '-Wctad-maybe-unsupported' compiler warning when CTAD is used in user code
+// with these classes.
+#  if _LIBCPP_STD_VER >= 17
+#    ifdef _LIBCPP_COMPILER_CLANG_BASED
+#      define _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(_ClassName)                                                              \
+        template <class... _Tag>                                                                                       \
+        [[maybe_unused]] _ClassName(typename _Tag::__allow_ctad...)->_ClassName<_Tag...>
+#    else
+#      define _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(ClassName)                                                               \
+        template <class... _Tag>                                                                                       \
+        ClassName(typename _Tag::__allow_ctad...)->ClassName<_Tag...>
+#    endif
+#  else
+#    define _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(_ClassName) static_assert(true, "")
+#  endif
+
+// TODO(varconst): currently, there are bugs in Clang's intrinsics when handling Objective-C++ `id`, so don't use
+// compiler intrinsics in the Objective-C++ mode.
+#  ifdef __OBJC__
+#    define _LIBCPP_WORKAROUND_OBJCXX_COMPILER_INTRINSICS
+#  endif
+
+#  define _PSTL_PRAGMA(x) _Pragma(#x)
+
+// Enable SIMD for compilers that support OpenMP 4.0
+#  if (defined(_OPENMP) && _OPENMP >= 201307)
+
+#    define _PSTL_UDR_PRESENT
+#    define _PSTL_PRAGMA_SIMD _PSTL_PRAGMA(omp simd)
+#    define _PSTL_PRAGMA_DECLARE_SIMD _PSTL_PRAGMA(omp declare simd)
+#    define _PSTL_PRAGMA_SIMD_REDUCTION(PRM) _PSTL_PRAGMA(omp simd reduction(PRM))
+#    define _PSTL_PRAGMA_SIMD_SCAN(PRM) _PSTL_PRAGMA(omp simd reduction(inscan, PRM))
+#    define _PSTL_PRAGMA_SIMD_INCLUSIVE_SCAN(PRM) _PSTL_PRAGMA(omp scan inclusive(PRM))
+#    define _PSTL_PRAGMA_SIMD_EXCLUSIVE_SCAN(PRM) _PSTL_PRAGMA(omp scan exclusive(PRM))
+
+// Declaration of reduction functor, where
+// NAME - the name of the functor
+// OP - type of the callable object with the reduction operation
+// omp_in - refers to the local partial result
+// omp_out - refers to the final value of the combiner operator
+// omp_priv - refers to the private copy of the initial value
+// omp_orig - refers to the original variable to be reduced
+#    define _PSTL_PRAGMA_DECLARE_REDUCTION(NAME, OP)                                                                   \
+      _PSTL_PRAGMA(omp declare reduction(NAME:OP : omp_out(omp_in)) initializer(omp_priv = omp_orig))
+
+#  elif defined(_LIBCPP_COMPILER_CLANG_BASED)
+
+#    define _PSTL_PRAGMA_SIMD _Pragma("clang loop vectorize(enable) interleave(enable)")
+#    define _PSTL_PRAGMA_DECLARE_SIMD
+#    define _PSTL_PRAGMA_SIMD_REDUCTION(PRM) _Pragma("clang loop vectorize(enable) interleave(enable)")
+#    define _PSTL_PRAGMA_SIMD_SCAN(PRM) _Pragma("clang loop vectorize(enable) interleave(enable)")
+#    define _PSTL_PRAGMA_SIMD_INCLUSIVE_SCAN(PRM)
+#    define _PSTL_PRAGMA_SIMD_EXCLUSIVE_SCAN(PRM)
+#    define _PSTL_PRAGMA_DECLARE_REDUCTION(NAME, OP)
+
+#  else // (defined(_OPENMP) && _OPENMP >= 201307)
+
+#    define _PSTL_PRAGMA_SIMD
+#    define _PSTL_PRAGMA_DECLARE_SIMD
+#    define _PSTL_PRAGMA_SIMD_REDUCTION(PRM)
+#    define _PSTL_PRAGMA_SIMD_SCAN(PRM)
+#    define _PSTL_PRAGMA_SIMD_INCLUSIVE_SCAN(PRM)
+#    define _PSTL_PRAGMA_SIMD_EXCLUSIVE_SCAN(PRM)
+#    define _PSTL_PRAGMA_DECLARE_REDUCTION(NAME, OP)
+
+#  endif // (defined(_OPENMP) && _OPENMP >= 201307)
+
+#  define _PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED
+
+// Optional attributes - these are useful for a better QoI, but not required to be available
+
+#  if __has_attribute(__no_sanitize__) && !defined(_LIBCPP_COMPILER_GCC)
+#    define _LIBCPP_NO_CFI __attribute__((__no_sanitize__("cfi")))
+#  else
+#    define _LIBCPP_NO_CFI
+#  endif
+
+#  if __has_attribute(__malloc__)
+#    define _LIBCPP_NOALIAS __attribute__((__malloc__))
+#  else
+#    define _LIBCPP_NOALIAS
+#  endif
+
+#  if __has_attribute(__using_if_exists__)
+#    define _LIBCPP_USING_IF_EXISTS __attribute__((__using_if_exists__))
+#  else
+#    define _LIBCPP_USING_IF_EXISTS
+#  endif
+
+#  if __has_cpp_attribute(__nodiscard__)
+#    define _LIBCPP_NODISCARD [[__nodiscard__]]
+#  else
+// We can't use GCC's [[gnu::warn_unused_result]] and
+// __attribute__((warn_unused_result)), because GCC does not silence them via
+// (void) cast.
+#    define _LIBCPP_NODISCARD
+#  endif
+
+#  if __has_attribute(__no_destroy__)
+#    define _LIBCPP_NO_DESTROY __attribute__((__no_destroy__))
+#  else
+#    define _LIBCPP_NO_DESTROY
+#  endif
+
+#  if __has_attribute(__diagnose_if__)
+#    define _LIBCPP_DIAGNOSE_WARNING(...) __attribute__((__diagnose_if__(__VA_ARGS__, "warning")))
+#  else
+#    define _LIBCPP_DIAGNOSE_WARNING(...)
+#  endif
+
+// Use a function like macro to imply that it must be followed by a semicolon
+#  if __has_cpp_attribute(fallthrough)
+#    define _LIBCPP_FALLTHROUGH() [[fallthrough]]
+#  elif __has_attribute(__fallthrough__)
+#    define _LIBCPP_FALLTHROUGH() __attribute__((__fallthrough__))
+#  else
+#    define _LIBCPP_FALLTHROUGH() ((void)0)
+#  endif
+
+#  if __has_cpp_attribute(_Clang::__lifetimebound__)
+#    define _LIBCPP_LIFETIMEBOUND [[_Clang::__lifetimebound__]]
+#  else
+#    define _LIBCPP_LIFETIMEBOUND
+#  endif
+
+#  if __has_attribute(__nodebug__)
+#    define _LIBCPP_NODEBUG __attribute__((__nodebug__))
+#  else
+#    define _LIBCPP_NODEBUG
+#  endif
+
+#  if __has_attribute(__standalone_debug__)
+#    define _LIBCPP_STANDALONE_DEBUG __attribute__((__standalone_debug__))
+#  else
+#    define _LIBCPP_STANDALONE_DEBUG
+#  endif
+
+#  if __has_attribute(__preferred_name__)
+#    define _LIBCPP_PREFERRED_NAME(x) __attribute__((__preferred_name__(x)))
+#  else
+#    define _LIBCPP_PREFERRED_NAME(x)
+#  endif
+
+#  if __has_attribute(__no_sanitize__)
+#    define _LIBCPP_NO_SANITIZE(...) __attribute__((__no_sanitize__(__VA_ARGS__)))
+#  else
+#    define _LIBCPP_NO_SANITIZE(...)
+#  endif
+
+#  if __has_attribute(__init_priority__)
+#    define _LIBCPP_INIT_PRIORITY_MAX __attribute__((__init_priority__(100)))
+#  else
+#    define _LIBCPP_INIT_PRIORITY_MAX
+#  endif
+
+#  if __has_attribute(__format__)
+// The attribute uses 1-based indices for ordinary and static member functions.
+// The attribute uses 2-based indices for non-static member functions.
+#    define _LIBCPP_ATTRIBUTE_FORMAT(archetype, format_string_index, first_format_arg_index)                           \
+      __attribute__((__format__(archetype, format_string_index, first_format_arg_index)))
+#  else
+#    define _LIBCPP_ATTRIBUTE_FORMAT(archetype, format_string_index, first_format_arg_index) /* nothing */
+#  endif
+
+#  if __has_attribute(__packed__)
+#    define _LIBCPP_PACKED __attribute__((__packed__))
+#  else
+#    define _LIBCPP_PACKED
+#  endif
+
+#  if defined(_LIBCPP_ABI_MICROSOFT) && __has_declspec_attribute(empty_bases)
+#    define _LIBCPP_DECLSPEC_EMPTY_BASES __declspec(empty_bases)
+#  else
+#    define _LIBCPP_DECLSPEC_EMPTY_BASES
+#  endif
+
+// Allow for build-time disabling of unsigned integer sanitization
+#  if __has_attribute(no_sanitize) && !defined(_LIBCPP_COMPILER_GCC)
+#    define _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK __attribute__((__no_sanitize__("unsigned-integer-overflow")))
+#  else
+#    define _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK
+#  endif
+
+// Clang-18 has support for deducing this, but it does not set the FTM.
+#  if defined(__cpp_explicit_this_parameter) || (defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1800)
+#    define _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
+#  endif
+
+#endif // __cplusplus
+
+#endif // _LIBCPP___CONFIG
diff --git a/libcxx/include/__cxx03/__config_site.in b/libcxx/include/__cxx03/__config_site.in
new file mode 100644
index 000000000000..67022146c908
--- /dev/null
+++ b/libcxx/include/__cxx03/__config_site.in
@@ -0,0 +1,56 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONFIG_SITE
+#define _LIBCPP___CONFIG_SITE
+
+#cmakedefine _LIBCPP_ABI_VERSION @_LIBCPP_ABI_VERSION@
+#cmakedefine _LIBCPP_ABI_NAMESPACE @_LIBCPP_ABI_NAMESPACE@
+#cmakedefine _LIBCPP_ABI_FORCE_ITANIUM
+#cmakedefine _LIBCPP_ABI_FORCE_MICROSOFT
+#cmakedefine _LIBCPP_HAS_NO_THREADS
+#cmakedefine _LIBCPP_HAS_NO_MONOTONIC_CLOCK
+#cmakedefine _LIBCPP_HAS_MUSL_LIBC
+#cmakedefine _LIBCPP_HAS_THREAD_API_PTHREAD
+#cmakedefine _LIBCPP_HAS_THREAD_API_EXTERNAL
+#cmakedefine _LIBCPP_HAS_THREAD_API_WIN32
+#cmakedefine _LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS
+#cmakedefine _LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS
+#cmakedefine _LIBCPP_NO_VCRUNTIME
+#cmakedefine _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION @_LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION@
+#cmakedefine _LIBCPP_HAS_NO_FILESYSTEM
+#cmakedefine _LIBCPP_HAS_NO_RANDOM_DEVICE
+#cmakedefine _LIBCPP_HAS_NO_LOCALIZATION
+#cmakedefine _LIBCPP_HAS_NO_UNICODE
+#cmakedefine _LIBCPP_HAS_NO_WIDE_CHARACTERS
+#cmakedefine _LIBCPP_HAS_NO_STD_MODULES
+#cmakedefine _LIBCPP_HAS_NO_TIME_ZONE_DATABASE
+#cmakedefine _LIBCPP_INSTRUMENTED_WITH_ASAN
+
+// PSTL backends
+#cmakedefine _LIBCPP_PSTL_BACKEND_SERIAL
+#cmakedefine _LIBCPP_PSTL_BACKEND_STD_THREAD
+#cmakedefine _LIBCPP_PSTL_BACKEND_LIBDISPATCH
+
+// Hardening.
+#cmakedefine _LIBCPP_HARDENING_MODE_DEFAULT @_LIBCPP_HARDENING_MODE_DEFAULT@
+
+// __USE_MINGW_ANSI_STDIO gets redefined on MinGW
+#ifdef __clang__
+#  pragma clang diagnostic push
+#  pragma clang diagnostic ignored "-Wmacro-redefined"
+#endif
+
+@_LIBCPP_ABI_DEFINES@
+@_LIBCPP_EXTRA_SITE_DEFINES@
+
+#ifdef __clang__
+#  pragma clang diagnostic pop
+#endif
+
+#endif // _LIBCPP___CONFIG_SITE
diff --git a/libcxx/include/__cxx03/__configuration/abi.h b/libcxx/include/__cxx03/__configuration/abi.h
new file mode 100644
index 000000000000..0422b645727d
--- /dev/null
+++ b/libcxx/include/__cxx03/__configuration/abi.h
@@ -0,0 +1,172 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONFIGURATION_ABI_H
+#define _LIBCPP___CONFIGURATION_ABI_H
+
+#include <__config_site>
+#include <__configuration/compiler.h>
+#include <__configuration/platform.h>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_ABI_VERSION >= 2
+// Change short string representation so that string data starts at offset 0,
+// improving its alignment in some cases.
+#  define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
+// Fix deque iterator type in order to support incomplete types.
+#  define _LIBCPP_ABI_INCOMPLETE_TYPES_IN_DEQUE
+// Fix undefined behavior in how std::list stores its linked nodes.
+#  define _LIBCPP_ABI_LIST_REMOVE_NODE_POINTER_UB
+// Fix undefined behavior in  how __tree stores its end and parent nodes.
+#  define _LIBCPP_ABI_TREE_REMOVE_NODE_POINTER_UB
+// Fix undefined behavior in how __hash_table stores its pointer types.
+#  define _LIBCPP_ABI_FIX_UNORDERED_NODE_POINTER_UB
+#  define _LIBCPP_ABI_FORWARD_LIST_REMOVE_NODE_POINTER_UB
+#  define _LIBCPP_ABI_FIX_UNORDERED_CONTAINER_SIZE_TYPE
+// Override the default return value of exception::what() for bad_function_call::what()
+// with a string that is specific to bad_function_call (see http://wg21.link/LWG2233).
+// This is an ABI break on platforms that sign and authenticate vtable function pointers
+// because it changes the mangling of the virtual function located in the vtable, which
+// changes how it gets signed.
+#  define _LIBCPP_ABI_BAD_FUNCTION_CALL_GOOD_WHAT_MESSAGE
+// Enable optimized version of __do_get_(un)signed which avoids redundant copies.
+#  define _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
+// Give reverse_iterator<T> one data member of type T, not two.
+// Also, in C++17 and later, don't derive iterator types from std::iterator.
+#  define _LIBCPP_ABI_NO_ITERATOR_BASES
+// Use the smallest possible integer type to represent the index of the variant.
+// Previously libc++ used "unsigned int" exclusively.
+#  define _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION
+// Unstable attempt to provide a more optimized std::function
+#  define _LIBCPP_ABI_OPTIMIZED_FUNCTION
+// All the regex constants must be distinct and nonzero.
+#  define _LIBCPP_ABI_REGEX_CONSTANTS_NONZERO
+// Re-worked external template instantiations for std::string with a focus on
+// performance and fast-path inlining.
+#  define _LIBCPP_ABI_STRING_OPTIMIZED_EXTERNAL_INSTANTIATION
+// Enable clang::trivial_abi on std::unique_ptr.
+#  define _LIBCPP_ABI_ENABLE_UNIQUE_PTR_TRIVIAL_ABI
+// Enable clang::trivial_abi on std::shared_ptr and std::weak_ptr
+#  define _LIBCPP_ABI_ENABLE_SHARED_PTR_TRIVIAL_ABI
+// std::random_device holds some state when it uses an implementation that gets
+// entropy from a file (see _LIBCPP_USING_DEV_RANDOM). When switching from this
+// implementation to another one on a platform that has already shipped
+// std::random_device, one needs to retain the same object layout to remain ABI
+// compatible. This switch removes these workarounds for platforms that don't care
+// about ABI compatibility.
+#  define _LIBCPP_ABI_NO_RANDOM_DEVICE_COMPATIBILITY_LAYOUT
+// Don't export the legacy __basic_string_common class and its methods from the built library.
+#  define _LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON
+// Don't export the legacy __vector_base_common class and its methods from the built library.
+#  define _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON
+// According to the Standard, `bitset::operator[] const` returns bool
+#  define _LIBCPP_ABI_BITSET_VECTOR_BOOL_CONST_SUBSCRIPT_RETURN_BOOL
+// Fix the implementation of CityHash used for std::hash<fundamental-type>.
+// This is an ABI break because `std::hash` will return a different result,
+// which means that hashing the same object in translation units built against
+// different versions of libc++ can return inconsistent results. This is especially
+// tricky since std::hash is used in the implementation of unordered containers.
+//
+// The incorrect implementation of CityHash has the problem that it drops some
+// bits on the floor.
+#  define _LIBCPP_ABI_FIX_CITYHASH_IMPLEMENTATION
+// Remove the base 10 implementation of std::to_chars from the dylib.
+// The implementation moved to the header, but we still export the symbols from
+// the dylib for backwards compatibility.
+#  define _LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10
+// Define std::array/std::string_view iterators to be __wrap_iters instead of raw
+// pointers, which prevents people from relying on a non-portable implementation
+// detail. This is especially useful because enabling bounded iterators hardening
+// requires code not to make these assumptions.
+#  define _LIBCPP_ABI_USE_WRAP_ITER_IN_STD_ARRAY
+#  define _LIBCPP_ABI_USE_WRAP_ITER_IN_STD_STRING_VIEW
+// Dont' add an inline namespace for `std::filesystem`
+#  define _LIBCPP_ABI_NO_FILESYSTEM_INLINE_NAMESPACE
+// std::basic_ios uses WEOF to indicate that the fill value is
+// uninitialized. However, on platforms where the size of char_type is
+// equal to or greater than the size of int_type and char_type is unsigned,
+// std::char_traits<char_type>::eq_int_type() cannot distinguish between WEOF
+// and WCHAR_MAX. This ABI setting determines whether we should instead track whether the fill
+// value has been initialized using a separate boolean, which changes the ABI.
+#  define _LIBCPP_ABI_IOS_ALLOW_ARBITRARY_FILL_VALUE
+// Make a std::pair of trivially copyable types trivially copyable.
+// While this technically doesn't change the layout of pair itself, other types may decide to programatically change
+// their representation based on whether something is trivially copyable.
+#  define _LIBCPP_ABI_TRIVIALLY_COPYABLE_PAIR
+#elif _LIBCPP_ABI_VERSION == 1
+#  if !(defined(_LIBCPP_OBJECT_FORMAT_COFF) || defined(_LIBCPP_OBJECT_FORMAT_XCOFF))
+// Enable compiling copies of now inline methods into the dylib to support
+// applications compiled against older libraries. This is unnecessary with
+// COFF dllexport semantics, since dllexport forces a non-inline definition
+// of inline functions to be emitted anyway. Our own non-inline copy would
+// conflict with the dllexport-emitted copy, so we disable it. For XCOFF,
+// the linker will take issue with the symbols in the shared object if the
+// weak inline methods get visibility (such as from -fvisibility-inlines-hidden),
+// so disable it.
+#    define _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS
+#  endif
+// Feature macros for disabling pre ABI v1 features. All of these options
+// are deprecated.
+#  if defined(__FreeBSD__) && __FreeBSD__ < 14
+#    define _LIBCPP_DEPRECATED_ABI_DISABLE_PAIR_TRIVIAL_COPY_CTOR
+#  endif
+#endif
+
+// We had some bugs where we use [[no_unique_address]] together with construct_at,
+// which causes UB as the call on construct_at could write to overlapping subobjects
+//
+// https://github.com/llvm/llvm-project/issues/70506
+// https://github.com/llvm/llvm-project/issues/70494
+//
+// To fix the bug we had to change the ABI of some classes to remove [[no_unique_address]] under certain conditions.
+// The macro below is used for all classes whose ABI have changed as part of fixing these bugs.
+#define _LIBCPP_ABI_LLVM18_NO_UNIQUE_ADDRESS __attribute__((__abi_tag__("llvm18_nua")))
+
+// Changes the iterator type of select containers (see below) to a bounded iterator that keeps track of whether it's
+// within the bounds of the original container and asserts it on every dereference.
+//
+// ABI impact: changes the iterator type of the relevant containers.
+//
+// Supported containers:
+// - `span`;
+// - `string_view`.
+// #define _LIBCPP_ABI_BOUNDED_ITERATORS
+
+// Changes the iterator type of `basic_string` to a bounded iterator that keeps track of whether it's within the bounds
+// of the original container and asserts it on every dereference and when performing iterator arithmetics.
+//
+// ABI impact: changes the iterator type of `basic_string` and its specializations, such as `string` and `wstring`.
+// #define _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
+
+// Changes the iterator type of `vector` to a bounded iterator that keeps track of whether it's within the bounds of the
+// original container and asserts it on every dereference and when performing iterator arithmetics. Note: this doesn't
+// yet affect `vector<bool>`.
+//
+// ABI impact: changes the iterator type of `vector` (except `vector<bool>`).
+// #define _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
+
+#if defined(_LIBCPP_COMPILER_CLANG_BASED)
+#  if defined(__APPLE__)
+#    if defined(__i386__) || defined(__x86_64__)
+// use old string layout on x86_64 and i386
+#    elif defined(__arm__)
+// use old string layout on arm (which does not include aarch64/arm64), except on watch ABIs
+#      if defined(__ARM_ARCH_7K__) && __ARM_ARCH_7K__ >= 2
+#        define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
+#      endif
+#    else
+#      define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
+#    endif
+#  endif
+#endif
+
+#endif // _LIBCPP___CONFIGURATION_ABI_H
diff --git a/libcxx/include/__cxx03/__configuration/availability.h b/libcxx/include/__cxx03/__configuration/availability.h
new file mode 100644
index 000000000000..ab483a07c9c1
--- /dev/null
+++ b/libcxx/include/__cxx03/__configuration/availability.h
@@ -0,0 +1,400 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONFIGURATION_AVAILABILITY_H
+#define _LIBCPP___CONFIGURATION_AVAILABILITY_H
+
+#include <__configuration/compiler.h>
+#include <__configuration/language.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+// Libc++ is shipped by various vendors. In particular, it is used as a system
+// library on macOS, iOS and other Apple platforms. In order for users to be
+// able to compile a binary that is intended to be deployed to an older version
+// of a platform, Clang provides availability attributes [1]. These attributes
+// can be placed on declarations and are used to describe the life cycle of a
+// symbol in the library.
+//
+// The main goal is to ensure a compile-time error if a symbol that hasn't been
+// introduced in a previously released library is used in a program that targets
+// that previously released library. Normally, this would be a load-time error
+// when one tries to launch the program against the older library.
+//
+// For example, the filesystem library was introduced in the dylib in LLVM 9.
+// On Apple platforms, this corresponds to macOS 10.15. If a user compiles on
+// a macOS 10.15 host but targets macOS 10.13 with their program, the compiler
+// would normally not complain (because the required declarations are in the
+// headers), but the dynamic loader would fail to find the symbols when actually
+// trying to launch the program on macOS 10.13. To turn this into a compile-time
+// issue instead, declarations are annotated with when they were introduced, and
+// the compiler can produce a diagnostic if the program references something that
+// isn't available on the deployment target.
+//
+// This mechanism is general in nature, and any vendor can add their markup to
+// the library (see below). Whenever a new feature is added that requires support
+// in the shared library, two macros are added below to allow marking the feature
+// as unavailable:
+// 1. A macro named `_LIBCPP_AVAILABILITY_HAS_<feature>` which must be defined
+//    to `_LIBCPP_INTRODUCED_IN_<version>` for the appropriate LLVM version.
+// 2. A macro named `_LIBCPP_AVAILABILITY_<feature>`, which must be defined to
+//    `_LIBCPP_INTRODUCED_IN_<version>_MARKUP` for the appropriate LLVM version.
+//
+// When vendors decide to ship the feature as part of their shared library, they
+// can update the `_LIBCPP_INTRODUCED_IN_<version>` macro (and the markup counterpart)
+// based on the platform version they shipped that version of LLVM in. The library
+// will then use this markup to provide an optimal user experience on these platforms.
+//
+// Furthermore, many features in the standard library have corresponding
+// feature-test macros. The `_LIBCPP_AVAILABILITY_HAS_<feature>` macros
+// are checked by the corresponding feature-test macros generated by
+// generate_feature_test_macro_components.py to ensure that the library
+// doesn't announce a feature as being implemented if it is unavailable on
+// the deployment target.
+//
+// Note that this mechanism is disabled by default in the "upstream" libc++.
+// Availability annotations are only meaningful when shipping libc++ inside
+// a platform (i.e. as a system library), and so vendors that want them should
+// turn those annotations on at CMake configuration time.
+//
+// [1]: https://clang.llvm.org/docs/AttributeReference.html#availability
+
+// For backwards compatibility, allow users to define _LIBCPP_DISABLE_AVAILABILITY
+// for a while.
+#if defined(_LIBCPP_DISABLE_AVAILABILITY)
+#  if !defined(_LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS)
+#    define _LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS
+#  endif
+#endif
+
+// Availability markup is disabled when building the library, or when a non-Clang
+// compiler is used because only Clang supports the necessary attributes.
+#if defined(_LIBCPP_BUILDING_LIBRARY) || defined(_LIBCXXABI_BUILDING_LIBRARY) || !defined(_LIBCPP_COMPILER_CLANG_BASED)
+#  if !defined(_LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS)
+#    define _LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS
+#  endif
+#endif
+
+// When availability annotations are disabled, we take for granted that features introduced
+// in all versions of the library are available.
+#if defined(_LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS)
+
+#  define _LIBCPP_INTRODUCED_IN_LLVM_19 1
+#  define _LIBCPP_INTRODUCED_IN_LLVM_19_ATTRIBUTE /* nothing */
+
+#  define _LIBCPP_INTRODUCED_IN_LLVM_18 1
+#  define _LIBCPP_INTRODUCED_IN_LLVM_18_ATTRIBUTE /* nothing */
+
+#  define _LIBCPP_INTRODUCED_IN_LLVM_17 1
+#  define _LIBCPP_INTRODUCED_IN_LLVM_17_ATTRIBUTE /* nothing */
+
+#  define _LIBCPP_INTRODUCED_IN_LLVM_16 1
+#  define _LIBCPP_INTRODUCED_IN_LLVM_16_ATTRIBUTE /* nothing */
+
+#  define _LIBCPP_INTRODUCED_IN_LLVM_15 1
+#  define _LIBCPP_INTRODUCED_IN_LLVM_15_ATTRIBUTE /* nothing */
+
+#  define _LIBCPP_INTRODUCED_IN_LLVM_14 1
+#  define _LIBCPP_INTRODUCED_IN_LLVM_14_ATTRIBUTE /* nothing */
+
+#  define _LIBCPP_INTRODUCED_IN_LLVM_13 1
+#  define _LIBCPP_INTRODUCED_IN_LLVM_13_ATTRIBUTE /* nothing */
+
+#  define _LIBCPP_INTRODUCED_IN_LLVM_12 1
+#  define _LIBCPP_INTRODUCED_IN_LLVM_12_ATTRIBUTE /* nothing */
+
+#  define _LIBCPP_INTRODUCED_IN_LLVM_11 1
+#  define _LIBCPP_INTRODUCED_IN_LLVM_11_ATTRIBUTE /* nothing */
+
+#  define _LIBCPP_INTRODUCED_IN_LLVM_10 1
+#  define _LIBCPP_INTRODUCED_IN_LLVM_10_ATTRIBUTE /* nothing */
+
+#  define _LIBCPP_INTRODUCED_IN_LLVM_9 1
+#  define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE      /* nothing */
+#  define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_PUSH /* nothing */
+#  define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_POP  /* nothing */
+
+#  define _LIBCPP_INTRODUCED_IN_LLVM_8 1
+#  define _LIBCPP_INTRODUCED_IN_LLVM_8_ATTRIBUTE /* nothing */
+
+#  define _LIBCPP_INTRODUCED_IN_LLVM_4 1
+#  define _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE /* nothing */
+
+#elif defined(__APPLE__)
+
+// clang-format off
+
+// LLVM 19
+// TODO: Fill this in
+#  define _LIBCPP_INTRODUCED_IN_LLVM_19 0
+#  define _LIBCPP_INTRODUCED_IN_LLVM_19_ATTRIBUTE __attribute__((unavailable))
+
+// LLVM 18
+// TODO: Fill this in
+#  define _LIBCPP_INTRODUCED_IN_LLVM_18 0
+#  define _LIBCPP_INTRODUCED_IN_LLVM_18_ATTRIBUTE __attribute__((unavailable))
+
+// LLVM 17
+#  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 140400) ||       \
+      (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 170400) ||     \
+      (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 170400) ||             \
+      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 100400)
+#    define _LIBCPP_INTRODUCED_IN_LLVM_17 0
+#  else
+#    define _LIBCPP_INTRODUCED_IN_LLVM_17 1
+#  endif
+#  define _LIBCPP_INTRODUCED_IN_LLVM_17_ATTRIBUTE                                                                 \
+    __attribute__((availability(macos, strict, introduced = 14.4)))                                               \
+    __attribute__((availability(ios, strict, introduced = 17.4)))                                                 \
+    __attribute__((availability(tvos, strict, introduced = 17.4)))                                                \
+    __attribute__((availability(watchos, strict, introduced = 10.4)))
+
+// LLVM 16
+#  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 140000) ||       \
+      (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 170000) ||     \
+      (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 170000) ||             \
+      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 100000)
+#    define _LIBCPP_INTRODUCED_IN_LLVM_16 0
+#  else
+#    define _LIBCPP_INTRODUCED_IN_LLVM_16 1
+#  endif
+#  define _LIBCPP_INTRODUCED_IN_LLVM_16_ATTRIBUTE                                                                 \
+    __attribute__((availability(macos, strict, introduced = 14.0)))                                               \
+    __attribute__((availability(ios, strict, introduced = 17.0)))                                                 \
+    __attribute__((availability(tvos, strict, introduced = 17.0)))                                                \
+    __attribute__((availability(watchos, strict, introduced = 10.0)))
+
+// LLVM 15
+#  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 130400) ||   \
+      (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 160500) || \
+      (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 160500) ||         \
+      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 90500)
+#    define _LIBCPP_INTRODUCED_IN_LLVM_15 0
+#  else
+#    define _LIBCPP_INTRODUCED_IN_LLVM_15 1
+#  endif
+#  define _LIBCPP_INTRODUCED_IN_LLVM_15_ATTRIBUTE                                                                 \
+    __attribute__((availability(macos, strict, introduced = 13.4)))                                               \
+    __attribute__((availability(ios, strict, introduced = 16.5)))                                                 \
+    __attribute__((availability(tvos, strict, introduced = 16.5)))                                                \
+    __attribute__((availability(watchos, strict, introduced = 9.5)))
+
+// LLVM 14
+#  define _LIBCPP_INTRODUCED_IN_LLVM_14 _LIBCPP_INTRODUCED_IN_LLVM_15
+#  define _LIBCPP_INTRODUCED_IN_LLVM_14_ATTRIBUTE _LIBCPP_INTRODUCED_IN_LLVM_15_ATTRIBUTE
+
+// LLVM 13
+#  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 130000) ||   \
+      (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 160000) || \
+      (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 160000) ||         \
+      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 90000)
+#    define _LIBCPP_INTRODUCED_IN_LLVM_13 0
+#  else
+#    define _LIBCPP_INTRODUCED_IN_LLVM_13 1
+#  endif
+#  define _LIBCPP_INTRODUCED_IN_LLVM_13_ATTRIBUTE                                                                 \
+    __attribute__((availability(macos, strict, introduced = 13.0)))                                               \
+    __attribute__((availability(ios, strict, introduced = 16.0)))                                                 \
+    __attribute__((availability(tvos, strict, introduced = 16.0)))                                                \
+    __attribute__((availability(watchos, strict, introduced = 9.0)))
+
+// LLVM 12
+#  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 120300)   ||     \
+      (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 150300) ||     \
+      (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 150300)         ||     \
+      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 80300)
+#    define _LIBCPP_INTRODUCED_IN_LLVM_12 0
+#  else
+#    define _LIBCPP_INTRODUCED_IN_LLVM_12 1
+#  endif
+#  define _LIBCPP_INTRODUCED_IN_LLVM_12_ATTRIBUTE                                                                 \
+    __attribute__((availability(macos, strict, introduced = 12.3)))                                               \
+    __attribute__((availability(ios, strict, introduced = 15.3)))                                                 \
+    __attribute__((availability(tvos, strict, introduced = 15.3)))                                                \
+    __attribute__((availability(watchos, strict, introduced = 8.3)))
+
+// LLVM 11
+#  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 110000) ||   \
+      (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 140000) || \
+      (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 140000) ||         \
+      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 70000)
+#    define _LIBCPP_INTRODUCED_IN_LLVM_11 0
+#  else
+#    define _LIBCPP_INTRODUCED_IN_LLVM_11 1
+#  endif
+#  define _LIBCPP_INTRODUCED_IN_LLVM_11_ATTRIBUTE                                                                 \
+    __attribute__((availability(macos, strict, introduced = 11.0)))                                               \
+    __attribute__((availability(ios, strict, introduced = 14.0)))                                                 \
+    __attribute__((availability(tvos, strict, introduced = 14.0)))                                                \
+    __attribute__((availability(watchos, strict, introduced = 7.0)))
+
+// LLVM 10
+#  define _LIBCPP_INTRODUCED_IN_LLVM_10 _LIBCPP_INTRODUCED_IN_LLVM_11
+#  define _LIBCPP_INTRODUCED_IN_LLVM_10_ATTRIBUTE _LIBCPP_INTRODUCED_IN_LLVM_11_ATTRIBUTE
+
+// LLVM 9
+#  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101500) ||   \
+      (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 130000) || \
+      (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 130000) ||         \
+      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 60000)
+#    define _LIBCPP_INTRODUCED_IN_LLVM_9 0
+#  else
+#    define _LIBCPP_INTRODUCED_IN_LLVM_9 1
+#  endif
+#  define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE                                                                  \
+    __attribute__((availability(macos, strict, introduced = 10.15)))                                              \
+    __attribute__((availability(ios, strict, introduced = 13.0)))                                                 \
+    __attribute__((availability(tvos, strict, introduced = 13.0)))                                                \
+    __attribute__((availability(watchos, strict, introduced = 6.0)))
+#  define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_PUSH                                                                            \
+    _Pragma("clang attribute push(__attribute__((availability(macos,strict,introduced=10.15))), apply_to=any(function,record))") \
+    _Pragma("clang attribute push(__attribute__((availability(ios,strict,introduced=13.0))), apply_to=any(function,record))")    \
+    _Pragma("clang attribute push(__attribute__((availability(tvos,strict,introduced=13.0))), apply_to=any(function,record))")   \
+    _Pragma("clang attribute push(__attribute__((availability(watchos,strict,introduced=6.0))), apply_to=any(function,record))")
+#  define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_POP                                                                    \
+    _Pragma("clang attribute pop") \
+    _Pragma("clang attribute pop") \
+    _Pragma("clang attribute pop") \
+    _Pragma("clang attribute pop")
+
+// LLVM 4
+#  if defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 50000
+#    define _LIBCPP_INTRODUCED_IN_LLVM_4 0
+#  else
+#    define _LIBCPP_INTRODUCED_IN_LLVM_4 1
+#  endif
+#  define _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE __attribute__((availability(watchos, strict, introduced = 5.0)))
+
+// clang-format on
+
+#else
+
+// ...New vendors can add availability markup here...
+
+#  error                                                                                                               \
+      "It looks like you're trying to enable vendor availability markup, but you haven't defined the corresponding macros yet!"
+
+#endif
+
+// These macros control the availability of std::bad_optional_access and
+// other exception types. These were put in the shared library to prevent
+// code bloat from every user program defining the vtable for these exception
+// types.
+//
+// Note that when exceptions are disabled, the methods that normally throw
+// these exceptions can be used even on older deployment targets, but those
+// methods will abort instead of throwing.
+#define _LIBCPP_AVAILABILITY_HAS_BAD_OPTIONAL_ACCESS _LIBCPP_INTRODUCED_IN_LLVM_4
+#define _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE
+
+#define _LIBCPP_AVAILABILITY_HAS_BAD_VARIANT_ACCESS _LIBCPP_INTRODUCED_IN_LLVM_4
+#define _LIBCPP_AVAILABILITY_BAD_VARIANT_ACCESS _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE
+
+#define _LIBCPP_AVAILABILITY_HAS_BAD_ANY_CAST _LIBCPP_INTRODUCED_IN_LLVM_4
+#define _LIBCPP_AVAILABILITY_BAD_ANY_CAST _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE
+
+// These macros control the availability of all parts of <filesystem> that
+// depend on something in the dylib.
+#define _LIBCPP_AVAILABILITY_HAS_FILESYSTEM_LIBRARY _LIBCPP_INTRODUCED_IN_LLVM_9
+#define _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE
+#define _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_PUSH
+#define _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_POP
+
+// This controls the availability of the C++20 synchronization library,
+// which requires shared library support for various operations
+// (see libcxx/src/atomic.cpp). This includes <barier>, <latch>,
+// <semaphore>, and notification functions on std::atomic.
+#define _LIBCPP_AVAILABILITY_HAS_SYNC _LIBCPP_INTRODUCED_IN_LLVM_11
+#define _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INTRODUCED_IN_LLVM_11_ATTRIBUTE
+
+// Enable additional explicit instantiations of iostreams components. This
+// reduces the number of weak definitions generated in programs that use
+// iostreams by providing a single strong definition in the shared library.
+//
+// TODO: Enable additional explicit instantiations on GCC once it supports exclude_from_explicit_instantiation,
+//       or once libc++ doesn't use the attribute anymore.
+// TODO: Enable them on Windows once https://llvm.org/PR41018 has been fixed.
+#if !defined(_LIBCPP_COMPILER_GCC) && !defined(_WIN32)
+#  define _LIBCPP_AVAILABILITY_HAS_ADDITIONAL_IOSTREAM_EXPLICIT_INSTANTIATIONS_1 _LIBCPP_INTRODUCED_IN_LLVM_12
+#else
+#  define _LIBCPP_AVAILABILITY_HAS_ADDITIONAL_IOSTREAM_EXPLICIT_INSTANTIATIONS_1 0
+#endif
+
+// This controls the availability of floating-point std::to_chars functions.
+// These overloads were added later than the integer overloads.
+#define _LIBCPP_AVAILABILITY_HAS_TO_CHARS_FLOATING_POINT _LIBCPP_INTRODUCED_IN_LLVM_14
+#define _LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_INTRODUCED_IN_LLVM_14_ATTRIBUTE
+
+// This controls whether the library claims to provide a default verbose
+// termination function, and consequently whether the headers will try
+// to use it when the mechanism isn't overriden at compile-time.
+#define _LIBCPP_AVAILABILITY_HAS_VERBOSE_ABORT _LIBCPP_INTRODUCED_IN_LLVM_15
+#define _LIBCPP_AVAILABILITY_VERBOSE_ABORT _LIBCPP_INTRODUCED_IN_LLVM_15_ATTRIBUTE
+
+// This controls the availability of the C++17 std::pmr library,
+// which is implemented in large part in the built library.
+//
+// TODO: Enable std::pmr markup once https://github.com/llvm/llvm-project/issues/40340 has been fixed
+//       Until then, it is possible for folks to try to use `std::pmr` when back-deploying to targets that don't support
+//       it and it'll be a load-time error, but we don't have a good alternative because the library won't compile if we
+//       use availability annotations until that bug has been fixed.
+#define _LIBCPP_AVAILABILITY_HAS_PMR _LIBCPP_INTRODUCED_IN_LLVM_16
+#define _LIBCPP_AVAILABILITY_PMR
+
+// These macros controls the availability of __cxa_init_primary_exception
+// in the built library, which std::make_exception_ptr might use
+// (see libcxx/include/__exception/exception_ptr.h).
+#define _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION _LIBCPP_INTRODUCED_IN_LLVM_18
+#define _LIBCPP_AVAILABILITY_INIT_PRIMARY_EXCEPTION _LIBCPP_INTRODUCED_IN_LLVM_18_ATTRIBUTE
+
+// This controls the availability of C++23 <print>, which
+// has a dependency on the built library (it needs access to
+// the underlying buffer types of std::cout, std::cerr, and std::clog.
+#define _LIBCPP_AVAILABILITY_HAS_PRINT _LIBCPP_INTRODUCED_IN_LLVM_18
+#define _LIBCPP_AVAILABILITY_PRINT _LIBCPP_INTRODUCED_IN_LLVM_18_ATTRIBUTE
+
+// This controls the availability of the C++20 time zone database.
+// The parser code is built in the library.
+#define _LIBCPP_AVAILABILITY_HAS_TZDB _LIBCPP_INTRODUCED_IN_LLVM_19
+#define _LIBCPP_AVAILABILITY_TZDB _LIBCPP_INTRODUCED_IN_LLVM_19_ATTRIBUTE
+
+// These macros determine whether we assume that std::bad_function_call and
+// std::bad_expected_access provide a key function in the dylib. This allows
+// centralizing their vtable and typeinfo instead of having all TUs provide
+// a weak definition that then gets deduplicated.
+#define _LIBCPP_AVAILABILITY_HAS_BAD_FUNCTION_CALL_KEY_FUNCTION _LIBCPP_INTRODUCED_IN_LLVM_19
+#define _LIBCPP_AVAILABILITY_BAD_FUNCTION_CALL_KEY_FUNCTION _LIBCPP_INTRODUCED_IN_LLVM_19_ATTRIBUTE
+#define _LIBCPP_AVAILABILITY_HAS_BAD_EXPECTED_ACCESS_KEY_FUNCTION _LIBCPP_INTRODUCED_IN_LLVM_19
+#define _LIBCPP_AVAILABILITY_BAD_EXPECTED_ACCESS_KEY_FUNCTION _LIBCPP_INTRODUCED_IN_LLVM_19_ATTRIBUTE
+
+// Define availability attributes that depend on _LIBCPP_HAS_NO_EXCEPTIONS.
+// Those are defined in terms of the availability attributes above, and
+// should not be vendor-specific.
+#if defined(_LIBCPP_HAS_NO_EXCEPTIONS)
+#  define _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST
+#  define _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS
+#  define _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS
+#else
+#  define _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _LIBCPP_AVAILABILITY_BAD_ANY_CAST
+#  define _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS
+#  define _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS _LIBCPP_AVAILABILITY_BAD_VARIANT_ACCESS
+#endif
+
+// Define availability attributes that depend on both
+// _LIBCPP_HAS_NO_EXCEPTIONS and _LIBCPP_HAS_NO_RTTI.
+#if defined(_LIBCPP_HAS_NO_EXCEPTIONS) || defined(_LIBCPP_HAS_NO_RTTI)
+#  undef _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION
+#  undef _LIBCPP_AVAILABILITY_INIT_PRIMARY_EXCEPTION
+#  define _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION 0
+#  define _LIBCPP_AVAILABILITY_INIT_PRIMARY_EXCEPTION
+#endif
+
+#endif // _LIBCPP___CONFIGURATION_AVAILABILITY_H
diff --git a/libcxx/include/__cxx03/__configuration/compiler.h b/libcxx/include/__cxx03/__configuration/compiler.h
new file mode 100644
index 000000000000..80ece22bb50b
--- /dev/null
+++ b/libcxx/include/__cxx03/__configuration/compiler.h
@@ -0,0 +1,51 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONFIGURATION_COMPILER_H
+#define _LIBCPP___CONFIGURATION_COMPILER_H
+
+#include <__config_site>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+#if defined(__apple_build_version__)
+// Given AppleClang XX.Y.Z, _LIBCPP_APPLE_CLANG_VER is XXYZ (e.g. AppleClang 14.0.3 => 1403)
+#  define _LIBCPP_COMPILER_CLANG_BASED
+#  define _LIBCPP_APPLE_CLANG_VER (__apple_build_version__ / 10000)
+#elif defined(__clang__)
+#  define _LIBCPP_COMPILER_CLANG_BASED
+#  define _LIBCPP_CLANG_VER (__clang_major__ * 100 + __clang_minor__)
+#elif defined(__GNUC__)
+#  define _LIBCPP_COMPILER_GCC
+#  define _LIBCPP_GCC_VER (__GNUC__ * 100 + __GNUC_MINOR__)
+#endif
+
+#ifdef __cplusplus
+
+// Warn if a compiler version is used that is not supported anymore
+// LLVM RELEASE Update the minimum compiler versions
+#  if defined(_LIBCPP_CLANG_VER)
+#    if _LIBCPP_CLANG_VER < 1700
+#      warning "Libc++ only supports Clang 17 and later"
+#    endif
+#  elif defined(_LIBCPP_APPLE_CLANG_VER)
+#    if _LIBCPP_APPLE_CLANG_VER < 1500
+#      warning "Libc++ only supports AppleClang 15 and later"
+#    endif
+#  elif defined(_LIBCPP_GCC_VER)
+#    if _LIBCPP_GCC_VER < 1400
+#      warning "Libc++ only supports GCC 14 and later"
+#    endif
+#  endif
+
+#endif
+
+#endif // _LIBCPP___CONFIGURATION_COMPILER_H
diff --git a/libcxx/include/__cxx03/__configuration/language.h b/libcxx/include/__cxx03/__configuration/language.h
new file mode 100644
index 000000000000..fa62a7b6f5c2
--- /dev/null
+++ b/libcxx/include/__cxx03/__configuration/language.h
@@ -0,0 +1,46 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONFIGURATION_LANGUAGE_H
+#define _LIBCPP___CONFIGURATION_LANGUAGE_H
+
+#include <__config_site>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+// NOLINTBEGIN(libcpp-cpp-version-check)
+#ifdef __cplusplus
+#  if __cplusplus <= 201103L
+#    define _LIBCPP_STD_VER 11
+#  elif __cplusplus <= 201402L
+#    define _LIBCPP_STD_VER 14
+#  elif __cplusplus <= 201703L
+#    define _LIBCPP_STD_VER 17
+#  elif __cplusplus <= 202002L
+#    define _LIBCPP_STD_VER 20
+#  elif __cplusplus <= 202302L
+#    define _LIBCPP_STD_VER 23
+#  else
+// Expected release year of the next C++ standard
+#    define _LIBCPP_STD_VER 26
+#  endif
+#endif // __cplusplus
+// NOLINTEND(libcpp-cpp-version-check)
+
+#if !defined(__cpp_rtti) || __cpp_rtti < 199711L
+#  define _LIBCPP_HAS_NO_RTTI
+#endif
+
+#if !defined(__cpp_exceptions) || __cpp_exceptions < 199711L
+#  define _LIBCPP_HAS_NO_EXCEPTIONS
+#endif
+
+#endif // _LIBCPP___CONFIGURATION_LANGUAGE_H
diff --git a/libcxx/include/__cxx03/__configuration/platform.h b/libcxx/include/__cxx03/__configuration/platform.h
new file mode 100644
index 000000000000..27f68d04e8a8
--- /dev/null
+++ b/libcxx/include/__cxx03/__configuration/platform.h
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CONFIGURATION_PLATFORM_H
+#define _LIBCPP___CONFIGURATION_PLATFORM_H
+
+#include <__config_site>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+#if defined(__ELF__)
+#  define _LIBCPP_OBJECT_FORMAT_ELF 1
+#elif defined(__MACH__)
+#  define _LIBCPP_OBJECT_FORMAT_MACHO 1
+#elif defined(_WIN32)
+#  define _LIBCPP_OBJECT_FORMAT_COFF 1
+#elif defined(__wasm__)
+#  define _LIBCPP_OBJECT_FORMAT_WASM 1
+#elif defined(_AIX)
+#  define _LIBCPP_OBJECT_FORMAT_XCOFF 1
+#else
+// ... add new file formats here ...
+#endif
+
+// Need to detect which libc we're using if we're on Linux.
+#if defined(__linux__)
+#  include <features.h>
+#  if defined(__GLIBC_PREREQ)
+#    define _LIBCPP_GLIBC_PREREQ(a, b) __GLIBC_PREREQ(a, b)
+#  else
+#    define _LIBCPP_GLIBC_PREREQ(a, b) 0
+#  endif // defined(__GLIBC_PREREQ)
+#endif   // defined(__linux__)
+
+#ifndef __BYTE_ORDER__
+#  error                                                                                                               \
+      "Your compiler doesn't seem to define __BYTE_ORDER__, which is required by libc++ to know the endianness of your target platform"
+#endif
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#  define _LIBCPP_LITTLE_ENDIAN
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#  define _LIBCPP_BIG_ENDIAN
+#endif // __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+
+#endif // _LIBCPP___CONFIGURATION_PLATFORM_H
diff --git a/libcxx/include/__cxx03/__coroutine/coroutine_handle.h b/libcxx/include/__cxx03/__coroutine/coroutine_handle.h
new file mode 100644
index 000000000000..4557a6643c23
--- /dev/null
+++ b/libcxx/include/__cxx03/__coroutine/coroutine_handle.h
@@ -0,0 +1,176 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COROUTINE_COROUTINE_HANDLE_H
+#define _LIBCPP___COROUTINE_COROUTINE_HANDLE_H
+
+#include <__assert>
+#include <__config>
+#include <__functional/hash.h>
+#include <__memory/addressof.h>
+#include <__type_traits/remove_cv.h>
+#include <compare>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// [coroutine.handle]
+template <class _Promise = void>
+struct _LIBCPP_TEMPLATE_VIS coroutine_handle;
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS coroutine_handle<void> {
+public:
+  // [coroutine.handle.con], construct/reset
+  constexpr coroutine_handle() noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr coroutine_handle(nullptr_t) noexcept {}
+
+  _LIBCPP_HIDE_FROM_ABI coroutine_handle& operator=(nullptr_t) noexcept {
+    __handle_ = nullptr;
+    return *this;
+  }
+
+  // [coroutine.handle.export.import], export/import
+  _LIBCPP_HIDE_FROM_ABI constexpr void* address() const noexcept { return __handle_; }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr coroutine_handle from_address(void* __addr) noexcept {
+    coroutine_handle __tmp;
+    __tmp.__handle_ = __addr;
+    return __tmp;
+  }
+
+  // [coroutine.handle.observers], observers
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return __handle_ != nullptr; }
+
+  _LIBCPP_HIDE_FROM_ABI bool done() const {
+    _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(__is_suspended(), "done() can be called only on suspended coroutines");
+    return __builtin_coro_done(__handle_);
+  }
+
+  // [coroutine.handle.resumption], resumption
+  _LIBCPP_HIDE_FROM_ABI void operator()() const { resume(); }
+
+  _LIBCPP_HIDE_FROM_ABI void resume() const {
+    _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(__is_suspended(), "resume() can be called only on suspended coroutines");
+    _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(!done(), "resume() has undefined behavior when the coroutine is done");
+    __builtin_coro_resume(__handle_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void destroy() const {
+    _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(__is_suspended(), "destroy() can be called only on suspended coroutines");
+    __builtin_coro_destroy(__handle_);
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI bool __is_suspended() const {
+    // FIXME actually implement a check for if the coro is suspended.
+    return __handle_ != nullptr;
+  }
+
+  void* __handle_ = nullptr;
+};
+
+// [coroutine.handle.compare]
+inline _LIBCPP_HIDE_FROM_ABI constexpr bool operator==(coroutine_handle<> __x, coroutine_handle<> __y) noexcept {
+  return __x.address() == __y.address();
+}
+inline _LIBCPP_HIDE_FROM_ABI constexpr strong_ordering
+operator<=>(coroutine_handle<> __x, coroutine_handle<> __y) noexcept {
+  return compare_three_way()(__x.address(), __y.address());
+}
+
+template <class _Promise>
+struct _LIBCPP_TEMPLATE_VIS coroutine_handle {
+public:
+  // [coroutine.handle.con], construct/reset
+  constexpr coroutine_handle() noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr coroutine_handle(nullptr_t) noexcept {}
+
+  _LIBCPP_HIDE_FROM_ABI static coroutine_handle from_promise(_Promise& __promise) {
+    using _RawPromise = __remove_cv_t<_Promise>;
+    coroutine_handle __tmp;
+    __tmp.__handle_ =
+        __builtin_coro_promise(std::addressof(const_cast<_RawPromise&>(__promise)), alignof(_Promise), true);
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI coroutine_handle& operator=(nullptr_t) noexcept {
+    __handle_ = nullptr;
+    return *this;
+  }
+
+  // [coroutine.handle.export.import], export/import
+  _LIBCPP_HIDE_FROM_ABI constexpr void* address() const noexcept { return __handle_; }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr coroutine_handle from_address(void* __addr) noexcept {
+    coroutine_handle __tmp;
+    __tmp.__handle_ = __addr;
+    return __tmp;
+  }
+
+  // [coroutine.handle.conv], conversion
+  _LIBCPP_HIDE_FROM_ABI constexpr operator coroutine_handle<>() const noexcept {
+    return coroutine_handle<>::from_address(address());
+  }
+
+  // [coroutine.handle.observers], observers
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return __handle_ != nullptr; }
+
+  _LIBCPP_HIDE_FROM_ABI bool done() const {
+    _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(__is_suspended(), "done() can be called only on suspended coroutines");
+    return __builtin_coro_done(__handle_);
+  }
+
+  // [coroutine.handle.resumption], resumption
+  _LIBCPP_HIDE_FROM_ABI void operator()() const { resume(); }
+
+  _LIBCPP_HIDE_FROM_ABI void resume() const {
+    _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(__is_suspended(), "resume() can be called only on suspended coroutines");
+    _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(!done(), "resume() has undefined behavior when the coroutine is done");
+    __builtin_coro_resume(__handle_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void destroy() const {
+    _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(__is_suspended(), "destroy() can be called only on suspended coroutines");
+    __builtin_coro_destroy(__handle_);
+  }
+
+  // [coroutine.handle.promise], promise access
+  _LIBCPP_HIDE_FROM_ABI _Promise& promise() const {
+    return *static_cast<_Promise*>(__builtin_coro_promise(this->__handle_, alignof(_Promise), false));
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI bool __is_suspended() const {
+    // FIXME actually implement a check for if the coro is suspended.
+    return __handle_ != nullptr;
+  }
+  void* __handle_ = nullptr;
+};
+
+// [coroutine.handle.hash]
+template <class _Tp>
+struct hash<coroutine_handle<_Tp>> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const coroutine_handle<_Tp>& __v) const noexcept {
+    return hash<void*>()(__v.address());
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // __LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___COROUTINE_COROUTINE_HANDLE_H
diff --git a/libcxx/include/__cxx03/__coroutine/coroutine_traits.h b/libcxx/include/__cxx03/__coroutine/coroutine_traits.h
new file mode 100644
index 000000000000..78f05341f748
--- /dev/null
+++ b/libcxx/include/__cxx03/__coroutine/coroutine_traits.h
@@ -0,0 +1,48 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COROUTINE_COROUTINE_TRAITS_H
+#define _LIBCPP___COROUTINE_COROUTINE_TRAITS_H
+
+#include <__config>
+#include <__type_traits/void_t.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// [coroutine.traits]
+// [coroutine.traits.primary]
+//   The header <coroutine> defined the primary template coroutine_traits such that
+// if ArgTypes is a parameter pack of types and if the qualified-id R::promise_type
+// is valid and denotes a type ([temp.deduct]), then coroutine_traits<R, ArgTypes...>
+// has the following publicly accessible memebr:
+//
+//    using promise_type = typename R::promise_type;
+//
+// Otherwise, coroutine_traits<R, ArgTypes...> has no members.
+template <class _Tp, class = void>
+struct __coroutine_traits_sfinae {};
+
+template <class _Tp>
+struct __coroutine_traits_sfinae< _Tp, __void_t<typename _Tp::promise_type> > {
+  using promise_type = typename _Tp::promise_type;
+};
+
+template <class _Ret, class... _Args>
+struct coroutine_traits : public __coroutine_traits_sfinae<_Ret> {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // __LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___COROUTINE_COROUTINE_TRAITS_H
diff --git a/libcxx/include/__cxx03/__coroutine/noop_coroutine_handle.h b/libcxx/include/__cxx03/__coroutine/noop_coroutine_handle.h
new file mode 100644
index 000000000000..da13d579604b
--- /dev/null
+++ b/libcxx/include/__cxx03/__coroutine/noop_coroutine_handle.h
@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___COROUTINE_NOOP_COROUTINE_HANDLE_H
+#define _LIBCPP___COROUTINE_NOOP_COROUTINE_HANDLE_H
+
+#include <__config>
+#include <__coroutine/coroutine_handle.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  if __has_builtin(__builtin_coro_noop) || defined(_LIBCPP_COMPILER_GCC)
+
+// [coroutine.noop]
+// [coroutine.promise.noop]
+struct noop_coroutine_promise {};
+
+// [coroutine.handle.noop]
+template <>
+struct _LIBCPP_TEMPLATE_VIS coroutine_handle<noop_coroutine_promise> {
+public:
+  // [coroutine.handle.noop.conv], conversion
+  _LIBCPP_HIDE_FROM_ABI constexpr operator coroutine_handle<>() const noexcept {
+    return coroutine_handle<>::from_address(address());
+  }
+
+  // [coroutine.handle.noop.observers], observers
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return true; }
+  _LIBCPP_HIDE_FROM_ABI constexpr bool done() const noexcept { return false; }
+
+  // [coroutine.handle.noop.resumption], resumption
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator()() const noexcept {}
+  _LIBCPP_HIDE_FROM_ABI constexpr void resume() const noexcept {}
+  _LIBCPP_HIDE_FROM_ABI constexpr void destroy() const noexcept {}
+
+  // [coroutine.handle.noop.promise], promise access
+  _LIBCPP_HIDE_FROM_ABI noop_coroutine_promise& promise() const noexcept {
+    return *static_cast<noop_coroutine_promise*>(
+        __builtin_coro_promise(this->__handle_, alignof(noop_coroutine_promise), false));
+  }
+
+  // [coroutine.handle.noop.address], address
+  _LIBCPP_HIDE_FROM_ABI constexpr void* address() const noexcept { return __handle_; }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI friend coroutine_handle<noop_coroutine_promise> noop_coroutine() noexcept;
+
+#    if __has_builtin(__builtin_coro_noop)
+  _LIBCPP_HIDE_FROM_ABI coroutine_handle() noexcept { this->__handle_ = __builtin_coro_noop(); }
+
+  void* __handle_ = nullptr;
+
+#    elif defined(_LIBCPP_COMPILER_GCC)
+  // GCC doesn't implement __builtin_coro_noop().
+  // Construct the coroutine frame manually instead.
+  struct __noop_coroutine_frame_ty_ {
+    static void __dummy_resume_destroy_func() {}
+
+    void (*__resume_)()  = __dummy_resume_destroy_func;
+    void (*__destroy_)() = __dummy_resume_destroy_func;
+    struct noop_coroutine_promise __promise_;
+  };
+
+  static __noop_coroutine_frame_ty_ __noop_coroutine_frame_;
+
+  void* __handle_ = &__noop_coroutine_frame_;
+
+  _LIBCPP_HIDE_FROM_ABI coroutine_handle() noexcept = default;
+
+#    endif // __has_builtin(__builtin_coro_noop)
+};
+
+using noop_coroutine_handle = coroutine_handle<noop_coroutine_promise>;
+
+#    if defined(_LIBCPP_COMPILER_GCC)
+inline noop_coroutine_handle::__noop_coroutine_frame_ty_ noop_coroutine_handle::__noop_coroutine_frame_{};
+#    endif
+
+// [coroutine.noop.coroutine]
+inline _LIBCPP_HIDE_FROM_ABI noop_coroutine_handle noop_coroutine() noexcept { return noop_coroutine_handle(); }
+
+#  endif // __has_builtin(__builtin_coro_noop) || defined(_LIBCPP_COMPILER_GCC)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // __LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___COROUTINE_NOOP_COROUTINE_HANDLE_H
diff --git a/libcxx/include/__cxx03/__coroutine/trivial_awaitables.h b/libcxx/include/__cxx03/__coroutine/trivial_awaitables.h
new file mode 100644
index 000000000000..b604bd3c2d8a
--- /dev/null
+++ b/libcxx/include/__cxx03/__coroutine/trivial_awaitables.h
@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __LIBCPP___COROUTINE_TRIVIAL_AWAITABLES_H
+#define __LIBCPP___COROUTINE_TRIVIAL_AWAITABLES_H
+
+#include <__config>
+#include <__coroutine/coroutine_handle.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// [coroutine.trivial.awaitables]
+struct suspend_never {
+  _LIBCPP_HIDE_FROM_ABI constexpr bool await_ready() const noexcept { return true; }
+  _LIBCPP_HIDE_FROM_ABI constexpr void await_suspend(coroutine_handle<>) const noexcept {}
+  _LIBCPP_HIDE_FROM_ABI constexpr void await_resume() const noexcept {}
+};
+
+struct suspend_always {
+  _LIBCPP_HIDE_FROM_ABI constexpr bool await_ready() const noexcept { return false; }
+  _LIBCPP_HIDE_FROM_ABI constexpr void await_suspend(coroutine_handle<>) const noexcept {}
+  _LIBCPP_HIDE_FROM_ABI constexpr void await_resume() const noexcept {}
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // __LIBCPP_STD_VER >= 20
+
+#endif // __LIBCPP___COROUTINE_TRIVIAL_AWAITABLES_H
diff --git a/libcxx/include/__cxx03/__debug_utils/randomize_range.h b/libcxx/include/__cxx03/__debug_utils/randomize_range.h
new file mode 100644
index 000000000000..7eb77d81ab2a
--- /dev/null
+++ b/libcxx/include/__cxx03/__debug_utils/randomize_range.h
@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LIBCXX_DEBUG_RANDOMIZE_RANGE_H
+#define _LIBCPP___LIBCXX_DEBUG_RANDOMIZE_RANGE_H
+
+#include <__config>
+
+#ifdef _LIBCPP_DEBUG_RANDOMIZE_UNSPECIFIED_STABILITY
+#  include <__algorithm/shuffle.h>
+#  include <__type_traits/is_constant_evaluated.h>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _AlgPolicy, class _Iterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __debug_randomize_range(_Iterator __first, _Sentinel __last) {
+#ifdef _LIBCPP_DEBUG_RANDOMIZE_UNSPECIFIED_STABILITY
+#  ifdef _LIBCPP_CXX03_LANG
+#    error Support for unspecified stability is only for C++11 and higher
+#  endif
+
+  if (!__libcpp_is_constant_evaluated())
+    std::__shuffle<_AlgPolicy>(__first, __last, __libcpp_debug_randomizer());
+#else
+  (void)__first;
+  (void)__last;
+#endif
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___LIBCXX_DEBUG_RANDOMIZE_RANGE_H
diff --git a/libcxx/include/__cxx03/__debug_utils/sanitizers.h b/libcxx/include/__cxx03/__debug_utils/sanitizers.h
new file mode 100644
index 000000000000..d8547e324933
--- /dev/null
+++ b/libcxx/include/__cxx03/__debug_utils/sanitizers.h
@@ -0,0 +1,104 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LIBCXX_DEBUG_UTILS_SANITIZERS_H
+#define _LIBCPP___LIBCXX_DEBUG_UTILS_SANITIZERS_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_constant_evaluated.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#ifndef _LIBCPP_HAS_NO_ASAN
+
+extern "C" {
+_LIBCPP_EXPORTED_FROM_ABI void
+__sanitizer_annotate_contiguous_container(const void*, const void*, const void*, const void*);
+_LIBCPP_EXPORTED_FROM_ABI void __sanitizer_annotate_double_ended_contiguous_container(
+    const void*, const void*, const void*, const void*, const void*, const void*);
+_LIBCPP_EXPORTED_FROM_ABI int
+__sanitizer_verify_double_ended_contiguous_container(const void*, const void*, const void*, const void*);
+}
+
+#endif // _LIBCPP_HAS_NO_ASAN
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// ASan choices
+#ifndef _LIBCPP_HAS_NO_ASAN
+#  define _LIBCPP_HAS_ASAN_CONTAINER_ANNOTATIONS_FOR_ALL_ALLOCATORS 1
+#endif
+
+#ifdef _LIBCPP_HAS_ASAN_CONTAINER_ANNOTATIONS_FOR_ALL_ALLOCATORS
+// __asan_annotate_container_with_allocator determines whether containers with custom allocators are annotated. This is
+// a public customization point to disable annotations if the custom allocator assumes that the memory isn't poisoned.
+// See the https://libcxx.llvm.org/UsingLibcxx.html#turning-off-asan-annotation-in-containers for more information.
+template <class _Alloc>
+struct __asan_annotate_container_with_allocator : true_type {};
+#endif
+
+// Annotate a double-ended contiguous range.
+// - [__first_storage, __last_storage) is the allocated memory region,
+// - [__first_old_contained, __last_old_contained) is the previously allowed (unpoisoned) range, and
+// - [__first_new_contained, __last_new_contained) is the new allowed (unpoisoned) range.
+template <class _Allocator>
+_LIBCPP_HIDE_FROM_ABI void __annotate_double_ended_contiguous_container(
+    const void* __first_storage,
+    const void* __last_storage,
+    const void* __first_old_contained,
+    const void* __last_old_contained,
+    const void* __first_new_contained,
+    const void* __last_new_contained) {
+#ifdef _LIBCPP_HAS_NO_ASAN
+  (void)__first_storage;
+  (void)__last_storage;
+  (void)__first_old_contained;
+  (void)__last_old_contained;
+  (void)__first_new_contained;
+  (void)__last_new_contained;
+#else
+  if (__asan_annotate_container_with_allocator<_Allocator>::value && __first_storage != nullptr)
+    __sanitizer_annotate_double_ended_contiguous_container(
+        __first_storage,
+        __last_storage,
+        __first_old_contained,
+        __last_old_contained,
+        __first_new_contained,
+        __last_new_contained);
+#endif
+}
+
+// Annotate a contiguous range.
+// [__first_storage, __last_storage) is the allocated memory region,
+// __old_last_contained is the previously last allowed (unpoisoned) element, and
+// __new_last_contained is the new last allowed (unpoisoned) element.
+template <class _Allocator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __annotate_contiguous_container(
+    const void* __first_storage,
+    const void* __last_storage,
+    const void* __old_last_contained,
+    const void* __new_last_contained) {
+#ifdef _LIBCPP_HAS_NO_ASAN
+  (void)__first_storage;
+  (void)__last_storage;
+  (void)__old_last_contained;
+  (void)__new_last_contained;
+#else
+  if (!__libcpp_is_constant_evaluated() && __asan_annotate_container_with_allocator<_Allocator>::value &&
+      __first_storage != nullptr)
+    __sanitizer_annotate_contiguous_container(
+        __first_storage, __last_storage, __old_last_contained, __new_last_contained);
+#endif
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___LIBCXX_DEBUG_UTILS_SANITIZERS_H
diff --git a/libcxx/include/__cxx03/__debug_utils/strict_weak_ordering_check.h b/libcxx/include/__cxx03/__debug_utils/strict_weak_ordering_check.h
new file mode 100644
index 000000000000..3a9d88728416
--- /dev/null
+++ b/libcxx/include/__cxx03/__debug_utils/strict_weak_ordering_check.h
@@ -0,0 +1,77 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LIBCXX_DEBUG_STRICT_WEAK_ORDERING_CHECK
+#define _LIBCPP___LIBCXX_DEBUG_STRICT_WEAK_ORDERING_CHECK
+
+#include <__config>
+
+#include <__algorithm/comp_ref_type.h>
+#include <__algorithm/is_sorted.h>
+#include <__assert>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/is_constant_evaluated.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _RandomAccessIterator, class _Comp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+__check_strict_weak_ordering_sorted(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp& __comp) {
+#if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG
+  using __diff_t  = __iter_diff_t<_RandomAccessIterator>;
+  using _Comp_ref = __comp_ref_type<_Comp>;
+  if (!__libcpp_is_constant_evaluated()) {
+    // Check if the range is actually sorted.
+    _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
+        (std::is_sorted<_RandomAccessIterator, _Comp_ref>(__first, __last, _Comp_ref(__comp))),
+        "The range is not sorted after the sort, your comparator is not a valid strict-weak ordering");
+    // Limit the number of elements we need to check.
+    __diff_t __size = __last - __first > __diff_t(100) ? __diff_t(100) : __last - __first;
+    __diff_t __p    = 0;
+    while (__p < __size) {
+      __diff_t __q = __p + __diff_t(1);
+      // Find first element that is greater than *(__first+__p).
+      while (__q < __size && !__comp(*(__first + __p), *(__first + __q))) {
+        ++__q;
+      }
+      // Check that the elements from __p to __q are equal between each other.
+      for (__diff_t __b = __p; __b < __q; ++__b) {
+        for (__diff_t __a = __p; __a <= __b; ++__a) {
+          _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
+              !__comp(*(__first + __a), *(__first + __b)), "Your comparator is not a valid strict-weak ordering");
+          _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
+              !__comp(*(__first + __b), *(__first + __a)), "Your comparator is not a valid strict-weak ordering");
+        }
+      }
+      // Check that elements between __p and __q are less than between __q and __size.
+      for (__diff_t __a = __p; __a < __q; ++__a) {
+        for (__diff_t __b = __q; __b < __size; ++__b) {
+          _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
+              __comp(*(__first + __a), *(__first + __b)), "Your comparator is not a valid strict-weak ordering");
+          _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
+              !__comp(*(__first + __b), *(__first + __a)), "Your comparator is not a valid strict-weak ordering");
+        }
+      }
+      // Skip these equal elements.
+      __p = __q;
+    }
+  }
+#else
+  (void)__first;
+  (void)__last;
+  (void)__comp;
+#endif // _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___LIBCXX_DEBUG_STRICT_WEAK_ORDERING_CHECK
diff --git a/libcxx/include/__cxx03/__exception/exception.h b/libcxx/include/__cxx03/__exception/exception.h
new file mode 100644
index 000000000000..e724e1b99bd1
--- /dev/null
+++ b/libcxx/include/__cxx03/__exception/exception.h
@@ -0,0 +1,94 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___EXCEPTION_EXCEPTION_H
+#define _LIBCPP___EXCEPTION_EXCEPTION_H
+
+#include <__config>
+
+// <vcruntime_exception.h> defines its own std::exception and std::bad_exception types,
+// which we use in order to be ABI-compatible with other STLs on Windows.
+#if defined(_LIBCPP_ABI_VCRUNTIME)
+#  include <vcruntime_exception.h>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+namespace std { // purposefully not using versioning namespace
+
+#if defined(_LIBCPP_ABI_VCRUNTIME) && (!defined(_HAS_EXCEPTIONS) || _HAS_EXCEPTIONS != 0)
+// The std::exception class was already included above, but we're explicit about this condition here for clarity.
+
+#elif defined(_LIBCPP_ABI_VCRUNTIME) && _HAS_EXCEPTIONS == 0
+// However, <vcruntime_exception.h> does not define std::exception and std::bad_exception
+// when _HAS_EXCEPTIONS == 0.
+//
+// Since libc++ still wants to provide the std::exception hierarchy even when _HAS_EXCEPTIONS == 0
+// (after all those are simply types like any other), we define an ABI-compatible version
+// of the VCRuntime std::exception and std::bad_exception types in that mode.
+
+struct __std_exception_data {
+  char const* _What;
+  bool _DoFree;
+};
+
+class exception { // base of all library exceptions
+public:
+  exception() _NOEXCEPT : __data_() {}
+
+  explicit exception(char const* __message) _NOEXCEPT : __data_() {
+    __data_._What   = __message;
+    __data_._DoFree = true;
+  }
+
+  exception(exception const&) _NOEXCEPT {}
+
+  exception& operator=(exception const&) _NOEXCEPT { return *this; }
+
+  virtual ~exception() _NOEXCEPT {}
+
+  virtual char const* what() const _NOEXCEPT { return __data_._What ? __data_._What : "Unknown exception"; }
+
+private:
+  __std_exception_data __data_;
+};
+
+class bad_exception : public exception {
+public:
+  bad_exception() _NOEXCEPT : exception("bad exception") {}
+};
+
+#else  // !defined(_LIBCPP_ABI_VCRUNTIME)
+// On all other platforms, we define our own std::exception and std::bad_exception types
+// regardless of whether exceptions are turned on as a language feature.
+
+class _LIBCPP_EXPORTED_FROM_ABI exception {
+public:
+  _LIBCPP_HIDE_FROM_ABI exception() _NOEXCEPT {}
+  _LIBCPP_HIDE_FROM_ABI exception(const exception&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI exception& operator=(const exception&) _NOEXCEPT = default;
+
+  virtual ~exception() _NOEXCEPT;
+  virtual const char* what() const _NOEXCEPT;
+};
+
+class _LIBCPP_EXPORTED_FROM_ABI bad_exception : public exception {
+public:
+  _LIBCPP_HIDE_FROM_ABI bad_exception() _NOEXCEPT {}
+  _LIBCPP_HIDE_FROM_ABI bad_exception(const bad_exception&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI bad_exception& operator=(const bad_exception&) _NOEXCEPT = default;
+  ~bad_exception() _NOEXCEPT override;
+  const char* what() const _NOEXCEPT override;
+};
+#endif // !_LIBCPP_ABI_VCRUNTIME
+
+} // namespace std
+
+#endif // _LIBCPP___EXCEPTION_EXCEPTION_H
diff --git a/libcxx/include/__cxx03/__exception/exception_ptr.h b/libcxx/include/__cxx03/__exception/exception_ptr.h
new file mode 100644
index 000000000000..beadd9212abd
--- /dev/null
+++ b/libcxx/include/__cxx03/__exception/exception_ptr.h
@@ -0,0 +1,177 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___EXCEPTION_EXCEPTION_PTR_H
+#define _LIBCPP___EXCEPTION_EXCEPTION_PTR_H
+
+#include <__config>
+#include <__exception/operations.h>
+#include <__memory/addressof.h>
+#include <__memory/construct_at.h>
+#include <__type_traits/decay.h>
+#include <cstddef>
+#include <cstdlib>
+#include <new>
+#include <typeinfo>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#ifndef _LIBCPP_ABI_MICROSOFT
+
+#  if _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION
+
+namespace __cxxabiv1 {
+
+extern "C" {
+_LIBCPP_OVERRIDABLE_FUNC_VIS void* __cxa_allocate_exception(size_t) throw();
+_LIBCPP_OVERRIDABLE_FUNC_VIS void __cxa_free_exception(void*) throw();
+
+struct __cxa_exception;
+_LIBCPP_OVERRIDABLE_FUNC_VIS __cxa_exception* __cxa_init_primary_exception(
+    void*,
+    std::type_info*,
+#    if defined(_WIN32)
+    void(__thiscall*)(void*)) throw();
+#    elif defined(__wasm__)
+    // In Wasm, a destructor returns its argument
+    void* (*)(void*)) throw();
+#    else
+    void (*)(void*)) throw();
+#    endif
+}
+
+} // namespace __cxxabiv1
+
+#  endif
+
+#endif
+
+namespace std { // purposefully not using versioning namespace
+
+#ifndef _LIBCPP_ABI_MICROSOFT
+
+class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
+  void* __ptr_;
+
+  static exception_ptr __from_native_exception_pointer(void*) _NOEXCEPT;
+
+  template <class _Ep>
+  friend _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep) _NOEXCEPT;
+
+public:
+  // exception_ptr is basically a COW string.
+  using __trivially_relocatable = exception_ptr;
+
+  _LIBCPP_HIDE_FROM_ABI exception_ptr() _NOEXCEPT : __ptr_() {}
+  _LIBCPP_HIDE_FROM_ABI exception_ptr(nullptr_t) _NOEXCEPT : __ptr_() {}
+
+  exception_ptr(const exception_ptr&) _NOEXCEPT;
+  exception_ptr& operator=(const exception_ptr&) _NOEXCEPT;
+  ~exception_ptr() _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return __ptr_ != nullptr; }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const exception_ptr& __x, const exception_ptr& __y) _NOEXCEPT {
+    return __x.__ptr_ == __y.__ptr_;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const exception_ptr& __x, const exception_ptr& __y) _NOEXCEPT {
+    return !(__x == __y);
+  }
+
+  friend _LIBCPP_EXPORTED_FROM_ABI exception_ptr current_exception() _NOEXCEPT;
+  friend _LIBCPP_EXPORTED_FROM_ABI void rethrow_exception(exception_ptr);
+};
+
+template <class _Ep>
+_LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#    if _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION && __cplusplus >= 201103L
+  using _Ep2 = __decay_t<_Ep>;
+
+  void* __ex = __cxxabiv1::__cxa_allocate_exception(sizeof(_Ep));
+#      ifdef __wasm__
+  // In Wasm, a destructor returns its argument
+  (void)__cxxabiv1::__cxa_init_primary_exception(
+      __ex, const_cast<std::type_info*>(&typeid(_Ep)), [](void* __p) -> void* {
+#      else
+  (void)__cxxabiv1::__cxa_init_primary_exception(__ex, const_cast<std::type_info*>(&typeid(_Ep)), [](void* __p) {
+#      endif
+        std::__destroy_at(static_cast<_Ep2*>(__p));
+#      ifdef __wasm__
+        return __p;
+#      endif
+      });
+
+  try {
+    ::new (__ex) _Ep2(__e);
+    return exception_ptr::__from_native_exception_pointer(__ex);
+  } catch (...) {
+    __cxxabiv1::__cxa_free_exception(__ex);
+    return current_exception();
+  }
+#    else
+  try {
+    throw __e;
+  } catch (...) {
+    return current_exception();
+  }
+#    endif
+#  else
+  ((void)__e);
+  std::abort();
+#  endif
+}
+
+#else // _LIBCPP_ABI_MICROSOFT
+
+class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
+  _LIBCPP_DIAGNOSTIC_PUSH
+  _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wunused-private-field")
+  void* __ptr1_;
+  void* __ptr2_;
+  _LIBCPP_DIAGNOSTIC_POP
+
+public:
+  exception_ptr() _NOEXCEPT;
+  exception_ptr(nullptr_t) _NOEXCEPT;
+  exception_ptr(const exception_ptr& __other) _NOEXCEPT;
+  exception_ptr& operator=(const exception_ptr& __other) _NOEXCEPT;
+  exception_ptr& operator=(nullptr_t) _NOEXCEPT;
+  ~exception_ptr() _NOEXCEPT;
+  explicit operator bool() const _NOEXCEPT;
+};
+
+_LIBCPP_EXPORTED_FROM_ABI bool operator==(const exception_ptr& __x, const exception_ptr& __y) _NOEXCEPT;
+
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const exception_ptr& __x, const exception_ptr& __y) _NOEXCEPT {
+  return !(__x == __y);
+}
+
+_LIBCPP_EXPORTED_FROM_ABI void swap(exception_ptr&, exception_ptr&) _NOEXCEPT;
+
+_LIBCPP_EXPORTED_FROM_ABI exception_ptr __copy_exception_ptr(void* __except, const void* __ptr);
+_LIBCPP_EXPORTED_FROM_ABI exception_ptr current_exception() _NOEXCEPT;
+_LIBCPP_NORETURN _LIBCPP_EXPORTED_FROM_ABI void rethrow_exception(exception_ptr);
+
+// This is a built-in template function which automagically extracts the required
+// information.
+template <class _E>
+void* __GetExceptionInfo(_E);
+
+template <class _Ep>
+_LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT {
+  return __copy_exception_ptr(std::addressof(__e), __GetExceptionInfo(__e));
+}
+
+#endif // _LIBCPP_ABI_MICROSOFT
+} // namespace std
+
+#endif // _LIBCPP___EXCEPTION_EXCEPTION_PTR_H
diff --git a/libcxx/include/__cxx03/__exception/nested_exception.h b/libcxx/include/__cxx03/__exception/nested_exception.h
new file mode 100644
index 000000000000..feb489f87f62
--- /dev/null
+++ b/libcxx/include/__cxx03/__exception/nested_exception.h
@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___EXCEPTION_NESTED_EXCEPTION_H
+#define _LIBCPP___EXCEPTION_NESTED_EXCEPTION_H
+
+#include <__config>
+#include <__exception/exception_ptr.h>
+#include <__memory/addressof.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_base_of.h>
+#include <__type_traits/is_class.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_final.h>
+#include <__type_traits/is_polymorphic.h>
+#include <__utility/forward.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+namespace std { // purposefully not using versioning namespace
+
+class _LIBCPP_EXPORTED_FROM_ABI nested_exception {
+  exception_ptr __ptr_;
+
+public:
+  nested_exception() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI nested_exception(const nested_exception&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI nested_exception& operator=(const nested_exception&) _NOEXCEPT = default;
+  virtual ~nested_exception() _NOEXCEPT;
+
+  // access functions
+  _LIBCPP_NORETURN void rethrow_nested() const;
+  _LIBCPP_HIDE_FROM_ABI exception_ptr nested_ptr() const _NOEXCEPT { return __ptr_; }
+};
+
+template <class _Tp>
+struct __nested : public _Tp, public nested_exception {
+  _LIBCPP_HIDE_FROM_ABI explicit __nested(const _Tp& __t) : _Tp(__t) {}
+};
+
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+template <class _Tp, class _Up, bool>
+struct __throw_with_nested;
+
+template <class _Tp, class _Up>
+struct __throw_with_nested<_Tp, _Up, true> {
+  _LIBCPP_NORETURN static inline _LIBCPP_HIDE_FROM_ABI void __do_throw(_Tp&& __t) {
+    throw __nested<_Up>(std::forward<_Tp>(__t));
+  }
+};
+
+template <class _Tp, class _Up>
+struct __throw_with_nested<_Tp, _Up, false> {
+  _LIBCPP_NORETURN static inline _LIBCPP_HIDE_FROM_ABI void __do_throw(_Tp&& __t) { throw std::forward<_Tp>(__t); }
+};
+#endif
+
+template <class _Tp>
+_LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI void throw_with_nested(_Tp&& __t) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  using _Up = __decay_t<_Tp>;
+  static_assert(is_copy_constructible<_Up>::value, "type thrown must be CopyConstructible");
+  __throw_with_nested<_Tp,
+                      _Up,
+                      is_class<_Up>::value && !is_base_of<nested_exception, _Up>::value &&
+                          !__libcpp_is_final<_Up>::value>::__do_throw(std::forward<_Tp>(__t));
+#else
+  ((void)__t);
+  // FIXME: Make this abort
+#endif
+}
+
+template <class _From, class _To>
+struct __can_dynamic_cast
+    : _BoolConstant< is_polymorphic<_From>::value &&
+                     (!is_base_of<_To, _From>::value || is_convertible<const _From*, const _To*>::value)> {};
+
+template <class _Ep, __enable_if_t< __can_dynamic_cast<_Ep, nested_exception>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI void rethrow_if_nested(const _Ep& __e) {
+  const nested_exception* __nep = dynamic_cast<const nested_exception*>(std::addressof(__e));
+  if (__nep)
+    __nep->rethrow_nested();
+}
+
+template <class _Ep, __enable_if_t<!__can_dynamic_cast<_Ep, nested_exception>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI void rethrow_if_nested(const _Ep&) {}
+
+} // namespace std
+
+#endif // _LIBCPP___EXCEPTION_NESTED_EXCEPTION_H
diff --git a/libcxx/include/__cxx03/__exception/operations.h b/libcxx/include/__cxx03/__exception/operations.h
new file mode 100644
index 000000000000..0a9c7a7c7f0d
--- /dev/null
+++ b/libcxx/include/__cxx03/__exception/operations.h
@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___EXCEPTION_OPERATIONS_H
+#define _LIBCPP___EXCEPTION_OPERATIONS_H
+
+#include <__config>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+namespace std { // purposefully not using versioning namespace
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_UNEXPECTED_FUNCTIONS) ||                             \
+    defined(_LIBCPP_BUILDING_LIBRARY)
+using unexpected_handler = void (*)();
+_LIBCPP_EXPORTED_FROM_ABI unexpected_handler set_unexpected(unexpected_handler) _NOEXCEPT;
+_LIBCPP_EXPORTED_FROM_ABI unexpected_handler get_unexpected() _NOEXCEPT;
+_LIBCPP_NORETURN _LIBCPP_EXPORTED_FROM_ABI void unexpected();
+#endif
+
+using terminate_handler = void (*)();
+_LIBCPP_EXPORTED_FROM_ABI terminate_handler set_terminate(terminate_handler) _NOEXCEPT;
+_LIBCPP_EXPORTED_FROM_ABI terminate_handler get_terminate() _NOEXCEPT;
+
+_LIBCPP_EXPORTED_FROM_ABI bool uncaught_exception() _NOEXCEPT;
+_LIBCPP_EXPORTED_FROM_ABI int uncaught_exceptions() _NOEXCEPT;
+
+class _LIBCPP_EXPORTED_FROM_ABI exception_ptr;
+
+_LIBCPP_EXPORTED_FROM_ABI exception_ptr current_exception() _NOEXCEPT;
+_LIBCPP_NORETURN _LIBCPP_EXPORTED_FROM_ABI void rethrow_exception(exception_ptr);
+} // namespace std
+
+#endif // _LIBCPP___EXCEPTION_OPERATIONS_H
diff --git a/libcxx/include/__cxx03/__exception/terminate.h b/libcxx/include/__cxx03/__exception/terminate.h
new file mode 100644
index 000000000000..e672471dc526
--- /dev/null
+++ b/libcxx/include/__cxx03/__exception/terminate.h
@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___EXCEPTION_TERMINATE_H
+#define _LIBCPP___EXCEPTION_TERMINATE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+namespace std { // purposefully not using versioning namespace
+_LIBCPP_NORETURN _LIBCPP_EXPORTED_FROM_ABI void terminate() _NOEXCEPT;
+} // namespace std
+
+#endif // _LIBCPP___EXCEPTION_TERMINATE_H
diff --git a/libcxx/include/__cxx03/__expected/bad_expected_access.h b/libcxx/include/__cxx03/__expected/bad_expected_access.h
new file mode 100644
index 000000000000..1b734389e831
--- /dev/null
+++ b/libcxx/include/__cxx03/__expected/bad_expected_access.h
@@ -0,0 +1,73 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef _LIBCPP___EXPECTED_BAD_EXPECTED_ACCESS_H
+#define _LIBCPP___EXPECTED_BAD_EXPECTED_ACCESS_H
+
+#include <__config>
+#include <__exception/exception.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Err>
+class bad_expected_access;
+
+_LIBCPP_DIAGNOSTIC_PUSH
+#  if !_LIBCPP_AVAILABILITY_HAS_BAD_EXPECTED_ACCESS_KEY_FUNCTION
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wweak-vtables")
+#  endif
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI bad_expected_access<void> : public exception {
+protected:
+  _LIBCPP_HIDE_FROM_ABI bad_expected_access() noexcept                                      = default;
+  _LIBCPP_HIDE_FROM_ABI bad_expected_access(const bad_expected_access&) noexcept            = default;
+  _LIBCPP_HIDE_FROM_ABI bad_expected_access(bad_expected_access&&) noexcept                 = default;
+  _LIBCPP_HIDE_FROM_ABI bad_expected_access& operator=(const bad_expected_access&) noexcept = default;
+  _LIBCPP_HIDE_FROM_ABI bad_expected_access& operator=(bad_expected_access&&) noexcept      = default;
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~bad_expected_access() override                             = default;
+
+public:
+#  if _LIBCPP_AVAILABILITY_HAS_BAD_EXPECTED_ACCESS_KEY_FUNCTION
+  const char* what() const noexcept override;
+#  else
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const char* what() const noexcept override { return "bad access to std::expected"; }
+#  endif
+};
+_LIBCPP_DIAGNOSTIC_POP
+
+template <class _Err>
+class bad_expected_access : public bad_expected_access<void> {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit bad_expected_access(_Err __e) : __unex_(std::move(__e)) {}
+
+  _LIBCPP_HIDE_FROM_ABI _Err& error() & noexcept { return __unex_; }
+  _LIBCPP_HIDE_FROM_ABI const _Err& error() const& noexcept { return __unex_; }
+  _LIBCPP_HIDE_FROM_ABI _Err&& error() && noexcept { return std::move(__unex_); }
+  _LIBCPP_HIDE_FROM_ABI const _Err&& error() const&& noexcept { return std::move(__unex_); }
+
+private:
+  _Err __unex_;
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___EXPECTED_BAD_EXPECTED_ACCESS_H
diff --git a/libcxx/include/__cxx03/__expected/expected.h b/libcxx/include/__cxx03/__expected/expected.h
new file mode 100644
index 000000000000..7a6f04c50dab
--- /dev/null
+++ b/libcxx/include/__cxx03/__expected/expected.h
@@ -0,0 +1,1874 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef _LIBCPP___EXPECTED_EXPECTED_H
+#define _LIBCPP___EXPECTED_EXPECTED_H
+
+#include <__assert>
+#include <__config>
+#include <__expected/bad_expected_access.h>
+#include <__expected/unexpect.h>
+#include <__expected/unexpected.h>
+#include <__functional/invoke.h>
+#include <__memory/addressof.h>
+#include <__memory/construct_at.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/disjunction.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_function.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_swappable.h>
+#include <__type_traits/is_trivially_constructible.h>
+#include <__type_traits/is_trivially_destructible.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__type_traits/is_void.h>
+#include <__type_traits/lazy.h>
+#include <__type_traits/negation.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/as_const.h>
+#include <__utility/exception_guard.h>
+#include <__utility/forward.h>
+#include <__utility/in_place.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+#include <__verbose_abort>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Err>
+class expected;
+
+template <class _Tp>
+struct __is_std_expected : false_type {};
+
+template <class _Tp, class _Err>
+struct __is_std_expected<expected<_Tp, _Err>> : true_type {};
+
+struct __expected_construct_in_place_from_invoke_tag {};
+struct __expected_construct_unexpected_from_invoke_tag {};
+
+template <class _Err, class _Arg>
+_LIBCPP_HIDE_FROM_ABI void __throw_bad_expected_access(_Arg&& __arg) {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw bad_expected_access<_Err>(std::forward<_Arg>(__arg));
+#  else
+  (void)__arg;
+  _LIBCPP_VERBOSE_ABORT("bad_expected_access was thrown in -fno-exceptions mode");
+#  endif
+}
+
+// If parameter type `_Tp` of `__conditional_no_unique_address` is neither
+// copyable nor movable, a constructor with this tag is provided. For that
+// constructor, the user has to provide a function and arguments. The function
+// must return an object of type `_Tp`. When the function is invoked by the
+// constructor, guaranteed copy elision kicks in and the `_Tp` is constructed
+// in place.
+struct __conditional_no_unique_address_invoke_tag {};
+
+// This class implements an object with `[[no_unique_address]]` conditionally applied to it,
+// based on the value of `_NoUnique`.
+//
+// A member of this class must always have `[[no_unique_address]]` applied to
+// it. Otherwise, the `[[no_unique_address]]` in the "`_NoUnique == true`" case
+// would not have any effect. In the `false` case, the `__v` is not
+// `[[no_unique_address]]`, so nullifies the effects of the "outer"
+// `[[no_unique_address]]` regarding data layout.
+//
+// If we had a language feature, this class would basically be replaced by `[[no_unique_address(condition)]]`.
+template <bool _NoUnique, class _Tp>
+struct __conditional_no_unique_address;
+
+template <class _Tp>
+struct __conditional_no_unique_address<true, _Tp> {
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __conditional_no_unique_address(in_place_t, _Args&&... __args)
+      : __v(std::forward<_Args>(__args)...) {}
+
+  template <class _Func, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __conditional_no_unique_address(
+      __conditional_no_unique_address_invoke_tag, _Func&& __f, _Args&&... __args)
+      : __v(std::invoke(std::forward<_Func>(__f), std::forward<_Args>(__args)...)) {}
+
+  _LIBCPP_NO_UNIQUE_ADDRESS _Tp __v;
+};
+
+template <class _Tp>
+struct __conditional_no_unique_address<false, _Tp> {
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __conditional_no_unique_address(in_place_t, _Args&&... __args)
+      : __v(std::forward<_Args>(__args)...) {}
+
+  template <class _Func, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __conditional_no_unique_address(
+      __conditional_no_unique_address_invoke_tag, _Func&& __f, _Args&&... __args)
+      : __v(std::invoke(std::forward<_Func>(__f), std::forward<_Args>(__args)...)) {}
+
+  _Tp __v;
+};
+
+// This function returns whether the type `_Second` can be stuffed into the tail padding
+// of the `_First` type if both of them are given `[[no_unique_address]]`.
+template <class _First, class _Second>
+inline constexpr bool __fits_in_tail_padding = []() {
+  struct __x {
+    _LIBCPP_NO_UNIQUE_ADDRESS _First __first;
+    _LIBCPP_NO_UNIQUE_ADDRESS _Second __second;
+  };
+  return sizeof(__x) == sizeof(_First);
+}();
+
+// This class implements the storage used by `std::expected`. We have a few
+// goals for this storage:
+// 1. Whenever the underlying {_Tp | _Unex} combination has free bytes in its
+//    tail padding, we should reuse it to store the bool discriminator of the
+//    expected, so as to save space.
+// 2. Whenever the `expected<_Tp, _Unex>` as a whole has free bytes in its tail
+//    padding, we should allow an object following the expected to be stored in
+//    its tail padding.
+// 3. However, we never want a user object (say `X`) that would follow an
+//    `expected<_Tp, _Unex>` to be stored in the padding bytes of the
+//    underlying {_Tp | _Unex} union, if any. That is because we use
+//    `construct_at` on that union, which would end up overwriting the `X`
+//    member if it is stored in the tail padding of the union.
+//
+// To achieve this, `__expected_base`'s logic is implemented in an inner
+// `__repr` class. `__expected_base` holds one `__repr` member which is
+// conditionally `[[no_unique_address]]`. The `__repr` class holds the
+// underlying {_Tp | _Unex} union and a boolean "has value" flag.
+//
+// Which one of the `__repr_`/`__union_` members is `[[no_unique_address]]`
+// depends on whether the "has value" boolean fits into the tail padding of
+// the underlying {_Tp | _Unex} union:
+//
+// - In case the "has value" bool fits into the tail padding of the union, the
+//   whole `__repr_` member is _not_ `[[no_unique_address]]` as it needs to be
+//   transparently replaced on `emplace()`/`swap()` etc.
+// - In case the "has value" bool does not fit into the tail padding of the
+//   union, only the union member must be transparently replaced (therefore is
+//   _not_ `[[no_unique_address]]`) and the "has value" flag must be adjusted
+//   manually.
+//
+// This way, the member that is transparently replaced on mutating operations
+// is never `[[no_unique_address]]`, satisfying the requirements from
+// "[basic.life]" in the standard.
+//
+// Stripped away of all superfluous elements, the layout of `__expected_base`
+// then looks like this:
+//
+//     template <class Tp, class Err>
+//     class expected_base {
+//       union union_t {
+//         [[no_unique_address]] Tp val;
+//         [[no_unique_address]] Err unex;
+//       };
+//
+//       static constexpr bool put_flag_in_tail                    = fits_in_tail_padding<union_t, bool>;
+//       static constexpr bool allow_reusing_expected_tail_padding = !put_flag_in_tail;
+//
+//       struct repr {
+//       private:
+//         // If "has value" fits into the tail, this should be
+//         // `[[no_unique_address]]`, otherwise not.
+//         [[no_unique_address]] conditional_no_unique_address<
+//             put_flag_in_tail,
+//             union_t>::type union_;
+//         [[no_unique_address]] bool has_val_;
+//       };
+//
+//     protected:
+//       // If "has value" fits into the tail, this must _not_ be
+//       // `[[no_unique_address]]` so that we fill out the
+//       // complete `expected` object.
+//       [[no_unique_address]] conditional_no_unique_address<
+//           allow_reusing_expected_tail_padding,
+//           repr>::type repr_;
+//     };
+//
+template <class _Tp, class _Err>
+class __expected_base {
+  // use named union because [[no_unique_address]] cannot be applied to an unnamed union,
+  // also guaranteed elision into a potentially-overlapping subobject is unsettled (and
+  // it's not clear that it's implementable, given that the function is allowed to clobber
+  // the tail padding) - see https://github.com/itanium-cxx-abi/cxx-abi/issues/107.
+  union __union_t {
+    _LIBCPP_HIDE_FROM_ABI constexpr __union_t(const __union_t&) = delete;
+    _LIBCPP_HIDE_FROM_ABI constexpr __union_t(const __union_t&)
+      requires(is_copy_constructible_v<_Tp> && is_copy_constructible_v<_Err> &&
+               is_trivially_copy_constructible_v<_Tp> && is_trivially_copy_constructible_v<_Err>)
+    = default;
+    _LIBCPP_HIDE_FROM_ABI constexpr __union_t(__union_t&&) = delete;
+    _LIBCPP_HIDE_FROM_ABI constexpr __union_t(__union_t&&)
+      requires(is_move_constructible_v<_Tp> && is_move_constructible_v<_Err> &&
+               is_trivially_move_constructible_v<_Tp> && is_trivially_move_constructible_v<_Err>)
+    = default;
+    _LIBCPP_HIDE_FROM_ABI constexpr __union_t& operator=(const __union_t&) = delete;
+    _LIBCPP_HIDE_FROM_ABI constexpr __union_t& operator=(__union_t&&)      = delete;
+
+    template <class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(in_place_t, _Args&&... __args)
+        : __val_(std::forward<_Args>(__args)...) {}
+
+    template <class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(unexpect_t, _Args&&... __args)
+        : __unex_(std::forward<_Args>(__args)...) {}
+
+    template <class _Func, class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(
+        std::__expected_construct_in_place_from_invoke_tag, _Func&& __f, _Args&&... __args)
+        : __val_(std::invoke(std::forward<_Func>(__f), std::forward<_Args>(__args)...)) {}
+
+    template <class _Func, class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(
+        std::__expected_construct_unexpected_from_invoke_tag, _Func&& __f, _Args&&... __args)
+        : __unex_(std::invoke(std::forward<_Func>(__f), std::forward<_Args>(__args)...)) {}
+
+    _LIBCPP_HIDE_FROM_ABI constexpr ~__union_t()
+      requires(is_trivially_destructible_v<_Tp> && is_trivially_destructible_v<_Err>)
+    = default;
+
+    // __repr's destructor handles this
+    _LIBCPP_HIDE_FROM_ABI constexpr ~__union_t() {}
+
+    _LIBCPP_NO_UNIQUE_ADDRESS _Tp __val_;
+    _LIBCPP_NO_UNIQUE_ADDRESS _Err __unex_;
+  };
+
+  static constexpr bool __put_flag_in_tail                    = __fits_in_tail_padding<__union_t, bool>;
+  static constexpr bool __allow_reusing_expected_tail_padding = !__put_flag_in_tail;
+
+  struct __repr {
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __repr() = delete;
+
+    template <class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __repr(in_place_t __tag, _Args&&... __args)
+        : __union_(in_place, __tag, std::forward<_Args>(__args)...), __has_val_(true) {}
+
+    template <class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __repr(unexpect_t __tag, _Args&&... __args)
+        : __union_(in_place, __tag, std::forward<_Args>(__args)...), __has_val_(false) {}
+
+    template <class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __repr(std::__expected_construct_in_place_from_invoke_tag __tag,
+                                                    _Args&&... __args)
+        : __union_(in_place, __tag, std::forward<_Args>(__args)...), __has_val_(true) {}
+
+    template <class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __repr(std::__expected_construct_unexpected_from_invoke_tag __tag,
+                                                    _Args&&... __args)
+        : __union_(in_place, __tag, std::forward<_Args>(__args)...), __has_val_(false) {}
+
+    // The return value of `__make_union` must be constructed in place in the
+    // `__v` member of `__union_`, relying on guaranteed copy elision. To do
+    // this, the `__conditional_no_unique_address_invoke_tag` constructor is
+    // called with a lambda that is immediately called inside
+    // `__conditional_no_unique_address`'s constructor.
+    template <class _OtherUnion>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __repr(bool __has_val, _OtherUnion&& __other)
+      requires(__allow_reusing_expected_tail_padding)
+        : __union_(__conditional_no_unique_address_invoke_tag{},
+                   [&] { return __make_union(__has_val, std::forward<_OtherUnion>(__other)); }),
+          __has_val_(__has_val) {}
+
+    _LIBCPP_HIDE_FROM_ABI constexpr __repr(const __repr&) = delete;
+    _LIBCPP_HIDE_FROM_ABI constexpr __repr(const __repr&)
+      requires(is_copy_constructible_v<_Tp> && is_copy_constructible_v<_Err> &&
+               is_trivially_copy_constructible_v<_Tp> && is_trivially_copy_constructible_v<_Err>)
+    = default;
+    _LIBCPP_HIDE_FROM_ABI constexpr __repr(__repr&&) = delete;
+    _LIBCPP_HIDE_FROM_ABI constexpr __repr(__repr&&)
+      requires(is_move_constructible_v<_Tp> && is_move_constructible_v<_Err> &&
+               is_trivially_move_constructible_v<_Tp> && is_trivially_move_constructible_v<_Err>)
+    = default;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr __repr& operator=(const __repr&) = delete;
+    _LIBCPP_HIDE_FROM_ABI constexpr __repr& operator=(__repr&&)      = delete;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr ~__repr()
+      requires(is_trivially_destructible_v<_Tp> && is_trivially_destructible_v<_Err>)
+    = default;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr ~__repr()
+      requires(!is_trivially_destructible_v<_Tp> || !is_trivially_destructible_v<_Err>)
+    {
+      __destroy_union_member();
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr void __destroy_union()
+      requires(__allow_reusing_expected_tail_padding &&
+               (is_trivially_destructible_v<_Tp> && is_trivially_destructible_v<_Err>))
+    {
+      // Note: Since the destructor of the union is trivial, this does nothing
+      // except to end the lifetime of the union.
+      std::destroy_at(&__union_.__v);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr void __destroy_union()
+      requires(__allow_reusing_expected_tail_padding &&
+               (!is_trivially_destructible_v<_Tp> || !is_trivially_destructible_v<_Err>))
+    {
+      __destroy_union_member();
+      std::destroy_at(&__union_.__v);
+    }
+
+    template <class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr void __construct_union(in_place_t, _Args&&... __args)
+      requires(__allow_reusing_expected_tail_padding)
+    {
+      std::construct_at(&__union_.__v, in_place, std::forward<_Args>(__args)...);
+      __has_val_ = true;
+    }
+
+    template <class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr void __construct_union(unexpect_t, _Args&&... __args)
+      requires(__allow_reusing_expected_tail_padding)
+    {
+      std::construct_at(&__union_.__v, unexpect, std::forward<_Args>(__args)...);
+      __has_val_ = false;
+    }
+
+  private:
+    template <class, class>
+    friend class __expected_base;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr void __destroy_union_member()
+      requires(!is_trivially_destructible_v<_Tp> || !is_trivially_destructible_v<_Err>)
+    {
+      if (__has_val_) {
+        std::destroy_at(std::addressof(__union_.__v.__val_));
+      } else {
+        std::destroy_at(std::addressof(__union_.__v.__unex_));
+      }
+    }
+
+    template <class _OtherUnion>
+    _LIBCPP_HIDE_FROM_ABI static constexpr __union_t __make_union(bool __has_val, _OtherUnion&& __other)
+      requires(__allow_reusing_expected_tail_padding)
+    {
+      if (__has_val)
+        return __union_t(in_place, std::forward<_OtherUnion>(__other).__val_);
+      else
+        return __union_t(unexpect, std::forward<_OtherUnion>(__other).__unex_);
+    }
+
+    _LIBCPP_NO_UNIQUE_ADDRESS __conditional_no_unique_address<__put_flag_in_tail, __union_t> __union_;
+    _LIBCPP_NO_UNIQUE_ADDRESS bool __has_val_;
+  };
+
+  template <class _OtherUnion>
+  _LIBCPP_HIDE_FROM_ABI static constexpr __repr __make_repr(bool __has_val, _OtherUnion&& __other)
+    requires(__put_flag_in_tail)
+  {
+    if (__has_val)
+      return __repr(in_place, std::forward<_OtherUnion>(__other).__val_);
+    else
+      return __repr(unexpect, std::forward<_OtherUnion>(__other).__unex_);
+  }
+
+protected:
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __expected_base(_Args&&... __args)
+      : __repr_(in_place, std::forward<_Args>(__args)...) {}
+
+  // In case we copy/move construct from another `expected` we need to create
+  // our `expected` so that it either has a value or not, depending on the "has
+  // value" flag of the other `expected`. To do this without falling back on
+  // `std::construct_at` we rely on guaranteed copy elision using two helper
+  // functions `__make_repr` and `__make_union`. There have to be two since
+  // there are two data layouts with different members being
+  // `[[no_unique_address]]`. GCC (as of version 13) does not do guaranteed
+  // copy elision when initializing `[[no_unique_address]]` members. The two
+  // cases are:
+  //
+  // - `__make_repr`: This is used when the "has value" flag lives in the tail
+  //   of the union. In this case, the `__repr` member is _not_
+  //   `[[no_unique_address]]`.
+  // - `__make_union`: When the "has value" flag does _not_ fit in the tail of
+  //   the union, the `__repr` member is `[[no_unique_address]]` and the union
+  //   is not.
+  //
+  // This constructor "catches" the first case and leaves the second case to
+  // `__union_t`, its constructors and `__make_union`.
+  template <class _OtherUnion>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __expected_base(bool __has_val, _OtherUnion&& __other)
+    requires(__put_flag_in_tail)
+      : __repr_(__conditional_no_unique_address_invoke_tag{},
+                [&] { return __make_repr(__has_val, std::forward<_OtherUnion>(__other)); }) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void __destroy() {
+    if constexpr (__put_flag_in_tail)
+      std::destroy_at(&__repr_.__v);
+    else
+      __repr_.__v.__destroy_union();
+  }
+
+  template <class _Tag, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr void __construct(_Tag __tag, _Args&&... __args) {
+    if constexpr (__put_flag_in_tail)
+      std::construct_at(&__repr_.__v, __tag, std::forward<_Args>(__args)...);
+    else
+      __repr_.__v.__construct_union(__tag, std::forward<_Args>(__args)...);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __has_val() const { return __repr_.__v.__has_val_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr __union_t& __union() { return __repr_.__v.__union_.__v; }
+  _LIBCPP_HIDE_FROM_ABI constexpr const __union_t& __union() const { return __repr_.__v.__union_.__v; }
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp& __val() { return __repr_.__v.__union_.__v.__val_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& __val() const { return __repr_.__v.__union_.__v.__val_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr _Err& __unex() { return __repr_.__v.__union_.__v.__unex_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Err& __unex() const { return __repr_.__v.__union_.__v.__unex_; }
+
+private:
+  _LIBCPP_NO_UNIQUE_ADDRESS __conditional_no_unique_address<__allow_reusing_expected_tail_padding, __repr> __repr_;
+};
+
+template <class _Tp, class _Err>
+class expected : private __expected_base<_Tp, _Err> {
+  static_assert(!is_reference_v<_Tp> && !is_function_v<_Tp> && !is_same_v<remove_cv_t<_Tp>, in_place_t> &&
+                    !is_same_v<remove_cv_t<_Tp>, unexpect_t> && !__is_std_unexpected<remove_cv_t<_Tp>>::value &&
+                    __valid_std_unexpected<_Err>::value,
+                "[expected.object.general] A program that instantiates the definition of template expected<T, E> for a "
+                "reference type, a function type, or for possibly cv-qualified types in_place_t, unexpect_t, or a "
+                "specialization of unexpected for the T parameter is ill-formed. A program that instantiates the "
+                "definition of the template expected<T, E> with a type for the E parameter that is not a valid "
+                "template argument for unexpected is ill-formed.");
+
+  template <class _Up, class _OtherErr>
+  friend class expected;
+
+  using __base = __expected_base<_Tp, _Err>;
+
+public:
+  using value_type      = _Tp;
+  using error_type      = _Err;
+  using unexpected_type = unexpected<_Err>;
+
+  using __trivially_relocatable =
+      __conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value && __libcpp_is_trivially_relocatable<_Err>::value,
+                      expected,
+                      void>;
+
+  template <class _Up>
+  using rebind = expected<_Up, error_type>;
+
+  // [expected.object.ctor], constructors
+  _LIBCPP_HIDE_FROM_ABI constexpr expected() noexcept(is_nothrow_default_constructible_v<_Tp>) // strengthened
+    requires is_default_constructible_v<_Tp>
+      : __base(in_place) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr expected(const expected&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr expected(const expected&)
+    requires(is_copy_constructible_v<_Tp> && is_copy_constructible_v<_Err> && is_trivially_copy_constructible_v<_Tp> &&
+             is_trivially_copy_constructible_v<_Err>)
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr expected(const expected& __other) noexcept(
+      is_nothrow_copy_constructible_v<_Tp> && is_nothrow_copy_constructible_v<_Err>) // strengthened
+    requires(is_copy_constructible_v<_Tp> && is_copy_constructible_v<_Err> &&
+             !(is_trivially_copy_constructible_v<_Tp> && is_trivially_copy_constructible_v<_Err>))
+      : __base(__other.__has_val(), __other.__union()) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr expected(expected&&)
+    requires(is_move_constructible_v<_Tp> && is_move_constructible_v<_Err> && is_trivially_move_constructible_v<_Tp> &&
+             is_trivially_move_constructible_v<_Err>)
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr expected(expected&& __other) noexcept(
+      is_nothrow_move_constructible_v<_Tp> && is_nothrow_move_constructible_v<_Err>)
+    requires(is_move_constructible_v<_Tp> && is_move_constructible_v<_Err> &&
+             !(is_trivially_move_constructible_v<_Tp> && is_trivially_move_constructible_v<_Err>))
+      : __base(__other.__has_val(), std::move(__other.__union())) {}
+
+private:
+  template <class _Up, class _OtherErr, class _UfQual, class _OtherErrQual>
+  using __can_convert = _And<
+      is_constructible<_Tp, _UfQual>,
+      is_constructible<_Err, _OtherErrQual>,
+      _If<_Not<is_same<remove_cv_t<_Tp>, bool>>::value,
+          _And< _Not<_And<is_same<_Tp, _Up>, is_same<_Err, _OtherErr>>>, // use the copy constructor instead, see #92676
+                _Not<is_constructible<_Tp, expected<_Up, _OtherErr>&>>,
+                _Not<is_constructible<_Tp, expected<_Up, _OtherErr>>>,
+                _Not<is_constructible<_Tp, const expected<_Up, _OtherErr>&>>,
+                _Not<is_constructible<_Tp, const expected<_Up, _OtherErr>>>,
+                _Not<is_convertible<expected<_Up, _OtherErr>&, _Tp>>,
+                _Not<is_convertible<expected<_Up, _OtherErr>&&, _Tp>>,
+                _Not<is_convertible<const expected<_Up, _OtherErr>&, _Tp>>,
+                _Not<is_convertible<const expected<_Up, _OtherErr>&&, _Tp>>>,
+          true_type>,
+      _Not<is_constructible<unexpected<_Err>, expected<_Up, _OtherErr>&>>,
+      _Not<is_constructible<unexpected<_Err>, expected<_Up, _OtherErr>>>,
+      _Not<is_constructible<unexpected<_Err>, const expected<_Up, _OtherErr>&>>,
+      _Not<is_constructible<unexpected<_Err>, const expected<_Up, _OtherErr>>> >;
+
+  template <class _Func, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(
+      std::__expected_construct_in_place_from_invoke_tag __tag, _Func&& __f, _Args&&... __args)
+      : __base(__tag, std::forward<_Func>(__f), std::forward<_Args>(__args)...) {}
+
+  template <class _Func, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(
+      std::__expected_construct_unexpected_from_invoke_tag __tag, _Func&& __f, _Args&&... __args)
+      : __base(__tag, std::forward<_Func>(__f), std::forward<_Args>(__args)...) {}
+
+public:
+  template <class _Up, class _OtherErr>
+    requires __can_convert<_Up, _OtherErr, const _Up&, const _OtherErr&>::value
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<const _Up&, _Tp> ||
+                                           !is_convertible_v<const _OtherErr&, _Err>)
+      expected(const expected<_Up, _OtherErr>& __other) noexcept(
+          is_nothrow_constructible_v<_Tp, const _Up&> &&
+          is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened
+      : __base(__other.__has_val(), __other.__union()) {}
+
+  template <class _Up, class _OtherErr>
+    requires __can_convert<_Up, _OtherErr, _Up, _OtherErr>::value
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_Up, _Tp> || !is_convertible_v<_OtherErr, _Err>)
+      expected(expected<_Up, _OtherErr>&& __other) noexcept(
+          is_nothrow_constructible_v<_Tp, _Up> && is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened
+      : __base(__other.__has_val(), std::move(__other.__union())) {}
+
+  template <class _Up = _Tp>
+    requires(!is_same_v<remove_cvref_t<_Up>, in_place_t> && !is_same_v<expected, remove_cvref_t<_Up>> &&
+             is_constructible_v<_Tp, _Up> && !__is_std_unexpected<remove_cvref_t<_Up>>::value &&
+             (!is_same_v<remove_cv_t<_Tp>, bool> || !__is_std_expected<remove_cvref_t<_Up>>::value))
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_Up, _Tp>)
+      expected(_Up&& __u) noexcept(is_nothrow_constructible_v<_Tp, _Up>) // strengthened
+      : __base(in_place, std::forward<_Up>(__u)) {}
+
+  template <class _OtherErr>
+    requires is_constructible_v<_Err, const _OtherErr&>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<const _OtherErr&, _Err>) expected(
+      const unexpected<_OtherErr>& __unex) noexcept(is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened
+      : __base(unexpect, __unex.error()) {}
+
+  template <class _OtherErr>
+    requires is_constructible_v<_Err, _OtherErr>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_OtherErr, _Err>)
+      expected(unexpected<_OtherErr>&& __unex) noexcept(is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened
+      : __base(unexpect, std::move(__unex.error())) {}
+
+  template <class... _Args>
+    requires is_constructible_v<_Tp, _Args...>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(in_place_t, _Args&&... __args) noexcept(
+      is_nothrow_constructible_v<_Tp, _Args...>) // strengthened
+      : __base(in_place, std::forward<_Args>(__args)...) {}
+
+  template <class _Up, class... _Args>
+    requires is_constructible_v< _Tp, initializer_list<_Up>&, _Args... >
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(in_place_t, initializer_list<_Up> __il, _Args&&... __args) noexcept(
+      is_nothrow_constructible_v<_Tp, initializer_list<_Up>&, _Args...>) // strengthened
+      : __base(in_place, __il, std::forward<_Args>(__args)...) {}
+
+  template <class... _Args>
+    requires is_constructible_v<_Err, _Args...>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(unexpect_t, _Args&&... __args) noexcept(
+      is_nothrow_constructible_v<_Err, _Args...>) // strengthened
+      : __base(unexpect, std::forward<_Args>(__args)...) {}
+
+  template <class _Up, class... _Args>
+    requires is_constructible_v< _Err, initializer_list<_Up>&, _Args... >
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(unexpect_t, initializer_list<_Up> __il, _Args&&... __args) noexcept(
+      is_nothrow_constructible_v<_Err, initializer_list<_Up>&, _Args...>) // strengthened
+      : __base(unexpect, __il, std::forward<_Args>(__args)...) {}
+
+  // [expected.object.dtor], destructor
+
+  _LIBCPP_HIDE_FROM_ABI constexpr ~expected() = default;
+
+private:
+  template <class _Tag, class _OtherTag, class _T1, class _T2, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr void __reinit_expected(_T2& __oldval, _Args&&... __args) {
+    if constexpr (is_nothrow_constructible_v<_T1, _Args...>) {
+      this->__destroy();
+      this->__construct(_Tag{}, std::forward<_Args>(__args)...);
+    } else if constexpr (is_nothrow_move_constructible_v<_T1>) {
+      _T1 __tmp(std::forward<_Args>(__args)...);
+      this->__destroy();
+      this->__construct(_Tag{}, std::move(__tmp));
+    } else {
+      static_assert(
+          is_nothrow_move_constructible_v<_T2>,
+          "To provide strong exception guarantee, T2 has to satisfy `is_nothrow_move_constructible_v` so that it can "
+          "be reverted to the previous state in case an exception is thrown during the assignment.");
+      _T2 __tmp(std::move(__oldval));
+      this->__destroy();
+      auto __trans = std::__make_exception_guard([&] { this->__construct(_OtherTag{}, std::move(__tmp)); });
+      this->__construct(_Tag{}, std::forward<_Args>(__args)...);
+      __trans.__complete();
+    }
+  }
+
+public:
+  // [expected.object.assign], assignment
+  _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(const expected&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(const expected& __rhs) noexcept(
+      is_nothrow_copy_assignable_v<_Tp> && is_nothrow_copy_constructible_v<_Tp> && is_nothrow_copy_assignable_v<_Err> &&
+      is_nothrow_copy_constructible_v<_Err>) // strengthened
+    requires(is_copy_assignable_v<_Tp> && is_copy_constructible_v<_Tp> && is_copy_assignable_v<_Err> &&
+             is_copy_constructible_v<_Err> &&
+             (is_nothrow_move_constructible_v<_Tp> || is_nothrow_move_constructible_v<_Err>))
+  {
+    if (this->__has_val() && __rhs.__has_val()) {
+      this->__val() = __rhs.__val();
+    } else if (this->__has_val()) {
+      __reinit_expected<unexpect_t, in_place_t, _Err, _Tp>(this->__val(), __rhs.__unex());
+    } else if (__rhs.__has_val()) {
+      __reinit_expected<in_place_t, unexpect_t, _Tp, _Err>(this->__unex(), __rhs.__val());
+    } else {
+      this->__unex() = __rhs.__unex();
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr expected&
+  operator=(expected&& __rhs) noexcept(is_nothrow_move_assignable_v<_Tp> && is_nothrow_move_constructible_v<_Tp> &&
+                                       is_nothrow_move_assignable_v<_Err> && is_nothrow_move_constructible_v<_Err>)
+    requires(is_move_constructible_v<_Tp> && is_move_assignable_v<_Tp> && is_move_constructible_v<_Err> &&
+             is_move_assignable_v<_Err> &&
+             (is_nothrow_move_constructible_v<_Tp> || is_nothrow_move_constructible_v<_Err>))
+  {
+    if (this->__has_val() && __rhs.__has_val()) {
+      this->__val() = std::move(__rhs.__val());
+    } else if (this->__has_val()) {
+      __reinit_expected<unexpect_t, in_place_t, _Err, _Tp>(this->__val(), std::move(__rhs.__unex()));
+    } else if (__rhs.__has_val()) {
+      __reinit_expected<in_place_t, unexpect_t, _Tp, _Err>(this->__unex(), std::move(__rhs.__val()));
+    } else {
+      this->__unex() = std::move(__rhs.__unex());
+    }
+    return *this;
+  }
+
+  template <class _Up = _Tp>
+  _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(_Up&& __v)
+    requires(!is_same_v<expected, remove_cvref_t<_Up>> && !__is_std_unexpected<remove_cvref_t<_Up>>::value &&
+             is_constructible_v<_Tp, _Up> && is_assignable_v<_Tp&, _Up> &&
+             (is_nothrow_constructible_v<_Tp, _Up> || is_nothrow_move_constructible_v<_Tp> ||
+              is_nothrow_move_constructible_v<_Err>))
+  {
+    if (this->__has_val()) {
+      this->__val() = std::forward<_Up>(__v);
+    } else {
+      __reinit_expected<in_place_t, unexpect_t, _Tp, _Err>(this->__unex(), std::forward<_Up>(__v));
+    }
+    return *this;
+  }
+
+private:
+  template <class _OtherErrQual>
+  static constexpr bool __can_assign_from_unexpected =
+      _And< is_constructible<_Err, _OtherErrQual>,
+            is_assignable<_Err&, _OtherErrQual>,
+            _Lazy<_Or,
+                  is_nothrow_constructible<_Err, _OtherErrQual>,
+                  is_nothrow_move_constructible<_Tp>,
+                  is_nothrow_move_constructible<_Err>> >::value;
+
+public:
+  template <class _OtherErr>
+    requires(__can_assign_from_unexpected<const _OtherErr&>)
+  _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(const unexpected<_OtherErr>& __un) {
+    if (this->__has_val()) {
+      __reinit_expected<unexpect_t, in_place_t, _Err, _Tp>(this->__val(), __un.error());
+    } else {
+      this->__unex() = __un.error();
+    }
+    return *this;
+  }
+
+  template <class _OtherErr>
+    requires(__can_assign_from_unexpected<_OtherErr>)
+  _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(unexpected<_OtherErr>&& __un) {
+    if (this->__has_val()) {
+      __reinit_expected<unexpect_t, in_place_t, _Err, _Tp>(this->__val(), std::move(__un.error()));
+    } else {
+      this->__unex() = std::move(__un.error());
+    }
+    return *this;
+  }
+
+  template <class... _Args>
+    requires is_nothrow_constructible_v<_Tp, _Args...>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp& emplace(_Args&&... __args) noexcept {
+    this->__destroy();
+    this->__construct(in_place, std::forward<_Args>(__args)...);
+    return this->__val();
+  }
+
+  template <class _Up, class... _Args>
+    requires is_nothrow_constructible_v<_Tp, initializer_list<_Up>&, _Args...>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp& emplace(initializer_list<_Up> __il, _Args&&... __args) noexcept {
+    this->__destroy();
+    this->__construct(in_place, __il, std::forward<_Args>(__args)...);
+    return this->__val();
+  }
+
+public:
+  // [expected.object.swap], swap
+  _LIBCPP_HIDE_FROM_ABI constexpr void
+  swap(expected& __rhs) noexcept(is_nothrow_move_constructible_v<_Tp> && is_nothrow_swappable_v<_Tp> &&
+                                 is_nothrow_move_constructible_v<_Err> && is_nothrow_swappable_v<_Err>)
+    requires(is_swappable_v<_Tp> && is_swappable_v<_Err> && is_move_constructible_v<_Tp> &&
+             is_move_constructible_v<_Err> &&
+             (is_nothrow_move_constructible_v<_Tp> || is_nothrow_move_constructible_v<_Err>))
+  {
+    auto __swap_val_unex_impl = [](expected& __with_val, expected& __with_err) {
+      if constexpr (is_nothrow_move_constructible_v<_Err>) {
+        _Err __tmp(std::move(__with_err.__unex()));
+        __with_err.__destroy();
+        auto __trans = std::__make_exception_guard([&] { __with_err.__construct(unexpect, std::move(__tmp)); });
+        __with_err.__construct(in_place, std::move(__with_val.__val()));
+        __trans.__complete();
+        __with_val.__destroy();
+        __with_val.__construct(unexpect, std::move(__tmp));
+      } else {
+        static_assert(is_nothrow_move_constructible_v<_Tp>,
+                      "To provide strong exception guarantee, Tp has to satisfy `is_nothrow_move_constructible_v` so "
+                      "that it can be reverted to the previous state in case an exception is thrown during swap.");
+        _Tp __tmp(std::move(__with_val.__val()));
+        __with_val.__destroy();
+        auto __trans = std::__make_exception_guard([&] { __with_val.__construct(in_place, std::move(__tmp)); });
+        __with_val.__construct(unexpect, std::move(__with_err.__unex()));
+        __trans.__complete();
+        __with_err.__destroy();
+        __with_err.__construct(in_place, std::move(__tmp));
+      }
+    };
+
+    if (this->__has_val()) {
+      if (__rhs.__has_val()) {
+        using std::swap;
+        swap(this->__val(), __rhs.__val());
+      } else {
+        __swap_val_unex_impl(*this, __rhs);
+      }
+    } else {
+      if (__rhs.__has_val()) {
+        __swap_val_unex_impl(__rhs, *this);
+      } else {
+        using std::swap;
+        swap(this->__unex(), __rhs.__unex());
+      }
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr void swap(expected& __x, expected& __y) noexcept(noexcept(__x.swap(__y)))
+    requires requires { __x.swap(__y); }
+  {
+    __x.swap(__y);
+  }
+
+  // [expected.object.obs], observers
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp* operator->() const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        this->__has_val(), "expected::operator-> requires the expected to contain a value");
+    return std::addressof(this->__val());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp* operator->() noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        this->__has_val(), "expected::operator-> requires the expected to contain a value");
+    return std::addressof(this->__val());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& operator*() const& noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        this->__has_val(), "expected::operator* requires the expected to contain a value");
+    return this->__val();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp& operator*() & noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        this->__has_val(), "expected::operator* requires the expected to contain a value");
+    return this->__val();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp&& operator*() const&& noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        this->__has_val(), "expected::operator* requires the expected to contain a value");
+    return std::move(this->__val());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp&& operator*() && noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        this->__has_val(), "expected::operator* requires the expected to contain a value");
+    return std::move(this->__val());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return this->__has_val(); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool has_value() const noexcept { return this->__has_val(); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& value() const& {
+    static_assert(is_copy_constructible_v<_Err>, "error_type has to be copy constructible");
+    if (!this->__has_val()) {
+      std::__throw_bad_expected_access<_Err>(std::as_const(error()));
+    }
+    return this->__val();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp& value() & {
+    static_assert(is_copy_constructible_v<_Err>, "error_type has to be copy constructible");
+    if (!this->__has_val()) {
+      std::__throw_bad_expected_access<_Err>(std::as_const(error()));
+    }
+    return this->__val();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp&& value() const&& {
+    static_assert(is_copy_constructible_v<_Err> && is_constructible_v<_Err, decltype(std::move(error()))>,
+                  "error_type has to be both copy constructible and constructible from decltype(std::move(error()))");
+    if (!this->__has_val()) {
+      std::__throw_bad_expected_access<_Err>(std::move(error()));
+    }
+    return std::move(this->__val());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp&& value() && {
+    static_assert(is_copy_constructible_v<_Err> && is_constructible_v<_Err, decltype(std::move(error()))>,
+                  "error_type has to be both copy constructible and constructible from decltype(std::move(error()))");
+    if (!this->__has_val()) {
+      std::__throw_bad_expected_access<_Err>(std::move(error()));
+    }
+    return std::move(this->__val());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Err& error() const& noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !this->__has_val(), "expected::error requires the expected to contain an error");
+    return this->__unex();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Err& error() & noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !this->__has_val(), "expected::error requires the expected to contain an error");
+    return this->__unex();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Err&& error() const&& noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !this->__has_val(), "expected::error requires the expected to contain an error");
+    return std::move(this->__unex());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Err&& error() && noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !this->__has_val(), "expected::error requires the expected to contain an error");
+    return std::move(this->__unex());
+  }
+
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp value_or(_Up&& __v) const& {
+    static_assert(is_copy_constructible_v<_Tp>, "value_type has to be copy constructible");
+    static_assert(is_convertible_v<_Up, _Tp>, "argument has to be convertible to value_type");
+    return this->__has_val() ? this->__val() : static_cast<_Tp>(std::forward<_Up>(__v));
+  }
+
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp value_or(_Up&& __v) && {
+    static_assert(is_move_constructible_v<_Tp>, "value_type has to be move constructible");
+    static_assert(is_convertible_v<_Up, _Tp>, "argument has to be convertible to value_type");
+    return this->__has_val() ? std::move(this->__val()) : static_cast<_Tp>(std::forward<_Up>(__v));
+  }
+
+  template <class _Up = _Err>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Err error_or(_Up&& __error) const& {
+    static_assert(is_copy_constructible_v<_Err>, "error_type has to be copy constructible");
+    static_assert(is_convertible_v<_Up, _Err>, "argument has to be convertible to error_type");
+    if (has_value())
+      return std::forward<_Up>(__error);
+    return error();
+  }
+
+  template <class _Up = _Err>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Err error_or(_Up&& __error) && {
+    static_assert(is_move_constructible_v<_Err>, "error_type has to be move constructible");
+    static_assert(is_convertible_v<_Up, _Err>, "argument has to be convertible to error_type");
+    if (has_value())
+      return std::forward<_Up>(__error);
+    return std::move(error());
+  }
+
+  // [expected.void.monadic], monadic
+  template <class _Func>
+    requires is_constructible_v<_Err, _Err&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) & {
+    using _Up = remove_cvref_t<invoke_result_t<_Func, _Tp&>>;
+    static_assert(__is_std_expected<_Up>::value, "The result of f(**this) must be a specialization of std::expected");
+    static_assert(is_same_v<typename _Up::error_type, _Err>,
+                  "The result of f(**this) must have the same error_type as this expected");
+    if (has_value()) {
+      return std::invoke(std::forward<_Func>(__f), this->__val());
+    }
+    return _Up(unexpect, error());
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Err, const _Err&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const& {
+    using _Up = remove_cvref_t<invoke_result_t<_Func, const _Tp&>>;
+    static_assert(__is_std_expected<_Up>::value, "The result of f(**this) must be a specialization of std::expected");
+    static_assert(is_same_v<typename _Up::error_type, _Err>,
+                  "The result of f(**this) must have the same error_type as this expected");
+    if (has_value()) {
+      return std::invoke(std::forward<_Func>(__f), this->__val());
+    }
+    return _Up(unexpect, error());
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Err, _Err&&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) && {
+    using _Up = remove_cvref_t<invoke_result_t<_Func, _Tp&&>>;
+    static_assert(
+        __is_std_expected<_Up>::value, "The result of f(std::move(**this)) must be a specialization of std::expected");
+    static_assert(is_same_v<typename _Up::error_type, _Err>,
+                  "The result of f(std::move(**this)) must have the same error_type as this expected");
+    if (has_value()) {
+      return std::invoke(std::forward<_Func>(__f), std::move(this->__val()));
+    }
+    return _Up(unexpect, std::move(error()));
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Err, const _Err&&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const&& {
+    using _Up = remove_cvref_t<invoke_result_t<_Func, const _Tp&&>>;
+    static_assert(
+        __is_std_expected<_Up>::value, "The result of f(std::move(**this)) must be a specialization of std::expected");
+    static_assert(is_same_v<typename _Up::error_type, _Err>,
+                  "The result of f(std::move(**this)) must have the same error_type as this expected");
+    if (has_value()) {
+      return std::invoke(std::forward<_Func>(__f), std::move(this->__val()));
+    }
+    return _Up(unexpect, std::move(error()));
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Tp, _Tp&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) & {
+    using _Gp = remove_cvref_t<invoke_result_t<_Func, _Err&>>;
+    static_assert(__is_std_expected<_Gp>::value, "The result of f(error()) must be a specialization of std::expected");
+    static_assert(is_same_v<typename _Gp::value_type, _Tp>,
+                  "The result of f(error()) must have the same value_type as this expected");
+    if (has_value()) {
+      return _Gp(in_place, this->__val());
+    }
+    return std::invoke(std::forward<_Func>(__f), error());
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Tp, const _Tp&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) const& {
+    using _Gp = remove_cvref_t<invoke_result_t<_Func, const _Err&>>;
+    static_assert(__is_std_expected<_Gp>::value, "The result of f(error()) must be a specialization of std::expected");
+    static_assert(is_same_v<typename _Gp::value_type, _Tp>,
+                  "The result of f(error()) must have the same value_type as this expected");
+    if (has_value()) {
+      return _Gp(in_place, this->__val());
+    }
+    return std::invoke(std::forward<_Func>(__f), error());
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Tp, _Tp&&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) && {
+    using _Gp = remove_cvref_t<invoke_result_t<_Func, _Err&&>>;
+    static_assert(
+        __is_std_expected<_Gp>::value, "The result of f(std::move(error())) must be a specialization of std::expected");
+    static_assert(is_same_v<typename _Gp::value_type, _Tp>,
+                  "The result of f(std::move(error())) must have the same value_type as this expected");
+    if (has_value()) {
+      return _Gp(in_place, std::move(this->__val()));
+    }
+    return std::invoke(std::forward<_Func>(__f), std::move(error()));
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Tp, const _Tp&&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) const&& {
+    using _Gp = remove_cvref_t<invoke_result_t<_Func, const _Err&&>>;
+    static_assert(
+        __is_std_expected<_Gp>::value, "The result of f(std::move(error())) must be a specialization of std::expected");
+    static_assert(is_same_v<typename _Gp::value_type, _Tp>,
+                  "The result of f(std::move(error())) must have the same value_type as this expected");
+    if (has_value()) {
+      return _Gp(in_place, std::move(this->__val()));
+    }
+    return std::invoke(std::forward<_Func>(__f), std::move(error()));
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Err, _Err&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) & {
+    using _Up = remove_cv_t<invoke_result_t<_Func, _Tp&>>;
+    if (!has_value()) {
+      return expected<_Up, _Err>(unexpect, error());
+    }
+    if constexpr (!is_void_v<_Up>) {
+      return expected<_Up, _Err>(
+          __expected_construct_in_place_from_invoke_tag{}, std::forward<_Func>(__f), this->__val());
+    } else {
+      std::invoke(std::forward<_Func>(__f), this->__val());
+      return expected<_Up, _Err>();
+    }
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Err, const _Err&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const& {
+    using _Up = remove_cv_t<invoke_result_t<_Func, const _Tp&>>;
+    if (!has_value()) {
+      return expected<_Up, _Err>(unexpect, error());
+    }
+    if constexpr (!is_void_v<_Up>) {
+      return expected<_Up, _Err>(
+          __expected_construct_in_place_from_invoke_tag{}, std::forward<_Func>(__f), this->__val());
+    } else {
+      std::invoke(std::forward<_Func>(__f), this->__val());
+      return expected<_Up, _Err>();
+    }
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Err, _Err&&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) && {
+    using _Up = remove_cv_t<invoke_result_t<_Func, _Tp&&>>;
+    if (!has_value()) {
+      return expected<_Up, _Err>(unexpect, std::move(error()));
+    }
+    if constexpr (!is_void_v<_Up>) {
+      return expected<_Up, _Err>(
+          __expected_construct_in_place_from_invoke_tag{}, std::forward<_Func>(__f), std::move(this->__val()));
+    } else {
+      std::invoke(std::forward<_Func>(__f), std::move(this->__val()));
+      return expected<_Up, _Err>();
+    }
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Err, const _Err&&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const&& {
+    using _Up = remove_cv_t<invoke_result_t<_Func, const _Tp&&>>;
+    if (!has_value()) {
+      return expected<_Up, _Err>(unexpect, std::move(error()));
+    }
+    if constexpr (!is_void_v<_Up>) {
+      return expected<_Up, _Err>(
+          __expected_construct_in_place_from_invoke_tag{}, std::forward<_Func>(__f), std::move(this->__val()));
+    } else {
+      std::invoke(std::forward<_Func>(__f), std::move(this->__val()));
+      return expected<_Up, _Err>();
+    }
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Tp, _Tp&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) & {
+    using _Gp = remove_cv_t<invoke_result_t<_Func, _Err&>>;
+    static_assert(__valid_std_unexpected<_Gp>::value,
+                  "The result of f(error()) must be a valid template argument for unexpected");
+    if (has_value()) {
+      return expected<_Tp, _Gp>(in_place, this->__val());
+    }
+    return expected<_Tp, _Gp>(__expected_construct_unexpected_from_invoke_tag{}, std::forward<_Func>(__f), error());
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Tp, const _Tp&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) const& {
+    using _Gp = remove_cv_t<invoke_result_t<_Func, const _Err&>>;
+    static_assert(__valid_std_unexpected<_Gp>::value,
+                  "The result of f(error()) must be a valid template argument for unexpected");
+    if (has_value()) {
+      return expected<_Tp, _Gp>(in_place, this->__val());
+    }
+    return expected<_Tp, _Gp>(__expected_construct_unexpected_from_invoke_tag{}, std::forward<_Func>(__f), error());
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Tp, _Tp&&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) && {
+    using _Gp = remove_cv_t<invoke_result_t<_Func, _Err&&>>;
+    static_assert(__valid_std_unexpected<_Gp>::value,
+                  "The result of f(std::move(error())) must be a valid template argument for unexpected");
+    if (has_value()) {
+      return expected<_Tp, _Gp>(in_place, std::move(this->__val()));
+    }
+    return expected<_Tp, _Gp>(
+        __expected_construct_unexpected_from_invoke_tag{}, std::forward<_Func>(__f), std::move(error()));
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Tp, const _Tp&&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) const&& {
+    using _Gp = remove_cv_t<invoke_result_t<_Func, const _Err&&>>;
+    static_assert(__valid_std_unexpected<_Gp>::value,
+                  "The result of f(std::move(error())) must be a valid template argument for unexpected");
+    if (has_value()) {
+      return expected<_Tp, _Gp>(in_place, std::move(this->__val()));
+    }
+    return expected<_Tp, _Gp>(
+        __expected_construct_unexpected_from_invoke_tag{}, std::forward<_Func>(__f), std::move(error()));
+  }
+
+  // [expected.object.eq], equality operators
+  template <class _T2, class _E2>
+    requires(!is_void_v<_T2>)
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const expected& __x, const expected<_T2, _E2>& __y) {
+    if (__x.__has_val() != __y.__has_val()) {
+      return false;
+    } else {
+      if (__x.__has_val()) {
+        return __x.__val() == __y.__val();
+      } else {
+        return __x.__unex() == __y.__unex();
+      }
+    }
+  }
+
+  template <class _T2>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const expected& __x, const _T2& __v) {
+    return __x.__has_val() && static_cast<bool>(__x.__val() == __v);
+  }
+
+  template <class _E2>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const expected& __x, const unexpected<_E2>& __e) {
+    return !__x.__has_val() && static_cast<bool>(__x.__unex() == __e.error());
+  }
+};
+
+template <class _Err>
+class __expected_void_base {
+  struct __empty_t {};
+  // use named union because [[no_unique_address]] cannot be applied to an unnamed union,
+  // also guaranteed elision into a potentially-overlapping subobject is unsettled (and
+  // it's not clear that it's implementable, given that the function is allowed to clobber
+  // the tail padding) - see https://github.com/itanium-cxx-abi/cxx-abi/issues/107.
+  union __union_t {
+    _LIBCPP_HIDE_FROM_ABI constexpr __union_t(const __union_t&) = delete;
+    _LIBCPP_HIDE_FROM_ABI constexpr __union_t(const __union_t&)
+      requires(is_copy_constructible_v<_Err> && is_trivially_copy_constructible_v<_Err>)
+    = default;
+    _LIBCPP_HIDE_FROM_ABI constexpr __union_t(__union_t&&) = delete;
+    _LIBCPP_HIDE_FROM_ABI constexpr __union_t(__union_t&&)
+      requires(is_move_constructible_v<_Err> && is_trivially_move_constructible_v<_Err>)
+    = default;
+    _LIBCPP_HIDE_FROM_ABI constexpr __union_t& operator=(const __union_t&) = delete;
+    _LIBCPP_HIDE_FROM_ABI constexpr __union_t& operator=(__union_t&&)      = delete;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(in_place_t) : __empty_() {}
+
+    template <class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(unexpect_t, _Args&&... __args)
+        : __unex_(std::forward<_Args>(__args)...) {}
+
+    template <class _Func, class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(
+        __expected_construct_unexpected_from_invoke_tag, _Func&& __f, _Args&&... __args)
+        : __unex_(std::invoke(std::forward<_Func>(__f), std::forward<_Args>(__args)...)) {}
+
+    _LIBCPP_HIDE_FROM_ABI constexpr ~__union_t()
+      requires(is_trivially_destructible_v<_Err>)
+    = default;
+
+    // __repr's destructor handles this
+    _LIBCPP_HIDE_FROM_ABI constexpr ~__union_t()
+      requires(!is_trivially_destructible_v<_Err>)
+    {}
+
+    _LIBCPP_NO_UNIQUE_ADDRESS __empty_t __empty_;
+    _LIBCPP_NO_UNIQUE_ADDRESS _Err __unex_;
+  };
+
+  static constexpr bool __put_flag_in_tail                    = __fits_in_tail_padding<__union_t, bool>;
+  static constexpr bool __allow_reusing_expected_tail_padding = !__put_flag_in_tail;
+
+  struct __repr {
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __repr() = delete;
+
+    template <class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __repr(in_place_t __tag) : __union_(in_place, __tag), __has_val_(true) {}
+
+    template <class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __repr(unexpect_t __tag, _Args&&... __args)
+        : __union_(in_place, __tag, std::forward<_Args>(__args)...), __has_val_(false) {}
+
+    template <class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __repr(std::__expected_construct_unexpected_from_invoke_tag __tag,
+                                                    _Args&&... __args)
+        : __union_(in_place, __tag, std::forward<_Args>(__args)...), __has_val_(false) {}
+
+    template <class _OtherUnion>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __repr(bool __has_val, _OtherUnion&& __other)
+      requires(__allow_reusing_expected_tail_padding)
+        : __union_(__conditional_no_unique_address_invoke_tag{},
+                   [&] { return __make_union(__has_val, std::forward<_OtherUnion>(__other)); }),
+          __has_val_(__has_val) {}
+
+    _LIBCPP_HIDE_FROM_ABI constexpr __repr(const __repr&) = delete;
+    _LIBCPP_HIDE_FROM_ABI constexpr __repr(const __repr&)
+      requires(is_copy_constructible_v<_Err> && is_trivially_copy_constructible_v<_Err>)
+    = default;
+    _LIBCPP_HIDE_FROM_ABI constexpr __repr(__repr&&) = delete;
+    _LIBCPP_HIDE_FROM_ABI constexpr __repr(__repr&&)
+      requires(is_move_constructible_v<_Err> && is_trivially_move_constructible_v<_Err>)
+    = default;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr __repr& operator=(const __repr&) = delete;
+    _LIBCPP_HIDE_FROM_ABI constexpr __repr& operator=(__repr&&)      = delete;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr ~__repr()
+      requires(is_trivially_destructible_v<_Err>)
+    = default;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr ~__repr()
+      requires(!is_trivially_destructible_v<_Err>)
+    {
+      __destroy_union_member();
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr void __destroy_union()
+      requires(__allow_reusing_expected_tail_padding && is_trivially_destructible_v<_Err>)
+    {
+      std::destroy_at(&__union_.__v);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr void __destroy_union()
+      requires(__allow_reusing_expected_tail_padding && !is_trivially_destructible_v<_Err>)
+    {
+      __destroy_union_member();
+      std::destroy_at(&__union_.__v);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr void __construct_union(in_place_t)
+      requires(__allow_reusing_expected_tail_padding)
+    {
+      std::construct_at(&__union_.__v, in_place);
+      __has_val_ = true;
+    }
+
+    template <class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr void __construct_union(unexpect_t, _Args&&... __args)
+      requires(__allow_reusing_expected_tail_padding)
+    {
+      std::construct_at(&__union_.__v, unexpect, std::forward<_Args>(__args)...);
+      __has_val_ = false;
+    }
+
+  private:
+    template <class>
+    friend class __expected_void_base;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr void __destroy_union_member()
+      requires(!is_trivially_destructible_v<_Err>)
+    {
+      if (!__has_val_)
+        std::destroy_at(std::addressof(__union_.__v.__unex_));
+    }
+
+    template <class _OtherUnion>
+    _LIBCPP_HIDE_FROM_ABI static constexpr __union_t __make_union(bool __has_val, _OtherUnion&& __other)
+      requires(__allow_reusing_expected_tail_padding)
+    {
+      if (__has_val)
+        return __union_t(in_place);
+      else
+        return __union_t(unexpect, std::forward<_OtherUnion>(__other).__unex_);
+    }
+
+    _LIBCPP_NO_UNIQUE_ADDRESS __conditional_no_unique_address<__put_flag_in_tail, __union_t> __union_;
+    _LIBCPP_NO_UNIQUE_ADDRESS bool __has_val_;
+  };
+
+  template <class _OtherUnion>
+  _LIBCPP_HIDE_FROM_ABI static constexpr __repr __make_repr(bool __has_val, _OtherUnion&& __other)
+    requires(__put_flag_in_tail)
+  {
+    if (__has_val)
+      return __repr(in_place);
+    else
+      return __repr(unexpect, std::forward<_OtherUnion>(__other).__unex_);
+  }
+
+protected:
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __expected_void_base(_Args&&... __args)
+      : __repr_(in_place, std::forward<_Args>(__args)...) {}
+
+  template <class _OtherUnion>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __expected_void_base(bool __has_val, _OtherUnion&& __other)
+    requires(__put_flag_in_tail)
+      : __repr_(__conditional_no_unique_address_invoke_tag{},
+                [&] { return __make_repr(__has_val, std::forward<_OtherUnion>(__other)); }) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void __destroy() {
+    if constexpr (__put_flag_in_tail)
+      std::destroy_at(&__repr_.__v);
+    else
+      __repr_.__v.__destroy_union();
+  }
+
+  template <class _Tag, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr void __construct(_Tag __tag, _Args&&... __args) {
+    if constexpr (__put_flag_in_tail)
+      std::construct_at(&__repr_.__v, __tag, std::forward<_Args>(__args)...);
+    else
+      __repr_.__v.__construct_union(__tag, std::forward<_Args>(__args)...);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __has_val() const { return __repr_.__v.__has_val_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr __union_t& __union() { return __repr_.__v.__union_.__v; }
+  _LIBCPP_HIDE_FROM_ABI constexpr const __union_t& __union() const { return __repr_.__v.__union_.__v; }
+  _LIBCPP_HIDE_FROM_ABI constexpr _Err& __unex() { return __repr_.__v.__union_.__v.__unex_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Err& __unex() const { return __repr_.__v.__union_.__v.__unex_; }
+
+private:
+  _LIBCPP_NO_UNIQUE_ADDRESS __conditional_no_unique_address<__allow_reusing_expected_tail_padding, __repr> __repr_;
+};
+
+template <class _Tp, class _Err>
+  requires is_void_v<_Tp>
+class expected<_Tp, _Err> : private __expected_void_base<_Err> {
+  static_assert(__valid_std_unexpected<_Err>::value,
+                "[expected.void.general] A program that instantiates expected<T, E> with a E that is not a "
+                "valid argument for unexpected<E> is ill-formed");
+
+  template <class, class>
+  friend class expected;
+
+  template <class _Up, class _OtherErr, class _OtherErrQual>
+  using __can_convert =
+      _And< is_void<_Up>,
+            is_constructible<_Err, _OtherErrQual>,
+            _Not<is_constructible<unexpected<_Err>, expected<_Up, _OtherErr>&>>,
+            _Not<is_constructible<unexpected<_Err>, expected<_Up, _OtherErr>>>,
+            _Not<is_constructible<unexpected<_Err>, const expected<_Up, _OtherErr>&>>,
+            _Not<is_constructible<unexpected<_Err>, const expected<_Up, _OtherErr>>>>;
+
+  using __base = __expected_void_base<_Err>;
+
+public:
+  using value_type      = _Tp;
+  using error_type      = _Err;
+  using unexpected_type = unexpected<_Err>;
+
+  template <class _Up>
+  using rebind = expected<_Up, error_type>;
+
+  // [expected.void.ctor], constructors
+  _LIBCPP_HIDE_FROM_ABI constexpr expected() noexcept : __base(in_place) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr expected(const expected&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr expected(const expected&)
+    requires(is_copy_constructible_v<_Err> && is_trivially_copy_constructible_v<_Err>)
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr expected(const expected& __rhs) noexcept(
+      is_nothrow_copy_constructible_v<_Err>) // strengthened
+    requires(is_copy_constructible_v<_Err> && !is_trivially_copy_constructible_v<_Err>)
+      : __base(__rhs.__has_val(), __rhs.__union()) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr expected(expected&&)
+    requires(is_move_constructible_v<_Err> && is_trivially_move_constructible_v<_Err>)
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr expected(expected&& __rhs) noexcept(is_nothrow_move_constructible_v<_Err>)
+    requires(is_move_constructible_v<_Err> && !is_trivially_move_constructible_v<_Err>)
+      : __base(__rhs.__has_val(), std::move(__rhs.__union())) {}
+
+  template <class _Up, class _OtherErr>
+    requires __can_convert<_Up, _OtherErr, const _OtherErr&>::value
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<const _OtherErr&, _Err>)
+      expected(const expected<_Up, _OtherErr>& __rhs) noexcept(
+          is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened
+      : __base(__rhs.__has_val(), __rhs.__union()) {}
+
+  template <class _Up, class _OtherErr>
+    requires __can_convert<_Up, _OtherErr, _OtherErr>::value
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_OtherErr, _Err>)
+      expected(expected<_Up, _OtherErr>&& __rhs) noexcept(is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened
+      : __base(__rhs.__has_val(), std::move(__rhs.__union())) {}
+
+  template <class _OtherErr>
+    requires is_constructible_v<_Err, const _OtherErr&>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<const _OtherErr&, _Err>) expected(
+      const unexpected<_OtherErr>& __unex) noexcept(is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened
+      : __base(unexpect, __unex.error()) {}
+
+  template <class _OtherErr>
+    requires is_constructible_v<_Err, _OtherErr>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_OtherErr, _Err>)
+      expected(unexpected<_OtherErr>&& __unex) noexcept(is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened
+      : __base(unexpect, std::move(__unex.error())) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(in_place_t) noexcept : __base(in_place) {}
+
+  template <class... _Args>
+    requires is_constructible_v<_Err, _Args...>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(unexpect_t, _Args&&... __args) noexcept(
+      is_nothrow_constructible_v<_Err, _Args...>) // strengthened
+      : __base(unexpect, std::forward<_Args>(__args)...) {}
+
+  template <class _Up, class... _Args>
+    requires is_constructible_v< _Err, initializer_list<_Up>&, _Args... >
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(unexpect_t, initializer_list<_Up> __il, _Args&&... __args) noexcept(
+      is_nothrow_constructible_v<_Err, initializer_list<_Up>&, _Args...>) // strengthened
+      : __base(unexpect, __il, std::forward<_Args>(__args)...) {}
+
+private:
+  template <class _Func, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(
+      __expected_construct_unexpected_from_invoke_tag __tag, _Func&& __f, _Args&&... __args)
+      : __base(__tag, std::forward<_Func>(__f), std::forward<_Args>(__args)...) {}
+
+public:
+  // [expected.void.dtor], destructor
+
+  _LIBCPP_HIDE_FROM_ABI constexpr ~expected() = default;
+
+private:
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr void __reinit_expected(unexpect_t, _Args&&... __args) {
+    _LIBCPP_ASSERT_INTERNAL(this->__has_val(), "__reinit_expected(unexpect_t, ...) needs value to be set");
+
+    this->__destroy();
+    auto __trans = std::__make_exception_guard([&] { this->__construct(in_place); });
+    this->__construct(unexpect, std::forward<_Args>(__args)...);
+    __trans.__complete();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void __reinit_expected(in_place_t) {
+    _LIBCPP_ASSERT_INTERNAL(!this->__has_val(), "__reinit_expected(in_place_t, ...) needs value to be unset");
+
+    this->__destroy();
+    this->__construct(in_place);
+  }
+
+public:
+  // [expected.void.assign], assignment
+  _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(const expected&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(const expected& __rhs) noexcept(
+      is_nothrow_copy_assignable_v<_Err> && is_nothrow_copy_constructible_v<_Err>) // strengthened
+    requires(is_copy_assignable_v<_Err> && is_copy_constructible_v<_Err>)
+  {
+    if (this->__has_val()) {
+      if (!__rhs.__has_val()) {
+        __reinit_expected(unexpect, __rhs.__unex());
+      }
+    } else {
+      if (__rhs.__has_val()) {
+        __reinit_expected(in_place);
+      } else {
+        this->__unex() = __rhs.__unex();
+      }
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(expected&&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr expected&
+  operator=(expected&& __rhs) noexcept(is_nothrow_move_assignable_v<_Err> && is_nothrow_move_constructible_v<_Err>)
+    requires(is_move_assignable_v<_Err> && is_move_constructible_v<_Err>)
+  {
+    if (this->__has_val()) {
+      if (!__rhs.__has_val()) {
+        __reinit_expected(unexpect, std::move(__rhs.__unex()));
+      }
+    } else {
+      if (__rhs.__has_val()) {
+        __reinit_expected(in_place);
+      } else {
+        this->__unex() = std::move(__rhs.__unex());
+      }
+    }
+    return *this;
+  }
+
+  template <class _OtherErr>
+    requires(is_constructible_v<_Err, const _OtherErr&> && is_assignable_v<_Err&, const _OtherErr&>)
+  _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(const unexpected<_OtherErr>& __un) {
+    if (this->__has_val()) {
+      __reinit_expected(unexpect, __un.error());
+    } else {
+      this->__unex() = __un.error();
+    }
+    return *this;
+  }
+
+  template <class _OtherErr>
+    requires(is_constructible_v<_Err, _OtherErr> && is_assignable_v<_Err&, _OtherErr>)
+  _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(unexpected<_OtherErr>&& __un) {
+    if (this->__has_val()) {
+      __reinit_expected(unexpect, std::move(__un.error()));
+    } else {
+      this->__unex() = std::move(__un.error());
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void emplace() noexcept {
+    if (!this->__has_val()) {
+      __reinit_expected(in_place);
+    }
+  }
+
+  // [expected.void.swap], swap
+  _LIBCPP_HIDE_FROM_ABI constexpr void
+  swap(expected& __rhs) noexcept(is_nothrow_move_constructible_v<_Err> && is_nothrow_swappable_v<_Err>)
+    requires(is_swappable_v<_Err> && is_move_constructible_v<_Err>)
+  {
+    auto __swap_val_unex_impl = [](expected& __with_val, expected& __with_err) {
+      // May throw, but will re-engage `__with_val` in that case.
+      __with_val.__reinit_expected(unexpect, std::move(__with_err.__unex()));
+      // Will not throw.
+      __with_err.__reinit_expected(in_place);
+    };
+
+    if (this->__has_val()) {
+      if (!__rhs.__has_val()) {
+        __swap_val_unex_impl(*this, __rhs);
+      }
+    } else {
+      if (__rhs.__has_val()) {
+        __swap_val_unex_impl(__rhs, *this);
+      } else {
+        using std::swap;
+        swap(this->__unex(), __rhs.__unex());
+      }
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr void swap(expected& __x, expected& __y) noexcept(noexcept(__x.swap(__y)))
+    requires requires { __x.swap(__y); }
+  {
+    __x.swap(__y);
+  }
+
+  // [expected.void.obs], observers
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return this->__has_val(); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool has_value() const noexcept { return this->__has_val(); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator*() const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        this->__has_val(), "expected::operator* requires the expected to contain a value");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void value() const& {
+    static_assert(is_copy_constructible_v<_Err>);
+    if (!this->__has_val()) {
+      std::__throw_bad_expected_access<_Err>(this->__unex());
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void value() && {
+    static_assert(is_copy_constructible_v<_Err> && is_move_constructible_v<_Err>);
+    if (!this->__has_val()) {
+      std::__throw_bad_expected_access<_Err>(std::move(this->__unex()));
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Err& error() const& noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !this->__has_val(), "expected::error requires the expected to contain an error");
+    return this->__unex();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Err& error() & noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !this->__has_val(), "expected::error requires the expected to contain an error");
+    return this->__unex();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Err&& error() const&& noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !this->__has_val(), "expected::error requires the expected to contain an error");
+    return std::move(this->__unex());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Err&& error() && noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !this->__has_val(), "expected::error requires the expected to contain an error");
+    return std::move(this->__unex());
+  }
+
+  template <class _Up = _Err>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Err error_or(_Up&& __error) const& {
+    static_assert(is_copy_constructible_v<_Err>, "error_type has to be copy constructible");
+    static_assert(is_convertible_v<_Up, _Err>, "argument has to be convertible to error_type");
+    if (has_value()) {
+      return std::forward<_Up>(__error);
+    }
+    return error();
+  }
+
+  template <class _Up = _Err>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Err error_or(_Up&& __error) && {
+    static_assert(is_move_constructible_v<_Err>, "error_type has to be move constructible");
+    static_assert(is_convertible_v<_Up, _Err>, "argument has to be convertible to error_type");
+    if (has_value()) {
+      return std::forward<_Up>(__error);
+    }
+    return std::move(error());
+  }
+
+  // [expected.void.monadic], monadic
+  template <class _Func>
+    requires is_constructible_v<_Err, _Err&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) & {
+    using _Up = remove_cvref_t<invoke_result_t<_Func>>;
+    static_assert(__is_std_expected<_Up>::value, "The result of f() must be a specialization of std::expected");
+    static_assert(
+        is_same_v<typename _Up::error_type, _Err>, "The result of f() must have the same error_type as this expected");
+    if (has_value()) {
+      return std::invoke(std::forward<_Func>(__f));
+    }
+    return _Up(unexpect, error());
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Err, const _Err&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const& {
+    using _Up = remove_cvref_t<invoke_result_t<_Func>>;
+    static_assert(__is_std_expected<_Up>::value, "The result of f() must be a specialization of std::expected");
+    static_assert(
+        is_same_v<typename _Up::error_type, _Err>, "The result of f() must have the same error_type as this expected");
+    if (has_value()) {
+      return std::invoke(std::forward<_Func>(__f));
+    }
+    return _Up(unexpect, error());
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Err, _Err&&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) && {
+    using _Up = remove_cvref_t<invoke_result_t<_Func>>;
+    static_assert(__is_std_expected<_Up>::value, "The result of f() must be a specialization of std::expected");
+    static_assert(
+        is_same_v<typename _Up::error_type, _Err>, "The result of f() must have the same error_type as this expected");
+    if (has_value()) {
+      return std::invoke(std::forward<_Func>(__f));
+    }
+    return _Up(unexpect, std::move(error()));
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Err, const _Err&&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const&& {
+    using _Up = remove_cvref_t<invoke_result_t<_Func>>;
+    static_assert(__is_std_expected<_Up>::value, "The result of f() must be a specialization of std::expected");
+    static_assert(
+        is_same_v<typename _Up::error_type, _Err>, "The result of f() must have the same error_type as this expected");
+    if (has_value()) {
+      return std::invoke(std::forward<_Func>(__f));
+    }
+    return _Up(unexpect, std::move(error()));
+  }
+
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) & {
+    using _Gp = remove_cvref_t<invoke_result_t<_Func, _Err&>>;
+    static_assert(__is_std_expected<_Gp>::value, "The result of f(error()) must be a specialization of std::expected");
+    static_assert(is_same_v<typename _Gp::value_type, _Tp>,
+                  "The result of f(error()) must have the same value_type as this expected");
+    if (has_value()) {
+      return _Gp();
+    }
+    return std::invoke(std::forward<_Func>(__f), error());
+  }
+
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) const& {
+    using _Gp = remove_cvref_t<invoke_result_t<_Func, const _Err&>>;
+    static_assert(__is_std_expected<_Gp>::value, "The result of f(error()) must be a specialization of std::expected");
+    static_assert(is_same_v<typename _Gp::value_type, _Tp>,
+                  "The result of f(error()) must have the same value_type as this expected");
+    if (has_value()) {
+      return _Gp();
+    }
+    return std::invoke(std::forward<_Func>(__f), error());
+  }
+
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) && {
+    using _Gp = remove_cvref_t<invoke_result_t<_Func, _Err&&>>;
+    static_assert(
+        __is_std_expected<_Gp>::value, "The result of f(std::move(error())) must be a specialization of std::expected");
+    static_assert(is_same_v<typename _Gp::value_type, _Tp>,
+                  "The result of f(std::move(error())) must have the same value_type as this expected");
+    if (has_value()) {
+      return _Gp();
+    }
+    return std::invoke(std::forward<_Func>(__f), std::move(error()));
+  }
+
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) const&& {
+    using _Gp = remove_cvref_t<invoke_result_t<_Func, const _Err&&>>;
+    static_assert(
+        __is_std_expected<_Gp>::value, "The result of f(std::move(error())) must be a specialization of std::expected");
+    static_assert(is_same_v<typename _Gp::value_type, _Tp>,
+                  "The result of f(std::move(error())) must have the same value_type as this expected");
+    if (has_value()) {
+      return _Gp();
+    }
+    return std::invoke(std::forward<_Func>(__f), std::move(error()));
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Err, _Err&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) & {
+    using _Up = remove_cv_t<invoke_result_t<_Func>>;
+    if (!has_value()) {
+      return expected<_Up, _Err>(unexpect, error());
+    }
+    if constexpr (!is_void_v<_Up>) {
+      return expected<_Up, _Err>(__expected_construct_in_place_from_invoke_tag{}, std::forward<_Func>(__f));
+    } else {
+      std::invoke(std::forward<_Func>(__f));
+      return expected<_Up, _Err>();
+    }
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Err, const _Err&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const& {
+    using _Up = remove_cv_t<invoke_result_t<_Func>>;
+    if (!has_value()) {
+      return expected<_Up, _Err>(unexpect, error());
+    }
+    if constexpr (!is_void_v<_Up>) {
+      return expected<_Up, _Err>(__expected_construct_in_place_from_invoke_tag{}, std::forward<_Func>(__f));
+    } else {
+      std::invoke(std::forward<_Func>(__f));
+      return expected<_Up, _Err>();
+    }
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Err, _Err&&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) && {
+    using _Up = remove_cv_t<invoke_result_t<_Func>>;
+    if (!has_value()) {
+      return expected<_Up, _Err>(unexpect, std::move(error()));
+    }
+    if constexpr (!is_void_v<_Up>) {
+      return expected<_Up, _Err>(__expected_construct_in_place_from_invoke_tag{}, std::forward<_Func>(__f));
+    } else {
+      std::invoke(std::forward<_Func>(__f));
+      return expected<_Up, _Err>();
+    }
+  }
+
+  template <class _Func>
+    requires is_constructible_v<_Err, const _Err&&>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const&& {
+    using _Up = remove_cv_t<invoke_result_t<_Func>>;
+    if (!has_value()) {
+      return expected<_Up, _Err>(unexpect, std::move(error()));
+    }
+    if constexpr (!is_void_v<_Up>) {
+      return expected<_Up, _Err>(__expected_construct_in_place_from_invoke_tag{}, std::forward<_Func>(__f));
+    } else {
+      std::invoke(std::forward<_Func>(__f));
+      return expected<_Up, _Err>();
+    }
+  }
+
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) & {
+    using _Gp = remove_cv_t<invoke_result_t<_Func, _Err&>>;
+    static_assert(__valid_std_unexpected<_Gp>::value,
+                  "The result of f(error()) must be a valid template argument for unexpected");
+    if (has_value()) {
+      return expected<_Tp, _Gp>();
+    }
+    return expected<_Tp, _Gp>(__expected_construct_unexpected_from_invoke_tag{}, std::forward<_Func>(__f), error());
+  }
+
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) const& {
+    using _Gp = remove_cv_t<invoke_result_t<_Func, const _Err&>>;
+    static_assert(__valid_std_unexpected<_Gp>::value,
+                  "The result of f(error()) must be a valid template argument for unexpected");
+    if (has_value()) {
+      return expected<_Tp, _Gp>();
+    }
+    return expected<_Tp, _Gp>(__expected_construct_unexpected_from_invoke_tag{}, std::forward<_Func>(__f), error());
+  }
+
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) && {
+    using _Gp = remove_cv_t<invoke_result_t<_Func, _Err&&>>;
+    static_assert(__valid_std_unexpected<_Gp>::value,
+                  "The result of f(std::move(error())) must be a valid template argument for unexpected");
+    if (has_value()) {
+      return expected<_Tp, _Gp>();
+    }
+    return expected<_Tp, _Gp>(
+        __expected_construct_unexpected_from_invoke_tag{}, std::forward<_Func>(__f), std::move(error()));
+  }
+
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) const&& {
+    using _Gp = remove_cv_t<invoke_result_t<_Func, const _Err&&>>;
+    static_assert(__valid_std_unexpected<_Gp>::value,
+                  "The result of f(std::move(error())) must be a valid template argument for unexpected");
+    if (has_value()) {
+      return expected<_Tp, _Gp>();
+    }
+    return expected<_Tp, _Gp>(
+        __expected_construct_unexpected_from_invoke_tag{}, std::forward<_Func>(__f), std::move(error()));
+  }
+
+  // [expected.void.eq], equality operators
+  template <class _T2, class _E2>
+    requires is_void_v<_T2>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const expected& __x, const expected<_T2, _E2>& __y) {
+    if (__x.__has_val() != __y.__has_val()) {
+      return false;
+    } else {
+      return __x.__has_val() || static_cast<bool>(__x.__unex() == __y.__unex());
+    }
+  }
+
+  template <class _E2>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const expected& __x, const unexpected<_E2>& __y) {
+    return !__x.__has_val() && static_cast<bool>(__x.__unex() == __y.error());
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___EXPECTED_EXPECTED_H
diff --git a/libcxx/include/__cxx03/__expected/unexpect.h b/libcxx/include/__cxx03/__expected/unexpect.h
new file mode 100644
index 000000000000..df52787d36fa
--- /dev/null
+++ b/libcxx/include/__cxx03/__expected/unexpect.h
@@ -0,0 +1,32 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef _LIBCPP___EXPECTED_UNEXPECT_H
+#define _LIBCPP___EXPECTED_UNEXPECT_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct unexpect_t {
+  explicit unexpect_t() = default;
+};
+
+inline constexpr unexpect_t unexpect{};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+#endif // _LIBCPP___EXPECTED_UNEXPECT_H
diff --git a/libcxx/include/__cxx03/__expected/unexpected.h b/libcxx/include/__cxx03/__expected/unexpected.h
new file mode 100644
index 000000000000..c7fe3c52e431
--- /dev/null
+++ b/libcxx/include/__cxx03/__expected/unexpected.h
@@ -0,0 +1,127 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef _LIBCPP___EXPECTED_UNEXPECTED_H
+#define _LIBCPP___EXPECTED_UNEXPECTED_H
+
+#include <__config>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_const.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_object.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_swappable.h>
+#include <__type_traits/is_volatile.h>
+#include <__type_traits/negation.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/forward.h>
+#include <__utility/in_place.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Err>
+class unexpected;
+
+template <class _Tp>
+struct __is_std_unexpected : false_type {};
+
+template <class _Err>
+struct __is_std_unexpected<unexpected<_Err>> : true_type {};
+
+template <class _Tp>
+using __valid_std_unexpected = _BoolConstant< //
+    is_object_v<_Tp> &&                       //
+    !is_array_v<_Tp> &&                       //
+    !__is_std_unexpected<_Tp>::value &&       //
+    !is_const_v<_Tp> &&                       //
+    !is_volatile_v<_Tp>                       //
+    >;
+
+template <class _Err>
+class unexpected {
+  static_assert(__valid_std_unexpected<_Err>::value,
+                "[expected.un.general] states a program that instantiates std::unexpected for a non-object type, an "
+                "array type, a specialization of unexpected, or a cv-qualified type is ill-formed.");
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr unexpected(const unexpected&) = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr unexpected(unexpected&&)      = default;
+
+  template <class _Error = _Err>
+    requires(!is_same_v<remove_cvref_t<_Error>, unexpected> && //
+             !is_same_v<remove_cvref_t<_Error>, in_place_t> && //
+             is_constructible_v<_Err, _Error>)
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit unexpected(_Error&& __error) //
+      noexcept(is_nothrow_constructible_v<_Err, _Error>)                // strengthened
+      : __unex_(std::forward<_Error>(__error)) {}
+
+  template <class... _Args>
+    requires is_constructible_v<_Err, _Args...>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit unexpected(in_place_t, _Args&&... __args) //
+      noexcept(is_nothrow_constructible_v<_Err, _Args...>)                           // strengthened
+      : __unex_(std::forward<_Args>(__args)...) {}
+
+  template <class _Up, class... _Args>
+    requires is_constructible_v<_Err, initializer_list<_Up>&, _Args...>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit unexpected(in_place_t, initializer_list<_Up> __il, _Args&&... __args) //
+      noexcept(is_nothrow_constructible_v<_Err, initializer_list<_Up>&, _Args...>) // strengthened
+      : __unex_(__il, std::forward<_Args>(__args)...) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr unexpected& operator=(const unexpected&) = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr unexpected& operator=(unexpected&&)      = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Err& error() const& noexcept { return __unex_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr _Err& error() & noexcept { return __unex_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Err&& error() const&& noexcept { return std::move(__unex_); }
+  _LIBCPP_HIDE_FROM_ABI constexpr _Err&& error() && noexcept { return std::move(__unex_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void swap(unexpected& __other) noexcept(is_nothrow_swappable_v<_Err>) {
+    static_assert(is_swappable_v<_Err>, "unexpected::swap requires is_swappable_v<E> to be true");
+    using std::swap;
+    swap(__unex_, __other.__unex_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr void swap(unexpected& __x, unexpected& __y) noexcept(noexcept(__x.swap(__y)))
+    requires is_swappable_v<_Err>
+  {
+    __x.swap(__y);
+  }
+
+  template <class _Err2>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const unexpected& __x, const unexpected<_Err2>& __y) {
+    return __x.__unex_ == __y.__unex_;
+  }
+
+private:
+  _Err __unex_;
+};
+
+template <class _Err>
+unexpected(_Err) -> unexpected<_Err>;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___EXPECTED_UNEXPECTED_H
diff --git a/libcxx/include/__cxx03/__filesystem/copy_options.h b/libcxx/include/__cxx03/__filesystem/copy_options.h
new file mode 100644
index 000000000000..097eebe61137
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/copy_options.h
@@ -0,0 +1,69 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_COPY_OPTIONS_H
+#define _LIBCPP___FILESYSTEM_COPY_OPTIONS_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+enum class copy_options : unsigned short {
+  none                = 0,
+  skip_existing       = 1,
+  overwrite_existing  = 2,
+  update_existing     = 4,
+  recursive           = 8,
+  copy_symlinks       = 16,
+  skip_symlinks       = 32,
+  directories_only    = 64,
+  create_symlinks     = 128,
+  create_hard_links   = 256,
+  __in_recursive_copy = 512,
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator&(copy_options __lhs, copy_options __rhs) {
+  return static_cast<copy_options>(static_cast<unsigned short>(__lhs) & static_cast<unsigned short>(__rhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator|(copy_options __lhs, copy_options __rhs) {
+  return static_cast<copy_options>(static_cast<unsigned short>(__lhs) | static_cast<unsigned short>(__rhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator^(copy_options __lhs, copy_options __rhs) {
+  return static_cast<copy_options>(static_cast<unsigned short>(__lhs) ^ static_cast<unsigned short>(__rhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator~(copy_options __lhs) {
+  return static_cast<copy_options>(~static_cast<unsigned short>(__lhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline copy_options& operator&=(copy_options& __lhs, copy_options __rhs) {
+  return __lhs = __lhs & __rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline copy_options& operator|=(copy_options& __lhs, copy_options __rhs) {
+  return __lhs = __lhs | __rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline copy_options& operator^=(copy_options& __lhs, copy_options __rhs) {
+  return __lhs = __lhs ^ __rhs;
+}
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___FILESYSTEM_COPY_OPTIONS_H
diff --git a/libcxx/include/__cxx03/__filesystem/directory_entry.h b/libcxx/include/__cxx03/__filesystem/directory_entry.h
new file mode 100644
index 000000000000..96d88dcd90b4
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/directory_entry.h
@@ -0,0 +1,435 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_DIRECTORY_ENTRY_H
+#define _LIBCPP___FILESYSTEM_DIRECTORY_ENTRY_H
+
+#include <__chrono/time_point.h>
+#include <__compare/ordering.h>
+#include <__config>
+#include <__filesystem/file_status.h>
+#include <__filesystem/file_time_type.h>
+#include <__filesystem/file_type.h>
+#include <__filesystem/filesystem_error.h>
+#include <__filesystem/operations.h>
+#include <__filesystem/path.h>
+#include <__filesystem/perms.h>
+#include <__system_error/errc.h>
+#include <__system_error/error_code.h>
+#include <__utility/move.h>
+#include <__utility/unreachable.h>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 17 && !defined(_LIBCPP_HAS_NO_FILESYSTEM)
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH
+
+class directory_entry {
+  typedef filesystem::path _Path;
+
+public:
+  // constructors and destructors
+  _LIBCPP_HIDE_FROM_ABI directory_entry() noexcept                  = default;
+  _LIBCPP_HIDE_FROM_ABI directory_entry(directory_entry const&)     = default;
+  _LIBCPP_HIDE_FROM_ABI directory_entry(directory_entry&&) noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI explicit directory_entry(_Path const& __p) : __p_(__p) {
+    error_code __ec;
+    __refresh(&__ec);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI directory_entry(_Path const& __p, error_code& __ec) : __p_(__p) { __refresh(&__ec); }
+
+  _LIBCPP_HIDE_FROM_ABI ~directory_entry() {}
+
+  _LIBCPP_HIDE_FROM_ABI directory_entry& operator=(directory_entry const&)     = default;
+  _LIBCPP_HIDE_FROM_ABI directory_entry& operator=(directory_entry&&) noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI void assign(_Path const& __p) {
+    __p_ = __p;
+    error_code __ec;
+    __refresh(&__ec);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void assign(_Path const& __p, error_code& __ec) {
+    __p_ = __p;
+    __refresh(&__ec);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void replace_filename(_Path const& __p) {
+    __p_.replace_filename(__p);
+    error_code __ec;
+    __refresh(&__ec);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void replace_filename(_Path const& __p, error_code& __ec) {
+    __p_ = __p_.parent_path() / __p;
+    __refresh(&__ec);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void refresh() { __refresh(); }
+
+  _LIBCPP_HIDE_FROM_ABI void refresh(error_code& __ec) noexcept { __refresh(&__ec); }
+
+  _LIBCPP_HIDE_FROM_ABI _Path const& path() const noexcept { return __p_; }
+
+  _LIBCPP_HIDE_FROM_ABI operator const _Path&() const noexcept { return __p_; }
+
+  _LIBCPP_HIDE_FROM_ABI bool exists() const { return filesystem::exists(file_status{__get_ft()}); }
+
+  _LIBCPP_HIDE_FROM_ABI bool exists(error_code& __ec) const noexcept {
+    return filesystem::exists(file_status{__get_ft(&__ec)});
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_block_file() const { return __get_ft() == file_type::block; }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_block_file(error_code& __ec) const noexcept {
+    return __get_ft(&__ec) == file_type::block;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_character_file() const { return __get_ft() == file_type::character; }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_character_file(error_code& __ec) const noexcept {
+    return __get_ft(&__ec) == file_type::character;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_directory() const { return __get_ft() == file_type::directory; }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_directory(error_code& __ec) const noexcept {
+    return __get_ft(&__ec) == file_type::directory;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_fifo() const { return __get_ft() == file_type::fifo; }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_fifo(error_code& __ec) const noexcept { return __get_ft(&__ec) == file_type::fifo; }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_other() const { return filesystem::is_other(file_status{__get_ft()}); }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_other(error_code& __ec) const noexcept {
+    return filesystem::is_other(file_status{__get_ft(&__ec)});
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_regular_file() const { return __get_ft() == file_type::regular; }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_regular_file(error_code& __ec) const noexcept {
+    return __get_ft(&__ec) == file_type::regular;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_socket() const { return __get_ft() == file_type::socket; }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_socket(error_code& __ec) const noexcept { return __get_ft(&__ec) == file_type::socket; }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_symlink() const { return __get_sym_ft() == file_type::symlink; }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_symlink(error_code& __ec) const noexcept {
+    return __get_sym_ft(&__ec) == file_type::symlink;
+  }
+  _LIBCPP_HIDE_FROM_ABI uintmax_t file_size() const { return __get_size(); }
+
+  _LIBCPP_HIDE_FROM_ABI uintmax_t file_size(error_code& __ec) const noexcept { return __get_size(&__ec); }
+
+  _LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count() const { return __get_nlink(); }
+
+  _LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count(error_code& __ec) const noexcept { return __get_nlink(&__ec); }
+
+  _LIBCPP_HIDE_FROM_ABI file_time_type last_write_time() const { return __get_write_time(); }
+
+  _LIBCPP_HIDE_FROM_ABI file_time_type last_write_time(error_code& __ec) const noexcept {
+    return __get_write_time(&__ec);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI file_status status() const { return __get_status(); }
+
+  _LIBCPP_HIDE_FROM_ABI file_status status(error_code& __ec) const noexcept { return __get_status(&__ec); }
+
+  _LIBCPP_HIDE_FROM_ABI file_status symlink_status() const { return __get_symlink_status(); }
+
+  _LIBCPP_HIDE_FROM_ABI file_status symlink_status(error_code& __ec) const noexcept {
+    return __get_symlink_status(&__ec);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool operator==(directory_entry const& __rhs) const noexcept { return __p_ == __rhs.__p_; }
+
+#  if _LIBCPP_STD_VER <= 17
+  _LIBCPP_HIDE_FROM_ABI bool operator!=(directory_entry const& __rhs) const noexcept { return __p_ != __rhs.__p_; }
+
+  _LIBCPP_HIDE_FROM_ABI bool operator<(directory_entry const& __rhs) const noexcept { return __p_ < __rhs.__p_; }
+
+  _LIBCPP_HIDE_FROM_ABI bool operator<=(directory_entry const& __rhs) const noexcept { return __p_ <= __rhs.__p_; }
+
+  _LIBCPP_HIDE_FROM_ABI bool operator>(directory_entry const& __rhs) const noexcept { return __p_ > __rhs.__p_; }
+
+  _LIBCPP_HIDE_FROM_ABI bool operator>=(directory_entry const& __rhs) const noexcept { return __p_ >= __rhs.__p_; }
+
+#  else // _LIBCPP_STD_VER <= 17
+
+  _LIBCPP_HIDE_FROM_ABI strong_ordering operator<=>(const directory_entry& __rhs) const noexcept {
+    return __p_ <=> __rhs.__p_;
+  }
+
+#  endif // _LIBCPP_STD_VER <= 17
+
+  template <class _CharT, class _Traits>
+  _LIBCPP_HIDE_FROM_ABI friend basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const directory_entry& __d) {
+    return __os << __d.path();
+  }
+
+private:
+  friend class directory_iterator;
+  friend class recursive_directory_iterator;
+  friend class _LIBCPP_HIDDEN __dir_stream;
+
+  enum _CacheType : unsigned char {
+    _Empty,
+    _IterSymlink,
+    _IterNonSymlink,
+    _RefreshSymlink,
+    _RefreshSymlinkUnresolved,
+    _RefreshNonSymlink
+  };
+
+  struct __cached_data {
+    uintmax_t __size_;
+    uintmax_t __nlink_;
+    file_time_type __write_time_;
+    perms __sym_perms_;
+    perms __non_sym_perms_;
+    file_type __type_;
+    _CacheType __cache_type_;
+
+    _LIBCPP_HIDE_FROM_ABI __cached_data() noexcept { __reset(); }
+
+    _LIBCPP_HIDE_FROM_ABI void __reset() {
+      __cache_type_ = _Empty;
+      __type_       = file_type::none;
+      __sym_perms_ = __non_sym_perms_ = perms::unknown;
+      __size_ = __nlink_ = uintmax_t(-1);
+      __write_time_      = file_time_type::min();
+    }
+  };
+
+  _LIBCPP_HIDE_FROM_ABI static __cached_data __create_iter_result(file_type __ft) {
+    __cached_data __data;
+    __data.__type_       = __ft;
+    __data.__cache_type_ = [&]() {
+      switch (__ft) {
+      case file_type::none:
+        return _Empty;
+      case file_type::symlink:
+        return _IterSymlink;
+      default:
+        return _IterNonSymlink;
+      }
+    }();
+    return __data;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __assign_iter_entry(_Path&& __p, __cached_data __dt) {
+    __p_    = std::move(__p);
+    __data_ = __dt;
+  }
+
+  _LIBCPP_EXPORTED_FROM_ABI error_code __do_refresh() noexcept;
+
+  _LIBCPP_HIDE_FROM_ABI static bool __is_dne_error(error_code const& __ec) {
+    if (!__ec)
+      return true;
+    switch (static_cast<errc>(__ec.value())) {
+    case errc::no_such_file_or_directory:
+    case errc::not_a_directory:
+      return true;
+    default:
+      return false;
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void
+  __handle_error(const char* __msg, error_code* __dest_ec, error_code const& __ec, bool __allow_dne = false) const {
+    if (__dest_ec) {
+      *__dest_ec = __ec;
+      return;
+    }
+    if (__ec && (!__allow_dne || !__is_dne_error(__ec)))
+      __throw_filesystem_error(__msg, __p_, __ec);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __refresh(error_code* __ec = nullptr) {
+    __handle_error("in directory_entry::refresh",
+                   __ec,
+                   __do_refresh(),
+                   /*allow_dne*/ true);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI file_type __get_sym_ft(error_code* __ec = nullptr) const {
+    switch (__data_.__cache_type_) {
+    case _Empty:
+      return __symlink_status(__p_, __ec).type();
+    case _IterSymlink:
+    case _RefreshSymlink:
+    case _RefreshSymlinkUnresolved:
+      if (__ec)
+        __ec->clear();
+      return file_type::symlink;
+    case _IterNonSymlink:
+    case _RefreshNonSymlink:
+      file_status __st(__data_.__type_);
+      if (__ec && !filesystem::exists(__st))
+        *__ec = make_error_code(errc::no_such_file_or_directory);
+      else if (__ec)
+        __ec->clear();
+      return __data_.__type_;
+    }
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI file_type __get_ft(error_code* __ec = nullptr) const {
+    switch (__data_.__cache_type_) {
+    case _Empty:
+    case _IterSymlink:
+    case _RefreshSymlinkUnresolved:
+      return __status(__p_, __ec).type();
+    case _IterNonSymlink:
+    case _RefreshNonSymlink:
+    case _RefreshSymlink: {
+      file_status __st(__data_.__type_);
+      if (__ec && !filesystem::exists(__st))
+        *__ec = make_error_code(errc::no_such_file_or_directory);
+      else if (__ec)
+        __ec->clear();
+      return __data_.__type_;
+    }
+    }
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI file_status __get_status(error_code* __ec = nullptr) const {
+    switch (__data_.__cache_type_) {
+    case _Empty:
+    case _IterNonSymlink:
+    case _IterSymlink:
+    case _RefreshSymlinkUnresolved:
+      return __status(__p_, __ec);
+    case _RefreshNonSymlink:
+    case _RefreshSymlink:
+      return file_status(__get_ft(__ec), __data_.__non_sym_perms_);
+    }
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI file_status __get_symlink_status(error_code* __ec = nullptr) const {
+    switch (__data_.__cache_type_) {
+    case _Empty:
+    case _IterNonSymlink:
+    case _IterSymlink:
+      return __symlink_status(__p_, __ec);
+    case _RefreshNonSymlink:
+      return file_status(__get_sym_ft(__ec), __data_.__non_sym_perms_);
+    case _RefreshSymlink:
+    case _RefreshSymlinkUnresolved:
+      return file_status(__get_sym_ft(__ec), __data_.__sym_perms_);
+    }
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI uintmax_t __get_size(error_code* __ec = nullptr) const {
+    switch (__data_.__cache_type_) {
+    case _Empty:
+    case _IterNonSymlink:
+    case _IterSymlink:
+    case _RefreshSymlinkUnresolved:
+      return filesystem::__file_size(__p_, __ec);
+    case _RefreshSymlink:
+    case _RefreshNonSymlink: {
+      error_code __m_ec;
+      file_status __st(__get_ft(&__m_ec));
+      __handle_error("in directory_entry::file_size", __ec, __m_ec);
+      if (filesystem::exists(__st) && !filesystem::is_regular_file(__st)) {
+        errc __err_kind = filesystem::is_directory(__st) ? errc::is_a_directory : errc::not_supported;
+        __handle_error("in directory_entry::file_size", __ec, make_error_code(__err_kind));
+      }
+      return __data_.__size_;
+    }
+    }
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI uintmax_t __get_nlink(error_code* __ec = nullptr) const {
+    switch (__data_.__cache_type_) {
+    case _Empty:
+    case _IterNonSymlink:
+    case _IterSymlink:
+    case _RefreshSymlinkUnresolved:
+      return filesystem::__hard_link_count(__p_, __ec);
+    case _RefreshSymlink:
+    case _RefreshNonSymlink: {
+      error_code __m_ec;
+      (void)__get_ft(&__m_ec);
+      __handle_error("in directory_entry::hard_link_count", __ec, __m_ec);
+      return __data_.__nlink_;
+    }
+    }
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI file_time_type __get_write_time(error_code* __ec = nullptr) const {
+    switch (__data_.__cache_type_) {
+    case _Empty:
+    case _IterNonSymlink:
+    case _IterSymlink:
+    case _RefreshSymlinkUnresolved:
+      return filesystem::__last_write_time(__p_, __ec);
+    case _RefreshSymlink:
+    case _RefreshNonSymlink: {
+      error_code __m_ec;
+      file_status __st(__get_ft(&__m_ec));
+      __handle_error("in directory_entry::last_write_time", __ec, __m_ec);
+      if (filesystem::exists(__st) && __data_.__write_time_ == file_time_type::min())
+        __handle_error("in directory_entry::last_write_time", __ec, make_error_code(errc::value_too_large));
+      return __data_.__write_time_;
+    }
+    }
+    __libcpp_unreachable();
+  }
+
+private:
+  _Path __p_;
+  __cached_data __data_;
+};
+
+class __dir_element_proxy {
+public:
+  inline _LIBCPP_HIDE_FROM_ABI directory_entry operator*() { return std::move(__elem_); }
+
+private:
+  friend class directory_iterator;
+  friend class recursive_directory_iterator;
+  _LIBCPP_HIDE_FROM_ABI explicit __dir_element_proxy(directory_entry const& __e) : __elem_(__e) {}
+  _LIBCPP_HIDE_FROM_ABI __dir_element_proxy(__dir_element_proxy&& __o) : __elem_(std::move(__o.__elem_)) {}
+  directory_entry __elem_;
+};
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_STD_VER >= 17 && !defined(_LIBCPP_HAS_NO_FILESYSTEM)
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FILESYSTEM_DIRECTORY_ENTRY_H
diff --git a/libcxx/include/__cxx03/__filesystem/directory_iterator.h b/libcxx/include/__cxx03/__filesystem/directory_iterator.h
new file mode 100644
index 000000000000..e0246d8001e1
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/directory_iterator.h
@@ -0,0 +1,151 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_DIRECTORY_ITERATOR_H
+#define _LIBCPP___FILESYSTEM_DIRECTORY_ITERATOR_H
+
+#include <__assert>
+#include <__config>
+#include <__filesystem/directory_entry.h>
+#include <__filesystem/directory_options.h>
+#include <__filesystem/path.h>
+#include <__iterator/default_sentinel.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/shared_ptr.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/enable_view.h>
+#include <__system_error/error_code.h>
+#include <__utility/move.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 17 && !defined(_LIBCPP_HAS_NO_FILESYSTEM)
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH
+
+class _LIBCPP_HIDDEN __dir_stream;
+class directory_iterator {
+public:
+  typedef directory_entry value_type;
+  typedef ptrdiff_t difference_type;
+  typedef value_type const* pointer;
+  typedef value_type const& reference;
+  typedef input_iterator_tag iterator_category;
+
+public:
+  // ctor & dtor
+  _LIBCPP_HIDE_FROM_ABI directory_iterator() noexcept {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit directory_iterator(const path& __p) : directory_iterator(__p, nullptr) {}
+
+  _LIBCPP_HIDE_FROM_ABI directory_iterator(const path& __p, directory_options __opts)
+      : directory_iterator(__p, nullptr, __opts) {}
+
+  _LIBCPP_HIDE_FROM_ABI directory_iterator(const path& __p, error_code& __ec) : directory_iterator(__p, &__ec) {}
+
+  _LIBCPP_HIDE_FROM_ABI directory_iterator(const path& __p, directory_options __opts, error_code& __ec)
+      : directory_iterator(__p, &__ec, __opts) {}
+
+  _LIBCPP_HIDE_FROM_ABI directory_iterator(const directory_iterator&)            = default;
+  _LIBCPP_HIDE_FROM_ABI directory_iterator(directory_iterator&&)                 = default;
+  _LIBCPP_HIDE_FROM_ABI directory_iterator& operator=(const directory_iterator&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI directory_iterator& operator=(directory_iterator&& __o) noexcept {
+    // non-default implementation provided to support self-move assign.
+    if (this != &__o) {
+      __imp_ = std::move(__o.__imp_);
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~directory_iterator() = default;
+
+  _LIBCPP_HIDE_FROM_ABI const directory_entry& operator*() const {
+    // Note: this check duplicates a check in `__dereference()`.
+    _LIBCPP_ASSERT_NON_NULL(__imp_, "The end iterator cannot be dereferenced");
+    return __dereference();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const directory_entry* operator->() const { return &**this; }
+
+  _LIBCPP_HIDE_FROM_ABI directory_iterator& operator++() { return __increment(); }
+
+  _LIBCPP_HIDE_FROM_ABI __dir_element_proxy operator++(int) {
+    __dir_element_proxy __p(**this);
+    __increment();
+    return __p;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI directory_iterator& increment(error_code& __ec) { return __increment(&__ec); }
+
+#  if _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI bool operator==(default_sentinel_t) const noexcept { return *this == directory_iterator(); }
+
+#  endif
+
+private:
+  inline _LIBCPP_HIDE_FROM_ABI friend bool
+  operator==(const directory_iterator& __lhs, const directory_iterator& __rhs) noexcept;
+
+  // construct the dir_stream
+  _LIBCPP_EXPORTED_FROM_ABI directory_iterator(const path&, error_code*, directory_options = directory_options::none);
+
+  _LIBCPP_EXPORTED_FROM_ABI directory_iterator& __increment(error_code* __ec = nullptr);
+
+  _LIBCPP_EXPORTED_FROM_ABI const directory_entry& __dereference() const;
+
+private:
+  shared_ptr<__dir_stream> __imp_;
+};
+
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const directory_iterator& __lhs, const directory_iterator& __rhs) noexcept {
+  return __lhs.__imp_ == __rhs.__imp_;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const directory_iterator& __lhs, const directory_iterator& __rhs) noexcept {
+  return !(__lhs == __rhs);
+}
+
+// enable directory_iterator range-based for statements
+inline _LIBCPP_HIDE_FROM_ABI directory_iterator begin(directory_iterator __iter) noexcept { return __iter; }
+
+inline _LIBCPP_HIDE_FROM_ABI directory_iterator end(directory_iterator) noexcept { return directory_iterator(); }
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#  if _LIBCPP_STD_VER >= 20
+
+template <>
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY inline constexpr bool
+    std::ranges::enable_borrowed_range<std::filesystem::directory_iterator> = true;
+
+template <>
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY inline constexpr bool
+    std::ranges::enable_view<std::filesystem::directory_iterator> = true;
+
+#  endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP_STD_VER >= 17 && !defined(_LIBCPP_HAS_NO_FILESYSTEM)
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FILESYSTEM_DIRECTORY_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__filesystem/directory_options.h b/libcxx/include/__cxx03/__filesystem/directory_options.h
new file mode 100644
index 000000000000..d0cd3ebfdaa7
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/directory_options.h
@@ -0,0 +1,57 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_DIRECTORY_OPTIONS_H
+#define _LIBCPP___FILESYSTEM_DIRECTORY_OPTIONS_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+enum class directory_options : unsigned char { none = 0, follow_directory_symlink = 1, skip_permission_denied = 2 };
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr directory_options operator&(directory_options __lhs, directory_options __rhs) {
+  return static_cast<directory_options>(static_cast<unsigned char>(__lhs) & static_cast<unsigned char>(__rhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr directory_options operator|(directory_options __lhs, directory_options __rhs) {
+  return static_cast<directory_options>(static_cast<unsigned char>(__lhs) | static_cast<unsigned char>(__rhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr directory_options operator^(directory_options __lhs, directory_options __rhs) {
+  return static_cast<directory_options>(static_cast<unsigned char>(__lhs) ^ static_cast<unsigned char>(__rhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr directory_options operator~(directory_options __lhs) {
+  return static_cast<directory_options>(~static_cast<unsigned char>(__lhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline directory_options& operator&=(directory_options& __lhs, directory_options __rhs) {
+  return __lhs = __lhs & __rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline directory_options& operator|=(directory_options& __lhs, directory_options __rhs) {
+  return __lhs = __lhs | __rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline directory_options& operator^=(directory_options& __lhs, directory_options __rhs) {
+  return __lhs = __lhs ^ __rhs;
+}
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___FILESYSTEM_DIRECTORY_OPTIONS_H
diff --git a/libcxx/include/__cxx03/__filesystem/file_status.h b/libcxx/include/__cxx03/__filesystem/file_status.h
new file mode 100644
index 000000000000..da316c8b0274
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/file_status.h
@@ -0,0 +1,67 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_FILE_STATUS_H
+#define _LIBCPP___FILESYSTEM_FILE_STATUS_H
+
+#include <__config>
+#include <__filesystem/file_type.h>
+#include <__filesystem/perms.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+class _LIBCPP_EXPORTED_FROM_ABI file_status {
+public:
+  // constructors
+  _LIBCPP_HIDE_FROM_ABI file_status() noexcept : file_status(file_type::none) {}
+  _LIBCPP_HIDE_FROM_ABI explicit file_status(file_type __ft, perms __prms = perms::unknown) noexcept
+      : __ft_(__ft), __prms_(__prms) {}
+
+  _LIBCPP_HIDE_FROM_ABI file_status(const file_status&) noexcept = default;
+  _LIBCPP_HIDE_FROM_ABI file_status(file_status&&) noexcept      = default;
+
+  _LIBCPP_HIDE_FROM_ABI ~file_status() {}
+
+  _LIBCPP_HIDE_FROM_ABI file_status& operator=(const file_status&) noexcept = default;
+  _LIBCPP_HIDE_FROM_ABI file_status& operator=(file_status&&) noexcept      = default;
+
+  // observers
+  _LIBCPP_HIDE_FROM_ABI file_type type() const noexcept { return __ft_; }
+
+  _LIBCPP_HIDE_FROM_ABI perms permissions() const noexcept { return __prms_; }
+
+  // modifiers
+  _LIBCPP_HIDE_FROM_ABI void type(file_type __ft) noexcept { __ft_ = __ft; }
+
+  _LIBCPP_HIDE_FROM_ABI void permissions(perms __p) noexcept { __prms_ = __p; }
+
+#  if _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI friend bool operator==(const file_status& __lhs, const file_status& __rhs) noexcept {
+    return __lhs.type() == __rhs.type() && __lhs.permissions() == __rhs.permissions();
+  }
+
+#  endif
+
+private:
+  file_type __ft_;
+  perms __prms_;
+};
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___FILESYSTEM_FILE_STATUS_H
diff --git a/libcxx/include/__cxx03/__filesystem/file_time_type.h b/libcxx/include/__cxx03/__filesystem/file_time_type.h
new file mode 100644
index 000000000000..63e4ae1578cf
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/file_time_type.h
@@ -0,0 +1,31 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_FILE_TIME_TYPE_H
+#define _LIBCPP___FILESYSTEM_FILE_TIME_TYPE_H
+
+#include <__chrono/file_clock.h>
+#include <__chrono/time_point.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+typedef chrono::time_point<_FilesystemClock> file_time_type;
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___FILESYSTEM_FILE_TIME_TYPE_H
diff --git a/libcxx/include/__cxx03/__filesystem/file_type.h b/libcxx/include/__cxx03/__filesystem/file_type.h
new file mode 100644
index 000000000000..e4ac1dfee9ed
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/file_type.h
@@ -0,0 +1,42 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_FILE_TYPE_H
+#define _LIBCPP___FILESYSTEM_FILE_TYPE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+// On Windows, the library never identifies files as block, character, fifo
+// or socket.
+enum class file_type : signed char {
+  none      = 0,
+  not_found = -1,
+  regular   = 1,
+  directory = 2,
+  symlink   = 3,
+  block     = 4,
+  character = 5,
+  fifo      = 6,
+  socket    = 7,
+  unknown   = 8
+};
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___FILESYSTEM_FILE_TYPE_H
diff --git a/libcxx/include/__cxx03/__filesystem/filesystem_error.h b/libcxx/include/__cxx03/__filesystem/filesystem_error.h
new file mode 100644
index 000000000000..80a11e3b1932
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/filesystem_error.h
@@ -0,0 +1,88 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_FILESYSTEM_ERROR_H
+#define _LIBCPP___FILESYSTEM_FILESYSTEM_ERROR_H
+
+#include <__config>
+#include <__filesystem/path.h>
+#include <__memory/shared_ptr.h>
+#include <__system_error/error_code.h>
+#include <__system_error/system_error.h>
+#include <__utility/forward.h>
+#include <__verbose_abort>
+#include <string>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+class _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_EXPORTED_FROM_ABI filesystem_error : public system_error {
+public:
+  _LIBCPP_HIDE_FROM_ABI filesystem_error(const string& __what, error_code __ec)
+      : system_error(__ec, __what), __storage_(make_shared<_Storage>(path(), path())) {
+    __create_what(0);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI filesystem_error(const string& __what, const path& __p1, error_code __ec)
+      : system_error(__ec, __what), __storage_(make_shared<_Storage>(__p1, path())) {
+    __create_what(1);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI filesystem_error(const string& __what, const path& __p1, const path& __p2, error_code __ec)
+      : system_error(__ec, __what), __storage_(make_shared<_Storage>(__p1, __p2)) {
+    __create_what(2);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const path& path1() const noexcept { return __storage_->__p1_; }
+
+  _LIBCPP_HIDE_FROM_ABI const path& path2() const noexcept { return __storage_->__p2_; }
+
+  _LIBCPP_HIDE_FROM_ABI filesystem_error(const filesystem_error&) = default;
+  ~filesystem_error() override; // key function
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL
+  const char* what() const noexcept override { return __storage_->__what_.c_str(); }
+
+  void __create_what(int __num_paths);
+
+private:
+  struct _LIBCPP_HIDDEN _Storage {
+    _LIBCPP_HIDE_FROM_ABI _Storage(const path& __p1, const path& __p2) : __p1_(__p1), __p2_(__p2) {}
+
+    path __p1_;
+    path __p2_;
+    string __what_;
+  };
+  shared_ptr<_Storage> __storage_;
+};
+
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+template <class... _Args>
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY void
+__throw_filesystem_error(_Args&&... __args) {
+  throw filesystem_error(std::forward<_Args>(__args)...);
+}
+#  else
+template <class... _Args>
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY void
+__throw_filesystem_error(_Args&&...) {
+  _LIBCPP_VERBOSE_ABORT("filesystem_error was thrown in -fno-exceptions mode");
+}
+#  endif
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___FILESYSTEM_FILESYSTEM_ERROR_H
diff --git a/libcxx/include/__cxx03/__filesystem/operations.h b/libcxx/include/__cxx03/__filesystem/operations.h
new file mode 100644
index 000000000000..f588189ed1d9
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/operations.h
@@ -0,0 +1,310 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_OPERATIONS_H
+#define _LIBCPP___FILESYSTEM_OPERATIONS_H
+
+#include <__chrono/time_point.h>
+#include <__config>
+#include <__filesystem/copy_options.h>
+#include <__filesystem/file_status.h>
+#include <__filesystem/file_time_type.h>
+#include <__filesystem/file_type.h>
+#include <__filesystem/path.h>
+#include <__filesystem/perm_options.h>
+#include <__filesystem/perms.h>
+#include <__filesystem/space_info.h>
+#include <__system_error/error_code.h>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17 && !defined(_LIBCPP_HAS_NO_FILESYSTEM)
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH
+
+_LIBCPP_EXPORTED_FROM_ABI path __absolute(const path&, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI path __canonical(const path&, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI bool
+__copy_file(const path& __from, const path& __to, copy_options __opt, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI void
+__copy_symlink(const path& __existing_symlink, const path& __new_symlink, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI void
+__copy(const path& __from, const path& __to, copy_options __opt, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI bool __create_directories(const path&, error_code* = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI void
+__create_directory_symlink(const path& __to, const path& __new_symlink, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI bool __create_directory(const path&, error_code* = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI bool __create_directory(const path&, const path& __attributes, error_code* = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI void
+__create_hard_link(const path& __to, const path& __new_hard_link, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI void
+__create_symlink(const path& __to, const path& __new_symlink, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI path __current_path(error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI void __current_path(const path&, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI bool __equivalent(const path&, const path&, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI file_status __status(const path&, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI uintmax_t __file_size(const path&, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI uintmax_t __hard_link_count(const path&, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI file_status __symlink_status(const path&, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI file_time_type __last_write_time(const path&, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI void __last_write_time(const path&, file_time_type __new_time, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI path __weakly_canonical(path const& __p, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI path __read_symlink(const path&, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI uintmax_t __remove_all(const path&, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI bool __remove(const path&, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI void __rename(const path& __from, const path& __to, error_code* __ec = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI void __resize_file(const path&, uintmax_t __size, error_code* = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI path __temp_directory_path(error_code* __ec = nullptr);
+
+inline _LIBCPP_HIDE_FROM_ABI path absolute(const path& __p) { return __absolute(__p); }
+inline _LIBCPP_HIDE_FROM_ABI path absolute(const path& __p, error_code& __ec) { return __absolute(__p, &__ec); }
+inline _LIBCPP_HIDE_FROM_ABI path canonical(const path& __p) { return __canonical(__p); }
+inline _LIBCPP_HIDE_FROM_ABI path canonical(const path& __p, error_code& __ec) { return __canonical(__p, &__ec); }
+inline _LIBCPP_HIDE_FROM_ABI bool copy_file(const path& __from, const path& __to) {
+  return __copy_file(__from, __to, copy_options::none);
+}
+inline _LIBCPP_HIDE_FROM_ABI bool copy_file(const path& __from, const path& __to, error_code& __ec) {
+  return __copy_file(__from, __to, copy_options::none, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI bool copy_file(const path& __from, const path& __to, copy_options __opt) {
+  return __copy_file(__from, __to, __opt);
+}
+inline _LIBCPP_HIDE_FROM_ABI bool
+copy_file(const path& __from, const path& __to, copy_options __opt, error_code& __ec) {
+  return __copy_file(__from, __to, __opt, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI void copy_symlink(const path& __from, const path& __to) { __copy_symlink(__from, __to); }
+inline _LIBCPP_HIDE_FROM_ABI void copy_symlink(const path& __from, const path& __to, error_code& __ec) noexcept {
+  __copy_symlink(__from, __to, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI void copy(const path& __from, const path& __to) {
+  __copy(__from, __to, copy_options::none);
+}
+inline _LIBCPP_HIDE_FROM_ABI void copy(const path& __from, const path& __to, error_code& __ec) {
+  __copy(__from, __to, copy_options::none, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI void copy(const path& __from, const path& __to, copy_options __opt) {
+  __copy(__from, __to, __opt);
+}
+inline _LIBCPP_HIDE_FROM_ABI void copy(const path& __from, const path& __to, copy_options __opt, error_code& __ec) {
+  __copy(__from, __to, __opt, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI bool create_directories(const path& __p) { return __create_directories(__p); }
+inline _LIBCPP_HIDE_FROM_ABI bool create_directories(const path& __p, error_code& __ec) {
+  return __create_directories(__p, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI void create_directory_symlink(const path& __target, const path& __link) {
+  __create_directory_symlink(__target, __link);
+}
+inline _LIBCPP_HIDE_FROM_ABI void
+create_directory_symlink(const path& __target, const path& __link, error_code& __ec) noexcept {
+  __create_directory_symlink(__target, __link, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI bool create_directory(const path& __p) { return __create_directory(__p); }
+inline _LIBCPP_HIDE_FROM_ABI bool create_directory(const path& __p, error_code& __ec) noexcept {
+  return __create_directory(__p, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI bool create_directory(const path& __p, const path& __attrs) {
+  return __create_directory(__p, __attrs);
+}
+inline _LIBCPP_HIDE_FROM_ABI bool create_directory(const path& __p, const path& __attrs, error_code& __ec) noexcept {
+  return __create_directory(__p, __attrs, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI void create_hard_link(const path& __target, const path& __link) {
+  __create_hard_link(__target, __link);
+}
+inline _LIBCPP_HIDE_FROM_ABI void
+create_hard_link(const path& __target, const path& __link, error_code& __ec) noexcept {
+  __create_hard_link(__target, __link, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI void create_symlink(const path& __target, const path& __link) {
+  __create_symlink(__target, __link);
+}
+inline _LIBCPP_HIDE_FROM_ABI void create_symlink(const path& __target, const path& __link, error_code& __ec) noexcept {
+  return __create_symlink(__target, __link, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI path current_path() { return __current_path(); }
+inline _LIBCPP_HIDE_FROM_ABI path current_path(error_code& __ec) { return __current_path(&__ec); }
+inline _LIBCPP_HIDE_FROM_ABI void current_path(const path& __p) { __current_path(__p); }
+inline _LIBCPP_HIDE_FROM_ABI void current_path(const path& __p, error_code& __ec) noexcept {
+  __current_path(__p, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI bool equivalent(const path& __p1, const path& __p2) { return __equivalent(__p1, __p2); }
+inline _LIBCPP_HIDE_FROM_ABI bool equivalent(const path& __p1, const path& __p2, error_code& __ec) noexcept {
+  return __equivalent(__p1, __p2, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI bool status_known(file_status __s) noexcept { return __s.type() != file_type::none; }
+inline _LIBCPP_HIDE_FROM_ABI bool exists(file_status __s) noexcept {
+  return status_known(__s) && __s.type() != file_type::not_found;
+}
+inline _LIBCPP_HIDE_FROM_ABI bool exists(const path& __p) { return exists(__status(__p)); }
+
+inline _LIBCPP_HIDE_FROM_ABI bool exists(const path& __p, error_code& __ec) noexcept {
+  auto __s = __status(__p, &__ec);
+  if (status_known(__s))
+    __ec.clear();
+  return exists(__s);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI uintmax_t file_size(const path& __p) { return __file_size(__p); }
+inline _LIBCPP_HIDE_FROM_ABI uintmax_t file_size(const path& __p, error_code& __ec) noexcept {
+  return __file_size(__p, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count(const path& __p) { return __hard_link_count(__p); }
+inline _LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count(const path& __p, error_code& __ec) noexcept {
+  return __hard_link_count(__p, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI bool is_block_file(file_status __s) noexcept { return __s.type() == file_type::block; }
+inline _LIBCPP_HIDE_FROM_ABI bool is_block_file(const path& __p) { return is_block_file(__status(__p)); }
+inline _LIBCPP_HIDE_FROM_ABI bool is_block_file(const path& __p, error_code& __ec) noexcept {
+  return is_block_file(__status(__p, &__ec));
+}
+inline _LIBCPP_HIDE_FROM_ABI bool is_character_file(file_status __s) noexcept {
+  return __s.type() == file_type::character;
+}
+inline _LIBCPP_HIDE_FROM_ABI bool is_character_file(const path& __p) { return is_character_file(__status(__p)); }
+inline _LIBCPP_HIDE_FROM_ABI bool is_character_file(const path& __p, error_code& __ec) noexcept {
+  return is_character_file(__status(__p, &__ec));
+}
+inline _LIBCPP_HIDE_FROM_ABI bool is_directory(file_status __s) noexcept { return __s.type() == file_type::directory; }
+inline _LIBCPP_HIDE_FROM_ABI bool is_directory(const path& __p) { return is_directory(__status(__p)); }
+inline _LIBCPP_HIDE_FROM_ABI bool is_directory(const path& __p, error_code& __ec) noexcept {
+  return is_directory(__status(__p, &__ec));
+}
+_LIBCPP_EXPORTED_FROM_ABI bool __fs_is_empty(const path& __p, error_code* __ec = nullptr);
+inline _LIBCPP_HIDE_FROM_ABI bool is_empty(const path& __p) { return __fs_is_empty(__p); }
+inline _LIBCPP_HIDE_FROM_ABI bool is_empty(const path& __p, error_code& __ec) { return __fs_is_empty(__p, &__ec); }
+inline _LIBCPP_HIDE_FROM_ABI bool is_fifo(file_status __s) noexcept { return __s.type() == file_type::fifo; }
+inline _LIBCPP_HIDE_FROM_ABI bool is_fifo(const path& __p) { return is_fifo(__status(__p)); }
+inline _LIBCPP_HIDE_FROM_ABI bool is_fifo(const path& __p, error_code& __ec) noexcept {
+  return is_fifo(__status(__p, &__ec));
+}
+inline _LIBCPP_HIDE_FROM_ABI bool is_regular_file(file_status __s) noexcept { return __s.type() == file_type::regular; }
+inline _LIBCPP_HIDE_FROM_ABI bool is_regular_file(const path& __p) { return is_regular_file(__status(__p)); }
+inline _LIBCPP_HIDE_FROM_ABI bool is_regular_file(const path& __p, error_code& __ec) noexcept {
+  return is_regular_file(__status(__p, &__ec));
+}
+inline _LIBCPP_HIDE_FROM_ABI bool is_symlink(file_status __s) noexcept { return __s.type() == file_type::symlink; }
+inline _LIBCPP_HIDE_FROM_ABI bool is_symlink(const path& __p) { return is_symlink(__symlink_status(__p)); }
+inline _LIBCPP_HIDE_FROM_ABI bool is_symlink(const path& __p, error_code& __ec) noexcept {
+  return is_symlink(__symlink_status(__p, &__ec));
+}
+inline _LIBCPP_HIDE_FROM_ABI bool is_other(file_status __s) noexcept {
+  return exists(__s) && !is_regular_file(__s) && !is_directory(__s) && !is_symlink(__s);
+}
+inline _LIBCPP_HIDE_FROM_ABI bool is_other(const path& __p) { return is_other(__status(__p)); }
+inline _LIBCPP_HIDE_FROM_ABI bool is_other(const path& __p, error_code& __ec) noexcept {
+  return is_other(__status(__p, &__ec));
+}
+inline _LIBCPP_HIDE_FROM_ABI bool is_socket(file_status __s) noexcept { return __s.type() == file_type::socket; }
+inline _LIBCPP_HIDE_FROM_ABI bool is_socket(const path& __p) { return is_socket(__status(__p)); }
+inline _LIBCPP_HIDE_FROM_ABI bool is_socket(const path& __p, error_code& __ec) noexcept {
+  return is_socket(__status(__p, &__ec));
+}
+inline _LIBCPP_HIDE_FROM_ABI file_time_type last_write_time(const path& __p) { return __last_write_time(__p); }
+inline _LIBCPP_HIDE_FROM_ABI file_time_type last_write_time(const path& __p, error_code& __ec) noexcept {
+  return __last_write_time(__p, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI void last_write_time(const path& __p, file_time_type __t) { __last_write_time(__p, __t); }
+inline _LIBCPP_HIDE_FROM_ABI void last_write_time(const path& __p, file_time_type __t, error_code& __ec) noexcept {
+  __last_write_time(__p, __t, &__ec);
+}
+_LIBCPP_EXPORTED_FROM_ABI void __permissions(const path&, perms, perm_options, error_code* = nullptr);
+inline _LIBCPP_HIDE_FROM_ABI void
+permissions(const path& __p, perms __prms, perm_options __opts = perm_options::replace) {
+  __permissions(__p, __prms, __opts);
+}
+inline _LIBCPP_HIDE_FROM_ABI void permissions(const path& __p, perms __prms, error_code& __ec) noexcept {
+  __permissions(__p, __prms, perm_options::replace, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI void permissions(const path& __p, perms __prms, perm_options __opts, error_code& __ec) {
+  __permissions(__p, __prms, __opts, &__ec);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, const path& __base, error_code& __ec) {
+  path __tmp = __weakly_canonical(__p, &__ec);
+  if (__ec)
+    return {};
+  path __tmp_base = __weakly_canonical(__base, &__ec);
+  if (__ec)
+    return {};
+  return __tmp.lexically_proximate(__tmp_base);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, error_code& __ec) {
+  return proximate(__p, current_path(), __ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, const path& __base = current_path()) {
+  return __weakly_canonical(__p).lexically_proximate(__weakly_canonical(__base));
+}
+inline _LIBCPP_HIDE_FROM_ABI path read_symlink(const path& __p) { return __read_symlink(__p); }
+inline _LIBCPP_HIDE_FROM_ABI path read_symlink(const path& __p, error_code& __ec) { return __read_symlink(__p, &__ec); }
+
+inline _LIBCPP_HIDE_FROM_ABI path relative(const path& __p, const path& __base, error_code& __ec) {
+  path __tmp = __weakly_canonical(__p, &__ec);
+  if (__ec)
+    return path();
+  path __tmpbase = __weakly_canonical(__base, &__ec);
+  if (__ec)
+    return path();
+  return __tmp.lexically_relative(__tmpbase);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI path relative(const path& __p, error_code& __ec) {
+  return relative(__p, current_path(), __ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI path relative(const path& __p, const path& __base = current_path()) {
+  return __weakly_canonical(__p).lexically_relative(__weakly_canonical(__base));
+}
+inline _LIBCPP_HIDE_FROM_ABI uintmax_t remove_all(const path& __p) { return __remove_all(__p); }
+inline _LIBCPP_HIDE_FROM_ABI uintmax_t remove_all(const path& __p, error_code& __ec) {
+  return __remove_all(__p, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI bool remove(const path& __p) { return __remove(__p); }
+inline _LIBCPP_HIDE_FROM_ABI bool remove(const path& __p, error_code& __ec) noexcept { return __remove(__p, &__ec); }
+inline _LIBCPP_HIDE_FROM_ABI void rename(const path& __from, const path& __to) { return __rename(__from, __to); }
+inline _LIBCPP_HIDE_FROM_ABI void rename(const path& __from, const path& __to, error_code& __ec) noexcept {
+  return __rename(__from, __to, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI void resize_file(const path& __p, uintmax_t __ns) { return __resize_file(__p, __ns); }
+inline _LIBCPP_HIDE_FROM_ABI void resize_file(const path& __p, uintmax_t __ns, error_code& __ec) noexcept {
+  return __resize_file(__p, __ns, &__ec);
+}
+_LIBCPP_EXPORTED_FROM_ABI space_info __space(const path&, error_code* __ec = nullptr);
+inline _LIBCPP_HIDE_FROM_ABI space_info space(const path& __p) { return __space(__p); }
+inline _LIBCPP_HIDE_FROM_ABI space_info space(const path& __p, error_code& __ec) noexcept {
+  return __space(__p, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI file_status status(const path& __p) { return __status(__p); }
+inline _LIBCPP_HIDE_FROM_ABI file_status status(const path& __p, error_code& __ec) noexcept {
+  return __status(__p, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI file_status symlink_status(const path& __p) { return __symlink_status(__p); }
+inline _LIBCPP_HIDE_FROM_ABI file_status symlink_status(const path& __p, error_code& __ec) noexcept {
+  return __symlink_status(__p, &__ec);
+}
+inline _LIBCPP_HIDE_FROM_ABI path temp_directory_path() { return __temp_directory_path(); }
+inline _LIBCPP_HIDE_FROM_ABI path temp_directory_path(error_code& __ec) { return __temp_directory_path(&__ec); }
+inline _LIBCPP_HIDE_FROM_ABI path weakly_canonical(path const& __p) { return __weakly_canonical(__p); }
+inline _LIBCPP_HIDE_FROM_ABI path weakly_canonical(path const& __p, error_code& __ec) {
+  return __weakly_canonical(__p, &__ec);
+}
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_STD_VER >= 17 && !defined(_LIBCPP_HAS_NO_FILESYSTEM)
+
+#endif // _LIBCPP___FILESYSTEM_OPERATIONS_H
diff --git a/libcxx/include/__cxx03/__filesystem/path.h b/libcxx/include/__cxx03/__filesystem/path.h
new file mode 100644
index 000000000000..ff468d517722
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/path.h
@@ -0,0 +1,931 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_PATH_H
+#define _LIBCPP___FILESYSTEM_PATH_H
+
+#include <__algorithm/replace.h>
+#include <__algorithm/replace_copy.h>
+#include <__config>
+#include <__functional/unary_function.h>
+#include <__fwd/functional.h>
+#include <__iterator/back_insert_iterator.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/remove_const.h>
+#include <__type_traits/remove_pointer.h>
+#include <cstddef>
+#include <string>
+#include <string_view>
+
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <iomanip> // for quoted
+#  include <locale>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH
+
+template <class _Tp>
+struct __can_convert_char {
+  static const bool value = false;
+};
+template <class _Tp>
+struct __can_convert_char<const _Tp> : public __can_convert_char<_Tp> {};
+template <>
+struct __can_convert_char<char> {
+  static const bool value = true;
+  using __char_type       = char;
+};
+template <>
+struct __can_convert_char<wchar_t> {
+  static const bool value = true;
+  using __char_type       = wchar_t;
+};
+#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+template <>
+struct __can_convert_char<char8_t> {
+  static const bool value = true;
+  using __char_type       = char8_t;
+};
+#  endif
+template <>
+struct __can_convert_char<char16_t> {
+  static const bool value = true;
+  using __char_type       = char16_t;
+};
+template <>
+struct __can_convert_char<char32_t> {
+  static const bool value = true;
+  using __char_type       = char32_t;
+};
+
+template <class _ECharT, __enable_if_t<__can_convert_char<_ECharT>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI bool __is_separator(_ECharT __e) {
+#  if defined(_LIBCPP_WIN32API)
+  return __e == _ECharT('/') || __e == _ECharT('\\');
+#  else
+  return __e == _ECharT('/');
+#  endif
+}
+
+#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+typedef u8string __u8_string;
+#  else
+typedef string __u8_string;
+#  endif
+
+struct _NullSentinel {};
+
+template <class _Tp>
+using _Void = void;
+
+template <class _Tp, class = void>
+struct __is_pathable_string : public false_type {};
+
+template <class _ECharT, class _Traits, class _Alloc>
+struct __is_pathable_string< basic_string<_ECharT, _Traits, _Alloc>,
+                             _Void<typename __can_convert_char<_ECharT>::__char_type> >
+    : public __can_convert_char<_ECharT> {
+  using _Str = basic_string<_ECharT, _Traits, _Alloc>;
+
+  _LIBCPP_HIDE_FROM_ABI static _ECharT const* __range_begin(_Str const& __s) { return __s.data(); }
+
+  _LIBCPP_HIDE_FROM_ABI static _ECharT const* __range_end(_Str const& __s) { return __s.data() + __s.length(); }
+
+  _LIBCPP_HIDE_FROM_ABI static _ECharT __first_or_null(_Str const& __s) { return __s.empty() ? _ECharT{} : __s[0]; }
+};
+
+template <class _ECharT, class _Traits>
+struct __is_pathable_string< basic_string_view<_ECharT, _Traits>,
+                             _Void<typename __can_convert_char<_ECharT>::__char_type> >
+    : public __can_convert_char<_ECharT> {
+  using _Str = basic_string_view<_ECharT, _Traits>;
+
+  _LIBCPP_HIDE_FROM_ABI static _ECharT const* __range_begin(_Str const& __s) { return __s.data(); }
+
+  _LIBCPP_HIDE_FROM_ABI static _ECharT const* __range_end(_Str const& __s) { return __s.data() + __s.length(); }
+
+  _LIBCPP_HIDE_FROM_ABI static _ECharT __first_or_null(_Str const& __s) { return __s.empty() ? _ECharT{} : __s[0]; }
+};
+
+template <class _Source,
+          class _DS            = __decay_t<_Source>,
+          class _UnqualPtrType = __remove_const_t<__remove_pointer_t<_DS> >,
+          bool _IsCharPtr      = is_pointer<_DS>::value && __can_convert_char<_UnqualPtrType>::value>
+struct __is_pathable_char_array : false_type {};
+
+template <class _Source, class _ECharT, class _UPtr>
+struct __is_pathable_char_array<_Source, _ECharT*, _UPtr, true> : __can_convert_char<__remove_const_t<_ECharT> > {
+  _LIBCPP_HIDE_FROM_ABI static _ECharT const* __range_begin(const _ECharT* __b) { return __b; }
+
+  _LIBCPP_HIDE_FROM_ABI static _ECharT const* __range_end(const _ECharT* __b) {
+    using _Iter              = const _ECharT*;
+    const _ECharT __sentinel = _ECharT{};
+    _Iter __e                = __b;
+    for (; *__e != __sentinel; ++__e)
+      ;
+    return __e;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static _ECharT __first_or_null(const _ECharT* __b) { return *__b; }
+};
+
+template <class _Iter, bool _IsIt = __has_input_iterator_category<_Iter>::value, class = void>
+struct __is_pathable_iter : false_type {};
+
+template <class _Iter>
+struct __is_pathable_iter<
+    _Iter,
+    true,
+    _Void<typename __can_convert_char< typename iterator_traits<_Iter>::value_type>::__char_type> >
+    : __can_convert_char<typename iterator_traits<_Iter>::value_type> {
+  using _ECharT = typename iterator_traits<_Iter>::value_type;
+
+  _LIBCPP_HIDE_FROM_ABI static _Iter __range_begin(_Iter __b) { return __b; }
+
+  _LIBCPP_HIDE_FROM_ABI static _NullSentinel __range_end(_Iter) { return _NullSentinel{}; }
+
+  _LIBCPP_HIDE_FROM_ABI static _ECharT __first_or_null(_Iter __b) { return *__b; }
+};
+
+template <class _Tp,
+          bool _IsStringT   = __is_pathable_string<_Tp>::value,
+          bool _IsCharIterT = __is_pathable_char_array<_Tp>::value,
+          bool _IsIterT     = !_IsCharIterT && __is_pathable_iter<_Tp>::value>
+struct __is_pathable : false_type {
+  static_assert(!_IsStringT && !_IsCharIterT && !_IsIterT, "Must all be false");
+};
+
+template <class _Tp>
+struct __is_pathable<_Tp, true, false, false> : __is_pathable_string<_Tp> {};
+
+template <class _Tp>
+struct __is_pathable<_Tp, false, true, false> : __is_pathable_char_array<_Tp> {};
+
+template <class _Tp>
+struct __is_pathable<_Tp, false, false, true> : __is_pathable_iter<_Tp> {};
+
+#  if defined(_LIBCPP_WIN32API)
+typedef wstring __path_string;
+typedef wchar_t __path_value;
+#  else
+typedef string __path_string;
+typedef char __path_value;
+#  endif
+
+#  if defined(_LIBCPP_WIN32API)
+_LIBCPP_EXPORTED_FROM_ABI size_t __wide_to_char(const wstring&, char*, size_t);
+_LIBCPP_EXPORTED_FROM_ABI size_t __char_to_wide(const string&, wchar_t*, size_t);
+#  endif
+
+template <class _ECharT>
+struct _PathCVT;
+
+#  if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+template <class _ECharT>
+struct _PathCVT {
+  static_assert(__can_convert_char<_ECharT>::value, "Char type not convertible");
+
+  typedef __narrow_to_utf8<sizeof(_ECharT) * __CHAR_BIT__> _Narrower;
+#    if defined(_LIBCPP_WIN32API)
+  typedef __widen_from_utf8<sizeof(wchar_t) * __CHAR_BIT__> _Widener;
+#    endif
+
+  _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _ECharT const* __b, _ECharT const* __e) {
+#    if defined(_LIBCPP_WIN32API)
+    string __utf8;
+    _Narrower()(back_inserter(__utf8), __b, __e);
+    _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size());
+#    else
+    _Narrower()(back_inserter(__dest), __b, __e);
+#    endif
+  }
+
+  template <class _Iter>
+  _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _Iter __e) {
+    static_assert(!is_same<_Iter, _ECharT*>::value, "Call const overload");
+    if (__b == __e)
+      return;
+    basic_string<_ECharT> __tmp(__b, __e);
+#    if defined(_LIBCPP_WIN32API)
+    string __utf8;
+    _Narrower()(back_inserter(__utf8), __tmp.data(), __tmp.data() + __tmp.length());
+    _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size());
+#    else
+    _Narrower()(back_inserter(__dest), __tmp.data(), __tmp.data() + __tmp.length());
+#    endif
+  }
+
+  template <class _Iter>
+  _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) {
+    static_assert(!is_same<_Iter, _ECharT*>::value, "Call const overload");
+    const _ECharT __sentinel = _ECharT{};
+    if (*__b == __sentinel)
+      return;
+    basic_string<_ECharT> __tmp;
+    for (; *__b != __sentinel; ++__b)
+      __tmp.push_back(*__b);
+#    if defined(_LIBCPP_WIN32API)
+    string __utf8;
+    _Narrower()(back_inserter(__utf8), __tmp.data(), __tmp.data() + __tmp.length());
+    _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size());
+#    else
+    _Narrower()(back_inserter(__dest), __tmp.data(), __tmp.data() + __tmp.length());
+#    endif
+  }
+
+  template <class _Source>
+  _LIBCPP_HIDE_FROM_ABI static void __append_source(__path_string& __dest, _Source const& __s) {
+    using _Traits = __is_pathable<_Source>;
+    __append_range(__dest, _Traits::__range_begin(__s), _Traits::__range_end(__s));
+  }
+};
+#  endif // !_LIBCPP_HAS_NO_LOCALIZATION
+
+template <>
+struct _PathCVT<__path_value> {
+  template <class _Iter, __enable_if_t<__has_exactly_input_iterator_category<_Iter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _Iter __e) {
+    for (; __b != __e; ++__b)
+      __dest.push_back(*__b);
+  }
+
+  template <class _Iter, __enable_if_t<__has_forward_iterator_category<_Iter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _Iter __e) {
+    __dest.append(__b, __e);
+  }
+
+  template <class _Iter>
+  _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) {
+    const char __sentinel = char{};
+    for (; *__b != __sentinel; ++__b)
+      __dest.push_back(*__b);
+  }
+
+  template <class _Source>
+  _LIBCPP_HIDE_FROM_ABI static void __append_source(__path_string& __dest, _Source const& __s) {
+    using _Traits = __is_pathable<_Source>;
+    __append_range(__dest, _Traits::__range_begin(__s), _Traits::__range_end(__s));
+  }
+};
+
+#  if defined(_LIBCPP_WIN32API)
+template <>
+struct _PathCVT<char> {
+  _LIBCPP_HIDE_FROM_ABI static void __append_string(__path_string& __dest, const basic_string<char>& __str) {
+    size_t __size = __char_to_wide(__str, nullptr, 0);
+    size_t __pos  = __dest.size();
+    __dest.resize(__pos + __size);
+    __char_to_wide(__str, const_cast<__path_value*>(__dest.data()) + __pos, __size);
+  }
+
+  template <class _Iter, __enable_if_t<__has_exactly_input_iterator_category<_Iter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _Iter __e) {
+    basic_string<char> __tmp(__b, __e);
+    __append_string(__dest, __tmp);
+  }
+
+  template <class _Iter, __enable_if_t<__has_forward_iterator_category<_Iter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _Iter __e) {
+    basic_string<char> __tmp(__b, __e);
+    __append_string(__dest, __tmp);
+  }
+
+  template <class _Iter>
+  _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) {
+    const char __sentinel = char{};
+    basic_string<char> __tmp;
+    for (; *__b != __sentinel; ++__b)
+      __tmp.push_back(*__b);
+    __append_string(__dest, __tmp);
+  }
+
+  template <class _Source>
+  _LIBCPP_HIDE_FROM_ABI static void __append_source(__path_string& __dest, _Source const& __s) {
+    using _Traits = __is_pathable<_Source>;
+    __append_range(__dest, _Traits::__range_begin(__s), _Traits::__range_end(__s));
+  }
+};
+
+template <class _ECharT>
+struct _PathExport {
+  typedef __narrow_to_utf8<sizeof(wchar_t) * __CHAR_BIT__> _Narrower;
+  typedef __widen_from_utf8<sizeof(_ECharT) * __CHAR_BIT__> _Widener;
+
+  template <class _Str>
+  _LIBCPP_HIDE_FROM_ABI static void __append(_Str& __dest, const __path_string& __src) {
+    string __utf8;
+    _Narrower()(back_inserter(__utf8), __src.data(), __src.data() + __src.size());
+    _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size());
+  }
+};
+
+template <>
+struct _PathExport<char> {
+  template <class _Str>
+  _LIBCPP_HIDE_FROM_ABI static void __append(_Str& __dest, const __path_string& __src) {
+    size_t __size = __wide_to_char(__src, nullptr, 0);
+    size_t __pos  = __dest.size();
+    __dest.resize(__size);
+    __wide_to_char(__src, const_cast<char*>(__dest.data()) + __pos, __size);
+  }
+};
+
+template <>
+struct _PathExport<wchar_t> {
+  template <class _Str>
+  _LIBCPP_HIDE_FROM_ABI static void __append(_Str& __dest, const __path_string& __src) {
+    __dest.append(__src.begin(), __src.end());
+  }
+};
+
+template <>
+struct _PathExport<char16_t> {
+  template <class _Str>
+  _LIBCPP_HIDE_FROM_ABI static void __append(_Str& __dest, const __path_string& __src) {
+    __dest.append(__src.begin(), __src.end());
+  }
+};
+
+#    ifndef _LIBCPP_HAS_NO_CHAR8_T
+template <>
+struct _PathExport<char8_t> {
+  typedef __narrow_to_utf8<sizeof(wchar_t) * __CHAR_BIT__> _Narrower;
+
+  template <class _Str>
+  _LIBCPP_HIDE_FROM_ABI static void __append(_Str& __dest, const __path_string& __src) {
+    _Narrower()(back_inserter(__dest), __src.data(), __src.data() + __src.size());
+  }
+};
+#    endif /* !_LIBCPP_HAS_NO_CHAR8_T */
+#  endif   /* _LIBCPP_WIN32API */
+
+class _LIBCPP_EXPORTED_FROM_ABI path {
+  template <class _SourceOrIter, class _Tp = path&>
+  using _EnableIfPathable = __enable_if_t<__is_pathable<_SourceOrIter>::value, _Tp>;
+
+  template <class _Tp>
+  using _SourceChar = typename __is_pathable<_Tp>::__char_type;
+
+  template <class _Tp>
+  using _SourceCVT = _PathCVT<_SourceChar<_Tp> >;
+
+public:
+#  if defined(_LIBCPP_WIN32API)
+  typedef wchar_t value_type;
+  static constexpr value_type preferred_separator = L'\\';
+#  else
+  typedef char value_type;
+  static constexpr value_type preferred_separator = '/';
+#  endif
+  typedef basic_string<value_type> string_type;
+  typedef basic_string_view<value_type> __string_view;
+
+  enum format : unsigned char { auto_format, native_format, generic_format };
+
+  // constructors and destructor
+  _LIBCPP_HIDE_FROM_ABI path() noexcept {}
+  _LIBCPP_HIDE_FROM_ABI path(const path& __p) : __pn_(__p.__pn_) {}
+  _LIBCPP_HIDE_FROM_ABI path(path&& __p) noexcept : __pn_(std::move(__p.__pn_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI path(string_type&& __s, format = format::auto_format) noexcept : __pn_(std::move(__s)) {}
+
+  template <class _Source, class = _EnableIfPathable<_Source, void> >
+  _LIBCPP_HIDE_FROM_ABI path(const _Source& __src, format = format::auto_format) {
+    _SourceCVT<_Source>::__append_source(__pn_, __src);
+  }
+
+  template <class _InputIt>
+  _LIBCPP_HIDE_FROM_ABI path(_InputIt __first, _InputIt __last, format = format::auto_format) {
+    typedef typename iterator_traits<_InputIt>::value_type _ItVal;
+    _PathCVT<_ItVal>::__append_range(__pn_, __first, __last);
+  }
+
+  /*
+  #if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+    // TODO Implement locale conversions.
+    template <class _Source, class = _EnableIfPathable<_Source, void> >
+    path(const _Source& __src, const locale& __loc, format = format::auto_format);
+    template <class _InputIt>
+    path(_InputIt __first, _InputIt _last, const locale& __loc,
+         format = format::auto_format);
+  #endif
+  */
+
+  _LIBCPP_HIDE_FROM_ABI ~path() = default;
+
+  // assignments
+  _LIBCPP_HIDE_FROM_ABI path& operator=(const path& __p) {
+    __pn_ = __p.__pn_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI path& operator=(path&& __p) noexcept {
+    __pn_ = std::move(__p.__pn_);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI path& operator=(string_type&& __s) noexcept {
+    __pn_ = std::move(__s);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI path& assign(string_type&& __s) noexcept {
+    __pn_ = std::move(__s);
+    return *this;
+  }
+
+  template <class _Source>
+  _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> operator=(const _Source& __src) {
+    return this->assign(__src);
+  }
+
+  template <class _Source>
+  _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> assign(const _Source& __src) {
+    __pn_.clear();
+    _SourceCVT<_Source>::__append_source(__pn_, __src);
+    return *this;
+  }
+
+  template <class _InputIt>
+  _LIBCPP_HIDE_FROM_ABI path& assign(_InputIt __first, _InputIt __last) {
+    typedef typename iterator_traits<_InputIt>::value_type _ItVal;
+    __pn_.clear();
+    _PathCVT<_ItVal>::__append_range(__pn_, __first, __last);
+    return *this;
+  }
+
+public:
+  // appends
+#  if defined(_LIBCPP_WIN32API)
+  _LIBCPP_HIDE_FROM_ABI path& operator/=(const path& __p) {
+    auto __p_root_name      = __p.__root_name();
+    auto __p_root_name_size = __p_root_name.size();
+    if (__p.is_absolute() || (!__p_root_name.empty() && __p_root_name != __string_view(root_name().__pn_))) {
+      __pn_ = __p.__pn_;
+      return *this;
+    }
+    if (__p.has_root_directory()) {
+      path __root_name_str = root_name();
+      __pn_                = __root_name_str.native();
+      __pn_ += __string_view(__p.__pn_).substr(__p_root_name_size);
+      return *this;
+    }
+    if (has_filename() || (!has_root_directory() && is_absolute()))
+      __pn_ += preferred_separator;
+    __pn_ += __string_view(__p.__pn_).substr(__p_root_name_size);
+    return *this;
+  }
+  template <class _Source>
+  _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> operator/=(const _Source& __src) {
+    return operator/=(path(__src));
+  }
+
+  template <class _Source>
+  _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> append(const _Source& __src) {
+    return operator/=(path(__src));
+  }
+
+  template <class _InputIt>
+  _LIBCPP_HIDE_FROM_ABI path& append(_InputIt __first, _InputIt __last) {
+    return operator/=(path(__first, __last));
+  }
+#  else
+  _LIBCPP_HIDE_FROM_ABI path& operator/=(const path& __p) {
+    if (__p.is_absolute()) {
+      __pn_ = __p.__pn_;
+      return *this;
+    }
+    if (has_filename())
+      __pn_ += preferred_separator;
+    __pn_ += __p.native();
+    return *this;
+  }
+
+  // FIXME: Use _LIBCPP_DIAGNOSE_WARNING to produce a diagnostic when __src
+  // is known at compile time to be "/' since the user almost certainly intended
+  // to append a separator instead of overwriting the path with "/"
+  template <class _Source>
+  _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> operator/=(const _Source& __src) {
+    return this->append(__src);
+  }
+
+  template <class _Source>
+  _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> append(const _Source& __src) {
+    using _Traits             = __is_pathable<_Source>;
+    using _CVT                = _PathCVT<_SourceChar<_Source> >;
+    bool __source_is_absolute = filesystem::__is_separator(_Traits::__first_or_null(__src));
+    if (__source_is_absolute)
+      __pn_.clear();
+    else if (has_filename())
+      __pn_ += preferred_separator;
+    _CVT::__append_source(__pn_, __src);
+    return *this;
+  }
+
+  template <class _InputIt>
+  _LIBCPP_HIDE_FROM_ABI path& append(_InputIt __first, _InputIt __last) {
+    typedef typename iterator_traits<_InputIt>::value_type _ItVal;
+    static_assert(__can_convert_char<_ItVal>::value, "Must convertible");
+    using _CVT = _PathCVT<_ItVal>;
+    if (__first != __last && filesystem::__is_separator(*__first))
+      __pn_.clear();
+    else if (has_filename())
+      __pn_ += preferred_separator;
+    _CVT::__append_range(__pn_, __first, __last);
+    return *this;
+  }
+#  endif
+
+  // concatenation
+  _LIBCPP_HIDE_FROM_ABI path& operator+=(const path& __x) {
+    __pn_ += __x.__pn_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI path& operator+=(const string_type& __x) {
+    __pn_ += __x;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI path& operator+=(__string_view __x) {
+    __pn_ += __x;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI path& operator+=(const value_type* __x) {
+    __pn_ += __x;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI path& operator+=(value_type __x) {
+    __pn_ += __x;
+    return *this;
+  }
+
+  template <class _ECharT, __enable_if_t<__can_convert_char<_ECharT>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI path& operator+=(_ECharT __x) {
+    _PathCVT<_ECharT>::__append_source(__pn_, basic_string_view<_ECharT>(&__x, 1));
+    return *this;
+  }
+
+  template <class _Source>
+  _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> operator+=(const _Source& __x) {
+    return this->concat(__x);
+  }
+
+  template <class _Source>
+  _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> concat(const _Source& __x) {
+    _SourceCVT<_Source>::__append_source(__pn_, __x);
+    return *this;
+  }
+
+  template <class _InputIt>
+  _LIBCPP_HIDE_FROM_ABI path& concat(_InputIt __first, _InputIt __last) {
+    typedef typename iterator_traits<_InputIt>::value_type _ItVal;
+    _PathCVT<_ItVal>::__append_range(__pn_, __first, __last);
+    return *this;
+  }
+
+  // modifiers
+  _LIBCPP_HIDE_FROM_ABI void clear() noexcept { __pn_.clear(); }
+
+  _LIBCPP_HIDE_FROM_ABI path& make_preferred() {
+#  if defined(_LIBCPP_WIN32API)
+    std::replace(__pn_.begin(), __pn_.end(), L'/', L'\\');
+#  endif
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI path& remove_filename() {
+    auto __fname = __filename();
+    if (!__fname.empty())
+      __pn_.erase(__fname.data() - __pn_.data());
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI path& replace_filename(const path& __replacement) {
+    remove_filename();
+    return (*this /= __replacement);
+  }
+
+  path& replace_extension(const path& __replacement = path());
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const path& __lhs, const path& __rhs) noexcept {
+    return __lhs.__compare(__rhs.__pn_) == 0;
+  }
+#  if _LIBCPP_STD_VER <= 17
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const path& __lhs, const path& __rhs) noexcept {
+    return __lhs.__compare(__rhs.__pn_) != 0;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator<(const path& __lhs, const path& __rhs) noexcept {
+    return __lhs.__compare(__rhs.__pn_) < 0;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator<=(const path& __lhs, const path& __rhs) noexcept {
+    return __lhs.__compare(__rhs.__pn_) <= 0;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator>(const path& __lhs, const path& __rhs) noexcept {
+    return __lhs.__compare(__rhs.__pn_) > 0;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator>=(const path& __lhs, const path& __rhs) noexcept {
+    return __lhs.__compare(__rhs.__pn_) >= 0;
+  }
+#  else  // _LIBCPP_STD_VER <= 17
+  friend _LIBCPP_HIDE_FROM_ABI strong_ordering operator<=>(const path& __lhs, const path& __rhs) noexcept {
+    return __lhs.__compare(__rhs.__pn_) <=> 0;
+  }
+#  endif // _LIBCPP_STD_VER <= 17
+
+  friend _LIBCPP_HIDE_FROM_ABI path operator/(const path& __lhs, const path& __rhs) {
+    path __result(__lhs);
+    __result /= __rhs;
+    return __result;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(path& __rhs) noexcept { __pn_.swap(__rhs.__pn_); }
+
+  // private helper to allow reserving memory in the path
+  _LIBCPP_HIDE_FROM_ABI void __reserve(size_t __s) { __pn_.reserve(__s); }
+
+  // native format observers
+  _LIBCPP_HIDE_FROM_ABI const string_type& native() const noexcept { return __pn_; }
+
+  _LIBCPP_HIDE_FROM_ABI const value_type* c_str() const noexcept { return __pn_.c_str(); }
+
+  _LIBCPP_HIDE_FROM_ABI operator string_type() const { return __pn_; }
+
+#  if defined(_LIBCPP_WIN32API)
+  _LIBCPP_HIDE_FROM_ABI std::wstring wstring() const { return __pn_; }
+
+  _LIBCPP_HIDE_FROM_ABI std::wstring generic_wstring() const {
+    std::wstring __s;
+    __s.resize(__pn_.size());
+    std::replace_copy(__pn_.begin(), __pn_.end(), __s.begin(), '\\', '/');
+    return __s;
+  }
+
+#    if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  template <class _ECharT, class _Traits = char_traits<_ECharT>, class _Allocator = allocator<_ECharT> >
+  _LIBCPP_HIDE_FROM_ABI basic_string<_ECharT, _Traits, _Allocator> string(const _Allocator& __a = _Allocator()) const {
+    using _Str = basic_string<_ECharT, _Traits, _Allocator>;
+    _Str __s(__a);
+    __s.reserve(__pn_.size());
+    _PathExport<_ECharT>::__append(__s, __pn_);
+    return __s;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI std::string string() const { return string<char>(); }
+  _LIBCPP_HIDE_FROM_ABI __u8_string u8string() const {
+    using _CVT = __narrow_to_utf8<sizeof(wchar_t) * __CHAR_BIT__>;
+    __u8_string __s;
+    __s.reserve(__pn_.size());
+    _CVT()(back_inserter(__s), __pn_.data(), __pn_.data() + __pn_.size());
+    return __s;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI std::u16string u16string() const { return string<char16_t>(); }
+  _LIBCPP_HIDE_FROM_ABI std::u32string u32string() const { return string<char32_t>(); }
+
+  // generic format observers
+  template <class _ECharT, class _Traits = char_traits<_ECharT>, class _Allocator = allocator<_ECharT> >
+  _LIBCPP_HIDE_FROM_ABI basic_string<_ECharT, _Traits, _Allocator>
+  generic_string(const _Allocator& __a = _Allocator()) const {
+    using _Str = basic_string<_ECharT, _Traits, _Allocator>;
+    _Str __s   = string<_ECharT, _Traits, _Allocator>(__a);
+    // Note: This (and generic_u8string below) is slightly suboptimal as
+    // it iterates twice over the string; once to convert it to the right
+    // character type, and once to replace path delimiters.
+    std::replace(__s.begin(), __s.end(), static_cast<_ECharT>('\\'), static_cast<_ECharT>('/'));
+    return __s;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI std::string generic_string() const { return generic_string<char>(); }
+  _LIBCPP_HIDE_FROM_ABI std::u16string generic_u16string() const { return generic_string<char16_t>(); }
+  _LIBCPP_HIDE_FROM_ABI std::u32string generic_u32string() const { return generic_string<char32_t>(); }
+  _LIBCPP_HIDE_FROM_ABI __u8_string generic_u8string() const {
+    __u8_string __s = u8string();
+    std::replace(__s.begin(), __s.end(), '\\', '/');
+    return __s;
+  }
+#    endif /* !_LIBCPP_HAS_NO_LOCALIZATION */
+#  else    /* _LIBCPP_WIN32API */
+
+  _LIBCPP_HIDE_FROM_ABI std::string string() const { return __pn_; }
+#    ifndef _LIBCPP_HAS_NO_CHAR8_T
+  _LIBCPP_HIDE_FROM_ABI std::u8string u8string() const { return std::u8string(__pn_.begin(), __pn_.end()); }
+#    else
+  _LIBCPP_HIDE_FROM_ABI std::string u8string() const { return __pn_; }
+#    endif
+
+#    if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  template <class _ECharT, class _Traits = char_traits<_ECharT>, class _Allocator = allocator<_ECharT> >
+  _LIBCPP_HIDE_FROM_ABI basic_string<_ECharT, _Traits, _Allocator> string(const _Allocator& __a = _Allocator()) const {
+    using _CVT = __widen_from_utf8<sizeof(_ECharT) * __CHAR_BIT__>;
+    using _Str = basic_string<_ECharT, _Traits, _Allocator>;
+    _Str __s(__a);
+    __s.reserve(__pn_.size());
+    _CVT()(std::back_inserter(__s), __pn_.data(), __pn_.data() + __pn_.size());
+    return __s;
+  }
+
+#      ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+  _LIBCPP_HIDE_FROM_ABI std::wstring wstring() const { return string<wchar_t>(); }
+#      endif
+  _LIBCPP_HIDE_FROM_ABI std::u16string u16string() const { return string<char16_t>(); }
+  _LIBCPP_HIDE_FROM_ABI std::u32string u32string() const { return string<char32_t>(); }
+#    endif /* !_LIBCPP_HAS_NO_LOCALIZATION */
+
+  // generic format observers
+  _LIBCPP_HIDE_FROM_ABI std::string generic_string() const { return __pn_; }
+#    ifndef _LIBCPP_HAS_NO_CHAR8_T
+  _LIBCPP_HIDE_FROM_ABI std::u8string generic_u8string() const { return std::u8string(__pn_.begin(), __pn_.end()); }
+#    else
+  _LIBCPP_HIDE_FROM_ABI std::string generic_u8string() const { return __pn_; }
+#    endif
+
+#    if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  template <class _ECharT, class _Traits = char_traits<_ECharT>, class _Allocator = allocator<_ECharT> >
+  _LIBCPP_HIDE_FROM_ABI basic_string<_ECharT, _Traits, _Allocator>
+  generic_string(const _Allocator& __a = _Allocator()) const {
+    return string<_ECharT, _Traits, _Allocator>(__a);
+  }
+
+#      ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+  _LIBCPP_HIDE_FROM_ABI std::wstring generic_wstring() const { return string<wchar_t>(); }
+#      endif
+  _LIBCPP_HIDE_FROM_ABI std::u16string generic_u16string() const { return string<char16_t>(); }
+  _LIBCPP_HIDE_FROM_ABI std::u32string generic_u32string() const { return string<char32_t>(); }
+#    endif /* !_LIBCPP_HAS_NO_LOCALIZATION */
+#  endif   /* !_LIBCPP_WIN32API */
+
+private:
+  int __compare(__string_view) const;
+  __string_view __root_name() const;
+  __string_view __root_directory() const;
+  __string_view __root_path_raw() const;
+  __string_view __relative_path() const;
+  __string_view __parent_path() const;
+  __string_view __filename() const;
+  __string_view __stem() const;
+  __string_view __extension() const;
+
+public:
+  // compare
+  _LIBCPP_HIDE_FROM_ABI int compare(const path& __p) const noexcept { return __compare(__p.__pn_); }
+  _LIBCPP_HIDE_FROM_ABI int compare(const string_type& __s) const { return __compare(__s); }
+  _LIBCPP_HIDE_FROM_ABI int compare(__string_view __s) const { return __compare(__s); }
+  _LIBCPP_HIDE_FROM_ABI int compare(const value_type* __s) const { return __compare(__s); }
+
+  // decomposition
+  _LIBCPP_HIDE_FROM_ABI path root_name() const { return string_type(__root_name()); }
+  _LIBCPP_HIDE_FROM_ABI path root_directory() const { return string_type(__root_directory()); }
+  _LIBCPP_HIDE_FROM_ABI path root_path() const {
+#  if defined(_LIBCPP_WIN32API)
+    return string_type(__root_path_raw());
+#  else
+    return root_name().append(string_type(__root_directory()));
+#  endif
+  }
+  _LIBCPP_HIDE_FROM_ABI path relative_path() const { return string_type(__relative_path()); }
+  _LIBCPP_HIDE_FROM_ABI path parent_path() const { return string_type(__parent_path()); }
+  _LIBCPP_HIDE_FROM_ABI path filename() const { return string_type(__filename()); }
+  _LIBCPP_HIDE_FROM_ABI path stem() const { return string_type(__stem()); }
+  _LIBCPP_HIDE_FROM_ABI path extension() const { return string_type(__extension()); }
+
+  // query
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const noexcept { return __pn_.empty(); }
+
+  _LIBCPP_HIDE_FROM_ABI bool has_root_name() const { return !__root_name().empty(); }
+  _LIBCPP_HIDE_FROM_ABI bool has_root_directory() const { return !__root_directory().empty(); }
+  _LIBCPP_HIDE_FROM_ABI bool has_root_path() const { return !__root_path_raw().empty(); }
+  _LIBCPP_HIDE_FROM_ABI bool has_relative_path() const { return !__relative_path().empty(); }
+  _LIBCPP_HIDE_FROM_ABI bool has_parent_path() const { return !__parent_path().empty(); }
+  _LIBCPP_HIDE_FROM_ABI bool has_filename() const { return !__filename().empty(); }
+  _LIBCPP_HIDE_FROM_ABI bool has_stem() const { return !__stem().empty(); }
+  _LIBCPP_HIDE_FROM_ABI bool has_extension() const { return !__extension().empty(); }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_absolute() const {
+#  if defined(_LIBCPP_WIN32API)
+    __string_view __root_name_str = __root_name();
+    __string_view __root_dir      = __root_directory();
+    if (__root_name_str.size() == 2 && __root_name_str[1] == ':') {
+      // A drive letter with no root directory is relative, e.g. x:example.
+      return !__root_dir.empty();
+    }
+    // If no root name, it's relative, e.g. \example is relative to the current drive
+    if (__root_name_str.empty())
+      return false;
+    if (__root_name_str.size() < 3)
+      return false;
+    // A server root name, like \\server, is always absolute
+    if (__root_name_str[0] != '/' && __root_name_str[0] != '\\')
+      return false;
+    if (__root_name_str[1] != '/' && __root_name_str[1] != '\\')
+      return false;
+    // Seems to be a server root name
+    return true;
+#  else
+    return has_root_directory();
+#  endif
+  }
+  _LIBCPP_HIDE_FROM_ABI bool is_relative() const { return !is_absolute(); }
+
+  // relative paths
+  path lexically_normal() const;
+  path lexically_relative(const path& __base) const;
+
+  _LIBCPP_HIDE_FROM_ABI path lexically_proximate(const path& __base) const {
+    path __result = this->lexically_relative(__base);
+    if (__result.native().empty())
+      return *this;
+    return __result;
+  }
+
+  // iterators
+  class _LIBCPP_EXPORTED_FROM_ABI iterator;
+  typedef iterator const_iterator;
+
+  iterator begin() const;
+  iterator end() const;
+
+#  if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  template <
+      class _CharT,
+      class _Traits,
+      __enable_if_t<is_same<_CharT, value_type>::value && is_same<_Traits, char_traits<value_type> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI friend basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const path& __p) {
+    __os << std::__quoted(__p.native());
+    return __os;
+  }
+
+  template <
+      class _CharT,
+      class _Traits,
+      __enable_if_t<!is_same<_CharT, value_type>::value || !is_same<_Traits, char_traits<value_type> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI friend basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const path& __p) {
+    __os << std::__quoted(__p.string<_CharT, _Traits>());
+    return __os;
+  }
+
+  template <class _CharT, class _Traits>
+  _LIBCPP_HIDE_FROM_ABI friend basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, path& __p) {
+    basic_string<_CharT, _Traits> __tmp;
+    __is >> std::__quoted(__tmp);
+    __p = __tmp;
+    return __is;
+  }
+#  endif // !_LIBCPP_HAS_NO_LOCALIZATION
+
+private:
+  inline _LIBCPP_HIDE_FROM_ABI path& __assign_view(__string_view const& __s) {
+    __pn_ = string_type(__s);
+    return *this;
+  }
+  string_type __pn_;
+};
+
+inline _LIBCPP_HIDE_FROM_ABI void swap(path& __lhs, path& __rhs) noexcept { __lhs.swap(__rhs); }
+
+_LIBCPP_EXPORTED_FROM_ABI size_t hash_value(const path& __p) noexcept;
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <>
+struct _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY hash<filesystem::path> : __unary_function<filesystem::path, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(filesystem::path const& __p) const noexcept {
+    return filesystem::hash_value(__p);
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FILESYSTEM_PATH_H
diff --git a/libcxx/include/__cxx03/__filesystem/path_iterator.h b/libcxx/include/__cxx03/__filesystem/path_iterator.h
new file mode 100644
index 000000000000..f4d486d86cf3
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/path_iterator.h
@@ -0,0 +1,115 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_PATH_ITERATOR_H
+#define _LIBCPP___FILESYSTEM_PATH_ITERATOR_H
+
+#include <__assert>
+#include <__config>
+#include <__filesystem/path.h>
+#include <__iterator/iterator_traits.h>
+#include <cstddef>
+#include <string>
+#include <string_view>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+class _LIBCPP_EXPORTED_FROM_ABI path::iterator {
+public:
+  enum _ParserState : unsigned char {
+    _Singular,
+    _BeforeBegin,
+    _InRootName,
+    _InRootDir,
+    _InFilenames,
+    _InTrailingSep,
+    _AtEnd
+  };
+
+public:
+  typedef input_iterator_tag iterator_category;
+  typedef bidirectional_iterator_tag iterator_concept;
+
+  typedef path value_type;
+  typedef ptrdiff_t difference_type;
+  typedef const path* pointer;
+  typedef path reference;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI iterator() : __stashed_elem_(), __path_ptr_(nullptr), __entry_(), __state_(_Singular) {}
+
+  _LIBCPP_HIDE_FROM_ABI iterator(const iterator&) = default;
+  _LIBCPP_HIDE_FROM_ABI ~iterator()               = default;
+
+  _LIBCPP_HIDE_FROM_ABI iterator& operator=(const iterator&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __stashed_elem_; }
+
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return &__stashed_elem_; }
+
+  _LIBCPP_HIDE_FROM_ABI iterator& operator++() {
+    _LIBCPP_ASSERT_NON_NULL(__state_ != _Singular, "attempting to increment a singular iterator");
+    _LIBCPP_ASSERT_UNCATEGORIZED(__state_ != _AtEnd, "attempting to increment the end iterator");
+    return __increment();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator operator++(int) {
+    iterator __it(*this);
+    this->operator++();
+    return __it;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator& operator--() {
+    _LIBCPP_ASSERT_NON_NULL(__state_ != _Singular, "attempting to decrement a singular iterator");
+    _LIBCPP_ASSERT_UNCATEGORIZED(
+        __entry_.data() != __path_ptr_->native().data(), "attempting to decrement the begin iterator");
+    return __decrement();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator operator--(int) {
+    iterator __it(*this);
+    this->operator--();
+    return __it;
+  }
+
+private:
+  friend class path;
+
+  inline _LIBCPP_HIDE_FROM_ABI friend bool operator==(const iterator&, const iterator&);
+
+  iterator& __increment();
+  iterator& __decrement();
+
+  path __stashed_elem_;
+  const path* __path_ptr_;
+  path::__string_view __entry_;
+  _ParserState __state_;
+};
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(const path::iterator& __lhs, const path::iterator& __rhs) {
+  return __lhs.__path_ptr_ == __rhs.__path_ptr_ && __lhs.__entry_.data() == __rhs.__entry_.data();
+}
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const path::iterator& __lhs, const path::iterator& __rhs) {
+  return !(__lhs == __rhs);
+}
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___FILESYSTEM_PATH_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__filesystem/perm_options.h b/libcxx/include/__cxx03/__filesystem/perm_options.h
new file mode 100644
index 000000000000..64c16ee60a17
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/perm_options.h
@@ -0,0 +1,57 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_PERM_OPTIONS_H
+#define _LIBCPP___FILESYSTEM_PERM_OPTIONS_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+enum class perm_options : unsigned char { replace = 1, add = 2, remove = 4, nofollow = 8 };
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator&(perm_options __lhs, perm_options __rhs) {
+  return static_cast<perm_options>(static_cast<unsigned>(__lhs) & static_cast<unsigned>(__rhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator|(perm_options __lhs, perm_options __rhs) {
+  return static_cast<perm_options>(static_cast<unsigned>(__lhs) | static_cast<unsigned>(__rhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator^(perm_options __lhs, perm_options __rhs) {
+  return static_cast<perm_options>(static_cast<unsigned>(__lhs) ^ static_cast<unsigned>(__rhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator~(perm_options __lhs) {
+  return static_cast<perm_options>(~static_cast<unsigned>(__lhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline perm_options& operator&=(perm_options& __lhs, perm_options __rhs) {
+  return __lhs = __lhs & __rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline perm_options& operator|=(perm_options& __lhs, perm_options __rhs) {
+  return __lhs = __lhs | __rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline perm_options& operator^=(perm_options& __lhs, perm_options __rhs) {
+  return __lhs = __lhs ^ __rhs;
+}
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___FILESYSTEM_PERM_OPTIONS_H
diff --git a/libcxx/include/__cxx03/__filesystem/perms.h b/libcxx/include/__cxx03/__filesystem/perms.h
new file mode 100644
index 000000000000..458f1e6e5348
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/perms.h
@@ -0,0 +1,80 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_PERMS_H
+#define _LIBCPP___FILESYSTEM_PERMS_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+// On Windows, these permission bits map to one single readonly flag per
+// file, and the executable bit is always returned as set. When setting
+// permissions, as long as the write bit is set for either owner, group or
+// others, the readonly flag is cleared.
+enum class perms : unsigned {
+  none = 0,
+
+  owner_read  = 0400,
+  owner_write = 0200,
+  owner_exec  = 0100,
+  owner_all   = 0700,
+
+  group_read  = 040,
+  group_write = 020,
+  group_exec  = 010,
+  group_all   = 070,
+
+  others_read  = 04,
+  others_write = 02,
+  others_exec  = 01,
+  others_all   = 07,
+
+  all = 0777,
+
+  set_uid    = 04000,
+  set_gid    = 02000,
+  sticky_bit = 01000,
+  mask       = 07777,
+  unknown    = 0xFFFF,
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr perms operator&(perms __lhs, perms __rhs) {
+  return static_cast<perms>(static_cast<unsigned>(__lhs) & static_cast<unsigned>(__rhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr perms operator|(perms __lhs, perms __rhs) {
+  return static_cast<perms>(static_cast<unsigned>(__lhs) | static_cast<unsigned>(__rhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr perms operator^(perms __lhs, perms __rhs) {
+  return static_cast<perms>(static_cast<unsigned>(__lhs) ^ static_cast<unsigned>(__rhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr perms operator~(perms __lhs) {
+  return static_cast<perms>(~static_cast<unsigned>(__lhs));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline perms& operator&=(perms& __lhs, perms __rhs) { return __lhs = __lhs & __rhs; }
+
+_LIBCPP_HIDE_FROM_ABI inline perms& operator|=(perms& __lhs, perms __rhs) { return __lhs = __lhs | __rhs; }
+
+_LIBCPP_HIDE_FROM_ABI inline perms& operator^=(perms& __lhs, perms __rhs) { return __lhs = __lhs ^ __rhs; }
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___FILESYSTEM_PERMS_H
diff --git a/libcxx/include/__cxx03/__filesystem/recursive_directory_iterator.h b/libcxx/include/__cxx03/__filesystem/recursive_directory_iterator.h
new file mode 100644
index 000000000000..caa1396eb301
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/recursive_directory_iterator.h
@@ -0,0 +1,164 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_RECURSIVE_DIRECTORY_ITERATOR_H
+#define _LIBCPP___FILESYSTEM_RECURSIVE_DIRECTORY_ITERATOR_H
+
+#include <__config>
+#include <__filesystem/directory_entry.h>
+#include <__filesystem/directory_options.h>
+#include <__filesystem/path.h>
+#include <__iterator/default_sentinel.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/shared_ptr.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/enable_view.h>
+#include <__system_error/error_code.h>
+#include <__utility/move.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 17 && !defined(_LIBCPP_HAS_NO_FILESYSTEM)
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH
+
+class recursive_directory_iterator {
+public:
+  using value_type        = directory_entry;
+  using difference_type   = ptrdiff_t;
+  using pointer           = directory_entry const*;
+  using reference         = directory_entry const&;
+  using iterator_category = input_iterator_tag;
+
+public:
+  // constructors and destructor
+  _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator() noexcept : __rec_(false) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit recursive_directory_iterator(
+      const path& __p, directory_options __xoptions = directory_options::none)
+      : recursive_directory_iterator(__p, __xoptions, nullptr) {}
+
+  _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator(const path& __p, directory_options __xoptions, error_code& __ec)
+      : recursive_directory_iterator(__p, __xoptions, &__ec) {}
+
+  _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator(const path& __p, error_code& __ec)
+      : recursive_directory_iterator(__p, directory_options::none, &__ec) {}
+
+  _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator(const recursive_directory_iterator&) = default;
+  _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator(recursive_directory_iterator&&)      = default;
+
+  _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator& operator=(const recursive_directory_iterator&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator& operator=(recursive_directory_iterator&& __o) noexcept {
+    // non-default implementation provided to support self-move assign.
+    if (this != &__o) {
+      __imp_ = std::move(__o.__imp_);
+      __rec_ = __o.__rec_;
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~recursive_directory_iterator() = default;
+
+  _LIBCPP_HIDE_FROM_ABI const directory_entry& operator*() const { return __dereference(); }
+
+  _LIBCPP_HIDE_FROM_ABI const directory_entry* operator->() const { return &__dereference(); }
+
+  _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator& operator++() { return __increment(); }
+
+  _LIBCPP_HIDE_FROM_ABI __dir_element_proxy operator++(int) {
+    __dir_element_proxy __p(**this);
+    __increment();
+    return __p;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator& increment(error_code& __ec) { return __increment(&__ec); }
+
+  _LIBCPP_EXPORTED_FROM_ABI directory_options options() const;
+  _LIBCPP_EXPORTED_FROM_ABI int depth() const;
+
+  _LIBCPP_HIDE_FROM_ABI void pop() { __pop(); }
+
+  _LIBCPP_HIDE_FROM_ABI void pop(error_code& __ec) { __pop(&__ec); }
+
+  _LIBCPP_HIDE_FROM_ABI bool recursion_pending() const { return __rec_; }
+
+  _LIBCPP_HIDE_FROM_ABI void disable_recursion_pending() { __rec_ = false; }
+
+#  if _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI bool operator==(default_sentinel_t) const noexcept {
+    return *this == recursive_directory_iterator();
+  }
+
+#  endif
+
+private:
+  _LIBCPP_EXPORTED_FROM_ABI recursive_directory_iterator(const path& __p, directory_options __opt, error_code* __ec);
+  _LIBCPP_EXPORTED_FROM_ABI const directory_entry& __dereference() const;
+  _LIBCPP_EXPORTED_FROM_ABI bool __try_recursion(error_code* __ec);
+  _LIBCPP_EXPORTED_FROM_ABI void __advance(error_code* __ec = nullptr);
+  _LIBCPP_EXPORTED_FROM_ABI recursive_directory_iterator& __increment(error_code* __ec = nullptr);
+  _LIBCPP_EXPORTED_FROM_ABI void __pop(error_code* __ec = nullptr);
+
+  inline _LIBCPP_HIDE_FROM_ABI friend bool
+  operator==(const recursive_directory_iterator&, const recursive_directory_iterator&) noexcept;
+
+  struct _LIBCPP_HIDDEN __shared_imp;
+  shared_ptr<__shared_imp> __imp_;
+  bool __rec_;
+}; // class recursive_directory_iterator
+
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const recursive_directory_iterator& __lhs, const recursive_directory_iterator& __rhs) noexcept {
+  return __lhs.__imp_ == __rhs.__imp_;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline bool
+operator!=(const recursive_directory_iterator& __lhs, const recursive_directory_iterator& __rhs) noexcept {
+  return !(__lhs == __rhs);
+}
+// enable recursive_directory_iterator range-based for statements
+inline _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator begin(recursive_directory_iterator __iter) noexcept {
+  return __iter;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator end(recursive_directory_iterator) noexcept {
+  return recursive_directory_iterator();
+}
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#  if _LIBCPP_STD_VER >= 20
+
+template <>
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY inline constexpr bool
+    std::ranges::enable_borrowed_range<std::filesystem::recursive_directory_iterator> = true;
+
+template <>
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY inline constexpr bool
+    std::ranges::enable_view<std::filesystem::recursive_directory_iterator> = true;
+
+#  endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP_STD_VER >= 17 && !defined(_LIBCPP_HAS_NO_FILESYSTEM)
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FILESYSTEM_RECURSIVE_DIRECTORY_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__filesystem/space_info.h b/libcxx/include/__cxx03/__filesystem/space_info.h
new file mode 100644
index 000000000000..3fa57d33096f
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/space_info.h
@@ -0,0 +1,38 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_SPACE_INFO_H
+#define _LIBCPP___FILESYSTEM_SPACE_INFO_H
+
+#include <__config>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+struct _LIBCPP_EXPORTED_FROM_ABI space_info {
+  uintmax_t capacity;
+  uintmax_t free;
+  uintmax_t available;
+
+#  if _LIBCPP_STD_VER >= 20
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const space_info&, const space_info&) = default;
+#  endif
+};
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___FILESYSTEM_SPACE_INFO_H
diff --git a/libcxx/include/__cxx03/__filesystem/u8path.h b/libcxx/include/__cxx03/__filesystem/u8path.h
new file mode 100644
index 000000000000..dae5823128f0
--- /dev/null
+++ b/libcxx/include/__cxx03/__filesystem/u8path.h
@@ -0,0 +1,100 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_U8PATH_H
+#define _LIBCPP___FILESYSTEM_U8PATH_H
+
+#include <__algorithm/unwrap_iter.h>
+#include <__config>
+#include <__filesystem/path.h>
+#include <string>
+
+// Only required on Windows for __widen_from_utf8, and included conservatively
+// because it requires support for localization.
+#if defined(_LIBCPP_WIN32API)
+#  include <locale>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH
+
+template <class _InputIt, __enable_if_t<__is_pathable<_InputIt>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(_InputIt __f, _InputIt __l) {
+  static_assert(
+#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+      is_same<typename __is_pathable<_InputIt>::__char_type, char8_t>::value ||
+#  endif
+          is_same<typename __is_pathable<_InputIt>::__char_type, char>::value,
+      "u8path(Iter, Iter) requires Iter have a value_type of type 'char'"
+      " or 'char8_t'");
+#  if defined(_LIBCPP_WIN32API)
+  string __tmp(__f, __l);
+  using _CVT = __widen_from_utf8<sizeof(wchar_t) * __CHAR_BIT__>;
+  std::wstring __w;
+  __w.reserve(__tmp.size());
+  _CVT()(back_inserter(__w), __tmp.data(), __tmp.data() + __tmp.size());
+  return path(__w);
+#  else
+  return path(__f, __l);
+#  endif /* !_LIBCPP_WIN32API */
+}
+
+#  if defined(_LIBCPP_WIN32API)
+template <class _InputIt, __enable_if_t<__is_pathable<_InputIt>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(_InputIt __f, _NullSentinel) {
+  static_assert(
+#    ifndef _LIBCPP_HAS_NO_CHAR8_T
+      is_same<typename __is_pathable<_InputIt>::__char_type, char8_t>::value ||
+#    endif
+          is_same<typename __is_pathable<_InputIt>::__char_type, char>::value,
+      "u8path(Iter, Iter) requires Iter have a value_type of type 'char'"
+      " or 'char8_t'");
+  string __tmp;
+  const char __sentinel = char{};
+  for (; *__f != __sentinel; ++__f)
+    __tmp.push_back(*__f);
+  using _CVT = __widen_from_utf8<sizeof(wchar_t) * __CHAR_BIT__>;
+  std::wstring __w;
+  __w.reserve(__tmp.size());
+  _CVT()(back_inserter(__w), __tmp.data(), __tmp.data() + __tmp.size());
+  return path(__w);
+}
+#  endif /* _LIBCPP_WIN32API */
+
+template <class _Source, __enable_if_t<__is_pathable<_Source>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(const _Source& __s) {
+  static_assert(
+#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+      is_same<typename __is_pathable<_Source>::__char_type, char8_t>::value ||
+#  endif
+          is_same<typename __is_pathable<_Source>::__char_type, char>::value,
+      "u8path(Source const&) requires Source have a character type of type "
+      "'char' or 'char8_t'");
+#  if defined(_LIBCPP_WIN32API)
+  using _Traits = __is_pathable<_Source>;
+  return u8path(std::__unwrap_iter(_Traits::__range_begin(__s)), std::__unwrap_iter(_Traits::__range_end(__s)));
+#  else
+  return path(__s);
+#  endif
+}
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___FILESYSTEM_U8PATH_H
diff --git a/libcxx/include/__cxx03/__format/buffer.h b/libcxx/include/__cxx03/__format/buffer.h
new file mode 100644
index 000000000000..8598f0a1c039
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/buffer.h
@@ -0,0 +1,655 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_BUFFER_H
+#define _LIBCPP___FORMAT_BUFFER_H
+
+#include <__algorithm/copy_n.h>
+#include <__algorithm/fill_n.h>
+#include <__algorithm/max.h>
+#include <__algorithm/min.h>
+#include <__algorithm/ranges_copy_n.h>
+#include <__algorithm/transform.h>
+#include <__algorithm/unwrap_iter.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/concepts.h>
+#include <__format/enable_insertable.h>
+#include <__format/format_to_n_result.h>
+#include <__iterator/back_insert_iterator.h>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/wrap_iter.h>
+#include <__memory/addressof.h>
+#include <__memory/allocate_at_least.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/construct_at.h>
+#include <__memory/ranges_construct_at.h>
+#include <__memory/uninitialized_algorithms.h>
+#include <__type_traits/add_pointer.h>
+#include <__type_traits/conditional.h>
+#include <__utility/exception_guard.h>
+#include <__utility/move.h>
+#include <cstddef>
+#include <string_view>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __format {
+
+/// A "buffer" that handles writing to the proper iterator.
+///
+/// This helper is used together with the @ref back_insert_iterator to offer
+/// type-erasure for the formatting functions. This reduces the number to
+/// template instantiations.
+template <__fmt_char_type _CharT>
+class _LIBCPP_TEMPLATE_VIS __output_buffer {
+public:
+  using value_type = _CharT;
+
+  template <class _Tp>
+  _LIBCPP_HIDE_FROM_ABI explicit __output_buffer(_CharT* __ptr, size_t __capacity, _Tp* __obj)
+      : __ptr_(__ptr),
+        __capacity_(__capacity),
+        __flush_([](_CharT* __p, size_t __n, void* __o) { static_cast<_Tp*>(__o)->__flush(__p, __n); }),
+        __obj_(__obj) {}
+
+  _LIBCPP_HIDE_FROM_ABI void __reset(_CharT* __ptr, size_t __capacity) {
+    __ptr_      = __ptr;
+    __capacity_ = __capacity;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI auto __make_output_iterator() { return std::back_insert_iterator{*this}; }
+
+  // Used in std::back_insert_iterator.
+  _LIBCPP_HIDE_FROM_ABI void push_back(_CharT __c) {
+    __ptr_[__size_++] = __c;
+
+    // Profiling showed flushing after adding is more efficient than flushing
+    // when entering the function.
+    if (__size_ == __capacity_)
+      __flush();
+  }
+
+  /// Copies the input __str to the buffer.
+  ///
+  /// Since some of the input is generated by std::to_chars, there needs to be a
+  /// conversion when _CharT is wchar_t.
+  template <__fmt_char_type _InCharT>
+  _LIBCPP_HIDE_FROM_ABI void __copy(basic_string_view<_InCharT> __str) {
+    // When the underlying iterator is a simple iterator the __capacity_ is
+    // infinite. For a string or container back_inserter it isn't. This means
+    // that adding a large string to the buffer can cause some overhead. In that
+    // case a better approach could be:
+    // - flush the buffer
+    // - container.append(__str.begin(), __str.end());
+    // The same holds true for the fill.
+    // For transform it might be slightly harder, however the use case for
+    // transform is slightly less common; it converts hexadecimal values to
+    // upper case. For integral these strings are short.
+    // TODO FMT Look at the improvements above.
+    size_t __n = __str.size();
+
+    __flush_on_overflow(__n);
+    if (__n < __capacity_) { //  push_back requires the buffer to have room for at least one character (so use <).
+      std::copy_n(__str.data(), __n, std::addressof(__ptr_[__size_]));
+      __size_ += __n;
+      return;
+    }
+
+    // The output doesn't fit in the internal buffer.
+    // Copy the data in "__capacity_" sized chunks.
+    _LIBCPP_ASSERT_INTERNAL(__size_ == 0, "the buffer should be flushed by __flush_on_overflow");
+    const _InCharT* __first = __str.data();
+    do {
+      size_t __chunk = std::min(__n, __capacity_);
+      std::copy_n(__first, __chunk, std::addressof(__ptr_[__size_]));
+      __size_ = __chunk;
+      __first += __chunk;
+      __n -= __chunk;
+      __flush();
+    } while (__n);
+  }
+
+  /// A std::transform wrapper.
+  ///
+  /// Like @ref __copy it may need to do type conversion.
+  template <contiguous_iterator _Iterator,
+            class _UnaryOperation,
+            __fmt_char_type _InCharT = typename iterator_traits<_Iterator>::value_type>
+  _LIBCPP_HIDE_FROM_ABI void __transform(_Iterator __first, _Iterator __last, _UnaryOperation __operation) {
+    _LIBCPP_ASSERT_INTERNAL(__first <= __last, "not a valid range");
+
+    size_t __n = static_cast<size_t>(__last - __first);
+    __flush_on_overflow(__n);
+    if (__n < __capacity_) { //  push_back requires the buffer to have room for at least one character (so use <).
+      std::transform(__first, __last, std::addressof(__ptr_[__size_]), std::move(__operation));
+      __size_ += __n;
+      return;
+    }
+
+    // The output doesn't fit in the internal buffer.
+    // Transform the data in "__capacity_" sized chunks.
+    _LIBCPP_ASSERT_INTERNAL(__size_ == 0, "the buffer should be flushed by __flush_on_overflow");
+    do {
+      size_t __chunk = std::min(__n, __capacity_);
+      std::transform(__first, __first + __chunk, std::addressof(__ptr_[__size_]), __operation);
+      __size_ = __chunk;
+      __first += __chunk;
+      __n -= __chunk;
+      __flush();
+    } while (__n);
+  }
+
+  /// A \c fill_n wrapper.
+  _LIBCPP_HIDE_FROM_ABI void __fill(size_t __n, _CharT __value) {
+    __flush_on_overflow(__n);
+    if (__n < __capacity_) { //  push_back requires the buffer to have room for at least one character (so use <).
+      std::fill_n(std::addressof(__ptr_[__size_]), __n, __value);
+      __size_ += __n;
+      return;
+    }
+
+    // The output doesn't fit in the internal buffer.
+    // Fill the buffer in "__capacity_" sized chunks.
+    _LIBCPP_ASSERT_INTERNAL(__size_ == 0, "the buffer should be flushed by __flush_on_overflow");
+    do {
+      size_t __chunk = std::min(__n, __capacity_);
+      std::fill_n(std::addressof(__ptr_[__size_]), __chunk, __value);
+      __size_ = __chunk;
+      __n -= __chunk;
+      __flush();
+    } while (__n);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __flush() {
+    __flush_(__ptr_, __size_, __obj_);
+    __size_ = 0;
+  }
+
+private:
+  _CharT* __ptr_;
+  size_t __capacity_;
+  size_t __size_{0};
+  void (*__flush_)(_CharT*, size_t, void*);
+  void* __obj_;
+
+  /// Flushes the buffer when the output operation would overflow the buffer.
+  ///
+  /// A simple approach for the overflow detection would be something along the
+  /// lines:
+  /// \code
+  /// // The internal buffer is large enough.
+  /// if (__n <= __capacity_) {
+  ///   // Flush when we really would overflow.
+  ///   if (__size_ + __n >= __capacity_)
+  ///     __flush();
+  ///   ...
+  /// }
+  /// \endcode
+  ///
+  /// This approach works for all cases but one:
+  /// A __format_to_n_buffer_base where \ref __enable_direct_output is true.
+  /// In that case the \ref __capacity_ of the buffer changes during the first
+  /// \ref __flush. During that operation the output buffer switches from its
+  /// __writer_ to its __storage_. The \ref __capacity_ of the former depends
+  /// on the value of n, of the latter is a fixed size. For example:
+  /// - a format_to_n call with a 10'000 char buffer,
+  /// - the buffer is filled with 9'500 chars,
+  /// - adding 1'000 elements would overflow the buffer so the buffer gets
+  ///   changed and the \ref __capacity_ decreases from 10'000 to
+  ///   __buffer_size (256 at the time of writing).
+  ///
+  /// This means that the \ref __flush for this class may need to copy a part of
+  /// the internal buffer to the proper output. In this example there will be
+  /// 500 characters that need this copy operation.
+  ///
+  /// Note it would be more efficient to write 500 chars directly and then swap
+  /// the buffers. This would make the code more complex and \ref format_to_n is
+  /// not the most common use case. Therefore the optimization isn't done.
+  _LIBCPP_HIDE_FROM_ABI void __flush_on_overflow(size_t __n) {
+    if (__size_ + __n >= __capacity_)
+      __flush();
+  }
+};
+
+/// A storage using an internal buffer.
+///
+/// This storage is used when writing a single element to the output iterator
+/// is expensive.
+template <__fmt_char_type _CharT>
+class _LIBCPP_TEMPLATE_VIS __internal_storage {
+public:
+  _LIBCPP_HIDE_FROM_ABI _CharT* __begin() { return __buffer_; }
+
+  static constexpr size_t __buffer_size = 256 / sizeof(_CharT);
+
+private:
+  _CharT __buffer_[__buffer_size];
+};
+
+/// A storage writing directly to the storage.
+///
+/// This requires the storage to be a contiguous buffer of \a _CharT.
+/// Since the output is directly written to the underlying storage this class
+/// is just an empty class.
+template <__fmt_char_type _CharT>
+class _LIBCPP_TEMPLATE_VIS __direct_storage {};
+
+template <class _OutIt, class _CharT>
+concept __enable_direct_output =
+    __fmt_char_type<_CharT> &&
+    (same_as<_OutIt, _CharT*>
+     // TODO(hardening): the following check might not apply to hardened iterators and might need to be wrapped in an
+     // `#ifdef`.
+     || same_as<_OutIt, __wrap_iter<_CharT*>>);
+
+/// Write policy for directly writing to the underlying output.
+template <class _OutIt, __fmt_char_type _CharT>
+class _LIBCPP_TEMPLATE_VIS __writer_direct {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __writer_direct(_OutIt __out_it) : __out_it_(__out_it) {}
+
+  _LIBCPP_HIDE_FROM_ABI _OutIt __out_it() { return __out_it_; }
+
+  _LIBCPP_HIDE_FROM_ABI void __flush(_CharT*, size_t __n) {
+    // _OutIt can be a __wrap_iter<CharT*>. Therefore the original iterator
+    // is adjusted.
+    __out_it_ += __n;
+  }
+
+private:
+  _OutIt __out_it_;
+};
+
+/// Write policy for copying the buffer to the output.
+template <class _OutIt, __fmt_char_type _CharT>
+class _LIBCPP_TEMPLATE_VIS __writer_iterator {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __writer_iterator(_OutIt __out_it) : __out_it_{std::move(__out_it)} {}
+
+  _LIBCPP_HIDE_FROM_ABI _OutIt __out_it() && { return std::move(__out_it_); }
+
+  _LIBCPP_HIDE_FROM_ABI void __flush(_CharT* __ptr, size_t __n) {
+    __out_it_ = std::ranges::copy_n(__ptr, __n, std::move(__out_it_)).out;
+  }
+
+private:
+  _OutIt __out_it_;
+};
+
+/// Concept to see whether a \a _Container is insertable.
+///
+/// The concept is used to validate whether multiple calls to a
+/// \ref back_insert_iterator can be replace by a call to \c _Container::insert.
+///
+/// \note a \a _Container needs to opt-in to the concept by specializing
+/// \ref __enable_insertable.
+template <class _Container>
+concept __insertable =
+    __enable_insertable<_Container> && __fmt_char_type<typename _Container::value_type> &&
+    requires(_Container& __t,
+             add_pointer_t<typename _Container::value_type> __first,
+             add_pointer_t<typename _Container::value_type> __last) { __t.insert(__t.end(), __first, __last); };
+
+/// Extract the container type of a \ref back_insert_iterator.
+template <class _It>
+struct _LIBCPP_TEMPLATE_VIS __back_insert_iterator_container {
+  using type = void;
+};
+
+template <__insertable _Container>
+struct _LIBCPP_TEMPLATE_VIS __back_insert_iterator_container<back_insert_iterator<_Container>> {
+  using type = _Container;
+};
+
+/// Write policy for inserting the buffer in a container.
+template <class _Container>
+class _LIBCPP_TEMPLATE_VIS __writer_container {
+public:
+  using _CharT = typename _Container::value_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __writer_container(back_insert_iterator<_Container> __out_it)
+      : __container_{__out_it.__get_container()} {}
+
+  _LIBCPP_HIDE_FROM_ABI auto __out_it() { return std::back_inserter(*__container_); }
+
+  _LIBCPP_HIDE_FROM_ABI void __flush(_CharT* __ptr, size_t __n) {
+    __container_->insert(__container_->end(), __ptr, __ptr + __n);
+  }
+
+private:
+  _Container* __container_;
+};
+
+/// Selects the type of the writer used for the output iterator.
+template <class _OutIt, class _CharT>
+class _LIBCPP_TEMPLATE_VIS __writer_selector {
+  using _Container = typename __back_insert_iterator_container<_OutIt>::type;
+
+public:
+  using type =
+      conditional_t<!same_as<_Container, void>,
+                    __writer_container<_Container>,
+                    conditional_t<__enable_direct_output<_OutIt, _CharT>,
+                                  __writer_direct<_OutIt, _CharT>,
+                                  __writer_iterator<_OutIt, _CharT>>>;
+};
+
+/// The generic formatting buffer.
+template <class _OutIt, __fmt_char_type _CharT>
+  requires(output_iterator<_OutIt, const _CharT&>)
+class _LIBCPP_TEMPLATE_VIS __format_buffer {
+  using _Storage =
+      conditional_t<__enable_direct_output<_OutIt, _CharT>, __direct_storage<_CharT>, __internal_storage<_CharT>>;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __format_buffer(_OutIt __out_it)
+    requires(same_as<_Storage, __internal_storage<_CharT>>)
+      : __output_(__storage_.__begin(), __storage_.__buffer_size, this), __writer_(std::move(__out_it)) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit __format_buffer(_OutIt __out_it)
+    requires(same_as<_Storage, __direct_storage<_CharT>>)
+      : __output_(std::__unwrap_iter(__out_it), size_t(-1), this), __writer_(std::move(__out_it)) {}
+
+  _LIBCPP_HIDE_FROM_ABI auto __make_output_iterator() { return __output_.__make_output_iterator(); }
+
+  _LIBCPP_HIDE_FROM_ABI void __flush(_CharT* __ptr, size_t __n) { __writer_.__flush(__ptr, __n); }
+
+  _LIBCPP_HIDE_FROM_ABI _OutIt __out_it() && {
+    __output_.__flush();
+    return std::move(__writer_).__out_it();
+  }
+
+private:
+  _LIBCPP_NO_UNIQUE_ADDRESS _Storage __storage_;
+  __output_buffer<_CharT> __output_;
+  typename __writer_selector<_OutIt, _CharT>::type __writer_;
+};
+
+/// A buffer that counts the number of insertions.
+///
+/// Since \ref formatted_size only needs to know the size, the output itself is
+/// discarded.
+template <__fmt_char_type _CharT>
+class _LIBCPP_TEMPLATE_VIS __formatted_size_buffer {
+public:
+  _LIBCPP_HIDE_FROM_ABI auto __make_output_iterator() { return __output_.__make_output_iterator(); }
+
+  _LIBCPP_HIDE_FROM_ABI void __flush(const _CharT*, size_t __n) { __size_ += __n; }
+
+  _LIBCPP_HIDE_FROM_ABI size_t __result() && {
+    __output_.__flush();
+    return __size_;
+  }
+
+private:
+  __internal_storage<_CharT> __storage_;
+  __output_buffer<_CharT> __output_{__storage_.__begin(), __storage_.__buffer_size, this};
+  size_t __size_{0};
+};
+
+/// The base of a buffer that counts and limits the number of insertions.
+template <class _OutIt, __fmt_char_type _CharT, bool>
+  requires(output_iterator<_OutIt, const _CharT&>)
+struct _LIBCPP_TEMPLATE_VIS __format_to_n_buffer_base {
+  using _Size = iter_difference_t<_OutIt>;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __format_to_n_buffer_base(_OutIt __out_it, _Size __max_size)
+      : __writer_(std::move(__out_it)), __max_size_(std::max(_Size(0), __max_size)) {}
+
+  _LIBCPP_HIDE_FROM_ABI void __flush(_CharT* __ptr, size_t __n) {
+    if (_Size(__size_) <= __max_size_)
+      __writer_.__flush(__ptr, std::min(_Size(__n), __max_size_ - __size_));
+    __size_ += __n;
+  }
+
+protected:
+  __internal_storage<_CharT> __storage_;
+  __output_buffer<_CharT> __output_{__storage_.__begin(), __storage_.__buffer_size, this};
+  typename __writer_selector<_OutIt, _CharT>::type __writer_;
+
+  _Size __max_size_;
+  _Size __size_{0};
+};
+
+/// The base of a buffer that counts and limits the number of insertions.
+///
+/// This version is used when \c __enable_direct_output<_OutIt, _CharT> == true.
+///
+/// This class limits the size available to the direct writer so it will not
+/// exceed the maximum number of code units.
+template <class _OutIt, __fmt_char_type _CharT>
+  requires(output_iterator<_OutIt, const _CharT&>)
+class _LIBCPP_TEMPLATE_VIS __format_to_n_buffer_base<_OutIt, _CharT, true> {
+  using _Size = iter_difference_t<_OutIt>;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __format_to_n_buffer_base(_OutIt __out_it, _Size __max_size)
+      : __output_(std::__unwrap_iter(__out_it), __max_size, this),
+        __writer_(std::move(__out_it)),
+        __max_size_(__max_size) {
+    if (__max_size <= 0) [[unlikely]]
+      __output_.__reset(__storage_.__begin(), __storage_.__buffer_size);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __flush(_CharT* __ptr, size_t __n) {
+    // A __flush to the direct writer happens in the following occasions:
+    // - The format function has written the maximum number of allowed code
+    //   units. At this point it's no longer valid to write to this writer. So
+    //   switch to the internal storage. This internal storage doesn't need to
+    //   be written anywhere so the __flush for that storage writes no output.
+    // - Like above, but the next "mass write" operation would overflow the
+    //   buffer. In that case the buffer is pre-emptively switched. The still
+    //   valid code units will be written separately.
+    // - The format_to_n function is finished. In this case there's no need to
+    //   switch the buffer, but for simplicity the buffers are still switched.
+    // When the __max_size <= 0 the constructor already switched the buffers.
+    if (__size_ == 0 && __ptr != __storage_.__begin()) {
+      __writer_.__flush(__ptr, __n);
+      __output_.__reset(__storage_.__begin(), __storage_.__buffer_size);
+    } else if (__size_ < __max_size_) {
+      // Copies a part of the internal buffer to the output up to n characters.
+      // See __output_buffer<_CharT>::__flush_on_overflow for more information.
+      _Size __s = std::min(_Size(__n), __max_size_ - __size_);
+      std::copy_n(__ptr, __s, __writer_.__out_it());
+      __writer_.__flush(__ptr, __s);
+    }
+
+    __size_ += __n;
+  }
+
+protected:
+  __internal_storage<_CharT> __storage_;
+  __output_buffer<_CharT> __output_;
+  __writer_direct<_OutIt, _CharT> __writer_;
+
+  _Size __max_size_;
+  _Size __size_{0};
+};
+
+/// The buffer that counts and limits the number of insertions.
+template <class _OutIt, __fmt_char_type _CharT>
+  requires(output_iterator<_OutIt, const _CharT&>)
+struct _LIBCPP_TEMPLATE_VIS __format_to_n_buffer final
+    : public __format_to_n_buffer_base< _OutIt, _CharT, __enable_direct_output<_OutIt, _CharT>> {
+  using _Base = __format_to_n_buffer_base<_OutIt, _CharT, __enable_direct_output<_OutIt, _CharT>>;
+  using _Size = iter_difference_t<_OutIt>;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __format_to_n_buffer(_OutIt __out_it, _Size __max_size)
+      : _Base(std::move(__out_it), __max_size) {}
+  _LIBCPP_HIDE_FROM_ABI auto __make_output_iterator() { return this->__output_.__make_output_iterator(); }
+
+  _LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> __result() && {
+    this->__output_.__flush();
+    return {std::move(this->__writer_).__out_it(), this->__size_};
+  }
+};
+
+// A dynamically growing buffer intended to be used for retargeting a context.
+//
+// P2286 Formatting ranges adds range formatting support. It allows the user to
+// specify the minimum width for the entire formatted range.  The width of the
+// range is not known until the range is formatted. Formatting is done to an
+// output_iterator so there's no guarantee it would be possible to add the fill
+// to the front of the output. Instead the range is formatted to a temporary
+// buffer and that buffer is formatted as a string.
+//
+// There is an issue with that approach, the format context used in
+// std::formatter<T>::format contains the output iterator used as part of its
+// type. So using this output iterator means there needs to be a new format
+// context and the format arguments need to be retargeted to the new context.
+// This retargeting is done by a basic_format_context specialized for the
+// __iterator of this container.
+//
+// This class uses its own buffer management, since using vector
+// would lead to a circular include with formatter for vector<bool>.
+template <__fmt_char_type _CharT>
+class _LIBCPP_TEMPLATE_VIS __retarget_buffer {
+  using _Alloc = allocator<_CharT>;
+
+public:
+  using value_type = _CharT;
+
+  struct __iterator {
+    using difference_type = ptrdiff_t;
+    using value_type      = _CharT;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __iterator(__retarget_buffer& __buffer)
+        : __buffer_(std::addressof(__buffer)) {}
+    _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator=(const _CharT& __c) {
+      __buffer_->push_back(__c);
+      return *this;
+    }
+    _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator=(_CharT&& __c) {
+      __buffer_->push_back(__c);
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator*() { return *this; }
+    _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() { return *this; }
+    _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator++(int) { return *this; }
+    __retarget_buffer* __buffer_;
+  };
+
+  __retarget_buffer(const __retarget_buffer&)            = delete;
+  __retarget_buffer& operator=(const __retarget_buffer&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __retarget_buffer(size_t __size_hint) {
+    // When the initial size is very small a lot of resizes happen
+    // when elements added. So use a hard-coded minimum size.
+    //
+    // Note a size < 2 will not work
+    // - 0 there is no buffer, while push_back requires 1 empty element.
+    // - 1 multiplied by the grow factor is 1 and thus the buffer never
+    //   grows.
+    auto __result = std::__allocate_at_least(__alloc_, std::max(__size_hint, 256 / sizeof(_CharT)));
+    __ptr_        = __result.ptr;
+    __capacity_   = __result.count;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~__retarget_buffer() {
+    ranges::destroy_n(__ptr_, __size_);
+    allocator_traits<_Alloc>::deallocate(__alloc_, __ptr_, __capacity_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __iterator __make_output_iterator() { return __iterator{*this}; }
+
+  _LIBCPP_HIDE_FROM_ABI void push_back(_CharT __c) {
+    std::construct_at(__ptr_ + __size_, __c);
+    ++__size_;
+
+    if (__size_ == __capacity_)
+      __grow_buffer();
+  }
+
+  template <__fmt_char_type _InCharT>
+  _LIBCPP_HIDE_FROM_ABI void __copy(basic_string_view<_InCharT> __str) {
+    size_t __n = __str.size();
+    if (__size_ + __n >= __capacity_)
+      // Push_back requires the buffer to have room for at least one character.
+      __grow_buffer(__size_ + __n + 1);
+
+    std::uninitialized_copy_n(__str.data(), __n, __ptr_ + __size_);
+    __size_ += __n;
+  }
+
+  template <contiguous_iterator _Iterator,
+            class _UnaryOperation,
+            __fmt_char_type _InCharT = typename iterator_traits<_Iterator>::value_type>
+  _LIBCPP_HIDE_FROM_ABI void __transform(_Iterator __first, _Iterator __last, _UnaryOperation __operation) {
+    _LIBCPP_ASSERT_INTERNAL(__first <= __last, "not a valid range");
+
+    size_t __n = static_cast<size_t>(__last - __first);
+    if (__size_ + __n >= __capacity_)
+      // Push_back requires the buffer to have room for at least one character.
+      __grow_buffer(__size_ + __n + 1);
+
+    std::uninitialized_default_construct_n(__ptr_ + __size_, __n);
+    std::transform(__first, __last, __ptr_ + __size_, std::move(__operation));
+    __size_ += __n;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __fill(size_t __n, _CharT __value) {
+    if (__size_ + __n >= __capacity_)
+      // Push_back requires the buffer to have room for at least one character.
+      __grow_buffer(__size_ + __n + 1);
+
+    std::uninitialized_fill_n(__ptr_ + __size_, __n, __value);
+    __size_ += __n;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI basic_string_view<_CharT> __view() { return {__ptr_, __size_}; }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI void __grow_buffer() { __grow_buffer(__capacity_ * 1.6); }
+
+  _LIBCPP_HIDE_FROM_ABI void __grow_buffer(size_t __capacity) {
+    _LIBCPP_ASSERT_INTERNAL(__capacity > __capacity_, "the buffer must grow");
+    auto __result = std::__allocate_at_least(__alloc_, __capacity);
+    auto __guard  = std::__make_exception_guard([&] {
+      allocator_traits<_Alloc>::deallocate(__alloc_, __result.ptr, __result.count);
+    });
+    // This shouldn't throw, but just to be safe. Note that at -O1 this
+    // guard is optimized away so there is no runtime overhead.
+    std::uninitialized_move_n(__ptr_, __size_, __result.ptr);
+    __guard.__complete();
+    ranges::destroy_n(__ptr_, __size_);
+    allocator_traits<_Alloc>::deallocate(__alloc_, __ptr_, __capacity_);
+
+    __ptr_      = __result.ptr;
+    __capacity_ = __result.count;
+  }
+  _LIBCPP_NO_UNIQUE_ADDRESS _Alloc __alloc_;
+  _CharT* __ptr_;
+  size_t __capacity_;
+  size_t __size_{0};
+};
+
+} // namespace __format
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FORMAT_BUFFER_H
diff --git a/libcxx/include/__cxx03/__format/concepts.h b/libcxx/include/__cxx03/__format/concepts.h
new file mode 100644
index 000000000000..13380e9b91af
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/concepts.h
@@ -0,0 +1,83 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_CONCEPTS_H
+#define _LIBCPP___FORMAT_CONCEPTS_H
+
+#include <__concepts/same_as.h>
+#include <__concepts/semiregular.h>
+#include <__config>
+#include <__format/format_parse_context.h>
+#include <__fwd/format.h>
+#include <__fwd/tuple.h>
+#include <__tuple/tuple_size.h>
+#include <__type_traits/is_specialization.h>
+#include <__type_traits/remove_const.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+/// The character type specializations of \ref formatter.
+template <class _CharT>
+concept __fmt_char_type =
+    same_as<_CharT, char>
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+    || same_as<_CharT, wchar_t>
+#  endif
+    ;
+
+// The output iterator isn't specified. A formatter should accept any
+// output_iterator. This iterator is a minimal iterator to test the concept.
+// (Note testing for (w)format_context would be a valid choice, but requires
+// selecting the proper one depending on the type of _CharT.)
+template <class _CharT>
+using __fmt_iter_for = _CharT*;
+
+template <class _Tp, class _Context, class _Formatter = typename _Context::template formatter_type<remove_const_t<_Tp>>>
+concept __formattable_with =
+    semiregular<_Formatter> &&
+    requires(_Formatter& __f,
+             const _Formatter& __cf,
+             _Tp&& __t,
+             _Context __fc,
+             basic_format_parse_context<typename _Context::char_type> __pc) {
+      { __f.parse(__pc) } -> same_as<typename decltype(__pc)::iterator>;
+      { __cf.format(__t, __fc) } -> same_as<typename _Context::iterator>;
+    };
+
+template <class _Tp, class _CharT>
+concept __formattable =
+    __formattable_with<remove_reference_t<_Tp>, basic_format_context<__fmt_iter_for<_CharT>, _CharT>>;
+
+#  if _LIBCPP_STD_VER >= 23
+template <class _Tp, class _CharT>
+concept formattable = __formattable<_Tp, _CharT>;
+
+// [tuple.like] defines a tuple-like exposition only concept. This concept is
+// not related to that. Therefore it uses a different name for the concept.
+//
+// TODO FMT Add a test to validate we fail when using that concept after P2165
+// has been implemented.
+template <class _Tp>
+concept __fmt_pair_like =
+    __is_specialization_v<_Tp, pair> || (__is_specialization_v<_Tp, tuple> && tuple_size_v<_Tp> == 2);
+
+#  endif //_LIBCPP_STD_VER >= 23
+#endif   //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_CONCEPTS_H
diff --git a/libcxx/include/__cxx03/__format/container_adaptor.h b/libcxx/include/__cxx03/__format/container_adaptor.h
new file mode 100644
index 000000000000..9f49ca03bf4f
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/container_adaptor.h
@@ -0,0 +1,73 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_CONTAINER_ADAPTOR_H
+#define _LIBCPP___FORMAT_CONTAINER_ADAPTOR_H
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#include <__config>
+#include <__format/concepts.h>
+#include <__format/formatter.h>
+#include <__format/range_default_formatter.h>
+#include <__fwd/queue.h>
+#include <__fwd/stack.h>
+#include <__ranges/ref_view.h>
+#include <__type_traits/is_const.h>
+#include <__type_traits/maybe_const.h>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+// [container.adaptors.format] only specifies the library should provide the
+// formatter specializations, not which header should provide them.
+// Since <format> includes a lot of headers, add these headers here instead of
+// adding more dependencies like, locale, optinal, string, tuple, etc. to the
+// adaptor headers. To use the format functions users already include <format>.
+
+template <class _Adaptor, class _CharT>
+struct _LIBCPP_TEMPLATE_VIS __formatter_container_adaptor {
+private:
+  using __maybe_const_container = __fmt_maybe_const<typename _Adaptor::container_type, _CharT>;
+  using __maybe_const_adaptor   = __maybe_const<is_const_v<__maybe_const_container>, _Adaptor>;
+  formatter<ranges::ref_view<__maybe_const_container>, _CharT> __underlying_;
+
+public:
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return __underlying_.parse(__ctx);
+  }
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator
+  format(__maybe_const_adaptor& __adaptor, _FormatContext& __ctx) const {
+    return __underlying_.format(__adaptor.__get_container(), __ctx);
+  }
+};
+
+template <class _CharT, class _Tp, formattable<_CharT> _Container>
+struct _LIBCPP_TEMPLATE_VIS formatter<queue<_Tp, _Container>, _CharT>
+    : public __formatter_container_adaptor<queue<_Tp, _Container>, _CharT> {};
+
+template <class _CharT, class _Tp, class _Container, class _Compare>
+struct _LIBCPP_TEMPLATE_VIS formatter<priority_queue<_Tp, _Container, _Compare>, _CharT>
+    : public __formatter_container_adaptor<priority_queue<_Tp, _Container, _Compare>, _CharT> {};
+
+template <class _CharT, class _Tp, formattable<_CharT> _Container>
+struct _LIBCPP_TEMPLATE_VIS formatter<stack<_Tp, _Container>, _CharT>
+    : public __formatter_container_adaptor<stack<_Tp, _Container>, _CharT> {};
+
+#endif //_LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_CONTAINER_ADAPTOR_H
diff --git a/libcxx/include/__cxx03/__format/enable_insertable.h b/libcxx/include/__cxx03/__format/enable_insertable.h
new file mode 100644
index 000000000000..86ef94a325b1
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/enable_insertable.h
@@ -0,0 +1,35 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_ENABLE_INSERTABLE_H
+#define _LIBCPP___FORMAT_ENABLE_INSERTABLE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __format {
+
+/// Opt-in to enable \ref __insertable for a \a _Container.
+template <class _Container>
+inline constexpr bool __enable_insertable = false;
+
+} // namespace __format
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_ENABLE_INSERTABLE_H
diff --git a/libcxx/include/__cxx03/__format/escaped_output_table.h b/libcxx/include/__cxx03/__format/escaped_output_table.h
new file mode 100644
index 000000000000..f7be2dc61f21
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/escaped_output_table.h
@@ -0,0 +1,863 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// WARNING, this entire header is generated by
+// utils/generate_escaped_output_table.py
+// DO NOT MODIFY!
+
+// UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
+//
+// See Terms of Use <https://www.unicode.org/copyright.html>
+// for definitions of Unicode Inc.'s Data Files and Software.
+//
+// NOTICE TO USER: Carefully read the following legal agreement.
+// BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S
+// DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"),
+// YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
+// TERMS AND CONDITIONS OF THIS AGREEMENT.
+// IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE
+// THE DATA FILES OR SOFTWARE.
+//
+// COPYRIGHT AND PERMISSION NOTICE
+//
+// Copyright (c) 1991-2022 Unicode, Inc. All rights reserved.
+// Distributed under the Terms of Use in https://www.unicode.org/copyright.html.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of the Unicode data files and any associated documentation
+// (the "Data Files") or Unicode software and any associated documentation
+// (the "Software") to deal in the Data Files or Software
+// without restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, and/or sell copies of
+// the Data Files or Software, and to permit persons to whom the Data Files
+// or Software are furnished to do so, provided that either
+// (a) this copyright and permission notice appear with all copies
+// of the Data Files or Software, or
+// (b) this copyright and permission notice appear in associated
+// Documentation.
+//
+// THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
+// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT OF THIRD PARTY RIGHTS.
+// IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
+// NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
+// DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+// DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+//
+// Except as contained in this notice, the name of a copyright holder
+// shall not be used in advertising or otherwise to promote the sale,
+// use or other dealings in these Data Files or Software without prior
+// written authorization of the copyright holder.
+
+#ifndef _LIBCPP___FORMAT_ESCAPED_OUTPUT_TABLE_H
+#define _LIBCPP___FORMAT_ESCAPED_OUTPUT_TABLE_H
+
+#include <__algorithm/ranges_upper_bound.h>
+#include <__config>
+#include <cstddef>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+namespace __escaped_output_table {
+// clang-format off
+
+/// The entries of the characters to escape in format's debug string.
+///
+/// Contains the entries for [format.string.escaped]/2.2.1.2.1
+///   CE is a Unicode encoding and C corresponds to a UCS scalar value whose
+///   Unicode property General_Category has a value in the groups Separator (Z)
+///   or Other (C), as described by table 12 of UAX #44
+///
+/// Separator (Z) consists of General_Category
+/// - Space_Separator,
+/// - Line_Separator,
+/// - Paragraph_Separator.
+///
+/// Other (C) consists of General_Category
+/// - Control,
+/// - Format,
+/// - Surrogate,
+/// - Private_Use,
+/// - Unassigned.
+///
+/// The data is generated from
+/// - https://www.unicode.org/Public/UCD/latest/ucd/extracted/DerivedGeneralCategory.txt
+///
+/// The table is similar to the table
+///  __extended_grapheme_custer_property_boundary::__entries
+/// which explains the details of these classes. The only difference is this
+/// table lacks a property, thus having more bits available for the size.
+///
+/// The data has 2 values:
+/// - bits [0, 13] The size of the range, allowing 16384 elements.
+/// - bits [14, 31] The lower bound code point of the range. The upper bound of
+///   the range is lower bound + size. Note the code expects code units the fit
+///   into 18 bits, instead of the 21 bits needed for the full Unicode range.
+_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
+    0x00000020 /* 00000000 - 00000020 [   33] */,
+    0x001fc021 /* 0000007f - 000000a0 [   34] */,
+    0x002b4000 /* 000000ad - 000000ad [    1] */,
+    0x00de0001 /* 00000378 - 00000379 [    2] */,
+    0x00e00003 /* 00000380 - 00000383 [    4] */,
+    0x00e2c000 /* 0000038b - 0000038b [    1] */,
+    0x00e34000 /* 0000038d - 0000038d [    1] */,
+    0x00e88000 /* 000003a2 - 000003a2 [    1] */,
+    0x014c0000 /* 00000530 - 00000530 [    1] */,
+    0x0155c001 /* 00000557 - 00000558 [    2] */,
+    0x0162c001 /* 0000058b - 0000058c [    2] */,
+    0x01640000 /* 00000590 - 00000590 [    1] */,
+    0x01720007 /* 000005c8 - 000005cf [    8] */,
+    0x017ac003 /* 000005eb - 000005ee [    4] */,
+    0x017d4010 /* 000005f5 - 00000605 [   17] */,
+    0x01870000 /* 0000061c - 0000061c [    1] */,
+    0x01b74000 /* 000006dd - 000006dd [    1] */,
+    0x01c38001 /* 0000070e - 0000070f [    2] */,
+    0x01d2c001 /* 0000074b - 0000074c [    2] */,
+    0x01ec800d /* 000007b2 - 000007bf [   14] */,
+    0x01fec001 /* 000007fb - 000007fc [    2] */,
+    0x020b8001 /* 0000082e - 0000082f [    2] */,
+    0x020fc000 /* 0000083f - 0000083f [    1] */,
+    0x02170001 /* 0000085c - 0000085d [    2] */,
+    0x0217c000 /* 0000085f - 0000085f [    1] */,
+    0x021ac004 /* 0000086b - 0000086f [    5] */,
+    0x0223c008 /* 0000088f - 00000897 [    9] */,
+    0x02388000 /* 000008e2 - 000008e2 [    1] */,
+    0x02610000 /* 00000984 - 00000984 [    1] */,
+    0x02634001 /* 0000098d - 0000098e [    2] */,
+    0x02644001 /* 00000991 - 00000992 [    2] */,
+    0x026a4000 /* 000009a9 - 000009a9 [    1] */,
+    0x026c4000 /* 000009b1 - 000009b1 [    1] */,
+    0x026cc002 /* 000009b3 - 000009b5 [    3] */,
+    0x026e8001 /* 000009ba - 000009bb [    2] */,
+    0x02714001 /* 000009c5 - 000009c6 [    2] */,
+    0x02724001 /* 000009c9 - 000009ca [    2] */,
+    0x0273c007 /* 000009cf - 000009d6 [    8] */,
+    0x02760003 /* 000009d8 - 000009db [    4] */,
+    0x02778000 /* 000009de - 000009de [    1] */,
+    0x02790001 /* 000009e4 - 000009e5 [    2] */,
+    0x027fc001 /* 000009ff - 00000a00 [    2] */,
+    0x02810000 /* 00000a04 - 00000a04 [    1] */,
+    0x0282c003 /* 00000a0b - 00000a0e [    4] */,
+    0x02844001 /* 00000a11 - 00000a12 [    2] */,
+    0x028a4000 /* 00000a29 - 00000a29 [    1] */,
+    0x028c4000 /* 00000a31 - 00000a31 [    1] */,
+    0x028d0000 /* 00000a34 - 00000a34 [    1] */,
+    0x028dc000 /* 00000a37 - 00000a37 [    1] */,
+    0x028e8001 /* 00000a3a - 00000a3b [    2] */,
+    0x028f4000 /* 00000a3d - 00000a3d [    1] */,
+    0x0290c003 /* 00000a43 - 00000a46 [    4] */,
+    0x02924001 /* 00000a49 - 00000a4a [    2] */,
+    0x02938002 /* 00000a4e - 00000a50 [    3] */,
+    0x02948006 /* 00000a52 - 00000a58 [    7] */,
+    0x02974000 /* 00000a5d - 00000a5d [    1] */,
+    0x0297c006 /* 00000a5f - 00000a65 [    7] */,
+    0x029dc009 /* 00000a77 - 00000a80 [   10] */,
+    0x02a10000 /* 00000a84 - 00000a84 [    1] */,
+    0x02a38000 /* 00000a8e - 00000a8e [    1] */,
+    0x02a48000 /* 00000a92 - 00000a92 [    1] */,
+    0x02aa4000 /* 00000aa9 - 00000aa9 [    1] */,
+    0x02ac4000 /* 00000ab1 - 00000ab1 [    1] */,
+    0x02ad0000 /* 00000ab4 - 00000ab4 [    1] */,
+    0x02ae8001 /* 00000aba - 00000abb [    2] */,
+    0x02b18000 /* 00000ac6 - 00000ac6 [    1] */,
+    0x02b28000 /* 00000aca - 00000aca [    1] */,
+    0x02b38001 /* 00000ace - 00000acf [    2] */,
+    0x02b4400e /* 00000ad1 - 00000adf [   15] */,
+    0x02b90001 /* 00000ae4 - 00000ae5 [    2] */,
+    0x02bc8006 /* 00000af2 - 00000af8 [    7] */,
+    0x02c00000 /* 00000b00 - 00000b00 [    1] */,
+    0x02c10000 /* 00000b04 - 00000b04 [    1] */,
+    0x02c34001 /* 00000b0d - 00000b0e [    2] */,
+    0x02c44001 /* 00000b11 - 00000b12 [    2] */,
+    0x02ca4000 /* 00000b29 - 00000b29 [    1] */,
+    0x02cc4000 /* 00000b31 - 00000b31 [    1] */,
+    0x02cd0000 /* 00000b34 - 00000b34 [    1] */,
+    0x02ce8001 /* 00000b3a - 00000b3b [    2] */,
+    0x02d14001 /* 00000b45 - 00000b46 [    2] */,
+    0x02d24001 /* 00000b49 - 00000b4a [    2] */,
+    0x02d38006 /* 00000b4e - 00000b54 [    7] */,
+    0x02d60003 /* 00000b58 - 00000b5b [    4] */,
+    0x02d78000 /* 00000b5e - 00000b5e [    1] */,
+    0x02d90001 /* 00000b64 - 00000b65 [    2] */,
+    0x02de0009 /* 00000b78 - 00000b81 [   10] */,
+    0x02e10000 /* 00000b84 - 00000b84 [    1] */,
+    0x02e2c002 /* 00000b8b - 00000b8d [    3] */,
+    0x02e44000 /* 00000b91 - 00000b91 [    1] */,
+    0x02e58002 /* 00000b96 - 00000b98 [    3] */,
+    0x02e6c000 /* 00000b9b - 00000b9b [    1] */,
+    0x02e74000 /* 00000b9d - 00000b9d [    1] */,
+    0x02e80002 /* 00000ba0 - 00000ba2 [    3] */,
+    0x02e94002 /* 00000ba5 - 00000ba7 [    3] */,
+    0x02eac002 /* 00000bab - 00000bad [    3] */,
+    0x02ee8003 /* 00000bba - 00000bbd [    4] */,
+    0x02f0c002 /* 00000bc3 - 00000bc5 [    3] */,
+    0x02f24000 /* 00000bc9 - 00000bc9 [    1] */,
+    0x02f38001 /* 00000bce - 00000bcf [    2] */,
+    0x02f44005 /* 00000bd1 - 00000bd6 [    6] */,
+    0x02f6000d /* 00000bd8 - 00000be5 [   14] */,
+    0x02fec004 /* 00000bfb - 00000bff [    5] */,
+    0x03034000 /* 00000c0d - 00000c0d [    1] */,
+    0x03044000 /* 00000c11 - 00000c11 [    1] */,
+    0x030a4000 /* 00000c29 - 00000c29 [    1] */,
+    0x030e8001 /* 00000c3a - 00000c3b [    2] */,
+    0x03114000 /* 00000c45 - 00000c45 [    1] */,
+    0x03124000 /* 00000c49 - 00000c49 [    1] */,
+    0x03138006 /* 00000c4e - 00000c54 [    7] */,
+    0x0315c000 /* 00000c57 - 00000c57 [    1] */,
+    0x0316c001 /* 00000c5b - 00000c5c [    2] */,
+    0x03178001 /* 00000c5e - 00000c5f [    2] */,
+    0x03190001 /* 00000c64 - 00000c65 [    2] */,
+    0x031c0006 /* 00000c70 - 00000c76 [    7] */,
+    0x03234000 /* 00000c8d - 00000c8d [    1] */,
+    0x03244000 /* 00000c91 - 00000c91 [    1] */,
+    0x032a4000 /* 00000ca9 - 00000ca9 [    1] */,
+    0x032d0000 /* 00000cb4 - 00000cb4 [    1] */,
+    0x032e8001 /* 00000cba - 00000cbb [    2] */,
+    0x03314000 /* 00000cc5 - 00000cc5 [    1] */,
+    0x03324000 /* 00000cc9 - 00000cc9 [    1] */,
+    0x03338006 /* 00000cce - 00000cd4 [    7] */,
+    0x0335c005 /* 00000cd7 - 00000cdc [    6] */,
+    0x0337c000 /* 00000cdf - 00000cdf [    1] */,
+    0x03390001 /* 00000ce4 - 00000ce5 [    2] */,
+    0x033c0000 /* 00000cf0 - 00000cf0 [    1] */,
+    0x033d000b /* 00000cf4 - 00000cff [   12] */,
+    0x03434000 /* 00000d0d - 00000d0d [    1] */,
+    0x03444000 /* 00000d11 - 00000d11 [    1] */,
+    0x03514000 /* 00000d45 - 00000d45 [    1] */,
+    0x03524000 /* 00000d49 - 00000d49 [    1] */,
+    0x03540003 /* 00000d50 - 00000d53 [    4] */,
+    0x03590001 /* 00000d64 - 00000d65 [    2] */,
+    0x03600000 /* 00000d80 - 00000d80 [    1] */,
+    0x03610000 /* 00000d84 - 00000d84 [    1] */,
+    0x0365c002 /* 00000d97 - 00000d99 [    3] */,
+    0x036c8000 /* 00000db2 - 00000db2 [    1] */,
+    0x036f0000 /* 00000dbc - 00000dbc [    1] */,
+    0x036f8001 /* 00000dbe - 00000dbf [    2] */,
+    0x0371c002 /* 00000dc7 - 00000dc9 [    3] */,
+    0x0372c003 /* 00000dcb - 00000dce [    4] */,
+    0x03754000 /* 00000dd5 - 00000dd5 [    1] */,
+    0x0375c000 /* 00000dd7 - 00000dd7 [    1] */,
+    0x03780005 /* 00000de0 - 00000de5 [    6] */,
+    0x037c0001 /* 00000df0 - 00000df1 [    2] */,
+    0x037d400b /* 00000df5 - 00000e00 [   12] */,
+    0x038ec003 /* 00000e3b - 00000e3e [    4] */,
+    0x03970024 /* 00000e5c - 00000e80 [   37] */,
+    0x03a0c000 /* 00000e83 - 00000e83 [    1] */,
+    0x03a14000 /* 00000e85 - 00000e85 [    1] */,
+    0x03a2c000 /* 00000e8b - 00000e8b [    1] */,
+    0x03a90000 /* 00000ea4 - 00000ea4 [    1] */,
+    0x03a98000 /* 00000ea6 - 00000ea6 [    1] */,
+    0x03af8001 /* 00000ebe - 00000ebf [    2] */,
+    0x03b14000 /* 00000ec5 - 00000ec5 [    1] */,
+    0x03b1c000 /* 00000ec7 - 00000ec7 [    1] */,
+    0x03b3c000 /* 00000ecf - 00000ecf [    1] */,
+    0x03b68001 /* 00000eda - 00000edb [    2] */,
+    0x03b8001f /* 00000ee0 - 00000eff [   32] */,
+    0x03d20000 /* 00000f48 - 00000f48 [    1] */,
+    0x03db4003 /* 00000f6d - 00000f70 [    4] */,
+    0x03e60000 /* 00000f98 - 00000f98 [    1] */,
+    0x03ef4000 /* 00000fbd - 00000fbd [    1] */,
+    0x03f34000 /* 00000fcd - 00000fcd [    1] */,
+    0x03f6c024 /* 00000fdb - 00000fff [   37] */,
+    0x04318000 /* 000010c6 - 000010c6 [    1] */,
+    0x04320004 /* 000010c8 - 000010cc [    5] */,
+    0x04338001 /* 000010ce - 000010cf [    2] */,
+    0x04924000 /* 00001249 - 00001249 [    1] */,
+    0x04938001 /* 0000124e - 0000124f [    2] */,
+    0x0495c000 /* 00001257 - 00001257 [    1] */,
+    0x04964000 /* 00001259 - 00001259 [    1] */,
+    0x04978001 /* 0000125e - 0000125f [    2] */,
+    0x04a24000 /* 00001289 - 00001289 [    1] */,
+    0x04a38001 /* 0000128e - 0000128f [    2] */,
+    0x04ac4000 /* 000012b1 - 000012b1 [    1] */,
+    0x04ad8001 /* 000012b6 - 000012b7 [    2] */,
+    0x04afc000 /* 000012bf - 000012bf [    1] */,
+    0x04b04000 /* 000012c1 - 000012c1 [    1] */,
+    0x04b18001 /* 000012c6 - 000012c7 [    2] */,
+    0x04b5c000 /* 000012d7 - 000012d7 [    1] */,
+    0x04c44000 /* 00001311 - 00001311 [    1] */,
+    0x04c58001 /* 00001316 - 00001317 [    2] */,
+    0x04d6c001 /* 0000135b - 0000135c [    2] */,
+    0x04df4002 /* 0000137d - 0000137f [    3] */,
+    0x04e68005 /* 0000139a - 0000139f [    6] */,
+    0x04fd8001 /* 000013f6 - 000013f7 [    2] */,
+    0x04ff8001 /* 000013fe - 000013ff [    2] */,
+    0x05a00000 /* 00001680 - 00001680 [    1] */,
+    0x05a74002 /* 0000169d - 0000169f [    3] */,
+    0x05be4006 /* 000016f9 - 000016ff [    7] */,
+    0x05c58008 /* 00001716 - 0000171e [    9] */,
+    0x05cdc008 /* 00001737 - 0000173f [    9] */,
+    0x05d5000b /* 00001754 - 0000175f [   12] */,
+    0x05db4000 /* 0000176d - 0000176d [    1] */,
+    0x05dc4000 /* 00001771 - 00001771 [    1] */,
+    0x05dd000b /* 00001774 - 0000177f [   12] */,
+    0x05f78001 /* 000017de - 000017df [    2] */,
+    0x05fa8005 /* 000017ea - 000017ef [    6] */,
+    0x05fe8005 /* 000017fa - 000017ff [    6] */,
+    0x06038000 /* 0000180e - 0000180e [    1] */,
+    0x06068005 /* 0000181a - 0000181f [    6] */,
+    0x061e4006 /* 00001879 - 0000187f [    7] */,
+    0x062ac004 /* 000018ab - 000018af [    5] */,
+    0x063d8009 /* 000018f6 - 000018ff [   10] */,
+    0x0647c000 /* 0000191f - 0000191f [    1] */,
+    0x064b0003 /* 0000192c - 0000192f [    4] */,
+    0x064f0003 /* 0000193c - 0000193f [    4] */,
+    0x06504002 /* 00001941 - 00001943 [    3] */,
+    0x065b8001 /* 0000196e - 0000196f [    2] */,
+    0x065d400a /* 00001975 - 0000197f [   11] */,
+    0x066b0003 /* 000019ac - 000019af [    4] */,
+    0x06728005 /* 000019ca - 000019cf [    6] */,
+    0x0676c002 /* 000019db - 000019dd [    3] */,
+    0x06870001 /* 00001a1c - 00001a1d [    2] */,
+    0x0697c000 /* 00001a5f - 00001a5f [    1] */,
+    0x069f4001 /* 00001a7d - 00001a7e [    2] */,
+    0x06a28005 /* 00001a8a - 00001a8f [    6] */,
+    0x06a68005 /* 00001a9a - 00001a9f [    6] */,
+    0x06ab8001 /* 00001aae - 00001aaf [    2] */,
+    0x06b3c030 /* 00001acf - 00001aff [   49] */,
+    0x06d34002 /* 00001b4d - 00001b4f [    3] */,
+    0x06dfc000 /* 00001b7f - 00001b7f [    1] */,
+    0x06fd0007 /* 00001bf4 - 00001bfb [    8] */,
+    0x070e0002 /* 00001c38 - 00001c3a [    3] */,
+    0x07128002 /* 00001c4a - 00001c4c [    3] */,
+    0x07224006 /* 00001c89 - 00001c8f [    7] */,
+    0x072ec001 /* 00001cbb - 00001cbc [    2] */,
+    0x07320007 /* 00001cc8 - 00001ccf [    8] */,
+    0x073ec004 /* 00001cfb - 00001cff [    5] */,
+    0x07c58001 /* 00001f16 - 00001f17 [    2] */,
+    0x07c78001 /* 00001f1e - 00001f1f [    2] */,
+    0x07d18001 /* 00001f46 - 00001f47 [    2] */,
+    0x07d38001 /* 00001f4e - 00001f4f [    2] */,
+    0x07d60000 /* 00001f58 - 00001f58 [    1] */,
+    0x07d68000 /* 00001f5a - 00001f5a [    1] */,
+    0x07d70000 /* 00001f5c - 00001f5c [    1] */,
+    0x07d78000 /* 00001f5e - 00001f5e [    1] */,
+    0x07df8001 /* 00001f7e - 00001f7f [    2] */,
+    0x07ed4000 /* 00001fb5 - 00001fb5 [    1] */,
+    0x07f14000 /* 00001fc5 - 00001fc5 [    1] */,
+    0x07f50001 /* 00001fd4 - 00001fd5 [    2] */,
+    0x07f70000 /* 00001fdc - 00001fdc [    1] */,
+    0x07fc0001 /* 00001ff0 - 00001ff1 [    2] */,
+    0x07fd4000 /* 00001ff5 - 00001ff5 [    1] */,
+    0x07ffc010 /* 00001fff - 0000200f [   17] */,
+    0x080a0007 /* 00002028 - 0000202f [    8] */,
+    0x0817c010 /* 0000205f - 0000206f [   17] */,
+    0x081c8001 /* 00002072 - 00002073 [    2] */,
+    0x0823c000 /* 0000208f - 0000208f [    1] */,
+    0x08274002 /* 0000209d - 0000209f [    3] */,
+    0x0830400e /* 000020c1 - 000020cf [   15] */,
+    0x083c400e /* 000020f1 - 000020ff [   15] */,
+    0x08630003 /* 0000218c - 0000218f [    4] */,
+    0x0909c018 /* 00002427 - 0000243f [   25] */,
+    0x0912c014 /* 0000244b - 0000245f [   21] */,
+    0x0add0001 /* 00002b74 - 00002b75 [    2] */,
+    0x0ae58000 /* 00002b96 - 00002b96 [    1] */,
+    0x0b3d0004 /* 00002cf4 - 00002cf8 [    5] */,
+    0x0b498000 /* 00002d26 - 00002d26 [    1] */,
+    0x0b4a0004 /* 00002d28 - 00002d2c [    5] */,
+    0x0b4b8001 /* 00002d2e - 00002d2f [    2] */,
+    0x0b5a0006 /* 00002d68 - 00002d6e [    7] */,
+    0x0b5c400d /* 00002d71 - 00002d7e [   14] */,
+    0x0b65c008 /* 00002d97 - 00002d9f [    9] */,
+    0x0b69c000 /* 00002da7 - 00002da7 [    1] */,
+    0x0b6bc000 /* 00002daf - 00002daf [    1] */,
+    0x0b6dc000 /* 00002db7 - 00002db7 [    1] */,
+    0x0b6fc000 /* 00002dbf - 00002dbf [    1] */,
+    0x0b71c000 /* 00002dc7 - 00002dc7 [    1] */,
+    0x0b73c000 /* 00002dcf - 00002dcf [    1] */,
+    0x0b75c000 /* 00002dd7 - 00002dd7 [    1] */,
+    0x0b77c000 /* 00002ddf - 00002ddf [    1] */,
+    0x0b978021 /* 00002e5e - 00002e7f [   34] */,
+    0x0ba68000 /* 00002e9a - 00002e9a [    1] */,
+    0x0bbd000b /* 00002ef4 - 00002eff [   12] */,
+    0x0bf58019 /* 00002fd6 - 00002fef [   26] */,
+    0x0c000000 /* 00003000 - 00003000 [    1] */,
+    0x0c100000 /* 00003040 - 00003040 [    1] */,
+    0x0c25c001 /* 00003097 - 00003098 [    2] */,
+    0x0c400004 /* 00003100 - 00003104 [    5] */,
+    0x0c4c0000 /* 00003130 - 00003130 [    1] */,
+    0x0c63c000 /* 0000318f - 0000318f [    1] */,
+    0x0c79000a /* 000031e4 - 000031ee [   11] */,
+    0x0c87c000 /* 0000321f - 0000321f [    1] */,
+    0x29234002 /* 0000a48d - 0000a48f [    3] */,
+    0x2931c008 /* 0000a4c7 - 0000a4cf [    9] */,
+    0x298b0013 /* 0000a62c - 0000a63f [   20] */,
+    0x29be0007 /* 0000a6f8 - 0000a6ff [    8] */,
+    0x29f2c004 /* 0000a7cb - 0000a7cf [    5] */,
+    0x29f48000 /* 0000a7d2 - 0000a7d2 [    1] */,
+    0x29f50000 /* 0000a7d4 - 0000a7d4 [    1] */,
+    0x29f68017 /* 0000a7da - 0000a7f1 [   24] */,
+    0x2a0b4002 /* 0000a82d - 0000a82f [    3] */,
+    0x2a0e8005 /* 0000a83a - 0000a83f [    6] */,
+    0x2a1e0007 /* 0000a878 - 0000a87f [    8] */,
+    0x2a318007 /* 0000a8c6 - 0000a8cd [    8] */,
+    0x2a368005 /* 0000a8da - 0000a8df [    6] */,
+    0x2a55000a /* 0000a954 - 0000a95e [   11] */,
+    0x2a5f4002 /* 0000a97d - 0000a97f [    3] */,
+    0x2a738000 /* 0000a9ce - 0000a9ce [    1] */,
+    0x2a768003 /* 0000a9da - 0000a9dd [    4] */,
+    0x2a7fc000 /* 0000a9ff - 0000a9ff [    1] */,
+    0x2a8dc008 /* 0000aa37 - 0000aa3f [    9] */,
+    0x2a938001 /* 0000aa4e - 0000aa4f [    2] */,
+    0x2a968001 /* 0000aa5a - 0000aa5b [    2] */,
+    0x2ab0c017 /* 0000aac3 - 0000aada [   24] */,
+    0x2abdc009 /* 0000aaf7 - 0000ab00 [   10] */,
+    0x2ac1c001 /* 0000ab07 - 0000ab08 [    2] */,
+    0x2ac3c001 /* 0000ab0f - 0000ab10 [    2] */,
+    0x2ac5c008 /* 0000ab17 - 0000ab1f [    9] */,
+    0x2ac9c000 /* 0000ab27 - 0000ab27 [    1] */,
+    0x2acbc000 /* 0000ab2f - 0000ab2f [    1] */,
+    0x2adb0003 /* 0000ab6c - 0000ab6f [    4] */,
+    0x2afb8001 /* 0000abee - 0000abef [    2] */,
+    0x2afe8005 /* 0000abfa - 0000abff [    6] */,
+    0x35e9000b /* 0000d7a4 - 0000d7af [   12] */,
+    0x35f1c003 /* 0000d7c7 - 0000d7ca [    4] */,
+    0x35ff2103 /* 0000d7fc - 0000f8ff [ 8452] */,
+    0x3e9b8001 /* 0000fa6e - 0000fa6f [    2] */,
+    0x3eb68025 /* 0000fada - 0000faff [   38] */,
+    0x3ec1c00b /* 0000fb07 - 0000fb12 [   12] */,
+    0x3ec60004 /* 0000fb18 - 0000fb1c [    5] */,
+    0x3ecdc000 /* 0000fb37 - 0000fb37 [    1] */,
+    0x3ecf4000 /* 0000fb3d - 0000fb3d [    1] */,
+    0x3ecfc000 /* 0000fb3f - 0000fb3f [    1] */,
+    0x3ed08000 /* 0000fb42 - 0000fb42 [    1] */,
+    0x3ed14000 /* 0000fb45 - 0000fb45 [    1] */,
+    0x3ef0c00f /* 0000fbc3 - 0000fbd2 [   16] */,
+    0x3f640001 /* 0000fd90 - 0000fd91 [    2] */,
+    0x3f720006 /* 0000fdc8 - 0000fdce [    7] */,
+    0x3f74001f /* 0000fdd0 - 0000fdef [   32] */,
+    0x3f868005 /* 0000fe1a - 0000fe1f [    6] */,
+    0x3f94c000 /* 0000fe53 - 0000fe53 [    1] */,
+    0x3f99c000 /* 0000fe67 - 0000fe67 [    1] */,
+    0x3f9b0003 /* 0000fe6c - 0000fe6f [    4] */,
+    0x3f9d4000 /* 0000fe75 - 0000fe75 [    1] */,
+    0x3fbf4003 /* 0000fefd - 0000ff00 [    4] */,
+    0x3fefc002 /* 0000ffbf - 0000ffc1 [    3] */,
+    0x3ff20001 /* 0000ffc8 - 0000ffc9 [    2] */,
+    0x3ff40001 /* 0000ffd0 - 0000ffd1 [    2] */,
+    0x3ff60001 /* 0000ffd8 - 0000ffd9 [    2] */,
+    0x3ff74002 /* 0000ffdd - 0000ffdf [    3] */,
+    0x3ff9c000 /* 0000ffe7 - 0000ffe7 [    1] */,
+    0x3ffbc00c /* 0000ffef - 0000fffb [   13] */,
+    0x3fff8001 /* 0000fffe - 0000ffff [    2] */,
+    0x40030000 /* 0001000c - 0001000c [    1] */,
+    0x4009c000 /* 00010027 - 00010027 [    1] */,
+    0x400ec000 /* 0001003b - 0001003b [    1] */,
+    0x400f8000 /* 0001003e - 0001003e [    1] */,
+    0x40138001 /* 0001004e - 0001004f [    2] */,
+    0x40178021 /* 0001005e - 0001007f [   34] */,
+    0x403ec004 /* 000100fb - 000100ff [    5] */,
+    0x4040c003 /* 00010103 - 00010106 [    4] */,
+    0x404d0002 /* 00010134 - 00010136 [    3] */,
+    0x4063c000 /* 0001018f - 0001018f [    1] */,
+    0x40674002 /* 0001019d - 0001019f [    3] */,
+    0x4068402e /* 000101a1 - 000101cf [   47] */,
+    0x407f8081 /* 000101fe - 0001027f [  130] */,
+    0x40a74002 /* 0001029d - 0001029f [    3] */,
+    0x40b4400e /* 000102d1 - 000102df [   15] */,
+    0x40bf0003 /* 000102fc - 000102ff [    4] */,
+    0x40c90008 /* 00010324 - 0001032c [    9] */,
+    0x40d2c004 /* 0001034b - 0001034f [    5] */,
+    0x40dec004 /* 0001037b - 0001037f [    5] */,
+    0x40e78000 /* 0001039e - 0001039e [    1] */,
+    0x40f10003 /* 000103c4 - 000103c7 [    4] */,
+    0x40f58029 /* 000103d6 - 000103ff [   42] */,
+    0x41278001 /* 0001049e - 0001049f [    2] */,
+    0x412a8005 /* 000104aa - 000104af [    6] */,
+    0x41350003 /* 000104d4 - 000104d7 [    4] */,
+    0x413f0003 /* 000104fc - 000104ff [    4] */,
+    0x414a0007 /* 00010528 - 0001052f [    8] */,
+    0x4159000a /* 00010564 - 0001056e [   11] */,
+    0x415ec000 /* 0001057b - 0001057b [    1] */,
+    0x4162c000 /* 0001058b - 0001058b [    1] */,
+    0x4164c000 /* 00010593 - 00010593 [    1] */,
+    0x41658000 /* 00010596 - 00010596 [    1] */,
+    0x41688000 /* 000105a2 - 000105a2 [    1] */,
+    0x416c8000 /* 000105b2 - 000105b2 [    1] */,
+    0x416e8000 /* 000105ba - 000105ba [    1] */,
+    0x416f4042 /* 000105bd - 000105ff [   67] */,
+    0x41cdc008 /* 00010737 - 0001073f [    9] */,
+    0x41d58009 /* 00010756 - 0001075f [   10] */,
+    0x41da0017 /* 00010768 - 0001077f [   24] */,
+    0x41e18000 /* 00010786 - 00010786 [    1] */,
+    0x41ec4000 /* 000107b1 - 000107b1 [    1] */,
+    0x41eec044 /* 000107bb - 000107ff [   69] */,
+    0x42018001 /* 00010806 - 00010807 [    2] */,
+    0x42024000 /* 00010809 - 00010809 [    1] */,
+    0x420d8000 /* 00010836 - 00010836 [    1] */,
+    0x420e4002 /* 00010839 - 0001083b [    3] */,
+    0x420f4001 /* 0001083d - 0001083e [    2] */,
+    0x42158000 /* 00010856 - 00010856 [    1] */,
+    0x4227c007 /* 0001089f - 000108a6 [    8] */,
+    0x422c002f /* 000108b0 - 000108df [   48] */,
+    0x423cc000 /* 000108f3 - 000108f3 [    1] */,
+    0x423d8004 /* 000108f6 - 000108fa [    5] */,
+    0x42470002 /* 0001091c - 0001091e [    3] */,
+    0x424e8004 /* 0001093a - 0001093e [    5] */,
+    0x4250003f /* 00010940 - 0001097f [   64] */,
+    0x426e0003 /* 000109b8 - 000109bb [    4] */,
+    0x42740001 /* 000109d0 - 000109d1 [    2] */,
+    0x42810000 /* 00010a04 - 00010a04 [    1] */,
+    0x4281c004 /* 00010a07 - 00010a0b [    5] */,
+    0x42850000 /* 00010a14 - 00010a14 [    1] */,
+    0x42860000 /* 00010a18 - 00010a18 [    1] */,
+    0x428d8001 /* 00010a36 - 00010a37 [    2] */,
+    0x428ec003 /* 00010a3b - 00010a3e [    4] */,
+    0x42924006 /* 00010a49 - 00010a4f [    7] */,
+    0x42964006 /* 00010a59 - 00010a5f [    7] */,
+    0x42a8001f /* 00010aa0 - 00010abf [   32] */,
+    0x42b9c003 /* 00010ae7 - 00010aea [    4] */,
+    0x42bdc008 /* 00010af7 - 00010aff [    9] */,
+    0x42cd8002 /* 00010b36 - 00010b38 [    3] */,
+    0x42d58001 /* 00010b56 - 00010b57 [    2] */,
+    0x42dcc004 /* 00010b73 - 00010b77 [    5] */,
+    0x42e48006 /* 00010b92 - 00010b98 [    7] */,
+    0x42e7400b /* 00010b9d - 00010ba8 [   12] */,
+    0x42ec004f /* 00010bb0 - 00010bff [   80] */,
+    0x43124036 /* 00010c49 - 00010c7f [   55] */,
+    0x432cc00c /* 00010cb3 - 00010cbf [   13] */,
+    0x433cc006 /* 00010cf3 - 00010cf9 [    7] */,
+    0x434a0007 /* 00010d28 - 00010d2f [    8] */,
+    0x434e8125 /* 00010d3a - 00010e5f [  294] */,
+    0x439fc000 /* 00010e7f - 00010e7f [    1] */,
+    0x43aa8000 /* 00010eaa - 00010eaa [    1] */,
+    0x43ab8001 /* 00010eae - 00010eaf [    2] */,
+    0x43ac804a /* 00010eb2 - 00010efc [   75] */,
+    0x43ca0007 /* 00010f28 - 00010f2f [    8] */,
+    0x43d68015 /* 00010f5a - 00010f6f [   22] */,
+    0x43e28025 /* 00010f8a - 00010faf [   38] */,
+    0x43f30013 /* 00010fcc - 00010fdf [   20] */,
+    0x43fdc008 /* 00010ff7 - 00010fff [    9] */,
+    0x44138003 /* 0001104e - 00011051 [    4] */,
+    0x441d8008 /* 00011076 - 0001107e [    9] */,
+    0x442f4000 /* 000110bd - 000110bd [    1] */,
+    0x4430c00c /* 000110c3 - 000110cf [   13] */,
+    0x443a4006 /* 000110e9 - 000110ef [    7] */,
+    0x443e8005 /* 000110fa - 000110ff [    6] */,
+    0x444d4000 /* 00011135 - 00011135 [    1] */,
+    0x44520007 /* 00011148 - 0001114f [    8] */,
+    0x445dc008 /* 00011177 - 0001117f [    9] */,
+    0x44780000 /* 000111e0 - 000111e0 [    1] */,
+    0x447d400a /* 000111f5 - 000111ff [   11] */,
+    0x44848000 /* 00011212 - 00011212 [    1] */,
+    0x4490803d /* 00011242 - 0001127f [   62] */,
+    0x44a1c000 /* 00011287 - 00011287 [    1] */,
+    0x44a24000 /* 00011289 - 00011289 [    1] */,
+    0x44a38000 /* 0001128e - 0001128e [    1] */,
+    0x44a78000 /* 0001129e - 0001129e [    1] */,
+    0x44aa8005 /* 000112aa - 000112af [    6] */,
+    0x44bac004 /* 000112eb - 000112ef [    5] */,
+    0x44be8005 /* 000112fa - 000112ff [    6] */,
+    0x44c10000 /* 00011304 - 00011304 [    1] */,
+    0x44c34001 /* 0001130d - 0001130e [    2] */,
+    0x44c44001 /* 00011311 - 00011312 [    2] */,
+    0x44ca4000 /* 00011329 - 00011329 [    1] */,
+    0x44cc4000 /* 00011331 - 00011331 [    1] */,
+    0x44cd0000 /* 00011334 - 00011334 [    1] */,
+    0x44ce8000 /* 0001133a - 0001133a [    1] */,
+    0x44d14001 /* 00011345 - 00011346 [    2] */,
+    0x44d24001 /* 00011349 - 0001134a [    2] */,
+    0x44d38001 /* 0001134e - 0001134f [    2] */,
+    0x44d44005 /* 00011351 - 00011356 [    6] */,
+    0x44d60004 /* 00011358 - 0001135c [    5] */,
+    0x44d90001 /* 00011364 - 00011365 [    2] */,
+    0x44db4002 /* 0001136d - 0001136f [    3] */,
+    0x44dd408a /* 00011375 - 000113ff [  139] */,
+    0x45170000 /* 0001145c - 0001145c [    1] */,
+    0x4518801d /* 00011462 - 0001147f [   30] */,
+    0x45320007 /* 000114c8 - 000114cf [    8] */,
+    0x453680a5 /* 000114da - 0001157f [  166] */,
+    0x456d8001 /* 000115b6 - 000115b7 [    2] */,
+    0x45778021 /* 000115de - 000115ff [   34] */,
+    0x4591400a /* 00011645 - 0001164f [   11] */,
+    0x45968005 /* 0001165a - 0001165f [    6] */,
+    0x459b4012 /* 0001166d - 0001167f [   19] */,
+    0x45ae8005 /* 000116ba - 000116bf [    6] */,
+    0x45b28035 /* 000116ca - 000116ff [   54] */,
+    0x45c6c001 /* 0001171b - 0001171c [    2] */,
+    0x45cb0003 /* 0001172c - 0001172f [    4] */,
+    0x45d1c0b8 /* 00011747 - 000117ff [  185] */,
+    0x460f0063 /* 0001183c - 0001189f [  100] */,
+    0x463cc00b /* 000118f3 - 000118fe [   12] */,
+    0x4641c001 /* 00011907 - 00011908 [    2] */,
+    0x46428001 /* 0001190a - 0001190b [    2] */,
+    0x46450000 /* 00011914 - 00011914 [    1] */,
+    0x4645c000 /* 00011917 - 00011917 [    1] */,
+    0x464d8000 /* 00011936 - 00011936 [    1] */,
+    0x464e4001 /* 00011939 - 0001193a [    2] */,
+    0x4651c008 /* 00011947 - 0001194f [    9] */,
+    0x46568045 /* 0001195a - 0001199f [   70] */,
+    0x466a0001 /* 000119a8 - 000119a9 [    2] */,
+    0x46760001 /* 000119d8 - 000119d9 [    2] */,
+    0x4679401a /* 000119e5 - 000119ff [   27] */,
+    0x46920007 /* 00011a48 - 00011a4f [    8] */,
+    0x46a8c00c /* 00011aa3 - 00011aaf [   13] */,
+    0x46be4006 /* 00011af9 - 00011aff [    7] */,
+    0x46c280f5 /* 00011b0a - 00011bff [  246] */,
+    0x47024000 /* 00011c09 - 00011c09 [    1] */,
+    0x470dc000 /* 00011c37 - 00011c37 [    1] */,
+    0x47118009 /* 00011c46 - 00011c4f [   10] */,
+    0x471b4002 /* 00011c6d - 00011c6f [    3] */,
+    0x47240001 /* 00011c90 - 00011c91 [    2] */,
+    0x472a0000 /* 00011ca8 - 00011ca8 [    1] */,
+    0x472dc048 /* 00011cb7 - 00011cff [   73] */,
+    0x4741c000 /* 00011d07 - 00011d07 [    1] */,
+    0x47428000 /* 00011d0a - 00011d0a [    1] */,
+    0x474dc002 /* 00011d37 - 00011d39 [    3] */,
+    0x474ec000 /* 00011d3b - 00011d3b [    1] */,
+    0x474f8000 /* 00011d3e - 00011d3e [    1] */,
+    0x47520007 /* 00011d48 - 00011d4f [    8] */,
+    0x47568005 /* 00011d5a - 00011d5f [    6] */,
+    0x47598000 /* 00011d66 - 00011d66 [    1] */,
+    0x475a4000 /* 00011d69 - 00011d69 [    1] */,
+    0x4763c000 /* 00011d8f - 00011d8f [    1] */,
+    0x47648000 /* 00011d92 - 00011d92 [    1] */,
+    0x47664006 /* 00011d99 - 00011d9f [    7] */,
+    0x476a8135 /* 00011daa - 00011edf [  310] */,
+    0x47be4006 /* 00011ef9 - 00011eff [    7] */,
+    0x47c44000 /* 00011f11 - 00011f11 [    1] */,
+    0x47cec002 /* 00011f3b - 00011f3d [    3] */,
+    0x47d68055 /* 00011f5a - 00011faf [   86] */,
+    0x47ec400e /* 00011fb1 - 00011fbf [   15] */,
+    0x47fc800c /* 00011ff2 - 00011ffe [   13] */,
+    0x48e68065 /* 0001239a - 000123ff [  102] */,
+    0x491bc000 /* 0001246f - 0001246f [    1] */,
+    0x491d400a /* 00012475 - 0001247f [   11] */,
+    0x49510a4b /* 00012544 - 00012f8f [ 2636] */,
+    0x4bfcc00c /* 00012ff3 - 00012fff [   13] */,
+    0x4d0c000f /* 00013430 - 0001343f [   16] */,
+    0x4d158fa9 /* 00013456 - 000143ff [ 4010] */,
+    0x5191e1b8 /* 00014647 - 000167ff [ 8633] */,
+    0x5a8e4006 /* 00016a39 - 00016a3f [    7] */,
+    0x5a97c000 /* 00016a5f - 00016a5f [    1] */,
+    0x5a9a8003 /* 00016a6a - 00016a6d [    4] */,
+    0x5aafc000 /* 00016abf - 00016abf [    1] */,
+    0x5ab28005 /* 00016aca - 00016acf [    6] */,
+    0x5abb8001 /* 00016aee - 00016aef [    2] */,
+    0x5abd8009 /* 00016af6 - 00016aff [   10] */,
+    0x5ad18009 /* 00016b46 - 00016b4f [   10] */,
+    0x5ad68000 /* 00016b5a - 00016b5a [    1] */,
+    0x5ad88000 /* 00016b62 - 00016b62 [    1] */,
+    0x5ade0004 /* 00016b78 - 00016b7c [    5] */,
+    0x5ae402af /* 00016b90 - 00016e3f [  688] */,
+    0x5ba6c064 /* 00016e9b - 00016eff [  101] */,
+    0x5bd2c003 /* 00016f4b - 00016f4e [    4] */,
+    0x5be20006 /* 00016f88 - 00016f8e [    7] */,
+    0x5be8003f /* 00016fa0 - 00016fdf [   64] */,
+    0x5bf9400a /* 00016fe5 - 00016fef [   11] */,
+    0x5bfc800d /* 00016ff2 - 00016fff [   14] */,
+    0x61fe0007 /* 000187f8 - 000187ff [    8] */,
+    0x63358029 /* 00018cd6 - 00018cff [   42] */,
+    0x634262e6 /* 00018d09 - 0001afef [ 8935] */,
+    0x6bfd0000 /* 0001aff4 - 0001aff4 [    1] */,
+    0x6bff0000 /* 0001affc - 0001affc [    1] */,
+    0x6bffc000 /* 0001afff - 0001afff [    1] */,
+    0x6c48c00e /* 0001b123 - 0001b131 [   15] */,
+    0x6c4cc01c /* 0001b133 - 0001b14f [   29] */,
+    0x6c54c001 /* 0001b153 - 0001b154 [    2] */,
+    0x6c55800d /* 0001b156 - 0001b163 [   14] */,
+    0x6c5a0007 /* 0001b168 - 0001b16f [    8] */,
+    0x6cbf0903 /* 0001b2fc - 0001bbff [ 2308] */,
+    0x6f1ac004 /* 0001bc6b - 0001bc6f [    5] */,
+    0x6f1f4002 /* 0001bc7d - 0001bc7f [    3] */,
+    0x6f224006 /* 0001bc89 - 0001bc8f [    7] */,
+    0x6f268001 /* 0001bc9a - 0001bc9b [    2] */,
+    0x6f28125f /* 0001bca0 - 0001ceff [ 4704] */,
+    0x73cb8001 /* 0001cf2e - 0001cf2f [    2] */,
+    0x73d1c008 /* 0001cf47 - 0001cf4f [    9] */,
+    0x73f1003b /* 0001cfc4 - 0001cfff [   60] */,
+    0x743d8009 /* 0001d0f6 - 0001d0ff [   10] */,
+    0x7449c001 /* 0001d127 - 0001d128 [    2] */,
+    0x745cc007 /* 0001d173 - 0001d17a [    8] */,
+    0x747ac014 /* 0001d1eb - 0001d1ff [   21] */,
+    0x74918079 /* 0001d246 - 0001d2bf [  122] */,
+    0x74b5000b /* 0001d2d4 - 0001d2df [   12] */,
+    0x74bd000b /* 0001d2f4 - 0001d2ff [   12] */,
+    0x74d5c008 /* 0001d357 - 0001d35f [    9] */,
+    0x74de4086 /* 0001d379 - 0001d3ff [  135] */,
+    0x75154000 /* 0001d455 - 0001d455 [    1] */,
+    0x75274000 /* 0001d49d - 0001d49d [    1] */,
+    0x75280001 /* 0001d4a0 - 0001d4a1 [    2] */,
+    0x7528c001 /* 0001d4a3 - 0001d4a4 [    2] */,
+    0x7529c001 /* 0001d4a7 - 0001d4a8 [    2] */,
+    0x752b4000 /* 0001d4ad - 0001d4ad [    1] */,
+    0x752e8000 /* 0001d4ba - 0001d4ba [    1] */,
+    0x752f0000 /* 0001d4bc - 0001d4bc [    1] */,
+    0x75310000 /* 0001d4c4 - 0001d4c4 [    1] */,
+    0x75418000 /* 0001d506 - 0001d506 [    1] */,
+    0x7542c001 /* 0001d50b - 0001d50c [    2] */,
+    0x75454000 /* 0001d515 - 0001d515 [    1] */,
+    0x75474000 /* 0001d51d - 0001d51d [    1] */,
+    0x754e8000 /* 0001d53a - 0001d53a [    1] */,
+    0x754fc000 /* 0001d53f - 0001d53f [    1] */,
+    0x75514000 /* 0001d545 - 0001d545 [    1] */,
+    0x7551c002 /* 0001d547 - 0001d549 [    3] */,
+    0x75544000 /* 0001d551 - 0001d551 [    1] */,
+    0x75a98001 /* 0001d6a6 - 0001d6a7 [    2] */,
+    0x75f30001 /* 0001d7cc - 0001d7cd [    2] */,
+    0x76a3000e /* 0001da8c - 0001da9a [   15] */,
+    0x76a80000 /* 0001daa0 - 0001daa0 [    1] */,
+    0x76ac044f /* 0001dab0 - 0001deff [ 1104] */,
+    0x77c7c005 /* 0001df1f - 0001df24 [    6] */,
+    0x77cac0d4 /* 0001df2b - 0001dfff [  213] */,
+    0x7801c000 /* 0001e007 - 0001e007 [    1] */,
+    0x78064001 /* 0001e019 - 0001e01a [    2] */,
+    0x78088000 /* 0001e022 - 0001e022 [    1] */,
+    0x78094000 /* 0001e025 - 0001e025 [    1] */,
+    0x780ac004 /* 0001e02b - 0001e02f [    5] */,
+    0x781b8020 /* 0001e06e - 0001e08e [   33] */,
+    0x7824006f /* 0001e090 - 0001e0ff [  112] */,
+    0x784b4002 /* 0001e12d - 0001e12f [    3] */,
+    0x784f8001 /* 0001e13e - 0001e13f [    2] */,
+    0x78528003 /* 0001e14a - 0001e14d [    4] */,
+    0x7854013f /* 0001e150 - 0001e28f [  320] */,
+    0x78abc010 /* 0001e2af - 0001e2bf [   17] */,
+    0x78be8004 /* 0001e2fa - 0001e2fe [    5] */,
+    0x78c001cf /* 0001e300 - 0001e4cf [  464] */,
+    0x793e82e5 /* 0001e4fa - 0001e7df [  742] */,
+    0x79f9c000 /* 0001e7e7 - 0001e7e7 [    1] */,
+    0x79fb0000 /* 0001e7ec - 0001e7ec [    1] */,
+    0x79fbc000 /* 0001e7ef - 0001e7ef [    1] */,
+    0x79ffc000 /* 0001e7ff - 0001e7ff [    1] */,
+    0x7a314001 /* 0001e8c5 - 0001e8c6 [    2] */,
+    0x7a35c028 /* 0001e8d7 - 0001e8ff [   41] */,
+    0x7a530003 /* 0001e94c - 0001e94f [    4] */,
+    0x7a568003 /* 0001e95a - 0001e95d [    4] */,
+    0x7a580310 /* 0001e960 - 0001ec70 [  785] */,
+    0x7b2d404b /* 0001ecb5 - 0001ed00 [   76] */,
+    0x7b4f80c1 /* 0001ed3e - 0001edff [  194] */,
+    0x7b810000 /* 0001ee04 - 0001ee04 [    1] */,
+    0x7b880000 /* 0001ee20 - 0001ee20 [    1] */,
+    0x7b88c000 /* 0001ee23 - 0001ee23 [    1] */,
+    0x7b894001 /* 0001ee25 - 0001ee26 [    2] */,
+    0x7b8a0000 /* 0001ee28 - 0001ee28 [    1] */,
+    0x7b8cc000 /* 0001ee33 - 0001ee33 [    1] */,
+    0x7b8e0000 /* 0001ee38 - 0001ee38 [    1] */,
+    0x7b8e8000 /* 0001ee3a - 0001ee3a [    1] */,
+    0x7b8f0005 /* 0001ee3c - 0001ee41 [    6] */,
+    0x7b90c003 /* 0001ee43 - 0001ee46 [    4] */,
+    0x7b920000 /* 0001ee48 - 0001ee48 [    1] */,
+    0x7b928000 /* 0001ee4a - 0001ee4a [    1] */,
+    0x7b930000 /* 0001ee4c - 0001ee4c [    1] */,
+    0x7b940000 /* 0001ee50 - 0001ee50 [    1] */,
+    0x7b94c000 /* 0001ee53 - 0001ee53 [    1] */,
+    0x7b954001 /* 0001ee55 - 0001ee56 [    2] */,
+    0x7b960000 /* 0001ee58 - 0001ee58 [    1] */,
+    0x7b968000 /* 0001ee5a - 0001ee5a [    1] */,
+    0x7b970000 /* 0001ee5c - 0001ee5c [    1] */,
+    0x7b978000 /* 0001ee5e - 0001ee5e [    1] */,
+    0x7b980000 /* 0001ee60 - 0001ee60 [    1] */,
+    0x7b98c000 /* 0001ee63 - 0001ee63 [    1] */,
+    0x7b994001 /* 0001ee65 - 0001ee66 [    2] */,
+    0x7b9ac000 /* 0001ee6b - 0001ee6b [    1] */,
+    0x7b9cc000 /* 0001ee73 - 0001ee73 [    1] */,
+    0x7b9e0000 /* 0001ee78 - 0001ee78 [    1] */,
+    0x7b9f4000 /* 0001ee7d - 0001ee7d [    1] */,
+    0x7b9fc000 /* 0001ee7f - 0001ee7f [    1] */,
+    0x7ba28000 /* 0001ee8a - 0001ee8a [    1] */,
+    0x7ba70004 /* 0001ee9c - 0001eea0 [    5] */,
+    0x7ba90000 /* 0001eea4 - 0001eea4 [    1] */,
+    0x7baa8000 /* 0001eeaa - 0001eeaa [    1] */,
+    0x7baf0033 /* 0001eebc - 0001eeef [   52] */,
+    0x7bbc810d /* 0001eef2 - 0001efff [  270] */,
+    0x7c0b0003 /* 0001f02c - 0001f02f [    4] */,
+    0x7c25000b /* 0001f094 - 0001f09f [   12] */,
+    0x7c2bc001 /* 0001f0af - 0001f0b0 [    2] */,
+    0x7c300000 /* 0001f0c0 - 0001f0c0 [    1] */,
+    0x7c340000 /* 0001f0d0 - 0001f0d0 [    1] */,
+    0x7c3d8009 /* 0001f0f6 - 0001f0ff [   10] */,
+    0x7c6b8037 /* 0001f1ae - 0001f1e5 [   56] */,
+    0x7c80c00c /* 0001f203 - 0001f20f [   13] */,
+    0x7c8f0003 /* 0001f23c - 0001f23f [    4] */,
+    0x7c924006 /* 0001f249 - 0001f24f [    7] */,
+    0x7c94800d /* 0001f252 - 0001f25f [   14] */,
+    0x7c998099 /* 0001f266 - 0001f2ff [  154] */,
+    0x7db60003 /* 0001f6d8 - 0001f6db [    4] */,
+    0x7dbb4002 /* 0001f6ed - 0001f6ef [    3] */,
+    0x7dbf4002 /* 0001f6fd - 0001f6ff [    3] */,
+    0x7dddc003 /* 0001f777 - 0001f77a [    4] */,
+    0x7df68005 /* 0001f7da - 0001f7df [    6] */,
+    0x7dfb0003 /* 0001f7ec - 0001f7ef [    4] */,
+    0x7dfc400e /* 0001f7f1 - 0001f7ff [   15] */,
+    0x7e030003 /* 0001f80c - 0001f80f [    4] */,
+    0x7e120007 /* 0001f848 - 0001f84f [    8] */,
+    0x7e168005 /* 0001f85a - 0001f85f [    6] */,
+    0x7e220007 /* 0001f888 - 0001f88f [    8] */,
+    0x7e2b8001 /* 0001f8ae - 0001f8af [    2] */,
+    0x7e2c804d /* 0001f8b2 - 0001f8ff [   78] */,
+    0x7e95000b /* 0001fa54 - 0001fa5f [   12] */,
+    0x7e9b8001 /* 0001fa6e - 0001fa6f [    2] */,
+    0x7e9f4002 /* 0001fa7d - 0001fa7f [    3] */,
+    0x7ea24006 /* 0001fa89 - 0001fa8f [    7] */,
+    0x7eaf8000 /* 0001fabe - 0001fabe [    1] */,
+    0x7eb18007 /* 0001fac6 - 0001facd [    8] */,
+    0x7eb70003 /* 0001fadc - 0001fadf [    4] */,
+    0x7eba4006 /* 0001fae9 - 0001faef [    7] */,
+    0x7ebe4006 /* 0001faf9 - 0001faff [    7] */,
+    0x7ee4c000 /* 0001fb93 - 0001fb93 [    1] */,
+    0x7ef2c024 /* 0001fbcb - 0001fbef [   37] */,
+    0x7efe8405 /* 0001fbfa - 0001ffff [ 1030] */,
+    0xa9b8001f /* 0002a6e0 - 0002a6ff [   32] */,
+    0xadce8005 /* 0002b73a - 0002b73f [    6] */,
+    0xae078001 /* 0002b81e - 0002b81f [    2] */,
+    0xb3a8800d /* 0002cea2 - 0002ceaf [   14] */,
+    0xbaf8400e /* 0002ebe1 - 0002ebef [   15] */,
+    0xbb9789a1 /* 0002ee5e - 0002f7ff [ 2466] */,
+    0xbe8785e1 /* 0002fa1e - 0002ffff [ 1506] */,
+    0xc4d2c004 /* 0003134b - 0003134f [    5] */};
+
+/// Returns whether the code unit needs to be escaped.
+///
+/// At the end of the valid Unicode code points space a lot of code points are
+/// either reserved or a noncharacter. Adding all these entries to the
+/// lookup table would greatly increase the size of the table. Instead these
+/// entries are manually processed. In this large area of reserved code points,
+/// there is a small area of extended graphemes that should not be escaped
+/// unconditionally. This is also manually coded. See the generation script for
+/// more details.
+
+///
+/// \\pre The code point is a valid Unicode code point.
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool __needs_escape(const char32_t __code_point) noexcept {
+
+  // The entries in the gap at the end.
+  if(__code_point >= 0x000e0100 && __code_point <= 0x000e01ef)
+     return false;
+
+  // The entries at the end.
+  if (__code_point >= 0x000323b0)
+    return true;
+
+  ptrdiff_t __i = std::ranges::upper_bound(__entries, (__code_point << 14) | 0x3fffu) - __entries;
+  if (__i == 0)
+    return false;
+
+  --__i;
+  uint32_t __upper_bound = (__entries[__i] >> 14) + (__entries[__i] & 0x3fffu);
+  return __code_point <= __upper_bound;
+}
+
+// clang-format on
+} // namespace __escaped_output_table
+
+#endif //_LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_ESCAPED_OUTPUT_TABLE_H
diff --git a/libcxx/include/__cxx03/__format/extended_grapheme_cluster_table.h b/libcxx/include/__cxx03/__format/extended_grapheme_cluster_table.h
new file mode 100644
index 000000000000..48581d8a5dde
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/extended_grapheme_cluster_table.h
@@ -0,0 +1,1663 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// WARNING, this entire header is generated by
+// utils/generate_extended_grapheme_cluster_table.py
+// DO NOT MODIFY!
+
+// UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
+//
+// See Terms of Use <https://www.unicode.org/copyright.html>
+// for definitions of Unicode Inc.'s Data Files and Software.
+//
+// NOTICE TO USER: Carefully read the following legal agreement.
+// BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S
+// DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"),
+// YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
+// TERMS AND CONDITIONS OF THIS AGREEMENT.
+// IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE
+// THE DATA FILES OR SOFTWARE.
+//
+// COPYRIGHT AND PERMISSION NOTICE
+//
+// Copyright (c) 1991-2022 Unicode, Inc. All rights reserved.
+// Distributed under the Terms of Use in https://www.unicode.org/copyright.html.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of the Unicode data files and any associated documentation
+// (the "Data Files") or Unicode software and any associated documentation
+// (the "Software") to deal in the Data Files or Software
+// without restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, and/or sell copies of
+// the Data Files or Software, and to permit persons to whom the Data Files
+// or Software are furnished to do so, provided that either
+// (a) this copyright and permission notice appear with all copies
+// of the Data Files or Software, or
+// (b) this copyright and permission notice appear in associated
+// Documentation.
+//
+// THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
+// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT OF THIRD PARTY RIGHTS.
+// IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
+// NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
+// DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+// DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+//
+// Except as contained in this notice, the name of a copyright holder
+// shall not be used in advertising or otherwise to promote the sale,
+// use or other dealings in these Data Files or Software without prior
+// written authorization of the copyright holder.
+
+#ifndef _LIBCPP___FORMAT_EXTENDED_GRAPHEME_CLUSTER_TABLE_H
+#define _LIBCPP___FORMAT_EXTENDED_GRAPHEME_CLUSTER_TABLE_H
+
+#include <__algorithm/ranges_upper_bound.h>
+#include <__config>
+#include <__iterator/access.h>
+#include <cstddef>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __extended_grapheme_custer_property_boundary {
+
+enum class __property : uint8_t {
+  // Values generated from the data files.
+  __CR,
+  __Control,
+  __Extend,
+  __Extended_Pictographic,
+  __L,
+  __LF,
+  __LV,
+  __LVT,
+  __Prepend,
+  __Regional_Indicator,
+  __SpacingMark,
+  __T,
+  __V,
+  __ZWJ,
+
+  // The properies below aren't stored in the "database".
+
+  // Text position properties.
+  __sot,
+  __eot,
+
+  // The code unit has none of above properties.
+  __none
+};
+
+/// The entries of the extended grapheme cluster bondary property table.
+///
+/// The data is generated from
+/// - https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakProperty.txt
+/// - https://www.unicode.org/Public/UCD/latest/ucd/emoji/emoji-data.txt
+///
+/// The data has 3 values
+/// - bits [0, 3] The property. One of the values generated from the datafiles
+///   of \ref __property
+/// - bits [4, 10] The size of the range.
+/// - bits [11, 31] The lower bound code point of the range. The upper bound of
+///   the range is lower bound + size.
+///
+/// The 7 bits for the size allow a maximum range of 128 elements. Some ranges
+/// in the Unicode tables are larger. They are stored in multiple consecutive
+/// ranges in the data table. An alternative would be to store the sizes in a
+/// separate 16-bit value. The original MSVC STL code had such an approach, but
+/// this approach uses less space for the data and is about 4% faster in the
+/// following benchmark.
+/// libcxx/benchmarks/std_format_spec_string_unicode.bench.cpp
+// clang-format off
+_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
+    0x00000091,
+    0x00005005,
+    0x00005811,
+    0x00006800,
+    0x00007111,
+    0x0003fa01,
+    0x00054803,
+    0x00056801,
+    0x00057003,
+    0x001806f2,
+    0x00241862,
+    0x002c8ac2,
+    0x002df802,
+    0x002e0812,
+    0x002e2012,
+    0x002e3802,
+    0x00300058,
+    0x003080a2,
+    0x0030e001,
+    0x00325942,
+    0x00338002,
+    0x0036b062,
+    0x0036e808,
+    0x0036f852,
+    0x00373812,
+    0x00375032,
+    0x00387808,
+    0x00388802,
+    0x003981a2,
+    0x003d30a2,
+    0x003f5882,
+    0x003fe802,
+    0x0040b032,
+    0x0040d882,
+    0x00412822,
+    0x00414842,
+    0x0042c822,
+    0x00448018,
+    0x0044c072,
+    0x00465172,
+    0x00471008,
+    0x004719f2,
+    0x0048180a,
+    0x0049d002,
+    0x0049d80a,
+    0x0049e002,
+    0x0049f02a,
+    0x004a0872,
+    0x004a483a,
+    0x004a6802,
+    0x004a701a,
+    0x004a8862,
+    0x004b1012,
+    0x004c0802,
+    0x004c101a,
+    0x004de002,
+    0x004df002,
+    0x004df81a,
+    0x004e0832,
+    0x004e381a,
+    0x004e581a,
+    0x004e6802,
+    0x004eb802,
+    0x004f1012,
+    0x004ff002,
+    0x00500812,
+    0x0050180a,
+    0x0051e002,
+    0x0051f02a,
+    0x00520812,
+    0x00523812,
+    0x00525822,
+    0x00528802,
+    0x00538012,
+    0x0053a802,
+    0x00540812,
+    0x0054180a,
+    0x0055e002,
+    0x0055f02a,
+    0x00560842,
+    0x00563812,
+    0x0056480a,
+    0x0056581a,
+    0x00566802,
+    0x00571012,
+    0x0057d052,
+    0x00580802,
+    0x0058101a,
+    0x0059e002,
+    0x0059f012,
+    0x005a000a,
+    0x005a0832,
+    0x005a381a,
+    0x005a581a,
+    0x005a6802,
+    0x005aa822,
+    0x005b1012,
+    0x005c1002,
+    0x005df002,
+    0x005df80a,
+    0x005e0002,
+    0x005e081a,
+    0x005e302a,
+    0x005e502a,
+    0x005e6802,
+    0x005eb802,
+    0x00600002,
+    0x0060082a,
+    0x00602002,
+    0x0061e002,
+    0x0061f022,
+    0x0062083a,
+    0x00623022,
+    0x00625032,
+    0x0062a812,
+    0x00631012,
+    0x00640802,
+    0x0064101a,
+    0x0065e002,
+    0x0065f00a,
+    0x0065f802,
+    0x0066001a,
+    0x00661002,
+    0x0066181a,
+    0x00663002,
+    0x0066381a,
+    0x0066501a,
+    0x00666012,
+    0x0066a812,
+    0x00671012,
+    0x0067980a,
+    0x00680012,
+    0x0068101a,
+    0x0069d812,
+    0x0069f002,
+    0x0069f81a,
+    0x006a0832,
+    0x006a302a,
+    0x006a502a,
+    0x006a6802,
+    0x006a7008,
+    0x006ab802,
+    0x006b1012,
+    0x006c0802,
+    0x006c101a,
+    0x006e5002,
+    0x006e7802,
+    0x006e801a,
+    0x006e9022,
+    0x006eb002,
+    0x006ec06a,
+    0x006ef802,
+    0x006f901a,
+    0x00718802,
+    0x0071980a,
+    0x0071a062,
+    0x00723872,
+    0x00758802,
+    0x0075980a,
+    0x0075a082,
+    0x00764062,
+    0x0078c012,
+    0x0079a802,
+    0x0079b802,
+    0x0079c802,
+    0x0079f01a,
+    0x007b88d2,
+    0x007bf80a,
+    0x007c0042,
+    0x007c3012,
+    0x007c68a2,
+    0x007cca32,
+    0x007e3002,
+    0x00816832,
+    0x0081880a,
+    0x00819052,
+    0x0081c812,
+    0x0081d81a,
+    0x0081e812,
+    0x0082b01a,
+    0x0082c012,
+    0x0082f022,
+    0x00838832,
+    0x00841002,
+    0x0084200a,
+    0x00842812,
+    0x00846802,
+    0x0084e802,
+    0x008805f4,
+    0x008b047c,
+    0x008d457b,
+    0x009ae822,
+    0x00b89022,
+    0x00b8a80a,
+    0x00b99012,
+    0x00b9a00a,
+    0x00ba9012,
+    0x00bb9012,
+    0x00bda012,
+    0x00bdb00a,
+    0x00bdb862,
+    0x00bdf07a,
+    0x00be3002,
+    0x00be381a,
+    0x00be48a2,
+    0x00bee802,
+    0x00c05822,
+    0x00c07001,
+    0x00c07802,
+    0x00c42812,
+    0x00c54802,
+    0x00c90022,
+    0x00c9183a,
+    0x00c93812,
+    0x00c9482a,
+    0x00c9801a,
+    0x00c99002,
+    0x00c9985a,
+    0x00c9c822,
+    0x00d0b812,
+    0x00d0c81a,
+    0x00d0d802,
+    0x00d2a80a,
+    0x00d2b002,
+    0x00d2b80a,
+    0x00d2c062,
+    0x00d30002,
+    0x00d31002,
+    0x00d32872,
+    0x00d3685a,
+    0x00d39892,
+    0x00d3f802,
+    0x00d581e2,
+    0x00d80032,
+    0x00d8200a,
+    0x00d9a062,
+    0x00d9d80a,
+    0x00d9e002,
+    0x00d9e84a,
+    0x00da1002,
+    0x00da181a,
+    0x00db5882,
+    0x00dc0012,
+    0x00dc100a,
+    0x00dd080a,
+    0x00dd1032,
+    0x00dd301a,
+    0x00dd4012,
+    0x00dd500a,
+    0x00dd5822,
+    0x00df3002,
+    0x00df380a,
+    0x00df4012,
+    0x00df502a,
+    0x00df6802,
+    0x00df700a,
+    0x00df7822,
+    0x00df901a,
+    0x00e1207a,
+    0x00e16072,
+    0x00e1a01a,
+    0x00e1b012,
+    0x00e68022,
+    0x00e6a0c2,
+    0x00e7080a,
+    0x00e71062,
+    0x00e76802,
+    0x00e7a002,
+    0x00e7b80a,
+    0x00e7c012,
+    0x00ee03f2,
+    0x01005801,
+    0x01006002,
+    0x0100680d,
+    0x01007011,
+    0x01014061,
+    0x0101e003,
+    0x01024803,
+    0x010300f1,
+    0x01068202,
+    0x01091003,
+    0x0109c803,
+    0x010ca053,
+    0x010d4813,
+    0x0118d013,
+    0x01194003,
+    0x011c4003,
+    0x011e7803,
+    0x011f48a3,
+    0x011fc023,
+    0x01261003,
+    0x012d5013,
+    0x012db003,
+    0x012e0003,
+    0x012fd833,
+    0x01300053,
+    0x013038b3,
+    0x0130a713,
+    0x01348753,
+    0x013840a3,
+    0x0138a003,
+    0x0138b003,
+    0x0138e803,
+    0x01390803,
+    0x01394003,
+    0x01399813,
+    0x013a2003,
+    0x013a3803,
+    0x013a6003,
+    0x013a7003,
+    0x013a9823,
+    0x013ab803,
+    0x013b1843,
+    0x013ca823,
+    0x013d0803,
+    0x013d8003,
+    0x013df803,
+    0x0149a013,
+    0x01582823,
+    0x0158d813,
+    0x015a8003,
+    0x015aa803,
+    0x01677822,
+    0x016bf802,
+    0x016f01f2,
+    0x01815052,
+    0x01818003,
+    0x0181e803,
+    0x0184c812,
+    0x0194b803,
+    0x0194c803,
+    0x05337832,
+    0x0533a092,
+    0x0534f012,
+    0x05378012,
+    0x05401002,
+    0x05403002,
+    0x05405802,
+    0x0541181a,
+    0x05412812,
+    0x0541380a,
+    0x05416002,
+    0x0544001a,
+    0x0545a0fa,
+    0x05462012,
+    0x05470112,
+    0x0547f802,
+    0x05493072,
+    0x054a38a2,
+    0x054a901a,
+    0x054b01c4,
+    0x054c0022,
+    0x054c180a,
+    0x054d9802,
+    0x054da01a,
+    0x054db032,
+    0x054dd01a,
+    0x054de012,
+    0x054df02a,
+    0x054f2802,
+    0x05514852,
+    0x0551781a,
+    0x05518812,
+    0x0551981a,
+    0x0551a812,
+    0x05521802,
+    0x05526002,
+    0x0552680a,
+    0x0553e002,
+    0x05558002,
+    0x05559022,
+    0x0555b812,
+    0x0555f012,
+    0x05560802,
+    0x0557580a,
+    0x05576012,
+    0x0557701a,
+    0x0557a80a,
+    0x0557b002,
+    0x055f181a,
+    0x055f2802,
+    0x055f301a,
+    0x055f4002,
+    0x055f481a,
+    0x055f600a,
+    0x055f6802,
+    0x05600006,
+    0x056009a7,
+    0x0560e006,
+    0x0560e9a7,
+    0x0561c006,
+    0x0561c9a7,
+    0x0562a006,
+    0x0562a9a7,
+    0x05638006,
+    0x056389a7,
+    0x05646006,
+    0x056469a7,
+    0x05654006,
+    0x056549a7,
+    0x05662006,
+    0x056629a7,
+    0x05670006,
+    0x056709a7,
+    0x0567e006,
+    0x0567e9a7,
+    0x0568c006,
+    0x0568c9a7,
+    0x0569a006,
+    0x0569a9a7,
+    0x056a8006,
+    0x056a89a7,
+    0x056b6006,
+    0x056b69a7,
+    0x056c4006,
+    0x056c49a7,
+    0x056d2006,
+    0x056d29a7,
+    0x056e0006,
+    0x056e09a7,
+    0x056ee006,
+    0x056ee9a7,
+    0x056fc006,
+    0x056fc9a7,
+    0x0570a006,
+    0x0570a9a7,
+    0x05718006,
+    0x057189a7,
+    0x05726006,
+    0x057269a7,
+    0x05734006,
+    0x057349a7,
+    0x05742006,
+    0x057429a7,
+    0x05750006,
+    0x057509a7,
+    0x0575e006,
+    0x0575e9a7,
+    0x0576c006,
+    0x0576c9a7,
+    0x0577a006,
+    0x0577a9a7,
+    0x05788006,
+    0x057889a7,
+    0x05796006,
+    0x057969a7,
+    0x057a4006,
+    0x057a49a7,
+    0x057b2006,
+    0x057b29a7,
+    0x057c0006,
+    0x057c09a7,
+    0x057ce006,
+    0x057ce9a7,
+    0x057dc006,
+    0x057dc9a7,
+    0x057ea006,
+    0x057ea9a7,
+    0x057f8006,
+    0x057f89a7,
+    0x05806006,
+    0x058069a7,
+    0x05814006,
+    0x058149a7,
+    0x05822006,
+    0x058229a7,
+    0x05830006,
+    0x058309a7,
+    0x0583e006,
+    0x0583e9a7,
+    0x0584c006,
+    0x0584c9a7,
+    0x0585a006,
+    0x0585a9a7,
+    0x05868006,
+    0x058689a7,
+    0x05876006,
+    0x058769a7,
+    0x05884006,
+    0x058849a7,
+    0x05892006,
+    0x058929a7,
+    0x058a0006,
+    0x058a09a7,
+    0x058ae006,
+    0x058ae9a7,
+    0x058bc006,
+    0x058bc9a7,
+    0x058ca006,
+    0x058ca9a7,
+    0x058d8006,
+    0x058d89a7,
+    0x058e6006,
+    0x058e69a7,
+    0x058f4006,
+    0x058f49a7,
+    0x05902006,
+    0x059029a7,
+    0x05910006,
+    0x059109a7,
+    0x0591e006,
+    0x0591e9a7,
+    0x0592c006,
+    0x0592c9a7,
+    0x0593a006,
+    0x0593a9a7,
+    0x05948006,
+    0x059489a7,
+    0x05956006,
+    0x059569a7,
+    0x05964006,
+    0x059649a7,
+    0x05972006,
+    0x059729a7,
+    0x05980006,
+    0x059809a7,
+    0x0598e006,
+    0x0598e9a7,
+    0x0599c006,
+    0x0599c9a7,
+    0x059aa006,
+    0x059aa9a7,
+    0x059b8006,
+    0x059b89a7,
+    0x059c6006,
+    0x059c69a7,
+    0x059d4006,
+    0x059d49a7,
+    0x059e2006,
+    0x059e29a7,
+    0x059f0006,
+    0x059f09a7,
+    0x059fe006,
+    0x059fe9a7,
+    0x05a0c006,
+    0x05a0c9a7,
+    0x05a1a006,
+    0x05a1a9a7,
+    0x05a28006,
+    0x05a289a7,
+    0x05a36006,
+    0x05a369a7,
+    0x05a44006,
+    0x05a449a7,
+    0x05a52006,
+    0x05a529a7,
+    0x05a60006,
+    0x05a609a7,
+    0x05a6e006,
+    0x05a6e9a7,
+    0x05a7c006,
+    0x05a7c9a7,
+    0x05a8a006,
+    0x05a8a9a7,
+    0x05a98006,
+    0x05a989a7,
+    0x05aa6006,
+    0x05aa69a7,
+    0x05ab4006,
+    0x05ab49a7,
+    0x05ac2006,
+    0x05ac29a7,
+    0x05ad0006,
+    0x05ad09a7,
+    0x05ade006,
+    0x05ade9a7,
+    0x05aec006,
+    0x05aec9a7,
+    0x05afa006,
+    0x05afa9a7,
+    0x05b08006,
+    0x05b089a7,
+    0x05b16006,
+    0x05b169a7,
+    0x05b24006,
+    0x05b249a7,
+    0x05b32006,
+    0x05b329a7,
+    0x05b40006,
+    0x05b409a7,
+    0x05b4e006,
+    0x05b4e9a7,
+    0x05b5c006,
+    0x05b5c9a7,
+    0x05b6a006,
+    0x05b6a9a7,
+    0x05b78006,
+    0x05b789a7,
+    0x05b86006,
+    0x05b869a7,
+    0x05b94006,
+    0x05b949a7,
+    0x05ba2006,
+    0x05ba29a7,
+    0x05bb0006,
+    0x05bb09a7,
+    0x05bbe006,
+    0x05bbe9a7,
+    0x05bcc006,
+    0x05bcc9a7,
+    0x05bda006,
+    0x05bda9a7,
+    0x05be8006,
+    0x05be89a7,
+    0x05bf6006,
+    0x05bf69a7,
+    0x05c04006,
+    0x05c049a7,
+    0x05c12006,
+    0x05c129a7,
+    0x05c20006,
+    0x05c209a7,
+    0x05c2e006,
+    0x05c2e9a7,
+    0x05c3c006,
+    0x05c3c9a7,
+    0x05c4a006,
+    0x05c4a9a7,
+    0x05c58006,
+    0x05c589a7,
+    0x05c66006,
+    0x05c669a7,
+    0x05c74006,
+    0x05c749a7,
+    0x05c82006,
+    0x05c829a7,
+    0x05c90006,
+    0x05c909a7,
+    0x05c9e006,
+    0x05c9e9a7,
+    0x05cac006,
+    0x05cac9a7,
+    0x05cba006,
+    0x05cba9a7,
+    0x05cc8006,
+    0x05cc89a7,
+    0x05cd6006,
+    0x05cd69a7,
+    0x05ce4006,
+    0x05ce49a7,
+    0x05cf2006,
+    0x05cf29a7,
+    0x05d00006,
+    0x05d009a7,
+    0x05d0e006,
+    0x05d0e9a7,
+    0x05d1c006,
+    0x05d1c9a7,
+    0x05d2a006,
+    0x05d2a9a7,
+    0x05d38006,
+    0x05d389a7,
+    0x05d46006,
+    0x05d469a7,
+    0x05d54006,
+    0x05d549a7,
+    0x05d62006,
+    0x05d629a7,
+    0x05d70006,
+    0x05d709a7,
+    0x05d7e006,
+    0x05d7e9a7,
+    0x05d8c006,
+    0x05d8c9a7,
+    0x05d9a006,
+    0x05d9a9a7,
+    0x05da8006,
+    0x05da89a7,
+    0x05db6006,
+    0x05db69a7,
+    0x05dc4006,
+    0x05dc49a7,
+    0x05dd2006,
+    0x05dd29a7,
+    0x05de0006,
+    0x05de09a7,
+    0x05dee006,
+    0x05dee9a7,
+    0x05dfc006,
+    0x05dfc9a7,
+    0x05e0a006,
+    0x05e0a9a7,
+    0x05e18006,
+    0x05e189a7,
+    0x05e26006,
+    0x05e269a7,
+    0x05e34006,
+    0x05e349a7,
+    0x05e42006,
+    0x05e429a7,
+    0x05e50006,
+    0x05e509a7,
+    0x05e5e006,
+    0x05e5e9a7,
+    0x05e6c006,
+    0x05e6c9a7,
+    0x05e7a006,
+    0x05e7a9a7,
+    0x05e88006,
+    0x05e889a7,
+    0x05e96006,
+    0x05e969a7,
+    0x05ea4006,
+    0x05ea49a7,
+    0x05eb2006,
+    0x05eb29a7,
+    0x05ec0006,
+    0x05ec09a7,
+    0x05ece006,
+    0x05ece9a7,
+    0x05edc006,
+    0x05edc9a7,
+    0x05eea006,
+    0x05eea9a7,
+    0x05ef8006,
+    0x05ef89a7,
+    0x05f06006,
+    0x05f069a7,
+    0x05f14006,
+    0x05f149a7,
+    0x05f22006,
+    0x05f229a7,
+    0x05f30006,
+    0x05f309a7,
+    0x05f3e006,
+    0x05f3e9a7,
+    0x05f4c006,
+    0x05f4c9a7,
+    0x05f5a006,
+    0x05f5a9a7,
+    0x05f68006,
+    0x05f689a7,
+    0x05f76006,
+    0x05f769a7,
+    0x05f84006,
+    0x05f849a7,
+    0x05f92006,
+    0x05f929a7,
+    0x05fa0006,
+    0x05fa09a7,
+    0x05fae006,
+    0x05fae9a7,
+    0x05fbc006,
+    0x05fbc9a7,
+    0x05fca006,
+    0x05fca9a7,
+    0x05fd8006,
+    0x05fd89a7,
+    0x05fe6006,
+    0x05fe69a7,
+    0x05ff4006,
+    0x05ff49a7,
+    0x06002006,
+    0x060029a7,
+    0x06010006,
+    0x060109a7,
+    0x0601e006,
+    0x0601e9a7,
+    0x0602c006,
+    0x0602c9a7,
+    0x0603a006,
+    0x0603a9a7,
+    0x06048006,
+    0x060489a7,
+    0x06056006,
+    0x060569a7,
+    0x06064006,
+    0x060649a7,
+    0x06072006,
+    0x060729a7,
+    0x06080006,
+    0x060809a7,
+    0x0608e006,
+    0x0608e9a7,
+    0x0609c006,
+    0x0609c9a7,
+    0x060aa006,
+    0x060aa9a7,
+    0x060b8006,
+    0x060b89a7,
+    0x060c6006,
+    0x060c69a7,
+    0x060d4006,
+    0x060d49a7,
+    0x060e2006,
+    0x060e29a7,
+    0x060f0006,
+    0x060f09a7,
+    0x060fe006,
+    0x060fe9a7,
+    0x0610c006,
+    0x0610c9a7,
+    0x0611a006,
+    0x0611a9a7,
+    0x06128006,
+    0x061289a7,
+    0x06136006,
+    0x061369a7,
+    0x06144006,
+    0x061449a7,
+    0x06152006,
+    0x061529a7,
+    0x06160006,
+    0x061609a7,
+    0x0616e006,
+    0x0616e9a7,
+    0x0617c006,
+    0x0617c9a7,
+    0x0618a006,
+    0x0618a9a7,
+    0x06198006,
+    0x061989a7,
+    0x061a6006,
+    0x061a69a7,
+    0x061b4006,
+    0x061b49a7,
+    0x061c2006,
+    0x061c29a7,
+    0x061d0006,
+    0x061d09a7,
+    0x061de006,
+    0x061de9a7,
+    0x061ec006,
+    0x061ec9a7,
+    0x061fa006,
+    0x061fa9a7,
+    0x06208006,
+    0x062089a7,
+    0x06216006,
+    0x062169a7,
+    0x06224006,
+    0x062249a7,
+    0x06232006,
+    0x062329a7,
+    0x06240006,
+    0x062409a7,
+    0x0624e006,
+    0x0624e9a7,
+    0x0625c006,
+    0x0625c9a7,
+    0x0626a006,
+    0x0626a9a7,
+    0x06278006,
+    0x062789a7,
+    0x06286006,
+    0x062869a7,
+    0x06294006,
+    0x062949a7,
+    0x062a2006,
+    0x062a29a7,
+    0x062b0006,
+    0x062b09a7,
+    0x062be006,
+    0x062be9a7,
+    0x062cc006,
+    0x062cc9a7,
+    0x062da006,
+    0x062da9a7,
+    0x062e8006,
+    0x062e89a7,
+    0x062f6006,
+    0x062f69a7,
+    0x06304006,
+    0x063049a7,
+    0x06312006,
+    0x063129a7,
+    0x06320006,
+    0x063209a7,
+    0x0632e006,
+    0x0632e9a7,
+    0x0633c006,
+    0x0633c9a7,
+    0x0634a006,
+    0x0634a9a7,
+    0x06358006,
+    0x063589a7,
+    0x06366006,
+    0x063669a7,
+    0x06374006,
+    0x063749a7,
+    0x06382006,
+    0x063829a7,
+    0x06390006,
+    0x063909a7,
+    0x0639e006,
+    0x0639e9a7,
+    0x063ac006,
+    0x063ac9a7,
+    0x063ba006,
+    0x063ba9a7,
+    0x063c8006,
+    0x063c89a7,
+    0x063d6006,
+    0x063d69a7,
+    0x063e4006,
+    0x063e49a7,
+    0x063f2006,
+    0x063f29a7,
+    0x06400006,
+    0x064009a7,
+    0x0640e006,
+    0x0640e9a7,
+    0x0641c006,
+    0x0641c9a7,
+    0x0642a006,
+    0x0642a9a7,
+    0x06438006,
+    0x064389a7,
+    0x06446006,
+    0x064469a7,
+    0x06454006,
+    0x064549a7,
+    0x06462006,
+    0x064629a7,
+    0x06470006,
+    0x064709a7,
+    0x0647e006,
+    0x0647e9a7,
+    0x0648c006,
+    0x0648c9a7,
+    0x0649a006,
+    0x0649a9a7,
+    0x064a8006,
+    0x064a89a7,
+    0x064b6006,
+    0x064b69a7,
+    0x064c4006,
+    0x064c49a7,
+    0x064d2006,
+    0x064d29a7,
+    0x064e0006,
+    0x064e09a7,
+    0x064ee006,
+    0x064ee9a7,
+    0x064fc006,
+    0x064fc9a7,
+    0x0650a006,
+    0x0650a9a7,
+    0x06518006,
+    0x065189a7,
+    0x06526006,
+    0x065269a7,
+    0x06534006,
+    0x065349a7,
+    0x06542006,
+    0x065429a7,
+    0x06550006,
+    0x065509a7,
+    0x0655e006,
+    0x0655e9a7,
+    0x0656c006,
+    0x0656c9a7,
+    0x0657a006,
+    0x0657a9a7,
+    0x06588006,
+    0x065889a7,
+    0x06596006,
+    0x065969a7,
+    0x065a4006,
+    0x065a49a7,
+    0x065b2006,
+    0x065b29a7,
+    0x065c0006,
+    0x065c09a7,
+    0x065ce006,
+    0x065ce9a7,
+    0x065dc006,
+    0x065dc9a7,
+    0x065ea006,
+    0x065ea9a7,
+    0x065f8006,
+    0x065f89a7,
+    0x06606006,
+    0x066069a7,
+    0x06614006,
+    0x066149a7,
+    0x06622006,
+    0x066229a7,
+    0x06630006,
+    0x066309a7,
+    0x0663e006,
+    0x0663e9a7,
+    0x0664c006,
+    0x0664c9a7,
+    0x0665a006,
+    0x0665a9a7,
+    0x06668006,
+    0x066689a7,
+    0x06676006,
+    0x066769a7,
+    0x06684006,
+    0x066849a7,
+    0x06692006,
+    0x066929a7,
+    0x066a0006,
+    0x066a09a7,
+    0x066ae006,
+    0x066ae9a7,
+    0x066bc006,
+    0x066bc9a7,
+    0x066ca006,
+    0x066ca9a7,
+    0x066d8006,
+    0x066d89a7,
+    0x066e6006,
+    0x066e69a7,
+    0x066f4006,
+    0x066f49a7,
+    0x06702006,
+    0x067029a7,
+    0x06710006,
+    0x067109a7,
+    0x0671e006,
+    0x0671e9a7,
+    0x0672c006,
+    0x0672c9a7,
+    0x0673a006,
+    0x0673a9a7,
+    0x06748006,
+    0x067489a7,
+    0x06756006,
+    0x067569a7,
+    0x06764006,
+    0x067649a7,
+    0x06772006,
+    0x067729a7,
+    0x06780006,
+    0x067809a7,
+    0x0678e006,
+    0x0678e9a7,
+    0x0679c006,
+    0x0679c9a7,
+    0x067aa006,
+    0x067aa9a7,
+    0x067b8006,
+    0x067b89a7,
+    0x067c6006,
+    0x067c69a7,
+    0x067d4006,
+    0x067d49a7,
+    0x067e2006,
+    0x067e29a7,
+    0x067f0006,
+    0x067f09a7,
+    0x067fe006,
+    0x067fe9a7,
+    0x0680c006,
+    0x0680c9a7,
+    0x0681a006,
+    0x0681a9a7,
+    0x06828006,
+    0x068289a7,
+    0x06836006,
+    0x068369a7,
+    0x06844006,
+    0x068449a7,
+    0x06852006,
+    0x068529a7,
+    0x06860006,
+    0x068609a7,
+    0x0686e006,
+    0x0686e9a7,
+    0x0687c006,
+    0x0687c9a7,
+    0x0688a006,
+    0x0688a9a7,
+    0x06898006,
+    0x068989a7,
+    0x068a6006,
+    0x068a69a7,
+    0x068b4006,
+    0x068b49a7,
+    0x068c2006,
+    0x068c29a7,
+    0x068d0006,
+    0x068d09a7,
+    0x068de006,
+    0x068de9a7,
+    0x068ec006,
+    0x068ec9a7,
+    0x068fa006,
+    0x068fa9a7,
+    0x06908006,
+    0x069089a7,
+    0x06916006,
+    0x069169a7,
+    0x06924006,
+    0x069249a7,
+    0x06932006,
+    0x069329a7,
+    0x06940006,
+    0x069409a7,
+    0x0694e006,
+    0x0694e9a7,
+    0x0695c006,
+    0x0695c9a7,
+    0x0696a006,
+    0x0696a9a7,
+    0x06978006,
+    0x069789a7,
+    0x06986006,
+    0x069869a7,
+    0x06994006,
+    0x069949a7,
+    0x069a2006,
+    0x069a29a7,
+    0x069b0006,
+    0x069b09a7,
+    0x069be006,
+    0x069be9a7,
+    0x069cc006,
+    0x069cc9a7,
+    0x069da006,
+    0x069da9a7,
+    0x069e8006,
+    0x069e89a7,
+    0x069f6006,
+    0x069f69a7,
+    0x06a04006,
+    0x06a049a7,
+    0x06a12006,
+    0x06a129a7,
+    0x06a20006,
+    0x06a209a7,
+    0x06a2e006,
+    0x06a2e9a7,
+    0x06a3c006,
+    0x06a3c9a7,
+    0x06a4a006,
+    0x06a4a9a7,
+    0x06a58006,
+    0x06a589a7,
+    0x06a66006,
+    0x06a669a7,
+    0x06a74006,
+    0x06a749a7,
+    0x06a82006,
+    0x06a829a7,
+    0x06a90006,
+    0x06a909a7,
+    0x06a9e006,
+    0x06a9e9a7,
+    0x06aac006,
+    0x06aac9a7,
+    0x06aba006,
+    0x06aba9a7,
+    0x06ac8006,
+    0x06ac89a7,
+    0x06ad6006,
+    0x06ad69a7,
+    0x06ae4006,
+    0x06ae49a7,
+    0x06af2006,
+    0x06af29a7,
+    0x06b00006,
+    0x06b009a7,
+    0x06b0e006,
+    0x06b0e9a7,
+    0x06b1c006,
+    0x06b1c9a7,
+    0x06b2a006,
+    0x06b2a9a7,
+    0x06b38006,
+    0x06b389a7,
+    0x06b46006,
+    0x06b469a7,
+    0x06b54006,
+    0x06b549a7,
+    0x06b62006,
+    0x06b629a7,
+    0x06b70006,
+    0x06b709a7,
+    0x06b7e006,
+    0x06b7e9a7,
+    0x06b8c006,
+    0x06b8c9a7,
+    0x06b9a006,
+    0x06b9a9a7,
+    0x06ba8006,
+    0x06ba89a7,
+    0x06bb6006,
+    0x06bb69a7,
+    0x06bc4006,
+    0x06bc49a7,
+    0x06bd816c,
+    0x06be5b0b,
+    0x07d8f002,
+    0x07f000f2,
+    0x07f100f2,
+    0x07f7f801,
+    0x07fcf012,
+    0x07ff80b1,
+    0x080fe802,
+    0x08170002,
+    0x081bb042,
+    0x08500822,
+    0x08502812,
+    0x08506032,
+    0x0851c022,
+    0x0851f802,
+    0x08572812,
+    0x08692032,
+    0x08755812,
+    0x0877e822,
+    0x087a30a2,
+    0x087c1032,
+    0x0880000a,
+    0x08800802,
+    0x0880100a,
+    0x0881c0e2,
+    0x08838002,
+    0x08839812,
+    0x0883f822,
+    0x0884100a,
+    0x0885802a,
+    0x08859832,
+    0x0885b81a,
+    0x0885c812,
+    0x0885e808,
+    0x08861002,
+    0x08866808,
+    0x08880022,
+    0x08893842,
+    0x0889600a,
+    0x08896872,
+    0x088a281a,
+    0x088b9802,
+    0x088c0012,
+    0x088c100a,
+    0x088d982a,
+    0x088db082,
+    0x088df81a,
+    0x088e1018,
+    0x088e4832,
+    0x088e700a,
+    0x088e7802,
+    0x0891602a,
+    0x08917822,
+    0x0891901a,
+    0x0891a002,
+    0x0891a80a,
+    0x0891b012,
+    0x0891f002,
+    0x08920802,
+    0x0896f802,
+    0x0897002a,
+    0x08971872,
+    0x08980012,
+    0x0898101a,
+    0x0899d812,
+    0x0899f002,
+    0x0899f80a,
+    0x089a0002,
+    0x089a083a,
+    0x089a381a,
+    0x089a582a,
+    0x089ab802,
+    0x089b101a,
+    0x089b3062,
+    0x089b8042,
+    0x08a1a82a,
+    0x08a1c072,
+    0x08a2001a,
+    0x08a21022,
+    0x08a2280a,
+    0x08a23002,
+    0x08a2f002,
+    0x08a58002,
+    0x08a5881a,
+    0x08a59852,
+    0x08a5c80a,
+    0x08a5d002,
+    0x08a5d81a,
+    0x08a5e802,
+    0x08a5f00a,
+    0x08a5f812,
+    0x08a6080a,
+    0x08a61012,
+    0x08ad7802,
+    0x08ad801a,
+    0x08ad9032,
+    0x08adc03a,
+    0x08ade012,
+    0x08adf00a,
+    0x08adf812,
+    0x08aee012,
+    0x08b1802a,
+    0x08b19872,
+    0x08b1d81a,
+    0x08b1e802,
+    0x08b1f00a,
+    0x08b1f812,
+    0x08b55802,
+    0x08b5600a,
+    0x08b56802,
+    0x08b5701a,
+    0x08b58052,
+    0x08b5b00a,
+    0x08b5b802,
+    0x08b8e822,
+    0x08b91032,
+    0x08b9300a,
+    0x08b93842,
+    0x08c1602a,
+    0x08c17882,
+    0x08c1c00a,
+    0x08c1c812,
+    0x08c98002,
+    0x08c9884a,
+    0x08c9b81a,
+    0x08c9d812,
+    0x08c9e80a,
+    0x08c9f002,
+    0x08c9f808,
+    0x08ca000a,
+    0x08ca0808,
+    0x08ca100a,
+    0x08ca1802,
+    0x08ce882a,
+    0x08cea032,
+    0x08ced012,
+    0x08cee03a,
+    0x08cf0002,
+    0x08cf200a,
+    0x08d00892,
+    0x08d19852,
+    0x08d1c80a,
+    0x08d1d008,
+    0x08d1d832,
+    0x08d23802,
+    0x08d28852,
+    0x08d2b81a,
+    0x08d2c822,
+    0x08d42058,
+    0x08d450c2,
+    0x08d4b80a,
+    0x08d4c012,
+    0x08e1780a,
+    0x08e18062,
+    0x08e1c052,
+    0x08e1f00a,
+    0x08e1f802,
+    0x08e49152,
+    0x08e5480a,
+    0x08e55062,
+    0x08e5880a,
+    0x08e59012,
+    0x08e5a00a,
+    0x08e5a812,
+    0x08e98852,
+    0x08e9d002,
+    0x08e9e012,
+    0x08e9f862,
+    0x08ea3008,
+    0x08ea3802,
+    0x08ec504a,
+    0x08ec8012,
+    0x08ec981a,
+    0x08eca802,
+    0x08ecb00a,
+    0x08ecb802,
+    0x08f79812,
+    0x08f7a81a,
+    0x08f80012,
+    0x08f81008,
+    0x08f8180a,
+    0x08f9a01a,
+    0x08f9b042,
+    0x08f9f01a,
+    0x08fa0002,
+    0x08fa080a,
+    0x08fa1002,
+    0x09a180f1,
+    0x09a20002,
+    0x09a238e2,
+    0x0b578042,
+    0x0b598062,
+    0x0b7a7802,
+    0x0b7a8b6a,
+    0x0b7c7832,
+    0x0b7f2002,
+    0x0b7f801a,
+    0x0de4e812,
+    0x0de50031,
+    0x0e7802d2,
+    0x0e798162,
+    0x0e8b2802,
+    0x0e8b300a,
+    0x0e8b3822,
+    0x0e8b680a,
+    0x0e8b7042,
+    0x0e8b9871,
+    0x0e8bd872,
+    0x0e8c2862,
+    0x0e8d5032,
+    0x0e921022,
+    0x0ed00362,
+    0x0ed1db12,
+    0x0ed3a802,
+    0x0ed42002,
+    0x0ed4d842,
+    0x0ed508e2,
+    0x0f000062,
+    0x0f004102,
+    0x0f00d862,
+    0x0f011812,
+    0x0f013042,
+    0x0f047802,
+    0x0f098062,
+    0x0f157002,
+    0x0f176032,
+    0x0f276032,
+    0x0f468062,
+    0x0f4a2062,
+    0x0f8007f3,
+    0x0f8407f3,
+    0x0f886823,
+    0x0f897803,
+    0x0f8b6053,
+    0x0f8bf013,
+    0x0f8c7003,
+    0x0f8c8893,
+    0x0f8d6b83,
+    0x0f8f3199,
+    0x0f9008e3,
+    0x0f90d003,
+    0x0f917803,
+    0x0f919083,
+    0x0f91e033,
+    0x0f924ff3,
+    0x0f964ff3,
+    0x0f9a4ff3,
+    0x0f9e4b13,
+    0x0f9fd842,
+    0x0fa007f3,
+    0x0fa407f3,
+    0x0fa803d3,
+    0x0faa37f3,
+    0x0fae37f3,
+    0x0fb23093,
+    0x0fb407f3,
+    0x0fbba0b3,
+    0x0fbeaaa3,
+    0x0fc06033,
+    0x0fc24073,
+    0x0fc2d053,
+    0x0fc44073,
+    0x0fc57513,
+    0x0fc862e3,
+    0x0fc9e093,
+    0x0fca3ff3,
+    0x0fce3ff3,
+    0x0fd23ff3,
+    0x0fd63b83,
+    0x0fe007f3,
+    0x0fe407f3,
+    0x0fe807f3,
+    0x0fec07f3,
+    0x0ff007f3,
+    0x0ff407f3,
+    0x0ff807f3,
+    0x0ffc07d3,
+    0x700001f1,
+    0x700105f2,
+    0x700407f1,
+    0x700807f2,
+    0x700c06f2,
+    0x700f87f1,
+    0x701387f1,
+    0x701787f1,
+    0x701b87f1,
+    0x701f87f1,
+    0x702387f1,
+    0x702787f1,
+    0x702b87f1,
+    0x702f87f1,
+    0x703387f1,
+    0x703787f1,
+    0x703b87f1,
+    0x703f87f1,
+    0x704387f1,
+    0x704787f1,
+    0x704b87f1,
+    0x704f87f1,
+    0x705387f1,
+    0x705787f1,
+    0x705b87f1,
+    0x705f87f1,
+    0x706387f1,
+    0x706787f1,
+    0x706b87f1,
+    0x706f87f1,
+    0x707387f1,
+    0x707787f1,
+    0x707b87f1,
+    0x707f80f1};
+// clang-format on
+
+/// Returns the extended grapheme cluster bondary property of a code point.
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __property __get_property(const char32_t __code_point) noexcept {
+  // The algorithm searches for the upper bound of the range and, when found,
+  // steps back one entry. This algorithm is used since the code point can be
+  // anywhere in the range. After a lower bound is found the next step is to
+  // compare whether the code unit is indeed in the range.
+  //
+  // Since the entry contains a code unit, size, and property the code point
+  // being sought needs to be adjusted. Just shifting the code point to the
+  // proper position doesn't work; suppose an entry has property 0, size 1,
+  // and lower bound 3. This results in the entry 0x1810.
+  // When searching for code point 3 it will search for 0x1800, find 0x1810
+  // and moves to the previous entry. Thus the lower bound value will never
+  // be found.
+  // The simple solution is to set the bits belonging to the property and
+  // size. Then the upper bound for code point 3 will return the entry after
+  // 0x1810. After moving to the previous entry the algorithm arrives at the
+  // correct entry.
+  ptrdiff_t __i = std::ranges::upper_bound(__entries, (__code_point << 11) | 0x7ffu) - __entries;
+  if (__i == 0)
+    return __property::__none;
+
+  --__i;
+  uint32_t __upper_bound = (__entries[__i] >> 11) + ((__entries[__i] >> 4) & 0x7f);
+  if (__code_point <= __upper_bound)
+    return static_cast<__property>(__entries[__i] & 0xf);
+
+  return __property::__none;
+}
+
+} // namespace __extended_grapheme_custer_property_boundary
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_EXTENDED_GRAPHEME_CLUSTER_TABLE_H
diff --git a/libcxx/include/__cxx03/__format/format_arg.h b/libcxx/include/__cxx03/__format/format_arg.h
new file mode 100644
index 000000000000..aa02f81dc40e
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/format_arg.h
@@ -0,0 +1,401 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMAT_ARG_H
+#define _LIBCPP___FORMAT_FORMAT_ARG_H
+
+#include <__assert>
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <__format/concepts.h>
+#include <__format/format_parse_context.h>
+#include <__functional/invoke.h>
+#include <__fwd/format.h>
+#include <__memory/addressof.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/remove_const.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/unreachable.h>
+#include <__variant/monostate.h>
+#include <cstdint>
+#include <string_view>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __format {
+/// The type stored in @ref basic_format_arg.
+///
+/// @note The 128-bit types are unconditionally in the list to avoid the values
+/// of the enums to depend on the availability of 128-bit integers.
+///
+/// @note The value is stored as a 5-bit value in the __packed_arg_t_bits. This
+/// limits the maximum number of elements to 32.
+/// When modifying update the test
+/// test/libcxx/utilities/format/format.arguments/format.arg/arg_t.compile.pass.cpp
+/// It could be packed in 4-bits but that means a new type directly becomes an
+/// ABI break. The packed type is 64-bit so this reduces the maximum number of
+/// packed elements from 16 to 12.
+///
+/// @note Some members of this enum are an extension. These extensions need
+/// special behaviour in visit_format_arg. There they need to be wrapped in a
+/// handle to satisfy the user observable behaviour. The internal function
+/// __visit_format_arg doesn't do this wrapping. So in the format functions
+/// this function is used to avoid unneeded overhead.
+enum class __arg_t : uint8_t {
+  __none,
+  __boolean,
+  __char_type,
+  __int,
+  __long_long,
+  __i128, // extension
+  __unsigned,
+  __unsigned_long_long,
+  __u128, // extension
+  __float,
+  __double,
+  __long_double,
+  __const_char_type_ptr,
+  __string_view,
+  __ptr,
+  __handle
+};
+
+inline constexpr unsigned __packed_arg_t_bits = 5;
+inline constexpr uint8_t __packed_arg_t_mask  = 0x1f;
+
+inline constexpr unsigned __packed_types_storage_bits = 64;
+inline constexpr unsigned __packed_types_max          = __packed_types_storage_bits / __packed_arg_t_bits;
+
+_LIBCPP_HIDE_FROM_ABI constexpr bool __use_packed_format_arg_store(size_t __size) {
+  return __size <= __packed_types_max;
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr __arg_t __get_packed_type(uint64_t __types, size_t __id) {
+  _LIBCPP_ASSERT_INTERNAL(__id <= __packed_types_max, "");
+
+  if (__id > 0)
+    __types >>= __id * __packed_arg_t_bits;
+
+  return static_cast<__format::__arg_t>(__types & __packed_arg_t_mask);
+}
+
+} // namespace __format
+
+// This function is not user observable, so it can directly use the non-standard
+// types of the "variant". See __arg_t for more details.
+template <class _Visitor, class _Context>
+_LIBCPP_HIDE_FROM_ABI decltype(auto) __visit_format_arg(_Visitor&& __vis, basic_format_arg<_Context> __arg) {
+  switch (__arg.__type_) {
+  case __format::__arg_t::__none:
+    return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__monostate_);
+  case __format::__arg_t::__boolean:
+    return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__boolean_);
+  case __format::__arg_t::__char_type:
+    return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__char_type_);
+  case __format::__arg_t::__int:
+    return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__int_);
+  case __format::__arg_t::__long_long:
+    return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__long_long_);
+  case __format::__arg_t::__i128:
+#  ifndef _LIBCPP_HAS_NO_INT128
+    return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__i128_);
+#  else
+    __libcpp_unreachable();
+#  endif
+  case __format::__arg_t::__unsigned:
+    return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__unsigned_);
+  case __format::__arg_t::__unsigned_long_long:
+    return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__unsigned_long_long_);
+  case __format::__arg_t::__u128:
+#  ifndef _LIBCPP_HAS_NO_INT128
+    return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__u128_);
+#  else
+    __libcpp_unreachable();
+#  endif
+  case __format::__arg_t::__float:
+    return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__float_);
+  case __format::__arg_t::__double:
+    return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__double_);
+  case __format::__arg_t::__long_double:
+    return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__long_double_);
+  case __format::__arg_t::__const_char_type_ptr:
+    return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__const_char_type_ptr_);
+  case __format::__arg_t::__string_view:
+    return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__string_view_);
+  case __format::__arg_t::__ptr:
+    return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__ptr_);
+  case __format::__arg_t::__handle:
+    return std::invoke(
+        std::forward<_Visitor>(__vis), typename basic_format_arg<_Context>::handle{__arg.__value_.__handle_});
+  }
+
+  __libcpp_unreachable();
+}
+
+#  if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+
+template <class _Rp, class _Visitor, class _Context>
+_LIBCPP_HIDE_FROM_ABI _Rp __visit_format_arg(_Visitor&& __vis, basic_format_arg<_Context> __arg) {
+  switch (__arg.__type_) {
+  case __format::__arg_t::__none:
+    return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__monostate_);
+  case __format::__arg_t::__boolean:
+    return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__boolean_);
+  case __format::__arg_t::__char_type:
+    return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__char_type_);
+  case __format::__arg_t::__int:
+    return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__int_);
+  case __format::__arg_t::__long_long:
+    return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__long_long_);
+  case __format::__arg_t::__i128:
+#    ifndef _LIBCPP_HAS_NO_INT128
+    return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__i128_);
+#    else
+    __libcpp_unreachable();
+#    endif
+  case __format::__arg_t::__unsigned:
+    return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__unsigned_);
+  case __format::__arg_t::__unsigned_long_long:
+    return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__unsigned_long_long_);
+  case __format::__arg_t::__u128:
+#    ifndef _LIBCPP_HAS_NO_INT128
+    return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__u128_);
+#    else
+    __libcpp_unreachable();
+#    endif
+  case __format::__arg_t::__float:
+    return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__float_);
+  case __format::__arg_t::__double:
+    return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__double_);
+  case __format::__arg_t::__long_double:
+    return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__long_double_);
+  case __format::__arg_t::__const_char_type_ptr:
+    return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__const_char_type_ptr_);
+  case __format::__arg_t::__string_view:
+    return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__string_view_);
+  case __format::__arg_t::__ptr:
+    return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__ptr_);
+  case __format::__arg_t::__handle:
+    return std::invoke_r<_Rp>(
+        std::forward<_Visitor>(__vis), typename basic_format_arg<_Context>::handle{__arg.__value_.__handle_});
+  }
+
+  __libcpp_unreachable();
+}
+
+#  endif // _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+
+/// Contains the values used in basic_format_arg.
+///
+/// This is a separate type so it's possible to store the values and types in
+/// separate arrays.
+template <class _Context>
+class __basic_format_arg_value {
+  using _CharT = typename _Context::char_type;
+
+public:
+  /// Contains the implementation for basic_format_arg::handle.
+  struct __handle {
+    template <class _Tp>
+    _LIBCPP_HIDE_FROM_ABI explicit __handle(_Tp& __v) noexcept
+        : __ptr_(std::addressof(__v)),
+          __format_([](basic_format_parse_context<_CharT>& __parse_ctx, _Context& __ctx, const void* __ptr) {
+            using _Dp = remove_const_t<_Tp>;
+            using _Qp = conditional_t<__formattable_with<const _Dp, _Context>, const _Dp, _Dp>;
+            static_assert(__formattable_with<_Qp, _Context>, "Mandated by [format.arg]/10");
+
+            typename _Context::template formatter_type<_Dp> __f;
+            __parse_ctx.advance_to(__f.parse(__parse_ctx));
+            __ctx.advance_to(__f.format(*const_cast<_Qp*>(static_cast<const _Dp*>(__ptr)), __ctx));
+          }) {}
+
+    const void* __ptr_;
+    void (*__format_)(basic_format_parse_context<_CharT>&, _Context&, const void*);
+  };
+
+  union {
+    monostate __monostate_;
+    bool __boolean_;
+    _CharT __char_type_;
+    int __int_;
+    unsigned __unsigned_;
+    long long __long_long_;
+    unsigned long long __unsigned_long_long_;
+#  ifndef _LIBCPP_HAS_NO_INT128
+    __int128_t __i128_;
+    __uint128_t __u128_;
+#  endif
+    float __float_;
+    double __double_;
+    long double __long_double_;
+    const _CharT* __const_char_type_ptr_;
+    basic_string_view<_CharT> __string_view_;
+    const void* __ptr_;
+    __handle __handle_;
+  };
+
+  // These constructors contain the exact storage type used. If adjustments are
+  // required, these will be done in __create_format_arg.
+
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value() noexcept : __monostate_() {}
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(bool __value) noexcept : __boolean_(__value) {}
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(_CharT __value) noexcept : __char_type_(__value) {}
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(int __value) noexcept : __int_(__value) {}
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(unsigned __value) noexcept : __unsigned_(__value) {}
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(long long __value) noexcept : __long_long_(__value) {}
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(unsigned long long __value) noexcept
+      : __unsigned_long_long_(__value) {}
+#  ifndef _LIBCPP_HAS_NO_INT128
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(__int128_t __value) noexcept : __i128_(__value) {}
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(__uint128_t __value) noexcept : __u128_(__value) {}
+#  endif
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(float __value) noexcept : __float_(__value) {}
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(double __value) noexcept : __double_(__value) {}
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(long double __value) noexcept : __long_double_(__value) {}
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(const _CharT* __value) noexcept : __const_char_type_ptr_(__value) {}
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(basic_string_view<_CharT> __value) noexcept
+      : __string_view_(__value) {}
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(const void* __value) noexcept : __ptr_(__value) {}
+  _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(__handle&& __value) noexcept : __handle_(std::move(__value)) {}
+};
+
+template <class _Context>
+class _LIBCPP_TEMPLATE_VIS basic_format_arg {
+public:
+  class _LIBCPP_TEMPLATE_VIS handle;
+
+  _LIBCPP_HIDE_FROM_ABI basic_format_arg() noexcept : __type_{__format::__arg_t::__none} {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit operator bool() const noexcept { return __type_ != __format::__arg_t::__none; }
+
+#  if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+
+  // This function is user facing, so it must wrap the non-standard types of
+  // the "variant" in a handle to stay conforming. See __arg_t for more details.
+  template <class _Visitor>
+  _LIBCPP_HIDE_FROM_ABI decltype(auto) visit(this basic_format_arg __arg, _Visitor&& __vis) {
+    switch (__arg.__type_) {
+#    ifndef _LIBCPP_HAS_NO_INT128
+    case __format::__arg_t::__i128: {
+      typename __basic_format_arg_value<_Context>::__handle __h{__arg.__value_.__i128_};
+      return std::invoke(std::forward<_Visitor>(__vis), typename basic_format_arg<_Context>::handle{__h});
+    }
+
+    case __format::__arg_t::__u128: {
+      typename __basic_format_arg_value<_Context>::__handle __h{__arg.__value_.__u128_};
+      return std::invoke(std::forward<_Visitor>(__vis), typename basic_format_arg<_Context>::handle{__h});
+    }
+#    endif
+    default:
+      return std::__visit_format_arg(std::forward<_Visitor>(__vis), __arg);
+    }
+  }
+
+  // This function is user facing, so it must wrap the non-standard types of
+  // the "variant" in a handle to stay conforming. See __arg_t for more details.
+  template <class _Rp, class _Visitor>
+  _LIBCPP_HIDE_FROM_ABI _Rp visit(this basic_format_arg __arg, _Visitor&& __vis) {
+    switch (__arg.__type_) {
+#    ifndef _LIBCPP_HAS_NO_INT128
+    case __format::__arg_t::__i128: {
+      typename __basic_format_arg_value<_Context>::__handle __h{__arg.__value_.__i128_};
+      return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), typename basic_format_arg<_Context>::handle{__h});
+    }
+
+    case __format::__arg_t::__u128: {
+      typename __basic_format_arg_value<_Context>::__handle __h{__arg.__value_.__u128_};
+      return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), typename basic_format_arg<_Context>::handle{__h});
+    }
+#    endif
+    default:
+      return std::__visit_format_arg<_Rp>(std::forward<_Visitor>(__vis), __arg);
+    }
+  }
+
+#  endif // _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+
+private:
+  using char_type = typename _Context::char_type;
+
+  // TODO FMT Implement constrain [format.arg]/4
+  // Constraints: The template specialization
+  //   typename Context::template formatter_type<T>
+  // meets the Formatter requirements ([formatter.requirements]).  The extent
+  // to which an implementation determines that the specialization meets the
+  // Formatter requirements is unspecified, except that as a minimum the
+  // expression
+  //   typename Context::template formatter_type<T>()
+  //    .format(declval<const T&>(), declval<Context&>())
+  // shall be well-formed when treated as an unevaluated operand.
+
+public:
+  __basic_format_arg_value<_Context> __value_;
+  __format::__arg_t __type_;
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_format_arg(__format::__arg_t __type,
+                                                  __basic_format_arg_value<_Context> __value) noexcept
+      : __value_(__value), __type_(__type) {}
+};
+
+template <class _Context>
+class _LIBCPP_TEMPLATE_VIS basic_format_arg<_Context>::handle {
+public:
+  _LIBCPP_HIDE_FROM_ABI void format(basic_format_parse_context<char_type>& __parse_ctx, _Context& __ctx) const {
+    __handle_.__format_(__parse_ctx, __ctx, __handle_.__ptr_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI explicit handle(typename __basic_format_arg_value<_Context>::__handle& __handle) noexcept
+      : __handle_(__handle) {}
+
+private:
+  typename __basic_format_arg_value<_Context>::__handle& __handle_;
+};
+
+// This function is user facing, so it must wrap the non-standard types of
+// the "variant" in a handle to stay conforming. See __arg_t for more details.
+template <class _Visitor, class _Context>
+#  if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+_LIBCPP_DEPRECATED_IN_CXX26
+#  endif
+    _LIBCPP_HIDE_FROM_ABI decltype(auto)
+    visit_format_arg(_Visitor&& __vis, basic_format_arg<_Context> __arg) {
+  switch (__arg.__type_) {
+#  ifndef _LIBCPP_HAS_NO_INT128
+  case __format::__arg_t::__i128: {
+    typename __basic_format_arg_value<_Context>::__handle __h{__arg.__value_.__i128_};
+    return std::invoke(std::forward<_Visitor>(__vis), typename basic_format_arg<_Context>::handle{__h});
+  }
+
+  case __format::__arg_t::__u128: {
+    typename __basic_format_arg_value<_Context>::__handle __h{__arg.__value_.__u128_};
+    return std::invoke(std::forward<_Visitor>(__vis), typename basic_format_arg<_Context>::handle{__h});
+  }
+#  endif // _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+  default:
+    return std::__visit_format_arg(std::forward<_Visitor>(__vis), __arg);
+  }
+}
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FORMAT_FORMAT_ARG_H
diff --git a/libcxx/include/__cxx03/__format/format_arg_store.h b/libcxx/include/__cxx03/__format/format_arg_store.h
new file mode 100644
index 000000000000..23a599e99575
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/format_arg_store.h
@@ -0,0 +1,266 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMAT_ARG_STORE_H
+#define _LIBCPP___FORMAT_FORMAT_ARG_STORE_H
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#include <__concepts/arithmetic.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/concepts.h>
+#include <__format/format_arg.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/extent.h>
+#include <__type_traits/remove_const.h>
+#include <string>
+#include <string_view>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __format {
+
+/// \returns The @c __arg_t based on the type of the formatting argument.
+///
+/// \pre \c __formattable<_Tp, typename _Context::char_type>
+template <class _Context, class _Tp>
+consteval __arg_t __determine_arg_t();
+
+// Boolean
+template <class, same_as<bool> _Tp>
+consteval __arg_t __determine_arg_t() {
+  return __arg_t::__boolean;
+}
+
+// Char
+template <class _Context, same_as<typename _Context::char_type> _Tp>
+consteval __arg_t __determine_arg_t() {
+  return __arg_t::__char_type;
+}
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class _Context, class _CharT>
+  requires(same_as<typename _Context::char_type, wchar_t> && same_as<_CharT, char>)
+consteval __arg_t __determine_arg_t() {
+  return __arg_t::__char_type;
+}
+#  endif
+
+// Signed integers
+template <class, __libcpp_signed_integer _Tp>
+consteval __arg_t __determine_arg_t() {
+  if constexpr (sizeof(_Tp) <= sizeof(int))
+    return __arg_t::__int;
+  else if constexpr (sizeof(_Tp) <= sizeof(long long))
+    return __arg_t::__long_long;
+#  ifndef _LIBCPP_HAS_NO_INT128
+  else if constexpr (sizeof(_Tp) == sizeof(__int128_t))
+    return __arg_t::__i128;
+#  endif
+  else
+    static_assert(sizeof(_Tp) == 0, "an unsupported signed integer was used");
+}
+
+// Unsigned integers
+template <class, __libcpp_unsigned_integer _Tp>
+consteval __arg_t __determine_arg_t() {
+  if constexpr (sizeof(_Tp) <= sizeof(unsigned))
+    return __arg_t::__unsigned;
+  else if constexpr (sizeof(_Tp) <= sizeof(unsigned long long))
+    return __arg_t::__unsigned_long_long;
+#  ifndef _LIBCPP_HAS_NO_INT128
+  else if constexpr (sizeof(_Tp) == sizeof(__uint128_t))
+    return __arg_t::__u128;
+#  endif
+  else
+    static_assert(sizeof(_Tp) == 0, "an unsupported unsigned integer was used");
+}
+
+// Floating-point
+template <class, same_as<float> _Tp>
+consteval __arg_t __determine_arg_t() {
+  return __arg_t::__float;
+}
+template <class, same_as<double> _Tp>
+consteval __arg_t __determine_arg_t() {
+  return __arg_t::__double;
+}
+template <class, same_as<long double> _Tp>
+consteval __arg_t __determine_arg_t() {
+  return __arg_t::__long_double;
+}
+
+// Char pointer
+template <class _Context, class _Tp>
+  requires(same_as<typename _Context::char_type*, _Tp> || same_as<const typename _Context::char_type*, _Tp>)
+consteval __arg_t __determine_arg_t() {
+  return __arg_t::__const_char_type_ptr;
+}
+
+// Char array
+template <class _Context, class _Tp>
+  requires(is_array_v<_Tp> && same_as<_Tp, typename _Context::char_type[extent_v<_Tp>]>)
+consteval __arg_t __determine_arg_t() {
+  return __arg_t::__string_view;
+}
+
+// String view
+template <class _Context, class _Tp>
+  requires(same_as<typename _Context::char_type, typename _Tp::value_type> &&
+           same_as<_Tp, basic_string_view<typename _Tp::value_type, typename _Tp::traits_type>>)
+consteval __arg_t __determine_arg_t() {
+  return __arg_t::__string_view;
+}
+
+// String
+template <class _Context, class _Tp>
+  requires(
+      same_as<typename _Context::char_type, typename _Tp::value_type> &&
+      same_as<_Tp, basic_string<typename _Tp::value_type, typename _Tp::traits_type, typename _Tp::allocator_type>>)
+consteval __arg_t __determine_arg_t() {
+  return __arg_t::__string_view;
+}
+
+// Pointers
+template <class, class _Ptr>
+  requires(same_as<_Ptr, void*> || same_as<_Ptr, const void*> || same_as<_Ptr, nullptr_t>)
+consteval __arg_t __determine_arg_t() {
+  return __arg_t::__ptr;
+}
+
+// Handle
+//
+// Note this version can't be constrained avoiding ambiguous overloads.
+// That means it can be instantiated by disabled formatters. To solve this, a
+// constrained version for not formattable formatters is added.
+template <class _Context, class _Tp>
+consteval __arg_t __determine_arg_t() {
+  return __arg_t::__handle;
+}
+
+// The overload for not formattable types allows triggering the static
+// assertion below.
+template <class _Context, class _Tp>
+  requires(!__formattable_with<_Tp, _Context>)
+consteval __arg_t __determine_arg_t() {
+  return __arg_t::__none;
+}
+
+// Pseudo constuctor for basic_format_arg
+//
+// Modeled after template<class T> explicit basic_format_arg(T& v) noexcept;
+// [format.arg]/4-6
+template <class _Context, class _Tp>
+_LIBCPP_HIDE_FROM_ABI basic_format_arg<_Context> __create_format_arg(_Tp& __value) noexcept {
+  using _Dp               = remove_const_t<_Tp>;
+  constexpr __arg_t __arg = __determine_arg_t<_Context, _Dp>();
+  static_assert(__arg != __arg_t::__none, "the supplied type is not formattable");
+  static_assert(__formattable_with<_Tp, _Context>);
+
+  // Not all types can be used to directly initialize the
+  // __basic_format_arg_value.  First handle all types needing adjustment, the
+  // final else requires no adjustment.
+  if constexpr (__arg == __arg_t::__char_type)
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+    if constexpr (same_as<typename _Context::char_type, wchar_t> && same_as<_Dp, char>)
+      return basic_format_arg<_Context>{__arg, static_cast<wchar_t>(static_cast<unsigned char>(__value))};
+    else
+#  endif
+      return basic_format_arg<_Context>{__arg, __value};
+  else if constexpr (__arg == __arg_t::__int)
+    return basic_format_arg<_Context>{__arg, static_cast<int>(__value)};
+  else if constexpr (__arg == __arg_t::__long_long)
+    return basic_format_arg<_Context>{__arg, static_cast<long long>(__value)};
+  else if constexpr (__arg == __arg_t::__unsigned)
+    return basic_format_arg<_Context>{__arg, static_cast<unsigned>(__value)};
+  else if constexpr (__arg == __arg_t::__unsigned_long_long)
+    return basic_format_arg<_Context>{__arg, static_cast<unsigned long long>(__value)};
+  else if constexpr (__arg == __arg_t::__string_view)
+    // Using std::size on a character array will add the NUL-terminator to the size.
+    if constexpr (is_array_v<_Dp>)
+      return basic_format_arg<_Context>{
+          __arg, basic_string_view<typename _Context::char_type>{__value, extent_v<_Dp> - 1}};
+    else
+      // When the _Traits or _Allocator are different an implicit conversion will
+      // fail.
+      return basic_format_arg<_Context>{
+          __arg, basic_string_view<typename _Context::char_type>{__value.data(), __value.size()}};
+  else if constexpr (__arg == __arg_t::__ptr)
+    return basic_format_arg<_Context>{__arg, static_cast<const void*>(__value)};
+  else if constexpr (__arg == __arg_t::__handle)
+    return basic_format_arg<_Context>{__arg, typename __basic_format_arg_value<_Context>::__handle{__value}};
+  else
+    return basic_format_arg<_Context>{__arg, __value};
+}
+
+template <class _Context, class... _Args>
+_LIBCPP_HIDE_FROM_ABI void
+__create_packed_storage(uint64_t& __types, __basic_format_arg_value<_Context>* __values, _Args&... __args) noexcept {
+  int __shift = 0;
+  (
+      [&] {
+        basic_format_arg<_Context> __arg = __format::__create_format_arg<_Context>(__args);
+        if (__shift != 0)
+          __types |= static_cast<uint64_t>(__arg.__type_) << __shift;
+        else
+          // Assigns the initial value.
+          __types = static_cast<uint64_t>(__arg.__type_);
+        __shift += __packed_arg_t_bits;
+        *__values++ = __arg.__value_;
+      }(),
+      ...);
+}
+
+template <class _Context, class... _Args>
+_LIBCPP_HIDE_FROM_ABI void __store_basic_format_arg(basic_format_arg<_Context>* __data, _Args&... __args) noexcept {
+  ([&] { *__data++ = __format::__create_format_arg<_Context>(__args); }(), ...);
+}
+
+template <class _Context, size_t _Np>
+struct __packed_format_arg_store {
+  __basic_format_arg_value<_Context> __values_[_Np];
+  uint64_t __types_ = 0;
+};
+
+template <class _Context, size_t _Np>
+struct __unpacked_format_arg_store {
+  basic_format_arg<_Context> __args_[_Np];
+};
+
+} // namespace __format
+
+template <class _Context, class... _Args>
+struct _LIBCPP_TEMPLATE_VIS __format_arg_store {
+  _LIBCPP_HIDE_FROM_ABI __format_arg_store(_Args&... __args) noexcept {
+    if constexpr (sizeof...(_Args) != 0) {
+      if constexpr (__format::__use_packed_format_arg_store(sizeof...(_Args)))
+        __format::__create_packed_storage(__storage.__types_, __storage.__values_, __args...);
+      else
+        __format::__store_basic_format_arg<_Context>(__storage.__args_, __args...);
+    }
+  }
+
+  using _Storage =
+      conditional_t<__format::__use_packed_format_arg_store(sizeof...(_Args)),
+                    __format::__packed_format_arg_store<_Context, sizeof...(_Args)>,
+                    __format::__unpacked_format_arg_store<_Context, sizeof...(_Args)>>;
+
+  _Storage __storage;
+};
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_FORMAT_ARG_STORE_H
diff --git a/libcxx/include/__cxx03/__format/format_args.h b/libcxx/include/__cxx03/__format/format_args.h
new file mode 100644
index 000000000000..07923570f389
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/format_args.h
@@ -0,0 +1,78 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMAT_ARGS_H
+#define _LIBCPP___FORMAT_FORMAT_ARGS_H
+
+#include <__config>
+#include <__format/format_arg.h>
+#include <__format/format_arg_store.h>
+#include <__fwd/format.h>
+#include <cstddef>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Context>
+class _LIBCPP_TEMPLATE_VIS basic_format_args {
+public:
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI basic_format_args(const __format_arg_store<_Context, _Args...>& __store) noexcept
+      : __size_(sizeof...(_Args)) {
+    if constexpr (sizeof...(_Args) != 0) {
+      if constexpr (__format::__use_packed_format_arg_store(sizeof...(_Args))) {
+        __values_ = __store.__storage.__values_;
+        __types_  = __store.__storage.__types_;
+      } else
+        __args_ = __store.__storage.__args_;
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI basic_format_arg<_Context> get(size_t __id) const noexcept {
+    if (__id >= __size_)
+      return basic_format_arg<_Context>{};
+
+    if (__format::__use_packed_format_arg_store(__size_))
+      return basic_format_arg<_Context>{__format::__get_packed_type(__types_, __id), __values_[__id]};
+
+    return __args_[__id];
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_t __size() const noexcept { return __size_; }
+
+private:
+  size_t __size_{0};
+  // [format.args]/5
+  // [Note 1: Implementations are encouraged to optimize the representation of
+  // basic_format_args for small number of formatting arguments by storing
+  // indices of type alternatives separately from values and packing the
+  // former. - end note]
+  union {
+    struct {
+      const __basic_format_arg_value<_Context>* __values_;
+      uint64_t __types_;
+    };
+    const basic_format_arg<_Context>* __args_;
+  };
+};
+
+template <class _Context, class... _Args>
+basic_format_args(__format_arg_store<_Context, _Args...>) -> basic_format_args<_Context>;
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_FORMAT_ARGS_H
diff --git a/libcxx/include/__cxx03/__format/format_context.h b/libcxx/include/__cxx03/__format/format_context.h
new file mode 100644
index 000000000000..20c07559eae4
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/format_context.h
@@ -0,0 +1,220 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMAT_CONTEXT_H
+#define _LIBCPP___FORMAT_FORMAT_CONTEXT_H
+
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/buffer.h>
+#include <__format/format_arg.h>
+#include <__format/format_arg_store.h>
+#include <__format/format_args.h>
+#include <__format/format_error.h>
+#include <__fwd/format.h>
+#include <__iterator/back_insert_iterator.h>
+#include <__iterator/concepts.h>
+#include <__memory/addressof.h>
+#include <__utility/move.h>
+#include <__variant/monostate.h>
+#include <cstddef>
+
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#  include <__locale>
+#  include <optional>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _OutIt, class _CharT>
+  requires output_iterator<_OutIt, const _CharT&>
+class _LIBCPP_TEMPLATE_VIS basic_format_context;
+
+#  ifndef _LIBCPP_HAS_NO_LOCALIZATION
+/**
+ * Helper to create a basic_format_context.
+ *
+ * This is needed since the constructor is private.
+ */
+template <class _OutIt, class _CharT>
+_LIBCPP_HIDE_FROM_ABI basic_format_context<_OutIt, _CharT>
+__format_context_create(_OutIt __out_it,
+                        basic_format_args<basic_format_context<_OutIt, _CharT>> __args,
+                        optional<std::locale>&& __loc = nullopt) {
+  return std::basic_format_context(std::move(__out_it), __args, std::move(__loc));
+}
+#  else
+template <class _OutIt, class _CharT>
+_LIBCPP_HIDE_FROM_ABI basic_format_context<_OutIt, _CharT>
+__format_context_create(_OutIt __out_it, basic_format_args<basic_format_context<_OutIt, _CharT>> __args) {
+  return std::basic_format_context(std::move(__out_it), __args);
+}
+#  endif
+
+using format_context = basic_format_context<back_insert_iterator<__format::__output_buffer<char>>, char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using wformat_context = basic_format_context< back_insert_iterator<__format::__output_buffer<wchar_t>>, wchar_t>;
+#  endif
+
+template <class _OutIt, class _CharT>
+  requires output_iterator<_OutIt, const _CharT&>
+class
+    // clang-format off
+    _LIBCPP_TEMPLATE_VIS
+    _LIBCPP_PREFERRED_NAME(format_context)
+    _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wformat_context))
+    // clang-format on
+    basic_format_context {
+public:
+  using iterator  = _OutIt;
+  using char_type = _CharT;
+  template <class _Tp>
+  using formatter_type = formatter<_Tp, _CharT>;
+
+  _LIBCPP_HIDE_FROM_ABI basic_format_arg<basic_format_context> arg(size_t __id) const noexcept {
+    return __args_.get(__id);
+  }
+#  ifndef _LIBCPP_HAS_NO_LOCALIZATION
+  _LIBCPP_HIDE_FROM_ABI std::locale locale() {
+    if (!__loc_)
+      __loc_ = std::locale{};
+    return *__loc_;
+  }
+#  endif
+  _LIBCPP_HIDE_FROM_ABI iterator out() { return std::move(__out_it_); }
+  _LIBCPP_HIDE_FROM_ABI void advance_to(iterator __it) { __out_it_ = std::move(__it); }
+
+private:
+  iterator __out_it_;
+  basic_format_args<basic_format_context> __args_;
+#  ifndef _LIBCPP_HAS_NO_LOCALIZATION
+
+  // The Standard doesn't specify how the locale is stored.
+  // [format.context]/6
+  // std::locale locale();
+  //   Returns: The locale passed to the formatting function if the latter
+  //   takes one, and std::locale() otherwise.
+  // This is done by storing the locale of the constructor in this optional. If
+  // locale() is called and the optional has no value the value will be created.
+  // This allows the implementation to lazily create the locale.
+  // TODO FMT Validate whether lazy creation is the best solution.
+  optional<std::locale> __loc_;
+
+  template <class _OtherOutIt, class _OtherCharT>
+  friend _LIBCPP_HIDE_FROM_ABI basic_format_context<_OtherOutIt, _OtherCharT> __format_context_create(
+      _OtherOutIt, basic_format_args<basic_format_context<_OtherOutIt, _OtherCharT>>, optional<std::locale>&&);
+
+  // Note: the Standard doesn't specify the required constructors.
+  _LIBCPP_HIDE_FROM_ABI explicit basic_format_context(
+      _OutIt __out_it, basic_format_args<basic_format_context> __args, optional<std::locale>&& __loc)
+      : __out_it_(std::move(__out_it)), __args_(__args), __loc_(std::move(__loc)) {}
+#  else
+  template <class _OtherOutIt, class _OtherCharT>
+  friend _LIBCPP_HIDE_FROM_ABI basic_format_context<_OtherOutIt, _OtherCharT>
+      __format_context_create(_OtherOutIt, basic_format_args<basic_format_context<_OtherOutIt, _OtherCharT>>);
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_format_context(_OutIt __out_it, basic_format_args<basic_format_context> __args)
+      : __out_it_(std::move(__out_it)), __args_(__args) {}
+#  endif
+
+  basic_format_context(const basic_format_context&)            = delete;
+  basic_format_context& operator=(const basic_format_context&) = delete;
+};
+
+// A specialization for __retarget_buffer
+//
+// See __retarget_buffer for the motivation for this specialization.
+//
+// This context holds a reference to the instance of the basic_format_context
+// that is retargeted. It converts a formatting argument when it is requested
+// during formatting. It is expected that the usage of the arguments is rare so
+// the lookups are not expected to be used often. An alternative would be to
+// convert all elements during construction.
+//
+// The elements of the retargets context are only used when an underlying
+// formatter uses a locale specific formatting or an formatting argument is
+// part for the format spec. For example
+//   format("{:256:{}}", input, 8);
+// Here the width of an element in input is determined dynamically.
+// Note when the top-level element has no width the retargeting is not needed.
+template <class _CharT>
+class _LIBCPP_TEMPLATE_VIS basic_format_context<typename __format::__retarget_buffer<_CharT>::__iterator, _CharT> {
+public:
+  using iterator  = typename __format::__retarget_buffer<_CharT>::__iterator;
+  using char_type = _CharT;
+  template <class _Tp>
+  using formatter_type = formatter<_Tp, _CharT>;
+
+  template <class _Context>
+  _LIBCPP_HIDE_FROM_ABI explicit basic_format_context(iterator __out_it, _Context& __ctx)
+      : __out_it_(std::move(__out_it)),
+#  ifndef _LIBCPP_HAS_NO_LOCALIZATION
+        __loc_([](void* __c) { return static_cast<_Context*>(__c)->locale(); }),
+#  endif
+        __ctx_(std::addressof(__ctx)),
+        __arg_([](void* __c, size_t __id) {
+          auto __visitor = [&](auto __arg) -> basic_format_arg<basic_format_context> {
+            if constexpr (same_as<decltype(__arg), monostate>)
+              return {};
+            else if constexpr (same_as<decltype(__arg), typename basic_format_arg<_Context>::handle>)
+              // At the moment it's not possible for formatting to use a re-targeted handle.
+              // TODO FMT add this when support is needed.
+              std::__throw_format_error("Re-targeting handle not supported");
+            else
+              return basic_format_arg<basic_format_context>{
+                  __format::__determine_arg_t<basic_format_context, decltype(__arg)>(),
+                  __basic_format_arg_value<basic_format_context>(__arg)};
+          };
+#  if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+          return static_cast<_Context*>(__c)->arg(__id).visit(std::move(__visitor));
+#  else
+          _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+          return std::visit_format_arg(std::move(__visitor), static_cast<_Context*>(__c)->arg(__id));
+          _LIBCPP_SUPPRESS_DEPRECATED_POP
+#  endif // _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+        }) {
+  }
+
+  _LIBCPP_HIDE_FROM_ABI basic_format_arg<basic_format_context> arg(size_t __id) const noexcept {
+    return __arg_(__ctx_, __id);
+  }
+#  ifndef _LIBCPP_HAS_NO_LOCALIZATION
+  _LIBCPP_HIDE_FROM_ABI std::locale locale() { return __loc_(__ctx_); }
+#  endif
+  _LIBCPP_HIDE_FROM_ABI iterator out() { return std::move(__out_it_); }
+  _LIBCPP_HIDE_FROM_ABI void advance_to(iterator __it) { __out_it_ = std::move(__it); }
+
+private:
+  iterator __out_it_;
+
+#  ifndef _LIBCPP_HAS_NO_LOCALIZATION
+  std::locale (*__loc_)(void* __ctx);
+#  endif
+
+  void* __ctx_;
+  basic_format_arg<basic_format_context> (*__arg_)(void* __ctx, size_t __id);
+};
+
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(basic_format_context);
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FORMAT_FORMAT_CONTEXT_H
diff --git a/libcxx/include/__cxx03/__format/format_error.h b/libcxx/include/__cxx03/__format/format_error.h
new file mode 100644
index 000000000000..ed40e395d6af
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/format_error.h
@@ -0,0 +1,50 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMAT_ERROR_H
+#define _LIBCPP___FORMAT_FORMAT_ERROR_H
+
+#include <__config>
+#include <__verbose_abort>
+#include <stdexcept>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wweak-vtables")
+class _LIBCPP_EXPORTED_FROM_ABI format_error : public runtime_error {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit format_error(const string& __s) : runtime_error(__s) {}
+  _LIBCPP_HIDE_FROM_ABI explicit format_error(const char* __s) : runtime_error(__s) {}
+  _LIBCPP_HIDE_FROM_ABI format_error(const format_error&)            = default;
+  _LIBCPP_HIDE_FROM_ABI format_error& operator=(const format_error&) = default;
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL
+  ~format_error() noexcept override = default;
+};
+_LIBCPP_DIAGNOSTIC_POP
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_format_error(const char* __s) {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw format_error(__s);
+#  else
+  _LIBCPP_VERBOSE_ABORT("format_error was thrown in -fno-exceptions mode with message \"%s\"", __s);
+#  endif
+}
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_FORMAT_ERROR_H
diff --git a/libcxx/include/__cxx03/__format/format_functions.h b/libcxx/include/__cxx03/__format/format_functions.h
new file mode 100644
index 000000000000..d14b49aff149
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/format_functions.h
@@ -0,0 +1,680 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMAT_FUNCTIONS
+#define _LIBCPP___FORMAT_FORMAT_FUNCTIONS
+
+#include <__algorithm/clamp.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/buffer.h>
+#include <__format/format_arg.h>
+#include <__format/format_arg_store.h>
+#include <__format/format_args.h>
+#include <__format/format_context.h>
+#include <__format/format_error.h>
+#include <__format/format_parse_context.h>
+#include <__format/format_string.h>
+#include <__format/format_to_n_result.h>
+#include <__format/formatter.h>
+#include <__format/formatter_bool.h>
+#include <__format/formatter_char.h>
+#include <__format/formatter_floating_point.h>
+#include <__format/formatter_integer.h>
+#include <__format/formatter_pointer.h>
+#include <__format/formatter_string.h>
+#include <__format/parser_std_format_spec.h>
+#include <__iterator/back_insert_iterator.h>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h> // iter_value_t
+#include <__variant/monostate.h>
+#include <array>
+#include <string>
+#include <string_view>
+
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#  include <__locale>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// TODO FMT Evaluate which templates should be external templates. This
+// improves the efficiency of the header. However since the header is still
+// under heavy development and not all classes are stable it makes no sense
+// to do this optimization now.
+
+using format_args = basic_format_args<format_context>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using wformat_args = basic_format_args<wformat_context>;
+#  endif
+
+template <class _Context = format_context, class... _Args>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI __format_arg_store<_Context, _Args...> make_format_args(_Args&... __args) {
+  return std::__format_arg_store<_Context, _Args...>(__args...);
+}
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class... _Args>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI __format_arg_store<wformat_context, _Args...> make_wformat_args(_Args&... __args) {
+  return std::__format_arg_store<wformat_context, _Args...>(__args...);
+}
+#  endif
+
+namespace __format {
+
+/// Helper class parse and handle argument.
+///
+/// When parsing a handle which is not enabled the code is ill-formed.
+/// This helper uses the parser of the appropriate formatter for the stored type.
+template <class _CharT>
+class _LIBCPP_TEMPLATE_VIS __compile_time_handle {
+public:
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr void __parse(_ParseContext& __ctx) const {
+    __parse_(__ctx);
+  }
+
+  template <class _Tp>
+  _LIBCPP_HIDE_FROM_ABI constexpr void __enable() {
+    __parse_ = [](basic_format_parse_context<_CharT>& __ctx) {
+      formatter<_Tp, _CharT> __f;
+      __ctx.advance_to(__f.parse(__ctx));
+    };
+  }
+
+  // Before calling __parse the proper handler needs to be set with __enable.
+  // The default handler isn't a core constant expression.
+  _LIBCPP_HIDE_FROM_ABI constexpr __compile_time_handle()
+      : __parse_([](basic_format_parse_context<_CharT>&) { std::__throw_format_error("Not a handle"); }) {}
+
+private:
+  void (*__parse_)(basic_format_parse_context<_CharT>&);
+};
+
+// Dummy format_context only providing the parts used during constant
+// validation of the basic_format_string.
+template <class _CharT>
+struct _LIBCPP_TEMPLATE_VIS __compile_time_basic_format_context {
+public:
+  using char_type = _CharT;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __compile_time_basic_format_context(
+      const __arg_t* __args, const __compile_time_handle<_CharT>* __handles, size_t __size)
+      : __args_(__args), __handles_(__handles), __size_(__size) {}
+
+  // During the compile-time validation nothing needs to be written.
+  // Therefore all operations of this iterator are a NOP.
+  struct iterator {
+    _LIBCPP_HIDE_FROM_ABI constexpr iterator& operator=(_CharT) { return *this; }
+    _LIBCPP_HIDE_FROM_ABI constexpr iterator& operator*() { return *this; }
+    _LIBCPP_HIDE_FROM_ABI constexpr iterator operator++(int) { return *this; }
+  };
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __arg_t arg(size_t __id) const {
+    if (__id >= __size_)
+      std::__throw_format_error("The argument index value is too large for the number of arguments supplied");
+    return __args_[__id];
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const __compile_time_handle<_CharT>& __handle(size_t __id) const {
+    if (__id >= __size_)
+      std::__throw_format_error("The argument index value is too large for the number of arguments supplied");
+    return __handles_[__id];
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator out() { return {}; }
+  _LIBCPP_HIDE_FROM_ABI constexpr void advance_to(iterator) {}
+
+private:
+  const __arg_t* __args_;
+  const __compile_time_handle<_CharT>* __handles_;
+  size_t __size_;
+};
+
+// [format.string.std]/8
+// If { arg-idopt } is used in a width or precision, the value of the
+// corresponding formatting argument is used in its place. If the
+// corresponding formatting argument is not of standard signed or unsigned
+// integer type, or its value is negative for precision or non-positive for
+// width, an exception of type format_error is thrown.
+//
+// _HasPrecision does the formatter have a precision?
+template <class _CharT, class _Tp, bool _HasPrecision = false>
+_LIBCPP_HIDE_FROM_ABI constexpr void __compile_time_validate_argument(
+    basic_format_parse_context<_CharT>& __parse_ctx, __compile_time_basic_format_context<_CharT>& __ctx) {
+  auto __validate_type = [](__arg_t __type) {
+    // LWG3720 originally allowed "signed or unsigned integer types", however
+    // the final version explicitly changed it to "*standard* signed or unsigned
+    // integer types". It's trivial to use 128-bit integrals in libc++'s
+    // implementation, but other implementations may not implement it.
+    // (Using a width or precision, that does not fit in 64-bits, sounds very
+    // unlikely in real world code.)
+    switch (__type) {
+    case __arg_t::__int:
+    case __arg_t::__long_long:
+    case __arg_t::__unsigned:
+    case __arg_t::__unsigned_long_long:
+      return;
+
+    default:
+      std::__throw_format_error("Replacement argument isn't a standard signed or unsigned integer type");
+    }
+  };
+
+  formatter<_Tp, _CharT> __formatter;
+  __parse_ctx.advance_to(__formatter.parse(__parse_ctx));
+  if (__formatter.__parser_.__width_as_arg_)
+    __validate_type(__ctx.arg(__formatter.__parser_.__width_));
+
+  if constexpr (_HasPrecision)
+    if (__formatter.__parser_.__precision_as_arg_)
+      __validate_type(__ctx.arg(__formatter.__parser_.__precision_));
+}
+
+// This function is not user facing, so it can directly use the non-standard types of the "variant".
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI constexpr void __compile_time_visit_format_arg(
+    basic_format_parse_context<_CharT>& __parse_ctx,
+    __compile_time_basic_format_context<_CharT>& __ctx,
+    __arg_t __type) {
+  switch (__type) {
+  case __arg_t::__none:
+    std::__throw_format_error("Invalid argument");
+  case __arg_t::__boolean:
+    return __format::__compile_time_validate_argument<_CharT, bool>(__parse_ctx, __ctx);
+  case __arg_t::__char_type:
+    return __format::__compile_time_validate_argument<_CharT, _CharT>(__parse_ctx, __ctx);
+  case __arg_t::__int:
+    return __format::__compile_time_validate_argument<_CharT, int>(__parse_ctx, __ctx);
+  case __arg_t::__long_long:
+    return __format::__compile_time_validate_argument<_CharT, long long>(__parse_ctx, __ctx);
+  case __arg_t::__i128:
+#  ifndef _LIBCPP_HAS_NO_INT128
+    return __format::__compile_time_validate_argument<_CharT, __int128_t>(__parse_ctx, __ctx);
+#  else
+    std::__throw_format_error("Invalid argument");
+#  endif
+    return;
+  case __arg_t::__unsigned:
+    return __format::__compile_time_validate_argument<_CharT, unsigned>(__parse_ctx, __ctx);
+  case __arg_t::__unsigned_long_long:
+    return __format::__compile_time_validate_argument<_CharT, unsigned long long>(__parse_ctx, __ctx);
+  case __arg_t::__u128:
+#  ifndef _LIBCPP_HAS_NO_INT128
+    return __format::__compile_time_validate_argument<_CharT, __uint128_t>(__parse_ctx, __ctx);
+#  else
+    std::__throw_format_error("Invalid argument");
+#  endif
+    return;
+  case __arg_t::__float:
+    return __format::__compile_time_validate_argument<_CharT, float, true>(__parse_ctx, __ctx);
+  case __arg_t::__double:
+    return __format::__compile_time_validate_argument<_CharT, double, true>(__parse_ctx, __ctx);
+  case __arg_t::__long_double:
+    return __format::__compile_time_validate_argument<_CharT, long double, true>(__parse_ctx, __ctx);
+  case __arg_t::__const_char_type_ptr:
+    return __format::__compile_time_validate_argument<_CharT, const _CharT*, true>(__parse_ctx, __ctx);
+  case __arg_t::__string_view:
+    return __format::__compile_time_validate_argument<_CharT, basic_string_view<_CharT>, true>(__parse_ctx, __ctx);
+  case __arg_t::__ptr:
+    return __format::__compile_time_validate_argument<_CharT, const void*>(__parse_ctx, __ctx);
+  case __arg_t::__handle:
+    std::__throw_format_error("Handle should use __compile_time_validate_handle_argument");
+  }
+  std::__throw_format_error("Invalid argument");
+}
+
+template <contiguous_iterator _Iterator, class _ParseCtx, class _Ctx>
+_LIBCPP_HIDE_FROM_ABI constexpr _Iterator
+__handle_replacement_field(_Iterator __begin, _Iterator __end, _ParseCtx& __parse_ctx, _Ctx& __ctx) {
+  using _CharT                        = iter_value_t<_Iterator>;
+  __format::__parse_number_result __r = __format::__parse_arg_id(__begin, __end, __parse_ctx);
+
+  if (__r.__last == __end)
+    std::__throw_format_error("The argument index should end with a ':' or a '}'");
+
+  bool __parse = *__r.__last == _CharT(':');
+  switch (*__r.__last) {
+  case _CharT(':'):
+    // The arg-id has a format-specifier, advance the input to the format-spec.
+    __parse_ctx.advance_to(__r.__last + 1);
+    break;
+  case _CharT('}'):
+    // The arg-id has no format-specifier.
+    __parse_ctx.advance_to(__r.__last);
+    break;
+  default:
+    std::__throw_format_error("The argument index should end with a ':' or a '}'");
+  }
+
+  if constexpr (same_as<_Ctx, __compile_time_basic_format_context<_CharT>>) {
+    __arg_t __type = __ctx.arg(__r.__value);
+    if (__type == __arg_t::__none)
+      std::__throw_format_error("The argument index value is too large for the number of arguments supplied");
+    else if (__type == __arg_t::__handle)
+      __ctx.__handle(__r.__value).__parse(__parse_ctx);
+    else if (__parse)
+      __format::__compile_time_visit_format_arg(__parse_ctx, __ctx, __type);
+  } else
+    std::__visit_format_arg(
+        [&](auto __arg) {
+          if constexpr (same_as<decltype(__arg), monostate>)
+            std::__throw_format_error("The argument index value is too large for the number of arguments supplied");
+          else if constexpr (same_as<decltype(__arg), typename basic_format_arg<_Ctx>::handle>)
+            __arg.format(__parse_ctx, __ctx);
+          else {
+            formatter<decltype(__arg), _CharT> __formatter;
+            if (__parse)
+              __parse_ctx.advance_to(__formatter.parse(__parse_ctx));
+            __ctx.advance_to(__formatter.format(__arg, __ctx));
+          }
+        },
+        __ctx.arg(__r.__value));
+
+  __begin = __parse_ctx.begin();
+  if (__begin == __end || *__begin != _CharT('}'))
+    std::__throw_format_error("The replacement field misses a terminating '}'");
+
+  return ++__begin;
+}
+
+template <class _ParseCtx, class _Ctx>
+_LIBCPP_HIDE_FROM_ABI constexpr typename _Ctx::iterator __vformat_to(_ParseCtx&& __parse_ctx, _Ctx&& __ctx) {
+  using _CharT = typename _ParseCtx::char_type;
+  static_assert(same_as<typename _Ctx::char_type, _CharT>);
+
+  auto __begin                     = __parse_ctx.begin();
+  auto __end                       = __parse_ctx.end();
+  typename _Ctx::iterator __out_it = __ctx.out();
+  while (__begin != __end) {
+    switch (*__begin) {
+    case _CharT('{'):
+      ++__begin;
+      if (__begin == __end)
+        std::__throw_format_error("The format string terminates at a '{'");
+
+      if (*__begin != _CharT('{')) [[likely]] {
+        __ctx.advance_to(std::move(__out_it));
+        __begin  = __format::__handle_replacement_field(__begin, __end, __parse_ctx, __ctx);
+        __out_it = __ctx.out();
+
+        // The output is written and __begin points to the next character. So
+        // start the next iteration.
+        continue;
+      }
+      // The string is an escape character.
+      break;
+
+    case _CharT('}'):
+      ++__begin;
+      if (__begin == __end || *__begin != _CharT('}'))
+        std::__throw_format_error("The format string contains an invalid escape sequence");
+
+      break;
+    }
+
+    // Copy the character to the output verbatim.
+    *__out_it++ = *__begin++;
+  }
+  return __out_it;
+}
+
+} // namespace __format
+
+#  if _LIBCPP_STD_VER >= 26
+template <class _CharT>
+struct _LIBCPP_TEMPLATE_VIS __runtime_format_string {
+private:
+  basic_string_view<_CharT> __str_;
+
+  template <class _Cp, class... _Args>
+  friend struct _LIBCPP_TEMPLATE_VIS basic_format_string;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __runtime_format_string(basic_string_view<_CharT> __s) noexcept : __str_(__s) {}
+
+  __runtime_format_string(const __runtime_format_string&)            = delete;
+  __runtime_format_string& operator=(const __runtime_format_string&) = delete;
+};
+
+_LIBCPP_HIDE_FROM_ABI inline __runtime_format_string<char> runtime_format(string_view __fmt) noexcept { return __fmt; }
+#    ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+_LIBCPP_HIDE_FROM_ABI inline __runtime_format_string<wchar_t> runtime_format(wstring_view __fmt) noexcept {
+  return __fmt;
+}
+#    endif
+#  endif //_LIBCPP_STD_VER >= 26
+
+template <class _CharT, class... _Args>
+struct _LIBCPP_TEMPLATE_VIS basic_format_string {
+  template <class _Tp>
+    requires convertible_to<const _Tp&, basic_string_view<_CharT>>
+  consteval basic_format_string(const _Tp& __str) : __str_{__str} {
+    __format::__vformat_to(basic_format_parse_context<_CharT>{__str_, sizeof...(_Args)},
+                           _Context{__types_.data(), __handles_.data(), sizeof...(_Args)});
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<_CharT> get() const noexcept { return __str_; }
+#  if _LIBCPP_STD_VER >= 26
+  _LIBCPP_HIDE_FROM_ABI basic_format_string(__runtime_format_string<_CharT> __s) noexcept : __str_(__s.__str_) {}
+#  endif
+
+private:
+  basic_string_view<_CharT> __str_;
+
+  using _Context = __format::__compile_time_basic_format_context<_CharT>;
+
+  static constexpr array<__format::__arg_t, sizeof...(_Args)> __types_{
+      __format::__determine_arg_t<_Context, remove_cvref_t<_Args>>()...};
+
+  static constexpr array<__format::__compile_time_handle<_CharT>, sizeof...(_Args)> __handles_{[] {
+    using _Tp = remove_cvref_t<_Args>;
+    __format::__compile_time_handle<_CharT> __handle;
+    if (__format::__determine_arg_t<_Context, _Tp>() == __format::__arg_t::__handle)
+      __handle.template __enable<_Tp>();
+
+    return __handle;
+  }()...};
+};
+
+template <class... _Args>
+using format_string = basic_format_string<char, type_identity_t<_Args>...>;
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class... _Args>
+using wformat_string = basic_format_string<wchar_t, type_identity_t<_Args>...>;
+#  endif
+
+template <class _OutIt, class _CharT, class _FormatOutIt>
+  requires(output_iterator<_OutIt, const _CharT&>)
+_LIBCPP_HIDE_FROM_ABI _OutIt __vformat_to(_OutIt __out_it,
+                                          basic_string_view<_CharT> __fmt,
+                                          basic_format_args<basic_format_context<_FormatOutIt, _CharT>> __args) {
+  if constexpr (same_as<_OutIt, _FormatOutIt>)
+    return std::__format::__vformat_to(
+        basic_format_parse_context{__fmt, __args.__size()}, std::__format_context_create(std::move(__out_it), __args));
+  else {
+    __format::__format_buffer<_OutIt, _CharT> __buffer{std::move(__out_it)};
+    std::__format::__vformat_to(basic_format_parse_context{__fmt, __args.__size()},
+                                std::__format_context_create(__buffer.__make_output_iterator(), __args));
+    return std::move(__buffer).__out_it();
+  }
+}
+
+// The function is _LIBCPP_ALWAYS_INLINE since the compiler is bad at inlining
+// https://reviews.llvm.org/D110499#inline-1180704
+// TODO FMT Evaluate whether we want to file a Clang bug report regarding this.
+template <output_iterator<const char&> _OutIt>
+_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt vformat_to(_OutIt __out_it, string_view __fmt, format_args __args) {
+  return std::__vformat_to(std::move(__out_it), __fmt, __args);
+}
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <output_iterator<const wchar_t&> _OutIt>
+_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt
+vformat_to(_OutIt __out_it, wstring_view __fmt, wformat_args __args) {
+  return std::__vformat_to(std::move(__out_it), __fmt, __args);
+}
+#  endif
+
+template <output_iterator<const char&> _OutIt, class... _Args>
+_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt
+format_to(_OutIt __out_it, format_string<_Args...> __fmt, _Args&&... __args) {
+  return std::vformat_to(std::move(__out_it), __fmt.get(), std::make_format_args(__args...));
+}
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <output_iterator<const wchar_t&> _OutIt, class... _Args>
+_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt
+format_to(_OutIt __out_it, wformat_string<_Args...> __fmt, _Args&&... __args) {
+  return std::vformat_to(std::move(__out_it), __fmt.get(), std::make_wformat_args(__args...));
+}
+#  endif
+
+// TODO FMT This needs to be a template or std::to_chars(floating-point) availability markup
+// fires too eagerly, see http://llvm.org/PR61563.
+template <class = void>
+[[nodiscard]] _LIBCPP_ALWAYS_INLINE inline _LIBCPP_HIDE_FROM_ABI string vformat(string_view __fmt, format_args __args) {
+  string __res;
+  std::vformat_to(std::back_inserter(__res), __fmt, __args);
+  return __res;
+}
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+// TODO FMT This needs to be a template or std::to_chars(floating-point) availability markup
+// fires too eagerly, see http://llvm.org/PR61563.
+template <class = void>
+[[nodiscard]] _LIBCPP_ALWAYS_INLINE inline _LIBCPP_HIDE_FROM_ABI wstring
+vformat(wstring_view __fmt, wformat_args __args) {
+  wstring __res;
+  std::vformat_to(std::back_inserter(__res), __fmt, __args);
+  return __res;
+}
+#  endif
+
+template <class... _Args>
+[[nodiscard]] _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI string
+format(format_string<_Args...> __fmt, _Args&&... __args) {
+  return std::vformat(__fmt.get(), std::make_format_args(__args...));
+}
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class... _Args>
+[[nodiscard]] _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI wstring
+format(wformat_string<_Args...> __fmt, _Args&&... __args) {
+  return std::vformat(__fmt.get(), std::make_wformat_args(__args...));
+}
+#  endif
+
+template <class _Context, class _OutIt, class _CharT>
+_LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt>
+__vformat_to_n(_OutIt __out_it,
+               iter_difference_t<_OutIt> __n,
+               basic_string_view<_CharT> __fmt,
+               basic_format_args<_Context> __args) {
+  __format::__format_to_n_buffer<_OutIt, _CharT> __buffer{std::move(__out_it), __n};
+  std::__format::__vformat_to(basic_format_parse_context{__fmt, __args.__size()},
+                              std::__format_context_create(__buffer.__make_output_iterator(), __args));
+  return std::move(__buffer).__result();
+}
+
+template <output_iterator<const char&> _OutIt, class... _Args>
+_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt>
+format_to_n(_OutIt __out_it, iter_difference_t<_OutIt> __n, format_string<_Args...> __fmt, _Args&&... __args) {
+  return std::__vformat_to_n<format_context>(std::move(__out_it), __n, __fmt.get(), std::make_format_args(__args...));
+}
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <output_iterator<const wchar_t&> _OutIt, class... _Args>
+_LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt>
+format_to_n(_OutIt __out_it, iter_difference_t<_OutIt> __n, wformat_string<_Args...> __fmt, _Args&&... __args) {
+  return std::__vformat_to_n<wformat_context>(std::move(__out_it), __n, __fmt.get(), std::make_wformat_args(__args...));
+}
+#  endif
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI size_t __vformatted_size(basic_string_view<_CharT> __fmt, auto __args) {
+  __format::__formatted_size_buffer<_CharT> __buffer;
+  std::__format::__vformat_to(basic_format_parse_context{__fmt, __args.__size()},
+                              std::__format_context_create(__buffer.__make_output_iterator(), __args));
+  return std::move(__buffer).__result();
+}
+
+template <class... _Args>
+[[nodiscard]] _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI size_t
+formatted_size(format_string<_Args...> __fmt, _Args&&... __args) {
+  return std::__vformatted_size(__fmt.get(), basic_format_args{std::make_format_args(__args...)});
+}
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class... _Args>
+[[nodiscard]] _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI size_t
+formatted_size(wformat_string<_Args...> __fmt, _Args&&... __args) {
+  return std::__vformatted_size(__fmt.get(), basic_format_args{std::make_wformat_args(__args...)});
+}
+#  endif
+
+#  ifndef _LIBCPP_HAS_NO_LOCALIZATION
+
+template <class _OutIt, class _CharT, class _FormatOutIt>
+  requires(output_iterator<_OutIt, const _CharT&>)
+_LIBCPP_HIDE_FROM_ABI _OutIt __vformat_to(
+    _OutIt __out_it,
+    locale __loc,
+    basic_string_view<_CharT> __fmt,
+    basic_format_args<basic_format_context<_FormatOutIt, _CharT>> __args) {
+  if constexpr (same_as<_OutIt, _FormatOutIt>)
+    return std::__format::__vformat_to(basic_format_parse_context{__fmt, __args.__size()},
+                                       std::__format_context_create(std::move(__out_it), __args, std::move(__loc)));
+  else {
+    __format::__format_buffer<_OutIt, _CharT> __buffer{std::move(__out_it)};
+    std::__format::__vformat_to(
+        basic_format_parse_context{__fmt, __args.__size()},
+        std::__format_context_create(__buffer.__make_output_iterator(), __args, std::move(__loc)));
+    return std::move(__buffer).__out_it();
+  }
+}
+
+template <output_iterator<const char&> _OutIt>
+_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt
+vformat_to(_OutIt __out_it, locale __loc, string_view __fmt, format_args __args) {
+  return std::__vformat_to(std::move(__out_it), std::move(__loc), __fmt, __args);
+}
+
+#    ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <output_iterator<const wchar_t&> _OutIt>
+_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt
+vformat_to(_OutIt __out_it, locale __loc, wstring_view __fmt, wformat_args __args) {
+  return std::__vformat_to(std::move(__out_it), std::move(__loc), __fmt, __args);
+}
+#    endif
+
+template <output_iterator<const char&> _OutIt, class... _Args>
+_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt
+format_to(_OutIt __out_it, locale __loc, format_string<_Args...> __fmt, _Args&&... __args) {
+  return std::vformat_to(std::move(__out_it), std::move(__loc), __fmt.get(), std::make_format_args(__args...));
+}
+
+#    ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <output_iterator<const wchar_t&> _OutIt, class... _Args>
+_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt
+format_to(_OutIt __out_it, locale __loc, wformat_string<_Args...> __fmt, _Args&&... __args) {
+  return std::vformat_to(std::move(__out_it), std::move(__loc), __fmt.get(), std::make_wformat_args(__args...));
+}
+#    endif
+
+// TODO FMT This needs to be a template or std::to_chars(floating-point) availability markup
+// fires too eagerly, see http://llvm.org/PR61563.
+template <class = void>
+[[nodiscard]] _LIBCPP_ALWAYS_INLINE inline _LIBCPP_HIDE_FROM_ABI string
+vformat(locale __loc, string_view __fmt, format_args __args) {
+  string __res;
+  std::vformat_to(std::back_inserter(__res), std::move(__loc), __fmt, __args);
+  return __res;
+}
+
+#    ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+// TODO FMT This needs to be a template or std::to_chars(floating-point) availability markup
+// fires too eagerly, see http://llvm.org/PR61563.
+template <class = void>
+[[nodiscard]] _LIBCPP_ALWAYS_INLINE inline _LIBCPP_HIDE_FROM_ABI wstring
+vformat(locale __loc, wstring_view __fmt, wformat_args __args) {
+  wstring __res;
+  std::vformat_to(std::back_inserter(__res), std::move(__loc), __fmt, __args);
+  return __res;
+}
+#    endif
+
+template <class... _Args>
+[[nodiscard]] _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI string
+format(locale __loc, format_string<_Args...> __fmt, _Args&&... __args) {
+  return std::vformat(std::move(__loc), __fmt.get(), std::make_format_args(__args...));
+}
+
+#    ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class... _Args>
+[[nodiscard]] _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI wstring
+format(locale __loc, wformat_string<_Args...> __fmt, _Args&&... __args) {
+  return std::vformat(std::move(__loc), __fmt.get(), std::make_wformat_args(__args...));
+}
+#    endif
+
+template <class _Context, class _OutIt, class _CharT>
+_LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> __vformat_to_n(
+    _OutIt __out_it,
+    iter_difference_t<_OutIt> __n,
+    locale __loc,
+    basic_string_view<_CharT> __fmt,
+    basic_format_args<_Context> __args) {
+  __format::__format_to_n_buffer<_OutIt, _CharT> __buffer{std::move(__out_it), __n};
+  std::__format::__vformat_to(
+      basic_format_parse_context{__fmt, __args.__size()},
+      std::__format_context_create(__buffer.__make_output_iterator(), __args, std::move(__loc)));
+  return std::move(__buffer).__result();
+}
+
+template <output_iterator<const char&> _OutIt, class... _Args>
+_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> format_to_n(
+    _OutIt __out_it, iter_difference_t<_OutIt> __n, locale __loc, format_string<_Args...> __fmt, _Args&&... __args) {
+  return std::__vformat_to_n<format_context>(
+      std::move(__out_it), __n, std::move(__loc), __fmt.get(), std::make_format_args(__args...));
+}
+
+#    ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <output_iterator<const wchar_t&> _OutIt, class... _Args>
+_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> format_to_n(
+    _OutIt __out_it, iter_difference_t<_OutIt> __n, locale __loc, wformat_string<_Args...> __fmt, _Args&&... __args) {
+  return std::__vformat_to_n<wformat_context>(
+      std::move(__out_it), __n, std::move(__loc), __fmt.get(), std::make_wformat_args(__args...));
+}
+#    endif
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI size_t __vformatted_size(locale __loc, basic_string_view<_CharT> __fmt, auto __args) {
+  __format::__formatted_size_buffer<_CharT> __buffer;
+  std::__format::__vformat_to(
+      basic_format_parse_context{__fmt, __args.__size()},
+      std::__format_context_create(__buffer.__make_output_iterator(), __args, std::move(__loc)));
+  return std::move(__buffer).__result();
+}
+
+template <class... _Args>
+[[nodiscard]] _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI size_t
+formatted_size(locale __loc, format_string<_Args...> __fmt, _Args&&... __args) {
+  return std::__vformatted_size(std::move(__loc), __fmt.get(), basic_format_args{std::make_format_args(__args...)});
+}
+
+#    ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class... _Args>
+[[nodiscard]] _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI size_t
+formatted_size(locale __loc, wformat_string<_Args...> __fmt, _Args&&... __args) {
+  return std::__vformatted_size(std::move(__loc), __fmt.get(), basic_format_args{std::make_wformat_args(__args...)});
+}
+#    endif
+
+#  endif // _LIBCPP_HAS_NO_LOCALIZATION
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FORMAT_FORMAT_FUNCTIONS
diff --git a/libcxx/include/__cxx03/__format/format_parse_context.h b/libcxx/include/__cxx03/__format/format_parse_context.h
new file mode 100644
index 000000000000..aefcd5497f3b
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/format_parse_context.h
@@ -0,0 +1,105 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMAT_PARSE_CONTEXT_H
+#define _LIBCPP___FORMAT_FORMAT_PARSE_CONTEXT_H
+
+#include <__config>
+#include <__format/format_error.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <string_view>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _CharT>
+class _LIBCPP_TEMPLATE_VIS basic_format_parse_context {
+public:
+  using char_type      = _CharT;
+  using const_iterator = typename basic_string_view<_CharT>::const_iterator;
+  using iterator       = const_iterator;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit basic_format_parse_context(
+      basic_string_view<_CharT> __fmt, size_t __num_args = 0) noexcept
+      : __begin_(__fmt.begin()),
+        __end_(__fmt.end()),
+        __indexing_(__unknown),
+        __next_arg_id_(0),
+        __num_args_(__num_args) {}
+
+  basic_format_parse_context(const basic_format_parse_context&)            = delete;
+  basic_format_parse_context& operator=(const basic_format_parse_context&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const_iterator begin() const noexcept { return __begin_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr const_iterator end() const noexcept { return __end_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr void advance_to(const_iterator __it) { __begin_ = __it; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr size_t next_arg_id() {
+    if (__indexing_ == __manual)
+      std::__throw_format_error("Using automatic argument numbering in manual argument numbering mode");
+
+    if (__indexing_ == __unknown)
+      __indexing_ = __automatic;
+
+    // Throws an exception to make the expression a non core constant
+    // expression as required by:
+    // [format.parse.ctx]/8
+    //   Remarks: Let cur-arg-id be the value of next_arg_id_ prior to this
+    //   call. Call expressions where cur-arg-id >= num_args_ is true are not
+    //   core constant expressions (7.7 [expr.const]).
+    // Note: the Throws clause [format.parse.ctx]/9 doesn't specify the
+    // behavior when id >= num_args_.
+    if (is_constant_evaluated() && __next_arg_id_ >= __num_args_)
+      std::__throw_format_error("Argument index outside the valid range");
+
+    return __next_arg_id_++;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr void check_arg_id(size_t __id) {
+    if (__indexing_ == __automatic)
+      std::__throw_format_error("Using manual argument numbering in automatic argument numbering mode");
+
+    if (__indexing_ == __unknown)
+      __indexing_ = __manual;
+
+    // Throws an exception to make the expression a non core constant
+    // expression as required by:
+    // [format.parse.ctx]/11
+    //   Remarks: Call expressions where id >= num_args_ are not core constant
+    //   expressions ([expr.const]).
+    // Note: the Throws clause [format.parse.ctx]/10 doesn't specify the
+    // behavior when id >= num_args_.
+    if (is_constant_evaluated() && __id >= __num_args_)
+      std::__throw_format_error("Argument index outside the valid range");
+  }
+
+private:
+  iterator __begin_;
+  iterator __end_;
+  enum _Indexing { __unknown, __manual, __automatic };
+  _Indexing __indexing_;
+  size_t __next_arg_id_;
+  size_t __num_args_;
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(basic_format_parse_context);
+
+using format_parse_context = basic_format_parse_context<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using wformat_parse_context = basic_format_parse_context<wchar_t>;
+#  endif
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_FORMAT_PARSE_CONTEXT_H
diff --git a/libcxx/include/__cxx03/__format/format_string.h b/libcxx/include/__cxx03/__format/format_string.h
new file mode 100644
index 000000000000..bdf3cff7f49b
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/format_string.h
@@ -0,0 +1,160 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMAT_STRING_H
+#define _LIBCPP___FORMAT_FORMAT_STRING_H
+
+#include <__assert>
+#include <__config>
+#include <__format/format_error.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h> // iter_value_t
+#include <cstddef>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __format {
+
+template <contiguous_iterator _Iterator>
+struct _LIBCPP_TEMPLATE_VIS __parse_number_result {
+  _Iterator __last;
+  uint32_t __value;
+};
+
+template <contiguous_iterator _Iterator>
+__parse_number_result(_Iterator, uint32_t) -> __parse_number_result<_Iterator>;
+
+template <contiguous_iterator _Iterator>
+_LIBCPP_HIDE_FROM_ABI constexpr __parse_number_result<_Iterator> __parse_number(_Iterator __begin, _Iterator __end);
+
+/**
+ * The maximum value of a numeric argument.
+ *
+ * This is used for:
+ * * arg-id
+ * * width as value or arg-id.
+ * * precision as value or arg-id.
+ *
+ * The value is compatible with the maximum formatting width and precision
+ * using the `%*` syntax on a 32-bit system.
+ */
+inline constexpr uint32_t __number_max = INT32_MAX;
+
+namespace __detail {
+template <contiguous_iterator _Iterator>
+_LIBCPP_HIDE_FROM_ABI constexpr __parse_number_result<_Iterator>
+__parse_zero(_Iterator __begin, _Iterator, auto& __parse_ctx) {
+  __parse_ctx.check_arg_id(0);
+  return {++__begin, 0}; // can never be larger than the maximum.
+}
+
+template <contiguous_iterator _Iterator>
+_LIBCPP_HIDE_FROM_ABI constexpr __parse_number_result<_Iterator>
+__parse_automatic(_Iterator __begin, _Iterator, auto& __parse_ctx) {
+  size_t __value = __parse_ctx.next_arg_id();
+  _LIBCPP_ASSERT_UNCATEGORIZED(__value <= __number_max, "Compilers don't support this number of arguments");
+
+  return {__begin, uint32_t(__value)};
+}
+
+template <contiguous_iterator _Iterator>
+_LIBCPP_HIDE_FROM_ABI constexpr __parse_number_result<_Iterator>
+__parse_manual(_Iterator __begin, _Iterator __end, auto& __parse_ctx) {
+  __parse_number_result<_Iterator> __r = __format::__parse_number(__begin, __end);
+  __parse_ctx.check_arg_id(__r.__value);
+  return __r;
+}
+
+} // namespace __detail
+
+/**
+ * Parses a number.
+ *
+ * The number is used for the 31-bit values @em width and @em precision. This
+ * allows a maximum value of 2147483647.
+ */
+template <contiguous_iterator _Iterator>
+_LIBCPP_HIDE_FROM_ABI constexpr __parse_number_result<_Iterator>
+__parse_number(_Iterator __begin, _Iterator __end_input) {
+  using _CharT = iter_value_t<_Iterator>;
+  static_assert(__format::__number_max == INT32_MAX, "The algorithm is implemented based on this value.");
+  /*
+   * Limit the input to 9 digits, otherwise we need two checks during every
+   * iteration:
+   * - Are we at the end of the input?
+   * - Does the value exceed width of an uint32_t? (Switching to uint64_t would
+   *   have the same issue, but with a higher maximum.)
+   */
+  _Iterator __end  = __end_input - __begin > 9 ? __begin + 9 : __end_input;
+  uint32_t __value = *__begin - _CharT('0');
+  while (++__begin != __end) {
+    if (*__begin < _CharT('0') || *__begin > _CharT('9'))
+      return {__begin, __value};
+
+    __value = __value * 10 + *__begin - _CharT('0');
+  }
+
+  if (__begin != __end_input && *__begin >= _CharT('0') && *__begin <= _CharT('9')) {
+    /*
+     * There are more than 9 digits, do additional validations:
+     * - Does the 10th digit exceed the maximum allowed value?
+     * - Are there more than 10 digits?
+     * (More than 10 digits always overflows the maximum.)
+     */
+    uint64_t __v = uint64_t(__value) * 10 + *__begin++ - _CharT('0');
+    if (__v > __number_max || (__begin != __end_input && *__begin >= _CharT('0') && *__begin <= _CharT('9')))
+      std::__throw_format_error("The numeric value of the format specifier is too large");
+
+    __value = __v;
+  }
+
+  return {__begin, __value};
+}
+
+/**
+ * Multiplexer for all parse functions.
+ *
+ * The parser will return a pointer beyond the last consumed character. This
+ * should be the closing '}' of the arg-id.
+ */
+template <contiguous_iterator _Iterator>
+_LIBCPP_HIDE_FROM_ABI constexpr __parse_number_result<_Iterator>
+__parse_arg_id(_Iterator __begin, _Iterator __end, auto& __parse_ctx) {
+  using _CharT = iter_value_t<_Iterator>;
+  switch (*__begin) {
+  case _CharT('0'):
+    return __detail::__parse_zero(__begin, __end, __parse_ctx);
+
+  case _CharT(':'):
+    // This case is conditionally valid. It's allowed in an arg-id in the
+    // replacement-field, but not in the std-format-spec. The caller can
+    // provide a better diagnostic, so accept it here unconditionally.
+  case _CharT('}'):
+    return __detail::__parse_automatic(__begin, __end, __parse_ctx);
+  }
+  if (*__begin < _CharT('0') || *__begin > _CharT('9'))
+    std::__throw_format_error("The argument index starts with an invalid character");
+
+  return __detail::__parse_manual(__begin, __end, __parse_ctx);
+}
+
+} // namespace __format
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_FORMAT_STRING_H
diff --git a/libcxx/include/__cxx03/__format/format_to_n_result.h b/libcxx/include/__cxx03/__format/format_to_n_result.h
new file mode 100644
index 000000000000..6f30546dec08
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/format_to_n_result.h
@@ -0,0 +1,35 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMAT_TO_N_RESULT_H
+#define _LIBCPP___FORMAT_FORMAT_TO_N_RESULT_H
+
+#include <__config>
+#include <__iterator/incrementable_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _OutIt>
+struct _LIBCPP_TEMPLATE_VIS format_to_n_result {
+  _OutIt out;
+  iter_difference_t<_OutIt> size;
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(format_to_n_result);
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_FORMAT_TO_N_RESULT_H
diff --git a/libcxx/include/__cxx03/__format/formatter.h b/libcxx/include/__cxx03/__format/formatter.h
new file mode 100644
index 000000000000..e2f418f936ee
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/formatter.h
@@ -0,0 +1,53 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMATTER_H
+#define _LIBCPP___FORMAT_FORMATTER_H
+
+#include <__config>
+#include <__fwd/format.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+/// The default formatter template.
+///
+/// [format.formatter.spec]/5
+/// If F is a disabled specialization of formatter, these values are false:
+/// - is_default_constructible_v<F>,
+/// - is_copy_constructible_v<F>,
+/// - is_move_constructible_v<F>,
+/// - is_copy_assignable<F>, and
+/// - is_move_assignable<F>.
+template <class _Tp, class _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter {
+  formatter()                            = delete;
+  formatter(const formatter&)            = delete;
+  formatter& operator=(const formatter&) = delete;
+};
+
+#  if _LIBCPP_STD_VER >= 23
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr void __set_debug_format(_Tp& __formatter) {
+  if constexpr (requires { __formatter.set_debug_format(); })
+    __formatter.set_debug_format();
+}
+
+#  endif // _LIBCPP_STD_VER >= 23
+#endif   // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_FORMATTER_H
diff --git a/libcxx/include/__cxx03/__format/formatter_bool.h b/libcxx/include/__cxx03/__format/formatter_bool.h
new file mode 100644
index 000000000000..17dc69541e8f
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/formatter_bool.h
@@ -0,0 +1,76 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMATTER_BOOL_H
+#define _LIBCPP___FORMAT_FORMATTER_BOOL_H
+
+#include <__algorithm/copy.h>
+#include <__assert>
+#include <__config>
+#include <__format/concepts.h>
+#include <__format/format_parse_context.h>
+#include <__format/formatter.h>
+#include <__format/formatter_integral.h>
+#include <__format/parser_std_format_spec.h>
+#include <__utility/unreachable.h>
+
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#  include <__locale>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<bool, _CharT> {
+public:
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    typename _ParseContext::iterator __result = __parser_.__parse(__ctx, __format_spec::__fields_integral);
+    __format_spec::__process_parsed_bool(__parser_, "a bool");
+    return __result;
+  }
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(bool __value, _FormatContext& __ctx) const {
+    switch (__parser_.__type_) {
+    case __format_spec::__type::__default:
+    case __format_spec::__type::__string:
+      return __formatter::__format_bool(__value, __ctx, __parser_.__get_parsed_std_specifications(__ctx));
+
+    case __format_spec::__type::__binary_lower_case:
+    case __format_spec::__type::__binary_upper_case:
+    case __format_spec::__type::__octal:
+    case __format_spec::__type::__decimal:
+    case __format_spec::__type::__hexadecimal_lower_case:
+    case __format_spec::__type::__hexadecimal_upper_case:
+      // Promotes bool to an integral type. This reduces the number of
+      // instantiations of __format_integer reducing code size.
+      return __formatter::__format_integer(
+          static_cast<unsigned>(__value), __ctx, __parser_.__get_parsed_std_specifications(__ctx));
+
+    default:
+      _LIBCPP_ASSERT_INTERNAL(false, "The parse function should have validated the type");
+      __libcpp_unreachable();
+    }
+  }
+
+  __format_spec::__parser<_CharT> __parser_;
+};
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_FORMATTER_BOOL_H
diff --git a/libcxx/include/__cxx03/__format/formatter_char.h b/libcxx/include/__cxx03/__format/formatter_char.h
new file mode 100644
index 000000000000..d33e84368a76
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/formatter_char.h
@@ -0,0 +1,93 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMATTER_CHAR_H
+#define _LIBCPP___FORMAT_FORMATTER_CHAR_H
+
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/concepts.h>
+#include <__format/format_parse_context.h>
+#include <__format/formatter.h>
+#include <__format/formatter_integral.h>
+#include <__format/formatter_output.h>
+#include <__format/parser_std_format_spec.h>
+#include <__format/write_escaped.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/make_unsigned.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS __formatter_char {
+public:
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    typename _ParseContext::iterator __result = __parser_.__parse(__ctx, __format_spec::__fields_integral);
+    __format_spec::__process_parsed_char(__parser_, "a character");
+    return __result;
+  }
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(_CharT __value, _FormatContext& __ctx) const {
+    if (__parser_.__type_ == __format_spec::__type::__default || __parser_.__type_ == __format_spec::__type::__char)
+      return __formatter::__format_char(__value, __ctx.out(), __parser_.__get_parsed_std_specifications(__ctx));
+
+#  if _LIBCPP_STD_VER >= 23
+    if (__parser_.__type_ == __format_spec::__type::__debug)
+      return __formatter::__format_escaped_char(__value, __ctx.out(), __parser_.__get_parsed_std_specifications(__ctx));
+#  endif
+
+    if constexpr (sizeof(_CharT) <= sizeof(unsigned))
+      return __formatter::__format_integer(
+          static_cast<unsigned>(static_cast<make_unsigned_t<_CharT>>(__value)),
+          __ctx,
+          __parser_.__get_parsed_std_specifications(__ctx));
+    else
+      return __formatter::__format_integer(
+          static_cast<make_unsigned_t<_CharT>>(__value), __ctx, __parser_.__get_parsed_std_specifications(__ctx));
+  }
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(char __value, _FormatContext& __ctx) const
+    requires(same_as<_CharT, wchar_t>)
+  {
+    return format(static_cast<wchar_t>(static_cast<unsigned char>(__value)), __ctx);
+  }
+
+#  if _LIBCPP_STD_VER >= 23
+  _LIBCPP_HIDE_FROM_ABI constexpr void set_debug_format() { __parser_.__type_ = __format_spec::__type::__debug; }
+#  endif
+
+  __format_spec::__parser<_CharT> __parser_;
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS formatter<char, char> : public __formatter_char<char> {};
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+struct _LIBCPP_TEMPLATE_VIS formatter<char, wchar_t> : public __formatter_char<wchar_t> {};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS formatter<wchar_t, wchar_t> : public __formatter_char<wchar_t> {};
+
+#  endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_FORMATTER_CHAR_H
diff --git a/libcxx/include/__cxx03/__format/formatter_floating_point.h b/libcxx/include/__cxx03/__format/formatter_floating_point.h
new file mode 100644
index 000000000000..fa42ba203b0b
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/formatter_floating_point.h
@@ -0,0 +1,783 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMATTER_FLOATING_POINT_H
+#define _LIBCPP___FORMAT_FORMATTER_FLOATING_POINT_H
+
+#include <__algorithm/copy_n.h>
+#include <__algorithm/find.h>
+#include <__algorithm/max.h>
+#include <__algorithm/min.h>
+#include <__algorithm/rotate.h>
+#include <__algorithm/transform.h>
+#include <__assert>
+#include <__charconv/chars_format.h>
+#include <__charconv/to_chars_floating_point.h>
+#include <__charconv/to_chars_result.h>
+#include <__concepts/arithmetic.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/concepts.h>
+#include <__format/format_parse_context.h>
+#include <__format/formatter.h>
+#include <__format/formatter_integral.h>
+#include <__format/formatter_output.h>
+#include <__format/parser_std_format_spec.h>
+#include <__iterator/concepts.h>
+#include <__memory/allocator.h>
+#include <__system_error/errc.h>
+#include <__type_traits/conditional.h>
+#include <__utility/move.h>
+#include <__utility/unreachable.h>
+#include <cmath>
+#include <cstddef>
+
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#  include <__locale>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __formatter {
+
+template <floating_point _Tp>
+_LIBCPP_HIDE_FROM_ABI char* __to_buffer(char* __first, char* __last, _Tp __value) {
+  to_chars_result __r = std::to_chars(__first, __last, __value);
+  _LIBCPP_ASSERT_INTERNAL(__r.ec == errc(0), "Internal buffer too small");
+  return __r.ptr;
+}
+
+template <floating_point _Tp>
+_LIBCPP_HIDE_FROM_ABI char* __to_buffer(char* __first, char* __last, _Tp __value, chars_format __fmt) {
+  to_chars_result __r = std::to_chars(__first, __last, __value, __fmt);
+  _LIBCPP_ASSERT_INTERNAL(__r.ec == errc(0), "Internal buffer too small");
+  return __r.ptr;
+}
+
+template <floating_point _Tp>
+_LIBCPP_HIDE_FROM_ABI char* __to_buffer(char* __first, char* __last, _Tp __value, chars_format __fmt, int __precision) {
+  to_chars_result __r = std::to_chars(__first, __last, __value, __fmt, __precision);
+  _LIBCPP_ASSERT_INTERNAL(__r.ec == errc(0), "Internal buffer too small");
+  return __r.ptr;
+}
+
+// https://en.cppreference.com/w/cpp/language/types#cite_note-1
+// float             min subnormal: +/-0x1p-149   max: +/- 3.402,823,4 10^38
+// double            min subnormal: +/-0x1p-1074  max  +/- 1.797,693,134,862,315,7 10^308
+// long double (x86) min subnormal: +/-0x1p-16446 max: +/- 1.189,731,495,357,231,765,021 10^4932
+//
+// The maximum number of digits required for the integral part is based on the
+// maximum's value power of 10. Every power of 10 requires one additional
+// decimal digit.
+// The maximum number of digits required for the fractional part is based on
+// the minimal subnormal hexadecimal output's power of 10. Every division of a
+// fraction's binary 1 by 2, requires one additional decimal digit.
+//
+// The maximum size of a formatted value depends on the selected output format.
+// Ignoring the fact the format string can request a precision larger than the
+// values maximum required, these values are:
+//
+// sign                    1 code unit
+// __max_integral
+// radix point             1 code unit
+// __max_fractional
+// exponent character      1 code unit
+// sign                    1 code unit
+// __max_fractional_value
+// -----------------------------------
+// total                   4 code units extra required.
+//
+// TODO FMT Optimize the storage to avoid storing digits that are known to be zero.
+// https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/
+
+// TODO FMT Add long double specialization when to_chars has proper long double support.
+template <class _Tp>
+struct __traits;
+
+template <floating_point _Fp>
+_LIBCPP_HIDE_FROM_ABI constexpr size_t __float_buffer_size(int __precision) {
+  using _Traits = __traits<_Fp>;
+  return 4 + _Traits::__max_integral + __precision + _Traits::__max_fractional_value;
+}
+
+template <>
+struct __traits<float> {
+  static constexpr int __max_integral         = 38;
+  static constexpr int __max_fractional       = 149;
+  static constexpr int __max_fractional_value = 3;
+  static constexpr size_t __stack_buffer_size = 256;
+
+  static constexpr int __hex_precision_digits = 3;
+};
+
+template <>
+struct __traits<double> {
+  static constexpr int __max_integral         = 308;
+  static constexpr int __max_fractional       = 1074;
+  static constexpr int __max_fractional_value = 4;
+  static constexpr size_t __stack_buffer_size = 1024;
+
+  static constexpr int __hex_precision_digits = 4;
+};
+
+/// Helper class to store the conversion buffer.
+///
+/// Depending on the maximum size required for a value, the buffer is allocated
+/// on the stack or the heap.
+template <floating_point _Fp>
+class _LIBCPP_TEMPLATE_VIS __float_buffer {
+  using _Traits = __traits<_Fp>;
+
+public:
+  // TODO FMT Improve this constructor to do a better estimate.
+  // When using a scientific formatting with a precision of 6 a stack buffer
+  // will always suffice. At the moment that isn't important since floats and
+  // doubles use a stack buffer, unless the precision used in the format string
+  // is large.
+  // When supporting long doubles the __max_integral part becomes 4932 which
+  // may be too much for some platforms. For these cases a better estimate is
+  // required.
+  explicit _LIBCPP_HIDE_FROM_ABI __float_buffer(int __precision)
+      : __precision_(__precision != -1 ? __precision : _Traits::__max_fractional) {
+    // When the precision is larger than _Traits::__max_fractional the digits in
+    // the range (_Traits::__max_fractional, precision] will contain the value
+    // zero. There's no need to request to_chars to write these zeros:
+    // - When the value is large a temporary heap buffer needs to be allocated.
+    // - When to_chars writes the values they need to be "copied" to the output:
+    //   - char: std::fill on the output iterator is faster than std::copy.
+    //   - wchar_t: same argument as char, but additional std::copy won't work.
+    //     The input is always a char buffer, so every char in the buffer needs
+    //     to be converted from a char to a wchar_t.
+    if (__precision_ > _Traits::__max_fractional) {
+      __num_trailing_zeros_ = __precision_ - _Traits::__max_fractional;
+      __precision_          = _Traits::__max_fractional;
+    }
+
+    __size_ = __formatter::__float_buffer_size<_Fp>(__precision_);
+    if (__size_ > _Traits::__stack_buffer_size)
+      // The allocated buffer's contents don't need initialization.
+      __begin_ = allocator<char>{}.allocate(__size_);
+    else
+      __begin_ = __buffer_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~__float_buffer() {
+    if (__size_ > _Traits::__stack_buffer_size)
+      allocator<char>{}.deallocate(__begin_, __size_);
+  }
+  _LIBCPP_HIDE_FROM_ABI __float_buffer(const __float_buffer&)            = delete;
+  _LIBCPP_HIDE_FROM_ABI __float_buffer& operator=(const __float_buffer&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI char* begin() const { return __begin_; }
+  _LIBCPP_HIDE_FROM_ABI char* end() const { return __begin_ + __size_; }
+
+  _LIBCPP_HIDE_FROM_ABI int __precision() const { return __precision_; }
+  _LIBCPP_HIDE_FROM_ABI int __num_trailing_zeros() const { return __num_trailing_zeros_; }
+  _LIBCPP_HIDE_FROM_ABI void __remove_trailing_zeros() { __num_trailing_zeros_ = 0; }
+  _LIBCPP_HIDE_FROM_ABI void __add_trailing_zeros(int __zeros) { __num_trailing_zeros_ += __zeros; }
+
+private:
+  int __precision_;
+  int __num_trailing_zeros_{0};
+  size_t __size_;
+  char* __begin_;
+  char __buffer_[_Traits::__stack_buffer_size];
+};
+
+struct __float_result {
+  /// Points at the beginning of the integral part in the buffer.
+  ///
+  /// When there's no sign character this points at the start of the buffer.
+  char* __integral;
+
+  /// Points at the radix point, when not present it's the same as \ref __last.
+  char* __radix_point;
+
+  /// Points at the exponent character, when not present it's the same as \ref __last.
+  char* __exponent;
+
+  /// Points beyond the last written element in the buffer.
+  char* __last;
+};
+
+/// Finds the position of the exponent character 'e' at the end of the buffer.
+///
+/// Assuming there is an exponent the input will terminate with
+/// eSdd and eSdddd (S = sign, d = digit)
+///
+/// \returns a pointer to the exponent or __last when not found.
+constexpr inline _LIBCPP_HIDE_FROM_ABI char* __find_exponent(char* __first, char* __last) {
+  ptrdiff_t __size = __last - __first;
+  if (__size >= 4) {
+    __first = __last - std::min(__size, ptrdiff_t(6));
+    for (; __first != __last - 3; ++__first) {
+      if (*__first == 'e')
+        return __first;
+    }
+  }
+  return __last;
+}
+
+template <class _Fp, class _Tp>
+_LIBCPP_HIDE_FROM_ABI __float_result
+__format_buffer_default(const __float_buffer<_Fp>& __buffer, _Tp __value, char* __integral) {
+  __float_result __result;
+  __result.__integral = __integral;
+  __result.__last     = __formatter::__to_buffer(__integral, __buffer.end(), __value);
+
+  __result.__exponent = __formatter::__find_exponent(__result.__integral, __result.__last);
+
+  // Constrains:
+  // - There's at least one decimal digit before the radix point.
+  // - The radix point, when present, is placed before the exponent.
+  __result.__radix_point = std::find(__result.__integral + 1, __result.__exponent, '.');
+
+  // When the radix point isn't found its position is the exponent instead of
+  // __result.__last.
+  if (__result.__radix_point == __result.__exponent)
+    __result.__radix_point = __result.__last;
+
+  // clang-format off
+  _LIBCPP_ASSERT_INTERNAL((__result.__integral != __result.__last) &&
+                          (__result.__radix_point == __result.__last || *__result.__radix_point == '.') &&
+                          (__result.__exponent == __result.__last || *__result.__exponent == 'e'),
+                          "Post-condition failure.");
+  // clang-format on
+
+  return __result;
+}
+
+template <class _Fp, class _Tp>
+_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_hexadecimal_lower_case(
+    const __float_buffer<_Fp>& __buffer, _Tp __value, int __precision, char* __integral) {
+  __float_result __result;
+  __result.__integral = __integral;
+  if (__precision == -1)
+    __result.__last = __formatter::__to_buffer(__integral, __buffer.end(), __value, chars_format::hex);
+  else
+    __result.__last = __formatter::__to_buffer(__integral, __buffer.end(), __value, chars_format::hex, __precision);
+
+  // H = one or more hex-digits
+  // S = sign
+  // D = one or more decimal-digits
+  // When the fractional part is zero and no precision the output is 0p+0
+  // else the output is                                              0.HpSD
+  // So testing the second position can differentiate between these two cases.
+  char* __first = __integral + 1;
+  if (*__first == '.') {
+    __result.__radix_point = __first;
+    // One digit is the minimum
+    // 0.hpSd
+    //       ^-- last
+    //     ^---- integral = end of search
+    // ^-------- start of search
+    // 0123456
+    //
+    // Four digits is the maximum
+    // 0.hpSdddd
+    //          ^-- last
+    //        ^---- integral = end of search
+    //    ^-------- start of search
+    // 0123456789
+    static_assert(__traits<_Fp>::__hex_precision_digits <= 4, "Guard against possible underflow.");
+
+    char* __last        = __result.__last - 2;
+    __first             = __last - __traits<_Fp>::__hex_precision_digits;
+    __result.__exponent = std::find(__first, __last, 'p');
+  } else {
+    __result.__radix_point = __result.__last;
+    __result.__exponent    = __first;
+  }
+
+  // clang-format off
+  _LIBCPP_ASSERT_INTERNAL((__result.__integral != __result.__last) &&
+                          (__result.__radix_point == __result.__last || *__result.__radix_point == '.') &&
+                          (__result.__exponent != __result.__last && *__result.__exponent == 'p'),
+                          "Post-condition failure.");
+  // clang-format on
+
+  return __result;
+}
+
+template <class _Fp, class _Tp>
+_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_hexadecimal_upper_case(
+    const __float_buffer<_Fp>& __buffer, _Tp __value, int __precision, char* __integral) {
+  __float_result __result =
+      __formatter::__format_buffer_hexadecimal_lower_case(__buffer, __value, __precision, __integral);
+  std::transform(__result.__integral, __result.__exponent, __result.__integral, __hex_to_upper);
+  *__result.__exponent = 'P';
+  return __result;
+}
+
+template <class _Fp, class _Tp>
+_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_scientific_lower_case(
+    const __float_buffer<_Fp>& __buffer, _Tp __value, int __precision, char* __integral) {
+  __float_result __result;
+  __result.__integral = __integral;
+  __result.__last =
+      __formatter::__to_buffer(__integral, __buffer.end(), __value, chars_format::scientific, __precision);
+
+  char* __first = __integral + 1;
+  _LIBCPP_ASSERT_INTERNAL(__first != __result.__last, "No exponent present");
+  if (*__first == '.') {
+    __result.__radix_point = __first;
+    __result.__exponent    = __formatter::__find_exponent(__first + 1, __result.__last);
+  } else {
+    __result.__radix_point = __result.__last;
+    __result.__exponent    = __first;
+  }
+
+  // clang-format off
+  _LIBCPP_ASSERT_INTERNAL((__result.__integral != __result.__last) &&
+                          (__result.__radix_point == __result.__last || *__result.__radix_point == '.') &&
+                          (__result.__exponent != __result.__last && *__result.__exponent == 'e'),
+                          "Post-condition failure.");
+  // clang-format on
+  return __result;
+}
+
+template <class _Fp, class _Tp>
+_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_scientific_upper_case(
+    const __float_buffer<_Fp>& __buffer, _Tp __value, int __precision, char* __integral) {
+  __float_result __result =
+      __formatter::__format_buffer_scientific_lower_case(__buffer, __value, __precision, __integral);
+  *__result.__exponent = 'E';
+  return __result;
+}
+
+template <class _Fp, class _Tp>
+_LIBCPP_HIDE_FROM_ABI __float_result
+__format_buffer_fixed(const __float_buffer<_Fp>& __buffer, _Tp __value, int __precision, char* __integral) {
+  __float_result __result;
+  __result.__integral = __integral;
+  __result.__last     = __formatter::__to_buffer(__integral, __buffer.end(), __value, chars_format::fixed, __precision);
+
+  // When there's no precision there's no radix point.
+  // Else the radix point is placed at __precision + 1 from the end.
+  // By converting __precision to a bool the subtraction can be done
+  // unconditionally.
+  __result.__radix_point = __result.__last - (__precision + bool(__precision));
+  __result.__exponent    = __result.__last;
+
+  // clang-format off
+  _LIBCPP_ASSERT_INTERNAL((__result.__integral != __result.__last) &&
+                          (__result.__radix_point == __result.__last || *__result.__radix_point == '.') &&
+                          (__result.__exponent == __result.__last),
+                          "Post-condition failure.");
+  // clang-format on
+  return __result;
+}
+
+template <class _Fp, class _Tp>
+_LIBCPP_HIDE_FROM_ABI __float_result
+__format_buffer_general_lower_case(__float_buffer<_Fp>& __buffer, _Tp __value, int __precision, char* __integral) {
+  __buffer.__remove_trailing_zeros();
+
+  __float_result __result;
+  __result.__integral = __integral;
+  __result.__last = __formatter::__to_buffer(__integral, __buffer.end(), __value, chars_format::general, __precision);
+
+  char* __first = __integral + 1;
+  if (__first == __result.__last) {
+    __result.__radix_point = __result.__last;
+    __result.__exponent    = __result.__last;
+  } else {
+    __result.__exponent = __formatter::__find_exponent(__first, __result.__last);
+    if (__result.__exponent != __result.__last)
+      // In scientific mode if there's a radix point it will always be after
+      // the first digit. (This is the position __first points at).
+      __result.__radix_point = *__first == '.' ? __first : __result.__last;
+    else {
+      // In fixed mode the algorithm truncates trailing spaces and possibly the
+      // radix point. There's no good guess for the position of the radix point
+      // therefore scan the output after the first digit.
+      __result.__radix_point = std::find(__first, __result.__last, '.');
+    }
+  }
+
+  // clang-format off
+  _LIBCPP_ASSERT_INTERNAL((__result.__integral != __result.__last) &&
+                          (__result.__radix_point == __result.__last || *__result.__radix_point == '.') &&
+                          (__result.__exponent == __result.__last || *__result.__exponent == 'e'),
+                          "Post-condition failure.");
+  // clang-format on
+
+  return __result;
+}
+
+template <class _Fp, class _Tp>
+_LIBCPP_HIDE_FROM_ABI __float_result
+__format_buffer_general_upper_case(__float_buffer<_Fp>& __buffer, _Tp __value, int __precision, char* __integral) {
+  __float_result __result = __formatter::__format_buffer_general_lower_case(__buffer, __value, __precision, __integral);
+  if (__result.__exponent != __result.__last)
+    *__result.__exponent = 'E';
+  return __result;
+}
+
+/// Fills the buffer with the data based on the requested formatting.
+///
+/// This function, when needed, turns the characters to upper case and
+/// determines the "interesting" locations which are returned to the caller.
+///
+/// This means the caller never has to convert the contents of the buffer to
+/// upper case or search for radix points and the location of the exponent.
+/// This gives a bit of overhead. The original code didn't do that, but due
+/// to the number of possible additional work needed to turn this number to
+/// the proper output the code was littered with tests for upper cases and
+/// searches for radix points and exponents.
+/// - When a precision larger than the type's precision is selected
+///   additional zero characters need to be written before the exponent.
+/// - alternate form needs to add a radix point when not present.
+/// - localization needs to do grouping in the integral part.
+template <class _Fp, class _Tp>
+// TODO FMT _Fp should just be _Tp when to_chars has proper long double support.
+_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer(
+    __float_buffer<_Fp>& __buffer,
+    _Tp __value,
+    bool __negative,
+    bool __has_precision,
+    __format_spec::__sign __sign,
+    __format_spec::__type __type) {
+  char* __first = __formatter::__insert_sign(__buffer.begin(), __negative, __sign);
+  switch (__type) {
+  case __format_spec::__type::__default:
+    if (__has_precision)
+      return __formatter::__format_buffer_general_lower_case(__buffer, __value, __buffer.__precision(), __first);
+    else
+      return __formatter::__format_buffer_default(__buffer, __value, __first);
+
+  case __format_spec::__type::__hexfloat_lower_case:
+    return __formatter::__format_buffer_hexadecimal_lower_case(
+        __buffer, __value, __has_precision ? __buffer.__precision() : -1, __first);
+
+  case __format_spec::__type::__hexfloat_upper_case:
+    return __formatter::__format_buffer_hexadecimal_upper_case(
+        __buffer, __value, __has_precision ? __buffer.__precision() : -1, __first);
+
+  case __format_spec::__type::__scientific_lower_case:
+    return __formatter::__format_buffer_scientific_lower_case(__buffer, __value, __buffer.__precision(), __first);
+
+  case __format_spec::__type::__scientific_upper_case:
+    return __formatter::__format_buffer_scientific_upper_case(__buffer, __value, __buffer.__precision(), __first);
+
+  case __format_spec::__type::__fixed_lower_case:
+  case __format_spec::__type::__fixed_upper_case:
+    return __formatter::__format_buffer_fixed(__buffer, __value, __buffer.__precision(), __first);
+
+  case __format_spec::__type::__general_lower_case:
+    return __formatter::__format_buffer_general_lower_case(__buffer, __value, __buffer.__precision(), __first);
+
+  case __format_spec::__type::__general_upper_case:
+    return __formatter::__format_buffer_general_upper_case(__buffer, __value, __buffer.__precision(), __first);
+
+  default:
+    _LIBCPP_ASSERT_INTERNAL(false, "The parser should have validated the type");
+    __libcpp_unreachable();
+  }
+}
+
+#  ifndef _LIBCPP_HAS_NO_LOCALIZATION
+template <class _OutIt, class _Fp, class _CharT>
+_LIBCPP_HIDE_FROM_ABI _OutIt __format_locale_specific_form(
+    _OutIt __out_it,
+    const __float_buffer<_Fp>& __buffer,
+    const __float_result& __result,
+    std::locale __loc,
+    __format_spec::__parsed_specifications<_CharT> __specs) {
+  const auto& __np  = std::use_facet<numpunct<_CharT>>(__loc);
+  string __grouping = __np.grouping();
+  char* __first     = __result.__integral;
+  // When no radix point or exponent are present __last will be __result.__last.
+  char* __last = std::min(__result.__radix_point, __result.__exponent);
+
+  ptrdiff_t __digits = __last - __first;
+  if (!__grouping.empty()) {
+    if (__digits <= __grouping[0])
+      __grouping.clear();
+    else
+      __grouping = __formatter::__determine_grouping(__digits, __grouping);
+  }
+
+  ptrdiff_t __size =
+      __result.__last - __buffer.begin() + // Formatted string
+      __buffer.__num_trailing_zeros() +    // Not yet rendered zeros
+      __grouping.size() -                  // Grouping contains one
+      !__grouping.empty();                 // additional character
+
+  __formatter::__padding_size_result __padding = {0, 0};
+  bool __zero_padding                          = __specs.__alignment_ == __format_spec::__alignment::__zero_padding;
+  if (__size < __specs.__width_) {
+    if (__zero_padding) {
+      __specs.__alignment_      = __format_spec::__alignment::__right;
+      __specs.__fill_.__data[0] = _CharT('0');
+    }
+
+    __padding = __formatter::__padding_size(__size, __specs.__width_, __specs.__alignment_);
+  }
+
+  // sign and (zero padding or alignment)
+  if (__zero_padding && __first != __buffer.begin())
+    *__out_it++ = *__buffer.begin();
+  __out_it = __formatter::__fill(std::move(__out_it), __padding.__before_, __specs.__fill_);
+  if (!__zero_padding && __first != __buffer.begin())
+    *__out_it++ = *__buffer.begin();
+
+  // integral part
+  if (__grouping.empty()) {
+    __out_it = __formatter::__copy(__first, __digits, std::move(__out_it));
+  } else {
+    auto __r     = __grouping.rbegin();
+    auto __e     = __grouping.rend() - 1;
+    _CharT __sep = __np.thousands_sep();
+    // The output is divided in small groups of numbers to write:
+    // - A group before the first separator.
+    // - A separator and a group, repeated for the number of separators.
+    // - A group after the last separator.
+    // This loop achieves that process by testing the termination condition
+    // midway in the loop.
+    while (true) {
+      __out_it = __formatter::__copy(__first, *__r, std::move(__out_it));
+      __first += *__r;
+
+      if (__r == __e)
+        break;
+
+      ++__r;
+      *__out_it++ = __sep;
+    }
+  }
+
+  // fractional part
+  if (__result.__radix_point != __result.__last) {
+    *__out_it++ = __np.decimal_point();
+    __out_it    = __formatter::__copy(__result.__radix_point + 1, __result.__exponent, std::move(__out_it));
+    __out_it    = __formatter::__fill(std::move(__out_it), __buffer.__num_trailing_zeros(), _CharT('0'));
+  }
+
+  // exponent
+  if (__result.__exponent != __result.__last)
+    __out_it = __formatter::__copy(__result.__exponent, __result.__last, std::move(__out_it));
+
+  // alignment
+  return __formatter::__fill(std::move(__out_it), __padding.__after_, __specs.__fill_);
+}
+#  endif // _LIBCPP_HAS_NO_LOCALIZATION
+
+template <class _OutIt, class _CharT>
+_LIBCPP_HIDE_FROM_ABI _OutIt __format_floating_point_non_finite(
+    _OutIt __out_it, __format_spec::__parsed_specifications<_CharT> __specs, bool __negative, bool __isnan) {
+  char __buffer[4];
+  char* __last = __formatter::__insert_sign(__buffer, __negative, __specs.__std_.__sign_);
+
+  // to_chars can return inf, infinity, nan, and nan(n-char-sequence).
+  // The format library requires inf and nan.
+  // All in one expression to avoid dangling references.
+  bool __upper_case =
+      __specs.__std_.__type_ == __format_spec::__type::__hexfloat_upper_case ||
+      __specs.__std_.__type_ == __format_spec::__type::__scientific_upper_case ||
+      __specs.__std_.__type_ == __format_spec::__type::__fixed_upper_case ||
+      __specs.__std_.__type_ == __format_spec::__type::__general_upper_case;
+  __last = std::copy_n(&("infnanINFNAN"[6 * __upper_case + 3 * __isnan]), 3, __last);
+
+  // [format.string.std]/13
+  // A zero (0) character preceding the width field pads the field with
+  // leading zeros (following any indication of sign or base) to the field
+  // width, except when applied to an infinity or NaN.
+  if (__specs.__alignment_ == __format_spec::__alignment::__zero_padding)
+    __specs.__alignment_ = __format_spec::__alignment::__right;
+
+  return __formatter::__write(__buffer, __last, std::move(__out_it), __specs);
+}
+
+/// Writes additional zero's for the precision before the exponent.
+/// This is used when the precision requested in the format string is larger
+/// than the maximum precision of the floating-point type. These precision
+/// digits are always 0.
+///
+/// \param __exponent           The location of the exponent character.
+/// \param __num_trailing_zeros The number of 0's to write before the exponent
+///                             character.
+template <class _CharT, class _ParserCharT>
+_LIBCPP_HIDE_FROM_ABI auto __write_using_trailing_zeros(
+    const _CharT* __first,
+    const _CharT* __last,
+    output_iterator<const _CharT&> auto __out_it,
+    __format_spec::__parsed_specifications<_ParserCharT> __specs,
+    size_t __size,
+    const _CharT* __exponent,
+    size_t __num_trailing_zeros) -> decltype(__out_it) {
+  _LIBCPP_ASSERT_INTERNAL(__first <= __last, "Not a valid range");
+  _LIBCPP_ASSERT_INTERNAL(__num_trailing_zeros > 0, "The overload not writing trailing zeros should have been used");
+
+  __padding_size_result __padding =
+      __formatter::__padding_size(__size + __num_trailing_zeros, __specs.__width_, __specs.__alignment_);
+  __out_it = __formatter::__fill(std::move(__out_it), __padding.__before_, __specs.__fill_);
+  __out_it = __formatter::__copy(__first, __exponent, std::move(__out_it));
+  __out_it = __formatter::__fill(std::move(__out_it), __num_trailing_zeros, _CharT('0'));
+  __out_it = __formatter::__copy(__exponent, __last, std::move(__out_it));
+  return __formatter::__fill(std::move(__out_it), __padding.__after_, __specs.__fill_);
+}
+
+template <floating_point _Tp, class _CharT, class _FormatContext>
+_LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator
+__format_floating_point(_Tp __value, _FormatContext& __ctx, __format_spec::__parsed_specifications<_CharT> __specs) {
+  bool __negative = std::signbit(__value);
+
+  if (!std::isfinite(__value)) [[unlikely]]
+    return __formatter::__format_floating_point_non_finite(__ctx.out(), __specs, __negative, std::isnan(__value));
+
+  // Depending on the std-format-spec string the sign and the value
+  // might not be outputted together:
+  // - zero-padding may insert additional '0' characters.
+  // Therefore the value is processed as a non negative value.
+  // The function @ref __insert_sign will insert a '-' when the value was
+  // negative.
+
+  if (__negative)
+    __value = -__value;
+
+  // TODO FMT _Fp should just be _Tp when to_chars has proper long double support.
+  using _Fp = conditional_t<same_as<_Tp, long double>, double, _Tp>;
+  // Force the type of the precision to avoid -1 to become an unsigned value.
+  __float_buffer<_Fp> __buffer(__specs.__precision_);
+  __float_result __result = __formatter::__format_buffer(
+      __buffer, __value, __negative, (__specs.__has_precision()), __specs.__std_.__sign_, __specs.__std_.__type_);
+
+  if (__specs.__std_.__alternate_form_) {
+    if (__result.__radix_point == __result.__last) {
+      *__result.__last++ = '.';
+
+      // When there is an exponent the point needs to be moved before the
+      // exponent. When there's no exponent the rotate does nothing. Since
+      // rotate tests whether the operation is a nop, call it unconditionally.
+      std::rotate(__result.__exponent, __result.__last - 1, __result.__last);
+      __result.__radix_point = __result.__exponent;
+
+      // The radix point is always placed before the exponent.
+      // - No exponent needs to point to the new last.
+      // - An exponent needs to move one position to the right.
+      // So it's safe to increment the value unconditionally.
+      ++__result.__exponent;
+    }
+
+    // [format.string.std]/6
+    //   In addition, for g and G conversions, trailing zeros are not removed
+    //   from the result.
+    //
+    // If the type option for a floating-point type is none it may use the
+    // general formatting, but it's not a g or G conversion. So in that case
+    // the formatting should not append trailing zeros.
+    bool __is_general = __specs.__std_.__type_ == __format_spec::__type::__general_lower_case ||
+                        __specs.__std_.__type_ == __format_spec::__type::__general_upper_case;
+
+    if (__is_general) {
+      // https://en.cppreference.com/w/c/io/fprintf
+      // Let P equal the precision if nonzero, 6 if the precision is not
+      // specified, or 1 if the precision is 0. Then, if a conversion with
+      // style E would have an exponent of X:
+      int __p = std::max<int>(1, (__specs.__has_precision() ? __specs.__precision_ : 6));
+      if (__result.__exponent == __result.__last)
+        // if P > X >= -4, the conversion is with style f or F and precision P - 1 - X.
+        // By including the radix point it calculates P - (1 + X)
+        __p -= __result.__radix_point - __result.__integral;
+      else
+        // otherwise, the conversion is with style e or E and precision P - 1.
+        --__p;
+
+      ptrdiff_t __precision = (__result.__exponent - __result.__radix_point) - 1;
+      if (__precision < __p)
+        __buffer.__add_trailing_zeros(__p - __precision);
+    }
+  }
+
+#  ifndef _LIBCPP_HAS_NO_LOCALIZATION
+  if (__specs.__std_.__locale_specific_form_)
+    return __formatter::__format_locale_specific_form(__ctx.out(), __buffer, __result, __ctx.locale(), __specs);
+#  endif
+
+  ptrdiff_t __size         = __result.__last - __buffer.begin();
+  int __num_trailing_zeros = __buffer.__num_trailing_zeros();
+  if (__size + __num_trailing_zeros >= __specs.__width_) {
+    if (__num_trailing_zeros && __result.__exponent != __result.__last)
+      // Insert trailing zeros before exponent character.
+      return __formatter::__copy(
+          __result.__exponent,
+          __result.__last,
+          __formatter::__fill(__formatter::__copy(__buffer.begin(), __result.__exponent, __ctx.out()),
+                              __num_trailing_zeros,
+                              _CharT('0')));
+
+    return __formatter::__fill(
+        __formatter::__copy(__buffer.begin(), __result.__last, __ctx.out()), __num_trailing_zeros, _CharT('0'));
+  }
+
+  auto __out_it = __ctx.out();
+  char* __first = __buffer.begin();
+  if (__specs.__alignment_ == __format_spec::__alignment ::__zero_padding) {
+    // When there is a sign output it before the padding. Note the __size
+    // doesn't need any adjustment, regardless whether the sign is written
+    // here or in __formatter::__write.
+    if (__first != __result.__integral)
+      *__out_it++ = *__first++;
+    // After the sign is written, zero padding is the same a right alignment
+    // with '0'.
+    __specs.__alignment_      = __format_spec::__alignment::__right;
+    __specs.__fill_.__data[0] = _CharT('0');
+  }
+
+  if (__num_trailing_zeros)
+    return __formatter::__write_using_trailing_zeros(
+        __first, __result.__last, std::move(__out_it), __specs, __size, __result.__exponent, __num_trailing_zeros);
+
+  return __formatter::__write(__first, __result.__last, std::move(__out_it), __specs, __size);
+}
+
+} // namespace __formatter
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS __formatter_floating_point {
+public:
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    typename _ParseContext::iterator __result = __parser_.__parse(__ctx, __format_spec::__fields_floating_point);
+    __format_spec::__process_parsed_floating_point(__parser_, "a floating-point");
+    return __result;
+  }
+
+  template <floating_point _Tp, class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(_Tp __value, _FormatContext& __ctx) const {
+    return __formatter::__format_floating_point(__value, __ctx, __parser_.__get_parsed_std_specifications(__ctx));
+  }
+
+  __format_spec::__parser<_CharT> __parser_;
+};
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<float, _CharT> : public __formatter_floating_point<_CharT> {};
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<double, _CharT> : public __formatter_floating_point<_CharT> {};
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<long double, _CharT> : public __formatter_floating_point<_CharT> {};
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FORMAT_FORMATTER_FLOATING_POINT_H
diff --git a/libcxx/include/__cxx03/__format/formatter_integer.h b/libcxx/include/__cxx03/__format/formatter_integer.h
new file mode 100644
index 000000000000..41400f00478e
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/formatter_integer.h
@@ -0,0 +1,95 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMATTER_INTEGER_H
+#define _LIBCPP___FORMAT_FORMATTER_INTEGER_H
+
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <__format/concepts.h>
+#include <__format/format_parse_context.h>
+#include <__format/formatter.h>
+#include <__format/formatter_integral.h>
+#include <__format/formatter_output.h>
+#include <__format/parser_std_format_spec.h>
+#include <__type_traits/is_void.h>
+#include <__type_traits/make_32_64_or_128_bit.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS __formatter_integer {
+public:
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    typename _ParseContext::iterator __result = __parser_.__parse(__ctx, __format_spec::__fields_integral);
+    __format_spec::__process_parsed_integer(__parser_, "an integer");
+    return __result;
+  }
+
+  template <integral _Tp, class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(_Tp __value, _FormatContext& __ctx) const {
+    __format_spec::__parsed_specifications<_CharT> __specs = __parser_.__get_parsed_std_specifications(__ctx);
+
+    if (__specs.__std_.__type_ == __format_spec::__type::__char)
+      return __formatter::__format_char(__value, __ctx.out(), __specs);
+
+    using _Type = __make_32_64_or_128_bit_t<_Tp>;
+    static_assert(!is_void<_Type>::value, "unsupported integral type used in __formatter_integer::__format");
+
+    // Reduce the number of instantiation of the integer formatter
+    return __formatter::__format_integer(static_cast<_Type>(__value), __ctx, __specs);
+  }
+
+  __format_spec::__parser<_CharT> __parser_;
+};
+
+// Signed integral types.
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<signed char, _CharT> : public __formatter_integer<_CharT> {};
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<short, _CharT> : public __formatter_integer<_CharT> {};
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<int, _CharT> : public __formatter_integer<_CharT> {};
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<long, _CharT> : public __formatter_integer<_CharT> {};
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<long long, _CharT> : public __formatter_integer<_CharT> {};
+#  ifndef _LIBCPP_HAS_NO_INT128
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<__int128_t, _CharT> : public __formatter_integer<_CharT> {};
+#  endif
+
+// Unsigned integral types.
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<unsigned char, _CharT> : public __formatter_integer<_CharT> {};
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<unsigned short, _CharT> : public __formatter_integer<_CharT> {};
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<unsigned, _CharT> : public __formatter_integer<_CharT> {};
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<unsigned long, _CharT> : public __formatter_integer<_CharT> {};
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<unsigned long long, _CharT> : public __formatter_integer<_CharT> {};
+#  ifndef _LIBCPP_HAS_NO_INT128
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<__uint128_t, _CharT> : public __formatter_integer<_CharT> {};
+#  endif
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_FORMATTER_INTEGER_H
diff --git a/libcxx/include/__cxx03/__format/formatter_integral.h b/libcxx/include/__cxx03/__format/formatter_integral.h
new file mode 100644
index 000000000000..eca966f8886f
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/formatter_integral.h
@@ -0,0 +1,445 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMATTER_INTEGRAL_H
+#define _LIBCPP___FORMAT_FORMATTER_INTEGRAL_H
+
+#include <__charconv/to_chars_integral.h>
+#include <__charconv/to_chars_result.h>
+#include <__charconv/traits.h>
+#include <__concepts/arithmetic.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/concepts.h>
+#include <__format/format_error.h>
+#include <__format/formatter_output.h>
+#include <__format/parser_std_format_spec.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/pointer_traits.h>
+#include <__system_error/errc.h>
+#include <__type_traits/make_unsigned.h>
+#include <__utility/unreachable.h>
+#include <array>
+#include <limits>
+#include <string>
+#include <string_view>
+
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#  include <__locale>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __formatter {
+
+//
+// Generic
+//
+
+template <contiguous_iterator _Iterator>
+  requires same_as<char, iter_value_t<_Iterator>>
+_LIBCPP_HIDE_FROM_ABI inline _Iterator __insert_sign(_Iterator __buf, bool __negative, __format_spec::__sign __sign) {
+  if (__negative)
+    *__buf++ = '-';
+  else
+    switch (__sign) {
+    case __format_spec::__sign::__default:
+    case __format_spec::__sign::__minus:
+      // No sign added.
+      break;
+    case __format_spec::__sign::__plus:
+      *__buf++ = '+';
+      break;
+    case __format_spec::__sign::__space:
+      *__buf++ = ' ';
+      break;
+    }
+
+  return __buf;
+}
+
+/**
+ * Determines the required grouping based on the size of the input.
+ *
+ * The grouping's last element will be repeated. For simplicity this repeating
+ * is unwrapped based on the length of the input. (When the input is short some
+ * groups are not processed.)
+ *
+ * @returns The size of the groups to write. This means the number of
+ * separator characters written is size() - 1.
+ *
+ * @note Since zero-sized groups cause issues they are silently ignored.
+ *
+ * @note The grouping field of the locale is always a @c std::string,
+ * regardless whether the @c std::numpunct's type is @c char or @c wchar_t.
+ */
+_LIBCPP_HIDE_FROM_ABI inline string __determine_grouping(ptrdiff_t __size, const string& __grouping) {
+  _LIBCPP_ASSERT_INTERNAL(!__grouping.empty() && __size > __grouping[0],
+                          "The slow grouping formatting is used while there will be no separators written");
+  string __r;
+  auto __end = __grouping.end() - 1;
+  auto __ptr = __grouping.begin();
+
+  while (true) {
+    __size -= *__ptr;
+    if (__size > 0)
+      __r.push_back(*__ptr);
+    else {
+      // __size <= 0 so the value pushed will be <= *__ptr.
+      __r.push_back(*__ptr + __size);
+      return __r;
+    }
+
+    // Proceed to the next group.
+    if (__ptr != __end) {
+      do {
+        ++__ptr;
+        // Skip grouping with a width of 0.
+      } while (*__ptr == 0 && __ptr != __end);
+    }
+  }
+
+  __libcpp_unreachable();
+}
+
+//
+// Char
+//
+
+template <__fmt_char_type _CharT>
+_LIBCPP_HIDE_FROM_ABI auto
+__format_char(integral auto __value,
+              output_iterator<const _CharT&> auto __out_it,
+              __format_spec::__parsed_specifications<_CharT> __specs) -> decltype(__out_it) {
+  using _Tp = decltype(__value);
+  if constexpr (!same_as<_CharT, _Tp>) {
+    // cmp_less and cmp_greater can't be used for character types.
+    if constexpr (signed_integral<_CharT> == signed_integral<_Tp>) {
+      if (__value < numeric_limits<_CharT>::min() || __value > numeric_limits<_CharT>::max())
+        std::__throw_format_error("Integral value outside the range of the char type");
+    } else if constexpr (signed_integral<_CharT>) {
+      // _CharT is signed _Tp is unsigned
+      if (__value > static_cast<make_unsigned_t<_CharT>>(numeric_limits<_CharT>::max()))
+        std::__throw_format_error("Integral value outside the range of the char type");
+    } else {
+      // _CharT is unsigned _Tp is signed
+      if (__value < 0 || static_cast<make_unsigned_t<_Tp>>(__value) > numeric_limits<_CharT>::max())
+        std::__throw_format_error("Integral value outside the range of the char type");
+    }
+  }
+
+  const auto __c = static_cast<_CharT>(__value);
+  return __formatter::__write(std::addressof(__c), std::addressof(__c) + 1, std::move(__out_it), __specs);
+}
+
+//
+// Integer
+//
+
+/** Wrapper around @ref to_chars, returning the output iterator. */
+template <contiguous_iterator _Iterator, integral _Tp>
+  requires same_as<char, iter_value_t<_Iterator>>
+_LIBCPP_HIDE_FROM_ABI _Iterator __to_buffer(_Iterator __first, _Iterator __last, _Tp __value, int __base) {
+  // TODO FMT Evaluate code overhead due to not calling the internal function
+  // directly. (Should be zero overhead.)
+  to_chars_result __r = std::to_chars(std::to_address(__first), std::to_address(__last), __value, __base);
+  _LIBCPP_ASSERT_INTERNAL(__r.ec == errc(0), "Internal buffer too small");
+  auto __diff = __r.ptr - std::to_address(__first);
+  return __first + __diff;
+}
+
+/**
+ * Helper to determine the buffer size to output a integer in Base @em x.
+ *
+ * There are several overloads for the supported bases. The function uses the
+ * base as template argument so it can be used in a constant expression.
+ */
+template <unsigned_integral _Tp, size_t _Base>
+consteval size_t __buffer_size() noexcept
+  requires(_Base == 2)
+{
+  return numeric_limits<_Tp>::digits // The number of binary digits.
+       + 2                           // Reserve space for the '0[Bb]' prefix.
+       + 1;                          // Reserve space for the sign.
+}
+
+template <unsigned_integral _Tp, size_t _Base>
+consteval size_t __buffer_size() noexcept
+  requires(_Base == 8)
+{
+  return numeric_limits<_Tp>::digits // The number of binary digits.
+           / 3                       // Adjust to octal.
+       + 1                           // Turn floor to ceil.
+       + 1                           // Reserve space for the '0' prefix.
+       + 1;                          // Reserve space for the sign.
+}
+
+template <unsigned_integral _Tp, size_t _Base>
+consteval size_t __buffer_size() noexcept
+  requires(_Base == 10)
+{
+  return numeric_limits<_Tp>::digits10 // The floored value.
+       + 1                             // Turn floor to ceil.
+       + 1;                            // Reserve space for the sign.
+}
+
+template <unsigned_integral _Tp, size_t _Base>
+consteval size_t __buffer_size() noexcept
+  requires(_Base == 16)
+{
+  return numeric_limits<_Tp>::digits // The number of binary digits.
+           / 4                       // Adjust to hexadecimal.
+       + 2                           // Reserve space for the '0[Xx]' prefix.
+       + 1;                          // Reserve space for the sign.
+}
+
+template <class _OutIt, contiguous_iterator _Iterator, class _CharT>
+  requires same_as<char, iter_value_t<_Iterator>>
+_LIBCPP_HIDE_FROM_ABI _OutIt __write_using_decimal_separators(
+    _OutIt __out_it,
+    _Iterator __begin,
+    _Iterator __first,
+    _Iterator __last,
+    string&& __grouping,
+    _CharT __sep,
+    __format_spec::__parsed_specifications<_CharT> __specs) {
+  int __size = (__first - __begin) +    // [sign][prefix]
+               (__last - __first) +     // data
+               (__grouping.size() - 1); // number of separator characters
+
+  __padding_size_result __padding = {0, 0};
+  if (__specs.__alignment_ == __format_spec::__alignment::__zero_padding) {
+    // Write [sign][prefix].
+    __out_it = __formatter::__copy(__begin, __first, std::move(__out_it));
+
+    if (__specs.__width_ > __size) {
+      // Write zero padding.
+      __padding.__before_ = __specs.__width_ - __size;
+      __out_it            = __formatter::__fill(std::move(__out_it), __specs.__width_ - __size, _CharT('0'));
+    }
+  } else {
+    if (__specs.__width_ > __size) {
+      // Determine padding and write padding.
+      __padding = __formatter::__padding_size(__size, __specs.__width_, __specs.__alignment_);
+
+      __out_it = __formatter::__fill(std::move(__out_it), __padding.__before_, __specs.__fill_);
+    }
+    // Write [sign][prefix].
+    __out_it = __formatter::__copy(__begin, __first, std::move(__out_it));
+  }
+
+  auto __r = __grouping.rbegin();
+  auto __e = __grouping.rend() - 1;
+  _LIBCPP_ASSERT_INTERNAL(
+      __r != __e, "The slow grouping formatting is used while there will be no separators written.");
+  // The output is divided in small groups of numbers to write:
+  // - A group before the first separator.
+  // - A separator and a group, repeated for the number of separators.
+  // - A group after the last separator.
+  // This loop achieves that process by testing the termination condition
+  // midway in the loop.
+  //
+  // TODO FMT This loop evaluates the loop invariant `__parser.__type !=
+  // _Flags::_Type::__hexadecimal_upper_case` for every iteration. (This test
+  // happens in the __write call.) Benchmark whether making two loops and
+  // hoisting the invariant is worth the effort.
+  while (true) {
+    if (__specs.__std_.__type_ == __format_spec::__type::__hexadecimal_upper_case) {
+      __last   = __first + *__r;
+      __out_it = __formatter::__transform(__first, __last, std::move(__out_it), __hex_to_upper);
+      __first  = __last;
+    } else {
+      __out_it = __formatter::__copy(__first, *__r, std::move(__out_it));
+      __first += *__r;
+    }
+
+    if (__r == __e)
+      break;
+
+    ++__r;
+    *__out_it++ = __sep;
+  }
+
+  return __formatter::__fill(std::move(__out_it), __padding.__after_, __specs.__fill_);
+}
+
+template <unsigned_integral _Tp, contiguous_iterator _Iterator, class _CharT, class _FormatContext>
+  requires same_as<char, iter_value_t<_Iterator>>
+_LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator __format_integer(
+    _Tp __value,
+    _FormatContext& __ctx,
+    __format_spec::__parsed_specifications<_CharT> __specs,
+    bool __negative,
+    _Iterator __begin,
+    _Iterator __end,
+    const char* __prefix,
+    int __base) {
+  _Iterator __first = __formatter::__insert_sign(__begin, __negative, __specs.__std_.__sign_);
+  if (__specs.__std_.__alternate_form_ && __prefix)
+    while (*__prefix)
+      *__first++ = *__prefix++;
+
+  _Iterator __last = __formatter::__to_buffer(__first, __end, __value, __base);
+
+#  ifndef _LIBCPP_HAS_NO_LOCALIZATION
+  if (__specs.__std_.__locale_specific_form_) {
+    const auto& __np  = std::use_facet<numpunct<_CharT>>(__ctx.locale());
+    string __grouping = __np.grouping();
+    ptrdiff_t __size  = __last - __first;
+    // Writing the grouped form has more overhead than the normal output
+    // routines. If there will be no separators written the locale-specific
+    // form is identical to the normal routine. Test whether to grouped form
+    // is required.
+    if (!__grouping.empty() && __size > __grouping[0])
+      return __formatter::__write_using_decimal_separators(
+          __ctx.out(),
+          __begin,
+          __first,
+          __last,
+          __formatter::__determine_grouping(__size, __grouping),
+          __np.thousands_sep(),
+          __specs);
+  }
+#  endif
+  auto __out_it = __ctx.out();
+  if (__specs.__alignment_ != __format_spec::__alignment::__zero_padding)
+    __first = __begin;
+  else {
+    // __buf contains [sign][prefix]data
+    //                              ^ location of __first
+    // The zero padding is done like:
+    // - Write [sign][prefix]
+    // - Write data right aligned with '0' as fill character.
+    __out_it                  = __formatter::__copy(__begin, __first, std::move(__out_it));
+    __specs.__alignment_      = __format_spec::__alignment::__right;
+    __specs.__fill_.__data[0] = _CharT('0');
+    int32_t __size            = __first - __begin;
+
+    __specs.__width_ -= std::min(__size, __specs.__width_);
+  }
+
+  if (__specs.__std_.__type_ != __format_spec::__type::__hexadecimal_upper_case) [[likely]]
+    return __formatter::__write(__first, __last, __ctx.out(), __specs);
+
+  return __formatter::__write_transformed(__first, __last, __ctx.out(), __specs, __formatter::__hex_to_upper);
+}
+
+template <unsigned_integral _Tp, class _CharT, class _FormatContext>
+_LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator
+__format_integer(_Tp __value,
+                 _FormatContext& __ctx,
+                 __format_spec::__parsed_specifications<_CharT> __specs,
+                 bool __negative = false) {
+  switch (__specs.__std_.__type_) {
+  case __format_spec::__type::__binary_lower_case: {
+    array<char, __formatter::__buffer_size<decltype(__value), 2>()> __array;
+    return __formatter::__format_integer(__value, __ctx, __specs, __negative, __array.begin(), __array.end(), "0b", 2);
+  }
+  case __format_spec::__type::__binary_upper_case: {
+    array<char, __formatter::__buffer_size<decltype(__value), 2>()> __array;
+    return __formatter::__format_integer(__value, __ctx, __specs, __negative, __array.begin(), __array.end(), "0B", 2);
+  }
+  case __format_spec::__type::__octal: {
+    // Octal is special; if __value == 0 there's no prefix.
+    array<char, __formatter::__buffer_size<decltype(__value), 8>()> __array;
+    return __formatter::__format_integer(
+        __value, __ctx, __specs, __negative, __array.begin(), __array.end(), __value != 0 ? "0" : nullptr, 8);
+  }
+  case __format_spec::__type::__default:
+  case __format_spec::__type::__decimal: {
+    array<char, __formatter::__buffer_size<decltype(__value), 10>()> __array;
+    return __formatter::__format_integer(
+        __value, __ctx, __specs, __negative, __array.begin(), __array.end(), nullptr, 10);
+  }
+  case __format_spec::__type::__hexadecimal_lower_case: {
+    array<char, __formatter::__buffer_size<decltype(__value), 16>()> __array;
+    return __formatter::__format_integer(__value, __ctx, __specs, __negative, __array.begin(), __array.end(), "0x", 16);
+  }
+  case __format_spec::__type::__hexadecimal_upper_case: {
+    array<char, __formatter::__buffer_size<decltype(__value), 16>()> __array;
+    return __formatter::__format_integer(__value, __ctx, __specs, __negative, __array.begin(), __array.end(), "0X", 16);
+  }
+  default:
+    _LIBCPP_ASSERT_INTERNAL(false, "The parse function should have validated the type");
+    __libcpp_unreachable();
+  }
+}
+
+template <signed_integral _Tp, class _CharT, class _FormatContext>
+_LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator
+__format_integer(_Tp __value, _FormatContext& __ctx, __format_spec::__parsed_specifications<_CharT> __specs) {
+  // Depending on the std-format-spec string the sign and the value
+  // might not be outputted together:
+  // - alternate form may insert a prefix string.
+  // - zero-padding may insert additional '0' characters.
+  // Therefore the value is processed as a positive unsigned value.
+  // The function @ref __insert_sign will a '-' when the value was negative.
+  auto __r        = std::__to_unsigned_like(__value);
+  bool __negative = __value < 0;
+  if (__negative)
+    __r = std::__complement(__r);
+
+  return __formatter::__format_integer(__r, __ctx, __specs, __negative);
+}
+
+//
+// Formatter arithmetic (bool)
+//
+
+template <class _CharT>
+struct _LIBCPP_TEMPLATE_VIS __bool_strings;
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS __bool_strings<char> {
+  static constexpr string_view __true{"true"};
+  static constexpr string_view __false{"false"};
+};
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+struct _LIBCPP_TEMPLATE_VIS __bool_strings<wchar_t> {
+  static constexpr wstring_view __true{L"true"};
+  static constexpr wstring_view __false{L"false"};
+};
+#  endif
+
+template <class _CharT, class _FormatContext>
+_LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator
+__format_bool(bool __value, _FormatContext& __ctx, __format_spec::__parsed_specifications<_CharT> __specs) {
+#  ifndef _LIBCPP_HAS_NO_LOCALIZATION
+  if (__specs.__std_.__locale_specific_form_) {
+    const auto& __np           = std::use_facet<numpunct<_CharT>>(__ctx.locale());
+    basic_string<_CharT> __str = __value ? __np.truename() : __np.falsename();
+    return __formatter::__write_string_no_precision(basic_string_view<_CharT>{__str}, __ctx.out(), __specs);
+  }
+#  endif
+  basic_string_view<_CharT> __str =
+      __value ? __formatter::__bool_strings<_CharT>::__true : __formatter::__bool_strings<_CharT>::__false;
+  return __formatter::__write(__str.begin(), __str.end(), __ctx.out(), __specs);
+}
+
+} // namespace __formatter
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FORMAT_FORMATTER_INTEGRAL_H
diff --git a/libcxx/include/__cxx03/__format/formatter_output.h b/libcxx/include/__cxx03/__format/formatter_output.h
new file mode 100644
index 000000000000..1498f64c4aef
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/formatter_output.h
@@ -0,0 +1,335 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMATTER_OUTPUT_H
+#define _LIBCPP___FORMAT_FORMATTER_OUTPUT_H
+
+#include <__algorithm/ranges_copy.h>
+#include <__algorithm/ranges_fill_n.h>
+#include <__algorithm/ranges_transform.h>
+#include <__bit/countl.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/buffer.h>
+#include <__format/concepts.h>
+#include <__format/formatter.h>
+#include <__format/parser_std_format_spec.h>
+#include <__format/unicode.h>
+#include <__iterator/back_insert_iterator.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__memory/pointer_traits.h>
+#include <__utility/move.h>
+#include <__utility/unreachable.h>
+#include <cstddef>
+#include <string_view>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __formatter {
+
+_LIBCPP_HIDE_FROM_ABI constexpr char __hex_to_upper(char __c) {
+  switch (__c) {
+  case 'a':
+    return 'A';
+  case 'b':
+    return 'B';
+  case 'c':
+    return 'C';
+  case 'd':
+    return 'D';
+  case 'e':
+    return 'E';
+  case 'f':
+    return 'F';
+  }
+  return __c;
+}
+
+struct _LIBCPP_EXPORTED_FROM_ABI __padding_size_result {
+  size_t __before_;
+  size_t __after_;
+};
+
+_LIBCPP_HIDE_FROM_ABI constexpr __padding_size_result
+__padding_size(size_t __size, size_t __width, __format_spec::__alignment __align) {
+  _LIBCPP_ASSERT_INTERNAL(__width > __size, "don't call this function when no padding is required");
+  _LIBCPP_ASSERT_INTERNAL(
+      __align != __format_spec::__alignment::__zero_padding, "the caller should have handled the zero-padding");
+
+  size_t __fill = __width - __size;
+  switch (__align) {
+  case __format_spec::__alignment::__zero_padding:
+    __libcpp_unreachable();
+
+  case __format_spec::__alignment::__left:
+    return {0, __fill};
+
+  case __format_spec::__alignment::__center: {
+    // The extra padding is divided per [format.string.std]/3
+    // __before = floor(__fill, 2);
+    // __after = ceil(__fill, 2);
+    size_t __before = __fill / 2;
+    size_t __after  = __fill - __before;
+    return {__before, __after};
+  }
+  case __format_spec::__alignment::__default:
+  case __format_spec::__alignment::__right:
+    return {__fill, 0};
+  }
+  __libcpp_unreachable();
+}
+
+/// Copy wrapper.
+///
+/// This uses a "mass output function" of __format::__output_buffer when possible.
+template <__fmt_char_type _CharT, __fmt_char_type _OutCharT = _CharT>
+_LIBCPP_HIDE_FROM_ABI auto
+__copy(basic_string_view<_CharT> __str, output_iterator<const _OutCharT&> auto __out_it) -> decltype(__out_it) {
+  if constexpr (std::same_as<decltype(__out_it), std::back_insert_iterator<__format::__output_buffer<_OutCharT>>>) {
+    __out_it.__get_container()->__copy(__str);
+    return __out_it;
+  } else if constexpr (std::same_as<decltype(__out_it), typename __format::__retarget_buffer<_OutCharT>::__iterator>) {
+    __out_it.__buffer_->__copy(__str);
+    return __out_it;
+  } else {
+    return std::ranges::copy(__str, std::move(__out_it)).out;
+  }
+}
+
+template <contiguous_iterator _Iterator,
+          __fmt_char_type _CharT    = typename iterator_traits<_Iterator>::value_type,
+          __fmt_char_type _OutCharT = _CharT>
+_LIBCPP_HIDE_FROM_ABI auto
+__copy(_Iterator __first, _Iterator __last, output_iterator<const _OutCharT&> auto __out_it) -> decltype(__out_it) {
+  return __formatter::__copy(basic_string_view{__first, __last}, std::move(__out_it));
+}
+
+template <contiguous_iterator _Iterator,
+          __fmt_char_type _CharT    = typename iterator_traits<_Iterator>::value_type,
+          __fmt_char_type _OutCharT = _CharT>
+_LIBCPP_HIDE_FROM_ABI auto
+__copy(_Iterator __first, size_t __n, output_iterator<const _OutCharT&> auto __out_it) -> decltype(__out_it) {
+  return __formatter::__copy(basic_string_view{std::to_address(__first), __n}, std::move(__out_it));
+}
+
+/// Transform wrapper.
+///
+/// This uses a "mass output function" of __format::__output_buffer when possible.
+template <contiguous_iterator _Iterator,
+          __fmt_char_type _CharT    = typename iterator_traits<_Iterator>::value_type,
+          __fmt_char_type _OutCharT = _CharT,
+          class _UnaryOperation>
+_LIBCPP_HIDE_FROM_ABI auto
+__transform(_Iterator __first,
+            _Iterator __last,
+            output_iterator<const _OutCharT&> auto __out_it,
+            _UnaryOperation __operation) -> decltype(__out_it) {
+  if constexpr (std::same_as<decltype(__out_it), std::back_insert_iterator<__format::__output_buffer<_OutCharT>>>) {
+    __out_it.__get_container()->__transform(__first, __last, std::move(__operation));
+    return __out_it;
+  } else if constexpr (std::same_as<decltype(__out_it), typename __format::__retarget_buffer<_OutCharT>::__iterator>) {
+    __out_it.__buffer_->__transform(__first, __last, std::move(__operation));
+    return __out_it;
+  } else {
+    return std::ranges::transform(__first, __last, std::move(__out_it), __operation).out;
+  }
+}
+
+/// Fill wrapper.
+///
+/// This uses a "mass output function" of __format::__output_buffer when possible.
+template <__fmt_char_type _CharT, output_iterator<const _CharT&> _OutIt>
+_LIBCPP_HIDE_FROM_ABI _OutIt __fill(_OutIt __out_it, size_t __n, _CharT __value) {
+  if constexpr (std::same_as<decltype(__out_it), std::back_insert_iterator<__format::__output_buffer<_CharT>>>) {
+    __out_it.__get_container()->__fill(__n, __value);
+    return __out_it;
+  } else if constexpr (std::same_as<decltype(__out_it), typename __format::__retarget_buffer<_CharT>::__iterator>) {
+    __out_it.__buffer_->__fill(__n, __value);
+    return __out_it;
+  } else {
+    return std::ranges::fill_n(std::move(__out_it), __n, __value);
+  }
+}
+
+#  ifndef _LIBCPP_HAS_NO_UNICODE
+template <__fmt_char_type _CharT, output_iterator<const _CharT&> _OutIt>
+  requires(same_as<_CharT, char>)
+_LIBCPP_HIDE_FROM_ABI _OutIt __fill(_OutIt __out_it, size_t __n, __format_spec::__code_point<_CharT> __value) {
+  std::size_t __bytes = std::countl_one(static_cast<unsigned char>(__value.__data[0]));
+  if (__bytes == 0)
+    return __formatter::__fill(std::move(__out_it), __n, __value.__data[0]);
+
+  for (size_t __i = 0; __i < __n; ++__i)
+    __out_it = __formatter::__copy(
+        std::addressof(__value.__data[0]), std::addressof(__value.__data[0]) + __bytes, std::move(__out_it));
+  return __out_it;
+}
+
+#    ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <__fmt_char_type _CharT, output_iterator<const _CharT&> _OutIt>
+  requires(same_as<_CharT, wchar_t> && sizeof(wchar_t) == 2)
+_LIBCPP_HIDE_FROM_ABI _OutIt __fill(_OutIt __out_it, size_t __n, __format_spec::__code_point<_CharT> __value) {
+  if (!__unicode::__is_high_surrogate(__value.__data[0]))
+    return __formatter::__fill(std::move(__out_it), __n, __value.__data[0]);
+
+  for (size_t __i = 0; __i < __n; ++__i)
+    __out_it = __formatter::__copy(
+        std::addressof(__value.__data[0]), std::addressof(__value.__data[0]) + 2, std::move(__out_it));
+  return __out_it;
+}
+
+template <__fmt_char_type _CharT, output_iterator<const _CharT&> _OutIt>
+  requires(same_as<_CharT, wchar_t> && sizeof(wchar_t) == 4)
+_LIBCPP_HIDE_FROM_ABI _OutIt __fill(_OutIt __out_it, size_t __n, __format_spec::__code_point<_CharT> __value) {
+  return __formatter::__fill(std::move(__out_it), __n, __value.__data[0]);
+}
+#    endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+#  else    // _LIBCPP_HAS_NO_UNICODE
+template <__fmt_char_type _CharT, output_iterator<const _CharT&> _OutIt>
+_LIBCPP_HIDE_FROM_ABI _OutIt __fill(_OutIt __out_it, size_t __n, __format_spec::__code_point<_CharT> __value) {
+  return __formatter::__fill(std::move(__out_it), __n, __value.__data[0]);
+}
+#  endif   // _LIBCPP_HAS_NO_UNICODE
+
+/// Writes the input to the output with the required padding.
+///
+/// Since the output column width is specified the function can be used for
+/// ASCII and Unicode output.
+///
+/// \pre \a __size <= \a __width. Using this function when this pre-condition
+///      doesn't hold incurs an unwanted overhead.
+///
+/// \param __str       The string to write.
+/// \param __out_it    The output iterator to write to.
+/// \param __specs     The parsed formatting specifications.
+/// \param __size      The (estimated) output column width. When the elements
+///                    to be written are ASCII the following condition holds
+///                    \a __size == \a __last - \a __first.
+///
+/// \returns           An iterator pointing beyond the last element written.
+///
+/// \note The type of the elements in range [\a __first, \a __last) can differ
+/// from the type of \a __specs. Integer output uses \c std::to_chars for its
+/// conversion, which means the [\a __first, \a __last) always contains elements
+/// of the type \c char.
+template <class _CharT, class _ParserCharT>
+_LIBCPP_HIDE_FROM_ABI auto
+__write(basic_string_view<_CharT> __str,
+        output_iterator<const _CharT&> auto __out_it,
+        __format_spec::__parsed_specifications<_ParserCharT> __specs,
+        ptrdiff_t __size) -> decltype(__out_it) {
+  if (__size >= __specs.__width_)
+    return __formatter::__copy(__str, std::move(__out_it));
+
+  __padding_size_result __padding = __formatter::__padding_size(__size, __specs.__width_, __specs.__std_.__alignment_);
+  __out_it                        = __formatter::__fill(std::move(__out_it), __padding.__before_, __specs.__fill_);
+  __out_it                        = __formatter::__copy(__str, std::move(__out_it));
+  return __formatter::__fill(std::move(__out_it), __padding.__after_, __specs.__fill_);
+}
+
+template <contiguous_iterator _Iterator, class _ParserCharT>
+_LIBCPP_HIDE_FROM_ABI auto
+__write(_Iterator __first,
+        _Iterator __last,
+        output_iterator<const iter_value_t<_Iterator>&> auto __out_it,
+        __format_spec::__parsed_specifications<_ParserCharT> __specs,
+        ptrdiff_t __size) -> decltype(__out_it) {
+  _LIBCPP_ASSERT_VALID_INPUT_RANGE(__first <= __last, "Not a valid range");
+  return __formatter::__write(basic_string_view{__first, __last}, std::move(__out_it), __specs, __size);
+}
+
+/// \overload
+///
+/// Calls the function above where \a __size = \a __last - \a __first.
+template <contiguous_iterator _Iterator, class _ParserCharT>
+_LIBCPP_HIDE_FROM_ABI auto
+__write(_Iterator __first,
+        _Iterator __last,
+        output_iterator<const iter_value_t<_Iterator>&> auto __out_it,
+        __format_spec::__parsed_specifications<_ParserCharT> __specs) -> decltype(__out_it) {
+  _LIBCPP_ASSERT_VALID_INPUT_RANGE(__first <= __last, "Not a valid range");
+  return __formatter::__write(__first, __last, std::move(__out_it), __specs, __last - __first);
+}
+
+template <contiguous_iterator _Iterator,
+          class _CharT = typename iterator_traits<_Iterator>::value_type,
+          class _ParserCharT,
+          class _UnaryOperation>
+_LIBCPP_HIDE_FROM_ABI auto __write_transformed(
+    _Iterator __first,
+    _Iterator __last,
+    output_iterator<const _CharT&> auto __out_it,
+    __format_spec::__parsed_specifications<_ParserCharT> __specs,
+    _UnaryOperation __op) -> decltype(__out_it) {
+  _LIBCPP_ASSERT_VALID_INPUT_RANGE(__first <= __last, "Not a valid range");
+
+  ptrdiff_t __size = __last - __first;
+  if (__size >= __specs.__width_)
+    return __formatter::__transform(__first, __last, std::move(__out_it), __op);
+
+  __padding_size_result __padding = __formatter::__padding_size(__size, __specs.__width_, __specs.__alignment_);
+  __out_it                        = __formatter::__fill(std::move(__out_it), __padding.__before_, __specs.__fill_);
+  __out_it                        = __formatter::__transform(__first, __last, std::move(__out_it), __op);
+  return __formatter::__fill(std::move(__out_it), __padding.__after_, __specs.__fill_);
+}
+
+/// Writes a string using format's width estimation algorithm.
+///
+/// \pre !__specs.__has_precision()
+///
+/// \note When \c _LIBCPP_HAS_NO_UNICODE is defined the function assumes the
+/// input is ASCII.
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI auto __write_string_no_precision(
+    basic_string_view<_CharT> __str,
+    output_iterator<const _CharT&> auto __out_it,
+    __format_spec::__parsed_specifications<_CharT> __specs) -> decltype(__out_it) {
+  _LIBCPP_ASSERT_INTERNAL(!__specs.__has_precision(), "use __write_string");
+
+  // No padding -> copy the string
+  if (!__specs.__has_width())
+    return __formatter::__copy(__str, std::move(__out_it));
+
+  // Note when the estimated width is larger than size there's no padding. So
+  // there's no reason to get the real size when the estimate is larger than or
+  // equal to the minimum field width.
+  size_t __size =
+      __format_spec::__estimate_column_width(__str, __specs.__width_, __format_spec::__column_width_rounding::__up)
+          .__width_;
+  return __formatter::__write(__str, std::move(__out_it), __specs, __size);
+}
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI int __truncate(basic_string_view<_CharT>& __str, int __precision) {
+  __format_spec::__column_width_result __result =
+      __format_spec::__estimate_column_width(__str, __precision, __format_spec::__column_width_rounding::__down);
+  __str = basic_string_view<_CharT>{__str.begin(), __result.__last_};
+  return __result.__width_;
+}
+
+} // namespace __formatter
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FORMAT_FORMATTER_OUTPUT_H
diff --git a/libcxx/include/__cxx03/__format/formatter_pointer.h b/libcxx/include/__cxx03/__format/formatter_pointer.h
new file mode 100644
index 000000000000..6941343efd91
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/formatter_pointer.h
@@ -0,0 +1,72 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMATTER_POINTER_H
+#define _LIBCPP___FORMAT_FORMATTER_POINTER_H
+
+#include <__config>
+#include <__format/concepts.h>
+#include <__format/format_parse_context.h>
+#include <__format/formatter.h>
+#include <__format/formatter_integral.h>
+#include <__format/formatter_output.h>
+#include <__format/parser_std_format_spec.h>
+#include <cstddef>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS __formatter_pointer {
+public:
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    typename _ParseContext::iterator __result = __parser_.__parse(__ctx, __format_spec::__fields_pointer);
+    __format_spec::__process_display_type_pointer(__parser_.__type_, "a pointer");
+    return __result;
+  }
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(const void* __ptr, _FormatContext& __ctx) const {
+    __format_spec::__parsed_specifications<_CharT> __specs = __parser_.__get_parsed_std_specifications(__ctx);
+    __specs.__std_.__alternate_form_                       = true;
+    __specs.__std_.__type_ =
+        __specs.__std_.__type_ == __format_spec::__type::__pointer_upper_case
+            ? __format_spec::__type::__hexadecimal_upper_case
+            : __format_spec::__type::__hexadecimal_lower_case;
+
+    return __formatter::__format_integer(reinterpret_cast<uintptr_t>(__ptr), __ctx, __specs);
+  }
+
+  __format_spec::__parser<_CharT> __parser_;
+};
+
+// [format.formatter.spec]/2.4
+// For each charT, the pointer type specializations template<>
+// - struct formatter<nullptr_t, charT>;
+// - template<> struct formatter<void*, charT>;
+// - template<> struct formatter<const void*, charT>;
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<nullptr_t, _CharT> : public __formatter_pointer<_CharT> {};
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<void*, _CharT> : public __formatter_pointer<_CharT> {};
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<const void*, _CharT> : public __formatter_pointer<_CharT> {};
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_FORMATTER_POINTER_H
diff --git a/libcxx/include/__cxx03/__format/formatter_string.h b/libcxx/include/__cxx03/__format/formatter_string.h
new file mode 100644
index 000000000000..347439fc8dff
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/formatter_string.h
@@ -0,0 +1,150 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMATTER_STRING_H
+#define _LIBCPP___FORMAT_FORMATTER_STRING_H
+
+#include <__config>
+#include <__format/concepts.h>
+#include <__format/format_parse_context.h>
+#include <__format/formatter.h>
+#include <__format/formatter_output.h>
+#include <__format/parser_std_format_spec.h>
+#include <__format/write_escaped.h>
+#include <string>
+#include <string_view>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS __formatter_string {
+public:
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    typename _ParseContext::iterator __result = __parser_.__parse(__ctx, __format_spec::__fields_string);
+    __format_spec::__process_display_type_string(__parser_.__type_);
+    return __result;
+  }
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator
+  format(basic_string_view<_CharT> __str, _FormatContext& __ctx) const {
+#  if _LIBCPP_STD_VER >= 23
+    if (__parser_.__type_ == __format_spec::__type::__debug)
+      return __formatter::__format_escaped_string(__str, __ctx.out(), __parser_.__get_parsed_std_specifications(__ctx));
+#  endif
+
+    return __formatter::__write_string(__str, __ctx.out(), __parser_.__get_parsed_std_specifications(__ctx));
+  }
+
+#  if _LIBCPP_STD_VER >= 23
+  _LIBCPP_HIDE_FROM_ABI constexpr void set_debug_format() { __parser_.__type_ = __format_spec::__type::__debug; }
+#  endif
+
+  __format_spec::__parser<_CharT> __parser_{.__alignment_ = __format_spec::__alignment::__left};
+};
+
+// Formatter const char*.
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<const _CharT*, _CharT> : public __formatter_string<_CharT> {
+  using _Base = __formatter_string<_CharT>;
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(const _CharT* __str, _FormatContext& __ctx) const {
+    _LIBCPP_ASSERT_INTERNAL(__str, "The basic_format_arg constructor should have prevented an invalid pointer.");
+
+    __format_spec::__parsed_specifications<_CharT> __specs = _Base::__parser_.__get_parsed_std_specifications(__ctx);
+#  if _LIBCPP_STD_VER >= 23
+    if (_Base::__parser_.__type_ == __format_spec::__type::__debug)
+      return __formatter::__format_escaped_string(basic_string_view<_CharT>{__str}, __ctx.out(), __specs);
+#  endif
+
+    // When using a center or right alignment and the width option the length
+    // of __str must be known to add the padding upfront. This case is handled
+    // by the base class by converting the argument to a basic_string_view.
+    //
+    // When using left alignment and the width option the padding is added
+    // after outputting __str so the length can be determined while outputting
+    // __str. The same holds true for the precision, during outputting __str it
+    // can be validated whether the precision threshold has been reached. For
+    // now these optimizations aren't implemented. Instead the base class
+    // handles these options.
+    // TODO FMT Implement these improvements.
+    if (__specs.__has_width() || __specs.__has_precision())
+      return __formatter::__write_string(basic_string_view<_CharT>{__str}, __ctx.out(), __specs);
+
+    // No formatting required, copy the string to the output.
+    auto __out_it = __ctx.out();
+    while (*__str)
+      *__out_it++ = *__str++;
+    return __out_it;
+  }
+};
+
+// Formatter char*.
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<_CharT*, _CharT> : public formatter<const _CharT*, _CharT> {
+  using _Base = formatter<const _CharT*, _CharT>;
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(_CharT* __str, _FormatContext& __ctx) const {
+    return _Base::format(__str, __ctx);
+  }
+};
+
+// Formatter char[].
+template <__fmt_char_type _CharT, size_t _Size>
+struct _LIBCPP_TEMPLATE_VIS formatter<_CharT[_Size], _CharT> : public __formatter_string<_CharT> {
+  using _Base = __formatter_string<_CharT>;
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator
+  format(const _CharT (&__str)[_Size], _FormatContext& __ctx) const {
+    return _Base::format(basic_string_view<_CharT>(__str, _Size), __ctx);
+  }
+};
+
+// Formatter std::string.
+template <__fmt_char_type _CharT, class _Traits, class _Allocator>
+struct _LIBCPP_TEMPLATE_VIS formatter<basic_string<_CharT, _Traits, _Allocator>, _CharT>
+    : public __formatter_string<_CharT> {
+  using _Base = __formatter_string<_CharT>;
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator
+  format(const basic_string<_CharT, _Traits, _Allocator>& __str, _FormatContext& __ctx) const {
+    // Drop _Traits and _Allocator to have one std::basic_string formatter.
+    return _Base::format(basic_string_view<_CharT>(__str.data(), __str.size()), __ctx);
+  }
+};
+
+// Formatter std::string_view.
+template <__fmt_char_type _CharT, class _Traits>
+struct _LIBCPP_TEMPLATE_VIS formatter<basic_string_view<_CharT, _Traits>, _CharT> : public __formatter_string<_CharT> {
+  using _Base = __formatter_string<_CharT>;
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator
+  format(basic_string_view<_CharT, _Traits> __str, _FormatContext& __ctx) const {
+    // Drop _Traits to have one std::basic_string_view formatter.
+    return _Base::format(basic_string_view<_CharT>(__str.data(), __str.size()), __ctx);
+  }
+};
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_FORMATTER_STRING_H
diff --git a/libcxx/include/__cxx03/__format/formatter_tuple.h b/libcxx/include/__cxx03/__format/formatter_tuple.h
new file mode 100644
index 000000000000..030097a8797d
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/formatter_tuple.h
@@ -0,0 +1,150 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMATTER_TUPLE_H
+#define _LIBCPP___FORMAT_FORMATTER_TUPLE_H
+
+#include <__algorithm/ranges_copy.h>
+#include <__chrono/statically_widen.h>
+#include <__config>
+#include <__format/buffer.h>
+#include <__format/concepts.h>
+#include <__format/format_context.h>
+#include <__format/format_error.h>
+#include <__format/format_parse_context.h>
+#include <__format/formatter.h>
+#include <__format/formatter_output.h>
+#include <__format/parser_std_format_spec.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/integer_sequence.h>
+#include <__utility/pair.h>
+#include <string_view>
+#include <tuple>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <__fmt_char_type _CharT, class _Tuple, formattable<_CharT>... _Args>
+struct _LIBCPP_TEMPLATE_VIS __formatter_tuple {
+  _LIBCPP_HIDE_FROM_ABI constexpr void set_separator(basic_string_view<_CharT> __separator) noexcept {
+    __separator_ = __separator;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr void
+  set_brackets(basic_string_view<_CharT> __opening_bracket, basic_string_view<_CharT> __closing_bracket) noexcept {
+    __opening_bracket_ = __opening_bracket;
+    __closing_bracket_ = __closing_bracket;
+  }
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    auto __begin = __parser_.__parse(__ctx, __format_spec::__fields_tuple);
+
+    auto __end = __ctx.end();
+    // Note 'n' is part of the type here
+    if (__parser_.__clear_brackets_)
+      set_brackets({}, {});
+    else if (__begin != __end && *__begin == _CharT('m')) {
+      if constexpr (sizeof...(_Args) == 2) {
+        set_separator(_LIBCPP_STATICALLY_WIDEN(_CharT, ": "));
+        set_brackets({}, {});
+        ++__begin;
+      } else
+        std::__throw_format_error("Type m requires a pair or a tuple with two elements");
+    }
+
+    if (__begin != __end && *__begin != _CharT('}'))
+      std::__throw_format_error("The format specifier should consume the input or end with a '}'");
+
+    __ctx.advance_to(__begin);
+
+    // [format.tuple]/7
+    //   ... For each element e in underlying_, if e.set_debug_format()
+    //   is a valid expression, calls e.set_debug_format().
+    std::__for_each_index_sequence(make_index_sequence<sizeof...(_Args)>(), [&]<size_t _Index> {
+      auto& __formatter = std::get<_Index>(__underlying_);
+      __formatter.parse(__ctx);
+      // Unlike the range_formatter we don't guard against evil parsers. Since
+      // this format-spec never has a format-spec for the underlying type
+      // adding the test would give additional overhead.
+      std::__set_debug_format(__formatter);
+    });
+
+    return __begin;
+  }
+
+  template <class _FormatContext>
+  typename _FormatContext::iterator _LIBCPP_HIDE_FROM_ABI
+  format(conditional_t<(formattable<const _Args, _CharT> && ...), const _Tuple&, _Tuple&> __tuple,
+         _FormatContext& __ctx) const {
+    __format_spec::__parsed_specifications<_CharT> __specs = __parser_.__get_parsed_std_specifications(__ctx);
+
+    if (!__specs.__has_width())
+      return __format_tuple(__tuple, __ctx);
+
+    // The size of the buffer needed is:
+    // - open bracket characters
+    // - close bracket character
+    // - n elements where every element may have a different size
+    // - (n -1) separators
+    // The size of the element is hard to predict, knowing the type helps but
+    // it depends on the format-spec. As an initial estimate we guess 6
+    // characters.
+    // Typically both brackets are 1 character and the separator is 2
+    // characters. Which means there will be
+    //   (n - 1) * 2 + 1 + 1 = n * 2 character
+    // So estimate 8 times the range size as buffer.
+    __format::__retarget_buffer<_CharT> __buffer{8 * tuple_size_v<_Tuple>};
+    basic_format_context<typename __format::__retarget_buffer<_CharT>::__iterator, _CharT> __c{
+        __buffer.__make_output_iterator(), __ctx};
+
+    __format_tuple(__tuple, __c);
+
+    return __formatter::__write_string_no_precision(basic_string_view{__buffer.__view()}, __ctx.out(), __specs);
+  }
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator __format_tuple(auto&& __tuple, _FormatContext& __ctx) const {
+    __ctx.advance_to(std::ranges::copy(__opening_bracket_, __ctx.out()).out);
+
+    std::__for_each_index_sequence(make_index_sequence<sizeof...(_Args)>(), [&]<size_t _Index> {
+      if constexpr (_Index)
+        __ctx.advance_to(std::ranges::copy(__separator_, __ctx.out()).out);
+      __ctx.advance_to(std::get<_Index>(__underlying_).format(std::get<_Index>(__tuple), __ctx));
+    });
+
+    return std::ranges::copy(__closing_bracket_, __ctx.out()).out;
+  }
+
+  __format_spec::__parser<_CharT> __parser_{.__alignment_ = __format_spec::__alignment::__left};
+
+private:
+  tuple<formatter<remove_cvref_t<_Args>, _CharT>...> __underlying_;
+  basic_string_view<_CharT> __separator_       = _LIBCPP_STATICALLY_WIDEN(_CharT, ", ");
+  basic_string_view<_CharT> __opening_bracket_ = _LIBCPP_STATICALLY_WIDEN(_CharT, "(");
+  basic_string_view<_CharT> __closing_bracket_ = _LIBCPP_STATICALLY_WIDEN(_CharT, ")");
+};
+
+template <__fmt_char_type _CharT, formattable<_CharT>... _Args>
+struct _LIBCPP_TEMPLATE_VIS formatter<pair<_Args...>, _CharT>
+    : public __formatter_tuple<_CharT, pair<_Args...>, _Args...> {};
+
+template <__fmt_char_type _CharT, formattable<_CharT>... _Args>
+struct _LIBCPP_TEMPLATE_VIS formatter<tuple<_Args...>, _CharT>
+    : public __formatter_tuple<_CharT, tuple<_Args...>, _Args...> {};
+
+#endif //_LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_FORMATTER_TUPLE_H
diff --git a/libcxx/include/__cxx03/__format/indic_conjunct_break_table.h b/libcxx/include/__cxx03/__format/indic_conjunct_break_table.h
new file mode 100644
index 000000000000..44521d27498c
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/indic_conjunct_break_table.h
@@ -0,0 +1,350 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// WARNING, this entire header is generated by
+// utils/generate_indic_conjunct_break_table.py
+// DO NOT MODIFY!
+
+// UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
+//
+// See Terms of Use <https://www.unicode.org/copyright.html>
+// for definitions of Unicode Inc.'s Data Files and Software.
+//
+// NOTICE TO USER: Carefully read the following legal agreement.
+// BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S
+// DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"),
+// YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
+// TERMS AND CONDITIONS OF THIS AGREEMENT.
+// IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE
+// THE DATA FILES OR SOFTWARE.
+//
+// COPYRIGHT AND PERMISSION NOTICE
+//
+// Copyright (c) 1991-2022 Unicode, Inc. All rights reserved.
+// Distributed under the Terms of Use in https://www.unicode.org/copyright.html.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of the Unicode data files and any associated documentation
+// (the "Data Files") or Unicode software and any associated documentation
+// (the "Software") to deal in the Data Files or Software
+// without restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, and/or sell copies of
+// the Data Files or Software, and to permit persons to whom the Data Files
+// or Software are furnished to do so, provided that either
+// (a) this copyright and permission notice appear with all copies
+// of the Data Files or Software, or
+// (b) this copyright and permission notice appear in associated
+// Documentation.
+//
+// THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
+// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT OF THIRD PARTY RIGHTS.
+// IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
+// NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
+// DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+// DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+//
+// Except as contained in this notice, the name of a copyright holder
+// shall not be used in advertising or otherwise to promote the sale,
+// use or other dealings in these Data Files or Software without prior
+// written authorization of the copyright holder.
+
+#ifndef _LIBCPP___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H
+#define _LIBCPP___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H
+
+#include <__algorithm/ranges_upper_bound.h>
+#include <__config>
+#include <__iterator/access.h>
+#include <cstddef>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __indic_conjunct_break {
+
+enum class __property : uint8_t {
+  // Values generated from the data files.
+  __Consonant,
+  __Extend,
+  __Linker,
+
+  // The code unit has none of above properties.
+  __none
+};
+
+/// The entries of the indic conjunct break property table.
+///
+/// The data is generated from
+/// -  https://www.unicode.org/Public/UCD/latest/ucd/DerivedCoreProperties.txt
+///
+/// The data has 3 values
+/// - bits [0, 1] The property. One of the values generated from the datafiles
+///   of \ref __property
+/// - bits [2, 10] The size of the range.
+/// - bits [11, 31] The lower bound code point of the range. The upper bound of
+///   the range is lower bound + size.
+///
+/// The 9 bits for the size allow a maximum range of 512 elements. Some ranges
+/// in the Unicode tables are larger. They are stored in multiple consecutive
+/// ranges in the data table. An alternative would be to store the sizes in a
+/// separate 16-bit value. The original MSVC STL code had such an approach, but
+/// this approach uses less space for the data and is about 4% faster in the
+/// following benchmark.
+/// libcxx/benchmarks/std_format_spec_string_unicode.bench.cpp
+// clang-format off
+_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[201] = {
+    0x00180139,
+    0x001a807d,
+    0x00241811,
+    0x002c88b1,
+    0x002df801,
+    0x002e0805,
+    0x002e2005,
+    0x002e3801,
+    0x00308029,
+    0x00325851,
+    0x00338001,
+    0x0036b019,
+    0x0036f815,
+    0x00373805,
+    0x0037500d,
+    0x00388801,
+    0x00398069,
+    0x003f5821,
+    0x003fe801,
+    0x0040b00d,
+    0x0040d821,
+    0x00412809,
+    0x00414811,
+    0x0042c809,
+    0x0044c01d,
+    0x0046505d,
+    0x00471871,
+    0x0048a890,
+    0x0049e001,
+    0x004a6802,
+    0x004a880d,
+    0x004ac01c,
+    0x004bc01c,
+    0x004ca84c,
+    0x004d5018,
+    0x004d9000,
+    0x004db00c,
+    0x004de001,
+    0x004e6802,
+    0x004ee004,
+    0x004ef800,
+    0x004f8004,
+    0x004ff001,
+    0x0051e001,
+    0x0054a84c,
+    0x00555018,
+    0x00559004,
+    0x0055a810,
+    0x0055e001,
+    0x00566802,
+    0x0057c800,
+    0x0058a84c,
+    0x00595018,
+    0x00599004,
+    0x0059a810,
+    0x0059e001,
+    0x005a6802,
+    0x005ae004,
+    0x005af800,
+    0x005b8800,
+    0x0060a84c,
+    0x0061503c,
+    0x0061e001,
+    0x00626802,
+    0x0062a805,
+    0x0062c008,
+    0x0065e001,
+    0x0068a894,
+    0x0069d805,
+    0x006a6802,
+    0x0071c009,
+    0x0072400d,
+    0x0075c009,
+    0x0076400d,
+    0x0078c005,
+    0x0079a801,
+    0x0079b801,
+    0x0079c801,
+    0x007b8805,
+    0x007ba001,
+    0x007bd00d,
+    0x007c0001,
+    0x007c1009,
+    0x007c3005,
+    0x007e3001,
+    0x0081b801,
+    0x0081c805,
+    0x00846801,
+    0x009ae809,
+    0x00b8a001,
+    0x00be9001,
+    0x00bee801,
+    0x00c54801,
+    0x00c9c809,
+    0x00d0b805,
+    0x00d30001,
+    0x00d3a81d,
+    0x00d3f801,
+    0x00d58035,
+    0x00d5f83d,
+    0x00d9a001,
+    0x00db5821,
+    0x00dd5801,
+    0x00df3001,
+    0x00e1b801,
+    0x00e68009,
+    0x00e6a031,
+    0x00e71019,
+    0x00e76801,
+    0x00e7a001,
+    0x00e7c005,
+    0x00ee00fd,
+    0x01006801,
+    0x01068031,
+    0x01070801,
+    0x0107282d,
+    0x01677809,
+    0x016bf801,
+    0x016f007d,
+    0x01815015,
+    0x0184c805,
+    0x05337801,
+    0x0533a025,
+    0x0534f005,
+    0x05378005,
+    0x05416001,
+    0x05470045,
+    0x05495809,
+    0x054d9801,
+    0x05558001,
+    0x05559009,
+    0x0555b805,
+    0x0555f005,
+    0x05560801,
+    0x0557b001,
+    0x055f6801,
+    0x07d8f001,
+    0x07f1003d,
+    0x080fe801,
+    0x08170001,
+    0x081bb011,
+    0x08506801,
+    0x08507801,
+    0x0851c009,
+    0x0851f801,
+    0x08572805,
+    0x0869200d,
+    0x08755805,
+    0x0877e809,
+    0x087a3029,
+    0x087c100d,
+    0x08838001,
+    0x0883f801,
+    0x0885d001,
+    0x08880009,
+    0x08899805,
+    0x088b9801,
+    0x088e5001,
+    0x0891b001,
+    0x08974805,
+    0x0899d805,
+    0x089b3019,
+    0x089b8011,
+    0x08a23001,
+    0x08a2f001,
+    0x08a61801,
+    0x08ae0001,
+    0x08b5b801,
+    0x08b95801,
+    0x08c1d001,
+    0x08c9f001,
+    0x08ca1801,
+    0x08d1a001,
+    0x08d23801,
+    0x08d4c801,
+    0x08ea1001,
+    0x08ea2005,
+    0x08ecb801,
+    0x08fa1001,
+    0x0b578011,
+    0x0b598019,
+    0x0de4f001,
+    0x0e8b2801,
+    0x0e8b3809,
+    0x0e8b7011,
+    0x0e8bd81d,
+    0x0e8c2819,
+    0x0e8d500d,
+    0x0e921009,
+    0x0f000019,
+    0x0f004041,
+    0x0f00d819,
+    0x0f011805,
+    0x0f013011,
+    0x0f047801,
+    0x0f098019,
+    0x0f157001,
+    0x0f17600d,
+    0x0f27600d,
+    0x0f468019,
+    0x0f4a2019};
+// clang-format on
+
+/// Returns the indic conjuct break property of a code point.
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __property __get_property(const char32_t __code_point) noexcept {
+  // The algorithm searches for the upper bound of the range and, when found,
+  // steps back one entry. This algorithm is used since the code point can be
+  // anywhere in the range. After a lower bound is found the next step is to
+  // compare whether the code unit is indeed in the range.
+  //
+  // Since the entry contains a code unit, size, and property the code point
+  // being sought needs to be adjusted. Just shifting the code point to the
+  // proper position doesn't work; suppose an entry has property 0, size 1,
+  // and lower bound 3. This results in the entry 0x1810.
+  // When searching for code point 3 it will search for 0x1800, find 0x1810
+  // and moves to the previous entry. Thus the lower bound value will never
+  // be found.
+  // The simple solution is to set the bits belonging to the property and
+  // size. Then the upper bound for code point 3 will return the entry after
+  // 0x1810. After moving to the previous entry the algorithm arrives at the
+  // correct entry.
+  ptrdiff_t __i = std::ranges::upper_bound(__entries, (__code_point << 11) | 0x7ffu) - __entries;
+  if (__i == 0)
+    return __property::__none;
+
+  --__i;
+  uint32_t __upper_bound = (__entries[__i] >> 11) + ((__entries[__i] >> 2) & 0b1'1111'1111);
+  if (__code_point <= __upper_bound)
+    return static_cast<__property>(__entries[__i] & 0b11);
+
+  return __property::__none;
+}
+
+} // namespace __indic_conjunct_break
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H
diff --git a/libcxx/include/__cxx03/__format/parser_std_format_spec.h b/libcxx/include/__cxx03/__format/parser_std_format_spec.h
new file mode 100644
index 000000000000..150bdde89f3b
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/parser_std_format_spec.h
@@ -0,0 +1,1172 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_PARSER_STD_FORMAT_SPEC_H
+#define _LIBCPP___FORMAT_PARSER_STD_FORMAT_SPEC_H
+
+/// \file Contains the std-format-spec parser.
+///
+/// Most of the code can be reused in the chrono-format-spec.
+/// This header has some support for the chrono-format-spec since it doesn't
+/// affect the std-format-spec.
+
+#include <__algorithm/copy_n.h>
+#include <__algorithm/min.h>
+#include <__assert>
+#include <__concepts/arithmetic.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/format_arg.h>
+#include <__format/format_error.h>
+#include <__format/format_parse_context.h>
+#include <__format/format_string.h>
+#include <__format/unicode.h>
+#include <__format/width_estimation_table.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h> // iter_value_t
+#include <__memory/addressof.h>
+#include <__type_traits/common_type.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_trivially_copyable.h>
+#include <__variant/monostate.h>
+#include <cstdint>
+#include <string>
+#include <string_view>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __format_spec {
+
+_LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI inline void
+__throw_invalid_option_format_error(const char* __id, const char* __option) {
+  std::__throw_format_error(
+      (string("The format specifier for ") + __id + " does not allow the " + __option + " option").c_str());
+}
+
+_LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI inline void __throw_invalid_type_format_error(const char* __id) {
+  std::__throw_format_error(
+      (string("The type option contains an invalid value for ") + __id + " formatting argument").c_str());
+}
+
+template <contiguous_iterator _Iterator, class _ParseContext>
+_LIBCPP_HIDE_FROM_ABI constexpr __format::__parse_number_result<_Iterator>
+__parse_arg_id(_Iterator __begin, _Iterator __end, _ParseContext& __ctx) {
+  using _CharT = iter_value_t<_Iterator>;
+  // This function is a wrapper to call the real parser. But it does the
+  // validation for the pre-conditions and post-conditions.
+  if (__begin == __end)
+    std::__throw_format_error("End of input while parsing an argument index");
+
+  __format::__parse_number_result __r = __format::__parse_arg_id(__begin, __end, __ctx);
+
+  if (__r.__last == __end || *__r.__last != _CharT('}'))
+    std::__throw_format_error("The argument index is invalid");
+
+  ++__r.__last;
+  return __r;
+}
+
+template <class _Context>
+_LIBCPP_HIDE_FROM_ABI constexpr uint32_t __substitute_arg_id(basic_format_arg<_Context> __format_arg) {
+  // [format.string.std]/8
+  //   If the corresponding formatting argument is not of integral type...
+  // This wording allows char and bool too. LWG-3720 changes the wording to
+  //    If the corresponding formatting argument is not of standard signed or
+  //    unsigned integer type,
+  // This means the 128-bit will not be valid anymore.
+  // TODO FMT Verify this resolution is accepted and add a test to verify
+  //          128-bit integrals fail and switch to visit_format_arg.
+  return std::__visit_format_arg(
+      [](auto __arg) -> uint32_t {
+        using _Type = decltype(__arg);
+        if constexpr (same_as<_Type, monostate>)
+          std::__throw_format_error("The argument index value is too large for the number of arguments supplied");
+
+        // [format.string.std]/8
+        // If { arg-idopt } is used in a width or precision, the value of the
+        // corresponding formatting argument is used in its place. If the
+        // corresponding formatting argument is not of standard signed or unsigned
+        // integer type, or its value is negative for precision or non-positive for
+        // width, an exception of type format_error is thrown.
+        //
+        // When an integral is used in a format function, it is stored as one of
+        // the types checked below. Other integral types are promoted. For example,
+        // a signed char is stored as an int.
+        if constexpr (same_as<_Type, int> || same_as<_Type, unsigned int> || //
+                      same_as<_Type, long long> || same_as<_Type, unsigned long long>) {
+          if constexpr (signed_integral<_Type>) {
+            if (__arg < 0)
+              std::__throw_format_error("An argument index may not have a negative value");
+          }
+
+          using _CT = common_type_t<_Type, decltype(__format::__number_max)>;
+          if (static_cast<_CT>(__arg) > static_cast<_CT>(__format::__number_max))
+            std::__throw_format_error("The value of the argument index exceeds its maximum value");
+
+          return __arg;
+        } else
+          std::__throw_format_error("Replacement argument isn't a standard signed or unsigned integer type");
+      },
+      __format_arg);
+}
+
+/// These fields are a filter for which elements to parse.
+///
+/// They default to false so when a new field is added it needs to be opted in
+/// explicitly.
+struct _LIBCPP_HIDE_FROM_ABI __fields {
+  uint16_t __sign_                 : 1 {false};
+  uint16_t __alternate_form_       : 1 {false};
+  uint16_t __zero_padding_         : 1 {false};
+  uint16_t __precision_            : 1 {false};
+  uint16_t __locale_specific_form_ : 1 {false};
+  uint16_t __type_                 : 1 {false};
+  // Determines the valid values for fill.
+  //
+  // Originally the fill could be any character except { and }. Range-based
+  // formatters use the colon to mark the beginning of the
+  // underlying-format-spec. To avoid parsing ambiguities these formatter
+  // specializations prohibit the use of the colon as a fill character.
+  uint16_t __use_range_fill_ : 1 {false};
+  uint16_t __clear_brackets_ : 1 {false};
+  uint16_t __consume_all_    : 1 {false};
+};
+
+// By not placing this constant in the formatter class it's not duplicated for
+// char and wchar_t.
+inline constexpr __fields __fields_bool{.__locale_specific_form_ = true, .__type_ = true, .__consume_all_ = true};
+inline constexpr __fields __fields_integral{
+    .__sign_                 = true,
+    .__alternate_form_       = true,
+    .__zero_padding_         = true,
+    .__locale_specific_form_ = true,
+    .__type_                 = true,
+    .__consume_all_          = true};
+inline constexpr __fields __fields_floating_point{
+    .__sign_                 = true,
+    .__alternate_form_       = true,
+    .__zero_padding_         = true,
+    .__precision_            = true,
+    .__locale_specific_form_ = true,
+    .__type_                 = true,
+    .__consume_all_          = true};
+inline constexpr __fields __fields_string{.__precision_ = true, .__type_ = true, .__consume_all_ = true};
+inline constexpr __fields __fields_pointer{.__zero_padding_ = true, .__type_ = true, .__consume_all_ = true};
+
+#  if _LIBCPP_STD_VER >= 23
+inline constexpr __fields __fields_tuple{.__use_range_fill_ = true, .__clear_brackets_ = true};
+inline constexpr __fields __fields_range{.__use_range_fill_ = true, .__clear_brackets_ = true};
+inline constexpr __fields __fields_fill_align_width{};
+#  endif
+
+enum class __alignment : uint8_t {
+  /// No alignment is set in the format string.
+  __default,
+  __left,
+  __center,
+  __right,
+  __zero_padding
+};
+
+enum class __sign : uint8_t {
+  /// No sign is set in the format string.
+  ///
+  /// The sign isn't allowed for certain format-types. By using this value
+  /// it's possible to detect whether or not the user explicitly set the sign
+  /// flag. For formatting purposes it behaves the same as \ref __minus.
+  __default,
+  __minus,
+  __plus,
+  __space
+};
+
+enum class __type : uint8_t {
+  __default = 0,
+  __string,
+  __binary_lower_case,
+  __binary_upper_case,
+  __octal,
+  __decimal,
+  __hexadecimal_lower_case,
+  __hexadecimal_upper_case,
+  __pointer_lower_case,
+  __pointer_upper_case,
+  __char,
+  __hexfloat_lower_case,
+  __hexfloat_upper_case,
+  __scientific_lower_case,
+  __scientific_upper_case,
+  __fixed_lower_case,
+  __fixed_upper_case,
+  __general_lower_case,
+  __general_upper_case,
+  __debug
+};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __create_type_mask(__type __t) {
+  uint32_t __shift = static_cast<uint32_t>(__t);
+  if (__shift == 0)
+    return 1;
+
+  if (__shift > 31)
+    std::__throw_format_error("The type does not fit in the mask");
+
+  return 1 << __shift;
+}
+
+inline constexpr uint32_t __type_mask_integer =
+    __create_type_mask(__type::__binary_lower_case) |      //
+    __create_type_mask(__type::__binary_upper_case) |      //
+    __create_type_mask(__type::__decimal) |                //
+    __create_type_mask(__type::__octal) |                  //
+    __create_type_mask(__type::__hexadecimal_lower_case) | //
+    __create_type_mask(__type::__hexadecimal_upper_case);
+
+struct __std {
+  __alignment __alignment_     : 3;
+  __sign __sign_               : 2;
+  bool __alternate_form_       : 1;
+  bool __locale_specific_form_ : 1;
+  __type __type_;
+};
+
+struct __chrono {
+  __alignment __alignment_     : 3;
+  bool __locale_specific_form_ : 1;
+  bool __hour_                 : 1;
+  bool __weekday_name_         : 1;
+  bool __weekday_              : 1;
+  bool __day_of_year_          : 1;
+  bool __week_of_year_         : 1;
+  bool __month_name_           : 1;
+};
+
+// The fill UCS scalar value.
+//
+// This is always an array, with 1, 2, or 4 elements.
+// The size of the data structure is always 32-bits.
+template <class _CharT>
+struct __code_point;
+
+template <>
+struct __code_point<char> {
+  char __data[4] = {' '};
+};
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+struct __code_point<wchar_t> {
+  wchar_t __data[4 / sizeof(wchar_t)] = {L' '};
+};
+#  endif
+
+/// Contains the parsed formatting specifications.
+///
+/// This contains information for both the std-format-spec and the
+/// chrono-format-spec. This results in some unused members for both
+/// specifications. However these unused members don't increase the size
+/// of the structure.
+///
+/// This struct doesn't cross ABI boundaries so its layout doesn't need to be
+/// kept stable.
+template <class _CharT>
+struct __parsed_specifications {
+  union {
+    // The field __alignment_ is the first element in __std_ and __chrono_.
+    // This allows the code to always inspect this value regards which member
+    // of the union is the active member [class.union.general]/2.
+    //
+    // This is needed since the generic output routines handle the alignment of
+    // the output.
+    __alignment __alignment_ : 3;
+    __std __std_;
+    __chrono __chrono_;
+  };
+
+  /// The requested width.
+  ///
+  /// When the format-spec used an arg-id for this field it has already been
+  /// replaced with the value of that arg-id.
+  int32_t __width_;
+
+  /// The requested precision.
+  ///
+  /// When the format-spec used an arg-id for this field it has already been
+  /// replaced with the value of that arg-id.
+  int32_t __precision_;
+
+  __code_point<_CharT> __fill_;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __has_width() const { return __width_ > 0; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __has_precision() const { return __precision_ >= 0; }
+};
+
+// Validate the struct is small and cheap to copy since the struct is passed by
+// value in formatting functions.
+static_assert(sizeof(__parsed_specifications<char>) == 16);
+static_assert(is_trivially_copyable_v<__parsed_specifications<char>>);
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+static_assert(sizeof(__parsed_specifications<wchar_t>) == 16);
+static_assert(is_trivially_copyable_v<__parsed_specifications<wchar_t>>);
+#  endif
+
+/// The parser for the std-format-spec.
+///
+/// Note this class is a member of std::formatter specializations. It's
+/// expected developers will create their own formatter specializations that
+/// inherit from the std::formatter specializations. This means this class
+/// must be ABI stable. To aid the stability the unused bits in the class are
+/// set to zero. That way they can be repurposed if a future revision of the
+/// Standards adds new fields to std-format-spec.
+template <class _CharT>
+class _LIBCPP_TEMPLATE_VIS __parser {
+public:
+  // Parses the format specification.
+  //
+  // Depending on whether the parsing is done compile-time or run-time
+  // the method slightly differs.
+  // - Only parses a field when it is in the __fields. Accepting all
+  //   fields and then validating the valid ones has a performance impact.
+  //   This is faster but gives slighly worse error messages.
+  // - At compile-time when a field is not accepted the parser will still
+  //   parse it and give an error when it's present. This gives a more
+  //   accurate error.
+  // The idea is that most times the format instead of the vformat
+  // functions are used. In that case the error will be detected during
+  // compilation and there is no need to pay for the run-time overhead.
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator __parse(_ParseContext& __ctx, __fields __fields) {
+    auto __begin = __ctx.begin();
+    auto __end   = __ctx.end();
+    if (__begin == __end || *__begin == _CharT('}') || (__fields.__use_range_fill_ && *__begin == _CharT(':')))
+      return __begin;
+
+    if (__parse_fill_align(__begin, __end) && __begin == __end)
+      return __begin;
+
+    if (__fields.__sign_) {
+      if (__parse_sign(__begin) && __begin == __end)
+        return __begin;
+    } else if (std::is_constant_evaluated() && __parse_sign(__begin)) {
+      std::__throw_format_error("The format specification does not allow the sign option");
+    }
+
+    if (__fields.__alternate_form_) {
+      if (__parse_alternate_form(__begin) && __begin == __end)
+        return __begin;
+    } else if (std::is_constant_evaluated() && __parse_alternate_form(__begin)) {
+      std::__throw_format_error("The format specifier does not allow the alternate form option");
+    }
+
+    if (__fields.__zero_padding_) {
+      if (__parse_zero_padding(__begin) && __begin == __end)
+        return __begin;
+    } else if (std::is_constant_evaluated() && __parse_zero_padding(__begin)) {
+      std::__throw_format_error("The format specifier does not allow the zero-padding option");
+    }
+
+    if (__parse_width(__begin, __end, __ctx) && __begin == __end)
+      return __begin;
+
+    if (__fields.__precision_) {
+      if (__parse_precision(__begin, __end, __ctx) && __begin == __end)
+        return __begin;
+    } else if (std::is_constant_evaluated() && __parse_precision(__begin, __end, __ctx)) {
+      std::__throw_format_error("The format specifier does not allow the precision option");
+    }
+
+    if (__fields.__locale_specific_form_) {
+      if (__parse_locale_specific_form(__begin) && __begin == __end)
+        return __begin;
+    } else if (std::is_constant_evaluated() && __parse_locale_specific_form(__begin)) {
+      std::__throw_format_error("The format specifier does not allow the locale-specific form option");
+    }
+
+    if (__fields.__clear_brackets_) {
+      if (__parse_clear_brackets(__begin) && __begin == __end)
+        return __begin;
+    } else if (std::is_constant_evaluated() && __parse_clear_brackets(__begin)) {
+      std::__throw_format_error("The format specifier does not allow the n option");
+    }
+
+    if (__fields.__type_)
+      __parse_type(__begin);
+
+    if (!__fields.__consume_all_)
+      return __begin;
+
+    if (__begin != __end && *__begin != _CharT('}'))
+      std::__throw_format_error("The format specifier should consume the input or end with a '}'");
+
+    return __begin;
+  }
+
+  // Validates the selected the parsed data.
+  //
+  // The valid fields in the parser may depend on the display type
+  // selected. But the type is the last optional field, so by the time
+  // it's known an option can't be used, it already has been parsed.
+  // This does the validation again.
+  //
+  // For example an integral may have a sign, zero-padding, or alternate
+  // form when the type option is not 'c'. So the generic approach is:
+  //
+  // typename _ParseContext::iterator __result = __parser_.__parse(__ctx, __format_spec::__fields_integral);
+  // if (__parser.__type_ == __format_spec::__type::__char) {
+  //   __parser.__validate((__format_spec::__fields_bool, "an integer");
+  //   ... // more char adjustments
+  // } else {
+  //   ... // validate an integral type.
+  // }
+  //
+  // For some types all valid options need a second validation run, like
+  // boolean types.
+  //
+  // Depending on whether the validation is done at compile-time or
+  // run-time the error differs
+  // - run-time the exception is thrown and contains the type of field
+  //   being validated.
+  // - at compile-time the line with `std::__throw_format_error` is shown
+  //   in the output. In that case it's important for the error to be on one
+  //   line.
+  // Note future versions of C++ may allow better compile-time error
+  // reporting.
+  _LIBCPP_HIDE_FROM_ABI constexpr void
+  __validate(__fields __fields, const char* __id, uint32_t __type_mask = -1) const {
+    if (!__fields.__sign_ && __sign_ != __sign::__default) {
+      if (std::is_constant_evaluated())
+        std::__throw_format_error("The format specifier does not allow the sign option");
+      else
+        __format_spec::__throw_invalid_option_format_error(__id, "sign");
+    }
+
+    if (!__fields.__alternate_form_ && __alternate_form_) {
+      if (std::is_constant_evaluated())
+        std::__throw_format_error("The format specifier does not allow the alternate form option");
+      else
+        __format_spec::__throw_invalid_option_format_error(__id, "alternate form");
+    }
+
+    if (!__fields.__zero_padding_ && __alignment_ == __alignment::__zero_padding) {
+      if (std::is_constant_evaluated())
+        std::__throw_format_error("The format specifier does not allow the zero-padding option");
+      else
+        __format_spec::__throw_invalid_option_format_error(__id, "zero-padding");
+    }
+
+    if (!__fields.__precision_ && __precision_ != -1) { // Works both when the precision has a value or an arg-id.
+      if (std::is_constant_evaluated())
+        std::__throw_format_error("The format specifier does not allow the precision option");
+      else
+        __format_spec::__throw_invalid_option_format_error(__id, "precision");
+    }
+
+    if (!__fields.__locale_specific_form_ && __locale_specific_form_) {
+      if (std::is_constant_evaluated())
+        std::__throw_format_error("The format specifier does not allow the locale-specific form option");
+      else
+        __format_spec::__throw_invalid_option_format_error(__id, "locale-specific form");
+    }
+
+    if ((__create_type_mask(__type_) & __type_mask) == 0) {
+      if (std::is_constant_evaluated())
+        std::__throw_format_error("The format specifier uses an invalid value for the type option");
+      else
+        __format_spec::__throw_invalid_type_format_error(__id);
+    }
+  }
+
+  /// \returns the `__parsed_specifications` with the resolved dynamic sizes..
+  _LIBCPP_HIDE_FROM_ABI __parsed_specifications<_CharT> __get_parsed_std_specifications(auto& __ctx) const {
+    return __parsed_specifications<_CharT>{
+        .__std_ = __std{.__alignment_            = __alignment_,
+                        .__sign_                 = __sign_,
+                        .__alternate_form_       = __alternate_form_,
+                        .__locale_specific_form_ = __locale_specific_form_,
+                        .__type_                 = __type_},
+        .__width_{__get_width(__ctx)},
+        .__precision_{__get_precision(__ctx)},
+        .__fill_{__fill_}};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __parsed_specifications<_CharT> __get_parsed_chrono_specifications(auto& __ctx) const {
+    return __parsed_specifications<_CharT>{
+        .__chrono_ =
+            __chrono{.__alignment_            = __alignment_,
+                     .__locale_specific_form_ = __locale_specific_form_,
+                     .__hour_                 = __hour_,
+                     .__weekday_name_         = __weekday_name_,
+                     .__weekday_              = __weekday_,
+                     .__day_of_year_          = __day_of_year_,
+                     .__week_of_year_         = __week_of_year_,
+                     .__month_name_           = __month_name_},
+        .__width_{__get_width(__ctx)},
+        .__precision_{__get_precision(__ctx)},
+        .__fill_{__fill_}};
+  }
+
+  __alignment __alignment_     : 3 {__alignment::__default};
+  __sign __sign_               : 2 {__sign::__default};
+  bool __alternate_form_       : 1 {false};
+  bool __locale_specific_form_ : 1 {false};
+  bool __clear_brackets_       : 1 {false};
+  __type __type_{__type::__default};
+
+  // These flags are only used for formatting chrono. Since the struct has
+  // padding space left it's added to this structure.
+  bool __hour_ : 1 {false};
+
+  bool __weekday_name_ : 1 {false};
+  bool __weekday_      : 1 {false};
+
+  bool __day_of_year_  : 1 {false};
+  bool __week_of_year_ : 1 {false};
+
+  bool __month_name_ : 1 {false};
+
+  uint8_t __reserved_0_ : 2 {0};
+  uint8_t __reserved_1_ : 6 {0};
+  // These two flags are only used internally and not part of the
+  // __parsed_specifications. Therefore put them at the end.
+  bool __width_as_arg_     : 1 {false};
+  bool __precision_as_arg_ : 1 {false};
+
+  /// The requested width, either the value or the arg-id.
+  int32_t __width_{0};
+
+  /// The requested precision, either the value or the arg-id.
+  int32_t __precision_{-1};
+
+  __code_point<_CharT> __fill_{};
+
+private:
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __parse_alignment(_CharT __c) {
+    switch (__c) {
+    case _CharT('<'):
+      __alignment_ = __alignment::__left;
+      return true;
+
+    case _CharT('^'):
+      __alignment_ = __alignment::__center;
+      return true;
+
+    case _CharT('>'):
+      __alignment_ = __alignment::__right;
+      return true;
+    }
+    return false;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void __validate_fill_character(_CharT __fill) {
+    // The forbidden fill characters all code points formed from a single code unit, thus the
+    // check can be omitted when more code units are used.
+    if (__fill == _CharT('{'))
+      std::__throw_format_error("The fill option contains an invalid value");
+  }
+
+#  ifndef _LIBCPP_HAS_NO_UNICODE
+  // range-fill and tuple-fill are identical
+  template <contiguous_iterator _Iterator>
+    requires same_as<_CharT, char>
+#    ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+          || (same_as<_CharT, wchar_t> && sizeof(wchar_t) == 2)
+#    endif
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __parse_fill_align(_Iterator& __begin, _Iterator __end) {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __begin != __end,
+        "when called with an empty input the function will cause "
+        "undefined behavior by evaluating data not in the input");
+    __unicode::__code_point_view<_CharT> __view{__begin, __end};
+    __unicode::__consume_result __consumed = __view.__consume();
+    if (__consumed.__status != __unicode::__consume_result::__ok)
+      std::__throw_format_error("The format specifier contains malformed Unicode characters");
+
+    if (__view.__position() < __end && __parse_alignment(*__view.__position())) {
+      ptrdiff_t __code_units = __view.__position() - __begin;
+      if (__code_units == 1)
+        // The forbidden fill characters all are code points encoded
+        // in one code unit, thus the check can be omitted when more
+        // code units are used.
+        __validate_fill_character(*__begin);
+
+      std::copy_n(__begin, __code_units, std::addressof(__fill_.__data[0]));
+      __begin += __code_units + 1;
+      return true;
+    }
+
+    if (!__parse_alignment(*__begin))
+      return false;
+
+    ++__begin;
+    return true;
+  }
+
+#    ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+  template <contiguous_iterator _Iterator>
+    requires(same_as<_CharT, wchar_t> && sizeof(wchar_t) == 4)
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __parse_fill_align(_Iterator& __begin, _Iterator __end) {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __begin != __end,
+        "when called with an empty input the function will cause "
+        "undefined behavior by evaluating data not in the input");
+    if (__begin + 1 != __end && __parse_alignment(*(__begin + 1))) {
+      if (!__unicode::__is_scalar_value(*__begin))
+        std::__throw_format_error("The fill option contains an invalid value");
+
+      __validate_fill_character(*__begin);
+
+      __fill_.__data[0] = *__begin;
+      __begin += 2;
+      return true;
+    }
+
+    if (!__parse_alignment(*__begin))
+      return false;
+
+    ++__begin;
+    return true;
+  }
+
+#    endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+#  else // _LIBCPP_HAS_NO_UNICODE
+  // range-fill and tuple-fill are identical
+  template <contiguous_iterator _Iterator>
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __parse_fill_align(_Iterator& __begin, _Iterator __end) {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __begin != __end,
+        "when called with an empty input the function will cause "
+        "undefined behavior by evaluating data not in the input");
+    if (__begin + 1 != __end) {
+      if (__parse_alignment(*(__begin + 1))) {
+        __validate_fill_character(*__begin);
+
+        __fill_.__data[0] = *__begin;
+        __begin += 2;
+        return true;
+      }
+    }
+
+    if (!__parse_alignment(*__begin))
+      return false;
+
+    ++__begin;
+    return true;
+  }
+
+#  endif // _LIBCPP_HAS_NO_UNICODE
+
+  template <contiguous_iterator _Iterator>
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __parse_sign(_Iterator& __begin) {
+    switch (*__begin) {
+    case _CharT('-'):
+      __sign_ = __sign::__minus;
+      break;
+    case _CharT('+'):
+      __sign_ = __sign::__plus;
+      break;
+    case _CharT(' '):
+      __sign_ = __sign::__space;
+      break;
+    default:
+      return false;
+    }
+    ++__begin;
+    return true;
+  }
+
+  template <contiguous_iterator _Iterator>
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __parse_alternate_form(_Iterator& __begin) {
+    if (*__begin != _CharT('#'))
+      return false;
+
+    __alternate_form_ = true;
+    ++__begin;
+    return true;
+  }
+
+  template <contiguous_iterator _Iterator>
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __parse_zero_padding(_Iterator& __begin) {
+    if (*__begin != _CharT('0'))
+      return false;
+
+    if (__alignment_ == __alignment::__default)
+      __alignment_ = __alignment::__zero_padding;
+    ++__begin;
+    return true;
+  }
+
+  template <contiguous_iterator _Iterator>
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __parse_width(_Iterator& __begin, _Iterator __end, auto& __ctx) {
+    if (*__begin == _CharT('0'))
+      std::__throw_format_error("The width option should not have a leading zero");
+
+    if (*__begin == _CharT('{')) {
+      __format::__parse_number_result __r = __format_spec::__parse_arg_id(++__begin, __end, __ctx);
+      __width_as_arg_                     = true;
+      __width_                            = __r.__value;
+      __begin                             = __r.__last;
+      return true;
+    }
+
+    if (*__begin < _CharT('0') || *__begin > _CharT('9'))
+      return false;
+
+    __format::__parse_number_result __r = __format::__parse_number(__begin, __end);
+    __width_                            = __r.__value;
+    _LIBCPP_ASSERT_INTERNAL(__width_ != 0,
+                            "A zero value isn't allowed and should be impossible, "
+                            "due to validations in this function");
+    __begin = __r.__last;
+    return true;
+  }
+
+  template <contiguous_iterator _Iterator>
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __parse_precision(_Iterator& __begin, _Iterator __end, auto& __ctx) {
+    if (*__begin != _CharT('.'))
+      return false;
+
+    ++__begin;
+    if (__begin == __end)
+      std::__throw_format_error("End of input while parsing format specifier precision");
+
+    if (*__begin == _CharT('{')) {
+      __format::__parse_number_result __arg_id = __format_spec::__parse_arg_id(++__begin, __end, __ctx);
+      __precision_as_arg_                      = true;
+      __precision_                             = __arg_id.__value;
+      __begin                                  = __arg_id.__last;
+      return true;
+    }
+
+    if (*__begin < _CharT('0') || *__begin > _CharT('9'))
+      std::__throw_format_error("The precision option does not contain a value or an argument index");
+
+    __format::__parse_number_result __r = __format::__parse_number(__begin, __end);
+    __precision_                        = __r.__value;
+    __precision_as_arg_                 = false;
+    __begin                             = __r.__last;
+    return true;
+  }
+
+  template <contiguous_iterator _Iterator>
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __parse_locale_specific_form(_Iterator& __begin) {
+    if (*__begin != _CharT('L'))
+      return false;
+
+    __locale_specific_form_ = true;
+    ++__begin;
+    return true;
+  }
+
+  template <contiguous_iterator _Iterator>
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __parse_clear_brackets(_Iterator& __begin) {
+    if (*__begin != _CharT('n'))
+      return false;
+
+    __clear_brackets_ = true;
+    ++__begin;
+    return true;
+  }
+
+  template <contiguous_iterator _Iterator>
+  _LIBCPP_HIDE_FROM_ABI constexpr void __parse_type(_Iterator& __begin) {
+    // Determines the type. It does not validate whether the selected type is
+    // valid. Most formatters have optional fields that are only allowed for
+    // certain types. These parsers need to do validation after the type has
+    // been parsed. So its easier to implement the validation for all types in
+    // the specific parse function.
+    switch (*__begin) {
+    case 'A':
+      __type_ = __type::__hexfloat_upper_case;
+      break;
+    case 'B':
+      __type_ = __type::__binary_upper_case;
+      break;
+    case 'E':
+      __type_ = __type::__scientific_upper_case;
+      break;
+    case 'F':
+      __type_ = __type::__fixed_upper_case;
+      break;
+    case 'G':
+      __type_ = __type::__general_upper_case;
+      break;
+    case 'X':
+      __type_ = __type::__hexadecimal_upper_case;
+      break;
+    case 'a':
+      __type_ = __type::__hexfloat_lower_case;
+      break;
+    case 'b':
+      __type_ = __type::__binary_lower_case;
+      break;
+    case 'c':
+      __type_ = __type::__char;
+      break;
+    case 'd':
+      __type_ = __type::__decimal;
+      break;
+    case 'e':
+      __type_ = __type::__scientific_lower_case;
+      break;
+    case 'f':
+      __type_ = __type::__fixed_lower_case;
+      break;
+    case 'g':
+      __type_ = __type::__general_lower_case;
+      break;
+    case 'o':
+      __type_ = __type::__octal;
+      break;
+    case 'p':
+      __type_ = __type::__pointer_lower_case;
+      break;
+    case 'P':
+      __type_ = __type::__pointer_upper_case;
+      break;
+    case 's':
+      __type_ = __type::__string;
+      break;
+    case 'x':
+      __type_ = __type::__hexadecimal_lower_case;
+      break;
+#  if _LIBCPP_STD_VER >= 23
+    case '?':
+      __type_ = __type::__debug;
+      break;
+#  endif
+    default:
+      return;
+    }
+    ++__begin;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int32_t __get_width(auto& __ctx) const {
+    if (!__width_as_arg_)
+      return __width_;
+
+    return __format_spec::__substitute_arg_id(__ctx.arg(__width_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int32_t __get_precision(auto& __ctx) const {
+    if (!__precision_as_arg_)
+      return __precision_;
+
+    return __format_spec::__substitute_arg_id(__ctx.arg(__precision_));
+  }
+};
+
+// Validates whether the reserved bitfields don't change the size.
+static_assert(sizeof(__parser<char>) == 16);
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+static_assert(sizeof(__parser<wchar_t>) == 16);
+#  endif
+
+_LIBCPP_HIDE_FROM_ABI constexpr void __process_display_type_string(__format_spec::__type __type) {
+  switch (__type) {
+  case __format_spec::__type::__default:
+  case __format_spec::__type::__string:
+  case __format_spec::__type::__debug:
+    break;
+
+  default:
+    std::__throw_format_error("The type option contains an invalid value for a string formatting argument");
+  }
+}
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI constexpr void __process_display_type_bool_string(__parser<_CharT>& __parser, const char* __id) {
+  __parser.__validate(__format_spec::__fields_bool, __id);
+  if (__parser.__alignment_ == __alignment::__default)
+    __parser.__alignment_ = __alignment::__left;
+}
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI constexpr void __process_display_type_char(__parser<_CharT>& __parser, const char* __id) {
+  __format_spec::__process_display_type_bool_string(__parser, __id);
+}
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI constexpr void __process_parsed_bool(__parser<_CharT>& __parser, const char* __id) {
+  switch (__parser.__type_) {
+  case __format_spec::__type::__default:
+  case __format_spec::__type::__string:
+    __format_spec::__process_display_type_bool_string(__parser, __id);
+    break;
+
+  case __format_spec::__type::__binary_lower_case:
+  case __format_spec::__type::__binary_upper_case:
+  case __format_spec::__type::__octal:
+  case __format_spec::__type::__decimal:
+  case __format_spec::__type::__hexadecimal_lower_case:
+  case __format_spec::__type::__hexadecimal_upper_case:
+    break;
+
+  default:
+    __format_spec::__throw_invalid_type_format_error(__id);
+  }
+}
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI constexpr void __process_parsed_char(__parser<_CharT>& __parser, const char* __id) {
+  switch (__parser.__type_) {
+  case __format_spec::__type::__default:
+  case __format_spec::__type::__char:
+  case __format_spec::__type::__debug:
+    __format_spec::__process_display_type_char(__parser, __id);
+    break;
+
+  case __format_spec::__type::__binary_lower_case:
+  case __format_spec::__type::__binary_upper_case:
+  case __format_spec::__type::__octal:
+  case __format_spec::__type::__decimal:
+  case __format_spec::__type::__hexadecimal_lower_case:
+  case __format_spec::__type::__hexadecimal_upper_case:
+    break;
+
+  default:
+    __format_spec::__throw_invalid_type_format_error(__id);
+  }
+}
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI constexpr void __process_parsed_integer(__parser<_CharT>& __parser, const char* __id) {
+  switch (__parser.__type_) {
+  case __format_spec::__type::__default:
+  case __format_spec::__type::__binary_lower_case:
+  case __format_spec::__type::__binary_upper_case:
+  case __format_spec::__type::__octal:
+  case __format_spec::__type::__decimal:
+  case __format_spec::__type::__hexadecimal_lower_case:
+  case __format_spec::__type::__hexadecimal_upper_case:
+    break;
+
+  case __format_spec::__type::__char:
+    __format_spec::__process_display_type_char(__parser, __id);
+    break;
+
+  default:
+    __format_spec::__throw_invalid_type_format_error(__id);
+  }
+}
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI constexpr void __process_parsed_floating_point(__parser<_CharT>& __parser, const char* __id) {
+  switch (__parser.__type_) {
+  case __format_spec::__type::__default:
+  case __format_spec::__type::__hexfloat_lower_case:
+  case __format_spec::__type::__hexfloat_upper_case:
+    // Precision specific behavior will be handled later.
+    break;
+  case __format_spec::__type::__scientific_lower_case:
+  case __format_spec::__type::__scientific_upper_case:
+  case __format_spec::__type::__fixed_lower_case:
+  case __format_spec::__type::__fixed_upper_case:
+  case __format_spec::__type::__general_lower_case:
+  case __format_spec::__type::__general_upper_case:
+    if (!__parser.__precision_as_arg_ && __parser.__precision_ == -1)
+      // Set the default precision for the call to to_chars.
+      __parser.__precision_ = 6;
+    break;
+
+  default:
+    __format_spec::__throw_invalid_type_format_error(__id);
+  }
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr void __process_display_type_pointer(__format_spec::__type __type, const char* __id) {
+  switch (__type) {
+  case __format_spec::__type::__default:
+  case __format_spec::__type::__pointer_lower_case:
+  case __format_spec::__type::__pointer_upper_case:
+    break;
+
+  default:
+    __format_spec::__throw_invalid_type_format_error(__id);
+  }
+}
+
+template <contiguous_iterator _Iterator>
+struct __column_width_result {
+  /// The number of output columns.
+  size_t __width_;
+  /// One beyond the last code unit used in the estimation.
+  ///
+  /// This limits the original output to fit in the wanted number of columns.
+  _Iterator __last_;
+};
+
+template <contiguous_iterator _Iterator>
+__column_width_result(size_t, _Iterator) -> __column_width_result<_Iterator>;
+
+/// Since a column width can be two it's possible that the requested column
+/// width can't be achieved. Depending on the intended usage the policy can be
+/// selected.
+/// - When used as precision the maximum width may not be exceeded and the
+///   result should be "rounded down" to the previous boundary.
+/// - When used as a width we're done once the minimum is reached, but
+///   exceeding is not an issue. Rounding down is an issue since that will
+///   result in writing fill characters. Therefore the result needs to be
+///   "rounded up".
+enum class __column_width_rounding { __down, __up };
+
+#  ifndef _LIBCPP_HAS_NO_UNICODE
+
+namespace __detail {
+template <contiguous_iterator _Iterator>
+_LIBCPP_HIDE_FROM_ABI constexpr __column_width_result<_Iterator> __estimate_column_width_grapheme_clustering(
+    _Iterator __first, _Iterator __last, size_t __maximum, __column_width_rounding __rounding) noexcept {
+  using _CharT = iter_value_t<_Iterator>;
+  __unicode::__extended_grapheme_cluster_view<_CharT> __view{__first, __last};
+
+  __column_width_result<_Iterator> __result{0, __first};
+  while (__result.__last_ != __last && __result.__width_ <= __maximum) {
+    typename __unicode::__extended_grapheme_cluster_view<_CharT>::__cluster __cluster = __view.__consume();
+    int __width = __width_estimation_table::__estimated_width(__cluster.__code_point_);
+
+    // When the next entry would exceed the maximum width the previous width
+    // might be returned. For example when a width of 100 is requested the
+    // returned width might be 99, since the next code point has an estimated
+    // column width of 2. This depends on the rounding flag.
+    // When the maximum is exceeded the loop will abort the next iteration.
+    if (__rounding == __column_width_rounding::__down && __result.__width_ + __width > __maximum)
+      return __result;
+
+    __result.__width_ += __width;
+    __result.__last_ = __cluster.__last_;
+  }
+
+  return __result;
+}
+
+} // namespace __detail
+
+// Unicode can be stored in several formats: UTF-8, UTF-16, and UTF-32.
+// Depending on format the relation between the number of code units stored and
+// the number of output columns differs. The first relation is the number of
+// code units forming a code point. (The text assumes the code units are
+// unsigned.)
+// - UTF-8 The number of code units is between one and four. The first 127
+//   Unicode code points match the ASCII character set. When the highest bit is
+//   set it means the code point has more than one code unit.
+// - UTF-16: The number of code units is between 1 and 2. When the first
+//   code unit is in the range [0xd800,0xdfff) it means the code point uses two
+//   code units.
+// - UTF-32: The number of code units is always one.
+//
+// The code point to the number of columns is specified in
+// [format.string.std]/11. This list might change in the future.
+//
+// Another thing to be taken into account is Grapheme clustering. This means
+// that in some cases multiple code points are combined one element in the
+// output. For example:
+// - an ASCII character with a combined diacritical mark
+// - an emoji with a skin tone modifier
+// - a group of combined people emoji to create a family
+// - a combination of flag emoji
+//
+// See also:
+// - [format.string.general]/11
+// - https://en.wikipedia.org/wiki/UTF-8#Encoding
+// - https://en.wikipedia.org/wiki/UTF-16#U+D800_to_U+DFFF
+
+_LIBCPP_HIDE_FROM_ABI constexpr bool __is_ascii(char32_t __c) { return __c < 0x80; }
+
+/// Determines the number of output columns needed to render the input.
+///
+/// \note When the scanner encounters malformed Unicode it acts as-if every
+/// code unit is a one column code point. Typically a terminal uses the same
+/// strategy and replaces every malformed code unit with a one column
+/// replacement character.
+///
+/// \param __first    Points to the first element of the input range.
+/// \param __last     Points beyond the last element of the input range.
+/// \param __maximum  The maximum number of output columns. The returned number
+///                   of estimated output columns will not exceed this value.
+/// \param __rounding Selects the rounding method.
+///                   \c __down result.__width_ <= __maximum
+///                   \c __up result.__width_ <= __maximum + 1
+template <class _CharT, class _Iterator = typename basic_string_view<_CharT>::const_iterator>
+_LIBCPP_HIDE_FROM_ABI constexpr __column_width_result<_Iterator> __estimate_column_width(
+    basic_string_view<_CharT> __str, size_t __maximum, __column_width_rounding __rounding) noexcept {
+  // The width estimation is done in two steps:
+  // - Quickly process for the ASCII part. ASCII has the following properties
+  //   - One code unit is one code point
+  //   - Every code point has an estimated width of one
+  // - When needed it will a Unicode Grapheme clustering algorithm to find
+  //   the proper place for truncation.
+
+  if (__str.empty() || __maximum == 0)
+    return {0, __str.begin()};
+
+  // ASCII has one caveat; when an ASCII character is followed by a non-ASCII
+  // character they might be part of an extended grapheme cluster. For example:
+  //   an ASCII letter and a COMBINING ACUTE ACCENT
+  // The truncate should happen after the COMBINING ACUTE ACCENT. Therefore we
+  // need to scan one code unit beyond the requested precision. When this code
+  // unit is non-ASCII we omit the current code unit and let the Grapheme
+  // clustering algorithm do its work.
+  auto __it = __str.begin();
+  if (__format_spec::__is_ascii(*__it)) {
+    do {
+      --__maximum;
+      ++__it;
+      if (__it == __str.end())
+        return {__str.size(), __str.end()};
+
+      if (__maximum == 0) {
+        if (__format_spec::__is_ascii(*__it))
+          return {static_cast<size_t>(__it - __str.begin()), __it};
+
+        break;
+      }
+    } while (__format_spec::__is_ascii(*__it));
+    --__it;
+    ++__maximum;
+  }
+
+  ptrdiff_t __ascii_size = __it - __str.begin();
+  __column_width_result __result =
+      __detail::__estimate_column_width_grapheme_clustering(__it, __str.end(), __maximum, __rounding);
+
+  __result.__width_ += __ascii_size;
+  return __result;
+}
+#  else // !defined(_LIBCPP_HAS_NO_UNICODE)
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI constexpr __column_width_result<typename basic_string_view<_CharT>::const_iterator>
+__estimate_column_width(basic_string_view<_CharT> __str, size_t __maximum, __column_width_rounding) noexcept {
+  // When Unicode isn't supported assume ASCII and every code unit is one code
+  // point. In ASCII the estimated column width is always one. Thus there's no
+  // need for rounding.
+  size_t __width = std::min(__str.size(), __maximum);
+  return {__width, __str.begin() + __width};
+}
+
+#  endif // !defined(_LIBCPP_HAS_NO_UNICODE)
+
+} // namespace __format_spec
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FORMAT_PARSER_STD_FORMAT_SPEC_H
diff --git a/libcxx/include/__cxx03/__format/range_default_formatter.h b/libcxx/include/__cxx03/__format/range_default_formatter.h
new file mode 100644
index 000000000000..b35223ae9332
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/range_default_formatter.h
@@ -0,0 +1,214 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_RANGE_DEFAULT_FORMATTER_H
+#define _LIBCPP___FORMAT_RANGE_DEFAULT_FORMATTER_H
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#include <__algorithm/ranges_copy.h>
+#include <__chrono/statically_widen.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/concepts.h>
+#include <__format/formatter.h>
+#include <__format/range_formatter.h>
+#include <__iterator/back_insert_iterator.h>
+#include <__ranges/concepts.h>
+#include <__ranges/data.h>
+#include <__ranges/from_range.h>
+#include <__ranges/size.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/pair.h>
+#include <string_view>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Rp, class _CharT>
+concept __const_formattable_range =
+    ranges::input_range<const _Rp> && formattable<ranges::range_reference_t<const _Rp>, _CharT>;
+
+template <class _Rp, class _CharT>
+using __fmt_maybe_const = conditional_t<__const_formattable_range<_Rp, _CharT>, const _Rp, _Rp>;
+
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wshadow")
+_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wshadow")
+// This shadows map, set, and string.
+enum class range_format { disabled, map, set, sequence, string, debug_string };
+_LIBCPP_DIAGNOSTIC_POP
+
+// There is no definition of this struct, it's purely intended to be used to
+// generate diagnostics.
+template <class _Rp>
+struct _LIBCPP_TEMPLATE_VIS __instantiated_the_primary_template_of_format_kind;
+
+template <class _Rp>
+constexpr range_format format_kind = [] {
+  // [format.range.fmtkind]/1
+  // A program that instantiates the primary template of format_kind is ill-formed.
+  static_assert(sizeof(_Rp) != sizeof(_Rp), "create a template specialization of format_kind for your type");
+  return range_format::disabled;
+}();
+
+template <ranges::input_range _Rp>
+  requires same_as<_Rp, remove_cvref_t<_Rp>>
+inline constexpr range_format format_kind<_Rp> = [] {
+  // [format.range.fmtkind]/2
+
+  // 2.1 If same_as<remove_cvref_t<ranges::range_reference_t<R>>, R> is true,
+  // Otherwise format_kind<R> is range_format::disabled.
+  if constexpr (same_as<remove_cvref_t<ranges::range_reference_t<_Rp>>, _Rp>)
+    return range_format::disabled;
+  // 2.2 Otherwise, if the qualified-id R::key_type is valid and denotes a type:
+  else if constexpr (requires { typename _Rp::key_type; }) {
+    // 2.2.1 If the qualified-id R::mapped_type is valid and denotes a type ...
+    if constexpr (requires { typename _Rp::mapped_type; } &&
+                  // 2.2.1 ... If either U is a specialization of pair or U is a specialization
+                  // of tuple and tuple_size_v<U> == 2
+                  __fmt_pair_like<remove_cvref_t<ranges::range_reference_t<_Rp>>>)
+      return range_format::map;
+    else
+      // 2.2.2 Otherwise format_kind<R> is range_format::set.
+      return range_format::set;
+  } else
+    // 2.3 Otherwise, format_kind<R> is range_format::sequence.
+    return range_format::sequence;
+}();
+
+template <range_format _Kp, ranges::input_range _Rp, class _CharT>
+struct _LIBCPP_TEMPLATE_VIS __range_default_formatter;
+
+// Required specializations
+
+template <ranges::input_range _Rp, class _CharT>
+struct _LIBCPP_TEMPLATE_VIS __range_default_formatter<range_format::sequence, _Rp, _CharT> {
+private:
+  using __maybe_const_r = __fmt_maybe_const<_Rp, _CharT>;
+  range_formatter<remove_cvref_t<ranges::range_reference_t<__maybe_const_r>>, _CharT> __underlying_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr void set_separator(basic_string_view<_CharT> __separator) noexcept {
+    __underlying_.set_separator(__separator);
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr void
+  set_brackets(basic_string_view<_CharT> __opening_bracket, basic_string_view<_CharT> __closing_bracket) noexcept {
+    __underlying_.set_brackets(__opening_bracket, __closing_bracket);
+  }
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return __underlying_.parse(__ctx);
+  }
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator
+  format(__maybe_const_r& __range, _FormatContext& __ctx) const {
+    return __underlying_.format(__range, __ctx);
+  }
+};
+
+template <ranges::input_range _Rp, class _CharT>
+struct _LIBCPP_TEMPLATE_VIS __range_default_formatter<range_format::map, _Rp, _CharT> {
+private:
+  using __maybe_const_map = __fmt_maybe_const<_Rp, _CharT>;
+  using __element_type    = remove_cvref_t<ranges::range_reference_t<__maybe_const_map>>;
+  range_formatter<__element_type, _CharT> __underlying_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr __range_default_formatter()
+    requires(__fmt_pair_like<__element_type>)
+  {
+    __underlying_.set_brackets(_LIBCPP_STATICALLY_WIDEN(_CharT, "{"), _LIBCPP_STATICALLY_WIDEN(_CharT, "}"));
+    __underlying_.underlying().set_brackets({}, {});
+    __underlying_.underlying().set_separator(_LIBCPP_STATICALLY_WIDEN(_CharT, ": "));
+  }
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return __underlying_.parse(__ctx);
+  }
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator
+  format(__maybe_const_map& __range, _FormatContext& __ctx) const {
+    return __underlying_.format(__range, __ctx);
+  }
+};
+
+template <ranges::input_range _Rp, class _CharT>
+struct _LIBCPP_TEMPLATE_VIS __range_default_formatter<range_format::set, _Rp, _CharT> {
+private:
+  using __maybe_const_set = __fmt_maybe_const<_Rp, _CharT>;
+  using __element_type    = remove_cvref_t<ranges::range_reference_t<__maybe_const_set>>;
+  range_formatter<__element_type, _CharT> __underlying_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr __range_default_formatter() {
+    __underlying_.set_brackets(_LIBCPP_STATICALLY_WIDEN(_CharT, "{"), _LIBCPP_STATICALLY_WIDEN(_CharT, "}"));
+  }
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return __underlying_.parse(__ctx);
+  }
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator
+  format(__maybe_const_set& __range, _FormatContext& __ctx) const {
+    return __underlying_.format(__range, __ctx);
+  }
+};
+
+template <range_format _Kp, ranges::input_range _Rp, class _CharT>
+  requires(_Kp == range_format::string || _Kp == range_format::debug_string)
+struct _LIBCPP_TEMPLATE_VIS __range_default_formatter<_Kp, _Rp, _CharT> {
+private:
+  // This deviates from the Standard, there the exposition only type is
+  //   formatter<basic_string<charT>, charT> underlying_;
+  // Using a string_view allows the format function to avoid a copy of the
+  // input range when it is a contigious range.
+  formatter<basic_string_view<_CharT>, _CharT> __underlying_;
+
+public:
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    typename _ParseContext::iterator __i = __underlying_.parse(__ctx);
+    if constexpr (_Kp == range_format::debug_string)
+      __underlying_.set_debug_format();
+    return __i;
+  }
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator
+  format(conditional_t<ranges::input_range<const _Rp>, const _Rp&, _Rp&> __range, _FormatContext& __ctx) const {
+    // When the range is contiguous use a basic_string_view instead to avoid a
+    // copy of the underlying data. The basic_string_view formatter
+    // specialization is the "basic" string formatter in libc++.
+    if constexpr (ranges::contiguous_range<_Rp> && std::ranges::sized_range<_Rp>)
+      return __underlying_.format(basic_string_view<_CharT>{ranges::data(__range), ranges::size(__range)}, __ctx);
+    else
+      return __underlying_.format(basic_string<_CharT>{from_range, __range}, __ctx);
+  }
+};
+
+template <ranges::input_range _Rp, class _CharT>
+  requires(format_kind<_Rp> != range_format::disabled && formattable<ranges::range_reference_t<_Rp>, _CharT>)
+struct _LIBCPP_TEMPLATE_VIS formatter<_Rp, _CharT> : __range_default_formatter<format_kind<_Rp>, _Rp, _CharT> {};
+
+#endif //_LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_RANGE_DEFAULT_FORMATTER_H
diff --git a/libcxx/include/__cxx03/__format/range_formatter.h b/libcxx/include/__cxx03/__format/range_formatter.h
new file mode 100644
index 000000000000..691563074349
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/range_formatter.h
@@ -0,0 +1,264 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_RANGE_FORMATTER_H
+#define _LIBCPP___FORMAT_RANGE_FORMATTER_H
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#include <__algorithm/ranges_copy.h>
+#include <__chrono/statically_widen.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/buffer.h>
+#include <__format/concepts.h>
+#include <__format/format_context.h>
+#include <__format/format_error.h>
+#include <__format/formatter.h>
+#include <__format/formatter_output.h>
+#include <__format/parser_std_format_spec.h>
+#include <__iterator/back_insert_iterator.h>
+#include <__ranges/concepts.h>
+#include <__ranges/data.h>
+#include <__ranges/from_range.h>
+#include <__ranges/size.h>
+#include <__type_traits/remove_cvref.h>
+#include <string_view>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Tp, class _CharT = char>
+  requires same_as<remove_cvref_t<_Tp>, _Tp> && formattable<_Tp, _CharT>
+struct _LIBCPP_TEMPLATE_VIS range_formatter {
+  _LIBCPP_HIDE_FROM_ABI constexpr void set_separator(basic_string_view<_CharT> __separator) noexcept {
+    __separator_ = __separator;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr void
+  set_brackets(basic_string_view<_CharT> __opening_bracket, basic_string_view<_CharT> __closing_bracket) noexcept {
+    __opening_bracket_ = __opening_bracket;
+    __closing_bracket_ = __closing_bracket;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr formatter<_Tp, _CharT>& underlying() noexcept { return __underlying_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr const formatter<_Tp, _CharT>& underlying() const noexcept { return __underlying_; }
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    auto __begin = __parser_.__parse(__ctx, __format_spec::__fields_range);
+    auto __end   = __ctx.end();
+    // Note the cases where __begin == __end in this code only happens when the
+    // replacement-field has no terminating }, or when the parse is manually
+    // called with a format-spec. The former is an error and the latter means
+    // using a formatter without the format functions or print.
+    if (__begin == __end) [[unlikely]]
+      return __parse_empty_range_underlying_spec(__ctx, __begin);
+
+    // The n field overrides a possible m type, therefore delay applying the
+    // effect of n until the type has been procesed.
+    __parse_type(__begin, __end);
+    if (__parser_.__clear_brackets_)
+      set_brackets({}, {});
+    if (__begin == __end) [[unlikely]]
+      return __parse_empty_range_underlying_spec(__ctx, __begin);
+
+    bool __has_range_underlying_spec = *__begin == _CharT(':');
+    if (__has_range_underlying_spec) {
+      // range-underlying-spec:
+      //   :  format-spec
+      ++__begin;
+    } else if (__begin != __end && *__begin != _CharT('}'))
+      // When there is no underlaying range the current parse should have
+      // consumed the format-spec. If not, the not consumed input will be
+      // processed by the underlying. For example {:-} for a range in invalid,
+      // the sign field is not present. Without this check the underlying_ will
+      // get -} as input which my be valid.
+      std::__throw_format_error("The format specifier should consume the input or end with a '}'");
+
+    __ctx.advance_to(__begin);
+    __begin = __underlying_.parse(__ctx);
+
+    // This test should not be required if __has_range_underlying_spec is false.
+    // However this test makes sure the underlying formatter left the parser in
+    // a valid state. (Note this is not a full protection against evil parsers.
+    // For example
+    //   } this is test for the next argument {}
+    //   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
+    // could consume more than it should.
+    if (__begin != __end && *__begin != _CharT('}'))
+      std::__throw_format_error("The format specifier should consume the input or end with a '}'");
+
+    if (__parser_.__type_ != __format_spec::__type::__default) {
+      // [format.range.formatter]/6
+      //   If the range-type is s or ?s, then there shall be no n option and no
+      //   range-underlying-spec.
+      if (__parser_.__clear_brackets_) {
+        if (__parser_.__type_ == __format_spec::__type::__string)
+          std::__throw_format_error("The n option and type s can't be used together");
+        std::__throw_format_error("The n option and type ?s can't be used together");
+      }
+      if (__has_range_underlying_spec) {
+        if (__parser_.__type_ == __format_spec::__type::__string)
+          std::__throw_format_error("Type s and an underlying format specification can't be used together");
+        std::__throw_format_error("Type ?s and an underlying format specification can't be used together");
+      }
+    } else if (!__has_range_underlying_spec)
+      std::__set_debug_format(__underlying_);
+
+    return __begin;
+  }
+
+  template <ranges::input_range _Rp, class _FormatContext>
+    requires formattable<ranges::range_reference_t<_Rp>, _CharT> &&
+             same_as<remove_cvref_t<ranges::range_reference_t<_Rp>>, _Tp>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(_Rp&& __range, _FormatContext& __ctx) const {
+    __format_spec::__parsed_specifications<_CharT> __specs = __parser_.__get_parsed_std_specifications(__ctx);
+
+    if (!__specs.__has_width())
+      return __format_range(__range, __ctx, __specs);
+
+    // The size of the buffer needed is:
+    // - open bracket characters
+    // - close bracket character
+    // - n elements where every element may have a different size
+    // - (n -1) separators
+    // The size of the element is hard to predict, knowing the type helps but
+    // it depends on the format-spec. As an initial estimate we guess 6
+    // characters.
+    // Typically both brackets are 1 character and the separator is 2
+    // characters. Which means there will be
+    //   (n - 1) * 2 + 1 + 1 = n * 2 character
+    // So estimate 8 times the range size as buffer.
+    std::size_t __capacity_hint = 0;
+    if constexpr (std::ranges::sized_range<_Rp>)
+      __capacity_hint = 8 * ranges::size(__range);
+    __format::__retarget_buffer<_CharT> __buffer{__capacity_hint};
+    basic_format_context<typename __format::__retarget_buffer<_CharT>::__iterator, _CharT> __c{
+        __buffer.__make_output_iterator(), __ctx};
+
+    __format_range(__range, __c, __specs);
+
+    return __formatter::__write_string_no_precision(__buffer.__view(), __ctx.out(), __specs);
+  }
+
+  template <ranges::input_range _Rp, class _FormatContext>
+  typename _FormatContext::iterator _LIBCPP_HIDE_FROM_ABI
+  __format_range(_Rp&& __range, _FormatContext& __ctx, __format_spec::__parsed_specifications<_CharT> __specs) const {
+    if constexpr (same_as<_Tp, _CharT>) {
+      switch (__specs.__std_.__type_) {
+      case __format_spec::__type::__string:
+      case __format_spec::__type::__debug:
+        return __format_as_string(__range, __ctx, __specs.__std_.__type_ == __format_spec::__type::__debug);
+      default:
+        return __format_as_sequence(__range, __ctx);
+      }
+    } else
+      return __format_as_sequence(__range, __ctx);
+  }
+
+  template <ranges::input_range _Rp, class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator
+  __format_as_string(_Rp&& __range, _FormatContext& __ctx, bool __debug_format) const {
+    // When the range is contiguous use a basic_string_view instead to avoid a
+    // copy of the underlying data. The basic_string_view formatter
+    // specialization is the "basic" string formatter in libc++.
+    if constexpr (ranges::contiguous_range<_Rp> && std::ranges::sized_range<_Rp>) {
+      std::formatter<basic_string_view<_CharT>, _CharT> __formatter;
+      if (__debug_format)
+        __formatter.set_debug_format();
+      return __formatter.format(
+          basic_string_view<_CharT>{
+              ranges::data(__range),
+              ranges::size(__range),
+          },
+          __ctx);
+    } else {
+      std::formatter<basic_string<_CharT>, _CharT> __formatter;
+      if (__debug_format)
+        __formatter.set_debug_format();
+      return __formatter.format(basic_string<_CharT>{from_range, __range}, __ctx);
+    }
+  }
+
+  template <ranges::input_range _Rp, class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator
+  __format_as_sequence(_Rp&& __range, _FormatContext& __ctx) const {
+    __ctx.advance_to(ranges::copy(__opening_bracket_, __ctx.out()).out);
+    bool __use_separator = false;
+    for (auto&& __e : __range) {
+      if (__use_separator)
+        __ctx.advance_to(ranges::copy(__separator_, __ctx.out()).out);
+      else
+        __use_separator = true;
+
+      __ctx.advance_to(__underlying_.format(__e, __ctx));
+    }
+
+    return ranges::copy(__closing_bracket_, __ctx.out()).out;
+  }
+
+  __format_spec::__parser<_CharT> __parser_{.__alignment_ = __format_spec::__alignment::__left};
+
+private:
+  template <contiguous_iterator _Iterator>
+  _LIBCPP_HIDE_FROM_ABI constexpr void __parse_type(_Iterator& __begin, _Iterator __end) {
+    switch (*__begin) {
+    case _CharT('m'):
+      if constexpr (__fmt_pair_like<_Tp>) {
+        set_brackets(_LIBCPP_STATICALLY_WIDEN(_CharT, "{"), _LIBCPP_STATICALLY_WIDEN(_CharT, "}"));
+        set_separator(_LIBCPP_STATICALLY_WIDEN(_CharT, ", "));
+        ++__begin;
+      } else
+        std::__throw_format_error("Type m requires a pair or a tuple with two elements");
+      break;
+
+    case _CharT('s'):
+      if constexpr (same_as<_Tp, _CharT>) {
+        __parser_.__type_ = __format_spec::__type::__string;
+        ++__begin;
+      } else
+        std::__throw_format_error("Type s requires character type as formatting argument");
+      break;
+
+    case _CharT('?'):
+      ++__begin;
+      if (__begin == __end || *__begin != _CharT('s'))
+        std::__throw_format_error("The format specifier should consume the input or end with a '}'");
+      if constexpr (same_as<_Tp, _CharT>) {
+        __parser_.__type_ = __format_spec::__type::__debug;
+        ++__begin;
+      } else
+        std::__throw_format_error("Type ?s requires character type as formatting argument");
+    }
+  }
+
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator
+  __parse_empty_range_underlying_spec(_ParseContext& __ctx, typename _ParseContext::iterator __begin) {
+    __ctx.advance_to(__begin);
+    [[maybe_unused]] typename _ParseContext::iterator __result = __underlying_.parse(__ctx);
+    _LIBCPP_ASSERT_INTERNAL(__result == __begin,
+                            "the underlying's parse function should not advance the input beyond the end of the input");
+    return __begin;
+  }
+
+  formatter<_Tp, _CharT> __underlying_;
+  basic_string_view<_CharT> __separator_       = _LIBCPP_STATICALLY_WIDEN(_CharT, ", ");
+  basic_string_view<_CharT> __opening_bracket_ = _LIBCPP_STATICALLY_WIDEN(_CharT, "[");
+  basic_string_view<_CharT> __closing_bracket_ = _LIBCPP_STATICALLY_WIDEN(_CharT, "]");
+};
+
+#endif //_LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_RANGE_FORMATTER_H
diff --git a/libcxx/include/__cxx03/__format/unicode.h b/libcxx/include/__cxx03/__format/unicode.h
new file mode 100644
index 000000000000..de7d0fea1df5
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/unicode.h
@@ -0,0 +1,602 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_UNICODE_H
+#define _LIBCPP___FORMAT_UNICODE_H
+
+#include <__assert>
+#include <__bit/countl.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/extended_grapheme_cluster_table.h>
+#include <__format/indic_conjunct_break_table.h>
+#include <__iterator/concepts.h>
+#include <__iterator/readable_traits.h> // iter_value_t
+#include <__utility/unreachable.h>
+#include <string_view>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __unicode {
+
+// Helper struct for the result of a consume operation.
+//
+// The status value for a correct code point is 0. This allows a valid value to
+// be used without masking.
+// When the decoding fails it know the number of code units affected. For the
+// current use-cases that value is not needed, therefore it is not stored.
+// The escape routine needs the number of code units for both a valid and
+// invalid character and keeps track of it itself. Doing it in this result
+// unconditionally would give some overhead when the value is unneeded.
+struct __consume_result {
+  // When __status == __ok it contains the decoded code point.
+  // Else it contains the replacement character U+FFFD
+  char32_t __code_point : 31;
+
+  enum : char32_t {
+    // Consumed a well-formed code point.
+    __ok = 0,
+    // Encountered invalid UTF-8
+    __error = 1
+  } __status : 1 {__ok};
+};
+static_assert(sizeof(__consume_result) == sizeof(char32_t));
+
+#  ifndef _LIBCPP_HAS_NO_UNICODE
+
+/// Implements the grapheme cluster boundary rules
+///
+/// These rules are used to implement format's width estimation as stated in
+/// [format.string.std]/11
+///
+/// The Standard refers to UAX \#29 for Unicode 12.0.0
+/// https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundary_Rules
+///
+/// The data tables used are
+/// https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakProperty.txt
+/// https://www.unicode.org/Public/UCD/latest/ucd/emoji/emoji-data.txt
+/// https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakTest.txt (for testing only)
+
+inline constexpr char32_t __replacement_character = U'\ufffd';
+
+// The error of a consume operation.
+//
+// This sets the code point to the replacement character. This code point does
+// not participate in the grapheme clustering, so grapheme clustering code can
+// ignore the error status and always use the code point.
+inline constexpr __consume_result __consume_result_error{__replacement_character, __consume_result::__error};
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool __is_high_surrogate(char32_t __value) {
+  return __value >= 0xd800 && __value <= 0xdbff;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool __is_low_surrogate(char32_t __value) {
+  return __value >= 0xdc00 && __value <= 0xdfff;
+}
+
+// https://www.unicode.org/glossary/#surrogate_code_point
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_surrogate(char32_t __value) {
+  return __value >= 0xd800 && __value <= 0xdfff;
+}
+
+// https://www.unicode.org/glossary/#code_point
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_code_point(char32_t __value) {
+  return __value <= 0x10ffff;
+}
+
+// https://www.unicode.org/glossary/#unicode_scalar_value
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_scalar_value(char32_t __value) {
+  return __unicode::__is_code_point(__value) && !__unicode::__is_surrogate(__value);
+}
+
+template <contiguous_iterator _Iterator>
+  requires same_as<iter_value_t<_Iterator>, char>
+_LIBCPP_HIDE_FROM_ABI constexpr bool __is_continuation(_Iterator __char, int __count) {
+  do {
+    if ((*__char & 0b1100'0000) != 0b1000'0000)
+      return false;
+    --__count;
+    ++__char;
+  } while (__count);
+  return true;
+}
+
+/// Helper class to extract a code unit from a Unicode character range.
+///
+/// The stored range is a view. There are multiple specialization for different
+/// character types.
+template <class _CharT>
+class __code_point_view;
+
+/// UTF-8 specialization.
+template <>
+class __code_point_view<char> {
+  using _Iterator = basic_string_view<char>::const_iterator;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last)
+      : __first_(__first), __last_(__last) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; }
+
+  // https://www.unicode.org/versions/latest/ch03.pdf#G7404
+  // Based on Table 3-7, Well-Formed UTF-8 Byte Sequences
+  //
+  // Code Points        First Byte Second Byte Third Byte Fourth Byte  Remarks
+  // U+0000..U+007F     00..7F                                         U+0000..U+007F 1 code unit range
+  //                    C0..C1     80..BF                              invalid overlong encoding
+  // U+0080..U+07FF     C2..DF     80..BF                              U+0080..U+07FF 2 code unit range
+  //                    E0         80..9F      80..BF                  invalid overlong encoding
+  // U+0800..U+0FFF     E0         A0..BF      80..BF                  U+0800..U+FFFF 3 code unit range
+  // U+1000..U+CFFF     E1..EC     80..BF      80..BF
+  // U+D000..U+D7FF     ED         80..9F      80..BF
+  // U+D800..U+DFFF     ED         A0..BF      80..BF                  invalid encoding of surrogate code point
+  // U+E000..U+FFFF     EE..EF     80..BF      80..BF
+  //                    F0         80..8F      80..BF     80..BF       invalid overlong encoding
+  // U+10000..U+3FFFF   F0         90..BF      80..BF     80..BF       U+10000..U+10FFFF 4 code unit range
+  // U+40000..U+FFFFF   F1..F3     80..BF      80..BF     80..BF
+  // U+100000..U+10FFFF F4         80..8F      80..BF     80..BF
+  //                    F4         90..BF      80..BF     80..BF       U+110000.. invalid code point range
+  //
+  // Unlike other parsers, these invalid entries are tested after decoding.
+  // - The parser always needs to consume these code units
+  // - The code is optimized for well-formed UTF-8
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept {
+    _LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input");
+
+    // Based on the number of leading 1 bits the number of code units in the
+    // code point can be determined. See
+    // https://en.wikipedia.org/wiki/UTF-8#Encoding
+    switch (std::countl_one(static_cast<unsigned char>(*__first_))) {
+    case 0:
+      return {static_cast<unsigned char>(*__first_++)};
+
+    case 2: {
+      if (__last_ - __first_ < 2 || !__unicode::__is_continuation(__first_ + 1, 1)) [[unlikely]]
+        break;
+
+      char32_t __value = static_cast<unsigned char>(*__first_++) & 0x1f;
+      __value <<= 6;
+      __value |= static_cast<unsigned char>(*__first_++) & 0x3f;
+
+      // These values should be encoded in 1 UTF-8 code unit.
+      if (__value < 0x0080) [[unlikely]]
+        return __consume_result_error;
+
+      return {__value};
+    }
+
+    case 3: {
+      if (__last_ - __first_ < 3 || !__unicode::__is_continuation(__first_ + 1, 2)) [[unlikely]]
+        break;
+
+      char32_t __value = static_cast<unsigned char>(*__first_++) & 0x0f;
+      __value <<= 6;
+      __value |= static_cast<unsigned char>(*__first_++) & 0x3f;
+      __value <<= 6;
+      __value |= static_cast<unsigned char>(*__first_++) & 0x3f;
+
+      // These values should be encoded in 1 or 2 UTF-8 code units.
+      if (__value < 0x0800) [[unlikely]]
+        return __consume_result_error;
+
+      // A surrogate value is always encoded in 3 UTF-8 code units.
+      if (__unicode::__is_surrogate(__value)) [[unlikely]]
+        return __consume_result_error;
+
+      return {__value};
+    }
+
+    case 4: {
+      if (__last_ - __first_ < 4 || !__unicode::__is_continuation(__first_ + 1, 3)) [[unlikely]]
+        break;
+
+      char32_t __value = static_cast<unsigned char>(*__first_++) & 0x07;
+      __value <<= 6;
+      __value |= static_cast<unsigned char>(*__first_++) & 0x3f;
+      __value <<= 6;
+      __value |= static_cast<unsigned char>(*__first_++) & 0x3f;
+      __value <<= 6;
+      __value |= static_cast<unsigned char>(*__first_++) & 0x3f;
+
+      // These values should be encoded in 1, 2, or 3 UTF-8 code units.
+      if (__value < 0x10000) [[unlikely]]
+        return __consume_result_error;
+
+      // A value too large is always encoded in 4 UTF-8 code units.
+      if (!__unicode::__is_code_point(__value)) [[unlikely]]
+        return __consume_result_error;
+
+      return {__value};
+    }
+    }
+    // An invalid number of leading ones can be garbage or a code unit in the
+    // middle of a code point. By consuming one code unit the parser may get
+    // "in sync" after a few code units.
+    ++__first_;
+    return __consume_result_error;
+  }
+
+private:
+  _Iterator __first_;
+  _Iterator __last_;
+};
+
+#    ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+_LIBCPP_HIDE_FROM_ABI constexpr bool __is_surrogate_pair_high(wchar_t __value) {
+  return __value >= 0xd800 && __value <= 0xdbff;
+}
+
+_LIBCPP_HIDE_FROM_ABI constexpr bool __is_surrogate_pair_low(wchar_t __value) {
+  return __value >= 0xdc00 && __value <= 0xdfff;
+}
+
+/// This specialization depends on the size of wchar_t
+/// - 2 UTF-16 (for example Windows and AIX)
+/// - 4 UTF-32 (for example Linux)
+template <>
+class __code_point_view<wchar_t> {
+  using _Iterator = typename basic_string_view<wchar_t>::const_iterator;
+
+public:
+  static_assert(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4, "sizeof(wchar_t) has a not implemented value");
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last)
+      : __first_(__first), __last_(__last) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept {
+    _LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input");
+
+    char32_t __value = static_cast<char32_t>(*__first_++);
+    if constexpr (sizeof(wchar_t) == 2) {
+      if (__unicode::__is_low_surrogate(__value)) [[unlikely]]
+        return __consume_result_error;
+
+      if (__unicode::__is_high_surrogate(__value)) {
+        if (__first_ == __last_ || !__unicode::__is_low_surrogate(static_cast<char32_t>(*__first_))) [[unlikely]]
+          return __consume_result_error;
+
+        __value -= 0xd800;
+        __value <<= 10;
+        __value += static_cast<char32_t>(*__first_++) - 0xdc00;
+        __value += 0x10000;
+
+        if (!__unicode::__is_code_point(__value)) [[unlikely]]
+          return __consume_result_error;
+      }
+    } else {
+      if (!__unicode::__is_scalar_value(__value)) [[unlikely]]
+        return __consume_result_error;
+    }
+
+    return {__value};
+  }
+
+private:
+  _Iterator __first_;
+  _Iterator __last_;
+};
+#    endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+// State machine to implement the Extended Grapheme Cluster Boundary
+//
+// The exact rules may change between Unicode versions.
+// This implements the extended rules see
+// https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries
+class __extended_grapheme_cluster_break {
+  using __EGC_property  = __extended_grapheme_custer_property_boundary::__property;
+  using __inCB_property = __indic_conjunct_break::__property;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __extended_grapheme_cluster_break(char32_t __first_code_point)
+      : __prev_code_point_(__first_code_point),
+        __prev_property_(__extended_grapheme_custer_property_boundary::__get_property(__first_code_point)) {
+    // Initializes the active rule.
+    if (__prev_property_ == __EGC_property::__Extended_Pictographic)
+      __active_rule_ = __rule::__GB11_emoji;
+    else if (__prev_property_ == __EGC_property::__Regional_Indicator)
+      __active_rule_ = __rule::__GB12_GB13_regional_indicator;
+    else if (__indic_conjunct_break::__get_property(__first_code_point) == __inCB_property::__Consonant)
+      __active_rule_ = __rule::__GB9c_indic_conjunct_break;
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(char32_t __next_code_point) {
+    __EGC_property __next_property = __extended_grapheme_custer_property_boundary::__get_property(__next_code_point);
+    bool __result                  = __evaluate(__next_code_point, __next_property);
+    __prev_code_point_             = __next_code_point;
+    __prev_property_               = __next_property;
+    return __result;
+  }
+
+  // The code point whose break propery are considered during the next
+  // evaluation cyle.
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr char32_t __current_code_point() const { return __prev_code_point_; }
+
+private:
+  // The naming of the identifiers matches the Unicode standard.
+  // NOLINTBEGIN(readability-identifier-naming)
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  __evaluate(char32_t __next_code_point, __EGC_property __next_property) {
+    switch (__active_rule_) {
+    case __rule::__none:
+      return __evaluate_none(__next_code_point, __next_property);
+    case __rule::__GB9c_indic_conjunct_break:
+      return __evaluate_GB9c_indic_conjunct_break(__next_code_point, __next_property);
+    case __rule::__GB11_emoji:
+      return __evaluate_GB11_emoji(__next_code_point, __next_property);
+    case __rule::__GB12_GB13_regional_indicator:
+      return __evaluate_GB12_GB13_regional_indicator(__next_code_point, __next_property);
+    }
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __evaluate_none(char32_t __next_code_point, __EGC_property __next_property) {
+    // *** Break at the start and end of text, unless the text is empty. ***
+
+    _LIBCPP_ASSERT_INTERNAL(__prev_property_ != __EGC_property::__sot, "should be handled in the constructor"); // GB1
+    _LIBCPP_ASSERT_INTERNAL(__prev_property_ != __EGC_property::__eot, "should be handled by our caller");      // GB2
+
+    // *** Do not break between a CR and LF. Otherwise, break before and after controls. ***
+    if (__prev_property_ == __EGC_property::__CR && __next_property == __EGC_property::__LF) // GB3
+      return false;
+
+    if (__prev_property_ == __EGC_property::__Control || __prev_property_ == __EGC_property::__CR ||
+        __prev_property_ == __EGC_property::__LF) // GB4
+      return true;
+
+    if (__next_property == __EGC_property::__Control || __next_property == __EGC_property::__CR ||
+        __next_property == __EGC_property::__LF) // GB5
+      return true;
+
+    // *** Do not break Hangul syllable sequences. ***
+    if (__prev_property_ == __EGC_property::__L &&
+        (__next_property == __EGC_property::__L || __next_property == __EGC_property::__V ||
+         __next_property == __EGC_property::__LV || __next_property == __EGC_property::__LVT)) // GB6
+      return false;
+
+    if ((__prev_property_ == __EGC_property::__LV || __prev_property_ == __EGC_property::__V) &&
+        (__next_property == __EGC_property::__V || __next_property == __EGC_property::__T)) // GB7
+      return false;
+
+    if ((__prev_property_ == __EGC_property::__LVT || __prev_property_ == __EGC_property::__T) &&
+        __next_property == __EGC_property::__T) // GB8
+      return false;
+
+    // *** Do not break before extending characters or ZWJ. ***
+    if (__next_property == __EGC_property::__Extend || __next_property == __EGC_property::__ZWJ)
+      return false; // GB9
+
+    // *** Do not break before SpacingMarks, or after Prepend characters. ***
+    if (__next_property == __EGC_property::__SpacingMark) // GB9a
+      return false;
+
+    if (__prev_property_ == __EGC_property::__Prepend) // GB9b
+      return false;
+
+    // *** Do not break within certain combinations with Indic_Conjunct_Break (InCB)=Linker. ***
+    if (__indic_conjunct_break::__get_property(__next_code_point) == __inCB_property::__Consonant) {
+      __active_rule_                     = __rule::__GB9c_indic_conjunct_break;
+      __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant;
+      return true;
+    }
+
+    // *** Do not break within emoji modifier sequences or emoji zwj sequences. ***
+    if (__next_property == __EGC_property::__Extended_Pictographic) {
+      __active_rule_      = __rule::__GB11_emoji;
+      __GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic;
+      return true;
+    }
+
+    // *** Do not break within emoji flag sequences ***
+
+    // That is, do not break between regional indicator (RI) symbols if there
+    // is an odd number of RI characters before the break point.
+    if (__next_property == __EGC_property::__Regional_Indicator) { // GB12 + GB13
+      __active_rule_ = __rule::__GB12_GB13_regional_indicator;
+      return true;
+    }
+
+    // *** Otherwise, break everywhere. ***
+    return true; // GB999
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  __evaluate_GB9c_indic_conjunct_break(char32_t __next_code_point, __EGC_property __next_property) {
+    __inCB_property __break = __indic_conjunct_break::__get_property(__next_code_point);
+    if (__break == __inCB_property::__none) {
+      __active_rule_ = __rule::__none;
+      return __evaluate_none(__next_code_point, __next_property);
+    }
+
+    switch (__GB9c_indic_conjunct_break_state_) {
+    case __GB9c_indic_conjunct_break_state::__Consonant:
+      if (__break == __inCB_property::__Extend) {
+        return false;
+      }
+      if (__break == __inCB_property::__Linker) {
+        __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Linker;
+        return false;
+      }
+      __active_rule_ = __rule::__none;
+      return __evaluate_none(__next_code_point, __next_property);
+
+    case __GB9c_indic_conjunct_break_state::__Linker:
+      if (__break == __inCB_property::__Extend) {
+        return false;
+      }
+      if (__break == __inCB_property::__Linker) {
+        return false;
+      }
+      if (__break == __inCB_property::__Consonant) {
+        __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant;
+        return false;
+      }
+      __active_rule_ = __rule::__none;
+      return __evaluate_none(__next_code_point, __next_property);
+    }
+    __libcpp_unreachable();
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  __evaluate_GB11_emoji(char32_t __next_code_point, __EGC_property __next_property) {
+    switch (__GB11_emoji_state_) {
+    case __GB11_emoji_state::__Extended_Pictographic:
+      if (__next_property == __EGC_property::__Extend) {
+        __GB11_emoji_state_ = __GB11_emoji_state::__Extend;
+        return false;
+      }
+      [[fallthrough]];
+    case __GB11_emoji_state::__Extend:
+      if (__next_property == __EGC_property::__ZWJ) {
+        __GB11_emoji_state_ = __GB11_emoji_state::__ZWJ;
+        return false;
+      }
+      if (__next_property == __EGC_property::__Extend)
+        return false;
+      __active_rule_ = __rule::__none;
+      return __evaluate_none(__next_code_point, __next_property);
+
+    case __GB11_emoji_state::__ZWJ:
+      if (__next_property == __EGC_property::__Extended_Pictographic) {
+        __GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic;
+        return false;
+      }
+      __active_rule_ = __rule::__none;
+      return __evaluate_none(__next_code_point, __next_property);
+    }
+    __libcpp_unreachable();
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
+  __evaluate_GB12_GB13_regional_indicator(char32_t __next_code_point, __EGC_property __next_property) {
+    __active_rule_ = __rule::__none;
+    if (__next_property == __EGC_property::__Regional_Indicator)
+      return false;
+    return __evaluate_none(__next_code_point, __next_property);
+  }
+
+  char32_t __prev_code_point_;
+  __EGC_property __prev_property_;
+
+  enum class __rule {
+    __none,
+    __GB9c_indic_conjunct_break,
+    __GB11_emoji,
+    __GB12_GB13_regional_indicator,
+  };
+  __rule __active_rule_ = __rule::__none;
+
+  enum class __GB11_emoji_state {
+    __Extended_Pictographic,
+    __Extend,
+    __ZWJ,
+  };
+  __GB11_emoji_state __GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic;
+
+  enum class __GB9c_indic_conjunct_break_state {
+    __Consonant,
+    __Linker,
+  };
+
+  __GB9c_indic_conjunct_break_state __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant;
+
+  // NOLINTEND(readability-identifier-naming)
+};
+
+/// Helper class to extract an extended grapheme cluster from a Unicode character range.
+///
+/// This function is used to determine the column width of an extended grapheme
+/// cluster. In order to do that only the first code point is evaluated.
+/// Therefore only this code point is extracted.
+template <class _CharT>
+class __extended_grapheme_cluster_view {
+  using _Iterator = typename basic_string_view<_CharT>::const_iterator;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __extended_grapheme_cluster_view(_Iterator __first, _Iterator __last)
+      : __code_point_view_(__first, __last), __at_break_(__code_point_view_.__consume().__code_point) {}
+
+  struct __cluster {
+    /// The first code point of the extended grapheme cluster.
+    ///
+    /// The first code point is used to estimate the width of the extended
+    /// grapheme cluster.
+    char32_t __code_point_;
+
+    /// Points one beyond the last code unit in the extended grapheme cluster.
+    ///
+    /// It's expected the caller has the start position and thus can determine
+    /// the code unit range of the extended grapheme cluster.
+    _Iterator __last_;
+  };
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __cluster __consume() {
+    char32_t __code_point = __at_break_.__current_code_point();
+    _Iterator __position  = __code_point_view_.__position();
+    while (!__code_point_view_.__at_end()) {
+      if (__at_break_(__code_point_view_.__consume().__code_point))
+        break;
+      __position = __code_point_view_.__position();
+    }
+    return {__code_point, __position};
+  }
+
+private:
+  __code_point_view<_CharT> __code_point_view_;
+  __extended_grapheme_cluster_break __at_break_;
+};
+
+template <contiguous_iterator _Iterator>
+__extended_grapheme_cluster_view(_Iterator, _Iterator) -> __extended_grapheme_cluster_view<iter_value_t<_Iterator>>;
+
+#  else //  _LIBCPP_HAS_NO_UNICODE
+
+// For ASCII every character is a "code point".
+// This makes it easier to write code agnostic of the _LIBCPP_HAS_NO_UNICODE define.
+template <class _CharT>
+class __code_point_view {
+  using _Iterator = typename basic_string_view<_CharT>::const_iterator;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last)
+      : __first_(__first), __last_(__last) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept {
+    _LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input");
+    return {static_cast<char32_t>(*__first_++)};
+  }
+
+private:
+  _Iterator __first_;
+  _Iterator __last_;
+};
+
+#  endif //  _LIBCPP_HAS_NO_UNICODE
+
+} // namespace __unicode
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_UNICODE_H
diff --git a/libcxx/include/__cxx03/__format/width_estimation_table.h b/libcxx/include/__cxx03/__format/width_estimation_table.h
new file mode 100644
index 000000000000..11f61dea18d6
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/width_estimation_table.h
@@ -0,0 +1,270 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// WARNING, this entire header is generated by
+// utils/generate_width_estimation_table.py
+// DO NOT MODIFY!
+
+// UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
+//
+// See Terms of Use <https://www.unicode.org/copyright.html>
+// for definitions of Unicode Inc.'s Data Files and Software.
+//
+// NOTICE TO USER: Carefully read the following legal agreement.
+// BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S
+// DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"),
+// YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
+// TERMS AND CONDITIONS OF THIS AGREEMENT.
+// IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE
+// THE DATA FILES OR SOFTWARE.
+//
+// COPYRIGHT AND PERMISSION NOTICE
+//
+// Copyright (c) 1991-2022 Unicode, Inc. All rights reserved.
+// Distributed under the Terms of Use in https://www.unicode.org/copyright.html.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of the Unicode data files and any associated documentation
+// (the "Data Files") or Unicode software and any associated documentation
+// (the "Software") to deal in the Data Files or Software
+// without restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, and/or sell copies of
+// the Data Files or Software, and to permit persons to whom the Data Files
+// or Software are furnished to do so, provided that either
+// (a) this copyright and permission notice appear with all copies
+// of the Data Files or Software, or
+// (b) this copyright and permission notice appear in associated
+// Documentation.
+//
+// THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
+// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT OF THIRD PARTY RIGHTS.
+// IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
+// NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
+// DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+// DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+//
+// Except as contained in this notice, the name of a copyright holder
+// shall not be used in advertising or otherwise to promote the sale,
+// use or other dealings in these Data Files or Software without prior
+// written authorization of the copyright holder.
+
+#ifndef _LIBCPP___FORMAT_WIDTH_ESTIMATION_TABLE_H
+#define _LIBCPP___FORMAT_WIDTH_ESTIMATION_TABLE_H
+
+#include <__algorithm/ranges_upper_bound.h>
+#include <__config>
+#include <cstddef>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace __width_estimation_table {
+
+/// The entries of the characters with an estimated width of 2.
+///
+/// Contains the entries for [format.string.std]/12
+///  -  Any code point with the East_Asian_Width="W" or East_Asian_Width="F"
+///     Derived Extracted Property as described by UAX #44
+/// - U+4DC0 - U+4DFF (Yijing Hexagram Symbols)
+/// - U+1F300 - U+1F5FF (Miscellaneous Symbols and Pictographs)
+/// - U+1F900 - U+1F9FF (Supplemental Symbols and Pictographs)
+///
+/// The data is generated from
+/// - https://www.unicode.org/Public/UCD/latest/ucd/EastAsianWidth.txt
+/// - The "overrides" in [format.string.std]/12
+///
+/// The format of EastAsianWidth.txt is two fields separated by a semicolon.
+/// Field 0: Unicode code point value or range of code point values
+/// Field 1: East_Asian_Width property, consisting of one of the following values:
+///         "A", "F", "H", "N", "Na", "W"
+///  - All code points, assigned or unassigned, that are not listed
+///      explicitly are given the value "N".
+///  - The unassigned code points in the following blocks default to "W":
+///         CJK Unified Ideographs Extension A: U+3400..U+4DBF
+///         CJK Unified Ideographs:             U+4E00..U+9FFF
+///         CJK Compatibility Ideographs:       U+F900..U+FAFF
+///  - All undesignated code points in Planes 2 and 3, whether inside or
+///      outside of allocated blocks, default to "W":
+///         Plane 2:                            U+20000..U+2FFFD
+///         Plane 3:                            U+30000..U+3FFFD
+///
+/// The table is similar to the table
+///  __extended_grapheme_custer_property_boundary::__entries
+/// which explains the details of these classes. The only difference is this
+/// table lacks a property, thus having more bits available for the size.
+///
+/// The maximum code point that has an estimated width of 2 is U+3FFFD. This
+/// value can be encoded in 18 bits. Thus the upper 3 bits of the code point
+/// are always 0. These 3 bits are used to enlarge the offset range. This
+/// optimization reduces the table in Unicode 15 from 184 to 104 entries,
+/// saving 320 bytes.
+///
+/// The data has 2 values:
+/// - bits [0, 13] The size of the range, allowing 16384 elements.
+/// - bits [14, 31] The lower bound code point of the range. The upper bound of
+///   the range is lower bound + size.
+_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[107] = {
+    0x0440005f /* 00001100 - 0000115f [   96] */, //
+    0x08c68001 /* 0000231a - 0000231b [    2] */, //
+    0x08ca4001 /* 00002329 - 0000232a [    2] */, //
+    0x08fa4003 /* 000023e9 - 000023ec [    4] */, //
+    0x08fc0000 /* 000023f0 - 000023f0 [    1] */, //
+    0x08fcc000 /* 000023f3 - 000023f3 [    1] */, //
+    0x097f4001 /* 000025fd - 000025fe [    2] */, //
+    0x09850001 /* 00002614 - 00002615 [    2] */, //
+    0x0992000b /* 00002648 - 00002653 [   12] */, //
+    0x099fc000 /* 0000267f - 0000267f [    1] */, //
+    0x09a4c000 /* 00002693 - 00002693 [    1] */, //
+    0x09a84000 /* 000026a1 - 000026a1 [    1] */, //
+    0x09aa8001 /* 000026aa - 000026ab [    2] */, //
+    0x09af4001 /* 000026bd - 000026be [    2] */, //
+    0x09b10001 /* 000026c4 - 000026c5 [    2] */, //
+    0x09b38000 /* 000026ce - 000026ce [    1] */, //
+    0x09b50000 /* 000026d4 - 000026d4 [    1] */, //
+    0x09ba8000 /* 000026ea - 000026ea [    1] */, //
+    0x09bc8001 /* 000026f2 - 000026f3 [    2] */, //
+    0x09bd4000 /* 000026f5 - 000026f5 [    1] */, //
+    0x09be8000 /* 000026fa - 000026fa [    1] */, //
+    0x09bf4000 /* 000026fd - 000026fd [    1] */, //
+    0x09c14000 /* 00002705 - 00002705 [    1] */, //
+    0x09c28001 /* 0000270a - 0000270b [    2] */, //
+    0x09ca0000 /* 00002728 - 00002728 [    1] */, //
+    0x09d30000 /* 0000274c - 0000274c [    1] */, //
+    0x09d38000 /* 0000274e - 0000274e [    1] */, //
+    0x09d4c002 /* 00002753 - 00002755 [    3] */, //
+    0x09d5c000 /* 00002757 - 00002757 [    1] */, //
+    0x09e54002 /* 00002795 - 00002797 [    3] */, //
+    0x09ec0000 /* 000027b0 - 000027b0 [    1] */, //
+    0x09efc000 /* 000027bf - 000027bf [    1] */, //
+    0x0ac6c001 /* 00002b1b - 00002b1c [    2] */, //
+    0x0ad40000 /* 00002b50 - 00002b50 [    1] */, //
+    0x0ad54000 /* 00002b55 - 00002b55 [    1] */, //
+    0x0ba00019 /* 00002e80 - 00002e99 [   26] */, //
+    0x0ba6c058 /* 00002e9b - 00002ef3 [   89] */, //
+    0x0bc000d5 /* 00002f00 - 00002fd5 [  214] */, //
+    0x0bfc004e /* 00002ff0 - 0000303e [   79] */, //
+    0x0c104055 /* 00003041 - 00003096 [   86] */, //
+    0x0c264066 /* 00003099 - 000030ff [  103] */, //
+    0x0c41402a /* 00003105 - 0000312f [   43] */, //
+    0x0c4c405d /* 00003131 - 0000318e [   94] */, //
+    0x0c640053 /* 00003190 - 000031e3 [   84] */, //
+    0x0c7bc02f /* 000031ef - 0000321e [   48] */, //
+    0x0c880027 /* 00003220 - 00003247 [   40] */, //
+    0x0c943fff /* 00003250 - 0000724f [16384] */, //
+    0x1c94323c /* 00007250 - 0000a48c [12861] */, //
+    0x29240036 /* 0000a490 - 0000a4c6 [   55] */, //
+    0x2a58001c /* 0000a960 - 0000a97c [   29] */, //
+    0x2b002ba3 /* 0000ac00 - 0000d7a3 [11172] */, //
+    0x3e4001ff /* 0000f900 - 0000faff [  512] */, //
+    0x3f840009 /* 0000fe10 - 0000fe19 [   10] */, //
+    0x3f8c0022 /* 0000fe30 - 0000fe52 [   35] */, //
+    0x3f950012 /* 0000fe54 - 0000fe66 [   19] */, //
+    0x3f9a0003 /* 0000fe68 - 0000fe6b [    4] */, //
+    0x3fc0405f /* 0000ff01 - 0000ff60 [   96] */, //
+    0x3ff80006 /* 0000ffe0 - 0000ffe6 [    7] */, //
+    0x5bf80004 /* 00016fe0 - 00016fe4 [    5] */, //
+    0x5bfc0001 /* 00016ff0 - 00016ff1 [    2] */, //
+    0x5c0017f7 /* 00017000 - 000187f7 [ 6136] */, //
+    0x620004d5 /* 00018800 - 00018cd5 [ 1238] */, //
+    0x63400008 /* 00018d00 - 00018d08 [    9] */, //
+    0x6bfc0003 /* 0001aff0 - 0001aff3 [    4] */, //
+    0x6bfd4006 /* 0001aff5 - 0001affb [    7] */, //
+    0x6bff4001 /* 0001affd - 0001affe [    2] */, //
+    0x6c000122 /* 0001b000 - 0001b122 [  291] */, //
+    0x6c4c8000 /* 0001b132 - 0001b132 [    1] */, //
+    0x6c540002 /* 0001b150 - 0001b152 [    3] */, //
+    0x6c554000 /* 0001b155 - 0001b155 [    1] */, //
+    0x6c590003 /* 0001b164 - 0001b167 [    4] */, //
+    0x6c5c018b /* 0001b170 - 0001b2fb [  396] */, //
+    0x7c010000 /* 0001f004 - 0001f004 [    1] */, //
+    0x7c33c000 /* 0001f0cf - 0001f0cf [    1] */, //
+    0x7c638000 /* 0001f18e - 0001f18e [    1] */, //
+    0x7c644009 /* 0001f191 - 0001f19a [   10] */, //
+    0x7c800002 /* 0001f200 - 0001f202 [    3] */, //
+    0x7c84002b /* 0001f210 - 0001f23b [   44] */, //
+    0x7c900008 /* 0001f240 - 0001f248 [    9] */, //
+    0x7c940001 /* 0001f250 - 0001f251 [    2] */, //
+    0x7c980005 /* 0001f260 - 0001f265 [    6] */, //
+    0x7cc0034f /* 0001f300 - 0001f64f [  848] */, //
+    0x7da00045 /* 0001f680 - 0001f6c5 [   70] */, //
+    0x7db30000 /* 0001f6cc - 0001f6cc [    1] */, //
+    0x7db40002 /* 0001f6d0 - 0001f6d2 [    3] */, //
+    0x7db54002 /* 0001f6d5 - 0001f6d7 [    3] */, //
+    0x7db70003 /* 0001f6dc - 0001f6df [    4] */, //
+    0x7dbac001 /* 0001f6eb - 0001f6ec [    2] */, //
+    0x7dbd0008 /* 0001f6f4 - 0001f6fc [    9] */, //
+    0x7df8000b /* 0001f7e0 - 0001f7eb [   12] */, //
+    0x7dfc0000 /* 0001f7f0 - 0001f7f0 [    1] */, //
+    0x7e4000ff /* 0001f900 - 0001f9ff [  256] */, //
+    0x7e9c000c /* 0001fa70 - 0001fa7c [   13] */, //
+    0x7ea00008 /* 0001fa80 - 0001fa88 [    9] */, //
+    0x7ea4002d /* 0001fa90 - 0001fabd [   46] */, //
+    0x7eafc006 /* 0001fabf - 0001fac5 [    7] */, //
+    0x7eb3800d /* 0001face - 0001fadb [   14] */, //
+    0x7eb80008 /* 0001fae0 - 0001fae8 [    9] */, //
+    0x7ebc0008 /* 0001faf0 - 0001faf8 [    9] */, //
+    0x80003fff /* 00020000 - 00023fff [16384] */, //
+    0x90003fff /* 00024000 - 00027fff [16384] */, //
+    0xa0003fff /* 00028000 - 0002bfff [16384] */, //
+    0xb0003ffd /* 0002c000 - 0002fffd [16382] */, //
+    0xc0003fff /* 00030000 - 00033fff [16384] */, //
+    0xd0003fff /* 00034000 - 00037fff [16384] */, //
+    0xe0003fff /* 00038000 - 0003bfff [16384] */, //
+    0xf0003ffd /* 0003c000 - 0003fffd [16382] */};
+
+/// The upper bound entry of EastAsianWidth.txt.
+///
+/// Values greater than this value may have more than 18 significant bits.
+/// They always have a width of 1. This property makes it possible to store
+/// the table in its compact form.
+inline constexpr uint32_t __table_upper_bound = 0x0003fffd;
+
+/// Returns the estimated width of a Unicode code point.
+///
+/// \\pre The code point is a valid Unicode code point.
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr int __estimated_width(const char32_t __code_point) noexcept {
+  // Since __table_upper_bound contains the unshifted range do the
+  // comparison without shifting.
+  if (__code_point > __table_upper_bound) [[unlikely]]
+    return 1;
+
+  // When the code-point is less than the first element in the table
+  // the lookup is quite expensive. Since quite some scripts are in
+  // that range, it makes sense to validate that first.
+  // The std_format_spec_string_unicode benchmark gives a measurable
+  // improvement.
+  if (__code_point < (__entries[0] >> 14))
+    return 1;
+
+  ptrdiff_t __i = std::ranges::upper_bound(__entries, (__code_point << 14) | 0x3fffu) - __entries;
+  if (__i == 0)
+    return 1;
+
+  --__i;
+  uint32_t __upper_bound = (__entries[__i] >> 14) + (__entries[__i] & 0x3fffu);
+  return 1 + (__code_point <= __upper_bound);
+}
+
+} // namespace __width_estimation_table
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FORMAT_WIDTH_ESTIMATION_TABLE_H
diff --git a/libcxx/include/__cxx03/__format/write_escaped.h b/libcxx/include/__cxx03/__format/write_escaped.h
new file mode 100644
index 000000000000..052ea98c3c3b
--- /dev/null
+++ b/libcxx/include/__cxx03/__format/write_escaped.h
@@ -0,0 +1,242 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_WRITE_ESCAPED_H
+#define _LIBCPP___FORMAT_WRITE_ESCAPED_H
+
+#include <__algorithm/ranges_copy.h>
+#include <__algorithm/ranges_for_each.h>
+#include <__charconv/to_chars_integral.h>
+#include <__charconv/to_chars_result.h>
+#include <__chrono/statically_widen.h>
+#include <__format/escaped_output_table.h>
+#include <__format/formatter_output.h>
+#include <__format/parser_std_format_spec.h>
+#include <__format/unicode.h>
+#include <__iterator/back_insert_iterator.h>
+#include <__memory/addressof.h>
+#include <__system_error/errc.h>
+#include <__type_traits/make_unsigned.h>
+#include <__utility/move.h>
+#include <string_view>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __formatter {
+
+#if _LIBCPP_STD_VER >= 20
+
+/// Writes a string using format's width estimation algorithm.
+///
+/// \note When \c _LIBCPP_HAS_NO_UNICODE is defined the function assumes the
+/// input is ASCII.
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI auto
+__write_string(basic_string_view<_CharT> __str,
+               output_iterator<const _CharT&> auto __out_it,
+               __format_spec::__parsed_specifications<_CharT> __specs) -> decltype(__out_it) {
+  if (!__specs.__has_precision())
+    return __formatter::__write_string_no_precision(__str, std::move(__out_it), __specs);
+
+  int __size = __formatter::__truncate(__str, __specs.__precision_);
+
+  return __formatter::__write(__str.begin(), __str.end(), std::move(__out_it), __specs, __size);
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+#if _LIBCPP_STD_VER >= 23
+
+struct __nul_terminator {};
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI bool operator==(const _CharT* __cstr, __nul_terminator) {
+  return *__cstr == _CharT('\0');
+}
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI void
+__write_escaped_code_unit(basic_string<_CharT>& __str, char32_t __value, const _CharT* __prefix) {
+  back_insert_iterator __out_it{__str};
+  std::ranges::copy(__prefix, __nul_terminator{}, __out_it);
+
+  char __buffer[8];
+  to_chars_result __r = std::to_chars(std::begin(__buffer), std::end(__buffer), __value, 16);
+  _LIBCPP_ASSERT_INTERNAL(__r.ec == errc(0), "Internal buffer too small");
+  std::ranges::copy(std::begin(__buffer), __r.ptr, __out_it);
+
+  __str += _CharT('}');
+}
+
+// [format.string.escaped]/2.2.1.2
+// ...
+// then the sequence \u{hex-digit-sequence} is appended to E, where
+// hex-digit-sequence is the shortest hexadecimal representation of C using
+// lower-case hexadecimal digits.
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI void __write_well_formed_escaped_code_unit(basic_string<_CharT>& __str, char32_t __value) {
+  __formatter::__write_escaped_code_unit(__str, __value, _LIBCPP_STATICALLY_WIDEN(_CharT, "\\u{"));
+}
+
+// [format.string.escaped]/2.2.3
+// Otherwise (X is a sequence of ill-formed code units), each code unit U is
+// appended to E in order as the sequence \x{hex-digit-sequence}, where
+// hex-digit-sequence is the shortest hexadecimal representation of U using
+// lower-case hexadecimal digits.
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI void __write_escape_ill_formed_code_unit(basic_string<_CharT>& __str, char32_t __value) {
+  __formatter::__write_escaped_code_unit(__str, __value, _LIBCPP_STATICALLY_WIDEN(_CharT, "\\x{"));
+}
+
+template <class _CharT>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool
+__is_escaped_sequence_written(basic_string<_CharT>& __str, bool __last_escaped, char32_t __value) {
+#  ifdef _LIBCPP_HAS_NO_UNICODE
+  // For ASCII assume everything above 127 is printable.
+  if (__value > 127)
+    return false;
+#  endif
+
+  // [format.string.escaped]/2.2.1.2.1
+  //   CE is UTF-8, UTF-16, or UTF-32 and C corresponds to a Unicode scalar
+  //   value whose Unicode property General_Category has a value in the groups
+  //   Separator (Z) or Other (C), as described by UAX #44 of the Unicode Standard,
+  if (!__escaped_output_table::__needs_escape(__value))
+    // [format.string.escaped]/2.2.1.2.2
+    //   CE is UTF-8, UTF-16, or UTF-32 and C corresponds to a Unicode scalar
+    //   value with the Unicode property Grapheme_Extend=Yes as described by UAX
+    //   #44 of the Unicode Standard and C is not immediately preceded in S by a
+    //   character P appended to E without translation to an escape sequence,
+    if (!__last_escaped || __extended_grapheme_custer_property_boundary::__get_property(__value) !=
+                               __extended_grapheme_custer_property_boundary::__property::__Extend)
+      return false;
+
+  __formatter::__write_well_formed_escaped_code_unit(__str, __value);
+  return true;
+}
+
+template <class _CharT>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr char32_t __to_char32(_CharT __value) {
+  return static_cast<make_unsigned_t<_CharT>>(__value);
+}
+
+enum class __escape_quotation_mark { __apostrophe, __double_quote };
+
+// [format.string.escaped]/2
+template <class _CharT>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool __is_escaped_sequence_written(
+    basic_string<_CharT>& __str, char32_t __value, bool __last_escaped, __escape_quotation_mark __mark) {
+  // 2.2.1.1 - Mapped character in [tab:format.escape.sequences]
+  switch (__value) {
+  case _CharT('\t'):
+    __str += _LIBCPP_STATICALLY_WIDEN(_CharT, "\\t");
+    return true;
+  case _CharT('\n'):
+    __str += _LIBCPP_STATICALLY_WIDEN(_CharT, "\\n");
+    return true;
+  case _CharT('\r'):
+    __str += _LIBCPP_STATICALLY_WIDEN(_CharT, "\\r");
+    return true;
+  case _CharT('\''):
+    if (__mark == __escape_quotation_mark::__apostrophe)
+      __str += _LIBCPP_STATICALLY_WIDEN(_CharT, R"(\')");
+    else
+      __str += __value;
+    return true;
+  case _CharT('"'):
+    if (__mark == __escape_quotation_mark::__double_quote)
+      __str += _LIBCPP_STATICALLY_WIDEN(_CharT, R"(\")");
+    else
+      __str += __value;
+    return true;
+  case _CharT('\\'):
+    __str += _LIBCPP_STATICALLY_WIDEN(_CharT, R"(\\)");
+    return true;
+
+  // 2.2.1.2 - Space
+  case _CharT(' '):
+    __str += __value;
+    return true;
+  }
+
+  // 2.2.2
+  //   Otherwise, if X is a shift sequence, the effect on E and further
+  //   decoding of S is unspecified.
+  // For now shift sequences are ignored and treated as Unicode. Other parts
+  // of the format library do the same. It's unknown how ostream treats them.
+  // TODO FMT determine what to do with shift sequences.
+
+  // 2.2.1.2.1 and 2.2.1.2.2 - Escape
+  return __formatter::__is_escaped_sequence_written(__str, __last_escaped, __formatter::__to_char32(__value));
+}
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI void
+__escape(basic_string<_CharT>& __str, basic_string_view<_CharT> __values, __escape_quotation_mark __mark) {
+  __unicode::__code_point_view<_CharT> __view{__values.begin(), __values.end()};
+
+  // When the first code unit has the property Grapheme_Extend=Yes it needs to
+  // be escaped. This happens when the previous code unit was also escaped.
+  bool __escape = true;
+  while (!__view.__at_end()) {
+    auto __first                                  = __view.__position();
+    typename __unicode::__consume_result __result = __view.__consume();
+    if (__result.__status == __unicode::__consume_result::__ok) {
+      __escape = __formatter::__is_escaped_sequence_written(__str, __result.__code_point, __escape, __mark);
+      if (!__escape)
+        // 2.2.1.3 - Add the character
+        ranges::copy(__first, __view.__position(), std::back_insert_iterator(__str));
+    } else {
+      // 2.2.3 sequence of ill-formed code units
+      ranges::for_each(__first, __view.__position(), [&](_CharT __value) {
+        __formatter::__write_escape_ill_formed_code_unit(__str, __formatter::__to_char32(__value));
+      });
+    }
+  }
+}
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI auto
+__format_escaped_char(_CharT __value,
+                      output_iterator<const _CharT&> auto __out_it,
+                      __format_spec::__parsed_specifications<_CharT> __specs) -> decltype(__out_it) {
+  basic_string<_CharT> __str;
+  __str += _CharT('\'');
+  __formatter::__escape(__str, basic_string_view{std::addressof(__value), 1}, __escape_quotation_mark::__apostrophe);
+  __str += _CharT('\'');
+  return __formatter::__write(__str.data(), __str.data() + __str.size(), std::move(__out_it), __specs, __str.size());
+}
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI auto
+__format_escaped_string(basic_string_view<_CharT> __values,
+                        output_iterator<const _CharT&> auto __out_it,
+                        __format_spec::__parsed_specifications<_CharT> __specs) -> decltype(__out_it) {
+  basic_string<_CharT> __str;
+  __str += _CharT('"');
+  __formatter::__escape(__str, __values, __escape_quotation_mark::__double_quote);
+  __str += _CharT('"');
+  return __formatter::__write_string(basic_string_view{__str}, std::move(__out_it), __specs);
+}
+
+#endif // _LIBCPP_STD_VER >= 23
+
+} // namespace __formatter
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FORMAT_WRITE_ESCAPED_H
diff --git a/libcxx/include/__cxx03/__functional/binary_function.h b/libcxx/include/__cxx03/__functional/binary_function.h
new file mode 100644
index 000000000000..ddee3b170311
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/binary_function.h
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_BINARY_FUNCTION_H
+#define _LIBCPP___FUNCTIONAL_BINARY_FUNCTION_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION)
+
+template <class _Arg1, class _Arg2, class _Result>
+struct _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 binary_function {
+  typedef _Arg1 first_argument_type;
+  typedef _Arg2 second_argument_type;
+  typedef _Result result_type;
+};
+
+#endif // _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION)
+
+template <class _Arg1, class _Arg2, class _Result>
+struct __binary_function_keep_layout_base {
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  using first_argument_type _LIBCPP_DEPRECATED_IN_CXX17  = _Arg1;
+  using second_argument_type _LIBCPP_DEPRECATED_IN_CXX17 = _Arg2;
+  using result_type _LIBCPP_DEPRECATED_IN_CXX17          = _Result;
+#endif
+};
+
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION)
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wdeprecated-declarations")
+template <class _Arg1, class _Arg2, class _Result>
+using __binary_function = binary_function<_Arg1, _Arg2, _Result>;
+_LIBCPP_DIAGNOSTIC_POP
+#else
+template <class _Arg1, class _Arg2, class _Result>
+using __binary_function = __binary_function_keep_layout_base<_Arg1, _Arg2, _Result>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_BINARY_FUNCTION_H
diff --git a/libcxx/include/__cxx03/__functional/binary_negate.h b/libcxx/include/__cxx03/__functional/binary_negate.h
new file mode 100644
index 000000000000..ce52b5ae9fc4
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/binary_negate.h
@@ -0,0 +1,51 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_BINARY_NEGATE_H
+#define _LIBCPP___FUNCTIONAL_BINARY_NEGATE_H
+
+#include <__config>
+#include <__functional/binary_function.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_NEGATORS)
+
+template <class _Predicate>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 binary_negate
+    : public __binary_function<typename _Predicate::first_argument_type,
+                               typename _Predicate::second_argument_type,
+                               bool> {
+  _Predicate __pred_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit _LIBCPP_CONSTEXPR_SINCE_CXX14 binary_negate(const _Predicate& __pred)
+      : __pred_(__pred) {}
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(
+      const typename _Predicate::first_argument_type& __x, const typename _Predicate::second_argument_type& __y) const {
+    return !__pred_(__x, __y);
+  }
+};
+
+template <class _Predicate>
+_LIBCPP_DEPRECATED_IN_CXX17 inline _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI binary_negate<_Predicate>
+not2(const _Predicate& __pred) {
+  return binary_negate<_Predicate>(__pred);
+}
+
+#endif // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_NEGATORS)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_BINARY_NEGATE_H
diff --git a/libcxx/include/__cxx03/__functional/bind.h b/libcxx/include/__cxx03/__functional/bind.h
new file mode 100644
index 000000000000..b4f46441da50
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/bind.h
@@ -0,0 +1,296 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_BIND_H
+#define _LIBCPP___FUNCTIONAL_BIND_H
+
+#include <__config>
+#include <__functional/invoke.h>
+#include <__functional/weak_result_type.h>
+#include <__fwd/functional.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_reference_wrapper.h>
+#include <__type_traits/is_void.h>
+#include <cstddef>
+#include <tuple>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct is_bind_expression
+    : _If< _IsSame<_Tp, __remove_cvref_t<_Tp> >::value, false_type, is_bind_expression<__remove_cvref_t<_Tp> > > {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_bind_expression_v = is_bind_expression<_Tp>::value;
+#endif
+
+template <class _Tp>
+struct is_placeholder
+    : _If< _IsSame<_Tp, __remove_cvref_t<_Tp> >::value,
+           integral_constant<int, 0>,
+           is_placeholder<__remove_cvref_t<_Tp> > > {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr int is_placeholder_v = is_placeholder<_Tp>::value;
+#endif
+
+namespace placeholders {
+
+template <int _Np>
+struct __ph {};
+
+// C++17 recommends that we implement placeholders as `inline constexpr`, but allows
+// implementing them as `extern <implementation-defined>`. Libc++ implements them as
+// `extern const` in all standard modes to avoid an ABI break in C++03: making them
+// `inline constexpr` requires removing their definition in the shared library to
+// avoid ODR violations, which is an ABI break.
+//
+// In practice, since placeholders are empty, `extern const` is almost impossible
+// to distinguish from `inline constexpr` from a usage stand point.
+_LIBCPP_EXPORTED_FROM_ABI extern const __ph<1> _1;
+_LIBCPP_EXPORTED_FROM_ABI extern const __ph<2> _2;
+_LIBCPP_EXPORTED_FROM_ABI extern const __ph<3> _3;
+_LIBCPP_EXPORTED_FROM_ABI extern const __ph<4> _4;
+_LIBCPP_EXPORTED_FROM_ABI extern const __ph<5> _5;
+_LIBCPP_EXPORTED_FROM_ABI extern const __ph<6> _6;
+_LIBCPP_EXPORTED_FROM_ABI extern const __ph<7> _7;
+_LIBCPP_EXPORTED_FROM_ABI extern const __ph<8> _8;
+_LIBCPP_EXPORTED_FROM_ABI extern const __ph<9> _9;
+_LIBCPP_EXPORTED_FROM_ABI extern const __ph<10> _10;
+
+} // namespace placeholders
+
+template <int _Np>
+struct is_placeholder<placeholders::__ph<_Np> > : public integral_constant<int, _Np> {};
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Uj>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& __mu(reference_wrapper<_Tp> __t, _Uj&) {
+  return __t.get();
+}
+
+template <class _Ti, class... _Uj, size_t... _Indx>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __invoke_of<_Ti&, _Uj...>::type
+__mu_expand(_Ti& __ti, tuple<_Uj...>& __uj, __tuple_indices<_Indx...>) {
+  return __ti(std::forward<_Uj>(std::get<_Indx>(__uj))...);
+}
+
+template <class _Ti, class... _Uj, __enable_if_t<is_bind_expression<_Ti>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __invoke_of<_Ti&, _Uj...>::type
+__mu(_Ti& __ti, tuple<_Uj...>& __uj) {
+  typedef typename __make_tuple_indices<sizeof...(_Uj)>::type __indices;
+  return std::__mu_expand(__ti, __uj, __indices());
+}
+
+template <bool _IsPh, class _Ti, class _Uj>
+struct __mu_return2 {};
+
+template <class _Ti, class _Uj>
+struct __mu_return2<true, _Ti, _Uj> {
+  typedef typename tuple_element<is_placeholder<_Ti>::value - 1, _Uj>::type type;
+};
+
+template <class _Ti, class _Uj, __enable_if_t<0 < is_placeholder<_Ti>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+typename __mu_return2<0 < is_placeholder<_Ti>::value, _Ti, _Uj>::type
+__mu(_Ti&, _Uj& __uj) {
+  const size_t __indx = is_placeholder<_Ti>::value - 1;
+  return std::forward<typename tuple_element<__indx, _Uj>::type>(std::get<__indx>(__uj));
+}
+
+template <class _Ti,
+          class _Uj,
+          __enable_if_t<!is_bind_expression<_Ti>::value && is_placeholder<_Ti>::value == 0 &&
+                            !__is_reference_wrapper<_Ti>::value,
+                        int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Ti& __mu(_Ti& __ti, _Uj&) {
+  return __ti;
+}
+
+template <class _Ti, bool _IsReferenceWrapper, bool _IsBindEx, bool _IsPh, class _TupleUj>
+struct __mu_return_impl;
+
+template <bool _Invokable, class _Ti, class... _Uj>
+struct __mu_return_invokable // false
+{
+  typedef __nat type;
+};
+
+template <class _Ti, class... _Uj>
+struct __mu_return_invokable<true, _Ti, _Uj...> {
+  typedef typename __invoke_of<_Ti&, _Uj...>::type type;
+};
+
+template <class _Ti, class... _Uj>
+struct __mu_return_impl<_Ti, false, true, false, tuple<_Uj...> >
+    : public __mu_return_invokable<__invokable<_Ti&, _Uj...>::value, _Ti, _Uj...> {};
+
+template <class _Ti, class _TupleUj>
+struct __mu_return_impl<_Ti, false, false, true, _TupleUj> {
+  typedef typename tuple_element<is_placeholder<_Ti>::value - 1, _TupleUj>::type&& type;
+};
+
+template <class _Ti, class _TupleUj>
+struct __mu_return_impl<_Ti, true, false, false, _TupleUj> {
+  typedef typename _Ti::type& type;
+};
+
+template <class _Ti, class _TupleUj>
+struct __mu_return_impl<_Ti, false, false, false, _TupleUj> {
+  typedef _Ti& type;
+};
+
+template <class _Ti, class _TupleUj>
+struct __mu_return
+    : public __mu_return_impl<
+          _Ti,
+          __is_reference_wrapper<_Ti>::value,
+          is_bind_expression<_Ti>::value,
+          0 < is_placeholder<_Ti>::value && is_placeholder<_Ti>::value <= tuple_size<_TupleUj>::value,
+          _TupleUj> {};
+
+template <class _Fp, class _BoundArgs, class _TupleUj>
+struct __is_valid_bind_return {
+  static const bool value = false;
+};
+
+template <class _Fp, class... _BoundArgs, class _TupleUj>
+struct __is_valid_bind_return<_Fp, tuple<_BoundArgs...>, _TupleUj> {
+  static const bool value = __invokable<_Fp, typename __mu_return<_BoundArgs, _TupleUj>::type...>::value;
+};
+
+template <class _Fp, class... _BoundArgs, class _TupleUj>
+struct __is_valid_bind_return<_Fp, const tuple<_BoundArgs...>, _TupleUj> {
+  static const bool value = __invokable<_Fp, typename __mu_return<const _BoundArgs, _TupleUj>::type...>::value;
+};
+
+template <class _Fp, class _BoundArgs, class _TupleUj, bool = __is_valid_bind_return<_Fp, _BoundArgs, _TupleUj>::value>
+struct __bind_return;
+
+template <class _Fp, class... _BoundArgs, class _TupleUj>
+struct __bind_return<_Fp, tuple<_BoundArgs...>, _TupleUj, true> {
+  typedef typename __invoke_of< _Fp&, typename __mu_return< _BoundArgs, _TupleUj >::type... >::type type;
+};
+
+template <class _Fp, class... _BoundArgs, class _TupleUj>
+struct __bind_return<_Fp, const tuple<_BoundArgs...>, _TupleUj, true> {
+  typedef typename __invoke_of< _Fp&, typename __mu_return< const _BoundArgs, _TupleUj >::type... >::type type;
+};
+
+template <class _Fp, class _BoundArgs, size_t... _Indx, class _Args>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __bind_return<_Fp, _BoundArgs, _Args>::type
+__apply_functor(_Fp& __f, _BoundArgs& __bound_args, __tuple_indices<_Indx...>, _Args&& __args) {
+  return std::__invoke(__f, std::__mu(std::get<_Indx>(__bound_args), __args)...);
+}
+
+template <class _Fp, class... _BoundArgs>
+class __bind : public __weak_result_type<__decay_t<_Fp> > {
+protected:
+  using _Fd = __decay_t<_Fp>;
+  typedef tuple<__decay_t<_BoundArgs>...> _Td;
+
+private:
+  _Fd __f_;
+  _Td __bound_args_;
+
+  typedef typename __make_tuple_indices<sizeof...(_BoundArgs)>::type __indices;
+
+public:
+  template <
+      class _Gp,
+      class... _BA,
+      __enable_if_t<is_constructible<_Fd, _Gp>::value && !is_same<__libcpp_remove_reference_t<_Gp>, __bind>::value,
+                    int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit __bind(_Gp&& __f, _BA&&... __bound_args)
+      : __f_(std::forward<_Gp>(__f)), __bound_args_(std::forward<_BA>(__bound_args)...) {}
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __bind_return<_Fd, _Td, tuple<_Args&&...> >::type
+  operator()(_Args&&... __args) {
+    return std::__apply_functor(__f_, __bound_args_, __indices(), tuple<_Args&&...>(std::forward<_Args>(__args)...));
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  typename __bind_return<const _Fd, const _Td, tuple<_Args&&...> >::type
+  operator()(_Args&&... __args) const {
+    return std::__apply_functor(__f_, __bound_args_, __indices(), tuple<_Args&&...>(std::forward<_Args>(__args)...));
+  }
+};
+
+template <class _Fp, class... _BoundArgs>
+struct is_bind_expression<__bind<_Fp, _BoundArgs...> > : public true_type {};
+
+template <class _Rp, class _Fp, class... _BoundArgs>
+class __bind_r : public __bind<_Fp, _BoundArgs...> {
+  typedef __bind<_Fp, _BoundArgs...> base;
+  typedef typename base::_Fd _Fd;
+  typedef typename base::_Td _Td;
+
+public:
+  typedef _Rp result_type;
+
+  template <
+      class _Gp,
+      class... _BA,
+      __enable_if_t<is_constructible<_Fd, _Gp>::value && !is_same<__libcpp_remove_reference_t<_Gp>, __bind_r>::value,
+                    int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit __bind_r(_Gp&& __f, _BA&&... __bound_args)
+      : base(std::forward<_Gp>(__f), std::forward<_BA>(__bound_args)...) {}
+
+  template <
+      class... _Args,
+      __enable_if_t<is_convertible<typename __bind_return<_Fd, _Td, tuple<_Args&&...> >::type, result_type>::value ||
+                        is_void<_Rp>::value,
+                    int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 result_type operator()(_Args&&... __args) {
+    typedef __invoke_void_return_wrapper<_Rp> _Invoker;
+    return _Invoker::__call(static_cast<base&>(*this), std::forward<_Args>(__args)...);
+  }
+
+  template <class... _Args,
+            __enable_if_t<is_convertible<typename __bind_return<const _Fd, const _Td, tuple<_Args&&...> >::type,
+                                         result_type>::value ||
+                              is_void<_Rp>::value,
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 result_type operator()(_Args&&... __args) const {
+    typedef __invoke_void_return_wrapper<_Rp> _Invoker;
+    return _Invoker::__call(static_cast<base const&>(*this), std::forward<_Args>(__args)...);
+  }
+};
+
+template <class _Rp, class _Fp, class... _BoundArgs>
+struct is_bind_expression<__bind_r<_Rp, _Fp, _BoundArgs...> > : public true_type {};
+
+template <class _Fp, class... _BoundArgs>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bind<_Fp, _BoundArgs...>
+bind(_Fp&& __f, _BoundArgs&&... __bound_args) {
+  typedef __bind<_Fp, _BoundArgs...> type;
+  return type(std::forward<_Fp>(__f), std::forward<_BoundArgs>(__bound_args)...);
+}
+
+template <class _Rp, class _Fp, class... _BoundArgs>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bind_r<_Rp, _Fp, _BoundArgs...>
+bind(_Fp&& __f, _BoundArgs&&... __bound_args) {
+  typedef __bind_r<_Rp, _Fp, _BoundArgs...> type;
+  return type(std::forward<_Fp>(__f), std::forward<_BoundArgs>(__bound_args)...);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_BIND_H
diff --git a/libcxx/include/__cxx03/__functional/bind_back.h b/libcxx/include/__cxx03/__functional/bind_back.h
new file mode 100644
index 000000000000..e44768d2283c
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/bind_back.h
@@ -0,0 +1,83 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_BIND_BACK_H
+#define _LIBCPP___FUNCTIONAL_BIND_BACK_H
+
+#include <__config>
+#include <__functional/invoke.h>
+#include <__functional/perfect_forward.h>
+#include <__type_traits/decay.h>
+#include <__utility/forward.h>
+#include <__utility/integer_sequence.h>
+#include <tuple>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <size_t _NBound, class = make_index_sequence<_NBound>>
+struct __bind_back_op;
+
+template <size_t _NBound, size_t... _Ip>
+struct __bind_back_op<_NBound, index_sequence<_Ip...>> {
+  template <class _Fn, class _BoundArgs, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Fn&& __f, _BoundArgs&& __bound_args, _Args&&... __args) const
+      noexcept(noexcept(std::invoke(std::forward<_Fn>(__f),
+                                    std::forward<_Args>(__args)...,
+                                    std::get<_Ip>(std::forward<_BoundArgs>(__bound_args))...)))
+          -> decltype(std::invoke(std::forward<_Fn>(__f),
+                                  std::forward<_Args>(__args)...,
+                                  std::get<_Ip>(std::forward<_BoundArgs>(__bound_args))...)) {
+    return std::invoke(std::forward<_Fn>(__f),
+                       std::forward<_Args>(__args)...,
+                       std::get<_Ip>(std::forward<_BoundArgs>(__bound_args))...);
+  }
+};
+
+template <class _Fn, class _BoundArgs>
+struct __bind_back_t : __perfect_forward<__bind_back_op<tuple_size_v<_BoundArgs>>, _Fn, _BoundArgs> {
+  using __perfect_forward<__bind_back_op<tuple_size_v<_BoundArgs>>, _Fn, _BoundArgs>::__perfect_forward;
+};
+
+template <class _Fn, class... _Args>
+  requires is_constructible_v<decay_t<_Fn>, _Fn> && is_move_constructible_v<decay_t<_Fn>> &&
+               (is_constructible_v<decay_t<_Args>, _Args> && ...) && (is_move_constructible_v<decay_t<_Args>> && ...)
+_LIBCPP_HIDE_FROM_ABI constexpr auto __bind_back(_Fn&& __f, _Args&&... __args) noexcept(
+    noexcept(__bind_back_t<decay_t<_Fn>, tuple<decay_t<_Args>...>>(
+        std::forward<_Fn>(__f), std::forward_as_tuple(std::forward<_Args>(__args)...))))
+    -> decltype(__bind_back_t<decay_t<_Fn>, tuple<decay_t<_Args>...>>(
+        std::forward<_Fn>(__f), std::forward_as_tuple(std::forward<_Args>(__args)...))) {
+  return __bind_back_t<decay_t<_Fn>, tuple<decay_t<_Args>...>>(
+      std::forward<_Fn>(__f), std::forward_as_tuple(std::forward<_Args>(__args)...));
+}
+
+#  if _LIBCPP_STD_VER >= 23
+template <class _Fn, class... _Args>
+_LIBCPP_HIDE_FROM_ABI constexpr auto bind_back(_Fn&& __f, _Args&&... __args) {
+  static_assert(is_constructible_v<decay_t<_Fn>, _Fn>, "bind_back requires decay_t<F> to be constructible from F");
+  static_assert(is_move_constructible_v<decay_t<_Fn>>, "bind_back requires decay_t<F> to be move constructible");
+  static_assert((is_constructible_v<decay_t<_Args>, _Args> && ...),
+                "bind_back requires all decay_t<Args> to be constructible from respective Args");
+  static_assert((is_move_constructible_v<decay_t<_Args>> && ...),
+                "bind_back requires all decay_t<Args> to be move constructible");
+  return __bind_back_t<decay_t<_Fn>, tuple<decay_t<_Args>...>>(
+      std::forward<_Fn>(__f), std::forward_as_tuple(std::forward<_Args>(__args)...));
+}
+#  endif // _LIBCPP_STD_VER >= 23
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_BIND_BACK_H
diff --git a/libcxx/include/__cxx03/__functional/bind_front.h b/libcxx/include/__cxx03/__functional/bind_front.h
new file mode 100644
index 000000000000..87ef3affe80b
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/bind_front.h
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_BIND_FRONT_H
+#define _LIBCPP___FUNCTIONAL_BIND_FRONT_H
+
+#include <__config>
+#include <__functional/invoke.h>
+#include <__functional/perfect_forward.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_constructible.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+struct __bind_front_op {
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Args&&... __args) const noexcept(
+      noexcept(std::invoke(std::forward<_Args>(__args)...))) -> decltype(std::invoke(std::forward<_Args>(__args)...)) {
+    return std::invoke(std::forward<_Args>(__args)...);
+  }
+};
+
+template <class _Fn, class... _BoundArgs>
+struct __bind_front_t : __perfect_forward<__bind_front_op, _Fn, _BoundArgs...> {
+  using __perfect_forward<__bind_front_op, _Fn, _BoundArgs...>::__perfect_forward;
+};
+
+template <class _Fn, class... _Args>
+  requires is_constructible_v<decay_t<_Fn>, _Fn> && is_move_constructible_v<decay_t<_Fn>> &&
+           (is_constructible_v<decay_t<_Args>, _Args> && ...) && (is_move_constructible_v<decay_t<_Args>> && ...)
+_LIBCPP_HIDE_FROM_ABI constexpr auto bind_front(_Fn&& __f, _Args&&... __args) {
+  return __bind_front_t<decay_t<_Fn>, decay_t<_Args>...>(std::forward<_Fn>(__f), std::forward<_Args>(__args)...);
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_BIND_FRONT_H
diff --git a/libcxx/include/__cxx03/__functional/binder1st.h b/libcxx/include/__cxx03/__functional/binder1st.h
new file mode 100644
index 000000000000..04b51fefab70
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/binder1st.h
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_BINDER1ST_H
+#define _LIBCPP___FUNCTIONAL_BINDER1ST_H
+
+#include <__config>
+#include <__functional/unary_function.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_BINDERS)
+
+template <class _Operation>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 binder1st
+    : public __unary_function<typename _Operation::second_argument_type, typename _Operation::result_type> {
+protected:
+  _Operation op;
+  typename _Operation::first_argument_type value;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI binder1st(const _Operation& __x, const typename _Operation::first_argument_type __y)
+      : op(__x), value(__y) {}
+  _LIBCPP_HIDE_FROM_ABI typename _Operation::result_type
+  operator()(typename _Operation::second_argument_type& __x) const {
+    return op(value, __x);
+  }
+  _LIBCPP_HIDE_FROM_ABI typename _Operation::result_type
+  operator()(const typename _Operation::second_argument_type& __x) const {
+    return op(value, __x);
+  }
+};
+
+template <class _Operation, class _Tp>
+_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI binder1st<_Operation>
+bind1st(const _Operation& __op, const _Tp& __x) {
+  return binder1st<_Operation>(__op, __x);
+}
+
+#endif // _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_BINDERS)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_BINDER1ST_H
diff --git a/libcxx/include/__cxx03/__functional/binder2nd.h b/libcxx/include/__cxx03/__functional/binder2nd.h
new file mode 100644
index 000000000000..9d22e4430b1b
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/binder2nd.h
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_BINDER2ND_H
+#define _LIBCPP___FUNCTIONAL_BINDER2ND_H
+
+#include <__config>
+#include <__functional/unary_function.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_BINDERS)
+
+template <class _Operation>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 binder2nd
+    : public __unary_function<typename _Operation::first_argument_type, typename _Operation::result_type> {
+protected:
+  _Operation op;
+  typename _Operation::second_argument_type value;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI binder2nd(const _Operation& __x, const typename _Operation::second_argument_type __y)
+      : op(__x), value(__y) {}
+  _LIBCPP_HIDE_FROM_ABI typename _Operation::result_type
+  operator()(typename _Operation::first_argument_type& __x) const {
+    return op(__x, value);
+  }
+  _LIBCPP_HIDE_FROM_ABI typename _Operation::result_type
+  operator()(const typename _Operation::first_argument_type& __x) const {
+    return op(__x, value);
+  }
+};
+
+template <class _Operation, class _Tp>
+_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI binder2nd<_Operation>
+bind2nd(const _Operation& __op, const _Tp& __x) {
+  return binder2nd<_Operation>(__op, __x);
+}
+
+#endif // _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_BINDERS)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_BINDER2ND_H
diff --git a/libcxx/include/__cxx03/__functional/boyer_moore_searcher.h b/libcxx/include/__cxx03/__functional/boyer_moore_searcher.h
new file mode 100644
index 000000000000..648b60c50521
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/boyer_moore_searcher.h
@@ -0,0 +1,306 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_BOYER_MOORE_SEARCHER_H
+#define _LIBCPP___FUNCTIONAL_BOYER_MOORE_SEARCHER_H
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#include <__algorithm/fill_n.h>
+#include <__config>
+#include <__functional/hash.h>
+#include <__functional/operations.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/shared_ptr.h>
+#include <__type_traits/make_unsigned.h>
+#include <__utility/pair.h>
+#include <array>
+#include <unordered_map>
+#include <vector>
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Key, class _Value, class _Hash, class _BinaryPredicate, bool /*useArray*/>
+class _BMSkipTable;
+
+// General case for BM data searching; use a map
+template <class _Key, class _Value, class _Hash, class _BinaryPredicate>
+class _BMSkipTable<_Key, _Value, _Hash, _BinaryPredicate, false> {
+private:
+  using value_type = _Value;
+  using key_type   = _Key;
+
+  const value_type __default_value_;
+  unordered_map<_Key, _Value, _Hash, _BinaryPredicate> __table_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit _BMSkipTable(
+      size_t __sz, value_type __default_value, _Hash __hash, _BinaryPredicate __pred)
+      : __default_value_(__default_value), __table_(__sz, __hash, __pred) {}
+
+  _LIBCPP_HIDE_FROM_ABI void insert(const key_type& __key, value_type __val) { __table_[__key] = __val; }
+
+  _LIBCPP_HIDE_FROM_ABI value_type operator[](const key_type& __key) const {
+    auto __it = __table_.find(__key);
+    return __it == __table_.end() ? __default_value_ : __it->second;
+  }
+};
+
+// Special case small numeric values; use an array
+template <class _Key, class _Value, class _Hash, class _BinaryPredicate>
+class _BMSkipTable<_Key, _Value, _Hash, _BinaryPredicate, true> {
+private:
+  using value_type = _Value;
+  using key_type   = _Key;
+
+  using unsigned_key_type = make_unsigned_t<key_type>;
+  std::array<value_type, 256> __table_;
+  static_assert(numeric_limits<unsigned_key_type>::max() < 256);
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit _BMSkipTable(size_t, value_type __default_value, _Hash, _BinaryPredicate) {
+    std::fill_n(__table_.data(), __table_.size(), __default_value);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void insert(key_type __key, value_type __val) {
+    __table_[static_cast<unsigned_key_type>(__key)] = __val;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI value_type operator[](key_type __key) const {
+    return __table_[static_cast<unsigned_key_type>(__key)];
+  }
+};
+
+template <class _RandomAccessIterator1,
+          class _Hash            = hash<typename iterator_traits<_RandomAccessIterator1>::value_type>,
+          class _BinaryPredicate = equal_to<>>
+class _LIBCPP_TEMPLATE_VIS boyer_moore_searcher {
+private:
+  using difference_type = typename std::iterator_traits<_RandomAccessIterator1>::difference_type;
+  using value_type      = typename std::iterator_traits<_RandomAccessIterator1>::value_type;
+  using __skip_table_type =
+      _BMSkipTable<value_type,
+                   difference_type,
+                   _Hash,
+                   _BinaryPredicate,
+                   is_integral_v<value_type> && sizeof(value_type) == 1 && is_same_v<_Hash, hash<value_type>> &&
+                       is_same_v<_BinaryPredicate, equal_to<>>>;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI boyer_moore_searcher(
+      _RandomAccessIterator1 __first,
+      _RandomAccessIterator1 __last,
+      _Hash __hash            = _Hash(),
+      _BinaryPredicate __pred = _BinaryPredicate())
+      : __first_(__first),
+        __last_(__last),
+        __pred_(__pred),
+        __pattern_length_(__last - __first),
+        __skip_table_(std::make_shared<__skip_table_type>(__pattern_length_, -1, __hash, __pred_)),
+        __suffix_(std::__allocate_shared_unbounded_array<difference_type[]>(
+            allocator<difference_type>(), __pattern_length_ + 1)) {
+    difference_type __i = 0;
+    while (__first != __last) {
+      __skip_table_->insert(*__first, __i);
+      ++__first;
+      ++__i;
+    }
+    __build_suffix_table(__first_, __last_, __pred_);
+  }
+
+  template <class _RandomAccessIterator2>
+  _LIBCPP_HIDE_FROM_ABI pair<_RandomAccessIterator2, _RandomAccessIterator2>
+  operator()(_RandomAccessIterator2 __first, _RandomAccessIterator2 __last) const {
+    static_assert(__is_same_uncvref<typename iterator_traits<_RandomAccessIterator1>::value_type,
+                                    typename iterator_traits<_RandomAccessIterator2>::value_type>::value,
+                  "Corpus and Pattern iterators must point to the same type");
+    if (__first == __last)
+      return std::make_pair(__last, __last);
+    if (__first_ == __last_)
+      return std::make_pair(__first, __first);
+
+    if (__pattern_length_ > (__last - __first))
+      return std::make_pair(__last, __last);
+    return __search(__first, __last);
+  }
+
+private:
+  _RandomAccessIterator1 __first_;
+  _RandomAccessIterator1 __last_;
+  _BinaryPredicate __pred_;
+  difference_type __pattern_length_;
+  shared_ptr<__skip_table_type> __skip_table_;
+  shared_ptr<difference_type[]> __suffix_;
+
+  template <class _RandomAccessIterator2>
+  _LIBCPP_HIDE_FROM_ABI pair<_RandomAccessIterator2, _RandomAccessIterator2>
+  __search(_RandomAccessIterator2 __f, _RandomAccessIterator2 __l) const {
+    _RandomAccessIterator2 __current      = __f;
+    const _RandomAccessIterator2 __last   = __l - __pattern_length_;
+    const __skip_table_type& __skip_table = *__skip_table_;
+
+    while (__current <= __last) {
+      difference_type __j = __pattern_length_;
+      while (__pred_(__first_[__j - 1], __current[__j - 1])) {
+        --__j;
+        if (__j == 0)
+          return std::make_pair(__current, __current + __pattern_length_);
+      }
+
+      difference_type __k = __skip_table[__current[__j - 1]];
+      difference_type __m = __j - __k - 1;
+      if (__k < __j && __m > __suffix_[__j])
+        __current += __m;
+      else
+        __current += __suffix_[__j];
+    }
+    return std::make_pair(__l, __l);
+  }
+
+  template <class _Iterator, class _Container>
+  _LIBCPP_HIDE_FROM_ABI void
+  __compute_bm_prefix(_Iterator __first, _Iterator __last, _BinaryPredicate __pred, _Container& __prefix) {
+    const size_t __count = __last - __first;
+
+    __prefix[0] = 0;
+    size_t __k  = 0;
+
+    for (size_t __i = 1; __i != __count; ++__i) {
+      while (__k > 0 && !__pred(__first[__k], __first[__i]))
+        __k = __prefix[__k - 1];
+
+      if (__pred(__first[__k], __first[__i]))
+        ++__k;
+      __prefix[__i] = __k;
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void
+  __build_suffix_table(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _BinaryPredicate __pred) {
+    const size_t __count = __last - __first;
+
+    if (__count == 0)
+      return;
+
+    vector<difference_type> __scratch(__count);
+
+    __compute_bm_prefix(__first, __last, __pred, __scratch);
+    for (size_t __i = 0; __i <= __count; ++__i)
+      __suffix_[__i] = __count - __scratch[__count - 1];
+
+    using _ReverseIter = reverse_iterator<_RandomAccessIterator1>;
+    __compute_bm_prefix(_ReverseIter(__last), _ReverseIter(__first), __pred, __scratch);
+
+    for (size_t __i = 0; __i != __count; ++__i) {
+      const size_t __j          = __count - __scratch[__i];
+      const difference_type __k = __i - __scratch[__i] + 1;
+
+      if (__suffix_[__j] > __k)
+        __suffix_[__j] = __k;
+    }
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(boyer_moore_searcher);
+
+template <class _RandomAccessIterator1,
+          class _Hash            = hash<typename iterator_traits<_RandomAccessIterator1>::value_type>,
+          class _BinaryPredicate = equal_to<>>
+class _LIBCPP_TEMPLATE_VIS boyer_moore_horspool_searcher {
+private:
+  using difference_type = typename iterator_traits<_RandomAccessIterator1>::difference_type;
+  using value_type      = typename iterator_traits<_RandomAccessIterator1>::value_type;
+  using __skip_table_type =
+      _BMSkipTable<value_type,
+                   difference_type,
+                   _Hash,
+                   _BinaryPredicate,
+                   is_integral_v<value_type> && sizeof(value_type) == 1 && is_same_v<_Hash, hash<value_type>> &&
+                       is_same_v<_BinaryPredicate, equal_to<>>>;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI boyer_moore_horspool_searcher(
+      _RandomAccessIterator1 __first,
+      _RandomAccessIterator1 __last,
+      _Hash __hash            = _Hash(),
+      _BinaryPredicate __pred = _BinaryPredicate())
+      : __first_(__first),
+        __last_(__last),
+        __pred_(__pred),
+        __pattern_length_(__last - __first),
+        __skip_table_(std::make_shared<__skip_table_type>(__pattern_length_, __pattern_length_, __hash, __pred_)) {
+    if (__first == __last)
+      return;
+    --__last;
+    difference_type __i = 0;
+    while (__first != __last) {
+      __skip_table_->insert(*__first, __pattern_length_ - 1 - __i);
+      ++__first;
+      ++__i;
+    }
+  }
+
+  template <class _RandomAccessIterator2>
+  _LIBCPP_HIDE_FROM_ABI pair<_RandomAccessIterator2, _RandomAccessIterator2>
+  operator()(_RandomAccessIterator2 __first, _RandomAccessIterator2 __last) const {
+    static_assert(__is_same_uncvref<typename std::iterator_traits<_RandomAccessIterator1>::value_type,
+                                    typename std::iterator_traits<_RandomAccessIterator2>::value_type>::value,
+                  "Corpus and Pattern iterators must point to the same type");
+    if (__first == __last)
+      return std::make_pair(__last, __last);
+    if (__first_ == __last_)
+      return std::make_pair(__first, __first);
+
+    if (__pattern_length_ > __last - __first)
+      return std::make_pair(__last, __last);
+
+    return __search(__first, __last);
+  }
+
+private:
+  _RandomAccessIterator1 __first_;
+  _RandomAccessIterator1 __last_;
+  _BinaryPredicate __pred_;
+  difference_type __pattern_length_;
+  shared_ptr<__skip_table_type> __skip_table_;
+
+  template <class _RandomAccessIterator2>
+  _LIBCPP_HIDE_FROM_ABI pair<_RandomAccessIterator2, _RandomAccessIterator2>
+  __search(_RandomAccessIterator2 __f, _RandomAccessIterator2 __l) const {
+    _RandomAccessIterator2 __current      = __f;
+    const _RandomAccessIterator2 __last   = __l - __pattern_length_;
+    const __skip_table_type& __skip_table = *__skip_table_;
+
+    while (__current <= __last) {
+      difference_type __j = __pattern_length_;
+      while (__pred_(__first_[__j - 1], __current[__j - 1])) {
+        --__j;
+        if (__j == 0)
+          return std::make_pair(__current, __current + __pattern_length_);
+      }
+      __current += __skip_table[__current[__pattern_length_ - 1]];
+    }
+    return std::make_pair(__l, __l);
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(boyer_moore_horspool_searcher);
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___FUNCTIONAL_BOYER_MOORE_SEARCHER_H
diff --git a/libcxx/include/__cxx03/__functional/compose.h b/libcxx/include/__cxx03/__functional/compose.h
new file mode 100644
index 000000000000..4b86dd37cd48
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/compose.h
@@ -0,0 +1,53 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_COMPOSE_H
+#define _LIBCPP___FUNCTIONAL_COMPOSE_H
+
+#include <__config>
+#include <__functional/invoke.h>
+#include <__functional/perfect_forward.h>
+#include <__type_traits/decay.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+struct __compose_op {
+  template <class _Fn1, class _Fn2, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Fn1&& __f1, _Fn2&& __f2, _Args&&... __args) const noexcept(noexcept(
+      std::invoke(std::forward<_Fn1>(__f1), std::invoke(std::forward<_Fn2>(__f2), std::forward<_Args>(__args)...))))
+      -> decltype(std::invoke(std::forward<_Fn1>(__f1),
+                              std::invoke(std::forward<_Fn2>(__f2), std::forward<_Args>(__args)...))) {
+    return std::invoke(std::forward<_Fn1>(__f1), std::invoke(std::forward<_Fn2>(__f2), std::forward<_Args>(__args)...));
+  }
+};
+
+template <class _Fn1, class _Fn2>
+struct __compose_t : __perfect_forward<__compose_op, _Fn1, _Fn2> {
+  using __perfect_forward<__compose_op, _Fn1, _Fn2>::__perfect_forward;
+};
+
+template <class _Fn1, class _Fn2>
+_LIBCPP_HIDE_FROM_ABI constexpr auto __compose(_Fn1&& __f1, _Fn2&& __f2) noexcept(
+    noexcept(__compose_t<decay_t<_Fn1>, decay_t<_Fn2>>(std::forward<_Fn1>(__f1), std::forward<_Fn2>(__f2))))
+    -> decltype(__compose_t<decay_t<_Fn1>, decay_t<_Fn2>>(std::forward<_Fn1>(__f1), std::forward<_Fn2>(__f2))) {
+  return __compose_t<decay_t<_Fn1>, decay_t<_Fn2>>(std::forward<_Fn1>(__f1), std::forward<_Fn2>(__f2));
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_COMPOSE_H
diff --git a/libcxx/include/__cxx03/__functional/default_searcher.h b/libcxx/include/__cxx03/__functional/default_searcher.h
new file mode 100644
index 000000000000..db89d10757c1
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/default_searcher.h
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_DEFAULT_SEARCHER_H
+#define _LIBCPP___FUNCTIONAL_DEFAULT_SEARCHER_H
+
+#include <__algorithm/search.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/operations.h>
+#include <__iterator/iterator_traits.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+// default searcher
+template <class _ForwardIterator, class _BinaryPredicate = equal_to<>>
+class _LIBCPP_TEMPLATE_VIS default_searcher {
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  default_searcher(_ForwardIterator __f, _ForwardIterator __l, _BinaryPredicate __p = _BinaryPredicate())
+      : __first_(__f), __last_(__l), __pred_(__p) {}
+
+  template <typename _ForwardIterator2>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_ForwardIterator2, _ForwardIterator2>
+  operator()(_ForwardIterator2 __f, _ForwardIterator2 __l) const {
+    auto __proj = __identity();
+    return std::__search_impl(__f, __l, __first_, __last_, __pred_, __proj, __proj);
+  }
+
+private:
+  _ForwardIterator __first_;
+  _ForwardIterator __last_;
+  _BinaryPredicate __pred_;
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(default_searcher);
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_DEFAULT_SEARCHER_H
diff --git a/libcxx/include/__cxx03/__functional/function.h b/libcxx/include/__cxx03/__functional/function.h
new file mode 100644
index 000000000000..c7b98035e34b
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/function.h
@@ -0,0 +1,1048 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_FUNCTION_H
+#define _LIBCPP___FUNCTIONAL_FUNCTION_H
+
+#include <__assert>
+#include <__config>
+#include <__exception/exception.h>
+#include <__functional/binary_function.h>
+#include <__functional/invoke.h>
+#include <__functional/unary_function.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__memory/allocator.h>
+#include <__memory/allocator_destructor.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/builtin_new_allocator.h>
+#include <__memory/compressed_pair.h>
+#include <__memory/unique_ptr.h>
+#include <__type_traits/aligned_storage.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_core_convertible.h>
+#include <__type_traits/is_scalar.h>
+#include <__type_traits/is_trivially_constructible.h>
+#include <__type_traits/is_trivially_destructible.h>
+#include <__type_traits/is_void.h>
+#include <__type_traits/strip_signature.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/piecewise_construct.h>
+#include <__utility/swap.h>
+#include <__verbose_abort>
+#include <new>
+#include <tuple>
+#include <typeinfo>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// bad_function_call
+
+_LIBCPP_DIAGNOSTIC_PUSH
+#  if !_LIBCPP_AVAILABILITY_HAS_BAD_FUNCTION_CALL_KEY_FUNCTION
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wweak-vtables")
+#  endif
+class _LIBCPP_EXPORTED_FROM_ABI bad_function_call : public exception {
+public:
+  _LIBCPP_HIDE_FROM_ABI bad_function_call() _NOEXCEPT                                    = default;
+  _LIBCPP_HIDE_FROM_ABI bad_function_call(const bad_function_call&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI bad_function_call& operator=(const bad_function_call&) _NOEXCEPT = default;
+// Note that when a key function is not used, every translation unit that uses
+// bad_function_call will end up containing a weak definition of the vtable and
+// typeinfo.
+#  if _LIBCPP_AVAILABILITY_HAS_BAD_FUNCTION_CALL_KEY_FUNCTION
+  ~bad_function_call() _NOEXCEPT override;
+#  else
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~bad_function_call() _NOEXCEPT override {}
+#  endif
+
+#  ifdef _LIBCPP_ABI_BAD_FUNCTION_CALL_GOOD_WHAT_MESSAGE
+  const char* what() const _NOEXCEPT override;
+#  endif
+};
+_LIBCPP_DIAGNOSTIC_POP
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_bad_function_call() {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw bad_function_call();
+#  else
+  _LIBCPP_VERBOSE_ABORT("bad_function_call was thrown in -fno-exceptions mode");
+#  endif
+}
+
+template <class _Fp>
+class _LIBCPP_TEMPLATE_VIS function; // undefined
+
+namespace __function {
+
+template <class _Rp>
+struct __maybe_derive_from_unary_function {};
+
+template <class _Rp, class _A1>
+struct __maybe_derive_from_unary_function<_Rp(_A1)> : public __unary_function<_A1, _Rp> {};
+
+template <class _Rp>
+struct __maybe_derive_from_binary_function {};
+
+template <class _Rp, class _A1, class _A2>
+struct __maybe_derive_from_binary_function<_Rp(_A1, _A2)> : public __binary_function<_A1, _A2, _Rp> {};
+
+template <class _Fp>
+_LIBCPP_HIDE_FROM_ABI bool __not_null(_Fp const&) {
+  return true;
+}
+
+template <class _Fp>
+_LIBCPP_HIDE_FROM_ABI bool __not_null(_Fp* __ptr) {
+  return __ptr;
+}
+
+template <class _Ret, class _Class>
+_LIBCPP_HIDE_FROM_ABI bool __not_null(_Ret _Class::*__ptr) {
+  return __ptr;
+}
+
+template <class _Fp>
+_LIBCPP_HIDE_FROM_ABI bool __not_null(function<_Fp> const& __f) {
+  return !!__f;
+}
+
+#  ifdef _LIBCPP_HAS_EXTENSION_BLOCKS
+template <class _Rp, class... _Args>
+_LIBCPP_HIDE_FROM_ABI bool __not_null(_Rp (^__p)(_Args...)) {
+  return __p;
+}
+#  endif
+
+} // namespace __function
+
+namespace __function {
+
+// __alloc_func holds a functor and an allocator.
+
+template <class _Fp, class _Ap, class _FB>
+class __alloc_func;
+template <class _Fp, class _FB>
+class __default_alloc_func;
+
+template <class _Fp, class _Ap, class _Rp, class... _ArgTypes>
+class __alloc_func<_Fp, _Ap, _Rp(_ArgTypes...)> {
+  __compressed_pair<_Fp, _Ap> __f_;
+
+public:
+  typedef _LIBCPP_NODEBUG _Fp _Target;
+  typedef _LIBCPP_NODEBUG _Ap _Alloc;
+
+  _LIBCPP_HIDE_FROM_ABI const _Target& __target() const { return __f_.first(); }
+
+  // WIN32 APIs may define __allocator, so use __get_allocator instead.
+  _LIBCPP_HIDE_FROM_ABI const _Alloc& __get_allocator() const { return __f_.second(); }
+
+  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(_Target&& __f)
+      : __f_(piecewise_construct, std::forward_as_tuple(std::move(__f)), std::forward_as_tuple()) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(const _Target& __f, const _Alloc& __a)
+      : __f_(piecewise_construct, std::forward_as_tuple(__f), std::forward_as_tuple(__a)) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(const _Target& __f, _Alloc&& __a)
+      : __f_(piecewise_construct, std::forward_as_tuple(__f), std::forward_as_tuple(std::move(__a))) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(_Target&& __f, _Alloc&& __a)
+      : __f_(piecewise_construct, std::forward_as_tuple(std::move(__f)), std::forward_as_tuple(std::move(__a))) {}
+
+  _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes&&... __arg) {
+    typedef __invoke_void_return_wrapper<_Rp> _Invoker;
+    return _Invoker::__call(__f_.first(), std::forward<_ArgTypes>(__arg)...);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __alloc_func* __clone() const {
+    typedef allocator_traits<_Alloc> __alloc_traits;
+    typedef __rebind_alloc<__alloc_traits, __alloc_func> _AA;
+    _AA __a(__f_.second());
+    typedef __allocator_destructor<_AA> _Dp;
+    unique_ptr<__alloc_func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1));
+    ::new ((void*)__hold.get()) __alloc_func(__f_.first(), _Alloc(__a));
+    return __hold.release();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void destroy() _NOEXCEPT { __f_.~__compressed_pair<_Target, _Alloc>(); }
+
+  _LIBCPP_HIDE_FROM_ABI static void __destroy_and_delete(__alloc_func* __f) {
+    typedef allocator_traits<_Alloc> __alloc_traits;
+    typedef __rebind_alloc<__alloc_traits, __alloc_func> _FunAlloc;
+    _FunAlloc __a(__f->__get_allocator());
+    __f->destroy();
+    __a.deallocate(__f, 1);
+  }
+};
+
+template <class _Fp, class _Rp, class... _ArgTypes>
+class __default_alloc_func<_Fp, _Rp(_ArgTypes...)> {
+  _Fp __f_;
+
+public:
+  typedef _LIBCPP_NODEBUG _Fp _Target;
+
+  _LIBCPP_HIDE_FROM_ABI const _Target& __target() const { return __f_; }
+
+  _LIBCPP_HIDE_FROM_ABI explicit __default_alloc_func(_Target&& __f) : __f_(std::move(__f)) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit __default_alloc_func(const _Target& __f) : __f_(__f) {}
+
+  _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes&&... __arg) {
+    typedef __invoke_void_return_wrapper<_Rp> _Invoker;
+    return _Invoker::__call(__f_, std::forward<_ArgTypes>(__arg)...);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __default_alloc_func* __clone() const {
+    __builtin_new_allocator::__holder_t __hold = __builtin_new_allocator::__allocate_type<__default_alloc_func>(1);
+    __default_alloc_func* __res                = ::new ((void*)__hold.get()) __default_alloc_func(__f_);
+    (void)__hold.release();
+    return __res;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void destroy() _NOEXCEPT { __f_.~_Target(); }
+
+  _LIBCPP_HIDE_FROM_ABI static void __destroy_and_delete(__default_alloc_func* __f) {
+    __f->destroy();
+    __builtin_new_allocator::__deallocate_type<__default_alloc_func>(__f, 1);
+  }
+};
+
+// __base provides an abstract interface for copyable functors.
+
+template <class _Fp>
+class _LIBCPP_TEMPLATE_VIS __base;
+
+template <class _Rp, class... _ArgTypes>
+class __base<_Rp(_ArgTypes...)> {
+public:
+  __base(const __base&)            = delete;
+  __base& operator=(const __base&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI __base() {}
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual ~__base() {}
+  virtual __base* __clone() const             = 0;
+  virtual void __clone(__base*) const         = 0;
+  virtual void destroy() _NOEXCEPT            = 0;
+  virtual void destroy_deallocate() _NOEXCEPT = 0;
+  virtual _Rp operator()(_ArgTypes&&...)      = 0;
+#  ifndef _LIBCPP_HAS_NO_RTTI
+  virtual const void* target(const type_info&) const _NOEXCEPT = 0;
+  virtual const std::type_info& target_type() const _NOEXCEPT  = 0;
+#  endif // _LIBCPP_HAS_NO_RTTI
+};
+
+// __func implements __base for a given functor type.
+
+template <class _FD, class _Alloc, class _FB>
+class __func;
+
+template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
+class __func<_Fp, _Alloc, _Rp(_ArgTypes...)> : public __base<_Rp(_ArgTypes...)> {
+  __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> __f_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __func(_Fp&& __f) : __f_(std::move(__f)) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit __func(const _Fp& __f, const _Alloc& __a) : __f_(__f, __a) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit __func(const _Fp& __f, _Alloc&& __a) : __f_(__f, std::move(__a)) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit __func(_Fp&& __f, _Alloc&& __a) : __f_(std::move(__f), std::move(__a)) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual __base<_Rp(_ArgTypes...)>* __clone() const;
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __clone(__base<_Rp(_ArgTypes...)>*) const;
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy_deallocate() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual _Rp operator()(_ArgTypes&&... __arg);
+#  ifndef _LIBCPP_HAS_NO_RTTI
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual const void* target(const type_info&) const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual const std::type_info& target_type() const _NOEXCEPT;
+#  endif // _LIBCPP_HAS_NO_RTTI
+};
+
+template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
+__base<_Rp(_ArgTypes...)>* __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::__clone() const {
+  typedef allocator_traits<_Alloc> __alloc_traits;
+  typedef __rebind_alloc<__alloc_traits, __func> _Ap;
+  _Ap __a(__f_.__get_allocator());
+  typedef __allocator_destructor<_Ap> _Dp;
+  unique_ptr<__func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1));
+  ::new ((void*)__hold.get()) __func(__f_.__target(), _Alloc(__a));
+  return __hold.release();
+}
+
+template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
+void __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::__clone(__base<_Rp(_ArgTypes...)>* __p) const {
+  ::new ((void*)__p) __func(__f_.__target(), __f_.__get_allocator());
+}
+
+template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
+void __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy() _NOEXCEPT {
+  __f_.destroy();
+}
+
+template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
+void __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy_deallocate() _NOEXCEPT {
+  typedef allocator_traits<_Alloc> __alloc_traits;
+  typedef __rebind_alloc<__alloc_traits, __func> _Ap;
+  _Ap __a(__f_.__get_allocator());
+  __f_.destroy();
+  __a.deallocate(this, 1);
+}
+
+template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
+_Rp __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::operator()(_ArgTypes&&... __arg) {
+  return __f_(std::forward<_ArgTypes>(__arg)...);
+}
+
+#  ifndef _LIBCPP_HAS_NO_RTTI
+
+template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
+const void* __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::target(const type_info& __ti) const _NOEXCEPT {
+  if (__ti == typeid(_Fp))
+    return std::addressof(__f_.__target());
+  return nullptr;
+}
+
+template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
+const std::type_info& __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::target_type() const _NOEXCEPT {
+  return typeid(_Fp);
+}
+
+#  endif // _LIBCPP_HAS_NO_RTTI
+
+// __value_func creates a value-type from a __func.
+
+template <class _Fp>
+class __value_func;
+
+template <class _Rp, class... _ArgTypes>
+class __value_func<_Rp(_ArgTypes...)> {
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  typename aligned_storage<3 * sizeof(void*)>::type __buf_;
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+  typedef __base<_Rp(_ArgTypes...)> __func;
+  __func* __f_;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI static __func* __as_base(void* __p) { return reinterpret_cast<__func*>(__p); }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __value_func() _NOEXCEPT : __f_(nullptr) {}
+
+  template <class _Fp, class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI __value_func(_Fp&& __f, const _Alloc& __a) : __f_(nullptr) {
+    typedef allocator_traits<_Alloc> __alloc_traits;
+    typedef __function::__func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun;
+    typedef __rebind_alloc<__alloc_traits, _Fun> _FunAlloc;
+
+    if (__function::__not_null(__f)) {
+      _FunAlloc __af(__a);
+      if (sizeof(_Fun) <= sizeof(__buf_) && is_nothrow_copy_constructible<_Fp>::value &&
+          is_nothrow_copy_constructible<_FunAlloc>::value) {
+        __f_ = ::new ((void*)&__buf_) _Fun(std::move(__f), _Alloc(__af));
+      } else {
+        typedef __allocator_destructor<_FunAlloc> _Dp;
+        unique_ptr<__func, _Dp> __hold(__af.allocate(1), _Dp(__af, 1));
+        ::new ((void*)__hold.get()) _Fun(std::move(__f), _Alloc(__a));
+        __f_ = __hold.release();
+      }
+    }
+  }
+
+  template <class _Fp, __enable_if_t<!is_same<__decay_t<_Fp>, __value_func>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit __value_func(_Fp&& __f) : __value_func(std::forward<_Fp>(__f), allocator<_Fp>()) {}
+
+  _LIBCPP_HIDE_FROM_ABI __value_func(const __value_func& __f) {
+    if (__f.__f_ == nullptr)
+      __f_ = nullptr;
+    else if ((void*)__f.__f_ == &__f.__buf_) {
+      __f_ = __as_base(&__buf_);
+      __f.__f_->__clone(__f_);
+    } else
+      __f_ = __f.__f_->__clone();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __value_func(__value_func&& __f) _NOEXCEPT {
+    if (__f.__f_ == nullptr)
+      __f_ = nullptr;
+    else if ((void*)__f.__f_ == &__f.__buf_) {
+      __f_ = __as_base(&__buf_);
+      __f.__f_->__clone(__f_);
+    } else {
+      __f_     = __f.__f_;
+      __f.__f_ = nullptr;
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~__value_func() {
+    if ((void*)__f_ == &__buf_)
+      __f_->destroy();
+    else if (__f_)
+      __f_->destroy_deallocate();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __value_func& operator=(__value_func&& __f) {
+    *this = nullptr;
+    if (__f.__f_ == nullptr)
+      __f_ = nullptr;
+    else if ((void*)__f.__f_ == &__f.__buf_) {
+      __f_ = __as_base(&__buf_);
+      __f.__f_->__clone(__f_);
+    } else {
+      __f_     = __f.__f_;
+      __f.__f_ = nullptr;
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __value_func& operator=(nullptr_t) {
+    __func* __f = __f_;
+    __f_        = nullptr;
+    if ((void*)__f == &__buf_)
+      __f->destroy();
+    else if (__f)
+      __f->destroy_deallocate();
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes&&... __args) const {
+    if (__f_ == nullptr)
+      __throw_bad_function_call();
+    return (*__f_)(std::forward<_ArgTypes>(__args)...);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(__value_func& __f) _NOEXCEPT {
+    if (&__f == this)
+      return;
+    if ((void*)__f_ == &__buf_ && (void*)__f.__f_ == &__f.__buf_) {
+      _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+      typename aligned_storage<sizeof(__buf_)>::type __tempbuf;
+      _LIBCPP_SUPPRESS_DEPRECATED_POP
+      __func* __t = __as_base(&__tempbuf);
+      __f_->__clone(__t);
+      __f_->destroy();
+      __f_ = nullptr;
+      __f.__f_->__clone(__as_base(&__buf_));
+      __f.__f_->destroy();
+      __f.__f_ = nullptr;
+      __f_     = __as_base(&__buf_);
+      __t->__clone(__as_base(&__f.__buf_));
+      __t->destroy();
+      __f.__f_ = __as_base(&__f.__buf_);
+    } else if ((void*)__f_ == &__buf_) {
+      __f_->__clone(__as_base(&__f.__buf_));
+      __f_->destroy();
+      __f_     = __f.__f_;
+      __f.__f_ = __as_base(&__f.__buf_);
+    } else if ((void*)__f.__f_ == &__f.__buf_) {
+      __f.__f_->__clone(__as_base(&__buf_));
+      __f.__f_->destroy();
+      __f.__f_ = __f_;
+      __f_     = __as_base(&__buf_);
+    } else
+      std::swap(__f_, __f.__f_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return __f_ != nullptr; }
+
+#  ifndef _LIBCPP_HAS_NO_RTTI
+  _LIBCPP_HIDE_FROM_ABI const std::type_info& target_type() const _NOEXCEPT {
+    if (__f_ == nullptr)
+      return typeid(void);
+    return __f_->target_type();
+  }
+
+  template <typename _Tp>
+  _LIBCPP_HIDE_FROM_ABI const _Tp* target() const _NOEXCEPT {
+    if (__f_ == nullptr)
+      return nullptr;
+    return (const _Tp*)__f_->target(typeid(_Tp));
+  }
+#  endif // _LIBCPP_HAS_NO_RTTI
+};
+
+// Storage for a functor object, to be used with __policy to manage copy and
+// destruction.
+union __policy_storage {
+  mutable char __small[sizeof(void*) * 2];
+  void* __large;
+};
+
+// True if _Fun can safely be held in __policy_storage.__small.
+template <typename _Fun>
+struct __use_small_storage
+    : public integral_constant<
+          bool,
+          sizeof(_Fun) <= sizeof(__policy_storage)&& _LIBCPP_ALIGNOF(_Fun) <= _LIBCPP_ALIGNOF(__policy_storage) &&
+              is_trivially_copy_constructible<_Fun>::value && is_trivially_destructible<_Fun>::value> {};
+
+// Policy contains information about how to copy, destroy, and move the
+// underlying functor. You can think of it as a vtable of sorts.
+struct __policy {
+  // Used to copy or destroy __large values. null for trivial objects.
+  void* (*const __clone)(const void*);
+  void (*const __destroy)(void*);
+
+  // True if this is the null policy (no value).
+  const bool __is_null;
+
+  // The target type. May be null if RTTI is disabled.
+  const std::type_info* const __type_info;
+
+  // Returns a pointer to a static policy object suitable for the functor
+  // type.
+  template <typename _Fun>
+  _LIBCPP_HIDE_FROM_ABI static const __policy* __create() {
+    return __choose_policy<_Fun>(__use_small_storage<_Fun>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static const __policy* __create_empty() {
+    static constexpr __policy __policy = {
+        nullptr,
+        nullptr,
+        true,
+#  ifndef _LIBCPP_HAS_NO_RTTI
+        &typeid(void)
+#  else
+        nullptr
+#  endif
+    };
+    return &__policy;
+  }
+
+private:
+  template <typename _Fun>
+  _LIBCPP_HIDE_FROM_ABI static void* __large_clone(const void* __s) {
+    const _Fun* __f = static_cast<const _Fun*>(__s);
+    return __f->__clone();
+  }
+
+  template <typename _Fun>
+  _LIBCPP_HIDE_FROM_ABI static void __large_destroy(void* __s) {
+    _Fun::__destroy_and_delete(static_cast<_Fun*>(__s));
+  }
+
+  template <typename _Fun>
+  _LIBCPP_HIDE_FROM_ABI static const __policy* __choose_policy(/* is_small = */ false_type) {
+    static constexpr __policy __policy = {
+        &__large_clone<_Fun>,
+        &__large_destroy<_Fun>,
+        false,
+#  ifndef _LIBCPP_HAS_NO_RTTI
+        &typeid(typename _Fun::_Target)
+#  else
+        nullptr
+#  endif
+    };
+    return &__policy;
+  }
+
+  template <typename _Fun>
+  _LIBCPP_HIDE_FROM_ABI static const __policy* __choose_policy(/* is_small = */ true_type) {
+    static constexpr __policy __policy = {
+        nullptr,
+        nullptr,
+        false,
+#  ifndef _LIBCPP_HAS_NO_RTTI
+        &typeid(typename _Fun::_Target)
+#  else
+        nullptr
+#  endif
+    };
+    return &__policy;
+  }
+};
+
+// Used to choose between perfect forwarding or pass-by-value. Pass-by-value is
+// faster for types that can be passed in registers.
+template <typename _Tp>
+using __fast_forward = __conditional_t<is_scalar<_Tp>::value, _Tp, _Tp&&>;
+
+// __policy_invoker calls an instance of __alloc_func held in __policy_storage.
+
+template <class _Fp>
+struct __policy_invoker;
+
+template <class _Rp, class... _ArgTypes>
+struct __policy_invoker<_Rp(_ArgTypes...)> {
+  typedef _Rp (*__Call)(const __policy_storage*, __fast_forward<_ArgTypes>...);
+
+  __Call __call_;
+
+  // Creates an invoker that throws bad_function_call.
+  _LIBCPP_HIDE_FROM_ABI __policy_invoker() : __call_(&__call_empty) {}
+
+  // Creates an invoker that calls the given instance of __func.
+  template <typename _Fun>
+  _LIBCPP_HIDE_FROM_ABI static __policy_invoker __create() {
+    return __policy_invoker(&__call_impl<_Fun>);
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI explicit __policy_invoker(__Call __c) : __call_(__c) {}
+
+  _LIBCPP_HIDE_FROM_ABI static _Rp __call_empty(const __policy_storage*, __fast_forward<_ArgTypes>...) {
+    __throw_bad_function_call();
+  }
+
+  template <typename _Fun>
+  _LIBCPP_HIDE_FROM_ABI static _Rp __call_impl(const __policy_storage* __buf, __fast_forward<_ArgTypes>... __args) {
+    _Fun* __f = reinterpret_cast<_Fun*>(__use_small_storage<_Fun>::value ? &__buf->__small : __buf->__large);
+    return (*__f)(std::forward<_ArgTypes>(__args)...);
+  }
+};
+
+// __policy_func uses a __policy and __policy_invoker to create a type-erased,
+// copyable functor.
+
+template <class _Fp>
+class __policy_func;
+
+template <class _Rp, class... _ArgTypes>
+class __policy_func<_Rp(_ArgTypes...)> {
+  // Inline storage for small objects.
+  __policy_storage __buf_;
+
+  // Calls the value stored in __buf_. This could technically be part of
+  // policy, but storing it here eliminates a level of indirection inside
+  // operator().
+  typedef __function::__policy_invoker<_Rp(_ArgTypes...)> __invoker;
+  __invoker __invoker_;
+
+  // The policy that describes how to move / copy / destroy __buf_. Never
+  // null, even if the function is empty.
+  const __policy* __policy_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __policy_func() : __policy_(__policy::__create_empty()) {}
+
+  template <class _Fp, class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI __policy_func(_Fp&& __f, const _Alloc& __a) : __policy_(__policy::__create_empty()) {
+    typedef __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun;
+    typedef allocator_traits<_Alloc> __alloc_traits;
+    typedef __rebind_alloc<__alloc_traits, _Fun> _FunAlloc;
+
+    if (__function::__not_null(__f)) {
+      __invoker_ = __invoker::template __create<_Fun>();
+      __policy_  = __policy::__create<_Fun>();
+
+      _FunAlloc __af(__a);
+      if (__use_small_storage<_Fun>()) {
+        ::new ((void*)&__buf_.__small) _Fun(std::move(__f), _Alloc(__af));
+      } else {
+        typedef __allocator_destructor<_FunAlloc> _Dp;
+        unique_ptr<_Fun, _Dp> __hold(__af.allocate(1), _Dp(__af, 1));
+        ::new ((void*)__hold.get()) _Fun(std::move(__f), _Alloc(__af));
+        __buf_.__large = __hold.release();
+      }
+    }
+  }
+
+  template <class _Fp, __enable_if_t<!is_same<__decay_t<_Fp>, __policy_func>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit __policy_func(_Fp&& __f) : __policy_(__policy::__create_empty()) {
+    typedef __default_alloc_func<_Fp, _Rp(_ArgTypes...)> _Fun;
+
+    if (__function::__not_null(__f)) {
+      __invoker_ = __invoker::template __create<_Fun>();
+      __policy_  = __policy::__create<_Fun>();
+      if (__use_small_storage<_Fun>()) {
+        ::new ((void*)&__buf_.__small) _Fun(std::move(__f));
+      } else {
+        __builtin_new_allocator::__holder_t __hold = __builtin_new_allocator::__allocate_type<_Fun>(1);
+        __buf_.__large                             = ::new ((void*)__hold.get()) _Fun(std::move(__f));
+        (void)__hold.release();
+      }
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __policy_func(const __policy_func& __f)
+      : __buf_(__f.__buf_), __invoker_(__f.__invoker_), __policy_(__f.__policy_) {
+    if (__policy_->__clone)
+      __buf_.__large = __policy_->__clone(__f.__buf_.__large);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __policy_func(__policy_func&& __f)
+      : __buf_(__f.__buf_), __invoker_(__f.__invoker_), __policy_(__f.__policy_) {
+    if (__policy_->__destroy) {
+      __f.__policy_  = __policy::__create_empty();
+      __f.__invoker_ = __invoker();
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~__policy_func() {
+    if (__policy_->__destroy)
+      __policy_->__destroy(__buf_.__large);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __policy_func& operator=(__policy_func&& __f) {
+    *this          = nullptr;
+    __buf_         = __f.__buf_;
+    __invoker_     = __f.__invoker_;
+    __policy_      = __f.__policy_;
+    __f.__policy_  = __policy::__create_empty();
+    __f.__invoker_ = __invoker();
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __policy_func& operator=(nullptr_t) {
+    const __policy* __p = __policy_;
+    __policy_           = __policy::__create_empty();
+    __invoker_          = __invoker();
+    if (__p->__destroy)
+      __p->__destroy(__buf_.__large);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes&&... __args) const {
+    return __invoker_.__call_(std::addressof(__buf_), std::forward<_ArgTypes>(__args)...);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(__policy_func& __f) {
+    std::swap(__invoker_, __f.__invoker_);
+    std::swap(__policy_, __f.__policy_);
+    std::swap(__buf_, __f.__buf_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return !__policy_->__is_null; }
+
+#  ifndef _LIBCPP_HAS_NO_RTTI
+  _LIBCPP_HIDE_FROM_ABI const std::type_info& target_type() const _NOEXCEPT { return *__policy_->__type_info; }
+
+  template <typename _Tp>
+  _LIBCPP_HIDE_FROM_ABI const _Tp* target() const _NOEXCEPT {
+    if (__policy_->__is_null || typeid(_Tp) != *__policy_->__type_info)
+      return nullptr;
+    if (__policy_->__clone) // Out of line storage.
+      return reinterpret_cast<const _Tp*>(__buf_.__large);
+    else
+      return reinterpret_cast<const _Tp*>(&__buf_.__small);
+  }
+#  endif // _LIBCPP_HAS_NO_RTTI
+};
+
+#  if defined(_LIBCPP_HAS_BLOCKS_RUNTIME)
+
+extern "C" void* _Block_copy(const void*);
+extern "C" void _Block_release(const void*);
+
+template <class _Rp1, class... _ArgTypes1, class _Alloc, class _Rp, class... _ArgTypes>
+class __func<_Rp1 (^)(_ArgTypes1...), _Alloc, _Rp(_ArgTypes...)> : public __base<_Rp(_ArgTypes...)> {
+  typedef _Rp1 (^__block_type)(_ArgTypes1...);
+  __block_type __f_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __func(__block_type const& __f)
+#    ifdef _LIBCPP_HAS_OBJC_ARC
+      : __f_(__f)
+#    else
+      : __f_(reinterpret_cast<__block_type>(__f ? _Block_copy(__f) : nullptr))
+#    endif
+  {
+  }
+
+  // [TODO] add && to save on a retain
+
+  _LIBCPP_HIDE_FROM_ABI explicit __func(__block_type __f, const _Alloc& /* unused */)
+#    ifdef _LIBCPP_HAS_OBJC_ARC
+      : __f_(__f)
+#    else
+      : __f_(reinterpret_cast<__block_type>(__f ? _Block_copy(__f) : nullptr))
+#    endif
+  {
+  }
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual __base<_Rp(_ArgTypes...)>* __clone() const {
+    _LIBCPP_ASSERT_INTERNAL(
+        false,
+        "Block pointers are just pointers, so they should always fit into "
+        "std::function's small buffer optimization. This function should "
+        "never be invoked.");
+    return nullptr;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __clone(__base<_Rp(_ArgTypes...)>* __p) const {
+    ::new ((void*)__p) __func(__f_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy() _NOEXCEPT {
+#    ifndef _LIBCPP_HAS_OBJC_ARC
+    if (__f_)
+      _Block_release(__f_);
+#    endif
+    __f_ = 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy_deallocate() _NOEXCEPT {
+    _LIBCPP_ASSERT_INTERNAL(
+        false,
+        "Block pointers are just pointers, so they should always fit into "
+        "std::function's small buffer optimization. This function should "
+        "never be invoked.");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual _Rp operator()(_ArgTypes&&... __arg) {
+    return std::__invoke(__f_, std::forward<_ArgTypes>(__arg)...);
+  }
+
+#    ifndef _LIBCPP_HAS_NO_RTTI
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual const void* target(type_info const& __ti) const _NOEXCEPT {
+    if (__ti == typeid(__func::__block_type))
+      return &__f_;
+    return (const void*)nullptr;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual const std::type_info& target_type() const _NOEXCEPT {
+    return typeid(__func::__block_type);
+  }
+#    endif // _LIBCPP_HAS_NO_RTTI
+};
+
+#  endif // _LIBCPP_HAS_EXTENSION_BLOCKS
+
+} // namespace __function
+
+template <class _Rp, class... _ArgTypes>
+class _LIBCPP_TEMPLATE_VIS function<_Rp(_ArgTypes...)>
+    : public __function::__maybe_derive_from_unary_function<_Rp(_ArgTypes...)>,
+      public __function::__maybe_derive_from_binary_function<_Rp(_ArgTypes...)> {
+#  ifndef _LIBCPP_ABI_OPTIMIZED_FUNCTION
+  typedef __function::__value_func<_Rp(_ArgTypes...)> __func;
+#  else
+  typedef __function::__policy_func<_Rp(_ArgTypes...)> __func;
+#  endif
+
+  __func __f_;
+
+  template <class _Fp,
+            bool = _And< _IsNotSame<__remove_cvref_t<_Fp>, function>, __invokable<_Fp, _ArgTypes...> >::value>
+  struct __callable;
+  template <class _Fp>
+  struct __callable<_Fp, true> {
+    static const bool value =
+        is_void<_Rp>::value || __is_core_convertible<typename __invoke_of<_Fp, _ArgTypes...>::type, _Rp>::value;
+  };
+  template <class _Fp>
+  struct __callable<_Fp, false> {
+    static const bool value = false;
+  };
+
+  template <class _Fp>
+  using _EnableIfLValueCallable = __enable_if_t<__callable<_Fp&>::value>;
+
+public:
+  typedef _Rp result_type;
+
+  // construct/copy/destroy:
+  _LIBCPP_HIDE_FROM_ABI function() _NOEXCEPT {}
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_HIDE_FROM_ABI function(nullptr_t) _NOEXCEPT {}
+  _LIBCPP_HIDE_FROM_ABI function(const function&);
+  _LIBCPP_HIDE_FROM_ABI function(function&&) _NOEXCEPT;
+  template <class _Fp, class = _EnableIfLValueCallable<_Fp>>
+  _LIBCPP_HIDE_FROM_ABI function(_Fp);
+
+#  if _LIBCPP_STD_VER <= 14
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI function(allocator_arg_t, const _Alloc&) _NOEXCEPT {}
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI function(allocator_arg_t, const _Alloc&, nullptr_t) _NOEXCEPT {}
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI function(allocator_arg_t, const _Alloc&, const function&);
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI function(allocator_arg_t, const _Alloc&, function&&);
+  template <class _Fp, class _Alloc, class = _EnableIfLValueCallable<_Fp>>
+  _LIBCPP_HIDE_FROM_ABI function(allocator_arg_t, const _Alloc& __a, _Fp __f);
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI function& operator=(const function&);
+  _LIBCPP_HIDE_FROM_ABI function& operator=(function&&) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI function& operator=(nullptr_t) _NOEXCEPT;
+  template <class _Fp, class = _EnableIfLValueCallable<__decay_t<_Fp>>>
+  _LIBCPP_HIDE_FROM_ABI function& operator=(_Fp&&);
+
+  _LIBCPP_HIDE_FROM_ABI ~function();
+
+  // function modifiers:
+  _LIBCPP_HIDE_FROM_ABI void swap(function&) _NOEXCEPT;
+
+#  if _LIBCPP_STD_VER <= 14
+  template <class _Fp, class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI void assign(_Fp&& __f, const _Alloc& __a) {
+    function(allocator_arg, __a, std::forward<_Fp>(__f)).swap(*this);
+  }
+#  endif
+
+  // function capacity:
+  _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return static_cast<bool>(__f_); }
+
+  // deleted overloads close possible hole in the type system
+  template <class _R2, class... _ArgTypes2>
+  bool operator==(const function<_R2(_ArgTypes2...)>&) const = delete;
+#  if _LIBCPP_STD_VER <= 17
+  template <class _R2, class... _ArgTypes2>
+  bool operator!=(const function<_R2(_ArgTypes2...)>&) const = delete;
+#  endif
+
+public:
+  // function invocation:
+  _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes...) const;
+
+#  ifndef _LIBCPP_HAS_NO_RTTI
+  // function target access:
+  _LIBCPP_HIDE_FROM_ABI const std::type_info& target_type() const _NOEXCEPT;
+  template <typename _Tp>
+  _LIBCPP_HIDE_FROM_ABI _Tp* target() _NOEXCEPT;
+  template <typename _Tp>
+  _LIBCPP_HIDE_FROM_ABI const _Tp* target() const _NOEXCEPT;
+#  endif // _LIBCPP_HAS_NO_RTTI
+};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Rp, class... _Ap>
+function(_Rp (*)(_Ap...)) -> function<_Rp(_Ap...)>;
+
+template <class _Fp, class _Stripped = typename __strip_signature<decltype(&_Fp::operator())>::type>
+function(_Fp) -> function<_Stripped>;
+#  endif // _LIBCPP_STD_VER >= 17
+
+template <class _Rp, class... _ArgTypes>
+function<_Rp(_ArgTypes...)>::function(const function& __f) : __f_(__f.__f_) {}
+
+#  if _LIBCPP_STD_VER <= 14
+template <class _Rp, class... _ArgTypes>
+template <class _Alloc>
+function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc&, const function& __f) : __f_(__f.__f_) {}
+#  endif
+
+template <class _Rp, class... _ArgTypes>
+function<_Rp(_ArgTypes...)>::function(function&& __f) _NOEXCEPT : __f_(std::move(__f.__f_)) {}
+
+#  if _LIBCPP_STD_VER <= 14
+template <class _Rp, class... _ArgTypes>
+template <class _Alloc>
+function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc&, function&& __f) : __f_(std::move(__f.__f_)) {}
+#  endif
+
+template <class _Rp, class... _ArgTypes>
+template <class _Fp, class>
+function<_Rp(_ArgTypes...)>::function(_Fp __f) : __f_(std::move(__f)) {}
+
+#  if _LIBCPP_STD_VER <= 14
+template <class _Rp, class... _ArgTypes>
+template <class _Fp, class _Alloc, class>
+function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc& __a, _Fp __f) : __f_(std::move(__f), __a) {}
+#  endif
+
+template <class _Rp, class... _ArgTypes>
+function<_Rp(_ArgTypes...)>& function<_Rp(_ArgTypes...)>::operator=(const function& __f) {
+  function(__f).swap(*this);
+  return *this;
+}
+
+template <class _Rp, class... _ArgTypes>
+function<_Rp(_ArgTypes...)>& function<_Rp(_ArgTypes...)>::operator=(function&& __f) _NOEXCEPT {
+  __f_ = std::move(__f.__f_);
+  return *this;
+}
+
+template <class _Rp, class... _ArgTypes>
+function<_Rp(_ArgTypes...)>& function<_Rp(_ArgTypes...)>::operator=(nullptr_t) _NOEXCEPT {
+  __f_ = nullptr;
+  return *this;
+}
+
+template <class _Rp, class... _ArgTypes>
+template <class _Fp, class>
+function<_Rp(_ArgTypes...)>& function<_Rp(_ArgTypes...)>::operator=(_Fp&& __f) {
+  function(std::forward<_Fp>(__f)).swap(*this);
+  return *this;
+}
+
+template <class _Rp, class... _ArgTypes>
+function<_Rp(_ArgTypes...)>::~function() {}
+
+template <class _Rp, class... _ArgTypes>
+void function<_Rp(_ArgTypes...)>::swap(function& __f) _NOEXCEPT {
+  __f_.swap(__f.__f_);
+}
+
+template <class _Rp, class... _ArgTypes>
+_Rp function<_Rp(_ArgTypes...)>::operator()(_ArgTypes... __arg) const {
+  return __f_(std::forward<_ArgTypes>(__arg)...);
+}
+
+#  ifndef _LIBCPP_HAS_NO_RTTI
+
+template <class _Rp, class... _ArgTypes>
+const std::type_info& function<_Rp(_ArgTypes...)>::target_type() const _NOEXCEPT {
+  return __f_.target_type();
+}
+
+template <class _Rp, class... _ArgTypes>
+template <typename _Tp>
+_Tp* function<_Rp(_ArgTypes...)>::target() _NOEXCEPT {
+  return (_Tp*)(__f_.template target<_Tp>());
+}
+
+template <class _Rp, class... _ArgTypes>
+template <typename _Tp>
+const _Tp* function<_Rp(_ArgTypes...)>::target() const _NOEXCEPT {
+  return __f_.template target<_Tp>();
+}
+
+#  endif // _LIBCPP_HAS_NO_RTTI
+
+template <class _Rp, class... _ArgTypes>
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(const function<_Rp(_ArgTypes...)>& __f, nullptr_t) _NOEXCEPT {
+  return !__f;
+}
+
+#  if _LIBCPP_STD_VER <= 17
+
+template <class _Rp, class... _ArgTypes>
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(nullptr_t, const function<_Rp(_ArgTypes...)>& __f) _NOEXCEPT {
+  return !__f;
+}
+
+template <class _Rp, class... _ArgTypes>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const function<_Rp(_ArgTypes...)>& __f, nullptr_t) _NOEXCEPT {
+  return (bool)__f;
+}
+
+template <class _Rp, class... _ArgTypes>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(nullptr_t, const function<_Rp(_ArgTypes...)>& __f) _NOEXCEPT {
+  return (bool)__f;
+}
+
+#  endif // _LIBCPP_STD_VER <= 17
+
+template <class _Rp, class... _ArgTypes>
+inline _LIBCPP_HIDE_FROM_ABI void swap(function<_Rp(_ArgTypes...)>& __x, function<_Rp(_ArgTypes...)>& __y) _NOEXCEPT {
+  return __x.swap(__y);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CXX03_LANG
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FUNCTIONAL_FUNCTION_H
diff --git a/libcxx/include/__cxx03/__functional/hash.h b/libcxx/include/__cxx03/__functional/hash.h
new file mode 100644
index 000000000000..a9e450edd39f
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/hash.h
@@ -0,0 +1,542 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_HASH_H
+#define _LIBCPP___FUNCTIONAL_HASH_H
+
+#include <__config>
+#include <__functional/unary_function.h>
+#include <__fwd/functional.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/invoke.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_enum.h>
+#include <__type_traits/underlying_type.h>
+#include <__utility/pair.h>
+#include <__utility/swap.h>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Size>
+inline _LIBCPP_HIDE_FROM_ABI _Size __loadword(const void* __p) {
+  _Size __r;
+  std::memcpy(&__r, __p, sizeof(__r));
+  return __r;
+}
+
+// We use murmur2 when size_t is 32 bits, and cityhash64 when size_t
+// is 64 bits.  This is because cityhash64 uses 64bit x 64bit
+// multiplication, which can be very slow on 32-bit systems.
+template <class _Size, size_t = sizeof(_Size) * __CHAR_BIT__>
+struct __murmur2_or_cityhash;
+
+template <class _Size>
+struct __murmur2_or_cityhash<_Size, 32> {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK _Size
+  operator()(const void* __key, _Size __len) const {
+    // murmur2
+    const _Size __m             = 0x5bd1e995;
+    const _Size __r             = 24;
+    _Size __h                   = __len;
+    const unsigned char* __data = static_cast<const unsigned char*>(__key);
+    for (; __len >= 4; __data += 4, __len -= 4) {
+      _Size __k = std::__loadword<_Size>(__data);
+      __k *= __m;
+      __k ^= __k >> __r;
+      __k *= __m;
+      __h *= __m;
+      __h ^= __k;
+    }
+    switch (__len) {
+    case 3:
+      __h ^= static_cast<_Size>(__data[2] << 16);
+      _LIBCPP_FALLTHROUGH();
+    case 2:
+      __h ^= static_cast<_Size>(__data[1] << 8);
+      _LIBCPP_FALLTHROUGH();
+    case 1:
+      __h ^= __data[0];
+      __h *= __m;
+    }
+    __h ^= __h >> 13;
+    __h *= __m;
+    __h ^= __h >> 15;
+    return __h;
+  }
+};
+
+template <class _Size>
+struct __murmur2_or_cityhash<_Size, 64> {
+  // cityhash64
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK _Size
+  operator()(const void* __key, _Size __len) const {
+    const char* __s = static_cast<const char*>(__key);
+    if (__len <= 32) {
+      if (__len <= 16) {
+        return __hash_len_0_to_16(__s, __len);
+      } else {
+        return __hash_len_17_to_32(__s, __len);
+      }
+    } else if (__len <= 64) {
+      return __hash_len_33_to_64(__s, __len);
+    }
+
+    // For strings over 64 bytes we hash the end first, and then as we
+    // loop we keep 56 bytes of state: v, w, x, y, and z.
+    _Size __x = std::__loadword<_Size>(__s + __len - 40);
+    _Size __y = std::__loadword<_Size>(__s + __len - 16) + std::__loadword<_Size>(__s + __len - 56);
+    _Size __z =
+        __hash_len_16(std::__loadword<_Size>(__s + __len - 48) + __len, std::__loadword<_Size>(__s + __len - 24));
+    pair<_Size, _Size> __v = __weak_hash_len_32_with_seeds(__s + __len - 64, __len, __z);
+    pair<_Size, _Size> __w = __weak_hash_len_32_with_seeds(__s + __len - 32, __y + __k1, __x);
+    __x                    = __x * __k1 + std::__loadword<_Size>(__s);
+
+    // Decrease len to the nearest multiple of 64, and operate on 64-byte chunks.
+    __len = (__len - 1) & ~static_cast<_Size>(63);
+    do {
+      __x = __rotate(__x + __y + __v.first + std::__loadword<_Size>(__s + 8), 37) * __k1;
+      __y = __rotate(__y + __v.second + std::__loadword<_Size>(__s + 48), 42) * __k1;
+      __x ^= __w.second;
+      __y += __v.first + std::__loadword<_Size>(__s + 40);
+      __z = __rotate(__z + __w.first, 33) * __k1;
+      __v = __weak_hash_len_32_with_seeds(__s, __v.second * __k1, __x + __w.first);
+      __w = __weak_hash_len_32_with_seeds(__s + 32, __z + __w.second, __y + std::__loadword<_Size>(__s + 16));
+      std::swap(__z, __x);
+      __s += 64;
+      __len -= 64;
+    } while (__len != 0);
+    return __hash_len_16(__hash_len_16(__v.first, __w.first) + __shift_mix(__y) * __k1 + __z,
+                         __hash_len_16(__v.second, __w.second) + __x);
+  }
+
+private:
+  // Some primes between 2^63 and 2^64.
+  static const _Size __k0 = 0xc3a5c85c97cb3127ULL;
+  static const _Size __k1 = 0xb492b66fbe98f273ULL;
+  static const _Size __k2 = 0x9ae16a3b2f90404fULL;
+  static const _Size __k3 = 0xc949d7c7509e6557ULL;
+
+  _LIBCPP_HIDE_FROM_ABI static _Size __rotate(_Size __val, int __shift) {
+    return __shift == 0 ? __val : ((__val >> __shift) | (__val << (64 - __shift)));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static _Size __rotate_by_at_least_1(_Size __val, int __shift) {
+    return (__val >> __shift) | (__val << (64 - __shift));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static _Size __shift_mix(_Size __val) { return __val ^ (__val >> 47); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK static _Size __hash_len_16(_Size __u, _Size __v) {
+    const _Size __mul = 0x9ddfea08eb382d69ULL;
+    _Size __a         = (__u ^ __v) * __mul;
+    __a ^= (__a >> 47);
+    _Size __b = (__v ^ __a) * __mul;
+    __b ^= (__b >> 47);
+    __b *= __mul;
+    return __b;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK static _Size
+  __hash_len_0_to_16(const char* __s, _Size __len) {
+    if (__len > 8) {
+      const _Size __a = std::__loadword<_Size>(__s);
+      const _Size __b = std::__loadword<_Size>(__s + __len - 8);
+      return __hash_len_16(__a, __rotate_by_at_least_1(__b + __len, __len)) ^ __b;
+    }
+    if (__len >= 4) {
+      const uint32_t __a = std::__loadword<uint32_t>(__s);
+      const uint32_t __b = std::__loadword<uint32_t>(__s + __len - 4);
+#ifdef _LIBCPP_ABI_FIX_CITYHASH_IMPLEMENTATION
+      return __hash_len_16(__len + (static_cast<_Size>(__a) << 3), __b);
+#else
+      return __hash_len_16(__len + (__a << 3), __b);
+#endif
+    }
+    if (__len > 0) {
+      const unsigned char __a = static_cast<unsigned char>(__s[0]);
+      const unsigned char __b = static_cast<unsigned char>(__s[__len >> 1]);
+      const unsigned char __c = static_cast<unsigned char>(__s[__len - 1]);
+      const uint32_t __y      = static_cast<uint32_t>(__a) + (static_cast<uint32_t>(__b) << 8);
+      const uint32_t __z      = __len + (static_cast<uint32_t>(__c) << 2);
+      return __shift_mix(__y * __k2 ^ __z * __k3) * __k2;
+    }
+    return __k2;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK static _Size
+  __hash_len_17_to_32(const char* __s, _Size __len) {
+    const _Size __a = std::__loadword<_Size>(__s) * __k1;
+    const _Size __b = std::__loadword<_Size>(__s + 8);
+    const _Size __c = std::__loadword<_Size>(__s + __len - 8) * __k2;
+    const _Size __d = std::__loadword<_Size>(__s + __len - 16) * __k0;
+    return __hash_len_16(
+        __rotate(__a - __b, 43) + __rotate(__c, 30) + __d, __a + __rotate(__b ^ __k3, 20) - __c + __len);
+  }
+
+  // Return a 16-byte hash for 48 bytes.  Quick and dirty.
+  // Callers do best to use "random-looking" values for a and b.
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK static pair<_Size, _Size>
+  __weak_hash_len_32_with_seeds(_Size __w, _Size __x, _Size __y, _Size __z, _Size __a, _Size __b) {
+    __a += __w;
+    __b             = __rotate(__b + __a + __z, 21);
+    const _Size __c = __a;
+    __a += __x;
+    __a += __y;
+    __b += __rotate(__a, 44);
+    return pair<_Size, _Size>(__a + __z, __b + __c);
+  }
+
+  // Return a 16-byte hash for s[0] ... s[31], a, and b.  Quick and dirty.
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK static pair<_Size, _Size>
+  __weak_hash_len_32_with_seeds(const char* __s, _Size __a, _Size __b) {
+    return __weak_hash_len_32_with_seeds(
+        std::__loadword<_Size>(__s),
+        std::__loadword<_Size>(__s + 8),
+        std::__loadword<_Size>(__s + 16),
+        std::__loadword<_Size>(__s + 24),
+        __a,
+        __b);
+  }
+
+  // Return an 8-byte hash for 33 to 64 bytes.
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK static _Size
+  __hash_len_33_to_64(const char* __s, size_t __len) {
+    _Size __z = std::__loadword<_Size>(__s + 24);
+    _Size __a = std::__loadword<_Size>(__s) + (__len + std::__loadword<_Size>(__s + __len - 16)) * __k0;
+    _Size __b = __rotate(__a + __z, 52);
+    _Size __c = __rotate(__a, 37);
+    __a += std::__loadword<_Size>(__s + 8);
+    __c += __rotate(__a, 7);
+    __a += std::__loadword<_Size>(__s + 16);
+    _Size __vf = __a + __z;
+    _Size __vs = __b + __rotate(__a, 31) + __c;
+    __a        = std::__loadword<_Size>(__s + 16) + std::__loadword<_Size>(__s + __len - 32);
+    __z += std::__loadword<_Size>(__s + __len - 8);
+    __b = __rotate(__a + __z, 52);
+    __c = __rotate(__a, 37);
+    __a += std::__loadword<_Size>(__s + __len - 24);
+    __c += __rotate(__a, 7);
+    __a += std::__loadword<_Size>(__s + __len - 16);
+    _Size __wf = __a + __z;
+    _Size __ws = __b + __rotate(__a, 31) + __c;
+    _Size __r  = __shift_mix((__vf + __ws) * __k2 + (__wf + __vs) * __k0);
+    return __shift_mix(__r * __k0 + __vs) * __k2;
+  }
+};
+
+template <class _Tp, size_t = sizeof(_Tp) / sizeof(size_t)>
+struct __scalar_hash;
+
+template <class _Tp>
+struct __scalar_hash<_Tp, 0> : public __unary_function<_Tp, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT {
+    union {
+      _Tp __t;
+      size_t __a;
+    } __u;
+    __u.__a = 0;
+    __u.__t = __v;
+    return __u.__a;
+  }
+};
+
+template <class _Tp>
+struct __scalar_hash<_Tp, 1> : public __unary_function<_Tp, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT {
+    union {
+      _Tp __t;
+      size_t __a;
+    } __u;
+    __u.__t = __v;
+    return __u.__a;
+  }
+};
+
+template <class _Tp>
+struct __scalar_hash<_Tp, 2> : public __unary_function<_Tp, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT {
+    union {
+      _Tp __t;
+      struct {
+        size_t __a;
+        size_t __b;
+      } __s;
+    } __u;
+    __u.__t = __v;
+    return __murmur2_or_cityhash<size_t>()(&__u, sizeof(__u));
+  }
+};
+
+template <class _Tp>
+struct __scalar_hash<_Tp, 3> : public __unary_function<_Tp, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT {
+    union {
+      _Tp __t;
+      struct {
+        size_t __a;
+        size_t __b;
+        size_t __c;
+      } __s;
+    } __u;
+    __u.__t = __v;
+    return __murmur2_or_cityhash<size_t>()(&__u, sizeof(__u));
+  }
+};
+
+template <class _Tp>
+struct __scalar_hash<_Tp, 4> : public __unary_function<_Tp, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT {
+    union {
+      _Tp __t;
+      struct {
+        size_t __a;
+        size_t __b;
+        size_t __c;
+        size_t __d;
+      } __s;
+    } __u;
+    __u.__t = __v;
+    return __murmur2_or_cityhash<size_t>()(&__u, sizeof(__u));
+  }
+};
+
+struct _PairT {
+  size_t first;
+  size_t second;
+};
+
+_LIBCPP_HIDE_FROM_ABI inline size_t __hash_combine(size_t __lhs, size_t __rhs) _NOEXCEPT {
+  typedef __scalar_hash<_PairT> _HashT;
+  const _PairT __p = {__lhs, __rhs};
+  return _HashT()(__p);
+}
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS hash<_Tp*> : public __unary_function<_Tp*, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp* __v) const _NOEXCEPT {
+    union {
+      _Tp* __t;
+      size_t __a;
+    } __u;
+    __u.__t = __v;
+    return __murmur2_or_cityhash<size_t>()(&__u, sizeof(__u));
+  }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<bool> : public __unary_function<bool, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(bool __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<char> : public __unary_function<char, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(char __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<signed char> : public __unary_function<signed char, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(signed char __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<unsigned char> : public __unary_function<unsigned char, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned char __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+};
+
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<char8_t> : public __unary_function<char8_t, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(char8_t __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+};
+#endif // !_LIBCPP_HAS_NO_CHAR8_T
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<char16_t> : public __unary_function<char16_t, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(char16_t __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<char32_t> : public __unary_function<char32_t, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(char32_t __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+};
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<wchar_t> : public __unary_function<wchar_t, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(wchar_t __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+};
+#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<short> : public __unary_function<short, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(short __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<unsigned short> : public __unary_function<unsigned short, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned short __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<int> : public __unary_function<int, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(int __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<unsigned int> : public __unary_function<unsigned int, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned int __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<long> : public __unary_function<long, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(long __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<unsigned long> : public __unary_function<unsigned long, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned long __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<long long> : public __scalar_hash<long long> {};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<unsigned long long> : public __scalar_hash<unsigned long long> {};
+
+#ifndef _LIBCPP_HAS_NO_INT128
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<__int128_t> : public __scalar_hash<__int128_t> {};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<__uint128_t> : public __scalar_hash<__uint128_t> {};
+
+#endif
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<float> : public __scalar_hash<float> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(float __v) const _NOEXCEPT {
+    // -0.0 and 0.0 should return same hash
+    if (__v == 0.0f)
+      return 0;
+    return __scalar_hash<float>::operator()(__v);
+  }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<double> : public __scalar_hash<double> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(double __v) const _NOEXCEPT {
+    // -0.0 and 0.0 should return same hash
+    if (__v == 0.0)
+      return 0;
+    return __scalar_hash<double>::operator()(__v);
+  }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<long double> : public __scalar_hash<long double> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(long double __v) const _NOEXCEPT {
+    // -0.0 and 0.0 should return same hash
+    if (__v == 0.0L)
+      return 0;
+#if defined(__i386__) || (defined(__x86_64__) && defined(__ILP32__))
+    // Zero out padding bits
+    union {
+      long double __t;
+      struct {
+        size_t __a;
+        size_t __b;
+        size_t __c;
+        size_t __d;
+      } __s;
+    } __u;
+    __u.__s.__a = 0;
+    __u.__s.__b = 0;
+    __u.__s.__c = 0;
+    __u.__s.__d = 0;
+    __u.__t     = __v;
+    return __u.__s.__a ^ __u.__s.__b ^ __u.__s.__c ^ __u.__s.__d;
+#elif defined(__x86_64__)
+    // Zero out padding bits
+    union {
+      long double __t;
+      struct {
+        size_t __a;
+        size_t __b;
+      } __s;
+    } __u;
+    __u.__s.__a = 0;
+    __u.__s.__b = 0;
+    __u.__t     = __v;
+    return __u.__s.__a ^ __u.__s.__b;
+#else
+    return __scalar_hash<long double>::operator()(__v);
+#endif
+  }
+};
+
+template <class _Tp, bool = is_enum<_Tp>::value>
+struct _LIBCPP_TEMPLATE_VIS __enum_hash : public __unary_function<_Tp, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT {
+    typedef typename underlying_type<_Tp>::type type;
+    return hash<type>()(static_cast<type>(__v));
+  }
+};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS __enum_hash<_Tp, false> {
+  __enum_hash()                              = delete;
+  __enum_hash(__enum_hash const&)            = delete;
+  __enum_hash& operator=(__enum_hash const&) = delete;
+};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS hash : public __enum_hash<_Tp> {};
+
+#if _LIBCPP_STD_VER >= 17
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<nullptr_t> : public __unary_function<nullptr_t, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(nullptr_t) const _NOEXCEPT { return 662607004ull; }
+};
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Key, class _Hash>
+using __check_hash_requirements _LIBCPP_NODEBUG =
+    integral_constant<bool,
+                      is_copy_constructible<_Hash>::value && is_move_constructible<_Hash>::value &&
+                          __invokable_r<size_t, _Hash, _Key const&>::value >;
+
+template <class _Key, class _Hash = hash<_Key> >
+using __has_enabled_hash _LIBCPP_NODEBUG =
+    integral_constant<bool, __check_hash_requirements<_Key, _Hash>::value && is_default_constructible<_Hash>::value >;
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Type, class>
+using __enable_hash_helper_imp _LIBCPP_NODEBUG = _Type;
+
+template <class _Type, class... _Keys>
+using __enable_hash_helper _LIBCPP_NODEBUG =
+    __enable_hash_helper_imp<_Type, __enable_if_t<__all<__has_enabled_hash<_Keys>::value...>::value> >;
+#  else
+template <class _Type, class...>
+using __enable_hash_helper _LIBCPP_NODEBUG = _Type;
+#  endif
+
+#endif // !_LIBCPP_CXX03_LANG
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_HASH_H
diff --git a/libcxx/include/__cxx03/__functional/identity.h b/libcxx/include/__cxx03/__functional/identity.h
new file mode 100644
index 000000000000..8468de3dae26
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/identity.h
@@ -0,0 +1,65 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_IDENTITY_H
+#define _LIBCPP___FUNCTIONAL_IDENTITY_H
+
+#include <__config>
+#include <__fwd/functional.h>
+#include <__type_traits/integral_constant.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct __is_identity : false_type {};
+
+struct __identity {
+  template <class _Tp>
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp&& operator()(_Tp&& __t) const _NOEXCEPT {
+    return std::forward<_Tp>(__t);
+  }
+
+  using is_transparent = void;
+};
+
+template <>
+struct __is_identity<__identity> : true_type {};
+template <>
+struct __is_identity<reference_wrapper<__identity> > : true_type {};
+template <>
+struct __is_identity<reference_wrapper<const __identity> > : true_type {};
+
+#if _LIBCPP_STD_VER >= 20
+
+struct identity {
+  template <class _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp&& operator()(_Tp&& __t) const noexcept {
+    return std::forward<_Tp>(__t);
+  }
+
+  using is_transparent = void;
+};
+
+template <>
+struct __is_identity<identity> : true_type {};
+template <>
+struct __is_identity<reference_wrapper<identity> > : true_type {};
+template <>
+struct __is_identity<reference_wrapper<const identity> > : true_type {};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_IDENTITY_H
diff --git a/libcxx/include/__cxx03/__functional/invoke.h b/libcxx/include/__cxx03/__functional/invoke.h
new file mode 100644
index 000000000000..ef4bf25f0775
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/invoke.h
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_INVOKE_H
+#define _LIBCPP___FUNCTIONAL_INVOKE_H
+
+#include <__config>
+#include <__type_traits/invoke.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _Fn, class... _Args>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 invoke_result_t<_Fn, _Args...>
+invoke(_Fn&& __f, _Args&&... __args) noexcept(is_nothrow_invocable_v<_Fn, _Args...>) {
+  return std::__invoke(std::forward<_Fn>(__f), std::forward<_Args>(__args)...);
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#if _LIBCPP_STD_VER >= 23
+template <class _Result, class _Fn, class... _Args>
+  requires is_invocable_r_v<_Result, _Fn, _Args...>
+_LIBCPP_HIDE_FROM_ABI constexpr _Result
+invoke_r(_Fn&& __f, _Args&&... __args) noexcept(is_nothrow_invocable_r_v<_Result, _Fn, _Args...>) {
+  if constexpr (is_void_v<_Result>) {
+    static_cast<void>(std::invoke(std::forward<_Fn>(__f), std::forward<_Args>(__args)...));
+  } else {
+    // TODO: Use reference_converts_from_temporary_v once implemented
+    // using _ImplicitInvokeResult = invoke_result_t<_Fn, _Args...>;
+    // static_assert(!reference_converts_from_temporary_v<_Result, _ImplicitInvokeResult>,
+    static_assert(true,
+                  "Returning from invoke_r would bind a temporary object to the reference return type, "
+                  "which would result in a dangling reference.");
+    return std::invoke(std::forward<_Fn>(__f), std::forward<_Args>(__args)...);
+  }
+}
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_INVOKE_H
diff --git a/libcxx/include/__cxx03/__functional/is_transparent.h b/libcxx/include/__cxx03/__functional/is_transparent.h
new file mode 100644
index 000000000000..b2d62f2e3ead
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/is_transparent.h
@@ -0,0 +1,34 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_IS_TRANSPARENT
+#define _LIBCPP___FUNCTIONAL_IS_TRANSPARENT
+
+#include <__config>
+#include <__type_traits/void_t.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 14
+
+template <class _Tp, class, class = void>
+inline const bool __is_transparent_v = false;
+
+template <class _Tp, class _Up>
+inline const bool __is_transparent_v<_Tp, _Up, __void_t<typename _Tp::is_transparent> > = true;
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_IS_TRANSPARENT
diff --git a/libcxx/include/__cxx03/__functional/mem_fn.h b/libcxx/include/__cxx03/__functional/mem_fn.h
new file mode 100644
index 000000000000..ee07a71774f9
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/mem_fn.h
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_MEM_FN_H
+#define _LIBCPP___FUNCTIONAL_MEM_FN_H
+
+#include <__config>
+#include <__functional/binary_function.h>
+#include <__functional/invoke.h>
+#include <__functional/weak_result_type.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+class __mem_fn : public __weak_result_type<_Tp> {
+public:
+  // types
+  typedef _Tp type;
+
+private:
+  type __f_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __mem_fn(type __f) _NOEXCEPT : __f_(__f) {}
+
+  // invoke
+  template <class... _ArgTypes>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+
+  typename __invoke_return<type, _ArgTypes...>::type
+  operator()(_ArgTypes&&... __args) const {
+    return std::__invoke(__f_, std::forward<_ArgTypes>(__args)...);
+  }
+};
+
+template <class _Rp, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __mem_fn<_Rp _Tp::*> mem_fn(_Rp _Tp::*__pm) _NOEXCEPT {
+  return __mem_fn<_Rp _Tp::*>(__pm);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_MEM_FN_H
diff --git a/libcxx/include/__cxx03/__functional/mem_fun_ref.h b/libcxx/include/__cxx03/__functional/mem_fun_ref.h
new file mode 100644
index 000000000000..c344420b0299
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/mem_fun_ref.h
@@ -0,0 +1,146 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_MEM_FUN_REF_H
+#define _LIBCPP___FUNCTIONAL_MEM_FUN_REF_H
+
+#include <__config>
+#include <__functional/binary_function.h>
+#include <__functional/unary_function.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_BINDERS)
+
+template <class _Sp, class _Tp>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun_t : public __unary_function<_Tp*, _Sp> {
+  _Sp (_Tp::*__p_)();
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit mem_fun_t(_Sp (_Tp::*__p)()) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI _Sp operator()(_Tp* __p) const { return (__p->*__p_)(); }
+};
+
+template <class _Sp, class _Tp, class _Ap>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun1_t : public __binary_function<_Tp*, _Ap, _Sp> {
+  _Sp (_Tp::*__p_)(_Ap);
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit mem_fun1_t(_Sp (_Tp::*__p)(_Ap)) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI _Sp operator()(_Tp* __p, _Ap __x) const { return (__p->*__p_)(__x); }
+};
+
+template <class _Sp, class _Tp>
+_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI mem_fun_t<_Sp, _Tp> mem_fun(_Sp (_Tp::*__f)()) {
+  return mem_fun_t<_Sp, _Tp>(__f);
+}
+
+template <class _Sp, class _Tp, class _Ap>
+_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI mem_fun1_t<_Sp, _Tp, _Ap> mem_fun(_Sp (_Tp::*__f)(_Ap)) {
+  return mem_fun1_t<_Sp, _Tp, _Ap>(__f);
+}
+
+template <class _Sp, class _Tp>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun_ref_t : public __unary_function<_Tp, _Sp> {
+  _Sp (_Tp::*__p_)();
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit mem_fun_ref_t(_Sp (_Tp::*__p)()) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI _Sp operator()(_Tp& __p) const { return (__p.*__p_)(); }
+};
+
+template <class _Sp, class _Tp, class _Ap>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun1_ref_t : public __binary_function<_Tp, _Ap, _Sp> {
+  _Sp (_Tp::*__p_)(_Ap);
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit mem_fun1_ref_t(_Sp (_Tp::*__p)(_Ap)) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI _Sp operator()(_Tp& __p, _Ap __x) const { return (__p.*__p_)(__x); }
+};
+
+template <class _Sp, class _Tp>
+_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI mem_fun_ref_t<_Sp, _Tp> mem_fun_ref(_Sp (_Tp::*__f)()) {
+  return mem_fun_ref_t<_Sp, _Tp>(__f);
+}
+
+template <class _Sp, class _Tp, class _Ap>
+_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI mem_fun1_ref_t<_Sp, _Tp, _Ap>
+mem_fun_ref(_Sp (_Tp::*__f)(_Ap)) {
+  return mem_fun1_ref_t<_Sp, _Tp, _Ap>(__f);
+}
+
+template <class _Sp, class _Tp>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 const_mem_fun_t : public __unary_function<const _Tp*, _Sp> {
+  _Sp (_Tp::*__p_)() const;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit const_mem_fun_t(_Sp (_Tp::*__p)() const) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI _Sp operator()(const _Tp* __p) const { return (__p->*__p_)(); }
+};
+
+template <class _Sp, class _Tp, class _Ap>
+class _LIBCPP_TEMPLATE_VIS
+_LIBCPP_DEPRECATED_IN_CXX11 const_mem_fun1_t : public __binary_function<const _Tp*, _Ap, _Sp> {
+  _Sp (_Tp::*__p_)(_Ap) const;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit const_mem_fun1_t(_Sp (_Tp::*__p)(_Ap) const) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI _Sp operator()(const _Tp* __p, _Ap __x) const { return (__p->*__p_)(__x); }
+};
+
+template <class _Sp, class _Tp>
+_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI const_mem_fun_t<_Sp, _Tp> mem_fun(_Sp (_Tp::*__f)() const) {
+  return const_mem_fun_t<_Sp, _Tp>(__f);
+}
+
+template <class _Sp, class _Tp, class _Ap>
+_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI const_mem_fun1_t<_Sp, _Tp, _Ap>
+mem_fun(_Sp (_Tp::*__f)(_Ap) const) {
+  return const_mem_fun1_t<_Sp, _Tp, _Ap>(__f);
+}
+
+template <class _Sp, class _Tp>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 const_mem_fun_ref_t : public __unary_function<_Tp, _Sp> {
+  _Sp (_Tp::*__p_)() const;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit const_mem_fun_ref_t(_Sp (_Tp::*__p)() const) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI _Sp operator()(const _Tp& __p) const { return (__p.*__p_)(); }
+};
+
+template <class _Sp, class _Tp, class _Ap>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 const_mem_fun1_ref_t : public __binary_function<_Tp, _Ap, _Sp> {
+  _Sp (_Tp::*__p_)(_Ap) const;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit const_mem_fun1_ref_t(_Sp (_Tp::*__p)(_Ap) const) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI _Sp operator()(const _Tp& __p, _Ap __x) const { return (__p.*__p_)(__x); }
+};
+
+template <class _Sp, class _Tp>
+_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI const_mem_fun_ref_t<_Sp, _Tp>
+mem_fun_ref(_Sp (_Tp::*__f)() const) {
+  return const_mem_fun_ref_t<_Sp, _Tp>(__f);
+}
+
+template <class _Sp, class _Tp, class _Ap>
+_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI const_mem_fun1_ref_t<_Sp, _Tp, _Ap>
+mem_fun_ref(_Sp (_Tp::*__f)(_Ap) const) {
+  return const_mem_fun1_ref_t<_Sp, _Tp, _Ap>(__f);
+}
+
+#endif // _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_BINDERS)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_MEM_FUN_REF_H
diff --git a/libcxx/include/__cxx03/__functional/not_fn.h b/libcxx/include/__cxx03/__functional/not_fn.h
new file mode 100644
index 000000000000..4b3ce5524a74
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/not_fn.h
@@ -0,0 +1,53 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_NOT_FN_H
+#define _LIBCPP___FUNCTIONAL_NOT_FN_H
+
+#include <__config>
+#include <__functional/invoke.h>
+#include <__functional/perfect_forward.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_constructible.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+struct __not_fn_op {
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 auto operator()(_Args&&... __args) const
+      noexcept(noexcept(!std::invoke(std::forward<_Args>(__args)...)))
+          -> decltype(!std::invoke(std::forward<_Args>(__args)...)) {
+    return !std::invoke(std::forward<_Args>(__args)...);
+  }
+};
+
+template <class _Fn>
+struct __not_fn_t : __perfect_forward<__not_fn_op, _Fn> {
+  using __perfect_forward<__not_fn_op, _Fn>::__perfect_forward;
+};
+
+template <class _Fn,
+          class = enable_if_t< is_constructible_v<decay_t<_Fn>, _Fn> && is_move_constructible_v<decay_t<_Fn>> >>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 auto not_fn(_Fn&& __f) {
+  return __not_fn_t<decay_t<_Fn>>(std::forward<_Fn>(__f));
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_NOT_FN_H
diff --git a/libcxx/include/__cxx03/__functional/operations.h b/libcxx/include/__cxx03/__functional/operations.h
new file mode 100644
index 000000000000..0a6320f19de3
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/operations.h
@@ -0,0 +1,541 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_OPERATIONS_H
+#define _LIBCPP___FUNCTIONAL_OPERATIONS_H
+
+#include <__config>
+#include <__functional/binary_function.h>
+#include <__functional/unary_function.h>
+#include <__type_traits/desugars_to.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Arithmetic operations
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS plus : __binary_function<_Tp, _Tp, _Tp> {
+  typedef _Tp __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x + __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(plus);
+
+// The non-transparent std::plus specialization is only equivalent to a raw plus
+// operator when we don't perform an implicit conversion when calling it.
+template <class _Tp>
+inline const bool __desugars_to_v<__plus_tag, plus<_Tp>, _Tp, _Tp> = true;
+
+template <class _Tp, class _Up>
+inline const bool __desugars_to_v<__plus_tag, plus<void>, _Tp, _Up> = true;
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS plus<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) + std::forward<_T2>(__u))) //
+      -> decltype(std::forward<_T1>(__t) + std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) + std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS minus : __binary_function<_Tp, _Tp, _Tp> {
+  typedef _Tp __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x - __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(minus);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS minus<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) - std::forward<_T2>(__u))) //
+      -> decltype(std::forward<_T1>(__t) - std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) - std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS multiplies : __binary_function<_Tp, _Tp, _Tp> {
+  typedef _Tp __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x * __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(multiplies);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS multiplies<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) * std::forward<_T2>(__u))) //
+      -> decltype(std::forward<_T1>(__t) * std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) * std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS divides : __binary_function<_Tp, _Tp, _Tp> {
+  typedef _Tp __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x / __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(divides);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS divides<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) / std::forward<_T2>(__u))) //
+      -> decltype(std::forward<_T1>(__t) / std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) / std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS modulus : __binary_function<_Tp, _Tp, _Tp> {
+  typedef _Tp __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x % __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(modulus);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS modulus<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) % std::forward<_T2>(__u))) //
+      -> decltype(std::forward<_T1>(__t) % std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) % std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS negate : __unary_function<_Tp, _Tp> {
+  typedef _Tp __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return -__x; }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(negate);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS negate<void> {
+  template <class _Tp>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_Tp&& __x) const
+      noexcept(noexcept(-std::forward<_Tp>(__x))) //
+      -> decltype(-std::forward<_Tp>(__x)) {
+    return -std::forward<_Tp>(__x);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+// Bitwise operations
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS bit_and : __binary_function<_Tp, _Tp, _Tp> {
+  typedef _Tp __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x & __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(bit_and);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS bit_and<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) &
+                        std::forward<_T2>(__u))) -> decltype(std::forward<_T1>(__t) & std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) & std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+struct _LIBCPP_TEMPLATE_VIS bit_not : __unary_function<_Tp, _Tp> {
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return ~__x; }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(bit_not);
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS bit_not<void> {
+  template <class _Tp>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_Tp&& __x) const
+      noexcept(noexcept(~std::forward<_Tp>(__x))) //
+      -> decltype(~std::forward<_Tp>(__x)) {
+    return ~std::forward<_Tp>(__x);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS bit_or : __binary_function<_Tp, _Tp, _Tp> {
+  typedef _Tp __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x | __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(bit_or);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS bit_or<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) | std::forward<_T2>(__u))) //
+      -> decltype(std::forward<_T1>(__t) | std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) | std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS bit_xor : __binary_function<_Tp, _Tp, _Tp> {
+  typedef _Tp __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x ^ __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(bit_xor);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS bit_xor<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) ^ std::forward<_T2>(__u))) //
+      -> decltype(std::forward<_T1>(__t) ^ std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) ^ std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+// Comparison operations
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS equal_to : __binary_function<_Tp, _Tp, bool> {
+  typedef bool __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x == __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(equal_to);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS equal_to<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) == std::forward<_T2>(__u))) //
+      -> decltype(std::forward<_T1>(__t) == std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) == std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+// The non-transparent std::equal_to specialization is only equivalent to a raw equality
+// comparison when we don't perform an implicit conversion when calling it.
+template <class _Tp>
+inline const bool __desugars_to_v<__equal_tag, equal_to<_Tp>, _Tp, _Tp> = true;
+
+// In the transparent case, we do not enforce that
+template <class _Tp, class _Up>
+inline const bool __desugars_to_v<__equal_tag, equal_to<void>, _Tp, _Up> = true;
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS not_equal_to : __binary_function<_Tp, _Tp, bool> {
+  typedef bool __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x != __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(not_equal_to);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS not_equal_to<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) != std::forward<_T2>(__u))) //
+      -> decltype(std::forward<_T1>(__t) != std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) != std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS less : __binary_function<_Tp, _Tp, bool> {
+  typedef bool __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x < __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(less);
+
+template <class _Tp>
+inline const bool __desugars_to_v<__less_tag, less<_Tp>, _Tp, _Tp> = true;
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS less<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) < std::forward<_T2>(__u))) //
+      -> decltype(std::forward<_T1>(__t) < std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) < std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+
+template <class _Tp>
+inline const bool __desugars_to_v<__less_tag, less<>, _Tp, _Tp> = true;
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS less_equal : __binary_function<_Tp, _Tp, bool> {
+  typedef bool __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x <= __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(less_equal);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS less_equal<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) <= std::forward<_T2>(__u))) //
+      -> decltype(std::forward<_T1>(__t) <= std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) <= std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS greater_equal : __binary_function<_Tp, _Tp, bool> {
+  typedef bool __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x >= __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(greater_equal);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS greater_equal<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) >=
+                        std::forward<_T2>(__u))) -> decltype(std::forward<_T1>(__t) >= std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) >= std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS greater : __binary_function<_Tp, _Tp, bool> {
+  typedef bool __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x > __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(greater);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS greater<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) > std::forward<_T2>(__u))) //
+      -> decltype(std::forward<_T1>(__t) > std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) > std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+// Logical operations
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS logical_and : __binary_function<_Tp, _Tp, bool> {
+  typedef bool __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x && __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(logical_and);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS logical_and<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) && std::forward<_T2>(__u))) //
+      -> decltype(std::forward<_T1>(__t) && std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) && std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS logical_not : __unary_function<_Tp, bool> {
+  typedef bool __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x) const { return !__x; }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(logical_not);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS logical_not<void> {
+  template <class _Tp>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_Tp&& __x) const
+      noexcept(noexcept(!std::forward<_Tp>(__x))) //
+      -> decltype(!std::forward<_Tp>(__x)) {
+    return !std::forward<_Tp>(__x);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp = void>
+#else
+template <class _Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS logical_or : __binary_function<_Tp, _Tp, bool> {
+  typedef bool __result_type; // used by valarray
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const {
+    return __x || __y;
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(logical_or);
+
+#if _LIBCPP_STD_VER >= 14
+template <>
+struct _LIBCPP_TEMPLATE_VIS logical_or<void> {
+  template <class _T1, class _T2>
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const
+      noexcept(noexcept(std::forward<_T1>(__t) || std::forward<_T2>(__u))) //
+      -> decltype(std::forward<_T1>(__t) || std::forward<_T2>(__u)) {
+    return std::forward<_T1>(__t) || std::forward<_T2>(__u);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_OPERATIONS_H
diff --git a/libcxx/include/__cxx03/__functional/perfect_forward.h b/libcxx/include/__cxx03/__functional/perfect_forward.h
new file mode 100644
index 000000000000..74177c789b4a
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/perfect_forward.h
@@ -0,0 +1,104 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_PERFECT_FORWARD_H
+#define _LIBCPP___FUNCTIONAL_PERFECT_FORWARD_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/invoke.h>
+#include <__type_traits/is_constructible.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/integer_sequence.h>
+#include <__utility/move.h>
+#include <tuple>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _Op, class _Indices, class... _BoundArgs>
+struct __perfect_forward_impl;
+
+template <class _Op, size_t... _Idx, class... _BoundArgs>
+struct __perfect_forward_impl<_Op, index_sequence<_Idx...>, _BoundArgs...> {
+private:
+  tuple<_BoundArgs...> __bound_args_;
+
+public:
+  template <class... _Args, class = enable_if_t< is_constructible_v<tuple<_BoundArgs...>, _Args&&...> >>
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr __perfect_forward_impl(_Args&&... __bound_args)
+      : __bound_args_(std::forward<_Args>(__bound_args)...) {}
+
+  _LIBCPP_HIDE_FROM_ABI __perfect_forward_impl(__perfect_forward_impl const&) = default;
+  _LIBCPP_HIDE_FROM_ABI __perfect_forward_impl(__perfect_forward_impl&&)      = default;
+
+  _LIBCPP_HIDE_FROM_ABI __perfect_forward_impl& operator=(__perfect_forward_impl const&) = default;
+  _LIBCPP_HIDE_FROM_ABI __perfect_forward_impl& operator=(__perfect_forward_impl&&)      = default;
+
+  template <class... _Args, class = enable_if_t<is_invocable_v<_Op, _BoundArgs&..., _Args...>>>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Args&&... __args) & noexcept(
+      noexcept(_Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...)))
+      -> decltype(_Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...)) {
+    return _Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...);
+  }
+
+  template <class... _Args, class = enable_if_t<!is_invocable_v<_Op, _BoundArgs&..., _Args...>>>
+  auto operator()(_Args&&...) & = delete;
+
+  template <class... _Args, class = enable_if_t<is_invocable_v<_Op, _BoundArgs const&..., _Args...>>>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Args&&... __args) const& noexcept(
+      noexcept(_Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...)))
+      -> decltype(_Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...)) {
+    return _Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...);
+  }
+
+  template <class... _Args, class = enable_if_t<!is_invocable_v<_Op, _BoundArgs const&..., _Args...>>>
+  auto operator()(_Args&&...) const& = delete;
+
+  template <class... _Args, class = enable_if_t<is_invocable_v<_Op, _BoundArgs..., _Args...>>>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Args&&... __args) && noexcept(
+      noexcept(_Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...)))
+      -> decltype(_Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...)) {
+    return _Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...);
+  }
+
+  template <class... _Args, class = enable_if_t<!is_invocable_v<_Op, _BoundArgs..., _Args...>>>
+  auto operator()(_Args&&...) && = delete;
+
+  template <class... _Args, class = enable_if_t<is_invocable_v<_Op, _BoundArgs const..., _Args...>>>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Args&&... __args) const&& noexcept(
+      noexcept(_Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...)))
+      -> decltype(_Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...)) {
+    return _Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...);
+  }
+
+  template <class... _Args, class = enable_if_t<!is_invocable_v<_Op, _BoundArgs const..., _Args...>>>
+  auto operator()(_Args&&...) const&& = delete;
+};
+
+// __perfect_forward implements a perfect-forwarding call wrapper as explained in [func.require].
+template <class _Op, class... _Args>
+using __perfect_forward = __perfect_forward_impl<_Op, index_sequence_for<_Args...>, _Args...>;
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FUNCTIONAL_PERFECT_FORWARD_H
diff --git a/libcxx/include/__cxx03/__functional/pointer_to_binary_function.h b/libcxx/include/__cxx03/__functional/pointer_to_binary_function.h
new file mode 100644
index 000000000000..e345250dcdd8
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/pointer_to_binary_function.h
@@ -0,0 +1,44 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_POINTER_TO_BINARY_FUNCTION_H
+#define _LIBCPP___FUNCTIONAL_POINTER_TO_BINARY_FUNCTION_H
+
+#include <__config>
+#include <__functional/binary_function.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_BINDERS)
+
+template <class _Arg1, class _Arg2, class _Result>
+class _LIBCPP_TEMPLATE_VIS
+_LIBCPP_DEPRECATED_IN_CXX11 pointer_to_binary_function : public __binary_function<_Arg1, _Arg2, _Result> {
+  _Result (*__f_)(_Arg1, _Arg2);
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit pointer_to_binary_function(_Result (*__f)(_Arg1, _Arg2)) : __f_(__f) {}
+  _LIBCPP_HIDE_FROM_ABI _Result operator()(_Arg1 __x, _Arg2 __y) const { return __f_(__x, __y); }
+};
+
+template <class _Arg1, class _Arg2, class _Result>
+_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI pointer_to_binary_function<_Arg1, _Arg2, _Result>
+ptr_fun(_Result (*__f)(_Arg1, _Arg2)) {
+  return pointer_to_binary_function<_Arg1, _Arg2, _Result>(__f);
+}
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_POINTER_TO_BINARY_FUNCTION_H
diff --git a/libcxx/include/__cxx03/__functional/pointer_to_unary_function.h b/libcxx/include/__cxx03/__functional/pointer_to_unary_function.h
new file mode 100644
index 000000000000..3a5d153d3617
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/pointer_to_unary_function.h
@@ -0,0 +1,44 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_POINTER_TO_UNARY_FUNCTION_H
+#define _LIBCPP___FUNCTIONAL_POINTER_TO_UNARY_FUNCTION_H
+
+#include <__config>
+#include <__functional/unary_function.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_BINDERS)
+
+template <class _Arg, class _Result>
+class _LIBCPP_TEMPLATE_VIS
+_LIBCPP_DEPRECATED_IN_CXX11 pointer_to_unary_function : public __unary_function<_Arg, _Result> {
+  _Result (*__f_)(_Arg);
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit pointer_to_unary_function(_Result (*__f)(_Arg)) : __f_(__f) {}
+  _LIBCPP_HIDE_FROM_ABI _Result operator()(_Arg __x) const { return __f_(__x); }
+};
+
+template <class _Arg, class _Result>
+_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI pointer_to_unary_function<_Arg, _Result>
+ptr_fun(_Result (*__f)(_Arg)) {
+  return pointer_to_unary_function<_Arg, _Result>(__f);
+}
+
+#endif // _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_BINDERS)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_POINTER_TO_UNARY_FUNCTION_H
diff --git a/libcxx/include/__cxx03/__functional/ranges_operations.h b/libcxx/include/__cxx03/__functional/ranges_operations.h
new file mode 100644
index 000000000000..27f06eadd0eb
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/ranges_operations.h
@@ -0,0 +1,109 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_RANGES_OPERATIONS_H
+#define _LIBCPP___FUNCTIONAL_RANGES_OPERATIONS_H
+
+#include <__concepts/equality_comparable.h>
+#include <__concepts/totally_ordered.h>
+#include <__config>
+#include <__type_traits/desugars_to.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+struct equal_to {
+  template <class _Tp, class _Up>
+    requires equality_comparable_with<_Tp, _Up>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t, _Up&& __u) const
+      noexcept(noexcept(bool(std::forward<_Tp>(__t) == std::forward<_Up>(__u)))) {
+    return std::forward<_Tp>(__t) == std::forward<_Up>(__u);
+  }
+
+  using is_transparent = void;
+};
+
+struct not_equal_to {
+  template <class _Tp, class _Up>
+    requires equality_comparable_with<_Tp, _Up>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t, _Up&& __u) const
+      noexcept(noexcept(bool(!(std::forward<_Tp>(__t) == std::forward<_Up>(__u))))) {
+    return !(std::forward<_Tp>(__t) == std::forward<_Up>(__u));
+  }
+
+  using is_transparent = void;
+};
+
+struct less {
+  template <class _Tp, class _Up>
+    requires totally_ordered_with<_Tp, _Up>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t, _Up&& __u) const
+      noexcept(noexcept(bool(std::forward<_Tp>(__t) < std::forward<_Up>(__u)))) {
+    return std::forward<_Tp>(__t) < std::forward<_Up>(__u);
+  }
+
+  using is_transparent = void;
+};
+
+struct less_equal {
+  template <class _Tp, class _Up>
+    requires totally_ordered_with<_Tp, _Up>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t, _Up&& __u) const
+      noexcept(noexcept(bool(!(std::forward<_Up>(__u) < std::forward<_Tp>(__t))))) {
+    return !(std::forward<_Up>(__u) < std::forward<_Tp>(__t));
+  }
+
+  using is_transparent = void;
+};
+
+struct greater {
+  template <class _Tp, class _Up>
+    requires totally_ordered_with<_Tp, _Up>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t, _Up&& __u) const
+      noexcept(noexcept(bool(std::forward<_Up>(__u) < std::forward<_Tp>(__t)))) {
+    return std::forward<_Up>(__u) < std::forward<_Tp>(__t);
+  }
+
+  using is_transparent = void;
+};
+
+struct greater_equal {
+  template <class _Tp, class _Up>
+    requires totally_ordered_with<_Tp, _Up>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t, _Up&& __u) const
+      noexcept(noexcept(bool(!(std::forward<_Tp>(__t) < std::forward<_Up>(__u))))) {
+    return !(std::forward<_Tp>(__t) < std::forward<_Up>(__u));
+  }
+
+  using is_transparent = void;
+};
+
+} // namespace ranges
+
+// For ranges we do not require that the types on each side of the equality
+// operator are of the same type
+template <class _Tp, class _Up>
+inline const bool __desugars_to_v<__equal_tag, ranges::equal_to, _Tp, _Up> = true;
+
+template <class _Tp, class _Up>
+inline const bool __desugars_to_v<__less_tag, ranges::less, _Tp, _Up> = true;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_RANGES_OPERATIONS_H
diff --git a/libcxx/include/__cxx03/__functional/reference_wrapper.h b/libcxx/include/__cxx03/__functional/reference_wrapper.h
new file mode 100644
index 000000000000..3570e2673c80
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/reference_wrapper.h
@@ -0,0 +1,154 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_REFERENCE_WRAPPER_H
+#define _LIBCPP___FUNCTIONAL_REFERENCE_WRAPPER_H
+
+#include <__compare/synth_three_way.h>
+#include <__concepts/boolean_testable.h>
+#include <__config>
+#include <__functional/invoke.h>
+#include <__functional/weak_result_type.h>
+#include <__memory/addressof.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_const.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS reference_wrapper : public __weak_result_type<_Tp> {
+public:
+  // types
+  typedef _Tp type;
+
+private:
+  type* __f_;
+
+  static void __fun(_Tp&) _NOEXCEPT;
+  static void __fun(_Tp&&) = delete; // NOLINT(modernize-use-equals-delete) ; This is llvm.org/PR54276
+
+public:
+  template <class _Up,
+            class = __void_t<decltype(__fun(std::declval<_Up>()))>,
+            __enable_if_t<!__is_same_uncvref<_Up, reference_wrapper>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper(_Up&& __u)
+      _NOEXCEPT_(noexcept(__fun(std::declval<_Up>()))) {
+    type& __f = static_cast<_Up&&>(__u);
+    __f_      = std::addressof(__f);
+  }
+
+  // access
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 operator type&() const _NOEXCEPT { return *__f_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 type& get() const _NOEXCEPT { return *__f_; }
+
+  // invoke
+  template <class... _ArgTypes>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __invoke_of<type&, _ArgTypes...>::type
+  operator()(_ArgTypes&&... __args) const
+#if _LIBCPP_STD_VER >= 17
+      // Since is_nothrow_invocable requires C++17 LWG3764 is not backported
+      // to earlier versions.
+      noexcept(is_nothrow_invocable_v<_Tp&, _ArgTypes...>)
+#endif
+  {
+    return std::__invoke(get(), std::forward<_ArgTypes>(__args)...);
+  }
+
+#if _LIBCPP_STD_VER >= 26
+
+  // [refwrap.comparisons], comparisons
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(reference_wrapper __x, reference_wrapper __y)
+    requires requires {
+      { __x.get() == __y.get() } -> __boolean_testable;
+    }
+  {
+    return __x.get() == __y.get();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(reference_wrapper __x, const _Tp& __y)
+    requires requires {
+      { __x.get() == __y } -> __boolean_testable;
+    }
+  {
+    return __x.get() == __y;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(reference_wrapper __x, reference_wrapper<const _Tp> __y)
+    requires(!is_const_v<_Tp>) && requires {
+      { __x.get() == __y.get() } -> __boolean_testable;
+    }
+  {
+    return __x.get() == __y.get();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr auto operator<=>(reference_wrapper __x, reference_wrapper __y)
+    requires requires { std::__synth_three_way(__x.get(), __y.get()); }
+  {
+    return std::__synth_three_way(__x.get(), __y.get());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr auto operator<=>(reference_wrapper __x, const _Tp& __y)
+    requires requires { std::__synth_three_way(__x.get(), __y); }
+  {
+    return std::__synth_three_way(__x.get(), __y);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr auto operator<=>(reference_wrapper __x, reference_wrapper<const _Tp> __y)
+    requires(!is_const_v<_Tp>) && requires { std::__synth_three_way(__x.get(), __y.get()); }
+  {
+    return std::__synth_three_way(__x.get(), __y.get());
+  }
+
+#endif // _LIBCPP_STD_VER >= 26
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+reference_wrapper(_Tp&) -> reference_wrapper<_Tp>;
+#endif
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<_Tp> ref(_Tp& __t) _NOEXCEPT {
+  return reference_wrapper<_Tp>(__t);
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<_Tp>
+ref(reference_wrapper<_Tp> __t) _NOEXCEPT {
+  return __t;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<const _Tp> cref(const _Tp& __t) _NOEXCEPT {
+  return reference_wrapper<const _Tp>(__t);
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<const _Tp>
+cref(reference_wrapper<_Tp> __t) _NOEXCEPT {
+  return __t;
+}
+
+template <class _Tp>
+void ref(const _Tp&&) = delete;
+template <class _Tp>
+void cref(const _Tp&&) = delete;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_REFERENCE_WRAPPER_H
diff --git a/libcxx/include/__cxx03/__functional/unary_function.h b/libcxx/include/__cxx03/__functional/unary_function.h
new file mode 100644
index 000000000000..69b1bc94220a
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/unary_function.h
@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_UNARY_FUNCTION_H
+#define _LIBCPP___FUNCTIONAL_UNARY_FUNCTION_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION)
+
+template <class _Arg, class _Result>
+struct _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 unary_function {
+  typedef _Arg argument_type;
+  typedef _Result result_type;
+};
+
+#endif // _LIBCPP_STD_VER <= 14
+
+template <class _Arg, class _Result>
+struct __unary_function_keep_layout_base {
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  using argument_type _LIBCPP_DEPRECATED_IN_CXX17 = _Arg;
+  using result_type _LIBCPP_DEPRECATED_IN_CXX17   = _Result;
+#endif
+};
+
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION)
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wdeprecated-declarations")
+template <class _Arg, class _Result>
+using __unary_function = unary_function<_Arg, _Result>;
+_LIBCPP_DIAGNOSTIC_POP
+#else
+template <class _Arg, class _Result>
+using __unary_function = __unary_function_keep_layout_base<_Arg, _Result>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_UNARY_FUNCTION_H
diff --git a/libcxx/include/__cxx03/__functional/unary_negate.h b/libcxx/include/__cxx03/__functional/unary_negate.h
new file mode 100644
index 000000000000..5bd487a97bcb
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/unary_negate.h
@@ -0,0 +1,48 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_UNARY_NEGATE_H
+#define _LIBCPP___FUNCTIONAL_UNARY_NEGATE_H
+
+#include <__config>
+#include <__functional/unary_function.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_NEGATORS)
+
+template <class _Predicate>
+class _LIBCPP_TEMPLATE_VIS
+_LIBCPP_DEPRECATED_IN_CXX17 unary_negate : public __unary_function<typename _Predicate::argument_type, bool> {
+  _Predicate __pred_;
+
+public:
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI explicit unary_negate(const _Predicate& __pred)
+      : __pred_(__pred) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+  operator()(const typename _Predicate::argument_type& __x) const {
+    return !__pred_(__x);
+  }
+};
+
+template <class _Predicate>
+_LIBCPP_DEPRECATED_IN_CXX17 inline _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI unary_negate<_Predicate>
+not1(const _Predicate& __pred) {
+  return unary_negate<_Predicate>(__pred);
+}
+
+#endif // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_NEGATORS)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_UNARY_NEGATE_H
diff --git a/libcxx/include/__cxx03/__functional/weak_result_type.h b/libcxx/include/__cxx03/__functional/weak_result_type.h
new file mode 100644
index 000000000000..ad7a8395186c
--- /dev/null
+++ b/libcxx/include/__cxx03/__functional/weak_result_type.h
@@ -0,0 +1,231 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_WEAK_RESULT_TYPE_H
+#define _LIBCPP___FUNCTIONAL_WEAK_RESULT_TYPE_H
+
+#include <__config>
+#include <__functional/binary_function.h>
+#include <__functional/invoke.h>
+#include <__functional/unary_function.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_same.h>
+#include <__utility/declval.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct __has_result_type {
+private:
+  template <class _Up>
+  static false_type __test(...);
+  template <class _Up>
+  static true_type __test(typename _Up::result_type* = 0);
+
+public:
+  static const bool value = decltype(__test<_Tp>(0))::value;
+};
+
+// __weak_result_type
+
+template <class _Tp>
+struct __derives_from_unary_function {
+private:
+  struct __two {
+    char __lx;
+    char __lxx;
+  };
+  static __two __test(...);
+  template <class _Ap, class _Rp>
+  static __unary_function<_Ap, _Rp> __test(const volatile __unary_function<_Ap, _Rp>*);
+
+public:
+  static const bool value = !is_same<decltype(__test((_Tp*)0)), __two>::value;
+  typedef decltype(__test((_Tp*)0)) type;
+};
+
+template <class _Tp>
+struct __derives_from_binary_function {
+private:
+  struct __two {
+    char __lx;
+    char __lxx;
+  };
+  static __two __test(...);
+  template <class _A1, class _A2, class _Rp>
+  static __binary_function<_A1, _A2, _Rp> __test(const volatile __binary_function<_A1, _A2, _Rp>*);
+
+public:
+  static const bool value = !is_same<decltype(__test((_Tp*)0)), __two>::value;
+  typedef decltype(__test((_Tp*)0)) type;
+};
+
+template <class _Tp, bool = __derives_from_unary_function<_Tp>::value>
+struct __maybe_derive_from_unary_function // bool is true
+    : public __derives_from_unary_function<_Tp>::type {};
+
+template <class _Tp>
+struct __maybe_derive_from_unary_function<_Tp, false> {};
+
+template <class _Tp, bool = __derives_from_binary_function<_Tp>::value>
+struct __maybe_derive_from_binary_function // bool is true
+    : public __derives_from_binary_function<_Tp>::type {};
+
+template <class _Tp>
+struct __maybe_derive_from_binary_function<_Tp, false> {};
+
+template <class _Tp, bool = __has_result_type<_Tp>::value>
+struct __weak_result_type_imp // bool is true
+    : public __maybe_derive_from_unary_function<_Tp>,
+      public __maybe_derive_from_binary_function<_Tp> {
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = typename _Tp::result_type;
+#endif
+};
+
+template <class _Tp>
+struct __weak_result_type_imp<_Tp, false>
+    : public __maybe_derive_from_unary_function<_Tp>, public __maybe_derive_from_binary_function<_Tp> {};
+
+template <class _Tp>
+struct __weak_result_type : public __weak_result_type_imp<_Tp> {};
+
+// 0 argument case
+
+template <class _Rp>
+struct __weak_result_type<_Rp()> {
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp;
+#endif
+};
+
+template <class _Rp>
+struct __weak_result_type<_Rp (&)()> {
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp;
+#endif
+};
+
+template <class _Rp>
+struct __weak_result_type<_Rp (*)()> {
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp;
+#endif
+};
+
+// 1 argument case
+
+template <class _Rp, class _A1>
+struct __weak_result_type<_Rp(_A1)> : public __unary_function<_A1, _Rp> {};
+
+template <class _Rp, class _A1>
+struct __weak_result_type<_Rp (&)(_A1)> : public __unary_function<_A1, _Rp> {};
+
+template <class _Rp, class _A1>
+struct __weak_result_type<_Rp (*)(_A1)> : public __unary_function<_A1, _Rp> {};
+
+template <class _Rp, class _Cp>
+struct __weak_result_type<_Rp (_Cp::*)()> : public __unary_function<_Cp*, _Rp> {};
+
+template <class _Rp, class _Cp>
+struct __weak_result_type<_Rp (_Cp::*)() const> : public __unary_function<const _Cp*, _Rp> {};
+
+template <class _Rp, class _Cp>
+struct __weak_result_type<_Rp (_Cp::*)() volatile> : public __unary_function<volatile _Cp*, _Rp> {};
+
+template <class _Rp, class _Cp>
+struct __weak_result_type<_Rp (_Cp::*)() const volatile> : public __unary_function<const volatile _Cp*, _Rp> {};
+
+// 2 argument case
+
+template <class _Rp, class _A1, class _A2>
+struct __weak_result_type<_Rp(_A1, _A2)> : public __binary_function<_A1, _A2, _Rp> {};
+
+template <class _Rp, class _A1, class _A2>
+struct __weak_result_type<_Rp (*)(_A1, _A2)> : public __binary_function<_A1, _A2, _Rp> {};
+
+template <class _Rp, class _A1, class _A2>
+struct __weak_result_type<_Rp (&)(_A1, _A2)> : public __binary_function<_A1, _A2, _Rp> {};
+
+template <class _Rp, class _Cp, class _A1>
+struct __weak_result_type<_Rp (_Cp::*)(_A1)> : public __binary_function<_Cp*, _A1, _Rp> {};
+
+template <class _Rp, class _Cp, class _A1>
+struct __weak_result_type<_Rp (_Cp::*)(_A1) const> : public __binary_function<const _Cp*, _A1, _Rp> {};
+
+template <class _Rp, class _Cp, class _A1>
+struct __weak_result_type<_Rp (_Cp::*)(_A1) volatile> : public __binary_function<volatile _Cp*, _A1, _Rp> {};
+
+template <class _Rp, class _Cp, class _A1>
+struct __weak_result_type<_Rp (_Cp::*)(_A1) const volatile> : public __binary_function<const volatile _Cp*, _A1, _Rp> {
+};
+
+// 3 or more arguments
+
+template <class _Rp, class _A1, class _A2, class _A3, class... _A4>
+struct __weak_result_type<_Rp(_A1, _A2, _A3, _A4...)> {
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp;
+#endif
+};
+
+template <class _Rp, class _A1, class _A2, class _A3, class... _A4>
+struct __weak_result_type<_Rp (&)(_A1, _A2, _A3, _A4...)> {
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp;
+#endif
+};
+
+template <class _Rp, class _A1, class _A2, class _A3, class... _A4>
+struct __weak_result_type<_Rp (*)(_A1, _A2, _A3, _A4...)> {
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp;
+#endif
+};
+
+template <class _Rp, class _Cp, class _A1, class _A2, class... _A3>
+struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...)> {
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp;
+#endif
+};
+
+template <class _Rp, class _Cp, class _A1, class _A2, class... _A3>
+struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...) const> {
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp;
+#endif
+};
+
+template <class _Rp, class _Cp, class _A1, class _A2, class... _A3>
+struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...) volatile> {
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp;
+#endif
+};
+
+template <class _Rp, class _Cp, class _A1, class _A2, class... _A3>
+struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...) const volatile> {
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp;
+#endif
+};
+
+template <class _Tp, class... _Args>
+struct __invoke_return {
+  typedef decltype(std::__invoke(std::declval<_Tp>(), std::declval<_Args>()...)) type;
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_WEAK_RESULT_TYPE_H
diff --git a/libcxx/include/__cxx03/__fwd/array.h b/libcxx/include/__cxx03/__fwd/array.h
new file mode 100644
index 000000000000..b429d0c5a954
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/array.h
@@ -0,0 +1,46 @@
+//===---------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_ARRAY_H
+#define _LIBCPP___FWD_ARRAY_H
+
+#include <__config>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, size_t _Size>
+struct _LIBCPP_TEMPLATE_VIS array;
+
+template <size_t _Ip, class _Tp, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp& get(array<_Tp, _Size>&) _NOEXCEPT;
+
+template <size_t _Ip, class _Tp, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp& get(const array<_Tp, _Size>&) _NOEXCEPT;
+
+#ifndef _LIBCPP_CXX03_LANG
+template <size_t _Ip, class _Tp, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp&& get(array<_Tp, _Size>&&) _NOEXCEPT;
+
+template <size_t _Ip, class _Tp, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp&& get(const array<_Tp, _Size>&&) _NOEXCEPT;
+#endif
+
+template <class>
+struct __is_std_array : false_type {};
+
+template <class _Tp, size_t _Size>
+struct __is_std_array<array<_Tp, _Size> > : true_type {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_ARRAY_H
diff --git a/libcxx/include/__cxx03/__fwd/bit_reference.h b/libcxx/include/__cxx03/__fwd/bit_reference.h
new file mode 100644
index 000000000000..237efb6db664
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/bit_reference.h
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_BIT_REFERENCE_H
+#define _LIBCPP___FWD_BIT_REFERENCE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Cp, bool _IsConst, typename _Cp::__storage_type = 0>
+class __bit_iterator;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_BIT_REFERENCE_H
diff --git a/libcxx/include/__cxx03/__fwd/complex.h b/libcxx/include/__cxx03/__fwd/complex.h
new file mode 100644
index 000000000000..22c78c5cc3c7
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/complex.h
@@ -0,0 +1,42 @@
+//===---------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_COMPLEX_H
+#define _LIBCPP___FWD_COMPLEX_H
+
+#include <__config>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS complex;
+
+#if _LIBCPP_STD_VER >= 26
+
+template <size_t _Ip, class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp& get(complex<_Tp>&) noexcept;
+
+template <size_t _Ip, class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp&& get(complex<_Tp>&&) noexcept;
+
+template <size_t _Ip, class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr const _Tp& get(const complex<_Tp>&) noexcept;
+
+template <size_t _Ip, class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr const _Tp&& get(const complex<_Tp>&&) noexcept;
+
+#endif // _LIBCPP_STD_VER >= 26
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_COMPLEX_H
diff --git a/libcxx/include/__cxx03/__fwd/deque.h b/libcxx/include/__cxx03/__fwd/deque.h
new file mode 100644
index 000000000000..fd2fb5bb4b8e
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/deque.h
@@ -0,0 +1,26 @@
+//===---------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_DEQUE_H
+#define _LIBCPP___FWD_DEQUE_H
+
+#include <__config>
+#include <__fwd/memory.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Allocator = allocator<_Tp> >
+class _LIBCPP_TEMPLATE_VIS deque;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_DEQUE_H
diff --git a/libcxx/include/__cxx03/__fwd/format.h b/libcxx/include/__cxx03/__fwd/format.h
new file mode 100644
index 000000000000..b30c220f8a04
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/format.h
@@ -0,0 +1,38 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_FORMAT_H
+#define _LIBCPP___FWD_FORMAT_H
+
+#include <__config>
+#include <__iterator/concepts.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Context>
+class _LIBCPP_TEMPLATE_VIS basic_format_arg;
+
+template <class _OutIt, class _CharT>
+  requires output_iterator<_OutIt, const _CharT&>
+class _LIBCPP_TEMPLATE_VIS basic_format_context;
+
+template <class _Tp, class _CharT = char>
+struct _LIBCPP_TEMPLATE_VIS formatter;
+
+#endif //_LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_FORMAT_H
diff --git a/libcxx/include/__cxx03/__fwd/fstream.h b/libcxx/include/__cxx03/__fwd/fstream.h
new file mode 100644
index 000000000000..b4a112bfd4de
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/fstream.h
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_FSTREAM_H
+#define _LIBCPP___FWD_FSTREAM_H
+
+#include <__config>
+#include <__fwd/string.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_filebuf;
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_ifstream;
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_ofstream;
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_fstream;
+
+using filebuf  = basic_filebuf<char>;
+using ifstream = basic_ifstream<char>;
+using ofstream = basic_ofstream<char>;
+using fstream  = basic_fstream<char>;
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using wfilebuf  = basic_filebuf<wchar_t>;
+using wifstream = basic_ifstream<wchar_t>;
+using wofstream = basic_ofstream<wchar_t>;
+using wfstream  = basic_fstream<wchar_t>;
+#endif
+
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(filebuf) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wfilebuf)) basic_filebuf;
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(ifstream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wifstream)) basic_ifstream;
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(ofstream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wofstream)) basic_ofstream;
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(fstream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wfstream)) basic_fstream;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_FSTREAM_H
diff --git a/libcxx/include/__cxx03/__fwd/functional.h b/libcxx/include/__cxx03/__fwd/functional.h
new file mode 100644
index 000000000000..32c9ef33e453
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/functional.h
@@ -0,0 +1,28 @@
+//===---------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_FUNCTIONAL_H
+#define _LIBCPP___FWD_FUNCTIONAL_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class>
+struct _LIBCPP_TEMPLATE_VIS hash;
+
+template <class>
+class _LIBCPP_TEMPLATE_VIS reference_wrapper;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_FUNCTIONAL_H
diff --git a/libcxx/include/__cxx03/__fwd/ios.h b/libcxx/include/__cxx03/__fwd/ios.h
new file mode 100644
index 000000000000..48350709d4ce
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/ios.h
@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_IOS_H
+#define _LIBCPP___FWD_IOS_H
+
+#include <__config>
+#include <__fwd/string.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+class _LIBCPP_EXPORTED_FROM_ABI ios_base;
+
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_ios;
+
+using ios = basic_ios<char>;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using wios = basic_ios<wchar_t>;
+#endif
+
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(ios) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wios)) basic_ios;
+
+#if defined(_NEWLIB_VERSION)
+// On newlib, off_t is 'long int'
+using streamoff = long int; // for char_traits in <string>
+#else
+using streamoff = long long; // for char_traits in <string>
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_IOS_H
diff --git a/libcxx/include/__cxx03/__fwd/istream.h b/libcxx/include/__cxx03/__fwd/istream.h
new file mode 100644
index 000000000000..a06907a6c8ef
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/istream.h
@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_ISTREAM_H
+#define _LIBCPP___FWD_ISTREAM_H
+
+#include <__config>
+#include <__fwd/string.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_istream;
+
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_iostream;
+
+using istream  = basic_istream<char>;
+using iostream = basic_iostream<char>;
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using wistream  = basic_istream<wchar_t>;
+using wiostream = basic_iostream<wchar_t>;
+#endif
+
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(istream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wistream)) basic_istream;
+
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(iostream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wiostream)) basic_iostream;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_ISTREAM_H
diff --git a/libcxx/include/__cxx03/__fwd/mdspan.h b/libcxx/include/__cxx03/__fwd/mdspan.h
new file mode 100644
index 000000000000..8889567a047f
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/mdspan.h
@@ -0,0 +1,57 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//                        Kokkos v. 4.0
+//       Copyright (2022) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MDSPAN_LAYOUTS_H
+#define _LIBCPP___MDSPAN_LAYOUTS_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+// Layout policy with a mapping which corresponds to FORTRAN-style array layouts
+struct layout_left {
+  template <class _Extents>
+  class mapping;
+};
+
+// Layout policy with a mapping which corresponds to C-style array layouts
+struct layout_right {
+  template <class _Extents>
+  class mapping;
+};
+
+// Layout policy with a unique mapping where strides are arbitrary
+struct layout_stride {
+  template <class _Extents>
+  class mapping;
+};
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MDSPAN_LAYOUTS_H
diff --git a/libcxx/include/__cxx03/__fwd/memory.h b/libcxx/include/__cxx03/__fwd/memory.h
new file mode 100644
index 000000000000..b9e151855ad7
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/memory.h
@@ -0,0 +1,25 @@
+//===---------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_MEMORY_H
+#define _LIBCPP___FWD_MEMORY_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS allocator;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_MEMORY_H
diff --git a/libcxx/include/__cxx03/__fwd/memory_resource.h b/libcxx/include/__cxx03/__fwd/memory_resource.h
new file mode 100644
index 000000000000..d68b2c2b6315
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/memory_resource.h
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_MEMORY_RESOURCE_H
+#define _LIBCPP___FWD_MEMORY_RESOURCE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace pmr {
+template <class _ValueType>
+class _LIBCPP_AVAILABILITY_PMR _LIBCPP_TEMPLATE_VIS polymorphic_allocator;
+} // namespace pmr
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_MEMORY_RESOURCE_H
diff --git a/libcxx/include/__cxx03/__fwd/ostream.h b/libcxx/include/__cxx03/__fwd/ostream.h
new file mode 100644
index 000000000000..3347e0f71d7a
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/ostream.h
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_OSTREAM_H
+#define _LIBCPP___FWD_OSTREAM_H
+
+#include <__config>
+#include <__fwd/string.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_ostream;
+
+using ostream = basic_ostream<char>;
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using wostream = basic_ostream<wchar_t>;
+#endif
+
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(ostream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wostream)) basic_ostream;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_OSTREAM_H
diff --git a/libcxx/include/__cxx03/__fwd/pair.h b/libcxx/include/__cxx03/__fwd/pair.h
new file mode 100644
index 000000000000..af32628fe1e0
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/pair.h
@@ -0,0 +1,45 @@
+//===---------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_PAIR_H
+#define _LIBCPP___FWD_PAIR_H
+
+#include <__config>
+#include <__fwd/tuple.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class, class>
+struct _LIBCPP_TEMPLATE_VIS pair;
+
+template <size_t _Ip, class _T1, class _T2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, pair<_T1, _T2> >::type&
+get(pair<_T1, _T2>&) _NOEXCEPT;
+
+template <size_t _Ip, class _T1, class _T2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, pair<_T1, _T2> >::type&
+get(const pair<_T1, _T2>&) _NOEXCEPT;
+
+#ifndef _LIBCPP_CXX03_LANG
+template <size_t _Ip, class _T1, class _T2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, pair<_T1, _T2> >::type&&
+get(pair<_T1, _T2>&&) _NOEXCEPT;
+
+template <size_t _Ip, class _T1, class _T2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, pair<_T1, _T2> >::type&&
+get(const pair<_T1, _T2>&&) _NOEXCEPT;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_PAIR_H
diff --git a/libcxx/include/__cxx03/__fwd/queue.h b/libcxx/include/__cxx03/__fwd/queue.h
new file mode 100644
index 000000000000..50d99ad9c29f
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/queue.h
@@ -0,0 +1,31 @@
+//===---------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_QUEUE_H
+#define _LIBCPP___FWD_QUEUE_H
+
+#include <__config>
+#include <__functional/operations.h>
+#include <__fwd/deque.h>
+#include <__fwd/vector.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Container = deque<_Tp> >
+class _LIBCPP_TEMPLATE_VIS queue;
+
+template <class _Tp, class _Container = vector<_Tp>, class _Compare = less<typename _Container::value_type> >
+class _LIBCPP_TEMPLATE_VIS priority_queue;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_QUEUE_H
diff --git a/libcxx/include/__cxx03/__fwd/span.h b/libcxx/include/__cxx03/__fwd/span.h
new file mode 100644
index 000000000000..8dafa742c19d
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/span.h
@@ -0,0 +1,38 @@
+// -*- C++ -*-
+//===---------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_SPAN_H
+#define _LIBCPP___FWD_SPAN_H
+
+#include <__config>
+#include <cstddef>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+inline constexpr size_t dynamic_extent = numeric_limits<size_t>::max();
+template <typename _Tp, size_t _Extent = dynamic_extent>
+class span;
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FWD_SPAN_H
diff --git a/libcxx/include/__cxx03/__fwd/sstream.h b/libcxx/include/__cxx03/__fwd/sstream.h
new file mode 100644
index 000000000000..39a9c3faf1f8
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/sstream.h
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_SSTREAM_H
+#define _LIBCPP___FWD_SSTREAM_H
+
+#include <__config>
+#include <__fwd/memory.h>
+#include <__fwd/string.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits = char_traits<_CharT>, class _Allocator = allocator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_stringbuf;
+
+template <class _CharT, class _Traits = char_traits<_CharT>, class _Allocator = allocator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_istringstream;
+template <class _CharT, class _Traits = char_traits<_CharT>, class _Allocator = allocator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_ostringstream;
+template <class _CharT, class _Traits = char_traits<_CharT>, class _Allocator = allocator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_stringstream;
+
+using stringbuf     = basic_stringbuf<char>;
+using istringstream = basic_istringstream<char>;
+using ostringstream = basic_ostringstream<char>;
+using stringstream  = basic_stringstream<char>;
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using wstringbuf     = basic_stringbuf<wchar_t>;
+using wistringstream = basic_istringstream<wchar_t>;
+using wostringstream = basic_ostringstream<wchar_t>;
+using wstringstream  = basic_stringstream<wchar_t>;
+#endif
+
+template <class _CharT, class _Traits, class _Allocator>
+class _LIBCPP_PREFERRED_NAME(stringbuf) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wstringbuf)) basic_stringbuf;
+template <class _CharT, class _Traits, class _Allocator>
+class _LIBCPP_PREFERRED_NAME(istringstream)
+    _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wistringstream)) basic_istringstream;
+template <class _CharT, class _Traits, class _Allocator>
+class _LIBCPP_PREFERRED_NAME(ostringstream)
+    _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wostringstream)) basic_ostringstream;
+template <class _CharT, class _Traits, class _Allocator>
+class _LIBCPP_PREFERRED_NAME(stringstream)
+    _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wstringstream)) basic_stringstream;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_SSTREAM_H
diff --git a/libcxx/include/__cxx03/__fwd/stack.h b/libcxx/include/__cxx03/__fwd/stack.h
new file mode 100644
index 000000000000..7dab6c1a4f4e
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/stack.h
@@ -0,0 +1,26 @@
+//===---------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_STACK_H
+#define _LIBCPP___FWD_STACK_H
+
+#include <__config>
+#include <__fwd/deque.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Container = deque<_Tp> >
+class _LIBCPP_TEMPLATE_VIS stack;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_STACK_H
diff --git a/libcxx/include/__cxx03/__fwd/streambuf.h b/libcxx/include/__cxx03/__fwd/streambuf.h
new file mode 100644
index 000000000000..b35afa6afe34
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/streambuf.h
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_STREAMBUF_H
+#define _LIBCPP___FWD_STREAMBUF_H
+
+#include <__config>
+#include <__fwd/string.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_streambuf;
+
+using streambuf = basic_streambuf<char>;
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using wstreambuf = basic_streambuf<wchar_t>;
+#endif
+
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(streambuf) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wstreambuf)) basic_streambuf;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_STREAMBUF_H
diff --git a/libcxx/include/__cxx03/__fwd/string.h b/libcxx/include/__cxx03/__fwd/string.h
new file mode 100644
index 000000000000..2418e1f9b23d
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/string.h
@@ -0,0 +1,107 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_STRING_H
+#define _LIBCPP___FWD_STRING_H
+
+#include <__config>
+#include <__fwd/memory.h>
+#include <__fwd/memory_resource.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT>
+struct _LIBCPP_TEMPLATE_VIS char_traits;
+template <>
+struct char_traits<char>;
+
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+template <>
+struct char_traits<char8_t>;
+#endif
+
+template <>
+struct char_traits<char16_t>;
+template <>
+struct char_traits<char32_t>;
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+struct char_traits<wchar_t>;
+#endif
+
+template <class _CharT, class _Traits = char_traits<_CharT>, class _Allocator = allocator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_string;
+
+using string = basic_string<char>;
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using wstring = basic_string<wchar_t>;
+#endif
+
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+using u8string = basic_string<char8_t>;
+#endif
+
+using u16string = basic_string<char16_t>;
+using u32string = basic_string<char32_t>;
+
+#if _LIBCPP_STD_VER >= 17
+
+namespace pmr {
+template <class _CharT, class _Traits = char_traits<_CharT>>
+using basic_string _LIBCPP_AVAILABILITY_PMR = std::basic_string<_CharT, _Traits, polymorphic_allocator<_CharT>>;
+
+using string _LIBCPP_AVAILABILITY_PMR = basic_string<char>;
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using wstring _LIBCPP_AVAILABILITY_PMR = basic_string<wchar_t>;
+#  endif
+
+#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+using u8string _LIBCPP_AVAILABILITY_PMR = basic_string<char8_t>;
+#  endif
+
+using u16string _LIBCPP_AVAILABILITY_PMR = basic_string<char16_t>;
+using u32string _LIBCPP_AVAILABILITY_PMR = basic_string<char32_t>;
+} // namespace pmr
+
+#endif // _LIBCPP_STD_VER >= 17
+
+// clang-format off
+template <class _CharT, class _Traits, class _Allocator>
+class _LIBCPP_PREFERRED_NAME(string)
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+      _LIBCPP_PREFERRED_NAME(wstring)
+#endif
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+      _LIBCPP_PREFERRED_NAME(u8string)
+#endif
+      _LIBCPP_PREFERRED_NAME(u16string)
+      _LIBCPP_PREFERRED_NAME(u32string)
+#if _LIBCPP_STD_VER >= 17
+      _LIBCPP_PREFERRED_NAME(pmr::string)
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+      _LIBCPP_PREFERRED_NAME(pmr::wstring)
+#  endif
+#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+      _LIBCPP_PREFERRED_NAME(pmr::u8string)
+#  endif
+      _LIBCPP_PREFERRED_NAME(pmr::u16string)
+      _LIBCPP_PREFERRED_NAME(pmr::u32string)
+#endif
+      basic_string;
+// clang-format on
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_STRING_H
diff --git a/libcxx/include/__cxx03/__fwd/string_view.h b/libcxx/include/__cxx03/__fwd/string_view.h
new file mode 100644
index 000000000000..72a64be5b00b
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/string_view.h
@@ -0,0 +1,50 @@
+// -*- C++ -*-
+//===---------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_STRING_VIEW_H
+#define _LIBCPP___FWD_STRING_VIEW_H
+
+#include <__config>
+#include <__fwd/string.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_string_view;
+
+typedef basic_string_view<char> string_view;
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+typedef basic_string_view<char8_t> u8string_view;
+#endif
+typedef basic_string_view<char16_t> u16string_view;
+typedef basic_string_view<char32_t> u32string_view;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+typedef basic_string_view<wchar_t> wstring_view;
+#endif
+
+// clang-format off
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(string_view)
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+      _LIBCPP_PREFERRED_NAME(wstring_view)
+#endif
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+      _LIBCPP_PREFERRED_NAME(u8string_view)
+#endif
+      _LIBCPP_PREFERRED_NAME(u16string_view)
+      _LIBCPP_PREFERRED_NAME(u32string_view)
+      basic_string_view;
+// clang-format on
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_STRING_VIEW_H
diff --git a/libcxx/include/__cxx03/__fwd/subrange.h b/libcxx/include/__cxx03/__fwd/subrange.h
new file mode 100644
index 000000000000..60a41da23dd4
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/subrange.h
@@ -0,0 +1,49 @@
+//===---------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_SUBRANGE_H
+#define _LIBCPP___FWD_SUBRANGE_H
+
+#include <__concepts/copyable.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+enum class subrange_kind : bool { unsized, sized };
+
+template <input_or_output_iterator _Iter, sentinel_for<_Iter> _Sent, subrange_kind _Kind>
+  requires(_Kind == subrange_kind::sized || !sized_sentinel_for<_Sent, _Iter>)
+class _LIBCPP_TEMPLATE_VIS subrange;
+
+template <size_t _Index, class _Iter, class _Sent, subrange_kind _Kind>
+  requires((_Index == 0 && copyable<_Iter>) || _Index == 1)
+_LIBCPP_HIDE_FROM_ABI constexpr auto get(const subrange<_Iter, _Sent, _Kind>&);
+
+template <size_t _Index, class _Iter, class _Sent, subrange_kind _Kind>
+  requires(_Index < 2)
+_LIBCPP_HIDE_FROM_ABI constexpr auto get(subrange<_Iter, _Sent, _Kind>&&);
+
+} // namespace ranges
+
+using ranges::get;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___FWD_SUBRANGE_H
diff --git a/libcxx/include/__cxx03/__fwd/tuple.h b/libcxx/include/__cxx03/__fwd/tuple.h
new file mode 100644
index 000000000000..902770c29555
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/tuple.h
@@ -0,0 +1,52 @@
+//===---------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_TUPLE_H
+#define _LIBCPP___FWD_TUPLE_H
+
+#include <__config>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <size_t, class>
+struct _LIBCPP_TEMPLATE_VIS tuple_element;
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class...>
+class _LIBCPP_TEMPLATE_VIS tuple;
+
+template <class>
+struct _LIBCPP_TEMPLATE_VIS tuple_size;
+
+template <size_t _Ip, class... _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, tuple<_Tp...> >::type&
+get(tuple<_Tp...>&) _NOEXCEPT;
+
+template <size_t _Ip, class... _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, tuple<_Tp...> >::type&
+get(const tuple<_Tp...>&) _NOEXCEPT;
+
+template <size_t _Ip, class... _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, tuple<_Tp...> >::type&&
+get(tuple<_Tp...>&&) _NOEXCEPT;
+
+template <size_t _Ip, class... _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, tuple<_Tp...> >::type&&
+get(const tuple<_Tp...>&&) _NOEXCEPT;
+
+#endif // _LIBCPP_CXX03_LANG
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_TUPLE_H
diff --git a/libcxx/include/__cxx03/__fwd/vector.h b/libcxx/include/__cxx03/__fwd/vector.h
new file mode 100644
index 000000000000..c9cc96137449
--- /dev/null
+++ b/libcxx/include/__cxx03/__fwd/vector.h
@@ -0,0 +1,26 @@
+//===---------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_VECTOR_H
+#define _LIBCPP___FWD_VECTOR_H
+
+#include <__config>
+#include <__fwd/memory.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Alloc = allocator<_Tp> >
+class _LIBCPP_TEMPLATE_VIS vector;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_VECTOR_H
diff --git a/libcxx/include/__cxx03/__hash_table b/libcxx/include/__cxx03/__hash_table
new file mode 100644
index 000000000000..025758528573
--- /dev/null
+++ b/libcxx/include/__cxx03/__hash_table
@@ -0,0 +1,2044 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___HASH_TABLE
+#define _LIBCPP___HASH_TABLE
+
+#include <__algorithm/max.h>
+#include <__algorithm/min.h>
+#include <__assert>
+#include <__bit/countl.h>
+#include <__config>
+#include <__functional/hash.h>
+#include <__functional/invoke.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/compressed_pair.h>
+#include <__memory/construct_at.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/swap_allocator.h>
+#include <__memory/unique_ptr.h>
+#include <__type_traits/can_extract_key.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/is_const.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/is_swappable.h>
+#include <__type_traits/remove_const.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <__utility/swap.h>
+#include <cmath>
+#include <cstring>
+#include <initializer_list>
+#include <new> // __launder
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Key, class _Tp>
+struct __hash_value_type;
+
+template <class _Tp>
+struct __is_hash_value_type_imp : false_type {};
+
+template <class _Key, class _Value>
+struct __is_hash_value_type_imp<__hash_value_type<_Key, _Value> > : true_type {};
+
+template <class... _Args>
+struct __is_hash_value_type : false_type {};
+
+template <class _One>
+struct __is_hash_value_type<_One> : __is_hash_value_type_imp<__remove_cvref_t<_One> > {};
+
+_LIBCPP_EXPORTED_FROM_ABI size_t __next_prime(size_t __n);
+
+template <class _NodePtr>
+struct __hash_node_base {
+  typedef typename pointer_traits<_NodePtr>::element_type __node_type;
+  typedef __hash_node_base __first_node;
+  typedef __rebind_pointer_t<_NodePtr, __first_node> __node_base_pointer;
+  typedef _NodePtr __node_pointer;
+
+#if defined(_LIBCPP_ABI_FIX_UNORDERED_NODE_POINTER_UB)
+  typedef __node_base_pointer __next_pointer;
+#else
+  typedef __conditional_t<is_pointer<__node_pointer>::value, __node_base_pointer, __node_pointer> __next_pointer;
+#endif
+
+  __next_pointer __next_;
+
+  _LIBCPP_HIDE_FROM_ABI __next_pointer __ptr() _NOEXCEPT {
+    return static_cast<__next_pointer>(pointer_traits<__node_base_pointer>::pointer_to(*this));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __node_pointer __upcast() _NOEXCEPT {
+    return static_cast<__node_pointer>(pointer_traits<__node_base_pointer>::pointer_to(*this));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_t __hash() const _NOEXCEPT { return static_cast<__node_type const&>(*this).__hash_; }
+
+  _LIBCPP_HIDE_FROM_ABI __hash_node_base() _NOEXCEPT : __next_(nullptr) {}
+  _LIBCPP_HIDE_FROM_ABI explicit __hash_node_base(__next_pointer __next) _NOEXCEPT : __next_(__next) {}
+};
+
+template <class _Tp, class _VoidPtr>
+struct __hash_node : public __hash_node_base< __rebind_pointer_t<_VoidPtr, __hash_node<_Tp, _VoidPtr> > > {
+  typedef _Tp __node_value_type;
+  using _Base          = __hash_node_base<__rebind_pointer_t<_VoidPtr, __hash_node<_Tp, _VoidPtr> > >;
+  using __next_pointer = typename _Base::__next_pointer;
+
+  size_t __hash_;
+
+  // We allow starting the lifetime of nodes without initializing the value held by the node,
+  // since that is handled by the hash table itself in order to be allocator-aware.
+#ifndef _LIBCPP_CXX03_LANG
+
+private:
+  union {
+    _Tp __value_;
+  };
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; }
+#else
+
+private:
+  _ALIGNAS_TYPE(_Tp) char __buffer_[sizeof(_Tp)];
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return *std::__launder(reinterpret_cast<_Tp*>(&__buffer_)); }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI explicit __hash_node(__next_pointer __next, size_t __hash) : _Base(__next), __hash_(__hash) {}
+  _LIBCPP_HIDE_FROM_ABI ~__hash_node() {}
+};
+
+inline _LIBCPP_HIDE_FROM_ABI bool __is_hash_power2(size_t __bc) { return __bc > 2 && !(__bc & (__bc - 1)); }
+
+inline _LIBCPP_HIDE_FROM_ABI size_t __constrain_hash(size_t __h, size_t __bc) {
+  return !(__bc & (__bc - 1)) ? __h & (__bc - 1) : (__h < __bc ? __h : __h % __bc);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI size_t __next_hash_pow2(size_t __n) {
+  return __n < 2 ? __n : (size_t(1) << (numeric_limits<size_t>::digits - __libcpp_clz(__n - 1)));
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+class __hash_table;
+
+template <class _NodePtr>
+class _LIBCPP_TEMPLATE_VIS __hash_iterator;
+template <class _ConstNodePtr>
+class _LIBCPP_TEMPLATE_VIS __hash_const_iterator;
+template <class _NodePtr>
+class _LIBCPP_TEMPLATE_VIS __hash_local_iterator;
+template <class _ConstNodePtr>
+class _LIBCPP_TEMPLATE_VIS __hash_const_local_iterator;
+template <class _HashIterator>
+class _LIBCPP_TEMPLATE_VIS __hash_map_iterator;
+template <class _HashIterator>
+class _LIBCPP_TEMPLATE_VIS __hash_map_const_iterator;
+
+template <class _Tp>
+struct __hash_key_value_types {
+  static_assert(!is_reference<_Tp>::value && !is_const<_Tp>::value, "");
+  typedef _Tp key_type;
+  typedef _Tp __node_value_type;
+  typedef _Tp __container_value_type;
+  static const bool __is_map = false;
+
+  _LIBCPP_HIDE_FROM_ABI static key_type const& __get_key(_Tp const& __v) { return __v; }
+  _LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(__node_value_type const& __v) { return __v; }
+  _LIBCPP_HIDE_FROM_ABI static __container_value_type* __get_ptr(__node_value_type& __n) { return std::addressof(__n); }
+  _LIBCPP_HIDE_FROM_ABI static __container_value_type&& __move(__node_value_type& __v) { return std::move(__v); }
+};
+
+template <class _Key, class _Tp>
+struct __hash_key_value_types<__hash_value_type<_Key, _Tp> > {
+  typedef _Key key_type;
+  typedef _Tp mapped_type;
+  typedef __hash_value_type<_Key, _Tp> __node_value_type;
+  typedef pair<const _Key, _Tp> __container_value_type;
+  typedef __container_value_type __map_value_type;
+  static const bool __is_map = true;
+
+  _LIBCPP_HIDE_FROM_ABI static key_type const& __get_key(__container_value_type const& __v) { return __v.first; }
+
+  template <class _Up, __enable_if_t<__is_same_uncvref<_Up, __node_value_type>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(_Up& __t) {
+    return __t.__get_value();
+  }
+
+  template <class _Up, __enable_if_t<__is_same_uncvref<_Up, __container_value_type>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(_Up& __t) {
+    return __t;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static __container_value_type* __get_ptr(__node_value_type& __n) {
+    return std::addressof(__n.__get_value());
+  }
+  _LIBCPP_HIDE_FROM_ABI static pair<key_type&&, mapped_type&&> __move(__node_value_type& __v) { return __v.__move(); }
+};
+
+template <class _Tp, class _AllocPtr, class _KVTypes = __hash_key_value_types<_Tp>, bool = _KVTypes::__is_map>
+struct __hash_map_pointer_types {};
+
+template <class _Tp, class _AllocPtr, class _KVTypes>
+struct __hash_map_pointer_types<_Tp, _AllocPtr, _KVTypes, true> {
+  typedef typename _KVTypes::__map_value_type _Mv;
+  typedef __rebind_pointer_t<_AllocPtr, _Mv> __map_value_type_pointer;
+  typedef __rebind_pointer_t<_AllocPtr, const _Mv> __const_map_value_type_pointer;
+};
+
+template <class _NodePtr, class _NodeT = typename pointer_traits<_NodePtr>::element_type>
+struct __hash_node_types;
+
+template <class _NodePtr, class _Tp, class _VoidPtr>
+struct __hash_node_types<_NodePtr, __hash_node<_Tp, _VoidPtr> >
+    : public __hash_key_value_types<_Tp>,
+      __hash_map_pointer_types<_Tp, _VoidPtr>
+
+{
+  typedef __hash_key_value_types<_Tp> __base;
+
+public:
+  typedef ptrdiff_t difference_type;
+  typedef size_t size_type;
+
+  typedef __rebind_pointer_t<_NodePtr, void> __void_pointer;
+
+  typedef typename pointer_traits<_NodePtr>::element_type __node_type;
+  typedef _NodePtr __node_pointer;
+
+  typedef __hash_node_base<__node_pointer> __node_base_type;
+  typedef __rebind_pointer_t<_NodePtr, __node_base_type> __node_base_pointer;
+
+  typedef typename __node_base_type::__next_pointer __next_pointer;
+
+  typedef _Tp __node_value_type;
+  typedef __rebind_pointer_t<_VoidPtr, __node_value_type> __node_value_type_pointer;
+  typedef __rebind_pointer_t<_VoidPtr, const __node_value_type> __const_node_value_type_pointer;
+
+private:
+  static_assert(!is_const<__node_type>::value, "_NodePtr should never be a pointer to const");
+  static_assert(is_same<typename pointer_traits<_VoidPtr>::element_type, void>::value,
+                "_VoidPtr does not point to unqualified void type");
+  static_assert(is_same<__rebind_pointer_t<_VoidPtr, __node_type>, _NodePtr>::value,
+                "_VoidPtr does not rebind to _NodePtr.");
+};
+
+template <class _HashIterator>
+struct __hash_node_types_from_iterator;
+template <class _NodePtr>
+struct __hash_node_types_from_iterator<__hash_iterator<_NodePtr> > : __hash_node_types<_NodePtr> {};
+template <class _NodePtr>
+struct __hash_node_types_from_iterator<__hash_const_iterator<_NodePtr> > : __hash_node_types<_NodePtr> {};
+template <class _NodePtr>
+struct __hash_node_types_from_iterator<__hash_local_iterator<_NodePtr> > : __hash_node_types<_NodePtr> {};
+template <class _NodePtr>
+struct __hash_node_types_from_iterator<__hash_const_local_iterator<_NodePtr> > : __hash_node_types<_NodePtr> {};
+
+template <class _NodeValueTp, class _VoidPtr>
+struct __make_hash_node_types {
+  typedef __hash_node<_NodeValueTp, _VoidPtr> _NodeTp;
+  typedef __rebind_pointer_t<_VoidPtr, _NodeTp> _NodePtr;
+  typedef __hash_node_types<_NodePtr> type;
+};
+
+template <class _NodePtr>
+class _LIBCPP_TEMPLATE_VIS __hash_iterator {
+  typedef __hash_node_types<_NodePtr> _NodeTypes;
+  typedef _NodePtr __node_pointer;
+  typedef typename _NodeTypes::__next_pointer __next_pointer;
+
+  __next_pointer __node_;
+
+public:
+  typedef forward_iterator_tag iterator_category;
+  typedef typename _NodeTypes::__node_value_type value_type;
+  typedef typename _NodeTypes::difference_type difference_type;
+  typedef value_type& reference;
+  typedef typename _NodeTypes::__node_value_type_pointer pointer;
+
+  _LIBCPP_HIDE_FROM_ABI __hash_iterator() _NOEXCEPT : __node_(nullptr) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const {
+    _LIBCPP_ASSERT_NON_NULL(
+        __node_ != nullptr, "Attempted to dereference a non-dereferenceable unordered container iterator");
+    return __node_->__upcast()->__get_value();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
+    _LIBCPP_ASSERT_NON_NULL(
+        __node_ != nullptr, "Attempted to dereference a non-dereferenceable unordered container iterator");
+    return pointer_traits<pointer>::pointer_to(__node_->__upcast()->__get_value());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __hash_iterator& operator++() {
+    _LIBCPP_ASSERT_NON_NULL(
+        __node_ != nullptr, "Attempted to increment a non-incrementable unordered container iterator");
+    __node_ = __node_->__next_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __hash_iterator operator++(int) {
+    __hash_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __hash_iterator& __x, const __hash_iterator& __y) {
+    return __x.__node_ == __y.__node_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __hash_iterator& __x, const __hash_iterator& __y) {
+    return !(__x == __y);
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI explicit __hash_iterator(__next_pointer __node) _NOEXCEPT : __node_(__node) {}
+
+  template <class, class, class, class>
+  friend class __hash_table;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_const_iterator;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_map_iterator;
+  template <class, class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_map;
+  template <class, class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_multimap;
+};
+
+template <class _NodePtr>
+class _LIBCPP_TEMPLATE_VIS __hash_const_iterator {
+  static_assert(!is_const<typename pointer_traits<_NodePtr>::element_type>::value, "");
+  typedef __hash_node_types<_NodePtr> _NodeTypes;
+  typedef _NodePtr __node_pointer;
+  typedef typename _NodeTypes::__next_pointer __next_pointer;
+
+  __next_pointer __node_;
+
+public:
+  typedef __hash_iterator<_NodePtr> __non_const_iterator;
+
+  typedef forward_iterator_tag iterator_category;
+  typedef typename _NodeTypes::__node_value_type value_type;
+  typedef typename _NodeTypes::difference_type difference_type;
+  typedef const value_type& reference;
+  typedef typename _NodeTypes::__const_node_value_type_pointer pointer;
+
+  _LIBCPP_HIDE_FROM_ABI __hash_const_iterator() _NOEXCEPT : __node_(nullptr) {}
+
+  _LIBCPP_HIDE_FROM_ABI __hash_const_iterator(const __non_const_iterator& __x) _NOEXCEPT : __node_(__x.__node_) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const {
+    _LIBCPP_ASSERT_NON_NULL(
+        __node_ != nullptr, "Attempted to dereference a non-dereferenceable unordered container const_iterator");
+    return __node_->__upcast()->__get_value();
+  }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
+    _LIBCPP_ASSERT_NON_NULL(
+        __node_ != nullptr, "Attempted to dereference a non-dereferenceable unordered container const_iterator");
+    return pointer_traits<pointer>::pointer_to(__node_->__upcast()->__get_value());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __hash_const_iterator& operator++() {
+    _LIBCPP_ASSERT_NON_NULL(
+        __node_ != nullptr, "Attempted to increment a non-incrementable unordered container const_iterator");
+    __node_ = __node_->__next_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __hash_const_iterator operator++(int) {
+    __hash_const_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __hash_const_iterator& __x, const __hash_const_iterator& __y) {
+    return __x.__node_ == __y.__node_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __hash_const_iterator& __x, const __hash_const_iterator& __y) {
+    return !(__x == __y);
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI explicit __hash_const_iterator(__next_pointer __node) _NOEXCEPT : __node_(__node) {}
+
+  template <class, class, class, class>
+  friend class __hash_table;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_map_const_iterator;
+  template <class, class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_map;
+  template <class, class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_multimap;
+};
+
+template <class _NodePtr>
+class _LIBCPP_TEMPLATE_VIS __hash_local_iterator {
+  typedef __hash_node_types<_NodePtr> _NodeTypes;
+  typedef _NodePtr __node_pointer;
+  typedef typename _NodeTypes::__next_pointer __next_pointer;
+
+  __next_pointer __node_;
+  size_t __bucket_;
+  size_t __bucket_count_;
+
+public:
+  typedef forward_iterator_tag iterator_category;
+  typedef typename _NodeTypes::__node_value_type value_type;
+  typedef typename _NodeTypes::difference_type difference_type;
+  typedef value_type& reference;
+  typedef typename _NodeTypes::__node_value_type_pointer pointer;
+
+  _LIBCPP_HIDE_FROM_ABI __hash_local_iterator() _NOEXCEPT : __node_(nullptr) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const {
+    _LIBCPP_ASSERT_NON_NULL(
+        __node_ != nullptr, "Attempted to dereference a non-dereferenceable unordered container local_iterator");
+    return __node_->__upcast()->__get_value();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
+    _LIBCPP_ASSERT_NON_NULL(
+        __node_ != nullptr, "Attempted to dereference a non-dereferenceable unordered container local_iterator");
+    return pointer_traits<pointer>::pointer_to(__node_->__upcast()->__get_value());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __hash_local_iterator& operator++() {
+    _LIBCPP_ASSERT_NON_NULL(
+        __node_ != nullptr, "Attempted to increment a non-incrementable unordered container local_iterator");
+    __node_ = __node_->__next_;
+    if (__node_ != nullptr && std::__constrain_hash(__node_->__hash(), __bucket_count_) != __bucket_)
+      __node_ = nullptr;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __hash_local_iterator operator++(int) {
+    __hash_local_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __hash_local_iterator& __x, const __hash_local_iterator& __y) {
+    return __x.__node_ == __y.__node_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __hash_local_iterator& __x, const __hash_local_iterator& __y) {
+    return !(__x == __y);
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI explicit __hash_local_iterator(
+      __next_pointer __node, size_t __bucket, size_t __bucket_count) _NOEXCEPT
+      : __node_(__node),
+        __bucket_(__bucket),
+        __bucket_count_(__bucket_count) {
+    if (__node_ != nullptr)
+      __node_ = __node_->__next_;
+  }
+
+  template <class, class, class, class>
+  friend class __hash_table;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_const_local_iterator;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_map_iterator;
+};
+
+template <class _ConstNodePtr>
+class _LIBCPP_TEMPLATE_VIS __hash_const_local_iterator {
+  typedef __hash_node_types<_ConstNodePtr> _NodeTypes;
+  typedef _ConstNodePtr __node_pointer;
+  typedef typename _NodeTypes::__next_pointer __next_pointer;
+
+  __next_pointer __node_;
+  size_t __bucket_;
+  size_t __bucket_count_;
+
+  typedef pointer_traits<__node_pointer> __pointer_traits;
+  typedef typename __pointer_traits::element_type __node;
+  typedef __remove_const_t<__node> __non_const_node;
+  typedef __rebind_pointer_t<__node_pointer, __non_const_node> __non_const_node_pointer;
+
+public:
+  typedef __hash_local_iterator<__non_const_node_pointer> __non_const_iterator;
+
+  typedef forward_iterator_tag iterator_category;
+  typedef typename _NodeTypes::__node_value_type value_type;
+  typedef typename _NodeTypes::difference_type difference_type;
+  typedef const value_type& reference;
+  typedef typename _NodeTypes::__const_node_value_type_pointer pointer;
+
+  _LIBCPP_HIDE_FROM_ABI __hash_const_local_iterator() _NOEXCEPT : __node_(nullptr) {}
+
+  _LIBCPP_HIDE_FROM_ABI __hash_const_local_iterator(const __non_const_iterator& __x) _NOEXCEPT
+      : __node_(__x.__node_),
+        __bucket_(__x.__bucket_),
+        __bucket_count_(__x.__bucket_count_) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const {
+    _LIBCPP_ASSERT_NON_NULL(
+        __node_ != nullptr, "Attempted to dereference a non-dereferenceable unordered container const_local_iterator");
+    return __node_->__upcast()->__get_value();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
+    _LIBCPP_ASSERT_NON_NULL(
+        __node_ != nullptr, "Attempted to dereference a non-dereferenceable unordered container const_local_iterator");
+    return pointer_traits<pointer>::pointer_to(__node_->__upcast()->__get_value());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __hash_const_local_iterator& operator++() {
+    _LIBCPP_ASSERT_NON_NULL(
+        __node_ != nullptr, "Attempted to increment a non-incrementable unordered container const_local_iterator");
+    __node_ = __node_->__next_;
+    if (__node_ != nullptr && std::__constrain_hash(__node_->__hash(), __bucket_count_) != __bucket_)
+      __node_ = nullptr;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __hash_const_local_iterator operator++(int) {
+    __hash_const_local_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const __hash_const_local_iterator& __x, const __hash_const_local_iterator& __y) {
+    return __x.__node_ == __y.__node_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator!=(const __hash_const_local_iterator& __x, const __hash_const_local_iterator& __y) {
+    return !(__x == __y);
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI explicit __hash_const_local_iterator(
+      __next_pointer __node_ptr, size_t __bucket, size_t __bucket_count) _NOEXCEPT
+      : __node_(__node_ptr),
+        __bucket_(__bucket),
+        __bucket_count_(__bucket_count) {
+    if (__node_ != nullptr)
+      __node_ = __node_->__next_;
+  }
+
+  template <class, class, class, class>
+  friend class __hash_table;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_map_const_iterator;
+};
+
+template <class _Alloc>
+class __bucket_list_deallocator {
+  typedef _Alloc allocator_type;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+  typedef typename __alloc_traits::size_type size_type;
+
+  __compressed_pair<size_type, allocator_type> __data_;
+
+public:
+  typedef typename __alloc_traits::pointer pointer;
+
+  _LIBCPP_HIDE_FROM_ABI __bucket_list_deallocator() _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
+      : __data_(0, __default_init_tag()) {}
+
+  _LIBCPP_HIDE_FROM_ABI __bucket_list_deallocator(const allocator_type& __a, size_type __size)
+      _NOEXCEPT_(is_nothrow_copy_constructible<allocator_type>::value)
+      : __data_(__size, __a) {}
+
+  _LIBCPP_HIDE_FROM_ABI __bucket_list_deallocator(__bucket_list_deallocator&& __x)
+      _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value)
+      : __data_(std::move(__x.__data_)) {
+    __x.size() = 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_type& size() _NOEXCEPT { return __data_.first(); }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __data_.first(); }
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type& __alloc() _NOEXCEPT { return __data_.second(); }
+  _LIBCPP_HIDE_FROM_ABI const allocator_type& __alloc() const _NOEXCEPT { return __data_.second(); }
+
+  _LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) _NOEXCEPT { __alloc_traits::deallocate(__alloc(), __p, size()); }
+};
+
+template <class _Alloc>
+class __hash_map_node_destructor;
+
+template <class _Alloc>
+class __hash_node_destructor {
+  typedef _Alloc allocator_type;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+
+public:
+  typedef typename __alloc_traits::pointer pointer;
+
+private:
+  typedef __hash_node_types<pointer> _NodeTypes;
+
+  allocator_type& __na_;
+
+public:
+  bool __value_constructed;
+
+  _LIBCPP_HIDE_FROM_ABI __hash_node_destructor(__hash_node_destructor const&)            = default;
+  _LIBCPP_HIDE_FROM_ABI __hash_node_destructor& operator=(const __hash_node_destructor&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __hash_node_destructor(allocator_type& __na, bool __constructed = false) _NOEXCEPT
+      : __na_(__na),
+        __value_constructed(__constructed) {}
+
+  _LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) _NOEXCEPT {
+    if (__value_constructed) {
+      __alloc_traits::destroy(__na_, _NodeTypes::__get_ptr(__p->__get_value()));
+      std::__destroy_at(std::addressof(*__p));
+    }
+    if (__p)
+      __alloc_traits::deallocate(__na_, __p, 1);
+  }
+
+  template <class>
+  friend class __hash_map_node_destructor;
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _NodeType, class _Alloc>
+struct __generic_container_node_destructor;
+
+template <class _Tp, class _VoidPtr, class _Alloc>
+struct __generic_container_node_destructor<__hash_node<_Tp, _VoidPtr>, _Alloc> : __hash_node_destructor<_Alloc> {
+  using __hash_node_destructor<_Alloc>::__hash_node_destructor;
+};
+#endif
+
+template <class _Key, class _Hash, class _Equal>
+struct __enforce_unordered_container_requirements {
+#ifndef _LIBCPP_CXX03_LANG
+  static_assert(__check_hash_requirements<_Key, _Hash>::value,
+                "the specified hash does not meet the Hash requirements");
+  static_assert(is_copy_constructible<_Equal>::value, "the specified comparator is required to be copy constructible");
+#endif
+  typedef int type;
+};
+
+template <class _Key, class _Hash, class _Equal>
+#ifndef _LIBCPP_CXX03_LANG
+_LIBCPP_DIAGNOSE_WARNING(!__invokable<_Equal const&, _Key const&, _Key const&>::value,
+                         "the specified comparator type does not provide a viable const call operator")
+_LIBCPP_DIAGNOSE_WARNING(!__invokable<_Hash const&, _Key const&>::value,
+                         "the specified hash functor does not provide a viable const call operator")
+#endif
+    typename __enforce_unordered_container_requirements<_Key, _Hash, _Equal>::type
+    __diagnose_unordered_container_requirements(int);
+
+// This dummy overload is used so that the compiler won't emit a spurious
+// "no matching function for call to __diagnose_unordered_xxx" diagnostic
+// when the overload above causes a hard error.
+template <class _Key, class _Hash, class _Equal>
+int __diagnose_unordered_container_requirements(void*);
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+class __hash_table {
+public:
+  typedef _Tp value_type;
+  typedef _Hash hasher;
+  typedef _Equal key_equal;
+  typedef _Alloc allocator_type;
+
+private:
+  typedef allocator_traits<allocator_type> __alloc_traits;
+  typedef typename __make_hash_node_types<value_type, typename __alloc_traits::void_pointer>::type _NodeTypes;
+
+public:
+  typedef typename _NodeTypes::__node_value_type __node_value_type;
+  typedef typename _NodeTypes::__container_value_type __container_value_type;
+  typedef typename _NodeTypes::key_type key_type;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+  typedef typename __alloc_traits::pointer pointer;
+  typedef typename __alloc_traits::const_pointer const_pointer;
+#ifndef _LIBCPP_ABI_FIX_UNORDERED_CONTAINER_SIZE_TYPE
+  typedef typename __alloc_traits::size_type size_type;
+#else
+  typedef typename _NodeTypes::size_type size_type;
+#endif
+  typedef typename _NodeTypes::difference_type difference_type;
+
+public:
+  // Create __node
+
+  typedef typename _NodeTypes::__node_type __node;
+  typedef __rebind_alloc<__alloc_traits, __node> __node_allocator;
+  typedef allocator_traits<__node_allocator> __node_traits;
+  typedef typename _NodeTypes::__void_pointer __void_pointer;
+  typedef typename _NodeTypes::__node_pointer __node_pointer;
+  typedef typename _NodeTypes::__node_pointer __node_const_pointer;
+  typedef typename _NodeTypes::__node_base_type __first_node;
+  typedef typename _NodeTypes::__node_base_pointer __node_base_pointer;
+  typedef typename _NodeTypes::__next_pointer __next_pointer;
+
+private:
+  // check for sane allocator pointer rebinding semantics. Rebinding the
+  // allocator for a new pointer type should be exactly the same as rebinding
+  // the pointer using 'pointer_traits'.
+  static_assert(is_same<__node_pointer, typename __node_traits::pointer>::value,
+                "Allocator does not rebind pointers in a sane manner.");
+  typedef __rebind_alloc<__node_traits, __first_node> __node_base_allocator;
+  typedef allocator_traits<__node_base_allocator> __node_base_traits;
+  static_assert(is_same<__node_base_pointer, typename __node_base_traits::pointer>::value,
+                "Allocator does not rebind pointers in a sane manner.");
+
+private:
+  typedef __rebind_alloc<__node_traits, __next_pointer> __pointer_allocator;
+  typedef __bucket_list_deallocator<__pointer_allocator> __bucket_list_deleter;
+  typedef unique_ptr<__next_pointer[], __bucket_list_deleter> __bucket_list;
+  typedef allocator_traits<__pointer_allocator> __pointer_alloc_traits;
+  typedef typename __bucket_list_deleter::pointer __node_pointer_pointer;
+
+  // --- Member data begin ---
+  __bucket_list __bucket_list_;
+  __compressed_pair<__first_node, __node_allocator> __p1_;
+  __compressed_pair<size_type, hasher> __p2_;
+  __compressed_pair<float, key_equal> __p3_;
+  // --- Member data end ---
+
+  _LIBCPP_HIDE_FROM_ABI size_type& size() _NOEXCEPT { return __p2_.first(); }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __p2_.first(); }
+
+  _LIBCPP_HIDE_FROM_ABI hasher& hash_function() _NOEXCEPT { return __p2_.second(); }
+  _LIBCPP_HIDE_FROM_ABI const hasher& hash_function() const _NOEXCEPT { return __p2_.second(); }
+
+  _LIBCPP_HIDE_FROM_ABI float& max_load_factor() _NOEXCEPT { return __p3_.first(); }
+  _LIBCPP_HIDE_FROM_ABI float max_load_factor() const _NOEXCEPT { return __p3_.first(); }
+
+  _LIBCPP_HIDE_FROM_ABI key_equal& key_eq() _NOEXCEPT { return __p3_.second(); }
+  _LIBCPP_HIDE_FROM_ABI const key_equal& key_eq() const _NOEXCEPT { return __p3_.second(); }
+
+  _LIBCPP_HIDE_FROM_ABI __node_allocator& __node_alloc() _NOEXCEPT { return __p1_.second(); }
+  _LIBCPP_HIDE_FROM_ABI const __node_allocator& __node_alloc() const _NOEXCEPT { return __p1_.second(); }
+
+public:
+  typedef __hash_iterator<__node_pointer> iterator;
+  typedef __hash_const_iterator<__node_pointer> const_iterator;
+  typedef __hash_local_iterator<__node_pointer> local_iterator;
+  typedef __hash_const_local_iterator<__node_pointer> const_local_iterator;
+
+  _LIBCPP_HIDE_FROM_ABI __hash_table() _NOEXCEPT_(
+      is_nothrow_default_constructible<__bucket_list>::value&& is_nothrow_default_constructible<__first_node>::value&&
+          is_nothrow_default_constructible<__node_allocator>::value&& is_nothrow_default_constructible<hasher>::value&&
+              is_nothrow_default_constructible<key_equal>::value);
+  _LIBCPP_HIDE_FROM_ABI __hash_table(const hasher& __hf, const key_equal& __eql);
+  _LIBCPP_HIDE_FROM_ABI __hash_table(const hasher& __hf, const key_equal& __eql, const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI explicit __hash_table(const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI __hash_table(const __hash_table& __u);
+  _LIBCPP_HIDE_FROM_ABI __hash_table(const __hash_table& __u, const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI __hash_table(__hash_table&& __u) _NOEXCEPT_(
+      is_nothrow_move_constructible<__bucket_list>::value&& is_nothrow_move_constructible<__first_node>::value&&
+          is_nothrow_move_constructible<__node_allocator>::value&& is_nothrow_move_constructible<hasher>::value&&
+              is_nothrow_move_constructible<key_equal>::value);
+  _LIBCPP_HIDE_FROM_ABI __hash_table(__hash_table&& __u, const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI ~__hash_table();
+
+  _LIBCPP_HIDE_FROM_ABI __hash_table& operator=(const __hash_table& __u);
+  _LIBCPP_HIDE_FROM_ABI __hash_table& operator=(__hash_table&& __u)
+      _NOEXCEPT_(__node_traits::propagate_on_container_move_assignment::value&&
+                     is_nothrow_move_assignable<__node_allocator>::value&& is_nothrow_move_assignable<hasher>::value&&
+                         is_nothrow_move_assignable<key_equal>::value);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void __assign_unique(_InputIterator __first, _InputIterator __last);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void __assign_multi(_InputIterator __first, _InputIterator __last);
+
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
+    return std::min<size_type>(__node_traits::max_size(__node_alloc()), numeric_limits<difference_type >::max());
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI __next_pointer __node_insert_multi_prepare(size_t __cp_hash, value_type& __cp_val);
+  _LIBCPP_HIDE_FROM_ABI void __node_insert_multi_perform(__node_pointer __cp, __next_pointer __pn) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI __next_pointer __node_insert_unique_prepare(size_t __nd_hash, value_type& __nd_val);
+  _LIBCPP_HIDE_FROM_ABI void __node_insert_unique_perform(__node_pointer __ptr) _NOEXCEPT;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __node_insert_unique(__node_pointer __nd);
+  _LIBCPP_HIDE_FROM_ABI iterator __node_insert_multi(__node_pointer __nd);
+  _LIBCPP_HIDE_FROM_ABI iterator __node_insert_multi(const_iterator __p, __node_pointer __nd);
+
+  template <class _Key, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_key_args(_Key const& __k, _Args&&... __args);
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_impl(_Args&&... __args);
+
+  template <class _Pp>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique(_Pp&& __x) {
+    return __emplace_unique_extract_key(std::forward<_Pp>(__x), __can_extract_key<_Pp, key_type>());
+  }
+
+  template <class _First,
+            class _Second,
+            __enable_if_t<__can_extract_map_key<_First, key_type, __container_value_type>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique(_First&& __f, _Second&& __s) {
+    return __emplace_unique_key_args(__f, std::forward<_First>(__f), std::forward<_Second>(__s));
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique(_Args&&... __args) {
+    return __emplace_unique_impl(std::forward<_Args>(__args)...);
+  }
+
+  template <class _Pp>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_extract_key(_Pp&& __x, __extract_key_fail_tag) {
+    return __emplace_unique_impl(std::forward<_Pp>(__x));
+  }
+  template <class _Pp>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_extract_key(_Pp&& __x, __extract_key_self_tag) {
+    return __emplace_unique_key_args(__x, std::forward<_Pp>(__x));
+  }
+  template <class _Pp>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_extract_key(_Pp&& __x, __extract_key_first_tag) {
+    return __emplace_unique_key_args(__x.first, std::forward<_Pp>(__x));
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator __emplace_multi(_Args&&... __args);
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator __emplace_hint_multi(const_iterator __p, _Args&&... __args);
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __insert_unique(__container_value_type&& __x) {
+    return __emplace_unique_key_args(_NodeTypes::__get_key(__x), std::move(__x));
+  }
+
+  template <class _Pp, __enable_if_t<!__is_same_uncvref<_Pp, __container_value_type>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __insert_unique(_Pp&& __x) {
+    return __emplace_unique(std::forward<_Pp>(__x));
+  }
+
+  template <class _Pp>
+  _LIBCPP_HIDE_FROM_ABI iterator __insert_multi(_Pp&& __x) {
+    return __emplace_multi(std::forward<_Pp>(__x));
+  }
+
+  template <class _Pp>
+  _LIBCPP_HIDE_FROM_ABI iterator __insert_multi(const_iterator __p, _Pp&& __x) {
+    return __emplace_hint_multi(__p, std::forward<_Pp>(__x));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __insert_unique(const __container_value_type& __x) {
+    return __emplace_unique_key_args(_NodeTypes::__get_key(__x), __x);
+  }
+
+#if _LIBCPP_STD_VER >= 17
+  template <class _NodeHandle, class _InsertReturnType>
+  _LIBCPP_HIDE_FROM_ABI _InsertReturnType __node_handle_insert_unique(_NodeHandle&& __nh);
+  template <class _NodeHandle>
+  _LIBCPP_HIDE_FROM_ABI iterator __node_handle_insert_unique(const_iterator __hint, _NodeHandle&& __nh);
+  template <class _Table>
+  _LIBCPP_HIDE_FROM_ABI void __node_handle_merge_unique(_Table& __source);
+
+  template <class _NodeHandle>
+  _LIBCPP_HIDE_FROM_ABI iterator __node_handle_insert_multi(_NodeHandle&& __nh);
+  template <class _NodeHandle>
+  _LIBCPP_HIDE_FROM_ABI iterator __node_handle_insert_multi(const_iterator __hint, _NodeHandle&& __nh);
+  template <class _Table>
+  _LIBCPP_HIDE_FROM_ABI void __node_handle_merge_multi(_Table& __source);
+
+  template <class _NodeHandle>
+  _LIBCPP_HIDE_FROM_ABI _NodeHandle __node_handle_extract(key_type const& __key);
+  template <class _NodeHandle>
+  _LIBCPP_HIDE_FROM_ABI _NodeHandle __node_handle_extract(const_iterator __it);
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI void __rehash_unique(size_type __n) { __rehash<true>(__n); }
+  _LIBCPP_HIDE_FROM_ABI void __rehash_multi(size_type __n) { __rehash<false>(__n); }
+  _LIBCPP_HIDE_FROM_ABI void __reserve_unique(size_type __n) {
+    __rehash_unique(static_cast<size_type>(std::ceil(__n / max_load_factor())));
+  }
+  _LIBCPP_HIDE_FROM_ABI void __reserve_multi(size_type __n) {
+    __rehash_multi(static_cast<size_type>(std::ceil(__n / max_load_factor())));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __bucket_list_.get_deleter().size(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT;
+
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI size_type bucket(const _Key& __k) const {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        bucket_count() > 0, "unordered container::bucket(key) called when bucket_count() == 0");
+    return std::__constrain_hash(hash_function()(__k), bucket_count());
+  }
+
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI iterator find(const _Key& __x);
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _Key& __x) const;
+
+  typedef __hash_node_destructor<__node_allocator> _Dp;
+  typedef unique_ptr<__node, _Dp> __node_holder;
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p);
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last);
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI size_type __erase_unique(const _Key& __k);
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI size_type __erase_multi(const _Key& __k);
+  _LIBCPP_HIDE_FROM_ABI __node_holder remove(const_iterator __p) _NOEXCEPT;
+
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI size_type __count_unique(const _Key& __k) const;
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI size_type __count_multi(const _Key& __k) const;
+
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> __equal_range_unique(const _Key& __k);
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> __equal_range_unique(const _Key& __k) const;
+
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> __equal_range_multi(const _Key& __k);
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> __equal_range_multi(const _Key& __k) const;
+
+  _LIBCPP_HIDE_FROM_ABI void swap(__hash_table& __u)
+#if _LIBCPP_STD_VER <= 11
+      _NOEXCEPT_(__is_nothrow_swappable_v<hasher>&& __is_nothrow_swappable_v<key_equal> &&
+                 (!allocator_traits<__pointer_allocator>::propagate_on_container_swap::value ||
+                  __is_nothrow_swappable_v<__pointer_allocator>) &&
+                 (!__node_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__node_allocator>));
+#else
+      _NOEXCEPT_(__is_nothrow_swappable_v<hasher>&& __is_nothrow_swappable_v<key_equal>);
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT { return max_size(); }
+  _LIBCPP_HIDE_FROM_ABI size_type bucket_size(size_type __n) const;
+  _LIBCPP_HIDE_FROM_ABI float load_factor() const _NOEXCEPT {
+    size_type __bc = bucket_count();
+    return __bc != 0 ? (float)size() / __bc : 0.f;
+  }
+  _LIBCPP_HIDE_FROM_ABI void max_load_factor(float __mlf) _NOEXCEPT {
+    // While passing a non-positive load factor is undefined behavior, in practice the result will be benign (the
+    // call will be equivalent to `max_load_factor(load_factor())`, which is also the case for passing a valid value
+    // less than the current `load_factor`).
+    _LIBCPP_ASSERT_PEDANTIC(__mlf > 0, "unordered container::max_load_factor(lf) called with lf <= 0");
+    max_load_factor() = std::max(__mlf, load_factor());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI local_iterator begin(size_type __n) {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __n < bucket_count(), "unordered container::begin(n) called with n >= bucket_count()");
+    return local_iterator(__bucket_list_[__n], __n, bucket_count());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI local_iterator end(size_type __n) {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __n < bucket_count(), "unordered container::end(n) called with n >= bucket_count()");
+    return local_iterator(nullptr, __n, bucket_count());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator cbegin(size_type __n) const {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __n < bucket_count(), "unordered container::cbegin(n) called with n >= bucket_count()");
+    return const_local_iterator(__bucket_list_[__n], __n, bucket_count());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator cend(size_type __n) const {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __n < bucket_count(), "unordered container::cend(n) called with n >= bucket_count()");
+    return const_local_iterator(nullptr, __n, bucket_count());
+  }
+
+private:
+  template <bool _UniqueKeys>
+  _LIBCPP_HIDE_FROM_ABI void __rehash(size_type __n);
+  template <bool _UniqueKeys>
+  _LIBCPP_HIDE_FROM_ABI void __do_rehash(size_type __n);
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI __node_holder __construct_node(_Args&&... __args);
+
+  template <class _First, class... _Rest>
+  _LIBCPP_HIDE_FROM_ABI __node_holder __construct_node_hash(size_t __hash, _First&& __f, _Rest&&... __rest);
+
+  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __hash_table& __u) {
+    __copy_assign_alloc(__u, integral_constant<bool, __node_traits::propagate_on_container_copy_assignment::value>());
+  }
+  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __hash_table& __u, true_type);
+  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __hash_table&, false_type) {}
+
+  _LIBCPP_HIDE_FROM_ABI void __move_assign(__hash_table& __u, false_type);
+  _LIBCPP_HIDE_FROM_ABI void __move_assign(__hash_table& __u, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<__node_allocator>::value&& is_nothrow_move_assignable<hasher>::value&&
+                     is_nothrow_move_assignable<key_equal>::value);
+  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__hash_table& __u) _NOEXCEPT_(
+      !__node_traits::propagate_on_container_move_assignment::value ||
+      (is_nothrow_move_assignable<__pointer_allocator>::value && is_nothrow_move_assignable<__node_allocator>::value)) {
+    __move_assign_alloc(__u, integral_constant<bool, __node_traits::propagate_on_container_move_assignment::value>());
+  }
+  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__hash_table& __u, true_type) _NOEXCEPT_(
+      is_nothrow_move_assignable<__pointer_allocator>::value&& is_nothrow_move_assignable<__node_allocator>::value) {
+    __bucket_list_.get_deleter().__alloc() = std::move(__u.__bucket_list_.get_deleter().__alloc());
+    __node_alloc()                         = std::move(__u.__node_alloc());
+  }
+  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__hash_table&, false_type) _NOEXCEPT {}
+
+  _LIBCPP_HIDE_FROM_ABI void __deallocate_node(__next_pointer __np) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI __next_pointer __detach() _NOEXCEPT;
+
+  template <class, class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_map;
+  template <class, class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_multimap;
+};
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+inline __hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table() _NOEXCEPT_(
+    is_nothrow_default_constructible<__bucket_list>::value&& is_nothrow_default_constructible<__first_node>::value&&
+        is_nothrow_default_constructible<__node_allocator>::value&& is_nothrow_default_constructible<hasher>::value&&
+            is_nothrow_default_constructible<key_equal>::value)
+    : __p2_(0, __default_init_tag()), __p3_(1.0f, __default_init_tag()) {}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+inline __hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const hasher& __hf, const key_equal& __eql)
+    : __bucket_list_(nullptr, __bucket_list_deleter()), __p1_(), __p2_(0, __hf), __p3_(1.0f, __eql) {}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(
+    const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
+    : __bucket_list_(nullptr, __bucket_list_deleter(__pointer_allocator(__a), 0)),
+      __p1_(__default_init_tag(), __node_allocator(__a)),
+      __p2_(0, __hf),
+      __p3_(1.0f, __eql) {}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const allocator_type& __a)
+    : __bucket_list_(nullptr, __bucket_list_deleter(__pointer_allocator(__a), 0)),
+      __p1_(__default_init_tag(), __node_allocator(__a)),
+      __p2_(0, __default_init_tag()),
+      __p3_(1.0f, __default_init_tag()) {}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __u)
+    : __bucket_list_(nullptr,
+                     __bucket_list_deleter(allocator_traits<__pointer_allocator>::select_on_container_copy_construction(
+                                               __u.__bucket_list_.get_deleter().__alloc()),
+                                           0)),
+      __p1_(__default_init_tag(),
+            allocator_traits<__node_allocator>::select_on_container_copy_construction(__u.__node_alloc())),
+      __p2_(0, __u.hash_function()),
+      __p3_(__u.__p3_) {}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __u, const allocator_type& __a)
+    : __bucket_list_(nullptr, __bucket_list_deleter(__pointer_allocator(__a), 0)),
+      __p1_(__default_init_tag(), __node_allocator(__a)),
+      __p2_(0, __u.hash_function()),
+      __p3_(__u.__p3_) {}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(__hash_table&& __u) _NOEXCEPT_(
+    is_nothrow_move_constructible<__bucket_list>::value&& is_nothrow_move_constructible<__first_node>::value&&
+        is_nothrow_move_constructible<__node_allocator>::value&& is_nothrow_move_constructible<hasher>::value&&
+            is_nothrow_move_constructible<key_equal>::value)
+    : __bucket_list_(std::move(__u.__bucket_list_)),
+      __p1_(std::move(__u.__p1_)),
+      __p2_(std::move(__u.__p2_)),
+      __p3_(std::move(__u.__p3_)) {
+  if (size() > 0) {
+    __bucket_list_[std::__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = __p1_.first().__ptr();
+    __u.__p1_.first().__next_                                                              = nullptr;
+    __u.size()                                                                             = 0;
+  }
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(__hash_table&& __u, const allocator_type& __a)
+    : __bucket_list_(nullptr, __bucket_list_deleter(__pointer_allocator(__a), 0)),
+      __p1_(__default_init_tag(), __node_allocator(__a)),
+      __p2_(0, std::move(__u.hash_function())),
+      __p3_(std::move(__u.__p3_)) {
+  if (__a == allocator_type(__u.__node_alloc())) {
+    __bucket_list_.reset(__u.__bucket_list_.release());
+    __bucket_list_.get_deleter().size()     = __u.__bucket_list_.get_deleter().size();
+    __u.__bucket_list_.get_deleter().size() = 0;
+    if (__u.size() > 0) {
+      __p1_.first().__next_     = __u.__p1_.first().__next_;
+      __u.__p1_.first().__next_ = nullptr;
+      __bucket_list_[std::__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = __p1_.first().__ptr();
+      size()                                                                                 = __u.size();
+      __u.size()                                                                             = 0;
+    }
+  }
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::~__hash_table() {
+#if defined(_LIBCPP_CXX03_LANG)
+  static_assert(is_copy_constructible<key_equal>::value, "Predicate must be copy-constructible.");
+  static_assert(is_copy_constructible<hasher>::value, "Hasher must be copy-constructible.");
+#endif
+
+  __deallocate_node(__p1_.first().__next_);
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__copy_assign_alloc(const __hash_table& __u, true_type) {
+  if (__node_alloc() != __u.__node_alloc()) {
+    clear();
+    __bucket_list_.reset();
+    __bucket_list_.get_deleter().size() = 0;
+  }
+  __bucket_list_.get_deleter().__alloc() = __u.__bucket_list_.get_deleter().__alloc();
+  __node_alloc()                         = __u.__node_alloc();
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+__hash_table<_Tp, _Hash, _Equal, _Alloc>& __hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(const __hash_table& __u) {
+  if (this != std::addressof(__u)) {
+    __copy_assign_alloc(__u);
+    hash_function()   = __u.hash_function();
+    key_eq()          = __u.key_eq();
+    max_load_factor() = __u.max_load_factor();
+    __assign_multi(__u.begin(), __u.end());
+  }
+  return *this;
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__deallocate_node(__next_pointer __np) _NOEXCEPT {
+  __node_allocator& __na = __node_alloc();
+  while (__np != nullptr) {
+    __next_pointer __next    = __np->__next_;
+    __node_pointer __real_np = __np->__upcast();
+    __node_traits::destroy(__na, _NodeTypes::__get_ptr(__real_np->__get_value()));
+    std::__destroy_at(std::addressof(*__real_np));
+    __node_traits::deallocate(__na, __real_np, 1);
+    __np = __next;
+  }
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__next_pointer
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__detach() _NOEXCEPT {
+  size_type __bc = bucket_count();
+  for (size_type __i = 0; __i < __bc; ++__i)
+    __bucket_list_[__i] = nullptr;
+  size()                 = 0;
+  __next_pointer __cache = __p1_.first().__next_;
+  __p1_.first().__next_  = nullptr;
+  return __cache;
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__move_assign(__hash_table& __u, true_type)
+    _NOEXCEPT_(is_nothrow_move_assignable<__node_allocator>::value&& is_nothrow_move_assignable<hasher>::value&&
+                   is_nothrow_move_assignable<key_equal>::value) {
+  clear();
+  __bucket_list_.reset(__u.__bucket_list_.release());
+  __bucket_list_.get_deleter().size()     = __u.__bucket_list_.get_deleter().size();
+  __u.__bucket_list_.get_deleter().size() = 0;
+  __move_assign_alloc(__u);
+  size()                = __u.size();
+  hash_function()       = std::move(__u.hash_function());
+  max_load_factor()     = __u.max_load_factor();
+  key_eq()              = std::move(__u.key_eq());
+  __p1_.first().__next_ = __u.__p1_.first().__next_;
+  if (size() > 0) {
+    __bucket_list_[std::__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = __p1_.first().__ptr();
+    __u.__p1_.first().__next_                                                              = nullptr;
+    __u.size()                                                                             = 0;
+  }
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__move_assign(__hash_table& __u, false_type) {
+  if (__node_alloc() == __u.__node_alloc())
+    __move_assign(__u, true_type());
+  else {
+    hash_function()   = std::move(__u.hash_function());
+    key_eq()          = std::move(__u.key_eq());
+    max_load_factor() = __u.max_load_factor();
+    if (bucket_count() != 0) {
+      __next_pointer __cache = __detach();
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+        const_iterator __i = __u.begin();
+        while (__cache != nullptr && __u.size() != 0) {
+          __cache->__upcast()->__get_value() = std::move(__u.remove(__i++)->__get_value());
+          __next_pointer __next              = __cache->__next_;
+          __node_insert_multi(__cache->__upcast());
+          __cache = __next;
+        }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      } catch (...) {
+        __deallocate_node(__cache);
+        throw;
+      }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      __deallocate_node(__cache);
+    }
+    const_iterator __i = __u.begin();
+    while (__u.size() != 0) {
+      __node_holder __h = __construct_node(_NodeTypes::__move(__u.remove(__i++)->__get_value()));
+      __node_insert_multi(__h.get());
+      __h.release();
+    }
+  }
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+inline __hash_table<_Tp, _Hash, _Equal, _Alloc>&
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(__hash_table&& __u) _NOEXCEPT_(
+    __node_traits::propagate_on_container_move_assignment::value&& is_nothrow_move_assignable<__node_allocator>::value&&
+        is_nothrow_move_assignable<hasher>::value&& is_nothrow_move_assignable<key_equal>::value) {
+  __move_assign(__u, integral_constant<bool, __node_traits::propagate_on_container_move_assignment::value>());
+  return *this;
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _InputIterator>
+void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__assign_unique(_InputIterator __first, _InputIterator __last) {
+  typedef iterator_traits<_InputIterator> _ITraits;
+  typedef typename _ITraits::value_type _ItValueType;
+  static_assert(is_same<_ItValueType, __container_value_type>::value,
+                "__assign_unique may only be called with the containers value type");
+
+  if (bucket_count() != 0) {
+    __next_pointer __cache = __detach();
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      for (; __cache != nullptr && __first != __last; ++__first) {
+        __cache->__upcast()->__get_value() = *__first;
+        __next_pointer __next              = __cache->__next_;
+        __node_insert_unique(__cache->__upcast());
+        __cache = __next;
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __deallocate_node(__cache);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __deallocate_node(__cache);
+  }
+  for (; __first != __last; ++__first)
+    __insert_unique(*__first);
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _InputIterator>
+void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__assign_multi(_InputIterator __first, _InputIterator __last) {
+  typedef iterator_traits<_InputIterator> _ITraits;
+  typedef typename _ITraits::value_type _ItValueType;
+  static_assert(
+      (is_same<_ItValueType, __container_value_type>::value || is_same<_ItValueType, __node_value_type>::value),
+      "__assign_multi may only be called with the containers value type"
+      " or the nodes value type");
+  if (bucket_count() != 0) {
+    __next_pointer __cache = __detach();
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      for (; __cache != nullptr && __first != __last; ++__first) {
+        __cache->__upcast()->__get_value() = *__first;
+        __next_pointer __next              = __cache->__next_;
+        __node_insert_multi(__cache->__upcast());
+        __cache = __next;
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __deallocate_node(__cache);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __deallocate_node(__cache);
+  }
+  for (; __first != __last; ++__first)
+    __insert_multi(_NodeTypes::__get_value(*__first));
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+inline typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::begin() _NOEXCEPT {
+  return iterator(__p1_.first().__next_);
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+inline typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::end() _NOEXCEPT {
+  return iterator(nullptr);
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+inline typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::begin() const _NOEXCEPT {
+  return const_iterator(__p1_.first().__next_);
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+inline typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::end() const _NOEXCEPT {
+  return const_iterator(nullptr);
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+void __hash_table<_Tp, _Hash, _Equal, _Alloc>::clear() _NOEXCEPT {
+  if (size() > 0) {
+    __deallocate_node(__p1_.first().__next_);
+    __p1_.first().__next_ = nullptr;
+    size_type __bc        = bucket_count();
+    for (size_type __i = 0; __i < __bc; ++__i)
+      __bucket_list_[__i] = nullptr;
+    size() = 0;
+  }
+}
+
+// Prepare the container for an insertion of the value __value with the hash
+// __hash. This does a lookup into the container to see if __value is already
+// present, and performs a rehash if necessary. Returns a pointer to the
+// existing element if it exists, otherwise nullptr.
+//
+// Note that this function does forward exceptions if key_eq() throws, and never
+// mutates __value or actually inserts into the map.
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+_LIBCPP_HIDE_FROM_ABI typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__next_pointer
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique_prepare(size_t __hash, value_type& __value) {
+  size_type __bc = bucket_count();
+
+  if (__bc != 0) {
+    size_t __chash         = std::__constrain_hash(__hash, __bc);
+    __next_pointer __ndptr = __bucket_list_[__chash];
+    if (__ndptr != nullptr) {
+      for (__ndptr = __ndptr->__next_;
+           __ndptr != nullptr &&
+           (__ndptr->__hash() == __hash || std::__constrain_hash(__ndptr->__hash(), __bc) == __chash);
+           __ndptr = __ndptr->__next_) {
+        if ((__ndptr->__hash() == __hash) && key_eq()(__ndptr->__upcast()->__get_value(), __value))
+          return __ndptr;
+      }
+    }
+  }
+  if (size() + 1 > __bc * max_load_factor() || __bc == 0) {
+    __rehash_unique(std::max<size_type>(
+        2 * __bc + !std::__is_hash_power2(__bc), size_type(std::ceil(float(size() + 1) / max_load_factor()))));
+  }
+  return nullptr;
+}
+
+// Insert the node __nd into the container by pushing it into the right bucket,
+// and updating size(). Assumes that __nd->__hash is up-to-date, and that
+// rehashing has already occurred and that no element with the same key exists
+// in the map.
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+_LIBCPP_HIDE_FROM_ABI void
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique_perform(__node_pointer __nd) _NOEXCEPT {
+  size_type __bc = bucket_count();
+  size_t __chash = std::__constrain_hash(__nd->__hash(), __bc);
+  // insert_after __bucket_list_[__chash], or __first_node if bucket is null
+  __next_pointer __pn = __bucket_list_[__chash];
+  if (__pn == nullptr) {
+    __pn          = __p1_.first().__ptr();
+    __nd->__next_ = __pn->__next_;
+    __pn->__next_ = __nd->__ptr();
+    // fix up __bucket_list_
+    __bucket_list_[__chash] = __pn;
+    if (__nd->__next_ != nullptr)
+      __bucket_list_[std::__constrain_hash(__nd->__next_->__hash(), __bc)] = __nd->__ptr();
+  } else {
+    __nd->__next_ = __pn->__next_;
+    __pn->__next_ = __nd->__ptr();
+  }
+  ++size();
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+pair<typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator, bool>
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique(__node_pointer __nd) {
+  __nd->__hash_                  = hash_function()(__nd->__get_value());
+  __next_pointer __existing_node = __node_insert_unique_prepare(__nd->__hash(), __nd->__get_value());
+
+  // Insert the node, unless it already exists in the container.
+  bool __inserted = false;
+  if (__existing_node == nullptr) {
+    __node_insert_unique_perform(__nd);
+    __existing_node = __nd->__ptr();
+    __inserted      = true;
+  }
+  return pair<iterator, bool>(iterator(__existing_node), __inserted);
+}
+
+// Prepare the container for an insertion of the value __cp_val with the hash
+// __cp_hash. This does a lookup into the container to see if __cp_value is
+// already present, and performs a rehash if necessary. Returns a pointer to the
+// last occurrence of __cp_val in the map.
+//
+// Note that this function does forward exceptions if key_eq() throws, and never
+// mutates __value or actually inserts into the map.
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__next_pointer
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi_prepare(size_t __cp_hash, value_type& __cp_val) {
+  size_type __bc = bucket_count();
+  if (size() + 1 > __bc * max_load_factor() || __bc == 0) {
+    __rehash_multi(std::max<size_type>(
+        2 * __bc + !std::__is_hash_power2(__bc), size_type(std::ceil(float(size() + 1) / max_load_factor()))));
+    __bc = bucket_count();
+  }
+  size_t __chash      = std::__constrain_hash(__cp_hash, __bc);
+  __next_pointer __pn = __bucket_list_[__chash];
+  if (__pn != nullptr) {
+    for (bool __found = false;
+         __pn->__next_ != nullptr && std::__constrain_hash(__pn->__next_->__hash(), __bc) == __chash;
+         __pn = __pn->__next_) {
+      //      __found    key_eq()     action
+      //      false       false       loop
+      //      true        true        loop
+      //      false       true        set __found to true
+      //      true        false       break
+      if (__found !=
+          (__pn->__next_->__hash() == __cp_hash && key_eq()(__pn->__next_->__upcast()->__get_value(), __cp_val))) {
+        if (!__found)
+          __found = true;
+        else
+          break;
+      }
+    }
+  }
+  return __pn;
+}
+
+// Insert the node __cp into the container after __pn (which is the last node in
+// the bucket that compares equal to __cp). Rehashing, and checking for
+// uniqueness has already been performed (in __node_insert_multi_prepare), so
+// all we need to do is update the bucket and size(). Assumes that __cp->__hash
+// is up-to-date.
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi_perform(
+    __node_pointer __cp, __next_pointer __pn) _NOEXCEPT {
+  size_type __bc = bucket_count();
+  size_t __chash = std::__constrain_hash(__cp->__hash_, __bc);
+  if (__pn == nullptr) {
+    __pn          = __p1_.first().__ptr();
+    __cp->__next_ = __pn->__next_;
+    __pn->__next_ = __cp->__ptr();
+    // fix up __bucket_list_
+    __bucket_list_[__chash] = __pn;
+    if (__cp->__next_ != nullptr)
+      __bucket_list_[std::__constrain_hash(__cp->__next_->__hash(), __bc)] = __cp->__ptr();
+  } else {
+    __cp->__next_ = __pn->__next_;
+    __pn->__next_ = __cp->__ptr();
+    if (__cp->__next_ != nullptr) {
+      size_t __nhash = std::__constrain_hash(__cp->__next_->__hash(), __bc);
+      if (__nhash != __chash)
+        __bucket_list_[__nhash] = __cp->__ptr();
+    }
+  }
+  ++size();
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi(__node_pointer __cp) {
+  __cp->__hash_       = hash_function()(__cp->__get_value());
+  __next_pointer __pn = __node_insert_multi_prepare(__cp->__hash(), __cp->__get_value());
+  __node_insert_multi_perform(__cp, __pn);
+
+  return iterator(__cp->__ptr());
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi(const_iterator __p, __node_pointer __cp) {
+  if (__p != end() && key_eq()(*__p, __cp->__get_value())) {
+    __next_pointer __np = __p.__node_;
+    __cp->__hash_       = __np->__hash();
+    size_type __bc      = bucket_count();
+    if (size() + 1 > __bc * max_load_factor() || __bc == 0) {
+      __rehash_multi(std::max<size_type>(
+          2 * __bc + !std::__is_hash_power2(__bc), size_type(std::ceil(float(size() + 1) / max_load_factor()))));
+      __bc = bucket_count();
+    }
+    size_t __chash      = std::__constrain_hash(__cp->__hash_, __bc);
+    __next_pointer __pp = __bucket_list_[__chash];
+    while (__pp->__next_ != __np)
+      __pp = __pp->__next_;
+    __cp->__next_ = __np;
+    __pp->__next_ = static_cast<__next_pointer>(__cp);
+    ++size();
+    return iterator(static_cast<__next_pointer>(__cp));
+  }
+  return __node_insert_multi(__cp);
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _Key, class... _Args>
+pair<typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator, bool>
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_unique_key_args(_Key const& __k, _Args&&... __args) {
+  size_t __hash   = hash_function()(__k);
+  size_type __bc  = bucket_count();
+  bool __inserted = false;
+  __next_pointer __nd;
+  size_t __chash;
+  if (__bc != 0) {
+    __chash = std::__constrain_hash(__hash, __bc);
+    __nd    = __bucket_list_[__chash];
+    if (__nd != nullptr) {
+      for (__nd = __nd->__next_;
+           __nd != nullptr && (__nd->__hash() == __hash || std::__constrain_hash(__nd->__hash(), __bc) == __chash);
+           __nd = __nd->__next_) {
+        if ((__nd->__hash() == __hash) && key_eq()(__nd->__upcast()->__get_value(), __k))
+          goto __done;
+      }
+    }
+  }
+  {
+    __node_holder __h = __construct_node_hash(__hash, std::forward<_Args>(__args)...);
+    if (size() + 1 > __bc * max_load_factor() || __bc == 0) {
+      __rehash_unique(std::max<size_type>(
+          2 * __bc + !std::__is_hash_power2(__bc), size_type(std::ceil(float(size() + 1) / max_load_factor()))));
+      __bc    = bucket_count();
+      __chash = std::__constrain_hash(__hash, __bc);
+    }
+    // insert_after __bucket_list_[__chash], or __first_node if bucket is null
+    __next_pointer __pn = __bucket_list_[__chash];
+    if (__pn == nullptr) {
+      __pn          = __p1_.first().__ptr();
+      __h->__next_  = __pn->__next_;
+      __pn->__next_ = __h.get()->__ptr();
+      // fix up __bucket_list_
+      __bucket_list_[__chash] = __pn;
+      if (__h->__next_ != nullptr)
+        __bucket_list_[std::__constrain_hash(__h->__next_->__hash(), __bc)] = __h.get()->__ptr();
+    } else {
+      __h->__next_  = __pn->__next_;
+      __pn->__next_ = static_cast<__next_pointer>(__h.get());
+    }
+    __nd = static_cast<__next_pointer>(__h.release());
+    // increment size
+    ++size();
+    __inserted = true;
+  }
+__done:
+  return pair<iterator, bool>(iterator(__nd), __inserted);
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class... _Args>
+pair<typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator, bool>
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_unique_impl(_Args&&... __args) {
+  __node_holder __h        = __construct_node(std::forward<_Args>(__args)...);
+  pair<iterator, bool> __r = __node_insert_unique(__h.get());
+  if (__r.second)
+    __h.release();
+  return __r;
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class... _Args>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_multi(_Args&&... __args) {
+  __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
+  iterator __r      = __node_insert_multi(__h.get());
+  __h.release();
+  return __r;
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class... _Args>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_hint_multi(const_iterator __p, _Args&&... __args) {
+  __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
+  iterator __r      = __node_insert_multi(__p, __h.get());
+  __h.release();
+  return __r;
+}
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _NodeHandle, class _InsertReturnType>
+_LIBCPP_HIDE_FROM_ABI _InsertReturnType
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_insert_unique(_NodeHandle&& __nh) {
+  if (__nh.empty())
+    return _InsertReturnType{end(), false, _NodeHandle()};
+  pair<iterator, bool> __result = __node_insert_unique(__nh.__ptr_);
+  if (__result.second)
+    __nh.__release_ptr();
+  return _InsertReturnType{__result.first, __result.second, std::move(__nh)};
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _NodeHandle>
+_LIBCPP_HIDE_FROM_ABI typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_insert_unique(const_iterator, _NodeHandle&& __nh) {
+  if (__nh.empty())
+    return end();
+  pair<iterator, bool> __result = __node_insert_unique(__nh.__ptr_);
+  if (__result.second)
+    __nh.__release_ptr();
+  return __result.first;
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _NodeHandle>
+_LIBCPP_HIDE_FROM_ABI _NodeHandle
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_extract(key_type const& __key) {
+  iterator __i = find(__key);
+  if (__i == end())
+    return _NodeHandle();
+  return __node_handle_extract<_NodeHandle>(__i);
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _NodeHandle>
+_LIBCPP_HIDE_FROM_ABI _NodeHandle __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_extract(const_iterator __p) {
+  allocator_type __alloc(__node_alloc());
+  return _NodeHandle(remove(__p).release(), __alloc);
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _Table>
+_LIBCPP_HIDE_FROM_ABI void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_merge_unique(_Table& __source) {
+  static_assert(is_same<__node, typename _Table::__node>::value, "");
+
+  for (typename _Table::iterator __it = __source.begin(); __it != __source.end();) {
+    __node_pointer __src_ptr       = __it.__node_->__upcast();
+    size_t __hash                  = hash_function()(__src_ptr->__get_value());
+    __next_pointer __existing_node = __node_insert_unique_prepare(__hash, __src_ptr->__get_value());
+    auto __prev_iter               = __it++;
+    if (__existing_node == nullptr) {
+      (void)__source.remove(__prev_iter).release();
+      __src_ptr->__hash_ = __hash;
+      __node_insert_unique_perform(__src_ptr);
+    }
+  }
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _NodeHandle>
+_LIBCPP_HIDE_FROM_ABI typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_insert_multi(_NodeHandle&& __nh) {
+  if (__nh.empty())
+    return end();
+  iterator __result = __node_insert_multi(__nh.__ptr_);
+  __nh.__release_ptr();
+  return __result;
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _NodeHandle>
+_LIBCPP_HIDE_FROM_ABI typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_insert_multi(const_iterator __hint, _NodeHandle&& __nh) {
+  if (__nh.empty())
+    return end();
+  iterator __result = __node_insert_multi(__hint, __nh.__ptr_);
+  __nh.__release_ptr();
+  return __result;
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _Table>
+_LIBCPP_HIDE_FROM_ABI void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_merge_multi(_Table& __source) {
+  static_assert(is_same<typename _Table::__node, __node>::value, "");
+
+  for (typename _Table::iterator __it = __source.begin(); __it != __source.end();) {
+    __node_pointer __src_ptr = __it.__node_->__upcast();
+    size_t __src_hash        = hash_function()(__src_ptr->__get_value());
+    __next_pointer __pn      = __node_insert_multi_prepare(__src_hash, __src_ptr->__get_value());
+    (void)__source.remove(__it++).release();
+    __src_ptr->__hash_ = __src_hash;
+    __node_insert_multi_perform(__src_ptr, __pn);
+  }
+}
+#endif // _LIBCPP_STD_VER >= 17
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <bool _UniqueKeys>
+void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__rehash(size_type __n) _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK {
+  if (__n == 1)
+    __n = 2;
+  else if (__n & (__n - 1))
+    __n = std::__next_prime(__n);
+  size_type __bc = bucket_count();
+  if (__n > __bc)
+    __do_rehash<_UniqueKeys>(__n);
+  else if (__n < __bc) {
+    __n = std::max<size_type>(
+        __n,
+        std::__is_hash_power2(__bc) ? std::__next_hash_pow2(size_t(std::ceil(float(size()) / max_load_factor())))
+                                    : std::__next_prime(size_t(std::ceil(float(size()) / max_load_factor()))));
+    if (__n < __bc)
+      __do_rehash<_UniqueKeys>(__n);
+  }
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <bool _UniqueKeys>
+void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__do_rehash(size_type __nbc) {
+  __pointer_allocator& __npa = __bucket_list_.get_deleter().__alloc();
+  __bucket_list_.reset(__nbc > 0 ? __pointer_alloc_traits::allocate(__npa, __nbc) : nullptr);
+  __bucket_list_.get_deleter().size() = __nbc;
+  if (__nbc > 0) {
+    for (size_type __i = 0; __i < __nbc; ++__i)
+      __bucket_list_[__i] = nullptr;
+    __next_pointer __pp = __p1_.first().__ptr();
+    __next_pointer __cp = __pp->__next_;
+    if (__cp != nullptr) {
+      size_type __chash       = std::__constrain_hash(__cp->__hash(), __nbc);
+      __bucket_list_[__chash] = __pp;
+      size_type __phash       = __chash;
+      for (__pp = __cp, void(), __cp = __cp->__next_; __cp != nullptr; __cp = __pp->__next_) {
+        __chash = std::__constrain_hash(__cp->__hash(), __nbc);
+        if (__chash == __phash)
+          __pp = __cp;
+        else {
+          if (__bucket_list_[__chash] == nullptr) {
+            __bucket_list_[__chash] = __pp;
+            __pp                    = __cp;
+            __phash                 = __chash;
+          } else {
+            __next_pointer __np = __cp;
+            if _LIBCPP_CONSTEXPR_SINCE_CXX17 (!_UniqueKeys) {
+              for (; __np->__next_ != nullptr &&
+                     key_eq()(__cp->__upcast()->__get_value(), __np->__next_->__upcast()->__get_value());
+                   __np = __np->__next_)
+                ;
+            }
+            __pp->__next_                    = __np->__next_;
+            __np->__next_                    = __bucket_list_[__chash]->__next_;
+            __bucket_list_[__chash]->__next_ = __cp;
+          }
+        }
+      }
+    }
+  }
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _Key>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::find(const _Key& __k) {
+  size_t __hash  = hash_function()(__k);
+  size_type __bc = bucket_count();
+  if (__bc != 0) {
+    size_t __chash      = std::__constrain_hash(__hash, __bc);
+    __next_pointer __nd = __bucket_list_[__chash];
+    if (__nd != nullptr) {
+      for (__nd = __nd->__next_;
+           __nd != nullptr && (__nd->__hash() == __hash || std::__constrain_hash(__nd->__hash(), __bc) == __chash);
+           __nd = __nd->__next_) {
+        if ((__nd->__hash() == __hash) && key_eq()(__nd->__upcast()->__get_value(), __k))
+          return iterator(__nd);
+      }
+    }
+  }
+  return end();
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _Key>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::find(const _Key& __k) const {
+  size_t __hash  = hash_function()(__k);
+  size_type __bc = bucket_count();
+  if (__bc != 0) {
+    size_t __chash      = std::__constrain_hash(__hash, __bc);
+    __next_pointer __nd = __bucket_list_[__chash];
+    if (__nd != nullptr) {
+      for (__nd = __nd->__next_;
+           __nd != nullptr && (__hash == __nd->__hash() || std::__constrain_hash(__nd->__hash(), __bc) == __chash);
+           __nd = __nd->__next_) {
+        if ((__nd->__hash() == __hash) && key_eq()(__nd->__upcast()->__get_value(), __k))
+          return const_iterator(__nd);
+      }
+    }
+  }
+  return end();
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class... _Args>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_holder
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__construct_node(_Args&&... __args) {
+  static_assert(!__is_hash_value_type<_Args...>::value, "Construct cannot be called with a hash value type");
+  __node_allocator& __na = __node_alloc();
+  __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
+
+  // Begin the lifetime of the node itself. Note that this doesn't begin the lifetime of the value
+  // held inside the node, since we need to use the allocator's construct() method for that.
+  //
+  // We don't use the allocator's construct() method to construct the node itself since the
+  // Cpp17FooInsertable named requirements don't require the allocator's construct() method
+  // to work on anything other than the value_type.
+  std::__construct_at(std::addressof(*__h), /* next = */ nullptr, /* hash = */ 0);
+
+  // Now construct the value_type using the allocator's construct() method.
+  __node_traits::construct(__na, _NodeTypes::__get_ptr(__h->__get_value()), std::forward<_Args>(__args)...);
+  __h.get_deleter().__value_constructed = true;
+
+  __h->__hash_ = hash_function()(__h->__get_value());
+  return __h;
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _First, class... _Rest>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_holder
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__construct_node_hash(size_t __hash, _First&& __f, _Rest&&... __rest) {
+  static_assert(!__is_hash_value_type<_First, _Rest...>::value, "Construct cannot be called with a hash value type");
+  __node_allocator& __na = __node_alloc();
+  __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
+  std::__construct_at(std::addressof(*__h), /* next = */ nullptr, /* hash = */ __hash);
+  __node_traits::construct(
+      __na, _NodeTypes::__get_ptr(__h->__get_value()), std::forward<_First>(__f), std::forward<_Rest>(__rest)...);
+  __h.get_deleter().__value_constructed = true;
+  return __h;
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::erase(const_iterator __p) {
+  __next_pointer __np = __p.__node_;
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+      __p != end(), "unordered container::erase(iterator) called with a non-dereferenceable iterator");
+  iterator __r(__np);
+  ++__r;
+  remove(__p);
+  return __r;
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::erase(const_iterator __first, const_iterator __last) {
+  for (const_iterator __p = __first; __first != __last; __p = __first) {
+    ++__first;
+    erase(__p);
+  }
+  __next_pointer __np = __last.__node_;
+  return iterator(__np);
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _Key>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::size_type
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__erase_unique(const _Key& __k) {
+  iterator __i = find(__k);
+  if (__i == end())
+    return 0;
+  erase(__i);
+  return 1;
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _Key>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::size_type
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__erase_multi(const _Key& __k) {
+  size_type __r = 0;
+  iterator __i  = find(__k);
+  if (__i != end()) {
+    iterator __e = end();
+    do {
+      erase(__i++);
+      ++__r;
+    } while (__i != __e && key_eq()(*__i, __k));
+  }
+  return __r;
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_holder
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::remove(const_iterator __p) _NOEXCEPT {
+  // current node
+  __next_pointer __cn = __p.__node_;
+  size_type __bc      = bucket_count();
+  size_t __chash      = std::__constrain_hash(__cn->__hash(), __bc);
+  // find previous node
+  __next_pointer __pn = __bucket_list_[__chash];
+  for (; __pn->__next_ != __cn; __pn = __pn->__next_)
+    ;
+  // Fix up __bucket_list_
+  // if __pn is not in same bucket (before begin is not in same bucket) &&
+  //    if __cn->__next_ is not in same bucket (nullptr is not in same bucket)
+  if (__pn == __p1_.first().__ptr() || std::__constrain_hash(__pn->__hash(), __bc) != __chash) {
+    if (__cn->__next_ == nullptr || std::__constrain_hash(__cn->__next_->__hash(), __bc) != __chash)
+      __bucket_list_[__chash] = nullptr;
+  }
+  // if __cn->__next_ is not in same bucket (nullptr is in same bucket)
+  if (__cn->__next_ != nullptr) {
+    size_t __nhash = std::__constrain_hash(__cn->__next_->__hash(), __bc);
+    if (__nhash != __chash)
+      __bucket_list_[__nhash] = __pn;
+  }
+  // remove __cn
+  __pn->__next_ = __cn->__next_;
+  __cn->__next_ = nullptr;
+  --size();
+  return __node_holder(__cn->__upcast(), _Dp(__node_alloc(), true));
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _Key>
+inline typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::size_type
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__count_unique(const _Key& __k) const {
+  return static_cast<size_type>(find(__k) != end());
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _Key>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::size_type
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__count_multi(const _Key& __k) const {
+  size_type __r      = 0;
+  const_iterator __i = find(__k);
+  if (__i != end()) {
+    const_iterator __e = end();
+    do {
+      ++__i;
+      ++__r;
+    } while (__i != __e && key_eq()(*__i, __k));
+  }
+  return __r;
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _Key>
+pair<typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator,
+     typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator>
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__equal_range_unique(const _Key& __k) {
+  iterator __i = find(__k);
+  iterator __j = __i;
+  if (__i != end())
+    ++__j;
+  return pair<iterator, iterator>(__i, __j);
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _Key>
+pair<typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator,
+     typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator>
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__equal_range_unique(const _Key& __k) const {
+  const_iterator __i = find(__k);
+  const_iterator __j = __i;
+  if (__i != end())
+    ++__j;
+  return pair<const_iterator, const_iterator>(__i, __j);
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _Key>
+pair<typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator,
+     typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator>
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__equal_range_multi(const _Key& __k) {
+  iterator __i = find(__k);
+  iterator __j = __i;
+  if (__i != end()) {
+    iterator __e = end();
+    do {
+      ++__j;
+    } while (__j != __e && key_eq()(*__j, __k));
+  }
+  return pair<iterator, iterator>(__i, __j);
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <class _Key>
+pair<typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator,
+     typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator>
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__equal_range_multi(const _Key& __k) const {
+  const_iterator __i = find(__k);
+  const_iterator __j = __i;
+  if (__i != end()) {
+    const_iterator __e = end();
+    do {
+      ++__j;
+    } while (__j != __e && key_eq()(*__j, __k));
+  }
+  return pair<const_iterator, const_iterator>(__i, __j);
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+void __hash_table<_Tp, _Hash, _Equal, _Alloc>::swap(__hash_table& __u)
+#if _LIBCPP_STD_VER <= 11
+    _NOEXCEPT_(__is_nothrow_swappable_v<hasher>&& __is_nothrow_swappable_v<key_equal> &&
+               (!allocator_traits<__pointer_allocator>::propagate_on_container_swap::value ||
+                __is_nothrow_swappable_v<__pointer_allocator>) &&
+               (!__node_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__node_allocator>))
+#else
+    _NOEXCEPT_(__is_nothrow_swappable_v<hasher>&& __is_nothrow_swappable_v<key_equal>)
+#endif
+{
+  _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+      __node_traits::propagate_on_container_swap::value || this->__node_alloc() == __u.__node_alloc(),
+      "unordered container::swap: Either propagate_on_container_swap "
+      "must be true or the allocators must compare equal");
+  {
+    __node_pointer_pointer __npp = __bucket_list_.release();
+    __bucket_list_.reset(__u.__bucket_list_.release());
+    __u.__bucket_list_.reset(__npp);
+  }
+  std::swap(__bucket_list_.get_deleter().size(), __u.__bucket_list_.get_deleter().size());
+  std::__swap_allocator(__bucket_list_.get_deleter().__alloc(), __u.__bucket_list_.get_deleter().__alloc());
+  std::__swap_allocator(__node_alloc(), __u.__node_alloc());
+  std::swap(__p1_.first().__next_, __u.__p1_.first().__next_);
+  __p2_.swap(__u.__p2_);
+  __p3_.swap(__u.__p3_);
+  if (size() > 0)
+    __bucket_list_[std::__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = __p1_.first().__ptr();
+  if (__u.size() > 0)
+    __u.__bucket_list_[std::__constrain_hash(__u.__p1_.first().__next_->__hash(), __u.bucket_count())] =
+        __u.__p1_.first().__ptr();
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::size_type
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::bucket_size(size_type __n) const {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+      __n < bucket_count(), "unordered container::bucket_size(n) called with n >= bucket_count()");
+  __next_pointer __np = __bucket_list_[__n];
+  size_type __bc      = bucket_count();
+  size_type __r       = 0;
+  if (__np != nullptr) {
+    for (__np = __np->__next_; __np != nullptr && std::__constrain_hash(__np->__hash(), __bc) == __n;
+         __np = __np->__next_, (void)++__r)
+      ;
+  }
+  return __r;
+}
+
+template <class _Tp, class _Hash, class _Equal, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(__hash_table<_Tp, _Hash, _Equal, _Alloc>& __x, __hash_table<_Tp, _Hash, _Equal, _Alloc>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___HASH_TABLE
diff --git a/libcxx/include/__cxx03/__ios/fpos.h b/libcxx/include/__cxx03/__ios/fpos.h
new file mode 100644
index 000000000000..1af1e23ee50d
--- /dev/null
+++ b/libcxx/include/__cxx03/__ios/fpos.h
@@ -0,0 +1,76 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___IOS_FPOS_H
+#define _LIBCPP___IOS_FPOS_H
+
+#include <__config>
+#include <__fwd/ios.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _StateT>
+class _LIBCPP_TEMPLATE_VIS fpos {
+private:
+  _StateT __st_;
+  streamoff __off_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI fpos(streamoff __off = streamoff()) : __st_(), __off_(__off) {}
+
+  _LIBCPP_HIDE_FROM_ABI operator streamoff() const { return __off_; }
+
+  _LIBCPP_HIDE_FROM_ABI _StateT state() const { return __st_; }
+  _LIBCPP_HIDE_FROM_ABI void state(_StateT __st) { __st_ = __st; }
+
+  _LIBCPP_HIDE_FROM_ABI fpos& operator+=(streamoff __off) {
+    __off_ += __off;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI fpos operator+(streamoff __off) const {
+    fpos __t(*this);
+    __t += __off;
+    return __t;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI fpos& operator-=(streamoff __off) {
+    __off_ -= __off;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI fpos operator-(streamoff __off) const {
+    fpos __t(*this);
+    __t -= __off;
+    return __t;
+  }
+};
+
+template <class _StateT>
+inline _LIBCPP_HIDE_FROM_ABI streamoff operator-(const fpos<_StateT>& __x, const fpos<_StateT>& __y) {
+  return streamoff(__x) - streamoff(__y);
+}
+
+template <class _StateT>
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(const fpos<_StateT>& __x, const fpos<_StateT>& __y) {
+  return streamoff(__x) == streamoff(__y);
+}
+
+template <class _StateT>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const fpos<_StateT>& __x, const fpos<_StateT>& __y) {
+  return streamoff(__x) != streamoff(__y);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___IOS_FPOS_H
diff --git a/libcxx/include/__cxx03/__iterator/access.h b/libcxx/include/__cxx03/__iterator/access.h
new file mode 100644
index 000000000000..acc4f60bf697
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/access.h
@@ -0,0 +1,95 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_ACCESS_H
+#define _LIBCPP___ITERATOR_ACCESS_H
+
+#include <__config>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, size_t _Np>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp* begin(_Tp (&__array)[_Np]) _NOEXCEPT {
+  return __array;
+}
+
+template <class _Tp, size_t _Np>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp* end(_Tp (&__array)[_Np]) _NOEXCEPT {
+  return __array + _Np;
+}
+
+#if !defined(_LIBCPP_CXX03_LANG)
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto begin(_Cp& __c) -> decltype(__c.begin()) {
+  return __c.begin();
+}
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto begin(const _Cp& __c) -> decltype(__c.begin()) {
+  return __c.begin();
+}
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto end(_Cp& __c) -> decltype(__c.end()) {
+  return __c.end();
+}
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto end(const _Cp& __c) -> decltype(__c.end()) {
+  return __c.end();
+}
+
+#  if _LIBCPP_STD_VER >= 14
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+cbegin(const _Cp& __c) noexcept(noexcept(std::begin(__c))) -> decltype(std::begin(__c)) {
+  return std::begin(__c);
+}
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI constexpr auto cend(const _Cp& __c) noexcept(noexcept(std::end(__c))) -> decltype(std::end(__c)) {
+  return std::end(__c);
+}
+
+#  endif
+
+#else // defined(_LIBCPP_CXX03_LANG)
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI typename _Cp::iterator begin(_Cp& __c) {
+  return __c.begin();
+}
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI typename _Cp::const_iterator begin(const _Cp& __c) {
+  return __c.begin();
+}
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI typename _Cp::iterator end(_Cp& __c) {
+  return __c.end();
+}
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI typename _Cp::const_iterator end(const _Cp& __c) {
+  return __c.end();
+}
+
+#endif // !defined(_LIBCPP_CXX03_LANG)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_ACCESS_H
diff --git a/libcxx/include/__cxx03/__iterator/advance.h b/libcxx/include/__cxx03/__iterator/advance.h
new file mode 100644
index 000000000000..296db1aaab65
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/advance.h
@@ -0,0 +1,205 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_ADVANCE_H
+#define _LIBCPP___ITERATOR_ADVANCE_H
+
+#include <__assert>
+#include <__concepts/assignable.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_integral.h>
+#include <__utility/convert_to_integral.h>
+#include <__utility/declval.h>
+#include <__utility/move.h>
+#include <__utility/unreachable.h>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIter>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 void
+__advance(_InputIter& __i, typename iterator_traits<_InputIter>::difference_type __n, input_iterator_tag) {
+  for (; __n > 0; --__n)
+    ++__i;
+}
+
+template <class _BiDirIter>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 void
+__advance(_BiDirIter& __i, typename iterator_traits<_BiDirIter>::difference_type __n, bidirectional_iterator_tag) {
+  if (__n >= 0)
+    for (; __n > 0; --__n)
+      ++__i;
+  else
+    for (; __n < 0; ++__n)
+      --__i;
+}
+
+template <class _RandIter>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 void
+__advance(_RandIter& __i, typename iterator_traits<_RandIter>::difference_type __n, random_access_iterator_tag) {
+  __i += __n;
+}
+
+template < class _InputIter,
+           class _Distance,
+           class _IntegralDistance = decltype(std::__convert_to_integral(std::declval<_Distance>())),
+           __enable_if_t<is_integral<_IntegralDistance>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 void advance(_InputIter& __i, _Distance __orig_n) {
+  typedef typename iterator_traits<_InputIter>::difference_type _Difference;
+  _Difference __n = static_cast<_Difference>(std::__convert_to_integral(__orig_n));
+  // Calling `advance` with a negative value on a non-bidirectional iterator is a no-op in the current implementation.
+  _LIBCPP_ASSERT_PEDANTIC(__n >= 0 || __has_bidirectional_iterator_category<_InputIter>::value,
+                          "Attempt to advance(it, n) with negative n on a non-bidirectional iterator");
+  std::__advance(__i, __n, typename iterator_traits<_InputIter>::iterator_category());
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+// [range.iter.op.advance]
+
+namespace ranges {
+namespace __advance {
+
+struct __fn {
+private:
+  template <class _Ip>
+  _LIBCPP_HIDE_FROM_ABI static constexpr void __advance_forward(_Ip& __i, iter_difference_t<_Ip> __n) {
+    while (__n > 0) {
+      --__n;
+      ++__i;
+    }
+  }
+
+  template <class _Ip>
+  _LIBCPP_HIDE_FROM_ABI static constexpr void __advance_backward(_Ip& __i, iter_difference_t<_Ip> __n) {
+    while (__n < 0) {
+      ++__n;
+      --__i;
+    }
+  }
+
+public:
+  // Preconditions: If `I` does not model `bidirectional_iterator`, `n` is not negative.
+  template <input_or_output_iterator _Ip>
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Ip& __i, iter_difference_t<_Ip> __n) const {
+    // Calling `advance` with a negative value on a non-bidirectional iterator is a no-op in the current implementation.
+    _LIBCPP_ASSERT_PEDANTIC(
+        __n >= 0 || bidirectional_iterator<_Ip>, "If `n < 0`, then `bidirectional_iterator<I>` must be true.");
+
+    // If `I` models `random_access_iterator`, equivalent to `i += n`.
+    if constexpr (random_access_iterator<_Ip>) {
+      __i += __n;
+      return;
+    } else if constexpr (bidirectional_iterator<_Ip>) {
+      // Otherwise, if `n` is non-negative, increments `i` by `n`.
+      __advance_forward(__i, __n);
+      // Otherwise, decrements `i` by `-n`.
+      __advance_backward(__i, __n);
+      return;
+    } else {
+      // Otherwise, if `n` is non-negative, increments `i` by `n`.
+      __advance_forward(__i, __n);
+      return;
+    }
+  }
+
+  // Preconditions: Either `assignable_from<I&, S> || sized_sentinel_for<S, I>` is modeled, or [i, bound_sentinel)
+  // denotes a range.
+  template <input_or_output_iterator _Ip, sentinel_for<_Ip> _Sp>
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Ip& __i, _Sp __bound_sentinel) const {
+    // If `I` and `S` model `assignable_from<I&, S>`, equivalent to `i = std::move(bound_sentinel)`.
+    if constexpr (assignable_from<_Ip&, _Sp>) {
+      __i = std::move(__bound_sentinel);
+    }
+    // Otherwise, if `S` and `I` model `sized_sentinel_for<S, I>`, equivalent to `ranges::advance(i, bound_sentinel -
+    // i)`.
+    else if constexpr (sized_sentinel_for<_Sp, _Ip>) {
+      (*this)(__i, __bound_sentinel - __i);
+    }
+    // Otherwise, while `bool(i != bound_sentinel)` is true, increments `i`.
+    else {
+      while (__i != __bound_sentinel) {
+        ++__i;
+      }
+    }
+  }
+
+  // Preconditions:
+  //   * If `n > 0`, [i, bound_sentinel) denotes a range.
+  //   * If `n == 0`, [i, bound_sentinel) or [bound_sentinel, i) denotes a range.
+  //   * If `n < 0`, [bound_sentinel, i) denotes a range, `I` models `bidirectional_iterator`, and `I` and `S` model
+  //   `same_as<I, S>`.
+  // Returns: `n - M`, where `M` is the difference between the ending and starting position.
+  template <input_or_output_iterator _Ip, sentinel_for<_Ip> _Sp>
+  _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Ip>
+  operator()(_Ip& __i, iter_difference_t<_Ip> __n, _Sp __bound_sentinel) const {
+    // Calling `advance` with a negative value on a non-bidirectional iterator is a no-op in the current implementation.
+    _LIBCPP_ASSERT_PEDANTIC((__n >= 0) || (bidirectional_iterator<_Ip> && same_as<_Ip, _Sp>),
+                            "If `n < 0`, then `bidirectional_iterator<I> && same_as<I, S>` must be true.");
+    // If `S` and `I` model `sized_sentinel_for<S, I>`:
+    if constexpr (sized_sentinel_for<_Sp, _Ip>) {
+      // If |n| >= |bound_sentinel - i|, equivalent to `ranges::advance(i, bound_sentinel)`.
+      // __magnitude_geq(a, b) returns |a| >= |b|, assuming they have the same sign.
+      auto __magnitude_geq = [](auto __a, auto __b) { return __a == 0 ? __b == 0 : __a > 0 ? __a >= __b : __a <= __b; };
+      if (const auto __m = __bound_sentinel - __i; __magnitude_geq(__n, __m)) {
+        (*this)(__i, __bound_sentinel);
+        return __n - __m;
+      }
+
+      // Otherwise, equivalent to `ranges::advance(i, n)`.
+      (*this)(__i, __n);
+      return 0;
+    } else {
+      // Otherwise, if `n` is non-negative, while `bool(i != bound_sentinel)` is true, increments `i` but at
+      // most `n` times.
+      while (__n > 0 && __i != __bound_sentinel) {
+        ++__i;
+        --__n;
+      }
+
+      // Otherwise, while `bool(i != bound_sentinel)` is true, decrements `i` but at most `-n` times.
+      if constexpr (bidirectional_iterator<_Ip> && same_as<_Ip, _Sp>) {
+        while (__n < 0 && __i != __bound_sentinel) {
+          --__i;
+          ++__n;
+        }
+      }
+      return __n;
+    }
+
+    __libcpp_unreachable();
+  }
+};
+
+} // namespace __advance
+
+inline namespace __cpo {
+inline constexpr auto advance = __advance::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ITERATOR_ADVANCE_H
diff --git a/libcxx/include/__cxx03/__iterator/aliasing_iterator.h b/libcxx/include/__cxx03/__iterator/aliasing_iterator.h
new file mode 100644
index 000000000000..94ba577078b5
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/aliasing_iterator.h
@@ -0,0 +1,127 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_ALIASING_ITERATOR_H
+#define _LIBCPP___ITERATOR_ALIASING_ITERATOR_H
+
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__memory/pointer_traits.h>
+#include <__type_traits/is_trivial.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+// This iterator wrapper is used to type-pun an iterator to return a different type. This is done without UB by not
+// actually punning the type, but instead inspecting the object representation of the base type and copying that into
+// an instance of the alias type. For that reason the alias type has to be trivial. The alias is returned as a prvalue
+// when derferencing the iterator, since it is temporary storage. This wrapper is used to vectorize some algorithms.
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _BaseIter, class _Alias>
+struct __aliasing_iterator_wrapper {
+  class __iterator {
+    _BaseIter __base_ = nullptr;
+
+    using __iter_traits     = iterator_traits<_BaseIter>;
+    using __base_value_type = typename __iter_traits::value_type;
+
+    static_assert(__has_random_access_iterator_category<_BaseIter>::value,
+                  "The base iterator has to be a random access iterator!");
+
+  public:
+    using iterator_category = random_access_iterator_tag;
+    using value_type        = _Alias;
+    using difference_type   = ptrdiff_t;
+    using reference         = value_type&;
+    using pointer           = value_type*;
+
+    static_assert(is_trivial<value_type>::value);
+    static_assert(sizeof(__base_value_type) == sizeof(value_type));
+
+    _LIBCPP_HIDE_FROM_ABI __iterator() = default;
+    _LIBCPP_HIDE_FROM_ABI __iterator(_BaseIter __base) _NOEXCEPT : __base_(__base) {}
+
+    _LIBCPP_HIDE_FROM_ABI __iterator& operator++() _NOEXCEPT {
+      ++__base_;
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator operator++(int) _NOEXCEPT {
+      __iterator __tmp(*this);
+      ++__base_;
+      return __tmp;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator& operator--() _NOEXCEPT {
+      --__base_;
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator operator--(int) _NOEXCEPT {
+      __iterator __tmp(*this);
+      --__base_;
+      return __tmp;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend __iterator operator+(__iterator __iter, difference_type __n) _NOEXCEPT {
+      return __iterator(__iter.__base_ + __n);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend __iterator operator+(difference_type __n, __iterator __iter) _NOEXCEPT {
+      return __iterator(__n + __iter.__base_);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator& operator+=(difference_type __n) _NOEXCEPT {
+      __base_ += __n;
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend __iterator operator-(__iterator __iter, difference_type __n) _NOEXCEPT {
+      return __iterator(__iter.__base_ - __n);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend difference_type operator-(__iterator __lhs, __iterator __rhs) _NOEXCEPT {
+      return __lhs.__base_ - __rhs.__base_;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator& operator-=(difference_type __n) _NOEXCEPT {
+      __base_ -= __n;
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _BaseIter __base() const _NOEXCEPT { return __base_; }
+
+    _LIBCPP_HIDE_FROM_ABI _Alias operator*() const _NOEXCEPT {
+      _Alias __val;
+      __builtin_memcpy(&__val, std::__to_address(__base_), sizeof(value_type));
+      return __val;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI value_type operator[](difference_type __n) const _NOEXCEPT { return *(*this + __n); }
+
+    _LIBCPP_HIDE_FROM_ABI friend bool operator==(const __iterator& __lhs, const __iterator& __rhs) _NOEXCEPT {
+      return __lhs.__base_ == __rhs.__base_;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend bool operator!=(const __iterator& __lhs, const __iterator& __rhs) _NOEXCEPT {
+      return __lhs.__base_ != __rhs.__base_;
+    }
+  };
+};
+
+// This is required to avoid ADL instantiations on _BaseT
+template <class _BaseT, class _Alias>
+using __aliasing_iterator = typename __aliasing_iterator_wrapper<_BaseT, _Alias>::__iterator;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_ALIASING_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/back_insert_iterator.h b/libcxx/include/__cxx03/__iterator/back_insert_iterator.h
new file mode 100644
index 000000000000..6d3dd4b12966
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/back_insert_iterator.h
@@ -0,0 +1,85 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_BACK_INSERT_ITERATOR_H
+#define _LIBCPP___ITERATOR_BACK_INSERT_ITERATOR_H
+
+#include <__config>
+#include <__iterator/iterator.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__utility/move.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Container>
+class _LIBCPP_TEMPLATE_VIS back_insert_iterator
+#if _LIBCPP_STD_VER <= 14 || !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
+    : public iterator<output_iterator_tag, void, void, void, void>
+#endif
+{
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  _Container* container;
+
+public:
+  typedef output_iterator_tag iterator_category;
+  typedef void value_type;
+#if _LIBCPP_STD_VER >= 20
+  typedef ptrdiff_t difference_type;
+#else
+  typedef void difference_type;
+#endif
+  typedef void pointer;
+  typedef void reference;
+  typedef _Container container_type;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit back_insert_iterator(_Container& __x)
+      : container(std::addressof(__x)) {}
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator&
+  operator=(const typename _Container::value_type& __value) {
+    container->push_back(__value);
+    return *this;
+  }
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator&
+  operator=(typename _Container::value_type&& __value) {
+    container->push_back(std::move(__value));
+    return *this;
+  }
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator& operator*() { return *this; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator& operator++() { return *this; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator operator++(int) { return *this; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Container* __get_container() const { return container; }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(back_insert_iterator);
+
+template <class _Container>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator<_Container>
+back_inserter(_Container& __x) {
+  return back_insert_iterator<_Container>(__x);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ITERATOR_BACK_INSERT_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/bounded_iter.h b/libcxx/include/__cxx03/__iterator/bounded_iter.h
new file mode 100644
index 000000000000..8a81c9ffbfc3
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/bounded_iter.h
@@ -0,0 +1,283 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_BOUNDED_ITER_H
+#define _LIBCPP___ITERATOR_BOUNDED_ITER_H
+
+#include <__assert>
+#include <__compare/ordering.h>
+#include <__compare/three_way_comparable.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__memory/pointer_traits.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_convertible.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Iterator wrapper that carries the valid range it is allowed to access.
+//
+// This is a simple iterator wrapper for contiguous iterators that points
+// within a [begin, end] range and carries these bounds with it. The iterator
+// ensures that it is pointing within [begin, end) range when it is
+// dereferenced. It also ensures that it is never iterated outside of
+// [begin, end]. This is important for two reasons:
+//
+// 1. It allows `operator*` and `operator++` bounds checks to be `iter != end`.
+//    This is both less for the optimizer to prove, and aligns with how callers
+//    typically use iterators.
+//
+// 2. Advancing an iterator out of bounds is undefined behavior (see the table
+//    in [input.iterators]). In particular, when the underlying iterator is a
+//    pointer, it is undefined at the language level (see [expr.add]). If
+//    bounded iterators exhibited this undefined behavior, we risk compiler
+//    optimizations deleting non-redundant bounds checks.
+template <class _Iterator, class = __enable_if_t< __libcpp_is_contiguous_iterator<_Iterator>::value > >
+struct __bounded_iter {
+  using value_type        = typename iterator_traits<_Iterator>::value_type;
+  using difference_type   = typename iterator_traits<_Iterator>::difference_type;
+  using pointer           = typename iterator_traits<_Iterator>::pointer;
+  using reference         = typename iterator_traits<_Iterator>::reference;
+  using iterator_category = typename iterator_traits<_Iterator>::iterator_category;
+#if _LIBCPP_STD_VER >= 20
+  using iterator_concept = contiguous_iterator_tag;
+#endif
+
+  // Create a singular iterator.
+  //
+  // Such an iterator points past the end of an empty span, so it is not dereferenceable.
+  // Observing operations like comparison and assignment are valid.
+  _LIBCPP_HIDE_FROM_ABI __bounded_iter() = default;
+
+  _LIBCPP_HIDE_FROM_ABI __bounded_iter(__bounded_iter const&) = default;
+  _LIBCPP_HIDE_FROM_ABI __bounded_iter(__bounded_iter&&)      = default;
+
+  template <class _OtherIterator, __enable_if_t< is_convertible<_OtherIterator, _Iterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __bounded_iter(__bounded_iter<_OtherIterator> const& __other) _NOEXCEPT
+      : __current_(__other.__current_),
+        __begin_(__other.__begin_),
+        __end_(__other.__end_) {}
+
+  // Assign a bounded iterator to another one, rebinding the bounds of the iterator as well.
+  _LIBCPP_HIDE_FROM_ABI __bounded_iter& operator=(__bounded_iter const&) = default;
+  _LIBCPP_HIDE_FROM_ABI __bounded_iter& operator=(__bounded_iter&&)      = default;
+
+private:
+  // Create an iterator wrapping the given iterator, and whose bounds are described
+  // by the provided [begin, end] range.
+  //
+  // The constructor does not check whether the resulting iterator is within its bounds. It is a
+  // responsibility of the container to ensure that the given bounds are valid.
+  //
+  // Since it is non-standard for iterators to have this constructor, __bounded_iter must
+  // be created via `std::__make_bounded_iter`.
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __bounded_iter(_Iterator __current, _Iterator __begin, _Iterator __end)
+      : __current_(__current), __begin_(__begin), __end_(__end) {
+    _LIBCPP_ASSERT_INTERNAL(
+        __begin <= __current, "__bounded_iter(current, begin, end): current and begin are inconsistent");
+    _LIBCPP_ASSERT_INTERNAL(
+        __current <= __end, "__bounded_iter(current, begin, end): current and end are inconsistent");
+  }
+
+  template <class _It>
+  friend _LIBCPP_CONSTEXPR __bounded_iter<_It> __make_bounded_iter(_It, _It, _It);
+
+public:
+  // Dereference and indexing operations.
+  //
+  // These operations check that the iterator is dereferenceable. Since the class invariant is
+  // that the iterator is always within `[begin, end]`, we only need to check it's not pointing to
+  // `end`. This is easier for the optimizer because it aligns with the `iter != container.end()`
+  // checks that typical callers already use (see
+  // https://github.com/llvm/llvm-project/issues/78829).
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 reference operator*() const _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __current_ != __end_, "__bounded_iter::operator*: Attempt to dereference an iterator at the end");
+    return *__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pointer operator->() const _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __current_ != __end_, "__bounded_iter::operator->: Attempt to dereference an iterator at the end");
+    return std::__to_address(__current_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 reference operator[](difference_type __n) const _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __n >= __begin_ - __current_, "__bounded_iter::operator[]: Attempt to index an iterator past the start");
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __n < __end_ - __current_, "__bounded_iter::operator[]: Attempt to index an iterator at or past the end");
+    return __current_[__n];
+  }
+
+  // Arithmetic operations.
+  //
+  // These operations check that the iterator remains within `[begin, end]`.
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __bounded_iter& operator++() _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __current_ != __end_, "__bounded_iter::operator++: Attempt to advance an iterator past the end");
+    ++__current_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __bounded_iter operator++(int) _NOEXCEPT {
+    __bounded_iter __tmp(*this);
+    ++*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __bounded_iter& operator--() _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __current_ != __begin_, "__bounded_iter::operator--: Attempt to rewind an iterator past the start");
+    --__current_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __bounded_iter operator--(int) _NOEXCEPT {
+    __bounded_iter __tmp(*this);
+    --*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __bounded_iter& operator+=(difference_type __n) _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __n >= __begin_ - __current_, "__bounded_iter::operator+=: Attempt to rewind an iterator past the start");
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __n <= __end_ - __current_, "__bounded_iter::operator+=: Attempt to advance an iterator past the end");
+    __current_ += __n;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 friend __bounded_iter
+  operator+(__bounded_iter const& __self, difference_type __n) _NOEXCEPT {
+    __bounded_iter __tmp(__self);
+    __tmp += __n;
+    return __tmp;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 friend __bounded_iter
+  operator+(difference_type __n, __bounded_iter const& __self) _NOEXCEPT {
+    __bounded_iter __tmp(__self);
+    __tmp += __n;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __bounded_iter& operator-=(difference_type __n) _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __n <= __current_ - __begin_, "__bounded_iter::operator-=: Attempt to rewind an iterator past the start");
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __n >= __current_ - __end_, "__bounded_iter::operator-=: Attempt to advance an iterator past the end");
+    __current_ -= __n;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 friend __bounded_iter
+  operator-(__bounded_iter const& __self, difference_type __n) _NOEXCEPT {
+    __bounded_iter __tmp(__self);
+    __tmp -= __n;
+    return __tmp;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 friend difference_type
+  operator-(__bounded_iter const& __x, __bounded_iter const& __y) _NOEXCEPT {
+    return __x.__current_ - __y.__current_;
+  }
+
+  // Comparison operations.
+  //
+  // These operations do not check whether the iterators are within their bounds.
+  // The valid range for each iterator is also not considered as part of the comparison,
+  // i.e. two iterators pointing to the same location will be considered equal even
+  // if they have different validity ranges.
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR friend bool
+  operator==(__bounded_iter const& __x, __bounded_iter const& __y) _NOEXCEPT {
+    return __x.__current_ == __y.__current_;
+  }
+
+#if _LIBCPP_STD_VER <= 17
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR friend bool
+  operator!=(__bounded_iter const& __x, __bounded_iter const& __y) _NOEXCEPT {
+    return __x.__current_ != __y.__current_;
+  }
+#endif
+
+  // TODO(mordante) disable these overloads in the LLVM 20 release.
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR friend bool
+  operator<(__bounded_iter const& __x, __bounded_iter const& __y) _NOEXCEPT {
+    return __x.__current_ < __y.__current_;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR friend bool
+  operator>(__bounded_iter const& __x, __bounded_iter const& __y) _NOEXCEPT {
+    return __x.__current_ > __y.__current_;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR friend bool
+  operator<=(__bounded_iter const& __x, __bounded_iter const& __y) _NOEXCEPT {
+    return __x.__current_ <= __y.__current_;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR friend bool
+  operator>=(__bounded_iter const& __x, __bounded_iter const& __y) _NOEXCEPT {
+    return __x.__current_ >= __y.__current_;
+  }
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI constexpr friend strong_ordering
+  operator<=>(__bounded_iter const& __x, __bounded_iter const& __y) noexcept {
+    if constexpr (three_way_comparable<_Iterator, strong_ordering>) {
+      return __x.__current_ <=> __y.__current_;
+    } else {
+      if (__x.__current_ < __y.__current_)
+        return strong_ordering::less;
+
+      if (__x.__current_ == __y.__current_)
+        return strong_ordering::equal;
+
+      return strong_ordering::greater;
+    }
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+private:
+  template <class>
+  friend struct pointer_traits;
+  template <class, class>
+  friend struct __bounded_iter;
+  _Iterator __current_;       // current iterator
+  _Iterator __begin_, __end_; // valid range represented as [begin, end]
+};
+
+template <class _It>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __bounded_iter<_It> __make_bounded_iter(_It __it, _It __begin, _It __end) {
+  return __bounded_iter<_It>(std::move(__it), std::move(__begin), std::move(__end));
+}
+
+#if _LIBCPP_STD_VER <= 17
+template <class _Iterator>
+struct __libcpp_is_contiguous_iterator<__bounded_iter<_Iterator> > : true_type {};
+#endif
+
+template <class _Iterator>
+struct pointer_traits<__bounded_iter<_Iterator> > {
+  using pointer         = __bounded_iter<_Iterator>;
+  using element_type    = typename pointer_traits<_Iterator>::element_type;
+  using difference_type = typename pointer_traits<_Iterator>::difference_type;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR static element_type* to_address(pointer __it) _NOEXCEPT {
+    return std::__to_address(__it.__current_);
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ITERATOR_BOUNDED_ITER_H
diff --git a/libcxx/include/__cxx03/__iterator/common_iterator.h b/libcxx/include/__cxx03/__iterator/common_iterator.h
new file mode 100644
index 000000000000..199de2cc7337
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/common_iterator.h
@@ -0,0 +1,299 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_COMMON_ITERATOR_H
+#define _LIBCPP___ITERATOR_COMMON_ITERATOR_H
+
+#include <__assert>
+#include <__concepts/assignable.h>
+#include <__concepts/constructible.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/copyable.h>
+#include <__concepts/derived_from.h>
+#include <__concepts/equality_comparable.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/iter_swap.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/readable_traits.h>
+#include <__memory/addressof.h>
+#include <__type_traits/is_pointer.h>
+#include <__utility/declval.h>
+#include <variant>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Iter>
+concept __can_use_postfix_proxy =
+    constructible_from<iter_value_t<_Iter>, iter_reference_t<_Iter>> && move_constructible<iter_value_t<_Iter>>;
+
+template <input_or_output_iterator _Iter, sentinel_for<_Iter> _Sent>
+  requires(!same_as<_Iter, _Sent> && copyable<_Iter>)
+class common_iterator {
+  struct __proxy {
+    _LIBCPP_HIDE_FROM_ABI constexpr const iter_value_t<_Iter>* operator->() const noexcept {
+      return std::addressof(__value_);
+    }
+    iter_value_t<_Iter> __value_;
+  };
+
+  struct __postfix_proxy {
+    _LIBCPP_HIDE_FROM_ABI constexpr const iter_value_t<_Iter>& operator*() const noexcept { return __value_; }
+    iter_value_t<_Iter> __value_;
+  };
+
+  variant<_Iter, _Sent> __hold_;
+  template <input_or_output_iterator _OtherIter, sentinel_for<_OtherIter> _OtherSent>
+    requires(!same_as<_OtherIter, _OtherSent> && copyable<_OtherIter>)
+  friend class common_iterator;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI common_iterator()
+    requires default_initializable<_Iter>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr common_iterator(_Iter __i) : __hold_(in_place_type<_Iter>, std::move(__i)) {}
+  _LIBCPP_HIDE_FROM_ABI constexpr common_iterator(_Sent __s) : __hold_(in_place_type<_Sent>, std::move(__s)) {}
+
+  template <class _I2, class _S2>
+    requires convertible_to<const _I2&, _Iter> && convertible_to<const _S2&, _Sent>
+  _LIBCPP_HIDE_FROM_ABI constexpr common_iterator(const common_iterator<_I2, _S2>& __other)
+      : __hold_([&]() -> variant<_Iter, _Sent> {
+          _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+              !__other.__hold_.valueless_by_exception(), "Attempted to construct from a valueless common_iterator");
+          if (__other.__hold_.index() == 0)
+            return variant<_Iter, _Sent>{in_place_index<0>, std::__unchecked_get<0>(__other.__hold_)};
+          return variant<_Iter, _Sent>{in_place_index<1>, std::__unchecked_get<1>(__other.__hold_)};
+        }()) {}
+
+  template <class _I2, class _S2>
+    requires convertible_to<const _I2&, _Iter> && convertible_to<const _S2&, _Sent> &&
+             assignable_from<_Iter&, const _I2&> && assignable_from<_Sent&, const _S2&>
+  _LIBCPP_HIDE_FROM_ABI common_iterator& operator=(const common_iterator<_I2, _S2>& __other) {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !__other.__hold_.valueless_by_exception(), "Attempted to assign from a valueless common_iterator");
+
+    auto __idx       = __hold_.index();
+    auto __other_idx = __other.__hold_.index();
+
+    // If they're the same index, just assign.
+    if (__idx == 0 && __other_idx == 0)
+      std::__unchecked_get<0>(__hold_) = std::__unchecked_get<0>(__other.__hold_);
+    else if (__idx == 1 && __other_idx == 1)
+      std::__unchecked_get<1>(__hold_) = std::__unchecked_get<1>(__other.__hold_);
+
+    // Otherwise replace with the oposite element.
+    else if (__other_idx == 1)
+      __hold_.template emplace<1>(std::__unchecked_get<1>(__other.__hold_));
+    else if (__other_idx == 0)
+      __hold_.template emplace<0>(std::__unchecked_get<0>(__other.__hold_));
+
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        std::holds_alternative<_Iter>(__hold_), "Attempted to dereference a non-dereferenceable common_iterator");
+    return *std::__unchecked_get<_Iter>(__hold_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() const
+    requires __dereferenceable<const _Iter>
+  {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        std::holds_alternative<_Iter>(__hold_), "Attempted to dereference a non-dereferenceable common_iterator");
+    return *std::__unchecked_get<_Iter>(__hold_);
+  }
+
+  template <class _I2 = _Iter>
+  _LIBCPP_HIDE_FROM_ABI auto operator->() const
+    requires indirectly_readable<const _I2> && (requires(const _I2& __i) {
+               __i.operator->();
+             } || is_reference_v<iter_reference_t<_I2>> || constructible_from<iter_value_t<_I2>, iter_reference_t<_I2>>)
+  {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        std::holds_alternative<_Iter>(__hold_), "Attempted to dereference a non-dereferenceable common_iterator");
+    if constexpr (is_pointer_v<_Iter> || requires(const _Iter& __i) { __i.operator->(); }) {
+      return std::__unchecked_get<_Iter>(__hold_);
+    } else if constexpr (is_reference_v<iter_reference_t<_Iter>>) {
+      auto&& __tmp = *std::__unchecked_get<_Iter>(__hold_);
+      return std::addressof(__tmp);
+    } else {
+      return __proxy{*std::__unchecked_get<_Iter>(__hold_)};
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI common_iterator& operator++() {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        std::holds_alternative<_Iter>(__hold_), "Attempted to increment a non-dereferenceable common_iterator");
+    ++std::__unchecked_get<_Iter>(__hold_);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI decltype(auto) operator++(int) {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        std::holds_alternative<_Iter>(__hold_), "Attempted to increment a non-dereferenceable common_iterator");
+    if constexpr (forward_iterator<_Iter>) {
+      auto __tmp = *this;
+      ++*this;
+      return __tmp;
+    } else if constexpr (requires(_Iter& __i) {
+                           { *__i++ } -> __can_reference;
+                         } || !__can_use_postfix_proxy<_Iter>) {
+      return std::__unchecked_get<_Iter>(__hold_)++;
+    } else {
+      auto __p = __postfix_proxy{**this};
+      ++*this;
+      return __p;
+    }
+  }
+
+  template <class _I2, sentinel_for<_Iter> _S2>
+    requires sentinel_for<_Sent, _I2>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool
+  operator==(const common_iterator& __x, const common_iterator<_I2, _S2>& __y) {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !__x.__hold_.valueless_by_exception(), "Attempted to compare a valueless common_iterator");
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !__y.__hold_.valueless_by_exception(), "Attempted to compare a valueless common_iterator");
+
+    auto __x_index = __x.__hold_.index();
+    auto __y_index = __y.__hold_.index();
+
+    if (__x_index == __y_index)
+      return true;
+
+    if (__x_index == 0)
+      return std::__unchecked_get<_Iter>(__x.__hold_) == std::__unchecked_get<_S2>(__y.__hold_);
+
+    return std::__unchecked_get<_Sent>(__x.__hold_) == std::__unchecked_get<_I2>(__y.__hold_);
+  }
+
+  template <class _I2, sentinel_for<_Iter> _S2>
+    requires sentinel_for<_Sent, _I2> && equality_comparable_with<_Iter, _I2>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool
+  operator==(const common_iterator& __x, const common_iterator<_I2, _S2>& __y) {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !__x.__hold_.valueless_by_exception(), "Attempted to compare a valueless common_iterator");
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !__y.__hold_.valueless_by_exception(), "Attempted to compare a valueless common_iterator");
+
+    auto __x_index = __x.__hold_.index();
+    auto __y_index = __y.__hold_.index();
+
+    if (__x_index == 1 && __y_index == 1)
+      return true;
+
+    if (__x_index == 0 && __y_index == 0)
+      return std::__unchecked_get<_Iter>(__x.__hold_) == std::__unchecked_get<_I2>(__y.__hold_);
+
+    if (__x_index == 0)
+      return std::__unchecked_get<_Iter>(__x.__hold_) == std::__unchecked_get<_S2>(__y.__hold_);
+
+    return std::__unchecked_get<_Sent>(__x.__hold_) == std::__unchecked_get<_I2>(__y.__hold_);
+  }
+
+  template <sized_sentinel_for<_Iter> _I2, sized_sentinel_for<_Iter> _S2>
+    requires sized_sentinel_for<_Sent, _I2>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr iter_difference_t<_I2>
+  operator-(const common_iterator& __x, const common_iterator<_I2, _S2>& __y) {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !__x.__hold_.valueless_by_exception(), "Attempted to subtract from a valueless common_iterator");
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !__y.__hold_.valueless_by_exception(), "Attempted to subtract a valueless common_iterator");
+
+    auto __x_index = __x.__hold_.index();
+    auto __y_index = __y.__hold_.index();
+
+    if (__x_index == 1 && __y_index == 1)
+      return 0;
+
+    if (__x_index == 0 && __y_index == 0)
+      return std::__unchecked_get<_Iter>(__x.__hold_) - std::__unchecked_get<_I2>(__y.__hold_);
+
+    if (__x_index == 0)
+      return std::__unchecked_get<_Iter>(__x.__hold_) - std::__unchecked_get<_S2>(__y.__hold_);
+
+    return std::__unchecked_get<_Sent>(__x.__hold_) - std::__unchecked_get<_I2>(__y.__hold_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr iter_rvalue_reference_t<_Iter>
+  iter_move(const common_iterator& __i) noexcept(noexcept(ranges::iter_move(std::declval<const _Iter&>())))
+    requires input_iterator<_Iter>
+  {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        std::holds_alternative<_Iter>(__i.__hold_), "Attempted to iter_move a non-dereferenceable common_iterator");
+    return ranges::iter_move(std::__unchecked_get<_Iter>(__i.__hold_));
+  }
+
+  template <indirectly_swappable<_Iter> _I2, class _S2>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr void
+  iter_swap(const common_iterator& __x, const common_iterator<_I2, _S2>& __y) noexcept(
+      noexcept(ranges::iter_swap(std::declval<const _Iter&>(), std::declval<const _I2&>()))) {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        std::holds_alternative<_Iter>(__x.__hold_), "Attempted to iter_swap a non-dereferenceable common_iterator");
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        std::holds_alternative<_I2>(__y.__hold_), "Attempted to iter_swap a non-dereferenceable common_iterator");
+    return ranges::iter_swap(std::__unchecked_get<_Iter>(__x.__hold_), std::__unchecked_get<_I2>(__y.__hold_));
+  }
+};
+
+template <class _Iter, class _Sent>
+struct incrementable_traits<common_iterator<_Iter, _Sent>> {
+  using difference_type = iter_difference_t<_Iter>;
+};
+
+template <class _Iter>
+concept __denotes_forward_iter = requires {
+  typename iterator_traits<_Iter>::iterator_category;
+} && derived_from<typename iterator_traits<_Iter>::iterator_category, forward_iterator_tag>;
+
+template <class _Iter, class _Sent>
+concept __common_iter_has_ptr_op = requires(const common_iterator<_Iter, _Sent>& __a) { __a.operator->(); };
+
+template <class, class>
+struct __arrow_type_or_void {
+  using type = void;
+};
+
+template <class _Iter, class _Sent>
+  requires __common_iter_has_ptr_op<_Iter, _Sent>
+struct __arrow_type_or_void<_Iter, _Sent> {
+  using type = decltype(std::declval<const common_iterator<_Iter, _Sent>&>().operator->());
+};
+
+template <input_iterator _Iter, class _Sent>
+struct iterator_traits<common_iterator<_Iter, _Sent>> {
+  using iterator_concept  = _If<forward_iterator<_Iter>, forward_iterator_tag, input_iterator_tag>;
+  using iterator_category = _If<__denotes_forward_iter<_Iter>, forward_iterator_tag, input_iterator_tag>;
+  using pointer           = typename __arrow_type_or_void<_Iter, _Sent>::type;
+  using value_type        = iter_value_t<_Iter>;
+  using difference_type   = iter_difference_t<_Iter>;
+  using reference         = iter_reference_t<_Iter>;
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ITERATOR_COMMON_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/concepts.h b/libcxx/include/__cxx03/__iterator/concepts.h
new file mode 100644
index 000000000000..0a4878308d55
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/concepts.h
@@ -0,0 +1,257 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_CONCEPTS_H
+#define _LIBCPP___ITERATOR_CONCEPTS_H
+
+#include <__concepts/arithmetic.h>
+#include <__concepts/assignable.h>
+#include <__concepts/common_reference_with.h>
+#include <__concepts/constructible.h>
+#include <__concepts/copyable.h>
+#include <__concepts/derived_from.h>
+#include <__concepts/equality_comparable.h>
+#include <__concepts/invocable.h>
+#include <__concepts/movable.h>
+#include <__concepts/predicate.h>
+#include <__concepts/regular.h>
+#include <__concepts/relation.h>
+#include <__concepts/same_as.h>
+#include <__concepts/semiregular.h>
+#include <__concepts/totally_ordered.h>
+#include <__config>
+#include <__functional/invoke.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/readable_traits.h>
+#include <__memory/pointer_traits.h>
+#include <__type_traits/add_pointer.h>
+#include <__type_traits/common_reference.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [iterator.concept.readable]
+template <class _In>
+concept __indirectly_readable_impl =
+    requires(const _In __i) {
+      typename iter_value_t<_In>;
+      typename iter_reference_t<_In>;
+      typename iter_rvalue_reference_t<_In>;
+      { *__i } -> same_as<iter_reference_t<_In>>;
+      { ranges::iter_move(__i) } -> same_as<iter_rvalue_reference_t<_In>>;
+    } && common_reference_with<iter_reference_t<_In>&&, iter_value_t<_In>&> &&
+    common_reference_with<iter_reference_t<_In>&&, iter_rvalue_reference_t<_In>&&> &&
+    common_reference_with<iter_rvalue_reference_t<_In>&&, const iter_value_t<_In>&>;
+
+template <class _In>
+concept indirectly_readable = __indirectly_readable_impl<remove_cvref_t<_In>>;
+
+template <indirectly_readable _Tp>
+using iter_common_reference_t = common_reference_t<iter_reference_t<_Tp>, iter_value_t<_Tp>&>;
+
+// [iterator.concept.writable]
+template <class _Out, class _Tp>
+concept indirectly_writable = requires(_Out&& __o, _Tp&& __t) {
+  *__o                                             = std::forward<_Tp>(__t); // not required to be equality-preserving
+  *std::forward<_Out>(__o)                         = std::forward<_Tp>(__t); // not required to be equality-preserving
+  const_cast<const iter_reference_t<_Out>&&>(*__o) = std::forward<_Tp>(__t); // not required to be equality-preserving
+  const_cast<const iter_reference_t<_Out>&&>(*std::forward<_Out>(__o)) =
+      std::forward<_Tp>(__t); // not required to be equality-preserving
+};
+
+// [iterator.concept.winc]
+template <class _Tp>
+concept __integer_like = integral<_Tp> && !same_as<_Tp, bool>;
+
+template <class _Tp>
+concept __signed_integer_like = signed_integral<_Tp>;
+
+template <class _Ip>
+concept weakly_incrementable =
+    // TODO: remove this once the clang bug is fixed (bugs.llvm.org/PR48173).
+    !same_as<_Ip, bool> && // Currently, clang does not handle bool correctly.
+    movable<_Ip> && requires(_Ip __i) {
+      typename iter_difference_t<_Ip>;
+      requires __signed_integer_like<iter_difference_t<_Ip>>;
+      { ++__i } -> same_as<_Ip&>; // not required to be equality-preserving
+      __i++;                      // not required to be equality-preserving
+    };
+
+// [iterator.concept.inc]
+template <class _Ip>
+concept incrementable = regular<_Ip> && weakly_incrementable<_Ip> && requires(_Ip __i) {
+  { __i++ } -> same_as<_Ip>;
+};
+
+// [iterator.concept.iterator]
+template <class _Ip>
+concept input_or_output_iterator = requires(_Ip __i) {
+  { *__i } -> __can_reference;
+} && weakly_incrementable<_Ip>;
+
+// [iterator.concept.sentinel]
+template <class _Sp, class _Ip>
+concept sentinel_for = semiregular<_Sp> && input_or_output_iterator<_Ip> && __weakly_equality_comparable_with<_Sp, _Ip>;
+
+template <class, class>
+inline constexpr bool disable_sized_sentinel_for = false;
+
+template <class _Sp, class _Ip>
+concept sized_sentinel_for =
+    sentinel_for<_Sp, _Ip> && !disable_sized_sentinel_for<remove_cv_t<_Sp>, remove_cv_t<_Ip>> &&
+    requires(const _Ip& __i, const _Sp& __s) {
+      { __s - __i } -> same_as<iter_difference_t<_Ip>>;
+      { __i - __s } -> same_as<iter_difference_t<_Ip>>;
+    };
+
+// [iterator.concept.input]
+template <class _Ip>
+concept input_iterator = input_or_output_iterator<_Ip> && indirectly_readable<_Ip> && requires {
+  typename _ITER_CONCEPT<_Ip>;
+} && derived_from<_ITER_CONCEPT<_Ip>, input_iterator_tag>;
+
+// [iterator.concept.output]
+template <class _Ip, class _Tp>
+concept output_iterator =
+    input_or_output_iterator<_Ip> && indirectly_writable<_Ip, _Tp> && requires(_Ip __it, _Tp&& __t) {
+      *__it++ = std::forward<_Tp>(__t); // not required to be equality-preserving
+    };
+
+// [iterator.concept.forward]
+template <class _Ip>
+concept forward_iterator =
+    input_iterator<_Ip> && derived_from<_ITER_CONCEPT<_Ip>, forward_iterator_tag> && incrementable<_Ip> &&
+    sentinel_for<_Ip, _Ip>;
+
+// [iterator.concept.bidir]
+template <class _Ip>
+concept bidirectional_iterator =
+    forward_iterator<_Ip> && derived_from<_ITER_CONCEPT<_Ip>, bidirectional_iterator_tag> && requires(_Ip __i) {
+      { --__i } -> same_as<_Ip&>;
+      { __i-- } -> same_as<_Ip>;
+    };
+
+template <class _Ip>
+concept random_access_iterator =
+    bidirectional_iterator<_Ip> && derived_from<_ITER_CONCEPT<_Ip>, random_access_iterator_tag> &&
+    totally_ordered<_Ip> && sized_sentinel_for<_Ip, _Ip> &&
+    requires(_Ip __i, const _Ip __j, const iter_difference_t<_Ip> __n) {
+      { __i += __n } -> same_as<_Ip&>;
+      { __j + __n } -> same_as<_Ip>;
+      { __n + __j } -> same_as<_Ip>;
+      { __i -= __n } -> same_as<_Ip&>;
+      { __j - __n } -> same_as<_Ip>;
+      { __j[__n] } -> same_as<iter_reference_t<_Ip>>;
+    };
+
+template <class _Ip>
+concept contiguous_iterator =
+    random_access_iterator<_Ip> && derived_from<_ITER_CONCEPT<_Ip>, contiguous_iterator_tag> &&
+    is_lvalue_reference_v<iter_reference_t<_Ip>> && same_as<iter_value_t<_Ip>, remove_cvref_t<iter_reference_t<_Ip>>> &&
+    requires(const _Ip& __i) {
+      { std::to_address(__i) } -> same_as<add_pointer_t<iter_reference_t<_Ip>>>;
+    };
+
+template <class _Ip>
+concept __has_arrow = input_iterator<_Ip> && (is_pointer_v<_Ip> || requires(_Ip __i) { __i.operator->(); });
+
+// [indirectcallable.indirectinvocable]
+template <class _Fp, class _It>
+concept indirectly_unary_invocable =
+    indirectly_readable<_It> && copy_constructible<_Fp> && invocable<_Fp&, iter_value_t<_It>&> &&
+    invocable<_Fp&, iter_reference_t<_It>> &&
+    common_reference_with< invoke_result_t<_Fp&, iter_value_t<_It>&>, invoke_result_t<_Fp&, iter_reference_t<_It>>>;
+
+template <class _Fp, class _It>
+concept indirectly_regular_unary_invocable =
+    indirectly_readable<_It> && copy_constructible<_Fp> && regular_invocable<_Fp&, iter_value_t<_It>&> &&
+    regular_invocable<_Fp&, iter_reference_t<_It>> &&
+    common_reference_with< invoke_result_t<_Fp&, iter_value_t<_It>&>, invoke_result_t<_Fp&, iter_reference_t<_It>>>;
+
+template <class _Fp, class _It>
+concept indirect_unary_predicate =
+    indirectly_readable<_It> && copy_constructible<_Fp> && predicate<_Fp&, iter_value_t<_It>&> &&
+    predicate<_Fp&, iter_reference_t<_It>>;
+
+template <class _Fp, class _It1, class _It2>
+concept indirect_binary_predicate =
+    indirectly_readable<_It1> && indirectly_readable<_It2> && copy_constructible<_Fp> &&
+    predicate<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&> &&
+    predicate<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>> &&
+    predicate<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&> &&
+    predicate<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>>;
+
+template <class _Fp, class _It1, class _It2 = _It1>
+concept indirect_equivalence_relation =
+    indirectly_readable<_It1> && indirectly_readable<_It2> && copy_constructible<_Fp> &&
+    equivalence_relation<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&> &&
+    equivalence_relation<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>> &&
+    equivalence_relation<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&> &&
+    equivalence_relation<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>>;
+
+template <class _Fp, class _It1, class _It2 = _It1>
+concept indirect_strict_weak_order =
+    indirectly_readable<_It1> && indirectly_readable<_It2> && copy_constructible<_Fp> &&
+    strict_weak_order<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&> &&
+    strict_weak_order<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>> &&
+    strict_weak_order<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&> &&
+    strict_weak_order<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>>;
+
+template <class _Fp, class... _Its>
+  requires(indirectly_readable<_Its> && ...) && invocable<_Fp, iter_reference_t<_Its>...>
+using indirect_result_t = invoke_result_t<_Fp, iter_reference_t<_Its>...>;
+
+template <class _In, class _Out>
+concept indirectly_movable = indirectly_readable<_In> && indirectly_writable<_Out, iter_rvalue_reference_t<_In>>;
+
+template <class _In, class _Out>
+concept indirectly_movable_storable =
+    indirectly_movable<_In, _Out> && indirectly_writable<_Out, iter_value_t<_In>> && movable<iter_value_t<_In>> &&
+    constructible_from<iter_value_t<_In>, iter_rvalue_reference_t<_In>> &&
+    assignable_from<iter_value_t<_In>&, iter_rvalue_reference_t<_In>>;
+
+template <class _In, class _Out>
+concept indirectly_copyable = indirectly_readable<_In> && indirectly_writable<_Out, iter_reference_t<_In>>;
+
+template <class _In, class _Out>
+concept indirectly_copyable_storable =
+    indirectly_copyable<_In, _Out> && indirectly_writable<_Out, iter_value_t<_In>&> &&
+    indirectly_writable<_Out, const iter_value_t<_In>&> && indirectly_writable<_Out, iter_value_t<_In>&&> &&
+    indirectly_writable<_Out, const iter_value_t<_In>&&> && copyable<iter_value_t<_In>> &&
+    constructible_from<iter_value_t<_In>, iter_reference_t<_In>> &&
+    assignable_from<iter_value_t<_In>&, iter_reference_t<_In>>;
+
+// Note: indirectly_swappable is located in iter_swap.h to prevent a dependency cycle
+// (both iter_swap and indirectly_swappable require indirectly_readable).
+
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+using __has_random_access_iterator_category_or_concept
+#if _LIBCPP_STD_VER >= 20
+    = integral_constant<bool, random_access_iterator<_Tp>>;
+#else  // _LIBCPP_STD_VER < 20
+    = __has_random_access_iterator_category<_Tp>;
+#endif // _LIBCPP_STD_VER
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_CONCEPTS_H
diff --git a/libcxx/include/__cxx03/__iterator/counted_iterator.h b/libcxx/include/__cxx03/__iterator/counted_iterator.h
new file mode 100644
index 000000000000..ea2832e3b978
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/counted_iterator.h
@@ -0,0 +1,289 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_COUNTED_ITERATOR_H
+#define _LIBCPP___ITERATOR_COUNTED_ITERATOR_H
+
+#include <__assert>
+#include <__concepts/assignable.h>
+#include <__concepts/common_with.h>
+#include <__concepts/constructible.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/default_sentinel.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/iter_swap.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/readable_traits.h>
+#include <__memory/pointer_traits.h>
+#include <__type_traits/add_pointer.h>
+#include <__type_traits/conditional.h>
+#include <__utility/move.h>
+#include <compare>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class>
+struct __counted_iterator_concept {};
+
+template <class _Iter>
+  requires requires { typename _Iter::iterator_concept; }
+struct __counted_iterator_concept<_Iter> {
+  using iterator_concept = typename _Iter::iterator_concept;
+};
+
+template <class>
+struct __counted_iterator_category {};
+
+template <class _Iter>
+  requires requires { typename _Iter::iterator_category; }
+struct __counted_iterator_category<_Iter> {
+  using iterator_category = typename _Iter::iterator_category;
+};
+
+template <class>
+struct __counted_iterator_value_type {};
+
+template <indirectly_readable _Iter>
+struct __counted_iterator_value_type<_Iter> {
+  using value_type = iter_value_t<_Iter>;
+};
+
+template <input_or_output_iterator _Iter>
+class counted_iterator
+    : public __counted_iterator_concept<_Iter>,
+      public __counted_iterator_category<_Iter>,
+      public __counted_iterator_value_type<_Iter> {
+public:
+  using iterator_type   = _Iter;
+  using difference_type = iter_difference_t<_Iter>;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator()
+    requires default_initializable<_Iter>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator(_Iter __iter, iter_difference_t<_Iter> __n)
+      : __current_(std::move(__iter)), __count_(__n) {
+    _LIBCPP_ASSERT_UNCATEGORIZED(__n >= 0, "__n must not be negative.");
+  }
+
+  template <class _I2>
+    requires convertible_to<const _I2&, _Iter>
+  _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator(const counted_iterator<_I2>& __other)
+      : __current_(__other.__current_), __count_(__other.__count_) {}
+
+  template <class _I2>
+    requires assignable_from<_Iter&, const _I2&>
+  _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator& operator=(const counted_iterator<_I2>& __other) {
+    __current_ = __other.__current_;
+    __count_   = __other.__count_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Iter& base() const& noexcept { return __current_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter base() && { return std::move(__current_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Iter> count() const noexcept { return __count_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__count_ > 0, "Iterator is equal to or past end.");
+    return *__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() const
+    requires __dereferenceable<const _Iter>
+  {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__count_ > 0, "Iterator is equal to or past end.");
+    return *__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator->() const noexcept
+    requires contiguous_iterator<_Iter>
+  {
+    return std::to_address(__current_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator& operator++() {
+    _LIBCPP_ASSERT_UNCATEGORIZED(__count_ > 0, "Iterator already at or past end.");
+    ++__current_;
+    --__count_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator++(int) {
+    _LIBCPP_ASSERT_UNCATEGORIZED(__count_ > 0, "Iterator already at or past end.");
+    --__count_;
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+      return __current_++;
+    } catch (...) {
+      ++__count_;
+      throw;
+    }
+#  else
+    return __current_++;
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator operator++(int)
+    requires forward_iterator<_Iter>
+  {
+    _LIBCPP_ASSERT_UNCATEGORIZED(__count_ > 0, "Iterator already at or past end.");
+    counted_iterator __tmp = *this;
+    ++*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator& operator--()
+    requires bidirectional_iterator<_Iter>
+  {
+    --__current_;
+    ++__count_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator operator--(int)
+    requires bidirectional_iterator<_Iter>
+  {
+    counted_iterator __tmp = *this;
+    --*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator operator+(iter_difference_t<_Iter> __n) const
+    requires random_access_iterator<_Iter>
+  {
+    return counted_iterator(__current_ + __n, __count_ - __n);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr counted_iterator
+  operator+(iter_difference_t<_Iter> __n, const counted_iterator& __x)
+    requires random_access_iterator<_Iter>
+  {
+    return __x + __n;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator& operator+=(iter_difference_t<_Iter> __n)
+    requires random_access_iterator<_Iter>
+  {
+    _LIBCPP_ASSERT_UNCATEGORIZED(__n <= __count_, "Cannot advance iterator past end.");
+    __current_ += __n;
+    __count_ -= __n;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator operator-(iter_difference_t<_Iter> __n) const
+    requires random_access_iterator<_Iter>
+  {
+    return counted_iterator(__current_ - __n, __count_ + __n);
+  }
+
+  template <common_with<_Iter> _I2>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr iter_difference_t<_I2>
+  operator-(const counted_iterator& __lhs, const counted_iterator<_I2>& __rhs) {
+    return __rhs.__count_ - __lhs.__count_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr iter_difference_t<_Iter>
+  operator-(const counted_iterator& __lhs, default_sentinel_t) {
+    return -__lhs.__count_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr iter_difference_t<_Iter>
+  operator-(default_sentinel_t, const counted_iterator& __rhs) {
+    return __rhs.__count_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator& operator-=(iter_difference_t<_Iter> __n)
+    requires random_access_iterator<_Iter>
+  {
+    _LIBCPP_ASSERT_UNCATEGORIZED(
+        -__n <= __count_,
+        "Attempt to subtract too large of a size: "
+        "counted_iterator would be decremented before the "
+        "first element of its range.");
+    __current_ -= __n;
+    __count_ += __n;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator[](iter_difference_t<_Iter> __n) const
+    requires random_access_iterator<_Iter>
+  {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < __count_, "Subscript argument must be less than size.");
+    return __current_[__n];
+  }
+
+  template <common_with<_Iter> _I2>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool
+  operator==(const counted_iterator& __lhs, const counted_iterator<_I2>& __rhs) {
+    return __lhs.__count_ == __rhs.__count_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const counted_iterator& __lhs, default_sentinel_t) {
+    return __lhs.__count_ == 0;
+  }
+
+  template <common_with<_Iter> _I2>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr strong_ordering
+  operator<=>(const counted_iterator& __lhs, const counted_iterator<_I2>& __rhs) {
+    return __rhs.__count_ <=> __lhs.__count_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr iter_rvalue_reference_t<_Iter>
+  iter_move(const counted_iterator& __i) noexcept(noexcept(ranges::iter_move(__i.__current_)))
+    requires input_iterator<_Iter>
+  {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__i.__count_ > 0, "Iterator must not be past end of range.");
+    return ranges::iter_move(__i.__current_);
+  }
+
+  template <indirectly_swappable<_Iter> _I2>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr void
+  iter_swap(const counted_iterator& __x,
+            const counted_iterator<_I2>& __y) noexcept(noexcept(ranges::iter_swap(__x.__current_, __y.__current_))) {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __x.__count_ > 0 && __y.__count_ > 0, "Iterators must not be past end of range.");
+    return ranges::iter_swap(__x.__current_, __y.__current_);
+  }
+
+private:
+  _LIBCPP_NO_UNIQUE_ADDRESS _Iter __current_ = _Iter();
+  iter_difference_t<_Iter> __count_          = 0;
+  template <input_or_output_iterator _OtherIter>
+  friend class counted_iterator;
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(counted_iterator);
+
+template <input_iterator _Iter>
+  requires same_as<_ITER_TRAITS<_Iter>, iterator_traits<_Iter>>
+struct iterator_traits<counted_iterator<_Iter>> : iterator_traits<_Iter> {
+  using pointer = conditional_t<contiguous_iterator<_Iter>, add_pointer_t<iter_reference_t<_Iter>>, void>;
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ITERATOR_COUNTED_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/cpp17_iterator_concepts.h b/libcxx/include/__cxx03/__iterator/cpp17_iterator_concepts.h
new file mode 100644
index 000000000000..ba3536b68609
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/cpp17_iterator_concepts.h
@@ -0,0 +1,190 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_CPP17_ITERATOR_CONCEPTS_H
+#define _LIBCPP___ITERATOR_CPP17_ITERATOR_CONCEPTS_H
+
+#include <__concepts/boolean_testable.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_signed.h>
+#include <__type_traits/is_void.h>
+#include <__utility/as_const.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+concept __cpp17_move_constructible = is_move_constructible_v<_Tp>;
+
+template <class _Tp>
+concept __cpp17_copy_constructible = __cpp17_move_constructible<_Tp> && is_copy_constructible_v<_Tp>;
+
+template <class _Tp>
+concept __cpp17_move_assignable = requires(_Tp __lhs, _Tp __rhs) {
+  { __lhs = std::move(__rhs) } -> same_as<_Tp&>;
+};
+
+template <class _Tp>
+concept __cpp17_copy_assignable = __cpp17_move_assignable<_Tp> && requires(_Tp __lhs, _Tp __rhs) {
+  { __lhs = __rhs } -> same_as<_Tp&>;
+  { __lhs = std::as_const(__rhs) } -> same_as<_Tp&>;
+};
+
+template <class _Tp>
+concept __cpp17_destructible = requires(_Tp __v) { __v.~_Tp(); };
+
+template <class _Tp>
+concept __cpp17_equality_comparable = requires(_Tp __lhs, _Tp __rhs) {
+  { __lhs == __rhs } -> __boolean_testable;
+  { std::as_const(__lhs) == __rhs } -> __boolean_testable;
+  { __lhs == std::as_const(__rhs) } -> __boolean_testable;
+  { std::as_const(__lhs) == std::as_const(__rhs) } -> __boolean_testable;
+};
+
+template <class _Tp>
+concept __cpp17_default_constructible = is_default_constructible_v<_Tp>;
+
+template <class _Iter>
+concept __cpp17_iterator =
+    __cpp17_copy_constructible<_Iter> && __cpp17_copy_assignable<_Iter> && __cpp17_destructible<_Iter> &&
+    (is_signed_v<__iter_diff_t<_Iter>> || is_void_v<__iter_diff_t<_Iter>>) && requires(_Iter __iter) {
+      { *__iter };
+      { ++__iter } -> same_as<_Iter&>;
+    };
+
+template <class _Iter>
+concept __cpp17_input_iterator =
+    __cpp17_iterator<_Iter> && __cpp17_equality_comparable<_Iter> && requires(_Iter __lhs, _Iter __rhs) {
+      { __lhs != __rhs } -> __boolean_testable;
+      { std::as_const(__lhs) != __rhs } -> __boolean_testable;
+      { __lhs != std::as_const(__rhs) } -> __boolean_testable;
+      { std::as_const(__lhs) != std::as_const(__rhs) } -> __boolean_testable;
+
+      { *__lhs } -> same_as<__iter_reference<_Iter>>;
+      { *std::as_const(__lhs) } -> same_as<__iter_reference<_Iter>>;
+
+      { ++__lhs } -> same_as<_Iter&>;
+      { (void)__lhs++ };
+      { *__lhs++ };
+    };
+
+template <class _Iter, class _WriteTo>
+concept __cpp17_output_iterator = __cpp17_iterator<_Iter> && requires(_Iter __iter, _WriteTo __write) {
+  { *__iter = std::forward<_WriteTo>(__write) };
+  { ++__iter } -> same_as<_Iter&>;
+  { __iter++ } -> convertible_to<const _Iter&>;
+  { *__iter++ = std::forward<_WriteTo>(__write) };
+};
+
+template <class _Iter>
+concept __cpp17_forward_iterator =
+    __cpp17_input_iterator<_Iter> && __cpp17_default_constructible<_Iter> && requires(_Iter __iter) {
+      { __iter++ } -> convertible_to<const _Iter&>;
+      { *__iter++ } -> same_as<__iter_reference<_Iter>>;
+    };
+
+template <class _Iter>
+concept __cpp17_bidirectional_iterator = __cpp17_forward_iterator<_Iter> && requires(_Iter __iter) {
+  { --__iter } -> same_as<_Iter&>;
+  { __iter-- } -> convertible_to<const _Iter&>;
+  { *__iter-- } -> same_as<__iter_reference<_Iter>>;
+};
+
+template <class _Iter>
+concept __cpp17_random_access_iterator =
+    __cpp17_bidirectional_iterator<_Iter> && requires(_Iter __iter, __iter_diff_t<_Iter> __n) {
+      { __iter += __n } -> same_as<_Iter&>;
+
+      { __iter + __n } -> same_as<_Iter>;
+      { __n + __iter } -> same_as<_Iter>;
+      { std::as_const(__iter) + __n } -> same_as<_Iter>;
+      { __n + std::as_const(__iter) } -> same_as<_Iter>;
+
+      { __iter -= __n } -> same_as<_Iter&>;
+      { __iter - __n } -> same_as<_Iter>;
+      { std::as_const(__iter) - __n } -> same_as<_Iter>;
+
+      { __iter - __iter } -> same_as<__iter_diff_t<_Iter>>;
+      { std::as_const(__iter) - __iter } -> same_as<__iter_diff_t<_Iter>>;
+      { __iter - std::as_const(__iter) } -> same_as<__iter_diff_t<_Iter>>;
+      { std::as_const(__iter) - std::as_const(__iter) } -> same_as<__iter_diff_t<_Iter>>;
+
+      { __iter[__n] } -> convertible_to<__iter_reference<_Iter>>;
+      { std::as_const(__iter)[__n] } -> convertible_to<__iter_reference<_Iter>>;
+
+      { __iter < __iter } -> __boolean_testable;
+      { std::as_const(__iter) < __iter } -> __boolean_testable;
+      { __iter < std::as_const(__iter) } -> __boolean_testable;
+      { std::as_const(__iter) < std::as_const(__iter) } -> __boolean_testable;
+
+      { __iter > __iter } -> __boolean_testable;
+      { std::as_const(__iter) > __iter } -> __boolean_testable;
+      { __iter > std::as_const(__iter) } -> __boolean_testable;
+      { std::as_const(__iter) > std::as_const(__iter) } -> __boolean_testable;
+
+      { __iter >= __iter } -> __boolean_testable;
+      { std::as_const(__iter) >= __iter } -> __boolean_testable;
+      { __iter >= std::as_const(__iter) } -> __boolean_testable;
+      { std::as_const(__iter) >= std::as_const(__iter) } -> __boolean_testable;
+
+      { __iter <= __iter } -> __boolean_testable;
+      { std::as_const(__iter) <= __iter } -> __boolean_testable;
+      { __iter <= std::as_const(__iter) } -> __boolean_testable;
+      { std::as_const(__iter) <= std::as_const(__iter) } -> __boolean_testable;
+    };
+
+_LIBCPP_END_NAMESPACE_STD
+
+#  ifndef _LIBCPP_DISABLE_ITERATOR_CHECKS
+#    define _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(iter_t, message)                                                      \
+      static_assert(::std::__cpp17_input_iterator<iter_t>, message)
+#    define _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(iter_t, write_t, message)                                            \
+      static_assert(::std::__cpp17_output_iterator<iter_t, write_t>, message)
+#    define _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(iter_t, message)                                                    \
+      static_assert(::std::__cpp17_forward_iterator<iter_t>, message)
+#    define _LIBCPP_REQUIRE_CPP17_BIDIRECTIONAL_ITERATOR(iter_t, message)                                              \
+      static_assert(::std::__cpp17_bidirectional_iterator<iter_t>, message)
+#    define _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(iter_t, message)                                              \
+      static_assert(::std::__cpp17_random_access_iterator<iter_t>, message)
+#  else
+#    define _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(iter_t, message) static_assert(true)
+#    define _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(iter_t, write_t, message) static_assert(true)
+#    define _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(iter_t, message) static_assert(true)
+#    define _LIBCPP_REQUIRE_CPP17_BIDIRECTIONAL_ITERATOR(iter_t, message) static_assert(true)
+#    define _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(iter_t, message) static_assert(true)
+#  endif
+
+#else // _LIBCPP_STD_VER >= 20
+
+#  define _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(iter_t, message) static_assert(true)
+#  define _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(iter_t, write_t, message) static_assert(true)
+#  define _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(iter_t, message) static_assert(true)
+#  define _LIBCPP_REQUIRE_CPP17_BIDIRECTIONAL_ITERATOR(iter_t, message) static_assert(true)
+#  define _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(iter_t, message) static_assert(true)
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ITERATOR_CPP17_ITERATOR_CONCEPTS_H
diff --git a/libcxx/include/__cxx03/__iterator/data.h b/libcxx/include/__cxx03/__iterator/data.h
new file mode 100644
index 000000000000..b7c1603652b0
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/data.h
@@ -0,0 +1,49 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_DATA_H
+#define _LIBCPP___ITERATOR_DATA_H
+
+#include <__config>
+#include <cstddef>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _Cont>
+constexpr _LIBCPP_HIDE_FROM_ABI auto data(_Cont& __c) noexcept(noexcept(__c.data())) -> decltype(__c.data()) {
+  return __c.data();
+}
+
+template <class _Cont>
+constexpr _LIBCPP_HIDE_FROM_ABI auto data(const _Cont& __c) noexcept(noexcept(__c.data())) -> decltype(__c.data()) {
+  return __c.data();
+}
+
+template <class _Tp, size_t _Sz>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp* data(_Tp (&__array)[_Sz]) noexcept {
+  return __array;
+}
+
+template <class _Ep>
+_LIBCPP_HIDE_FROM_ABI constexpr const _Ep* data(initializer_list<_Ep> __il) noexcept {
+  return __il.begin();
+}
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_DATA_H
diff --git a/libcxx/include/__cxx03/__iterator/default_sentinel.h b/libcxx/include/__cxx03/__iterator/default_sentinel.h
new file mode 100644
index 000000000000..3b65f442f1a8
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/default_sentinel.h
@@ -0,0 +1,30 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_DEFAULT_SENTINEL_H
+#define _LIBCPP___ITERATOR_DEFAULT_SENTINEL_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+struct default_sentinel_t {};
+inline constexpr default_sentinel_t default_sentinel{};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_DEFAULT_SENTINEL_H
diff --git a/libcxx/include/__cxx03/__iterator/distance.h b/libcxx/include/__cxx03/__iterator/distance.h
new file mode 100644
index 000000000000..75bd49c9ae73
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/distance.h
@@ -0,0 +1,99 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_DISTANCE_H
+#define _LIBCPP___ITERATOR_DISTANCE_H
+
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/size.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/remove_cvref.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIter>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_InputIter>::difference_type
+__distance(_InputIter __first, _InputIter __last, input_iterator_tag) {
+  typename iterator_traits<_InputIter>::difference_type __r(0);
+  for (; __first != __last; ++__first)
+    ++__r;
+  return __r;
+}
+
+template <class _RandIter>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_RandIter>::difference_type
+__distance(_RandIter __first, _RandIter __last, random_access_iterator_tag) {
+  return __last - __first;
+}
+
+template <class _InputIter>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_InputIter>::difference_type
+distance(_InputIter __first, _InputIter __last) {
+  return std::__distance(__first, __last, typename iterator_traits<_InputIter>::iterator_category());
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+// [range.iter.op.distance]
+
+namespace ranges {
+namespace __distance {
+
+struct __fn {
+  template <class _Ip, sentinel_for<_Ip> _Sp>
+    requires(!sized_sentinel_for<_Sp, _Ip>)
+  _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Ip> operator()(_Ip __first, _Sp __last) const {
+    iter_difference_t<_Ip> __n = 0;
+    while (__first != __last) {
+      ++__first;
+      ++__n;
+    }
+    return __n;
+  }
+
+  template <class _Ip, sized_sentinel_for<decay_t<_Ip>> _Sp>
+  _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Ip> operator()(_Ip&& __first, _Sp __last) const {
+    if constexpr (sized_sentinel_for<_Sp, __remove_cvref_t<_Ip>>) {
+      return __last - __first;
+    } else {
+      return __last - decay_t<_Ip>(__first);
+    }
+  }
+
+  template <range _Rp>
+  _LIBCPP_HIDE_FROM_ABI constexpr range_difference_t<_Rp> operator()(_Rp&& __r) const {
+    if constexpr (sized_range<_Rp>) {
+      return static_cast<range_difference_t<_Rp>>(ranges::size(__r));
+    } else {
+      return operator()(ranges::begin(__r), ranges::end(__r));
+    }
+  }
+};
+
+} // namespace __distance
+
+inline namespace __cpo {
+inline constexpr auto distance = __distance::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_DISTANCE_H
diff --git a/libcxx/include/__cxx03/__iterator/empty.h b/libcxx/include/__cxx03/__iterator/empty.h
new file mode 100644
index 000000000000..773f2776955b
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/empty.h
@@ -0,0 +1,45 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_EMPTY_H
+#define _LIBCPP___ITERATOR_EMPTY_H
+
+#include <__config>
+#include <cstddef>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _Cont>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto
+empty(const _Cont& __c) noexcept(noexcept(__c.empty())) -> decltype(__c.empty()) {
+  return __c.empty();
+}
+
+template <class _Tp, size_t _Sz>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool empty(const _Tp (&)[_Sz]) noexcept {
+  return false;
+}
+
+template <class _Ep>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool empty(initializer_list<_Ep> __il) noexcept {
+  return __il.size() == 0;
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_EMPTY_H
diff --git a/libcxx/include/__cxx03/__iterator/erase_if_container.h b/libcxx/include/__cxx03/__iterator/erase_if_container.h
new file mode 100644
index 000000000000..0f87f50cd1c1
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/erase_if_container.h
@@ -0,0 +1,43 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_ERASE_IF_CONTAINER_H
+#define _LIBCPP___ITERATOR_ERASE_IF_CONTAINER_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Container, class _Predicate>
+_LIBCPP_HIDE_FROM_ABI typename _Container::size_type __libcpp_erase_if_container(_Container& __c, _Predicate& __pred) {
+  typename _Container::size_type __old_size = __c.size();
+
+  const typename _Container::iterator __last = __c.end();
+  for (typename _Container::iterator __iter = __c.begin(); __iter != __last;) {
+    if (__pred(*__iter))
+      __iter = __c.erase(__iter);
+    else
+      ++__iter;
+  }
+
+  return __old_size - __c.size();
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ITERATOR_ERASE_IF_CONTAINER_H
diff --git a/libcxx/include/__cxx03/__iterator/front_insert_iterator.h b/libcxx/include/__cxx03/__iterator/front_insert_iterator.h
new file mode 100644
index 000000000000..7f2c54ec8744
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/front_insert_iterator.h
@@ -0,0 +1,83 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_FRONT_INSERT_ITERATOR_H
+#define _LIBCPP___ITERATOR_FRONT_INSERT_ITERATOR_H
+
+#include <__config>
+#include <__iterator/iterator.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__utility/move.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Container>
+class _LIBCPP_TEMPLATE_VIS front_insert_iterator
+#if _LIBCPP_STD_VER <= 14 || !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
+    : public iterator<output_iterator_tag, void, void, void, void>
+#endif
+{
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  _Container* container;
+
+public:
+  typedef output_iterator_tag iterator_category;
+  typedef void value_type;
+#if _LIBCPP_STD_VER >= 20
+  typedef ptrdiff_t difference_type;
+#else
+  typedef void difference_type;
+#endif
+  typedef void pointer;
+  typedef void reference;
+  typedef _Container container_type;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit front_insert_iterator(_Container& __x)
+      : container(std::addressof(__x)) {}
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator&
+  operator=(const typename _Container::value_type& __value) {
+    container->push_front(__value);
+    return *this;
+  }
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator&
+  operator=(typename _Container::value_type&& __value) {
+    container->push_front(std::move(__value));
+    return *this;
+  }
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator& operator*() { return *this; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator& operator++() { return *this; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator operator++(int) { return *this; }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(front_insert_iterator);
+
+template <class _Container>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator<_Container>
+front_inserter(_Container& __x) {
+  return front_insert_iterator<_Container>(__x);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ITERATOR_FRONT_INSERT_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/incrementable_traits.h b/libcxx/include/__cxx03/__iterator/incrementable_traits.h
new file mode 100644
index 000000000000..a228b228f6e5
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/incrementable_traits.h
@@ -0,0 +1,79 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_INCREMENTABLE_TRAITS_H
+#define _LIBCPP___ITERATOR_INCREMENTABLE_TRAITS_H
+
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <__type_traits/conditional.h>
+#include <__type_traits/is_object.h>
+#include <__type_traits/is_primary_template.h>
+#include <__type_traits/make_signed.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/declval.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [incrementable.traits]
+template <class>
+struct incrementable_traits {};
+
+template <class _Tp>
+  requires is_object_v<_Tp>
+struct incrementable_traits<_Tp*> {
+  using difference_type = ptrdiff_t;
+};
+
+template <class _Ip>
+struct incrementable_traits<const _Ip> : incrementable_traits<_Ip> {};
+
+template <class _Tp>
+concept __has_member_difference_type = requires { typename _Tp::difference_type; };
+
+template <__has_member_difference_type _Tp>
+struct incrementable_traits<_Tp> {
+  using difference_type = typename _Tp::difference_type;
+};
+
+template <class _Tp>
+concept __has_integral_minus = requires(const _Tp& __x, const _Tp& __y) {
+  { __x - __y } -> integral;
+};
+
+template <__has_integral_minus _Tp>
+  requires(!__has_member_difference_type<_Tp>)
+struct incrementable_traits<_Tp> {
+  using difference_type = make_signed_t<decltype(std::declval<_Tp>() - std::declval<_Tp>())>;
+};
+
+template <class>
+struct iterator_traits;
+
+// Let `RI` be `remove_cvref_t<I>`. The type `iter_difference_t<I>` denotes
+// `incrementable_traits<RI>::difference_type` if `iterator_traits<RI>` names a specialization
+// generated from the primary template, and `iterator_traits<RI>::difference_type` otherwise.
+template <class _Ip>
+using iter_difference_t =
+    typename conditional_t<__is_primary_template<iterator_traits<remove_cvref_t<_Ip> > >::value,
+                           incrementable_traits<remove_cvref_t<_Ip> >,
+                           iterator_traits<remove_cvref_t<_Ip> > >::difference_type;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_INCREMENTABLE_TRAITS_H
diff --git a/libcxx/include/__cxx03/__iterator/indirectly_comparable.h b/libcxx/include/__cxx03/__iterator/indirectly_comparable.h
new file mode 100644
index 000000000000..e8a7398bacd2
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/indirectly_comparable.h
@@ -0,0 +1,33 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_INDIRECTLY_COMPARABLE_H
+#define _LIBCPP___ITERATOR_INDIRECTLY_COMPARABLE_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _I1, class _I2, class _Rp, class _P1 = identity, class _P2 = identity>
+concept indirectly_comparable = indirect_binary_predicate<_Rp, projected<_I1, _P1>, projected<_I2, _P2>>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_INDIRECTLY_COMPARABLE_H
diff --git a/libcxx/include/__cxx03/__iterator/insert_iterator.h b/libcxx/include/__cxx03/__iterator/insert_iterator.h
new file mode 100644
index 000000000000..8b7574dc9ec0
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/insert_iterator.h
@@ -0,0 +1,95 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_INSERT_ITERATOR_H
+#define _LIBCPP___ITERATOR_INSERT_ITERATOR_H
+
+#include <__config>
+#include <__iterator/iterator.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__ranges/access.h>
+#include <__utility/move.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Container>
+using __insert_iterator_iter_t = ranges::iterator_t<_Container>;
+#else
+template <class _Container>
+using __insert_iterator_iter_t = typename _Container::iterator;
+#endif
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Container>
+class _LIBCPP_TEMPLATE_VIS insert_iterator
+#if _LIBCPP_STD_VER <= 14 || !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
+    : public iterator<output_iterator_tag, void, void, void, void>
+#endif
+{
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  _Container* container;
+  __insert_iterator_iter_t<_Container> iter;
+
+public:
+  typedef output_iterator_tag iterator_category;
+  typedef void value_type;
+#if _LIBCPP_STD_VER >= 20
+  typedef ptrdiff_t difference_type;
+#else
+  typedef void difference_type;
+#endif
+  typedef void pointer;
+  typedef void reference;
+  typedef _Container container_type;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  insert_iterator(_Container& __x, __insert_iterator_iter_t<_Container> __i)
+      : container(std::addressof(__x)), iter(__i) {}
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator&
+  operator=(const typename _Container::value_type& __value) {
+    iter = container->insert(iter, __value);
+    ++iter;
+    return *this;
+  }
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator&
+  operator=(typename _Container::value_type&& __value) {
+    iter = container->insert(iter, std::move(__value));
+    ++iter;
+    return *this;
+  }
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator& operator*() { return *this; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator& operator++() { return *this; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator& operator++(int) { return *this; }
+};
+
+template <class _Container>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator<_Container>
+inserter(_Container& __x, __insert_iterator_iter_t<_Container> __i) {
+  return insert_iterator<_Container>(__x, __i);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ITERATOR_INSERT_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/istream_iterator.h b/libcxx/include/__cxx03/__iterator/istream_iterator.h
new file mode 100644
index 000000000000..58c9ac6d4ccc
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/istream_iterator.h
@@ -0,0 +1,101 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_ISTREAM_ITERATOR_H
+#define _LIBCPP___ITERATOR_ISTREAM_ITERATOR_H
+
+#include <__config>
+#include <__fwd/istream.h>
+#include <__fwd/string.h>
+#include <__iterator/default_sentinel.h>
+#include <__iterator/iterator.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Tp, class _CharT = char, class _Traits = char_traits<_CharT>, class _Distance = ptrdiff_t>
+class _LIBCPP_TEMPLATE_VIS istream_iterator
+#if _LIBCPP_STD_VER <= 14 || !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
+    : public iterator<input_iterator_tag, _Tp, _Distance, const _Tp*, const _Tp&>
+#endif
+{
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+public:
+  typedef input_iterator_tag iterator_category;
+  typedef _Tp value_type;
+  typedef _Distance difference_type;
+  typedef const _Tp* pointer;
+  typedef const _Tp& reference;
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef basic_istream<_CharT, _Traits> istream_type;
+
+private:
+  istream_type* __in_stream_;
+  _Tp __value_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR istream_iterator() : __in_stream_(nullptr), __value_() {}
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI constexpr istream_iterator(default_sentinel_t) : istream_iterator() {}
+#endif // _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI istream_iterator(istream_type& __s) : __in_stream_(std::addressof(__s)) {
+    if (!(*__in_stream_ >> __value_))
+      __in_stream_ = nullptr;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const _Tp& operator*() const { return __value_; }
+  _LIBCPP_HIDE_FROM_ABI const _Tp* operator->() const { return std::addressof((operator*())); }
+  _LIBCPP_HIDE_FROM_ABI istream_iterator& operator++() {
+    if (!(*__in_stream_ >> __value_))
+      __in_stream_ = nullptr;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI istream_iterator operator++(int) {
+    istream_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  template <class _Up, class _CharU, class _TraitsU, class _DistanceU>
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const istream_iterator<_Up, _CharU, _TraitsU, _DistanceU>& __x,
+                                               const istream_iterator<_Up, _CharU, _TraitsU, _DistanceU>& __y);
+
+#if _LIBCPP_STD_VER >= 20
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const istream_iterator& __i, default_sentinel_t) {
+    return __i.__in_stream_ == nullptr;
+  }
+#endif // _LIBCPP_STD_VER >= 20
+};
+
+template <class _Tp, class _CharT, class _Traits, class _Distance>
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __x,
+                                             const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __y) {
+  return __x.__in_stream_ == __y.__in_stream_;
+}
+
+#if _LIBCPP_STD_VER <= 17
+template <class _Tp, class _CharT, class _Traits, class _Distance>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __x,
+                                             const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __y) {
+  return !(__x == __y);
+}
+#endif // _LIBCPP_STD_VER <= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_ISTREAM_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/istreambuf_iterator.h b/libcxx/include/__cxx03/__iterator/istreambuf_iterator.h
new file mode 100644
index 000000000000..51c4ecff351f
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/istreambuf_iterator.h
@@ -0,0 +1,109 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_ISTREAMBUF_ITERATOR_H
+#define _LIBCPP___ITERATOR_ISTREAMBUF_ITERATOR_H
+
+#include <__config>
+#include <__fwd/istream.h>
+#include <__fwd/streambuf.h>
+#include <__iterator/default_sentinel.h>
+#include <__iterator/iterator.h>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS istreambuf_iterator
+#if _LIBCPP_STD_VER <= 14 || !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
+    : public iterator<input_iterator_tag, _CharT, typename _Traits::off_type, _CharT*, _CharT>
+#endif
+{
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+public:
+  typedef input_iterator_tag iterator_category;
+  typedef _CharT value_type;
+  typedef typename _Traits::off_type difference_type;
+  typedef _CharT* pointer;
+  typedef _CharT reference;
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef typename _Traits::int_type int_type;
+  typedef basic_streambuf<_CharT, _Traits> streambuf_type;
+  typedef basic_istream<_CharT, _Traits> istream_type;
+
+private:
+  mutable streambuf_type* __sbuf_;
+
+  class __proxy {
+    char_type __keep_;
+    streambuf_type* __sbuf_;
+    _LIBCPP_HIDE_FROM_ABI explicit __proxy(char_type __c, streambuf_type* __s) : __keep_(__c), __sbuf_(__s) {}
+    friend class istreambuf_iterator;
+
+  public:
+    _LIBCPP_HIDE_FROM_ABI char_type operator*() const { return __keep_; }
+  };
+
+  _LIBCPP_HIDE_FROM_ABI bool __test_for_eof() const {
+    if (__sbuf_ && traits_type::eq_int_type(__sbuf_->sgetc(), traits_type::eof()))
+      __sbuf_ = nullptr;
+    return __sbuf_ == nullptr;
+  }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR istreambuf_iterator() _NOEXCEPT : __sbuf_(nullptr) {}
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI constexpr istreambuf_iterator(default_sentinel_t) noexcept : istreambuf_iterator() {}
+#endif // _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI istreambuf_iterator(istream_type& __s) _NOEXCEPT : __sbuf_(__s.rdbuf()) {}
+  _LIBCPP_HIDE_FROM_ABI istreambuf_iterator(streambuf_type* __s) _NOEXCEPT : __sbuf_(__s) {}
+  _LIBCPP_HIDE_FROM_ABI istreambuf_iterator(const __proxy& __p) _NOEXCEPT : __sbuf_(__p.__sbuf_) {}
+
+  _LIBCPP_HIDE_FROM_ABI char_type operator*() const { return static_cast<char_type>(__sbuf_->sgetc()); }
+  _LIBCPP_HIDE_FROM_ABI istreambuf_iterator& operator++() {
+    __sbuf_->sbumpc();
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __proxy operator++(int) { return __proxy(__sbuf_->sbumpc(), __sbuf_); }
+
+  _LIBCPP_HIDE_FROM_ABI bool equal(const istreambuf_iterator& __b) const {
+    return __test_for_eof() == __b.__test_for_eof();
+  }
+
+#if _LIBCPP_STD_VER >= 20
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const istreambuf_iterator& __i, default_sentinel_t) {
+    return __i.__test_for_eof();
+  }
+#endif // _LIBCPP_STD_VER >= 20
+};
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const istreambuf_iterator<_CharT, _Traits>& __a, const istreambuf_iterator<_CharT, _Traits>& __b) {
+  return __a.equal(__b);
+}
+
+#if _LIBCPP_STD_VER <= 17
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const istreambuf_iterator<_CharT, _Traits>& __a, const istreambuf_iterator<_CharT, _Traits>& __b) {
+  return !__a.equal(__b);
+}
+#endif // _LIBCPP_STD_VER <= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_ISTREAMBUF_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/iter_move.h b/libcxx/include/__cxx03/__iterator/iter_move.h
new file mode 100644
index 000000000000..ba8aed3c0ffb
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/iter_move.h
@@ -0,0 +1,103 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_ITER_MOVE_H
+#define _LIBCPP___ITERATOR_ITER_MOVE_H
+
+#include <__concepts/class_or_enum.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [iterator.cust.move]
+
+namespace ranges {
+namespace __iter_move {
+
+void iter_move() = delete;
+
+template <class _Tp>
+concept __unqualified_iter_move = __class_or_enum<remove_cvref_t<_Tp>> && requires(_Tp&& __t) {
+  // NOLINTNEXTLINE(libcpp-robust-against-adl) iter_swap ADL calls should only be made through ranges::iter_swap
+  iter_move(std::forward<_Tp>(__t));
+};
+
+template <class _Tp>
+concept __move_deref = !__unqualified_iter_move<_Tp> && requires(_Tp&& __t) {
+  *__t;
+  requires is_lvalue_reference_v<decltype(*__t)>;
+};
+
+template <class _Tp>
+concept __just_deref = !__unqualified_iter_move<_Tp> && !__move_deref<_Tp> && requires(_Tp&& __t) {
+  *__t;
+  requires(!is_lvalue_reference_v<decltype(*__t)>);
+};
+
+// [iterator.cust.move]
+
+struct __fn {
+  // NOLINTBEGIN(libcpp-robust-against-adl) iter_move ADL calls should only be made through ranges::iter_move
+  template <class _Ip>
+    requires __unqualified_iter_move<_Ip>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator()(_Ip&& __i) const
+      noexcept(noexcept(iter_move(std::forward<_Ip>(__i)))) {
+    return iter_move(std::forward<_Ip>(__i));
+  }
+  // NOLINTEND(libcpp-robust-against-adl)
+
+  template <class _Ip>
+    requires __move_deref<_Ip>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Ip&& __i) const
+      noexcept(noexcept(std::move(*std::forward<_Ip>(__i)))) -> decltype(std::move(*std::forward<_Ip>(__i))) {
+    return std::move(*std::forward<_Ip>(__i));
+  }
+
+  template <class _Ip>
+    requires __just_deref<_Ip>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Ip&& __i) const
+      noexcept(noexcept(*std::forward<_Ip>(__i))) -> decltype(*std::forward<_Ip>(__i)) {
+    return *std::forward<_Ip>(__i);
+  }
+};
+} // namespace __iter_move
+
+inline namespace __cpo {
+inline constexpr auto iter_move = __iter_move::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+template <__dereferenceable _Tp>
+  requires requires(_Tp& __t) {
+    { ranges::iter_move(__t) } -> __can_reference;
+  }
+using iter_rvalue_reference_t = decltype(ranges::iter_move(std::declval<_Tp&>()));
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ITERATOR_ITER_MOVE_H
diff --git a/libcxx/include/__cxx03/__iterator/iter_swap.h b/libcxx/include/__cxx03/__iterator/iter_swap.h
new file mode 100644
index 000000000000..01ab1b97d650
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/iter_swap.h
@@ -0,0 +1,108 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_ITER_SWAP_H
+#define _LIBCPP___ITERATOR_ITER_SWAP_H
+
+#include <__concepts/class_or_enum.h>
+#include <__concepts/swappable.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/readable_traits.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [iter.cust.swap]
+
+namespace ranges {
+namespace __iter_swap {
+template <class _I1, class _I2>
+void iter_swap(_I1, _I2) = delete;
+
+template <class _T1, class _T2>
+concept __unqualified_iter_swap =
+    (__class_or_enum<remove_cvref_t<_T1>> || __class_or_enum<remove_cvref_t<_T2>>) && requires(_T1&& __x, _T2&& __y) {
+      // NOLINTNEXTLINE(libcpp-robust-against-adl) iter_swap ADL calls should only be made through ranges::iter_swap
+      iter_swap(std::forward<_T1>(__x), std::forward<_T2>(__y));
+    };
+
+template <class _T1, class _T2>
+concept __readable_swappable =
+    indirectly_readable<_T1> && indirectly_readable<_T2> &&
+    swappable_with<iter_reference_t<_T1>, iter_reference_t<_T2>>;
+
+struct __fn {
+  // NOLINTBEGIN(libcpp-robust-against-adl) iter_swap ADL calls should only be made through ranges::iter_swap
+  template <class _T1, class _T2>
+    requires __unqualified_iter_swap<_T1, _T2>
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_T1&& __x, _T2&& __y) const
+      noexcept(noexcept(iter_swap(std::forward<_T1>(__x), std::forward<_T2>(__y)))) {
+    (void)iter_swap(std::forward<_T1>(__x), std::forward<_T2>(__y));
+  }
+  // NOLINTEND(libcpp-robust-against-adl)
+
+  template <class _T1, class _T2>
+    requires(!__unqualified_iter_swap<_T1, _T2>) && __readable_swappable<_T1, _T2>
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_T1&& __x, _T2&& __y) const
+      noexcept(noexcept(ranges::swap(*std::forward<_T1>(__x), *std::forward<_T2>(__y)))) {
+    ranges::swap(*std::forward<_T1>(__x), *std::forward<_T2>(__y));
+  }
+
+  template <class _T1, class _T2>
+    requires(!__unqualified_iter_swap<_T1, _T2> &&   //
+             !__readable_swappable<_T1, _T2>) &&     //
+            indirectly_movable_storable<_T1, _T2> && //
+            indirectly_movable_storable<_T2, _T1>
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_T1&& __x, _T2&& __y) const
+      noexcept(noexcept(iter_value_t<_T2>(ranges::iter_move(__y))) && //
+               noexcept(*__y = ranges::iter_move(__x)) &&             //
+               noexcept(*std::forward<_T1>(__x) = std::declval<iter_value_t<_T2>>())) {
+    iter_value_t<_T2> __old(ranges::iter_move(__y));
+    *__y                    = ranges::iter_move(__x);
+    *std::forward<_T1>(__x) = std::move(__old);
+  }
+};
+} // namespace __iter_swap
+
+inline namespace __cpo {
+inline constexpr auto iter_swap = __iter_swap::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+template <class _I1, class _I2 = _I1>
+concept indirectly_swappable =
+    indirectly_readable<_I1> && indirectly_readable<_I2> && requires(const _I1 __i1, const _I2 __i2) {
+      ranges::iter_swap(__i1, __i1);
+      ranges::iter_swap(__i2, __i2);
+      ranges::iter_swap(__i1, __i2);
+      ranges::iter_swap(__i2, __i1);
+    };
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ITERATOR_ITER_SWAP_H
diff --git a/libcxx/include/__cxx03/__iterator/iterator.h b/libcxx/include/__cxx03/__iterator/iterator.h
new file mode 100644
index 000000000000..ba9308f3c224
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/iterator.h
@@ -0,0 +1,33 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_ITERATOR_H
+#define _LIBCPP___ITERATOR_ITERATOR_H
+
+#include <__config>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Category, class _Tp, class _Distance = ptrdiff_t, class _Pointer = _Tp*, class _Reference = _Tp&>
+struct _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 iterator {
+  typedef _Tp value_type;
+  typedef _Distance difference_type;
+  typedef _Pointer pointer;
+  typedef _Reference reference;
+  typedef _Category iterator_category;
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/iterator_traits.h b/libcxx/include/__cxx03/__iterator/iterator_traits.h
new file mode 100644
index 000000000000..11af9e301842
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/iterator_traits.h
@@ -0,0 +1,528 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_ITERATOR_TRAITS_H
+#define _LIBCPP___ITERATOR_ITERATOR_TRAITS_H
+
+#include <__concepts/arithmetic.h>
+#include <__concepts/constructible.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/copyable.h>
+#include <__concepts/equality_comparable.h>
+#include <__concepts/same_as.h>
+#include <__concepts/totally_ordered.h>
+#include <__config>
+#include <__fwd/pair.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/readable_traits.h>
+#include <__type_traits/common_reference.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/disjunction.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_object.h>
+#include <__type_traits/is_primary_template.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/is_valid_expansion.h>
+#include <__type_traits/remove_const.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+using __with_reference = _Tp&;
+
+template <class _Tp>
+concept __can_reference = requires { typename __with_reference<_Tp>; };
+
+template <class _Tp>
+concept __dereferenceable = requires(_Tp& __t) {
+  { *__t } -> __can_reference; // not required to be equality-preserving
+};
+
+// [iterator.traits]
+template <__dereferenceable _Tp>
+using iter_reference_t = decltype(*std::declval<_Tp&>());
+
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _Iter>
+struct _LIBCPP_TEMPLATE_VIS iterator_traits;
+
+struct _LIBCPP_TEMPLATE_VIS input_iterator_tag {};
+struct _LIBCPP_TEMPLATE_VIS output_iterator_tag {};
+struct _LIBCPP_TEMPLATE_VIS forward_iterator_tag : public input_iterator_tag {};
+struct _LIBCPP_TEMPLATE_VIS bidirectional_iterator_tag : public forward_iterator_tag {};
+struct _LIBCPP_TEMPLATE_VIS random_access_iterator_tag : public bidirectional_iterator_tag {};
+#if _LIBCPP_STD_VER >= 20
+struct _LIBCPP_TEMPLATE_VIS contiguous_iterator_tag : public random_access_iterator_tag {};
+#endif
+
+template <class _Iter>
+struct __iter_traits_cache {
+  using type = _If< __is_primary_template<iterator_traits<_Iter> >::value, _Iter, iterator_traits<_Iter> >;
+};
+template <class _Iter>
+using _ITER_TRAITS = typename __iter_traits_cache<_Iter>::type;
+
+struct __iter_concept_concept_test {
+  template <class _Iter>
+  using _Apply = typename _ITER_TRAITS<_Iter>::iterator_concept;
+};
+struct __iter_concept_category_test {
+  template <class _Iter>
+  using _Apply = typename _ITER_TRAITS<_Iter>::iterator_category;
+};
+struct __iter_concept_random_fallback {
+  template <class _Iter>
+  using _Apply = __enable_if_t< __is_primary_template<iterator_traits<_Iter> >::value, random_access_iterator_tag >;
+};
+
+template <class _Iter, class _Tester>
+struct __test_iter_concept : _IsValidExpansion<_Tester::template _Apply, _Iter>, _Tester {};
+
+template <class _Iter>
+struct __iter_concept_cache {
+  using type = _Or< __test_iter_concept<_Iter, __iter_concept_concept_test>,
+                    __test_iter_concept<_Iter, __iter_concept_category_test>,
+                    __test_iter_concept<_Iter, __iter_concept_random_fallback> >;
+};
+
+template <class _Iter>
+using _ITER_CONCEPT = typename __iter_concept_cache<_Iter>::type::template _Apply<_Iter>;
+
+template <class _Tp>
+struct __has_iterator_typedefs {
+private:
+  template <class _Up>
+  static false_type __test(...);
+  template <class _Up>
+  static true_type
+  __test(__void_t<typename _Up::iterator_category>* = nullptr,
+         __void_t<typename _Up::difference_type>*   = nullptr,
+         __void_t<typename _Up::value_type>*        = nullptr,
+         __void_t<typename _Up::reference>*         = nullptr,
+         __void_t<typename _Up::pointer>*           = nullptr);
+
+public:
+  static const bool value = decltype(__test<_Tp>(nullptr, nullptr, nullptr, nullptr, nullptr))::value;
+};
+
+template <class _Tp>
+struct __has_iterator_category {
+private:
+  template <class _Up>
+  static false_type __test(...);
+  template <class _Up>
+  static true_type __test(typename _Up::iterator_category* = nullptr);
+
+public:
+  static const bool value = decltype(__test<_Tp>(nullptr))::value;
+};
+
+template <class _Tp>
+struct __has_iterator_concept {
+private:
+  template <class _Up>
+  static false_type __test(...);
+  template <class _Up>
+  static true_type __test(typename _Up::iterator_concept* = nullptr);
+
+public:
+  static const bool value = decltype(__test<_Tp>(nullptr))::value;
+};
+
+#if _LIBCPP_STD_VER >= 20
+
+// The `cpp17-*-iterator` exposition-only concepts have very similar names to the `Cpp17*Iterator` named requirements
+// from `[iterator.cpp17]`. To avoid confusion between the two, the exposition-only concepts have been banished to
+// a "detail" namespace indicating they have a niche use-case.
+namespace __iterator_traits_detail {
+template <class _Ip>
+concept __cpp17_iterator = requires(_Ip __i) {
+  { *__i } -> __can_reference;
+  { ++__i } -> same_as<_Ip&>;
+  { *__i++ } -> __can_reference;
+} && copyable<_Ip>;
+
+template <class _Ip>
+concept __cpp17_input_iterator = __cpp17_iterator<_Ip> && equality_comparable<_Ip> && requires(_Ip __i) {
+  typename incrementable_traits<_Ip>::difference_type;
+  typename indirectly_readable_traits<_Ip>::value_type;
+  typename common_reference_t<iter_reference_t<_Ip>&&, typename indirectly_readable_traits<_Ip>::value_type&>;
+  typename common_reference_t<decltype(*__i++)&&, typename indirectly_readable_traits<_Ip>::value_type&>;
+  requires signed_integral<typename incrementable_traits<_Ip>::difference_type>;
+};
+
+template <class _Ip>
+concept __cpp17_forward_iterator =
+    __cpp17_input_iterator<_Ip> && constructible_from<_Ip> && is_reference_v<iter_reference_t<_Ip>> &&
+    same_as<remove_cvref_t<iter_reference_t<_Ip>>, typename indirectly_readable_traits<_Ip>::value_type> &&
+    requires(_Ip __i) {
+      { __i++ } -> convertible_to<_Ip const&>;
+      { *__i++ } -> same_as<iter_reference_t<_Ip>>;
+    };
+
+template <class _Ip>
+concept __cpp17_bidirectional_iterator = __cpp17_forward_iterator<_Ip> && requires(_Ip __i) {
+  { --__i } -> same_as<_Ip&>;
+  { __i-- } -> convertible_to<_Ip const&>;
+  { *__i-- } -> same_as<iter_reference_t<_Ip>>;
+};
+
+template <class _Ip>
+concept __cpp17_random_access_iterator =
+    __cpp17_bidirectional_iterator<_Ip> && totally_ordered<_Ip> &&
+    requires(_Ip __i, typename incrementable_traits<_Ip>::difference_type __n) {
+      { __i += __n } -> same_as<_Ip&>;
+      { __i -= __n } -> same_as<_Ip&>;
+      { __i + __n } -> same_as<_Ip>;
+      { __n + __i } -> same_as<_Ip>;
+      { __i - __n } -> same_as<_Ip>;
+      { __i - __i } -> same_as<decltype(__n)>; // NOLINT(misc-redundant-expression) ; This is llvm.org/PR54114
+      { __i[__n] } -> convertible_to<iter_reference_t<_Ip>>;
+    };
+} // namespace __iterator_traits_detail
+
+template <class _Ip>
+concept __has_member_reference = requires { typename _Ip::reference; };
+
+template <class _Ip>
+concept __has_member_pointer = requires { typename _Ip::pointer; };
+
+template <class _Ip>
+concept __has_member_iterator_category = requires { typename _Ip::iterator_category; };
+
+template <class _Ip>
+concept __specifies_members = requires {
+  typename _Ip::value_type;
+  typename _Ip::difference_type;
+  requires __has_member_reference<_Ip>;
+  requires __has_member_iterator_category<_Ip>;
+};
+
+template <class>
+struct __iterator_traits_member_pointer_or_void {
+  using type = void;
+};
+
+template <__has_member_pointer _Tp>
+struct __iterator_traits_member_pointer_or_void<_Tp> {
+  using type = typename _Tp::pointer;
+};
+
+template <class _Tp>
+concept __cpp17_iterator_missing_members = !__specifies_members<_Tp> && __iterator_traits_detail::__cpp17_iterator<_Tp>;
+
+template <class _Tp>
+concept __cpp17_input_iterator_missing_members =
+    __cpp17_iterator_missing_members<_Tp> && __iterator_traits_detail::__cpp17_input_iterator<_Tp>;
+
+// Otherwise, `pointer` names `void`.
+template <class>
+struct __iterator_traits_member_pointer_or_arrow_or_void {
+  using type = void;
+};
+
+// [iterator.traits]/3.2.1
+// If the qualified-id `I::pointer` is valid and denotes a type, `pointer` names that type.
+template <__has_member_pointer _Ip>
+struct __iterator_traits_member_pointer_or_arrow_or_void<_Ip> {
+  using type = typename _Ip::pointer;
+};
+
+// Otherwise, if `decltype(declval<I&>().operator->())` is well-formed, then `pointer` names that
+// type.
+template <class _Ip>
+  requires requires(_Ip& __i) { __i.operator->(); } && (!__has_member_pointer<_Ip>)
+struct __iterator_traits_member_pointer_or_arrow_or_void<_Ip> {
+  using type = decltype(std::declval<_Ip&>().operator->());
+};
+
+// Otherwise, `reference` names `iter-reference-t<I>`.
+template <class _Ip>
+struct __iterator_traits_member_reference {
+  using type = iter_reference_t<_Ip>;
+};
+
+// [iterator.traits]/3.2.2
+// If the qualified-id `I::reference` is valid and denotes a type, `reference` names that type.
+template <__has_member_reference _Ip>
+struct __iterator_traits_member_reference<_Ip> {
+  using type = typename _Ip::reference;
+};
+
+// [iterator.traits]/3.2.3.4
+// input_iterator_tag
+template <class _Ip>
+struct __deduce_iterator_category {
+  using type = input_iterator_tag;
+};
+
+// [iterator.traits]/3.2.3.1
+// `random_access_iterator_tag` if `I` satisfies `cpp17-random-access-iterator`, or otherwise
+template <__iterator_traits_detail::__cpp17_random_access_iterator _Ip>
+struct __deduce_iterator_category<_Ip> {
+  using type = random_access_iterator_tag;
+};
+
+// [iterator.traits]/3.2.3.2
+// `bidirectional_iterator_tag` if `I` satisfies `cpp17-bidirectional-iterator`, or otherwise
+template <__iterator_traits_detail::__cpp17_bidirectional_iterator _Ip>
+struct __deduce_iterator_category<_Ip> {
+  using type = bidirectional_iterator_tag;
+};
+
+// [iterator.traits]/3.2.3.3
+// `forward_iterator_tag` if `I` satisfies `cpp17-forward-iterator`, or otherwise
+template <__iterator_traits_detail::__cpp17_forward_iterator _Ip>
+struct __deduce_iterator_category<_Ip> {
+  using type = forward_iterator_tag;
+};
+
+template <class _Ip>
+struct __iterator_traits_iterator_category : __deduce_iterator_category<_Ip> {};
+
+// [iterator.traits]/3.2.3
+// If the qualified-id `I::iterator-category` is valid and denotes a type, `iterator-category` names
+// that type.
+template <__has_member_iterator_category _Ip>
+struct __iterator_traits_iterator_category<_Ip> {
+  using type = typename _Ip::iterator_category;
+};
+
+// otherwise, it names void.
+template <class>
+struct __iterator_traits_difference_type {
+  using type = void;
+};
+
+// If the qualified-id `incrementable_traits<I>::difference_type` is valid and denotes a type, then
+// `difference_type` names that type;
+template <class _Ip>
+  requires requires { typename incrementable_traits<_Ip>::difference_type; }
+struct __iterator_traits_difference_type<_Ip> {
+  using type = typename incrementable_traits<_Ip>::difference_type;
+};
+
+// [iterator.traits]/3.4
+// Otherwise, `iterator_traits<I>` has no members by any of the above names.
+template <class>
+struct __iterator_traits {};
+
+// [iterator.traits]/3.1
+// If `I` has valid ([temp.deduct]) member types `difference-type`, `value-type`, `reference`, and
+// `iterator-category`, then `iterator-traits<I>` has the following publicly accessible members:
+template <__specifies_members _Ip>
+struct __iterator_traits<_Ip> {
+  using iterator_category = typename _Ip::iterator_category;
+  using value_type        = typename _Ip::value_type;
+  using difference_type   = typename _Ip::difference_type;
+  using pointer           = typename __iterator_traits_member_pointer_or_void<_Ip>::type;
+  using reference         = typename _Ip::reference;
+};
+
+// [iterator.traits]/3.2
+// Otherwise, if `I` satisfies the exposition-only concept `cpp17-input-iterator`,
+// `iterator-traits<I>` has the following publicly accessible members:
+template <__cpp17_input_iterator_missing_members _Ip>
+struct __iterator_traits<_Ip> {
+  using iterator_category = typename __iterator_traits_iterator_category<_Ip>::type;
+  using value_type        = typename indirectly_readable_traits<_Ip>::value_type;
+  using difference_type   = typename incrementable_traits<_Ip>::difference_type;
+  using pointer           = typename __iterator_traits_member_pointer_or_arrow_or_void<_Ip>::type;
+  using reference         = typename __iterator_traits_member_reference<_Ip>::type;
+};
+
+// Otherwise, if `I` satisfies the exposition-only concept `cpp17-iterator`, then
+// `iterator_traits<I>` has the following publicly accessible members:
+template <__cpp17_iterator_missing_members _Ip>
+struct __iterator_traits<_Ip> {
+  using iterator_category = output_iterator_tag;
+  using value_type        = void;
+  using difference_type   = typename __iterator_traits_difference_type<_Ip>::type;
+  using pointer           = void;
+  using reference         = void;
+};
+
+template <class _Ip>
+struct iterator_traits : __iterator_traits<_Ip> {
+  using __primary_template = iterator_traits;
+};
+
+#else  // _LIBCPP_STD_VER >= 20
+
+template <class _Iter, bool>
+struct __iterator_traits {};
+
+template <class _Iter, bool>
+struct __iterator_traits_impl {};
+
+template <class _Iter>
+struct __iterator_traits_impl<_Iter, true> {
+  typedef typename _Iter::difference_type difference_type;
+  typedef typename _Iter::value_type value_type;
+  typedef typename _Iter::pointer pointer;
+  typedef typename _Iter::reference reference;
+  typedef typename _Iter::iterator_category iterator_category;
+};
+
+template <class _Iter>
+struct __iterator_traits<_Iter, true>
+    : __iterator_traits_impl< _Iter,
+                              is_convertible<typename _Iter::iterator_category, input_iterator_tag>::value ||
+                                  is_convertible<typename _Iter::iterator_category, output_iterator_tag>::value > {};
+
+// iterator_traits<Iterator> will only have the nested types if Iterator::iterator_category
+//    exists.  Else iterator_traits<Iterator> will be an empty class.  This is a
+//    conforming extension which allows some programs to compile and behave as
+//    the client expects instead of failing at compile time.
+
+template <class _Iter>
+struct _LIBCPP_TEMPLATE_VIS iterator_traits : __iterator_traits<_Iter, __has_iterator_typedefs<_Iter>::value> {
+  using __primary_template = iterator_traits;
+};
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+#if _LIBCPP_STD_VER >= 20
+  requires is_object_v<_Tp>
+#endif
+struct _LIBCPP_TEMPLATE_VIS iterator_traits<_Tp*> {
+  typedef ptrdiff_t difference_type;
+  typedef __remove_cv_t<_Tp> value_type;
+  typedef _Tp* pointer;
+  typedef _Tp& reference;
+  typedef random_access_iterator_tag iterator_category;
+#if _LIBCPP_STD_VER >= 20
+  typedef contiguous_iterator_tag iterator_concept;
+#endif
+};
+
+template <class _Tp, class _Up, bool = __has_iterator_category<iterator_traits<_Tp> >::value>
+struct __has_iterator_category_convertible_to : is_convertible<typename iterator_traits<_Tp>::iterator_category, _Up> {
+};
+
+template <class _Tp, class _Up>
+struct __has_iterator_category_convertible_to<_Tp, _Up, false> : false_type {};
+
+template <class _Tp, class _Up, bool = __has_iterator_concept<_Tp>::value>
+struct __has_iterator_concept_convertible_to : is_convertible<typename _Tp::iterator_concept, _Up> {};
+
+template <class _Tp, class _Up>
+struct __has_iterator_concept_convertible_to<_Tp, _Up, false> : false_type {};
+
+template <class _Tp>
+using __has_input_iterator_category = __has_iterator_category_convertible_to<_Tp, input_iterator_tag>;
+
+template <class _Tp>
+using __has_forward_iterator_category = __has_iterator_category_convertible_to<_Tp, forward_iterator_tag>;
+
+template <class _Tp>
+using __has_bidirectional_iterator_category = __has_iterator_category_convertible_to<_Tp, bidirectional_iterator_tag>;
+
+template <class _Tp>
+using __has_random_access_iterator_category = __has_iterator_category_convertible_to<_Tp, random_access_iterator_tag>;
+
+// __libcpp_is_contiguous_iterator determines if an iterator is known by
+// libc++ to be contiguous, either because it advertises itself as such
+// (in C++20) or because it is a pointer type or a known trivial wrapper
+// around a (possibly fancy) pointer type, such as __wrap_iter<T*>.
+// Such iterators receive special "contiguous" optimizations in
+// std::copy and std::sort.
+//
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp>
+struct __libcpp_is_contiguous_iterator
+    : _Or< __has_iterator_category_convertible_to<_Tp, contiguous_iterator_tag>,
+           __has_iterator_concept_convertible_to<_Tp, contiguous_iterator_tag> > {};
+#else
+template <class _Tp>
+struct __libcpp_is_contiguous_iterator : false_type {};
+#endif
+
+// Any native pointer which is an iterator is also a contiguous iterator.
+template <class _Up>
+struct __libcpp_is_contiguous_iterator<_Up*> : true_type {};
+
+template <class _Iter>
+class __wrap_iter;
+
+template <class _Tp>
+using __has_exactly_input_iterator_category =
+    integral_constant<bool,
+                      __has_iterator_category_convertible_to<_Tp, input_iterator_tag>::value &&
+                          !__has_iterator_category_convertible_to<_Tp, forward_iterator_tag>::value>;
+
+template <class _Tp>
+using __has_exactly_forward_iterator_category =
+    integral_constant<bool,
+                      __has_iterator_category_convertible_to<_Tp, forward_iterator_tag>::value &&
+                          !__has_iterator_category_convertible_to<_Tp, bidirectional_iterator_tag>::value>;
+
+template <class _Tp>
+using __has_exactly_bidirectional_iterator_category =
+    integral_constant<bool,
+                      __has_iterator_category_convertible_to<_Tp, bidirectional_iterator_tag>::value &&
+                          !__has_iterator_category_convertible_to<_Tp, random_access_iterator_tag>::value>;
+
+template <class _InputIterator>
+using __iter_value_type = typename iterator_traits<_InputIterator>::value_type;
+
+template <class _InputIterator>
+using __iter_key_type = __remove_const_t<typename iterator_traits<_InputIterator>::value_type::first_type>;
+
+template <class _InputIterator>
+using __iter_mapped_type = typename iterator_traits<_InputIterator>::value_type::second_type;
+
+template <class _InputIterator>
+using __iter_to_alloc_type =
+    pair<const typename iterator_traits<_InputIterator>::value_type::first_type,
+         typename iterator_traits<_InputIterator>::value_type::second_type>;
+
+template <class _Iter>
+using __iterator_category_type = typename iterator_traits<_Iter>::iterator_category;
+
+template <class _Iter>
+using __iterator_pointer_type = typename iterator_traits<_Iter>::pointer;
+
+template <class _Iter>
+using __iter_diff_t = typename iterator_traits<_Iter>::difference_type;
+
+template <class _Iter>
+using __iter_reference = typename iterator_traits<_Iter>::reference;
+
+#if _LIBCPP_STD_VER >= 20
+
+// [readable.traits]
+
+// Let `RI` be `remove_cvref_t<I>`. The type `iter_value_t<I>` denotes
+// `indirectly_readable_traits<RI>::value_type` if `iterator_traits<RI>` names a specialization
+// generated from the primary template, and `iterator_traits<RI>::value_type` otherwise.
+// This has to be in this file and not readable_traits.h to break the include cycle between the two.
+template <class _Ip>
+using iter_value_t =
+    typename conditional_t<__is_primary_template<iterator_traits<remove_cvref_t<_Ip> > >::value,
+                           indirectly_readable_traits<remove_cvref_t<_Ip> >,
+                           iterator_traits<remove_cvref_t<_Ip> > >::value_type;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_ITERATOR_TRAITS_H
diff --git a/libcxx/include/__cxx03/__iterator/iterator_with_data.h b/libcxx/include/__cxx03/__iterator/iterator_with_data.h
new file mode 100644
index 000000000000..afdc0a4e12e2
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/iterator_with_data.h
@@ -0,0 +1,105 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_ITERATOR_WITH_DATA_H
+#define _LIBCPP___ITERATOR_ITERATOR_WITH_DATA_H
+
+#include <__compare/compare_three_way_result.h>
+#include <__compare/three_way_comparable.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/iter_swap.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/readable_traits.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <forward_iterator _Iterator, class _Data>
+class __iterator_with_data {
+  _Iterator __iter_{};
+  _Data __data_{};
+
+public:
+  using value_type      = iter_value_t<_Iterator>;
+  using difference_type = iter_difference_t<_Iterator>;
+
+  _LIBCPP_HIDE_FROM_ABI __iterator_with_data() = default;
+
+  constexpr _LIBCPP_HIDE_FROM_ABI __iterator_with_data(_Iterator __iter, _Data __data)
+      : __iter_(std::move(__iter)), __data_(std::move(__data)) {}
+
+  constexpr _LIBCPP_HIDE_FROM_ABI _Iterator __get_iter() const { return __iter_; }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI _Data __get_data() && { return std::move(__data_); }
+
+  friend constexpr _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const __iterator_with_data& __lhs, const __iterator_with_data& __rhs) {
+    return __lhs.__iter_ == __rhs.__iter_;
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI __iterator_with_data& operator++() {
+    ++__iter_;
+    return *this;
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI __iterator_with_data operator++(int) {
+    auto __tmp = *this;
+    __iter_++;
+    return __tmp;
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI __iterator_with_data& operator--()
+    requires bidirectional_iterator<_Iterator>
+  {
+    --__iter_;
+    return *this;
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI __iterator_with_data operator--(int)
+    requires bidirectional_iterator<_Iterator>
+  {
+    auto __tmp = *this;
+    --__iter_;
+    return __tmp;
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI iter_reference_t<_Iterator> operator*() const { return *__iter_; }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr iter_rvalue_reference_t<_Iterator>
+  iter_move(const __iterator_with_data& __iter) noexcept(noexcept(ranges::iter_move(__iter.__iter_))) {
+    return ranges::iter_move(__iter.__iter_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr void
+  iter_swap(const __iterator_with_data& __lhs,
+            const __iterator_with_data& __rhs) noexcept(noexcept(ranges::iter_swap(__lhs.__iter_, __rhs.__iter_)))
+    requires indirectly_swappable<_Iterator>
+  {
+    return ranges::iter_swap(__lhs.__data_, __rhs.__iter_);
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ITERATOR_ITERATOR_WITH_DATA_H
diff --git a/libcxx/include/__cxx03/__iterator/mergeable.h b/libcxx/include/__cxx03/__iterator/mergeable.h
new file mode 100644
index 000000000000..7976d751095e
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/mergeable.h
@@ -0,0 +1,42 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_MERGEABLE_H
+#define _LIBCPP___ITERATOR_MERGEABLE_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/projected.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Input1,
+          class _Input2,
+          class _Output,
+          class _Comp  = ranges::less,
+          class _Proj1 = identity,
+          class _Proj2 = identity>
+concept mergeable =
+    input_iterator<_Input1> && input_iterator<_Input2> && weakly_incrementable<_Output> &&
+    indirectly_copyable<_Input1, _Output> && indirectly_copyable<_Input2, _Output> &&
+    indirect_strict_weak_order<_Comp, projected<_Input1, _Proj1>, projected<_Input2, _Proj2>>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_MERGEABLE_H
diff --git a/libcxx/include/__cxx03/__iterator/move_iterator.h b/libcxx/include/__cxx03/__iterator/move_iterator.h
new file mode 100644
index 000000000000..a1c53e9bd2b5
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/move_iterator.h
@@ -0,0 +1,347 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_MOVE_ITERATOR_H
+#define _LIBCPP___ITERATOR_MOVE_ITERATOR_H
+
+#include <__compare/compare_three_way_result.h>
+#include <__compare/three_way_comparable.h>
+#include <__concepts/assignable.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/derived_from.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/iter_swap.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/move_sentinel.h>
+#include <__iterator/readable_traits.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/declval.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Iter, class = void>
+struct __move_iter_category_base {};
+
+template <class _Iter>
+  requires requires { typename iterator_traits<_Iter>::iterator_category; }
+struct __move_iter_category_base<_Iter> {
+  using iterator_category =
+      _If< derived_from<typename iterator_traits<_Iter>::iterator_category, random_access_iterator_tag>,
+           random_access_iterator_tag,
+           typename iterator_traits<_Iter>::iterator_category >;
+};
+
+template <class _Iter, class _Sent>
+concept __move_iter_comparable = requires {
+  { std::declval<const _Iter&>() == std::declval<_Sent>() } -> convertible_to<bool>;
+};
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _Iter>
+class _LIBCPP_TEMPLATE_VIS move_iterator
+#if _LIBCPP_STD_VER >= 20
+    : public __move_iter_category_base<_Iter>
+#endif
+{
+#if _LIBCPP_STD_VER >= 20
+
+private:
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __get_iter_concept() {
+    if constexpr (random_access_iterator<_Iter>) {
+      return random_access_iterator_tag{};
+    } else if constexpr (bidirectional_iterator<_Iter>) {
+      return bidirectional_iterator_tag{};
+    } else if constexpr (forward_iterator<_Iter>) {
+      return forward_iterator_tag{};
+    } else {
+      return input_iterator_tag{};
+    }
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+public:
+#if _LIBCPP_STD_VER >= 20
+  using iterator_type    = _Iter;
+  using iterator_concept = decltype(__get_iter_concept());
+  // iterator_category is inherited and not always present
+  using value_type      = iter_value_t<_Iter>;
+  using difference_type = iter_difference_t<_Iter>;
+  using pointer         = _Iter;
+  using reference       = iter_rvalue_reference_t<_Iter>;
+#else
+  typedef _Iter iterator_type;
+  typedef _If< __has_random_access_iterator_category<_Iter>::value,
+               random_access_iterator_tag,
+               typename iterator_traits<_Iter>::iterator_category >
+      iterator_category;
+  typedef typename iterator_traits<iterator_type>::value_type value_type;
+  typedef typename iterator_traits<iterator_type>::difference_type difference_type;
+  typedef iterator_type pointer;
+
+  typedef typename iterator_traits<iterator_type>::reference __reference;
+  typedef __conditional_t<is_reference<__reference>::value, __libcpp_remove_reference_t<__reference>&&, __reference>
+      reference;
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 explicit move_iterator(_Iter __i) : __current_(std::move(__i)) {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator& operator++() {
+    ++__current_;
+    return *this;
+  }
+
+  _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 pointer operator->() const {
+    return __current_;
+  }
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI constexpr move_iterator()
+    requires is_constructible_v<_Iter>
+      : __current_() {}
+
+  template <class _Up>
+    requires(!_IsSame<_Up, _Iter>::value) && convertible_to<const _Up&, _Iter>
+  _LIBCPP_HIDE_FROM_ABI constexpr move_iterator(const move_iterator<_Up>& __u) : __current_(__u.base()) {}
+
+  template <class _Up>
+    requires(!_IsSame<_Up, _Iter>::value) && convertible_to<const _Up&, _Iter> && assignable_from<_Iter&, const _Up&>
+  _LIBCPP_HIDE_FROM_ABI constexpr move_iterator& operator=(const move_iterator<_Up>& __u) {
+    __current_ = __u.base();
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Iter& base() const& noexcept { return __current_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter base() && { return std::move(__current_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr reference operator*() const { return ranges::iter_move(__current_); }
+  _LIBCPP_HIDE_FROM_ABI constexpr reference operator[](difference_type __n) const {
+    return ranges::iter_move(__current_ + __n);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator++(int)
+    requires forward_iterator<_Iter>
+  {
+    move_iterator __tmp(*this);
+    ++__current_;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator++(int) { ++__current_; }
+#else
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator() : __current_() {}
+
+  template <class _Up, __enable_if_t< !is_same<_Up, _Iter>::value && is_convertible<const _Up&, _Iter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator(const move_iterator<_Up>& __u)
+      : __current_(__u.base()) {}
+
+  template <class _Up,
+            __enable_if_t< !is_same<_Up, _Iter>::value && is_convertible<const _Up&, _Iter>::value &&
+                               is_assignable<_Iter&, const _Up&>::value,
+                           int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator& operator=(const move_iterator<_Up>& __u) {
+    __current_ = __u.base();
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _Iter base() const { return __current_; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference operator*() const {
+    return static_cast<reference>(*__current_);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference operator[](difference_type __n) const {
+    return static_cast<reference>(__current_[__n]);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator operator++(int) {
+    move_iterator __tmp(*this);
+    ++__current_;
+    return __tmp;
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator& operator--() {
+    --__current_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator operator--(int) {
+    move_iterator __tmp(*this);
+    --__current_;
+    return __tmp;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator operator+(difference_type __n) const {
+    return move_iterator(__current_ + __n);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator& operator+=(difference_type __n) {
+    __current_ += __n;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator operator-(difference_type __n) const {
+    return move_iterator(__current_ - __n);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator& operator-=(difference_type __n) {
+    __current_ -= __n;
+    return *this;
+  }
+
+#if _LIBCPP_STD_VER >= 20
+  template <sentinel_for<_Iter> _Sent>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr bool operator==(const move_iterator& __x, const move_sentinel<_Sent>& __y)
+    requires __move_iter_comparable<_Iter, _Sent>
+  {
+    return __x.base() == __y.base();
+  }
+
+  template <sized_sentinel_for<_Iter> _Sent>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Iter>
+  operator-(const move_sentinel<_Sent>& __x, const move_iterator& __y) {
+    return __x.base() - __y.base();
+  }
+
+  template <sized_sentinel_for<_Iter> _Sent>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Iter>
+  operator-(const move_iterator& __x, const move_sentinel<_Sent>& __y) {
+    return __x.base() - __y.base();
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI constexpr iter_rvalue_reference_t<_Iter>
+  iter_move(const move_iterator& __i) noexcept(noexcept(ranges::iter_move(__i.__current_))) {
+    return ranges::iter_move(__i.__current_);
+  }
+
+  template <indirectly_swappable<_Iter> _It2>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr void
+  iter_swap(const move_iterator& __x,
+            const move_iterator<_It2>& __y) noexcept(noexcept(ranges::iter_swap(__x.__current_, __y.__current_))) {
+    return ranges::iter_swap(__x.__current_, __y.__current_);
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+private:
+  template <class _It2>
+  friend class move_iterator;
+
+  _Iter __current_;
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(move_iterator);
+
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool
+operator==(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) {
+  return __x.base() == __y.base();
+}
+
+#if _LIBCPP_STD_VER <= 17
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool
+operator!=(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) {
+  return __x.base() != __y.base();
+}
+#endif // _LIBCPP_STD_VER <= 17
+
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool
+operator<(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) {
+  return __x.base() < __y.base();
+}
+
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool
+operator>(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) {
+  return __x.base() > __y.base();
+}
+
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool
+operator<=(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) {
+  return __x.base() <= __y.base();
+}
+
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool
+operator>=(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) {
+  return __x.base() >= __y.base();
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Iter1, three_way_comparable_with<_Iter1> _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI constexpr auto
+operator<=>(const move_iterator<_Iter1>& __x,
+            const move_iterator<_Iter2>& __y) -> compare_three_way_result_t<_Iter1, _Iter2> {
+  return __x.base() <=> __y.base();
+}
+#endif // _LIBCPP_STD_VER >= 20
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto
+operator-(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) -> decltype(__x.base() - __y.base()) {
+  return __x.base() - __y.base();
+}
+#else
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI typename move_iterator<_Iter1>::difference_type
+operator-(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) {
+  return __x.base() - __y.base();
+}
+#endif // !_LIBCPP_CXX03_LANG
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Iter>
+inline _LIBCPP_HIDE_FROM_ABI constexpr move_iterator<_Iter>
+operator+(iter_difference_t<_Iter> __n, const move_iterator<_Iter>& __x)
+  requires requires {
+    { __x.base() + __n } -> same_as<_Iter>;
+  }
+{
+  return __x + __n;
+}
+#else
+template <class _Iter>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator<_Iter>
+operator+(typename move_iterator<_Iter>::difference_type __n, const move_iterator<_Iter>& __x) {
+  return move_iterator<_Iter>(__x.base() + __n);
+}
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Iter1, class _Iter2>
+  requires(!sized_sentinel_for<_Iter1, _Iter2>)
+inline constexpr bool disable_sized_sentinel_for<move_iterator<_Iter1>, move_iterator<_Iter2>> = true;
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _Iter>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator<_Iter> make_move_iterator(_Iter __i) {
+  return move_iterator<_Iter>(std::move(__i));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ITERATOR_MOVE_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/move_sentinel.h b/libcxx/include/__cxx03/__iterator/move_sentinel.h
new file mode 100644
index 000000000000..4a2a09ef0611
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/move_sentinel.h
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_MOVE_SENTINEL_H
+#define _LIBCPP___ITERATOR_MOVE_SENTINEL_H
+
+#include <__concepts/assignable.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/semiregular.h>
+#include <__config>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <semiregular _Sent>
+class _LIBCPP_TEMPLATE_VIS move_sentinel {
+public:
+  _LIBCPP_HIDE_FROM_ABI move_sentinel() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit move_sentinel(_Sent __s) : __last_(std::move(__s)) {}
+
+  template <class _S2>
+    requires convertible_to<const _S2&, _Sent>
+  _LIBCPP_HIDE_FROM_ABI constexpr move_sentinel(const move_sentinel<_S2>& __s) : __last_(__s.base()) {}
+
+  template <class _S2>
+    requires assignable_from<_Sent&, const _S2&>
+  _LIBCPP_HIDE_FROM_ABI constexpr move_sentinel& operator=(const move_sentinel<_S2>& __s) {
+    __last_ = __s.base();
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Sent base() const { return __last_; }
+
+private:
+  _Sent __last_ = _Sent();
+};
+
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(move_sentinel);
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___ITERATOR_MOVE_SENTINEL_H
diff --git a/libcxx/include/__cxx03/__iterator/next.h b/libcxx/include/__cxx03/__iterator/next.h
new file mode 100644
index 000000000000..21d3688ad9eb
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/next.h
@@ -0,0 +1,83 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_NEXT_H
+#define _LIBCPP___ITERATOR_NEXT_H
+
+#include <__assert>
+#include <__config>
+#include <__iterator/advance.h>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/enable_if.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _InputIter
+next(_InputIter __x, typename iterator_traits<_InputIter>::difference_type __n = 1) {
+  // Calling `advance` with a negative value on a non-bidirectional iterator is a no-op in the current implementation.
+  // Note that this check duplicates the similar check in `std::advance`.
+  _LIBCPP_ASSERT_PEDANTIC(__n >= 0 || __has_bidirectional_iterator_category<_InputIter>::value,
+                          "Attempt to next(it, n) with negative n on a non-bidirectional iterator");
+
+  std::advance(__x, __n);
+  return __x;
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+// [range.iter.op.next]
+
+namespace ranges {
+namespace __next {
+
+struct __fn {
+  template <input_or_output_iterator _Ip>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Ip operator()(_Ip __x) const {
+    ++__x;
+    return __x;
+  }
+
+  template <input_or_output_iterator _Ip>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Ip operator()(_Ip __x, iter_difference_t<_Ip> __n) const {
+    ranges::advance(__x, __n);
+    return __x;
+  }
+
+  template <input_or_output_iterator _Ip, sentinel_for<_Ip> _Sp>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Ip operator()(_Ip __x, _Sp __bound_sentinel) const {
+    ranges::advance(__x, __bound_sentinel);
+    return __x;
+  }
+
+  template <input_or_output_iterator _Ip, sentinel_for<_Ip> _Sp>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Ip operator()(_Ip __x, iter_difference_t<_Ip> __n, _Sp __bound_sentinel) const {
+    ranges::advance(__x, __n, __bound_sentinel);
+    return __x;
+  }
+};
+
+} // namespace __next
+
+inline namespace __cpo {
+inline constexpr auto next = __next::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_NEXT_H
diff --git a/libcxx/include/__cxx03/__iterator/ostream_iterator.h b/libcxx/include/__cxx03/__iterator/ostream_iterator.h
new file mode 100644
index 000000000000..05697e62d9dc
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/ostream_iterator.h
@@ -0,0 +1,75 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_OSTREAM_ITERATOR_H
+#define _LIBCPP___ITERATOR_OSTREAM_ITERATOR_H
+
+#include <__config>
+#include <__fwd/ostream.h>
+#include <__fwd/string.h>
+#include <__iterator/iterator.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Tp, class _CharT = char, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS ostream_iterator
+#if _LIBCPP_STD_VER <= 14 || !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
+    : public iterator<output_iterator_tag, void, void, void, void>
+#endif
+{
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+public:
+  typedef output_iterator_tag iterator_category;
+  typedef void value_type;
+#if _LIBCPP_STD_VER >= 20
+  typedef ptrdiff_t difference_type;
+#else
+  typedef void difference_type;
+#endif
+  typedef void pointer;
+  typedef void reference;
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef basic_ostream<_CharT, _Traits> ostream_type;
+
+private:
+  ostream_type* __out_stream_;
+  const char_type* __delim_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI ostream_iterator(ostream_type& __s) _NOEXCEPT
+      : __out_stream_(std::addressof(__s)),
+        __delim_(nullptr) {}
+  _LIBCPP_HIDE_FROM_ABI ostream_iterator(ostream_type& __s, const _CharT* __delimiter) _NOEXCEPT
+      : __out_stream_(std::addressof(__s)),
+        __delim_(__delimiter) {}
+  _LIBCPP_HIDE_FROM_ABI ostream_iterator& operator=(const _Tp& __value) {
+    *__out_stream_ << __value;
+    if (__delim_)
+      *__out_stream_ << __delim_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ostream_iterator& operator*() { return *this; }
+  _LIBCPP_HIDE_FROM_ABI ostream_iterator& operator++() { return *this; }
+  _LIBCPP_HIDE_FROM_ABI ostream_iterator& operator++(int) { return *this; }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_OSTREAM_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/ostreambuf_iterator.h b/libcxx/include/__cxx03/__iterator/ostreambuf_iterator.h
new file mode 100644
index 000000000000..dda0094dc3f5
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/ostreambuf_iterator.h
@@ -0,0 +1,72 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_OSTREAMBUF_ITERATOR_H
+#define _LIBCPP___ITERATOR_OSTREAMBUF_ITERATOR_H
+
+#include <__config>
+#include <__iterator/iterator.h>
+#include <__iterator/iterator_traits.h>
+#include <cstddef>
+#include <iosfwd> // for forward declaration of basic_streambuf
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS ostreambuf_iterator
+#if _LIBCPP_STD_VER <= 14 || !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
+    : public iterator<output_iterator_tag, void, void, void, void>
+#endif
+{
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+public:
+  typedef output_iterator_tag iterator_category;
+  typedef void value_type;
+#if _LIBCPP_STD_VER >= 20
+  typedef ptrdiff_t difference_type;
+#else
+  typedef void difference_type;
+#endif
+  typedef void pointer;
+  typedef void reference;
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef basic_streambuf<_CharT, _Traits> streambuf_type;
+  typedef basic_ostream<_CharT, _Traits> ostream_type;
+
+private:
+  streambuf_type* __sbuf_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator(ostream_type& __s) _NOEXCEPT : __sbuf_(__s.rdbuf()) {}
+  _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator(streambuf_type* __s) _NOEXCEPT : __sbuf_(__s) {}
+  _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator& operator=(_CharT __c) {
+    if (__sbuf_ && traits_type::eq_int_type(__sbuf_->sputc(__c), traits_type::eof()))
+      __sbuf_ = nullptr;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator& operator*() { return *this; }
+  _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator& operator++() { return *this; }
+  _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator& operator++(int) { return *this; }
+  _LIBCPP_HIDE_FROM_ABI bool failed() const _NOEXCEPT { return __sbuf_ == nullptr; }
+
+  template <class _Ch, class _Tr>
+  friend _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator<_Ch, _Tr> __pad_and_output(
+      ostreambuf_iterator<_Ch, _Tr> __s, const _Ch* __ob, const _Ch* __op, const _Ch* __oe, ios_base& __iob, _Ch __fl);
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_OSTREAMBUF_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/permutable.h b/libcxx/include/__cxx03/__iterator/permutable.h
new file mode 100644
index 000000000000..f65ba3bfbbaa
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/permutable.h
@@ -0,0 +1,34 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_PERMUTABLE_H
+#define _LIBCPP___ITERATOR_PERMUTABLE_H
+
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iter_swap.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Iterator>
+concept permutable =
+    forward_iterator<_Iterator> && indirectly_movable_storable<_Iterator, _Iterator> &&
+    indirectly_swappable<_Iterator, _Iterator>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_PERMUTABLE_H
diff --git a/libcxx/include/__cxx03/__iterator/prev.h b/libcxx/include/__cxx03/__iterator/prev.h
new file mode 100644
index 000000000000..2f0e6a088edb
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/prev.h
@@ -0,0 +1,76 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_PREV_H
+#define _LIBCPP___ITERATOR_PREV_H
+
+#include <__assert>
+#include <__config>
+#include <__iterator/advance.h>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/enable_if.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _InputIter
+prev(_InputIter __x, typename iterator_traits<_InputIter>::difference_type __n = 1) {
+  // Calling `advance` with a negative value on a non-bidirectional iterator is a no-op in the current implementation.
+  // Note that this check duplicates the similar check in `std::advance`.
+  _LIBCPP_ASSERT_PEDANTIC(__n <= 0 || __has_bidirectional_iterator_category<_InputIter>::value,
+                          "Attempt to prev(it, n) with a positive n on a non-bidirectional iterator");
+  std::advance(__x, -__n);
+  return __x;
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+// [range.iter.op.prev]
+
+namespace ranges {
+namespace __prev {
+
+struct __fn {
+  template <bidirectional_iterator _Ip>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Ip operator()(_Ip __x) const {
+    --__x;
+    return __x;
+  }
+
+  template <bidirectional_iterator _Ip>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Ip operator()(_Ip __x, iter_difference_t<_Ip> __n) const {
+    ranges::advance(__x, -__n);
+    return __x;
+  }
+
+  template <bidirectional_iterator _Ip>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Ip operator()(_Ip __x, iter_difference_t<_Ip> __n, _Ip __bound_iter) const {
+    ranges::advance(__x, -__n, __bound_iter);
+    return __x;
+  }
+};
+
+} // namespace __prev
+
+inline namespace __cpo {
+inline constexpr auto prev = __prev::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_PREV_H
diff --git a/libcxx/include/__cxx03/__iterator/projected.h b/libcxx/include/__cxx03/__iterator/projected.h
new file mode 100644
index 000000000000..463d07b0d33c
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/projected.h
@@ -0,0 +1,53 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_PROJECTED_H
+#define _LIBCPP___ITERATOR_PROJECTED_H
+
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h> // iter_difference_t
+#include <__type_traits/remove_cvref.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _It, class _Proj>
+struct __projected_impl {
+  struct __type {
+    using value_type = remove_cvref_t<indirect_result_t<_Proj&, _It>>;
+    indirect_result_t<_Proj&, _It> operator*() const; // not defined
+  };
+};
+
+template <weakly_incrementable _It, class _Proj>
+struct __projected_impl<_It, _Proj> {
+  struct __type {
+    using value_type      = remove_cvref_t<indirect_result_t<_Proj&, _It>>;
+    using difference_type = iter_difference_t<_It>;
+    indirect_result_t<_Proj&, _It> operator*() const; // not defined
+  };
+};
+
+// Note that we implement std::projected in a way that satisfies P2538R1 even in standard
+// modes before C++26 to avoid breaking the ABI between standard modes (even though ABI
+// breaks with std::projected are expected to have essentially no impact).
+template <indirectly_readable _It, indirectly_regular_unary_invocable<_It> _Proj>
+using projected = typename __projected_impl<_It, _Proj>::__type;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_PROJECTED_H
diff --git a/libcxx/include/__cxx03/__iterator/ranges_iterator_traits.h b/libcxx/include/__cxx03/__iterator/ranges_iterator_traits.h
new file mode 100644
index 000000000000..859e7082048a
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/ranges_iterator_traits.h
@@ -0,0 +1,40 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_RANGES_ITERATOR_TRAITS_H
+#define _LIBCPP___ITERATOR_RANGES_ITERATOR_TRAITS_H
+
+#include <__config>
+#include <__fwd/pair.h>
+#include <__ranges/concepts.h>
+#include <__type_traits/remove_const.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <ranges::input_range _Range>
+using __range_key_type = __remove_const_t<typename ranges::range_value_t<_Range>::first_type>;
+
+template <ranges::input_range _Range>
+using __range_mapped_type = typename ranges::range_value_t<_Range>::second_type;
+
+template <ranges::input_range _Range>
+using __range_to_alloc_type =
+    pair<const typename ranges::range_value_t<_Range>::first_type, typename ranges::range_value_t<_Range>::second_type>;
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_RANGES_ITERATOR_TRAITS_H
diff --git a/libcxx/include/__cxx03/__iterator/readable_traits.h b/libcxx/include/__cxx03/__iterator/readable_traits.h
new file mode 100644
index 000000000000..25e74567fff1
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/readable_traits.h
@@ -0,0 +1,81 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_READABLE_TRAITS_H
+#define _LIBCPP___ITERATOR_READABLE_TRAITS_H
+
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__type_traits/conditional.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_object.h>
+#include <__type_traits/is_primary_template.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/remove_extent.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [readable.traits]
+template <class>
+struct __cond_value_type {};
+
+template <class _Tp>
+  requires is_object_v<_Tp>
+struct __cond_value_type<_Tp> {
+  using value_type = remove_cv_t<_Tp>;
+};
+
+template <class _Tp>
+concept __has_member_value_type = requires { typename _Tp::value_type; };
+
+template <class _Tp>
+concept __has_member_element_type = requires { typename _Tp::element_type; };
+
+template <class>
+struct indirectly_readable_traits {};
+
+template <class _Ip>
+  requires is_array_v<_Ip>
+struct indirectly_readable_traits<_Ip> {
+  using value_type = remove_cv_t<remove_extent_t<_Ip>>;
+};
+
+template <class _Ip>
+struct indirectly_readable_traits<const _Ip> : indirectly_readable_traits<_Ip> {};
+
+template <class _Tp>
+struct indirectly_readable_traits<_Tp*> : __cond_value_type<_Tp> {};
+
+template <__has_member_value_type _Tp>
+struct indirectly_readable_traits<_Tp> : __cond_value_type<typename _Tp::value_type> {};
+
+template <__has_member_element_type _Tp>
+struct indirectly_readable_traits<_Tp> : __cond_value_type<typename _Tp::element_type> {};
+
+template <__has_member_value_type _Tp>
+  requires __has_member_element_type<_Tp>
+struct indirectly_readable_traits<_Tp> {};
+
+template <__has_member_value_type _Tp>
+  requires __has_member_element_type<_Tp> &&
+           same_as<remove_cv_t<typename _Tp::element_type>, remove_cv_t<typename _Tp::value_type>>
+struct indirectly_readable_traits<_Tp> : __cond_value_type<typename _Tp::value_type> {};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_READABLE_TRAITS_H
diff --git a/libcxx/include/__cxx03/__iterator/reverse_access.h b/libcxx/include/__cxx03/__iterator/reverse_access.h
new file mode 100644
index 000000000000..54d7270b04a5
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/reverse_access.h
@@ -0,0 +1,80 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_REVERSE_ACCESS_H
+#define _LIBCPP___ITERATOR_REVERSE_ACCESS_H
+
+#include <__config>
+#include <__iterator/reverse_iterator.h>
+#include <cstddef>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 14
+
+template <class _Tp, size_t _Np>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator<_Tp*> rbegin(_Tp (&__array)[_Np]) {
+  return reverse_iterator<_Tp*>(__array + _Np);
+}
+
+template <class _Tp, size_t _Np>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator<_Tp*> rend(_Tp (&__array)[_Np]) {
+  return reverse_iterator<_Tp*>(__array);
+}
+
+template <class _Ep>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator<const _Ep*> rbegin(initializer_list<_Ep> __il) {
+  return reverse_iterator<const _Ep*>(__il.end());
+}
+
+template <class _Ep>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator<const _Ep*> rend(initializer_list<_Ep> __il) {
+  return reverse_iterator<const _Ep*>(__il.begin());
+}
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto rbegin(_Cp& __c) -> decltype(__c.rbegin()) {
+  return __c.rbegin();
+}
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto rbegin(const _Cp& __c) -> decltype(__c.rbegin()) {
+  return __c.rbegin();
+}
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto rend(_Cp& __c) -> decltype(__c.rend()) {
+  return __c.rend();
+}
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto rend(const _Cp& __c) -> decltype(__c.rend()) {
+  return __c.rend();
+}
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto crbegin(const _Cp& __c) -> decltype(std::rbegin(__c)) {
+  return std::rbegin(__c);
+}
+
+template <class _Cp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto crend(const _Cp& __c) -> decltype(std::rend(__c)) {
+  return std::rend(__c);
+}
+
+#endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_REVERSE_ACCESS_H
diff --git a/libcxx/include/__cxx03/__iterator/reverse_iterator.h b/libcxx/include/__cxx03/__iterator/reverse_iterator.h
new file mode 100644
index 000000000000..50c0f21eaa28
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/reverse_iterator.h
@@ -0,0 +1,346 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_REVERSE_ITERATOR_H
+#define _LIBCPP___ITERATOR_REVERSE_ITERATOR_H
+
+#include <__algorithm/unwrap_iter.h>
+#include <__compare/compare_three_way_result.h>
+#include <__compare/three_way_comparable.h>
+#include <__concepts/convertible_to.h>
+#include <__config>
+#include <__iterator/advance.h>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/iter_swap.h>
+#include <__iterator/iterator.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/prev.h>
+#include <__iterator/readable_traits.h>
+#include <__iterator/segmented_iterator.h>
+#include <__memory/addressof.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/subrange.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/is_same.h>
+#include <__utility/declval.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Iter>
+class _LIBCPP_TEMPLATE_VIS reverse_iterator
+#if _LIBCPP_STD_VER <= 14 || !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
+    : public iterator<typename iterator_traits<_Iter>::iterator_category,
+                      typename iterator_traits<_Iter>::value_type,
+                      typename iterator_traits<_Iter>::difference_type,
+                      typename iterator_traits<_Iter>::pointer,
+                      typename iterator_traits<_Iter>::reference>
+#endif
+{
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+private:
+#ifndef _LIBCPP_ABI_NO_ITERATOR_BASES
+  _Iter __t_; // no longer used as of LWG #2360, not removed due to ABI break
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+  static_assert(__has_bidirectional_iterator_category<_Iter>::value || bidirectional_iterator<_Iter>,
+                "reverse_iterator<It> requires It to be a bidirectional iterator.");
+#endif // _LIBCPP_STD_VER >= 20
+
+protected:
+  _Iter current;
+
+public:
+  using iterator_type = _Iter;
+
+  using iterator_category =
+      _If<__has_random_access_iterator_category<_Iter>::value,
+          random_access_iterator_tag,
+          typename iterator_traits<_Iter>::iterator_category>;
+  using pointer = typename iterator_traits<_Iter>::pointer;
+#if _LIBCPP_STD_VER >= 20
+  using iterator_concept = _If<random_access_iterator<_Iter>, random_access_iterator_tag, bidirectional_iterator_tag>;
+  using value_type       = iter_value_t<_Iter>;
+  using difference_type  = iter_difference_t<_Iter>;
+  using reference        = iter_reference_t<_Iter>;
+#else
+  using value_type      = typename iterator_traits<_Iter>::value_type;
+  using difference_type = typename iterator_traits<_Iter>::difference_type;
+  using reference       = typename iterator_traits<_Iter>::reference;
+#endif
+
+#ifndef _LIBCPP_ABI_NO_ITERATOR_BASES
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator() : __t_(), current() {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 explicit reverse_iterator(_Iter __x) : __t_(__x), current(__x) {}
+
+  template <class _Up, __enable_if_t<!is_same<_Up, _Iter>::value && is_convertible<_Up const&, _Iter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator(const reverse_iterator<_Up>& __u)
+      : __t_(__u.base()), current(__u.base()) {}
+
+  template <class _Up,
+            __enable_if_t<!is_same<_Up, _Iter>::value && is_convertible<_Up const&, _Iter>::value &&
+                              is_assignable<_Iter&, _Up const&>::value,
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator& operator=(const reverse_iterator<_Up>& __u) {
+    __t_ = current = __u.base();
+    return *this;
+  }
+#else
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator() : current() {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 explicit reverse_iterator(_Iter __x) : current(__x) {}
+
+  template <class _Up, __enable_if_t<!is_same<_Up, _Iter>::value && is_convertible<_Up const&, _Iter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator(const reverse_iterator<_Up>& __u)
+      : current(__u.base()) {}
+
+  template <class _Up,
+            __enable_if_t<!is_same<_Up, _Iter>::value && is_convertible<_Up const&, _Iter>::value &&
+                              is_assignable<_Iter&, _Up const&>::value,
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator& operator=(const reverse_iterator<_Up>& __u) {
+    current = __u.base();
+    return *this;
+  }
+#endif
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _Iter base() const { return current; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference operator*() const {
+    _Iter __tmp = current;
+    return *--__tmp;
+  }
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI constexpr pointer operator->() const
+    requires is_pointer_v<_Iter> || requires(const _Iter __i) { __i.operator->(); }
+  {
+    if constexpr (is_pointer_v<_Iter>) {
+      return std::prev(current);
+    } else {
+      return std::prev(current).operator->();
+    }
+  }
+#else
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 pointer operator->() const { return std::addressof(operator*()); }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator& operator++() {
+    --current;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator operator++(int) {
+    reverse_iterator __tmp(*this);
+    --current;
+    return __tmp;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator& operator--() {
+    ++current;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator operator--(int) {
+    reverse_iterator __tmp(*this);
+    ++current;
+    return __tmp;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator operator+(difference_type __n) const {
+    return reverse_iterator(current - __n);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator& operator+=(difference_type __n) {
+    current -= __n;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator operator-(difference_type __n) const {
+    return reverse_iterator(current + __n);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator& operator-=(difference_type __n) {
+    current += __n;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference operator[](difference_type __n) const {
+    return *(*this + __n);
+  }
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI friend constexpr iter_rvalue_reference_t<_Iter> iter_move(const reverse_iterator& __i) noexcept(
+      is_nothrow_copy_constructible_v<_Iter> && noexcept(ranges::iter_move(--std::declval<_Iter&>()))) {
+    auto __tmp = __i.base();
+    return ranges::iter_move(--__tmp);
+  }
+
+  template <indirectly_swappable<_Iter> _Iter2>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr void
+  iter_swap(const reverse_iterator& __x, const reverse_iterator<_Iter2>& __y) noexcept(
+      is_nothrow_copy_constructible_v<_Iter> && is_nothrow_copy_constructible_v<_Iter2> &&
+      noexcept(ranges::iter_swap(--std::declval<_Iter&>(), --std::declval<_Iter2&>()))) {
+    auto __xtmp = __x.base();
+    auto __ytmp = __y.base();
+    ranges::iter_swap(--__xtmp, --__ytmp);
+  }
+#endif // _LIBCPP_STD_VER >= 20
+};
+
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool
+operator==(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y)
+#if _LIBCPP_STD_VER >= 20
+  requires requires {
+    { __x.base() == __y.base() } -> convertible_to<bool>;
+  }
+#endif // _LIBCPP_STD_VER >= 20
+{
+  return __x.base() == __y.base();
+}
+
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool
+operator<(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y)
+#if _LIBCPP_STD_VER >= 20
+  requires requires {
+    { __x.base() > __y.base() } -> convertible_to<bool>;
+  }
+#endif // _LIBCPP_STD_VER >= 20
+{
+  return __x.base() > __y.base();
+}
+
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool
+operator!=(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y)
+#if _LIBCPP_STD_VER >= 20
+  requires requires {
+    { __x.base() != __y.base() } -> convertible_to<bool>;
+  }
+#endif // _LIBCPP_STD_VER >= 20
+{
+  return __x.base() != __y.base();
+}
+
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool
+operator>(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y)
+#if _LIBCPP_STD_VER >= 20
+  requires requires {
+    { __x.base() < __y.base() } -> convertible_to<bool>;
+  }
+#endif // _LIBCPP_STD_VER >= 20
+{
+  return __x.base() < __y.base();
+}
+
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool
+operator>=(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y)
+#if _LIBCPP_STD_VER >= 20
+  requires requires {
+    { __x.base() <= __y.base() } -> convertible_to<bool>;
+  }
+#endif // _LIBCPP_STD_VER >= 20
+{
+  return __x.base() <= __y.base();
+}
+
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool
+operator<=(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y)
+#if _LIBCPP_STD_VER >= 20
+  requires requires {
+    { __x.base() >= __y.base() } -> convertible_to<bool>;
+  }
+#endif // _LIBCPP_STD_VER >= 20
+{
+  return __x.base() >= __y.base();
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Iter1, three_way_comparable_with<_Iter1> _Iter2>
+_LIBCPP_HIDE_FROM_ABI constexpr compare_three_way_result_t<_Iter1, _Iter2>
+operator<=>(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y) {
+  return __y.base() <=> __x.base();
+}
+#endif // _LIBCPP_STD_VER >= 20
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto
+operator-(const reverse_iterator<_Iter1>& __x,
+          const reverse_iterator<_Iter2>& __y) -> decltype(__y.base() - __x.base()) {
+  return __y.base() - __x.base();
+}
+#else
+template <class _Iter1, class _Iter2>
+inline _LIBCPP_HIDE_FROM_ABI typename reverse_iterator<_Iter1>::difference_type
+operator-(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y) {
+  return __y.base() - __x.base();
+}
+#endif
+
+template <class _Iter>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator<_Iter>
+operator+(typename reverse_iterator<_Iter>::difference_type __n, const reverse_iterator<_Iter>& __x) {
+  return reverse_iterator<_Iter>(__x.base() - __n);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Iter1, class _Iter2>
+  requires(!sized_sentinel_for<_Iter1, _Iter2>)
+inline constexpr bool disable_sized_sentinel_for<reverse_iterator<_Iter1>, reverse_iterator<_Iter2>> = true;
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Iter>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator<_Iter> make_reverse_iterator(_Iter __i) {
+  return reverse_iterator<_Iter>(__i);
+}
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+template <ranges::bidirectional_range _Range>
+_LIBCPP_HIDE_FROM_ABI constexpr ranges::subrange<reverse_iterator<ranges::iterator_t<_Range>>,
+                                                 reverse_iterator<ranges::iterator_t<_Range>>>
+__reverse_range(_Range&& __range) {
+  auto __first = ranges::begin(__range);
+  return {std::make_reverse_iterator(ranges::next(__first, ranges::end(__range))), std::make_reverse_iterator(__first)};
+}
+#endif
+
+template <class _Iter, bool __b>
+struct __unwrap_iter_impl<reverse_iterator<reverse_iterator<_Iter> >, __b> {
+  using _UnwrappedIter  = decltype(__unwrap_iter_impl<_Iter>::__unwrap(std::declval<_Iter>()));
+  using _ReverseWrapper = reverse_iterator<reverse_iterator<_Iter> >;
+
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ReverseWrapper
+  __rewrap(_ReverseWrapper __orig_iter, _UnwrappedIter __unwrapped_iter) {
+    return _ReverseWrapper(
+        reverse_iterator<_Iter>(__unwrap_iter_impl<_Iter>::__rewrap(__orig_iter.base().base(), __unwrapped_iter)));
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _UnwrappedIter __unwrap(_ReverseWrapper __i) _NOEXCEPT {
+    return __unwrap_iter_impl<_Iter>::__unwrap(__i.base().base());
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_REVERSE_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/segmented_iterator.h b/libcxx/include/__cxx03/__iterator/segmented_iterator.h
new file mode 100644
index 000000000000..f3cd1e5fa1f5
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/segmented_iterator.h
@@ -0,0 +1,79 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___SEGMENTED_ITERATOR_H
+#define _LIBCPP___SEGMENTED_ITERATOR_H
+
+// Segmented iterators are iterators over (not necessarily contiguous) sub-ranges.
+//
+// For example, std::deque stores its data into multiple blocks of contiguous memory,
+// which are not stored contiguously themselves. The concept of segmented iterators
+// allows algorithms to operate over these multi-level iterators natively, opening the
+// door to various optimizations. See http://lafstern.org/matt/segmented.pdf for details.
+//
+// If __segmented_iterator_traits can be instantiated, the following functions and associated types must be provided:
+// - Traits::__local_iterator
+//   The type of iterators used to iterate inside a segment.
+//
+// - Traits::__segment_iterator
+//   The type of iterators used to iterate over segments.
+//   Segment iterators can be forward iterators or bidirectional iterators, depending on the
+//   underlying data structure.
+//
+// - static __segment_iterator Traits::__segment(It __it)
+//   Returns an iterator to the segment that the provided iterator is in.
+//
+// - static __local_iterator Traits::__local(It __it)
+//   Returns the local iterator pointing to the element that the provided iterator points to.
+//
+// - static __local_iterator Traits::__begin(__segment_iterator __it)
+//   Returns the local iterator to the beginning of the segment that the provided iterator is pointing into.
+//
+// - static __local_iterator Traits::__end(__segment_iterator __it)
+//   Returns the one-past-the-end local iterator to the segment that the provided iterator is pointing into.
+//
+// - static It Traits::__compose(__segment_iterator, __local_iterator)
+//   Returns the iterator composed of the segment iterator and local iterator.
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Iterator>
+struct __segmented_iterator_traits;
+/* exposition-only:
+{
+  using __segment_iterator = ...;
+  using __local_iterator   = ...;
+
+  static __segment_iterator __segment(_Iterator);
+  static __local_iterator __local(_Iterator);
+  static __local_iterator __begin(__segment_iterator);
+  static __local_iterator __end(__segment_iterator);
+  static _Iterator __compose(__segment_iterator, __local_iterator);
+};
+*/
+
+template <class _Tp, size_t = 0>
+struct __has_specialization : false_type {};
+
+template <class _Tp>
+struct __has_specialization<_Tp, sizeof(_Tp) * 0> : true_type {};
+
+template <class _Iterator>
+using __is_segmented_iterator = __has_specialization<__segmented_iterator_traits<_Iterator> >;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___SEGMENTED_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/size.h b/libcxx/include/__cxx03/__iterator/size.h
new file mode 100644
index 000000000000..876e6963f77d
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/size.h
@@ -0,0 +1,59 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_SIZE_H
+#define _LIBCPP___ITERATOR_SIZE_H
+
+#include <__config>
+#include <__type_traits/common_type.h>
+#include <__type_traits/make_signed.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _Cont>
+_LIBCPP_HIDE_FROM_ABI constexpr auto size(const _Cont& __c) noexcept(noexcept(__c.size())) -> decltype(__c.size()) {
+  return __c.size();
+}
+
+template <class _Tp, size_t _Sz>
+_LIBCPP_HIDE_FROM_ABI constexpr size_t size(const _Tp (&)[_Sz]) noexcept {
+  return _Sz;
+}
+
+#  if _LIBCPP_STD_VER >= 20
+template <class _Cont>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+ssize(const _Cont& __c) noexcept(noexcept(static_cast<common_type_t<ptrdiff_t, make_signed_t<decltype(__c.size())>>>(
+    __c.size()))) -> common_type_t<ptrdiff_t, make_signed_t<decltype(__c.size())>> {
+  return static_cast<common_type_t<ptrdiff_t, make_signed_t<decltype(__c.size())>>>(__c.size());
+}
+
+// GCC complains about the implicit conversion from ptrdiff_t to size_t in
+// the array bound.
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wsign-conversion")
+template <class _Tp, ptrdiff_t _Sz>
+_LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t ssize(const _Tp (&)[_Sz]) noexcept {
+  return _Sz;
+}
+_LIBCPP_DIAGNOSTIC_POP
+#  endif
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_SIZE_H
diff --git a/libcxx/include/__cxx03/__iterator/sortable.h b/libcxx/include/__cxx03/__iterator/sortable.h
new file mode 100644
index 000000000000..1444860f2aa1
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/sortable.h
@@ -0,0 +1,35 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_SORTABLE_H
+#define _LIBCPP___ITERATOR_SORTABLE_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/permutable.h>
+#include <__iterator/projected.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Iter, class _Comp = ranges::less, class _Proj = identity>
+concept sortable = permutable<_Iter> && indirect_strict_weak_order<_Comp, projected<_Iter, _Proj>>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_SORTABLE_H
diff --git a/libcxx/include/__cxx03/__iterator/unreachable_sentinel.h b/libcxx/include/__cxx03/__iterator/unreachable_sentinel.h
new file mode 100644
index 000000000000..77e663da4b3a
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/unreachable_sentinel.h
@@ -0,0 +1,37 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_UNREACHABLE_SENTINEL_H
+#define _LIBCPP___ITERATOR_UNREACHABLE_SENTINEL_H
+
+#include <__config>
+#include <__iterator/concepts.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+struct unreachable_sentinel_t {
+  template <weakly_incrementable _Iter>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(unreachable_sentinel_t, const _Iter&) noexcept {
+    return false;
+  }
+};
+
+inline constexpr unreachable_sentinel_t unreachable_sentinel{};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_UNREACHABLE_SENTINEL_H
diff --git a/libcxx/include/__cxx03/__iterator/wrap_iter.h b/libcxx/include/__cxx03/__iterator/wrap_iter.h
new file mode 100644
index 000000000000..56183c0ee794
--- /dev/null
+++ b/libcxx/include/__cxx03/__iterator/wrap_iter.h
@@ -0,0 +1,244 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_WRAP_ITER_H
+#define _LIBCPP___ITERATOR_WRAP_ITER_H
+
+#include <__compare/ordering.h>
+#include <__compare/three_way_comparable.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__memory/pointer_traits.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_convertible.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Iter>
+class __wrap_iter {
+public:
+  typedef _Iter iterator_type;
+  typedef typename iterator_traits<iterator_type>::value_type value_type;
+  typedef typename iterator_traits<iterator_type>::difference_type difference_type;
+  typedef typename iterator_traits<iterator_type>::pointer pointer;
+  typedef typename iterator_traits<iterator_type>::reference reference;
+  typedef typename iterator_traits<iterator_type>::iterator_category iterator_category;
+#if _LIBCPP_STD_VER >= 20
+  typedef contiguous_iterator_tag iterator_concept;
+#endif
+
+private:
+  iterator_type __i_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __wrap_iter() _NOEXCEPT : __i_() {}
+  template <class _Up, __enable_if_t<is_convertible<_Up, iterator_type>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __wrap_iter(const __wrap_iter<_Up>& __u) _NOEXCEPT
+      : __i_(__u.base()) {}
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 reference operator*() const _NOEXCEPT { return *__i_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pointer operator->() const _NOEXCEPT {
+    return std::__to_address(__i_);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __wrap_iter& operator++() _NOEXCEPT {
+    ++__i_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __wrap_iter operator++(int) _NOEXCEPT {
+    __wrap_iter __tmp(*this);
+    ++(*this);
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __wrap_iter& operator--() _NOEXCEPT {
+    --__i_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __wrap_iter operator--(int) _NOEXCEPT {
+    __wrap_iter __tmp(*this);
+    --(*this);
+    return __tmp;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __wrap_iter operator+(difference_type __n) const _NOEXCEPT {
+    __wrap_iter __w(*this);
+    __w += __n;
+    return __w;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __wrap_iter& operator+=(difference_type __n) _NOEXCEPT {
+    __i_ += __n;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __wrap_iter operator-(difference_type __n) const _NOEXCEPT {
+    return *this + (-__n);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __wrap_iter& operator-=(difference_type __n) _NOEXCEPT {
+    *this += -__n;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 reference operator[](difference_type __n) const _NOEXCEPT {
+    return __i_[__n];
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 iterator_type base() const _NOEXCEPT { return __i_; }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __wrap_iter(iterator_type __x) _NOEXCEPT : __i_(__x) {}
+
+  template <class _Up>
+  friend class __wrap_iter;
+  template <class _CharT, class _Traits, class _Alloc>
+  friend class basic_string;
+  template <class _CharT, class _Traits>
+  friend class basic_string_view;
+  template <class _Tp, class _Alloc>
+  friend class _LIBCPP_TEMPLATE_VIS vector;
+  template <class _Tp, size_t>
+  friend class _LIBCPP_TEMPLATE_VIS span;
+  template <class _Tp, size_t _Size>
+  friend struct array;
+};
+
+template <class _Iter1>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator==(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter1>& __y) _NOEXCEPT {
+  return __x.base() == __y.base();
+}
+
+template <class _Iter1, class _Iter2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator==(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) _NOEXCEPT {
+  return __x.base() == __y.base();
+}
+
+template <class _Iter1>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator<(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter1>& __y) _NOEXCEPT {
+  return __x.base() < __y.base();
+}
+
+template <class _Iter1, class _Iter2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator<(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) _NOEXCEPT {
+  return __x.base() < __y.base();
+}
+
+#if _LIBCPP_STD_VER <= 17
+template <class _Iter1>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator!=(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter1>& __y) _NOEXCEPT {
+  return !(__x == __y);
+}
+
+template <class _Iter1, class _Iter2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator!=(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) _NOEXCEPT {
+  return !(__x == __y);
+}
+#endif
+
+// TODO(mordante) disable these overloads in the LLVM 20 release.
+template <class _Iter1>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator>(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter1>& __y) _NOEXCEPT {
+  return __y < __x;
+}
+
+template <class _Iter1, class _Iter2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator>(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) _NOEXCEPT {
+  return __y < __x;
+}
+
+template <class _Iter1>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator>=(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter1>& __y) _NOEXCEPT {
+  return !(__x < __y);
+}
+
+template <class _Iter1, class _Iter2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator>=(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) _NOEXCEPT {
+  return !(__x < __y);
+}
+
+template <class _Iter1>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator<=(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter1>& __y) _NOEXCEPT {
+  return !(__y < __x);
+}
+
+template <class _Iter1, class _Iter2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
+operator<=(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) _NOEXCEPT {
+  return !(__y < __x);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Iter1, class _Iter2>
+_LIBCPP_HIDE_FROM_ABI constexpr strong_ordering
+operator<=>(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) noexcept {
+  if constexpr (three_way_comparable_with<_Iter1, _Iter2, strong_ordering>) {
+    return __x.base() <=> __y.base();
+  } else {
+    if (__x.base() < __y.base())
+      return strong_ordering::less;
+
+    if (__x.base() == __y.base())
+      return strong_ordering::equal;
+
+    return strong_ordering::greater;
+  }
+}
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _Iter1, class _Iter2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+#ifndef _LIBCPP_CXX03_LANG
+    auto
+    operator-(const __wrap_iter<_Iter1>& __x,
+              const __wrap_iter<_Iter2>& __y) _NOEXCEPT->decltype(__x.base() - __y.base())
+#else
+typename __wrap_iter<_Iter1>::difference_type
+operator-(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) _NOEXCEPT
+#endif // C++03
+{
+  return __x.base() - __y.base();
+}
+
+template <class _Iter1>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __wrap_iter<_Iter1>
+operator+(typename __wrap_iter<_Iter1>::difference_type __n, __wrap_iter<_Iter1> __x) _NOEXCEPT {
+  __x += __n;
+  return __x;
+}
+
+#if _LIBCPP_STD_VER <= 17
+template <class _It>
+struct __libcpp_is_contiguous_iterator<__wrap_iter<_It> > : true_type {};
+#endif
+
+template <class _It>
+struct _LIBCPP_TEMPLATE_VIS pointer_traits<__wrap_iter<_It> > {
+  typedef __wrap_iter<_It> pointer;
+  typedef typename pointer_traits<_It>::element_type element_type;
+  typedef typename pointer_traits<_It>::difference_type difference_type;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR static element_type* to_address(pointer __w) _NOEXCEPT {
+    return std::__to_address(__w.base());
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_WRAP_ITER_H
diff --git a/libcxx/include/__cxx03/__locale b/libcxx/include/__cxx03/__locale
new file mode 100644
index 000000000000..4b382764b446
--- /dev/null
+++ b/libcxx/include/__cxx03/__locale
@@ -0,0 +1,1513 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE
+#define _LIBCPP___LOCALE
+
+#include <__config>
+#include <__locale_dir/locale_base_api.h>
+#include <__memory/shared_ptr.h> // __shared_count
+#include <__mutex/once_flag.h>
+#include <__type_traits/make_unsigned.h>
+#include <__utility/no_destroy.h>
+#include <__utility/private_constructor_tag.h>
+#include <cctype>
+#include <clocale>
+#include <cstdint>
+#include <cstdlib>
+#include <string>
+
+// Some platforms require more includes than others. Keep the includes on all plaforms for now.
+#include <cstddef>
+#include <cstring>
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+#  include <cwchar>
+#else
+#  include <__std_mbstate_t.h>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+class _LIBCPP_EXPORTED_FROM_ABI locale;
+
+template <class _Facet>
+_LIBCPP_HIDE_FROM_ABI bool has_facet(const locale&) _NOEXCEPT;
+
+template <class _Facet>
+_LIBCPP_HIDE_FROM_ABI const _Facet& use_facet(const locale&);
+
+class _LIBCPP_EXPORTED_FROM_ABI locale {
+public:
+  // locale is essentially a shared_ptr that doesn't support weak_ptrs and never got a move constructor.
+  using __trivially_relocatable = locale;
+
+  // types:
+  class _LIBCPP_EXPORTED_FROM_ABI facet;
+  class _LIBCPP_EXPORTED_FROM_ABI id;
+
+  typedef int category;
+
+  static const category // values assigned here are for exposition only
+      none    = 0,
+      collate = LC_COLLATE_MASK, ctype = LC_CTYPE_MASK, monetary = LC_MONETARY_MASK, numeric = LC_NUMERIC_MASK,
+      time = LC_TIME_MASK, messages = LC_MESSAGES_MASK, all = collate | ctype | monetary | numeric | time | messages;
+
+  // construct/copy/destroy:
+  locale() _NOEXCEPT;
+  locale(const locale&) _NOEXCEPT;
+  explicit locale(const char*);
+  explicit locale(const string&);
+  locale(const locale&, const char*, category);
+  locale(const locale&, const string&, category);
+  template <class _Facet>
+  _LIBCPP_HIDE_FROM_ABI locale(const locale&, _Facet*);
+  locale(const locale&, const locale&, category);
+
+  ~locale();
+
+  const locale& operator=(const locale&) _NOEXCEPT;
+
+  template <class _Facet>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS locale combine(const locale&) const;
+
+  // locale operations:
+  string name() const;
+  bool operator==(const locale&) const;
+#if _LIBCPP_STD_VER <= 17
+  _LIBCPP_HIDE_FROM_ABI bool operator!=(const locale& __y) const { return !(*this == __y); }
+#endif
+  template <class _CharT, class _Traits, class _Allocator>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS bool
+  operator()(const basic_string<_CharT, _Traits, _Allocator>&, const basic_string<_CharT, _Traits, _Allocator>&) const;
+
+  // global locale objects:
+  static locale global(const locale&);
+  static const locale& classic();
+
+private:
+  class __imp;
+  __imp* __locale_;
+
+  template <class>
+  friend struct __no_destroy;
+  _LIBCPP_HIDE_FROM_ABI explicit locale(__private_constructor_tag, __imp* __loc) : __locale_(__loc) {}
+
+  void __install_ctor(const locale&, facet*, long);
+  static locale& __global();
+  bool has_facet(id&) const;
+  const facet* use_facet(id&) const;
+
+  template <class _Facet>
+  friend bool has_facet(const locale&) _NOEXCEPT;
+  template <class _Facet>
+  friend const _Facet& use_facet(const locale&);
+};
+
+class _LIBCPP_EXPORTED_FROM_ABI locale::facet : public __shared_count {
+protected:
+  _LIBCPP_HIDE_FROM_ABI explicit facet(size_t __refs = 0) : __shared_count(static_cast<long>(__refs) - 1) {}
+
+  ~facet() override;
+
+  //    facet(const facet&) = delete;     // effectively done in __shared_count
+  //    void operator=(const facet&) = delete;
+
+private:
+  void __on_zero_shared() _NOEXCEPT override;
+};
+
+class _LIBCPP_EXPORTED_FROM_ABI locale::id {
+  once_flag __flag_;
+  int32_t __id_;
+
+  static int32_t __next_id;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR id() : __id_(0) {}
+  void operator=(const id&) = delete;
+  id(const id&)             = delete;
+
+public: // only needed for tests
+  long __get();
+
+  friend class locale;
+  friend class locale::__imp;
+};
+
+template <class _Facet>
+inline _LIBCPP_HIDE_FROM_ABI locale::locale(const locale& __other, _Facet* __f) {
+  __install_ctor(__other, __f, __f ? __f->id.__get() : 0);
+}
+
+template <class _Facet>
+locale locale::combine(const locale& __other) const {
+  if (!std::has_facet<_Facet>(__other))
+    __throw_runtime_error("locale::combine: locale missing facet");
+
+  return locale(*this, &const_cast<_Facet&>(std::use_facet<_Facet>(__other)));
+}
+
+template <class _Facet>
+inline _LIBCPP_HIDE_FROM_ABI bool has_facet(const locale& __l) _NOEXCEPT {
+  return __l.has_facet(_Facet::id);
+}
+
+template <class _Facet>
+inline _LIBCPP_HIDE_FROM_ABI const _Facet& use_facet(const locale& __l) {
+  return static_cast<const _Facet&>(*__l.use_facet(_Facet::id));
+}
+
+// template <class _CharT> class collate;
+
+template <class _CharT>
+class _LIBCPP_TEMPLATE_VIS collate : public locale::facet {
+public:
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit collate(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI int
+  compare(const char_type* __lo1, const char_type* __hi1, const char_type* __lo2, const char_type* __hi2) const {
+    return do_compare(__lo1, __hi1, __lo2, __hi2);
+  }
+
+  // FIXME(EricWF): The _LIBCPP_ALWAYS_INLINE is needed on Windows to work
+  // around a dllimport bug that expects an external instantiation.
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE string_type
+  transform(const char_type* __lo, const char_type* __hi) const {
+    return do_transform(__lo, __hi);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI long hash(const char_type* __lo, const char_type* __hi) const { return do_hash(__lo, __hi); }
+
+  static locale::id id;
+
+protected:
+  ~collate() override;
+  virtual int
+  do_compare(const char_type* __lo1, const char_type* __hi1, const char_type* __lo2, const char_type* __hi2) const;
+  virtual string_type do_transform(const char_type* __lo, const char_type* __hi) const {
+    return string_type(__lo, __hi);
+  }
+  virtual long do_hash(const char_type* __lo, const char_type* __hi) const;
+};
+
+template <class _CharT>
+locale::id collate<_CharT>::id;
+
+template <class _CharT>
+collate<_CharT>::~collate() {}
+
+template <class _CharT>
+int collate<_CharT>::do_compare(
+    const char_type* __lo1, const char_type* __hi1, const char_type* __lo2, const char_type* __hi2) const {
+  for (; __lo2 != __hi2; ++__lo1, ++__lo2) {
+    if (__lo1 == __hi1 || *__lo1 < *__lo2)
+      return -1;
+    if (*__lo2 < *__lo1)
+      return 1;
+  }
+  return __lo1 != __hi1;
+}
+
+template <class _CharT>
+long collate<_CharT>::do_hash(const char_type* __lo, const char_type* __hi) const {
+  size_t __h          = 0;
+  const size_t __sr   = __CHAR_BIT__ * sizeof(size_t) - 8;
+  const size_t __mask = size_t(0xF) << (__sr + 4);
+  for (const char_type* __p = __lo; __p != __hi; ++__p) {
+    __h        = (__h << 4) + static_cast<size_t>(*__p);
+    size_t __g = __h & __mask;
+    __h ^= __g | (__g >> __sr);
+  }
+  return static_cast<long>(__h);
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS collate<char>;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS collate<wchar_t>;
+#endif
+
+// template <class CharT> class collate_byname;
+
+template <class _CharT>
+class _LIBCPP_TEMPLATE_VIS collate_byname;
+
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI collate_byname<char> : public collate<char> {
+  locale_t __l_;
+
+public:
+  typedef char char_type;
+  typedef basic_string<char_type> string_type;
+
+  explicit collate_byname(const char* __n, size_t __refs = 0);
+  explicit collate_byname(const string& __n, size_t __refs = 0);
+
+protected:
+  ~collate_byname() override;
+  int do_compare(
+      const char_type* __lo1, const char_type* __hi1, const char_type* __lo2, const char_type* __hi2) const override;
+  string_type do_transform(const char_type* __lo, const char_type* __hi) const override;
+};
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI collate_byname<wchar_t> : public collate<wchar_t> {
+  locale_t __l_;
+
+public:
+  typedef wchar_t char_type;
+  typedef basic_string<char_type> string_type;
+
+  explicit collate_byname(const char* __n, size_t __refs = 0);
+  explicit collate_byname(const string& __n, size_t __refs = 0);
+
+protected:
+  ~collate_byname() override;
+
+  int do_compare(
+      const char_type* __lo1, const char_type* __hi1, const char_type* __lo2, const char_type* __hi2) const override;
+  string_type do_transform(const char_type* __lo, const char_type* __hi) const override;
+};
+#endif
+
+template <class _CharT, class _Traits, class _Allocator>
+bool locale::operator()(const basic_string<_CharT, _Traits, _Allocator>& __x,
+                        const basic_string<_CharT, _Traits, _Allocator>& __y) const {
+  return std::use_facet<std::collate<_CharT> >(*this).compare(
+             __x.data(), __x.data() + __x.size(), __y.data(), __y.data() + __y.size()) < 0;
+}
+
+// template <class charT> class ctype
+
+class _LIBCPP_EXPORTED_FROM_ABI ctype_base {
+public:
+#if defined(_LIBCPP_PROVIDES_DEFAULT_RUNE_TABLE)
+  typedef unsigned long mask;
+  static const mask space  = 1 << 0;
+  static const mask print  = 1 << 1;
+  static const mask cntrl  = 1 << 2;
+  static const mask upper  = 1 << 3;
+  static const mask lower  = 1 << 4;
+  static const mask alpha  = 1 << 5;
+  static const mask digit  = 1 << 6;
+  static const mask punct  = 1 << 7;
+  static const mask xdigit = 1 << 8;
+  static const mask blank  = 1 << 9;
+#  if defined(__BIONIC__)
+  // Historically this was a part of regex_traits rather than ctype_base. The
+  // historical value of the constant is preserved for ABI compatibility.
+  static const mask __regex_word = 0x8000;
+#  else
+  static const mask __regex_word = 1 << 10;
+#  endif // defined(__BIONIC__)
+#elif defined(__GLIBC__)
+  typedef unsigned short mask;
+  static const mask space  = _ISspace;
+  static const mask print  = _ISprint;
+  static const mask cntrl  = _IScntrl;
+  static const mask upper  = _ISupper;
+  static const mask lower  = _ISlower;
+  static const mask alpha  = _ISalpha;
+  static const mask digit  = _ISdigit;
+  static const mask punct  = _ISpunct;
+  static const mask xdigit = _ISxdigit;
+  static const mask blank  = _ISblank;
+#  if defined(__mips__) || (BYTE_ORDER == BIG_ENDIAN)
+  static const mask __regex_word = static_cast<mask>(_ISbit(15));
+#  else
+  static const mask __regex_word = 0x80;
+#  endif
+#elif defined(_LIBCPP_MSVCRT_LIKE)
+  typedef unsigned short mask;
+  static const mask space        = _SPACE;
+  static const mask print        = _BLANK | _PUNCT | _ALPHA | _DIGIT;
+  static const mask cntrl        = _CONTROL;
+  static const mask upper        = _UPPER;
+  static const mask lower        = _LOWER;
+  static const mask alpha        = _ALPHA;
+  static const mask digit        = _DIGIT;
+  static const mask punct        = _PUNCT;
+  static const mask xdigit       = _HEX;
+  static const mask blank        = _BLANK;
+  static const mask __regex_word = 0x4000; // 0x8000 and 0x0100 and 0x00ff are used
+#  define _LIBCPP_CTYPE_MASK_IS_COMPOSITE_PRINT
+#  define _LIBCPP_CTYPE_MASK_IS_COMPOSITE_ALPHA
+#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__)
+#  ifdef __APPLE__
+  typedef __uint32_t mask;
+#  elif defined(__FreeBSD__)
+  typedef unsigned long mask;
+#  elif defined(__NetBSD__)
+  typedef unsigned short mask;
+#  endif
+  static const mask space  = _CTYPE_S;
+  static const mask print  = _CTYPE_R;
+  static const mask cntrl  = _CTYPE_C;
+  static const mask upper  = _CTYPE_U;
+  static const mask lower  = _CTYPE_L;
+  static const mask alpha  = _CTYPE_A;
+  static const mask digit  = _CTYPE_D;
+  static const mask punct  = _CTYPE_P;
+  static const mask xdigit = _CTYPE_X;
+
+#  if defined(__NetBSD__)
+  static const mask blank = _CTYPE_BL;
+  // NetBSD defines classes up to 0x2000
+  // see sys/ctype_bits.h, _CTYPE_Q
+  static const mask __regex_word = 0x8000;
+#  else
+  static const mask blank        = _CTYPE_B;
+  static const mask __regex_word = 0x80;
+#  endif
+#elif defined(_AIX)
+  typedef unsigned int mask;
+  static const mask space        = _ISSPACE;
+  static const mask print        = _ISPRINT;
+  static const mask cntrl        = _ISCNTRL;
+  static const mask upper        = _ISUPPER;
+  static const mask lower        = _ISLOWER;
+  static const mask alpha        = _ISALPHA;
+  static const mask digit        = _ISDIGIT;
+  static const mask punct        = _ISPUNCT;
+  static const mask xdigit       = _ISXDIGIT;
+  static const mask blank        = _ISBLANK;
+  static const mask __regex_word = 0x8000;
+#elif defined(_NEWLIB_VERSION)
+  // Same type as Newlib's _ctype_ array in newlib/libc/include/ctype.h.
+  typedef char mask;
+  // In case char is signed, static_cast is needed to avoid warning on
+  // positive value becomming negative.
+  static const mask space  = static_cast<mask>(_S);
+  static const mask print  = static_cast<mask>(_P | _U | _L | _N | _B);
+  static const mask cntrl  = static_cast<mask>(_C);
+  static const mask upper  = static_cast<mask>(_U);
+  static const mask lower  = static_cast<mask>(_L);
+  static const mask alpha  = static_cast<mask>(_U | _L);
+  static const mask digit  = static_cast<mask>(_N);
+  static const mask punct  = static_cast<mask>(_P);
+  static const mask xdigit = static_cast<mask>(_X | _N);
+  static const mask blank  = static_cast<mask>(_B);
+  // mask is already fully saturated, use a different type in regex_type_traits.
+  static const unsigned short __regex_word = 0x100;
+#  define _LIBCPP_CTYPE_MASK_IS_COMPOSITE_PRINT
+#  define _LIBCPP_CTYPE_MASK_IS_COMPOSITE_ALPHA
+#  define _LIBCPP_CTYPE_MASK_IS_COMPOSITE_XDIGIT
+#elif defined(__MVS__)
+#  if defined(__NATIVE_ASCII_F)
+  typedef unsigned int mask;
+  static const mask space  = _ISSPACE_A;
+  static const mask print  = _ISPRINT_A;
+  static const mask cntrl  = _ISCNTRL_A;
+  static const mask upper  = _ISUPPER_A;
+  static const mask lower  = _ISLOWER_A;
+  static const mask alpha  = _ISALPHA_A;
+  static const mask digit  = _ISDIGIT_A;
+  static const mask punct  = _ISPUNCT_A;
+  static const mask xdigit = _ISXDIGIT_A;
+  static const mask blank  = _ISBLANK_A;
+#  else
+  typedef unsigned short mask;
+  static const mask space  = __ISSPACE;
+  static const mask print  = __ISPRINT;
+  static const mask cntrl  = __ISCNTRL;
+  static const mask upper  = __ISUPPER;
+  static const mask lower  = __ISLOWER;
+  static const mask alpha  = __ISALPHA;
+  static const mask digit  = __ISDIGIT;
+  static const mask punct  = __ISPUNCT;
+  static const mask xdigit = __ISXDIGIT;
+  static const mask blank  = __ISBLANK;
+#  endif
+  static const mask __regex_word = 0x8000;
+#else
+#  error unknown rune table for this platform -- do you mean to define _LIBCPP_PROVIDES_DEFAULT_RUNE_TABLE?
+#endif
+  static const mask alnum = alpha | digit;
+  static const mask graph = alnum | punct;
+
+  _LIBCPP_HIDE_FROM_ABI ctype_base() {}
+
+  static_assert((__regex_word & ~(std::make_unsigned<mask>::type)(space | print | cntrl | upper | lower | alpha |
+                                                                  digit | punct | xdigit | blank)) == __regex_word,
+                "__regex_word can't overlap other bits");
+};
+
+template <class _CharT>
+class _LIBCPP_TEMPLATE_VIS ctype;
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI ctype<wchar_t> : public locale::facet, public ctype_base {
+public:
+  typedef wchar_t char_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit ctype(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI bool is(mask __m, char_type __c) const { return do_is(__m, __c); }
+
+  _LIBCPP_HIDE_FROM_ABI const char_type* is(const char_type* __low, const char_type* __high, mask* __vec) const {
+    return do_is(__low, __high, __vec);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const char_type* scan_is(mask __m, const char_type* __low, const char_type* __high) const {
+    return do_scan_is(__m, __low, __high);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const char_type* scan_not(mask __m, const char_type* __low, const char_type* __high) const {
+    return do_scan_not(__m, __low, __high);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI char_type toupper(char_type __c) const { return do_toupper(__c); }
+
+  _LIBCPP_HIDE_FROM_ABI const char_type* toupper(char_type* __low, const char_type* __high) const {
+    return do_toupper(__low, __high);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI char_type tolower(char_type __c) const { return do_tolower(__c); }
+
+  _LIBCPP_HIDE_FROM_ABI const char_type* tolower(char_type* __low, const char_type* __high) const {
+    return do_tolower(__low, __high);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI char_type widen(char __c) const { return do_widen(__c); }
+
+  _LIBCPP_HIDE_FROM_ABI const char* widen(const char* __low, const char* __high, char_type* __to) const {
+    return do_widen(__low, __high, __to);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI char narrow(char_type __c, char __dfault) const { return do_narrow(__c, __dfault); }
+
+  _LIBCPP_HIDE_FROM_ABI const char_type*
+  narrow(const char_type* __low, const char_type* __high, char __dfault, char* __to) const {
+    return do_narrow(__low, __high, __dfault, __to);
+  }
+
+  static locale::id id;
+
+protected:
+  ~ctype() override;
+  virtual bool do_is(mask __m, char_type __c) const;
+  virtual const char_type* do_is(const char_type* __low, const char_type* __high, mask* __vec) const;
+  virtual const char_type* do_scan_is(mask __m, const char_type* __low, const char_type* __high) const;
+  virtual const char_type* do_scan_not(mask __m, const char_type* __low, const char_type* __high) const;
+  virtual char_type do_toupper(char_type) const;
+  virtual const char_type* do_toupper(char_type* __low, const char_type* __high) const;
+  virtual char_type do_tolower(char_type) const;
+  virtual const char_type* do_tolower(char_type* __low, const char_type* __high) const;
+  virtual char_type do_widen(char) const;
+  virtual const char* do_widen(const char* __low, const char* __high, char_type* __dest) const;
+  virtual char do_narrow(char_type, char __dfault) const;
+  virtual const char_type*
+  do_narrow(const char_type* __low, const char_type* __high, char __dfault, char* __dest) const;
+};
+#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI ctype<char> : public locale::facet, public ctype_base {
+  const mask* __tab_;
+  bool __del_;
+
+public:
+  typedef char char_type;
+
+  explicit ctype(const mask* __tab = nullptr, bool __del = false, size_t __refs = 0);
+
+  _LIBCPP_HIDE_FROM_ABI bool is(mask __m, char_type __c) const {
+    return isascii(__c) ? (__tab_[static_cast<int>(__c)] & __m) != 0 : false;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const char_type* is(const char_type* __low, const char_type* __high, mask* __vec) const {
+    for (; __low != __high; ++__low, ++__vec)
+      *__vec = isascii(*__low) ? __tab_[static_cast<int>(*__low)] : 0;
+    return __low;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const char_type* scan_is(mask __m, const char_type* __low, const char_type* __high) const {
+    for (; __low != __high; ++__low)
+      if (isascii(*__low) && (__tab_[static_cast<int>(*__low)] & __m))
+        break;
+    return __low;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const char_type* scan_not(mask __m, const char_type* __low, const char_type* __high) const {
+    for (; __low != __high; ++__low)
+      if (!isascii(*__low) || !(__tab_[static_cast<int>(*__low)] & __m))
+        break;
+    return __low;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI char_type toupper(char_type __c) const { return do_toupper(__c); }
+
+  _LIBCPP_HIDE_FROM_ABI const char_type* toupper(char_type* __low, const char_type* __high) const {
+    return do_toupper(__low, __high);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI char_type tolower(char_type __c) const { return do_tolower(__c); }
+
+  _LIBCPP_HIDE_FROM_ABI const char_type* tolower(char_type* __low, const char_type* __high) const {
+    return do_tolower(__low, __high);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI char_type widen(char __c) const { return do_widen(__c); }
+
+  _LIBCPP_HIDE_FROM_ABI const char* widen(const char* __low, const char* __high, char_type* __to) const {
+    return do_widen(__low, __high, __to);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI char narrow(char_type __c, char __dfault) const { return do_narrow(__c, __dfault); }
+
+  _LIBCPP_HIDE_FROM_ABI const char*
+  narrow(const char_type* __low, const char_type* __high, char __dfault, char* __to) const {
+    return do_narrow(__low, __high, __dfault, __to);
+  }
+
+  static locale::id id;
+
+#ifdef _CACHED_RUNES
+  static const size_t table_size = _CACHED_RUNES;
+#else
+  static const size_t table_size = 256; // FIXME: Don't hardcode this.
+#endif
+  _LIBCPP_HIDE_FROM_ABI const mask* table() const _NOEXCEPT { return __tab_; }
+  static const mask* classic_table() _NOEXCEPT;
+#if defined(__GLIBC__) || defined(__EMSCRIPTEN__)
+  static const int* __classic_upper_table() _NOEXCEPT;
+  static const int* __classic_lower_table() _NOEXCEPT;
+#endif
+#if defined(__NetBSD__)
+  static const short* __classic_upper_table() _NOEXCEPT;
+  static const short* __classic_lower_table() _NOEXCEPT;
+#endif
+#if defined(__MVS__)
+  static const unsigned short* __classic_upper_table() _NOEXCEPT;
+  static const unsigned short* __classic_lower_table() _NOEXCEPT;
+#endif
+
+protected:
+  ~ctype() override;
+  virtual char_type do_toupper(char_type __c) const;
+  virtual const char_type* do_toupper(char_type* __low, const char_type* __high) const;
+  virtual char_type do_tolower(char_type __c) const;
+  virtual const char_type* do_tolower(char_type* __low, const char_type* __high) const;
+  virtual char_type do_widen(char __c) const;
+  virtual const char* do_widen(const char* __low, const char* __high, char_type* __to) const;
+  virtual char do_narrow(char_type __c, char __dfault) const;
+  virtual const char* do_narrow(const char_type* __low, const char_type* __high, char __dfault, char* __to) const;
+};
+
+// template <class CharT> class ctype_byname;
+
+template <class _CharT>
+class _LIBCPP_TEMPLATE_VIS ctype_byname;
+
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI ctype_byname<char> : public ctype<char> {
+  locale_t __l_;
+
+public:
+  explicit ctype_byname(const char*, size_t = 0);
+  explicit ctype_byname(const string&, size_t = 0);
+
+protected:
+  ~ctype_byname() override;
+  char_type do_toupper(char_type) const override;
+  const char_type* do_toupper(char_type* __low, const char_type* __high) const override;
+  char_type do_tolower(char_type) const override;
+  const char_type* do_tolower(char_type* __low, const char_type* __high) const override;
+};
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI ctype_byname<wchar_t> : public ctype<wchar_t> {
+  locale_t __l_;
+
+public:
+  explicit ctype_byname(const char*, size_t = 0);
+  explicit ctype_byname(const string&, size_t = 0);
+
+protected:
+  ~ctype_byname() override;
+  bool do_is(mask __m, char_type __c) const override;
+  const char_type* do_is(const char_type* __low, const char_type* __high, mask* __vec) const override;
+  const char_type* do_scan_is(mask __m, const char_type* __low, const char_type* __high) const override;
+  const char_type* do_scan_not(mask __m, const char_type* __low, const char_type* __high) const override;
+  char_type do_toupper(char_type) const override;
+  const char_type* do_toupper(char_type* __low, const char_type* __high) const override;
+  char_type do_tolower(char_type) const override;
+  const char_type* do_tolower(char_type* __low, const char_type* __high) const override;
+  char_type do_widen(char) const override;
+  const char* do_widen(const char* __low, const char* __high, char_type* __dest) const override;
+  char do_narrow(char_type, char __dfault) const override;
+  const char_type*
+  do_narrow(const char_type* __low, const char_type* __high, char __dfault, char* __dest) const override;
+};
+#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI bool isspace(_CharT __c, const locale& __loc) {
+  return std::use_facet<ctype<_CharT> >(__loc).is(ctype_base::space, __c);
+}
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI bool isprint(_CharT __c, const locale& __loc) {
+  return std::use_facet<ctype<_CharT> >(__loc).is(ctype_base::print, __c);
+}
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI bool iscntrl(_CharT __c, const locale& __loc) {
+  return std::use_facet<ctype<_CharT> >(__loc).is(ctype_base::cntrl, __c);
+}
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI bool isupper(_CharT __c, const locale& __loc) {
+  return std::use_facet<ctype<_CharT> >(__loc).is(ctype_base::upper, __c);
+}
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI bool islower(_CharT __c, const locale& __loc) {
+  return std::use_facet<ctype<_CharT> >(__loc).is(ctype_base::lower, __c);
+}
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI bool isalpha(_CharT __c, const locale& __loc) {
+  return std::use_facet<ctype<_CharT> >(__loc).is(ctype_base::alpha, __c);
+}
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI bool isdigit(_CharT __c, const locale& __loc) {
+  return std::use_facet<ctype<_CharT> >(__loc).is(ctype_base::digit, __c);
+}
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI bool ispunct(_CharT __c, const locale& __loc) {
+  return std::use_facet<ctype<_CharT> >(__loc).is(ctype_base::punct, __c);
+}
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI bool isxdigit(_CharT __c, const locale& __loc) {
+  return std::use_facet<ctype<_CharT> >(__loc).is(ctype_base::xdigit, __c);
+}
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI bool isalnum(_CharT __c, const locale& __loc) {
+  return std::use_facet<ctype<_CharT> >(__loc).is(ctype_base::alnum, __c);
+}
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI bool isgraph(_CharT __c, const locale& __loc) {
+  return std::use_facet<ctype<_CharT> >(__loc).is(ctype_base::graph, __c);
+}
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI bool isblank(_CharT __c, const locale& __loc) {
+  return std::use_facet<ctype<_CharT> >(__loc).is(ctype_base::blank, __c);
+}
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI _CharT toupper(_CharT __c, const locale& __loc) {
+  return std::use_facet<ctype<_CharT> >(__loc).toupper(__c);
+}
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI _CharT tolower(_CharT __c, const locale& __loc) {
+  return std::use_facet<ctype<_CharT> >(__loc).tolower(__c);
+}
+
+// codecvt_base
+
+class _LIBCPP_EXPORTED_FROM_ABI codecvt_base {
+public:
+  _LIBCPP_HIDE_FROM_ABI codecvt_base() {}
+  enum result { ok, partial, error, noconv };
+};
+
+// template <class internT, class externT, class stateT> class codecvt;
+
+template <class _InternT, class _ExternT, class _StateT>
+class _LIBCPP_TEMPLATE_VIS codecvt;
+
+// template <> class codecvt<char, char, mbstate_t>
+
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI codecvt<char, char, mbstate_t> : public locale::facet, public codecvt_base {
+public:
+  typedef char intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit codecvt(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI result
+  out(state_type& __st,
+      const intern_type* __frm,
+      const intern_type* __frm_end,
+      const intern_type*& __frm_nxt,
+      extern_type* __to,
+      extern_type* __to_end,
+      extern_type*& __to_nxt) const {
+    return do_out(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI result
+  unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const {
+    return do_unshift(__st, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI result
+  in(state_type& __st,
+     const extern_type* __frm,
+     const extern_type* __frm_end,
+     const extern_type*& __frm_nxt,
+     intern_type* __to,
+     intern_type* __to_end,
+     intern_type*& __to_nxt) const {
+    return do_in(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int encoding() const _NOEXCEPT { return do_encoding(); }
+
+  _LIBCPP_HIDE_FROM_ABI bool always_noconv() const _NOEXCEPT { return do_always_noconv(); }
+
+  _LIBCPP_HIDE_FROM_ABI int
+  length(state_type& __st, const extern_type* __frm, const extern_type* __end, size_t __mx) const {
+    return do_length(__st, __frm, __end, __mx);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int max_length() const _NOEXCEPT { return do_max_length(); }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI explicit codecvt(const char*, size_t __refs = 0) : locale::facet(__refs) {}
+
+  ~codecvt() override;
+
+  virtual result
+  do_out(state_type& __st,
+         const intern_type* __frm,
+         const intern_type* __frm_end,
+         const intern_type*& __frm_nxt,
+         extern_type* __to,
+         extern_type* __to_end,
+         extern_type*& __to_nxt) const;
+  virtual result
+  do_in(state_type& __st,
+        const extern_type* __frm,
+        const extern_type* __frm_end,
+        const extern_type*& __frm_nxt,
+        intern_type* __to,
+        intern_type* __to_end,
+        intern_type*& __to_nxt) const;
+  virtual result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const;
+  virtual int do_encoding() const _NOEXCEPT;
+  virtual bool do_always_noconv() const _NOEXCEPT;
+  virtual int do_length(state_type& __st, const extern_type* __frm, const extern_type* __end, size_t __mx) const;
+  virtual int do_max_length() const _NOEXCEPT;
+};
+
+// template <> class codecvt<wchar_t, char, mbstate_t>
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI codecvt<wchar_t, char, mbstate_t> : public locale::facet, public codecvt_base {
+  locale_t __l_;
+
+public:
+  typedef wchar_t intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  explicit codecvt(size_t __refs = 0);
+
+  _LIBCPP_HIDE_FROM_ABI result
+  out(state_type& __st,
+      const intern_type* __frm,
+      const intern_type* __frm_end,
+      const intern_type*& __frm_nxt,
+      extern_type* __to,
+      extern_type* __to_end,
+      extern_type*& __to_nxt) const {
+    return do_out(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI result
+  unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const {
+    return do_unshift(__st, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI result
+  in(state_type& __st,
+     const extern_type* __frm,
+     const extern_type* __frm_end,
+     const extern_type*& __frm_nxt,
+     intern_type* __to,
+     intern_type* __to_end,
+     intern_type*& __to_nxt) const {
+    return do_in(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int encoding() const _NOEXCEPT { return do_encoding(); }
+
+  _LIBCPP_HIDE_FROM_ABI bool always_noconv() const _NOEXCEPT { return do_always_noconv(); }
+
+  _LIBCPP_HIDE_FROM_ABI int
+  length(state_type& __st, const extern_type* __frm, const extern_type* __end, size_t __mx) const {
+    return do_length(__st, __frm, __end, __mx);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int max_length() const _NOEXCEPT { return do_max_length(); }
+
+  static locale::id id;
+
+protected:
+  explicit codecvt(const char*, size_t __refs = 0);
+
+  ~codecvt() override;
+
+  virtual result
+  do_out(state_type& __st,
+         const intern_type* __frm,
+         const intern_type* __frm_end,
+         const intern_type*& __frm_nxt,
+         extern_type* __to,
+         extern_type* __to_end,
+         extern_type*& __to_nxt) const;
+  virtual result
+  do_in(state_type& __st,
+        const extern_type* __frm,
+        const extern_type* __frm_end,
+        const extern_type*& __frm_nxt,
+        intern_type* __to,
+        intern_type* __to_end,
+        intern_type*& __to_nxt) const;
+  virtual result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const;
+  virtual int do_encoding() const _NOEXCEPT;
+  virtual bool do_always_noconv() const _NOEXCEPT;
+  virtual int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const;
+  virtual int do_max_length() const _NOEXCEPT;
+};
+#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+// template <> class codecvt<char16_t, char, mbstate_t> // deprecated in C++20
+
+template <>
+class _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_EXPORTED_FROM_ABI codecvt<char16_t, char, mbstate_t>
+    : public locale::facet, public codecvt_base {
+public:
+  typedef char16_t intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit codecvt(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI result
+  out(state_type& __st,
+      const intern_type* __frm,
+      const intern_type* __frm_end,
+      const intern_type*& __frm_nxt,
+      extern_type* __to,
+      extern_type* __to_end,
+      extern_type*& __to_nxt) const {
+    return do_out(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI result
+  unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const {
+    return do_unshift(__st, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI result
+  in(state_type& __st,
+     const extern_type* __frm,
+     const extern_type* __frm_end,
+     const extern_type*& __frm_nxt,
+     intern_type* __to,
+     intern_type* __to_end,
+     intern_type*& __to_nxt) const {
+    return do_in(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int encoding() const _NOEXCEPT { return do_encoding(); }
+
+  _LIBCPP_HIDE_FROM_ABI bool always_noconv() const _NOEXCEPT { return do_always_noconv(); }
+
+  _LIBCPP_HIDE_FROM_ABI int
+  length(state_type& __st, const extern_type* __frm, const extern_type* __end, size_t __mx) const {
+    return do_length(__st, __frm, __end, __mx);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int max_length() const _NOEXCEPT { return do_max_length(); }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI explicit codecvt(const char*, size_t __refs = 0) : locale::facet(__refs) {}
+
+  ~codecvt() override;
+
+  virtual result
+  do_out(state_type& __st,
+         const intern_type* __frm,
+         const intern_type* __frm_end,
+         const intern_type*& __frm_nxt,
+         extern_type* __to,
+         extern_type* __to_end,
+         extern_type*& __to_nxt) const;
+  virtual result
+  do_in(state_type& __st,
+        const extern_type* __frm,
+        const extern_type* __frm_end,
+        const extern_type*& __frm_nxt,
+        intern_type* __to,
+        intern_type* __to_end,
+        intern_type*& __to_nxt) const;
+  virtual result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const;
+  virtual int do_encoding() const _NOEXCEPT;
+  virtual bool do_always_noconv() const _NOEXCEPT;
+  virtual int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const;
+  virtual int do_max_length() const _NOEXCEPT;
+};
+
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+
+// template <> class codecvt<char16_t, char8_t, mbstate_t> // C++20
+
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI codecvt<char16_t, char8_t, mbstate_t> : public locale::facet, public codecvt_base {
+public:
+  typedef char16_t intern_type;
+  typedef char8_t extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit codecvt(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI result
+  out(state_type& __st,
+      const intern_type* __frm,
+      const intern_type* __frm_end,
+      const intern_type*& __frm_nxt,
+      extern_type* __to,
+      extern_type* __to_end,
+      extern_type*& __to_nxt) const {
+    return do_out(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI result
+  unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const {
+    return do_unshift(__st, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI result
+  in(state_type& __st,
+     const extern_type* __frm,
+     const extern_type* __frm_end,
+     const extern_type*& __frm_nxt,
+     intern_type* __to,
+     intern_type* __to_end,
+     intern_type*& __to_nxt) const {
+    return do_in(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int encoding() const _NOEXCEPT { return do_encoding(); }
+
+  _LIBCPP_HIDE_FROM_ABI bool always_noconv() const _NOEXCEPT { return do_always_noconv(); }
+
+  _LIBCPP_HIDE_FROM_ABI int
+  length(state_type& __st, const extern_type* __frm, const extern_type* __end, size_t __mx) const {
+    return do_length(__st, __frm, __end, __mx);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int max_length() const _NOEXCEPT { return do_max_length(); }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI explicit codecvt(const char*, size_t __refs = 0) : locale::facet(__refs) {}
+
+  ~codecvt() override;
+
+  virtual result
+  do_out(state_type& __st,
+         const intern_type* __frm,
+         const intern_type* __frm_end,
+         const intern_type*& __frm_nxt,
+         extern_type* __to,
+         extern_type* __to_end,
+         extern_type*& __to_nxt) const;
+  virtual result
+  do_in(state_type& __st,
+        const extern_type* __frm,
+        const extern_type* __frm_end,
+        const extern_type*& __frm_nxt,
+        intern_type* __to,
+        intern_type* __to_end,
+        intern_type*& __to_nxt) const;
+  virtual result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const;
+  virtual int do_encoding() const _NOEXCEPT;
+  virtual bool do_always_noconv() const _NOEXCEPT;
+  virtual int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const;
+  virtual int do_max_length() const _NOEXCEPT;
+};
+
+#endif
+
+// template <> class codecvt<char32_t, char, mbstate_t> // deprecated in C++20
+
+template <>
+class _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_EXPORTED_FROM_ABI codecvt<char32_t, char, mbstate_t>
+    : public locale::facet, public codecvt_base {
+public:
+  typedef char32_t intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit codecvt(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI result
+  out(state_type& __st,
+      const intern_type* __frm,
+      const intern_type* __frm_end,
+      const intern_type*& __frm_nxt,
+      extern_type* __to,
+      extern_type* __to_end,
+      extern_type*& __to_nxt) const {
+    return do_out(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI result
+  unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const {
+    return do_unshift(__st, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI result
+  in(state_type& __st,
+     const extern_type* __frm,
+     const extern_type* __frm_end,
+     const extern_type*& __frm_nxt,
+     intern_type* __to,
+     intern_type* __to_end,
+     intern_type*& __to_nxt) const {
+    return do_in(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int encoding() const _NOEXCEPT { return do_encoding(); }
+
+  _LIBCPP_HIDE_FROM_ABI bool always_noconv() const _NOEXCEPT { return do_always_noconv(); }
+
+  _LIBCPP_HIDE_FROM_ABI int
+  length(state_type& __st, const extern_type* __frm, const extern_type* __end, size_t __mx) const {
+    return do_length(__st, __frm, __end, __mx);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int max_length() const _NOEXCEPT { return do_max_length(); }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI explicit codecvt(const char*, size_t __refs = 0) : locale::facet(__refs) {}
+
+  ~codecvt() override;
+
+  virtual result
+  do_out(state_type& __st,
+         const intern_type* __frm,
+         const intern_type* __frm_end,
+         const intern_type*& __frm_nxt,
+         extern_type* __to,
+         extern_type* __to_end,
+         extern_type*& __to_nxt) const;
+  virtual result
+  do_in(state_type& __st,
+        const extern_type* __frm,
+        const extern_type* __frm_end,
+        const extern_type*& __frm_nxt,
+        intern_type* __to,
+        intern_type* __to_end,
+        intern_type*& __to_nxt) const;
+  virtual result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const;
+  virtual int do_encoding() const _NOEXCEPT;
+  virtual bool do_always_noconv() const _NOEXCEPT;
+  virtual int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const;
+  virtual int do_max_length() const _NOEXCEPT;
+};
+
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+
+// template <> class codecvt<char32_t, char8_t, mbstate_t> // C++20
+
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI codecvt<char32_t, char8_t, mbstate_t> : public locale::facet, public codecvt_base {
+public:
+  typedef char32_t intern_type;
+  typedef char8_t extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit codecvt(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI result
+  out(state_type& __st,
+      const intern_type* __frm,
+      const intern_type* __frm_end,
+      const intern_type*& __frm_nxt,
+      extern_type* __to,
+      extern_type* __to_end,
+      extern_type*& __to_nxt) const {
+    return do_out(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI result
+  unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const {
+    return do_unshift(__st, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI result
+  in(state_type& __st,
+     const extern_type* __frm,
+     const extern_type* __frm_end,
+     const extern_type*& __frm_nxt,
+     intern_type* __to,
+     intern_type* __to_end,
+     intern_type*& __to_nxt) const {
+    return do_in(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int encoding() const _NOEXCEPT { return do_encoding(); }
+
+  _LIBCPP_HIDE_FROM_ABI bool always_noconv() const _NOEXCEPT { return do_always_noconv(); }
+
+  _LIBCPP_HIDE_FROM_ABI int
+  length(state_type& __st, const extern_type* __frm, const extern_type* __end, size_t __mx) const {
+    return do_length(__st, __frm, __end, __mx);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int max_length() const _NOEXCEPT { return do_max_length(); }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI explicit codecvt(const char*, size_t __refs = 0) : locale::facet(__refs) {}
+
+  ~codecvt() override;
+
+  virtual result
+  do_out(state_type& __st,
+         const intern_type* __frm,
+         const intern_type* __frm_end,
+         const intern_type*& __frm_nxt,
+         extern_type* __to,
+         extern_type* __to_end,
+         extern_type*& __to_nxt) const;
+  virtual result
+  do_in(state_type& __st,
+        const extern_type* __frm,
+        const extern_type* __frm_end,
+        const extern_type*& __frm_nxt,
+        intern_type* __to,
+        intern_type* __to_end,
+        intern_type*& __to_nxt) const;
+  virtual result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const;
+  virtual int do_encoding() const _NOEXCEPT;
+  virtual bool do_always_noconv() const _NOEXCEPT;
+  virtual int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const;
+  virtual int do_max_length() const _NOEXCEPT;
+};
+
+#endif
+
+// template <class _InternT, class _ExternT, class _StateT> class codecvt_byname
+
+template <class _InternT, class _ExternT, class _StateT>
+class _LIBCPP_TEMPLATE_VIS codecvt_byname : public codecvt<_InternT, _ExternT, _StateT> {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit codecvt_byname(const char* __nm, size_t __refs = 0)
+      : codecvt<_InternT, _ExternT, _StateT>(__nm, __refs) {}
+  _LIBCPP_HIDE_FROM_ABI explicit codecvt_byname(const string& __nm, size_t __refs = 0)
+      : codecvt<_InternT, _ExternT, _StateT>(__nm.c_str(), __refs) {}
+
+protected:
+  ~codecvt_byname() override;
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _InternT, class _ExternT, class _StateT>
+codecvt_byname<_InternT, _ExternT, _StateT>::~codecvt_byname() {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<char, char, mbstate_t>;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<wchar_t, char, mbstate_t>;
+#endif
+extern template class _LIBCPP_DEPRECATED_IN_CXX20
+_LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<char16_t, char, mbstate_t>; // deprecated in C++20
+extern template class _LIBCPP_DEPRECATED_IN_CXX20
+_LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<char32_t, char, mbstate_t>; // deprecated in C++20
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<char16_t, char8_t, mbstate_t>; // C++20
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<char32_t, char8_t, mbstate_t>; // C++20
+#endif
+
+template <size_t _Np>
+struct __narrow_to_utf8 {
+  template <class _OutputIterator, class _CharT>
+  _OutputIterator operator()(_OutputIterator __s, const _CharT* __wb, const _CharT* __we) const;
+};
+
+template <>
+struct __narrow_to_utf8<8> {
+  template <class _OutputIterator, class _CharT>
+  _LIBCPP_HIDE_FROM_ABI _OutputIterator operator()(_OutputIterator __s, const _CharT* __wb, const _CharT* __we) const {
+    for (; __wb < __we; ++__wb, ++__s)
+      *__s = *__wb;
+    return __s;
+  }
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <>
+struct _LIBCPP_EXPORTED_FROM_ABI __narrow_to_utf8<16> : public codecvt<char16_t, char, mbstate_t> {
+  _LIBCPP_HIDE_FROM_ABI __narrow_to_utf8() : codecvt<char16_t, char, mbstate_t>(1) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+  ~__narrow_to_utf8() override;
+
+  template <class _OutputIterator, class _CharT>
+  _LIBCPP_HIDE_FROM_ABI _OutputIterator operator()(_OutputIterator __s, const _CharT* __wb, const _CharT* __we) const {
+    result __r = ok;
+    mbstate_t __mb;
+    while (__wb < __we && __r != error) {
+      const int __sz = 32;
+      char __buf[__sz];
+      char* __bn;
+      const char16_t* __wn = (const char16_t*)__wb;
+      __r = do_out(__mb, (const char16_t*)__wb, (const char16_t*)__we, __wn, __buf, __buf + __sz, __bn);
+      if (__r == codecvt_base::error || __wn == (const char16_t*)__wb)
+        __throw_runtime_error("locale not supported");
+      for (const char* __p = __buf; __p < __bn; ++__p, ++__s)
+        *__s = *__p;
+      __wb = (const _CharT*)__wn;
+    }
+    return __s;
+  }
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <>
+struct _LIBCPP_EXPORTED_FROM_ABI __narrow_to_utf8<32> : public codecvt<char32_t, char, mbstate_t> {
+  _LIBCPP_HIDE_FROM_ABI __narrow_to_utf8() : codecvt<char32_t, char, mbstate_t>(1) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+  ~__narrow_to_utf8() override;
+
+  template <class _OutputIterator, class _CharT>
+  _LIBCPP_HIDE_FROM_ABI _OutputIterator operator()(_OutputIterator __s, const _CharT* __wb, const _CharT* __we) const {
+    result __r = ok;
+    mbstate_t __mb;
+    while (__wb < __we && __r != error) {
+      const int __sz = 32;
+      char __buf[__sz];
+      char* __bn;
+      const char32_t* __wn = (const char32_t*)__wb;
+      __r = do_out(__mb, (const char32_t*)__wb, (const char32_t*)__we, __wn, __buf, __buf + __sz, __bn);
+      if (__r == codecvt_base::error || __wn == (const char32_t*)__wb)
+        __throw_runtime_error("locale not supported");
+      for (const char* __p = __buf; __p < __bn; ++__p, ++__s)
+        *__s = *__p;
+      __wb = (const _CharT*)__wn;
+    }
+    return __s;
+  }
+};
+
+template <size_t _Np>
+struct __widen_from_utf8 {
+  template <class _OutputIterator>
+  _OutputIterator operator()(_OutputIterator __s, const char* __nb, const char* __ne) const;
+};
+
+template <>
+struct __widen_from_utf8<8> {
+  template <class _OutputIterator>
+  _LIBCPP_HIDE_FROM_ABI _OutputIterator operator()(_OutputIterator __s, const char* __nb, const char* __ne) const {
+    for (; __nb < __ne; ++__nb, ++__s)
+      *__s = *__nb;
+    return __s;
+  }
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <>
+struct _LIBCPP_EXPORTED_FROM_ABI __widen_from_utf8<16> : public codecvt<char16_t, char, mbstate_t> {
+  _LIBCPP_HIDE_FROM_ABI __widen_from_utf8() : codecvt<char16_t, char, mbstate_t>(1) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+  ~__widen_from_utf8() override;
+
+  template <class _OutputIterator>
+  _LIBCPP_HIDE_FROM_ABI _OutputIterator operator()(_OutputIterator __s, const char* __nb, const char* __ne) const {
+    result __r = ok;
+    mbstate_t __mb;
+    while (__nb < __ne && __r != error) {
+      const int __sz = 32;
+      char16_t __buf[__sz];
+      char16_t* __bn;
+      const char* __nn = __nb;
+      __r              = do_in(__mb, __nb, __ne - __nb > __sz ? __nb + __sz : __ne, __nn, __buf, __buf + __sz, __bn);
+      if (__r == codecvt_base::error || __nn == __nb)
+        __throw_runtime_error("locale not supported");
+      for (const char16_t* __p = __buf; __p < __bn; ++__p, ++__s)
+        *__s = *__p;
+      __nb = __nn;
+    }
+    return __s;
+  }
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <>
+struct _LIBCPP_EXPORTED_FROM_ABI __widen_from_utf8<32> : public codecvt<char32_t, char, mbstate_t> {
+  _LIBCPP_HIDE_FROM_ABI __widen_from_utf8() : codecvt<char32_t, char, mbstate_t>(1) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+  ~__widen_from_utf8() override;
+
+  template <class _OutputIterator>
+  _LIBCPP_HIDE_FROM_ABI _OutputIterator operator()(_OutputIterator __s, const char* __nb, const char* __ne) const {
+    result __r = ok;
+    mbstate_t __mb;
+    while (__nb < __ne && __r != error) {
+      const int __sz = 32;
+      char32_t __buf[__sz];
+      char32_t* __bn;
+      const char* __nn = __nb;
+      __r              = do_in(__mb, __nb, __ne - __nb > __sz ? __nb + __sz : __ne, __nn, __buf, __buf + __sz, __bn);
+      if (__r == codecvt_base::error || __nn == __nb)
+        __throw_runtime_error("locale not supported");
+      for (const char32_t* __p = __buf; __p < __bn; ++__p, ++__s)
+        *__s = *__p;
+      __nb = __nn;
+    }
+    return __s;
+  }
+};
+
+// template <class charT> class numpunct
+
+template <class _CharT>
+class _LIBCPP_TEMPLATE_VIS numpunct;
+
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI numpunct<char> : public locale::facet {
+public:
+  typedef char char_type;
+  typedef basic_string<char_type> string_type;
+
+  explicit numpunct(size_t __refs = 0);
+
+  _LIBCPP_HIDE_FROM_ABI char_type decimal_point() const { return do_decimal_point(); }
+  _LIBCPP_HIDE_FROM_ABI char_type thousands_sep() const { return do_thousands_sep(); }
+  _LIBCPP_HIDE_FROM_ABI string grouping() const { return do_grouping(); }
+  _LIBCPP_HIDE_FROM_ABI string_type truename() const { return do_truename(); }
+  _LIBCPP_HIDE_FROM_ABI string_type falsename() const { return do_falsename(); }
+
+  static locale::id id;
+
+protected:
+  ~numpunct() override;
+  virtual char_type do_decimal_point() const;
+  virtual char_type do_thousands_sep() const;
+  virtual string do_grouping() const;
+  virtual string_type do_truename() const;
+  virtual string_type do_falsename() const;
+
+  char_type __decimal_point_;
+  char_type __thousands_sep_;
+  string __grouping_;
+};
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI numpunct<wchar_t> : public locale::facet {
+public:
+  typedef wchar_t char_type;
+  typedef basic_string<char_type> string_type;
+
+  explicit numpunct(size_t __refs = 0);
+
+  _LIBCPP_HIDE_FROM_ABI char_type decimal_point() const { return do_decimal_point(); }
+  _LIBCPP_HIDE_FROM_ABI char_type thousands_sep() const { return do_thousands_sep(); }
+  _LIBCPP_HIDE_FROM_ABI string grouping() const { return do_grouping(); }
+  _LIBCPP_HIDE_FROM_ABI string_type truename() const { return do_truename(); }
+  _LIBCPP_HIDE_FROM_ABI string_type falsename() const { return do_falsename(); }
+
+  static locale::id id;
+
+protected:
+  ~numpunct() override;
+  virtual char_type do_decimal_point() const;
+  virtual char_type do_thousands_sep() const;
+  virtual string do_grouping() const;
+  virtual string_type do_truename() const;
+  virtual string_type do_falsename() const;
+
+  char_type __decimal_point_;
+  char_type __thousands_sep_;
+  string __grouping_;
+};
+#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+// template <class charT> class numpunct_byname
+
+template <class _CharT>
+class _LIBCPP_TEMPLATE_VIS numpunct_byname;
+
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI numpunct_byname<char> : public numpunct<char> {
+public:
+  typedef char char_type;
+  typedef basic_string<char_type> string_type;
+
+  explicit numpunct_byname(const char* __nm, size_t __refs = 0);
+  explicit numpunct_byname(const string& __nm, size_t __refs = 0);
+
+protected:
+  ~numpunct_byname() override;
+
+private:
+  void __init(const char*);
+};
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI numpunct_byname<wchar_t> : public numpunct<wchar_t> {
+public:
+  typedef wchar_t char_type;
+  typedef basic_string<char_type> string_type;
+
+  explicit numpunct_byname(const char* __nm, size_t __refs = 0);
+  explicit numpunct_byname(const string& __nm, size_t __refs = 0);
+
+protected:
+  ~numpunct_byname() override;
+
+private:
+  void __init(const char*);
+};
+#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___LOCALE
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api.h
new file mode 100644
index 000000000000..8c000c558c52
--- /dev/null
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api.h
@@ -0,0 +1,98 @@
+//===-----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_H
+#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_H
+
+#if defined(_LIBCPP_MSVCRT_LIKE)
+#  include <__locale_dir/locale_base_api/win32.h>
+#elif defined(_AIX) || defined(__MVS__)
+#  include <__locale_dir/locale_base_api/ibm.h>
+#elif defined(__ANDROID__)
+#  include <__locale_dir/locale_base_api/android.h>
+#elif defined(__sun__)
+#  include <__locale_dir/locale_base_api/solaris.h>
+#elif defined(_NEWLIB_VERSION)
+#  include <__locale_dir/locale_base_api/newlib.h>
+#elif defined(__OpenBSD__)
+#  include <__locale_dir/locale_base_api/openbsd.h>
+#elif defined(__Fuchsia__)
+#  include <__locale_dir/locale_base_api/fuchsia.h>
+#elif defined(__wasi__) || defined(_LIBCPP_HAS_MUSL_LIBC)
+#  include <__locale_dir/locale_base_api/musl.h>
+#elif defined(__APPLE__) || defined(__FreeBSD__)
+#  include <xlocale.h>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+/*
+The platform-specific headers have to provide the following interface:
+
+// TODO: rename this to __libcpp_locale_t
+using locale_t = implementation-defined;
+
+implementation-defined __libcpp_mb_cur_max_l(locale_t);
+wint_t __libcpp_btowc_l(int, locale_t);
+int __libcpp_wctob_l(wint_t, locale_t);
+size_t __libcpp_wcsnrtombs_l(char* dest, const wchar_t** src, size_t wide_char_count, size_t len, mbstate_t, locale_t);
+size_t __libcpp_wcrtomb_l(char* str, wchar_t wide_char, mbstate_t*, locale_t);
+size_t __libcpp_mbsnrtowcs_l(wchar_t* dest, const char** src, size_t max_out, size_t len, mbstate_t*, locale_t);
+size_t __libcpp_mbrtowc_l(wchar_t* dest, cosnt char* src, size_t count, mbstate_t*, locale_t);
+int __libcpp_mbtowc_l(wchar_t* dest, const char* src, size_t count, locale_t);
+size_t __libcpp_mbrlen_l(const char* str, size_t count, mbstate_t*, locale_t);
+lconv* __libcpp_localeconv_l(locale_t);
+size_t __libcpp_mbsrtowcs_l(wchar_t* dest, const char** src, size_t len, mbstate_t*, locale_t);
+int __libcpp_snprintf_l(char* dest, size_t buff_size, locale_t, const char* format, ...);
+int __libcpp_asprintf_l(char** dest, locale_t, const char* format, ...);
+int __libcpp_sscanf_l(const char* dest, locale_t, const char* format, ...);
+
+// TODO: change these to reserved names
+float strtof_l(const char* str, char** str_end, locale_t);
+double strtod_l(const char* str, char** str_end, locale_t);
+long double strtold_l(const char* str, char** str_end, locale_t);
+long long strtoll_l(const char* str, char** str_end, locale_t);
+unsigned long long strtoull_l(const char* str, char** str_end, locale_t);
+
+locale_t newlocale(int category_mask, const char* locale, locale_t base);
+void freelocale(locale_t);
+
+int islower_l(int ch, locale_t);
+int isupper_l(int ch, locale_t);
+int isdigit_l(int ch, locale_t);
+int isxdigit_l(int ch, locale_t);
+int strcoll_l(const char* lhs, const char* rhs, locale_t);
+size_t strxfrm_l(char* dst, const char* src, size_t n, locale_t);
+int wcscoll_l(const char* lhs, const char* rhs, locale_t);
+size_t wcsxfrm_l(wchar_t* dst, const wchar_t* src, size_t n, locale_t);
+int toupper_l(int ch, locale_t);
+int tolower_l(int ch, locale_t);
+int iswspace_l(wint_t ch, locale_t);
+int iswprint_l(wint_t ch, locale_t);
+int iswcntrl_l(wint_t ch, locale_t);
+int iswupper_l(wint_t ch, locale_t);
+int iswlower_l(wint_t ch, locale_t);
+int iswalpha_l(wint_t ch, locale_t);
+int iswblank_l(wint_t ch, locale_t);
+int iswdigit_l(wint_t ch, locale_t);
+int iswpunct_l(wint_t ch, locale_t);
+int iswxdigit_l(wint_t ch, locale_t);
+wint_t towupper_l(wint_t ch, locale_t);
+wint_t towlower_l(wint_t ch, locale_t);
+size_t strftime_l(char* str, size_t len, const char* format, const tm*, locale_t);
+
+
+These functions are equivalent to their C counterparts,
+except that locale_t is used instead of the current global locale.
+
+The variadic functions may be implemented as templates with a parameter pack instead of variadic functions.
+*/
+
+#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/android.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/android.h
new file mode 100644
index 000000000000..9965d8bbf6a2
--- /dev/null
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/android.h
@@ -0,0 +1,50 @@
+// -*- C++ -*-
+//===-----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_ANDROID_H
+#define _LIBCPP___LOCALE_LOCALE_BASE_API_ANDROID_H
+
+#include <stdlib.h>
+
+// FIXME: Is this actually required?
+extern "C" {
+#include <xlocale.h>
+}
+
+#include <android/api-level.h>
+#if __ANDROID_API__ < 21
+#  include <__support/xlocale/__posix_l_fallback.h>
+#endif
+
+// If we do not have this header, we are in a platform build rather than an NDK
+// build, which will always be at least as new as the ToT NDK, in which case we
+// don't need any of the inlines below since libc provides them.
+#if __has_include(<android/ndk-version.h>)
+#  include <android/ndk-version.h>
+// In NDK versions later than 16, locale-aware functions are provided by
+// legacy_stdlib_inlines.h
+#  if __NDK_MAJOR__ <= 16
+#    if __ANDROID_API__ < 21
+#      include <__support/xlocale/__strtonum_fallback.h>
+#    elif __ANDROID_API__ < 26
+
+inline _LIBCPP_HIDE_FROM_ABI float strtof_l(const char* __nptr, char** __endptr, locale_t) {
+  return ::strtof(__nptr, __endptr);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI double strtod_l(const char* __nptr, char** __endptr, locale_t) {
+  return ::strtod(__nptr, __endptr);
+}
+
+#    endif // __ANDROID_API__ < 26
+
+#  endif // __NDK_MAJOR__ <= 16
+#endif   // __has_include(<android/ndk-version.h>)
+
+#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_ANDROID_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_defaults.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_defaults.h
new file mode 100644
index 000000000000..1f9607209842
--- /dev/null
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_defaults.h
@@ -0,0 +1,36 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// The BSDs have lots of *_l functions.  We don't want to define those symbols
+// on other platforms though, for fear of conflicts with user code.  So here,
+// we will define the mapping from an internal macro to the real BSD symbol.
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_DEFAULTS_H
+#define _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_DEFAULTS_H
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#define __libcpp_mb_cur_max_l(loc) MB_CUR_MAX_L(loc)
+#define __libcpp_btowc_l(ch, loc) btowc_l(ch, loc)
+#define __libcpp_wctob_l(wch, loc) wctob_l(wch, loc)
+#define __libcpp_wcsnrtombs_l(dst, src, nwc, len, ps, loc) wcsnrtombs_l(dst, src, nwc, len, ps, loc)
+#define __libcpp_wcrtomb_l(src, wc, ps, loc) wcrtomb_l(src, wc, ps, loc)
+#define __libcpp_mbsnrtowcs_l(dst, src, nms, len, ps, loc) mbsnrtowcs_l(dst, src, nms, len, ps, loc)
+#define __libcpp_mbrtowc_l(pwc, s, n, ps, l) mbrtowc_l(pwc, s, n, ps, l)
+#define __libcpp_mbtowc_l(pwc, pmb, max, l) mbtowc_l(pwc, pmb, max, l)
+#define __libcpp_mbrlen_l(s, n, ps, l) mbrlen_l(s, n, ps, l)
+#define __libcpp_localeconv_l(l) localeconv_l(l)
+#define __libcpp_mbsrtowcs_l(dest, src, len, ps, l) mbsrtowcs_l(dest, src, len, ps, l)
+#define __libcpp_snprintf_l(...) snprintf_l(__VA_ARGS__)
+#define __libcpp_asprintf_l(...) asprintf_l(__VA_ARGS__)
+#define __libcpp_sscanf_l(...) sscanf_l(__VA_ARGS__)
+
+#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_DEFAULTS_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_fallbacks.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_fallbacks.h
new file mode 100644
index 000000000000..76b94287cd6c
--- /dev/null
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_fallbacks.h
@@ -0,0 +1,126 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// The BSDs have lots of *_l functions.  This file provides reimplementations
+// of those functions for non-BSD platforms.
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
+#define _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
+
+#include <__locale_dir/locale_base_api/locale_guard.h>
+#include <cstdio>
+#include <stdarg.h>
+#include <stdlib.h>
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+#  include <cwchar>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+inline _LIBCPP_HIDE_FROM_ABI decltype(MB_CUR_MAX) __libcpp_mb_cur_max_l(locale_t __l) {
+  __libcpp_locale_guard __current(__l);
+  return MB_CUR_MAX;
+}
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+inline _LIBCPP_HIDE_FROM_ABI wint_t __libcpp_btowc_l(int __c, locale_t __l) {
+  __libcpp_locale_guard __current(__l);
+  return btowc(__c);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_wctob_l(wint_t __c, locale_t __l) {
+  __libcpp_locale_guard __current(__l);
+  return wctob(__c);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI size_t
+__libcpp_wcsnrtombs_l(char* __dest, const wchar_t** __src, size_t __nwc, size_t __len, mbstate_t* __ps, locale_t __l) {
+  __libcpp_locale_guard __current(__l);
+  return wcsnrtombs(__dest, __src, __nwc, __len, __ps);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI size_t __libcpp_wcrtomb_l(char* __s, wchar_t __wc, mbstate_t* __ps, locale_t __l) {
+  __libcpp_locale_guard __current(__l);
+  return wcrtomb(__s, __wc, __ps);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI size_t
+__libcpp_mbsnrtowcs_l(wchar_t* __dest, const char** __src, size_t __nms, size_t __len, mbstate_t* __ps, locale_t __l) {
+  __libcpp_locale_guard __current(__l);
+  return mbsnrtowcs(__dest, __src, __nms, __len, __ps);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI size_t
+__libcpp_mbrtowc_l(wchar_t* __pwc, const char* __s, size_t __n, mbstate_t* __ps, locale_t __l) {
+  __libcpp_locale_guard __current(__l);
+  return mbrtowc(__pwc, __s, __n, __ps);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_mbtowc_l(wchar_t* __pwc, const char* __pmb, size_t __max, locale_t __l) {
+  __libcpp_locale_guard __current(__l);
+  return mbtowc(__pwc, __pmb, __max);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI size_t __libcpp_mbrlen_l(const char* __s, size_t __n, mbstate_t* __ps, locale_t __l) {
+  __libcpp_locale_guard __current(__l);
+  return mbrlen(__s, __n, __ps);
+}
+#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+inline _LIBCPP_HIDE_FROM_ABI lconv* __libcpp_localeconv_l(locale_t __l) {
+  __libcpp_locale_guard __current(__l);
+  return localeconv();
+}
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+inline _LIBCPP_HIDE_FROM_ABI size_t
+__libcpp_mbsrtowcs_l(wchar_t* __dest, const char** __src, size_t __len, mbstate_t* __ps, locale_t __l) {
+  __libcpp_locale_guard __current(__l);
+  return mbsrtowcs(__dest, __src, __len, __ps);
+}
+#endif
+
+inline _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 4, 5) int __libcpp_snprintf_l(
+    char* __s, size_t __n, locale_t __l, const char* __format, ...) {
+  va_list __va;
+  va_start(__va, __format);
+  __libcpp_locale_guard __current(__l);
+  int __res = vsnprintf(__s, __n, __format, __va);
+  va_end(__va);
+  return __res;
+}
+
+inline _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 3, 4) int __libcpp_asprintf_l(
+    char** __s, locale_t __l, const char* __format, ...) {
+  va_list __va;
+  va_start(__va, __format);
+  __libcpp_locale_guard __current(__l);
+  int __res = vasprintf(__s, __format, __va);
+  va_end(__va);
+  return __res;
+}
+
+inline _LIBCPP_ATTRIBUTE_FORMAT(__scanf__, 3, 4) int __libcpp_sscanf_l(
+    const char* __s, locale_t __l, const char* __format, ...) {
+  va_list __va;
+  va_start(__va, __format);
+  __libcpp_locale_guard __current(__l);
+  int __res = vsscanf(__s, __format, __va);
+  va_end(__va);
+  return __res;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/fuchsia.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/fuchsia.h
new file mode 100644
index 000000000000..4c3440f981c6
--- /dev/null
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/fuchsia.h
@@ -0,0 +1,18 @@
+// -*- C++ -*-
+//===-----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_FUCHSIA_H
+#define _LIBCPP___LOCALE_LOCALE_BASE_API_FUCHSIA_H
+
+#include <__support/xlocale/__posix_l_fallback.h>
+#include <__support/xlocale/__strtonum_fallback.h>
+#include <cstdlib>
+#include <cwchar>
+
+#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_FUCHSIA_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/ibm.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/ibm.h
new file mode 100644
index 000000000000..01af20194428
--- /dev/null
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/ibm.h
@@ -0,0 +1,108 @@
+// -*- C++ -*-
+//===-----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_IBM_H
+#define _LIBCPP___LOCALE_LOCALE_BASE_API_IBM_H
+
+#if defined(__MVS__)
+#  include <__support/ibm/locale_mgmt_zos.h>
+#endif // defined(__MVS__)
+
+#include <locale.h>
+#include <stdarg.h>
+#include <stdio.h>
+
+#include "cstdlib"
+
+#if defined(__MVS__)
+#  include <wctype.h>
+// POSIX routines
+#  include <__support/xlocale/__posix_l_fallback.h>
+#endif // defined(__MVS__)
+
+namespace {
+
+struct __setAndRestore {
+  explicit __setAndRestore(locale_t locale) {
+    if (locale == (locale_t)0) {
+      __cloc   = newlocale(LC_ALL_MASK, "C", /* base */ (locale_t)0);
+      __stored = uselocale(__cloc);
+    } else {
+      __stored = uselocale(locale);
+    }
+  }
+
+  ~__setAndRestore() {
+    uselocale(__stored);
+    if (__cloc)
+      freelocale(__cloc);
+  }
+
+private:
+  locale_t __stored = (locale_t)0;
+  locale_t __cloc   = (locale_t)0;
+};
+
+} // namespace
+
+// The following are not POSIX routines.  These are quick-and-dirty hacks
+// to make things pretend to work
+inline _LIBCPP_HIDE_FROM_ABI long long strtoll_l(const char* __nptr, char** __endptr, int __base, locale_t locale) {
+  __setAndRestore __newloc(locale);
+  return ::strtoll(__nptr, __endptr, __base);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI double strtod_l(const char* __nptr, char** __endptr, locale_t locale) {
+  __setAndRestore __newloc(locale);
+  return ::strtod(__nptr, __endptr);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI float strtof_l(const char* __nptr, char** __endptr, locale_t locale) {
+  __setAndRestore __newloc(locale);
+  return ::strtof(__nptr, __endptr);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double strtold_l(const char* __nptr, char** __endptr, locale_t locale) {
+  __setAndRestore __newloc(locale);
+  return ::strtold(__nptr, __endptr);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI unsigned long long
+strtoull_l(const char* __nptr, char** __endptr, int __base, locale_t locale) {
+  __setAndRestore __newloc(locale);
+  return ::strtoull(__nptr, __endptr, __base);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI
+_LIBCPP_ATTRIBUTE_FORMAT(__printf__, 2, 0) int vasprintf(char** strp, const char* fmt, va_list ap) {
+  const size_t buff_size = 256;
+  if ((*strp = (char*)malloc(buff_size)) == NULL) {
+    return -1;
+  }
+
+  va_list ap_copy;
+  // va_copy may not be provided by the C library in C++03 mode.
+#if defined(_LIBCPP_CXX03_LANG) && __has_builtin(__builtin_va_copy)
+  __builtin_va_copy(ap_copy, ap);
+#else
+  va_copy(ap_copy, ap);
+#endif
+  int str_size = vsnprintf(*strp, buff_size, fmt, ap_copy);
+  va_end(ap_copy);
+
+  if ((size_t)str_size >= buff_size) {
+    if ((*strp = (char*)realloc(*strp, str_size + 1)) == NULL) {
+      return -1;
+    }
+    str_size = vsnprintf(*strp, str_size + 1, fmt, ap);
+  }
+  return str_size;
+}
+
+#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_IBM_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/locale_guard.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/locale_guard.h
new file mode 100644
index 000000000000..2baacb51cd06
--- /dev/null
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/locale_guard.h
@@ -0,0 +1,78 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_LOCALE_GUARD_H
+#define _LIBCPP___LOCALE_LOCALE_BASE_API_LOCALE_GUARD_H
+
+#include <__config>
+#include <__locale> // for locale_t
+#include <clocale>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if !defined(_LIBCPP_LOCALE__L_EXTENSIONS)
+struct __libcpp_locale_guard {
+  _LIBCPP_HIDE_FROM_ABI __libcpp_locale_guard(locale_t& __loc) : __old_loc_(uselocale(__loc)) {}
+
+  _LIBCPP_HIDE_FROM_ABI ~__libcpp_locale_guard() {
+    if (__old_loc_)
+      uselocale(__old_loc_);
+  }
+
+  locale_t __old_loc_;
+
+  __libcpp_locale_guard(__libcpp_locale_guard const&)            = delete;
+  __libcpp_locale_guard& operator=(__libcpp_locale_guard const&) = delete;
+};
+#elif defined(_LIBCPP_MSVCRT_LIKE)
+struct __libcpp_locale_guard {
+  __libcpp_locale_guard(locale_t __l) : __status(_configthreadlocale(_ENABLE_PER_THREAD_LOCALE)) {
+    // Setting the locale can be expensive even when the locale given is
+    // already the current locale, so do an explicit check to see if the
+    // current locale is already the one we want.
+    const char* __lc = __setlocale(nullptr);
+    // If every category is the same, the locale string will simply be the
+    // locale name, otherwise it will be a semicolon-separated string listing
+    // each category.  In the second case, we know at least one category won't
+    // be what we want, so we only have to check the first case.
+    if (std::strcmp(__l.__get_locale(), __lc) != 0) {
+      __locale_all = _strdup(__lc);
+      if (__locale_all == nullptr)
+        __throw_bad_alloc();
+      __setlocale(__l.__get_locale());
+    }
+  }
+  ~__libcpp_locale_guard() {
+    // The CRT documentation doesn't explicitly say, but setlocale() does the
+    // right thing when given a semicolon-separated list of locale settings
+    // for the different categories in the same format as returned by
+    // setlocale(LC_ALL, nullptr).
+    if (__locale_all != nullptr) {
+      __setlocale(__locale_all);
+      free(__locale_all);
+    }
+    _configthreadlocale(__status);
+  }
+  static const char* __setlocale(const char* __locale) {
+    const char* __new_locale = setlocale(LC_ALL, __locale);
+    if (__new_locale == nullptr)
+      __throw_bad_alloc();
+    return __new_locale;
+  }
+  int __status;
+  char* __locale_all = nullptr;
+};
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_LOCALE_GUARD_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/musl.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/musl.h
new file mode 100644
index 000000000000..bf7b849d5863
--- /dev/null
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/musl.h
@@ -0,0 +1,31 @@
+// -*- C++ -*-
+//===-----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// This adds support for the extended locale functions that are currently
+// missing from the Musl C library.
+//
+// This only works when the specified locale is "C" or "POSIX", but that's
+// about as good as we can do without implementing full xlocale support
+// in Musl.
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_MUSL_H
+#define _LIBCPP___LOCALE_LOCALE_BASE_API_MUSL_H
+
+#include <cstdlib>
+#include <cwchar>
+
+inline _LIBCPP_HIDE_FROM_ABI long long strtoll_l(const char* __nptr, char** __endptr, int __base, locale_t) {
+  return ::strtoll(__nptr, __endptr, __base);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI unsigned long long strtoull_l(const char* __nptr, char** __endptr, int __base, locale_t) {
+  return ::strtoull(__nptr, __endptr, __base);
+}
+
+#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_MUSL_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/newlib.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/newlib.h
new file mode 100644
index 000000000000..a8c1cff16e6d
--- /dev/null
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/newlib.h
@@ -0,0 +1,12 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_NEWLIB_H
+#define _LIBCPP___LOCALE_LOCALE_BASE_API_NEWLIB_H
+
+#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_NEWLIB_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/openbsd.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/openbsd.h
new file mode 100644
index 000000000000..0c05d6a0f788
--- /dev/null
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/openbsd.h
@@ -0,0 +1,19 @@
+// -*- C++ -*-
+//===-----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_OPENBSD_H
+#define _LIBCPP___LOCALE_LOCALE_BASE_API_OPENBSD_H
+
+#include <__support/xlocale/__strtonum_fallback.h>
+#include <clocale>
+#include <cstdlib>
+#include <ctype.h>
+#include <cwctype>
+
+#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_OPENBSD_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/win32.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/win32.h
new file mode 100644
index 000000000000..f66baffb6920
--- /dev/null
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/win32.h
@@ -0,0 +1,235 @@
+// -*- C++ -*-
+//===-----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_WIN32_H
+#define _LIBCPP___LOCALE_LOCALE_BASE_API_WIN32_H
+
+#include <__config>
+#include <cstddef>
+#include <locale.h> // _locale_t
+#include <stdio.h>
+#include <string>
+
+#define _X_ALL LC_ALL
+#define _X_COLLATE LC_COLLATE
+#define _X_CTYPE LC_CTYPE
+#define _X_MONETARY LC_MONETARY
+#define _X_NUMERIC LC_NUMERIC
+#define _X_TIME LC_TIME
+#define _X_MAX LC_MAX
+#define _X_MESSAGES 6
+#define _NCAT (_X_MESSAGES + 1)
+
+#define _CATMASK(n) ((1 << (n)) >> 1)
+#define _M_COLLATE _CATMASK(_X_COLLATE)
+#define _M_CTYPE _CATMASK(_X_CTYPE)
+#define _M_MONETARY _CATMASK(_X_MONETARY)
+#define _M_NUMERIC _CATMASK(_X_NUMERIC)
+#define _M_TIME _CATMASK(_X_TIME)
+#define _M_MESSAGES _CATMASK(_X_MESSAGES)
+#define _M_ALL (_CATMASK(_NCAT) - 1)
+
+#define LC_COLLATE_MASK _M_COLLATE
+#define LC_CTYPE_MASK _M_CTYPE
+#define LC_MONETARY_MASK _M_MONETARY
+#define LC_NUMERIC_MASK _M_NUMERIC
+#define LC_TIME_MASK _M_TIME
+#define LC_MESSAGES_MASK _M_MESSAGES
+#define LC_ALL_MASK                                                                                                    \
+  (LC_COLLATE_MASK | LC_CTYPE_MASK | LC_MESSAGES_MASK | LC_MONETARY_MASK | LC_NUMERIC_MASK | LC_TIME_MASK)
+
+class __lconv_storage {
+public:
+  __lconv_storage(const lconv* __lc_input) {
+    __lc_ = *__lc_input;
+
+    __decimal_point_     = __lc_input->decimal_point;
+    __thousands_sep_     = __lc_input->thousands_sep;
+    __grouping_          = __lc_input->grouping;
+    __int_curr_symbol_   = __lc_input->int_curr_symbol;
+    __currency_symbol_   = __lc_input->currency_symbol;
+    __mon_decimal_point_ = __lc_input->mon_decimal_point;
+    __mon_thousands_sep_ = __lc_input->mon_thousands_sep;
+    __mon_grouping_      = __lc_input->mon_grouping;
+    __positive_sign_     = __lc_input->positive_sign;
+    __negative_sign_     = __lc_input->negative_sign;
+
+    __lc_.decimal_point     = const_cast<char*>(__decimal_point_.c_str());
+    __lc_.thousands_sep     = const_cast<char*>(__thousands_sep_.c_str());
+    __lc_.grouping          = const_cast<char*>(__grouping_.c_str());
+    __lc_.int_curr_symbol   = const_cast<char*>(__int_curr_symbol_.c_str());
+    __lc_.currency_symbol   = const_cast<char*>(__currency_symbol_.c_str());
+    __lc_.mon_decimal_point = const_cast<char*>(__mon_decimal_point_.c_str());
+    __lc_.mon_thousands_sep = const_cast<char*>(__mon_thousands_sep_.c_str());
+    __lc_.mon_grouping      = const_cast<char*>(__mon_grouping_.c_str());
+    __lc_.positive_sign     = const_cast<char*>(__positive_sign_.c_str());
+    __lc_.negative_sign     = const_cast<char*>(__negative_sign_.c_str());
+  }
+
+  lconv* __get() { return &__lc_; }
+
+private:
+  lconv __lc_;
+  std::string __decimal_point_;
+  std::string __thousands_sep_;
+  std::string __grouping_;
+  std::string __int_curr_symbol_;
+  std::string __currency_symbol_;
+  std::string __mon_decimal_point_;
+  std::string __mon_thousands_sep_;
+  std::string __mon_grouping_;
+  std::string __positive_sign_;
+  std::string __negative_sign_;
+};
+
+class locale_t {
+public:
+  locale_t() : __locale_(nullptr), __locale_str_(nullptr), __lc_(nullptr) {}
+  locale_t(std::nullptr_t) : __locale_(nullptr), __locale_str_(nullptr), __lc_(nullptr) {}
+  locale_t(_locale_t __xlocale, const char* __xlocale_str)
+      : __locale_(__xlocale), __locale_str_(__xlocale_str), __lc_(nullptr) {}
+  locale_t(const locale_t& __l) : __locale_(__l.__locale_), __locale_str_(__l.__locale_str_), __lc_(nullptr) {}
+
+  ~locale_t() { delete __lc_; }
+
+  locale_t& operator=(const locale_t& __l) {
+    __locale_     = __l.__locale_;
+    __locale_str_ = __l.__locale_str_;
+    // __lc_ not copied
+    return *this;
+  }
+
+  friend bool operator==(const locale_t& __left, const locale_t& __right) {
+    return __left.__locale_ == __right.__locale_;
+  }
+
+  friend bool operator==(const locale_t& __left, int __right) { return __left.__locale_ == nullptr && __right == 0; }
+
+  friend bool operator==(const locale_t& __left, long long __right) {
+    return __left.__locale_ == nullptr && __right == 0;
+  }
+
+  friend bool operator==(const locale_t& __left, std::nullptr_t) { return __left.__locale_ == nullptr; }
+
+  friend bool operator==(int __left, const locale_t& __right) { return __left == 0 && nullptr == __right.__locale_; }
+
+  friend bool operator==(std::nullptr_t, const locale_t& __right) { return nullptr == __right.__locale_; }
+
+  friend bool operator!=(const locale_t& __left, const locale_t& __right) { return !(__left == __right); }
+
+  friend bool operator!=(const locale_t& __left, int __right) { return !(__left == __right); }
+
+  friend bool operator!=(const locale_t& __left, long long __right) { return !(__left == __right); }
+
+  friend bool operator!=(const locale_t& __left, std::nullptr_t __right) { return !(__left == __right); }
+
+  friend bool operator!=(int __left, const locale_t& __right) { return !(__left == __right); }
+
+  friend bool operator!=(std::nullptr_t __left, const locale_t& __right) { return !(__left == __right); }
+
+  operator bool() const { return __locale_ != nullptr; }
+
+  const char* __get_locale() const { return __locale_str_; }
+
+  operator _locale_t() const { return __locale_; }
+
+  lconv* __store_lconv(const lconv* __input_lc) {
+    delete __lc_;
+    __lc_ = new __lconv_storage(__input_lc);
+    return __lc_->__get();
+  }
+
+private:
+  _locale_t __locale_;
+  const char* __locale_str_;
+  __lconv_storage* __lc_ = nullptr;
+};
+
+// Locale management functions
+#define freelocale _free_locale
+// FIXME: base currently unused. Needs manual work to construct the new locale
+locale_t newlocale(int __mask, const char* __locale, locale_t __base);
+// uselocale can't be implemented on Windows because Windows allows partial modification
+// of thread-local locale and so _get_current_locale() returns a copy while uselocale does
+// not create any copies.
+// We can still implement raii even without uselocale though.
+
+lconv* localeconv_l(locale_t& __loc);
+size_t mbrlen_l(const char* __restrict __s, size_t __n, mbstate_t* __restrict __ps, locale_t __loc);
+size_t mbsrtowcs_l(
+    wchar_t* __restrict __dst, const char** __restrict __src, size_t __len, mbstate_t* __restrict __ps, locale_t __loc);
+size_t wcrtomb_l(char* __restrict __s, wchar_t __wc, mbstate_t* __restrict __ps, locale_t __loc);
+size_t mbrtowc_l(
+    wchar_t* __restrict __pwc, const char* __restrict __s, size_t __n, mbstate_t* __restrict __ps, locale_t __loc);
+size_t mbsnrtowcs_l(wchar_t* __restrict __dst,
+                    const char** __restrict __src,
+                    size_t __nms,
+                    size_t __len,
+                    mbstate_t* __restrict __ps,
+                    locale_t __loc);
+size_t wcsnrtombs_l(char* __restrict __dst,
+                    const wchar_t** __restrict __src,
+                    size_t __nwc,
+                    size_t __len,
+                    mbstate_t* __restrict __ps,
+                    locale_t __loc);
+wint_t btowc_l(int __c, locale_t __loc);
+int wctob_l(wint_t __c, locale_t __loc);
+
+decltype(MB_CUR_MAX) MB_CUR_MAX_L(locale_t __l);
+
+// the *_l functions are prefixed on Windows, only available for msvcr80+, VS2005+
+#define mbtowc_l _mbtowc_l
+#define strtoll_l _strtoi64_l
+#define strtoull_l _strtoui64_l
+#define strtod_l _strtod_l
+#if defined(_LIBCPP_MSVCRT)
+#  define strtof_l _strtof_l
+#  define strtold_l _strtold_l
+#else
+_LIBCPP_EXPORTED_FROM_ABI float strtof_l(const char*, char**, locale_t);
+_LIBCPP_EXPORTED_FROM_ABI long double strtold_l(const char*, char**, locale_t);
+#endif
+inline _LIBCPP_HIDE_FROM_ABI int islower_l(int __c, _locale_t __loc) { return _islower_l((int)__c, __loc); }
+
+inline _LIBCPP_HIDE_FROM_ABI int isupper_l(int __c, _locale_t __loc) { return _isupper_l((int)__c, __loc); }
+
+#define isdigit_l _isdigit_l
+#define isxdigit_l _isxdigit_l
+#define strcoll_l _strcoll_l
+#define strxfrm_l _strxfrm_l
+#define wcscoll_l _wcscoll_l
+#define wcsxfrm_l _wcsxfrm_l
+#define toupper_l _toupper_l
+#define tolower_l _tolower_l
+#define iswspace_l _iswspace_l
+#define iswprint_l _iswprint_l
+#define iswcntrl_l _iswcntrl_l
+#define iswupper_l _iswupper_l
+#define iswlower_l _iswlower_l
+#define iswalpha_l _iswalpha_l
+#define iswdigit_l _iswdigit_l
+#define iswpunct_l _iswpunct_l
+#define iswxdigit_l _iswxdigit_l
+#define towupper_l _towupper_l
+#define towlower_l _towlower_l
+#if defined(__MINGW32__) && __MSVCRT_VERSION__ < 0x0800
+_LIBCPP_EXPORTED_FROM_ABI size_t strftime_l(char* ret, size_t n, const char* format, const struct tm* tm, locale_t loc);
+#else
+#  define strftime_l _strftime_l
+#endif
+#define sscanf_l(__s, __l, __f, ...) _sscanf_l(__s, __f, __l, __VA_ARGS__)
+_LIBCPP_EXPORTED_FROM_ABI int snprintf_l(char* __ret, size_t __n, locale_t __loc, const char* __format, ...);
+_LIBCPP_EXPORTED_FROM_ABI int asprintf_l(char** __ret, locale_t __loc, const char* __format, ...);
+_LIBCPP_EXPORTED_FROM_ABI int vasprintf_l(char** __ret, locale_t __loc, const char* __format, va_list __ap);
+
+// not-so-pressing FIXME: use locale to determine blank characters
+inline int iswblank_l(wint_t __c, locale_t /*loc*/) { return (__c == L' ' || __c == L'\t'); }
+
+#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_WIN32_H
diff --git a/libcxx/include/__cxx03/__math/abs.h b/libcxx/include/__cxx03/__math/abs.h
new file mode 100644
index 000000000000..ab82a2800f53
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/abs.h
@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_ABS_H
+#define _LIBCPP___MATH_ABS_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_integral.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// fabs
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI float fabs(float __x) _NOEXCEPT { return __builtin_fabsf(__x); }
+
+template <class = int>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI double fabs(double __x) _NOEXCEPT {
+  return __builtin_fabs(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI long double fabs(long double __x) _NOEXCEPT {
+  return __builtin_fabsl(__x);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI double fabs(_A1 __x) _NOEXCEPT {
+  return __builtin_fabs((double)__x);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_ABS_H
diff --git a/libcxx/include/__cxx03/__math/copysign.h b/libcxx/include/__cxx03/__math/copysign.h
new file mode 100644
index 000000000000..b38690bb581a
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/copysign.h
@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_COPYSIGN_H
+#define _LIBCPP___MATH_COPYSIGN_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/promote.h>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// copysign
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI float copysign(float __x, float __y) _NOEXCEPT {
+  return ::__builtin_copysignf(__x, __y);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI long double copysign(long double __x, long double __y) _NOEXCEPT {
+  return ::__builtin_copysignl(__x, __y);
+}
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::type copysign(_A1 __x, _A2 __y) _NOEXCEPT {
+  return ::__builtin_copysign(__x, __y);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_COPYSIGN_H
diff --git a/libcxx/include/__cxx03/__math/error_functions.h b/libcxx/include/__cxx03/__math/error_functions.h
new file mode 100644
index 000000000000..6b528bb29000
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/error_functions.h
@@ -0,0 +1,60 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_ERROR_FUNCTIONS_H
+#define _LIBCPP___MATH_ERROR_FUNCTIONS_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_integral.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// erf
+
+inline _LIBCPP_HIDE_FROM_ABI float erf(float __x) _NOEXCEPT { return __builtin_erff(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double erf(double __x) _NOEXCEPT {
+  return __builtin_erf(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double erf(long double __x) _NOEXCEPT { return __builtin_erfl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double erf(_A1 __x) _NOEXCEPT {
+  return __builtin_erf((double)__x);
+}
+
+// erfc
+
+inline _LIBCPP_HIDE_FROM_ABI float erfc(float __x) _NOEXCEPT { return __builtin_erfcf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double erfc(double __x) _NOEXCEPT {
+  return __builtin_erfc(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double erfc(long double __x) _NOEXCEPT { return __builtin_erfcl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double erfc(_A1 __x) _NOEXCEPT {
+  return __builtin_erfc((double)__x);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_ERROR_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__math/exponential_functions.h b/libcxx/include/__cxx03/__math/exponential_functions.h
new file mode 100644
index 000000000000..109c3349970f
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/exponential_functions.h
@@ -0,0 +1,171 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_EXPONENTIAL_FUNCTIONS_H
+#define _LIBCPP___MATH_EXPONENTIAL_FUNCTIONS_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/promote.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// exp
+
+inline _LIBCPP_HIDE_FROM_ABI float exp(float __x) _NOEXCEPT { return __builtin_expf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double exp(double __x) _NOEXCEPT {
+  return __builtin_exp(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double exp(long double __x) _NOEXCEPT { return __builtin_expl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double exp(_A1 __x) _NOEXCEPT {
+  return __builtin_exp((double)__x);
+}
+
+// frexp
+
+inline _LIBCPP_HIDE_FROM_ABI float frexp(float __x, int* __e) _NOEXCEPT { return __builtin_frexpf(__x, __e); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double frexp(double __x, int* __e) _NOEXCEPT {
+  return __builtin_frexp(__x, __e);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double frexp(long double __x, int* __e) _NOEXCEPT {
+  return __builtin_frexpl(__x, __e);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double frexp(_A1 __x, int* __e) _NOEXCEPT {
+  return __builtin_frexp((double)__x, __e);
+}
+
+// ldexp
+
+inline _LIBCPP_HIDE_FROM_ABI float ldexp(float __x, int __e) _NOEXCEPT { return __builtin_ldexpf(__x, __e); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double ldexp(double __x, int __e) _NOEXCEPT {
+  return __builtin_ldexp(__x, __e);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double ldexp(long double __x, int __e) _NOEXCEPT {
+  return __builtin_ldexpl(__x, __e);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double ldexp(_A1 __x, int __e) _NOEXCEPT {
+  return __builtin_ldexp((double)__x, __e);
+}
+
+// exp2
+
+inline _LIBCPP_HIDE_FROM_ABI float exp2(float __x) _NOEXCEPT { return __builtin_exp2f(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double exp2(double __x) _NOEXCEPT {
+  return __builtin_exp2(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double exp2(long double __x) _NOEXCEPT { return __builtin_exp2l(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double exp2(_A1 __x) _NOEXCEPT {
+  return __builtin_exp2((double)__x);
+}
+
+// expm1
+
+inline _LIBCPP_HIDE_FROM_ABI float expm1(float __x) _NOEXCEPT { return __builtin_expm1f(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double expm1(double __x) _NOEXCEPT {
+  return __builtin_expm1(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double expm1(long double __x) _NOEXCEPT { return __builtin_expm1l(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double expm1(_A1 __x) _NOEXCEPT {
+  return __builtin_expm1((double)__x);
+}
+
+// scalbln
+
+inline _LIBCPP_HIDE_FROM_ABI float scalbln(float __x, long __y) _NOEXCEPT { return __builtin_scalblnf(__x, __y); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double scalbln(double __x, long __y) _NOEXCEPT {
+  return __builtin_scalbln(__x, __y);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double scalbln(long double __x, long __y) _NOEXCEPT {
+  return __builtin_scalblnl(__x, __y);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double scalbln(_A1 __x, long __y) _NOEXCEPT {
+  return __builtin_scalbln((double)__x, __y);
+}
+
+// scalbn
+
+inline _LIBCPP_HIDE_FROM_ABI float scalbn(float __x, int __y) _NOEXCEPT { return __builtin_scalbnf(__x, __y); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double scalbn(double __x, int __y) _NOEXCEPT {
+  return __builtin_scalbn(__x, __y);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double scalbn(long double __x, int __y) _NOEXCEPT {
+  return __builtin_scalbnl(__x, __y);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double scalbn(_A1 __x, int __y) _NOEXCEPT {
+  return __builtin_scalbn((double)__x, __y);
+}
+
+// pow
+
+inline _LIBCPP_HIDE_FROM_ABI float pow(float __x, float __y) _NOEXCEPT { return __builtin_powf(__x, __y); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double pow(double __x, double __y) _NOEXCEPT {
+  return __builtin_pow(__x, __y);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double pow(long double __x, long double __y) _NOEXCEPT {
+  return __builtin_powl(__x, __y);
+}
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::type pow(_A1 __x, _A2 __y) _NOEXCEPT {
+  using __result_type = typename __promote<_A1, _A2>::type;
+  static_assert(!(_IsSame<_A1, __result_type>::value && _IsSame<_A2, __result_type>::value), "");
+  return __math::pow((__result_type)__x, (__result_type)__y);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_EXPONENTIAL_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__math/fdim.h b/libcxx/include/__cxx03/__math/fdim.h
new file mode 100644
index 000000000000..dc1b4ecc07dc
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/fdim.h
@@ -0,0 +1,48 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_FDIM_H
+#define _LIBCPP___MATH_FDIM_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/promote.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+inline _LIBCPP_HIDE_FROM_ABI float fdim(float __x, float __y) _NOEXCEPT { return __builtin_fdimf(__x, __y); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double fdim(double __x, double __y) _NOEXCEPT {
+  return __builtin_fdim(__x, __y);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double fdim(long double __x, long double __y) _NOEXCEPT {
+  return __builtin_fdiml(__x, __y);
+}
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::type fdim(_A1 __x, _A2 __y) _NOEXCEPT {
+  using __result_type = typename __promote<_A1, _A2>::type;
+  static_assert(!(_IsSame<_A1, __result_type>::value && _IsSame<_A2, __result_type>::value), "");
+  return __math::fdim((__result_type)__x, (__result_type)__y);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_FDIM_H
diff --git a/libcxx/include/__cxx03/__math/fma.h b/libcxx/include/__cxx03/__math/fma.h
new file mode 100644
index 000000000000..6ba7a5a2d26d
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/fma.h
@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_FMA_H
+#define _LIBCPP___MATH_FMA_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/promote.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+inline _LIBCPP_HIDE_FROM_ABI float fma(float __x, float __y, float __z) _NOEXCEPT {
+  return __builtin_fmaf(__x, __y, __z);
+}
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double fma(double __x, double __y, double __z) _NOEXCEPT {
+  return __builtin_fma(__x, __y, __z);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double fma(long double __x, long double __y, long double __z) _NOEXCEPT {
+  return __builtin_fmal(__x, __y, __z);
+}
+
+template <class _A1,
+          class _A2,
+          class _A3,
+          __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value && is_arithmetic<_A3>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2, _A3>::type fma(_A1 __x, _A2 __y, _A3 __z) _NOEXCEPT {
+  using __result_type = typename __promote<_A1, _A2, _A3>::type;
+  static_assert(
+      !(_IsSame<_A1, __result_type>::value && _IsSame<_A2, __result_type>::value && _IsSame<_A3, __result_type>::value),
+      "");
+  return __builtin_fma((__result_type)__x, (__result_type)__y, (__result_type)__z);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_FMA_H
diff --git a/libcxx/include/__cxx03/__math/gamma.h b/libcxx/include/__cxx03/__math/gamma.h
new file mode 100644
index 000000000000..693e111a84e9
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/gamma.h
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_GAMMA_H
+#define _LIBCPP___MATH_GAMMA_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_integral.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// lgamma
+
+inline _LIBCPP_HIDE_FROM_ABI float lgamma(float __x) _NOEXCEPT { return __builtin_lgammaf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double lgamma(double __x) _NOEXCEPT {
+  return __builtin_lgamma(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double lgamma(long double __x) _NOEXCEPT { return __builtin_lgammal(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double lgamma(_A1 __x) _NOEXCEPT {
+  return __builtin_lgamma((double)__x);
+}
+
+// nan
+
+// tgamma
+
+inline _LIBCPP_HIDE_FROM_ABI float tgamma(float __x) _NOEXCEPT { return __builtin_tgammaf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double tgamma(double __x) _NOEXCEPT {
+  return __builtin_tgamma(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double tgamma(long double __x) _NOEXCEPT { return __builtin_tgammal(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double tgamma(_A1 __x) _NOEXCEPT {
+  return __builtin_tgamma((double)__x);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_GAMMA_H
diff --git a/libcxx/include/__cxx03/__math/hyperbolic_functions.h b/libcxx/include/__cxx03/__math/hyperbolic_functions.h
new file mode 100644
index 000000000000..78832bae70c9
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/hyperbolic_functions.h
@@ -0,0 +1,76 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_HYPERBOLIC_FUNCTIONS_H
+#define _LIBCPP___MATH_HYPERBOLIC_FUNCTIONS_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_integral.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// cosh
+
+inline _LIBCPP_HIDE_FROM_ABI float cosh(float __x) _NOEXCEPT { return __builtin_coshf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double cosh(double __x) _NOEXCEPT {
+  return __builtin_cosh(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double cosh(long double __x) _NOEXCEPT { return __builtin_coshl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double cosh(_A1 __x) _NOEXCEPT {
+  return __builtin_cosh((double)__x);
+}
+
+// sinh
+
+inline _LIBCPP_HIDE_FROM_ABI float sinh(float __x) _NOEXCEPT { return __builtin_sinhf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double sinh(double __x) _NOEXCEPT {
+  return __builtin_sinh(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double sinh(long double __x) _NOEXCEPT { return __builtin_sinhl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double sinh(_A1 __x) _NOEXCEPT {
+  return __builtin_sinh((double)__x);
+}
+
+// tanh
+
+inline _LIBCPP_HIDE_FROM_ABI float tanh(float __x) _NOEXCEPT { return __builtin_tanhf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double tanh(double __x) _NOEXCEPT {
+  return __builtin_tanh(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double tanh(long double __x) _NOEXCEPT { return __builtin_tanhl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double tanh(_A1 __x) _NOEXCEPT {
+  return __builtin_tanh((double)__x);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_HYPERBOLIC_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__math/hypot.h b/libcxx/include/__cxx03/__math/hypot.h
new file mode 100644
index 000000000000..b99216371101
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/hypot.h
@@ -0,0 +1,109 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_HYPOT_H
+#define _LIBCPP___MATH_HYPOT_H
+
+#include <__algorithm/max.h>
+#include <__config>
+#include <__math/abs.h>
+#include <__math/exponential_functions.h>
+#include <__math/roots.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/promote.h>
+#include <__utility/pair.h>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+inline _LIBCPP_HIDE_FROM_ABI float hypot(float __x, float __y) _NOEXCEPT { return __builtin_hypotf(__x, __y); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double hypot(double __x, double __y) _NOEXCEPT {
+  return __builtin_hypot(__x, __y);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double hypot(long double __x, long double __y) _NOEXCEPT {
+  return __builtin_hypotl(__x, __y);
+}
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::type hypot(_A1 __x, _A2 __y) _NOEXCEPT {
+  using __result_type = typename __promote<_A1, _A2>::type;
+  static_assert(!(_IsSame<_A1, __result_type>::value && _IsSame<_A2, __result_type>::value), "");
+  return __math::hypot((__result_type)__x, (__result_type)__y);
+}
+
+#if _LIBCPP_STD_VER >= 17
+// Computes the three-dimensional hypotenuse: `std::hypot(x,y,z)`.
+// The naive implementation might over-/underflow which is why this implementation is more involved:
+//    If the square of an argument might run into issues, we scale the arguments appropriately.
+// See https://github.com/llvm/llvm-project/issues/92782 for a detailed discussion and summary.
+template <class _Real>
+_LIBCPP_HIDE_FROM_ABI _Real __hypot(_Real __x, _Real __y, _Real __z) {
+  // Factors needed to determine if over-/underflow might happen
+  constexpr int __exp              = std::numeric_limits<_Real>::max_exponent / 2;
+  const _Real __overflow_threshold = __math::ldexp(_Real(1), __exp);
+  const _Real __overflow_scale     = __math::ldexp(_Real(1), -(__exp + 20));
+
+  // Scale arguments depending on their size
+  const _Real __max_abs = std::max(__math::fabs(__x), std::max(__math::fabs(__y), __math::fabs(__z)));
+  _Real __scale;
+  if (__max_abs > __overflow_threshold) { // x*x + y*y + z*z might overflow
+    __scale = __overflow_scale;
+  } else if (__max_abs < 1 / __overflow_threshold) { // x*x + y*y + z*z might underflow
+    __scale = 1 / __overflow_scale;
+  } else {
+    __scale = 1;
+  }
+  __x *= __scale;
+  __y *= __scale;
+  __z *= __scale;
+
+  // Compute hypot of scaled arguments and undo scaling
+  return __math::sqrt(__x * __x + __y * __y + __z * __z) / __scale;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI float hypot(float __x, float __y, float __z) { return __math::__hypot(__x, __y, __z); }
+
+inline _LIBCPP_HIDE_FROM_ABI double hypot(double __x, double __y, double __z) { return __math::__hypot(__x, __y, __z); }
+
+inline _LIBCPP_HIDE_FROM_ABI long double hypot(long double __x, long double __y, long double __z) {
+  return __math::__hypot(__x, __y, __z);
+}
+
+template <class _A1,
+          class _A2,
+          class _A3,
+          std::enable_if_t< is_arithmetic_v<_A1> && is_arithmetic_v<_A2> && is_arithmetic_v<_A3>, int> = 0 >
+_LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2, _A3>::type hypot(_A1 __x, _A2 __y, _A3 __z) _NOEXCEPT {
+  using __result_type = typename __promote<_A1, _A2, _A3>::type;
+  static_assert(!(
+      std::is_same_v<_A1, __result_type> && std::is_same_v<_A2, __result_type> && std::is_same_v<_A3, __result_type>));
+  return __math::__hypot(
+      static_cast<__result_type>(__x), static_cast<__result_type>(__y), static_cast<__result_type>(__z));
+}
+#endif
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MATH_HYPOT_H
diff --git a/libcxx/include/__cxx03/__math/inverse_hyperbolic_functions.h b/libcxx/include/__cxx03/__math/inverse_hyperbolic_functions.h
new file mode 100644
index 000000000000..4660a58e4eba
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/inverse_hyperbolic_functions.h
@@ -0,0 +1,76 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_INVERSE_HYPERBOLIC_FUNCTIONS_H
+#define _LIBCPP___MATH_INVERSE_HYPERBOLIC_FUNCTIONS_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_integral.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// acosh
+
+inline _LIBCPP_HIDE_FROM_ABI float acosh(float __x) _NOEXCEPT { return __builtin_acoshf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double acosh(double __x) _NOEXCEPT {
+  return __builtin_acosh(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double acosh(long double __x) _NOEXCEPT { return __builtin_acoshl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double acosh(_A1 __x) _NOEXCEPT {
+  return __builtin_acosh((double)__x);
+}
+
+// asinh
+
+inline _LIBCPP_HIDE_FROM_ABI float asinh(float __x) _NOEXCEPT { return __builtin_asinhf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double asinh(double __x) _NOEXCEPT {
+  return __builtin_asinh(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double asinh(long double __x) _NOEXCEPT { return __builtin_asinhl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double asinh(_A1 __x) _NOEXCEPT {
+  return __builtin_asinh((double)__x);
+}
+
+// atanh
+
+inline _LIBCPP_HIDE_FROM_ABI float atanh(float __x) _NOEXCEPT { return __builtin_atanhf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double atanh(double __x) _NOEXCEPT {
+  return __builtin_atanh(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double atanh(long double __x) _NOEXCEPT { return __builtin_atanhl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double atanh(_A1 __x) _NOEXCEPT {
+  return __builtin_atanh((double)__x);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_INVERSE_HYPERBOLIC_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__math/inverse_trigonometric_functions.h b/libcxx/include/__cxx03/__math/inverse_trigonometric_functions.h
new file mode 100644
index 000000000000..cd98b46a6aab
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/inverse_trigonometric_functions.h
@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_INVERSE_TRIGONOMETRIC_FUNCTIONS_H
+#define _LIBCPP___MATH_INVERSE_TRIGONOMETRIC_FUNCTIONS_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/promote.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// acos
+
+inline _LIBCPP_HIDE_FROM_ABI float acos(float __x) _NOEXCEPT { return __builtin_acosf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double acos(double __x) _NOEXCEPT {
+  return __builtin_acos(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double acos(long double __x) _NOEXCEPT { return __builtin_acosl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double acos(_A1 __x) _NOEXCEPT {
+  return __builtin_acos((double)__x);
+}
+
+// asin
+
+inline _LIBCPP_HIDE_FROM_ABI float asin(float __x) _NOEXCEPT { return __builtin_asinf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double asin(double __x) _NOEXCEPT {
+  return __builtin_asin(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double asin(long double __x) _NOEXCEPT { return __builtin_asinl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double asin(_A1 __x) _NOEXCEPT {
+  return __builtin_asin((double)__x);
+}
+
+// atan
+
+inline _LIBCPP_HIDE_FROM_ABI float atan(float __x) _NOEXCEPT { return __builtin_atanf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double atan(double __x) _NOEXCEPT {
+  return __builtin_atan(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double atan(long double __x) _NOEXCEPT { return __builtin_atanl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double atan(_A1 __x) _NOEXCEPT {
+  return __builtin_atan((double)__x);
+}
+
+// atan2
+
+inline _LIBCPP_HIDE_FROM_ABI float atan2(float __y, float __x) _NOEXCEPT { return __builtin_atan2f(__y, __x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double atan2(double __x, double __y) _NOEXCEPT {
+  return __builtin_atan2(__x, __y);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double atan2(long double __y, long double __x) _NOEXCEPT {
+  return __builtin_atan2l(__y, __x);
+}
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::type atan2(_A1 __y, _A2 __x) _NOEXCEPT {
+  using __result_type = typename __promote<_A1, _A2>::type;
+  static_assert(!(_IsSame<_A1, __result_type>::value && _IsSame<_A2, __result_type>::value), "");
+  return __math::atan2((__result_type)__y, (__result_type)__x);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_INVERSE_TRIGONOMETRIC_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__math/logarithms.h b/libcxx/include/__cxx03/__math/logarithms.h
new file mode 100644
index 000000000000..5f5f943977a5
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/logarithms.h
@@ -0,0 +1,124 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_LOGARITHMS_H
+#define _LIBCPP___MATH_LOGARITHMS_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_integral.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// log
+
+inline _LIBCPP_HIDE_FROM_ABI float log(float __x) _NOEXCEPT { return __builtin_logf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double log(double __x) _NOEXCEPT {
+  return __builtin_log(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double log(long double __x) _NOEXCEPT { return __builtin_logl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double log(_A1 __x) _NOEXCEPT {
+  return __builtin_log((double)__x);
+}
+
+// log10
+
+inline _LIBCPP_HIDE_FROM_ABI float log10(float __x) _NOEXCEPT { return __builtin_log10f(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double log10(double __x) _NOEXCEPT {
+  return __builtin_log10(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double log10(long double __x) _NOEXCEPT { return __builtin_log10l(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double log10(_A1 __x) _NOEXCEPT {
+  return __builtin_log10((double)__x);
+}
+
+// ilogb
+
+inline _LIBCPP_HIDE_FROM_ABI int ilogb(float __x) _NOEXCEPT { return __builtin_ilogbf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double ilogb(double __x) _NOEXCEPT {
+  return __builtin_ilogb(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI int ilogb(long double __x) _NOEXCEPT { return __builtin_ilogbl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI int ilogb(_A1 __x) _NOEXCEPT {
+  return __builtin_ilogb((double)__x);
+}
+
+// log1p
+
+inline _LIBCPP_HIDE_FROM_ABI float log1p(float __x) _NOEXCEPT { return __builtin_log1pf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double log1p(double __x) _NOEXCEPT {
+  return __builtin_log1p(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double log1p(long double __x) _NOEXCEPT { return __builtin_log1pl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double log1p(_A1 __x) _NOEXCEPT {
+  return __builtin_log1p((double)__x);
+}
+
+// log2
+
+inline _LIBCPP_HIDE_FROM_ABI float log2(float __x) _NOEXCEPT { return __builtin_log2f(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double log2(double __x) _NOEXCEPT {
+  return __builtin_log2(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double log2(long double __x) _NOEXCEPT { return __builtin_log2l(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double log2(_A1 __x) _NOEXCEPT {
+  return __builtin_log2((double)__x);
+}
+
+// logb
+
+inline _LIBCPP_HIDE_FROM_ABI float logb(float __x) _NOEXCEPT { return __builtin_logbf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double logb(double __x) _NOEXCEPT {
+  return __builtin_logb(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double logb(long double __x) _NOEXCEPT { return __builtin_logbl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double logb(_A1 __x) _NOEXCEPT {
+  return __builtin_logb((double)__x);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_LOGARITHMS_H
diff --git a/libcxx/include/__cxx03/__math/min_max.h b/libcxx/include/__cxx03/__math/min_max.h
new file mode 100644
index 000000000000..27997b44910a
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/min_max.h
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_MIN_MAX_H
+#define _LIBCPP___MATH_MIN_MAX_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/promote.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// fmax
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI float fmax(float __x, float __y) _NOEXCEPT {
+  return __builtin_fmaxf(__x, __y);
+}
+
+template <class = int>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI double fmax(double __x, double __y) _NOEXCEPT {
+  return __builtin_fmax(__x, __y);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI long double fmax(long double __x, long double __y) _NOEXCEPT {
+  return __builtin_fmaxl(__x, __y);
+}
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::type fmax(_A1 __x, _A2 __y) _NOEXCEPT {
+  using __result_type = typename __promote<_A1, _A2>::type;
+  static_assert(!(_IsSame<_A1, __result_type>::value && _IsSame<_A2, __result_type>::value), "");
+  return __math::fmax((__result_type)__x, (__result_type)__y);
+}
+
+// fmin
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI float fmin(float __x, float __y) _NOEXCEPT {
+  return __builtin_fminf(__x, __y);
+}
+
+template <class = int>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI double fmin(double __x, double __y) _NOEXCEPT {
+  return __builtin_fmin(__x, __y);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI long double fmin(long double __x, long double __y) _NOEXCEPT {
+  return __builtin_fminl(__x, __y);
+}
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::type fmin(_A1 __x, _A2 __y) _NOEXCEPT {
+  using __result_type = typename __promote<_A1, _A2>::type;
+  static_assert(!(_IsSame<_A1, __result_type>::value && _IsSame<_A2, __result_type>::value), "");
+  return __math::fmin((__result_type)__x, (__result_type)__y);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_MIN_MAX_H
diff --git a/libcxx/include/__cxx03/__math/modulo.h b/libcxx/include/__cxx03/__math/modulo.h
new file mode 100644
index 000000000000..c8ea506f37d7
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/modulo.h
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_MODULO_H
+#define _LIBCPP___MATH_MODULO_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/promote.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// fmod
+
+inline _LIBCPP_HIDE_FROM_ABI float fmod(float __x, float __y) _NOEXCEPT { return __builtin_fmodf(__x, __y); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double fmod(double __x, double __y) _NOEXCEPT {
+  return __builtin_fmod(__x, __y);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double fmod(long double __x, long double __y) _NOEXCEPT {
+  return __builtin_fmodl(__x, __y);
+}
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::type fmod(_A1 __x, _A2 __y) _NOEXCEPT {
+  using __result_type = typename __promote<_A1, _A2>::type;
+  static_assert(!(_IsSame<_A1, __result_type>::value && _IsSame<_A2, __result_type>::value), "");
+  return __math::fmod((__result_type)__x, (__result_type)__y);
+}
+
+// modf
+
+inline _LIBCPP_HIDE_FROM_ABI float modf(float __x, float* __y) _NOEXCEPT { return __builtin_modff(__x, __y); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double modf(double __x, double* __y) _NOEXCEPT {
+  return __builtin_modf(__x, __y);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double modf(long double __x, long double* __y) _NOEXCEPT {
+  return __builtin_modfl(__x, __y);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_MODULO_H
diff --git a/libcxx/include/__cxx03/__math/remainder.h b/libcxx/include/__cxx03/__math/remainder.h
new file mode 100644
index 000000000000..0fbf0b8ef97b
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/remainder.h
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_REMAINDER_H
+#define _LIBCPP___MATH_REMAINDER_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/promote.h>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// remainder
+
+inline _LIBCPP_HIDE_FROM_ABI float remainder(float __x, float __y) _NOEXCEPT { return __builtin_remainderf(__x, __y); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double remainder(double __x, double __y) _NOEXCEPT {
+  return __builtin_remainder(__x, __y);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double remainder(long double __x, long double __y) _NOEXCEPT {
+  return __builtin_remainderl(__x, __y);
+}
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::type remainder(_A1 __x, _A2 __y) _NOEXCEPT {
+  using __result_type = typename __promote<_A1, _A2>::type;
+  static_assert(!(_IsSame<_A1, __result_type>::value && _IsSame<_A2, __result_type>::value), "");
+  return __math::remainder((__result_type)__x, (__result_type)__y);
+}
+
+// remquo
+
+inline _LIBCPP_HIDE_FROM_ABI float remquo(float __x, float __y, int* __z) _NOEXCEPT {
+  return __builtin_remquof(__x, __y, __z);
+}
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double remquo(double __x, double __y, int* __z) _NOEXCEPT {
+  return __builtin_remquo(__x, __y, __z);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double remquo(long double __x, long double __y, int* __z) _NOEXCEPT {
+  return __builtin_remquol(__x, __y, __z);
+}
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::type remquo(_A1 __x, _A2 __y, int* __z) _NOEXCEPT {
+  using __result_type = typename __promote<_A1, _A2>::type;
+  static_assert(!(_IsSame<_A1, __result_type>::value && _IsSame<_A2, __result_type>::value), "");
+  return __math::remquo((__result_type)__x, (__result_type)__y, __z);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_REMAINDER_H
diff --git a/libcxx/include/__cxx03/__math/roots.h b/libcxx/include/__cxx03/__math/roots.h
new file mode 100644
index 000000000000..359fd747cfbe
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/roots.h
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_ROOTS_H
+#define _LIBCPP___MATH_ROOTS_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_integral.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// sqrt
+
+inline _LIBCPP_HIDE_FROM_ABI float sqrt(float __x) _NOEXCEPT { return __builtin_sqrtf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double sqrt(double __x) _NOEXCEPT {
+  return __builtin_sqrt(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double sqrt(long double __x) _NOEXCEPT { return __builtin_sqrtl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double sqrt(_A1 __x) _NOEXCEPT {
+  return __builtin_sqrt((double)__x);
+}
+
+// cbrt
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI float cbrt(float __x) _NOEXCEPT { return __builtin_cbrtf(__x); }
+
+template <class = int>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI double cbrt(double __x) _NOEXCEPT {
+  return __builtin_cbrt(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI long double cbrt(long double __x) _NOEXCEPT {
+  return __builtin_cbrtl(__x);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI double cbrt(_A1 __x) _NOEXCEPT {
+  return __builtin_cbrt((double)__x);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_ROOTS_H
diff --git a/libcxx/include/__cxx03/__math/rounding_functions.h b/libcxx/include/__cxx03/__math/rounding_functions.h
new file mode 100644
index 000000000000..f7246ba7fed0
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/rounding_functions.h
@@ -0,0 +1,245 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_ROUNDING_FUNCTIONS_H
+#define _LIBCPP___MATH_ROUNDING_FUNCTIONS_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/promote.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// ceil
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI float ceil(float __x) _NOEXCEPT { return __builtin_ceilf(__x); }
+
+template <class = int>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI double ceil(double __x) _NOEXCEPT {
+  return __builtin_ceil(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI long double ceil(long double __x) _NOEXCEPT {
+  return __builtin_ceill(__x);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI double ceil(_A1 __x) _NOEXCEPT {
+  return __builtin_ceil((double)__x);
+}
+
+// floor
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI float floor(float __x) _NOEXCEPT { return __builtin_floorf(__x); }
+
+template <class = int>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI double floor(double __x) _NOEXCEPT {
+  return __builtin_floor(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI long double floor(long double __x) _NOEXCEPT {
+  return __builtin_floorl(__x);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI double floor(_A1 __x) _NOEXCEPT {
+  return __builtin_floor((double)__x);
+}
+
+// llrint
+
+inline _LIBCPP_HIDE_FROM_ABI long long llrint(float __x) _NOEXCEPT { return __builtin_llrintf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI long long llrint(double __x) _NOEXCEPT {
+  return __builtin_llrint(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long long llrint(long double __x) _NOEXCEPT { return __builtin_llrintl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI long long llrint(_A1 __x) _NOEXCEPT {
+  return __builtin_llrint((double)__x);
+}
+
+// llround
+
+inline _LIBCPP_HIDE_FROM_ABI long long llround(float __x) _NOEXCEPT { return __builtin_llroundf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI long long llround(double __x) _NOEXCEPT {
+  return __builtin_llround(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long long llround(long double __x) _NOEXCEPT { return __builtin_llroundl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI long long llround(_A1 __x) _NOEXCEPT {
+  return __builtin_llround((double)__x);
+}
+
+// lrint
+
+inline _LIBCPP_HIDE_FROM_ABI long lrint(float __x) _NOEXCEPT { return __builtin_lrintf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI long lrint(double __x) _NOEXCEPT {
+  return __builtin_lrint(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long lrint(long double __x) _NOEXCEPT { return __builtin_lrintl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI long lrint(_A1 __x) _NOEXCEPT {
+  return __builtin_lrint((double)__x);
+}
+
+// lround
+
+inline _LIBCPP_HIDE_FROM_ABI long lround(float __x) _NOEXCEPT { return __builtin_lroundf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI long lround(double __x) _NOEXCEPT {
+  return __builtin_lround(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long lround(long double __x) _NOEXCEPT { return __builtin_lroundl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI long lround(_A1 __x) _NOEXCEPT {
+  return __builtin_lround((double)__x);
+}
+
+// nearbyint
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI float nearbyint(float __x) _NOEXCEPT {
+  return __builtin_nearbyintf(__x);
+}
+
+template <class = int>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI double nearbyint(double __x) _NOEXCEPT {
+  return __builtin_nearbyint(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI long double nearbyint(long double __x) _NOEXCEPT {
+  return __builtin_nearbyintl(__x);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI double nearbyint(_A1 __x) _NOEXCEPT {
+  return __builtin_nearbyint((double)__x);
+}
+
+// nextafter
+
+inline _LIBCPP_HIDE_FROM_ABI float nextafter(float __x, float __y) _NOEXCEPT { return __builtin_nextafterf(__x, __y); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double nextafter(double __x, double __y) _NOEXCEPT {
+  return __builtin_nextafter(__x, __y);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double nextafter(long double __x, long double __y) _NOEXCEPT {
+  return __builtin_nextafterl(__x, __y);
+}
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::type nextafter(_A1 __x, _A2 __y) _NOEXCEPT {
+  using __result_type = typename __promote<_A1, _A2>::type;
+  static_assert(!(_IsSame<_A1, __result_type>::value && _IsSame<_A2, __result_type>::value), "");
+  return __math::nextafter((__result_type)__x, (__result_type)__y);
+}
+
+// nexttoward
+
+inline _LIBCPP_HIDE_FROM_ABI float nexttoward(float __x, long double __y) _NOEXCEPT {
+  return __builtin_nexttowardf(__x, __y);
+}
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double nexttoward(double __x, long double __y) _NOEXCEPT {
+  return __builtin_nexttoward(__x, __y);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double nexttoward(long double __x, long double __y) _NOEXCEPT {
+  return __builtin_nexttowardl(__x, __y);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double nexttoward(_A1 __x, long double __y) _NOEXCEPT {
+  return __builtin_nexttoward((double)__x, __y);
+}
+
+// rint
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI float rint(float __x) _NOEXCEPT { return __builtin_rintf(__x); }
+
+template <class = int>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI double rint(double __x) _NOEXCEPT {
+  return __builtin_rint(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI long double rint(long double __x) _NOEXCEPT {
+  return __builtin_rintl(__x);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI double rint(_A1 __x) _NOEXCEPT {
+  return __builtin_rint((double)__x);
+}
+
+// round
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI float round(float __x) _NOEXCEPT { return __builtin_round(__x); }
+
+template <class = int>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI double round(double __x) _NOEXCEPT {
+  return __builtin_round(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI long double round(long double __x) _NOEXCEPT {
+  return __builtin_roundl(__x);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI double round(_A1 __x) _NOEXCEPT {
+  return __builtin_round((double)__x);
+}
+
+// trunc
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI float trunc(float __x) _NOEXCEPT { return __builtin_trunc(__x); }
+
+template <class = int>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI double trunc(double __x) _NOEXCEPT {
+  return __builtin_trunc(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI long double trunc(long double __x) _NOEXCEPT {
+  return __builtin_truncl(__x);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI double trunc(_A1 __x) _NOEXCEPT {
+  return __builtin_trunc((double)__x);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_ROUNDING_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__math/special_functions.h b/libcxx/include/__cxx03/__math/special_functions.h
new file mode 100644
index 000000000000..0b1c753a659a
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/special_functions.h
@@ -0,0 +1,84 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_SPECIAL_FUNCTIONS_H
+#define _LIBCPP___MATH_SPECIAL_FUNCTIONS_H
+
+#include <__config>
+#include <__math/copysign.h>
+#include <__math/traits.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_integral.h>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _Real>
+_LIBCPP_HIDE_FROM_ABI _Real __hermite(unsigned __n, _Real __x) {
+  // The Hermite polynomial H_n(x).
+  // The implementation is based on the recurrence formula: H_{n+1}(x) = 2x H_n(x) - 2n H_{n-1}.
+  // Press, William H., et al. Numerical recipes 3rd edition: The art of scientific computing.
+  // Cambridge university press, 2007, p. 183.
+
+  // NOLINTBEGIN(readability-identifier-naming)
+  if (__math::isnan(__x))
+    return __x;
+
+  _Real __H_0{1};
+  if (__n == 0)
+    return __H_0;
+
+  _Real __H_n_prev = __H_0;
+  _Real __H_n      = 2 * __x;
+  for (unsigned __i = 1; __i < __n; ++__i) {
+    _Real __H_n_next = 2 * (__x * __H_n - __i * __H_n_prev);
+    __H_n_prev       = __H_n;
+    __H_n            = __H_n_next;
+  }
+
+  if (!__math::isfinite(__H_n)) {
+    // Overflow occured. Two possible cases:
+    //    n is odd:  return infinity of the same sign as x.
+    //    n is even: return +Inf
+    _Real __inf = std::numeric_limits<_Real>::infinity();
+    return (__n & 1) ? __math::copysign(__inf, __x) : __inf;
+  }
+  return __H_n;
+  // NOLINTEND(readability-identifier-naming)
+}
+
+inline _LIBCPP_HIDE_FROM_ABI double hermite(unsigned __n, double __x) { return std::__hermite(__n, __x); }
+
+inline _LIBCPP_HIDE_FROM_ABI float hermite(unsigned __n, float __x) {
+  // use double internally -- float is too prone to overflow!
+  return static_cast<float>(std::hermite(__n, static_cast<double>(__x)));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double hermite(unsigned __n, long double __x) { return std::__hermite(__n, __x); }
+
+inline _LIBCPP_HIDE_FROM_ABI float hermitef(unsigned __n, float __x) { return std::hermite(__n, __x); }
+
+inline _LIBCPP_HIDE_FROM_ABI long double hermitel(unsigned __n, long double __x) { return std::hermite(__n, __x); }
+
+template <class _Integer, std::enable_if_t<std::is_integral_v<_Integer>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI double hermite(unsigned __n, _Integer __x) {
+  return std::hermite(__n, static_cast<double>(__x));
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_SPECIAL_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__math/traits.h b/libcxx/include/__cxx03/__math/traits.h
new file mode 100644
index 000000000000..27ec52ecef02
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/traits.h
@@ -0,0 +1,188 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_TRAITS_H
+#define _LIBCPP___MATH_TRAITS_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_floating_point.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_signed.h>
+#include <__type_traits/promote.h>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// signbit
+
+template <class _A1, __enable_if_t<is_floating_point<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI bool signbit(_A1 __x) _NOEXCEPT {
+  return __builtin_signbit(__x);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value && is_signed<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI bool signbit(_A1 __x) _NOEXCEPT {
+  return __x < 0;
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value && !is_signed<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI bool signbit(_A1) _NOEXCEPT {
+  return false;
+}
+
+// isfinite
+
+template <class _A1, __enable_if_t<is_arithmetic<_A1>::value && numeric_limits<_A1>::has_infinity, int> = 0>
+_LIBCPP_NODISCARD _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isfinite(_A1 __x) _NOEXCEPT {
+  return __builtin_isfinite((typename __promote<_A1>::type)__x);
+}
+
+template <class _A1, __enable_if_t<is_arithmetic<_A1>::value && !numeric_limits<_A1>::has_infinity, int> = 0>
+_LIBCPP_NODISCARD _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isfinite(_A1) _NOEXCEPT {
+  return true;
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isfinite(float __x) _NOEXCEPT {
+  return __builtin_isfinite(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isfinite(double __x) _NOEXCEPT {
+  return __builtin_isfinite(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isfinite(long double __x) _NOEXCEPT {
+  return __builtin_isfinite(__x);
+}
+
+// isinf
+
+template <class _A1, __enable_if_t<is_arithmetic<_A1>::value && numeric_limits<_A1>::has_infinity, int> = 0>
+_LIBCPP_NODISCARD _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isinf(_A1 __x) _NOEXCEPT {
+  return __builtin_isinf((typename __promote<_A1>::type)__x);
+}
+
+template <class _A1, __enable_if_t<is_arithmetic<_A1>::value && !numeric_limits<_A1>::has_infinity, int> = 0>
+_LIBCPP_NODISCARD _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isinf(_A1) _NOEXCEPT {
+  return false;
+}
+
+#ifdef _LIBCPP_PREFERRED_OVERLOAD
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isinf(float __x) _NOEXCEPT {
+  return __builtin_isinf(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD bool
+isinf(double __x) _NOEXCEPT {
+  return __builtin_isinf(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isinf(long double __x) _NOEXCEPT {
+  return __builtin_isinf(__x);
+}
+#endif
+
+// isnan
+
+template <class _A1, __enable_if_t<is_floating_point<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isnan(_A1 __x) _NOEXCEPT {
+  return __builtin_isnan(__x);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isnan(_A1) _NOEXCEPT {
+  return false;
+}
+
+#ifdef _LIBCPP_PREFERRED_OVERLOAD
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isnan(float __x) _NOEXCEPT {
+  return __builtin_isnan(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD bool
+isnan(double __x) _NOEXCEPT {
+  return __builtin_isnan(__x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isnan(long double __x) _NOEXCEPT {
+  return __builtin_isnan(__x);
+}
+#endif
+
+// isnormal
+
+template <class _A1, __enable_if_t<is_floating_point<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isnormal(_A1 __x) _NOEXCEPT {
+  return __builtin_isnormal(__x);
+}
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isnormal(_A1 __x) _NOEXCEPT {
+  return __x != 0;
+}
+
+// isgreater
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI bool isgreater(_A1 __x, _A2 __y) _NOEXCEPT {
+  using type = typename __promote<_A1, _A2>::type;
+  return __builtin_isgreater((type)__x, (type)__y);
+}
+
+// isgreaterequal
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI bool isgreaterequal(_A1 __x, _A2 __y) _NOEXCEPT {
+  using type = typename __promote<_A1, _A2>::type;
+  return __builtin_isgreaterequal((type)__x, (type)__y);
+}
+
+// isless
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI bool isless(_A1 __x, _A2 __y) _NOEXCEPT {
+  using type = typename __promote<_A1, _A2>::type;
+  return __builtin_isless((type)__x, (type)__y);
+}
+
+// islessequal
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI bool islessequal(_A1 __x, _A2 __y) _NOEXCEPT {
+  using type = typename __promote<_A1, _A2>::type;
+  return __builtin_islessequal((type)__x, (type)__y);
+}
+
+// islessgreater
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI bool islessgreater(_A1 __x, _A2 __y) _NOEXCEPT {
+  using type = typename __promote<_A1, _A2>::type;
+  return __builtin_islessgreater((type)__x, (type)__y);
+}
+
+// isunordered
+
+template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_arithmetic<_A2>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI bool isunordered(_A1 __x, _A2 __y) _NOEXCEPT {
+  using type = typename __promote<_A1, _A2>::type;
+  return __builtin_isunordered((type)__x, (type)__y);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_TRAITS_H
diff --git a/libcxx/include/__cxx03/__math/trigonometric_functions.h b/libcxx/include/__cxx03/__math/trigonometric_functions.h
new file mode 100644
index 000000000000..0ad91c763160
--- /dev/null
+++ b/libcxx/include/__cxx03/__math/trigonometric_functions.h
@@ -0,0 +1,76 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MATH_TRIGONOMETRIC_FUNCTIONS_H
+#define _LIBCPP___MATH_TRIGONOMETRIC_FUNCTIONS_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_integral.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// cos
+
+inline _LIBCPP_HIDE_FROM_ABI float cos(float __x) _NOEXCEPT { return __builtin_cosf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double cos(double __x) _NOEXCEPT {
+  return __builtin_cos(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double cos(long double __x) _NOEXCEPT { return __builtin_cosl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double cos(_A1 __x) _NOEXCEPT {
+  return __builtin_cos((double)__x);
+}
+
+// sin
+
+inline _LIBCPP_HIDE_FROM_ABI float sin(float __x) _NOEXCEPT { return __builtin_sinf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double sin(double __x) _NOEXCEPT {
+  return __builtin_sin(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double sin(long double __x) _NOEXCEPT { return __builtin_sinl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double sin(_A1 __x) _NOEXCEPT {
+  return __builtin_sin((double)__x);
+}
+
+// tan
+
+inline _LIBCPP_HIDE_FROM_ABI float tan(float __x) _NOEXCEPT { return __builtin_tanf(__x); }
+
+template <class = int>
+_LIBCPP_HIDE_FROM_ABI double tan(double __x) _NOEXCEPT {
+  return __builtin_tan(__x);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double tan(long double __x) _NOEXCEPT { return __builtin_tanl(__x); }
+
+template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double tan(_A1 __x) _NOEXCEPT {
+  return __builtin_tan((double)__x);
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MATH_TRIGONOMETRIC_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__mbstate_t.h b/libcxx/include/__cxx03/__mbstate_t.h
new file mode 100644
index 000000000000..bfa6d617e2b8
--- /dev/null
+++ b/libcxx/include/__cxx03/__mbstate_t.h
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MBSTATE_T_H
+#define _LIBCPP___MBSTATE_T_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+// The goal of this header is to provide mbstate_t without requiring all of
+// <uchar.h> or <wchar.h>. It's also used by the libc++ versions of <uchar.h>
+// and <wchar.h> to get mbstate_t when the C library doesn't provide <uchar.h>
+// or <wchar.h>, hence the #include_next of those headers instead of #include.
+// (e.g. if <wchar.h> isn't present in the C library, the libc++ <wchar.h>
+// will include this header. This header needs to not turn around and cyclically
+// include <wchar.h>, but fall through to <uchar.h>.)
+//
+// This does not define std::mbstate_t -- this only brings in the declaration
+// in the global namespace.
+
+// We define this here to support older versions of glibc <wchar.h> that do
+// not define this for clang. This is also set in libc++'s <wchar.h> header,
+// and we need to do so here too to avoid a different function signature given
+// a different include order.
+#ifdef __cplusplus
+#  define __CORRECT_ISO_CPP_WCHAR_H_PROTO
+#endif
+
+#if defined(_LIBCPP_HAS_MUSL_LIBC)
+#  define __NEED_mbstate_t
+#  include <bits/alltypes.h>
+#  undef __NEED_mbstate_t
+#elif __has_include(<bits/types/mbstate_t.h>)
+#  include <bits/types/mbstate_t.h> // works on most Unixes
+#elif __has_include(<sys/_types/_mbstate_t.h>)
+#  include <sys/_types/_mbstate_t.h> // works on Darwin
+#elif !defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS) && __has_include_next(<wchar.h>)
+#  include_next <wchar.h> // fall back to the C standard provider of mbstate_t
+#elif __has_include_next(<uchar.h>)
+#  include_next <uchar.h> // <uchar.h> is also required to make mbstate_t visible
+#else
+#  error "We don't know how to get the definition of mbstate_t without <wchar.h> on your platform."
+#endif
+
+#endif // _LIBCPP___MBSTATE_T_H
diff --git a/libcxx/include/__cxx03/__mdspan/default_accessor.h b/libcxx/include/__cxx03/__mdspan/default_accessor.h
new file mode 100644
index 000000000000..1cc5f15545fc
--- /dev/null
+++ b/libcxx/include/__cxx03/__mdspan/default_accessor.h
@@ -0,0 +1,66 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//                        Kokkos v. 4.0
+//       Copyright (2022) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MDSPAN_DEFAULT_ACCESSOR_H
+#define _LIBCPP___MDSPAN_DEFAULT_ACCESSOR_H
+
+#include <__config>
+#include <__type_traits/is_abstract.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/remove_const.h>
+#include <cinttypes>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class _ElementType>
+struct default_accessor {
+  static_assert(!is_array_v<_ElementType>, "default_accessor: template argument may not be an array type");
+  static_assert(!is_abstract_v<_ElementType>, "default_accessor: template argument may not be an abstract class");
+
+  using offset_policy    = default_accessor;
+  using element_type     = _ElementType;
+  using reference        = _ElementType&;
+  using data_handle_type = _ElementType*;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr default_accessor() noexcept = default;
+  template <class _OtherElementType>
+    requires(is_convertible_v<_OtherElementType (*)[], element_type (*)[]>)
+  _LIBCPP_HIDE_FROM_ABI constexpr default_accessor(default_accessor<_OtherElementType>) noexcept {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr reference access(data_handle_type __p, size_t __i) const noexcept { return __p[__i]; }
+  _LIBCPP_HIDE_FROM_ABI constexpr data_handle_type offset(data_handle_type __p, size_t __i) const noexcept {
+    return __p + __i;
+  }
+};
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MDSPAN_DEFAULT_ACCESSOR_H
diff --git a/libcxx/include/__cxx03/__mdspan/extents.h b/libcxx/include/__cxx03/__mdspan/extents.h
new file mode 100644
index 000000000000..95082ef3d11a
--- /dev/null
+++ b/libcxx/include/__cxx03/__mdspan/extents.h
@@ -0,0 +1,532 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//                        Kokkos v. 4.0
+//       Copyright (2022) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MDSPAN_EXTENTS_H
+#define _LIBCPP___MDSPAN_EXTENTS_H
+
+#include <__assert>
+#include <__config>
+#include <__type_traits/common_type.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/make_unsigned.h>
+#include <__utility/integer_sequence.h>
+#include <__utility/unreachable.h>
+#include <array>
+#include <cinttypes>
+#include <concepts>
+#include <cstddef>
+#include <limits>
+#include <span>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+namespace __mdspan_detail {
+
+// ------------------------------------------------------------------
+// ------------ __static_array --------------------------------------
+// ------------------------------------------------------------------
+// array like class which provides an array of static values with get
+template <class _Tp, _Tp... _Values>
+struct __static_array {
+  static constexpr array<_Tp, sizeof...(_Values)> __array = {_Values...};
+
+public:
+  _LIBCPP_HIDE_FROM_ABI static constexpr size_t __size() { return sizeof...(_Values); }
+  _LIBCPP_HIDE_FROM_ABI static constexpr _Tp __get(size_t __index) noexcept { return __array[__index]; }
+
+  template <size_t _Index>
+  _LIBCPP_HIDE_FROM_ABI static constexpr _Tp __get() {
+    return __get(_Index);
+  }
+};
+
+// ------------------------------------------------------------------
+// ------------ __possibly_empty_array  -----------------------------
+// ------------------------------------------------------------------
+
+// array like class which provides get function and operator [], and
+// has a specialization for the size 0 case.
+// This is needed to make the __maybe_static_array be truly empty, for
+// all static values.
+
+template <class _Tp, size_t _Size>
+struct __possibly_empty_array {
+  _Tp __vals_[_Size];
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp& operator[](size_t __index) { return __vals_[__index]; }
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& operator[](size_t __index) const { return __vals_[__index]; }
+};
+
+template <class _Tp>
+struct __possibly_empty_array<_Tp, 0> {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp& operator[](size_t) { __libcpp_unreachable(); }
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& operator[](size_t) const { __libcpp_unreachable(); }
+};
+
+// ------------------------------------------------------------------
+// ------------ static_partial_sums ---------------------------------
+// ------------------------------------------------------------------
+
+// Provides a compile time partial sum one can index into
+
+template <size_t... _Values>
+struct __static_partial_sums {
+  _LIBCPP_HIDE_FROM_ABI static constexpr array<size_t, sizeof...(_Values)> __static_partial_sums_impl() {
+    array<size_t, sizeof...(_Values)> __values{_Values...};
+    array<size_t, sizeof...(_Values)> __partial_sums{{}};
+    size_t __running_sum = 0;
+    for (int __i = 0; __i != sizeof...(_Values); ++__i) {
+      __partial_sums[__i] = __running_sum;
+      __running_sum += __values[__i];
+    }
+    return __partial_sums;
+  }
+  static constexpr array<size_t, sizeof...(_Values)> __result{__static_partial_sums_impl()};
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr size_t __get(size_t __index) { return __result[__index]; }
+};
+
+// ------------------------------------------------------------------
+// ------------ __maybe_static_array --------------------------------
+// ------------------------------------------------------------------
+
+// array like class which has a mix of static and runtime values but
+// only stores the runtime values.
+// The type of the static and the runtime values can be different.
+// The position of a dynamic value is indicated through a tag value.
+template <class _TDynamic, class _TStatic, _TStatic _DynTag, _TStatic... _Values>
+struct __maybe_static_array {
+  static_assert(is_convertible<_TStatic, _TDynamic>::value,
+                "__maybe_static_array: _TStatic must be convertible to _TDynamic");
+  static_assert(is_convertible<_TDynamic, _TStatic>::value,
+                "__maybe_static_array: _TDynamic must be convertible to _TStatic");
+
+private:
+  // Static values member
+  static constexpr size_t __size_         = sizeof...(_Values);
+  static constexpr size_t __size_dynamic_ = ((_Values == _DynTag) + ... + 0);
+  using _StaticValues                     = __static_array<_TStatic, _Values...>;
+  using _DynamicValues                    = __possibly_empty_array<_TDynamic, __size_dynamic_>;
+
+  // Dynamic values member
+  _LIBCPP_NO_UNIQUE_ADDRESS _DynamicValues __dyn_vals_;
+
+  // static mapping of indices to the position in the dynamic values array
+  using _DynamicIdxMap = __static_partial_sums<static_cast<size_t>(_Values == _DynTag)...>;
+
+  template <size_t... _Indices>
+  _LIBCPP_HIDE_FROM_ABI static constexpr _DynamicValues __zeros(index_sequence<_Indices...>) noexcept {
+    return _DynamicValues{((void)_Indices, 0)...};
+  }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr __maybe_static_array() noexcept
+      : __dyn_vals_{__zeros(make_index_sequence<__size_dynamic_>())} {}
+
+  // constructors from dynamic values only -- this covers the case for rank() == 0
+  template <class... _DynVals>
+    requires(sizeof...(_DynVals) == __size_dynamic_)
+  _LIBCPP_HIDE_FROM_ABI constexpr __maybe_static_array(_DynVals... __vals)
+      : __dyn_vals_{static_cast<_TDynamic>(__vals)...} {}
+
+  template <class _Tp, size_t _Size >
+    requires(_Size == __size_dynamic_)
+  _LIBCPP_HIDE_FROM_ABI constexpr __maybe_static_array([[maybe_unused]] const span<_Tp, _Size>& __vals) {
+    if constexpr (_Size > 0) {
+      for (size_t __i = 0; __i < _Size; __i++)
+        __dyn_vals_[__i] = static_cast<_TDynamic>(__vals[__i]);
+    }
+  }
+
+  // constructors from all values -- here rank will be greater than 0
+  template <class... _DynVals>
+    requires(sizeof...(_DynVals) != __size_dynamic_)
+  _LIBCPP_HIDE_FROM_ABI constexpr __maybe_static_array(_DynVals... __vals) {
+    static_assert(sizeof...(_DynVals) == __size_, "Invalid number of values.");
+    _TDynamic __values[__size_] = {static_cast<_TDynamic>(__vals)...};
+    for (size_t __i = 0; __i < __size_; __i++) {
+      _TStatic __static_val = _StaticValues::__get(__i);
+      if (__static_val == _DynTag) {
+        __dyn_vals_[_DynamicIdxMap::__get(__i)] = __values[__i];
+      } else
+        // Not catching this could lead to out of bounds errors later
+        // e.g. using my_mdspan_t = mdspan<int, extents<int, 10>>; my_mdspan_t = m(new int[5], 5);
+        // Right-hand-side construction looks ok with allocation and size matching,
+        // but since (potentially elsewhere defined) my_mdspan_t has static size m now thinks its range is 10 not 5
+        _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+            __values[__i] == static_cast<_TDynamic>(__static_val),
+            "extents construction: mismatch of provided arguments with static extents.");
+    }
+  }
+
+  template <class _Tp, size_t _Size>
+    requires(_Size != __size_dynamic_)
+  _LIBCPP_HIDE_FROM_ABI constexpr __maybe_static_array(const span<_Tp, _Size>& __vals) {
+    static_assert(_Size == __size_ || __size_ == dynamic_extent);
+    for (size_t __i = 0; __i < __size_; __i++) {
+      _TStatic __static_val = _StaticValues::__get(__i);
+      if (__static_val == _DynTag) {
+        __dyn_vals_[_DynamicIdxMap::__get(__i)] = static_cast<_TDynamic>(__vals[__i]);
+      } else
+        // Not catching this could lead to out of bounds errors later
+        // e.g. using my_mdspan_t = mdspan<int, extents<int, 10>>; my_mdspan_t = m(new int[N], span<int,1>(&N));
+        // Right-hand-side construction looks ok with allocation and size matching,
+        // but since (potentially elsewhere defined) my_mdspan_t has static size m now thinks its range is 10 not N
+        _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+            static_cast<_TDynamic>(__vals[__i]) == static_cast<_TDynamic>(__static_val),
+            "extents construction: mismatch of provided arguments with static extents.");
+    }
+  }
+
+  // access functions
+  _LIBCPP_HIDE_FROM_ABI static constexpr _TStatic __static_value(size_t __i) noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__i < __size_, "extents access: index must be less than rank");
+    return _StaticValues::__get(__i);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _TDynamic __value(size_t __i) const {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__i < __size_, "extents access: index must be less than rank");
+    _TStatic __static_val = _StaticValues::__get(__i);
+    return __static_val == _DynTag ? __dyn_vals_[_DynamicIdxMap::__get(__i)] : static_cast<_TDynamic>(__static_val);
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr _TDynamic operator[](size_t __i) const {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__i < __size_, "extents access: index must be less than rank");
+    return __value(__i);
+  }
+
+  // observers
+  _LIBCPP_HIDE_FROM_ABI static constexpr size_t __size() { return __size_; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr size_t __size_dynamic() { return __size_dynamic_; }
+};
+
+// Function to check whether a value is representable as another type
+// value must be a positive integer otherwise returns false
+// if _From is not an integral, we just check positivity
+template <integral _To, class _From>
+  requires(integral<_From>)
+_LIBCPP_HIDE_FROM_ABI constexpr bool __is_representable_as(_From __value) {
+  using _To_u   = make_unsigned_t<_To>;
+  using _From_u = make_unsigned_t<_From>;
+  if constexpr (is_signed_v<_From>) {
+    if (__value < 0)
+      return false;
+  }
+  if constexpr (static_cast<_To_u>(numeric_limits<_To>::max()) >= static_cast<_From_u>(numeric_limits<_From>::max())) {
+    return true;
+  } else {
+    return static_cast<_To_u>(numeric_limits<_To>::max()) >= static_cast<_From_u>(__value);
+  }
+}
+
+template <integral _To, class _From>
+  requires(!integral<_From>)
+_LIBCPP_HIDE_FROM_ABI constexpr bool __is_representable_as(_From __value) {
+  if constexpr (is_signed_v<_To>) {
+    if (static_cast<_To>(__value) < 0)
+      return false;
+  }
+  return true;
+}
+
+template <integral _To, class... _From>
+_LIBCPP_HIDE_FROM_ABI constexpr bool __are_representable_as(_From... __values) {
+  return (__mdspan_detail::__is_representable_as<_To>(__values) && ... && true);
+}
+
+template <integral _To, class _From, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI constexpr bool __are_representable_as(span<_From, _Size> __values) {
+  for (size_t __i = 0; __i < _Size; __i++)
+    if (!__mdspan_detail::__is_representable_as<_To>(__values[__i]))
+      return false;
+  return true;
+}
+
+} // namespace __mdspan_detail
+
+// ------------------------------------------------------------------
+// ------------ extents ---------------------------------------------
+// ------------------------------------------------------------------
+
+// Class to describe the extents of a multi dimensional array.
+// Used by mdspan, mdarray and layout mappings.
+// See ISO C++ standard [mdspan.extents]
+
+template <class _IndexType, size_t... _Extents>
+class extents {
+public:
+  // typedefs for integral types used
+  using index_type = _IndexType;
+  using size_type  = make_unsigned_t<index_type>;
+  using rank_type  = size_t;
+
+  static_assert(is_integral<index_type>::value && !is_same<index_type, bool>::value,
+                "extents::index_type must be a signed or unsigned integer type");
+  static_assert(((__mdspan_detail::__is_representable_as<index_type>(_Extents) || (_Extents == dynamic_extent)) && ...),
+                "extents ctor: arguments must be representable as index_type and nonnegative");
+
+private:
+  static constexpr rank_type __rank_         = sizeof...(_Extents);
+  static constexpr rank_type __rank_dynamic_ = ((_Extents == dynamic_extent) + ... + 0);
+
+  // internal storage type using __maybe_static_array
+  using _Values = __mdspan_detail::__maybe_static_array<_IndexType, size_t, dynamic_extent, _Extents...>;
+  [[no_unique_address]] _Values __vals_;
+
+public:
+  // [mdspan.extents.obs], observers of multidimensional index space
+  _LIBCPP_HIDE_FROM_ABI static constexpr rank_type rank() noexcept { return __rank_; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr rank_type rank_dynamic() noexcept { return __rank_dynamic_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr index_type extent(rank_type __r) const noexcept { return __vals_.__value(__r); }
+  _LIBCPP_HIDE_FROM_ABI static constexpr size_t static_extent(rank_type __r) noexcept {
+    return _Values::__static_value(__r);
+  }
+
+  // [mdspan.extents.cons], constructors
+  _LIBCPP_HIDE_FROM_ABI constexpr extents() noexcept = default;
+
+  // Construction from just dynamic or all values.
+  // Precondition check is deferred to __maybe_static_array constructor
+  template <class... _OtherIndexTypes>
+    requires((is_convertible_v<_OtherIndexTypes, index_type> && ...) &&
+             (is_nothrow_constructible_v<index_type, _OtherIndexTypes> && ...) &&
+             (sizeof...(_OtherIndexTypes) == __rank_ || sizeof...(_OtherIndexTypes) == __rank_dynamic_))
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit extents(_OtherIndexTypes... __dynvals) noexcept
+      : __vals_(static_cast<index_type>(__dynvals)...) {
+    // Not catching this could lead to out of bounds errors later
+    // e.g. mdspan m(ptr, dextents<char, 1>(200u)); leads to an extent of -56 on m
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__mdspan_detail::__are_representable_as<index_type>(__dynvals...),
+                                        "extents ctor: arguments must be representable as index_type and nonnegative");
+  }
+
+  template <class _OtherIndexType, size_t _Size>
+    requires(is_convertible_v<const _OtherIndexType&, index_type> &&
+             is_nothrow_constructible_v<index_type, const _OtherIndexType&> &&
+             (_Size == __rank_ || _Size == __rank_dynamic_))
+  explicit(_Size != __rank_dynamic_)
+      _LIBCPP_HIDE_FROM_ABI constexpr extents(const array<_OtherIndexType, _Size>& __exts) noexcept
+      : __vals_(span(__exts)) {
+    // Not catching this could lead to out of bounds errors later
+    // e.g. mdspan m(ptr, dextents<char, 1>(array<unsigned,1>(200))); leads to an extent of -56 on m
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__mdspan_detail::__are_representable_as<index_type>(span(__exts)),
+                                        "extents ctor: arguments must be representable as index_type and nonnegative");
+  }
+
+  template <class _OtherIndexType, size_t _Size>
+    requires(is_convertible_v<const _OtherIndexType&, index_type> &&
+             is_nothrow_constructible_v<index_type, const _OtherIndexType&> &&
+             (_Size == __rank_ || _Size == __rank_dynamic_))
+  explicit(_Size != __rank_dynamic_)
+      _LIBCPP_HIDE_FROM_ABI constexpr extents(const span<_OtherIndexType, _Size>& __exts) noexcept
+      : __vals_(__exts) {
+    // Not catching this could lead to out of bounds errors later
+    // e.g. array a{200u}; mdspan<int, dextents<char,1>> m(ptr, extents(span<unsigned,1>(a))); leads to an extent of -56
+    // on m
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__mdspan_detail::__are_representable_as<index_type>(__exts),
+                                        "extents ctor: arguments must be representable as index_type and nonnegative");
+  }
+
+private:
+  // Function to construct extents storage from other extents.
+  template <size_t _DynCount, size_t _Idx, class _OtherExtents, class... _DynamicValues>
+    requires(_Idx < __rank_)
+  _LIBCPP_HIDE_FROM_ABI constexpr _Values __construct_vals_from_extents(
+      integral_constant<size_t, _DynCount>,
+      integral_constant<size_t, _Idx>,
+      const _OtherExtents& __exts,
+      _DynamicValues... __dynamic_values) noexcept {
+    if constexpr (static_extent(_Idx) == dynamic_extent)
+      return __construct_vals_from_extents(
+          integral_constant<size_t, _DynCount + 1>(),
+          integral_constant<size_t, _Idx + 1>(),
+          __exts,
+          __dynamic_values...,
+          __exts.extent(_Idx));
+    else
+      return __construct_vals_from_extents(
+          integral_constant<size_t, _DynCount>(), integral_constant<size_t, _Idx + 1>(), __exts, __dynamic_values...);
+  }
+
+  template <size_t _DynCount, size_t _Idx, class _OtherExtents, class... _DynamicValues>
+    requires((_Idx == __rank_) && (_DynCount == __rank_dynamic_))
+  _LIBCPP_HIDE_FROM_ABI constexpr _Values __construct_vals_from_extents(
+      integral_constant<size_t, _DynCount>,
+      integral_constant<size_t, _Idx>,
+      const _OtherExtents&,
+      _DynamicValues... __dynamic_values) noexcept {
+    return _Values{static_cast<index_type>(__dynamic_values)...};
+  }
+
+public:
+  // Converting constructor from other extents specializations
+  template <class _OtherIndexType, size_t... _OtherExtents>
+    requires((sizeof...(_OtherExtents) == sizeof...(_Extents)) &&
+             ((_OtherExtents == dynamic_extent || _Extents == dynamic_extent || _OtherExtents == _Extents) && ...))
+  explicit((((_Extents != dynamic_extent) && (_OtherExtents == dynamic_extent)) || ...) ||
+           (static_cast<make_unsigned_t<index_type>>(numeric_limits<index_type>::max()) <
+            static_cast<make_unsigned_t<_OtherIndexType>>(numeric_limits<_OtherIndexType>::max())))
+      _LIBCPP_HIDE_FROM_ABI constexpr extents(const extents<_OtherIndexType, _OtherExtents...>& __other) noexcept
+      : __vals_(
+            __construct_vals_from_extents(integral_constant<size_t, 0>(), integral_constant<size_t, 0>(), __other)) {
+    if constexpr (rank() > 0) {
+      for (size_t __r = 0; __r < rank(); __r++) {
+        if constexpr (static_cast<make_unsigned_t<index_type>>(numeric_limits<index_type>::max()) <
+                      static_cast<make_unsigned_t<_OtherIndexType>>(numeric_limits<_OtherIndexType>::max())) {
+          // Not catching this could lead to out of bounds errors later
+          // e.g. dextents<char,1>> e(dextents<unsigned,1>(200)) leads to an extent of -56 on e
+          _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+              __mdspan_detail::__is_representable_as<index_type>(__other.extent(__r)),
+              "extents ctor: arguments must be representable as index_type and nonnegative");
+        }
+        // Not catching this could lead to out of bounds errors later
+        // e.g. mdspan<int, extents<int, 10>> m = mdspan<int, dextents<int, 1>>(new int[5], 5);
+        // Right-hand-side construction was ok, but m now thinks its range is 10 not 5
+        _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+            (_Values::__static_value(__r) == dynamic_extent) ||
+                (static_cast<index_type>(__other.extent(__r)) == static_cast<index_type>(_Values::__static_value(__r))),
+            "extents construction: mismatch of provided arguments with static extents.");
+      }
+    }
+  }
+
+  // Comparison operator
+  template <class _OtherIndexType, size_t... _OtherExtents>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool
+  operator==(const extents& __lhs, const extents<_OtherIndexType, _OtherExtents...>& __rhs) noexcept {
+    if constexpr (rank() != sizeof...(_OtherExtents)) {
+      return false;
+    } else {
+      for (rank_type __r = 0; __r < __rank_; __r++) {
+        // avoid warning when comparing signed and unsigner integers and pick the wider of two types
+        using _CommonType = common_type_t<index_type, _OtherIndexType>;
+        if (static_cast<_CommonType>(__lhs.extent(__r)) != static_cast<_CommonType>(__rhs.extent(__r))) {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+};
+
+// Recursive helper classes to implement dextents alias for extents
+namespace __mdspan_detail {
+
+template <class _IndexType, size_t _Rank, class _Extents = extents<_IndexType>>
+struct __make_dextents;
+
+template <class _IndexType, size_t _Rank, size_t... _ExtentsPack>
+struct __make_dextents< _IndexType, _Rank, extents<_IndexType, _ExtentsPack...>> {
+  using type =
+      typename __make_dextents< _IndexType, _Rank - 1, extents<_IndexType, dynamic_extent, _ExtentsPack...>>::type;
+};
+
+template <class _IndexType, size_t... _ExtentsPack>
+struct __make_dextents< _IndexType, 0, extents<_IndexType, _ExtentsPack...>> {
+  using type = extents<_IndexType, _ExtentsPack...>;
+};
+
+} // end namespace __mdspan_detail
+
+// [mdspan.extents.dextents], alias template
+template <class _IndexType, size_t _Rank>
+using dextents = typename __mdspan_detail::__make_dextents<_IndexType, _Rank>::type;
+
+#  if _LIBCPP_STD_VER >= 26
+// [mdspan.extents.dims], alias template `dims`
+template <size_t _Rank, class _IndexType = size_t>
+using dims = dextents<_IndexType, _Rank>;
+#  endif
+
+// Deduction guide for extents
+#  if _LIBCPP_STD_VER >= 26
+template <class... _IndexTypes>
+  requires(is_convertible_v<_IndexTypes, size_t> && ...)
+explicit extents(_IndexTypes...) -> extents<size_t, __maybe_static_ext<_IndexTypes>...>;
+#  else
+template <class... _IndexTypes>
+  requires(is_convertible_v<_IndexTypes, size_t> && ...)
+explicit extents(_IndexTypes...) -> extents<size_t, size_t(((void)sizeof(_IndexTypes), dynamic_extent))...>;
+#  endif
+
+namespace __mdspan_detail {
+
+// Helper type traits for identifying a class as extents.
+template <class _Tp>
+struct __is_extents : false_type {};
+
+template <class _IndexType, size_t... _ExtentsPack>
+struct __is_extents<extents<_IndexType, _ExtentsPack...>> : true_type {};
+
+template <class _Tp>
+inline constexpr bool __is_extents_v = __is_extents<_Tp>::value;
+
+// Function to check whether a set of indices are a multidimensional
+// index into extents. This is a word of power in the C++ standard
+// requiring that the indices are larger than 0 and smaller than
+// the respective extents.
+
+template <integral _IndexType, class _From>
+  requires(integral<_From>)
+_LIBCPP_HIDE_FROM_ABI constexpr bool __is_index_in_extent(_IndexType __extent, _From __value) {
+  if constexpr (is_signed_v<_From>) {
+    if (__value < 0)
+      return false;
+  }
+  using _Tp = common_type_t<_IndexType, _From>;
+  return static_cast<_Tp>(__value) < static_cast<_Tp>(__extent);
+}
+
+template <integral _IndexType, class _From>
+  requires(!integral<_From>)
+_LIBCPP_HIDE_FROM_ABI constexpr bool __is_index_in_extent(_IndexType __extent, _From __value) {
+  if constexpr (is_signed_v<_IndexType>) {
+    if (static_cast<_IndexType>(__value) < 0)
+      return false;
+  }
+  return static_cast<_IndexType>(__value) < __extent;
+}
+
+template <size_t... _Idxs, class _Extents, class... _From>
+_LIBCPP_HIDE_FROM_ABI constexpr bool
+__is_multidimensional_index_in_impl(index_sequence<_Idxs...>, const _Extents& __ext, _From... __values) {
+  return (__mdspan_detail::__is_index_in_extent(__ext.extent(_Idxs), __values) && ...);
+}
+
+template <class _Extents, class... _From>
+_LIBCPP_HIDE_FROM_ABI constexpr bool __is_multidimensional_index_in(const _Extents& __ext, _From... __values) {
+  return __mdspan_detail::__is_multidimensional_index_in_impl(
+      make_index_sequence<_Extents::rank()>(), __ext, __values...);
+}
+
+} // namespace __mdspan_detail
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MDSPAN_EXTENTS_H
diff --git a/libcxx/include/__cxx03/__mdspan/layout_left.h b/libcxx/include/__cxx03/__mdspan/layout_left.h
new file mode 100644
index 000000000000..d058cbccffd9
--- /dev/null
+++ b/libcxx/include/__cxx03/__mdspan/layout_left.h
@@ -0,0 +1,204 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//                        Kokkos v. 4.0
+//       Copyright (2022) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MDSPAN_LAYOUT_LEFT_H
+#define _LIBCPP___MDSPAN_LAYOUT_LEFT_H
+
+#include <__assert>
+#include <__config>
+#include <__fwd/mdspan.h>
+#include <__mdspan/extents.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__utility/integer_sequence.h>
+#include <array>
+#include <cinttypes>
+#include <cstddef>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Extents>
+class layout_left::mapping {
+public:
+  static_assert(__mdspan_detail::__is_extents<_Extents>::value,
+                "layout_left::mapping template argument must be a specialization of extents.");
+
+  using extents_type = _Extents;
+  using index_type   = typename extents_type::index_type;
+  using size_type    = typename extents_type::size_type;
+  using rank_type    = typename extents_type::rank_type;
+  using layout_type  = layout_left;
+
+private:
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __required_span_size_is_representable(const extents_type& __ext) {
+    if constexpr (extents_type::rank() == 0)
+      return true;
+
+    index_type __prod = __ext.extent(0);
+    for (rank_type __r = 1; __r < extents_type::rank(); __r++) {
+      bool __overflowed = __builtin_mul_overflow(__prod, __ext.extent(__r), &__prod);
+      if (__overflowed)
+        return false;
+    }
+    return true;
+  }
+
+  static_assert(extents_type::rank_dynamic() > 0 || __required_span_size_is_representable(extents_type()),
+                "layout_left::mapping product of static extents must be representable as index_type.");
+
+public:
+  // [mdspan.layout.left.cons], constructors
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping() noexcept               = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping(const mapping&) noexcept = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping(const extents_type& __ext) noexcept : __extents_(__ext) {
+    // not catching this could lead to out-of-bounds access later when used inside mdspan
+    // mapping<dextents<char, 2>> map(dextents<char, 2>(40,40)); map(10, 3) == -126
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __required_span_size_is_representable(__ext),
+        "layout_left::mapping extents ctor: product of extents must be representable as index_type.");
+  }
+
+  template <class _OtherExtents>
+    requires(is_constructible_v<extents_type, _OtherExtents>)
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_OtherExtents, extents_type>)
+      mapping(const mapping<_OtherExtents>& __other) noexcept
+      : __extents_(__other.extents()) {
+    // not catching this could lead to out-of-bounds access later when used inside mdspan
+    // mapping<dextents<char, 2>> map(mapping<dextents<int, 2>>(dextents<int, 2>(40,40))); map(10, 3) == -126
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __mdspan_detail::__is_representable_as<index_type>(__other.required_span_size()),
+        "layout_left::mapping converting ctor: other.required_span_size() must be representable as index_type.");
+  }
+
+  template <class _OtherExtents>
+    requires(is_constructible_v<extents_type, _OtherExtents> && _OtherExtents::rank() <= 1)
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_OtherExtents, extents_type>)
+      mapping(const layout_right::mapping<_OtherExtents>& __other) noexcept
+      : __extents_(__other.extents()) {
+    // not catching this could lead to out-of-bounds access later when used inside mdspan
+    // Note: since this is constraint to rank 1, extents itself would catch the invalid conversion first
+    //       and thus this assertion should never be triggered, but keeping it here for consistency
+    // layout_left::mapping<dextents<char, 1>> map(
+    //           layout_right::mapping<dextents<unsigned, 1>>(dextents<unsigned, 1>(200))); map.extents().extent(0) ==
+    //           -56
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __mdspan_detail::__is_representable_as<index_type>(__other.required_span_size()),
+        "layout_left::mapping converting ctor: other.required_span_size() must be representable as index_type.");
+  }
+
+  template <class _OtherExtents>
+    requires(is_constructible_v<extents_type, _OtherExtents>)
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(extents_type::rank() > 0)
+      mapping(const layout_stride::mapping<_OtherExtents>& __other) noexcept
+      : __extents_(__other.extents()) {
+    if constexpr (extents_type::rank() > 0) {
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          ([&]() {
+            using _CommonType = common_type_t<typename extents_type::index_type, typename _OtherExtents::index_type>;
+            for (rank_type __r = 0; __r < extents_type::rank(); __r++)
+              if (static_cast<_CommonType>(stride(__r)) != static_cast<_CommonType>(__other.stride(__r)))
+                return false;
+            return true;
+          }()),
+          "layout_left::mapping from layout_stride ctor: strides are not compatible with layout_left.");
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          __mdspan_detail::__is_representable_as<index_type>(__other.required_span_size()),
+          "layout_left::mapping from layout_stride ctor: other.required_span_size() must be representable as "
+          "index_type.");
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping& operator=(const mapping&) noexcept = default;
+
+  // [mdspan.layout.left.obs], observers
+  _LIBCPP_HIDE_FROM_ABI constexpr const extents_type& extents() const noexcept { return __extents_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr index_type required_span_size() const noexcept {
+    index_type __size = 1;
+    for (size_t __r = 0; __r < extents_type::rank(); __r++)
+      __size *= __extents_.extent(__r);
+    return __size;
+  }
+
+  template <class... _Indices>
+    requires((sizeof...(_Indices) == extents_type::rank()) && (is_convertible_v<_Indices, index_type> && ...) &&
+             (is_nothrow_constructible_v<index_type, _Indices> && ...))
+  _LIBCPP_HIDE_FROM_ABI constexpr index_type operator()(_Indices... __idx) const noexcept {
+    // Mappings are generally meant to be used for accessing allocations and are meant to guarantee to never
+    // return a value exceeding required_span_size(), which is used to know how large an allocation one needs
+    // Thus, this is a canonical point in multi-dimensional data structures to make invalid element access checks
+    // However, mdspan does check this on its own, so for now we avoid double checking in hardened mode
+    _LIBCPP_ASSERT_UNCATEGORIZED(__mdspan_detail::__is_multidimensional_index_in(__extents_, __idx...),
+                                 "layout_left::mapping: out of bounds indexing");
+    array<index_type, extents_type::rank()> __idx_a{static_cast<index_type>(__idx)...};
+    return [&]<size_t... _Pos>(index_sequence<_Pos...>) {
+      index_type __res = 0;
+      ((__res = __idx_a[extents_type::rank() - 1 - _Pos] + __extents_.extent(extents_type::rank() - 1 - _Pos) * __res),
+       ...);
+      return __res;
+    }(make_index_sequence<sizeof...(_Indices)>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_unique() noexcept { return true; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_exhaustive() noexcept { return true; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_strided() noexcept { return true; }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_unique() noexcept { return true; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_exhaustive() noexcept { return true; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_strided() noexcept { return true; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr index_type stride(rank_type __r) const noexcept
+    requires(extents_type::rank() > 0)
+  {
+    // While it would be caught by extents itself too, using a too large __r
+    // is functionally an out of bounds access on the stored information needed to compute strides
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __r < extents_type::rank(), "layout_left::mapping::stride(): invalid rank index");
+    index_type __s = 1;
+    for (rank_type __i = 0; __i < __r; __i++)
+      __s *= __extents_.extent(__i);
+    return __s;
+  }
+
+  template <class _OtherExtents>
+    requires(_OtherExtents::rank() == extents_type::rank())
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool
+  operator==(const mapping& __lhs, const mapping<_OtherExtents>& __rhs) noexcept {
+    return __lhs.extents() == __rhs.extents();
+  }
+
+private:
+  _LIBCPP_NO_UNIQUE_ADDRESS extents_type __extents_{};
+};
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MDSPAN_LAYOUT_LEFT_H
diff --git a/libcxx/include/__cxx03/__mdspan/layout_right.h b/libcxx/include/__cxx03/__mdspan/layout_right.h
new file mode 100644
index 000000000000..6842e9dc37fd
--- /dev/null
+++ b/libcxx/include/__cxx03/__mdspan/layout_right.h
@@ -0,0 +1,201 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//                        Kokkos v. 4.0
+//       Copyright (2022) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MDSPAN_LAYOUT_RIGHT_H
+#define _LIBCPP___MDSPAN_LAYOUT_RIGHT_H
+
+#include <__assert>
+#include <__config>
+#include <__fwd/mdspan.h>
+#include <__mdspan/extents.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__utility/integer_sequence.h>
+#include <cinttypes>
+#include <cstddef>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Extents>
+class layout_right::mapping {
+public:
+  static_assert(__mdspan_detail::__is_extents<_Extents>::value,
+                "layout_right::mapping template argument must be a specialization of extents.");
+
+  using extents_type = _Extents;
+  using index_type   = typename extents_type::index_type;
+  using size_type    = typename extents_type::size_type;
+  using rank_type    = typename extents_type::rank_type;
+  using layout_type  = layout_right;
+
+private:
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __required_span_size_is_representable(const extents_type& __ext) {
+    if constexpr (extents_type::rank() == 0)
+      return true;
+
+    index_type __prod = __ext.extent(0);
+    for (rank_type __r = 1; __r < extents_type::rank(); __r++) {
+      bool __overflowed = __builtin_mul_overflow(__prod, __ext.extent(__r), &__prod);
+      if (__overflowed)
+        return false;
+    }
+    return true;
+  }
+
+  static_assert(extents_type::rank_dynamic() > 0 || __required_span_size_is_representable(extents_type()),
+                "layout_right::mapping product of static extents must be representable as index_type.");
+
+public:
+  // [mdspan.layout.right.cons], constructors
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping() noexcept               = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping(const mapping&) noexcept = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping(const extents_type& __ext) noexcept : __extents_(__ext) {
+    // not catching this could lead to out-of-bounds access later when used inside mdspan
+    // mapping<dextents<char, 2>> map(dextents<char, 2>(40,40)); map(3, 10) == -126
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __required_span_size_is_representable(__ext),
+        "layout_right::mapping extents ctor: product of extents must be representable as index_type.");
+  }
+
+  template <class _OtherExtents>
+    requires(is_constructible_v<extents_type, _OtherExtents>)
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_OtherExtents, extents_type>)
+      mapping(const mapping<_OtherExtents>& __other) noexcept
+      : __extents_(__other.extents()) {
+    // not catching this could lead to out-of-bounds access later when used inside mdspan
+    // mapping<dextents<char, 2>> map(mapping<dextents<int, 2>>(dextents<int, 2>(40,40))); map(3, 10) == -126
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __mdspan_detail::__is_representable_as<index_type>(__other.required_span_size()),
+        "layout_right::mapping converting ctor: other.required_span_size() must be representable as index_type.");
+  }
+
+  template <class _OtherExtents>
+    requires(is_constructible_v<extents_type, _OtherExtents> && _OtherExtents::rank() <= 1)
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_OtherExtents, extents_type>)
+      mapping(const layout_left::mapping<_OtherExtents>& __other) noexcept
+      : __extents_(__other.extents()) {
+    // not catching this could lead to out-of-bounds access later when used inside mdspan
+    // Note: since this is constraint to rank 1, extents itself would catch the invalid conversion first
+    //       and thus this assertion should never be triggered, but keeping it here for consistency
+    // layout_right::mapping<dextents<char, 1>> map(
+    //           layout_left::mapping<dextents<unsigned, 1>>(dextents<unsigned, 1>(200))); map.extents().extent(0) ==
+    //           -56
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __mdspan_detail::__is_representable_as<index_type>(__other.required_span_size()),
+        "layout_right::mapping converting ctor: other.required_span_size() must be representable as index_type.");
+  }
+
+  template <class _OtherExtents>
+    requires(is_constructible_v<extents_type, _OtherExtents>)
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(extents_type::rank() > 0)
+      mapping(const layout_stride::mapping<_OtherExtents>& __other) noexcept
+      : __extents_(__other.extents()) {
+    if constexpr (extents_type::rank() > 0) {
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          ([&]() {
+            using _CommonType = common_type_t<typename extents_type::index_type, typename _OtherExtents::index_type>;
+            for (rank_type __r = 0; __r < extents_type::rank(); __r++)
+              if (static_cast<_CommonType>(stride(__r)) != static_cast<_CommonType>(__other.stride(__r)))
+                return false;
+            return true;
+          }()),
+          "layout_right::mapping from layout_stride ctor: strides are not compatible with layout_right.");
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          __mdspan_detail::__is_representable_as<index_type>(__other.required_span_size()),
+          "layout_right::mapping from layout_stride ctor: other.required_span_size() must be representable as "
+          "index_type.");
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping& operator=(const mapping&) noexcept = default;
+
+  // [mdspan.layout.right.obs], observers
+  _LIBCPP_HIDE_FROM_ABI constexpr const extents_type& extents() const noexcept { return __extents_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr index_type required_span_size() const noexcept {
+    index_type __size = 1;
+    for (size_t __r = 0; __r < extents_type::rank(); __r++)
+      __size *= __extents_.extent(__r);
+    return __size;
+  }
+
+  template <class... _Indices>
+    requires((sizeof...(_Indices) == extents_type::rank()) && (is_convertible_v<_Indices, index_type> && ...) &&
+             (is_nothrow_constructible_v<index_type, _Indices> && ...))
+  _LIBCPP_HIDE_FROM_ABI constexpr index_type operator()(_Indices... __idx) const noexcept {
+    // Mappings are generally meant to be used for accessing allocations and are meant to guarantee to never
+    // return a value exceeding required_span_size(), which is used to know how large an allocation one needs
+    // Thus, this is a canonical point in multi-dimensional data structures to make invalid element access checks
+    // However, mdspan does check this on its own, so for now we avoid double checking in hardened mode
+    _LIBCPP_ASSERT_UNCATEGORIZED(__mdspan_detail::__is_multidimensional_index_in(__extents_, __idx...),
+                                 "layout_right::mapping: out of bounds indexing");
+    return [&]<size_t... _Pos>(index_sequence<_Pos...>) {
+      index_type __res = 0;
+      ((__res = static_cast<index_type>(__idx) + __extents_.extent(_Pos) * __res), ...);
+      return __res;
+    }(make_index_sequence<sizeof...(_Indices)>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_unique() noexcept { return true; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_exhaustive() noexcept { return true; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_strided() noexcept { return true; }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_unique() noexcept { return true; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_exhaustive() noexcept { return true; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_strided() noexcept { return true; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr index_type stride(rank_type __r) const noexcept
+    requires(extents_type::rank() > 0)
+  {
+    // While it would be caught by extents itself too, using a too large __r
+    // is functionally an out of bounds access on the stored information needed to compute strides
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __r < extents_type::rank(), "layout_right::mapping::stride(): invalid rank index");
+    index_type __s = 1;
+    for (rank_type __i = extents_type::rank() - 1; __i > __r; __i--)
+      __s *= __extents_.extent(__i);
+    return __s;
+  }
+
+  template <class _OtherExtents>
+    requires(_OtherExtents::rank() == extents_type::rank())
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool
+  operator==(const mapping& __lhs, const mapping<_OtherExtents>& __rhs) noexcept {
+    return __lhs.extents() == __rhs.extents();
+  }
+
+private:
+  _LIBCPP_NO_UNIQUE_ADDRESS extents_type __extents_{};
+};
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MDSPAN_LAYOUT_RIGHT_H
diff --git a/libcxx/include/__cxx03/__mdspan/layout_stride.h b/libcxx/include/__cxx03/__mdspan/layout_stride.h
new file mode 100644
index 000000000000..86148ac849ec
--- /dev/null
+++ b/libcxx/include/__cxx03/__mdspan/layout_stride.h
@@ -0,0 +1,366 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//                        Kokkos v. 4.0
+//       Copyright (2022) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MDSPAN_LAYOUT_STRIDE_H
+#define _LIBCPP___MDSPAN_LAYOUT_STRIDE_H
+
+#include <__assert>
+#include <__config>
+#include <__fwd/mdspan.h>
+#include <__mdspan/extents.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__utility/as_const.h>
+#include <__utility/integer_sequence.h>
+#include <__utility/swap.h>
+#include <array>
+#include <cinttypes>
+#include <cstddef>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+namespace __mdspan_detail {
+template <class _Layout, class _Mapping>
+constexpr bool __is_mapping_of =
+    is_same_v<typename _Layout::template mapping<typename _Mapping::extents_type>, _Mapping>;
+
+template <class _Mapping>
+concept __layout_mapping_alike = requires {
+  requires __is_mapping_of<typename _Mapping::layout_type, _Mapping>;
+  requires __is_extents_v<typename _Mapping::extents_type>;
+  { _Mapping::is_always_strided() } -> same_as<bool>;
+  { _Mapping::is_always_exhaustive() } -> same_as<bool>;
+  { _Mapping::is_always_unique() } -> same_as<bool>;
+  bool_constant<_Mapping::is_always_strided()>::value;
+  bool_constant<_Mapping::is_always_exhaustive()>::value;
+  bool_constant<_Mapping::is_always_unique()>::value;
+};
+} // namespace __mdspan_detail
+
+template <class _Extents>
+class layout_stride::mapping {
+public:
+  static_assert(__mdspan_detail::__is_extents<_Extents>::value,
+                "layout_stride::mapping template argument must be a specialization of extents.");
+
+  using extents_type = _Extents;
+  using index_type   = typename extents_type::index_type;
+  using size_type    = typename extents_type::size_type;
+  using rank_type    = typename extents_type::rank_type;
+  using layout_type  = layout_stride;
+
+private:
+  static constexpr rank_type __rank_ = extents_type::rank();
+
+  // Used for default construction check and mandates
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __required_span_size_is_representable(const extents_type& __ext) {
+    if constexpr (__rank_ == 0)
+      return true;
+
+    index_type __prod = __ext.extent(0);
+    for (rank_type __r = 1; __r < __rank_; __r++) {
+      bool __overflowed = __builtin_mul_overflow(__prod, __ext.extent(__r), &__prod);
+      if (__overflowed)
+        return false;
+    }
+    return true;
+  }
+
+  template <class _OtherIndexType>
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool
+  __required_span_size_is_representable(const extents_type& __ext, span<_OtherIndexType, __rank_> __strides) {
+    if constexpr (__rank_ == 0)
+      return true;
+
+    index_type __size = 1;
+    for (rank_type __r = 0; __r < __rank_; __r++) {
+      // We can only check correct conversion of _OtherIndexType if it is an integral
+      if constexpr (is_integral_v<_OtherIndexType>) {
+        using _CommonType = common_type_t<index_type, _OtherIndexType>;
+        if (static_cast<_CommonType>(__strides[__r]) > static_cast<_CommonType>(numeric_limits<index_type>::max()))
+          return false;
+      }
+      if (__ext.extent(__r) == static_cast<index_type>(0))
+        return true;
+      index_type __prod     = (__ext.extent(__r) - 1);
+      bool __overflowed_mul = __builtin_mul_overflow(__prod, static_cast<index_type>(__strides[__r]), &__prod);
+      if (__overflowed_mul)
+        return false;
+      bool __overflowed_add = __builtin_add_overflow(__size, __prod, &__size);
+      if (__overflowed_add)
+        return false;
+    }
+    return true;
+  }
+
+  // compute offset of a strided layout mapping
+  template <class _StridedMapping>
+  _LIBCPP_HIDE_FROM_ABI static constexpr index_type __offset(const _StridedMapping& __mapping) {
+    if constexpr (_StridedMapping::extents_type::rank() == 0) {
+      return static_cast<index_type>(__mapping());
+    } else if (__mapping.required_span_size() == static_cast<typename _StridedMapping::index_type>(0)) {
+      return static_cast<index_type>(0);
+    } else {
+      return [&]<size_t... _Pos>(index_sequence<_Pos...>) {
+        return static_cast<index_type>(__mapping((_Pos ? 0 : 0)...));
+      }(make_index_sequence<__rank_>());
+    }
+  }
+
+  // compute the permutation for sorting the stride array
+  // we never actually sort the stride array
+  _LIBCPP_HIDE_FROM_ABI constexpr void __bubble_sort_by_strides(array<rank_type, __rank_>& __permute) const {
+    for (rank_type __i = __rank_ - 1; __i > 0; __i--) {
+      for (rank_type __r = 0; __r < __i; __r++) {
+        if (__strides_[__permute[__r]] > __strides_[__permute[__r + 1]]) {
+          swap(__permute[__r], __permute[__r + 1]);
+        } else {
+          // if two strides are the same then one of the associated extents must be 1 or 0
+          // both could be, but you can't have one larger than 1 come first
+          if ((__strides_[__permute[__r]] == __strides_[__permute[__r + 1]]) &&
+              (__extents_.extent(__permute[__r]) > static_cast<index_type>(1)))
+            swap(__permute[__r], __permute[__r + 1]);
+        }
+      }
+    }
+  }
+
+  static_assert(extents_type::rank_dynamic() > 0 || __required_span_size_is_representable(extents_type()),
+                "layout_stride::mapping product of static extents must be representable as index_type.");
+
+public:
+  // [mdspan.layout.stride.cons], constructors
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping() noexcept : __extents_(extents_type()) {
+    // Note the nominal precondition is covered by above static assert since
+    // if rank_dynamic is != 0 required_span_size is zero for default construction
+    if constexpr (__rank_ > 0) {
+      index_type __stride = 1;
+      for (rank_type __r = __rank_ - 1; __r > static_cast<rank_type>(0); __r--) {
+        __strides_[__r] = __stride;
+        __stride *= __extents_.extent(__r);
+      }
+      __strides_[0] = __stride;
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping(const mapping&) noexcept = default;
+
+  template <class _OtherIndexType>
+    requires(is_convertible_v<const _OtherIndexType&, index_type> &&
+             is_nothrow_constructible_v<index_type, const _OtherIndexType&>)
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping(const extents_type& __ext, span<_OtherIndexType, __rank_> __strides) noexcept
+      : __extents_(__ext), __strides_([&]<size_t... _Pos>(index_sequence<_Pos...>) {
+          return __mdspan_detail::__possibly_empty_array<index_type, __rank_>{
+              static_cast<index_type>(std::as_const(__strides[_Pos]))...};
+        }(make_index_sequence<__rank_>())) {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        ([&]<size_t... _Pos>(index_sequence<_Pos...>) {
+          // For integrals we can do a pre-conversion check, for other types not
+          if constexpr (is_integral_v<_OtherIndexType>) {
+            return ((__strides[_Pos] > static_cast<_OtherIndexType>(0)) && ... && true);
+          } else {
+            return ((static_cast<index_type>(__strides[_Pos]) > static_cast<index_type>(0)) && ... && true);
+          }
+        }(make_index_sequence<__rank_>())),
+        "layout_stride::mapping ctor: all strides must be greater than 0");
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __required_span_size_is_representable(__ext, __strides),
+        "layout_stride::mapping ctor: required span size is not representable as index_type.");
+    if constexpr (__rank_ > 1) {
+      _LIBCPP_ASSERT_UNCATEGORIZED(
+          ([&]<size_t... _Pos>(index_sequence<_Pos...>) {
+            // basically sort the dimensions based on strides and extents, sorting is represented in permute array
+            array<rank_type, __rank_> __permute{_Pos...};
+            __bubble_sort_by_strides(__permute);
+
+            // check that this permutations represents a growing set
+            for (rank_type __i = 1; __i < __rank_; __i++)
+              if (static_cast<index_type>(__strides[__permute[__i]]) <
+                  static_cast<index_type>(__strides[__permute[__i - 1]]) * __extents_.extent(__permute[__i - 1]))
+                return false;
+            return true;
+          }(make_index_sequence<__rank_>())),
+          "layout_stride::mapping ctor: the provided extents and strides lead to a non-unique mapping");
+    }
+  }
+
+  template <class _OtherIndexType>
+    requires(is_convertible_v<const _OtherIndexType&, index_type> &&
+             is_nothrow_constructible_v<index_type, const _OtherIndexType&>)
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping(const extents_type& __ext,
+                                          const array<_OtherIndexType, __rank_>& __strides) noexcept
+      : mapping(__ext, span(__strides)) {}
+
+  template <class _StridedLayoutMapping>
+    requires(__mdspan_detail::__layout_mapping_alike<_StridedLayoutMapping> &&
+             is_constructible_v<extents_type, typename _StridedLayoutMapping::extents_type> &&
+             _StridedLayoutMapping::is_always_unique() && _StridedLayoutMapping::is_always_strided())
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(
+      !(is_convertible_v<typename _StridedLayoutMapping::extents_type, extents_type> &&
+        (__mdspan_detail::__is_mapping_of<layout_left, _StridedLayoutMapping> ||
+         __mdspan_detail::__is_mapping_of<layout_right, _StridedLayoutMapping> ||
+         __mdspan_detail::__is_mapping_of<layout_stride, _StridedLayoutMapping>)))
+      mapping(const _StridedLayoutMapping& __other) noexcept
+      : __extents_(__other.extents()), __strides_([&]<size_t... _Pos>(index_sequence<_Pos...>) {
+          // stride() only compiles for rank > 0
+          if constexpr (__rank_ > 0) {
+            return __mdspan_detail::__possibly_empty_array<index_type, __rank_>{
+                static_cast<index_type>(__other.stride(_Pos))...};
+          } else {
+            return __mdspan_detail::__possibly_empty_array<index_type, 0>{};
+          }
+        }(make_index_sequence<__rank_>())) {
+    // stride() only compiles for rank > 0
+    if constexpr (__rank_ > 0) {
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          ([&]<size_t... _Pos>(index_sequence<_Pos...>) {
+            return ((static_cast<index_type>(__other.stride(_Pos)) > static_cast<index_type>(0)) && ... && true);
+          }(make_index_sequence<__rank_>())),
+          "layout_stride::mapping converting ctor: all strides must be greater than 0");
+    }
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __mdspan_detail::__is_representable_as<index_type>(__other.required_span_size()),
+        "layout_stride::mapping converting ctor: other.required_span_size() must be representable as index_type.");
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(static_cast<index_type>(0) == __offset(__other),
+                                        "layout_stride::mapping converting ctor: base offset of mapping must be zero.");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping& operator=(const mapping&) noexcept = default;
+
+  // [mdspan.layout.stride.obs], observers
+  _LIBCPP_HIDE_FROM_ABI constexpr const extents_type& extents() const noexcept { return __extents_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr array<index_type, __rank_> strides() const noexcept {
+    return [&]<size_t... _Pos>(index_sequence<_Pos...>) {
+      return array<index_type, __rank_>{__strides_[_Pos]...};
+    }(make_index_sequence<__rank_>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr index_type required_span_size() const noexcept {
+    if constexpr (__rank_ == 0) {
+      return static_cast<index_type>(1);
+    } else {
+      return [&]<size_t... _Pos>(index_sequence<_Pos...>) {
+        if ((__extents_.extent(_Pos) * ... * 1) == 0)
+          return static_cast<index_type>(0);
+        else
+          return static_cast<index_type>(
+              static_cast<index_type>(1) +
+              (((__extents_.extent(_Pos) - static_cast<index_type>(1)) * __strides_[_Pos]) + ... +
+               static_cast<index_type>(0)));
+      }(make_index_sequence<__rank_>());
+    }
+  }
+
+  template <class... _Indices>
+    requires((sizeof...(_Indices) == __rank_) && (is_convertible_v<_Indices, index_type> && ...) &&
+             (is_nothrow_constructible_v<index_type, _Indices> && ...))
+  _LIBCPP_HIDE_FROM_ABI constexpr index_type operator()(_Indices... __idx) const noexcept {
+    // Mappings are generally meant to be used for accessing allocations and are meant to guarantee to never
+    // return a value exceeding required_span_size(), which is used to know how large an allocation one needs
+    // Thus, this is a canonical point in multi-dimensional data structures to make invalid element access checks
+    // However, mdspan does check this on its own, so for now we avoid double checking in hardened mode
+    _LIBCPP_ASSERT_UNCATEGORIZED(__mdspan_detail::__is_multidimensional_index_in(__extents_, __idx...),
+                                 "layout_stride::mapping: out of bounds indexing");
+    return [&]<size_t... _Pos>(index_sequence<_Pos...>) {
+      return ((static_cast<index_type>(__idx) * __strides_[_Pos]) + ... + index_type(0));
+    }(make_index_sequence<sizeof...(_Indices)>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_unique() noexcept { return true; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_exhaustive() noexcept { return false; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_strided() noexcept { return true; }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_unique() noexcept { return true; }
+  // The answer of this function is fairly complex in the case where one or more
+  // extents are zero.
+  // Technically it is meaningless to query is_exhaustive() in that case, but unfortunately
+  // the way the standard defines this function, we can't give a simple true or false then.
+  _LIBCPP_HIDE_FROM_ABI constexpr bool is_exhaustive() const noexcept {
+    if constexpr (__rank_ == 0)
+      return true;
+    else {
+      index_type __span_size = required_span_size();
+      if (__span_size == static_cast<index_type>(0)) {
+        if constexpr (__rank_ == 1)
+          return __strides_[0] == 1;
+        else {
+          rank_type __r_largest = 0;
+          for (rank_type __r = 1; __r < __rank_; __r++)
+            if (__strides_[__r] > __strides_[__r_largest])
+              __r_largest = __r;
+          for (rank_type __r = 0; __r < __rank_; __r++)
+            if (__extents_.extent(__r) == 0 && __r != __r_largest)
+              return false;
+          return true;
+        }
+      } else {
+        return required_span_size() == [&]<size_t... _Pos>(index_sequence<_Pos...>) {
+          return (__extents_.extent(_Pos) * ... * static_cast<index_type>(1));
+        }(make_index_sequence<__rank_>());
+      }
+    }
+  }
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_strided() noexcept { return true; }
+
+  // according to the standard layout_stride does not have a constraint on stride(r) for rank>0
+  // it still has the precondition though
+  _LIBCPP_HIDE_FROM_ABI constexpr index_type stride(rank_type __r) const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__r < __rank_, "layout_stride::mapping::stride(): invalid rank index");
+    return __strides_[__r];
+  }
+
+  template <class _OtherMapping>
+    requires(__mdspan_detail::__layout_mapping_alike<_OtherMapping> &&
+             (_OtherMapping::extents_type::rank() == __rank_) && _OtherMapping::is_always_strided())
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const mapping& __lhs, const _OtherMapping& __rhs) noexcept {
+    if (__offset(__rhs))
+      return false;
+    if constexpr (__rank_ == 0)
+      return true;
+    else {
+      return __lhs.extents() == __rhs.extents() && [&]<size_t... _Pos>(index_sequence<_Pos...>) {
+        // avoid warning when comparing signed and unsigner integers and pick the wider of two types
+        using _CommonType = common_type_t<index_type, typename _OtherMapping::index_type>;
+        return ((static_cast<_CommonType>(__lhs.stride(_Pos)) == static_cast<_CommonType>(__rhs.stride(_Pos))) && ... &&
+                true);
+      }(make_index_sequence<__rank_>());
+    }
+  }
+
+private:
+  _LIBCPP_NO_UNIQUE_ADDRESS extents_type __extents_{};
+  _LIBCPP_NO_UNIQUE_ADDRESS __mdspan_detail::__possibly_empty_array<index_type, __rank_> __strides_{};
+};
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MDSPAN_LAYOUT_STRIDE_H
diff --git a/libcxx/include/__cxx03/__mdspan/mdspan.h b/libcxx/include/__cxx03/__mdspan/mdspan.h
new file mode 100644
index 000000000000..1ff4fd4ba4a8
--- /dev/null
+++ b/libcxx/include/__cxx03/__mdspan/mdspan.h
@@ -0,0 +1,319 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//                        Kokkos v. 4.0
+//       Copyright (2022) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MDSPAN_MDSPAN_H
+#define _LIBCPP___MDSPAN_MDSPAN_H
+
+#include <__assert>
+#include <__config>
+#include <__fwd/mdspan.h>
+#include <__mdspan/default_accessor.h>
+#include <__mdspan/extents.h>
+#include <__type_traits/extent.h>
+#include <__type_traits/is_abstract.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/rank.h>
+#include <__type_traits/remove_all_extents.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/remove_pointer.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/integer_sequence.h>
+#include <array>
+#include <cinttypes>
+#include <cstddef>
+#include <limits>
+#include <span>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+// Helper for lightweight test checking that one did pass a layout policy as LayoutPolicy template argument
+namespace __mdspan_detail {
+template <class _Layout, class _Extents>
+concept __has_invalid_mapping = !requires { typename _Layout::template mapping<_Extents>; };
+} // namespace __mdspan_detail
+
+template <class _ElementType,
+          class _Extents,
+          class _LayoutPolicy   = layout_right,
+          class _AccessorPolicy = default_accessor<_ElementType> >
+class mdspan {
+private:
+  static_assert(__mdspan_detail::__is_extents_v<_Extents>,
+                "mdspan: Extents template parameter must be a specialization of extents.");
+  static_assert(!is_array_v<_ElementType>, "mdspan: ElementType template parameter may not be an array type");
+  static_assert(!is_abstract_v<_ElementType>, "mdspan: ElementType template parameter may not be an abstract class");
+  static_assert(is_same_v<_ElementType, typename _AccessorPolicy::element_type>,
+                "mdspan: ElementType template parameter must match AccessorPolicy::element_type");
+  static_assert(!__mdspan_detail::__has_invalid_mapping<_LayoutPolicy, _Extents>,
+                "mdspan: LayoutPolicy template parameter is invalid. A common mistake is to pass a layout mapping "
+                "instead of a layout policy");
+
+public:
+  using extents_type     = _Extents;
+  using layout_type      = _LayoutPolicy;
+  using accessor_type    = _AccessorPolicy;
+  using mapping_type     = typename layout_type::template mapping<extents_type>;
+  using element_type     = _ElementType;
+  using value_type       = remove_cv_t<element_type>;
+  using index_type       = typename extents_type::index_type;
+  using size_type        = typename extents_type::size_type;
+  using rank_type        = typename extents_type::rank_type;
+  using data_handle_type = typename accessor_type::data_handle_type;
+  using reference        = typename accessor_type::reference;
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr rank_type rank() noexcept { return extents_type::rank(); }
+  _LIBCPP_HIDE_FROM_ABI static constexpr rank_type rank_dynamic() noexcept { return extents_type::rank_dynamic(); }
+  _LIBCPP_HIDE_FROM_ABI static constexpr size_t static_extent(rank_type __r) noexcept {
+    return extents_type::static_extent(__r);
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr index_type extent(rank_type __r) const noexcept {
+    return __map_.extents().extent(__r);
+  };
+
+public:
+  //--------------------------------------------------------------------------------
+  // [mdspan.mdspan.cons], mdspan constructors, assignment, and destructor
+
+  _LIBCPP_HIDE_FROM_ABI constexpr mdspan()
+    requires((extents_type::rank_dynamic() > 0) && is_default_constructible_v<data_handle_type> &&
+             is_default_constructible_v<mapping_type> && is_default_constructible_v<accessor_type>)
+  = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr mdspan(const mdspan&) = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr mdspan(mdspan&&)      = default;
+
+  template <class... _OtherIndexTypes>
+    requires((is_convertible_v<_OtherIndexTypes, index_type> && ...) &&
+             (is_nothrow_constructible_v<index_type, _OtherIndexTypes> && ...) &&
+             ((sizeof...(_OtherIndexTypes) == rank()) || (sizeof...(_OtherIndexTypes) == rank_dynamic())) &&
+             is_constructible_v<mapping_type, extents_type> && is_default_constructible_v<accessor_type>)
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr mdspan(data_handle_type __p, _OtherIndexTypes... __exts)
+      : __ptr_(std::move(__p)), __map_(extents_type(static_cast<index_type>(std::move(__exts))...)), __acc_{} {}
+
+  template <class _OtherIndexType, size_t _Size>
+    requires(is_convertible_v<const _OtherIndexType&, index_type> &&
+             is_nothrow_constructible_v<index_type, const _OtherIndexType&> &&
+             ((_Size == rank()) || (_Size == rank_dynamic())) && is_constructible_v<mapping_type, extents_type> &&
+             is_default_constructible_v<accessor_type>)
+  explicit(_Size != rank_dynamic())
+      _LIBCPP_HIDE_FROM_ABI constexpr mdspan(data_handle_type __p, const array<_OtherIndexType, _Size>& __exts)
+      : __ptr_(std::move(__p)), __map_(extents_type(__exts)), __acc_{} {}
+
+  template <class _OtherIndexType, size_t _Size>
+    requires(is_convertible_v<const _OtherIndexType&, index_type> &&
+             is_nothrow_constructible_v<index_type, const _OtherIndexType&> &&
+             ((_Size == rank()) || (_Size == rank_dynamic())) && is_constructible_v<mapping_type, extents_type> &&
+             is_default_constructible_v<accessor_type>)
+  explicit(_Size != rank_dynamic())
+      _LIBCPP_HIDE_FROM_ABI constexpr mdspan(data_handle_type __p, span<_OtherIndexType, _Size> __exts)
+      : __ptr_(std::move(__p)), __map_(extents_type(__exts)), __acc_{} {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr mdspan(data_handle_type __p, const extents_type& __exts)
+    requires(is_default_constructible_v<accessor_type> && is_constructible_v<mapping_type, const extents_type&>)
+      : __ptr_(std::move(__p)), __map_(__exts), __acc_{} {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr mdspan(data_handle_type __p, const mapping_type& __m)
+    requires(is_default_constructible_v<accessor_type>)
+      : __ptr_(std::move(__p)), __map_(__m), __acc_{} {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr mdspan(data_handle_type __p, const mapping_type& __m, const accessor_type& __a)
+      : __ptr_(std::move(__p)), __map_(__m), __acc_(__a) {}
+
+  template <class _OtherElementType, class _OtherExtents, class _OtherLayoutPolicy, class _OtherAccessor>
+    requires(is_constructible_v<mapping_type, const typename _OtherLayoutPolicy::template mapping<_OtherExtents>&> &&
+             is_constructible_v<accessor_type, const _OtherAccessor&>)
+  explicit(!is_convertible_v<const typename _OtherLayoutPolicy::template mapping<_OtherExtents>&, mapping_type> ||
+           !is_convertible_v<const _OtherAccessor&, accessor_type>)
+      _LIBCPP_HIDE_FROM_ABI constexpr mdspan(
+          const mdspan<_OtherElementType, _OtherExtents, _OtherLayoutPolicy, _OtherAccessor>& __other)
+      : __ptr_(__other.__ptr_), __map_(__other.__map_), __acc_(__other.__acc_) {
+    static_assert(is_constructible_v<data_handle_type, const typename _OtherAccessor::data_handle_type&>,
+                  "mdspan: incompatible data_handle_type for mdspan construction");
+    static_assert(
+        is_constructible_v<extents_type, _OtherExtents>, "mdspan: incompatible extents for mdspan construction");
+
+    // The following precondition is part of the standard, but is unlikely to be triggered.
+    // The extents constructor checks this and the mapping must be storing the extents, since
+    // its extents() function returns a const reference to extents_type.
+    // The only way this can be triggered is if the mapping conversion constructor would for example
+    // always construct its extents() only from the dynamic extents, instead of from the other extents.
+    if constexpr (rank() > 0) {
+      for (size_t __r = 0; __r < rank(); __r++) {
+        // Not catching this could lead to out of bounds errors later
+        // e.g. mdspan<int, dextents<char,1>, non_checking_layout> m =
+        //        mdspan<int, dextents<unsigned, 1>, non_checking_layout>(ptr, 200); leads to an extent of -56 on m
+        _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+            (static_extent(__r) == dynamic_extent) ||
+                (static_cast<index_type>(__other.extent(__r)) == static_cast<index_type>(static_extent(__r))),
+            "mdspan: conversion mismatch of source dynamic extents with static extents");
+      }
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr mdspan& operator=(const mdspan&) = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr mdspan& operator=(mdspan&&)      = default;
+
+  //--------------------------------------------------------------------------------
+  // [mdspan.mdspan.members], members
+
+  template <class... _OtherIndexTypes>
+    requires((is_convertible_v<_OtherIndexTypes, index_type> && ...) &&
+             (is_nothrow_constructible_v<index_type, _OtherIndexTypes> && ...) &&
+             (sizeof...(_OtherIndexTypes) == rank()))
+  _LIBCPP_HIDE_FROM_ABI constexpr reference operator[](_OtherIndexTypes... __indices) const {
+    // Note the standard layouts would also check this, but user provided ones may not, so we
+    // check the precondition here
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__mdspan_detail::__is_multidimensional_index_in(extents(), __indices...),
+                                        "mdspan: operator[] out of bounds access");
+    return __acc_.access(__ptr_, __map_(static_cast<index_type>(std::move(__indices))...));
+  }
+
+  template <class _OtherIndexType>
+    requires(is_convertible_v<const _OtherIndexType&, index_type> &&
+             is_nothrow_constructible_v<index_type, const _OtherIndexType&>)
+  _LIBCPP_HIDE_FROM_ABI constexpr reference operator[](const array< _OtherIndexType, rank()>& __indices) const {
+    return __acc_.access(__ptr_, [&]<size_t... _Idxs>(index_sequence<_Idxs...>) {
+      return __map_(__indices[_Idxs]...);
+    }(make_index_sequence<rank()>()));
+  }
+
+  template <class _OtherIndexType>
+    requires(is_convertible_v<const _OtherIndexType&, index_type> &&
+             is_nothrow_constructible_v<index_type, const _OtherIndexType&>)
+  _LIBCPP_HIDE_FROM_ABI constexpr reference operator[](span<_OtherIndexType, rank()> __indices) const {
+    return __acc_.access(__ptr_, [&]<size_t... _Idxs>(index_sequence<_Idxs...>) {
+      return __map_(__indices[_Idxs]...);
+    }(make_index_sequence<rank()>()));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr size_type size() const noexcept {
+    // Could leave this as only checked in debug mode: semantically size() is never
+    // guaranteed to be related to any accessible range
+    _LIBCPP_ASSERT_UNCATEGORIZED(
+        false == ([&]<size_t... _Idxs>(index_sequence<_Idxs...>) {
+          size_type __prod = 1;
+          return (__builtin_mul_overflow(__prod, extent(_Idxs), &__prod) || ... || false);
+        }(make_index_sequence<rank()>())),
+        "mdspan: size() is not representable as size_type");
+    return [&]<size_t... _Idxs>(index_sequence<_Idxs...>) {
+      return ((static_cast<size_type>(__map_.extents().extent(_Idxs))) * ... * size_type(1));
+    }(make_index_sequence<rank()>());
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool empty() const noexcept {
+    return [&]<size_t... _Idxs>(index_sequence<_Idxs...>) {
+      return (rank() > 0) && ((__map_.extents().extent(_Idxs) == index_type(0)) || ... || false);
+    }(make_index_sequence<rank()>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr void swap(mdspan& __x, mdspan& __y) noexcept {
+    swap(__x.__ptr_, __y.__ptr_);
+    swap(__x.__map_, __y.__map_);
+    swap(__x.__acc_, __y.__acc_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const extents_type& extents() const noexcept { return __map_.extents(); };
+  _LIBCPP_HIDE_FROM_ABI constexpr const data_handle_type& data_handle() const noexcept { return __ptr_; };
+  _LIBCPP_HIDE_FROM_ABI constexpr const mapping_type& mapping() const noexcept { return __map_; };
+  _LIBCPP_HIDE_FROM_ABI constexpr const accessor_type& accessor() const noexcept { return __acc_; };
+
+  // per LWG-4021 "mdspan::is_always_meow() should be noexcept"
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_unique() noexcept { return mapping_type::is_always_unique(); };
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_exhaustive() noexcept {
+    return mapping_type::is_always_exhaustive();
+  };
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_strided() noexcept {
+    return mapping_type::is_always_strided();
+  };
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool is_unique() const { return __map_.is_unique(); };
+  _LIBCPP_HIDE_FROM_ABI constexpr bool is_exhaustive() const { return __map_.is_exhaustive(); };
+  _LIBCPP_HIDE_FROM_ABI constexpr bool is_strided() const { return __map_.is_strided(); };
+  _LIBCPP_HIDE_FROM_ABI constexpr index_type stride(rank_type __r) const { return __map_.stride(__r); };
+
+private:
+  _LIBCPP_NO_UNIQUE_ADDRESS data_handle_type __ptr_{};
+  _LIBCPP_NO_UNIQUE_ADDRESS mapping_type __map_{};
+  _LIBCPP_NO_UNIQUE_ADDRESS accessor_type __acc_{};
+
+  template <class, class, class, class>
+  friend class mdspan;
+};
+
+#  if _LIBCPP_STD_VER >= 26
+template <class _ElementType, class... _OtherIndexTypes>
+  requires((is_convertible_v<_OtherIndexTypes, size_t> && ...) && (sizeof...(_OtherIndexTypes) > 0))
+explicit mdspan(_ElementType*,
+                _OtherIndexTypes...) -> mdspan<_ElementType, extents<size_t, __maybe_static_ext<_OtherIndexTypes>...>>;
+#  else
+template <class _ElementType, class... _OtherIndexTypes>
+  requires((is_convertible_v<_OtherIndexTypes, size_t> && ...) && (sizeof...(_OtherIndexTypes) > 0))
+explicit mdspan(_ElementType*,
+                _OtherIndexTypes...) -> mdspan<_ElementType, dextents<size_t, sizeof...(_OtherIndexTypes)>>;
+#  endif
+
+template <class _Pointer>
+  requires(is_pointer_v<remove_reference_t<_Pointer>>)
+mdspan(_Pointer&&) -> mdspan<remove_pointer_t<remove_reference_t<_Pointer>>, extents<size_t>>;
+
+template <class _CArray>
+  requires(is_array_v<_CArray> && (rank_v<_CArray> == 1))
+mdspan(_CArray&) -> mdspan<remove_all_extents_t<_CArray>, extents<size_t, extent_v<_CArray, 0>>>;
+
+template <class _ElementType, class _OtherIndexType, size_t _Size>
+mdspan(_ElementType*, const array<_OtherIndexType, _Size>&) -> mdspan<_ElementType, dextents<size_t, _Size>>;
+
+template <class _ElementType, class _OtherIndexType, size_t _Size>
+mdspan(_ElementType*, span<_OtherIndexType, _Size>) -> mdspan<_ElementType, dextents<size_t, _Size>>;
+
+// This one is necessary because all the constructors take `data_handle_type`s, not
+// `_ElementType*`s, and `data_handle_type` is taken from `accessor_type::data_handle_type`, which
+// seems to throw off automatic deduction guides.
+template <class _ElementType, class _OtherIndexType, size_t... _ExtentsPack>
+mdspan(_ElementType*, const extents<_OtherIndexType, _ExtentsPack...>&)
+    -> mdspan<_ElementType, extents<_OtherIndexType, _ExtentsPack...>>;
+
+template <class _ElementType, class _MappingType>
+mdspan(_ElementType*, const _MappingType&)
+    -> mdspan<_ElementType, typename _MappingType::extents_type, typename _MappingType::layout_type>;
+
+template <class _MappingType, class _AccessorType>
+mdspan(const typename _AccessorType::data_handle_type, const _MappingType&, const _AccessorType&)
+    -> mdspan<typename _AccessorType::element_type,
+              typename _MappingType::extents_type,
+              typename _MappingType::layout_type,
+              _AccessorType>;
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MDSPAN_MDSPAN_H
diff --git a/libcxx/include/__cxx03/__memory/addressof.h b/libcxx/include/__cxx03/__memory/addressof.h
new file mode 100644
index 000000000000..fa590212c49b
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/addressof.h
@@ -0,0 +1,61 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ADDRESSOF_H
+#define _LIBCPP___MEMORY_ADDRESSOF_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_NO_CFI _LIBCPP_HIDE_FROM_ABI _Tp* addressof(_Tp& __x) _NOEXCEPT {
+  return __builtin_addressof(__x);
+}
+
+#if defined(_LIBCPP_HAS_OBJC_ARC) && !defined(_LIBCPP_PREDEFINED_OBJC_ARC_ADDRESSOF)
+// Objective-C++ Automatic Reference Counting uses qualified pointers
+// that require special addressof() signatures. When
+// _LIBCPP_PREDEFINED_OBJC_ARC_ADDRESSOF is defined, the compiler
+// itself is providing these definitions. Otherwise, we provide them.
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI __strong _Tp* addressof(__strong _Tp& __x) _NOEXCEPT {
+  return &__x;
+}
+
+#  ifdef _LIBCPP_HAS_OBJC_ARC_WEAK
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI __weak _Tp* addressof(__weak _Tp& __x) _NOEXCEPT {
+  return &__x;
+}
+#  endif
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI __autoreleasing _Tp* addressof(__autoreleasing _Tp& __x) _NOEXCEPT {
+  return &__x;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI __unsafe_unretained _Tp* addressof(__unsafe_unretained _Tp& __x) _NOEXCEPT {
+  return &__x;
+}
+#endif
+
+#if !defined(_LIBCPP_CXX03_LANG)
+template <class _Tp>
+_Tp* addressof(const _Tp&&) noexcept = delete;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_ADDRESSOF_H
diff --git a/libcxx/include/__cxx03/__memory/align.h b/libcxx/include/__cxx03/__memory/align.h
new file mode 100644
index 000000000000..bbb995f4a8c8
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/align.h
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ALIGN_H
+#define _LIBCPP___MEMORY_ALIGN_H
+
+#include <__config>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space);
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_ALIGN_H
diff --git a/libcxx/include/__cxx03/__memory/aligned_alloc.h b/libcxx/include/__cxx03/__memory/aligned_alloc.h
new file mode 100644
index 000000000000..cb424328bcaf
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/aligned_alloc.h
@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ALIGNED_ALLOC_H
+#define _LIBCPP___MEMORY_ALIGNED_ALLOC_H
+
+#include <__config>
+#include <cstddef>
+#include <cstdlib>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifndef _LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+
+// Low-level helpers to call the aligned allocation and deallocation functions
+// on the target platform. This is used to implement libc++'s own memory
+// allocation routines -- if you need to allocate memory inside the library,
+// chances are that you want to use `__libcpp_allocate` instead.
+//
+// Returns the allocated memory, or `nullptr` on failure.
+inline _LIBCPP_HIDE_FROM_ABI void* __libcpp_aligned_alloc(std::size_t __alignment, std::size_t __size) {
+#  if defined(_LIBCPP_MSVCRT_LIKE)
+  return ::_aligned_malloc(__size, __alignment);
+#  elif _LIBCPP_STD_VER >= 17 && !defined(_LIBCPP_HAS_NO_C11_ALIGNED_ALLOC)
+  // aligned_alloc() requires that __size is a multiple of __alignment,
+  // but for C++ [new.delete.general], only states "if the value of an
+  // alignment argument passed to any of these functions is not a valid
+  // alignment value, the behavior is undefined".
+  // To handle calls such as ::operator new(1, std::align_val_t(128)), we
+  // round __size up to the next multiple of __alignment.
+  size_t __rounded_size = (__size + __alignment - 1) & ~(__alignment - 1);
+  // Rounding up could have wrapped around to zero, so we have to add another
+  // max() ternary to the actual call site to avoid succeeded in that case.
+  return ::aligned_alloc(__alignment, __size > __rounded_size ? __size : __rounded_size);
+#  else
+  void* __result = nullptr;
+  (void)::posix_memalign(&__result, __alignment, __size);
+  // If posix_memalign fails, __result is unmodified so we still return `nullptr`.
+  return __result;
+#  endif
+}
+
+inline _LIBCPP_HIDE_FROM_ABI void __libcpp_aligned_free(void* __ptr) {
+#  if defined(_LIBCPP_MSVCRT_LIKE)
+  ::_aligned_free(__ptr);
+#  else
+  ::free(__ptr);
+#  endif
+}
+
+#endif // !_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_ALIGNED_ALLOC_H
diff --git a/libcxx/include/__cxx03/__memory/allocate_at_least.h b/libcxx/include/__cxx03/__memory/allocate_at_least.h
new file mode 100644
index 000000000000..df73d9a2e94a
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/allocate_at_least.h
@@ -0,0 +1,48 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ALLOCATE_AT_LEAST_H
+#define _LIBCPP___MEMORY_ALLOCATE_AT_LEAST_H
+
+#include <__config>
+#include <__memory/allocator_traits.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Alloc>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto __allocate_at_least(_Alloc& __alloc, size_t __n) {
+  return std::allocator_traits<_Alloc>::allocate_at_least(__alloc, __n);
+}
+
+#else
+
+template <class _Pointer>
+struct __allocation_result {
+  _Pointer ptr;
+  size_t count;
+};
+
+template <class _Alloc>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI
+_LIBCPP_CONSTEXPR __allocation_result<typename allocator_traits<_Alloc>::pointer>
+__allocate_at_least(_Alloc& __alloc, size_t __n) {
+  return {__alloc.allocate(__n), __n};
+}
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_ALLOCATE_AT_LEAST_H
diff --git a/libcxx/include/__cxx03/__memory/allocation_guard.h b/libcxx/include/__cxx03/__memory/allocation_guard.h
new file mode 100644
index 000000000000..cb870af7be67
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/allocation_guard.h
@@ -0,0 +1,108 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ALLOCATION_GUARD_H
+#define _LIBCPP___MEMORY_ALLOCATION_GUARD_H
+
+#include <__config>
+#include <__memory/addressof.h>
+#include <__memory/allocator_traits.h>
+#include <__utility/move.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Helper class to allocate memory using an Allocator in an exception safe
+// manner.
+//
+// The intended usage of this class is as follows:
+//
+// 0
+// 1     __allocation_guard<SomeAllocator> guard(alloc, 10);
+// 2     do_some_initialization_that_may_throw(guard.__get());
+// 3     save_allocated_pointer_in_a_noexcept_operation(guard.__release_ptr());
+// 4
+//
+// If line (2) throws an exception during initialization of the memory, the
+// guard's destructor will be called, and the memory will be released using
+// Allocator deallocation. Otherwise, we release the memory from the guard on
+// line (3) in an operation that can't throw -- after that, the guard is not
+// responsible for the memory anymore.
+//
+// This is similar to a unique_ptr, except it's easier to use with a
+// custom allocator.
+template <class _Alloc>
+struct __allocation_guard {
+  using _Pointer = typename allocator_traits<_Alloc>::pointer;
+  using _Size    = typename allocator_traits<_Alloc>::size_type;
+
+  template <class _AllocT> // we perform the allocator conversion inside the constructor
+  _LIBCPP_HIDE_FROM_ABI explicit __allocation_guard(_AllocT __alloc, _Size __n)
+      : __alloc_(std::move(__alloc)),
+        __n_(__n),
+        __ptr_(allocator_traits<_Alloc>::allocate(__alloc_, __n_)) // initialization order is important
+  {}
+
+  _LIBCPP_HIDE_FROM_ABI ~__allocation_guard() _NOEXCEPT { __destroy(); }
+
+  _LIBCPP_HIDE_FROM_ABI __allocation_guard(const __allocation_guard&) = delete;
+  _LIBCPP_HIDE_FROM_ABI __allocation_guard(__allocation_guard&& __other) _NOEXCEPT
+      : __alloc_(std::move(__other.__alloc_)),
+        __n_(__other.__n_),
+        __ptr_(__other.__ptr_) {
+    __other.__ptr_ = nullptr;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __allocation_guard& operator=(const __allocation_guard& __other) = delete;
+  _LIBCPP_HIDE_FROM_ABI __allocation_guard& operator=(__allocation_guard&& __other) _NOEXCEPT {
+    if (std::addressof(__other) != this) {
+      __destroy();
+
+      __alloc_       = std::move(__other.__alloc_);
+      __n_           = __other.__n_;
+      __ptr_         = __other.__ptr_;
+      __other.__ptr_ = nullptr;
+    }
+
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Pointer
+  __release_ptr() _NOEXCEPT { // not called __release() because it's a keyword in objective-c++
+    _Pointer __tmp = __ptr_;
+    __ptr_         = nullptr;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Pointer __get() const _NOEXCEPT { return __ptr_; }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI void __destroy() _NOEXCEPT {
+    if (__ptr_ != nullptr) {
+      allocator_traits<_Alloc>::deallocate(__alloc_, __ptr_, __n_);
+    }
+  }
+
+  _Alloc __alloc_;
+  _Size __n_;
+  _Pointer __ptr_;
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_ALLOCATION_GUARD_H
diff --git a/libcxx/include/__cxx03/__memory/allocator.h b/libcxx/include/__cxx03/__memory/allocator.h
new file mode 100644
index 000000000000..2d8624e771bc
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/allocator.h
@@ -0,0 +1,268 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ALLOCATOR_H
+#define _LIBCPP___MEMORY_ALLOCATOR_H
+
+#include <__config>
+#include <__memory/addressof.h>
+#include <__memory/allocate_at_least.h>
+#include <__memory/allocator_traits.h>
+#include <__type_traits/is_const.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_void.h>
+#include <__type_traits/is_volatile.h>
+#include <__utility/forward.h>
+#include <cstddef>
+#include <new>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+class allocator;
+
+#if _LIBCPP_STD_VER <= 17
+// These specializations shouldn't be marked _LIBCPP_DEPRECATED_IN_CXX17.
+// Specializing allocator<void> is deprecated, but not using it.
+template <>
+class _LIBCPP_TEMPLATE_VIS allocator<void> {
+public:
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef void* pointer;
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef const void* const_pointer;
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef void value_type;
+
+  template <class _Up>
+  struct _LIBCPP_DEPRECATED_IN_CXX17 rebind {
+    typedef allocator<_Up> other;
+  };
+};
+
+// TODO(LLVM 20): Remove the escape hatch
+#  ifdef _LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST
+template <>
+class _LIBCPP_TEMPLATE_VIS allocator<const void> {
+public:
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef const void* pointer;
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef const void* const_pointer;
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef const void value_type;
+
+  template <class _Up>
+  struct _LIBCPP_DEPRECATED_IN_CXX17 rebind {
+    typedef allocator<_Up> other;
+  };
+};
+#  endif // _LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST
+#endif   // _LIBCPP_STD_VER <= 17
+
+// This class provides a non-trivial default constructor to the class that derives from it
+// if the condition is satisfied.
+//
+// The second template parameter exists to allow giving a unique type to __non_trivial_if,
+// which makes it possible to avoid breaking the ABI when making this a base class of an
+// existing class. Without that, imagine we have classes D1 and D2, both of which used to
+// have no base classes, but which now derive from __non_trivial_if. The layout of a class
+// that inherits from both D1 and D2 will change because the two __non_trivial_if base
+// classes are not allowed to share the same address.
+//
+// By making those __non_trivial_if base classes unique, we work around this problem and
+// it is safe to start deriving from __non_trivial_if in existing classes.
+template <bool _Cond, class _Unique>
+struct __non_trivial_if {};
+
+template <class _Unique>
+struct __non_trivial_if<true, _Unique> {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __non_trivial_if() _NOEXCEPT {}
+};
+
+// allocator
+//
+// Note: For ABI compatibility between C++20 and previous standards, we make
+//       allocator<void> trivial in C++20.
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS allocator : private __non_trivial_if<!is_void<_Tp>::value, allocator<_Tp> > {
+  static_assert(!is_const<_Tp>::value, "std::allocator does not support const types");
+  static_assert(!is_volatile<_Tp>::value, "std::allocator does not support volatile types");
+
+public:
+  typedef size_t size_type;
+  typedef ptrdiff_t difference_type;
+  typedef _Tp value_type;
+  typedef true_type propagate_on_container_move_assignment;
+#if _LIBCPP_STD_VER <= 23 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_ALLOCATOR_MEMBERS)
+  _LIBCPP_DEPRECATED_IN_CXX23 typedef true_type is_always_equal;
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator() _NOEXCEPT = default;
+
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator(const allocator<_Up>&) _NOEXCEPT {}
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* allocate(size_t __n) {
+    if (__n > allocator_traits<allocator>::max_size(*this))
+      __throw_bad_array_new_length();
+    if (__libcpp_is_constant_evaluated()) {
+      return static_cast<_Tp*>(::operator new(__n * sizeof(_Tp)));
+    } else {
+      return static_cast<_Tp*>(std::__libcpp_allocate(__n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp)));
+    }
+  }
+
+#if _LIBCPP_STD_VER >= 23
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr allocation_result<_Tp*> allocate_at_least(size_t __n) {
+    return {allocate(__n), __n};
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void deallocate(_Tp* __p, size_t __n) _NOEXCEPT {
+    if (__libcpp_is_constant_evaluated()) {
+      ::operator delete(__p);
+    } else {
+      std::__libcpp_deallocate((void*)__p, __n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp));
+    }
+  }
+
+  // C++20 Removed members
+#if _LIBCPP_STD_VER <= 17
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef _Tp* pointer;
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp* const_pointer;
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef _Tp& reference;
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp& const_reference;
+
+  template <class _Up>
+  struct _LIBCPP_DEPRECATED_IN_CXX17 rebind {
+    typedef allocator<_Up> other;
+  };
+
+  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI pointer address(reference __x) const _NOEXCEPT {
+    return std::addressof(__x);
+  }
+  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI const_pointer address(const_reference __x) const _NOEXCEPT {
+    return std::addressof(__x);
+  }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_IN_CXX17 _Tp* allocate(size_t __n, const void*) {
+    return allocate(__n);
+  }
+
+  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
+    return size_type(~0) / sizeof(_Tp);
+  }
+
+  template <class _Up, class... _Args>
+  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI void construct(_Up* __p, _Args&&... __args) {
+    ::new ((void*)__p) _Up(std::forward<_Args>(__args)...);
+  }
+
+  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI void destroy(pointer __p) { __p->~_Tp(); }
+#endif
+};
+
+// TODO(LLVM 20): Remove the escape hatch
+#ifdef _LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS allocator<const _Tp>
+    : private __non_trivial_if<!is_void<_Tp>::value, allocator<const _Tp> > {
+  static_assert(!is_volatile<_Tp>::value, "std::allocator does not support volatile types");
+
+public:
+  typedef size_t size_type;
+  typedef ptrdiff_t difference_type;
+  typedef const _Tp value_type;
+  typedef true_type propagate_on_container_move_assignment;
+#  if _LIBCPP_STD_VER <= 23 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_ALLOCATOR_MEMBERS)
+  _LIBCPP_DEPRECATED_IN_CXX23 typedef true_type is_always_equal;
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator() _NOEXCEPT = default;
+
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator(const allocator<_Up>&) _NOEXCEPT {}
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const _Tp* allocate(size_t __n) {
+    if (__n > allocator_traits<allocator>::max_size(*this))
+      __throw_bad_array_new_length();
+    if (__libcpp_is_constant_evaluated()) {
+      return static_cast<const _Tp*>(::operator new(__n * sizeof(_Tp)));
+    } else {
+      return static_cast<const _Tp*>(std::__libcpp_allocate(__n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp)));
+    }
+  }
+
+#  if _LIBCPP_STD_VER >= 23
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr allocation_result<const _Tp*> allocate_at_least(size_t __n) {
+    return {allocate(__n), __n};
+  }
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void deallocate(const _Tp* __p, size_t __n) {
+    if (__libcpp_is_constant_evaluated()) {
+      ::operator delete(const_cast<_Tp*>(__p));
+    } else {
+      std::__libcpp_deallocate((void*)const_cast<_Tp*>(__p), __n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp));
+    }
+  }
+
+  // C++20 Removed members
+#  if _LIBCPP_STD_VER <= 17
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp* pointer;
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp* const_pointer;
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp& reference;
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp& const_reference;
+
+  template <class _Up>
+  struct _LIBCPP_DEPRECATED_IN_CXX17 rebind {
+    typedef allocator<_Up> other;
+  };
+
+  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI const_pointer address(const_reference __x) const _NOEXCEPT {
+    return std::addressof(__x);
+  }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_IN_CXX17 const _Tp* allocate(size_t __n, const void*) {
+    return allocate(__n);
+  }
+
+  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
+    return size_type(~0) / sizeof(_Tp);
+  }
+
+  template <class _Up, class... _Args>
+  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI void construct(_Up* __p, _Args&&... __args) {
+    ::new ((void*)__p) _Up(std::forward<_Args>(__args)...);
+  }
+
+  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI void destroy(pointer __p) { __p->~_Tp(); }
+#  endif
+};
+#endif // _LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST
+
+template <class _Tp, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+operator==(const allocator<_Tp>&, const allocator<_Up>&) _NOEXCEPT {
+  return true;
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const allocator<_Tp>&, const allocator<_Up>&) _NOEXCEPT {
+  return false;
+}
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_ALLOCATOR_H
diff --git a/libcxx/include/__cxx03/__memory/allocator_arg_t.h b/libcxx/include/__cxx03/__memory/allocator_arg_t.h
new file mode 100644
index 000000000000..7e66da740cd4
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/allocator_arg_t.h
@@ -0,0 +1,75 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FUNCTIONAL_ALLOCATOR_ARG_T_H
+#define _LIBCPP___FUNCTIONAL_ALLOCATOR_ARG_T_H
+
+#include <__config>
+#include <__memory/uses_allocator.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct _LIBCPP_TEMPLATE_VIS allocator_arg_t {
+  explicit allocator_arg_t() = default;
+};
+
+#if _LIBCPP_STD_VER >= 17
+inline constexpr allocator_arg_t allocator_arg = allocator_arg_t();
+#elif !defined(_LIBCPP_CXX03_LANG)
+constexpr allocator_arg_t allocator_arg = allocator_arg_t();
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+
+// allocator construction
+
+template <class _Tp, class _Alloc, class... _Args>
+struct __uses_alloc_ctor_imp {
+  typedef _LIBCPP_NODEBUG __remove_cvref_t<_Alloc> _RawAlloc;
+  static const bool __ua = uses_allocator<_Tp, _RawAlloc>::value;
+  static const bool __ic = is_constructible<_Tp, allocator_arg_t, _Alloc, _Args...>::value;
+  static const int value = __ua ? 2 - __ic : 0;
+};
+
+template <class _Tp, class _Alloc, class... _Args>
+struct __uses_alloc_ctor : integral_constant<int, __uses_alloc_ctor_imp<_Tp, _Alloc, _Args...>::value> {};
+
+template <class _Tp, class _Allocator, class... _Args>
+inline _LIBCPP_HIDE_FROM_ABI void
+__user_alloc_construct_impl(integral_constant<int, 0>, _Tp* __storage, const _Allocator&, _Args&&... __args) {
+  new (__storage) _Tp(std::forward<_Args>(__args)...);
+}
+
+// FIXME: This should have a version which takes a non-const alloc.
+template <class _Tp, class _Allocator, class... _Args>
+inline _LIBCPP_HIDE_FROM_ABI void
+__user_alloc_construct_impl(integral_constant<int, 1>, _Tp* __storage, const _Allocator& __a, _Args&&... __args) {
+  new (__storage) _Tp(allocator_arg, __a, std::forward<_Args>(__args)...);
+}
+
+// FIXME: This should have a version which takes a non-const alloc.
+template <class _Tp, class _Allocator, class... _Args>
+inline _LIBCPP_HIDE_FROM_ABI void
+__user_alloc_construct_impl(integral_constant<int, 2>, _Tp* __storage, const _Allocator& __a, _Args&&... __args) {
+  new (__storage) _Tp(std::forward<_Args>(__args)..., __a);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FUNCTIONAL_ALLOCATOR_ARG_T_H
diff --git a/libcxx/include/__cxx03/__memory/allocator_destructor.h b/libcxx/include/__cxx03/__memory/allocator_destructor.h
new file mode 100644
index 000000000000..ed3d8918f5fe
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/allocator_destructor.h
@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ALLOCATOR_DESTRUCTOR_H
+#define _LIBCPP___MEMORY_ALLOCATOR_DESTRUCTOR_H
+
+#include <__config>
+#include <__memory/allocator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Alloc>
+class __allocator_destructor {
+  typedef _LIBCPP_NODEBUG allocator_traits<_Alloc> __alloc_traits;
+
+public:
+  typedef _LIBCPP_NODEBUG typename __alloc_traits::pointer pointer;
+  typedef _LIBCPP_NODEBUG typename __alloc_traits::size_type size_type;
+
+private:
+  _Alloc& __alloc_;
+  size_type __s_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __allocator_destructor(_Alloc& __a, size_type __s) _NOEXCEPT : __alloc_(__a), __s_(__s) {}
+  _LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) _NOEXCEPT { __alloc_traits::deallocate(__alloc_, __p, __s_); }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_ALLOCATOR_DESTRUCTOR_H
diff --git a/libcxx/include/__cxx03/__memory/allocator_traits.h b/libcxx/include/__cxx03/__memory/allocator_traits.h
new file mode 100644
index 000000000000..c5fcc89327b8
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/allocator_traits.h
@@ -0,0 +1,424 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ALLOCATOR_TRAITS_H
+#define _LIBCPP___MEMORY_ALLOCATOR_TRAITS_H
+
+#include <__config>
+#include <__memory/construct_at.h>
+#include <__memory/pointer_traits.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_empty.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/make_unsigned.h>
+#include <__type_traits/remove_reference.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <cstddef>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#define _LIBCPP_ALLOCATOR_TRAITS_HAS_XXX(NAME, PROPERTY)                                                               \
+  template <class _Tp, class = void>                                                                                   \
+  struct NAME : false_type {};                                                                                         \
+  template <class _Tp>                                                                                                 \
+  struct NAME<_Tp, __void_t<typename _Tp::PROPERTY > > : true_type {}
+
+// __pointer
+template <class _Tp,
+          class _Alloc,
+          class _RawAlloc = __libcpp_remove_reference_t<_Alloc>,
+          bool            = __has_pointer<_RawAlloc>::value>
+struct __pointer {
+  using type _LIBCPP_NODEBUG = typename _RawAlloc::pointer;
+};
+template <class _Tp, class _Alloc, class _RawAlloc>
+struct __pointer<_Tp, _Alloc, _RawAlloc, false> {
+  using type _LIBCPP_NODEBUG = _Tp*;
+};
+
+// __const_pointer
+_LIBCPP_ALLOCATOR_TRAITS_HAS_XXX(__has_const_pointer, const_pointer);
+template <class _Tp, class _Ptr, class _Alloc, bool = __has_const_pointer<_Alloc>::value>
+struct __const_pointer {
+  using type _LIBCPP_NODEBUG = typename _Alloc::const_pointer;
+};
+template <class _Tp, class _Ptr, class _Alloc>
+struct __const_pointer<_Tp, _Ptr, _Alloc, false> {
+#ifdef _LIBCPP_CXX03_LANG
+  using type = typename pointer_traits<_Ptr>::template rebind<const _Tp>::other;
+#else
+  using type _LIBCPP_NODEBUG = typename pointer_traits<_Ptr>::template rebind<const _Tp>;
+#endif
+};
+
+// __void_pointer
+_LIBCPP_ALLOCATOR_TRAITS_HAS_XXX(__has_void_pointer, void_pointer);
+template <class _Ptr, class _Alloc, bool = __has_void_pointer<_Alloc>::value>
+struct __void_pointer {
+  using type _LIBCPP_NODEBUG = typename _Alloc::void_pointer;
+};
+template <class _Ptr, class _Alloc>
+struct __void_pointer<_Ptr, _Alloc, false> {
+#ifdef _LIBCPP_CXX03_LANG
+  using type _LIBCPP_NODEBUG = typename pointer_traits<_Ptr>::template rebind<void>::other;
+#else
+  using type _LIBCPP_NODEBUG = typename pointer_traits<_Ptr>::template rebind<void>;
+#endif
+};
+
+// __const_void_pointer
+_LIBCPP_ALLOCATOR_TRAITS_HAS_XXX(__has_const_void_pointer, const_void_pointer);
+template <class _Ptr, class _Alloc, bool = __has_const_void_pointer<_Alloc>::value>
+struct __const_void_pointer {
+  using type _LIBCPP_NODEBUG = typename _Alloc::const_void_pointer;
+};
+template <class _Ptr, class _Alloc>
+struct __const_void_pointer<_Ptr, _Alloc, false> {
+#ifdef _LIBCPP_CXX03_LANG
+  using type _LIBCPP_NODEBUG = typename pointer_traits<_Ptr>::template rebind<const void>::other;
+#else
+  using type _LIBCPP_NODEBUG = typename pointer_traits<_Ptr>::template rebind<const void>;
+#endif
+};
+
+// __size_type
+_LIBCPP_ALLOCATOR_TRAITS_HAS_XXX(__has_size_type, size_type);
+template <class _Alloc, class _DiffType, bool = __has_size_type<_Alloc>::value>
+struct __size_type : make_unsigned<_DiffType> {};
+template <class _Alloc, class _DiffType>
+struct __size_type<_Alloc, _DiffType, true> {
+  using type _LIBCPP_NODEBUG = typename _Alloc::size_type;
+};
+
+// __alloc_traits_difference_type
+_LIBCPP_ALLOCATOR_TRAITS_HAS_XXX(__has_alloc_traits_difference_type, difference_type);
+template <class _Alloc, class _Ptr, bool = __has_alloc_traits_difference_type<_Alloc>::value>
+struct __alloc_traits_difference_type {
+  using type _LIBCPP_NODEBUG = typename pointer_traits<_Ptr>::difference_type;
+};
+template <class _Alloc, class _Ptr>
+struct __alloc_traits_difference_type<_Alloc, _Ptr, true> {
+  using type _LIBCPP_NODEBUG = typename _Alloc::difference_type;
+};
+
+// __propagate_on_container_copy_assignment
+_LIBCPP_ALLOCATOR_TRAITS_HAS_XXX(__has_propagate_on_container_copy_assignment, propagate_on_container_copy_assignment);
+template <class _Alloc, bool = __has_propagate_on_container_copy_assignment<_Alloc>::value>
+struct __propagate_on_container_copy_assignment : false_type {};
+template <class _Alloc>
+struct __propagate_on_container_copy_assignment<_Alloc, true> {
+  using type _LIBCPP_NODEBUG = typename _Alloc::propagate_on_container_copy_assignment;
+};
+
+// __propagate_on_container_move_assignment
+_LIBCPP_ALLOCATOR_TRAITS_HAS_XXX(__has_propagate_on_container_move_assignment, propagate_on_container_move_assignment);
+template <class _Alloc, bool = __has_propagate_on_container_move_assignment<_Alloc>::value>
+struct __propagate_on_container_move_assignment : false_type {};
+template <class _Alloc>
+struct __propagate_on_container_move_assignment<_Alloc, true> {
+  using type _LIBCPP_NODEBUG = typename _Alloc::propagate_on_container_move_assignment;
+};
+
+// __propagate_on_container_swap
+_LIBCPP_ALLOCATOR_TRAITS_HAS_XXX(__has_propagate_on_container_swap, propagate_on_container_swap);
+template <class _Alloc, bool = __has_propagate_on_container_swap<_Alloc>::value>
+struct __propagate_on_container_swap : false_type {};
+template <class _Alloc>
+struct __propagate_on_container_swap<_Alloc, true> {
+  using type _LIBCPP_NODEBUG = typename _Alloc::propagate_on_container_swap;
+};
+
+// __is_always_equal
+_LIBCPP_ALLOCATOR_TRAITS_HAS_XXX(__has_is_always_equal, is_always_equal);
+template <class _Alloc, bool = __has_is_always_equal<_Alloc>::value>
+struct __is_always_equal : is_empty<_Alloc> {};
+template <class _Alloc>
+struct __is_always_equal<_Alloc, true> {
+  using type _LIBCPP_NODEBUG = typename _Alloc::is_always_equal;
+};
+
+// __allocator_traits_rebind
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Tp, class _Up, class = void>
+struct __has_rebind_other : false_type {};
+template <class _Tp, class _Up>
+struct __has_rebind_other<_Tp, _Up, __void_t<typename _Tp::template rebind<_Up>::other> > : true_type {};
+
+template <class _Tp, class _Up, bool = __has_rebind_other<_Tp, _Up>::value>
+struct __allocator_traits_rebind {
+  static_assert(__has_rebind_other<_Tp, _Up>::value, "This allocator has to implement rebind");
+  using type _LIBCPP_NODEBUG = typename _Tp::template rebind<_Up>::other;
+};
+template <template <class, class...> class _Alloc, class _Tp, class... _Args, class _Up>
+struct __allocator_traits_rebind<_Alloc<_Tp, _Args...>, _Up, true> {
+  using type _LIBCPP_NODEBUG = typename _Alloc<_Tp, _Args...>::template rebind<_Up>::other;
+};
+template <template <class, class...> class _Alloc, class _Tp, class... _Args, class _Up>
+struct __allocator_traits_rebind<_Alloc<_Tp, _Args...>, _Up, false> {
+  using type _LIBCPP_NODEBUG = _Alloc<_Up, _Args...>;
+};
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+template <class _Alloc, class _Tp>
+using __allocator_traits_rebind_t = typename __allocator_traits_rebind<_Alloc, _Tp>::type;
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+
+// __has_allocate_hint
+template <class _Alloc, class _SizeType, class _ConstVoidPtr, class = void>
+struct __has_allocate_hint : false_type {};
+
+template <class _Alloc, class _SizeType, class _ConstVoidPtr>
+struct __has_allocate_hint<
+    _Alloc,
+    _SizeType,
+    _ConstVoidPtr,
+    decltype((void)std::declval<_Alloc>().allocate(std::declval<_SizeType>(), std::declval<_ConstVoidPtr>()))>
+    : true_type {};
+
+// __has_construct
+template <class, class _Alloc, class... _Args>
+struct __has_construct_impl : false_type {};
+
+template <class _Alloc, class... _Args>
+struct __has_construct_impl<decltype((void)std::declval<_Alloc>().construct(std::declval<_Args>()...)),
+                            _Alloc,
+                            _Args...> : true_type {};
+
+template <class _Alloc, class... _Args>
+struct __has_construct : __has_construct_impl<void, _Alloc, _Args...> {};
+
+// __has_destroy
+template <class _Alloc, class _Pointer, class = void>
+struct __has_destroy : false_type {};
+
+template <class _Alloc, class _Pointer>
+struct __has_destroy<_Alloc, _Pointer, decltype((void)std::declval<_Alloc>().destroy(std::declval<_Pointer>()))>
+    : true_type {};
+
+// __has_max_size
+template <class _Alloc, class = void>
+struct __has_max_size : false_type {};
+
+template <class _Alloc>
+struct __has_max_size<_Alloc, decltype((void)std::declval<_Alloc&>().max_size())> : true_type {};
+
+// __has_select_on_container_copy_construction
+template <class _Alloc, class = void>
+struct __has_select_on_container_copy_construction : false_type {};
+
+template <class _Alloc>
+struct __has_select_on_container_copy_construction<
+    _Alloc,
+    decltype((void)std::declval<_Alloc>().select_on_container_copy_construction())> : true_type {};
+
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Pointer, class _SizeType = size_t>
+struct allocation_result {
+  _Pointer ptr;
+  _SizeType count;
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(allocation_result);
+
+#endif // _LIBCPP_STD_VER
+
+template <class _Alloc>
+struct _LIBCPP_TEMPLATE_VIS allocator_traits {
+  using allocator_type     = _Alloc;
+  using value_type         = typename allocator_type::value_type;
+  using pointer            = typename __pointer<value_type, allocator_type>::type;
+  using const_pointer      = typename __const_pointer<value_type, pointer, allocator_type>::type;
+  using void_pointer       = typename __void_pointer<pointer, allocator_type>::type;
+  using const_void_pointer = typename __const_void_pointer<pointer, allocator_type>::type;
+  using difference_type    = typename __alloc_traits_difference_type<allocator_type, pointer>::type;
+  using size_type          = typename __size_type<allocator_type, difference_type>::type;
+  using propagate_on_container_copy_assignment =
+      typename __propagate_on_container_copy_assignment<allocator_type>::type;
+  using propagate_on_container_move_assignment =
+      typename __propagate_on_container_move_assignment<allocator_type>::type;
+  using propagate_on_container_swap = typename __propagate_on_container_swap<allocator_type>::type;
+  using is_always_equal             = typename __is_always_equal<allocator_type>::type;
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class _Tp>
+  using rebind_alloc = __allocator_traits_rebind_t<allocator_type, _Tp>;
+  template <class _Tp>
+  using rebind_traits = allocator_traits<rebind_alloc<_Tp> >;
+#else  // _LIBCPP_CXX03_LANG
+  template <class _Tp>
+  struct rebind_alloc {
+    using other = __allocator_traits_rebind_t<allocator_type, _Tp>;
+  };
+  template <class _Tp>
+  struct rebind_traits {
+    using other = allocator_traits<typename rebind_alloc<_Tp>::other>;
+  };
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static pointer
+  allocate(allocator_type& __a, size_type __n) {
+    return __a.allocate(__n);
+  }
+
+  template <class _Ap = _Alloc, __enable_if_t<__has_allocate_hint<_Ap, size_type, const_void_pointer>::value, int> = 0>
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static pointer
+  allocate(allocator_type& __a, size_type __n, const_void_pointer __hint) {
+    _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+    return __a.allocate(__n, __hint);
+    _LIBCPP_SUPPRESS_DEPRECATED_POP
+  }
+  template <class _Ap                                                                           = _Alloc,
+            class                                                                               = void,
+            __enable_if_t<!__has_allocate_hint<_Ap, size_type, const_void_pointer>::value, int> = 0>
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static pointer
+  allocate(allocator_type& __a, size_type __n, const_void_pointer) {
+    return __a.allocate(__n);
+  }
+
+#if _LIBCPP_STD_VER >= 23
+  template <class _Ap = _Alloc>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr allocation_result<pointer, size_type>
+  allocate_at_least(_Ap& __alloc, size_type __n) {
+    if constexpr (requires { __alloc.allocate_at_least(__n); }) {
+      return __alloc.allocate_at_least(__n);
+    } else {
+      return {__alloc.allocate(__n), __n};
+    }
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void
+  deallocate(allocator_type& __a, pointer __p, size_type __n) _NOEXCEPT {
+    __a.deallocate(__p, __n);
+  }
+
+  template <class _Tp, class... _Args, __enable_if_t<__has_construct<allocator_type, _Tp*, _Args...>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void
+  construct(allocator_type& __a, _Tp* __p, _Args&&... __args) {
+    _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+    __a.construct(__p, std::forward<_Args>(__args)...);
+    _LIBCPP_SUPPRESS_DEPRECATED_POP
+  }
+  template <class _Tp,
+            class... _Args,
+            class                                                                       = void,
+            __enable_if_t<!__has_construct<allocator_type, _Tp*, _Args...>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void
+  construct(allocator_type&, _Tp* __p, _Args&&... __args) {
+    std::__construct_at(__p, std::forward<_Args>(__args)...);
+  }
+
+  template <class _Tp, __enable_if_t<__has_destroy<allocator_type, _Tp*>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void destroy(allocator_type& __a, _Tp* __p) {
+    _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+    __a.destroy(__p);
+    _LIBCPP_SUPPRESS_DEPRECATED_POP
+  }
+  template <class _Tp, class = void, __enable_if_t<!__has_destroy<allocator_type, _Tp*>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void destroy(allocator_type&, _Tp* __p) {
+    std::__destroy_at(__p);
+  }
+
+  template <class _Ap = _Alloc, __enable_if_t<__has_max_size<const _Ap>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static size_type max_size(const allocator_type& __a) _NOEXCEPT {
+    _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+    return __a.max_size();
+    _LIBCPP_SUPPRESS_DEPRECATED_POP
+  }
+  template <class _Ap = _Alloc, class = void, __enable_if_t<!__has_max_size<const _Ap>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static size_type max_size(const allocator_type&) _NOEXCEPT {
+    return numeric_limits<size_type>::max() / sizeof(value_type);
+  }
+
+  template <class _Ap = _Alloc, __enable_if_t<__has_select_on_container_copy_construction<const _Ap>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static allocator_type
+  select_on_container_copy_construction(const allocator_type& __a) {
+    return __a.select_on_container_copy_construction();
+  }
+  template <class _Ap                                                                          = _Alloc,
+            class                                                                              = void,
+            __enable_if_t<!__has_select_on_container_copy_construction<const _Ap>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static allocator_type
+  select_on_container_copy_construction(const allocator_type& __a) {
+    return __a;
+  }
+};
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Traits, class _Tp>
+using __rebind_alloc _LIBCPP_NODEBUG = typename _Traits::template rebind_alloc<_Tp>;
+#else
+template <class _Traits, class _Tp>
+using __rebind_alloc = typename _Traits::template rebind_alloc<_Tp>::other;
+#endif
+
+template <class _Alloc>
+struct __check_valid_allocator : true_type {
+  using _Traits = std::allocator_traits<_Alloc>;
+  static_assert(is_same<_Alloc, __rebind_alloc<_Traits, typename _Traits::value_type> >::value,
+                "[allocator.requirements] states that rebinding an allocator to the same type should result in the "
+                "original allocator");
+};
+
+// __is_default_allocator
+template <class _Tp>
+struct __is_default_allocator : false_type {};
+
+template <class>
+class allocator;
+
+template <class _Tp>
+struct __is_default_allocator<allocator<_Tp> > : true_type {};
+
+// __is_cpp17_move_insertable
+template <class _Alloc, class = void>
+struct __is_cpp17_move_insertable : is_move_constructible<typename _Alloc::value_type> {};
+
+template <class _Alloc>
+struct __is_cpp17_move_insertable<
+    _Alloc,
+    __enable_if_t< !__is_default_allocator<_Alloc>::value &&
+                   __has_construct<_Alloc, typename _Alloc::value_type*, typename _Alloc::value_type&&>::value > >
+    : true_type {};
+
+// __is_cpp17_copy_insertable
+template <class _Alloc, class = void>
+struct __is_cpp17_copy_insertable
+    : integral_constant<bool,
+                        is_copy_constructible<typename _Alloc::value_type>::value &&
+                            __is_cpp17_move_insertable<_Alloc>::value > {};
+
+template <class _Alloc>
+struct __is_cpp17_copy_insertable<
+    _Alloc,
+    __enable_if_t< !__is_default_allocator<_Alloc>::value &&
+                   __has_construct<_Alloc, typename _Alloc::value_type*, const typename _Alloc::value_type&>::value > >
+    : __is_cpp17_move_insertable<_Alloc> {};
+
+#undef _LIBCPP_ALLOCATOR_TRAITS_HAS_XXX
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_ALLOCATOR_TRAITS_H
diff --git a/libcxx/include/__cxx03/__memory/assume_aligned.h b/libcxx/include/__cxx03/__memory/assume_aligned.h
new file mode 100644
index 000000000000..526eb3334f95
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/assume_aligned.h
@@ -0,0 +1,50 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ASSUME_ALIGNED_H
+#define _LIBCPP___MEMORY_ASSUME_ALIGNED_H
+
+#include <__assert>
+#include <__config>
+#include <__type_traits/is_constant_evaluated.h>
+#include <cstddef>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <size_t _Np, class _Tp>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __assume_aligned(_Tp* __ptr) {
+  static_assert(_Np != 0 && (_Np & (_Np - 1)) == 0, "std::assume_aligned<N>(p) requires N to be a power of two");
+
+  if (__libcpp_is_constant_evaluated()) {
+    (void)__builtin_assume_aligned(__ptr, _Np);
+    return __ptr;
+  } else {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        reinterpret_cast<uintptr_t>(__ptr) % _Np == 0, "Alignment assumption is violated");
+    return static_cast<_Tp*>(__builtin_assume_aligned(__ptr, _Np));
+  }
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+template <size_t _Np, class _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp* assume_aligned(_Tp* __ptr) {
+  return std::__assume_aligned<_Np>(__ptr);
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_ASSUME_ALIGNED_H
diff --git a/libcxx/include/__cxx03/__memory/auto_ptr.h b/libcxx/include/__cxx03/__memory/auto_ptr.h
new file mode 100644
index 000000000000..752143616bb2
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/auto_ptr.h
@@ -0,0 +1,92 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_AUTO_PTR_H
+#define _LIBCPP___MEMORY_AUTO_PTR_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR)
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_DEPRECATED_IN_CXX11 auto_ptr_ref {
+  _Tp* __ptr_;
+};
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 auto_ptr {
+private:
+  _Tp* __ptr_;
+
+public:
+  typedef _Tp element_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit auto_ptr(_Tp* __p = 0) _NOEXCEPT : __ptr_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI auto_ptr(auto_ptr& __p) _NOEXCEPT : __ptr_(__p.release()) {}
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI auto_ptr(auto_ptr<_Up>& __p) _NOEXCEPT : __ptr_(__p.release()) {}
+  _LIBCPP_HIDE_FROM_ABI auto_ptr& operator=(auto_ptr& __p) _NOEXCEPT {
+    reset(__p.release());
+    return *this;
+  }
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI auto_ptr& operator=(auto_ptr<_Up>& __p) _NOEXCEPT {
+    reset(__p.release());
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI auto_ptr& operator=(auto_ptr_ref<_Tp> __p) _NOEXCEPT {
+    reset(__p.__ptr_);
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI ~auto_ptr() _NOEXCEPT { delete __ptr_; }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp& operator*() const _NOEXCEPT { return *__ptr_; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator->() const _NOEXCEPT { return __ptr_; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* get() const _NOEXCEPT { return __ptr_; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* release() _NOEXCEPT {
+    _Tp* __t = __ptr_;
+    __ptr_   = nullptr;
+    return __t;
+  }
+  _LIBCPP_HIDE_FROM_ABI void reset(_Tp* __p = 0) _NOEXCEPT {
+    if (__ptr_ != __p)
+      delete __ptr_;
+    __ptr_ = __p;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI auto_ptr(auto_ptr_ref<_Tp> __p) _NOEXCEPT : __ptr_(__p.__ptr_) {}
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI operator auto_ptr_ref<_Up>() _NOEXCEPT {
+    auto_ptr_ref<_Up> __t;
+    __t.__ptr_ = release();
+    return __t;
+  }
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI operator auto_ptr<_Up>() _NOEXCEPT {
+    return auto_ptr<_Up>(release());
+  }
+};
+
+template <>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 auto_ptr<void> {
+public:
+  typedef void element_type;
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR)
+
+#endif // _LIBCPP___MEMORY_AUTO_PTR_H
diff --git a/libcxx/include/__cxx03/__memory/builtin_new_allocator.h b/libcxx/include/__cxx03/__memory/builtin_new_allocator.h
new file mode 100644
index 000000000000..c6f7f3c5ff52
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/builtin_new_allocator.h
@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_BUILTIN_NEW_ALLOCATOR_H
+#define _LIBCPP___MEMORY_BUILTIN_NEW_ALLOCATOR_H
+
+#include <__config>
+#include <__memory/unique_ptr.h>
+#include <cstddef>
+#include <new>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// __builtin_new_allocator -- A non-templated helper for allocating and
+// deallocating memory using __builtin_operator_new and
+// __builtin_operator_delete. It should be used in preference to
+// `std::allocator<T>` to avoid additional instantiations.
+struct __builtin_new_allocator {
+  struct __builtin_new_deleter {
+    typedef void* pointer_type;
+
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __builtin_new_deleter(size_t __size, size_t __align)
+        : __size_(__size), __align_(__align) {}
+
+    _LIBCPP_HIDE_FROM_ABI void operator()(void* __p) const _NOEXCEPT {
+      std::__libcpp_deallocate(__p, __size_, __align_);
+    }
+
+  private:
+    size_t __size_;
+    size_t __align_;
+  };
+
+  typedef unique_ptr<void, __builtin_new_deleter> __holder_t;
+
+  _LIBCPP_HIDE_FROM_ABI static __holder_t __allocate_bytes(size_t __s, size_t __align) {
+    return __holder_t(std::__libcpp_allocate(__s, __align), __builtin_new_deleter(__s, __align));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static void __deallocate_bytes(void* __p, size_t __s, size_t __align) _NOEXCEPT {
+    std::__libcpp_deallocate(__p, __s, __align);
+  }
+
+  template <class _Tp>
+  _LIBCPP_NODEBUG _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI static __holder_t __allocate_type(size_t __n) {
+    return __allocate_bytes(__n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp));
+  }
+
+  template <class _Tp>
+  _LIBCPP_NODEBUG _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI static void
+  __deallocate_type(void* __p, size_t __n) _NOEXCEPT {
+    __deallocate_bytes(__p, __n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp));
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_BUILTIN_NEW_ALLOCATOR_H
diff --git a/libcxx/include/__cxx03/__memory/compressed_pair.h b/libcxx/include/__cxx03/__memory/compressed_pair.h
new file mode 100644
index 000000000000..40e5cfc35fb0
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/compressed_pair.h
@@ -0,0 +1,171 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_COMPRESSED_PAIR_H
+#define _LIBCPP___MEMORY_COMPRESSED_PAIR_H
+
+#include <__config>
+#include <__fwd/tuple.h>
+#include <__tuple/tuple_indices.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/dependent_type.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_empty.h>
+#include <__type_traits/is_final.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_swappable.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/piecewise_construct.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Tag used to default initialize one or both of the pair's elements.
+struct __default_init_tag {};
+struct __value_init_tag {};
+
+template <class _Tp, int _Idx, bool _CanBeEmptyBase = is_empty<_Tp>::value && !__libcpp_is_final<_Tp>::value>
+struct __compressed_pair_elem {
+  using _ParamT         = _Tp;
+  using reference       = _Tp&;
+  using const_reference = const _Tp&;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(__default_init_tag) {}
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(__value_init_tag) : __value_() {}
+
+  template <class _Up, __enable_if_t<!is_same<__compressed_pair_elem, __decay_t<_Up> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(_Up&& __u)
+      : __value_(std::forward<_Up>(__u)) {}
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class... _Args, size_t... _Indices>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 explicit __compressed_pair_elem(
+      piecewise_construct_t, tuple<_Args...> __args, __tuple_indices<_Indices...>)
+      : __value_(std::forward<_Args>(std::get<_Indices>(__args))...) {}
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 reference __get() _NOEXCEPT { return __value_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR const_reference __get() const _NOEXCEPT { return __value_; }
+
+private:
+  _Tp __value_;
+};
+
+template <class _Tp, int _Idx>
+struct __compressed_pair_elem<_Tp, _Idx, true> : private _Tp {
+  using _ParamT         = _Tp;
+  using reference       = _Tp&;
+  using const_reference = const _Tp&;
+  using __value_type    = _Tp;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem() = default;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(__default_init_tag) {}
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(__value_init_tag) : __value_type() {}
+
+  template <class _Up, __enable_if_t<!is_same<__compressed_pair_elem, __decay_t<_Up> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(_Up&& __u)
+      : __value_type(std::forward<_Up>(__u)) {}
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class... _Args, size_t... _Indices>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17
+  __compressed_pair_elem(piecewise_construct_t, tuple<_Args...> __args, __tuple_indices<_Indices...>)
+      : __value_type(std::forward<_Args>(std::get<_Indices>(__args))...) {}
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 reference __get() _NOEXCEPT { return *this; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR const_reference __get() const _NOEXCEPT { return *this; }
+};
+
+template <class _T1, class _T2>
+class __compressed_pair : private __compressed_pair_elem<_T1, 0>, private __compressed_pair_elem<_T2, 1> {
+public:
+  // NOTE: This static assert should never fire because __compressed_pair
+  // is *almost never* used in a scenario where it's possible for T1 == T2.
+  // (The exception is std::function where it is possible that the function
+  //  object and the allocator have the same type).
+  static_assert(
+      (!is_same<_T1, _T2>::value),
+      "__compressed_pair cannot be instantiated when T1 and T2 are the same type; "
+      "The current implementation is NOT ABI-compatible with the previous implementation for this configuration");
+
+  using _Base1 _LIBCPP_NODEBUG = __compressed_pair_elem<_T1, 0>;
+  using _Base2 _LIBCPP_NODEBUG = __compressed_pair_elem<_T2, 1>;
+
+  template <bool _Dummy         = true,
+            __enable_if_t< __dependent_type<is_default_constructible<_T1>, _Dummy>::value &&
+                               __dependent_type<is_default_constructible<_T2>, _Dummy>::value,
+                           int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair()
+      : _Base1(__value_init_tag()), _Base2(__value_init_tag()) {}
+
+  template <class _U1, class _U2>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair(_U1&& __t1, _U2&& __t2)
+      : _Base1(std::forward<_U1>(__t1)), _Base2(std::forward<_U2>(__t2)) {}
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class... _Args1, class... _Args2>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 explicit __compressed_pair(
+      piecewise_construct_t __pc, tuple<_Args1...> __first_args, tuple<_Args2...> __second_args)
+      : _Base1(__pc, std::move(__first_args), typename __make_tuple_indices<sizeof...(_Args1)>::type()),
+        _Base2(__pc, std::move(__second_args), typename __make_tuple_indices<sizeof...(_Args2)>::type()) {}
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename _Base1::reference first() _NOEXCEPT {
+    return static_cast<_Base1&>(*this).__get();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename _Base1::const_reference first() const _NOEXCEPT {
+    return static_cast<_Base1 const&>(*this).__get();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename _Base2::reference second() _NOEXCEPT {
+    return static_cast<_Base2&>(*this).__get();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename _Base2::const_reference second() const _NOEXCEPT {
+    return static_cast<_Base2 const&>(*this).__get();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR static _Base1* __get_first_base(__compressed_pair* __pair) _NOEXCEPT {
+    return static_cast<_Base1*>(__pair);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR static _Base2* __get_second_base(__compressed_pair* __pair) _NOEXCEPT {
+    return static_cast<_Base2*>(__pair);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void swap(__compressed_pair& __x)
+      _NOEXCEPT_(__is_nothrow_swappable_v<_T1>&& __is_nothrow_swappable_v<_T2>) {
+    using std::swap;
+    swap(first(), __x.first());
+    swap(second(), __x.second());
+  }
+};
+
+template <class _T1, class _T2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+swap(__compressed_pair<_T1, _T2>& __x, __compressed_pair<_T1, _T2>& __y)
+    _NOEXCEPT_(__is_nothrow_swappable_v<_T1>&& __is_nothrow_swappable_v<_T2>) {
+  __x.swap(__y);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_COMPRESSED_PAIR_H
diff --git a/libcxx/include/__cxx03/__memory/concepts.h b/libcxx/include/__cxx03/__memory/concepts.h
new file mode 100644
index 000000000000..216144aad748
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/concepts.h
@@ -0,0 +1,63 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_CONCEPTS_H
+#define _LIBCPP___MEMORY_CONCEPTS_H
+
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/readable_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/remove_reference.h> // TODO(modules): This should not be required
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+// [special.mem.concepts]
+
+// This concept ensures that uninitialized algorithms can construct an object
+// at the address pointed-to by the iterator, which requires an lvalue.
+template <class _Ip>
+concept __nothrow_input_iterator =
+    input_iterator<_Ip> && is_lvalue_reference_v<iter_reference_t<_Ip>> &&
+    same_as<remove_cvref_t<iter_reference_t<_Ip>>, iter_value_t<_Ip>>;
+
+template <class _Sp, class _Ip>
+concept __nothrow_sentinel_for = sentinel_for<_Sp, _Ip>;
+
+template <class _Rp>
+concept __nothrow_input_range =
+    range<_Rp> && __nothrow_input_iterator<iterator_t<_Rp>> && __nothrow_sentinel_for<sentinel_t<_Rp>, iterator_t<_Rp>>;
+
+template <class _Ip>
+concept __nothrow_forward_iterator =
+    __nothrow_input_iterator<_Ip> && forward_iterator<_Ip> && __nothrow_sentinel_for<_Ip, _Ip>;
+
+template <class _Rp>
+concept __nothrow_forward_range = __nothrow_input_range<_Rp> && __nothrow_forward_iterator<iterator_t<_Rp>>;
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_CONCEPTS_H
diff --git a/libcxx/include/__cxx03/__memory/construct_at.h b/libcxx/include/__cxx03/__memory/construct_at.h
new file mode 100644
index 000000000000..eb0213248006
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/construct_at.h
@@ -0,0 +1,128 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_CONSTRUCT_AT_H
+#define _LIBCPP___MEMORY_CONSTRUCT_AT_H
+
+#include <__assert>
+#include <__config>
+#include <__iterator/access.h>
+#include <__memory/addressof.h>
+#include <__memory/voidify.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_array.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <new>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// construct_at
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp, class... _Args, class = decltype(::new(std::declval<void*>()) _Tp(std::declval<_Args>()...))>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp* construct_at(_Tp* __location, _Args&&... __args) {
+  _LIBCPP_ASSERT_NON_NULL(__location != nullptr, "null pointer given to construct_at");
+  return ::new (std::__voidify(*__location)) _Tp(std::forward<_Args>(__args)...);
+}
+
+#endif
+
+template <class _Tp, class... _Args, class = decltype(::new(std::declval<void*>()) _Tp(std::declval<_Args>()...))>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __construct_at(_Tp* __location, _Args&&... __args) {
+#if _LIBCPP_STD_VER >= 20
+  return std::construct_at(__location, std::forward<_Args>(__args)...);
+#else
+  return _LIBCPP_ASSERT_NON_NULL(__location != nullptr, "null pointer given to construct_at"),
+         ::new (std::__voidify(*__location)) _Tp(std::forward<_Args>(__args)...);
+#endif
+}
+
+// destroy_at
+
+// The internal functions are available regardless of the language version (with the exception of the `__destroy_at`
+// taking an array).
+
+template <class _ForwardIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator __destroy(_ForwardIterator, _ForwardIterator);
+
+template <class _Tp, __enable_if_t<!is_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __destroy_at(_Tp* __loc) {
+  _LIBCPP_ASSERT_NON_NULL(__loc != nullptr, "null pointer given to destroy_at");
+  __loc->~_Tp();
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp, __enable_if_t<is_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __destroy_at(_Tp* __loc) {
+  _LIBCPP_ASSERT_NON_NULL(__loc != nullptr, "null pointer given to destroy_at");
+  std::__destroy(std::begin(*__loc), std::end(*__loc));
+}
+#endif
+
+template <class _ForwardIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
+__destroy(_ForwardIterator __first, _ForwardIterator __last) {
+  for (; __first != __last; ++__first)
+    std::__destroy_at(std::addressof(*__first));
+  return __first;
+}
+
+template <class _BidirectionalIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _BidirectionalIterator
+__reverse_destroy(_BidirectionalIterator __first, _BidirectionalIterator __last) {
+  while (__last != __first) {
+    --__last;
+    std::__destroy_at(std::addressof(*__last));
+  }
+  return __last;
+}
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _Tp, enable_if_t<!is_array_v<_Tp>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void destroy_at(_Tp* __loc) {
+  std::__destroy_at(__loc);
+}
+
+#  if _LIBCPP_STD_VER >= 20
+template <class _Tp, enable_if_t<is_array_v<_Tp>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void destroy_at(_Tp* __loc) {
+  std::__destroy_at(__loc);
+}
+#  endif
+
+template <class _ForwardIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void destroy(_ForwardIterator __first, _ForwardIterator __last) {
+  (void)std::__destroy(std::move(__first), std::move(__last));
+}
+
+template <class _ForwardIterator, class _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator destroy_n(_ForwardIterator __first, _Size __n) {
+  for (; __n > 0; (void)++__first, --__n)
+    std::__destroy_at(std::addressof(*__first));
+  return __first;
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_CONSTRUCT_AT_H
diff --git a/libcxx/include/__cxx03/__memory/destruct_n.h b/libcxx/include/__cxx03/__memory/destruct_n.h
new file mode 100644
index 000000000000..78635ad0af04
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/destruct_n.h
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_DESTRUCT_N_H
+#define _LIBCPP___MEMORY_DESTRUCT_N_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_trivially_destructible.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct __destruct_n {
+private:
+  size_t __size_;
+
+  template <class _Tp>
+  _LIBCPP_HIDE_FROM_ABI void __process(_Tp* __p, false_type) _NOEXCEPT {
+    for (size_t __i = 0; __i < __size_; ++__i, ++__p)
+      __p->~_Tp();
+  }
+
+  template <class _Tp>
+  _LIBCPP_HIDE_FROM_ABI void __process(_Tp*, true_type) _NOEXCEPT {}
+
+  _LIBCPP_HIDE_FROM_ABI void __incr(false_type) _NOEXCEPT { ++__size_; }
+  _LIBCPP_HIDE_FROM_ABI void __incr(true_type) _NOEXCEPT {}
+
+  _LIBCPP_HIDE_FROM_ABI void __set(size_t __s, false_type) _NOEXCEPT { __size_ = __s; }
+  _LIBCPP_HIDE_FROM_ABI void __set(size_t, true_type) _NOEXCEPT {}
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __destruct_n(size_t __s) _NOEXCEPT : __size_(__s) {}
+
+  template <class _Tp>
+  _LIBCPP_HIDE_FROM_ABI void __incr() _NOEXCEPT {
+    __incr(integral_constant<bool, is_trivially_destructible<_Tp>::value>());
+  }
+
+  template <class _Tp>
+  _LIBCPP_HIDE_FROM_ABI void __set(size_t __s, _Tp*) _NOEXCEPT {
+    __set(__s, integral_constant<bool, is_trivially_destructible<_Tp>::value>());
+  }
+
+  template <class _Tp>
+  _LIBCPP_HIDE_FROM_ABI void operator()(_Tp* __p) _NOEXCEPT {
+    __process(__p, integral_constant<bool, is_trivially_destructible<_Tp>::value>());
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_DESTRUCT_N_H
diff --git a/libcxx/include/__cxx03/__memory/inout_ptr.h b/libcxx/include/__cxx03/__memory/inout_ptr.h
new file mode 100644
index 000000000000..e5f3ac5d027e
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/inout_ptr.h
@@ -0,0 +1,109 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___INOUT_PTR_H
+#define _LIBCPP___INOUT_PTR_H
+
+#include <__config>
+#include <__memory/addressof.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/shared_ptr.h>
+#include <__memory/unique_ptr.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_specialization.h>
+#include <__type_traits/is_void.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <tuple>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Smart, class _Pointer, class... _Args>
+class _LIBCPP_TEMPLATE_VIS inout_ptr_t {
+  static_assert(!__is_specialization_v<_Smart, shared_ptr>, "std::shared_ptr<> is not supported with std::inout_ptr.");
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit inout_ptr_t(_Smart& __smart, _Args... __args)
+      : __s_(__smart), __a_(std::forward<_Args>(__args)...), __p_([&__smart] {
+          if constexpr (is_pointer_v<_Smart>) {
+            return __smart;
+          } else {
+            return __smart.get();
+          }
+        }()) {
+    if constexpr (requires { __s_.release(); }) {
+      __s_.release();
+    } else {
+      __s_ = _Smart();
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI inout_ptr_t(const inout_ptr_t&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI ~inout_ptr_t() {
+    // LWG-3897 inout_ptr will not update raw pointer to null
+    if constexpr (!is_pointer_v<_Smart>) {
+      if (!__p_) {
+        return;
+      }
+    }
+
+    using _SmartPtr = __pointer_of_or_t<_Smart, _Pointer>;
+    if constexpr (is_pointer_v<_Smart>) {
+      std::apply([&](auto&&... __args) { __s_ = _Smart(static_cast<_SmartPtr>(__p_), std::forward<_Args>(__args)...); },
+                 std::move(__a_));
+    } else if constexpr (__resettable_smart_pointer_with_args<_Smart, _Pointer, _Args...>) {
+      std::apply([&](auto&&... __args) { __s_.reset(static_cast<_SmartPtr>(__p_), std::forward<_Args>(__args)...); },
+                 std::move(__a_));
+    } else {
+      static_assert(is_constructible_v<_Smart, _SmartPtr, _Args...>,
+                    "The smart pointer must be constructible from arguments of types _Smart, _Pointer, _Args...");
+      std::apply([&](auto&&... __args) { __s_ = _Smart(static_cast<_SmartPtr>(__p_), std::forward<_Args>(__args)...); },
+                 std::move(__a_));
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI operator _Pointer*() const noexcept { return std::addressof(const_cast<_Pointer&>(__p_)); }
+
+  _LIBCPP_HIDE_FROM_ABI operator void**() const noexcept
+    requires(!is_same_v<_Pointer, void*>)
+  {
+    static_assert(is_pointer_v<_Pointer>, "The conversion to void** requires _Pointer to be a raw pointer.");
+
+    return reinterpret_cast<void**>(static_cast<_Pointer*>(*this));
+  }
+
+private:
+  _Smart& __s_;
+  tuple<_Args...> __a_;
+  _Pointer __p_;
+};
+
+template <class _Pointer = void, class _Smart, class... _Args>
+_LIBCPP_HIDE_FROM_ABI auto inout_ptr(_Smart& __s, _Args&&... __args) {
+  using _Ptr = conditional_t<is_void_v<_Pointer>, __pointer_of_t<_Smart>, _Pointer>;
+  return std::inout_ptr_t<_Smart, _Ptr, _Args&&...>(__s, std::forward<_Args>(__args)...);
+}
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___INOUT_PTR_H
diff --git a/libcxx/include/__cxx03/__memory/out_ptr.h b/libcxx/include/__cxx03/__memory/out_ptr.h
new file mode 100644
index 000000000000..fd99110790cc
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/out_ptr.h
@@ -0,0 +1,101 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___OUT_PTR_H
+#define _LIBCPP___OUT_PTR_H
+
+#include <__config>
+#include <__memory/addressof.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/shared_ptr.h>
+#include <__memory/unique_ptr.h>
+#include <__type_traits/is_specialization.h>
+#include <__type_traits/is_void.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <tuple>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Smart, class _Pointer, class... _Args>
+class _LIBCPP_TEMPLATE_VIS out_ptr_t {
+  static_assert(!__is_specialization_v<_Smart, shared_ptr> || sizeof...(_Args) > 0,
+                "Using std::shared_ptr<> without a deleter in std::out_ptr is not supported.");
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit out_ptr_t(_Smart& __smart, _Args... __args)
+      : __s_(__smart), __a_(std::forward<_Args>(__args)...), __p_() {
+    using _Ptr = decltype(__smart);
+    if constexpr (__resettable_smart_pointer<_Ptr>) {
+      __s_.reset();
+    } else if constexpr (is_constructible_v<_Smart>) {
+      __s_ = _Smart();
+    } else {
+      static_assert(__resettable_smart_pointer<_Ptr> || is_constructible_v<_Smart>,
+                    "The adapted pointer type must have a reset() member function or be default constructible.");
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI out_ptr_t(const out_ptr_t&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI ~out_ptr_t() {
+    if (!__p_) {
+      return;
+    }
+
+    using _SmartPtr = __pointer_of_or_t<_Smart, _Pointer>;
+    if constexpr (__resettable_smart_pointer_with_args<_Smart, _Pointer, _Args...>) {
+      std::apply([&](auto&&... __args) { __s_.reset(static_cast<_SmartPtr>(__p_), std::forward<_Args>(__args)...); },
+                 std::move(__a_));
+    } else {
+      static_assert(is_constructible_v<_Smart, _SmartPtr, _Args...>,
+                    "The smart pointer must be constructible from arguments of types _Smart, _Pointer, _Args...");
+      std::apply([&](auto&&... __args) { __s_ = _Smart(static_cast<_SmartPtr>(__p_), std::forward<_Args>(__args)...); },
+                 std::move(__a_));
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI operator _Pointer*() const noexcept { return std::addressof(const_cast<_Pointer&>(__p_)); }
+
+  _LIBCPP_HIDE_FROM_ABI operator void**() const noexcept
+    requires(!is_same_v<_Pointer, void*>)
+  {
+    static_assert(is_pointer_v<_Pointer>, "The conversion to void** requires _Pointer to be a raw pointer.");
+
+    return reinterpret_cast<void**>(static_cast<_Pointer*>(*this));
+  }
+
+private:
+  _Smart& __s_;
+  tuple<_Args...> __a_;
+  _Pointer __p_ = _Pointer();
+};
+
+template <class _Pointer = void, class _Smart, class... _Args>
+_LIBCPP_HIDE_FROM_ABI auto out_ptr(_Smart& __s, _Args&&... __args) {
+  using _Ptr = conditional_t<is_void_v<_Pointer>, __pointer_of_t<_Smart>, _Pointer>;
+  return std::out_ptr_t<_Smart, _Ptr, _Args&&...>(__s, std::forward<_Args>(__args)...);
+}
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___OUT_PTR_H
diff --git a/libcxx/include/__cxx03/__memory/pointer_traits.h b/libcxx/include/__cxx03/__memory/pointer_traits.h
new file mode 100644
index 000000000000..0914aceb318b
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/pointer_traits.h
@@ -0,0 +1,307 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_POINTER_TRAITS_H
+#define _LIBCPP___MEMORY_POINTER_TRAITS_H
+
+#include <__config>
+#include <__memory/addressof.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_class.h>
+#include <__type_traits/is_function.h>
+#include <__type_traits/is_void.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// clang-format off
+#define _LIBCPP_CLASS_TRAITS_HAS_XXX(NAME, PROPERTY)                                                                   \
+  template <class _Tp, class = void>                                                                                   \
+  struct NAME : false_type {};                                                                                         \
+  template <class _Tp>                                                                                                 \
+  struct NAME<_Tp, __void_t<typename _Tp::PROPERTY> > : true_type {}
+// clang-format on
+
+_LIBCPP_CLASS_TRAITS_HAS_XXX(__has_pointer, pointer);
+_LIBCPP_CLASS_TRAITS_HAS_XXX(__has_element_type, element_type);
+
+template <class _Ptr, bool = __has_element_type<_Ptr>::value>
+struct __pointer_traits_element_type {};
+
+template <class _Ptr>
+struct __pointer_traits_element_type<_Ptr, true> {
+  typedef _LIBCPP_NODEBUG typename _Ptr::element_type type;
+};
+
+template <template <class, class...> class _Sp, class _Tp, class... _Args>
+struct __pointer_traits_element_type<_Sp<_Tp, _Args...>, true> {
+  typedef _LIBCPP_NODEBUG typename _Sp<_Tp, _Args...>::element_type type;
+};
+
+template <template <class, class...> class _Sp, class _Tp, class... _Args>
+struct __pointer_traits_element_type<_Sp<_Tp, _Args...>, false> {
+  typedef _LIBCPP_NODEBUG _Tp type;
+};
+
+template <class _Tp, class = void>
+struct __has_difference_type : false_type {};
+
+template <class _Tp>
+struct __has_difference_type<_Tp, __void_t<typename _Tp::difference_type> > : true_type {};
+
+template <class _Ptr, bool = __has_difference_type<_Ptr>::value>
+struct __pointer_traits_difference_type {
+  typedef _LIBCPP_NODEBUG ptrdiff_t type;
+};
+
+template <class _Ptr>
+struct __pointer_traits_difference_type<_Ptr, true> {
+  typedef _LIBCPP_NODEBUG typename _Ptr::difference_type type;
+};
+
+template <class _Tp, class _Up>
+struct __has_rebind {
+private:
+  template <class _Xp>
+  static false_type __test(...);
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  template <class _Xp>
+  static true_type __test(typename _Xp::template rebind<_Up>* = 0);
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+public:
+  static const bool value = decltype(__test<_Tp>(0))::value;
+};
+
+template <class _Tp, class _Up, bool = __has_rebind<_Tp, _Up>::value>
+struct __pointer_traits_rebind {
+#ifndef _LIBCPP_CXX03_LANG
+  typedef _LIBCPP_NODEBUG typename _Tp::template rebind<_Up> type;
+#else
+  typedef _LIBCPP_NODEBUG typename _Tp::template rebind<_Up>::other type;
+#endif
+};
+
+template <template <class, class...> class _Sp, class _Tp, class... _Args, class _Up>
+struct __pointer_traits_rebind<_Sp<_Tp, _Args...>, _Up, true> {
+#ifndef _LIBCPP_CXX03_LANG
+  typedef _LIBCPP_NODEBUG typename _Sp<_Tp, _Args...>::template rebind<_Up> type;
+#else
+  typedef _LIBCPP_NODEBUG typename _Sp<_Tp, _Args...>::template rebind<_Up>::other type;
+#endif
+};
+
+template <template <class, class...> class _Sp, class _Tp, class... _Args, class _Up>
+struct __pointer_traits_rebind<_Sp<_Tp, _Args...>, _Up, false> {
+  typedef _Sp<_Up, _Args...> type;
+};
+
+template <class _Ptr, class = void>
+struct __pointer_traits_impl {};
+
+template <class _Ptr>
+struct __pointer_traits_impl<_Ptr, __void_t<typename __pointer_traits_element_type<_Ptr>::type> > {
+  typedef _Ptr pointer;
+  typedef typename __pointer_traits_element_type<pointer>::type element_type;
+  typedef typename __pointer_traits_difference_type<pointer>::type difference_type;
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class _Up>
+  using rebind = typename __pointer_traits_rebind<pointer, _Up>::type;
+#else
+  template <class _Up>
+  struct rebind {
+    typedef typename __pointer_traits_rebind<pointer, _Up>::type other;
+  };
+#endif // _LIBCPP_CXX03_LANG
+
+private:
+  struct __nat {};
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static pointer
+  pointer_to(__conditional_t<is_void<element_type>::value, __nat, element_type>& __r) {
+    return pointer::pointer_to(__r);
+  }
+};
+
+template <class _Ptr>
+struct _LIBCPP_TEMPLATE_VIS pointer_traits : __pointer_traits_impl<_Ptr> {};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS pointer_traits<_Tp*> {
+  typedef _Tp* pointer;
+  typedef _Tp element_type;
+  typedef ptrdiff_t difference_type;
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class _Up>
+  using rebind = _Up*;
+#else
+  template <class _Up>
+  struct rebind {
+    typedef _Up* other;
+  };
+#endif
+
+private:
+  struct __nat {};
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static pointer
+  pointer_to(__conditional_t<is_void<element_type>::value, __nat, element_type>& __r) _NOEXCEPT {
+    return std::addressof(__r);
+  }
+};
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _From, class _To>
+using __rebind_pointer_t = typename pointer_traits<_From>::template rebind<_To>;
+#else
+template <class _From, class _To>
+using __rebind_pointer_t = typename pointer_traits<_From>::template rebind<_To>::other;
+#endif
+
+// to_address
+
+template <class _Pointer, class = void>
+struct __to_address_helper;
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp* __to_address(_Tp* __p) _NOEXCEPT {
+  static_assert(!is_function<_Tp>::value, "_Tp is a function type");
+  return __p;
+}
+
+template <class _Pointer, class = void>
+struct _HasToAddress : false_type {};
+
+template <class _Pointer>
+struct _HasToAddress<_Pointer, decltype((void)pointer_traits<_Pointer>::to_address(std::declval<const _Pointer&>())) >
+    : true_type {};
+
+template <class _Pointer, class = void>
+struct _HasArrow : false_type {};
+
+template <class _Pointer>
+struct _HasArrow<_Pointer, decltype((void)std::declval<const _Pointer&>().operator->()) > : true_type {};
+
+template <class _Pointer>
+struct _IsFancyPointer {
+  static const bool value = _HasArrow<_Pointer>::value || _HasToAddress<_Pointer>::value;
+};
+
+// enable_if is needed here to avoid instantiating checks for fancy pointers on raw pointers
+template <class _Pointer, __enable_if_t< _And<is_class<_Pointer>, _IsFancyPointer<_Pointer> >::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI
+_LIBCPP_CONSTEXPR __decay_t<decltype(__to_address_helper<_Pointer>::__call(std::declval<const _Pointer&>()))>
+__to_address(const _Pointer& __p) _NOEXCEPT {
+  return __to_address_helper<_Pointer>::__call(__p);
+}
+
+template <class _Pointer, class>
+struct __to_address_helper {
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR static decltype(std::__to_address(std::declval<const _Pointer&>().operator->()))
+  __call(const _Pointer& __p) _NOEXCEPT {
+    return std::__to_address(__p.operator->());
+  }
+};
+
+template <class _Pointer>
+struct __to_address_helper<_Pointer,
+                           decltype((void)pointer_traits<_Pointer>::to_address(std::declval<const _Pointer&>()))> {
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR static decltype(pointer_traits<_Pointer>::to_address(std::declval<const _Pointer&>()))
+  __call(const _Pointer& __p) _NOEXCEPT {
+    return pointer_traits<_Pointer>::to_address(__p);
+  }
+};
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI constexpr auto to_address(_Tp* __p) noexcept {
+  return std::__to_address(__p);
+}
+
+template <class _Pointer>
+inline _LIBCPP_HIDE_FROM_ABI constexpr auto
+to_address(const _Pointer& __p) noexcept -> decltype(std::__to_address(__p)) {
+  return std::__to_address(__p);
+}
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Tp>
+struct __pointer_of {};
+
+template <class _Tp>
+  requires(__has_pointer<_Tp>::value)
+struct __pointer_of<_Tp> {
+  using type = typename _Tp::pointer;
+};
+
+template <class _Tp>
+  requires(!__has_pointer<_Tp>::value && __has_element_type<_Tp>::value)
+struct __pointer_of<_Tp> {
+  using type = typename _Tp::element_type*;
+};
+
+template <class _Tp>
+  requires(!__has_pointer<_Tp>::value && !__has_element_type<_Tp>::value &&
+           __has_element_type<pointer_traits<_Tp>>::value)
+struct __pointer_of<_Tp> {
+  using type = typename pointer_traits<_Tp>::element_type*;
+};
+
+template <typename _Tp>
+using __pointer_of_t = typename __pointer_of<_Tp>::type;
+
+template <class _Tp, class _Up>
+struct __pointer_of_or {
+  using type _LIBCPP_NODEBUG = _Up;
+};
+
+template <class _Tp, class _Up>
+  requires requires { typename __pointer_of_t<_Tp>; }
+struct __pointer_of_or<_Tp, _Up> {
+  using type _LIBCPP_NODEBUG = __pointer_of_t<_Tp>;
+};
+
+template <typename _Tp, typename _Up>
+using __pointer_of_or_t = typename __pointer_of_or<_Tp, _Up>::type;
+
+template <class _Smart>
+concept __resettable_smart_pointer = requires(_Smart __s) { __s.reset(); };
+
+template <class _Smart, class _Pointer, class... _Args>
+concept __resettable_smart_pointer_with_args = requires(_Smart __s, _Pointer __p, _Args... __args) {
+  __s.reset(static_cast<__pointer_of_or_t<_Smart, _Pointer>>(__p), std::forward<_Args>(__args)...);
+};
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_POINTER_TRAITS_H
diff --git a/libcxx/include/__cxx03/__memory/ranges_construct_at.h b/libcxx/include/__cxx03/__memory/ranges_construct_at.h
new file mode 100644
index 000000000000..f731e75e7bdc
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/ranges_construct_at.h
@@ -0,0 +1,124 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_RANGES_CONSTRUCT_AT_H
+#define _LIBCPP___MEMORY_RANGES_CONSTRUCT_AT_H
+
+#include <__concepts/destructible.h>
+#include <__config>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/concepts.h>
+#include <__memory/construct_at.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <new>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+namespace ranges {
+
+// construct_at
+
+namespace __construct_at {
+
+struct __fn {
+  template <class _Tp, class... _Args, class = decltype(::new(std::declval<void*>()) _Tp(std::declval<_Args>()...))>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp* operator()(_Tp* __location, _Args&&... __args) const {
+    return std::construct_at(__location, std::forward<_Args>(__args)...);
+  }
+};
+
+} // namespace __construct_at
+
+inline namespace __cpo {
+inline constexpr auto construct_at = __construct_at::__fn{};
+} // namespace __cpo
+
+// destroy_at
+
+namespace __destroy_at {
+
+struct __fn {
+  template <destructible _Tp>
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Tp* __location) const noexcept {
+    std::destroy_at(__location);
+  }
+};
+
+} // namespace __destroy_at
+
+inline namespace __cpo {
+inline constexpr auto destroy_at = __destroy_at::__fn{};
+} // namespace __cpo
+
+// destroy
+
+namespace __destroy {
+
+struct __fn {
+  template <__nothrow_input_iterator _InputIterator, __nothrow_sentinel_for<_InputIterator> _Sentinel>
+    requires destructible<iter_value_t<_InputIterator>>
+  _LIBCPP_HIDE_FROM_ABI constexpr _InputIterator operator()(_InputIterator __first, _Sentinel __last) const noexcept {
+    return std::__destroy(std::move(__first), std::move(__last));
+  }
+
+  template <__nothrow_input_range _InputRange>
+    requires destructible<range_value_t<_InputRange>>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_InputRange> operator()(_InputRange&& __range) const noexcept {
+    return (*this)(ranges::begin(__range), ranges::end(__range));
+  }
+};
+
+} // namespace __destroy
+
+inline namespace __cpo {
+inline constexpr auto destroy = __destroy::__fn{};
+} // namespace __cpo
+
+// destroy_n
+
+namespace __destroy_n {
+
+struct __fn {
+  template <__nothrow_input_iterator _InputIterator>
+    requires destructible<iter_value_t<_InputIterator>>
+  _LIBCPP_HIDE_FROM_ABI constexpr _InputIterator
+  operator()(_InputIterator __first, iter_difference_t<_InputIterator> __n) const noexcept {
+    return std::destroy_n(std::move(__first), __n);
+  }
+};
+
+} // namespace __destroy_n
+
+inline namespace __cpo {
+inline constexpr auto destroy_n = __destroy_n::__fn{};
+} // namespace __cpo
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_RANGES_CONSTRUCT_AT_H
diff --git a/libcxx/include/__cxx03/__memory/ranges_uninitialized_algorithms.h b/libcxx/include/__cxx03/__memory/ranges_uninitialized_algorithms.h
new file mode 100644
index 000000000000..90090055bbbb
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/ranges_uninitialized_algorithms.h
@@ -0,0 +1,325 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_RANGES_UNINITIALIZED_ALGORITHMS_H
+#define _LIBCPP___MEMORY_RANGES_UNINITIALIZED_ALGORITHMS_H
+
+#include <__algorithm/in_out_result.h>
+#include <__concepts/constructible.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/readable_traits.h>
+#include <__memory/concepts.h>
+#include <__memory/uninitialized_algorithms.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/move.h>
+#include <new>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+// uninitialized_default_construct
+
+namespace __uninitialized_default_construct {
+
+struct __fn {
+  template <__nothrow_forward_iterator _ForwardIterator, __nothrow_sentinel_for<_ForwardIterator> _Sentinel>
+    requires default_initializable<iter_value_t<_ForwardIterator>>
+  _LIBCPP_HIDE_FROM_ABI _ForwardIterator operator()(_ForwardIterator __first, _Sentinel __last) const {
+    using _ValueType = remove_reference_t<iter_reference_t<_ForwardIterator>>;
+    return std::__uninitialized_default_construct<_ValueType>(std::move(__first), std::move(__last));
+  }
+
+  template <__nothrow_forward_range _ForwardRange>
+    requires default_initializable<range_value_t<_ForwardRange>>
+  _LIBCPP_HIDE_FROM_ABI borrowed_iterator_t<_ForwardRange> operator()(_ForwardRange&& __range) const {
+    return (*this)(ranges::begin(__range), ranges::end(__range));
+  }
+};
+
+} // namespace __uninitialized_default_construct
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_default_construct = __uninitialized_default_construct::__fn{};
+} // namespace __cpo
+
+// uninitialized_default_construct_n
+
+namespace __uninitialized_default_construct_n {
+
+struct __fn {
+  template <__nothrow_forward_iterator _ForwardIterator>
+    requires default_initializable<iter_value_t<_ForwardIterator>>
+  _LIBCPP_HIDE_FROM_ABI _ForwardIterator
+  operator()(_ForwardIterator __first, iter_difference_t<_ForwardIterator> __n) const {
+    using _ValueType = remove_reference_t<iter_reference_t<_ForwardIterator>>;
+    return std::__uninitialized_default_construct_n<_ValueType>(std::move(__first), __n);
+  }
+};
+
+} // namespace __uninitialized_default_construct_n
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_default_construct_n = __uninitialized_default_construct_n::__fn{};
+} // namespace __cpo
+
+// uninitialized_value_construct
+
+namespace __uninitialized_value_construct {
+
+struct __fn {
+  template <__nothrow_forward_iterator _ForwardIterator, __nothrow_sentinel_for<_ForwardIterator> _Sentinel>
+    requires default_initializable<iter_value_t<_ForwardIterator>>
+  _LIBCPP_HIDE_FROM_ABI _ForwardIterator operator()(_ForwardIterator __first, _Sentinel __last) const {
+    using _ValueType = remove_reference_t<iter_reference_t<_ForwardIterator>>;
+    return std::__uninitialized_value_construct<_ValueType>(std::move(__first), std::move(__last));
+  }
+
+  template <__nothrow_forward_range _ForwardRange>
+    requires default_initializable<range_value_t<_ForwardRange>>
+  _LIBCPP_HIDE_FROM_ABI borrowed_iterator_t<_ForwardRange> operator()(_ForwardRange&& __range) const {
+    return (*this)(ranges::begin(__range), ranges::end(__range));
+  }
+};
+
+} // namespace __uninitialized_value_construct
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_value_construct = __uninitialized_value_construct::__fn{};
+} // namespace __cpo
+
+// uninitialized_value_construct_n
+
+namespace __uninitialized_value_construct_n {
+
+struct __fn {
+  template <__nothrow_forward_iterator _ForwardIterator>
+    requires default_initializable<iter_value_t<_ForwardIterator>>
+  _LIBCPP_HIDE_FROM_ABI _ForwardIterator
+  operator()(_ForwardIterator __first, iter_difference_t<_ForwardIterator> __n) const {
+    using _ValueType = remove_reference_t<iter_reference_t<_ForwardIterator>>;
+    return std::__uninitialized_value_construct_n<_ValueType>(std::move(__first), __n);
+  }
+};
+
+} // namespace __uninitialized_value_construct_n
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_value_construct_n = __uninitialized_value_construct_n::__fn{};
+} // namespace __cpo
+
+// uninitialized_fill
+
+namespace __uninitialized_fill {
+
+struct __fn {
+  template <__nothrow_forward_iterator _ForwardIterator, __nothrow_sentinel_for<_ForwardIterator> _Sentinel, class _Tp>
+    requires constructible_from<iter_value_t<_ForwardIterator>, const _Tp&>
+  _LIBCPP_HIDE_FROM_ABI _ForwardIterator operator()(_ForwardIterator __first, _Sentinel __last, const _Tp& __x) const {
+    using _ValueType = remove_reference_t<iter_reference_t<_ForwardIterator>>;
+    return std::__uninitialized_fill<_ValueType>(std::move(__first), std::move(__last), __x);
+  }
+
+  template <__nothrow_forward_range _ForwardRange, class _Tp>
+    requires constructible_from<range_value_t<_ForwardRange>, const _Tp&>
+  _LIBCPP_HIDE_FROM_ABI borrowed_iterator_t<_ForwardRange> operator()(_ForwardRange&& __range, const _Tp& __x) const {
+    return (*this)(ranges::begin(__range), ranges::end(__range), __x);
+  }
+};
+
+} // namespace __uninitialized_fill
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_fill = __uninitialized_fill::__fn{};
+} // namespace __cpo
+
+// uninitialized_fill_n
+
+namespace __uninitialized_fill_n {
+
+struct __fn {
+  template <__nothrow_forward_iterator _ForwardIterator, class _Tp>
+    requires constructible_from<iter_value_t<_ForwardIterator>, const _Tp&>
+  _LIBCPP_HIDE_FROM_ABI _ForwardIterator
+  operator()(_ForwardIterator __first, iter_difference_t<_ForwardIterator> __n, const _Tp& __x) const {
+    using _ValueType = remove_reference_t<iter_reference_t<_ForwardIterator>>;
+    return std::__uninitialized_fill_n<_ValueType>(std::move(__first), __n, __x);
+  }
+};
+
+} // namespace __uninitialized_fill_n
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_fill_n = __uninitialized_fill_n::__fn{};
+} // namespace __cpo
+
+// uninitialized_copy
+
+template <class _InputIterator, class _OutputIterator>
+using uninitialized_copy_result = in_out_result<_InputIterator, _OutputIterator>;
+
+namespace __uninitialized_copy {
+
+struct __fn {
+  template <input_iterator _InputIterator,
+            sentinel_for<_InputIterator> _Sentinel1,
+            __nothrow_forward_iterator _OutputIterator,
+            __nothrow_sentinel_for<_OutputIterator> _Sentinel2>
+    requires constructible_from<iter_value_t<_OutputIterator>, iter_reference_t<_InputIterator>>
+  _LIBCPP_HIDE_FROM_ABI uninitialized_copy_result<_InputIterator, _OutputIterator>
+  operator()(_InputIterator __ifirst, _Sentinel1 __ilast, _OutputIterator __ofirst, _Sentinel2 __olast) const {
+    using _ValueType = remove_reference_t<iter_reference_t<_OutputIterator>>;
+
+    auto __stop_copying = [&__olast](auto&& __out_iter) -> bool { return __out_iter == __olast; };
+    auto __result       = std::__uninitialized_copy<_ValueType>(
+        std::move(__ifirst), std::move(__ilast), std::move(__ofirst), __stop_copying);
+    return {std::move(__result.first), std::move(__result.second)};
+  }
+
+  template <input_range _InputRange, __nothrow_forward_range _OutputRange>
+    requires constructible_from<range_value_t<_OutputRange>, range_reference_t<_InputRange>>
+  _LIBCPP_HIDE_FROM_ABI uninitialized_copy_result<borrowed_iterator_t<_InputRange>, borrowed_iterator_t<_OutputRange>>
+  operator()(_InputRange&& __in_range, _OutputRange&& __out_range) const {
+    return (*this)(
+        ranges::begin(__in_range), ranges::end(__in_range), ranges::begin(__out_range), ranges::end(__out_range));
+  }
+};
+
+} // namespace __uninitialized_copy
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_copy = __uninitialized_copy::__fn{};
+} // namespace __cpo
+
+// uninitialized_copy_n
+
+template <class _InputIterator, class _OutputIterator>
+using uninitialized_copy_n_result = in_out_result<_InputIterator, _OutputIterator>;
+
+namespace __uninitialized_copy_n {
+
+struct __fn {
+  template <input_iterator _InputIterator,
+            __nothrow_forward_iterator _OutputIterator,
+            __nothrow_sentinel_for<_OutputIterator> _Sentinel>
+    requires constructible_from<iter_value_t<_OutputIterator>, iter_reference_t<_InputIterator>>
+  _LIBCPP_HIDE_FROM_ABI uninitialized_copy_n_result<_InputIterator, _OutputIterator>
+  operator()(_InputIterator __ifirst,
+             iter_difference_t<_InputIterator> __n,
+             _OutputIterator __ofirst,
+             _Sentinel __olast) const {
+    using _ValueType    = remove_reference_t<iter_reference_t<_OutputIterator>>;
+    auto __stop_copying = [&__olast](auto&& __out_iter) -> bool { return __out_iter == __olast; };
+    auto __result =
+        std::__uninitialized_copy_n<_ValueType>(std::move(__ifirst), __n, std::move(__ofirst), __stop_copying);
+    return {std::move(__result.first), std::move(__result.second)};
+  }
+};
+
+} // namespace __uninitialized_copy_n
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_copy_n = __uninitialized_copy_n::__fn{};
+} // namespace __cpo
+
+// uninitialized_move
+
+template <class _InputIterator, class _OutputIterator>
+using uninitialized_move_result = in_out_result<_InputIterator, _OutputIterator>;
+
+namespace __uninitialized_move {
+
+struct __fn {
+  template <input_iterator _InputIterator,
+            sentinel_for<_InputIterator> _Sentinel1,
+            __nothrow_forward_iterator _OutputIterator,
+            __nothrow_sentinel_for<_OutputIterator> _Sentinel2>
+    requires constructible_from<iter_value_t<_OutputIterator>, iter_rvalue_reference_t<_InputIterator>>
+  _LIBCPP_HIDE_FROM_ABI uninitialized_move_result<_InputIterator, _OutputIterator>
+  operator()(_InputIterator __ifirst, _Sentinel1 __ilast, _OutputIterator __ofirst, _Sentinel2 __olast) const {
+    using _ValueType   = remove_reference_t<iter_reference_t<_OutputIterator>>;
+    auto __iter_move   = [](auto&& __iter) -> decltype(auto) { return ranges::iter_move(__iter); };
+    auto __stop_moving = [&__olast](auto&& __out_iter) -> bool { return __out_iter == __olast; };
+    auto __result      = std::__uninitialized_move<_ValueType>(
+        std::move(__ifirst), std::move(__ilast), std::move(__ofirst), __stop_moving, __iter_move);
+    return {std::move(__result.first), std::move(__result.second)};
+  }
+
+  template <input_range _InputRange, __nothrow_forward_range _OutputRange>
+    requires constructible_from<range_value_t<_OutputRange>, range_rvalue_reference_t<_InputRange>>
+  _LIBCPP_HIDE_FROM_ABI uninitialized_move_result<borrowed_iterator_t<_InputRange>, borrowed_iterator_t<_OutputRange>>
+  operator()(_InputRange&& __in_range, _OutputRange&& __out_range) const {
+    return (*this)(
+        ranges::begin(__in_range), ranges::end(__in_range), ranges::begin(__out_range), ranges::end(__out_range));
+  }
+};
+
+} // namespace __uninitialized_move
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_move = __uninitialized_move::__fn{};
+} // namespace __cpo
+
+// uninitialized_move_n
+
+template <class _InputIterator, class _OutputIterator>
+using uninitialized_move_n_result = in_out_result<_InputIterator, _OutputIterator>;
+
+namespace __uninitialized_move_n {
+
+struct __fn {
+  template <input_iterator _InputIterator,
+            __nothrow_forward_iterator _OutputIterator,
+            __nothrow_sentinel_for<_OutputIterator> _Sentinel>
+    requires constructible_from<iter_value_t<_OutputIterator>, iter_rvalue_reference_t<_InputIterator>>
+  _LIBCPP_HIDE_FROM_ABI uninitialized_move_n_result<_InputIterator, _OutputIterator>
+  operator()(_InputIterator __ifirst,
+             iter_difference_t<_InputIterator> __n,
+             _OutputIterator __ofirst,
+             _Sentinel __olast) const {
+    using _ValueType   = remove_reference_t<iter_reference_t<_OutputIterator>>;
+    auto __iter_move   = [](auto&& __iter) -> decltype(auto) { return ranges::iter_move(__iter); };
+    auto __stop_moving = [&__olast](auto&& __out_iter) -> bool { return __out_iter == __olast; };
+    auto __result      = std::__uninitialized_move_n<_ValueType>(
+        std::move(__ifirst), __n, std::move(__ofirst), __stop_moving, __iter_move);
+    return {std::move(__result.first), std::move(__result.second)};
+  }
+};
+
+} // namespace __uninitialized_move_n
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_move_n = __uninitialized_move_n::__fn{};
+} // namespace __cpo
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_RANGES_UNINITIALIZED_ALGORITHMS_H
diff --git a/libcxx/include/__cxx03/__memory/raw_storage_iterator.h b/libcxx/include/__cxx03/__memory/raw_storage_iterator.h
new file mode 100644
index 000000000000..774878aa1c5e
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/raw_storage_iterator.h
@@ -0,0 +1,87 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_RAW_STORAGE_ITERATOR_H
+#define _LIBCPP___MEMORY_RAW_STORAGE_ITERATOR_H
+
+#include <__config>
+#include <__iterator/iterator.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__utility/move.h>
+#include <cstddef>
+#include <new>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_RAW_STORAGE_ITERATOR)
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _OutputIterator, class _Tp>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 raw_storage_iterator
+#  if _LIBCPP_STD_VER <= 14 || !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
+    : public iterator<output_iterator_tag, void, void, void, void>
+#  endif
+{
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+private:
+  _OutputIterator __x_;
+
+public:
+  typedef output_iterator_tag iterator_category;
+  typedef void value_type;
+#  if _LIBCPP_STD_VER >= 20
+  typedef ptrdiff_t difference_type;
+#  else
+  typedef void difference_type;
+#  endif
+  typedef void pointer;
+  typedef void reference;
+
+  _LIBCPP_HIDE_FROM_ABI explicit raw_storage_iterator(_OutputIterator __x) : __x_(__x) {}
+  _LIBCPP_HIDE_FROM_ABI raw_storage_iterator& operator*() { return *this; }
+  _LIBCPP_HIDE_FROM_ABI raw_storage_iterator& operator=(const _Tp& __element) {
+    ::new ((void*)std::addressof(*__x_)) _Tp(__element);
+    return *this;
+  }
+#  if _LIBCPP_STD_VER >= 14
+  _LIBCPP_HIDE_FROM_ABI raw_storage_iterator& operator=(_Tp&& __element) {
+    ::new ((void*)std::addressof(*__x_)) _Tp(std::move(__element));
+    return *this;
+  }
+#  endif
+  _LIBCPP_HIDE_FROM_ABI raw_storage_iterator& operator++() {
+    ++__x_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI raw_storage_iterator operator++(int) {
+    raw_storage_iterator __t(*this);
+    ++__x_;
+    return __t;
+  }
+#  if _LIBCPP_STD_VER >= 14
+  _LIBCPP_HIDE_FROM_ABI _OutputIterator base() const { return __x_; }
+#  endif
+};
+
+#endif // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_RAW_STORAGE_ITERATOR)
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_RAW_STORAGE_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__memory/shared_ptr.h b/libcxx/include/__cxx03/__memory/shared_ptr.h
new file mode 100644
index 000000000000..d487e4fbe3a9
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/shared_ptr.h
@@ -0,0 +1,1694 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_SHARED_PTR_H
+#define _LIBCPP___MEMORY_SHARED_PTR_H
+
+#include <__compare/compare_three_way.h>
+#include <__compare/ordering.h>
+#include <__config>
+#include <__exception/exception.h>
+#include <__functional/binary_function.h>
+#include <__functional/operations.h>
+#include <__functional/reference_wrapper.h>
+#include <__fwd/ostream.h>
+#include <__iterator/access.h>
+#include <__memory/addressof.h>
+#include <__memory/allocation_guard.h>
+#include <__memory/allocator.h>
+#include <__memory/allocator_destructor.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/auto_ptr.h>
+#include <__memory/compressed_pair.h>
+#include <__memory/construct_at.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/uninitialized_algorithms.h>
+#include <__memory/unique_ptr.h>
+#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/disjunction.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_bounded_array.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/is_unbounded_array.h>
+#include <__type_traits/nat.h>
+#include <__type_traits/negation.h>
+#include <__type_traits/remove_extent.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+#include <__verbose_abort>
+#include <cstddef>
+#include <new>
+#include <typeinfo>
+#if !defined(_LIBCPP_HAS_NO_ATOMIC_HEADER)
+#  include <__atomic/memory_order.h>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
+// should be sufficient for thread safety.
+// See https://llvm.org/PR22803
+#if defined(__clang__) && __has_builtin(__atomic_add_fetch) && defined(__ATOMIC_RELAXED) && defined(__ATOMIC_ACQ_REL)
+#  define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT
+#elif defined(_LIBCPP_COMPILER_GCC)
+#  define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT
+#endif
+
+template <class _ValueType>
+inline _LIBCPP_HIDE_FROM_ABI _ValueType __libcpp_relaxed_load(_ValueType const* __value) {
+#if !defined(_LIBCPP_HAS_NO_THREADS) && defined(__ATOMIC_RELAXED) &&                                                   \
+    (__has_builtin(__atomic_load_n) || defined(_LIBCPP_COMPILER_GCC))
+  return __atomic_load_n(__value, __ATOMIC_RELAXED);
+#else
+  return *__value;
+#endif
+}
+
+template <class _ValueType>
+inline _LIBCPP_HIDE_FROM_ABI _ValueType __libcpp_acquire_load(_ValueType const* __value) {
+#if !defined(_LIBCPP_HAS_NO_THREADS) && defined(__ATOMIC_ACQUIRE) &&                                                   \
+    (__has_builtin(__atomic_load_n) || defined(_LIBCPP_COMPILER_GCC))
+  return __atomic_load_n(__value, __ATOMIC_ACQUIRE);
+#else
+  return *__value;
+#endif
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _Tp __libcpp_atomic_refcount_increment(_Tp& __t) _NOEXCEPT {
+#if defined(_LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT) && !defined(_LIBCPP_HAS_NO_THREADS)
+  return __atomic_add_fetch(&__t, 1, __ATOMIC_RELAXED);
+#else
+  return __t += 1;
+#endif
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _Tp __libcpp_atomic_refcount_decrement(_Tp& __t) _NOEXCEPT {
+#if defined(_LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT) && !defined(_LIBCPP_HAS_NO_THREADS)
+  return __atomic_add_fetch(&__t, -1, __ATOMIC_ACQ_REL);
+#else
+  return __t -= 1;
+#endif
+}
+
+class _LIBCPP_EXPORTED_FROM_ABI bad_weak_ptr : public std::exception {
+public:
+  _LIBCPP_HIDE_FROM_ABI bad_weak_ptr() _NOEXCEPT                               = default;
+  _LIBCPP_HIDE_FROM_ABI bad_weak_ptr(const bad_weak_ptr&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI bad_weak_ptr& operator=(const bad_weak_ptr&) _NOEXCEPT = default;
+  ~bad_weak_ptr() _NOEXCEPT override;
+  const char* what() const _NOEXCEPT override;
+};
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_bad_weak_ptr() {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw bad_weak_ptr();
+#else
+  _LIBCPP_VERBOSE_ABORT("bad_weak_ptr was thrown in -fno-exceptions mode");
+#endif
+}
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS weak_ptr;
+
+class _LIBCPP_EXPORTED_FROM_ABI __shared_count {
+  __shared_count(const __shared_count&);
+  __shared_count& operator=(const __shared_count&);
+
+protected:
+  long __shared_owners_;
+  virtual ~__shared_count();
+
+private:
+  virtual void __on_zero_shared() _NOEXCEPT = 0;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __shared_count(long __refs = 0) _NOEXCEPT : __shared_owners_(__refs) {}
+
+#if defined(_LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS)
+  void __add_shared() noexcept;
+  bool __release_shared() noexcept;
+#else
+  _LIBCPP_HIDE_FROM_ABI void __add_shared() _NOEXCEPT { __libcpp_atomic_refcount_increment(__shared_owners_); }
+  _LIBCPP_HIDE_FROM_ABI bool __release_shared() _NOEXCEPT {
+    if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1) {
+      __on_zero_shared();
+      return true;
+    }
+    return false;
+  }
+#endif
+  _LIBCPP_HIDE_FROM_ABI long use_count() const _NOEXCEPT { return __libcpp_relaxed_load(&__shared_owners_) + 1; }
+};
+
+class _LIBCPP_EXPORTED_FROM_ABI __shared_weak_count : private __shared_count {
+  long __shared_weak_owners_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __shared_weak_count(long __refs = 0) _NOEXCEPT
+      : __shared_count(__refs),
+        __shared_weak_owners_(__refs) {}
+
+protected:
+  ~__shared_weak_count() override;
+
+public:
+#if defined(_LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS)
+  void __add_shared() noexcept;
+  void __add_weak() noexcept;
+  void __release_shared() noexcept;
+#else
+  _LIBCPP_HIDE_FROM_ABI void __add_shared() _NOEXCEPT { __shared_count::__add_shared(); }
+  _LIBCPP_HIDE_FROM_ABI void __add_weak() _NOEXCEPT { __libcpp_atomic_refcount_increment(__shared_weak_owners_); }
+  _LIBCPP_HIDE_FROM_ABI void __release_shared() _NOEXCEPT {
+    if (__shared_count::__release_shared())
+      __release_weak();
+  }
+#endif
+  void __release_weak() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI long use_count() const _NOEXCEPT { return __shared_count::use_count(); }
+  __shared_weak_count* lock() _NOEXCEPT;
+
+  virtual const void* __get_deleter(const type_info&) const _NOEXCEPT;
+
+private:
+  virtual void __on_zero_shared_weak() _NOEXCEPT = 0;
+};
+
+template <class _Tp, class _Dp, class _Alloc>
+class __shared_ptr_pointer : public __shared_weak_count {
+  __compressed_pair<__compressed_pair<_Tp, _Dp>, _Alloc> __data_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __shared_ptr_pointer(_Tp __p, _Dp __d, _Alloc __a)
+      : __data_(__compressed_pair<_Tp, _Dp>(__p, std::move(__d)), std::move(__a)) {}
+
+#ifndef _LIBCPP_HAS_NO_RTTI
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const void* __get_deleter(const type_info&) const _NOEXCEPT override;
+#endif
+
+private:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared() _NOEXCEPT override;
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared_weak() _NOEXCEPT override;
+};
+
+#ifndef _LIBCPP_HAS_NO_RTTI
+
+template <class _Tp, class _Dp, class _Alloc>
+const void* __shared_ptr_pointer<_Tp, _Dp, _Alloc>::__get_deleter(const type_info& __t) const _NOEXCEPT {
+  return __t == typeid(_Dp) ? std::addressof(__data_.first().second()) : nullptr;
+}
+
+#endif // _LIBCPP_HAS_NO_RTTI
+
+template <class _Tp, class _Dp, class _Alloc>
+void __shared_ptr_pointer<_Tp, _Dp, _Alloc>::__on_zero_shared() _NOEXCEPT {
+  __data_.first().second()(__data_.first().first());
+  __data_.first().second().~_Dp();
+}
+
+template <class _Tp, class _Dp, class _Alloc>
+void __shared_ptr_pointer<_Tp, _Dp, _Alloc>::__on_zero_shared_weak() _NOEXCEPT {
+  typedef typename __allocator_traits_rebind<_Alloc, __shared_ptr_pointer>::type _Al;
+  typedef allocator_traits<_Al> _ATraits;
+  typedef pointer_traits<typename _ATraits::pointer> _PTraits;
+
+  _Al __a(__data_.second());
+  __data_.second().~_Alloc();
+  __a.deallocate(_PTraits::pointer_to(*this), 1);
+}
+
+// This tag is used to instantiate an allocator type. The various shared_ptr control blocks
+// detect that the allocator has been instantiated for this type and perform alternative
+// initialization/destruction based on that.
+struct __for_overwrite_tag {};
+
+template <class _Tp, class _Alloc>
+struct __shared_ptr_emplace : __shared_weak_count {
+  template <class... _Args,
+            class _Allocator                                                                         = _Alloc,
+            __enable_if_t<is_same<typename _Allocator::value_type, __for_overwrite_tag>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit __shared_ptr_emplace(_Alloc __a, _Args&&...) : __storage_(std::move(__a)) {
+    static_assert(
+        sizeof...(_Args) == 0, "No argument should be provided to the control block when using _for_overwrite");
+    ::new ((void*)__get_elem()) _Tp;
+  }
+
+  template <class... _Args,
+            class _Allocator                                                                          = _Alloc,
+            __enable_if_t<!is_same<typename _Allocator::value_type, __for_overwrite_tag>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit __shared_ptr_emplace(_Alloc __a, _Args&&... __args) : __storage_(std::move(__a)) {
+    using _TpAlloc = typename __allocator_traits_rebind<_Alloc, __remove_cv_t<_Tp> >::type;
+    _TpAlloc __tmp(*__get_alloc());
+    allocator_traits<_TpAlloc>::construct(__tmp, __get_elem(), std::forward<_Args>(__args)...);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Alloc* __get_alloc() _NOEXCEPT { return __storage_.__get_alloc(); }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp* __get_elem() _NOEXCEPT { return __storage_.__get_elem(); }
+
+private:
+  template <class _Allocator                                                                         = _Alloc,
+            __enable_if_t<is_same<typename _Allocator::value_type, __for_overwrite_tag>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void __on_zero_shared_impl() _NOEXCEPT {
+    __get_elem()->~_Tp();
+  }
+
+  template <class _Allocator                                                                          = _Alloc,
+            __enable_if_t<!is_same<typename _Allocator::value_type, __for_overwrite_tag>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void __on_zero_shared_impl() _NOEXCEPT {
+    using _TpAlloc = typename __allocator_traits_rebind<_Allocator, __remove_cv_t<_Tp> >::type;
+    _TpAlloc __tmp(*__get_alloc());
+    allocator_traits<_TpAlloc>::destroy(__tmp, __get_elem());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared() _NOEXCEPT override { __on_zero_shared_impl(); }
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared_weak() _NOEXCEPT override {
+    using _ControlBlockAlloc   = typename __allocator_traits_rebind<_Alloc, __shared_ptr_emplace>::type;
+    using _ControlBlockPointer = typename allocator_traits<_ControlBlockAlloc>::pointer;
+    _ControlBlockAlloc __tmp(*__get_alloc());
+    __storage_.~_Storage();
+    allocator_traits<_ControlBlockAlloc>::deallocate(__tmp, pointer_traits<_ControlBlockPointer>::pointer_to(*this), 1);
+  }
+
+  // This class implements the control block for non-array shared pointers created
+  // through `std::allocate_shared` and `std::make_shared`.
+  //
+  // In previous versions of the library, we used a compressed pair to store
+  // both the _Alloc and the _Tp. This implies using EBO, which is incompatible
+  // with Allocator construction for _Tp. To allow implementing P0674 in C++20,
+  // we now use a properly aligned char buffer while making sure that we maintain
+  // the same layout that we had when we used a compressed pair.
+  using _CompressedPair = __compressed_pair<_Alloc, _Tp>;
+  struct _ALIGNAS_TYPE(_CompressedPair) _Storage {
+    char __blob_[sizeof(_CompressedPair)];
+
+    _LIBCPP_HIDE_FROM_ABI explicit _Storage(_Alloc&& __a) { ::new ((void*)__get_alloc()) _Alloc(std::move(__a)); }
+    _LIBCPP_HIDE_FROM_ABI ~_Storage() { __get_alloc()->~_Alloc(); }
+    _LIBCPP_HIDE_FROM_ABI _Alloc* __get_alloc() _NOEXCEPT {
+      _CompressedPair* __as_pair                = reinterpret_cast<_CompressedPair*>(__blob_);
+      typename _CompressedPair::_Base1* __first = _CompressedPair::__get_first_base(__as_pair);
+      _Alloc* __alloc                           = reinterpret_cast<_Alloc*>(__first);
+      return __alloc;
+    }
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI _Tp* __get_elem() _NOEXCEPT {
+      _CompressedPair* __as_pair                 = reinterpret_cast<_CompressedPair*>(__blob_);
+      typename _CompressedPair::_Base2* __second = _CompressedPair::__get_second_base(__as_pair);
+      _Tp* __elem                                = reinterpret_cast<_Tp*>(__second);
+      return __elem;
+    }
+  };
+
+  static_assert(_LIBCPP_ALIGNOF(_Storage) == _LIBCPP_ALIGNOF(_CompressedPair), "");
+  static_assert(sizeof(_Storage) == sizeof(_CompressedPair), "");
+  _Storage __storage_;
+};
+
+struct __shared_ptr_dummy_rebind_allocator_type;
+template <>
+class _LIBCPP_TEMPLATE_VIS allocator<__shared_ptr_dummy_rebind_allocator_type> {
+public:
+  template <class _Other>
+  struct rebind {
+    typedef allocator<_Other> other;
+  };
+};
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS enable_shared_from_this;
+
+// http://eel.is/c++draft/util.sharedptr#util.smartptr.shared.general-6
+// A pointer type Y* is said to be compatible with a pointer type T*
+// when either Y* is convertible to T* or Y is U[N] and T is cv U[].
+#if _LIBCPP_STD_VER >= 17
+template <class _Yp, class _Tp>
+struct __bounded_convertible_to_unbounded : false_type {};
+
+template <class _Up, std::size_t _Np, class _Tp>
+struct __bounded_convertible_to_unbounded<_Up[_Np], _Tp> : is_same<__remove_cv_t<_Tp>, _Up[]> {};
+
+template <class _Yp, class _Tp>
+struct __compatible_with : _Or< is_convertible<_Yp*, _Tp*>, __bounded_convertible_to_unbounded<_Yp, _Tp> > {};
+#else
+template <class _Yp, class _Tp>
+struct __compatible_with : is_convertible<_Yp*, _Tp*> {};
+#endif // _LIBCPP_STD_VER >= 17
+
+// Constructors that take raw pointers have a different set of "compatible" constraints
+// http://eel.is/c++draft/util.sharedptr#util.smartptr.shared.const-9.1
+// - If T is an array type, then either T is U[N] and Y(*)[N] is convertible to T*,
+//   or T is U[] and Y(*)[] is convertible to T*.
+// - If T is not an array type, then Y* is convertible to T*.
+#if _LIBCPP_STD_VER >= 17
+template <class _Yp, class _Tp, class = void>
+struct __raw_pointer_compatible_with : _And< _Not<is_array<_Tp>>, is_convertible<_Yp*, _Tp*> > {};
+
+template <class _Yp, class _Up, std::size_t _Np>
+struct __raw_pointer_compatible_with<_Yp, _Up[_Np], __enable_if_t< is_convertible<_Yp (*)[_Np], _Up (*)[_Np]>::value> >
+    : true_type {};
+
+template <class _Yp, class _Up>
+struct __raw_pointer_compatible_with<_Yp, _Up[], __enable_if_t< is_convertible<_Yp (*)[], _Up (*)[]>::value> >
+    : true_type {};
+
+#else
+template <class _Yp, class _Tp>
+struct __raw_pointer_compatible_with : is_convertible<_Yp*, _Tp*> {};
+#endif // _LIBCPP_STD_VER >= 17
+
+template <class _Ptr, class = void>
+struct __is_deletable : false_type {};
+template <class _Ptr>
+struct __is_deletable<_Ptr, decltype(delete std::declval<_Ptr>())> : true_type {};
+
+template <class _Ptr, class = void>
+struct __is_array_deletable : false_type {};
+template <class _Ptr>
+struct __is_array_deletable<_Ptr, decltype(delete[] std::declval<_Ptr>())> : true_type {};
+
+template <class _Dp, class _Pt, class = decltype(std::declval<_Dp>()(std::declval<_Pt>()))>
+true_type __well_formed_deleter_test(int);
+
+template <class, class>
+false_type __well_formed_deleter_test(...);
+
+template <class _Dp, class _Pt>
+struct __well_formed_deleter : decltype(std::__well_formed_deleter_test<_Dp, _Pt>(0)) {};
+
+template <class _Dp, class _Yp, class _Tp>
+struct __shared_ptr_deleter_ctor_reqs {
+  static const bool value = __raw_pointer_compatible_with<_Yp, _Tp>::value && is_move_constructible<_Dp>::value &&
+                            __well_formed_deleter<_Dp, _Yp*>::value;
+};
+
+template <class _Dp>
+using __shared_ptr_nullptr_deleter_ctor_reqs = _And<is_move_constructible<_Dp>, __well_formed_deleter<_Dp, nullptr_t> >;
+
+#if defined(_LIBCPP_ABI_ENABLE_SHARED_PTR_TRIVIAL_ABI)
+#  define _LIBCPP_SHARED_PTR_TRIVIAL_ABI __attribute__((__trivial_abi__))
+#else
+#  define _LIBCPP_SHARED_PTR_TRIVIAL_ABI
+#endif
+
+template <class _Tp>
+class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS shared_ptr {
+  struct __nullptr_sfinae_tag {};
+
+public:
+#if _LIBCPP_STD_VER >= 17
+  typedef weak_ptr<_Tp> weak_type;
+  typedef remove_extent_t<_Tp> element_type;
+#else
+  typedef _Tp element_type;
+#endif
+
+  // A shared_ptr contains only two raw pointers which point to the heap and move constructing already doesn't require
+  // any bookkeeping, so it's always trivially relocatable.
+  using __trivially_relocatable = shared_ptr;
+
+private:
+  element_type* __ptr_;
+  __shared_weak_count* __cntrl_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR shared_ptr() _NOEXCEPT : __ptr_(nullptr), __cntrl_(nullptr) {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR shared_ptr(nullptr_t) _NOEXCEPT : __ptr_(nullptr), __cntrl_(nullptr) {}
+
+  template <class _Yp,
+            __enable_if_t< _And< __raw_pointer_compatible_with<_Yp, _Tp>
+  // In C++03 we get errors when trying to do SFINAE with the
+  // delete operator, so we always pretend that it's deletable.
+  // The same happens on GCC.
+#if !defined(_LIBCPP_CXX03_LANG) && !defined(_LIBCPP_COMPILER_GCC)
+                                 ,
+                                 _If<is_array<_Tp>::value, __is_array_deletable<_Yp*>, __is_deletable<_Yp*> >
+#endif
+                                 >::value,
+                           int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit shared_ptr(_Yp* __p) : __ptr_(__p) {
+    unique_ptr<_Yp> __hold(__p);
+    typedef typename __shared_ptr_default_allocator<_Yp>::type _AllocT;
+    typedef __shared_ptr_pointer<_Yp*, __shared_ptr_default_delete<_Tp, _Yp>, _AllocT> _CntrlBlk;
+    __cntrl_ = new _CntrlBlk(__p, __shared_ptr_default_delete<_Tp, _Yp>(), _AllocT());
+    __hold.release();
+    __enable_weak_this(__p, __p);
+  }
+
+  template <class _Yp, class _Dp, __enable_if_t<__shared_ptr_deleter_ctor_reqs<_Dp, _Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI shared_ptr(_Yp* __p, _Dp __d) : __ptr_(__p) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      typedef typename __shared_ptr_default_allocator<_Yp>::type _AllocT;
+      typedef __shared_ptr_pointer<_Yp*, _Dp, _AllocT> _CntrlBlk;
+#ifndef _LIBCPP_CXX03_LANG
+      __cntrl_ = new _CntrlBlk(__p, std::move(__d), _AllocT());
+#else
+    __cntrl_ = new _CntrlBlk(__p, __d, _AllocT());
+#endif // not _LIBCPP_CXX03_LANG
+      __enable_weak_this(__p, __p);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __d(__p);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+
+  template <class _Yp,
+            class _Dp,
+            class _Alloc,
+            __enable_if_t<__shared_ptr_deleter_ctor_reqs<_Dp, _Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI shared_ptr(_Yp* __p, _Dp __d, _Alloc __a) : __ptr_(__p) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      typedef __shared_ptr_pointer<_Yp*, _Dp, _Alloc> _CntrlBlk;
+      typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _A2;
+      typedef __allocator_destructor<_A2> _D2;
+      _A2 __a2(__a);
+      unique_ptr<_CntrlBlk, _D2> __hold2(__a2.allocate(1), _D2(__a2, 1));
+      ::new ((void*)std::addressof(*__hold2.get()))
+#ifndef _LIBCPP_CXX03_LANG
+          _CntrlBlk(__p, std::move(__d), __a);
+#else
+        _CntrlBlk(__p, __d, __a);
+#endif // not _LIBCPP_CXX03_LANG
+      __cntrl_ = std::addressof(*__hold2.release());
+      __enable_weak_this(__p, __p);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __d(__p);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+
+  template <class _Dp>
+  _LIBCPP_HIDE_FROM_ABI shared_ptr(
+      nullptr_t __p,
+      _Dp __d,
+      __enable_if_t<__shared_ptr_nullptr_deleter_ctor_reqs<_Dp>::value, __nullptr_sfinae_tag> = __nullptr_sfinae_tag())
+      : __ptr_(nullptr) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      typedef typename __shared_ptr_default_allocator<_Tp>::type _AllocT;
+      typedef __shared_ptr_pointer<nullptr_t, _Dp, _AllocT> _CntrlBlk;
+#ifndef _LIBCPP_CXX03_LANG
+      __cntrl_ = new _CntrlBlk(__p, std::move(__d), _AllocT());
+#else
+    __cntrl_ = new _CntrlBlk(__p, __d, _AllocT());
+#endif // not _LIBCPP_CXX03_LANG
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __d(__p);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+
+  template <class _Dp, class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI shared_ptr(
+      nullptr_t __p,
+      _Dp __d,
+      _Alloc __a,
+      __enable_if_t<__shared_ptr_nullptr_deleter_ctor_reqs<_Dp>::value, __nullptr_sfinae_tag> = __nullptr_sfinae_tag())
+      : __ptr_(nullptr) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      typedef __shared_ptr_pointer<nullptr_t, _Dp, _Alloc> _CntrlBlk;
+      typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _A2;
+      typedef __allocator_destructor<_A2> _D2;
+      _A2 __a2(__a);
+      unique_ptr<_CntrlBlk, _D2> __hold2(__a2.allocate(1), _D2(__a2, 1));
+      ::new ((void*)std::addressof(*__hold2.get()))
+#ifndef _LIBCPP_CXX03_LANG
+          _CntrlBlk(__p, std::move(__d), __a);
+#else
+        _CntrlBlk(__p, __d, __a);
+#endif // not _LIBCPP_CXX03_LANG
+      __cntrl_ = std::addressof(*__hold2.release());
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __d(__p);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+
+  template <class _Yp>
+  _LIBCPP_HIDE_FROM_ABI shared_ptr(const shared_ptr<_Yp>& __r, element_type* __p) _NOEXCEPT
+      : __ptr_(__p),
+        __cntrl_(__r.__cntrl_) {
+    if (__cntrl_)
+      __cntrl_->__add_shared();
+  }
+
+// LWG-2996
+// We don't backport because it is an evolutionary change.
+#if _LIBCPP_STD_VER >= 20
+  template <class _Yp>
+  _LIBCPP_HIDE_FROM_ABI shared_ptr(shared_ptr<_Yp>&& __r, element_type* __p) noexcept
+      : __ptr_(__p), __cntrl_(__r.__cntrl_) {
+    __r.__ptr_   = nullptr;
+    __r.__cntrl_ = nullptr;
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI shared_ptr(const shared_ptr& __r) _NOEXCEPT : __ptr_(__r.__ptr_), __cntrl_(__r.__cntrl_) {
+    if (__cntrl_)
+      __cntrl_->__add_shared();
+  }
+
+  template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI shared_ptr(const shared_ptr<_Yp>& __r) _NOEXCEPT : __ptr_(__r.__ptr_), __cntrl_(__r.__cntrl_) {
+    if (__cntrl_)
+      __cntrl_->__add_shared();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI shared_ptr(shared_ptr&& __r) _NOEXCEPT : __ptr_(__r.__ptr_), __cntrl_(__r.__cntrl_) {
+    __r.__ptr_   = nullptr;
+    __r.__cntrl_ = nullptr;
+  }
+
+  template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI shared_ptr(shared_ptr<_Yp>&& __r) _NOEXCEPT : __ptr_(__r.__ptr_), __cntrl_(__r.__cntrl_) {
+    __r.__ptr_   = nullptr;
+    __r.__cntrl_ = nullptr;
+  }
+
+  template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit shared_ptr(const weak_ptr<_Yp>& __r)
+      : __ptr_(__r.__ptr_), __cntrl_(__r.__cntrl_ ? __r.__cntrl_->lock() : __r.__cntrl_) {
+    if (__cntrl_ == nullptr)
+      __throw_bad_weak_ptr();
+  }
+
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR)
+  template <class _Yp, __enable_if_t<is_convertible<_Yp*, element_type*>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI shared_ptr(auto_ptr<_Yp>&& __r) : __ptr_(__r.get()) {
+    typedef __shared_ptr_pointer<_Yp*, default_delete<_Yp>, allocator<__remove_cv_t<_Yp> > > _CntrlBlk;
+    __cntrl_ = new _CntrlBlk(__r.get(), default_delete<_Yp>(), allocator<__remove_cv_t<_Yp> >());
+    __enable_weak_this(__r.get(), __r.get());
+    __r.release();
+  }
+#endif
+
+  template <class _Yp,
+            class _Dp,
+            __enable_if_t<!is_lvalue_reference<_Dp>::value && __compatible_with<_Yp, _Tp>::value &&
+                              is_convertible<typename unique_ptr<_Yp, _Dp>::pointer, element_type*>::value,
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI shared_ptr(unique_ptr<_Yp, _Dp>&& __r) : __ptr_(__r.get()) {
+#if _LIBCPP_STD_VER >= 14
+    if (__ptr_ == nullptr)
+      __cntrl_ = nullptr;
+    else
+#endif
+    {
+      typedef typename __shared_ptr_default_allocator<_Yp>::type _AllocT;
+      typedef __shared_ptr_pointer<typename unique_ptr<_Yp, _Dp>::pointer, _Dp, _AllocT> _CntrlBlk;
+      __cntrl_ = new _CntrlBlk(__r.get(), std::move(__r.get_deleter()), _AllocT());
+      __enable_weak_this(__r.get(), __r.get());
+    }
+    __r.release();
+  }
+
+  template <class _Yp,
+            class _Dp,
+            class              = void,
+            __enable_if_t<is_lvalue_reference<_Dp>::value && __compatible_with<_Yp, _Tp>::value &&
+                              is_convertible<typename unique_ptr<_Yp, _Dp>::pointer, element_type*>::value,
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI shared_ptr(unique_ptr<_Yp, _Dp>&& __r) : __ptr_(__r.get()) {
+#if _LIBCPP_STD_VER >= 14
+    if (__ptr_ == nullptr)
+      __cntrl_ = nullptr;
+    else
+#endif
+    {
+      typedef typename __shared_ptr_default_allocator<_Yp>::type _AllocT;
+      typedef __shared_ptr_pointer<typename unique_ptr<_Yp, _Dp>::pointer,
+                                   reference_wrapper<__libcpp_remove_reference_t<_Dp> >,
+                                   _AllocT>
+          _CntrlBlk;
+      __cntrl_ = new _CntrlBlk(__r.get(), std::ref(__r.get_deleter()), _AllocT());
+      __enable_weak_this(__r.get(), __r.get());
+    }
+    __r.release();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~shared_ptr() {
+    if (__cntrl_)
+      __cntrl_->__release_shared();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>& operator=(const shared_ptr& __r) _NOEXCEPT {
+    shared_ptr(__r).swap(*this);
+    return *this;
+  }
+
+  template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>& operator=(const shared_ptr<_Yp>& __r) _NOEXCEPT {
+    shared_ptr(__r).swap(*this);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>& operator=(shared_ptr&& __r) _NOEXCEPT {
+    shared_ptr(std::move(__r)).swap(*this);
+    return *this;
+  }
+
+  template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>& operator=(shared_ptr<_Yp>&& __r) {
+    shared_ptr(std::move(__r)).swap(*this);
+    return *this;
+  }
+
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR)
+  template <class _Yp,
+            __enable_if_t<!is_array<_Yp>::value && is_convertible<_Yp*, typename shared_ptr<_Tp>::element_type*>::value,
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>& operator=(auto_ptr<_Yp>&& __r) {
+    shared_ptr(std::move(__r)).swap(*this);
+    return *this;
+  }
+#endif
+
+  template <class _Yp,
+            class _Dp,
+            __enable_if_t<_And< __compatible_with<_Yp, _Tp>,
+                                is_convertible<typename unique_ptr<_Yp, _Dp>::pointer, element_type*> >::value,
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>& operator=(unique_ptr<_Yp, _Dp>&& __r) {
+    shared_ptr(std::move(__r)).swap(*this);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(shared_ptr& __r) _NOEXCEPT {
+    std::swap(__ptr_, __r.__ptr_);
+    std::swap(__cntrl_, __r.__cntrl_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void reset() _NOEXCEPT { shared_ptr().swap(*this); }
+
+  template <class _Yp, __enable_if_t<__raw_pointer_compatible_with<_Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void reset(_Yp* __p) {
+    shared_ptr(__p).swap(*this);
+  }
+
+  template <class _Yp, class _Dp, __enable_if_t<__shared_ptr_deleter_ctor_reqs<_Dp, _Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void reset(_Yp* __p, _Dp __d) {
+    shared_ptr(__p, __d).swap(*this);
+  }
+
+  template <class _Yp,
+            class _Dp,
+            class _Alloc,
+            __enable_if_t<__shared_ptr_deleter_ctor_reqs<_Dp, _Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void reset(_Yp* __p, _Dp __d, _Alloc __a) {
+    shared_ptr(__p, __d, __a).swap(*this);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI element_type* get() const _NOEXCEPT { return __ptr_; }
+
+  _LIBCPP_HIDE_FROM_ABI __add_lvalue_reference_t<element_type> operator*() const _NOEXCEPT { return *__ptr_; }
+
+  _LIBCPP_HIDE_FROM_ABI element_type* operator->() const _NOEXCEPT {
+    static_assert(!is_array<_Tp>::value, "std::shared_ptr<T>::operator-> is only valid when T is not an array type.");
+    return __ptr_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI long use_count() const _NOEXCEPT { return __cntrl_ ? __cntrl_->use_count() : 0; }
+
+#if _LIBCPP_STD_VER < 20 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_SHARED_PTR_UNIQUE)
+  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI bool unique() const _NOEXCEPT { return use_count() == 1; }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return get() != nullptr; }
+
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI bool owner_before(shared_ptr<_Up> const& __p) const _NOEXCEPT {
+    return __cntrl_ < __p.__cntrl_;
+  }
+
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI bool owner_before(weak_ptr<_Up> const& __p) const _NOEXCEPT {
+    return __cntrl_ < __p.__cntrl_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool __owner_equivalent(const shared_ptr& __p) const { return __cntrl_ == __p.__cntrl_; }
+
+#if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI __add_lvalue_reference_t<element_type> operator[](ptrdiff_t __i) const {
+    static_assert(is_array<_Tp>::value, "std::shared_ptr<T>::operator[] is only valid when T is an array type.");
+    return __ptr_[__i];
+  }
+#endif
+
+#ifndef _LIBCPP_HAS_NO_RTTI
+  template <class _Dp>
+  _LIBCPP_HIDE_FROM_ABI _Dp* __get_deleter() const _NOEXCEPT {
+    return static_cast<_Dp*>(__cntrl_ ? const_cast<void*>(__cntrl_->__get_deleter(typeid(_Dp))) : nullptr);
+  }
+#endif // _LIBCPP_HAS_NO_RTTI
+
+  template <class _Yp, class _CntrlBlk>
+  _LIBCPP_HIDE_FROM_ABI static shared_ptr<_Tp> __create_with_control_block(_Yp* __p, _CntrlBlk* __cntrl) _NOEXCEPT {
+    shared_ptr<_Tp> __r;
+    __r.__ptr_   = __p;
+    __r.__cntrl_ = __cntrl;
+    __r.__enable_weak_this(__r.__ptr_, __r.__ptr_);
+    return __r;
+  }
+
+private:
+  template <class _Yp, bool = is_function<_Yp>::value>
+  struct __shared_ptr_default_allocator {
+    typedef allocator<__remove_cv_t<_Yp> > type;
+  };
+
+  template <class _Yp>
+  struct __shared_ptr_default_allocator<_Yp, true> {
+    typedef allocator<__shared_ptr_dummy_rebind_allocator_type> type;
+  };
+
+  template <class _Yp,
+            class _OrigPtr,
+            __enable_if_t<is_convertible<_OrigPtr*, const enable_shared_from_this<_Yp>*>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void __enable_weak_this(const enable_shared_from_this<_Yp>* __e, _OrigPtr* __ptr) _NOEXCEPT {
+    typedef __remove_cv_t<_Yp> _RawYp;
+    if (__e && __e->__weak_this_.expired()) {
+      __e->__weak_this_ = shared_ptr<_RawYp>(*this, const_cast<_RawYp*>(static_cast<const _Yp*>(__ptr)));
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __enable_weak_this(...) _NOEXCEPT {}
+
+  template <class, class _Yp>
+  struct __shared_ptr_default_delete : default_delete<_Yp> {};
+
+  template <class _Yp, class _Un, size_t _Sz>
+  struct __shared_ptr_default_delete<_Yp[_Sz], _Un> : default_delete<_Yp[]> {};
+
+  template <class _Yp, class _Un>
+  struct __shared_ptr_default_delete<_Yp[], _Un> : default_delete<_Yp[]> {};
+
+  template <class _Up>
+  friend class _LIBCPP_TEMPLATE_VIS shared_ptr;
+  template <class _Up>
+  friend class _LIBCPP_TEMPLATE_VIS weak_ptr;
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+shared_ptr(weak_ptr<_Tp>) -> shared_ptr<_Tp>;
+template <class _Tp, class _Dp>
+shared_ptr(unique_ptr<_Tp, _Dp>) -> shared_ptr<_Tp>;
+#endif
+
+//
+// std::allocate_shared and std::make_shared
+//
+template <class _Tp, class _Alloc, class... _Args, __enable_if_t<!is_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared(const _Alloc& __a, _Args&&... __args) {
+  using _ControlBlock          = __shared_ptr_emplace<_Tp, _Alloc>;
+  using _ControlBlockAllocator = typename __allocator_traits_rebind<_Alloc, _ControlBlock>::type;
+  __allocation_guard<_ControlBlockAllocator> __guard(__a, 1);
+  ::new ((void*)std::addressof(*__guard.__get())) _ControlBlock(__a, std::forward<_Args>(__args)...);
+  auto __control_block = __guard.__release_ptr();
+  return shared_ptr<_Tp>::__create_with_control_block(
+      (*__control_block).__get_elem(), std::addressof(*__control_block));
+}
+
+template <class _Tp, class... _Args, __enable_if_t<!is_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared(_Args&&... __args) {
+  return std::allocate_shared<_Tp>(allocator<__remove_cv_t<_Tp> >(), std::forward<_Args>(__args)...);
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp, class _Alloc, __enable_if_t<!is_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared_for_overwrite(const _Alloc& __a) {
+  using _ForOverwriteAllocator = __allocator_traits_rebind_t<_Alloc, __for_overwrite_tag>;
+  _ForOverwriteAllocator __alloc(__a);
+  return std::allocate_shared<_Tp>(__alloc);
+}
+
+template <class _Tp, __enable_if_t<!is_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared_for_overwrite() {
+  return std::allocate_shared_for_overwrite<_Tp>(allocator<__remove_cv_t<_Tp>>());
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 17
+
+template <size_t _Alignment>
+struct __sp_aligned_storage {
+  alignas(_Alignment) char __storage[_Alignment];
+};
+
+template <class _Tp, class _Alloc>
+struct __unbounded_array_control_block;
+
+template <class _Tp, class _Alloc>
+struct __unbounded_array_control_block<_Tp[], _Alloc> : __shared_weak_count {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp* __get_data() noexcept { return __data_; }
+
+  _LIBCPP_HIDE_FROM_ABI explicit __unbounded_array_control_block(
+      _Alloc const& __alloc, size_t __count, _Tp const& __arg)
+      : __alloc_(__alloc), __count_(__count) {
+    std::__uninitialized_allocator_fill_n_multidimensional(__alloc_, std::begin(__data_), __count_, __arg);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI explicit __unbounded_array_control_block(_Alloc const& __alloc, size_t __count)
+      : __alloc_(__alloc), __count_(__count) {
+#  if _LIBCPP_STD_VER >= 20
+    if constexpr (is_same_v<typename _Alloc::value_type, __for_overwrite_tag>) {
+      // We are purposefully not using an allocator-aware default construction because the spec says so.
+      // There's currently no way of expressing default initialization in an allocator-aware manner anyway.
+      std::uninitialized_default_construct_n(std::begin(__data_), __count_);
+    } else {
+      std::__uninitialized_allocator_value_construct_n_multidimensional(__alloc_, std::begin(__data_), __count_);
+    }
+#  else
+    std::__uninitialized_allocator_value_construct_n_multidimensional(__alloc_, std::begin(__data_), __count_);
+#  endif
+  }
+
+  // Returns the number of bytes required to store a control block followed by the given number
+  // of elements of _Tp, with the whole storage being aligned to a multiple of _Tp's alignment.
+  _LIBCPP_HIDE_FROM_ABI static constexpr size_t __bytes_for(size_t __elements) {
+    // When there's 0 elements, the control block alone is enough since it holds one element.
+    // Otherwise, we allocate one fewer element than requested because the control block already
+    // holds one. Also, we use the bitwise formula below to ensure that we allocate enough bytes
+    // for the whole allocation to be a multiple of _Tp's alignment. That formula is taken from [1].
+    //
+    // [1]: https://en.wikipedia.org/wiki/Data_structure_alignment#Computing_padding
+    size_t __bytes           = __elements == 0 ? sizeof(__unbounded_array_control_block)
+                                               : (__elements - 1) * sizeof(_Tp) + sizeof(__unbounded_array_control_block);
+    constexpr size_t __align = alignof(_Tp);
+    return (__bytes + __align - 1) & ~(__align - 1);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL
+  ~__unbounded_array_control_block() override {
+  } // can't be `= default` because of the sometimes-non-trivial union member __data_
+
+private:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared() _NOEXCEPT override {
+#  if _LIBCPP_STD_VER >= 20
+    if constexpr (is_same_v<typename _Alloc::value_type, __for_overwrite_tag>) {
+      std::__reverse_destroy(__data_, __data_ + __count_);
+    } else {
+      __allocator_traits_rebind_t<_Alloc, _Tp> __value_alloc(__alloc_);
+      std::__allocator_destroy_multidimensional(__value_alloc, __data_, __data_ + __count_);
+    }
+#  else
+    __allocator_traits_rebind_t<_Alloc, _Tp> __value_alloc(__alloc_);
+    std::__allocator_destroy_multidimensional(__value_alloc, __data_, __data_ + __count_);
+#  endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared_weak() _NOEXCEPT override {
+    using _AlignedStorage = __sp_aligned_storage<alignof(__unbounded_array_control_block)>;
+    using _StorageAlloc   = __allocator_traits_rebind_t<_Alloc, _AlignedStorage>;
+    using _PointerTraits  = pointer_traits<typename allocator_traits<_StorageAlloc>::pointer>;
+
+    _StorageAlloc __tmp(__alloc_);
+    __alloc_.~_Alloc();
+    size_t __size              = __unbounded_array_control_block::__bytes_for(__count_);
+    _AlignedStorage* __storage = reinterpret_cast<_AlignedStorage*>(this);
+    allocator_traits<_StorageAlloc>::deallocate(
+        __tmp, _PointerTraits::pointer_to(*__storage), __size / sizeof(_AlignedStorage));
+  }
+
+  _LIBCPP_NO_UNIQUE_ADDRESS _Alloc __alloc_;
+  size_t __count_;
+  union {
+    _Tp __data_[1];
+  };
+};
+
+template <class _Array, class _Alloc, class... _Arg>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Array>
+__allocate_shared_unbounded_array(const _Alloc& __a, size_t __n, _Arg&&... __arg) {
+  static_assert(__libcpp_is_unbounded_array<_Array>::value);
+  // We compute the number of bytes necessary to hold the control block and the
+  // array elements. Then, we allocate an array of properly-aligned dummy structs
+  // large enough to hold the control block and array. This allows shifting the
+  // burden of aligning memory properly from us to the allocator.
+  using _ControlBlock   = __unbounded_array_control_block<_Array, _Alloc>;
+  using _AlignedStorage = __sp_aligned_storage<alignof(_ControlBlock)>;
+  using _StorageAlloc   = __allocator_traits_rebind_t<_Alloc, _AlignedStorage>;
+  __allocation_guard<_StorageAlloc> __guard(__a, _ControlBlock::__bytes_for(__n) / sizeof(_AlignedStorage));
+  _ControlBlock* __control_block = reinterpret_cast<_ControlBlock*>(std::addressof(*__guard.__get()));
+  std::__construct_at(__control_block, __a, __n, std::forward<_Arg>(__arg)...);
+  __guard.__release_ptr();
+  return shared_ptr<_Array>::__create_with_control_block(__control_block->__get_data(), __control_block);
+}
+
+template <class _Tp, class _Alloc>
+struct __bounded_array_control_block;
+
+template <class _Tp, size_t _Count, class _Alloc>
+struct __bounded_array_control_block<_Tp[_Count], _Alloc> : __shared_weak_count {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp* __get_data() noexcept { return __data_; }
+
+  _LIBCPP_HIDE_FROM_ABI explicit __bounded_array_control_block(_Alloc const& __alloc, _Tp const& __arg)
+      : __alloc_(__alloc) {
+    std::__uninitialized_allocator_fill_n_multidimensional(__alloc_, std::addressof(__data_[0]), _Count, __arg);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI explicit __bounded_array_control_block(_Alloc const& __alloc) : __alloc_(__alloc) {
+#  if _LIBCPP_STD_VER >= 20
+    if constexpr (is_same_v<typename _Alloc::value_type, __for_overwrite_tag>) {
+      // We are purposefully not using an allocator-aware default construction because the spec says so.
+      // There's currently no way of expressing default initialization in an allocator-aware manner anyway.
+      std::uninitialized_default_construct_n(std::addressof(__data_[0]), _Count);
+    } else {
+      std::__uninitialized_allocator_value_construct_n_multidimensional(__alloc_, std::addressof(__data_[0]), _Count);
+    }
+#  else
+    std::__uninitialized_allocator_value_construct_n_multidimensional(__alloc_, std::addressof(__data_[0]), _Count);
+#  endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL
+  ~__bounded_array_control_block() override {
+  } // can't be `= default` because of the sometimes-non-trivial union member __data_
+
+private:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared() _NOEXCEPT override {
+#  if _LIBCPP_STD_VER >= 20
+    if constexpr (is_same_v<typename _Alloc::value_type, __for_overwrite_tag>) {
+      std::__reverse_destroy(__data_, __data_ + _Count);
+    } else {
+      __allocator_traits_rebind_t<_Alloc, _Tp> __value_alloc(__alloc_);
+      std::__allocator_destroy_multidimensional(__value_alloc, __data_, __data_ + _Count);
+    }
+#  else
+    __allocator_traits_rebind_t<_Alloc, _Tp> __value_alloc(__alloc_);
+    std::__allocator_destroy_multidimensional(__value_alloc, __data_, __data_ + _Count);
+#  endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared_weak() _NOEXCEPT override {
+    using _ControlBlockAlloc = __allocator_traits_rebind_t<_Alloc, __bounded_array_control_block>;
+    using _PointerTraits     = pointer_traits<typename allocator_traits<_ControlBlockAlloc>::pointer>;
+
+    _ControlBlockAlloc __tmp(__alloc_);
+    __alloc_.~_Alloc();
+    allocator_traits<_ControlBlockAlloc>::deallocate(__tmp, _PointerTraits::pointer_to(*this), 1);
+  }
+
+  _LIBCPP_NO_UNIQUE_ADDRESS _Alloc __alloc_;
+  union {
+    _Tp __data_[_Count];
+  };
+};
+
+template <class _Array, class _Alloc, class... _Arg>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Array> __allocate_shared_bounded_array(const _Alloc& __a, _Arg&&... __arg) {
+  static_assert(__libcpp_is_bounded_array<_Array>::value);
+  using _ControlBlock      = __bounded_array_control_block<_Array, _Alloc>;
+  using _ControlBlockAlloc = __allocator_traits_rebind_t<_Alloc, _ControlBlock>;
+
+  __allocation_guard<_ControlBlockAlloc> __guard(__a, 1);
+  _ControlBlock* __control_block = reinterpret_cast<_ControlBlock*>(std::addressof(*__guard.__get()));
+  std::__construct_at(__control_block, __a, std::forward<_Arg>(__arg)...);
+  __guard.__release_ptr();
+  return shared_ptr<_Array>::__create_with_control_block(__control_block->__get_data(), __control_block);
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#if _LIBCPP_STD_VER >= 20
+
+// bounded array variants
+template <class _Tp, class _Alloc, __enable_if_t<is_bounded_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared(const _Alloc& __a) {
+  return std::__allocate_shared_bounded_array<_Tp>(__a);
+}
+
+template <class _Tp, class _Alloc, __enable_if_t<is_bounded_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared(const _Alloc& __a, const remove_extent_t<_Tp>& __u) {
+  return std::__allocate_shared_bounded_array<_Tp>(__a, __u);
+}
+
+template <class _Tp, class _Alloc, __enable_if_t<is_bounded_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared_for_overwrite(const _Alloc& __a) {
+  using _ForOverwriteAllocator = __allocator_traits_rebind_t<_Alloc, __for_overwrite_tag>;
+  _ForOverwriteAllocator __alloc(__a);
+  return std::__allocate_shared_bounded_array<_Tp>(__alloc);
+}
+
+template <class _Tp, __enable_if_t<is_bounded_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared() {
+  return std::__allocate_shared_bounded_array<_Tp>(allocator<_Tp>());
+}
+
+template <class _Tp, __enable_if_t<is_bounded_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared(const remove_extent_t<_Tp>& __u) {
+  return std::__allocate_shared_bounded_array<_Tp>(allocator<_Tp>(), __u);
+}
+
+template <class _Tp, __enable_if_t<is_bounded_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared_for_overwrite() {
+  return std::__allocate_shared_bounded_array<_Tp>(allocator<__for_overwrite_tag>());
+}
+
+// unbounded array variants
+template <class _Tp, class _Alloc, __enable_if_t<is_unbounded_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared(const _Alloc& __a, size_t __n) {
+  return std::__allocate_shared_unbounded_array<_Tp>(__a, __n);
+}
+
+template <class _Tp, class _Alloc, __enable_if_t<is_unbounded_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared(const _Alloc& __a, size_t __n, const remove_extent_t<_Tp>& __u) {
+  return std::__allocate_shared_unbounded_array<_Tp>(__a, __n, __u);
+}
+
+template <class _Tp, class _Alloc, __enable_if_t<is_unbounded_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared_for_overwrite(const _Alloc& __a, size_t __n) {
+  using _ForOverwriteAllocator = __allocator_traits_rebind_t<_Alloc, __for_overwrite_tag>;
+  _ForOverwriteAllocator __alloc(__a);
+  return std::__allocate_shared_unbounded_array<_Tp>(__alloc, __n);
+}
+
+template <class _Tp, __enable_if_t<is_unbounded_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared(size_t __n) {
+  return std::__allocate_shared_unbounded_array<_Tp>(allocator<_Tp>(), __n);
+}
+
+template <class _Tp, __enable_if_t<is_unbounded_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared(size_t __n, const remove_extent_t<_Tp>& __u) {
+  return std::__allocate_shared_unbounded_array<_Tp>(allocator<_Tp>(), __n, __u);
+}
+
+template <class _Tp, __enable_if_t<is_unbounded_array<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared_for_overwrite(size_t __n) {
+  return std::__allocate_shared_unbounded_array<_Tp>(allocator<__for_overwrite_tag>(), __n);
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _Tp, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(const shared_ptr<_Tp>& __x, const shared_ptr<_Up>& __y) _NOEXCEPT {
+  return __x.get() == __y.get();
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const shared_ptr<_Tp>& __x, const shared_ptr<_Up>& __y) _NOEXCEPT {
+  return !(__x == __y);
+}
+
+template <class _Tp, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(const shared_ptr<_Tp>& __x, const shared_ptr<_Up>& __y) _NOEXCEPT {
+#  if _LIBCPP_STD_VER <= 11
+  typedef typename common_type<_Tp*, _Up*>::type _Vp;
+  return less<_Vp>()(__x.get(), __y.get());
+#  else
+  return less<>()(__x.get(), __y.get());
+#  endif
+}
+
+template <class _Tp, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>(const shared_ptr<_Tp>& __x, const shared_ptr<_Up>& __y) _NOEXCEPT {
+  return __y < __x;
+}
+
+template <class _Tp, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const shared_ptr<_Tp>& __x, const shared_ptr<_Up>& __y) _NOEXCEPT {
+  return !(__y < __x);
+}
+
+template <class _Tp, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>=(const shared_ptr<_Tp>& __x, const shared_ptr<_Up>& __y) _NOEXCEPT {
+  return !(__x < __y);
+}
+
+#endif // _LIBCPP_STD_VER <= 17
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI strong_ordering operator<=>(shared_ptr<_Tp> const& __x, shared_ptr<_Up> const& __y) noexcept {
+  return compare_three_way()(__x.get(), __y.get());
+}
+#endif
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(const shared_ptr<_Tp>& __x, nullptr_t) _NOEXCEPT {
+  return !__x;
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(nullptr_t, const shared_ptr<_Tp>& __x) _NOEXCEPT {
+  return !__x;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const shared_ptr<_Tp>& __x, nullptr_t) _NOEXCEPT {
+  return static_cast<bool>(__x);
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(nullptr_t, const shared_ptr<_Tp>& __x) _NOEXCEPT {
+  return static_cast<bool>(__x);
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(const shared_ptr<_Tp>& __x, nullptr_t) _NOEXCEPT {
+  return less<typename shared_ptr<_Tp>::element_type*>()(__x.get(), nullptr);
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(nullptr_t, const shared_ptr<_Tp>& __x) _NOEXCEPT {
+  return less<typename shared_ptr<_Tp>::element_type*>()(nullptr, __x.get());
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>(const shared_ptr<_Tp>& __x, nullptr_t) _NOEXCEPT {
+  return nullptr < __x;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>(nullptr_t, const shared_ptr<_Tp>& __x) _NOEXCEPT {
+  return __x < nullptr;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const shared_ptr<_Tp>& __x, nullptr_t) _NOEXCEPT {
+  return !(nullptr < __x);
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<=(nullptr_t, const shared_ptr<_Tp>& __x) _NOEXCEPT {
+  return !(__x < nullptr);
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>=(const shared_ptr<_Tp>& __x, nullptr_t) _NOEXCEPT {
+  return !(__x < nullptr);
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>=(nullptr_t, const shared_ptr<_Tp>& __x) _NOEXCEPT {
+  return !(nullptr < __x);
+}
+
+#endif // _LIBCPP_STD_VER <= 17
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI strong_ordering operator<=>(shared_ptr<_Tp> const& __x, nullptr_t) noexcept {
+  return compare_three_way()(__x.get(), static_cast<typename shared_ptr<_Tp>::element_type*>(nullptr));
+}
+#endif
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI void swap(shared_ptr<_Tp>& __x, shared_ptr<_Tp>& __y) _NOEXCEPT {
+  __x.swap(__y);
+}
+
+template <class _Tp, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> static_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
+  return shared_ptr<_Tp>(__r, static_cast< typename shared_ptr<_Tp>::element_type*>(__r.get()));
+}
+
+// LWG-2996
+// We don't backport because it is an evolutionary change.
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> static_pointer_cast(shared_ptr<_Up>&& __r) noexcept {
+  return shared_ptr<_Tp>(std::move(__r), static_cast<typename shared_ptr<_Tp>::element_type*>(__r.get()));
+}
+#endif
+
+template <class _Tp, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> dynamic_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
+  typedef typename shared_ptr<_Tp>::element_type _ET;
+  _ET* __p = dynamic_cast<_ET*>(__r.get());
+  return __p ? shared_ptr<_Tp>(__r, __p) : shared_ptr<_Tp>();
+}
+
+// LWG-2996
+// We don't backport because it is an evolutionary change.
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> dynamic_pointer_cast(shared_ptr<_Up>&& __r) noexcept {
+  auto* __p = dynamic_cast<typename shared_ptr<_Tp>::element_type*>(__r.get());
+  return __p ? shared_ptr<_Tp>(std::move(__r), __p) : shared_ptr<_Tp>();
+}
+#endif
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> const_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
+  typedef typename shared_ptr<_Tp>::element_type _RTp;
+  return shared_ptr<_Tp>(__r, const_cast<_RTp*>(__r.get()));
+}
+
+// LWG-2996
+// We don't backport because it is an evolutionary change.
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> const_pointer_cast(shared_ptr<_Up>&& __r) noexcept {
+  return shared_ptr<_Tp>(std::move(__r), const_cast<typename shared_ptr<_Tp>::element_type*>(__r.get()));
+}
+#endif
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> reinterpret_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
+  return shared_ptr<_Tp>(__r, reinterpret_cast< typename shared_ptr<_Tp>::element_type*>(__r.get()));
+}
+
+// LWG-2996
+// We don't backport because it is an evolutionary change.
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> reinterpret_pointer_cast(shared_ptr<_Up>&& __r) noexcept {
+  return shared_ptr<_Tp>(std::move(__r), reinterpret_cast<typename shared_ptr<_Tp>::element_type*>(__r.get()));
+}
+#endif
+
+#ifndef _LIBCPP_HAS_NO_RTTI
+
+template <class _Dp, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _Dp* get_deleter(const shared_ptr<_Tp>& __p) _NOEXCEPT {
+  return __p.template __get_deleter<_Dp>();
+}
+
+#endif // _LIBCPP_HAS_NO_RTTI
+
+template <class _Tp>
+class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS weak_ptr {
+public:
+#if _LIBCPP_STD_VER >= 17
+  typedef remove_extent_t<_Tp> element_type;
+#else
+  typedef _Tp element_type;
+#endif
+
+  // A weak_ptr contains only two raw pointers which point to the heap and move constructing already doesn't require
+  // any bookkeeping, so it's always trivially relocatable.
+  using __trivially_relocatable = weak_ptr;
+
+private:
+  element_type* __ptr_;
+  __shared_weak_count* __cntrl_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR weak_ptr() _NOEXCEPT;
+
+  template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI weak_ptr(shared_ptr<_Yp> const& __r) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI weak_ptr(weak_ptr const& __r) _NOEXCEPT;
+
+  template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI weak_ptr(weak_ptr<_Yp> const& __r) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI weak_ptr(weak_ptr&& __r) _NOEXCEPT;
+
+  template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI weak_ptr(weak_ptr<_Yp>&& __r) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI ~weak_ptr();
+
+  _LIBCPP_HIDE_FROM_ABI weak_ptr& operator=(weak_ptr const& __r) _NOEXCEPT;
+  template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI weak_ptr& operator=(weak_ptr<_Yp> const& __r) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI weak_ptr& operator=(weak_ptr&& __r) _NOEXCEPT;
+  template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI weak_ptr& operator=(weak_ptr<_Yp>&& __r) _NOEXCEPT;
+
+  template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI weak_ptr& operator=(shared_ptr<_Yp> const& __r) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI void swap(weak_ptr& __r) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI void reset() _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI long use_count() const _NOEXCEPT { return __cntrl_ ? __cntrl_->use_count() : 0; }
+  _LIBCPP_HIDE_FROM_ABI bool expired() const _NOEXCEPT { return __cntrl_ == nullptr || __cntrl_->use_count() == 0; }
+  _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> lock() const _NOEXCEPT;
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI bool owner_before(const shared_ptr<_Up>& __r) const _NOEXCEPT {
+    return __cntrl_ < __r.__cntrl_;
+  }
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI bool owner_before(const weak_ptr<_Up>& __r) const _NOEXCEPT {
+    return __cntrl_ < __r.__cntrl_;
+  }
+
+  template <class _Up>
+  friend class _LIBCPP_TEMPLATE_VIS weak_ptr;
+  template <class _Up>
+  friend class _LIBCPP_TEMPLATE_VIS shared_ptr;
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+weak_ptr(shared_ptr<_Tp>) -> weak_ptr<_Tp>;
+#endif
+
+template <class _Tp>
+inline _LIBCPP_CONSTEXPR weak_ptr<_Tp>::weak_ptr() _NOEXCEPT : __ptr_(nullptr), __cntrl_(nullptr) {}
+
+template <class _Tp>
+inline weak_ptr<_Tp>::weak_ptr(weak_ptr const& __r) _NOEXCEPT : __ptr_(__r.__ptr_), __cntrl_(__r.__cntrl_) {
+  if (__cntrl_)
+    __cntrl_->__add_weak();
+}
+
+template <class _Tp>
+template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> >
+inline weak_ptr<_Tp>::weak_ptr(shared_ptr<_Yp> const& __r) _NOEXCEPT : __ptr_(__r.__ptr_), __cntrl_(__r.__cntrl_) {
+  if (__cntrl_)
+    __cntrl_->__add_weak();
+}
+
+template <class _Tp>
+template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> >
+inline weak_ptr<_Tp>::weak_ptr(weak_ptr<_Yp> const& __r) _NOEXCEPT : __ptr_(nullptr), __cntrl_(nullptr) {
+  shared_ptr<_Yp> __s = __r.lock();
+  *this               = weak_ptr<_Tp>(__s);
+}
+
+template <class _Tp>
+inline weak_ptr<_Tp>::weak_ptr(weak_ptr&& __r) _NOEXCEPT : __ptr_(__r.__ptr_), __cntrl_(__r.__cntrl_) {
+  __r.__ptr_   = nullptr;
+  __r.__cntrl_ = nullptr;
+}
+
+template <class _Tp>
+template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> >
+inline weak_ptr<_Tp>::weak_ptr(weak_ptr<_Yp>&& __r) _NOEXCEPT : __ptr_(nullptr), __cntrl_(nullptr) {
+  shared_ptr<_Yp> __s = __r.lock();
+  *this               = weak_ptr<_Tp>(__s);
+  __r.reset();
+}
+
+template <class _Tp>
+weak_ptr<_Tp>::~weak_ptr() {
+  if (__cntrl_)
+    __cntrl_->__release_weak();
+}
+
+template <class _Tp>
+inline weak_ptr<_Tp>& weak_ptr<_Tp>::operator=(weak_ptr const& __r) _NOEXCEPT {
+  weak_ptr(__r).swap(*this);
+  return *this;
+}
+
+template <class _Tp>
+template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> >
+inline weak_ptr<_Tp>& weak_ptr<_Tp>::operator=(weak_ptr<_Yp> const& __r) _NOEXCEPT {
+  weak_ptr(__r).swap(*this);
+  return *this;
+}
+
+template <class _Tp>
+inline weak_ptr<_Tp>& weak_ptr<_Tp>::operator=(weak_ptr&& __r) _NOEXCEPT {
+  weak_ptr(std::move(__r)).swap(*this);
+  return *this;
+}
+
+template <class _Tp>
+template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> >
+inline weak_ptr<_Tp>& weak_ptr<_Tp>::operator=(weak_ptr<_Yp>&& __r) _NOEXCEPT {
+  weak_ptr(std::move(__r)).swap(*this);
+  return *this;
+}
+
+template <class _Tp>
+template <class _Yp, __enable_if_t<__compatible_with<_Yp, _Tp>::value, int> >
+inline weak_ptr<_Tp>& weak_ptr<_Tp>::operator=(shared_ptr<_Yp> const& __r) _NOEXCEPT {
+  weak_ptr(__r).swap(*this);
+  return *this;
+}
+
+template <class _Tp>
+inline void weak_ptr<_Tp>::swap(weak_ptr& __r) _NOEXCEPT {
+  std::swap(__ptr_, __r.__ptr_);
+  std::swap(__cntrl_, __r.__cntrl_);
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI void swap(weak_ptr<_Tp>& __x, weak_ptr<_Tp>& __y) _NOEXCEPT {
+  __x.swap(__y);
+}
+
+template <class _Tp>
+inline void weak_ptr<_Tp>::reset() _NOEXCEPT {
+  weak_ptr().swap(*this);
+}
+
+template <class _Tp>
+shared_ptr<_Tp> weak_ptr<_Tp>::lock() const _NOEXCEPT {
+  shared_ptr<_Tp> __r;
+  __r.__cntrl_ = __cntrl_ ? __cntrl_->lock() : __cntrl_;
+  if (__r.__cntrl_)
+    __r.__ptr_ = __ptr_;
+  return __r;
+}
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp = void>
+struct owner_less;
+#else
+template <class _Tp>
+struct owner_less;
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS owner_less<shared_ptr<_Tp> > : __binary_function<shared_ptr<_Tp>, shared_ptr<_Tp>, bool> {
+  _LIBCPP_HIDE_FROM_ABI bool operator()(shared_ptr<_Tp> const& __x, shared_ptr<_Tp> const& __y) const _NOEXCEPT {
+    return __x.owner_before(__y);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(shared_ptr<_Tp> const& __x, weak_ptr<_Tp> const& __y) const _NOEXCEPT {
+    return __x.owner_before(__y);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(weak_ptr<_Tp> const& __x, shared_ptr<_Tp> const& __y) const _NOEXCEPT {
+    return __x.owner_before(__y);
+  }
+};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS owner_less<weak_ptr<_Tp> > : __binary_function<weak_ptr<_Tp>, weak_ptr<_Tp>, bool> {
+  _LIBCPP_HIDE_FROM_ABI bool operator()(weak_ptr<_Tp> const& __x, weak_ptr<_Tp> const& __y) const _NOEXCEPT {
+    return __x.owner_before(__y);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(shared_ptr<_Tp> const& __x, weak_ptr<_Tp> const& __y) const _NOEXCEPT {
+    return __x.owner_before(__y);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(weak_ptr<_Tp> const& __x, shared_ptr<_Tp> const& __y) const _NOEXCEPT {
+    return __x.owner_before(__y);
+  }
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <>
+struct _LIBCPP_TEMPLATE_VIS owner_less<void> {
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(shared_ptr<_Tp> const& __x, shared_ptr<_Up> const& __y) const _NOEXCEPT {
+    return __x.owner_before(__y);
+  }
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(shared_ptr<_Tp> const& __x, weak_ptr<_Up> const& __y) const _NOEXCEPT {
+    return __x.owner_before(__y);
+  }
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(weak_ptr<_Tp> const& __x, shared_ptr<_Up> const& __y) const _NOEXCEPT {
+    return __x.owner_before(__y);
+  }
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(weak_ptr<_Tp> const& __x, weak_ptr<_Up> const& __y) const _NOEXCEPT {
+    return __x.owner_before(__y);
+  }
+  typedef void is_transparent;
+};
+#endif
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS enable_shared_from_this {
+  mutable weak_ptr<_Tp> __weak_this_;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR enable_shared_from_this() _NOEXCEPT {}
+  _LIBCPP_HIDE_FROM_ABI enable_shared_from_this(enable_shared_from_this const&) _NOEXCEPT {}
+  _LIBCPP_HIDE_FROM_ABI enable_shared_from_this& operator=(enable_shared_from_this const&) _NOEXCEPT { return *this; }
+  _LIBCPP_HIDE_FROM_ABI ~enable_shared_from_this() {}
+
+public:
+  _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> shared_from_this() { return shared_ptr<_Tp>(__weak_this_); }
+  _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp const> shared_from_this() const { return shared_ptr<const _Tp>(__weak_this_); }
+
+#if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI weak_ptr<_Tp> weak_from_this() _NOEXCEPT { return __weak_this_; }
+
+  _LIBCPP_HIDE_FROM_ABI weak_ptr<const _Tp> weak_from_this() const _NOEXCEPT { return __weak_this_; }
+#endif // _LIBCPP_STD_VER >= 17
+
+  template <class _Up>
+  friend class shared_ptr;
+};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS hash;
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS hash<shared_ptr<_Tp> > {
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef shared_ptr<_Tp> argument_type;
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef size_t result_type;
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const shared_ptr<_Tp>& __ptr) const _NOEXCEPT {
+    return hash<typename shared_ptr<_Tp>::element_type*>()(__ptr.get());
+  }
+};
+
+template <class _CharT, class _Traits, class _Yp>
+inline _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, shared_ptr<_Yp> const& __p);
+
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+
+class _LIBCPP_EXPORTED_FROM_ABI __sp_mut {
+  void* __lx_;
+
+public:
+  void lock() _NOEXCEPT;
+  void unlock() _NOEXCEPT;
+
+private:
+  _LIBCPP_CONSTEXPR __sp_mut(void*) _NOEXCEPT;
+  __sp_mut(const __sp_mut&);
+  __sp_mut& operator=(const __sp_mut&);
+
+  friend _LIBCPP_EXPORTED_FROM_ABI __sp_mut& __get_sp_mut(const void*);
+};
+
+_LIBCPP_EXPORTED_FROM_ABI __sp_mut& __get_sp_mut(const void*);
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const shared_ptr<_Tp>*) {
+  return false;
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_load(const shared_ptr<_Tp>* __p) {
+  __sp_mut& __m = std::__get_sp_mut(__p);
+  __m.lock();
+  shared_ptr<_Tp> __q = *__p;
+  __m.unlock();
+  return __q;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_load_explicit(const shared_ptr<_Tp>* __p, memory_order) {
+  return std::atomic_load(__p);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void atomic_store(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) {
+  __sp_mut& __m = std::__get_sp_mut(__p);
+  __m.lock();
+  __p->swap(__r);
+  __m.unlock();
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI void atomic_store_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r, memory_order) {
+  std::atomic_store(__p, __r);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_exchange(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) {
+  __sp_mut& __m = std::__get_sp_mut(__p);
+  __m.lock();
+  __p->swap(__r);
+  __m.unlock();
+  return __r;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
+atomic_exchange_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r, memory_order) {
+  return std::atomic_exchange(__p, __r);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool
+atomic_compare_exchange_strong(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w) {
+  shared_ptr<_Tp> __temp;
+  __sp_mut& __m = std::__get_sp_mut(__p);
+  __m.lock();
+  if (__p->__owner_equivalent(*__v)) {
+    std::swap(__temp, *__p);
+    *__p = __w;
+    __m.unlock();
+    return true;
+  }
+  std::swap(__temp, *__v);
+  *__v = *__p;
+  __m.unlock();
+  return false;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool
+atomic_compare_exchange_weak(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w) {
+  return std::atomic_compare_exchange_strong(__p, __v, __w);
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(
+    shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w, memory_order, memory_order) {
+  return std::atomic_compare_exchange_strong(__p, __v, __w);
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(
+    shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w, memory_order, memory_order) {
+  return std::atomic_compare_exchange_weak(__p, __v, __w);
+}
+
+#endif // !defined(_LIBCPP_HAS_NO_THREADS)
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_SHARED_PTR_H
diff --git a/libcxx/include/__cxx03/__memory/swap_allocator.h b/libcxx/include/__cxx03/__memory/swap_allocator.h
new file mode 100644
index 000000000000..b17e082a43c9
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/swap_allocator.h
@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_SWAP_ALLOCATOR_H
+#define _LIBCPP___MEMORY_SWAP_ALLOCATOR_H
+
+#include <__config>
+#include <__memory/allocator_traits.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_swappable.h>
+#include <__utility/swap.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <typename _Alloc>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __swap_allocator(_Alloc& __a1, _Alloc& __a2, true_type)
+#if _LIBCPP_STD_VER >= 14
+    _NOEXCEPT
+#else
+    _NOEXCEPT_(__is_nothrow_swappable_v<_Alloc>)
+#endif
+{
+  using std::swap;
+  swap(__a1, __a2);
+}
+
+template <typename _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+__swap_allocator(_Alloc&, _Alloc&, false_type) _NOEXCEPT {}
+
+template <typename _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __swap_allocator(_Alloc& __a1, _Alloc& __a2)
+#if _LIBCPP_STD_VER >= 14
+    _NOEXCEPT
+#else
+    _NOEXCEPT_(__is_nothrow_swappable_v<_Alloc>)
+#endif
+{
+  std::__swap_allocator(
+      __a1, __a2, integral_constant<bool, allocator_traits<_Alloc>::propagate_on_container_swap::value>());
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_SWAP_ALLOCATOR_H
diff --git a/libcxx/include/__cxx03/__memory/temp_value.h b/libcxx/include/__cxx03/__memory/temp_value.h
new file mode 100644
index 000000000000..4a133b3fbcf6
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/temp_value.h
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_TEMP_VALUE_H
+#define _LIBCPP___MEMORY_TEMP_VALUE_H
+
+#include <__config>
+#include <__memory/addressof.h>
+#include <__memory/allocator_traits.h>
+#include <__type_traits/aligned_storage.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Alloc>
+struct __temp_value {
+  typedef allocator_traits<_Alloc> _Traits;
+
+#ifdef _LIBCPP_CXX03_LANG
+  typename aligned_storage<sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp)>::type __v;
+#else
+  union {
+    _Tp __v;
+  };
+#endif
+  _Alloc& __a;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __addr() {
+#ifdef _LIBCPP_CXX03_LANG
+    return reinterpret_cast<_Tp*>(std::addressof(__v));
+#else
+    return std::addressof(__v);
+#endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& get() { return *__addr(); }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI _LIBCPP_CONSTEXPR_SINCE_CXX20 __temp_value(_Alloc& __alloc, _Args&&... __args)
+      : __a(__alloc) {
+    _Traits::construct(__a, __addr(), std::forward<_Args>(__args)...);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__temp_value() { _Traits::destroy(__a, __addr()); }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_TEMP_VALUE_H
diff --git a/libcxx/include/__cxx03/__memory/temporary_buffer.h b/libcxx/include/__cxx03/__memory/temporary_buffer.h
new file mode 100644
index 000000000000..88799ca95c1f
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/temporary_buffer.h
@@ -0,0 +1,75 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_TEMPORARY_BUFFER_H
+#define _LIBCPP___MEMORY_TEMPORARY_BUFFER_H
+
+#include <__config>
+#include <__utility/pair.h>
+#include <cstddef>
+#include <new>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI _LIBCPP_DEPRECATED_IN_CXX17 pair<_Tp*, ptrdiff_t>
+get_temporary_buffer(ptrdiff_t __n) _NOEXCEPT {
+  pair<_Tp*, ptrdiff_t> __r(0, 0);
+  const ptrdiff_t __m =
+      (~ptrdiff_t(0) ^ ptrdiff_t(ptrdiff_t(1) << (sizeof(ptrdiff_t) * __CHAR_BIT__ - 1))) / sizeof(_Tp);
+  if (__n > __m)
+    __n = __m;
+  while (__n > 0) {
+#if !defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION)
+    if (__is_overaligned_for_new(_LIBCPP_ALIGNOF(_Tp))) {
+      align_val_t __al = align_val_t(_LIBCPP_ALIGNOF(_Tp));
+      __r.first        = static_cast<_Tp*>(::operator new(__n * sizeof(_Tp), __al, nothrow));
+    } else {
+      __r.first = static_cast<_Tp*>(::operator new(__n * sizeof(_Tp), nothrow));
+    }
+#else
+    if (__is_overaligned_for_new(_LIBCPP_ALIGNOF(_Tp))) {
+      // Since aligned operator new is unavailable, return an empty
+      // buffer rather than one with invalid alignment.
+      return __r;
+    }
+
+    __r.first = static_cast<_Tp*>(::operator new(__n * sizeof(_Tp), nothrow));
+#endif
+
+    if (__r.first) {
+      __r.second = __n;
+      break;
+    }
+    __n /= 2;
+  }
+  return __r;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_IN_CXX17 void return_temporary_buffer(_Tp* __p) _NOEXCEPT {
+  std::__libcpp_deallocate_unsized((void*)__p, _LIBCPP_ALIGNOF(_Tp));
+}
+
+struct __return_temporary_buffer {
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  template <class _Tp>
+  _LIBCPP_HIDE_FROM_ABI void operator()(_Tp* __p) const {
+    std::return_temporary_buffer(__p);
+  }
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_TEMPORARY_BUFFER_H
diff --git a/libcxx/include/__cxx03/__memory/uninitialized_algorithms.h b/libcxx/include/__cxx03/__memory/uninitialized_algorithms.h
new file mode 100644
index 000000000000..7475ef5cf85d
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/uninitialized_algorithms.h
@@ -0,0 +1,653 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_UNINITIALIZED_ALGORITHMS_H
+#define _LIBCPP___MEMORY_UNINITIALIZED_ALGORITHMS_H
+
+#include <__algorithm/copy.h>
+#include <__algorithm/move.h>
+#include <__algorithm/unwrap_iter.h>
+#include <__algorithm/unwrap_range.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/reverse_iterator.h>
+#include <__memory/addressof.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/construct_at.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/voidify.h>
+#include <__type_traits/extent.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_trivially_assignable.h>
+#include <__type_traits/is_trivially_constructible.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__type_traits/is_unbounded_array.h>
+#include <__type_traits/negation.h>
+#include <__type_traits/remove_const.h>
+#include <__type_traits/remove_extent.h>
+#include <__utility/exception_guard.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <new>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct __always_false {
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator()(_Args&&...) const _NOEXCEPT {
+    return false;
+  }
+};
+
+// uninitialized_copy
+
+template <class _ValueType, class _InputIterator, class _Sentinel1, class _ForwardIterator, class _EndPredicate>
+inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitialized_copy(
+    _InputIterator __ifirst, _Sentinel1 __ilast, _ForwardIterator __ofirst, _EndPredicate __stop_copying) {
+  _ForwardIterator __idx = __ofirst;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif
+    for (; __ifirst != __ilast && !__stop_copying(__idx); ++__ifirst, (void)++__idx)
+      ::new (std::__voidify(*__idx)) _ValueType(*__ifirst);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    std::__destroy(__ofirst, __idx);
+    throw;
+  }
+#endif
+
+  return pair<_InputIterator, _ForwardIterator>(std::move(__ifirst), std::move(__idx));
+}
+
+template <class _InputIterator, class _ForwardIterator>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator
+uninitialized_copy(_InputIterator __ifirst, _InputIterator __ilast, _ForwardIterator __ofirst) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  auto __result = std::__uninitialized_copy<_ValueType>(
+      std::move(__ifirst), std::move(__ilast), std::move(__ofirst), __always_false());
+  return std::move(__result.second);
+}
+
+// uninitialized_copy_n
+
+template <class _ValueType, class _InputIterator, class _Size, class _ForwardIterator, class _EndPredicate>
+inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator>
+__uninitialized_copy_n(_InputIterator __ifirst, _Size __n, _ForwardIterator __ofirst, _EndPredicate __stop_copying) {
+  _ForwardIterator __idx = __ofirst;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif
+    for (; __n > 0 && !__stop_copying(__idx); ++__ifirst, (void)++__idx, (void)--__n)
+      ::new (std::__voidify(*__idx)) _ValueType(*__ifirst);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    std::__destroy(__ofirst, __idx);
+    throw;
+  }
+#endif
+
+  return pair<_InputIterator, _ForwardIterator>(std::move(__ifirst), std::move(__idx));
+}
+
+template <class _InputIterator, class _Size, class _ForwardIterator>
+inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
+uninitialized_copy_n(_InputIterator __ifirst, _Size __n, _ForwardIterator __ofirst) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  auto __result =
+      std::__uninitialized_copy_n<_ValueType>(std::move(__ifirst), __n, std::move(__ofirst), __always_false());
+  return std::move(__result.second);
+}
+
+// uninitialized_fill
+
+template <class _ValueType, class _ForwardIterator, class _Sentinel, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
+__uninitialized_fill(_ForwardIterator __first, _Sentinel __last, const _Tp& __x) {
+  _ForwardIterator __idx = __first;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif
+    for (; __idx != __last; ++__idx)
+      ::new (std::__voidify(*__idx)) _ValueType(__x);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    std::__destroy(__first, __idx);
+    throw;
+  }
+#endif
+
+  return __idx;
+}
+
+template <class _ForwardIterator, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI void
+uninitialized_fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __x) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  (void)std::__uninitialized_fill<_ValueType>(__first, __last, __x);
+}
+
+// uninitialized_fill_n
+
+template <class _ValueType, class _ForwardIterator, class _Size, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
+__uninitialized_fill_n(_ForwardIterator __first, _Size __n, const _Tp& __x) {
+  _ForwardIterator __idx = __first;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif
+    for (; __n > 0; ++__idx, (void)--__n)
+      ::new (std::__voidify(*__idx)) _ValueType(__x);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    std::__destroy(__first, __idx);
+    throw;
+  }
+#endif
+
+  return __idx;
+}
+
+template <class _ForwardIterator, class _Size, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
+uninitialized_fill_n(_ForwardIterator __first, _Size __n, const _Tp& __x) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  return std::__uninitialized_fill_n<_ValueType>(__first, __n, __x);
+}
+
+#if _LIBCPP_STD_VER >= 17
+
+// uninitialized_default_construct
+
+template <class _ValueType, class _ForwardIterator, class _Sentinel>
+inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
+__uninitialized_default_construct(_ForwardIterator __first, _Sentinel __last) {
+  auto __idx = __first;
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif
+    for (; __idx != __last; ++__idx)
+      ::new (std::__voidify(*__idx)) _ValueType;
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    std::__destroy(__first, __idx);
+    throw;
+  }
+#  endif
+
+  return __idx;
+}
+
+template <class _ForwardIterator>
+inline _LIBCPP_HIDE_FROM_ABI void uninitialized_default_construct(_ForwardIterator __first, _ForwardIterator __last) {
+  using _ValueType = typename iterator_traits<_ForwardIterator>::value_type;
+  (void)std::__uninitialized_default_construct<_ValueType>(std::move(__first), std::move(__last));
+}
+
+// uninitialized_default_construct_n
+
+template <class _ValueType, class _ForwardIterator, class _Size>
+inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator __uninitialized_default_construct_n(_ForwardIterator __first, _Size __n) {
+  auto __idx = __first;
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif
+    for (; __n > 0; ++__idx, (void)--__n)
+      ::new (std::__voidify(*__idx)) _ValueType;
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    std::__destroy(__first, __idx);
+    throw;
+  }
+#  endif
+
+  return __idx;
+}
+
+template <class _ForwardIterator, class _Size>
+inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator uninitialized_default_construct_n(_ForwardIterator __first, _Size __n) {
+  using _ValueType = typename iterator_traits<_ForwardIterator>::value_type;
+  return std::__uninitialized_default_construct_n<_ValueType>(std::move(__first), __n);
+}
+
+// uninitialized_value_construct
+
+template <class _ValueType, class _ForwardIterator, class _Sentinel>
+inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
+__uninitialized_value_construct(_ForwardIterator __first, _Sentinel __last) {
+  auto __idx = __first;
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif
+    for (; __idx != __last; ++__idx)
+      ::new (std::__voidify(*__idx)) _ValueType();
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    std::__destroy(__first, __idx);
+    throw;
+  }
+#  endif
+
+  return __idx;
+}
+
+template <class _ForwardIterator>
+inline _LIBCPP_HIDE_FROM_ABI void uninitialized_value_construct(_ForwardIterator __first, _ForwardIterator __last) {
+  using _ValueType = typename iterator_traits<_ForwardIterator>::value_type;
+  (void)std::__uninitialized_value_construct<_ValueType>(std::move(__first), std::move(__last));
+}
+
+// uninitialized_value_construct_n
+
+template <class _ValueType, class _ForwardIterator, class _Size>
+inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator __uninitialized_value_construct_n(_ForwardIterator __first, _Size __n) {
+  auto __idx = __first;
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif
+    for (; __n > 0; ++__idx, (void)--__n)
+      ::new (std::__voidify(*__idx)) _ValueType();
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    std::__destroy(__first, __idx);
+    throw;
+  }
+#  endif
+
+  return __idx;
+}
+
+template <class _ForwardIterator, class _Size>
+inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator uninitialized_value_construct_n(_ForwardIterator __first, _Size __n) {
+  using _ValueType = typename iterator_traits<_ForwardIterator>::value_type;
+  return std::__uninitialized_value_construct_n<_ValueType>(std::move(__first), __n);
+}
+
+// uninitialized_move
+
+template <class _ValueType,
+          class _InputIterator,
+          class _Sentinel1,
+          class _ForwardIterator,
+          class _EndPredicate,
+          class _IterMove>
+inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitialized_move(
+    _InputIterator __ifirst,
+    _Sentinel1 __ilast,
+    _ForwardIterator __ofirst,
+    _EndPredicate __stop_moving,
+    _IterMove __iter_move) {
+  auto __idx = __ofirst;
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif
+    for (; __ifirst != __ilast && !__stop_moving(__idx); ++__idx, (void)++__ifirst) {
+      ::new (std::__voidify(*__idx)) _ValueType(__iter_move(__ifirst));
+    }
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    std::__destroy(__ofirst, __idx);
+    throw;
+  }
+#  endif
+
+  return {std::move(__ifirst), std::move(__idx)};
+}
+
+template <class _InputIterator, class _ForwardIterator>
+inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
+uninitialized_move(_InputIterator __ifirst, _InputIterator __ilast, _ForwardIterator __ofirst) {
+  using _ValueType = typename iterator_traits<_ForwardIterator>::value_type;
+  auto __iter_move = [](auto&& __iter) -> decltype(auto) { return std::move(*__iter); };
+
+  auto __result = std::__uninitialized_move<_ValueType>(
+      std::move(__ifirst), std::move(__ilast), std::move(__ofirst), __always_false(), __iter_move);
+  return std::move(__result.second);
+}
+
+// uninitialized_move_n
+
+template <class _ValueType,
+          class _InputIterator,
+          class _Size,
+          class _ForwardIterator,
+          class _EndPredicate,
+          class _IterMove>
+inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitialized_move_n(
+    _InputIterator __ifirst, _Size __n, _ForwardIterator __ofirst, _EndPredicate __stop_moving, _IterMove __iter_move) {
+  auto __idx = __ofirst;
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif
+    for (; __n > 0 && !__stop_moving(__idx); ++__idx, (void)++__ifirst, --__n)
+      ::new (std::__voidify(*__idx)) _ValueType(__iter_move(__ifirst));
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    std::__destroy(__ofirst, __idx);
+    throw;
+  }
+#  endif
+
+  return {std::move(__ifirst), std::move(__idx)};
+}
+
+template <class _InputIterator, class _Size, class _ForwardIterator>
+inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator>
+uninitialized_move_n(_InputIterator __ifirst, _Size __n, _ForwardIterator __ofirst) {
+  using _ValueType = typename iterator_traits<_ForwardIterator>::value_type;
+  auto __iter_move = [](auto&& __iter) -> decltype(auto) { return std::move(*__iter); };
+
+  return std::__uninitialized_move_n<_ValueType>(
+      std::move(__ifirst), __n, std::move(__ofirst), __always_false(), __iter_move);
+}
+
+// TODO: Rewrite this to iterate left to right and use reverse_iterators when calling
+// Destroys every element in the range [first, last) FROM RIGHT TO LEFT using allocator
+// destruction. If elements are themselves C-style arrays, they are recursively destroyed
+// in the same manner.
+//
+// This function assumes that destructors do not throw, and that the allocator is bound to
+// the correct type.
+template <class _Alloc,
+          class _BidirIter,
+          __enable_if_t<__has_bidirectional_iterator_category<_BidirIter>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr void
+__allocator_destroy_multidimensional(_Alloc& __alloc, _BidirIter __first, _BidirIter __last) noexcept {
+  using _ValueType = typename iterator_traits<_BidirIter>::value_type;
+  static_assert(is_same_v<typename allocator_traits<_Alloc>::value_type, _ValueType>,
+                "The allocator should already be rebound to the correct type");
+
+  if (__first == __last)
+    return;
+
+  if constexpr (is_array_v<_ValueType>) {
+    static_assert(!__libcpp_is_unbounded_array<_ValueType>::value,
+                  "arrays of unbounded arrays don't exist, but if they did we would mess up here");
+
+    using _Element = remove_extent_t<_ValueType>;
+    __allocator_traits_rebind_t<_Alloc, _Element> __elem_alloc(__alloc);
+    do {
+      --__last;
+      decltype(auto) __array = *__last;
+      std::__allocator_destroy_multidimensional(__elem_alloc, __array, __array + extent_v<_ValueType>);
+    } while (__last != __first);
+  } else {
+    do {
+      --__last;
+      allocator_traits<_Alloc>::destroy(__alloc, std::addressof(*__last));
+    } while (__last != __first);
+  }
+}
+
+// Constructs the object at the given location using the allocator's construct method.
+//
+// If the object being constructed is an array, each element of the array is allocator-constructed,
+// recursively. If an exception is thrown during the construction of an array, the initialized
+// elements are destroyed in reverse order of initialization using allocator destruction.
+//
+// This function assumes that the allocator is bound to the correct type.
+template <class _Alloc, class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr void __allocator_construct_at_multidimensional(_Alloc& __alloc, _Tp* __loc) {
+  static_assert(is_same_v<typename allocator_traits<_Alloc>::value_type, _Tp>,
+                "The allocator should already be rebound to the correct type");
+
+  if constexpr (is_array_v<_Tp>) {
+    using _Element = remove_extent_t<_Tp>;
+    __allocator_traits_rebind_t<_Alloc, _Element> __elem_alloc(__alloc);
+    size_t __i   = 0;
+    _Tp& __array = *__loc;
+
+    // If an exception is thrown, destroy what we have constructed so far in reverse order.
+    auto __guard = std::__make_exception_guard([&]() {
+      std::__allocator_destroy_multidimensional(__elem_alloc, __array, __array + __i);
+    });
+
+    for (; __i != extent_v<_Tp>; ++__i) {
+      std::__allocator_construct_at_multidimensional(__elem_alloc, std::addressof(__array[__i]));
+    }
+    __guard.__complete();
+  } else {
+    allocator_traits<_Alloc>::construct(__alloc, __loc);
+  }
+}
+
+// Constructs the object at the given location using the allocator's construct method, passing along
+// the provided argument.
+//
+// If the object being constructed is an array, the argument is also assumed to be an array. Each
+// each element of the array being constructed is allocator-constructed from the corresponding
+// element of the argument array. If an exception is thrown during the construction of an array,
+// the initialized elements are destroyed in reverse order of initialization using allocator
+// destruction.
+//
+// This function assumes that the allocator is bound to the correct type.
+template <class _Alloc, class _Tp, class _Arg>
+_LIBCPP_HIDE_FROM_ABI constexpr void
+__allocator_construct_at_multidimensional(_Alloc& __alloc, _Tp* __loc, _Arg const& __arg) {
+  static_assert(is_same_v<typename allocator_traits<_Alloc>::value_type, _Tp>,
+                "The allocator should already be rebound to the correct type");
+
+  if constexpr (is_array_v<_Tp>) {
+    static_assert(is_array_v<_Arg>,
+                  "Provided non-array initialization argument to __allocator_construct_at_multidimensional when "
+                  "trying to construct an array.");
+
+    using _Element = remove_extent_t<_Tp>;
+    __allocator_traits_rebind_t<_Alloc, _Element> __elem_alloc(__alloc);
+    size_t __i   = 0;
+    _Tp& __array = *__loc;
+
+    // If an exception is thrown, destroy what we have constructed so far in reverse order.
+    auto __guard = std::__make_exception_guard([&]() {
+      std::__allocator_destroy_multidimensional(__elem_alloc, __array, __array + __i);
+    });
+    for (; __i != extent_v<_Tp>; ++__i) {
+      std::__allocator_construct_at_multidimensional(__elem_alloc, std::addressof(__array[__i]), __arg[__i]);
+    }
+    __guard.__complete();
+  } else {
+    allocator_traits<_Alloc>::construct(__alloc, __loc, __arg);
+  }
+}
+
+// Given a range starting at it and containing n elements, initializes each element in the
+// range from left to right using the construct method of the allocator (rebound to the
+// correct type).
+//
+// If an exception is thrown, the initialized elements are destroyed in reverse order of
+// initialization using allocator_traits destruction. If the elements in the range are C-style
+// arrays, they are initialized element-wise using allocator construction, and recursively so.
+template <class _Alloc,
+          class _BidirIter,
+          class _Tp,
+          class _Size = typename iterator_traits<_BidirIter>::difference_type>
+_LIBCPP_HIDE_FROM_ABI constexpr void
+__uninitialized_allocator_fill_n_multidimensional(_Alloc& __alloc, _BidirIter __it, _Size __n, _Tp const& __value) {
+  using _ValueType = typename iterator_traits<_BidirIter>::value_type;
+  __allocator_traits_rebind_t<_Alloc, _ValueType> __value_alloc(__alloc);
+  _BidirIter __begin = __it;
+
+  // If an exception is thrown, destroy what we have constructed so far in reverse order.
+  auto __guard =
+      std::__make_exception_guard([&]() { std::__allocator_destroy_multidimensional(__value_alloc, __begin, __it); });
+  for (; __n != 0; --__n, ++__it) {
+    std::__allocator_construct_at_multidimensional(__value_alloc, std::addressof(*__it), __value);
+  }
+  __guard.__complete();
+}
+
+// Same as __uninitialized_allocator_fill_n_multidimensional, but doesn't pass any initialization argument
+// to the allocator's construct method, which results in value initialization.
+template <class _Alloc, class _BidirIter, class _Size = typename iterator_traits<_BidirIter>::difference_type>
+_LIBCPP_HIDE_FROM_ABI constexpr void
+__uninitialized_allocator_value_construct_n_multidimensional(_Alloc& __alloc, _BidirIter __it, _Size __n) {
+  using _ValueType = typename iterator_traits<_BidirIter>::value_type;
+  __allocator_traits_rebind_t<_Alloc, _ValueType> __value_alloc(__alloc);
+  _BidirIter __begin = __it;
+
+  // If an exception is thrown, destroy what we have constructed so far in reverse order.
+  auto __guard =
+      std::__make_exception_guard([&]() { std::__allocator_destroy_multidimensional(__value_alloc, __begin, __it); });
+  for (; __n != 0; --__n, ++__it) {
+    std::__allocator_construct_at_multidimensional(__value_alloc, std::addressof(*__it));
+  }
+  __guard.__complete();
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+// Destroy all elements in [__first, __last) from left to right using allocator destruction.
+template <class _Alloc, class _Iter, class _Sent>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+__allocator_destroy(_Alloc& __alloc, _Iter __first, _Sent __last) {
+  for (; __first != __last; ++__first)
+    allocator_traits<_Alloc>::destroy(__alloc, std::__to_address(__first));
+}
+
+template <class _Alloc, class _Iter>
+class _AllocatorDestroyRangeReverse {
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+  _AllocatorDestroyRangeReverse(_Alloc& __alloc, _Iter& __first, _Iter& __last)
+      : __alloc_(__alloc), __first_(__first), __last_(__last) {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void operator()() const {
+    std::__allocator_destroy(__alloc_, std::reverse_iterator<_Iter>(__last_), std::reverse_iterator<_Iter>(__first_));
+  }
+
+private:
+  _Alloc& __alloc_;
+  _Iter& __first_;
+  _Iter& __last_;
+};
+
+// Copy-construct [__first1, __last1) in [__first2, __first2 + N), where N is distance(__first1, __last1).
+//
+// The caller has to ensure that __first2 can hold at least N uninitialized elements. If an exception is thrown the
+// already copied elements are destroyed in reverse order of their construction.
+template <class _Alloc, class _Iter1, class _Sent1, class _Iter2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Iter2
+__uninitialized_allocator_copy_impl(_Alloc& __alloc, _Iter1 __first1, _Sent1 __last1, _Iter2 __first2) {
+  auto __destruct_first = __first2;
+  auto __guard =
+      std::__make_exception_guard(_AllocatorDestroyRangeReverse<_Alloc, _Iter2>(__alloc, __destruct_first, __first2));
+  while (__first1 != __last1) {
+    allocator_traits<_Alloc>::construct(__alloc, std::__to_address(__first2), *__first1);
+    ++__first1;
+    ++__first2;
+  }
+  __guard.__complete();
+  return __first2;
+}
+
+template <class _Alloc, class _Type>
+struct __allocator_has_trivial_copy_construct : _Not<__has_construct<_Alloc, _Type*, const _Type&> > {};
+
+template <class _Type>
+struct __allocator_has_trivial_copy_construct<allocator<_Type>, _Type> : true_type {};
+
+template <class _Alloc,
+          class _In,
+          class _RawTypeIn = __remove_const_t<_In>,
+          class _Out,
+          __enable_if_t<
+              // using _RawTypeIn because of the allocator<T const> extension
+              is_trivially_copy_constructible<_RawTypeIn>::value && is_trivially_copy_assignable<_RawTypeIn>::value &&
+                  is_same<__remove_const_t<_In>, __remove_const_t<_Out> >::value &&
+                  __allocator_has_trivial_copy_construct<_Alloc, _RawTypeIn>::value,
+              int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Out*
+__uninitialized_allocator_copy_impl(_Alloc&, _In* __first1, _In* __last1, _Out* __first2) {
+  // TODO: Remove the const_cast once we drop support for std::allocator<T const>
+  if (__libcpp_is_constant_evaluated()) {
+    while (__first1 != __last1) {
+      std::__construct_at(std::__to_address(__first2), *__first1);
+      ++__first1;
+      ++__first2;
+    }
+    return __first2;
+  } else {
+    return std::copy(__first1, __last1, const_cast<_RawTypeIn*>(__first2));
+  }
+}
+
+template <class _Alloc, class _Iter1, class _Sent1, class _Iter2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Iter2
+__uninitialized_allocator_copy(_Alloc& __alloc, _Iter1 __first1, _Sent1 __last1, _Iter2 __first2) {
+  auto __unwrapped_range = std::__unwrap_range(__first1, __last1);
+  auto __result          = std::__uninitialized_allocator_copy_impl(
+      __alloc, __unwrapped_range.first, __unwrapped_range.second, std::__unwrap_iter(__first2));
+  return std::__rewrap_iter(__first2, __result);
+}
+
+template <class _Alloc, class _Type>
+struct __allocator_has_trivial_move_construct : _Not<__has_construct<_Alloc, _Type*, _Type&&> > {};
+
+template <class _Type>
+struct __allocator_has_trivial_move_construct<allocator<_Type>, _Type> : true_type {};
+
+template <class _Alloc, class _Tp>
+struct __allocator_has_trivial_destroy : _Not<__has_destroy<_Alloc, _Tp*> > {};
+
+template <class _Tp, class _Up>
+struct __allocator_has_trivial_destroy<allocator<_Tp>, _Up> : true_type {};
+
+// __uninitialized_allocator_relocate relocates the objects in [__first, __last) into __result.
+// Relocation means that the objects in [__first, __last) are placed into __result as-if by move-construct and destroy,
+// except that the move constructor and destructor may never be called if they are known to be equivalent to a memcpy.
+//
+// Preconditions:  __result doesn't contain any objects and [__first, __last) contains objects
+// Postconditions: __result contains the objects from [__first, __last) and
+//                 [__first, __last) doesn't contain any objects
+//
+// The strong exception guarantee is provided if any of the following are true:
+// - is_nothrow_move_constructible<_Tp>
+// - is_copy_constructible<_Tp>
+// - __libcpp_is_trivially_relocatable<_Tp>
+template <class _Alloc, class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+__uninitialized_allocator_relocate(_Alloc& __alloc, _Tp* __first, _Tp* __last, _Tp* __result) {
+  static_assert(__is_cpp17_move_insertable<_Alloc>::value,
+                "The specified type does not meet the requirements of Cpp17MoveInsertable");
+  if (__libcpp_is_constant_evaluated() || !__libcpp_is_trivially_relocatable<_Tp>::value ||
+      !__allocator_has_trivial_move_construct<_Alloc, _Tp>::value ||
+      !__allocator_has_trivial_destroy<_Alloc, _Tp>::value) {
+    auto __destruct_first = __result;
+    auto __guard =
+        std::__make_exception_guard(_AllocatorDestroyRangeReverse<_Alloc, _Tp*>(__alloc, __destruct_first, __result));
+    auto __iter = __first;
+    while (__iter != __last) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      allocator_traits<_Alloc>::construct(__alloc, __result, std::move_if_noexcept(*__iter));
+#else
+      allocator_traits<_Alloc>::construct(__alloc, __result, std::move(*__iter));
+#endif
+      ++__iter;
+      ++__result;
+    }
+    __guard.__complete();
+    std::__allocator_destroy(__alloc, __first, __last);
+  } else {
+    __builtin_memcpy(const_cast<__remove_const_t<_Tp>*>(__result), __first, sizeof(_Tp) * (__last - __first));
+  }
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_UNINITIALIZED_ALGORITHMS_H
diff --git a/libcxx/include/__cxx03/__memory/unique_ptr.h b/libcxx/include/__cxx03/__memory/unique_ptr.h
new file mode 100644
index 000000000000..f75259473efb
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/unique_ptr.h
@@ -0,0 +1,693 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_UNIQUE_PTR_H
+#define _LIBCPP___MEMORY_UNIQUE_PTR_H
+
+#include <__compare/compare_three_way.h>
+#include <__compare/compare_three_way_result.h>
+#include <__compare/three_way_comparable.h>
+#include <__config>
+#include <__functional/hash.h>
+#include <__functional/operations.h>
+#include <__memory/allocator_traits.h> // __pointer
+#include <__memory/auto_ptr.h>
+#include <__memory/compressed_pair.h>
+#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/common_type.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/dependent_type.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_function.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_swappable.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__type_traits/is_void.h>
+#include <__type_traits/remove_extent.h>
+#include <__type_traits/remove_pointer.h>
+#include <__type_traits/type_identity.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Ptr>
+struct __is_noexcept_deref_or_void {
+  static constexpr bool value = noexcept(*std::declval<_Ptr>());
+};
+
+template <>
+struct __is_noexcept_deref_or_void<void*> : true_type {};
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS default_delete {
+  static_assert(!is_function<_Tp>::value, "default_delete cannot be instantiated for function types");
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI constexpr default_delete() _NOEXCEPT = default;
+#else
+  _LIBCPP_HIDE_FROM_ABI default_delete() {}
+#endif
+  template <class _Up, __enable_if_t<is_convertible<_Up*, _Tp*>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 default_delete(const default_delete<_Up>&) _NOEXCEPT {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void operator()(_Tp* __ptr) const _NOEXCEPT {
+    static_assert(sizeof(_Tp) >= 0, "cannot delete an incomplete type");
+    static_assert(!is_void<_Tp>::value, "cannot delete an incomplete type");
+    delete __ptr;
+  }
+};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS default_delete<_Tp[]> {
+private:
+  template <class _Up>
+  struct _EnableIfConvertible : enable_if<is_convertible<_Up (*)[], _Tp (*)[]>::value> {};
+
+public:
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI constexpr default_delete() _NOEXCEPT = default;
+#else
+  _LIBCPP_HIDE_FROM_ABI default_delete() {}
+#endif
+
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23
+  default_delete(const default_delete<_Up[]>&, typename _EnableIfConvertible<_Up>::type* = 0) _NOEXCEPT {}
+
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 typename _EnableIfConvertible<_Up>::type
+  operator()(_Up* __ptr) const _NOEXCEPT {
+    static_assert(sizeof(_Up) >= 0, "cannot delete an incomplete type");
+    delete[] __ptr;
+  }
+};
+
+template <class _Deleter>
+struct __unique_ptr_deleter_sfinae {
+  static_assert(!is_reference<_Deleter>::value, "incorrect specialization");
+  typedef const _Deleter& __lval_ref_type;
+  typedef _Deleter&& __good_rval_ref_type;
+  typedef true_type __enable_rval_overload;
+};
+
+template <class _Deleter>
+struct __unique_ptr_deleter_sfinae<_Deleter const&> {
+  typedef const _Deleter& __lval_ref_type;
+  typedef const _Deleter&& __bad_rval_ref_type;
+  typedef false_type __enable_rval_overload;
+};
+
+template <class _Deleter>
+struct __unique_ptr_deleter_sfinae<_Deleter&> {
+  typedef _Deleter& __lval_ref_type;
+  typedef _Deleter&& __bad_rval_ref_type;
+  typedef false_type __enable_rval_overload;
+};
+
+#if defined(_LIBCPP_ABI_ENABLE_UNIQUE_PTR_TRIVIAL_ABI)
+#  define _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI __attribute__((__trivial_abi__))
+#else
+#  define _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI
+#endif
+
+template <class _Tp, class _Dp = default_delete<_Tp> >
+class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr {
+public:
+  typedef _Tp element_type;
+  typedef _Dp deleter_type;
+  typedef _LIBCPP_NODEBUG typename __pointer<_Tp, deleter_type>::type pointer;
+
+  static_assert(!is_rvalue_reference<deleter_type>::value, "the specified deleter type cannot be an rvalue reference");
+
+  // A unique_ptr contains the following members which may be trivially relocatable:
+  // - pointer : this may be trivially relocatable, so it's checked
+  // - deleter_type: this may be trivially relocatable, so it's checked
+  //
+  // This unique_ptr implementation only contains a pointer to the unique object and a deleter, so there are no
+  // references to itself. This means that the entire structure is trivially relocatable if its members are.
+  using __trivially_relocatable = __conditional_t<
+      __libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<deleter_type>::value,
+      unique_ptr,
+      void>;
+
+private:
+  __compressed_pair<pointer, deleter_type> __ptr_;
+
+  typedef _LIBCPP_NODEBUG __unique_ptr_deleter_sfinae<_Dp> _DeleterSFINAE;
+
+  template <bool _Dummy>
+  using _LValRefType _LIBCPP_NODEBUG = typename __dependent_type<_DeleterSFINAE, _Dummy>::__lval_ref_type;
+
+  template <bool _Dummy>
+  using _GoodRValRefType _LIBCPP_NODEBUG = typename __dependent_type<_DeleterSFINAE, _Dummy>::__good_rval_ref_type;
+
+  template <bool _Dummy>
+  using _BadRValRefType _LIBCPP_NODEBUG = typename __dependent_type<_DeleterSFINAE, _Dummy>::__bad_rval_ref_type;
+
+  template <bool _Dummy, class _Deleter = typename __dependent_type< __type_identity<deleter_type>, _Dummy>::type>
+  using _EnableIfDeleterDefaultConstructible _LIBCPP_NODEBUG =
+      __enable_if_t<is_default_constructible<_Deleter>::value && !is_pointer<_Deleter>::value>;
+
+  template <class _ArgType>
+  using _EnableIfDeleterConstructible _LIBCPP_NODEBUG = __enable_if_t<is_constructible<deleter_type, _ArgType>::value>;
+
+  template <class _UPtr, class _Up>
+  using _EnableIfMoveConvertible _LIBCPP_NODEBUG =
+      __enable_if_t< is_convertible<typename _UPtr::pointer, pointer>::value && !is_array<_Up>::value >;
+
+  template <class _UDel>
+  using _EnableIfDeleterConvertible _LIBCPP_NODEBUG =
+      __enable_if_t< (is_reference<_Dp>::value && is_same<_Dp, _UDel>::value) ||
+                     (!is_reference<_Dp>::value && is_convertible<_UDel, _Dp>::value) >;
+
+  template <class _UDel>
+  using _EnableIfDeleterAssignable = __enable_if_t< is_assignable<_Dp&, _UDel&&>::value >;
+
+public:
+  template <bool _Dummy = true, class = _EnableIfDeleterDefaultConstructible<_Dummy> >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR unique_ptr() _NOEXCEPT : __ptr_(__value_init_tag(), __value_init_tag()) {}
+
+  template <bool _Dummy = true, class = _EnableIfDeleterDefaultConstructible<_Dummy> >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR unique_ptr(nullptr_t) _NOEXCEPT
+      : __ptr_(__value_init_tag(), __value_init_tag()) {}
+
+  template <bool _Dummy = true, class = _EnableIfDeleterDefaultConstructible<_Dummy> >
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX23 explicit unique_ptr(pointer __p) _NOEXCEPT : __ptr_(__p, __value_init_tag()) {}
+
+  template <bool _Dummy = true, class = _EnableIfDeleterConstructible<_LValRefType<_Dummy> > >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr(pointer __p, _LValRefType<_Dummy> __d) _NOEXCEPT
+      : __ptr_(__p, __d) {}
+
+  template <bool _Dummy = true, class = _EnableIfDeleterConstructible<_GoodRValRefType<_Dummy> > >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr(pointer __p, _GoodRValRefType<_Dummy> __d) _NOEXCEPT
+      : __ptr_(__p, std::move(__d)) {
+    static_assert(!is_reference<deleter_type>::value, "rvalue deleter bound to reference");
+  }
+
+  template <bool _Dummy = true, class = _EnableIfDeleterConstructible<_BadRValRefType<_Dummy> > >
+  _LIBCPP_HIDE_FROM_ABI unique_ptr(pointer __p, _BadRValRefType<_Dummy> __d) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr(unique_ptr&& __u) _NOEXCEPT
+      : __ptr_(__u.release(), std::forward<deleter_type>(__u.get_deleter())) {}
+
+  template <class _Up,
+            class _Ep,
+            class = _EnableIfMoveConvertible<unique_ptr<_Up, _Ep>, _Up>,
+            class = _EnableIfDeleterConvertible<_Ep> >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr(unique_ptr<_Up, _Ep>&& __u) _NOEXCEPT
+      : __ptr_(__u.release(), std::forward<_Ep>(__u.get_deleter())) {}
+
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR)
+  template <class _Up,
+            __enable_if_t<is_convertible<_Up*, _Tp*>::value && is_same<_Dp, default_delete<_Tp> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI unique_ptr(auto_ptr<_Up>&& __p) _NOEXCEPT : __ptr_(__p.release(), __value_init_tag()) {}
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr& operator=(unique_ptr&& __u) _NOEXCEPT {
+    reset(__u.release());
+    __ptr_.second() = std::forward<deleter_type>(__u.get_deleter());
+    return *this;
+  }
+
+  template <class _Up,
+            class _Ep,
+            class = _EnableIfMoveConvertible<unique_ptr<_Up, _Ep>, _Up>,
+            class = _EnableIfDeleterAssignable<_Ep> >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr& operator=(unique_ptr<_Up, _Ep>&& __u) _NOEXCEPT {
+    reset(__u.release());
+    __ptr_.second() = std::forward<_Ep>(__u.get_deleter());
+    return *this;
+  }
+
+#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR)
+  template <class _Up,
+            __enable_if_t<is_convertible<_Up*, _Tp*>::value && is_same<_Dp, default_delete<_Tp> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI unique_ptr& operator=(auto_ptr<_Up> __p) {
+    reset(__p.release());
+    return *this;
+  }
+#endif
+
+#ifdef _LIBCPP_CXX03_LANG
+  unique_ptr(unique_ptr const&)            = delete;
+  unique_ptr& operator=(unique_ptr const&) = delete;
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 ~unique_ptr() { reset(); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr& operator=(nullptr_t) _NOEXCEPT {
+    reset();
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 __add_lvalue_reference_t<_Tp> operator*() const
+      _NOEXCEPT_(__is_noexcept_deref_or_void<pointer>::value) {
+    return *__ptr_.first();
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 pointer operator->() const _NOEXCEPT { return __ptr_.first(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 pointer get() const _NOEXCEPT { return __ptr_.first(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 deleter_type& get_deleter() _NOEXCEPT { return __ptr_.second(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 const deleter_type& get_deleter() const _NOEXCEPT {
+    return __ptr_.second();
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 explicit operator bool() const _NOEXCEPT {
+    return __ptr_.first() != nullptr;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 pointer release() _NOEXCEPT {
+    pointer __t    = __ptr_.first();
+    __ptr_.first() = pointer();
+    return __t;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void reset(pointer __p = pointer()) _NOEXCEPT {
+    pointer __tmp  = __ptr_.first();
+    __ptr_.first() = __p;
+    if (__tmp)
+      __ptr_.second()(__tmp);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void swap(unique_ptr& __u) _NOEXCEPT { __ptr_.swap(__u.__ptr_); }
+};
+
+template <class _Tp, class _Dp>
+class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp> {
+public:
+  typedef _Tp element_type;
+  typedef _Dp deleter_type;
+  typedef typename __pointer<_Tp, deleter_type>::type pointer;
+
+  // A unique_ptr contains the following members which may be trivially relocatable:
+  // - pointer : this may be trivially relocatable, so it's checked
+  // - deleter_type: this may be trivially relocatable, so it's checked
+  //
+  // This unique_ptr implementation only contains a pointer to the unique object and a deleter, so there are no
+  // references to itself. This means that the entire structure is trivially relocatable if its members are.
+  using __trivially_relocatable = __conditional_t<
+      __libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<deleter_type>::value,
+      unique_ptr,
+      void>;
+
+private:
+  __compressed_pair<pointer, deleter_type> __ptr_;
+
+  template <class _From>
+  struct _CheckArrayPointerConversion : is_same<_From, pointer> {};
+
+  template <class _FromElem>
+  struct _CheckArrayPointerConversion<_FromElem*>
+      : integral_constant<bool,
+                          is_same<_FromElem*, pointer>::value ||
+                              (is_same<pointer, element_type*>::value &&
+                               is_convertible<_FromElem (*)[], element_type (*)[]>::value) > {};
+
+  typedef __unique_ptr_deleter_sfinae<_Dp> _DeleterSFINAE;
+
+  template <bool _Dummy>
+  using _LValRefType _LIBCPP_NODEBUG = typename __dependent_type<_DeleterSFINAE, _Dummy>::__lval_ref_type;
+
+  template <bool _Dummy>
+  using _GoodRValRefType _LIBCPP_NODEBUG = typename __dependent_type<_DeleterSFINAE, _Dummy>::__good_rval_ref_type;
+
+  template <bool _Dummy>
+  using _BadRValRefType _LIBCPP_NODEBUG = typename __dependent_type<_DeleterSFINAE, _Dummy>::__bad_rval_ref_type;
+
+  template <bool _Dummy, class _Deleter = typename __dependent_type< __type_identity<deleter_type>, _Dummy>::type>
+  using _EnableIfDeleterDefaultConstructible _LIBCPP_NODEBUG =
+      __enable_if_t<is_default_constructible<_Deleter>::value && !is_pointer<_Deleter>::value>;
+
+  template <class _ArgType>
+  using _EnableIfDeleterConstructible _LIBCPP_NODEBUG = __enable_if_t<is_constructible<deleter_type, _ArgType>::value>;
+
+  template <class _Pp>
+  using _EnableIfPointerConvertible _LIBCPP_NODEBUG = __enable_if_t< _CheckArrayPointerConversion<_Pp>::value >;
+
+  template <class _UPtr, class _Up, class _ElemT = typename _UPtr::element_type>
+  using _EnableIfMoveConvertible _LIBCPP_NODEBUG =
+      __enable_if_t< is_array<_Up>::value && is_same<pointer, element_type*>::value &&
+                     is_same<typename _UPtr::pointer, _ElemT*>::value &&
+                     is_convertible<_ElemT (*)[], element_type (*)[]>::value >;
+
+  template <class _UDel>
+  using _EnableIfDeleterConvertible _LIBCPP_NODEBUG =
+      __enable_if_t< (is_reference<_Dp>::value && is_same<_Dp, _UDel>::value) ||
+                     (!is_reference<_Dp>::value && is_convertible<_UDel, _Dp>::value) >;
+
+  template <class _UDel>
+  using _EnableIfDeleterAssignable _LIBCPP_NODEBUG = __enable_if_t< is_assignable<_Dp&, _UDel&&>::value >;
+
+public:
+  template <bool _Dummy = true, class = _EnableIfDeleterDefaultConstructible<_Dummy> >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR unique_ptr() _NOEXCEPT : __ptr_(__value_init_tag(), __value_init_tag()) {}
+
+  template <bool _Dummy = true, class = _EnableIfDeleterDefaultConstructible<_Dummy> >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR unique_ptr(nullptr_t) _NOEXCEPT
+      : __ptr_(__value_init_tag(), __value_init_tag()) {}
+
+  template <class _Pp,
+            bool _Dummy = true,
+            class       = _EnableIfDeleterDefaultConstructible<_Dummy>,
+            class       = _EnableIfPointerConvertible<_Pp> >
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX23 explicit unique_ptr(_Pp __p) _NOEXCEPT : __ptr_(__p, __value_init_tag()) {}
+
+  template <class _Pp,
+            bool _Dummy = true,
+            class       = _EnableIfDeleterConstructible<_LValRefType<_Dummy> >,
+            class       = _EnableIfPointerConvertible<_Pp> >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr(_Pp __p, _LValRefType<_Dummy> __d) _NOEXCEPT
+      : __ptr_(__p, __d) {}
+
+  template <bool _Dummy = true, class = _EnableIfDeleterConstructible<_LValRefType<_Dummy> > >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr(nullptr_t, _LValRefType<_Dummy> __d) _NOEXCEPT
+      : __ptr_(nullptr, __d) {}
+
+  template <class _Pp,
+            bool _Dummy = true,
+            class       = _EnableIfDeleterConstructible<_GoodRValRefType<_Dummy> >,
+            class       = _EnableIfPointerConvertible<_Pp> >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr(_Pp __p, _GoodRValRefType<_Dummy> __d) _NOEXCEPT
+      : __ptr_(__p, std::move(__d)) {
+    static_assert(!is_reference<deleter_type>::value, "rvalue deleter bound to reference");
+  }
+
+  template <bool _Dummy = true, class = _EnableIfDeleterConstructible<_GoodRValRefType<_Dummy> > >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr(nullptr_t, _GoodRValRefType<_Dummy> __d) _NOEXCEPT
+      : __ptr_(nullptr, std::move(__d)) {
+    static_assert(!is_reference<deleter_type>::value, "rvalue deleter bound to reference");
+  }
+
+  template <class _Pp,
+            bool _Dummy = true,
+            class       = _EnableIfDeleterConstructible<_BadRValRefType<_Dummy> >,
+            class       = _EnableIfPointerConvertible<_Pp> >
+  _LIBCPP_HIDE_FROM_ABI unique_ptr(_Pp __p, _BadRValRefType<_Dummy> __d) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr(unique_ptr&& __u) _NOEXCEPT
+      : __ptr_(__u.release(), std::forward<deleter_type>(__u.get_deleter())) {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr& operator=(unique_ptr&& __u) _NOEXCEPT {
+    reset(__u.release());
+    __ptr_.second() = std::forward<deleter_type>(__u.get_deleter());
+    return *this;
+  }
+
+  template <class _Up,
+            class _Ep,
+            class = _EnableIfMoveConvertible<unique_ptr<_Up, _Ep>, _Up>,
+            class = _EnableIfDeleterConvertible<_Ep> >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr(unique_ptr<_Up, _Ep>&& __u) _NOEXCEPT
+      : __ptr_(__u.release(), std::forward<_Ep>(__u.get_deleter())) {}
+
+  template <class _Up,
+            class _Ep,
+            class = _EnableIfMoveConvertible<unique_ptr<_Up, _Ep>, _Up>,
+            class = _EnableIfDeleterAssignable<_Ep> >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr& operator=(unique_ptr<_Up, _Ep>&& __u) _NOEXCEPT {
+    reset(__u.release());
+    __ptr_.second() = std::forward<_Ep>(__u.get_deleter());
+    return *this;
+  }
+
+#ifdef _LIBCPP_CXX03_LANG
+  unique_ptr(unique_ptr const&)            = delete;
+  unique_ptr& operator=(unique_ptr const&) = delete;
+#endif
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 ~unique_ptr() { reset(); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr& operator=(nullptr_t) _NOEXCEPT {
+    reset();
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 __add_lvalue_reference_t<_Tp> operator[](size_t __i) const {
+    return __ptr_.first()[__i];
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 pointer get() const _NOEXCEPT { return __ptr_.first(); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 deleter_type& get_deleter() _NOEXCEPT { return __ptr_.second(); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 const deleter_type& get_deleter() const _NOEXCEPT {
+    return __ptr_.second();
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 explicit operator bool() const _NOEXCEPT {
+    return __ptr_.first() != nullptr;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 pointer release() _NOEXCEPT {
+    pointer __t    = __ptr_.first();
+    __ptr_.first() = pointer();
+    return __t;
+  }
+
+  template <class _Pp, __enable_if_t<_CheckArrayPointerConversion<_Pp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void reset(_Pp __p) _NOEXCEPT {
+    pointer __tmp  = __ptr_.first();
+    __ptr_.first() = __p;
+    if (__tmp)
+      __ptr_.second()(__tmp);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void reset(nullptr_t = nullptr) _NOEXCEPT {
+    pointer __tmp  = __ptr_.first();
+    __ptr_.first() = nullptr;
+    if (__tmp)
+      __ptr_.second()(__tmp);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void swap(unique_ptr& __u) _NOEXCEPT { __ptr_.swap(__u.__ptr_); }
+};
+
+template <class _Tp, class _Dp, __enable_if_t<__is_swappable_v<_Dp>, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void
+swap(unique_ptr<_Tp, _Dp>& __x, unique_ptr<_Tp, _Dp>& __y) _NOEXCEPT {
+  __x.swap(__y);
+}
+
+template <class _T1, class _D1, class _T2, class _D2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool
+operator==(const unique_ptr<_T1, _D1>& __x, const unique_ptr<_T2, _D2>& __y) {
+  return __x.get() == __y.get();
+}
+
+#if _LIBCPP_STD_VER <= 17
+template <class _T1, class _D1, class _T2, class _D2>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unique_ptr<_T1, _D1>& __x, const unique_ptr<_T2, _D2>& __y) {
+  return !(__x == __y);
+}
+#endif
+
+template <class _T1, class _D1, class _T2, class _D2>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(const unique_ptr<_T1, _D1>& __x, const unique_ptr<_T2, _D2>& __y) {
+  typedef typename unique_ptr<_T1, _D1>::pointer _P1;
+  typedef typename unique_ptr<_T2, _D2>::pointer _P2;
+  typedef typename common_type<_P1, _P2>::type _Vp;
+  return less<_Vp>()(__x.get(), __y.get());
+}
+
+template <class _T1, class _D1, class _T2, class _D2>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>(const unique_ptr<_T1, _D1>& __x, const unique_ptr<_T2, _D2>& __y) {
+  return __y < __x;
+}
+
+template <class _T1, class _D1, class _T2, class _D2>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const unique_ptr<_T1, _D1>& __x, const unique_ptr<_T2, _D2>& __y) {
+  return !(__y < __x);
+}
+
+template <class _T1, class _D1, class _T2, class _D2>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>=(const unique_ptr<_T1, _D1>& __x, const unique_ptr<_T2, _D2>& __y) {
+  return !(__x < __y);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _T1, class _D1, class _T2, class _D2>
+  requires three_way_comparable_with<typename unique_ptr<_T1, _D1>::pointer, typename unique_ptr<_T2, _D2>::pointer>
+_LIBCPP_HIDE_FROM_ABI
+compare_three_way_result_t<typename unique_ptr<_T1, _D1>::pointer, typename unique_ptr<_T2, _D2>::pointer>
+operator<=>(const unique_ptr<_T1, _D1>& __x, const unique_ptr<_T2, _D2>& __y) {
+  return compare_three_way()(__x.get(), __y.get());
+}
+#endif
+
+template <class _T1, class _D1>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool
+operator==(const unique_ptr<_T1, _D1>& __x, nullptr_t) _NOEXCEPT {
+  return !__x;
+}
+
+#if _LIBCPP_STD_VER <= 17
+template <class _T1, class _D1>
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(nullptr_t, const unique_ptr<_T1, _D1>& __x) _NOEXCEPT {
+  return !__x;
+}
+
+template <class _T1, class _D1>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unique_ptr<_T1, _D1>& __x, nullptr_t) _NOEXCEPT {
+  return static_cast<bool>(__x);
+}
+
+template <class _T1, class _D1>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(nullptr_t, const unique_ptr<_T1, _D1>& __x) _NOEXCEPT {
+  return static_cast<bool>(__x);
+}
+#endif // _LIBCPP_STD_VER <= 17
+
+template <class _T1, class _D1>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool operator<(const unique_ptr<_T1, _D1>& __x, nullptr_t) {
+  typedef typename unique_ptr<_T1, _D1>::pointer _P1;
+  return less<_P1>()(__x.get(), nullptr);
+}
+
+template <class _T1, class _D1>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool operator<(nullptr_t, const unique_ptr<_T1, _D1>& __x) {
+  typedef typename unique_ptr<_T1, _D1>::pointer _P1;
+  return less<_P1>()(nullptr, __x.get());
+}
+
+template <class _T1, class _D1>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool operator>(const unique_ptr<_T1, _D1>& __x, nullptr_t) {
+  return nullptr < __x;
+}
+
+template <class _T1, class _D1>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool operator>(nullptr_t, const unique_ptr<_T1, _D1>& __x) {
+  return __x < nullptr;
+}
+
+template <class _T1, class _D1>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool operator<=(const unique_ptr<_T1, _D1>& __x, nullptr_t) {
+  return !(nullptr < __x);
+}
+
+template <class _T1, class _D1>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool operator<=(nullptr_t, const unique_ptr<_T1, _D1>& __x) {
+  return !(__x < nullptr);
+}
+
+template <class _T1, class _D1>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool operator>=(const unique_ptr<_T1, _D1>& __x, nullptr_t) {
+  return !(__x < nullptr);
+}
+
+template <class _T1, class _D1>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool operator>=(nullptr_t, const unique_ptr<_T1, _D1>& __x) {
+  return !(nullptr < __x);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _T1, class _D1>
+  requires three_way_comparable< typename unique_ptr<_T1, _D1>::pointer>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 compare_three_way_result_t<typename unique_ptr<_T1, _D1>::pointer>
+operator<=>(const unique_ptr<_T1, _D1>& __x, nullptr_t) {
+  return compare_three_way()(__x.get(), static_cast<typename unique_ptr<_T1, _D1>::pointer>(nullptr));
+}
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+
+template <class _Tp>
+struct __unique_if {
+  typedef unique_ptr<_Tp> __unique_single;
+};
+
+template <class _Tp>
+struct __unique_if<_Tp[]> {
+  typedef unique_ptr<_Tp[]> __unique_array_unknown_bound;
+};
+
+template <class _Tp, size_t _Np>
+struct __unique_if<_Tp[_Np]> {
+  typedef void __unique_array_known_bound;
+};
+
+template <class _Tp, class... _Args>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 typename __unique_if<_Tp>::__unique_single
+make_unique(_Args&&... __args) {
+  return unique_ptr<_Tp>(new _Tp(std::forward<_Args>(__args)...));
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 typename __unique_if<_Tp>::__unique_array_unknown_bound
+make_unique(size_t __n) {
+  typedef __remove_extent_t<_Tp> _Up;
+  return unique_ptr<_Tp>(new _Up[__n]());
+}
+
+template <class _Tp, class... _Args>
+typename __unique_if<_Tp>::__unique_array_known_bound make_unique(_Args&&...) = delete;
+
+#endif // _LIBCPP_STD_VER >= 14
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 typename __unique_if<_Tp>::__unique_single
+make_unique_for_overwrite() {
+  return unique_ptr<_Tp>(new _Tp);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 typename __unique_if<_Tp>::__unique_array_unknown_bound
+make_unique_for_overwrite(size_t __n) {
+  return unique_ptr<_Tp>(new __remove_extent_t<_Tp>[__n]);
+}
+
+template <class _Tp, class... _Args>
+typename __unique_if<_Tp>::__unique_array_known_bound make_unique_for_overwrite(_Args&&...) = delete;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS hash;
+
+template <class _Tp, class _Dp>
+#ifdef _LIBCPP_CXX03_LANG
+struct _LIBCPP_TEMPLATE_VIS hash<unique_ptr<_Tp, _Dp> >
+#else
+struct _LIBCPP_TEMPLATE_VIS hash<__enable_hash_helper< unique_ptr<_Tp, _Dp>, typename unique_ptr<_Tp, _Dp>::pointer> >
+#endif
+{
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef unique_ptr<_Tp, _Dp> argument_type;
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef size_t result_type;
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const unique_ptr<_Tp, _Dp>& __ptr) const {
+    typedef typename unique_ptr<_Tp, _Dp>::pointer pointer;
+    return hash<pointer>()(__ptr.get());
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_UNIQUE_PTR_H
diff --git a/libcxx/include/__cxx03/__memory/uses_allocator.h b/libcxx/include/__cxx03/__memory/uses_allocator.h
new file mode 100644
index 000000000000..84310c3fa567
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/uses_allocator.h
@@ -0,0 +1,52 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_USES_ALLOCATOR_H
+#define _LIBCPP___MEMORY_USES_ALLOCATOR_H
+
+#include <__config>
+#include <__type_traits/is_convertible.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct __has_allocator_type {
+private:
+  template <class _Up>
+  static false_type __test(...);
+  template <class _Up>
+  static true_type __test(typename _Up::allocator_type* = 0);
+
+public:
+  static const bool value = decltype(__test<_Tp>(0))::value;
+};
+
+template <class _Tp, class _Alloc, bool = __has_allocator_type<_Tp>::value>
+struct __uses_allocator : public integral_constant<bool, is_convertible<_Alloc, typename _Tp::allocator_type>::value> {
+};
+
+template <class _Tp, class _Alloc>
+struct __uses_allocator<_Tp, _Alloc, false> : public false_type {};
+
+template <class _Tp, class _Alloc>
+struct _LIBCPP_TEMPLATE_VIS uses_allocator : public __uses_allocator<_Tp, _Alloc> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp, class _Alloc>
+inline constexpr bool uses_allocator_v = uses_allocator<_Tp, _Alloc>::value;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_USES_ALLOCATOR_H
diff --git a/libcxx/include/__cxx03/__memory/uses_allocator_construction.h b/libcxx/include/__cxx03/__memory/uses_allocator_construction.h
new file mode 100644
index 000000000000..5e5819d4c281
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/uses_allocator_construction.h
@@ -0,0 +1,247 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_USES_ALLOCATOR_CONSTRUCTION_H
+#define _LIBCPP___MEMORY_USES_ALLOCATOR_CONSTRUCTION_H
+
+#include <__config>
+#include <__memory/construct_at.h>
+#include <__memory/uses_allocator.h>
+#include <__tuple/tuple_like_no_subrange.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_cv.h>
+#include <__utility/declval.h>
+#include <__utility/pair.h>
+#include <tuple>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _Type>
+inline constexpr bool __is_std_pair = false;
+
+template <class _Type1, class _Type2>
+inline constexpr bool __is_std_pair<pair<_Type1, _Type2>> = true;
+
+template <class _Tp>
+inline constexpr bool __is_cv_std_pair = __is_std_pair<remove_cv_t<_Tp>>;
+
+template <class _Type, class _Alloc, class... _Args, __enable_if_t<!__is_cv_std_pair<_Type>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+__uses_allocator_construction_args(const _Alloc& __alloc, _Args&&... __args) noexcept {
+  if constexpr (!uses_allocator_v<remove_cv_t<_Type>, _Alloc> && is_constructible_v<_Type, _Args...>) {
+    return std::forward_as_tuple(std::forward<_Args>(__args)...);
+  } else if constexpr (uses_allocator_v<remove_cv_t<_Type>, _Alloc> &&
+                       is_constructible_v<_Type, allocator_arg_t, const _Alloc&, _Args...>) {
+    return tuple<allocator_arg_t, const _Alloc&, _Args&&...>(allocator_arg, __alloc, std::forward<_Args>(__args)...);
+  } else if constexpr (uses_allocator_v<remove_cv_t<_Type>, _Alloc> &&
+                       is_constructible_v<_Type, _Args..., const _Alloc&>) {
+    return std::forward_as_tuple(std::forward<_Args>(__args)..., __alloc);
+  } else {
+    static_assert(
+        sizeof(_Type) + 1 == 0, "If uses_allocator_v<Type> is true, the type has to be allocator-constructible");
+  }
+}
+
+template <class _Pair, class _Alloc, class _Tuple1, class _Tuple2, __enable_if_t<__is_cv_std_pair<_Pair>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr auto __uses_allocator_construction_args(
+    const _Alloc& __alloc, piecewise_construct_t, _Tuple1&& __x, _Tuple2&& __y) noexcept {
+  return std::make_tuple(
+      piecewise_construct,
+      std::apply(
+          [&__alloc](auto&&... __args1) {
+            return std::__uses_allocator_construction_args<typename _Pair::first_type>(
+                __alloc, std::forward<decltype(__args1)>(__args1)...);
+          },
+          std::forward<_Tuple1>(__x)),
+      std::apply(
+          [&__alloc](auto&&... __args2) {
+            return std::__uses_allocator_construction_args<typename _Pair::second_type>(
+                __alloc, std::forward<decltype(__args2)>(__args2)...);
+          },
+          std::forward<_Tuple2>(__y)));
+}
+
+template <class _Pair, class _Alloc, __enable_if_t<__is_cv_std_pair<_Pair>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr auto __uses_allocator_construction_args(const _Alloc& __alloc) noexcept {
+  return std::__uses_allocator_construction_args<_Pair>(__alloc, piecewise_construct, tuple<>{}, tuple<>{});
+}
+
+template <class _Pair, class _Alloc, class _Up, class _Vp, __enable_if_t<__is_cv_std_pair<_Pair>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+__uses_allocator_construction_args(const _Alloc& __alloc, _Up&& __u, _Vp&& __v) noexcept {
+  return std::__uses_allocator_construction_args<_Pair>(
+      __alloc,
+      piecewise_construct,
+      std::forward_as_tuple(std::forward<_Up>(__u)),
+      std::forward_as_tuple(std::forward<_Vp>(__v)));
+}
+
+#  if _LIBCPP_STD_VER >= 23
+template <class _Pair, class _Alloc, class _Up, class _Vp, __enable_if_t<__is_cv_std_pair<_Pair>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+__uses_allocator_construction_args(const _Alloc& __alloc, pair<_Up, _Vp>& __pair) noexcept {
+  return std::__uses_allocator_construction_args<_Pair>(
+      __alloc, piecewise_construct, std::forward_as_tuple(__pair.first), std::forward_as_tuple(__pair.second));
+}
+#  endif
+
+template <class _Pair, class _Alloc, class _Up, class _Vp, __enable_if_t<__is_cv_std_pair<_Pair>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+__uses_allocator_construction_args(const _Alloc& __alloc, const pair<_Up, _Vp>& __pair) noexcept {
+  return std::__uses_allocator_construction_args<_Pair>(
+      __alloc, piecewise_construct, std::forward_as_tuple(__pair.first), std::forward_as_tuple(__pair.second));
+}
+
+template <class _Pair, class _Alloc, class _Up, class _Vp, __enable_if_t<__is_cv_std_pair<_Pair>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+__uses_allocator_construction_args(const _Alloc& __alloc, pair<_Up, _Vp>&& __pair) noexcept {
+  return std::__uses_allocator_construction_args<_Pair>(
+      __alloc,
+      piecewise_construct,
+      std::forward_as_tuple(std::get<0>(std::move(__pair))),
+      std::forward_as_tuple(std::get<1>(std::move(__pair))));
+}
+
+#  if _LIBCPP_STD_VER >= 23
+template <class _Pair, class _Alloc, class _Up, class _Vp, __enable_if_t<__is_cv_std_pair<_Pair>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+__uses_allocator_construction_args(const _Alloc& __alloc, const pair<_Up, _Vp>&& __pair) noexcept {
+  return std::__uses_allocator_construction_args<_Pair>(
+      __alloc,
+      piecewise_construct,
+      std::forward_as_tuple(std::get<0>(std::move(__pair))),
+      std::forward_as_tuple(std::get<1>(std::move(__pair))));
+}
+
+template <class _Pair, class _Alloc, __pair_like_no_subrange _PairLike, __enable_if_t<__is_cv_std_pair<_Pair>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+__uses_allocator_construction_args(const _Alloc& __alloc, _PairLike&& __p) noexcept {
+  return std::__uses_allocator_construction_args<_Pair>(
+      __alloc,
+      piecewise_construct,
+      std::forward_as_tuple(std::get<0>(std::forward<_PairLike>(__p))),
+      std::forward_as_tuple(std::get<1>(std::forward<_PairLike>(__p))));
+}
+#  endif
+
+namespace __uses_allocator_detail {
+
+template <class _Ap, class _Bp>
+void __fun(const pair<_Ap, _Bp>&);
+
+template <class _Tp>
+decltype(__uses_allocator_detail::__fun(std::declval<_Tp>()), true_type()) __convertible_to_const_pair_ref_impl(int);
+
+template <class>
+false_type __convertible_to_const_pair_ref_impl(...);
+
+template <class _Tp>
+inline constexpr bool __convertible_to_const_pair_ref =
+    decltype(__uses_allocator_detail::__convertible_to_const_pair_ref_impl<_Tp>(0))::value;
+
+#  if _LIBCPP_STD_VER >= 23
+template <class _Tp, class _Up>
+inline constexpr bool __uses_allocator_constraints =
+    __is_cv_std_pair<_Tp> && !__pair_like_no_subrange<_Up> && !__convertible_to_const_pair_ref<_Up>;
+#  else
+template <class _Tp, class _Up>
+inline constexpr bool __uses_allocator_constraints = __is_cv_std_pair<_Tp> && !__convertible_to_const_pair_ref<_Up>;
+#  endif
+
+} // namespace __uses_allocator_detail
+
+template < class _Pair,
+           class _Alloc,
+           class _Type,
+           __enable_if_t<__uses_allocator_detail::__uses_allocator_constraints<_Pair, _Type>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+__uses_allocator_construction_args(const _Alloc& __alloc, _Type&& __value) noexcept;
+
+template <class _Type, class _Alloc, class... _Args>
+_LIBCPP_HIDE_FROM_ABI constexpr _Type __make_obj_using_allocator(const _Alloc& __alloc, _Args&&... __args);
+
+template < class _Pair,
+           class _Alloc,
+           class _Type,
+           __enable_if_t< __uses_allocator_detail::__uses_allocator_constraints<_Pair, _Type>, int>>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+__uses_allocator_construction_args(const _Alloc& __alloc, _Type&& __value) noexcept {
+  struct __pair_constructor {
+    using _PairMutable = remove_cv_t<_Pair>;
+
+    _LIBCPP_HIDDEN constexpr auto __do_construct(const _PairMutable& __pair) const {
+      return std::__make_obj_using_allocator<_PairMutable>(__alloc_, __pair);
+    }
+
+    _LIBCPP_HIDDEN constexpr auto __do_construct(_PairMutable&& __pair) const {
+      return std::__make_obj_using_allocator<_PairMutable>(__alloc_, std::move(__pair));
+    }
+
+    const _Alloc& __alloc_;
+    _Type& __value_;
+
+    _LIBCPP_HIDDEN constexpr operator _PairMutable() const { return __do_construct(std::forward<_Type>(__value_)); }
+  };
+
+  return std::make_tuple(__pair_constructor{__alloc, __value});
+}
+
+template <class _Type, class _Alloc, class... _Args>
+_LIBCPP_HIDE_FROM_ABI constexpr _Type __make_obj_using_allocator(const _Alloc& __alloc, _Args&&... __args) {
+  return std::make_from_tuple<_Type>(
+      std::__uses_allocator_construction_args<_Type>(__alloc, std::forward<_Args>(__args)...));
+}
+
+template <class _Type, class _Alloc, class... _Args>
+_LIBCPP_HIDE_FROM_ABI constexpr _Type*
+__uninitialized_construct_using_allocator(_Type* __ptr, const _Alloc& __alloc, _Args&&... __args) {
+  return std::apply(
+      [&__ptr](auto&&... __xs) { return std::__construct_at(__ptr, std::forward<decltype(__xs)>(__xs)...); },
+      std::__uses_allocator_construction_args<_Type>(__alloc, std::forward<_Args>(__args)...));
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Type, class _Alloc, class... _Args>
+_LIBCPP_HIDE_FROM_ABI constexpr auto uses_allocator_construction_args(const _Alloc& __alloc, _Args&&... __args) noexcept
+    -> decltype(std::__uses_allocator_construction_args<_Type>(__alloc, std::forward<_Args>(__args)...)) {
+  return /*--*/ std::__uses_allocator_construction_args<_Type>(__alloc, std::forward<_Args>(__args)...);
+}
+
+template <class _Type, class _Alloc, class... _Args>
+_LIBCPP_HIDE_FROM_ABI constexpr auto make_obj_using_allocator(const _Alloc& __alloc, _Args&&... __args)
+    -> decltype(std::__make_obj_using_allocator<_Type>(__alloc, std::forward<_Args>(__args)...)) {
+  return /*--*/ std::__make_obj_using_allocator<_Type>(__alloc, std::forward<_Args>(__args)...);
+}
+
+template <class _Type, class _Alloc, class... _Args>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+uninitialized_construct_using_allocator(_Type* __ptr, const _Alloc& __alloc, _Args&&... __args)
+    -> decltype(std::__uninitialized_construct_using_allocator(__ptr, __alloc, std::forward<_Args>(__args)...)) {
+  return /*--*/ std::__uninitialized_construct_using_allocator(__ptr, __alloc, std::forward<_Args>(__args)...);
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_USES_ALLOCATOR_CONSTRUCTION_H
diff --git a/libcxx/include/__cxx03/__memory/voidify.h b/libcxx/include/__cxx03/__memory/voidify.h
new file mode 100644
index 000000000000..dbd083bd8c1e
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory/voidify.h
@@ -0,0 +1,30 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_VOIDIFY_H
+#define _LIBCPP___MEMORY_VOIDIFY_H
+
+#include <__config>
+#include <__memory/addressof.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <typename _Tp>
+_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void* __voidify(_Tp& __from) {
+  // Cast away cv-qualifiers to allow modifying elements of a range through const iterators.
+  return const_cast<void*>(static_cast<const volatile void*>(std::addressof(__from)));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_VOIDIFY_H
diff --git a/libcxx/include/__cxx03/__memory_resource/memory_resource.h b/libcxx/include/__cxx03/__memory_resource/memory_resource.h
new file mode 100644
index 000000000000..ea85e50cd568
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory_resource/memory_resource.h
@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_RESOURCE_MEMORY_RESOURCE_H
+#define _LIBCPP___MEMORY_RESOURCE_MEMORY_RESOURCE_H
+
+#include <__config>
+#include <__fwd/memory_resource.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace pmr {
+
+// [mem.res.class]
+
+class _LIBCPP_AVAILABILITY_PMR _LIBCPP_EXPORTED_FROM_ABI memory_resource {
+  static const size_t __max_align = alignof(max_align_t);
+
+public:
+  virtual ~memory_resource();
+
+  [[nodiscard]] [[using __gnu__: __returns_nonnull__, __alloc_size__(2), __alloc_align__(3)]]
+  _LIBCPP_HIDE_FROM_ABI void* allocate(size_t __bytes, size_t __align = __max_align) {
+    return do_allocate(__bytes, __align);
+  }
+
+  [[__gnu__::__nonnull__]] _LIBCPP_HIDE_FROM_ABI void
+  deallocate(void* __p, size_t __bytes, size_t __align = __max_align) {
+    do_deallocate(__p, __bytes, __align);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool is_equal(const memory_resource& __other) const noexcept { return do_is_equal(__other); }
+
+private:
+  virtual void* do_allocate(size_t, size_t)                       = 0;
+  virtual void do_deallocate(void*, size_t, size_t)               = 0;
+  virtual bool do_is_equal(memory_resource const&) const noexcept = 0;
+};
+
+// [mem.res.eq]
+
+inline _LIBCPP_AVAILABILITY_PMR _LIBCPP_HIDE_FROM_ABI bool
+operator==(const memory_resource& __lhs, const memory_resource& __rhs) noexcept {
+  return &__lhs == &__rhs || __lhs.is_equal(__rhs);
+}
+
+#  if _LIBCPP_STD_VER <= 17
+
+inline _LIBCPP_AVAILABILITY_PMR _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const memory_resource& __lhs, const memory_resource& __rhs) noexcept {
+  return !(__lhs == __rhs);
+}
+
+#  endif
+
+// [mem.res.global]
+
+[[__gnu__::__returns_nonnull__]] _LIBCPP_AVAILABILITY_PMR _LIBCPP_EXPORTED_FROM_ABI memory_resource*
+get_default_resource() noexcept;
+
+[[__gnu__::__returns_nonnull__]] _LIBCPP_AVAILABILITY_PMR _LIBCPP_EXPORTED_FROM_ABI memory_resource*
+set_default_resource(memory_resource*) noexcept;
+
+[[using __gnu__: __returns_nonnull__, __const__]] _LIBCPP_AVAILABILITY_PMR _LIBCPP_EXPORTED_FROM_ABI memory_resource*
+new_delete_resource() noexcept;
+
+[[using __gnu__: __returns_nonnull__, __const__]] _LIBCPP_AVAILABILITY_PMR _LIBCPP_EXPORTED_FROM_ABI memory_resource*
+null_memory_resource() noexcept;
+
+} // namespace pmr
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___MEMORY_RESOURCE_MEMORY_RESOURCE_H
diff --git a/libcxx/include/__cxx03/__memory_resource/monotonic_buffer_resource.h b/libcxx/include/__cxx03/__memory_resource/monotonic_buffer_resource.h
new file mode 100644
index 000000000000..f45b30fdb386
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory_resource/monotonic_buffer_resource.h
@@ -0,0 +1,122 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_RESOURCE_MONOTONIC_BUFFER_RESOURCE_H
+#define _LIBCPP___MEMORY_RESOURCE_MONOTONIC_BUFFER_RESOURCE_H
+
+#include <__config>
+#include <__memory/addressof.h>
+#include <__memory_resource/memory_resource.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace pmr {
+
+// [mem.res.monotonic.buffer]
+
+class _LIBCPP_AVAILABILITY_PMR _LIBCPP_EXPORTED_FROM_ABI monotonic_buffer_resource : public memory_resource {
+  static const size_t __default_buffer_capacity  = 1024;
+  static const size_t __default_buffer_alignment = 16;
+
+  struct __chunk_footer {
+    __chunk_footer* __next_;
+    char* __start_;
+    char* __cur_;
+    size_t __align_;
+    _LIBCPP_HIDE_FROM_ABI size_t __allocation_size() {
+      return (reinterpret_cast<char*>(this) - __start_) + sizeof(*this);
+    }
+    void* __try_allocate_from_chunk(size_t, size_t);
+  };
+
+  struct __initial_descriptor {
+    char* __start_;
+    char* __cur_;
+    union {
+      char* __end_;
+      size_t __size_;
+    };
+    void* __try_allocate_from_chunk(size_t, size_t);
+  };
+
+public:
+  _LIBCPP_HIDE_FROM_ABI monotonic_buffer_resource()
+      : monotonic_buffer_resource(nullptr, __default_buffer_capacity, get_default_resource()) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit monotonic_buffer_resource(size_t __initial_size)
+      : monotonic_buffer_resource(nullptr, __initial_size, get_default_resource()) {}
+
+  _LIBCPP_HIDE_FROM_ABI monotonic_buffer_resource(void* __buffer, size_t __buffer_size)
+      : monotonic_buffer_resource(__buffer, __buffer_size, get_default_resource()) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit monotonic_buffer_resource(memory_resource* __upstream)
+      : monotonic_buffer_resource(nullptr, __default_buffer_capacity, __upstream) {}
+
+  _LIBCPP_HIDE_FROM_ABI monotonic_buffer_resource(size_t __initial_size, memory_resource* __upstream)
+      : monotonic_buffer_resource(nullptr, __initial_size, __upstream) {}
+
+  _LIBCPP_HIDE_FROM_ABI monotonic_buffer_resource(void* __buffer, size_t __buffer_size, memory_resource* __upstream)
+      : __res_(__upstream) {
+    __initial_.__start_ = static_cast<char*>(__buffer);
+    if (__buffer != nullptr) {
+      __initial_.__cur_ = static_cast<char*>(__buffer) + __buffer_size;
+      __initial_.__end_ = static_cast<char*>(__buffer) + __buffer_size;
+    } else {
+      __initial_.__cur_  = nullptr;
+      __initial_.__size_ = __buffer_size;
+    }
+    __chunks_ = nullptr;
+  }
+
+  monotonic_buffer_resource(const monotonic_buffer_resource&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~monotonic_buffer_resource() override { release(); }
+
+  monotonic_buffer_resource& operator=(const monotonic_buffer_resource&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI void release() {
+    if (__initial_.__start_ != nullptr)
+      __initial_.__cur_ = __initial_.__end_;
+    while (__chunks_ != nullptr) {
+      __chunk_footer* __next = __chunks_->__next_;
+      __res_->deallocate(__chunks_->__start_, __chunks_->__allocation_size(), __chunks_->__align_);
+      __chunks_ = __next;
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI memory_resource* upstream_resource() const { return __res_; }
+
+protected:
+  void* do_allocate(size_t __bytes, size_t __alignment) override; // key function
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void do_deallocate(void*, size_t, size_t) override {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL bool do_is_equal(const memory_resource& __other) const _NOEXCEPT override {
+    return this == std::addressof(__other);
+  }
+
+private:
+  __initial_descriptor __initial_;
+  __chunk_footer* __chunks_;
+  memory_resource* __res_;
+};
+
+} // namespace pmr
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___MEMORY_RESOURCE_MONOTONIC_BUFFER_RESOURCE_H
diff --git a/libcxx/include/__cxx03/__memory_resource/polymorphic_allocator.h b/libcxx/include/__cxx03/__memory_resource/polymorphic_allocator.h
new file mode 100644
index 000000000000..a71096d3e478
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory_resource/polymorphic_allocator.h
@@ -0,0 +1,231 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_RESOURCE_POLYMORPHIC_ALLOCATOR_H
+#define _LIBCPP___MEMORY_RESOURCE_POLYMORPHIC_ALLOCATOR_H
+
+#include <__assert>
+#include <__config>
+#include <__fwd/pair.h>
+#include <__memory_resource/memory_resource.h>
+#include <__utility/exception_guard.h>
+#include <cstddef>
+#include <limits>
+#include <new>
+#include <tuple>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace pmr {
+
+// [mem.poly.allocator.class]
+
+template <class _ValueType
+#  if _LIBCPP_STD_VER >= 20
+          = byte
+#  endif
+          >
+class _LIBCPP_AVAILABILITY_PMR _LIBCPP_TEMPLATE_VIS polymorphic_allocator {
+
+public:
+  using value_type = _ValueType;
+
+  // [mem.poly.allocator.ctor]
+
+  _LIBCPP_HIDE_FROM_ABI polymorphic_allocator() noexcept : __res_(std::pmr::get_default_resource()) {}
+
+  _LIBCPP_HIDE_FROM_ABI polymorphic_allocator(memory_resource* __r) noexcept : __res_(__r) {}
+
+  _LIBCPP_HIDE_FROM_ABI polymorphic_allocator(const polymorphic_allocator&) = default;
+
+  template <class _Tp>
+  _LIBCPP_HIDE_FROM_ABI polymorphic_allocator(const polymorphic_allocator<_Tp>& __other) noexcept
+      : __res_(__other.resource()) {}
+
+  polymorphic_allocator& operator=(const polymorphic_allocator&) = delete;
+
+  // [mem.poly.allocator.mem]
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI _ValueType* allocate(size_t __n) {
+    if (__n > __max_size()) {
+      __throw_bad_array_new_length();
+    }
+    return static_cast<_ValueType*>(__res_->allocate(__n * sizeof(_ValueType), alignof(_ValueType)));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void deallocate(_ValueType* __p, size_t __n) {
+    _LIBCPP_ASSERT_VALID_DEALLOCATION(
+        __n <= __max_size(),
+        "deallocate() called for a size which exceeds max_size(), leading to a memory leak "
+        "(the argument will overflow and result in too few objects being deleted)");
+    __res_->deallocate(__p, __n * sizeof(_ValueType), alignof(_ValueType));
+  }
+
+#  if _LIBCPP_STD_VER >= 20
+
+  [[nodiscard]] [[using __gnu__: __alloc_size__(2), __alloc_align__(3)]] _LIBCPP_HIDE_FROM_ABI void*
+  allocate_bytes(size_t __nbytes, size_t __alignment = alignof(max_align_t)) {
+    return __res_->allocate(__nbytes, __alignment);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void deallocate_bytes(void* __ptr, size_t __nbytes, size_t __alignment = alignof(max_align_t)) {
+    __res_->deallocate(__ptr, __nbytes, __alignment);
+  }
+
+  template <class _Type>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI _Type* allocate_object(size_t __n = 1) {
+    if (numeric_limits<size_t>::max() / sizeof(_Type) < __n)
+      std::__throw_bad_array_new_length();
+    return static_cast<_Type*>(allocate_bytes(__n * sizeof(_Type), alignof(_Type)));
+  }
+
+  template <class _Type>
+  _LIBCPP_HIDE_FROM_ABI void deallocate_object(_Type* __ptr, size_t __n = 1) {
+    deallocate_bytes(__ptr, __n * sizeof(_Type), alignof(_Type));
+  }
+
+  template <class _Type, class... _CtorArgs>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI _Type* new_object(_CtorArgs&&... __ctor_args) {
+    _Type* __ptr = allocate_object<_Type>();
+    auto __guard = std::__make_exception_guard([&] { deallocate_object(__ptr); });
+    construct(__ptr, std::forward<_CtorArgs>(__ctor_args)...);
+    __guard.__complete();
+    return __ptr;
+  }
+
+  template <class _Type>
+  _LIBCPP_HIDE_FROM_ABI void delete_object(_Type* __ptr) {
+    destroy(__ptr);
+    deallocate_object(__ptr);
+  }
+
+#  endif // _LIBCPP_STD_VER >= 20
+
+  template <class _Tp, class... _Ts>
+  _LIBCPP_HIDE_FROM_ABI void construct(_Tp* __p, _Ts&&... __args) {
+    std::__user_alloc_construct_impl(
+        typename __uses_alloc_ctor<_Tp, polymorphic_allocator&, _Ts...>::type(),
+        __p,
+        *this,
+        std::forward<_Ts>(__args)...);
+  }
+
+  template <class _T1, class _T2, class... _Args1, class... _Args2>
+  _LIBCPP_HIDE_FROM_ABI void
+  construct(pair<_T1, _T2>* __p, piecewise_construct_t, tuple<_Args1...> __x, tuple<_Args2...> __y) {
+    ::new ((void*)__p) pair<_T1, _T2>(
+        piecewise_construct,
+        __transform_tuple(typename __uses_alloc_ctor< _T1, polymorphic_allocator&, _Args1... >::type(),
+                          std::move(__x),
+                          typename __make_tuple_indices<sizeof...(_Args1)>::type{}),
+        __transform_tuple(typename __uses_alloc_ctor< _T2, polymorphic_allocator&, _Args2... >::type(),
+                          std::move(__y),
+                          typename __make_tuple_indices<sizeof...(_Args2)>::type{}));
+  }
+
+  template <class _T1, class _T2>
+  _LIBCPP_HIDE_FROM_ABI void construct(pair<_T1, _T2>* __p) {
+    construct(__p, piecewise_construct, tuple<>(), tuple<>());
+  }
+
+  template <class _T1, class _T2, class _Up, class _Vp>
+  _LIBCPP_HIDE_FROM_ABI void construct(pair<_T1, _T2>* __p, _Up&& __u, _Vp&& __v) {
+    construct(__p,
+              piecewise_construct,
+              std::forward_as_tuple(std::forward<_Up>(__u)),
+              std::forward_as_tuple(std::forward<_Vp>(__v)));
+  }
+
+  template <class _T1, class _T2, class _U1, class _U2>
+  _LIBCPP_HIDE_FROM_ABI void construct(pair<_T1, _T2>* __p, const pair<_U1, _U2>& __pr) {
+    construct(__p, piecewise_construct, std::forward_as_tuple(__pr.first), std::forward_as_tuple(__pr.second));
+  }
+
+  template <class _T1, class _T2, class _U1, class _U2>
+  _LIBCPP_HIDE_FROM_ABI void construct(pair<_T1, _T2>* __p, pair<_U1, _U2>&& __pr) {
+    construct(__p,
+              piecewise_construct,
+              std::forward_as_tuple(std::forward<_U1>(__pr.first)),
+              std::forward_as_tuple(std::forward<_U2>(__pr.second)));
+  }
+
+  template <class _Tp>
+  _LIBCPP_HIDE_FROM_ABI void destroy(_Tp* __p) {
+    __p->~_Tp();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI polymorphic_allocator select_on_container_copy_construction() const noexcept {
+    return polymorphic_allocator();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI memory_resource* resource() const noexcept { return __res_; }
+
+private:
+  template <class... _Args, size_t... _Is>
+  _LIBCPP_HIDE_FROM_ABI tuple<_Args&&...>
+  __transform_tuple(integral_constant<int, 0>, tuple<_Args...>&& __t, __tuple_indices<_Is...>) {
+    return std::forward_as_tuple(std::get<_Is>(std::move(__t))...);
+  }
+
+  template <class... _Args, size_t... _Is>
+  _LIBCPP_HIDE_FROM_ABI tuple<allocator_arg_t const&, polymorphic_allocator&, _Args&&...>
+  __transform_tuple(integral_constant<int, 1>, tuple<_Args...>&& __t, __tuple_indices<_Is...>) {
+    using _Tup = tuple<allocator_arg_t const&, polymorphic_allocator&, _Args&&...>;
+    return _Tup(allocator_arg, *this, std::get<_Is>(std::move(__t))...);
+  }
+
+  template <class... _Args, size_t... _Is>
+  _LIBCPP_HIDE_FROM_ABI tuple<_Args&&..., polymorphic_allocator&>
+  __transform_tuple(integral_constant<int, 2>, tuple<_Args...>&& __t, __tuple_indices<_Is...>) {
+    using _Tup = tuple<_Args&&..., polymorphic_allocator&>;
+    return _Tup(std::get<_Is>(std::move(__t))..., *this);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_t __max_size() const noexcept {
+    return numeric_limits<size_t>::max() / sizeof(value_type);
+  }
+
+  memory_resource* __res_;
+};
+
+// [mem.poly.allocator.eq]
+
+template <class _Tp, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const polymorphic_allocator<_Tp>& __lhs, const polymorphic_allocator<_Up>& __rhs) noexcept {
+  return *__lhs.resource() == *__rhs.resource();
+}
+
+#  if _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const polymorphic_allocator<_Tp>& __lhs, const polymorphic_allocator<_Up>& __rhs) noexcept {
+  return !(__lhs == __rhs);
+}
+
+#  endif
+
+} // namespace pmr
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_RESOURCE_POLYMORPHIC_ALLOCATOR_H
diff --git a/libcxx/include/__cxx03/__memory_resource/pool_options.h b/libcxx/include/__cxx03/__memory_resource/pool_options.h
new file mode 100644
index 000000000000..442959836c7e
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory_resource/pool_options.h
@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_RESOURCE_POOL_OPTIONS_H
+#define _LIBCPP___MEMORY_RESOURCE_POOL_OPTIONS_H
+
+#include <__config>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace pmr {
+
+// [mem.res.pool.options]
+
+struct _LIBCPP_EXPORTED_FROM_ABI pool_options {
+  size_t max_blocks_per_chunk        = 0;
+  size_t largest_required_pool_block = 0;
+};
+
+} // namespace pmr
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___MEMORY_RESOURCE_POOL_OPTIONS_H
diff --git a/libcxx/include/__cxx03/__memory_resource/synchronized_pool_resource.h b/libcxx/include/__cxx03/__memory_resource/synchronized_pool_resource.h
new file mode 100644
index 000000000000..50a673c2861d
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory_resource/synchronized_pool_resource.h
@@ -0,0 +1,92 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_RESOURCE_SYNCHRONIZED_POOL_RESOURCE_H
+#define _LIBCPP___MEMORY_RESOURCE_SYNCHRONIZED_POOL_RESOURCE_H
+
+#include <__config>
+#include <__memory_resource/memory_resource.h>
+#include <__memory_resource/pool_options.h>
+#include <__memory_resource/unsynchronized_pool_resource.h>
+#include <cstddef>
+#include <mutex>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace pmr {
+
+// [mem.res.pool.overview]
+
+class _LIBCPP_AVAILABILITY_PMR _LIBCPP_EXPORTED_FROM_ABI synchronized_pool_resource : public memory_resource {
+public:
+  _LIBCPP_HIDE_FROM_ABI synchronized_pool_resource(const pool_options& __opts, memory_resource* __upstream)
+      : __unsync_(__opts, __upstream) {}
+
+  _LIBCPP_HIDE_FROM_ABI synchronized_pool_resource()
+      : synchronized_pool_resource(pool_options(), get_default_resource()) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit synchronized_pool_resource(memory_resource* __upstream)
+      : synchronized_pool_resource(pool_options(), __upstream) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit synchronized_pool_resource(const pool_options& __opts)
+      : synchronized_pool_resource(__opts, get_default_resource()) {}
+
+  synchronized_pool_resource(const synchronized_pool_resource&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~synchronized_pool_resource() override = default;
+
+  synchronized_pool_resource& operator=(const synchronized_pool_resource&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI void release() {
+#  if !defined(_LIBCPP_HAS_NO_THREADS)
+    unique_lock<mutex> __lk(__mut_);
+#  endif
+    __unsync_.release();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI memory_resource* upstream_resource() const { return __unsync_.upstream_resource(); }
+
+  _LIBCPP_HIDE_FROM_ABI pool_options options() const { return __unsync_.options(); }
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void* do_allocate(size_t __bytes, size_t __align) override {
+#  if !defined(_LIBCPP_HAS_NO_THREADS)
+    unique_lock<mutex> __lk(__mut_);
+#  endif
+    return __unsync_.allocate(__bytes, __align);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void do_deallocate(void* __p, size_t __bytes, size_t __align) override {
+#  if !defined(_LIBCPP_HAS_NO_THREADS)
+    unique_lock<mutex> __lk(__mut_);
+#  endif
+    return __unsync_.deallocate(__p, __bytes, __align);
+  }
+
+  bool do_is_equal(const memory_resource& __other) const noexcept override; // key function
+
+private:
+#  if !defined(_LIBCPP_HAS_NO_THREADS)
+  mutex __mut_;
+#  endif
+  unsynchronized_pool_resource __unsync_;
+};
+
+} // namespace pmr
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___MEMORY_RESOURCE_SYNCHRONIZED_POOL_RESOURCE_H
diff --git a/libcxx/include/__cxx03/__memory_resource/unsynchronized_pool_resource.h b/libcxx/include/__cxx03/__memory_resource/unsynchronized_pool_resource.h
new file mode 100644
index 000000000000..783db84262af
--- /dev/null
+++ b/libcxx/include/__cxx03/__memory_resource/unsynchronized_pool_resource.h
@@ -0,0 +1,106 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_RESOURCE_UNSYNCHRONIZED_POOL_RESOURCE_H
+#define _LIBCPP___MEMORY_RESOURCE_UNSYNCHRONIZED_POOL_RESOURCE_H
+
+#include <__config>
+#include <__memory_resource/memory_resource.h>
+#include <__memory_resource/pool_options.h>
+#include <cstddef>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace pmr {
+
+// [mem.res.pool.overview]
+
+class _LIBCPP_AVAILABILITY_PMR _LIBCPP_EXPORTED_FROM_ABI unsynchronized_pool_resource : public memory_resource {
+  class __fixed_pool;
+
+  class __adhoc_pool {
+    struct __chunk_footer;
+    __chunk_footer* __first_;
+
+  public:
+    _LIBCPP_HIDE_FROM_ABI explicit __adhoc_pool() : __first_(nullptr) {}
+
+    void __release_ptr(memory_resource* __upstream);
+    void* __do_allocate(memory_resource* __upstream, size_t __bytes, size_t __align);
+    void __do_deallocate(memory_resource* __upstream, void* __p, size_t __bytes, size_t __align);
+  };
+
+  static const size_t __min_blocks_per_chunk = 16;
+  static const size_t __min_bytes_per_chunk  = 1024;
+  static const size_t __max_blocks_per_chunk = (size_t(1) << 20);
+  static const size_t __max_bytes_per_chunk  = (size_t(1) << 30);
+
+  static const int __log2_smallest_block_size      = 3;
+  static const size_t __smallest_block_size        = 8;
+  static const size_t __default_largest_block_size = (size_t(1) << 20);
+  static const size_t __max_largest_block_size     = (size_t(1) << 30);
+
+  size_t __pool_block_size(int __i) const;
+  int __log2_pool_block_size(int __i) const;
+  int __pool_index(size_t __bytes, size_t __align) const;
+
+public:
+  unsynchronized_pool_resource(const pool_options& __opts, memory_resource* __upstream);
+
+  _LIBCPP_HIDE_FROM_ABI unsynchronized_pool_resource()
+      : unsynchronized_pool_resource(pool_options(), get_default_resource()) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit unsynchronized_pool_resource(memory_resource* __upstream)
+      : unsynchronized_pool_resource(pool_options(), __upstream) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit unsynchronized_pool_resource(const pool_options& __opts)
+      : unsynchronized_pool_resource(__opts, get_default_resource()) {}
+
+  unsynchronized_pool_resource(const unsynchronized_pool_resource&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~unsynchronized_pool_resource() override { release(); }
+
+  unsynchronized_pool_resource& operator=(const unsynchronized_pool_resource&) = delete;
+
+  void release();
+
+  _LIBCPP_HIDE_FROM_ABI memory_resource* upstream_resource() const { return __res_; }
+
+  [[__gnu__::__pure__]] pool_options options() const;
+
+protected:
+  void* do_allocate(size_t __bytes, size_t __align) override; // key function
+
+  void do_deallocate(void* __p, size_t __bytes, size_t __align) override;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL bool do_is_equal(const memory_resource& __other) const _NOEXCEPT override {
+    return &__other == this;
+  }
+
+private:
+  memory_resource* __res_;
+  __adhoc_pool __adhoc_pool_;
+  __fixed_pool* __fixed_pools_;
+  int __num_fixed_pools_;
+  uint32_t __options_max_blocks_per_chunk_;
+};
+
+} // namespace pmr
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___MEMORY_RESOURCE_UNSYNCHRONIZED_POOL_RESOURCE_H
diff --git a/libcxx/include/__cxx03/__mutex/lock_guard.h b/libcxx/include/__cxx03/__mutex/lock_guard.h
new file mode 100644
index 000000000000..ef56896be9f6
--- /dev/null
+++ b/libcxx/include/__cxx03/__mutex/lock_guard.h
@@ -0,0 +1,48 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MUTEX_LOCK_GUARD_H
+#define _LIBCPP___MUTEX_LOCK_GUARD_H
+
+#include <__config>
+#include <__mutex/tag_types.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Mutex>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_THREAD_SAFETY_ANNOTATION(scoped_lockable) lock_guard {
+public:
+  typedef _Mutex mutex_type;
+
+private:
+  mutex_type& __m_;
+
+public:
+  _LIBCPP_NODISCARD
+  _LIBCPP_HIDE_FROM_ABI explicit lock_guard(mutex_type& __m) _LIBCPP_THREAD_SAFETY_ANNOTATION(acquire_capability(__m))
+      : __m_(__m) {
+    __m_.lock();
+  }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI lock_guard(mutex_type& __m, adopt_lock_t)
+      _LIBCPP_THREAD_SAFETY_ANNOTATION(requires_capability(__m))
+      : __m_(__m) {}
+  _LIBCPP_HIDE_FROM_ABI ~lock_guard() _LIBCPP_THREAD_SAFETY_ANNOTATION(release_capability()) { __m_.unlock(); }
+
+  lock_guard(lock_guard const&)            = delete;
+  lock_guard& operator=(lock_guard const&) = delete;
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(lock_guard);
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MUTEX_LOCK_GUARD_H
diff --git a/libcxx/include/__cxx03/__mutex/mutex.h b/libcxx/include/__cxx03/__mutex/mutex.h
new file mode 100644
index 000000000000..1ed01547126f
--- /dev/null
+++ b/libcxx/include/__cxx03/__mutex/mutex.h
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MUTEX_MUTEX_H
+#define _LIBCPP___MUTEX_MUTEX_H
+
+#include <__config>
+#include <__thread/support.h>
+#include <__type_traits/is_nothrow_constructible.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#ifndef _LIBCPP_HAS_NO_THREADS
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_THREAD_SAFETY_ANNOTATION(capability("mutex")) mutex {
+  __libcpp_mutex_t __m_ = _LIBCPP_MUTEX_INITIALIZER;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR mutex() = default;
+
+  mutex(const mutex&)            = delete;
+  mutex& operator=(const mutex&) = delete;
+
+#  if defined(_LIBCPP_HAS_TRIVIAL_MUTEX_DESTRUCTION)
+  _LIBCPP_HIDE_FROM_ABI ~mutex() = default;
+#  else
+  ~mutex() _NOEXCEPT;
+#  endif
+
+  void lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(acquire_capability());
+  bool try_lock() _NOEXCEPT _LIBCPP_THREAD_SAFETY_ANNOTATION(try_acquire_capability(true));
+  void unlock() _NOEXCEPT _LIBCPP_THREAD_SAFETY_ANNOTATION(release_capability());
+
+  typedef __libcpp_mutex_t* native_handle_type;
+  _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; }
+};
+
+static_assert(is_nothrow_default_constructible<mutex>::value, "the default constructor for std::mutex must be nothrow");
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_NO_THREADS
+
+#endif // _LIBCPP___MUTEX_MUTEX_H
diff --git a/libcxx/include/__cxx03/__mutex/once_flag.h b/libcxx/include/__cxx03/__mutex/once_flag.h
new file mode 100644
index 000000000000..9d7baecbc708
--- /dev/null
+++ b/libcxx/include/__cxx03/__mutex/once_flag.h
@@ -0,0 +1,159 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MUTEX_ONCE_FLAG_H
+#define _LIBCPP___MUTEX_ONCE_FLAG_H
+
+#include <__config>
+#include <__functional/invoke.h>
+#include <__memory/shared_ptr.h> // __libcpp_acquire_load
+#include <__tuple/tuple_indices.h>
+#include <__tuple/tuple_size.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <cstdint>
+#ifndef _LIBCPP_CXX03_LANG
+#  include <tuple>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct _LIBCPP_TEMPLATE_VIS once_flag;
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Callable, class... _Args>
+_LIBCPP_HIDE_FROM_ABI void call_once(once_flag&, _Callable&&, _Args&&...);
+
+#else // _LIBCPP_CXX03_LANG
+
+template <class _Callable>
+_LIBCPP_HIDE_FROM_ABI void call_once(once_flag&, _Callable&);
+
+template <class _Callable>
+_LIBCPP_HIDE_FROM_ABI void call_once(once_flag&, const _Callable&);
+
+#endif // _LIBCPP_CXX03_LANG
+
+struct _LIBCPP_TEMPLATE_VIS once_flag {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR once_flag() _NOEXCEPT : __state_(_Unset) {}
+  once_flag(const once_flag&)            = delete;
+  once_flag& operator=(const once_flag&) = delete;
+
+#if defined(_LIBCPP_ABI_MICROSOFT)
+  typedef uintptr_t _State_type;
+#else
+  typedef unsigned long _State_type;
+#endif
+
+  static const _State_type _Unset    = 0;
+  static const _State_type _Pending  = 1;
+  static const _State_type _Complete = ~_State_type(0);
+
+private:
+  _State_type __state_;
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class _Callable, class... _Args>
+  friend void call_once(once_flag&, _Callable&&, _Args&&...);
+#else  // _LIBCPP_CXX03_LANG
+  template <class _Callable>
+  friend void call_once(once_flag&, _Callable&);
+
+  template <class _Callable>
+  friend void call_once(once_flag&, const _Callable&);
+#endif // _LIBCPP_CXX03_LANG
+};
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Fp>
+class __call_once_param {
+  _Fp& __f_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __call_once_param(_Fp& __f) : __f_(__f) {}
+
+  _LIBCPP_HIDE_FROM_ABI void operator()() {
+    typedef typename __make_tuple_indices<tuple_size<_Fp>::value, 1>::type _Index;
+    __execute(_Index());
+  }
+
+private:
+  template <size_t... _Indices>
+  _LIBCPP_HIDE_FROM_ABI void __execute(__tuple_indices<_Indices...>) {
+    std::__invoke(std::get<0>(std::move(__f_)), std::get<_Indices>(std::move(__f_))...);
+  }
+};
+
+#else
+
+template <class _Fp>
+class __call_once_param {
+  _Fp& __f_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __call_once_param(_Fp& __f) : __f_(__f) {}
+
+  _LIBCPP_HIDE_FROM_ABI void operator()() { __f_(); }
+};
+
+#endif
+
+template <class _Fp>
+void _LIBCPP_HIDE_FROM_ABI __call_once_proxy(void* __vp) {
+  __call_once_param<_Fp>* __p = static_cast<__call_once_param<_Fp>*>(__vp);
+  (*__p)();
+}
+
+_LIBCPP_EXPORTED_FROM_ABI void __call_once(volatile once_flag::_State_type&, void*, void (*)(void*));
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Callable, class... _Args>
+inline _LIBCPP_HIDE_FROM_ABI void call_once(once_flag& __flag, _Callable&& __func, _Args&&... __args) {
+  if (__libcpp_acquire_load(&__flag.__state_) != once_flag::_Complete) {
+    typedef tuple<_Callable&&, _Args&&...> _Gp;
+    _Gp __f(std::forward<_Callable>(__func), std::forward<_Args>(__args)...);
+    __call_once_param<_Gp> __p(__f);
+    std::__call_once(__flag.__state_, &__p, &__call_once_proxy<_Gp>);
+  }
+}
+
+#else // _LIBCPP_CXX03_LANG
+
+template <class _Callable>
+inline _LIBCPP_HIDE_FROM_ABI void call_once(once_flag& __flag, _Callable& __func) {
+  if (__libcpp_acquire_load(&__flag.__state_) != once_flag::_Complete) {
+    __call_once_param<_Callable> __p(__func);
+    std::__call_once(__flag.__state_, &__p, &__call_once_proxy<_Callable>);
+  }
+}
+
+template <class _Callable>
+inline _LIBCPP_HIDE_FROM_ABI void call_once(once_flag& __flag, const _Callable& __func) {
+  if (__libcpp_acquire_load(&__flag.__state_) != once_flag::_Complete) {
+    __call_once_param<const _Callable> __p(__func);
+    std::__call_once(__flag.__state_, &__p, &__call_once_proxy<const _Callable>);
+  }
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MUTEX_ONCE_FLAG_H
diff --git a/libcxx/include/__cxx03/__mutex/tag_types.h b/libcxx/include/__cxx03/__mutex/tag_types.h
new file mode 100644
index 000000000000..2b2dd58ee4e8
--- /dev/null
+++ b/libcxx/include/__cxx03/__mutex/tag_types.h
@@ -0,0 +1,44 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MUTEX_TAG_TYPES_H
+#define _LIBCPP___MUTEX_TAG_TYPES_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct _LIBCPP_EXPORTED_FROM_ABI defer_lock_t {
+  explicit defer_lock_t() = default;
+};
+
+struct _LIBCPP_EXPORTED_FROM_ABI try_to_lock_t {
+  explicit try_to_lock_t() = default;
+};
+
+struct _LIBCPP_EXPORTED_FROM_ABI adopt_lock_t {
+  explicit adopt_lock_t() = default;
+};
+
+#if _LIBCPP_STD_VER >= 17
+inline constexpr defer_lock_t defer_lock   = defer_lock_t();
+inline constexpr try_to_lock_t try_to_lock = try_to_lock_t();
+inline constexpr adopt_lock_t adopt_lock   = adopt_lock_t();
+#elif !defined(_LIBCPP_CXX03_LANG)
+constexpr defer_lock_t defer_lock   = defer_lock_t();
+constexpr try_to_lock_t try_to_lock = try_to_lock_t();
+constexpr adopt_lock_t adopt_lock   = adopt_lock_t();
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MUTEX_TAG_TYPES_H
diff --git a/libcxx/include/__cxx03/__mutex/unique_lock.h b/libcxx/include/__cxx03/__mutex/unique_lock.h
new file mode 100644
index 000000000000..4a616ba51ee1
--- /dev/null
+++ b/libcxx/include/__cxx03/__mutex/unique_lock.h
@@ -0,0 +1,177 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MUTEX_UNIQUE_LOCK_H
+#define _LIBCPP___MUTEX_UNIQUE_LOCK_H
+
+#include <__chrono/duration.h>
+#include <__chrono/time_point.h>
+#include <__config>
+#include <__memory/addressof.h>
+#include <__mutex/tag_types.h>
+#include <__system_error/system_error.h>
+#include <__utility/swap.h>
+#include <cerrno>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#ifndef _LIBCPP_HAS_NO_THREADS
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Mutex>
+class _LIBCPP_TEMPLATE_VIS unique_lock {
+public:
+  typedef _Mutex mutex_type;
+
+private:
+  mutex_type* __m_;
+  bool __owns_;
+
+public:
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI unique_lock() _NOEXCEPT : __m_(nullptr), __owns_(false) {}
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI explicit unique_lock(mutex_type& __m)
+      : __m_(std::addressof(__m)), __owns_(true) {
+    __m_->lock();
+  }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI unique_lock(mutex_type& __m, defer_lock_t) _NOEXCEPT
+      : __m_(std::addressof(__m)),
+        __owns_(false) {}
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI unique_lock(mutex_type& __m, try_to_lock_t)
+      : __m_(std::addressof(__m)), __owns_(__m.try_lock()) {}
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI unique_lock(mutex_type& __m, adopt_lock_t)
+      : __m_(std::addressof(__m)), __owns_(true) {}
+
+  template <class _Clock, class _Duration>
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI unique_lock(mutex_type& __m, const chrono::time_point<_Clock, _Duration>& __t)
+      : __m_(std::addressof(__m)), __owns_(__m.try_lock_until(__t)) {}
+
+  template <class _Rep, class _Period>
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI unique_lock(mutex_type& __m, const chrono::duration<_Rep, _Period>& __d)
+      : __m_(std::addressof(__m)), __owns_(__m.try_lock_for(__d)) {}
+
+  _LIBCPP_HIDE_FROM_ABI ~unique_lock() {
+    if (__owns_)
+      __m_->unlock();
+  }
+
+  unique_lock(unique_lock const&)            = delete;
+  unique_lock& operator=(unique_lock const&) = delete;
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI unique_lock(unique_lock&& __u) _NOEXCEPT
+      : __m_(__u.__m_),
+        __owns_(__u.__owns_) {
+    __u.__m_    = nullptr;
+    __u.__owns_ = false;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI unique_lock& operator=(unique_lock&& __u) _NOEXCEPT {
+    if (__owns_)
+      __m_->unlock();
+
+    __m_        = __u.__m_;
+    __owns_     = __u.__owns_;
+    __u.__m_    = nullptr;
+    __u.__owns_ = false;
+    return *this;
+  }
+
+  void lock();
+  bool try_lock();
+
+  template <class _Rep, class _Period>
+  bool try_lock_for(const chrono::duration<_Rep, _Period>& __d);
+
+  template <class _Clock, class _Duration>
+  bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __t);
+
+  void unlock();
+
+  _LIBCPP_HIDE_FROM_ABI void swap(unique_lock& __u) _NOEXCEPT {
+    std::swap(__m_, __u.__m_);
+    std::swap(__owns_, __u.__owns_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI mutex_type* release() _NOEXCEPT {
+    mutex_type* __m = __m_;
+    __m_            = nullptr;
+    __owns_         = false;
+    return __m;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool owns_lock() const _NOEXCEPT { return __owns_; }
+  _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return __owns_; }
+  _LIBCPP_HIDE_FROM_ABI mutex_type* mutex() const _NOEXCEPT { return __m_; }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(unique_lock);
+
+template <class _Mutex>
+void unique_lock<_Mutex>::lock() {
+  if (__m_ == nullptr)
+    __throw_system_error(EPERM, "unique_lock::lock: references null mutex");
+  if (__owns_)
+    __throw_system_error(EDEADLK, "unique_lock::lock: already locked");
+  __m_->lock();
+  __owns_ = true;
+}
+
+template <class _Mutex>
+bool unique_lock<_Mutex>::try_lock() {
+  if (__m_ == nullptr)
+    __throw_system_error(EPERM, "unique_lock::try_lock: references null mutex");
+  if (__owns_)
+    __throw_system_error(EDEADLK, "unique_lock::try_lock: already locked");
+  __owns_ = __m_->try_lock();
+  return __owns_;
+}
+
+template <class _Mutex>
+template <class _Rep, class _Period>
+bool unique_lock<_Mutex>::try_lock_for(const chrono::duration<_Rep, _Period>& __d) {
+  if (__m_ == nullptr)
+    __throw_system_error(EPERM, "unique_lock::try_lock_for: references null mutex");
+  if (__owns_)
+    __throw_system_error(EDEADLK, "unique_lock::try_lock_for: already locked");
+  __owns_ = __m_->try_lock_for(__d);
+  return __owns_;
+}
+
+template <class _Mutex>
+template <class _Clock, class _Duration>
+bool unique_lock<_Mutex>::try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) {
+  if (__m_ == nullptr)
+    __throw_system_error(EPERM, "unique_lock::try_lock_until: references null mutex");
+  if (__owns_)
+    __throw_system_error(EDEADLK, "unique_lock::try_lock_until: already locked");
+  __owns_ = __m_->try_lock_until(__t);
+  return __owns_;
+}
+
+template <class _Mutex>
+void unique_lock<_Mutex>::unlock() {
+  if (!__owns_)
+    __throw_system_error(EPERM, "unique_lock::unlock: not locked");
+  __m_->unlock();
+  __owns_ = false;
+}
+
+template <class _Mutex>
+inline _LIBCPP_HIDE_FROM_ABI void swap(unique_lock<_Mutex>& __x, unique_lock<_Mutex>& __y) _NOEXCEPT {
+  __x.swap(__y);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_NO_THREADS
+
+#endif // _LIBCPP___MUTEX_UNIQUE_LOCK_H
diff --git a/libcxx/include/__cxx03/__node_handle b/libcxx/include/__cxx03/__node_handle
new file mode 100644
index 000000000000..d0b35bfd1934
--- /dev/null
+++ b/libcxx/include/__cxx03/__node_handle
@@ -0,0 +1,209 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NODE_HANDLE
+#define _LIBCPP___NODE_HANDLE
+
+/*
+
+template<unspecified>
+class node-handle {
+public:
+  using value_type     = see below;     // not present for map containers
+  using key_type       = see below;     // not present for set containers
+  using mapped_type    = see below;     // not present for set containers
+  using allocator_type = see below;
+
+private:
+  using container_node_type = unspecified;                  // exposition only
+  using ator_traits = allocator_traits<allocator_type>;     // exposition only
+
+  typename ator_traits::template
+    rebind_traits<container_node_type>::pointer ptr_;       // exposition only
+  optional<allocator_type> alloc_;                          // exposition only
+
+public:
+  // [container.node.cons], constructors, copy, and assignment
+  constexpr node-handle() noexcept : ptr_(), alloc_() {}
+  node-handle(node-handle&&) noexcept;
+  node-handle& operator=(node-handle&&);
+
+  // [container.node.dtor], destructor
+  ~node-handle();
+
+  // [container.node.observers], observers
+  value_type& value() const;            // not present for map containers
+  key_type& key() const;                // not present for set containers
+  mapped_type& mapped() const;          // not present for set containers
+
+  allocator_type get_allocator() const;
+  explicit operator bool() const noexcept;
+  [[nodiscard]] bool empty() const noexcept; // nodiscard since C++20
+
+  // [container.node.modifiers], modifiers
+  void swap(node-handle&)
+    noexcept(ator_traits::propagate_on_container_swap::value ||
+             ator_traits::is_always_equal::value);
+
+  friend void swap(node-handle& x, node-handle& y) noexcept(noexcept(x.swap(y))) {
+    x.swap(y);
+  }
+};
+
+*/
+
+#include <__assert>
+#include <__config>
+#include <__memory/allocator_traits.h>
+#include <__memory/pointer_traits.h>
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+// Specialized in __tree & __hash_table for their _NodeType.
+template <class _NodeType, class _Alloc>
+struct __generic_container_node_destructor;
+
+template <class _NodeType, class _Alloc, template <class, class> class _MapOrSetSpecifics>
+class _LIBCPP_TEMPLATE_VIS __basic_node_handle
+    : public _MapOrSetSpecifics< _NodeType, __basic_node_handle<_NodeType, _Alloc, _MapOrSetSpecifics>> {
+  template <class _Tp, class _Compare, class _Allocator>
+  friend class __tree;
+  template <class _Tp, class _Hash, class _Equal, class _Allocator>
+  friend class __hash_table;
+  friend struct _MapOrSetSpecifics< _NodeType, __basic_node_handle<_NodeType, _Alloc, _MapOrSetSpecifics>>;
+
+  typedef allocator_traits<_Alloc> __alloc_traits;
+  typedef __rebind_pointer_t<typename __alloc_traits::void_pointer, _NodeType> __node_pointer_type;
+
+public:
+  typedef _Alloc allocator_type;
+
+private:
+  __node_pointer_type __ptr_ = nullptr;
+  optional<allocator_type> __alloc_;
+
+  _LIBCPP_HIDE_FROM_ABI void __release_ptr() {
+    __ptr_   = nullptr;
+    __alloc_ = std::nullopt;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __destroy_node_pointer() {
+    if (__ptr_ != nullptr) {
+      typedef typename __allocator_traits_rebind< allocator_type, _NodeType>::type __node_alloc_type;
+      __node_alloc_type __alloc(*__alloc_);
+      __generic_container_node_destructor<_NodeType, __node_alloc_type>(__alloc, true)(__ptr_);
+      __ptr_ = nullptr;
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __basic_node_handle(__node_pointer_type __ptr, allocator_type const& __alloc)
+      : __ptr_(__ptr), __alloc_(__alloc) {}
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __basic_node_handle() = default;
+
+  _LIBCPP_HIDE_FROM_ABI __basic_node_handle(__basic_node_handle&& __other) noexcept
+      : __ptr_(__other.__ptr_), __alloc_(std::move(__other.__alloc_)) {
+    __other.__ptr_   = nullptr;
+    __other.__alloc_ = std::nullopt;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __basic_node_handle& operator=(__basic_node_handle&& __other) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __alloc_ == std::nullopt || __alloc_traits::propagate_on_container_move_assignment::value ||
+            __alloc_ == __other.__alloc_,
+        "node_type with incompatible allocator passed to "
+        "node_type::operator=(node_type&&)");
+
+    __destroy_node_pointer();
+    __ptr_ = __other.__ptr_;
+
+    if (__alloc_traits::propagate_on_container_move_assignment::value || __alloc_ == std::nullopt)
+      __alloc_ = std::move(__other.__alloc_);
+
+    __other.__ptr_   = nullptr;
+    __other.__alloc_ = std::nullopt;
+
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const { return *__alloc_; }
+
+  _LIBCPP_HIDE_FROM_ABI explicit operator bool() const { return __ptr_ != nullptr; }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool empty() const { return __ptr_ == nullptr; }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(__basic_node_handle& __other) noexcept(
+      __alloc_traits::propagate_on_container_swap::value || __alloc_traits::is_always_equal::value) {
+    using std::swap;
+    swap(__ptr_, __other.__ptr_);
+    if (__alloc_traits::propagate_on_container_swap::value || __alloc_ == std::nullopt ||
+        __other.__alloc_ == std::nullopt)
+      swap(__alloc_, __other.__alloc_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend void
+  swap(__basic_node_handle& __a, __basic_node_handle& __b) noexcept(noexcept(__a.swap(__b))) {
+    __a.swap(__b);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~__basic_node_handle() { __destroy_node_pointer(); }
+};
+
+template <class _NodeType, class _Derived>
+struct __set_node_handle_specifics {
+  typedef typename _NodeType::__node_value_type value_type;
+
+  _LIBCPP_HIDE_FROM_ABI value_type& value() const { return static_cast<_Derived const*>(this)->__ptr_->__get_value(); }
+};
+
+template <class _NodeType, class _Derived>
+struct __map_node_handle_specifics {
+  typedef typename _NodeType::__node_value_type::key_type key_type;
+  typedef typename _NodeType::__node_value_type::mapped_type mapped_type;
+
+  _LIBCPP_HIDE_FROM_ABI key_type& key() const {
+    return static_cast<_Derived const*>(this)->__ptr_->__get_value().__ref().first;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI mapped_type& mapped() const {
+    return static_cast<_Derived const*>(this)->__ptr_->__get_value().__ref().second;
+  }
+};
+
+template <class _NodeType, class _Alloc>
+using __set_node_handle = __basic_node_handle< _NodeType, _Alloc, __set_node_handle_specifics>;
+
+template <class _NodeType, class _Alloc>
+using __map_node_handle = __basic_node_handle< _NodeType, _Alloc, __map_node_handle_specifics>;
+
+template <class _Iterator, class _NodeType>
+struct _LIBCPP_TEMPLATE_VIS __insert_return_type {
+  _Iterator position;
+  bool inserted;
+  _NodeType node;
+};
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___NODE_HANDLE
diff --git a/libcxx/include/__cxx03/__numeric/accumulate.h b/libcxx/include/__cxx03/__numeric/accumulate.h
new file mode 100644
index 000000000000..fedc1c46dfd0
--- /dev/null
+++ b/libcxx/include/__cxx03/__numeric/accumulate.h
@@ -0,0 +1,53 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NUMERIC_ACCUMULATE_H
+#define _LIBCPP___NUMERIC_ACCUMULATE_H
+
+#include <__config>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp
+accumulate(_InputIterator __first, _InputIterator __last, _Tp __init) {
+  for (; __first != __last; ++__first)
+#if _LIBCPP_STD_VER >= 20
+    __init = std::move(__init) + *__first;
+#else
+    __init = __init + *__first;
+#endif
+  return __init;
+}
+
+template <class _InputIterator, class _Tp, class _BinaryOperation>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp
+accumulate(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op) {
+  for (; __first != __last; ++__first)
+#if _LIBCPP_STD_VER >= 20
+    __init = __binary_op(std::move(__init), *__first);
+#else
+    __init = __binary_op(__init, *__first);
+#endif
+  return __init;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___NUMERIC_ACCUMULATE_H
diff --git a/libcxx/include/__cxx03/__numeric/adjacent_difference.h b/libcxx/include/__cxx03/__numeric/adjacent_difference.h
new file mode 100644
index 000000000000..62b53342d9a4
--- /dev/null
+++ b/libcxx/include/__cxx03/__numeric/adjacent_difference.h
@@ -0,0 +1,68 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NUMERIC_ADJACENT_DIFFERENCE_H
+#define _LIBCPP___NUMERIC_ADJACENT_DIFFERENCE_H
+
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _OutputIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+adjacent_difference(_InputIterator __first, _InputIterator __last, _OutputIterator __result) {
+  if (__first != __last) {
+    typename iterator_traits<_InputIterator>::value_type __acc(*__first);
+    *__result = __acc;
+    for (++__first, (void)++__result; __first != __last; ++__first, (void)++__result) {
+      typename iterator_traits<_InputIterator>::value_type __val(*__first);
+#if _LIBCPP_STD_VER >= 20
+      *__result = __val - std::move(__acc);
+#else
+      *__result = __val - __acc;
+#endif
+      __acc = std::move(__val);
+    }
+  }
+  return __result;
+}
+
+template <class _InputIterator, class _OutputIterator, class _BinaryOperation>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator adjacent_difference(
+    _InputIterator __first, _InputIterator __last, _OutputIterator __result, _BinaryOperation __binary_op) {
+  if (__first != __last) {
+    typename iterator_traits<_InputIterator>::value_type __acc(*__first);
+    *__result = __acc;
+    for (++__first, (void)++__result; __first != __last; ++__first, (void)++__result) {
+      typename iterator_traits<_InputIterator>::value_type __val(*__first);
+#if _LIBCPP_STD_VER >= 20
+      *__result = __binary_op(__val, std::move(__acc));
+#else
+      *__result = __binary_op(__val, __acc);
+#endif
+      __acc = std::move(__val);
+    }
+  }
+  return __result;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___NUMERIC_ADJACENT_DIFFERENCE_H
diff --git a/libcxx/include/__cxx03/__numeric/exclusive_scan.h b/libcxx/include/__cxx03/__numeric/exclusive_scan.h
new file mode 100644
index 000000000000..af85b477dfe6
--- /dev/null
+++ b/libcxx/include/__cxx03/__numeric/exclusive_scan.h
@@ -0,0 +1,58 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NUMERIC_EXCLUSIVE_SCAN_H
+#define _LIBCPP___NUMERIC_EXCLUSIVE_SCAN_H
+
+#include <__config>
+#include <__functional/operations.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+exclusive_scan(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _Tp __init, _BinaryOp __b) {
+  if (__first != __last) {
+    _Tp __tmp(__b(__init, *__first));
+    while (true) {
+      *__result = std::move(__init);
+      ++__result;
+      ++__first;
+      if (__first == __last)
+        break;
+      __init = std::move(__tmp);
+      __tmp  = __b(__init, *__first);
+    }
+  }
+  return __result;
+}
+
+template <class _InputIterator, class _OutputIterator, class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+exclusive_scan(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _Tp __init) {
+  return std::exclusive_scan(__first, __last, __result, __init, std::plus<>());
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___NUMERIC_EXCLUSIVE_SCAN_H
diff --git a/libcxx/include/__cxx03/__numeric/gcd_lcm.h b/libcxx/include/__cxx03/__numeric/gcd_lcm.h
new file mode 100644
index 000000000000..9be6cf8516b1
--- /dev/null
+++ b/libcxx/include/__cxx03/__numeric/gcd_lcm.h
@@ -0,0 +1,132 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NUMERIC_GCD_LCM_H
+#define _LIBCPP___NUMERIC_GCD_LCM_H
+
+#include <__algorithm/min.h>
+#include <__assert>
+#include <__bit/countr.h>
+#include <__config>
+#include <__type_traits/common_type.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_signed.h>
+#include <__type_traits/make_unsigned.h>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <typename _Result, typename _Source, bool _IsSigned = is_signed<_Source>::value>
+struct __ct_abs;
+
+template <typename _Result, typename _Source>
+struct __ct_abs<_Result, _Source, true> {
+  constexpr _LIBCPP_HIDE_FROM_ABI _Result operator()(_Source __t) const noexcept {
+    if (__t >= 0)
+      return __t;
+    if (__t == numeric_limits<_Source>::min())
+      return -static_cast<_Result>(__t);
+    return -__t;
+  }
+};
+
+template <typename _Result, typename _Source>
+struct __ct_abs<_Result, _Source, false> {
+  constexpr _LIBCPP_HIDE_FROM_ABI _Result operator()(_Source __t) const noexcept { return __t; }
+};
+
+template <class _Tp>
+constexpr _LIBCPP_HIDDEN _Tp __gcd(_Tp __a, _Tp __b) {
+  static_assert(!is_signed<_Tp>::value, "");
+
+  // From: https://lemire.me/blog/2013/12/26/fastest-way-to-compute-the-greatest-common-divisor
+  //
+  // If power of two divides both numbers, we can push it out.
+  // - gcd( 2^x * a, 2^x * b) = 2^x * gcd(a, b)
+  //
+  // If and only if exactly one number is even, we can divide that number by that power.
+  // - if a, b are odd, then gcd(2^x * a, b) = gcd(a, b)
+  //
+  // And standard gcd algorithm where instead of modulo, minus is used.
+
+  if (__a < __b) {
+    _Tp __tmp = __b;
+    __b       = __a;
+    __a       = __tmp;
+  }
+  if (__b == 0)
+    return __a;
+  __a %= __b; // Make both argument of the same size, and early result in the easy case.
+  if (__a == 0)
+    return __b;
+
+  int __az    = std::__countr_zero(__a);
+  int __bz    = std::__countr_zero(__b);
+  int __shift = std::min(__az, __bz);
+  __a >>= __az;
+  __b >>= __bz;
+  do {
+    _Tp __diff = __a - __b;
+    if (__a > __b) {
+      __a = __b;
+      __b = __diff;
+    } else {
+      __b = __b - __a;
+    }
+    if (__diff != 0)
+      __b >>= std::__countr_zero(__diff);
+  } while (__b != 0);
+  return __a << __shift;
+}
+
+template <class _Tp, class _Up>
+constexpr _LIBCPP_HIDE_FROM_ABI common_type_t<_Tp, _Up> gcd(_Tp __m, _Up __n) {
+  static_assert(is_integral<_Tp>::value && is_integral<_Up>::value, "Arguments to gcd must be integer types");
+  static_assert(!is_same<__remove_cv_t<_Tp>, bool>::value, "First argument to gcd cannot be bool");
+  static_assert(!is_same<__remove_cv_t<_Up>, bool>::value, "Second argument to gcd cannot be bool");
+  using _Rp = common_type_t<_Tp, _Up>;
+  using _Wp = make_unsigned_t<_Rp>;
+  return static_cast<_Rp>(
+      std::__gcd(static_cast<_Wp>(__ct_abs<_Rp, _Tp>()(__m)), static_cast<_Wp>(__ct_abs<_Rp, _Up>()(__n))));
+}
+
+template <class _Tp, class _Up>
+constexpr _LIBCPP_HIDE_FROM_ABI common_type_t<_Tp, _Up> lcm(_Tp __m, _Up __n) {
+  static_assert(is_integral<_Tp>::value && is_integral<_Up>::value, "Arguments to lcm must be integer types");
+  static_assert(!is_same<__remove_cv_t<_Tp>, bool>::value, "First argument to lcm cannot be bool");
+  static_assert(!is_same<__remove_cv_t<_Up>, bool>::value, "Second argument to lcm cannot be bool");
+  if (__m == 0 || __n == 0)
+    return 0;
+
+  using _Rp  = common_type_t<_Tp, _Up>;
+  _Rp __val1 = __ct_abs<_Rp, _Tp>()(__m) / std::gcd(__m, __n);
+  _Rp __val2 = __ct_abs<_Rp, _Up>()(__n);
+  _Rp __res;
+  [[maybe_unused]] bool __overflow = __builtin_mul_overflow(__val1, __val2, &__res);
+  _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(!__overflow, "Overflow in lcm");
+  return __res;
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___NUMERIC_GCD_LCM_H
diff --git a/libcxx/include/__cxx03/__numeric/inclusive_scan.h b/libcxx/include/__cxx03/__numeric/inclusive_scan.h
new file mode 100644
index 000000000000..d714f102d74e
--- /dev/null
+++ b/libcxx/include/__cxx03/__numeric/inclusive_scan.h
@@ -0,0 +1,59 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NUMERIC_INCLUSIVE_SCAN_H
+#define _LIBCPP___NUMERIC_INCLUSIVE_SCAN_H
+
+#include <__config>
+#include <__functional/operations.h>
+#include <__iterator/iterator_traits.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+inclusive_scan(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _BinaryOp __b, _Tp __init) {
+  for (; __first != __last; ++__first, (void)++__result) {
+    __init    = __b(__init, *__first);
+    *__result = __init;
+  }
+  return __result;
+}
+
+template <class _InputIterator, class _OutputIterator, class _BinaryOp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+inclusive_scan(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _BinaryOp __b) {
+  if (__first != __last) {
+    typename iterator_traits<_InputIterator>::value_type __init = *__first;
+    *__result++                                                 = __init;
+    if (++__first != __last)
+      return std::inclusive_scan(__first, __last, __result, __b, __init);
+  }
+
+  return __result;
+}
+
+template <class _InputIterator, class _OutputIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+inclusive_scan(_InputIterator __first, _InputIterator __last, _OutputIterator __result) {
+  return std::inclusive_scan(__first, __last, __result, std::plus<>());
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___NUMERIC_INCLUSIVE_SCAN_H
diff --git a/libcxx/include/__cxx03/__numeric/inner_product.h b/libcxx/include/__cxx03/__numeric/inner_product.h
new file mode 100644
index 000000000000..0deab3d421b7
--- /dev/null
+++ b/libcxx/include/__cxx03/__numeric/inner_product.h
@@ -0,0 +1,58 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NUMERIC_INNER_PRODUCT_H
+#define _LIBCPP___NUMERIC_INNER_PRODUCT_H
+
+#include <__config>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator1, class _InputIterator2, class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp
+inner_product(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _Tp __init) {
+  for (; __first1 != __last1; ++__first1, (void)++__first2)
+#if _LIBCPP_STD_VER >= 20
+    __init = std::move(__init) + *__first1 * *__first2;
+#else
+    __init = __init + *__first1 * *__first2;
+#endif
+  return __init;
+}
+
+template <class _InputIterator1, class _InputIterator2, class _Tp, class _BinaryOperation1, class _BinaryOperation2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp inner_product(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _Tp __init,
+    _BinaryOperation1 __binary_op1,
+    _BinaryOperation2 __binary_op2) {
+  for (; __first1 != __last1; ++__first1, (void)++__first2)
+#if _LIBCPP_STD_VER >= 20
+    __init = __binary_op1(std::move(__init), __binary_op2(*__first1, *__first2));
+#else
+    __init = __binary_op1(__init, __binary_op2(*__first1, *__first2));
+#endif
+  return __init;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___NUMERIC_INNER_PRODUCT_H
diff --git a/libcxx/include/__cxx03/__numeric/iota.h b/libcxx/include/__cxx03/__numeric/iota.h
new file mode 100644
index 000000000000..27bd84e395a3
--- /dev/null
+++ b/libcxx/include/__cxx03/__numeric/iota.h
@@ -0,0 +1,30 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NUMERIC_IOTA_H
+#define _LIBCPP___NUMERIC_IOTA_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ForwardIterator, class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+iota(_ForwardIterator __first, _ForwardIterator __last, _Tp __value) {
+  for (; __first != __last; ++__first, (void)++__value)
+    *__first = __value;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___NUMERIC_IOTA_H
diff --git a/libcxx/include/__cxx03/__numeric/midpoint.h b/libcxx/include/__cxx03/__numeric/midpoint.h
new file mode 100644
index 000000000000..5ef30d4ec50f
--- /dev/null
+++ b/libcxx/include/__cxx03/__numeric/midpoint.h
@@ -0,0 +1,88 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NUMERIC_MIDPOINT_H
+#define _LIBCPP___NUMERIC_MIDPOINT_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_floating_point.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_null_pointer.h>
+#include <__type_traits/is_object.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_void.h>
+#include <__type_traits/make_unsigned.h>
+#include <__type_traits/remove_pointer.h>
+#include <cstddef>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<is_integral_v<_Tp> && !is_same_v<bool, _Tp> && !is_null_pointer_v<_Tp>, _Tp>
+midpoint(_Tp __a, _Tp __b) noexcept _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK {
+  using _Up                = make_unsigned_t<_Tp>;
+  constexpr _Up __bitshift = numeric_limits<_Up>::digits - 1;
+
+  _Up __diff     = _Up(__b) - _Up(__a);
+  _Up __sign_bit = __b < __a;
+
+  _Up __half_diff = (__diff / 2) + (__sign_bit << __bitshift) + (__sign_bit & __diff);
+
+  return __a + __half_diff;
+}
+
+template <class _Tp, enable_if_t<is_object_v<_Tp> && !is_void_v<_Tp> && (sizeof(_Tp) > 0), int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp* midpoint(_Tp* __a, _Tp* __b) noexcept {
+  return __a + std::midpoint(ptrdiff_t(0), __b - __a);
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr int __sign(_Tp __val) {
+  return (_Tp(0) < __val) - (__val < _Tp(0));
+}
+
+template <typename _Fp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Fp __fp_abs(_Fp __f) {
+  return __f >= 0 ? __f : -__f;
+}
+
+template <class _Fp>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<is_floating_point_v<_Fp>, _Fp> midpoint(_Fp __a, _Fp __b) noexcept {
+  constexpr _Fp __lo = numeric_limits<_Fp>::min() * 2;
+  constexpr _Fp __hi = numeric_limits<_Fp>::max() / 2;
+
+  // typical case: overflow is impossible
+  if (std::__fp_abs(__a) <= __hi && std::__fp_abs(__b) <= __hi)
+    return (__a + __b) / 2; // always correctly rounded
+  if (std::__fp_abs(__a) < __lo)
+    return __a + __b / 2; // not safe to halve a
+  if (std::__fp_abs(__b) < __lo)
+    return __a / 2 + __b; // not safe to halve b
+
+  return __a / 2 + __b / 2; // otherwise correctly rounded
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___NUMERIC_MIDPOINT_H
diff --git a/libcxx/include/__cxx03/__numeric/partial_sum.h b/libcxx/include/__cxx03/__numeric/partial_sum.h
new file mode 100644
index 000000000000..7c3cf7fb20b0
--- /dev/null
+++ b/libcxx/include/__cxx03/__numeric/partial_sum.h
@@ -0,0 +1,66 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NUMERIC_PARTIAL_SUM_H
+#define _LIBCPP___NUMERIC_PARTIAL_SUM_H
+
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _InputIterator, class _OutputIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+partial_sum(_InputIterator __first, _InputIterator __last, _OutputIterator __result) {
+  if (__first != __last) {
+    typename iterator_traits<_InputIterator>::value_type __t(*__first);
+    *__result = __t;
+    for (++__first, (void)++__result; __first != __last; ++__first, (void)++__result) {
+#if _LIBCPP_STD_VER >= 20
+      __t = std::move(__t) + *__first;
+#else
+      __t = __t + *__first;
+#endif
+      *__result = __t;
+    }
+  }
+  return __result;
+}
+
+template <class _InputIterator, class _OutputIterator, class _BinaryOperation>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+partial_sum(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _BinaryOperation __binary_op) {
+  if (__first != __last) {
+    typename iterator_traits<_InputIterator>::value_type __t(*__first);
+    *__result = __t;
+    for (++__first, (void)++__result; __first != __last; ++__first, (void)++__result) {
+#if _LIBCPP_STD_VER >= 20
+      __t = __binary_op(std::move(__t), *__first);
+#else
+      __t = __binary_op(__t, *__first);
+#endif
+      *__result = __t;
+    }
+  }
+  return __result;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___NUMERIC_PARTIAL_SUM_H
diff --git a/libcxx/include/__cxx03/__numeric/pstl.h b/libcxx/include/__cxx03/__numeric/pstl.h
new file mode 100644
index 000000000000..7557686a3663
--- /dev/null
+++ b/libcxx/include/__cxx03/__numeric/pstl.h
@@ -0,0 +1,174 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NUMERIC_PSTL_H
+#define _LIBCPP___NUMERIC_PSTL_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
+
+#  include <__functional/identity.h>
+#  include <__functional/operations.h>
+#  include <__iterator/cpp17_iterator_concepts.h>
+#  include <__iterator/iterator_traits.h>
+#  include <__pstl/backend.h>
+#  include <__pstl/dispatch.h>
+#  include <__pstl/handle_exception.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/is_execution_policy.h>
+#  include <__type_traits/remove_cvref.h>
+#  include <__utility/forward.h>
+#  include <__utility/move.h>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Tp,
+          class _BinaryOperation,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp reduce(
+    _ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Tp __init, _BinaryOperation __op) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "reduce requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__reduce, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first),
+      std::move(__last),
+      std::move(__init),
+      std::move(__op));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Tp,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp
+reduce(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Tp __init) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "reduce requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__reduce, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__init), plus{});
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI __iter_value_type<_ForwardIterator>
+reduce(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "reduce requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__reduce, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first),
+      std::move(__last),
+      __iter_value_type<_ForwardIterator>(),
+      plus{});
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _Tp,
+          class _BinaryOperation1,
+          class _BinaryOperation2,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp transform_reduce(
+    _ExecutionPolicy&& __policy,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _Tp __init,
+    _BinaryOperation1 __reduce,
+    _BinaryOperation2 __transform) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "transform_reduce requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "transform_reduce requires ForwardIterators");
+  using _Implementation =
+      __pstl::__dispatch<__pstl::__transform_reduce_binary, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__init),
+      std::move(__reduce),
+      std::move(__transform));
+}
+
+// This overload doesn't get a customization point because it's trivial to detect (through e.g.
+// __desugars_to_v) when specializing the more general variant, which should always be preferred
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _Tp,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp transform_reduce(
+    _ExecutionPolicy&& __policy,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _Tp __init) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "transform_reduce requires ForwardIterators");
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "transform_reduce requires ForwardIterators");
+  using _Implementation =
+      __pstl::__dispatch<__pstl::__transform_reduce_binary, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first1),
+      std::move(__last1),
+      std::move(__first2),
+      std::move(__init),
+      plus{},
+      multiplies{});
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Tp,
+          class _BinaryOperation,
+          class _UnaryOperation,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp transform_reduce(
+    _ExecutionPolicy&& __policy,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _Tp __init,
+    _BinaryOperation __reduce,
+    _UnaryOperation __transform) {
+  _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "transform_reduce requires ForwardIterators");
+  using _Implementation = __pstl::__dispatch<__pstl::__transform_reduce, __pstl::__current_configuration, _RawPolicy>;
+  return __pstl::__handle_exception<_Implementation>(
+      std::forward<_ExecutionPolicy>(__policy),
+      std::move(__first),
+      std::move(__last),
+      std::move(__init),
+      std::move(__reduce),
+      std::move(__transform));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___NUMERIC_PSTL_H
diff --git a/libcxx/include/__cxx03/__numeric/reduce.h b/libcxx/include/__cxx03/__numeric/reduce.h
new file mode 100644
index 000000000000..6c205bf581fb
--- /dev/null
+++ b/libcxx/include/__cxx03/__numeric/reduce.h
@@ -0,0 +1,53 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NUMERIC_REDUCE_H
+#define _LIBCPP___NUMERIC_REDUCE_H
+
+#include <__config>
+#include <__functional/operations.h>
+#include <__iterator/iterator_traits.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator, class _Tp, class _BinaryOp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp
+reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOp __b) {
+  for (; __first != __last; ++__first)
+    __init = __b(std::move(__init), *__first);
+  return __init;
+}
+
+template <class _InputIterator, class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp
+reduce(_InputIterator __first, _InputIterator __last, _Tp __init) {
+  return std::reduce(__first, __last, __init, std::plus<>());
+}
+
+template <class _InputIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename iterator_traits<_InputIterator>::value_type
+reduce(_InputIterator __first, _InputIterator __last) {
+  return std::reduce(__first, __last, typename iterator_traits<_InputIterator>::value_type{});
+}
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___NUMERIC_REDUCE_H
diff --git a/libcxx/include/__cxx03/__numeric/saturation_arithmetic.h b/libcxx/include/__cxx03/__numeric/saturation_arithmetic.h
new file mode 100644
index 000000000000..2390b42aaec3
--- /dev/null
+++ b/libcxx/include/__cxx03/__numeric/saturation_arithmetic.h
@@ -0,0 +1,145 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NUMERIC_SATURATION_ARITHMETIC_H
+#define _LIBCPP___NUMERIC_SATURATION_ARITHMETIC_H
+
+#include <__assert>
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <__utility/cmp.h>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <__libcpp_integer _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp __add_sat(_Tp __x, _Tp __y) noexcept {
+  if (_Tp __sum; !__builtin_add_overflow(__x, __y, &__sum))
+    return __sum;
+  // Handle overflow
+  if constexpr (__libcpp_unsigned_integer<_Tp>) {
+    return std::numeric_limits<_Tp>::max();
+  } else {
+    // Signed addition overflow
+    if (__x > 0)
+      // Overflows if (x > 0 && y > 0)
+      return std::numeric_limits<_Tp>::max();
+    else
+      // Overflows if  (x < 0 && y < 0)
+      return std::numeric_limits<_Tp>::min();
+  }
+}
+
+template <__libcpp_integer _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp __sub_sat(_Tp __x, _Tp __y) noexcept {
+  if (_Tp __sub; !__builtin_sub_overflow(__x, __y, &__sub))
+    return __sub;
+  // Handle overflow
+  if constexpr (__libcpp_unsigned_integer<_Tp>) {
+    // Overflows if (x < y)
+    return std::numeric_limits<_Tp>::min();
+  } else {
+    // Signed subtration overflow
+    if (__x >= 0)
+      // Overflows if (x >= 0 && y < 0)
+      return std::numeric_limits<_Tp>::max();
+    else
+      // Overflows if (x < 0 && y > 0)
+      return std::numeric_limits<_Tp>::min();
+  }
+}
+
+template <__libcpp_integer _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp __mul_sat(_Tp __x, _Tp __y) noexcept {
+  if (_Tp __mul; !__builtin_mul_overflow(__x, __y, &__mul))
+    return __mul;
+  // Handle overflow
+  if constexpr (__libcpp_unsigned_integer<_Tp>) {
+    return std::numeric_limits<_Tp>::max();
+  } else {
+    // Signed multiplication overflow
+    if ((__x > 0 && __y > 0) || (__x < 0 && __y < 0))
+      return std::numeric_limits<_Tp>::max();
+    // Overflows if (x < 0 && y > 0) || (x > 0 && y < 0)
+    return std::numeric_limits<_Tp>::min();
+  }
+}
+
+template <__libcpp_integer _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp __div_sat(_Tp __x, _Tp __y) noexcept {
+  _LIBCPP_ASSERT_UNCATEGORIZED(__y != 0, "Division by 0 is undefined");
+  if constexpr (__libcpp_unsigned_integer<_Tp>) {
+    return __x / __y;
+  } else {
+    // Handle signed division overflow
+    if (__x == std::numeric_limits<_Tp>::min() && __y == _Tp{-1})
+      return std::numeric_limits<_Tp>::max();
+    return __x / __y;
+  }
+}
+
+template <__libcpp_integer _Rp, __libcpp_integer _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Rp __saturate_cast(_Tp __x) noexcept {
+  // Saturation is impossible edge case when ((min _Rp) < (min _Tp) && (max _Rp) > (max _Tp)) and it is expected to be
+  // optimized out by the compiler.
+
+  // Handle overflow
+  if (std::cmp_less(__x, std::numeric_limits<_Rp>::min()))
+    return std::numeric_limits<_Rp>::min();
+  if (std::cmp_greater(__x, std::numeric_limits<_Rp>::max()))
+    return std::numeric_limits<_Rp>::max();
+  // No overflow
+  return static_cast<_Rp>(__x);
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 26
+
+template <__libcpp_integer _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp add_sat(_Tp __x, _Tp __y) noexcept {
+  return std::__add_sat(__x, __y);
+}
+
+template <__libcpp_integer _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp sub_sat(_Tp __x, _Tp __y) noexcept {
+  return std::__sub_sat(__x, __y);
+}
+
+template <__libcpp_integer _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp mul_sat(_Tp __x, _Tp __y) noexcept {
+  return std::__mul_sat(__x, __y);
+}
+
+template <__libcpp_integer _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp div_sat(_Tp __x, _Tp __y) noexcept {
+  return std::__div_sat(__x, __y);
+}
+
+template <__libcpp_integer _Rp, __libcpp_integer _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Rp saturate_cast(_Tp __x) noexcept {
+  return std::__saturate_cast<_Rp>(__x);
+}
+
+#endif // _LIBCPP_STD_VER >= 26
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___NUMERIC_SATURATION_ARITHMETIC_H
diff --git a/libcxx/include/__cxx03/__numeric/transform_exclusive_scan.h b/libcxx/include/__cxx03/__numeric/transform_exclusive_scan.h
new file mode 100644
index 000000000000..fb88aa52cd10
--- /dev/null
+++ b/libcxx/include/__cxx03/__numeric/transform_exclusive_scan.h
@@ -0,0 +1,42 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NUMERIC_TRANSFORM_EXCLUSIVE_SCAN_H
+#define _LIBCPP___NUMERIC_TRANSFORM_EXCLUSIVE_SCAN_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp, class _UnaryOp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator transform_exclusive_scan(
+    _InputIterator __first, _InputIterator __last, _OutputIterator __result, _Tp __init, _BinaryOp __b, _UnaryOp __u) {
+  if (__first != __last) {
+    _Tp __saved = __init;
+    do {
+      __init    = __b(__init, __u(*__first));
+      *__result = __saved;
+      __saved   = __init;
+      ++__result;
+    } while (++__first != __last);
+  }
+  return __result;
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___NUMERIC_TRANSFORM_EXCLUSIVE_SCAN_H
diff --git a/libcxx/include/__cxx03/__numeric/transform_inclusive_scan.h b/libcxx/include/__cxx03/__numeric/transform_inclusive_scan.h
new file mode 100644
index 000000000000..2eab1a142439
--- /dev/null
+++ b/libcxx/include/__cxx03/__numeric/transform_inclusive_scan.h
@@ -0,0 +1,52 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NUMERIC_TRANSFORM_INCLUSIVE_SCAN_H
+#define _LIBCPP___NUMERIC_TRANSFORM_INCLUSIVE_SCAN_H
+
+#include <__config>
+#include <__iterator/iterator_traits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp, class _UnaryOp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator transform_inclusive_scan(
+    _InputIterator __first, _InputIterator __last, _OutputIterator __result, _BinaryOp __b, _UnaryOp __u, _Tp __init) {
+  for (; __first != __last; ++__first, (void)++__result) {
+    __init    = __b(__init, __u(*__first));
+    *__result = __init;
+  }
+
+  return __result;
+}
+
+template <class _InputIterator, class _OutputIterator, class _BinaryOp, class _UnaryOp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator transform_inclusive_scan(
+    _InputIterator __first, _InputIterator __last, _OutputIterator __result, _BinaryOp __b, _UnaryOp __u) {
+  if (__first != __last) {
+    typename iterator_traits<_InputIterator>::value_type __init = __u(*__first);
+    *__result++                                                 = __init;
+    if (++__first != __last)
+      return std::transform_inclusive_scan(__first, __last, __result, __b, __u, __init);
+  }
+
+  return __result;
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___NUMERIC_TRANSFORM_INCLUSIVE_SCAN_H
diff --git a/libcxx/include/__cxx03/__numeric/transform_reduce.h b/libcxx/include/__cxx03/__numeric/transform_reduce.h
new file mode 100644
index 000000000000..f1150510f0c3
--- /dev/null
+++ b/libcxx/include/__cxx03/__numeric/transform_reduce.h
@@ -0,0 +1,59 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___NUMERIC_TRANSFORM_REDUCE_H
+#define _LIBCPP___NUMERIC_TRANSFORM_REDUCE_H
+
+#include <__config>
+#include <__functional/operations.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator, class _Tp, class _BinaryOp, class _UnaryOp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp
+transform_reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOp __b, _UnaryOp __u) {
+  for (; __first != __last; ++__first)
+    __init = __b(std::move(__init), __u(*__first));
+  return __init;
+}
+
+template <class _InputIterator1, class _InputIterator2, class _Tp, class _BinaryOp1, class _BinaryOp2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp transform_reduce(
+    _InputIterator1 __first1,
+    _InputIterator1 __last1,
+    _InputIterator2 __first2,
+    _Tp __init,
+    _BinaryOp1 __b1,
+    _BinaryOp2 __b2) {
+  for (; __first1 != __last1; ++__first1, (void)++__first2)
+    __init = __b1(std::move(__init), __b2(*__first1, *__first2));
+  return __init;
+}
+
+template <class _InputIterator1, class _InputIterator2, class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp
+transform_reduce(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _Tp __init) {
+  return std::transform_reduce(__first1, __last1, __first2, std::move(__init), std::plus<>(), std::multiplies<>());
+}
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___NUMERIC_TRANSFORM_REDUCE_H
diff --git a/libcxx/include/__cxx03/__ostream/basic_ostream.h b/libcxx/include/__cxx03/__ostream/basic_ostream.h
new file mode 100644
index 000000000000..178359d68156
--- /dev/null
+++ b/libcxx/include/__cxx03/__ostream/basic_ostream.h
@@ -0,0 +1,860 @@
+//===---------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___OSTREAM_BASIC_OSTREAM_H
+#define _LIBCPP___OSTREAM_BASIC_OSTREAM_H
+
+#include <__config>
+#include <__exception/operations.h>
+#include <__memory/shared_ptr.h>
+#include <__memory/unique_ptr.h>
+#include <__system_error/error_code.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_base_of.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+#include <bitset>
+#include <cstddef>
+#include <ios>
+#include <locale>
+#include <new> // for __throw_bad_alloc
+#include <streambuf>
+#include <string_view>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS basic_ostream : virtual public basic_ios<_CharT, _Traits> {
+public:
+  // types (inherited from basic_ios (27.5.4)):
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+
+  // 27.7.2.2 Constructor/destructor:
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 explicit basic_ostream(basic_streambuf<char_type, traits_type>* __sb) {
+    this->init(__sb);
+  }
+  ~basic_ostream() override;
+
+  basic_ostream(const basic_ostream& __rhs)            = delete;
+  basic_ostream& operator=(const basic_ostream& __rhs) = delete;
+
+protected:
+  inline _LIBCPP_HIDE_FROM_ABI basic_ostream(basic_ostream&& __rhs);
+
+  // 27.7.2.3 Assign/swap
+  inline _LIBCPP_HIDE_FROM_ABI basic_ostream& operator=(basic_ostream&& __rhs);
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void swap(basic_ostream& __rhs) {
+    basic_ios<char_type, traits_type>::swap(__rhs);
+  }
+
+public:
+  // 27.7.2.4 Prefix/suffix:
+  class _LIBCPP_TEMPLATE_VIS sentry;
+
+  // 27.7.2.6 Formatted output:
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_ostream& operator<<(basic_ostream& (*__pf)(basic_ostream&)) {
+    return __pf(*this);
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_ostream&
+  operator<<(basic_ios<char_type, traits_type>& (*__pf)(basic_ios<char_type, traits_type>&)) {
+    __pf(*this);
+    return *this;
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_ostream& operator<<(ios_base& (*__pf)(ios_base&)) {
+    __pf(*this);
+    return *this;
+  }
+
+  basic_ostream& operator<<(bool __n);
+  basic_ostream& operator<<(short __n);
+  basic_ostream& operator<<(unsigned short __n);
+  basic_ostream& operator<<(int __n);
+  basic_ostream& operator<<(unsigned int __n);
+  basic_ostream& operator<<(long __n);
+  basic_ostream& operator<<(unsigned long __n);
+  basic_ostream& operator<<(long long __n);
+  basic_ostream& operator<<(unsigned long long __n);
+  basic_ostream& operator<<(float __f);
+  basic_ostream& operator<<(double __f);
+  basic_ostream& operator<<(long double __f);
+  basic_ostream& operator<<(const void* __p);
+
+#if _LIBCPP_STD_VER >= 23
+  _LIBCPP_HIDE_FROM_ABI basic_ostream& operator<<(const volatile void* __p) {
+    return operator<<(const_cast<const void*>(__p));
+  }
+#endif
+
+  basic_ostream& operator<<(basic_streambuf<char_type, traits_type>* __sb);
+
+#if _LIBCPP_STD_VER >= 17
+  // LWG 2221 - nullptr. This is not backported to older standards modes.
+  // See https://reviews.llvm.org/D127033 for more info on the rationale.
+  _LIBCPP_HIDE_FROM_ABI basic_ostream& operator<<(nullptr_t) { return *this << "nullptr"; }
+#endif
+
+  // 27.7.2.7 Unformatted output:
+  basic_ostream& put(char_type __c);
+  basic_ostream& write(const char_type* __s, streamsize __n);
+  basic_ostream& flush();
+
+  // 27.7.2.5 seeks:
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 pos_type tellp();
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_ostream& seekp(pos_type __pos);
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_ostream& seekp(off_type __off, ios_base::seekdir __dir);
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI basic_ostream() {} // extension, intentially does not initialize
+};
+
+template <class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS basic_ostream<_CharT, _Traits>::sentry {
+  bool __ok_;
+  basic_ostream<_CharT, _Traits>& __os_;
+
+public:
+  explicit sentry(basic_ostream<_CharT, _Traits>& __os);
+  ~sentry();
+  sentry(const sentry&)            = delete;
+  sentry& operator=(const sentry&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI explicit operator bool() const { return __ok_; }
+};
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>::sentry::sentry(basic_ostream<_CharT, _Traits>& __os) : __ok_(false), __os_(__os) {
+  if (__os.good()) {
+    if (__os.tie())
+      __os.tie()->flush();
+    __ok_ = true;
+  }
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>::sentry::~sentry() {
+  if (__os_.rdbuf() && __os_.good() && (__os_.flags() & ios_base::unitbuf) && !uncaught_exception()) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      if (__os_.rdbuf()->pubsync() == -1)
+        __os_.setstate(ios_base::badbit);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>::basic_ostream(basic_ostream&& __rhs) {
+  this->move(__rhs);
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator=(basic_ostream&& __rhs) {
+  swap(__rhs);
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>::~basic_ostream() {}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>&
+basic_ostream<_CharT, _Traits>::operator<<(basic_streambuf<char_type, traits_type>* __sb) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __s(*this);
+    if (__s) {
+      if (__sb) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+        try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+          typedef istreambuf_iterator<_CharT, _Traits> _Ip;
+          typedef ostreambuf_iterator<_CharT, _Traits> _Op;
+          _Ip __i(__sb);
+          _Ip __eof;
+          _Op __o(*this);
+          size_t __c = 0;
+          for (; __i != __eof; ++__i, ++__o, ++__c) {
+            *__o = *__i;
+            if (__o.failed())
+              break;
+          }
+          if (__c == 0)
+            this->setstate(ios_base::failbit);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+        } catch (...) {
+          this->__set_failbit_and_consider_rethrow();
+        }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      } else
+        this->setstate(ios_base::badbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(bool __n) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __s(*this);
+    if (__s) {
+      typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
+      const _Fp& __f = std::use_facet<_Fp>(this->getloc());
+      if (__f.put(*this, *this, this->fill(), __n).failed())
+        this->setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(short __n) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __s(*this);
+    if (__s) {
+      ios_base::fmtflags __flags = ios_base::flags() & ios_base::basefield;
+      typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
+      const _Fp& __f = std::use_facet<_Fp>(this->getloc());
+      if (__f.put(*this,
+                  *this,
+                  this->fill(),
+                  __flags == ios_base::oct || __flags == ios_base::hex
+                      ? static_cast<long>(static_cast<unsigned short>(__n))
+                      : static_cast<long>(__n))
+              .failed())
+        this->setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(unsigned short __n) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __s(*this);
+    if (__s) {
+      typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
+      const _Fp& __f = std::use_facet<_Fp>(this->getloc());
+      if (__f.put(*this, *this, this->fill(), static_cast<unsigned long>(__n)).failed())
+        this->setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(int __n) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __s(*this);
+    if (__s) {
+      ios_base::fmtflags __flags = ios_base::flags() & ios_base::basefield;
+      typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
+      const _Fp& __f = std::use_facet<_Fp>(this->getloc());
+      if (__f.put(*this,
+                  *this,
+                  this->fill(),
+                  __flags == ios_base::oct || __flags == ios_base::hex
+                      ? static_cast<long>(static_cast<unsigned int>(__n))
+                      : static_cast<long>(__n))
+              .failed())
+        this->setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(unsigned int __n) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __s(*this);
+    if (__s) {
+      typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
+      const _Fp& __f = std::use_facet<_Fp>(this->getloc());
+      if (__f.put(*this, *this, this->fill(), static_cast<unsigned long>(__n)).failed())
+        this->setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(long __n) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __s(*this);
+    if (__s) {
+      typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
+      const _Fp& __f = std::use_facet<_Fp>(this->getloc());
+      if (__f.put(*this, *this, this->fill(), __n).failed())
+        this->setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(unsigned long __n) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __s(*this);
+    if (__s) {
+      typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
+      const _Fp& __f = std::use_facet<_Fp>(this->getloc());
+      if (__f.put(*this, *this, this->fill(), __n).failed())
+        this->setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(long long __n) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __s(*this);
+    if (__s) {
+      typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
+      const _Fp& __f = std::use_facet<_Fp>(this->getloc());
+      if (__f.put(*this, *this, this->fill(), __n).failed())
+        this->setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(unsigned long long __n) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __s(*this);
+    if (__s) {
+      typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
+      const _Fp& __f = std::use_facet<_Fp>(this->getloc());
+      if (__f.put(*this, *this, this->fill(), __n).failed())
+        this->setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(float __n) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __s(*this);
+    if (__s) {
+      typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
+      const _Fp& __f = std::use_facet<_Fp>(this->getloc());
+      if (__f.put(*this, *this, this->fill(), static_cast<double>(__n)).failed())
+        this->setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(double __n) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __s(*this);
+    if (__s) {
+      typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
+      const _Fp& __f = std::use_facet<_Fp>(this->getloc());
+      if (__f.put(*this, *this, this->fill(), __n).failed())
+        this->setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(long double __n) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __s(*this);
+    if (__s) {
+      typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
+      const _Fp& __f = std::use_facet<_Fp>(this->getloc());
+      if (__f.put(*this, *this, this->fill(), __n).failed())
+        this->setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(const void* __n) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __s(*this);
+    if (__s) {
+      typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
+      const _Fp& __f = std::use_facet<_Fp>(this->getloc());
+      if (__f.put(*this, *this, this->fill(), __n).failed())
+        this->setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+__put_character_sequence(basic_ostream<_CharT, _Traits>& __os, const _CharT* __str, size_t __len) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    typename basic_ostream<_CharT, _Traits>::sentry __s(__os);
+    if (__s) {
+      typedef ostreambuf_iterator<_CharT, _Traits> _Ip;
+      if (std::__pad_and_output(
+              _Ip(__os),
+              __str,
+              (__os.flags() & ios_base::adjustfield) == ios_base::left ? __str + __len : __str,
+              __str + __len,
+              __os,
+              __os.fill())
+              .failed())
+        __os.setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    __os.__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return __os;
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>& operator<<(basic_ostream<_CharT, _Traits>& __os, _CharT __c) {
+  return std::__put_character_sequence(__os, &__c, 1);
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>& operator<<(basic_ostream<_CharT, _Traits>& __os, char __cn) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    typename basic_ostream<_CharT, _Traits>::sentry __s(__os);
+    if (__s) {
+      _CharT __c = __os.widen(__cn);
+      typedef ostreambuf_iterator<_CharT, _Traits> _Ip;
+      if (std::__pad_and_output(
+              _Ip(__os),
+              &__c,
+              (__os.flags() & ios_base::adjustfield) == ios_base::left ? &__c + 1 : &__c,
+              &__c + 1,
+              __os,
+              __os.fill())
+              .failed())
+        __os.setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    __os.__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return __os;
+}
+
+template <class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<char, _Traits>& operator<<(basic_ostream<char, _Traits>& __os, char __c) {
+  return std::__put_character_sequence(__os, &__c, 1);
+}
+
+template <class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<char, _Traits>& operator<<(basic_ostream<char, _Traits>& __os, signed char __c) {
+  return std::__put_character_sequence(__os, (char*)&__c, 1);
+}
+
+template <class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<char, _Traits>& operator<<(basic_ostream<char, _Traits>& __os, unsigned char __c) {
+  return std::__put_character_sequence(__os, (char*)&__c, 1);
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const _CharT* __str) {
+  return std::__put_character_sequence(__os, __str, _Traits::length(__str));
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const char* __strn) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    typename basic_ostream<_CharT, _Traits>::sentry __s(__os);
+    if (__s) {
+      typedef ostreambuf_iterator<_CharT, _Traits> _Ip;
+      size_t __len   = char_traits<char>::length(__strn);
+      const int __bs = 100;
+      _CharT __wbb[__bs];
+      _CharT* __wb = __wbb;
+      unique_ptr<_CharT, void (*)(void*)> __h(0, free);
+      if (__len > __bs) {
+        __wb = (_CharT*)malloc(__len * sizeof(_CharT));
+        if (__wb == 0)
+          __throw_bad_alloc();
+        __h.reset(__wb);
+      }
+      for (_CharT* __p = __wb; *__strn != '\0'; ++__strn, ++__p)
+        *__p = __os.widen(*__strn);
+      if (std::__pad_and_output(
+              _Ip(__os),
+              __wb,
+              (__os.flags() & ios_base::adjustfield) == ios_base::left ? __wb + __len : __wb,
+              __wb + __len,
+              __os,
+              __os.fill())
+              .failed())
+        __os.setstate(ios_base::badbit | ios_base::failbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    __os.__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return __os;
+}
+
+template <class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<char, _Traits>& operator<<(basic_ostream<char, _Traits>& __os, const char* __str) {
+  return std::__put_character_sequence(__os, __str, _Traits::length(__str));
+}
+
+template <class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<char, _Traits>&
+operator<<(basic_ostream<char, _Traits>& __os, const signed char* __str) {
+  const char* __s = (const char*)__str;
+  return std::__put_character_sequence(__os, __s, _Traits::length(__s));
+}
+
+template <class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<char, _Traits>&
+operator<<(basic_ostream<char, _Traits>& __os, const unsigned char* __str) {
+  const char* __s = (const char*)__str;
+  return std::__put_character_sequence(__os, __s, _Traits::length(__s));
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::put(char_type __c) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __s(*this);
+    if (__s) {
+      typedef ostreambuf_iterator<_CharT, _Traits> _Op;
+      _Op __o(*this);
+      *__o = __c;
+      if (__o.failed())
+        this->setstate(ios_base::badbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::write(const char_type* __s, streamsize __n) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    sentry __sen(*this);
+    if (__sen && __n) {
+      if (this->rdbuf()->sputn(__s, __n) != __n)
+        this->setstate(ios_base::badbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::flush() {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    if (this->rdbuf()) {
+      sentry __s(*this);
+      if (__s) {
+        if (this->rdbuf()->pubsync() == -1)
+          this->setstate(ios_base::badbit);
+      }
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+typename basic_ostream<_CharT, _Traits>::pos_type basic_ostream<_CharT, _Traits>::tellp() {
+  if (this->fail())
+    return pos_type(-1);
+  return this->rdbuf()->pubseekoff(0, ios_base::cur, ios_base::out);
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::seekp(pos_type __pos) {
+  sentry __s(*this);
+  if (!this->fail()) {
+    if (this->rdbuf()->pubseekpos(__pos, ios_base::out) == pos_type(-1))
+      this->setstate(ios_base::failbit);
+  }
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::seekp(off_type __off, ios_base::seekdir __dir) {
+  sentry __s(*this);
+  if (!this->fail()) {
+    if (this->rdbuf()->pubseekoff(__off, __dir, ios_base::out) == pos_type(-1))
+      this->setstate(ios_base::failbit);
+  }
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI inline basic_ostream<_CharT, _Traits>& endl(basic_ostream<_CharT, _Traits>& __os) {
+  __os.put(__os.widen('\n'));
+  __os.flush();
+  return __os;
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI inline basic_ostream<_CharT, _Traits>& ends(basic_ostream<_CharT, _Traits>& __os) {
+  __os.put(_CharT());
+  return __os;
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI inline basic_ostream<_CharT, _Traits>& flush(basic_ostream<_CharT, _Traits>& __os) {
+  __os.flush();
+  return __os;
+}
+
+template <class _Stream, class _Tp, class = void>
+struct __is_ostreamable : false_type {};
+
+template <class _Stream, class _Tp>
+struct __is_ostreamable<_Stream, _Tp, decltype(std::declval<_Stream>() << std::declval<_Tp>(), void())> : true_type {};
+
+template <class _Stream,
+          class _Tp,
+          __enable_if_t<_And<is_base_of<ios_base, _Stream>, __is_ostreamable<_Stream&, const _Tp&> >::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Stream&& operator<<(_Stream&& __os, const _Tp& __x) {
+  __os << __x;
+  return std::move(__os);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const basic_string<_CharT, _Traits, _Allocator>& __str) {
+  return std::__put_character_sequence(__os, __str.data(), __str.size());
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, basic_string_view<_CharT, _Traits> __sv) {
+  return std::__put_character_sequence(__os, __sv.data(), __sv.size());
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const error_code& __ec) {
+  return __os << __ec.category().name() << ':' << __ec.value();
+}
+
+template <class _CharT, class _Traits, class _Yp>
+inline _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, shared_ptr<_Yp> const& __p) {
+  return __os << __p.get();
+}
+
+template <
+    class _CharT,
+    class _Traits,
+    class _Yp,
+    class _Dp,
+    __enable_if_t<is_same<void,
+                          __void_t<decltype((std::declval<basic_ostream<_CharT, _Traits>&>()
+                                             << std::declval<typename unique_ptr<_Yp, _Dp>::pointer>()))> >::value,
+                  int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, unique_ptr<_Yp, _Dp> const& __p) {
+  return __os << __p.get();
+}
+
+template <class _CharT, class _Traits, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const bitset<_Size>& __x) {
+  return __os << __x.template to_string<_CharT, _Traits>(std::use_facet<ctype<_CharT> >(__os.getloc()).widen('0'),
+                                                         std::use_facet<ctype<_CharT> >(__os.getloc()).widen('1'));
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class _Traits>
+basic_ostream<char, _Traits>& operator<<(basic_ostream<char, _Traits>&, wchar_t) = delete;
+
+template <class _Traits>
+basic_ostream<char, _Traits>& operator<<(basic_ostream<char, _Traits>&, const wchar_t*) = delete;
+
+template <class _Traits>
+basic_ostream<wchar_t, _Traits>& operator<<(basic_ostream<wchar_t, _Traits>&, char16_t) = delete;
+
+template <class _Traits>
+basic_ostream<wchar_t, _Traits>& operator<<(basic_ostream<wchar_t, _Traits>&, char32_t) = delete;
+
+template <class _Traits>
+basic_ostream<wchar_t, _Traits>& operator<<(basic_ostream<wchar_t, _Traits>&, const char16_t*) = delete;
+
+template <class _Traits>
+basic_ostream<wchar_t, _Traits>& operator<<(basic_ostream<wchar_t, _Traits>&, const char32_t*) = delete;
+
+#  endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+template <class _Traits>
+basic_ostream<char, _Traits>& operator<<(basic_ostream<char, _Traits>&, char8_t) = delete;
+
+template <class _Traits>
+basic_ostream<wchar_t, _Traits>& operator<<(basic_ostream<wchar_t, _Traits>&, char8_t) = delete;
+
+template <class _Traits>
+basic_ostream<char, _Traits>& operator<<(basic_ostream<char, _Traits>&, const char8_t*) = delete;
+
+template <class _Traits>
+basic_ostream<wchar_t, _Traits>& operator<<(basic_ostream<wchar_t, _Traits>&, const char8_t*) = delete;
+#  endif
+
+template <class _Traits>
+basic_ostream<char, _Traits>& operator<<(basic_ostream<char, _Traits>&, char16_t) = delete;
+
+template <class _Traits>
+basic_ostream<char, _Traits>& operator<<(basic_ostream<char, _Traits>&, char32_t) = delete;
+
+template <class _Traits>
+basic_ostream<char, _Traits>& operator<<(basic_ostream<char, _Traits>&, const char16_t*) = delete;
+
+template <class _Traits>
+basic_ostream<char, _Traits>& operator<<(basic_ostream<char, _Traits>&, const char32_t*) = delete;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_ostream<char>;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_ostream<wchar_t>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___OSTREAM_BASIC_OSTREAM_H
diff --git a/libcxx/include/__cxx03/__ostream/print.h b/libcxx/include/__cxx03/__ostream/print.h
new file mode 100644
index 000000000000..8265ac00777e
--- /dev/null
+++ b/libcxx/include/__cxx03/__ostream/print.h
@@ -0,0 +1,179 @@
+//===---------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___OSTREAM_PRINT_H
+#define _LIBCPP___OSTREAM_PRINT_H
+
+#include <__config>
+#include <__fwd/ostream.h>
+#include <__iterator/ostreambuf_iterator.h>
+#include <__ostream/basic_ostream.h>
+#include <format>
+#include <ios>
+#include <locale>
+#include <print>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
+_LIBCPP_HIDE_FROM_ABI inline void
+__vprint_nonunicode(ostream& __os, string_view __fmt, format_args __args, bool __write_nl) {
+  // [ostream.formatted.print]/3
+  // Effects: Behaves as a formatted output function
+  // ([ostream.formatted.reqmts]) of os, except that:
+  // - failure to generate output is reported as specified below, and
+  // - any exception thrown by the call to vformat is propagated without regard
+  //   to the value of os.exceptions() and without turning on ios_base::badbit
+  //   in the error state of os.
+  // After constructing a sentry object, the function initializes an automatic
+  // variable via
+  //   string out = vformat(os.getloc(), fmt, args);
+
+  ostream::sentry __s(__os);
+  if (__s) {
+    string __o = std::vformat(__os.getloc(), __fmt, __args);
+    if (__write_nl)
+      __o += '\n';
+
+    const char* __str = __o.data();
+    size_t __len      = __o.size();
+
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      typedef ostreambuf_iterator<char> _Ip;
+      if (std::__pad_and_output(
+              _Ip(__os),
+              __str,
+              (__os.flags() & ios_base::adjustfield) == ios_base::left ? __str + __len : __str,
+              __str + __len,
+              __os,
+              __os.fill())
+              .failed())
+        __os.setstate(ios_base::badbit | ios_base::failbit);
+
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __os.__set_badbit_and_consider_rethrow();
+    }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+}
+
+template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
+_LIBCPP_HIDE_FROM_ABI inline void vprint_nonunicode(ostream& __os, string_view __fmt, format_args __args) {
+  std::__vprint_nonunicode(__os, __fmt, __args, false);
+}
+
+// Returns the FILE* associated with the __os.
+// Returns a nullptr when no FILE* is associated with __os.
+// This function is in the dylib since the type of the buffer associated
+// with std::cout, std::cerr, and std::clog is only known in the dylib.
+//
+// This function implements part of the implementation-defined behavior
+// of [ostream.formatted.print]/3
+//   If the function is vprint_unicode and os is a stream that refers to
+//   a terminal capable of displaying Unicode which is determined in an
+//   implementation-defined manner, writes out to the terminal using the
+//   native Unicode API;
+// Whether the returned FILE* is "a terminal capable of displaying Unicode"
+// is determined in the same way as the print(FILE*, ...) overloads.
+_LIBCPP_EXPORTED_FROM_ABI FILE* __get_ostream_file(ostream& __os);
+
+#  ifndef _LIBCPP_HAS_NO_UNICODE
+template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
+_LIBCPP_HIDE_FROM_ABI void __vprint_unicode(ostream& __os, string_view __fmt, format_args __args, bool __write_nl) {
+#    if _LIBCPP_AVAILABILITY_HAS_PRINT == 0
+  return std::__vprint_nonunicode(__os, __fmt, __args, __write_nl);
+#    else
+  FILE* __file = std::__get_ostream_file(__os);
+  if (!__file || !__print::__is_terminal(__file))
+    return std::__vprint_nonunicode(__os, __fmt, __args, __write_nl);
+
+  // [ostream.formatted.print]/3
+  //    If the function is vprint_unicode and os is a stream that refers to a
+  //    terminal capable of displaying Unicode which is determined in an
+  //    implementation-defined manner, writes out to the terminal using the
+  //    native Unicode API; if out contains invalid code units, the behavior is
+  //    undefined and implementations are encouraged to diagnose it. If the
+  //    native Unicode API is used, the function flushes os before writing out.
+  //
+  // This is the path for the native API, start with flushing.
+  __os.flush();
+
+#      ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#      endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    ostream::sentry __s(__os);
+    if (__s) {
+#      ifndef _LIBCPP_WIN32API
+      __print::__vprint_unicode_posix(__file, __fmt, __args, __write_nl, true);
+#      elif !defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS)
+    __print::__vprint_unicode_windows(__file, __fmt, __args, __write_nl, true);
+#      else
+#        error "Windows builds with wchar_t disabled are not supported."
+#      endif
+    }
+
+#      ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    __os.__set_badbit_and_consider_rethrow();
+  }
+#      endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#    endif   // _LIBCPP_AVAILABILITY_HAS_PRINT
+}
+
+template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
+_LIBCPP_HIDE_FROM_ABI inline void vprint_unicode(ostream& __os, string_view __fmt, format_args __args) {
+  std::__vprint_unicode(__os, __fmt, __args, false);
+}
+#  endif // _LIBCPP_HAS_NO_UNICODE
+
+template <class... _Args>
+_LIBCPP_HIDE_FROM_ABI void print(ostream& __os, format_string<_Args...> __fmt, _Args&&... __args) {
+#  ifndef _LIBCPP_HAS_NO_UNICODE
+  if constexpr (__print::__use_unicode_execution_charset)
+    std::__vprint_unicode(__os, __fmt.get(), std::make_format_args(__args...), false);
+  else
+    std::__vprint_nonunicode(__os, __fmt.get(), std::make_format_args(__args...), false);
+#  else  // _LIBCPP_HAS_NO_UNICODE
+  std::__vprint_nonunicode(__os, __fmt.get(), std::make_format_args(__args...), false);
+#  endif // _LIBCPP_HAS_NO_UNICODE
+}
+
+template <class... _Args>
+_LIBCPP_HIDE_FROM_ABI void println(ostream& __os, format_string<_Args...> __fmt, _Args&&... __args) {
+#  ifndef _LIBCPP_HAS_NO_UNICODE
+  // Note the wording in the Standard is inefficient. The output of
+  // std::format is a std::string which is then copied. This solution
+  // just appends a newline at the end of the output.
+  if constexpr (__print::__use_unicode_execution_charset)
+    std::__vprint_unicode(__os, __fmt.get(), std::make_format_args(__args...), true);
+  else
+    std::__vprint_nonunicode(__os, __fmt.get(), std::make_format_args(__args...), true);
+#  else  // _LIBCPP_HAS_NO_UNICODE
+  std::__vprint_nonunicode(__os, __fmt.get(), std::make_format_args(__args...), true);
+#  endif // _LIBCPP_HAS_NO_UNICODE
+}
+
+template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
+_LIBCPP_HIDE_FROM_ABI inline void println(ostream& __os) {
+  std::print(__os, "\n");
+}
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___OSTREAM_PRINT_H
diff --git a/libcxx/include/__cxx03/__pstl/backend.h b/libcxx/include/__cxx03/__pstl/backend.h
new file mode 100644
index 000000000000..86d9f28c77fa
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/backend.h
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_BACKEND_H
+#define _LIBCPP___PSTL_BACKEND_H
+
+#include <__config>
+#include <__pstl/backend_fwd.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if defined(_LIBCPP_PSTL_BACKEND_SERIAL)
+#  include <__pstl/backends/default.h>
+#  include <__pstl/backends/serial.h>
+#elif defined(_LIBCPP_PSTL_BACKEND_STD_THREAD)
+#  include <__pstl/backends/default.h>
+#  include <__pstl/backends/std_thread.h>
+#elif defined(_LIBCPP_PSTL_BACKEND_LIBDISPATCH)
+#  include <__pstl/backends/default.h>
+#  include <__pstl/backends/libdispatch.h>
+#endif
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_BACKEND_H
diff --git a/libcxx/include/__cxx03/__pstl/backend_fwd.h b/libcxx/include/__cxx03/__pstl/backend_fwd.h
new file mode 100644
index 000000000000..32c5da576fb3
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/backend_fwd.h
@@ -0,0 +1,301 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_BACKEND_FWD_H
+#define _LIBCPP___PSTL_BACKEND_FWD_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+//
+// This header declares available PSTL backends and the functions that must be implemented in order for the
+// PSTL algorithms to be provided.
+//
+// Backends often do not implement the full set of functions themselves -- a configuration of the PSTL is
+// usually a set of backends "stacked" together which each implement some algorithms under some execution
+// policies. It is only necessary for the "stack" of backends to implement all algorithms under all execution
+// policies, but a single backend is not required to implement everything on its own.
+//
+// The signatures used by each backend function are documented below.
+//
+// Exception handling
+// ==================
+//
+// PSTL backends are expected to report errors (i.e. failure to allocate) by returning a disengaged `optional` from
+// their implementation. Exceptions shouldn't be used to report an internal failure-to-allocate, since all exceptions
+// are turned into a program termination at the front-end level. When a backend returns a disengaged `optional` to the
+// frontend, the frontend will turn that into a call to `std::__throw_bad_alloc();` to report the internal failure to
+// the user.
+//
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+template <class... _Backends>
+struct __backend_configuration;
+
+struct __default_backend_tag;
+struct __libdispatch_backend_tag;
+struct __serial_backend_tag;
+struct __std_thread_backend_tag;
+
+#if defined(_LIBCPP_PSTL_BACKEND_SERIAL)
+using __current_configuration = __backend_configuration<__serial_backend_tag, __default_backend_tag>;
+#elif defined(_LIBCPP_PSTL_BACKEND_STD_THREAD)
+using __current_configuration = __backend_configuration<__std_thread_backend_tag, __default_backend_tag>;
+#elif defined(_LIBCPP_PSTL_BACKEND_LIBDISPATCH)
+using __current_configuration = __backend_configuration<__libdispatch_backend_tag, __default_backend_tag>;
+#else
+
+// ...New vendors can add parallel backends here...
+
+#  error "Invalid PSTL backend configuration"
+#endif
+
+template <class _Backend, class _ExecutionPolicy>
+struct __find_if;
+// template <class _Policy, class _ForwardIterator, class _Predicate>
+// optional<_ForwardIterator>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __find_if_not;
+// template <class _Policy, class _ForwardIterator, class _Predicate>
+// optional<_ForwardIterator>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __find;
+// template <class _Policy, class _ForwardIterator, class _Tp>
+// optional<_ForwardIterator>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __any_of;
+// template <class _Policy, class _ForwardIterator, class _Predicate>
+// optional<bool>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __all_of;
+// template <class _Policy, class _ForwardIterator, class _Predicate>
+// optional<bool>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __none_of;
+// template <class _Policy, class _ForwardIterator, class _Predicate>
+// optional<bool>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __is_partitioned;
+// template <class _Policy, class _ForwardIterator, class _Predicate>
+// optional<bool>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __for_each;
+// template <class _Policy, class _ForwardIterator, class _Function>
+// optional<__empty>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Function __func) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __for_each_n;
+// template <class _Policy, class _ForwardIterator, class _Size, class _Function>
+// optional<__empty>
+// operator()(_Policy&&, _ForwardIterator __first, _Size __size, _Function __func) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __fill;
+// template <class _Policy, class _ForwardIterator, class _Tp>
+// optional<__empty>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Tp const& __value) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __fill_n;
+// template <class _Policy, class _ForwardIterator, class _Size, class _Tp>
+// optional<__empty>
+// operator()(_Policy&&, _ForwardIterator __first, _Size __n, _Tp const& __value) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __replace;
+// template <class _Policy, class _ForwardIterator, class _Tp>
+// optional<__empty>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last,
+//                       _Tp const& __old, _Tp const& __new) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __replace_if;
+// template <class _Policy, class _ForwardIterator, class _Predicate, class _Tp>
+// optional<__empty>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last,
+//                       _Predicate __pred, _Tp const& __new_value) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __generate;
+// template <class _Policy, class _ForwardIterator, class _Generator>
+// optional<__empty>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Generator __gen) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __generate_n;
+// template <class _Policy, class _ForwardIterator, class _Size, class _Generator>
+// optional<__empty>
+// operator()(_Policy&&, _ForwardIterator __first, _Size __n, _Generator __gen) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __merge;
+// template <class _Policy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardOutIterator, class _Comp>
+// optional<_ForwardOutIterator>
+// operator()(_Policy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
+//                       _ForwardIterator2 __first2, _ForwardIterator2 __last2,
+//                       _ForwardOutIterator __result, _Comp __comp) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __stable_sort;
+// template <class _Policy, class _RandomAccessIterator, class _Comp>
+// optional<__empty>
+// operator()(_Policy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __sort;
+// template <class _Policy, class _RandomAccessIterator, class _Comp>
+// optional<__empty>
+// operator()(_Policy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __transform;
+// template <class _Policy, class _ForwardIterator, class _ForwardOutIterator, class _UnaryOperation>
+// optional<_ForwardOutIterator>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last,
+//                       _ForwardOutIterator __result,
+//                       _UnaryOperation __op) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __transform_binary;
+// template <class _Policy, class _ForwardIterator1, class _ForwardIterator2,
+//                          class _ForwardOutIterator,
+//                          class _BinaryOperation>
+// optional<_ForwardOutIterator>
+// operator()(_Policy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
+//                       _ForwardIterator2 __first2,
+//                       _ForwardOutIterator __result,
+//                       _BinaryOperation __op) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __replace_copy_if;
+// template <class _Policy, class _ForwardIterator, class _ForwardOutIterator, class _Predicate, class _Tp>
+// optional<__empty>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last,
+//                       _ForwardOutIterator __out_it,
+//                       _Predicate __pred,
+//                       _Tp const& __new_value) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __replace_copy;
+// template <class _Policy, class _ForwardIterator, class _ForwardOutIterator, class _Tp>
+// optional<__empty>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last,
+//                       _ForwardOutIterator __out_it,
+//                       _Tp const& __old_value,
+//                       _Tp const& __new_value) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __move;
+// template <class _Policy, class _ForwardIterator, class _ForwardOutIterator>
+// optional<_ForwardOutIterator>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last,
+//                       _ForwardOutIterator __out_it) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __copy;
+// template <class _Policy, class _ForwardIterator, class _ForwardOutIterator>
+// optional<_ForwardOutIterator>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last,
+//                       _ForwardOutIterator __out_it) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __copy_n;
+// template <class _Policy, class _ForwardIterator, class _Size, class _ForwardOutIterator>
+// optional<_ForwardOutIterator>
+// operator()(_Policy&&, _ForwardIterator __first, _Size __n, _ForwardOutIterator __out_it) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __rotate_copy;
+// template <class _Policy, class _ForwardIterator, class _ForwardOutIterator>
+// optional<_ForwardOutIterator>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
+//                       _ForwardOutIterator __out_it) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __transform_reduce;
+// template <class _Policy, class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
+// optional<_Tp>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last,
+//                       _Tp __init,
+//                       _BinaryOperation __reduce,
+//                       _UnaryOperation __transform) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __transform_reduce_binary;
+// template <class _Policy, class _ForwardIterator1, class _ForwardIterator2,
+//           class _Tp, class _BinaryOperation1, class _BinaryOperation2>
+// optional<_Tp> operator()(_Policy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
+//                                     _ForwardIterator2 __first2,
+//                                     _Tp __init,
+//                                     _BinaryOperation1 __reduce,
+//                                     _BinaryOperation2 __transform) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __count_if;
+// template <class _Policy, class _ForwardIterator, class _Predicate>
+// optional<__iter_diff_t<_ForwardIterator>>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __count;
+// template <class _Policy, class _ForwardIterator, class _Tp>
+// optional<__iter_diff_t<_ForwardIterator>>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Tp const& __value) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __equal_3leg;
+// template <class _Policy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
+// optional<bool>
+// operator()(_Policy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
+//                       _ForwardIterator2 __first2,
+//                       _Predicate __pred) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __equal;
+// template <class _Policy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
+// optional<bool>
+// operator()(_Policy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
+//                       _ForwardIterator2 __first2, _ForwardIterator2 __last2,
+//                       _Predicate __pred) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __reduce;
+// template <class _Policy, class _ForwardIterator, class _Tp, class _BinaryOperation>
+// optional<_Tp>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last,
+//                       _Tp __init, _BinaryOperation __op) const noexcept;
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_BACKEND_FWD_H
diff --git a/libcxx/include/__cxx03/__pstl/backends/default.h b/libcxx/include/__cxx03/__pstl/backends/default.h
new file mode 100644
index 000000000000..61a128805f85
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/backends/default.h
@@ -0,0 +1,503 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_BACKENDS_DEFAULT_H
+#define _LIBCPP___PSTL_BACKENDS_DEFAULT_H
+
+#include <__algorithm/copy_n.h>
+#include <__algorithm/equal.h>
+#include <__algorithm/fill_n.h>
+#include <__algorithm/for_each_n.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/not_fn.h>
+#include <__functional/operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__pstl/backend_fwd.h>
+#include <__pstl/dispatch.h>
+#include <__utility/empty.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+//
+// This file provides an incomplete PSTL backend that implements all of the PSTL algorithms
+// based on a smaller set of basis operations.
+//
+// It is intended as a building block for other PSTL backends that implement some operations more
+// efficiently but may not want to define the full set of PSTL algorithms.
+//
+// This backend implements all the PSTL algorithms based on the following basis operations:
+//
+// find_if family
+// --------------
+// - find
+// - find_if_not
+// - any_of
+// - all_of
+// - none_of
+// - is_partitioned
+//
+// for_each family
+// ---------------
+// - for_each_n
+// - fill
+// - fill_n
+// - replace
+// - replace_if
+// - generate
+// - generate_n
+//
+// merge family
+// ------------
+// No other algorithms based on merge
+//
+// stable_sort family
+// ------------------
+// - sort
+//
+// transform_reduce and transform_reduce_binary family
+// ---------------------------------------------------
+// - count_if
+// - count
+// - equal(3 legs)
+// - equal
+// - reduce
+//
+// transform and transform_binary family
+// -------------------------------------
+// - replace_copy_if
+// - replace_copy
+// - move
+// - copy
+// - copy_n
+// - rotate_copy
+//
+
+//////////////////////////////////////////////////////////////
+// find_if family
+//////////////////////////////////////////////////////////////
+template <class _ExecutionPolicy>
+struct __find<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardIterator>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) const noexcept {
+    using _FindIf = __dispatch<__find_if, __current_configuration, _ExecutionPolicy>;
+    return _FindIf()(
+        __policy, std::move(__first), std::move(__last), [&](__iter_reference<_ForwardIterator> __element) {
+          return __element == __value;
+        });
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __find_if_not<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardIterator>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred) const noexcept {
+    using _FindIf = __dispatch<__find_if, __current_configuration, _ExecutionPolicy>;
+    return _FindIf()(__policy, __first, __last, std::not_fn(std::forward<_Pred>(__pred)));
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __any_of<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<bool>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred) const noexcept {
+    using _FindIf = __dispatch<__find_if, __current_configuration, _ExecutionPolicy>;
+    auto __res    = _FindIf()(__policy, __first, __last, std::forward<_Pred>(__pred));
+    if (!__res)
+      return nullopt;
+    return *__res != __last;
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __all_of<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<bool>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred) const noexcept {
+    using _AnyOf = __dispatch<__any_of, __current_configuration, _ExecutionPolicy>;
+    auto __res   = _AnyOf()(__policy, __first, __last, [&](__iter_reference<_ForwardIterator> __value) {
+      return !__pred(__value);
+    });
+    if (!__res)
+      return nullopt;
+    return !*__res;
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __none_of<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<bool>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred) const noexcept {
+    using _AnyOf = __dispatch<__any_of, __current_configuration, _ExecutionPolicy>;
+    auto __res   = _AnyOf()(__policy, __first, __last, std::forward<_Pred>(__pred));
+    if (!__res)
+      return nullopt;
+    return !*__res;
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __is_partitioned<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<bool>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred) const noexcept {
+    using _FindIfNot   = __dispatch<__find_if_not, __current_configuration, _ExecutionPolicy>;
+    auto __maybe_first = _FindIfNot()(__policy, std::move(__first), std::move(__last), __pred);
+    if (__maybe_first == nullopt)
+      return nullopt;
+
+    __first = *__maybe_first;
+    if (__first == __last)
+      return true;
+    ++__first;
+    using _NoneOf = __dispatch<__none_of, __current_configuration, _ExecutionPolicy>;
+    return _NoneOf()(__policy, std::move(__first), std::move(__last), __pred);
+  }
+};
+
+//////////////////////////////////////////////////////////////
+// for_each family
+//////////////////////////////////////////////////////////////
+template <class _ExecutionPolicy>
+struct __for_each_n<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Size, class _Function>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _Size __size, _Function __func) const noexcept {
+    if constexpr (__has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
+      using _ForEach          = __dispatch<__for_each, __current_configuration, _ExecutionPolicy>;
+      _ForwardIterator __last = __first + __size;
+      return _ForEach()(__policy, std::move(__first), std::move(__last), std::move(__func));
+    } else {
+      // Otherwise, use the serial algorithm to avoid doing two passes over the input
+      std::for_each_n(std::move(__first), __size, std::move(__func));
+      return __empty{};
+    }
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __fill<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Tp const& __value) const noexcept {
+    using _ForEach = __dispatch<__for_each, __current_configuration, _ExecutionPolicy>;
+    using _Ref     = __iter_reference<_ForwardIterator>;
+    return _ForEach()(__policy, std::move(__first), std::move(__last), [&](_Ref __element) { __element = __value; });
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __fill_n<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Size, class _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _Size __n, _Tp const& __value) const noexcept {
+    if constexpr (__has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
+      using _Fill             = __dispatch<__fill, __current_configuration, _ExecutionPolicy>;
+      _ForwardIterator __last = __first + __n;
+      return _Fill()(__policy, std::move(__first), std::move(__last), __value);
+    } else {
+      // Otherwise, use the serial algorithm to avoid doing two passes over the input
+      std::fill_n(std::move(__first), __n, __value);
+      return optional<__empty>{__empty{}};
+    }
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __replace<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Tp const& __old, _Tp const& __new)
+      const noexcept {
+    using _ReplaceIf = __dispatch<__replace_if, __current_configuration, _ExecutionPolicy>;
+    using _Ref       = __iter_reference<_ForwardIterator>;
+    return _ReplaceIf()(
+        __policy, std::move(__first), std::move(__last), [&](_Ref __element) { return __element == __old; }, __new);
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __replace_if<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Pred, class _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> operator()(
+      _Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred, _Tp const& __new_value)
+      const noexcept {
+    using _ForEach = __dispatch<__for_each, __current_configuration, _ExecutionPolicy>;
+    using _Ref     = __iter_reference<_ForwardIterator>;
+    return _ForEach()(__policy, std::move(__first), std::move(__last), [&](_Ref __element) {
+      if (__pred(__element))
+        __element = __new_value;
+    });
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __generate<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Generator>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Generator&& __gen) const noexcept {
+    using _ForEach = __dispatch<__for_each, __current_configuration, _ExecutionPolicy>;
+    using _Ref     = __iter_reference<_ForwardIterator>;
+    return _ForEach()(__policy, std::move(__first), std::move(__last), [&](_Ref __element) { __element = __gen(); });
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __generate_n<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Size, class _Generator>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _Size __n, _Generator&& __gen) const noexcept {
+    using _ForEachN = __dispatch<__for_each_n, __current_configuration, _ExecutionPolicy>;
+    using _Ref      = __iter_reference<_ForwardIterator>;
+    return _ForEachN()(__policy, std::move(__first), __n, [&](_Ref __element) { __element = __gen(); });
+  }
+};
+
+//////////////////////////////////////////////////////////////
+// stable_sort family
+//////////////////////////////////////////////////////////////
+template <class _ExecutionPolicy>
+struct __sort<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _RandomAccessIterator, class _Comp>
+  _LIBCPP_HIDE_FROM_ABI optional<__empty> operator()(
+      _Policy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp&& __comp) const noexcept {
+    using _StableSort = __dispatch<__stable_sort, __current_configuration, _ExecutionPolicy>;
+    return _StableSort()(__policy, std::move(__first), std::move(__last), std::forward<_Comp>(__comp));
+  }
+};
+
+//////////////////////////////////////////////////////////////
+// transform_reduce family
+//////////////////////////////////////////////////////////////
+template <class _ExecutionPolicy>
+struct __count_if<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Predicate>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__iter_diff_t<_ForwardIterator>> operator()(
+      _Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate&& __pred) const noexcept {
+    using _TransformReduce = __dispatch<__transform_reduce, __current_configuration, _ExecutionPolicy>;
+    using _DiffT           = __iter_diff_t<_ForwardIterator>;
+    using _Ref             = __iter_reference<_ForwardIterator>;
+    return _TransformReduce()(
+        __policy, std::move(__first), std::move(__last), _DiffT{}, std::plus{}, [&](_Ref __element) -> _DiffT {
+          return __pred(__element) ? _DiffT(1) : _DiffT(0);
+        });
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __count<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__iter_diff_t<_ForwardIterator>>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Tp const& __value) const noexcept {
+    using _CountIf = __dispatch<__count_if, __current_configuration, _ExecutionPolicy>;
+    using _Ref     = __iter_reference<_ForwardIterator>;
+    return _CountIf()(__policy, std::move(__first), std::move(__last), [&](_Ref __element) -> bool {
+      return __element == __value;
+    });
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __equal_3leg<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<bool>
+  operator()(_Policy&& __policy,
+             _ForwardIterator1 __first1,
+             _ForwardIterator1 __last1,
+             _ForwardIterator2 __first2,
+             _Predicate&& __pred) const noexcept {
+    using _TransformReduce = __dispatch<__transform_reduce_binary, __current_configuration, _ExecutionPolicy>;
+    return _TransformReduce()(
+        __policy,
+        std::move(__first1),
+        std::move(__last1),
+        std::move(__first2),
+        true,
+        std::logical_and{},
+        std::forward<_Predicate>(__pred));
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __equal<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<bool>
+  operator()(_Policy&& __policy,
+             _ForwardIterator1 __first1,
+             _ForwardIterator1 __last1,
+             _ForwardIterator2 __first2,
+             _ForwardIterator2 __last2,
+             _Predicate&& __pred) const noexcept {
+    if constexpr (__has_random_access_iterator_category<_ForwardIterator1>::value &&
+                  __has_random_access_iterator_category<_ForwardIterator2>::value) {
+      if (__last1 - __first1 != __last2 - __first2)
+        return false;
+      // Fall back to the 3 legged algorithm
+      using _Equal3Leg = __dispatch<__equal_3leg, __current_configuration, _ExecutionPolicy>;
+      return _Equal3Leg()(
+          __policy, std::move(__first1), std::move(__last1), std::move(__first2), std::forward<_Predicate>(__pred));
+    } else {
+      // If we don't have random access, fall back to the serial algorithm cause we can't do much
+      return std::equal(
+          std::move(__first1),
+          std::move(__last1),
+          std::move(__first2),
+          std::move(__last2),
+          std::forward<_Predicate>(__pred));
+    }
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __reduce<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Tp, class _BinaryOperation>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_Tp>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Tp __init, _BinaryOperation&& __op)
+      const noexcept {
+    using _TransformReduce = __dispatch<__transform_reduce, __current_configuration, _ExecutionPolicy>;
+    return _TransformReduce()(
+        __policy,
+        std::move(__first),
+        std::move(__last),
+        std::move(__init),
+        std::forward<_BinaryOperation>(__op),
+        __identity{});
+  }
+};
+
+//////////////////////////////////////////////////////////////
+// transform family
+//////////////////////////////////////////////////////////////
+template <class _ExecutionPolicy>
+struct __replace_copy_if<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _ForwardOutIterator, class _Pred, class _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty>
+  operator()(_Policy&& __policy,
+             _ForwardIterator __first,
+             _ForwardIterator __last,
+             _ForwardOutIterator __out_it,
+             _Pred&& __pred,
+             _Tp const& __new_value) const noexcept {
+    using _Transform = __dispatch<__transform, __current_configuration, _ExecutionPolicy>;
+    using _Ref       = __iter_reference<_ForwardIterator>;
+    auto __res =
+        _Transform()(__policy, std::move(__first), std::move(__last), std::move(__out_it), [&](_Ref __element) {
+          return __pred(__element) ? __new_value : __element;
+        });
+    if (__res == nullopt)
+      return nullopt;
+    return __empty{};
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __replace_copy<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _ForwardOutIterator, class _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty>
+  operator()(_Policy&& __policy,
+             _ForwardIterator __first,
+             _ForwardIterator __last,
+             _ForwardOutIterator __out_it,
+             _Tp const& __old_value,
+             _Tp const& __new_value) const noexcept {
+    using _ReplaceCopyIf = __dispatch<__replace_copy_if, __current_configuration, _ExecutionPolicy>;
+    using _Ref           = __iter_reference<_ForwardIterator>;
+    return _ReplaceCopyIf()(
+        __policy,
+        std::move(__first),
+        std::move(__last),
+        std::move(__out_it),
+        [&](_Ref __element) { return __element == __old_value; },
+        __new_value);
+  }
+};
+
+// TODO: Use the std::copy/move shenanigans to forward to std::memmove
+//       Investigate whether we want to still forward to std::transform(policy)
+//       in that case for the execution::par part, or whether we actually want
+//       to run everything serially in that case.
+template <class _ExecutionPolicy>
+struct __move<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _ForwardOutIterator>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _ForwardOutIterator __out_it)
+      const noexcept {
+    using _Transform = __dispatch<__transform, __current_configuration, _ExecutionPolicy>;
+    return _Transform()(__policy, std::move(__first), std::move(__last), std::move(__out_it), [&](auto&& __element) {
+      return std::move(__element);
+    });
+  }
+};
+
+// TODO: Use the std::copy/move shenanigans to forward to std::memmove
+template <class _ExecutionPolicy>
+struct __copy<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _ForwardOutIterator>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _ForwardOutIterator __out_it)
+      const noexcept {
+    using _Transform = __dispatch<__transform, __current_configuration, _ExecutionPolicy>;
+    return _Transform()(__policy, std::move(__first), std::move(__last), std::move(__out_it), __identity());
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __copy_n<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Size, class _ForwardOutIterator>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _Size __n, _ForwardOutIterator __out_it) const noexcept {
+    if constexpr (__has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
+      using _Copy             = __dispatch<__copy, __current_configuration, _ExecutionPolicy>;
+      _ForwardIterator __last = __first + __n;
+      return _Copy()(__policy, std::move(__first), std::move(__last), std::move(__out_it));
+    } else {
+      // Otherwise, use the serial algorithm to avoid doing two passes over the input
+      return std::copy_n(std::move(__first), __n, std::move(__out_it));
+    }
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __rotate_copy<__default_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _ForwardOutIterator>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator>
+  operator()(_Policy&& __policy,
+             _ForwardIterator __first,
+             _ForwardIterator __middle,
+             _ForwardIterator __last,
+             _ForwardOutIterator __out_it) const noexcept {
+    using _Copy       = __dispatch<__copy, __current_configuration, _ExecutionPolicy>;
+    auto __result_mid = _Copy()(__policy, __middle, std::move(__last), std::move(__out_it));
+    if (__result_mid == nullopt)
+      return nullopt;
+    return _Copy()(__policy, std::move(__first), std::move(__middle), *std::move(__result_mid));
+  }
+};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_BACKENDS_DEFAULT_H
diff --git a/libcxx/include/__cxx03/__pstl/backends/libdispatch.h b/libcxx/include/__cxx03/__pstl/backends/libdispatch.h
new file mode 100644
index 000000000000..a0c3ad980ed1
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/backends/libdispatch.h
@@ -0,0 +1,397 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_BACKENDS_LIBDISPATCH_H
+#define _LIBCPP___PSTL_BACKENDS_LIBDISPATCH_H
+
+#include <__algorithm/inplace_merge.h>
+#include <__algorithm/lower_bound.h>
+#include <__algorithm/max.h>
+#include <__algorithm/merge.h>
+#include <__algorithm/upper_bound.h>
+#include <__atomic/atomic.h>
+#include <__config>
+#include <__exception/terminate.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/move_iterator.h>
+#include <__memory/allocator.h>
+#include <__memory/construct_at.h>
+#include <__memory/unique_ptr.h>
+#include <__numeric/reduce.h>
+#include <__pstl/backend_fwd.h>
+#include <__pstl/cpu_algos/any_of.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
+#include <__pstl/cpu_algos/fill.h>
+#include <__pstl/cpu_algos/find_if.h>
+#include <__pstl/cpu_algos/for_each.h>
+#include <__pstl/cpu_algos/merge.h>
+#include <__pstl/cpu_algos/stable_sort.h>
+#include <__pstl/cpu_algos/transform.h>
+#include <__pstl/cpu_algos/transform_reduce.h>
+#include <__utility/empty.h>
+#include <__utility/exception_guard.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <cstddef>
+#include <new>
+#include <optional>
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+namespace __libdispatch {
+// ::dispatch_apply is marked as __attribute__((nothrow)) because it doesn't let exceptions propagate, and neither do
+// we.
+// TODO: Do we want to add [[_Clang::__callback__(__func, __context, __)]]?
+_LIBCPP_EXPORTED_FROM_ABI void
+__dispatch_apply(size_t __chunk_count, void* __context, void (*__func)(void* __context, size_t __chunk)) noexcept;
+
+template <class _Func>
+_LIBCPP_HIDE_FROM_ABI void __dispatch_apply(size_t __chunk_count, _Func __func) noexcept {
+  __libdispatch::__dispatch_apply(__chunk_count, &__func, [](void* __context, size_t __chunk) {
+    (*static_cast<_Func*>(__context))(__chunk);
+  });
+}
+
+struct __chunk_partitions {
+  ptrdiff_t __chunk_count_; // includes the first chunk
+  ptrdiff_t __chunk_size_;
+  ptrdiff_t __first_chunk_size_;
+};
+
+[[__gnu__::__const__]] _LIBCPP_EXPORTED_FROM_ABI __chunk_partitions __partition_chunks(ptrdiff_t __size) noexcept;
+
+template <class _RandomAccessIterator, class _Functor>
+_LIBCPP_HIDE_FROM_ABI optional<__empty>
+__dispatch_parallel_for(__chunk_partitions __partitions, _RandomAccessIterator __first, _Functor __func) {
+  // Perform the chunked execution.
+  __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __chunk) {
+    auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_;
+    auto __index =
+        __chunk == 0
+            ? 0
+            : (__chunk * __partitions.__chunk_size_) + (__partitions.__first_chunk_size_ - __partitions.__chunk_size_);
+    __func(__first + __index, __first + __index + __this_chunk_size);
+  });
+
+  return __empty{};
+}
+} // namespace __libdispatch
+
+template <>
+struct __cpu_traits<__libdispatch_backend_tag> {
+  template <class _RandomAccessIterator, class _Functor>
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+  __for_each(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func) {
+    return __libdispatch::__dispatch_parallel_for(
+        __libdispatch::__partition_chunks(__last - __first), std::move(__first), std::move(__func));
+  }
+
+  template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIteratorOut>
+  struct __merge_range {
+    __merge_range(_RandomAccessIterator1 __mid1, _RandomAccessIterator2 __mid2, _RandomAccessIteratorOut __result)
+        : __mid1_(__mid1), __mid2_(__mid2), __result_(__result) {}
+
+    _RandomAccessIterator1 __mid1_;
+    _RandomAccessIterator2 __mid2_;
+    _RandomAccessIteratorOut __result_;
+  };
+
+  template <typename _RandomAccessIterator1,
+            typename _RandomAccessIterator2,
+            typename _RandomAccessIterator3,
+            typename _Compare,
+            typename _LeafMerge>
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+  __merge(_RandomAccessIterator1 __first1,
+          _RandomAccessIterator1 __last1,
+          _RandomAccessIterator2 __first2,
+          _RandomAccessIterator2 __last2,
+          _RandomAccessIterator3 __result,
+          _Compare __comp,
+          _LeafMerge __leaf_merge) noexcept {
+    __libdispatch::__chunk_partitions __partitions =
+        __libdispatch::__partition_chunks(std::max<ptrdiff_t>(__last1 - __first1, __last2 - __first2));
+
+    if (__partitions.__chunk_count_ == 0)
+      return __empty{};
+
+    if (__partitions.__chunk_count_ == 1) {
+      __leaf_merge(__first1, __last1, __first2, __last2, __result, __comp);
+      return __empty{};
+    }
+
+    using __merge_range_t = __merge_range<_RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator3>;
+    auto const __n_ranges = __partitions.__chunk_count_ + 1;
+
+    // TODO: use __uninitialized_buffer
+    auto __destroy = [=](__merge_range_t* __ptr) {
+      std::destroy_n(__ptr, __n_ranges);
+      std::allocator<__merge_range_t>().deallocate(__ptr, __n_ranges);
+    };
+
+    unique_ptr<__merge_range_t[], decltype(__destroy)> __ranges(
+        [&]() -> __merge_range_t* {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+          try {
+#endif
+            return std::allocator<__merge_range_t>().allocate(__n_ranges);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+          } catch (const std::bad_alloc&) {
+            return nullptr;
+          }
+#endif
+        }(),
+        __destroy);
+
+    if (!__ranges)
+      return nullopt;
+
+    // TODO: Improve the case where the smaller range is merged into just a few (or even one) chunks of the larger case
+    __merge_range_t* __r = __ranges.get();
+    std::__construct_at(__r++, __first1, __first2, __result);
+
+    bool __iterate_first_range = __last1 - __first1 > __last2 - __first2;
+
+    auto __compute_chunk = [&](size_t __chunk_size) -> __merge_range_t {
+      auto [__mid1, __mid2] = [&] {
+        if (__iterate_first_range) {
+          auto __m1 = __first1 + __chunk_size;
+          auto __m2 = std::lower_bound(__first2, __last2, __m1[-1], __comp);
+          return std::make_pair(__m1, __m2);
+        } else {
+          auto __m2 = __first2 + __chunk_size;
+          auto __m1 = std::lower_bound(__first1, __last1, __m2[-1], __comp);
+          return std::make_pair(__m1, __m2);
+        }
+      }();
+
+      __result += (__mid1 - __first1) + (__mid2 - __first2);
+      __first1 = __mid1;
+      __first2 = __mid2;
+      return {std::move(__mid1), std::move(__mid2), __result};
+    };
+
+    // handle first chunk
+    std::__construct_at(__r++, __compute_chunk(__partitions.__first_chunk_size_));
+
+    // handle 2 -> N - 1 chunks
+    for (ptrdiff_t __i = 0; __i != __partitions.__chunk_count_ - 2; ++__i)
+      std::__construct_at(__r++, __compute_chunk(__partitions.__chunk_size_));
+
+    // handle last chunk
+    std::__construct_at(__r, __last1, __last2, __result);
+
+    __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __index) {
+      auto __first_iters = __ranges[__index];
+      auto __last_iters  = __ranges[__index + 1];
+      __leaf_merge(
+          __first_iters.__mid1_,
+          __last_iters.__mid1_,
+          __first_iters.__mid2_,
+          __last_iters.__mid2_,
+          __first_iters.__result_,
+          __comp);
+    });
+
+    return __empty{};
+  }
+
+  template <class _RandomAccessIterator, class _Transform, class _Value, class _Combiner, class _Reduction>
+  _LIBCPP_HIDE_FROM_ABI static optional<_Value> __transform_reduce(
+      _RandomAccessIterator __first,
+      _RandomAccessIterator __last,
+      _Transform __transform,
+      _Value __init,
+      _Combiner __combiner,
+      _Reduction __reduction) {
+    if (__first == __last)
+      return __init;
+
+    auto __partitions = __libdispatch::__partition_chunks(__last - __first);
+
+    auto __destroy = [__count = __partitions.__chunk_count_](_Value* __ptr) {
+      std::destroy_n(__ptr, __count);
+      std::allocator<_Value>().deallocate(__ptr, __count);
+    };
+
+    // TODO: use __uninitialized_buffer
+    // TODO: allocate one element per worker instead of one element per chunk
+    unique_ptr<_Value[], decltype(__destroy)> __values(
+        std::allocator<_Value>().allocate(__partitions.__chunk_count_), __destroy);
+
+    // __dispatch_apply is noexcept
+    __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __chunk) {
+      auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_;
+      auto __index           = __chunk == 0 ? 0
+                                            : (__chunk * __partitions.__chunk_size_) +
+                                        (__partitions.__first_chunk_size_ - __partitions.__chunk_size_);
+      if (__this_chunk_size != 1) {
+        std::__construct_at(
+            __values.get() + __chunk,
+            __reduction(__first + __index + 2,
+                        __first + __index + __this_chunk_size,
+                        __combiner(__transform(__first + __index), __transform(__first + __index + 1))));
+      } else {
+        std::__construct_at(__values.get() + __chunk, __transform(__first + __index));
+      }
+    });
+
+    return std::reduce(
+        std::make_move_iterator(__values.get()),
+        std::make_move_iterator(__values.get() + __partitions.__chunk_count_),
+        std::move(__init),
+        __combiner);
+  }
+
+  template <class _RandomAccessIterator, class _Comp, class _LeafSort>
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+  __stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp, _LeafSort __leaf_sort) {
+    const auto __size = __last - __first;
+    auto __partitions = __libdispatch::__partition_chunks(__size);
+
+    if (__partitions.__chunk_count_ == 0)
+      return __empty{};
+
+    if (__partitions.__chunk_count_ == 1) {
+      __leaf_sort(__first, __last, __comp);
+      return __empty{};
+    }
+
+    using _Value = __iter_value_type<_RandomAccessIterator>;
+
+    auto __destroy = [__size](_Value* __ptr) {
+      std::destroy_n(__ptr, __size);
+      std::allocator<_Value>().deallocate(__ptr, __size);
+    };
+
+    // TODO: use __uninitialized_buffer
+    unique_ptr<_Value[], decltype(__destroy)> __values(std::allocator<_Value>().allocate(__size), __destroy);
+
+    // Initialize all elements to a moved-from state
+    // TODO: Don't do this - this can be done in the first merge - see https://llvm.org/PR63928
+    std::__construct_at(__values.get(), std::move(*__first));
+    for (__iter_diff_t<_RandomAccessIterator> __i = 1; __i != __size; ++__i) {
+      std::__construct_at(__values.get() + __i, std::move(__values.get()[__i - 1]));
+    }
+    *__first = std::move(__values.get()[__size - 1]);
+
+    __libdispatch::__dispatch_parallel_for(
+        __partitions,
+        __first,
+        [&__leaf_sort, &__comp](_RandomAccessIterator __chunk_first, _RandomAccessIterator __chunk_last) {
+          __leaf_sort(std::move(__chunk_first), std::move(__chunk_last), __comp);
+        });
+
+    bool __objects_are_in_buffer = false;
+    do {
+      const auto __old_chunk_size = __partitions.__chunk_size_;
+      if (__partitions.__chunk_count_ % 2 == 1) {
+        auto __inplace_merge_chunks = [&__comp, &__partitions](auto __first_chunk_begin) {
+          std::inplace_merge(
+              __first_chunk_begin,
+              __first_chunk_begin + __partitions.__first_chunk_size_,
+              __first_chunk_begin + __partitions.__first_chunk_size_ + __partitions.__chunk_size_,
+              __comp);
+        };
+        if (__objects_are_in_buffer)
+          __inplace_merge_chunks(__values.get());
+        else
+          __inplace_merge_chunks(__first);
+        __partitions.__first_chunk_size_ += 2 * __partitions.__chunk_size_;
+      } else {
+        __partitions.__first_chunk_size_ += __partitions.__chunk_size_;
+      }
+
+      __partitions.__chunk_size_ *= 2;
+      __partitions.__chunk_count_ /= 2;
+
+      auto __merge_chunks = [__partitions, __old_chunk_size, &__comp](auto __from_first, auto __to_first) {
+        __libdispatch::__dispatch_parallel_for(
+            __partitions,
+            __from_first,
+            [__old_chunk_size, &__from_first, &__to_first, &__comp](auto __chunk_first, auto __chunk_last) {
+              std::merge(std::make_move_iterator(__chunk_first),
+                         std::make_move_iterator(__chunk_last - __old_chunk_size),
+                         std::make_move_iterator(__chunk_last - __old_chunk_size),
+                         std::make_move_iterator(__chunk_last),
+                         __to_first + (__chunk_first - __from_first),
+                         __comp);
+            });
+      };
+
+      if (__objects_are_in_buffer)
+        __merge_chunks(__values.get(), __first);
+      else
+        __merge_chunks(__first, __values.get());
+      __objects_are_in_buffer = !__objects_are_in_buffer;
+    } while (__partitions.__chunk_count_ > 1);
+
+    if (__objects_are_in_buffer) {
+      std::move(__values.get(), __values.get() + __size, __first);
+    }
+
+    return __empty{};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static void __cancel_execution() {}
+
+  static constexpr size_t __lane_size = 64;
+};
+
+// Mandatory implementations of the computational basis
+template <class _ExecutionPolicy>
+struct __find_if<__libdispatch_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_find_if<__libdispatch_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __for_each<__libdispatch_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_for_each<__libdispatch_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __merge<__libdispatch_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_merge<__libdispatch_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __stable_sort<__libdispatch_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_stable_sort<__libdispatch_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __transform<__libdispatch_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_transform<__libdispatch_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __transform_binary<__libdispatch_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_transform_binary<__libdispatch_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __transform_reduce<__libdispatch_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_transform_reduce<__libdispatch_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __transform_reduce_binary<__libdispatch_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_transform_reduce_binary<__libdispatch_backend_tag, _ExecutionPolicy> {};
+
+// Not mandatory, but better optimized
+template <class _ExecutionPolicy>
+struct __any_of<__libdispatch_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_any_of<__libdispatch_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __fill<__libdispatch_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_fill<__libdispatch_backend_tag, _ExecutionPolicy> {};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_BACKENDS_LIBDISPATCH_H
diff --git a/libcxx/include/__cxx03/__pstl/backends/serial.h b/libcxx/include/__cxx03/__pstl/backends/serial.h
new file mode 100644
index 000000000000..5f24499899bd
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/backends/serial.h
@@ -0,0 +1,181 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_BACKENDS_SERIAL_H
+#define _LIBCPP___PSTL_BACKENDS_SERIAL_H
+
+#include <__algorithm/find_if.h>
+#include <__algorithm/for_each.h>
+#include <__algorithm/merge.h>
+#include <__algorithm/stable_sort.h>
+#include <__algorithm/transform.h>
+#include <__config>
+#include <__numeric/transform_reduce.h>
+#include <__pstl/backend_fwd.h>
+#include <__utility/empty.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+//
+// This partial PSTL backend runs everything serially.
+//
+// TODO: Right now, the serial backend must be used with another backend
+//       like the "default backend" because it doesn't implement all the
+//       necessary PSTL operations. It would be better to dispatch all
+//       algorithms to their serial counterpart directly, since this can
+//       often be more efficient than the "default backend"'s implementation
+//       if we end up running serially anyways.
+//
+
+template <class _ExecutionPolicy>
+struct __find_if<__serial_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Pred>
+  _LIBCPP_HIDE_FROM_ABI optional<_ForwardIterator>
+  operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred) const noexcept {
+    return std::find_if(std::move(__first), std::move(__last), std::forward<_Pred>(__pred));
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __for_each<__serial_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Function>
+  _LIBCPP_HIDE_FROM_ABI optional<__empty>
+  operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Function&& __func) const noexcept {
+    std::for_each(std::move(__first), std::move(__last), std::forward<_Function>(__func));
+    return __empty{};
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __merge<__serial_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardOutIterator, class _Comp>
+  _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> operator()(
+      _Policy&&,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _ForwardIterator2 __last2,
+      _ForwardOutIterator __outit,
+      _Comp&& __comp) const noexcept {
+    return std::merge(
+        std::move(__first1),
+        std::move(__last1),
+        std::move(__first2),
+        std::move(__last2),
+        std::move(__outit),
+        std::forward<_Comp>(__comp));
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __stable_sort<__serial_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _RandomAccessIterator, class _Comp>
+  _LIBCPP_HIDE_FROM_ABI optional<__empty>
+  operator()(_Policy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp&& __comp) const noexcept {
+    std::stable_sort(std::move(__first), std::move(__last), std::forward<_Comp>(__comp));
+    return __empty{};
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __transform<__serial_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _ForwardOutIterator, class _UnaryOperation>
+  _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> operator()(
+      _Policy&&, _ForwardIterator __first, _ForwardIterator __last, _ForwardOutIterator __outit, _UnaryOperation&& __op)
+      const noexcept {
+    return std::transform(
+        std::move(__first), std::move(__last), std::move(__outit), std::forward<_UnaryOperation>(__op));
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __transform_binary<__serial_backend_tag, _ExecutionPolicy> {
+  template <class _Policy,
+            class _ForwardIterator1,
+            class _ForwardIterator2,
+            class _ForwardOutIterator,
+            class _BinaryOperation>
+  _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator>
+  operator()(_Policy&&,
+             _ForwardIterator1 __first1,
+             _ForwardIterator1 __last1,
+             _ForwardIterator2 __first2,
+             _ForwardOutIterator __outit,
+             _BinaryOperation&& __op) const noexcept {
+    return std::transform(
+        std::move(__first1),
+        std::move(__last1),
+        std::move(__first2),
+        std::move(__outit),
+        std::forward<_BinaryOperation>(__op));
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __transform_reduce<__serial_backend_tag, _ExecutionPolicy> {
+  template <class _Policy, class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
+  _LIBCPP_HIDE_FROM_ABI optional<_Tp>
+  operator()(_Policy&&,
+             _ForwardIterator __first,
+             _ForwardIterator __last,
+             _Tp __init,
+             _BinaryOperation&& __reduce,
+             _UnaryOperation&& __transform) const noexcept {
+    return std::transform_reduce(
+        std::move(__first),
+        std::move(__last),
+        std::move(__init),
+        std::forward<_BinaryOperation>(__reduce),
+        std::forward<_UnaryOperation>(__transform));
+  }
+};
+
+template <class _ExecutionPolicy>
+struct __transform_reduce_binary<__serial_backend_tag, _ExecutionPolicy> {
+  template <class _Policy,
+            class _ForwardIterator1,
+            class _ForwardIterator2,
+            class _Tp,
+            class _BinaryOperation1,
+            class _BinaryOperation2>
+  _LIBCPP_HIDE_FROM_ABI optional<_Tp> operator()(
+      _Policy&&,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _Tp __init,
+      _BinaryOperation1&& __reduce,
+      _BinaryOperation2&& __transform) const noexcept {
+    return std::transform_reduce(
+        std::move(__first1),
+        std::move(__last1),
+        std::move(__first2),
+        std::move(__init),
+        std::forward<_BinaryOperation1>(__reduce),
+        std::forward<_BinaryOperation2>(__transform));
+  }
+};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_BACKENDS_SERIAL_H
diff --git a/libcxx/include/__cxx03/__pstl/backends/std_thread.h b/libcxx/include/__cxx03/__pstl/backends/std_thread.h
new file mode 100644
index 000000000000..49570bd30b08
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/backends/std_thread.h
@@ -0,0 +1,136 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_BACKENDS_STD_THREAD_H
+#define _LIBCPP___PSTL_BACKENDS_STD_THREAD_H
+
+#include <__config>
+#include <__pstl/backend_fwd.h>
+#include <__pstl/cpu_algos/any_of.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
+#include <__pstl/cpu_algos/fill.h>
+#include <__pstl/cpu_algos/find_if.h>
+#include <__pstl/cpu_algos/for_each.h>
+#include <__pstl/cpu_algos/merge.h>
+#include <__pstl/cpu_algos/stable_sort.h>
+#include <__pstl/cpu_algos/transform.h>
+#include <__pstl/cpu_algos/transform_reduce.h>
+#include <__utility/empty.h>
+#include <__utility/move.h>
+#include <cstddef>
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+//
+// This partial backend implementation is for testing purposes only and not meant for production use. This will be
+// replaced by a proper implementation once the PSTL implementation is somewhat stable.
+//
+// This is intended to be used on top of the "default backend".
+//
+
+template <>
+struct __cpu_traits<__std_thread_backend_tag> {
+  template <class _RandomAccessIterator, class _Fp>
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+  __for_each(_RandomAccessIterator __first, _RandomAccessIterator __last, _Fp __f) {
+    __f(__first, __last);
+    return __empty{};
+  }
+
+  template <class _Index, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduce>
+  _LIBCPP_HIDE_FROM_ABI static optional<_Tp>
+  __transform_reduce(_Index __first, _Index __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduce __reduce) {
+    return __reduce(std::move(__first), std::move(__last), std::move(__init));
+  }
+
+  template <class _RandomAccessIterator, class _Compare, class _LeafSort>
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+  __stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort) {
+    __leaf_sort(__first, __last, __comp);
+    return __empty{};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static void __cancel_execution() {}
+
+  template <class _RandomAccessIterator1,
+            class _RandomAccessIterator2,
+            class _RandomAccessIterator3,
+            class _Compare,
+            class _LeafMerge>
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+  __merge(_RandomAccessIterator1 __first1,
+          _RandomAccessIterator1 __last1,
+          _RandomAccessIterator2 __first2,
+          _RandomAccessIterator2 __last2,
+          _RandomAccessIterator3 __outit,
+          _Compare __comp,
+          _LeafMerge __leaf_merge) {
+    __leaf_merge(__first1, __last1, __first2, __last2, __outit, __comp);
+    return __empty{};
+  }
+
+  static constexpr size_t __lane_size = 64;
+};
+
+// Mandatory implementations of the computational basis
+template <class _ExecutionPolicy>
+struct __find_if<__std_thread_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_find_if<__std_thread_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __for_each<__std_thread_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_for_each<__std_thread_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __merge<__std_thread_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_merge<__std_thread_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __stable_sort<__std_thread_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_stable_sort<__std_thread_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __transform<__std_thread_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_transform<__std_thread_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __transform_binary<__std_thread_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_transform_binary<__std_thread_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __transform_reduce<__std_thread_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_transform_reduce<__std_thread_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __transform_reduce_binary<__std_thread_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_transform_reduce_binary<__std_thread_backend_tag, _ExecutionPolicy> {};
+
+// Not mandatory, but better optimized
+template <class _ExecutionPolicy>
+struct __any_of<__std_thread_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_any_of<__std_thread_backend_tag, _ExecutionPolicy> {};
+
+template <class _ExecutionPolicy>
+struct __fill<__std_thread_backend_tag, _ExecutionPolicy>
+    : __cpu_parallel_fill<__std_thread_backend_tag, _ExecutionPolicy> {};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_BACKENDS_STD_THREAD_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/any_of.h b/libcxx/include/__cxx03/__pstl/cpu_algos/any_of.h
new file mode 100644
index 000000000000..b33c787a29db
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/any_of.h
@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_CPU_ALGOS_ANY_OF_H
+#define _LIBCPP___PSTL_CPU_ALGOS_ANY_OF_H
+
+#include <__algorithm/any_of.h>
+#include <__assert>
+#include <__atomic/atomic.h>
+#include <__atomic/memory_order.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__pstl/backend_fwd.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
+#include <__type_traits/is_execution_policy.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <cstdint>
+#include <optional>
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+template <class _Backend, class _Index, class _Brick>
+_LIBCPP_HIDE_FROM_ABI optional<bool> __parallel_or(_Index __first, _Index __last, _Brick __f) {
+  std::atomic<bool> __found(false);
+  auto __ret = __cpu_traits<_Backend>::__for_each(__first, __last, [__f, &__found](_Index __i, _Index __j) {
+    if (!__found.load(std::memory_order_relaxed) && __f(__i, __j)) {
+      __found.store(true, std::memory_order_relaxed);
+      __cpu_traits<_Backend>::__cancel_execution();
+    }
+  });
+  if (!__ret)
+    return nullopt;
+  return static_cast<bool>(__found);
+}
+
+// TODO: check whether __simd_first() can be used here
+template <class _Index, class _DifferenceType, class _Pred>
+_LIBCPP_HIDE_FROM_ABI bool __simd_or(_Index __first, _DifferenceType __n, _Pred __pred) noexcept {
+  _DifferenceType __block_size = 4 < __n ? 4 : __n;
+  const _Index __last          = __first + __n;
+  while (__last != __first) {
+    int32_t __flag = 1;
+    _PSTL_PRAGMA_SIMD_REDUCTION(& : __flag)
+    for (_DifferenceType __i = 0; __i < __block_size; ++__i)
+      if (__pred(*(__first + __i)))
+        __flag = 0;
+    if (!__flag)
+      return true;
+
+    __first += __block_size;
+    if (__last - __first >= __block_size << 1) {
+      // Double the block _Size.  Any unnecessary iterations can be amortized against work done so far.
+      __block_size <<= 1;
+    } else {
+      __block_size = __last - __first;
+    }
+  }
+  return false;
+}
+
+template <class _Backend, class _RawExecutionPolicy>
+struct __cpu_parallel_any_of {
+  template <class _Policy, class _ForwardIterator, class _Predicate>
+  _LIBCPP_HIDE_FROM_ABI optional<bool>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept {
+    if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> &&
+                  __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
+      return __pstl::__parallel_or<_Backend>(
+          __first, __last, [&__policy, &__pred](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
+            using _AnyOfUnseq = __pstl::__any_of<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>;
+            auto __res = _AnyOfUnseq()(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __pred);
+            _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!");
+            return *std::move(__res);
+          });
+    } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> &&
+                         __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
+      return __pstl::__simd_or(__first, __last - __first, __pred);
+    } else {
+      return std::any_of(__first, __last, __pred);
+    }
+  }
+};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_CPU_ALGOS_ANY_OF_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/cpu_traits.h b/libcxx/include/__cxx03/__pstl/cpu_algos/cpu_traits.h
new file mode 100644
index 000000000000..0483d6918fd0
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/cpu_traits.h
@@ -0,0 +1,86 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_CPU_ALGOS_CPU_TRAITS_H
+#define _LIBCPP___PSTL_CPU_ALGOS_CPU_TRAITS_H
+
+#include <__config>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+// __cpu_traits
+//
+// This traits class encapsulates the basis operations for a CPU-based implementation of the PSTL.
+// All the operations in the PSTL can be implemented from these basis operations, so a pure CPU backend
+// only needs to customize these traits in order to get an implementation of the whole PSTL.
+//
+// Basis operations
+// ================
+//
+//  template <class _RandomAccessIterator, class _Functor>
+//  optional<__empty> __for_each(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func);
+//    - __func must take a subrange of [__first, __last) that should be executed in serial
+//
+//  template <class _Iterator, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduction>
+//  optional<_Tp> __transform_reduce(_Iterator __first, _Iterator __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduction);
+//
+//  template <class _RandomAccessIterator1,
+//            class _RandomAccessIterator2,
+//            class _RandomAccessIterator3,
+//            class _Compare,
+//            class _LeafMerge>
+//  optional<_RandomAccessIterator3> __merge(_RandomAccessIterator1 __first1,
+//                                           _RandomAccessIterator1 __last1,
+//                                           _RandomAccessIterator2 __first2,
+//                                           _RandomAccessIterator2 __last2,
+//                                           _RandomAccessIterator3 __outit,
+//                                           _Compare __comp,
+//                                           _LeafMerge __leaf_merge);
+//
+//  template <class _RandomAccessIterator, class _Comp, class _LeafSort>
+//  optional<__empty> __stable_sort(_RandomAccessIterator __first,
+//                                  _RandomAccessIterator __last,
+//                                  _Comp __comp,
+//                                  _LeafSort __leaf_sort);
+//
+//   void __cancel_execution();
+//      Cancel the execution of other jobs - they aren't needed anymore. This is not a binding request,
+//      some backends may not actually be able to cancel jobs.
+//
+//   constexpr size_t __lane_size;
+//      Size of SIMD lanes.
+//      TODO: Merge this with __native_vector_size from __algorithm/simd_utils.h
+//
+//
+// Exception handling
+// ==================
+//
+// CPU backends are expected to report errors (i.e. failure to allocate) by returning a disengaged `optional` from their
+// implementation. Exceptions shouldn't be used to report an internal failure-to-allocate, since all exceptions are
+// turned into a program termination at the front-end level. When a backend returns a disengaged `optional` to the
+// frontend, the frontend will turn that into a call to `std::__throw_bad_alloc();` to report the internal failure to
+// the user.
+
+template <class _Backend>
+struct __cpu_traits;
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_CPU_ALGOS_CPU_TRAITS_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/fill.h b/libcxx/include/__cxx03/__pstl/cpu_algos/fill.h
new file mode 100644
index 000000000000..4e6d29b30cc6
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/fill.h
@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_CPU_ALGOS_FILL_H
+#define _LIBCPP___PSTL_CPU_ALGOS_FILL_H
+
+#include <__algorithm/fill.h>
+#include <__assert>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__pstl/backend_fwd.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
+#include <__type_traits/is_execution_policy.h>
+#include <__utility/empty.h>
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+template <class _Index, class _DifferenceType, class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Index __simd_fill_n(_Index __first, _DifferenceType __n, const _Tp& __value) noexcept {
+  _PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED
+  _PSTL_PRAGMA_SIMD
+  for (_DifferenceType __i = 0; __i < __n; ++__i)
+    __first[__i] = __value;
+  return __first + __n;
+}
+
+template <class _Backend, class _RawExecutionPolicy>
+struct __cpu_parallel_fill {
+  template <class _Policy, class _ForwardIterator, class _Tp>
+  _LIBCPP_HIDE_FROM_ABI optional<__empty>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) const noexcept {
+    if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> &&
+                  __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
+      return __cpu_traits<_Backend>::__for_each(
+          __first, __last, [&__policy, &__value](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
+            using _FillUnseq = __pstl::__fill<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>;
+            [[maybe_unused]] auto __res =
+                _FillUnseq()(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __value);
+            _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!");
+          });
+    } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> &&
+                         __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
+      __pstl::__simd_fill_n(__first, __last - __first, __value);
+      return __empty{};
+    } else {
+      std::fill(__first, __last, __value);
+      return __empty{};
+    }
+  }
+};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___PSTL_CPU_ALGOS_FILL_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/find_if.h b/libcxx/include/__cxx03/__pstl/cpu_algos/find_if.h
new file mode 100644
index 000000000000..12b2e88971df
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/find_if.h
@@ -0,0 +1,137 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_CPU_ALGOS_FIND_IF_H
+#define _LIBCPP___PSTL_CPU_ALGOS_FIND_IF_H
+
+#include <__algorithm/find_if.h>
+#include <__assert>
+#include <__atomic/atomic.h>
+#include <__config>
+#include <__functional/operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__pstl/backend_fwd.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
+#include <__type_traits/is_execution_policy.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <cstddef>
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+template <class _Backend, class _Index, class _Brick, class _Compare>
+_LIBCPP_HIDE_FROM_ABI optional<_Index>
+__parallel_find(_Index __first, _Index __last, _Brick __f, _Compare __comp, bool __b_first) {
+  typedef typename std::iterator_traits<_Index>::difference_type _DifferenceType;
+  const _DifferenceType __n      = __last - __first;
+  _DifferenceType __initial_dist = __b_first ? __n : -1;
+  std::atomic<_DifferenceType> __extremum(__initial_dist);
+  // TODO: find out what is better here: parallel_for or parallel_reduce
+  auto __res =
+      __cpu_traits<_Backend>::__for_each(__first, __last, [__comp, __f, __first, &__extremum](_Index __i, _Index __j) {
+        // See "Reducing Contention Through Priority Updates", PPoPP '13, for discussion of
+        // why using a shared variable scales fairly well in this situation.
+        if (__comp(__i - __first, __extremum)) {
+          _Index __result = __f(__i, __j);
+          // If not '__last' returned then we found what we want so put this to extremum
+          if (__result != __j) {
+            const _DifferenceType __k = __result - __first;
+            for (_DifferenceType __old = __extremum; __comp(__k, __old); __old = __extremum) {
+              __extremum.compare_exchange_weak(__old, __k);
+            }
+          }
+        }
+      });
+  if (!__res)
+    return nullopt;
+  return __extremum.load() != __initial_dist ? __first + __extremum.load() : __last;
+}
+
+template <class _Backend, class _Index, class _DifferenceType, class _Compare>
+_LIBCPP_HIDE_FROM_ABI _Index
+__simd_first(_Index __first, _DifferenceType __begin, _DifferenceType __end, _Compare __comp) noexcept {
+  // Experiments show good block sizes like this
+  const _DifferenceType __block_size                                                = 8;
+  alignas(__cpu_traits<_Backend>::__lane_size) _DifferenceType __lane[__block_size] = {0};
+  while (__end - __begin >= __block_size) {
+    _DifferenceType __found = 0;
+    _PSTL_PRAGMA_SIMD_REDUCTION(| : __found) for (_DifferenceType __i = __begin; __i < __begin + __block_size; ++__i) {
+      const _DifferenceType __t = __comp(__first, __i);
+      __lane[__i - __begin]     = __t;
+      __found |= __t;
+    }
+    if (__found) {
+      _DifferenceType __i;
+      // This will vectorize
+      for (__i = 0; __i < __block_size; ++__i) {
+        if (__lane[__i]) {
+          break;
+        }
+      }
+      return __first + __begin + __i;
+    }
+    __begin += __block_size;
+  }
+
+  // Keep remainder scalar
+  while (__begin != __end) {
+    if (__comp(__first, __begin)) {
+      return __first + __begin;
+    }
+    ++__begin;
+  }
+  return __first + __end;
+}
+
+template <class _Backend, class _RawExecutionPolicy>
+struct __cpu_parallel_find_if {
+  template <class _Policy, class _ForwardIterator, class _Predicate>
+  _LIBCPP_HIDE_FROM_ABI optional<_ForwardIterator>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept {
+    if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> &&
+                  __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
+      return __pstl::__parallel_find<_Backend>(
+          __first,
+          __last,
+          [&__policy, &__pred](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
+            using _FindIfUnseq = __pstl::__find_if<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>;
+            auto __res = _FindIfUnseq()(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __pred);
+            _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!");
+            return *std::move(__res);
+          },
+          less<>{},
+          true);
+    } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> &&
+                         __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
+      using __diff_t = __iter_diff_t<_ForwardIterator>;
+      return __pstl::__simd_first<_Backend>(
+          __first, __diff_t(0), __last - __first, [&__pred](_ForwardIterator __iter, __diff_t __i) {
+            return __pred(__iter[__i]);
+          });
+    } else {
+      return std::find_if(__first, __last, __pred);
+    }
+  }
+};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_CPU_ALGOS_FIND_IF_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/for_each.h b/libcxx/include/__cxx03/__pstl/cpu_algos/for_each.h
new file mode 100644
index 000000000000..d4d7862135ff
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/for_each.h
@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_CPU_ALGOS_FOR_EACH_H
+#define _LIBCPP___PSTL_CPU_ALGOS_FOR_EACH_H
+
+#include <__algorithm/for_each.h>
+#include <__assert>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__pstl/backend_fwd.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
+#include <__type_traits/is_execution_policy.h>
+#include <__utility/empty.h>
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+template <class _Iterator, class _DifferenceType, class _Function>
+_LIBCPP_HIDE_FROM_ABI _Iterator __simd_for_each(_Iterator __first, _DifferenceType __n, _Function __f) noexcept {
+  _PSTL_PRAGMA_SIMD
+  for (_DifferenceType __i = 0; __i < __n; ++__i)
+    __f(__first[__i]);
+
+  return __first + __n;
+}
+
+template <class _Backend, class _RawExecutionPolicy>
+struct __cpu_parallel_for_each {
+  template <class _Policy, class _ForwardIterator, class _Function>
+  _LIBCPP_HIDE_FROM_ABI optional<__empty>
+  operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Function __func) const noexcept {
+    if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> &&
+                  __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
+      return __cpu_traits<_Backend>::__for_each(
+          __first, __last, [&__policy, __func](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
+            using _ForEachUnseq = __pstl::__for_each<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>;
+            [[maybe_unused]] auto __res =
+                _ForEachUnseq()(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __func);
+            _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!");
+          });
+    } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> &&
+                         __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
+      __pstl::__simd_for_each(__first, __last - __first, __func);
+      return __empty{};
+    } else {
+      std::for_each(__first, __last, __func);
+      return __empty{};
+    }
+  }
+};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___PSTL_CPU_ALGOS_FOR_EACH_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/merge.h b/libcxx/include/__cxx03/__pstl/cpu_algos/merge.h
new file mode 100644
index 000000000000..dfa4cbf69b14
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/merge.h
@@ -0,0 +1,85 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_CPU_ALGOS_MERGE_H
+#define _LIBCPP___PSTL_CPU_ALGOS_MERGE_H
+
+#include <__algorithm/merge.h>
+#include <__assert>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__pstl/backend_fwd.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
+#include <__type_traits/is_execution_policy.h>
+#include <__utility/move.h>
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+template <class _Backend, class _RawExecutionPolicy>
+struct __cpu_parallel_merge {
+  template <class _Policy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardOutIterator, class _Comp>
+  _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> operator()(
+      _Policy&& __policy,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _ForwardIterator2 __last2,
+      _ForwardOutIterator __result,
+      _Comp __comp) const noexcept {
+    if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> &&
+                  __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
+                  __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value &&
+                  __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) {
+      auto __res = __cpu_traits<_Backend>::__merge(
+          __first1,
+          __last1,
+          __first2,
+          __last2,
+          __result,
+          __comp,
+          [&__policy](_ForwardIterator1 __g_first1,
+                      _ForwardIterator1 __g_last1,
+                      _ForwardIterator2 __g_first2,
+                      _ForwardIterator2 __g_last2,
+                      _ForwardOutIterator __g_result,
+                      _Comp __g_comp) {
+            using _MergeUnseq             = __pstl::__merge<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>;
+            [[maybe_unused]] auto __g_res = _MergeUnseq()(
+                std::__remove_parallel_policy(__policy),
+                std::move(__g_first1),
+                std::move(__g_last1),
+                std::move(__g_first2),
+                std::move(__g_last2),
+                std::move(__g_result),
+                std::move(__g_comp));
+            _LIBCPP_ASSERT_INTERNAL(__g_res, "unsed/sed should never try to allocate!");
+          });
+      if (!__res)
+        return nullopt;
+      return __result + (__last1 - __first1) + (__last2 - __first2);
+    } else {
+      return std::merge(__first1, __last1, __first2, __last2, __result, __comp);
+    }
+  }
+};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_CPU_ALGOS_MERGE_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/stable_sort.h b/libcxx/include/__cxx03/__pstl/cpu_algos/stable_sort.h
new file mode 100644
index 000000000000..8e64f3e537c0
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/stable_sort.h
@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_CPU_ALGOS_STABLE_SORT_H
+#define _LIBCPP___PSTL_CPU_ALGOS_STABLE_SORT_H
+
+#include <__algorithm/stable_sort.h>
+#include <__config>
+#include <__pstl/backend_fwd.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
+#include <__type_traits/is_execution_policy.h>
+#include <__utility/empty.h>
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+template <class _Backend, class _RawExecutionPolicy>
+struct __cpu_parallel_stable_sort {
+  template <class _Policy, class _RandomAccessIterator, class _Comp>
+  _LIBCPP_HIDE_FROM_ABI optional<__empty>
+  operator()(_Policy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) const noexcept {
+    if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy>) {
+      return __cpu_traits<_Backend>::__stable_sort(
+          __first, __last, __comp, [](_RandomAccessIterator __g_first, _RandomAccessIterator __g_last, _Comp __g_comp) {
+            std::stable_sort(__g_first, __g_last, __g_comp);
+          });
+    } else {
+      std::stable_sort(__first, __last, __comp);
+      return __empty{};
+    }
+  }
+};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___PSTL_CPU_ALGOS_STABLE_SORT_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/transform.h b/libcxx/include/__cxx03/__pstl/cpu_algos/transform.h
new file mode 100644
index 000000000000..27ce8e27b242
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/transform.h
@@ -0,0 +1,153 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_H
+#define _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_H
+
+#include <__algorithm/transform.h>
+#include <__assert>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__pstl/backend_fwd.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
+#include <__type_traits/is_execution_policy.h>
+#include <__utility/move.h>
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+template <class _Iterator1, class _DifferenceType, class _Iterator2, class _Function>
+_LIBCPP_HIDE_FROM_ABI _Iterator2
+__simd_transform(_Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _Function __f) noexcept {
+  _PSTL_PRAGMA_SIMD
+  for (_DifferenceType __i = 0; __i < __n; ++__i)
+    __f(__first1[__i], __first2[__i]);
+  return __first2 + __n;
+}
+
+template <class _Iterator1, class _DifferenceType, class _Iterator2, class _Iterator3, class _Function>
+_LIBCPP_HIDE_FROM_ABI _Iterator3 __simd_transform(
+    _Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _Iterator3 __first3, _Function __f) noexcept {
+  _PSTL_PRAGMA_SIMD
+  for (_DifferenceType __i = 0; __i < __n; ++__i)
+    __f(__first1[__i], __first2[__i], __first3[__i]);
+  return __first3 + __n;
+}
+
+template <class _Backend, class _RawExecutionPolicy>
+struct __cpu_parallel_transform {
+  template <class _Policy, class _ForwardIterator, class _ForwardOutIterator, class _UnaryOperation>
+  _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator>
+  operator()(_Policy&& __policy,
+             _ForwardIterator __first,
+             _ForwardIterator __last,
+             _ForwardOutIterator __result,
+             _UnaryOperation __op) const noexcept {
+    if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> &&
+                  __has_random_access_iterator_category_or_concept<_ForwardIterator>::value &&
+                  __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) {
+      __cpu_traits<_Backend>::__for_each(
+          __first,
+          __last,
+          [&__policy, __op, __first, __result](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
+            using _TransformUnseq = __pstl::__transform<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>;
+            auto __res            = _TransformUnseq()(
+                std::__remove_parallel_policy(__policy),
+                __brick_first,
+                __brick_last,
+                __result + (__brick_first - __first),
+                __op);
+            _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!");
+            return *std::move(__res);
+          });
+      return __result + (__last - __first);
+    } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> &&
+                         __has_random_access_iterator_category_or_concept<_ForwardIterator>::value &&
+                         __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) {
+      return __pstl::__simd_transform(
+          __first,
+          __last - __first,
+          __result,
+          [&](__iter_reference<_ForwardIterator> __in_value, __iter_reference<_ForwardOutIterator> __out_value) {
+            __out_value = __op(__in_value);
+          });
+    } else {
+      return std::transform(__first, __last, __result, __op);
+    }
+  }
+};
+
+template <class _Backend, class _RawExecutionPolicy>
+struct __cpu_parallel_transform_binary {
+  template <class _Policy,
+            class _ForwardIterator1,
+            class _ForwardIterator2,
+            class _ForwardOutIterator,
+            class _BinaryOperation>
+  _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator>
+  operator()(_Policy&& __policy,
+             _ForwardIterator1 __first1,
+             _ForwardIterator1 __last1,
+             _ForwardIterator2 __first2,
+             _ForwardOutIterator __result,
+             _BinaryOperation __op) const noexcept {
+    if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> &&
+                  __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
+                  __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value &&
+                  __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) {
+      auto __res = __cpu_traits<_Backend>::__for_each(
+          __first1,
+          __last1,
+          [&__policy, __op, __first1, __first2, __result](
+              _ForwardIterator1 __brick_first, _ForwardIterator1 __brick_last) {
+            using _TransformBinaryUnseq =
+                __pstl::__transform_binary<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>;
+            return _TransformBinaryUnseq()(
+                std::__remove_parallel_policy(__policy),
+                __brick_first,
+                __brick_last,
+                __first2 + (__brick_first - __first1),
+                __result + (__brick_first - __first1),
+                __op);
+          });
+      if (!__res)
+        return nullopt;
+      return __result + (__last1 - __first1);
+    } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> &&
+                         __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
+                         __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value &&
+                         __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) {
+      return __pstl::__simd_transform(
+          __first1,
+          __last1 - __first1,
+          __first2,
+          __result,
+          [&](__iter_reference<_ForwardIterator1> __in1,
+              __iter_reference<_ForwardIterator2> __in2,
+              __iter_reference<_ForwardOutIterator> __out_value) { __out_value = __op(__in1, __in2); });
+    } else {
+      return std::transform(__first1, __last1, __first2, __result, __op);
+    }
+  }
+};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/transform_reduce.h b/libcxx/include/__cxx03/__pstl/cpu_algos/transform_reduce.h
new file mode 100644
index 000000000000..36ac1a9072a8
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/transform_reduce.h
@@ -0,0 +1,216 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_REDUCE_H
+#define _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_REDUCE_H
+
+#include <__assert>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__numeric/transform_reduce.h>
+#include <__pstl/backend_fwd.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
+#include <__type_traits/desugars_to.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_execution_policy.h>
+#include <__utility/move.h>
+#include <cstddef>
+#include <new>
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+template <typename _Backend,
+          typename _DifferenceType,
+          typename _Tp,
+          typename _BinaryOperation,
+          typename _UnaryOperation,
+          typename _UnaryResult = invoke_result_t<_UnaryOperation, _DifferenceType>,
+          __enable_if_t<__desugars_to_v<__plus_tag, _BinaryOperation, _Tp, _UnaryResult> && is_arithmetic_v<_Tp> &&
+                            is_arithmetic_v<_UnaryResult>,
+                        int>    = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__simd_transform_reduce(_DifferenceType __n, _Tp __init, _BinaryOperation, _UnaryOperation __f) noexcept {
+  _PSTL_PRAGMA_SIMD_REDUCTION(+ : __init)
+  for (_DifferenceType __i = 0; __i < __n; ++__i)
+    __init += __f(__i);
+  return __init;
+}
+
+template <typename _Backend,
+          typename _Size,
+          typename _Tp,
+          typename _BinaryOperation,
+          typename _UnaryOperation,
+          typename _UnaryResult = invoke_result_t<_UnaryOperation, _Size>,
+          __enable_if_t<!(__desugars_to_v<__plus_tag, _BinaryOperation, _Tp, _UnaryResult> && is_arithmetic_v<_Tp> &&
+                          is_arithmetic_v<_UnaryResult>),
+                        int>    = 0>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__simd_transform_reduce(_Size __n, _Tp __init, _BinaryOperation __binary_op, _UnaryOperation __f) noexcept {
+  constexpr size_t __lane_size = __cpu_traits<_Backend>::__lane_size;
+  const _Size __block_size     = __lane_size / sizeof(_Tp);
+  if (__n > 2 * __block_size && __block_size > 1) {
+    alignas(__lane_size) char __lane_buffer[__lane_size];
+    _Tp* __lane = reinterpret_cast<_Tp*>(__lane_buffer);
+
+    // initializer
+    _PSTL_PRAGMA_SIMD
+    for (_Size __i = 0; __i < __block_size; ++__i) {
+      ::new (__lane + __i) _Tp(__binary_op(__f(__i), __f(__block_size + __i)));
+    }
+    // main loop
+    _Size __i                    = 2 * __block_size;
+    const _Size __last_iteration = __block_size * (__n / __block_size);
+    for (; __i < __last_iteration; __i += __block_size) {
+      _PSTL_PRAGMA_SIMD
+      for (_Size __j = 0; __j < __block_size; ++__j) {
+        __lane[__j] = __binary_op(std::move(__lane[__j]), __f(__i + __j));
+      }
+    }
+    // remainder
+    _PSTL_PRAGMA_SIMD
+    for (_Size __j = 0; __j < __n - __last_iteration; ++__j) {
+      __lane[__j] = __binary_op(std::move(__lane[__j]), __f(__last_iteration + __j));
+    }
+    // combiner
+    for (_Size __j = 0; __j < __block_size; ++__j) {
+      __init = __binary_op(std::move(__init), std::move(__lane[__j]));
+    }
+    // destroyer
+    _PSTL_PRAGMA_SIMD
+    for (_Size __j = 0; __j < __block_size; ++__j) {
+      __lane[__j].~_Tp();
+    }
+  } else {
+    for (_Size __i = 0; __i < __n; ++__i) {
+      __init = __binary_op(std::move(__init), __f(__i));
+    }
+  }
+  return __init;
+}
+
+template <class _Backend, class _RawExecutionPolicy>
+struct __cpu_parallel_transform_reduce_binary {
+  template <class _Policy,
+            class _ForwardIterator1,
+            class _ForwardIterator2,
+            class _Tp,
+            class _BinaryOperation1,
+            class _BinaryOperation2>
+  _LIBCPP_HIDE_FROM_ABI optional<_Tp> operator()(
+      _Policy&& __policy,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _Tp __init,
+      _BinaryOperation1 __reduce,
+      _BinaryOperation2 __transform) const noexcept {
+    if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> &&
+                  __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
+                  __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value) {
+      return __cpu_traits<_Backend>::__transform_reduce(
+          __first1,
+          std::move(__last1),
+          [__first1, __first2, __transform](_ForwardIterator1 __iter) {
+            return __transform(*__iter, *(__first2 + (__iter - __first1)));
+          },
+          std::move(__init),
+          std::move(__reduce),
+          [&__policy, __first1, __first2, __reduce, __transform](
+              _ForwardIterator1 __brick_first, _ForwardIterator1 __brick_last, _Tp __brick_init) {
+            using _TransformReduceBinaryUnseq =
+                __pstl::__transform_reduce_binary<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>;
+            return *_TransformReduceBinaryUnseq()(
+                std::__remove_parallel_policy(__policy),
+                __brick_first,
+                std::move(__brick_last),
+                __first2 + (__brick_first - __first1),
+                std::move(__brick_init),
+                std::move(__reduce),
+                std::move(__transform));
+          });
+    } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> &&
+                         __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
+                         __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value) {
+      return __pstl::__simd_transform_reduce<_Backend>(
+          __last1 - __first1, std::move(__init), std::move(__reduce), [&](__iter_diff_t<_ForwardIterator1> __i) {
+            return __transform(__first1[__i], __first2[__i]);
+          });
+    } else {
+      return std::transform_reduce(
+          std::move(__first1),
+          std::move(__last1),
+          std::move(__first2),
+          std::move(__init),
+          std::move(__reduce),
+          std::move(__transform));
+    }
+  }
+};
+
+template <class _Backend, class _RawExecutionPolicy>
+struct __cpu_parallel_transform_reduce {
+  template <class _Policy, class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
+  _LIBCPP_HIDE_FROM_ABI optional<_Tp>
+  operator()(_Policy&& __policy,
+             _ForwardIterator __first,
+             _ForwardIterator __last,
+             _Tp __init,
+             _BinaryOperation __reduce,
+             _UnaryOperation __transform) const noexcept {
+    if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> &&
+                  __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
+      return __cpu_traits<_Backend>::__transform_reduce(
+          std::move(__first),
+          std::move(__last),
+          [__transform](_ForwardIterator __iter) { return __transform(*__iter); },
+          std::move(__init),
+          __reduce,
+          [&__policy, __transform, __reduce](auto __brick_first, auto __brick_last, _Tp __brick_init) {
+            using _TransformReduceUnseq =
+                __pstl::__transform_reduce<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>;
+            auto __res = _TransformReduceUnseq()(
+                std::__remove_parallel_policy(__policy),
+                std::move(__brick_first),
+                std::move(__brick_last),
+                std::move(__brick_init),
+                std::move(__reduce),
+                std::move(__transform));
+            _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!");
+            return *std::move(__res);
+          });
+    } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> &&
+                         __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
+      return __pstl::__simd_transform_reduce<_Backend>(
+          __last - __first,
+          std::move(__init),
+          std::move(__reduce),
+          [=, &__transform](__iter_diff_t<_ForwardIterator> __i) { return __transform(__first[__i]); });
+    } else {
+      return std::transform_reduce(
+          std::move(__first), std::move(__last), std::move(__init), std::move(__reduce), std::move(__transform));
+    }
+  }
+};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_REDUCE_H
diff --git a/libcxx/include/__cxx03/__pstl/dispatch.h b/libcxx/include/__cxx03/__pstl/dispatch.h
new file mode 100644
index 000000000000..5e903f7524fe
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/dispatch.h
@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_DISPATCH_H
+#define _LIBCPP___PSTL_DISPATCH_H
+
+#include <__config>
+#include <__pstl/backend_fwd.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/type_identity.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+template <template <class, class> class _Algorithm, class _Backend, class _ExecutionPolicy, class = void>
+constexpr bool __is_implemented_v = false;
+
+template <template <class, class> class _Algorithm, class _Backend, class _ExecutionPolicy>
+constexpr bool __is_implemented_v<_Algorithm,
+                                  _Backend,
+                                  _ExecutionPolicy,
+                                  __enable_if_t<sizeof(_Algorithm<_Backend, _ExecutionPolicy>)>> = true;
+
+// Helpful to provide better error messages. This will show the algorithm and the execution policy
+// in the compiler diagnostic.
+template <template <class, class> class _Algorithm, class _ExecutionPolicy>
+constexpr bool __cant_find_backend_for = false;
+
+template <template <class, class> class _Algorithm, class _BackendConfiguration, class _ExecutionPolicy>
+struct __find_first_implemented;
+
+template <template <class, class> class _Algorithm, class _ExecutionPolicy>
+struct __find_first_implemented<_Algorithm, __backend_configuration<>, _ExecutionPolicy> {
+  static_assert(__cant_find_backend_for<_Algorithm, _ExecutionPolicy>,
+                "Could not find a PSTL backend for the given algorithm and execution policy");
+};
+
+template <template <class, class> class _Algorithm, class _B1, class... _Bn, class _ExecutionPolicy>
+struct __find_first_implemented<_Algorithm, __backend_configuration<_B1, _Bn...>, _ExecutionPolicy>
+    : _If<__is_implemented_v<_Algorithm, _B1, _ExecutionPolicy>,
+          __type_identity<_Algorithm<_B1, _ExecutionPolicy>>,
+          __find_first_implemented<_Algorithm, __backend_configuration<_Bn...>, _ExecutionPolicy> > {};
+
+template <template <class, class> class _Algorithm, class _BackendConfiguration, class _ExecutionPolicy>
+using __dispatch = typename __find_first_implemented<_Algorithm, _BackendConfiguration, _ExecutionPolicy>::type;
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_DISPATCH_H
diff --git a/libcxx/include/__cxx03/__pstl/handle_exception.h b/libcxx/include/__cxx03/__pstl/handle_exception.h
new file mode 100644
index 000000000000..d6270958c3a7
--- /dev/null
+++ b/libcxx/include/__cxx03/__pstl/handle_exception.h
@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_HANDLE_EXCEPTION_H
+#define _LIBCPP___PSTL_HANDLE_EXCEPTION_H
+
+#include <__config>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <new> // __throw_bad_alloc
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+template <class _BackendFunction, class... _Args>
+_LIBCPP_HIDE_FROM_ABI auto __handle_exception_impl(_Args&&... __args) noexcept {
+  return _BackendFunction{}(std::forward<_Args>(__args)...);
+}
+
+// This function is used to call a backend PSTL algorithm from a frontend algorithm.
+//
+// All PSTL backend algorithms return an optional denoting whether there was an
+// "infrastructure"-level failure (aka failure to allocate). This function takes
+// care of unwrapping that and throwing `bad_alloc()` in case there was a problem
+// in the underlying implementation.
+//
+// We must also be careful not to call any user code that could throw an exception
+// (such as moving or copying iterators) in here since that should terminate the
+// program, which is why we delegate to a noexcept helper below.
+template <class _BackendFunction, class... _Args>
+_LIBCPP_HIDE_FROM_ABI auto __handle_exception(_Args&&... __args) {
+  auto __result = __pstl::__handle_exception_impl<_BackendFunction>(std::forward<_Args>(__args)...);
+  if (__result == nullopt)
+    std::__throw_bad_alloc();
+  else
+    return std::move(*__result);
+}
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_HANDLE_EXCEPTION_H
diff --git a/libcxx/include/__cxx03/__random/bernoulli_distribution.h b/libcxx/include/__cxx03/__random/bernoulli_distribution.h
new file mode 100644
index 000000000000..4f33dca132d1
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/bernoulli_distribution.h
@@ -0,0 +1,123 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_BERNOULLI_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_BERNOULLI_DISTRIBUTION_H
+
+#include <__config>
+#include <__random/is_valid.h>
+#include <__random/uniform_real_distribution.h>
+#include <iosfwd>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+class _LIBCPP_TEMPLATE_VIS bernoulli_distribution {
+public:
+  // types
+  typedef bool result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    double __p_;
+
+  public:
+    typedef bernoulli_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(double __p = 0.5) : __p_(__p) {}
+
+    _LIBCPP_HIDE_FROM_ABI double p() const { return __p_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__p_ == __y.__p_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructors and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI bernoulli_distribution() : bernoulli_distribution(0.5) {}
+  _LIBCPP_HIDE_FROM_ABI explicit bernoulli_distribution(double __p) : __p_(param_type(__p)) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit bernoulli_distribution(double __p = 0.5) : __p_(param_type(__p)) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit bernoulli_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI double p() const { return __p_.p(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return false; }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return true; }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const bernoulli_distribution& __x, const bernoulli_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const bernoulli_distribution& __x, const bernoulli_distribution& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _URNG>
+inline bernoulli_distribution::result_type bernoulli_distribution::operator()(_URNG& __g, const param_type& __p) {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  uniform_real_distribution<double> __gen;
+  return __gen(__g) < __p.p();
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const bernoulli_distribution& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  return __os << __x.p();
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, bernoulli_distribution& __x) {
+  typedef bernoulli_distribution _Eng;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  double __p;
+  __is >> __p;
+  if (!__is.fail())
+    __x.param(param_type(__p));
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_BERNOULLI_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/binomial_distribution.h b/libcxx/include/__cxx03/__random/binomial_distribution.h
new file mode 100644
index 000000000000..e8774bb8d67e
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/binomial_distribution.h
@@ -0,0 +1,198 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_BINOMIAL_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_BINOMIAL_DISTRIBUTION_H
+
+#include <__config>
+#include <__random/is_valid.h>
+#include <__random/uniform_real_distribution.h>
+#include <cmath>
+#include <iosfwd>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _IntType = int>
+class _LIBCPP_TEMPLATE_VIS binomial_distribution {
+  static_assert(__libcpp_random_is_valid_inttype<_IntType>::value, "IntType must be a supported integer type");
+
+public:
+  // types
+  typedef _IntType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    result_type __t_;
+    double __p_;
+    double __pr_;
+    double __odds_ratio_;
+    result_type __r0_;
+
+  public:
+    typedef binomial_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(result_type __t = 1, double __p = 0.5);
+
+    _LIBCPP_HIDE_FROM_ABI result_type t() const { return __t_; }
+    _LIBCPP_HIDE_FROM_ABI double p() const { return __p_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__t_ == __y.__t_ && __x.__p_ == __y.__p_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+
+    friend class binomial_distribution;
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructors and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI binomial_distribution() : binomial_distribution(1) {}
+  _LIBCPP_HIDE_FROM_ABI explicit binomial_distribution(result_type __t, double __p = 0.5)
+      : __p_(param_type(__t, __p)) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit binomial_distribution(result_type __t = 1, double __p = 0.5)
+      : __p_(param_type(__t, __p)) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit binomial_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI result_type t() const { return __p_.t(); }
+  _LIBCPP_HIDE_FROM_ABI double p() const { return __p_.p(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return 0; }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return t(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const binomial_distribution& __x, const binomial_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const binomial_distribution& __x, const binomial_distribution& __y) {
+    return !(__x == __y);
+  }
+};
+
+#ifndef _LIBCPP_MSVCRT_LIKE
+extern "C" double lgamma_r(double, int*);
+#endif
+
+inline _LIBCPP_HIDE_FROM_ABI double __libcpp_lgamma(double __d) {
+#if defined(_LIBCPP_MSVCRT_LIKE)
+  return lgamma(__d);
+#else
+  int __sign;
+  return lgamma_r(__d, &__sign);
+#endif
+}
+
+template <class _IntType>
+binomial_distribution<_IntType>::param_type::param_type(result_type __t, double __p) : __t_(__t), __p_(__p) {
+  if (0 < __p_ && __p_ < 1) {
+    __r0_ = static_cast<result_type>((__t_ + 1) * __p_);
+    __pr_ = std::exp(
+        std::__libcpp_lgamma(__t_ + 1.) - std::__libcpp_lgamma(__r0_ + 1.) - std::__libcpp_lgamma(__t_ - __r0_ + 1.) +
+        __r0_ * std::log(__p_) + (__t_ - __r0_) * std::log(1 - __p_));
+    __odds_ratio_ = __p_ / (1 - __p_);
+  }
+}
+
+// Reference: Kemp, C.D. (1986). `A modal method for generating binomial
+//           variables', Commun. Statist. - Theor. Meth. 15(3), 805-813.
+template <class _IntType>
+template <class _URNG>
+_IntType binomial_distribution<_IntType>::operator()(_URNG& __g, const param_type& __pr) {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  if (__pr.__t_ == 0 || __pr.__p_ == 0)
+    return 0;
+  if (__pr.__p_ == 1)
+    return __pr.__t_;
+  uniform_real_distribution<double> __gen;
+  double __u = __gen(__g) - __pr.__pr_;
+  if (__u < 0)
+    return __pr.__r0_;
+  double __pu      = __pr.__pr_;
+  double __pd      = __pu;
+  result_type __ru = __pr.__r0_;
+  result_type __rd = __ru;
+  while (true) {
+    bool __break = true;
+    if (__rd >= 1) {
+      __pd *= __rd / (__pr.__odds_ratio_ * (__pr.__t_ - __rd + 1));
+      __u -= __pd;
+      __break = false;
+      if (__u < 0)
+        return __rd - 1;
+    }
+    if (__rd != 0)
+      --__rd;
+    ++__ru;
+    if (__ru <= __pr.__t_) {
+      __pu *= (__pr.__t_ - __ru + 1) * __pr.__odds_ratio_ / __ru;
+      __u -= __pu;
+      __break = false;
+      if (__u < 0)
+        return __ru;
+    }
+    if (__break)
+      return 0;
+  }
+}
+
+template <class _CharT, class _Traits, class _IntType>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const binomial_distribution<_IntType>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  return __os << __x.t() << __sp << __x.p();
+}
+
+template <class _CharT, class _Traits, class _IntType>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, binomial_distribution<_IntType>& __x) {
+  typedef binomial_distribution<_IntType> _Eng;
+  typedef typename _Eng::result_type result_type;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  result_type __t;
+  double __p;
+  __is >> __t >> __p;
+  if (!__is.fail())
+    __x.param(param_type(__t, __p));
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_BINOMIAL_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/cauchy_distribution.h b/libcxx/include/__cxx03/__random/cauchy_distribution.h
new file mode 100644
index 000000000000..bd341427a152
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/cauchy_distribution.h
@@ -0,0 +1,139 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_CAUCHY_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_CAUCHY_DISTRIBUTION_H
+
+#include <__config>
+#include <__random/is_valid.h>
+#include <__random/uniform_real_distribution.h>
+#include <cmath>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _RealType = double>
+class _LIBCPP_TEMPLATE_VIS cauchy_distribution {
+  static_assert(__libcpp_random_is_valid_realtype<_RealType>::value,
+                "RealType must be a supported floating-point type");
+
+public:
+  // types
+  typedef _RealType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    result_type __a_;
+    result_type __b_;
+
+  public:
+    typedef cauchy_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(result_type __a = 0, result_type __b = 1) : __a_(__a), __b_(__b) {}
+
+    _LIBCPP_HIDE_FROM_ABI result_type a() const { return __a_; }
+    _LIBCPP_HIDE_FROM_ABI result_type b() const { return __b_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__a_ == __y.__a_ && __x.__b_ == __y.__b_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructor and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI cauchy_distribution() : cauchy_distribution(0) {}
+  _LIBCPP_HIDE_FROM_ABI explicit cauchy_distribution(result_type __a, result_type __b = 1)
+      : __p_(param_type(__a, __b)) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit cauchy_distribution(result_type __a = 0, result_type __b = 1)
+      : __p_(param_type(__a, __b)) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit cauchy_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI result_type a() const { return __p_.a(); }
+  _LIBCPP_HIDE_FROM_ABI result_type b() const { return __p_.b(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return -numeric_limits<result_type>::infinity(); }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return numeric_limits<result_type>::infinity(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const cauchy_distribution& __x, const cauchy_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const cauchy_distribution& __x, const cauchy_distribution& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _RealType>
+template <class _URNG>
+inline _RealType cauchy_distribution<_RealType>::operator()(_URNG& __g, const param_type& __p) {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  uniform_real_distribution<result_type> __gen;
+  // purposefully let tan arg get as close to pi/2 as it wants, tan will return a finite
+  return __p.a() + __p.b() * std::tan(3.1415926535897932384626433832795 * __gen(__g));
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const cauchy_distribution<_RT>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  __os << __x.a() << __sp << __x.b();
+  return __os;
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, cauchy_distribution<_RT>& __x) {
+  typedef cauchy_distribution<_RT> _Eng;
+  typedef typename _Eng::result_type result_type;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  result_type __a;
+  result_type __b;
+  __is >> __a >> __b;
+  if (!__is.fail())
+    __x.param(param_type(__a, __b));
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_CAUCHY_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/chi_squared_distribution.h b/libcxx/include/__cxx03/__random/chi_squared_distribution.h
new file mode 100644
index 000000000000..efa96dcdaafb
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/chi_squared_distribution.h
@@ -0,0 +1,125 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_CHI_SQUARED_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_CHI_SQUARED_DISTRIBUTION_H
+
+#include <__config>
+#include <__random/gamma_distribution.h>
+#include <__random/is_valid.h>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _RealType = double>
+class _LIBCPP_TEMPLATE_VIS chi_squared_distribution {
+  static_assert(__libcpp_random_is_valid_realtype<_RealType>::value,
+                "RealType must be a supported floating-point type");
+
+public:
+  // types
+  typedef _RealType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    result_type __n_;
+
+  public:
+    typedef chi_squared_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(result_type __n = 1) : __n_(__n) {}
+
+    _LIBCPP_HIDE_FROM_ABI result_type n() const { return __n_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__n_ == __y.__n_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructor and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI chi_squared_distribution() : chi_squared_distribution(1) {}
+  _LIBCPP_HIDE_FROM_ABI explicit chi_squared_distribution(result_type __n) : __p_(param_type(__n)) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit chi_squared_distribution(result_type __n = 1) : __p_(param_type(__n)) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit chi_squared_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p) {
+    return gamma_distribution<result_type>(__p.n() / 2, 2)(__g);
+  }
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI result_type n() const { return __p_.n(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return 0; }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return numeric_limits<result_type>::infinity(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const chi_squared_distribution& __x, const chi_squared_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator!=(const chi_squared_distribution& __x, const chi_squared_distribution& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const chi_squared_distribution<_RT>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  __os << __x.n();
+  return __os;
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, chi_squared_distribution<_RT>& __x) {
+  typedef chi_squared_distribution<_RT> _Eng;
+  typedef typename _Eng::result_type result_type;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  result_type __n;
+  __is >> __n;
+  if (!__is.fail())
+    __x.param(param_type(__n));
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_CHI_SQUARED_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/clamp_to_integral.h b/libcxx/include/__cxx03/__random/clamp_to_integral.h
new file mode 100644
index 000000000000..d9bfd31b7f01
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/clamp_to_integral.h
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_CLAMP_TO_INTEGRAL_H
+#define _LIBCPP___RANDOM_CLAMP_TO_INTEGRAL_H
+
+#include <__config>
+#include <cmath>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _IntT,
+          class _FloatT,
+          bool _FloatBigger = (numeric_limits<_FloatT>::digits > numeric_limits<_IntT>::digits),
+          int _Bits         = (numeric_limits<_IntT>::digits - numeric_limits<_FloatT>::digits)>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _IntT __max_representable_int_for_float() _NOEXCEPT {
+  static_assert(is_floating_point<_FloatT>::value, "must be a floating point type");
+  static_assert(is_integral<_IntT>::value, "must be an integral type");
+  static_assert(numeric_limits<_FloatT>::radix == 2, "FloatT has incorrect radix");
+  static_assert(
+      (_IsSame<_FloatT, float>::value || _IsSame<_FloatT, double>::value || _IsSame<_FloatT, long double>::value),
+      "unsupported floating point type");
+  return _FloatBigger ? numeric_limits<_IntT>::max() : (numeric_limits<_IntT>::max() >> _Bits << _Bits);
+}
+
+// Convert a floating point number to the specified integral type after
+// clamping to the integral type's representable range.
+//
+// The behavior is undefined if `__r` is NaN.
+template <class _IntT, class _RealT>
+_LIBCPP_HIDE_FROM_ABI _IntT __clamp_to_integral(_RealT __r) _NOEXCEPT {
+  using _Lim            = numeric_limits<_IntT>;
+  const _IntT __max_val = __max_representable_int_for_float<_IntT, _RealT>();
+  if (__r >= ::nextafter(static_cast<_RealT>(__max_val), INFINITY)) {
+    return _Lim::max();
+  } else if (__r <= _Lim::lowest()) {
+    return _Lim::min();
+  }
+  return static_cast<_IntT>(__r);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_CLAMP_TO_INTEGRAL_H
diff --git a/libcxx/include/__cxx03/__random/default_random_engine.h b/libcxx/include/__cxx03/__random/default_random_engine.h
new file mode 100644
index 000000000000..89792f4f0d43
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/default_random_engine.h
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_DEFAULT_RANDOM_ENGINE_H
+#define _LIBCPP___RANDOM_DEFAULT_RANDOM_ENGINE_H
+
+#include <__config>
+#include <__random/linear_congruential_engine.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+typedef minstd_rand default_random_engine;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANDOM_DEFAULT_RANDOM_ENGINE_H
diff --git a/libcxx/include/__cxx03/__random/discard_block_engine.h b/libcxx/include/__cxx03/__random/discard_block_engine.h
new file mode 100644
index 000000000000..07f599067279
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/discard_block_engine.h
@@ -0,0 +1,172 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_DISCARD_BLOCK_ENGINE_H
+#define _LIBCPP___RANDOM_DISCARD_BLOCK_ENGINE_H
+
+#include <__config>
+#include <__random/is_seed_sequence.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_convertible.h>
+#include <__utility/move.h>
+#include <cstddef>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Engine, size_t __p, size_t __r>
+class _LIBCPP_TEMPLATE_VIS discard_block_engine {
+  _Engine __e_;
+  int __n_;
+
+  static_assert(0 < __r, "discard_block_engine invalid parameters");
+  static_assert(__r <= __p, "discard_block_engine invalid parameters");
+#ifndef _LIBCPP_CXX03_LANG // numeric_limits::max() is not constexpr in C++03
+  static_assert(__r <= numeric_limits<int>::max(), "discard_block_engine invalid parameters");
+#endif
+
+public:
+  // types
+  typedef typename _Engine::result_type result_type;
+
+  // engine characteristics
+  static _LIBCPP_CONSTEXPR const size_t block_size = __p;
+  static _LIBCPP_CONSTEXPR const size_t used_block = __r;
+
+#ifdef _LIBCPP_CXX03_LANG
+  static const result_type _Min = _Engine::_Min;
+  static const result_type _Max = _Engine::_Max;
+#else
+  static constexpr result_type _Min = _Engine::min();
+  static constexpr result_type _Max = _Engine::max();
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR result_type min() { return _Engine::min(); }
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR result_type max() { return _Engine::max(); }
+
+  // constructors and seeding functions
+  _LIBCPP_HIDE_FROM_ABI discard_block_engine() : __n_(0) {}
+  _LIBCPP_HIDE_FROM_ABI explicit discard_block_engine(const _Engine& __e) : __e_(__e), __n_(0) {}
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI explicit discard_block_engine(_Engine&& __e) : __e_(std::move(__e)), __n_(0) {}
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI explicit discard_block_engine(result_type __sd) : __e_(__sd), __n_(0) {}
+  template <
+      class _Sseq,
+      __enable_if_t<__is_seed_sequence<_Sseq, discard_block_engine>::value && !is_convertible<_Sseq, _Engine>::value,
+                    int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit discard_block_engine(_Sseq& __q) : __e_(__q), __n_(0) {}
+  _LIBCPP_HIDE_FROM_ABI void seed() {
+    __e_.seed();
+    __n_ = 0;
+  }
+  _LIBCPP_HIDE_FROM_ABI void seed(result_type __sd) {
+    __e_.seed(__sd);
+    __n_ = 0;
+  }
+  template <class _Sseq, __enable_if_t<__is_seed_sequence<_Sseq, discard_block_engine>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void seed(_Sseq& __q) {
+    __e_.seed(__q);
+    __n_ = 0;
+  }
+
+  // generating functions
+  _LIBCPP_HIDE_FROM_ABI result_type operator()();
+  _LIBCPP_HIDE_FROM_ABI void discard(unsigned long long __z) {
+    for (; __z; --__z)
+      operator()();
+  }
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI const _Engine& base() const _NOEXCEPT { return __e_; }
+
+  template <class _Eng, size_t _Pp, size_t _Rp>
+  friend bool
+  operator==(const discard_block_engine<_Eng, _Pp, _Rp>& __x, const discard_block_engine<_Eng, _Pp, _Rp>& __y);
+
+  template <class _Eng, size_t _Pp, size_t _Rp>
+  friend bool
+  operator!=(const discard_block_engine<_Eng, _Pp, _Rp>& __x, const discard_block_engine<_Eng, _Pp, _Rp>& __y);
+
+  template <class _CharT, class _Traits, class _Eng, size_t _Pp, size_t _Rp>
+  friend basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const discard_block_engine<_Eng, _Pp, _Rp>& __x);
+
+  template <class _CharT, class _Traits, class _Eng, size_t _Pp, size_t _Rp>
+  friend basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, discard_block_engine<_Eng, _Pp, _Rp>& __x);
+};
+
+template <class _Engine, size_t __p, size_t __r>
+_LIBCPP_CONSTEXPR const size_t discard_block_engine<_Engine, __p, __r>::block_size;
+
+template <class _Engine, size_t __p, size_t __r>
+_LIBCPP_CONSTEXPR const size_t discard_block_engine<_Engine, __p, __r>::used_block;
+
+template <class _Engine, size_t __p, size_t __r>
+typename discard_block_engine<_Engine, __p, __r>::result_type discard_block_engine<_Engine, __p, __r>::operator()() {
+  if (__n_ >= static_cast<int>(__r)) {
+    __e_.discard(__p - __r);
+    __n_ = 0;
+  }
+  ++__n_;
+  return __e_();
+}
+
+template <class _Eng, size_t _Pp, size_t _Rp>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const discard_block_engine<_Eng, _Pp, _Rp>& __x, const discard_block_engine<_Eng, _Pp, _Rp>& __y) {
+  return __x.__n_ == __y.__n_ && __x.__e_ == __y.__e_;
+}
+
+template <class _Eng, size_t _Pp, size_t _Rp>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const discard_block_engine<_Eng, _Pp, _Rp>& __x, const discard_block_engine<_Eng, _Pp, _Rp>& __y) {
+  return !(__x == __y);
+}
+
+template <class _CharT, class _Traits, class _Eng, size_t _Pp, size_t _Rp>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const discard_block_engine<_Eng, _Pp, _Rp>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _Ostream;
+  __os.flags(_Ostream::dec | _Ostream::left);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  return __os << __x.__e_ << __sp << __x.__n_;
+}
+
+template <class _CharT, class _Traits, class _Eng, size_t _Pp, size_t _Rp>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, discard_block_engine<_Eng, _Pp, _Rp>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  _Eng __e;
+  int __n;
+  __is >> __e >> __n;
+  if (!__is.fail()) {
+    __x.__e_ = __e;
+    __x.__n_ = __n;
+  }
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_DISCARD_BLOCK_ENGINE_H
diff --git a/libcxx/include/__cxx03/__random/discrete_distribution.h b/libcxx/include/__cxx03/__random/discrete_distribution.h
new file mode 100644
index 000000000000..bb72dd6cb507
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/discrete_distribution.h
@@ -0,0 +1,212 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_DISCRETE_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_DISCRETE_DISTRIBUTION_H
+
+#include <__algorithm/upper_bound.h>
+#include <__config>
+#include <__random/is_valid.h>
+#include <__random/uniform_real_distribution.h>
+#include <cstddef>
+#include <iosfwd>
+#include <numeric>
+#include <vector>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _IntType = int>
+class _LIBCPP_TEMPLATE_VIS discrete_distribution {
+  static_assert(__libcpp_random_is_valid_inttype<_IntType>::value, "IntType must be a supported integer type");
+
+public:
+  // types
+  typedef _IntType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    vector<double> __p_;
+
+  public:
+    typedef discrete_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI param_type() {}
+    template <class _InputIterator>
+    _LIBCPP_HIDE_FROM_ABI param_type(_InputIterator __f, _InputIterator __l) : __p_(__f, __l) {
+      __init();
+    }
+#ifndef _LIBCPP_CXX03_LANG
+    _LIBCPP_HIDE_FROM_ABI param_type(initializer_list<double> __wl) : __p_(__wl.begin(), __wl.end()) { __init(); }
+#endif // _LIBCPP_CXX03_LANG
+    template <class _UnaryOperation>
+    _LIBCPP_HIDE_FROM_ABI param_type(size_t __nw, double __xmin, double __xmax, _UnaryOperation __fw);
+
+    _LIBCPP_HIDE_FROM_ABI vector<double> probabilities() const;
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__p_ == __y.__p_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+
+  private:
+    _LIBCPP_HIDE_FROM_ABI void __init();
+
+    friend class discrete_distribution;
+
+    template <class _CharT, class _Traits, class _IT>
+    friend basic_ostream<_CharT, _Traits>&
+    operator<<(basic_ostream<_CharT, _Traits>& __os, const discrete_distribution<_IT>& __x);
+
+    template <class _CharT, class _Traits, class _IT>
+    friend basic_istream<_CharT, _Traits>&
+    operator>>(basic_istream<_CharT, _Traits>& __is, discrete_distribution<_IT>& __x);
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructor and reset functions
+  _LIBCPP_HIDE_FROM_ABI discrete_distribution() {}
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI discrete_distribution(_InputIterator __f, _InputIterator __l) : __p_(__f, __l) {}
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI discrete_distribution(initializer_list<double> __wl) : __p_(__wl) {}
+#endif // _LIBCPP_CXX03_LANG
+  template <class _UnaryOperation>
+  _LIBCPP_HIDE_FROM_ABI discrete_distribution(size_t __nw, double __xmin, double __xmax, _UnaryOperation __fw)
+      : __p_(__nw, __xmin, __xmax, __fw) {}
+  _LIBCPP_HIDE_FROM_ABI explicit discrete_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI vector<double> probabilities() const { return __p_.probabilities(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return 0; }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return __p_.__p_.size(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const discrete_distribution& __x, const discrete_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const discrete_distribution& __x, const discrete_distribution& __y) {
+    return !(__x == __y);
+  }
+
+  template <class _CharT, class _Traits, class _IT>
+  friend basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const discrete_distribution<_IT>& __x);
+
+  template <class _CharT, class _Traits, class _IT>
+  friend basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, discrete_distribution<_IT>& __x);
+};
+
+template <class _IntType>
+template <class _UnaryOperation>
+discrete_distribution<_IntType>::param_type::param_type(
+    size_t __nw, double __xmin, double __xmax, _UnaryOperation __fw) {
+  if (__nw > 1) {
+    __p_.reserve(__nw - 1);
+    double __d  = (__xmax - __xmin) / __nw;
+    double __d2 = __d / 2;
+    for (size_t __k = 0; __k < __nw; ++__k)
+      __p_.push_back(__fw(__xmin + __k * __d + __d2));
+    __init();
+  }
+}
+
+template <class _IntType>
+void discrete_distribution<_IntType>::param_type::__init() {
+  if (!__p_.empty()) {
+    if (__p_.size() > 1) {
+      double __s = std::accumulate(__p_.begin(), __p_.end(), 0.0);
+      for (vector<double>::iterator __i = __p_.begin(), __e = __p_.end(); __i < __e; ++__i)
+        *__i /= __s;
+      vector<double> __t(__p_.size() - 1);
+      std::partial_sum(__p_.begin(), __p_.end() - 1, __t.begin());
+      swap(__p_, __t);
+    } else {
+      __p_.clear();
+      __p_.shrink_to_fit();
+    }
+  }
+}
+
+template <class _IntType>
+vector<double> discrete_distribution<_IntType>::param_type::probabilities() const {
+  size_t __n = __p_.size();
+  vector<double> __p(__n + 1);
+  std::adjacent_difference(__p_.begin(), __p_.end(), __p.begin());
+  if (__n > 0)
+    __p[__n] = 1 - __p_[__n - 1];
+  else
+    __p[0] = 1;
+  return __p;
+}
+
+template <class _IntType>
+template <class _URNG>
+_IntType discrete_distribution<_IntType>::operator()(_URNG& __g, const param_type& __p) {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  uniform_real_distribution<double> __gen;
+  return static_cast<_IntType>(std::upper_bound(__p.__p_.begin(), __p.__p_.end(), __gen(__g)) - __p.__p_.begin());
+}
+
+template <class _CharT, class _Traits, class _IT>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const discrete_distribution<_IT>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  size_t __n = __x.__p_.__p_.size();
+  __os << __n;
+  for (size_t __i = 0; __i < __n; ++__i)
+    __os << __sp << __x.__p_.__p_[__i];
+  return __os;
+}
+
+template <class _CharT, class _Traits, class _IT>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, discrete_distribution<_IT>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  size_t __n;
+  __is >> __n;
+  vector<double> __p(__n);
+  for (size_t __i = 0; __i < __n; ++__i)
+    __is >> __p[__i];
+  if (!__is.fail())
+    swap(__x.__p_.__p_, __p);
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_DISCRETE_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/exponential_distribution.h b/libcxx/include/__cxx03/__random/exponential_distribution.h
new file mode 100644
index 000000000000..e0e38841172f
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/exponential_distribution.h
@@ -0,0 +1,132 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_EXPONENTIAL_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_EXPONENTIAL_DISTRIBUTION_H
+
+#include <__config>
+#include <__random/generate_canonical.h>
+#include <__random/is_valid.h>
+#include <__random/uniform_real_distribution.h>
+#include <cmath>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _RealType = double>
+class _LIBCPP_TEMPLATE_VIS exponential_distribution {
+  static_assert(__libcpp_random_is_valid_realtype<_RealType>::value,
+                "RealType must be a supported floating-point type");
+
+public:
+  // types
+  typedef _RealType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    result_type __lambda_;
+
+  public:
+    typedef exponential_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(result_type __lambda = 1) : __lambda_(__lambda) {}
+
+    _LIBCPP_HIDE_FROM_ABI result_type lambda() const { return __lambda_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__lambda_ == __y.__lambda_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructors and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI exponential_distribution() : exponential_distribution(1) {}
+  _LIBCPP_HIDE_FROM_ABI explicit exponential_distribution(result_type __lambda) : __p_(param_type(__lambda)) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit exponential_distribution(result_type __lambda = 1) : __p_(param_type(__lambda)) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit exponential_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI result_type lambda() const { return __p_.lambda(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return 0; }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return numeric_limits<result_type>::infinity(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const exponential_distribution& __x, const exponential_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator!=(const exponential_distribution& __x, const exponential_distribution& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _RealType>
+template <class _URNG>
+_RealType exponential_distribution<_RealType>::operator()(_URNG& __g, const param_type& __p) {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  return -std::log(result_type(1) - std::generate_canonical<result_type, numeric_limits<result_type>::digits>(__g)) /
+         __p.lambda();
+}
+
+template <class _CharT, class _Traits, class _RealType>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const exponential_distribution<_RealType>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  return __os << __x.lambda();
+}
+
+template <class _CharT, class _Traits, class _RealType>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, exponential_distribution<_RealType>& __x) {
+  typedef exponential_distribution<_RealType> _Eng;
+  typedef typename _Eng::result_type result_type;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  result_type __lambda;
+  __is >> __lambda;
+  if (!__is.fail())
+    __x.param(param_type(__lambda));
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_EXPONENTIAL_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/extreme_value_distribution.h b/libcxx/include/__cxx03/__random/extreme_value_distribution.h
new file mode 100644
index 000000000000..5505f93274f5
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/extreme_value_distribution.h
@@ -0,0 +1,139 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_EXTREME_VALUE_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_EXTREME_VALUE_DISTRIBUTION_H
+
+#include <__config>
+#include <__random/is_valid.h>
+#include <__random/uniform_real_distribution.h>
+#include <cmath>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _RealType = double>
+class _LIBCPP_TEMPLATE_VIS extreme_value_distribution {
+  static_assert(__libcpp_random_is_valid_realtype<_RealType>::value,
+                "RealType must be a supported floating-point type");
+
+public:
+  // types
+  typedef _RealType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    result_type __a_;
+    result_type __b_;
+
+  public:
+    typedef extreme_value_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(result_type __a = 0, result_type __b = 1) : __a_(__a), __b_(__b) {}
+
+    _LIBCPP_HIDE_FROM_ABI result_type a() const { return __a_; }
+    _LIBCPP_HIDE_FROM_ABI result_type b() const { return __b_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__a_ == __y.__a_ && __x.__b_ == __y.__b_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructor and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI extreme_value_distribution() : extreme_value_distribution(0) {}
+  _LIBCPP_HIDE_FROM_ABI explicit extreme_value_distribution(result_type __a, result_type __b = 1)
+      : __p_(param_type(__a, __b)) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit extreme_value_distribution(result_type __a = 0, result_type __b = 1)
+      : __p_(param_type(__a, __b)) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit extreme_value_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI result_type a() const { return __p_.a(); }
+  _LIBCPP_HIDE_FROM_ABI result_type b() const { return __p_.b(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return -numeric_limits<result_type>::infinity(); }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return numeric_limits<result_type>::infinity(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const extreme_value_distribution& __x, const extreme_value_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator!=(const extreme_value_distribution& __x, const extreme_value_distribution& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _RealType>
+template <class _URNG>
+_RealType extreme_value_distribution<_RealType>::operator()(_URNG& __g, const param_type& __p) {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  return __p.a() - __p.b() * std::log(-std::log(1 - uniform_real_distribution<result_type>()(__g)));
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const extreme_value_distribution<_RT>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  __os << __x.a() << __sp << __x.b();
+  return __os;
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, extreme_value_distribution<_RT>& __x) {
+  typedef extreme_value_distribution<_RT> _Eng;
+  typedef typename _Eng::result_type result_type;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  result_type __a;
+  result_type __b;
+  __is >> __a >> __b;
+  if (!__is.fail())
+    __x.param(param_type(__a, __b));
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_EXTREME_VALUE_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/fisher_f_distribution.h b/libcxx/include/__cxx03/__random/fisher_f_distribution.h
new file mode 100644
index 000000000000..cd170b3af388
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/fisher_f_distribution.h
@@ -0,0 +1,138 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_FISHER_F_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_FISHER_F_DISTRIBUTION_H
+
+#include <__config>
+#include <__random/gamma_distribution.h>
+#include <__random/is_valid.h>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _RealType = double>
+class _LIBCPP_TEMPLATE_VIS fisher_f_distribution {
+  static_assert(__libcpp_random_is_valid_realtype<_RealType>::value,
+                "RealType must be a supported floating-point type");
+
+public:
+  // types
+  typedef _RealType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    result_type __m_;
+    result_type __n_;
+
+  public:
+    typedef fisher_f_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(result_type __m = 1, result_type __n = 1) : __m_(__m), __n_(__n) {}
+
+    _LIBCPP_HIDE_FROM_ABI result_type m() const { return __m_; }
+    _LIBCPP_HIDE_FROM_ABI result_type n() const { return __n_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__m_ == __y.__m_ && __x.__n_ == __y.__n_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructor and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI fisher_f_distribution() : fisher_f_distribution(1) {}
+  _LIBCPP_HIDE_FROM_ABI explicit fisher_f_distribution(result_type __m, result_type __n = 1)
+      : __p_(param_type(__m, __n)) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit fisher_f_distribution(result_type __m = 1, result_type __n = 1)
+      : __p_(param_type(__m, __n)) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit fisher_f_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI result_type m() const { return __p_.m(); }
+  _LIBCPP_HIDE_FROM_ABI result_type n() const { return __p_.n(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return 0; }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return numeric_limits<result_type>::infinity(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const fisher_f_distribution& __x, const fisher_f_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const fisher_f_distribution& __x, const fisher_f_distribution& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _RealType>
+template <class _URNG>
+_RealType fisher_f_distribution<_RealType>::operator()(_URNG& __g, const param_type& __p) {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  gamma_distribution<result_type> __gdm(__p.m() * result_type(.5));
+  gamma_distribution<result_type> __gdn(__p.n() * result_type(.5));
+  return __p.n() * __gdm(__g) / (__p.m() * __gdn(__g));
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const fisher_f_distribution<_RT>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  __os << __x.m() << __sp << __x.n();
+  return __os;
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, fisher_f_distribution<_RT>& __x) {
+  typedef fisher_f_distribution<_RT> _Eng;
+  typedef typename _Eng::result_type result_type;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  result_type __m;
+  result_type __n;
+  __is >> __m >> __n;
+  if (!__is.fail())
+    __x.param(param_type(__m, __n));
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_FISHER_F_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/gamma_distribution.h b/libcxx/include/__cxx03/__random/gamma_distribution.h
new file mode 100644
index 000000000000..986e42c1c7f5
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/gamma_distribution.h
@@ -0,0 +1,181 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_GAMMA_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_GAMMA_DISTRIBUTION_H
+
+#include <__config>
+#include <__random/exponential_distribution.h>
+#include <__random/is_valid.h>
+#include <__random/uniform_real_distribution.h>
+#include <cmath>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _RealType = double>
+class _LIBCPP_TEMPLATE_VIS gamma_distribution {
+  static_assert(__libcpp_random_is_valid_realtype<_RealType>::value,
+                "RealType must be a supported floating-point type");
+
+public:
+  // types
+  typedef _RealType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    result_type __alpha_;
+    result_type __beta_;
+
+  public:
+    typedef gamma_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(result_type __alpha = 1, result_type __beta = 1)
+        : __alpha_(__alpha), __beta_(__beta) {}
+
+    _LIBCPP_HIDE_FROM_ABI result_type alpha() const { return __alpha_; }
+    _LIBCPP_HIDE_FROM_ABI result_type beta() const { return __beta_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__alpha_ == __y.__alpha_ && __x.__beta_ == __y.__beta_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructors and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI gamma_distribution() : gamma_distribution(1) {}
+  _LIBCPP_HIDE_FROM_ABI explicit gamma_distribution(result_type __alpha, result_type __beta = 1)
+      : __p_(param_type(__alpha, __beta)) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit gamma_distribution(result_type __alpha = 1, result_type __beta = 1)
+      : __p_(param_type(__alpha, __beta)) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit gamma_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI result_type alpha() const { return __p_.alpha(); }
+  _LIBCPP_HIDE_FROM_ABI result_type beta() const { return __p_.beta(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return 0; }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return numeric_limits<result_type>::infinity(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const gamma_distribution& __x, const gamma_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const gamma_distribution& __x, const gamma_distribution& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _RealType>
+template <class _URNG>
+_RealType gamma_distribution<_RealType>::operator()(_URNG& __g, const param_type& __p) {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  result_type __a = __p.alpha();
+  uniform_real_distribution<result_type> __gen(0, 1);
+  exponential_distribution<result_type> __egen;
+  result_type __x;
+  if (__a == 1)
+    __x = __egen(__g);
+  else if (__a > 1) {
+    const result_type __b = __a - 1;
+    const result_type __c = 3 * __a - result_type(0.75);
+    while (true) {
+      const result_type __u = __gen(__g);
+      const result_type __v = __gen(__g);
+      const result_type __w = __u * (1 - __u);
+      if (__w != 0) {
+        const result_type __y = std::sqrt(__c / __w) * (__u - result_type(0.5));
+        __x                   = __b + __y;
+        if (__x >= 0) {
+          const result_type __z = 64 * __w * __w * __w * __v * __v;
+          if (__z <= 1 - 2 * __y * __y / __x)
+            break;
+          if (std::log(__z) <= 2 * (__b * std::log(__x / __b) - __y))
+            break;
+        }
+      }
+    }
+  } else // __a < 1
+  {
+    while (true) {
+      const result_type __u  = __gen(__g);
+      const result_type __es = __egen(__g);
+      if (__u <= 1 - __a) {
+        __x = std::pow(__u, 1 / __a);
+        if (__x <= __es)
+          break;
+      } else {
+        const result_type __e = -std::log((1 - __u) / __a);
+        __x                   = std::pow(1 - __a + __a * __e, 1 / __a);
+        if (__x <= __e + __es)
+          break;
+      }
+    }
+  }
+  return __x * __p.beta();
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const gamma_distribution<_RT>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  __os << __x.alpha() << __sp << __x.beta();
+  return __os;
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, gamma_distribution<_RT>& __x) {
+  typedef gamma_distribution<_RT> _Eng;
+  typedef typename _Eng::result_type result_type;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  result_type __alpha;
+  result_type __beta;
+  __is >> __alpha >> __beta;
+  if (!__is.fail())
+    __x.param(param_type(__alpha, __beta));
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_GAMMA_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/generate_canonical.h b/libcxx/include/__cxx03/__random/generate_canonical.h
new file mode 100644
index 000000000000..738de1517e28
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/generate_canonical.h
@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_GENERATE_CANONICAL_H
+#define _LIBCPP___RANDOM_GENERATE_CANONICAL_H
+
+#include <__config>
+#include <__random/log2.h>
+#include <cstdint>
+#include <initializer_list>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// generate_canonical
+
+template <class _RealType, size_t __bits, class _URNG>
+_LIBCPP_HIDE_FROM_ABI _RealType generate_canonical(_URNG& __g) {
+  const size_t __dt = numeric_limits<_RealType>::digits;
+  const size_t __b  = __dt < __bits ? __dt : __bits;
+#ifdef _LIBCPP_CXX03_LANG
+  const size_t __log_r = __log2<uint64_t, _URNG::_Max - _URNG::_Min + uint64_t(1)>::value;
+#else
+  const size_t __log_r = __log2<uint64_t, _URNG::max() - _URNG::min() + uint64_t(1)>::value;
+#endif
+  const size_t __k     = __b / __log_r + (__b % __log_r != 0) + (__b == 0);
+  const _RealType __rp = static_cast<_RealType>(_URNG::max() - _URNG::min()) + _RealType(1);
+  _RealType __base     = __rp;
+  _RealType __sp       = __g() - _URNG::min();
+  for (size_t __i = 1; __i < __k; ++__i, __base *= __rp)
+    __sp += (__g() - _URNG::min()) * __base;
+  return __sp / __base;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_GENERATE_CANONICAL_H
diff --git a/libcxx/include/__cxx03/__random/geometric_distribution.h b/libcxx/include/__cxx03/__random/geometric_distribution.h
new file mode 100644
index 000000000000..cecd7e57cfa6
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/geometric_distribution.h
@@ -0,0 +1,120 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_GEOMETRIC_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_GEOMETRIC_DISTRIBUTION_H
+
+#include <__config>
+#include <__random/is_valid.h>
+#include <__random/negative_binomial_distribution.h>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _IntType = int>
+class _LIBCPP_TEMPLATE_VIS geometric_distribution {
+  static_assert(__libcpp_random_is_valid_inttype<_IntType>::value, "IntType must be a supported integer type");
+
+public:
+  // types
+  typedef _IntType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    double __p_;
+
+  public:
+    typedef geometric_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(double __p = 0.5) : __p_(__p) {}
+
+    _LIBCPP_HIDE_FROM_ABI double p() const { return __p_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__p_ == __y.__p_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructors and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI geometric_distribution() : geometric_distribution(0.5) {}
+  _LIBCPP_HIDE_FROM_ABI explicit geometric_distribution(double __p) : __p_(__p) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit geometric_distribution(double __p = 0.5) : __p_(__p) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit geometric_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p) {
+    return negative_binomial_distribution<result_type>(1, __p.p())(__g);
+  }
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI double p() const { return __p_.p(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return 0; }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return numeric_limits<result_type>::max(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const geometric_distribution& __x, const geometric_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const geometric_distribution& __x, const geometric_distribution& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _CharT, class _Traits, class _IntType>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const geometric_distribution<_IntType>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  return __os << __x.p();
+}
+
+template <class _CharT, class _Traits, class _IntType>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, geometric_distribution<_IntType>& __x) {
+  typedef geometric_distribution<_IntType> _Eng;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  double __p;
+  __is >> __p;
+  if (!__is.fail())
+    __x.param(param_type(__p));
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_GEOMETRIC_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/independent_bits_engine.h b/libcxx/include/__cxx03/__random/independent_bits_engine.h
new file mode 100644
index 000000000000..0f4a7b82b98f
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/independent_bits_engine.h
@@ -0,0 +1,205 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_INDEPENDENT_BITS_ENGINE_H
+#define _LIBCPP___RANDOM_INDEPENDENT_BITS_ENGINE_H
+
+#include <__config>
+#include <__fwd/istream.h>
+#include <__fwd/ostream.h>
+#include <__random/is_seed_sequence.h>
+#include <__random/log2.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_convertible.h>
+#include <__utility/move.h>
+#include <cstddef>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Engine, size_t __w, class _UIntType>
+class _LIBCPP_TEMPLATE_VIS independent_bits_engine {
+  template <class _UInt, _UInt _R0, size_t _Wp, size_t _Mp>
+  class __get_n {
+    static _LIBCPP_CONSTEXPR const size_t _Dt = numeric_limits<_UInt>::digits;
+    static _LIBCPP_CONSTEXPR const size_t _Np = _Wp / _Mp + (_Wp % _Mp != 0);
+    static _LIBCPP_CONSTEXPR const size_t _W0 = _Wp / _Np;
+    static _LIBCPP_CONSTEXPR const _UInt _Y0  = _W0 >= _Dt ? 0 : (_R0 >> _W0) << _W0;
+
+  public:
+    static _LIBCPP_CONSTEXPR const size_t value = _R0 - _Y0 > _Y0 / _Np ? _Np + 1 : _Np;
+  };
+
+public:
+  // types
+  typedef _UIntType result_type;
+
+private:
+  _Engine __e_;
+
+  static _LIBCPP_CONSTEXPR const result_type _Dt = numeric_limits<result_type>::digits;
+  static_assert(0 < __w, "independent_bits_engine invalid parameters");
+  static_assert(__w <= _Dt, "independent_bits_engine invalid parameters");
+
+  typedef typename _Engine::result_type _Engine_result_type;
+  typedef __conditional_t<sizeof(_Engine_result_type) <= sizeof(result_type), result_type, _Engine_result_type>
+      _Working_result_type;
+#ifdef _LIBCPP_CXX03_LANG
+  static const _Working_result_type _Rp = _Engine::_Max - _Engine::_Min + _Working_result_type(1);
+#else
+  static _LIBCPP_CONSTEXPR const _Working_result_type _Rp = _Engine::max() - _Engine::min() + _Working_result_type(1);
+#endif
+  static _LIBCPP_CONSTEXPR const size_t __m                = __log2<_Working_result_type, _Rp>::value;
+  static _LIBCPP_CONSTEXPR const size_t __n                = __get_n<_Working_result_type, _Rp, __w, __m>::value;
+  static _LIBCPP_CONSTEXPR const size_t __w0               = __w / __n;
+  static _LIBCPP_CONSTEXPR const size_t __n0               = __n - __w % __n;
+  static _LIBCPP_CONSTEXPR const size_t _WDt               = numeric_limits<_Working_result_type>::digits;
+  static _LIBCPP_CONSTEXPR const size_t _EDt               = numeric_limits<_Engine_result_type>::digits;
+  static _LIBCPP_CONSTEXPR const _Working_result_type __y0 = __w0 >= _WDt ? 0 : (_Rp >> __w0) << __w0;
+  static _LIBCPP_CONSTEXPR const _Working_result_type __y1 = __w0 >= _WDt - 1 ? 0 : (_Rp >> (__w0 + 1)) << (__w0 + 1);
+  static _LIBCPP_CONSTEXPR const
+      _Engine_result_type __mask0 = __w0 > 0 ? _Engine_result_type(~0) >> (_EDt - __w0) : _Engine_result_type(0);
+  static _LIBCPP_CONSTEXPR const _Engine_result_type __mask1 =
+      __w0 < _EDt - 1 ? _Engine_result_type(~0) >> (_EDt - (__w0 + 1)) : _Engine_result_type(~0);
+
+public:
+  static _LIBCPP_CONSTEXPR const result_type _Min = 0;
+  static _LIBCPP_CONSTEXPR const result_type _Max =
+      __w == _Dt ? result_type(~0) : (result_type(1) << __w) - result_type(1);
+  static_assert(_Min < _Max, "independent_bits_engine invalid parameters");
+
+  // engine characteristics
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR result_type min() { return _Min; }
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR result_type max() { return _Max; }
+
+  // constructors and seeding functions
+  _LIBCPP_HIDE_FROM_ABI independent_bits_engine() {}
+  _LIBCPP_HIDE_FROM_ABI explicit independent_bits_engine(const _Engine& __e) : __e_(__e) {}
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI explicit independent_bits_engine(_Engine&& __e) : __e_(std::move(__e)) {}
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI explicit independent_bits_engine(result_type __sd) : __e_(__sd) {}
+  template <
+      class _Sseq,
+      __enable_if_t<__is_seed_sequence<_Sseq, independent_bits_engine>::value && !is_convertible<_Sseq, _Engine>::value,
+                    int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit independent_bits_engine(_Sseq& __q) : __e_(__q) {}
+  _LIBCPP_HIDE_FROM_ABI void seed() { __e_.seed(); }
+  _LIBCPP_HIDE_FROM_ABI void seed(result_type __sd) { __e_.seed(__sd); }
+  template <class _Sseq, __enable_if_t<__is_seed_sequence<_Sseq, independent_bits_engine>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void seed(_Sseq& __q) {
+    __e_.seed(__q);
+  }
+
+  // generating functions
+  _LIBCPP_HIDE_FROM_ABI result_type operator()() { return __eval(integral_constant<bool, _Rp != 0>()); }
+  _LIBCPP_HIDE_FROM_ABI void discard(unsigned long long __z) {
+    for (; __z; --__z)
+      operator()();
+  }
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI const _Engine& base() const _NOEXCEPT { return __e_; }
+
+  template <class _Eng, size_t _Wp, class _UInt>
+  friend bool operator==(const independent_bits_engine<_Eng, _Wp, _UInt>& __x,
+                         const independent_bits_engine<_Eng, _Wp, _UInt>& __y);
+
+  template <class _Eng, size_t _Wp, class _UInt>
+  friend bool operator!=(const independent_bits_engine<_Eng, _Wp, _UInt>& __x,
+                         const independent_bits_engine<_Eng, _Wp, _UInt>& __y);
+
+  template <class _CharT, class _Traits, class _Eng, size_t _Wp, class _UInt>
+  friend basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const independent_bits_engine<_Eng, _Wp, _UInt>& __x);
+
+  template <class _CharT, class _Traits, class _Eng, size_t _Wp, class _UInt>
+  friend basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, independent_bits_engine<_Eng, _Wp, _UInt>& __x);
+
+private:
+  _LIBCPP_HIDE_FROM_ABI result_type __eval(false_type);
+  _LIBCPP_HIDE_FROM_ABI result_type __eval(true_type);
+
+  template <size_t __count,
+            __enable_if_t<__count< _Dt, int> = 0> _LIBCPP_HIDE_FROM_ABI static result_type __lshift(result_type __x) {
+    return __x << __count;
+  }
+
+  template <size_t __count, __enable_if_t<(__count >= _Dt), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI static result_type __lshift(result_type) {
+    return result_type(0);
+  }
+};
+
+template <class _Engine, size_t __w, class _UIntType>
+inline _UIntType independent_bits_engine<_Engine, __w, _UIntType>::__eval(false_type) {
+  return static_cast<result_type>(__e_() & __mask0);
+}
+
+template <class _Engine, size_t __w, class _UIntType>
+_UIntType independent_bits_engine<_Engine, __w, _UIntType>::__eval(true_type) {
+  result_type __sp = 0;
+  for (size_t __k = 0; __k < __n0; ++__k) {
+    _Engine_result_type __u;
+    do {
+      __u = __e_() - _Engine::min();
+    } while (__u >= __y0);
+    __sp = static_cast<result_type>(__lshift<__w0>(__sp) + (__u & __mask0));
+  }
+  for (size_t __k = __n0; __k < __n; ++__k) {
+    _Engine_result_type __u;
+    do {
+      __u = __e_() - _Engine::min();
+    } while (__u >= __y1);
+    __sp = static_cast<result_type>(__lshift<__w0 + 1>(__sp) + (__u & __mask1));
+  }
+  return __sp;
+}
+
+template <class _Eng, size_t _Wp, class _UInt>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const independent_bits_engine<_Eng, _Wp, _UInt>& __x, const independent_bits_engine<_Eng, _Wp, _UInt>& __y) {
+  return __x.base() == __y.base();
+}
+
+template <class _Eng, size_t _Wp, class _UInt>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const independent_bits_engine<_Eng, _Wp, _UInt>& __x, const independent_bits_engine<_Eng, _Wp, _UInt>& __y) {
+  return !(__x == __y);
+}
+
+template <class _CharT, class _Traits, class _Eng, size_t _Wp, class _UInt>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const independent_bits_engine<_Eng, _Wp, _UInt>& __x) {
+  return __os << __x.base();
+}
+
+template <class _CharT, class _Traits, class _Eng, size_t _Wp, class _UInt>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, independent_bits_engine<_Eng, _Wp, _UInt>& __x) {
+  _Eng __e;
+  __is >> __e;
+  if (!__is.fail())
+    __x.__e_ = __e;
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_INDEPENDENT_BITS_ENGINE_H
diff --git a/libcxx/include/__cxx03/__random/is_seed_sequence.h b/libcxx/include/__cxx03/__random/is_seed_sequence.h
new file mode 100644
index 000000000000..c7171cff2eda
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/is_seed_sequence.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_IS_SEED_SEQUENCE_H
+#define _LIBCPP___RANDOM_IS_SEED_SEQUENCE_H
+
+#include <__config>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_cv.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Sseq, class _Engine>
+struct __is_seed_sequence {
+  static _LIBCPP_CONSTEXPR const bool value =
+      !is_convertible<_Sseq, typename _Engine::result_type>::value && !is_same<__remove_cv_t<_Sseq>, _Engine>::value;
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANDOM_IS_SEED_SEQUENCE_H
diff --git a/libcxx/include/__cxx03/__random/is_valid.h b/libcxx/include/__cxx03/__random/is_valid.h
new file mode 100644
index 000000000000..a3e0f143ae86
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/is_valid.h
@@ -0,0 +1,94 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_IS_VALID_H
+#define _LIBCPP___RANDOM_IS_VALID_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_unsigned.h>
+#include <__utility/declval.h>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// [rand.req.genl]/1.4:
+// The effect of instantiating a template that has a template type parameter
+// named RealType is undefined unless the corresponding template argument is
+// cv-unqualified and is one of float, double, or long double.
+
+template <class>
+struct __libcpp_random_is_valid_realtype : false_type {};
+template <>
+struct __libcpp_random_is_valid_realtype<float> : true_type {};
+template <>
+struct __libcpp_random_is_valid_realtype<double> : true_type {};
+template <>
+struct __libcpp_random_is_valid_realtype<long double> : true_type {};
+
+// [rand.req.genl]/1.5:
+// The effect of instantiating a template that has a template type parameter
+// named IntType is undefined unless the corresponding template argument is
+// cv-unqualified and is one of short, int, long, long long, unsigned short,
+// unsigned int, unsigned long, or unsigned long long.
+
+template <class>
+struct __libcpp_random_is_valid_inttype : false_type {};
+template <>
+struct __libcpp_random_is_valid_inttype<int8_t> : true_type {}; // extension
+template <>
+struct __libcpp_random_is_valid_inttype<short> : true_type {};
+template <>
+struct __libcpp_random_is_valid_inttype<int> : true_type {};
+template <>
+struct __libcpp_random_is_valid_inttype<long> : true_type {};
+template <>
+struct __libcpp_random_is_valid_inttype<long long> : true_type {};
+template <>
+struct __libcpp_random_is_valid_inttype<uint8_t> : true_type {}; // extension
+template <>
+struct __libcpp_random_is_valid_inttype<unsigned short> : true_type {};
+template <>
+struct __libcpp_random_is_valid_inttype<unsigned int> : true_type {};
+template <>
+struct __libcpp_random_is_valid_inttype<unsigned long> : true_type {};
+template <>
+struct __libcpp_random_is_valid_inttype<unsigned long long> : true_type {};
+
+#ifndef _LIBCPP_HAS_NO_INT128
+template <>
+struct __libcpp_random_is_valid_inttype<__int128_t> : true_type {}; // extension
+template <>
+struct __libcpp_random_is_valid_inttype<__uint128_t> : true_type {}; // extension
+#endif                                                               // _LIBCPP_HAS_NO_INT128
+
+// [rand.req.urng]/3:
+// A class G meets the uniform random bit generator requirements if G models
+// uniform_random_bit_generator, invoke_result_t<G&> is an unsigned integer type,
+// and G provides a nested typedef-name result_type that denotes the same type
+// as invoke_result_t<G&>.
+// (In particular, reject URNGs with signed result_types; our distributions cannot
+// handle such generator types.)
+
+template <class, class = void>
+struct __libcpp_random_is_valid_urng : false_type {};
+template <class _Gp>
+struct __libcpp_random_is_valid_urng<
+    _Gp,
+    __enable_if_t< is_unsigned<typename _Gp::result_type>::value &&
+                   _IsSame<decltype(std::declval<_Gp&>()()), typename _Gp::result_type>::value > > : true_type {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANDOM_IS_VALID_H
diff --git a/libcxx/include/__cxx03/__random/knuth_b.h b/libcxx/include/__cxx03/__random/knuth_b.h
new file mode 100644
index 000000000000..f5b31cb64fa4
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/knuth_b.h
@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_KNUTH_B_H
+#define _LIBCPP___RANDOM_KNUTH_B_H
+
+#include <__config>
+#include <__random/linear_congruential_engine.h>
+#include <__random/shuffle_order_engine.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+typedef shuffle_order_engine<minstd_rand0, 256> knuth_b;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANDOM_KNUTH_B_H
diff --git a/libcxx/include/__cxx03/__random/linear_congruential_engine.h b/libcxx/include/__cxx03/__random/linear_congruential_engine.h
new file mode 100644
index 000000000000..9d77649e9cfc
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/linear_congruential_engine.h
@@ -0,0 +1,387 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_LINEAR_CONGRUENTIAL_ENGINE_H
+#define _LIBCPP___RANDOM_LINEAR_CONGRUENTIAL_ENGINE_H
+
+#include <__config>
+#include <__random/is_seed_sequence.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_unsigned.h>
+#include <cstdint>
+#include <iosfwd>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+enum __lce_alg_type {
+  _LCE_Full,
+  _LCE_Part,
+  _LCE_Schrage,
+  _LCE_Promote,
+};
+
+template <unsigned long long __a,
+          unsigned long long __c,
+          unsigned long long __m,
+          unsigned long long _Mp,
+          bool _HasOverflow = (__a != 0ull && (__m & (__m - 1ull)) != 0ull),      // a != 0, m != 0, m != 2^n
+          bool _Full        = (!_HasOverflow || __m - 1ull <= (_Mp - __c) / __a), // (a * x + c) % m works
+          bool _Part        = (!_HasOverflow || __m - 1ull <= _Mp / __a),         // (a * x) % m works
+          bool _Schrage     = (_HasOverflow && __m % __a <= __m / __a)>               // r <= q
+struct __lce_alg_picker {
+  static _LIBCPP_CONSTEXPR const __lce_alg_type __mode =
+      _Full      ? _LCE_Full
+      : _Part    ? _LCE_Part
+      : _Schrage ? _LCE_Schrage
+                 : _LCE_Promote;
+
+#ifdef _LIBCPP_HAS_NO_INT128
+  static_assert(_Mp != (unsigned long long)(-1) || _Full || _Part || _Schrage,
+                "The current values for a, c, and m are not currently supported on platforms without __int128");
+#endif
+};
+
+template <unsigned long long __a,
+          unsigned long long __c,
+          unsigned long long __m,
+          unsigned long long _Mp,
+          __lce_alg_type _Mode = __lce_alg_picker<__a, __c, __m, _Mp>::__mode>
+struct __lce_ta;
+
+// 64
+
+#ifndef _LIBCPP_HAS_NO_INT128
+template <unsigned long long _Ap, unsigned long long _Cp, unsigned long long _Mp>
+struct __lce_ta<_Ap, _Cp, _Mp, (unsigned long long)(-1), _LCE_Promote> {
+  typedef unsigned long long result_type;
+  _LIBCPP_HIDE_FROM_ABI static result_type next(result_type __xp) {
+    __extension__ using __calc_type = unsigned __int128;
+    const __calc_type __a           = static_cast<__calc_type>(_Ap);
+    const __calc_type __c           = static_cast<__calc_type>(_Cp);
+    const __calc_type __m           = static_cast<__calc_type>(_Mp);
+    const __calc_type __x           = static_cast<__calc_type>(__xp);
+    return static_cast<result_type>((__a * __x + __c) % __m);
+  }
+};
+#endif
+
+template <unsigned long long __a, unsigned long long __c, unsigned long long __m>
+struct __lce_ta<__a, __c, __m, (unsigned long long)(-1), _LCE_Schrage> {
+  typedef unsigned long long result_type;
+  _LIBCPP_HIDE_FROM_ABI static result_type next(result_type __x) {
+    // Schrage's algorithm
+    const result_type __q  = __m / __a;
+    const result_type __r  = __m % __a;
+    const result_type __t0 = __a * (__x % __q);
+    const result_type __t1 = __r * (__x / __q);
+    __x                    = __t0 + (__t0 < __t1) * __m - __t1;
+    __x += __c - (__x >= __m - __c) * __m;
+    return __x;
+  }
+};
+
+template <unsigned long long __a, unsigned long long __m>
+struct __lce_ta<__a, 0ull, __m, (unsigned long long)(-1), _LCE_Schrage> {
+  typedef unsigned long long result_type;
+  _LIBCPP_HIDE_FROM_ABI static result_type next(result_type __x) {
+    // Schrage's algorithm
+    const result_type __q  = __m / __a;
+    const result_type __r  = __m % __a;
+    const result_type __t0 = __a * (__x % __q);
+    const result_type __t1 = __r * (__x / __q);
+    __x                    = __t0 + (__t0 < __t1) * __m - __t1;
+    return __x;
+  }
+};
+
+template <unsigned long long __a, unsigned long long __c, unsigned long long __m>
+struct __lce_ta<__a, __c, __m, (unsigned long long)(-1), _LCE_Part> {
+  typedef unsigned long long result_type;
+  _LIBCPP_HIDE_FROM_ABI static result_type next(result_type __x) {
+    // Use (((a*x) % m) + c) % m
+    __x = (__a * __x) % __m;
+    __x += __c - (__x >= __m - __c) * __m;
+    return __x;
+  }
+};
+
+template <unsigned long long __a, unsigned long long __c, unsigned long long __m>
+struct __lce_ta<__a, __c, __m, (unsigned long long)(-1), _LCE_Full> {
+  typedef unsigned long long result_type;
+  _LIBCPP_HIDE_FROM_ABI static result_type next(result_type __x) { return (__a * __x + __c) % __m; }
+};
+
+template <unsigned long long __a, unsigned long long __c>
+struct __lce_ta<__a, __c, 0ull, (unsigned long long)(-1), _LCE_Full> {
+  typedef unsigned long long result_type;
+  _LIBCPP_HIDE_FROM_ABI static result_type next(result_type __x) { return __a * __x + __c; }
+};
+
+// 32
+
+template <unsigned long long __a, unsigned long long __c, unsigned long long __m>
+struct __lce_ta<__a, __c, __m, unsigned(-1), _LCE_Promote> {
+  typedef unsigned result_type;
+  _LIBCPP_HIDE_FROM_ABI static result_type next(result_type __x) {
+    return static_cast<result_type>(__lce_ta<__a, __c, __m, (unsigned long long)(-1)>::next(__x));
+  }
+};
+
+template <unsigned long long _Ap, unsigned long long _Cp, unsigned long long _Mp>
+struct __lce_ta<_Ap, _Cp, _Mp, unsigned(-1), _LCE_Schrage> {
+  typedef unsigned result_type;
+  _LIBCPP_HIDE_FROM_ABI static result_type next(result_type __x) {
+    const result_type __a = static_cast<result_type>(_Ap);
+    const result_type __c = static_cast<result_type>(_Cp);
+    const result_type __m = static_cast<result_type>(_Mp);
+    // Schrage's algorithm
+    const result_type __q  = __m / __a;
+    const result_type __r  = __m % __a;
+    const result_type __t0 = __a * (__x % __q);
+    const result_type __t1 = __r * (__x / __q);
+    __x                    = __t0 + (__t0 < __t1) * __m - __t1;
+    __x += __c - (__x >= __m - __c) * __m;
+    return __x;
+  }
+};
+
+template <unsigned long long _Ap, unsigned long long _Mp>
+struct __lce_ta<_Ap, 0ull, _Mp, unsigned(-1), _LCE_Schrage> {
+  typedef unsigned result_type;
+  _LIBCPP_HIDE_FROM_ABI static result_type next(result_type __x) {
+    const result_type __a = static_cast<result_type>(_Ap);
+    const result_type __m = static_cast<result_type>(_Mp);
+    // Schrage's algorithm
+    const result_type __q  = __m / __a;
+    const result_type __r  = __m % __a;
+    const result_type __t0 = __a * (__x % __q);
+    const result_type __t1 = __r * (__x / __q);
+    __x                    = __t0 + (__t0 < __t1) * __m - __t1;
+    return __x;
+  }
+};
+
+template <unsigned long long _Ap, unsigned long long _Cp, unsigned long long _Mp>
+struct __lce_ta<_Ap, _Cp, _Mp, unsigned(-1), _LCE_Part> {
+  typedef unsigned result_type;
+  _LIBCPP_HIDE_FROM_ABI static result_type next(result_type __x) {
+    const result_type __a = static_cast<result_type>(_Ap);
+    const result_type __c = static_cast<result_type>(_Cp);
+    const result_type __m = static_cast<result_type>(_Mp);
+    // Use (((a*x) % m) + c) % m
+    __x = (__a * __x) % __m;
+    __x += __c - (__x >= __m - __c) * __m;
+    return __x;
+  }
+};
+
+template <unsigned long long _Ap, unsigned long long _Cp, unsigned long long _Mp>
+struct __lce_ta<_Ap, _Cp, _Mp, unsigned(-1), _LCE_Full> {
+  typedef unsigned result_type;
+  _LIBCPP_HIDE_FROM_ABI static result_type next(result_type __x) {
+    const result_type __a = static_cast<result_type>(_Ap);
+    const result_type __c = static_cast<result_type>(_Cp);
+    const result_type __m = static_cast<result_type>(_Mp);
+    return (__a * __x + __c) % __m;
+  }
+};
+
+template <unsigned long long _Ap, unsigned long long _Cp>
+struct __lce_ta<_Ap, _Cp, 0ull, unsigned(-1), _LCE_Full> {
+  typedef unsigned result_type;
+  _LIBCPP_HIDE_FROM_ABI static result_type next(result_type __x) {
+    const result_type __a = static_cast<result_type>(_Ap);
+    const result_type __c = static_cast<result_type>(_Cp);
+    return __a * __x + __c;
+  }
+};
+
+// 16
+
+template <unsigned long long __a, unsigned long long __c, unsigned long long __m, __lce_alg_type __mode>
+struct __lce_ta<__a, __c, __m, (unsigned short)(-1), __mode> {
+  typedef unsigned short result_type;
+  _LIBCPP_HIDE_FROM_ABI static result_type next(result_type __x) {
+    return static_cast<result_type>(__lce_ta<__a, __c, __m, unsigned(-1)>::next(__x));
+  }
+};
+
+template <class _UIntType, _UIntType __a, _UIntType __c, _UIntType __m>
+class _LIBCPP_TEMPLATE_VIS linear_congruential_engine;
+
+template <class _CharT, class _Traits, class _Up, _Up _Ap, _Up _Cp, _Up _Np>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const linear_congruential_engine<_Up, _Ap, _Cp, _Np>&);
+
+template <class _CharT, class _Traits, class _Up, _Up _Ap, _Up _Cp, _Up _Np>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, linear_congruential_engine<_Up, _Ap, _Cp, _Np>& __x);
+
+template <class _UIntType, _UIntType __a, _UIntType __c, _UIntType __m>
+class _LIBCPP_TEMPLATE_VIS linear_congruential_engine {
+public:
+  // types
+  typedef _UIntType result_type;
+
+private:
+  result_type __x_;
+
+  static _LIBCPP_CONSTEXPR const result_type _Mp = result_type(-1);
+
+  static_assert(__m == 0 || __a < __m, "linear_congruential_engine invalid parameters");
+  static_assert(__m == 0 || __c < __m, "linear_congruential_engine invalid parameters");
+  static_assert(is_unsigned<_UIntType>::value, "_UIntType must be unsigned type");
+
+public:
+  static _LIBCPP_CONSTEXPR const result_type _Min = __c == 0u ? 1u : 0u;
+  static _LIBCPP_CONSTEXPR const result_type _Max = __m - _UIntType(1u);
+  static_assert(_Min < _Max, "linear_congruential_engine invalid parameters");
+
+  // engine characteristics
+  static _LIBCPP_CONSTEXPR const result_type multiplier = __a;
+  static _LIBCPP_CONSTEXPR const result_type increment  = __c;
+  static _LIBCPP_CONSTEXPR const result_type modulus    = __m;
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR result_type min() { return _Min; }
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR result_type max() { return _Max; }
+  static _LIBCPP_CONSTEXPR const result_type default_seed = 1u;
+
+  // constructors and seeding functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI linear_congruential_engine() : linear_congruential_engine(default_seed) {}
+  _LIBCPP_HIDE_FROM_ABI explicit linear_congruential_engine(result_type __s) { seed(__s); }
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit linear_congruential_engine(result_type __s = default_seed) { seed(__s); }
+#endif
+  template <class _Sseq, __enable_if_t<__is_seed_sequence<_Sseq, linear_congruential_engine>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit linear_congruential_engine(_Sseq& __q) {
+    seed(__q);
+  }
+  _LIBCPP_HIDE_FROM_ABI void seed(result_type __s = default_seed) {
+    seed(integral_constant<bool, __m == 0>(), integral_constant<bool, __c == 0>(), __s);
+  }
+  template <class _Sseq, __enable_if_t<__is_seed_sequence<_Sseq, linear_congruential_engine>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void seed(_Sseq& __q) {
+    __seed(
+        __q,
+        integral_constant<unsigned,
+                          1 + (__m == 0 ? (sizeof(result_type) * __CHAR_BIT__ - 1) / 32 : (__m > 0x100000000ull))>());
+  }
+
+  // generating functions
+  _LIBCPP_HIDE_FROM_ABI result_type operator()() {
+    return __x_ = static_cast<result_type>(__lce_ta<__a, __c, __m, _Mp>::next(__x_));
+  }
+  _LIBCPP_HIDE_FROM_ABI void discard(unsigned long long __z) {
+    for (; __z; --__z)
+      operator()();
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const linear_congruential_engine& __x, const linear_congruential_engine& __y) {
+    return __x.__x_ == __y.__x_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator!=(const linear_congruential_engine& __x, const linear_congruential_engine& __y) {
+    return !(__x == __y);
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI void seed(true_type, true_type, result_type __s) { __x_ = __s == 0 ? 1 : __s; }
+  _LIBCPP_HIDE_FROM_ABI void seed(true_type, false_type, result_type __s) { __x_ = __s; }
+  _LIBCPP_HIDE_FROM_ABI void seed(false_type, true_type, result_type __s) { __x_ = __s % __m == 0 ? 1 : __s % __m; }
+  _LIBCPP_HIDE_FROM_ABI void seed(false_type, false_type, result_type __s) { __x_ = __s % __m; }
+
+  template <class _Sseq>
+  _LIBCPP_HIDE_FROM_ABI void __seed(_Sseq& __q, integral_constant<unsigned, 1>);
+  template <class _Sseq>
+  _LIBCPP_HIDE_FROM_ABI void __seed(_Sseq& __q, integral_constant<unsigned, 2>);
+
+  template <class _CharT, class _Traits, class _Up, _Up _Ap, _Up _Cp, _Up _Np>
+  friend basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const linear_congruential_engine<_Up, _Ap, _Cp, _Np>&);
+
+  template <class _CharT, class _Traits, class _Up, _Up _Ap, _Up _Cp, _Up _Np>
+  friend basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, linear_congruential_engine<_Up, _Ap, _Cp, _Np>& __x);
+};
+
+template <class _UIntType, _UIntType __a, _UIntType __c, _UIntType __m>
+_LIBCPP_CONSTEXPR const typename linear_congruential_engine<_UIntType, __a, __c, __m>::result_type
+    linear_congruential_engine<_UIntType, __a, __c, __m>::multiplier;
+
+template <class _UIntType, _UIntType __a, _UIntType __c, _UIntType __m>
+_LIBCPP_CONSTEXPR const typename linear_congruential_engine<_UIntType, __a, __c, __m>::result_type
+    linear_congruential_engine<_UIntType, __a, __c, __m>::increment;
+
+template <class _UIntType, _UIntType __a, _UIntType __c, _UIntType __m>
+_LIBCPP_CONSTEXPR const typename linear_congruential_engine<_UIntType, __a, __c, __m>::result_type
+    linear_congruential_engine<_UIntType, __a, __c, __m>::modulus;
+
+template <class _UIntType, _UIntType __a, _UIntType __c, _UIntType __m>
+_LIBCPP_CONSTEXPR const typename linear_congruential_engine<_UIntType, __a, __c, __m>::result_type
+    linear_congruential_engine<_UIntType, __a, __c, __m>::default_seed;
+
+template <class _UIntType, _UIntType __a, _UIntType __c, _UIntType __m>
+template <class _Sseq>
+void linear_congruential_engine<_UIntType, __a, __c, __m>::__seed(_Sseq& __q, integral_constant<unsigned, 1>) {
+  const unsigned __k = 1;
+  uint32_t __ar[__k + 3];
+  __q.generate(__ar, __ar + __k + 3);
+  result_type __s = static_cast<result_type>(__ar[3] % __m);
+  __x_            = __c == 0 && __s == 0 ? result_type(1) : __s;
+}
+
+template <class _UIntType, _UIntType __a, _UIntType __c, _UIntType __m>
+template <class _Sseq>
+void linear_congruential_engine<_UIntType, __a, __c, __m>::__seed(_Sseq& __q, integral_constant<unsigned, 2>) {
+  const unsigned __k = 2;
+  uint32_t __ar[__k + 3];
+  __q.generate(__ar, __ar + __k + 3);
+  result_type __s = static_cast<result_type>((__ar[3] + ((uint64_t)__ar[4] << 32)) % __m);
+  __x_            = __c == 0 && __s == 0 ? result_type(1) : __s;
+}
+
+template <class _CharT, class _Traits, class _UIntType, _UIntType __a, _UIntType __c, _UIntType __m>
+inline _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const linear_congruential_engine<_UIntType, __a, __c, __m>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _Ostream;
+  __os.flags(_Ostream::dec | _Ostream::left);
+  __os.fill(__os.widen(' '));
+  return __os << __x.__x_;
+}
+
+template <class _CharT, class _Traits, class _UIntType, _UIntType __a, _UIntType __c, _UIntType __m>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, linear_congruential_engine<_UIntType, __a, __c, __m>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  _UIntType __t;
+  __is >> __t;
+  if (!__is.fail())
+    __x.__x_ = __t;
+  return __is;
+}
+
+typedef linear_congruential_engine<uint_fast32_t, 16807, 0, 2147483647> minstd_rand0;
+typedef linear_congruential_engine<uint_fast32_t, 48271, 0, 2147483647> minstd_rand;
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_LINEAR_CONGRUENTIAL_ENGINE_H
diff --git a/libcxx/include/__cxx03/__random/log2.h b/libcxx/include/__cxx03/__random/log2.h
new file mode 100644
index 000000000000..74b4889c6402
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/log2.h
@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_LOG2_H
+#define _LIBCPP___RANDOM_LOG2_H
+
+#include <__config>
+#include <__type_traits/conditional.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _UIntType, _UIntType _Xp, size_t _Rp>
+struct __log2_imp;
+
+template <unsigned long long _Xp, size_t _Rp>
+struct __log2_imp<unsigned long long, _Xp, _Rp> {
+  static const size_t value =
+      _Xp & ((unsigned long long)(1) << _Rp) ? _Rp : __log2_imp<unsigned long long, _Xp, _Rp - 1>::value;
+};
+
+template <unsigned long long _Xp>
+struct __log2_imp<unsigned long long, _Xp, 0> {
+  static const size_t value = 0;
+};
+
+template <size_t _Rp>
+struct __log2_imp<unsigned long long, 0, _Rp> {
+  static const size_t value = _Rp + 1;
+};
+
+#ifndef _LIBCPP_HAS_NO_INT128
+
+template <__uint128_t _Xp, size_t _Rp>
+struct __log2_imp<__uint128_t, _Xp, _Rp> {
+  static const size_t value =
+      (_Xp >> 64) ? (64 + __log2_imp<unsigned long long, (_Xp >> 64), 63>::value)
+                  : __log2_imp<unsigned long long, _Xp, 63>::value;
+};
+
+#endif // _LIBCPP_HAS_NO_INT128
+
+template <class _UIntType, _UIntType _Xp>
+struct __log2 {
+  static const size_t value = __log2_imp<
+#ifndef _LIBCPP_HAS_NO_INT128
+      __conditional_t<sizeof(_UIntType) <= sizeof(unsigned long long), unsigned long long, __uint128_t>,
+#else
+      unsigned long long,
+#endif // _LIBCPP_HAS_NO_INT128
+      _Xp,
+      sizeof(_UIntType) * __CHAR_BIT__ - 1>::value;
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANDOM_LOG2_H
diff --git a/libcxx/include/__cxx03/__random/lognormal_distribution.h b/libcxx/include/__cxx03/__random/lognormal_distribution.h
new file mode 100644
index 000000000000..d8724f8bc5ce
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/lognormal_distribution.h
@@ -0,0 +1,126 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_LOGNORMAL_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_LOGNORMAL_DISTRIBUTION_H
+
+#include <__config>
+#include <__random/is_valid.h>
+#include <__random/normal_distribution.h>
+#include <cmath>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _RealType = double>
+class _LIBCPP_TEMPLATE_VIS lognormal_distribution {
+  static_assert(__libcpp_random_is_valid_realtype<_RealType>::value,
+                "RealType must be a supported floating-point type");
+
+public:
+  // types
+  typedef _RealType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    result_type __m_;
+    result_type __s_;
+
+  public:
+    typedef lognormal_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(result_type __m = 0, result_type __s = 1) : __m_(__m), __s_(__s) {}
+
+    _LIBCPP_HIDE_FROM_ABI result_type m() const { return __m_; }
+    _LIBCPP_HIDE_FROM_ABI result_type s() const { return __s_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__m_ == __y.__m_ && __x.__s_ == __y.__s_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+  };
+
+private:
+  normal_distribution<result_type> __nd_;
+
+public:
+  // constructor and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI lognormal_distribution() : lognormal_distribution(0) {}
+  _LIBCPP_HIDE_FROM_ABI explicit lognormal_distribution(result_type __m, result_type __s = 1) : __nd_(__m, __s) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit lognormal_distribution(result_type __m = 0, result_type __s = 1) : __nd_(__m, __s) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit lognormal_distribution(const param_type& __p) : __nd_(__p.m(), __p.s()) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() { __nd_.reset(); }
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return std::exp(__nd_(__g));
+  }
+
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p) {
+    typename normal_distribution<result_type>::param_type __pn(__p.m(), __p.s());
+    return std::exp(__nd_(__g, __pn));
+  }
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI result_type m() const { return __nd_.mean(); }
+  _LIBCPP_HIDE_FROM_ABI result_type s() const { return __nd_.stddev(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return param_type(__nd_.mean(), __nd_.stddev()); }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) {
+    typename normal_distribution<result_type>::param_type __pn(__p.m(), __p.s());
+    __nd_.param(__pn);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return 0; }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return numeric_limits<result_type>::infinity(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const lognormal_distribution& __x, const lognormal_distribution& __y) {
+    return __x.__nd_ == __y.__nd_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const lognormal_distribution& __x, const lognormal_distribution& __y) {
+    return !(__x == __y);
+  }
+
+  template <class _CharT, class _Traits, class _RT>
+  friend basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const lognormal_distribution<_RT>& __x);
+
+  template <class _CharT, class _Traits, class _RT>
+  friend basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, lognormal_distribution<_RT>& __x);
+};
+
+template <class _CharT, class _Traits, class _RT>
+inline _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const lognormal_distribution<_RT>& __x) {
+  return __os << __x.__nd_;
+}
+
+template <class _CharT, class _Traits, class _RT>
+inline _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, lognormal_distribution<_RT>& __x) {
+  return __is >> __x.__nd_;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_LOGNORMAL_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/mersenne_twister_engine.h b/libcxx/include/__cxx03/__random/mersenne_twister_engine.h
new file mode 100644
index 000000000000..65280d7c5505
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/mersenne_twister_engine.h
@@ -0,0 +1,914 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_MERSENNE_TWISTER_ENGINE_H
+#define _LIBCPP___RANDOM_MERSENNE_TWISTER_ENGINE_H
+
+#include <__algorithm/equal.h>
+#include <__algorithm/min.h>
+#include <__config>
+#include <__random/is_seed_sequence.h>
+#include <cstddef>
+#include <cstdint>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+class _LIBCPP_TEMPLATE_VIS mersenne_twister_engine;
+
+template <class _UInt,
+          size_t _Wp,
+          size_t _Np,
+          size_t _Mp,
+          size_t _Rp,
+          _UInt _Ap,
+          size_t _Up,
+          _UInt _Dp,
+          size_t _Sp,
+          _UInt _Bp,
+          size_t _Tp,
+          _UInt _Cp,
+          size_t _Lp,
+          _UInt _Fp>
+_LIBCPP_HIDE_FROM_ABI bool
+operator==(const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x,
+           const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __y);
+
+template <class _UInt,
+          size_t _Wp,
+          size_t _Np,
+          size_t _Mp,
+          size_t _Rp,
+          _UInt _Ap,
+          size_t _Up,
+          _UInt _Dp,
+          size_t _Sp,
+          _UInt _Bp,
+          size_t _Tp,
+          _UInt _Cp,
+          size_t _Lp,
+          _UInt _Fp>
+_LIBCPP_HIDE_FROM_ABI bool
+operator!=(const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x,
+           const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __y);
+
+template <class _CharT,
+          class _Traits,
+          class _UInt,
+          size_t _Wp,
+          size_t _Np,
+          size_t _Mp,
+          size_t _Rp,
+          _UInt _Ap,
+          size_t _Up,
+          _UInt _Dp,
+          size_t _Sp,
+          _UInt _Bp,
+          size_t _Tp,
+          _UInt _Cp,
+          size_t _Lp,
+          _UInt _Fp>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os,
+           const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x);
+
+template <class _CharT,
+          class _Traits,
+          class _UInt,
+          size_t _Wp,
+          size_t _Np,
+          size_t _Mp,
+          size_t _Rp,
+          _UInt _Ap,
+          size_t _Up,
+          _UInt _Dp,
+          size_t _Sp,
+          _UInt _Bp,
+          size_t _Tp,
+          _UInt _Cp,
+          size_t _Lp,
+          _UInt _Fp>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is,
+           mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x);
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+class _LIBCPP_TEMPLATE_VIS mersenne_twister_engine {
+public:
+  // types
+  typedef _UIntType result_type;
+
+private:
+  result_type __x_[__n];
+  size_t __i_;
+
+  static_assert(0 < __m, "mersenne_twister_engine invalid parameters");
+  static_assert(__m <= __n, "mersenne_twister_engine invalid parameters");
+  static _LIBCPP_CONSTEXPR const result_type _Dt = numeric_limits<result_type>::digits;
+  static_assert(__w <= _Dt, "mersenne_twister_engine invalid parameters");
+  static_assert(2 <= __w, "mersenne_twister_engine invalid parameters");
+  static_assert(__r <= __w, "mersenne_twister_engine invalid parameters");
+  static_assert(__u <= __w, "mersenne_twister_engine invalid parameters");
+  static_assert(__s <= __w, "mersenne_twister_engine invalid parameters");
+  static_assert(__t <= __w, "mersenne_twister_engine invalid parameters");
+  static_assert(__l <= __w, "mersenne_twister_engine invalid parameters");
+
+public:
+  static _LIBCPP_CONSTEXPR const result_type _Min = 0;
+  static _LIBCPP_CONSTEXPR const result_type _Max =
+      __w == _Dt ? result_type(~0) : (result_type(1) << __w) - result_type(1);
+  static_assert(_Min < _Max, "mersenne_twister_engine invalid parameters");
+  static_assert(__a <= _Max, "mersenne_twister_engine invalid parameters");
+  static_assert(__b <= _Max, "mersenne_twister_engine invalid parameters");
+  static_assert(__c <= _Max, "mersenne_twister_engine invalid parameters");
+  static_assert(__d <= _Max, "mersenne_twister_engine invalid parameters");
+  static_assert(__f <= _Max, "mersenne_twister_engine invalid parameters");
+
+  // engine characteristics
+  static _LIBCPP_CONSTEXPR const size_t word_size                      = __w;
+  static _LIBCPP_CONSTEXPR const size_t state_size                     = __n;
+  static _LIBCPP_CONSTEXPR const size_t shift_size                     = __m;
+  static _LIBCPP_CONSTEXPR const size_t mask_bits                      = __r;
+  static _LIBCPP_CONSTEXPR const result_type xor_mask                  = __a;
+  static _LIBCPP_CONSTEXPR const size_t tempering_u                    = __u;
+  static _LIBCPP_CONSTEXPR const result_type tempering_d               = __d;
+  static _LIBCPP_CONSTEXPR const size_t tempering_s                    = __s;
+  static _LIBCPP_CONSTEXPR const result_type tempering_b               = __b;
+  static _LIBCPP_CONSTEXPR const size_t tempering_t                    = __t;
+  static _LIBCPP_CONSTEXPR const result_type tempering_c               = __c;
+  static _LIBCPP_CONSTEXPR const size_t tempering_l                    = __l;
+  static _LIBCPP_CONSTEXPR const result_type initialization_multiplier = __f;
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR result_type min() { return _Min; }
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR result_type max() { return _Max; }
+  static _LIBCPP_CONSTEXPR const result_type default_seed = 5489u;
+
+  // constructors and seeding functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI mersenne_twister_engine() : mersenne_twister_engine(default_seed) {}
+  _LIBCPP_HIDE_FROM_ABI explicit mersenne_twister_engine(result_type __sd) { seed(__sd); }
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit mersenne_twister_engine(result_type __sd = default_seed) { seed(__sd); }
+#endif
+  template <class _Sseq, __enable_if_t<__is_seed_sequence<_Sseq, mersenne_twister_engine>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit mersenne_twister_engine(_Sseq& __q) {
+    seed(__q);
+  }
+  _LIBCPP_HIDE_FROM_ABI void seed(result_type __sd = default_seed);
+  template <class _Sseq, __enable_if_t<__is_seed_sequence<_Sseq, mersenne_twister_engine>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void seed(_Sseq& __q) {
+    __seed(__q, integral_constant<unsigned, 1 + (__w - 1) / 32>());
+  }
+
+  // generating functions
+  _LIBCPP_HIDE_FROM_ABI result_type operator()();
+  _LIBCPP_HIDE_FROM_ABI void discard(unsigned long long __z) {
+    for (; __z; --__z)
+      operator()();
+  }
+
+  template <class _UInt,
+            size_t _Wp,
+            size_t _Np,
+            size_t _Mp,
+            size_t _Rp,
+            _UInt _Ap,
+            size_t _Up,
+            _UInt _Dp,
+            size_t _Sp,
+            _UInt _Bp,
+            size_t _Tp,
+            _UInt _Cp,
+            size_t _Lp,
+            _UInt _Fp>
+  friend bool operator==(
+      const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x,
+      const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __y);
+
+  template <class _UInt,
+            size_t _Wp,
+            size_t _Np,
+            size_t _Mp,
+            size_t _Rp,
+            _UInt _Ap,
+            size_t _Up,
+            _UInt _Dp,
+            size_t _Sp,
+            _UInt _Bp,
+            size_t _Tp,
+            _UInt _Cp,
+            size_t _Lp,
+            _UInt _Fp>
+  friend bool operator!=(
+      const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x,
+      const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __y);
+
+  template <class _CharT,
+            class _Traits,
+            class _UInt,
+            size_t _Wp,
+            size_t _Np,
+            size_t _Mp,
+            size_t _Rp,
+            _UInt _Ap,
+            size_t _Up,
+            _UInt _Dp,
+            size_t _Sp,
+            _UInt _Bp,
+            size_t _Tp,
+            _UInt _Cp,
+            size_t _Lp,
+            _UInt _Fp>
+  friend basic_ostream<_CharT, _Traits>& operator<<(
+      basic_ostream<_CharT, _Traits>& __os,
+      const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x);
+
+  template <class _CharT,
+            class _Traits,
+            class _UInt,
+            size_t _Wp,
+            size_t _Np,
+            size_t _Mp,
+            size_t _Rp,
+            _UInt _Ap,
+            size_t _Up,
+            _UInt _Dp,
+            size_t _Sp,
+            _UInt _Bp,
+            size_t _Tp,
+            _UInt _Cp,
+            size_t _Lp,
+            _UInt _Fp>
+  friend basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is,
+             mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x);
+
+private:
+  template <class _Sseq>
+  _LIBCPP_HIDE_FROM_ABI void __seed(_Sseq& __q, integral_constant<unsigned, 1>);
+  template <class _Sseq>
+  _LIBCPP_HIDE_FROM_ABI void __seed(_Sseq& __q, integral_constant<unsigned, 2>);
+
+  template <size_t __count,
+            __enable_if_t<__count< __w, int> = 0> _LIBCPP_HIDE_FROM_ABI static result_type __lshift(result_type __x) {
+    return (__x << __count) & _Max;
+  }
+
+  template <size_t __count, __enable_if_t<(__count >= __w), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI static result_type __lshift(result_type) {
+    return result_type(0);
+  }
+
+  template <size_t __count,
+            __enable_if_t<__count< _Dt, int> = 0> _LIBCPP_HIDE_FROM_ABI static result_type __rshift(result_type __x) {
+    return __x >> __count;
+  }
+
+  template <size_t __count, __enable_if_t<(__count >= _Dt), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI static result_type __rshift(result_type) {
+    return result_type(0);
+  }
+};
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+_LIBCPP_CONSTEXPR const size_t
+    mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::word_size;
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+_LIBCPP_CONSTEXPR const size_t
+    mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::state_size;
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+_LIBCPP_CONSTEXPR const size_t
+    mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::shift_size;
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+_LIBCPP_CONSTEXPR const size_t
+    mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::mask_bits;
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+_LIBCPP_CONSTEXPR const typename mersenne_twister_engine<
+    _UIntType,
+    __w,
+    __n,
+    __m,
+    __r,
+    __a,
+    __u,
+    __d,
+    __s,
+    __b,
+    __t,
+    __c,
+    __l,
+    __f>::result_type
+    mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::xor_mask;
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+_LIBCPP_CONSTEXPR const size_t
+    mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::tempering_u;
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+_LIBCPP_CONSTEXPR const typename mersenne_twister_engine<
+    _UIntType,
+    __w,
+    __n,
+    __m,
+    __r,
+    __a,
+    __u,
+    __d,
+    __s,
+    __b,
+    __t,
+    __c,
+    __l,
+    __f>::result_type
+    mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::tempering_d;
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+_LIBCPP_CONSTEXPR const size_t
+    mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::tempering_s;
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+_LIBCPP_CONSTEXPR const typename mersenne_twister_engine<
+    _UIntType,
+    __w,
+    __n,
+    __m,
+    __r,
+    __a,
+    __u,
+    __d,
+    __s,
+    __b,
+    __t,
+    __c,
+    __l,
+    __f>::result_type
+    mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::tempering_b;
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+_LIBCPP_CONSTEXPR const size_t
+    mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::tempering_t;
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+_LIBCPP_CONSTEXPR const typename mersenne_twister_engine<
+    _UIntType,
+    __w,
+    __n,
+    __m,
+    __r,
+    __a,
+    __u,
+    __d,
+    __s,
+    __b,
+    __t,
+    __c,
+    __l,
+    __f>::result_type
+    mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::tempering_c;
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+_LIBCPP_CONSTEXPR const size_t
+    mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::tempering_l;
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+_LIBCPP_CONSTEXPR const typename mersenne_twister_engine<
+    _UIntType,
+    __w,
+    __n,
+    __m,
+    __r,
+    __a,
+    __u,
+    __d,
+    __s,
+    __b,
+    __t,
+    __c,
+    __l,
+    __f>::result_type
+    mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::
+        initialization_multiplier;
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+_LIBCPP_CONSTEXPR const typename mersenne_twister_engine<
+    _UIntType,
+    __w,
+    __n,
+    __m,
+    __r,
+    __a,
+    __u,
+    __d,
+    __s,
+    __b,
+    __t,
+    __c,
+    __l,
+    __f>::result_type
+    mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::default_seed;
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+void mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::seed(
+    result_type __sd) _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK { // __w >= 2
+  __x_[0] = __sd & _Max;
+  for (size_t __i = 1; __i < __n; ++__i)
+    __x_[__i] = (__f * (__x_[__i - 1] ^ __rshift<__w - 2>(__x_[__i - 1])) + __i) & _Max;
+  __i_ = 0;
+}
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+template <class _Sseq>
+void mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::__seed(
+    _Sseq& __q, integral_constant<unsigned, 1>) {
+  const unsigned __k = 1;
+  uint32_t __ar[__n * __k];
+  __q.generate(__ar, __ar + __n * __k);
+  for (size_t __i = 0; __i < __n; ++__i)
+    __x_[__i] = static_cast<result_type>(__ar[__i] & _Max);
+  const result_type __mask = __r == _Dt ? result_type(~0) : (result_type(1) << __r) - result_type(1);
+  __i_                     = 0;
+  if ((__x_[0] & ~__mask) == 0) {
+    for (size_t __i = 1; __i < __n; ++__i)
+      if (__x_[__i] != 0)
+        return;
+    __x_[0] = result_type(1) << (__w - 1);
+  }
+}
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+template <class _Sseq>
+void mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::__seed(
+    _Sseq& __q, integral_constant<unsigned, 2>) {
+  const unsigned __k = 2;
+  uint32_t __ar[__n * __k];
+  __q.generate(__ar, __ar + __n * __k);
+  for (size_t __i = 0; __i < __n; ++__i)
+    __x_[__i] = static_cast<result_type>((__ar[2 * __i] + ((uint64_t)__ar[2 * __i + 1] << 32)) & _Max);
+  const result_type __mask = __r == _Dt ? result_type(~0) : (result_type(1) << __r) - result_type(1);
+  __i_                     = 0;
+  if ((__x_[0] & ~__mask) == 0) {
+    for (size_t __i = 1; __i < __n; ++__i)
+      if (__x_[__i] != 0)
+        return;
+    __x_[0] = result_type(1) << (__w - 1);
+  }
+}
+
+template <class _UIntType,
+          size_t __w,
+          size_t __n,
+          size_t __m,
+          size_t __r,
+          _UIntType __a,
+          size_t __u,
+          _UIntType __d,
+          size_t __s,
+          _UIntType __b,
+          size_t __t,
+          _UIntType __c,
+          size_t __l,
+          _UIntType __f>
+_UIntType
+mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::operator()() {
+  const size_t __j         = (__i_ + 1) % __n;
+  const result_type __mask = __r == _Dt ? result_type(~0) : (result_type(1) << __r) - result_type(1);
+  const result_type __yp   = (__x_[__i_] & ~__mask) | (__x_[__j] & __mask);
+  const size_t __k         = (__i_ + __m) % __n;
+  __x_[__i_]               = __x_[__k] ^ __rshift<1>(__yp) ^ (__a * (__yp & 1));
+  result_type __z          = __x_[__i_] ^ (__rshift<__u>(__x_[__i_]) & __d);
+  __i_                     = __j;
+  __z ^= __lshift<__s>(__z) & __b;
+  __z ^= __lshift<__t>(__z) & __c;
+  return __z ^ __rshift<__l>(__z);
+}
+
+template <class _UInt,
+          size_t _Wp,
+          size_t _Np,
+          size_t _Mp,
+          size_t _Rp,
+          _UInt _Ap,
+          size_t _Up,
+          _UInt _Dp,
+          size_t _Sp,
+          _UInt _Bp,
+          size_t _Tp,
+          _UInt _Cp,
+          size_t _Lp,
+          _UInt _Fp>
+_LIBCPP_HIDE_FROM_ABI bool
+operator==(const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x,
+           const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __y) {
+  if (__x.__i_ == __y.__i_)
+    return std::equal(__x.__x_, __x.__x_ + _Np, __y.__x_);
+  if (__x.__i_ == 0 || __y.__i_ == 0) {
+    size_t __j = std::min(_Np - __x.__i_, _Np - __y.__i_);
+    if (!std::equal(__x.__x_ + __x.__i_, __x.__x_ + __x.__i_ + __j, __y.__x_ + __y.__i_))
+      return false;
+    if (__x.__i_ == 0)
+      return std::equal(__x.__x_ + __j, __x.__x_ + _Np, __y.__x_);
+    return std::equal(__x.__x_, __x.__x_ + (_Np - __j), __y.__x_ + __j);
+  }
+  if (__x.__i_ < __y.__i_) {
+    size_t __j = _Np - __y.__i_;
+    if (!std::equal(__x.__x_ + __x.__i_, __x.__x_ + (__x.__i_ + __j), __y.__x_ + __y.__i_))
+      return false;
+    if (!std::equal(__x.__x_ + (__x.__i_ + __j), __x.__x_ + _Np, __y.__x_))
+      return false;
+    return std::equal(__x.__x_, __x.__x_ + __x.__i_, __y.__x_ + (_Np - (__x.__i_ + __j)));
+  }
+  size_t __j = _Np - __x.__i_;
+  if (!std::equal(__y.__x_ + __y.__i_, __y.__x_ + (__y.__i_ + __j), __x.__x_ + __x.__i_))
+    return false;
+  if (!std::equal(__y.__x_ + (__y.__i_ + __j), __y.__x_ + _Np, __x.__x_))
+    return false;
+  return std::equal(__y.__x_, __y.__x_ + __y.__i_, __x.__x_ + (_Np - (__y.__i_ + __j)));
+}
+
+template <class _UInt,
+          size_t _Wp,
+          size_t _Np,
+          size_t _Mp,
+          size_t _Rp,
+          _UInt _Ap,
+          size_t _Up,
+          _UInt _Dp,
+          size_t _Sp,
+          _UInt _Bp,
+          size_t _Tp,
+          _UInt _Cp,
+          size_t _Lp,
+          _UInt _Fp>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x,
+           const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __y) {
+  return !(__x == __y);
+}
+
+template <class _CharT,
+          class _Traits,
+          class _UInt,
+          size_t _Wp,
+          size_t _Np,
+          size_t _Mp,
+          size_t _Rp,
+          _UInt _Ap,
+          size_t _Up,
+          _UInt _Dp,
+          size_t _Sp,
+          _UInt _Bp,
+          size_t _Tp,
+          _UInt _Cp,
+          size_t _Lp,
+          _UInt _Fp>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os,
+           const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _Ostream;
+  __os.flags(_Ostream::dec | _Ostream::left);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  __os << __x.__x_[__x.__i_];
+  for (size_t __j = __x.__i_ + 1; __j < _Np; ++__j)
+    __os << __sp << __x.__x_[__j];
+  for (size_t __j = 0; __j < __x.__i_; ++__j)
+    __os << __sp << __x.__x_[__j];
+  return __os;
+}
+
+template <class _CharT,
+          class _Traits,
+          class _UInt,
+          size_t _Wp,
+          size_t _Np,
+          size_t _Mp,
+          size_t _Rp,
+          _UInt _Ap,
+          size_t _Up,
+          _UInt _Dp,
+          size_t _Sp,
+          _UInt _Bp,
+          size_t _Tp,
+          _UInt _Cp,
+          size_t _Lp,
+          _UInt _Fp>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is,
+           mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  _UInt __t[_Np];
+  for (size_t __i = 0; __i < _Np; ++__i)
+    __is >> __t[__i];
+  if (!__is.fail()) {
+    for (size_t __i = 0; __i < _Np; ++__i)
+      __x.__x_[__i] = __t[__i];
+    __x.__i_ = 0;
+  }
+  return __is;
+}
+
+typedef mersenne_twister_engine<
+    uint_fast32_t,
+    32,
+    624,
+    397,
+    31,
+    0x9908b0df,
+    11,
+    0xffffffff,
+    7,
+    0x9d2c5680,
+    15,
+    0xefc60000,
+    18,
+    1812433253>
+    mt19937;
+typedef mersenne_twister_engine<
+    uint_fast64_t,
+    64,
+    312,
+    156,
+    31,
+    0xb5026f5aa96619e9ULL,
+    29,
+    0x5555555555555555ULL,
+    17,
+    0x71d67fffeda60000ULL,
+    37,
+    0xfff7eee000000000ULL,
+    43,
+    6364136223846793005ULL>
+    mt19937_64;
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_MERSENNE_TWISTER_ENGINE_H
diff --git a/libcxx/include/__cxx03/__random/negative_binomial_distribution.h b/libcxx/include/__cxx03/__random/negative_binomial_distribution.h
new file mode 100644
index 000000000000..6d0055d01ed4
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/negative_binomial_distribution.h
@@ -0,0 +1,157 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_NEGATIVE_BINOMIAL_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_NEGATIVE_BINOMIAL_DISTRIBUTION_H
+
+#include <__assert>
+#include <__config>
+#include <__random/bernoulli_distribution.h>
+#include <__random/gamma_distribution.h>
+#include <__random/is_valid.h>
+#include <__random/poisson_distribution.h>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _IntType = int>
+class _LIBCPP_TEMPLATE_VIS negative_binomial_distribution {
+  static_assert(__libcpp_random_is_valid_inttype<_IntType>::value, "IntType must be a supported integer type");
+
+public:
+  // types
+  typedef _IntType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    result_type __k_;
+    double __p_;
+
+  public:
+    typedef negative_binomial_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(result_type __k = 1, double __p = 0.5) : __k_(__k), __p_(__p) {}
+
+    _LIBCPP_HIDE_FROM_ABI result_type k() const { return __k_; }
+    _LIBCPP_HIDE_FROM_ABI double p() const { return __p_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__k_ == __y.__k_ && __x.__p_ == __y.__p_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructor and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI negative_binomial_distribution() : negative_binomial_distribution(1) {}
+  _LIBCPP_HIDE_FROM_ABI explicit negative_binomial_distribution(result_type __k, double __p = 0.5) : __p_(__k, __p) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit negative_binomial_distribution(result_type __k = 1, double __p = 0.5)
+      : __p_(__k, __p) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit negative_binomial_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI result_type k() const { return __p_.k(); }
+  _LIBCPP_HIDE_FROM_ABI double p() const { return __p_.p(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return 0; }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return numeric_limits<result_type>::max(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const negative_binomial_distribution& __x, const negative_binomial_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator!=(const negative_binomial_distribution& __x, const negative_binomial_distribution& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _IntType>
+template <class _URNG>
+_IntType negative_binomial_distribution<_IntType>::operator()(_URNG& __urng, const param_type& __pr) {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  result_type __k = __pr.k();
+  double __p      = __pr.p();
+  // When the number of bits in _IntType is small, we are too likely to
+  // overflow __f below to use this technique.
+  if (__k <= 21 * __p && sizeof(_IntType) > 1) {
+    bernoulli_distribution __gen(__p);
+    result_type __f = 0;
+    result_type __s = 0;
+    while (__s < __k) {
+      if (__gen(__urng))
+        ++__s;
+      else
+        ++__f;
+    }
+    _LIBCPP_ASSERT_INTERNAL(__f >= 0,
+                            "std::negative_binomial_distribution should never produce negative values. "
+                            "This is almost certainly a signed integer overflow issue on __f.");
+    return __f;
+  }
+  return poisson_distribution<result_type>(gamma_distribution<double>(__k, (1 - __p) / __p)(__urng))(__urng);
+}
+
+template <class _CharT, class _Traits, class _IntType>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const negative_binomial_distribution<_IntType>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  return __os << __x.k() << __sp << __x.p();
+}
+
+template <class _CharT, class _Traits, class _IntType>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, negative_binomial_distribution<_IntType>& __x) {
+  typedef negative_binomial_distribution<_IntType> _Eng;
+  typedef typename _Eng::result_type result_type;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  result_type __k;
+  double __p;
+  __is >> __k >> __p;
+  if (!__is.fail())
+    __x.param(param_type(__k, __p));
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_NEGATIVE_BINOMIAL_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/normal_distribution.h b/libcxx/include/__cxx03/__random/normal_distribution.h
new file mode 100644
index 000000000000..889f189e4161
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/normal_distribution.h
@@ -0,0 +1,176 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_NORMAL_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_NORMAL_DISTRIBUTION_H
+
+#include <__config>
+#include <__random/is_valid.h>
+#include <__random/uniform_real_distribution.h>
+#include <cmath>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _RealType = double>
+class _LIBCPP_TEMPLATE_VIS normal_distribution {
+  static_assert(__libcpp_random_is_valid_realtype<_RealType>::value,
+                "RealType must be a supported floating-point type");
+
+public:
+  // types
+  typedef _RealType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    result_type __mean_;
+    result_type __stddev_;
+
+  public:
+    typedef normal_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(result_type __mean = 0, result_type __stddev = 1)
+        : __mean_(__mean), __stddev_(__stddev) {}
+
+    _LIBCPP_HIDE_FROM_ABI result_type mean() const { return __mean_; }
+    _LIBCPP_HIDE_FROM_ABI result_type stddev() const { return __stddev_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__mean_ == __y.__mean_ && __x.__stddev_ == __y.__stddev_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+  };
+
+private:
+  param_type __p_;
+  result_type __v_;
+  bool __v_hot_;
+
+public:
+  // constructors and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI normal_distribution() : normal_distribution(0) {}
+  _LIBCPP_HIDE_FROM_ABI explicit normal_distribution(result_type __mean, result_type __stddev = 1)
+      : __p_(param_type(__mean, __stddev)), __v_hot_(false) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit normal_distribution(result_type __mean = 0, result_type __stddev = 1)
+      : __p_(param_type(__mean, __stddev)), __v_hot_(false) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit normal_distribution(const param_type& __p) : __p_(__p), __v_hot_(false) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() { __v_hot_ = false; }
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI result_type mean() const { return __p_.mean(); }
+  _LIBCPP_HIDE_FROM_ABI result_type stddev() const { return __p_.stddev(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return -numeric_limits<result_type>::infinity(); }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return numeric_limits<result_type>::infinity(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const normal_distribution& __x, const normal_distribution& __y) {
+    return __x.__p_ == __y.__p_ && __x.__v_hot_ == __y.__v_hot_ && (!__x.__v_hot_ || __x.__v_ == __y.__v_);
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const normal_distribution& __x, const normal_distribution& __y) {
+    return !(__x == __y);
+  }
+
+  template <class _CharT, class _Traits, class _RT>
+  friend basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const normal_distribution<_RT>& __x);
+
+  template <class _CharT, class _Traits, class _RT>
+  friend basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, normal_distribution<_RT>& __x);
+};
+
+template <class _RealType>
+template <class _URNG>
+_RealType normal_distribution<_RealType>::operator()(_URNG& __g, const param_type& __p) {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  result_type __up;
+  if (__v_hot_) {
+    __v_hot_ = false;
+    __up     = __v_;
+  } else {
+    uniform_real_distribution<result_type> __uni(-1, 1);
+    result_type __u;
+    result_type __v;
+    result_type __s;
+    do {
+      __u = __uni(__g);
+      __v = __uni(__g);
+      __s = __u * __u + __v * __v;
+    } while (__s > 1 || __s == 0);
+    result_type __fp = std::sqrt(-2 * std::log(__s) / __s);
+    __v_             = __v * __fp;
+    __v_hot_         = true;
+    __up             = __u * __fp;
+  }
+  return __up * __p.stddev() + __p.mean();
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const normal_distribution<_RT>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  __os << __x.mean() << __sp << __x.stddev() << __sp << __x.__v_hot_;
+  if (__x.__v_hot_)
+    __os << __sp << __x.__v_;
+  return __os;
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, normal_distribution<_RT>& __x) {
+  typedef normal_distribution<_RT> _Eng;
+  typedef typename _Eng::result_type result_type;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  result_type __mean;
+  result_type __stddev;
+  result_type __vp = 0;
+  bool __v_hot     = false;
+  __is >> __mean >> __stddev >> __v_hot;
+  if (__v_hot)
+    __is >> __vp;
+  if (!__is.fail()) {
+    __x.param(param_type(__mean, __stddev));
+    __x.__v_hot_ = __v_hot;
+    __x.__v_     = __vp;
+  }
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_NORMAL_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/piecewise_constant_distribution.h b/libcxx/include/__cxx03/__random/piecewise_constant_distribution.h
new file mode 100644
index 000000000000..e19380f97c35
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/piecewise_constant_distribution.h
@@ -0,0 +1,302 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_PIECEWISE_CONSTANT_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_PIECEWISE_CONSTANT_DISTRIBUTION_H
+
+#include <__algorithm/upper_bound.h>
+#include <__config>
+#include <__random/is_valid.h>
+#include <__random/uniform_real_distribution.h>
+#include <iosfwd>
+#include <numeric>
+#include <vector>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _RealType = double>
+class _LIBCPP_TEMPLATE_VIS piecewise_constant_distribution {
+  static_assert(__libcpp_random_is_valid_realtype<_RealType>::value,
+                "RealType must be a supported floating-point type");
+
+public:
+  // types
+  typedef _RealType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    vector<result_type> __b_;
+    vector<result_type> __densities_;
+    vector<result_type> __areas_;
+
+  public:
+    typedef piecewise_constant_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI param_type();
+    template <class _InputIteratorB, class _InputIteratorW>
+    _LIBCPP_HIDE_FROM_ABI param_type(_InputIteratorB __f_b, _InputIteratorB __l_b, _InputIteratorW __f_w);
+#ifndef _LIBCPP_CXX03_LANG
+    template <class _UnaryOperation>
+    _LIBCPP_HIDE_FROM_ABI param_type(initializer_list<result_type> __bl, _UnaryOperation __fw);
+#endif // _LIBCPP_CXX03_LANG
+    template <class _UnaryOperation>
+    _LIBCPP_HIDE_FROM_ABI param_type(size_t __nw, result_type __xmin, result_type __xmax, _UnaryOperation __fw);
+    _LIBCPP_HIDE_FROM_ABI param_type(param_type const&) = default;
+    _LIBCPP_HIDE_FROM_ABI param_type& operator=(const param_type& __rhs);
+
+    _LIBCPP_HIDE_FROM_ABI vector<result_type> intervals() const { return __b_; }
+    _LIBCPP_HIDE_FROM_ABI vector<result_type> densities() const { return __densities_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__densities_ == __y.__densities_ && __x.__b_ == __y.__b_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+
+  private:
+    _LIBCPP_HIDE_FROM_ABI void __init();
+
+    friend class piecewise_constant_distribution;
+
+    template <class _CharT, class _Traits, class _RT>
+    friend basic_ostream<_CharT, _Traits>&
+    operator<<(basic_ostream<_CharT, _Traits>& __os, const piecewise_constant_distribution<_RT>& __x);
+
+    template <class _CharT, class _Traits, class _RT>
+    friend basic_istream<_CharT, _Traits>&
+    operator>>(basic_istream<_CharT, _Traits>& __is, piecewise_constant_distribution<_RT>& __x);
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructor and reset functions
+  _LIBCPP_HIDE_FROM_ABI piecewise_constant_distribution() {}
+  template <class _InputIteratorB, class _InputIteratorW>
+  _LIBCPP_HIDE_FROM_ABI
+  piecewise_constant_distribution(_InputIteratorB __f_b, _InputIteratorB __l_b, _InputIteratorW __f_w)
+      : __p_(__f_b, __l_b, __f_w) {}
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class _UnaryOperation>
+  _LIBCPP_HIDE_FROM_ABI piecewise_constant_distribution(initializer_list<result_type> __bl, _UnaryOperation __fw)
+      : __p_(__bl, __fw) {}
+#endif // _LIBCPP_CXX03_LANG
+
+  template <class _UnaryOperation>
+  _LIBCPP_HIDE_FROM_ABI
+  piecewise_constant_distribution(size_t __nw, result_type __xmin, result_type __xmax, _UnaryOperation __fw)
+      : __p_(__nw, __xmin, __xmax, __fw) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit piecewise_constant_distribution(const param_type& __p) : __p_(__p) {}
+
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI vector<result_type> intervals() const { return __p_.intervals(); }
+  _LIBCPP_HIDE_FROM_ABI vector<result_type> densities() const { return __p_.densities(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return __p_.__b_.front(); }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return __p_.__b_.back(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const piecewise_constant_distribution& __x, const piecewise_constant_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator!=(const piecewise_constant_distribution& __x, const piecewise_constant_distribution& __y) {
+    return !(__x == __y);
+  }
+
+  template <class _CharT, class _Traits, class _RT>
+  friend basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const piecewise_constant_distribution<_RT>& __x);
+
+  template <class _CharT, class _Traits, class _RT>
+  friend basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, piecewise_constant_distribution<_RT>& __x);
+};
+
+template <class _RealType>
+typename piecewise_constant_distribution<_RealType>::param_type&
+piecewise_constant_distribution<_RealType>::param_type::operator=(const param_type& __rhs) {
+  //  These can throw
+  __b_.reserve(__rhs.__b_.size());
+  __densities_.reserve(__rhs.__densities_.size());
+  __areas_.reserve(__rhs.__areas_.size());
+
+  //  These can not throw
+  __b_         = __rhs.__b_;
+  __densities_ = __rhs.__densities_;
+  __areas_     = __rhs.__areas_;
+  return *this;
+}
+
+template <class _RealType>
+void piecewise_constant_distribution<_RealType>::param_type::__init() {
+  // __densities_ contains non-normalized areas
+  result_type __total_area = std::accumulate(__densities_.begin(), __densities_.end(), result_type());
+  for (size_t __i = 0; __i < __densities_.size(); ++__i)
+    __densities_[__i] /= __total_area;
+  // __densities_ contains normalized areas
+  __areas_.assign(__densities_.size(), result_type());
+  std::partial_sum(__densities_.begin(), __densities_.end() - 1, __areas_.begin() + 1);
+  // __areas_ contains partial sums of normalized areas: [0, __densities_ - 1]
+  __densities_.back() = 1 - __areas_.back(); // correct round off error
+  for (size_t __i = 0; __i < __densities_.size(); ++__i)
+    __densities_[__i] /= (__b_[__i + 1] - __b_[__i]);
+  // __densities_ now contains __densities_
+}
+
+template <class _RealType>
+piecewise_constant_distribution<_RealType>::param_type::param_type() : __b_(2), __densities_(1, 1.0), __areas_(1, 0.0) {
+  __b_[1] = 1;
+}
+
+template <class _RealType>
+template <class _InputIteratorB, class _InputIteratorW>
+piecewise_constant_distribution<_RealType>::param_type::param_type(
+    _InputIteratorB __f_b, _InputIteratorB __l_b, _InputIteratorW __f_w)
+    : __b_(__f_b, __l_b) {
+  if (__b_.size() < 2) {
+    __b_.resize(2);
+    __b_[0] = 0;
+    __b_[1] = 1;
+    __densities_.assign(1, 1.0);
+    __areas_.assign(1, 0.0);
+  } else {
+    __densities_.reserve(__b_.size() - 1);
+    for (size_t __i = 0; __i < __b_.size() - 1; ++__i, ++__f_w)
+      __densities_.push_back(*__f_w);
+    __init();
+  }
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _RealType>
+template <class _UnaryOperation>
+piecewise_constant_distribution<_RealType>::param_type::param_type(
+    initializer_list<result_type> __bl, _UnaryOperation __fw)
+    : __b_(__bl.begin(), __bl.end()) {
+  if (__b_.size() < 2) {
+    __b_.resize(2);
+    __b_[0] = 0;
+    __b_[1] = 1;
+    __densities_.assign(1, 1.0);
+    __areas_.assign(1, 0.0);
+  } else {
+    __densities_.reserve(__b_.size() - 1);
+    for (size_t __i = 0; __i < __b_.size() - 1; ++__i)
+      __densities_.push_back(__fw((__b_[__i + 1] + __b_[__i]) * .5));
+    __init();
+  }
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _RealType>
+template <class _UnaryOperation>
+piecewise_constant_distribution<_RealType>::param_type::param_type(
+    size_t __nw, result_type __xmin, result_type __xmax, _UnaryOperation __fw)
+    : __b_(__nw == 0 ? 2 : __nw + 1) {
+  size_t __n      = __b_.size() - 1;
+  result_type __d = (__xmax - __xmin) / __n;
+  __densities_.reserve(__n);
+  for (size_t __i = 0; __i < __n; ++__i) {
+    __b_[__i] = __xmin + __i * __d;
+    __densities_.push_back(__fw(__b_[__i] + __d * .5));
+  }
+  __b_[__n] = __xmax;
+  __init();
+}
+
+template <class _RealType>
+template <class _URNG>
+_RealType piecewise_constant_distribution<_RealType>::operator()(_URNG& __g, const param_type& __p) {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  typedef uniform_real_distribution<result_type> _Gen;
+  result_type __u = _Gen()(__g);
+  ptrdiff_t __k   = std::upper_bound(__p.__areas_.begin(), __p.__areas_.end(), __u) - __p.__areas_.begin() - 1;
+  return (__u - __p.__areas_[__k]) / __p.__densities_[__k] + __p.__b_[__k];
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const piecewise_constant_distribution<_RT>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  size_t __n = __x.__p_.__b_.size();
+  __os << __n;
+  for (size_t __i = 0; __i < __n; ++__i)
+    __os << __sp << __x.__p_.__b_[__i];
+  __n = __x.__p_.__densities_.size();
+  __os << __sp << __n;
+  for (size_t __i = 0; __i < __n; ++__i)
+    __os << __sp << __x.__p_.__densities_[__i];
+  __n = __x.__p_.__areas_.size();
+  __os << __sp << __n;
+  for (size_t __i = 0; __i < __n; ++__i)
+    __os << __sp << __x.__p_.__areas_[__i];
+  return __os;
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, piecewise_constant_distribution<_RT>& __x) {
+  typedef piecewise_constant_distribution<_RT> _Eng;
+  typedef typename _Eng::result_type result_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  size_t __n;
+  __is >> __n;
+  vector<result_type> __b(__n);
+  for (size_t __i = 0; __i < __n; ++__i)
+    __is >> __b[__i];
+  __is >> __n;
+  vector<result_type> __densities(__n);
+  for (size_t __i = 0; __i < __n; ++__i)
+    __is >> __densities[__i];
+  __is >> __n;
+  vector<result_type> __areas(__n);
+  for (size_t __i = 0; __i < __n; ++__i)
+    __is >> __areas[__i];
+  if (!__is.fail()) {
+    swap(__x.__p_.__b_, __b);
+    swap(__x.__p_.__densities_, __densities);
+    swap(__x.__p_.__areas_, __areas);
+  }
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_PIECEWISE_CONSTANT_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/piecewise_linear_distribution.h b/libcxx/include/__cxx03/__random/piecewise_linear_distribution.h
new file mode 100644
index 000000000000..43769dc825e6
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/piecewise_linear_distribution.h
@@ -0,0 +1,315 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_PIECEWISE_LINEAR_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_PIECEWISE_LINEAR_DISTRIBUTION_H
+
+#include <__algorithm/upper_bound.h>
+#include <__config>
+#include <__random/is_valid.h>
+#include <__random/uniform_real_distribution.h>
+#include <cmath>
+#include <iosfwd>
+#include <vector>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _RealType = double>
+class _LIBCPP_TEMPLATE_VIS piecewise_linear_distribution {
+  static_assert(__libcpp_random_is_valid_realtype<_RealType>::value,
+                "RealType must be a supported floating-point type");
+
+public:
+  // types
+  typedef _RealType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    vector<result_type> __b_;
+    vector<result_type> __densities_;
+    vector<result_type> __areas_;
+
+  public:
+    typedef piecewise_linear_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI param_type();
+    template <class _InputIteratorB, class _InputIteratorW>
+    _LIBCPP_HIDE_FROM_ABI param_type(_InputIteratorB __f_b, _InputIteratorB __l_b, _InputIteratorW __f_w);
+#ifndef _LIBCPP_CXX03_LANG
+    template <class _UnaryOperation>
+    _LIBCPP_HIDE_FROM_ABI param_type(initializer_list<result_type> __bl, _UnaryOperation __fw);
+#endif // _LIBCPP_CXX03_LANG
+    template <class _UnaryOperation>
+    _LIBCPP_HIDE_FROM_ABI param_type(size_t __nw, result_type __xmin, result_type __xmax, _UnaryOperation __fw);
+    _LIBCPP_HIDE_FROM_ABI param_type(param_type const&) = default;
+    _LIBCPP_HIDE_FROM_ABI param_type& operator=(const param_type& __rhs);
+
+    _LIBCPP_HIDE_FROM_ABI vector<result_type> intervals() const { return __b_; }
+    _LIBCPP_HIDE_FROM_ABI vector<result_type> densities() const { return __densities_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__densities_ == __y.__densities_ && __x.__b_ == __y.__b_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+
+  private:
+    _LIBCPP_HIDE_FROM_ABI void __init();
+
+    friend class piecewise_linear_distribution;
+
+    template <class _CharT, class _Traits, class _RT>
+    friend basic_ostream<_CharT, _Traits>&
+    operator<<(basic_ostream<_CharT, _Traits>& __os, const piecewise_linear_distribution<_RT>& __x);
+
+    template <class _CharT, class _Traits, class _RT>
+    friend basic_istream<_CharT, _Traits>&
+    operator>>(basic_istream<_CharT, _Traits>& __is, piecewise_linear_distribution<_RT>& __x);
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructor and reset functions
+  _LIBCPP_HIDE_FROM_ABI piecewise_linear_distribution() {}
+  template <class _InputIteratorB, class _InputIteratorW>
+  _LIBCPP_HIDE_FROM_ABI
+  piecewise_linear_distribution(_InputIteratorB __f_b, _InputIteratorB __l_b, _InputIteratorW __f_w)
+      : __p_(__f_b, __l_b, __f_w) {}
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class _UnaryOperation>
+  _LIBCPP_HIDE_FROM_ABI piecewise_linear_distribution(initializer_list<result_type> __bl, _UnaryOperation __fw)
+      : __p_(__bl, __fw) {}
+#endif // _LIBCPP_CXX03_LANG
+
+  template <class _UnaryOperation>
+  _LIBCPP_HIDE_FROM_ABI
+  piecewise_linear_distribution(size_t __nw, result_type __xmin, result_type __xmax, _UnaryOperation __fw)
+      : __p_(__nw, __xmin, __xmax, __fw) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit piecewise_linear_distribution(const param_type& __p) : __p_(__p) {}
+
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI vector<result_type> intervals() const { return __p_.intervals(); }
+  _LIBCPP_HIDE_FROM_ABI vector<result_type> densities() const { return __p_.densities(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return __p_.__b_.front(); }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return __p_.__b_.back(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const piecewise_linear_distribution& __x, const piecewise_linear_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator!=(const piecewise_linear_distribution& __x, const piecewise_linear_distribution& __y) {
+    return !(__x == __y);
+  }
+
+  template <class _CharT, class _Traits, class _RT>
+  friend basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const piecewise_linear_distribution<_RT>& __x);
+
+  template <class _CharT, class _Traits, class _RT>
+  friend basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, piecewise_linear_distribution<_RT>& __x);
+};
+
+template <class _RealType>
+typename piecewise_linear_distribution<_RealType>::param_type&
+piecewise_linear_distribution<_RealType>::param_type::operator=(const param_type& __rhs) {
+  //  These can throw
+  __b_.reserve(__rhs.__b_.size());
+  __densities_.reserve(__rhs.__densities_.size());
+  __areas_.reserve(__rhs.__areas_.size());
+
+  //  These can not throw
+  __b_         = __rhs.__b_;
+  __densities_ = __rhs.__densities_;
+  __areas_     = __rhs.__areas_;
+  return *this;
+}
+
+template <class _RealType>
+void piecewise_linear_distribution<_RealType>::param_type::__init() {
+  __areas_.assign(__densities_.size() - 1, result_type());
+  result_type __sp = 0;
+  for (size_t __i = 0; __i < __areas_.size(); ++__i) {
+    __areas_[__i] = (__densities_[__i + 1] + __densities_[__i]) * (__b_[__i + 1] - __b_[__i]) * .5;
+    __sp += __areas_[__i];
+  }
+  for (size_t __i = __areas_.size(); __i > 1;) {
+    --__i;
+    __areas_[__i] = __areas_[__i - 1] / __sp;
+  }
+  __areas_[0] = 0;
+  for (size_t __i = 1; __i < __areas_.size(); ++__i)
+    __areas_[__i] += __areas_[__i - 1];
+  for (size_t __i = 0; __i < __densities_.size(); ++__i)
+    __densities_[__i] /= __sp;
+}
+
+template <class _RealType>
+piecewise_linear_distribution<_RealType>::param_type::param_type() : __b_(2), __densities_(2, 1.0), __areas_(1, 0.0) {
+  __b_[1] = 1;
+}
+
+template <class _RealType>
+template <class _InputIteratorB, class _InputIteratorW>
+piecewise_linear_distribution<_RealType>::param_type::param_type(
+    _InputIteratorB __f_b, _InputIteratorB __l_b, _InputIteratorW __f_w)
+    : __b_(__f_b, __l_b) {
+  if (__b_.size() < 2) {
+    __b_.resize(2);
+    __b_[0] = 0;
+    __b_[1] = 1;
+    __densities_.assign(2, 1.0);
+    __areas_.assign(1, 0.0);
+  } else {
+    __densities_.reserve(__b_.size());
+    for (size_t __i = 0; __i < __b_.size(); ++__i, ++__f_w)
+      __densities_.push_back(*__f_w);
+    __init();
+  }
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _RealType>
+template <class _UnaryOperation>
+piecewise_linear_distribution<_RealType>::param_type::param_type(
+    initializer_list<result_type> __bl, _UnaryOperation __fw)
+    : __b_(__bl.begin(), __bl.end()) {
+  if (__b_.size() < 2) {
+    __b_.resize(2);
+    __b_[0] = 0;
+    __b_[1] = 1;
+    __densities_.assign(2, 1.0);
+    __areas_.assign(1, 0.0);
+  } else {
+    __densities_.reserve(__b_.size());
+    for (size_t __i = 0; __i < __b_.size(); ++__i)
+      __densities_.push_back(__fw(__b_[__i]));
+    __init();
+  }
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _RealType>
+template <class _UnaryOperation>
+piecewise_linear_distribution<_RealType>::param_type::param_type(
+    size_t __nw, result_type __xmin, result_type __xmax, _UnaryOperation __fw)
+    : __b_(__nw == 0 ? 2 : __nw + 1) {
+  size_t __n      = __b_.size() - 1;
+  result_type __d = (__xmax - __xmin) / __n;
+  __densities_.reserve(__b_.size());
+  for (size_t __i = 0; __i < __n; ++__i) {
+    __b_[__i] = __xmin + __i * __d;
+    __densities_.push_back(__fw(__b_[__i]));
+  }
+  __b_[__n] = __xmax;
+  __densities_.push_back(__fw(__b_[__n]));
+  __init();
+}
+
+template <class _RealType>
+template <class _URNG>
+_RealType piecewise_linear_distribution<_RealType>::operator()(_URNG& __g, const param_type& __p) {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  typedef uniform_real_distribution<result_type> _Gen;
+  result_type __u = _Gen()(__g);
+  ptrdiff_t __k   = std::upper_bound(__p.__areas_.begin(), __p.__areas_.end(), __u) - __p.__areas_.begin() - 1;
+  __u -= __p.__areas_[__k];
+  const result_type __dk     = __p.__densities_[__k];
+  const result_type __dk1    = __p.__densities_[__k + 1];
+  const result_type __deltad = __dk1 - __dk;
+  const result_type __bk     = __p.__b_[__k];
+  if (__deltad == 0)
+    return __u / __dk + __bk;
+  const result_type __bk1    = __p.__b_[__k + 1];
+  const result_type __deltab = __bk1 - __bk;
+  return (__bk * __dk1 - __bk1 * __dk + std::sqrt(__deltab * (__deltab * __dk * __dk + 2 * __deltad * __u))) / __deltad;
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const piecewise_linear_distribution<_RT>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  size_t __n = __x.__p_.__b_.size();
+  __os << __n;
+  for (size_t __i = 0; __i < __n; ++__i)
+    __os << __sp << __x.__p_.__b_[__i];
+  __n = __x.__p_.__densities_.size();
+  __os << __sp << __n;
+  for (size_t __i = 0; __i < __n; ++__i)
+    __os << __sp << __x.__p_.__densities_[__i];
+  __n = __x.__p_.__areas_.size();
+  __os << __sp << __n;
+  for (size_t __i = 0; __i < __n; ++__i)
+    __os << __sp << __x.__p_.__areas_[__i];
+  return __os;
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, piecewise_linear_distribution<_RT>& __x) {
+  typedef piecewise_linear_distribution<_RT> _Eng;
+  typedef typename _Eng::result_type result_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  size_t __n;
+  __is >> __n;
+  vector<result_type> __b(__n);
+  for (size_t __i = 0; __i < __n; ++__i)
+    __is >> __b[__i];
+  __is >> __n;
+  vector<result_type> __densities(__n);
+  for (size_t __i = 0; __i < __n; ++__i)
+    __is >> __densities[__i];
+  __is >> __n;
+  vector<result_type> __areas(__n);
+  for (size_t __i = 0; __i < __n; ++__i)
+    __is >> __areas[__i];
+  if (!__is.fail()) {
+    swap(__x.__p_.__b_, __b);
+    swap(__x.__p_.__densities_, __densities);
+    swap(__x.__p_.__areas_, __areas);
+  }
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_PIECEWISE_LINEAR_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/poisson_distribution.h b/libcxx/include/__cxx03/__random/poisson_distribution.h
new file mode 100644
index 000000000000..61a092ef9dd4
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/poisson_distribution.h
@@ -0,0 +1,241 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_POISSON_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_POISSON_DISTRIBUTION_H
+
+#include <__config>
+#include <__random/clamp_to_integral.h>
+#include <__random/exponential_distribution.h>
+#include <__random/is_valid.h>
+#include <__random/normal_distribution.h>
+#include <__random/uniform_real_distribution.h>
+#include <cmath>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _IntType = int>
+class _LIBCPP_TEMPLATE_VIS poisson_distribution {
+  static_assert(__libcpp_random_is_valid_inttype<_IntType>::value, "IntType must be a supported integer type");
+
+public:
+  // types
+  typedef _IntType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    double __mean_;
+    double __s_;
+    double __d_;
+    double __l_;
+    double __omega_;
+    double __c0_;
+    double __c1_;
+    double __c2_;
+    double __c3_;
+    double __c_;
+
+  public:
+    typedef poisson_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(double __mean = 1.0);
+
+    _LIBCPP_HIDE_FROM_ABI double mean() const { return __mean_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__mean_ == __y.__mean_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+
+    friend class poisson_distribution;
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructors and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI poisson_distribution() : poisson_distribution(1.0) {}
+  _LIBCPP_HIDE_FROM_ABI explicit poisson_distribution(double __mean) : __p_(__mean) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit poisson_distribution(double __mean = 1.0) : __p_(__mean) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit poisson_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI double mean() const { return __p_.mean(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return 0; }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return numeric_limits<result_type>::max(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const poisson_distribution& __x, const poisson_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const poisson_distribution& __x, const poisson_distribution& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _IntType>
+poisson_distribution<_IntType>::param_type::param_type(double __mean)
+    // According to the standard `inf` is a valid input, but it causes the
+    // distribution to hang, so we replace it with the maximum representable
+    // mean.
+    : __mean_(isinf(__mean) ? numeric_limits<double>::max() : __mean) {
+  if (__mean_ < 10) {
+    __s_     = 0;
+    __d_     = 0;
+    __l_     = std::exp(-__mean_);
+    __omega_ = 0;
+    __c3_    = 0;
+    __c2_    = 0;
+    __c1_    = 0;
+    __c0_    = 0;
+    __c_     = 0;
+  } else {
+    __s_        = std::sqrt(__mean_);
+    __d_        = 6 * __mean_ * __mean_;
+    __l_        = std::trunc(__mean_ - 1.1484);
+    __omega_    = .3989423 / __s_;
+    double __b1 = .4166667E-1 / __mean_;
+    double __b2 = .3 * __b1 * __b1;
+    __c3_       = .1428571 * __b1 * __b2;
+    __c2_       = __b2 - 15. * __c3_;
+    __c1_       = __b1 - 6. * __b2 + 45. * __c3_;
+    __c0_       = 1. - __b1 + 3. * __b2 - 15. * __c3_;
+    __c_        = .1069 / __mean_;
+  }
+}
+
+template <class _IntType>
+template <class _URNG>
+_IntType poisson_distribution<_IntType>::operator()(_URNG& __urng, const param_type& __pr) {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  double __tx;
+  uniform_real_distribution<double> __urd;
+  if (__pr.__mean_ < 10) {
+    __tx = 0;
+    for (double __p = __urd(__urng); __p > __pr.__l_; ++__tx)
+      __p *= __urd(__urng);
+  } else {
+    double __difmuk;
+    double __g = __pr.__mean_ + __pr.__s_ * normal_distribution<double>()(__urng);
+    double __u;
+    if (__g > 0) {
+      __tx = std::trunc(__g);
+      if (__tx >= __pr.__l_)
+        return std::__clamp_to_integral<result_type>(__tx);
+      __difmuk = __pr.__mean_ - __tx;
+      __u      = __urd(__urng);
+      if (__pr.__d_ * __u >= __difmuk * __difmuk * __difmuk)
+        return std::__clamp_to_integral<result_type>(__tx);
+    }
+    exponential_distribution<double> __edist;
+    for (bool __using_exp_dist = false; true; __using_exp_dist = true) {
+      double __e;
+      if (__using_exp_dist || __g <= 0) {
+        double __t;
+        do {
+          __e = __edist(__urng);
+          __u = __urd(__urng);
+          __u += __u - 1;
+          __t = 1.8 + (__u < 0 ? -__e : __e);
+        } while (__t <= -.6744);
+        __tx             = std::trunc(__pr.__mean_ + __pr.__s_ * __t);
+        __difmuk         = __pr.__mean_ - __tx;
+        __using_exp_dist = true;
+      }
+      double __px;
+      double __py;
+      if (__tx < 10 && __tx >= 0) {
+        const double __fac[] = {1, 1, 2, 6, 24, 120, 720, 5040, 40320, 362880};
+        __px                 = -__pr.__mean_;
+        __py                 = std::pow(__pr.__mean_, (double)__tx) / __fac[static_cast<int>(__tx)];
+      } else {
+        double __del = .8333333E-1 / __tx;
+        __del -= 4.8 * __del * __del * __del;
+        double __v = __difmuk / __tx;
+        if (std::abs(__v) > 0.25)
+          __px = __tx * std::log(1 + __v) - __difmuk - __del;
+        else
+          __px = __tx * __v * __v *
+                     (((((((.1250060 * __v + -.1384794) * __v + .1421878) * __v + -.1661269) * __v + .2000118) * __v +
+                        -.2500068) *
+                           __v +
+                       .3333333) *
+                          __v +
+                      -.5) -
+                 __del;
+        __py = .3989423 / std::sqrt(__tx);
+      }
+      double __r  = (0.5 - __difmuk) / __pr.__s_;
+      double __r2 = __r * __r;
+      double __fx = -0.5 * __r2;
+      double __fy = __pr.__omega_ * (((__pr.__c3_ * __r2 + __pr.__c2_) * __r2 + __pr.__c1_) * __r2 + __pr.__c0_);
+      if (__using_exp_dist) {
+        if (__pr.__c_ * std::abs(__u) <= __py * std::exp(__px + __e) - __fy * std::exp(__fx + __e))
+          break;
+      } else {
+        if (__fy - __u * __fy <= __py * std::exp(__px - __fx))
+          break;
+      }
+    }
+  }
+  return std::__clamp_to_integral<result_type>(__tx);
+}
+
+template <class _CharT, class _Traits, class _IntType>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const poisson_distribution<_IntType>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  return __os << __x.mean();
+}
+
+template <class _CharT, class _Traits, class _IntType>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, poisson_distribution<_IntType>& __x) {
+  typedef poisson_distribution<_IntType> _Eng;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  double __mean;
+  __is >> __mean;
+  if (!__is.fail())
+    __x.param(param_type(__mean));
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_POISSON_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/random_device.h b/libcxx/include/__cxx03/__random/random_device.h
new file mode 100644
index 000000000000..52407943d2ec
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/random_device.h
@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_RANDOM_DEVICE_H
+#define _LIBCPP___RANDOM_RANDOM_DEVICE_H
+
+#include <__config>
+#include <string>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if !defined(_LIBCPP_HAS_NO_RANDOM_DEVICE)
+
+class _LIBCPP_EXPORTED_FROM_ABI random_device {
+#  ifdef _LIBCPP_USING_DEV_RANDOM
+  int __f_;
+#  elif !defined(_LIBCPP_ABI_NO_RANDOM_DEVICE_COMPATIBILITY_LAYOUT)
+  _LIBCPP_DIAGNOSTIC_PUSH
+  _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wunused-private-field")
+
+  // Apple platforms used to use the `_LIBCPP_USING_DEV_RANDOM` code path, and now
+  // use `arc4random()` as of this comment. In order to avoid breaking the ABI, we
+  // retain the same layout as before.
+#    if defined(__APPLE__)
+  int __padding_; // padding to fake the `__f_` field above
+#    endif
+
+  // ... vendors can add workarounds here if they switch to a different representation ...
+
+  _LIBCPP_DIAGNOSTIC_POP
+#  endif
+
+public:
+  // types
+  typedef unsigned result_type;
+
+  // generator characteristics
+  static _LIBCPP_CONSTEXPR const result_type _Min = 0;
+  static _LIBCPP_CONSTEXPR const result_type _Max = 0xFFFFFFFFu;
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR result_type min() { return _Min; }
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR result_type max() { return _Max; }
+
+  // constructors
+#  ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI random_device() : random_device("/dev/urandom") {}
+  explicit random_device(const string& __token);
+#  else
+  explicit random_device(const string& __token = "/dev/urandom");
+#  endif
+  ~random_device();
+
+  // generating functions
+  result_type operator()();
+
+  // property functions
+  double entropy() const _NOEXCEPT;
+
+  random_device(const random_device&)  = delete;
+  void operator=(const random_device&) = delete;
+};
+
+#endif // !_LIBCPP_HAS_NO_RANDOM_DEVICE
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_RANDOM_DEVICE_H
diff --git a/libcxx/include/__cxx03/__random/ranlux.h b/libcxx/include/__cxx03/__random/ranlux.h
new file mode 100644
index 000000000000..952afde91b10
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/ranlux.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_RANLUX_H
+#define _LIBCPP___RANDOM_RANLUX_H
+
+#include <__config>
+#include <__random/discard_block_engine.h>
+#include <__random/subtract_with_carry_engine.h>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+typedef subtract_with_carry_engine<uint_fast32_t, 24, 10, 24> ranlux24_base;
+typedef subtract_with_carry_engine<uint_fast64_t, 48, 5, 12> ranlux48_base;
+
+typedef discard_block_engine<ranlux24_base, 223, 23> ranlux24;
+typedef discard_block_engine<ranlux48_base, 389, 11> ranlux48;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANDOM_RANLUX_H
diff --git a/libcxx/include/__cxx03/__random/seed_seq.h b/libcxx/include/__cxx03/__random/seed_seq.h
new file mode 100644
index 000000000000..5cf84aeb8a72
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/seed_seq.h
@@ -0,0 +1,166 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_SEED_SEQ_H
+#define _LIBCPP___RANDOM_SEED_SEQ_H
+
+#include <__algorithm/copy.h>
+#include <__algorithm/fill.h>
+#include <__algorithm/max.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/is_unsigned.h>
+#include <cstdint>
+#include <initializer_list>
+#include <vector>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+class _LIBCPP_TEMPLATE_VIS seed_seq {
+public:
+  // types
+  typedef uint32_t result_type;
+
+  // constructors
+  _LIBCPP_HIDE_FROM_ABI seed_seq() _NOEXCEPT {}
+#ifndef _LIBCPP_CXX03_LANG
+  template <class _Tp, __enable_if_t<is_integral<_Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI seed_seq(initializer_list<_Tp> __il) {
+    __init(__il.begin(), __il.end());
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI seed_seq(_InputIterator __first, _InputIterator __last) {
+    static_assert(is_integral<typename iterator_traits<_InputIterator>::value_type>::value,
+                  "Mandates: iterator_traits<InputIterator>::value_type is an integer type");
+    __init(__first, __last);
+  }
+
+  // generating functions
+  template <class _RandomAccessIterator>
+  _LIBCPP_HIDE_FROM_ABI void generate(_RandomAccessIterator __first, _RandomAccessIterator __last);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI size_t size() const _NOEXCEPT { return __v_.size(); }
+  template <class _OutputIterator>
+  _LIBCPP_HIDE_FROM_ABI void param(_OutputIterator __dest) const {
+    std::copy(__v_.begin(), __v_.end(), __dest);
+  }
+
+  seed_seq(const seed_seq&)       = delete;
+  void operator=(const seed_seq&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI static result_type _Tp(result_type __x) { return __x ^ (__x >> 27); }
+
+private:
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void __init(_InputIterator __first, _InputIterator __last);
+
+  vector<result_type> __v_;
+};
+
+template <class _InputIterator>
+void seed_seq::__init(_InputIterator __first, _InputIterator __last) {
+  for (_InputIterator __s = __first; __s != __last; ++__s)
+    __v_.push_back(*__s & 0xFFFFFFFF);
+}
+
+template <class _RandomAccessIterator>
+void seed_seq::generate(_RandomAccessIterator __first, _RandomAccessIterator __last) {
+  using _ValueType = typename iterator_traits<_RandomAccessIterator>::value_type;
+  static_assert(is_unsigned<_ValueType>::value && sizeof(_ValueType) >= sizeof(uint32_t),
+                "[rand.util.seedseq]/7 requires the value_type of the iterator to be an unsigned "
+                "integer capable of accommodating 32-bit quantities.");
+
+  if (__first != __last) {
+    std::fill(__first, __last, 0x8b8b8b8b);
+    const size_t __n = static_cast<size_t>(__last - __first);
+    const size_t __s = __v_.size();
+    const size_t __t = (__n >= 623) ? 11 : (__n >= 68) ? 7 : (__n >= 39) ? 5 : (__n >= 7) ? 3 : (__n - 1) / 2;
+    const size_t __p = (__n - __t) / 2;
+    const size_t __q = __p + __t;
+    const size_t __m = std::max(__s + 1, __n);
+    // __k = 0;
+    {
+      result_type __r = 1664525 * _Tp(__first[0] ^ __first[__p] ^ __first[__n - 1]);
+      __first[__p] += __r;
+      __r += __s;
+      __first[__q] += __r;
+      __first[0] = __r;
+    }
+    // Initialize indexing terms used with if statements as an optimization to
+    // avoid calculating modulo n on every loop iteration for each term.
+    size_t __kmodn  = 0;         // __k % __n
+    size_t __k1modn = __n - 1;   // (__k - 1) % __n
+    size_t __kpmodn = __p % __n; // (__k + __p) % __n
+    size_t __kqmodn = __q % __n; // (__k + __q) % __n
+
+    for (size_t __k = 1; __k <= __s; ++__k) {
+      if (++__kmodn == __n)
+        __kmodn = 0;
+      if (++__k1modn == __n)
+        __k1modn = 0;
+      if (++__kpmodn == __n)
+        __kpmodn = 0;
+      if (++__kqmodn == __n)
+        __kqmodn = 0;
+
+      result_type __r = 1664525 * _Tp(__first[__kmodn] ^ __first[__kpmodn] ^ __first[__k1modn]);
+      __first[__kpmodn] += __r;
+      __r += __kmodn + __v_[__k - 1];
+      __first[__kqmodn] += __r;
+      __first[__kmodn] = __r;
+    }
+    for (size_t __k = __s + 1; __k < __m; ++__k) {
+      if (++__kmodn == __n)
+        __kmodn = 0;
+      if (++__k1modn == __n)
+        __k1modn = 0;
+      if (++__kpmodn == __n)
+        __kpmodn = 0;
+      if (++__kqmodn == __n)
+        __kqmodn = 0;
+
+      result_type __r = 1664525 * _Tp(__first[__kmodn] ^ __first[__kpmodn] ^ __first[__k1modn]);
+      __first[__kpmodn] += __r;
+      __r += __kmodn;
+      __first[__kqmodn] += __r;
+      __first[__kmodn] = __r;
+    }
+    for (size_t __k = __m; __k < __m + __n; ++__k) {
+      if (++__kmodn == __n)
+        __kmodn = 0;
+      if (++__k1modn == __n)
+        __k1modn = 0;
+      if (++__kpmodn == __n)
+        __kpmodn = 0;
+      if (++__kqmodn == __n)
+        __kqmodn = 0;
+
+      result_type __r = 1566083941 * _Tp(__first[__kmodn] + __first[__kpmodn] + __first[__k1modn]);
+      __first[__kpmodn] ^= __r;
+      __r -= __kmodn;
+      __first[__kqmodn] ^= __r;
+      __first[__kmodn] = __r;
+    }
+  }
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_SEED_SEQ_H
diff --git a/libcxx/include/__cxx03/__random/shuffle_order_engine.h b/libcxx/include/__cxx03/__random/shuffle_order_engine.h
new file mode 100644
index 000000000000..f54ed17e3838
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/shuffle_order_engine.h
@@ -0,0 +1,230 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_SHUFFLE_ORDER_ENGINE_H
+#define _LIBCPP___RANDOM_SHUFFLE_ORDER_ENGINE_H
+
+#include <__algorithm/equal.h>
+#include <__config>
+#include <__random/is_seed_sequence.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_convertible.h>
+#include <__utility/move.h>
+#include <cstddef>
+#include <cstdint>
+#include <iosfwd>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <uint64_t _Xp, uint64_t _Yp>
+struct __ugcd {
+  static _LIBCPP_CONSTEXPR const uint64_t value = __ugcd<_Yp, _Xp % _Yp>::value;
+};
+
+template <uint64_t _Xp>
+struct __ugcd<_Xp, 0> {
+  static _LIBCPP_CONSTEXPR const uint64_t value = _Xp;
+};
+
+template <uint64_t _Np, uint64_t _Dp>
+class __uratio {
+  static_assert(_Dp != 0, "__uratio divide by 0");
+  static _LIBCPP_CONSTEXPR const uint64_t __gcd = __ugcd<_Np, _Dp>::value;
+
+public:
+  static _LIBCPP_CONSTEXPR const uint64_t num = _Np / __gcd;
+  static _LIBCPP_CONSTEXPR const uint64_t den = _Dp / __gcd;
+
+  typedef __uratio<num, den> type;
+};
+
+template <class _Engine, size_t __k>
+class _LIBCPP_TEMPLATE_VIS shuffle_order_engine {
+  static_assert(0 < __k, "shuffle_order_engine invalid parameters");
+
+public:
+  // types
+  typedef typename _Engine::result_type result_type;
+
+private:
+  _Engine __e_;
+  result_type __v_[__k];
+  result_type __y_;
+
+public:
+  // engine characteristics
+  static _LIBCPP_CONSTEXPR const size_t table_size = __k;
+
+#ifdef _LIBCPP_CXX03_LANG
+  static const result_type _Min = _Engine::_Min;
+  static const result_type _Max = _Engine::_Max;
+#else
+  static _LIBCPP_CONSTEXPR const result_type _Min = _Engine::min();
+  static _LIBCPP_CONSTEXPR const result_type _Max = _Engine::max();
+#endif
+  static_assert(_Min < _Max, "shuffle_order_engine invalid parameters");
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR result_type min() { return _Min; }
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR result_type max() { return _Max; }
+
+  static _LIBCPP_CONSTEXPR const unsigned long long _Rp = _Max - _Min + 1ull;
+
+  // constructors and seeding functions
+  _LIBCPP_HIDE_FROM_ABI shuffle_order_engine() { __init(); }
+  _LIBCPP_HIDE_FROM_ABI explicit shuffle_order_engine(const _Engine& __e) : __e_(__e) { __init(); }
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI explicit shuffle_order_engine(_Engine&& __e) : __e_(std::move(__e)) { __init(); }
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI explicit shuffle_order_engine(result_type __sd) : __e_(__sd) { __init(); }
+  template <
+      class _Sseq,
+      __enable_if_t<__is_seed_sequence<_Sseq, shuffle_order_engine>::value && !is_convertible<_Sseq, _Engine>::value,
+                    int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit shuffle_order_engine(_Sseq& __q) : __e_(__q) {
+    __init();
+  }
+  _LIBCPP_HIDE_FROM_ABI void seed() {
+    __e_.seed();
+    __init();
+  }
+  _LIBCPP_HIDE_FROM_ABI void seed(result_type __sd) {
+    __e_.seed(__sd);
+    __init();
+  }
+  template <class _Sseq, __enable_if_t<__is_seed_sequence<_Sseq, shuffle_order_engine>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void seed(_Sseq& __q) {
+    __e_.seed(__q);
+    __init();
+  }
+
+  // generating functions
+  _LIBCPP_HIDE_FROM_ABI result_type operator()() { return __eval(integral_constant<bool, _Rp != 0>()); }
+  _LIBCPP_HIDE_FROM_ABI void discard(unsigned long long __z) {
+    for (; __z; --__z)
+      operator()();
+  }
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI const _Engine& base() const _NOEXCEPT { return __e_; }
+
+private:
+  template <class _Eng, size_t _Kp>
+  friend bool operator==(const shuffle_order_engine<_Eng, _Kp>& __x, const shuffle_order_engine<_Eng, _Kp>& __y);
+
+  template <class _Eng, size_t _Kp>
+  friend bool operator!=(const shuffle_order_engine<_Eng, _Kp>& __x, const shuffle_order_engine<_Eng, _Kp>& __y);
+
+  template <class _CharT, class _Traits, class _Eng, size_t _Kp>
+  friend basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const shuffle_order_engine<_Eng, _Kp>& __x);
+
+  template <class _CharT, class _Traits, class _Eng, size_t _Kp>
+  friend basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, shuffle_order_engine<_Eng, _Kp>& __x);
+
+  _LIBCPP_HIDE_FROM_ABI void __init() {
+    for (size_t __i = 0; __i < __k; ++__i)
+      __v_[__i] = __e_();
+    __y_ = __e_();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI result_type __eval(false_type) { return __eval2(integral_constant<bool, __k & 1>()); }
+  _LIBCPP_HIDE_FROM_ABI result_type __eval(true_type) { return __eval(__uratio<__k, _Rp>()); }
+
+  _LIBCPP_HIDE_FROM_ABI result_type __eval2(false_type) { return __eval(__uratio<__k / 2, 0x8000000000000000ull>()); }
+  _LIBCPP_HIDE_FROM_ABI result_type __eval2(true_type) { return __evalf<__k, 0>(); }
+
+  template <uint64_t _Np,
+            uint64_t _Dp,
+            __enable_if_t<(__uratio<_Np, _Dp>::num > 0xFFFFFFFFFFFFFFFFull / (_Max - _Min)), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI result_type __eval(__uratio<_Np, _Dp>) {
+    return __evalf<__uratio<_Np, _Dp>::num, __uratio<_Np, _Dp>::den>();
+  }
+
+  template <uint64_t _Np,
+            uint64_t _Dp,
+            __enable_if_t<__uratio<_Np, _Dp>::num <= 0xFFFFFFFFFFFFFFFFull / (_Max - _Min), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI result_type __eval(__uratio<_Np, _Dp>) {
+    const size_t __j = static_cast<size_t>(__uratio<_Np, _Dp>::num * (__y_ - _Min) / __uratio<_Np, _Dp>::den);
+    __y_             = __v_[__j];
+    __v_[__j]        = __e_();
+    return __y_;
+  }
+
+  template <uint64_t __n, uint64_t __d>
+  _LIBCPP_HIDE_FROM_ABI result_type __evalf() {
+    const double __fp = __d == 0 ? __n / (2. * 0x8000000000000000ull) : __n / (double)__d;
+    const size_t __j  = static_cast<size_t>(__fp * (__y_ - _Min));
+    __y_              = __v_[__j];
+    __v_[__j]         = __e_();
+    return __y_;
+  }
+};
+
+template <class _Engine, size_t __k>
+_LIBCPP_CONSTEXPR const size_t shuffle_order_engine<_Engine, __k>::table_size;
+
+template <class _Eng, size_t _Kp>
+_LIBCPP_HIDE_FROM_ABI bool
+operator==(const shuffle_order_engine<_Eng, _Kp>& __x, const shuffle_order_engine<_Eng, _Kp>& __y) {
+  return __x.__y_ == __y.__y_ && std::equal(__x.__v_, __x.__v_ + _Kp, __y.__v_) && __x.__e_ == __y.__e_;
+}
+
+template <class _Eng, size_t _Kp>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const shuffle_order_engine<_Eng, _Kp>& __x, const shuffle_order_engine<_Eng, _Kp>& __y) {
+  return !(__x == __y);
+}
+
+template <class _CharT, class _Traits, class _Eng, size_t _Kp>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const shuffle_order_engine<_Eng, _Kp>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _Ostream;
+  __os.flags(_Ostream::dec | _Ostream::left);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  __os << __x.__e_ << __sp << __x.__v_[0];
+  for (size_t __i = 1; __i < _Kp; ++__i)
+    __os << __sp << __x.__v_[__i];
+  return __os << __sp << __x.__y_;
+}
+
+template <class _CharT, class _Traits, class _Eng, size_t _Kp>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, shuffle_order_engine<_Eng, _Kp>& __x) {
+  typedef typename shuffle_order_engine<_Eng, _Kp>::result_type result_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  _Eng __e;
+  result_type __vp[_Kp + 1];
+  __is >> __e;
+  for (size_t __i = 0; __i < _Kp + 1; ++__i)
+    __is >> __vp[__i];
+  if (!__is.fail()) {
+    __x.__e_ = __e;
+    for (size_t __i = 0; __i < _Kp; ++__i)
+      __x.__v_[__i] = __vp[__i];
+    __x.__y_ = __vp[_Kp];
+  }
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_SHUFFLE_ORDER_ENGINE_H
diff --git a/libcxx/include/__cxx03/__random/student_t_distribution.h b/libcxx/include/__cxx03/__random/student_t_distribution.h
new file mode 100644
index 000000000000..110a856ee658
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/student_t_distribution.h
@@ -0,0 +1,132 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_STUDENT_T_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_STUDENT_T_DISTRIBUTION_H
+
+#include <__config>
+#include <__random/gamma_distribution.h>
+#include <__random/is_valid.h>
+#include <__random/normal_distribution.h>
+#include <cmath>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _RealType = double>
+class _LIBCPP_TEMPLATE_VIS student_t_distribution {
+  static_assert(__libcpp_random_is_valid_realtype<_RealType>::value,
+                "RealType must be a supported floating-point type");
+
+public:
+  // types
+  typedef _RealType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    result_type __n_;
+
+  public:
+    typedef student_t_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(result_type __n = 1) : __n_(__n) {}
+
+    _LIBCPP_HIDE_FROM_ABI result_type n() const { return __n_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__n_ == __y.__n_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+  };
+
+private:
+  param_type __p_;
+  normal_distribution<result_type> __nd_;
+
+public:
+  // constructor and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI student_t_distribution() : student_t_distribution(1) {}
+  _LIBCPP_HIDE_FROM_ABI explicit student_t_distribution(result_type __n) : __p_(param_type(__n)) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit student_t_distribution(result_type __n = 1) : __p_(param_type(__n)) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit student_t_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() { __nd_.reset(); }
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI result_type n() const { return __p_.n(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return -numeric_limits<result_type>::infinity(); }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return numeric_limits<result_type>::infinity(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const student_t_distribution& __x, const student_t_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const student_t_distribution& __x, const student_t_distribution& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _RealType>
+template <class _URNG>
+_RealType student_t_distribution<_RealType>::operator()(_URNG& __g, const param_type& __p) {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  gamma_distribution<result_type> __gd(__p.n() * .5, 2);
+  return __nd_(__g) * std::sqrt(__p.n() / __gd(__g));
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const student_t_distribution<_RT>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  __os << __x.n();
+  return __os;
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, student_t_distribution<_RT>& __x) {
+  typedef student_t_distribution<_RT> _Eng;
+  typedef typename _Eng::result_type result_type;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  result_type __n;
+  __is >> __n;
+  if (!__is.fail())
+    __x.param(param_type(__n));
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_STUDENT_T_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/subtract_with_carry_engine.h b/libcxx/include/__cxx03/__random/subtract_with_carry_engine.h
new file mode 100644
index 000000000000..ec25fed49f94
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/subtract_with_carry_engine.h
@@ -0,0 +1,276 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_SUBTRACT_WITH_CARRY_ENGINE_H
+#define _LIBCPP___RANDOM_SUBTRACT_WITH_CARRY_ENGINE_H
+
+#include <__algorithm/equal.h>
+#include <__algorithm/min.h>
+#include <__config>
+#include <__random/is_seed_sequence.h>
+#include <__random/linear_congruential_engine.h>
+#include <cstddef>
+#include <cstdint>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _UIntType, size_t __w, size_t __s, size_t __r>
+class _LIBCPP_TEMPLATE_VIS subtract_with_carry_engine;
+
+template <class _UInt, size_t _Wp, size_t _Sp, size_t _Rp>
+_LIBCPP_HIDE_FROM_ABI bool operator==(const subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __x,
+                                      const subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __y);
+
+template <class _UInt, size_t _Wp, size_t _Sp, size_t _Rp>
+_LIBCPP_HIDE_FROM_ABI bool operator!=(const subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __x,
+                                      const subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __y);
+
+template <class _CharT, class _Traits, class _UInt, size_t _Wp, size_t _Sp, size_t _Rp>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __x);
+
+template <class _CharT, class _Traits, class _UInt, size_t _Wp, size_t _Sp, size_t _Rp>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __x);
+
+template <class _UIntType, size_t __w, size_t __s, size_t __r>
+class _LIBCPP_TEMPLATE_VIS subtract_with_carry_engine {
+public:
+  // types
+  typedef _UIntType result_type;
+
+private:
+  result_type __x_[__r];
+  result_type __c_;
+  size_t __i_;
+
+  static _LIBCPP_CONSTEXPR const result_type _Dt = numeric_limits<result_type>::digits;
+  static_assert(0 < __w, "subtract_with_carry_engine invalid parameters");
+  static_assert(__w <= _Dt, "subtract_with_carry_engine invalid parameters");
+  static_assert(0 < __s, "subtract_with_carry_engine invalid parameters");
+  static_assert(__s < __r, "subtract_with_carry_engine invalid parameters");
+
+public:
+  static _LIBCPP_CONSTEXPR const result_type _Min = 0;
+  static _LIBCPP_CONSTEXPR const result_type _Max =
+      __w == _Dt ? result_type(~0) : (result_type(1) << __w) - result_type(1);
+  static_assert(_Min < _Max, "subtract_with_carry_engine invalid parameters");
+
+  // engine characteristics
+  static _LIBCPP_CONSTEXPR const size_t word_size = __w;
+  static _LIBCPP_CONSTEXPR const size_t short_lag = __s;
+  static _LIBCPP_CONSTEXPR const size_t long_lag  = __r;
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR result_type min() { return _Min; }
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR result_type max() { return _Max; }
+  static _LIBCPP_CONSTEXPR const result_type default_seed = 19780503u;
+
+  // constructors and seeding functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI subtract_with_carry_engine() : subtract_with_carry_engine(default_seed) {}
+  _LIBCPP_HIDE_FROM_ABI explicit subtract_with_carry_engine(result_type __sd) { seed(__sd); }
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit subtract_with_carry_engine(result_type __sd = default_seed) { seed(__sd); }
+#endif
+  template <class _Sseq, __enable_if_t<__is_seed_sequence<_Sseq, subtract_with_carry_engine>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit subtract_with_carry_engine(_Sseq& __q) {
+    seed(__q);
+  }
+  _LIBCPP_HIDE_FROM_ABI void seed(result_type __sd = default_seed) {
+    seed(__sd, integral_constant<unsigned, 1 + (__w - 1) / 32>());
+  }
+  template <class _Sseq, __enable_if_t<__is_seed_sequence<_Sseq, subtract_with_carry_engine>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void seed(_Sseq& __q) {
+    __seed(__q, integral_constant<unsigned, 1 + (__w - 1) / 32>());
+  }
+
+  // generating functions
+  _LIBCPP_HIDE_FROM_ABI result_type operator()();
+  _LIBCPP_HIDE_FROM_ABI void discard(unsigned long long __z) {
+    for (; __z; --__z)
+      operator()();
+  }
+
+  template <class _UInt, size_t _Wp, size_t _Sp, size_t _Rp>
+  friend bool operator==(const subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __x,
+                         const subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __y);
+
+  template <class _UInt, size_t _Wp, size_t _Sp, size_t _Rp>
+  friend bool operator!=(const subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __x,
+                         const subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __y);
+
+  template <class _CharT, class _Traits, class _UInt, size_t _Wp, size_t _Sp, size_t _Rp>
+  friend basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __x);
+
+  template <class _CharT, class _Traits, class _UInt, size_t _Wp, size_t _Sp, size_t _Rp>
+  friend basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __x);
+
+private:
+  _LIBCPP_HIDE_FROM_ABI void seed(result_type __sd, integral_constant<unsigned, 1>);
+  _LIBCPP_HIDE_FROM_ABI void seed(result_type __sd, integral_constant<unsigned, 2>);
+  template <class _Sseq>
+  _LIBCPP_HIDE_FROM_ABI void __seed(_Sseq& __q, integral_constant<unsigned, 1>);
+  template <class _Sseq>
+  _LIBCPP_HIDE_FROM_ABI void __seed(_Sseq& __q, integral_constant<unsigned, 2>);
+};
+
+template <class _UIntType, size_t __w, size_t __s, size_t __r>
+_LIBCPP_CONSTEXPR const size_t subtract_with_carry_engine<_UIntType, __w, __s, __r>::word_size;
+
+template <class _UIntType, size_t __w, size_t __s, size_t __r>
+_LIBCPP_CONSTEXPR const size_t subtract_with_carry_engine<_UIntType, __w, __s, __r>::short_lag;
+
+template <class _UIntType, size_t __w, size_t __s, size_t __r>
+_LIBCPP_CONSTEXPR const size_t subtract_with_carry_engine<_UIntType, __w, __s, __r>::long_lag;
+
+template <class _UIntType, size_t __w, size_t __s, size_t __r>
+_LIBCPP_CONSTEXPR const typename subtract_with_carry_engine<_UIntType, __w, __s, __r>::result_type
+    subtract_with_carry_engine<_UIntType, __w, __s, __r>::default_seed;
+
+template <class _UIntType, size_t __w, size_t __s, size_t __r>
+void subtract_with_carry_engine<_UIntType, __w, __s, __r>::seed(result_type __sd, integral_constant<unsigned, 1>) {
+  linear_congruential_engine<result_type, 40014u, 0u, 2147483563u> __e(__sd == 0u ? default_seed : __sd);
+  for (size_t __i = 0; __i < __r; ++__i)
+    __x_[__i] = static_cast<result_type>(__e() & _Max);
+  __c_ = __x_[__r - 1] == 0;
+  __i_ = 0;
+}
+
+template <class _UIntType, size_t __w, size_t __s, size_t __r>
+void subtract_with_carry_engine<_UIntType, __w, __s, __r>::seed(result_type __sd, integral_constant<unsigned, 2>) {
+  linear_congruential_engine<result_type, 40014u, 0u, 2147483563u> __e(__sd == 0u ? default_seed : __sd);
+  for (size_t __i = 0; __i < __r; ++__i) {
+    result_type __e0 = __e();
+    __x_[__i]        = static_cast<result_type>((__e0 + ((uint64_t)__e() << 32)) & _Max);
+  }
+  __c_ = __x_[__r - 1] == 0;
+  __i_ = 0;
+}
+
+template <class _UIntType, size_t __w, size_t __s, size_t __r>
+template <class _Sseq>
+void subtract_with_carry_engine<_UIntType, __w, __s, __r>::__seed(_Sseq& __q, integral_constant<unsigned, 1>) {
+  const unsigned __k = 1;
+  uint32_t __ar[__r * __k];
+  __q.generate(__ar, __ar + __r * __k);
+  for (size_t __i = 0; __i < __r; ++__i)
+    __x_[__i] = static_cast<result_type>(__ar[__i] & _Max);
+  __c_ = __x_[__r - 1] == 0;
+  __i_ = 0;
+}
+
+template <class _UIntType, size_t __w, size_t __s, size_t __r>
+template <class _Sseq>
+void subtract_with_carry_engine<_UIntType, __w, __s, __r>::__seed(_Sseq& __q, integral_constant<unsigned, 2>) {
+  const unsigned __k = 2;
+  uint32_t __ar[__r * __k];
+  __q.generate(__ar, __ar + __r * __k);
+  for (size_t __i = 0; __i < __r; ++__i)
+    __x_[__i] = static_cast<result_type>((__ar[2 * __i] + ((uint64_t)__ar[2 * __i + 1] << 32)) & _Max);
+  __c_ = __x_[__r - 1] == 0;
+  __i_ = 0;
+}
+
+template <class _UIntType, size_t __w, size_t __s, size_t __r>
+_UIntType subtract_with_carry_engine<_UIntType, __w, __s, __r>::operator()() {
+  const result_type& __xs = __x_[(__i_ + (__r - __s)) % __r];
+  result_type& __xr       = __x_[__i_];
+  result_type __new_c     = __c_ == 0 ? __xs < __xr : __xs != 0 ? __xs <= __xr : 1;
+  __xr                    = (__xs - __xr - __c_) & _Max;
+  __c_                    = __new_c;
+  __i_                    = (__i_ + 1) % __r;
+  return __xr;
+}
+
+template <class _UInt, size_t _Wp, size_t _Sp, size_t _Rp>
+_LIBCPP_HIDE_FROM_ABI bool operator==(const subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __x,
+                                      const subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __y) {
+  if (__x.__c_ != __y.__c_)
+    return false;
+  if (__x.__i_ == __y.__i_)
+    return std::equal(__x.__x_, __x.__x_ + _Rp, __y.__x_);
+  if (__x.__i_ == 0 || __y.__i_ == 0) {
+    size_t __j = std::min(_Rp - __x.__i_, _Rp - __y.__i_);
+    if (!std::equal(__x.__x_ + __x.__i_, __x.__x_ + __x.__i_ + __j, __y.__x_ + __y.__i_))
+      return false;
+    if (__x.__i_ == 0)
+      return std::equal(__x.__x_ + __j, __x.__x_ + _Rp, __y.__x_);
+    return std::equal(__x.__x_, __x.__x_ + (_Rp - __j), __y.__x_ + __j);
+  }
+  if (__x.__i_ < __y.__i_) {
+    size_t __j = _Rp - __y.__i_;
+    if (!std::equal(__x.__x_ + __x.__i_, __x.__x_ + (__x.__i_ + __j), __y.__x_ + __y.__i_))
+      return false;
+    if (!std::equal(__x.__x_ + (__x.__i_ + __j), __x.__x_ + _Rp, __y.__x_))
+      return false;
+    return std::equal(__x.__x_, __x.__x_ + __x.__i_, __y.__x_ + (_Rp - (__x.__i_ + __j)));
+  }
+  size_t __j = _Rp - __x.__i_;
+  if (!std::equal(__y.__x_ + __y.__i_, __y.__x_ + (__y.__i_ + __j), __x.__x_ + __x.__i_))
+    return false;
+  if (!std::equal(__y.__x_ + (__y.__i_ + __j), __y.__x_ + _Rp, __x.__x_))
+    return false;
+  return std::equal(__y.__x_, __y.__x_ + __y.__i_, __x.__x_ + (_Rp - (__y.__i_ + __j)));
+}
+
+template <class _UInt, size_t _Wp, size_t _Sp, size_t _Rp>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __x,
+                                             const subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __y) {
+  return !(__x == __y);
+}
+
+template <class _CharT, class _Traits, class _UInt, size_t _Wp, size_t _Sp, size_t _Rp>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _Ostream;
+  __os.flags(_Ostream::dec | _Ostream::left);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  __os << __x.__x_[__x.__i_];
+  for (size_t __j = __x.__i_ + 1; __j < _Rp; ++__j)
+    __os << __sp << __x.__x_[__j];
+  for (size_t __j = 0; __j < __x.__i_; ++__j)
+    __os << __sp << __x.__x_[__j];
+  __os << __sp << __x.__c_;
+  return __os;
+}
+
+template <class _CharT, class _Traits, class _UInt, size_t _Wp, size_t _Sp, size_t _Rp>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, subtract_with_carry_engine<_UInt, _Wp, _Sp, _Rp>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  _UInt __t[_Rp + 1];
+  for (size_t __i = 0; __i < _Rp + 1; ++__i)
+    __is >> __t[__i];
+  if (!__is.fail()) {
+    for (size_t __i = 0; __i < _Rp; ++__i)
+      __x.__x_[__i] = __t[__i];
+    __x.__c_ = __t[_Rp];
+    __x.__i_ = 0;
+  }
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_SUBTRACT_WITH_CARRY_ENGINE_H
diff --git a/libcxx/include/__cxx03/__random/uniform_int_distribution.h b/libcxx/include/__cxx03/__random/uniform_int_distribution.h
new file mode 100644
index 000000000000..4e3ca3efe568
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/uniform_int_distribution.h
@@ -0,0 +1,264 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_UNIFORM_INT_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_UNIFORM_INT_DISTRIBUTION_H
+
+#include <__bit/countl.h>
+#include <__config>
+#include <__random/is_valid.h>
+#include <__random/log2.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/make_unsigned.h>
+#include <cstddef>
+#include <cstdint>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Engine, class _UIntType>
+class __independent_bits_engine {
+public:
+  // types
+  typedef _UIntType result_type;
+
+private:
+  typedef typename _Engine::result_type _Engine_result_type;
+  typedef __conditional_t<sizeof(_Engine_result_type) <= sizeof(result_type), result_type, _Engine_result_type>
+      _Working_result_type;
+
+  _Engine& __e_;
+  size_t __w_;
+  size_t __w0_;
+  size_t __n_;
+  size_t __n0_;
+  _Working_result_type __y0_;
+  _Working_result_type __y1_;
+  _Engine_result_type __mask0_;
+  _Engine_result_type __mask1_;
+
+#ifdef _LIBCPP_CXX03_LANG
+  static const _Working_result_type _Rp = _Engine::_Max - _Engine::_Min + _Working_result_type(1);
+#else
+  static _LIBCPP_CONSTEXPR const _Working_result_type _Rp = _Engine::max() - _Engine::min() + _Working_result_type(1);
+#endif
+  static _LIBCPP_CONSTEXPR const size_t __m  = __log2<_Working_result_type, _Rp>::value;
+  static _LIBCPP_CONSTEXPR const size_t _WDt = numeric_limits<_Working_result_type>::digits;
+  static _LIBCPP_CONSTEXPR const size_t _EDt = numeric_limits<_Engine_result_type>::digits;
+
+public:
+  // constructors and seeding functions
+  _LIBCPP_HIDE_FROM_ABI __independent_bits_engine(_Engine& __e, size_t __w);
+
+  // generating functions
+  _LIBCPP_HIDE_FROM_ABI result_type operator()() { return __eval(integral_constant<bool, _Rp != 0>()); }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI result_type __eval(false_type);
+  _LIBCPP_HIDE_FROM_ABI result_type __eval(true_type);
+};
+
+template <class _Engine, class _UIntType>
+__independent_bits_engine<_Engine, _UIntType>::__independent_bits_engine(_Engine& __e, size_t __w)
+    : __e_(__e), __w_(__w) {
+  __n_  = __w_ / __m + (__w_ % __m != 0);
+  __w0_ = __w_ / __n_;
+  if (_Rp == 0)
+    __y0_ = _Rp;
+  else if (__w0_ < _WDt)
+    __y0_ = (_Rp >> __w0_) << __w0_;
+  else
+    __y0_ = 0;
+  if (_Rp - __y0_ > __y0_ / __n_) {
+    ++__n_;
+    __w0_ = __w_ / __n_;
+    if (__w0_ < _WDt)
+      __y0_ = (_Rp >> __w0_) << __w0_;
+    else
+      __y0_ = 0;
+  }
+  __n0_ = __n_ - __w_ % __n_;
+  if (__w0_ < _WDt - 1)
+    __y1_ = (_Rp >> (__w0_ + 1)) << (__w0_ + 1);
+  else
+    __y1_ = 0;
+  __mask0_ = __w0_ > 0 ? _Engine_result_type(~0) >> (_EDt - __w0_) : _Engine_result_type(0);
+  __mask1_ = __w0_ < _EDt - 1 ? _Engine_result_type(~0) >> (_EDt - (__w0_ + 1)) : _Engine_result_type(~0);
+}
+
+template <class _Engine, class _UIntType>
+inline _UIntType __independent_bits_engine<_Engine, _UIntType>::__eval(false_type) {
+  return static_cast<result_type>(__e_() & __mask0_);
+}
+
+template <class _Engine, class _UIntType>
+_UIntType __independent_bits_engine<_Engine, _UIntType>::__eval(true_type) {
+  const size_t __w_rt = numeric_limits<result_type>::digits;
+  result_type __sp    = 0;
+  for (size_t __k = 0; __k < __n0_; ++__k) {
+    _Engine_result_type __u;
+    do {
+      __u = __e_() - _Engine::min();
+    } while (__u >= __y0_);
+    if (__w0_ < __w_rt)
+      __sp <<= __w0_;
+    else
+      __sp = 0;
+    __sp += __u & __mask0_;
+  }
+  for (size_t __k = __n0_; __k < __n_; ++__k) {
+    _Engine_result_type __u;
+    do {
+      __u = __e_() - _Engine::min();
+    } while (__u >= __y1_);
+    if (__w0_ < __w_rt - 1)
+      __sp <<= __w0_ + 1;
+    else
+      __sp = 0;
+    __sp += __u & __mask1_;
+  }
+  return __sp;
+}
+
+template <class _IntType = int>
+class uniform_int_distribution {
+  static_assert(__libcpp_random_is_valid_inttype<_IntType>::value, "IntType must be a supported integer type");
+
+public:
+  // types
+  typedef _IntType result_type;
+
+  class param_type {
+    result_type __a_;
+    result_type __b_;
+
+  public:
+    typedef uniform_int_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(result_type __a = 0, result_type __b = numeric_limits<result_type>::max())
+        : __a_(__a), __b_(__b) {}
+
+    _LIBCPP_HIDE_FROM_ABI result_type a() const { return __a_; }
+    _LIBCPP_HIDE_FROM_ABI result_type b() const { return __b_; }
+
+    _LIBCPP_HIDE_FROM_ABI friend bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__a_ == __y.__a_ && __x.__b_ == __y.__b_;
+    }
+    _LIBCPP_HIDE_FROM_ABI friend bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructors and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI uniform_int_distribution() : uniform_int_distribution(0) {}
+  _LIBCPP_HIDE_FROM_ABI explicit uniform_int_distribution(
+      result_type __a, result_type __b = numeric_limits<result_type>::max())
+      : __p_(param_type(__a, __b)) {}
+#else
+  explicit uniform_int_distribution(result_type __a = 0, result_type __b = numeric_limits<result_type>::max())
+      : __p_(param_type(__a, __b)) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit uniform_int_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI result_type a() const { return __p_.a(); }
+  _LIBCPP_HIDE_FROM_ABI result_type b() const { return __p_.b(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return a(); }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return b(); }
+
+  _LIBCPP_HIDE_FROM_ABI friend bool
+  operator==(const uniform_int_distribution& __x, const uniform_int_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  _LIBCPP_HIDE_FROM_ABI friend bool
+  operator!=(const uniform_int_distribution& __x, const uniform_int_distribution& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _IntType>
+template <class _URNG>
+typename uniform_int_distribution<_IntType>::result_type uniform_int_distribution<_IntType>::operator()(
+    _URNG& __g, const param_type& __p) _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  typedef __conditional_t<sizeof(result_type) <= sizeof(uint32_t), uint32_t, __make_unsigned_t<result_type> > _UIntType;
+  const _UIntType __rp = _UIntType(__p.b()) - _UIntType(__p.a()) + _UIntType(1);
+  if (__rp == 1)
+    return __p.a();
+  const size_t __dt = numeric_limits<_UIntType>::digits;
+  typedef __independent_bits_engine<_URNG, _UIntType> _Eng;
+  if (__rp == 0)
+    return static_cast<result_type>(_Eng(__g, __dt)());
+  size_t __w = __dt - std::__countl_zero(__rp) - 1;
+  if ((__rp & (numeric_limits<_UIntType>::max() >> (__dt - __w))) != 0)
+    ++__w;
+  _Eng __e(__g, __w);
+  _UIntType __u;
+  do {
+    __u = __e();
+  } while (__u >= __rp);
+  return static_cast<result_type>(__u + __p.a());
+}
+
+template <class _CharT, class _Traits, class _IT>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const uniform_int_distribution<_IT>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _Ostream;
+  __os.flags(_Ostream::dec | _Ostream::left);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  return __os << __x.a() << __sp << __x.b();
+}
+
+template <class _CharT, class _Traits, class _IT>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, uniform_int_distribution<_IT>& __x) {
+  typedef uniform_int_distribution<_IT> _Eng;
+  typedef typename _Eng::result_type result_type;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  result_type __a;
+  result_type __b;
+  __is >> __a >> __b;
+  if (!__is.fail())
+    __x.param(param_type(__a, __b));
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_UNIFORM_INT_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/uniform_random_bit_generator.h b/libcxx/include/__cxx03/__random/uniform_random_bit_generator.h
new file mode 100644
index 000000000000..4076f19b2cb2
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/uniform_random_bit_generator.h
@@ -0,0 +1,44 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_UNIFORM_RANDOM_BIT_GENERATOR_H
+#define _LIBCPP___RANDOM_UNIFORM_RANDOM_BIT_GENERATOR_H
+
+#include <__concepts/arithmetic.h>
+#include <__concepts/invocable.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__functional/invoke.h>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [rand.req.urng]
+template <class _Gen>
+concept uniform_random_bit_generator = invocable<_Gen&> && unsigned_integral<invoke_result_t<_Gen&>> && requires {
+  { _Gen::min() } -> same_as<invoke_result_t<_Gen&>>;
+  { _Gen::max() } -> same_as<invoke_result_t<_Gen&>>;
+  requires bool_constant<(_Gen::min() < _Gen::max())>::value;
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_UNIFORM_RANDOM_BIT_GENERATOR_H
diff --git a/libcxx/include/__cxx03/__random/uniform_real_distribution.h b/libcxx/include/__cxx03/__random/uniform_real_distribution.h
new file mode 100644
index 000000000000..250cb8bab58c
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/uniform_real_distribution.h
@@ -0,0 +1,138 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_UNIFORM_REAL_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_UNIFORM_REAL_DISTRIBUTION_H
+
+#include <__config>
+#include <__random/generate_canonical.h>
+#include <__random/is_valid.h>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _RealType = double>
+class _LIBCPP_TEMPLATE_VIS uniform_real_distribution {
+  static_assert(__libcpp_random_is_valid_realtype<_RealType>::value,
+                "RealType must be a supported floating-point type");
+
+public:
+  // types
+  typedef _RealType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    result_type __a_;
+    result_type __b_;
+
+  public:
+    typedef uniform_real_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(result_type __a = 0, result_type __b = 1) : __a_(__a), __b_(__b) {}
+
+    _LIBCPP_HIDE_FROM_ABI result_type a() const { return __a_; }
+    _LIBCPP_HIDE_FROM_ABI result_type b() const { return __b_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__a_ == __y.__a_ && __x.__b_ == __y.__b_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructors and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI uniform_real_distribution() : uniform_real_distribution(0) {}
+  _LIBCPP_HIDE_FROM_ABI explicit uniform_real_distribution(result_type __a, result_type __b = 1)
+      : __p_(param_type(__a, __b)) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit uniform_real_distribution(result_type __a = 0, result_type __b = 1)
+      : __p_(param_type(__a, __b)) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit uniform_real_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p);
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI result_type a() const { return __p_.a(); }
+  _LIBCPP_HIDE_FROM_ABI result_type b() const { return __p_.b(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return a(); }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return b(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const uniform_real_distribution& __x, const uniform_real_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator!=(const uniform_real_distribution& __x, const uniform_real_distribution& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _RealType>
+template <class _URNG>
+inline typename uniform_real_distribution<_RealType>::result_type
+uniform_real_distribution<_RealType>::operator()(_URNG& __g, const param_type& __p) {
+  static_assert(__libcpp_random_is_valid_urng<_URNG>::value, "");
+  return (__p.b() - __p.a()) * std::generate_canonical<_RealType, numeric_limits<_RealType>::digits>(__g) + __p.a();
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const uniform_real_distribution<_RT>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  return __os << __x.a() << __sp << __x.b();
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, uniform_real_distribution<_RT>& __x) {
+  typedef uniform_real_distribution<_RT> _Eng;
+  typedef typename _Eng::result_type result_type;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  result_type __a;
+  result_type __b;
+  __is >> __a >> __b;
+  if (!__is.fail())
+    __x.param(param_type(__a, __b));
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_UNIFORM_REAL_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/weibull_distribution.h b/libcxx/include/__cxx03/__random/weibull_distribution.h
new file mode 100644
index 000000000000..aa3d63c8e866
--- /dev/null
+++ b/libcxx/include/__cxx03/__random/weibull_distribution.h
@@ -0,0 +1,132 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_WEIBULL_DISTRIBUTION_H
+#define _LIBCPP___RANDOM_WEIBULL_DISTRIBUTION_H
+
+#include <__config>
+#include <__random/exponential_distribution.h>
+#include <__random/is_valid.h>
+#include <cmath>
+#include <iosfwd>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _RealType = double>
+class _LIBCPP_TEMPLATE_VIS weibull_distribution {
+  static_assert(__libcpp_random_is_valid_realtype<_RealType>::value,
+                "RealType must be a supported floating-point type");
+
+public:
+  // types
+  typedef _RealType result_type;
+
+  class _LIBCPP_TEMPLATE_VIS param_type {
+    result_type __a_;
+    result_type __b_;
+
+  public:
+    typedef weibull_distribution distribution_type;
+
+    _LIBCPP_HIDE_FROM_ABI explicit param_type(result_type __a = 1, result_type __b = 1) : __a_(__a), __b_(__b) {}
+
+    _LIBCPP_HIDE_FROM_ABI result_type a() const { return __a_; }
+    _LIBCPP_HIDE_FROM_ABI result_type b() const { return __b_; }
+
+    friend _LIBCPP_HIDE_FROM_ABI bool operator==(const param_type& __x, const param_type& __y) {
+      return __x.__a_ == __y.__a_ && __x.__b_ == __y.__b_;
+    }
+    friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const param_type& __x, const param_type& __y) { return !(__x == __y); }
+  };
+
+private:
+  param_type __p_;
+
+public:
+  // constructor and reset functions
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI weibull_distribution() : weibull_distribution(1) {}
+  _LIBCPP_HIDE_FROM_ABI explicit weibull_distribution(result_type __a, result_type __b = 1)
+      : __p_(param_type(__a, __b)) {}
+#else
+  _LIBCPP_HIDE_FROM_ABI explicit weibull_distribution(result_type __a = 1, result_type __b = 1)
+      : __p_(param_type(__a, __b)) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI explicit weibull_distribution(const param_type& __p) : __p_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI void reset() {}
+
+  // generating functions
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g) {
+    return (*this)(__g, __p_);
+  }
+  template <class _URNG>
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(_URNG& __g, const param_type& __p) {
+    return __p.b() * std::pow(exponential_distribution<result_type>()(__g), 1 / __p.a());
+  }
+
+  // property functions
+  _LIBCPP_HIDE_FROM_ABI result_type a() const { return __p_.a(); }
+  _LIBCPP_HIDE_FROM_ABI result_type b() const { return __p_.b(); }
+
+  _LIBCPP_HIDE_FROM_ABI param_type param() const { return __p_; }
+  _LIBCPP_HIDE_FROM_ABI void param(const param_type& __p) { __p_ = __p; }
+
+  _LIBCPP_HIDE_FROM_ABI result_type min() const { return 0; }
+  _LIBCPP_HIDE_FROM_ABI result_type max() const { return numeric_limits<result_type>::infinity(); }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const weibull_distribution& __x, const weibull_distribution& __y) {
+    return __x.__p_ == __y.__p_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const weibull_distribution& __x, const weibull_distribution& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const weibull_distribution<_RT>& __x) {
+  __save_flags<_CharT, _Traits> __lx(__os);
+  typedef basic_ostream<_CharT, _Traits> _OStream;
+  __os.flags(_OStream::dec | _OStream::left | _OStream::fixed | _OStream::scientific);
+  _CharT __sp = __os.widen(' ');
+  __os.fill(__sp);
+  __os << __x.a() << __sp << __x.b();
+  return __os;
+}
+
+template <class _CharT, class _Traits, class _RT>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, weibull_distribution<_RT>& __x) {
+  typedef weibull_distribution<_RT> _Eng;
+  typedef typename _Eng::result_type result_type;
+  typedef typename _Eng::param_type param_type;
+  __save_flags<_CharT, _Traits> __lx(__is);
+  typedef basic_istream<_CharT, _Traits> _Istream;
+  __is.flags(_Istream::dec | _Istream::skipws);
+  result_type __a;
+  result_type __b;
+  __is >> __a >> __b;
+  if (!__is.fail())
+    __x.param(param_type(__a, __b));
+  return __is;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_WEIBULL_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__ranges/access.h b/libcxx/include/__cxx03/__ranges/access.h
new file mode 100644
index 000000000000..c0a40c5e1017
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/access.h
@@ -0,0 +1,210 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_ACCESS_H
+#define _LIBCPP___RANGES_ACCESS_H
+
+#include <__concepts/class_or_enum.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/readable_traits.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/auto_cast.h>
+#include <__utility/declval.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+template <class _Tp>
+concept __can_borrow = is_lvalue_reference_v<_Tp> || enable_borrowed_range<remove_cvref_t<_Tp>>;
+} // namespace ranges
+
+// [range.access.begin]
+
+namespace ranges {
+namespace __begin {
+template <class _Tp>
+concept __member_begin = __can_borrow<_Tp> && requires(_Tp&& __t) {
+  { _LIBCPP_AUTO_CAST(__t.begin()) } -> input_or_output_iterator;
+};
+
+void begin() = delete;
+
+template <class _Tp>
+concept __unqualified_begin =
+    !__member_begin<_Tp> && __can_borrow<_Tp> && __class_or_enum<remove_cvref_t<_Tp>> && requires(_Tp&& __t) {
+      { _LIBCPP_AUTO_CAST(begin(__t)) } -> input_or_output_iterator;
+    };
+
+struct __fn {
+  template <class _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp (&__t)[]) const noexcept
+    requires(sizeof(_Tp) >= 0) // Disallow incomplete element types.
+  {
+    return __t + 0;
+  }
+
+  template <class _Tp, size_t _Np>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp (&__t)[_Np]) const noexcept
+    requires(sizeof(_Tp) >= 0) // Disallow incomplete element types.
+  {
+    return __t + 0;
+  }
+
+  template <class _Tp>
+    requires __member_begin<_Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(__t.begin()))) {
+    return _LIBCPP_AUTO_CAST(__t.begin());
+  }
+
+  template <class _Tp>
+    requires __unqualified_begin<_Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(begin(__t)))) {
+    return _LIBCPP_AUTO_CAST(begin(__t));
+  }
+
+  void operator()(auto&&) const = delete;
+};
+} // namespace __begin
+
+inline namespace __cpo {
+inline constexpr auto begin = __begin::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+// [range.range]
+
+namespace ranges {
+template <class _Tp>
+using iterator_t = decltype(ranges::begin(std::declval<_Tp&>()));
+} // namespace ranges
+
+// [range.access.end]
+
+namespace ranges {
+namespace __end {
+template <class _Tp>
+concept __member_end = __can_borrow<_Tp> && requires(_Tp&& __t) {
+  typename iterator_t<_Tp>;
+  { _LIBCPP_AUTO_CAST(__t.end()) } -> sentinel_for<iterator_t<_Tp>>;
+};
+
+void end() = delete;
+
+template <class _Tp>
+concept __unqualified_end =
+    !__member_end<_Tp> && __can_borrow<_Tp> && __class_or_enum<remove_cvref_t<_Tp>> && requires(_Tp&& __t) {
+      typename iterator_t<_Tp>;
+      { _LIBCPP_AUTO_CAST(end(__t)) } -> sentinel_for<iterator_t<_Tp>>;
+    };
+
+struct __fn {
+  template <class _Tp, size_t _Np>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp (&__t)[_Np]) const noexcept
+    requires(sizeof(_Tp) >= 0) // Disallow incomplete element types.
+  {
+    return __t + _Np;
+  }
+
+  template <class _Tp>
+    requires __member_end<_Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(__t.end()))) {
+    return _LIBCPP_AUTO_CAST(__t.end());
+  }
+
+  template <class _Tp>
+    requires __unqualified_end<_Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(end(__t)))) {
+    return _LIBCPP_AUTO_CAST(end(__t));
+  }
+
+  void operator()(auto&&) const = delete;
+};
+} // namespace __end
+
+inline namespace __cpo {
+inline constexpr auto end = __end::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+// [range.access.cbegin]
+
+namespace ranges {
+namespace __cbegin {
+struct __fn {
+  template <class _Tp>
+    requires is_lvalue_reference_v<_Tp&&>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(ranges::begin(static_cast<const remove_reference_t<_Tp>&>(__t))))
+          -> decltype(ranges::begin(static_cast<const remove_reference_t<_Tp>&>(__t))) {
+    return ranges::begin(static_cast<const remove_reference_t<_Tp>&>(__t));
+  }
+
+  template <class _Tp>
+    requires is_rvalue_reference_v<_Tp&&>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(ranges::begin(static_cast<const _Tp&&>(__t))))
+          -> decltype(ranges::begin(static_cast<const _Tp&&>(__t))) {
+    return ranges::begin(static_cast<const _Tp&&>(__t));
+  }
+};
+} // namespace __cbegin
+
+inline namespace __cpo {
+inline constexpr auto cbegin = __cbegin::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+// [range.access.cend]
+
+namespace ranges {
+namespace __cend {
+struct __fn {
+  template <class _Tp>
+    requires is_lvalue_reference_v<_Tp&&>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(ranges::end(static_cast<const remove_reference_t<_Tp>&>(__t))))
+          -> decltype(ranges::end(static_cast<const remove_reference_t<_Tp>&>(__t))) {
+    return ranges::end(static_cast<const remove_reference_t<_Tp>&>(__t));
+  }
+
+  template <class _Tp>
+    requires is_rvalue_reference_v<_Tp&&>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const noexcept(
+      noexcept(ranges::end(static_cast<const _Tp&&>(__t)))) -> decltype(ranges::end(static_cast<const _Tp&&>(__t))) {
+    return ranges::end(static_cast<const _Tp&&>(__t));
+  }
+};
+} // namespace __cend
+
+inline namespace __cpo {
+inline constexpr auto cend = __cend::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_ACCESS_H
diff --git a/libcxx/include/__cxx03/__ranges/all.h b/libcxx/include/__cxx03/__ranges/all.h
new file mode 100644
index 000000000000..023cee6caa9a
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/all.h
@@ -0,0 +1,78 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_ALL_H
+#define _LIBCPP___RANGES_ALL_H
+
+#include <__config>
+#include <__functional/compose.h>         // TODO(modules): Those should not be required
+#include <__functional/perfect_forward.h> //
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/owning_view.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/ref_view.h>
+#include <__type_traits/decay.h>
+#include <__utility/auto_cast.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges::views {
+
+namespace __all {
+struct __fn : __range_adaptor_closure<__fn> {
+  template <class _Tp>
+    requires ranges::view<decay_t<_Tp>>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const noexcept(
+      noexcept(_LIBCPP_AUTO_CAST(std::forward<_Tp>(__t)))) -> decltype(_LIBCPP_AUTO_CAST(std::forward<_Tp>(__t))) {
+    return _LIBCPP_AUTO_CAST(std::forward<_Tp>(__t));
+  }
+
+  template <class _Tp>
+    requires(!ranges::view<decay_t<_Tp>>) && requires(_Tp&& __t) { ranges::ref_view{std::forward<_Tp>(__t)}; }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(ranges::ref_view{std::forward<_Tp>(__t)})) {
+    return ranges::ref_view{std::forward<_Tp>(__t)};
+  }
+
+  template <class _Tp>
+    requires(
+        !ranges::view<decay_t<_Tp>> && !requires(_Tp&& __t) { ranges::ref_view{std::forward<_Tp>(__t)}; } &&
+        requires(_Tp&& __t) { ranges::owning_view{std::forward<_Tp>(__t)}; })
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(ranges::owning_view{std::forward<_Tp>(__t)})) {
+    return ranges::owning_view{std::forward<_Tp>(__t)};
+  }
+};
+} // namespace __all
+
+inline namespace __cpo {
+inline constexpr auto all = __all::__fn{};
+} // namespace __cpo
+
+template <ranges::viewable_range _Range>
+using all_t = decltype(views::all(std::declval<_Range>()));
+
+} // namespace ranges::views
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_ALL_H
diff --git a/libcxx/include/__cxx03/__ranges/as_rvalue_view.h b/libcxx/include/__cxx03/__ranges/as_rvalue_view.h
new file mode 100644
index 000000000000..5849a6c36839
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/as_rvalue_view.h
@@ -0,0 +1,142 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_AS_RVALUE_H
+#define _LIBCPP___RANGES_AS_RVALUE_H
+
+#include <__concepts/constructible.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__iterator/move_iterator.h>
+#include <__iterator/move_sentinel.h>
+#include <__ranges/access.h>
+#include <__ranges/all.h>
+#include <__ranges/concepts.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/size.h>
+#include <__ranges/view_interface.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+template <view _View>
+  requires input_range<_View>
+class as_rvalue_view : public view_interface<as_rvalue_view<_View>> {
+  _LIBCPP_NO_UNIQUE_ADDRESS _View __base_ = _View();
+
+public:
+  _LIBCPP_HIDE_FROM_ABI as_rvalue_view()
+    requires default_initializable<_View>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit as_rvalue_view(_View __base) : __base_(std::move(__base)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
+    requires copy_constructible<_View>
+  {
+    return __base_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin()
+    requires(!__simple_view<_View>)
+  {
+    return move_iterator(ranges::begin(__base_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
+    requires range<const _View>
+  {
+    return move_iterator(ranges::begin(__base_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end()
+    requires(!__simple_view<_View>)
+  {
+    if constexpr (common_range<_View>) {
+      return move_iterator(ranges::end(__base_));
+    } else {
+      return move_sentinel(ranges::end(__base_));
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
+    requires range<const _View>
+  {
+    if constexpr (common_range<const _View>) {
+      return move_iterator(ranges::end(__base_));
+    } else {
+      return move_sentinel(ranges::end(__base_));
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size()
+    requires sized_range<_View>
+  {
+    return ranges::size(__base_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
+    requires sized_range<const _View>
+  {
+    return ranges::size(__base_);
+  }
+};
+
+template <class _Range>
+as_rvalue_view(_Range&&) -> as_rvalue_view<views::all_t<_Range>>;
+
+template <class _View>
+inline constexpr bool enable_borrowed_range<as_rvalue_view<_View>> = enable_borrowed_range<_View>;
+
+namespace views {
+namespace __as_rvalue {
+struct __fn : __range_adaptor_closure<__fn> {
+  template <class _Range>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr auto
+  operator()(_Range&& __range) noexcept(noexcept(as_rvalue_view(std::forward<_Range>(__range))))
+      -> decltype(/*--------------------------*/ as_rvalue_view(std::forward<_Range>(__range))) {
+    return /*---------------------------------*/ as_rvalue_view(std::forward<_Range>(__range));
+  }
+
+  template <class _Range>
+    requires same_as<range_rvalue_reference_t<_Range>, range_reference_t<_Range>>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr auto
+  operator()(_Range&& __range) noexcept(noexcept(views::all(std::forward<_Range>(__range))))
+      -> decltype(/*--------------------------*/ views::all(std::forward<_Range>(__range))) {
+    return /*---------------------------------*/ views::all(std::forward<_Range>(__range));
+  }
+};
+} // namespace __as_rvalue
+
+inline namespace __cpo {
+inline constexpr auto as_rvalue = __as_rvalue::__fn{};
+} // namespace __cpo
+} // namespace views
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_AS_RVALUE_H
diff --git a/libcxx/include/__cxx03/__ranges/chunk_by_view.h b/libcxx/include/__cxx03/__ranges/chunk_by_view.h
new file mode 100644
index 000000000000..00014d9f10ae
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/chunk_by_view.h
@@ -0,0 +1,235 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_CHUNK_BY_VIEW_H
+#define _LIBCPP___RANGES_CHUNK_BY_VIEW_H
+
+#include <__algorithm/ranges_adjacent_find.h>
+#include <__assert>
+#include <__concepts/constructible.h>
+#include <__config>
+#include <__functional/bind_back.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__iterator/default_sentinel.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/prev.h>
+#include <__memory/addressof.h>
+#include <__ranges/access.h>
+#include <__ranges/all.h>
+#include <__ranges/concepts.h>
+#include <__ranges/movable_box.h>
+#include <__ranges/non_propagating_cache.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/reverse_view.h>
+#include <__ranges/subrange.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_object.h>
+#include <__utility/forward.h>
+#include <__utility/in_place.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+namespace ranges {
+
+template <forward_range _View, indirect_binary_predicate<iterator_t<_View>, iterator_t<_View>> _Pred>
+  requires view<_View> && is_object_v<_Pred>
+class _LIBCPP_ABI_LLVM18_NO_UNIQUE_ADDRESS chunk_by_view : public view_interface<chunk_by_view<_View, _Pred>> {
+  _LIBCPP_NO_UNIQUE_ADDRESS _View __base_ = _View();
+  _LIBCPP_NO_UNIQUE_ADDRESS __movable_box<_Pred> __pred_;
+
+  // We cache the result of begin() to allow providing an amortized O(1).
+  using _Cache = __non_propagating_cache<iterator_t<_View>>;
+  _Cache __cached_begin_;
+
+  class __iterator;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_View> __find_next(iterator_t<_View> __current) {
+    // Note: this duplicates a check in `optional` but provides a better error message.
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __pred_.__has_value(), "Trying to call __find_next() on a chunk_by_view that does not have a valid predicate.");
+    auto __reversed_pred = [this]<class _Tp, class _Up>(_Tp&& __x, _Up&& __y) -> bool {
+      return !std::invoke(*__pred_, std::forward<_Tp>(__x), std::forward<_Up>(__y));
+    };
+    return ranges::next(
+        ranges::adjacent_find(__current, ranges::end(__base_), __reversed_pred), 1, ranges::end(__base_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_View> __find_prev(iterator_t<_View> __current)
+    requires bidirectional_range<_View>
+  {
+    // Attempting to decrement a begin iterator is a no-op (`__find_prev` would return the same argument given to it).
+    _LIBCPP_ASSERT_PEDANTIC(__current != ranges::begin(__base_), "Trying to call __find_prev() on a begin iterator.");
+    // Note: this duplicates a check in `optional` but provides a better error message.
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __pred_.__has_value(), "Trying to call __find_prev() on a chunk_by_view that does not have a valid predicate.");
+
+    auto __first = ranges::begin(__base_);
+    reverse_view __reversed{subrange{__first, __current}};
+    auto __reversed_pred = [this]<class _Tp, class _Up>(_Tp&& __x, _Up&& __y) -> bool {
+      return !std::invoke(*__pred_, std::forward<_Up>(__y), std::forward<_Tp>(__x));
+    };
+    return ranges::prev(ranges::adjacent_find(__reversed, __reversed_pred).base(), 1, std::move(__first));
+  }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI chunk_by_view()
+    requires default_initializable<_View> && default_initializable<_Pred>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit chunk_by_view(_View __base, _Pred __pred)
+      : __base_(std::move(__base)), __pred_(in_place, std::move(__pred)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
+    requires copy_constructible<_View>
+  {
+    return __base_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Pred& pred() const { return *__pred_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() {
+    // Note: this duplicates a check in `optional` but provides a better error message.
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __pred_.__has_value(), "Trying to call begin() on a chunk_by_view that does not have a valid predicate.");
+
+    auto __first = ranges::begin(__base_);
+    if (!__cached_begin_.__has_value()) {
+      __cached_begin_.__emplace(__find_next(__first));
+    }
+    return {*this, std::move(__first), *__cached_begin_};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() {
+    if constexpr (common_range<_View>) {
+      return __iterator{*this, ranges::end(__base_), ranges::end(__base_)};
+    } else {
+      return default_sentinel;
+    }
+  }
+};
+
+template <class _Range, class _Pred>
+chunk_by_view(_Range&&, _Pred) -> chunk_by_view<views::all_t<_Range>, _Pred>;
+
+template <forward_range _View, indirect_binary_predicate<iterator_t<_View>, iterator_t<_View>> _Pred>
+  requires view<_View> && is_object_v<_Pred>
+class chunk_by_view<_View, _Pred>::__iterator {
+  friend chunk_by_view;
+
+  chunk_by_view* __parent_                               = nullptr;
+  _LIBCPP_NO_UNIQUE_ADDRESS iterator_t<_View> __current_ = iterator_t<_View>();
+  _LIBCPP_NO_UNIQUE_ADDRESS iterator_t<_View> __next_    = iterator_t<_View>();
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator(
+      chunk_by_view& __parent, iterator_t<_View> __current, iterator_t<_View> __next)
+      : __parent_(std::addressof(__parent)), __current_(__current), __next_(__next) {}
+
+public:
+  using value_type        = subrange<iterator_t<_View>>;
+  using difference_type   = range_difference_t<_View>;
+  using iterator_category = input_iterator_tag;
+  using iterator_concept  = conditional_t<bidirectional_range<_View>, bidirectional_iterator_tag, forward_iterator_tag>;
+
+  _LIBCPP_HIDE_FROM_ABI __iterator() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr value_type operator*() const {
+    // If the iterator is at end, this would return an empty range which can be checked by the calling code and doesn't
+    // necessarily lead to a bad access.
+    _LIBCPP_ASSERT_PEDANTIC(__current_ != __next_, "Trying to dereference past-the-end chunk_by_view iterator.");
+    return {__current_, __next_};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() {
+    // Attempting to increment an end iterator is a no-op (`__find_next` would return the same argument given to it).
+    _LIBCPP_ASSERT_PEDANTIC(__current_ != __next_, "Trying to increment past end chunk_by_view iterator.");
+    __current_ = __next_;
+    __next_    = __parent_->__find_next(__current_);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator++(int) {
+    auto __tmp = *this;
+    ++*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator--()
+    requires bidirectional_range<_View>
+  {
+    __next_    = __current_;
+    __current_ = __parent_->__find_prev(__next_);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator--(int)
+    requires bidirectional_range<_View>
+  {
+    auto __tmp = *this;
+    --*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __iterator& __y) {
+    return __x.__current_ == __y.__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, default_sentinel_t) {
+    return __x.__current_ == __x.__next_;
+  }
+};
+
+namespace views {
+namespace __chunk_by {
+struct __fn {
+  template <class _Range, class _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range, _Pred&& __pred) const
+      noexcept(noexcept(/**/ chunk_by_view(std::forward<_Range>(__range), std::forward<_Pred>(__pred))))
+          -> decltype(/*--*/ chunk_by_view(std::forward<_Range>(__range), std::forward<_Pred>(__pred))) {
+    return /*-------------*/ chunk_by_view(std::forward<_Range>(__range), std::forward<_Pred>(__pred));
+  }
+
+  template <class _Pred>
+    requires constructible_from<decay_t<_Pred>, _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Pred&& __pred) const
+      noexcept(is_nothrow_constructible_v<decay_t<_Pred>, _Pred>) {
+    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Pred>(__pred)));
+  }
+};
+} // namespace __chunk_by
+
+inline namespace __cpo {
+inline constexpr auto chunk_by = __chunk_by::__fn{};
+} // namespace __cpo
+} // namespace views
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_CHUNK_BY_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/common_view.h b/libcxx/include/__cxx03/__ranges/common_view.h
new file mode 100644
index 000000000000..133236dd1d78
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/common_view.h
@@ -0,0 +1,142 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_COMMON_VIEW_H
+#define _LIBCPP___RANGES_COMMON_VIEW_H
+
+#include <__concepts/constructible.h>
+#include <__concepts/copyable.h>
+#include <__config>
+#include <__iterator/common_iterator.h>
+#include <__iterator/iterator_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/all.h>
+#include <__ranges/concepts.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/size.h>
+#include <__ranges/view_interface.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+template <view _View>
+  requires(!common_range<_View> && copyable<iterator_t<_View>>)
+class common_view : public view_interface<common_view<_View>> {
+  _View __base_ = _View();
+
+public:
+  _LIBCPP_HIDE_FROM_ABI common_view()
+    requires default_initializable<_View>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit common_view(_View __v) : __base_(std::move(__v)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
+    requires copy_constructible<_View>
+  {
+    return __base_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() {
+    if constexpr (random_access_range<_View> && sized_range<_View>)
+      return ranges::begin(__base_);
+    else
+      return common_iterator<iterator_t<_View>, sentinel_t<_View>>(ranges::begin(__base_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
+    requires range<const _View>
+  {
+    if constexpr (random_access_range<const _View> && sized_range<const _View>)
+      return ranges::begin(__base_);
+    else
+      return common_iterator<iterator_t<const _View>, sentinel_t<const _View>>(ranges::begin(__base_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() {
+    if constexpr (random_access_range<_View> && sized_range<_View>)
+      return ranges::begin(__base_) + ranges::size(__base_);
+    else
+      return common_iterator<iterator_t<_View>, sentinel_t<_View>>(ranges::end(__base_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
+    requires range<const _View>
+  {
+    if constexpr (random_access_range<const _View> && sized_range<const _View>)
+      return ranges::begin(__base_) + ranges::size(__base_);
+    else
+      return common_iterator<iterator_t<const _View>, sentinel_t<const _View>>(ranges::end(__base_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size()
+    requires sized_range<_View>
+  {
+    return ranges::size(__base_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
+    requires sized_range<const _View>
+  {
+    return ranges::size(__base_);
+  }
+};
+
+template <class _Range>
+common_view(_Range&&) -> common_view<views::all_t<_Range>>;
+
+template <class _View>
+inline constexpr bool enable_borrowed_range<common_view<_View>> = enable_borrowed_range<_View>;
+
+namespace views {
+namespace __common {
+struct __fn : __range_adaptor_closure<__fn> {
+  template <class _Range>
+    requires common_range<_Range>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range) const noexcept(
+      noexcept(views::all(std::forward<_Range>(__range)))) -> decltype(views::all(std::forward<_Range>(__range))) {
+    return views::all(std::forward<_Range>(__range));
+  }
+
+  template <class _Range>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range) const noexcept(noexcept(common_view{
+      std::forward<_Range>(__range)})) -> decltype(common_view{std::forward<_Range>(__range)}) {
+    return common_view{std::forward<_Range>(__range)};
+  }
+};
+} // namespace __common
+
+inline namespace __cpo {
+inline constexpr auto common = __common::__fn{};
+} // namespace __cpo
+} // namespace views
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_COMMON_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/concepts.h b/libcxx/include/__cxx03/__ranges/concepts.h
new file mode 100644
index 000000000000..674a3f359ff9
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/concepts.h
@@ -0,0 +1,142 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_CONCEPTS_H
+#define _LIBCPP___RANGES_CONCEPTS_H
+
+#include <__concepts/constructible.h>
+#include <__concepts/movable.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/readable_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/data.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/enable_view.h>
+#include <__ranges/size.h>
+#include <__type_traits/add_pointer.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/declval.h>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+// [range.range]
+
+template <class _Tp>
+concept range = requires(_Tp& __t) {
+  ranges::begin(__t); // sometimes equality-preserving
+  ranges::end(__t);
+};
+
+template <class _Tp>
+concept input_range = range<_Tp> && input_iterator<iterator_t<_Tp>>;
+
+template <class _Range>
+concept borrowed_range =
+    range<_Range> && (is_lvalue_reference_v<_Range> || enable_borrowed_range<remove_cvref_t<_Range>>);
+
+// `iterator_t` defined in <__ranges/access.h>
+
+template <range _Rp>
+using sentinel_t = decltype(ranges::end(std::declval<_Rp&>()));
+
+template <range _Rp>
+using range_difference_t = iter_difference_t<iterator_t<_Rp>>;
+
+template <range _Rp>
+using range_value_t = iter_value_t<iterator_t<_Rp>>;
+
+template <range _Rp>
+using range_reference_t = iter_reference_t<iterator_t<_Rp>>;
+
+template <range _Rp>
+using range_rvalue_reference_t = iter_rvalue_reference_t<iterator_t<_Rp>>;
+
+template <range _Rp>
+using range_common_reference_t = iter_common_reference_t<iterator_t<_Rp>>;
+
+// [range.sized]
+template <class _Tp>
+concept sized_range = range<_Tp> && requires(_Tp& __t) { ranges::size(__t); };
+
+template <sized_range _Rp>
+using range_size_t = decltype(ranges::size(std::declval<_Rp&>()));
+
+// `disable_sized_range` defined in `<__ranges/size.h>`
+
+// [range.view], views
+
+// `enable_view` defined in <__ranges/enable_view.h>
+// `view_base` defined in <__ranges/enable_view.h>
+
+template <class _Tp>
+concept view = range<_Tp> && movable<_Tp> && enable_view<_Tp>;
+
+template <class _Range>
+concept __simple_view =
+    view<_Range> && range<const _Range> && same_as<iterator_t<_Range>, iterator_t<const _Range>> &&
+    same_as<sentinel_t<_Range>, sentinel_t<const _Range>>;
+
+// [range.refinements], other range refinements
+template <class _Rp, class _Tp>
+concept output_range = range<_Rp> && output_iterator<iterator_t<_Rp>, _Tp>;
+
+template <class _Tp>
+concept forward_range = input_range<_Tp> && forward_iterator<iterator_t<_Tp>>;
+
+template <class _Tp>
+concept bidirectional_range = forward_range<_Tp> && bidirectional_iterator<iterator_t<_Tp>>;
+
+template <class _Tp>
+concept random_access_range = bidirectional_range<_Tp> && random_access_iterator<iterator_t<_Tp>>;
+
+template <class _Tp>
+concept contiguous_range = random_access_range<_Tp> && contiguous_iterator<iterator_t<_Tp>> && requires(_Tp& __t) {
+  { ranges::data(__t) } -> same_as<add_pointer_t<range_reference_t<_Tp>>>;
+};
+
+template <class _Tp>
+concept common_range = range<_Tp> && same_as<iterator_t<_Tp>, sentinel_t<_Tp>>;
+
+template <class _Tp>
+inline constexpr bool __is_std_initializer_list = false;
+
+template <class _Ep>
+inline constexpr bool __is_std_initializer_list<initializer_list<_Ep>> = true;
+
+template <class _Tp>
+concept viewable_range =
+    range<_Tp> &&
+    ((view<remove_cvref_t<_Tp>> && constructible_from<remove_cvref_t<_Tp>, _Tp>) ||
+     (!view<remove_cvref_t<_Tp>> &&
+      (is_lvalue_reference_v<_Tp> ||
+       (movable<remove_reference_t<_Tp>> && !__is_std_initializer_list<remove_cvref_t<_Tp>>))));
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_CONCEPTS_H
diff --git a/libcxx/include/__cxx03/__ranges/container_compatible_range.h b/libcxx/include/__cxx03/__ranges/container_compatible_range.h
new file mode 100644
index 000000000000..a58f1119885e
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/container_compatible_range.h
@@ -0,0 +1,33 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_CONTAINER_COMPATIBLE_RANGE_H
+#define _LIBCPP___RANGES_CONTAINER_COMPATIBLE_RANGE_H
+
+#include <__concepts/convertible_to.h>
+#include <__config>
+#include <__ranges/concepts.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Range, class _Tp>
+concept _ContainerCompatibleRange =
+    ranges::input_range<_Range> && convertible_to<ranges::range_reference_t<_Range>, _Tp>;
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_CONTAINER_COMPATIBLE_RANGE_H
diff --git a/libcxx/include/__cxx03/__ranges/counted.h b/libcxx/include/__cxx03/__ranges/counted.h
new file mode 100644
index 000000000000..e365deca4e63
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/counted.h
@@ -0,0 +1,89 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_COUNTED_H
+#define _LIBCPP___RANGES_COUNTED_H
+
+#include <__concepts/convertible_to.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/counted_iterator.h>
+#include <__iterator/default_sentinel.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/pointer_traits.h>
+#include <__ranges/subrange.h>
+#include <__type_traits/decay.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <cstddef>
+#include <span>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges::views {
+
+namespace __counted {
+
+struct __fn {
+  template <contiguous_iterator _It>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto
+  __go(_It __it,
+       iter_difference_t<_It> __count) noexcept(noexcept(span(std::to_address(__it), static_cast<size_t>(__count))))
+  // Deliberately omit return-type SFINAE, because to_address is not SFINAE-friendly
+  {
+    return span(std::to_address(__it), static_cast<size_t>(__count));
+  }
+
+  template <random_access_iterator _It>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_It __it, iter_difference_t<_It> __count) noexcept(
+      noexcept(subrange(__it, __it + __count))) -> decltype(subrange(__it, __it + __count)) {
+    return subrange(__it, __it + __count);
+  }
+
+  template <class _It>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_It __it, iter_difference_t<_It> __count) noexcept(
+      noexcept(subrange(counted_iterator(std::move(__it), __count), default_sentinel)))
+      -> decltype(subrange(counted_iterator(std::move(__it), __count), default_sentinel)) {
+    return subrange(counted_iterator(std::move(__it), __count), default_sentinel);
+  }
+
+  template <class _It, convertible_to<iter_difference_t<_It>> _Diff>
+    requires input_or_output_iterator<decay_t<_It>>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_It&& __it, _Diff&& __count) const
+      noexcept(noexcept(__go(std::forward<_It>(__it), std::forward<_Diff>(__count))))
+          -> decltype(__go(std::forward<_It>(__it), std::forward<_Diff>(__count))) {
+    return __go(std::forward<_It>(__it), std::forward<_Diff>(__count));
+  }
+};
+
+} // namespace __counted
+
+inline namespace __cpo {
+inline constexpr auto counted = __counted::__fn{};
+} // namespace __cpo
+
+} // namespace ranges::views
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_COUNTED_H
diff --git a/libcxx/include/__cxx03/__ranges/dangling.h b/libcxx/include/__cxx03/__ranges/dangling.h
new file mode 100644
index 000000000000..613084d5fb9f
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/dangling.h
@@ -0,0 +1,42 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_DANGLING_H
+#define _LIBCPP___RANGES_DANGLING_H
+
+#include <__config>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__type_traits/conditional.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+struct dangling {
+  dangling() = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr dangling(auto&&...) noexcept {}
+};
+
+template <range _Rp>
+using borrowed_iterator_t = _If<borrowed_range<_Rp>, iterator_t<_Rp>, dangling>;
+
+// borrowed_subrange_t defined in <__ranges/subrange.h>
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_DANGLING_H
diff --git a/libcxx/include/__cxx03/__ranges/data.h b/libcxx/include/__cxx03/__ranges/data.h
new file mode 100644
index 000000000000..50db3cffeeed
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/data.h
@@ -0,0 +1,102 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_DATA_H
+#define _LIBCPP___RANGES_DATA_H
+
+#include <__concepts/class_or_enum.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/pointer_traits.h>
+#include <__ranges/access.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_object.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/remove_pointer.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/auto_cast.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [range.prim.data]
+
+namespace ranges {
+namespace __data {
+template <class _Tp>
+concept __ptr_to_object = is_pointer_v<_Tp> && is_object_v<remove_pointer_t<_Tp>>;
+
+template <class _Tp>
+concept __member_data = __can_borrow<_Tp> && requires(_Tp&& __t) {
+  { _LIBCPP_AUTO_CAST(__t.data()) } -> __ptr_to_object;
+};
+
+template <class _Tp>
+concept __ranges_begin_invocable = !__member_data<_Tp> && __can_borrow<_Tp> && requires(_Tp&& __t) {
+  { ranges::begin(__t) } -> contiguous_iterator;
+};
+
+struct __fn {
+  template <__member_data _Tp>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const noexcept(noexcept(__t.data())) {
+    return __t.data();
+  }
+
+  template <__ranges_begin_invocable _Tp>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(std::to_address(ranges::begin(__t)))) {
+    return std::to_address(ranges::begin(__t));
+  }
+};
+} // namespace __data
+
+inline namespace __cpo {
+inline constexpr auto data = __data::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+// [range.prim.cdata]
+
+namespace ranges {
+namespace __cdata {
+struct __fn {
+  template <class _Tp>
+    requires is_lvalue_reference_v<_Tp&&>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(ranges::data(static_cast<const remove_reference_t<_Tp>&>(__t))))
+          -> decltype(ranges::data(static_cast<const remove_reference_t<_Tp>&>(__t))) {
+    return ranges::data(static_cast<const remove_reference_t<_Tp>&>(__t));
+  }
+
+  template <class _Tp>
+    requires is_rvalue_reference_v<_Tp&&>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const noexcept(
+      noexcept(ranges::data(static_cast<const _Tp&&>(__t)))) -> decltype(ranges::data(static_cast<const _Tp&&>(__t))) {
+    return ranges::data(static_cast<const _Tp&&>(__t));
+  }
+};
+} // namespace __cdata
+
+inline namespace __cpo {
+inline constexpr auto cdata = __cdata::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_DATA_H
diff --git a/libcxx/include/__cxx03/__ranges/drop_view.h b/libcxx/include/__cxx03/__ranges/drop_view.h
new file mode 100644
index 000000000000..853e22a402ca
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/drop_view.h
@@ -0,0 +1,329 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_DROP_VIEW_H
+#define _LIBCPP___RANGES_DROP_VIEW_H
+
+#include <__algorithm/min.h>
+#include <__assert>
+#include <__concepts/constructible.h>
+#include <__concepts/convertible_to.h>
+#include <__config>
+#include <__functional/bind_back.h>
+#include <__fwd/span.h>
+#include <__fwd/string_view.h>
+#include <__iterator/concepts.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__ranges/access.h>
+#include <__ranges/all.h>
+#include <__ranges/concepts.h>
+#include <__ranges/empty_view.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/iota_view.h>
+#include <__ranges/non_propagating_cache.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/repeat_view.h>
+#include <__ranges/size.h>
+#include <__ranges/subrange.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/make_unsigned.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/auto_cast.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+template <view _View>
+class drop_view : public view_interface<drop_view<_View>> {
+  // We cache begin() whenever ranges::next is not guaranteed O(1) to provide an
+  // amortized O(1) begin() method. If this is an input_range, then we cannot cache
+  // begin because begin is not equality preserving.
+  // Note: drop_view<input-range>::begin() is still trivially amortized O(1) because
+  // one can't call begin() on it more than once.
+  static constexpr bool _UseCache = forward_range<_View> && !(random_access_range<_View> && sized_range<_View>);
+  using _Cache                    = _If<_UseCache, __non_propagating_cache<iterator_t<_View>>, __empty_cache>;
+  _LIBCPP_NO_UNIQUE_ADDRESS _Cache __cached_begin_ = _Cache();
+  range_difference_t<_View> __count_               = 0;
+  _View __base_                                    = _View();
+
+public:
+  _LIBCPP_HIDE_FROM_ABI drop_view()
+    requires default_initializable<_View>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23
+  drop_view(_View __base, range_difference_t<_View> __count)
+      : __count_(__count), __base_(std::move(__base)) {
+    _LIBCPP_ASSERT_UNCATEGORIZED(__count_ >= 0, "count must be greater than or equal to zero.");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
+    requires copy_constructible<_View>
+  {
+    return __base_;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin()
+    requires(!(__simple_view<_View> && random_access_range<const _View> && sized_range<const _View>))
+  {
+    if constexpr (random_access_range<_View> && sized_range<_View>) {
+      const auto __dist = std::min(ranges::distance(__base_), __count_);
+      return ranges::begin(__base_) + __dist;
+    }
+    if constexpr (_UseCache)
+      if (__cached_begin_.__has_value())
+        return *__cached_begin_;
+
+    auto __tmp = ranges::next(ranges::begin(__base_), __count_, ranges::end(__base_));
+    if constexpr (_UseCache)
+      __cached_begin_.__emplace(__tmp);
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
+    requires random_access_range<const _View> && sized_range<const _View>
+  {
+    const auto __dist = std::min(ranges::distance(__base_), __count_);
+    return ranges::begin(__base_) + __dist;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end()
+    requires(!__simple_view<_View>)
+  {
+    return ranges::end(__base_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
+    requires range<const _View>
+  {
+    return ranges::end(__base_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __size(auto& __self) {
+    const auto __s = ranges::size(__self.__base_);
+    const auto __c = static_cast<decltype(__s)>(__self.__count_);
+    return __s < __c ? 0 : __s - __c;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size()
+    requires sized_range<_View>
+  {
+    return __size(*this);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
+    requires sized_range<const _View>
+  {
+    return __size(*this);
+  }
+};
+
+template <class _Range>
+drop_view(_Range&&, range_difference_t<_Range>) -> drop_view<views::all_t<_Range>>;
+
+template <class _Tp>
+inline constexpr bool enable_borrowed_range<drop_view<_Tp>> = enable_borrowed_range<_Tp>;
+
+namespace views {
+namespace __drop {
+
+template <class _Tp>
+inline constexpr bool __is_empty_view = false;
+
+template <class _Tp>
+inline constexpr bool __is_empty_view<empty_view<_Tp>> = true;
+
+template <class _Tp>
+inline constexpr bool __is_passthrough_specialization = false;
+
+template <class _Tp, size_t _Extent>
+inline constexpr bool __is_passthrough_specialization<span<_Tp, _Extent>> = true;
+
+template <class _CharT, class _Traits>
+inline constexpr bool __is_passthrough_specialization<basic_string_view<_CharT, _Traits>> = true;
+
+template <class _Np, class _Bound>
+inline constexpr bool __is_passthrough_specialization<iota_view<_Np, _Bound>> = true;
+
+template <class _Iter, class _Sent, subrange_kind _Kind>
+inline constexpr bool __is_passthrough_specialization<subrange<_Iter, _Sent, _Kind>> =
+    !subrange<_Iter, _Sent, _Kind>::_StoreSize;
+
+template <class _Tp>
+inline constexpr bool __is_subrange_specialization_with_store_size = false;
+
+template <class _Iter, class _Sent, subrange_kind _Kind>
+inline constexpr bool __is_subrange_specialization_with_store_size<subrange<_Iter, _Sent, _Kind>> =
+    subrange<_Iter, _Sent, _Kind>::_StoreSize;
+
+template <class _Tp>
+struct __passthrough_type;
+
+template <class _Tp, size_t _Extent>
+struct __passthrough_type<span<_Tp, _Extent>> {
+  using type = span<_Tp>;
+};
+
+template <class _CharT, class _Traits>
+struct __passthrough_type<basic_string_view<_CharT, _Traits>> {
+  using type = basic_string_view<_CharT, _Traits>;
+};
+
+template <class _Np, class _Bound>
+struct __passthrough_type<iota_view<_Np, _Bound>> {
+  using type = iota_view<_Np, _Bound>;
+};
+
+template <class _Iter, class _Sent, subrange_kind _Kind>
+struct __passthrough_type<subrange<_Iter, _Sent, _Kind>> {
+  using type = subrange<_Iter, _Sent, _Kind>;
+};
+
+template <class _Tp>
+using __passthrough_type_t = typename __passthrough_type<_Tp>::type;
+
+struct __fn {
+  // [range.drop.overview]: the `empty_view` case.
+  template <class _Range, convertible_to<range_difference_t<_Range>> _Np>
+    requires __is_empty_view<remove_cvref_t<_Range>>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range, _Np&&) const
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(std::forward<_Range>(__range))))
+          -> decltype(_LIBCPP_AUTO_CAST(std::forward<_Range>(__range))) {
+    return _LIBCPP_AUTO_CAST(std::forward<_Range>(__range));
+  }
+
+  // [range.drop.overview]: the `span | basic_string_view | iota_view | subrange (StoreSize == false)` case.
+  template <class _Range,
+            convertible_to<range_difference_t<_Range>> _Np,
+            class _RawRange = remove_cvref_t<_Range>,
+            class _Dist     = range_difference_t<_Range>>
+    requires(!__is_empty_view<_RawRange> && random_access_range<_RawRange> && sized_range<_RawRange> &&
+             __is_passthrough_specialization<_RawRange>)
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __rng, _Np&& __n) const
+      noexcept(noexcept(__passthrough_type_t<_RawRange>(
+          ranges::begin(__rng) + std::min<_Dist>(ranges::distance(__rng), std::forward<_Np>(__n)), ranges::end(__rng))))
+          -> decltype(__passthrough_type_t<_RawRange>(
+              // Note: deliberately not forwarding `__rng` to guard against double moves.
+              ranges::begin(__rng) + std::min<_Dist>(ranges::distance(__rng), std::forward<_Np>(__n)),
+              ranges::end(__rng))) {
+    return __passthrough_type_t<_RawRange>(
+        ranges::begin(__rng) + std::min<_Dist>(ranges::distance(__rng), std::forward<_Np>(__n)), ranges::end(__rng));
+  }
+
+  // [range.drop.overview]: the `subrange (StoreSize == true)` case.
+  template <class _Range,
+            convertible_to<range_difference_t<_Range>> _Np,
+            class _RawRange = remove_cvref_t<_Range>,
+            class _Dist     = range_difference_t<_Range>>
+    requires(!__is_empty_view<_RawRange> && random_access_range<_RawRange> && sized_range<_RawRange> &&
+             __is_subrange_specialization_with_store_size<_RawRange>)
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __rng, _Np&& __n) const noexcept(noexcept(
+      _RawRange(ranges::begin(__rng) + std::min<_Dist>(ranges::distance(__rng), std::forward<_Np>(__n)),
+                ranges::end(__rng),
+                std::__to_unsigned_like(ranges::distance(__rng) -
+                                        std::min<_Dist>(ranges::distance(__rng), std::forward<_Np>(__n))))))
+      -> decltype(_RawRange(
+          // Note: deliberately not forwarding `__rng` to guard against double moves.
+          ranges::begin(__rng) + std::min<_Dist>(ranges::distance(__rng), std::forward<_Np>(__n)),
+          ranges::end(__rng),
+          std::__to_unsigned_like(ranges::distance(__rng) -
+                                  std::min<_Dist>(ranges::distance(__rng), std::forward<_Np>(__n))))) {
+    // Introducing local variables avoids calculating `min` and `distance` twice (at the cost of diverging from the
+    // expression used in the `noexcept` clause and the return statement).
+    auto __dist    = ranges::distance(__rng);
+    auto __clamped = std::min<_Dist>(__dist, std::forward<_Np>(__n));
+    return _RawRange(ranges::begin(__rng) + __clamped, ranges::end(__rng), std::__to_unsigned_like(__dist - __clamped));
+  }
+  // clang-format off
+#if _LIBCPP_STD_VER >= 23
+  // [range.drop.overview]: the `repeat_view` "_RawRange models sized_range" case.
+  template <class _Range,
+            convertible_to<range_difference_t<_Range>> _Np,
+            class _RawRange = remove_cvref_t<_Range>,
+            class _Dist     = range_difference_t<_Range>>
+    requires (__is_repeat_specialization<_RawRange> && sized_range<_RawRange>)
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range, _Np&& __n) const
+    noexcept(noexcept(views::repeat(*__range.__value_, ranges::distance(__range) - std::min<_Dist>(ranges::distance(__range), std::forward<_Np>(__n)))))
+    -> decltype(      views::repeat(*__range.__value_, ranges::distance(__range) - std::min<_Dist>(ranges::distance(__range), std::forward<_Np>(__n))))
+    { return          views::repeat(*__range.__value_, ranges::distance(__range) - std::min<_Dist>(ranges::distance(__range), std::forward<_Np>(__n))); }
+
+  // [range.drop.overview]: the `repeat_view` "otherwise" case.
+  template <class _Range,
+            convertible_to<range_difference_t<_Range>> _Np,
+            class _RawRange = remove_cvref_t<_Range>,
+            class _Dist     = range_difference_t<_Range>>
+    requires (__is_repeat_specialization<_RawRange> && !sized_range<_RawRange>)
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI
+  constexpr auto operator()(_Range&& __range, _Np&&) const
+    noexcept(noexcept(_LIBCPP_AUTO_CAST(std::forward<_Range>(__range))))
+    -> decltype(      _LIBCPP_AUTO_CAST(std::forward<_Range>(__range)))
+    { return          _LIBCPP_AUTO_CAST(std::forward<_Range>(__range)); }
+#endif
+  // clang-format on
+
+  // [range.drop.overview]: the "otherwise" case.
+  template <class _Range, convertible_to<range_difference_t<_Range>> _Np, class _RawRange = remove_cvref_t<_Range>>
+  // Note: without specifically excluding the other cases, GCC sees this overload as ambiguous with the other
+  // overloads.
+    requires(!(__is_empty_view<_RawRange> ||
+#  if _LIBCPP_STD_VER >= 23
+               __is_repeat_specialization<_RawRange> ||
+#  endif
+               (__is_subrange_specialization_with_store_size<_RawRange> && sized_range<_RawRange> &&
+                random_access_range<_RawRange>) ||
+               (__is_passthrough_specialization<_RawRange> && sized_range<_RawRange> &&
+                random_access_range<_RawRange>)))
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range, _Np&& __n) const
+      noexcept(noexcept(drop_view(std::forward<_Range>(__range), std::forward<_Np>(__n))))
+          -> decltype(drop_view(std::forward<_Range>(__range), std::forward<_Np>(__n))) {
+    return drop_view(std::forward<_Range>(__range), std::forward<_Np>(__n));
+  }
+
+  template <class _Np>
+    requires constructible_from<decay_t<_Np>, _Np>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Np&& __n) const
+      noexcept(is_nothrow_constructible_v<decay_t<_Np>, _Np>) {
+    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Np>(__n)));
+  }
+};
+
+} // namespace __drop
+
+inline namespace __cpo {
+inline constexpr auto drop = __drop::__fn{};
+} // namespace __cpo
+} // namespace views
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_DROP_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/drop_while_view.h b/libcxx/include/__cxx03/__ranges/drop_while_view.h
new file mode 100644
index 000000000000..92f48bd0ecfb
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/drop_while_view.h
@@ -0,0 +1,136 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_DROP_WHILE_VIEW_H
+#define _LIBCPP___RANGES_DROP_WHILE_VIEW_H
+
+#include <__algorithm/ranges_find_if_not.h>
+#include <__assert>
+#include <__concepts/constructible.h>
+#include <__config>
+#include <__functional/bind_back.h>
+#include <__functional/reference_wrapper.h>
+#include <__iterator/concepts.h>
+#include <__ranges/access.h>
+#include <__ranges/all.h>
+#include <__ranges/concepts.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/movable_box.h>
+#include <__ranges/non_propagating_cache.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_object.h>
+#include <__utility/forward.h>
+#include <__utility/in_place.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+template <view _View, class _Pred>
+  requires input_range<_View> && is_object_v<_Pred> && indirect_unary_predicate<const _Pred, iterator_t<_View>>
+class _LIBCPP_ABI_LLVM18_NO_UNIQUE_ADDRESS drop_while_view : public view_interface<drop_while_view<_View, _Pred>> {
+public:
+  _LIBCPP_HIDE_FROM_ABI drop_while_view()
+    requires default_initializable<_View> && default_initializable<_Pred>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23 drop_while_view(_View __base, _Pred __pred)
+      : __base_(std::move(__base)), __pred_(std::in_place, std::move(__pred)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
+    requires copy_constructible<_View>
+  {
+    return __base_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Pred& pred() const { return *__pred_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() {
+    // Note: this duplicates a check in `optional` but provides a better error message.
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __pred_.__has_value(),
+        "drop_while_view needs to have a non-empty predicate before calling begin() -- did a previous "
+        "assignment to this drop_while_view fail?");
+    if constexpr (_UseCache) {
+      if (!__cached_begin_.__has_value()) {
+        __cached_begin_.__emplace(ranges::find_if_not(__base_, std::cref(*__pred_)));
+      }
+      return *__cached_begin_;
+    } else {
+      return ranges::find_if_not(__base_, std::cref(*__pred_));
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() { return ranges::end(__base_); }
+
+private:
+  _LIBCPP_NO_UNIQUE_ADDRESS _View __base_ = _View();
+  _LIBCPP_NO_UNIQUE_ADDRESS __movable_box<_Pred> __pred_;
+
+  static constexpr bool _UseCache = forward_range<_View>;
+  using _Cache                    = _If<_UseCache, __non_propagating_cache<iterator_t<_View>>, __empty_cache>;
+  _LIBCPP_NO_UNIQUE_ADDRESS _Cache __cached_begin_ = _Cache();
+};
+
+template <class _View, class _Pred>
+inline constexpr bool enable_borrowed_range<drop_while_view<_View, _Pred>> = enable_borrowed_range<_View>;
+
+template <class _Range, class _Pred>
+drop_while_view(_Range&&, _Pred) -> drop_while_view<views::all_t<_Range>, _Pred>;
+
+namespace views {
+namespace __drop_while {
+
+struct __fn {
+  template <class _Range, class _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range, _Pred&& __pred) const
+      noexcept(noexcept(/**/ drop_while_view(std::forward<_Range>(__range), std::forward<_Pred>(__pred))))
+          -> decltype(/*--*/ drop_while_view(std::forward<_Range>(__range), std::forward<_Pred>(__pred))) {
+    return /*-------------*/ drop_while_view(std::forward<_Range>(__range), std::forward<_Pred>(__pred));
+  }
+
+  template <class _Pred>
+    requires constructible_from<decay_t<_Pred>, _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Pred&& __pred) const
+      noexcept(is_nothrow_constructible_v<decay_t<_Pred>, _Pred>) {
+    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Pred>(__pred)));
+  }
+};
+
+} // namespace __drop_while
+
+inline namespace __cpo {
+inline constexpr auto drop_while = __drop_while::__fn{};
+} // namespace __cpo
+} // namespace views
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_DROP_WHILE_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/elements_view.h b/libcxx/include/__cxx03/__ranges/elements_view.h
new file mode 100644
index 000000000000..f159f53dc0a8
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/elements_view.h
@@ -0,0 +1,418 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_ELEMENTS_VIEW_H
+#define _LIBCPP___RANGES_ELEMENTS_VIEW_H
+
+#include <__compare/three_way_comparable.h>
+#include <__concepts/constructible.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/derived_from.h>
+#include <__concepts/equality_comparable.h>
+#include <__config>
+#include <__fwd/complex.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/all.h>
+#include <__ranges/concepts.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/size.h>
+#include <__ranges/view_interface.h>
+#include <__tuple/tuple_element.h>
+#include <__tuple/tuple_like.h>
+#include <__tuple/tuple_size.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/maybe_const.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+template <class _Tp, size_t _Np>
+concept __has_tuple_element = __tuple_like<_Tp> && _Np < tuple_size<_Tp>::value;
+
+template <class _Tp, size_t _Np>
+concept __returnable_element = is_reference_v<_Tp> || move_constructible<tuple_element_t<_Np, _Tp>>;
+
+template <input_range _View, size_t _Np>
+  requires view<_View> && __has_tuple_element<range_value_t<_View>, _Np> &&
+           __has_tuple_element<remove_reference_t<range_reference_t<_View>>, _Np> &&
+           __returnable_element<range_reference_t<_View>, _Np>
+class elements_view : public view_interface<elements_view<_View, _Np>> {
+private:
+  template <bool>
+  class __iterator;
+
+  template <bool>
+  class __sentinel;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI elements_view()
+    requires default_initializable<_View>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit elements_view(_View __base) : __base_(std::move(__base)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
+    requires copy_constructible<_View>
+  {
+    return __base_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin()
+    requires(!__simple_view<_View>)
+  {
+    return __iterator</*_Const=*/false>(ranges::begin(__base_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
+    requires range<const _View>
+  {
+    return __iterator</*_Const=*/true>(ranges::begin(__base_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end()
+    requires(!__simple_view<_View> && !common_range<_View>)
+  {
+    return __sentinel</*_Const=*/false>{ranges::end(__base_)};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end()
+    requires(!__simple_view<_View> && common_range<_View>)
+  {
+    return __iterator</*_Const=*/false>{ranges::end(__base_)};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
+    requires range<const _View>
+  {
+    return __sentinel</*_Const=*/true>{ranges::end(__base_)};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
+    requires common_range<const _View>
+  {
+    return __iterator</*_Const=*/true>{ranges::end(__base_)};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size()
+    requires sized_range<_View>
+  {
+    return ranges::size(__base_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
+    requires sized_range<const _View>
+  {
+    return ranges::size(__base_);
+  }
+
+private:
+  _LIBCPP_NO_UNIQUE_ADDRESS _View __base_ = _View();
+};
+
+template <class, size_t>
+struct __elements_view_iterator_category_base {};
+
+template <forward_range _Base, size_t _Np>
+struct __elements_view_iterator_category_base<_Base, _Np> {
+  static consteval auto __get_iterator_category() {
+    using _Result = decltype(std::get<_Np>(*std::declval<iterator_t<_Base>>()));
+    using _Cat    = typename iterator_traits<iterator_t<_Base>>::iterator_category;
+
+    if constexpr (!is_lvalue_reference_v<_Result>) {
+      return input_iterator_tag{};
+    } else if constexpr (derived_from<_Cat, random_access_iterator_tag>) {
+      return random_access_iterator_tag{};
+    } else {
+      return _Cat{};
+    }
+  }
+
+  using iterator_category = decltype(__get_iterator_category());
+};
+
+template <input_range _View, size_t _Np>
+  requires view<_View> && __has_tuple_element<range_value_t<_View>, _Np> &&
+           __has_tuple_element<remove_reference_t<range_reference_t<_View>>, _Np> &&
+           __returnable_element<range_reference_t<_View>, _Np>
+template <bool _Const>
+class elements_view<_View, _Np>::__iterator
+    : public __elements_view_iterator_category_base<__maybe_const<_Const, _View>, _Np> {
+  template <bool>
+  friend class __iterator;
+
+  template <bool>
+  friend class __sentinel;
+
+  using _Base = __maybe_const<_Const, _View>;
+
+  iterator_t<_Base> __current_ = iterator_t<_Base>();
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr decltype(auto) __get_element(const iterator_t<_Base>& __i) {
+    if constexpr (is_reference_v<range_reference_t<_Base>>) {
+      return std::get<_Np>(*__i);
+    } else {
+      using _Element = remove_cv_t<tuple_element_t<_Np, range_reference_t<_Base>>>;
+      return static_cast<_Element>(std::get<_Np>(*__i));
+    }
+  }
+
+  static consteval auto __get_iterator_concept() {
+    if constexpr (random_access_range<_Base>) {
+      return random_access_iterator_tag{};
+    } else if constexpr (bidirectional_range<_Base>) {
+      return bidirectional_iterator_tag{};
+    } else if constexpr (forward_range<_Base>) {
+      return forward_iterator_tag{};
+    } else {
+      return input_iterator_tag{};
+    }
+  }
+
+public:
+  using iterator_concept = decltype(__get_iterator_concept());
+  using value_type       = remove_cvref_t<tuple_element_t<_Np, range_value_t<_Base>>>;
+  using difference_type  = range_difference_t<_Base>;
+
+  _LIBCPP_HIDE_FROM_ABI __iterator()
+    requires default_initializable<iterator_t<_Base>>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __iterator(iterator_t<_Base> __current) : __current_(std::move(__current)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator(__iterator<!_Const> __i)
+    requires _Const && convertible_to<iterator_t<_View>, iterator_t<_Base>>
+      : __current_(std::move(__i.__current_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const iterator_t<_Base>& base() const& noexcept { return __current_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_Base> base() && { return std::move(__current_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() const { return __get_element(__current_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() {
+    ++__current_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator++(int) { ++__current_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator++(int)
+    requires forward_range<_Base>
+  {
+    auto __temp = *this;
+    ++__current_;
+    return __temp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator--()
+    requires bidirectional_range<_Base>
+  {
+    --__current_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator--(int)
+    requires bidirectional_range<_Base>
+  {
+    auto __temp = *this;
+    --__current_;
+    return __temp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator+=(difference_type __n)
+    requires random_access_range<_Base>
+  {
+    __current_ += __n;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator-=(difference_type __n)
+    requires random_access_range<_Base>
+  {
+    __current_ -= __n;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator[](difference_type __n) const
+    requires random_access_range<_Base>
+  {
+    return __get_element(__current_ + __n);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __iterator& __y)
+    requires equality_comparable<iterator_t<_Base>>
+  {
+    return __x.__current_ == __y.__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(const __iterator& __x, const __iterator& __y)
+    requires random_access_range<_Base>
+  {
+    return __x.__current_ < __y.__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(const __iterator& __x, const __iterator& __y)
+    requires random_access_range<_Base>
+  {
+    return __y < __x;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(const __iterator& __x, const __iterator& __y)
+    requires random_access_range<_Base>
+  {
+    return !(__y < __x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(const __iterator& __x, const __iterator& __y)
+    requires random_access_range<_Base>
+  {
+    return !(__x < __y);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr auto operator<=>(const __iterator& __x, const __iterator& __y)
+    requires random_access_range<_Base> && three_way_comparable<iterator_t<_Base>>
+  {
+    return __x.__current_ <=> __y.__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(const __iterator& __x, difference_type __y)
+    requires random_access_range<_Base>
+  {
+    return __iterator{__x} += __y;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(difference_type __x, const __iterator& __y)
+    requires random_access_range<_Base>
+  {
+    return __y + __x;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator-(const __iterator& __x, difference_type __y)
+    requires random_access_range<_Base>
+  {
+    return __iterator{__x} -= __y;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr difference_type operator-(const __iterator& __x, const __iterator& __y)
+    requires sized_sentinel_for<iterator_t<_Base>, iterator_t<_Base>>
+  {
+    return __x.__current_ - __y.__current_;
+  }
+};
+
+template <input_range _View, size_t _Np>
+  requires view<_View> && __has_tuple_element<range_value_t<_View>, _Np> &&
+           __has_tuple_element<remove_reference_t<range_reference_t<_View>>, _Np> &&
+           __returnable_element<range_reference_t<_View>, _Np>
+template <bool _Const>
+class elements_view<_View, _Np>::__sentinel {
+private:
+  using _Base                                        = __maybe_const<_Const, _View>;
+  _LIBCPP_NO_UNIQUE_ADDRESS sentinel_t<_Base> __end_ = sentinel_t<_Base>();
+
+  template <bool>
+  friend class __sentinel;
+
+  template <bool _AnyConst>
+  _LIBCPP_HIDE_FROM_ABI static constexpr decltype(auto) __get_current(const __iterator<_AnyConst>& __iter) {
+    return (__iter.__current_);
+  }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __sentinel() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __sentinel(sentinel_t<_Base> __end) : __end_(std::move(__end)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __sentinel(__sentinel<!_Const> __other)
+    requires _Const && convertible_to<sentinel_t<_View>, sentinel_t<_Base>>
+      : __end_(std::move(__other.__end_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr sentinel_t<_Base> base() const { return __end_; }
+
+  template <bool _OtherConst>
+    requires sentinel_for<sentinel_t<_Base>, iterator_t<__maybe_const<_OtherConst, _View>>>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator<_OtherConst>& __x, const __sentinel& __y) {
+    return __get_current(__x) == __y.__end_;
+  }
+
+  template <bool _OtherConst>
+    requires sized_sentinel_for<sentinel_t<_Base>, iterator_t<__maybe_const<_OtherConst, _View>>>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr range_difference_t<__maybe_const<_OtherConst, _View>>
+  operator-(const __iterator<_OtherConst>& __x, const __sentinel& __y) {
+    return __get_current(__x) - __y.__end_;
+  }
+
+  template <bool _OtherConst>
+    requires sized_sentinel_for<sentinel_t<_Base>, iterator_t<__maybe_const<_OtherConst, _View>>>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr range_difference_t<__maybe_const<_OtherConst, _View>>
+  operator-(const __sentinel& __x, const __iterator<_OtherConst>& __y) {
+    return __x.__end_ - __get_current(__y);
+  }
+};
+
+template <class _Tp, size_t _Np>
+inline constexpr bool enable_borrowed_range<elements_view<_Tp, _Np>> = enable_borrowed_range<_Tp>;
+
+template <class _Tp>
+using keys_view = elements_view<_Tp, 0>;
+template <class _Tp>
+using values_view = elements_view<_Tp, 1>;
+
+namespace views {
+namespace __elements {
+
+template <size_t _Np>
+struct __fn : __range_adaptor_closure<__fn<_Np>> {
+  template <class _Range>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range) const
+      /**/ noexcept(noexcept(elements_view<all_t<_Range&&>, _Np>(std::forward<_Range>(__range))))
+      /*------*/ -> decltype(elements_view<all_t<_Range&&>, _Np>(std::forward<_Range>(__range))) {
+    /*-------------*/ return elements_view<all_t<_Range&&>, _Np>(std::forward<_Range>(__range));
+  }
+};
+} // namespace __elements
+
+inline namespace __cpo {
+template <size_t _Np>
+inline constexpr auto elements = __elements::__fn<_Np>{};
+inline constexpr auto keys     = elements<0>;
+inline constexpr auto values   = elements<1>;
+} // namespace __cpo
+} // namespace views
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_ELEMENTS_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/empty.h b/libcxx/include/__cxx03/__ranges/empty.h
new file mode 100644
index 000000000000..5c1004042aba
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/empty.h
@@ -0,0 +1,71 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_EMPTY_H
+#define _LIBCPP___RANGES_EMPTY_H
+
+#include <__concepts/class_or_enum.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__ranges/access.h>
+#include <__ranges/size.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [range.prim.empty]
+
+namespace ranges {
+namespace __empty {
+template <class _Tp>
+concept __member_empty = requires(_Tp&& __t) { bool(__t.empty()); };
+
+template <class _Tp>
+concept __can_invoke_size = !__member_empty<_Tp> && requires(_Tp&& __t) { ranges::size(__t); };
+
+template <class _Tp>
+concept __can_compare_begin_end = !__member_empty<_Tp> && !__can_invoke_size<_Tp> && requires(_Tp&& __t) {
+  bool(ranges::begin(__t) == ranges::end(__t));
+  { ranges::begin(__t) } -> forward_iterator;
+};
+
+struct __fn {
+  template <__member_empty _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t) const noexcept(noexcept(bool(__t.empty()))) {
+    return bool(__t.empty());
+  }
+
+  template <__can_invoke_size _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t) const noexcept(noexcept(ranges::size(__t))) {
+    return ranges::size(__t) == 0;
+  }
+
+  template <__can_compare_begin_end _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t) const
+      noexcept(noexcept(bool(ranges::begin(__t) == ranges::end(__t)))) {
+    return ranges::begin(__t) == ranges::end(__t);
+  }
+};
+} // namespace __empty
+
+inline namespace __cpo {
+inline constexpr auto empty = __empty::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_EMPTY_H
diff --git a/libcxx/include/__cxx03/__ranges/empty_view.h b/libcxx/include/__cxx03/__ranges/empty_view.h
new file mode 100644
index 000000000000..6c04b0200c35
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/empty_view.h
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_EMPTY_VIEW_H
+#define _LIBCPP___RANGES_EMPTY_VIEW_H
+
+#include <__config>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/is_object.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+template <class _Tp>
+  requires is_object_v<_Tp>
+class empty_view : public view_interface<empty_view<_Tp>> {
+public:
+  _LIBCPP_HIDE_FROM_ABI static constexpr _Tp* begin() noexcept { return nullptr; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr _Tp* end() noexcept { return nullptr; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr _Tp* data() noexcept { return nullptr; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr size_t size() noexcept { return 0; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool empty() noexcept { return true; }
+};
+
+template <class _Tp>
+inline constexpr bool enable_borrowed_range<empty_view<_Tp>> = true;
+
+namespace views {
+
+template <class _Tp>
+inline constexpr empty_view<_Tp> empty{};
+
+} // namespace views
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_EMPTY_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/enable_borrowed_range.h b/libcxx/include/__cxx03/__ranges/enable_borrowed_range.h
new file mode 100644
index 000000000000..1d068335e20a
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/enable_borrowed_range.h
@@ -0,0 +1,40 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_ENABLE_BORROWED_RANGE_H
+#define _LIBCPP___RANGES_ENABLE_BORROWED_RANGE_H
+
+// These customization variables are used in <span> and <string_view>. The
+// separate header is used to avoid including the entire <ranges> header in
+// <span> and <string_view>.
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+// [range.range], ranges
+
+template <class>
+inline constexpr bool enable_borrowed_range = false;
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_ENABLE_BORROWED_RANGE_H
diff --git a/libcxx/include/__cxx03/__ranges/enable_view.h b/libcxx/include/__cxx03/__ranges/enable_view.h
new file mode 100644
index 000000000000..f570926eb67c
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/enable_view.h
@@ -0,0 +1,51 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_ENABLE_VIEW_H
+#define _LIBCPP___RANGES_ENABLE_VIEW_H
+
+#include <__concepts/derived_from.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__type_traits/is_class.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/remove_cv.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+struct view_base {};
+
+template <class _Derived>
+  requires is_class_v<_Derived> && same_as<_Derived, remove_cv_t<_Derived>>
+class view_interface;
+
+template <class _Op, class _Yp>
+  requires is_convertible_v<_Op*, view_interface<_Yp>*>
+void __is_derived_from_view_interface(const _Op*, const view_interface<_Yp>*);
+
+template <class _Tp>
+inline constexpr bool enable_view = derived_from<_Tp, view_base> || requires {
+  ranges::__is_derived_from_view_interface((_Tp*)nullptr, (_Tp*)nullptr);
+};
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_ENABLE_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/filter_view.h b/libcxx/include/__cxx03/__ranges/filter_view.h
new file mode 100644
index 000000000000..5b938dd4c16e
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/filter_view.h
@@ -0,0 +1,260 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_FILTER_VIEW_H
+#define _LIBCPP___RANGES_FILTER_VIEW_H
+
+#include <__algorithm/ranges_find_if.h>
+#include <__assert>
+#include <__concepts/constructible.h>
+#include <__concepts/copyable.h>
+#include <__concepts/derived_from.h>
+#include <__concepts/equality_comparable.h>
+#include <__config>
+#include <__functional/bind_back.h>
+#include <__functional/invoke.h>
+#include <__functional/reference_wrapper.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/iter_swap.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__ranges/access.h>
+#include <__ranges/all.h>
+#include <__ranges/concepts.h>
+#include <__ranges/movable_box.h>
+#include <__ranges/non_propagating_cache.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_object.h>
+#include <__utility/forward.h>
+#include <__utility/in_place.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+template <input_range _View, indirect_unary_predicate<iterator_t<_View>> _Pred>
+  requires view<_View> && is_object_v<_Pred>
+class _LIBCPP_ABI_LLVM18_NO_UNIQUE_ADDRESS filter_view : public view_interface<filter_view<_View, _Pred>> {
+  _LIBCPP_NO_UNIQUE_ADDRESS _View __base_ = _View();
+  _LIBCPP_NO_UNIQUE_ADDRESS __movable_box<_Pred> __pred_;
+
+  // We cache the result of begin() to allow providing an amortized O(1) begin() whenever
+  // the underlying range is at least a forward_range.
+  static constexpr bool _UseCache = forward_range<_View>;
+  using _Cache                    = _If<_UseCache, __non_propagating_cache<iterator_t<_View>>, __empty_cache>;
+  _LIBCPP_NO_UNIQUE_ADDRESS _Cache __cached_begin_ = _Cache();
+
+  class __iterator;
+  class __sentinel;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI filter_view()
+    requires default_initializable<_View> && default_initializable<_Pred>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23 filter_view(_View __base, _Pred __pred)
+      : __base_(std::move(__base)), __pred_(in_place, std::move(__pred)) {}
+
+  template <class _Vp = _View>
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
+    requires copy_constructible<_Vp>
+  {
+    return __base_;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Pred const& pred() const { return *__pred_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() {
+    // Note: this duplicates a check in `optional` but provides a better error message.
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __pred_.__has_value(), "Trying to call begin() on a filter_view that does not have a valid predicate.");
+    if constexpr (_UseCache) {
+      if (!__cached_begin_.__has_value()) {
+        __cached_begin_.__emplace(ranges::find_if(__base_, std::ref(*__pred_)));
+      }
+      return {*this, *__cached_begin_};
+    } else {
+      return {*this, ranges::find_if(__base_, std::ref(*__pred_))};
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() {
+    if constexpr (common_range<_View>)
+      return __iterator{*this, ranges::end(__base_)};
+    else
+      return __sentinel{*this};
+  }
+};
+
+template <class _Range, class _Pred>
+filter_view(_Range&&, _Pred) -> filter_view<views::all_t<_Range>, _Pred>;
+
+template <class _View>
+struct __filter_iterator_category {};
+
+template <forward_range _View>
+struct __filter_iterator_category<_View> {
+  using _Cat = typename iterator_traits<iterator_t<_View>>::iterator_category;
+  using iterator_category =
+      _If<derived_from<_Cat, bidirectional_iterator_tag>,
+          bidirectional_iterator_tag,
+          _If<derived_from<_Cat, forward_iterator_tag>,
+              forward_iterator_tag,
+              /* else */ _Cat >>;
+};
+
+template <input_range _View, indirect_unary_predicate<iterator_t<_View>> _Pred>
+  requires view<_View> && is_object_v<_Pred>
+class filter_view<_View, _Pred>::__iterator : public __filter_iterator_category<_View> {
+public:
+  _LIBCPP_NO_UNIQUE_ADDRESS iterator_t<_View> __current_ = iterator_t<_View>();
+  _LIBCPP_NO_UNIQUE_ADDRESS filter_view* __parent_       = nullptr;
+
+  using iterator_concept =
+      _If<bidirectional_range<_View>,
+          bidirectional_iterator_tag,
+          _If<forward_range<_View>,
+              forward_iterator_tag,
+              /* else */ input_iterator_tag >>;
+  // using iterator_category = inherited;
+  using value_type      = range_value_t<_View>;
+  using difference_type = range_difference_t<_View>;
+
+  _LIBCPP_HIDE_FROM_ABI __iterator()
+    requires default_initializable<iterator_t<_View>>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator(filter_view& __parent, iterator_t<_View> __current)
+      : __current_(std::move(__current)), __parent_(std::addressof(__parent)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_View> const& base() const& noexcept { return __current_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_View> base() && { return std::move(__current_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr range_reference_t<_View> operator*() const { return *__current_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_View> operator->() const
+    requires __has_arrow<iterator_t<_View>> && copyable<iterator_t<_View>>
+  {
+    return __current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() {
+    __current_ =
+        ranges::find_if(std::move(++__current_), ranges::end(__parent_->__base_), std::ref(*__parent_->__pred_));
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator++(int) { ++*this; }
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator++(int)
+    requires forward_range<_View>
+  {
+    auto __tmp = *this;
+    ++*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator--()
+    requires bidirectional_range<_View>
+  {
+    do {
+      --__current_;
+    } while (!std::invoke(*__parent_->__pred_, *__current_));
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator--(int)
+    requires bidirectional_range<_View>
+  {
+    auto __tmp = *this;
+    --*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(__iterator const& __x, __iterator const& __y)
+    requires equality_comparable<iterator_t<_View>>
+  {
+    return __x.__current_ == __y.__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr range_rvalue_reference_t<_View>
+  iter_move(__iterator const& __it) noexcept(noexcept(ranges::iter_move(__it.__current_))) {
+    return ranges::iter_move(__it.__current_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr void
+  iter_swap(__iterator const& __x,
+            __iterator const& __y) noexcept(noexcept(ranges::iter_swap(__x.__current_, __y.__current_)))
+    requires indirectly_swappable<iterator_t<_View>>
+  {
+    return ranges::iter_swap(__x.__current_, __y.__current_);
+  }
+};
+
+template <input_range _View, indirect_unary_predicate<iterator_t<_View>> _Pred>
+  requires view<_View> && is_object_v<_Pred>
+class filter_view<_View, _Pred>::__sentinel {
+public:
+  sentinel_t<_View> __end_ = sentinel_t<_View>();
+
+  _LIBCPP_HIDE_FROM_ABI __sentinel() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __sentinel(filter_view& __parent) : __end_(ranges::end(__parent.__base_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr sentinel_t<_View> base() const { return __end_; }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(__iterator const& __x, __sentinel const& __y) {
+    return __x.__current_ == __y.__end_;
+  }
+};
+
+namespace views {
+namespace __filter {
+struct __fn {
+  template <class _Range, class _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range, _Pred&& __pred) const
+      noexcept(noexcept(filter_view(std::forward<_Range>(__range), std::forward<_Pred>(__pred))))
+          -> decltype(filter_view(std::forward<_Range>(__range), std::forward<_Pred>(__pred))) {
+    return filter_view(std::forward<_Range>(__range), std::forward<_Pred>(__pred));
+  }
+
+  template <class _Pred>
+    requires constructible_from<decay_t<_Pred>, _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Pred&& __pred) const
+      noexcept(is_nothrow_constructible_v<decay_t<_Pred>, _Pred>) {
+    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Pred>(__pred)));
+  }
+};
+} // namespace __filter
+
+inline namespace __cpo {
+inline constexpr auto filter = __filter::__fn{};
+} // namespace __cpo
+} // namespace views
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_FILTER_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/from_range.h b/libcxx/include/__cxx03/__ranges/from_range.h
new file mode 100644
index 000000000000..a6cb9e3d439e
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/from_range.h
@@ -0,0 +1,33 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_FROM_RANGE_H
+#define _LIBCPP___RANGES_FROM_RANGE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+struct from_range_t {
+  explicit from_range_t() = default;
+};
+
+inline constexpr from_range_t from_range{};
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_FROM_RANGE_H
diff --git a/libcxx/include/__cxx03/__ranges/iota_view.h b/libcxx/include/__cxx03/__ranges/iota_view.h
new file mode 100644
index 000000000000..b2fa958a0f56
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/iota_view.h
@@ -0,0 +1,404 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_IOTA_VIEW_H
+#define _LIBCPP___RANGES_IOTA_VIEW_H
+
+#include <__assert>
+#include <__compare/three_way_comparable.h>
+#include <__concepts/arithmetic.h>
+#include <__concepts/constructible.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/copyable.h>
+#include <__concepts/equality_comparable.h>
+#include <__concepts/invocable.h>
+#include <__concepts/same_as.h>
+#include <__concepts/semiregular.h>
+#include <__concepts/totally_ordered.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/unreachable_sentinel.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/movable_box.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/make_unsigned.h>
+#include <__type_traits/type_identity.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+template <class _Int>
+struct __get_wider_signed {
+  consteval static auto __call() {
+    if constexpr (sizeof(_Int) < sizeof(short))
+      return type_identity<short>{};
+    else if constexpr (sizeof(_Int) < sizeof(int))
+      return type_identity<int>{};
+    else if constexpr (sizeof(_Int) < sizeof(long))
+      return type_identity<long>{};
+    else
+      return type_identity<long long>{};
+
+    static_assert(
+        sizeof(_Int) <= sizeof(long long), "Found integer-like type that is bigger than largest integer like type.");
+  }
+
+  using type = typename decltype(__call())::type;
+};
+
+template <class _Start>
+using _IotaDiffT =
+    typename _If< (!integral<_Start> || sizeof(iter_difference_t<_Start>) > sizeof(_Start)),
+                  type_identity<iter_difference_t<_Start>>,
+                  __get_wider_signed<_Start> >::type;
+
+template <class _Iter>
+concept __decrementable = incrementable<_Iter> && requires(_Iter __i) {
+  { --__i } -> same_as<_Iter&>;
+  { __i-- } -> same_as<_Iter>;
+};
+
+template <class _Iter>
+concept __advanceable =
+    __decrementable<_Iter> && totally_ordered<_Iter> &&
+    requires(_Iter __i, const _Iter __j, const _IotaDiffT<_Iter> __n) {
+      { __i += __n } -> same_as<_Iter&>;
+      { __i -= __n } -> same_as<_Iter&>;
+      _Iter(__j + __n);
+      _Iter(__n + __j);
+      _Iter(__j - __n);
+      { __j - __j } -> convertible_to<_IotaDiffT<_Iter>>;
+    };
+
+template <class>
+struct __iota_iterator_category {};
+
+template <incrementable _Tp>
+struct __iota_iterator_category<_Tp> {
+  using iterator_category = input_iterator_tag;
+};
+
+template <weakly_incrementable _Start, semiregular _BoundSentinel = unreachable_sentinel_t>
+  requires __weakly_equality_comparable_with<_Start, _BoundSentinel> && copyable<_Start>
+class iota_view : public view_interface<iota_view<_Start, _BoundSentinel>> {
+  struct __iterator : public __iota_iterator_category<_Start> {
+    friend class iota_view;
+
+    using iterator_concept =
+        _If<__advanceable<_Start>,
+            random_access_iterator_tag,
+            _If<__decrementable<_Start>,
+                bidirectional_iterator_tag,
+                _If<incrementable<_Start>,
+                    forward_iterator_tag,
+                    /*Else*/ input_iterator_tag>>>;
+
+    using value_type      = _Start;
+    using difference_type = _IotaDiffT<_Start>;
+
+    _Start __value_ = _Start();
+
+    _LIBCPP_HIDE_FROM_ABI __iterator()
+      requires default_initializable<_Start>
+    = default;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __iterator(_Start __value) : __value_(std::move(__value)) {}
+
+    _LIBCPP_HIDE_FROM_ABI constexpr _Start operator*() const noexcept(is_nothrow_copy_constructible_v<_Start>) {
+      return __value_;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() {
+      ++__value_;
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr void operator++(int) { ++*this; }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator++(int)
+      requires incrementable<_Start>
+    {
+      auto __tmp = *this;
+      ++*this;
+      return __tmp;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator--()
+      requires __decrementable<_Start>
+    {
+      --__value_;
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator--(int)
+      requires __decrementable<_Start>
+    {
+      auto __tmp = *this;
+      --*this;
+      return __tmp;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator+=(difference_type __n)
+      requires __advanceable<_Start>
+    {
+      if constexpr (__integer_like<_Start> && !__signed_integer_like<_Start>) {
+        if (__n >= difference_type(0)) {
+          __value_ += static_cast<_Start>(__n);
+        } else {
+          __value_ -= static_cast<_Start>(-__n);
+        }
+      } else {
+        __value_ += __n;
+      }
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator-=(difference_type __n)
+      requires __advanceable<_Start>
+    {
+      if constexpr (__integer_like<_Start> && !__signed_integer_like<_Start>) {
+        if (__n >= difference_type(0)) {
+          __value_ -= static_cast<_Start>(__n);
+        } else {
+          __value_ += static_cast<_Start>(-__n);
+        }
+      } else {
+        __value_ -= __n;
+      }
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr _Start operator[](difference_type __n) const
+      requires __advanceable<_Start>
+    {
+      return _Start(__value_ + __n);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __iterator& __y)
+      requires equality_comparable<_Start>
+    {
+      return __x.__value_ == __y.__value_;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(const __iterator& __x, const __iterator& __y)
+      requires totally_ordered<_Start>
+    {
+      return __x.__value_ < __y.__value_;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(const __iterator& __x, const __iterator& __y)
+      requires totally_ordered<_Start>
+    {
+      return __y < __x;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(const __iterator& __x, const __iterator& __y)
+      requires totally_ordered<_Start>
+    {
+      return !(__y < __x);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(const __iterator& __x, const __iterator& __y)
+      requires totally_ordered<_Start>
+    {
+      return !(__x < __y);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr auto operator<=>(const __iterator& __x, const __iterator& __y)
+      requires totally_ordered<_Start> && three_way_comparable<_Start>
+    {
+      return __x.__value_ <=> __y.__value_;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(__iterator __i, difference_type __n)
+      requires __advanceable<_Start>
+    {
+      __i += __n;
+      return __i;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(difference_type __n, __iterator __i)
+      requires __advanceable<_Start>
+    {
+      return __i + __n;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator-(__iterator __i, difference_type __n)
+      requires __advanceable<_Start>
+    {
+      __i -= __n;
+      return __i;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr difference_type operator-(const __iterator& __x, const __iterator& __y)
+      requires __advanceable<_Start>
+    {
+      if constexpr (__integer_like<_Start>) {
+        if constexpr (__signed_integer_like<_Start>) {
+          return difference_type(difference_type(__x.__value_) - difference_type(__y.__value_));
+        }
+        if (__y.__value_ > __x.__value_) {
+          return difference_type(-difference_type(__y.__value_ - __x.__value_));
+        }
+        return difference_type(__x.__value_ - __y.__value_);
+      }
+      return __x.__value_ - __y.__value_;
+    }
+  };
+
+  struct __sentinel {
+    friend class iota_view;
+
+  private:
+    _BoundSentinel __bound_sentinel_ = _BoundSentinel();
+
+  public:
+    _LIBCPP_HIDE_FROM_ABI __sentinel() = default;
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __sentinel(_BoundSentinel __bound_sentinel)
+        : __bound_sentinel_(std::move(__bound_sentinel)) {}
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __sentinel& __y) {
+      return __x.__value_ == __y.__bound_sentinel_;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr iter_difference_t<_Start>
+    operator-(const __iterator& __x, const __sentinel& __y)
+      requires sized_sentinel_for<_BoundSentinel, _Start>
+    {
+      return __x.__value_ - __y.__bound_sentinel_;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr iter_difference_t<_Start>
+    operator-(const __sentinel& __x, const __iterator& __y)
+      requires sized_sentinel_for<_BoundSentinel, _Start>
+    {
+      return -(__y - __x);
+    }
+  };
+
+  _Start __value_                  = _Start();
+  _BoundSentinel __bound_sentinel_ = _BoundSentinel();
+
+public:
+  _LIBCPP_HIDE_FROM_ABI iota_view()
+    requires default_initializable<_Start>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit iota_view(_Start __value) : __value_(std::move(__value)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23
+  iota_view(type_identity_t<_Start> __value, type_identity_t<_BoundSentinel> __bound_sentinel)
+      : __value_(std::move(__value)), __bound_sentinel_(std::move(__bound_sentinel)) {
+    // Validate the precondition if possible.
+    if constexpr (totally_ordered_with<_Start, _BoundSentinel>) {
+      _LIBCPP_ASSERT_VALID_INPUT_RANGE(
+          bool(__value_ <= __bound_sentinel_), "iota_view: bound must be reachable from value");
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23 iota_view(__iterator __first, __iterator __last)
+    requires same_as<_Start, _BoundSentinel>
+      : iota_view(std::move(__first.__value_), std::move(__last.__value_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23 iota_view(__iterator __first, _BoundSentinel __last)
+    requires same_as<_BoundSentinel, unreachable_sentinel_t>
+      : iota_view(std::move(__first.__value_), std::move(__last)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23 iota_view(__iterator __first, __sentinel __last)
+    requires(!same_as<_Start, _BoundSentinel> && !same_as<_BoundSentinel, unreachable_sentinel_t>)
+      : iota_view(std::move(__first.__value_), std::move(__last.__bound_sentinel_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() const { return __iterator{__value_}; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() const {
+    if constexpr (same_as<_BoundSentinel, unreachable_sentinel_t>)
+      return unreachable_sentinel;
+    else
+      return __sentinel{__bound_sentinel_};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator end() const
+    requires same_as<_Start, _BoundSentinel>
+  {
+    return __iterator{__bound_sentinel_};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool empty() const { return __value_ == __bound_sentinel_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
+    requires(same_as<_Start, _BoundSentinel> && __advanceable<_Start>) ||
+            (integral<_Start> && integral<_BoundSentinel>) || sized_sentinel_for<_BoundSentinel, _Start>
+  {
+    if constexpr (__integer_like<_Start> && __integer_like<_BoundSentinel>) {
+      return (__value_ < 0)
+               ? ((__bound_sentinel_ < 0)
+                      ? std::__to_unsigned_like(-__value_) - std::__to_unsigned_like(-__bound_sentinel_)
+                      : std::__to_unsigned_like(__bound_sentinel_) + std::__to_unsigned_like(-__value_))
+               : std::__to_unsigned_like(__bound_sentinel_) - std::__to_unsigned_like(__value_);
+    } else {
+      return std::__to_unsigned_like(__bound_sentinel_ - __value_);
+    }
+  }
+};
+
+template <class _Start, class _BoundSentinel>
+  requires(!__integer_like<_Start> || !__integer_like<_BoundSentinel> ||
+           (__signed_integer_like<_Start> == __signed_integer_like<_BoundSentinel>))
+iota_view(_Start, _BoundSentinel) -> iota_view<_Start, _BoundSentinel>;
+
+template <class _Start, class _BoundSentinel>
+inline constexpr bool enable_borrowed_range<iota_view<_Start, _BoundSentinel>> = true;
+
+namespace views {
+namespace __iota {
+struct __fn {
+  template <class _Start>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Start&& __start) const
+      noexcept(noexcept(ranges::iota_view(std::forward<_Start>(__start))))
+          -> decltype(ranges::iota_view(std::forward<_Start>(__start))) {
+    return ranges::iota_view(std::forward<_Start>(__start));
+  }
+
+  template <class _Start, class _BoundSentinel>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Start&& __start, _BoundSentinel&& __bound_sentinel) const noexcept(
+      noexcept(ranges::iota_view(std::forward<_Start>(__start), std::forward<_BoundSentinel>(__bound_sentinel))))
+      -> decltype(ranges::iota_view(std::forward<_Start>(__start), std::forward<_BoundSentinel>(__bound_sentinel))) {
+    return ranges::iota_view(std::forward<_Start>(__start), std::forward<_BoundSentinel>(__bound_sentinel));
+  }
+};
+} // namespace __iota
+
+inline namespace __cpo {
+inline constexpr auto iota = __iota::__fn{};
+} // namespace __cpo
+} // namespace views
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_IOTA_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/istream_view.h b/libcxx/include/__cxx03/__ranges/istream_view.h
new file mode 100644
index 000000000000..cd7096d35c2c
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/istream_view.h
@@ -0,0 +1,141 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_ISTREAM_VIEW_H
+#define _LIBCPP___RANGES_ISTREAM_VIEW_H
+
+#include <__concepts/constructible.h>
+#include <__concepts/derived_from.h>
+#include <__concepts/movable.h>
+#include <__config>
+#include <__fwd/istream.h>
+#include <__fwd/string.h>
+#include <__iterator/default_sentinel.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/forward.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace ranges {
+
+template <class _Val, class _CharT, class _Traits>
+concept __stream_extractable = requires(basic_istream<_CharT, _Traits>& __is, _Val& __t) { __is >> __t; };
+
+template <movable _Val, class _CharT, class _Traits = char_traits<_CharT>>
+  requires default_initializable<_Val> && __stream_extractable<_Val, _CharT, _Traits>
+class basic_istream_view : public view_interface<basic_istream_view<_Val, _CharT, _Traits>> {
+  class __iterator;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit basic_istream_view(basic_istream<_CharT, _Traits>& __stream)
+      : __stream_(std::addressof(__stream)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() {
+    *__stream_ >> __value_;
+    return __iterator{*this};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr default_sentinel_t end() const noexcept { return default_sentinel; }
+
+private:
+  basic_istream<_CharT, _Traits>* __stream_;
+  _LIBCPP_NO_UNIQUE_ADDRESS _Val __value_ = _Val();
+};
+
+template <movable _Val, class _CharT, class _Traits>
+  requires default_initializable<_Val> && __stream_extractable<_Val, _CharT, _Traits>
+class basic_istream_view<_Val, _CharT, _Traits>::__iterator {
+public:
+  using iterator_concept = input_iterator_tag;
+  using difference_type  = ptrdiff_t;
+  using value_type       = _Val;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __iterator(basic_istream_view<_Val, _CharT, _Traits>& __parent) noexcept
+      : __parent_(std::addressof(__parent)) {}
+
+  __iterator(const __iterator&)                  = delete;
+  _LIBCPP_HIDE_FROM_ABI __iterator(__iterator&&) = default;
+
+  __iterator& operator=(const __iterator&)                  = delete;
+  _LIBCPP_HIDE_FROM_ABI __iterator& operator=(__iterator&&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI __iterator& operator++() {
+    *__parent_->__stream_ >> __parent_->__value_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void operator++(int) { ++*this; }
+
+  _LIBCPP_HIDE_FROM_ABI _Val& operator*() const { return __parent_->__value_; }
+
+  _LIBCPP_HIDE_FROM_ABI friend bool operator==(const __iterator& __x, default_sentinel_t) {
+    return !*__x.__get_parent_stream();
+  }
+
+private:
+  basic_istream_view<_Val, _CharT, _Traits>* __parent_;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr basic_istream<_CharT, _Traits>* __get_parent_stream() const {
+    return __parent_->__stream_;
+  }
+};
+
+template <class _Val>
+using istream_view = basic_istream_view<_Val, char>;
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class _Val>
+using wistream_view = basic_istream_view<_Val, wchar_t>;
+#  endif
+
+namespace views {
+namespace __istream {
+
+// clang-format off
+template <class _Tp>
+struct __fn {
+  template <class _Up, class _UnCVRef = remove_cvref_t<_Up>>
+    requires derived_from<_UnCVRef, basic_istream<typename _UnCVRef::char_type,
+                                                  typename _UnCVRef::traits_type>>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Up&& __u) const
+    noexcept(noexcept(basic_istream_view<_Tp, typename _UnCVRef::char_type,
+                                              typename _UnCVRef::traits_type>(std::forward<_Up>(__u))))
+    -> decltype(      basic_istream_view<_Tp, typename _UnCVRef::char_type,
+                                              typename _UnCVRef::traits_type>(std::forward<_Up>(__u)))
+    {   return        basic_istream_view<_Tp, typename _UnCVRef::char_type,
+                                              typename _UnCVRef::traits_type>(std::forward<_Up>(__u));
+    }
+};
+// clang-format on
+
+} // namespace __istream
+
+inline namespace __cpo {
+template <class _Tp>
+inline constexpr auto istream = __istream::__fn<_Tp>{};
+} // namespace __cpo
+} // namespace views
+
+} // namespace ranges
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___RANGES_ISTREAM_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/join_view.h b/libcxx/include/__cxx03/__ranges/join_view.h
new file mode 100644
index 000000000000..9c2c77995539
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/join_view.h
@@ -0,0 +1,423 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_JOIN_VIEW_H
+#define _LIBCPP___RANGES_JOIN_VIEW_H
+
+#include <__concepts/constructible.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/copyable.h>
+#include <__concepts/derived_from.h>
+#include <__concepts/equality_comparable.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/iter_swap.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/iterator_with_data.h>
+#include <__iterator/segmented_iterator.h>
+#include <__memory/addressof.h>
+#include <__ranges/access.h>
+#include <__ranges/all.h>
+#include <__ranges/concepts.h>
+#include <__ranges/empty.h>
+#include <__ranges/non_propagating_cache.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/common_type.h>
+#include <__type_traits/maybe_const.h>
+#include <__utility/as_lvalue.h>
+#include <__utility/empty.h>
+#include <__utility/forward.h>
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+template <class>
+struct __join_view_iterator_category {};
+
+template <class _View>
+  requires is_reference_v<range_reference_t<_View>> && forward_range<_View> && forward_range<range_reference_t<_View>>
+struct __join_view_iterator_category<_View> {
+  using _OuterC = typename iterator_traits<iterator_t<_View>>::iterator_category;
+  using _InnerC = typename iterator_traits<iterator_t<range_reference_t<_View>>>::iterator_category;
+
+  using iterator_category =
+      _If< derived_from<_OuterC, bidirectional_iterator_tag> && derived_from<_InnerC, bidirectional_iterator_tag> &&
+               common_range<range_reference_t<_View>>,
+           bidirectional_iterator_tag,
+           _If< derived_from<_OuterC, forward_iterator_tag> && derived_from<_InnerC, forward_iterator_tag>,
+                forward_iterator_tag,
+                input_iterator_tag > >;
+};
+
+template <input_range _View>
+  requires view<_View> && input_range<range_reference_t<_View>>
+class join_view : public view_interface<join_view<_View>> {
+private:
+  using _InnerRange = range_reference_t<_View>;
+
+  template <bool>
+  struct __iterator;
+
+  template <bool>
+  struct __sentinel;
+
+  template <class>
+  friend struct std::__segmented_iterator_traits;
+
+  _LIBCPP_NO_UNIQUE_ADDRESS _View __base_ = _View();
+
+  static constexpr bool _UseOuterCache = !forward_range<_View>;
+  using _OuterCache                    = _If<_UseOuterCache, __non_propagating_cache<iterator_t<_View>>, __empty_cache>;
+  _LIBCPP_NO_UNIQUE_ADDRESS _OuterCache __outer_;
+
+  static constexpr bool _UseInnerCache = !is_reference_v<_InnerRange>;
+  using _InnerCache = _If<_UseInnerCache, __non_propagating_cache<remove_cvref_t<_InnerRange>>, __empty_cache>;
+  _LIBCPP_NO_UNIQUE_ADDRESS _InnerCache __inner_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI join_view()
+    requires default_initializable<_View>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit join_view(_View __base) : __base_(std::move(__base)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
+    requires copy_constructible<_View>
+  {
+    return __base_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() {
+    if constexpr (forward_range<_View>) {
+      constexpr bool __use_const = __simple_view<_View> && is_reference_v<range_reference_t<_View>>;
+      return __iterator<__use_const>{*this, ranges::begin(__base_)};
+    } else {
+      __outer_.__emplace(ranges::begin(__base_));
+      return __iterator<false>{*this};
+    }
+  }
+
+  template <class _V2 = _View>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
+    requires forward_range<const _V2> && is_reference_v<range_reference_t<const _V2>> &&
+             input_range<range_reference_t<const _V2>>
+  {
+    return __iterator<true>{*this, ranges::begin(__base_)};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() {
+    if constexpr (forward_range<_View> && is_reference_v<_InnerRange> && forward_range<_InnerRange> &&
+                  common_range<_View> && common_range<_InnerRange>)
+      return __iterator<__simple_view<_View>>{*this, ranges::end(__base_)};
+    else
+      return __sentinel<__simple_view<_View>>{*this};
+  }
+
+  template <class _V2 = _View>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
+    requires forward_range<const _V2> && is_reference_v<range_reference_t<const _V2>> &&
+             input_range<range_reference_t<const _V2>>
+  {
+    using _ConstInnerRange = range_reference_t<const _View>;
+    if constexpr (forward_range<_ConstInnerRange> && common_range<const _View> && common_range<_ConstInnerRange>) {
+      return __iterator<true>{*this, ranges::end(__base_)};
+    } else {
+      return __sentinel<true>{*this};
+    }
+  }
+};
+
+template <input_range _View>
+  requires view<_View> && input_range<range_reference_t<_View>>
+template <bool _Const>
+struct join_view<_View>::__sentinel {
+private:
+  template <bool>
+  friend struct __sentinel;
+
+  using _Parent            = __maybe_const<_Const, join_view>;
+  using _Base              = __maybe_const<_Const, _View>;
+  sentinel_t<_Base> __end_ = sentinel_t<_Base>();
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __sentinel() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __sentinel(_Parent& __parent) : __end_(ranges::end(__parent.__base_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __sentinel(__sentinel<!_Const> __s)
+    requires _Const && convertible_to<sentinel_t<_View>, sentinel_t<_Base>>
+      : __end_(std::move(__s.__end_)) {}
+
+  template <bool _OtherConst>
+    requires sentinel_for<sentinel_t<_Base>, iterator_t<__maybe_const<_OtherConst, _View>>>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator<_OtherConst>& __x, const __sentinel& __y) {
+    return __x.__get_outer() == __y.__end_;
+  }
+};
+
+// https://reviews.llvm.org/D142811#inline-1383022
+// To simplify the segmented iterator traits specialization,
+// make the iterator `final`
+template <input_range _View>
+  requires view<_View> && input_range<range_reference_t<_View>>
+template <bool _Const>
+struct join_view<_View>::__iterator final : public __join_view_iterator_category<__maybe_const<_Const, _View>> {
+  friend join_view;
+
+  template <class>
+  friend struct std::__segmented_iterator_traits;
+
+  static constexpr bool __is_join_view_iterator = true;
+
+private:
+  using _Parent     = __maybe_const<_Const, join_view<_View>>;
+  using _Base       = __maybe_const<_Const, _View>;
+  using _Outer      = iterator_t<_Base>;
+  using _Inner      = iterator_t<range_reference_t<_Base>>;
+  using _InnerRange = range_reference_t<_View>;
+
+  static_assert(!_Const || forward_range<_Base>, "Const can only be true when Base models forward_range.");
+
+  static constexpr bool __ref_is_glvalue = is_reference_v<range_reference_t<_Base>>;
+
+  static constexpr bool _OuterPresent           = forward_range<_Base>;
+  using _OuterType                              = _If<_OuterPresent, _Outer, std::__empty>;
+  _LIBCPP_NO_UNIQUE_ADDRESS _OuterType __outer_ = _OuterType();
+
+  optional<_Inner> __inner_;
+  _Parent* __parent_ = nullptr;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void __satisfy() {
+    for (; __get_outer() != ranges::end(__parent_->__base_); ++__get_outer()) {
+      auto&& __inner = [this]() -> auto&& {
+        if constexpr (__ref_is_glvalue)
+          return *__get_outer();
+        else
+          return __parent_->__inner_.__emplace_from([&]() -> decltype(auto) { return *__get_outer(); });
+      }();
+      __inner_ = ranges::begin(__inner);
+      if (*__inner_ != ranges::end(__inner))
+        return;
+    }
+
+    if constexpr (__ref_is_glvalue)
+      __inner_.reset();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Outer& __get_outer() {
+    if constexpr (forward_range<_Base>) {
+      return __outer_;
+    } else {
+      return *__parent_->__outer_;
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Outer& __get_outer() const {
+    if constexpr (forward_range<_Base>) {
+      return __outer_;
+    } else {
+      return *__parent_->__outer_;
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator(_Parent& __parent, _Outer __outer)
+    requires forward_range<_Base>
+      : __outer_(std::move(__outer)), __parent_(std::addressof(__parent)) {
+    __satisfy();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __iterator(_Parent& __parent)
+    requires(!forward_range<_Base>)
+      : __parent_(std::addressof(__parent)) {
+    __satisfy();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator(_Parent* __parent, _Outer __outer, _Inner __inner)
+    requires forward_range<_Base>
+      : __outer_(std::move(__outer)), __inner_(std::move(__inner)), __parent_(__parent) {}
+
+public:
+  using iterator_concept =
+      _If< __ref_is_glvalue && bidirectional_range<_Base> && bidirectional_range<range_reference_t<_Base>> &&
+               common_range<range_reference_t<_Base>>,
+           bidirectional_iterator_tag,
+           _If< __ref_is_glvalue && forward_range<_Base> && forward_range<range_reference_t<_Base>>,
+                forward_iterator_tag,
+                input_iterator_tag > >;
+
+  using value_type = range_value_t<range_reference_t<_Base>>;
+
+  using difference_type = common_type_t< range_difference_t<_Base>, range_difference_t<range_reference_t<_Base>>>;
+
+  _LIBCPP_HIDE_FROM_ABI __iterator() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator(__iterator<!_Const> __i)
+    requires _Const && convertible_to<iterator_t<_View>, _Outer> && convertible_to<iterator_t<_InnerRange>, _Inner>
+      : __outer_(std::move(__i.__outer_)), __inner_(std::move(__i.__inner_)), __parent_(__i.__parent_) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() const { return **__inner_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Inner operator->() const
+    requires __has_arrow<_Inner> && copyable<_Inner>
+  {
+    return *__inner_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() {
+    auto __get_inner_range = [&]() -> decltype(auto) {
+      if constexpr (__ref_is_glvalue)
+        return *__get_outer();
+      else
+        return *__parent_->__inner_;
+    };
+    if (++*__inner_ == ranges::end(std::__as_lvalue(__get_inner_range()))) {
+      ++__get_outer();
+      __satisfy();
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator++(int) { ++*this; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator++(int)
+    requires __ref_is_glvalue && forward_range<_Base> && forward_range<range_reference_t<_Base>>
+  {
+    auto __tmp = *this;
+    ++*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator--()
+    requires __ref_is_glvalue && bidirectional_range<_Base> && bidirectional_range<range_reference_t<_Base>> &&
+             common_range<range_reference_t<_Base>>
+  {
+    if (__outer_ == ranges::end(__parent_->__base_))
+      __inner_ = ranges::end(std::__as_lvalue(*--__outer_));
+
+    // Skip empty inner ranges when going backwards.
+    while (*__inner_ == ranges::begin(std::__as_lvalue(*__outer_))) {
+      __inner_ = ranges::end(std::__as_lvalue(*--__outer_));
+    }
+
+    --*__inner_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator--(int)
+    requires __ref_is_glvalue && bidirectional_range<_Base> && bidirectional_range<range_reference_t<_Base>> &&
+             common_range<range_reference_t<_Base>>
+  {
+    auto __tmp = *this;
+    --*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __iterator& __y)
+    requires __ref_is_glvalue && forward_range<_Base> && equality_comparable<iterator_t<range_reference_t<_Base>>>
+  {
+    return __x.__outer_ == __y.__outer_ && __x.__inner_ == __y.__inner_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr decltype(auto)
+  iter_move(const __iterator& __i) noexcept(noexcept(ranges::iter_move(*__i.__inner_))) {
+    return ranges::iter_move(*__i.__inner_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr void
+  iter_swap(const __iterator& __x,
+            const __iterator& __y) noexcept(noexcept(ranges::iter_swap(*__x.__inner_, *__y.__inner_)))
+    requires indirectly_swappable<_Inner>
+  {
+    return ranges::iter_swap(*__x.__inner_, *__y.__inner_);
+  }
+};
+
+template <class _Range>
+explicit join_view(_Range&&) -> join_view<views::all_t<_Range>>;
+
+namespace views {
+namespace __join_view {
+struct __fn : __range_adaptor_closure<__fn> {
+  template <class _Range>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range) const
+      noexcept(noexcept(join_view<all_t<_Range&&>>(std::forward<_Range>(__range))))
+          -> decltype(join_view<all_t<_Range&&>>(std::forward<_Range>(__range))) {
+    return join_view<all_t<_Range&&>>(std::forward<_Range>(__range));
+  }
+};
+} // namespace __join_view
+inline namespace __cpo {
+inline constexpr auto join = __join_view::__fn{};
+} // namespace __cpo
+} // namespace views
+} // namespace ranges
+
+template <class _JoinViewIterator>
+  requires(_JoinViewIterator::__is_join_view_iterator && ranges::common_range<typename _JoinViewIterator::_Parent> &&
+           __has_random_access_iterator_category<typename _JoinViewIterator::_Outer>::value &&
+           __has_random_access_iterator_category<typename _JoinViewIterator::_Inner>::value)
+struct __segmented_iterator_traits<_JoinViewIterator> {
+  using __segment_iterator =
+      _LIBCPP_NODEBUG __iterator_with_data<typename _JoinViewIterator::_Outer, typename _JoinViewIterator::_Parent*>;
+  using __local_iterator = typename _JoinViewIterator::_Inner;
+
+  // TODO: Would it make sense to enable the optimization for other iterator types?
+
+  static constexpr _LIBCPP_HIDE_FROM_ABI __segment_iterator __segment(_JoinViewIterator __iter) {
+    if (ranges::empty(__iter.__parent_->__base_))
+      return {};
+    if (!__iter.__inner_.has_value())
+      return __segment_iterator(--__iter.__outer_, __iter.__parent_);
+    return __segment_iterator(__iter.__outer_, __iter.__parent_);
+  }
+
+  static constexpr _LIBCPP_HIDE_FROM_ABI __local_iterator __local(_JoinViewIterator __iter) {
+    if (ranges::empty(__iter.__parent_->__base_))
+      return {};
+    if (!__iter.__inner_.has_value())
+      return ranges::end(*--__iter.__outer_);
+    return *__iter.__inner_;
+  }
+
+  static constexpr _LIBCPP_HIDE_FROM_ABI __local_iterator __begin(__segment_iterator __iter) {
+    return ranges::begin(*__iter.__get_iter());
+  }
+
+  static constexpr _LIBCPP_HIDE_FROM_ABI __local_iterator __end(__segment_iterator __iter) {
+    return ranges::end(*__iter.__get_iter());
+  }
+
+  static constexpr _LIBCPP_HIDE_FROM_ABI _JoinViewIterator
+  __compose(__segment_iterator __seg_iter, __local_iterator __local_iter) {
+    return _JoinViewIterator(
+        std::move(__seg_iter).__get_data(), std::move(__seg_iter).__get_iter(), std::move(__local_iter));
+  }
+};
+
+#endif // #if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_JOIN_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/lazy_split_view.h b/libcxx/include/__cxx03/__ranges/lazy_split_view.h
new file mode 100644
index 000000000000..db031fe5f8b4
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/lazy_split_view.h
@@ -0,0 +1,441 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_LAZY_SPLIT_VIEW_H
+#define _LIBCPP___RANGES_LAZY_SPLIT_VIEW_H
+
+#include <__algorithm/ranges_find.h>
+#include <__algorithm/ranges_mismatch.h>
+#include <__assert>
+#include <__concepts/constructible.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/derived_from.h>
+#include <__config>
+#include <__functional/bind_back.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/default_sentinel.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/indirectly_comparable.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/iter_swap.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__ranges/access.h>
+#include <__ranges/all.h>
+#include <__ranges/concepts.h>
+#include <__ranges/non_propagating_cache.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/single_view.h>
+#include <__ranges/subrange.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/maybe_const.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+template <auto>
+struct __require_constant;
+
+template <class _Range>
+concept __tiny_range = sized_range<_Range> && requires {
+  typename __require_constant<remove_reference_t<_Range>::size()>;
+} && (remove_reference_t<_Range>::size() <= 1);
+
+template <input_range _View, forward_range _Pattern>
+  requires view<_View> && view<_Pattern> &&
+           indirectly_comparable<iterator_t<_View>, iterator_t<_Pattern>, ranges::equal_to> &&
+           (forward_range<_View> || __tiny_range<_Pattern>)
+class lazy_split_view : public view_interface<lazy_split_view<_View, _Pattern>> {
+  _LIBCPP_NO_UNIQUE_ADDRESS _View __base_       = _View();
+  _LIBCPP_NO_UNIQUE_ADDRESS _Pattern __pattern_ = _Pattern();
+
+  using _MaybeCurrent = _If<!forward_range<_View>, __non_propagating_cache<iterator_t<_View>>, __empty_cache>;
+  _LIBCPP_NO_UNIQUE_ADDRESS _MaybeCurrent __current_ = _MaybeCurrent();
+
+  template <bool>
+  struct __outer_iterator;
+  template <bool>
+  struct __inner_iterator;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI lazy_split_view()
+    requires default_initializable<_View> && default_initializable<_Pattern>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23 lazy_split_view(_View __base, _Pattern __pattern)
+      : __base_(std::move(__base)), __pattern_(std::move(__pattern)) {}
+
+  template <input_range _Range>
+    requires constructible_from<_View, views::all_t<_Range>> &&
+                 constructible_from<_Pattern, single_view<range_value_t<_Range>>>
+  _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23 lazy_split_view(_Range&& __r, range_value_t<_Range> __e)
+      : __base_(views::all(std::forward<_Range>(__r))), __pattern_(views::single(std::move(__e))) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
+    requires copy_constructible<_View>
+  {
+    return __base_;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() {
+    if constexpr (forward_range<_View>) {
+      return __outer_iterator < __simple_view<_View> && __simple_view < _Pattern >> {*this, ranges::begin(__base_)};
+    } else {
+      __current_.__emplace(ranges::begin(__base_));
+      return __outer_iterator<false>{*this};
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
+    requires forward_range<_View> && forward_range<const _View>
+  {
+    return __outer_iterator<true>{*this, ranges::begin(__base_)};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end()
+    requires forward_range<_View> && common_range<_View>
+  {
+    return __outer_iterator < __simple_view<_View> && __simple_view < _Pattern >> {*this, ranges::end(__base_)};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() const {
+    if constexpr (forward_range<_View> && forward_range<const _View> && common_range<const _View>) {
+      return __outer_iterator<true>{*this, ranges::end(__base_)};
+    } else {
+      return default_sentinel;
+    }
+  }
+
+private:
+  template <class>
+  struct __outer_iterator_category {};
+
+  template <forward_range _Tp>
+  struct __outer_iterator_category<_Tp> {
+    using iterator_category = input_iterator_tag;
+  };
+
+  template <bool _Const>
+  struct __outer_iterator : __outer_iterator_category<__maybe_const<_Const, _View>> {
+  private:
+    template <bool>
+    friend struct __inner_iterator;
+    friend __outer_iterator<true>;
+
+    using _Parent = __maybe_const<_Const, lazy_split_view>;
+    using _Base   = __maybe_const<_Const, _View>;
+
+    _Parent* __parent_                                 = nullptr;
+    using _MaybeCurrent                                = _If<forward_range<_View>, iterator_t<_Base>, __empty_cache>;
+    _LIBCPP_NO_UNIQUE_ADDRESS _MaybeCurrent __current_ = _MaybeCurrent();
+    bool __trailing_empty_                             = false;
+
+    [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto& __current() noexcept {
+      if constexpr (forward_range<_View>) {
+        return __current_;
+      } else {
+        return *__parent_->__current_;
+      }
+    }
+
+    [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const auto& __current() const noexcept {
+      if constexpr (forward_range<_View>) {
+        return __current_;
+      } else {
+        return *__parent_->__current_;
+      }
+    }
+
+    // Workaround for the GCC issue that doesn't allow calling `__parent_->__base_` from friend functions (because
+    // `__base_` is private).
+    [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto& __parent_base() const noexcept { return __parent_->__base_; }
+
+  public:
+    // using iterator_category = inherited;
+    using iterator_concept = conditional_t<forward_range<_Base>, forward_iterator_tag, input_iterator_tag>;
+    using difference_type  = range_difference_t<_Base>;
+
+    struct value_type : view_interface<value_type> {
+    private:
+      __outer_iterator __i_ = __outer_iterator();
+
+    public:
+      _LIBCPP_HIDE_FROM_ABI value_type() = default;
+      _LIBCPP_HIDE_FROM_ABI constexpr explicit value_type(__outer_iterator __i) : __i_(std::move(__i)) {}
+
+      _LIBCPP_HIDE_FROM_ABI constexpr __inner_iterator<_Const> begin() const { return __inner_iterator<_Const>{__i_}; }
+      _LIBCPP_HIDE_FROM_ABI constexpr default_sentinel_t end() const noexcept { return default_sentinel; }
+    };
+
+    _LIBCPP_HIDE_FROM_ABI __outer_iterator() = default;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __outer_iterator(_Parent& __parent)
+      requires(!forward_range<_Base>)
+        : __parent_(std::addressof(__parent)) {}
+
+    _LIBCPP_HIDE_FROM_ABI constexpr __outer_iterator(_Parent& __parent, iterator_t<_Base> __current)
+      requires forward_range<_Base>
+        : __parent_(std::addressof(__parent)), __current_(std::move(__current)) {}
+
+    _LIBCPP_HIDE_FROM_ABI constexpr __outer_iterator(__outer_iterator<!_Const> __i)
+      requires _Const && convertible_to<iterator_t<_View>, iterator_t<_Base>>
+        : __parent_(__i.__parent_), __current_(std::move(__i.__current_)) {}
+
+    _LIBCPP_HIDE_FROM_ABI constexpr value_type operator*() const { return value_type{*this}; }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr __outer_iterator& operator++() {
+      const auto __end = ranges::end(__parent_->__base_);
+      if (__current() == __end) {
+        __trailing_empty_ = false;
+        return *this;
+      }
+
+      const auto [__pbegin, __pend] = ranges::subrange{__parent_->__pattern_};
+      if (__pbegin == __pend) {
+        // Empty pattern: split on every element in the input range
+        ++__current();
+
+      } else if constexpr (__tiny_range<_Pattern>) {
+        // One-element pattern: we can use `ranges::find`.
+        __current() = ranges::find(std::move(__current()), __end, *__pbegin);
+        if (__current() != __end) {
+          // Make sure we point to after the separator we just found.
+          ++__current();
+          if (__current() == __end)
+            __trailing_empty_ = true;
+        }
+
+      } else {
+        // General case for n-element pattern.
+        do {
+          const auto [__b, __p] = ranges::mismatch(__current(), __end, __pbegin, __pend);
+          if (__p == __pend) {
+            __current() = __b;
+            if (__current() == __end) {
+              __trailing_empty_ = true;
+            }
+            break; // The pattern matched; skip it.
+          }
+        } while (++__current() != __end);
+      }
+
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator++(int) {
+      if constexpr (forward_range<_Base>) {
+        auto __tmp = *this;
+        ++*this;
+        return __tmp;
+
+      } else {
+        ++*this;
+      }
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __outer_iterator& __x, const __outer_iterator& __y)
+      requires forward_range<_Base>
+    {
+      return __x.__current_ == __y.__current_ && __x.__trailing_empty_ == __y.__trailing_empty_;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __outer_iterator& __x, default_sentinel_t) {
+      _LIBCPP_ASSERT_NON_NULL(__x.__parent_ != nullptr, "Cannot call comparison on a default-constructed iterator.");
+      return __x.__current() == ranges::end(__x.__parent_base()) && !__x.__trailing_empty_;
+    }
+  };
+
+  template <class>
+  struct __inner_iterator_category {};
+
+  template <forward_range _Tp>
+  struct __inner_iterator_category<_Tp> {
+    using iterator_category =
+        _If< derived_from<typename iterator_traits<iterator_t<_Tp>>::iterator_category, forward_iterator_tag>,
+             forward_iterator_tag,
+             typename iterator_traits<iterator_t<_Tp>>::iterator_category >;
+  };
+
+  template <bool _Const>
+  struct __inner_iterator : __inner_iterator_category<__maybe_const<_Const, _View>> {
+  private:
+    using _Base = __maybe_const<_Const, _View>;
+    // Workaround for a GCC issue.
+    static constexpr bool _OuterConst = _Const;
+    __outer_iterator<_Const> __i_     = __outer_iterator<_OuterConst>();
+    bool __incremented_               = false;
+
+    // Note: these private functions are necessary because GCC doesn't allow calls to private members of `__i_` from
+    // free functions that are friends of `inner-iterator`.
+
+    _LIBCPP_HIDE_FROM_ABI constexpr bool __is_done() const {
+      _LIBCPP_ASSERT_NON_NULL(__i_.__parent_ != nullptr, "Cannot call comparison on a default-constructed iterator.");
+
+      auto [__pcur, __pend] = ranges::subrange{__i_.__parent_->__pattern_};
+      auto __end            = ranges::end(__i_.__parent_->__base_);
+
+      if constexpr (__tiny_range<_Pattern>) {
+        const auto& __cur = __i_.__current();
+        if (__cur == __end)
+          return true;
+        if (__pcur == __pend)
+          return __incremented_;
+
+        return *__cur == *__pcur;
+
+      } else {
+        auto __cur = __i_.__current();
+        if (__cur == __end)
+          return true;
+        if (__pcur == __pend)
+          return __incremented_;
+
+        do {
+          if (*__cur != *__pcur)
+            return false;
+          if (++__pcur == __pend)
+            return true;
+        } while (++__cur != __end);
+
+        return false;
+      }
+    }
+
+    [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto& __outer_current() noexcept { return __i_.__current(); }
+
+    [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const auto& __outer_current() const noexcept {
+      return __i_.__current();
+    }
+
+  public:
+    // using iterator_category = inherited;
+    using iterator_concept = typename __outer_iterator<_Const>::iterator_concept;
+    using value_type       = range_value_t<_Base>;
+    using difference_type  = range_difference_t<_Base>;
+
+    _LIBCPP_HIDE_FROM_ABI __inner_iterator() = default;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __inner_iterator(__outer_iterator<_Const> __i) : __i_(std::move(__i)) {}
+
+    _LIBCPP_HIDE_FROM_ABI constexpr const iterator_t<_Base>& base() const& noexcept { return __i_.__current(); }
+    _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_Base> base() &&
+      requires forward_range<_View>
+    {
+      return std::move(__i_.__current());
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() const { return *__i_.__current(); }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr __inner_iterator& operator++() {
+      __incremented_ = true;
+
+      if constexpr (!forward_range<_Base>) {
+        if constexpr (_Pattern::size() == 0) {
+          return *this;
+        }
+      }
+
+      ++__i_.__current();
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator++(int) {
+      if constexpr (forward_range<_Base>) {
+        auto __tmp = *this;
+        ++*this;
+        return __tmp;
+
+      } else {
+        ++*this;
+      }
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __inner_iterator& __x, const __inner_iterator& __y)
+      requires forward_range<_Base>
+    {
+      return __x.__outer_current() == __y.__outer_current();
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __inner_iterator& __x, default_sentinel_t) {
+      return __x.__is_done();
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr decltype(auto)
+    iter_move(const __inner_iterator& __i) noexcept(noexcept(ranges::iter_move(__i.__outer_current()))) {
+      return ranges::iter_move(__i.__outer_current());
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr void iter_swap(
+        const __inner_iterator& __x,
+        const __inner_iterator& __y) noexcept(noexcept(ranges::iter_swap(__x.__outer_current(), __y.__outer_current())))
+      requires indirectly_swappable<iterator_t<_Base>>
+    {
+      ranges::iter_swap(__x.__outer_current(), __y.__outer_current());
+    }
+  };
+};
+
+template <class _Range, class _Pattern>
+lazy_split_view(_Range&&, _Pattern&&) -> lazy_split_view<views::all_t<_Range>, views::all_t<_Pattern>>;
+
+template <input_range _Range>
+lazy_split_view(_Range&&,
+                range_value_t<_Range>) -> lazy_split_view<views::all_t<_Range>, single_view<range_value_t<_Range>>>;
+
+namespace views {
+namespace __lazy_split_view {
+struct __fn {
+  template <class _Range, class _Pattern>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range, _Pattern&& __pattern) const
+      noexcept(noexcept(lazy_split_view(std::forward<_Range>(__range), std::forward<_Pattern>(__pattern))))
+          -> decltype(lazy_split_view(std::forward<_Range>(__range), std::forward<_Pattern>(__pattern))) {
+    return lazy_split_view(std::forward<_Range>(__range), std::forward<_Pattern>(__pattern));
+  }
+
+  template <class _Pattern>
+    requires constructible_from<decay_t<_Pattern>, _Pattern>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Pattern&& __pattern) const
+      noexcept(is_nothrow_constructible_v<decay_t<_Pattern>, _Pattern>) {
+    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Pattern>(__pattern)));
+  }
+};
+} // namespace __lazy_split_view
+
+inline namespace __cpo {
+inline constexpr auto lazy_split = __lazy_split_view::__fn{};
+} // namespace __cpo
+} // namespace views
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_LAZY_SPLIT_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/movable_box.h b/libcxx/include/__cxx03/__ranges/movable_box.h
new file mode 100644
index 000000000000..5a456cc3a1b6
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/movable_box.h
@@ -0,0 +1,247 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_MOVABLE_BOX_H
+#define _LIBCPP___RANGES_MOVABLE_BOX_H
+
+#include <__concepts/constructible.h>
+#include <__concepts/copyable.h>
+#include <__concepts/movable.h>
+#include <__config>
+#include <__memory/addressof.h>
+#include <__memory/construct_at.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__utility/move.h>
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// __movable_box allows turning a type that is move-constructible (but maybe not move-assignable) into
+// a type that is both move-constructible and move-assignable. It does that by introducing an empty state
+// and basically doing destroy-then-copy-construct in the assignment operator. The empty state is necessary
+// to handle the case where the copy construction fails after destroying the object.
+//
+// In some cases, we can completely avoid the use of an empty state; we provide a specialization of
+// __movable_box that does this, see below for the details.
+
+// until C++23, `__movable_box` was named `__copyable_box` and required the stored type to be copy-constructible, not
+// just move-constructible; we preserve the old behavior in pre-C++23 modes.
+template <class _Tp>
+concept __movable_box_object =
+#  if _LIBCPP_STD_VER >= 23
+    move_constructible<_Tp>
+#  else
+    copy_constructible<_Tp>
+#  endif
+    && is_object_v<_Tp>;
+
+namespace ranges {
+// Primary template - uses std::optional and introduces an empty state in case assignment fails.
+template <__movable_box_object _Tp>
+class __movable_box {
+  _LIBCPP_NO_UNIQUE_ADDRESS optional<_Tp> __val_;
+
+public:
+  template <class... _Args>
+    requires is_constructible_v<_Tp, _Args...>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __movable_box(in_place_t, _Args&&... __args) noexcept(
+      is_nothrow_constructible_v<_Tp, _Args...>)
+      : __val_(in_place, std::forward<_Args>(__args)...) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __movable_box() noexcept(is_nothrow_default_constructible_v<_Tp>)
+    requires default_initializable<_Tp>
+      : __val_(in_place) {}
+
+  _LIBCPP_HIDE_FROM_ABI __movable_box(__movable_box const&) = default;
+  _LIBCPP_HIDE_FROM_ABI __movable_box(__movable_box&&)      = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __movable_box&
+  operator=(__movable_box const& __other) noexcept(is_nothrow_copy_constructible_v<_Tp>)
+#  if _LIBCPP_STD_VER >= 23
+    requires copy_constructible<_Tp>
+#  endif
+  {
+    if (this != std::addressof(__other)) {
+      if (__other.__has_value())
+        __val_.emplace(*__other);
+      else
+        __val_.reset();
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __movable_box& operator=(__movable_box&&)
+    requires movable<_Tp>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __movable_box&
+  operator=(__movable_box&& __other) noexcept(is_nothrow_move_constructible_v<_Tp>) {
+    if (this != std::addressof(__other)) {
+      if (__other.__has_value())
+        __val_.emplace(std::move(*__other));
+      else
+        __val_.reset();
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp const& operator*() const noexcept { return *__val_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp& operator*() noexcept { return *__val_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp* operator->() const noexcept { return __val_.operator->(); }
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp* operator->() noexcept { return __val_.operator->(); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __has_value() const noexcept { return __val_.has_value(); }
+};
+
+// This partial specialization implements an optimization for when we know we don't need to store
+// an empty state to represent failure to perform an assignment. For copy-assignment, this happens:
+//
+// 1. If the type is copyable (which includes copy-assignment), we can use the type's own assignment operator
+//    directly and avoid using std::optional.
+// 2. If the type is not copyable, but it is nothrow-copy-constructible, then we can implement assignment as
+//    destroy-and-then-construct and we know it will never fail, so we don't need an empty state.
+//
+// The exact same reasoning can be applied for move-assignment, with copyable replaced by movable and
+// nothrow-copy-constructible replaced by nothrow-move-constructible. This specialization is enabled
+// whenever we can apply any of these optimizations for both the copy assignment and the move assignment
+// operator.
+
+#  if _LIBCPP_STD_VER >= 23
+template <class _Tp>
+concept __doesnt_need_empty_state =
+    (copy_constructible<_Tp>
+         // 1. If copy_constructible<T> is true, movable-box<T> should store only a T if either T models
+         //    copyable, or is_nothrow_move_constructible_v<T> && is_nothrow_copy_constructible_v<T> is true.
+         ? copyable<_Tp> || (is_nothrow_move_constructible_v<_Tp> && is_nothrow_copy_constructible_v<_Tp>)
+         // 2. Otherwise, movable-box<T> should store only a T if either T models movable or
+         //    is_nothrow_move_constructible_v<T> is true.
+         : movable<_Tp> || is_nothrow_move_constructible_v<_Tp>);
+
+// When _Tp doesn't have an assignment operator, we must implement __movable_box's assignment operator
+// by doing destroy_at followed by construct_at. However, that implementation strategy leads to UB if the nested
+// _Tp is potentially overlapping, as it is doing a non-transparent replacement of the sub-object, which means that
+// we're not considered "nested" inside the movable-box anymore, and since we're not nested within it, [basic.life]/1.5
+// says that we essentially just reused the storage of the movable-box for a completely unrelated object and ended the
+// movable-box's lifetime.
+// https://github.com/llvm/llvm-project/issues/70494#issuecomment-1845646490
+//
+// Hence, when the _Tp doesn't have an assignment operator, we can't risk making it a potentially-overlapping
+// subobject because of the above, and we don't use [[no_unique_address]] in that case.
+template <class _Tp>
+concept __can_use_no_unique_address = (copy_constructible<_Tp> ? copyable<_Tp> : movable<_Tp>);
+
+#  else
+
+template <class _Tp>
+concept __doesnt_need_empty_state_for_copy = copyable<_Tp> || is_nothrow_copy_constructible_v<_Tp>;
+
+template <class _Tp>
+concept __doesnt_need_empty_state_for_move = movable<_Tp> || is_nothrow_move_constructible_v<_Tp>;
+
+template <class _Tp>
+concept __doesnt_need_empty_state = __doesnt_need_empty_state_for_copy<_Tp> && __doesnt_need_empty_state_for_move<_Tp>;
+
+template <class _Tp>
+concept __can_use_no_unique_address = copyable<_Tp>;
+#  endif
+
+template <class _Tp>
+struct __movable_box_holder {
+  _Tp __val_;
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __movable_box_holder(in_place_t, _Args&&... __args)
+      : __val_(std::forward<_Args>(__args)...) {}
+};
+
+template <class _Tp>
+  requires __can_use_no_unique_address<_Tp>
+struct __movable_box_holder<_Tp> {
+  _LIBCPP_NO_UNIQUE_ADDRESS _Tp __val_;
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __movable_box_holder(in_place_t, _Args&&... __args)
+      : __val_(std::forward<_Args>(__args)...) {}
+};
+
+template <__movable_box_object _Tp>
+  requires __doesnt_need_empty_state<_Tp>
+class __movable_box<_Tp> {
+  _LIBCPP_NO_UNIQUE_ADDRESS __movable_box_holder<_Tp> __holder_;
+
+public:
+  template <class... _Args>
+    requires is_constructible_v<_Tp, _Args...>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __movable_box(in_place_t __inplace, _Args&&... __args) noexcept(
+      is_nothrow_constructible_v<_Tp, _Args...>)
+      : __holder_(__inplace, std::forward<_Args>(__args)...) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __movable_box() noexcept(is_nothrow_default_constructible_v<_Tp>)
+    requires default_initializable<_Tp>
+      : __holder_(in_place_t{}) {}
+
+  _LIBCPP_HIDE_FROM_ABI __movable_box(__movable_box const&) = default;
+  _LIBCPP_HIDE_FROM_ABI __movable_box(__movable_box&&)      = default;
+
+  // Implementation of assignment operators in case we perform optimization (1)
+  _LIBCPP_HIDE_FROM_ABI __movable_box& operator=(__movable_box const&)
+    requires copyable<_Tp>
+  = default;
+  _LIBCPP_HIDE_FROM_ABI __movable_box& operator=(__movable_box&&)
+    requires movable<_Tp>
+  = default;
+
+  // Implementation of assignment operators in case we perform optimization (2)
+  _LIBCPP_HIDE_FROM_ABI constexpr __movable_box& operator=(__movable_box const& __other) noexcept {
+    static_assert(is_nothrow_copy_constructible_v<_Tp>);
+    static_assert(!__can_use_no_unique_address<_Tp>);
+    if (this != std::addressof(__other)) {
+      std::destroy_at(std::addressof(__holder_.__val_));
+      std::construct_at(std::addressof(__holder_.__val_), __other.__holder_.__val_);
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __movable_box& operator=(__movable_box&& __other) noexcept {
+    static_assert(is_nothrow_move_constructible_v<_Tp>);
+    static_assert(!__can_use_no_unique_address<_Tp>);
+    if (this != std::addressof(__other)) {
+      std::destroy_at(std::addressof(__holder_.__val_));
+      std::construct_at(std::addressof(__holder_.__val_), std::move(__other.__holder_.__val_));
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp const& operator*() const noexcept { return __holder_.__val_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp& operator*() noexcept { return __holder_.__val_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp* operator->() const noexcept { return std::addressof(__holder_.__val_); }
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp* operator->() noexcept { return std::addressof(__holder_.__val_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __has_value() const noexcept { return true; }
+};
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_MOVABLE_BOX_H
diff --git a/libcxx/include/__cxx03/__ranges/non_propagating_cache.h b/libcxx/include/__cxx03/__ranges/non_propagating_cache.h
new file mode 100644
index 000000000000..b2de2d2ae1cb
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/non_propagating_cache.h
@@ -0,0 +1,103 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_NON_PROPAGATING_CACHE_H
+#define _LIBCPP___RANGES_NON_PROPAGATING_CACHE_H
+
+#include <__config>
+#include <__iterator/concepts.h>        // indirectly_readable
+#include <__iterator/iterator_traits.h> // iter_reference_t
+#include <__memory/addressof.h>
+#include <__utility/forward.h>
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+// __non_propagating_cache is a helper type that allows storing an optional value in it,
+// but which does not copy the source's value when it is copy constructed/assigned to,
+// and which resets the source's value when it is moved-from.
+//
+// This type is used as an implementation detail of some views that need to cache the
+// result of `begin()` in order to provide an amortized O(1) begin() method. Typically,
+// we don't want to propagate the value of the cache upon copy because the cached iterator
+// may refer to internal details of the source view.
+template <class _Tp>
+  requires is_object_v<_Tp>
+class _LIBCPP_TEMPLATE_VIS __non_propagating_cache {
+  struct __from_tag {};
+  struct __forward_tag {};
+
+  // This helper class is needed to perform copy and move elision when
+  // constructing the contained type from an iterator.
+  struct __wrapper {
+    template <class... _Args>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __wrapper(__forward_tag, _Args&&... __args)
+        : __t_(std::forward<_Args>(__args)...) {}
+    template <class _Fn>
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit __wrapper(__from_tag, _Fn const& __f) : __t_(__f()) {}
+    _Tp __t_;
+  };
+
+  optional<__wrapper> __value_ = nullopt;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __non_propagating_cache() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __non_propagating_cache(__non_propagating_cache const&) noexcept
+      : __value_(nullopt) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __non_propagating_cache(__non_propagating_cache&& __other) noexcept
+      : __value_(nullopt) {
+    __other.__value_.reset();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __non_propagating_cache& operator=(__non_propagating_cache const& __other) noexcept {
+    if (this != std::addressof(__other)) {
+      __value_.reset();
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __non_propagating_cache& operator=(__non_propagating_cache&& __other) noexcept {
+    __value_.reset();
+    __other.__value_.reset();
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp& operator*() { return __value_->__t_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp const& operator*() const { return __value_->__t_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __has_value() const { return __value_.has_value(); }
+
+  template <class _Fn>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp& __emplace_from(_Fn const& __f) {
+    return __value_.emplace(__from_tag{}, __f).__t_;
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp& __emplace(_Args&&... __args) {
+    return __value_.emplace(__forward_tag{}, std::forward<_Args>(__args)...).__t_;
+  }
+};
+
+struct __empty_cache {};
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_NON_PROPAGATING_CACHE_H
diff --git a/libcxx/include/__cxx03/__ranges/owning_view.h b/libcxx/include/__cxx03/__ranges/owning_view.h
new file mode 100644
index 000000000000..254bdb432911
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/owning_view.h
@@ -0,0 +1,116 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_OWNING_VIEW_H
+#define _LIBCPP___RANGES_OWNING_VIEW_H
+
+#include <__concepts/constructible.h>
+#include <__concepts/movable.h>
+#include <__config>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/data.h>
+#include <__ranges/empty.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/size.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+template <range _Rp>
+  requires movable<_Rp> && (!__is_std_initializer_list<remove_cvref_t<_Rp>>)
+class owning_view : public view_interface<owning_view<_Rp>> {
+  _Rp __r_ = _Rp();
+
+public:
+  _LIBCPP_HIDE_FROM_ABI owning_view()
+    requires default_initializable<_Rp>
+  = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr owning_view(_Rp&& __r) : __r_(std::move(__r)) {}
+
+  _LIBCPP_HIDE_FROM_ABI owning_view(owning_view&&)            = default;
+  _LIBCPP_HIDE_FROM_ABI owning_view& operator=(owning_view&&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Rp& base() & noexcept { return __r_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Rp& base() const& noexcept { return __r_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr _Rp&& base() && noexcept { return std::move(__r_); }
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Rp&& base() const&& noexcept { return std::move(__r_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_Rp> begin() { return ranges::begin(__r_); }
+  _LIBCPP_HIDE_FROM_ABI constexpr sentinel_t<_Rp> end() { return ranges::end(__r_); }
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
+    requires range<const _Rp>
+  {
+    return ranges::begin(__r_);
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
+    requires range<const _Rp>
+  {
+    return ranges::end(__r_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool empty()
+    requires requires { ranges::empty(__r_); }
+  {
+    return ranges::empty(__r_);
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr bool empty() const
+    requires requires { ranges::empty(__r_); }
+  {
+    return ranges::empty(__r_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size()
+    requires sized_range<_Rp>
+  {
+    return ranges::size(__r_);
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
+    requires sized_range<const _Rp>
+  {
+    return ranges::size(__r_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto data()
+    requires contiguous_range<_Rp>
+  {
+    return ranges::data(__r_);
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr auto data() const
+    requires contiguous_range<const _Rp>
+  {
+    return ranges::data(__r_);
+  }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(owning_view);
+
+template <class _Tp>
+inline constexpr bool enable_borrowed_range<owning_view<_Tp>> = enable_borrowed_range<_Tp>;
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_OWNING_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/range_adaptor.h b/libcxx/include/__cxx03/__ranges/range_adaptor.h
new file mode 100644
index 000000000000..2da246f24e1d
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/range_adaptor.h
@@ -0,0 +1,101 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_RANGE_ADAPTOR_H
+#define _LIBCPP___RANGES_RANGE_ADAPTOR_H
+
+#include <__concepts/constructible.h>
+#include <__concepts/derived_from.h>
+#include <__concepts/invocable.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__functional/compose.h>
+#include <__functional/invoke.h>
+#include <__ranges/concepts.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_class.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+// CRTP base that one can derive from in order to be considered a range adaptor closure
+// by the library. When deriving from this class, a pipe operator will be provided to
+// make the following hold:
+// - `x | f` is equivalent to `f(x)`
+// - `f1 | f2` is an adaptor closure `g` such that `g(x)` is equivalent to `f2(f1(x))`
+template <class _Tp>
+  requires is_class_v<_Tp> && same_as<_Tp, remove_cv_t<_Tp>>
+struct __range_adaptor_closure;
+
+// Type that wraps an arbitrary function object and makes it into a range adaptor closure,
+// i.e. something that can be called via the `x | f` notation.
+template <class _Fn>
+struct __range_adaptor_closure_t : _Fn, __range_adaptor_closure<__range_adaptor_closure_t<_Fn>> {
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __range_adaptor_closure_t(_Fn&& __f) : _Fn(std::move(__f)) {}
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(__range_adaptor_closure_t);
+
+template <class _Tp>
+_Tp __derived_from_range_adaptor_closure(__range_adaptor_closure<_Tp>*);
+
+template <class _Tp>
+concept _RangeAdaptorClosure = !ranges::range<remove_cvref_t<_Tp>> && requires {
+  // Ensure that `remove_cvref_t<_Tp>` is derived from `__range_adaptor_closure<remove_cvref_t<_Tp>>` and isn't derived
+  // from `__range_adaptor_closure<U>` for any other type `U`.
+  { ranges::__derived_from_range_adaptor_closure((remove_cvref_t<_Tp>*)nullptr) } -> same_as<remove_cvref_t<_Tp>>;
+};
+
+template <ranges::range _Range, _RangeAdaptorClosure _Closure>
+  requires invocable<_Closure, _Range>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto)
+operator|(_Range&& __range, _Closure&& __closure) noexcept(is_nothrow_invocable_v<_Closure, _Range>) {
+  return std::invoke(std::forward<_Closure>(__closure), std::forward<_Range>(__range));
+}
+
+template <_RangeAdaptorClosure _Closure, _RangeAdaptorClosure _OtherClosure>
+  requires constructible_from<decay_t<_Closure>, _Closure> && constructible_from<decay_t<_OtherClosure>, _OtherClosure>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator|(_Closure&& __c1, _OtherClosure&& __c2) noexcept(
+    is_nothrow_constructible_v<decay_t<_Closure>, _Closure> &&
+    is_nothrow_constructible_v<decay_t<_OtherClosure>, _OtherClosure>) {
+  return __range_adaptor_closure_t(std::__compose(std::forward<_OtherClosure>(__c2), std::forward<_Closure>(__c1)));
+}
+
+template <class _Tp>
+  requires is_class_v<_Tp> && same_as<_Tp, remove_cv_t<_Tp>>
+struct __range_adaptor_closure {};
+
+#  if _LIBCPP_STD_VER >= 23
+template <class _Tp>
+  requires is_class_v<_Tp> && same_as<_Tp, remove_cv_t<_Tp>>
+class range_adaptor_closure : public __range_adaptor_closure<_Tp> {};
+#  endif // _LIBCPP_STD_VER >= 23
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_RANGE_ADAPTOR_H
diff --git a/libcxx/include/__cxx03/__ranges/rbegin.h b/libcxx/include/__cxx03/__ranges/rbegin.h
new file mode 100644
index 000000000000..12e739e1a2b8
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/rbegin.h
@@ -0,0 +1,120 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_RBEGIN_H
+#define _LIBCPP___RANGES_RBEGIN_H
+
+#include <__concepts/class_or_enum.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/readable_traits.h>
+#include <__iterator/reverse_iterator.h>
+#include <__ranges/access.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/auto_cast.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [ranges.access.rbegin]
+
+namespace ranges {
+namespace __rbegin {
+template <class _Tp>
+concept __member_rbegin = __can_borrow<_Tp> && requires(_Tp&& __t) {
+  { _LIBCPP_AUTO_CAST(__t.rbegin()) } -> input_or_output_iterator;
+};
+
+void rbegin() = delete;
+
+template <class _Tp>
+concept __unqualified_rbegin =
+    !__member_rbegin<_Tp> && __can_borrow<_Tp> && __class_or_enum<remove_cvref_t<_Tp>> && requires(_Tp&& __t) {
+      { _LIBCPP_AUTO_CAST(rbegin(__t)) } -> input_or_output_iterator;
+    };
+
+template <class _Tp>
+concept __can_reverse =
+    __can_borrow<_Tp> && !__member_rbegin<_Tp> && !__unqualified_rbegin<_Tp> && requires(_Tp&& __t) {
+      { ranges::begin(__t) } -> same_as<decltype(ranges::end(__t))>;
+      { ranges::begin(__t) } -> bidirectional_iterator;
+    };
+
+struct __fn {
+  template <class _Tp>
+    requires __member_rbegin<_Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(__t.rbegin()))) {
+    return _LIBCPP_AUTO_CAST(__t.rbegin());
+  }
+
+  template <class _Tp>
+    requires __unqualified_rbegin<_Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(rbegin(__t)))) {
+    return _LIBCPP_AUTO_CAST(rbegin(__t));
+  }
+
+  template <class _Tp>
+    requires __can_reverse<_Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const noexcept(noexcept(ranges::end(__t))) {
+    return std::make_reverse_iterator(ranges::end(__t));
+  }
+
+  void operator()(auto&&) const = delete;
+};
+} // namespace __rbegin
+
+inline namespace __cpo {
+inline constexpr auto rbegin = __rbegin::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+// [range.access.crbegin]
+
+namespace ranges {
+namespace __crbegin {
+struct __fn {
+  template <class _Tp>
+    requires is_lvalue_reference_v<_Tp&&>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(ranges::rbegin(static_cast<const remove_reference_t<_Tp>&>(__t))))
+          -> decltype(ranges::rbegin(static_cast<const remove_reference_t<_Tp>&>(__t))) {
+    return ranges::rbegin(static_cast<const remove_reference_t<_Tp>&>(__t));
+  }
+
+  template <class _Tp>
+    requires is_rvalue_reference_v<_Tp&&>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(ranges::rbegin(static_cast<const _Tp&&>(__t))))
+          -> decltype(ranges::rbegin(static_cast<const _Tp&&>(__t))) {
+    return ranges::rbegin(static_cast<const _Tp&&>(__t));
+  }
+};
+} // namespace __crbegin
+
+inline namespace __cpo {
+inline constexpr auto crbegin = __crbegin::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_RBEGIN_H
diff --git a/libcxx/include/__cxx03/__ranges/ref_view.h b/libcxx/include/__cxx03/__ranges/ref_view.h
new file mode 100644
index 000000000000..5329d778dd30
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/ref_view.h
@@ -0,0 +1,89 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_REF_VIEW_H
+#define _LIBCPP___RANGES_REF_VIEW_H
+
+#include <__concepts/convertible_to.h>
+#include <__concepts/different_from.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/data.h>
+#include <__ranges/empty.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/size.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/is_object.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+template <range _Range>
+  requires is_object_v<_Range>
+class ref_view : public view_interface<ref_view<_Range>> {
+  _Range* __range_;
+
+  static void __fun(_Range&);
+  static void __fun(_Range&&) = delete; // NOLINT(modernize-use-equals-delete) ; This is llvm.org/PR54276
+
+public:
+  template <class _Tp>
+    requires __different_from<_Tp, ref_view> && convertible_to<_Tp, _Range&> && requires { __fun(std::declval<_Tp>()); }
+  _LIBCPP_HIDE_FROM_ABI constexpr ref_view(_Tp&& __t)
+      : __range_(std::addressof(static_cast<_Range&>(std::forward<_Tp>(__t)))) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Range& base() const { return *__range_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_Range> begin() const { return ranges::begin(*__range_); }
+  _LIBCPP_HIDE_FROM_ABI constexpr sentinel_t<_Range> end() const { return ranges::end(*__range_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool empty() const
+    requires requires { ranges::empty(*__range_); }
+  {
+    return ranges::empty(*__range_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
+    requires sized_range<_Range>
+  {
+    return ranges::size(*__range_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto data() const
+    requires contiguous_range<_Range>
+  {
+    return ranges::data(*__range_);
+  }
+};
+
+template <class _Range>
+ref_view(_Range&) -> ref_view<_Range>;
+
+template <class _Tp>
+inline constexpr bool enable_borrowed_range<ref_view<_Tp>> = true;
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_REF_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/rend.h b/libcxx/include/__cxx03/__ranges/rend.h
new file mode 100644
index 000000000000..02b4c5999a7e
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/rend.h
@@ -0,0 +1,123 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_REND_H
+#define _LIBCPP___RANGES_REND_H
+
+#include <__concepts/class_or_enum.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/readable_traits.h>
+#include <__iterator/reverse_iterator.h>
+#include <__ranges/access.h>
+#include <__ranges/rbegin.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/auto_cast.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// [range.access.rend]
+
+namespace ranges {
+namespace __rend {
+template <class _Tp>
+concept __member_rend = __can_borrow<_Tp> && requires(_Tp&& __t) {
+  ranges::rbegin(__t);
+  { _LIBCPP_AUTO_CAST(__t.rend()) } -> sentinel_for<decltype(ranges::rbegin(__t))>;
+};
+
+void rend() = delete;
+
+template <class _Tp>
+concept __unqualified_rend =
+    !__member_rend<_Tp> && __can_borrow<_Tp> && __class_or_enum<remove_cvref_t<_Tp>> && requires(_Tp&& __t) {
+      ranges::rbegin(__t);
+      { _LIBCPP_AUTO_CAST(rend(__t)) } -> sentinel_for<decltype(ranges::rbegin(__t))>;
+    };
+
+template <class _Tp>
+concept __can_reverse = __can_borrow<_Tp> && !__member_rend<_Tp> && !__unqualified_rend<_Tp> && requires(_Tp&& __t) {
+  { ranges::begin(__t) } -> same_as<decltype(ranges::end(__t))>;
+  { ranges::begin(__t) } -> bidirectional_iterator;
+};
+
+class __fn {
+public:
+  template <class _Tp>
+    requires __member_rend<_Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(__t.rend()))) {
+    return _LIBCPP_AUTO_CAST(__t.rend());
+  }
+
+  template <class _Tp>
+    requires __unqualified_rend<_Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(rend(__t)))) {
+    return _LIBCPP_AUTO_CAST(rend(__t));
+  }
+
+  template <class _Tp>
+    requires __can_reverse<_Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(ranges::begin(__t))) {
+    return std::make_reverse_iterator(ranges::begin(__t));
+  }
+
+  void operator()(auto&&) const = delete;
+};
+} // namespace __rend
+
+inline namespace __cpo {
+inline constexpr auto rend = __rend::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+// [range.access.crend]
+
+namespace ranges {
+namespace __crend {
+struct __fn {
+  template <class _Tp>
+    requires is_lvalue_reference_v<_Tp&&>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(ranges::rend(static_cast<const remove_reference_t<_Tp>&>(__t))))
+          -> decltype(ranges::rend(static_cast<const remove_reference_t<_Tp>&>(__t))) {
+    return ranges::rend(static_cast<const remove_reference_t<_Tp>&>(__t));
+  }
+
+  template <class _Tp>
+    requires is_rvalue_reference_v<_Tp&&>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const noexcept(
+      noexcept(ranges::rend(static_cast<const _Tp&&>(__t)))) -> decltype(ranges::rend(static_cast<const _Tp&&>(__t))) {
+    return ranges::rend(static_cast<const _Tp&&>(__t));
+  }
+};
+} // namespace __crend
+
+inline namespace __cpo {
+inline constexpr auto crend = __crend::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_REND_H
diff --git a/libcxx/include/__cxx03/__ranges/repeat_view.h b/libcxx/include/__cxx03/__ranges/repeat_view.h
new file mode 100644
index 000000000000..53e4beb270ad
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/repeat_view.h
@@ -0,0 +1,266 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_REPEAT_VIEW_H
+#define _LIBCPP___RANGES_REPEAT_VIEW_H
+
+#include <__assert>
+#include <__concepts/constructible.h>
+#include <__concepts/same_as.h>
+#include <__concepts/semiregular.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/unreachable_sentinel.h>
+#include <__memory/addressof.h>
+#include <__ranges/iota_view.h>
+#include <__ranges/movable_box.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_object.h>
+#include <__type_traits/make_unsigned.h>
+#include <__type_traits/remove_cv.h>
+#include <__utility/forward.h>
+#include <__utility/in_place.h>
+#include <__utility/move.h>
+#include <__utility/piecewise_construct.h>
+#include <tuple>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+namespace ranges {
+
+template <class _Tp>
+concept __integer_like_with_usable_difference_type =
+    __signed_integer_like<_Tp> || (__integer_like<_Tp> && weakly_incrementable<_Tp>);
+
+template <class _Tp>
+struct __repeat_view_iterator_difference {
+  using type = _IotaDiffT<_Tp>;
+};
+
+template <__signed_integer_like _Tp>
+struct __repeat_view_iterator_difference<_Tp> {
+  using type = _Tp;
+};
+
+template <class _Tp>
+using __repeat_view_iterator_difference_t = typename __repeat_view_iterator_difference<_Tp>::type;
+
+namespace views::__drop {
+struct __fn;
+} // namespace views::__drop
+
+namespace views::__take {
+struct __fn;
+} // namespace views::__take
+
+template <move_constructible _Tp, semiregular _Bound = unreachable_sentinel_t>
+  requires(is_object_v<_Tp> && same_as<_Tp, remove_cv_t<_Tp>> &&
+           (__integer_like_with_usable_difference_type<_Bound> || same_as<_Bound, unreachable_sentinel_t>))
+class _LIBCPP_ABI_LLVM18_NO_UNIQUE_ADDRESS repeat_view : public view_interface<repeat_view<_Tp, _Bound>> {
+  friend struct views::__take::__fn;
+  friend struct views::__drop::__fn;
+  class __iterator;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI repeat_view()
+    requires default_initializable<_Tp>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit repeat_view(const _Tp& __value, _Bound __bound_sentinel = _Bound())
+    requires copy_constructible<_Tp>
+      : __value_(in_place, __value), __bound_(__bound_sentinel) {
+    if constexpr (!same_as<_Bound, unreachable_sentinel_t>)
+      _LIBCPP_ASSERT_UNCATEGORIZED(__bound_ >= 0, "The value of bound must be greater than or equal to 0");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit repeat_view(_Tp&& __value, _Bound __bound_sentinel = _Bound())
+      : __value_(in_place, std::move(__value)), __bound_(__bound_sentinel) {
+    if constexpr (!same_as<_Bound, unreachable_sentinel_t>)
+      _LIBCPP_ASSERT_UNCATEGORIZED(__bound_ >= 0, "The value of bound must be greater than or equal to 0");
+  }
+
+  template <class... _TpArgs, class... _BoundArgs>
+    requires(constructible_from<_Tp, _TpArgs...> && constructible_from<_Bound, _BoundArgs...>)
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit repeat_view(
+      piecewise_construct_t, tuple<_TpArgs...> __value_args, tuple<_BoundArgs...> __bound_args = tuple<>{})
+      : __value_(in_place, std::make_from_tuple<_Tp>(std::move(__value_args))),
+        __bound_(std::make_from_tuple<_Bound>(std::move(__bound_args))) {
+    if constexpr (!same_as<_Bound, unreachable_sentinel_t>)
+      _LIBCPP_ASSERT_UNCATEGORIZED(
+          __bound_ >= 0, "The behavior is undefined if Bound is not unreachable_sentinel_t and bound is negative");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() const { return __iterator(std::addressof(*__value_)); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator end() const
+    requires(!same_as<_Bound, unreachable_sentinel_t>)
+  {
+    return __iterator(std::addressof(*__value_), __bound_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr unreachable_sentinel_t end() const noexcept { return unreachable_sentinel; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
+    requires(!same_as<_Bound, unreachable_sentinel_t>)
+  {
+    return std::__to_unsigned_like(__bound_);
+  }
+
+private:
+  _LIBCPP_NO_UNIQUE_ADDRESS __movable_box<_Tp> __value_;
+  _LIBCPP_NO_UNIQUE_ADDRESS _Bound __bound_ = _Bound();
+};
+
+template <class _Tp, class _Bound = unreachable_sentinel_t>
+repeat_view(_Tp, _Bound = _Bound()) -> repeat_view<_Tp, _Bound>;
+
+// [range.repeat.iterator]
+template <move_constructible _Tp, semiregular _Bound>
+  requires(is_object_v<_Tp> && same_as<_Tp, remove_cv_t<_Tp>> &&
+           (__integer_like_with_usable_difference_type<_Bound> || same_as<_Bound, unreachable_sentinel_t>))
+class repeat_view<_Tp, _Bound>::__iterator {
+  friend class repeat_view;
+
+  using _IndexT = conditional_t<same_as<_Bound, unreachable_sentinel_t>, ptrdiff_t, _Bound>;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __iterator(const _Tp* __value, _IndexT __bound_sentinel = _IndexT())
+      : __value_(__value), __current_(__bound_sentinel) {}
+
+public:
+  using iterator_concept  = random_access_iterator_tag;
+  using iterator_category = random_access_iterator_tag;
+  using value_type        = _Tp;
+  using difference_type   = __repeat_view_iterator_difference_t<_IndexT>;
+
+  _LIBCPP_HIDE_FROM_ABI __iterator() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& operator*() const noexcept { return *__value_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() {
+    ++__current_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator++(int) {
+    auto __tmp = *this;
+    ++*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator--() {
+    if constexpr (!same_as<_Bound, unreachable_sentinel_t>)
+      _LIBCPP_ASSERT_UNCATEGORIZED(__current_ > 0, "The value of bound must be greater than or equal to 0");
+    --__current_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator--(int) {
+    auto __tmp = *this;
+    --*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator+=(difference_type __n) {
+    if constexpr (!same_as<_Bound, unreachable_sentinel_t>)
+      _LIBCPP_ASSERT_UNCATEGORIZED(__current_ + __n >= 0, "The value of bound must be greater than or equal to 0");
+    __current_ += __n;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator-=(difference_type __n) {
+    if constexpr (!same_as<_Bound, unreachable_sentinel_t>)
+      _LIBCPP_ASSERT_UNCATEGORIZED(__current_ - __n >= 0, "The value of bound must be greater than or equal to 0");
+    __current_ -= __n;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& operator[](difference_type __n) const noexcept { return *(*this + __n); }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __iterator& __y) {
+    return __x.__current_ == __y.__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr auto operator<=>(const __iterator& __x, const __iterator& __y) {
+    return __x.__current_ <=> __y.__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(__iterator __i, difference_type __n) {
+    __i += __n;
+    return __i;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(difference_type __n, __iterator __i) {
+    __i += __n;
+    return __i;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator-(__iterator __i, difference_type __n) {
+    __i -= __n;
+    return __i;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr difference_type operator-(const __iterator& __x, const __iterator& __y) {
+    return static_cast<difference_type>(__x.__current_) - static_cast<difference_type>(__y.__current_);
+  }
+
+private:
+  const _Tp* __value_ = nullptr;
+  _IndexT __current_  = _IndexT();
+};
+
+// clang-format off
+namespace views {
+namespace __repeat {
+struct __fn {
+  template <class _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()(_Tp&& __value)
+    noexcept(noexcept(ranges::repeat_view<decay_t<_Tp>>(std::forward<_Tp>(__value))))
+    -> decltype(      ranges::repeat_view<decay_t<_Tp>>(std::forward<_Tp>(__value)))
+    { return          ranges::repeat_view<decay_t<_Tp>>(std::forward<_Tp>(__value)); }
+
+  template <class _Tp, class _Bound>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()(_Tp&& __value, _Bound&& __bound_sentinel)
+    noexcept(noexcept(ranges::repeat_view(std::forward<_Tp>(__value), std::forward<_Bound>(__bound_sentinel))))
+    -> decltype(      ranges::repeat_view(std::forward<_Tp>(__value), std::forward<_Bound>(__bound_sentinel)))
+    { return          ranges::repeat_view(std::forward<_Tp>(__value), std::forward<_Bound>(__bound_sentinel)); }
+};
+} // namespace __repeat
+// clang-format on
+
+inline namespace __cpo {
+inline constexpr auto repeat = __repeat::__fn{};
+} // namespace __cpo
+} // namespace views
+
+template <class _Tp>
+inline constexpr bool __is_repeat_specialization = false;
+
+template <class _Tp, class _Bound>
+inline constexpr bool __is_repeat_specialization<repeat_view<_Tp, _Bound>> = true;
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_REPEAT_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/reverse_view.h b/libcxx/include/__cxx03/__ranges/reverse_view.h
new file mode 100644
index 000000000000..796f5be22328
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/reverse_view.h
@@ -0,0 +1,203 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_REVERSE_VIEW_H
+#define _LIBCPP___RANGES_REVERSE_VIEW_H
+
+#include <__concepts/constructible.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/next.h>
+#include <__iterator/reverse_iterator.h>
+#include <__ranges/access.h>
+#include <__ranges/all.h>
+#include <__ranges/concepts.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/non_propagating_cache.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/size.h>
+#include <__ranges/subrange.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+template <view _View>
+  requires bidirectional_range<_View>
+class reverse_view : public view_interface<reverse_view<_View>> {
+  // We cache begin() whenever ranges::next is not guaranteed O(1) to provide an
+  // amortized O(1) begin() method.
+  static constexpr bool _UseCache = !random_access_range<_View> && !common_range<_View>;
+  using _Cache = _If<_UseCache, __non_propagating_cache<reverse_iterator<iterator_t<_View>>>, __empty_cache>;
+  _LIBCPP_NO_UNIQUE_ADDRESS _Cache __cached_begin_ = _Cache();
+  _LIBCPP_NO_UNIQUE_ADDRESS _View __base_          = _View();
+
+public:
+  _LIBCPP_HIDE_FROM_ABI reverse_view()
+    requires default_initializable<_View>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit reverse_view(_View __view) : __base_(std::move(__view)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
+    requires copy_constructible<_View>
+  {
+    return __base_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator<iterator_t<_View>> begin() {
+    if constexpr (_UseCache)
+      if (__cached_begin_.__has_value())
+        return *__cached_begin_;
+
+    auto __tmp = std::make_reverse_iterator(ranges::next(ranges::begin(__base_), ranges::end(__base_)));
+    if constexpr (_UseCache)
+      __cached_begin_.__emplace(__tmp);
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator<iterator_t<_View>> begin()
+    requires common_range<_View>
+  {
+    return std::make_reverse_iterator(ranges::end(__base_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
+    requires common_range<const _View>
+  {
+    return std::make_reverse_iterator(ranges::end(__base_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator<iterator_t<_View>> end() {
+    return std::make_reverse_iterator(ranges::begin(__base_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
+    requires common_range<const _View>
+  {
+    return std::make_reverse_iterator(ranges::begin(__base_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size()
+    requires sized_range<_View>
+  {
+    return ranges::size(__base_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
+    requires sized_range<const _View>
+  {
+    return ranges::size(__base_);
+  }
+};
+
+template <class _Range>
+reverse_view(_Range&&) -> reverse_view<views::all_t<_Range>>;
+
+template <class _Tp>
+inline constexpr bool enable_borrowed_range<reverse_view<_Tp>> = enable_borrowed_range<_Tp>;
+
+namespace views {
+namespace __reverse {
+template <class _Tp>
+inline constexpr bool __is_reverse_view = false;
+
+template <class _Tp>
+inline constexpr bool __is_reverse_view<reverse_view<_Tp>> = true;
+
+template <class _Tp>
+inline constexpr bool __is_sized_reverse_subrange = false;
+
+template <class _Iter>
+inline constexpr bool
+    __is_sized_reverse_subrange<subrange<reverse_iterator<_Iter>, reverse_iterator<_Iter>, subrange_kind::sized>> =
+        true;
+
+template <class _Tp>
+inline constexpr bool __is_unsized_reverse_subrange = false;
+
+template <class _Iter, subrange_kind _Kind>
+inline constexpr bool __is_unsized_reverse_subrange<subrange<reverse_iterator<_Iter>, reverse_iterator<_Iter>, _Kind>> =
+    _Kind == subrange_kind::unsized;
+
+template <class _Tp>
+struct __unwrapped_reverse_subrange {
+  using type =
+      void; // avoid SFINAE-ing out the overload below -- let the concept requirements do it for better diagnostics
+};
+
+template <class _Iter, subrange_kind _Kind>
+struct __unwrapped_reverse_subrange<subrange<reverse_iterator<_Iter>, reverse_iterator<_Iter>, _Kind>> {
+  using type = subrange<_Iter, _Iter, _Kind>;
+};
+
+struct __fn : __range_adaptor_closure<__fn> {
+  template <class _Range>
+    requires __is_reverse_view<remove_cvref_t<_Range>>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range) const
+      noexcept(noexcept(std::forward<_Range>(__range).base())) -> decltype(std::forward<_Range>(__range).base()) {
+    return std::forward<_Range>(__range).base();
+  }
+
+  template <class _Range,
+            class _UnwrappedSubrange = typename __unwrapped_reverse_subrange<remove_cvref_t<_Range>>::type>
+    requires __is_sized_reverse_subrange<remove_cvref_t<_Range>>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range) const
+      noexcept(noexcept(_UnwrappedSubrange(__range.end().base(), __range.begin().base(), __range.size())))
+          -> decltype(_UnwrappedSubrange(__range.end().base(), __range.begin().base(), __range.size())) {
+    return _UnwrappedSubrange(__range.end().base(), __range.begin().base(), __range.size());
+  }
+
+  template <class _Range,
+            class _UnwrappedSubrange = typename __unwrapped_reverse_subrange<remove_cvref_t<_Range>>::type>
+    requires __is_unsized_reverse_subrange<remove_cvref_t<_Range>>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range) const
+      noexcept(noexcept(_UnwrappedSubrange(__range.end().base(), __range.begin().base())))
+          -> decltype(_UnwrappedSubrange(__range.end().base(), __range.begin().base())) {
+    return _UnwrappedSubrange(__range.end().base(), __range.begin().base());
+  }
+
+  template <class _Range>
+    requires(!__is_reverse_view<remove_cvref_t<_Range>> && !__is_sized_reverse_subrange<remove_cvref_t<_Range>> &&
+             !__is_unsized_reverse_subrange<remove_cvref_t<_Range>>)
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range) const noexcept(noexcept(reverse_view{
+      std::forward<_Range>(__range)})) -> decltype(reverse_view{std::forward<_Range>(__range)}) {
+    return reverse_view{std::forward<_Range>(__range)};
+  }
+};
+} // namespace __reverse
+
+inline namespace __cpo {
+inline constexpr auto reverse = __reverse::__fn{};
+} // namespace __cpo
+} // namespace views
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_REVERSE_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/single_view.h b/libcxx/include/__cxx03/__ranges/single_view.h
new file mode 100644
index 000000000000..45244f34994d
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/single_view.h
@@ -0,0 +1,111 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_SINGLE_VIEW_H
+#define _LIBCPP___RANGES_SINGLE_VIEW_H
+
+#include <__concepts/constructible.h>
+#include <__config>
+#include <__ranges/movable_box.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_object.h>
+#include <__utility/forward.h>
+#include <__utility/in_place.h>
+#include <__utility/move.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+#  if _LIBCPP_STD_VER >= 23
+template <move_constructible _Tp>
+#  else
+template <copy_constructible _Tp>
+#  endif
+  requires is_object_v<_Tp>
+class _LIBCPP_ABI_LLVM18_NO_UNIQUE_ADDRESS single_view : public view_interface<single_view<_Tp>> {
+  _LIBCPP_NO_UNIQUE_ADDRESS __movable_box<_Tp> __value_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI single_view()
+    requires default_initializable<_Tp>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit single_view(const _Tp& __t)
+#  if _LIBCPP_STD_VER >= 23
+    requires copy_constructible<_Tp>
+#  endif
+      : __value_(in_place, __t) {
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit single_view(_Tp&& __t) : __value_(in_place, std::move(__t)) {}
+
+  template <class... _Args>
+    requires constructible_from<_Tp, _Args...>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit single_view(in_place_t, _Args&&... __args)
+      : __value_{in_place, std::forward<_Args>(__args)...} {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp* begin() noexcept { return data(); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp* begin() const noexcept { return data(); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp* end() noexcept { return data() + 1; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp* end() const noexcept { return data() + 1; }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool empty() noexcept { return false; }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr size_t size() noexcept { return 1; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Tp* data() noexcept { return __value_.operator->(); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp* data() const noexcept { return __value_.operator->(); }
+};
+
+template <class _Tp>
+single_view(_Tp) -> single_view<_Tp>;
+
+namespace views {
+namespace __single_view {
+
+struct __fn : __range_adaptor_closure<__fn> {
+  template <class _Range>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range) const
+      noexcept(noexcept(single_view<decay_t<_Range&&>>(std::forward<_Range>(__range))))
+          -> decltype(single_view<decay_t<_Range&&>>(std::forward<_Range>(__range))) {
+    return single_view<decay_t<_Range&&>>(std::forward<_Range>(__range));
+  }
+};
+} // namespace __single_view
+
+inline namespace __cpo {
+inline constexpr auto single = __single_view::__fn{};
+} // namespace __cpo
+
+} // namespace views
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_SINGLE_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/size.h b/libcxx/include/__cxx03/__ranges/size.h
new file mode 100644
index 000000000000..40b0c6b6aad7
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/size.h
@@ -0,0 +1,137 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_SIZE_H
+#define _LIBCPP___RANGES_SIZE_H
+
+#include <__concepts/arithmetic.h>
+#include <__concepts/class_or_enum.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__ranges/access.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/make_signed.h>
+#include <__type_traits/make_unsigned.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/auto_cast.h>
+#include <__utility/declval.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+template <class>
+inline constexpr bool disable_sized_range = false;
+} // namespace ranges
+
+// [range.prim.size]
+
+namespace ranges {
+namespace __size {
+void size() = delete;
+
+template <class _Tp>
+concept __size_enabled = !disable_sized_range<remove_cvref_t<_Tp>>;
+
+template <class _Tp>
+concept __member_size = __size_enabled<_Tp> && requires(_Tp&& __t) {
+  { _LIBCPP_AUTO_CAST(__t.size()) } -> __integer_like;
+};
+
+template <class _Tp>
+concept __unqualified_size =
+    __size_enabled<_Tp> && !__member_size<_Tp> && __class_or_enum<remove_cvref_t<_Tp>> && requires(_Tp&& __t) {
+      { _LIBCPP_AUTO_CAST(size(__t)) } -> __integer_like;
+    };
+
+template <class _Tp>
+concept __difference =
+    !__member_size<_Tp> && !__unqualified_size<_Tp> && __class_or_enum<remove_cvref_t<_Tp>> && requires(_Tp&& __t) {
+      { ranges::begin(__t) } -> forward_iterator;
+      { ranges::end(__t) } -> sized_sentinel_for<decltype(ranges::begin(std::declval<_Tp>()))>;
+    };
+
+struct __fn {
+  // `[range.prim.size]`: the array case (for rvalues).
+  template <class _Tp, size_t _Sz>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr size_t operator()(_Tp (&&)[_Sz]) const noexcept {
+    return _Sz;
+  }
+
+  // `[range.prim.size]`: the array case (for lvalues).
+  template <class _Tp, size_t _Sz>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr size_t operator()(_Tp (&)[_Sz]) const noexcept {
+    return _Sz;
+  }
+
+  // `[range.prim.size]`: `auto(t.size())` is a valid expression.
+  template <__member_size _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __integer_like auto operator()(_Tp&& __t) const
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(__t.size()))) {
+    return _LIBCPP_AUTO_CAST(__t.size());
+  }
+
+  // `[range.prim.size]`: `auto(size(t))` is a valid expression.
+  template <__unqualified_size _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __integer_like auto operator()(_Tp&& __t) const
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(size(__t)))) {
+    return _LIBCPP_AUTO_CAST(size(__t));
+  }
+
+  // [range.prim.size]: the `to-unsigned-like` case.
+  template <__difference _Tp>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
+      noexcept(noexcept(std::__to_unsigned_like(ranges::end(__t) - ranges::begin(__t))))
+          -> decltype(std::__to_unsigned_like(ranges::end(__t) - ranges::begin(__t))) {
+    return std::__to_unsigned_like(ranges::end(__t) - ranges::begin(__t));
+  }
+};
+
+} // namespace __size
+
+inline namespace __cpo {
+inline constexpr auto size = __size::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+// [range.prim.ssize]
+
+namespace ranges {
+namespace __ssize {
+struct __fn {
+  template <class _Tp>
+    requires requires(_Tp&& __t) { ranges::size(__t); }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr integral auto operator()(_Tp&& __t) const
+      noexcept(noexcept(ranges::size(__t))) {
+    using _Signed = make_signed_t<decltype(ranges::size(__t))>;
+    if constexpr (sizeof(ptrdiff_t) > sizeof(_Signed))
+      return static_cast<ptrdiff_t>(ranges::size(__t));
+    else
+      return static_cast<_Signed>(ranges::size(__t));
+  }
+};
+} // namespace __ssize
+
+inline namespace __cpo {
+inline constexpr auto ssize = __ssize::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_SIZE_H
diff --git a/libcxx/include/__cxx03/__ranges/split_view.h b/libcxx/include/__cxx03/__ranges/split_view.h
new file mode 100644
index 000000000000..ce3606aedfef
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/split_view.h
@@ -0,0 +1,232 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_SPLIT_VIEW_H
+#define _LIBCPP___RANGES_SPLIT_VIEW_H
+
+#include <__algorithm/ranges_search.h>
+#include <__concepts/constructible.h>
+#include <__config>
+#include <__functional/bind_back.h>
+#include <__functional/ranges_operations.h>
+#include <__iterator/indirectly_comparable.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__ranges/access.h>
+#include <__ranges/all.h>
+#include <__ranges/concepts.h>
+#include <__ranges/empty.h>
+#include <__ranges/non_propagating_cache.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/single_view.h>
+#include <__ranges/subrange.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+template <forward_range _View, forward_range _Pattern>
+  requires view<_View> && view<_Pattern> &&
+           indirectly_comparable<iterator_t<_View>, iterator_t<_Pattern>, ranges::equal_to>
+class split_view : public view_interface<split_view<_View, _Pattern>> {
+private:
+  _LIBCPP_NO_UNIQUE_ADDRESS _View __base_       = _View();
+  _LIBCPP_NO_UNIQUE_ADDRESS _Pattern __pattern_ = _Pattern();
+  using _Cache                                  = __non_propagating_cache<subrange<iterator_t<_View>>>;
+  _Cache __cached_begin_                        = _Cache();
+
+  template <class, class>
+  friend struct __iterator;
+
+  template <class, class>
+  friend struct __sentinel;
+
+  struct __iterator;
+  struct __sentinel;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr subrange<iterator_t<_View>> __find_next(iterator_t<_View> __it) {
+    auto [__begin, __end] = ranges::search(subrange(__it, ranges::end(__base_)), __pattern_);
+    if (__begin != ranges::end(__base_) && ranges::empty(__pattern_)) {
+      ++__begin;
+      ++__end;
+    }
+    return {__begin, __end};
+  }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI split_view()
+    requires default_initializable<_View> && default_initializable<_Pattern>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23 split_view(_View __base, _Pattern __pattern)
+      : __base_(std::move(__base)), __pattern_(std::move((__pattern))) {}
+
+  template <forward_range _Range>
+    requires constructible_from<_View, views::all_t<_Range>> &&
+                 constructible_from<_Pattern, single_view<range_value_t<_Range>>>
+  _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23
+  split_view(_Range&& __range, range_value_t<_Range> __elem)
+      : __base_(views::all(std::forward<_Range>(__range))), __pattern_(views::single(std::move(__elem))) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
+    requires copy_constructible<_View>
+  {
+    return __base_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() {
+    if (!__cached_begin_.__has_value()) {
+      __cached_begin_.__emplace(__find_next(ranges::begin(__base_)));
+    }
+    return {*this, ranges::begin(__base_), *__cached_begin_};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() {
+    if constexpr (common_range<_View>) {
+      return __iterator{*this, ranges::end(__base_), {}};
+    } else {
+      return __sentinel{*this};
+    }
+  }
+};
+
+template <class _Range, class _Pattern>
+split_view(_Range&&, _Pattern&&) -> split_view<views::all_t<_Range>, views::all_t<_Pattern>>;
+
+template <forward_range _Range>
+split_view(_Range&&, range_value_t<_Range>) -> split_view<views::all_t<_Range>, single_view<range_value_t<_Range>>>;
+
+template <forward_range _View, forward_range _Pattern>
+  requires view<_View> && view<_Pattern> &&
+           indirectly_comparable<iterator_t<_View>, iterator_t<_Pattern>, ranges::equal_to>
+struct split_view<_View, _Pattern>::__iterator {
+private:
+  split_view* __parent_                                         = nullptr;
+  _LIBCPP_NO_UNIQUE_ADDRESS iterator_t<_View> __cur_            = iterator_t<_View>();
+  _LIBCPP_NO_UNIQUE_ADDRESS subrange<iterator_t<_View>> __next_ = subrange<iterator_t<_View>>();
+  bool __trailing_empty_                                        = false;
+
+  friend struct __sentinel;
+
+public:
+  using iterator_concept  = forward_iterator_tag;
+  using iterator_category = input_iterator_tag;
+  using value_type        = subrange<iterator_t<_View>>;
+  using difference_type   = range_difference_t<_View>;
+
+  _LIBCPP_HIDE_FROM_ABI __iterator() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator(
+      split_view<_View, _Pattern>& __parent, iterator_t<_View> __current, subrange<iterator_t<_View>> __next)
+      : __parent_(std::addressof(__parent)), __cur_(std::move(__current)), __next_(std::move(__next)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_View> base() const { return __cur_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr value_type operator*() const { return {__cur_, __next_.begin()}; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() {
+    __cur_ = __next_.begin();
+    if (__cur_ != ranges::end(__parent_->__base_)) {
+      __cur_ = __next_.end();
+      if (__cur_ == ranges::end(__parent_->__base_)) {
+        __trailing_empty_ = true;
+        __next_           = {__cur_, __cur_};
+      } else {
+        __next_ = __parent_->__find_next(__cur_);
+      }
+    } else {
+      __trailing_empty_ = false;
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator++(int) {
+    auto __tmp = *this;
+    ++*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __iterator& __y) {
+    return __x.__cur_ == __y.__cur_ && __x.__trailing_empty_ == __y.__trailing_empty_;
+  }
+};
+
+template <forward_range _View, forward_range _Pattern>
+  requires view<_View> && view<_Pattern> &&
+           indirectly_comparable<iterator_t<_View>, iterator_t<_Pattern>, ranges::equal_to>
+struct split_view<_View, _Pattern>::__sentinel {
+private:
+  _LIBCPP_NO_UNIQUE_ADDRESS sentinel_t<_View> __end_ = sentinel_t<_View>();
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __equals(const __iterator& __x, const __sentinel& __y) {
+    return __x.__cur_ == __y.__end_ && !__x.__trailing_empty_;
+  }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __sentinel() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __sentinel(split_view<_View, _Pattern>& __parent)
+      : __end_(ranges::end(__parent.__base_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __sentinel& __y) {
+    return __equals(__x, __y);
+  }
+};
+
+namespace views {
+namespace __split_view {
+struct __fn {
+  // clang-format off
+  template <class _Range, class _Pattern>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI
+  constexpr auto operator()(_Range&& __range, _Pattern&& __pattern) const
+    noexcept(noexcept(split_view(std::forward<_Range>(__range), std::forward<_Pattern>(__pattern))))
+    -> decltype(      split_view(std::forward<_Range>(__range), std::forward<_Pattern>(__pattern)))
+    { return          split_view(std::forward<_Range>(__range), std::forward<_Pattern>(__pattern)); }
+  // clang-format on
+
+  template <class _Pattern>
+    requires constructible_from<decay_t<_Pattern>, _Pattern>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Pattern&& __pattern) const
+      noexcept(is_nothrow_constructible_v<decay_t<_Pattern>, _Pattern>) {
+    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Pattern>(__pattern)));
+  }
+};
+} // namespace __split_view
+
+inline namespace __cpo {
+inline constexpr auto split = __split_view::__fn{};
+} // namespace __cpo
+} // namespace views
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_SPLIT_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/subrange.h b/libcxx/include/__cxx03/__ranges/subrange.h
new file mode 100644
index 000000000000..aba584ef9335
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/subrange.h
@@ -0,0 +1,273 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_SUBRANGE_H
+#define _LIBCPP___RANGES_SUBRANGE_H
+
+#include <__assert>
+#include <__concepts/constructible.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/copyable.h>
+#include <__concepts/derived_from.h>
+#include <__concepts/different_from.h>
+#include <__config>
+#include <__fwd/subrange.h>
+#include <__iterator/advance.h>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/size.h>
+#include <__ranges/view_interface.h>
+#include <__tuple/tuple_element.h>
+#include <__tuple/tuple_like_no_subrange.h>
+#include <__tuple/tuple_size.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/make_unsigned.h>
+#include <__type_traits/remove_const.h>
+#include <__type_traits/remove_pointer.h>
+#include <__utility/move.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+template <class _From, class _To>
+concept __uses_nonqualification_pointer_conversion =
+    is_pointer_v<_From> && is_pointer_v<_To> &&
+    !convertible_to<remove_pointer_t<_From> (*)[], remove_pointer_t<_To> (*)[]>;
+
+template <class _From, class _To>
+concept __convertible_to_non_slicing =
+    convertible_to<_From, _To> && !__uses_nonqualification_pointer_conversion<decay_t<_From>, decay_t<_To>>;
+
+template <class _Pair, class _Iter, class _Sent>
+concept __pair_like_convertible_from =
+    !range<_Pair> && __pair_like_no_subrange<_Pair> && constructible_from<_Pair, _Iter, _Sent> &&
+    __convertible_to_non_slicing<_Iter, tuple_element_t<0, _Pair>> && convertible_to<_Sent, tuple_element_t<1, _Pair>>;
+
+template <input_or_output_iterator _Iter,
+          sentinel_for<_Iter> _Sent = _Iter,
+          subrange_kind _Kind       = sized_sentinel_for<_Sent, _Iter> ? subrange_kind::sized : subrange_kind::unsized>
+  requires(_Kind == subrange_kind::sized || !sized_sentinel_for<_Sent, _Iter>)
+class _LIBCPP_TEMPLATE_VIS subrange : public view_interface<subrange<_Iter, _Sent, _Kind>> {
+public:
+  // Note: this is an internal implementation detail that is public only for internal usage.
+  static constexpr bool _StoreSize = (_Kind == subrange_kind::sized && !sized_sentinel_for<_Sent, _Iter>);
+
+private:
+  static constexpr bool _MustProvideSizeAtConstruction = !_StoreSize; // just to improve compiler diagnostics
+  struct _Empty {
+    _LIBCPP_HIDE_FROM_ABI constexpr _Empty(auto) noexcept {}
+  };
+  using _Size = conditional_t<_StoreSize, make_unsigned_t<iter_difference_t<_Iter>>, _Empty>;
+  _LIBCPP_NO_UNIQUE_ADDRESS _Iter __begin_ = _Iter();
+  _LIBCPP_NO_UNIQUE_ADDRESS _Sent __end_   = _Sent();
+  _LIBCPP_NO_UNIQUE_ADDRESS _Size __size_  = 0;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI subrange()
+    requires default_initializable<_Iter>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr subrange(__convertible_to_non_slicing<_Iter> auto __iter, _Sent __sent)
+    requires _MustProvideSizeAtConstruction
+      : __begin_(std::move(__iter)), __end_(std::move(__sent)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr subrange(
+      __convertible_to_non_slicing<_Iter> auto __iter, _Sent __sent, make_unsigned_t<iter_difference_t<_Iter>> __n)
+    requires(_Kind == subrange_kind::sized)
+      : __begin_(std::move(__iter)), __end_(std::move(__sent)), __size_(__n) {
+    if constexpr (sized_sentinel_for<_Sent, _Iter>)
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS((__end_ - __begin_) == static_cast<iter_difference_t<_Iter>>(__n),
+                                          "std::ranges::subrange was passed an invalid size hint");
+  }
+
+  template <__different_from<subrange> _Range>
+    requires borrowed_range<_Range> && __convertible_to_non_slicing<iterator_t<_Range>, _Iter> &&
+             convertible_to<sentinel_t<_Range>, _Sent>
+             _LIBCPP_HIDE_FROM_ABI constexpr subrange(_Range&& __range)
+               requires(!_StoreSize)
+      : subrange(ranges::begin(__range), ranges::end(__range)) {}
+
+  template <__different_from<subrange> _Range>
+    requires borrowed_range<_Range> && __convertible_to_non_slicing<iterator_t<_Range>, _Iter> &&
+             convertible_to<sentinel_t<_Range>, _Sent>
+             _LIBCPP_HIDE_FROM_ABI constexpr subrange(_Range&& __range)
+               requires _StoreSize && sized_range<_Range>
+      : subrange(__range, ranges::size(__range)) {}
+
+  template <borrowed_range _Range>
+    requires __convertible_to_non_slicing<iterator_t<_Range>, _Iter> &&
+             convertible_to<sentinel_t<_Range>, _Sent>
+             _LIBCPP_HIDE_FROM_ABI constexpr subrange(_Range&& __range, make_unsigned_t<iter_difference_t<_Iter>> __n)
+               requires(_Kind == subrange_kind::sized)
+      : subrange(ranges::begin(__range), ranges::end(__range), __n) {}
+
+  template <__pair_like_convertible_from<const _Iter&, const _Sent&> _Pair>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator _Pair() const {
+    return _Pair(__begin_, __end_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter begin() const
+    requires copyable<_Iter>
+  {
+    return __begin_;
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Iter begin()
+    requires(!copyable<_Iter>)
+  {
+    return std::move(__begin_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Sent end() const { return __end_; }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool empty() const { return __begin_ == __end_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr make_unsigned_t<iter_difference_t<_Iter>> size() const
+    requires(_Kind == subrange_kind::sized)
+  {
+    if constexpr (_StoreSize)
+      return __size_;
+    else
+      return std::__to_unsigned_like(__end_ - __begin_);
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr subrange next(iter_difference_t<_Iter> __n = 1) const&
+    requires forward_iterator<_Iter>
+  {
+    auto __tmp = *this;
+    __tmp.advance(__n);
+    return __tmp;
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr subrange next(iter_difference_t<_Iter> __n = 1) && {
+    advance(__n);
+    return std::move(*this);
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr subrange prev(iter_difference_t<_Iter> __n = 1) const
+    requires bidirectional_iterator<_Iter>
+  {
+    auto __tmp = *this;
+    __tmp.advance(-__n);
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr subrange& advance(iter_difference_t<_Iter> __n) {
+    if constexpr (bidirectional_iterator<_Iter>) {
+      if (__n < 0) {
+        ranges::advance(__begin_, __n);
+        if constexpr (_StoreSize)
+          __size_ += std::__to_unsigned_like(-__n);
+        return *this;
+      }
+    }
+
+    auto __d = __n - ranges::advance(__begin_, __n, __end_);
+    if constexpr (_StoreSize)
+      __size_ -= std::__to_unsigned_like(__d);
+    return *this;
+  }
+};
+
+template <input_or_output_iterator _Iter, sentinel_for<_Iter> _Sent>
+subrange(_Iter, _Sent) -> subrange<_Iter, _Sent>;
+
+template <input_or_output_iterator _Iter, sentinel_for<_Iter> _Sent>
+subrange(_Iter, _Sent, make_unsigned_t<iter_difference_t<_Iter>>) -> subrange<_Iter, _Sent, subrange_kind::sized>;
+
+template <borrowed_range _Range>
+subrange(_Range&&) -> subrange<iterator_t<_Range>,
+                               sentinel_t<_Range>,
+                               (sized_range<_Range> || sized_sentinel_for<sentinel_t<_Range>, iterator_t<_Range>>)
+                                   ? subrange_kind::sized
+                                   : subrange_kind::unsized>;
+
+template <borrowed_range _Range>
+subrange(_Range&&, make_unsigned_t<range_difference_t<_Range>>)
+    -> subrange<iterator_t<_Range>, sentinel_t<_Range>, subrange_kind::sized>;
+
+template <size_t _Index, class _Iter, class _Sent, subrange_kind _Kind>
+  requires((_Index == 0 && copyable<_Iter>) || _Index == 1)
+_LIBCPP_HIDE_FROM_ABI constexpr auto get(const subrange<_Iter, _Sent, _Kind>& __subrange) {
+  if constexpr (_Index == 0)
+    return __subrange.begin();
+  else
+    return __subrange.end();
+}
+
+template <size_t _Index, class _Iter, class _Sent, subrange_kind _Kind>
+  requires(_Index < 2)
+_LIBCPP_HIDE_FROM_ABI constexpr auto get(subrange<_Iter, _Sent, _Kind>&& __subrange) {
+  if constexpr (_Index == 0)
+    return __subrange.begin();
+  else
+    return __subrange.end();
+}
+
+template <class _Ip, class _Sp, subrange_kind _Kp>
+inline constexpr bool enable_borrowed_range<subrange<_Ip, _Sp, _Kp>> = true;
+
+template <range _Rp>
+using borrowed_subrange_t = _If<borrowed_range<_Rp>, subrange<iterator_t<_Rp>>, dangling>;
+} // namespace ranges
+
+// [range.subrange.general]
+
+using ranges::get;
+
+// [ranges.syn]
+
+template <class _Ip, class _Sp, ranges::subrange_kind _Kp>
+struct tuple_size<ranges::subrange<_Ip, _Sp, _Kp>> : integral_constant<size_t, 2> {};
+
+template <class _Ip, class _Sp, ranges::subrange_kind _Kp>
+struct tuple_element<0, ranges::subrange<_Ip, _Sp, _Kp>> {
+  using type = _Ip;
+};
+
+template <class _Ip, class _Sp, ranges::subrange_kind _Kp>
+struct tuple_element<1, ranges::subrange<_Ip, _Sp, _Kp>> {
+  using type = _Sp;
+};
+
+template <class _Ip, class _Sp, ranges::subrange_kind _Kp>
+struct tuple_element<0, const ranges::subrange<_Ip, _Sp, _Kp>> {
+  using type = _Ip;
+};
+
+template <class _Ip, class _Sp, ranges::subrange_kind _Kp>
+struct tuple_element<1, const ranges::subrange<_Ip, _Sp, _Kp>> {
+  using type = _Sp;
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_SUBRANGE_H
diff --git a/libcxx/include/__cxx03/__ranges/take_view.h b/libcxx/include/__cxx03/__ranges/take_view.h
new file mode 100644
index 000000000000..27ca8155a69b
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/take_view.h
@@ -0,0 +1,369 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_TAKE_VIEW_H
+#define _LIBCPP___RANGES_TAKE_VIEW_H
+
+#include <__algorithm/min.h>
+#include <__algorithm/ranges_min.h>
+#include <__assert>
+#include <__concepts/constructible.h>
+#include <__concepts/convertible_to.h>
+#include <__config>
+#include <__functional/bind_back.h>
+#include <__fwd/span.h>
+#include <__fwd/string_view.h>
+#include <__iterator/concepts.h>
+#include <__iterator/counted_iterator.h>
+#include <__iterator/default_sentinel.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/all.h>
+#include <__ranges/concepts.h>
+#include <__ranges/empty_view.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/iota_view.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/repeat_view.h>
+#include <__ranges/size.h>
+#include <__ranges/subrange.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/maybe_const.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/auto_cast.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+template <view _View>
+class take_view : public view_interface<take_view<_View>> {
+  _LIBCPP_NO_UNIQUE_ADDRESS _View __base_ = _View();
+  range_difference_t<_View> __count_      = 0;
+
+  template <bool>
+  class __sentinel;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI take_view()
+    requires default_initializable<_View>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23
+  take_view(_View __base, range_difference_t<_View> __count)
+      : __base_(std::move(__base)), __count_(__count) {
+    _LIBCPP_ASSERT_UNCATEGORIZED(__count >= 0, "count has to be greater than or equal to zero");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
+    requires copy_constructible<_View>
+  {
+    return __base_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin()
+    requires(!__simple_view<_View>)
+  {
+    if constexpr (sized_range<_View>) {
+      if constexpr (random_access_range<_View>) {
+        return ranges::begin(__base_);
+      } else {
+        using _DifferenceT = range_difference_t<_View>;
+        auto __size        = size();
+        return counted_iterator(ranges::begin(__base_), static_cast<_DifferenceT>(__size));
+      }
+    } else {
+      return counted_iterator(ranges::begin(__base_), __count_);
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
+    requires range<const _View>
+  {
+    if constexpr (sized_range<const _View>) {
+      if constexpr (random_access_range<const _View>) {
+        return ranges::begin(__base_);
+      } else {
+        using _DifferenceT = range_difference_t<const _View>;
+        auto __size        = size();
+        return counted_iterator(ranges::begin(__base_), static_cast<_DifferenceT>(__size));
+      }
+    } else {
+      return counted_iterator(ranges::begin(__base_), __count_);
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end()
+    requires(!__simple_view<_View>)
+  {
+    if constexpr (sized_range<_View>) {
+      if constexpr (random_access_range<_View>) {
+        return ranges::begin(__base_) + size();
+      } else {
+        return default_sentinel;
+      }
+    } else {
+      return __sentinel<false>{ranges::end(__base_)};
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
+    requires range<const _View>
+  {
+    if constexpr (sized_range<const _View>) {
+      if constexpr (random_access_range<const _View>) {
+        return ranges::begin(__base_) + size();
+      } else {
+        return default_sentinel;
+      }
+    } else {
+      return __sentinel<true>{ranges::end(__base_)};
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size()
+    requires sized_range<_View>
+  {
+    auto __n = ranges::size(__base_);
+    return ranges::min(__n, static_cast<decltype(__n)>(__count_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
+    requires sized_range<const _View>
+  {
+    auto __n = ranges::size(__base_);
+    return ranges::min(__n, static_cast<decltype(__n)>(__count_));
+  }
+};
+
+template <view _View>
+template <bool _Const>
+class take_view<_View>::__sentinel {
+  using _Base = __maybe_const<_Const, _View>;
+  template <bool _OtherConst>
+  using _Iter                                        = counted_iterator<iterator_t<__maybe_const<_OtherConst, _View>>>;
+  _LIBCPP_NO_UNIQUE_ADDRESS sentinel_t<_Base> __end_ = sentinel_t<_Base>();
+
+  template <bool>
+  friend class take_view<_View>::__sentinel;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __sentinel() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __sentinel(sentinel_t<_Base> __end) : __end_(std::move(__end)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __sentinel(__sentinel<!_Const> __s)
+    requires _Const && convertible_to<sentinel_t<_View>, sentinel_t<_Base>>
+      : __end_(std::move(__s.__end_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr sentinel_t<_Base> base() const { return __end_; }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const _Iter<_Const>& __lhs, const __sentinel& __rhs) {
+    return __lhs.count() == 0 || __lhs.base() == __rhs.__end_;
+  }
+
+  template <bool _OtherConst = !_Const>
+    requires sentinel_for<sentinel_t<_Base>, iterator_t<__maybe_const<_OtherConst, _View>>>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const _Iter<_OtherConst>& __lhs, const __sentinel& __rhs) {
+    return __lhs.count() == 0 || __lhs.base() == __rhs.__end_;
+  }
+};
+
+template <class _Range>
+take_view(_Range&&, range_difference_t<_Range>) -> take_view<views::all_t<_Range>>;
+
+template <class _Tp>
+inline constexpr bool enable_borrowed_range<take_view<_Tp>> = enable_borrowed_range<_Tp>;
+
+namespace views {
+namespace __take {
+
+template <class _Tp>
+inline constexpr bool __is_empty_view = false;
+
+template <class _Tp>
+inline constexpr bool __is_empty_view<empty_view<_Tp>> = true;
+
+template <class _Tp>
+inline constexpr bool __is_passthrough_specialization = false;
+
+template <class _Tp, size_t _Extent>
+inline constexpr bool __is_passthrough_specialization<span<_Tp, _Extent>> = true;
+
+template <class _CharT, class _Traits>
+inline constexpr bool __is_passthrough_specialization<basic_string_view<_CharT, _Traits>> = true;
+
+template <class _Iter, class _Sent, subrange_kind _Kind>
+inline constexpr bool __is_passthrough_specialization<subrange<_Iter, _Sent, _Kind>> = true;
+
+template <class _Tp>
+inline constexpr bool __is_iota_specialization = false;
+
+template <class _Np, class _Bound>
+inline constexpr bool __is_iota_specialization<iota_view<_Np, _Bound>> = true;
+
+template <class _Tp>
+struct __passthrough_type;
+
+template <class _Tp, size_t _Extent>
+struct __passthrough_type<span<_Tp, _Extent>> {
+  using type = span<_Tp>;
+};
+
+template <class _CharT, class _Traits>
+struct __passthrough_type<basic_string_view<_CharT, _Traits>> {
+  using type = basic_string_view<_CharT, _Traits>;
+};
+
+template <class _Iter, class _Sent, subrange_kind _Kind>
+  requires requires { typename subrange<_Iter>; }
+struct __passthrough_type<subrange<_Iter, _Sent, _Kind>> {
+  using type = subrange<_Iter>;
+};
+
+template <class _Tp>
+using __passthrough_type_t = typename __passthrough_type<_Tp>::type;
+
+struct __fn {
+  // [range.take.overview]: the `empty_view` case.
+  template <class _Range, convertible_to<range_difference_t<_Range>> _Np>
+    requires __is_empty_view<remove_cvref_t<_Range>>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range, _Np&&) const
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(std::forward<_Range>(__range))))
+          -> decltype(_LIBCPP_AUTO_CAST(std::forward<_Range>(__range))) {
+    return _LIBCPP_AUTO_CAST(std::forward<_Range>(__range));
+  }
+
+  // [range.take.overview]: the `span | basic_string_view | subrange` case.
+  template <class _Range,
+            convertible_to<range_difference_t<_Range>> _Np,
+            class _RawRange = remove_cvref_t<_Range>,
+            class _Dist     = range_difference_t<_Range>>
+    requires(!__is_empty_view<_RawRange> && random_access_range<_RawRange> && sized_range<_RawRange> &&
+             __is_passthrough_specialization<_RawRange>)
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto
+  operator()(_Range&& __rng, _Np&& __n) const noexcept(noexcept(__passthrough_type_t<_RawRange>(
+      ranges::begin(__rng), ranges::begin(__rng) + std::min<_Dist>(ranges::distance(__rng), std::forward<_Np>(__n)))))
+      -> decltype(__passthrough_type_t<_RawRange>(
+          // Note: deliberately not forwarding `__rng` to guard against double moves.
+          ranges::begin(__rng),
+          ranges::begin(__rng) + std::min<_Dist>(ranges::distance(__rng), std::forward<_Np>(__n)))) {
+    return __passthrough_type_t<_RawRange>(
+        ranges::begin(__rng), ranges::begin(__rng) + std::min<_Dist>(ranges::distance(__rng), std::forward<_Np>(__n)));
+  }
+
+  // [range.take.overview]: the `iota_view` case.
+  // clang-format off
+  template <class _Range,
+            convertible_to<range_difference_t<_Range>> _Np,
+            class _RawRange = remove_cvref_t<_Range>,
+            class _Dist     = range_difference_t<_Range>>
+    requires (!__is_empty_view<_RawRange> &&
+              random_access_range<_RawRange> &&
+              sized_range<_RawRange> &&
+              __is_iota_specialization<_RawRange>)
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI
+  constexpr auto operator()(_Range&& __rng, _Np&& __n) const
+    noexcept(noexcept(ranges::iota_view(
+                              *ranges::begin(__rng),
+                              *(ranges::begin(__rng) + std::min<_Dist>(ranges::distance(__rng), std::forward<_Np>(__n)))
+                              )))
+    -> decltype(      ranges::iota_view(
+                              // Note: deliberately not forwarding `__rng` to guard against double moves.
+                              *ranges::begin(__rng),
+                              *(ranges::begin(__rng) + std::min<_Dist>(ranges::distance(__rng), std::forward<_Np>(__n)))
+                              ))
+    { return          ranges::iota_view(
+                              *ranges::begin(__rng),
+                              *(ranges::begin(__rng) + std::min<_Dist>(ranges::distance(__rng), std::forward<_Np>(__n)))
+                              ); }
+
+#if _LIBCPP_STD_VER >= 23
+  // [range.take.overview]: the `repeat_view` "_RawRange models sized_range" case.
+  template <class _Range,
+            convertible_to<range_difference_t<_Range>> _Np,
+            class _RawRange = remove_cvref_t<_Range>,
+            class _Dist     = range_difference_t<_Range>>
+    requires(__is_repeat_specialization<_RawRange> && sized_range<_RawRange>)
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range, _Np&& __n) const
+    noexcept(noexcept(views::repeat(*__range.__value_, std::min<_Dist>(ranges::distance(__range), std::forward<_Np>(__n)))))
+    -> decltype(      views::repeat(*__range.__value_, std::min<_Dist>(ranges::distance(__range), std::forward<_Np>(__n))))
+    { return          views::repeat(*__range.__value_, std::min<_Dist>(ranges::distance(__range), std::forward<_Np>(__n))); }
+
+  // [range.take.overview]: the `repeat_view` "otherwise" case.
+  template <class _Range,
+            convertible_to<range_difference_t<_Range>> _Np,
+            class _RawRange = remove_cvref_t<_Range>,
+            class _Dist     = range_difference_t<_Range>>
+    requires(__is_repeat_specialization<_RawRange> && !sized_range<_RawRange>)
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range, _Np&& __n) const
+    noexcept(noexcept(views::repeat(*__range.__value_, static_cast<_Dist>(__n))))
+    -> decltype(      views::repeat(*__range.__value_, static_cast<_Dist>(__n)))
+    { return          views::repeat(*__range.__value_, static_cast<_Dist>(__n)); }
+#endif
+  // clang-format on
+
+  // [range.take.overview]: the "otherwise" case.
+  template <class _Range, convertible_to<range_difference_t<_Range>> _Np, class _RawRange = remove_cvref_t<_Range>>
+  // Note: without specifically excluding the other cases, GCC sees this overload as ambiguous with the other
+  // overloads.
+    requires(!(__is_empty_view<_RawRange> ||
+#  if _LIBCPP_STD_VER >= 23
+               __is_repeat_specialization<_RawRange> ||
+#  endif
+               (__is_iota_specialization<_RawRange> && sized_range<_RawRange> && random_access_range<_RawRange>) ||
+               (__is_passthrough_specialization<_RawRange> && sized_range<_RawRange> &&
+                random_access_range<_RawRange>)))
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range, _Np&& __n) const
+      noexcept(noexcept(take_view(std::forward<_Range>(__range), std::forward<_Np>(__n))))
+          -> decltype(take_view(std::forward<_Range>(__range), std::forward<_Np>(__n))) {
+    return take_view(std::forward<_Range>(__range), std::forward<_Np>(__n));
+  }
+
+  template <class _Np>
+    requires constructible_from<decay_t<_Np>, _Np>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Np&& __n) const
+      noexcept(is_nothrow_constructible_v<decay_t<_Np>, _Np>) {
+    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Np>(__n)));
+  }
+};
+
+} // namespace __take
+
+inline namespace __cpo {
+inline constexpr auto take = __take::__fn{};
+} // namespace __cpo
+} // namespace views
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_TAKE_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/take_while_view.h b/libcxx/include/__cxx03/__ranges/take_while_view.h
new file mode 100644
index 000000000000..77ea9f7bb813
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/take_while_view.h
@@ -0,0 +1,170 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_TAKE_WHILE_VIEW_H
+#define _LIBCPP___RANGES_TAKE_WHILE_VIEW_H
+
+#include <__concepts/constructible.h>
+#include <__concepts/convertible_to.h>
+#include <__config>
+#include <__functional/bind_back.h>
+#include <__functional/invoke.h>
+#include <__iterator/concepts.h>
+#include <__memory/addressof.h>
+#include <__ranges/access.h>
+#include <__ranges/all.h>
+#include <__ranges/concepts.h>
+#include <__ranges/movable_box.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_object.h>
+#include <__type_traits/maybe_const.h>
+#include <__utility/forward.h>
+#include <__utility/in_place.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+template <view _View, class _Pred>
+  requires input_range<_View> && is_object_v<_Pred> && indirect_unary_predicate<const _Pred, iterator_t<_View>>
+class _LIBCPP_ABI_LLVM18_NO_UNIQUE_ADDRESS take_while_view : public view_interface<take_while_view<_View, _Pred>> {
+  template <bool>
+  class __sentinel;
+
+  _LIBCPP_NO_UNIQUE_ADDRESS _View __base_ = _View();
+  _LIBCPP_NO_UNIQUE_ADDRESS __movable_box<_Pred> __pred_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI take_while_view()
+    requires default_initializable<_View> && default_initializable<_Pred>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23 take_while_view(_View __base, _Pred __pred)
+      : __base_(std::move(__base)), __pred_(std::in_place, std::move(__pred)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
+    requires copy_constructible<_View>
+  {
+    return __base_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Pred& pred() const { return *__pred_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin()
+    requires(!__simple_view<_View>)
+  {
+    return ranges::begin(__base_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
+    requires range<const _View> && indirect_unary_predicate<const _Pred, iterator_t<const _View>>
+  {
+    return ranges::begin(__base_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end()
+    requires(!__simple_view<_View>)
+  {
+    return __sentinel</*_Const=*/false>(ranges::end(__base_), std::addressof(*__pred_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
+    requires range<const _View> && indirect_unary_predicate<const _Pred, iterator_t<const _View>>
+  {
+    return __sentinel</*_Const=*/true>(ranges::end(__base_), std::addressof(*__pred_));
+  }
+};
+
+template <class _Range, class _Pred>
+take_while_view(_Range&&, _Pred) -> take_while_view<views::all_t<_Range>, _Pred>;
+
+template <view _View, class _Pred>
+  requires input_range<_View> && is_object_v<_Pred> && indirect_unary_predicate<const _Pred, iterator_t<_View>>
+template <bool _Const>
+class take_while_view<_View, _Pred>::__sentinel {
+  using _Base = __maybe_const<_Const, _View>;
+
+  sentinel_t<_Base> __end_ = sentinel_t<_Base>();
+  const _Pred* __pred_     = nullptr;
+
+  friend class __sentinel<!_Const>;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __sentinel() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __sentinel(sentinel_t<_Base> __end, const _Pred* __pred)
+      : __end_(std::move(__end)), __pred_(__pred) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __sentinel(__sentinel<!_Const> __s)
+    requires _Const && convertible_to<sentinel_t<_View>, sentinel_t<_Base>>
+      : __end_(std::move(__s.__end_)), __pred_(__s.__pred_) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr sentinel_t<_Base> base() const { return __end_; }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const iterator_t<_Base>& __x, const __sentinel& __y) {
+    return __x == __y.__end_ || !std::invoke(*__y.__pred_, *__x);
+  }
+
+  template <bool _OtherConst = !_Const>
+    requires sentinel_for<sentinel_t<_Base>, iterator_t<__maybe_const<_OtherConst, _View>>>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool
+  operator==(const iterator_t<__maybe_const<_OtherConst, _View>>& __x, const __sentinel& __y) {
+    return __x == __y.__end_ || !std::invoke(*__y.__pred_, *__x);
+  }
+};
+
+namespace views {
+namespace __take_while {
+
+struct __fn {
+  template <class _Range, class _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range, _Pred&& __pred) const
+      noexcept(noexcept(/**/ take_while_view(std::forward<_Range>(__range), std::forward<_Pred>(__pred))))
+          -> decltype(/*--*/ take_while_view(std::forward<_Range>(__range), std::forward<_Pred>(__pred))) {
+    return /*-------------*/ take_while_view(std::forward<_Range>(__range), std::forward<_Pred>(__pred));
+  }
+
+  template <class _Pred>
+    requires constructible_from<decay_t<_Pred>, _Pred>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Pred&& __pred) const
+      noexcept(is_nothrow_constructible_v<decay_t<_Pred>, _Pred>) {
+    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Pred>(__pred)));
+  }
+};
+
+} // namespace __take_while
+
+inline namespace __cpo {
+inline constexpr auto take_while = __take_while::__fn{};
+} // namespace __cpo
+} // namespace views
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_TAKE_WHILE_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/to.h b/libcxx/include/__cxx03/__ranges/to.h
new file mode 100644
index 000000000000..e0abe6290b8f
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/to.h
@@ -0,0 +1,245 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_TO_H
+#define _LIBCPP___RANGES_TO_H
+
+#include <__algorithm/ranges_copy.h>
+#include <__concepts/constructible.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/derived_from.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__functional/bind_back.h>
+#include <__iterator/back_insert_iterator.h>
+#include <__iterator/insert_iterator.h>
+#include <__iterator/iterator_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/from_range.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/ref_view.h>
+#include <__ranges/size.h>
+#include <__ranges/transform_view.h>
+#include <__type_traits/add_pointer.h>
+#include <__type_traits/is_const.h>
+#include <__type_traits/is_volatile.h>
+#include <__type_traits/type_identity.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+namespace ranges {
+
+template <class _Container>
+constexpr bool __reservable_container =
+    sized_range<_Container> && requires(_Container& __c, range_size_t<_Container> __n) {
+      __c.reserve(__n);
+      { __c.capacity() } -> same_as<decltype(__n)>;
+      { __c.max_size() } -> same_as<decltype(__n)>;
+    };
+
+template <class _Container, class _Ref>
+constexpr bool __container_insertable = requires(_Container& __c, _Ref&& __ref) {
+  requires(
+      requires { __c.push_back(std::forward<_Ref>(__ref)); } ||
+      requires { __c.insert(__c.end(), std::forward<_Ref>(__ref)); });
+};
+
+template <class _Ref, class _Container>
+_LIBCPP_HIDE_FROM_ABI constexpr auto __container_inserter(_Container& __c) {
+  if constexpr (requires { __c.push_back(std::declval<_Ref>()); }) {
+    return std::back_inserter(__c);
+  } else {
+    return std::inserter(__c, __c.end());
+  }
+}
+
+// Note: making this a concept allows short-circuiting the second condition.
+template <class _Container, class _Range>
+concept __try_non_recursive_conversion =
+    !input_range<_Container> || convertible_to<range_reference_t<_Range>, range_value_t<_Container>>;
+
+template <class _Container, class _Range, class... _Args>
+concept __constructible_from_iter_pair =
+    common_range<_Range> && requires { typename iterator_traits<iterator_t<_Range>>::iterator_category; } &&
+    derived_from<typename iterator_traits<iterator_t<_Range>>::iterator_category, input_iterator_tag> &&
+    constructible_from<_Container, iterator_t<_Range>, sentinel_t<_Range>, _Args...>;
+
+template <class>
+concept __always_false = false;
+
+// `ranges::to` base template -- the `_Container` type is a simple type template parameter.
+template <class _Container, input_range _Range, class... _Args>
+  requires(!view<_Container>)
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Container to(_Range&& __range, _Args&&... __args) {
+  // Mandates: C is a cv-unqualified class type.
+  static_assert(!is_const_v<_Container>, "The target container cannot be const-qualified, please remove the const");
+  static_assert(
+      !is_volatile_v<_Container>, "The target container cannot be volatile-qualified, please remove the volatile");
+
+  // First see if the non-recursive case applies -- the conversion target is either:
+  // - a range with a convertible value type;
+  // - a non-range type which might support being created from the input argument(s) (e.g. an `optional`).
+  if constexpr (__try_non_recursive_conversion<_Container, _Range>) {
+    // Case 1 -- construct directly from the given range.
+    if constexpr (constructible_from<_Container, _Range, _Args...>) {
+      return _Container(std::forward<_Range>(__range), std::forward<_Args>(__args)...);
+    }
+
+    // Case 2 -- construct using the `from_range_t` tagged constructor.
+    else if constexpr (constructible_from<_Container, from_range_t, _Range, _Args...>) {
+      return _Container(from_range, std::forward<_Range>(__range), std::forward<_Args>(__args)...);
+    }
+
+    // Case 3 -- construct from a begin-end iterator pair.
+    else if constexpr (__constructible_from_iter_pair<_Container, _Range, _Args...>) {
+      return _Container(ranges::begin(__range), ranges::end(__range), std::forward<_Args>(__args)...);
+    }
+
+    // Case 4 -- default-construct (or construct from the extra arguments) and insert, reserving the size if possible.
+    else if constexpr (constructible_from<_Container, _Args...> &&
+                       __container_insertable<_Container, range_reference_t<_Range>>) {
+      _Container __result(std::forward<_Args>(__args)...);
+      if constexpr (sized_range<_Range> && __reservable_container<_Container>) {
+        __result.reserve(static_cast<range_size_t<_Container>>(ranges::size(__range)));
+      }
+
+      ranges::copy(__range, ranges::__container_inserter<range_reference_t<_Range>>(__result));
+
+      return __result;
+
+    } else {
+      static_assert(__always_false<_Container>, "ranges::to: unable to convert to the given container type.");
+    }
+
+    // Try the recursive case.
+  } else if constexpr (input_range<range_reference_t<_Range>>) {
+    return ranges::to<_Container>(
+        ref_view(__range) | views::transform([](auto&& __elem) {
+          return ranges::to<range_value_t<_Container>>(std::forward<decltype(__elem)>(__elem));
+        }),
+        std::forward<_Args>(__args)...);
+
+  } else {
+    static_assert(__always_false<_Container>, "ranges::to: unable to convert to the given container type.");
+  }
+}
+
+template <class _Range>
+struct __minimal_input_iterator {
+  using iterator_category = input_iterator_tag;
+  using value_type        = range_value_t<_Range>;
+  using difference_type   = ptrdiff_t;
+  using pointer           = add_pointer_t<range_reference_t<_Range>>;
+  using reference         = range_reference_t<_Range>;
+
+  reference operator*() const;
+  pointer operator->() const;
+  __minimal_input_iterator& operator++();
+  __minimal_input_iterator operator++(int);
+  bool operator==(const __minimal_input_iterator&) const;
+};
+
+// Deduces the full type of the container from the given template template parameter.
+template <template <class...> class _Container, input_range _Range, class... _Args>
+struct _Deducer {
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __deduce_func() {
+    using _InputIter = __minimal_input_iterator<_Range>;
+
+    // Case 1 -- can construct directly from the given range.
+    if constexpr (requires { _Container(std::declval<_Range>(), std::declval<_Args>()...); }) {
+      using _Result = decltype( //
+          _Container(std::declval<_Range>(), std::declval<_Args>()...));
+      return type_identity<_Result>{};
+
+      // Case 2 -- can construct from the given range using the `from_range_t` tagged constructor.
+    } else if constexpr ( //
+        requires { _Container(from_range, std::declval<_Range>(), std::declval<_Args>()...); }) {
+      using _Result = //
+          decltype(_Container(from_range, std::declval<_Range>(), std::declval<_Args>()...));
+      return type_identity<_Result>{};
+
+      // Case 3 -- can construct from a begin-end iterator pair.
+    } else if constexpr ( //
+        requires { _Container(std::declval<_InputIter>(), std::declval<_InputIter>(), std::declval<_Args>()...); }) {
+      using _Result =
+          decltype(_Container(std::declval<_InputIter>(), std::declval<_InputIter>(), std::declval<_Args>()...));
+      return type_identity<_Result>{};
+
+    } else {
+      static_assert(__always_false<_Range>,
+                    "ranges::to: unable to deduce the container type from the template template argument.");
+    }
+  }
+
+  using type = typename decltype(__deduce_func())::type;
+};
+
+// `ranges::to` specialization -- `_Container` is a template template parameter requiring deduction to figure out the
+// container element type.
+template <template <class...> class _Container, input_range _Range, class... _Args>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto to(_Range&& __range, _Args&&... __args) {
+  using _DeduceExpr = typename _Deducer<_Container, _Range, _Args...>::type;
+  return ranges::to<_DeduceExpr>(std::forward<_Range>(__range), std::forward<_Args>(__args)...);
+}
+
+// Range adaptor closure object 1 -- wrapping the `ranges::to` version where `_Container` is a simple type template
+// parameter.
+template <class _Container, class... _Args>
+  requires(!view<_Container>)
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto to(_Args&&... __args) {
+  // Mandates: C is a cv-unqualified class type.
+  static_assert(!is_const_v<_Container>, "The target container cannot be const-qualified, please remove the const");
+  static_assert(
+      !is_volatile_v<_Container>, "The target container cannot be volatile-qualified, please remove the volatile");
+
+  auto __to_func = []<input_range _Range, class... _Tail>(_Range&& __range, _Tail&&... __tail) static
+    requires requires { //
+      /**/ ranges::to<_Container>(std::forward<_Range>(__range), std::forward<_Tail>(__tail)...);
+    }
+  { return ranges::to<_Container>(std::forward<_Range>(__range), std::forward<_Tail>(__tail)...); };
+
+  return __range_adaptor_closure_t(std::__bind_back(__to_func, std::forward<_Args>(__args)...));
+}
+
+// Range adaptor closure object 2 -- wrapping the `ranges::to` version where `_Container` is a template template
+// parameter.
+template <template <class...> class _Container, class... _Args>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto to(_Args&&... __args) {
+  // clang-format off
+  auto __to_func = []<input_range _Range, class... _Tail,
+                      class _DeducedExpr = typename _Deducer<_Container, _Range, _Tail...>::type>
+    (_Range&& __range, _Tail&& ... __tail) static
+      requires requires { //
+      /**/ ranges::to<_DeducedExpr>(std::forward<_Range>(__range), std::forward<_Tail>(__tail)...);
+    }
+  {
+    return ranges::to<_DeducedExpr>(std::forward<_Range>(__range), std::forward<_Tail>(__tail)...);
+  };
+  // clang-format on
+
+  return __range_adaptor_closure_t(std::__bind_back(__to_func, std::forward<_Args>(__args)...));
+}
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_TO_H
diff --git a/libcxx/include/__cxx03/__ranges/transform_view.h b/libcxx/include/__cxx03/__ranges/transform_view.h
new file mode 100644
index 000000000000..bcce389c0e68
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/transform_view.h
@@ -0,0 +1,417 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_TRANSFORM_VIEW_H
+#define _LIBCPP___RANGES_TRANSFORM_VIEW_H
+
+#include <__compare/three_way_comparable.h>
+#include <__concepts/constructible.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/copyable.h>
+#include <__concepts/derived_from.h>
+#include <__concepts/equality_comparable.h>
+#include <__concepts/invocable.h>
+#include <__config>
+#include <__functional/bind_back.h>
+#include <__functional/invoke.h>
+#include <__functional/perfect_forward.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__ranges/access.h>
+#include <__ranges/all.h>
+#include <__ranges/concepts.h>
+#include <__ranges/empty.h>
+#include <__ranges/movable_box.h>
+#include <__ranges/range_adaptor.h>
+#include <__ranges/size.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_object.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/maybe_const.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/forward.h>
+#include <__utility/in_place.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+template <class _Fn, class _View>
+concept __regular_invocable_with_range_ref = regular_invocable<_Fn, range_reference_t<_View>>;
+
+template <class _View, class _Fn>
+concept __transform_view_constraints =
+    view<_View> && is_object_v<_Fn> && regular_invocable<_Fn&, range_reference_t<_View>> &&
+    __can_reference<invoke_result_t<_Fn&, range_reference_t<_View>>>;
+
+#  if _LIBCPP_STD_VER >= 23
+template <input_range _View, move_constructible _Fn>
+#  else
+template <input_range _View, copy_constructible _Fn>
+#  endif
+  requires __transform_view_constraints<_View, _Fn>
+class _LIBCPP_ABI_LLVM18_NO_UNIQUE_ADDRESS transform_view : public view_interface<transform_view<_View, _Fn>> {
+  template <bool>
+  class __iterator;
+  template <bool>
+  class __sentinel;
+
+  _LIBCPP_NO_UNIQUE_ADDRESS __movable_box<_Fn> __func_;
+  _LIBCPP_NO_UNIQUE_ADDRESS _View __base_ = _View();
+
+public:
+  _LIBCPP_HIDE_FROM_ABI transform_view()
+    requires default_initializable<_View> && default_initializable<_Fn>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23 transform_view(_View __base, _Fn __func)
+      : __func_(std::in_place, std::move(__func)), __base_(std::move(__base)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
+    requires copy_constructible<_View>
+  {
+    return __base_;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator<false> begin() { return __iterator<false>{*this, ranges::begin(__base_)}; }
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator<true> begin() const
+    requires range<const _View> && __regular_invocable_with_range_ref<const _Fn&, const _View>
+  {
+    return __iterator<true>(*this, ranges::begin(__base_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __sentinel<false> end() { return __sentinel<false>(ranges::end(__base_)); }
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator<false> end()
+    requires common_range<_View>
+  {
+    return __iterator<false>(*this, ranges::end(__base_));
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr __sentinel<true> end() const
+    requires range<const _View> && __regular_invocable_with_range_ref<const _Fn&, const _View>
+  {
+    return __sentinel<true>(ranges::end(__base_));
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator<true> end() const
+    requires common_range<const _View> && __regular_invocable_with_range_ref<const _Fn&, const _View>
+  {
+    return __iterator<true>(*this, ranges::end(__base_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size()
+    requires sized_range<_View>
+  {
+    return ranges::size(__base_);
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
+    requires sized_range<const _View>
+  {
+    return ranges::size(__base_);
+  }
+};
+
+template <class _Range, class _Fn>
+transform_view(_Range&&, _Fn) -> transform_view<views::all_t<_Range>, _Fn>;
+
+template <class _View>
+struct __transform_view_iterator_concept {
+  using type = input_iterator_tag;
+};
+
+template <random_access_range _View>
+struct __transform_view_iterator_concept<_View> {
+  using type = random_access_iterator_tag;
+};
+
+template <bidirectional_range _View>
+struct __transform_view_iterator_concept<_View> {
+  using type = bidirectional_iterator_tag;
+};
+
+template <forward_range _View>
+struct __transform_view_iterator_concept<_View> {
+  using type = forward_iterator_tag;
+};
+
+template <class, class>
+struct __transform_view_iterator_category_base {};
+
+template <forward_range _View, class _Fn>
+struct __transform_view_iterator_category_base<_View, _Fn> {
+  using _Cat = typename iterator_traits<iterator_t<_View>>::iterator_category;
+
+  using iterator_category =
+      conditional_t< is_reference_v<invoke_result_t<_Fn&, range_reference_t<_View>>>,
+                     conditional_t< derived_from<_Cat, contiguous_iterator_tag>, random_access_iterator_tag, _Cat >,
+                     input_iterator_tag >;
+};
+
+#  if _LIBCPP_STD_VER >= 23
+template <input_range _View, move_constructible _Fn>
+#  else
+template <input_range _View, copy_constructible _Fn>
+#  endif
+  requires __transform_view_constraints<_View, _Fn>
+template <bool _Const>
+class transform_view<_View, _Fn>::__iterator : public __transform_view_iterator_category_base<_View, _Fn> {
+
+  using _Parent = __maybe_const<_Const, transform_view>;
+  using _Base   = __maybe_const<_Const, _View>;
+
+  _Parent* __parent_ = nullptr;
+
+  template <bool>
+  friend class transform_view<_View, _Fn>::__iterator;
+
+  template <bool>
+  friend class transform_view<_View, _Fn>::__sentinel;
+
+public:
+  iterator_t<_Base> __current_ = iterator_t<_Base>();
+
+  using iterator_concept = typename __transform_view_iterator_concept<_View>::type;
+  using value_type       = remove_cvref_t<invoke_result_t<_Fn&, range_reference_t<_Base>>>;
+  using difference_type  = range_difference_t<_Base>;
+
+  _LIBCPP_HIDE_FROM_ABI __iterator()
+    requires default_initializable<iterator_t<_Base>>
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator(_Parent& __parent, iterator_t<_Base> __current)
+      : __parent_(std::addressof(__parent)), __current_(std::move(__current)) {}
+
+  // Note: `__i` should always be `__iterator<false>`, but directly using
+  // `__iterator<false>` is ill-formed when `_Const` is false
+  // (see http://wg21.link/class.copy.ctor#5).
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator(__iterator<!_Const> __i)
+    requires _Const && convertible_to<iterator_t<_View>, iterator_t<_Base>>
+      : __parent_(__i.__parent_), __current_(std::move(__i.__current_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const iterator_t<_Base>& base() const& noexcept { return __current_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_Base> base() && { return std::move(__current_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() const
+      noexcept(noexcept(std::invoke(*__parent_->__func_, *__current_))) {
+    return std::invoke(*__parent_->__func_, *__current_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() {
+    ++__current_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator++(int) { ++__current_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator++(int)
+    requires forward_range<_Base>
+  {
+    auto __tmp = *this;
+    ++*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator--()
+    requires bidirectional_range<_Base>
+  {
+    --__current_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator--(int)
+    requires bidirectional_range<_Base>
+  {
+    auto __tmp = *this;
+    --*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator+=(difference_type __n)
+    requires random_access_range<_Base>
+  {
+    __current_ += __n;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator-=(difference_type __n)
+    requires random_access_range<_Base>
+  {
+    __current_ -= __n;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator[](difference_type __n) const
+      noexcept(noexcept(std::invoke(*__parent_->__func_, __current_[__n])))
+    requires random_access_range<_Base>
+  {
+    return std::invoke(*__parent_->__func_, __current_[__n]);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __iterator& __y)
+    requires equality_comparable<iterator_t<_Base>>
+  {
+    return __x.__current_ == __y.__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(const __iterator& __x, const __iterator& __y)
+    requires random_access_range<_Base>
+  {
+    return __x.__current_ < __y.__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(const __iterator& __x, const __iterator& __y)
+    requires random_access_range<_Base>
+  {
+    return __x.__current_ > __y.__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(const __iterator& __x, const __iterator& __y)
+    requires random_access_range<_Base>
+  {
+    return __x.__current_ <= __y.__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(const __iterator& __x, const __iterator& __y)
+    requires random_access_range<_Base>
+  {
+    return __x.__current_ >= __y.__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr auto operator<=>(const __iterator& __x, const __iterator& __y)
+    requires random_access_range<_Base> && three_way_comparable<iterator_t<_Base>>
+  {
+    return __x.__current_ <=> __y.__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(__iterator __i, difference_type __n)
+    requires random_access_range<_Base>
+  {
+    return __iterator{*__i.__parent_, __i.__current_ + __n};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(difference_type __n, __iterator __i)
+    requires random_access_range<_Base>
+  {
+    return __iterator{*__i.__parent_, __i.__current_ + __n};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator-(__iterator __i, difference_type __n)
+    requires random_access_range<_Base>
+  {
+    return __iterator{*__i.__parent_, __i.__current_ - __n};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr difference_type operator-(const __iterator& __x, const __iterator& __y)
+    requires sized_sentinel_for<iterator_t<_Base>, iterator_t<_Base>>
+  {
+    return __x.__current_ - __y.__current_;
+  }
+};
+
+#  if _LIBCPP_STD_VER >= 23
+template <input_range _View, move_constructible _Fn>
+#  else
+template <input_range _View, copy_constructible _Fn>
+#  endif
+  requires __transform_view_constraints<_View, _Fn>
+template <bool _Const>
+class transform_view<_View, _Fn>::__sentinel {
+  using _Parent = __maybe_const<_Const, transform_view>;
+  using _Base   = __maybe_const<_Const, _View>;
+
+  sentinel_t<_Base> __end_ = sentinel_t<_Base>();
+
+  template <bool>
+  friend class transform_view<_View, _Fn>::__iterator;
+
+  template <bool>
+  friend class transform_view<_View, _Fn>::__sentinel;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __sentinel() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __sentinel(sentinel_t<_Base> __end) : __end_(__end) {}
+
+  // Note: `__i` should always be `__sentinel<false>`, but directly using
+  // `__sentinel<false>` is ill-formed when `_Const` is false
+  // (see http://wg21.link/class.copy.ctor#5).
+  _LIBCPP_HIDE_FROM_ABI constexpr __sentinel(__sentinel<!_Const> __i)
+    requires _Const && convertible_to<sentinel_t<_View>, sentinel_t<_Base>>
+      : __end_(std::move(__i.__end_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr sentinel_t<_Base> base() const { return __end_; }
+
+  template <bool _OtherConst>
+    requires sentinel_for<sentinel_t<_Base>, iterator_t<__maybe_const<_OtherConst, _View>>>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator<_OtherConst>& __x, const __sentinel& __y) {
+    return __x.__current_ == __y.__end_;
+  }
+
+  template <bool _OtherConst>
+    requires sized_sentinel_for<sentinel_t<_Base>, iterator_t<__maybe_const<_OtherConst, _View>>>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr range_difference_t<__maybe_const<_OtherConst, _View>>
+  operator-(const __iterator<_OtherConst>& __x, const __sentinel& __y) {
+    return __x.__current_ - __y.__end_;
+  }
+
+  template <bool _OtherConst>
+    requires sized_sentinel_for<sentinel_t<_Base>, iterator_t<__maybe_const<_OtherConst, _View>>>
+  _LIBCPP_HIDE_FROM_ABI friend constexpr range_difference_t<__maybe_const<_OtherConst, _View>>
+  operator-(const __sentinel& __x, const __iterator<_OtherConst>& __y) {
+    return __x.__end_ - __y.__current_;
+  }
+};
+
+namespace views {
+namespace __transform {
+struct __fn {
+  template <class _Range, class _Fn>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range, _Fn&& __f) const
+      noexcept(noexcept(transform_view(std::forward<_Range>(__range), std::forward<_Fn>(__f))))
+          -> decltype(transform_view(std::forward<_Range>(__range), std::forward<_Fn>(__f))) {
+    return transform_view(std::forward<_Range>(__range), std::forward<_Fn>(__f));
+  }
+
+  template <class _Fn>
+    requires constructible_from<decay_t<_Fn>, _Fn>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Fn&& __f) const
+      noexcept(is_nothrow_constructible_v<decay_t<_Fn>, _Fn>) {
+    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Fn>(__f)));
+  }
+};
+} // namespace __transform
+
+inline namespace __cpo {
+inline constexpr auto transform = __transform::__fn{};
+} // namespace __cpo
+} // namespace views
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_TRANSFORM_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/view_interface.h b/libcxx/include/__cxx03/__ranges/view_interface.h
new file mode 100644
index 000000000000..3bcfbaf3a2f9
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/view_interface.h
@@ -0,0 +1,170 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_VIEW_INTERFACE_H
+#define _LIBCPP___RANGES_VIEW_INTERFACE_H
+
+#include <__assert>
+#include <__concepts/derived_from.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/prev.h>
+#include <__memory/pointer_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/empty.h>
+#include <__ranges/size.h>
+#include <__type_traits/is_class.h>
+#include <__type_traits/make_unsigned.h>
+#include <__type_traits/remove_cv.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+template <class _Derived>
+  requires is_class_v<_Derived> && same_as<_Derived, remove_cv_t<_Derived>>
+class view_interface {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Derived& __derived() noexcept {
+    static_assert(sizeof(_Derived) && derived_from<_Derived, view_interface> && view<_Derived>);
+    return static_cast<_Derived&>(*this);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr _Derived const& __derived() const noexcept {
+    static_assert(sizeof(_Derived) && derived_from<_Derived, view_interface> && view<_Derived>);
+    return static_cast<_Derived const&>(*this);
+  }
+
+public:
+  template <class _D2 = _Derived>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool empty()
+    requires sized_range<_D2> || forward_range<_D2>
+  {
+    if constexpr (sized_range<_D2>) {
+      return ranges::size(__derived()) == 0;
+    } else {
+      return ranges::begin(__derived()) == ranges::end(__derived());
+    }
+  }
+
+  template <class _D2 = _Derived>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool empty() const
+    requires sized_range<const _D2> || forward_range<const _D2>
+  {
+    if constexpr (sized_range<const _D2>) {
+      return ranges::size(__derived()) == 0;
+    } else {
+      return ranges::begin(__derived()) == ranges::end(__derived());
+    }
+  }
+
+  template <class _D2 = _Derived>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool()
+    requires requires(_D2& __t) { ranges::empty(__t); }
+  {
+    return !ranges::empty(__derived());
+  }
+
+  template <class _D2 = _Derived>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const
+    requires requires(const _D2& __t) { ranges::empty(__t); }
+  {
+    return !ranges::empty(__derived());
+  }
+
+  template <class _D2 = _Derived>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto data()
+    requires contiguous_iterator<iterator_t<_D2>>
+  {
+    return std::to_address(ranges::begin(__derived()));
+  }
+
+  template <class _D2 = _Derived>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto data() const
+    requires range<const _D2> && contiguous_iterator<iterator_t<const _D2>>
+  {
+    return std::to_address(ranges::begin(__derived()));
+  }
+
+  template <class _D2 = _Derived>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size()
+    requires forward_range<_D2> && sized_sentinel_for<sentinel_t<_D2>, iterator_t<_D2>>
+  {
+    return std::__to_unsigned_like(ranges::end(__derived()) - ranges::begin(__derived()));
+  }
+
+  template <class _D2 = _Derived>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
+    requires forward_range<const _D2> && sized_sentinel_for<sentinel_t<const _D2>, iterator_t<const _D2>>
+  {
+    return std::__to_unsigned_like(ranges::end(__derived()) - ranges::begin(__derived()));
+  }
+
+  template <class _D2 = _Derived>
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) front()
+    requires forward_range<_D2>
+  {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !empty(), "Precondition `!empty()` not satisfied. `.front()` called on an empty view.");
+    return *ranges::begin(__derived());
+  }
+
+  template <class _D2 = _Derived>
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) front() const
+    requires forward_range<const _D2>
+  {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !empty(), "Precondition `!empty()` not satisfied. `.front()` called on an empty view.");
+    return *ranges::begin(__derived());
+  }
+
+  template <class _D2 = _Derived>
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) back()
+    requires bidirectional_range<_D2> && common_range<_D2>
+  {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !empty(), "Precondition `!empty()` not satisfied. `.back()` called on an empty view.");
+    return *ranges::prev(ranges::end(__derived()));
+  }
+
+  template <class _D2 = _Derived>
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) back() const
+    requires bidirectional_range<const _D2> && common_range<const _D2>
+  {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        !empty(), "Precondition `!empty()` not satisfied. `.back()` called on an empty view.");
+    return *ranges::prev(ranges::end(__derived()));
+  }
+
+  template <random_access_range _RARange = _Derived>
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator[](range_difference_t<_RARange> __index) {
+    return ranges::begin(__derived())[__index];
+  }
+
+  template <random_access_range _RARange = const _Derived>
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator[](range_difference_t<_RARange> __index) const {
+    return ranges::begin(__derived())[__index];
+  }
+};
+
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_VIEW_INTERFACE_H
diff --git a/libcxx/include/__cxx03/__ranges/views.h b/libcxx/include/__cxx03/__ranges/views.h
new file mode 100644
index 000000000000..a4de2a5e52a9
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/views.h
@@ -0,0 +1,35 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_VIEWS
+#define _LIBCPP___RANGES_VIEWS
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+namespace ranges {
+
+namespace views {}
+
+} // namespace ranges
+
+namespace views = ranges::views;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RANGES_VIEWS
diff --git a/libcxx/include/__cxx03/__ranges/zip_view.h b/libcxx/include/__cxx03/__ranges/zip_view.h
new file mode 100644
index 000000000000..fe3c87a9306f
--- /dev/null
+++ b/libcxx/include/__cxx03/__ranges/zip_view.h
@@ -0,0 +1,516 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANGES_ZIP_VIEW_H
+#define _LIBCPP___RANGES_ZIP_VIEW_H
+
+#include <__config>
+
+#include <__algorithm/ranges_min.h>
+#include <__compare/three_way_comparable.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/equality_comparable.h>
+#include <__functional/invoke.h>
+#include <__functional/operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iter_move.h>
+#include <__iterator/iter_swap.h>
+#include <__iterator/iterator_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/all.h>
+#include <__ranges/concepts.h>
+#include <__ranges/empty_view.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/size.h>
+#include <__ranges/view_interface.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/make_unsigned.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/integer_sequence.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <tuple>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+namespace ranges {
+
+template <class... _Ranges>
+concept __zip_is_common =
+    (sizeof...(_Ranges) == 1 && (common_range<_Ranges> && ...)) ||
+    (!(bidirectional_range<_Ranges> && ...) && (common_range<_Ranges> && ...)) ||
+    ((random_access_range<_Ranges> && ...) && (sized_range<_Ranges> && ...));
+
+template <typename _Tp, typename _Up>
+auto __tuple_or_pair_test() -> pair<_Tp, _Up>;
+
+template <typename... _Types>
+  requires(sizeof...(_Types) != 2)
+auto __tuple_or_pair_test() -> tuple<_Types...>;
+
+template <class... _Types>
+using __tuple_or_pair = decltype(__tuple_or_pair_test<_Types...>());
+
+template <class _Fun, class _Tuple>
+_LIBCPP_HIDE_FROM_ABI constexpr auto __tuple_transform(_Fun&& __f, _Tuple&& __tuple) {
+  return std::apply(
+      [&]<class... _Types>(_Types&&... __elements) {
+        return __tuple_or_pair<invoke_result_t<_Fun&, _Types>...>(
+            std::invoke(__f, std::forward<_Types>(__elements))...);
+      },
+      std::forward<_Tuple>(__tuple));
+}
+
+template <class _Fun, class _Tuple>
+_LIBCPP_HIDE_FROM_ABI constexpr void __tuple_for_each(_Fun&& __f, _Tuple&& __tuple) {
+  std::apply(
+      [&]<class... _Types>(_Types&&... __elements) {
+        (static_cast<void>(std::invoke(__f, std::forward<_Types>(__elements))), ...);
+      },
+      std::forward<_Tuple>(__tuple));
+}
+
+template <class _Fun, class _Tuple1, class _Tuple2, size_t... _Indices>
+_LIBCPP_HIDE_FROM_ABI constexpr __tuple_or_pair<
+    invoke_result_t<_Fun&,
+                    typename tuple_element<_Indices, remove_cvref_t<_Tuple1>>::type,
+                    typename tuple_element<_Indices, remove_cvref_t<_Tuple2>>::type>...>
+__tuple_zip_transform(_Fun&& __f, _Tuple1&& __tuple1, _Tuple2&& __tuple2, index_sequence<_Indices...>) {
+  return {std::invoke(__f,
+                      std::get<_Indices>(std::forward<_Tuple1>(__tuple1)),
+                      std::get<_Indices>(std::forward<_Tuple2>(__tuple2)))...};
+}
+
+template <class _Fun, class _Tuple1, class _Tuple2>
+_LIBCPP_HIDE_FROM_ABI constexpr auto __tuple_zip_transform(_Fun&& __f, _Tuple1&& __tuple1, _Tuple2&& __tuple2) {
+  return ranges::__tuple_zip_transform(
+      __f,
+      std::forward<_Tuple1>(__tuple1),
+      std::forward<_Tuple2>(__tuple2),
+      std::make_index_sequence<tuple_size<remove_cvref_t<_Tuple1>>::value>());
+}
+
+template <class _Fun, class _Tuple1, class _Tuple2, size_t... _Indices>
+_LIBCPP_HIDE_FROM_ABI constexpr void
+__tuple_zip_for_each(_Fun&& __f, _Tuple1&& __tuple1, _Tuple2&& __tuple2, index_sequence<_Indices...>) {
+  (std::invoke(
+       __f, std::get<_Indices>(std::forward<_Tuple1>(__tuple1)), std::get<_Indices>(std::forward<_Tuple2>(__tuple2))),
+   ...);
+}
+
+template <class _Fun, class _Tuple1, class _Tuple2>
+_LIBCPP_HIDE_FROM_ABI constexpr auto __tuple_zip_for_each(_Fun&& __f, _Tuple1&& __tuple1, _Tuple2&& __tuple2) {
+  return ranges::__tuple_zip_for_each(
+      __f,
+      std::forward<_Tuple1>(__tuple1),
+      std::forward<_Tuple2>(__tuple2),
+      std::make_index_sequence<tuple_size<remove_cvref_t<_Tuple1>>::value>());
+}
+
+template <class _Tuple1, class _Tuple2>
+_LIBCPP_HIDE_FROM_ABI constexpr bool __tuple_any_equals(const _Tuple1& __tuple1, const _Tuple2& __tuple2) {
+  const auto __equals = ranges::__tuple_zip_transform(std::equal_to<>(), __tuple1, __tuple2);
+  return std::apply([](auto... __bools) { return (__bools || ...); }, __equals);
+}
+
+// abs in cstdlib is not constexpr
+// TODO : remove __abs once P0533R9 is implemented.
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp __abs(_Tp __t) {
+  return __t < 0 ? -__t : __t;
+}
+
+template <input_range... _Views>
+  requires(view<_Views> && ...) && (sizeof...(_Views) > 0)
+class zip_view : public view_interface<zip_view<_Views...>> {
+  _LIBCPP_NO_UNIQUE_ADDRESS tuple<_Views...> __views_;
+
+  template <bool>
+  class __iterator;
+
+  template <bool>
+  class __sentinel;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI zip_view() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit zip_view(_Views... __views) : __views_(std::move(__views)...) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin()
+    requires(!(__simple_view<_Views> && ...))
+  {
+    return __iterator<false>(ranges::__tuple_transform(ranges::begin, __views_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
+    requires(range<const _Views> && ...)
+  {
+    return __iterator<true>(ranges::__tuple_transform(ranges::begin, __views_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end()
+    requires(!(__simple_view<_Views> && ...))
+  {
+    if constexpr (!__zip_is_common<_Views...>) {
+      return __sentinel<false>(ranges::__tuple_transform(ranges::end, __views_));
+    } else if constexpr ((random_access_range<_Views> && ...)) {
+      return begin() + iter_difference_t<__iterator<false>>(size());
+    } else {
+      return __iterator<false>(ranges::__tuple_transform(ranges::end, __views_));
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
+    requires(range<const _Views> && ...)
+  {
+    if constexpr (!__zip_is_common<const _Views...>) {
+      return __sentinel<true>(ranges::__tuple_transform(ranges::end, __views_));
+    } else if constexpr ((random_access_range<const _Views> && ...)) {
+      return begin() + iter_difference_t<__iterator<true>>(size());
+    } else {
+      return __iterator<true>(ranges::__tuple_transform(ranges::end, __views_));
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size()
+    requires(sized_range<_Views> && ...)
+  {
+    return std::apply(
+        [](auto... __sizes) {
+          using _CT = make_unsigned_t<common_type_t<decltype(__sizes)...>>;
+          return ranges::min({_CT(__sizes)...});
+        },
+        ranges::__tuple_transform(ranges::size, __views_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
+    requires(sized_range<const _Views> && ...)
+  {
+    return std::apply(
+        [](auto... __sizes) {
+          using _CT = make_unsigned_t<common_type_t<decltype(__sizes)...>>;
+          return ranges::min({_CT(__sizes)...});
+        },
+        ranges::__tuple_transform(ranges::size, __views_));
+  }
+};
+
+template <class... _Ranges>
+zip_view(_Ranges&&...) -> zip_view<views::all_t<_Ranges>...>;
+
+template <bool _Const, class... _Views>
+concept __zip_all_random_access = (random_access_range<__maybe_const<_Const, _Views>> && ...);
+
+template <bool _Const, class... _Views>
+concept __zip_all_bidirectional = (bidirectional_range<__maybe_const<_Const, _Views>> && ...);
+
+template <bool _Const, class... _Views>
+concept __zip_all_forward = (forward_range<__maybe_const<_Const, _Views>> && ...);
+
+template <bool _Const, class... _Views>
+consteval auto __get_zip_view_iterator_tag() {
+  if constexpr (__zip_all_random_access<_Const, _Views...>) {
+    return random_access_iterator_tag();
+  } else if constexpr (__zip_all_bidirectional<_Const, _Views...>) {
+    return bidirectional_iterator_tag();
+  } else if constexpr (__zip_all_forward<_Const, _Views...>) {
+    return forward_iterator_tag();
+  } else {
+    return input_iterator_tag();
+  }
+}
+
+template <bool _Const, class... _Views>
+struct __zip_view_iterator_category_base {};
+
+template <bool _Const, class... _Views>
+  requires __zip_all_forward<_Const, _Views...>
+struct __zip_view_iterator_category_base<_Const, _Views...> {
+  using iterator_category = input_iterator_tag;
+};
+
+template <input_range... _Views>
+  requires(view<_Views> && ...) && (sizeof...(_Views) > 0)
+template <bool _Const>
+class zip_view<_Views...>::__iterator : public __zip_view_iterator_category_base<_Const, _Views...> {
+  __tuple_or_pair<iterator_t<__maybe_const<_Const, _Views>>...> __current_;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __iterator(
+      __tuple_or_pair<iterator_t<__maybe_const<_Const, _Views>>...> __current)
+      : __current_(std::move(__current)) {}
+
+  template <bool>
+  friend class zip_view<_Views...>::__iterator;
+
+  template <bool>
+  friend class zip_view<_Views...>::__sentinel;
+
+  friend class zip_view<_Views...>;
+
+public:
+  using iterator_concept = decltype(__get_zip_view_iterator_tag<_Const, _Views...>());
+  using value_type       = __tuple_or_pair<range_value_t<__maybe_const<_Const, _Views>>...>;
+  using difference_type  = common_type_t<range_difference_t<__maybe_const<_Const, _Views>>...>;
+
+  _LIBCPP_HIDE_FROM_ABI __iterator() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator(__iterator<!_Const> __i)
+    requires _Const && (convertible_to<iterator_t<_Views>, iterator_t<__maybe_const<_Const, _Views>>> && ...)
+      : __current_(std::move(__i.__current_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator*() const {
+    return ranges::__tuple_transform([](auto& __i) -> decltype(auto) { return *__i; }, __current_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() {
+    ranges::__tuple_for_each([](auto& __i) { ++__i; }, __current_);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator++(int) { ++*this; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator++(int)
+    requires __zip_all_forward<_Const, _Views...>
+  {
+    auto __tmp = *this;
+    ++*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator--()
+    requires __zip_all_bidirectional<_Const, _Views...>
+  {
+    ranges::__tuple_for_each([](auto& __i) { --__i; }, __current_);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator operator--(int)
+    requires __zip_all_bidirectional<_Const, _Views...>
+  {
+    auto __tmp = *this;
+    --*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator+=(difference_type __x)
+    requires __zip_all_random_access<_Const, _Views...>
+  {
+    ranges::__tuple_for_each([&]<class _Iter>(_Iter& __i) { __i += iter_difference_t<_Iter>(__x); }, __current_);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator-=(difference_type __x)
+    requires __zip_all_random_access<_Const, _Views...>
+  {
+    ranges::__tuple_for_each([&]<class _Iter>(_Iter& __i) { __i -= iter_difference_t<_Iter>(__x); }, __current_);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto operator[](difference_type __n) const
+    requires __zip_all_random_access<_Const, _Views...>
+  {
+    return ranges::__tuple_transform(
+        [&]<class _Iter>(_Iter& __i) -> decltype(auto) { return __i[iter_difference_t<_Iter>(__n)]; }, __current_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __iterator& __y)
+    requires(equality_comparable<iterator_t<__maybe_const<_Const, _Views>>> && ...)
+  {
+    if constexpr (__zip_all_bidirectional<_Const, _Views...>) {
+      return __x.__current_ == __y.__current_;
+    } else {
+      return ranges::__tuple_any_equals(__x.__current_, __y.__current_);
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(const __iterator& __x, const __iterator& __y)
+    requires __zip_all_random_access<_Const, _Views...>
+  {
+    return __x.__current_ < __y.__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(const __iterator& __x, const __iterator& __y)
+    requires __zip_all_random_access<_Const, _Views...>
+  {
+    return __y < __x;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(const __iterator& __x, const __iterator& __y)
+    requires __zip_all_random_access<_Const, _Views...>
+  {
+    return !(__y < __x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(const __iterator& __x, const __iterator& __y)
+    requires __zip_all_random_access<_Const, _Views...>
+  {
+    return !(__x < __y);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr auto operator<=>(const __iterator& __x, const __iterator& __y)
+    requires __zip_all_random_access<_Const, _Views...> &&
+             (three_way_comparable<iterator_t<__maybe_const<_Const, _Views>>> && ...)
+  {
+    return __x.__current_ <=> __y.__current_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(const __iterator& __i, difference_type __n)
+    requires __zip_all_random_access<_Const, _Views...>
+  {
+    auto __r = __i;
+    __r += __n;
+    return __r;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(difference_type __n, const __iterator& __i)
+    requires __zip_all_random_access<_Const, _Views...>
+  {
+    return __i + __n;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator-(const __iterator& __i, difference_type __n)
+    requires __zip_all_random_access<_Const, _Views...>
+  {
+    auto __r = __i;
+    __r -= __n;
+    return __r;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr difference_type operator-(const __iterator& __x, const __iterator& __y)
+    requires(sized_sentinel_for<iterator_t<__maybe_const<_Const, _Views>>, iterator_t<__maybe_const<_Const, _Views>>> &&
+             ...)
+  {
+    const auto __diffs = ranges::__tuple_zip_transform(minus<>(), __x.__current_, __y.__current_);
+    return std::apply(
+        [](auto... __ds) {
+          return ranges::min({difference_type(__ds)...}, [](auto __a, auto __b) {
+            return ranges::__abs(__a) < ranges::__abs(__b);
+          });
+        },
+        __diffs);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr auto iter_move(const __iterator& __i) noexcept(
+      (noexcept(ranges::iter_move(std::declval<const iterator_t<__maybe_const<_Const, _Views>>&>())) && ...) &&
+      (is_nothrow_move_constructible_v<range_rvalue_reference_t<__maybe_const<_Const, _Views>>> && ...)) {
+    return ranges::__tuple_transform(ranges::iter_move, __i.__current_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr void iter_swap(const __iterator& __l, const __iterator& __r) noexcept(
+      (noexcept(ranges::iter_swap(std::declval<const iterator_t<__maybe_const<_Const, _Views>>&>(),
+                                  std::declval<const iterator_t<__maybe_const<_Const, _Views>>&>())) &&
+       ...))
+    requires(indirectly_swappable<iterator_t<__maybe_const<_Const, _Views>>> && ...)
+  {
+    ranges::__tuple_zip_for_each(ranges::iter_swap, __l.__current_, __r.__current_);
+  }
+};
+
+template <input_range... _Views>
+  requires(view<_Views> && ...) && (sizeof...(_Views) > 0)
+template <bool _Const>
+class zip_view<_Views...>::__sentinel {
+  __tuple_or_pair<sentinel_t<__maybe_const<_Const, _Views>>...> __end_;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __sentinel(
+      __tuple_or_pair<sentinel_t<__maybe_const<_Const, _Views>>...> __end)
+      : __end_(__end) {}
+
+  friend class zip_view<_Views...>;
+
+  // hidden friend cannot access private member of iterator because they are friends of friends
+  template <bool _OtherConst>
+  _LIBCPP_HIDE_FROM_ABI static constexpr decltype(auto)
+  __iter_current(zip_view<_Views...>::__iterator<_OtherConst> const& __it) {
+    return (__it.__current_);
+  }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __sentinel() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __sentinel(__sentinel<!_Const> __i)
+    requires _Const && (convertible_to<sentinel_t<_Views>, sentinel_t<__maybe_const<_Const, _Views>>> && ...)
+      : __end_(std::move(__i.__end_)) {}
+
+  template <bool _OtherConst>
+    requires(sentinel_for<sentinel_t<__maybe_const<_Const, _Views>>, iterator_t<__maybe_const<_OtherConst, _Views>>> &&
+             ...)
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator<_OtherConst>& __x, const __sentinel& __y) {
+    return ranges::__tuple_any_equals(__iter_current(__x), __y.__end_);
+  }
+
+  template <bool _OtherConst>
+    requires(
+        sized_sentinel_for<sentinel_t<__maybe_const<_Const, _Views>>, iterator_t<__maybe_const<_OtherConst, _Views>>> &&
+        ...)
+  _LIBCPP_HIDE_FROM_ABI friend constexpr common_type_t<range_difference_t<__maybe_const<_OtherConst, _Views>>...>
+  operator-(const __iterator<_OtherConst>& __x, const __sentinel& __y) {
+    const auto __diffs = ranges::__tuple_zip_transform(minus<>(), __iter_current(__x), __y.__end_);
+    return std::apply(
+        [](auto... __ds) {
+          using _Diff = common_type_t<range_difference_t<__maybe_const<_OtherConst, _Views>>...>;
+          return ranges::min({_Diff(__ds)...}, [](auto __a, auto __b) {
+            return ranges::__abs(__a) < ranges::__abs(__b);
+          });
+        },
+        __diffs);
+  }
+
+  template <bool _OtherConst>
+    requires(
+        sized_sentinel_for<sentinel_t<__maybe_const<_Const, _Views>>, iterator_t<__maybe_const<_OtherConst, _Views>>> &&
+        ...)
+  _LIBCPP_HIDE_FROM_ABI friend constexpr common_type_t<range_difference_t<__maybe_const<_OtherConst, _Views>>...>
+  operator-(const __sentinel& __y, const __iterator<_OtherConst>& __x) {
+    return -(__x - __y);
+  }
+};
+
+template <class... _Views>
+inline constexpr bool enable_borrowed_range<zip_view<_Views...>> = (enable_borrowed_range<_Views> && ...);
+
+namespace views {
+namespace __zip {
+
+struct __fn {
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()() noexcept { return empty_view<tuple<>>{}; }
+
+  template <class... _Ranges>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto
+  operator()(_Ranges&&... __rs) noexcept(noexcept(zip_view<all_t<_Ranges&&>...>(std::forward<_Ranges>(__rs)...)))
+      -> decltype(zip_view<all_t<_Ranges&&>...>(std::forward<_Ranges>(__rs)...)) {
+    return zip_view<all_t<_Ranges>...>(std::forward<_Ranges>(__rs)...);
+  }
+};
+
+} // namespace __zip
+inline namespace __cpo {
+inline constexpr auto zip = __zip::__fn{};
+} // namespace __cpo
+} // namespace views
+} // namespace ranges
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANGES_ZIP_VIEW_H
diff --git a/libcxx/include/__cxx03/__split_buffer b/libcxx/include/__cxx03/__split_buffer
new file mode 100644
index 000000000000..bab724d1b896
--- /dev/null
+++ b/libcxx/include/__cxx03/__split_buffer
@@ -0,0 +1,585 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___SPLIT_BUFFER
+#define _LIBCPP___SPLIT_BUFFER
+
+#include <__algorithm/max.h>
+#include <__algorithm/move.h>
+#include <__algorithm/move_backward.h>
+#include <__config>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/move_iterator.h>
+#include <__memory/allocate_at_least.h>
+#include <__memory/allocator.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/compressed_pair.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/swap_allocator.h>
+#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_swappable.h>
+#include <__type_traits/is_trivially_destructible.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// __split_buffer allocates a contiguous chunk of memory and stores objects in the range [__begin_, __end_).
+// It has uninitialized memory in the ranges  [__first_, __begin_) and [__end_, __end_cap_.first()). That allows
+// it to grow both in the front and back without having to move the data.
+
+template <class _Tp, class _Allocator = allocator<_Tp> >
+struct __split_buffer {
+public:
+  using value_type      = _Tp;
+  using allocator_type  = _Allocator;
+  using __alloc_rr      = __libcpp_remove_reference_t<allocator_type>;
+  using __alloc_traits  = allocator_traits<__alloc_rr>;
+  using reference       = value_type&;
+  using const_reference = const value_type&;
+  using size_type       = typename __alloc_traits::size_type;
+  using difference_type = typename __alloc_traits::difference_type;
+  using pointer         = typename __alloc_traits::pointer;
+  using const_pointer   = typename __alloc_traits::const_pointer;
+  using iterator        = pointer;
+  using const_iterator  = const_pointer;
+
+  // A __split_buffer contains the following members which may be trivially relocatable:
+  // - pointer: may be trivially relocatable, so it's checked
+  // - allocator_type: may be trivially relocatable, so it's checked
+  // __split_buffer doesn't have any self-references, so it's trivially relocatable if its members are.
+  using __trivially_relocatable = __conditional_t<
+      __libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<allocator_type>::value,
+      __split_buffer,
+      void>;
+
+  pointer __first_;
+  pointer __begin_;
+  pointer __end_;
+  __compressed_pair<pointer, allocator_type> __end_cap_;
+
+  using __alloc_ref       = __add_lvalue_reference_t<allocator_type>;
+  using __alloc_const_ref = __add_lvalue_reference_t<allocator_type>;
+
+  __split_buffer(const __split_buffer&)            = delete;
+  __split_buffer& operator=(const __split_buffer&) = delete;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer()
+      _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
+      : __first_(nullptr), __begin_(nullptr), __end_(nullptr), __end_cap_(nullptr, __default_init_tag()) {}
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __split_buffer(__alloc_rr& __a)
+      : __first_(nullptr), __begin_(nullptr), __end_(nullptr), __end_cap_(nullptr, __a) {}
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __split_buffer(const __alloc_rr& __a)
+      : __first_(nullptr), __begin_(nullptr), __end_(nullptr), __end_cap_(nullptr, __a) {}
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+  __split_buffer(size_type __cap, size_type __start, __alloc_rr& __a);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer(__split_buffer&& __c)
+      _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer(__split_buffer&& __c, const __alloc_rr& __a);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer& operator=(__split_buffer&& __c)
+      _NOEXCEPT_((__alloc_traits::propagate_on_container_move_assignment::value &&
+                  is_nothrow_move_assignable<allocator_type>::value) ||
+                 !__alloc_traits::propagate_on_container_move_assignment::value);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~__split_buffer();
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __alloc_rr& __alloc() _NOEXCEPT { return __end_cap_.second(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const __alloc_rr& __alloc() const _NOEXCEPT {
+    return __end_cap_.second();
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer& __end_cap() _NOEXCEPT { return __end_cap_.first(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const pointer& __end_cap() const _NOEXCEPT {
+    return __end_cap_.first();
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __begin_; }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __begin_; }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __end_; }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __end_; }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __destruct_at_end(__begin_); }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const {
+    return static_cast<size_type>(__end_ - __begin_);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool empty() const { return __end_ == __begin_; }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const {
+    return static_cast<size_type>(__end_cap() - __first_);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __front_spare() const {
+    return static_cast<size_type>(__begin_ - __first_);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __back_spare() const {
+    return static_cast<size_type>(__end_cap() - __end_);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference front() { return *__begin_; }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference front() const { return *__begin_; }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() { return *(__end_ - 1); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const { return *(__end_ - 1); }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void reserve(size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void shrink_to_fit() _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_front(const_reference __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_back(const_reference __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_front(value_type&& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __x);
+
+  template <class... _Args>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void emplace_back(_Args&&... __args);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_front() { __destruct_at_begin(__begin_ + 1); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_back() { __destruct_at_end(__end_ - 1); }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_at_end(size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_at_end(size_type __n, const_reference __x);
+
+  template <class _InputIter, __enable_if_t<__has_exactly_input_iterator_category<_InputIter>::value, int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_at_end(_InputIter __first, _InputIter __last);
+
+  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __construct_at_end(_ForwardIterator __first, _ForwardIterator __last);
+
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __construct_at_end_with_sentinel(_Iterator __first, _Sentinel __last);
+
+  template <class _Iterator>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __construct_at_end_with_size(_Iterator __first, size_type __n);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __destruct_at_begin(pointer __new_begin) {
+    __destruct_at_begin(__new_begin, is_trivially_destructible<value_type>());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __destruct_at_begin(pointer __new_begin, false_type);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __destruct_at_begin(pointer __new_begin, true_type);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __destruct_at_end(pointer __new_last) _NOEXCEPT {
+    __destruct_at_end(__new_last, false_type());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __destruct_at_end(pointer __new_last, false_type) _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __destruct_at_end(pointer __new_last, true_type) _NOEXCEPT;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void swap(__split_buffer& __x)
+      _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__alloc_rr>);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __invariants() const;
+
+private:
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__split_buffer& __c, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
+    __alloc() = std::move(__c.__alloc());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__split_buffer&, false_type) _NOEXCEPT {}
+
+  struct _ConstructTransaction {
+    _LIBCPP_CONSTEXPR_SINCE_CXX20
+    _LIBCPP_HIDE_FROM_ABI explicit _ConstructTransaction(pointer* __p, size_type __n) _NOEXCEPT
+        : __pos_(*__p),
+          __end_(*__p + __n),
+          __dest_(__p) {}
+
+    _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~_ConstructTransaction() { *__dest_ = __pos_; }
+
+    pointer __pos_;
+    const pointer __end_;
+
+  private:
+    pointer* __dest_;
+  };
+};
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 bool __split_buffer<_Tp, _Allocator>::__invariants() const {
+  if (__first_ == nullptr) {
+    if (__begin_ != nullptr)
+      return false;
+    if (__end_ != nullptr)
+      return false;
+    if (__end_cap() != nullptr)
+      return false;
+  } else {
+    if (__begin_ < __first_)
+      return false;
+    if (__end_ < __begin_)
+      return false;
+    if (__end_cap() < __end_)
+      return false;
+  }
+  return true;
+}
+
+//  Default constructs __n objects starting at __end_
+//  throws if construction throws
+//  Precondition:  __n > 0
+//  Precondition:  size() + __n <= capacity()
+//  Postcondition:  size() == size() + __n
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::__construct_at_end(size_type __n) {
+  _ConstructTransaction __tx(&this->__end_, __n);
+  for (; __tx.__pos_ != __tx.__end_; ++__tx.__pos_) {
+    __alloc_traits::construct(this->__alloc(), std::__to_address(__tx.__pos_));
+  }
+}
+
+//  Copy constructs __n objects starting at __end_ from __x
+//  throws if construction throws
+//  Precondition:  __n > 0
+//  Precondition:  size() + __n <= capacity()
+//  Postcondition:  size() == old size() + __n
+//  Postcondition:  [i] == __x for all i in [size() - __n, __n)
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+__split_buffer<_Tp, _Allocator>::__construct_at_end(size_type __n, const_reference __x) {
+  _ConstructTransaction __tx(&this->__end_, __n);
+  for (; __tx.__pos_ != __tx.__end_; ++__tx.__pos_) {
+    __alloc_traits::construct(this->__alloc(), std::__to_address(__tx.__pos_), __x);
+  }
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIter, __enable_if_t<__has_exactly_input_iterator_category<_InputIter>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+__split_buffer<_Tp, _Allocator>::__construct_at_end(_InputIter __first, _InputIter __last) {
+  __construct_at_end_with_sentinel(__first, __last);
+}
+
+template <class _Tp, class _Allocator>
+template <class _Iterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+__split_buffer<_Tp, _Allocator>::__construct_at_end_with_sentinel(_Iterator __first, _Sentinel __last) {
+  __alloc_rr& __a = this->__alloc();
+  for (; __first != __last; ++__first) {
+    if (__end_ == __end_cap()) {
+      size_type __old_cap = __end_cap() - __first_;
+      size_type __new_cap = std::max<size_type>(2 * __old_cap, 8);
+      __split_buffer __buf(__new_cap, 0, __a);
+      for (pointer __p = __begin_; __p != __end_; ++__p, (void)++__buf.__end_)
+        __alloc_traits::construct(__buf.__alloc(), std::__to_address(__buf.__end_), std::move(*__p));
+      swap(__buf);
+    }
+    __alloc_traits::construct(__a, std::__to_address(this->__end_), *__first);
+    ++this->__end_;
+  }
+}
+template <class _Tp, class _Allocator>
+template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+__split_buffer<_Tp, _Allocator>::__construct_at_end(_ForwardIterator __first, _ForwardIterator __last) {
+  __construct_at_end_with_size(__first, std::distance(__first, __last));
+}
+
+template <class _Tp, class _Allocator>
+template <class _ForwardIterator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+__split_buffer<_Tp, _Allocator>::__construct_at_end_with_size(_ForwardIterator __first, size_type __n) {
+  _ConstructTransaction __tx(&this->__end_, __n);
+  for (; __tx.__pos_ != __tx.__end_; ++__tx.__pos_, (void)++__first) {
+    __alloc_traits::construct(this->__alloc(), std::__to_address(__tx.__pos_), *__first);
+  }
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void
+__split_buffer<_Tp, _Allocator>::__destruct_at_begin(pointer __new_begin, false_type) {
+  while (__begin_ != __new_begin)
+    __alloc_traits::destroy(__alloc(), std::__to_address(__begin_++));
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void
+__split_buffer<_Tp, _Allocator>::__destruct_at_begin(pointer __new_begin, true_type) {
+  __begin_ = __new_begin;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void
+__split_buffer<_Tp, _Allocator>::__destruct_at_end(pointer __new_last, false_type) _NOEXCEPT {
+  while (__new_last != __end_)
+    __alloc_traits::destroy(__alloc(), std::__to_address(--__end_));
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void
+__split_buffer<_Tp, _Allocator>::__destruct_at_end(pointer __new_last, true_type) _NOEXCEPT {
+  __end_ = __new_last;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+__split_buffer<_Tp, _Allocator>::__split_buffer(size_type __cap, size_type __start, __alloc_rr& __a)
+    : __end_cap_(nullptr, __a) {
+  if (__cap == 0) {
+    __first_ = nullptr;
+  } else {
+    auto __allocation = std::__allocate_at_least(__alloc(), __cap);
+    __first_          = __allocation.ptr;
+    __cap             = __allocation.count;
+  }
+  __begin_ = __end_ = __first_ + __start;
+  __end_cap()       = __first_ + __cap;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 __split_buffer<_Tp, _Allocator>::~__split_buffer() {
+  clear();
+  if (__first_)
+    __alloc_traits::deallocate(__alloc(), __first_, capacity());
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 __split_buffer<_Tp, _Allocator>::__split_buffer(__split_buffer&& __c)
+    _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value)
+    : __first_(std::move(__c.__first_)),
+      __begin_(std::move(__c.__begin_)),
+      __end_(std::move(__c.__end_)),
+      __end_cap_(std::move(__c.__end_cap_)) {
+  __c.__first_    = nullptr;
+  __c.__begin_    = nullptr;
+  __c.__end_      = nullptr;
+  __c.__end_cap() = nullptr;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+__split_buffer<_Tp, _Allocator>::__split_buffer(__split_buffer&& __c, const __alloc_rr& __a)
+    : __end_cap_(nullptr, __a) {
+  if (__a == __c.__alloc()) {
+    __first_        = __c.__first_;
+    __begin_        = __c.__begin_;
+    __end_          = __c.__end_;
+    __end_cap()     = __c.__end_cap();
+    __c.__first_    = nullptr;
+    __c.__begin_    = nullptr;
+    __c.__end_      = nullptr;
+    __c.__end_cap() = nullptr;
+  } else {
+    auto __allocation = std::__allocate_at_least(__alloc(), __c.size());
+    __first_          = __allocation.ptr;
+    __begin_ = __end_ = __first_;
+    __end_cap()       = __first_ + __allocation.count;
+    typedef move_iterator<iterator> _Ip;
+    __construct_at_end(_Ip(__c.begin()), _Ip(__c.end()));
+  }
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 __split_buffer<_Tp, _Allocator>&
+__split_buffer<_Tp, _Allocator>::operator=(__split_buffer&& __c)
+    _NOEXCEPT_((__alloc_traits::propagate_on_container_move_assignment::value &&
+                is_nothrow_move_assignable<allocator_type>::value) ||
+               !__alloc_traits::propagate_on_container_move_assignment::value) {
+  clear();
+  shrink_to_fit();
+  __first_    = __c.__first_;
+  __begin_    = __c.__begin_;
+  __end_      = __c.__end_;
+  __end_cap() = __c.__end_cap();
+  __move_assign_alloc(__c, integral_constant<bool, __alloc_traits::propagate_on_container_move_assignment::value>());
+  __c.__first_ = __c.__begin_ = __c.__end_ = __c.__end_cap() = nullptr;
+  return *this;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::swap(__split_buffer& __x)
+    _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__alloc_rr>) {
+  std::swap(__first_, __x.__first_);
+  std::swap(__begin_, __x.__begin_);
+  std::swap(__end_, __x.__end_);
+  std::swap(__end_cap(), __x.__end_cap());
+  std::__swap_allocator(__alloc(), __x.__alloc());
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::reserve(size_type __n) {
+  if (__n < capacity()) {
+    __split_buffer<value_type, __alloc_rr&> __t(__n, 0, __alloc());
+    __t.__construct_at_end(move_iterator<pointer>(__begin_), move_iterator<pointer>(__end_));
+    std::swap(__first_, __t.__first_);
+    std::swap(__begin_, __t.__begin_);
+    std::swap(__end_, __t.__end_);
+    std::swap(__end_cap(), __t.__end_cap());
+  }
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::shrink_to_fit() _NOEXCEPT {
+  if (capacity() > size()) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      __split_buffer<value_type, __alloc_rr&> __t(size(), 0, __alloc());
+      __t.__construct_at_end(move_iterator<pointer>(__begin_), move_iterator<pointer>(__end_));
+      __t.__end_ = __t.__begin_ + (__end_ - __begin_);
+      std::swap(__first_, __t.__first_);
+      std::swap(__begin_, __t.__begin_);
+      std::swap(__end_, __t.__end_);
+      std::swap(__end_cap(), __t.__end_cap());
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::push_front(const_reference __x) {
+  if (__begin_ == __first_) {
+    if (__end_ < __end_cap()) {
+      difference_type __d = __end_cap() - __end_;
+      __d                 = (__d + 1) / 2;
+      __begin_            = std::move_backward(__begin_, __end_, __end_ + __d);
+      __end_ += __d;
+    } else {
+      size_type __c = std::max<size_type>(2 * static_cast<size_t>(__end_cap() - __first_), 1);
+      __split_buffer<value_type, __alloc_rr&> __t(__c, (__c + 3) / 4, __alloc());
+      __t.__construct_at_end(move_iterator<pointer>(__begin_), move_iterator<pointer>(__end_));
+      std::swap(__first_, __t.__first_);
+      std::swap(__begin_, __t.__begin_);
+      std::swap(__end_, __t.__end_);
+      std::swap(__end_cap(), __t.__end_cap());
+    }
+  }
+  __alloc_traits::construct(__alloc(), std::__to_address(__begin_ - 1), __x);
+  --__begin_;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::push_front(value_type&& __x) {
+  if (__begin_ == __first_) {
+    if (__end_ < __end_cap()) {
+      difference_type __d = __end_cap() - __end_;
+      __d                 = (__d + 1) / 2;
+      __begin_            = std::move_backward(__begin_, __end_, __end_ + __d);
+      __end_ += __d;
+    } else {
+      size_type __c = std::max<size_type>(2 * static_cast<size_t>(__end_cap() - __first_), 1);
+      __split_buffer<value_type, __alloc_rr&> __t(__c, (__c + 3) / 4, __alloc());
+      __t.__construct_at_end(move_iterator<pointer>(__begin_), move_iterator<pointer>(__end_));
+      std::swap(__first_, __t.__first_);
+      std::swap(__begin_, __t.__begin_);
+      std::swap(__end_, __t.__end_);
+      std::swap(__end_cap(), __t.__end_cap());
+    }
+  }
+  __alloc_traits::construct(__alloc(), std::__to_address(__begin_ - 1), std::move(__x));
+  --__begin_;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void
+__split_buffer<_Tp, _Allocator>::push_back(const_reference __x) {
+  if (__end_ == __end_cap()) {
+    if (__begin_ > __first_) {
+      difference_type __d = __begin_ - __first_;
+      __d                 = (__d + 1) / 2;
+      __end_              = std::move(__begin_, __end_, __begin_ - __d);
+      __begin_ -= __d;
+    } else {
+      size_type __c = std::max<size_type>(2 * static_cast<size_t>(__end_cap() - __first_), 1);
+      __split_buffer<value_type, __alloc_rr&> __t(__c, __c / 4, __alloc());
+      __t.__construct_at_end(move_iterator<pointer>(__begin_), move_iterator<pointer>(__end_));
+      std::swap(__first_, __t.__first_);
+      std::swap(__begin_, __t.__begin_);
+      std::swap(__end_, __t.__end_);
+      std::swap(__end_cap(), __t.__end_cap());
+    }
+  }
+  __alloc_traits::construct(__alloc(), std::__to_address(__end_), __x);
+  ++__end_;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::push_back(value_type&& __x) {
+  if (__end_ == __end_cap()) {
+    if (__begin_ > __first_) {
+      difference_type __d = __begin_ - __first_;
+      __d                 = (__d + 1) / 2;
+      __end_              = std::move(__begin_, __end_, __begin_ - __d);
+      __begin_ -= __d;
+    } else {
+      size_type __c = std::max<size_type>(2 * static_cast<size_t>(__end_cap() - __first_), 1);
+      __split_buffer<value_type, __alloc_rr&> __t(__c, __c / 4, __alloc());
+      __t.__construct_at_end(move_iterator<pointer>(__begin_), move_iterator<pointer>(__end_));
+      std::swap(__first_, __t.__first_);
+      std::swap(__begin_, __t.__begin_);
+      std::swap(__end_, __t.__end_);
+      std::swap(__end_cap(), __t.__end_cap());
+    }
+  }
+  __alloc_traits::construct(__alloc(), std::__to_address(__end_), std::move(__x));
+  ++__end_;
+}
+
+template <class _Tp, class _Allocator>
+template <class... _Args>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::emplace_back(_Args&&... __args) {
+  if (__end_ == __end_cap()) {
+    if (__begin_ > __first_) {
+      difference_type __d = __begin_ - __first_;
+      __d                 = (__d + 1) / 2;
+      __end_              = std::move(__begin_, __end_, __begin_ - __d);
+      __begin_ -= __d;
+    } else {
+      size_type __c = std::max<size_type>(2 * static_cast<size_t>(__end_cap() - __first_), 1);
+      __split_buffer<value_type, __alloc_rr&> __t(__c, __c / 4, __alloc());
+      __t.__construct_at_end(move_iterator<pointer>(__begin_), move_iterator<pointer>(__end_));
+      std::swap(__first_, __t.__first_);
+      std::swap(__begin_, __t.__begin_);
+      std::swap(__end_, __t.__end_);
+      std::swap(__end_cap(), __t.__end_cap());
+    }
+  }
+  __alloc_traits::construct(__alloc(), std::__to_address(__end_), std::forward<_Args>(__args)...);
+  ++__end_;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void
+swap(__split_buffer<_Tp, _Allocator>& __x, __split_buffer<_Tp, _Allocator>& __y) _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___SPLIT_BUFFER
diff --git a/libcxx/include/__cxx03/__std_clang_module b/libcxx/include/__cxx03/__std_clang_module
new file mode 100644
index 000000000000..18d6ce6b46c1
--- /dev/null
+++ b/libcxx/include/__cxx03/__std_clang_module
@@ -0,0 +1,215 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// WARNING, this entire header is generated by
+// utils/generate_std_clang_module_header.py
+// DO NOT MODIFY!
+
+// This header should not be directly included, it's exclusively to import all
+// of the libc++ public clang modules for the `std` clang module to export. In
+// other words, it's to facilitate `@import std;` in Objective-C++ and `import std`
+// in Swift to expose all of the libc++ interfaces. This is generally not
+// recommended, however there are some clients that need to import all of libc++
+// without knowing what "all" is.
+#if !__building_module(std)
+#  error "Do not include this header directly, include individual headers instead"
+#endif
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#include <algorithm>
+#include <any>
+#include <array>
+#if !defined(_LIBCPP_HAS_NO_ATOMIC_HEADER)
+#  include <atomic>
+#endif
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+#  include <barrier>
+#endif
+#include <bit>
+#include <bitset>
+#include <cassert>
+#include <ccomplex>
+#include <cctype>
+#include <cerrno>
+#include <cfenv>
+#include <cfloat>
+#include <charconv>
+#include <chrono>
+#include <cinttypes>
+#include <ciso646>
+#include <climits>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <clocale>
+#endif
+#include <cmath>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <codecvt>
+#endif
+#include <compare>
+#include <complex.h>
+#include <complex>
+#include <concepts>
+#include <condition_variable>
+#include <coroutine>
+#include <csetjmp>
+#include <csignal>
+#include <cstdarg>
+#include <cstdbool>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <ctgmath>
+#include <ctime>
+#include <ctype.h>
+#include <cuchar>
+#if !defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS)
+#  include <cwchar>
+#endif
+#if !defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS)
+#  include <cwctype>
+#endif
+#include <deque>
+#include <errno.h>
+#include <exception>
+#include <execution>
+#include <expected>
+#include <experimental/iterator>
+#include <experimental/memory>
+#include <experimental/propagate_const>
+#include <experimental/simd>
+#include <experimental/type_traits>
+#include <experimental/utility>
+#include <fenv.h>
+#include <filesystem>
+#include <float.h>
+#include <format>
+#include <forward_list>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <fstream>
+#endif
+#include <functional>
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+#  include <future>
+#endif
+#include <initializer_list>
+#include <inttypes.h>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <iomanip>
+#endif
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <ios>
+#endif
+#include <iosfwd>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <iostream>
+#endif
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <istream>
+#endif
+#include <iterator>
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+#  include <latch>
+#endif
+#include <limits>
+#include <list>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <locale.h>
+#endif
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <locale>
+#endif
+#include <map>
+#include <math.h>
+#include <mdspan>
+#include <memory>
+#include <memory_resource>
+#include <mutex>
+#include <new>
+#include <numbers>
+#include <numeric>
+#include <optional>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <ostream>
+#endif
+#include <print>
+#include <queue>
+#include <random>
+#include <ranges>
+#include <ratio>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <regex>
+#endif
+#include <scoped_allocator>
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+#  include <semaphore>
+#endif
+#include <set>
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+#  include <shared_mutex>
+#endif
+#include <source_location>
+#include <span>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <sstream>
+#endif
+#include <stack>
+#if !defined(_LIBCPP_HAS_NO_ATOMIC_HEADER)
+#  include <stdatomic.h>
+#endif
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdexcept>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+#  include <stop_token>
+#endif
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <streambuf>
+#endif
+#include <string.h>
+#include <string>
+#include <string_view>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <strstream>
+#endif
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <syncstream>
+#endif
+#include <system_error>
+#include <tgmath.h>
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+#  include <thread>
+#endif
+#include <tuple>
+#include <type_traits>
+#include <typeindex>
+#include <typeinfo>
+#include <uchar.h>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <valarray>
+#include <variant>
+#include <vector>
+#include <version>
+#if !defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS)
+#  include <wchar.h>
+#endif
+#if !defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS)
+#  include <wctype.h>
+#endif
diff --git a/libcxx/include/__cxx03/__std_mbstate_t.h b/libcxx/include/__cxx03/__std_mbstate_t.h
new file mode 100644
index 000000000000..e79cc789fddf
--- /dev/null
+++ b/libcxx/include/__cxx03/__std_mbstate_t.h
@@ -0,0 +1,29 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___STD_MBSTATE_T_H
+#define _LIBCPP___STD_MBSTATE_T_H
+
+#include <__config>
+#include <__mbstate_t.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+// The goal of this header is to provide std::mbstate_t without requiring all
+// of <cuchar> or <cwchar>.
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::mbstate_t _LIBCPP_USING_IF_EXISTS;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___STD_MBSTATE_T_H
diff --git a/libcxx/include/__cxx03/__stop_token/atomic_unique_lock.h b/libcxx/include/__cxx03/__stop_token/atomic_unique_lock.h
new file mode 100644
index 000000000000..13e59f9f0dce
--- /dev/null
+++ b/libcxx/include/__cxx03/__stop_token/atomic_unique_lock.h
@@ -0,0 +1,140 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___STOP_TOKEN_ATOMIC_UNIQUE_GUARD_H
+#define _LIBCPP___STOP_TOKEN_ATOMIC_UNIQUE_GUARD_H
+
+#include <__bit/popcount.h>
+#include <__config>
+#include <atomic>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// This class implements an RAII unique_lock without a mutex.
+// It uses std::atomic<State>,
+// where State contains a lock bit and might contain other data,
+// and LockedBit is the value of State when the lock bit is set, e.g  1 << 2
+template <class _State, _State _LockedBit>
+class _LIBCPP_AVAILABILITY_SYNC __atomic_unique_lock {
+  static_assert(std::__libcpp_popcount(static_cast<unsigned long long>(_LockedBit)) == 1,
+                "LockedBit must be an integer where only one bit is set");
+
+  std::atomic<_State>& __state_;
+  bool __is_locked_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __atomic_unique_lock(std::atomic<_State>& __state) noexcept
+      : __state_(__state), __is_locked_(true) {
+    __lock();
+  }
+
+  template <class _Pred>
+  _LIBCPP_HIDE_FROM_ABI __atomic_unique_lock(std::atomic<_State>& __state, _Pred&& __give_up_locking) noexcept
+      : __state_(__state), __is_locked_(false) {
+    __is_locked_ = __lock_impl(__give_up_locking, __set_locked_bit, std::memory_order_acquire);
+  }
+
+  template <class _Pred, class _UnaryFunction>
+  _LIBCPP_HIDE_FROM_ABI __atomic_unique_lock(
+      std::atomic<_State>& __state,
+      _Pred&& __give_up_locking,
+      _UnaryFunction&& __state_after_lock,
+      std::memory_order __locked_ordering) noexcept
+      : __state_(__state), __is_locked_(false) {
+    __is_locked_ = __lock_impl(__give_up_locking, __state_after_lock, __locked_ordering);
+  }
+
+  __atomic_unique_lock(const __atomic_unique_lock&)            = delete;
+  __atomic_unique_lock(__atomic_unique_lock&&)                 = delete;
+  __atomic_unique_lock& operator=(const __atomic_unique_lock&) = delete;
+  __atomic_unique_lock& operator=(__atomic_unique_lock&&)      = delete;
+
+  _LIBCPP_HIDE_FROM_ABI ~__atomic_unique_lock() {
+    if (__is_locked_) {
+      __unlock();
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool __owns_lock() const noexcept { return __is_locked_; }
+
+  _LIBCPP_HIDE_FROM_ABI void __lock() noexcept {
+    const auto __never_give_up_locking = [](_State) { return false; };
+    // std::memory_order_acquire because we'd like to make sure that all the read operations after the lock can read the
+    // up-to-date values.
+    __lock_impl(__never_give_up_locking, __set_locked_bit, std::memory_order_acquire);
+    __is_locked_ = true;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __unlock() noexcept {
+    // unset the _LockedBit. `memory_order_release` because we need to make sure all the write operations before calling
+    // `__unlock` will be made visible to other threads
+    __state_.fetch_and(static_cast<_State>(~_LockedBit), std::memory_order_release);
+    __state_.notify_all();
+    __is_locked_ = false;
+  }
+
+private:
+  template <class _Pred, class _UnaryFunction>
+  _LIBCPP_HIDE_FROM_ABI bool
+  __lock_impl(_Pred&& __give_up_locking, // while trying to lock the state, if the predicate returns true, give up
+                                         // locking and return
+              _UnaryFunction&& __state_after_lock,
+              std::memory_order __locked_ordering) noexcept {
+    // At this stage, until we exit the inner while loop, other than the atomic state, we are not reading any order
+    // dependent values that is written on other threads, or writing anything that needs to be seen on other threads.
+    // Therefore `memory_order_relaxed` is enough.
+    _State __current_state = __state_.load(std::memory_order_relaxed);
+    do {
+      while (true) {
+        if (__give_up_locking(__current_state)) {
+          // user provided early return condition. fail to lock
+          return false;
+        } else if ((__current_state & _LockedBit) != 0) {
+          // another thread has locked the state, we need to wait
+          __state_.wait(__current_state, std::memory_order_relaxed);
+          // when it is woken up by notifyAll or spuriously, the __state_
+          // might have changed. reload the state
+          // Note that the new state's _LockedBit may or may not equal to 0
+          __current_state = __state_.load(std::memory_order_relaxed);
+        } else {
+          // at least for now, it is not locked. we can try `compare_exchange_weak` to lock it.
+          // Note that the variable `__current_state`'s lock bit has to be 0 at this point.
+          break;
+        }
+      }
+    } while (!__state_.compare_exchange_weak(
+        __current_state, // if __state_ has the same value of __current_state, lock bit must be zero before exchange and
+                         // we are good to lock/exchange and return. If _state has a different value, because other
+                         // threads locked it between the `break` statement above and this statement, exchange will fail
+                         // and go back to the inner while loop above.
+        __state_after_lock(__current_state), // state after lock. Usually it should be __current_state | _LockedBit.
+                                             // Some use cases need to set other bits at the same time as an atomic
+                                             // operation therefore we accept a function
+        __locked_ordering,        // sucessful exchange order. Usually it should be std::memory_order_acquire.
+                                  // Some use cases need more strict ordering therefore we accept it as a parameter
+        std::memory_order_relaxed // fail to exchange order. We don't need any ordering as we are going back to the
+                                  // inner while loop
+        ));
+    return true;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __set_locked_bit = [](_State __state) { return __state | _LockedBit; };
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___STOP_TOKEN_ATOMIC_UNIQUE_GUARD_H
diff --git a/libcxx/include/__cxx03/__stop_token/intrusive_list_view.h b/libcxx/include/__cxx03/__stop_token/intrusive_list_view.h
new file mode 100644
index 000000000000..11a3e267e7c6
--- /dev/null
+++ b/libcxx/include/__cxx03/__stop_token/intrusive_list_view.h
@@ -0,0 +1,85 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___STOP_TOKEN_INTRUSIVE_LIST_VIEW_H
+#define _LIBCPP___STOP_TOKEN_INTRUSIVE_LIST_VIEW_H
+
+#include <__assert>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Derived>
+struct __intrusive_node_base {
+  _Derived* __next_ = nullptr;
+  _Derived* __prev_ = nullptr;
+};
+
+// This class is a view of underlying double-linked list.
+// It does not own the nodes. It provides user-friendly
+// operations on the linked list.
+template <class _Node>
+struct __intrusive_list_view {
+  _LIBCPP_HIDE_FROM_ABI __intrusive_list_view()                                        = default;
+  _LIBCPP_HIDE_FROM_ABI __intrusive_list_view(__intrusive_list_view const&)            = default;
+  _LIBCPP_HIDE_FROM_ABI __intrusive_list_view(__intrusive_list_view&&)                 = default;
+  _LIBCPP_HIDE_FROM_ABI __intrusive_list_view& operator=(__intrusive_list_view const&) = default;
+  _LIBCPP_HIDE_FROM_ABI __intrusive_list_view& operator=(__intrusive_list_view&&)      = default;
+  _LIBCPP_HIDE_FROM_ABI ~__intrusive_list_view()                                       = default;
+
+  _LIBCPP_HIDE_FROM_ABI bool __empty() const noexcept { return __head_ == nullptr; }
+
+  _LIBCPP_HIDE_FROM_ABI void __push_front(_Node* __node) noexcept {
+    __node->__next_ = __head_;
+    if (__head_) {
+      __head_->__prev_ = __node;
+    }
+    __head_ = __node;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Node* __pop_front() noexcept {
+    _Node* __front = __head_;
+    __head_        = __head_->__next_;
+    if (__head_) {
+      __head_->__prev_ = nullptr;
+    }
+    // OK not to set __front->__next_ = nullptr as __front is not part of the list anymore
+    return __front;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __remove(_Node* __node) noexcept {
+    if (__node->__prev_) {
+      // prev exists, set its next to our next to skip __node
+      __node->__prev_->__next_ = __node->__next_;
+      if (__node->__next_) {
+        __node->__next_->__prev_ = __node->__prev_;
+      }
+    } else {
+      _LIBCPP_ASSERT_INTERNAL(__node == __head_, "Node to be removed has no prev node, so it has to be the head");
+      __pop_front();
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool __is_head(_Node* __node) noexcept { return __node == __head_; }
+
+private:
+  _Node* __head_ = nullptr;
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___STOP_TOKEN_INTRUSIVE_LIST_VIEW_H
diff --git a/libcxx/include/__cxx03/__stop_token/intrusive_shared_ptr.h b/libcxx/include/__cxx03/__stop_token/intrusive_shared_ptr.h
new file mode 100644
index 000000000000..f00cea5bc2b6
--- /dev/null
+++ b/libcxx/include/__cxx03/__stop_token/intrusive_shared_ptr.h
@@ -0,0 +1,134 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___STOP_TOKEN_INTRUSIVE_SHARED_PTR_H
+#define _LIBCPP___STOP_TOKEN_INTRUSIVE_SHARED_PTR_H
+
+#include <__atomic/atomic.h>
+#include <__atomic/memory_order.h>
+#include <__config>
+#include <__type_traits/is_reference.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+// For intrusive_shared_ptr to work with a type T, specialize __intrusive_shared_ptr_traits<T> and implement
+// the following function:
+//
+// static std::atomic<U>& __get_atomic_ref_count(T&);
+//
+// where U must be an integral type representing the number of references to the object.
+template <class _Tp>
+struct __intrusive_shared_ptr_traits;
+
+// A reference counting shared_ptr for types whose reference counter
+// is stored inside the class _Tp itself.
+// When the reference count goes to zero, the destructor of _Tp will be called
+template <class _Tp>
+struct __intrusive_shared_ptr {
+  _LIBCPP_HIDE_FROM_ABI __intrusive_shared_ptr() = default;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __intrusive_shared_ptr(_Tp* __raw_ptr) : __raw_ptr_(__raw_ptr) {
+    if (__raw_ptr_)
+      __increment_ref_count(*__raw_ptr_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __intrusive_shared_ptr(const __intrusive_shared_ptr& __other) noexcept
+      : __raw_ptr_(__other.__raw_ptr_) {
+    if (__raw_ptr_)
+      __increment_ref_count(*__raw_ptr_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __intrusive_shared_ptr(__intrusive_shared_ptr&& __other) noexcept
+      : __raw_ptr_(__other.__raw_ptr_) {
+    __other.__raw_ptr_ = nullptr;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __intrusive_shared_ptr& operator=(const __intrusive_shared_ptr& __other) noexcept {
+    if (__other.__raw_ptr_ != __raw_ptr_) {
+      if (__other.__raw_ptr_) {
+        __increment_ref_count(*__other.__raw_ptr_);
+      }
+      if (__raw_ptr_) {
+        __decrement_ref_count(*__raw_ptr_);
+      }
+      __raw_ptr_ = __other.__raw_ptr_;
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __intrusive_shared_ptr& operator=(__intrusive_shared_ptr&& __other) noexcept {
+    __intrusive_shared_ptr(std::move(__other)).swap(*this);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~__intrusive_shared_ptr() {
+    if (__raw_ptr_) {
+      __decrement_ref_count(*__raw_ptr_);
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator->() const noexcept { return __raw_ptr_; }
+  _LIBCPP_HIDE_FROM_ABI _Tp& operator*() const noexcept { return *__raw_ptr_; }
+  _LIBCPP_HIDE_FROM_ABI explicit operator bool() const noexcept { return __raw_ptr_ != nullptr; }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(__intrusive_shared_ptr& __other) { std::swap(__raw_ptr_, __other.__raw_ptr_); }
+
+  _LIBCPP_HIDE_FROM_ABI friend void swap(__intrusive_shared_ptr& __lhs, __intrusive_shared_ptr& __rhs) {
+    __lhs.swap(__rhs);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend bool constexpr
+  operator==(const __intrusive_shared_ptr&, const __intrusive_shared_ptr&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI friend bool constexpr operator==(const __intrusive_shared_ptr& __ptr, std::nullptr_t) {
+    return __ptr.__raw_ptr_ == nullptr;
+  }
+
+private:
+  _Tp* __raw_ptr_ = nullptr;
+
+  // the memory order for increment/decrement the counter is the same for shared_ptr
+  // increment is relaxed and decrement is acq_rel
+  _LIBCPP_HIDE_FROM_ABI static void __increment_ref_count(_Tp& __obj) {
+    __get_atomic_ref_count(__obj).fetch_add(1, std::memory_order_relaxed);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static void __decrement_ref_count(_Tp& __obj) {
+    if (__get_atomic_ref_count(__obj).fetch_sub(1, std::memory_order_acq_rel) == 1) {
+      delete &__obj;
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static decltype(auto) __get_atomic_ref_count(_Tp& __obj) {
+    using __ret_type = decltype(__intrusive_shared_ptr_traits<_Tp>::__get_atomic_ref_count(__obj));
+    static_assert(
+        std::is_reference_v<__ret_type>, "__get_atomic_ref_count should return a reference to the atomic counter");
+    return __intrusive_shared_ptr_traits<_Tp>::__get_atomic_ref_count(__obj);
+  }
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___STOP_TOKEN_INTRUSIVE_SHARED_PTR_H
diff --git a/libcxx/include/__cxx03/__stop_token/stop_callback.h b/libcxx/include/__cxx03/__stop_token/stop_callback.h
new file mode 100644
index 000000000000..760cf2bb55b0
--- /dev/null
+++ b/libcxx/include/__cxx03/__stop_token/stop_callback.h
@@ -0,0 +1,102 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___STOP_TOKEN_STOP_CALLBACK_H
+#define _LIBCPP___STOP_TOKEN_STOP_CALLBACK_H
+
+#include <__concepts/constructible.h>
+#include <__concepts/destructible.h>
+#include <__concepts/invocable.h>
+#include <__config>
+#include <__stop_token/intrusive_shared_ptr.h>
+#include <__stop_token/stop_state.h>
+#include <__stop_token/stop_token.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/private_constructor_tag.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN) && !defined(_LIBCPP_HAS_NO_THREADS)
+
+template <class _Callback>
+class _LIBCPP_AVAILABILITY_SYNC stop_callback : private __stop_callback_base {
+  static_assert(invocable<_Callback>,
+                "Mandates: stop_callback is instantiated with an argument for the template parameter Callback that "
+                "satisfies invocable.");
+  static_assert(destructible<_Callback>,
+                "Mandates: stop_callback is instantiated with an argument for the template parameter Callback that "
+                "satisfies destructible.");
+
+public:
+  using callback_type = _Callback;
+
+  template <class _Cb>
+    requires constructible_from<_Callback, _Cb>
+  _LIBCPP_HIDE_FROM_ABI explicit stop_callback(const stop_token& __st,
+                                               _Cb&& __cb) noexcept(is_nothrow_constructible_v<_Callback, _Cb>)
+      : stop_callback(__private_constructor_tag{}, __st.__state_, std::forward<_Cb>(__cb)) {}
+
+  template <class _Cb>
+    requires constructible_from<_Callback, _Cb>
+  _LIBCPP_HIDE_FROM_ABI explicit stop_callback(stop_token&& __st,
+                                               _Cb&& __cb) noexcept(is_nothrow_constructible_v<_Callback, _Cb>)
+      : stop_callback(__private_constructor_tag{}, std::move(__st.__state_), std::forward<_Cb>(__cb)) {}
+
+  _LIBCPP_HIDE_FROM_ABI ~stop_callback() {
+    if (__state_) {
+      __state_->__remove_callback(this);
+    }
+  }
+
+  stop_callback(const stop_callback&)            = delete;
+  stop_callback(stop_callback&&)                 = delete;
+  stop_callback& operator=(const stop_callback&) = delete;
+  stop_callback& operator=(stop_callback&&)      = delete;
+
+private:
+  _LIBCPP_NO_UNIQUE_ADDRESS _Callback __callback_;
+  __intrusive_shared_ptr<__stop_state> __state_;
+
+  friend __stop_callback_base;
+
+  template <class _StatePtr, class _Cb>
+  _LIBCPP_HIDE_FROM_ABI explicit stop_callback(__private_constructor_tag, _StatePtr&& __state, _Cb&& __cb) noexcept(
+      is_nothrow_constructible_v<_Callback, _Cb>)
+      : __stop_callback_base([](__stop_callback_base* __cb_base) noexcept {
+          // stop callback is supposed to only be called once
+          std::forward<_Callback>(static_cast<stop_callback*>(__cb_base)->__callback_)();
+        }),
+        __callback_(std::forward<_Cb>(__cb)),
+        __state_() {
+    if (__state && __state->__add_callback(this)) {
+      // st.stop_requested() was false and this is successfully added to the linked list
+      __state_ = std::forward<_StatePtr>(__state);
+    }
+  }
+};
+
+template <class _Callback>
+_LIBCPP_AVAILABILITY_SYNC stop_callback(stop_token, _Callback) -> stop_callback<_Callback>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN) && !defined(_LIBCPP_HAS_NO_THREADS)
diff --git a/libcxx/include/__cxx03/__stop_token/stop_source.h b/libcxx/include/__cxx03/__stop_token/stop_source.h
new file mode 100644
index 000000000000..70697462784a
--- /dev/null
+++ b/libcxx/include/__cxx03/__stop_token/stop_source.h
@@ -0,0 +1,91 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___STOP_TOKEN_STOP_SOURCE_H
+#define _LIBCPP___STOP_TOKEN_STOP_SOURCE_H
+
+#include <__config>
+#include <__stop_token/intrusive_shared_ptr.h>
+#include <__stop_token/stop_state.h>
+#include <__stop_token/stop_token.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN) && !defined(_LIBCPP_HAS_NO_THREADS)
+
+struct nostopstate_t {
+  explicit nostopstate_t() = default;
+};
+
+inline constexpr nostopstate_t nostopstate{};
+
+class _LIBCPP_AVAILABILITY_SYNC stop_source {
+public:
+  _LIBCPP_HIDE_FROM_ABI stop_source() : __state_(new __stop_state()) { __state_->__increment_stop_source_counter(); }
+
+  _LIBCPP_HIDE_FROM_ABI explicit stop_source(nostopstate_t) noexcept : __state_(nullptr) {}
+
+  _LIBCPP_HIDE_FROM_ABI stop_source(const stop_source& __other) noexcept : __state_(__other.__state_) {
+    if (__state_) {
+      __state_->__increment_stop_source_counter();
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI stop_source(stop_source&& __other) noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI stop_source& operator=(const stop_source& __other) noexcept {
+    // increment `__other` first so that we don't hit 0 in case of self-assignment
+    if (__other.__state_) {
+      __other.__state_->__increment_stop_source_counter();
+    }
+    if (__state_) {
+      __state_->__decrement_stop_source_counter();
+    }
+    __state_ = __other.__state_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI stop_source& operator=(stop_source&&) noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI ~stop_source() {
+    if (__state_) {
+      __state_->__decrement_stop_source_counter();
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(stop_source& __other) noexcept { __state_.swap(__other.__state_); }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI stop_token get_token() const noexcept { return stop_token(__state_); }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool stop_possible() const noexcept { return __state_ != nullptr; }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool stop_requested() const noexcept {
+    return __state_ != nullptr && __state_->__stop_requested();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool request_stop() noexcept { return __state_ && __state_->__request_stop(); }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend bool operator==(const stop_source&, const stop_source&) noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI friend void swap(stop_source& __lhs, stop_source& __rhs) noexcept { __lhs.swap(__rhs); }
+
+private:
+  __intrusive_shared_ptr<__stop_state> __state_;
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN) && !defined(_LIBCPP_HAS_NO_THREADS)
diff --git a/libcxx/include/__cxx03/__stop_token/stop_state.h b/libcxx/include/__cxx03/__stop_token/stop_state.h
new file mode 100644
index 000000000000..b0eed13a143c
--- /dev/null
+++ b/libcxx/include/__cxx03/__stop_token/stop_state.h
@@ -0,0 +1,236 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___STOP_TOKEN_STOP_STATE_H
+#define _LIBCPP___STOP_TOKEN_STOP_STATE_H
+
+#include <__assert>
+#include <__config>
+#include <__stop_token/atomic_unique_lock.h>
+#include <__stop_token/intrusive_list_view.h>
+#include <__thread/id.h>
+#include <atomic>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_THREADS)
+
+struct __stop_callback_base : __intrusive_node_base<__stop_callback_base> {
+  using __callback_fn_t = void(__stop_callback_base*) noexcept;
+  _LIBCPP_HIDE_FROM_ABI explicit __stop_callback_base(__callback_fn_t* __callback_fn) : __callback_fn_(__callback_fn) {}
+
+  _LIBCPP_HIDE_FROM_ABI void __invoke() noexcept { __callback_fn_(this); }
+
+  __callback_fn_t* __callback_fn_;
+  atomic<bool> __completed_ = false;
+  bool* __destroyed_        = nullptr;
+};
+
+class __stop_state {
+  static constexpr uint32_t __stop_requested_bit        = 1;
+  static constexpr uint32_t __callback_list_locked_bit  = 1 << 1;
+  static constexpr uint32_t __stop_source_counter_shift = 2;
+
+  // The "stop_source counter" is not used for lifetime reference counting.
+  // When the number of stop_source reaches 0, the remaining stop_tokens's
+  // stop_possible will return false. We need this counter to track this.
+  //
+  // The "callback list locked" bit implements the atomic_unique_lock to
+  // guard the operations on the callback list
+  //
+  //       31 - 2          |  1                   |    0           |
+  //  stop_source counter  | callback list locked | stop_requested |
+  atomic<uint32_t> __state_ = 0;
+
+  // Reference count for stop_token + stop_callback + stop_source
+  // When the counter reaches zero, the state is destroyed
+  // It is used by __intrusive_shared_ptr, but it is stored here for better layout
+  atomic<uint32_t> __ref_count_ = 0;
+
+  using __state_t            = uint32_t;
+  using __callback_list_lock = __atomic_unique_lock<__state_t, __callback_list_locked_bit>;
+  using __callback_list      = __intrusive_list_view<__stop_callback_base>;
+
+  __callback_list __callback_list_;
+  __thread_id __requesting_thread_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __stop_state() noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI void __increment_stop_source_counter() noexcept {
+    _LIBCPP_ASSERT_UNCATEGORIZED(
+        __state_.load(std::memory_order_relaxed) <= static_cast<__state_t>(~(1 << __stop_source_counter_shift)),
+        "stop_source's counter reaches the maximum. Incrementing the counter will overflow");
+    __state_.fetch_add(1 << __stop_source_counter_shift, std::memory_order_relaxed);
+  }
+
+  // We are not destroying the object after counter decrements to zero, nor do we have
+  // operations depend on the ordering of decrementing the counter. relaxed is enough.
+  _LIBCPP_HIDE_FROM_ABI void __decrement_stop_source_counter() noexcept {
+    _LIBCPP_ASSERT_UNCATEGORIZED(
+        __state_.load(std::memory_order_relaxed) >= static_cast<__state_t>(1 << __stop_source_counter_shift),
+        "stop_source's counter is 0. Decrementing the counter will underflow");
+    __state_.fetch_sub(1 << __stop_source_counter_shift, std::memory_order_relaxed);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool __stop_requested() const noexcept {
+    // acquire because [thread.stoptoken.intro] A call to request_stop that returns true
+    // synchronizes with a call to stop_requested on an associated stop_token or stop_source
+    // object that returns true.
+    // request_stop's compare_exchange_weak has release which syncs with this acquire
+    return (__state_.load(std::memory_order_acquire) & __stop_requested_bit) != 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool __stop_possible_for_stop_token() const noexcept {
+    // [stoptoken.mem] false if "a stop request was not made and there are no associated stop_source objects"
+    // Todo: Can this be std::memory_order_relaxed as the standard does not say anything except not to introduce data
+    // race?
+    __state_t __curent_state = __state_.load(std::memory_order_acquire);
+    return ((__curent_state & __stop_requested_bit) != 0) || ((__curent_state >> __stop_source_counter_shift) != 0);
+  }
+
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool __request_stop() noexcept {
+    auto __cb_list_lock = __try_lock_for_request_stop();
+    if (!__cb_list_lock.__owns_lock()) {
+      return false;
+    }
+    __requesting_thread_ = this_thread::get_id();
+
+    while (!__callback_list_.__empty()) {
+      auto __cb = __callback_list_.__pop_front();
+
+      // allow other callbacks to be removed while invoking the current callback
+      __cb_list_lock.__unlock();
+
+      bool __destroyed   = false;
+      __cb->__destroyed_ = &__destroyed;
+
+      __cb->__invoke();
+
+      // __cb's invoke function could potentially delete itself. We need to check before accessing __cb's member
+      if (!__destroyed) {
+        // needs to set __destroyed_ pointer to nullptr, otherwise it points to a local variable
+        // which is to be destroyed at the end of the loop
+        __cb->__destroyed_ = nullptr;
+
+        // [stopcallback.cons] If callback is concurrently executing on another thread, then the return
+        // from the invocation of callback strongly happens before ([intro.races]) callback is destroyed.
+        // this release syncs with the acquire in the remove_callback
+        __cb->__completed_.store(true, std::memory_order_release);
+        __cb->__completed_.notify_all();
+      }
+
+      __cb_list_lock.__lock();
+    }
+
+    return true;
+  }
+
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool __add_callback(__stop_callback_base* __cb) noexcept {
+    // If it is already stop_requested. Do not try to request it again.
+    const auto __give_up_trying_to_lock_condition = [__cb](__state_t __state) {
+      if ((__state & __stop_requested_bit) != 0) {
+        // already stop requested, synchronously run the callback and no need to lock the list again
+        __cb->__invoke();
+        return true;
+      }
+      // no stop source. no need to lock the list to add the callback as it can never be invoked
+      return (__state >> __stop_source_counter_shift) == 0;
+    };
+
+    __callback_list_lock __cb_list_lock(__state_, __give_up_trying_to_lock_condition);
+
+    if (!__cb_list_lock.__owns_lock()) {
+      return false;
+    }
+
+    __callback_list_.__push_front(__cb);
+
+    return true;
+    // unlock here: [thread.stoptoken.intro] Registration of a callback synchronizes with the invocation of
+    // that callback.
+    // Note: this release sync with the acquire in the request_stop' __try_lock_for_request_stop
+  }
+
+  // called by the destructor of stop_callback
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __remove_callback(__stop_callback_base* __cb) noexcept {
+    __callback_list_lock __cb_list_lock(__state_);
+
+    // under below condition, the request_stop call just popped __cb from the list and could execute it now
+    bool __potentially_executing_now = __cb->__prev_ == nullptr && !__callback_list_.__is_head(__cb);
+
+    if (__potentially_executing_now) {
+      auto __requested_thread = __requesting_thread_;
+      __cb_list_lock.__unlock();
+
+      if (std::this_thread::get_id() != __requested_thread) {
+        // [stopcallback.cons] If callback is concurrently executing on another thread, then the return
+        // from the invocation of callback strongly happens before ([intro.races]) callback is destroyed.
+        __cb->__completed_.wait(false, std::memory_order_acquire);
+      } else {
+        // The destructor of stop_callback runs on the same thread of the thread that invokes the callback.
+        // The callback is potentially invoking its own destuctor. Set the flag to avoid accessing destroyed
+        // members on the invoking side
+        if (__cb->__destroyed_) {
+          *__cb->__destroyed_ = true;
+        }
+      }
+    } else {
+      __callback_list_.__remove(__cb);
+    }
+  }
+
+private:
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI __callback_list_lock __try_lock_for_request_stop() noexcept {
+    // If it is already stop_requested, do not try to request stop or lock the list again.
+    const auto __lock_fail_condition = [](__state_t __state) { return (__state & __stop_requested_bit) != 0; };
+
+    // set locked and requested bit at the same time
+    const auto __after_lock_state = [](__state_t __state) {
+      return __state | __callback_list_locked_bit | __stop_requested_bit;
+    };
+
+    // acq because [thread.stoptoken.intro] Registration of a callback synchronizes with the invocation of that
+    //     callback. We are going to invoke the callback after getting the lock, acquire so that we can see the
+    //     registration of a callback (and other writes that happens-before the add_callback)
+    //     Note: the rel (unlock) in the add_callback syncs with this acq
+    // rel because [thread.stoptoken.intro] A call to request_stop that returns true synchronizes with a call
+    //     to stop_requested on an associated stop_token or stop_source object that returns true.
+    //     We need to make sure that all writes (including user code) before request_stop will be made visible
+    //     to the threads that waiting for `stop_requested == true`
+    //     Note: this rel syncs with the acq in `stop_requested`
+    const auto __locked_ordering = std::memory_order_acq_rel;
+
+    return __callback_list_lock(__state_, __lock_fail_condition, __after_lock_state, __locked_ordering);
+  }
+
+  template <class _Tp>
+  friend struct __intrusive_shared_ptr_traits;
+};
+
+template <class _Tp>
+struct __intrusive_shared_ptr_traits;
+
+template <>
+struct __intrusive_shared_ptr_traits<__stop_state> {
+  _LIBCPP_HIDE_FROM_ABI static atomic<uint32_t>& __get_atomic_ref_count(__stop_state& __state) {
+    return __state.__ref_count_;
+  }
+};
+
+#endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_THREADS)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___STOP_TOKEN_STOP_STATE_H
diff --git a/libcxx/include/__cxx03/__stop_token/stop_token.h b/libcxx/include/__cxx03/__stop_token/stop_token.h
new file mode 100644
index 000000000000..1bd75cbbf6f8
--- /dev/null
+++ b/libcxx/include/__cxx03/__stop_token/stop_token.h
@@ -0,0 +1,63 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___STOP_TOKEN_STOP_TOKEN_H
+#define _LIBCPP___STOP_TOKEN_STOP_TOKEN_H
+
+#include <__config>
+#include <__stop_token/intrusive_shared_ptr.h>
+#include <__stop_token/stop_state.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN) && !defined(_LIBCPP_HAS_NO_THREADS)
+
+class _LIBCPP_AVAILABILITY_SYNC stop_token {
+public:
+  _LIBCPP_HIDE_FROM_ABI stop_token() noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI stop_token(const stop_token&) noexcept            = default;
+  _LIBCPP_HIDE_FROM_ABI stop_token(stop_token&&) noexcept                 = default;
+  _LIBCPP_HIDE_FROM_ABI stop_token& operator=(const stop_token&) noexcept = default;
+  _LIBCPP_HIDE_FROM_ABI stop_token& operator=(stop_token&&) noexcept      = default;
+  _LIBCPP_HIDE_FROM_ABI ~stop_token()                                     = default;
+
+  _LIBCPP_HIDE_FROM_ABI void swap(stop_token& __other) noexcept { __state_.swap(__other.__state_); }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool stop_requested() const noexcept {
+    return __state_ != nullptr && __state_->__stop_requested();
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool stop_possible() const noexcept {
+    return __state_ != nullptr && __state_->__stop_possible_for_stop_token();
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend bool operator==(const stop_token&, const stop_token&) noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI friend void swap(stop_token& __lhs, stop_token& __rhs) noexcept { __lhs.swap(__rhs); }
+
+private:
+  __intrusive_shared_ptr<__stop_state> __state_;
+
+  friend class stop_source;
+  template <class _Tp>
+  friend class stop_callback;
+
+  _LIBCPP_HIDE_FROM_ABI explicit stop_token(const __intrusive_shared_ptr<__stop_state>& __state) : __state_(__state) {}
+};
+
+#endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN) && !defined(_LIBCPP_HAS_NO_THREADS)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___STOP_TOKEN_STOP_TOKEN_H
diff --git a/libcxx/include/__cxx03/__string/char_traits.h b/libcxx/include/__cxx03/__string/char_traits.h
new file mode 100644
index 000000000000..2660ac2ede2d
--- /dev/null
+++ b/libcxx/include/__cxx03/__string/char_traits.h
@@ -0,0 +1,543 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___STRING_CHAR_TRAITS_H
+#define _LIBCPP___STRING_CHAR_TRAITS_H
+
+#include <__algorithm/fill_n.h>
+#include <__algorithm/find.h>
+#include <__algorithm/find_end.h>
+#include <__algorithm/find_first_of.h>
+#include <__algorithm/min.h>
+#include <__assert>
+#include <__compare/ordering.h>
+#include <__config>
+#include <__functional/hash.h>
+#include <__functional/identity.h>
+#include <__iterator/iterator_traits.h>
+#include <__string/constexpr_c_functions.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__utility/is_pointer_in_range.h>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <iosfwd>
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+#  include <cwchar> // for wmemcpy
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT>
+struct char_traits;
+/*
+The Standard does not define the base template for char_traits because it is impossible to provide
+a correct definition for arbitrary character types. Instead, it requires implementations to provide
+specializations for predefined character types like `char`, `wchar_t` and others. We provide this as
+exposition-only to document what members a char_traits specialization should provide:
+{
+    using char_type  = _CharT;
+    using int_type   = ...;
+    using off_type   = ...;
+    using pos_type   = ...;
+    using state_type = ...;
+
+    static void assign(char_type&, const char_type&);
+    static bool eq(char_type, char_type);
+    static bool lt(char_type, char_type);
+
+    static int              compare(const char_type*, const char_type*, size_t);
+    static size_t           length(const char_type*);
+    static const char_type* find(const char_type*, size_t, const char_type&);
+    static char_type*       move(char_type*, const char_type*, size_t);
+    static char_type*       copy(char_type*, const char_type*, size_t);
+    static char_type*       assign(char_type*, size_t, char_type);
+
+    static int_type  not_eof(int_type);
+    static char_type to_char_type(int_type);
+    static int_type  to_int_type(char_type);
+    static bool      eq_int_type(int_type, int_type);
+    static int_type  eof();
+};
+*/
+
+// char_traits<char>
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS char_traits<char> {
+  using char_type  = char;
+  using int_type   = int;
+  using off_type   = streamoff;
+  using pos_type   = streampos;
+  using state_type = mbstate_t;
+#if _LIBCPP_STD_VER >= 20
+  using comparison_category = strong_ordering;
+#endif
+
+  static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 void
+  assign(char_type& __c1, const char_type& __c2) _NOEXCEPT {
+    __c1 = __c2;
+  }
+
+  // TODO: Make this _LIBCPP_HIDE_FROM_ABI
+  static inline _LIBCPP_HIDDEN _LIBCPP_CONSTEXPR bool eq(char_type __c1, char_type __c2) _NOEXCEPT {
+    return __c1 == __c2;
+  }
+  static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool lt(char_type __c1, char_type __c2) _NOEXCEPT {
+    return (unsigned char)__c1 < (unsigned char)__c2;
+  }
+
+  // __constexpr_memcmp requires a trivially lexicographically comparable type, but char is not when char is a signed
+  // type
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 int
+  compare(const char_type* __lhs, const char_type* __rhs, size_t __count) _NOEXCEPT {
+    if (__libcpp_is_constant_evaluated()) {
+#ifdef _LIBCPP_COMPILER_CLANG_BASED
+      return __builtin_memcmp(__lhs, __rhs, __count);
+#else
+      while (__count != 0) {
+        if (lt(*__lhs, *__rhs))
+          return -1;
+        if (lt(*__rhs, *__lhs))
+          return 1;
+
+        __count -= sizeof(char_type);
+        ++__lhs;
+        ++__rhs;
+      }
+      return 0;
+#endif // _LIBCPP_COMPILER_CLANG_BASED
+    } else {
+      return __builtin_memcmp(__lhs, __rhs, __count);
+    }
+  }
+
+  static inline _LIBCPP_HIDE_FROM_ABI size_t _LIBCPP_CONSTEXPR_SINCE_CXX17 length(const char_type* __s) _NOEXCEPT {
+    return std::__constexpr_strlen(__s);
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const char_type*
+  find(const char_type* __s, size_t __n, const char_type& __a) _NOEXCEPT {
+    if (__n == 0)
+      return nullptr;
+    return std::__constexpr_memchr(__s, __a, __n);
+  }
+
+  static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 char_type*
+  move(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT {
+    return std::__constexpr_memmove(__s1, __s2, __element_count(__n));
+  }
+
+  static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 char_type*
+  copy(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(!std::__is_pointer_in_range(__s1, __s1 + __n, __s2),
+                                          "char_traits::copy: source and destination ranges overlap");
+    std::__constexpr_memmove(__s1, __s2, __element_count(__n));
+    return __s1;
+  }
+
+  static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 char_type*
+  assign(char_type* __s, size_t __n, char_type __a) _NOEXCEPT {
+    std::fill_n(__s, __n, __a);
+    return __s;
+  }
+
+  static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int_type not_eof(int_type __c) _NOEXCEPT {
+    return eq_int_type(__c, eof()) ? ~eof() : __c;
+  }
+  static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR char_type to_char_type(int_type __c) _NOEXCEPT {
+    return char_type(__c);
+  }
+  static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int_type to_int_type(char_type __c) _NOEXCEPT {
+    return int_type((unsigned char)__c);
+  }
+  static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool eq_int_type(int_type __c1, int_type __c2) _NOEXCEPT {
+    return __c1 == __c2;
+  }
+  static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int_type eof() _NOEXCEPT { return int_type(EOF); }
+};
+
+template <class _CharT, class _IntT, _IntT _EOFVal>
+struct __char_traits_base {
+  using char_type  = _CharT;
+  using int_type   = _IntT;
+  using off_type   = streamoff;
+  using state_type = mbstate_t;
+#if _LIBCPP_STD_VER >= 20
+  using comparison_category = strong_ordering;
+#endif
+
+  // There are different aliases for the different char types, but they are all aliases to this type
+  using pos_type = fpos<mbstate_t>;
+
+  _LIBCPP_HIDE_FROM_ABI static inline _LIBCPP_CONSTEXPR_SINCE_CXX17 void
+  assign(char_type& __lhs, const char_type& __rhs) _NOEXCEPT {
+    __lhs = __rhs;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR bool eq(char_type __lhs, char_type __rhs) _NOEXCEPT {
+    return __lhs == __rhs;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR bool lt(char_type __lhs, char_type __rhs) _NOEXCEPT {
+    return __lhs < __rhs;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX20 char_type*
+  move(char_type* __dest, const char_type* __src, size_t __n) _NOEXCEPT {
+    return std::__constexpr_memmove(__dest, __src, __element_count(__n));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX20 char_type*
+  copy(char_type* __dest, const char_type* __src, size_t __n) _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(!std::__is_pointer_in_range(__dest, __dest + __n, __src),
+                                          "char_traits::copy: source and destination ranges overlap");
+    return std::__constexpr_memmove(__dest, __src, __element_count(__n));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX20 char_type*
+  assign(char_type* __str, size_t __n, char_type __fill_char) _NOEXCEPT {
+    std::fill_n(__str, __n, __fill_char);
+    return __str;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR char_type to_char_type(int_type __c) _NOEXCEPT {
+    return char_type(__c);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR int_type to_int_type(char_type __c) _NOEXCEPT { return int_type(__c); }
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR bool eq_int_type(int_type __lhs, int_type __rhs) _NOEXCEPT {
+    return __lhs == __rhs;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR int_type eof() _NOEXCEPT { return _EOFVal; }
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR int_type not_eof(int_type __c) _NOEXCEPT {
+    return eq_int_type(__c, eof()) ? static_cast<int_type>(~eof()) : __c;
+  }
+};
+
+// char_traits<wchar_t>
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+struct _LIBCPP_TEMPLATE_VIS char_traits<wchar_t> : __char_traits_base<wchar_t, wint_t, static_cast<wint_t>(WEOF)> {
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 int
+  compare(const char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT {
+    if (__n == 0)
+      return 0;
+    return std::__constexpr_wmemcmp(__s1, __s2, __n);
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 size_t length(const char_type* __s) _NOEXCEPT {
+    return std::__constexpr_wcslen(__s);
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const char_type*
+  find(const char_type* __s, size_t __n, const char_type& __a) _NOEXCEPT {
+    if (__n == 0)
+      return nullptr;
+    return std::__constexpr_wmemchr(__s, __a, __n);
+  }
+};
+#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS char_traits<char8_t>
+    : __char_traits_base<char8_t, unsigned int, static_cast<unsigned int>(EOF)> {
+  static _LIBCPP_HIDE_FROM_ABI constexpr int
+  compare(const char_type* __s1, const char_type* __s2, size_t __n) noexcept {
+    return std::__constexpr_memcmp(__s1, __s2, __element_count(__n));
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI constexpr size_t length(const char_type* __str) noexcept {
+    return std::__constexpr_strlen(__str);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr const char_type*
+  find(const char_type* __s, size_t __n, const char_type& __a) noexcept {
+    return std::__constexpr_memchr(__s, __a, __n);
+  }
+};
+
+#endif // _LIBCPP_HAS_NO_CHAR8_T
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS char_traits<char16_t>
+    : __char_traits_base<char16_t, uint_least16_t, static_cast<uint_least16_t>(0xFFFF)> {
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 int
+  compare(const char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 size_t length(const char_type* __s) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 const char_type*
+  find(const char_type* __s, size_t __n, const char_type& __a) _NOEXCEPT {
+    __identity __proj;
+    const char_type* __match = std::__find(__s, __s + __n, __a, __proj);
+    if (__match == __s + __n)
+      return nullptr;
+    return __match;
+  }
+};
+
+inline _LIBCPP_CONSTEXPR_SINCE_CXX17 int
+char_traits<char16_t>::compare(const char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT {
+  for (; __n; --__n, ++__s1, ++__s2) {
+    if (lt(*__s1, *__s2))
+      return -1;
+    if (lt(*__s2, *__s1))
+      return 1;
+  }
+  return 0;
+}
+
+inline _LIBCPP_CONSTEXPR_SINCE_CXX17 size_t char_traits<char16_t>::length(const char_type* __s) _NOEXCEPT {
+  size_t __len = 0;
+  for (; !eq(*__s, char_type(0)); ++__s)
+    ++__len;
+  return __len;
+}
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS char_traits<char32_t>
+    : __char_traits_base<char32_t, uint_least32_t, static_cast<uint_least32_t>(0xFFFFFFFF)> {
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 int
+  compare(const char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 size_t length(const char_type* __s) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 const char_type*
+  find(const char_type* __s, size_t __n, const char_type& __a) _NOEXCEPT {
+    __identity __proj;
+    const char_type* __match = std::__find(__s, __s + __n, __a, __proj);
+    if (__match == __s + __n)
+      return nullptr;
+    return __match;
+  }
+};
+
+inline _LIBCPP_CONSTEXPR_SINCE_CXX17 int
+char_traits<char32_t>::compare(const char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT {
+  for (; __n; --__n, ++__s1, ++__s2) {
+    if (lt(*__s1, *__s2))
+      return -1;
+    if (lt(*__s2, *__s1))
+      return 1;
+  }
+  return 0;
+}
+
+inline _LIBCPP_CONSTEXPR_SINCE_CXX17 size_t char_traits<char32_t>::length(const char_type* __s) _NOEXCEPT {
+  size_t __len = 0;
+  for (; !eq(*__s, char_type(0)); ++__s)
+    ++__len;
+  return __len;
+}
+
+// helper fns for basic_string and string_view
+
+// __str_find
+template <class _CharT, class _SizeT, class _Traits, _SizeT __npos>
+inline _SizeT _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+__str_find(const _CharT* __p, _SizeT __sz, _CharT __c, _SizeT __pos) _NOEXCEPT {
+  if (__pos >= __sz)
+    return __npos;
+  const _CharT* __r = _Traits::find(__p + __pos, __sz - __pos, __c);
+  if (__r == nullptr)
+    return __npos;
+  return static_cast<_SizeT>(__r - __p);
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR_SINCE_CXX14 const _CharT* __search_substring(
+    const _CharT* __first1, const _CharT* __last1, const _CharT* __first2, const _CharT* __last2) _NOEXCEPT {
+  // Take advantage of knowing source and pattern lengths.
+  // Stop short when source is smaller than pattern.
+  const ptrdiff_t __len2 = __last2 - __first2;
+  if (__len2 == 0)
+    return __first1;
+
+  ptrdiff_t __len1 = __last1 - __first1;
+  if (__len1 < __len2)
+    return __last1;
+
+  // First element of __first2 is loop invariant.
+  _CharT __f2 = *__first2;
+  while (true) {
+    __len1 = __last1 - __first1;
+    // Check whether __first1 still has at least __len2 bytes.
+    if (__len1 < __len2)
+      return __last1;
+
+    // Find __f2 the first byte matching in __first1.
+    __first1 = _Traits::find(__first1, __len1 - __len2 + 1, __f2);
+    if (__first1 == nullptr)
+      return __last1;
+
+    // It is faster to compare from the first byte of __first1 even if we
+    // already know that it matches the first byte of __first2: this is because
+    // __first2 is most likely aligned, as it is user's "pattern" string, and
+    // __first1 + 1 is most likely not aligned, as the match is in the middle of
+    // the string.
+    if (_Traits::compare(__first1, __first2, __len2) == 0)
+      return __first1;
+
+    ++__first1;
+  }
+}
+
+template <class _CharT, class _SizeT, class _Traits, _SizeT __npos>
+inline _SizeT _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+__str_find(const _CharT* __p, _SizeT __sz, const _CharT* __s, _SizeT __pos, _SizeT __n) _NOEXCEPT {
+  if (__pos > __sz)
+    return __npos;
+
+  if (__n == 0) // There is nothing to search, just return __pos.
+    return __pos;
+
+  const _CharT* __r = std::__search_substring<_CharT, _Traits>(__p + __pos, __p + __sz, __s, __s + __n);
+
+  if (__r == __p + __sz)
+    return __npos;
+  return static_cast<_SizeT>(__r - __p);
+}
+
+// __str_rfind
+
+template <class _CharT, class _SizeT, class _Traits, _SizeT __npos>
+inline _SizeT _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+__str_rfind(const _CharT* __p, _SizeT __sz, _CharT __c, _SizeT __pos) _NOEXCEPT {
+  if (__sz < 1)
+    return __npos;
+  if (__pos < __sz)
+    ++__pos;
+  else
+    __pos = __sz;
+  for (const _CharT* __ps = __p + __pos; __ps != __p;) {
+    if (_Traits::eq(*--__ps, __c))
+      return static_cast<_SizeT>(__ps - __p);
+  }
+  return __npos;
+}
+
+template <class _CharT, class _SizeT, class _Traits, _SizeT __npos>
+inline _SizeT _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+__str_rfind(const _CharT* __p, _SizeT __sz, const _CharT* __s, _SizeT __pos, _SizeT __n) _NOEXCEPT {
+  __pos = std::min(__pos, __sz);
+  if (__n < __sz - __pos)
+    __pos += __n;
+  else
+    __pos = __sz;
+  const _CharT* __r = std::__find_end_classic(__p, __p + __pos, __s, __s + __n, _Traits::eq);
+  if (__n > 0 && __r == __p + __pos)
+    return __npos;
+  return static_cast<_SizeT>(__r - __p);
+}
+
+// __str_find_first_of
+template <class _CharT, class _SizeT, class _Traits, _SizeT __npos>
+inline _SizeT _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+__str_find_first_of(const _CharT* __p, _SizeT __sz, const _CharT* __s, _SizeT __pos, _SizeT __n) _NOEXCEPT {
+  if (__pos >= __sz || __n == 0)
+    return __npos;
+  const _CharT* __r = std::__find_first_of_ce(__p + __pos, __p + __sz, __s, __s + __n, _Traits::eq);
+  if (__r == __p + __sz)
+    return __npos;
+  return static_cast<_SizeT>(__r - __p);
+}
+
+// __str_find_last_of
+template <class _CharT, class _SizeT, class _Traits, _SizeT __npos>
+inline _SizeT _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+__str_find_last_of(const _CharT* __p, _SizeT __sz, const _CharT* __s, _SizeT __pos, _SizeT __n) _NOEXCEPT {
+  if (__n != 0) {
+    if (__pos < __sz)
+      ++__pos;
+    else
+      __pos = __sz;
+    for (const _CharT* __ps = __p + __pos; __ps != __p;) {
+      const _CharT* __r = _Traits::find(__s, __n, *--__ps);
+      if (__r)
+        return static_cast<_SizeT>(__ps - __p);
+    }
+  }
+  return __npos;
+}
+
+// __str_find_first_not_of
+template <class _CharT, class _SizeT, class _Traits, _SizeT __npos>
+inline _SizeT _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+__str_find_first_not_of(const _CharT* __p, _SizeT __sz, const _CharT* __s, _SizeT __pos, _SizeT __n) _NOEXCEPT {
+  if (__pos < __sz) {
+    const _CharT* __pe = __p + __sz;
+    for (const _CharT* __ps = __p + __pos; __ps != __pe; ++__ps)
+      if (_Traits::find(__s, __n, *__ps) == nullptr)
+        return static_cast<_SizeT>(__ps - __p);
+  }
+  return __npos;
+}
+
+template <class _CharT, class _SizeT, class _Traits, _SizeT __npos>
+inline _SizeT _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+__str_find_first_not_of(const _CharT* __p, _SizeT __sz, _CharT __c, _SizeT __pos) _NOEXCEPT {
+  if (__pos < __sz) {
+    const _CharT* __pe = __p + __sz;
+    for (const _CharT* __ps = __p + __pos; __ps != __pe; ++__ps)
+      if (!_Traits::eq(*__ps, __c))
+        return static_cast<_SizeT>(__ps - __p);
+  }
+  return __npos;
+}
+
+// __str_find_last_not_of
+template <class _CharT, class _SizeT, class _Traits, _SizeT __npos>
+inline _SizeT _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+__str_find_last_not_of(const _CharT* __p, _SizeT __sz, const _CharT* __s, _SizeT __pos, _SizeT __n) _NOEXCEPT {
+  if (__pos < __sz)
+    ++__pos;
+  else
+    __pos = __sz;
+  for (const _CharT* __ps = __p + __pos; __ps != __p;)
+    if (_Traits::find(__s, __n, *--__ps) == nullptr)
+      return static_cast<_SizeT>(__ps - __p);
+  return __npos;
+}
+
+template <class _CharT, class _SizeT, class _Traits, _SizeT __npos>
+inline _SizeT _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+__str_find_last_not_of(const _CharT* __p, _SizeT __sz, _CharT __c, _SizeT __pos) _NOEXCEPT {
+  if (__pos < __sz)
+    ++__pos;
+  else
+    __pos = __sz;
+  for (const _CharT* __ps = __p + __pos; __ps != __p;)
+    if (!_Traits::eq(*--__ps, __c))
+      return static_cast<_SizeT>(__ps - __p);
+  return __npos;
+}
+
+template <class _Ptr>
+inline _LIBCPP_HIDE_FROM_ABI size_t __do_string_hash(_Ptr __p, _Ptr __e) {
+  typedef typename iterator_traits<_Ptr>::value_type value_type;
+  return __murmur2_or_cityhash<size_t>()(__p, (__e - __p) * sizeof(value_type));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___STRING_CHAR_TRAITS_H
diff --git a/libcxx/include/__cxx03/__string/constexpr_c_functions.h b/libcxx/include/__cxx03/__string/constexpr_c_functions.h
new file mode 100644
index 000000000000..a978f816f189
--- /dev/null
+++ b/libcxx/include/__cxx03/__string/constexpr_c_functions.h
@@ -0,0 +1,234 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___STRING_CONSTEXPR_C_FUNCTIONS_H
+#define _LIBCPP___STRING_CONSTEXPR_C_FUNCTIONS_H
+
+#include <__config>
+#include <__memory/addressof.h>
+#include <__memory/construct_at.h>
+#include <__type_traits/datasizeof.h>
+#include <__type_traits/is_always_bitcastable.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_equality_comparable.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_trivially_copyable.h>
+#include <__type_traits/is_trivially_lexicographically_comparable.h>
+#include <__type_traits/remove_cv.h>
+#include <__utility/is_pointer_in_range.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Type used to encode that a function takes an integer that represents a number
+// of elements as opposed to a number of bytes.
+enum class __element_count : size_t {};
+
+template <class _Tp>
+inline const bool __is_char_type = false;
+
+template <>
+inline const bool __is_char_type<char> = true;
+
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+template <>
+inline const bool __is_char_type<char8_t> = true;
+#endif
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 size_t __constexpr_strlen(const _Tp* __str) _NOEXCEPT {
+  static_assert(__is_char_type<_Tp>, "__constexpr_strlen only works with char and char8_t");
+  // GCC currently doesn't support __builtin_strlen for heap-allocated memory during constant evaluation.
+  // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70816
+  if (__libcpp_is_constant_evaluated()) {
+#if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_COMPILER_CLANG_BASED)
+    if constexpr (is_same_v<_Tp, char>)
+      return __builtin_strlen(__str);
+#endif
+    size_t __i = 0;
+    for (; __str[__i] != '\0'; ++__i)
+      ;
+    return __i;
+  }
+  return __builtin_strlen(reinterpret_cast<const char*>(__str));
+}
+
+// Because of __libcpp_is_trivially_lexicographically_comparable we know that comparing the object representations is
+// equivalent to a std::memcmp. Since we have multiple objects contiguously in memory, we can call memcmp once instead
+// of invoking it on every object individually.
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int
+__constexpr_memcmp(const _Tp* __lhs, const _Up* __rhs, __element_count __n) {
+  static_assert(__libcpp_is_trivially_lexicographically_comparable<_Tp, _Up>::value,
+                "_Tp and _Up have to be trivially lexicographically comparable");
+
+  auto __count = static_cast<size_t>(__n);
+
+  if (__libcpp_is_constant_evaluated()) {
+#ifdef _LIBCPP_COMPILER_CLANG_BASED
+    if (sizeof(_Tp) == 1 && !is_same<_Tp, bool>::value)
+      return __builtin_memcmp(__lhs, __rhs, __count * sizeof(_Tp));
+#endif
+
+    while (__count != 0) {
+      if (*__lhs < *__rhs)
+        return -1;
+      if (*__rhs < *__lhs)
+        return 1;
+
+      --__count;
+      ++__lhs;
+      ++__rhs;
+    }
+    return 0;
+  } else {
+    return __builtin_memcmp(__lhs, __rhs, __count * sizeof(_Tp));
+  }
+}
+
+// Because of __libcpp_is_trivially_equality_comparable we know that comparing the object representations is equivalent
+// to a std::memcmp(...) == 0. Since we have multiple objects contiguously in memory, we can call memcmp once instead
+// of invoking it on every object individually.
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+__constexpr_memcmp_equal(const _Tp* __lhs, const _Up* __rhs, __element_count __n) {
+  static_assert(__libcpp_is_trivially_equality_comparable<_Tp, _Up>::value,
+                "_Tp and _Up have to be trivially equality comparable");
+
+  auto __count = static_cast<size_t>(__n);
+
+  if (__libcpp_is_constant_evaluated()) {
+#ifdef _LIBCPP_COMPILER_CLANG_BASED
+    if (sizeof(_Tp) == 1 && is_integral<_Tp>::value && !is_same<_Tp, bool>::value)
+      return __builtin_memcmp(__lhs, __rhs, __count * sizeof(_Tp)) == 0;
+#endif
+    while (__count != 0) {
+      if (*__lhs != *__rhs)
+        return false;
+
+      --__count;
+      ++__lhs;
+      ++__rhs;
+    }
+    return true;
+  } else {
+    return ::__builtin_memcmp(__lhs, __rhs, __count * sizeof(_Tp)) == 0;
+  }
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __constexpr_memchr(_Tp* __str, _Up __value, size_t __count) {
+  static_assert(sizeof(_Tp) == 1 && __libcpp_is_trivially_equality_comparable<_Tp, _Up>::value,
+                "Calling memchr on non-trivially equality comparable types is unsafe.");
+
+  if (__libcpp_is_constant_evaluated()) {
+// use __builtin_char_memchr to optimize constexpr evaluation if we can
+#if _LIBCPP_STD_VER >= 17 && __has_builtin(__builtin_char_memchr)
+    if constexpr (is_same_v<remove_cv_t<_Tp>, char> && is_same_v<remove_cv_t<_Up>, char>)
+      return __builtin_char_memchr(__str, __value, __count);
+#endif
+
+    for (; __count; --__count) {
+      if (*__str == __value)
+        return __str;
+      ++__str;
+    }
+    return nullptr;
+  } else {
+    char __value_buffer = 0;
+    __builtin_memcpy(&__value_buffer, &__value, sizeof(char));
+    return static_cast<_Tp*>(__builtin_memchr(__str, __value_buffer, __count));
+  }
+}
+
+// This function performs an assignment to an existing, already alive TriviallyCopyable object
+// from another TriviallyCopyable object.
+//
+// It basically works around the fact that TriviallyCopyable objects are not required to be
+// syntactically copy/move constructible or copy/move assignable. Technically, only one of the
+// four operations is required to be syntactically valid -- but at least one definitely has to
+// be valid.
+//
+// This is necessary in order to implement __constexpr_memmove below in a way that mirrors as
+// closely as possible what the compiler's __builtin_memmove is able to do.
+template <class _Tp, class _Up, __enable_if_t<is_assignable<_Tp&, _Up const&>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp& __assign_trivially_copyable(_Tp& __dest, _Up const& __src) {
+  __dest = __src;
+  return __dest;
+}
+
+// clang-format off
+template <class _Tp, class _Up, __enable_if_t<!is_assignable<_Tp&, _Up const&>::value &&
+                                               is_assignable<_Tp&, _Up&&>::value, int> = 0>
+// clang-format on
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp& __assign_trivially_copyable(_Tp& __dest, _Up& __src) {
+  __dest =
+      static_cast<_Up&&>(__src); // this is safe, we're not actually moving anything since the assignment is trivial
+  return __dest;
+}
+
+// clang-format off
+template <class _Tp, class _Up, __enable_if_t<!is_assignable<_Tp&, _Up const&>::value &&
+                                              !is_assignable<_Tp&, _Up&&>::value &&
+                                               is_constructible<_Tp, _Up const&>::value, int> = 0>
+// clang-format on
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& __assign_trivially_copyable(_Tp& __dest, _Up const& __src) {
+  // _Tp is trivially destructible, so we don't need to call its destructor to end the lifetime of the object
+  // that was there previously
+  std::__construct_at(std::addressof(__dest), __src);
+  return __dest;
+}
+
+// clang-format off
+template <class _Tp, class _Up, __enable_if_t<!is_assignable<_Tp&, _Up const&>::value &&
+                                              !is_assignable<_Tp&, _Up&&>::value &&
+                                              !is_constructible<_Tp, _Up const&>::value &&
+                                               is_constructible<_Tp, _Up&&>::value, int> = 0>
+// clang-format on
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& __assign_trivially_copyable(_Tp& __dest, _Up& __src) {
+  // _Tp is trivially destructible, so we don't need to call its destructor to end the lifetime of the object
+  // that was there previously
+  std::__construct_at(
+      std::addressof(__dest),
+      static_cast<_Up&&>(__src)); // this is safe, we're not actually moving anything since the constructor is trivial
+  return __dest;
+}
+
+template <class _Tp, class _Up, __enable_if_t<__is_always_bitcastable<_Up, _Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp*
+__constexpr_memmove(_Tp* __dest, _Up* __src, __element_count __n) {
+  size_t __count = static_cast<size_t>(__n);
+  if (__libcpp_is_constant_evaluated()) {
+#ifdef _LIBCPP_COMPILER_CLANG_BASED
+    if (is_same<__remove_cv_t<_Tp>, __remove_cv_t<_Up> >::value) {
+      ::__builtin_memmove(__dest, __src, __count * sizeof(_Tp));
+      return __dest;
+    }
+#endif
+    if (std::__is_pointer_in_range(__src, __src + __count, __dest)) {
+      for (; __count > 0; --__count)
+        std::__assign_trivially_copyable(__dest[__count - 1], __src[__count - 1]);
+    } else {
+      for (size_t __i = 0; __i != __count; ++__i)
+        std::__assign_trivially_copyable(__dest[__i], __src[__i]);
+    }
+  } else if (__count > 0) {
+    ::__builtin_memmove(__dest, __src, (__count - 1) * sizeof(_Tp) + __datasizeof_v<_Tp>);
+  }
+  return __dest;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___STRING_CONSTEXPR_C_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__string/extern_template_lists.h b/libcxx/include/__cxx03/__string/extern_template_lists.h
new file mode 100644
index 000000000000..cc536e514d4f
--- /dev/null
+++ b/libcxx/include/__cxx03/__string/extern_template_lists.h
@@ -0,0 +1,133 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___STRING_EXTERN_TEMPLATE_LISTS_H
+#define _LIBCPP___STRING_EXTERN_TEMPLATE_LISTS_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+// clang-format off
+
+// We maintain 2 ABI lists:
+// - _LIBCPP_STRING_V1_EXTERN_TEMPLATE_LIST
+// - _LIBCPP_STRING_UNSTABLE_EXTERN_TEMPLATE_LIST
+// As the name implies, the ABI lists define the V1 (Stable) and unstable ABI.
+//
+// For unstable, we may explicitly remove function that are external in V1,
+// and add (new) external functions to better control inlining and compiler
+// optimization opportunities.
+//
+// For stable, the ABI list should rarely change, except for adding new
+// functions supporting new c++ version / API changes. Typically entries
+// must never be removed from the stable list.
+#define _LIBCPP_STRING_V1_EXTERN_TEMPLATE_LIST(_Func, _CharType) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::replace(size_type, size_type, value_type const*, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::rfind(value_type const*, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::__init(value_type const*, size_type, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::basic_string(basic_string const&)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::replace(size_type, size_type, value_type const*)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::basic_string(basic_string const&, allocator<_CharType> const&)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::find_last_not_of(value_type const*, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::~basic_string()) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::find_first_not_of(value_type const*, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::insert(size_type, size_type, value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::operator=(value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::__init(value_type const*, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI const _CharType& basic_string<_CharType>::at(size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::insert(size_type, value_type const*, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::find_first_of(value_type const*, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::replace(size_type, size_type, size_type, value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::assign(value_type const*, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::reserve(size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::append(value_type const*, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::assign(basic_string const&, size_type, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::copy(value_type*, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::basic_string(basic_string const&, size_type, size_type, allocator<_CharType> const&)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::find(value_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::__init(size_type, value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::insert(size_type, value_type const*)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::find_last_of(value_type const*, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::__grow_by(size_type, size_type, size_type, size_type, size_type, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::__grow_by_and_replace(size_type, size_type, size_type, size_type, size_type, size_type, value_type const*)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::push_back(value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::append(size_type, value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::rfind(value_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI const basic_string<_CharType>::size_type basic_string<_CharType>::npos) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::assign(size_type, value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::erase(size_type, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::append(basic_string const&, size_type, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI int basic_string<_CharType>::compare(value_type const*) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI int basic_string<_CharType>::compare(size_type, size_type, value_type const*) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI _CharType& basic_string<_CharType>::at(size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::assign(value_type const*)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::find(value_type const*, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI int basic_string<_CharType>::compare(size_type, size_type, basic_string const&, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI int basic_string<_CharType>::compare(size_type, size_type, value_type const*, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::operator=(basic_string const&)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::append(value_type const*)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::replace(size_type, size_type, basic_string const&, size_type, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::iterator basic_string<_CharType>::insert(basic_string::const_iterator, value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::resize(size_type, value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::insert(size_type, basic_string const&, size_type, size_type))
+
+#define _LIBCPP_STRING_UNSTABLE_EXTERN_TEMPLATE_LIST(_Func, _CharType) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::replace(size_type, size_type, value_type const*, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::rfind(value_type const*, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::__init(value_type const*, size_type, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::replace(size_type, size_type, value_type const*)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::find_last_not_of(value_type const*, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::~basic_string()) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::find_first_not_of(value_type const*, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::insert(size_type, size_type, value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::operator=(value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::__init(value_type const*, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::__init_copy_ctor_external(value_type const*, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI const _CharType& basic_string<_CharType>::at(size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::insert(size_type, value_type const*, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::find_first_of(value_type const*, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::replace(size_type, size_type, size_type, value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::__assign_external(value_type const*, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::__assign_external(value_type const*)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::reserve(size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::append(value_type const*, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::assign(basic_string const&, size_type, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::copy(value_type*, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::basic_string(basic_string const&, size_type, size_type, allocator<_CharType> const&)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::find(value_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::__init(size_type, value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::insert(size_type, value_type const*)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::find_last_of(value_type const*, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::__grow_by_and_replace(size_type, size_type, size_type, size_type, size_type, size_type, value_type const*)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::__assign_no_alias<false>(value_type const*, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::__assign_no_alias<true>(value_type const*, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::push_back(value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::append(size_type, value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::rfind(value_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI const basic_string<_CharType>::size_type basic_string<_CharType>::npos) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::assign(size_type, value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::__erase_external_with_move(size_type, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::append(basic_string const&, size_type, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI int basic_string<_CharType>::compare(value_type const*) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI int basic_string<_CharType>::compare(size_type, size_type, value_type const*) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI _CharType& basic_string<_CharType>::at(size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::size_type basic_string<_CharType>::find(value_type const*, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI int basic_string<_CharType>::compare(size_type, size_type, basic_string const&, size_type, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI int basic_string<_CharType>::compare(size_type, size_type, value_type const*, size_type) const) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::append(value_type const*)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::replace(size_type, size_type, basic_string const&, size_type, size_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>::iterator basic_string<_CharType>::insert(basic_string::const_iterator, value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI void basic_string<_CharType>::resize(size_type, value_type)) \
+  _Func(_LIBCPP_EXPORTED_FROM_ABI basic_string<_CharType>& basic_string<_CharType>::insert(size_type, basic_string const&, size_type, size_type))
+
+// clang-format on
+
+#endif // _LIBCPP___STRING_EXTERN_TEMPLATE_LISTS_H
diff --git a/libcxx/include/__cxx03/__support/ibm/gettod_zos.h b/libcxx/include/__cxx03/__support/ibm/gettod_zos.h
new file mode 100644
index 000000000000..bd7e46773669
--- /dev/null
+++ b/libcxx/include/__cxx03/__support/ibm/gettod_zos.h
@@ -0,0 +1,52 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___SUPPORT_IBM_GETTOD_ZOS_H
+#define _LIBCPP___SUPPORT_IBM_GETTOD_ZOS_H
+
+#include <time.h>
+
+inline _LIBCPP_HIDE_FROM_ABI int gettimeofdayMonotonic(struct timespec64* Output) {
+  // The POSIX gettimeofday() function is not available on z/OS. Therefore,
+  // we will call stcke and other hardware instructions in implement equivalent.
+  // Note that nanoseconds alone will overflow when reaching new epoch in 2042.
+
+  struct _t {
+    uint64_t Hi;
+    uint64_t Lo;
+  };
+  struct _t Value = {0, 0};
+  uint64_t CC     = 0;
+  asm(" stcke %0\n"
+      " ipm %1\n"
+      " srlg %1,%1,28\n"
+      : "=m"(Value), "+r"(CC)::);
+
+  if (CC != 0) {
+    errno = EMVSTODNOTSET;
+    return CC;
+  }
+  uint64_t us = (Value.Hi >> 4);
+  uint64_t ns = ((Value.Hi & 0x0F) << 8) + (Value.Lo >> 56);
+  ns          = (ns * 1000) >> 12;
+  us          = us - 2208988800000000;
+
+  register uint64_t DivPair0 asm("r0"); // dividend (upper half), remainder
+  DivPair0 = 0;
+  register uint64_t DivPair1 asm("r1"); // dividend (lower half), quotient
+  DivPair1         = us;
+  uint64_t Divisor = 1000000;
+  asm(" dlgr %0,%2" : "+r"(DivPair0), "+r"(DivPair1) : "r"(Divisor) :);
+
+  Output->tv_sec  = DivPair1;
+  Output->tv_nsec = DivPair0 * 1000 + ns;
+  return 0;
+}
+
+#endif // _LIBCPP___SUPPORT_IBM_GETTOD_ZOS_H
diff --git a/libcxx/include/__cxx03/__support/ibm/locale_mgmt_zos.h b/libcxx/include/__cxx03/__support/ibm/locale_mgmt_zos.h
new file mode 100644
index 000000000000..5fc04b6b4b29
--- /dev/null
+++ b/libcxx/include/__cxx03/__support/ibm/locale_mgmt_zos.h
@@ -0,0 +1,53 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___SUPPORT_IBM_LOCALE_MGMT_ZOS_H
+#define _LIBCPP___SUPPORT_IBM_LOCALE_MGMT_ZOS_H
+
+#if defined(__MVS__)
+#  include <locale.h>
+#  include <string>
+
+#  ifdef __cplusplus
+extern "C" {
+#  endif
+
+#  define _LC_MAX LC_MESSAGES /* highest real category */
+#  define _NCAT (_LC_MAX + 1) /* maximum + 1 */
+
+#  define _CATMASK(n) (1 << (n))
+#  define LC_COLLATE_MASK _CATMASK(LC_COLLATE)
+#  define LC_CTYPE_MASK _CATMASK(LC_CTYPE)
+#  define LC_MONETARY_MASK _CATMASK(LC_MONETARY)
+#  define LC_NUMERIC_MASK _CATMASK(LC_NUMERIC)
+#  define LC_TIME_MASK _CATMASK(LC_TIME)
+#  define LC_MESSAGES_MASK _CATMASK(LC_MESSAGES)
+#  define LC_ALL_MASK (_CATMASK(_NCAT) - 1)
+
+typedef struct locale_struct {
+  int category_mask;
+  std::string lc_collate;
+  std::string lc_ctype;
+  std::string lc_monetary;
+  std::string lc_numeric;
+  std::string lc_time;
+  std::string lc_messages;
+}* locale_t;
+
+// z/OS does not have newlocale, freelocale and uselocale.
+// The functions below are workarounds in single thread mode.
+locale_t newlocale(int category_mask, const char* locale, locale_t base);
+void freelocale(locale_t locobj);
+locale_t uselocale(locale_t newloc);
+
+#  ifdef __cplusplus
+}
+#  endif
+#endif // defined(__MVS__)
+#endif // _LIBCPP___SUPPORT_IBM_LOCALE_MGMT_ZOS_H
diff --git a/libcxx/include/__cxx03/__support/ibm/nanosleep.h b/libcxx/include/__cxx03/__support/ibm/nanosleep.h
new file mode 100644
index 000000000000..fadc784c0297
--- /dev/null
+++ b/libcxx/include/__cxx03/__support/ibm/nanosleep.h
@@ -0,0 +1,55 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___SUPPORT_IBM_NANOSLEEP_H
+#define _LIBCPP___SUPPORT_IBM_NANOSLEEP_H
+
+#include <unistd.h>
+
+inline int nanosleep(const struct timespec* __req, struct timespec* __rem) {
+  // The nanosleep() function is not available on z/OS. Therefore, we will call
+  // sleep() to sleep for whole seconds and usleep() to sleep for any remaining
+  // fraction of a second. Any remaining nanoseconds will round up to the next
+  // microsecond.
+  if (__req->tv_sec < 0 || __req->tv_nsec < 0 || __req->tv_nsec > 999999999) {
+    errno = EINVAL;
+    return -1;
+  }
+  long __micro_sec = (__req->tv_nsec + 999) / 1000;
+  time_t __sec     = __req->tv_sec;
+  if (__micro_sec > 999999) {
+    ++__sec;
+    __micro_sec -= 1000000;
+  }
+  __sec = static_cast<time_t>(sleep(static_cast<unsigned int>(__sec)));
+  if (__sec) {
+    if (__rem) {
+      // Updating the remaining time to sleep in case of unsuccessful call to sleep().
+      __rem->tv_sec  = __sec;
+      __rem->tv_nsec = __micro_sec * 1000;
+    }
+    errno = EINTR;
+    return -1;
+  }
+  if (__micro_sec) {
+    int __rt = usleep(static_cast<unsigned int>(__micro_sec));
+    if (__rt != 0 && __rem) {
+      // The usleep() does not provide the amount of remaining time upon its failure,
+      // so the time slept will be ignored.
+      __rem->tv_sec  = 0;
+      __rem->tv_nsec = __micro_sec * 1000;
+      // The errno is already set.
+      return -1;
+    }
+    return __rt;
+  }
+  return 0;
+}
+
+#endif // _LIBCPP___SUPPORT_IBM_NANOSLEEP_H
diff --git a/libcxx/include/__cxx03/__support/xlocale/__nop_locale_mgmt.h b/libcxx/include/__cxx03/__support/xlocale/__nop_locale_mgmt.h
new file mode 100644
index 000000000000..b9ffcbe1622d
--- /dev/null
+++ b/libcxx/include/__cxx03/__support/xlocale/__nop_locale_mgmt.h
@@ -0,0 +1,35 @@
+// -*- C++ -*-
+//===-----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___SUPPORT_XLOCALE_NOP_LOCALE_MGMT_H
+#define _LIBCPP___SUPPORT_XLOCALE_NOP_LOCALE_MGMT_H
+
+#include <__config>
+
+// Patch over lack of extended locale support
+typedef void* locale_t;
+
+inline _LIBCPP_HIDE_FROM_ABI locale_t duplocale(locale_t) { return NULL; }
+
+inline _LIBCPP_HIDE_FROM_ABI void freelocale(locale_t) {}
+
+inline _LIBCPP_HIDE_FROM_ABI locale_t newlocale(int, const char*, locale_t) { return NULL; }
+
+inline _LIBCPP_HIDE_FROM_ABI locale_t uselocale(locale_t) { return NULL; }
+
+#define LC_COLLATE_MASK (1 << LC_COLLATE)
+#define LC_CTYPE_MASK (1 << LC_CTYPE)
+#define LC_MESSAGES_MASK (1 << LC_MESSAGES)
+#define LC_MONETARY_MASK (1 << LC_MONETARY)
+#define LC_NUMERIC_MASK (1 << LC_NUMERIC)
+#define LC_TIME_MASK (1 << LC_TIME)
+#define LC_ALL_MASK                                                                                                    \
+  (LC_COLLATE_MASK | LC_CTYPE_MASK | LC_MONETARY_MASK | LC_NUMERIC_MASK | LC_TIME_MASK | LC_MESSAGES_MASK)
+
+#endif // _LIBCPP___SUPPORT_XLOCALE_NOP_LOCALE_MGMT_H
diff --git a/libcxx/include/__cxx03/__support/xlocale/__posix_l_fallback.h b/libcxx/include/__cxx03/__support/xlocale/__posix_l_fallback.h
new file mode 100644
index 000000000000..8a3a6f27f48d
--- /dev/null
+++ b/libcxx/include/__cxx03/__support/xlocale/__posix_l_fallback.h
@@ -0,0 +1,107 @@
+// -*- C++ -*-
+//===-----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// These are reimplementations of some extended locale functions ( *_l ) that
+// are normally part of POSIX.  This shared implementation provides parts of the
+// extended locale support for libc's that normally don't have any (like
+// Android's bionic and Newlib).
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___SUPPORT_XLOCALE_POSIX_L_FALLBACK_H
+#define _LIBCPP___SUPPORT_XLOCALE_POSIX_L_FALLBACK_H
+
+#include <__config>
+#include <ctype.h>
+#include <string.h>
+#include <time.h>
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+#  include <wchar.h>
+#  include <wctype.h>
+#endif
+
+inline _LIBCPP_HIDE_FROM_ABI int isalnum_l(int __c, locale_t) { return ::isalnum(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int isalpha_l(int __c, locale_t) { return ::isalpha(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int iscntrl_l(int __c, locale_t) { return ::iscntrl(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int isdigit_l(int __c, locale_t) { return ::isdigit(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int isgraph_l(int __c, locale_t) { return ::isgraph(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int islower_l(int __c, locale_t) { return ::islower(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int isprint_l(int __c, locale_t) { return ::isprint(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int ispunct_l(int __c, locale_t) { return ::ispunct(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int isspace_l(int __c, locale_t) { return ::isspace(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int isupper_l(int __c, locale_t) { return ::isupper(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int isxdigit_l(int __c, locale_t) { return ::isxdigit(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int toupper_l(int __c, locale_t) { return ::toupper(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int tolower_l(int __c, locale_t) { return ::tolower(__c); }
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+inline _LIBCPP_HIDE_FROM_ABI int iswalnum_l(wint_t __c, locale_t) { return ::iswalnum(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int iswalpha_l(wint_t __c, locale_t) { return ::iswalpha(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int iswblank_l(wint_t __c, locale_t) { return ::iswblank(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int iswcntrl_l(wint_t __c, locale_t) { return ::iswcntrl(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int iswdigit_l(wint_t __c, locale_t) { return ::iswdigit(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int iswgraph_l(wint_t __c, locale_t) { return ::iswgraph(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int iswlower_l(wint_t __c, locale_t) { return ::iswlower(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int iswprint_l(wint_t __c, locale_t) { return ::iswprint(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int iswpunct_l(wint_t __c, locale_t) { return ::iswpunct(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int iswspace_l(wint_t __c, locale_t) { return ::iswspace(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int iswupper_l(wint_t __c, locale_t) { return ::iswupper(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI int iswxdigit_l(wint_t __c, locale_t) { return ::iswxdigit(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI wint_t towupper_l(wint_t __c, locale_t) { return ::towupper(__c); }
+
+inline _LIBCPP_HIDE_FROM_ABI wint_t towlower_l(wint_t __c, locale_t) { return ::towlower(__c); }
+#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+inline _LIBCPP_HIDE_FROM_ABI int strcoll_l(const char* __s1, const char* __s2, locale_t) {
+  return ::strcoll(__s1, __s2);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI size_t strxfrm_l(char* __dest, const char* __src, size_t __n, locale_t) {
+  return ::strxfrm(__dest, __src, __n);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI size_t
+strftime_l(char* __s, size_t __max, const char* __format, const struct tm* __tm, locale_t) {
+  return ::strftime(__s, __max, __format, __tm);
+}
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+inline _LIBCPP_HIDE_FROM_ABI int wcscoll_l(const wchar_t* __ws1, const wchar_t* __ws2, locale_t) {
+  return ::wcscoll(__ws1, __ws2);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI size_t wcsxfrm_l(wchar_t* __dest, const wchar_t* __src, size_t __n, locale_t) {
+  return ::wcsxfrm(__dest, __src, __n);
+}
+#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+#endif // _LIBCPP___SUPPORT_XLOCALE_POSIX_L_FALLBACK_H
diff --git a/libcxx/include/__cxx03/__support/xlocale/__strtonum_fallback.h b/libcxx/include/__cxx03/__support/xlocale/__strtonum_fallback.h
new file mode 100644
index 000000000000..b7eef5210ed3
--- /dev/null
+++ b/libcxx/include/__cxx03/__support/xlocale/__strtonum_fallback.h
@@ -0,0 +1,45 @@
+// -*- C++ -*-
+//===-----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// These are reimplementations of some extended locale functions ( *_l ) that
+// aren't part of POSIX.  They are widely available though (GLIBC, BSD, maybe
+// others).  The unifying aspect in this case is that all of these functions
+// convert strings to some numeric type.
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___SUPPORT_XLOCALE_STRTONUM_FALLBACK_H
+#define _LIBCPP___SUPPORT_XLOCALE_STRTONUM_FALLBACK_H
+
+#include <__config>
+#include <stdlib.h>
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+#  include <wchar.h>
+#endif
+
+inline _LIBCPP_HIDE_FROM_ABI float strtof_l(const char* __nptr, char** __endptr, locale_t) {
+  return ::strtof(__nptr, __endptr);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI double strtod_l(const char* __nptr, char** __endptr, locale_t) {
+  return ::strtod(__nptr, __endptr);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long double strtold_l(const char* __nptr, char** __endptr, locale_t) {
+  return ::strtold(__nptr, __endptr);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI long long strtoll_l(const char* __nptr, char** __endptr, int __base, locale_t) {
+  return ::strtoll(__nptr, __endptr, __base);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI unsigned long long strtoull_l(const char* __nptr, char** __endptr, int __base, locale_t) {
+  return ::strtoull(__nptr, __endptr, __base);
+}
+
+#endif // _LIBCPP___SUPPORT_XLOCALE_STRTONUM_FALLBACK_H
diff --git a/libcxx/include/__cxx03/__system_error/errc.h b/libcxx/include/__cxx03/__system_error/errc.h
new file mode 100644
index 000000000000..0004c46e2279
--- /dev/null
+++ b/libcxx/include/__cxx03/__system_error/errc.h
@@ -0,0 +1,266 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ERRC
+#define _LIBCPP___ERRC
+
+/*
+    system_error synopsis
+
+namespace std
+{
+
+enum class errc
+{
+    address_family_not_supported,       // EAFNOSUPPORT
+    address_in_use,                     // EADDRINUSE
+    address_not_available,              // EADDRNOTAVAIL
+    already_connected,                  // EISCONN
+    argument_list_too_long,             // E2BIG
+    argument_out_of_domain,             // EDOM
+    bad_address,                        // EFAULT
+    bad_file_descriptor,                // EBADF
+    bad_message,                        // EBADMSG
+    broken_pipe,                        // EPIPE
+    connection_aborted,                 // ECONNABORTED
+    connection_already_in_progress,     // EALREADY
+    connection_refused,                 // ECONNREFUSED
+    connection_reset,                   // ECONNRESET
+    cross_device_link,                  // EXDEV
+    destination_address_required,       // EDESTADDRREQ
+    device_or_resource_busy,            // EBUSY
+    directory_not_empty,                // ENOTEMPTY
+    executable_format_error,            // ENOEXEC
+    file_exists,                        // EEXIST
+    file_too_large,                     // EFBIG
+    filename_too_long,                  // ENAMETOOLONG
+    function_not_supported,             // ENOSYS
+    host_unreachable,                   // EHOSTUNREACH
+    identifier_removed,                 // EIDRM
+    illegal_byte_sequence,              // EILSEQ
+    inappropriate_io_control_operation, // ENOTTY
+    interrupted,                        // EINTR
+    invalid_argument,                   // EINVAL
+    invalid_seek,                       // ESPIPE
+    io_error,                           // EIO
+    is_a_directory,                     // EISDIR
+    message_size,                       // EMSGSIZE
+    network_down,                       // ENETDOWN
+    network_reset,                      // ENETRESET
+    network_unreachable,                // ENETUNREACH
+    no_buffer_space,                    // ENOBUFS
+    no_child_process,                   // ECHILD
+    no_link,                            // ENOLINK
+    no_lock_available,                  // ENOLCK
+    no_message_available,               // ENODATA         // deprecated
+    no_message,                         // ENOMSG
+    no_protocol_option,                 // ENOPROTOOPT
+    no_space_on_device,                 // ENOSPC
+    no_stream_resources,                // ENOSR           // deprecated
+    no_such_device_or_address,          // ENXIO
+    no_such_device,                     // ENODEV
+    no_such_file_or_directory,          // ENOENT
+    no_such_process,                    // ESRCH
+    not_a_directory,                    // ENOTDIR
+    not_a_socket,                       // ENOTSOCK
+    not_a_stream,                       // ENOSTR          // deprecated
+    not_connected,                      // ENOTCONN
+    not_enough_memory,                  // ENOMEM
+    not_supported,                      // ENOTSUP
+    operation_canceled,                 // ECANCELED
+    operation_in_progress,              // EINPROGRESS
+    operation_not_permitted,            // EPERM
+    operation_not_supported,            // EOPNOTSUPP
+    operation_would_block,              // EWOULDBLOCK
+    owner_dead,                         // EOWNERDEAD
+    permission_denied,                  // EACCES
+    protocol_error,                     // EPROTO
+    protocol_not_supported,             // EPROTONOSUPPORT
+    read_only_file_system,              // EROFS
+    resource_deadlock_would_occur,      // EDEADLK
+    resource_unavailable_try_again,     // EAGAIN
+    result_out_of_range,                // ERANGE
+    state_not_recoverable,              // ENOTRECOVERABLE
+    stream_timeout,                     // ETIME           // deprecated
+    text_file_busy,                     // ETXTBSY
+    timed_out,                          // ETIMEDOUT
+    too_many_files_open_in_system,      // ENFILE
+    too_many_files_open,                // EMFILE
+    too_many_links,                     // EMLINK
+    too_many_symbolic_link_levels,      // ELOOP
+    value_too_large,                    // EOVERFLOW
+    wrong_protocol_type                 // EPROTOTYPE
+};
+
+*/
+
+#include <__config>
+#include <cerrno>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+// The method of pushing and popping the diagnostics fails for GCC.  GCC does
+// not recognize the pragma's used to generate deprecated diagnostics for
+// macros. So GCC does not need the pushing and popping.
+//
+// TODO Remove this when the deprecated constants are removed.
+//
+// Note based on the post-review comments in
+// https://github.com/llvm/llvm-project/pull/80542 libc++ no longer deprecates
+// the macros. Since C libraries may start to deprecate these POSIX macros the
+// deprecation warning avoidance is kept.
+#if defined(_LIBCPP_COMPILER_CLANG_BASED)
+#  define _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+#  define _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP _LIBCPP_SUPPRESS_DEPRECATED_POP
+#else
+#  define _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH
+#  define _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Some error codes are not present on all platforms, so we provide equivalents
+// for them:
+
+// enum class errc
+//
+// LWG3869 deprecates the UNIX STREAMS macros and enum values.
+// This makes the code clumbersome:
+// - the enum value is deprecated and should show a diagnostic,
+// - the macro is deprecated and should _not_ show a diagnostic in this
+//   context, and
+// - the macro is not always available.
+// This leads to the odd pushing and popping of the deprecated
+// diagnostic.
+_LIBCPP_DECLARE_STRONG_ENUM(errc){
+    address_family_not_supported       = EAFNOSUPPORT,
+    address_in_use                     = EADDRINUSE,
+    address_not_available              = EADDRNOTAVAIL,
+    already_connected                  = EISCONN,
+    argument_list_too_long             = E2BIG,
+    argument_out_of_domain             = EDOM,
+    bad_address                        = EFAULT,
+    bad_file_descriptor                = EBADF,
+    bad_message                        = EBADMSG,
+    broken_pipe                        = EPIPE,
+    connection_aborted                 = ECONNABORTED,
+    connection_already_in_progress     = EALREADY,
+    connection_refused                 = ECONNREFUSED,
+    connection_reset                   = ECONNRESET,
+    cross_device_link                  = EXDEV,
+    destination_address_required       = EDESTADDRREQ,
+    device_or_resource_busy            = EBUSY,
+    directory_not_empty                = ENOTEMPTY,
+    executable_format_error            = ENOEXEC,
+    file_exists                        = EEXIST,
+    file_too_large                     = EFBIG,
+    filename_too_long                  = ENAMETOOLONG,
+    function_not_supported             = ENOSYS,
+    host_unreachable                   = EHOSTUNREACH,
+    identifier_removed                 = EIDRM,
+    illegal_byte_sequence              = EILSEQ,
+    inappropriate_io_control_operation = ENOTTY,
+    interrupted                        = EINTR,
+    invalid_argument                   = EINVAL,
+    invalid_seek                       = ESPIPE,
+    io_error                           = EIO,
+    is_a_directory                     = EISDIR,
+    message_size                       = EMSGSIZE,
+    network_down                       = ENETDOWN,
+    network_reset                      = ENETRESET,
+    network_unreachable                = ENETUNREACH,
+    no_buffer_space                    = ENOBUFS,
+    no_child_process                   = ECHILD,
+    no_link                            = ENOLINK,
+    no_lock_available                  = ENOLCK,
+    // clang-format off
+    no_message_available _LIBCPP_DEPRECATED =
+    _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH
+#ifdef ENODATA
+                                              ENODATA
+#else
+                                              ENOMSG
+#endif
+    _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP
+    ,
+    // clang-format on
+    no_message         = ENOMSG,
+    no_protocol_option = ENOPROTOOPT,
+    no_space_on_device = ENOSPC,
+    // clang-format off
+    no_stream_resources _LIBCPP_DEPRECATED =
+    _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH
+#ifdef ENOSR
+                                              ENOSR
+#else
+                                              ENOMEM
+#endif
+    _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP
+    ,
+    // clang-format on
+    no_such_device_or_address = ENXIO,
+    no_such_device            = ENODEV,
+    no_such_file_or_directory = ENOENT,
+    no_such_process           = ESRCH,
+    not_a_directory           = ENOTDIR,
+    not_a_socket              = ENOTSOCK,
+    // clang-format off
+    not_a_stream _LIBCPP_DEPRECATED =
+    _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH
+#ifdef ENOSTR
+                                      ENOSTR
+#else
+                                      EINVAL
+#endif
+    _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP
+    ,
+    // clang-format on
+    not_connected                  = ENOTCONN,
+    not_enough_memory              = ENOMEM,
+    not_supported                  = ENOTSUP,
+    operation_canceled             = ECANCELED,
+    operation_in_progress          = EINPROGRESS,
+    operation_not_permitted        = EPERM,
+    operation_not_supported        = EOPNOTSUPP,
+    operation_would_block          = EWOULDBLOCK,
+    owner_dead                     = EOWNERDEAD,
+    permission_denied              = EACCES,
+    protocol_error                 = EPROTO,
+    protocol_not_supported         = EPROTONOSUPPORT,
+    read_only_file_system          = EROFS,
+    resource_deadlock_would_occur  = EDEADLK,
+    resource_unavailable_try_again = EAGAIN,
+    result_out_of_range            = ERANGE,
+    state_not_recoverable          = ENOTRECOVERABLE,
+    // clang-format off
+    stream_timeout _LIBCPP_DEPRECATED =
+    _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH
+#ifdef ETIME
+                                        ETIME
+#else
+                                        ETIMEDOUT
+#endif
+    _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP
+    ,
+    // clang-format on
+    text_file_busy                = ETXTBSY,
+    timed_out                     = ETIMEDOUT,
+    too_many_files_open_in_system = ENFILE,
+    too_many_files_open           = EMFILE,
+    too_many_links                = EMLINK,
+    too_many_symbolic_link_levels = ELOOP,
+    value_too_large               = EOVERFLOW,
+    wrong_protocol_type           = EPROTOTYPE};
+_LIBCPP_DECLARE_STRONG_ENUM_EPILOG(errc)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ERRC
diff --git a/libcxx/include/__cxx03/__system_error/error_category.h b/libcxx/include/__cxx03/__system_error/error_category.h
new file mode 100644
index 000000000000..bfe7bc24a5d3
--- /dev/null
+++ b/libcxx/include/__cxx03/__system_error/error_category.h
@@ -0,0 +1,75 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___SYSTEM_ERROR_ERROR_CATEGORY_H
+#define _LIBCPP___SYSTEM_ERROR_ERROR_CATEGORY_H
+
+#include <__compare/ordering.h>
+#include <__config>
+#include <string>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+class _LIBCPP_EXPORTED_FROM_ABI error_condition;
+class _LIBCPP_EXPORTED_FROM_ABI error_code;
+
+class _LIBCPP_HIDDEN __do_message;
+
+class _LIBCPP_EXPORTED_FROM_ABI error_category {
+public:
+  virtual ~error_category() _NOEXCEPT;
+
+#if defined(_LIBCPP_ERROR_CATEGORY_DEFINE_LEGACY_INLINE_FUNCTIONS)
+  error_category() noexcept;
+#else
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 error_category() _NOEXCEPT = default;
+#endif
+  error_category(const error_category&)            = delete;
+  error_category& operator=(const error_category&) = delete;
+
+  virtual const char* name() const _NOEXCEPT = 0;
+  virtual error_condition default_error_condition(int __ev) const _NOEXCEPT;
+  virtual bool equivalent(int __code, const error_condition& __condition) const _NOEXCEPT;
+  virtual bool equivalent(const error_code& __code, int __condition) const _NOEXCEPT;
+  virtual string message(int __ev) const = 0;
+
+  _LIBCPP_HIDE_FROM_ABI bool operator==(const error_category& __rhs) const _NOEXCEPT { return this == &__rhs; }
+
+#if _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI strong_ordering operator<=>(const error_category& __rhs) const noexcept {
+    return compare_three_way()(this, std::addressof(__rhs));
+  }
+
+#else // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI bool operator!=(const error_category& __rhs) const _NOEXCEPT { return !(*this == __rhs); }
+
+  _LIBCPP_HIDE_FROM_ABI bool operator<(const error_category& __rhs) const _NOEXCEPT { return this < &__rhs; }
+
+#endif // _LIBCPP_STD_VER >= 20
+
+  friend class _LIBCPP_HIDDEN __do_message;
+};
+
+class _LIBCPP_HIDDEN __do_message : public error_category {
+public:
+  string message(int __ev) const override;
+};
+
+__attribute__((__const__)) _LIBCPP_EXPORTED_FROM_ABI const error_category& generic_category() _NOEXCEPT;
+__attribute__((__const__)) _LIBCPP_EXPORTED_FROM_ABI const error_category& system_category() _NOEXCEPT;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___SYSTEM_ERROR_ERROR_CATEGORY_H
diff --git a/libcxx/include/__cxx03/__system_error/error_code.h b/libcxx/include/__cxx03/__system_error/error_code.h
new file mode 100644
index 000000000000..475f2bb96a56
--- /dev/null
+++ b/libcxx/include/__cxx03/__system_error/error_code.h
@@ -0,0 +1,143 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___SYSTEM_ERROR_ERROR_CODE_H
+#define _LIBCPP___SYSTEM_ERROR_ERROR_CODE_H
+
+#include <__compare/ordering.h>
+#include <__config>
+#include <__functional/hash.h>
+#include <__functional/unary_function.h>
+#include <__system_error/errc.h>
+#include <__system_error/error_category.h>
+#include <__system_error/error_condition.h>
+#include <cstddef>
+#include <string>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_error_code_enum : public false_type {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_error_code_enum_v = is_error_code_enum<_Tp>::value;
+#endif
+
+namespace __adl_only {
+// Those cause ADL to trigger but they are not viable candidates,
+// so they are never actually selected.
+void make_error_code() = delete;
+} // namespace __adl_only
+
+class _LIBCPP_EXPORTED_FROM_ABI error_code {
+  int __val_;
+  const error_category* __cat_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI error_code() _NOEXCEPT : __val_(0), __cat_(&system_category()) {}
+
+  _LIBCPP_HIDE_FROM_ABI error_code(int __val, const error_category& __cat) _NOEXCEPT : __val_(__val), __cat_(&__cat) {}
+
+  template <class _Ep, __enable_if_t<is_error_code_enum<_Ep>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI error_code(_Ep __e) _NOEXCEPT {
+    using __adl_only::make_error_code;
+    *this = make_error_code(__e);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void assign(int __val, const error_category& __cat) _NOEXCEPT {
+    __val_ = __val;
+    __cat_ = &__cat;
+  }
+
+  template <class _Ep, __enable_if_t<is_error_code_enum<_Ep>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI error_code& operator=(_Ep __e) _NOEXCEPT {
+    using __adl_only::make_error_code;
+    *this = make_error_code(__e);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT {
+    __val_ = 0;
+    __cat_ = &system_category();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int value() const _NOEXCEPT { return __val_; }
+
+  _LIBCPP_HIDE_FROM_ABI const error_category& category() const _NOEXCEPT { return *__cat_; }
+
+  _LIBCPP_HIDE_FROM_ABI error_condition default_error_condition() const _NOEXCEPT {
+    return __cat_->default_error_condition(__val_);
+  }
+
+  string message() const;
+
+  _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return __val_ != 0; }
+};
+
+inline _LIBCPP_HIDE_FROM_ABI error_code make_error_code(errc __e) _NOEXCEPT {
+  return error_code(static_cast<int>(__e), generic_category());
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(const error_code& __x, const error_code& __y) _NOEXCEPT {
+  return __x.category() == __y.category() && __x.value() == __y.value();
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(const error_code& __x, const error_condition& __y) _NOEXCEPT {
+  return __x.category().equivalent(__x.value(), __y) || __y.category().equivalent(__x, __y.value());
+}
+
+#if _LIBCPP_STD_VER <= 17
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(const error_condition& __x, const error_code& __y) _NOEXCEPT {
+  return __y == __x;
+}
+#endif
+
+#if _LIBCPP_STD_VER <= 17
+
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const error_code& __x, const error_code& __y) _NOEXCEPT {
+  return !(__x == __y);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const error_code& __x, const error_condition& __y) _NOEXCEPT {
+  return !(__x == __y);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const error_condition& __x, const error_code& __y) _NOEXCEPT {
+  return !(__x == __y);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(const error_code& __x, const error_code& __y) _NOEXCEPT {
+  return __x.category() < __y.category() || (__x.category() == __y.category() && __x.value() < __y.value());
+}
+
+#else // _LIBCPP_STD_VER <= 17
+
+inline _LIBCPP_HIDE_FROM_ABI strong_ordering operator<=>(const error_code& __x, const error_code& __y) noexcept {
+  if (auto __c = __x.category() <=> __y.category(); __c != 0)
+    return __c;
+  return __x.value() <=> __y.value();
+}
+
+#endif // _LIBCPP_STD_VER <= 17
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<error_code> : public __unary_function<error_code, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const error_code& __ec) const _NOEXCEPT {
+    return static_cast<size_t>(__ec.value());
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___SYSTEM_ERROR_ERROR_CODE_H
diff --git a/libcxx/include/__cxx03/__system_error/error_condition.h b/libcxx/include/__cxx03/__system_error/error_condition.h
new file mode 100644
index 000000000000..42898c1f0e90
--- /dev/null
+++ b/libcxx/include/__cxx03/__system_error/error_condition.h
@@ -0,0 +1,130 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___SYSTEM_ERROR_ERROR_CONDITION_H
+#define _LIBCPP___SYSTEM_ERROR_ERROR_CONDITION_H
+
+#include <__compare/ordering.h>
+#include <__config>
+#include <__functional/hash.h>
+#include <__functional/unary_function.h>
+#include <__system_error/errc.h>
+#include <__system_error/error_category.h>
+#include <cstddef>
+#include <string>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_error_condition_enum : public false_type {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_error_condition_enum_v = is_error_condition_enum<_Tp>::value;
+#endif
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS is_error_condition_enum<errc> : true_type {};
+
+#ifdef _LIBCPP_CXX03_LANG
+template <>
+struct _LIBCPP_TEMPLATE_VIS is_error_condition_enum<errc::__lx> : true_type {};
+#endif
+
+namespace __adl_only {
+// Those cause ADL to trigger but they are not viable candidates,
+// so they are never actually selected.
+void make_error_condition() = delete;
+} // namespace __adl_only
+
+class _LIBCPP_EXPORTED_FROM_ABI error_condition {
+  int __val_;
+  const error_category* __cat_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI error_condition() _NOEXCEPT : __val_(0), __cat_(&generic_category()) {}
+
+  _LIBCPP_HIDE_FROM_ABI error_condition(int __val, const error_category& __cat) _NOEXCEPT
+      : __val_(__val),
+        __cat_(&__cat) {}
+
+  template <class _Ep, __enable_if_t<is_error_condition_enum<_Ep>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI error_condition(_Ep __e) _NOEXCEPT {
+    using __adl_only::make_error_condition;
+    *this = make_error_condition(__e);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void assign(int __val, const error_category& __cat) _NOEXCEPT {
+    __val_ = __val;
+    __cat_ = &__cat;
+  }
+
+  template <class _Ep, __enable_if_t<is_error_condition_enum<_Ep>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI error_condition& operator=(_Ep __e) _NOEXCEPT {
+    using __adl_only::make_error_condition;
+    *this = make_error_condition(__e);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT {
+    __val_ = 0;
+    __cat_ = &generic_category();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI int value() const _NOEXCEPT { return __val_; }
+
+  _LIBCPP_HIDE_FROM_ABI const error_category& category() const _NOEXCEPT { return *__cat_; }
+  string message() const;
+
+  _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return __val_ != 0; }
+};
+
+inline _LIBCPP_HIDE_FROM_ABI error_condition make_error_condition(errc __e) _NOEXCEPT {
+  return error_condition(static_cast<int>(__e), generic_category());
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(const error_condition& __x, const error_condition& __y) _NOEXCEPT {
+  return __x.category() == __y.category() && __x.value() == __y.value();
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const error_condition& __x, const error_condition& __y) _NOEXCEPT {
+  return !(__x == __y);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(const error_condition& __x, const error_condition& __y) _NOEXCEPT {
+  return __x.category() < __y.category() || (__x.category() == __y.category() && __x.value() < __y.value());
+}
+
+#else // _LIBCPP_STD_VER <= 17
+
+inline _LIBCPP_HIDE_FROM_ABI strong_ordering
+operator<=>(const error_condition& __x, const error_condition& __y) noexcept {
+  if (auto __c = __x.category() <=> __y.category(); __c != 0)
+    return __c;
+  return __x.value() <=> __y.value();
+}
+
+#endif // _LIBCPP_STD_VER <= 17
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<error_condition> : public __unary_function<error_condition, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const error_condition& __ec) const _NOEXCEPT {
+    return static_cast<size_t>(__ec.value());
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___SYSTEM_ERROR_ERROR_CONDITION_H
diff --git a/libcxx/include/__cxx03/__system_error/system_error.h b/libcxx/include/__cxx03/__system_error/system_error.h
new file mode 100644
index 000000000000..362e67505658
--- /dev/null
+++ b/libcxx/include/__cxx03/__system_error/system_error.h
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___SYSTEM_ERROR_SYSTEM_ERROR_H
+#define _LIBCPP___SYSTEM_ERROR_SYSTEM_ERROR_H
+
+#include <__config>
+#include <__system_error/error_category.h>
+#include <__system_error/error_code.h>
+#include <__verbose_abort>
+#include <stdexcept>
+#include <string>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+class _LIBCPP_EXPORTED_FROM_ABI system_error : public runtime_error {
+  error_code __ec_;
+
+public:
+  system_error(error_code __ec, const string& __what_arg);
+  system_error(error_code __ec, const char* __what_arg);
+  system_error(error_code __ec);
+  system_error(int __ev, const error_category& __ecat, const string& __what_arg);
+  system_error(int __ev, const error_category& __ecat, const char* __what_arg);
+  system_error(int __ev, const error_category& __ecat);
+  _LIBCPP_HIDE_FROM_ABI system_error(const system_error&) _NOEXCEPT = default;
+  ~system_error() _NOEXCEPT override;
+
+  _LIBCPP_HIDE_FROM_ABI const error_code& code() const _NOEXCEPT { return __ec_; }
+};
+
+_LIBCPP_NORETURN _LIBCPP_EXPORTED_FROM_ABI void __throw_system_error(int __ev, const char* __what_arg);
+_LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI inline void __throw_system_error(error_code __ec, const char* __what_arg) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw system_error(__ec, __what_arg);
+#else
+  _LIBCPP_VERBOSE_ABORT(
+      "system_error was thrown in -fno-exceptions mode with error %i and message \"%s\"", __ec.value(), __what_arg);
+#endif
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___SYSTEM_ERROR_SYSTEM_ERROR_H
diff --git a/libcxx/include/__cxx03/__thread/formatter.h b/libcxx/include/__cxx03/__thread/formatter.h
new file mode 100644
index 000000000000..9b54036dcab3
--- /dev/null
+++ b/libcxx/include/__cxx03/__thread/formatter.h
@@ -0,0 +1,80 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___THREAD_FORMATTER_H
+#define _LIBCPP___THREAD_FORMATTER_H
+
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <__format/concepts.h>
+#include <__format/format_parse_context.h>
+#include <__format/formatter.h>
+#include <__format/formatter_integral.h>
+#include <__format/parser_std_format_spec.h>
+#include <__thread/id.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/is_same.h>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  ifndef _LIBCPP_HAS_NO_THREADS
+
+template <__fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<__thread_id, _CharT> {
+public:
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return __parser_.__parse(__ctx, __format_spec::__fields_fill_align_width);
+  }
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(__thread_id __id, _FormatContext& __ctx) const {
+    // In __thread/support/pthread.h, __libcpp_thread_id is either a
+    // unsigned long long or a pthread_t.
+    //
+    // The type of pthread_t is left unspecified in POSIX so it can be any
+    // type. The most logical types are an integral or pointer.
+    // On Linux systems pthread_t is an unsigned long long.
+    // On Apple systems pthread_t is a pointer type.
+    //
+    // Note the output should match what the stream operator does. Since
+    // the ostream operator has been shipped years before this formatter
+    // was added to the Standard, this formatter does what the stream
+    // operator does. This may require platform specific changes.
+
+    using _Tp = decltype(__get_underlying_id(__id));
+    using _Cp = conditional_t<integral<_Tp>, _Tp, conditional_t<is_pointer_v<_Tp>, uintptr_t, void>>;
+    static_assert(!is_same_v<_Cp, void>, "unsupported thread::id type, please file a bug report");
+
+    __format_spec::__parsed_specifications<_CharT> __specs = __parser_.__get_parsed_std_specifications(__ctx);
+    if constexpr (is_pointer_v<_Tp>) {
+      __specs.__std_.__alternate_form_ = true;
+      __specs.__std_.__type_           = __format_spec::__type::__hexadecimal_lower_case;
+    }
+    return __formatter::__format_integer(reinterpret_cast<_Cp>(__get_underlying_id(__id)), __ctx, __specs);
+  }
+
+  __format_spec::__parser<_CharT> __parser_{.__alignment_ = __format_spec::__alignment::__right};
+};
+
+#  endif // !_LIBCPP_HAS_NO_THREADS
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+#endif // _LIBCPP___THREAD_FORMATTER_H
diff --git a/libcxx/include/__cxx03/__thread/id.h b/libcxx/include/__cxx03/__thread/id.h
new file mode 100644
index 000000000000..6db0ccbfe569
--- /dev/null
+++ b/libcxx/include/__cxx03/__thread/id.h
@@ -0,0 +1,121 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___THREAD_ID_H
+#define _LIBCPP___THREAD_ID_H
+
+#include <__compare/ordering.h>
+#include <__config>
+#include <__fwd/functional.h>
+#include <__fwd/ostream.h>
+#include <__thread/support.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifndef _LIBCPP_HAS_NO_THREADS
+class _LIBCPP_EXPORTED_FROM_ABI __thread_id;
+
+namespace this_thread {
+
+_LIBCPP_HIDE_FROM_ABI __thread_id get_id() _NOEXCEPT;
+
+} // namespace this_thread
+
+template <>
+struct hash<__thread_id>;
+
+class _LIBCPP_TEMPLATE_VIS __thread_id {
+  // FIXME: pthread_t is a pointer on Darwin but a long on Linux.
+  // NULL is the no-thread value on Darwin.  Someone needs to check
+  // on other platforms.  We assume 0 works everywhere for now.
+  __libcpp_thread_id __id_;
+
+  static _LIBCPP_HIDE_FROM_ABI bool
+  __lt_impl(__thread_id __x, __thread_id __y) _NOEXCEPT { // id==0 is always less than any other thread_id
+    if (__x.__id_ == 0)
+      return __y.__id_ != 0;
+    if (__y.__id_ == 0)
+      return false;
+    return __libcpp_thread_id_less(__x.__id_, __y.__id_);
+  }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __thread_id() _NOEXCEPT : __id_(0) {}
+
+  _LIBCPP_HIDE_FROM_ABI void __reset() { __id_ = 0; }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(__thread_id __x, __thread_id __y) _NOEXCEPT;
+#  if _LIBCPP_STD_VER <= 17
+  friend _LIBCPP_HIDE_FROM_ABI bool operator<(__thread_id __x, __thread_id __y) _NOEXCEPT;
+#  else  // _LIBCPP_STD_VER <= 17
+  friend _LIBCPP_HIDE_FROM_ABI strong_ordering operator<=>(__thread_id __x, __thread_id __y) noexcept;
+#  endif // _LIBCPP_STD_VER <= 17
+
+  template <class _CharT, class _Traits>
+  friend _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, __thread_id __id);
+
+private:
+  _LIBCPP_HIDE_FROM_ABI __thread_id(__libcpp_thread_id __id) : __id_(__id) {}
+
+  _LIBCPP_HIDE_FROM_ABI friend __libcpp_thread_id __get_underlying_id(const __thread_id __id) { return __id.__id_; }
+
+  friend __thread_id this_thread::get_id() _NOEXCEPT;
+  friend class _LIBCPP_EXPORTED_FROM_ABI thread;
+  friend struct _LIBCPP_TEMPLATE_VIS hash<__thread_id>;
+};
+
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(__thread_id __x, __thread_id __y) _NOEXCEPT {
+  // Don't pass id==0 to underlying routines
+  if (__x.__id_ == 0)
+    return __y.__id_ == 0;
+  if (__y.__id_ == 0)
+    return false;
+  return __libcpp_thread_id_equal(__x.__id_, __y.__id_);
+}
+
+#  if _LIBCPP_STD_VER <= 17
+
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(__thread_id __x, __thread_id __y) _NOEXCEPT { return !(__x == __y); }
+
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(__thread_id __x, __thread_id __y) _NOEXCEPT {
+  return __thread_id::__lt_impl(__x.__id_, __y.__id_);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool operator<=(__thread_id __x, __thread_id __y) _NOEXCEPT { return !(__y < __x); }
+inline _LIBCPP_HIDE_FROM_ABI bool operator>(__thread_id __x, __thread_id __y) _NOEXCEPT { return __y < __x; }
+inline _LIBCPP_HIDE_FROM_ABI bool operator>=(__thread_id __x, __thread_id __y) _NOEXCEPT { return !(__x < __y); }
+
+#  else // _LIBCPP_STD_VER <= 17
+
+inline _LIBCPP_HIDE_FROM_ABI strong_ordering operator<=>(__thread_id __x, __thread_id __y) noexcept {
+  if (__x == __y)
+    return strong_ordering::equal;
+  if (__thread_id::__lt_impl(__x, __y))
+    return strong_ordering::less;
+  return strong_ordering::greater;
+}
+
+#  endif // _LIBCPP_STD_VER <= 17
+
+namespace this_thread {
+
+inline _LIBCPP_HIDE_FROM_ABI __thread_id get_id() _NOEXCEPT { return __libcpp_thread_get_current_id(); }
+
+} // namespace this_thread
+
+#endif // !_LIBCPP_HAS_NO_THREADS
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___THREAD_ID_H
diff --git a/libcxx/include/__cxx03/__thread/jthread.h b/libcxx/include/__cxx03/__thread/jthread.h
new file mode 100644
index 000000000000..b3d5c25fb71c
--- /dev/null
+++ b/libcxx/include/__cxx03/__thread/jthread.h
@@ -0,0 +1,134 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___THREAD_JTHREAD_H
+#define _LIBCPP___THREAD_JTHREAD_H
+
+#include <__config>
+#include <__functional/invoke.h>
+#include <__stop_token/stop_source.h>
+#include <__stop_token/stop_token.h>
+#include <__thread/support.h>
+#include <__thread/thread.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN)
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+class _LIBCPP_AVAILABILITY_SYNC jthread {
+public:
+  // types
+  using id                 = thread::id;
+  using native_handle_type = thread::native_handle_type;
+
+  // [thread.jthread.cons], constructors, move, and assignment
+  _LIBCPP_HIDE_FROM_ABI jthread() noexcept : __stop_source_(std::nostopstate) {}
+
+  template <class _Fun, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI explicit jthread(_Fun&& __fun, _Args&&... __args)
+    requires(!std::is_same_v<remove_cvref_t<_Fun>, jthread>)
+      : __stop_source_(),
+        __thread_(__init_thread(__stop_source_, std::forward<_Fun>(__fun), std::forward<_Args>(__args)...)) {
+    static_assert(is_constructible_v<decay_t<_Fun>, _Fun>);
+    static_assert((is_constructible_v<decay_t<_Args>, _Args> && ...));
+    static_assert(is_invocable_v<decay_t<_Fun>, decay_t<_Args>...> ||
+                  is_invocable_v<decay_t<_Fun>, stop_token, decay_t<_Args>...>);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~jthread() {
+    if (joinable()) {
+      request_stop();
+      join();
+    }
+  }
+
+  jthread(const jthread&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI jthread(jthread&&) noexcept = default;
+
+  jthread& operator=(const jthread&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI jthread& operator=(jthread&& __other) noexcept {
+    if (this != &__other) {
+      if (joinable()) {
+        request_stop();
+        join();
+      }
+      __stop_source_ = std::move(__other.__stop_source_);
+      __thread_      = std::move(__other.__thread_);
+    }
+
+    return *this;
+  }
+
+  // [thread.jthread.mem], members
+  _LIBCPP_HIDE_FROM_ABI void swap(jthread& __other) noexcept {
+    std::swap(__stop_source_, __other.__stop_source_);
+    std::swap(__thread_, __other.__thread_);
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool joinable() const noexcept { return get_id() != id(); }
+
+  _LIBCPP_HIDE_FROM_ABI void join() { __thread_.join(); }
+
+  _LIBCPP_HIDE_FROM_ABI void detach() { __thread_.detach(); }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI id get_id() const noexcept { return __thread_.get_id(); }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return __thread_.native_handle(); }
+
+  // [thread.jthread.stop], stop token handling
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI stop_source get_stop_source() noexcept { return __stop_source_; }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI stop_token get_stop_token() const noexcept { return __stop_source_.get_token(); }
+
+  _LIBCPP_HIDE_FROM_ABI bool request_stop() noexcept { return __stop_source_.request_stop(); }
+
+  // [thread.jthread.special], specialized algorithms
+  _LIBCPP_HIDE_FROM_ABI friend void swap(jthread& __lhs, jthread& __rhs) noexcept { __lhs.swap(__rhs); }
+
+  // [thread.jthread.static], static members
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI static unsigned int hardware_concurrency() noexcept {
+    return thread::hardware_concurrency();
+  }
+
+private:
+  template <class _Fun, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI static thread __init_thread(const stop_source& __ss, _Fun&& __fun, _Args&&... __args) {
+    if constexpr (is_invocable_v<decay_t<_Fun>, stop_token, decay_t<_Args>...>) {
+      return thread(std::forward<_Fun>(__fun), __ss.get_token(), std::forward<_Args>(__args)...);
+    } else {
+      return thread(std::forward<_Fun>(__fun), std::forward<_Args>(__args)...);
+    }
+  }
+
+  stop_source __stop_source_;
+  thread __thread_;
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN)
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___THREAD_JTHREAD_H
diff --git a/libcxx/include/__cxx03/__thread/poll_with_backoff.h b/libcxx/include/__cxx03/__thread/poll_with_backoff.h
new file mode 100644
index 000000000000..4f961fe3f762
--- /dev/null
+++ b/libcxx/include/__cxx03/__thread/poll_with_backoff.h
@@ -0,0 +1,67 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___THREAD_POLL_WITH_BACKOFF_H
+#define _LIBCPP___THREAD_POLL_WITH_BACKOFF_H
+
+#include <__chrono/duration.h>
+#include <__chrono/high_resolution_clock.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+static _LIBCPP_CONSTEXPR const int __libcpp_polling_count = 64;
+
+// Polls a thread for a condition given by a predicate, and backs off based on a backoff policy
+// before polling again.
+//
+// - __poll is the "test function" that should return true if polling succeeded, and false if it failed.
+//
+// - __backoff is the "backoff policy", which is called with the duration since we started polling. It should
+//   return false in order to resume polling, and true if polling should stop entirely for some reason.
+//   In general, backoff policies sleep for some time before returning control to the polling loop.
+//
+// - __max_elapsed is the maximum duration to try polling for. If the maximum duration is exceeded,
+//   the polling loop will return false to report a timeout.
+template <class _Poll, class _Backoff>
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool __libcpp_thread_poll_with_backoff(
+    _Poll&& __poll, _Backoff&& __backoff, chrono::nanoseconds __max_elapsed = chrono::nanoseconds::zero()) {
+  auto const __start = chrono::high_resolution_clock::now();
+  for (int __count = 0;;) {
+    if (__poll())
+      return true; // __poll completion means success
+    if (__count < __libcpp_polling_count) {
+      __count += 1;
+      continue;
+    }
+    chrono::nanoseconds const __elapsed = chrono::high_resolution_clock::now() - __start;
+    if (__max_elapsed != chrono::nanoseconds::zero() && __max_elapsed < __elapsed)
+      return false; // timeout failure
+    if (__backoff(__elapsed))
+      return false; // __backoff completion means failure
+  }
+}
+
+// A trivial backoff policy that always immediately returns the control to
+// the polling loop.
+//
+// This is not very well-behaved since it will cause the polling loop to spin,
+// so this should most likely only be used on single-threaded systems where there
+// are no other threads to compete with.
+struct __spinning_backoff_policy {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator()(chrono::nanoseconds const&) const { return false; }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___THREAD_POLL_WITH_BACKOFF_H
diff --git a/libcxx/include/__cxx03/__thread/support.h b/libcxx/include/__cxx03/__thread/support.h
new file mode 100644
index 000000000000..92f1c4415e4d
--- /dev/null
+++ b/libcxx/include/__cxx03/__thread/support.h
@@ -0,0 +1,123 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___THREAD_SUPPORT_H
+#define _LIBCPP___THREAD_SUPPORT_H
+
+#include <__config>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+/*
+
+//
+// The library supports multiple implementations of the basic threading functionality.
+// The following functionality must be provided by any implementation:
+//
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using __libcpp_timespec_t = ...;
+
+//
+// Mutex
+//
+using __libcpp_mutex_t = ...;
+#define _LIBCPP_MUTEX_INITIALIZER ...
+
+using __libcpp_recursive_mutex_t = ...;
+
+int __libcpp_recursive_mutex_init(__libcpp_recursive_mutex_t*);
+_LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_recursive_mutex_lock(__libcpp_recursive_mutex_t*);
+_LIBCPP_NO_THREAD_SAFETY_ANALYSIS bool __libcpp_recursive_mutex_trylock(__libcpp_recursive_mutex_t*);
+_LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_recursive_mutex_unlock(__libcpp_recursive_mutex_t*);
+int __libcpp_recursive_mutex_destroy(__libcpp_recursive_mutex_t*);
+
+_LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_mutex_lock(__libcpp_mutex_t*);
+_LIBCPP_NO_THREAD_SAFETY_ANALYSIS bool __libcpp_mutex_trylock(__libcpp_mutex_t*);
+_LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_mutex_unlock(__libcpp_mutex_t*);
+int __libcpp_mutex_destroy(__libcpp_mutex_t*);
+
+//
+// Condition Variable
+//
+using __libcpp_condvar_t = ...;
+#define _LIBCPP_CONDVAR_INITIALIZER ...
+
+int __libcpp_condvar_signal(__libcpp_condvar_t*);
+int __libcpp_condvar_broadcast(__libcpp_condvar_t*);
+_LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_condvar_wait(__libcpp_condvar_t*, __libcpp_mutex_t*);
+_LIBCPP_NO_THREAD_SAFETY_ANALYSIS
+int __libcpp_condvar_timedwait(__libcpp_condvar_t*, __libcpp_mutex_t*, __libcpp_timespec_t*);
+int __libcpp_condvar_destroy(__libcpp_condvar_t*);
+
+//
+// Execute once
+//
+using __libcpp_exec_once_flag = ...;
+#define _LIBCPP_EXEC_ONCE_INITIALIZER ...
+
+int __libcpp_execute_once(__libcpp_exec_once_flag*, void (*__init_routine)());
+
+//
+// Thread id
+//
+using __libcpp_thread_id = ...;
+
+bool __libcpp_thread_id_equal(__libcpp_thread_id, __libcpp_thread_id);
+bool __libcpp_thread_id_less(__libcpp_thread_id, __libcpp_thread_id);
+
+//
+// Thread
+//
+#define _LIBCPP_NULL_THREAD ...
+using __libcpp_thread_t = ...;
+
+bool __libcpp_thread_isnull(const __libcpp_thread_t*);
+int __libcpp_thread_create(__libcpp_thread_t*, void* (*__func)(void*), void* __arg);
+__libcpp_thread_id __libcpp_thread_get_current_id();
+__libcpp_thread_id __libcpp_thread_get_id(const __libcpp_thread_t*);
+int __libcpp_thread_join(__libcpp_thread_t*);
+int __libcpp_thread_detach(__libcpp_thread_t*);
+void __libcpp_thread_yield();
+void __libcpp_thread_sleep_for(const chrono::nanoseconds&);
+
+//
+// Thread local storage
+//
+#define _LIBCPP_TLS_DESTRUCTOR_CC ...
+using __libcpp_tls_key = ...;
+
+int __libcpp_tls_create(__libcpp_tls_key*, void (*__at_exit)(void*));
+void* __libcpp_tls_get(__libcpp_tls_key);
+int __libcpp_tls_set(__libcpp_tls_key, void*);
+
+_LIBCPP_END_NAMESPACE_STD
+
+*/
+
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+
+#  if defined(_LIBCPP_HAS_THREAD_API_EXTERNAL)
+#    include <__thread/support/external.h>
+#  elif defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
+#    include <__thread/support/pthread.h>
+#  elif defined(_LIBCPP_HAS_THREAD_API_C11)
+#    include <__thread/support/c11.h>
+#  elif defined(_LIBCPP_HAS_THREAD_API_WIN32)
+#    include <__thread/support/windows.h>
+#  else
+#    error "No threading API was selected"
+#  endif
+
+#endif // !_LIBCPP_HAS_NO_THREADS
+
+#endif // _LIBCPP___THREAD_SUPPORT_H
diff --git a/libcxx/include/__cxx03/__thread/support/c11.h b/libcxx/include/__cxx03/__thread/support/c11.h
new file mode 100644
index 000000000000..fe00a2d97fad
--- /dev/null
+++ b/libcxx/include/__cxx03/__thread/support/c11.h
@@ -0,0 +1,191 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___THREAD_SUPPORT_C11_H
+#define _LIBCPP___THREAD_SUPPORT_C11_H
+
+#include <__chrono/convert_to_timespec.h>
+#include <__chrono/duration.h>
+#include <__config>
+#include <ctime>
+#include <errno.h>
+#include <threads.h>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using __libcpp_timespec_t = ::timespec;
+
+//
+// Mutex
+//
+typedef mtx_t __libcpp_mutex_t;
+// mtx_t is a struct so using {} for initialization is valid.
+#define _LIBCPP_MUTEX_INITIALIZER                                                                                      \
+  {}
+
+typedef mtx_t __libcpp_recursive_mutex_t;
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_recursive_mutex_init(__libcpp_recursive_mutex_t* __m) {
+  return mtx_init(__m, mtx_plain | mtx_recursive) == thrd_success ? 0 : EINVAL;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
+__libcpp_recursive_mutex_lock(__libcpp_recursive_mutex_t* __m) {
+  return mtx_lock(__m) == thrd_success ? 0 : EINVAL;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS bool
+__libcpp_recursive_mutex_trylock(__libcpp_recursive_mutex_t* __m) {
+  return mtx_trylock(__m) == thrd_success;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
+__libcpp_recursive_mutex_unlock(__libcpp_recursive_mutex_t* __m) {
+  return mtx_unlock(__m) == thrd_success ? 0 : EINVAL;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_recursive_mutex_destroy(__libcpp_recursive_mutex_t* __m) {
+  mtx_destroy(__m);
+  return 0;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_mutex_lock(__libcpp_mutex_t* __m) {
+  return mtx_lock(__m) == thrd_success ? 0 : EINVAL;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS bool __libcpp_mutex_trylock(__libcpp_mutex_t* __m) {
+  return mtx_trylock(__m) == thrd_success;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_mutex_unlock(__libcpp_mutex_t* __m) {
+  return mtx_unlock(__m) == thrd_success ? 0 : EINVAL;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_mutex_destroy(__libcpp_mutex_t* __m) {
+  mtx_destroy(__m);
+  return 0;
+}
+
+//
+// Condition Variable
+//
+typedef cnd_t __libcpp_condvar_t;
+// cnd_t is a struct so using {} for initialization is valid.
+#define _LIBCPP_CONDVAR_INITIALIZER                                                                                    \
+  {}
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_condvar_signal(__libcpp_condvar_t* __cv) {
+  return cnd_signal(__cv) == thrd_success ? 0 : EINVAL;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_condvar_broadcast(__libcpp_condvar_t* __cv) {
+  return cnd_broadcast(__cv) == thrd_success ? 0 : EINVAL;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
+__libcpp_condvar_wait(__libcpp_condvar_t* __cv, __libcpp_mutex_t* __m) {
+  return cnd_wait(__cv, __m) == thrd_success ? 0 : EINVAL;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
+__libcpp_condvar_timedwait(__libcpp_condvar_t* __cv, __libcpp_mutex_t* __m, timespec* __ts) {
+  int __ec = cnd_timedwait(__cv, __m, __ts);
+  return __ec == thrd_timedout ? ETIMEDOUT : __ec;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_condvar_destroy(__libcpp_condvar_t* __cv) {
+  cnd_destroy(__cv);
+  return 0;
+}
+
+//
+// Execute once
+//
+typedef ::once_flag __libcpp_exec_once_flag;
+#define _LIBCPP_EXEC_ONCE_INITIALIZER ONCE_FLAG_INIT
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_execute_once(__libcpp_exec_once_flag* flag, void (*init_routine)(void)) {
+  ::call_once(flag, init_routine);
+  return 0;
+}
+
+//
+// Thread id
+//
+typedef thrd_t __libcpp_thread_id;
+
+// Returns non-zero if the thread ids are equal, otherwise 0
+inline _LIBCPP_HIDE_FROM_ABI bool __libcpp_thread_id_equal(__libcpp_thread_id t1, __libcpp_thread_id t2) {
+  return thrd_equal(t1, t2) != 0;
+}
+
+// Returns non-zero if t1 < t2, otherwise 0
+inline _LIBCPP_HIDE_FROM_ABI bool __libcpp_thread_id_less(__libcpp_thread_id t1, __libcpp_thread_id t2) {
+  return t1 < t2;
+}
+
+//
+// Thread
+//
+#define _LIBCPP_NULL_THREAD 0U
+
+typedef thrd_t __libcpp_thread_t;
+
+inline _LIBCPP_HIDE_FROM_ABI __libcpp_thread_id __libcpp_thread_get_id(const __libcpp_thread_t* __t) { return *__t; }
+
+inline _LIBCPP_HIDE_FROM_ABI bool __libcpp_thread_isnull(const __libcpp_thread_t* __t) {
+  return __libcpp_thread_get_id(__t) == 0;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_thread_create(__libcpp_thread_t* __t, void* (*__func)(void*), void* __arg) {
+  int __ec = thrd_create(__t, reinterpret_cast<thrd_start_t>(__func), __arg);
+  return __ec == thrd_nomem ? ENOMEM : __ec;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI __libcpp_thread_id __libcpp_thread_get_current_id() { return thrd_current(); }
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_thread_join(__libcpp_thread_t* __t) {
+  return thrd_join(*__t, nullptr) == thrd_success ? 0 : EINVAL;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_thread_detach(__libcpp_thread_t* __t) {
+  return thrd_detach(*__t) == thrd_success ? 0 : EINVAL;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI void __libcpp_thread_yield() { thrd_yield(); }
+
+inline _LIBCPP_HIDE_FROM_ABI void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns) {
+  __libcpp_timespec_t __ts = std::__convert_to_timespec<__libcpp_timespec_t>(__ns);
+  thrd_sleep(&__ts, nullptr);
+}
+
+//
+// Thread local storage
+//
+#define _LIBCPP_TLS_DESTRUCTOR_CC /* nothing */
+
+typedef tss_t __libcpp_tls_key;
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_tls_create(__libcpp_tls_key* __key, void (*__at_exit)(void*)) {
+  return tss_create(__key, __at_exit) == thrd_success ? 0 : EINVAL;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI void* __libcpp_tls_get(__libcpp_tls_key __key) { return tss_get(__key); }
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_tls_set(__libcpp_tls_key __key, void* __p) {
+  return tss_set(__key, __p) == thrd_success ? 0 : EINVAL;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___THREAD_SUPPORT_C11_H
diff --git a/libcxx/include/__cxx03/__thread/support/external.h b/libcxx/include/__cxx03/__thread/support/external.h
new file mode 100644
index 000000000000..d5e212491cfd
--- /dev/null
+++ b/libcxx/include/__cxx03/__thread/support/external.h
@@ -0,0 +1,21 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___THREAD_SUPPORT_EXTERNAL_H
+#define _LIBCPP___THREAD_SUPPORT_EXTERNAL_H
+
+#include <__config>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+#include <__external_threading>
+
+#endif // _LIBCPP___THREAD_SUPPORT_EXTERNAL_H
diff --git a/libcxx/include/__cxx03/__thread/support/pthread.h b/libcxx/include/__cxx03/__thread/support/pthread.h
new file mode 100644
index 000000000000..531f3e71de83
--- /dev/null
+++ b/libcxx/include/__cxx03/__thread/support/pthread.h
@@ -0,0 +1,221 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___THREAD_SUPPORT_PTHREAD_H
+#define _LIBCPP___THREAD_SUPPORT_PTHREAD_H
+
+#include <__chrono/convert_to_timespec.h>
+#include <__chrono/duration.h>
+#include <__config>
+#include <ctime>
+#include <errno.h>
+#include <pthread.h>
+#include <sched.h>
+
+#ifdef __MVS__
+#  include <__support/ibm/nanosleep.h>
+#endif
+
+// Some platforms require <bits/atomic_wide_counter.h> in order for
+// PTHREAD_COND_INITIALIZER to be expanded. Normally that would come
+// in via <pthread.h>, but it's a non-modular header on those platforms,
+// so libc++'s <math.h> usually absorbs atomic_wide_counter.h into the
+// module with <math.h> and makes atomic_wide_counter.h invisible.
+// Include <math.h> here to work around that.
+// This checks wheter a Clang module is built
+#if __building_module(std)
+#  include <math.h>
+#endif
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using __libcpp_timespec_t = ::timespec;
+
+//
+// Mutex
+//
+typedef pthread_mutex_t __libcpp_mutex_t;
+#define _LIBCPP_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
+
+typedef pthread_mutex_t __libcpp_recursive_mutex_t;
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_recursive_mutex_init(__libcpp_recursive_mutex_t* __m) {
+  pthread_mutexattr_t __attr;
+  int __ec = pthread_mutexattr_init(&__attr);
+  if (__ec)
+    return __ec;
+  __ec = pthread_mutexattr_settype(&__attr, PTHREAD_MUTEX_RECURSIVE);
+  if (__ec) {
+    pthread_mutexattr_destroy(&__attr);
+    return __ec;
+  }
+  __ec = pthread_mutex_init(__m, &__attr);
+  if (__ec) {
+    pthread_mutexattr_destroy(&__attr);
+    return __ec;
+  }
+  __ec = pthread_mutexattr_destroy(&__attr);
+  if (__ec) {
+    pthread_mutex_destroy(__m);
+    return __ec;
+  }
+  return 0;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
+__libcpp_recursive_mutex_lock(__libcpp_recursive_mutex_t* __m) {
+  return pthread_mutex_lock(__m);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS bool
+__libcpp_recursive_mutex_trylock(__libcpp_recursive_mutex_t* __m) {
+  return pthread_mutex_trylock(__m) == 0;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
+__libcpp_recursive_mutex_unlock(__libcpp_recursive_mutex_t* __m) {
+  return pthread_mutex_unlock(__m);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_recursive_mutex_destroy(__libcpp_recursive_mutex_t* __m) {
+  return pthread_mutex_destroy(__m);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_mutex_lock(__libcpp_mutex_t* __m) {
+  return pthread_mutex_lock(__m);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS bool __libcpp_mutex_trylock(__libcpp_mutex_t* __m) {
+  return pthread_mutex_trylock(__m) == 0;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_mutex_unlock(__libcpp_mutex_t* __m) {
+  return pthread_mutex_unlock(__m);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_mutex_destroy(__libcpp_mutex_t* __m) { return pthread_mutex_destroy(__m); }
+
+//
+// Condition Variable
+//
+typedef pthread_cond_t __libcpp_condvar_t;
+#define _LIBCPP_CONDVAR_INITIALIZER PTHREAD_COND_INITIALIZER
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_condvar_signal(__libcpp_condvar_t* __cv) { return pthread_cond_signal(__cv); }
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_condvar_broadcast(__libcpp_condvar_t* __cv) {
+  return pthread_cond_broadcast(__cv);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
+__libcpp_condvar_wait(__libcpp_condvar_t* __cv, __libcpp_mutex_t* __m) {
+  return pthread_cond_wait(__cv, __m);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
+__libcpp_condvar_timedwait(__libcpp_condvar_t* __cv, __libcpp_mutex_t* __m, __libcpp_timespec_t* __ts) {
+  return pthread_cond_timedwait(__cv, __m, __ts);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_condvar_destroy(__libcpp_condvar_t* __cv) {
+  return pthread_cond_destroy(__cv);
+}
+
+//
+// Execute once
+//
+typedef pthread_once_t __libcpp_exec_once_flag;
+#define _LIBCPP_EXEC_ONCE_INITIALIZER PTHREAD_ONCE_INIT
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_execute_once(__libcpp_exec_once_flag* __flag, void (*__init_routine)()) {
+  return pthread_once(__flag, __init_routine);
+}
+
+//
+// Thread id
+//
+#if defined(__MVS__)
+typedef unsigned long long __libcpp_thread_id;
+#else
+typedef pthread_t __libcpp_thread_id;
+#endif
+
+// Returns non-zero if the thread ids are equal, otherwise 0
+inline _LIBCPP_HIDE_FROM_ABI bool __libcpp_thread_id_equal(__libcpp_thread_id __t1, __libcpp_thread_id __t2) {
+  return __t1 == __t2;
+}
+
+// Returns non-zero if t1 < t2, otherwise 0
+inline _LIBCPP_HIDE_FROM_ABI bool __libcpp_thread_id_less(__libcpp_thread_id __t1, __libcpp_thread_id __t2) {
+  return __t1 < __t2;
+}
+
+//
+// Thread
+//
+#define _LIBCPP_NULL_THREAD ((__libcpp_thread_t()))
+typedef pthread_t __libcpp_thread_t;
+
+inline _LIBCPP_HIDE_FROM_ABI __libcpp_thread_id __libcpp_thread_get_id(const __libcpp_thread_t* __t) {
+#if defined(__MVS__)
+  return __t->__;
+#else
+  return *__t;
+#endif
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool __libcpp_thread_isnull(const __libcpp_thread_t* __t) {
+  return __libcpp_thread_get_id(__t) == 0;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_thread_create(__libcpp_thread_t* __t, void* (*__func)(void*), void* __arg) {
+  return pthread_create(__t, nullptr, __func, __arg);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI __libcpp_thread_id __libcpp_thread_get_current_id() {
+  const __libcpp_thread_t __current_thread = pthread_self();
+  return __libcpp_thread_get_id(&__current_thread);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_thread_join(__libcpp_thread_t* __t) { return pthread_join(*__t, nullptr); }
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_thread_detach(__libcpp_thread_t* __t) { return pthread_detach(*__t); }
+
+inline _LIBCPP_HIDE_FROM_ABI void __libcpp_thread_yield() { sched_yield(); }
+
+inline _LIBCPP_HIDE_FROM_ABI void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns) {
+  __libcpp_timespec_t __ts = std::__convert_to_timespec<__libcpp_timespec_t>(__ns);
+  while (nanosleep(&__ts, &__ts) == -1 && errno == EINTR)
+    ;
+}
+
+//
+// Thread local storage
+//
+#define _LIBCPP_TLS_DESTRUCTOR_CC /* nothing */
+
+typedef pthread_key_t __libcpp_tls_key;
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_tls_create(__libcpp_tls_key* __key, void (*__at_exit)(void*)) {
+  return pthread_key_create(__key, __at_exit);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI void* __libcpp_tls_get(__libcpp_tls_key __key) { return pthread_getspecific(__key); }
+
+inline _LIBCPP_HIDE_FROM_ABI int __libcpp_tls_set(__libcpp_tls_key __key, void* __p) {
+  return pthread_setspecific(__key, __p);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___THREAD_SUPPORT_PTHREAD_H
diff --git a/libcxx/include/__cxx03/__thread/support/windows.h b/libcxx/include/__cxx03/__thread/support/windows.h
new file mode 100644
index 000000000000..5dc4fa14f45b
--- /dev/null
+++ b/libcxx/include/__cxx03/__thread/support/windows.h
@@ -0,0 +1,133 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___THREAD_SUPPORT_WINDOWS_H
+#define _LIBCPP___THREAD_SUPPORT_WINDOWS_H
+
+#include <__chrono/duration.h>
+#include <__config>
+#include <ctime>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using __libcpp_timespec_t = ::timespec;
+
+//
+// Mutex
+//
+typedef void* __libcpp_mutex_t;
+#define _LIBCPP_MUTEX_INITIALIZER 0
+
+#if defined(_M_IX86) || defined(__i386__) || defined(_M_ARM) || defined(__arm__)
+typedef void* __libcpp_recursive_mutex_t[6];
+#elif defined(_M_AMD64) || defined(__x86_64__) || defined(_M_ARM64) || defined(__aarch64__)
+typedef void* __libcpp_recursive_mutex_t[5];
+#else
+#  error Unsupported architecture
+#endif
+
+_LIBCPP_EXPORTED_FROM_ABI int __libcpp_recursive_mutex_init(__libcpp_recursive_mutex_t* __m);
+
+_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
+__libcpp_recursive_mutex_lock(__libcpp_recursive_mutex_t* __m);
+
+_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS bool
+__libcpp_recursive_mutex_trylock(__libcpp_recursive_mutex_t* __m);
+
+_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
+__libcpp_recursive_mutex_unlock(__libcpp_recursive_mutex_t* __m);
+
+_LIBCPP_EXPORTED_FROM_ABI int __libcpp_recursive_mutex_destroy(__libcpp_recursive_mutex_t* __m);
+
+_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_mutex_lock(__libcpp_mutex_t* __m);
+
+_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS bool __libcpp_mutex_trylock(__libcpp_mutex_t* __m);
+
+_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_mutex_unlock(__libcpp_mutex_t* __m);
+
+_LIBCPP_EXPORTED_FROM_ABI int __libcpp_mutex_destroy(__libcpp_mutex_t* __m);
+
+//
+// Condition variable
+//
+typedef void* __libcpp_condvar_t;
+#define _LIBCPP_CONDVAR_INITIALIZER 0
+
+_LIBCPP_EXPORTED_FROM_ABI int __libcpp_condvar_signal(__libcpp_condvar_t* __cv);
+
+_LIBCPP_EXPORTED_FROM_ABI int __libcpp_condvar_broadcast(__libcpp_condvar_t* __cv);
+
+_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
+__libcpp_condvar_wait(__libcpp_condvar_t* __cv, __libcpp_mutex_t* __m);
+
+_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
+__libcpp_condvar_timedwait(__libcpp_condvar_t* __cv, __libcpp_mutex_t* __m, __libcpp_timespec_t* __ts);
+
+_LIBCPP_EXPORTED_FROM_ABI int __libcpp_condvar_destroy(__libcpp_condvar_t* __cv);
+
+//
+// Execute once
+//
+typedef void* __libcpp_exec_once_flag;
+#define _LIBCPP_EXEC_ONCE_INITIALIZER 0
+
+_LIBCPP_EXPORTED_FROM_ABI int __libcpp_execute_once(__libcpp_exec_once_flag* __flag, void (*__init_routine)());
+
+//
+// Thread id
+//
+typedef long __libcpp_thread_id;
+
+_LIBCPP_EXPORTED_FROM_ABI bool __libcpp_thread_id_equal(__libcpp_thread_id __t1, __libcpp_thread_id __t2);
+
+_LIBCPP_EXPORTED_FROM_ABI bool __libcpp_thread_id_less(__libcpp_thread_id __t1, __libcpp_thread_id __t2);
+
+//
+// Thread
+//
+#define _LIBCPP_NULL_THREAD 0U
+typedef void* __libcpp_thread_t;
+
+_LIBCPP_EXPORTED_FROM_ABI bool __libcpp_thread_isnull(const __libcpp_thread_t* __t);
+
+_LIBCPP_EXPORTED_FROM_ABI int __libcpp_thread_create(__libcpp_thread_t* __t, void* (*__func)(void*), void* __arg);
+
+_LIBCPP_EXPORTED_FROM_ABI __libcpp_thread_id __libcpp_thread_get_current_id();
+
+_LIBCPP_EXPORTED_FROM_ABI __libcpp_thread_id __libcpp_thread_get_id(const __libcpp_thread_t* __t);
+
+_LIBCPP_EXPORTED_FROM_ABI int __libcpp_thread_join(__libcpp_thread_t* __t);
+
+_LIBCPP_EXPORTED_FROM_ABI int __libcpp_thread_detach(__libcpp_thread_t* __t);
+
+_LIBCPP_EXPORTED_FROM_ABI void __libcpp_thread_yield();
+
+_LIBCPP_EXPORTED_FROM_ABI void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns);
+
+//
+// Thread local storage
+//
+typedef long __libcpp_tls_key;
+
+#define _LIBCPP_TLS_DESTRUCTOR_CC __stdcall
+
+_LIBCPP_EXPORTED_FROM_ABI int
+__libcpp_tls_create(__libcpp_tls_key* __key, void(_LIBCPP_TLS_DESTRUCTOR_CC* __at_exit)(void*));
+
+_LIBCPP_EXPORTED_FROM_ABI void* __libcpp_tls_get(__libcpp_tls_key __key);
+
+_LIBCPP_EXPORTED_FROM_ABI int __libcpp_tls_set(__libcpp_tls_key __key, void* __p);
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___THREAD_SUPPORT_WINDOWS_H
diff --git a/libcxx/include/__cxx03/__thread/this_thread.h b/libcxx/include/__cxx03/__thread/this_thread.h
new file mode 100644
index 000000000000..de7eea282c87
--- /dev/null
+++ b/libcxx/include/__cxx03/__thread/this_thread.h
@@ -0,0 +1,74 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___THREAD_THIS_THREAD_H
+#define _LIBCPP___THREAD_THIS_THREAD_H
+
+#include <__chrono/steady_clock.h>
+#include <__chrono/time_point.h>
+#include <__condition_variable/condition_variable.h>
+#include <__config>
+#include <__mutex/mutex.h>
+#include <__mutex/unique_lock.h>
+#include <__thread/support.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace this_thread {
+
+_LIBCPP_EXPORTED_FROM_ABI void sleep_for(const chrono::nanoseconds& __ns);
+
+template <class _Rep, class _Period>
+_LIBCPP_HIDE_FROM_ABI void sleep_for(const chrono::duration<_Rep, _Period>& __d) {
+  if (__d > chrono::duration<_Rep, _Period>::zero()) {
+    // The standard guarantees a 64bit signed integer resolution for nanoseconds,
+    // so use INT64_MAX / 1e9 as cut-off point. Use a constant to avoid <climits>
+    // and issues with long double folding on PowerPC with GCC.
+    _LIBCPP_CONSTEXPR chrono::duration<long double> __max = chrono::duration<long double>(9223372036.0L);
+    chrono::nanoseconds __ns;
+    if (__d < __max) {
+      __ns = chrono::duration_cast<chrono::nanoseconds>(__d);
+      if (__ns < __d)
+        ++__ns;
+    } else
+      __ns = chrono::nanoseconds::max();
+    this_thread::sleep_for(__ns);
+  }
+}
+
+template <class _Clock, class _Duration>
+_LIBCPP_HIDE_FROM_ABI void sleep_until(const chrono::time_point<_Clock, _Duration>& __t) {
+  mutex __mut;
+  condition_variable __cv;
+  unique_lock<mutex> __lk(__mut);
+  while (_Clock::now() < __t)
+    __cv.wait_until(__lk, __t);
+}
+
+template <class _Duration>
+inline _LIBCPP_HIDE_FROM_ABI void sleep_until(const chrono::time_point<chrono::steady_clock, _Duration>& __t) {
+  this_thread::sleep_for(__t - chrono::steady_clock::now());
+}
+
+inline _LIBCPP_HIDE_FROM_ABI void yield() _NOEXCEPT { __libcpp_thread_yield(); }
+
+} // namespace this_thread
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___THREAD_THIS_THREAD_H
diff --git a/libcxx/include/__cxx03/__thread/thread.h b/libcxx/include/__cxx03/__thread/thread.h
new file mode 100644
index 000000000000..d2254a695f5e
--- /dev/null
+++ b/libcxx/include/__cxx03/__thread/thread.h
@@ -0,0 +1,258 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___THREAD_THREAD_H
+#define _LIBCPP___THREAD_THREAD_H
+
+#include <__condition_variable/condition_variable.h>
+#include <__config>
+#include <__exception/terminate.h>
+#include <__functional/hash.h>
+#include <__functional/unary_function.h>
+#include <__memory/unique_ptr.h>
+#include <__mutex/mutex.h>
+#include <__system_error/system_error.h>
+#include <__thread/id.h>
+#include <__thread/support.h>
+#include <__utility/forward.h>
+#include <tuple>
+
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#  include <locale>
+#  include <sstream>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+class __thread_specific_ptr;
+class _LIBCPP_EXPORTED_FROM_ABI __thread_struct;
+class _LIBCPP_HIDDEN __thread_struct_imp;
+class __assoc_sub_state;
+
+_LIBCPP_EXPORTED_FROM_ABI __thread_specific_ptr<__thread_struct>& __thread_local_data();
+
+class _LIBCPP_EXPORTED_FROM_ABI __thread_struct {
+  __thread_struct_imp* __p_;
+
+  __thread_struct(const __thread_struct&);
+  __thread_struct& operator=(const __thread_struct&);
+
+public:
+  __thread_struct();
+  ~__thread_struct();
+
+  void notify_all_at_thread_exit(condition_variable*, mutex*);
+  void __make_ready_at_thread_exit(__assoc_sub_state*);
+};
+
+template <class _Tp>
+class __thread_specific_ptr {
+  __libcpp_tls_key __key_;
+
+  // Only __thread_local_data() may construct a __thread_specific_ptr
+  // and only with _Tp == __thread_struct.
+  static_assert(is_same<_Tp, __thread_struct>::value, "");
+  __thread_specific_ptr();
+  friend _LIBCPP_EXPORTED_FROM_ABI __thread_specific_ptr<__thread_struct>& __thread_local_data();
+
+  _LIBCPP_HIDDEN static void _LIBCPP_TLS_DESTRUCTOR_CC __at_thread_exit(void*);
+
+public:
+  typedef _Tp* pointer;
+
+  __thread_specific_ptr(const __thread_specific_ptr&)            = delete;
+  __thread_specific_ptr& operator=(const __thread_specific_ptr&) = delete;
+  ~__thread_specific_ptr();
+
+  _LIBCPP_HIDE_FROM_ABI pointer get() const { return static_cast<_Tp*>(__libcpp_tls_get(__key_)); }
+  _LIBCPP_HIDE_FROM_ABI pointer operator*() const { return *get(); }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return get(); }
+  void set_pointer(pointer __p);
+};
+
+template <class _Tp>
+void _LIBCPP_TLS_DESTRUCTOR_CC __thread_specific_ptr<_Tp>::__at_thread_exit(void* __p) {
+  delete static_cast<pointer>(__p);
+}
+
+template <class _Tp>
+__thread_specific_ptr<_Tp>::__thread_specific_ptr() {
+  int __ec = __libcpp_tls_create(&__key_, &__thread_specific_ptr::__at_thread_exit);
+  if (__ec)
+    __throw_system_error(__ec, "__thread_specific_ptr construction failed");
+}
+
+template <class _Tp>
+__thread_specific_ptr<_Tp>::~__thread_specific_ptr() {
+  // __thread_specific_ptr is only created with a static storage duration
+  // so this destructor is only invoked during program termination. Invoking
+  // pthread_key_delete(__key_) may prevent other threads from deleting their
+  // thread local data. For this reason we leak the key.
+}
+
+template <class _Tp>
+void __thread_specific_ptr<_Tp>::set_pointer(pointer __p) {
+  _LIBCPP_ASSERT_INTERNAL(get() == nullptr, "Attempting to overwrite thread local data");
+  std::__libcpp_tls_set(__key_, __p);
+}
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<__thread_id> : public __unary_function<__thread_id, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(__thread_id __v) const _NOEXCEPT {
+    return hash<__libcpp_thread_id>()(__v.__id_);
+  }
+};
+
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, __thread_id __id) {
+  // [thread.thread.id]/9
+  //   Effects: Inserts the text representation for charT of id into out.
+  //
+  // [thread.thread.id]/2
+  //   The text representation for the character type charT of an
+  //   object of type thread::id is an unspecified sequence of charT
+  //   such that, for two objects of type thread::id x and y, if
+  //   x == y is true, the thread::id objects have the same text
+  //   representation, and if x != y is true, the thread::id objects
+  //   have distinct text representations.
+  //
+  // Since various flags in the output stream can affect how the
+  // thread id is represented (e.g. numpunct or showbase), we
+  // use a temporary stream instead and just output the thread
+  // id representation as a string.
+
+  basic_ostringstream<_CharT, _Traits> __sstr;
+  __sstr.imbue(locale::classic());
+  __sstr << __id.__id_;
+  return __os << __sstr.str();
+}
+#endif // _LIBCPP_HAS_NO_LOCALIZATION
+
+class _LIBCPP_EXPORTED_FROM_ABI thread {
+  __libcpp_thread_t __t_;
+
+  thread(const thread&);
+  thread& operator=(const thread&);
+
+public:
+  typedef __thread_id id;
+  typedef __libcpp_thread_t native_handle_type;
+
+  _LIBCPP_HIDE_FROM_ABI thread() _NOEXCEPT : __t_(_LIBCPP_NULL_THREAD) {}
+#ifndef _LIBCPP_CXX03_LANG
+  template <class _Fp, class... _Args, __enable_if_t<!is_same<__remove_cvref_t<_Fp>, thread>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS explicit thread(_Fp&& __f, _Args&&... __args);
+#else // _LIBCPP_CXX03_LANG
+  template <class _Fp>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS explicit thread(_Fp __f);
+#endif
+  ~thread();
+
+  _LIBCPP_HIDE_FROM_ABI thread(thread&& __t) _NOEXCEPT : __t_(__t.__t_) { __t.__t_ = _LIBCPP_NULL_THREAD; }
+
+  _LIBCPP_HIDE_FROM_ABI thread& operator=(thread&& __t) _NOEXCEPT {
+    if (!__libcpp_thread_isnull(&__t_))
+      terminate();
+    __t_     = __t.__t_;
+    __t.__t_ = _LIBCPP_NULL_THREAD;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(thread& __t) _NOEXCEPT { std::swap(__t_, __t.__t_); }
+
+  _LIBCPP_HIDE_FROM_ABI bool joinable() const _NOEXCEPT { return !__libcpp_thread_isnull(&__t_); }
+  void join();
+  void detach();
+  _LIBCPP_HIDE_FROM_ABI id get_id() const _NOEXCEPT { return __libcpp_thread_get_id(&__t_); }
+  _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() _NOEXCEPT { return __t_; }
+
+  static unsigned hardware_concurrency() _NOEXCEPT;
+};
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _TSp, class _Fp, class... _Args, size_t... _Indices>
+inline _LIBCPP_HIDE_FROM_ABI void __thread_execute(tuple<_TSp, _Fp, _Args...>& __t, __tuple_indices<_Indices...>) {
+  std::__invoke(std::move(std::get<1>(__t)), std::move(std::get<_Indices>(__t))...);
+}
+
+template <class _Fp>
+_LIBCPP_HIDE_FROM_ABI void* __thread_proxy(void* __vp) {
+  // _Fp = tuple< unique_ptr<__thread_struct>, Functor, Args...>
+  unique_ptr<_Fp> __p(static_cast<_Fp*>(__vp));
+  __thread_local_data().set_pointer(std::get<0>(*__p.get()).release());
+  typedef typename __make_tuple_indices<tuple_size<_Fp>::value, 2>::type _Index;
+  std::__thread_execute(*__p.get(), _Index());
+  return nullptr;
+}
+
+template <class _Fp, class... _Args, __enable_if_t<!is_same<__remove_cvref_t<_Fp>, thread>::value, int> >
+thread::thread(_Fp&& __f, _Args&&... __args) {
+  typedef unique_ptr<__thread_struct> _TSPtr;
+  _TSPtr __tsp(new __thread_struct);
+  typedef tuple<_TSPtr, __decay_t<_Fp>, __decay_t<_Args>...> _Gp;
+  unique_ptr<_Gp> __p(new _Gp(std::move(__tsp), std::forward<_Fp>(__f), std::forward<_Args>(__args)...));
+  int __ec = std::__libcpp_thread_create(&__t_, &__thread_proxy<_Gp>, __p.get());
+  if (__ec == 0)
+    __p.release();
+  else
+    __throw_system_error(__ec, "thread constructor failed");
+}
+
+#else // _LIBCPP_CXX03_LANG
+
+template <class _Fp>
+struct __thread_invoke_pair {
+  // This type is used to pass memory for thread local storage and a functor
+  // to a newly created thread because std::pair doesn't work with
+  // std::unique_ptr in C++03.
+  _LIBCPP_HIDE_FROM_ABI __thread_invoke_pair(_Fp& __f) : __tsp_(new __thread_struct), __fn_(__f) {}
+  unique_ptr<__thread_struct> __tsp_;
+  _Fp __fn_;
+};
+
+template <class _Fp>
+_LIBCPP_HIDE_FROM_ABI void* __thread_proxy_cxx03(void* __vp) {
+  unique_ptr<_Fp> __p(static_cast<_Fp*>(__vp));
+  __thread_local_data().set_pointer(__p->__tsp_.release());
+  (__p->__fn_)();
+  return nullptr;
+}
+
+template <class _Fp>
+thread::thread(_Fp __f) {
+  typedef __thread_invoke_pair<_Fp> _InvokePair;
+  typedef unique_ptr<_InvokePair> _PairPtr;
+  _PairPtr __pp(new _InvokePair(__f));
+  int __ec = std::__libcpp_thread_create(&__t_, &__thread_proxy_cxx03<_InvokePair>, __pp.get());
+  if (__ec == 0)
+    __pp.release();
+  else
+    __throw_system_error(__ec, "thread constructor failed");
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+inline _LIBCPP_HIDE_FROM_ABI void swap(thread& __x, thread& __y) _NOEXCEPT { __x.swap(__y); }
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___THREAD_THREAD_H
diff --git a/libcxx/include/__cxx03/__thread/timed_backoff_policy.h b/libcxx/include/__cxx03/__thread/timed_backoff_policy.h
new file mode 100644
index 000000000000..838c918a57ef
--- /dev/null
+++ b/libcxx/include/__cxx03/__thread/timed_backoff_policy.h
@@ -0,0 +1,44 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___THREAD_TIMED_BACKOFF_POLICY_H
+#define _LIBCPP___THREAD_TIMED_BACKOFF_POLICY_H
+
+#include <__config>
+
+#ifndef _LIBCPP_HAS_NO_THREADS
+
+#  include <__chrono/duration.h>
+#  include <__thread/support.h>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct __libcpp_timed_backoff_policy {
+  _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const {
+    if (__elapsed > chrono::milliseconds(128))
+      __libcpp_thread_sleep_for(chrono::milliseconds(8));
+    else if (__elapsed > chrono::microseconds(64))
+      __libcpp_thread_sleep_for(__elapsed / 2);
+    else if (__elapsed > chrono::microseconds(4))
+      __libcpp_thread_yield();
+    else {
+    } // poll
+    return false;
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_NO_THREADS
+
+#endif // _LIBCPP___THREAD_TIMED_BACKOFF_POLICY_H
diff --git a/libcxx/include/__cxx03/__tree b/libcxx/include/__cxx03/__tree
new file mode 100644
index 000000000000..1990fa602d39
--- /dev/null
+++ b/libcxx/include/__cxx03/__tree
@@ -0,0 +1,2299 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TREE
+#define _LIBCPP___TREE
+
+#include <__algorithm/min.h>
+#include <__assert>
+#include <__config>
+#include <__functional/invoke.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__memory/addressof.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/compressed_pair.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/swap_allocator.h>
+#include <__memory/unique_ptr.h>
+#include <__type_traits/can_extract_key.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/is_const.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_swappable.h>
+#include <__type_traits/remove_const_ref.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <__utility/swap.h>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class, class, class, class>
+class _LIBCPP_TEMPLATE_VIS map;
+template <class, class, class, class>
+class _LIBCPP_TEMPLATE_VIS multimap;
+template <class, class, class>
+class _LIBCPP_TEMPLATE_VIS set;
+template <class, class, class>
+class _LIBCPP_TEMPLATE_VIS multiset;
+
+template <class _Tp, class _Compare, class _Allocator>
+class __tree;
+template <class _Tp, class _NodePtr, class _DiffType>
+class _LIBCPP_TEMPLATE_VIS __tree_iterator;
+template <class _Tp, class _ConstNodePtr, class _DiffType>
+class _LIBCPP_TEMPLATE_VIS __tree_const_iterator;
+
+template <class _Pointer>
+class __tree_end_node;
+template <class _VoidPtr>
+class __tree_node_base;
+template <class _Tp, class _VoidPtr>
+class __tree_node;
+
+template <class _Key, class _Value>
+struct __value_type;
+
+template <class _Allocator>
+class __map_node_destructor;
+template <class _TreeIterator>
+class _LIBCPP_TEMPLATE_VIS __map_iterator;
+template <class _TreeIterator>
+class _LIBCPP_TEMPLATE_VIS __map_const_iterator;
+
+/*
+
+_NodePtr algorithms
+
+The algorithms taking _NodePtr are red black tree algorithms.  Those
+algorithms taking a parameter named __root should assume that __root
+points to a proper red black tree (unless otherwise specified).
+
+Each algorithm herein assumes that __root->__parent_ points to a non-null
+structure which has a member __left_ which points back to __root.  No other
+member is read or written to at __root->__parent_.
+
+__root->__parent_ will be referred to below (in comments only) as end_node.
+end_node->__left_ is an externably accessible lvalue for __root, and can be
+changed by node insertion and removal (without explicit reference to end_node).
+
+All nodes (with the exception of end_node), even the node referred to as
+__root, have a non-null __parent_ field.
+
+*/
+
+// Returns:  true if __x is a left child of its parent, else false
+// Precondition:  __x != nullptr.
+template <class _NodePtr>
+inline _LIBCPP_HIDE_FROM_ABI bool __tree_is_left_child(_NodePtr __x) _NOEXCEPT {
+  return __x == __x->__parent_->__left_;
+}
+
+// Determines if the subtree rooted at __x is a proper red black subtree.  If
+//    __x is a proper subtree, returns the black height (null counts as 1).  If
+//    __x is an improper subtree, returns 0.
+template <class _NodePtr>
+unsigned __tree_sub_invariant(_NodePtr __x) {
+  if (__x == nullptr)
+    return 1;
+  // parent consistency checked by caller
+  // check __x->__left_ consistency
+  if (__x->__left_ != nullptr && __x->__left_->__parent_ != __x)
+    return 0;
+  // check __x->__right_ consistency
+  if (__x->__right_ != nullptr && __x->__right_->__parent_ != __x)
+    return 0;
+  // check __x->__left_ != __x->__right_ unless both are nullptr
+  if (__x->__left_ == __x->__right_ && __x->__left_ != nullptr)
+    return 0;
+  // If this is red, neither child can be red
+  if (!__x->__is_black_) {
+    if (__x->__left_ && !__x->__left_->__is_black_)
+      return 0;
+    if (__x->__right_ && !__x->__right_->__is_black_)
+      return 0;
+  }
+  unsigned __h = std::__tree_sub_invariant(__x->__left_);
+  if (__h == 0)
+    return 0; // invalid left subtree
+  if (__h != std::__tree_sub_invariant(__x->__right_))
+    return 0;                    // invalid or different height right subtree
+  return __h + __x->__is_black_; // return black height of this node
+}
+
+// Determines if the red black tree rooted at __root is a proper red black tree.
+//    __root == nullptr is a proper tree.  Returns true is __root is a proper
+//    red black tree, else returns false.
+template <class _NodePtr>
+_LIBCPP_HIDE_FROM_ABI bool __tree_invariant(_NodePtr __root) {
+  if (__root == nullptr)
+    return true;
+  // check __x->__parent_ consistency
+  if (__root->__parent_ == nullptr)
+    return false;
+  if (!std::__tree_is_left_child(__root))
+    return false;
+  // root must be black
+  if (!__root->__is_black_)
+    return false;
+  // do normal node checks
+  return std::__tree_sub_invariant(__root) != 0;
+}
+
+// Returns:  pointer to the left-most node under __x.
+template <class _NodePtr>
+inline _LIBCPP_HIDE_FROM_ABI _NodePtr __tree_min(_NodePtr __x) _NOEXCEPT {
+  _LIBCPP_ASSERT_INTERNAL(__x != nullptr, "Root node shouldn't be null");
+  while (__x->__left_ != nullptr)
+    __x = __x->__left_;
+  return __x;
+}
+
+// Returns:  pointer to the right-most node under __x.
+template <class _NodePtr>
+inline _LIBCPP_HIDE_FROM_ABI _NodePtr __tree_max(_NodePtr __x) _NOEXCEPT {
+  _LIBCPP_ASSERT_INTERNAL(__x != nullptr, "Root node shouldn't be null");
+  while (__x->__right_ != nullptr)
+    __x = __x->__right_;
+  return __x;
+}
+
+// Returns:  pointer to the next in-order node after __x.
+template <class _NodePtr>
+_LIBCPP_HIDE_FROM_ABI _NodePtr __tree_next(_NodePtr __x) _NOEXCEPT {
+  _LIBCPP_ASSERT_INTERNAL(__x != nullptr, "node shouldn't be null");
+  if (__x->__right_ != nullptr)
+    return std::__tree_min(__x->__right_);
+  while (!std::__tree_is_left_child(__x))
+    __x = __x->__parent_unsafe();
+  return __x->__parent_unsafe();
+}
+
+template <class _EndNodePtr, class _NodePtr>
+inline _LIBCPP_HIDE_FROM_ABI _EndNodePtr __tree_next_iter(_NodePtr __x) _NOEXCEPT {
+  _LIBCPP_ASSERT_INTERNAL(__x != nullptr, "node shouldn't be null");
+  if (__x->__right_ != nullptr)
+    return static_cast<_EndNodePtr>(std::__tree_min(__x->__right_));
+  while (!std::__tree_is_left_child(__x))
+    __x = __x->__parent_unsafe();
+  return static_cast<_EndNodePtr>(__x->__parent_);
+}
+
+// Returns:  pointer to the previous in-order node before __x.
+// Note: __x may be the end node.
+template <class _NodePtr, class _EndNodePtr>
+inline _LIBCPP_HIDE_FROM_ABI _NodePtr __tree_prev_iter(_EndNodePtr __x) _NOEXCEPT {
+  _LIBCPP_ASSERT_INTERNAL(__x != nullptr, "node shouldn't be null");
+  if (__x->__left_ != nullptr)
+    return std::__tree_max(__x->__left_);
+  _NodePtr __xx = static_cast<_NodePtr>(__x);
+  while (std::__tree_is_left_child(__xx))
+    __xx = __xx->__parent_unsafe();
+  return __xx->__parent_unsafe();
+}
+
+// Returns:  pointer to a node which has no children
+template <class _NodePtr>
+_LIBCPP_HIDE_FROM_ABI _NodePtr __tree_leaf(_NodePtr __x) _NOEXCEPT {
+  _LIBCPP_ASSERT_INTERNAL(__x != nullptr, "node shouldn't be null");
+  while (true) {
+    if (__x->__left_ != nullptr) {
+      __x = __x->__left_;
+      continue;
+    }
+    if (__x->__right_ != nullptr) {
+      __x = __x->__right_;
+      continue;
+    }
+    break;
+  }
+  return __x;
+}
+
+// Effects:  Makes __x->__right_ the subtree root with __x as its left child
+//           while preserving in-order order.
+template <class _NodePtr>
+_LIBCPP_HIDE_FROM_ABI void __tree_left_rotate(_NodePtr __x) _NOEXCEPT {
+  _LIBCPP_ASSERT_INTERNAL(__x != nullptr, "node shouldn't be null");
+  _LIBCPP_ASSERT_INTERNAL(__x->__right_ != nullptr, "node should have a right child");
+  _NodePtr __y  = __x->__right_;
+  __x->__right_ = __y->__left_;
+  if (__x->__right_ != nullptr)
+    __x->__right_->__set_parent(__x);
+  __y->__parent_ = __x->__parent_;
+  if (std::__tree_is_left_child(__x))
+    __x->__parent_->__left_ = __y;
+  else
+    __x->__parent_unsafe()->__right_ = __y;
+  __y->__left_ = __x;
+  __x->__set_parent(__y);
+}
+
+// Effects:  Makes __x->__left_ the subtree root with __x as its right child
+//           while preserving in-order order.
+template <class _NodePtr>
+_LIBCPP_HIDE_FROM_ABI void __tree_right_rotate(_NodePtr __x) _NOEXCEPT {
+  _LIBCPP_ASSERT_INTERNAL(__x != nullptr, "node shouldn't be null");
+  _LIBCPP_ASSERT_INTERNAL(__x->__left_ != nullptr, "node should have a left child");
+  _NodePtr __y = __x->__left_;
+  __x->__left_ = __y->__right_;
+  if (__x->__left_ != nullptr)
+    __x->__left_->__set_parent(__x);
+  __y->__parent_ = __x->__parent_;
+  if (std::__tree_is_left_child(__x))
+    __x->__parent_->__left_ = __y;
+  else
+    __x->__parent_unsafe()->__right_ = __y;
+  __y->__right_ = __x;
+  __x->__set_parent(__y);
+}
+
+// Effects:  Rebalances __root after attaching __x to a leaf.
+// Precondition:  __x has no children.
+//                __x == __root or == a direct or indirect child of __root.
+//                If __x were to be unlinked from __root (setting __root to
+//                  nullptr if __root == __x), __tree_invariant(__root) == true.
+// Postcondition: __tree_invariant(end_node->__left_) == true.  end_node->__left_
+//                may be different than the value passed in as __root.
+template <class _NodePtr>
+_LIBCPP_HIDE_FROM_ABI void __tree_balance_after_insert(_NodePtr __root, _NodePtr __x) _NOEXCEPT {
+  _LIBCPP_ASSERT_INTERNAL(__root != nullptr, "Root of the tree shouldn't be null");
+  _LIBCPP_ASSERT_INTERNAL(__x != nullptr, "Can't attach null node to a leaf");
+  __x->__is_black_ = __x == __root;
+  while (__x != __root && !__x->__parent_unsafe()->__is_black_) {
+    // __x->__parent_ != __root because __x->__parent_->__is_black == false
+    if (std::__tree_is_left_child(__x->__parent_unsafe())) {
+      _NodePtr __y = __x->__parent_unsafe()->__parent_unsafe()->__right_;
+      if (__y != nullptr && !__y->__is_black_) {
+        __x              = __x->__parent_unsafe();
+        __x->__is_black_ = true;
+        __x              = __x->__parent_unsafe();
+        __x->__is_black_ = __x == __root;
+        __y->__is_black_ = true;
+      } else {
+        if (!std::__tree_is_left_child(__x)) {
+          __x = __x->__parent_unsafe();
+          std::__tree_left_rotate(__x);
+        }
+        __x              = __x->__parent_unsafe();
+        __x->__is_black_ = true;
+        __x              = __x->__parent_unsafe();
+        __x->__is_black_ = false;
+        std::__tree_right_rotate(__x);
+        break;
+      }
+    } else {
+      _NodePtr __y = __x->__parent_unsafe()->__parent_->__left_;
+      if (__y != nullptr && !__y->__is_black_) {
+        __x              = __x->__parent_unsafe();
+        __x->__is_black_ = true;
+        __x              = __x->__parent_unsafe();
+        __x->__is_black_ = __x == __root;
+        __y->__is_black_ = true;
+      } else {
+        if (std::__tree_is_left_child(__x)) {
+          __x = __x->__parent_unsafe();
+          std::__tree_right_rotate(__x);
+        }
+        __x              = __x->__parent_unsafe();
+        __x->__is_black_ = true;
+        __x              = __x->__parent_unsafe();
+        __x->__is_black_ = false;
+        std::__tree_left_rotate(__x);
+        break;
+      }
+    }
+  }
+}
+
+// Precondition:  __z == __root or == a direct or indirect child of __root.
+// Effects:  unlinks __z from the tree rooted at __root, rebalancing as needed.
+// Postcondition: __tree_invariant(end_node->__left_) == true && end_node->__left_
+//                nor any of its children refer to __z.  end_node->__left_
+//                may be different than the value passed in as __root.
+template <class _NodePtr>
+_LIBCPP_HIDE_FROM_ABI void __tree_remove(_NodePtr __root, _NodePtr __z) _NOEXCEPT {
+  _LIBCPP_ASSERT_INTERNAL(__root != nullptr, "Root node should not be null");
+  _LIBCPP_ASSERT_INTERNAL(__z != nullptr, "The node to remove should not be null");
+  _LIBCPP_ASSERT_INTERNAL(std::__tree_invariant(__root), "The tree invariants should hold");
+  // __z will be removed from the tree.  Client still needs to destruct/deallocate it
+  // __y is either __z, or if __z has two children, __tree_next(__z).
+  // __y will have at most one child.
+  // __y will be the initial hole in the tree (make the hole at a leaf)
+  _NodePtr __y = (__z->__left_ == nullptr || __z->__right_ == nullptr) ? __z : std::__tree_next(__z);
+  // __x is __y's possibly null single child
+  _NodePtr __x = __y->__left_ != nullptr ? __y->__left_ : __y->__right_;
+  // __w is __x's possibly null uncle (will become __x's sibling)
+  _NodePtr __w = nullptr;
+  // link __x to __y's parent, and find __w
+  if (__x != nullptr)
+    __x->__parent_ = __y->__parent_;
+  if (std::__tree_is_left_child(__y)) {
+    __y->__parent_->__left_ = __x;
+    if (__y != __root)
+      __w = __y->__parent_unsafe()->__right_;
+    else
+      __root = __x; // __w == nullptr
+  } else {
+    __y->__parent_unsafe()->__right_ = __x;
+    // __y can't be root if it is a right child
+    __w = __y->__parent_->__left_;
+  }
+  bool __removed_black = __y->__is_black_;
+  // If we didn't remove __z, do so now by splicing in __y for __z,
+  //    but copy __z's color.  This does not impact __x or __w.
+  if (__y != __z) {
+    // __z->__left_ != nulptr but __z->__right_ might == __x == nullptr
+    __y->__parent_ = __z->__parent_;
+    if (std::__tree_is_left_child(__z))
+      __y->__parent_->__left_ = __y;
+    else
+      __y->__parent_unsafe()->__right_ = __y;
+    __y->__left_ = __z->__left_;
+    __y->__left_->__set_parent(__y);
+    __y->__right_ = __z->__right_;
+    if (__y->__right_ != nullptr)
+      __y->__right_->__set_parent(__y);
+    __y->__is_black_ = __z->__is_black_;
+    if (__root == __z)
+      __root = __y;
+  }
+  // There is no need to rebalance if we removed a red, or if we removed
+  //     the last node.
+  if (__removed_black && __root != nullptr) {
+    // Rebalance:
+    // __x has an implicit black color (transferred from the removed __y)
+    //    associated with it, no matter what its color is.
+    // If __x is __root (in which case it can't be null), it is supposed
+    //    to be black anyway, and if it is doubly black, then the double
+    //    can just be ignored.
+    // If __x is red (in which case it can't be null), then it can absorb
+    //    the implicit black just by setting its color to black.
+    // Since __y was black and only had one child (which __x points to), __x
+    //   is either red with no children, else null, otherwise __y would have
+    //   different black heights under left and right pointers.
+    // if (__x == __root || __x != nullptr && !__x->__is_black_)
+    if (__x != nullptr)
+      __x->__is_black_ = true;
+    else {
+      //  Else __x isn't root, and is "doubly black", even though it may
+      //     be null.  __w can not be null here, else the parent would
+      //     see a black height >= 2 on the __x side and a black height
+      //     of 1 on the __w side (__w must be a non-null black or a red
+      //     with a non-null black child).
+      while (true) {
+        if (!std::__tree_is_left_child(__w)) // if x is left child
+        {
+          if (!__w->__is_black_) {
+            __w->__is_black_                    = true;
+            __w->__parent_unsafe()->__is_black_ = false;
+            std::__tree_left_rotate(__w->__parent_unsafe());
+            // __x is still valid
+            // reset __root only if necessary
+            if (__root == __w->__left_)
+              __root = __w;
+            // reset sibling, and it still can't be null
+            __w = __w->__left_->__right_;
+          }
+          // __w->__is_black_ is now true, __w may have null children
+          if ((__w->__left_ == nullptr || __w->__left_->__is_black_) &&
+              (__w->__right_ == nullptr || __w->__right_->__is_black_)) {
+            __w->__is_black_ = false;
+            __x              = __w->__parent_unsafe();
+            // __x can no longer be null
+            if (__x == __root || !__x->__is_black_) {
+              __x->__is_black_ = true;
+              break;
+            }
+            // reset sibling, and it still can't be null
+            __w = std::__tree_is_left_child(__x) ? __x->__parent_unsafe()->__right_ : __x->__parent_->__left_;
+            // continue;
+          } else // __w has a red child
+          {
+            if (__w->__right_ == nullptr || __w->__right_->__is_black_) {
+              // __w left child is non-null and red
+              __w->__left_->__is_black_ = true;
+              __w->__is_black_          = false;
+              std::__tree_right_rotate(__w);
+              // __w is known not to be root, so root hasn't changed
+              // reset sibling, and it still can't be null
+              __w = __w->__parent_unsafe();
+            }
+            // __w has a right red child, left child may be null
+            __w->__is_black_                    = __w->__parent_unsafe()->__is_black_;
+            __w->__parent_unsafe()->__is_black_ = true;
+            __w->__right_->__is_black_          = true;
+            std::__tree_left_rotate(__w->__parent_unsafe());
+            break;
+          }
+        } else {
+          if (!__w->__is_black_) {
+            __w->__is_black_                    = true;
+            __w->__parent_unsafe()->__is_black_ = false;
+            std::__tree_right_rotate(__w->__parent_unsafe());
+            // __x is still valid
+            // reset __root only if necessary
+            if (__root == __w->__right_)
+              __root = __w;
+            // reset sibling, and it still can't be null
+            __w = __w->__right_->__left_;
+          }
+          // __w->__is_black_ is now true, __w may have null children
+          if ((__w->__left_ == nullptr || __w->__left_->__is_black_) &&
+              (__w->__right_ == nullptr || __w->__right_->__is_black_)) {
+            __w->__is_black_ = false;
+            __x              = __w->__parent_unsafe();
+            // __x can no longer be null
+            if (!__x->__is_black_ || __x == __root) {
+              __x->__is_black_ = true;
+              break;
+            }
+            // reset sibling, and it still can't be null
+            __w = std::__tree_is_left_child(__x) ? __x->__parent_unsafe()->__right_ : __x->__parent_->__left_;
+            // continue;
+          } else // __w has a red child
+          {
+            if (__w->__left_ == nullptr || __w->__left_->__is_black_) {
+              // __w right child is non-null and red
+              __w->__right_->__is_black_ = true;
+              __w->__is_black_           = false;
+              std::__tree_left_rotate(__w);
+              // __w is known not to be root, so root hasn't changed
+              // reset sibling, and it still can't be null
+              __w = __w->__parent_unsafe();
+            }
+            // __w has a left red child, right child may be null
+            __w->__is_black_                    = __w->__parent_unsafe()->__is_black_;
+            __w->__parent_unsafe()->__is_black_ = true;
+            __w->__left_->__is_black_           = true;
+            std::__tree_right_rotate(__w->__parent_unsafe());
+            break;
+          }
+        }
+      }
+    }
+  }
+}
+
+// node traits
+
+template <class _Tp>
+struct __is_tree_value_type_imp : false_type {};
+
+template <class _Key, class _Value>
+struct __is_tree_value_type_imp<__value_type<_Key, _Value> > : true_type {};
+
+template <class... _Args>
+struct __is_tree_value_type : false_type {};
+
+template <class _One>
+struct __is_tree_value_type<_One> : __is_tree_value_type_imp<__remove_cvref_t<_One> > {};
+
+template <class _Tp>
+struct __tree_key_value_types {
+  typedef _Tp key_type;
+  typedef _Tp __node_value_type;
+  typedef _Tp __container_value_type;
+  static const bool __is_map = false;
+
+  _LIBCPP_HIDE_FROM_ABI static key_type const& __get_key(_Tp const& __v) { return __v; }
+  _LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(__node_value_type const& __v) { return __v; }
+  _LIBCPP_HIDE_FROM_ABI static __container_value_type* __get_ptr(__node_value_type& __n) { return std::addressof(__n); }
+  _LIBCPP_HIDE_FROM_ABI static __container_value_type&& __move(__node_value_type& __v) { return std::move(__v); }
+};
+
+template <class _Key, class _Tp>
+struct __tree_key_value_types<__value_type<_Key, _Tp> > {
+  typedef _Key key_type;
+  typedef _Tp mapped_type;
+  typedef __value_type<_Key, _Tp> __node_value_type;
+  typedef pair<const _Key, _Tp> __container_value_type;
+  typedef __container_value_type __map_value_type;
+  static const bool __is_map = true;
+
+  _LIBCPP_HIDE_FROM_ABI static key_type const& __get_key(__node_value_type const& __t) {
+    return __t.__get_value().first;
+  }
+
+  template <class _Up, __enable_if_t<__is_same_uncvref<_Up, __container_value_type>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI static key_type const& __get_key(_Up& __t) {
+    return __t.first;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(__node_value_type const& __t) {
+    return __t.__get_value();
+  }
+
+  template <class _Up, __enable_if_t<__is_same_uncvref<_Up, __container_value_type>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(_Up& __t) {
+    return __t;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static __container_value_type* __get_ptr(__node_value_type& __n) {
+    return std::addressof(__n.__get_value());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static pair<key_type&&, mapped_type&&> __move(__node_value_type& __v) { return __v.__move(); }
+};
+
+template <class _VoidPtr>
+struct __tree_node_base_types {
+  typedef _VoidPtr __void_pointer;
+
+  typedef __tree_node_base<__void_pointer> __node_base_type;
+  typedef __rebind_pointer_t<_VoidPtr, __node_base_type> __node_base_pointer;
+
+  typedef __tree_end_node<__node_base_pointer> __end_node_type;
+  typedef __rebind_pointer_t<_VoidPtr, __end_node_type> __end_node_pointer;
+#if defined(_LIBCPP_ABI_TREE_REMOVE_NODE_POINTER_UB)
+  typedef __end_node_pointer __parent_pointer;
+#else
+  typedef __conditional_t< is_pointer<__end_node_pointer>::value, __end_node_pointer, __node_base_pointer>
+      __parent_pointer;
+#endif
+
+private:
+  static_assert(is_same<typename pointer_traits<_VoidPtr>::element_type, void>::value,
+                "_VoidPtr does not point to unqualified void type");
+};
+
+template <class _Tp, class _AllocPtr, class _KVTypes = __tree_key_value_types<_Tp>, bool = _KVTypes::__is_map>
+struct __tree_map_pointer_types {};
+
+template <class _Tp, class _AllocPtr, class _KVTypes>
+struct __tree_map_pointer_types<_Tp, _AllocPtr, _KVTypes, true> {
+  typedef typename _KVTypes::__map_value_type _Mv;
+  typedef __rebind_pointer_t<_AllocPtr, _Mv> __map_value_type_pointer;
+  typedef __rebind_pointer_t<_AllocPtr, const _Mv> __const_map_value_type_pointer;
+};
+
+template <class _NodePtr, class _NodeT = typename pointer_traits<_NodePtr>::element_type>
+struct __tree_node_types;
+
+template <class _NodePtr, class _Tp, class _VoidPtr>
+struct __tree_node_types<_NodePtr, __tree_node<_Tp, _VoidPtr> >
+    : public __tree_node_base_types<_VoidPtr>, __tree_key_value_types<_Tp>, __tree_map_pointer_types<_Tp, _VoidPtr> {
+  typedef __tree_node_base_types<_VoidPtr> __base;
+  typedef __tree_key_value_types<_Tp> __key_base;
+  typedef __tree_map_pointer_types<_Tp, _VoidPtr> __map_pointer_base;
+
+public:
+  typedef typename pointer_traits<_NodePtr>::element_type __node_type;
+  typedef _NodePtr __node_pointer;
+
+  typedef _Tp __node_value_type;
+  typedef __rebind_pointer_t<_VoidPtr, __node_value_type> __node_value_type_pointer;
+  typedef __rebind_pointer_t<_VoidPtr, const __node_value_type> __const_node_value_type_pointer;
+#if defined(_LIBCPP_ABI_TREE_REMOVE_NODE_POINTER_UB)
+  typedef typename __base::__end_node_pointer __iter_pointer;
+#else
+  typedef __conditional_t< is_pointer<__node_pointer>::value, typename __base::__end_node_pointer, __node_pointer>
+      __iter_pointer;
+#endif
+
+private:
+  static_assert(!is_const<__node_type>::value, "_NodePtr should never be a pointer to const");
+  static_assert(is_same<__rebind_pointer_t<_VoidPtr, __node_type>, _NodePtr>::value,
+                "_VoidPtr does not rebind to _NodePtr.");
+};
+
+template <class _ValueTp, class _VoidPtr>
+struct __make_tree_node_types {
+  typedef __rebind_pointer_t<_VoidPtr, __tree_node<_ValueTp, _VoidPtr> > _NodePtr;
+  typedef __tree_node_types<_NodePtr> type;
+};
+
+// node
+
+template <class _Pointer>
+class __tree_end_node {
+public:
+  typedef _Pointer pointer;
+  pointer __left_;
+
+  _LIBCPP_HIDE_FROM_ABI __tree_end_node() _NOEXCEPT : __left_() {}
+};
+
+template <class _VoidPtr>
+class _LIBCPP_STANDALONE_DEBUG __tree_node_base : public __tree_node_base_types<_VoidPtr>::__end_node_type {
+  typedef __tree_node_base_types<_VoidPtr> _NodeBaseTypes;
+
+public:
+  typedef typename _NodeBaseTypes::__node_base_pointer pointer;
+  typedef typename _NodeBaseTypes::__parent_pointer __parent_pointer;
+
+  pointer __right_;
+  __parent_pointer __parent_;
+  bool __is_black_;
+
+  _LIBCPP_HIDE_FROM_ABI pointer __parent_unsafe() const { return static_cast<pointer>(__parent_); }
+
+  _LIBCPP_HIDE_FROM_ABI void __set_parent(pointer __p) { __parent_ = static_cast<__parent_pointer>(__p); }
+
+  ~__tree_node_base()                                  = delete;
+  __tree_node_base(__tree_node_base const&)            = delete;
+  __tree_node_base& operator=(__tree_node_base const&) = delete;
+};
+
+template <class _Tp, class _VoidPtr>
+class _LIBCPP_STANDALONE_DEBUG __tree_node : public __tree_node_base<_VoidPtr> {
+public:
+  typedef _Tp __node_value_type;
+
+  __node_value_type __value_;
+
+  _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; }
+
+  ~__tree_node()                             = delete;
+  __tree_node(__tree_node const&)            = delete;
+  __tree_node& operator=(__tree_node const&) = delete;
+};
+
+template <class _Allocator>
+class __tree_node_destructor {
+  typedef _Allocator allocator_type;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+
+public:
+  typedef typename __alloc_traits::pointer pointer;
+
+private:
+  typedef __tree_node_types<pointer> _NodeTypes;
+  allocator_type& __na_;
+
+public:
+  bool __value_constructed;
+
+  _LIBCPP_HIDE_FROM_ABI __tree_node_destructor(const __tree_node_destructor&) = default;
+  __tree_node_destructor& operator=(const __tree_node_destructor&)            = delete;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __tree_node_destructor(allocator_type& __na, bool __val = false) _NOEXCEPT
+      : __na_(__na),
+        __value_constructed(__val) {}
+
+  _LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) _NOEXCEPT {
+    if (__value_constructed)
+      __alloc_traits::destroy(__na_, _NodeTypes::__get_ptr(__p->__value_));
+    if (__p)
+      __alloc_traits::deallocate(__na_, __p, 1);
+  }
+
+  template <class>
+  friend class __map_node_destructor;
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _NodeType, class _Alloc>
+struct __generic_container_node_destructor;
+template <class _Tp, class _VoidPtr, class _Alloc>
+struct __generic_container_node_destructor<__tree_node<_Tp, _VoidPtr>, _Alloc> : __tree_node_destructor<_Alloc> {
+  using __tree_node_destructor<_Alloc>::__tree_node_destructor;
+};
+#endif
+
+template <class _Tp, class _NodePtr, class _DiffType>
+class _LIBCPP_TEMPLATE_VIS __tree_iterator {
+  typedef __tree_node_types<_NodePtr> _NodeTypes;
+  typedef _NodePtr __node_pointer;
+  typedef typename _NodeTypes::__node_base_pointer __node_base_pointer;
+  typedef typename _NodeTypes::__end_node_pointer __end_node_pointer;
+  typedef typename _NodeTypes::__iter_pointer __iter_pointer;
+  typedef pointer_traits<__node_pointer> __pointer_traits;
+
+  __iter_pointer __ptr_;
+
+public:
+  typedef bidirectional_iterator_tag iterator_category;
+  typedef _Tp value_type;
+  typedef _DiffType difference_type;
+  typedef value_type& reference;
+  typedef typename _NodeTypes::__node_value_type_pointer pointer;
+
+  _LIBCPP_HIDE_FROM_ABI __tree_iterator() _NOEXCEPT
+#if _LIBCPP_STD_VER >= 14
+      : __ptr_(nullptr)
+#endif
+  {
+  }
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __get_np()->__value_; }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return pointer_traits<pointer>::pointer_to(__get_np()->__value_); }
+
+  _LIBCPP_HIDE_FROM_ABI __tree_iterator& operator++() {
+    __ptr_ = static_cast<__iter_pointer>(
+        std::__tree_next_iter<__end_node_pointer>(static_cast<__node_base_pointer>(__ptr_)));
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __tree_iterator operator++(int) {
+    __tree_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __tree_iterator& operator--() {
+    __ptr_ = static_cast<__iter_pointer>(
+        std::__tree_prev_iter<__node_base_pointer>(static_cast<__end_node_pointer>(__ptr_)));
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __tree_iterator operator--(int) {
+    __tree_iterator __t(*this);
+    --(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __tree_iterator& __x, const __tree_iterator& __y) {
+    return __x.__ptr_ == __y.__ptr_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __tree_iterator& __x, const __tree_iterator& __y) {
+    return !(__x == __y);
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI explicit __tree_iterator(__node_pointer __p) _NOEXCEPT : __ptr_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI explicit __tree_iterator(__end_node_pointer __p) _NOEXCEPT : __ptr_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI __node_pointer __get_np() const { return static_cast<__node_pointer>(__ptr_); }
+  template <class, class, class>
+  friend class __tree;
+  template <class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS __tree_const_iterator;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __map_iterator;
+  template <class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS map;
+  template <class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS multimap;
+  template <class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS set;
+  template <class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS multiset;
+};
+
+template <class _Tp, class _NodePtr, class _DiffType>
+class _LIBCPP_TEMPLATE_VIS __tree_const_iterator {
+  typedef __tree_node_types<_NodePtr> _NodeTypes;
+  typedef typename _NodeTypes::__node_pointer __node_pointer;
+  typedef typename _NodeTypes::__node_base_pointer __node_base_pointer;
+  typedef typename _NodeTypes::__end_node_pointer __end_node_pointer;
+  typedef typename _NodeTypes::__iter_pointer __iter_pointer;
+  typedef pointer_traits<__node_pointer> __pointer_traits;
+
+  __iter_pointer __ptr_;
+
+public:
+  typedef bidirectional_iterator_tag iterator_category;
+  typedef _Tp value_type;
+  typedef _DiffType difference_type;
+  typedef const value_type& reference;
+  typedef typename _NodeTypes::__const_node_value_type_pointer pointer;
+
+  _LIBCPP_HIDE_FROM_ABI __tree_const_iterator() _NOEXCEPT
+#if _LIBCPP_STD_VER >= 14
+      : __ptr_(nullptr)
+#endif
+  {
+  }
+
+private:
+  typedef __tree_iterator<value_type, __node_pointer, difference_type> __non_const_iterator;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __tree_const_iterator(__non_const_iterator __p) _NOEXCEPT : __ptr_(__p.__ptr_) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __get_np()->__value_; }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return pointer_traits<pointer>::pointer_to(__get_np()->__value_); }
+
+  _LIBCPP_HIDE_FROM_ABI __tree_const_iterator& operator++() {
+    __ptr_ = static_cast<__iter_pointer>(
+        std::__tree_next_iter<__end_node_pointer>(static_cast<__node_base_pointer>(__ptr_)));
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __tree_const_iterator operator++(int) {
+    __tree_const_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __tree_const_iterator& operator--() {
+    __ptr_ = static_cast<__iter_pointer>(
+        std::__tree_prev_iter<__node_base_pointer>(static_cast<__end_node_pointer>(__ptr_)));
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __tree_const_iterator operator--(int) {
+    __tree_const_iterator __t(*this);
+    --(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __tree_const_iterator& __x, const __tree_const_iterator& __y) {
+    return __x.__ptr_ == __y.__ptr_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __tree_const_iterator& __x, const __tree_const_iterator& __y) {
+    return !(__x == __y);
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI explicit __tree_const_iterator(__node_pointer __p) _NOEXCEPT : __ptr_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI explicit __tree_const_iterator(__end_node_pointer __p) _NOEXCEPT : __ptr_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI __node_pointer __get_np() const { return static_cast<__node_pointer>(__ptr_); }
+
+  template <class, class, class>
+  friend class __tree;
+  template <class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS map;
+  template <class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS multimap;
+  template <class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS set;
+  template <class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS multiset;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __map_const_iterator;
+};
+
+template <class _Tp, class _Compare>
+#ifndef _LIBCPP_CXX03_LANG
+_LIBCPP_DIAGNOSE_WARNING(!__invokable<_Compare const&, _Tp const&, _Tp const&>::value,
+                         "the specified comparator type does not provide a viable const call operator")
+#endif
+int __diagnose_non_const_comparator();
+
+template <class _Tp, class _Compare, class _Allocator>
+class __tree {
+public:
+  typedef _Tp value_type;
+  typedef _Compare value_compare;
+  typedef _Allocator allocator_type;
+
+private:
+  typedef allocator_traits<allocator_type> __alloc_traits;
+  typedef typename __make_tree_node_types<value_type, typename __alloc_traits::void_pointer>::type _NodeTypes;
+  typedef typename _NodeTypes::key_type key_type;
+
+public:
+  typedef typename _NodeTypes::__node_value_type __node_value_type;
+  typedef typename _NodeTypes::__container_value_type __container_value_type;
+
+  typedef typename __alloc_traits::pointer pointer;
+  typedef typename __alloc_traits::const_pointer const_pointer;
+  typedef typename __alloc_traits::size_type size_type;
+  typedef typename __alloc_traits::difference_type difference_type;
+
+public:
+  typedef typename _NodeTypes::__void_pointer __void_pointer;
+
+  typedef typename _NodeTypes::__node_type __node;
+  typedef typename _NodeTypes::__node_pointer __node_pointer;
+
+  typedef typename _NodeTypes::__node_base_type __node_base;
+  typedef typename _NodeTypes::__node_base_pointer __node_base_pointer;
+
+  typedef typename _NodeTypes::__end_node_type __end_node_t;
+  typedef typename _NodeTypes::__end_node_pointer __end_node_ptr;
+
+  typedef typename _NodeTypes::__parent_pointer __parent_pointer;
+  typedef typename _NodeTypes::__iter_pointer __iter_pointer;
+
+  typedef __rebind_alloc<__alloc_traits, __node> __node_allocator;
+  typedef allocator_traits<__node_allocator> __node_traits;
+
+private:
+  // check for sane allocator pointer rebinding semantics. Rebinding the
+  // allocator for a new pointer type should be exactly the same as rebinding
+  // the pointer using 'pointer_traits'.
+  static_assert(is_same<__node_pointer, typename __node_traits::pointer>::value,
+                "Allocator does not rebind pointers in a sane manner.");
+  typedef __rebind_alloc<__node_traits, __node_base> __node_base_allocator;
+  typedef allocator_traits<__node_base_allocator> __node_base_traits;
+  static_assert(is_same<__node_base_pointer, typename __node_base_traits::pointer>::value,
+                "Allocator does not rebind pointers in a sane manner.");
+
+private:
+  __iter_pointer __begin_node_;
+  __compressed_pair<__end_node_t, __node_allocator> __pair1_;
+  __compressed_pair<size_type, value_compare> __pair3_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __iter_pointer __end_node() _NOEXCEPT {
+    return static_cast<__iter_pointer>(pointer_traits<__end_node_ptr>::pointer_to(__pair1_.first()));
+  }
+  _LIBCPP_HIDE_FROM_ABI __iter_pointer __end_node() const _NOEXCEPT {
+    return static_cast<__iter_pointer>(
+        pointer_traits<__end_node_ptr>::pointer_to(const_cast<__end_node_t&>(__pair1_.first())));
+  }
+  _LIBCPP_HIDE_FROM_ABI __node_allocator& __node_alloc() _NOEXCEPT { return __pair1_.second(); }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI const __node_allocator& __node_alloc() const _NOEXCEPT { return __pair1_.second(); }
+  _LIBCPP_HIDE_FROM_ABI __iter_pointer& __begin_node() _NOEXCEPT { return __begin_node_; }
+  _LIBCPP_HIDE_FROM_ABI const __iter_pointer& __begin_node() const _NOEXCEPT { return __begin_node_; }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI allocator_type __alloc() const _NOEXCEPT { return allocator_type(__node_alloc()); }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI size_type& size() _NOEXCEPT { return __pair3_.first(); }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI const size_type& size() const _NOEXCEPT { return __pair3_.first(); }
+  _LIBCPP_HIDE_FROM_ABI value_compare& value_comp() _NOEXCEPT { return __pair3_.second(); }
+  _LIBCPP_HIDE_FROM_ABI const value_compare& value_comp() const _NOEXCEPT { return __pair3_.second(); }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __node_pointer __root() const _NOEXCEPT {
+    return static_cast<__node_pointer>(__end_node()->__left_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __node_base_pointer* __root_ptr() const _NOEXCEPT {
+    return std::addressof(__end_node()->__left_);
+  }
+
+  typedef __tree_iterator<value_type, __node_pointer, difference_type> iterator;
+  typedef __tree_const_iterator<value_type, __node_pointer, difference_type> const_iterator;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __tree(const value_compare& __comp) _NOEXCEPT_(
+      is_nothrow_default_constructible<__node_allocator>::value&& is_nothrow_copy_constructible<value_compare>::value);
+  _LIBCPP_HIDE_FROM_ABI explicit __tree(const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI __tree(const value_compare& __comp, const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI __tree(const __tree& __t);
+  _LIBCPP_HIDE_FROM_ABI __tree& operator=(const __tree& __t);
+  template <class _ForwardIterator>
+  _LIBCPP_HIDE_FROM_ABI void __assign_unique(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void __assign_multi(_InputIterator __first, _InputIterator __last);
+  _LIBCPP_HIDE_FROM_ABI __tree(__tree&& __t) _NOEXCEPT_(
+      is_nothrow_move_constructible<__node_allocator>::value&& is_nothrow_move_constructible<value_compare>::value);
+  _LIBCPP_HIDE_FROM_ABI __tree(__tree&& __t, const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI __tree& operator=(__tree&& __t) _NOEXCEPT_(
+      __node_traits::propagate_on_container_move_assignment::value&& is_nothrow_move_assignable<value_compare>::value&&
+          is_nothrow_move_assignable<__node_allocator>::value);
+  _LIBCPP_HIDE_FROM_ABI ~__tree();
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return iterator(__begin_node()); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return const_iterator(__begin_node()); }
+  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return iterator(__end_node()); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return const_iterator(__end_node()); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
+    return std::min<size_type>(__node_traits::max_size(__node_alloc()), numeric_limits<difference_type >::max());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI void swap(__tree& __t)
+#if _LIBCPP_STD_VER <= 11
+      _NOEXCEPT_(__is_nothrow_swappable_v<value_compare> &&
+                 (!__node_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__node_allocator>));
+#else
+      _NOEXCEPT_(__is_nothrow_swappable_v<value_compare>);
+#endif
+
+  template <class _Key, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_key_args(_Key const&, _Args&&... __args);
+  template <class _Key, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_hint_unique_key_args(const_iterator, _Key const&, _Args&&...);
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_impl(_Args&&... __args);
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator __emplace_hint_unique_impl(const_iterator __p, _Args&&... __args);
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator __emplace_multi(_Args&&... __args);
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator __emplace_hint_multi(const_iterator __p, _Args&&... __args);
+
+  template <class _Pp>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique(_Pp&& __x) {
+    return __emplace_unique_extract_key(std::forward<_Pp>(__x), __can_extract_key<_Pp, key_type>());
+  }
+
+  template <class _First,
+            class _Second,
+            __enable_if_t<__can_extract_map_key<_First, key_type, __container_value_type>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique(_First&& __f, _Second&& __s) {
+    return __emplace_unique_key_args(__f, std::forward<_First>(__f), std::forward<_Second>(__s));
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique(_Args&&... __args) {
+    return __emplace_unique_impl(std::forward<_Args>(__args)...);
+  }
+
+  template <class _Pp>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_extract_key(_Pp&& __x, __extract_key_fail_tag) {
+    return __emplace_unique_impl(std::forward<_Pp>(__x));
+  }
+
+  template <class _Pp>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_extract_key(_Pp&& __x, __extract_key_self_tag) {
+    return __emplace_unique_key_args(__x, std::forward<_Pp>(__x));
+  }
+
+  template <class _Pp>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_extract_key(_Pp&& __x, __extract_key_first_tag) {
+    return __emplace_unique_key_args(__x.first, std::forward<_Pp>(__x));
+  }
+
+  template <class _Pp>
+  _LIBCPP_HIDE_FROM_ABI iterator __emplace_hint_unique(const_iterator __p, _Pp&& __x) {
+    return __emplace_hint_unique_extract_key(__p, std::forward<_Pp>(__x), __can_extract_key<_Pp, key_type>());
+  }
+
+  template <class _First,
+            class _Second,
+            __enable_if_t<__can_extract_map_key<_First, key_type, __container_value_type>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator __emplace_hint_unique(const_iterator __p, _First&& __f, _Second&& __s) {
+    return __emplace_hint_unique_key_args(__p, __f, std::forward<_First>(__f), std::forward<_Second>(__s)).first;
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator __emplace_hint_unique(const_iterator __p, _Args&&... __args) {
+    return __emplace_hint_unique_impl(__p, std::forward<_Args>(__args)...);
+  }
+
+  template <class _Pp>
+  _LIBCPP_HIDE_FROM_ABI iterator
+  __emplace_hint_unique_extract_key(const_iterator __p, _Pp&& __x, __extract_key_fail_tag) {
+    return __emplace_hint_unique_impl(__p, std::forward<_Pp>(__x));
+  }
+
+  template <class _Pp>
+  _LIBCPP_HIDE_FROM_ABI iterator
+  __emplace_hint_unique_extract_key(const_iterator __p, _Pp&& __x, __extract_key_self_tag) {
+    return __emplace_hint_unique_key_args(__p, __x, std::forward<_Pp>(__x)).first;
+  }
+
+  template <class _Pp>
+  _LIBCPP_HIDE_FROM_ABI iterator
+  __emplace_hint_unique_extract_key(const_iterator __p, _Pp&& __x, __extract_key_first_tag) {
+    return __emplace_hint_unique_key_args(__p, __x.first, std::forward<_Pp>(__x)).first;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __insert_unique(const __container_value_type& __v) {
+    return __emplace_unique_key_args(_NodeTypes::__get_key(__v), __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator __insert_unique(const_iterator __p, const __container_value_type& __v) {
+    return __emplace_hint_unique_key_args(__p, _NodeTypes::__get_key(__v), __v).first;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __insert_unique(__container_value_type&& __v) {
+    return __emplace_unique_key_args(_NodeTypes::__get_key(__v), std::move(__v));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator __insert_unique(const_iterator __p, __container_value_type&& __v) {
+    return __emplace_hint_unique_key_args(__p, _NodeTypes::__get_key(__v), std::move(__v)).first;
+  }
+
+  template <class _Vp, __enable_if_t<!is_same<__remove_const_ref_t<_Vp>, __container_value_type>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __insert_unique(_Vp&& __v) {
+    return __emplace_unique(std::forward<_Vp>(__v));
+  }
+
+  template <class _Vp, __enable_if_t<!is_same<__remove_const_ref_t<_Vp>, __container_value_type>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator __insert_unique(const_iterator __p, _Vp&& __v) {
+    return __emplace_hint_unique(__p, std::forward<_Vp>(__v));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator __insert_multi(__container_value_type&& __v) {
+    return __emplace_multi(std::move(__v));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator __insert_multi(const_iterator __p, __container_value_type&& __v) {
+    return __emplace_hint_multi(__p, std::move(__v));
+  }
+
+  template <class _Vp>
+  _LIBCPP_HIDE_FROM_ABI iterator __insert_multi(_Vp&& __v) {
+    return __emplace_multi(std::forward<_Vp>(__v));
+  }
+
+  template <class _Vp>
+  _LIBCPP_HIDE_FROM_ABI iterator __insert_multi(const_iterator __p, _Vp&& __v) {
+    return __emplace_hint_multi(__p, std::forward<_Vp>(__v));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool>
+  __node_assign_unique(const __container_value_type& __v, __node_pointer __dest);
+
+  _LIBCPP_HIDE_FROM_ABI iterator __node_insert_multi(__node_pointer __nd);
+  _LIBCPP_HIDE_FROM_ABI iterator __node_insert_multi(const_iterator __p, __node_pointer __nd);
+
+  _LIBCPP_HIDE_FROM_ABI iterator __remove_node_pointer(__node_pointer) _NOEXCEPT;
+
+#if _LIBCPP_STD_VER >= 17
+  template <class _NodeHandle, class _InsertReturnType>
+  _LIBCPP_HIDE_FROM_ABI _InsertReturnType __node_handle_insert_unique(_NodeHandle&&);
+  template <class _NodeHandle>
+  _LIBCPP_HIDE_FROM_ABI iterator __node_handle_insert_unique(const_iterator, _NodeHandle&&);
+  template <class _Tree>
+  _LIBCPP_HIDE_FROM_ABI void __node_handle_merge_unique(_Tree& __source);
+
+  template <class _NodeHandle>
+  _LIBCPP_HIDE_FROM_ABI iterator __node_handle_insert_multi(_NodeHandle&&);
+  template <class _NodeHandle>
+  _LIBCPP_HIDE_FROM_ABI iterator __node_handle_insert_multi(const_iterator, _NodeHandle&&);
+  template <class _Tree>
+  _LIBCPP_HIDE_FROM_ABI void __node_handle_merge_multi(_Tree& __source);
+
+  template <class _NodeHandle>
+  _LIBCPP_HIDE_FROM_ABI _NodeHandle __node_handle_extract(key_type const&);
+  template <class _NodeHandle>
+  _LIBCPP_HIDE_FROM_ABI _NodeHandle __node_handle_extract(const_iterator);
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p);
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __f, const_iterator __l);
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI size_type __erase_unique(const _Key& __k);
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI size_type __erase_multi(const _Key& __k);
+
+  _LIBCPP_HIDE_FROM_ABI void
+  __insert_node_at(__parent_pointer __parent, __node_base_pointer& __child, __node_base_pointer __new_node) _NOEXCEPT;
+
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI iterator find(const _Key& __v);
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _Key& __v) const;
+
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI size_type __count_unique(const _Key& __k) const;
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI size_type __count_multi(const _Key& __k) const;
+
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _Key& __v) {
+    return __lower_bound(__v, __root(), __end_node());
+  }
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI iterator __lower_bound(const _Key& __v, __node_pointer __root, __iter_pointer __result);
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _Key& __v) const {
+    return __lower_bound(__v, __root(), __end_node());
+  }
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI const_iterator
+  __lower_bound(const _Key& __v, __node_pointer __root, __iter_pointer __result) const;
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _Key& __v) {
+    return __upper_bound(__v, __root(), __end_node());
+  }
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI iterator __upper_bound(const _Key& __v, __node_pointer __root, __iter_pointer __result);
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _Key& __v) const {
+    return __upper_bound(__v, __root(), __end_node());
+  }
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI const_iterator
+  __upper_bound(const _Key& __v, __node_pointer __root, __iter_pointer __result) const;
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> __equal_range_unique(const _Key& __k);
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> __equal_range_unique(const _Key& __k) const;
+
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> __equal_range_multi(const _Key& __k);
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> __equal_range_multi(const _Key& __k) const;
+
+  typedef __tree_node_destructor<__node_allocator> _Dp;
+  typedef unique_ptr<__node, _Dp> __node_holder;
+
+  _LIBCPP_HIDE_FROM_ABI __node_holder remove(const_iterator __p) _NOEXCEPT;
+
+private:
+  _LIBCPP_HIDE_FROM_ABI __node_base_pointer& __find_leaf_low(__parent_pointer& __parent, const key_type& __v);
+  _LIBCPP_HIDE_FROM_ABI __node_base_pointer& __find_leaf_high(__parent_pointer& __parent, const key_type& __v);
+  _LIBCPP_HIDE_FROM_ABI __node_base_pointer&
+  __find_leaf(const_iterator __hint, __parent_pointer& __parent, const key_type& __v);
+  // FIXME: Make this function const qualified. Unfortunately doing so
+  // breaks existing code which uses non-const callable comparators.
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI __node_base_pointer& __find_equal(__parent_pointer& __parent, const _Key& __v);
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI __node_base_pointer& __find_equal(__parent_pointer& __parent, const _Key& __v) const {
+    return const_cast<__tree*>(this)->__find_equal(__parent, __v);
+  }
+  template <class _Key>
+  _LIBCPP_HIDE_FROM_ABI __node_base_pointer&
+  __find_equal(const_iterator __hint, __parent_pointer& __parent, __node_base_pointer& __dummy, const _Key& __v);
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI __node_holder __construct_node(_Args&&... __args);
+
+  // TODO: Make this _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDDEN void destroy(__node_pointer __nd) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __tree& __t) {
+    __copy_assign_alloc(__t, integral_constant<bool, __node_traits::propagate_on_container_copy_assignment::value>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __tree& __t, true_type) {
+    if (__node_alloc() != __t.__node_alloc())
+      clear();
+    __node_alloc() = __t.__node_alloc();
+  }
+  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __tree&, false_type) {}
+
+  _LIBCPP_HIDE_FROM_ABI void __move_assign(__tree& __t, false_type);
+  _LIBCPP_HIDE_FROM_ABI void __move_assign(__tree& __t, true_type) _NOEXCEPT_(
+      is_nothrow_move_assignable<value_compare>::value&& is_nothrow_move_assignable<__node_allocator>::value);
+
+  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__tree& __t)
+      _NOEXCEPT_(!__node_traits::propagate_on_container_move_assignment::value ||
+                 is_nothrow_move_assignable<__node_allocator>::value) {
+    __move_assign_alloc(__t, integral_constant<bool, __node_traits::propagate_on_container_move_assignment::value>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__tree& __t, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<__node_allocator>::value) {
+    __node_alloc() = std::move(__t.__node_alloc());
+  }
+  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__tree&, false_type) _NOEXCEPT {}
+
+  struct _DetachedTreeCache {
+    _LIBCPP_HIDE_FROM_ABI explicit _DetachedTreeCache(__tree* __t) _NOEXCEPT
+        : __t_(__t),
+          __cache_root_(__detach_from_tree(__t)) {
+      __advance();
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __node_pointer __get() const _NOEXCEPT { return __cache_elem_; }
+
+    _LIBCPP_HIDE_FROM_ABI void __advance() _NOEXCEPT {
+      __cache_elem_ = __cache_root_;
+      if (__cache_root_) {
+        __cache_root_ = __detach_next(__cache_root_);
+      }
+    }
+
+    _LIBCPP_HIDE_FROM_ABI ~_DetachedTreeCache() {
+      __t_->destroy(__cache_elem_);
+      if (__cache_root_) {
+        while (__cache_root_->__parent_ != nullptr)
+          __cache_root_ = static_cast<__node_pointer>(__cache_root_->__parent_);
+        __t_->destroy(__cache_root_);
+      }
+    }
+
+    _DetachedTreeCache(_DetachedTreeCache const&)            = delete;
+    _DetachedTreeCache& operator=(_DetachedTreeCache const&) = delete;
+
+  private:
+    _LIBCPP_HIDE_FROM_ABI static __node_pointer __detach_from_tree(__tree* __t) _NOEXCEPT;
+    _LIBCPP_HIDE_FROM_ABI static __node_pointer __detach_next(__node_pointer) _NOEXCEPT;
+
+    __tree* __t_;
+    __node_pointer __cache_root_;
+    __node_pointer __cache_elem_;
+  };
+
+  template <class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS map;
+  template <class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS multimap;
+};
+
+template <class _Tp, class _Compare, class _Allocator>
+__tree<_Tp, _Compare, _Allocator>::__tree(const value_compare& __comp) _NOEXCEPT_(
+    is_nothrow_default_constructible<__node_allocator>::value&& is_nothrow_copy_constructible<value_compare>::value)
+    : __pair3_(0, __comp) {
+  __begin_node() = __end_node();
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+__tree<_Tp, _Compare, _Allocator>::__tree(const allocator_type& __a)
+    : __begin_node_(__iter_pointer()),
+      __pair1_(__default_init_tag(), __node_allocator(__a)),
+      __pair3_(0, __default_init_tag()) {
+  __begin_node() = __end_node();
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+__tree<_Tp, _Compare, _Allocator>::__tree(const value_compare& __comp, const allocator_type& __a)
+    : __begin_node_(__iter_pointer()), __pair1_(__default_init_tag(), __node_allocator(__a)), __pair3_(0, __comp) {
+  __begin_node() = __end_node();
+}
+
+// Precondition:  size() != 0
+template <class _Tp, class _Compare, class _Allocator>
+typename __tree<_Tp, _Compare, _Allocator>::__node_pointer
+__tree<_Tp, _Compare, _Allocator>::_DetachedTreeCache::__detach_from_tree(__tree* __t) _NOEXCEPT {
+  __node_pointer __cache                = static_cast<__node_pointer>(__t->__begin_node());
+  __t->__begin_node()                   = __t->__end_node();
+  __t->__end_node()->__left_->__parent_ = nullptr;
+  __t->__end_node()->__left_            = nullptr;
+  __t->size()                           = 0;
+  // __cache->__left_ == nullptr
+  if (__cache->__right_ != nullptr)
+    __cache = static_cast<__node_pointer>(__cache->__right_);
+  // __cache->__left_ == nullptr
+  // __cache->__right_ == nullptr
+  return __cache;
+}
+
+// Precondition:  __cache != nullptr
+//    __cache->left_ == nullptr
+//    __cache->right_ == nullptr
+//    This is no longer a red-black tree
+template <class _Tp, class _Compare, class _Allocator>
+typename __tree<_Tp, _Compare, _Allocator>::__node_pointer
+__tree<_Tp, _Compare, _Allocator>::_DetachedTreeCache::__detach_next(__node_pointer __cache) _NOEXCEPT {
+  if (__cache->__parent_ == nullptr)
+    return nullptr;
+  if (std::__tree_is_left_child(static_cast<__node_base_pointer>(__cache))) {
+    __cache->__parent_->__left_ = nullptr;
+    __cache                     = static_cast<__node_pointer>(__cache->__parent_);
+    if (__cache->__right_ == nullptr)
+      return __cache;
+    return static_cast<__node_pointer>(std::__tree_leaf(__cache->__right_));
+  }
+  // __cache is right child
+  __cache->__parent_unsafe()->__right_ = nullptr;
+  __cache                              = static_cast<__node_pointer>(__cache->__parent_);
+  if (__cache->__left_ == nullptr)
+    return __cache;
+  return static_cast<__node_pointer>(std::__tree_leaf(__cache->__left_));
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+__tree<_Tp, _Compare, _Allocator>& __tree<_Tp, _Compare, _Allocator>::operator=(const __tree& __t) {
+  if (this != std::addressof(__t)) {
+    value_comp() = __t.value_comp();
+    __copy_assign_alloc(__t);
+    __assign_multi(__t.begin(), __t.end());
+  }
+  return *this;
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _ForwardIterator>
+void __tree<_Tp, _Compare, _Allocator>::__assign_unique(_ForwardIterator __first, _ForwardIterator __last) {
+  typedef iterator_traits<_ForwardIterator> _ITraits;
+  typedef typename _ITraits::value_type _ItValueType;
+  static_assert(is_same<_ItValueType, __container_value_type>::value,
+                "__assign_unique may only be called with the containers value type");
+  static_assert(
+      __has_forward_iterator_category<_ForwardIterator>::value, "__assign_unique requires a forward iterator");
+  if (size() != 0) {
+    _DetachedTreeCache __cache(this);
+    for (; __cache.__get() != nullptr && __first != __last; ++__first) {
+      if (__node_assign_unique(*__first, __cache.__get()).second)
+        __cache.__advance();
+    }
+  }
+  for (; __first != __last; ++__first)
+    __insert_unique(*__first);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _InputIterator>
+void __tree<_Tp, _Compare, _Allocator>::__assign_multi(_InputIterator __first, _InputIterator __last) {
+  typedef iterator_traits<_InputIterator> _ITraits;
+  typedef typename _ITraits::value_type _ItValueType;
+  static_assert(
+      (is_same<_ItValueType, __container_value_type>::value || is_same<_ItValueType, __node_value_type>::value),
+      "__assign_multi may only be called with the containers value type"
+      " or the nodes value type");
+  if (size() != 0) {
+    _DetachedTreeCache __cache(this);
+    for (; __cache.__get() && __first != __last; ++__first) {
+      __cache.__get()->__value_ = *__first;
+      __node_insert_multi(__cache.__get());
+      __cache.__advance();
+    }
+  }
+  for (; __first != __last; ++__first)
+    __insert_multi(_NodeTypes::__get_value(*__first));
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+__tree<_Tp, _Compare, _Allocator>::__tree(const __tree& __t)
+    : __begin_node_(__iter_pointer()),
+      __pair1_(__default_init_tag(), __node_traits::select_on_container_copy_construction(__t.__node_alloc())),
+      __pair3_(0, __t.value_comp()) {
+  __begin_node() = __end_node();
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+__tree<_Tp, _Compare, _Allocator>::__tree(__tree&& __t) _NOEXCEPT_(
+    is_nothrow_move_constructible<__node_allocator>::value&& is_nothrow_move_constructible<value_compare>::value)
+    : __begin_node_(std::move(__t.__begin_node_)),
+      __pair1_(std::move(__t.__pair1_)),
+      __pair3_(std::move(__t.__pair3_)) {
+  if (size() == 0)
+    __begin_node() = __end_node();
+  else {
+    __end_node()->__left_->__parent_ = static_cast<__parent_pointer>(__end_node());
+    __t.__begin_node()               = __t.__end_node();
+    __t.__end_node()->__left_        = nullptr;
+    __t.size()                       = 0;
+  }
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+__tree<_Tp, _Compare, _Allocator>::__tree(__tree&& __t, const allocator_type& __a)
+    : __pair1_(__default_init_tag(), __node_allocator(__a)), __pair3_(0, std::move(__t.value_comp())) {
+  if (__a == __t.__alloc()) {
+    if (__t.size() == 0)
+      __begin_node() = __end_node();
+    else {
+      __begin_node()                   = __t.__begin_node();
+      __end_node()->__left_            = __t.__end_node()->__left_;
+      __end_node()->__left_->__parent_ = static_cast<__parent_pointer>(__end_node());
+      size()                           = __t.size();
+      __t.__begin_node()               = __t.__end_node();
+      __t.__end_node()->__left_        = nullptr;
+      __t.size()                       = 0;
+    }
+  } else {
+    __begin_node() = __end_node();
+  }
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+void __tree<_Tp, _Compare, _Allocator>::__move_assign(__tree& __t, true_type)
+    _NOEXCEPT_(is_nothrow_move_assignable<value_compare>::value&& is_nothrow_move_assignable<__node_allocator>::value) {
+  destroy(static_cast<__node_pointer>(__end_node()->__left_));
+  __begin_node_    = __t.__begin_node_;
+  __pair1_.first() = __t.__pair1_.first();
+  __move_assign_alloc(__t);
+  __pair3_ = std::move(__t.__pair3_);
+  if (size() == 0)
+    __begin_node() = __end_node();
+  else {
+    __end_node()->__left_->__parent_ = static_cast<__parent_pointer>(__end_node());
+    __t.__begin_node()               = __t.__end_node();
+    __t.__end_node()->__left_        = nullptr;
+    __t.size()                       = 0;
+  }
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+void __tree<_Tp, _Compare, _Allocator>::__move_assign(__tree& __t, false_type) {
+  if (__node_alloc() == __t.__node_alloc())
+    __move_assign(__t, true_type());
+  else {
+    value_comp()       = std::move(__t.value_comp());
+    const_iterator __e = end();
+    if (size() != 0) {
+      _DetachedTreeCache __cache(this);
+      while (__cache.__get() != nullptr && __t.size() != 0) {
+        __cache.__get()->__value_ = std::move(__t.remove(__t.begin())->__value_);
+        __node_insert_multi(__cache.__get());
+        __cache.__advance();
+      }
+    }
+    while (__t.size() != 0)
+      __insert_multi(__e, _NodeTypes::__move(__t.remove(__t.begin())->__value_));
+  }
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+__tree<_Tp, _Compare, _Allocator>& __tree<_Tp, _Compare, _Allocator>::operator=(__tree&& __t) _NOEXCEPT_(
+    __node_traits::propagate_on_container_move_assignment::value&& is_nothrow_move_assignable<value_compare>::value&&
+        is_nothrow_move_assignable<__node_allocator>::value)
+
+{
+  __move_assign(__t, integral_constant<bool, __node_traits::propagate_on_container_move_assignment::value>());
+  return *this;
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+__tree<_Tp, _Compare, _Allocator>::~__tree() {
+  static_assert(is_copy_constructible<value_compare>::value, "Comparator must be copy-constructible.");
+  destroy(__root());
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+void __tree<_Tp, _Compare, _Allocator>::destroy(__node_pointer __nd) _NOEXCEPT {
+  if (__nd != nullptr) {
+    destroy(static_cast<__node_pointer>(__nd->__left_));
+    destroy(static_cast<__node_pointer>(__nd->__right_));
+    __node_allocator& __na = __node_alloc();
+    __node_traits::destroy(__na, _NodeTypes::__get_ptr(__nd->__value_));
+    __node_traits::deallocate(__na, __nd, 1);
+  }
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+void __tree<_Tp, _Compare, _Allocator>::swap(__tree& __t)
+#if _LIBCPP_STD_VER <= 11
+    _NOEXCEPT_(__is_nothrow_swappable_v<value_compare> &&
+               (!__node_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__node_allocator>))
+#else
+    _NOEXCEPT_(__is_nothrow_swappable_v<value_compare>)
+#endif
+{
+  using std::swap;
+  swap(__begin_node_, __t.__begin_node_);
+  swap(__pair1_.first(), __t.__pair1_.first());
+  std::__swap_allocator(__node_alloc(), __t.__node_alloc());
+  __pair3_.swap(__t.__pair3_);
+  if (size() == 0)
+    __begin_node() = __end_node();
+  else
+    __end_node()->__left_->__parent_ = static_cast<__parent_pointer>(__end_node());
+  if (__t.size() == 0)
+    __t.__begin_node() = __t.__end_node();
+  else
+    __t.__end_node()->__left_->__parent_ = static_cast<__parent_pointer>(__t.__end_node());
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+void __tree<_Tp, _Compare, _Allocator>::clear() _NOEXCEPT {
+  destroy(__root());
+  size()                = 0;
+  __begin_node()        = __end_node();
+  __end_node()->__left_ = nullptr;
+}
+
+// Find lower_bound place to insert
+// Set __parent to parent of null leaf
+// Return reference to null leaf
+template <class _Tp, class _Compare, class _Allocator>
+typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer&
+__tree<_Tp, _Compare, _Allocator>::__find_leaf_low(__parent_pointer& __parent, const key_type& __v) {
+  __node_pointer __nd = __root();
+  if (__nd != nullptr) {
+    while (true) {
+      if (value_comp()(__nd->__value_, __v)) {
+        if (__nd->__right_ != nullptr)
+          __nd = static_cast<__node_pointer>(__nd->__right_);
+        else {
+          __parent = static_cast<__parent_pointer>(__nd);
+          return __nd->__right_;
+        }
+      } else {
+        if (__nd->__left_ != nullptr)
+          __nd = static_cast<__node_pointer>(__nd->__left_);
+        else {
+          __parent = static_cast<__parent_pointer>(__nd);
+          return __parent->__left_;
+        }
+      }
+    }
+  }
+  __parent = static_cast<__parent_pointer>(__end_node());
+  return __parent->__left_;
+}
+
+// Find upper_bound place to insert
+// Set __parent to parent of null leaf
+// Return reference to null leaf
+template <class _Tp, class _Compare, class _Allocator>
+typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer&
+__tree<_Tp, _Compare, _Allocator>::__find_leaf_high(__parent_pointer& __parent, const key_type& __v) {
+  __node_pointer __nd = __root();
+  if (__nd != nullptr) {
+    while (true) {
+      if (value_comp()(__v, __nd->__value_)) {
+        if (__nd->__left_ != nullptr)
+          __nd = static_cast<__node_pointer>(__nd->__left_);
+        else {
+          __parent = static_cast<__parent_pointer>(__nd);
+          return __parent->__left_;
+        }
+      } else {
+        if (__nd->__right_ != nullptr)
+          __nd = static_cast<__node_pointer>(__nd->__right_);
+        else {
+          __parent = static_cast<__parent_pointer>(__nd);
+          return __nd->__right_;
+        }
+      }
+    }
+  }
+  __parent = static_cast<__parent_pointer>(__end_node());
+  return __parent->__left_;
+}
+
+// Find leaf place to insert closest to __hint
+// First check prior to __hint.
+// Next check after __hint.
+// Next do O(log N) search.
+// Set __parent to parent of null leaf
+// Return reference to null leaf
+template <class _Tp, class _Compare, class _Allocator>
+typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer&
+__tree<_Tp, _Compare, _Allocator>::__find_leaf(const_iterator __hint, __parent_pointer& __parent, const key_type& __v) {
+  if (__hint == end() || !value_comp()(*__hint, __v)) // check before
+  {
+    // __v <= *__hint
+    const_iterator __prior = __hint;
+    if (__prior == begin() || !value_comp()(__v, *--__prior)) {
+      // *prev(__hint) <= __v <= *__hint
+      if (__hint.__ptr_->__left_ == nullptr) {
+        __parent = static_cast<__parent_pointer>(__hint.__ptr_);
+        return __parent->__left_;
+      } else {
+        __parent = static_cast<__parent_pointer>(__prior.__ptr_);
+        return static_cast<__node_base_pointer>(__prior.__ptr_)->__right_;
+      }
+    }
+    // __v < *prev(__hint)
+    return __find_leaf_high(__parent, __v);
+  }
+  // else __v > *__hint
+  return __find_leaf_low(__parent, __v);
+}
+
+// Find place to insert if __v doesn't exist
+// Set __parent to parent of null leaf
+// Return reference to null leaf
+// If __v exists, set parent to node of __v and return reference to node of __v
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer&
+__tree<_Tp, _Compare, _Allocator>::__find_equal(__parent_pointer& __parent, const _Key& __v) {
+  __node_pointer __nd           = __root();
+  __node_base_pointer* __nd_ptr = __root_ptr();
+  if (__nd != nullptr) {
+    while (true) {
+      if (value_comp()(__v, __nd->__value_)) {
+        if (__nd->__left_ != nullptr) {
+          __nd_ptr = std::addressof(__nd->__left_);
+          __nd     = static_cast<__node_pointer>(__nd->__left_);
+        } else {
+          __parent = static_cast<__parent_pointer>(__nd);
+          return __parent->__left_;
+        }
+      } else if (value_comp()(__nd->__value_, __v)) {
+        if (__nd->__right_ != nullptr) {
+          __nd_ptr = std::addressof(__nd->__right_);
+          __nd     = static_cast<__node_pointer>(__nd->__right_);
+        } else {
+          __parent = static_cast<__parent_pointer>(__nd);
+          return __nd->__right_;
+        }
+      } else {
+        __parent = static_cast<__parent_pointer>(__nd);
+        return *__nd_ptr;
+      }
+    }
+  }
+  __parent = static_cast<__parent_pointer>(__end_node());
+  return __parent->__left_;
+}
+
+// Find place to insert if __v doesn't exist
+// First check prior to __hint.
+// Next check after __hint.
+// Next do O(log N) search.
+// Set __parent to parent of null leaf
+// Return reference to null leaf
+// If __v exists, set parent to node of __v and return reference to node of __v
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer& __tree<_Tp, _Compare, _Allocator>::__find_equal(
+    const_iterator __hint, __parent_pointer& __parent, __node_base_pointer& __dummy, const _Key& __v) {
+  if (__hint == end() || value_comp()(__v, *__hint)) // check before
+  {
+    // __v < *__hint
+    const_iterator __prior = __hint;
+    if (__prior == begin() || value_comp()(*--__prior, __v)) {
+      // *prev(__hint) < __v < *__hint
+      if (__hint.__ptr_->__left_ == nullptr) {
+        __parent = static_cast<__parent_pointer>(__hint.__ptr_);
+        return __parent->__left_;
+      } else {
+        __parent = static_cast<__parent_pointer>(__prior.__ptr_);
+        return static_cast<__node_base_pointer>(__prior.__ptr_)->__right_;
+      }
+    }
+    // __v <= *prev(__hint)
+    return __find_equal(__parent, __v);
+  } else if (value_comp()(*__hint, __v)) // check after
+  {
+    // *__hint < __v
+    const_iterator __next = std::next(__hint);
+    if (__next == end() || value_comp()(__v, *__next)) {
+      // *__hint < __v < *std::next(__hint)
+      if (__hint.__get_np()->__right_ == nullptr) {
+        __parent = static_cast<__parent_pointer>(__hint.__ptr_);
+        return static_cast<__node_base_pointer>(__hint.__ptr_)->__right_;
+      } else {
+        __parent = static_cast<__parent_pointer>(__next.__ptr_);
+        return __parent->__left_;
+      }
+    }
+    // *next(__hint) <= __v
+    return __find_equal(__parent, __v);
+  }
+  // else __v == *__hint
+  __parent = static_cast<__parent_pointer>(__hint.__ptr_);
+  __dummy  = static_cast<__node_base_pointer>(__hint.__ptr_);
+  return __dummy;
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+void __tree<_Tp, _Compare, _Allocator>::__insert_node_at(
+    __parent_pointer __parent, __node_base_pointer& __child, __node_base_pointer __new_node) _NOEXCEPT {
+  __new_node->__left_   = nullptr;
+  __new_node->__right_  = nullptr;
+  __new_node->__parent_ = __parent;
+  // __new_node->__is_black_ is initialized in __tree_balance_after_insert
+  __child = __new_node;
+  if (__begin_node()->__left_ != nullptr)
+    __begin_node() = static_cast<__iter_pointer>(__begin_node()->__left_);
+  std::__tree_balance_after_insert(__end_node()->__left_, __child);
+  ++size();
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key, class... _Args>
+pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, bool>
+__tree<_Tp, _Compare, _Allocator>::__emplace_unique_key_args(_Key const& __k, _Args&&... __args) {
+  __parent_pointer __parent;
+  __node_base_pointer& __child = __find_equal(__parent, __k);
+  __node_pointer __r           = static_cast<__node_pointer>(__child);
+  bool __inserted              = false;
+  if (__child == nullptr) {
+    __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
+    __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
+    __r        = __h.release();
+    __inserted = true;
+  }
+  return pair<iterator, bool>(iterator(__r), __inserted);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key, class... _Args>
+pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, bool>
+__tree<_Tp, _Compare, _Allocator>::__emplace_hint_unique_key_args(
+    const_iterator __p, _Key const& __k, _Args&&... __args) {
+  __parent_pointer __parent;
+  __node_base_pointer __dummy;
+  __node_base_pointer& __child = __find_equal(__p, __parent, __dummy, __k);
+  __node_pointer __r           = static_cast<__node_pointer>(__child);
+  bool __inserted              = false;
+  if (__child == nullptr) {
+    __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
+    __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
+    __r        = __h.release();
+    __inserted = true;
+  }
+  return pair<iterator, bool>(iterator(__r), __inserted);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class... _Args>
+typename __tree<_Tp, _Compare, _Allocator>::__node_holder
+__tree<_Tp, _Compare, _Allocator>::__construct_node(_Args&&... __args) {
+  static_assert(!__is_tree_value_type<_Args...>::value, "Cannot construct from __value_type");
+  __node_allocator& __na = __node_alloc();
+  __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
+  __node_traits::construct(__na, _NodeTypes::__get_ptr(__h->__value_), std::forward<_Args>(__args)...);
+  __h.get_deleter().__value_constructed = true;
+  return __h;
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class... _Args>
+pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, bool>
+__tree<_Tp, _Compare, _Allocator>::__emplace_unique_impl(_Args&&... __args) {
+  __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
+  __parent_pointer __parent;
+  __node_base_pointer& __child = __find_equal(__parent, __h->__value_);
+  __node_pointer __r           = static_cast<__node_pointer>(__child);
+  bool __inserted              = false;
+  if (__child == nullptr) {
+    __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
+    __r        = __h.release();
+    __inserted = true;
+  }
+  return pair<iterator, bool>(iterator(__r), __inserted);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class... _Args>
+typename __tree<_Tp, _Compare, _Allocator>::iterator
+__tree<_Tp, _Compare, _Allocator>::__emplace_hint_unique_impl(const_iterator __p, _Args&&... __args) {
+  __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
+  __parent_pointer __parent;
+  __node_base_pointer __dummy;
+  __node_base_pointer& __child = __find_equal(__p, __parent, __dummy, __h->__value_);
+  __node_pointer __r           = static_cast<__node_pointer>(__child);
+  if (__child == nullptr) {
+    __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
+    __r = __h.release();
+  }
+  return iterator(__r);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class... _Args>
+typename __tree<_Tp, _Compare, _Allocator>::iterator
+__tree<_Tp, _Compare, _Allocator>::__emplace_multi(_Args&&... __args) {
+  __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
+  __parent_pointer __parent;
+  __node_base_pointer& __child = __find_leaf_high(__parent, _NodeTypes::__get_key(__h->__value_));
+  __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
+  return iterator(static_cast<__node_pointer>(__h.release()));
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class... _Args>
+typename __tree<_Tp, _Compare, _Allocator>::iterator
+__tree<_Tp, _Compare, _Allocator>::__emplace_hint_multi(const_iterator __p, _Args&&... __args) {
+  __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
+  __parent_pointer __parent;
+  __node_base_pointer& __child = __find_leaf(__p, __parent, _NodeTypes::__get_key(__h->__value_));
+  __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
+  return iterator(static_cast<__node_pointer>(__h.release()));
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, bool>
+__tree<_Tp, _Compare, _Allocator>::__node_assign_unique(const __container_value_type& __v, __node_pointer __nd) {
+  __parent_pointer __parent;
+  __node_base_pointer& __child = __find_equal(__parent, _NodeTypes::__get_key(__v));
+  __node_pointer __r           = static_cast<__node_pointer>(__child);
+  bool __inserted              = false;
+  if (__child == nullptr) {
+    __nd->__value_ = __v;
+    __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd));
+    __r        = __nd;
+    __inserted = true;
+  }
+  return pair<iterator, bool>(iterator(__r), __inserted);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+typename __tree<_Tp, _Compare, _Allocator>::iterator
+__tree<_Tp, _Compare, _Allocator>::__node_insert_multi(__node_pointer __nd) {
+  __parent_pointer __parent;
+  __node_base_pointer& __child = __find_leaf_high(__parent, _NodeTypes::__get_key(__nd->__value_));
+  __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd));
+  return iterator(__nd);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+typename __tree<_Tp, _Compare, _Allocator>::iterator
+__tree<_Tp, _Compare, _Allocator>::__node_insert_multi(const_iterator __p, __node_pointer __nd) {
+  __parent_pointer __parent;
+  __node_base_pointer& __child = __find_leaf(__p, __parent, _NodeTypes::__get_key(__nd->__value_));
+  __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd));
+  return iterator(__nd);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+typename __tree<_Tp, _Compare, _Allocator>::iterator
+__tree<_Tp, _Compare, _Allocator>::__remove_node_pointer(__node_pointer __ptr) _NOEXCEPT {
+  iterator __r(__ptr);
+  ++__r;
+  if (__begin_node() == __ptr)
+    __begin_node() = __r.__ptr_;
+  --size();
+  std::__tree_remove(__end_node()->__left_, static_cast<__node_base_pointer>(__ptr));
+  return __r;
+}
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp, class _Compare, class _Allocator>
+template <class _NodeHandle, class _InsertReturnType>
+_LIBCPP_HIDE_FROM_ABI _InsertReturnType
+__tree<_Tp, _Compare, _Allocator>::__node_handle_insert_unique(_NodeHandle&& __nh) {
+  if (__nh.empty())
+    return _InsertReturnType{end(), false, _NodeHandle()};
+
+  __node_pointer __ptr = __nh.__ptr_;
+  __parent_pointer __parent;
+  __node_base_pointer& __child = __find_equal(__parent, __ptr->__value_);
+  if (__child != nullptr)
+    return _InsertReturnType{iterator(static_cast<__node_pointer>(__child)), false, std::move(__nh)};
+
+  __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__ptr));
+  __nh.__release_ptr();
+  return _InsertReturnType{iterator(__ptr), true, _NodeHandle()};
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _NodeHandle>
+_LIBCPP_HIDE_FROM_ABI typename __tree<_Tp, _Compare, _Allocator>::iterator
+__tree<_Tp, _Compare, _Allocator>::__node_handle_insert_unique(const_iterator __hint, _NodeHandle&& __nh) {
+  if (__nh.empty())
+    return end();
+
+  __node_pointer __ptr = __nh.__ptr_;
+  __parent_pointer __parent;
+  __node_base_pointer __dummy;
+  __node_base_pointer& __child = __find_equal(__hint, __parent, __dummy, __ptr->__value_);
+  __node_pointer __r           = static_cast<__node_pointer>(__child);
+  if (__child == nullptr) {
+    __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__ptr));
+    __r = __ptr;
+    __nh.__release_ptr();
+  }
+  return iterator(__r);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _NodeHandle>
+_LIBCPP_HIDE_FROM_ABI _NodeHandle __tree<_Tp, _Compare, _Allocator>::__node_handle_extract(key_type const& __key) {
+  iterator __it = find(__key);
+  if (__it == end())
+    return _NodeHandle();
+  return __node_handle_extract<_NodeHandle>(__it);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _NodeHandle>
+_LIBCPP_HIDE_FROM_ABI _NodeHandle __tree<_Tp, _Compare, _Allocator>::__node_handle_extract(const_iterator __p) {
+  __node_pointer __np = __p.__get_np();
+  __remove_node_pointer(__np);
+  return _NodeHandle(__np, __alloc());
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Tree>
+_LIBCPP_HIDE_FROM_ABI void __tree<_Tp, _Compare, _Allocator>::__node_handle_merge_unique(_Tree& __source) {
+  static_assert(is_same<typename _Tree::__node_pointer, __node_pointer>::value, "");
+
+  for (typename _Tree::iterator __i = __source.begin(); __i != __source.end();) {
+    __node_pointer __src_ptr = __i.__get_np();
+    __parent_pointer __parent;
+    __node_base_pointer& __child = __find_equal(__parent, _NodeTypes::__get_key(__src_ptr->__value_));
+    ++__i;
+    if (__child != nullptr)
+      continue;
+    __source.__remove_node_pointer(__src_ptr);
+    __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__src_ptr));
+  }
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _NodeHandle>
+_LIBCPP_HIDE_FROM_ABI typename __tree<_Tp, _Compare, _Allocator>::iterator
+__tree<_Tp, _Compare, _Allocator>::__node_handle_insert_multi(_NodeHandle&& __nh) {
+  if (__nh.empty())
+    return end();
+  __node_pointer __ptr = __nh.__ptr_;
+  __parent_pointer __parent;
+  __node_base_pointer& __child = __find_leaf_high(__parent, _NodeTypes::__get_key(__ptr->__value_));
+  __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__ptr));
+  __nh.__release_ptr();
+  return iterator(__ptr);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _NodeHandle>
+_LIBCPP_HIDE_FROM_ABI typename __tree<_Tp, _Compare, _Allocator>::iterator
+__tree<_Tp, _Compare, _Allocator>::__node_handle_insert_multi(const_iterator __hint, _NodeHandle&& __nh) {
+  if (__nh.empty())
+    return end();
+
+  __node_pointer __ptr = __nh.__ptr_;
+  __parent_pointer __parent;
+  __node_base_pointer& __child = __find_leaf(__hint, __parent, _NodeTypes::__get_key(__ptr->__value_));
+  __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__ptr));
+  __nh.__release_ptr();
+  return iterator(__ptr);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Tree>
+_LIBCPP_HIDE_FROM_ABI void __tree<_Tp, _Compare, _Allocator>::__node_handle_merge_multi(_Tree& __source) {
+  static_assert(is_same<typename _Tree::__node_pointer, __node_pointer>::value, "");
+
+  for (typename _Tree::iterator __i = __source.begin(); __i != __source.end();) {
+    __node_pointer __src_ptr = __i.__get_np();
+    __parent_pointer __parent;
+    __node_base_pointer& __child = __find_leaf_high(__parent, _NodeTypes::__get_key(__src_ptr->__value_));
+    ++__i;
+    __source.__remove_node_pointer(__src_ptr);
+    __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__src_ptr));
+  }
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+template <class _Tp, class _Compare, class _Allocator>
+typename __tree<_Tp, _Compare, _Allocator>::iterator __tree<_Tp, _Compare, _Allocator>::erase(const_iterator __p) {
+  __node_pointer __np    = __p.__get_np();
+  iterator __r           = __remove_node_pointer(__np);
+  __node_allocator& __na = __node_alloc();
+  __node_traits::destroy(__na, _NodeTypes::__get_ptr(const_cast<__node_value_type&>(*__p)));
+  __node_traits::deallocate(__na, __np, 1);
+  return __r;
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+typename __tree<_Tp, _Compare, _Allocator>::iterator
+__tree<_Tp, _Compare, _Allocator>::erase(const_iterator __f, const_iterator __l) {
+  while (__f != __l)
+    __f = erase(__f);
+  return iterator(__l.__ptr_);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+typename __tree<_Tp, _Compare, _Allocator>::size_type
+__tree<_Tp, _Compare, _Allocator>::__erase_unique(const _Key& __k) {
+  iterator __i = find(__k);
+  if (__i == end())
+    return 0;
+  erase(__i);
+  return 1;
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+typename __tree<_Tp, _Compare, _Allocator>::size_type
+__tree<_Tp, _Compare, _Allocator>::__erase_multi(const _Key& __k) {
+  pair<iterator, iterator> __p = __equal_range_multi(__k);
+  size_type __r                = 0;
+  for (; __p.first != __p.second; ++__r)
+    __p.first = erase(__p.first);
+  return __r;
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+typename __tree<_Tp, _Compare, _Allocator>::iterator __tree<_Tp, _Compare, _Allocator>::find(const _Key& __v) {
+  iterator __p = __lower_bound(__v, __root(), __end_node());
+  if (__p != end() && !value_comp()(__v, *__p))
+    return __p;
+  return end();
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+typename __tree<_Tp, _Compare, _Allocator>::const_iterator
+__tree<_Tp, _Compare, _Allocator>::find(const _Key& __v) const {
+  const_iterator __p = __lower_bound(__v, __root(), __end_node());
+  if (__p != end() && !value_comp()(__v, *__p))
+    return __p;
+  return end();
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+typename __tree<_Tp, _Compare, _Allocator>::size_type
+__tree<_Tp, _Compare, _Allocator>::__count_unique(const _Key& __k) const {
+  __node_pointer __rt = __root();
+  while (__rt != nullptr) {
+    if (value_comp()(__k, __rt->__value_)) {
+      __rt = static_cast<__node_pointer>(__rt->__left_);
+    } else if (value_comp()(__rt->__value_, __k))
+      __rt = static_cast<__node_pointer>(__rt->__right_);
+    else
+      return 1;
+  }
+  return 0;
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+typename __tree<_Tp, _Compare, _Allocator>::size_type
+__tree<_Tp, _Compare, _Allocator>::__count_multi(const _Key& __k) const {
+  __iter_pointer __result = __end_node();
+  __node_pointer __rt     = __root();
+  while (__rt != nullptr) {
+    if (value_comp()(__k, __rt->__value_)) {
+      __result = static_cast<__iter_pointer>(__rt);
+      __rt     = static_cast<__node_pointer>(__rt->__left_);
+    } else if (value_comp()(__rt->__value_, __k))
+      __rt = static_cast<__node_pointer>(__rt->__right_);
+    else
+      return std::distance(
+          __lower_bound(__k, static_cast<__node_pointer>(__rt->__left_), static_cast<__iter_pointer>(__rt)),
+          __upper_bound(__k, static_cast<__node_pointer>(__rt->__right_), __result));
+  }
+  return 0;
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+typename __tree<_Tp, _Compare, _Allocator>::iterator
+__tree<_Tp, _Compare, _Allocator>::__lower_bound(const _Key& __v, __node_pointer __root, __iter_pointer __result) {
+  while (__root != nullptr) {
+    if (!value_comp()(__root->__value_, __v)) {
+      __result = static_cast<__iter_pointer>(__root);
+      __root   = static_cast<__node_pointer>(__root->__left_);
+    } else
+      __root = static_cast<__node_pointer>(__root->__right_);
+  }
+  return iterator(__result);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+typename __tree<_Tp, _Compare, _Allocator>::const_iterator __tree<_Tp, _Compare, _Allocator>::__lower_bound(
+    const _Key& __v, __node_pointer __root, __iter_pointer __result) const {
+  while (__root != nullptr) {
+    if (!value_comp()(__root->__value_, __v)) {
+      __result = static_cast<__iter_pointer>(__root);
+      __root   = static_cast<__node_pointer>(__root->__left_);
+    } else
+      __root = static_cast<__node_pointer>(__root->__right_);
+  }
+  return const_iterator(__result);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+typename __tree<_Tp, _Compare, _Allocator>::iterator
+__tree<_Tp, _Compare, _Allocator>::__upper_bound(const _Key& __v, __node_pointer __root, __iter_pointer __result) {
+  while (__root != nullptr) {
+    if (value_comp()(__v, __root->__value_)) {
+      __result = static_cast<__iter_pointer>(__root);
+      __root   = static_cast<__node_pointer>(__root->__left_);
+    } else
+      __root = static_cast<__node_pointer>(__root->__right_);
+  }
+  return iterator(__result);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+typename __tree<_Tp, _Compare, _Allocator>::const_iterator __tree<_Tp, _Compare, _Allocator>::__upper_bound(
+    const _Key& __v, __node_pointer __root, __iter_pointer __result) const {
+  while (__root != nullptr) {
+    if (value_comp()(__v, __root->__value_)) {
+      __result = static_cast<__iter_pointer>(__root);
+      __root   = static_cast<__node_pointer>(__root->__left_);
+    } else
+      __root = static_cast<__node_pointer>(__root->__right_);
+  }
+  return const_iterator(__result);
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, typename __tree<_Tp, _Compare, _Allocator>::iterator>
+__tree<_Tp, _Compare, _Allocator>::__equal_range_unique(const _Key& __k) {
+  typedef pair<iterator, iterator> _Pp;
+  __iter_pointer __result = __end_node();
+  __node_pointer __rt     = __root();
+  while (__rt != nullptr) {
+    if (value_comp()(__k, __rt->__value_)) {
+      __result = static_cast<__iter_pointer>(__rt);
+      __rt     = static_cast<__node_pointer>(__rt->__left_);
+    } else if (value_comp()(__rt->__value_, __k))
+      __rt = static_cast<__node_pointer>(__rt->__right_);
+    else
+      return _Pp(iterator(__rt),
+                 iterator(__rt->__right_ != nullptr ? static_cast<__iter_pointer>(std::__tree_min(__rt->__right_))
+                                                    : __result));
+  }
+  return _Pp(iterator(__result), iterator(__result));
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+pair<typename __tree<_Tp, _Compare, _Allocator>::const_iterator,
+     typename __tree<_Tp, _Compare, _Allocator>::const_iterator>
+__tree<_Tp, _Compare, _Allocator>::__equal_range_unique(const _Key& __k) const {
+  typedef pair<const_iterator, const_iterator> _Pp;
+  __iter_pointer __result = __end_node();
+  __node_pointer __rt     = __root();
+  while (__rt != nullptr) {
+    if (value_comp()(__k, __rt->__value_)) {
+      __result = static_cast<__iter_pointer>(__rt);
+      __rt     = static_cast<__node_pointer>(__rt->__left_);
+    } else if (value_comp()(__rt->__value_, __k))
+      __rt = static_cast<__node_pointer>(__rt->__right_);
+    else
+      return _Pp(
+          const_iterator(__rt),
+          const_iterator(
+              __rt->__right_ != nullptr ? static_cast<__iter_pointer>(std::__tree_min(__rt->__right_)) : __result));
+  }
+  return _Pp(const_iterator(__result), const_iterator(__result));
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, typename __tree<_Tp, _Compare, _Allocator>::iterator>
+__tree<_Tp, _Compare, _Allocator>::__equal_range_multi(const _Key& __k) {
+  typedef pair<iterator, iterator> _Pp;
+  __iter_pointer __result = __end_node();
+  __node_pointer __rt     = __root();
+  while (__rt != nullptr) {
+    if (value_comp()(__k, __rt->__value_)) {
+      __result = static_cast<__iter_pointer>(__rt);
+      __rt     = static_cast<__node_pointer>(__rt->__left_);
+    } else if (value_comp()(__rt->__value_, __k))
+      __rt = static_cast<__node_pointer>(__rt->__right_);
+    else
+      return _Pp(__lower_bound(__k, static_cast<__node_pointer>(__rt->__left_), static_cast<__iter_pointer>(__rt)),
+                 __upper_bound(__k, static_cast<__node_pointer>(__rt->__right_), __result));
+  }
+  return _Pp(iterator(__result), iterator(__result));
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+template <class _Key>
+pair<typename __tree<_Tp, _Compare, _Allocator>::const_iterator,
+     typename __tree<_Tp, _Compare, _Allocator>::const_iterator>
+__tree<_Tp, _Compare, _Allocator>::__equal_range_multi(const _Key& __k) const {
+  typedef pair<const_iterator, const_iterator> _Pp;
+  __iter_pointer __result = __end_node();
+  __node_pointer __rt     = __root();
+  while (__rt != nullptr) {
+    if (value_comp()(__k, __rt->__value_)) {
+      __result = static_cast<__iter_pointer>(__rt);
+      __rt     = static_cast<__node_pointer>(__rt->__left_);
+    } else if (value_comp()(__rt->__value_, __k))
+      __rt = static_cast<__node_pointer>(__rt->__right_);
+    else
+      return _Pp(__lower_bound(__k, static_cast<__node_pointer>(__rt->__left_), static_cast<__iter_pointer>(__rt)),
+                 __upper_bound(__k, static_cast<__node_pointer>(__rt->__right_), __result));
+  }
+  return _Pp(const_iterator(__result), const_iterator(__result));
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+typename __tree<_Tp, _Compare, _Allocator>::__node_holder
+__tree<_Tp, _Compare, _Allocator>::remove(const_iterator __p) _NOEXCEPT {
+  __node_pointer __np = __p.__get_np();
+  if (__begin_node() == __p.__ptr_) {
+    if (__np->__right_ != nullptr)
+      __begin_node() = static_cast<__iter_pointer>(__np->__right_);
+    else
+      __begin_node() = static_cast<__iter_pointer>(__np->__parent_);
+  }
+  --size();
+  std::__tree_remove(__end_node()->__left_, static_cast<__node_base_pointer>(__np));
+  return __node_holder(__np, _Dp(__node_alloc(), true));
+}
+
+template <class _Tp, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI void swap(__tree<_Tp, _Compare, _Allocator>& __x, __tree<_Tp, _Compare, _Allocator>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___TREE
diff --git a/libcxx/include/__cxx03/__tuple/find_index.h b/libcxx/include/__cxx03/__tuple/find_index.h
new file mode 100644
index 000000000000..133b00419d0c
--- /dev/null
+++ b/libcxx/include/__cxx03/__tuple/find_index.h
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TUPLE_FIND_INDEX_H
+#define _LIBCPP___TUPLE_FIND_INDEX_H
+
+#include <__config>
+#include <__type_traits/is_same.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __find_detail {
+
+static constexpr size_t __not_found = static_cast<size_t>(-1);
+static constexpr size_t __ambiguous = __not_found - 1;
+
+inline _LIBCPP_HIDE_FROM_ABI constexpr size_t __find_idx_return(size_t __curr_i, size_t __res, bool __matches) {
+  return !__matches ? __res : (__res == __not_found ? __curr_i : __ambiguous);
+}
+
+template <size_t _Nx>
+inline _LIBCPP_HIDE_FROM_ABI constexpr size_t __find_idx(size_t __i, const bool (&__matches)[_Nx]) {
+  return __i == _Nx
+           ? __not_found
+           : __find_detail::__find_idx_return(__i, __find_detail::__find_idx(__i + 1, __matches), __matches[__i]);
+}
+
+template <class _T1, class... _Args>
+struct __find_exactly_one_checked {
+  static constexpr bool __matches[sizeof...(_Args)] = {is_same<_T1, _Args>::value...};
+  static constexpr size_t value                     = __find_detail::__find_idx(0, __matches);
+  static_assert(value != __not_found, "type not found in type list");
+  static_assert(value != __ambiguous, "type occurs more than once in type list");
+};
+
+template <class _T1>
+struct __find_exactly_one_checked<_T1> {
+  static_assert(!is_same<_T1, _T1>::value, "type not in empty type list");
+};
+
+} // namespace __find_detail
+
+template <typename _T1, typename... _Args>
+struct __find_exactly_one_t : public __find_detail::__find_exactly_one_checked<_T1, _Args...> {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 14
+
+#endif // _LIBCPP___TUPLE_FIND_INDEX_H
diff --git a/libcxx/include/__cxx03/__tuple/ignore.h b/libcxx/include/__cxx03/__tuple/ignore.h
new file mode 100644
index 000000000000..43cce5387411
--- /dev/null
+++ b/libcxx/include/__cxx03/__tuple/ignore.h
@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TUPLE_IGNORE_H
+#define _LIBCPP___TUPLE_IGNORE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct __ignore_type {
+  template <class _Tp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const __ignore_type& operator=(const _Tp&) const noexcept {
+    return *this;
+  }
+};
+
+#  if _LIBCPP_STD_VER >= 17
+inline constexpr __ignore_type ignore;
+#  else
+constexpr __ignore_type ignore;
+#  endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___TUPLE_IGNORE_H
diff --git a/libcxx/include/__cxx03/__tuple/make_tuple_types.h b/libcxx/include/__cxx03/__tuple/make_tuple_types.h
new file mode 100644
index 000000000000..9e0fefae2f2f
--- /dev/null
+++ b/libcxx/include/__cxx03/__tuple/make_tuple_types.h
@@ -0,0 +1,80 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TUPLE_MAKE_TUPLE_TYPES_H
+#define _LIBCPP___TUPLE_MAKE_TUPLE_TYPES_H
+
+#include <__config>
+#include <__fwd/array.h>
+#include <__fwd/tuple.h>
+#include <__tuple/tuple_element.h>
+#include <__tuple/tuple_indices.h>
+#include <__tuple/tuple_size.h>
+#include <__tuple/tuple_types.h>
+#include <__type_traits/copy_cvref.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/remove_reference.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// __make_tuple_types<_Tuple<_Types...>, _Ep, _Sp>::type is a
+// __tuple_types<_Types...> using only those _Types in the range [_Sp, _Ep).
+// _Sp defaults to 0 and _Ep defaults to tuple_size<_Tuple>.  If _Tuple is a
+// lvalue_reference type, then __tuple_types<_Types&...> is the result.
+
+template <class _TupleTypes, class _TupleIndices>
+struct __make_tuple_types_flat;
+
+template <template <class...> class _Tuple, class... _Types, size_t... _Idx>
+struct __make_tuple_types_flat<_Tuple<_Types...>, __tuple_indices<_Idx...>> {
+  // Specialization for pair, tuple, and __tuple_types
+  template <class _Tp>
+  using __apply_quals _LIBCPP_NODEBUG = __tuple_types<__copy_cvref_t<_Tp, __type_pack_element<_Idx, _Types...>>...>;
+};
+
+template <class _Vt, size_t _Np, size_t... _Idx>
+struct __make_tuple_types_flat<array<_Vt, _Np>, __tuple_indices<_Idx...>> {
+  template <size_t>
+  using __value_type = _Vt;
+  template <class _Tp>
+  using __apply_quals = __tuple_types<__copy_cvref_t<_Tp, __value_type<_Idx>>...>;
+};
+
+template <class _Tp,
+          size_t _Ep     = tuple_size<__libcpp_remove_reference_t<_Tp> >::value,
+          size_t _Sp     = 0,
+          bool _SameSize = (_Ep == tuple_size<__libcpp_remove_reference_t<_Tp> >::value)>
+struct __make_tuple_types {
+  static_assert(_Sp <= _Ep, "__make_tuple_types input error");
+  using _RawTp = __remove_cv_t<__libcpp_remove_reference_t<_Tp> >;
+  using _Maker = __make_tuple_types_flat<_RawTp, typename __make_tuple_indices<_Ep, _Sp>::type>;
+  using type   = typename _Maker::template __apply_quals<_Tp>;
+};
+
+template <class... _Types, size_t _Ep>
+struct __make_tuple_types<tuple<_Types...>, _Ep, 0, true> {
+  typedef _LIBCPP_NODEBUG __tuple_types<_Types...> type;
+};
+
+template <class... _Types, size_t _Ep>
+struct __make_tuple_types<__tuple_types<_Types...>, _Ep, 0, true> {
+  typedef _LIBCPP_NODEBUG __tuple_types<_Types...> type;
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___TUPLE_MAKE_TUPLE_TYPES_H
diff --git a/libcxx/include/__cxx03/__tuple/sfinae_helpers.h b/libcxx/include/__cxx03/__tuple/sfinae_helpers.h
new file mode 100644
index 000000000000..c7145e0b011a
--- /dev/null
+++ b/libcxx/include/__cxx03/__tuple/sfinae_helpers.h
@@ -0,0 +1,142 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TUPLE_SFINAE_HELPERS_H
+#define _LIBCPP___TUPLE_SFINAE_HELPERS_H
+
+#include <__config>
+#include <__fwd/tuple.h>
+#include <__tuple/make_tuple_types.h>
+#include <__tuple/tuple_element.h>
+#include <__tuple/tuple_like_ext.h>
+#include <__tuple/tuple_size.h>
+#include <__tuple/tuple_types.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/remove_reference.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifndef _LIBCPP_CXX03_LANG
+
+struct __tuple_sfinae_base {
+  template <template <class, class...> class _Trait, class... _LArgs, class... _RArgs>
+  static auto __do_test(__tuple_types<_LArgs...>,
+                        __tuple_types<_RArgs...>) -> __all<__enable_if_t<_Trait<_LArgs, _RArgs>::value, bool>{true}...>;
+  template <template <class...> class>
+  static auto __do_test(...) -> false_type;
+
+  template <class _FromArgs, class _ToArgs>
+  using __constructible = decltype(__do_test<is_constructible>(_ToArgs{}, _FromArgs{}));
+};
+
+// __tuple_constructible
+
+template <class _Tp,
+          class _Up,
+          bool = __tuple_like_ext<__libcpp_remove_reference_t<_Tp> >::value,
+          bool = __tuple_like_ext<_Up>::value>
+struct __tuple_constructible : public false_type {};
+
+template <class _Tp, class _Up>
+struct __tuple_constructible<_Tp, _Up, true, true>
+    : public __tuple_sfinae_base::__constructible< typename __make_tuple_types<_Tp>::type,
+                                                   typename __make_tuple_types<_Up>::type > {};
+
+template <size_t _Ip, class... _Tp>
+struct _LIBCPP_TEMPLATE_VIS tuple_element<_Ip, tuple<_Tp...> > {
+  typedef _LIBCPP_NODEBUG typename tuple_element<_Ip, __tuple_types<_Tp...> >::type type;
+};
+
+struct _LIBCPP_EXPORTED_FROM_ABI __check_tuple_constructor_fail {
+  static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_explicit_default() { return false; }
+  static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_implicit_default() { return false; }
+  template <class...>
+  static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_explicit() {
+    return false;
+  }
+  template <class...>
+  static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_implicit() {
+    return false;
+  }
+  template <class...>
+  static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_assign() {
+    return false;
+  }
+};
+#endif // !defined(_LIBCPP_CXX03_LANG)
+
+#if _LIBCPP_STD_VER >= 17
+
+template <bool _CanCopy, bool _CanMove>
+struct __sfinae_ctor_base {};
+template <>
+struct __sfinae_ctor_base<false, false> {
+  __sfinae_ctor_base()                                     = default;
+  __sfinae_ctor_base(__sfinae_ctor_base const&)            = delete;
+  __sfinae_ctor_base(__sfinae_ctor_base&&)                 = delete;
+  __sfinae_ctor_base& operator=(__sfinae_ctor_base const&) = default;
+  __sfinae_ctor_base& operator=(__sfinae_ctor_base&&)      = default;
+};
+template <>
+struct __sfinae_ctor_base<true, false> {
+  __sfinae_ctor_base()                                     = default;
+  __sfinae_ctor_base(__sfinae_ctor_base const&)            = default;
+  __sfinae_ctor_base(__sfinae_ctor_base&&)                 = delete;
+  __sfinae_ctor_base& operator=(__sfinae_ctor_base const&) = default;
+  __sfinae_ctor_base& operator=(__sfinae_ctor_base&&)      = default;
+};
+template <>
+struct __sfinae_ctor_base<false, true> {
+  __sfinae_ctor_base()                                     = default;
+  __sfinae_ctor_base(__sfinae_ctor_base const&)            = delete;
+  __sfinae_ctor_base(__sfinae_ctor_base&&)                 = default;
+  __sfinae_ctor_base& operator=(__sfinae_ctor_base const&) = default;
+  __sfinae_ctor_base& operator=(__sfinae_ctor_base&&)      = default;
+};
+
+template <bool _CanCopy, bool _CanMove>
+struct __sfinae_assign_base {};
+template <>
+struct __sfinae_assign_base<false, false> {
+  __sfinae_assign_base()                                       = default;
+  __sfinae_assign_base(__sfinae_assign_base const&)            = default;
+  __sfinae_assign_base(__sfinae_assign_base&&)                 = default;
+  __sfinae_assign_base& operator=(__sfinae_assign_base const&) = delete;
+  __sfinae_assign_base& operator=(__sfinae_assign_base&&)      = delete;
+};
+template <>
+struct __sfinae_assign_base<true, false> {
+  __sfinae_assign_base()                                       = default;
+  __sfinae_assign_base(__sfinae_assign_base const&)            = default;
+  __sfinae_assign_base(__sfinae_assign_base&&)                 = default;
+  __sfinae_assign_base& operator=(__sfinae_assign_base const&) = default;
+  __sfinae_assign_base& operator=(__sfinae_assign_base&&)      = delete;
+};
+template <>
+struct __sfinae_assign_base<false, true> {
+  __sfinae_assign_base()                                       = default;
+  __sfinae_assign_base(__sfinae_assign_base const&)            = default;
+  __sfinae_assign_base(__sfinae_assign_base&&)                 = default;
+  __sfinae_assign_base& operator=(__sfinae_assign_base const&) = delete;
+  __sfinae_assign_base& operator=(__sfinae_assign_base&&)      = default;
+};
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TUPLE_SFINAE_HELPERS_H
diff --git a/libcxx/include/__cxx03/__tuple/tuple_element.h b/libcxx/include/__cxx03/__tuple/tuple_element.h
new file mode 100644
index 000000000000..9127c47dc8f1
--- /dev/null
+++ b/libcxx/include/__cxx03/__tuple/tuple_element.h
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TUPLE_TUPLE_ELEMENT_H
+#define _LIBCPP___TUPLE_TUPLE_ELEMENT_H
+
+#include <__config>
+#include <__tuple/tuple_indices.h>
+#include <__tuple/tuple_types.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <size_t _Ip, class _Tp>
+struct _LIBCPP_TEMPLATE_VIS tuple_element;
+
+template <size_t _Ip, class _Tp>
+struct _LIBCPP_TEMPLATE_VIS tuple_element<_Ip, const _Tp> {
+  typedef _LIBCPP_NODEBUG const typename tuple_element<_Ip, _Tp>::type type;
+};
+
+template <size_t _Ip, class _Tp>
+struct _LIBCPP_TEMPLATE_VIS tuple_element<_Ip, volatile _Tp> {
+  typedef _LIBCPP_NODEBUG volatile typename tuple_element<_Ip, _Tp>::type type;
+};
+
+template <size_t _Ip, class _Tp>
+struct _LIBCPP_TEMPLATE_VIS tuple_element<_Ip, const volatile _Tp> {
+  typedef _LIBCPP_NODEBUG const volatile typename tuple_element<_Ip, _Tp>::type type;
+};
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <size_t _Ip, class... _Types>
+struct _LIBCPP_TEMPLATE_VIS tuple_element<_Ip, __tuple_types<_Types...> > {
+  static_assert(_Ip < sizeof...(_Types), "tuple_element index out of range");
+  typedef _LIBCPP_NODEBUG __type_pack_element<_Ip, _Types...> type;
+};
+
+#  if _LIBCPP_STD_VER >= 14
+template <size_t _Ip, class... _Tp>
+using tuple_element_t _LIBCPP_NODEBUG = typename tuple_element<_Ip, _Tp...>::type;
+#  endif
+
+#endif // _LIBCPP_CXX03_LANG
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TUPLE_TUPLE_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__tuple/tuple_indices.h b/libcxx/include/__cxx03/__tuple/tuple_indices.h
new file mode 100644
index 000000000000..501e711255ec
--- /dev/null
+++ b/libcxx/include/__cxx03/__tuple/tuple_indices.h
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TUPLE_MAKE_TUPLE_INDICES_H
+#define _LIBCPP___TUPLE_MAKE_TUPLE_INDICES_H
+
+#include <__config>
+#include <__utility/integer_sequence.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <size_t...>
+struct __tuple_indices {};
+
+template <size_t _Ep, size_t _Sp = 0>
+struct __make_tuple_indices {
+  static_assert(_Sp <= _Ep, "__make_tuple_indices input error");
+  typedef __make_indices_imp<_Ep, _Sp> type;
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___TUPLE_MAKE_TUPLE_INDICES_H
diff --git a/libcxx/include/__cxx03/__tuple/tuple_like.h b/libcxx/include/__cxx03/__tuple/tuple_like.h
new file mode 100644
index 000000000000..c080a3dcf1e2
--- /dev/null
+++ b/libcxx/include/__cxx03/__tuple/tuple_like.h
@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TUPLE_TUPLE_LIKE_H
+#define _LIBCPP___TUPLE_TUPLE_LIKE_H
+
+#include <__config>
+#include <__fwd/subrange.h>
+#include <__tuple/tuple_like_no_subrange.h>
+#include <__tuple/tuple_size.h>
+#include <__type_traits/remove_cvref.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+inline constexpr bool __is_ranges_subrange_v = false;
+
+template <class _Iter, class _Sent, ranges::subrange_kind _Kind>
+inline constexpr bool __is_ranges_subrange_v<ranges::subrange<_Iter, _Sent, _Kind>> = true;
+
+template <class _Tp>
+concept __tuple_like = __tuple_like_no_subrange<_Tp> || __is_ranges_subrange_v<remove_cvref_t<_Tp>>;
+
+// As of writing this comment every use of `pair-like` in the standard excludes `ranges::subrange`, so
+// you most likely want to use `__pair_like_no_subrange` if you're looking for `pair-like`.
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TUPLE_TUPLE_LIKE_H
diff --git a/libcxx/include/__cxx03/__tuple/tuple_like_ext.h b/libcxx/include/__cxx03/__tuple/tuple_like_ext.h
new file mode 100644
index 000000000000..0cc21e0b75fd
--- /dev/null
+++ b/libcxx/include/__cxx03/__tuple/tuple_like_ext.h
@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TUPLE_TUPLE_LIKE_EXT_H
+#define _LIBCPP___TUPLE_TUPLE_LIKE_EXT_H
+
+#include <__config>
+#include <__fwd/array.h>
+#include <__fwd/pair.h>
+#include <__fwd/tuple.h>
+#include <__tuple/tuple_types.h>
+#include <__type_traits/integral_constant.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct __tuple_like_ext : false_type {};
+
+template <class _Tp>
+struct __tuple_like_ext<const _Tp> : public __tuple_like_ext<_Tp> {};
+template <class _Tp>
+struct __tuple_like_ext<volatile _Tp> : public __tuple_like_ext<_Tp> {};
+template <class _Tp>
+struct __tuple_like_ext<const volatile _Tp> : public __tuple_like_ext<_Tp> {};
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class... _Tp>
+struct __tuple_like_ext<tuple<_Tp...> > : true_type {};
+#endif
+
+template <class _T1, class _T2>
+struct __tuple_like_ext<pair<_T1, _T2> > : true_type {};
+
+template <class _Tp, size_t _Size>
+struct __tuple_like_ext<array<_Tp, _Size> > : true_type {};
+
+template <class... _Tp>
+struct __tuple_like_ext<__tuple_types<_Tp...> > : true_type {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TUPLE_TUPLE_LIKE_EXT_H
diff --git a/libcxx/include/__cxx03/__tuple/tuple_like_no_subrange.h b/libcxx/include/__cxx03/__tuple/tuple_like_no_subrange.h
new file mode 100644
index 000000000000..274b0bf188e1
--- /dev/null
+++ b/libcxx/include/__cxx03/__tuple/tuple_like_no_subrange.h
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TUPLE_TUPLE_LIKE_NO_SUBRANGE_H
+#define _LIBCPP___TUPLE_TUPLE_LIKE_NO_SUBRANGE_H
+
+#include <__config>
+#include <__fwd/array.h>
+#include <__fwd/complex.h>
+#include <__fwd/pair.h>
+#include <__fwd/tuple.h>
+#include <__tuple/tuple_size.h>
+#include <__type_traits/remove_cvref.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+inline constexpr bool __tuple_like_no_subrange_impl = false;
+
+template <class... _Tp>
+inline constexpr bool __tuple_like_no_subrange_impl<tuple<_Tp...>> = true;
+
+template <class _T1, class _T2>
+inline constexpr bool __tuple_like_no_subrange_impl<pair<_T1, _T2>> = true;
+
+template <class _Tp, size_t _Size>
+inline constexpr bool __tuple_like_no_subrange_impl<array<_Tp, _Size>> = true;
+
+#  if _LIBCPP_STD_VER >= 26
+
+template <class _Tp>
+inline constexpr bool __tuple_like_no_subrange_impl<complex<_Tp>> = true;
+
+#  endif
+
+template <class _Tp>
+concept __tuple_like_no_subrange = __tuple_like_no_subrange_impl<remove_cvref_t<_Tp>>;
+
+// This is equivalent to the exposition-only type trait `pair-like`, except that it is false for specializations of
+// `ranges::subrange`. This is more useful than the pair-like concept in the standard because every use of `pair-like`
+// excludes `ranges::subrange`.
+template <class _Tp>
+concept __pair_like_no_subrange = __tuple_like_no_subrange<_Tp> && tuple_size<remove_cvref_t<_Tp>>::value == 2;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TUPLE_TUPLE_LIKE_NO_SUBRANGE_H
diff --git a/libcxx/include/__cxx03/__tuple/tuple_size.h b/libcxx/include/__cxx03/__tuple/tuple_size.h
new file mode 100644
index 000000000000..18a17fd4d587
--- /dev/null
+++ b/libcxx/include/__cxx03/__tuple/tuple_size.h
@@ -0,0 +1,75 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TUPLE_TUPLE_SIZE_H
+#define _LIBCPP___TUPLE_TUPLE_SIZE_H
+
+#include <__config>
+#include <__fwd/tuple.h>
+#include <__tuple/tuple_types.h>
+#include <__type_traits/is_const.h>
+#include <__type_traits/is_volatile.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS tuple_size;
+
+#if !defined(_LIBCPP_CXX03_LANG)
+template <class _Tp, class...>
+using __enable_if_tuple_size_imp = _Tp;
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS tuple_size<__enable_if_tuple_size_imp< const _Tp,
+                                                                   __enable_if_t<!is_volatile<_Tp>::value>,
+                                                                   integral_constant<size_t, sizeof(tuple_size<_Tp>)>>>
+    : public integral_constant<size_t, tuple_size<_Tp>::value> {};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS tuple_size<__enable_if_tuple_size_imp< volatile _Tp,
+                                                                   __enable_if_t<!is_const<_Tp>::value>,
+                                                                   integral_constant<size_t, sizeof(tuple_size<_Tp>)>>>
+    : public integral_constant<size_t, tuple_size<_Tp>::value> {};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS
+tuple_size<__enable_if_tuple_size_imp<const volatile _Tp, integral_constant<size_t, sizeof(tuple_size<_Tp>)>>>
+    : public integral_constant<size_t, tuple_size<_Tp>::value> {};
+
+#else
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS tuple_size<const _Tp> : public tuple_size<_Tp> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS tuple_size<volatile _Tp> : public tuple_size<_Tp> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS tuple_size<const volatile _Tp> : public tuple_size<_Tp> {};
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class... _Tp>
+struct _LIBCPP_TEMPLATE_VIS tuple_size<tuple<_Tp...> > : public integral_constant<size_t, sizeof...(_Tp)> {};
+
+template <class... _Tp>
+struct _LIBCPP_TEMPLATE_VIS tuple_size<__tuple_types<_Tp...> > : public integral_constant<size_t, sizeof...(_Tp)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr size_t tuple_size_v = tuple_size<_Tp>::value;
+#  endif
+
+#endif // _LIBCPP_CXX03_LANG
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TUPLE_TUPLE_SIZE_H
diff --git a/libcxx/include/__cxx03/__tuple/tuple_types.h b/libcxx/include/__cxx03/__tuple/tuple_types.h
new file mode 100644
index 000000000000..7e1256cf8790
--- /dev/null
+++ b/libcxx/include/__cxx03/__tuple/tuple_types.h
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TUPLE_TUPLE_TYPES_H
+#define _LIBCPP___TUPLE_TUPLE_TYPES_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class... _Tp>
+struct __tuple_types {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TUPLE_TUPLE_TYPES_H
diff --git a/libcxx/include/__cxx03/__type_traits/add_const.h b/libcxx/include/__cxx03/__type_traits/add_const.h
new file mode 100644
index 000000000000..9a6f1c10299f
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/add_const.h
@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_ADD_CONST_H
+#define _LIBCPP___TYPE_TRAITS_ADD_CONST_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS add_const {
+  typedef _LIBCPP_NODEBUG const _Tp type;
+};
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using add_const_t = typename add_const<_Tp>::type;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_ADD_CONST_H
diff --git a/libcxx/include/__cxx03/__type_traits/add_cv.h b/libcxx/include/__cxx03/__type_traits/add_cv.h
new file mode 100644
index 000000000000..9e23e5ceb7a3
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/add_cv.h
@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_ADD_CV_H
+#define _LIBCPP___TYPE_TRAITS_ADD_CV_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS add_cv {
+  typedef _LIBCPP_NODEBUG const volatile _Tp type;
+};
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using add_cv_t = typename add_cv<_Tp>::type;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_ADD_CV_H
diff --git a/libcxx/include/__cxx03/__type_traits/add_lvalue_reference.h b/libcxx/include/__cxx03/__type_traits/add_lvalue_reference.h
new file mode 100644
index 000000000000..a633e3904532
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/add_lvalue_reference.h
@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_ADD_LVALUE_REFERENCE_H
+#define _LIBCPP___TYPE_TRAITS_ADD_LVALUE_REFERENCE_H
+
+#include <__config>
+#include <__type_traits/is_referenceable.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__add_lvalue_reference)
+
+template <class _Tp>
+using __add_lvalue_reference_t = __add_lvalue_reference(_Tp);
+
+#else
+
+template <class _Tp, bool = __libcpp_is_referenceable<_Tp>::value>
+struct __add_lvalue_reference_impl {
+  typedef _LIBCPP_NODEBUG _Tp type;
+};
+template <class _Tp >
+struct __add_lvalue_reference_impl<_Tp, true> {
+  typedef _LIBCPP_NODEBUG _Tp& type;
+};
+
+template <class _Tp>
+using __add_lvalue_reference_t = typename __add_lvalue_reference_impl<_Tp>::type;
+
+#endif // __has_builtin(__add_lvalue_reference)
+
+template <class _Tp>
+struct add_lvalue_reference {
+  using type _LIBCPP_NODEBUG = __add_lvalue_reference_t<_Tp>;
+};
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using add_lvalue_reference_t = __add_lvalue_reference_t<_Tp>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_ADD_LVALUE_REFERENCE_H
diff --git a/libcxx/include/__cxx03/__type_traits/add_pointer.h b/libcxx/include/__cxx03/__type_traits/add_pointer.h
new file mode 100644
index 000000000000..5aac7d5cfa90
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/add_pointer.h
@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_ADD_POINTER_H
+#define _LIBCPP___TYPE_TRAITS_ADD_POINTER_H
+
+#include <__config>
+#include <__type_traits/is_referenceable.h>
+#include <__type_traits/is_void.h>
+#include <__type_traits/remove_reference.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if !defined(_LIBCPP_WORKAROUND_OBJCXX_COMPILER_INTRINSICS) && __has_builtin(__add_pointer)
+
+template <class _Tp>
+using __add_pointer_t = __add_pointer(_Tp);
+
+#else
+template <class _Tp, bool = __libcpp_is_referenceable<_Tp>::value || is_void<_Tp>::value>
+struct __add_pointer_impl {
+  typedef _LIBCPP_NODEBUG __libcpp_remove_reference_t<_Tp>* type;
+};
+template <class _Tp>
+struct __add_pointer_impl<_Tp, false> {
+  typedef _LIBCPP_NODEBUG _Tp type;
+};
+
+template <class _Tp>
+using __add_pointer_t = typename __add_pointer_impl<_Tp>::type;
+
+#endif // !defined(_LIBCPP_WORKAROUND_OBJCXX_COMPILER_INTRINSICS) && __has_builtin(__add_pointer)
+
+template <class _Tp>
+struct add_pointer {
+  using type _LIBCPP_NODEBUG = __add_pointer_t<_Tp>;
+};
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using add_pointer_t = __add_pointer_t<_Tp>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_ADD_POINTER_H
diff --git a/libcxx/include/__cxx03/__type_traits/add_rvalue_reference.h b/libcxx/include/__cxx03/__type_traits/add_rvalue_reference.h
new file mode 100644
index 000000000000..a54aae7ec8de
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/add_rvalue_reference.h
@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_ADD_RVALUE_REFERENCE_H
+#define _LIBCPP___TYPE_TRAITS_ADD_RVALUE_REFERENCE_H
+
+#include <__config>
+#include <__type_traits/is_referenceable.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__add_rvalue_reference)
+
+template <class _Tp>
+using __add_rvalue_reference_t = __add_rvalue_reference(_Tp);
+
+#else
+
+template <class _Tp, bool = __libcpp_is_referenceable<_Tp>::value>
+struct __add_rvalue_reference_impl {
+  typedef _LIBCPP_NODEBUG _Tp type;
+};
+template <class _Tp >
+struct __add_rvalue_reference_impl<_Tp, true> {
+  typedef _LIBCPP_NODEBUG _Tp&& type;
+};
+
+template <class _Tp>
+using __add_rvalue_reference_t = typename __add_rvalue_reference_impl<_Tp>::type;
+
+#endif // __has_builtin(__add_rvalue_reference)
+
+template <class _Tp>
+struct add_rvalue_reference {
+  using type = __add_rvalue_reference_t<_Tp>;
+};
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using add_rvalue_reference_t = __add_rvalue_reference_t<_Tp>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_ADD_RVALUE_REFERENCE_H
diff --git a/libcxx/include/__cxx03/__type_traits/add_volatile.h b/libcxx/include/__cxx03/__type_traits/add_volatile.h
new file mode 100644
index 000000000000..56b7dfaac026
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/add_volatile.h
@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_ADD_VOLATILE_H
+#define _LIBCPP___TYPE_TRAITS_ADD_VOLATILE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS add_volatile {
+  typedef _LIBCPP_NODEBUG volatile _Tp type;
+};
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using add_volatile_t = typename add_volatile<_Tp>::type;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_ADD_VOLATILE_H
diff --git a/libcxx/include/__cxx03/__type_traits/aligned_storage.h b/libcxx/include/__cxx03/__type_traits/aligned_storage.h
new file mode 100644
index 000000000000..46aae12832f8
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/aligned_storage.h
@@ -0,0 +1,138 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_ALIGNED_STORAGE_H
+#define _LIBCPP___TYPE_TRAITS_ALIGNED_STORAGE_H
+
+#include <__config>
+#include <__type_traits/conditional.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/nat.h>
+#include <__type_traits/type_list.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct __align_type {
+  static const size_t value = _LIBCPP_PREFERRED_ALIGNOF(_Tp);
+  typedef _Tp type;
+};
+
+struct __struct_double {
+  long double __lx;
+};
+struct __struct_double4 {
+  double __lx[4];
+};
+
+// clang-format off
+typedef __type_list<__align_type<unsigned char>,
+        __type_list<__align_type<unsigned short>,
+        __type_list<__align_type<unsigned int>,
+        __type_list<__align_type<unsigned long>,
+        __type_list<__align_type<unsigned long long>,
+        __type_list<__align_type<double>,
+        __type_list<__align_type<long double>,
+        __type_list<__align_type<__struct_double>,
+        __type_list<__align_type<__struct_double4>,
+        __type_list<__align_type<int*>,
+        __nat
+        > > > > > > > > > > __all_types;
+// clang-format on
+
+template <size_t _Align>
+struct _ALIGNAS(_Align) __fallback_overaligned {};
+
+template <class _TL, size_t _Align>
+struct __find_pod;
+
+template <class _Hp, size_t _Align>
+struct __find_pod<__type_list<_Hp, __nat>, _Align> {
+  typedef __conditional_t<_Align == _Hp::value, typename _Hp::type, __fallback_overaligned<_Align> > type;
+};
+
+template <class _Hp, class _Tp, size_t _Align>
+struct __find_pod<__type_list<_Hp, _Tp>, _Align> {
+  typedef __conditional_t<_Align == _Hp::value, typename _Hp::type, typename __find_pod<_Tp, _Align>::type> type;
+};
+
+template <class _TL, size_t _Len>
+struct __find_max_align;
+
+template <class _Hp, size_t _Len>
+struct __find_max_align<__type_list<_Hp, __nat>, _Len> : public integral_constant<size_t, _Hp::value> {};
+
+template <size_t _Len, size_t _A1, size_t _A2>
+struct __select_align {
+private:
+  static const size_t __min = _A2 < _A1 ? _A2 : _A1;
+  static const size_t __max = _A1 < _A2 ? _A2 : _A1;
+
+public:
+  static const size_t value = _Len < __max ? __min : __max;
+};
+
+template <class _Hp, class _Tp, size_t _Len>
+struct __find_max_align<__type_list<_Hp, _Tp>, _Len>
+    : public integral_constant<size_t, __select_align<_Len, _Hp::value, __find_max_align<_Tp, _Len>::value>::value> {};
+
+template <size_t _Len, size_t _Align = __find_max_align<__all_types, _Len>::value>
+struct _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_TEMPLATE_VIS aligned_storage {
+  typedef typename __find_pod<__all_types, _Align>::type _Aligner;
+  union type {
+    _Aligner __align;
+    unsigned char __data[(_Len + _Align - 1) / _Align * _Align];
+  };
+};
+
+#if _LIBCPP_STD_VER >= 14
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <size_t _Len, size_t _Align = __find_max_align<__all_types, _Len>::value>
+using aligned_storage_t _LIBCPP_DEPRECATED_IN_CXX23 = typename aligned_storage<_Len, _Align>::type;
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+#endif
+
+#define _CREATE_ALIGNED_STORAGE_SPECIALIZATION(n)                                                                      \
+  template <size_t _Len>                                                                                               \
+  struct _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_TEMPLATE_VIS aligned_storage<_Len, n> {                                   \
+    struct _ALIGNAS(n) type {                                                                                          \
+      unsigned char __lx[(_Len + n - 1) / n * n];                                                                      \
+    };                                                                                                                 \
+  }
+
+_CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x1);
+_CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x2);
+_CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x4);
+_CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x8);
+_CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x10);
+_CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x20);
+_CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x40);
+_CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x80);
+_CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x100);
+_CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x200);
+_CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x400);
+_CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x800);
+_CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x1000);
+_CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x2000);
+// PE/COFF does not support alignment beyond 8192 (=0x2000)
+#if !defined(_LIBCPP_OBJECT_FORMAT_COFF)
+_CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x4000);
+#endif // !defined(_LIBCPP_OBJECT_FORMAT_COFF)
+
+#undef _CREATE_ALIGNED_STORAGE_SPECIALIZATION
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_ALIGNED_STORAGE_H
diff --git a/libcxx/include/__cxx03/__type_traits/aligned_union.h b/libcxx/include/__cxx03/__type_traits/aligned_union.h
new file mode 100644
index 000000000000..005ed9a096ea
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/aligned_union.h
@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_ALIGNED_UNION_H
+#define _LIBCPP___TYPE_TRAITS_ALIGNED_UNION_H
+
+#include <__config>
+#include <__type_traits/aligned_storage.h>
+#include <__type_traits/integral_constant.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <size_t _I0, size_t... _In>
+struct __static_max;
+
+template <size_t _I0>
+struct __static_max<_I0> {
+  static const size_t value = _I0;
+};
+
+template <size_t _I0, size_t _I1, size_t... _In>
+struct __static_max<_I0, _I1, _In...> {
+  static const size_t value = _I0 >= _I1 ? __static_max<_I0, _In...>::value : __static_max<_I1, _In...>::value;
+};
+
+template <size_t _Len, class _Type0, class... _Types>
+struct _LIBCPP_DEPRECATED_IN_CXX23 aligned_union {
+  static const size_t alignment_value =
+      __static_max<_LIBCPP_PREFERRED_ALIGNOF(_Type0), _LIBCPP_PREFERRED_ALIGNOF(_Types)...>::value;
+  static const size_t __len = __static_max<_Len, sizeof(_Type0), sizeof(_Types)...>::value;
+  typedef typename aligned_storage<__len, alignment_value>::type type;
+};
+
+#if _LIBCPP_STD_VER >= 14
+template <size_t _Len, class... _Types>
+using aligned_union_t _LIBCPP_DEPRECATED_IN_CXX23 = typename aligned_union<_Len, _Types...>::type;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_ALIGNED_UNION_H
diff --git a/libcxx/include/__cxx03/__type_traits/alignment_of.h b/libcxx/include/__cxx03/__type_traits/alignment_of.h
new file mode 100644
index 000000000000..f2d069bf2488
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/alignment_of.h
@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_ALIGNMENT_OF_H
+#define _LIBCPP___TYPE_TRAITS_ALIGNMENT_OF_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS alignment_of : public integral_constant<size_t, _LIBCPP_ALIGNOF(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr size_t alignment_of_v = _LIBCPP_ALIGNOF(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_ALIGNMENT_OF_H
diff --git a/libcxx/include/__cxx03/__type_traits/can_extract_key.h b/libcxx/include/__cxx03/__type_traits/can_extract_key.h
new file mode 100644
index 000000000000..b8359d070881
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/can_extract_key.h
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_CAN_EXTRACT_KEY_H
+#define _LIBCPP___TYPE_TRAITS_CAN_EXTRACT_KEY_H
+
+#include <__config>
+#include <__fwd/pair.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_const.h>
+#include <__type_traits/remove_const_ref.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// These traits are used in __tree and __hash_table
+struct __extract_key_fail_tag {};
+struct __extract_key_self_tag {};
+struct __extract_key_first_tag {};
+
+template <class _ValTy, class _Key, class _RawValTy = __remove_const_ref_t<_ValTy> >
+struct __can_extract_key
+    : __conditional_t<_IsSame<_RawValTy, _Key>::value, __extract_key_self_tag, __extract_key_fail_tag> {};
+
+template <class _Pair, class _Key, class _First, class _Second>
+struct __can_extract_key<_Pair, _Key, pair<_First, _Second> >
+    : __conditional_t<_IsSame<__remove_const_t<_First>, _Key>::value, __extract_key_first_tag, __extract_key_fail_tag> {
+};
+
+// __can_extract_map_key uses true_type/false_type instead of the tags.
+// It returns true if _Key != _ContainerValueTy (the container is a map not a set)
+// and _ValTy == _Key.
+template <class _ValTy, class _Key, class _ContainerValueTy, class _RawValTy = __remove_const_ref_t<_ValTy> >
+struct __can_extract_map_key : integral_constant<bool, _IsSame<_RawValTy, _Key>::value> {};
+
+// This specialization returns __extract_key_fail_tag for non-map containers
+// because _Key == _ContainerValueTy
+template <class _ValTy, class _Key, class _RawValTy>
+struct __can_extract_map_key<_ValTy, _Key, _Key, _RawValTy> : false_type {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_CAN_EXTRACT_KEY_H
diff --git a/libcxx/include/__cxx03/__type_traits/common_reference.h b/libcxx/include/__cxx03/__type_traits/common_reference.h
new file mode 100644
index 000000000000..c802902eb19f
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/common_reference.h
@@ -0,0 +1,193 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_COMMON_REFERENCE_H
+#define _LIBCPP___TYPE_TRAITS_COMMON_REFERENCE_H
+
+#include <__config>
+#include <__type_traits/common_type.h>
+#include <__type_traits/copy_cv.h>
+#include <__type_traits/copy_cvref.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/declval.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// common_reference
+#if _LIBCPP_STD_VER >= 20
+// Let COND_RES(X, Y) be:
+template <class _Xp, class _Yp>
+using __cond_res = decltype(false ? std::declval<_Xp (&)()>()() : std::declval<_Yp (&)()>()());
+
+// Let `XREF(A)` denote a unary alias template `T` such that `T<U>` denotes the same type as `U`
+// with the addition of `A`'s cv and reference qualifiers, for a non-reference cv-unqualified type
+// `U`.
+// [Note: `XREF(A)` is `__xref<A>::template __apply`]
+template <class _Tp>
+struct __xref {
+  template <class _Up>
+  using __apply = __copy_cvref_t<_Tp, _Up>;
+};
+
+// Given types A and B, let X be remove_reference_t<A>, let Y be remove_reference_t<B>,
+// and let COMMON-REF(A, B) be:
+template <class _Ap, class _Bp, class _Xp = remove_reference_t<_Ap>, class _Yp = remove_reference_t<_Bp>>
+struct __common_ref;
+
+template <class _Xp, class _Yp>
+using __common_ref_t = typename __common_ref<_Xp, _Yp>::__type;
+
+template <class _Xp, class _Yp>
+using __cv_cond_res = __cond_res<__copy_cv_t<_Xp, _Yp>&, __copy_cv_t<_Yp, _Xp>&>;
+
+//    If A and B are both lvalue reference types, COMMON-REF(A, B) is
+//    COND-RES(COPYCV(X, Y)&, COPYCV(Y, X)&) if that type exists and is a reference type.
+// clang-format off
+template <class _Ap, class _Bp, class _Xp, class _Yp>
+  requires
+    requires { typename __cv_cond_res<_Xp, _Yp>; } &&
+    is_reference_v<__cv_cond_res<_Xp, _Yp>>
+struct __common_ref<_Ap&, _Bp&, _Xp, _Yp> {
+  using __type = __cv_cond_res<_Xp, _Yp>;
+};
+// clang-format on
+
+//    Otherwise, let C be remove_reference_t<COMMON-REF(X&, Y&)>&&. ...
+template <class _Xp, class _Yp>
+using __common_ref_C = remove_reference_t<__common_ref_t<_Xp&, _Yp&>>&&;
+
+//    .... If A and B are both rvalue reference types, C is well-formed, and
+//    is_convertible_v<A, C> && is_convertible_v<B, C> is true, then COMMON-REF(A, B) is C.
+// clang-format off
+template <class _Ap, class _Bp, class _Xp, class _Yp>
+  requires
+    requires { typename __common_ref_C<_Xp, _Yp>; } &&
+    is_convertible_v<_Ap&&, __common_ref_C<_Xp, _Yp>> &&
+    is_convertible_v<_Bp&&, __common_ref_C<_Xp, _Yp>>
+struct __common_ref<_Ap&&, _Bp&&, _Xp, _Yp> {
+  using __type = __common_ref_C<_Xp, _Yp>;
+};
+// clang-format on
+
+//    Otherwise, let D be COMMON-REF(const X&, Y&). ...
+template <class _Tp, class _Up>
+using __common_ref_D = __common_ref_t<const _Tp&, _Up&>;
+
+//    ... If A is an rvalue reference and B is an lvalue reference and D is well-formed and
+//    is_convertible_v<A, D> is true, then COMMON-REF(A, B) is D.
+// clang-format off
+template <class _Ap, class _Bp, class _Xp, class _Yp>
+  requires
+    requires { typename __common_ref_D<_Xp, _Yp>; } &&
+    is_convertible_v<_Ap&&, __common_ref_D<_Xp, _Yp>>
+struct __common_ref<_Ap&&, _Bp&, _Xp, _Yp> {
+  using __type = __common_ref_D<_Xp, _Yp>;
+};
+// clang-format on
+
+//    Otherwise, if A is an lvalue reference and B is an rvalue reference, then
+//    COMMON-REF(A, B) is COMMON-REF(B, A).
+template <class _Ap, class _Bp, class _Xp, class _Yp>
+struct __common_ref<_Ap&, _Bp&&, _Xp, _Yp> : __common_ref<_Bp&&, _Ap&> {};
+
+//    Otherwise, COMMON-REF(A, B) is ill-formed.
+template <class _Ap, class _Bp, class _Xp, class _Yp>
+struct __common_ref {};
+
+// Note C: For the common_reference trait applied to a parameter pack [...]
+
+template <class...>
+struct common_reference;
+
+template <class... _Types>
+using common_reference_t = typename common_reference<_Types...>::type;
+
+// bullet 1 - sizeof...(T) == 0
+template <>
+struct common_reference<> {};
+
+// bullet 2 - sizeof...(T) == 1
+template <class _Tp>
+struct common_reference<_Tp> {
+  using type = _Tp;
+};
+
+// bullet 3 - sizeof...(T) == 2
+template <class _Tp, class _Up>
+struct __common_reference_sub_bullet3;
+template <class _Tp, class _Up>
+struct __common_reference_sub_bullet2 : __common_reference_sub_bullet3<_Tp, _Up> {};
+template <class _Tp, class _Up>
+struct __common_reference_sub_bullet1 : __common_reference_sub_bullet2<_Tp, _Up> {};
+
+// sub-bullet 1 - If T1 and T2 are reference types and COMMON-REF(T1, T2) is well-formed, then
+// the member typedef `type` denotes that type.
+template <class _Tp, class _Up>
+struct common_reference<_Tp, _Up> : __common_reference_sub_bullet1<_Tp, _Up> {};
+
+template <class _Tp, class _Up>
+  requires is_reference_v<_Tp> && is_reference_v<_Up> && requires { typename __common_ref_t<_Tp, _Up>; }
+struct __common_reference_sub_bullet1<_Tp, _Up> {
+  using type = __common_ref_t<_Tp, _Up>;
+};
+
+// sub-bullet 2 - Otherwise, if basic_common_reference<remove_cvref_t<T1>, remove_cvref_t<T2>, XREF(T1), XREF(T2)>::type
+// is well-formed, then the member typedef `type` denotes that type.
+template <class, class, template <class> class, template <class> class>
+struct basic_common_reference {};
+
+template <class _Tp, class _Up>
+using __basic_common_reference_t =
+    typename basic_common_reference<remove_cvref_t<_Tp>,
+                                    remove_cvref_t<_Up>,
+                                    __xref<_Tp>::template __apply,
+                                    __xref<_Up>::template __apply>::type;
+
+template <class _Tp, class _Up>
+  requires requires { typename __basic_common_reference_t<_Tp, _Up>; }
+struct __common_reference_sub_bullet2<_Tp, _Up> {
+  using type = __basic_common_reference_t<_Tp, _Up>;
+};
+
+// sub-bullet 3 - Otherwise, if COND-RES(T1, T2) is well-formed,
+// then the member typedef `type` denotes that type.
+template <class _Tp, class _Up>
+  requires requires { typename __cond_res<_Tp, _Up>; }
+struct __common_reference_sub_bullet3<_Tp, _Up> {
+  using type = __cond_res<_Tp, _Up>;
+};
+
+// sub-bullet 4 & 5 - Otherwise, if common_type_t<T1, T2> is well-formed,
+//                    then the member typedef `type` denotes that type.
+//                  - Otherwise, there shall be no member `type`.
+template <class _Tp, class _Up>
+struct __common_reference_sub_bullet3 : common_type<_Tp, _Up> {};
+
+// bullet 4 - If there is such a type `C`, the member typedef type shall denote the same type, if
+//            any, as `common_reference_t<C, Rest...>`.
+template <class _Tp, class _Up, class _Vp, class... _Rest>
+  requires requires { typename common_reference_t<_Tp, _Up>; }
+struct common_reference<_Tp, _Up, _Vp, _Rest...> : common_reference<common_reference_t<_Tp, _Up>, _Vp, _Rest...> {};
+
+// bullet 5 - Otherwise, there shall be no member `type`.
+template <class...>
+struct common_reference {};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_COMMON_REFERENCE_H
diff --git a/libcxx/include/__cxx03/__type_traits/common_type.h b/libcxx/include/__cxx03/__type_traits/common_type.h
new file mode 100644
index 000000000000..f6bd9ed71b7a
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/common_type.h
@@ -0,0 +1,102 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_COMMON_TYPE_H
+#define _LIBCPP___TYPE_TRAITS_COMMON_TYPE_H
+
+#include <__config>
+#include <__type_traits/conditional.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+// Let COND_RES(X, Y) be:
+template <class _Tp, class _Up>
+using __cond_type = decltype(false ? std::declval<_Tp>() : std::declval<_Up>());
+
+template <class _Tp, class _Up, class = void>
+struct __common_type3 {};
+
+// sub-bullet 4 - "if COND_RES(CREF(D1), CREF(D2)) denotes a type..."
+template <class _Tp, class _Up>
+struct __common_type3<_Tp, _Up, void_t<__cond_type<const _Tp&, const _Up&>>> {
+  using type = remove_cvref_t<__cond_type<const _Tp&, const _Up&>>;
+};
+
+template <class _Tp, class _Up, class = void>
+struct __common_type2_imp : __common_type3<_Tp, _Up> {};
+#else
+template <class _Tp, class _Up, class = void>
+struct __common_type2_imp {};
+#endif
+
+// sub-bullet 3 - "if decay_t<decltype(false ? declval<D1>() : declval<D2>())> ..."
+template <class _Tp, class _Up>
+struct __common_type2_imp<_Tp, _Up, __void_t<decltype(true ? std::declval<_Tp>() : std::declval<_Up>())> > {
+  typedef _LIBCPP_NODEBUG __decay_t<decltype(true ? std::declval<_Tp>() : std::declval<_Up>())> type;
+};
+
+template <class, class = void>
+struct __common_type_impl {};
+
+template <class... _Tp>
+struct __common_types;
+template <class... _Tp>
+struct _LIBCPP_TEMPLATE_VIS common_type;
+
+template <class _Tp, class _Up>
+struct __common_type_impl< __common_types<_Tp, _Up>, __void_t<typename common_type<_Tp, _Up>::type> > {
+  typedef typename common_type<_Tp, _Up>::type type;
+};
+
+template <class _Tp, class _Up, class _Vp, class... _Rest>
+struct __common_type_impl<__common_types<_Tp, _Up, _Vp, _Rest...>, __void_t<typename common_type<_Tp, _Up>::type> >
+    : __common_type_impl<__common_types<typename common_type<_Tp, _Up>::type, _Vp, _Rest...> > {};
+
+// bullet 1 - sizeof...(Tp) == 0
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS common_type<> {};
+
+// bullet 2 - sizeof...(Tp) == 1
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS common_type<_Tp> : public common_type<_Tp, _Tp> {};
+
+// bullet 3 - sizeof...(Tp) == 2
+
+// sub-bullet 1 - "If is_same_v<T1, D1> is false or ..."
+template <class _Tp, class _Up>
+struct _LIBCPP_TEMPLATE_VIS common_type<_Tp, _Up>
+    : __conditional_t<_IsSame<_Tp, __decay_t<_Tp> >::value && _IsSame<_Up, __decay_t<_Up> >::value,
+                      __common_type2_imp<_Tp, _Up>,
+                      common_type<__decay_t<_Tp>, __decay_t<_Up> > > {};
+
+// bullet 4 - sizeof...(Tp) > 2
+
+template <class _Tp, class _Up, class _Vp, class... _Rest>
+struct _LIBCPP_TEMPLATE_VIS common_type<_Tp, _Up, _Vp, _Rest...>
+    : __common_type_impl<__common_types<_Tp, _Up, _Vp, _Rest...> > {};
+
+#if _LIBCPP_STD_VER >= 14
+template <class... _Tp>
+using common_type_t = typename common_type<_Tp...>::type;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_COMMON_TYPE_H
diff --git a/libcxx/include/__cxx03/__type_traits/conditional.h b/libcxx/include/__cxx03/__type_traits/conditional.h
new file mode 100644
index 000000000000..5b5445a83742
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/conditional.h
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_CONDITIONAL_H
+#define _LIBCPP___TYPE_TRAITS_CONDITIONAL_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <bool>
+struct _IfImpl;
+
+template <>
+struct _IfImpl<true> {
+  template <class _IfRes, class _ElseRes>
+  using _Select _LIBCPP_NODEBUG = _IfRes;
+};
+
+template <>
+struct _IfImpl<false> {
+  template <class _IfRes, class _ElseRes>
+  using _Select _LIBCPP_NODEBUG = _ElseRes;
+};
+
+template <bool _Cond, class _IfRes, class _ElseRes>
+using _If _LIBCPP_NODEBUG = typename _IfImpl<_Cond>::template _Select<_IfRes, _ElseRes>;
+
+template <bool _Bp, class _If, class _Then>
+struct _LIBCPP_TEMPLATE_VIS conditional {
+  using type _LIBCPP_NODEBUG = _If;
+};
+template <class _If, class _Then>
+struct _LIBCPP_TEMPLATE_VIS conditional<false, _If, _Then> {
+  using type _LIBCPP_NODEBUG = _Then;
+};
+
+#if _LIBCPP_STD_VER >= 14
+template <bool _Bp, class _IfRes, class _ElseRes>
+using conditional_t _LIBCPP_NODEBUG = typename conditional<_Bp, _IfRes, _ElseRes>::type;
+#endif
+
+// Helper so we can use "conditional_t" in all language versions.
+template <bool _Bp, class _If, class _Then>
+using __conditional_t _LIBCPP_NODEBUG = typename conditional<_Bp, _If, _Then>::type;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_CONDITIONAL_H
diff --git a/libcxx/include/__cxx03/__type_traits/conjunction.h b/libcxx/include/__cxx03/__type_traits/conjunction.h
new file mode 100644
index 000000000000..c2995591bbc2
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/conjunction.h
@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_CONJUNCTION_H
+#define _LIBCPP___TYPE_TRAITS_CONJUNCTION_H
+
+#include <__config>
+#include <__type_traits/conditional.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_same.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class...>
+using __expand_to_true = true_type;
+
+template <class... _Pred>
+__expand_to_true<__enable_if_t<_Pred::value>...> __and_helper(int);
+
+template <class...>
+false_type __and_helper(...);
+
+// _And always performs lazy evaluation of its arguments.
+//
+// However, `_And<_Pred...>` itself will evaluate its result immediately (without having to
+// be instantiated) since it is an alias, unlike `conjunction<_Pred...>`, which is a struct.
+// If you want to defer the evaluation of `_And<_Pred...>` itself, use `_Lazy<_And, _Pred...>`.
+template <class... _Pred>
+using _And _LIBCPP_NODEBUG = decltype(std::__and_helper<_Pred...>(0));
+
+template <bool... _Preds>
+struct __all_dummy;
+
+template <bool... _Pred>
+struct __all : _IsSame<__all_dummy<_Pred...>, __all_dummy<((void)_Pred, true)...> > {};
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class...>
+struct conjunction : true_type {};
+
+template <class _Arg>
+struct conjunction<_Arg> : _Arg {};
+
+template <class _Arg, class... _Args>
+struct conjunction<_Arg, _Args...> : conditional_t<!bool(_Arg::value), _Arg, conjunction<_Args...>> {};
+
+template <class... _Args>
+inline constexpr bool conjunction_v = conjunction<_Args...>::value;
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_CONJUNCTION_H
diff --git a/libcxx/include/__cxx03/__type_traits/copy_cv.h b/libcxx/include/__cxx03/__type_traits/copy_cv.h
new file mode 100644
index 000000000000..d482cb42bffe
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/copy_cv.h
@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_COPY_CV_H
+#define _LIBCPP___TYPE_TRAITS_COPY_CV_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Let COPYCV(FROM, TO) be an alias for type TO with the addition of FROM's
+// top-level cv-qualifiers.
+template <class _From>
+struct __copy_cv {
+  template <class _To>
+  using __apply = _To;
+};
+
+template <class _From>
+struct __copy_cv<const _From> {
+  template <class _To>
+  using __apply = const _To;
+};
+
+template <class _From>
+struct __copy_cv<volatile _From> {
+  template <class _To>
+  using __apply = volatile _To;
+};
+
+template <class _From>
+struct __copy_cv<const volatile _From> {
+  template <class _To>
+  using __apply = const volatile _To;
+};
+
+template <class _From, class _To>
+using __copy_cv_t = typename __copy_cv<_From>::template __apply<_To>;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_COPY_CV_H
diff --git a/libcxx/include/__cxx03/__type_traits/copy_cvref.h b/libcxx/include/__cxx03/__type_traits/copy_cvref.h
new file mode 100644
index 000000000000..8bbf8efdf44d
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/copy_cvref.h
@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_COPY_CVREF_H
+#define _LIBCPP___TYPE_TRAITS_COPY_CVREF_H
+
+#include <__config>
+#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/copy_cv.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _From, class _To>
+struct __copy_cvref {
+  using type = __copy_cv_t<_From, _To>;
+};
+
+template <class _From, class _To>
+struct __copy_cvref<_From&, _To> {
+  using type = __add_lvalue_reference_t<__copy_cv_t<_From, _To> >;
+};
+
+template <class _From, class _To>
+struct __copy_cvref<_From&&, _To> {
+  using type = __add_rvalue_reference_t<__copy_cv_t<_From, _To> >;
+};
+
+template <class _From, class _To>
+using __copy_cvref_t = typename __copy_cvref<_From, _To>::type;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_COPY_CVREF_H
diff --git a/libcxx/include/__cxx03/__type_traits/datasizeof.h b/libcxx/include/__cxx03/__type_traits/datasizeof.h
new file mode 100644
index 000000000000..a27baf67cc2d
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/datasizeof.h
@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_DATASIZEOF_H
+#define _LIBCPP___TYPE_TRAITS_DATASIZEOF_H
+
+#include <__config>
+#include <__type_traits/is_class.h>
+#include <__type_traits/is_final.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+// This trait provides the size of a type excluding any tail padding.
+//
+// It is useful in contexts where performing an operation using the full size of the class (including padding) may
+// have unintended side effects, such as overwriting a derived class' member when writing the tail padding of a class
+// through a pointer-to-base.
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_keyword(__datasizeof) || __has_extension(datasizeof)
+template <class _Tp>
+inline const size_t __datasizeof_v = __datasizeof(_Tp);
+#else
+// NOLINTNEXTLINE(readability-redundant-preprocessor) This is https://llvm.org/PR64825
+#  if __has_cpp_attribute(__no_unique_address__)
+template <class _Tp>
+struct _FirstPaddingByte {
+  [[__no_unique_address__]] _Tp __v_;
+  char __first_padding_byte_;
+};
+#  else
+template <class _Tp, bool = __libcpp_is_final<_Tp>::value || !is_class<_Tp>::value>
+struct _FirstPaddingByte : _Tp {
+  char __first_padding_byte_;
+};
+
+template <class _Tp>
+struct _FirstPaddingByte<_Tp, true> {
+  _Tp __v_;
+  char __first_padding_byte_;
+};
+#  endif // __has_cpp_attribute(__no_unique_address__)
+
+// _FirstPaddingByte<> is sometimes non-standard layout. Using `offsetof` is UB in that case, but GCC and Clang allow
+// the use as an extension.
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Winvalid-offsetof")
+_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Winvalid-offsetof")
+template <class _Tp>
+inline const size_t __datasizeof_v = offsetof(_FirstPaddingByte<_Tp>, __first_padding_byte_);
+_LIBCPP_DIAGNOSTIC_POP
+#endif   // __has_extension(datasizeof)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_DATASIZEOF_H
diff --git a/libcxx/include/__cxx03/__type_traits/decay.h b/libcxx/include/__cxx03/__type_traits/decay.h
new file mode 100644
index 000000000000..7412044f9317
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/decay.h
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_DECAY_H
+#define _LIBCPP___TYPE_TRAITS_DECAY_H
+
+#include <__config>
+#include <__type_traits/add_pointer.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_function.h>
+#include <__type_traits/is_referenceable.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/remove_extent.h>
+#include <__type_traits/remove_reference.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__decay)
+template <class _Tp>
+using __decay_t _LIBCPP_NODEBUG = __decay(_Tp);
+
+template <class _Tp>
+struct decay {
+  using type _LIBCPP_NODEBUG = __decay_t<_Tp>;
+};
+
+#else
+template <class _Up, bool>
+struct __decay {
+  typedef _LIBCPP_NODEBUG __remove_cv_t<_Up> type;
+};
+
+template <class _Up>
+struct __decay<_Up, true> {
+public:
+  typedef _LIBCPP_NODEBUG
+      __conditional_t<is_array<_Up>::value,
+                      __add_pointer_t<__remove_extent_t<_Up> >,
+                      __conditional_t<is_function<_Up>::value, typename add_pointer<_Up>::type, __remove_cv_t<_Up> > >
+          type;
+};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS decay {
+private:
+  typedef _LIBCPP_NODEBUG __libcpp_remove_reference_t<_Tp> _Up;
+
+public:
+  typedef _LIBCPP_NODEBUG typename __decay<_Up, __libcpp_is_referenceable<_Up>::value>::type type;
+};
+
+template <class _Tp>
+using __decay_t = typename decay<_Tp>::type;
+#endif // __has_builtin(__decay)
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using decay_t = __decay_t<_Tp>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_DECAY_H
diff --git a/libcxx/include/__cxx03/__type_traits/dependent_type.h b/libcxx/include/__cxx03/__type_traits/dependent_type.h
new file mode 100644
index 000000000000..db8a869820db
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/dependent_type.h
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_DEPENDENT_TYPE_H
+#define _LIBCPP___TYPE_TRAITS_DEPENDENT_TYPE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, bool>
+struct _LIBCPP_TEMPLATE_VIS __dependent_type : public _Tp {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_DEPENDENT_TYPE_H
diff --git a/libcxx/include/__cxx03/__type_traits/desugars_to.h b/libcxx/include/__cxx03/__type_traits/desugars_to.h
new file mode 100644
index 000000000000..97a2ee5448f2
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/desugars_to.h
@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_DESUGARS_TO_H
+#define _LIBCPP___TYPE_TRAITS_DESUGARS_TO_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Tags to represent the canonical operations
+struct __equal_tag {};
+struct __plus_tag {};
+struct __less_tag {};
+
+// This class template is used to determine whether an operation "desugars"
+// (or boils down) to a given canonical operation.
+//
+// For example, `std::equal_to<>`, our internal `std::__equal_to` helper and
+// `ranges::equal_to` are all just fancy ways of representing a transparent
+// equality operation, so they all desugar to `__equal_tag`.
+//
+// This is useful to optimize some functions in cases where we know e.g. the
+// predicate being passed is actually going to call a builtin operator, or has
+// some specific semantics.
+template <class _CanonicalTag, class _Operation, class... _Args>
+inline const bool __desugars_to_v = false;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_DESUGARS_TO_H
diff --git a/libcxx/include/__cxx03/__type_traits/disjunction.h b/libcxx/include/__cxx03/__type_traits/disjunction.h
new file mode 100644
index 000000000000..2c89528d9f2f
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/disjunction.h
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_DISJUNCTION_H
+#define _LIBCPP___TYPE_TRAITS_DISJUNCTION_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <bool>
+struct _OrImpl;
+
+template <>
+struct _OrImpl<true> {
+  template <class _Res, class _First, class... _Rest>
+  using _Result _LIBCPP_NODEBUG =
+      typename _OrImpl<!bool(_First::value) && sizeof...(_Rest) != 0>::template _Result<_First, _Rest...>;
+};
+
+template <>
+struct _OrImpl<false> {
+  template <class _Res, class...>
+  using _Result = _Res;
+};
+
+// _Or always performs lazy evaluation of its arguments.
+//
+// However, `_Or<_Pred...>` itself will evaluate its result immediately (without having to
+// be instantiated) since it is an alias, unlike `disjunction<_Pred...>`, which is a struct.
+// If you want to defer the evaluation of `_Or<_Pred...>` itself, use `_Lazy<_Or, _Pred...>`
+// or `disjunction<_Pred...>` directly.
+template <class... _Args>
+using _Or _LIBCPP_NODEBUG = typename _OrImpl<sizeof...(_Args) != 0>::template _Result<false_type, _Args...>;
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class... _Args>
+struct disjunction : _Or<_Args...> {};
+
+template <class... _Args>
+inline constexpr bool disjunction_v = _Or<_Args...>::value;
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_DISJUNCTION_H
diff --git a/libcxx/include/__cxx03/__type_traits/enable_if.h b/libcxx/include/__cxx03/__type_traits/enable_if.h
new file mode 100644
index 000000000000..77da9622ca28
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/enable_if.h
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_ENABLE_IF_H
+#define _LIBCPP___TYPE_TRAITS_ENABLE_IF_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <bool, class _Tp = void>
+struct _LIBCPP_TEMPLATE_VIS enable_if {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS enable_if<true, _Tp> {
+  typedef _Tp type;
+};
+
+template <bool _Bp, class _Tp = void>
+using __enable_if_t _LIBCPP_NODEBUG = typename enable_if<_Bp, _Tp>::type;
+
+#if _LIBCPP_STD_VER >= 14
+template <bool _Bp, class _Tp = void>
+using enable_if_t = typename enable_if<_Bp, _Tp>::type;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_ENABLE_IF_H
diff --git a/libcxx/include/__cxx03/__type_traits/extent.h b/libcxx/include/__cxx03/__type_traits/extent.h
new file mode 100644
index 000000000000..bab03fe997eb
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/extent.h
@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_EXTENT_H
+#define _LIBCPP___TYPE_TRAITS_EXTENT_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__array_extent)
+
+template <class _Tp, size_t _Dim = 0>
+struct _LIBCPP_TEMPLATE_VIS extent : integral_constant<size_t, __array_extent(_Tp, _Dim)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp, unsigned _Ip = 0>
+inline constexpr size_t extent_v = __array_extent(_Tp, _Ip);
+#  endif
+
+#else // __has_builtin(__array_extent)
+
+template <class _Tp, unsigned _Ip = 0>
+struct _LIBCPP_TEMPLATE_VIS extent : public integral_constant<size_t, 0> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS extent<_Tp[], 0> : public integral_constant<size_t, 0> {};
+template <class _Tp, unsigned _Ip>
+struct _LIBCPP_TEMPLATE_VIS extent<_Tp[], _Ip> : public integral_constant<size_t, extent<_Tp, _Ip - 1>::value> {};
+template <class _Tp, size_t _Np>
+struct _LIBCPP_TEMPLATE_VIS extent<_Tp[_Np], 0> : public integral_constant<size_t, _Np> {};
+template <class _Tp, size_t _Np, unsigned _Ip>
+struct _LIBCPP_TEMPLATE_VIS extent<_Tp[_Np], _Ip> : public integral_constant<size_t, extent<_Tp, _Ip - 1>::value> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp, unsigned _Ip = 0>
+inline constexpr size_t extent_v = extent<_Tp, _Ip>::value;
+#  endif
+
+#endif // __has_builtin(__array_extent)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_EXTENT_H
diff --git a/libcxx/include/__cxx03/__type_traits/has_unique_object_representation.h b/libcxx/include/__cxx03/__type_traits/has_unique_object_representation.h
new file mode 100644
index 000000000000..98c440c16bf2
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/has_unique_object_representation.h
@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_HAS_UNIQUE_OBJECT_REPRESENTATION_H
+#define _LIBCPP___TYPE_TRAITS_HAS_UNIQUE_OBJECT_REPRESENTATION_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/remove_all_extents.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS has_unique_object_representations
+    // TODO: We work around a Clang and GCC bug in __has_unique_object_representations by using remove_all_extents
+    //       even though it should not be necessary. This was reported to the compilers:
+    //         - Clang: https://github.com/llvm/llvm-project/issues/95311
+    //         - GCC: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115476
+    //       remove_all_extents_t can be removed once all the compilers we support have fixed this bug.
+    : public integral_constant<bool, __has_unique_object_representations(remove_all_extents_t<_Tp>)> {};
+
+template <class _Tp>
+inline constexpr bool has_unique_object_representations_v = __has_unique_object_representations(_Tp);
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_HAS_UNIQUE_OBJECT_REPRESENTATION_H
diff --git a/libcxx/include/__cxx03/__type_traits/has_virtual_destructor.h b/libcxx/include/__cxx03/__type_traits/has_virtual_destructor.h
new file mode 100644
index 000000000000..4ce96e649e67
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/has_virtual_destructor.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_HAS_VIRTUAL_DESTRUCTOR_H
+#define _LIBCPP___TYPE_TRAITS_HAS_VIRTUAL_DESTRUCTOR_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS has_virtual_destructor : public integral_constant<bool, __has_virtual_destructor(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool has_virtual_destructor_v = __has_virtual_destructor(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_HAS_VIRTUAL_DESTRUCTOR_H
diff --git a/libcxx/include/__cxx03/__type_traits/integral_constant.h b/libcxx/include/__cxx03/__type_traits/integral_constant.h
new file mode 100644
index 000000000000..23e87e27feff
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/integral_constant.h
@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_INTEGRAL_CONSTANT_H
+#define _LIBCPP___TYPE_TRAITS_INTEGRAL_CONSTANT_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, _Tp __v>
+struct _LIBCPP_TEMPLATE_VIS integral_constant {
+  static _LIBCPP_CONSTEXPR const _Tp value = __v;
+  typedef _Tp value_type;
+  typedef integral_constant type;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR operator value_type() const _NOEXCEPT { return value; }
+#if _LIBCPP_STD_VER >= 14
+  _LIBCPP_HIDE_FROM_ABI constexpr value_type operator()() const _NOEXCEPT { return value; }
+#endif
+};
+
+template <class _Tp, _Tp __v>
+_LIBCPP_CONSTEXPR const _Tp integral_constant<_Tp, __v>::value;
+
+typedef integral_constant<bool, true> true_type;
+typedef integral_constant<bool, false> false_type;
+
+template <bool _Val>
+using _BoolConstant _LIBCPP_NODEBUG = integral_constant<bool, _Val>;
+
+#if _LIBCPP_STD_VER >= 17
+template <bool __b>
+using bool_constant = integral_constant<bool, __b>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_INTEGRAL_CONSTANT_H
diff --git a/libcxx/include/__cxx03/__type_traits/invoke.h b/libcxx/include/__cxx03/__type_traits/invoke.h
new file mode 100644
index 000000000000..71db32ae6a3c
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/invoke.h
@@ -0,0 +1,270 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_INVOKE_H
+#define _LIBCPP___TYPE_TRAITS_INVOKE_H
+
+#include <__config>
+#include <__type_traits/conditional.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_base_of.h>
+#include <__type_traits/is_core_convertible.h>
+#include <__type_traits/is_member_pointer.h>
+#include <__type_traits/is_reference_wrapper.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_void.h>
+#include <__type_traits/nat.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _DecayedFp>
+struct __member_pointer_class_type {};
+
+template <class _Ret, class _ClassType>
+struct __member_pointer_class_type<_Ret _ClassType::*> {
+  typedef _ClassType type;
+};
+
+template <class _Fp,
+          class _A0,
+          class _DecayFp = __decay_t<_Fp>,
+          class _DecayA0 = __decay_t<_A0>,
+          class _ClassT  = typename __member_pointer_class_type<_DecayFp>::type>
+using __enable_if_bullet1 =
+    __enable_if_t<is_member_function_pointer<_DecayFp>::value &&
+                  (is_same<_ClassT, _DecayA0>::value || is_base_of<_ClassT, _DecayA0>::value)>;
+
+template <class _Fp, class _A0, class _DecayFp = __decay_t<_Fp>, class _DecayA0 = __decay_t<_A0> >
+using __enable_if_bullet2 =
+    __enable_if_t<is_member_function_pointer<_DecayFp>::value && __is_reference_wrapper<_DecayA0>::value>;
+
+template <class _Fp,
+          class _A0,
+          class _DecayFp = __decay_t<_Fp>,
+          class _DecayA0 = __decay_t<_A0>,
+          class _ClassT  = typename __member_pointer_class_type<_DecayFp>::type>
+using __enable_if_bullet3 =
+    __enable_if_t<is_member_function_pointer<_DecayFp>::value &&
+                  !(is_same<_ClassT, _DecayA0>::value || is_base_of<_ClassT, _DecayA0>::value) &&
+                  !__is_reference_wrapper<_DecayA0>::value>;
+
+template <class _Fp,
+          class _A0,
+          class _DecayFp = __decay_t<_Fp>,
+          class _DecayA0 = __decay_t<_A0>,
+          class _ClassT  = typename __member_pointer_class_type<_DecayFp>::type>
+using __enable_if_bullet4 =
+    __enable_if_t<is_member_object_pointer<_DecayFp>::value &&
+                  (is_same<_ClassT, _DecayA0>::value || is_base_of<_ClassT, _DecayA0>::value)>;
+
+template <class _Fp, class _A0, class _DecayFp = __decay_t<_Fp>, class _DecayA0 = __decay_t<_A0> >
+using __enable_if_bullet5 =
+    __enable_if_t<is_member_object_pointer<_DecayFp>::value && __is_reference_wrapper<_DecayA0>::value>;
+
+template <class _Fp,
+          class _A0,
+          class _DecayFp = __decay_t<_Fp>,
+          class _DecayA0 = __decay_t<_A0>,
+          class _ClassT  = typename __member_pointer_class_type<_DecayFp>::type>
+using __enable_if_bullet6 =
+    __enable_if_t<is_member_object_pointer<_DecayFp>::value &&
+                  !(is_same<_ClassT, _DecayA0>::value || is_base_of<_ClassT, _DecayA0>::value) &&
+                  !__is_reference_wrapper<_DecayA0>::value>;
+
+// __invoke forward declarations
+
+// fall back - none of the bullets
+
+template <class... _Args>
+__nat __invoke(_Args&&... __args);
+
+// bullets 1, 2 and 3
+
+// clang-format off
+template <class _Fp, class _A0, class... _Args, class = __enable_if_bullet1<_Fp, _A0> >
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
+decltype((std::declval<_A0>().*std::declval<_Fp>())(std::declval<_Args>()...))
+__invoke(_Fp&& __f, _A0&& __a0, _Args&&... __args)
+    _NOEXCEPT_(noexcept((static_cast<_A0&&>(__a0).*__f)(static_cast<_Args&&>(__args)...)))
+               { return (static_cast<_A0&&>(__a0).*__f)(static_cast<_Args&&>(__args)...); }
+
+template <class _Fp, class _A0, class... _Args, class = __enable_if_bullet2<_Fp, _A0> >
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
+decltype((std::declval<_A0>().get().*std::declval<_Fp>())(std::declval<_Args>()...))
+__invoke(_Fp&& __f, _A0&& __a0, _Args&&... __args)
+    _NOEXCEPT_(noexcept((__a0.get().*__f)(static_cast<_Args&&>(__args)...)))
+               { return (__a0.get().*__f)(static_cast<_Args&&>(__args)...); }
+
+template <class _Fp, class _A0, class... _Args, class = __enable_if_bullet3<_Fp, _A0> >
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
+decltype(((*std::declval<_A0>()).*std::declval<_Fp>())(std::declval<_Args>()...))
+__invoke(_Fp&& __f, _A0&& __a0, _Args&&... __args)
+    _NOEXCEPT_(noexcept(((*static_cast<_A0&&>(__a0)).*__f)(static_cast<_Args&&>(__args)...)))
+               { return ((*static_cast<_A0&&>(__a0)).*__f)(static_cast<_Args&&>(__args)...); }
+
+// bullets 4, 5 and 6
+
+template <class _Fp, class _A0, class = __enable_if_bullet4<_Fp, _A0> >
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
+decltype(std::declval<_A0>().*std::declval<_Fp>())
+__invoke(_Fp&& __f, _A0&& __a0)
+    _NOEXCEPT_(noexcept(static_cast<_A0&&>(__a0).*__f))
+               { return static_cast<_A0&&>(__a0).*__f; }
+
+template <class _Fp, class _A0, class = __enable_if_bullet5<_Fp, _A0> >
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
+decltype(std::declval<_A0>().get().*std::declval<_Fp>())
+__invoke(_Fp&& __f, _A0&& __a0)
+    _NOEXCEPT_(noexcept(__a0.get().*__f))
+               { return __a0.get().*__f; }
+
+template <class _Fp, class _A0, class = __enable_if_bullet6<_Fp, _A0> >
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
+decltype((*std::declval<_A0>()).*std::declval<_Fp>())
+__invoke(_Fp&& __f, _A0&& __a0)
+    _NOEXCEPT_(noexcept((*static_cast<_A0&&>(__a0)).*__f))
+               { return (*static_cast<_A0&&>(__a0)).*__f; }
+
+// bullet 7
+
+template <class _Fp, class... _Args>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
+decltype(std::declval<_Fp>()(std::declval<_Args>()...))
+__invoke(_Fp&& __f, _Args&&... __args)
+    _NOEXCEPT_(noexcept(static_cast<_Fp&&>(__f)(static_cast<_Args&&>(__args)...)))
+               { return static_cast<_Fp&&>(__f)(static_cast<_Args&&>(__args)...); }
+// clang-format on
+
+// __invokable
+template <class _Ret, class _Fp, class... _Args>
+struct __invokable_r {
+  template <class _XFp, class... _XArgs>
+  static decltype(std::__invoke(std::declval<_XFp>(), std::declval<_XArgs>()...)) __try_call(int);
+  template <class _XFp, class... _XArgs>
+  static __nat __try_call(...);
+
+  // FIXME: Check that _Ret, _Fp, and _Args... are all complete types, cv void,
+  // or incomplete array types as required by the standard.
+  using _Result = decltype(__try_call<_Fp, _Args...>(0));
+
+  using type              = __conditional_t<_IsNotSame<_Result, __nat>::value,
+                                            __conditional_t<is_void<_Ret>::value, true_type, __is_core_convertible<_Result, _Ret> >,
+                                            false_type>;
+  static const bool value = type::value;
+};
+template <class _Fp, class... _Args>
+using __invokable = __invokable_r<void, _Fp, _Args...>;
+
+template <bool _IsInvokable, bool _IsCVVoid, class _Ret, class _Fp, class... _Args>
+struct __nothrow_invokable_r_imp {
+  static const bool value = false;
+};
+
+template <class _Ret, class _Fp, class... _Args>
+struct __nothrow_invokable_r_imp<true, false, _Ret, _Fp, _Args...> {
+  typedef __nothrow_invokable_r_imp _ThisT;
+
+  template <class _Tp>
+  static void __test_noexcept(_Tp) _NOEXCEPT;
+
+#ifdef _LIBCPP_CXX03_LANG
+  static const bool value = false;
+#else
+  static const bool value =
+      noexcept(_ThisT::__test_noexcept<_Ret>(std::__invoke(std::declval<_Fp>(), std::declval<_Args>()...)));
+#endif
+};
+
+template <class _Ret, class _Fp, class... _Args>
+struct __nothrow_invokable_r_imp<true, true, _Ret, _Fp, _Args...> {
+#ifdef _LIBCPP_CXX03_LANG
+  static const bool value = false;
+#else
+  static const bool value = noexcept(std::__invoke(std::declval<_Fp>(), std::declval<_Args>()...));
+#endif
+};
+
+template <class _Ret, class _Fp, class... _Args>
+using __nothrow_invokable_r =
+    __nothrow_invokable_r_imp<__invokable_r<_Ret, _Fp, _Args...>::value, is_void<_Ret>::value, _Ret, _Fp, _Args...>;
+
+template <class _Fp, class... _Args>
+using __nothrow_invokable = __nothrow_invokable_r_imp<__invokable<_Fp, _Args...>::value, true, void, _Fp, _Args...>;
+
+template <class _Fp, class... _Args>
+struct __invoke_of
+    : public enable_if<__invokable<_Fp, _Args...>::value, typename __invokable_r<void, _Fp, _Args...>::_Result> {};
+
+template <class _Ret, bool = is_void<_Ret>::value>
+struct __invoke_void_return_wrapper {
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static _Ret __call(_Args&&... __args) {
+    return std::__invoke(std::forward<_Args>(__args)...);
+  }
+};
+
+template <class _Ret>
+struct __invoke_void_return_wrapper<_Ret, true> {
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void __call(_Args&&... __args) {
+    std::__invoke(std::forward<_Args>(__args)...);
+  }
+};
+
+#if _LIBCPP_STD_VER >= 17
+
+// is_invocable
+
+template <class _Fn, class... _Args>
+struct _LIBCPP_TEMPLATE_VIS is_invocable : integral_constant<bool, __invokable<_Fn, _Args...>::value> {};
+
+template <class _Ret, class _Fn, class... _Args>
+struct _LIBCPP_TEMPLATE_VIS is_invocable_r : integral_constant<bool, __invokable_r<_Ret, _Fn, _Args...>::value> {};
+
+template <class _Fn, class... _Args>
+inline constexpr bool is_invocable_v = is_invocable<_Fn, _Args...>::value;
+
+template <class _Ret, class _Fn, class... _Args>
+inline constexpr bool is_invocable_r_v = is_invocable_r<_Ret, _Fn, _Args...>::value;
+
+// is_nothrow_invocable
+
+template <class _Fn, class... _Args>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_invocable : integral_constant<bool, __nothrow_invokable<_Fn, _Args...>::value> {
+};
+
+template <class _Ret, class _Fn, class... _Args>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_invocable_r
+    : integral_constant<bool, __nothrow_invokable_r<_Ret, _Fn, _Args...>::value> {};
+
+template <class _Fn, class... _Args>
+inline constexpr bool is_nothrow_invocable_v = is_nothrow_invocable<_Fn, _Args...>::value;
+
+template <class _Ret, class _Fn, class... _Args>
+inline constexpr bool is_nothrow_invocable_r_v = is_nothrow_invocable_r<_Ret, _Fn, _Args...>::value;
+
+template <class _Fn, class... _Args>
+struct _LIBCPP_TEMPLATE_VIS invoke_result : __invoke_of<_Fn, _Args...> {};
+
+template <class _Fn, class... _Args>
+using invoke_result_t = typename invoke_result<_Fn, _Args...>::type;
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_INVOKE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_abstract.h b/libcxx/include/__cxx03/__type_traits/is_abstract.h
new file mode 100644
index 000000000000..4aa456be1c48
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_abstract.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_ABSTRACT_H
+#define _LIBCPP___TYPE_TRAITS_IS_ABSTRACT_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_abstract : public integral_constant<bool, __is_abstract(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_abstract_v = __is_abstract(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_ABSTRACT_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_aggregate.h b/libcxx/include/__cxx03/__type_traits/is_aggregate.h
new file mode 100644
index 000000000000..4e0988071ade
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_aggregate.h
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_AGGREGATE_H
+#define _LIBCPP___TYPE_TRAITS_IS_AGGREGATE_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_aggregate : public integral_constant<bool, __is_aggregate(_Tp)> {};
+
+template <class _Tp>
+inline constexpr bool is_aggregate_v = __is_aggregate(_Tp);
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_AGGREGATE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_allocator.h b/libcxx/include/__cxx03/__type_traits/is_allocator.h
new file mode 100644
index 000000000000..144ffac4d7ce
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_allocator.h
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_IS_ALLOCATOR_H
+#define _LIBCPP___TYPE_IS_ALLOCATOR_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <typename _Alloc, typename = void, typename = void>
+struct __is_allocator : false_type {};
+
+template <typename _Alloc>
+struct __is_allocator<_Alloc,
+                      __void_t<typename _Alloc::value_type>,
+                      __void_t<decltype(std::declval<_Alloc&>().allocate(size_t(0)))> > : true_type {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_IS_ALLOCATOR_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_always_bitcastable.h b/libcxx/include/__cxx03/__type_traits/is_always_bitcastable.h
new file mode 100644
index 000000000000..5bc650b41358
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_always_bitcastable.h
@@ -0,0 +1,85 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_ALWAYS_BITCASTABLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_ALWAYS_BITCASTABLE_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_object.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_trivially_copyable.h>
+#include <__type_traits/remove_cv.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Checks whether an object of type `From` can always be bit-cast to an object of type `To` and represent a valid value
+// of type `To`. In other words, `From` and `To` have the same value representation and the set of values of `From` is
+// a subset of the set of values of `To`.
+//
+// Note that types that cannot be assigned to each other using built-in assignment (e.g. arrays) might still be
+// considered bit-castable.
+template <class _From, class _To>
+struct __is_always_bitcastable {
+  using _UnqualFrom = __remove_cv_t<_From>;
+  using _UnqualTo   = __remove_cv_t<_To>;
+
+  // clang-format off
+  static const bool value =
+      // First, the simple case -- `From` and `To` are the same object type.
+      (is_same<_UnqualFrom, _UnqualTo>::value && is_trivially_copyable<_UnqualFrom>::value) ||
+
+      // Beyond the simple case, we say that one type is "always bit-castable" to another if:
+      // - (1) `From` and `To` have the same value representation, and in addition every possible value of `From` has
+      //   a corresponding value in the `To` type (in other words, the set of values of `To` is a superset of the set of
+      //   values of `From`);
+      // - (2) When the corresponding values are not the same value (as, for example, between an unsigned and a signed
+      //   integer, where a large positive value of the unsigned integer corresponds to a negative value in the signed
+      //   integer type), the value of `To` that results from a bitwise copy of `From` is the same what would be
+      //   produced by the built-in assignment (if it were defined for the two types, to which there are minor
+      //   exceptions, e.g. built-in arrays).
+      //
+      // In practice, that means:
+      // - all integral types (except `bool`, see below) -- that is, character types and `int` types, both signed and
+      //   unsigned...
+      // - as well as arrays of such types...
+      // - ...that have the same size.
+      //
+      // Other trivially-copyable types can't be validly bit-cast outside of their own type:
+      // - floating-point types normally have different sizes and thus aren't bit-castable between each other (fails
+      // #1);
+      // - integral types and floating-point types use different representations, so for example bit-casting an integral
+      //   `1` to `float` results in a very small less-than-one value, unlike built-in assignment that produces `1.0`
+      //   (fails #2);
+      // - booleans normally use only a single bit of their object representation; bit-casting an integer to a boolean
+      //   will result in a boolean object with an incorrect representation, which is undefined behavior (fails #2).
+      //   Bit-casting from a boolean into an integer, however, is valid;
+      // - enumeration types may have different ranges of possible values (fails #1);
+      // - for pointers, it is not guaranteed that pointers to different types use the same set of values to represent
+      //   addresses, and the conversion results are explicitly unspecified for types with different alignments
+      //   (fails #1);
+      // - for structs and unions it is impossible to determine whether the set of values of one of them is a subset of
+      //   the other (fails #1);
+      // - there is no need to consider `nullptr_t` for practical purposes.
+      (
+        sizeof(_From) == sizeof(_To) &&
+        is_integral<_From>::value &&
+        is_integral<_To>::value &&
+        !is_same<_UnqualTo, bool>::value
+      );
+  // clang-format on
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_ALWAYS_BITCASTABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_arithmetic.h b/libcxx/include/__cxx03/__type_traits/is_arithmetic.h
new file mode 100644
index 000000000000..c9713e1840a7
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_arithmetic.h
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_ARITHMETIC_H
+#define _LIBCPP___TYPE_TRAITS_IS_ARITHMETIC_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_floating_point.h>
+#include <__type_traits/is_integral.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_arithmetic
+    : public integral_constant<bool, is_integral<_Tp>::value || is_floating_point<_Tp>::value> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_arithmetic_v = is_arithmetic<_Tp>::value;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_ARITHMETIC_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_array.h b/libcxx/include/__cxx03/__type_traits/is_array.h
new file mode 100644
index 000000000000..dc23de28d2c6
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_array.h
@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_ARRAY_H
+#define _LIBCPP___TYPE_TRAITS_IS_ARRAY_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__is_array) &&                                                                                       \
+    (!defined(_LIBCPP_COMPILER_CLANG_BASED) || (defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1900))
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_array : _BoolConstant<__is_array(_Tp)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_array_v = __is_array(_Tp);
+#  endif
+
+#else
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_array : public false_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_array<_Tp[]> : public true_type {};
+template <class _Tp, size_t _Np>
+struct _LIBCPP_TEMPLATE_VIS is_array<_Tp[_Np]> : public true_type {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_array_v = is_array<_Tp>::value;
+#  endif
+
+#endif // __has_builtin(__is_array)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_ARRAY_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_assignable.h b/libcxx/include/__cxx03/__type_traits/is_assignable.h
new file mode 100644
index 000000000000..cfb469977787
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_assignable.h
@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_ASSIGNABLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_ASSIGNABLE_H
+
+#include <__config>
+#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Up>
+struct _LIBCPP_TEMPLATE_VIS is_assignable : _BoolConstant<__is_assignable(_Tp, _Up)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp, class _Arg>
+inline constexpr bool is_assignable_v = __is_assignable(_Tp, _Arg);
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_copy_assignable
+    : public integral_constant<bool,
+                               __is_assignable(__add_lvalue_reference_t<_Tp>, __add_lvalue_reference_t<const _Tp>)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_copy_assignable_v = is_copy_assignable<_Tp>::value;
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_move_assignable
+    : public integral_constant<bool, __is_assignable(__add_lvalue_reference_t<_Tp>, __add_rvalue_reference_t<_Tp>)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_move_assignable_v = is_move_assignable<_Tp>::value;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_ASSIGNABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_base_of.h b/libcxx/include/__cxx03/__type_traits/is_base_of.h
new file mode 100644
index 000000000000..090abeeb54dc
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_base_of.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_BASE_OF_H
+#define _LIBCPP___TYPE_TRAITS_IS_BASE_OF_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Bp, class _Dp>
+struct _LIBCPP_TEMPLATE_VIS is_base_of : public integral_constant<bool, __is_base_of(_Bp, _Dp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Bp, class _Dp>
+inline constexpr bool is_base_of_v = __is_base_of(_Bp, _Dp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_BASE_OF_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_bounded_array.h b/libcxx/include/__cxx03/__type_traits/is_bounded_array.h
new file mode 100644
index 000000000000..211403d638d0
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_bounded_array.h
@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_BOUNDED_ARRAY_H
+#define _LIBCPP___TYPE_TRAITS_IS_BOUNDED_ARRAY_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class>
+struct _LIBCPP_TEMPLATE_VIS __libcpp_is_bounded_array : false_type {};
+template <class _Tp, size_t _Np>
+struct _LIBCPP_TEMPLATE_VIS __libcpp_is_bounded_array<_Tp[_Np]> : true_type {};
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class>
+struct _LIBCPP_TEMPLATE_VIS is_bounded_array : false_type {};
+template <class _Tp, size_t _Np>
+struct _LIBCPP_TEMPLATE_VIS is_bounded_array<_Tp[_Np]> : true_type {};
+
+template <class _Tp>
+inline constexpr bool is_bounded_array_v = is_bounded_array<_Tp>::value;
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_BOUNDED_ARRAY_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_callable.h b/libcxx/include/__cxx03/__type_traits/is_callable.h
new file mode 100644
index 000000000000..49724fe892ee
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_callable.h
@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_CALLABLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_CALLABLE_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__utility/declval.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Func, class... _Args, class = decltype(std::declval<_Func>()(std::declval<_Args>()...))>
+true_type __is_callable_helper(int);
+template <class...>
+false_type __is_callable_helper(...);
+
+template <class _Func, class... _Args>
+struct __is_callable : decltype(std::__is_callable_helper<_Func, _Args...>(0)) {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_CALLABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_char_like_type.h b/libcxx/include/__cxx03/__type_traits/is_char_like_type.h
new file mode 100644
index 000000000000..26205843047c
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_char_like_type.h
@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_CHAR_LIKE_TYPE_H
+#define _LIBCPP___TYPE_TRAITS_IS_CHAR_LIKE_TYPE_H
+
+#include <__config>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/is_standard_layout.h>
+#include <__type_traits/is_trivial.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT>
+using _IsCharLikeType = _And<is_standard_layout<_CharT>, is_trivial<_CharT> >;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_CHAR_LIKE_TYPE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_class.h b/libcxx/include/__cxx03/__type_traits/is_class.h
new file mode 100644
index 000000000000..034f76a7865e
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_class.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_CLASS_H
+#define _LIBCPP___TYPE_TRAITS_IS_CLASS_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_class : public integral_constant<bool, __is_class(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_class_v = __is_class(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_CLASS_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_compound.h b/libcxx/include/__cxx03/__type_traits/is_compound.h
new file mode 100644
index 000000000000..cd208ceab288
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_compound.h
@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_COMPOUND_H
+#define _LIBCPP___TYPE_TRAITS_IS_COMPOUND_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_fundamental.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__is_compound)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_compound : _BoolConstant<__is_compound(_Tp)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_compound_v = __is_compound(_Tp);
+#  endif
+
+#else // __has_builtin(__is_compound)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_compound : public integral_constant<bool, !is_fundamental<_Tp>::value> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_compound_v = is_compound<_Tp>::value;
+#  endif
+
+#endif // __has_builtin(__is_compound)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_COMPOUND_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_const.h b/libcxx/include/__cxx03/__type_traits/is_const.h
new file mode 100644
index 000000000000..47ef70872b79
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_const.h
@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_CONST_H
+#define _LIBCPP___TYPE_TRAITS_IS_CONST_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__is_const)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_const : _BoolConstant<__is_const(_Tp)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_const_v = __is_const(_Tp);
+#  endif
+
+#else
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_const : public false_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_const<_Tp const> : public true_type {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_const_v = is_const<_Tp>::value;
+#  endif
+
+#endif // __has_builtin(__is_const)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_CONST_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_constant_evaluated.h b/libcxx/include/__cxx03/__type_traits/is_constant_evaluated.h
new file mode 100644
index 000000000000..05e070a74788
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_constant_evaluated.h
@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_CONSTANT_EVALUATED_H
+#define _LIBCPP___TYPE_TRAITS_IS_CONSTANT_EVALUATED_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_constant_evaluated() noexcept {
+  return __builtin_is_constant_evaluated();
+}
+#endif
+
+_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR bool __libcpp_is_constant_evaluated() _NOEXCEPT {
+  return __builtin_is_constant_evaluated();
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_CONSTANT_EVALUATED_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_constructible.h b/libcxx/include/__cxx03/__type_traits/is_constructible.h
new file mode 100644
index 000000000000..567bd165c715
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_constructible.h
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_IS_CONSTRUCTIBLE_H
+#define _LIBCPP___TYPE_IS_CONSTRUCTIBLE_H
+
+#include <__config>
+#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class... _Args>
+struct _LIBCPP_TEMPLATE_VIS is_constructible : public integral_constant<bool, __is_constructible(_Tp, _Args...)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp, class... _Args>
+inline constexpr bool is_constructible_v = __is_constructible(_Tp, _Args...);
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_copy_constructible
+    : public integral_constant<bool, __is_constructible(_Tp, __add_lvalue_reference_t<const _Tp>)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_copy_constructible_v = is_copy_constructible<_Tp>::value;
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_move_constructible
+    : public integral_constant<bool, __is_constructible(_Tp, __add_rvalue_reference_t<_Tp>)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_move_constructible_v = is_move_constructible<_Tp>::value;
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_default_constructible : public integral_constant<bool, __is_constructible(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_default_constructible_v = __is_constructible(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_IS_CONSTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_convertible.h b/libcxx/include/__cxx03/__type_traits/is_convertible.h
new file mode 100644
index 000000000000..414c2a6d6a0d
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_convertible.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_CONVERTIBLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_CONVERTIBLE_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _T1, class _T2>
+struct _LIBCPP_TEMPLATE_VIS is_convertible : public integral_constant<bool, __is_convertible(_T1, _T2)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _From, class _To>
+inline constexpr bool is_convertible_v = __is_convertible(_From, _To);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_CONVERTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_core_convertible.h b/libcxx/include/__cxx03/__type_traits/is_core_convertible.h
new file mode 100644
index 000000000000..0de177c7771f
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_core_convertible.h
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_CORE_CONVERTIBLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_CORE_CONVERTIBLE_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// [conv.general]/3 says "E is convertible to T" whenever "T t=E;" is well-formed.
+// We can't test for that, but we can test implicit convertibility by passing it
+// to a function. Notice that __is_core_convertible<void,void> is false,
+// and __is_core_convertible<immovable-type,immovable-type> is true in C++17 and later.
+
+template <class _Tp, class _Up, class = void>
+struct __is_core_convertible : public false_type {};
+
+template <class _Tp, class _Up>
+struct __is_core_convertible<_Tp, _Up, decltype(static_cast<void (*)(_Up)>(0)(static_cast<_Tp (*)()>(0)()))>
+    : public true_type {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_CORE_CONVERTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_destructible.h b/libcxx/include/__cxx03/__type_traits/is_destructible.h
new file mode 100644
index 000000000000..3248b07d36ee
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_destructible.h
@@ -0,0 +1,97 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_DESTRUCTIBLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_DESTRUCTIBLE_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_function.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/remove_all_extents.h>
+#include <__utility/declval.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__is_destructible)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_destructible : _BoolConstant<__is_destructible(_Tp)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_destructible_v = __is_destructible(_Tp);
+#  endif
+
+#else // __has_builtin(__is_destructible)
+
+//  if it's a reference, return true
+//  if it's a function, return false
+//  if it's   void,     return false
+//  if it's an array of unknown bound, return false
+//  Otherwise, return "declval<_Up&>().~_Up()" is well-formed
+//    where _Up is remove_all_extents<_Tp>::type
+
+template <class>
+struct __is_destructible_apply {
+  typedef int type;
+};
+
+template <typename _Tp>
+struct __is_destructor_wellformed {
+  template <typename _Tp1>
+  static true_type __test(typename __is_destructible_apply<decltype(std::declval<_Tp1&>().~_Tp1())>::type);
+
+  template <typename _Tp1>
+  static false_type __test(...);
+
+  static const bool value = decltype(__test<_Tp>(12))::value;
+};
+
+template <class _Tp, bool>
+struct __destructible_imp;
+
+template <class _Tp>
+struct __destructible_imp<_Tp, false>
+    : public integral_constant<bool, __is_destructor_wellformed<__remove_all_extents_t<_Tp> >::value> {};
+
+template <class _Tp>
+struct __destructible_imp<_Tp, true> : public true_type {};
+
+template <class _Tp, bool>
+struct __destructible_false;
+
+template <class _Tp>
+struct __destructible_false<_Tp, false> : public __destructible_imp<_Tp, is_reference<_Tp>::value> {};
+
+template <class _Tp>
+struct __destructible_false<_Tp, true> : public false_type {};
+
+template <class _Tp>
+struct is_destructible : public __destructible_false<_Tp, is_function<_Tp>::value> {};
+
+template <class _Tp>
+struct is_destructible<_Tp[]> : public false_type {};
+
+template <>
+struct is_destructible<void> : public false_type {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_destructible_v = is_destructible<_Tp>::value;
+#  endif
+
+#endif // __has_builtin(__is_destructible)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_DESTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_empty.h b/libcxx/include/__cxx03/__type_traits/is_empty.h
new file mode 100644
index 000000000000..951d93b5a2f1
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_empty.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_EMPTY_H
+#define _LIBCPP___TYPE_TRAITS_IS_EMPTY_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_empty : public integral_constant<bool, __is_empty(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_empty_v = __is_empty(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_EMPTY_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_enum.h b/libcxx/include/__cxx03/__type_traits/is_enum.h
new file mode 100644
index 000000000000..2fab6db2c8d5
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_enum.h
@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_ENUM_H
+#define _LIBCPP___TYPE_TRAITS_IS_ENUM_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_enum : public integral_constant<bool, __is_enum(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_enum_v = __is_enum(_Tp);
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_scoped_enum : bool_constant<__is_scoped_enum(_Tp)> {};
+
+template <class _Tp>
+inline constexpr bool is_scoped_enum_v = __is_scoped_enum(_Tp);
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_ENUM_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_equality_comparable.h b/libcxx/include/__cxx03/__type_traits/is_equality_comparable.h
new file mode 100644
index 000000000000..4397f743e5ee
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_equality_comparable.h
@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_EQUALITY_COMPARABLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_EQUALITY_COMPARABLE_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_signed.h>
+#include <__type_traits/is_void.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Up, class = void>
+struct __is_equality_comparable : false_type {};
+
+template <class _Tp, class _Up>
+struct __is_equality_comparable<_Tp, _Up, __void_t<decltype(std::declval<_Tp>() == std::declval<_Up>())> > : true_type {
+};
+
+// A type is_trivially_equality_comparable if the expression `a == b` is equivalent to `std::memcmp(&a, &b, sizeof(T))`
+// (with `a` and `b` being of type `T`). For the case where we compare two object of the same type, we can use
+// __is_trivially_equality_comparable. We have special-casing for pointers which point to the same type ignoring
+// cv-qualifications and comparing to void-pointers.
+//
+// The following types are not trivially equality comparable:
+// floating-point types: different bit-patterns can compare equal. (e.g 0.0 and -0.0)
+// enums: The user is allowed to specialize operator== for enums
+// pointers that don't have the same type (ignoring cv-qualifiers): pointers to virtual bases are equality comparable,
+//   but don't have the same bit-pattern. An exception to this is comparing to a void-pointer. There the bit-pattern is
+//   always compared.
+// objects with padding bytes: since objects with padding bytes may compare equal, even though their object
+//   representation may not be equivalent.
+
+template <class _Tp, class _Up, class = void>
+struct __libcpp_is_trivially_equality_comparable_impl : false_type {};
+
+template <class _Tp>
+struct __libcpp_is_trivially_equality_comparable_impl<_Tp, _Tp>
+#if __has_builtin(__is_trivially_equality_comparable)
+    : integral_constant<bool, __is_trivially_equality_comparable(_Tp) && __is_equality_comparable<_Tp, _Tp>::value> {
+};
+#else
+    : is_integral<_Tp> {
+};
+#endif // __has_builtin(__is_trivially_equality_comparable)
+
+template <class _Tp, class _Up>
+struct __libcpp_is_trivially_equality_comparable_impl<
+    _Tp,
+    _Up,
+    __enable_if_t<is_integral<_Tp>::value && is_integral<_Up>::value && !is_same<_Tp, _Up>::value &&
+                  is_signed<_Tp>::value == is_signed<_Up>::value && sizeof(_Tp) == sizeof(_Up)> > : true_type {};
+
+template <class _Tp>
+struct __libcpp_is_trivially_equality_comparable_impl<_Tp*, _Tp*> : true_type {};
+
+// TODO: Use is_pointer_inverconvertible_base_of
+template <class _Tp, class _Up>
+struct __libcpp_is_trivially_equality_comparable_impl<_Tp*, _Up*>
+    : integral_constant<
+          bool,
+          __is_equality_comparable<_Tp*, _Up*>::value &&
+              (is_same<__remove_cv_t<_Tp>, __remove_cv_t<_Up> >::value || is_void<_Tp>::value || is_void<_Up>::value)> {
+};
+
+template <class _Tp, class _Up>
+using __libcpp_is_trivially_equality_comparable =
+    __libcpp_is_trivially_equality_comparable_impl<__remove_cv_t<_Tp>, __remove_cv_t<_Up> >;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_EQUALITY_COMPARABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_execution_policy.h b/libcxx/include/__cxx03/__type_traits/is_execution_policy.h
new file mode 100644
index 000000000000..6884f17ba16c
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_execution_policy.h
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_EXECUTION_POLICY_H
+#define _LIBCPP___TYPE_TRAITS_IS_EXECUTION_POLICY_H
+
+#include <__config>
+#include <__type_traits/remove_cvref.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class>
+inline constexpr bool is_execution_policy_v = false;
+
+template <class>
+inline constexpr bool __is_unsequenced_execution_policy_impl = false;
+
+template <class _Tp>
+inline constexpr bool __is_unsequenced_execution_policy_v =
+    __is_unsequenced_execution_policy_impl<__remove_cvref_t<_Tp>>;
+
+template <class>
+inline constexpr bool __is_parallel_execution_policy_impl = false;
+
+template <class _Tp>
+inline constexpr bool __is_parallel_execution_policy_v = __is_parallel_execution_policy_impl<__remove_cvref_t<_Tp>>;
+
+namespace execution {
+struct __disable_user_instantiations_tag {
+  explicit __disable_user_instantiations_tag() = default;
+};
+} // namespace execution
+
+// TODO: Remove default argument once algorithms are using the new backend dispatching
+template <class _ExecutionPolicy>
+_LIBCPP_HIDE_FROM_ABI auto
+__remove_parallel_policy(const _ExecutionPolicy& = _ExecutionPolicy{execution::__disable_user_instantiations_tag{}});
+
+// Removes the "parallel" part of an execution policy.
+// For example, turns par_unseq into unseq, and par into seq.
+template <class _ExecutionPolicy>
+using __remove_parallel_policy_t = decltype(std::__remove_parallel_policy<_ExecutionPolicy>());
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_EXECUTION_POLICY_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_final.h b/libcxx/include/__cxx03/__type_traits/is_final.h
new file mode 100644
index 000000000000..499c5e3a1edc
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_final.h
@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_FINAL_H
+#define _LIBCPP___TYPE_TRAITS_IS_FINAL_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS __libcpp_is_final : public integral_constant<bool, __is_final(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_final : public integral_constant<bool, __is_final(_Tp)> {};
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_final_v = __is_final(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_FINAL_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_floating_point.h b/libcxx/include/__cxx03/__type_traits/is_floating_point.h
new file mode 100644
index 000000000000..add34782dfa0
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_floating_point.h
@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_FLOATING_POINT_H
+#define _LIBCPP___TYPE_TRAITS_IS_FLOATING_POINT_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/remove_cv.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// clang-format off
+template <class _Tp> struct __libcpp_is_floating_point              : public false_type {};
+template <>          struct __libcpp_is_floating_point<float>       : public true_type {};
+template <>          struct __libcpp_is_floating_point<double>      : public true_type {};
+template <>          struct __libcpp_is_floating_point<long double> : public true_type {};
+// clang-format on
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_floating_point : public __libcpp_is_floating_point<__remove_cv_t<_Tp> > {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_floating_point_v = is_floating_point<_Tp>::value;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_FLOATING_POINT_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_function.h b/libcxx/include/__cxx03/__type_traits/is_function.h
new file mode 100644
index 000000000000..98fedd0ad96d
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_function.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_FUNCTIONAL_H
+#define _LIBCPP___TYPE_TRAITS_IS_FUNCTIONAL_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_function : integral_constant<bool, __is_function(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_function_v = __is_function(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_FUNCTIONAL_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_fundamental.h b/libcxx/include/__cxx03/__type_traits/is_fundamental.h
new file mode 100644
index 000000000000..55f8e41f75f4
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_fundamental.h
@@ -0,0 +1,48 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_FUNDAMENTAL_H
+#define _LIBCPP___TYPE_TRAITS_IS_FUNDAMENTAL_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_null_pointer.h>
+#include <__type_traits/is_void.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__is_fundamental)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_fundamental : _BoolConstant<__is_fundamental(_Tp)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_fundamental_v = __is_fundamental(_Tp);
+#  endif
+
+#else // __has_builtin(__is_fundamental)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_fundamental
+    : public integral_constant<bool, is_void<_Tp>::value || __is_null_pointer_v<_Tp> || is_arithmetic<_Tp>::value> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_fundamental_v = is_fundamental<_Tp>::value;
+#  endif
+
+#endif // __has_builtin(__is_fundamental)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_FUNDAMENTAL_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_implicitly_default_constructible.h b/libcxx/include/__cxx03/__type_traits/is_implicitly_default_constructible.h
new file mode 100644
index 000000000000..d5dadd7b870d
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_implicitly_default_constructible.h
@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_IMPLICITLY_DEFAULT_CONSTRUCTIBLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_IMPLICITLY_DEFAULT_CONSTRUCTIBLE_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_constructible.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifndef _LIBCPP_CXX03_LANG
+// First of all, we can't implement this check in C++03 mode because the {}
+// default initialization syntax isn't valid.
+// Second, we implement the trait in a funny manner with two defaulted template
+// arguments to workaround Clang's PR43454.
+template <class _Tp>
+void __test_implicit_default_constructible(_Tp);
+
+template <class _Tp, class = void, class = typename is_default_constructible<_Tp>::type>
+struct __is_implicitly_default_constructible : false_type {};
+
+template <class _Tp>
+struct __is_implicitly_default_constructible<_Tp,
+                                             decltype(std::__test_implicit_default_constructible<_Tp const&>({})),
+                                             true_type> : true_type {};
+
+template <class _Tp>
+struct __is_implicitly_default_constructible<_Tp,
+                                             decltype(std::__test_implicit_default_constructible<_Tp const&>({})),
+                                             false_type> : false_type {};
+#endif // !C++03
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_IMPLICITLY_DEFAULT_CONSTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_integral.h b/libcxx/include/__cxx03/__type_traits/is_integral.h
new file mode 100644
index 000000000000..26969885af8d
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_integral.h
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_INTEGRAL_H
+#define _LIBCPP___TYPE_TRAITS_IS_INTEGRAL_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/remove_cv.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// clang-format off
+template <class _Tp> struct __libcpp_is_integral                     { enum { value = 0 }; };
+template <>          struct __libcpp_is_integral<bool>               { enum { value = 1 }; };
+template <>          struct __libcpp_is_integral<char>               { enum { value = 1 }; };
+template <>          struct __libcpp_is_integral<signed char>        { enum { value = 1 }; };
+template <>          struct __libcpp_is_integral<unsigned char>      { enum { value = 1 }; };
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>          struct __libcpp_is_integral<wchar_t>            { enum { value = 1 }; };
+#endif
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+template <>          struct __libcpp_is_integral<char8_t>            { enum { value = 1 }; };
+#endif
+template <>          struct __libcpp_is_integral<char16_t>           { enum { value = 1 }; };
+template <>          struct __libcpp_is_integral<char32_t>           { enum { value = 1 }; };
+template <>          struct __libcpp_is_integral<short>              { enum { value = 1 }; };
+template <>          struct __libcpp_is_integral<unsigned short>     { enum { value = 1 }; };
+template <>          struct __libcpp_is_integral<int>                { enum { value = 1 }; };
+template <>          struct __libcpp_is_integral<unsigned int>       { enum { value = 1 }; };
+template <>          struct __libcpp_is_integral<long>               { enum { value = 1 }; };
+template <>          struct __libcpp_is_integral<unsigned long>      { enum { value = 1 }; };
+template <>          struct __libcpp_is_integral<long long>          { enum { value = 1 }; };
+template <>          struct __libcpp_is_integral<unsigned long long> { enum { value = 1 }; };
+#ifndef _LIBCPP_HAS_NO_INT128
+template <>          struct __libcpp_is_integral<__int128_t>         { enum { value = 1 }; };
+template <>          struct __libcpp_is_integral<__uint128_t>        { enum { value = 1 }; };
+#endif
+// clang-format on
+
+#if __has_builtin(__is_integral)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_integral : _BoolConstant<__is_integral(_Tp)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_integral_v = __is_integral(_Tp);
+#  endif
+
+#else
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_integral : public _BoolConstant<__libcpp_is_integral<__remove_cv_t<_Tp> >::value> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_integral_v = is_integral<_Tp>::value;
+#  endif
+
+#endif // __has_builtin(__is_integral)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_INTEGRAL_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_literal_type.h b/libcxx/include/__cxx03/__type_traits/is_literal_type.h
new file mode 100644
index 000000000000..10e23bceffbd
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_literal_type.h
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_LITERAL_TYPE
+#define _LIBCPP___TYPE_TRAITS_IS_LITERAL_TYPE
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_TYPE_TRAITS)
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS
+_LIBCPP_DEPRECATED_IN_CXX17 is_literal_type : public integral_constant<bool, __is_literal_type(_Tp)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+_LIBCPP_DEPRECATED_IN_CXX17 inline constexpr bool is_literal_type_v = __is_literal_type(_Tp);
+#  endif // _LIBCPP_STD_VER >= 17
+#endif   // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_TYPE_TRAITS)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_LITERAL_TYPE
diff --git a/libcxx/include/__cxx03/__type_traits/is_member_pointer.h b/libcxx/include/__cxx03/__type_traits/is_member_pointer.h
new file mode 100644
index 000000000000..3e2753ac4228
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_member_pointer.h
@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_MEMBER_POINTER_H
+#define _LIBCPP___TYPE_TRAITS_IS_MEMBER_POINTER_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_member_pointer : _BoolConstant<__is_member_pointer(_Tp)> {};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_member_object_pointer : _BoolConstant<__is_member_object_pointer(_Tp)> {};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_member_function_pointer : _BoolConstant<__is_member_function_pointer(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_member_pointer_v = __is_member_pointer(_Tp);
+
+template <class _Tp>
+inline constexpr bool is_member_object_pointer_v = __is_member_object_pointer(_Tp);
+
+template <class _Tp>
+inline constexpr bool is_member_function_pointer_v = __is_member_function_pointer(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_MEMBER_POINTER_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_nothrow_assignable.h b/libcxx/include/__cxx03/__type_traits/is_nothrow_assignable.h
new file mode 100644
index 000000000000..7e00c741f83e
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_nothrow_assignable.h
@@ -0,0 +1,56 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_NOTHROW_ASSIGNABLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_NOTHROW_ASSIGNABLE_H
+
+#include <__config>
+#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Arg>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_assignable : public integral_constant<bool, __is_nothrow_assignable(_Tp, _Arg)> {
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp, class _Arg>
+inline constexpr bool is_nothrow_assignable_v = __is_nothrow_assignable(_Tp, _Arg);
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_copy_assignable
+    : public integral_constant<
+          bool,
+          __is_nothrow_assignable(__add_lvalue_reference_t<_Tp>, __add_lvalue_reference_t<const _Tp>)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_nothrow_copy_assignable_v = is_nothrow_copy_assignable<_Tp>::value;
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_move_assignable
+    : public integral_constant<bool,
+                               __is_nothrow_assignable(__add_lvalue_reference_t<_Tp>, __add_rvalue_reference_t<_Tp>)> {
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_nothrow_move_assignable_v = is_nothrow_move_assignable<_Tp>::value;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_NOTHROW_ASSIGNABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_nothrow_constructible.h b/libcxx/include/__cxx03/__type_traits/is_nothrow_constructible.h
new file mode 100644
index 000000000000..58d2b2475140
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_nothrow_constructible.h
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_NOTHROW_CONSTRUCTIBLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_NOTHROW_CONSTRUCTIBLE_H
+
+#include <__config>
+#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template < class _Tp, class... _Args>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_constructible
+    : public integral_constant<bool, __is_nothrow_constructible(_Tp, _Args...)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp, class... _Args>
+inline constexpr bool is_nothrow_constructible_v = is_nothrow_constructible<_Tp, _Args...>::value;
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_copy_constructible
+    : public integral_constant< bool, __is_nothrow_constructible(_Tp, __add_lvalue_reference_t<const _Tp>)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_nothrow_copy_constructible_v = is_nothrow_copy_constructible<_Tp>::value;
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_move_constructible
+    : public integral_constant<bool, __is_nothrow_constructible(_Tp, __add_rvalue_reference_t<_Tp>)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_nothrow_move_constructible_v = is_nothrow_move_constructible<_Tp>::value;
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_default_constructible
+    : public integral_constant<bool, __is_nothrow_constructible(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_nothrow_default_constructible_v = __is_nothrow_constructible(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_NOTHROW_CONSTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_nothrow_convertible.h b/libcxx/include/__cxx03/__type_traits/is_nothrow_convertible.h
new file mode 100644
index 000000000000..bfc5a94cbade
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_nothrow_convertible.h
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_NOTHROW_CONVERTIBLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_NOTHROW_CONVERTIBLE_H
+
+#include <__config>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/disjunction.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_void.h>
+#include <__type_traits/lazy.h>
+#include <__utility/declval.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+#  if __has_builtin(__is_nothrow_convertible)
+
+template <class _Tp, class _Up>
+struct is_nothrow_convertible : bool_constant<__is_nothrow_convertible(_Tp, _Up)> {};
+
+template <class _Tp, class _Up>
+inline constexpr bool is_nothrow_convertible_v = __is_nothrow_convertible(_Tp, _Up);
+
+#  else // __has_builtin(__is_nothrow_convertible)
+
+template <typename _Tp>
+void __test_noexcept(_Tp) noexcept;
+
+template <typename _Fm, typename _To>
+bool_constant<noexcept(std::__test_noexcept<_To>(std::declval<_Fm>()))> __is_nothrow_convertible_test();
+
+template <typename _Fm, typename _To>
+struct __is_nothrow_convertible_helper : decltype(__is_nothrow_convertible_test<_Fm, _To>()) {};
+
+template <typename _Fm, typename _To>
+struct is_nothrow_convertible
+    : _Or<_And<is_void<_To>, is_void<_Fm>>,
+          _Lazy<_And, is_convertible<_Fm, _To>, __is_nothrow_convertible_helper<_Fm, _To> > >::type {};
+
+template <typename _Fm, typename _To>
+inline constexpr bool is_nothrow_convertible_v = is_nothrow_convertible<_Fm, _To>::value;
+
+#  endif // __has_builtin(__is_nothrow_convertible)
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_NOTHROW_CONVERTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_nothrow_destructible.h b/libcxx/include/__cxx03/__type_traits/is_nothrow_destructible.h
new file mode 100644
index 000000000000..c2d5ca87232a
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_nothrow_destructible.h
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_NOTHROW_DESTRUCTIBLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_NOTHROW_DESTRUCTIBLE_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_destructible.h>
+#include <__utility/declval.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__is_nothrow_destructible)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible : integral_constant<bool, __is_nothrow_destructible(_Tp)> {};
+
+#else
+
+template <bool, class _Tp>
+struct __libcpp_is_nothrow_destructible;
+
+template <class _Tp>
+struct __libcpp_is_nothrow_destructible<false, _Tp> : public false_type {};
+
+template <class _Tp>
+struct __libcpp_is_nothrow_destructible<true, _Tp>
+    : public integral_constant<bool, noexcept(std::declval<_Tp>().~_Tp()) > {};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible
+    : public __libcpp_is_nothrow_destructible<is_destructible<_Tp>::value, _Tp> {};
+
+template <class _Tp, size_t _Ns>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible<_Tp[_Ns]> : public is_nothrow_destructible<_Tp> {};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible<_Tp&> : public true_type {};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible<_Tp&&> : public true_type {};
+
+#endif // __has_builtin(__is_nothrow_destructible)
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_nothrow_destructible_v = is_nothrow_destructible<_Tp>::value;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_NOTHROW_DESTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_null_pointer.h b/libcxx/include/__cxx03/__type_traits/is_null_pointer.h
new file mode 100644
index 000000000000..9f5697e23268
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_null_pointer.h
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_NULL_POINTER_H
+#define _LIBCPP___TYPE_TRAITS_IS_NULL_POINTER_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+inline const bool __is_null_pointer_v = __is_same(__remove_cv(_Tp), nullptr_t);
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_null_pointer : integral_constant<bool, __is_null_pointer_v<_Tp>> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_null_pointer_v = __is_null_pointer_v<_Tp>;
+#  endif
+#endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_NULL_POINTER_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_object.h b/libcxx/include/__cxx03/__type_traits/is_object.h
new file mode 100644
index 000000000000..ec04508402ce
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_object.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_OBJECT_H
+#define _LIBCPP___TYPE_TRAITS_IS_OBJECT_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_object : _BoolConstant<__is_object(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_object_v = __is_object(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_OBJECT_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_pod.h b/libcxx/include/__cxx03/__type_traits/is_pod.h
new file mode 100644
index 000000000000..5888fbf457d8
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_pod.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_POD_H
+#define _LIBCPP___TYPE_TRAITS_IS_POD_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_pod : public integral_constant<bool, __is_pod(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_pod_v = __is_pod(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_POD_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_pointer.h b/libcxx/include/__cxx03/__type_traits/is_pointer.h
new file mode 100644
index 000000000000..38eb7996c686
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_pointer.h
@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_POINTER_H
+#define _LIBCPP___TYPE_TRAITS_IS_POINTER_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/remove_cv.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__is_pointer)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_pointer : _BoolConstant<__is_pointer(_Tp)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_pointer_v = __is_pointer(_Tp);
+#  endif
+
+#else // __has_builtin(__is_pointer)
+
+template <class _Tp>
+struct __libcpp_is_pointer : public false_type {};
+template <class _Tp>
+struct __libcpp_is_pointer<_Tp*> : public true_type {};
+
+template <class _Tp>
+struct __libcpp_remove_objc_qualifiers {
+  typedef _Tp type;
+};
+#  if defined(_LIBCPP_HAS_OBJC_ARC)
+// clang-format off
+template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __strong> { typedef _Tp type; };
+template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __weak> { typedef _Tp type; };
+template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __autoreleasing> { typedef _Tp type; };
+template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __unsafe_unretained> { typedef _Tp type; };
+// clang-format on
+#  endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_pointer
+    : public __libcpp_is_pointer<typename __libcpp_remove_objc_qualifiers<__remove_cv_t<_Tp> >::type> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_pointer_v = is_pointer<_Tp>::value;
+#  endif
+
+#endif // __has_builtin(__is_pointer)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_POINTER_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_polymorphic.h b/libcxx/include/__cxx03/__type_traits/is_polymorphic.h
new file mode 100644
index 000000000000..d122e1c87775
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_polymorphic.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_POLYMORPHIC_H
+#define _LIBCPP___TYPE_TRAITS_IS_POLYMORPHIC_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_polymorphic : public integral_constant<bool, __is_polymorphic(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_polymorphic_v = __is_polymorphic(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_POLYMORPHIC_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_primary_template.h b/libcxx/include/__cxx03/__type_traits/is_primary_template.h
new file mode 100644
index 000000000000..f308dfadc8ec
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_primary_template.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_PRIMARY_TEMPLATE_H
+#define _LIBCPP___TYPE_TRAITS_IS_PRIMARY_TEMPLATE_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_valid_expansion.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+using __test_for_primary_template = __enable_if_t<_IsSame<_Tp, typename _Tp::__primary_template>::value>;
+
+template <class _Tp>
+using __is_primary_template = _IsValidExpansion<__test_for_primary_template, _Tp>;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_PRIMARY_TEMPLATE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_reference.h b/libcxx/include/__cxx03/__type_traits/is_reference.h
new file mode 100644
index 000000000000..cc157a438e49
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_reference.h
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_REFERENCE_H
+#define _LIBCPP___TYPE_TRAITS_IS_REFERENCE_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_reference : _BoolConstant<__is_reference(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_reference_v = __is_reference(_Tp);
+#endif
+
+#if __has_builtin(__is_lvalue_reference) && __has_builtin(__is_rvalue_reference)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_lvalue_reference : _BoolConstant<__is_lvalue_reference(_Tp)> {};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_rvalue_reference : _BoolConstant<__is_rvalue_reference(_Tp)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_lvalue_reference_v = __is_lvalue_reference(_Tp);
+template <class _Tp>
+inline constexpr bool is_rvalue_reference_v = __is_rvalue_reference(_Tp);
+#  endif
+
+#else // __has_builtin(__is_lvalue_reference)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_lvalue_reference : public false_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_lvalue_reference<_Tp&> : public true_type {};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_rvalue_reference : public false_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_rvalue_reference<_Tp&&> : public true_type {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_lvalue_reference_v = is_lvalue_reference<_Tp>::value;
+
+template <class _Tp>
+inline constexpr bool is_rvalue_reference_v = is_rvalue_reference<_Tp>::value;
+#  endif
+
+#endif // __has_builtin(__is_lvalue_reference)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_REFERENCE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_reference_wrapper.h b/libcxx/include/__cxx03/__type_traits/is_reference_wrapper.h
new file mode 100644
index 000000000000..310a910040e8
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_reference_wrapper.h
@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_REFERENCE_WRAPPER_H
+#define _LIBCPP___TYPE_TRAITS_IS_REFERENCE_WRAPPER_H
+
+#include <__config>
+#include <__fwd/functional.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/remove_cv.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct __is_reference_wrapper_impl : public false_type {};
+template <class _Tp>
+struct __is_reference_wrapper_impl<reference_wrapper<_Tp> > : public true_type {};
+template <class _Tp>
+struct __is_reference_wrapper : public __is_reference_wrapper_impl<__remove_cv_t<_Tp> > {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_ENABLE_IF_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_referenceable.h b/libcxx/include/__cxx03/__type_traits/is_referenceable.h
new file mode 100644
index 000000000000..4b34ec257231
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_referenceable.h
@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_REFERENCEABLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_REFERENCEABLE_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_same.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__is_referenceable)
+template <class _Tp>
+struct __libcpp_is_referenceable : integral_constant<bool, __is_referenceable(_Tp)> {};
+#else
+struct __libcpp_is_referenceable_impl {
+  template <class _Tp>
+  static _Tp& __test(int);
+  template <class _Tp>
+  static false_type __test(...);
+};
+
+template <class _Tp>
+struct __libcpp_is_referenceable
+    : integral_constant<bool, _IsNotSame<decltype(__libcpp_is_referenceable_impl::__test<_Tp>(0)), false_type>::value> {
+};
+#endif // __has_builtin(__is_referenceable)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_REFERENCEABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_same.h b/libcxx/include/__cxx03/__type_traits/is_same.h
new file mode 100644
index 000000000000..9561b7b5d6da
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_same.h
@@ -0,0 +1,44 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_SAME_H
+#define _LIBCPP___TYPE_TRAITS_IS_SAME_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Up>
+struct _LIBCPP_TEMPLATE_VIS is_same : _BoolConstant<__is_same(_Tp, _Up)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp, class _Up>
+inline constexpr bool is_same_v = __is_same(_Tp, _Up);
+#endif
+
+// _IsSame<T,U> has the same effect as is_same<T,U> but instantiates fewer types:
+// is_same<A,B> and is_same<C,D> are guaranteed to be different types, but
+// _IsSame<A,B> and _IsSame<C,D> are the same type (namely, false_type).
+// Neither GCC nor Clang can mangle the __is_same builtin, so _IsSame
+// mustn't be directly used anywhere that contributes to name-mangling
+// (such as in a dependent return type).
+
+template <class _Tp, class _Up>
+using _IsSame = _BoolConstant<__is_same(_Tp, _Up)>;
+
+template <class _Tp, class _Up>
+using _IsNotSame = _BoolConstant<!__is_same(_Tp, _Up)>;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_SAME_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_scalar.h b/libcxx/include/__cxx03/__type_traits/is_scalar.h
new file mode 100644
index 000000000000..455200de4720
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_scalar.h
@@ -0,0 +1,69 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_SCALAR_H
+#define _LIBCPP___TYPE_TRAITS_IS_SCALAR_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_enum.h>
+#include <__type_traits/is_member_pointer.h>
+#include <__type_traits/is_null_pointer.h>
+#include <__type_traits/is_pointer.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__is_scalar)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_scalar : _BoolConstant<__is_scalar(_Tp)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_scalar_v = __is_scalar(_Tp);
+#  endif
+
+#else // __has_builtin(__is_scalar)
+
+template <class _Tp>
+struct __is_block : false_type {};
+#  if defined(_LIBCPP_HAS_EXTENSION_BLOCKS)
+template <class _Rp, class... _Args>
+struct __is_block<_Rp (^)(_Args...)> : true_type {};
+#  endif
+
+// clang-format off
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_scalar
+    : public integral_constant<
+          bool, is_arithmetic<_Tp>::value ||
+                is_member_pointer<_Tp>::value ||
+                is_pointer<_Tp>::value ||
+                __is_null_pointer_v<_Tp> ||
+                __is_block<_Tp>::value ||
+                is_enum<_Tp>::value> {};
+// clang-format on
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS is_scalar<nullptr_t> : public true_type {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_scalar_v = is_scalar<_Tp>::value;
+#  endif
+
+#endif // __has_builtin(__is_scalar)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_SCALAR_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_signed.h b/libcxx/include/__cxx03/__type_traits/is_signed.h
new file mode 100644
index 000000000000..fd6f93e18236
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_signed.h
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_SIGNED_H
+#define _LIBCPP___TYPE_TRAITS_IS_SIGNED_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_integral.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__is_signed)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_signed : _BoolConstant<__is_signed(_Tp)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_signed_v = __is_signed(_Tp);
+#  endif
+
+#else // __has_builtin(__is_signed)
+
+template <class _Tp, bool = is_integral<_Tp>::value>
+struct __libcpp_is_signed_impl : public _BoolConstant<(_Tp(-1) < _Tp(0))> {};
+
+template <class _Tp>
+struct __libcpp_is_signed_impl<_Tp, false> : public true_type {}; // floating point
+
+template <class _Tp, bool = is_arithmetic<_Tp>::value>
+struct __libcpp_is_signed : public __libcpp_is_signed_impl<_Tp> {};
+
+template <class _Tp>
+struct __libcpp_is_signed<_Tp, false> : public false_type {};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_signed : public __libcpp_is_signed<_Tp> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_signed_v = is_signed<_Tp>::value;
+#  endif
+
+#endif // __has_builtin(__is_signed)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_SIGNED_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_signed_integer.h b/libcxx/include/__cxx03/__type_traits/is_signed_integer.h
new file mode 100644
index 000000000000..467548d0aaaf
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_signed_integer.h
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_SIGNED_INTEGER_H
+#define _LIBCPP___TYPE_TRAITS_IS_SIGNED_INTEGER_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// clang-format off
+template <class _Tp> struct __libcpp_is_signed_integer                   : public false_type {};
+template <>          struct __libcpp_is_signed_integer<signed char>      : public true_type {};
+template <>          struct __libcpp_is_signed_integer<signed short>     : public true_type {};
+template <>          struct __libcpp_is_signed_integer<signed int>       : public true_type {};
+template <>          struct __libcpp_is_signed_integer<signed long>      : public true_type {};
+template <>          struct __libcpp_is_signed_integer<signed long long> : public true_type {};
+#ifndef _LIBCPP_HAS_NO_INT128
+template <>          struct __libcpp_is_signed_integer<__int128_t>       : public true_type {};
+#endif
+// clang-format on
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_SIGNED_INTEGER_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_specialization.h b/libcxx/include/__cxx03/__type_traits/is_specialization.h
new file mode 100644
index 000000000000..9b75636b1a51
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_specialization.h
@@ -0,0 +1,45 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_SPECIALIZATION
+#define _LIBCPP___TYPE_TRAITS_IS_SPECIALIZATION
+
+// This contains parts of P2098R1 but is based on MSVC STL's implementation.
+//
+// The paper has been rejected
+//   We will not pursue P2098R0 (std::is_specialization_of) at this time; we'd
+//   like to see a solution to this problem, but it requires language evolution
+//   too.
+//
+// Since it is expected a real solution will be provided in the future only the
+// minimal part is implemented.
+//
+// Note a cvref qualified _Tp is never considered a specialization.
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _Tp, template <class...> class _Template>
+inline constexpr bool __is_specialization_v = false; // true if and only if _Tp is a specialization of _Template
+
+template <template <class...> class _Template, class... _Args>
+inline constexpr bool __is_specialization_v<_Template<_Args...>, _Template> = true;
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_SPECIALIZATION
diff --git a/libcxx/include/__cxx03/__type_traits/is_standard_layout.h b/libcxx/include/__cxx03/__type_traits/is_standard_layout.h
new file mode 100644
index 000000000000..76484f3e2a30
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_standard_layout.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_STANDARD_LAYOUT_H
+#define _LIBCPP___TYPE_TRAITS_IS_STANDARD_LAYOUT_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_standard_layout : public integral_constant<bool, __is_standard_layout(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_standard_layout_v = __is_standard_layout(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_STANDARD_LAYOUT_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_swappable.h b/libcxx/include/__cxx03/__type_traits/is_swappable.h
new file mode 100644
index 000000000000..0b817e650993
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_swappable.h
@@ -0,0 +1,104 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_SWAPPABLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_SWAPPABLE_H
+
+#include <__config>
+#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Up, class = void>
+inline const bool __is_swappable_with_v = false;
+
+template <class _Tp>
+inline const bool __is_swappable_v = __is_swappable_with_v<_Tp&, _Tp&>;
+
+template <class _Tp, class _Up, bool = __is_swappable_with_v<_Tp, _Up> >
+inline const bool __is_nothrow_swappable_with_v = false;
+
+template <class _Tp>
+inline const bool __is_nothrow_swappable_v = __is_nothrow_swappable_with_v<_Tp&, _Tp&>;
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Tp>
+using __swap_result_t = __enable_if_t<is_move_constructible<_Tp>::value && is_move_assignable<_Tp>::value>;
+#else
+template <class>
+using __swap_result_t = void;
+#endif
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __swap_result_t<_Tp> swap(_Tp& __x, _Tp& __y)
+    _NOEXCEPT_(is_nothrow_move_constructible<_Tp>::value&& is_nothrow_move_assignable<_Tp>::value);
+
+template <class _Tp, size_t _Np, __enable_if_t<__is_swappable_v<_Tp>, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(_Tp (&__a)[_Np], _Tp (&__b)[_Np]) _NOEXCEPT_(__is_nothrow_swappable_v<_Tp>);
+
+// ALL generic swap overloads MUST already have a declaration available at this point.
+
+template <class _Tp, class _Up>
+inline const bool __is_swappable_with_v<_Tp,
+                                        _Up,
+                                        __void_t<decltype(swap(std::declval<_Tp>(), std::declval<_Up>())),
+                                                 decltype(swap(std::declval<_Up>(), std::declval<_Tp>()))> > = true;
+
+#ifndef _LIBCPP_CXX03_LANG // C++03 doesn't have noexcept, so things are never nothrow swappable
+template <class _Tp, class _Up>
+inline const bool __is_nothrow_swappable_with_v<_Tp, _Up, true> =
+    noexcept(swap(std::declval<_Tp>(), std::declval<_Up>())) &&
+    noexcept(swap(std::declval<_Up>(), std::declval<_Tp>()));
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+template <class _Tp, class _Up>
+inline constexpr bool is_swappable_with_v = __is_swappable_with_v<_Tp, _Up>;
+
+template <class _Tp, class _Up>
+struct _LIBCPP_TEMPLATE_VIS is_swappable_with : bool_constant<is_swappable_with_v<_Tp, _Up>> {};
+
+template <class _Tp>
+inline constexpr bool is_swappable_v =
+    is_swappable_with_v<__add_lvalue_reference_t<_Tp>, __add_lvalue_reference_t<_Tp>>;
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_swappable : bool_constant<is_swappable_v<_Tp>> {};
+
+template <class _Tp, class _Up>
+inline constexpr bool is_nothrow_swappable_with_v = __is_nothrow_swappable_with_v<_Tp, _Up>;
+
+template <class _Tp, class _Up>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_swappable_with : bool_constant<is_nothrow_swappable_with_v<_Tp, _Up>> {};
+
+template <class _Tp>
+inline constexpr bool is_nothrow_swappable_v =
+    is_nothrow_swappable_with_v<__add_lvalue_reference_t<_Tp>, __add_lvalue_reference_t<_Tp>>;
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_swappable : bool_constant<is_nothrow_swappable_v<_Tp>> {};
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_SWAPPABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_trivial.h b/libcxx/include/__cxx03/__type_traits/is_trivial.h
new file mode 100644
index 000000000000..0007c7446d5e
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_trivial.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_TRIVIAL_H
+#define _LIBCPP___TYPE_TRAITS_IS_TRIVIAL_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_trivial : public integral_constant<bool, __is_trivial(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_trivial_v = __is_trivial(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_TRIVIAL_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_trivially_assignable.h b/libcxx/include/__cxx03/__type_traits/is_trivially_assignable.h
new file mode 100644
index 000000000000..201333b0fa0b
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_trivially_assignable.h
@@ -0,0 +1,56 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_ASSIGNABLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_ASSIGNABLE_H
+
+#include <__config>
+#include <__type_traits/add_const.h>
+#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Arg>
+struct is_trivially_assignable : integral_constant<bool, __is_trivially_assignable(_Tp, _Arg)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp, class _Arg>
+inline constexpr bool is_trivially_assignable_v = __is_trivially_assignable(_Tp, _Arg);
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_trivially_copy_assignable
+    : public integral_constant<
+          bool,
+          __is_trivially_assignable(__add_lvalue_reference_t<_Tp>, __add_lvalue_reference_t<const _Tp>)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_trivially_copy_assignable_v = is_trivially_copy_assignable<_Tp>::value;
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_trivially_move_assignable
+    : public integral_constant<
+          bool,
+          __is_trivially_assignable(__add_lvalue_reference_t<_Tp>, __add_rvalue_reference_t<_Tp>)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_trivially_move_assignable_v = is_trivially_move_assignable<_Tp>::value;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_ASSIGNABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_trivially_constructible.h b/libcxx/include/__cxx03/__type_traits/is_trivially_constructible.h
new file mode 100644
index 000000000000..3a77e9fe164d
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_trivially_constructible.h
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_CONSTRUCTIBLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_CONSTRUCTIBLE_H
+
+#include <__config>
+#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class... _Args>
+struct _LIBCPP_TEMPLATE_VIS is_trivially_constructible
+    : integral_constant<bool, __is_trivially_constructible(_Tp, _Args...)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp, class... _Args>
+inline constexpr bool is_trivially_constructible_v = __is_trivially_constructible(_Tp, _Args...);
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_trivially_copy_constructible
+    : public integral_constant<bool, __is_trivially_constructible(_Tp, __add_lvalue_reference_t<const _Tp>)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_trivially_copy_constructible_v = is_trivially_copy_constructible<_Tp>::value;
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_trivially_move_constructible
+    : public integral_constant<bool, __is_trivially_constructible(_Tp, __add_rvalue_reference_t<_Tp>)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_trivially_move_constructible_v = is_trivially_move_constructible<_Tp>::value;
+#endif
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_trivially_default_constructible
+    : public integral_constant<bool, __is_trivially_constructible(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_trivially_default_constructible_v = __is_trivially_constructible(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_CONSTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_trivially_copyable.h b/libcxx/include/__cxx03/__type_traits/is_trivially_copyable.h
new file mode 100644
index 000000000000..e92af126ee94
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_trivially_copyable.h
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_COPYABLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_COPYABLE_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_trivially_copyable : public integral_constant<bool, __is_trivially_copyable(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_trivially_copyable_v = __is_trivially_copyable(_Tp);
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp>
+inline constexpr bool __is_cheap_to_copy = is_trivially_copyable_v<_Tp> && sizeof(_Tp) <= sizeof(std::intmax_t);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_COPYABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_trivially_destructible.h b/libcxx/include/__cxx03/__type_traits/is_trivially_destructible.h
new file mode 100644
index 000000000000..5f9652f2a501
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_trivially_destructible.h
@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_DESTRUCTIBLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_DESTRUCTIBLE_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_destructible.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__is_trivially_destructible)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_trivially_destructible
+    : public integral_constant<bool, __is_trivially_destructible(_Tp)> {};
+
+#elif __has_builtin(__has_trivial_destructor)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_trivially_destructible
+    : public integral_constant<bool, is_destructible<_Tp>::value&& __has_trivial_destructor(_Tp)> {};
+
+#else
+
+#  error is_trivially_destructible is not implemented
+
+#endif // __has_builtin(__is_trivially_destructible)
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_trivially_destructible_v = is_trivially_destructible<_Tp>::value;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_DESTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_trivially_lexicographically_comparable.h b/libcxx/include/__cxx03/__type_traits/is_trivially_lexicographically_comparable.h
new file mode 100644
index 000000000000..a310ea1b87e3
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_trivially_lexicographically_comparable.h
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_LEXICOGRAPHICALLY_COMPARABLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_LEXICOGRAPHICALLY_COMPARABLE_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_unsigned.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// A type is_trivially_lexicographically_comparable if the expression `a <=> b` (or their pre-C++20 equivalents) is
+// equivalent to `std::memcmp(&a, &b, sizeof(T))` (with `a` and `b` being of type `T`). There is currently no builtin to
+// tell us whether that's the case for arbitrary types, so we can only do this for known types. Specifically, these are
+// currently unsigned integer types with a sizeof(T) == 1.
+//
+// bool is trivially lexicographically comparable, because e.g. false <=> true is valid code. Furthermore, the standard
+// says that [basic.fundamental] "Type bool is a distinct type that has the same object representation, value
+// representation, and alignment requirements as an implementation-defined unsigned integer type. The values of type
+// bool are true and false."
+// This means that bool has to be unsigned and has exactly two values. This means that having anything other than the
+// `true` or `false` value representations in a bool is UB.
+//
+// The following types are not trivially lexicographically comparable:
+// signed integer types: `char(-1) < char(1)`, but memcmp compares `unsigned char`s
+// unsigned integer types with sizeof(T) > 1: depending on the endianness, the LSB might be the first byte to be
+//                                            compared. This means that when comparing unsigned(129) and unsigned(2)
+//                                            using memcmp(), the result would be that 2 > 129.
+//                                            TODO: Do we want to enable this on big-endian systems?
+
+template <class _Tp, class _Up>
+struct __libcpp_is_trivially_lexicographically_comparable
+    : integral_constant<bool,
+                        is_same<__remove_cv_t<_Tp>, __remove_cv_t<_Up> >::value && sizeof(_Tp) == 1 &&
+                            is_unsigned<_Tp>::value> {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_LEXICOGRAPHICALLY_COMPARABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_trivially_relocatable.h b/libcxx/include/__cxx03/__type_traits/is_trivially_relocatable.h
new file mode 100644
index 000000000000..c0871731cc00
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_trivially_relocatable.h
@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_RELOCATABLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_RELOCATABLE_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_trivially_copyable.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// A type is trivially relocatable if a move construct + destroy of the original object is equivalent to
+// `memcpy(dst, src, sizeof(T))`.
+
+#if __has_builtin(__is_trivially_relocatable)
+template <class _Tp, class = void>
+struct __libcpp_is_trivially_relocatable : integral_constant<bool, __is_trivially_relocatable(_Tp)> {};
+#else
+template <class _Tp, class = void>
+struct __libcpp_is_trivially_relocatable : is_trivially_copyable<_Tp> {};
+#endif
+
+template <class _Tp>
+struct __libcpp_is_trivially_relocatable<_Tp,
+                                         __enable_if_t<is_same<_Tp, typename _Tp::__trivially_relocatable>::value> >
+    : true_type {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_RELOCATABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_unbounded_array.h b/libcxx/include/__cxx03/__type_traits/is_unbounded_array.h
new file mode 100644
index 000000000000..d58bb09e1042
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_unbounded_array.h
@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_UNBOUNDED_ARRAY_H
+#define _LIBCPP___TYPE_TRAITS_IS_UNBOUNDED_ARRAY_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class>
+struct _LIBCPP_TEMPLATE_VIS __libcpp_is_unbounded_array : false_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS __libcpp_is_unbounded_array<_Tp[]> : true_type {};
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class>
+struct _LIBCPP_TEMPLATE_VIS is_unbounded_array : false_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_unbounded_array<_Tp[]> : true_type {};
+
+template <class _Tp>
+inline constexpr bool is_unbounded_array_v = is_unbounded_array<_Tp>::value;
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_UNBOUNDED_ARRAY_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_union.h b/libcxx/include/__cxx03/__type_traits/is_union.h
new file mode 100644
index 000000000000..1f009d993545
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_union.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_UNION_H
+#define _LIBCPP___TYPE_TRAITS_IS_UNION_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_union : public integral_constant<bool, __is_union(_Tp)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_union_v = __is_union(_Tp);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_UNION_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_unsigned.h b/libcxx/include/__cxx03/__type_traits/is_unsigned.h
new file mode 100644
index 000000000000..48c5751ed70d
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_unsigned.h
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_UNSIGNED_H
+#define _LIBCPP___TYPE_TRAITS_IS_UNSIGNED_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_integral.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__is_unsigned)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_unsigned : _BoolConstant<__is_unsigned(_Tp)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_unsigned_v = __is_unsigned(_Tp);
+#  endif
+
+#else // __has_builtin(__is_unsigned)
+
+template <class _Tp, bool = is_integral<_Tp>::value>
+struct __libcpp_is_unsigned_impl : public _BoolConstant<(_Tp(0) < _Tp(-1))> {};
+
+template <class _Tp>
+struct __libcpp_is_unsigned_impl<_Tp, false> : public false_type {}; // floating point
+
+template <class _Tp, bool = is_arithmetic<_Tp>::value>
+struct __libcpp_is_unsigned : public __libcpp_is_unsigned_impl<_Tp> {};
+
+template <class _Tp>
+struct __libcpp_is_unsigned<_Tp, false> : public false_type {};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_unsigned : public __libcpp_is_unsigned<_Tp> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_unsigned_v = is_unsigned<_Tp>::value;
+#  endif
+
+#endif // __has_builtin(__is_unsigned)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_UNSIGNED_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_unsigned_integer.h b/libcxx/include/__cxx03/__type_traits/is_unsigned_integer.h
new file mode 100644
index 000000000000..265894b32d4f
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_unsigned_integer.h
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_UNSIGNED_INTEGER_H
+#define _LIBCPP___TYPE_TRAITS_IS_UNSIGNED_INTEGER_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// clang-format off
+template <class _Tp> struct __libcpp_is_unsigned_integer                     : public false_type {};
+template <>          struct __libcpp_is_unsigned_integer<unsigned char>      : public true_type {};
+template <>          struct __libcpp_is_unsigned_integer<unsigned short>     : public true_type {};
+template <>          struct __libcpp_is_unsigned_integer<unsigned int>       : public true_type {};
+template <>          struct __libcpp_is_unsigned_integer<unsigned long>      : public true_type {};
+template <>          struct __libcpp_is_unsigned_integer<unsigned long long> : public true_type {};
+#ifndef _LIBCPP_HAS_NO_INT128
+template <>          struct __libcpp_is_unsigned_integer<__uint128_t>        : public true_type {};
+#endif
+// clang-format on
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_UNSIGNED_INTEGER_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_valid_expansion.h b/libcxx/include/__cxx03/__type_traits/is_valid_expansion.h
new file mode 100644
index 000000000000..346bc98b110f
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_valid_expansion.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_VALID_EXPANSION_H
+#define _LIBCPP___TYPE_TRAITS_IS_VALID_EXPANSION_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <template <class...> class _Templ, class... _Args, class = _Templ<_Args...> >
+true_type __sfinae_test_impl(int);
+template <template <class...> class, class...>
+false_type __sfinae_test_impl(...);
+
+template <template <class...> class _Templ, class... _Args>
+using _IsValidExpansion _LIBCPP_NODEBUG = decltype(std::__sfinae_test_impl<_Templ, _Args...>(0));
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_VALID_EXPANSION_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_void.h b/libcxx/include/__cxx03/__type_traits/is_void.h
new file mode 100644
index 000000000000..562faae9fba2
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_void.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_VOID_H
+#define _LIBCPP___TYPE_TRAITS_IS_VOID_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_void : _BoolConstant<__is_same(__remove_cv(_Tp), void)> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_void_v = __is_same(__remove_cv(_Tp), void);
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_VOID_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_volatile.h b/libcxx/include/__cxx03/__type_traits/is_volatile.h
new file mode 100644
index 000000000000..87960a819c8f
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/is_volatile.h
@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_VOLATILE_H
+#define _LIBCPP___TYPE_TRAITS_IS_VOLATILE_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__is_volatile)
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_volatile : _BoolConstant<__is_volatile(_Tp)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_volatile_v = __is_volatile(_Tp);
+#  endif
+
+#else
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_volatile : public false_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_volatile<_Tp volatile> : public true_type {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr bool is_volatile_v = is_volatile<_Tp>::value;
+#  endif
+
+#endif // __has_builtin(__is_volatile)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_VOLATILE_H
diff --git a/libcxx/include/__cxx03/__type_traits/lazy.h b/libcxx/include/__cxx03/__type_traits/lazy.h
new file mode 100644
index 000000000000..80826f1d64f6
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/lazy.h
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_LAZY_H
+#define _LIBCPP___TYPE_TRAITS_LAZY_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <template <class...> class _Func, class... _Args>
+struct _Lazy : _Func<_Args...> {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_LAZY_H
diff --git a/libcxx/include/__cxx03/__type_traits/make_32_64_or_128_bit.h b/libcxx/include/__cxx03/__type_traits/make_32_64_or_128_bit.h
new file mode 100644
index 000000000000..f7f2e81735da
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/make_32_64_or_128_bit.h
@@ -0,0 +1,49 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_MAKE_32_64_OR_128_BIT_H
+#define _LIBCPP___TYPE_TRAITS_MAKE_32_64_OR_128_BIT_H
+
+#include <__config>
+#include <__type_traits/conditional.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_signed.h>
+#include <__type_traits/is_unsigned.h>
+#include <__type_traits/make_unsigned.h>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+/// Helper to promote an integral to smallest 32, 64, or 128 bit representation.
+///
+/// The restriction is the same as the integral version of to_char.
+template <class _Tp>
+#if _LIBCPP_STD_VER >= 20
+  requires(is_signed_v<_Tp> || is_unsigned_v<_Tp> || is_same_v<_Tp, char>)
+#endif
+// clang-format off
+using __make_32_64_or_128_bit_t =
+    __copy_unsigned_t<_Tp,
+        __conditional_t<sizeof(_Tp) <= sizeof(int32_t),    int32_t,
+        __conditional_t<sizeof(_Tp) <= sizeof(int64_t),    int64_t,
+#ifndef _LIBCPP_HAS_NO_INT128
+        __conditional_t<sizeof(_Tp) <= sizeof(__int128_t), __int128_t,
+        /* else */                                         void>
+#else
+        /* else */                                         void
+#endif
+    > > >;
+// clang-format on
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_MAKE_32_64_OR_128_BIT_H
diff --git a/libcxx/include/__cxx03/__type_traits/make_const_lvalue_ref.h b/libcxx/include/__cxx03/__type_traits/make_const_lvalue_ref.h
new file mode 100644
index 000000000000..469d4cb31ef7
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/make_const_lvalue_ref.h
@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_MAKE_CONST_LVALUE_REF_H
+#define _LIBCPP___TYPE_TRAITS_MAKE_CONST_LVALUE_REF_H
+
+#include <__config>
+#include <__type_traits/remove_reference.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+using __make_const_lvalue_ref = const __libcpp_remove_reference_t<_Tp>&;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_MAKE_CONST_LVALUE_REF_H
diff --git a/libcxx/include/__cxx03/__type_traits/make_signed.h b/libcxx/include/__cxx03/__type_traits/make_signed.h
new file mode 100644
index 000000000000..d09d6ed4a1e7
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/make_signed.h
@@ -0,0 +1,89 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_MAKE_SIGNED_H
+#define _LIBCPP___TYPE_TRAITS_MAKE_SIGNED_H
+
+#include <__config>
+#include <__type_traits/copy_cv.h>
+#include <__type_traits/is_enum.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/nat.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/type_list.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__make_signed)
+
+template <class _Tp>
+using __make_signed_t = __make_signed(_Tp);
+
+#else
+// clang-format off
+typedef __type_list<signed char,
+        __type_list<signed short,
+        __type_list<signed int,
+        __type_list<signed long,
+        __type_list<signed long long,
+#  ifndef _LIBCPP_HAS_NO_INT128
+        __type_list<__int128_t,
+#  endif
+        __nat
+#  ifndef _LIBCPP_HAS_NO_INT128
+        >
+#  endif
+        > > > > > __signed_types;
+// clang-format on
+
+template <class _Tp, bool = is_integral<_Tp>::value || is_enum<_Tp>::value>
+struct __make_signed{};
+
+template <class _Tp>
+struct __make_signed<_Tp, true> {
+  typedef typename __find_first<__signed_types, sizeof(_Tp)>::type type;
+};
+
+// clang-format off
+template <> struct __make_signed<bool,               true> {};
+template <> struct __make_signed<  signed short,     true> {typedef short     type;};
+template <> struct __make_signed<unsigned short,     true> {typedef short     type;};
+template <> struct __make_signed<  signed int,       true> {typedef int       type;};
+template <> struct __make_signed<unsigned int,       true> {typedef int       type;};
+template <> struct __make_signed<  signed long,      true> {typedef long      type;};
+template <> struct __make_signed<unsigned long,      true> {typedef long      type;};
+template <> struct __make_signed<  signed long long, true> {typedef long long type;};
+template <> struct __make_signed<unsigned long long, true> {typedef long long type;};
+#  ifndef _LIBCPP_HAS_NO_INT128
+template <> struct __make_signed<__int128_t,         true> {typedef __int128_t type;};
+template <> struct __make_signed<__uint128_t,        true> {typedef __int128_t type;};
+#  endif
+// clang-format on
+
+template <class _Tp>
+using __make_signed_t = __copy_cv_t<_Tp, typename __make_signed<__remove_cv_t<_Tp> >::type>;
+
+#endif // __has_builtin(__make_signed)
+
+template <class _Tp>
+struct make_signed {
+  using type _LIBCPP_NODEBUG = __make_signed_t<_Tp>;
+};
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using make_signed_t = __make_signed_t<_Tp>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_MAKE_SIGNED_H
diff --git a/libcxx/include/__cxx03/__type_traits/make_unsigned.h b/libcxx/include/__cxx03/__type_traits/make_unsigned.h
new file mode 100644
index 000000000000..282cd2d91131
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/make_unsigned.h
@@ -0,0 +1,101 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_MAKE_UNSIGNED_H
+#define _LIBCPP___TYPE_TRAITS_MAKE_UNSIGNED_H
+
+#include <__config>
+#include <__type_traits/conditional.h>
+#include <__type_traits/copy_cv.h>
+#include <__type_traits/is_enum.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_unsigned.h>
+#include <__type_traits/nat.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/type_list.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__make_unsigned)
+
+template <class _Tp>
+using __make_unsigned_t = __make_unsigned(_Tp);
+
+#else
+// clang-format off
+typedef __type_list<unsigned char,
+        __type_list<unsigned short,
+        __type_list<unsigned int,
+        __type_list<unsigned long,
+        __type_list<unsigned long long,
+#  ifndef _LIBCPP_HAS_NO_INT128
+        __type_list<__uint128_t,
+#  endif
+        __nat
+#  ifndef _LIBCPP_HAS_NO_INT128
+        >
+#  endif
+        > > > > > __unsigned_types;
+// clang-format on
+
+template <class _Tp, bool = is_integral<_Tp>::value || is_enum<_Tp>::value>
+struct __make_unsigned{};
+
+template <class _Tp>
+struct __make_unsigned<_Tp, true> {
+  typedef typename __find_first<__unsigned_types, sizeof(_Tp)>::type type;
+};
+
+// clang-format off
+template <> struct __make_unsigned<bool,               true> {};
+template <> struct __make_unsigned<  signed short,     true> {typedef unsigned short     type;};
+template <> struct __make_unsigned<unsigned short,     true> {typedef unsigned short     type;};
+template <> struct __make_unsigned<  signed int,       true> {typedef unsigned int       type;};
+template <> struct __make_unsigned<unsigned int,       true> {typedef unsigned int       type;};
+template <> struct __make_unsigned<  signed long,      true> {typedef unsigned long      type;};
+template <> struct __make_unsigned<unsigned long,      true> {typedef unsigned long      type;};
+template <> struct __make_unsigned<  signed long long, true> {typedef unsigned long long type;};
+template <> struct __make_unsigned<unsigned long long, true> {typedef unsigned long long type;};
+#  ifndef _LIBCPP_HAS_NO_INT128
+template <> struct __make_unsigned<__int128_t,         true> {typedef __uint128_t        type;};
+template <> struct __make_unsigned<__uint128_t,        true> {typedef __uint128_t        type;};
+#  endif
+// clang-format on
+
+template <class _Tp>
+using __make_unsigned_t = __copy_cv_t<_Tp, typename __make_unsigned<__remove_cv_t<_Tp> >::type>;
+
+#endif // __has_builtin(__make_unsigned)
+
+template <class _Tp>
+struct make_unsigned {
+  using type _LIBCPP_NODEBUG = __make_unsigned_t<_Tp>;
+};
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using make_unsigned_t = __make_unsigned_t<_Tp>;
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr __make_unsigned_t<_Tp> __to_unsigned_like(_Tp __x) noexcept {
+  return static_cast<__make_unsigned_t<_Tp> >(__x);
+}
+#endif
+
+template <class _Tp, class _Up>
+using __copy_unsigned_t = __conditional_t<is_unsigned<_Tp>::value, __make_unsigned_t<_Up>, _Up>;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_MAKE_UNSIGNED_H
diff --git a/libcxx/include/__cxx03/__type_traits/maybe_const.h b/libcxx/include/__cxx03/__type_traits/maybe_const.h
new file mode 100644
index 000000000000..25fba58fb773
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/maybe_const.h
@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_MAYBE_CONST_H
+#define _LIBCPP___TYPE_TRAITS_MAYBE_CONST_H
+
+#include <__config>
+#include <__type_traits/conditional.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <bool _Const, class _Tp>
+using __maybe_const = __conditional_t<_Const, const _Tp, _Tp>;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_MAYBE_CONST_H
diff --git a/libcxx/include/__cxx03/__type_traits/nat.h b/libcxx/include/__cxx03/__type_traits/nat.h
new file mode 100644
index 000000000000..9f39b806814e
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/nat.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_NAT_H
+#define _LIBCPP___TYPE_TRAITS_NAT_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct __nat {
+#ifndef _LIBCPP_CXX03_LANG
+  __nat()                        = delete;
+  __nat(const __nat&)            = delete;
+  __nat& operator=(const __nat&) = delete;
+  ~__nat()                       = delete;
+#endif
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_NAT_H
diff --git a/libcxx/include/__cxx03/__type_traits/negation.h b/libcxx/include/__cxx03/__type_traits/negation.h
new file mode 100644
index 000000000000..a72e62d3f96e
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/negation.h
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_NEGATION_H
+#define _LIBCPP___TYPE_TRAITS_NEGATION_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Pred>
+struct _Not : _BoolConstant<!_Pred::value> {};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+struct negation : _Not<_Tp> {};
+template <class _Tp>
+inline constexpr bool negation_v = !_Tp::value;
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_NEGATION_H
diff --git a/libcxx/include/__cxx03/__type_traits/noexcept_move_assign_container.h b/libcxx/include/__cxx03/__type_traits/noexcept_move_assign_container.h
new file mode 100644
index 000000000000..baaf36d9980e
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/noexcept_move_assign_container.h
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_NOEXCEPT_MOVE_ASSIGN_CONTAINER_H
+#define _LIBCPP___TYPE_TRAITS_NOEXCEPT_MOVE_ASSIGN_CONTAINER_H
+
+#include <__config>
+#include <__memory/allocator_traits.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_nothrow_assignable.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <typename _Alloc, typename _Traits = allocator_traits<_Alloc> >
+struct __noexcept_move_assign_container
+    : public integral_constant<bool,
+                               _Traits::propagate_on_container_move_assignment::value
+#if _LIBCPP_STD_VER >= 17
+                                   || _Traits::is_always_equal::value
+#else
+                                   && is_nothrow_move_assignable<_Alloc>::value
+#endif
+                               > {
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_NOEXCEPT_MOVE_ASSIGN_CONTAINER_H
diff --git a/libcxx/include/__cxx03/__type_traits/promote.h b/libcxx/include/__cxx03/__type_traits/promote.h
new file mode 100644
index 000000000000..2b2a6843b915
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/promote.h
@@ -0,0 +1,128 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_PROMOTE_H
+#define _LIBCPP___TYPE_TRAITS_PROMOTE_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_arithmetic.h>
+
+#if defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER == 1700
+#  include <__type_traits/is_same.h>
+#  include <__utility/declval.h>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// TODO(LLVM-20): Remove this workaround
+#if !defined(_LIBCPP_CLANG_VER) || _LIBCPP_CLANG_VER != 1700
+
+template <class... _Args>
+class __promote {
+  static_assert((is_arithmetic<_Args>::value && ...));
+
+  static float __test(float);
+  static double __test(char);
+  static double __test(int);
+  static double __test(unsigned);
+  static double __test(long);
+  static double __test(unsigned long);
+  static double __test(long long);
+  static double __test(unsigned long long);
+#  ifndef _LIBCPP_HAS_NO_INT128
+  static double __test(__int128_t);
+  static double __test(__uint128_t);
+#  endif
+  static double __test(double);
+  static long double __test(long double);
+
+public:
+  using type = decltype((__test(_Args()) + ...));
+};
+
+#else
+
+template <class _Tp>
+struct __numeric_type {
+  static void __test(...);
+  static float __test(float);
+  static double __test(char);
+  static double __test(int);
+  static double __test(unsigned);
+  static double __test(long);
+  static double __test(unsigned long);
+  static double __test(long long);
+  static double __test(unsigned long long);
+#  ifndef _LIBCPP_HAS_NO_INT128
+  static double __test(__int128_t);
+  static double __test(__uint128_t);
+#  endif
+  static double __test(double);
+  static long double __test(long double);
+
+  typedef decltype(__test(std::declval<_Tp>())) type;
+  static const bool value = _IsNotSame<type, void>::value;
+};
+
+template <>
+struct __numeric_type<void> {
+  static const bool value = true;
+};
+
+template <class _A1,
+          class _A2 = void,
+          class _A3 = void,
+          bool      = __numeric_type<_A1>::value && __numeric_type<_A2>::value && __numeric_type<_A3>::value>
+class __promote_imp {
+public:
+  static const bool value = false;
+};
+
+template <class _A1, class _A2, class _A3>
+class __promote_imp<_A1, _A2, _A3, true> {
+private:
+  typedef typename __promote_imp<_A1>::type __type1;
+  typedef typename __promote_imp<_A2>::type __type2;
+  typedef typename __promote_imp<_A3>::type __type3;
+
+public:
+  typedef decltype(__type1() + __type2() + __type3()) type;
+  static const bool value = true;
+};
+
+template <class _A1, class _A2>
+class __promote_imp<_A1, _A2, void, true> {
+private:
+  typedef typename __promote_imp<_A1>::type __type1;
+  typedef typename __promote_imp<_A2>::type __type2;
+
+public:
+  typedef decltype(__type1() + __type2()) type;
+  static const bool value = true;
+};
+
+template <class _A1>
+class __promote_imp<_A1, void, void, true> {
+public:
+  typedef typename __numeric_type<_A1>::type type;
+  static const bool value = true;
+};
+
+template <class _A1, class _A2 = void, class _A3 = void>
+class __promote : public __promote_imp<_A1, _A2, _A3> {};
+
+#endif // !defined(_LIBCPP_CLANG_VER) || _LIBCPP_CLANG_VER >= 1700
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_PROMOTE_H
diff --git a/libcxx/include/__cxx03/__type_traits/rank.h b/libcxx/include/__cxx03/__type_traits/rank.h
new file mode 100644
index 000000000000..7f6fad1c5402
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/rank.h
@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_RANK_H
+#define _LIBCPP___TYPE_TRAITS_RANK_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// TODO: Enable using the builtin __array_rank when https://llvm.org/PR57133 is resolved
+#if __has_builtin(__array_rank) && 0
+
+template <class _Tp>
+struct rank : integral_constant<size_t, __array_rank(_Tp)> {};
+
+#else
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS rank : public integral_constant<size_t, 0> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS rank<_Tp[]> : public integral_constant<size_t, rank<_Tp>::value + 1> {};
+template <class _Tp, size_t _Np>
+struct _LIBCPP_TEMPLATE_VIS rank<_Tp[_Np]> : public integral_constant<size_t, rank<_Tp>::value + 1> {};
+
+#endif // __has_builtin(__array_rank)
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+inline constexpr size_t rank_v = rank<_Tp>::value;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_RANK_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_all_extents.h b/libcxx/include/__cxx03/__type_traits/remove_all_extents.h
new file mode 100644
index 000000000000..d5373b51f522
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/remove_all_extents.h
@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_ALL_EXTENTS_H
+#define _LIBCPP___TYPE_TRAITS_REMOVE_ALL_EXTENTS_H
+
+#include <__config>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__remove_all_extents)
+template <class _Tp>
+struct remove_all_extents {
+  using type _LIBCPP_NODEBUG = __remove_all_extents(_Tp);
+};
+
+template <class _Tp>
+using __remove_all_extents_t = __remove_all_extents(_Tp);
+#else
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_all_extents {
+  typedef _Tp type;
+};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_all_extents<_Tp[]> {
+  typedef typename remove_all_extents<_Tp>::type type;
+};
+template <class _Tp, size_t _Np>
+struct _LIBCPP_TEMPLATE_VIS remove_all_extents<_Tp[_Np]> {
+  typedef typename remove_all_extents<_Tp>::type type;
+};
+
+template <class _Tp>
+using __remove_all_extents_t = typename remove_all_extents<_Tp>::type;
+#endif // __has_builtin(__remove_all_extents)
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using remove_all_extents_t = __remove_all_extents_t<_Tp>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_REMOVE_ALL_EXTENTS_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_const.h b/libcxx/include/__cxx03/__type_traits/remove_const.h
new file mode 100644
index 000000000000..a3f0648c4785
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/remove_const.h
@@ -0,0 +1,49 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_CONST_H
+#define _LIBCPP___TYPE_TRAITS_REMOVE_CONST_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__remove_const)
+template <class _Tp>
+struct remove_const {
+  using type _LIBCPP_NODEBUG = __remove_const(_Tp);
+};
+
+template <class _Tp>
+using __remove_const_t = __remove_const(_Tp);
+#else
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_const {
+  typedef _Tp type;
+};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_const<const _Tp> {
+  typedef _Tp type;
+};
+
+template <class _Tp>
+using __remove_const_t = typename remove_const<_Tp>::type;
+#endif // __has_builtin(__remove_const)
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using remove_const_t = __remove_const_t<_Tp>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_REMOVE_CONST_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_const_ref.h b/libcxx/include/__cxx03/__type_traits/remove_const_ref.h
new file mode 100644
index 000000000000..d3b334935a5b
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/remove_const_ref.h
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_CONST_REF_H
+#define _LIBCPP___TYPE_TRAITS_REMOVE_CONST_REF_H
+
+#include <__config>
+#include <__type_traits/remove_const.h>
+#include <__type_traits/remove_reference.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+using __remove_const_ref_t = __remove_const_t<__libcpp_remove_reference_t<_Tp> >;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_REMOVE_CONST_REF_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_cv.h b/libcxx/include/__cxx03/__type_traits/remove_cv.h
new file mode 100644
index 000000000000..c4bf612794bd
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/remove_cv.h
@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_CV_H
+#define _LIBCPP___TYPE_TRAITS_REMOVE_CV_H
+
+#include <__config>
+#include <__type_traits/remove_const.h>
+#include <__type_traits/remove_volatile.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__remove_cv) && !defined(_LIBCPP_COMPILER_GCC)
+template <class _Tp>
+struct remove_cv {
+  using type _LIBCPP_NODEBUG = __remove_cv(_Tp);
+};
+
+template <class _Tp>
+using __remove_cv_t = __remove_cv(_Tp);
+#else
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_cv {
+  typedef __remove_volatile_t<__remove_const_t<_Tp> > type;
+};
+
+template <class _Tp>
+using __remove_cv_t = __remove_volatile_t<__remove_const_t<_Tp> >;
+#endif // __has_builtin(__remove_cv)
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using remove_cv_t = __remove_cv_t<_Tp>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_REMOVE_CV_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_cvref.h b/libcxx/include/__cxx03/__type_traits/remove_cvref.h
new file mode 100644
index 000000000000..e8e8745ab096
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/remove_cvref.h
@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_CVREF_H
+#define _LIBCPP___TYPE_TRAITS_REMOVE_CVREF_H
+
+#include <__config>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/remove_reference.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__remove_cvref) && !defined(_LIBCPP_COMPILER_GCC)
+template <class _Tp>
+using __remove_cvref_t _LIBCPP_NODEBUG = __remove_cvref(_Tp);
+#else
+template <class _Tp>
+using __remove_cvref_t _LIBCPP_NODEBUG = __remove_cv_t<__libcpp_remove_reference_t<_Tp> >;
+#endif // __has_builtin(__remove_cvref)
+
+template <class _Tp, class _Up>
+struct __is_same_uncvref : _IsSame<__remove_cvref_t<_Tp>, __remove_cvref_t<_Up> > {};
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp>
+struct remove_cvref {
+  using type _LIBCPP_NODEBUG = __remove_cvref_t<_Tp>;
+};
+
+template <class _Tp>
+using remove_cvref_t = __remove_cvref_t<_Tp>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_REMOVE_CVREF_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_extent.h b/libcxx/include/__cxx03/__type_traits/remove_extent.h
new file mode 100644
index 000000000000..fe37b5c7266c
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/remove_extent.h
@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_EXTENT_H
+#define _LIBCPP___TYPE_TRAITS_REMOVE_EXTENT_H
+
+#include <__config>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__remove_extent)
+template <class _Tp>
+struct remove_extent {
+  using type _LIBCPP_NODEBUG = __remove_extent(_Tp);
+};
+
+template <class _Tp>
+using __remove_extent_t = __remove_extent(_Tp);
+#else
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_extent {
+  typedef _Tp type;
+};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_extent<_Tp[]> {
+  typedef _Tp type;
+};
+template <class _Tp, size_t _Np>
+struct _LIBCPP_TEMPLATE_VIS remove_extent<_Tp[_Np]> {
+  typedef _Tp type;
+};
+
+template <class _Tp>
+using __remove_extent_t = typename remove_extent<_Tp>::type;
+#endif // __has_builtin(__remove_extent)
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using remove_extent_t = __remove_extent_t<_Tp>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_REMOVE_EXTENT_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_pointer.h b/libcxx/include/__cxx03/__type_traits/remove_pointer.h
new file mode 100644
index 000000000000..1048f67055a2
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/remove_pointer.h
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_POINTER_H
+#define _LIBCPP___TYPE_TRAITS_REMOVE_POINTER_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if !defined(_LIBCPP_WORKAROUND_OBJCXX_COMPILER_INTRINSICS) && __has_builtin(__remove_pointer)
+template <class _Tp>
+struct remove_pointer {
+  using type _LIBCPP_NODEBUG = __remove_pointer(_Tp);
+};
+
+#  ifdef _LIBCPP_COMPILER_GCC
+template <class _Tp>
+using __remove_pointer_t = typename remove_pointer<_Tp>::type;
+#  else
+template <class _Tp>
+using __remove_pointer_t = __remove_pointer(_Tp);
+#  endif
+#else
+// clang-format off
+template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_pointer                      {typedef _LIBCPP_NODEBUG _Tp type;};
+template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_pointer<_Tp*>                {typedef _LIBCPP_NODEBUG _Tp type;};
+template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_pointer<_Tp* const>          {typedef _LIBCPP_NODEBUG _Tp type;};
+template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_pointer<_Tp* volatile>       {typedef _LIBCPP_NODEBUG _Tp type;};
+template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_pointer<_Tp* const volatile> {typedef _LIBCPP_NODEBUG _Tp type;};
+// clang-format on
+
+template <class _Tp>
+using __remove_pointer_t = typename remove_pointer<_Tp>::type;
+#endif // !defined(_LIBCPP_WORKAROUND_OBJCXX_COMPILER_INTRINSICS) && __has_builtin(__remove_pointer)
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using remove_pointer_t = __remove_pointer_t<_Tp>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_REMOVE_POINTER_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_reference.h b/libcxx/include/__cxx03/__type_traits/remove_reference.h
new file mode 100644
index 000000000000..ba67891758ad
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/remove_reference.h
@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_REFERENCE_H
+#define _LIBCPP___TYPE_TRAITS_REMOVE_REFERENCE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__remove_reference_t)
+template <class _Tp>
+struct remove_reference {
+  using type _LIBCPP_NODEBUG = __remove_reference_t(_Tp);
+};
+
+template <class _Tp>
+using __libcpp_remove_reference_t = __remove_reference_t(_Tp);
+#elif __has_builtin(__remove_reference)
+template <class _Tp>
+struct remove_reference {
+  using type _LIBCPP_NODEBUG = __remove_reference(_Tp);
+};
+
+template <class _Tp>
+using __libcpp_remove_reference_t = typename remove_reference<_Tp>::type;
+#else
+#  error "remove_reference not implemented!"
+#endif // __has_builtin(__remove_reference_t)
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using remove_reference_t = __libcpp_remove_reference_t<_Tp>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_REMOVE_REFERENCE_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_volatile.h b/libcxx/include/__cxx03/__type_traits/remove_volatile.h
new file mode 100644
index 000000000000..7600ae0ec516
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/remove_volatile.h
@@ -0,0 +1,49 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_VOLATILE_H
+#define _LIBCPP___TYPE_TRAITS_REMOVE_VOLATILE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if __has_builtin(__remove_volatile)
+template <class _Tp>
+struct remove_volatile {
+  using type _LIBCPP_NODEBUG = __remove_volatile(_Tp);
+};
+
+template <class _Tp>
+using __remove_volatile_t = __remove_volatile(_Tp);
+#else
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_volatile {
+  typedef _Tp type;
+};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_volatile<volatile _Tp> {
+  typedef _Tp type;
+};
+
+template <class _Tp>
+using __remove_volatile_t = typename remove_volatile<_Tp>::type;
+#endif // __has_builtin(__remove_volatile)
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using remove_volatile_t = __remove_volatile_t<_Tp>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_REMOVE_VOLATILE_H
diff --git a/libcxx/include/__cxx03/__type_traits/result_of.h b/libcxx/include/__cxx03/__type_traits/result_of.h
new file mode 100644
index 000000000000..f00fa8e9be7f
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/result_of.h
@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_RESULT_OF_H
+#define _LIBCPP___TYPE_TRAITS_RESULT_OF_H
+
+#include <__config>
+#include <__functional/invoke.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// result_of
+
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_TYPE_TRAITS)
+template <class _Callable>
+class _LIBCPP_DEPRECATED_IN_CXX17 result_of;
+
+template <class _Fp, class... _Args>
+class _LIBCPP_TEMPLATE_VIS result_of<_Fp(_Args...)> : public __invoke_of<_Fp, _Args...> {};
+
+#  if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using result_of_t _LIBCPP_DEPRECATED_IN_CXX17 = typename result_of<_Tp>::type;
+#  endif // _LIBCPP_STD_VER >= 14
+#endif   // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_TYPE_TRAITS)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_RESULT_OF_H
diff --git a/libcxx/include/__cxx03/__type_traits/strip_signature.h b/libcxx/include/__cxx03/__type_traits/strip_signature.h
new file mode 100644
index 000000000000..3fe79592f55b
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/strip_signature.h
@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_STRIP_SIGNATURE_H
+#define _LIBCPP___TYPE_TRAITS_STRIP_SIGNATURE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Fp>
+struct __strip_signature;
+
+#  if defined(__cpp_static_call_operator) && __cpp_static_call_operator >= 202207L
+
+template <class _Rp, class... _Args>
+struct __strip_signature<_Rp (*)(_Args...)> {
+  using type = _Rp(_Args...);
+};
+
+template <class _Rp, class... _Args>
+struct __strip_signature<_Rp (*)(_Args...) noexcept> {
+  using type = _Rp(_Args...);
+};
+
+#  endif // defined(__cpp_static_call_operator) && __cpp_static_call_operator >= 202207L
+
+// clang-format off
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...)> { using type = _Rp(_Ap...); };
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...) const> { using type = _Rp(_Ap...); };
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...) volatile> { using type = _Rp(_Ap...); };
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...) const volatile> { using type = _Rp(_Ap...); };
+
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...) &> { using type = _Rp(_Ap...); };
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...) const &> { using type = _Rp(_Ap...); };
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...) volatile &> { using type = _Rp(_Ap...); };
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...) const volatile &> { using type = _Rp(_Ap...); };
+
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...) noexcept> { using type = _Rp(_Ap...); };
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...) const noexcept> { using type = _Rp(_Ap...); };
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...) volatile noexcept> { using type = _Rp(_Ap...); };
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...) const volatile noexcept> { using type = _Rp(_Ap...); };
+
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...) & noexcept> { using type = _Rp(_Ap...); };
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...) const & noexcept> { using type = _Rp(_Ap...); };
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...) volatile & noexcept> { using type = _Rp(_Ap...); };
+template<class _Rp, class _Gp, class ..._Ap>
+struct __strip_signature<_Rp (_Gp::*) (_Ap...) const volatile & noexcept> { using type = _Rp(_Ap...); };
+// clang-format on
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___TYPE_TRAITS_STRIP_SIGNATURE_H
diff --git a/libcxx/include/__cxx03/__type_traits/type_identity.h b/libcxx/include/__cxx03/__type_traits/type_identity.h
new file mode 100644
index 000000000000..b0b5a1277d59
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/type_identity.h
@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_TYPE_IDENTITY_H
+#define _LIBCPP___TYPE_TRAITS_TYPE_IDENTITY_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct __type_identity {
+  typedef _Tp type;
+};
+
+template <class _Tp>
+using __type_identity_t _LIBCPP_NODEBUG = typename __type_identity<_Tp>::type;
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp>
+struct type_identity {
+  typedef _Tp type;
+};
+template <class _Tp>
+using type_identity_t = typename type_identity<_Tp>::type;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_TYPE_IDENTITY_H
diff --git a/libcxx/include/__cxx03/__type_traits/type_list.h b/libcxx/include/__cxx03/__type_traits/type_list.h
new file mode 100644
index 000000000000..02905707ee37
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/type_list.h
@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_TYPE_LIST_H
+#define _LIBCPP___TYPE_TRAITS_TYPE_LIST_H
+
+#include <__config>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Hp, class _Tp>
+struct __type_list {
+  typedef _Hp _Head;
+  typedef _Tp _Tail;
+};
+
+template <class _TypeList, size_t _Size, bool = _Size <= sizeof(typename _TypeList::_Head)>
+struct __find_first;
+
+template <class _Hp, class _Tp, size_t _Size>
+struct __find_first<__type_list<_Hp, _Tp>, _Size, true> {
+  typedef _LIBCPP_NODEBUG _Hp type;
+};
+
+template <class _Hp, class _Tp, size_t _Size>
+struct __find_first<__type_list<_Hp, _Tp>, _Size, false> {
+  typedef _LIBCPP_NODEBUG typename __find_first<_Tp, _Size>::type type;
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_TYPE_LIST_H
diff --git a/libcxx/include/__cxx03/__type_traits/underlying_type.h b/libcxx/include/__cxx03/__type_traits/underlying_type.h
new file mode 100644
index 000000000000..16e7501dee17
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/underlying_type.h
@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_UNDERLYING_TYPE_H
+#define _LIBCPP___TYPE_TRAITS_UNDERLYING_TYPE_H
+
+#include <__config>
+#include <__type_traits/is_enum.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, bool = is_enum<_Tp>::value>
+struct __underlying_type_impl;
+
+template <class _Tp>
+struct __underlying_type_impl<_Tp, false> {};
+
+template <class _Tp>
+struct __underlying_type_impl<_Tp, true> {
+  typedef __underlying_type(_Tp) type;
+};
+
+template <class _Tp>
+struct underlying_type : __underlying_type_impl<_Tp, is_enum<_Tp>::value> {};
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using underlying_type_t = typename underlying_type<_Tp>::type;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_UNDERLYING_TYPE_H
diff --git a/libcxx/include/__cxx03/__type_traits/unwrap_ref.h b/libcxx/include/__cxx03/__type_traits/unwrap_ref.h
new file mode 100644
index 000000000000..6bd74550f309
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/unwrap_ref.h
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_UNWRAP_REF_H
+#define _LIBCPP___TYPE_TRAITS_UNWRAP_REF_H
+
+#include <__config>
+#include <__fwd/functional.h>
+#include <__type_traits/decay.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct __unwrap_reference {
+  typedef _LIBCPP_NODEBUG _Tp type;
+};
+
+template <class _Tp>
+struct __unwrap_reference<reference_wrapper<_Tp> > {
+  typedef _LIBCPP_NODEBUG _Tp& type;
+};
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp>
+struct unwrap_reference : __unwrap_reference<_Tp> {};
+
+template <class _Tp>
+using unwrap_reference_t = typename unwrap_reference<_Tp>::type;
+
+template <class _Tp>
+struct unwrap_ref_decay : unwrap_reference<__decay_t<_Tp> > {};
+
+template <class _Tp>
+using unwrap_ref_decay_t = typename unwrap_ref_decay<_Tp>::type;
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+struct __unwrap_ref_decay
+#if _LIBCPP_STD_VER >= 20
+    : unwrap_ref_decay<_Tp>
+#else
+    : __unwrap_reference<__decay_t<_Tp> >
+#endif
+{
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_UNWRAP_REF_H
diff --git a/libcxx/include/__cxx03/__type_traits/void_t.h b/libcxx/include/__cxx03/__type_traits/void_t.h
new file mode 100644
index 000000000000..985bba02e72f
--- /dev/null
+++ b/libcxx/include/__cxx03/__type_traits/void_t.h
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_VOID_T_H
+#define _LIBCPP___TYPE_TRAITS_VOID_T_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+template <class...>
+using void_t = void;
+#endif
+
+template <class...>
+using __void_t = void;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_VOID_T_H
diff --git a/libcxx/include/__cxx03/__undef_macros b/libcxx/include/__cxx03/__undef_macros
new file mode 100644
index 000000000000..29ab327e1c37
--- /dev/null
+++ b/libcxx/include/__cxx03/__undef_macros
@@ -0,0 +1,28 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef min
+#  undef min
+#endif
+
+#ifdef max
+#  undef max
+#endif
+
+#ifdef refresh
+#  undef refresh
+#endif
+
+#ifdef move
+#  undef move
+#endif
+
+#ifdef erase
+#  undef erase
+#endif
diff --git a/libcxx/include/__cxx03/__utility/as_const.h b/libcxx/include/__cxx03/__utility/as_const.h
new file mode 100644
index 000000000000..582dd42f4079
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/as_const.h
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_AS_CONST_H
+#define _LIBCPP___UTILITY_AS_CONST_H
+
+#include <__config>
+#include <__type_traits/add_const.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr add_const_t<_Tp>& as_const(_Tp& __t) noexcept {
+  return __t;
+}
+
+template <class _Tp>
+void as_const(const _Tp&&) = delete;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_AS_CONST_H
diff --git a/libcxx/include/__cxx03/__utility/as_lvalue.h b/libcxx/include/__cxx03/__utility/as_lvalue.h
new file mode 100644
index 000000000000..159f45dad4d4
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/as_lvalue.h
@@ -0,0 +1,37 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_AS_LVALUE_H
+#define _LIBCPP___UTILITY_AS_LVALUE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp& __as_lvalue(_LIBCPP_LIFETIMEBOUND _Tp&& __t) {
+  return static_cast<_Tp&>(__t);
+}
+
+#endif // !_LIBCPP_CXX03_LANG
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___UTILITY_AS_LVALUE_H
diff --git a/libcxx/include/__cxx03/__utility/auto_cast.h b/libcxx/include/__cxx03/__utility/auto_cast.h
new file mode 100644
index 000000000000..06715b3438f9
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/auto_cast.h
@@ -0,0 +1,22 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_AUTO_CAST_H
+#define _LIBCPP___UTILITY_AUTO_CAST_H
+
+#include <__config>
+#include <__type_traits/decay.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#define _LIBCPP_AUTO_CAST(expr) static_cast<::std::__decay_t<decltype((expr))> >(expr)
+
+#endif // _LIBCPP___UTILITY_AUTO_CAST_H
diff --git a/libcxx/include/__cxx03/__utility/cmp.h b/libcxx/include/__cxx03/__utility/cmp.h
new file mode 100644
index 000000000000..b7c1ed614dfc
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/cmp.h
@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_CMP_H
+#define _LIBCPP___UTILITY_CMP_H
+
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <__type_traits/is_signed.h>
+#include <__type_traits/make_unsigned.h>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <__libcpp_integer _Tp, __libcpp_integer _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool cmp_equal(_Tp __t, _Up __u) noexcept {
+  if constexpr (is_signed_v<_Tp> == is_signed_v<_Up>)
+    return __t == __u;
+  else if constexpr (is_signed_v<_Tp>)
+    return __t < 0 ? false : make_unsigned_t<_Tp>(__t) == __u;
+  else
+    return __u < 0 ? false : __t == make_unsigned_t<_Up>(__u);
+}
+
+template <__libcpp_integer _Tp, __libcpp_integer _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool cmp_not_equal(_Tp __t, _Up __u) noexcept {
+  return !std::cmp_equal(__t, __u);
+}
+
+template <__libcpp_integer _Tp, __libcpp_integer _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool cmp_less(_Tp __t, _Up __u) noexcept {
+  if constexpr (is_signed_v<_Tp> == is_signed_v<_Up>)
+    return __t < __u;
+  else if constexpr (is_signed_v<_Tp>)
+    return __t < 0 ? true : make_unsigned_t<_Tp>(__t) < __u;
+  else
+    return __u < 0 ? false : __t < make_unsigned_t<_Up>(__u);
+}
+
+template <__libcpp_integer _Tp, __libcpp_integer _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool cmp_greater(_Tp __t, _Up __u) noexcept {
+  return std::cmp_less(__u, __t);
+}
+
+template <__libcpp_integer _Tp, __libcpp_integer _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool cmp_less_equal(_Tp __t, _Up __u) noexcept {
+  return !std::cmp_greater(__t, __u);
+}
+
+template <__libcpp_integer _Tp, __libcpp_integer _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool cmp_greater_equal(_Tp __t, _Up __u) noexcept {
+  return !std::cmp_less(__t, __u);
+}
+
+template <__libcpp_integer _Tp, __libcpp_integer _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool in_range(_Up __u) noexcept {
+  return std::cmp_less_equal(__u, numeric_limits<_Tp>::max()) &&
+         std::cmp_greater_equal(__u, numeric_limits<_Tp>::min());
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___UTILITY_CMP_H
diff --git a/libcxx/include/__cxx03/__utility/convert_to_integral.h b/libcxx/include/__cxx03/__utility/convert_to_integral.h
new file mode 100644
index 000000000000..f1fcdd98010c
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/convert_to_integral.h
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_CONVERT_TO_INTEGRAL_H
+#define _LIBCPP___UTILITY_CONVERT_TO_INTEGRAL_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_enum.h>
+#include <__type_traits/is_floating_point.h>
+#include <__type_traits/underlying_type.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __convert_to_integral(int __val) { return __val; }
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR unsigned __convert_to_integral(unsigned __val) { return __val; }
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR long __convert_to_integral(long __val) { return __val; }
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR unsigned long __convert_to_integral(unsigned long __val) {
+  return __val;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR long long __convert_to_integral(long long __val) { return __val; }
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR unsigned long long __convert_to_integral(unsigned long long __val) {
+  return __val;
+}
+
+template <typename _Fp, __enable_if_t<is_floating_point<_Fp>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR long long __convert_to_integral(_Fp __val) {
+  return __val;
+}
+
+#ifndef _LIBCPP_HAS_NO_INT128
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __int128_t __convert_to_integral(__int128_t __val) { return __val; }
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __uint128_t __convert_to_integral(__uint128_t __val) { return __val; }
+#endif
+
+template <class _Tp, bool = is_enum<_Tp>::value>
+struct __sfinae_underlying_type {
+  typedef typename underlying_type<_Tp>::type type;
+  typedef decltype(((type)1) + 0) __promoted_type;
+};
+
+template <class _Tp>
+struct __sfinae_underlying_type<_Tp, false> {};
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename __sfinae_underlying_type<_Tp>::__promoted_type
+__convert_to_integral(_Tp __val) {
+  return __val;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_CONVERT_TO_INTEGRAL_H
diff --git a/libcxx/include/__cxx03/__utility/declval.h b/libcxx/include/__cxx03/__utility/declval.h
new file mode 100644
index 000000000000..d0856b8afa4d
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/declval.h
@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_DECLVAL_H
+#define _LIBCPP___UTILITY_DECLVAL_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Suppress deprecation notice for volatile-qualified return type resulting
+// from volatile-qualified types _Tp.
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Tp>
+_Tp&& __declval(int);
+template <class _Tp>
+_Tp __declval(long);
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI decltype(std::__declval<_Tp>(0)) declval() _NOEXCEPT {
+  static_assert(!__is_same(_Tp, _Tp),
+                "std::declval can only be used in an unevaluated context. "
+                "It's likely that your current usage is trying to extract a value from the function.");
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_DECLVAL_H
diff --git a/libcxx/include/__cxx03/__utility/empty.h b/libcxx/include/__cxx03/__utility/empty.h
new file mode 100644
index 000000000000..8cca197145c7
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/empty.h
@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_EMPTY_H
+#define _LIBCPP___UTILITY_EMPTY_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct __empty {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_EMPTY_H
diff --git a/libcxx/include/__cxx03/__utility/exception_guard.h b/libcxx/include/__cxx03/__utility/exception_guard.h
new file mode 100644
index 000000000000..9f732ca265c8
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/exception_guard.h
@@ -0,0 +1,144 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_TRANSACTION_H
+#define _LIBCPP___UTILITY_TRANSACTION_H
+
+#include <__assert>
+#include <__config>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__utility/exchange.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// __exception_guard is a helper class for writing code with the strong exception guarantee.
+//
+// When writing code that can throw an exception, one can store rollback instructions in an
+// exception guard so that if an exception is thrown at any point during the lifetime of the
+// exception guard, it will be rolled back automatically. When the exception guard is done, one
+// must mark it as being complete so it isn't rolled back when the exception guard is destroyed.
+//
+// Exception guards are not default constructible, they can't be copied or assigned to, but
+// they can be moved around for convenience.
+//
+// __exception_guard is a no-op in -fno-exceptions mode to produce better code-gen. This means
+// that we don't provide the strong exception guarantees. However, Clang doesn't generate cleanup
+// code with exceptions disabled, so even if we wanted to provide the strong exception guarantees
+// we couldn't. This is also only relevant for constructs with a stack of
+// -fexceptions > -fno-exceptions > -fexceptions code, since the exception can't be caught where
+// exceptions are disabled. While -fexceptions > -fno-exceptions is quite common
+// (e.g. libc++.dylib > -fno-exceptions), having another layer with exceptions enabled seems a lot
+// less common, especially one that tries to catch an exception through -fno-exceptions code.
+//
+// __exception_guard can help greatly simplify code that would normally be cluttered by
+// `#if _LIBCPP_HAS_NO_EXCEPTIONS`. For example:
+//
+//    template <class Iterator, class Size, class OutputIterator>
+//    Iterator uninitialized_copy_n(Iterator iter, Size n, OutputIterator out) {
+//        typedef typename iterator_traits<Iterator>::value_type value_type;
+//        __exception_guard guard([start=out, &out] {
+//            std::destroy(start, out);
+//        });
+//
+//        for (; n > 0; ++iter, ++out, --n) {
+//            ::new ((void*)std::addressof(*out)) value_type(*iter);
+//        }
+//        guard.__complete();
+//        return out;
+//    }
+//
+
+template <class _Rollback>
+struct __exception_guard_exceptions {
+  __exception_guard_exceptions() = delete;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit __exception_guard_exceptions(_Rollback __rollback)
+      : __rollback_(std::move(__rollback)), __completed_(false) {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  __exception_guard_exceptions(__exception_guard_exceptions&& __other)
+      _NOEXCEPT_(is_nothrow_move_constructible<_Rollback>::value)
+      : __rollback_(std::move(__other.__rollback_)), __completed_(__other.__completed_) {
+    __other.__completed_ = true;
+  }
+
+  __exception_guard_exceptions(__exception_guard_exceptions const&)            = delete;
+  __exception_guard_exceptions& operator=(__exception_guard_exceptions const&) = delete;
+  __exception_guard_exceptions& operator=(__exception_guard_exceptions&&)      = delete;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __complete() _NOEXCEPT { __completed_ = true; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__exception_guard_exceptions() {
+    if (!__completed_)
+      __rollback_();
+  }
+
+private:
+  _Rollback __rollback_;
+  bool __completed_;
+};
+
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(__exception_guard_exceptions);
+
+template <class _Rollback>
+struct __exception_guard_noexceptions {
+  __exception_guard_noexceptions() = delete;
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NODEBUG explicit __exception_guard_noexceptions(_Rollback) {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NODEBUG
+  __exception_guard_noexceptions(__exception_guard_noexceptions&& __other)
+      _NOEXCEPT_(is_nothrow_move_constructible<_Rollback>::value)
+      : __completed_(__other.__completed_) {
+    __other.__completed_ = true;
+  }
+
+  __exception_guard_noexceptions(__exception_guard_noexceptions const&)            = delete;
+  __exception_guard_noexceptions& operator=(__exception_guard_noexceptions const&) = delete;
+  __exception_guard_noexceptions& operator=(__exception_guard_noexceptions&&)      = delete;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NODEBUG void __complete() _NOEXCEPT {
+    __completed_ = true;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NODEBUG ~__exception_guard_noexceptions() {
+    _LIBCPP_ASSERT_INTERNAL(__completed_, "__exception_guard not completed with exceptions disabled");
+  }
+
+private:
+  bool __completed_ = false;
+};
+
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(__exception_guard_noexceptions);
+
+#ifdef _LIBCPP_HAS_NO_EXCEPTIONS
+template <class _Rollback>
+using __exception_guard = __exception_guard_noexceptions<_Rollback>;
+#else
+template <class _Rollback>
+using __exception_guard = __exception_guard_exceptions<_Rollback>;
+#endif
+
+template <class _Rollback>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __exception_guard<_Rollback> __make_exception_guard(_Rollback __rollback) {
+  return __exception_guard<_Rollback>(std::move(__rollback));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___UTILITY_TRANSACTION_H
diff --git a/libcxx/include/__cxx03/__utility/exchange.h b/libcxx/include/__cxx03/__utility/exchange.h
new file mode 100644
index 000000000000..957e9d0acaa6
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/exchange.h
@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_EXCHANGE_H
+#define _LIBCPP___UTILITY_EXCHANGE_H
+
+#include <__config>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 14
+template <class _T1, class _T2 = _T1>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _T1 exchange(_T1& __obj, _T2&& __new_value) noexcept(
+    is_nothrow_move_constructible<_T1>::value && is_nothrow_assignable<_T1&, _T2>::value) {
+  _T1 __old_value = std::move(__obj);
+  __obj           = std::forward<_T2>(__new_value);
+  return __old_value;
+}
+#endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___UTILITY_EXCHANGE_H
diff --git a/libcxx/include/__cxx03/__utility/forward.h b/libcxx/include/__cxx03/__utility/forward.h
new file mode 100644
index 000000000000..d5275dcbd0ed
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/forward.h
@@ -0,0 +1,38 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_FORWARD_H
+#define _LIBCPP___UTILITY_FORWARD_H
+
+#include <__config>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/remove_reference.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp&&
+forward(_LIBCPP_LIFETIMEBOUND __libcpp_remove_reference_t<_Tp>& __t) _NOEXCEPT {
+  return static_cast<_Tp&&>(__t);
+}
+
+template <class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp&&
+forward(_LIBCPP_LIFETIMEBOUND __libcpp_remove_reference_t<_Tp>&& __t) _NOEXCEPT {
+  static_assert(!is_lvalue_reference<_Tp>::value, "cannot forward an rvalue as an lvalue");
+  return static_cast<_Tp&&>(__t);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_FORWARD_H
diff --git a/libcxx/include/__cxx03/__utility/forward_like.h b/libcxx/include/__cxx03/__utility/forward_like.h
new file mode 100644
index 000000000000..0206ce23a566
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/forward_like.h
@@ -0,0 +1,46 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_FORWARD_LIKE_H
+#define _LIBCPP___UTILITY_FORWARD_LIKE_H
+
+#include <__config>
+#include <__type_traits/conditional.h>
+#include <__type_traits/is_const.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/remove_reference.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+template <class _Ap, class _Bp>
+using _CopyConst = _If<is_const_v<_Ap>, const _Bp, _Bp>;
+
+template <class _Ap, class _Bp>
+using _OverrideRef = _If<is_rvalue_reference_v<_Ap>, remove_reference_t<_Bp>&&, _Bp&>;
+
+template <class _Ap, class _Bp>
+using _ForwardLike = _OverrideRef<_Ap&&, _CopyConst<remove_reference_t<_Ap>, remove_reference_t<_Bp>>>;
+
+template <class _Tp, class _Up>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto
+forward_like(_LIBCPP_LIFETIMEBOUND _Up&& __ux) noexcept -> _ForwardLike<_Tp, _Up> {
+  return static_cast<_ForwardLike<_Tp, _Up>>(__ux);
+}
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_FORWARD_LIKE_H
diff --git a/libcxx/include/__cxx03/__utility/in_place.h b/libcxx/include/__cxx03/__utility/in_place.h
new file mode 100644
index 000000000000..fa7a2f4bfd4a
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/in_place.h
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_IN_PLACE_H
+#define _LIBCPP___UTILITY_IN_PLACE_H
+
+#include <__config>
+#include <__type_traits/remove_cvref.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+struct _LIBCPP_EXPORTED_FROM_ABI in_place_t {
+  explicit in_place_t() = default;
+};
+inline constexpr in_place_t in_place{};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS in_place_type_t {
+  _LIBCPP_HIDE_FROM_ABI explicit in_place_type_t() = default;
+};
+template <class _Tp>
+inline constexpr in_place_type_t<_Tp> in_place_type{};
+
+template <size_t _Idx>
+struct _LIBCPP_TEMPLATE_VIS in_place_index_t {
+  _LIBCPP_HIDE_FROM_ABI explicit in_place_index_t() = default;
+};
+template <size_t _Idx>
+inline constexpr in_place_index_t<_Idx> in_place_index{};
+
+template <class _Tp>
+struct __is_inplace_type_imp : false_type {};
+template <class _Tp>
+struct __is_inplace_type_imp<in_place_type_t<_Tp>> : true_type {};
+
+template <class _Tp>
+using __is_inplace_type = __is_inplace_type_imp<__remove_cvref_t<_Tp>>;
+
+template <class _Tp>
+struct __is_inplace_index_imp : false_type {};
+template <size_t _Idx>
+struct __is_inplace_index_imp<in_place_index_t<_Idx>> : true_type {};
+
+template <class _Tp>
+using __is_inplace_index = __is_inplace_index_imp<__remove_cvref_t<_Tp>>;
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_IN_PLACE_H
diff --git a/libcxx/include/__cxx03/__utility/integer_sequence.h b/libcxx/include/__cxx03/__utility/integer_sequence.h
new file mode 100644
index 000000000000..ccce9433e7a8
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/integer_sequence.h
@@ -0,0 +1,90 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_INTEGER_SEQUENCE_H
+#define _LIBCPP___UTILITY_INTEGER_SEQUENCE_H
+
+#include <__config>
+#include <__type_traits/is_integral.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <size_t...>
+struct __tuple_indices;
+
+template <class _IdxType, _IdxType... _Values>
+struct __integer_sequence {
+  template <template <class _OIdxType, _OIdxType...> class _ToIndexSeq, class _ToIndexType>
+  using __convert = _ToIndexSeq<_ToIndexType, _Values...>;
+
+  template <size_t _Sp>
+  using __to_tuple_indices = __tuple_indices<(_Values + _Sp)...>;
+};
+
+#if __has_builtin(__make_integer_seq)
+template <size_t _Ep, size_t _Sp>
+using __make_indices_imp =
+    typename __make_integer_seq<__integer_sequence, size_t, _Ep - _Sp>::template __to_tuple_indices<_Sp>;
+#elif __has_builtin(__integer_pack)
+template <size_t _Ep, size_t _Sp>
+using __make_indices_imp =
+    typename __integer_sequence<size_t, __integer_pack(_Ep - _Sp)...>::template __to_tuple_indices<_Sp>;
+#else
+#  error "No known way to get an integer pack from the compiler"
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+
+template <class _Tp, _Tp... _Ip>
+struct _LIBCPP_TEMPLATE_VIS integer_sequence {
+  typedef _Tp value_type;
+  static_assert(is_integral<_Tp>::value, "std::integer_sequence can only be instantiated with an integral type");
+  static _LIBCPP_HIDE_FROM_ABI constexpr size_t size() noexcept { return sizeof...(_Ip); }
+};
+
+template <size_t... _Ip>
+using index_sequence = integer_sequence<size_t, _Ip...>;
+
+#  if __has_builtin(__make_integer_seq)
+
+template <class _Tp, _Tp _Ep>
+using make_integer_sequence _LIBCPP_NODEBUG = __make_integer_seq<integer_sequence, _Tp, _Ep>;
+
+#  elif __has_builtin(__integer_pack)
+
+template <class _Tp, _Tp _SequenceSize>
+using make_integer_sequence _LIBCPP_NODEBUG = integer_sequence<_Tp, __integer_pack(_SequenceSize)...>;
+
+#  else
+#    error "No known way to get an integer pack from the compiler"
+#  endif
+
+template <size_t _Np>
+using make_index_sequence = make_integer_sequence<size_t, _Np>;
+
+template <class... _Tp>
+using index_sequence_for = make_index_sequence<sizeof...(_Tp)>;
+
+#  if _LIBCPP_STD_VER >= 20
+// Executes __func for every element in an index_sequence.
+template <size_t... _Index, class _Function>
+_LIBCPP_HIDE_FROM_ABI constexpr void __for_each_index_sequence(index_sequence<_Index...>, _Function __func) {
+  (__func.template operator()<_Index>(), ...);
+}
+#  endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_INTEGER_SEQUENCE_H
diff --git a/libcxx/include/__cxx03/__utility/is_pointer_in_range.h b/libcxx/include/__cxx03/__utility/is_pointer_in_range.h
new file mode 100644
index 000000000000..4130b4ac7070
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/is_pointer_in_range.h
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_IS_POINTER_IN_RANGE_H
+#define _LIBCPP___UTILITY_IS_POINTER_IN_RANGE_H
+
+#include <__algorithm/comp.h>
+#include <__assert>
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+#include <__utility/is_valid_range.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Up, class = void>
+struct __is_less_than_comparable : false_type {};
+
+template <class _Tp, class _Up>
+struct __is_less_than_comparable<_Tp, _Up, __void_t<decltype(std::declval<_Tp>() < std::declval<_Up>())> > : true_type {
+};
+
+template <class _Tp, class _Up, __enable_if_t<__is_less_than_comparable<const _Tp*, const _Up*>::value, int> = 0>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_SANITIZE("address") bool
+__is_pointer_in_range(const _Tp* __begin, const _Tp* __end, const _Up* __ptr) {
+  _LIBCPP_ASSERT_VALID_INPUT_RANGE(std::__is_valid_range(__begin, __end), "[__begin, __end) is not a valid range");
+
+  if (__libcpp_is_constant_evaluated()) {
+    // If this is not a constant during constant evaluation we know that __ptr is not part of the allocation where
+    // [__begin, __end) is.
+    if (!__builtin_constant_p(__begin <= __ptr && __ptr < __end))
+      return false;
+  }
+
+  return !__less<>()(__ptr, __begin) && __less<>()(__ptr, __end);
+}
+
+template <class _Tp, class _Up, __enable_if_t<!__is_less_than_comparable<const _Tp*, const _Up*>::value, int> = 0>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_SANITIZE("address") bool
+__is_pointer_in_range(const _Tp* __begin, const _Tp* __end, const _Up* __ptr) {
+  if (__libcpp_is_constant_evaluated())
+    return false;
+
+  return reinterpret_cast<const char*>(__begin) <= reinterpret_cast<const char*>(__ptr) &&
+         reinterpret_cast<const char*>(__ptr) < reinterpret_cast<const char*>(__end);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_IS_POINTER_IN_RANGE_H
diff --git a/libcxx/include/__cxx03/__utility/is_valid_range.h b/libcxx/include/__cxx03/__utility/is_valid_range.h
new file mode 100644
index 000000000000..7286662dbf30
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/is_valid_range.h
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_IS_VALID_RANGE_H
+#define _LIBCPP___UTILITY_IS_VALID_RANGE_H
+
+#include <__algorithm/comp.h>
+#include <__config>
+#include <__type_traits/is_constant_evaluated.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_SANITIZE("address") bool
+__is_valid_range(const _Tp* __first, const _Tp* __last) {
+  if (__libcpp_is_constant_evaluated()) {
+    // If this is not a constant during constant evaluation, that is because __first and __last are not
+    // part of the same allocation. If they are part of the same allocation, we must still make sure they
+    // are ordered properly.
+    return __builtin_constant_p(__first <= __last) && __first <= __last;
+  }
+
+  return !__less<>()(__last, __first);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_IS_VALID_RANGE_H
diff --git a/libcxx/include/__cxx03/__utility/move.h b/libcxx/include/__cxx03/__utility/move.h
new file mode 100644
index 000000000000..b6a42db0545e
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/move.h
@@ -0,0 +1,49 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_MOVE_H
+#define _LIBCPP___UTILITY_MOVE_H
+
+#include <__config>
+#include <__type_traits/conditional.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/remove_reference.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __libcpp_remove_reference_t<_Tp>&&
+move(_LIBCPP_LIFETIMEBOUND _Tp&& __t) _NOEXCEPT {
+  typedef _LIBCPP_NODEBUG __libcpp_remove_reference_t<_Tp> _Up;
+  return static_cast<_Up&&>(__t);
+}
+
+template <class _Tp>
+using __move_if_noexcept_result_t =
+    __conditional_t<!is_nothrow_move_constructible<_Tp>::value && is_copy_constructible<_Tp>::value, const _Tp&, _Tp&&>;
+
+template <class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __move_if_noexcept_result_t<_Tp>
+move_if_noexcept(_LIBCPP_LIFETIMEBOUND _Tp& __x) _NOEXCEPT {
+  return std::move(__x);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___UTILITY_MOVE_H
diff --git a/libcxx/include/__cxx03/__utility/no_destroy.h b/libcxx/include/__cxx03/__utility/no_destroy.h
new file mode 100644
index 000000000000..8edd194577d7
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/no_destroy.h
@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_NO_DESTROY_H
+#define _LIBCPP___UTILITY_NO_DESTROY_H
+
+#include <__config>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__utility/forward.h>
+#include <new>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct __uninitialized_tag {};
+
+// This class stores an object of type _Tp but never destroys it.
+//
+// This is akin to using __attribute__((no_destroy)), except that it is possible
+// to control the lifetime of the object with more flexibility by deciding e.g.
+// whether to initialize the object at construction or to defer to a later
+// initialization using __emplace.
+template <class _Tp>
+struct __no_destroy {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __no_destroy(__uninitialized_tag) : __obj_() {}
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI explicit __no_destroy(_Args&&... __args) {
+    ::new ((void*)__obj_) _Tp(std::forward<_Args>(__args)...);
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _Tp& __emplace(_Args&&... __args) {
+    return *(::new ((void*)__obj_) _Tp(std::forward<_Args>(__args)...));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp& __get() { return *reinterpret_cast<_Tp*>(__obj_); }
+  _LIBCPP_HIDE_FROM_ABI _Tp const& __get() const { return *reinterpret_cast<const _Tp*>(__obj_); }
+
+private:
+  _ALIGNAS_TYPE(_Tp) char __obj_[sizeof(_Tp)];
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_NO_DESTROY_H
diff --git a/libcxx/include/__cxx03/__utility/pair.h b/libcxx/include/__cxx03/__utility/pair.h
new file mode 100644
index 000000000000..c0002b7abb3c
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/pair.h
@@ -0,0 +1,718 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_PAIR_H
+#define _LIBCPP___UTILITY_PAIR_H
+
+#include <__compare/common_comparison_category.h>
+#include <__compare/synth_three_way.h>
+#include <__concepts/different_from.h>
+#include <__config>
+#include <__fwd/array.h>
+#include <__fwd/pair.h>
+#include <__fwd/tuple.h>
+#include <__tuple/sfinae_helpers.h>
+#include <__tuple/tuple_element.h>
+#include <__tuple/tuple_indices.h>
+#include <__tuple/tuple_like_no_subrange.h>
+#include <__tuple/tuple_size.h>
+#include <__type_traits/common_reference.h>
+#include <__type_traits/common_type.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_implicitly_default_constructible.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_swappable.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__type_traits/nat.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/unwrap_ref.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/piecewise_construct.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class, class>
+struct __non_trivially_copyable_base {
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI __non_trivially_copyable_base() _NOEXCEPT {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+  __non_trivially_copyable_base(__non_trivially_copyable_base const&) _NOEXCEPT {}
+};
+
+template <class _T1, class _T2>
+struct _LIBCPP_TEMPLATE_VIS pair
+#if defined(_LIBCPP_DEPRECATED_ABI_DISABLE_PAIR_TRIVIAL_COPY_CTOR)
+    : private __non_trivially_copyable_base<_T1, _T2>
+#endif
+{
+  using first_type  = _T1;
+  using second_type = _T2;
+
+  _T1 first;
+  _T2 second;
+
+  using __trivially_relocatable =
+      __conditional_t<__libcpp_is_trivially_relocatable<_T1>::value && __libcpp_is_trivially_relocatable<_T2>::value,
+                      pair,
+                      void>;
+
+  _LIBCPP_HIDE_FROM_ABI pair(pair const&) = default;
+  _LIBCPP_HIDE_FROM_ABI pair(pair&&)      = default;
+
+  // When we are requested for pair to be trivially copyable by the ABI macro, we use defaulted members
+  // if it is both legal to do it (i.e. no references) and we have a way to actually implement it, which requires
+  // the __enable_if__ attribute before C++20.
+#ifdef _LIBCPP_ABI_TRIVIALLY_COPYABLE_PAIR
+  // FIXME: This should really just be a static constexpr variable. It's in a struct to avoid gdb printing the value
+  // when printing a pair
+  struct __has_defaulted_members {
+    static const bool value = !is_reference<first_type>::value && !is_reference<second_type>::value;
+  };
+#  if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI constexpr pair& operator=(const pair&)
+    requires __has_defaulted_members::value
+  = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr pair& operator=(pair&&)
+    requires __has_defaulted_members::value
+  = default;
+#  elif __has_attribute(__enable_if__)
+  _LIBCPP_HIDE_FROM_ABI pair& operator=(const pair&)
+      __attribute__((__enable_if__(__has_defaulted_members::value, ""))) = default;
+
+  _LIBCPP_HIDE_FROM_ABI pair& operator=(pair&&)
+      __attribute__((__enable_if__(__has_defaulted_members::value, ""))) = default;
+#  else
+#    error "_LIBCPP_ABI_TRIVIALLY_COPYABLE_PAIR isn't supported with this compiler"
+#  endif
+#else
+  struct __has_defaulted_members {
+    static const bool value = false;
+  };
+#endif // defined(_LIBCPP_ABI_TRIVIALLY_COPYABLE_PAIR) && __has_attribute(__enable_if__)
+
+#ifdef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI pair() : first(), second() {}
+
+  _LIBCPP_HIDE_FROM_ABI pair(_T1 const& __t1, _T2 const& __t2) : first(__t1), second(__t2) {}
+
+  template <class _U1, class _U2>
+  _LIBCPP_HIDE_FROM_ABI pair(const pair<_U1, _U2>& __p) : first(__p.first), second(__p.second) {}
+
+  _LIBCPP_HIDE_FROM_ABI pair& operator=(pair const& __p) {
+    first  = __p.first;
+    second = __p.second;
+    return *this;
+  }
+
+  // Extension: This is provided in C++03 because it allows properly handling the
+  //            assignment to a pair containing references, which would be a hard
+  //            error otherwise.
+  template <
+      class _U1,
+      class _U2,
+      __enable_if_t<is_assignable<first_type&, _U1 const&>::value && is_assignable<second_type&, _U2 const&>::value,
+                    int> = 0>
+  _LIBCPP_HIDE_FROM_ABI pair& operator=(pair<_U1, _U2> const& __p) {
+    first  = __p.first;
+    second = __p.second;
+    return *this;
+  }
+#else
+  struct _CheckArgs {
+    template <int&...>
+    static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_implicit_default() {
+      return __is_implicitly_default_constructible<_T1>::value && __is_implicitly_default_constructible<_T2>::value;
+    }
+
+    template <int&...>
+    static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_default() {
+      return is_default_constructible<_T1>::value && is_default_constructible<_T2>::value;
+    }
+
+    template <class _U1, class _U2>
+    static _LIBCPP_HIDE_FROM_ABI constexpr bool __is_pair_constructible() {
+      return is_constructible<first_type, _U1>::value && is_constructible<second_type, _U2>::value;
+    }
+
+    template <class _U1, class _U2>
+    static _LIBCPP_HIDE_FROM_ABI constexpr bool __is_implicit() {
+      return is_convertible<_U1, first_type>::value && is_convertible<_U2, second_type>::value;
+    }
+  };
+
+  template <bool _MaybeEnable>
+  using _CheckArgsDep _LIBCPP_NODEBUG =
+      typename conditional< _MaybeEnable, _CheckArgs, __check_tuple_constructor_fail>::type;
+
+  template <bool _Dummy = true, __enable_if_t<_CheckArgsDep<_Dummy>::__enable_default(), int> = 0>
+  explicit(!_CheckArgsDep<_Dummy>::__enable_implicit_default()) _LIBCPP_HIDE_FROM_ABI constexpr pair() noexcept(
+      is_nothrow_default_constructible<first_type>::value && is_nothrow_default_constructible<second_type>::value)
+      : first(), second() {}
+
+  template <bool _Dummy = true,
+            __enable_if_t<_CheckArgsDep<_Dummy>::template __is_pair_constructible<_T1 const&, _T2 const&>(), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(!_CheckArgsDep<_Dummy>::template __is_implicit<_T1 const&, _T2 const&>())
+      pair(_T1 const& __t1, _T2 const& __t2) noexcept(is_nothrow_copy_constructible<first_type>::value &&
+                                                      is_nothrow_copy_constructible<second_type>::value)
+      : first(__t1), second(__t2) {}
+
+  template <
+#  if _LIBCPP_STD_VER >= 23 // http://wg21.link/P1951
+      class _U1 = _T1,
+      class _U2 = _T2,
+#  else
+      class _U1,
+      class _U2,
+#  endif
+      __enable_if_t<_CheckArgs::template __is_pair_constructible<_U1, _U2>(), int> = 0 >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(!_CheckArgs::template __is_implicit<_U1, _U2>())
+      pair(_U1&& __u1, _U2&& __u2) noexcept(is_nothrow_constructible<first_type, _U1>::value &&
+                                            is_nothrow_constructible<second_type, _U2>::value)
+      : first(std::forward<_U1>(__u1)), second(std::forward<_U2>(__u2)) {
+  }
+
+#  if _LIBCPP_STD_VER >= 23
+  template <class _U1, class _U2, __enable_if_t<_CheckArgs::template __is_pair_constructible<_U1&, _U2&>(), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_CheckArgs::template __is_implicit<_U1&, _U2&>())
+      pair(pair<_U1, _U2>& __p) noexcept((is_nothrow_constructible<first_type, _U1&>::value &&
+                                          is_nothrow_constructible<second_type, _U2&>::value))
+      : first(__p.first), second(__p.second) {}
+#  endif
+
+  template <class _U1,
+            class _U2,
+            __enable_if_t<_CheckArgs::template __is_pair_constructible<_U1 const&, _U2 const&>(), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(!_CheckArgs::template __is_implicit<_U1 const&, _U2 const&>())
+      pair(pair<_U1, _U2> const& __p) noexcept(is_nothrow_constructible<first_type, _U1 const&>::value &&
+                                               is_nothrow_constructible<second_type, _U2 const&>::value)
+      : first(__p.first), second(__p.second) {}
+
+  template <class _U1, class _U2, __enable_if_t<_CheckArgs::template __is_pair_constructible<_U1, _U2>(), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(!_CheckArgs::template __is_implicit<_U1, _U2>())
+      pair(pair<_U1, _U2>&& __p) noexcept(is_nothrow_constructible<first_type, _U1&&>::value &&
+                                          is_nothrow_constructible<second_type, _U2&&>::value)
+      : first(std::forward<_U1>(__p.first)), second(std::forward<_U2>(__p.second)) {}
+
+#  if _LIBCPP_STD_VER >= 23
+  template <class _U1,
+            class _U2,
+            __enable_if_t<_CheckArgs::template __is_pair_constructible<const _U1&&, const _U2&&>(), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_CheckArgs::template __is_implicit<const _U1&&, const _U2&&>())
+      pair(const pair<_U1, _U2>&& __p) noexcept(is_nothrow_constructible<first_type, const _U1&&>::value &&
+                                                is_nothrow_constructible<second_type, const _U2&&>::value)
+      : first(std::move(__p.first)), second(std::move(__p.second)) {}
+#  endif
+
+#  if _LIBCPP_STD_VER >= 23
+  // TODO: Remove this workaround in LLVM 20. The bug got fixed in Clang 18.
+  // This is a workaround for http://llvm.org/PR60710. We should be able to remove it once Clang is fixed.
+  template <class _PairLike>
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __pair_like_explicit_wknd() {
+    if constexpr (__pair_like_no_subrange<_PairLike>) {
+      return !is_convertible_v<decltype(std::get<0>(std::declval<_PairLike&&>())), first_type> ||
+             !is_convertible_v<decltype(std::get<1>(std::declval<_PairLike&&>())), second_type>;
+    }
+    return false;
+  }
+
+  template <__pair_like_no_subrange _PairLike>
+    requires(is_constructible_v<first_type, decltype(std::get<0>(std::declval<_PairLike &&>()))> &&
+             is_constructible_v<second_type, decltype(std::get<1>(std::declval<_PairLike &&>()))>)
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(__pair_like_explicit_wknd<_PairLike>()) pair(_PairLike&& __p)
+      : first(std::get<0>(std::forward<_PairLike>(__p))), second(std::get<1>(std::forward<_PairLike>(__p))) {}
+#  endif
+
+  template <class... _Args1, class... _Args2>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  pair(piecewise_construct_t __pc, tuple<_Args1...> __first_args, tuple<_Args2...> __second_args) noexcept(
+      is_nothrow_constructible<first_type, _Args1...>::value && is_nothrow_constructible<second_type, _Args2...>::value)
+      : pair(__pc,
+             __first_args,
+             __second_args,
+             typename __make_tuple_indices<sizeof...(_Args1)>::type(),
+             typename __make_tuple_indices<sizeof...(_Args2) >::type()) {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair&
+  operator=(__conditional_t<!__has_defaulted_members::value && is_copy_assignable<first_type>::value &&
+                                is_copy_assignable<second_type>::value,
+                            pair,
+                            __nat> const& __p) noexcept(is_nothrow_copy_assignable<first_type>::value &&
+                                                        is_nothrow_copy_assignable<second_type>::value) {
+    first  = __p.first;
+    second = __p.second;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair&
+  operator=(__conditional_t<!__has_defaulted_members::value && is_move_assignable<first_type>::value &&
+                                is_move_assignable<second_type>::value,
+                            pair,
+                            __nat>&& __p) noexcept(is_nothrow_move_assignable<first_type>::value &&
+                                                   is_nothrow_move_assignable<second_type>::value) {
+    first  = std::forward<first_type>(__p.first);
+    second = std::forward<second_type>(__p.second);
+    return *this;
+  }
+
+  template <
+      class _U1,
+      class _U2,
+      __enable_if_t<is_assignable<first_type&, _U1 const&>::value && is_assignable<second_type&, _U2 const&>::value,
+                    int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair& operator=(pair<_U1, _U2> const& __p) {
+    first  = __p.first;
+    second = __p.second;
+    return *this;
+  }
+
+  template <class _U1,
+            class _U2,
+            __enable_if_t<is_assignable<first_type&, _U1>::value && is_assignable<second_type&, _U2>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair& operator=(pair<_U1, _U2>&& __p) {
+    first  = std::forward<_U1>(__p.first);
+    second = std::forward<_U2>(__p.second);
+    return *this;
+  }
+
+#  if _LIBCPP_STD_VER >= 23
+  template <class = void>
+  _LIBCPP_HIDE_FROM_ABI constexpr const pair& operator=(pair const& __p) const
+      noexcept(is_nothrow_copy_assignable_v<const first_type> && is_nothrow_copy_assignable_v<const second_type>)
+    requires(is_copy_assignable_v<const first_type> && is_copy_assignable_v<const second_type>)
+  {
+    first  = __p.first;
+    second = __p.second;
+    return *this;
+  }
+
+  template <class = void>
+  _LIBCPP_HIDE_FROM_ABI constexpr const pair& operator=(pair&& __p) const
+      noexcept(is_nothrow_assignable_v<const first_type&, first_type> &&
+               is_nothrow_assignable_v<const second_type&, second_type>)
+    requires(is_assignable_v<const first_type&, first_type> && is_assignable_v<const second_type&, second_type>)
+  {
+    first  = std::forward<first_type>(__p.first);
+    second = std::forward<second_type>(__p.second);
+    return *this;
+  }
+
+  template <class _U1, class _U2>
+  _LIBCPP_HIDE_FROM_ABI constexpr const pair& operator=(const pair<_U1, _U2>& __p) const
+    requires(is_assignable_v<const first_type&, const _U1&> && is_assignable_v<const second_type&, const _U2&>)
+  {
+    first  = __p.first;
+    second = __p.second;
+    return *this;
+  }
+
+  template <class _U1, class _U2>
+  _LIBCPP_HIDE_FROM_ABI constexpr const pair& operator=(pair<_U1, _U2>&& __p) const
+    requires(is_assignable_v<const first_type&, _U1> && is_assignable_v<const second_type&, _U2>)
+  {
+    first  = std::forward<_U1>(__p.first);
+    second = std::forward<_U2>(__p.second);
+    return *this;
+  }
+
+  template <__pair_like_no_subrange _PairLike>
+    requires(__different_from<_PairLike, pair> &&
+             is_assignable_v<first_type&, decltype(std::get<0>(std::declval<_PairLike>()))> &&
+             is_assignable_v<second_type&, decltype(std::get<1>(std::declval<_PairLike>()))>)
+  _LIBCPP_HIDE_FROM_ABI constexpr pair& operator=(_PairLike&& __p) {
+    first  = std::get<0>(std::forward<_PairLike>(__p));
+    second = std::get<1>(std::forward<_PairLike>(__p));
+    return *this;
+  }
+
+  template <__pair_like_no_subrange _PairLike>
+    requires(__different_from<_PairLike, pair> &&
+             is_assignable_v<first_type const&, decltype(std::get<0>(std::declval<_PairLike>()))> &&
+             is_assignable_v<second_type const&, decltype(std::get<1>(std::declval<_PairLike>()))>)
+  _LIBCPP_HIDE_FROM_ABI constexpr pair const& operator=(_PairLike&& __p) const {
+    first  = std::get<0>(std::forward<_PairLike>(__p));
+    second = std::get<1>(std::forward<_PairLike>(__p));
+    return *this;
+  }
+#  endif // _LIBCPP_STD_VER >= 23
+
+  // Prior to C++23, we provide an approximation of constructors and assignment operators from
+  // pair-like types. This was historically provided as an extension.
+#  if _LIBCPP_STD_VER < 23
+  // from std::tuple
+  template <class _U1,
+            class _U2,
+            __enable_if_t<is_convertible<_U1 const&, _T1>::value && is_convertible<_U2 const&, _T2>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair(tuple<_U1, _U2> const& __p)
+      : first(std::get<0>(__p)), second(std::get<1>(__p)) {}
+
+  template < class _U1,
+             class _U2,
+             __enable_if_t<is_constructible<_T1, _U1 const&>::value && is_constructible<_T2, _U2 const&>::value &&
+                               !(is_convertible<_U1 const&, _T1>::value && is_convertible<_U2 const&, _T2>::value),
+                           int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit pair(tuple<_U1, _U2> const& __p)
+      : first(std::get<0>(__p)), second(std::get<1>(__p)) {}
+
+  template <class _U1,
+            class _U2,
+            __enable_if_t<is_convertible<_U1, _T1>::value && is_convertible<_U2, _T2>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair(tuple<_U1, _U2>&& __p)
+      : first(std::get<0>(std::move(__p))), second(std::get<1>(std::move(__p))) {}
+
+  template <class _U1,
+            class _U2,
+            __enable_if_t<is_constructible<_T1, _U1>::value && is_constructible<_T2, _U2>::value &&
+                          !(is_convertible<_U1, _T1>::value && is_convertible<_U2, _T2>::value) > = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit pair(tuple<_U1, _U2>&& __p)
+      : first(std::get<0>(std::move(__p))), second(std::get<1>(std::move(__p))) {}
+
+  template <class _U1,
+            class _U2,
+            __enable_if_t<is_assignable<_T1&, _U1 const&>::value && is_assignable<_T2&, _U2 const&>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair& operator=(tuple<_U1, _U2> const& __p) {
+    first  = std::get<0>(__p);
+    second = std::get<1>(__p);
+    return *this;
+  }
+
+  template <class _U1,
+            class _U2,
+            __enable_if_t<is_assignable<_T1&, _U1&&>::value && is_assignable<_T2&, _U2&&>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair& operator=(tuple<_U1, _U2>&& __p) {
+    first  = std::get<0>(std::move(__p));
+    second = std::get<1>(std::move(__p));
+    return *this;
+  }
+
+  // from std::array
+  template <class _Up,
+            __enable_if_t<is_convertible<_Up const&, _T1>::value && is_convertible<_Up const&, _T2>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair(array<_Up, 2> const& __p) : first(__p[0]), second(__p[1]) {}
+
+  template <class _Up,
+            __enable_if_t<is_constructible<_T1, _Up const&>::value && is_constructible<_T2, _Up const&>::value &&
+                              !(is_convertible<_Up const&, _T1>::value && is_convertible<_Up const&, _T2>::value),
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit pair(array<_Up, 2> const& __p)
+      : first(__p[0]), second(__p[1]) {}
+
+  template <class _Up, __enable_if_t< is_convertible<_Up, _T1>::value && is_convertible<_Up, _T2>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair(array<_Up, 2>&& __p)
+      : first(std::move(__p)[0]), second(std::move(__p)[1]) {}
+
+  template <class _Up,
+            __enable_if_t<is_constructible<_T1, _Up>::value && is_constructible<_T2, _Up>::value &&
+                              !(is_convertible<_Up, _T1>::value && is_convertible<_Up, _T2>::value),
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit pair(array<_Up, 2>&& __p)
+      : first(std::move(__p)[0]), second(std::move(__p)[1]) {}
+
+  template <class _Up,
+            __enable_if_t<is_assignable<_T1&, _Up const&>::value && is_assignable<_T2&, _Up const&>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair& operator=(array<_Up, 2> const& __p) {
+    first  = std::get<0>(__p);
+    second = std::get<1>(__p);
+    return *this;
+  }
+
+  template <class _Up, __enable_if_t<is_assignable<_T1&, _Up>::value && is_assignable<_T2&, _Up>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair& operator=(array<_Up, 2>&& __p) {
+    first  = std::get<0>(std::move(__p));
+    second = std::get<1>(std::move(__p));
+    return *this;
+  }
+#  endif // _LIBCPP_STD_VER < 23
+#endif   // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(pair& __p)
+      _NOEXCEPT_(__is_nothrow_swappable_v<first_type>&& __is_nothrow_swappable_v<second_type>) {
+    using std::swap;
+    swap(first, __p.first);
+    swap(second, __p.second);
+  }
+
+#if _LIBCPP_STD_VER >= 23
+  _LIBCPP_HIDE_FROM_ABI constexpr void swap(const pair& __p) const
+      noexcept(__is_nothrow_swappable_v<const first_type> && __is_nothrow_swappable_v<const second_type>) {
+    using std::swap;
+    swap(first, __p.first);
+    swap(second, __p.second);
+  }
+#endif
+
+private:
+#ifndef _LIBCPP_CXX03_LANG
+  template <class... _Args1, class... _Args2, size_t... _I1, size_t... _I2>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  pair(piecewise_construct_t,
+       tuple<_Args1...>& __first_args,
+       tuple<_Args2...>& __second_args,
+       __tuple_indices<_I1...>,
+       __tuple_indices<_I2...>)
+      : first(std::forward<_Args1>(std::get<_I1>(__first_args))...),
+        second(std::forward<_Args2>(std::get<_I2>(__second_args))...) {}
+#endif
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _T1, class _T2>
+pair(_T1, _T2) -> pair<_T1, _T2>;
+#endif
+
+// [pairs.spec], specialized algorithms
+
+template <class _T1, class _T2, class _U1, class _U2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator==(const pair<_T1, _T2>& __x, const pair<_U1, _U2>& __y) {
+  return __x.first == __y.first && __x.second == __y.second;
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _T1, class _T2, class _U1, class _U2>
+_LIBCPP_HIDE_FROM_ABI constexpr common_comparison_category_t< __synth_three_way_result<_T1, _U1>,
+                                                              __synth_three_way_result<_T2, _U2> >
+operator<=>(const pair<_T1, _T2>& __x, const pair<_U1, _U2>& __y) {
+  if (auto __c = std::__synth_three_way(__x.first, __y.first); __c != 0) {
+    return __c;
+  }
+  return std::__synth_three_way(__x.second, __y.second);
+}
+
+#else // _LIBCPP_STD_VER >= 20
+
+template <class _T1, class _T2, class _U1, class _U2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator!=(const pair<_T1, _T2>& __x, const pair<_U1, _U2>& __y) {
+  return !(__x == __y);
+}
+
+template <class _T1, class _T2, class _U1, class _U2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator<(const pair<_T1, _T2>& __x, const pair<_U1, _U2>& __y) {
+  return __x.first < __y.first || (!(__y.first < __x.first) && __x.second < __y.second);
+}
+
+template <class _T1, class _T2, class _U1, class _U2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator>(const pair<_T1, _T2>& __x, const pair<_U1, _U2>& __y) {
+  return __y < __x;
+}
+
+template <class _T1, class _T2, class _U1, class _U2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator>=(const pair<_T1, _T2>& __x, const pair<_U1, _U2>& __y) {
+  return !(__x < __y);
+}
+
+template <class _T1, class _T2, class _U1, class _U2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator<=(const pair<_T1, _T2>& __x, const pair<_U1, _U2>& __y) {
+  return !(__y < __x);
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 23
+template <class _T1, class _T2, class _U1, class _U2, template <class> class _TQual, template <class> class _UQual>
+  requires requires {
+    typename pair<common_reference_t<_TQual<_T1>, _UQual<_U1>>, common_reference_t<_TQual<_T2>, _UQual<_U2>>>;
+  }
+struct basic_common_reference<pair<_T1, _T2>, pair<_U1, _U2>, _TQual, _UQual> {
+  using type = pair<common_reference_t<_TQual<_T1>, _UQual<_U1>>, common_reference_t<_TQual<_T2>, _UQual<_U2>>>;
+};
+
+template <class _T1, class _T2, class _U1, class _U2>
+  requires requires { typename pair<common_type_t<_T1, _U1>, common_type_t<_T2, _U2>>; }
+struct common_type<pair<_T1, _T2>, pair<_U1, _U2>> {
+  using type = pair<common_type_t<_T1, _U1>, common_type_t<_T2, _U2>>;
+};
+#endif // _LIBCPP_STD_VER >= 23
+
+template <class _T1, class _T2, __enable_if_t<__is_swappable_v<_T1> && __is_swappable_v<_T2>, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(pair<_T1, _T2>& __x, pair<_T1, _T2>& __y)
+    _NOEXCEPT_(__is_nothrow_swappable_v<_T1>&& __is_nothrow_swappable_v<_T2>) {
+  __x.swap(__y);
+}
+
+#if _LIBCPP_STD_VER >= 23
+template <class _T1, class _T2>
+  requires(__is_swappable_v<const _T1> && __is_swappable_v<const _T2>)
+_LIBCPP_HIDE_FROM_ABI constexpr void
+swap(const pair<_T1, _T2>& __x, const pair<_T1, _T2>& __y) noexcept(noexcept(__x.swap(__y))) {
+  __x.swap(__y);
+}
+#endif
+
+template <class _T1, class _T2>
+inline _LIBCPP_HIDE_FROM_ABI
+_LIBCPP_CONSTEXPR_SINCE_CXX14 pair<typename __unwrap_ref_decay<_T1>::type, typename __unwrap_ref_decay<_T2>::type>
+make_pair(_T1&& __t1, _T2&& __t2) {
+  return pair<typename __unwrap_ref_decay<_T1>::type, typename __unwrap_ref_decay<_T2>::type>(
+      std::forward<_T1>(__t1), std::forward<_T2>(__t2));
+}
+
+template <class _T1, class _T2>
+struct _LIBCPP_TEMPLATE_VIS tuple_size<pair<_T1, _T2> > : public integral_constant<size_t, 2> {};
+
+template <size_t _Ip, class _T1, class _T2>
+struct _LIBCPP_TEMPLATE_VIS tuple_element<_Ip, pair<_T1, _T2> > {
+  static_assert(_Ip < 2, "Index out of bounds in std::tuple_element<std::pair<T1, T2>>");
+};
+
+template <class _T1, class _T2>
+struct _LIBCPP_TEMPLATE_VIS tuple_element<0, pair<_T1, _T2> > {
+  using type _LIBCPP_NODEBUG = _T1;
+};
+
+template <class _T1, class _T2>
+struct _LIBCPP_TEMPLATE_VIS tuple_element<1, pair<_T1, _T2> > {
+  using type _LIBCPP_NODEBUG = _T2;
+};
+
+template <size_t _Ip>
+struct __get_pair;
+
+template <>
+struct __get_pair<0> {
+  template <class _T1, class _T2>
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _T1& get(pair<_T1, _T2>& __p) _NOEXCEPT {
+    return __p.first;
+  }
+
+  template <class _T1, class _T2>
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _T1& get(const pair<_T1, _T2>& __p) _NOEXCEPT {
+    return __p.first;
+  }
+
+  template <class _T1, class _T2>
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _T1&& get(pair<_T1, _T2>&& __p) _NOEXCEPT {
+    return std::forward<_T1>(__p.first);
+  }
+
+  template <class _T1, class _T2>
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _T1&& get(const pair<_T1, _T2>&& __p) _NOEXCEPT {
+    return std::forward<const _T1>(__p.first);
+  }
+};
+
+template <>
+struct __get_pair<1> {
+  template <class _T1, class _T2>
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _T2& get(pair<_T1, _T2>& __p) _NOEXCEPT {
+    return __p.second;
+  }
+
+  template <class _T1, class _T2>
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _T2& get(const pair<_T1, _T2>& __p) _NOEXCEPT {
+    return __p.second;
+  }
+
+  template <class _T1, class _T2>
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _T2&& get(pair<_T1, _T2>&& __p) _NOEXCEPT {
+    return std::forward<_T2>(__p.second);
+  }
+
+  template <class _T1, class _T2>
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _T2&& get(const pair<_T1, _T2>&& __p) _NOEXCEPT {
+    return std::forward<const _T2>(__p.second);
+  }
+};
+
+template <size_t _Ip, class _T1, class _T2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, pair<_T1, _T2> >::type&
+get(pair<_T1, _T2>& __p) _NOEXCEPT {
+  return __get_pair<_Ip>::get(__p);
+}
+
+template <size_t _Ip, class _T1, class _T2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, pair<_T1, _T2> >::type&
+get(const pair<_T1, _T2>& __p) _NOEXCEPT {
+  return __get_pair<_Ip>::get(__p);
+}
+
+template <size_t _Ip, class _T1, class _T2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, pair<_T1, _T2> >::type&&
+get(pair<_T1, _T2>&& __p) _NOEXCEPT {
+  return __get_pair<_Ip>::get(std::move(__p));
+}
+
+template <size_t _Ip, class _T1, class _T2>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, pair<_T1, _T2> >::type&&
+get(const pair<_T1, _T2>&& __p) _NOEXCEPT {
+  return __get_pair<_Ip>::get(std::move(__p));
+}
+
+#if _LIBCPP_STD_VER >= 14
+template <class _T1, class _T2>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _T1& get(pair<_T1, _T2>& __p) _NOEXCEPT {
+  return __get_pair<0>::get(__p);
+}
+
+template <class _T1, class _T2>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const& get(pair<_T1, _T2> const& __p) _NOEXCEPT {
+  return __get_pair<0>::get(__p);
+}
+
+template <class _T1, class _T2>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _T1&& get(pair<_T1, _T2>&& __p) _NOEXCEPT {
+  return __get_pair<0>::get(std::move(__p));
+}
+
+template <class _T1, class _T2>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const&& get(pair<_T1, _T2> const&& __p) _NOEXCEPT {
+  return __get_pair<0>::get(std::move(__p));
+}
+
+template <class _T1, class _T2>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _T1& get(pair<_T2, _T1>& __p) _NOEXCEPT {
+  return __get_pair<1>::get(__p);
+}
+
+template <class _T1, class _T2>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const& get(pair<_T2, _T1> const& __p) _NOEXCEPT {
+  return __get_pair<1>::get(__p);
+}
+
+template <class _T1, class _T2>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _T1&& get(pair<_T2, _T1>&& __p) _NOEXCEPT {
+  return __get_pair<1>::get(std::move(__p));
+}
+
+template <class _T1, class _T2>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const&& get(pair<_T2, _T1> const&& __p) _NOEXCEPT {
+  return __get_pair<1>::get(std::move(__p));
+}
+
+#endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___UTILITY_PAIR_H
diff --git a/libcxx/include/__cxx03/__utility/piecewise_construct.h b/libcxx/include/__cxx03/__utility/piecewise_construct.h
new file mode 100644
index 000000000000..52b19d791e10
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/piecewise_construct.h
@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_PIECEWISE_CONSTRUCT_H
+#define _LIBCPP___UTILITY_PIECEWISE_CONSTRUCT_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct _LIBCPP_TEMPLATE_VIS piecewise_construct_t {
+  explicit piecewise_construct_t() = default;
+};
+
+#if _LIBCPP_STD_VER >= 17
+inline constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t();
+#elif !defined(_LIBCPP_CXX03_LANG)
+constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t();
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_PIECEWISE_CONSTRUCT_H
diff --git a/libcxx/include/__cxx03/__utility/priority_tag.h b/libcxx/include/__cxx03/__utility/priority_tag.h
new file mode 100644
index 000000000000..a159ce7f1afb
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/priority_tag.h
@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_PRIORITY_TAG_H
+#define _LIBCPP___UTILITY_PRIORITY_TAG_H
+
+#include <__config>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <size_t _Ip>
+struct __priority_tag : __priority_tag<_Ip - 1> {};
+template <>
+struct __priority_tag<0> {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_PRIORITY_TAG_H
diff --git a/libcxx/include/__cxx03/__utility/private_constructor_tag.h b/libcxx/include/__cxx03/__utility/private_constructor_tag.h
new file mode 100644
index 000000000000..462cab48c9ed
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/private_constructor_tag.h
@@ -0,0 +1,28 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP__UTILITY_PRIVATE_CONSTRUCTOR_TAG_H
+#define _LIBCPP__UTILITY_PRIVATE_CONSTRUCTOR_TAG_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// This tag allows defining non-standard exposition-only constructors while
+// preventing users from being able to use them, since this reserved-name tag
+// needs to be used.
+struct __private_constructor_tag {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP__UTILITY_PRIVATE_CONSTRUCTOR_TAG_H
diff --git a/libcxx/include/__cxx03/__utility/rel_ops.h b/libcxx/include/__cxx03/__utility/rel_ops.h
new file mode 100644
index 000000000000..a8caf5bdeaf2
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/rel_ops.h
@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_REL_OPS_H
+#define _LIBCPP___UTILITY_REL_OPS_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace rel_ops {
+
+template <class _Tp>
+inline _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool operator!=(const _Tp& __x, const _Tp& __y) {
+  return !(__x == __y);
+}
+
+template <class _Tp>
+inline _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool operator>(const _Tp& __x, const _Tp& __y) {
+  return __y < __x;
+}
+
+template <class _Tp>
+inline _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool operator<=(const _Tp& __x, const _Tp& __y) {
+  return !(__y < __x);
+}
+
+template <class _Tp>
+inline _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool operator>=(const _Tp& __x, const _Tp& __y) {
+  return !(__x < __y);
+}
+
+} // namespace rel_ops
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_REL_OPS_H
diff --git a/libcxx/include/__cxx03/__utility/small_buffer.h b/libcxx/include/__cxx03/__utility/small_buffer.h
new file mode 100644
index 000000000000..9e13797573d2
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/small_buffer.h
@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_SMALL_BUFFER_H
+#define _LIBCPP___UTILITY_SMALL_BUFFER_H
+
+#include <__config>
+#include <__memory/construct_at.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_trivially_constructible.h>
+#include <__type_traits/is_trivially_destructible.h>
+#include <__utility/exception_guard.h>
+#include <__utility/forward.h>
+#include <cstddef>
+#include <new>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+
+// __small_buffer is a helper class to perform the well known SBO (small buffer optimization). It is mainly useful to
+// allow type-erasing classes like move_only_function to store small objects in a local buffer without requiring an
+// allocation.
+//
+// This small buffer class only allows storing trivially relocatable objects inside the local storage to allow
+// __small_buffer to be trivially relocatable itself. Since the buffer doesn't know what's stored inside it, the user
+// has to manage the object's lifetime, in particular the destruction of the object.
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <size_t _BufferSize, size_t _BufferAlignment>
+  requires(_BufferSize > 0 && _BufferAlignment > 0)
+class __small_buffer {
+public:
+  template <class _Tp, class _Decayed = decay_t<_Tp>>
+  static constexpr bool __fits_in_buffer =
+      is_trivially_move_constructible_v<_Decayed> && is_trivially_destructible_v<_Decayed> &&
+      sizeof(_Decayed) <= _BufferSize && alignof(_Decayed) <= _BufferAlignment;
+
+  _LIBCPP_HIDE_FROM_ABI __small_buffer()           = default;
+  __small_buffer(const __small_buffer&)            = delete;
+  __small_buffer& operator=(const __small_buffer&) = delete;
+  _LIBCPP_HIDE_FROM_ABI ~__small_buffer()          = default;
+
+  // Relocates the buffer - __delete() should never be called on a moved-from __small_buffer
+  _LIBCPP_HIDE_FROM_ABI __small_buffer(__small_buffer&&)            = default;
+  _LIBCPP_HIDE_FROM_ABI __small_buffer& operator=(__small_buffer&&) = default;
+
+  template <class _Stored>
+  _LIBCPP_HIDE_FROM_ABI _Stored* __get() {
+    if constexpr (__fits_in_buffer<_Stored>)
+      return std::launder(reinterpret_cast<_Stored*>(__buffer_));
+    else
+      return *std::launder(reinterpret_cast<_Stored**>(__buffer_));
+  }
+
+  template <class _Stored>
+  _LIBCPP_HIDE_FROM_ABI _Stored* __alloc() {
+    if constexpr (__fits_in_buffer<_Stored>) {
+      return std::launder(reinterpret_cast<_Stored*>(__buffer_));
+    } else {
+      byte* __allocation = static_cast<byte*>(::operator new[](sizeof(_Stored), align_val_t{alignof(_Stored)}));
+      std::construct_at(reinterpret_cast<byte**>(__buffer_), __allocation);
+      return std::launder(reinterpret_cast<_Stored*>(__allocation));
+    }
+  }
+
+  template <class _Stored>
+  _LIBCPP_HIDE_FROM_ABI void __dealloc() noexcept {
+    if constexpr (!__fits_in_buffer<_Stored>)
+      ::operator delete[](*reinterpret_cast<void**>(__buffer_), sizeof(_Stored), align_val_t{alignof(_Stored)});
+  }
+
+  template <class _Stored, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI void __construct(_Args&&... __args) {
+    _Stored* __buffer = __alloc<_Stored>();
+    auto __guard      = std::__make_exception_guard([&] { __dealloc<_Stored>(); });
+    std::construct_at(__buffer, std::forward<_Args>(__args)...);
+    __guard.__complete();
+  }
+
+private:
+  alignas(_BufferAlignment) byte __buffer_[_BufferSize];
+};
+
+#  undef _LIBCPP_SMALL_BUFFER_TRIVIAL_ABI
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+#endif // _LIBCPP___UTILITY_SMALL_BUFFER_H
diff --git a/libcxx/include/__cxx03/__utility/swap.h b/libcxx/include/__cxx03/__utility/swap.h
new file mode 100644
index 000000000000..ab88b8e0a0b5
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/swap.h
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_SWAP_H
+#define _LIBCPP___UTILITY_SWAP_H
+
+#include <__config>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_swappable.h>
+#include <__utility/declval.h>
+#include <__utility/move.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Tp>
+using __swap_result_t = __enable_if_t<is_move_constructible<_Tp>::value && is_move_assignable<_Tp>::value>;
+#else
+template <class>
+using __swap_result_t = void;
+#endif
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI __swap_result_t<_Tp> _LIBCPP_CONSTEXPR_SINCE_CXX20 swap(_Tp& __x, _Tp& __y)
+    _NOEXCEPT_(is_nothrow_move_constructible<_Tp>::value&& is_nothrow_move_assignable<_Tp>::value) {
+  _Tp __t(std::move(__x));
+  __x = std::move(__y);
+  __y = std::move(__t);
+}
+
+template <class _Tp, size_t _Np, __enable_if_t<__is_swappable_v<_Tp>, int> >
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(_Tp (&__a)[_Np], _Tp (&__b)[_Np])
+    _NOEXCEPT_(__is_nothrow_swappable_v<_Tp>) {
+  for (size_t __i = 0; __i != _Np; ++__i) {
+    swap(__a[__i], __b[__i]);
+  }
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___UTILITY_SWAP_H
diff --git a/libcxx/include/__cxx03/__utility/to_underlying.h b/libcxx/include/__cxx03/__utility/to_underlying.h
new file mode 100644
index 000000000000..77587108f20d
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/to_underlying.h
@@ -0,0 +1,38 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_TO_UNDERLYING_H
+#define _LIBCPP___UTILITY_TO_UNDERLYING_H
+
+#include <__config>
+#include <__type_traits/underlying_type.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr typename underlying_type<_Tp>::type __to_underlying(_Tp __val) noexcept {
+  return static_cast<typename underlying_type<_Tp>::type>(__val);
+}
+#endif // !_LIBCPP_CXX03_LANG
+
+#if _LIBCPP_STD_VER >= 23
+template <class _Tp>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr underlying_type_t<_Tp> to_underlying(_Tp __val) noexcept {
+  return std::__to_underlying(__val);
+}
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_TO_UNDERLYING_H
diff --git a/libcxx/include/__cxx03/__utility/unreachable.h b/libcxx/include/__cxx03/__utility/unreachable.h
new file mode 100644
index 000000000000..d833f74c2e4f
--- /dev/null
+++ b/libcxx/include/__cxx03/__utility/unreachable.h
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_UNREACHABLE_H
+#define _LIBCPP___UTILITY_UNREACHABLE_H
+
+#include <__assert>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI inline void __libcpp_unreachable() {
+  _LIBCPP_ASSERT_INTERNAL(false, "std::unreachable() was reached");
+  __builtin_unreachable();
+}
+
+#if _LIBCPP_STD_VER >= 23
+
+[[noreturn]] _LIBCPP_HIDE_FROM_ABI inline void unreachable() { __libcpp_unreachable(); }
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_UNREACHABLE_H
diff --git a/libcxx/include/__cxx03/__variant/monostate.h b/libcxx/include/__cxx03/__variant/monostate.h
new file mode 100644
index 000000000000..16f156609eb7
--- /dev/null
+++ b/libcxx/include/__cxx03/__variant/monostate.h
@@ -0,0 +1,64 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___VARIANT_MONOSTATE_H
+#define _LIBCPP___VARIANT_MONOSTATE_H
+
+#include <__compare/ordering.h>
+#include <__config>
+#include <__functional/hash.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+struct _LIBCPP_TEMPLATE_VIS monostate {};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(monostate, monostate) noexcept { return true; }
+
+#  if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering operator<=>(monostate, monostate) noexcept {
+  return strong_ordering::equal;
+}
+
+#  else // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator!=(monostate, monostate) noexcept { return false; }
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator<(monostate, monostate) noexcept { return false; }
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator>(monostate, monostate) noexcept { return false; }
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator<=(monostate, monostate) noexcept { return true; }
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator>=(monostate, monostate) noexcept { return true; }
+
+#  endif // _LIBCPP_STD_VER >= 20
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<monostate> {
+  using argument_type = monostate;
+  using result_type   = size_t;
+
+  inline _LIBCPP_HIDE_FROM_ABI result_type operator()(const argument_type&) const _NOEXCEPT {
+    return 66740831; // return a fundamentally attractive random value.
+  }
+};
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___VARIANT_MONOSTATE_H
diff --git a/libcxx/include/__cxx03/__verbose_abort b/libcxx/include/__cxx03/__verbose_abort
new file mode 100644
index 000000000000..195ce65b721f
--- /dev/null
+++ b/libcxx/include/__cxx03/__verbose_abort
@@ -0,0 +1,52 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___VERBOSE_ABORT
+#define _LIBCPP___VERBOSE_ABORT
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// This function should never be called directly from the code -- it should only be called through
+// the _LIBCPP_VERBOSE_ABORT macro.
+_LIBCPP_NORETURN _LIBCPP_AVAILABILITY_VERBOSE_ABORT _LIBCPP_OVERRIDABLE_FUNC_VIS
+_LIBCPP_ATTRIBUTE_FORMAT(__printf__, 1, 2) void __libcpp_verbose_abort(const char* __format, ...);
+
+// _LIBCPP_VERBOSE_ABORT(format, args...)
+//
+// This macro is used to abort the program abnormally while providing additional diagnostic information.
+//
+// The first argument is a printf-style format string, and the remaining arguments are values to format
+// into the format-string. This macro can be customized by users to provide fine-grained control over
+// how verbose termination is triggered.
+//
+// If the user does not supply their own version of the _LIBCPP_VERBOSE_ABORT macro, we pick the default
+// behavior based on whether we know the built library we're running against provides support for the
+// verbose termination handler or not. If it does, we call it. If it doesn't, we call __builtin_abort to
+// make sure that the program terminates but without taking any complex dependencies in this header.
+#if !defined(_LIBCPP_VERBOSE_ABORT)
+
+#  if !_LIBCPP_AVAILABILITY_HAS_VERBOSE_ABORT
+// The decltype is there to suppress -Wunused warnings in this configuration.
+void __use(const char*, ...);
+#    define _LIBCPP_VERBOSE_ABORT(...) (decltype(::std::__use(__VA_ARGS__))(), __builtin_abort())
+#  else
+#    define _LIBCPP_VERBOSE_ABORT(...) ::std::__libcpp_verbose_abort(__VA_ARGS__)
+#  endif
+
+#endif // !defined(_LIBCPP_VERBOSE_ABORT)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___VERBOSE_ABORT
diff --git a/libcxx/include/__cxx03/algorithm b/libcxx/include/__cxx03/algorithm
new file mode 100644
index 000000000000..698e6f5cb7ad
--- /dev/null
+++ b/libcxx/include/__cxx03/algorithm
@@ -0,0 +1,2049 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_ALGORITHM
+#define _LIBCPP_ALGORITHM
+
+/*
+    algorithm synopsis
+
+#include <initializer_list>
+
+namespace std
+{
+
+namespace ranges {
+
+  // [algorithms.results], algorithm result types
+  template <class I, class F>
+    struct in_fun_result;                // since C++20
+
+  template <class I1, class I2>
+    struct in_in_result;                 // since C++20
+
+  template <class I, class O>
+    struct in_out_result;                // since C++20
+
+  template <class I1, class I2, class O>
+    struct in_in_out_result;             // since C++20
+
+  template <class I, class O1, class O2>
+    struct in_out_out_result;            // since C++20
+
+  template <class I1, class I2>
+    struct min_max_result;               // since C++20
+
+  template <class I>
+    struct in_found_result;              // since C++20
+
+  template <class I, class T>
+    struct in_value_result;              // since C++23
+
+  template<forward_iterator I, sentinel_for<I> S, class Proj = identity,
+    indirect_strict_weak_order<projected<I, Proj>> Comp = ranges::less>                                   // since C++20
+  constexpr I min_element(I first, S last, Comp comp = {}, Proj proj = {});
+
+  template<forward_range R, class Proj = identity,
+    indirect_strict_weak_order<projected<iterator_t<R>, Proj>> Comp = ranges::less>                       // since C++20
+  constexpr borrowed_iterator_t<R> min_element(R&& r, Comp comp = {}, Proj proj = {});
+
+  template<forward_iterator I, sentinel_for<I> S, class Proj = identity,
+    indirect_strict_weak_order<projected<I, Proj>> Comp = ranges::less>
+  constexpr I ranges::max_element(I first, S last, Comp comp = {}, Proj proj = {});                       // since C++20
+
+  template<forward_range R, class Proj = identity,
+    indirect_strict_weak_order<projected<iterator_t<R>, Proj>> Comp = ranges::less>
+  constexpr borrowed_iterator_t<R> ranges::max_element(R&& r, Comp comp = {}, Proj proj = {});            // since C++20
+
+  template<class I1, class I2>
+    using mismatch_result = in_in_result<I1, I2>;
+
+  template <input_iterator I1, sentinel_for<_I1> S1, input_iterator I2, sentinel_for<_I2> S2,
+          class Pred = ranges::equal_to, class Proj1 = identity, class Proj2 = identity>
+    requires indirectly_comparable<I1, I2, Pred, Proj1, Proj2>
+  constexpr mismatch_result<_I1, _I2>                                                                     // since C++20
+  mismatch()(I1 first1, S1 last1, I2 first2, S2 last2, Pred pred = {}, Proj1 proj1 = {}, Proj2 proj2 = {})
+
+  template <input_range R1, input_range R2,
+          class Pred = ranges::equal_to, class Proj1 = identity, class Proj2 = identity>
+    requires indirectly_comparable<iterator_t<R1>, iterator_t<R2>, Pred, Proj1, Proj2>
+  constexpr mismatch_result<borrowed_iterator_t<R1>, borrowed_iterator_t<R2>>
+  mismatch(R1&& r1, R2&& r2, Pred pred = {}, Proj1 proj1 = {}, Proj2 proj2 = {})                          // since C++20
+
+    requires indirect_binary_predicate<ranges::equal_to, projected<I, Proj>, const T*>
+    constexpr I find(I first, S last, const T& value, Proj proj = {});                                    // since C++20
+
+  template<input_range R, class T, class Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<iterator_t<R>, Proj>, const T*>
+    constexpr borrowed_iterator_t<R>
+      find(R&& r, const T& value, Proj proj = {});                                                        // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    constexpr I find_if(I first, S last, Pred pred, Proj proj = {});                                      // since C++20
+
+  template<input_range R, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    constexpr borrowed_iterator_t<R>
+      find_if(R&& r, Pred pred, Proj proj = {});                                                          // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    constexpr I find_if_not(I first, S last, Pred pred, Proj proj = {});                                  // since C++20
+
+  template<input_range R, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    constexpr borrowed_iterator_t<R>
+      find_if_not(R&& r, Pred pred, Proj proj = {});                                                      // since C++20
+
+  template<forward_iterator I, sentinel_for<I> S, class T, class Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<I, Proj>, const T*>
+    constexpr subrange<I> find_last(I first, S last, const T& value, Proj proj = {});                     // since C++23
+
+  template<forward_range R, class T, class Proj = identity>
+    requires
+      indirect_binary_predicate<ranges::equal_to, projected<iterator_t<R>, Proj>, const T*>
+    constexpr borrowed_subrange_t<R> find_last(R&& r, const T& value, Proj proj = {});                   // since C++23
+
+  template<forward_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    constexpr subrange<I> find_last_if(I first, S last, Pred pred, Proj proj = {});                      // since C++23
+
+  template<forward_range R, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    constexpr borrowed_subrange_t<R> find_last_if(R&& r, Pred pred, Proj proj = {});                     // since C++23
+
+  template<forward_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    constexpr subrange<I> find_last_if_not(I first, S last, Pred pred, Proj proj = {});                  // since C++23
+
+  template<forward_range R, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    constexpr borrowed_subrange_t<R> find_last_if_not(R&& r, Pred pred, Proj proj = {});                 // since C++23
+
+  template<class T, class Proj = identity,
+           indirect_strict_weak_order<projected<const T*, Proj>> Comp = ranges::less>
+    constexpr const T& min(const T& a, const T& b, Comp comp = {}, Proj proj = {});                       // since C++20
+
+  template<copyable T, class Proj = identity,
+           indirect_strict_weak_order<projected<const T*, Proj>> Comp = ranges::less>
+    constexpr T min(initializer_list<T> r, Comp comp = {}, Proj proj = {});                               // since C++20
+
+ template<input_range R, class Proj = identity,
+          indirect_strict_weak_order<projected<iterator_t<R>, Proj>> Comp = ranges::less>
+   requires indirectly_copyable_storable<iterator_t<R>, range_value_t<R>*>
+   constexpr range_value_t<R>
+     min(R&& r, Comp comp = {}, Proj proj = {});                                                          // since C++20
+
+  template<class T, class Proj = identity,
+           indirect_strict_weak_order<projected<const T*, Proj>> Comp = ranges::less>
+    constexpr const T& max(const T& a, const T& b, Comp comp = {}, Proj proj = {});                       // since C++20
+
+  template<copyable T, class Proj = identity,
+           indirect_strict_weak_order<projected<const T*, Proj>> Comp = ranges::less>
+    constexpr T max(initializer_list<T> r, Comp comp = {}, Proj proj = {});                               // since C++20
+
+  template<input_range R, class Proj = identity,
+           indirect_strict_weak_order<projected<iterator_t<R>, Proj>> Comp = ranges::less>
+    requires indirectly_copyable_storable<iterator_t<R>, range_value_t<R>*>
+    constexpr range_value_t<R>
+      max(R&& r, Comp comp = {}, Proj proj = {});                                                         // since C++20
+
+  template<class I, class O>
+    using unary_transform_result = in_out_result<I, O>;                                                   // since C++20
+
+  template<class I1, class I2, class O>
+    using binary_transform_result = in_in_out_result<I1, I2, O>;                                          // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, weakly_incrementable O,
+           copy_constructible F, class Proj = identity>
+    requires indirectly_writable<O, indirect_result_t<F&, projected<I, Proj>>>
+    constexpr ranges::unary_transform_result<I, O>
+      transform(I first1, S last1, O result, F op, Proj proj = {});                                       // since C++20
+
+  template<input_range R, weakly_incrementable O, copy_constructible F,
+           class Proj = identity>
+    requires indirectly_writable<O, indirect_result_t<F&, projected<iterator_t<R>, Proj>>>
+    constexpr ranges::unary_transform_result<borrowed_iterator_t<R>, O>
+      transform(R&& r, O result, F op, Proj proj = {});                                                   // since C++20
+
+  template<input_iterator I1, sentinel_for<I1> S1, input_iterator I2, sentinel_for<I2> S2,
+           weakly_incrementable O, copy_constructible F, class Proj1 = identity,
+           class Proj2 = identity>
+    requires indirectly_writable<O, indirect_result_t<F&, projected<I1, Proj1>,
+                                           projected<I2, Proj2>>>
+    constexpr ranges::binary_transform_result<I1, I2, O>
+      transform(I1 first1, S1 last1, I2 first2, S2 last2, O result,
+                        F binary_op, Proj1 proj1 = {}, Proj2 proj2 = {});                                 // since C++20
+
+  template<input_range R1, input_range R2, weakly_incrementable O,
+           copy_constructible F, class Proj1 = identity, class Proj2 = identity>
+    requires indirectly_writable<O, indirect_result_t<F&, projected<iterator_t<R1>, Proj1>,
+                                           projected<iterator_t<R2>, Proj2>>>
+    constexpr ranges::binary_transform_result<borrowed_iterator_t<R1>, borrowed_iterator_t<R2>, O>
+      transform(R1&& r1, R2&& r2, O result,
+                        F binary_op, Proj1 proj1 = {}, Proj2 proj2 = {});                                 // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, class T, class Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<I, Proj>, const T*>
+    constexpr iter_difference_t<I>
+      count(I first, S last, const T& value, Proj proj = {});                                             // since C++20
+
+  template<input_range R, class T, class Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<iterator_t<R>, Proj>, const T*>
+    constexpr range_difference_t<R>
+      count(R&& r, const T& value, Proj proj = {});                                                       // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    constexpr iter_difference_t<I>
+      count_if(I first, S last, Pred pred, Proj proj = {});                                               // since C++20
+
+  template<input_range R, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    constexpr range_difference_t<R>
+      count_if(R&& r, Pred pred, Proj proj = {});                                                         // since C++20
+
+  template<class T>
+  using minmax_result = min_max_result<T>;
+
+  template<class T, class Proj = identity,
+           indirect_strict_weak_order<projected<const T*, Proj>> Comp = ranges::less>
+    constexpr ranges::minmax_result<const T&>
+      minmax(const T& a, const T& b, Comp comp = {}, Proj proj = {});                                     // since C++20
+
+  template<copyable T, class Proj = identity,
+           indirect_strict_weak_order<projected<const T*, Proj>> Comp = ranges::less>
+    constexpr ranges::minmax_result<T>
+      minmax(initializer_list<T> r, Comp comp = {}, Proj proj = {});                                      // since C++20
+
+  template<input_range R, class Proj = identity,
+           indirect_strict_weak_order<projected<iterator_t<R>, Proj>> Comp = ranges::less>
+    requires indirectly_copyable_storable<iterator_t<R>, range_value_t<R>*>
+    constexpr ranges::minmax_result<range_value_t<R>>
+      minmax(R&& r, Comp comp = {}, Proj proj = {});                                                      // since C++20
+
+  template<class I>
+  using minmax_element_result = min_max_result<I>;
+
+  template<forward_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_strict_weak_order<projected<I, Proj>> Comp = ranges::less>
+    constexpr ranges::minmax_element_result<I>
+      minmax_element(I first, S last, Comp comp = {}, Proj proj = {});                                    // since C++20
+
+  template<forward_range R, class Proj = identity,
+           indirect_strict_weak_order<projected<iterator_t<R>, Proj>> Comp = ranges::less>
+    constexpr ranges::minmax_element_result<borrowed_iterator_t<R>>
+      minmax_element(R&& r, Comp comp = {}, Proj proj = {});                                              // since C++20
+
+  template<forward_iterator I1, sentinel_for<I1> S1,
+           forward_iterator I2, sentinel_for<I2> S2,
+           class Pred = ranges::equal_to, class Proj1 = identity, class Proj2 = identity>
+    requires indirectly_comparable<I1, I2, Pred, Proj1, Proj2>
+    constexpr bool contains_subrange(I1 first1, S1 last1, I2 first2, S2 last2,
+                                     Pred pred = {}, Proj1 proj1 = {}, Proj2 proj2 = {});                // since C++23
+
+  template<forward_range R1, forward_range R2,
+           class Pred = ranges::equal_to, class Proj1 = identity, class Proj2 = identity>
+    requires indirectly_comparable<iterator_t<R1>, iterator_t<R2>, Pred, Proj1, Proj2>
+    constexpr bool contains_subrange(R1&& r1, R2&& r2, Pred pred = {},
+                                     Proj1 proj1 = {}, Proj2 proj2 = {});                                // since C++23
+
+  template<class I, class O>
+    using copy_result = in_out_result<I, O>;                                                // since C++20
+
+  template<class I, class O>
+    using copy_n_result = in_out_result<I, O>;                                              // since C++20
+
+  template<class I, class O>
+    using copy_if_result = in_out_result<I, O>;                                             // since C++20
+
+  template<class I1, class I2>
+    using copy_backward_result = in_out_result<I1, I2>;                                     // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, class T, class Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<I, Proj>, const T*>
+    constexpr bool ranges::contains(I first, S last, const T& value, Proj proj = {});       // since C++23
+
+  template<input_range R, class T, class Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<iterator_t<R>, Proj>, const T*>
+    constexpr bool ranges::contains(R&& r, const T& value, Proj proj = {});                 // since C++23
+
+  template<input_iterator I, sentinel_for<I> S, weakly_incrementable O>
+    requires indirectly_copyable<I, O>
+    constexpr ranges::copy_result<I, O> ranges::copy(I first, S last, O result);            // since C++20
+
+  template<input_range R, weakly_incrementable O>
+    requires indirectly_copyable<iterator_t<R>, O>
+    constexpr ranges::copy_result<borrowed_iterator_t<R>, O> ranges::copy(R&& r, O result); // since C++20
+
+  template<input_iterator I, weakly_incrementable O>
+    requires indirectly_copyable<I, O>
+    constexpr ranges::copy_n_result<I, O>
+      ranges::copy_n(I first, iter_difference_t<I> n, O result);                            // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, weakly_incrementable O, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    requires indirectly_copyable<I, O>
+    constexpr ranges::copy_if_result<I, O>
+      ranges::copy_if(I first, S last, O result, Pred pred, Proj proj = {});                // since C++20
+
+  template<input_range R, weakly_incrementable O, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    requires indirectly_copyable<iterator_t<R>, O>
+    constexpr ranges::copy_if_result<borrowed_iterator_t<R>, O>
+      ranges::copy_if(R&& r, O result, Pred pred, Proj proj = {});                          // since C++20
+
+  template<bidirectional_iterator I1, sentinel_for<I1> S1, bidirectional_iterator I2>
+    requires indirectly_copyable<I1, I2>
+    constexpr ranges::copy_backward_result<I1, I2>
+      ranges::copy_backward(I1 first, S1 last, I2 result);                                  // since C++20
+
+  template<bidirectional_range R, bidirectional_iterator I>
+    requires indirectly_copyable<iterator_t<R>, I>
+    constexpr ranges::copy_backward_result<borrowed_iterator_t<R>, I>
+      ranges::copy_backward(R&& r, I result);                                               // since C++20
+
+  template<class I, class F>
+    using for_each_result = in_fun_result<I, F>;                                            // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirectly_unary_invocable<projected<I, Proj>> Fun>
+    constexpr ranges::for_each_result<I, Fun>
+      ranges::for_each(I first, S last, Fun f, Proj proj = {});                             // since C++20
+
+  template<input_range R, class Proj = identity,
+           indirectly_unary_invocable<projected<iterator_t<R>, Proj>> Fun>
+    constexpr ranges::for_each_result<borrowed_iterator_t<R>, Fun>
+      ranges::for_each(R&& r, Fun f, Proj proj = {});                                       // since C++20
+
+  template<input_iterator I, class Proj = identity,
+           indirectly_unary_invocable<projected<I, Proj>> Fun>
+    constexpr ranges::for_each_n_result<I, Fun>
+      ranges::for_each_n(I first, iter_difference_t<I> n, Fun f, Proj proj = {});           // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    constexpr bool ranges::is_partitioned(I first, S last, Pred pred, Proj proj = {});      // since C++20
+
+  template<input_range R, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    constexpr bool ranges::is_partitioned(R&& r, Pred pred, Proj proj = {});                // since C++20
+
+  template<random_access_iterator I, sentinel_for<I> S, class Comp = ranges::less,
+          class Proj = identity>
+    requires sortable<I, Comp, Proj>
+    constexpr I
+      ranges::push_heap(I first, S last, Comp comp = {}, Proj proj = {});                   // since C++20
+
+  template<random_access_range R, class Comp = ranges::less, class Proj = identity>
+    requires sortable<iterator_t<R>, Comp, Proj>
+    constexpr borrowed_iterator_t<R>
+      ranges::push_heap(R&& r, Comp comp = {}, Proj proj = {});                             // since C++20
+
+  template<random_access_iterator I, sentinel_for<I> S, class Comp = ranges::less,
+          class Proj = identity>
+    requires sortable<I, Comp, Proj>
+    constexpr I
+      ranges::pop_heap(I first, S last, Comp comp = {}, Proj proj = {});                    // since C++20
+
+  template<random_access_range R, class Comp = ranges::less, class Proj = identity>
+    requires sortable<iterator_t<R>, Comp, Proj>
+    constexpr borrowed_iterator_t<R>
+      ranges::pop_heap(R&& r, Comp comp = {}, Proj proj = {});                              // since C++20
+
+  template<random_access_iterator I, sentinel_for<I> S, class Comp = ranges::less,
+          class Proj = identity>
+    requires sortable<I, Comp, Proj>
+    constexpr I
+      ranges::make_heap(I first, S last, Comp comp = {}, Proj proj = {});                   // since C++20
+
+  template<random_access_range R, class Comp = ranges::less, class Proj = identity>
+    requires sortable<iterator_t<R>, Comp, Proj>
+    constexpr borrowed_iterator_t<R>
+      ranges::make_heap(R&& r, Comp comp = {}, Proj proj = {});                             // since C++20
+
+  template<random_access_iterator I, sentinel_for<I> S, class Comp = ranges::less,
+          class Proj = identity>
+    requires sortable<I, Comp, Proj>
+    constexpr I
+      ranges::sort_heap(I first, S last, Comp comp = {}, Proj proj = {});                   // since C++20
+
+  template<random_access_range R, class Comp = ranges::less, class Proj = identity>
+    requires sortable<iterator_t<R>, Comp, Proj>
+    constexpr borrowed_iterator_t<R>
+      ranges::sort_heap(R&& r, Comp comp = {}, Proj proj = {});                             // since C++20
+
+  template<random_access_iterator I, sentinel_for<I> S, class Proj = identity,
+            indirect_strict_weak_order<projected<I, Proj>> Comp = ranges::less>
+    constexpr bool is_heap(I first, S last, Comp comp = {}, Proj proj = {});                // since C++20
+
+  template<random_access_range R, class Proj = identity,
+            indirect_strict_weak_order<projected<iterator_t<R>, Proj>> Comp = ranges::less>
+    constexpr bool is_heap(R&& r, Comp comp = {}, Proj proj = {});                          // since C++20
+
+  template<random_access_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_strict_weak_order<projected<I, Proj>> Comp = ranges::less>
+    constexpr I is_heap_until(I first, S last, Comp comp = {}, Proj proj = {});             // since C++20
+
+  template<random_access_range R, class Proj = identity,
+           indirect_strict_weak_order<projected<iterator_t<R>, Proj>> Comp = ranges::less>
+    constexpr borrowed_iterator_t<R>
+      is_heap_until(R&& r, Comp comp = {}, Proj proj = {});                                 // since C++20
+
+  template<bidirectional_iterator I, sentinel_for<I> S>
+    requires permutable<I>
+    constexpr I ranges::reverse(I first, S last);                                           // since C++20
+
+  template<bidirectional_range R>
+    requires permutable<iterator_t<R>>
+    constexpr borrowed_iterator_t<R> ranges::reverse(R&& r);                                // since C++20
+
+  template<random_access_iterator I, sentinel_for<I> S, class Comp = ranges::less,
+            class Proj = identity>
+    requires sortable<I, Comp, Proj>
+    constexpr I
+      ranges::sort(I first, S last, Comp comp = {}, Proj proj = {});                        // since C++20
+
+  template<random_access_range R, class Comp = ranges::less, class Proj = identity>
+    requires sortable<iterator_t<R>, Comp, Proj>
+    constexpr borrowed_iterator_t<R>
+      ranges::sort(R&& r, Comp comp = {}, Proj proj = {});                                  // since C++20
+
+  template<random_access_iterator I, sentinel_for<I> S, class Comp = ranges::less,
+          class Proj = identity>
+    requires sortable<I, Comp, Proj>
+    I ranges::stable_sort(I first, S last, Comp comp = {}, Proj proj = {});                 // since C++20
+
+  template<random_access_range R, class Comp = ranges::less, class Proj = identity>
+    requires sortable<iterator_t<R>, Comp, Proj>
+    borrowed_iterator_t<R>
+      ranges::stable_sort(R&& r, Comp comp = {}, Proj proj = {});                           // since C++20
+
+  template<random_access_iterator I, sentinel_for<I> S, class Comp = ranges::less,
+           class Proj = identity>
+    requires sortable<I, Comp, Proj>
+    constexpr I
+      ranges::partial_sort(I first, I middle, S last, Comp comp = {}, Proj proj = {});      // since C++20
+
+  template<random_access_range R, class Comp = ranges::less, class Proj = identity>
+    requires sortable<iterator_t<R>, Comp, Proj>
+    constexpr borrowed_iterator_t<R>
+      ranges::partial_sort(R&& r, iterator_t<R> middle, Comp comp = {}, Proj proj = {});    // since C++20
+
+  template<class T, output_iterator<const T&> O, sentinel_for<O> S>
+    constexpr O ranges::fill(O first, S last, const T& value);                              // since C++20
+
+  template<class T, output_range<const T&> R>
+    constexpr borrowed_iterator_t<R> ranges::fill(R&& r, const T& value);                   // since C++20
+
+  template<class T, output_iterator<const T&> O>
+    constexpr O ranges::fill_n(O first, iter_difference_t<O> n, const T& value);            // since C++20
+
+  template<input_or_output_iterator O, sentinel_for<O> S, copy_constructible F>
+    requires invocable<F&> && indirectly_writable<O, invoke_result_t<F&>>
+    constexpr O generate(O first, S last, F gen);                                           // since C++20
+
+  template<class ExecutionPolicy, class ForwardIterator, class Generator>
+    void generate(ExecutionPolicy&& exec,
+                  ForwardIterator first, ForwardIterator last,
+                  Generator gen);                                                           // since C++17
+
+  template<class R, copy_constructible F>
+    requires invocable<F&> && output_range<R, invoke_result_t<F&>>
+    constexpr borrowed_iterator_t<R> generate(R&& r, F gen);                                // since C++20
+
+  template<input_or_output_iterator O, copy_constructible F>
+    requires invocable<F&> && indirectly_writable<O, invoke_result_t<F&>>
+    constexpr O generate_n(O first, iter_difference_t<O> n, F gen);                         // since C++20
+
+  template<class ExecutionPolicy, class ForwardIterator, class Size, class Generator>
+    ForwardIterator generate_n(ExecutionPolicy&& exec,
+                               ForwardIterator first, Size n, Generator gen);               // since C++17
+
+ template<input_iterator I1, sentinel_for<I1> S1, input_iterator I2, sentinel_for<I2> S2,
+          class Pred = ranges::equal_to, class Proj1 = identity, class Proj2 = identity>
+   requires indirectly_comparable<I1, I2, Pred, Proj1, Proj2>
+   constexpr bool ranges::equal(I1 first1, S1 last1, I2 first2, S2 last2,
+                                Pred pred = {},
+                                Proj1 proj1 = {}, Proj2 proj2 = {});                        // since C++20
+
+ template<input_range R1, input_range R2, class Pred = ranges::equal_to,
+          class Proj1 = identity, class Proj2 = identity>
+   requires indirectly_comparable<iterator_t<R1>, iterator_t<R2>, Pred, Proj1, Proj2>
+   constexpr bool ranges::equal(R1&& r1, R2&& r2, Pred pred = {},
+                                Proj1 proj1 = {}, Proj2 proj2 = {});                        // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    constexpr bool ranges::all_of(I first, S last, Pred pred, Proj proj = {});              // since C++20
+
+  template<input_range R, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    constexpr bool ranges::all_of(R&& r, Pred pred, Proj proj = {});                        // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    constexpr bool ranges::any_of(I first, S last, Pred pred, Proj proj = {});              // since C++20
+
+  template<input_range R, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    constexpr bool ranges::any_of(R&& r, Pred pred, Proj proj = {});                        // since C++20
+
+  template<input_iterator I1, sentinel_for<I1> S1, input_iterator I2, sentinel_for<I2> S2,
+          class Pred = ranges::equal_to, class Proj1 = identity, class Proj2 = identity>
+    requires (forward_iterator<I1> || sized_sentinel_for<S1, I1>) &&
+           (forward_iterator<I2> || sized_sentinel_for<S2, I2>) &&
+           indirectly_comparable<I1, I2, Pred, Proj1, Proj2>
+    constexpr bool ranges::ends_with(I1 first1, S1 last1, I2 first2, S2 last2, Pred pred = {},
+                                   Proj1 proj1 = {}, Proj2 proj2 = {});                     // since C++23
+
+  template<input_range R1, input_range R2, class Pred = ranges::equal_to, class Proj1 = identity,
+          class Proj2 = identity>
+    requires (forward_range<R1> || sized_range<R1>) &&
+           (forward_range<R2> || sized_range<R2>) &&
+           indirectly_comparable<iterator_t<R1>, iterator_t<R2>, Pred, Proj1, Proj2>
+    constexpr bool ranges::ends_with(R1&& r1, R2&& r2, Pred pred = {},
+                                   Proj1 proj1 = {}, Proj2 proj2 = {});                     // since C++23
+
+  template<input_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    constexpr bool ranges::none_of(I first, S last, Pred pred, Proj proj = {});             // since C++20
+
+  template<input_range R, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    constexpr bool ranges::none_of(R&& r, Pred pred, Proj proj = {});                       // since C++20
+
+  template<input_iterator I1, sentinel_for<I1> S1, input_iterator I2, sentinel_for<I2> S2,
+          class Pred = ranges::equal_to, class Proj1 = identity, class Proj2 = identity>
+    requires indirectly_comparable<I1, I2, Pred, Proj1, Proj2>
+    constexpr bool ranges::starts_with(I1 first1, S1 last1, I2 first2, S2 last2, Pred pred = {},
+                                      Proj1 proj1 = {}, Proj2 proj2 = {});                 // since C++23
+
+  template<input_range R1, input_range R2, class Pred = ranges::equal_to, class Proj1 = identity,
+          class Proj2 = identity>
+    requires indirectly_comparable<iterator_t<R1>, iterator_t<R2>, Pred, Proj1, Proj2>
+    constexpr bool ranges::starts_with(R1&& r1, R2&& r2, Pred pred = {},
+                                      Proj1 proj1 = {}, Proj2 proj2 = {});                // since C++23
+
+  template<input_iterator I1, sentinel_for<I1> S1,
+          random_access_iterator I2, sentinel_for<I2> S2,
+          class Comp = ranges::less, class Proj1 = identity, class Proj2 = identity>
+    requires indirectly_copyable<I1, I2> && sortable<I2, Comp, Proj2> &&
+            indirect_strict_weak_order<Comp, projected<I1, Proj1>, projected<I2, Proj2>>
+    constexpr partial_sort_copy_result<I1, I2>
+      partial_sort_copy(I1 first, S1 last, I2 result_first, S2 result_last,
+                        Comp comp = {}, Proj1 proj1 = {}, Proj2 proj2 = {});                // since C++20
+
+  template<input_range R1, random_access_range R2, class Comp = ranges::less,
+          class Proj1 = identity, class Proj2 = identity>
+    requires indirectly_copyable<iterator_t<R1>, iterator_t<R2>> &&
+            sortable<iterator_t<R2>, Comp, Proj2> &&
+            indirect_strict_weak_order<Comp, projected<iterator_t<R1>, Proj1>,
+                                        projected<iterator_t<R2>, Proj2>>
+    constexpr partial_sort_copy_result<borrowed_iterator_t<R1>, borrowed_iterator_t<R2>>
+      partial_sort_copy(R1&& r, R2&& result_r, Comp comp = {},
+                        Proj1 proj1 = {}, Proj2 proj2 = {});                                // since C++20
+
+  template<forward_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_strict_weak_order<projected<I, Proj>> Comp = ranges::less>
+    constexpr bool ranges::is_sorted(I first, S last, Comp comp = {}, Proj proj = {});      // since C++20
+
+  template<forward_range R, class Proj = identity,
+           indirect_strict_weak_order<projected<iterator_t<R>, Proj>> Comp = ranges::less>
+    constexpr bool ranges::is_sorted(R&& r, Comp comp = {}, Proj proj = {});                // since C++20
+
+  template<forward_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_strict_weak_order<projected<I, Proj>> Comp = ranges::less>
+    constexpr I ranges::is_sorted_until(I first, S last, Comp comp = {}, Proj proj = {});   // since C++20
+
+  template<forward_range R, class Proj = identity,
+           indirect_strict_weak_order<projected<iterator_t<R>, Proj>> Comp = ranges::less>
+    constexpr borrowed_iterator_t<R>
+      ranges::is_sorted_until(R&& r, Comp comp = {}, Proj proj = {});                       // since C++20
+
+  template<random_access_iterator I, sentinel_for<I> S, class Comp = ranges::less,
+          class Proj = identity>
+    requires sortable<I, Comp, Proj>
+    constexpr I
+      ranges::nth_element(I first, I nth, S last, Comp comp = {}, Proj proj = {});          // since C++20
+
+  template<random_access_range R, class Comp = ranges::less, class Proj = identity>
+    requires sortable<iterator_t<R>, Comp, Proj>
+    constexpr borrowed_iterator_t<R>
+      ranges::nth_element(R&& r, iterator_t<R> nth, Comp comp = {}, Proj proj = {});        // since C++20
+
+  template<forward_iterator I, sentinel_for<I> S, class T, class Proj = identity,
+           indirect_strict_weak_order<const T*, projected<I, Proj>> Comp = ranges::less>    // since C++20
+    constexpr I upper_bound(I first, S last, const T& value, Comp comp = {}, Proj proj = {});
+
+  template<forward_range R, class T, class Proj = identity,
+           indirect_strict_weak_order<const T*, projected<iterator_t<R>, Proj>> Comp =
+             ranges::less>
+    constexpr borrowed_iterator_t<R>
+      upper_bound(R&& r, const T& value, Comp comp = {}, Proj proj = {});                   // since C++20
+
+  template<forward_iterator I, sentinel_for<I> S, class T, class Proj = identity,
+           indirect_strict_weak_order<const T*, projected<I, Proj>> Comp = ranges::less>
+    constexpr I lower_bound(I first, S last, const T& value, Comp comp = {},
+                                    Proj proj = {});                                        // since C++20
+  template<forward_range R, class T, class Proj = identity,
+           indirect_strict_weak_order<const T*, projected<iterator_t<R>, Proj>> Comp =
+             ranges::less>
+    constexpr borrowed_iterator_t<R>
+      lower_bound(R&& r, const T& value, Comp comp = {}, Proj proj = {});                   // since C++20
+
+  template<forward_iterator I, sentinel_for<I> S, class T, class Proj = identity,
+           indirect_strict_weak_order<const T*, projected<I, Proj>> Comp = ranges::less>
+    constexpr bool binary_search(I first, S last, const T& value, Comp comp = {},
+                                         Proj proj = {});                                   // since C++20
+
+  template<forward_range R, class T, class Proj = identity,
+           indirect_strict_weak_order<const T*, projected<iterator_t<R>, Proj>> Comp =
+             ranges::less>
+    constexpr bool binary_search(R&& r, const T& value, Comp comp = {},
+                                         Proj proj = {});                                   // since C++20
+
+  template<permutable I, sentinel_for<I> S, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    constexpr subrange<I>
+      partition(I first, S last, Pred pred, Proj proj = {});                                // since C++20
+
+  template<forward_range R, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    requires permutable<iterator_t<R>>
+    constexpr borrowed_subrange_t<R>
+      partition(R&& r, Pred pred, Proj proj = {});                                          // since C++20
+
+  template<bidirectional_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    requires permutable<I>
+    subrange<I> stable_partition(I first, S last, Pred pred, Proj proj = {});               // since C++20
+
+  template<bidirectional_range R, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    requires permutable<iterator_t<R>>
+    borrowed_subrange_t<R> stable_partition(R&& r, Pred pred, Proj proj = {});              // since C++20
+
+  template<input_iterator I1, sentinel_for<I1> S1, forward_iterator I2, sentinel_for<I2> S2,
+           class Pred = ranges::equal_to, class Proj1 = identity, class Proj2 = identity>
+    requires indirectly_comparable<I1, I2, Pred, Proj1, Proj2>
+    constexpr I1 ranges::find_first_of(I1 first1, S1 last1, I2 first2, S2 last2,
+                                       Pred pred = {},
+                                       Proj1 proj1 = {}, Proj2 proj2 = {});                 // since C++20
+
+  template<input_range R1, forward_range R2,
+           class Pred = ranges::equal_to, class Proj1 = identity, class Proj2 = identity>
+    requires indirectly_comparable<iterator_t<R1>, iterator_t<R2>, Pred, Proj1, Proj2>
+    constexpr borrowed_iterator_t<R1>
+      ranges::find_first_of(R1&& r1, R2&& r2,
+                            Pred pred = {},
+                            Proj1 proj1 = {}, Proj2 proj2 = {});                            // since C++20
+
+  template<forward_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_binary_predicate<projected<I, Proj>,
+                                     projected<I, Proj>> Pred = ranges::equal_to>
+    constexpr I ranges::adjacent_find(I first, S last, Pred pred = {}, Proj proj = {});     // since C++20
+
+  template<forward_range R, class Proj = identity,
+           indirect_binary_predicate<projected<iterator_t<R>, Proj>,
+                                     projected<iterator_t<R>, Proj>> Pred = ranges::equal_to>
+    constexpr borrowed_iterator_t<R> ranges::adjacent_find(R&& r, Pred pred = {}, Proj proj = {});  // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, class T1, class T2, class Proj = identity>
+    requires indirectly_writable<I, const T2&> &&
+             indirect_binary_predicate<ranges::equal_to, projected<I, Proj>, const T1*>
+    constexpr I
+      ranges::replace(I first, S last, const T1& old_value, const T2& new_value, Proj proj = {});   // since C++20
+
+  template<input_range R, class T1, class T2, class Proj = identity>
+    requires indirectly_writable<iterator_t<R>, const T2&> &&
+             indirect_binary_predicate<ranges::equal_to, projected<iterator_t<R>, Proj>, const T1*>
+    constexpr borrowed_iterator_t<R>
+      ranges::replace(R&& r, const T1& old_value, const T2& new_value, Proj proj = {});             // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, class T, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    requires indirectly_writable<I, const T&>
+    constexpr I ranges::replace_if(I first, S last, Pred pred, const T& new_value, Proj proj = {}); // since C++20
+
+  template<input_range R, class T, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    requires indirectly_writable<iterator_t<R>, const T&>
+    constexpr borrowed_iterator_t<R>
+      ranges::replace_if(R&& r, Pred pred, const T& new_value, Proj proj = {});                     // since C++20
+
+  template<class T, class Proj = identity,
+           indirect_strict_weak_order<projected<const T*, Proj>> Comp = ranges::less>
+    constexpr const T&
+      ranges::clamp(const T& v, const T& lo, const T& hi, Comp comp = {}, Proj proj = {});          // since C++20
+
+  template<input_iterator I1, sentinel_for<I1> S1, input_iterator I2, sentinel_for<I2> S2,
+           class Proj1 = identity, class Proj2 = identity,
+           indirect_strict_weak_order<projected<I1, Proj1>,
+                                      projected<I2, Proj2>> Comp = ranges::less>
+    constexpr bool
+      ranges::lexicographical_compare(I1 first1, S1 last1, I2 first2, S2 last2,
+                                      Comp comp = {}, Proj1 proj1 = {}, Proj2 proj2 = {});          // since C++20
+
+  template<input_range R1, input_range R2, class Proj1 = identity,
+           class Proj2 = identity,
+           indirect_strict_weak_order<projected<iterator_t<R1>, Proj1>,
+                                      projected<iterator_t<R2>, Proj2>> Comp = ranges::less>
+    constexpr bool
+      ranges::lexicographical_compare(R1&& r1, R2&& r2, Comp comp = {},
+                                      Proj1 proj1 = {}, Proj2 proj2 = {});                          // since C++20
+
+  template<bidirectional_iterator I1, sentinel_for<I1> S1, bidirectional_iterator I2>
+    requires indirectly_movable<I1, I2>
+    constexpr ranges::move_backward_result<I1, I2>
+      ranges::move_backward(I1 first, S1 last, I2 result);                                          // since C++20
+
+  template<bidirectional_range R, bidirectional_iterator I>
+    requires indirectly_movable<iterator_t<R>, I>
+    constexpr ranges::move_backward_result<borrowed_iterator_t<R>, I>
+      ranges::move_backward(R&& r, I result);                                                       // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, weakly_incrementable O>
+    requires indirectly_movable<I, O>
+    constexpr ranges::move_result<I, O>
+      ranges::move(I first, S last, O result);                                                      // since C++20
+
+  template<input_range R, weakly_incrementable O>
+    requires indirectly_movable<iterator_t<R>, O>
+    constexpr ranges::move_result<borrowed_iterator_t<R>, O>
+      ranges::move(R&& r, O result);                                                                // since C++20
+
+  template<class I, class O1, class O2>
+      using partition_copy_result = in_out_out_result<I, O1, O2>;                                   // since C++20
+
+  template<input_iterator I, sentinel_for<I> S,
+          weakly_incrementable O1, weakly_incrementable O2,
+          class Proj = identity, indirect_unary_predicate<projected<I, Proj>> Pred>
+    requires indirectly_copyable<I, O1> && indirectly_copyable<I, O2>
+    constexpr partition_copy_result<I, O1, O2>
+      partition_copy(I first, S last, O1 out_true, O2 out_false, Pred pred,
+                    Proj proj = {});                                                                // since C++20
+
+  template<input_range R, weakly_incrementable O1, weakly_incrementable O2,
+          class Proj = identity,
+          indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    requires indirectly_copyable<iterator_t<R>, O1> &&
+            indirectly_copyable<iterator_t<R>, O2>
+    constexpr partition_copy_result<borrowed_iterator_t<R>, O1, O2>
+      partition_copy(R&& r, O1 out_true, O2 out_false, Pred pred, Proj proj = {});                  // since C++20
+
+  template<forward_iterator I, sentinel_for<I> S, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    constexpr I partition_point(I first, S last, Pred pred, Proj proj = {});                        // since C++20
+
+  template<forward_range R, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    constexpr borrowed_iterator_t<R>
+      partition_point(R&& r, Pred pred, Proj proj = {});                                            // since C++20
+
+  template<class I1, class I2, class O>
+    using merge_result = in_in_out_result<I1, I2, O>;                                               // since C++20
+
+  template<input_iterator I1, sentinel_for<I1> S1, input_iterator I2, sentinel_for<I2> S2,
+           weakly_incrementable O, class Comp = ranges::less, class Proj1 = identity,
+           class Proj2 = identity>
+    requires mergeable<I1, I2, O, Comp, Proj1, Proj2>
+    constexpr merge_result<I1, I2, O>
+      merge(I1 first1, S1 last1, I2 first2, S2 last2, O result,
+            Comp comp = {}, Proj1 proj1 = {}, Proj2 proj2 = {});                                    // since C++20
+
+  template<input_range R1, input_range R2, weakly_incrementable O, class Comp = ranges::less,
+           class Proj1 = identity, class Proj2 = identity>
+    requires mergeable<iterator_t<R1>, iterator_t<R2>, O, Comp, Proj1, Proj2>
+    constexpr merge_result<borrowed_iterator_t<R1>, borrowed_iterator_t<R2>, O>
+      merge(R1&& r1, R2&& r2, O result,
+            Comp comp = {}, Proj1 proj1 = {}, Proj2 proj2 = {});                                    // since C++20
+
+  template<permutable I, sentinel_for<I> S, class T, class Proj = identity>
+    requires indirect_binary_predicate<ranges::equal_to, projected<I, Proj>, const T*>
+    constexpr subrange<I> ranges::remove(I first, S last, const T& value, Proj proj = {});          // since C++20
+
+  template<forward_range R, class T, class Proj = identity>
+    requires permutable<iterator_t<R>> &&
+             indirect_binary_predicate<ranges::equal_to, projected<iterator_t<R>, Proj>, const T*>
+    constexpr borrowed_subrange_t<R>
+      ranges::remove(R&& r, const T& value, Proj proj = {});                                        // since C++20
+
+  template<permutable I, sentinel_for<I> S, class Proj = identity,
+           indirect_unary_predicate<projected<I, Proj>> Pred>
+    constexpr subrange<I> ranges::remove_if(I first, S last, Pred pred, Proj proj = {});            // since C++20
+
+  template<forward_range R, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    requires permutable<iterator_t<R>>
+    constexpr borrowed_subrange_t<R>
+      ranges::remove_if(R&& r, Pred pred, Proj proj = {});                                          // since C++20
+
+  template<class I, class O>
+    using set_difference_result = in_out_result<I, O>;                                              // since C++20
+
+  template<input_iterator I1, sentinel_for<I1> S1, input_iterator I2, sentinel_for<I2> S2,
+           weakly_incrementable O, class Comp = ranges::less,
+           class Proj1 = identity, class Proj2 = identity>
+    requires mergeable<I1, I2, O, Comp, Proj1, Proj2>
+    constexpr set_difference_result<I1, O>
+      set_difference(I1 first1, S1 last1, I2 first2, S2 last2, O result,
+                     Comp comp = {}, Proj1 proj1 = {}, Proj2 proj2 = {});                           // since C++20
+
+  template<input_range R1, input_range R2, weakly_incrementable O,
+           class Comp = ranges::less, class Proj1 = identity, class Proj2 = identity>
+    requires mergeable<iterator_t<R1>, iterator_t<R2>, O, Comp, Proj1, Proj2>
+    constexpr set_difference_result<borrowed_iterator_t<R1>, O>
+      set_difference(R1&& r1, R2&& r2, O result,
+                     Comp comp = {}, Proj1 proj1 = {}, Proj2 proj2 = {});                           // since C++20
+
+  template<class I1, class I2, class O>
+    using set_intersection_result = in_in_out_result<I1, I2, O>;                                    // since C++20
+
+  template<input_iterator I1, sentinel_for<I1> S1, input_iterator I2, sentinel_for<I2> S2,
+           weakly_incrementable O, class Comp = ranges::less,
+           class Proj1 = identity, class Proj2 = identity>
+    requires mergeable<I1, I2, O, Comp, Proj1, Proj2>
+    constexpr set_intersection_result<I1, I2, O>
+      set_intersection(I1 first1, S1 last1, I2 first2, S2 last2, O result,
+                       Comp comp = {}, Proj1 proj1 = {}, Proj2 proj2 = {});                         // since C++20
+
+  template<input_iterator I1, sentinel_for<I1> S1, input_iterator I2, sentinel_for<I2> S2,
+           weakly_incrementable O, class Comp = ranges::less,
+           class Proj1 = identity, class Proj2 = identity>
+    requires mergeable<I1, I2, O, Comp, Proj1, Proj2>
+    constexpr set_intersection_result<borrowed_iterator_t<R1>, borrowed_iterator_t<R2>, O>
+      set_intersection(R1&& r1, R2&& r2, O result,
+                       Comp comp = {}, Proj1 proj1 = {}, Proj2 proj2 = {});                         // since C++20
+
+  template <class _InIter, class _OutIter>
+  using reverse_copy_result = in_out_result<_InIter, _OutIter>;                                     // since C++20
+
+  template<bidirectional_iterator I, sentinel_for<I> S, weakly_incrementable O>
+    requires indirectly_copyable<I, O>
+    constexpr ranges::reverse_copy_result<I, O>
+      ranges::reverse_copy(I first, S last, O result);                                              // since C++20
+
+  template<bidirectional_range R, weakly_incrementable O>
+    requires indirectly_copyable<iterator_t<R>, O>
+    constexpr ranges::reverse_copy_result<borrowed_iterator_t<R>, O>
+      ranges::reverse_copy(R&& r, O result);                                                        // since C++20
+
+  template<permutable I, sentinel_for<I> S>
+    constexpr subrange<I> rotate(I first, I middle, S last);                                        // since C++20
+
+  template<forward_range R>
+    requires permutable<iterator_t<R>>
+    constexpr borrowed_subrange_t<R> rotate(R&& r, iterator_t<R> middle);                           // since C++20
+
+  template <class _InIter, class _OutIter>
+  using rotate_copy_result = in_out_result<_InIter, _OutIter>;                                      // since C++20
+
+  template<forward_iterator I, sentinel_for<I> S, weakly_incrementable O>
+    requires indirectly_copyable<I, O>
+    constexpr ranges::rotate_copy_result<I, O>
+      ranges::rotate_copy(I first, I middle, S last, O result);                                     // since C++20
+
+  template<forward_range R, weakly_incrementable O>
+    requires indirectly_copyable<iterator_t<R>, O>
+    constexpr ranges::rotate_copy_result<borrowed_iterator_t<R>, O>
+      ranges::rotate_copy(R&& r, iterator_t<R> middle, O result);                                   // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, weakly_incrementable O, class Gen>
+    requires (forward_iterator<I> || random_access_iterator<O>) &&
+            indirectly_copyable<I, O> &&
+            uniform_random_bit_generator<remove_reference_t<Gen>>
+    O sample(I first, S last, O out, iter_difference_t<I> n, Gen&& g);                              // since C++20
+
+  template<input_range R, weakly_incrementable O, class Gen>
+    requires (forward_range<R> || random_access_iterator<O>) &&
+            indirectly_copyable<iterator_t<R>, O> &&
+            uniform_random_bit_generator<remove_reference_t<Gen>>
+    O sample(R&& r, O out, range_difference_t<R> n, Gen&& g);                                       // since C++20
+
+  template<random_access_iterator I, sentinel_for<I> S, class Gen>
+    requires permutable<I> &&
+            uniform_random_bit_generator<remove_reference_t<Gen>>
+    I shuffle(I first, S last, Gen&& g);                                                            // since C++20
+
+  template<random_access_range R, class Gen>
+    requires permutable<iterator_t<R>> &&
+            uniform_random_bit_generator<remove_reference_t<Gen>>
+    borrowed_iterator_t<R> shuffle(R&& r, Gen&& g);                                                 // since C++20
+
+  template<forward_iterator I1, sentinel_for<I1> S1, forward_iterator I2,
+         sentinel_for<I2> S2, class Proj1 = identity, class Proj2 = identity,
+         indirect_equivalence_relation<projected<I1, Proj1>,
+                                       projected<I2, Proj2>> Pred = ranges::equal_to>
+  constexpr bool ranges::is_permutation(I1 first1, S1 last1, I2 first2, S2 last2,
+                                        Pred pred = {},
+                                        Proj1 proj1 = {}, Proj2 proj2 = {});                       // since C++20
+
+  template<forward_range R1, forward_range R2,
+         class Proj1 = identity, class Proj2 = identity,
+         indirect_equivalence_relation<projected<iterator_t<R1>, Proj1>,
+                                       projected<iterator_t<R2>, Proj2>> Pred = ranges::equal_to>
+  constexpr bool ranges::is_permutation(R1&& r1, R2&& r2, Pred pred = {},
+                                        Proj1 proj1 = {}, Proj2 proj2 = {});                       // since C++20
+
+  template<forward_iterator I1, sentinel_for<I1> S1, forward_iterator I2,
+           sentinel_for<I2> S2, class Pred = ranges::equal_to,
+           class Proj1 = identity, class Proj2 = identity>
+    requires indirectly_comparable<I1, I2, Pred, Proj1, Proj2>
+    constexpr subrange<I1>
+      ranges::search(I1 first1, S1 last1, I2 first2, S2 last2, Pred pred = {},
+                     Proj1 proj1 = {}, Proj2 proj2 = {});                                           // since C++20
+
+  template<forward_range R1, forward_range R2, class Pred = ranges::equal_to,
+           class Proj1 = identity, class Proj2 = identity>
+    requires indirectly_comparable<iterator_t<R1>, iterator_t<R2>, Pred, Proj1, Proj2>
+    constexpr borrowed_subrange_t<R1>
+      ranges::search(R1&& r1, R2&& r2, Pred pred = {},
+                     Proj1 proj1 = {}, Proj2 proj2 = {});                                           // since C++20
+
+  template<forward_iterator I, sentinel_for<I> S, class T,
+           class Pred = ranges::equal_to, class Proj = identity>
+    requires indirectly_comparable<I, const T*, Pred, Proj>
+    constexpr subrange<I>
+      ranges::search_n(I first, S last, iter_difference_t<I> count,
+                       const T& value, Pred pred = {}, Proj proj = {});                             // since C++20
+
+  template<forward_range R, class T, class Pred = ranges::equal_to,
+           class Proj = identity>
+    requires indirectly_comparable<iterator_t<R>, const T*, Pred, Proj>
+    constexpr borrowed_subrange_t<R>
+      ranges::search_n(R&& r, range_difference_t<R> count,
+                       const T& value, Pred pred = {}, Proj proj = {});                             // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, class T,
+           indirectly-binary-left-foldable<T, I> F>
+    constexpr auto ranges::fold_left(I first, S last, T init, F f);                                 // since C++23
+
+  template<input_range R, class T, indirectly-binary-left-foldable<T, iterator_t<R>> F>
+    constexpr auto fold_left(R&& r, T init, F f);                                                   // since C++23
+
+  template<class I, class T>
+    using fold_left_with_iter_result = in_value_result<I, T>;                                       // since C++23
+
+  template<input_iterator I, sentinel_for<I> S, class T,
+           indirectly-binary-left-foldable<T, I> F>
+    constexpr see below fold_left_with_iter(I first, S last, T init, F f);                          // since C++23
+
+  template<input_range R, class T, indirectly-binary-left-foldable<T, iterator_t<R>> F>
+    constexpr see below fold_left_with_iter(R&& r, T init, F f);                                    // since C++23
+
+  template<forward_iterator I1, sentinel_for<I1> S1, forward_iterator I2, sentinel_for<I2> S2,
+           class Pred = ranges::equal_to, class Proj1 = identity, class Proj2 = identity>
+    requires indirectly_comparable<I1, I2, Pred, Proj1, Proj2>
+    constexpr subrange<I1>
+      ranges::find_end(I1 first1, S1 last1, I2 first2, S2 last2, Pred pred = {},
+                       Proj1 proj1 = {}, Proj2 proj2 = {});                                         // since C++20
+
+  template<forward_range R1, forward_range R2,
+           class Pred = ranges::equal_to, class Proj1 = identity, class Proj2 = identity>
+    requires indirectly_comparable<iterator_t<R1>, iterator_t<R2>, Pred, Proj1, Proj2>
+    constexpr borrowed_subrange_t<R1>
+      ranges::find_end(R1&& r1, R2&& r2, Pred pred = {},
+                       Proj1 proj1 = {}, Proj2 proj2 = {});                                         // since C++20
+
+  template<class I1, class I2, class O>
+    using set_symmetric_difference_result = in_in_out_result<I1, I2, O>;                            // since C++20
+
+  template<input_iterator I1, sentinel_for<I1> S1, input_iterator I2, sentinel_for<I2> S2,
+           weakly_incrementable O, class Comp = ranges::less,
+           class Proj1 = identity, class Proj2 = identity>
+    requires mergeable<I1, I2, O, Comp, Proj1, Proj2>
+    constexpr set_symmetric_difference_result<I1, I2, O>
+      set_symmetric_difference(I1 first1, S1 last1, I2 first2, S2 last2, O result,
+                               Comp comp = {}, Proj1 proj1 = {},
+                               Proj2 proj2 = {});                                                   // since C++20
+
+  template<input_range R1, input_range R2, weakly_incrementable O,
+           class Comp = ranges::less, class Proj1 = identity, class Proj2 = identity>
+    requires mergeable<iterator_t<R1>, iterator_t<R2>, O, Comp, Proj1, Proj2>
+    constexpr set_symmetric_difference_result<borrowed_iterator_t<R1>,
+                                                      borrowed_iterator_t<R2>, O>
+      set_symmetric_difference(R1&& r1, R2&& r2, O result, Comp comp = {},
+                               Proj1 proj1 = {}, Proj2 proj2 = {});                                 // since C++20
+
+  template<forward_iterator I, sentinel_for<I> S, class T, class Proj = identity,
+           indirect_strict_weak_order<const T*, projected<I, Proj>> Comp = ranges::less>
+    constexpr subrange<I>
+      equal_range(I first, S last, const T& value, Comp comp = {}, Proj proj = {});                 // since C++20
+
+  template<forward_range R, class T, class Proj = identity,
+           indirect_strict_weak_order<const T*, projected<iterator_t<R>, Proj>> Comp =
+             ranges::less>
+    constexpr borrowed_subrange_t<R>
+      equal_range(R&& r, const T& value, Comp comp = {}, Proj proj = {});                           // since C++20
+
+  template<class I1, class I2, class O>
+    using set_union_result = in_in_out_result<I1, I2, O>;                                           // since C++20
+
+  template<input_iterator I1, sentinel_for<I1> S1, input_iterator I2, sentinel_for<I2> S2,
+           weakly_incrementable O, class Comp = ranges::less,
+           class Proj1 = identity, class Proj2 = identity>
+    requires mergeable<I1, I2, O, Comp, Proj1, Proj2>
+    constexpr set_union_result<I1, I2, O>
+      set_union(I1 first1, S1 last1, I2 first2, S2 last2, O result, Comp comp = {},
+                Proj1 proj1 = {}, Proj2 proj2 = {});                                                // since C++20
+
+  template<input_range R1, input_range R2, weakly_incrementable O,
+           class Comp = ranges::less, class Proj1 = identity, class Proj2 = identity>
+    requires mergeable<iterator_t<R1>, iterator_t<R2>, O, Comp, Proj1, Proj2>
+    constexpr set_union_result<borrowed_iterator_t<R1>, borrowed_iterator_t<R2>, O>
+      set_union(R1&& r1, R2&& r2, O result, Comp comp = {},
+                Proj1 proj1 = {}, Proj2 proj2 = {});                                                // since C++20
+
+  template<input_iterator I1, sentinel_for<I1> S1, input_iterator I2, sentinel_for<I2> S2,
+           class Proj1 = identity, class Proj2 = identity,
+           indirect_strict_weak_order<projected<I1, Proj1>, projected<I2, Proj2>> Comp =
+             ranges::less>
+    constexpr bool includes(I1 first1, S1 last1, I2 first2, S2 last2, Comp comp = {},
+                            Proj1 proj1 = {}, Proj2 proj2 = {});                                   // since C++20
+
+  template<input_range R1, input_range R2, class Proj1 = identity,
+           class Proj2 = identity,
+           indirect_strict_weak_order<projected<iterator_t<R1>, Proj1>,
+                                      projected<iterator_t<R2>, Proj2>> Comp = ranges::less>
+    constexpr bool includes(R1&& r1, R2&& r2, Comp comp = {},
+                            Proj1 proj1 = {}, Proj2 proj2 = {});                                   // since C++20
+
+  template<bidirectional_iterator I, sentinel_for<I> S, class Comp = ranges::less,
+           class Proj = identity>
+    requires sortable<I, Comp, Proj>
+    I inplace_merge(I first, I middle, S last, Comp comp = {}, Proj proj = {});                    // since C++20
+
+  template<bidirectional_range R, class Comp = ranges::less, class Proj = identity>
+    requires sortable<iterator_t<R>, Comp, Proj>
+    borrowed_iterator_t<R>
+      inplace_merge(R&& r, iterator_t<R> middle, Comp comp = {},
+                    Proj proj = {});                                                               // since C++20
+
+  template<permutable I, sentinel_for<I> S, class Proj = identity,
+           indirect_equivalence_relation<projected<I, Proj>> C = ranges::equal_to>
+    constexpr subrange<I> unique(I first, S last, C comp = {}, Proj proj = {});                    // since C++20
+
+  template<forward_range R, class Proj = identity,
+           indirect_equivalence_relation<projected<iterator_t<R>, Proj>> C = ranges::equal_to>
+    requires permutable<iterator_t<R>>
+    constexpr borrowed_subrange_t<R>
+      unique(R&& r, C comp = {}, Proj proj = {});                                                  // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, weakly_incrementable O, class Proj = identity,
+           indirect_equivalence_relation<projected<I, Proj>> C = ranges::equal_to>
+    requires indirectly_copyable<I, O> &&
+             (forward_iterator<I> ||
+              (input_iterator<O> && same_as<iter_value_t<I>, iter_value_t<O>>) ||
+              indirectly_copyable_storable<I, O>)
+    constexpr unique_copy_result<I, O>
+      unique_copy(I first, S last, O result, C comp = {}, Proj proj = {});                         // since C++20
+
+  template<input_range R, weakly_incrementable O, class Proj = identity,
+           indirect_equivalence_relation<projected<iterator_t<R>, Proj>> C = ranges::equal_to>
+    requires indirectly_copyable<iterator_t<R>, O> &&
+             (forward_iterator<iterator_t<R>> ||
+              (input_iterator<O> && same_as<range_value_t<R>, iter_value_t<O>>) ||
+              indirectly_copyable_storable<iterator_t<R>, O>)
+    constexpr unique_copy_result<borrowed_iterator_t<R>, O>
+      unique_copy(R&& r, O result, C comp = {}, Proj proj = {});                                   // since C++20
+
+  template<class I, class O>
+      using remove_copy_result = in_out_result<I, O>;                                              // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, weakly_incrementable O, class T,
+           class Proj = identity>
+             indirect_binary_predicate<ranges::equal_to, projected<I, Proj>, const T*>
+    constexpr remove_copy_result<I, O>
+      remove_copy(I first, S last, O result, const T& value, Proj proj = {});                      // since C++20
+
+  template<input_range R, weakly_incrementable O, class T, class Proj = identity>
+    requires indirectly_copyable<iterator_t<R>, O> &&
+             indirect_binary_predicate<ranges::equal_to,
+                                       projected<iterator_t<R>, Proj>, const T*>
+    constexpr remove_copy_result<borrowed_iterator_t<R>, O>
+      remove_copy(R&& r, O result, const T& value, Proj proj = {});                                // since C++20
+
+  template<class I, class O>
+      using remove_copy_if_result = in_out_result<I, O>;                                           // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, weakly_incrementable O,
+           class Proj = identity, indirect_unary_predicate<projected<I, Proj>> Pred>
+    requires indirectly_copyable<I, O>
+    constexpr remove_copy_if_result<I, O>
+      remove_copy_if(I first, S last, O result, Pred pred, Proj proj = {});                        // since C++20
+
+  template<input_range R, weakly_incrementable O, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    requires indirectly_copyable<iterator_t<R>, O>
+    constexpr remove_copy_if_result<borrowed_iterator_t<R>, O>
+      remove_copy_if(R&& r, O result, Pred pred, Proj proj = {});                                  // since C++20
+
+  template<class I, class O>
+      using replace_copy_result = in_out_result<I, O>;                                             // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, class T1, class T2,
+           output_iterator<const T2&> O, class Proj = identity>
+    requires indirectly_copyable<I, O> &&
+             indirect_binary_predicate<ranges::equal_to, projected<I, Proj>, const T1*>
+    constexpr replace_copy_result<I, O>
+      replace_copy(I first, S last, O result, const T1& old_value, const T2& new_value,
+                   Proj proj = {});                                                                // since C++20
+
+  template<input_range R, class T1, class T2, output_iterator<const T2&> O,
+           class Proj = identity>
+    requires indirectly_copyable<iterator_t<R>, O> &&
+             indirect_binary_predicate<ranges::equal_to,
+                                       projected<iterator_t<R>, Proj>, const T1*>
+    constexpr replace_copy_result<borrowed_iterator_t<R>, O>
+      replace_copy(R&& r, O result, const T1& old_value, const T2& new_value,
+                   Proj proj = {});                                                                // since C++20
+
+  template<class I, class O>
+      using replace_copy_if_result = in_out_result<I, O>;                                          // since C++20
+
+  template<input_iterator I, sentinel_for<I> S, class T, output_iterator<const T&> O,
+           class Proj = identity, indirect_unary_predicate<projected<I, Proj>> Pred>
+    requires indirectly_copyable<I, O>
+    constexpr replace_copy_if_result<I, O>
+      replace_copy_if(I first, S last, O result, Pred pred, const T& new_value,
+                      Proj proj = {});                                                             // since C++20
+
+  template<input_range R, class T, output_iterator<const T&> O, class Proj = identity,
+           indirect_unary_predicate<projected<iterator_t<R>, Proj>> Pred>
+    requires indirectly_copyable<iterator_t<R>, O>
+    constexpr replace_copy_if_result<borrowed_iterator_t<R>, O>
+      replace_copy_if(R&& r, O result, Pred pred, const T& new_value,
+                      Proj proj = {});                                                             // since C++20
+
+  template<class I>
+    using prev_permutation_result = in_found_result<I>;                                            // since C++20
+
+  template<bidirectional_iterator I, sentinel_for<I> S, class Comp = ranges::less,
+           class Proj = identity>
+    requires sortable<I, Comp, Proj>
+    constexpr ranges::prev_permutation_result<I>
+      ranges::prev_permutation(I first, S last, Comp comp = {}, Proj proj = {});                   // since C++20
+
+  template<bidirectional_range R, class Comp = ranges::less,
+           class Proj = identity>
+    requires sortable<iterator_t<R>, Comp, Proj>
+    constexpr ranges::prev_permutation_result<borrowed_iterator_t<R>>
+      ranges::prev_permutation(R&& r, Comp comp = {}, Proj proj = {});                             // since C++20
+
+  template<class I>
+    using next_permutation_result = in_found_result<I>;                                            // since C++20
+
+  template<bidirectional_iterator I, sentinel_for<I> S, class Comp = ranges::less,
+           class Proj = identity>
+    requires sortable<I, Comp, Proj>
+    constexpr ranges::next_permutation_result<I>
+      ranges::next_permutation(I first, S last, Comp comp = {}, Proj proj = {});                   // since C++20
+
+  template<bidirectional_range R, class Comp = ranges::less,
+           class Proj = identity>
+    requires sortable<iterator_t<R>, Comp, Proj>
+    constexpr ranges::next_permutation_result<borrowed_iterator_t<R>>
+      ranges::next_permutation(R&& r, Comp comp = {}, Proj proj = {});                             // since C++20
+
+}
+
+template <class InputIterator, class Predicate>
+    constexpr bool     // constexpr in C++20
+    all_of(InputIterator first, InputIterator last, Predicate pred);
+
+template <class InputIterator, class Predicate>
+    constexpr bool     // constexpr in C++20
+    any_of(InputIterator first, InputIterator last, Predicate pred);
+
+template <class InputIterator, class Predicate>
+    constexpr bool     // constexpr in C++20
+    none_of(InputIterator first, InputIterator last, Predicate pred);
+
+template <class InputIterator, class Function>
+    constexpr Function          // constexpr in C++20
+    for_each(InputIterator first, InputIterator last, Function f);
+
+template<class InputIterator, class Size, class Function>
+    constexpr InputIterator     // constexpr in C++20
+    for_each_n(InputIterator first, Size n, Function f); // C++17
+
+template <class InputIterator, class T>
+    constexpr InputIterator     // constexpr in C++20
+    find(InputIterator first, InputIterator last, const T& value);
+
+template <class InputIterator, class Predicate>
+    constexpr InputIterator     // constexpr in C++20
+    find_if(InputIterator first, InputIterator last, Predicate pred);
+
+template<class InputIterator, class Predicate>
+    constexpr InputIterator     // constexpr in C++20
+    find_if_not(InputIterator first, InputIterator last, Predicate pred);
+
+template <class ForwardIterator1, class ForwardIterator2>
+    constexpr ForwardIterator1  // constexpr in C++20
+    find_end(ForwardIterator1 first1, ForwardIterator1 last1,
+             ForwardIterator2 first2, ForwardIterator2 last2);
+
+template <class ForwardIterator1, class ForwardIterator2, class BinaryPredicate>
+    constexpr ForwardIterator1  // constexpr in C++20
+    find_end(ForwardIterator1 first1, ForwardIterator1 last1,
+             ForwardIterator2 first2, ForwardIterator2 last2, BinaryPredicate pred);
+
+template <class ForwardIterator1, class ForwardIterator2>
+    constexpr ForwardIterator1  // constexpr in C++20
+    find_first_of(ForwardIterator1 first1, ForwardIterator1 last1,
+                  ForwardIterator2 first2, ForwardIterator2 last2);
+
+template <class ForwardIterator1, class ForwardIterator2, class BinaryPredicate>
+    constexpr ForwardIterator1  // constexpr in C++20
+    find_first_of(ForwardIterator1 first1, ForwardIterator1 last1,
+                  ForwardIterator2 first2, ForwardIterator2 last2, BinaryPredicate pred);
+
+template <class ForwardIterator>
+    constexpr ForwardIterator   // constexpr in C++20
+    adjacent_find(ForwardIterator first, ForwardIterator last);
+
+template <class ForwardIterator, class BinaryPredicate>
+    constexpr ForwardIterator   // constexpr in C++20
+    adjacent_find(ForwardIterator first, ForwardIterator last, BinaryPredicate pred);
+
+template <class InputIterator, class T>
+    constexpr typename iterator_traits<InputIterator>::difference_type  // constexpr in C++20
+    count(InputIterator first, InputIterator last, const T& value);
+
+template <class InputIterator, class Predicate>
+    constexpr typename iterator_traits<InputIterator>::difference_type // constexpr in C++20
+    count_if(InputIterator first, InputIterator last, Predicate pred);
+
+template <class InputIterator1, class InputIterator2>
+    constexpr pair<InputIterator1, InputIterator2>   // constexpr in C++20
+    mismatch(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2);
+
+template <class InputIterator1, class InputIterator2>
+    constexpr pair<InputIterator1, InputIterator2>   // constexpr in C++20
+    mismatch(InputIterator1 first1, InputIterator1 last1,
+             InputIterator2 first2, InputIterator2 last2); // **C++14**
+
+template <class InputIterator1, class InputIterator2, class BinaryPredicate>
+    constexpr pair<InputIterator1, InputIterator2>   // constexpr in C++20
+    mismatch(InputIterator1 first1, InputIterator1 last1,
+             InputIterator2 first2, BinaryPredicate pred);
+
+template <class InputIterator1, class InputIterator2, class BinaryPredicate>
+    constexpr pair<InputIterator1, InputIterator2>   // constexpr in C++20
+    mismatch(InputIterator1 first1, InputIterator1 last1,
+             InputIterator2 first2, InputIterator2 last2,
+             BinaryPredicate pred); // **C++14**
+
+template <class InputIterator1, class InputIterator2>
+    constexpr bool      // constexpr in C++20
+    equal(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2);
+
+template <class InputIterator1, class InputIterator2>
+    constexpr bool      // constexpr in C++20
+    equal(InputIterator1 first1, InputIterator1 last1,
+          InputIterator2 first2, InputIterator2 last2); // **C++14**
+
+template <class InputIterator1, class InputIterator2, class BinaryPredicate>
+    constexpr bool      // constexpr in C++20
+    equal(InputIterator1 first1, InputIterator1 last1,
+          InputIterator2 first2, BinaryPredicate pred);
+
+template <class InputIterator1, class InputIterator2, class BinaryPredicate>
+    constexpr bool      // constexpr in C++20
+    equal(InputIterator1 first1, InputIterator1 last1,
+          InputIterator2 first2, InputIterator2 last2,
+          BinaryPredicate pred); // **C++14**
+
+template<class ForwardIterator1, class ForwardIterator2>
+    constexpr bool      // constexpr in C++20
+    is_permutation(ForwardIterator1 first1, ForwardIterator1 last1,
+                   ForwardIterator2 first2);
+
+template<class ForwardIterator1, class ForwardIterator2>
+    constexpr bool      // constexpr in C++20
+    is_permutation(ForwardIterator1 first1, ForwardIterator1 last1,
+                   ForwardIterator2 first2, ForwardIterator2 last2); // **C++14**
+
+template<class ForwardIterator1, class ForwardIterator2, class BinaryPredicate>
+    constexpr bool      // constexpr in C++20
+    is_permutation(ForwardIterator1 first1, ForwardIterator1 last1,
+                   ForwardIterator2 first2, BinaryPredicate pred);
+
+template<class ForwardIterator1, class ForwardIterator2, class BinaryPredicate>
+    constexpr bool      // constexpr in C++20
+    is_permutation(ForwardIterator1 first1, ForwardIterator1 last1,
+                   ForwardIterator2 first2, ForwardIterator2 last2,
+                   BinaryPredicate pred);  // **C++14**
+
+template <class ForwardIterator1, class ForwardIterator2>
+    constexpr ForwardIterator1      // constexpr in C++20
+    search(ForwardIterator1 first1, ForwardIterator1 last1,
+           ForwardIterator2 first2, ForwardIterator2 last2);
+
+template <class ForwardIterator1, class ForwardIterator2, class BinaryPredicate>
+    constexpr ForwardIterator1      // constexpr in C++20
+    search(ForwardIterator1 first1, ForwardIterator1 last1,
+           ForwardIterator2 first2, ForwardIterator2 last2, BinaryPredicate pred);
+
+template <class ForwardIterator, class Size, class T>
+    constexpr ForwardIterator       // constexpr in C++20
+    search_n(ForwardIterator first, ForwardIterator last, Size count, const T& value);
+
+template <class ForwardIterator, class Size, class T, class BinaryPredicate>
+    constexpr ForwardIterator       // constexpr in C++20
+    search_n(ForwardIterator first, ForwardIterator last,
+             Size count, const T& value, BinaryPredicate pred);
+
+template <class InputIterator, class OutputIterator>
+    constexpr OutputIterator      // constexpr in C++20
+    copy(InputIterator first, InputIterator last, OutputIterator result);
+
+template<class InputIterator, class OutputIterator, class Predicate>
+    constexpr OutputIterator      // constexpr in C++20
+    copy_if(InputIterator first, InputIterator last,
+            OutputIterator result, Predicate pred);
+
+template<class InputIterator, class Size, class OutputIterator>
+    constexpr OutputIterator      // constexpr in C++20
+    copy_n(InputIterator first, Size n, OutputIterator result);
+
+template <class BidirectionalIterator1, class BidirectionalIterator2>
+    constexpr BidirectionalIterator2      // constexpr in C++20
+    copy_backward(BidirectionalIterator1 first, BidirectionalIterator1 last,
+                  BidirectionalIterator2 result);
+
+// [alg.move], move
+template<class InputIterator, class OutputIterator>
+    constexpr OutputIterator move(InputIterator first, InputIterator last,
+                                OutputIterator result);
+
+template<class BidirectionalIterator1, class BidirectionalIterator2>
+    constexpr BidirectionalIterator2
+    move_backward(BidirectionalIterator1 first, BidirectionalIterator1 last,
+                  BidirectionalIterator2 result);
+
+template <class ForwardIterator1, class ForwardIterator2>
+    constexpr ForwardIterator2    // constexpr in C++20
+    swap_ranges(ForwardIterator1 first1, ForwardIterator1 last1, ForwardIterator2 first2);
+
+namespace ranges {
+    template<class I1, class I2>
+    using swap_ranges_result = in_in_result<I1, I2>;
+
+template<input_iterator I1, sentinel_for<I1> S1, input_iterator I2, sentinel_for<I2> S2>
+        requires indirectly_swappable<I1, I2>
+    constexpr ranges::swap_ranges_result<I1, I2>
+        swap_ranges(I1 first1, S1 last1, I2 first2, S2 last2);
+
+template<input_range R1, input_range R2>
+        requires indirectly_swappable<iterator_t<R1>, iterator_t<R2>>
+    constexpr ranges::swap_ranges_result<borrowed_iterator_t<R1>, borrowed_iterator_t<R2>>
+        swap_ranges(R1&& r1, R2&& r2);
+}
+
+template <class ForwardIterator1, class ForwardIterator2>
+    constexpr void                // constexpr in C++20
+    iter_swap(ForwardIterator1 a, ForwardIterator2 b);
+
+template <class InputIterator, class OutputIterator, class UnaryOperation>
+    constexpr OutputIterator      // constexpr in C++20
+    transform(InputIterator first, InputIterator last, OutputIterator result, UnaryOperation op);
+
+template <class InputIterator1, class InputIterator2, class OutputIterator, class BinaryOperation>
+    constexpr OutputIterator      // constexpr in C++20
+    transform(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2,
+              OutputIterator result, BinaryOperation binary_op);
+
+template <class ForwardIterator, class T>
+    constexpr void      // constexpr in C++20
+    replace(ForwardIterator first, ForwardIterator last, const T& old_value, const T& new_value);
+
+template <class ForwardIterator, class Predicate, class T>
+    constexpr void      // constexpr in C++20
+    replace_if(ForwardIterator first, ForwardIterator last, Predicate pred, const T& new_value);
+
+template <class InputIterator, class OutputIterator, class T>
+    constexpr OutputIterator      // constexpr in C++20
+    replace_copy(InputIterator first, InputIterator last, OutputIterator result,
+                 const T& old_value, const T& new_value);
+
+template <class InputIterator, class OutputIterator, class Predicate, class T>
+    constexpr OutputIterator      // constexpr in C++20
+    replace_copy_if(InputIterator first, InputIterator last, OutputIterator result, Predicate pred, const T& new_value);
+
+template <class ForwardIterator, class T>
+    constexpr void      // constexpr in C++20
+    fill(ForwardIterator first, ForwardIterator last, const T& value);
+
+template <class OutputIterator, class Size, class T>
+    constexpr OutputIterator      // constexpr in C++20
+    fill_n(OutputIterator first, Size n, const T& value);
+
+template <class ForwardIterator, class Generator>
+    constexpr void      // constexpr in C++20
+    generate(ForwardIterator first, ForwardIterator last, Generator gen);
+
+template <class OutputIterator, class Size, class Generator>
+    constexpr OutputIterator      // constexpr in C++20
+    generate_n(OutputIterator first, Size n, Generator gen);
+
+template <class ForwardIterator, class T>
+    constexpr ForwardIterator     // constexpr in C++20
+    remove(ForwardIterator first, ForwardIterator last, const T& value);
+
+template <class ForwardIterator, class Predicate>
+    constexpr ForwardIterator     // constexpr in C++20
+    remove_if(ForwardIterator first, ForwardIterator last, Predicate pred);
+
+template <class InputIterator, class OutputIterator, class T>
+    constexpr OutputIterator     // constexpr in C++20
+    remove_copy(InputIterator first, InputIterator last, OutputIterator result, const T& value);
+
+template <class InputIterator, class OutputIterator, class Predicate>
+    constexpr OutputIterator     // constexpr in C++20
+    remove_copy_if(InputIterator first, InputIterator last, OutputIterator result, Predicate pred);
+
+template <class ForwardIterator>
+    constexpr ForwardIterator    // constexpr in C++20
+    unique(ForwardIterator first, ForwardIterator last);
+
+template <class ForwardIterator, class BinaryPredicate>
+    constexpr ForwardIterator    // constexpr in C++20
+    unique(ForwardIterator first, ForwardIterator last, BinaryPredicate pred);
+
+template <class InputIterator, class OutputIterator>
+    constexpr OutputIterator     // constexpr in C++20
+    unique_copy(InputIterator first, InputIterator last, OutputIterator result);
+
+template <class InputIterator, class OutputIterator, class BinaryPredicate>
+    constexpr OutputIterator     // constexpr in C++20
+    unique_copy(InputIterator first, InputIterator last, OutputIterator result, BinaryPredicate pred);
+
+template <class BidirectionalIterator>
+    constexpr void               // constexpr in C++20
+    reverse(BidirectionalIterator first, BidirectionalIterator last);
+
+template <class BidirectionalIterator, class OutputIterator>
+    constexpr OutputIterator       // constexpr in C++20
+    reverse_copy(BidirectionalIterator first, BidirectionalIterator last, OutputIterator result);
+
+template <class ForwardIterator>
+    constexpr ForwardIterator      // constexpr in C++20
+    rotate(ForwardIterator first, ForwardIterator middle, ForwardIterator last);
+
+template <class ForwardIterator, class OutputIterator>
+    constexpr OutputIterator       // constexpr in C++20
+    rotate_copy(ForwardIterator first, ForwardIterator middle, ForwardIterator last, OutputIterator result);
+
+template <class RandomAccessIterator>
+    void
+    random_shuffle(RandomAccessIterator first, RandomAccessIterator last); // deprecated in C++14, removed in C++17
+
+template <class RandomAccessIterator, class RandomNumberGenerator>
+    void
+    random_shuffle(RandomAccessIterator first, RandomAccessIterator last,
+                   RandomNumberGenerator& rand);  // deprecated in C++14, removed in C++17
+
+template<class PopulationIterator, class SampleIterator,
+         class Distance, class UniformRandomBitGenerator>
+    SampleIterator sample(PopulationIterator first, PopulationIterator last,
+                          SampleIterator out, Distance n,
+                          UniformRandomBitGenerator&& g); // C++17
+
+template<class RandomAccessIterator, class UniformRandomNumberGenerator>
+    void shuffle(RandomAccessIterator first, RandomAccessIterator last,
+                 UniformRandomNumberGenerator&& g);
+
+template<class ForwardIterator>
+  constexpr ForwardIterator
+    shift_left(ForwardIterator first, ForwardIterator last,
+               typename iterator_traits<ForwardIterator>::difference_type n); // C++20
+
+template<class ForwardIterator>
+  constexpr ForwardIterator
+    shift_right(ForwardIterator first, ForwardIterator last,
+                typename iterator_traits<ForwardIterator>::difference_type n); // C++20
+
+template <class InputIterator, class Predicate>
+    constexpr bool  // constexpr in C++20
+    is_partitioned(InputIterator first, InputIterator last, Predicate pred);
+
+template <class ForwardIterator, class Predicate>
+    constexpr ForwardIterator  // constexpr in C++20
+    partition(ForwardIterator first, ForwardIterator last, Predicate pred);
+
+template <class InputIterator, class OutputIterator1,
+          class OutputIterator2, class Predicate>
+    constexpr pair<OutputIterator1, OutputIterator2>   // constexpr in C++20
+    partition_copy(InputIterator first, InputIterator last,
+                   OutputIterator1 out_true, OutputIterator2 out_false,
+                   Predicate pred);
+
+template <class ForwardIterator, class Predicate>
+    ForwardIterator
+    stable_partition(ForwardIterator first, ForwardIterator last, Predicate pred);
+
+template<class ForwardIterator, class Predicate>
+    constexpr ForwardIterator  // constexpr in C++20
+    partition_point(ForwardIterator first, ForwardIterator last, Predicate pred);
+
+template <class ForwardIterator>
+    constexpr bool  // constexpr in C++20
+    is_sorted(ForwardIterator first, ForwardIterator last);
+
+template <class ForwardIterator, class Compare>
+    constexpr bool  // constexpr in C++20
+    is_sorted(ForwardIterator first, ForwardIterator last, Compare comp);
+
+template<class ForwardIterator>
+    constexpr ForwardIterator    // constexpr in C++20
+    is_sorted_until(ForwardIterator first, ForwardIterator last);
+
+template <class ForwardIterator, class Compare>
+    constexpr ForwardIterator    // constexpr in C++20
+    is_sorted_until(ForwardIterator first, ForwardIterator last, Compare comp);
+
+template <class RandomAccessIterator>
+    constexpr void               // constexpr in C++20
+    sort(RandomAccessIterator first, RandomAccessIterator last);
+
+template <class RandomAccessIterator, class Compare>
+    constexpr void               // constexpr in C++20
+    sort(RandomAccessIterator first, RandomAccessIterator last, Compare comp);
+
+template <class RandomAccessIterator>
+    void
+    stable_sort(RandomAccessIterator first, RandomAccessIterator last);
+
+template <class RandomAccessIterator, class Compare>
+    void
+    stable_sort(RandomAccessIterator first, RandomAccessIterator last, Compare comp);
+
+template <class RandomAccessIterator>
+    constexpr void                    // constexpr in C++20
+    partial_sort(RandomAccessIterator first, RandomAccessIterator middle, RandomAccessIterator last);
+
+template <class RandomAccessIterator, class Compare>
+    constexpr void                    // constexpr in C++20
+    partial_sort(RandomAccessIterator first, RandomAccessIterator middle, RandomAccessIterator last, Compare comp);
+
+template <class InputIterator, class RandomAccessIterator>
+    constexpr RandomAccessIterator    // constexpr in C++20
+    partial_sort_copy(InputIterator first, InputIterator last,
+                      RandomAccessIterator result_first, RandomAccessIterator result_last);
+
+template <class InputIterator, class RandomAccessIterator, class Compare>
+    constexpr RandomAccessIterator    // constexpr in C++20
+    partial_sort_copy(InputIterator first, InputIterator last,
+                      RandomAccessIterator result_first, RandomAccessIterator result_last, Compare comp);
+
+template <class RandomAccessIterator>
+    constexpr void                    // constexpr in C++20
+    nth_element(RandomAccessIterator first, RandomAccessIterator nth, RandomAccessIterator last);
+
+template <class RandomAccessIterator, class Compare>
+    constexpr void                    // constexpr in C++20
+    nth_element(RandomAccessIterator first, RandomAccessIterator nth, RandomAccessIterator last, Compare comp);
+
+template <class ForwardIterator, class T>
+    constexpr ForwardIterator                         // constexpr in C++20
+    lower_bound(ForwardIterator first, ForwardIterator last, const T& value);
+
+template <class ForwardIterator, class T, class Compare>
+    constexpr ForwardIterator                         // constexpr in C++20
+    lower_bound(ForwardIterator first, ForwardIterator last, const T& value, Compare comp);
+
+template <class ForwardIterator, class T>
+    constexpr ForwardIterator                         // constexpr in C++20
+    upper_bound(ForwardIterator first, ForwardIterator last, const T& value);
+
+template <class ForwardIterator, class T, class Compare>
+    constexpr ForwardIterator                         // constexpr in C++20
+    upper_bound(ForwardIterator first, ForwardIterator last, const T& value, Compare comp);
+
+template <class ForwardIterator, class T>
+    constexpr pair<ForwardIterator, ForwardIterator>  // constexpr in C++20
+    equal_range(ForwardIterator first, ForwardIterator last, const T& value);
+
+template <class ForwardIterator, class T, class Compare>
+    constexpr pair<ForwardIterator, ForwardIterator>  // constexpr in C++20
+    equal_range(ForwardIterator first, ForwardIterator last, const T& value, Compare comp);
+
+template <class ForwardIterator, class T>
+    constexpr bool                                    // constexpr in C++20
+    binary_search(ForwardIterator first, ForwardIterator last, const T& value);
+
+template <class ForwardIterator, class T, class Compare>
+    constexpr bool                                    // constexpr in C++20
+    binary_search(ForwardIterator first, ForwardIterator last, const T& value, Compare comp);
+
+template <class InputIterator1, class InputIterator2, class OutputIterator>
+    constexpr OutputIterator                          // constexpr in C++20
+    merge(InputIterator1 first1, InputIterator1 last1,
+          InputIterator2 first2, InputIterator2 last2, OutputIterator result);
+
+template <class InputIterator1, class InputIterator2, class OutputIterator, class Compare>
+    constexpr OutputIterator                          // constexpr in C++20
+    merge(InputIterator1 first1, InputIterator1 last1,
+          InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp);
+
+template <class BidirectionalIterator>
+    void
+    inplace_merge(BidirectionalIterator first, BidirectionalIterator middle, BidirectionalIterator last);
+
+template <class BidirectionalIterator, class Compare>
+    void
+    inplace_merge(BidirectionalIterator first, BidirectionalIterator middle, BidirectionalIterator last, Compare comp);
+
+template <class InputIterator1, class InputIterator2>
+    constexpr bool                                    // constexpr in C++20
+    includes(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2);
+
+template <class InputIterator1, class InputIterator2, class Compare>
+    constexpr bool                                    // constexpr in C++20
+    includes(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2, Compare comp);
+
+template <class InputIterator1, class InputIterator2, class OutputIterator>
+    constexpr OutputIterator                          // constexpr in C++20
+    set_union(InputIterator1 first1, InputIterator1 last1,
+              InputIterator2 first2, InputIterator2 last2, OutputIterator result);
+
+template <class InputIterator1, class InputIterator2, class OutputIterator, class Compare>
+    constexpr OutputIterator                          // constexpr in C++20
+    set_union(InputIterator1 first1, InputIterator1 last1,
+              InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp);
+
+template <class InputIterator1, class InputIterator2, class OutputIterator>
+    constexpr OutputIterator                         // constexpr in C++20
+    set_intersection(InputIterator1 first1, InputIterator1 last1,
+                     InputIterator2 first2, InputIterator2 last2, OutputIterator result);
+
+template <class InputIterator1, class InputIterator2, class OutputIterator, class Compare>
+    constexpr OutputIterator                         // constexpr in C++20
+    set_intersection(InputIterator1 first1, InputIterator1 last1,
+                     InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp);
+
+template <class InputIterator1, class InputIterator2, class OutputIterator>
+    constexpr OutputIterator                         // constexpr in C++20
+    set_difference(InputIterator1 first1, InputIterator1 last1,
+                   InputIterator2 first2, InputIterator2 last2, OutputIterator result);
+
+template <class InputIterator1, class InputIterator2, class OutputIterator, class Compare>
+    constexpr OutputIterator                         // constexpr in C++20
+    set_difference(InputIterator1 first1, InputIterator1 last1,
+                   InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp);
+
+template <class InputIterator1, class InputIterator2, class OutputIterator>
+    constexpr OutputIterator                         // constexpr in C++20
+    set_symmetric_difference(InputIterator1 first1, InputIterator1 last1,
+                             InputIterator2 first2, InputIterator2 last2, OutputIterator result);
+
+template <class InputIterator1, class InputIterator2, class OutputIterator, class Compare>
+    constexpr OutputIterator                         // constexpr in C++20
+    set_symmetric_difference(InputIterator1 first1, InputIterator1 last1,
+                             InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp);
+
+template <class RandomAccessIterator>
+    constexpr void                                   // constexpr in C++20
+    push_heap(RandomAccessIterator first, RandomAccessIterator last);
+
+template <class RandomAccessIterator, class Compare>
+    constexpr void                                   // constexpr in C++20
+    push_heap(RandomAccessIterator first, RandomAccessIterator last, Compare comp);
+
+template <class RandomAccessIterator>
+    constexpr void                                   // constexpr in C++20
+    pop_heap(RandomAccessIterator first, RandomAccessIterator last);
+
+template <class RandomAccessIterator, class Compare>
+    constexpr void                                   // constexpr in C++20
+    pop_heap(RandomAccessIterator first, RandomAccessIterator last, Compare comp);
+
+template <class RandomAccessIterator>
+    constexpr void                                   // constexpr in C++20
+    make_heap(RandomAccessIterator first, RandomAccessIterator last);
+
+template <class RandomAccessIterator, class Compare>
+    constexpr void                                   // constexpr in C++20
+    make_heap(RandomAccessIterator first, RandomAccessIterator last, Compare comp);
+
+template <class RandomAccessIterator>
+    constexpr void                                   // constexpr in C++20
+    sort_heap(RandomAccessIterator first, RandomAccessIterator last);
+
+template <class RandomAccessIterator, class Compare>
+    constexpr void                                   // constexpr in C++20
+    sort_heap(RandomAccessIterator first, RandomAccessIterator last, Compare comp);
+
+template <class RandomAccessIterator>
+    constexpr bool   // constexpr in C++20
+    is_heap(RandomAccessIterator first, RandomAccessiterator last);
+
+template <class RandomAccessIterator, class Compare>
+    constexpr bool   // constexpr in C++20
+    is_heap(RandomAccessIterator first, RandomAccessiterator last, Compare comp);
+
+template <class RandomAccessIterator>
+    constexpr RandomAccessIterator   // constexpr in C++20
+    is_heap_until(RandomAccessIterator first, RandomAccessiterator last);
+
+template <class RandomAccessIterator, class Compare>
+    constexpr RandomAccessIterator   // constexpr in C++20
+    is_heap_until(RandomAccessIterator first, RandomAccessiterator last, Compare comp);
+
+template <class ForwardIterator>
+    constexpr ForwardIterator        // constexpr in C++14
+    min_element(ForwardIterator first, ForwardIterator last);
+
+template <class ForwardIterator, class Compare>
+    constexpr ForwardIterator        // constexpr in C++14
+    min_element(ForwardIterator first, ForwardIterator last, Compare comp);
+
+template <class T>
+    constexpr const T&               // constexpr in C++14
+    min(const T& a, const T& b);
+
+template <class T, class Compare>
+    constexpr const T&               // constexpr in C++14
+    min(const T& a, const T& b, Compare comp);
+
+template<class T>
+    constexpr T                      // constexpr in C++14
+    min(initializer_list<T> t);
+
+template<class T, class Compare>
+    constexpr T                      // constexpr in C++14
+    min(initializer_list<T> t, Compare comp);
+
+template<class T>
+    constexpr const T& clamp(const T& v, const T& lo, const T& hi);               // C++17
+
+template<class T, class Compare>
+    constexpr const T& clamp(const T& v, const T& lo, const T& hi, Compare comp); // C++17
+
+template <class ForwardIterator>
+    constexpr ForwardIterator        // constexpr in C++14
+    max_element(ForwardIterator first, ForwardIterator last);
+
+template <class ForwardIterator, class Compare>
+    constexpr ForwardIterator        // constexpr in C++14
+    max_element(ForwardIterator first, ForwardIterator last, Compare comp);
+
+template <class T>
+    constexpr const T&               // constexpr in C++14
+    max(const T& a, const T& b);
+
+template <class T, class Compare>
+    constexpr const T&               // constexpr in C++14
+    max(const T& a, const T& b, Compare comp);
+
+template<class T>
+    constexpr T                      // constexpr in C++14
+    max(initializer_list<T> t);
+
+template<class T, class Compare>
+    constexpr T                      // constexpr in C++14
+    max(initializer_list<T> t, Compare comp);
+
+template<class ForwardIterator>
+    constexpr pair<ForwardIterator, ForwardIterator>  // constexpr in C++14
+    minmax_element(ForwardIterator first, ForwardIterator last);
+
+template<class ForwardIterator, class Compare>
+    constexpr pair<ForwardIterator, ForwardIterator>  // constexpr in C++14
+    minmax_element(ForwardIterator first, ForwardIterator last, Compare comp);
+
+template<class T>
+    constexpr pair<const T&, const T&>  // constexpr in C++14
+    minmax(const T& a, const T& b);
+
+template<class T, class Compare>
+    constexpr pair<const T&, const T&>  // constexpr in C++14
+    minmax(const T& a, const T& b, Compare comp);
+
+template<class T>
+    constexpr pair<T, T>                // constexpr in C++14
+    minmax(initializer_list<T> t);
+
+template<class T, class Compare>
+    constexpr pair<T, T>                // constexpr in C++14
+    minmax(initializer_list<T> t, Compare comp);
+
+template <class InputIterator1, class InputIterator2>
+    constexpr bool     // constexpr in C++20
+    lexicographical_compare(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2);
+
+template <class InputIterator1, class InputIterator2, class Compare>
+    constexpr bool     // constexpr in C++20
+    lexicographical_compare(InputIterator1 first1, InputIterator1 last1,
+                            InputIterator2 first2, InputIterator2 last2, Compare comp);
+
+template<class InputIterator1, class InputIterator2, class Cmp>
+    constexpr auto
+    lexicographical_compare_three_way(InputIterator1 first1, InputIterator1 last1,
+                                      InputIterator2 first2, InputIterator2 last2,
+                                      Cmp comp)
+      -> decltype(comp(*b1, *b2));                                                        // since C++20
+
+template<class InputIterator1, class InputIterator2>
+    constexpr auto
+    lexicographical_compare_three_way(InputIterator1 first1, InputIterator1 last1,
+                                      InputIterator2 first2, InputIterator2 last2);      // since C++20
+
+template <class BidirectionalIterator>
+    constexpr bool     // constexpr in C++20
+    next_permutation(BidirectionalIterator first, BidirectionalIterator last);
+
+template <class BidirectionalIterator, class Compare>
+    constexpr bool     // constexpr in C++20
+    next_permutation(BidirectionalIterator first, BidirectionalIterator last, Compare comp);
+
+template <class BidirectionalIterator>
+    constexpr bool     // constexpr in C++20
+    prev_permutation(BidirectionalIterator first, BidirectionalIterator last);
+
+template <class BidirectionalIterator, class Compare>
+    constexpr bool     // constexpr in C++20
+    prev_permutation(BidirectionalIterator first, BidirectionalIterator last, Compare comp);
+}  // std
+
+*/
+
+#include <__config>
+
+#include <__algorithm/adjacent_find.h>
+#include <__algorithm/all_of.h>
+#include <__algorithm/any_of.h>
+#include <__algorithm/binary_search.h>
+#include <__algorithm/copy.h>
+#include <__algorithm/copy_backward.h>
+#include <__algorithm/copy_if.h>
+#include <__algorithm/copy_n.h>
+#include <__algorithm/count.h>
+#include <__algorithm/count_if.h>
+#include <__algorithm/equal.h>
+#include <__algorithm/equal_range.h>
+#include <__algorithm/fill.h>
+#include <__algorithm/fill_n.h>
+#include <__algorithm/find.h>
+#include <__algorithm/find_end.h>
+#include <__algorithm/find_first_of.h>
+#include <__algorithm/find_if.h>
+#include <__algorithm/find_if_not.h>
+#include <__algorithm/for_each.h>
+#include <__algorithm/generate.h>
+#include <__algorithm/generate_n.h>
+#include <__algorithm/includes.h>
+#include <__algorithm/inplace_merge.h>
+#include <__algorithm/is_heap.h>
+#include <__algorithm/is_heap_until.h>
+#include <__algorithm/is_partitioned.h>
+#include <__algorithm/is_permutation.h>
+#include <__algorithm/is_sorted.h>
+#include <__algorithm/is_sorted_until.h>
+#include <__algorithm/iter_swap.h>
+#include <__algorithm/lexicographical_compare.h>
+#include <__algorithm/lower_bound.h>
+#include <__algorithm/make_heap.h>
+#include <__algorithm/max.h>
+#include <__algorithm/max_element.h>
+#include <__algorithm/merge.h>
+#include <__algorithm/min.h>
+#include <__algorithm/min_element.h>
+#include <__algorithm/minmax.h>
+#include <__algorithm/minmax_element.h>
+#include <__algorithm/mismatch.h>
+#include <__algorithm/move.h>
+#include <__algorithm/move_backward.h>
+#include <__algorithm/next_permutation.h>
+#include <__algorithm/none_of.h>
+#include <__algorithm/nth_element.h>
+#include <__algorithm/partial_sort.h>
+#include <__algorithm/partial_sort_copy.h>
+#include <__algorithm/partition.h>
+#include <__algorithm/partition_copy.h>
+#include <__algorithm/partition_point.h>
+#include <__algorithm/pop_heap.h>
+#include <__algorithm/prev_permutation.h>
+#include <__algorithm/push_heap.h>
+#include <__algorithm/remove.h>
+#include <__algorithm/remove_copy.h>
+#include <__algorithm/remove_copy_if.h>
+#include <__algorithm/remove_if.h>
+#include <__algorithm/replace.h>
+#include <__algorithm/replace_copy.h>
+#include <__algorithm/replace_copy_if.h>
+#include <__algorithm/replace_if.h>
+#include <__algorithm/reverse.h>
+#include <__algorithm/reverse_copy.h>
+#include <__algorithm/rotate.h>
+#include <__algorithm/rotate_copy.h>
+#include <__algorithm/search.h>
+#include <__algorithm/search_n.h>
+#include <__algorithm/set_difference.h>
+#include <__algorithm/set_intersection.h>
+#include <__algorithm/set_symmetric_difference.h>
+#include <__algorithm/set_union.h>
+#include <__algorithm/shuffle.h>
+#include <__algorithm/sort.h>
+#include <__algorithm/sort_heap.h>
+#include <__algorithm/stable_partition.h>
+#include <__algorithm/stable_sort.h>
+#include <__algorithm/swap_ranges.h>
+#include <__algorithm/transform.h>
+#include <__algorithm/unique.h>
+#include <__algorithm/unique_copy.h>
+#include <__algorithm/upper_bound.h>
+
+#if _LIBCPP_STD_VER >= 17
+#  include <__algorithm/clamp.h>
+#  include <__algorithm/for_each_n.h>
+#  include <__algorithm/pstl.h>
+#  include <__algorithm/sample.h>
+#endif // _LIBCPP_STD_VER >= 17
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__algorithm/in_found_result.h>
+#  include <__algorithm/in_fun_result.h>
+#  include <__algorithm/in_in_out_result.h>
+#  include <__algorithm/in_in_result.h>
+#  include <__algorithm/in_out_out_result.h>
+#  include <__algorithm/in_out_result.h>
+#  include <__algorithm/lexicographical_compare_three_way.h>
+#  include <__algorithm/min_max_result.h>
+#  include <__algorithm/ranges_adjacent_find.h>
+#  include <__algorithm/ranges_all_of.h>
+#  include <__algorithm/ranges_any_of.h>
+#  include <__algorithm/ranges_binary_search.h>
+#  include <__algorithm/ranges_clamp.h>
+#  include <__algorithm/ranges_contains.h>
+#  include <__algorithm/ranges_copy.h>
+#  include <__algorithm/ranges_copy_backward.h>
+#  include <__algorithm/ranges_copy_if.h>
+#  include <__algorithm/ranges_copy_n.h>
+#  include <__algorithm/ranges_count.h>
+#  include <__algorithm/ranges_count_if.h>
+#  include <__algorithm/ranges_equal.h>
+#  include <__algorithm/ranges_equal_range.h>
+#  include <__algorithm/ranges_fill.h>
+#  include <__algorithm/ranges_fill_n.h>
+#  include <__algorithm/ranges_find.h>
+#  include <__algorithm/ranges_find_end.h>
+#  include <__algorithm/ranges_find_first_of.h>
+#  include <__algorithm/ranges_find_if.h>
+#  include <__algorithm/ranges_find_if_not.h>
+#  include <__algorithm/ranges_for_each.h>
+#  include <__algorithm/ranges_for_each_n.h>
+#  include <__algorithm/ranges_generate.h>
+#  include <__algorithm/ranges_generate_n.h>
+#  include <__algorithm/ranges_includes.h>
+#  include <__algorithm/ranges_inplace_merge.h>
+#  include <__algorithm/ranges_is_heap.h>
+#  include <__algorithm/ranges_is_heap_until.h>
+#  include <__algorithm/ranges_is_partitioned.h>
+#  include <__algorithm/ranges_is_permutation.h>
+#  include <__algorithm/ranges_is_sorted.h>
+#  include <__algorithm/ranges_is_sorted_until.h>
+#  include <__algorithm/ranges_lexicographical_compare.h>
+#  include <__algorithm/ranges_lower_bound.h>
+#  include <__algorithm/ranges_make_heap.h>
+#  include <__algorithm/ranges_max.h>
+#  include <__algorithm/ranges_max_element.h>
+#  include <__algorithm/ranges_merge.h>
+#  include <__algorithm/ranges_min.h>
+#  include <__algorithm/ranges_min_element.h>
+#  include <__algorithm/ranges_minmax.h>
+#  include <__algorithm/ranges_minmax_element.h>
+#  include <__algorithm/ranges_mismatch.h>
+#  include <__algorithm/ranges_move.h>
+#  include <__algorithm/ranges_move_backward.h>
+#  include <__algorithm/ranges_next_permutation.h>
+#  include <__algorithm/ranges_none_of.h>
+#  include <__algorithm/ranges_nth_element.h>
+#  include <__algorithm/ranges_partial_sort.h>
+#  include <__algorithm/ranges_partial_sort_copy.h>
+#  include <__algorithm/ranges_partition.h>
+#  include <__algorithm/ranges_partition_copy.h>
+#  include <__algorithm/ranges_partition_point.h>
+#  include <__algorithm/ranges_pop_heap.h>
+#  include <__algorithm/ranges_prev_permutation.h>
+#  include <__algorithm/ranges_push_heap.h>
+#  include <__algorithm/ranges_remove.h>
+#  include <__algorithm/ranges_remove_copy.h>
+#  include <__algorithm/ranges_remove_copy_if.h>
+#  include <__algorithm/ranges_remove_if.h>
+#  include <__algorithm/ranges_replace.h>
+#  include <__algorithm/ranges_replace_copy.h>
+#  include <__algorithm/ranges_replace_copy_if.h>
+#  include <__algorithm/ranges_replace_if.h>
+#  include <__algorithm/ranges_reverse.h>
+#  include <__algorithm/ranges_reverse_copy.h>
+#  include <__algorithm/ranges_rotate.h>
+#  include <__algorithm/ranges_rotate_copy.h>
+#  include <__algorithm/ranges_sample.h>
+#  include <__algorithm/ranges_search.h>
+#  include <__algorithm/ranges_search_n.h>
+#  include <__algorithm/ranges_set_difference.h>
+#  include <__algorithm/ranges_set_intersection.h>
+#  include <__algorithm/ranges_set_symmetric_difference.h>
+#  include <__algorithm/ranges_set_union.h>
+#  include <__algorithm/ranges_shuffle.h>
+#  include <__algorithm/ranges_sort.h>
+#  include <__algorithm/ranges_sort_heap.h>
+#  include <__algorithm/ranges_stable_partition.h>
+#  include <__algorithm/ranges_stable_sort.h>
+#  include <__algorithm/ranges_swap_ranges.h>
+#  include <__algorithm/ranges_transform.h>
+#  include <__algorithm/ranges_unique.h>
+#  include <__algorithm/ranges_unique_copy.h>
+#  include <__algorithm/ranges_upper_bound.h>
+#  include <__algorithm/shift_left.h>
+#  include <__algorithm/shift_right.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+#  include <__algorithm/fold.h>
+#  include <__algorithm/ranges_contains_subrange.h>
+#  include <__algorithm/ranges_ends_with.h>
+#  include <__algorithm/ranges_find_last.h>
+#  include <__algorithm/ranges_starts_with.h>
+#endif // _LIBCPP_STD_VER >= 23
+
+#include <version>
+
+// standard-mandated includes
+
+// [algorithm.syn]
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER == 14
+#  include <execution>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <bit>
+#  include <concepts>
+#  include <cstdlib>
+#  include <cstring>
+#  include <iterator>
+#  include <memory>
+#  include <stdexcept>
+#  include <type_traits>
+#  include <utility>
+#endif
+
+#endif // _LIBCPP_ALGORITHM
diff --git a/libcxx/include/__cxx03/any b/libcxx/include/__cxx03/any
new file mode 100644
index 000000000000..5def14dc87e6
--- /dev/null
+++ b/libcxx/include/__cxx03/any
@@ -0,0 +1,614 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_ANY
+#define _LIBCPP_ANY
+
+/*
+   any synopsis
+
+namespace std {
+
+  class bad_any_cast : public bad_cast
+  {
+  public:
+    virtual const char* what() const noexcept;
+  };
+
+  class any
+  {
+  public:
+
+    // 6.3.1 any construct/destruct
+    any() noexcept;
+
+    any(const any& other);
+    any(any&& other) noexcept;
+
+    template <class ValueType>
+      any(ValueType&& value);
+
+    ~any();
+
+    // 6.3.2 any assignments
+    any& operator=(const any& rhs);
+    any& operator=(any&& rhs) noexcept;
+
+    template <class ValueType>
+      any& operator=(ValueType&& rhs);
+
+    // 6.3.3 any modifiers
+    template <class ValueType, class... Args>
+      decay_t<ValueType>& emplace(Args&&... args);
+    template <class ValueType, class U, class... Args>
+      decay_t<ValueType>& emplace(initializer_list<U>, Args&&...);
+    void reset() noexcept;
+    void swap(any& rhs) noexcept;
+
+    // 6.3.4 any observers
+    bool has_value() const noexcept;
+    const type_info& type() const noexcept;
+  };
+
+   // 6.4 Non-member functions
+  void swap(any& x, any& y) noexcept;
+
+  template <class T, class ...Args>
+    any make_any(Args&& ...args);
+  template <class T, class U, class ...Args>
+    any make_any(initializer_list<U>, Args&& ...args);
+
+  template<class ValueType>
+    ValueType any_cast(const any& operand);
+  template<class ValueType>
+    ValueType any_cast(any& operand);
+  template<class ValueType>
+    ValueType any_cast(any&& operand);
+
+  template<class ValueType>
+    const ValueType* any_cast(const any* operand) noexcept;
+  template<class ValueType>
+    ValueType* any_cast(any* operand) noexcept;
+
+} // namespace std
+
+*/
+
+#include <__config>
+#include <__memory/allocator.h>
+#include <__memory/allocator_destructor.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/unique_ptr.h>
+#include <__type_traits/add_const.h>
+#include <__type_traits/add_pointer.h>
+#include <__type_traits/aligned_storage.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_function.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_void.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/forward.h>
+#include <__utility/in_place.h>
+#include <__utility/move.h>
+#include <__utility/unreachable.h>
+#include <__verbose_abort>
+#include <initializer_list>
+#include <typeinfo>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+namespace std {
+class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_AVAILABILITY_BAD_ANY_CAST bad_any_cast : public bad_cast {
+public:
+  const char* what() const _NOEXCEPT override;
+};
+} // namespace std
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST void __throw_bad_any_cast() {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw bad_any_cast();
+#  else
+  _LIBCPP_VERBOSE_ABORT("bad_any_cast was thrown in -fno-exceptions mode");
+#  endif
+}
+
+// Forward declarations
+class _LIBCPP_TEMPLATE_VIS any;
+
+template <class _ValueType>
+_LIBCPP_HIDE_FROM_ABI add_pointer_t<add_const_t<_ValueType>> any_cast(any const*) _NOEXCEPT;
+
+template <class _ValueType>
+_LIBCPP_HIDE_FROM_ABI add_pointer_t<_ValueType> any_cast(any*) _NOEXCEPT;
+
+namespace __any_imp {
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+using _Buffer = aligned_storage_t<3 * sizeof(void*), alignof(void*)>;
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+template <class _Tp>
+using _IsSmallObject =
+    integral_constant<bool,
+                      sizeof(_Tp) <= sizeof(_Buffer) && alignof(_Buffer) % alignof(_Tp) == 0 &&
+                          is_nothrow_move_constructible<_Tp>::value >;
+
+enum class _Action { _Destroy, _Copy, _Move, _Get, _TypeInfo };
+
+template <class _Tp>
+struct _SmallHandler;
+template <class _Tp>
+struct _LargeHandler;
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS __unique_typeinfo {
+  static constexpr int __id = 0;
+};
+template <class _Tp>
+constexpr int __unique_typeinfo<_Tp>::__id;
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI constexpr const void* __get_fallback_typeid() {
+  return &__unique_typeinfo<remove_cv_t<remove_reference_t<_Tp>>>::__id;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI bool __compare_typeid(type_info const* __id, const void* __fallback_id) {
+#  if !defined(_LIBCPP_HAS_NO_RTTI)
+  if (__id && *__id == typeid(_Tp))
+    return true;
+#  endif
+  return !__id && __fallback_id == __any_imp::__get_fallback_typeid<_Tp>();
+}
+
+template <class _Tp>
+using _Handler = conditional_t< _IsSmallObject<_Tp>::value, _SmallHandler<_Tp>, _LargeHandler<_Tp>>;
+
+} // namespace __any_imp
+
+class _LIBCPP_TEMPLATE_VIS any {
+public:
+  // construct/destruct
+  _LIBCPP_HIDE_FROM_ABI constexpr any() _NOEXCEPT : __h_(nullptr) {}
+
+  _LIBCPP_HIDE_FROM_ABI any(any const& __other) : __h_(nullptr) {
+    if (__other.__h_)
+      __other.__call(_Action::_Copy, this);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI any(any&& __other) _NOEXCEPT : __h_(nullptr) {
+    if (__other.__h_)
+      __other.__call(_Action::_Move, this);
+  }
+
+  template < class _ValueType,
+             class _Tp = decay_t<_ValueType>,
+             class     = enable_if_t< !is_same<_Tp, any>::value && !__is_inplace_type<_ValueType>::value &&
+                                      is_copy_constructible<_Tp>::value> >
+  _LIBCPP_HIDE_FROM_ABI any(_ValueType&& __value);
+
+  template <class _ValueType,
+            class... _Args,
+            class _Tp = decay_t<_ValueType>,
+            class     = enable_if_t< is_constructible<_Tp, _Args...>::value && is_copy_constructible<_Tp>::value > >
+  _LIBCPP_HIDE_FROM_ABI explicit any(in_place_type_t<_ValueType>, _Args&&... __args);
+
+  template <class _ValueType,
+            class _Up,
+            class... _Args,
+            class _Tp = decay_t<_ValueType>,
+            class     = enable_if_t< is_constructible<_Tp, initializer_list<_Up>&, _Args...>::value &&
+                                     is_copy_constructible<_Tp>::value> >
+  _LIBCPP_HIDE_FROM_ABI explicit any(in_place_type_t<_ValueType>, initializer_list<_Up>, _Args&&... __args);
+
+  _LIBCPP_HIDE_FROM_ABI ~any() { this->reset(); }
+
+  // assignments
+  _LIBCPP_HIDE_FROM_ABI any& operator=(any const& __rhs) {
+    any(__rhs).swap(*this);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI any& operator=(any&& __rhs) _NOEXCEPT {
+    any(std::move(__rhs)).swap(*this);
+    return *this;
+  }
+
+  template < class _ValueType,
+             class _Tp = decay_t<_ValueType>,
+             class     = enable_if_t< !is_same<_Tp, any>::value && is_copy_constructible<_Tp>::value> >
+  _LIBCPP_HIDE_FROM_ABI any& operator=(_ValueType&& __rhs);
+
+  template <class _ValueType,
+            class... _Args,
+            class _Tp = decay_t<_ValueType>,
+            class     = enable_if_t< is_constructible<_Tp, _Args...>::value && is_copy_constructible<_Tp>::value> >
+  _LIBCPP_HIDE_FROM_ABI _Tp& emplace(_Args&&...);
+
+  template <class _ValueType,
+            class _Up,
+            class... _Args,
+            class _Tp = decay_t<_ValueType>,
+            class     = enable_if_t< is_constructible<_Tp, initializer_list<_Up>&, _Args...>::value &&
+                                     is_copy_constructible<_Tp>::value> >
+  _LIBCPP_HIDE_FROM_ABI _Tp& emplace(initializer_list<_Up>, _Args&&...);
+
+  // 6.3.3 any modifiers
+  _LIBCPP_HIDE_FROM_ABI void reset() _NOEXCEPT {
+    if (__h_)
+      this->__call(_Action::_Destroy);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(any& __rhs) _NOEXCEPT;
+
+  // 6.3.4 any observers
+  _LIBCPP_HIDE_FROM_ABI bool has_value() const _NOEXCEPT { return __h_ != nullptr; }
+
+#  if !defined(_LIBCPP_HAS_NO_RTTI)
+  _LIBCPP_HIDE_FROM_ABI const type_info& type() const _NOEXCEPT {
+    if (__h_) {
+      return *static_cast<type_info const*>(this->__call(_Action::_TypeInfo));
+    } else {
+      return typeid(void);
+    }
+  }
+#  endif
+
+private:
+  typedef __any_imp::_Action _Action;
+  using _HandleFuncPtr = void* (*)(_Action, any const*, any*, const type_info*, const void* __fallback_info);
+
+  union _Storage {
+    _LIBCPP_HIDE_FROM_ABI constexpr _Storage() : __ptr(nullptr) {}
+    void* __ptr;
+    __any_imp::_Buffer __buf;
+  };
+
+  _LIBCPP_HIDE_FROM_ABI void*
+  __call(_Action __a, any* __other = nullptr, type_info const* __info = nullptr, const void* __fallback_info = nullptr)
+      const {
+    return __h_(__a, this, __other, __info, __fallback_info);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void* __call(
+      _Action __a, any* __other = nullptr, type_info const* __info = nullptr, const void* __fallback_info = nullptr) {
+    return __h_(__a, this, __other, __info, __fallback_info);
+  }
+
+  template <class>
+  friend struct __any_imp::_SmallHandler;
+  template <class>
+  friend struct __any_imp::_LargeHandler;
+
+  template <class _ValueType>
+  friend add_pointer_t<add_const_t<_ValueType>> any_cast(any const*) _NOEXCEPT;
+
+  template <class _ValueType>
+  friend add_pointer_t<_ValueType> any_cast(any*) _NOEXCEPT;
+
+  _HandleFuncPtr __h_ = nullptr;
+  _Storage __s_;
+};
+
+namespace __any_imp {
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS _SmallHandler {
+  _LIBCPP_HIDE_FROM_ABI static void*
+  __handle(_Action __act, any const* __this, any* __other, type_info const* __info, const void* __fallback_info) {
+    switch (__act) {
+    case _Action::_Destroy:
+      __destroy(const_cast<any&>(*__this));
+      return nullptr;
+    case _Action::_Copy:
+      __copy(*__this, *__other);
+      return nullptr;
+    case _Action::_Move:
+      __move(const_cast<any&>(*__this), *__other);
+      return nullptr;
+    case _Action::_Get:
+      return __get(const_cast<any&>(*__this), __info, __fallback_info);
+    case _Action::_TypeInfo:
+      return __type_info();
+    }
+    __libcpp_unreachable();
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI static _Tp& __create(any& __dest, _Args&&... __args) {
+    typedef allocator<_Tp> _Alloc;
+    typedef allocator_traits<_Alloc> _ATraits;
+    _Alloc __a;
+    _Tp* __ret = static_cast<_Tp*>(static_cast<void*>(&__dest.__s_.__buf));
+    _ATraits::construct(__a, __ret, std::forward<_Args>(__args)...);
+    __dest.__h_ = &_SmallHandler::__handle;
+    return *__ret;
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI static void __destroy(any& __this) {
+    typedef allocator<_Tp> _Alloc;
+    typedef allocator_traits<_Alloc> _ATraits;
+    _Alloc __a;
+    _Tp* __p = static_cast<_Tp*>(static_cast<void*>(&__this.__s_.__buf));
+    _ATraits::destroy(__a, __p);
+    __this.__h_ = nullptr;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static void __copy(any const& __this, any& __dest) {
+    _SmallHandler::__create(__dest, *static_cast<_Tp const*>(static_cast<void const*>(&__this.__s_.__buf)));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static void __move(any& __this, any& __dest) {
+    _SmallHandler::__create(__dest, std::move(*static_cast<_Tp*>(static_cast<void*>(&__this.__s_.__buf))));
+    __destroy(__this);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static void* __get(any& __this, type_info const* __info, const void* __fallback_id) {
+    if (__any_imp::__compare_typeid<_Tp>(__info, __fallback_id))
+      return static_cast<void*>(&__this.__s_.__buf);
+    return nullptr;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static void* __type_info() {
+#  if !defined(_LIBCPP_HAS_NO_RTTI)
+    return const_cast<void*>(static_cast<void const*>(&typeid(_Tp)));
+#  else
+    return nullptr;
+#  endif
+  }
+};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS _LargeHandler {
+  _LIBCPP_HIDE_FROM_ABI static void*
+  __handle(_Action __act, any const* __this, any* __other, type_info const* __info, void const* __fallback_info) {
+    switch (__act) {
+    case _Action::_Destroy:
+      __destroy(const_cast<any&>(*__this));
+      return nullptr;
+    case _Action::_Copy:
+      __copy(*__this, *__other);
+      return nullptr;
+    case _Action::_Move:
+      __move(const_cast<any&>(*__this), *__other);
+      return nullptr;
+    case _Action::_Get:
+      return __get(const_cast<any&>(*__this), __info, __fallback_info);
+    case _Action::_TypeInfo:
+      return __type_info();
+    }
+    __libcpp_unreachable();
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI static _Tp& __create(any& __dest, _Args&&... __args) {
+    typedef allocator<_Tp> _Alloc;
+    typedef allocator_traits<_Alloc> _ATraits;
+    typedef __allocator_destructor<_Alloc> _Dp;
+    _Alloc __a;
+    unique_ptr<_Tp, _Dp> __hold(_ATraits::allocate(__a, 1), _Dp(__a, 1));
+    _Tp* __ret = __hold.get();
+    _ATraits::construct(__a, __ret, std::forward<_Args>(__args)...);
+    __dest.__s_.__ptr = __hold.release();
+    __dest.__h_       = &_LargeHandler::__handle;
+    return *__ret;
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI static void __destroy(any& __this) {
+    typedef allocator<_Tp> _Alloc;
+    typedef allocator_traits<_Alloc> _ATraits;
+    _Alloc __a;
+    _Tp* __p = static_cast<_Tp*>(__this.__s_.__ptr);
+    _ATraits::destroy(__a, __p);
+    _ATraits::deallocate(__a, __p, 1);
+    __this.__h_ = nullptr;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static void __copy(any const& __this, any& __dest) {
+    _LargeHandler::__create(__dest, *static_cast<_Tp const*>(__this.__s_.__ptr));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static void __move(any& __this, any& __dest) {
+    __dest.__s_.__ptr = __this.__s_.__ptr;
+    __dest.__h_       = &_LargeHandler::__handle;
+    __this.__h_       = nullptr;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static void* __get(any& __this, type_info const* __info, void const* __fallback_info) {
+    if (__any_imp::__compare_typeid<_Tp>(__info, __fallback_info))
+      return static_cast<void*>(__this.__s_.__ptr);
+    return nullptr;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static void* __type_info() {
+#  if !defined(_LIBCPP_HAS_NO_RTTI)
+    return const_cast<void*>(static_cast<void const*>(&typeid(_Tp)));
+#  else
+    return nullptr;
+#  endif
+  }
+};
+
+} // namespace __any_imp
+
+template <class _ValueType, class _Tp, class>
+any::any(_ValueType&& __v) : __h_(nullptr) {
+  __any_imp::_Handler<_Tp>::__create(*this, std::forward<_ValueType>(__v));
+}
+
+template <class _ValueType, class... _Args, class _Tp, class>
+any::any(in_place_type_t<_ValueType>, _Args&&... __args) {
+  __any_imp::_Handler<_Tp>::__create(*this, std::forward<_Args>(__args)...);
+}
+
+template <class _ValueType, class _Up, class... _Args, class _Tp, class>
+any::any(in_place_type_t<_ValueType>, initializer_list<_Up> __il, _Args&&... __args) {
+  __any_imp::_Handler<_Tp>::__create(*this, __il, std::forward<_Args>(__args)...);
+}
+
+template <class _ValueType, class, class>
+inline _LIBCPP_HIDE_FROM_ABI any& any::operator=(_ValueType&& __v) {
+  any(std::forward<_ValueType>(__v)).swap(*this);
+  return *this;
+}
+
+template <class _ValueType, class... _Args, class _Tp, class>
+inline _LIBCPP_HIDE_FROM_ABI _Tp& any::emplace(_Args&&... __args) {
+  reset();
+  return __any_imp::_Handler<_Tp>::__create(*this, std::forward<_Args>(__args)...);
+}
+
+template <class _ValueType, class _Up, class... _Args, class _Tp, class>
+inline _LIBCPP_HIDE_FROM_ABI _Tp& any::emplace(initializer_list<_Up> __il, _Args&&... __args) {
+  reset();
+  return __any_imp::_Handler<_Tp>::__create(*this, __il, std::forward<_Args>(__args)...);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI void any::swap(any& __rhs) _NOEXCEPT {
+  if (this == &__rhs)
+    return;
+  if (__h_ && __rhs.__h_) {
+    any __tmp;
+    __rhs.__call(_Action::_Move, &__tmp);
+    this->__call(_Action::_Move, &__rhs);
+    __tmp.__call(_Action::_Move, this);
+  } else if (__h_) {
+    this->__call(_Action::_Move, &__rhs);
+  } else if (__rhs.__h_) {
+    __rhs.__call(_Action::_Move, this);
+  }
+}
+
+// 6.4 Non-member functions
+
+inline _LIBCPP_HIDE_FROM_ABI void swap(any& __lhs, any& __rhs) _NOEXCEPT { __lhs.swap(__rhs); }
+
+template <class _Tp, class... _Args>
+inline _LIBCPP_HIDE_FROM_ABI any make_any(_Args&&... __args) {
+  return any(in_place_type<_Tp>, std::forward<_Args>(__args)...);
+}
+
+template <class _Tp, class _Up, class... _Args>
+inline _LIBCPP_HIDE_FROM_ABI any make_any(initializer_list<_Up> __il, _Args&&... __args) {
+  return any(in_place_type<_Tp>, __il, std::forward<_Args>(__args)...);
+}
+
+template <class _ValueType>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _ValueType any_cast(any const& __v) {
+  using _RawValueType = __remove_cvref_t<_ValueType>;
+  static_assert(is_constructible<_ValueType, _RawValueType const&>::value,
+                "ValueType is required to be a const lvalue reference "
+                "or a CopyConstructible type");
+  auto __tmp = std::any_cast<add_const_t<_RawValueType>>(&__v);
+  if (__tmp == nullptr)
+    __throw_bad_any_cast();
+  return static_cast<_ValueType>(*__tmp);
+}
+
+template <class _ValueType>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _ValueType any_cast(any& __v) {
+  using _RawValueType = __remove_cvref_t<_ValueType>;
+  static_assert(is_constructible<_ValueType, _RawValueType&>::value,
+                "ValueType is required to be an lvalue reference "
+                "or a CopyConstructible type");
+  auto __tmp = std::any_cast<_RawValueType>(&__v);
+  if (__tmp == nullptr)
+    __throw_bad_any_cast();
+  return static_cast<_ValueType>(*__tmp);
+}
+
+template <class _ValueType>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _ValueType any_cast(any&& __v) {
+  using _RawValueType = __remove_cvref_t<_ValueType>;
+  static_assert(is_constructible<_ValueType, _RawValueType>::value,
+                "ValueType is required to be an rvalue reference "
+                "or a CopyConstructible type");
+  auto __tmp = std::any_cast<_RawValueType>(&__v);
+  if (__tmp == nullptr)
+    __throw_bad_any_cast();
+  return static_cast<_ValueType>(std::move(*__tmp));
+}
+
+template <class _ValueType>
+inline _LIBCPP_HIDE_FROM_ABI add_pointer_t<add_const_t<_ValueType>> any_cast(any const* __any) _NOEXCEPT {
+  static_assert(!is_void_v<_ValueType>, "_ValueType may not be void.");
+  static_assert(!is_reference<_ValueType>::value, "_ValueType may not be a reference.");
+  return std::any_cast<_ValueType>(const_cast<any*>(__any));
+}
+
+template <class _RetType>
+inline _LIBCPP_HIDE_FROM_ABI _RetType __pointer_or_func_cast(void* __p, /*IsFunction*/ false_type) noexcept {
+  return static_cast<_RetType>(__p);
+}
+
+template <class _RetType>
+inline _LIBCPP_HIDE_FROM_ABI _RetType __pointer_or_func_cast(void*, /*IsFunction*/ true_type) noexcept {
+  return nullptr;
+}
+
+template <class _ValueType>
+_LIBCPP_HIDE_FROM_ABI add_pointer_t<_ValueType> any_cast(any* __any) _NOEXCEPT {
+  using __any_imp::_Action;
+  static_assert(!is_void_v<_ValueType>, "_ValueType may not be void.");
+  static_assert(!is_reference<_ValueType>::value, "_ValueType may not be a reference.");
+  typedef add_pointer_t<_ValueType> _ReturnType;
+  if (__any && __any->__h_) {
+    void* __p = __any->__call(
+        _Action::_Get,
+        nullptr,
+#  if !defined(_LIBCPP_HAS_NO_RTTI)
+        &typeid(_ValueType),
+#  else
+        nullptr,
+#  endif
+        __any_imp::__get_fallback_typeid<_ValueType>());
+    return std::__pointer_or_func_cast<_ReturnType>(__p, is_function<_ValueType>{});
+  }
+  return nullptr;
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#  include <chrono>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <concepts>
+#  include <cstdlib>
+#  include <iosfwd>
+#  include <iterator>
+#  include <memory>
+#  include <stdexcept>
+#  include <type_traits>
+#  include <variant>
+#endif
+
+#endif // _LIBCPP_ANY
diff --git a/libcxx/include/__cxx03/array b/libcxx/include/__cxx03/array
new file mode 100644
index 000000000000..b8ad28852438
--- /dev/null
+++ b/libcxx/include/__cxx03/array
@@ -0,0 +1,514 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_ARRAY
+#define _LIBCPP_ARRAY
+
+/*
+    array synopsis
+
+namespace std
+{
+template <class T, size_t N >
+struct array
+{
+    // types:
+    typedef T & reference;
+    typedef const T & const_reference;
+    typedef implementation defined iterator;
+    typedef implementation defined const_iterator;
+    typedef size_t size_type;
+    typedef ptrdiff_t difference_type;
+    typedef T value_type;
+    typedef T* pointer;
+    typedef const T* const_pointer;
+    typedef std::reverse_iterator<iterator> reverse_iterator;
+    typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+
+    // No explicit construct/copy/destroy for aggregate type
+    void fill(const T& u);                                      // constexpr in C++20
+    void swap(array& a) noexcept(is_nothrow_swappable_v<T>);    // constexpr in C++20
+
+    // iterators:
+    iterator begin() noexcept;                                  // constexpr in C++17
+    const_iterator begin() const noexcept;                      // constexpr in C++17
+    iterator end() noexcept;                                    // constexpr in C++17
+    const_iterator end() const noexcept;                        // constexpr in C++17
+
+    reverse_iterator rbegin() noexcept;                         // constexpr in C++17
+    const_reverse_iterator rbegin() const noexcept;             // constexpr in C++17
+    reverse_iterator rend() noexcept;                           // constexpr in C++17
+    const_reverse_iterator rend() const noexcept;               // constexpr in C++17
+
+    const_iterator cbegin() const noexcept;                     // constexpr in C++17
+    const_iterator cend() const noexcept;                       // constexpr in C++17
+    const_reverse_iterator crbegin() const noexcept;            // constexpr in C++17
+    const_reverse_iterator crend() const noexcept;              // constexpr in C++17
+
+    // capacity:
+    constexpr size_type size() const noexcept;
+    constexpr size_type max_size() const noexcept;
+    constexpr bool empty() const noexcept;
+
+    // element access:
+    reference operator[](size_type n);                          // constexpr in C++17
+    const_reference operator[](size_type n) const;              // constexpr in C++14
+    reference at(size_type n);                                  // constexpr in C++17
+    const_reference at(size_type n) const;                      // constexpr in C++14
+
+    reference front();                                          // constexpr in C++17
+    const_reference front() const;                              // constexpr in C++14
+    reference back();                                           // constexpr in C++17
+    const_reference back() const;                               // constexpr in C++14
+
+    T* data() noexcept;                                         // constexpr in C++17
+    const T* data() const noexcept;                             // constexpr in C++17
+};
+
+template <class T, class... U>
+  array(T, U...) -> array<T, 1 + sizeof...(U)>;                 // C++17
+
+template <class T, size_t N>
+  bool operator==(const array<T,N>& x, const array<T,N>& y);    // constexpr in C++20
+template <class T, size_t N>
+  bool operator!=(const array<T,N>& x, const array<T,N>& y);    // removed in C++20
+template <class T, size_t N>
+  bool operator<(const array<T,N>& x, const array<T,N>& y);     // removed in C++20
+template <class T, size_t N>
+  bool operator>(const array<T,N>& x, const array<T,N>& y);     // removed in C++20
+template <class T, size_t N>
+  bool operator<=(const array<T,N>& x, const array<T,N>& y);    // removed in C++20
+template <class T, size_t N>
+  bool operator>=(const array<T,N>& x, const array<T,N>& y);    // removed in C++20
+template<class T, size_t N>
+  constexpr synth-three-way-result<T>
+    operator<=>(const array<T, N>& x, const array<T, N>& y);    // since C++20
+
+template <class T, size_t N >
+  void swap(array<T,N>& x, array<T,N>& y) noexcept(noexcept(x.swap(y))); // constexpr in C++20
+
+template <class T, size_t N>
+  constexpr array<remove_cv_t<T>, N> to_array(T (&a)[N]);  // C++20
+template <class T, size_t N>
+  constexpr array<remove_cv_t<T>, N> to_array(T (&&a)[N]); // C++20
+
+template <class T> struct tuple_size;
+template <size_t I, class T> struct tuple_element;
+template <class T, size_t N> struct tuple_size<array<T, N>>;
+template <size_t I, class T, size_t N> struct tuple_element<I, array<T, N>>;
+template <size_t I, class T, size_t N> T& get(array<T, N>&) noexcept;               // constexpr in C++14
+template <size_t I, class T, size_t N> const T& get(const array<T, N>&) noexcept;   // constexpr in C++14
+template <size_t I, class T, size_t N> T&& get(array<T, N>&&) noexcept;             // constexpr in C++14
+template <size_t I, class T, size_t N> const T&& get(const array<T, N>&&) noexcept; // constexpr in C++14
+
+}  // std
+
+*/
+
+#include <__algorithm/equal.h>
+#include <__algorithm/fill_n.h>
+#include <__algorithm/lexicographical_compare.h>
+#include <__algorithm/lexicographical_compare_three_way.h>
+#include <__algorithm/swap_ranges.h>
+#include <__assert>
+#include <__config>
+#include <__fwd/array.h>
+#include <__iterator/reverse_iterator.h>
+#include <__iterator/wrap_iter.h>
+#include <__tuple/sfinae_helpers.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_const.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_swappable.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__type_traits/remove_cv.h>
+#include <__utility/empty.h>
+#include <__utility/integer_sequence.h>
+#include <__utility/move.h>
+#include <__utility/unreachable.h>
+#include <stdexcept>
+#include <version>
+
+// standard-mandated includes
+
+// [iterator.range]
+#include <__iterator/access.h>
+#include <__iterator/data.h>
+#include <__iterator/empty.h>
+#include <__iterator/reverse_access.h>
+#include <__iterator/size.h>
+
+// [array.syn]
+#include <compare>
+#include <initializer_list>
+
+// [tuple.helper]
+#include <__tuple/tuple_element.h>
+#include <__tuple/tuple_size.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, size_t _Size>
+struct _LIBCPP_TEMPLATE_VIS array {
+  using __trivially_relocatable = __conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value, array, void>;
+
+  // types:
+  using __self          = array;
+  using value_type      = _Tp;
+  using reference       = value_type&;
+  using const_reference = const value_type&;
+  using pointer         = value_type*;
+  using const_pointer   = const value_type*;
+#if defined(_LIBCPP_ABI_USE_WRAP_ITER_IN_STD_ARRAY)
+  using iterator       = __wrap_iter<pointer>;
+  using const_iterator = __wrap_iter<const_pointer>;
+#else
+  using iterator       = pointer;
+  using const_iterator = const_pointer;
+#endif
+  using size_type              = size_t;
+  using difference_type        = ptrdiff_t;
+  using reverse_iterator       = std::reverse_iterator<iterator>;
+  using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+
+  _Tp __elems_[_Size];
+
+  // No explicit construct/copy/destroy for aggregate type
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void fill(const value_type& __u) {
+    std::fill_n(data(), _Size, __u);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(array& __a) _NOEXCEPT_(__is_nothrow_swappable_v<_Tp>) {
+    std::swap_ranges(data(), data() + _Size, __a.data());
+  }
+
+  // iterators:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator begin() _NOEXCEPT { return iterator(data()); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator begin() const _NOEXCEPT {
+    return const_iterator(data());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator end() _NOEXCEPT { return iterator(data() + _Size); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator end() const _NOEXCEPT {
+    return const_iterator(data() + _Size);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rbegin() _NOEXCEPT {
+    return reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator rbegin() const _NOEXCEPT {
+    return const_reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rend() _NOEXCEPT {
+    return reverse_iterator(begin());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator rend() const _NOEXCEPT {
+    return const_reverse_iterator(begin());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator cbegin() const _NOEXCEPT { return begin(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator cend() const _NOEXCEPT { return end(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator crbegin() const _NOEXCEPT {
+    return rbegin();
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
+
+  // capacity:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_type size() const _NOEXCEPT { return _Size; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_type max_size() const _NOEXCEPT { return _Size; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool empty() const _NOEXCEPT { return _Size == 0; }
+
+  // element access:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference operator[](size_type __n) _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < _Size, "out-of-bounds access in std::array<T, N>");
+    return __elems_[__n];
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference operator[](size_type __n) const _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < _Size, "out-of-bounds access in std::array<T, N>");
+    return __elems_[__n];
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference at(size_type __n) {
+    if (__n >= _Size)
+      __throw_out_of_range("array::at");
+    return __elems_[__n];
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference at(size_type __n) const {
+    if (__n >= _Size)
+      __throw_out_of_range("array::at");
+    return __elems_[__n];
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference front() _NOEXCEPT { return (*this)[0]; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference front() const _NOEXCEPT { return (*this)[0]; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference back() _NOEXCEPT { return (*this)[_Size - 1]; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference back() const _NOEXCEPT {
+    return (*this)[_Size - 1];
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 value_type* data() _NOEXCEPT { return __elems_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const value_type* data() const _NOEXCEPT { return __elems_; }
+};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS array<_Tp, 0> {
+  // types:
+  typedef array __self;
+  typedef _Tp value_type;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+  typedef value_type* iterator;
+  typedef const value_type* const_iterator;
+  typedef value_type* pointer;
+  typedef const value_type* const_pointer;
+  typedef size_t size_type;
+  typedef ptrdiff_t difference_type;
+  typedef std::reverse_iterator<iterator> reverse_iterator;
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+
+  typedef __conditional_t<is_const<_Tp>::value, const __empty, __empty> _EmptyType;
+
+  struct _ArrayInStructT {
+    _Tp __data_[1];
+  };
+  _ALIGNAS_TYPE(_ArrayInStructT) _EmptyType __elems_[sizeof(_ArrayInStructT)];
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 value_type* data() _NOEXCEPT { return nullptr; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const value_type* data() const _NOEXCEPT { return nullptr; }
+
+  // No explicit construct/copy/destroy for aggregate type
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void fill(const value_type&) {
+    static_assert(!is_const<_Tp>::value, "cannot fill zero-sized array of type 'const T'");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(array&) _NOEXCEPT {
+    static_assert(!is_const<_Tp>::value, "cannot swap zero-sized array of type 'const T'");
+  }
+
+  // iterators:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator begin() _NOEXCEPT { return iterator(data()); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator begin() const _NOEXCEPT {
+    return const_iterator(data());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator end() _NOEXCEPT { return iterator(data()); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator end() const _NOEXCEPT {
+    return const_iterator(data());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rbegin() _NOEXCEPT {
+    return reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator rbegin() const _NOEXCEPT {
+    return const_reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rend() _NOEXCEPT {
+    return reverse_iterator(begin());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator rend() const _NOEXCEPT {
+    return const_reverse_iterator(begin());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator cbegin() const _NOEXCEPT { return begin(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator cend() const _NOEXCEPT { return end(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator crbegin() const _NOEXCEPT {
+    return rbegin();
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
+
+  // capacity:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_type size() const _NOEXCEPT { return 0; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_type max_size() const _NOEXCEPT { return 0; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool empty() const _NOEXCEPT { return true; }
+
+  // element access:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference operator[](size_type) _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(false, "cannot call array<T, 0>::operator[] on a zero-sized array");
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference operator[](size_type) const _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(false, "cannot call array<T, 0>::operator[] on a zero-sized array");
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference at(size_type) {
+    __throw_out_of_range("array<T, 0>::at");
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference at(size_type) const {
+    __throw_out_of_range("array<T, 0>::at");
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference front() _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(false, "cannot call array<T, 0>::front() on a zero-sized array");
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference front() const _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(false, "cannot call array<T, 0>::front() on a zero-sized array");
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference back() _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(false, "cannot call array<T, 0>::back() on a zero-sized array");
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference back() const _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(false, "cannot call array<T, 0>::back() on a zero-sized array");
+    __libcpp_unreachable();
+  }
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp, class... _Args, class = enable_if_t<__all<_IsSame<_Tp, _Args>::value...>::value> >
+array(_Tp, _Args...) -> array<_Tp, 1 + sizeof...(_Args)>;
+#endif
+
+template <class _Tp, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+operator==(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) {
+  return std::equal(__x.begin(), __x.end(), __y.begin());
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Tp, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) {
+  return !(__x == __y);
+}
+
+template <class _Tp, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) {
+  return std::lexicographical_compare(__x.begin(), __x.end(), __y.begin(), __y.end());
+}
+
+template <class _Tp, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) {
+  return __y < __x;
+}
+
+template <class _Tp, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) {
+  return !(__y < __x);
+}
+
+template <class _Tp, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>=(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) {
+  return !(__x < __y);
+}
+
+#else // _LIBCPP_STD_VER <= 17
+
+template <class _Tp, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI constexpr __synth_three_way_result<_Tp>
+operator<=>(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) {
+  return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
+}
+
+#endif // _LIBCPP_STD_VER <= 17
+
+template <class _Tp, size_t _Size, __enable_if_t<_Size == 0 || __is_swappable_v<_Tp>, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(array<_Tp, _Size>& __x, array<_Tp, _Size>& __y)
+    _NOEXCEPT_(noexcept(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+template <class _Tp, size_t _Size>
+struct _LIBCPP_TEMPLATE_VIS tuple_size<array<_Tp, _Size> > : public integral_constant<size_t, _Size> {};
+
+template <size_t _Ip, class _Tp, size_t _Size>
+struct _LIBCPP_TEMPLATE_VIS tuple_element<_Ip, array<_Tp, _Size> > {
+  static_assert(_Ip < _Size, "Index out of bounds in std::tuple_element<> (std::array)");
+  typedef _Tp type;
+};
+
+template <size_t _Ip, class _Tp, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp& get(array<_Tp, _Size>& __a) _NOEXCEPT {
+  static_assert(_Ip < _Size, "Index out of bounds in std::get<> (std::array)");
+  return __a.__elems_[_Ip];
+}
+
+template <size_t _Ip, class _Tp, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp& get(const array<_Tp, _Size>& __a) _NOEXCEPT {
+  static_assert(_Ip < _Size, "Index out of bounds in std::get<> (const std::array)");
+  return __a.__elems_[_Ip];
+}
+
+template <size_t _Ip, class _Tp, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp&& get(array<_Tp, _Size>&& __a) _NOEXCEPT {
+  static_assert(_Ip < _Size, "Index out of bounds in std::get<> (std::array &&)");
+  return std::move(__a.__elems_[_Ip]);
+}
+
+template <size_t _Ip, class _Tp, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp&& get(const array<_Tp, _Size>&& __a) _NOEXCEPT {
+  static_assert(_Ip < _Size, "Index out of bounds in std::get<> (const std::array &&)");
+  return std::move(__a.__elems_[_Ip]);
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+template <typename _Tp, size_t _Size, size_t... _Index>
+_LIBCPP_HIDE_FROM_ABI constexpr array<remove_cv_t<_Tp>, _Size>
+__to_array_lvalue_impl(_Tp (&__arr)[_Size], index_sequence<_Index...>) {
+  return {{__arr[_Index]...}};
+}
+
+template <typename _Tp, size_t _Size, size_t... _Index>
+_LIBCPP_HIDE_FROM_ABI constexpr array<remove_cv_t<_Tp>, _Size>
+__to_array_rvalue_impl(_Tp (&&__arr)[_Size], index_sequence<_Index...>) {
+  return {{std::move(__arr[_Index])...}};
+}
+
+template <typename _Tp, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI constexpr array<remove_cv_t<_Tp>, _Size>
+to_array(_Tp (&__arr)[_Size]) noexcept(is_nothrow_constructible_v<_Tp, _Tp&>) {
+  static_assert(!is_array_v<_Tp>, "[array.creation]/1: to_array does not accept multidimensional arrays.");
+  static_assert(is_constructible_v<_Tp, _Tp&>, "[array.creation]/1: to_array requires copy constructible elements.");
+  return std::__to_array_lvalue_impl(__arr, make_index_sequence<_Size>());
+}
+
+template <typename _Tp, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI constexpr array<remove_cv_t<_Tp>, _Size>
+to_array(_Tp (&&__arr)[_Size]) noexcept(is_nothrow_move_constructible_v<_Tp>) {
+  static_assert(!is_array_v<_Tp>, "[array.creation]/4: to_array does not accept multidimensional arrays.");
+  static_assert(is_move_constructible_v<_Tp>, "[array.creation]/4: to_array requires move constructible elements.");
+  return std::__to_array_rvalue_impl(std::move(__arr), make_index_sequence<_Size>());
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <algorithm>
+#  include <concepts>
+#  include <cstdlib>
+#  include <iterator>
+#  include <type_traits>
+#  include <utility>
+#endif
+
+#endif // _LIBCPP_ARRAY
diff --git a/libcxx/include/__cxx03/atomic b/libcxx/include/__cxx03/atomic
new file mode 100644
index 000000000000..0d13619d6ce4
--- /dev/null
+++ b/libcxx/include/__cxx03/atomic
@@ -0,0 +1,632 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_ATOMIC
+#define _LIBCPP_ATOMIC
+
+/*
+    atomic synopsis
+
+namespace std
+{
+
+// feature test macro [version.syn]
+
+#define __cpp_lib_atomic_is_always_lock_free
+#define __cpp_lib_atomic_flag_test
+#define __cpp_lib_atomic_lock_free_type_aliases
+#define __cpp_lib_atomic_wait
+
+ // order and consistency
+
+ enum memory_order: unspecified // enum class in C++20
+ {
+    relaxed,
+    consume, // load-consume
+    acquire, // load-acquire
+    release, // store-release
+    acq_rel, // store-release load-acquire
+    seq_cst // store-release load-acquire
+ };
+
+ inline constexpr auto memory_order_relaxed = memory_order::relaxed;
+ inline constexpr auto memory_order_consume = memory_order::consume;
+ inline constexpr auto memory_order_acquire = memory_order::acquire;
+ inline constexpr auto memory_order_release = memory_order::release;
+ inline constexpr auto memory_order_acq_rel = memory_order::acq_rel;
+ inline constexpr auto memory_order_seq_cst = memory_order::seq_cst;
+
+template <class T> T kill_dependency(T y) noexcept;
+
+// lock-free property
+
+#define ATOMIC_BOOL_LOCK_FREE unspecified
+#define ATOMIC_CHAR_LOCK_FREE unspecified
+#define ATOMIC_CHAR8_T_LOCK_FREE unspecified // C++20
+#define ATOMIC_CHAR16_T_LOCK_FREE unspecified
+#define ATOMIC_CHAR32_T_LOCK_FREE unspecified
+#define ATOMIC_WCHAR_T_LOCK_FREE unspecified
+#define ATOMIC_SHORT_LOCK_FREE unspecified
+#define ATOMIC_INT_LOCK_FREE unspecified
+#define ATOMIC_LONG_LOCK_FREE unspecified
+#define ATOMIC_LLONG_LOCK_FREE unspecified
+#define ATOMIC_POINTER_LOCK_FREE unspecified
+
+template <class T>
+struct atomic
+{
+    using value_type = T;
+
+    static constexpr bool is_always_lock_free;
+    bool is_lock_free() const volatile noexcept;
+    bool is_lock_free() const noexcept;
+
+    atomic() noexcept = default; // until C++20
+    constexpr atomic() noexcept(is_nothrow_default_constructible_v<T>); // since C++20
+    constexpr atomic(T desr) noexcept;
+    atomic(const atomic&) = delete;
+    atomic& operator=(const atomic&) = delete;
+    atomic& operator=(const atomic&) volatile = delete;
+
+    T load(memory_order m = memory_order_seq_cst) const volatile noexcept;
+    T load(memory_order m = memory_order_seq_cst) const noexcept;
+    operator T() const volatile noexcept;
+    operator T() const noexcept;
+    void store(T desr, memory_order m = memory_order_seq_cst) volatile noexcept;
+    void store(T desr, memory_order m = memory_order_seq_cst) noexcept;
+    T operator=(T) volatile noexcept;
+    T operator=(T) noexcept;
+
+    T exchange(T desr, memory_order m = memory_order_seq_cst) volatile noexcept;
+    T exchange(T desr, memory_order m = memory_order_seq_cst) noexcept;
+    bool compare_exchange_weak(T& expc, T desr,
+                               memory_order s, memory_order f) volatile noexcept;
+    bool compare_exchange_weak(T& expc, T desr, memory_order s, memory_order f) noexcept;
+    bool compare_exchange_strong(T& expc, T desr,
+                                 memory_order s, memory_order f) volatile noexcept;
+    bool compare_exchange_strong(T& expc, T desr,
+                                 memory_order s, memory_order f) noexcept;
+    bool compare_exchange_weak(T& expc, T desr,
+                               memory_order m = memory_order_seq_cst) volatile noexcept;
+    bool compare_exchange_weak(T& expc, T desr,
+                               memory_order m = memory_order_seq_cst) noexcept;
+    bool compare_exchange_strong(T& expc, T desr,
+                                memory_order m = memory_order_seq_cst) volatile noexcept;
+    bool compare_exchange_strong(T& expc, T desr,
+                                 memory_order m = memory_order_seq_cst) noexcept;
+
+    void wait(T, memory_order = memory_order::seq_cst) const volatile noexcept;
+    void wait(T, memory_order = memory_order::seq_cst) const noexcept;
+    void notify_one() volatile noexcept;
+    void notify_one() noexcept;
+    void notify_all() volatile noexcept;
+    void notify_all() noexcept;
+};
+
+template <>
+struct atomic<integral>
+{
+    using value_type = integral;
+    using difference_type = value_type;
+
+    static constexpr bool is_always_lock_free;
+    bool is_lock_free() const volatile noexcept;
+    bool is_lock_free() const noexcept;
+
+    atomic() noexcept = default;
+    constexpr atomic(integral desr) noexcept;
+    atomic(const atomic&) = delete;
+    atomic& operator=(const atomic&) = delete;
+    atomic& operator=(const atomic&) volatile = delete;
+
+    integral load(memory_order m = memory_order_seq_cst) const volatile noexcept;
+    integral load(memory_order m = memory_order_seq_cst) const noexcept;
+    operator integral() const volatile noexcept;
+    operator integral() const noexcept;
+    void store(integral desr, memory_order m = memory_order_seq_cst) volatile noexcept;
+    void store(integral desr, memory_order m = memory_order_seq_cst) noexcept;
+    integral operator=(integral desr) volatile noexcept;
+    integral operator=(integral desr) noexcept;
+
+    integral exchange(integral desr,
+                      memory_order m = memory_order_seq_cst) volatile noexcept;
+    integral exchange(integral desr, memory_order m = memory_order_seq_cst) noexcept;
+    bool compare_exchange_weak(integral& expc, integral desr,
+                               memory_order s, memory_order f) volatile noexcept;
+    bool compare_exchange_weak(integral& expc, integral desr,
+                               memory_order s, memory_order f) noexcept;
+    bool compare_exchange_strong(integral& expc, integral desr,
+                                 memory_order s, memory_order f) volatile noexcept;
+    bool compare_exchange_strong(integral& expc, integral desr,
+                                 memory_order s, memory_order f) noexcept;
+    bool compare_exchange_weak(integral& expc, integral desr,
+                               memory_order m = memory_order_seq_cst) volatile noexcept;
+    bool compare_exchange_weak(integral& expc, integral desr,
+                               memory_order m = memory_order_seq_cst) noexcept;
+    bool compare_exchange_strong(integral& expc, integral desr,
+                                memory_order m = memory_order_seq_cst) volatile noexcept;
+    bool compare_exchange_strong(integral& expc, integral desr,
+                                 memory_order m = memory_order_seq_cst) noexcept;
+
+    integral fetch_add(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
+    integral fetch_add(integral op, memory_order m = memory_order_seq_cst) noexcept;
+    integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
+    integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) noexcept;
+    integral fetch_and(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
+    integral fetch_and(integral op, memory_order m = memory_order_seq_cst) noexcept;
+    integral fetch_or(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
+    integral fetch_or(integral op, memory_order m = memory_order_seq_cst) noexcept;
+    integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
+    integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) noexcept;
+
+    integral operator++(int) volatile noexcept;
+    integral operator++(int) noexcept;
+    integral operator--(int) volatile noexcept;
+    integral operator--(int) noexcept;
+    integral operator++() volatile noexcept;
+    integral operator++() noexcept;
+    integral operator--() volatile noexcept;
+    integral operator--() noexcept;
+    integral operator+=(integral op) volatile noexcept;
+    integral operator+=(integral op) noexcept;
+    integral operator-=(integral op) volatile noexcept;
+    integral operator-=(integral op) noexcept;
+    integral operator&=(integral op) volatile noexcept;
+    integral operator&=(integral op) noexcept;
+    integral operator|=(integral op) volatile noexcept;
+    integral operator|=(integral op) noexcept;
+    integral operator^=(integral op) volatile noexcept;
+    integral operator^=(integral op) noexcept;
+
+    void wait(integral, memory_order = memory_order::seq_cst) const volatile noexcept;
+    void wait(integral, memory_order = memory_order::seq_cst) const noexcept;
+    void notify_one() volatile noexcept;
+    void notify_one() noexcept;
+    void notify_all() volatile noexcept;
+    void notify_all() noexcept;
+};
+
+template <class T>
+struct atomic<T*>
+{
+    using value_type = T*;
+    using difference_type = ptrdiff_t;
+
+    static constexpr bool is_always_lock_free;
+    bool is_lock_free() const volatile noexcept;
+    bool is_lock_free() const noexcept;
+
+    atomic() noexcept = default; // until C++20
+    constexpr atomic() noexcept; // since C++20
+    constexpr atomic(T* desr) noexcept;
+    atomic(const atomic&) = delete;
+    atomic& operator=(const atomic&) = delete;
+    atomic& operator=(const atomic&) volatile = delete;
+
+    T* load(memory_order m = memory_order_seq_cst) const volatile noexcept;
+    T* load(memory_order m = memory_order_seq_cst) const noexcept;
+    operator T*() const volatile noexcept;
+    operator T*() const noexcept;
+    void store(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept;
+    void store(T* desr, memory_order m = memory_order_seq_cst) noexcept;
+    T* operator=(T*) volatile noexcept;
+    T* operator=(T*) noexcept;
+
+    T* exchange(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept;
+    T* exchange(T* desr, memory_order m = memory_order_seq_cst) noexcept;
+    bool compare_exchange_weak(T*& expc, T* desr,
+                               memory_order s, memory_order f) volatile noexcept;
+    bool compare_exchange_weak(T*& expc, T* desr,
+                               memory_order s, memory_order f) noexcept;
+    bool compare_exchange_strong(T*& expc, T* desr,
+                                 memory_order s, memory_order f) volatile noexcept;
+    bool compare_exchange_strong(T*& expc, T* desr,
+                                 memory_order s, memory_order f) noexcept;
+    bool compare_exchange_weak(T*& expc, T* desr,
+                               memory_order m = memory_order_seq_cst) volatile noexcept;
+    bool compare_exchange_weak(T*& expc, T* desr,
+                               memory_order m = memory_order_seq_cst) noexcept;
+    bool compare_exchange_strong(T*& expc, T* desr,
+                                memory_order m = memory_order_seq_cst) volatile noexcept;
+    bool compare_exchange_strong(T*& expc, T* desr,
+                                 memory_order m = memory_order_seq_cst) noexcept;
+    T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept;
+    T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept;
+    T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept;
+    T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept;
+
+    T* operator++(int) volatile noexcept;
+    T* operator++(int) noexcept;
+    T* operator--(int) volatile noexcept;
+    T* operator--(int) noexcept;
+    T* operator++() volatile noexcept;
+    T* operator++() noexcept;
+    T* operator--() volatile noexcept;
+    T* operator--() noexcept;
+    T* operator+=(ptrdiff_t op) volatile noexcept;
+    T* operator+=(ptrdiff_t op) noexcept;
+    T* operator-=(ptrdiff_t op) volatile noexcept;
+    T* operator-=(ptrdiff_t op) noexcept;
+
+    void wait(T*, memory_order = memory_order::seq_cst) const volatile noexcept;
+    void wait(T*, memory_order = memory_order::seq_cst) const noexcept;
+    void notify_one() volatile noexcept;
+    void notify_one() noexcept;
+    void notify_all() volatile noexcept;
+    void notify_all() noexcept;
+};
+
+template<>
+struct atomic<floating-point-type> {  // since C++20
+  using value_type = floating-point-type;
+  using difference_type = value_type;
+
+  static constexpr bool is_always_lock_free = implementation-defined;
+  bool is_lock_free() const volatile noexcept;
+  bool is_lock_free() const noexcept;
+
+  constexpr atomic() noexcept;
+  constexpr atomic(floating-point-type) noexcept;
+  atomic(const atomic&) = delete;
+  atomic& operator=(const atomic&) = delete;
+  atomic& operator=(const atomic&) volatile = delete;
+
+  void store(floating-point-type, memory_order = memory_order::seq_cst) volatile noexcept;
+  void store(floating-point-type, memory_order = memory_order::seq_cst) noexcept;
+  floating-point-type operator=(floating-point-type) volatile noexcept;
+  floating-point-type operator=(floating-point-type) noexcept;
+  floating-point-type load(memory_order = memory_order::seq_cst) volatile noexcept;
+  floating-point-type load(memory_order = memory_order::seq_cst) noexcept;
+  operator floating-point-type() volatile noexcept;
+  operator floating-point-type() noexcept;
+
+  floating-point-type exchange(floating-point-type,
+                               memory_order = memory_order::seq_cst) volatile noexcept;
+  floating-point-type exchange(floating-point-type,
+                               memory_order = memory_order::seq_cst) noexcept;
+  bool compare_exchange_weak(floating-point-type&, floating-point-type,
+                             memory_order, memory_order) volatile noexcept;
+  bool compare_exchange_weak(floating-point-type&, floating-point-type,
+                             memory_order, memory_order) noexcept;
+  bool compare_exchange_strong(floating-point-type&, floating-point-type,
+                               memory_order, memory_order) volatile noexcept;
+  bool compare_exchange_strong(floating-point-type&, floating-point-type,
+                               memory_order, memory_order) noexcept;
+  bool compare_exchange_weak(floating-point-type&, floating-point-type,
+                             memory_order = memory_order::seq_cst) volatile noexcept;
+  bool compare_exchange_weak(floating-point-type&, floating-point-type,
+                             memory_order = memory_order::seq_cst) noexcept;
+  bool compare_exchange_strong(floating-point-type&, floating-point-type,
+                               memory_order = memory_order::seq_cst) volatile noexcept;
+  bool compare_exchange_strong(floating-point-type&, floating-point-type,
+                               memory_order = memory_order::seq_cst) noexcept;
+
+  floating-point-type fetch_add(floating-point-type,
+                                memory_order = memory_order::seq_cst) volatile noexcept;
+  floating-point-type fetch_add(floating-point-type,
+                                memory_order = memory_order::seq_cst) noexcept;
+  floating-point-type fetch_sub(floating-point-type,
+                                memory_order = memory_order::seq_cst) volatile noexcept;
+  floating-point-type fetch_sub(floating-point-type,
+                                memory_order = memory_order::seq_cst) noexcept;
+
+  floating-point-type operator+=(floating-point-type) volatile noexcept;
+  floating-point-type operator+=(floating-point-type) noexcept;
+  floating-point-type operator-=(floating-point-type) volatile noexcept;
+  floating-point-type operator-=(floating-point-type) noexcept;
+
+  void wait(floating-point-type, memory_order = memory_order::seq_cst) const volatile noexcept;
+  void wait(floating-point-type, memory_order = memory_order::seq_cst) const noexcept;
+  void notify_one() volatile noexcept;
+  void notify_one() noexcept;
+  void notify_all() volatile noexcept;
+  void notify_all() noexcept;
+};
+
+// [atomics.nonmembers], non-member functions
+template<class T>
+  bool atomic_is_lock_free(const volatile atomic<T>*) noexcept;
+template<class T>
+  bool atomic_is_lock_free(const atomic<T>*) noexcept;
+template<class T>
+  void atomic_store(volatile atomic<T>*, atomic<T>::value_type) noexcept;
+template<class T>
+  void atomic_store(atomic<T>*, atomic<T>::value_type) noexcept;
+template<class T>
+  void atomic_store_explicit(volatile atomic<T>*, atomic<T>::value_type,
+                             memory_order) noexcept;
+template<class T>
+  void atomic_store_explicit(atomic<T>*, atomic<T>::value_type,
+                             memory_order) noexcept;
+template<class T>
+  T atomic_load(const volatile atomic<T>*) noexcept;
+template<class T>
+  T atomic_load(const atomic<T>*) noexcept;
+template<class T>
+  T atomic_load_explicit(const volatile atomic<T>*, memory_order) noexcept;
+template<class T>
+  T atomic_load_explicit(const atomic<T>*, memory_order) noexcept;
+template<class T>
+  T atomic_exchange(volatile atomic<T>*, atomic<T>::value_type) noexcept;
+template<class T>
+  T atomic_exchange(atomic<T>*, atomic<T>::value_type) noexcept;
+template<class T>
+  T atomic_exchange_explicit(volatile atomic<T>*, atomic<T>::value_type,
+                             memory_order) noexcept;
+template<class T>
+  T atomic_exchange_explicit(atomic<T>*, atomic<T>::value_type,
+                             memory_order) noexcept;
+template<class T>
+  bool atomic_compare_exchange_weak(volatile atomic<T>*, atomic<T>::value_type*,
+                                    atomic<T>::value_type) noexcept;
+template<class T>
+  bool atomic_compare_exchange_weak(atomic<T>*, atomic<T>::value_type*,
+                                    atomic<T>::value_type) noexcept;
+template<class T>
+  bool atomic_compare_exchange_strong(volatile atomic<T>*, atomic<T>::value_type*,
+                                      atomic<T>::value_type) noexcept;
+template<class T>
+  bool atomic_compare_exchange_strong(atomic<T>*, atomic<T>::value_type*,
+                                      atomic<T>::value_type) noexcept;
+template<class T>
+  bool atomic_compare_exchange_weak_explicit(volatile atomic<T>*, atomic<T>::value_type*,
+                                             atomic<T>::value_type,
+                                             memory_order, memory_order) noexcept;
+template<class T>
+  bool atomic_compare_exchange_weak_explicit(atomic<T>*, atomic<T>::value_type*,
+                                             atomic<T>::value_type,
+                                             memory_order, memory_order) noexcept;
+template<class T>
+  bool atomic_compare_exchange_strong_explicit(volatile atomic<T>*, atomic<T>::value_type*,
+                                               atomic<T>::value_type,
+                                               memory_order, memory_order) noexcept;
+template<class T>
+  bool atomic_compare_exchange_strong_explicit(atomic<T>*, atomic<T>::value_type*,
+                                               atomic<T>::value_type,
+                                               memory_order, memory_order) noexcept;
+
+template<class T>
+  T atomic_fetch_add(volatile atomic<T>*, atomic<T>::difference_type) noexcept;
+template<class T>
+  T atomic_fetch_add(atomic<T>*, atomic<T>::difference_type) noexcept;
+template<class T>
+  T atomic_fetch_add_explicit(volatile atomic<T>*, atomic<T>::difference_type,
+                              memory_order) noexcept;
+template<class T>
+  T atomic_fetch_add_explicit(atomic<T>*, atomic<T>::difference_type,
+                              memory_order) noexcept;
+template<class T>
+  T atomic_fetch_sub(volatile atomic<T>*, atomic<T>::difference_type) noexcept;
+template<class T>
+  T atomic_fetch_sub(atomic<T>*, atomic<T>::difference_type) noexcept;
+template<class T>
+  T atomic_fetch_sub_explicit(volatile atomic<T>*, atomic<T>::difference_type,
+                              memory_order) noexcept;
+template<class T>
+  T atomic_fetch_sub_explicit(atomic<T>*, atomic<T>::difference_type,
+                              memory_order) noexcept;
+template<class T>
+  T atomic_fetch_and(volatile atomic<T>*, atomic<T>::value_type) noexcept;
+template<class T>
+  T atomic_fetch_and(atomic<T>*, atomic<T>::value_type) noexcept;
+template<class T>
+  T atomic_fetch_and_explicit(volatile atomic<T>*, atomic<T>::value_type,
+                              memory_order) noexcept;
+template<class T>
+  T atomic_fetch_and_explicit(atomic<T>*, atomic<T>::value_type,
+                              memory_order) noexcept;
+template<class T>
+  T atomic_fetch_or(volatile atomic<T>*, atomic<T>::value_type) noexcept;
+template<class T>
+  T atomic_fetch_or(atomic<T>*, atomic<T>::value_type) noexcept;
+template<class T>
+  T atomic_fetch_or_explicit(volatile atomic<T>*, atomic<T>::value_type,
+                             memory_order) noexcept;
+template<class T>
+  T atomic_fetch_or_explicit(atomic<T>*, atomic<T>::value_type,
+                             memory_order) noexcept;
+template<class T>
+  T atomic_fetch_xor(volatile atomic<T>*, atomic<T>::value_type) noexcept;
+template<class T>
+  T atomic_fetch_xor(atomic<T>*, atomic<T>::value_type) noexcept;
+template<class T>
+  T atomic_fetch_xor_explicit(volatile atomic<T>*, atomic<T>::value_type,
+                              memory_order) noexcept;
+template<class T>
+  T atomic_fetch_xor_explicit(atomic<T>*, atomic<T>::value_type,
+                              memory_order) noexcept;
+
+template<class T>
+  void atomic_wait(const volatile atomic<T>*, atomic<T>::value_type) noexcept;
+template<class T>
+  void atomic_wait(const atomic<T>*, atomic<T>::value_type) noexcept;
+template<class T>
+  void atomic_wait_explicit(const volatile atomic<T>*, atomic<T>::value_type,
+                            memory_order) noexcept;
+template<class T>
+  void atomic_wait_explicit(const atomic<T>*, atomic<T>::value_type,
+                            memory_order) noexcept;
+template<class T>
+  void atomic_notify_one(volatile atomic<T>*) noexcept;
+template<class T>
+  void atomic_notify_one(atomic<T>*) noexcept;
+template<class T>
+  void atomic_notify_all(volatile atomic<T>*) noexcept;
+template<class T>
+  void atomic_notify_all(atomic<T>*) noexcept;
+
+// Atomics for standard typedef types
+
+typedef atomic<bool>               atomic_bool;
+typedef atomic<char>               atomic_char;
+typedef atomic<signed char>        atomic_schar;
+typedef atomic<unsigned char>      atomic_uchar;
+typedef atomic<short>              atomic_short;
+typedef atomic<unsigned short>     atomic_ushort;
+typedef atomic<int>                atomic_int;
+typedef atomic<unsigned int>       atomic_uint;
+typedef atomic<long>               atomic_long;
+typedef atomic<unsigned long>      atomic_ulong;
+typedef atomic<long long>          atomic_llong;
+typedef atomic<unsigned long long> atomic_ullong;
+typedef atomic<char8_t>            atomic_char8_t; // C++20
+typedef atomic<char16_t>           atomic_char16_t;
+typedef atomic<char32_t>           atomic_char32_t;
+typedef atomic<wchar_t>            atomic_wchar_t;
+
+typedef atomic<int_least8_t>   atomic_int_least8_t;
+typedef atomic<uint_least8_t>  atomic_uint_least8_t;
+typedef atomic<int_least16_t>  atomic_int_least16_t;
+typedef atomic<uint_least16_t> atomic_uint_least16_t;
+typedef atomic<int_least32_t>  atomic_int_least32_t;
+typedef atomic<uint_least32_t> atomic_uint_least32_t;
+typedef atomic<int_least64_t>  atomic_int_least64_t;
+typedef atomic<uint_least64_t> atomic_uint_least64_t;
+
+typedef atomic<int_fast8_t>   atomic_int_fast8_t;
+typedef atomic<uint_fast8_t>  atomic_uint_fast8_t;
+typedef atomic<int_fast16_t>  atomic_int_fast16_t;
+typedef atomic<uint_fast16_t> atomic_uint_fast16_t;
+typedef atomic<int_fast32_t>  atomic_int_fast32_t;
+typedef atomic<uint_fast32_t> atomic_uint_fast32_t;
+typedef atomic<int_fast64_t>  atomic_int_fast64_t;
+typedef atomic<uint_fast64_t> atomic_uint_fast64_t;
+
+typedef atomic<int8_t>   atomic_int8_t;
+typedef atomic<uint8_t>  atomic_uint8_t;
+typedef atomic<int16_t>  atomic_int16_t;
+typedef atomic<uint16_t> atomic_uint16_t;
+typedef atomic<int32_t>  atomic_int32_t;
+typedef atomic<uint32_t> atomic_uint32_t;
+typedef atomic<int64_t>  atomic_int64_t;
+typedef atomic<uint64_t> atomic_uint64_t;
+
+typedef atomic<intptr_t>  atomic_intptr_t;
+typedef atomic<uintptr_t> atomic_uintptr_t;
+typedef atomic<size_t>    atomic_size_t;
+typedef atomic<ptrdiff_t> atomic_ptrdiff_t;
+typedef atomic<intmax_t>  atomic_intmax_t;
+typedef atomic<uintmax_t> atomic_uintmax_t;
+
+typedef see-below         atomic_signed_lock_free;   // since C++20
+typedef see-below         atomic_unsigned_lock_free; // since C++20
+
+// flag type and operations
+
+typedef struct atomic_flag
+{
+    atomic_flag() noexcept = default; // until C++20
+    constexpr atomic_flag() noexcept; // since C++20
+    atomic_flag(const atomic_flag&) = delete;
+    atomic_flag& operator=(const atomic_flag&) = delete;
+    atomic_flag& operator=(const atomic_flag&) volatile = delete;
+
+    bool test(memory_order m = memory_order_seq_cst) volatile noexcept;
+    bool test(memory_order m = memory_order_seq_cst) noexcept;
+    bool test_and_set(memory_order m = memory_order_seq_cst) volatile noexcept;
+    bool test_and_set(memory_order m = memory_order_seq_cst) noexcept;
+    void clear(memory_order m = memory_order_seq_cst) volatile noexcept;
+    void clear(memory_order m = memory_order_seq_cst) noexcept;
+
+    void wait(bool, memory_order = memory_order::seq_cst) const volatile noexcept;
+    void wait(bool, memory_order = memory_order::seq_cst) const noexcept;
+    void notify_one() volatile noexcept;
+    void notify_one() noexcept;
+    void notify_all() volatile noexcept;
+    void notify_all() noexcept;
+} atomic_flag;
+
+bool atomic_flag_test(volatile atomic_flag* obj) noexcept;
+bool atomic_flag_test(atomic_flag* obj) noexcept;
+bool atomic_flag_test_explicit(volatile atomic_flag* obj,
+                               memory_order m) noexcept;
+bool atomic_flag_test_explicit(atomic_flag* obj, memory_order m) noexcept;
+bool atomic_flag_test_and_set(volatile atomic_flag* obj) noexcept;
+bool atomic_flag_test_and_set(atomic_flag* obj) noexcept;
+bool atomic_flag_test_and_set_explicit(volatile atomic_flag* obj,
+                                       memory_order m) noexcept;
+bool atomic_flag_test_and_set_explicit(atomic_flag* obj, memory_order m) noexcept;
+void atomic_flag_clear(volatile atomic_flag* obj) noexcept;
+void atomic_flag_clear(atomic_flag* obj) noexcept;
+void atomic_flag_clear_explicit(volatile atomic_flag* obj, memory_order m) noexcept;
+void atomic_flag_clear_explicit(atomic_flag* obj, memory_order m) noexcept;
+
+void atomic_wait(const volatile atomic_flag* obj, T old) noexcept;
+void atomic_wait(const atomic_flag* obj, T old) noexcept;
+void atomic_wait_explicit(const volatile atomic_flag* obj, T old, memory_order m) noexcept;
+void atomic_wait_explicit(const atomic_flag* obj, T old, memory_order m) noexcept;
+void atomic_one(volatile atomic_flag* obj) noexcept;
+void atomic_one(atomic_flag* obj) noexcept;
+void atomic_all(volatile atomic_flag* obj) noexcept;
+void atomic_all(atomic_flag* obj) noexcept;
+
+// fences
+
+void atomic_thread_fence(memory_order m) noexcept;
+void atomic_signal_fence(memory_order m) noexcept;
+
+// deprecated
+
+template <class T>
+  void atomic_init(volatile atomic<T>* obj, atomic<T>::value_type desr) noexcept;
+
+template <class T>
+  void atomic_init(atomic<T>* obj, atomic<T>::value_type desr) noexcept;
+
+#define ATOMIC_VAR_INIT(value) see below
+
+#define ATOMIC_FLAG_INIT see below
+
+}  // std
+
+*/
+
+#include <__config>
+
+#if _LIBCPP_STD_VER < 23 && defined(_LIBCPP_STDATOMIC_H)
+#  error <atomic> is incompatible with <stdatomic.h> before C++23. Please compile with -std=c++23.
+#endif
+
+#include <__atomic/aliases.h>
+#include <__atomic/atomic.h>
+#include <__atomic/atomic_base.h>
+#include <__atomic/atomic_flag.h>
+#include <__atomic/atomic_init.h>
+#include <__atomic/atomic_lock_free.h>
+#include <__atomic/atomic_sync.h>
+#include <__atomic/check_memory_order.h>
+#include <__atomic/contention_t.h>
+#include <__atomic/cxx_atomic_impl.h>
+#include <__atomic/fence.h>
+#include <__atomic/is_always_lock_free.h>
+#include <__atomic/kill_dependency.h>
+#include <__atomic/memory_order.h>
+#include <version>
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__atomic/atomic_ref.h>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#ifdef _LIBCPP_HAS_NO_ATOMIC_HEADER
+#  error <atomic> is not implemented
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cmath>
+#  include <compare>
+#  include <cstdlib>
+#  include <cstring>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_ATOMIC
diff --git a/libcxx/include/__cxx03/barrier b/libcxx/include/__cxx03/barrier
new file mode 100644
index 000000000000..edee181273e2
--- /dev/null
+++ b/libcxx/include/__cxx03/barrier
@@ -0,0 +1,308 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_BARRIER
+#define _LIBCPP_BARRIER
+
+/*
+    barrier synopsis
+
+namespace std
+{
+
+  template<class CompletionFunction = see below>
+  class barrier
+  {
+  public:
+    using arrival_token = see below;
+
+    static constexpr ptrdiff_t max() noexcept;
+
+    constexpr explicit barrier(ptrdiff_t phase_count,
+                               CompletionFunction f = CompletionFunction());
+    ~barrier();
+
+    barrier(const barrier&) = delete;
+    barrier& operator=(const barrier&) = delete;
+
+    [[nodiscard]] arrival_token arrive(ptrdiff_t update = 1);
+    void wait(arrival_token&& arrival) const;
+
+    void arrive_and_wait();
+    void arrive_and_drop();
+
+  private:
+    CompletionFunction completion; // exposition only
+  };
+
+}
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+
+#  include <__assert>
+#  include <__atomic/atomic_base.h>
+#  include <__atomic/memory_order.h>
+#  include <__memory/unique_ptr.h>
+#  include <__thread/poll_with_backoff.h>
+#  include <__thread/timed_backoff_policy.h>
+#  include <__utility/move.h>
+#  include <cstddef>
+#  include <cstdint>
+#  include <limits>
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+#  if _LIBCPP_STD_VER >= 14
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct __empty_completion {
+  inline _LIBCPP_HIDE_FROM_ABI void operator()() noexcept {}
+};
+
+#    ifndef _LIBCPP_HAS_NO_TREE_BARRIER
+
+/*
+
+The default implementation of __barrier_base is a classic tree barrier.
+
+It looks different from literature pseudocode for two main reasons:
+ 1. Threads that call into std::barrier functions do not provide indices,
+    so a numbering step is added before the actual barrier algorithm,
+    appearing as an N+1 round to the N rounds of the tree barrier.
+ 2. A great deal of attention has been paid to avoid cache line thrashing
+    by flattening the tree structure into cache-line sized arrays, that
+    are indexed in an efficient way.
+
+*/
+
+using __barrier_phase_t = uint8_t;
+
+class __barrier_algorithm_base;
+
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __barrier_algorithm_base*
+__construct_barrier_algorithm_base(ptrdiff_t& __expected);
+
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI bool
+__arrive_barrier_algorithm_base(__barrier_algorithm_base* __barrier, __barrier_phase_t __old_phase) noexcept;
+
+_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
+__destroy_barrier_algorithm_base(__barrier_algorithm_base* __barrier) noexcept;
+
+template <class _CompletionF>
+class __barrier_base {
+  ptrdiff_t __expected_;
+  unique_ptr<__barrier_algorithm_base, void (*)(__barrier_algorithm_base*)> __base_;
+  __atomic_base<ptrdiff_t> __expected_adjustment_;
+  _CompletionF __completion_;
+  __atomic_base<__barrier_phase_t> __phase_;
+
+public:
+  using arrival_token = __barrier_phase_t;
+
+  static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { return numeric_limits<ptrdiff_t>::max(); }
+
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI
+  __barrier_base(ptrdiff_t __expected, _CompletionF __completion = _CompletionF())
+      : __expected_(__expected),
+        __base_(std::__construct_barrier_algorithm_base(this->__expected_), &__destroy_barrier_algorithm_base),
+        __expected_adjustment_(0),
+        __completion_(std::move(__completion)),
+        __phase_(0) {}
+  _LIBCPP_NODISCARD _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI arrival_token arrive(ptrdiff_t __update) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __update <= __expected_, "update is greater than the expected count for the current barrier phase");
+
+    auto const __old_phase = __phase_.load(memory_order_relaxed);
+    for (; __update; --__update)
+      if (__arrive_barrier_algorithm_base(__base_.get(), __old_phase)) {
+        __completion_();
+        __expected_ += __expected_adjustment_.load(memory_order_relaxed);
+        __expected_adjustment_.store(0, memory_order_relaxed);
+        __phase_.store(__old_phase + 2, memory_order_release);
+        __phase_.notify_all();
+      }
+    return __old_phase;
+  }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(arrival_token&& __old_phase) const {
+    auto const __test_fn = [this, __old_phase]() -> bool { return __phase_.load(memory_order_acquire) != __old_phase; };
+    std::__libcpp_thread_poll_with_backoff(__test_fn, __libcpp_timed_backoff_policy());
+  }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_drop() {
+    __expected_adjustment_.fetch_sub(1, memory_order_relaxed);
+    (void)arrive(1);
+  }
+};
+
+#    else
+
+/*
+
+The alternative implementation of __barrier_base is a central barrier.
+
+Two versions of this algorithm are provided:
+ 1. A fairly straightforward implementation of the litterature for the
+    general case where the completion function is not empty.
+ 2. An optimized implementation that exploits 2's complement arithmetic
+    and well-defined overflow in atomic arithmetic, to handle the phase
+    roll-over for free.
+
+*/
+
+template <class _CompletionF>
+class __barrier_base {
+  __atomic_base<ptrdiff_t> __expected;
+  __atomic_base<ptrdiff_t> __arrived;
+  _CompletionF __completion;
+  __atomic_base<bool> __phase;
+
+public:
+  using arrival_token = bool;
+
+  static constexpr ptrdiff_t max() noexcept { return numeric_limits<ptrdiff_t>::max(); }
+
+  _LIBCPP_HIDE_FROM_ABI __barrier_base(ptrdiff_t __expected, _CompletionF __completion = _CompletionF())
+      : __expected(__expected), __arrived(__expected), __completion(std::move(__completion)), __phase(false) {}
+  [[nodiscard]] _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI arrival_token arrive(ptrdiff_t update) {
+    auto const __old_phase  = __phase.load(memory_order_relaxed);
+    auto const __result     = __arrived.fetch_sub(update, memory_order_acq_rel) - update;
+    auto const new_expected = __expected.load(memory_order_relaxed);
+
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        update <= new_expected, "update is greater than the expected count for the current barrier phase");
+
+    if (0 == __result) {
+      __completion();
+      __arrived.store(new_expected, memory_order_relaxed);
+      __phase.store(!__old_phase, memory_order_release);
+      __phase.notify_all();
+    }
+    return __old_phase;
+  }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(arrival_token&& __old_phase) const {
+    __phase.wait(__old_phase, memory_order_acquire);
+  }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_drop() {
+    __expected.fetch_sub(1, memory_order_relaxed);
+    (void)arrive(1);
+  }
+};
+
+template <>
+class __barrier_base<__empty_completion> {
+  static constexpr uint64_t __expected_unit = 1ull;
+  static constexpr uint64_t __arrived_unit  = 1ull << 32;
+  static constexpr uint64_t __expected_mask = __arrived_unit - 1;
+  static constexpr uint64_t __phase_bit     = 1ull << 63;
+  static constexpr uint64_t __arrived_mask  = (__phase_bit - 1) & ~__expected_mask;
+
+  __atomic_base<uint64_t> __phase_arrived_expected;
+
+  static _LIBCPP_HIDE_FROM_ABI constexpr uint64_t __init(ptrdiff_t __count) _NOEXCEPT {
+    return ((uint64_t(1u << 31) - __count) << 32) | (uint64_t(1u << 31) - __count);
+  }
+
+public:
+  using arrival_token = uint64_t;
+
+  static constexpr ptrdiff_t max() noexcept { return ptrdiff_t(1u << 31) - 1; }
+
+  _LIBCPP_HIDE_FROM_ABI explicit inline __barrier_base(ptrdiff_t __count, __empty_completion = __empty_completion())
+      : __phase_arrived_expected(__init(__count)) {}
+  [[nodiscard]] inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI arrival_token arrive(ptrdiff_t update) {
+    auto const __inc = __arrived_unit * update;
+    auto const __old = __phase_arrived_expected.fetch_add(__inc, memory_order_acq_rel);
+
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        update <= __old, "update is greater than the expected count for the current barrier phase");
+
+    if ((__old ^ (__old + __inc)) & __phase_bit) {
+      __phase_arrived_expected.fetch_add((__old & __expected_mask) << 32, memory_order_relaxed);
+      __phase_arrived_expected.notify_all();
+    }
+    return __old & __phase_bit;
+  }
+  inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(arrival_token&& __phase) const {
+    auto const __test_fn = [=]() -> bool {
+      uint64_t const __current = __phase_arrived_expected.load(memory_order_acquire);
+      return ((__current & __phase_bit) != __phase);
+    };
+    __libcpp_thread_poll_with_backoff(__test_fn, __libcpp_timed_backoff_policy());
+  }
+  inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_drop() {
+    __phase_arrived_expected.fetch_add(__expected_unit, memory_order_relaxed);
+    (void)arrive(1);
+  }
+};
+
+#    endif // !_LIBCPP_HAS_NO_TREE_BARRIER
+
+template <class _CompletionF = __empty_completion>
+class _LIBCPP_DEPRECATED_ATOMIC_SYNC barrier {
+  __barrier_base<_CompletionF> __b_;
+
+public:
+  using arrival_token = typename __barrier_base<_CompletionF>::arrival_token;
+
+  static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { return __barrier_base<_CompletionF>::max(); }
+
+  _LIBCPP_AVAILABILITY_SYNC
+  _LIBCPP_HIDE_FROM_ABI explicit barrier(ptrdiff_t __count, _CompletionF __completion = _CompletionF())
+      : __b_(__count, std::move(__completion)) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __count >= 0,
+        "barrier::barrier(ptrdiff_t, CompletionFunction): barrier cannot be initialized with a negative value");
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __count <= max(),
+        "barrier::barrier(ptrdiff_t, CompletionFunction): barrier cannot be initialized with "
+        "a value greater than max()");
+  }
+
+  barrier(barrier const&)            = delete;
+  barrier& operator=(barrier const&) = delete;
+
+  _LIBCPP_NODISCARD _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI arrival_token arrive(ptrdiff_t __update = 1) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__update > 0, "barrier:arrive must be called with a value greater than 0");
+    return __b_.arrive(__update);
+  }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(arrival_token&& __phase) const {
+    __b_.wait(std::move(__phase));
+  }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_wait() { wait(arrive()); }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_drop() { __b_.arrive_and_drop(); }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#  endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_POP_MACROS
+
+#endif // !defined(_LIBCPP_HAS_NO_THREADS)
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <concepts>
+#  include <iterator>
+#  include <memory>
+#  include <stdexcept>
+#  include <variant>
+#endif
+
+#endif //_LIBCPP_BARRIER
diff --git a/libcxx/include/__cxx03/bit b/libcxx/include/__cxx03/bit
new file mode 100644
index 000000000000..94387d101a39
--- /dev/null
+++ b/libcxx/include/__cxx03/bit
@@ -0,0 +1,101 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_BIT
+#define _LIBCPP_BIT
+
+/*
+    bit synopsis
+
+namespace std {
+  // [bit.cast], bit_cast
+  template<class To, class From>
+    constexpr To bit_cast(const From& from) noexcept; // C++20
+
+  // [bit.byteswap], byteswap
+  template<class T>
+    constexpr T byteswap(T value) noexcept;      // C++23
+
+  // [bit.pow.two], integral powers of 2
+  template <class T>
+    constexpr bool has_single_bit(T x) noexcept; // C++20
+  template <class T>
+    constexpr T bit_ceil(T x);                   // C++20
+  template <class T>
+    constexpr T bit_floor(T x) noexcept;         // C++20
+  template <class T>
+    constexpr int bit_width(T x) noexcept;       // C++20
+
+  // [bit.rotate], rotating
+  template<class T>
+    constexpr T rotl(T x, int s) noexcept; // C++20
+  template<class T>
+    constexpr T rotr(T x, int s) noexcept; // C++20
+
+  // [bit.count], counting
+  template<class T>
+    constexpr int countl_zero(T x) noexcept;  // C++20
+  template<class T>
+    constexpr int countl_one(T x) noexcept;   // C++20
+  template<class T>
+    constexpr int countr_zero(T x) noexcept;  // C++20
+  template<class T>
+    constexpr int countr_one(T x) noexcept;   // C++20
+  template<class T>
+    constexpr int popcount(T x) noexcept;     // C++20
+
+  // [bit.endian], endian
+  enum class endian {
+    little = see below,        // C++20
+    big = see below,           // C++20
+    native = see below         // C++20
+  };
+
+} // namespace std
+
+*/
+
+#include <__config>
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__bit/bit_cast.h>
+#  include <__bit/bit_ceil.h>
+#  include <__bit/bit_floor.h>
+#  include <__bit/bit_log2.h>
+#  include <__bit/bit_width.h>
+#  include <__bit/countl.h>
+#  include <__bit/countr.h>
+#  include <__bit/endian.h>
+#  include <__bit/has_single_bit.h>
+#  include <__bit/popcount.h>
+#  include <__bit/rotate.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+#  include <__bit/byteswap.h>
+#endif
+
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#  include <cstdint>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstdlib>
+#  include <iosfwd>
+#  include <limits>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_BIT
diff --git a/libcxx/include/__cxx03/bitset b/libcxx/include/__cxx03/bitset
new file mode 100644
index 000000000000..6bd7bfe585f3
--- /dev/null
+++ b/libcxx/include/__cxx03/bitset
@@ -0,0 +1,969 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_BITSET
+#define _LIBCPP_BITSET
+
+// clang-format off
+
+/*
+    bitset synopsis
+
+namespace std
+{
+
+namespace std {
+
+template <size_t N>
+class bitset
+{
+public:
+    // bit reference:
+    class reference
+    {
+        friend class bitset;
+        reference() noexcept;
+    public:
+        ~reference() noexcept;
+        reference& operator=(bool x) noexcept;           // for b[i] = x;
+        reference& operator=(const reference&) noexcept; // for b[i] = b[j];
+        bool operator~() const noexcept;                 // flips the bit
+        operator bool() const noexcept;                  // for x = b[i];
+        reference& flip() noexcept;                      // for b[i].flip();
+    };
+
+    // 23.3.5.1 constructors:
+    constexpr bitset() noexcept;
+    constexpr bitset(unsigned long long val) noexcept;
+    template <class charT>
+        constexpr explicit bitset(const charT* str,
+            typename basic_string<charT>::size_type n = basic_string<charT>::npos,
+            charT zero = charT('0'), charT one = charT('1'));                                // until C++26, constexpr since C++23
+    template <class charT>
+        constexpr explicit bitset(const charT* str,
+            typename basic_string_view<charT>::size_type n = basic_string_view<charT>::npos,
+            charT zero = charT('0'), charT one = charT('1'));                                // since C++26
+    template<class charT, class traits>
+        explicit bitset(
+            const basic_string_view<charT,traits>& str,
+            typename basic_string_view<charT,traits>::size_type pos = 0,
+            typename basic_string_view<charT,traits>::size_type n = basic_string_view<charT,traits>::npos,
+            charT zero = charT('0'), charT one = charT('1'));                                // since C++26
+    template<class charT, class traits, class Allocator>
+        constexpr explicit bitset(
+            const basic_string<charT,traits,Allocator>& str,
+            typename basic_string<charT,traits,Allocator>::size_type pos = 0,
+            typename basic_string<charT,traits,Allocator>::size_type n = basic_string<charT,traits,Allocator>::npos,
+            charT zero = charT('0'), charT one = charT('1'));                                // constexpr since C++23
+
+    // 23.3.5.2 bitset operations:
+    bitset& operator&=(const bitset& rhs) noexcept; // constexpr since C++23
+    bitset& operator|=(const bitset& rhs) noexcept; // constexpr since C++23
+    bitset& operator^=(const bitset& rhs) noexcept; // constexpr since C++23
+    bitset& operator<<=(size_t pos) noexcept;       // constexpr since C++23
+    bitset& operator>>=(size_t pos) noexcept;       // constexpr since C++23
+    bitset& set() noexcept;                         // constexpr since C++23
+    bitset& set(size_t pos, bool val = true);       // constexpr since C++23
+    bitset& reset() noexcept;                       // constexpr since C++23
+    bitset& reset(size_t pos);                      // constexpr since C++23
+    bitset operator~() const noexcept;              // constexpr since C++23
+    bitset& flip() noexcept;                        // constexpr since C++23
+    bitset& flip(size_t pos);                       // constexpr since C++23
+
+    // element access:
+    constexpr bool operator[](size_t pos) const;
+    reference operator[](size_t pos);            // constexpr since C++23
+    unsigned long to_ulong() const;              // constexpr since C++23
+    unsigned long long to_ullong() const;        // constexpr since C++23
+    template <class charT, class traits, class Allocator> // constexpr since C++23
+        basic_string<charT, traits, Allocator> to_string(charT zero = charT('0'), charT one = charT('1')) const;
+    template <class charT, class traits> // constexpr since C++23
+        basic_string<charT, traits, allocator<charT> > to_string(charT zero = charT('0'), charT one = charT('1')) const;
+    template <class charT> // constexpr since C++23
+        basic_string<charT, char_traits<charT>, allocator<charT> > to_string(charT zero = charT('0'), charT one = charT('1')) const;
+    basic_string<char, char_traits<char>, allocator<char> > to_string(char zero = '0', char one = '1') const; // constexpr since C++23
+    size_t count() const noexcept;                     // constexpr since C++23
+    constexpr size_t size() const noexcept;            // constexpr since C++23
+    bool operator==(const bitset& rhs) const noexcept; // constexpr since C++23
+    bool operator!=(const bitset& rhs) const noexcept; // removed in C++20
+    bool test(size_t pos) const;                       // constexpr since C++23
+    bool all() const noexcept;                         // constexpr since C++23
+    bool any() const noexcept;                         // constexpr since C++23
+    bool none() const noexcept;                        // constexpr since C++23
+    bitset<N> operator<<(size_t pos) const noexcept;   // constexpr since C++23
+    bitset<N> operator>>(size_t pos) const noexcept;   // constexpr since C++23
+};
+
+// 23.3.5.3 bitset operators:
+template <size_t N>
+bitset<N> operator&(const bitset<N>&, const bitset<N>&) noexcept; // constexpr since C++23
+
+template <size_t N>
+bitset<N> operator|(const bitset<N>&, const bitset<N>&) noexcept; // constexpr since C++23
+
+template <size_t N>
+bitset<N> operator^(const bitset<N>&, const bitset<N>&) noexcept; // constexpr since C++23
+
+template <class charT, class traits, size_t N>
+basic_istream<charT, traits>&
+operator>>(basic_istream<charT, traits>& is, bitset<N>& x);
+
+template <class charT, class traits, size_t N>
+basic_ostream<charT, traits>&
+operator<<(basic_ostream<charT, traits>& os, const bitset<N>& x);
+
+template <size_t N> struct hash<std::bitset<N>>;
+
+}  // std
+
+*/
+
+// clang-format on
+
+#include <__algorithm/count.h>
+#include <__algorithm/fill.h>
+#include <__algorithm/find.h>
+#include <__bit_reference>
+#include <__config>
+#include <__functional/hash.h>
+#include <__functional/unary_function.h>
+#include <__type_traits/is_char_like_type.h>
+#include <climits>
+#include <cstddef>
+#include <stdexcept>
+#include <string_view>
+#include <version>
+
+// standard-mandated includes
+
+// [bitset.syn]
+#include <iosfwd>
+#include <string>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <size_t _N_words, size_t _Size>
+class __bitset;
+
+template <size_t _N_words, size_t _Size>
+struct __has_storage_type<__bitset<_N_words, _Size> > {
+  static const bool value = true;
+};
+
+template <size_t _N_words, size_t _Size>
+class __bitset {
+public:
+  typedef ptrdiff_t difference_type;
+  typedef size_t size_type;
+  typedef size_type __storage_type;
+
+protected:
+  typedef __bitset __self;
+  typedef __storage_type* __storage_pointer;
+  typedef const __storage_type* __const_storage_pointer;
+  static const unsigned __bits_per_word = static_cast<unsigned>(sizeof(__storage_type) * CHAR_BIT);
+
+  friend class __bit_reference<__bitset>;
+  friend class __bit_const_reference<__bitset>;
+  friend class __bit_iterator<__bitset, false>;
+  friend class __bit_iterator<__bitset, true>;
+  friend struct __bit_array<__bitset>;
+
+  __storage_type __first_[_N_words];
+
+  typedef __bit_reference<__bitset> reference;
+  typedef __bit_const_reference<__bitset> const_reference;
+  typedef __bit_iterator<__bitset, false> iterator;
+  typedef __bit_iterator<__bitset, true> const_iterator;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __bitset() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI explicit _LIBCPP_CONSTEXPR __bitset(unsigned long long __v) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 reference __make_ref(size_t __pos) _NOEXCEPT {
+    return reference(__first_ + __pos / __bits_per_word, __storage_type(1) << __pos % __bits_per_word);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR const_reference __make_ref(size_t __pos) const _NOEXCEPT {
+    return const_reference(__first_ + __pos / __bits_per_word, __storage_type(1) << __pos % __bits_per_word);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 iterator __make_iter(size_t __pos) _NOEXCEPT {
+    return iterator(__first_ + __pos / __bits_per_word, __pos % __bits_per_word);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 const_iterator __make_iter(size_t __pos) const _NOEXCEPT {
+    return const_iterator(__first_ + __pos / __bits_per_word, __pos % __bits_per_word);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void operator&=(const __bitset& __v) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void operator|=(const __bitset& __v) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void operator^=(const __bitset& __v) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void flip() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong() const {
+    return to_ulong(integral_constant < bool, _Size< sizeof(unsigned long) * CHAR_BIT>());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong() const {
+    return to_ullong(integral_constant < bool, _Size< sizeof(unsigned long long) * CHAR_BIT>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool all() const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool any() const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI size_t __hash_code() const _NOEXCEPT;
+
+private:
+#ifdef _LIBCPP_CXX03_LANG
+  void __init(unsigned long long __v, false_type) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI void __init(unsigned long long __v, true_type) _NOEXCEPT;
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong(false_type) const;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong(true_type) const;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong(false_type) const;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong(true_type) const;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong(true_type, false_type) const;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong(true_type, true_type) const;
+};
+
+template <size_t _N_words, size_t _Size>
+inline _LIBCPP_CONSTEXPR __bitset<_N_words, _Size>::__bitset() _NOEXCEPT
+#ifndef _LIBCPP_CXX03_LANG
+    : __first_{0}
+#endif
+{
+#ifdef _LIBCPP_CXX03_LANG
+  std::fill_n(__first_, _N_words, __storage_type(0));
+#endif
+}
+
+#ifdef _LIBCPP_CXX03_LANG
+
+template <size_t _N_words, size_t _Size>
+void __bitset<_N_words, _Size>::__init(unsigned long long __v, false_type) _NOEXCEPT {
+  __storage_type __t[sizeof(unsigned long long) / sizeof(__storage_type)];
+  size_t __sz = _Size;
+  for (size_t __i = 0; __i < sizeof(__t) / sizeof(__t[0]); ++__i, __v >>= __bits_per_word, __sz -= __bits_per_word)
+    if (__sz < __bits_per_word)
+      __t[__i] = static_cast<__storage_type>(__v) & (1ULL << __sz) - 1;
+    else
+      __t[__i] = static_cast<__storage_type>(__v);
+
+  std::copy(__t, __t + sizeof(__t) / sizeof(__t[0]), __first_);
+  std::fill(
+      __first_ + sizeof(__t) / sizeof(__t[0]), __first_ + sizeof(__first_) / sizeof(__first_[0]), __storage_type(0));
+}
+
+template <size_t _N_words, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI void __bitset<_N_words, _Size>::__init(unsigned long long __v, true_type) _NOEXCEPT {
+  __first_[0] = __v;
+  if (_Size < __bits_per_word)
+    __first_[0] &= (1ULL << _Size) - 1;
+
+  std::fill(__first_ + 1, __first_ + sizeof(__first_) / sizeof(__first_[0]), __storage_type(0));
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <size_t _N_words, size_t _Size>
+inline _LIBCPP_CONSTEXPR __bitset<_N_words, _Size>::__bitset(unsigned long long __v) _NOEXCEPT
+#ifndef _LIBCPP_CXX03_LANG
+#  if __SIZEOF_SIZE_T__ == 8
+    : __first_{__v}
+#  elif __SIZEOF_SIZE_T__ == 4
+    : __first_{static_cast<__storage_type>(__v),
+               _Size >= 2 * __bits_per_word
+                   ? static_cast<__storage_type>(__v >> __bits_per_word)
+                   : static_cast<__storage_type>((__v >> __bits_per_word) &
+                                                 (__storage_type(1) << (_Size - __bits_per_word)) - 1)}
+#  else
+#    error This constructor has not been ported to this platform
+#  endif
+#endif
+{
+#ifdef _LIBCPP_CXX03_LANG
+  __init(__v, integral_constant<bool, sizeof(unsigned long long) == sizeof(__storage_type)>());
+#endif
+}
+
+template <size_t _N_words, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void
+__bitset<_N_words, _Size>::operator&=(const __bitset& __v) _NOEXCEPT {
+  for (size_type __i = 0; __i < _N_words; ++__i)
+    __first_[__i] &= __v.__first_[__i];
+}
+
+template <size_t _N_words, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void
+__bitset<_N_words, _Size>::operator|=(const __bitset& __v) _NOEXCEPT {
+  for (size_type __i = 0; __i < _N_words; ++__i)
+    __first_[__i] |= __v.__first_[__i];
+}
+
+template <size_t _N_words, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void
+__bitset<_N_words, _Size>::operator^=(const __bitset& __v) _NOEXCEPT {
+  for (size_type __i = 0; __i < _N_words; ++__i)
+    __first_[__i] ^= __v.__first_[__i];
+}
+
+template <size_t _N_words, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void __bitset<_N_words, _Size>::flip() _NOEXCEPT {
+  // do middle whole words
+  size_type __n         = _Size;
+  __storage_pointer __p = __first_;
+  for (; __n >= __bits_per_word; ++__p, __n -= __bits_per_word)
+    *__p = ~*__p;
+  // do last partial word
+  if (__n > 0) {
+    __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+    __storage_type __b = *__p & __m;
+    *__p &= ~__m;
+    *__p |= ~__b & __m;
+  }
+}
+
+template <size_t _N_words, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long
+__bitset<_N_words, _Size>::to_ulong(false_type) const {
+  const_iterator __e = __make_iter(_Size);
+  const_iterator __i = std::find(__make_iter(sizeof(unsigned long) * CHAR_BIT), __e, true);
+  if (__i != __e)
+    __throw_overflow_error("bitset to_ulong overflow error");
+
+  return __first_[0];
+}
+
+template <size_t _N_words, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long
+__bitset<_N_words, _Size>::to_ulong(true_type) const {
+  return __first_[0];
+}
+
+template <size_t _N_words, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long
+__bitset<_N_words, _Size>::to_ullong(false_type) const {
+  const_iterator __e = __make_iter(_Size);
+  const_iterator __i = std::find(__make_iter(sizeof(unsigned long long) * CHAR_BIT), __e, true);
+  if (__i != __e)
+    __throw_overflow_error("bitset to_ullong overflow error");
+
+  return to_ullong(true_type());
+}
+
+template <size_t _N_words, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long
+__bitset<_N_words, _Size>::to_ullong(true_type) const {
+  return to_ullong(true_type(), integral_constant<bool, sizeof(__storage_type) < sizeof(unsigned long long)>());
+}
+
+template <size_t _N_words, size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long
+__bitset<_N_words, _Size>::to_ullong(true_type, false_type) const {
+  return __first_[0];
+}
+
+template <size_t _N_words, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long
+__bitset<_N_words, _Size>::to_ullong(true_type, true_type) const {
+  unsigned long long __r = __first_[0];
+  _LIBCPP_DIAGNOSTIC_PUSH
+  _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wshift-count-overflow")
+  for (size_t __i = 1; __i < sizeof(unsigned long long) / sizeof(__storage_type); ++__i)
+    __r |= static_cast<unsigned long long>(__first_[__i]) << (sizeof(__storage_type) * CHAR_BIT);
+  _LIBCPP_DIAGNOSTIC_POP
+  return __r;
+}
+
+template <size_t _N_words, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool __bitset<_N_words, _Size>::all() const _NOEXCEPT {
+  // do middle whole words
+  size_type __n               = _Size;
+  __const_storage_pointer __p = __first_;
+  for (; __n >= __bits_per_word; ++__p, __n -= __bits_per_word)
+    if (~*__p)
+      return false;
+  // do last partial word
+  if (__n > 0) {
+    __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+    if (~*__p & __m)
+      return false;
+  }
+  return true;
+}
+
+template <size_t _N_words, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool __bitset<_N_words, _Size>::any() const _NOEXCEPT {
+  // do middle whole words
+  size_type __n               = _Size;
+  __const_storage_pointer __p = __first_;
+  for (; __n >= __bits_per_word; ++__p, __n -= __bits_per_word)
+    if (*__p)
+      return true;
+  // do last partial word
+  if (__n > 0) {
+    __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+    if (*__p & __m)
+      return true;
+  }
+  return false;
+}
+
+template <size_t _N_words, size_t _Size>
+inline size_t __bitset<_N_words, _Size>::__hash_code() const _NOEXCEPT {
+  size_t __h = 0;
+  for (size_type __i = 0; __i < _N_words; ++__i)
+    __h ^= __first_[__i];
+  return __h;
+}
+
+template <size_t _Size>
+class __bitset<1, _Size> {
+public:
+  typedef ptrdiff_t difference_type;
+  typedef size_t size_type;
+  typedef size_type __storage_type;
+
+protected:
+  typedef __bitset __self;
+  typedef __storage_type* __storage_pointer;
+  typedef const __storage_type* __const_storage_pointer;
+  static const unsigned __bits_per_word = static_cast<unsigned>(sizeof(__storage_type) * CHAR_BIT);
+
+  friend class __bit_reference<__bitset>;
+  friend class __bit_const_reference<__bitset>;
+  friend class __bit_iterator<__bitset, false>;
+  friend class __bit_iterator<__bitset, true>;
+  friend struct __bit_array<__bitset>;
+
+  __storage_type __first_;
+
+  typedef __bit_reference<__bitset> reference;
+  typedef __bit_const_reference<__bitset> const_reference;
+  typedef __bit_iterator<__bitset, false> iterator;
+  typedef __bit_iterator<__bitset, true> const_iterator;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __bitset() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI explicit _LIBCPP_CONSTEXPR __bitset(unsigned long long __v) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 reference __make_ref(size_t __pos) _NOEXCEPT {
+    return reference(&__first_, __storage_type(1) << __pos);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR const_reference __make_ref(size_t __pos) const _NOEXCEPT {
+    return const_reference(&__first_, __storage_type(1) << __pos);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 iterator __make_iter(size_t __pos) _NOEXCEPT {
+    return iterator(&__first_ + __pos / __bits_per_word, __pos % __bits_per_word);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 const_iterator __make_iter(size_t __pos) const _NOEXCEPT {
+    return const_iterator(&__first_ + __pos / __bits_per_word, __pos % __bits_per_word);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void operator&=(const __bitset& __v) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void operator|=(const __bitset& __v) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void operator^=(const __bitset& __v) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void flip() _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong() const;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong() const;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool all() const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool any() const _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI size_t __hash_code() const _NOEXCEPT;
+};
+
+template <size_t _Size>
+inline _LIBCPP_CONSTEXPR __bitset<1, _Size>::__bitset() _NOEXCEPT : __first_(0) {}
+
+template <size_t _Size>
+inline _LIBCPP_CONSTEXPR __bitset<1, _Size>::__bitset(unsigned long long __v) _NOEXCEPT
+    : __first_(_Size == __bits_per_word ? static_cast<__storage_type>(__v)
+                                        : static_cast<__storage_type>(__v) & ((__storage_type(1) << _Size) - 1)) {}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void
+__bitset<1, _Size>::operator&=(const __bitset& __v) _NOEXCEPT {
+  __first_ &= __v.__first_;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void
+__bitset<1, _Size>::operator|=(const __bitset& __v) _NOEXCEPT {
+  __first_ |= __v.__first_;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void
+__bitset<1, _Size>::operator^=(const __bitset& __v) _NOEXCEPT {
+  __first_ ^= __v.__first_;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void __bitset<1, _Size>::flip() _NOEXCEPT {
+  __storage_type __m = ~__storage_type(0) >> (__bits_per_word - _Size);
+  __first_           = ~__first_;
+  __first_ &= __m;
+}
+
+template <size_t _Size>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long __bitset<1, _Size>::to_ulong() const {
+  return __first_;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long __bitset<1, _Size>::to_ullong() const {
+  return __first_;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool __bitset<1, _Size>::all() const _NOEXCEPT {
+  __storage_type __m = ~__storage_type(0) >> (__bits_per_word - _Size);
+  return !(~__first_ & __m);
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool __bitset<1, _Size>::any() const _NOEXCEPT {
+  __storage_type __m = ~__storage_type(0) >> (__bits_per_word - _Size);
+  return __first_ & __m;
+}
+
+template <size_t _Size>
+inline size_t __bitset<1, _Size>::__hash_code() const _NOEXCEPT {
+  return __first_;
+}
+
+template <>
+class __bitset<0, 0> {
+public:
+  typedef ptrdiff_t difference_type;
+  typedef size_t size_type;
+  typedef size_type __storage_type;
+
+protected:
+  typedef __bitset __self;
+  typedef __storage_type* __storage_pointer;
+  typedef const __storage_type* __const_storage_pointer;
+  static const unsigned __bits_per_word = static_cast<unsigned>(sizeof(__storage_type) * CHAR_BIT);
+
+  friend class __bit_reference<__bitset>;
+  friend class __bit_const_reference<__bitset>;
+  friend class __bit_iterator<__bitset, false>;
+  friend class __bit_iterator<__bitset, true>;
+  friend struct __bit_array<__bitset>;
+
+  typedef __bit_reference<__bitset> reference;
+  typedef __bit_const_reference<__bitset> const_reference;
+  typedef __bit_iterator<__bitset, false> iterator;
+  typedef __bit_iterator<__bitset, true> const_iterator;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __bitset() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI explicit _LIBCPP_CONSTEXPR __bitset(unsigned long long) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 reference __make_ref(size_t) _NOEXCEPT {
+    return reference(nullptr, 1);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR const_reference __make_ref(size_t) const _NOEXCEPT {
+    return const_reference(nullptr, 1);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 iterator __make_iter(size_t) _NOEXCEPT {
+    return iterator(nullptr, 0);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 const_iterator __make_iter(size_t) const _NOEXCEPT {
+    return const_iterator(nullptr, 0);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void operator&=(const __bitset&) _NOEXCEPT {}
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void operator|=(const __bitset&) _NOEXCEPT {}
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void operator^=(const __bitset&) _NOEXCEPT {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void flip() _NOEXCEPT {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong() const { return 0; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong() const { return 0; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool all() const _NOEXCEPT { return true; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool any() const _NOEXCEPT { return false; }
+
+  _LIBCPP_HIDE_FROM_ABI size_t __hash_code() const _NOEXCEPT { return 0; }
+};
+
+inline _LIBCPP_CONSTEXPR __bitset<0, 0>::__bitset() _NOEXCEPT {}
+
+inline _LIBCPP_CONSTEXPR __bitset<0, 0>::__bitset(unsigned long long) _NOEXCEPT {}
+
+template <size_t _Size>
+class _LIBCPP_TEMPLATE_VIS bitset;
+template <size_t _Size>
+struct hash<bitset<_Size> >;
+
+template <size_t _Size>
+class _LIBCPP_TEMPLATE_VIS bitset
+    : private __bitset<_Size == 0 ? 0 : (_Size - 1) / (sizeof(size_t) * CHAR_BIT) + 1, _Size> {
+public:
+  static const unsigned __n_words = _Size == 0 ? 0 : (_Size - 1) / (sizeof(size_t) * CHAR_BIT) + 1;
+  typedef __bitset<__n_words, _Size> base;
+
+public:
+  typedef typename base::reference reference;
+  typedef typename base::const_reference const_reference;
+
+  // 23.3.5.1 constructors:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bitset() _NOEXCEPT {}
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bitset(unsigned long long __v) _NOEXCEPT : base(__v) {}
+  template <class _CharT, __enable_if_t<_IsCharLikeType<_CharT>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 explicit bitset(
+      const _CharT* __str,
+#if _LIBCPP_STD_VER >= 26
+      typename basic_string_view<_CharT>::size_type __n = basic_string_view<_CharT>::npos,
+#else
+      typename basic_string<_CharT>::size_type __n = basic_string<_CharT>::npos,
+#endif
+      _CharT __zero = _CharT('0'),
+      _CharT __one  = _CharT('1')) {
+
+    size_t __rlen = std::min(__n, char_traits<_CharT>::length(__str));
+    __init_from_string_view(basic_string_view<_CharT>(__str, __rlen), __zero, __one);
+  }
+#if _LIBCPP_STD_VER >= 26
+  template <class _CharT, class _Traits>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit bitset(
+      basic_string_view<_CharT, _Traits> __str,
+      typename basic_string_view<_CharT, _Traits>::size_type __pos = 0,
+      typename basic_string_view<_CharT, _Traits>::size_type __n   = basic_string_view<_CharT, _Traits>::npos,
+      _CharT __zero                                                = _CharT('0'),
+      _CharT __one                                                 = _CharT('1')) {
+    if (__pos > __str.size())
+      __throw_out_of_range("bitset string pos out of range");
+
+    size_t __rlen = std::min(__n, __str.size() - __pos);
+    __init_from_string_view(basic_string_view<_CharT, _Traits>(__str.data() + __pos, __rlen), __zero, __one);
+  }
+#endif
+  template <class _CharT, class _Traits, class _Allocator>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 explicit bitset(
+      const basic_string<_CharT, _Traits, _Allocator>& __str,
+      typename basic_string<_CharT, _Traits, _Allocator>::size_type __pos = 0,
+      typename basic_string<_CharT, _Traits, _Allocator>::size_type __n =
+          basic_string<_CharT, _Traits, _Allocator>::npos,
+      _CharT __zero = _CharT('0'),
+      _CharT __one  = _CharT('1')) {
+    if (__pos > __str.size())
+      std::__throw_out_of_range("bitset string pos out of range");
+
+    size_t __rlen = std::min(__n, __str.size() - __pos);
+    __init_from_string_view(basic_string_view<_CharT, _Traits>(__str.data() + __pos, __rlen), __zero, __one);
+  }
+
+  // 23.3.5.2 bitset operations:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& operator&=(const bitset& __rhs) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& operator|=(const bitset& __rhs) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& operator^=(const bitset& __rhs) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& operator<<=(size_t __pos) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& operator>>=(size_t __pos) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& set() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& set(size_t __pos, bool __val = true);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& reset() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& reset(size_t __pos);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset operator~() const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& flip() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& flip(size_t __pos);
+
+  // element access:
+#ifdef _LIBCPP_ABI_BITSET_VECTOR_BOOL_CONST_SUBSCRIPT_RETURN_BOOL
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator[](size_t __p) const { return base::__make_ref(__p); }
+#else
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR const_reference operator[](size_t __p) const { return base::__make_ref(__p); }
+#endif
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 reference operator[](size_t __p) { return base::__make_ref(__p); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong() const;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong() const;
+  template <class _CharT, class _Traits, class _Allocator>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<_CharT, _Traits, _Allocator>
+  to_string(_CharT __zero = _CharT('0'), _CharT __one = _CharT('1')) const;
+  template <class _CharT, class _Traits>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<_CharT, _Traits, allocator<_CharT> >
+  to_string(_CharT __zero = _CharT('0'), _CharT __one = _CharT('1')) const;
+  template <class _CharT>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<_CharT, char_traits<_CharT>, allocator<_CharT> >
+  to_string(_CharT __zero = _CharT('0'), _CharT __one = _CharT('1')) const;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<char, char_traits<char>, allocator<char> >
+  to_string(char __zero = '0', char __one = '1') const;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 size_t count() const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_t size() const _NOEXCEPT { return _Size; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool operator==(const bitset& __rhs) const _NOEXCEPT;
+#if _LIBCPP_STD_VER <= 17
+  _LIBCPP_HIDE_FROM_ABI bool operator!=(const bitset& __rhs) const _NOEXCEPT;
+#endif
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool test(size_t __pos) const;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool all() const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool any() const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool none() const _NOEXCEPT { return !any(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset operator<<(size_t __pos) const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset operator>>(size_t __pos) const _NOEXCEPT;
+
+private:
+  template <class _CharT, class _Traits>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void
+  __init_from_string_view(basic_string_view<_CharT, _Traits> __str, _CharT __zero, _CharT __one) {
+    for (size_t __i = 0; __i < __str.size(); ++__i)
+      if (!_Traits::eq(__str[__i], __zero) && !_Traits::eq(__str[__i], __one))
+        std::__throw_invalid_argument("bitset string ctor has invalid argument");
+
+    size_t __mp = std::min(__str.size(), _Size);
+    size_t __i  = 0;
+    for (; __i < __mp; ++__i) {
+      _CharT __c   = __str[__mp - 1 - __i];
+      (*this)[__i] = _Traits::eq(__c, __one);
+    }
+    std::fill(base::__make_iter(__i), base::__make_iter(_Size), false);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_t __hash_code() const _NOEXCEPT { return base::__hash_code(); }
+
+  friend struct hash<bitset>;
+};
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>&
+bitset<_Size>::operator&=(const bitset& __rhs) _NOEXCEPT {
+  base::operator&=(__rhs);
+  return *this;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>&
+bitset<_Size>::operator|=(const bitset& __rhs) _NOEXCEPT {
+  base::operator|=(__rhs);
+  return *this;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>&
+bitset<_Size>::operator^=(const bitset& __rhs) _NOEXCEPT {
+  base::operator^=(__rhs);
+  return *this;
+}
+
+template <size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>& bitset<_Size>::operator<<=(size_t __pos) _NOEXCEPT {
+  __pos = std::min(__pos, _Size);
+  std::copy_backward(base::__make_iter(0), base::__make_iter(_Size - __pos), base::__make_iter(_Size));
+  std::fill_n(base::__make_iter(0), __pos, false);
+  return *this;
+}
+
+template <size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>& bitset<_Size>::operator>>=(size_t __pos) _NOEXCEPT {
+  __pos = std::min(__pos, _Size);
+  std::copy(base::__make_iter(__pos), base::__make_iter(_Size), base::__make_iter(0));
+  std::fill_n(base::__make_iter(_Size - __pos), __pos, false);
+  return *this;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>& bitset<_Size>::set() _NOEXCEPT {
+  std::fill_n(base::__make_iter(0), _Size, true);
+  return *this;
+}
+
+template <size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>& bitset<_Size>::set(size_t __pos, bool __val) {
+  if (__pos >= _Size)
+    __throw_out_of_range("bitset set argument out of range");
+
+  (*this)[__pos] = __val;
+  return *this;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>& bitset<_Size>::reset() _NOEXCEPT {
+  std::fill_n(base::__make_iter(0), _Size, false);
+  return *this;
+}
+
+template <size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>& bitset<_Size>::reset(size_t __pos) {
+  if (__pos >= _Size)
+    __throw_out_of_range("bitset reset argument out of range");
+
+  (*this)[__pos] = false;
+  return *this;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size> bitset<_Size>::operator~() const _NOEXCEPT {
+  bitset __x(*this);
+  __x.flip();
+  return __x;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>& bitset<_Size>::flip() _NOEXCEPT {
+  base::flip();
+  return *this;
+}
+
+template <size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>& bitset<_Size>::flip(size_t __pos) {
+  if (__pos >= _Size)
+    __throw_out_of_range("bitset flip argument out of range");
+
+  reference __r = base::__make_ref(__pos);
+  __r           = ~__r;
+  return *this;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long bitset<_Size>::to_ulong() const {
+  return base::to_ulong();
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long bitset<_Size>::to_ullong() const {
+  return base::to_ullong();
+}
+
+template <size_t _Size>
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<_CharT, _Traits, _Allocator>
+bitset<_Size>::to_string(_CharT __zero, _CharT __one) const {
+  basic_string<_CharT, _Traits, _Allocator> __r(_Size, __zero);
+  for (size_t __i = 0; __i != _Size; ++__i) {
+    if ((*this)[__i])
+      __r[_Size - 1 - __i] = __one;
+  }
+  return __r;
+}
+
+template <size_t _Size>
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<_CharT, _Traits, allocator<_CharT> >
+bitset<_Size>::to_string(_CharT __zero, _CharT __one) const {
+  return to_string<_CharT, _Traits, allocator<_CharT> >(__zero, __one);
+}
+
+template <size_t _Size>
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<_CharT, char_traits<_CharT>, allocator<_CharT> >
+bitset<_Size>::to_string(_CharT __zero, _CharT __one) const {
+  return to_string<_CharT, char_traits<_CharT>, allocator<_CharT> >(__zero, __one);
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<char, char_traits<char>, allocator<char> >
+bitset<_Size>::to_string(char __zero, char __one) const {
+  return to_string<char, char_traits<char>, allocator<char> >(__zero, __one);
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 size_t bitset<_Size>::count() const _NOEXCEPT {
+  return static_cast<size_t>(std::count(base::__make_iter(0), base::__make_iter(_Size), true));
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool
+bitset<_Size>::operator==(const bitset& __rhs) const _NOEXCEPT {
+  return std::equal(base::__make_iter(0), base::__make_iter(_Size), __rhs.__make_iter(0));
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI bool bitset<_Size>::operator!=(const bitset& __rhs) const _NOEXCEPT {
+  return !(*this == __rhs);
+}
+
+#endif
+
+template <size_t _Size>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool bitset<_Size>::test(size_t __pos) const {
+  if (__pos >= _Size)
+    __throw_out_of_range("bitset test argument out of range");
+
+  return (*this)[__pos];
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool bitset<_Size>::all() const _NOEXCEPT {
+  return base::all();
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool bitset<_Size>::any() const _NOEXCEPT {
+  return base::any();
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>
+bitset<_Size>::operator<<(size_t __pos) const _NOEXCEPT {
+  bitset __r = *this;
+  __r <<= __pos;
+  return __r;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>
+bitset<_Size>::operator>>(size_t __pos) const _NOEXCEPT {
+  bitset __r = *this;
+  __r >>= __pos;
+  return __r;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>
+operator&(const bitset<_Size>& __x, const bitset<_Size>& __y) _NOEXCEPT {
+  bitset<_Size> __r = __x;
+  __r &= __y;
+  return __r;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>
+operator|(const bitset<_Size>& __x, const bitset<_Size>& __y) _NOEXCEPT {
+  bitset<_Size> __r = __x;
+  __r |= __y;
+  return __r;
+}
+
+template <size_t _Size>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>
+operator^(const bitset<_Size>& __x, const bitset<_Size>& __y) _NOEXCEPT {
+  bitset<_Size> __r = __x;
+  __r ^= __y;
+  return __r;
+}
+
+template <size_t _Size>
+struct _LIBCPP_TEMPLATE_VIS hash<bitset<_Size> > : public __unary_function<bitset<_Size>, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const bitset<_Size>& __bs) const _NOEXCEPT { return __bs.__hash_code(); }
+};
+
+template <class _CharT, class _Traits, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, bitset<_Size>& __x);
+
+template <class _CharT, class _Traits, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const bitset<_Size>& __x);
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <concepts>
+#  include <cstdlib>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_BITSET
diff --git a/libcxx/include/__cxx03/cassert b/libcxx/include/__cxx03/cassert
new file mode 100644
index 000000000000..6fec37dc6376
--- /dev/null
+++ b/libcxx/include/__cxx03/cassert
@@ -0,0 +1,31 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+/*
+    cassert synopsis
+
+Macros:
+
+    assert
+
+*/
+
+#include <__config>
+
+// <assert.h> is not provided by libc++
+#if __has_include(<assert.h>)
+#  include <assert.h>
+#  ifdef _LIBCPP_ASSERT_H
+#    error "If libc++ starts defining <assert.h>, the __has_include check should move to libc++'s <assert.h>"
+#  endif
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
diff --git a/libcxx/include/__cxx03/ccomplex b/libcxx/include/__cxx03/ccomplex
new file mode 100644
index 000000000000..94d2c8d7d003
--- /dev/null
+++ b/libcxx/include/__cxx03/ccomplex
@@ -0,0 +1,26 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CCOMPLEX
+#define _LIBCPP_CCOMPLEX
+
+/*
+    ccomplex synopsis
+
+#include <complex>
+
+*/
+
+#include <complex>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#endif // _LIBCPP_CCOMPLEX
diff --git a/libcxx/include/__cxx03/cctype b/libcxx/include/__cxx03/cctype
new file mode 100644
index 000000000000..d7af7e084aa2
--- /dev/null
+++ b/libcxx/include/__cxx03/cctype
@@ -0,0 +1,127 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CCTYPE
+#define _LIBCPP_CCTYPE
+
+/*
+    cctype synopsis
+
+namespace std
+{
+
+int isalnum(int c);
+int isalpha(int c);
+int isblank(int c);  // C99
+int iscntrl(int c);
+int isdigit(int c);
+int isgraph(int c);
+int islower(int c);
+int isprint(int c);
+int ispunct(int c);
+int isspace(int c);
+int isupper(int c);
+int isxdigit(int c);
+int tolower(int c);
+int toupper(int c);
+
+}  // std
+*/
+
+#include <__config>
+
+#include <ctype.h>
+
+#ifndef _LIBCPP_CTYPE_H
+#   error <cctype> tried including <ctype.h> but didn't find libc++'s <ctype.h> header. \
+          This usually means that your header search paths are not configured properly.  \
+          The header search paths should contain the C++ Standard Library headers before \
+          any C Standard Library.
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifdef isalnum
+#  undef isalnum
+#endif
+
+#ifdef isalpha
+#  undef isalpha
+#endif
+
+#ifdef isblank
+#  undef isblank
+#endif
+
+#ifdef iscntrl
+#  undef iscntrl
+#endif
+
+#ifdef isdigit
+#  undef isdigit
+#endif
+
+#ifdef isgraph
+#  undef isgraph
+#endif
+
+#ifdef islower
+#  undef islower
+#endif
+
+#ifdef isprint
+#  undef isprint
+#endif
+
+#ifdef ispunct
+#  undef ispunct
+#endif
+
+#ifdef isspace
+#  undef isspace
+#endif
+
+#ifdef isupper
+#  undef isupper
+#endif
+
+#ifdef isxdigit
+#  undef isxdigit
+#endif
+
+#ifdef tolower
+#  undef tolower
+#endif
+
+#ifdef toupper
+#  undef toupper
+#endif
+
+using ::isalnum _LIBCPP_USING_IF_EXISTS;
+using ::isalpha _LIBCPP_USING_IF_EXISTS;
+using ::isblank _LIBCPP_USING_IF_EXISTS;
+using ::iscntrl _LIBCPP_USING_IF_EXISTS;
+using ::isdigit _LIBCPP_USING_IF_EXISTS;
+using ::isgraph _LIBCPP_USING_IF_EXISTS;
+using ::islower _LIBCPP_USING_IF_EXISTS;
+using ::isprint _LIBCPP_USING_IF_EXISTS;
+using ::ispunct _LIBCPP_USING_IF_EXISTS;
+using ::isspace _LIBCPP_USING_IF_EXISTS;
+using ::isupper _LIBCPP_USING_IF_EXISTS;
+using ::isxdigit _LIBCPP_USING_IF_EXISTS;
+using ::tolower _LIBCPP_USING_IF_EXISTS;
+using ::toupper _LIBCPP_USING_IF_EXISTS;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CCTYPE
diff --git a/libcxx/include/__cxx03/cerrno b/libcxx/include/__cxx03/cerrno
new file mode 100644
index 000000000000..f1295680fed6
--- /dev/null
+++ b/libcxx/include/__cxx03/cerrno
@@ -0,0 +1,48 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CERRNO
+#define _LIBCPP_CERRNO
+
+/*
+    cerrno synopsis
+
+Macros:
+
+    EDOM
+    EILSEQ  // C99
+    ERANGE
+    errno
+
+*/
+
+#include <__config>
+
+#include <errno.h>
+
+#ifndef _LIBCPP_ERRNO_H
+#   error <cerrno> tried including <errno.h> but didn't find libc++'s <errno.h> header. \
+          This usually means that your header search paths are not configured properly. \
+          The header search paths should contain the C++ Standard Library headers before \
+          any C Standard Library, and you are probably using compiler flags that make that \
+          not be the case.
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+// LWG3869 Deprecate std::errc constants related to UNIX STREAMS
+//
+// This LWG issue deprecates the POSIX macros ENODATA, ENOSR, ENOSTR, and ETIME. These were
+// deprecated in libc++ in https://github.com/llvm/llvm-project/pull/80542.
+// Based on the post commit feedback the macro are no longer deprecated.
+// Instead libc++ leaves the deprecation to the provider of errno.h.
+
+#endif // _LIBCPP_CERRNO
diff --git a/libcxx/include/__cxx03/cfenv b/libcxx/include/__cxx03/cfenv
new file mode 100644
index 000000000000..f8cacd562f76
--- /dev/null
+++ b/libcxx/include/__cxx03/cfenv
@@ -0,0 +1,90 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CFENV
+#define _LIBCPP_CFENV
+
+/*
+    cfenv synopsis
+
+This entire header is C99 / C++0X
+
+Macros:
+
+    FE_DIVBYZERO
+    FE_INEXACT
+    FE_INVALID
+    FE_OVERFLOW
+    FE_UNDERFLOW
+    FE_ALL_EXCEPT
+    FE_DOWNWARD
+    FE_TONEAREST
+    FE_TOWARDZERO
+    FE_UPWARD
+    FE_DFL_ENV
+
+namespace std
+{
+
+Types:
+
+    fenv_t
+    fexcept_t
+
+int feclearexcept(int excepts);
+int fegetexceptflag(fexcept_t* flagp, int excepts);
+int feraiseexcept(int excepts);
+int fesetexceptflag(const fexcept_t* flagp, int excepts);
+int fetestexcept(int excepts);
+int fegetround();
+int fesetround(int round);
+int fegetenv(fenv_t* envp);
+int feholdexcept(fenv_t* envp);
+int fesetenv(const fenv_t* envp);
+int feupdateenv(const fenv_t* envp);
+
+}  // std
+*/
+
+#include <__config>
+
+#include <fenv.h>
+
+#ifndef _LIBCPP_FENV_H
+#   error <cfenv> tried including <fenv.h> but didn't find libc++'s <fenv.h> header. \
+          This usually means that your header search paths are not configured properly. \
+          The header search paths should contain the C++ Standard Library headers before \
+          any C Standard Library, and you are probably using compiler flags that make that \
+          not be the case.
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::fenv_t _LIBCPP_USING_IF_EXISTS;
+using ::fexcept_t _LIBCPP_USING_IF_EXISTS;
+
+using ::feclearexcept _LIBCPP_USING_IF_EXISTS;
+using ::fegetexceptflag _LIBCPP_USING_IF_EXISTS;
+using ::feraiseexcept _LIBCPP_USING_IF_EXISTS;
+using ::fesetexceptflag _LIBCPP_USING_IF_EXISTS;
+using ::fetestexcept _LIBCPP_USING_IF_EXISTS;
+using ::fegetround _LIBCPP_USING_IF_EXISTS;
+using ::fesetround _LIBCPP_USING_IF_EXISTS;
+using ::fegetenv _LIBCPP_USING_IF_EXISTS;
+using ::feholdexcept _LIBCPP_USING_IF_EXISTS;
+using ::fesetenv _LIBCPP_USING_IF_EXISTS;
+using ::feupdateenv _LIBCPP_USING_IF_EXISTS;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CFENV
diff --git a/libcxx/include/__cxx03/cfloat b/libcxx/include/__cxx03/cfloat
new file mode 100644
index 000000000000..5d1b38c557dc
--- /dev/null
+++ b/libcxx/include/__cxx03/cfloat
@@ -0,0 +1,88 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CFLOAT
+#define _LIBCPP_CFLOAT
+
+/*
+    cfloat synopsis
+
+Macros:
+
+    FLT_ROUNDS
+    FLT_EVAL_METHOD     // C99
+    FLT_RADIX
+
+    FLT_HAS_SUBNORM     // C11
+    DBL_HAS_SUBNORM     // C11
+    LDBL_HAS_SUBNORM    // C11
+
+    FLT_MANT_DIG
+    DBL_MANT_DIG
+    LDBL_MANT_DIG
+
+    DECIMAL_DIG         // C99
+    FLT_DECIMAL_DIG     // C11
+    DBL_DECIMAL_DIG     // C11
+    LDBL_DECIMAL_DIG    // C11
+
+    FLT_DIG
+    DBL_DIG
+    LDBL_DIG
+
+    FLT_MIN_EXP
+    DBL_MIN_EXP
+    LDBL_MIN_EXP
+
+    FLT_MIN_10_EXP
+    DBL_MIN_10_EXP
+    LDBL_MIN_10_EXP
+
+    FLT_MAX_EXP
+    DBL_MAX_EXP
+    LDBL_MAX_EXP
+
+    FLT_MAX_10_EXP
+    DBL_MAX_10_EXP
+    LDBL_MAX_10_EXP
+
+    FLT_MAX
+    DBL_MAX
+    LDBL_MAX
+
+    FLT_EPSILON
+    DBL_EPSILON
+    LDBL_EPSILON
+
+    FLT_MIN
+    DBL_MIN
+    LDBL_MIN
+
+    FLT_TRUE_MIN        // C11
+    DBL_TRUE_MIN        // C11
+    LDBL_TRUE_MIN       // C11
+*/
+
+#include <__config>
+
+#include <float.h>
+
+#ifndef _LIBCPP_FLOAT_H
+#   error <cfloat> tried including <float.h> but didn't find libc++'s <float.h> header. \
+          This usually means that your header search paths are not configured properly. \
+          The header search paths should contain the C++ Standard Library headers before \
+          any C Standard Library, and you are probably using compiler flags that make that \
+          not be the case.
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#endif // _LIBCPP_CFLOAT
diff --git a/libcxx/include/__cxx03/charconv b/libcxx/include/__cxx03/charconv
new file mode 100644
index 000000000000..a2e270e9316d
--- /dev/null
+++ b/libcxx/include/__cxx03/charconv
@@ -0,0 +1,115 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CHARCONV
+#define _LIBCPP_CHARCONV
+
+/*
+    charconv synopsis
+
+namespace std {
+
+  // floating-point format for primitive numerical conversion
+  enum class chars_format {
+    scientific = unspecified,
+    fixed = unspecified,
+    hex = unspecified,
+    general = fixed | scientific
+  };
+
+  // 23.20.2, primitive numerical output conversion
+  struct to_chars_result {
+    char* ptr;
+    errc ec;
+    friend bool operator==(const to_chars_result&, const to_chars_result&) = default; // since C++20
+    constexpr explicit operator bool() const noexcept { return ec == errc{}; }        // since C++26
+  };
+
+  constexpr to_chars_result to_chars(char* first, char* last, see below value,
+                                     int base = 10);                                  // constexpr since C++23
+  to_chars_result to_chars(char* first, char* last, bool value,
+                           int base = 10) = delete;
+
+  to_chars_result to_chars(char* first, char* last, float value);
+  to_chars_result to_chars(char* first, char* last, double value);
+  to_chars_result to_chars(char* first, char* last, long double value);
+
+  to_chars_result to_chars(char* first, char* last, float value,
+                           chars_format fmt);
+  to_chars_result to_chars(char* first, char* last, double value,
+                           chars_format fmt);
+  to_chars_result to_chars(char* first, char* last, long double value,
+                           chars_format fmt);
+
+  to_chars_result to_chars(char* first, char* last, float value,
+                           chars_format fmt, int precision);
+  to_chars_result to_chars(char* first, char* last, double value,
+                           chars_format fmt, int precision);
+  to_chars_result to_chars(char* first, char* last, long double value,
+                           chars_format fmt, int precision);
+
+  // 23.20.3, primitive numerical input conversion
+  struct from_chars_result {
+    const char* ptr;
+    errc ec;
+    friend bool operator==(const from_chars_result&, const from_chars_result&) = default; // since C++20
+    constexpr explicit operator bool() const noexcept { return ec == errc{}; }            // since C++26
+  };
+
+  constexpr from_chars_result from_chars(const char* first, const char* last,
+                               see below& value, int base = 10);                         // constexpr since C++23
+
+} // namespace std
+
+*/
+
+#include <__config>
+
+#if _LIBCPP_STD_VER >= 17
+#  include <__charconv/chars_format.h>
+#  include <__charconv/from_chars_integral.h>
+#  include <__charconv/from_chars_result.h>
+#  include <__charconv/tables.h>
+#  include <__charconv/to_chars.h>
+#  include <__charconv/to_chars_base_10.h>
+#  include <__charconv/to_chars_floating_point.h>
+#  include <__charconv/to_chars_integral.h>
+#  include <__charconv/to_chars_result.h>
+#  include <__charconv/traits.h>
+#endif // _LIBCPP_STD_VER >= 17
+
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 14
+#  include <cerrno>
+#  include <cstddef>
+#  include <initializer_list>
+#  include <new>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cmath>
+#  include <concepts>
+#  include <cstdint>
+#  include <cstdlib>
+#  include <cstring>
+#  include <iosfwd>
+#  include <limits>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_CHARCONV
diff --git a/libcxx/include/__cxx03/chrono b/libcxx/include/__cxx03/chrono
new file mode 100644
index 000000000000..7bec5e5a26ef
--- /dev/null
+++ b/libcxx/include/__cxx03/chrono
@@ -0,0 +1,1022 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CHRONO
+#define _LIBCPP_CHRONO
+
+// clang-format off
+
+/*
+    chrono synopsis
+
+#include <compare> // C++20
+
+namespace std
+{
+namespace chrono
+{
+
+template <class ToDuration, class Rep, class Period>
+constexpr
+ToDuration
+duration_cast(const duration<Rep, Period>& fd);
+
+template <class Rep> struct treat_as_floating_point : is_floating_point<Rep> {};
+
+template <class Rep> inline constexpr bool treat_as_floating_point_v
+    = treat_as_floating_point<Rep>::value;                       // C++17
+
+template <class Rep>
+struct duration_values
+{
+public:
+    static constexpr Rep zero(); // noexcept in C++20
+    static constexpr Rep max();  // noexcept in C++20
+    static constexpr Rep min();  // noexcept in C++20
+};
+
+// duration
+
+template <class Rep, class Period = ratio<1>>
+class duration
+{
+    static_assert(!__is_duration<Rep>::value, "A duration representation can not be a duration");
+    static_assert(__is_ratio<Period>::value, "Second template parameter of duration must be a std::ratio");
+    static_assert(Period::num > 0, "duration period must be positive");
+public:
+    typedef Rep rep;
+    typedef typename _Period::type period;
+
+    constexpr duration() = default;
+    template <class Rep2>
+        constexpr explicit duration(const Rep2& r,
+            typename enable_if
+            <
+               is_convertible<const Rep2&, rep>::value &&
+               (treat_as_floating_point<rep>::value ||
+               !treat_as_floating_point<rep>::value && !treat_as_floating_point<Rep2>::value)
+            >::type* = 0);
+
+    // conversions
+    template <class Rep2, class Period2>
+        constexpr duration(const duration<Rep2, Period2>& d,
+            typename enable_if
+            <
+                treat_as_floating_point<rep>::value ||
+                ratio_divide<Period2, period>::type::den == 1
+            >::type* = 0);
+
+    // observer
+
+    constexpr rep count() const;
+
+    // arithmetic
+
+    constexpr common_type<duration>::type  operator+() const;
+    constexpr common_type<duration>::type  operator-() const;
+    constexpr duration& operator++();    // constexpr in C++17
+    constexpr duration  operator++(int); // constexpr in C++17
+    constexpr duration& operator--();    // constexpr in C++17
+    constexpr duration  operator--(int); // constexpr in C++17
+
+    constexpr duration& operator+=(const duration& d);  // constexpr in C++17
+    constexpr duration& operator-=(const duration& d);  // constexpr in C++17
+
+    duration& operator*=(const rep& rhs);       // constexpr in C++17
+    duration& operator/=(const rep& rhs);       // constexpr in C++17
+    duration& operator%=(const rep& rhs);       // constexpr in C++17
+    duration& operator%=(const duration& rhs);  // constexpr in C++17
+
+    // special values
+
+    static constexpr duration zero(); // noexcept in C++20
+    static constexpr duration min();  // noexcept in C++20
+    static constexpr duration max();  // noexcept in C++20
+};
+
+typedef duration<long long,         nano> nanoseconds;
+typedef duration<long long,        micro> microseconds;
+typedef duration<long long,        milli> milliseconds;
+typedef duration<long long              > seconds;
+typedef duration<     long, ratio<  60> > minutes;
+typedef duration<     long, ratio<3600> > hours;
+
+template <class Clock, class Duration = typename Clock::duration>
+class time_point
+{
+public:
+    typedef Clock                     clock;
+    typedef Duration                  duration;
+    typedef typename duration::rep    rep;
+    typedef typename duration::period period;
+private:
+    duration d_;  // exposition only
+
+public:
+    time_point();  // has value "epoch" // constexpr in C++14
+    explicit time_point(const duration& d);  // same as time_point() + d // constexpr in C++14
+
+    // conversions
+    template <class Duration2>
+       time_point(const time_point<clock, Duration2>& t); // constexpr in C++14
+
+    // observer
+
+    duration time_since_epoch() const; // constexpr in C++14
+
+    // arithmetic
+
+    time_point& operator+=(const duration& d); // constexpr in C++17
+    time_point& operator-=(const duration& d); // constexpr in C++17
+
+    // special values
+
+    static constexpr time_point min();  // noexcept in C++20
+    static constexpr time_point max();  // noexcept in C++20
+};
+
+} // chrono
+
+// common_type traits
+template <class Rep1, class Period1, class Rep2, class Period2>
+  struct common_type<chrono::duration<Rep1, Period1>, chrono::duration<Rep2, Period2>>;
+
+template <class Clock, class Duration1, class Duration2>
+  struct common_type<chrono::time_point<Clock, Duration1>, chrono::time_point<Clock, Duration2>>;
+
+namespace chrono {
+
+// duration arithmetic
+template <class Rep1, class Period1, class Rep2, class Period2>
+  constexpr
+  typename common_type<duration<Rep1, Period1>, duration<Rep2, Period2>>::type
+  operator+(const duration<Rep1, Period1>& lhs, const duration<Rep2, Period2>& rhs);
+template <class Rep1, class Period1, class Rep2, class Period2>
+  constexpr
+  typename common_type<duration<Rep1, Period1>, duration<Rep2, Period2>>::type
+  operator-(const duration<Rep1, Period1>& lhs, const duration<Rep2, Period2>& rhs);
+template <class Rep1, class Period, class Rep2>
+  constexpr
+  duration<typename common_type<Rep1, Rep2>::type, Period>
+  operator*(const duration<Rep1, Period>& d, const Rep2& s);
+template <class Rep1, class Period, class Rep2>
+  constexpr
+  duration<typename common_type<Rep1, Rep2>::type, Period>
+  operator*(const Rep1& s, const duration<Rep2, Period>& d);
+template <class Rep1, class Period, class Rep2>
+  constexpr
+  duration<typename common_type<Rep1, Rep2>::type, Period>
+  operator/(const duration<Rep1, Period>& d, const Rep2& s);
+template <class Rep1, class Period1, class Rep2, class Period2>
+  constexpr
+  typename common_type<Rep1, Rep2>::type
+  operator/(const duration<Rep1, Period1>& lhs, const duration<Rep2, Period2>& rhs);
+
+// duration comparisons
+template <class Rep1, class Period1, class Rep2, class Period2>
+   constexpr
+   bool operator==(const duration<Rep1, Period1>& lhs, const duration<Rep2, Period2>& rhs);
+template <class Rep1, class Period1, class Rep2, class Period2>
+   constexpr
+   bool operator!=(const duration<Rep1, Period1>& lhs, const duration<Rep2, Period2>& rhs); // removed in C++20
+template <class Rep1, class Period1, class Rep2, class Period2>
+   constexpr
+   bool operator< (const duration<Rep1, Period1>& lhs, const duration<Rep2, Period2>& rhs);
+template <class Rep1, class Period1, class Rep2, class Period2>
+   constexpr
+   bool operator<=(const duration<Rep1, Period1>& lhs, const duration<Rep2, Period2>& rhs);
+template <class Rep1, class Period1, class Rep2, class Period2>
+   constexpr
+   bool operator> (const duration<Rep1, Period1>& lhs, const duration<Rep2, Period2>& rhs);
+template <class Rep1, class Period1, class Rep2, class Period2>
+   constexpr
+   bool operator>=(const duration<Rep1, Period1>& lhs, const duration<Rep2, Period2>& rhs);
+template<class Rep1, class Period1, class Rep2, class Period2>
+  requires three_way_comparable<typename CT::rep>
+  constexpr auto operator<=>(const duration<Rep1, Period1>& lhs,
+                             const duration<Rep2, Period2>& rhs);                           // since C++20
+
+// duration_cast
+template <class ToDuration, class Rep, class Period>
+  ToDuration duration_cast(const duration<Rep, Period>& d);
+
+template <class ToDuration, class Rep, class Period>
+    constexpr ToDuration floor(const duration<Rep, Period>& d);    // C++17
+template <class ToDuration, class Rep, class Period>
+    constexpr ToDuration ceil(const duration<Rep, Period>& d);     // C++17
+template <class ToDuration, class Rep, class Period>
+    constexpr ToDuration round(const duration<Rep, Period>& d);    // C++17
+
+// duration I/O
+template<class charT, class traits, class Rep, class Period>       // C++20
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const duration<Rep, Period>& d);
+
+// time_point arithmetic (all constexpr in C++14)
+template <class Clock, class Duration1, class Rep2, class Period2>
+  time_point<Clock, typename common_type<Duration1, duration<Rep2, Period2>>::type>
+  operator+(const time_point<Clock, Duration1>& lhs, const duration<Rep2, Period2>& rhs);
+template <class Rep1, class Period1, class Clock, class Duration2>
+  time_point<Clock, typename common_type<duration<Rep1, Period1>, Duration2>::type>
+  operator+(const duration<Rep1, Period1>& lhs, const time_point<Clock, Duration2>& rhs);
+template <class Clock, class Duration1, class Rep2, class Period2>
+  time_point<Clock, typename common_type<Duration1, duration<Rep2, Period2>>::type>
+  operator-(const time_point<Clock, Duration1>& lhs, const duration<Rep2, Period2>& rhs);
+template <class Clock, class Duration1, class Duration2>
+  typename common_type<Duration1, Duration2>::type
+  operator-(const time_point<Clock, Duration1>& lhs, const time_point<Clock, Duration2>& rhs);
+
+// time_point comparisons (all constexpr in C++14)
+template <class Clock, class Duration1, class Duration2>
+   bool operator==(const time_point<Clock, Duration1>& lhs, const time_point<Clock, Duration2>& rhs);
+template <class Clock, class Duration1, class Duration2>
+   bool operator!=(const time_point<Clock, Duration1>& lhs, const time_point<Clock, Duration2>& rhs); // removed in C++20
+template <class Clock, class Duration1, class Duration2>
+   bool operator< (const time_point<Clock, Duration1>& lhs, const time_point<Clock, Duration2>& rhs);
+template <class Clock, class Duration1, class Duration2>
+   bool operator<=(const time_point<Clock, Duration1>& lhs, const time_point<Clock, Duration2>& rhs);
+template <class Clock, class Duration1, class Duration2>
+   bool operator> (const time_point<Clock, Duration1>& lhs, const time_point<Clock, Duration2>& rhs);
+template <class Clock, class Duration1, class Duration2>
+   bool operator>=(const time_point<Clock, Duration1>& lhs, const time_point<Clock, Duration2>& rhs);
+template<class Clock, class Duration1,
+         three_way_comparable_with<Duration1> Duration2>
+  constexpr auto operator<=>(const time_point<Clock, Duration1>& lhs,
+                             const time_point<Clock, Duration2>& rhs);                                // since C++20
+
+// time_point_cast (constexpr in C++14)
+
+template <class ToDuration, class Clock, class Duration>
+  time_point<Clock, ToDuration> time_point_cast(const time_point<Clock, Duration>& t);
+
+template <class ToDuration, class Clock, class Duration>
+    constexpr time_point<Clock, ToDuration>
+    floor(const time_point<Clock, Duration>& tp);                  // C++17
+
+template <class ToDuration, class Clock, class Duration>
+    constexpr time_point<Clock, ToDuration>
+    ceil(const time_point<Clock, Duration>& tp);                   // C++17
+
+template <class ToDuration, class Clock, class Duration>
+    constexpr time_point<Clock, ToDuration>
+    round(const time_point<Clock, Duration>& tp);                  // C++17
+
+template <class Rep, class Period>
+    constexpr duration<Rep, Period> abs(duration<Rep, Period> d);  // C++17
+
+// Clocks
+
+class system_clock
+{
+public:
+    typedef microseconds                     duration;
+    typedef duration::rep                    rep;
+    typedef duration::period                 period;
+    typedef chrono::time_point<system_clock> time_point;
+    static const bool is_steady =            false; // constexpr in C++14
+
+    static time_point now() noexcept;
+    static time_t     to_time_t  (const time_point& __t) noexcept;
+    static time_point from_time_t(time_t __t) noexcept;
+};
+
+template <class Duration>
+  using sys_time  = time_point<system_clock, Duration>; // C++20
+using sys_seconds = sys_time<seconds>;                  // C++20
+using sys_days    = sys_time<days>;                     // C++20
+
+template<class charT, class traits, class Duration>     // C++20
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const sys_time<Duration>& tp);
+
+template<class charT, class traits>                    // C++20
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const sys_days& dp);
+
+class file_clock                                        // C++20
+{
+public:
+    typedef see-below                      rep;
+    typedef nano                           period;
+    typedef chrono::duration<rep, period>  duration;
+    typedef chrono::time_point<file_clock> time_point;
+    static constexpr bool is_steady =      false;
+
+    static time_point now() noexcept;
+
+    template<class Duration>
+    static sys_time<see-below> to_sys(const file_time<Duration>&);
+
+    template<class Duration>
+    static file_time<see-below> from_sys(const sys_time<Duration>&);
+};
+
+template<class Duration>
+  using file_time = time_point<file_clock, Duration>;   // C++20
+
+template<class charT, class traits, class Duration>     // C++20
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const file_time<Duration>& tp);
+
+class steady_clock
+{
+public:
+    typedef nanoseconds                                   duration;
+    typedef duration::rep                                 rep;
+    typedef duration::period                              period;
+    typedef chrono::time_point<steady_clock, duration>    time_point;
+    static const bool is_steady =                         true; // constexpr in C++14
+
+    static time_point now() noexcept;
+};
+
+typedef steady_clock high_resolution_clock;
+
+// 25.7.8, local time           // C++20
+struct local_t {};
+template<class Duration>
+  using local_time  = time_point<local_t, Duration>;
+using local_seconds = local_time<seconds>;
+using local_days    = local_time<days>;
+
+template<class charT, class traits, class Duration>     // C++20
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const local_time<Duration>& tp);
+
+// 25.8.2, class last_spec    // C++20
+struct last_spec;
+
+// 25.8.3, class day          // C++20
+
+class day;
+constexpr bool operator==(const day& x, const day& y) noexcept;
+constexpr strong_ordering operator<=>(const day& x, const day& y) noexcept;
+constexpr day  operator+(const day&  x, const days& y) noexcept;
+constexpr day  operator+(const days& x, const day&  y) noexcept;
+constexpr day  operator-(const day&  x, const days& y) noexcept;
+constexpr days operator-(const day&  x, const day&  y) noexcept;
+template<class charT, class traits>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const day& d);
+
+// 25.8.4, class month    // C++20
+class month;
+constexpr bool operator==(const month& x, const month& y) noexcept;
+constexpr strong_ordering operator<=>(const month& x, const month& y) noexcept;
+
+constexpr month  operator+(const month&  x, const months& y) noexcept;
+constexpr month  operator+(const months& x,  const month& y) noexcept;
+constexpr month  operator-(const month&  x, const months& y) noexcept;
+constexpr months operator-(const month&  x,  const month& y) noexcept;
+template<class charT, class traits>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const month& m);
+
+// 25.8.5, class year    // C++20
+class year;
+constexpr bool operator==(const year& x, const year& y) noexcept;
+constexpr strong_ordering operator<=>(const year& x, const year& y) noexcept;
+
+constexpr year  operator+(const year&  x, const years& y) noexcept;
+constexpr year  operator+(const years& x, const year&  y) noexcept;
+constexpr year  operator-(const year&  x, const years& y) noexcept;
+constexpr years operator-(const year&  x, const year&  y) noexcept;
+template<class charT, class traits>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const year& y);
+
+// 25.8.6, class weekday    // C++20
+class weekday;
+
+constexpr bool operator==(const weekday& x, const weekday& y) noexcept;
+constexpr weekday operator+(const weekday& x, const days&    y) noexcept;
+constexpr weekday operator+(const days&    x, const weekday& y) noexcept;
+constexpr weekday operator-(const weekday& x, const days&    y) noexcept;
+constexpr days    operator-(const weekday& x, const weekday& y) noexcept;
+template<class charT, class traits>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const weekday& wd);
+
+// 25.8.7, class weekday_indexed    // C++20
+
+class weekday_indexed;
+constexpr bool operator==(const weekday_indexed& x, const weekday_indexed& y) noexcept;
+
+template<class charT, class traits>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const weekday_indexed& wdi);
+
+// 25.8.8, class weekday_last    // C++20
+class weekday_last;
+
+constexpr bool operator==(const weekday_last& x, const weekday_last& y) noexcept;
+
+template<class charT, class traits>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const weekday_last& wdl);
+
+// 25.8.9, class month_day    // C++20
+class month_day;
+
+constexpr bool operator==(const month_day& x, const month_day& y) noexcept;
+constexpr strong_ordering operator<=>(const month_day& x, const month_day& y) noexcept;
+
+template<class charT, class traits>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const month_day& md);
+
+// 25.8.10, class month_day_last    // C++20
+class month_day_last;
+
+constexpr bool operator==(const month_day_last& x, const month_day_last& y) noexcept;
+constexpr strong_ordering operator<=>(const month_day_last& x, const month_day_last& y) noexcept;
+
+template<class charT, class traits>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const month_day_last& mdl);
+
+// 25.8.11, class month_weekday    // C++20
+class month_weekday;
+
+constexpr bool operator==(const month_weekday& x, const month_weekday& y) noexcept;
+
+template<class charT, class traits>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const month_weekday& mwd);
+
+// 25.8.12, class month_weekday_last    // C++20
+class month_weekday_last;
+
+constexpr bool operator==(const month_weekday_last& x, const month_weekday_last& y) noexcept;
+
+template<class charT, class traits>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const month_weekday_last& mwdl);
+
+
+// 25.8.13, class year_month    // C++20
+class year_month;
+
+constexpr bool operator==(const year_month& x, const year_month& y) noexcept;
+constexpr strong_ordering operator<=>(const year_month& x, const year_month& y) noexcept;
+
+constexpr year_month operator+(const year_month& ym, const months& dm) noexcept;
+constexpr year_month operator+(const months& dm, const year_month& ym) noexcept;
+constexpr year_month operator-(const year_month& ym, const months& dm) noexcept;
+constexpr months operator-(const year_month& x, const year_month& y) noexcept;
+constexpr year_month operator+(const year_month& ym, const years& dy) noexcept;
+constexpr year_month operator+(const years& dy, const year_month& ym) noexcept;
+constexpr year_month operator-(const year_month& ym, const years& dy) noexcept;
+
+template<class charT, class traits>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const year_month& ym);
+
+// 25.8.14, class year_month_day class    // C++20
+year_month_day;
+
+constexpr bool operator==(const year_month_day& x, const year_month_day& y) noexcept;
+constexpr strong_ordering operator<=>(const year_month_day& x, const year_month_day& y) noexcept;
+
+constexpr year_month_day operator+(const year_month_day& ymd, const months& dm) noexcept;
+constexpr year_month_day operator+(const months& dm, const year_month_day& ymd) noexcept;
+constexpr year_month_day operator+(const year_month_day& ymd, const years& dy) noexcept;
+constexpr year_month_day operator+(const years& dy, const year_month_day& ymd) noexcept;
+constexpr year_month_day operator-(const year_month_day& ymd, const months& dm) noexcept;
+constexpr year_month_day operator-(const year_month_day& ymd, const years& dy) noexcept;
+
+template<class charT, class traits>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const year_month_day& ymd);
+
+// 25.8.15, class year_month_day_last    // C++20
+class year_month_day_last;
+
+constexpr bool operator==(const year_month_day_last& x, const year_month_day_last& y) noexcept;
+constexpr strong_ordering operator<=>(const year_month_day_last_day& x, const year_month_day_last_day& y) noexcept;
+
+constexpr year_month_day_last
+  operator+(const year_month_day_last& ymdl, const months& dm) noexcept;
+constexpr year_month_day_last
+  operator+(const months& dm, const year_month_day_last& ymdl) noexcept;
+constexpr year_month_day_last
+  operator+(const year_month_day_last& ymdl, const years& dy) noexcept;
+constexpr year_month_day_last
+  operator+(const years& dy, const year_month_day_last& ymdl) noexcept;
+constexpr year_month_day_last
+  operator-(const year_month_day_last& ymdl, const months& dm) noexcept;
+constexpr year_month_day_last
+  operator-(const year_month_day_last& ymdl, const years& dy) noexcept;
+
+template<class charT, class traits>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const year_month_day_last& ymdl);
+
+// 25.8.16, class year_month_weekday    // C++20
+class year_month_weekday;
+
+constexpr bool operator==(const year_month_weekday& x,
+                          const year_month_weekday& y) noexcept;
+
+constexpr year_month_weekday
+  operator+(const year_month_weekday& ymwd, const months& dm) noexcept;
+constexpr year_month_weekday
+  operator+(const months& dm, const year_month_weekday& ymwd) noexcept;
+constexpr year_month_weekday
+  operator+(const year_month_weekday& ymwd, const years& dy) noexcept;
+constexpr year_month_weekday
+  operator+(const years& dy, const year_month_weekday& ymwd) noexcept;
+constexpr year_month_weekday
+  operator-(const year_month_weekday& ymwd, const months& dm) noexcept;
+constexpr year_month_weekday
+  operator-(const year_month_weekday& ymwd, const years& dy) noexcept;
+
+template<class charT, class traits>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const year_month_weekday& ymwd);
+
+// 25.8.17, class year_month_weekday_last    // C++20
+class year_month_weekday_last;
+
+constexpr bool operator==(const year_month_weekday_last& x,
+                          const year_month_weekday_last& y) noexcept;
+constexpr year_month_weekday_last
+  operator+(const year_month_weekday_last& ymwdl, const months& dm) noexcept;
+constexpr year_month_weekday_last
+  operator+(const months& dm, const year_month_weekday_last& ymwdl) noexcept;
+constexpr year_month_weekday_last
+  operator+(const year_month_weekday_last& ymwdl, const years& dy) noexcept;
+constexpr year_month_weekday_last
+  operator+(const years& dy, const year_month_weekday_last& ymwdl) noexcept;
+constexpr year_month_weekday_last
+  operator-(const year_month_weekday_last& ymwdl, const months& dm) noexcept;
+constexpr year_month_weekday_last
+  operator-(const year_month_weekday_last& ymwdl, const years& dy) noexcept;
+
+template<class charT, class traits>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const year_month_weekday_last& ymwdl);
+
+// 25.8.18, civil calendar conventional syntax operators    // C++20
+constexpr year_month
+  operator/(const year& y, const month& m) noexcept;
+constexpr year_month
+  operator/(const year& y, int m) noexcept;
+constexpr month_day
+  operator/(const month& m, const day& d) noexcept;
+constexpr month_day
+  operator/(const month& m, int d) noexcept;
+constexpr month_day
+  operator/(int m, const day& d) noexcept;
+constexpr month_day
+  operator/(const day& d, const month& m) noexcept;
+constexpr month_day
+  operator/(const day& d, int m) noexcept;
+constexpr month_day_last
+  operator/(const month& m, last_spec) noexcept;
+constexpr month_day_last
+  operator/(int m, last_spec) noexcept;
+constexpr month_day_last
+  operator/(last_spec, const month& m) noexcept;
+constexpr month_day_last
+  operator/(last_spec, int m) noexcept;
+constexpr month_weekday
+  operator/(const month& m, const weekday_indexed& wdi) noexcept;
+constexpr month_weekday
+  operator/(int m, const weekday_indexed& wdi) noexcept;
+constexpr month_weekday
+  operator/(const weekday_indexed& wdi, const month& m) noexcept;
+constexpr month_weekday
+  operator/(const weekday_indexed& wdi, int m) noexcept;
+constexpr month_weekday_last
+  operator/(const month& m, const weekday_last& wdl) noexcept;
+constexpr month_weekday_last
+  operator/(int m, const weekday_last& wdl) noexcept;
+constexpr month_weekday_last
+  operator/(const weekday_last& wdl, const month& m) noexcept;
+constexpr month_weekday_last
+  operator/(const weekday_last& wdl, int m) noexcept;
+constexpr year_month_day
+  operator/(const year_month& ym, const day& d) noexcept;
+constexpr year_month_day
+  operator/(const year_month& ym, int d) noexcept;
+constexpr year_month_day
+  operator/(const year& y, const month_day& md) noexcept;
+constexpr year_month_day
+  operator/(int y, const month_day& md) noexcept;
+constexpr year_month_day
+  operator/(const month_day& md, const year& y) noexcept;
+constexpr year_month_day
+  operator/(const month_day& md, int y) noexcept;
+constexpr year_month_day_last
+  operator/(const year_month& ym, last_spec) noexcept;
+constexpr year_month_day_last
+  operator/(const year& y, const month_day_last& mdl) noexcept;
+constexpr year_month_day_last
+  operator/(int y, const month_day_last& mdl) noexcept;
+constexpr year_month_day_last
+  operator/(const month_day_last& mdl, const year& y) noexcept;
+constexpr year_month_day_last
+  operator/(const month_day_last& mdl, int y) noexcept;
+constexpr year_month_weekday
+  operator/(const year_month& ym, const weekday_indexed& wdi) noexcept;
+constexpr year_month_weekday
+  operator/(const year& y, const month_weekday& mwd) noexcept;
+constexpr year_month_weekday
+  operator/(int y, const month_weekday& mwd) noexcept;
+constexpr year_month_weekday
+  operator/(const month_weekday& mwd, const year& y) noexcept;
+constexpr year_month_weekday
+  operator/(const month_weekday& mwd, int y) noexcept;
+constexpr year_month_weekday_last
+  operator/(const year_month& ym, const weekday_last& wdl) noexcept;
+constexpr year_month_weekday_last
+  operator/(const year& y, const month_weekday_last& mwdl) noexcept;
+constexpr year_month_weekday_last
+  operator/(int y, const month_weekday_last& mwdl) noexcept;
+constexpr year_month_weekday_last
+  operator/(const month_weekday_last& mwdl, const year& y) noexcept;
+constexpr year_month_weekday_last
+  operator/(const month_weekday_last& mwdl, int y) noexcept;
+
+// 26.9, class template hh_mm_ss
+template <class Duration>
+class hh_mm_ss
+{
+    bool            is_neg; // exposition only
+    chrono::hours   h;      // exposition only
+    chrono::minutes m;      // exposition only
+    chrono::seconds s;      // exposition only
+    precision       ss;     // exposition only
+
+public:
+    static unsigned constexpr fractional_width = see below;
+    using precision                            = see below;
+
+    constexpr hh_mm_ss() noexcept : hh_mm_ss{Duration::zero()} {}
+    constexpr explicit hh_mm_ss(Duration d) noexcept;
+
+    constexpr bool is_negative() const noexcept;
+    constexpr chrono::hours hours() const noexcept;
+    constexpr chrono::minutes minutes() const noexcept;
+    constexpr chrono::seconds seconds() const noexcept;
+    constexpr precision subseconds() const noexcept;
+
+    constexpr explicit operator  precision()   const noexcept;
+    constexpr          precision to_duration() const noexcept;
+};
+
+template<class charT, class traits, class Duration>
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const hh_mm_ss<Duration>& hms); // C++20
+
+// 26.10, 12/24 hour functions
+constexpr bool is_am(hours const& h) noexcept;
+constexpr bool is_pm(hours const& h) noexcept;
+constexpr hours make12(const hours& h) noexcept;
+constexpr hours make24(const hours& h, bool is_pm) noexcept;
+
+// [time.zone.db], time zone database
+struct tzdb {                                                                    // C++20
+  string                 version;
+  vector<time_zone>      zones;
+  vector<time_zone_link> links;
+  vector<leap_second>    leap_seconds;
+
+  const time_zone* locate_zone(string_view tz_name) const;
+  const time_zone* current_zone() const;
+};
+
+class tzdb_list {                                                                // C++20
+public:
+  tzdb_list(const tzdb_list&) = delete;
+  tzdb_list& operator=(const tzdb_list&) = delete;
+
+  // unspecified additional constructors
+
+  class const_iterator;
+
+  const tzdb& front() const noexcept;
+
+  const_iterator erase_after(const_iterator p);
+
+  const_iterator begin() const noexcept;
+  const_iterator end()   const noexcept;
+
+  const_iterator cbegin() const noexcept;
+  const_iterator cend()   const noexcept;
+};
+
+// [time.zone.db.access], time zone database access
+const tzdb& get_tzdb();                                                          // C++20
+tzdb_list& get_tzdb_list();                                                      // C++20
+const time_zone* locate_zone(string_view tz_name);                               // C++20
+const time_zone* current_zone()                                                  // C++20
+
+// [time.zone.db.remote], remote time zone database support
+const tzdb& reload_tzdb();                                                       // C++20
+string remote_version();                                                         // C++20
+
+// [time.zone.exception], exception classes
+class nonexistent_local_time;                                                    // C++20
+class ambiguous_local_time;                                                      // C++20
+
+// [time.zone.info], information classes
+struct sys_info {                                                                // C++20
+  sys_seconds   begin;
+  sys_seconds   end;
+  seconds       offset;
+  minutes       save;
+  string        abbrev;
+};
+
+template<class charT, class traits>                                              // C++20
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const sys_info& si);
+
+struct local_info {                                                             // C++20
+  static constexpr int unique      = 0;
+  static constexpr int nonexistent = 1;
+  static constexpr int ambiguous   = 2;
+
+  int result;
+  sys_info first;
+  sys_info second;
+};
+
+template<class charT, class traits>                                              // C++20
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const local_info& li);
+
+// 25.10.5, class time_zone                                                      // C++20
+enum class choose {earliest, latest};
+class time_zone {
+  time_zone(time_zone&&) = default;
+  time_zone& operator=(time_zone&&) = default;
+
+  // unspecified additional constructors
+
+  string_view name() const noexcept;
+
+  template<class Duration>
+  sys_info get_info(const sys_time<Duration>& st) const;
+
+  template<class Duration>
+  local_info get_info(const local_time<Duration>& tp) const;
+
+  template<class Duration>
+  sys_time<common_type_t<Duration, seconds>>
+    to_sys(const local_time<Duration>& tp) const;
+
+  template<class Duration>
+  sys_time<common_type_t<Duration, seconds>>
+    to_sys(const local_time<Duration>& tp, choose z) const;
+
+  template<class Duration>
+  local_time<common_type_t<Duration, seconds>>
+    to_local(const sys_time<Duration>& tp) const;
+};
+bool operator==(const time_zone& x, const time_zone& y) noexcept;                // C++20
+strong_ordering operator<=>(const time_zone& x, const time_zone& y) noexcept;    // C++20
+
+// [time.zone.zonedtraits], class template zoned_traits
+template<class T> struct zoned_traits;                                           // C++20
+
+// [time.zone.zonedtime], class template zoned_time
+template<class Duration, class TimeZonePtr = const time_zone*>                   // C++20
+class zoned_time;
+
+using zoned_seconds = zoned_time<seconds>;                                       // C++20
+
+template<class Duration1, class Duration2, class TimeZonePtr>                    // C++20
+  bool operator==(const zoned_time<Duration1, TimeZonePtr>& x,
+                  const zoned_time<Duration2, TimeZonePtr>& y);
+
+template<class charT, class traits, class Duration, class TimeZonePtr>           // C++20
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const zoned_time<Duration, TimeZonePtr>& t);
+
+// [time.zone.leap], leap second support
+class leap_second {                                                              // C++20
+public:
+  leap_second(const leap_second&)            = default;
+  leap_second& operator=(const leap_second&) = default;
+
+  // unspecified additional constructors
+
+  constexpr sys_seconds date() const noexcept;
+  constexpr seconds value() const noexcept;
+};
+
+constexpr bool operator==(const leap_second& x, const leap_second& y);           // C++20
+constexpr strong_ordering operator<=>(const leap_second& x, const leap_second& y);
+
+template<class Duration>                                                         // C++20
+  constexpr bool operator==(const leap_second& x, const sys_time<Duration>& y);
+template<class Duration>                                                         // C++20
+  constexpr bool operator< (const leap_second& x, const sys_time<Duration>& y);
+template<class Duration>                                                         // C++20
+  constexpr bool operator< (const sys_time<Duration>& x, const leap_second& y);
+template<class Duration>                                                         // C++20
+  constexpr bool operator> (const leap_second& x, const sys_time<Duration>& y);
+template<class Duration>                                                         // C++20
+  constexpr bool operator> (const sys_time<Duration>& x, const leap_second& y);
+template<class Duration>                                                         // C++20
+  constexpr bool operator<=(const leap_second& x, const sys_time<Duration>& y);
+template<class Duration>                                                         // C++20
+  constexpr bool operator<=(const sys_time<Duration>& x, const leap_second& y);
+template<class Duration>                                                         // C++20
+  constexpr bool operator>=(const leap_second& x, const sys_time<Duration>& y);
+template<class Duration>                                                         // C++20
+  constexpr bool operator>=(const sys_time<Duration>& x, const leap_second& y);
+template<class Duration>                                                         // C++20
+  requires three_way_comparable_with<sys_seconds, sys_time<Duration>>
+  constexpr auto operator<=>(const leap_second& x, const sys_time<Duration>& y);
+
+// [time.zone.link], class time_zone_link
+class time_zone_link {                                                           // C++20
+public:
+  time_zone_link(time_zone_link&&)            = default;
+  time_zone_link& operator=(time_zone_link&&) = default;
+
+  // unspecified additional constructors
+
+  string_view name()   const noexcept;
+  string_view target() const noexcept;
+};
+
+bool operator==(const time_zone_link& x, const time_zone_link& y);               // C++20
+strong_ordering operator<=>(const time_zone_link& x, const time_zone_link& y);   // C++20
+
+}  // chrono
+
+namespace std {
+  template<class Duration, class charT>
+    struct formatter<chrono::sys_time<Duration>, charT>;                          // C++20
+  template<class Duration, class charT>
+    struct formatter<chrono::filetime<Duration>, charT>;                          // C++20
+  template<class Duration, class charT>
+    struct formatter<chrono::local_time<Duration>, charT>;                        // C++20
+  template<class Rep, class Period, class charT>
+    struct formatter<chrono::duration<Rep, Period>, charT>;                       // C++20
+  template<class charT> struct formatter<chrono::day, charT>;                     // C++20
+  template<class charT> struct formatter<chrono::month, charT>;                   // C++20
+  template<class charT> struct formatter<chrono::year, charT>;                    // C++20
+  template<class charT> struct formatter<chrono::weekday, charT>;                 // C++20
+  template<class charT> struct formatter<chrono::weekday_indexed, charT>;         // C++20
+  template<class charT> struct formatter<chrono::weekday_last, charT>;            // C++20
+  template<class charT> struct formatter<chrono::month_day, charT>;               // C++20
+  template<class charT> struct formatter<chrono::month_day_last, charT>;          // C++20
+  template<class charT> struct formatter<chrono::month_weekday, charT>;           // C++20
+  template<class charT> struct formatter<chrono::month_weekday_last, charT>;      // C++20
+  template<class charT> struct formatter<chrono::year_month, charT>;              // C++20
+  template<class charT> struct formatter<chrono::year_month_day, charT>;          // C++20
+  template<class charT> struct formatter<chrono::year_month_day_last, charT>;     // C++20
+  template<class charT> struct formatter<chrono::year_month_weekday, charT>;      // C++20
+  template<class charT> struct formatter<chrono::year_month_weekday_last, charT>; // C++20
+  template<class Rep, class Period, class charT>
+    struct formatter<chrono::hh_mm_ss<duration<Rep, Period>>, charT>;             // C++20
+  template<class charT> struct formatter<chrono::sys_info, charT>;                // C++20
+  template<class charT> struct formatter<chrono::local_info, charT>;              // C++20
+  template<class Duration, class TimeZonePtr, class charT>                        // C++20
+    struct formatter<chrono::zoned_time<Duration, TimeZonePtr>, charT>;
+} // namespace std
+
+namespace chrono {
+// calendrical constants
+inline constexpr last_spec                              last{};       // C++20
+inline constexpr chrono::weekday                        Sunday{0};    // C++20
+inline constexpr chrono::weekday                        Monday{1};    // C++20
+inline constexpr chrono::weekday                        Tuesday{2};   // C++20
+inline constexpr chrono::weekday                        Wednesday{3}; // C++20
+inline constexpr chrono::weekday                        Thursday{4};  // C++20
+inline constexpr chrono::weekday                        Friday{5};    // C++20
+inline constexpr chrono::weekday                        Saturday{6};  // C++20
+
+inline constexpr chrono::month                          January{1};   // C++20
+inline constexpr chrono::month                          February{2};  // C++20
+inline constexpr chrono::month                          March{3};     // C++20
+inline constexpr chrono::month                          April{4};     // C++20
+inline constexpr chrono::month                          May{5};       // C++20
+inline constexpr chrono::month                          June{6};      // C++20
+inline constexpr chrono::month                          July{7};      // C++20
+inline constexpr chrono::month                          August{8};    // C++20
+inline constexpr chrono::month                          September{9}; // C++20
+inline constexpr chrono::month                          October{10};  // C++20
+inline constexpr chrono::month                          November{11}; // C++20
+inline constexpr chrono::month                          December{12}; // C++20
+}  // chrono
+
+inline namespace literals {
+  inline namespace chrono_literals {
+constexpr chrono::hours                                 operator ""h(unsigned long long); // C++14
+constexpr chrono::duration<unspecified , ratio<3600,1>> operator ""h(long double); // C++14
+constexpr chrono::minutes                               operator ""min(unsigned long long); // C++14
+constexpr chrono::duration<unspecified , ratio<60,1>>   operator ""min(long double); // C++14
+constexpr chrono::seconds                               operator ""s(unsigned long long); // C++14
+constexpr chrono::duration<unspecified >                operator ""s(long double); // C++14
+constexpr chrono::milliseconds                          operator ""ms(unsigned long long); // C++14
+constexpr chrono::duration<unspecified , milli>         operator ""ms(long double); // C++14
+constexpr chrono::microseconds                          operator ""us(unsigned long long); // C++14
+constexpr chrono::duration<unspecified , micro>         operator ""us(long double); // C++14
+constexpr chrono::nanoseconds                           operator ""ns(unsigned long long); // C++14
+constexpr chrono::duration<unspecified , nano>          operator ""ns(long double); // C++14
+constexpr chrono::day                                   operator ""d(unsigned long long d) noexcept; // C++20
+constexpr chrono::year                                  operator ""y(unsigned long long y) noexcept; // C++20
+}  // chrono_literals
+}  // literals
+
+}  // std
+*/
+
+// clang-format on
+
+#include <__config>
+
+#include <__chrono/duration.h>
+#include <__chrono/file_clock.h>
+#include <__chrono/high_resolution_clock.h>
+#include <__chrono/steady_clock.h>
+#include <__chrono/system_clock.h>
+#include <__chrono/time_point.h>
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__chrono/calendar.h>
+#  include <__chrono/day.h>
+#  include <__chrono/exception.h>
+#  include <__chrono/hh_mm_ss.h>
+#  include <__chrono/literals.h>
+#  include <__chrono/local_info.h>
+#  include <__chrono/month.h>
+#  include <__chrono/month_weekday.h>
+#  include <__chrono/monthday.h>
+#  include <__chrono/sys_info.h>
+#  include <__chrono/weekday.h>
+#  include <__chrono/year.h>
+#  include <__chrono/year_month.h>
+#  include <__chrono/year_month_day.h>
+#  include <__chrono/year_month_weekday.h>
+
+#  if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#    include <__chrono/formatter.h>
+#    include <__chrono/ostream.h>
+#    include <__chrono/parser_std_format_spec.h>
+#    include <__chrono/statically_widen.h>
+#  endif
+
+#  if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                            \
+      !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#    include <__chrono/leap_second.h>
+#    include <__chrono/time_zone.h>
+#    include <__chrono/time_zone_link.h>
+#    include <__chrono/tzdb.h>
+#    include <__chrono/tzdb_list.h>
+#    include <__chrono/zoned_time.h>
+#  endif
+
+#endif
+
+#include <version>
+
+// standard-mandated includes
+
+// [time.syn]
+#include <compare>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#  include <cstdint>
+#  include <stdexcept>
+#  include <string_view>
+#  include <vector>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <bit>
+#  include <concepts>
+#  include <cstring>
+#  include <forward_list>
+#  include <string>
+#  include <tuple>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER == 20
+#  include <charconv>
+#  if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#    include <locale>
+#    include <ostream>
+#  endif
+#endif
+
+#endif // _LIBCPP_CHRONO
diff --git a/libcxx/include/__cxx03/cinttypes b/libcxx/include/__cxx03/cinttypes
new file mode 100644
index 000000000000..52663a4f35fa
--- /dev/null
+++ b/libcxx/include/__cxx03/cinttypes
@@ -0,0 +1,270 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CINTTYPES
+#define _LIBCPP_CINTTYPES
+
+/*
+    cinttypes synopsis
+
+This entire header is C99 / C++0X
+
+#include <cstdint>  // <cinttypes> includes <cstdint>
+
+Macros:
+
+    PRId8
+    PRId16
+    PRId32
+    PRId64
+
+    PRIdLEAST8
+    PRIdLEAST16
+    PRIdLEAST32
+    PRIdLEAST64
+
+    PRIdFAST8
+    PRIdFAST16
+    PRIdFAST32
+    PRIdFAST64
+
+    PRIdMAX
+    PRIdPTR
+
+    PRIi8
+    PRIi16
+    PRIi32
+    PRIi64
+
+    PRIiLEAST8
+    PRIiLEAST16
+    PRIiLEAST32
+    PRIiLEAST64
+
+    PRIiFAST8
+    PRIiFAST16
+    PRIiFAST32
+    PRIiFAST64
+
+    PRIiMAX
+    PRIiPTR
+
+    PRIo8
+    PRIo16
+    PRIo32
+    PRIo64
+
+    PRIoLEAST8
+    PRIoLEAST16
+    PRIoLEAST32
+    PRIoLEAST64
+
+    PRIoFAST8
+    PRIoFAST16
+    PRIoFAST32
+    PRIoFAST64
+
+    PRIoMAX
+    PRIoPTR
+
+    PRIu8
+    PRIu16
+    PRIu32
+    PRIu64
+
+    PRIuLEAST8
+    PRIuLEAST16
+    PRIuLEAST32
+    PRIuLEAST64
+
+    PRIuFAST8
+    PRIuFAST16
+    PRIuFAST32
+    PRIuFAST64
+
+    PRIuMAX
+    PRIuPTR
+
+    PRIx8
+    PRIx16
+    PRIx32
+    PRIx64
+
+    PRIxLEAST8
+    PRIxLEAST16
+    PRIxLEAST32
+    PRIxLEAST64
+
+    PRIxFAST8
+    PRIxFAST16
+    PRIxFAST32
+    PRIxFAST64
+
+    PRIxMAX
+    PRIxPTR
+
+    PRIX8
+    PRIX16
+    PRIX32
+    PRIX64
+
+    PRIXLEAST8
+    PRIXLEAST16
+    PRIXLEAST32
+    PRIXLEAST64
+
+    PRIXFAST8
+    PRIXFAST16
+    PRIXFAST32
+    PRIXFAST64
+
+    PRIXMAX
+    PRIXPTR
+
+    SCNd8
+    SCNd16
+    SCNd32
+    SCNd64
+
+    SCNdLEAST8
+    SCNdLEAST16
+    SCNdLEAST32
+    SCNdLEAST64
+
+    SCNdFAST8
+    SCNdFAST16
+    SCNdFAST32
+    SCNdFAST64
+
+    SCNdMAX
+    SCNdPTR
+
+    SCNi8
+    SCNi16
+    SCNi32
+    SCNi64
+
+    SCNiLEAST8
+    SCNiLEAST16
+    SCNiLEAST32
+    SCNiLEAST64
+
+    SCNiFAST8
+    SCNiFAST16
+    SCNiFAST32
+    SCNiFAST64
+
+    SCNiMAX
+    SCNiPTR
+
+    SCNo8
+    SCNo16
+    SCNo32
+    SCNo64
+
+    SCNoLEAST8
+    SCNoLEAST16
+    SCNoLEAST32
+    SCNoLEAST64
+
+    SCNoFAST8
+    SCNoFAST16
+    SCNoFAST32
+    SCNoFAST64
+
+    SCNoMAX
+    SCNoPTR
+
+    SCNu8
+    SCNu16
+    SCNu32
+    SCNu64
+
+    SCNuLEAST8
+    SCNuLEAST16
+    SCNuLEAST32
+    SCNuLEAST64
+
+    SCNuFAST8
+    SCNuFAST16
+    SCNuFAST32
+    SCNuFAST64
+
+    SCNuMAX
+    SCNuPTR
+
+    SCNx8
+    SCNx16
+    SCNx32
+    SCNx64
+
+    SCNxLEAST8
+    SCNxLEAST16
+    SCNxLEAST32
+    SCNxLEAST64
+
+    SCNxFAST8
+    SCNxFAST16
+    SCNxFAST32
+    SCNxFAST64
+
+    SCNxMAX
+    SCNxPTR
+
+namespace std
+{
+
+Types:
+
+    imaxdiv_t
+
+intmax_t  imaxabs(intmax_t j);
+imaxdiv_t imaxdiv(intmax_t numer, intmax_t denom);
+intmax_t  strtoimax(const char* restrict nptr, char** restrict endptr, int base);
+uintmax_t strtoumax(const char* restrict nptr, char** restrict endptr, int base);
+intmax_t  wcstoimax(const wchar_t* restrict nptr, wchar_t** restrict endptr, int base);
+uintmax_t wcstoumax(const wchar_t* restrict nptr, wchar_t** restrict endptr, int base);
+
+}  // std
+*/
+
+#include <__config>
+
+// standard-mandated includes
+
+// [cinttypes.syn]
+#include <cstdint>
+
+#include <inttypes.h>
+
+#ifndef _LIBCPP_INTTYPES_H
+#   error <cinttypes> tried including <inttypes.h> but didn't find libc++'s <inttypes.h> header. \
+          This usually means that your header search paths are not configured properly. \
+          The header search paths should contain the C++ Standard Library headers before \
+          any C Standard Library, and you are probably using compiler flags that make that \
+          not be the case.
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::imaxdiv_t _LIBCPP_USING_IF_EXISTS;
+using ::imaxabs _LIBCPP_USING_IF_EXISTS;
+using ::imaxdiv _LIBCPP_USING_IF_EXISTS;
+using ::strtoimax _LIBCPP_USING_IF_EXISTS;
+using ::strtoumax _LIBCPP_USING_IF_EXISTS;
+using ::wcstoimax _LIBCPP_USING_IF_EXISTS;
+using ::wcstoumax _LIBCPP_USING_IF_EXISTS;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CINTTYPES
diff --git a/libcxx/include/__cxx03/ciso646 b/libcxx/include/__cxx03/ciso646
new file mode 100644
index 000000000000..1d859f08fac5
--- /dev/null
+++ b/libcxx/include/__cxx03/ciso646
@@ -0,0 +1,24 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CISO646
+#define _LIBCPP_CISO646
+
+/*
+    ciso646 synopsis
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#endif // _LIBCPP_CISO646
diff --git a/libcxx/include/__cxx03/climits b/libcxx/include/__cxx03/climits
new file mode 100644
index 000000000000..bcd8b4a56a07
--- /dev/null
+++ b/libcxx/include/__cxx03/climits
@@ -0,0 +1,48 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CLIMITS
+#define _LIBCPP_CLIMITS
+
+/*
+    climits synopsis
+
+Macros:
+
+    CHAR_BIT
+    SCHAR_MIN
+    SCHAR_MAX
+    UCHAR_MAX
+    CHAR_MIN
+    CHAR_MAX
+    MB_LEN_MAX
+    SHRT_MIN
+    SHRT_MAX
+    USHRT_MAX
+    INT_MIN
+    INT_MAX
+    UINT_MAX
+    LONG_MIN
+    LONG_MAX
+    ULONG_MAX
+    LLONG_MIN   // C99
+    LLONG_MAX   // C99
+    ULLONG_MAX  // C99
+
+*/
+
+#include <__config>
+
+#include <limits.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#endif // _LIBCPP_CLIMITS
diff --git a/libcxx/include/__cxx03/clocale b/libcxx/include/__cxx03/clocale
new file mode 100644
index 000000000000..c689a64be288
--- /dev/null
+++ b/libcxx/include/__cxx03/clocale
@@ -0,0 +1,61 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CLOCALE
+#define _LIBCPP_CLOCALE
+
+/*
+    clocale synopsis
+
+Macros:
+
+    LC_ALL
+    LC_COLLATE
+    LC_CTYPE
+    LC_MONETARY
+    LC_NUMERIC
+    LC_TIME
+    NULL
+
+namespace std
+{
+
+struct lconv;
+char* setlocale(int category, const char* locale);
+lconv* localeconv();
+
+}  // std
+
+*/
+
+#include <__config>
+
+#include <locale.h>
+
+#ifndef _LIBCPP_LOCALE_H
+#   error <clocale> tried including <locale.h> but didn't find libc++'s <locale.h> header. \
+          This usually means that your header search paths are not configured properly. \
+          The header search paths should contain the C++ Standard Library headers before \
+          any C Standard Library, and you are probably using compiler flags that make that \
+          not be the case.
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::lconv _LIBCPP_USING_IF_EXISTS;
+using ::setlocale _LIBCPP_USING_IF_EXISTS;
+using ::localeconv _LIBCPP_USING_IF_EXISTS;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CLOCALE
diff --git a/libcxx/include/__cxx03/cmath b/libcxx/include/__cxx03/cmath
new file mode 100644
index 000000000000..6480c4678ce3
--- /dev/null
+++ b/libcxx/include/__cxx03/cmath
@@ -0,0 +1,646 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CMATH
+#define _LIBCPP_CMATH
+
+/*
+    cmath synopsis
+
+Macros:
+
+    HUGE_VAL
+    HUGE_VALF               // C99
+    HUGE_VALL               // C99
+    INFINITY                // C99
+    NAN                     // C99
+    FP_INFINITE             // C99
+    FP_NAN                  // C99
+    FP_NORMAL               // C99
+    FP_SUBNORMAL            // C99
+    FP_ZERO                 // C99
+    FP_FAST_FMA             // C99
+    FP_FAST_FMAF            // C99
+    FP_FAST_FMAL            // C99
+    FP_ILOGB0               // C99
+    FP_ILOGBNAN             // C99
+    MATH_ERRNO              // C99
+    MATH_ERREXCEPT          // C99
+    math_errhandling        // C99
+
+namespace std
+{
+
+Types:
+
+    float_t                 // C99
+    double_t                // C99
+
+// C90
+
+floating_point abs(floating_point x);
+
+floating_point acos (arithmetic x);
+float          acosf(float x);
+long double    acosl(long double x);
+
+floating_point asin (arithmetic x);
+float          asinf(float x);
+long double    asinl(long double x);
+
+floating_point atan (arithmetic x);
+float          atanf(float x);
+long double    atanl(long double x);
+
+floating_point atan2 (arithmetic y, arithmetic x);
+float          atan2f(float y, float x);
+long double    atan2l(long double y, long double x);
+
+floating_point ceil (arithmetic x);
+float          ceilf(float x);
+long double    ceill(long double x);
+
+floating_point cos (arithmetic x);
+float          cosf(float x);
+long double    cosl(long double x);
+
+floating_point cosh (arithmetic x);
+float          coshf(float x);
+long double    coshl(long double x);
+
+floating_point exp (arithmetic x);
+float          expf(float x);
+long double    expl(long double x);
+
+floating_point fabs (arithmetic x);
+float          fabsf(float x);
+long double    fabsl(long double x);
+
+floating_point floor (arithmetic x);
+float          floorf(float x);
+long double    floorl(long double x);
+
+floating_point fmod (arithmetic x, arithmetic y);
+float          fmodf(float x, float y);
+long double    fmodl(long double x, long double y);
+
+floating_point frexp (arithmetic value, int* exp);
+float          frexpf(float value, int* exp);
+long double    frexpl(long double value, int* exp);
+
+floating_point ldexp (arithmetic value, int exp);
+float          ldexpf(float value, int exp);
+long double    ldexpl(long double value, int exp);
+
+floating_point log (arithmetic x);
+float          logf(float x);
+long double    logl(long double x);
+
+floating_point log10 (arithmetic x);
+float          log10f(float x);
+long double    log10l(long double x);
+
+floating_point modf (floating_point value, floating_point* iptr);
+float          modff(float value, float* iptr);
+long double    modfl(long double value, long double* iptr);
+
+floating_point pow (arithmetic x, arithmetic y);
+float          powf(float x, float y);
+long double    powl(long double x, long double y);
+
+floating_point sin (arithmetic x);
+float          sinf(float x);
+long double    sinl(long double x);
+
+floating_point sinh (arithmetic x);
+float          sinhf(float x);
+long double    sinhl(long double x);
+
+floating_point sqrt (arithmetic x);
+float          sqrtf(float x);
+long double    sqrtl(long double x);
+
+floating_point tan (arithmetic x);
+float          tanf(float x);
+long double    tanl(long double x);
+
+floating_point tanh (arithmetic x);
+float          tanhf(float x);
+long double    tanhl(long double x);
+
+//  C99
+
+bool signbit(arithmetic x);
+
+int fpclassify(arithmetic x);
+
+bool isfinite(arithmetic x);
+bool isinf(arithmetic x);
+bool isnan(arithmetic x);
+bool isnormal(arithmetic x);
+
+bool isgreater(arithmetic x, arithmetic y);
+bool isgreaterequal(arithmetic x, arithmetic y);
+bool isless(arithmetic x, arithmetic y);
+bool islessequal(arithmetic x, arithmetic y);
+bool islessgreater(arithmetic x, arithmetic y);
+bool isunordered(arithmetic x, arithmetic y);
+
+floating_point acosh (arithmetic x);
+float          acoshf(float x);
+long double    acoshl(long double x);
+
+floating_point asinh (arithmetic x);
+float          asinhf(float x);
+long double    asinhl(long double x);
+
+floating_point atanh (arithmetic x);
+float          atanhf(float x);
+long double    atanhl(long double x);
+
+floating_point cbrt (arithmetic x);
+float          cbrtf(float x);
+long double    cbrtl(long double x);
+
+floating_point copysign (arithmetic x, arithmetic y);
+float          copysignf(float x, float y);
+long double    copysignl(long double x, long double y);
+
+floating_point erf (arithmetic x);
+float          erff(float x);
+long double    erfl(long double x);
+
+floating_point erfc (arithmetic x);
+float          erfcf(float x);
+long double    erfcl(long double x);
+
+floating_point exp2 (arithmetic x);
+float          exp2f(float x);
+long double    exp2l(long double x);
+
+floating_point expm1 (arithmetic x);
+float          expm1f(float x);
+long double    expm1l(long double x);
+
+floating_point fdim (arithmetic x, arithmetic y);
+float          fdimf(float x, float y);
+long double    fdiml(long double x, long double y);
+
+floating_point fma (arithmetic x, arithmetic y, arithmetic z);
+float          fmaf(float x, float y, float z);
+long double    fmal(long double x, long double y, long double z);
+
+floating_point fmax (arithmetic x, arithmetic y);
+float          fmaxf(float x, float y);
+long double    fmaxl(long double x, long double y);
+
+floating_point fmin (arithmetic x, arithmetic y);
+float          fminf(float x, float y);
+long double    fminl(long double x, long double y);
+
+double         hermite(unsigned n, double x);                    // C++17
+float          hermite(unsigned n, float x);                     // C++17
+long double    hermite(unsigned n, long double x);               // C++17
+float          hermitef(unsigned n, float x);                    // C++17
+long double    hermitel(unsigned n, long double x);              // C++17
+template <class Integer>
+double         hermite(unsigned n, Integer x);                   // C++17
+
+floating_point hypot (arithmetic x, arithmetic y);
+float          hypotf(float x, float y);
+long double    hypotl(long double x, long double y);
+
+double       hypot(double x, double y, double z);                // C++17
+float        hypot(float x, float y, float z);                   // C++17
+long double  hypot(long double x, long double y, long double z); // C++17
+
+int ilogb (arithmetic x);
+int ilogbf(float x);
+int ilogbl(long double x);
+
+floating_point lgamma (arithmetic x);
+float          lgammaf(float x);
+long double    lgammal(long double x);
+
+long long llrint (arithmetic x);
+long long llrintf(float x);
+long long llrintl(long double x);
+
+long long llround (arithmetic x);
+long long llroundf(float x);
+long long llroundl(long double x);
+
+floating_point log1p (arithmetic x);
+float          log1pf(float x);
+long double    log1pl(long double x);
+
+floating_point log2 (arithmetic x);
+float          log2f(float x);
+long double    log2l(long double x);
+
+floating_point logb (arithmetic x);
+float          logbf(float x);
+long double    logbl(long double x);
+
+long lrint (arithmetic x);
+long lrintf(float x);
+long lrintl(long double x);
+
+long lround (arithmetic x);
+long lroundf(float x);
+long lroundl(long double x);
+
+double      nan (const char* str);
+float       nanf(const char* str);
+long double nanl(const char* str);
+
+floating_point nearbyint (arithmetic x);
+float          nearbyintf(float x);
+long double    nearbyintl(long double x);
+
+floating_point nextafter (arithmetic x, arithmetic y);
+float          nextafterf(float x, float y);
+long double    nextafterl(long double x, long double y);
+
+floating_point nexttoward (arithmetic x, long double y);
+float          nexttowardf(float x, long double y);
+long double    nexttowardl(long double x, long double y);
+
+floating_point remainder (arithmetic x, arithmetic y);
+float          remainderf(float x, float y);
+long double    remainderl(long double x, long double y);
+
+floating_point remquo (arithmetic x, arithmetic y, int* pquo);
+float          remquof(float x, float y, int* pquo);
+long double    remquol(long double x, long double y, int* pquo);
+
+floating_point rint (arithmetic x);
+float          rintf(float x);
+long double    rintl(long double x);
+
+floating_point round (arithmetic x);
+float          roundf(float x);
+long double    roundl(long double x);
+
+floating_point scalbln (arithmetic x, long ex);
+float          scalblnf(float x, long ex);
+long double    scalblnl(long double x, long ex);
+
+floating_point scalbn (arithmetic x, int ex);
+float          scalbnf(float x, int ex);
+long double    scalbnl(long double x, int ex);
+
+floating_point tgamma (arithmetic x);
+float          tgammaf(float x);
+long double    tgammal(long double x);
+
+floating_point trunc (arithmetic x);
+float          truncf(float x);
+long double    truncl(long double x);
+
+constexpr float       lerp(float a, float b, float t) noexcept;                   // C++20
+constexpr double      lerp(double a, double b, double t) noexcept;                // C++20
+constexpr long double lerp(long double a, long double b, long double t) noexcept; // C++20
+
+}  // std
+
+*/
+
+#include <__config>
+#include <__math/hypot.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_floating_point.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/promote.h>
+#include <__type_traits/remove_cv.h>
+#include <limits>
+#include <version>
+
+#include <__math/special_functions.h>
+#include <math.h>
+
+#ifndef _LIBCPP_MATH_H
+#   error <cmath> tried including <math.h> but didn't find libc++'s <math.h> header. \
+          This usually means that your header search paths are not configured properly. \
+          The header search paths should contain the C++ Standard Library headers before \
+          any C Standard Library, and you are probably using compiler flags that make that \
+          not be the case.
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::signbit _LIBCPP_USING_IF_EXISTS;
+using ::fpclassify _LIBCPP_USING_IF_EXISTS;
+using ::isfinite _LIBCPP_USING_IF_EXISTS;
+using ::isinf _LIBCPP_USING_IF_EXISTS;
+using ::isnan _LIBCPP_USING_IF_EXISTS;
+using ::isnormal _LIBCPP_USING_IF_EXISTS;
+using ::isgreater _LIBCPP_USING_IF_EXISTS;
+using ::isgreaterequal _LIBCPP_USING_IF_EXISTS;
+using ::isless _LIBCPP_USING_IF_EXISTS;
+using ::islessequal _LIBCPP_USING_IF_EXISTS;
+using ::islessgreater _LIBCPP_USING_IF_EXISTS;
+using ::isunordered _LIBCPP_USING_IF_EXISTS;
+using ::isunordered _LIBCPP_USING_IF_EXISTS;
+
+using ::float_t _LIBCPP_USING_IF_EXISTS;
+using ::double_t _LIBCPP_USING_IF_EXISTS;
+
+using ::abs _LIBCPP_USING_IF_EXISTS;
+
+using ::acos _LIBCPP_USING_IF_EXISTS;
+using ::acosf _LIBCPP_USING_IF_EXISTS;
+using ::asin _LIBCPP_USING_IF_EXISTS;
+using ::asinf _LIBCPP_USING_IF_EXISTS;
+using ::atan _LIBCPP_USING_IF_EXISTS;
+using ::atanf _LIBCPP_USING_IF_EXISTS;
+using ::atan2 _LIBCPP_USING_IF_EXISTS;
+using ::atan2f _LIBCPP_USING_IF_EXISTS;
+using ::ceil _LIBCPP_USING_IF_EXISTS;
+using ::ceilf _LIBCPP_USING_IF_EXISTS;
+using ::cos _LIBCPP_USING_IF_EXISTS;
+using ::cosf _LIBCPP_USING_IF_EXISTS;
+using ::cosh _LIBCPP_USING_IF_EXISTS;
+using ::coshf _LIBCPP_USING_IF_EXISTS;
+
+using ::exp _LIBCPP_USING_IF_EXISTS;
+using ::expf _LIBCPP_USING_IF_EXISTS;
+
+using ::fabs _LIBCPP_USING_IF_EXISTS;
+using ::fabsf _LIBCPP_USING_IF_EXISTS;
+using ::floor _LIBCPP_USING_IF_EXISTS;
+using ::floorf _LIBCPP_USING_IF_EXISTS;
+
+using ::fmod _LIBCPP_USING_IF_EXISTS;
+using ::fmodf _LIBCPP_USING_IF_EXISTS;
+
+using ::frexp _LIBCPP_USING_IF_EXISTS;
+using ::frexpf _LIBCPP_USING_IF_EXISTS;
+using ::ldexp _LIBCPP_USING_IF_EXISTS;
+using ::ldexpf _LIBCPP_USING_IF_EXISTS;
+
+using ::log _LIBCPP_USING_IF_EXISTS;
+using ::logf _LIBCPP_USING_IF_EXISTS;
+
+using ::log10 _LIBCPP_USING_IF_EXISTS;
+using ::log10f _LIBCPP_USING_IF_EXISTS;
+using ::modf _LIBCPP_USING_IF_EXISTS;
+using ::modff _LIBCPP_USING_IF_EXISTS;
+
+using ::pow _LIBCPP_USING_IF_EXISTS;
+using ::powf _LIBCPP_USING_IF_EXISTS;
+
+using ::sin _LIBCPP_USING_IF_EXISTS;
+using ::sinf _LIBCPP_USING_IF_EXISTS;
+using ::sinh _LIBCPP_USING_IF_EXISTS;
+using ::sinhf _LIBCPP_USING_IF_EXISTS;
+
+using ::sqrt _LIBCPP_USING_IF_EXISTS;
+using ::sqrtf _LIBCPP_USING_IF_EXISTS;
+using ::tan _LIBCPP_USING_IF_EXISTS;
+using ::tanf _LIBCPP_USING_IF_EXISTS;
+
+using ::tanh _LIBCPP_USING_IF_EXISTS;
+using ::tanhf _LIBCPP_USING_IF_EXISTS;
+
+using ::acosh _LIBCPP_USING_IF_EXISTS;
+using ::acoshf _LIBCPP_USING_IF_EXISTS;
+using ::asinh _LIBCPP_USING_IF_EXISTS;
+using ::asinhf _LIBCPP_USING_IF_EXISTS;
+using ::atanh _LIBCPP_USING_IF_EXISTS;
+using ::atanhf _LIBCPP_USING_IF_EXISTS;
+using ::cbrt _LIBCPP_USING_IF_EXISTS;
+using ::cbrtf _LIBCPP_USING_IF_EXISTS;
+
+using ::copysign _LIBCPP_USING_IF_EXISTS;
+using ::copysignf _LIBCPP_USING_IF_EXISTS;
+
+using ::erf _LIBCPP_USING_IF_EXISTS;
+using ::erff _LIBCPP_USING_IF_EXISTS;
+using ::erfc _LIBCPP_USING_IF_EXISTS;
+using ::erfcf _LIBCPP_USING_IF_EXISTS;
+using ::exp2 _LIBCPP_USING_IF_EXISTS;
+using ::exp2f _LIBCPP_USING_IF_EXISTS;
+using ::expm1 _LIBCPP_USING_IF_EXISTS;
+using ::expm1f _LIBCPP_USING_IF_EXISTS;
+using ::fdim _LIBCPP_USING_IF_EXISTS;
+using ::fdimf _LIBCPP_USING_IF_EXISTS;
+using ::fmaf _LIBCPP_USING_IF_EXISTS;
+using ::fma _LIBCPP_USING_IF_EXISTS;
+using ::fmax _LIBCPP_USING_IF_EXISTS;
+using ::fmaxf _LIBCPP_USING_IF_EXISTS;
+using ::fmin _LIBCPP_USING_IF_EXISTS;
+using ::fminf _LIBCPP_USING_IF_EXISTS;
+using ::hypot _LIBCPP_USING_IF_EXISTS;
+using ::hypotf _LIBCPP_USING_IF_EXISTS;
+using ::ilogb _LIBCPP_USING_IF_EXISTS;
+using ::ilogbf _LIBCPP_USING_IF_EXISTS;
+using ::lgamma _LIBCPP_USING_IF_EXISTS;
+using ::lgammaf _LIBCPP_USING_IF_EXISTS;
+using ::llrint _LIBCPP_USING_IF_EXISTS;
+using ::llrintf _LIBCPP_USING_IF_EXISTS;
+using ::llround _LIBCPP_USING_IF_EXISTS;
+using ::llroundf _LIBCPP_USING_IF_EXISTS;
+using ::log1p _LIBCPP_USING_IF_EXISTS;
+using ::log1pf _LIBCPP_USING_IF_EXISTS;
+using ::log2 _LIBCPP_USING_IF_EXISTS;
+using ::log2f _LIBCPP_USING_IF_EXISTS;
+using ::logb _LIBCPP_USING_IF_EXISTS;
+using ::logbf _LIBCPP_USING_IF_EXISTS;
+using ::lrint _LIBCPP_USING_IF_EXISTS;
+using ::lrintf _LIBCPP_USING_IF_EXISTS;
+using ::lround _LIBCPP_USING_IF_EXISTS;
+using ::lroundf _LIBCPP_USING_IF_EXISTS;
+
+using ::nan _LIBCPP_USING_IF_EXISTS;
+using ::nanf _LIBCPP_USING_IF_EXISTS;
+
+using ::nearbyint _LIBCPP_USING_IF_EXISTS;
+using ::nearbyintf _LIBCPP_USING_IF_EXISTS;
+using ::nextafter _LIBCPP_USING_IF_EXISTS;
+using ::nextafterf _LIBCPP_USING_IF_EXISTS;
+using ::nexttoward _LIBCPP_USING_IF_EXISTS;
+using ::nexttowardf _LIBCPP_USING_IF_EXISTS;
+using ::remainder _LIBCPP_USING_IF_EXISTS;
+using ::remainderf _LIBCPP_USING_IF_EXISTS;
+using ::remquo _LIBCPP_USING_IF_EXISTS;
+using ::remquof _LIBCPP_USING_IF_EXISTS;
+using ::rint _LIBCPP_USING_IF_EXISTS;
+using ::rintf _LIBCPP_USING_IF_EXISTS;
+using ::round _LIBCPP_USING_IF_EXISTS;
+using ::roundf _LIBCPP_USING_IF_EXISTS;
+using ::scalbln _LIBCPP_USING_IF_EXISTS;
+using ::scalblnf _LIBCPP_USING_IF_EXISTS;
+using ::scalbn _LIBCPP_USING_IF_EXISTS;
+using ::scalbnf _LIBCPP_USING_IF_EXISTS;
+using ::tgamma _LIBCPP_USING_IF_EXISTS;
+using ::tgammaf _LIBCPP_USING_IF_EXISTS;
+using ::trunc _LIBCPP_USING_IF_EXISTS;
+using ::truncf _LIBCPP_USING_IF_EXISTS;
+
+using ::acosl _LIBCPP_USING_IF_EXISTS;
+using ::asinl _LIBCPP_USING_IF_EXISTS;
+using ::atanl _LIBCPP_USING_IF_EXISTS;
+using ::atan2l _LIBCPP_USING_IF_EXISTS;
+using ::ceill _LIBCPP_USING_IF_EXISTS;
+using ::cosl _LIBCPP_USING_IF_EXISTS;
+using ::coshl _LIBCPP_USING_IF_EXISTS;
+using ::expl _LIBCPP_USING_IF_EXISTS;
+using ::fabsl _LIBCPP_USING_IF_EXISTS;
+using ::floorl _LIBCPP_USING_IF_EXISTS;
+using ::fmodl _LIBCPP_USING_IF_EXISTS;
+using ::frexpl _LIBCPP_USING_IF_EXISTS;
+using ::ldexpl _LIBCPP_USING_IF_EXISTS;
+using ::logl _LIBCPP_USING_IF_EXISTS;
+using ::log10l _LIBCPP_USING_IF_EXISTS;
+using ::modfl _LIBCPP_USING_IF_EXISTS;
+using ::powl _LIBCPP_USING_IF_EXISTS;
+using ::sinl _LIBCPP_USING_IF_EXISTS;
+using ::sinhl _LIBCPP_USING_IF_EXISTS;
+using ::sqrtl _LIBCPP_USING_IF_EXISTS;
+using ::tanl _LIBCPP_USING_IF_EXISTS;
+
+using ::tanhl _LIBCPP_USING_IF_EXISTS;
+using ::acoshl _LIBCPP_USING_IF_EXISTS;
+using ::asinhl _LIBCPP_USING_IF_EXISTS;
+using ::atanhl _LIBCPP_USING_IF_EXISTS;
+using ::cbrtl _LIBCPP_USING_IF_EXISTS;
+
+using ::copysignl _LIBCPP_USING_IF_EXISTS;
+
+using ::erfl _LIBCPP_USING_IF_EXISTS;
+using ::erfcl _LIBCPP_USING_IF_EXISTS;
+using ::exp2l _LIBCPP_USING_IF_EXISTS;
+using ::expm1l _LIBCPP_USING_IF_EXISTS;
+using ::fdiml _LIBCPP_USING_IF_EXISTS;
+using ::fmal _LIBCPP_USING_IF_EXISTS;
+using ::fmaxl _LIBCPP_USING_IF_EXISTS;
+using ::fminl _LIBCPP_USING_IF_EXISTS;
+using ::hypotl _LIBCPP_USING_IF_EXISTS;
+using ::ilogbl _LIBCPP_USING_IF_EXISTS;
+using ::lgammal _LIBCPP_USING_IF_EXISTS;
+using ::llrintl _LIBCPP_USING_IF_EXISTS;
+using ::llroundl _LIBCPP_USING_IF_EXISTS;
+using ::log1pl _LIBCPP_USING_IF_EXISTS;
+using ::log2l _LIBCPP_USING_IF_EXISTS;
+using ::logbl _LIBCPP_USING_IF_EXISTS;
+using ::lrintl _LIBCPP_USING_IF_EXISTS;
+using ::lroundl _LIBCPP_USING_IF_EXISTS;
+using ::nanl _LIBCPP_USING_IF_EXISTS;
+using ::nearbyintl _LIBCPP_USING_IF_EXISTS;
+using ::nextafterl _LIBCPP_USING_IF_EXISTS;
+using ::nexttowardl _LIBCPP_USING_IF_EXISTS;
+using ::remainderl _LIBCPP_USING_IF_EXISTS;
+using ::remquol _LIBCPP_USING_IF_EXISTS;
+using ::rintl _LIBCPP_USING_IF_EXISTS;
+using ::roundl _LIBCPP_USING_IF_EXISTS;
+using ::scalblnl _LIBCPP_USING_IF_EXISTS;
+using ::scalbnl _LIBCPP_USING_IF_EXISTS;
+using ::tgammal _LIBCPP_USING_IF_EXISTS;
+using ::truncl _LIBCPP_USING_IF_EXISTS;
+
+template <class _A1, __enable_if_t<is_floating_point<_A1>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __constexpr_isnan(_A1 __lcpp_x) _NOEXCEPT {
+#if __has_builtin(__builtin_isnan)
+  return __builtin_isnan(__lcpp_x);
+#else
+  return isnan(__lcpp_x);
+#endif
+}
+
+template <class _A1, __enable_if_t<!is_floating_point<_A1>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __constexpr_isnan(_A1 __lcpp_x) _NOEXCEPT {
+  return std::isnan(__lcpp_x);
+}
+
+template <class _A1, __enable_if_t<is_floating_point<_A1>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __constexpr_isinf(_A1 __lcpp_x) _NOEXCEPT {
+#if __has_builtin(__builtin_isinf)
+  return __builtin_isinf(__lcpp_x);
+#else
+  return isinf(__lcpp_x);
+#endif
+}
+
+template <class _A1, __enable_if_t<!is_floating_point<_A1>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __constexpr_isinf(_A1 __lcpp_x) _NOEXCEPT {
+  return std::isinf(__lcpp_x);
+}
+
+template <class _A1, __enable_if_t<is_floating_point<_A1>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __constexpr_isfinite(_A1 __lcpp_x) _NOEXCEPT {
+#if __has_builtin(__builtin_isfinite)
+  return __builtin_isfinite(__lcpp_x);
+#else
+  return isfinite(__lcpp_x);
+#endif
+}
+
+template <class _A1, __enable_if_t<!is_floating_point<_A1>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __constexpr_isfinite(_A1 __lcpp_x) _NOEXCEPT {
+  return __builtin_isfinite(__lcpp_x);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <typename _Fp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Fp __lerp(_Fp __a, _Fp __b, _Fp __t) noexcept {
+  if ((__a <= 0 && __b >= 0) || (__a >= 0 && __b <= 0))
+    return __t * __b + (1 - __t) * __a;
+
+  if (__t == 1)
+    return __b;
+  const _Fp __x = __a + __t * (__b - __a);
+  if ((__t > 1) == (__b > __a))
+    return __b < __x ? __x : __b;
+  else
+    return __x < __b ? __x : __b;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr float lerp(float __a, float __b, float __t) _NOEXCEPT {
+  return __lerp(__a, __b, __t);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr double lerp(double __a, double __b, double __t) _NOEXCEPT {
+  return __lerp(__a, __b, __t);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr long double lerp(long double __a, long double __b, long double __t) _NOEXCEPT {
+  return __lerp(__a, __b, __t);
+}
+
+template <class _A1, class _A2, class _A3>
+inline _LIBCPP_HIDE_FROM_ABI constexpr
+    typename enable_if_t< is_arithmetic<_A1>::value && is_arithmetic<_A2>::value && is_arithmetic<_A3>::value,
+                          __promote<_A1, _A2, _A3> >::type
+    lerp(_A1 __a, _A2 __b, _A3 __t) noexcept {
+  typedef typename __promote<_A1, _A2, _A3>::type __result_type;
+  static_assert(!(
+      _IsSame<_A1, __result_type>::value && _IsSame<_A2, __result_type>::value && _IsSame<_A3, __result_type>::value));
+  return std::__lerp((__result_type)__a, (__result_type)__b, (__result_type)__t);
+}
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_CMATH
diff --git a/libcxx/include/__cxx03/codecvt b/libcxx/include/__cxx03/codecvt
new file mode 100644
index 000000000000..65cd752d6946
--- /dev/null
+++ b/libcxx/include/__cxx03/codecvt
@@ -0,0 +1,597 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CODECVT
+#define _LIBCPP_CODECVT
+
+/*
+    codecvt synopsis
+
+namespace std
+{
+
+enum codecvt_mode
+{
+    consume_header = 4,
+    generate_header = 2,
+    little_endian = 1
+};
+
+template <class Elem, unsigned long Maxcode = 0x10ffff,
+          codecvt_mode Mode = (codecvt_mode)0>
+class codecvt_utf8
+    : public codecvt<Elem, char, mbstate_t>
+{
+    explicit codecvt_utf8(size_t refs = 0);
+    ~codecvt_utf8();
+};
+
+template <class Elem, unsigned long Maxcode = 0x10ffff,
+          codecvt_mode Mode = (codecvt_mode)0>
+class codecvt_utf16
+    : public codecvt<Elem, char, mbstate_t>
+{
+    explicit codecvt_utf16(size_t refs = 0);
+    ~codecvt_utf16();
+};
+
+template <class Elem, unsigned long Maxcode = 0x10ffff,
+          codecvt_mode Mode = (codecvt_mode)0>
+class codecvt_utf8_utf16
+    : public codecvt<Elem, char, mbstate_t>
+{
+    explicit codecvt_utf8_utf16(size_t refs = 0);
+    ~codecvt_utf8_utf16();
+};
+
+}  // std
+
+*/
+
+#include <__config>
+#include <__locale>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_BUILDING_LIBRARY) || defined(_LIBCPP_ENABLE_CXX26_REMOVED_CODECVT)
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+enum _LIBCPP_DEPRECATED_IN_CXX17 codecvt_mode { consume_header = 4, generate_header = 2, little_endian = 1 };
+
+// codecvt_utf8
+
+template <class _Elem>
+class __codecvt_utf8;
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI __codecvt_utf8<wchar_t> : public codecvt<wchar_t, char, mbstate_t> {
+  unsigned long __maxcode_;
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  codecvt_mode __mode_;
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+public:
+  typedef wchar_t intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  _LIBCPP_HIDE_FROM_ABI explicit __codecvt_utf8(size_t __refs, unsigned long __maxcode, codecvt_mode __mode)
+      : codecvt<wchar_t, char, mbstate_t>(__refs), __maxcode_(__maxcode), __mode_(__mode) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  result do_out(state_type& __st,
+                const intern_type* __frm,
+                const intern_type* __frm_end,
+                const intern_type*& __frm_nxt,
+                extern_type* __to,
+                extern_type* __to_end,
+                extern_type*& __to_nxt) const override;
+  result do_in(state_type& __st,
+               const extern_type* __frm,
+               const extern_type* __frm_end,
+               const extern_type*& __frm_nxt,
+               intern_type* __to,
+               intern_type* __to_end,
+               intern_type*& __to_nxt) const override;
+  result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const override;
+  int do_encoding() const _NOEXCEPT override;
+  bool do_always_noconv() const _NOEXCEPT override;
+  int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const override;
+  int do_max_length() const _NOEXCEPT override;
+};
+#  endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI __codecvt_utf8<char16_t> : public codecvt<char16_t, char, mbstate_t> {
+  unsigned long __maxcode_;
+  codecvt_mode __mode_;
+
+public:
+  typedef char16_t intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __codecvt_utf8(size_t __refs, unsigned long __maxcode, codecvt_mode __mode)
+      : codecvt<char16_t, char, mbstate_t>(__refs), __maxcode_(__maxcode), __mode_(__mode) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  result do_out(state_type& __st,
+                const intern_type* __frm,
+                const intern_type* __frm_end,
+                const intern_type*& __frm_nxt,
+                extern_type* __to,
+                extern_type* __to_end,
+                extern_type*& __to_nxt) const override;
+  result do_in(state_type& __st,
+               const extern_type* __frm,
+               const extern_type* __frm_end,
+               const extern_type*& __frm_nxt,
+               intern_type* __to,
+               intern_type* __to_end,
+               intern_type*& __to_nxt) const override;
+  result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const override;
+  int do_encoding() const _NOEXCEPT override;
+  bool do_always_noconv() const _NOEXCEPT override;
+  int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const override;
+  int do_max_length() const _NOEXCEPT override;
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI __codecvt_utf8<char32_t> : public codecvt<char32_t, char, mbstate_t> {
+  unsigned long __maxcode_;
+  codecvt_mode __mode_;
+
+public:
+  typedef char32_t intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __codecvt_utf8(size_t __refs, unsigned long __maxcode, codecvt_mode __mode)
+      : codecvt<char32_t, char, mbstate_t>(__refs), __maxcode_(__maxcode), __mode_(__mode) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  result do_out(state_type& __st,
+                const intern_type* __frm,
+                const intern_type* __frm_end,
+                const intern_type*& __frm_nxt,
+                extern_type* __to,
+                extern_type* __to_end,
+                extern_type*& __to_nxt) const override;
+  result do_in(state_type& __st,
+               const extern_type* __frm,
+               const extern_type* __frm_end,
+               const extern_type*& __frm_nxt,
+               intern_type* __to,
+               intern_type* __to_end,
+               intern_type*& __to_nxt) const override;
+  result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const override;
+  int do_encoding() const _NOEXCEPT override;
+  bool do_always_noconv() const _NOEXCEPT override;
+  int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const override;
+  int do_max_length() const _NOEXCEPT override;
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Elem, unsigned long _Maxcode = 0x10ffff, codecvt_mode _Mode = (codecvt_mode)0>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 codecvt_utf8 : public __codecvt_utf8<_Elem> {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit codecvt_utf8(size_t __refs = 0) : __codecvt_utf8<_Elem>(__refs, _Maxcode, _Mode) {}
+
+  _LIBCPP_HIDE_FROM_ABI ~codecvt_utf8() {}
+};
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+// codecvt_utf16
+
+template <class _Elem, bool _LittleEndian>
+class __codecvt_utf16;
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI __codecvt_utf16<wchar_t, false> : public codecvt<wchar_t, char, mbstate_t> {
+  unsigned long __maxcode_;
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  codecvt_mode __mode_;
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+public:
+  typedef wchar_t intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  _LIBCPP_HIDE_FROM_ABI explicit __codecvt_utf16(size_t __refs, unsigned long __maxcode, codecvt_mode __mode)
+      : codecvt<wchar_t, char, mbstate_t>(__refs), __maxcode_(__maxcode), __mode_(__mode) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  result do_out(state_type& __st,
+                const intern_type* __frm,
+                const intern_type* __frm_end,
+                const intern_type*& __frm_nxt,
+                extern_type* __to,
+                extern_type* __to_end,
+                extern_type*& __to_nxt) const override;
+  result do_in(state_type& __st,
+               const extern_type* __frm,
+               const extern_type* __frm_end,
+               const extern_type*& __frm_nxt,
+               intern_type* __to,
+               intern_type* __to_end,
+               intern_type*& __to_nxt) const override;
+  result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const override;
+  int do_encoding() const _NOEXCEPT override;
+  bool do_always_noconv() const _NOEXCEPT override;
+  int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const override;
+  int do_max_length() const _NOEXCEPT override;
+};
+
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI __codecvt_utf16<wchar_t, true> : public codecvt<wchar_t, char, mbstate_t> {
+  unsigned long __maxcode_;
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  codecvt_mode __mode_;
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+public:
+  typedef wchar_t intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  _LIBCPP_HIDE_FROM_ABI explicit __codecvt_utf16(size_t __refs, unsigned long __maxcode, codecvt_mode __mode)
+      : codecvt<wchar_t, char, mbstate_t>(__refs), __maxcode_(__maxcode), __mode_(__mode) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  result do_out(state_type& __st,
+                const intern_type* __frm,
+                const intern_type* __frm_end,
+                const intern_type*& __frm_nxt,
+                extern_type* __to,
+                extern_type* __to_end,
+                extern_type*& __to_nxt) const override;
+  result do_in(state_type& __st,
+               const extern_type* __frm,
+               const extern_type* __frm_end,
+               const extern_type*& __frm_nxt,
+               intern_type* __to,
+               intern_type* __to_end,
+               intern_type*& __to_nxt) const override;
+  result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const override;
+  int do_encoding() const _NOEXCEPT override;
+  bool do_always_noconv() const _NOEXCEPT override;
+  int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const override;
+  int do_max_length() const _NOEXCEPT override;
+};
+#  endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI __codecvt_utf16<char16_t, false> : public codecvt<char16_t, char, mbstate_t> {
+  unsigned long __maxcode_;
+  codecvt_mode __mode_;
+
+public:
+  typedef char16_t intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __codecvt_utf16(size_t __refs, unsigned long __maxcode, codecvt_mode __mode)
+      : codecvt<char16_t, char, mbstate_t>(__refs), __maxcode_(__maxcode), __mode_(__mode) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  result do_out(state_type& __st,
+                const intern_type* __frm,
+                const intern_type* __frm_end,
+                const intern_type*& __frm_nxt,
+                extern_type* __to,
+                extern_type* __to_end,
+                extern_type*& __to_nxt) const override;
+  result do_in(state_type& __st,
+               const extern_type* __frm,
+               const extern_type* __frm_end,
+               const extern_type*& __frm_nxt,
+               intern_type* __to,
+               intern_type* __to_end,
+               intern_type*& __to_nxt) const override;
+  result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const override;
+  int do_encoding() const _NOEXCEPT override;
+  bool do_always_noconv() const _NOEXCEPT override;
+  int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const override;
+  int do_max_length() const _NOEXCEPT override;
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI __codecvt_utf16<char16_t, true> : public codecvt<char16_t, char, mbstate_t> {
+  unsigned long __maxcode_;
+  codecvt_mode __mode_;
+
+public:
+  typedef char16_t intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __codecvt_utf16(size_t __refs, unsigned long __maxcode, codecvt_mode __mode)
+      : codecvt<char16_t, char, mbstate_t>(__refs), __maxcode_(__maxcode), __mode_(__mode) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  result do_out(state_type& __st,
+                const intern_type* __frm,
+                const intern_type* __frm_end,
+                const intern_type*& __frm_nxt,
+                extern_type* __to,
+                extern_type* __to_end,
+                extern_type*& __to_nxt) const override;
+  result do_in(state_type& __st,
+               const extern_type* __frm,
+               const extern_type* __frm_end,
+               const extern_type*& __frm_nxt,
+               intern_type* __to,
+               intern_type* __to_end,
+               intern_type*& __to_nxt) const override;
+  result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const override;
+  int do_encoding() const _NOEXCEPT override;
+  bool do_always_noconv() const _NOEXCEPT override;
+  int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const override;
+  int do_max_length() const _NOEXCEPT override;
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI __codecvt_utf16<char32_t, false> : public codecvt<char32_t, char, mbstate_t> {
+  unsigned long __maxcode_;
+  codecvt_mode __mode_;
+
+public:
+  typedef char32_t intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __codecvt_utf16(size_t __refs, unsigned long __maxcode, codecvt_mode __mode)
+      : codecvt<char32_t, char, mbstate_t>(__refs), __maxcode_(__maxcode), __mode_(__mode) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  result do_out(state_type& __st,
+                const intern_type* __frm,
+                const intern_type* __frm_end,
+                const intern_type*& __frm_nxt,
+                extern_type* __to,
+                extern_type* __to_end,
+                extern_type*& __to_nxt) const override;
+  result do_in(state_type& __st,
+               const extern_type* __frm,
+               const extern_type* __frm_end,
+               const extern_type*& __frm_nxt,
+               intern_type* __to,
+               intern_type* __to_end,
+               intern_type*& __to_nxt) const override;
+  result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const override;
+  int do_encoding() const _NOEXCEPT override;
+  bool do_always_noconv() const _NOEXCEPT override;
+  int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const override;
+  int do_max_length() const _NOEXCEPT override;
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI __codecvt_utf16<char32_t, true> : public codecvt<char32_t, char, mbstate_t> {
+  unsigned long __maxcode_;
+  codecvt_mode __mode_;
+
+public:
+  typedef char32_t intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __codecvt_utf16(size_t __refs, unsigned long __maxcode, codecvt_mode __mode)
+      : codecvt<char32_t, char, mbstate_t>(__refs), __maxcode_(__maxcode), __mode_(__mode) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  result do_out(state_type& __st,
+                const intern_type* __frm,
+                const intern_type* __frm_end,
+                const intern_type*& __frm_nxt,
+                extern_type* __to,
+                extern_type* __to_end,
+                extern_type*& __to_nxt) const override;
+  result do_in(state_type& __st,
+               const extern_type* __frm,
+               const extern_type* __frm_end,
+               const extern_type*& __frm_nxt,
+               intern_type* __to,
+               intern_type* __to_end,
+               intern_type*& __to_nxt) const override;
+  result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const override;
+  int do_encoding() const _NOEXCEPT override;
+  bool do_always_noconv() const _NOEXCEPT override;
+  int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const override;
+  int do_max_length() const _NOEXCEPT override;
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Elem, unsigned long _Maxcode = 0x10ffff, codecvt_mode _Mode = (codecvt_mode)0>
+class _LIBCPP_TEMPLATE_VIS
+_LIBCPP_DEPRECATED_IN_CXX17 codecvt_utf16 : public __codecvt_utf16<_Elem, _Mode & little_endian> {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit codecvt_utf16(size_t __refs = 0)
+      : __codecvt_utf16<_Elem, _Mode & little_endian>(__refs, _Maxcode, _Mode) {}
+
+  _LIBCPP_HIDE_FROM_ABI ~codecvt_utf16() {}
+};
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+// codecvt_utf8_utf16
+
+template <class _Elem>
+class __codecvt_utf8_utf16;
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI __codecvt_utf8_utf16<wchar_t> : public codecvt<wchar_t, char, mbstate_t> {
+  unsigned long __maxcode_;
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  codecvt_mode __mode_;
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+public:
+  typedef wchar_t intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  _LIBCPP_HIDE_FROM_ABI explicit __codecvt_utf8_utf16(size_t __refs, unsigned long __maxcode, codecvt_mode __mode)
+      : codecvt<wchar_t, char, mbstate_t>(__refs), __maxcode_(__maxcode), __mode_(__mode) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  result do_out(state_type& __st,
+                const intern_type* __frm,
+                const intern_type* __frm_end,
+                const intern_type*& __frm_nxt,
+                extern_type* __to,
+                extern_type* __to_end,
+                extern_type*& __to_nxt) const override;
+  result do_in(state_type& __st,
+               const extern_type* __frm,
+               const extern_type* __frm_end,
+               const extern_type*& __frm_nxt,
+               intern_type* __to,
+               intern_type* __to_end,
+               intern_type*& __to_nxt) const override;
+  result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const override;
+  int do_encoding() const _NOEXCEPT override;
+  bool do_always_noconv() const _NOEXCEPT override;
+  int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const override;
+  int do_max_length() const _NOEXCEPT override;
+};
+#  endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI __codecvt_utf8_utf16<char32_t> : public codecvt<char32_t, char, mbstate_t> {
+  unsigned long __maxcode_;
+  codecvt_mode __mode_;
+
+public:
+  typedef char32_t intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __codecvt_utf8_utf16(size_t __refs, unsigned long __maxcode, codecvt_mode __mode)
+      : codecvt<char32_t, char, mbstate_t>(__refs), __maxcode_(__maxcode), __mode_(__mode) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  result do_out(state_type& __st,
+                const intern_type* __frm,
+                const intern_type* __frm_end,
+                const intern_type*& __frm_nxt,
+                extern_type* __to,
+                extern_type* __to_end,
+                extern_type*& __to_nxt) const override;
+  result do_in(state_type& __st,
+               const extern_type* __frm,
+               const extern_type* __frm_end,
+               const extern_type*& __frm_nxt,
+               intern_type* __to,
+               intern_type* __to_end,
+               intern_type*& __to_nxt) const override;
+  result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const override;
+  int do_encoding() const _NOEXCEPT override;
+  bool do_always_noconv() const _NOEXCEPT override;
+  int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const override;
+  int do_max_length() const _NOEXCEPT override;
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI __codecvt_utf8_utf16<char16_t> : public codecvt<char16_t, char, mbstate_t> {
+  unsigned long __maxcode_;
+  codecvt_mode __mode_;
+
+public:
+  typedef char16_t intern_type;
+  typedef char extern_type;
+  typedef mbstate_t state_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __codecvt_utf8_utf16(size_t __refs, unsigned long __maxcode, codecvt_mode __mode)
+      : codecvt<char16_t, char, mbstate_t>(__refs), __maxcode_(__maxcode), __mode_(__mode) {}
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  result do_out(state_type& __st,
+                const intern_type* __frm,
+                const intern_type* __frm_end,
+                const intern_type*& __frm_nxt,
+                extern_type* __to,
+                extern_type* __to_end,
+                extern_type*& __to_nxt) const override;
+  result do_in(state_type& __st,
+               const extern_type* __frm,
+               const extern_type* __frm_end,
+               const extern_type*& __frm_nxt,
+               intern_type* __to,
+               intern_type* __to_end,
+               intern_type*& __to_nxt) const override;
+  result do_unshift(state_type& __st, extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const override;
+  int do_encoding() const _NOEXCEPT override;
+  bool do_always_noconv() const _NOEXCEPT override;
+  int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const override;
+  int do_max_length() const _NOEXCEPT override;
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Elem, unsigned long _Maxcode = 0x10ffff, codecvt_mode _Mode = (codecvt_mode)0>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 codecvt_utf8_utf16 : public __codecvt_utf8_utf16<_Elem> {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit codecvt_utf8_utf16(size_t __refs = 0)
+      : __codecvt_utf8_utf16<_Elem>(__refs, _Maxcode, _Mode) {}
+
+  _LIBCPP_HIDE_FROM_ABI ~codecvt_utf8_utf16() {}
+};
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_BUILDING_LIBRARY) || defined(_LIBCPP_ENABLE_CXX26_REMOVED_CODECVT)
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <concepts>
+#  include <cstddef>
+#  include <cstdlib>
+#  include <cstring>
+#  include <initializer_list>
+#  include <iosfwd>
+#  include <limits>
+#  include <mutex>
+#  include <new>
+#  include <stdexcept>
+#  include <type_traits>
+#  include <typeinfo>
+#endif
+
+#endif // _LIBCPP_CODECVT
diff --git a/libcxx/include/__cxx03/compare b/libcxx/include/__cxx03/compare
new file mode 100644
index 000000000000..8a41835b1489
--- /dev/null
+++ b/libcxx/include/__cxx03/compare
@@ -0,0 +1,178 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_COMPARE
+#define _LIBCPP_COMPARE
+
+/*
+    compare synopsis
+
+namespace std {
+  // [cmp.categories], comparison category types
+  class partial_ordering;
+  class weak_ordering;
+  class strong_ordering;
+
+  // named comparison functions
+  constexpr bool is_eq  (partial_ordering cmp) noexcept { return cmp == 0; }
+  constexpr bool is_neq (partial_ordering cmp) noexcept { return cmp != 0; }
+  constexpr bool is_lt  (partial_ordering cmp) noexcept { return cmp < 0; }
+  constexpr bool is_lteq(partial_ordering cmp) noexcept { return cmp <= 0; }
+  constexpr bool is_gt  (partial_ordering cmp) noexcept { return cmp > 0; }
+  constexpr bool is_gteq(partial_ordering cmp) noexcept { return cmp >= 0; }
+
+  // [cmp.common], common comparison category type
+  template<class... Ts>
+  struct common_comparison_category {
+    using type = see below;
+  };
+  template<class... Ts>
+    using common_comparison_category_t = typename common_comparison_category<Ts...>::type;
+
+  // [cmp.concept], concept three_way_comparable
+  template<class T, class Cat = partial_ordering>
+    concept three_way_comparable = see below;
+  template<class T, class U, class Cat = partial_ordering>
+    concept three_way_comparable_with = see below;
+
+  // [cmp.result], result of three-way comparison
+  template<class T, class U = T> struct compare_three_way_result;
+
+  template<class T, class U = T>
+    using compare_three_way_result_t = typename compare_three_way_result<T, U>::type;
+
+  // [comparisons.three.way], class compare_three_way
+  struct compare_three_way; // C++20
+
+  // [cmp.alg], comparison algorithms
+  inline namespace unspecified {
+    inline constexpr unspecified strong_order = unspecified;
+    inline constexpr unspecified weak_order = unspecified;
+    inline constexpr unspecified partial_order = unspecified;
+    inline constexpr unspecified compare_strong_order_fallback = unspecified;
+    inline constexpr unspecified compare_weak_order_fallback = unspecified;
+    inline constexpr unspecified compare_partial_order_fallback = unspecified;
+  }
+
+  // [cmp.partialord], Class partial_ordering
+  class partial_ordering {
+  public:
+    // valid values
+    static const partial_ordering less;
+    static const partial_ordering equivalent;
+    static const partial_ordering greater;
+    static const partial_ordering unordered;
+
+    // comparisons
+    friend constexpr bool operator==(partial_ordering v, unspecified) noexcept;
+    friend constexpr bool operator==(partial_ordering v, partial_ordering w) noexcept = default;
+    friend constexpr bool operator< (partial_ordering v, unspecified) noexcept;
+    friend constexpr bool operator> (partial_ordering v, unspecified) noexcept;
+    friend constexpr bool operator<=(partial_ordering v, unspecified) noexcept;
+    friend constexpr bool operator>=(partial_ordering v, unspecified) noexcept;
+    friend constexpr bool operator< (unspecified, partial_ordering v) noexcept;
+    friend constexpr bool operator> (unspecified, partial_ordering v) noexcept;
+    friend constexpr bool operator<=(unspecified, partial_ordering v) noexcept;
+    friend constexpr bool operator>=(unspecified, partial_ordering v) noexcept;
+    friend constexpr partial_ordering operator<=>(partial_ordering v, unspecified) noexcept;
+    friend constexpr partial_ordering operator<=>(unspecified, partial_ordering v) noexcept;
+  };
+
+  // [cmp.weakord], Class weak_ordering
+  class weak_ordering {
+  public:
+    // valid values
+    static const weak_ordering less;
+    static const weak_ordering equivalent;
+    static const weak_ordering greater;
+
+    // conversions
+    constexpr operator partial_ordering() const noexcept;
+
+    // comparisons
+    friend constexpr bool operator==(weak_ordering v, unspecified) noexcept;
+    friend constexpr bool operator==(weak_ordering v, weak_ordering w) noexcept = default;
+    friend constexpr bool operator< (weak_ordering v, unspecified) noexcept;
+    friend constexpr bool operator> (weak_ordering v, unspecified) noexcept;
+    friend constexpr bool operator<=(weak_ordering v, unspecified) noexcept;
+    friend constexpr bool operator>=(weak_ordering v, unspecified) noexcept;
+    friend constexpr bool operator< (unspecified, weak_ordering v) noexcept;
+    friend constexpr bool operator> (unspecified, weak_ordering v) noexcept;
+    friend constexpr bool operator<=(unspecified, weak_ordering v) noexcept;
+    friend constexpr bool operator>=(unspecified, weak_ordering v) noexcept;
+    friend constexpr weak_ordering operator<=>(weak_ordering v, unspecified) noexcept;
+    friend constexpr weak_ordering operator<=>(unspecified, weak_ordering v) noexcept;
+  };
+
+  // [cmp.strongord], Class strong_ordering
+  class strong_ordering {
+  public:
+    // valid values
+    static const strong_ordering less;
+    static const strong_ordering equal;
+    static const strong_ordering equivalent;
+    static const strong_ordering greater;
+
+    // conversions
+    constexpr operator partial_ordering() const noexcept;
+    constexpr operator weak_ordering() const noexcept;
+
+    // comparisons
+    friend constexpr bool operator==(strong_ordering v, unspecified) noexcept;
+    friend constexpr bool operator==(strong_ordering v, strong_ordering w) noexcept = default;
+    friend constexpr bool operator< (strong_ordering v, unspecified) noexcept;
+    friend constexpr bool operator> (strong_ordering v, unspecified) noexcept;
+    friend constexpr bool operator<=(strong_ordering v, unspecified) noexcept;
+    friend constexpr bool operator>=(strong_ordering v, unspecified) noexcept;
+    friend constexpr bool operator< (unspecified, strong_ordering v) noexcept;
+    friend constexpr bool operator> (unspecified, strong_ordering v) noexcept;
+    friend constexpr bool operator<=(unspecified, strong_ordering v) noexcept;
+    friend constexpr bool operator>=(unspecified, strong_ordering v) noexcept;
+    friend constexpr strong_ordering operator<=>(strong_ordering v, unspecified) noexcept;
+    friend constexpr strong_ordering operator<=>(unspecified, strong_ordering v) noexcept;
+  };
+}
+*/
+
+#include <__config>
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__compare/common_comparison_category.h>
+#  include <__compare/compare_partial_order_fallback.h>
+#  include <__compare/compare_strong_order_fallback.h>
+#  include <__compare/compare_three_way.h>
+#  include <__compare/compare_three_way_result.h>
+#  include <__compare/compare_weak_order_fallback.h>
+#  include <__compare/is_eq.h>
+#  include <__compare/ordering.h>
+#  include <__compare/partial_order.h>
+#  include <__compare/strong_order.h>
+#  include <__compare/synth_three_way.h>
+#  include <__compare/three_way_comparable.h>
+#  include <__compare/weak_order.h>
+#endif // _LIBCPP_STD_VER >= 20
+
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#  include <cstddef>
+#  include <cstdint>
+#  include <limits>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cmath>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_COMPARE
diff --git a/libcxx/include/__cxx03/complex b/libcxx/include/__cxx03/complex
new file mode 100644
index 000000000000..e6534025de57
--- /dev/null
+++ b/libcxx/include/__cxx03/complex
@@ -0,0 +1,1480 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_COMPLEX
+#define _LIBCPP_COMPLEX
+
+/*
+    complex synopsis
+
+namespace std
+{
+
+template<class T>
+class complex
+{
+public:
+    typedef T value_type;
+
+    complex(const T& re = T(), const T& im = T()); // constexpr in C++14
+    complex(const complex&);  // constexpr in C++14
+    template<class X> complex(const complex<X>&);  // constexpr in C++14
+
+    T real() const; // constexpr in C++14
+    T imag() const; // constexpr in C++14
+
+    void real(T); // constexpr in C++20
+    void imag(T); // constexpr in C++20
+
+    complex<T>& operator= (const T&); // constexpr in C++20
+    complex<T>& operator+=(const T&); // constexpr in C++20
+    complex<T>& operator-=(const T&); // constexpr in C++20
+    complex<T>& operator*=(const T&); // constexpr in C++20
+    complex<T>& operator/=(const T&); // constexpr in C++20
+
+    complex& operator=(const complex&); // constexpr in C++20
+    template<class X> complex<T>& operator= (const complex<X>&); // constexpr in C++20
+    template<class X> complex<T>& operator+=(const complex<X>&); // constexpr in C++20
+    template<class X> complex<T>& operator-=(const complex<X>&); // constexpr in C++20
+    template<class X> complex<T>& operator*=(const complex<X>&); // constexpr in C++20
+    template<class X> complex<T>& operator/=(const complex<X>&); // constexpr in C++20
+};
+
+template<>
+class complex<float>
+{
+public:
+    typedef float value_type;
+
+    constexpr complex(float re = 0.0f, float im = 0.0f);
+    explicit constexpr complex(const complex<double>&);
+    explicit constexpr complex(const complex<long double>&);
+
+    constexpr float real() const;
+    void real(float); // constexpr in C++20
+    constexpr float imag() const;
+    void imag(float); // constexpr in C++20
+
+    complex<float>& operator= (float); // constexpr in C++20
+    complex<float>& operator+=(float); // constexpr in C++20
+    complex<float>& operator-=(float); // constexpr in C++20
+    complex<float>& operator*=(float); // constexpr in C++20
+    complex<float>& operator/=(float); // constexpr in C++20
+
+    complex<float>& operator=(const complex<float>&); // constexpr in C++20
+    template<class X> complex<float>& operator= (const complex<X>&); // constexpr in C++20
+    template<class X> complex<float>& operator+=(const complex<X>&); // constexpr in C++20
+    template<class X> complex<float>& operator-=(const complex<X>&); // constexpr in C++20
+    template<class X> complex<float>& operator*=(const complex<X>&); // constexpr in C++20
+    template<class X> complex<float>& operator/=(const complex<X>&); // constexpr in C++20
+};
+
+template<>
+class complex<double>
+{
+public:
+    typedef double value_type;
+
+    constexpr complex(double re = 0.0, double im = 0.0);
+    constexpr complex(const complex<float>&);
+    explicit constexpr complex(const complex<long double>&);
+
+    constexpr double real() const;
+    void real(double); // constexpr in C++20
+    constexpr double imag() const;
+    void imag(double); // constexpr in C++20
+
+    complex<double>& operator= (double); // constexpr in C++20
+    complex<double>& operator+=(double); // constexpr in C++20
+    complex<double>& operator-=(double); // constexpr in C++20
+    complex<double>& operator*=(double); // constexpr in C++20
+    complex<double>& operator/=(double); // constexpr in C++20
+    complex<double>& operator=(const complex<double>&); // constexpr in C++20
+
+    template<class X> complex<double>& operator= (const complex<X>&); // constexpr in C++20
+    template<class X> complex<double>& operator+=(const complex<X>&); // constexpr in C++20
+    template<class X> complex<double>& operator-=(const complex<X>&); // constexpr in C++20
+    template<class X> complex<double>& operator*=(const complex<X>&); // constexpr in C++20
+    template<class X> complex<double>& operator/=(const complex<X>&); // constexpr in C++20
+};
+
+template<>
+class complex<long double>
+{
+public:
+    typedef long double value_type;
+
+    constexpr complex(long double re = 0.0L, long double im = 0.0L);
+    constexpr complex(const complex<float>&);
+    constexpr complex(const complex<double>&);
+
+    constexpr long double real() const;
+    void real(long double); // constexpr in C++20
+    constexpr long double imag() const;
+    void imag(long double); // constexpr in C++20
+
+    complex<long double>& operator=(const complex<long double>&); // constexpr in C++20
+    complex<long double>& operator= (long double); // constexpr in C++20
+    complex<long double>& operator+=(long double); // constexpr in C++20
+    complex<long double>& operator-=(long double); // constexpr in C++20
+    complex<long double>& operator*=(long double); // constexpr in C++20
+    complex<long double>& operator/=(long double); // constexpr in C++20
+
+    template<class X> complex<long double>& operator= (const complex<X>&); // constexpr in C++20
+    template<class X> complex<long double>& operator+=(const complex<X>&); // constexpr in C++20
+    template<class X> complex<long double>& operator-=(const complex<X>&); // constexpr in C++20
+    template<class X> complex<long double>& operator*=(const complex<X>&); // constexpr in C++20
+    template<class X> complex<long double>& operator/=(const complex<X>&); // constexpr in C++20
+};
+
+// 26.3.6 operators:
+template<class T> complex<T> operator+(const complex<T>&, const complex<T>&); // constexpr in C++20
+template<class T> complex<T> operator+(const complex<T>&, const T&);          // constexpr in C++20
+template<class T> complex<T> operator+(const T&, const complex<T>&);          // constexpr in C++20
+template<class T> complex<T> operator-(const complex<T>&, const complex<T>&); // constexpr in C++20
+template<class T> complex<T> operator-(const complex<T>&, const T&);          // constexpr in C++20
+template<class T> complex<T> operator-(const T&, const complex<T>&);          // constexpr in C++20
+template<class T> complex<T> operator*(const complex<T>&, const complex<T>&); // constexpr in C++20
+template<class T> complex<T> operator*(const complex<T>&, const T&);          // constexpr in C++20
+template<class T> complex<T> operator*(const T&, const complex<T>&);          // constexpr in C++20
+template<class T> complex<T> operator/(const complex<T>&, const complex<T>&); // constexpr in C++20
+template<class T> complex<T> operator/(const complex<T>&, const T&);          // constexpr in C++20
+template<class T> complex<T> operator/(const T&, const complex<T>&);          // constexpr in C++20
+template<class T> complex<T> operator+(const complex<T>&);                    // constexpr in C++20
+template<class T> complex<T> operator-(const complex<T>&);                    // constexpr in C++20
+template<class T> bool operator==(const complex<T>&, const complex<T>&);      // constexpr in C++14
+template<class T> bool operator==(const complex<T>&, const T&);               // constexpr in C++14
+template<class T> bool operator==(const T&, const complex<T>&);               // constexpr in C++14, removed in C++20
+template<class T> bool operator!=(const complex<T>&, const complex<T>&);      // constexpr in C++14, removed in C++20
+template<class T> bool operator!=(const complex<T>&, const T&);               // constexpr in C++14, removed in C++20
+template<class T> bool operator!=(const T&, const complex<T>&);               // constexpr in C++14, removed in C++20
+
+template<class T, class charT, class traits>
+  basic_istream<charT, traits>&
+  operator>>(basic_istream<charT, traits>&, complex<T>&);
+template<class T, class charT, class traits>
+  basic_ostream<charT, traits>&
+  operator<<(basic_ostream<charT, traits>&, const complex<T>&);
+
+// 26.3.7 values:
+
+template<class T>              T real(const complex<T>&); // constexpr in C++14
+                     long double real(long double);       // constexpr in C++14
+                          double real(double);            // constexpr in C++14
+template<Integral T>      double real(T);                 // constexpr in C++14
+                          float  real(float);             // constexpr in C++14
+
+template<class T>              T imag(const complex<T>&); // constexpr in C++14
+                     long double imag(long double);       // constexpr in C++14
+                          double imag(double);            // constexpr in C++14
+template<Integral T>      double imag(T);                 // constexpr in C++14
+                          float  imag(float);             // constexpr in C++14
+
+template<class T> T abs(const complex<T>&);
+
+template<class T>              T arg(const complex<T>&);
+                     long double arg(long double);
+                          double arg(double);
+template<Integral T>      double arg(T);
+                          float  arg(float);
+
+template<class T>              T norm(const complex<T>&); // constexpr in C++20
+                     long double norm(long double);       // constexpr in C++20
+                          double norm(double);            // constexpr in C++20
+template<Integral T>      double norm(T);                 // constexpr in C++20
+                          float  norm(float);             // constexpr in C++20
+
+template<class T>      complex<T>           conj(const complex<T>&); // constexpr in C++20
+                       complex<long double> conj(long double);       // constexpr in C++20
+                       complex<double>      conj(double);            // constexpr in C++20
+template<Integral T>   complex<double>      conj(T);                 // constexpr in C++20
+                       complex<float>       conj(float);             // constexpr in C++20
+
+template<class T>    complex<T>           proj(const complex<T>&);
+                     complex<long double> proj(long double);
+                     complex<double>      proj(double);
+template<Integral T> complex<double>      proj(T);
+                     complex<float>       proj(float);
+
+template<class T> complex<T> polar(const T&, const T& = T());
+
+// 26.3.8 transcendentals:
+template<class T> complex<T> acos(const complex<T>&);
+template<class T> complex<T> asin(const complex<T>&);
+template<class T> complex<T> atan(const complex<T>&);
+template<class T> complex<T> acosh(const complex<T>&);
+template<class T> complex<T> asinh(const complex<T>&);
+template<class T> complex<T> atanh(const complex<T>&);
+template<class T> complex<T> cos (const complex<T>&);
+template<class T> complex<T> cosh (const complex<T>&);
+template<class T> complex<T> exp (const complex<T>&);
+template<class T> complex<T> log (const complex<T>&);
+template<class T> complex<T> log10(const complex<T>&);
+
+template<class T> complex<T> pow(const complex<T>&, const T&);
+template<class T> complex<T> pow(const complex<T>&, const complex<T>&);
+template<class T> complex<T> pow(const T&, const complex<T>&);
+
+template<class T> complex<T> sin (const complex<T>&);
+template<class T> complex<T> sinh (const complex<T>&);
+template<class T> complex<T> sqrt (const complex<T>&);
+template<class T> complex<T> tan (const complex<T>&);
+template<class T> complex<T> tanh (const complex<T>&);
+
+  // [complex.tuple], tuple interface
+  template<class T> struct tuple_size;                               // Since C++26
+  template<size_t I, class T> struct tuple_element;                  // Since C++26
+  template<class T> struct tuple_size<complex<T>>;                   // Since C++26
+  template<size_t I, class T> struct tuple_element<I, complex<T>>;   // Since C++26
+  template<size_t I, class T>
+    constexpr T& get(complex<T>&) noexcept;                          // Since C++26
+  template<size_t I, class T>
+    constexpr T&& get(complex<T>&&) noexcept;                        // Since C++26
+  template<size_t I, class T>
+    constexpr const T& get(const complex<T>&) noexcept;              // Since C++26
+  template<size_t I, class T>
+    constexpr const T&& get(const complex<T>&&) noexcept;            // Since C++26
+
+  // [complex.literals], complex literals
+  inline namespace literals {
+  inline namespace complex_literals {
+    constexpr complex<long double> operator""il(long double);        // Since C++14
+    constexpr complex<long double> operator""il(unsigned long long); // Since C++14
+    constexpr complex<double> operator""i(long double);              // Since C++14
+    constexpr complex<double> operator""i(unsigned long long);       // Since C++14
+    constexpr complex<float> operator""if(long double);              // Since C++14
+    constexpr complex<float> operator""if(unsigned long long);       // Since C++14
+  }
+  }
+}  // std
+
+*/
+
+#include <__config>
+#include <__fwd/complex.h>
+#include <__fwd/tuple.h>
+#include <__tuple/tuple_element.h>
+#include <__tuple/tuple_size.h>
+#include <__type_traits/conditional.h>
+#include <__utility/move.h>
+#include <cmath>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <sstream> // for std::basic_ostringstream
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS complex;
+
+template <class _Tp, __enable_if_t<is_floating_point<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator*(const complex<_Tp>& __z, const complex<_Tp>& __w);
+
+template <class _Tp, __enable_if_t<!is_floating_point<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator*(const complex<_Tp>& __z, const complex<_Tp>& __w);
+
+template <class _Tp, __enable_if_t<is_floating_point<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator/(const complex<_Tp>& __x, const complex<_Tp>& __y);
+
+template <class _Tp, __enable_if_t<!is_floating_point<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator/(const complex<_Tp>& __x, const complex<_Tp>& __y);
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS complex {
+public:
+  typedef _Tp value_type;
+
+private:
+  value_type __re_;
+  value_type __im_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+  complex(const value_type& __re = value_type(), const value_type& __im = value_type())
+      : __re_(__re), __im_(__im) {}
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 complex(const complex<_Xp>& __c)
+      : __re_(__c.real()), __im_(__c.imag()) {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 value_type real() const { return __re_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 value_type imag() const { return __im_; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void real(value_type __re) { __re_ = __re; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void imag(value_type __im) { __im_ = __im; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator=(const value_type& __re) {
+    __re_ = __re;
+    __im_ = value_type();
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator+=(const value_type& __re) {
+    __re_ += __re;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator-=(const value_type& __re) {
+    __re_ -= __re;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator*=(const value_type& __re) {
+    __re_ *= __re;
+    __im_ *= __re;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator/=(const value_type& __re) {
+    __re_ /= __re;
+    __im_ /= __re;
+    return *this;
+  }
+
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator=(const complex<_Xp>& __c) {
+    __re_ = __c.real();
+    __im_ = __c.imag();
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator+=(const complex<_Xp>& __c) {
+    __re_ += __c.real();
+    __im_ += __c.imag();
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator-=(const complex<_Xp>& __c) {
+    __re_ -= __c.real();
+    __im_ -= __c.imag();
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator*=(const complex<_Xp>& __c) {
+    *this = *this * complex(__c.real(), __c.imag());
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator/=(const complex<_Xp>& __c) {
+    *this = *this / complex(__c.real(), __c.imag());
+    return *this;
+  }
+
+#if _LIBCPP_STD_VER >= 26
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr _Xp& get(complex<_Xp>&) noexcept;
+
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr _Xp&& get(complex<_Xp>&&) noexcept;
+
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr const _Xp& get(const complex<_Xp>&) noexcept;
+
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr const _Xp&& get(const complex<_Xp>&&) noexcept;
+#endif
+};
+
+template <>
+class _LIBCPP_TEMPLATE_VIS complex<double>;
+template <>
+class _LIBCPP_TEMPLATE_VIS complex<long double>;
+
+struct __from_builtin_tag {};
+
+template <class _Tp>
+using __complex_t =
+    __conditional_t<is_same<_Tp, float>::value,
+                    _Complex float,
+                    __conditional_t<is_same<_Tp, double>::value, _Complex double, _Complex long double> >;
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __complex_t<_Tp> __make_complex(_Tp __re, _Tp __im) {
+#if __has_builtin(__builtin_complex)
+  return __builtin_complex(__re, __im);
+#else
+  return __complex_t<_Tp>{__re, __im};
+#endif
+}
+
+template <>
+class _LIBCPP_TEMPLATE_VIS complex<float> {
+  float __re_;
+  float __im_;
+
+public:
+  typedef float value_type;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR complex(float __re = 0.0f, float __im = 0.0f) : __re_(__re), __im_(__im) {}
+
+  template <class _Tag, __enable_if_t<_IsSame<_Tag, __from_builtin_tag>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit complex(_Tag, _Complex float __v)
+      : __re_(__real__ __v), __im_(__imag__ __v) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit _LIBCPP_CONSTEXPR complex(const complex<double>& __c);
+  _LIBCPP_HIDE_FROM_ABI explicit _LIBCPP_CONSTEXPR complex(const complex<long double>& __c);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR float real() const { return __re_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR float imag() const { return __im_; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void real(value_type __re) { __re_ = __re; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void imag(value_type __im) { __im_ = __im; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Complex float __builtin() const { return std::__make_complex(__re_, __im_); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __builtin(_Complex float __f) {
+    __re_ = __real__ __f;
+    __im_ = __imag__ __f;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator=(float __re) {
+    __re_ = __re;
+    __im_ = value_type();
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator+=(float __re) {
+    __re_ += __re;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator-=(float __re) {
+    __re_ -= __re;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator*=(float __re) {
+    __re_ *= __re;
+    __im_ *= __re;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator/=(float __re) {
+    __re_ /= __re;
+    __im_ /= __re;
+    return *this;
+  }
+
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator=(const complex<_Xp>& __c) {
+    __re_ = __c.real();
+    __im_ = __c.imag();
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator+=(const complex<_Xp>& __c) {
+    __re_ += __c.real();
+    __im_ += __c.imag();
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator-=(const complex<_Xp>& __c) {
+    __re_ -= __c.real();
+    __im_ -= __c.imag();
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator*=(const complex<_Xp>& __c) {
+    *this = *this * complex(__c.real(), __c.imag());
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator/=(const complex<_Xp>& __c) {
+    *this = *this / complex(__c.real(), __c.imag());
+    return *this;
+  }
+
+#if _LIBCPP_STD_VER >= 26
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr _Xp& get(complex<_Xp>&) noexcept;
+
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr _Xp&& get(complex<_Xp>&&) noexcept;
+
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr const _Xp& get(const complex<_Xp>&) noexcept;
+
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr const _Xp&& get(const complex<_Xp>&&) noexcept;
+#endif
+};
+
+template <>
+class _LIBCPP_TEMPLATE_VIS complex<double> {
+  double __re_;
+  double __im_;
+
+public:
+  typedef double value_type;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR complex(double __re = 0.0, double __im = 0.0) : __re_(__re), __im_(__im) {}
+
+  template <class _Tag, __enable_if_t<_IsSame<_Tag, __from_builtin_tag>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit complex(_Tag, _Complex double __v)
+      : __re_(__real__ __v), __im_(__imag__ __v) {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR complex(const complex<float>& __c);
+  _LIBCPP_HIDE_FROM_ABI explicit _LIBCPP_CONSTEXPR complex(const complex<long double>& __c);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR double real() const { return __re_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR double imag() const { return __im_; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void real(value_type __re) { __re_ = __re; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void imag(value_type __im) { __im_ = __im; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Complex double __builtin() const {
+    return std::__make_complex(__re_, __im_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __builtin(_Complex double __f) {
+    __re_ = __real__ __f;
+    __im_ = __imag__ __f;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator=(double __re) {
+    __re_ = __re;
+    __im_ = value_type();
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator+=(double __re) {
+    __re_ += __re;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator-=(double __re) {
+    __re_ -= __re;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator*=(double __re) {
+    __re_ *= __re;
+    __im_ *= __re;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator/=(double __re) {
+    __re_ /= __re;
+    __im_ /= __re;
+    return *this;
+  }
+
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator=(const complex<_Xp>& __c) {
+    __re_ = __c.real();
+    __im_ = __c.imag();
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator+=(const complex<_Xp>& __c) {
+    __re_ += __c.real();
+    __im_ += __c.imag();
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator-=(const complex<_Xp>& __c) {
+    __re_ -= __c.real();
+    __im_ -= __c.imag();
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator*=(const complex<_Xp>& __c) {
+    *this = *this * complex(__c.real(), __c.imag());
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator/=(const complex<_Xp>& __c) {
+    *this = *this / complex(__c.real(), __c.imag());
+    return *this;
+  }
+
+#if _LIBCPP_STD_VER >= 26
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr _Xp& get(complex<_Xp>&) noexcept;
+
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr _Xp&& get(complex<_Xp>&&) noexcept;
+
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr const _Xp& get(const complex<_Xp>&) noexcept;
+
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr const _Xp&& get(const complex<_Xp>&&) noexcept;
+#endif
+};
+
+template <>
+class _LIBCPP_TEMPLATE_VIS complex<long double> {
+  long double __re_;
+  long double __im_;
+
+public:
+  typedef long double value_type;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR complex(long double __re = 0.0L, long double __im = 0.0L)
+      : __re_(__re), __im_(__im) {}
+
+  template <class _Tag, __enable_if_t<_IsSame<_Tag, __from_builtin_tag>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit complex(_Tag, _Complex long double __v)
+      : __re_(__real__ __v), __im_(__imag__ __v) {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR complex(const complex<float>& __c);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR complex(const complex<double>& __c);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR long double real() const { return __re_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR long double imag() const { return __im_; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void real(value_type __re) { __re_ = __re; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void imag(value_type __im) { __im_ = __im; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Complex long double __builtin() const {
+    return std::__make_complex(__re_, __im_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __builtin(_Complex long double __f) {
+    __re_ = __real__ __f;
+    __im_ = __imag__ __f;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator=(long double __re) {
+    __re_ = __re;
+    __im_ = value_type();
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator+=(long double __re) {
+    __re_ += __re;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator-=(long double __re) {
+    __re_ -= __re;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator*=(long double __re) {
+    __re_ *= __re;
+    __im_ *= __re;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator/=(long double __re) {
+    __re_ /= __re;
+    __im_ /= __re;
+    return *this;
+  }
+
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator=(const complex<_Xp>& __c) {
+    __re_ = __c.real();
+    __im_ = __c.imag();
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator+=(const complex<_Xp>& __c) {
+    __re_ += __c.real();
+    __im_ += __c.imag();
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator-=(const complex<_Xp>& __c) {
+    __re_ -= __c.real();
+    __im_ -= __c.imag();
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator*=(const complex<_Xp>& __c) {
+    *this = *this * complex(__c.real(), __c.imag());
+    return *this;
+  }
+  template <class _Xp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex& operator/=(const complex<_Xp>& __c) {
+    *this = *this / complex(__c.real(), __c.imag());
+    return *this;
+  }
+
+#if _LIBCPP_STD_VER >= 26
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr _Xp& get(complex<_Xp>&) noexcept;
+
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr _Xp&& get(complex<_Xp>&&) noexcept;
+
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr const _Xp& get(const complex<_Xp>&) noexcept;
+
+  template <size_t _Ip, class _Xp>
+  friend _LIBCPP_HIDE_FROM_ABI constexpr const _Xp&& get(const complex<_Xp>&&) noexcept;
+#endif
+};
+
+inline _LIBCPP_CONSTEXPR complex<float>::complex(const complex<double>& __c) : __re_(__c.real()), __im_(__c.imag()) {}
+
+inline _LIBCPP_CONSTEXPR complex<float>::complex(const complex<long double>& __c)
+    : __re_(__c.real()), __im_(__c.imag()) {}
+
+inline _LIBCPP_CONSTEXPR complex<double>::complex(const complex<float>& __c) : __re_(__c.real()), __im_(__c.imag()) {}
+
+inline _LIBCPP_CONSTEXPR complex<double>::complex(const complex<long double>& __c)
+    : __re_(__c.real()), __im_(__c.imag()) {}
+
+inline _LIBCPP_CONSTEXPR complex<long double>::complex(const complex<float>& __c)
+    : __re_(__c.real()), __im_(__c.imag()) {}
+
+inline _LIBCPP_CONSTEXPR complex<long double>::complex(const complex<double>& __c)
+    : __re_(__c.real()), __im_(__c.imag()) {}
+
+// 26.3.6 operators:
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator+(const complex<_Tp>& __x, const complex<_Tp>& __y) {
+  complex<_Tp> __t(__x);
+  __t += __y;
+  return __t;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator+(const complex<_Tp>& __x, const _Tp& __y) {
+  complex<_Tp> __t(__x);
+  __t += __y;
+  return __t;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator+(const _Tp& __x, const complex<_Tp>& __y) {
+  complex<_Tp> __t(__y);
+  __t += __x;
+  return __t;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator-(const complex<_Tp>& __x, const complex<_Tp>& __y) {
+  complex<_Tp> __t(__x);
+  __t -= __y;
+  return __t;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator-(const complex<_Tp>& __x, const _Tp& __y) {
+  complex<_Tp> __t(__x);
+  __t -= __y;
+  return __t;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator-(const _Tp& __x, const complex<_Tp>& __y) {
+  complex<_Tp> __t(-__y);
+  __t += __x;
+  return __t;
+}
+
+template <class _Tp, __enable_if_t<is_floating_point<_Tp>::value, int> >
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator*(const complex<_Tp>& __lhs, const complex<_Tp>& __rhs) {
+  return complex<_Tp>(__from_builtin_tag(), __lhs.__builtin() * __rhs.__builtin());
+}
+
+template <class _Tp, __enable_if_t<!is_floating_point<_Tp>::value, int> >
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator*(const complex<_Tp>& __z, const complex<_Tp>& __w) {
+  _Tp __a = __z.real();
+  _Tp __b = __z.imag();
+  _Tp __c = __w.real();
+  _Tp __d = __w.imag();
+
+  return complex<_Tp>((__a * __c) - (__b * __d), (__a * __d) + (__b * __c));
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator*(const complex<_Tp>& __x, const _Tp& __y) {
+  complex<_Tp> __t(__x);
+  __t *= __y;
+  return __t;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator*(const _Tp& __x, const complex<_Tp>& __y) {
+  complex<_Tp> __t(__y);
+  __t *= __x;
+  return __t;
+}
+
+template <class _Tp, __enable_if_t<is_floating_point<_Tp>::value, int> >
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator/(const complex<_Tp>& __lhs, const complex<_Tp>& __rhs) {
+  return complex<_Tp>(__from_builtin_tag(), __lhs.__builtin() / __rhs.__builtin());
+}
+
+template <class _Tp, __enable_if_t<!is_floating_point<_Tp>::value, int> >
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator/(const complex<_Tp>& __z, const complex<_Tp>& __w) {
+  _Tp __a = __z.real();
+  _Tp __b = __z.imag();
+  _Tp __c = __w.real();
+  _Tp __d = __w.imag();
+
+  _Tp __denom = __c * __c + __d * __d;
+  return complex<_Tp>((__a * __c + __b * __d) / __denom, (__b * __c - __a * __d) / __denom);
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator/(const complex<_Tp>& __x, const _Tp& __y) {
+  return complex<_Tp>(__x.real() / __y, __x.imag() / __y);
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
+operator/(const _Tp& __x, const complex<_Tp>& __y) {
+  complex<_Tp> __t(__x);
+  __t /= __y;
+  return __t;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp> operator+(const complex<_Tp>& __x) {
+  return __x;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp> operator-(const complex<_Tp>& __x) {
+  return complex<_Tp>(-__x.real(), -__x.imag());
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator==(const complex<_Tp>& __x, const complex<_Tp>& __y) {
+  return __x.real() == __y.real() && __x.imag() == __y.imag();
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator==(const complex<_Tp>& __x, const _Tp& __y) {
+  return __x.real() == __y && __x.imag() == 0;
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator==(const _Tp& __x, const complex<_Tp>& __y) {
+  return __x == __y.real() && 0 == __y.imag();
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator!=(const complex<_Tp>& __x, const complex<_Tp>& __y) {
+  return !(__x == __y);
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator!=(const complex<_Tp>& __x, const _Tp& __y) {
+  return !(__x == __y);
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator!=(const _Tp& __x, const complex<_Tp>& __y) {
+  return !(__x == __y);
+}
+
+#endif
+
+// 26.3.7 values:
+
+template <class _Tp, bool = is_integral<_Tp>::value, bool = is_floating_point<_Tp>::value >
+struct __libcpp_complex_overload_traits {};
+
+// Integral Types
+template <class _Tp>
+struct __libcpp_complex_overload_traits<_Tp, true, false> {
+  typedef double _ValueType;
+  typedef complex<double> _ComplexType;
+};
+
+// Floating point types
+template <class _Tp>
+struct __libcpp_complex_overload_traits<_Tp, false, true> {
+  typedef _Tp _ValueType;
+  typedef complex<_Tp> _ComplexType;
+};
+
+// real
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp real(const complex<_Tp>& __c) {
+  return __c.real();
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename __libcpp_complex_overload_traits<_Tp>::_ValueType
+real(_Tp __re) {
+  return __re;
+}
+
+// imag
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp imag(const complex<_Tp>& __c) {
+  return __c.imag();
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename __libcpp_complex_overload_traits<_Tp>::_ValueType
+imag(_Tp) {
+  return 0;
+}
+
+// abs
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _Tp abs(const complex<_Tp>& __c) {
+  return std::hypot(__c.real(), __c.imag());
+}
+
+// arg
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _Tp arg(const complex<_Tp>& __c) {
+  return std::atan2(__c.imag(), __c.real());
+}
+
+template <class _Tp, __enable_if_t<is_same<_Tp, long double>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI long double arg(_Tp __re) {
+  return std::atan2l(0.L, __re);
+}
+
+template <class _Tp, __enable_if_t<is_integral<_Tp>::value || is_same<_Tp, double>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI double arg(_Tp __re) {
+  return std::atan2(0., __re);
+}
+
+template <class _Tp, __enable_if_t<is_same<_Tp, float>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI float arg(_Tp __re) {
+  return std::atan2f(0.F, __re);
+}
+
+// norm
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp norm(const complex<_Tp>& __c) {
+  if (std::__constexpr_isinf(__c.real()))
+    return std::abs(__c.real());
+  if (std::__constexpr_isinf(__c.imag()))
+    return std::abs(__c.imag());
+  return __c.real() * __c.real() + __c.imag() * __c.imag();
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __libcpp_complex_overload_traits<_Tp>::_ValueType
+norm(_Tp __re) {
+  typedef typename __libcpp_complex_overload_traits<_Tp>::_ValueType _ValueType;
+  return static_cast<_ValueType>(__re) * __re;
+}
+
+// conj
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp> conj(const complex<_Tp>& __c) {
+  return complex<_Tp>(__c.real(), -__c.imag());
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __libcpp_complex_overload_traits<_Tp>::_ComplexType
+conj(_Tp __re) {
+  typedef typename __libcpp_complex_overload_traits<_Tp>::_ComplexType _ComplexType;
+  return _ComplexType(__re);
+}
+
+// proj
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> proj(const complex<_Tp>& __c) {
+  complex<_Tp> __r = __c;
+  if (std::__constexpr_isinf(__c.real()) || std::__constexpr_isinf(__c.imag()))
+    __r = complex<_Tp>(INFINITY, std::copysign(_Tp(0), __c.imag()));
+  return __r;
+}
+
+template <class _Tp, __enable_if_t<is_floating_point<_Tp>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI typename __libcpp_complex_overload_traits<_Tp>::_ComplexType proj(_Tp __re) {
+  if (std::__constexpr_isinf(__re))
+    __re = std::abs(__re);
+  return complex<_Tp>(__re);
+}
+
+template <class _Tp, __enable_if_t<is_integral<_Tp>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI typename __libcpp_complex_overload_traits<_Tp>::_ComplexType proj(_Tp __re) {
+  typedef typename __libcpp_complex_overload_traits<_Tp>::_ComplexType _ComplexType;
+  return _ComplexType(__re);
+}
+
+// polar
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI complex<_Tp> polar(const _Tp& __rho, const _Tp& __theta = _Tp()) {
+  if (std::__constexpr_isnan(__rho) || std::signbit(__rho))
+    return complex<_Tp>(_Tp(NAN), _Tp(NAN));
+  if (std::__constexpr_isnan(__theta)) {
+    if (std::__constexpr_isinf(__rho))
+      return complex<_Tp>(__rho, __theta);
+    return complex<_Tp>(__theta, __theta);
+  }
+  if (std::__constexpr_isinf(__theta)) {
+    if (std::__constexpr_isinf(__rho))
+      return complex<_Tp>(__rho, _Tp(NAN));
+    return complex<_Tp>(_Tp(NAN), _Tp(NAN));
+  }
+  _Tp __x = __rho * std::cos(__theta);
+  if (std::__constexpr_isnan(__x))
+    __x = 0;
+  _Tp __y = __rho * std::sin(__theta);
+  if (std::__constexpr_isnan(__y))
+    __y = 0;
+  return complex<_Tp>(__x, __y);
+}
+
+// log
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> log(const complex<_Tp>& __x) {
+  return complex<_Tp>(std::log(std::abs(__x)), std::arg(__x));
+}
+
+// log10
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> log10(const complex<_Tp>& __x) {
+  return std::log(__x) / std::log(_Tp(10));
+}
+
+// sqrt
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI complex<_Tp> sqrt(const complex<_Tp>& __x) {
+  if (std::__constexpr_isinf(__x.imag()))
+    return complex<_Tp>(_Tp(INFINITY), __x.imag());
+  if (std::__constexpr_isinf(__x.real())) {
+    if (__x.real() > _Tp(0))
+      return complex<_Tp>(
+          __x.real(), std::__constexpr_isnan(__x.imag()) ? __x.imag() : std::copysign(_Tp(0), __x.imag()));
+    return complex<_Tp>(
+        std::__constexpr_isnan(__x.imag()) ? __x.imag() : _Tp(0), std::copysign(__x.real(), __x.imag()));
+  }
+  return std::polar(std::sqrt(std::abs(__x)), std::arg(__x) / _Tp(2));
+}
+
+// exp
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI complex<_Tp> exp(const complex<_Tp>& __x) {
+  _Tp __i = __x.imag();
+  if (__i == 0) {
+    return complex<_Tp>(std::exp(__x.real()), std::copysign(_Tp(0), __x.imag()));
+  }
+  if (std::__constexpr_isinf(__x.real())) {
+    if (__x.real() < _Tp(0)) {
+      if (!std::__constexpr_isfinite(__i))
+        __i = _Tp(1);
+    } else if (__i == 0 || !std::__constexpr_isfinite(__i)) {
+      if (std::__constexpr_isinf(__i))
+        __i = _Tp(NAN);
+      return complex<_Tp>(__x.real(), __i);
+    }
+  }
+  _Tp __e = std::exp(__x.real());
+  return complex<_Tp>(__e * std::cos(__i), __e * std::sin(__i));
+}
+
+// pow
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> pow(const complex<_Tp>& __x, const complex<_Tp>& __y) {
+  return std::exp(__y * std::log(__x));
+}
+
+template <class _Tp, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI complex<typename __promote<_Tp, _Up>::type>
+pow(const complex<_Tp>& __x, const complex<_Up>& __y) {
+  typedef complex<typename __promote<_Tp, _Up>::type> result_type;
+  return std::pow(result_type(__x), result_type(__y));
+}
+
+template <class _Tp, class _Up, __enable_if_t<is_arithmetic<_Up>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI complex<typename __promote<_Tp, _Up>::type> pow(const complex<_Tp>& __x, const _Up& __y) {
+  typedef complex<typename __promote<_Tp, _Up>::type> result_type;
+  return std::pow(result_type(__x), result_type(__y));
+}
+
+template <class _Tp, class _Up, __enable_if_t<is_arithmetic<_Tp>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI complex<typename __promote<_Tp, _Up>::type> pow(const _Tp& __x, const complex<_Up>& __y) {
+  typedef complex<typename __promote<_Tp, _Up>::type> result_type;
+  return std::pow(result_type(__x), result_type(__y));
+}
+
+// __sqr, computes pow(x, 2)
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> __sqr(const complex<_Tp>& __x) {
+  return complex<_Tp>((__x.real() - __x.imag()) * (__x.real() + __x.imag()), _Tp(2) * __x.real() * __x.imag());
+}
+
+// asinh
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI complex<_Tp> asinh(const complex<_Tp>& __x) {
+  const _Tp __pi(atan2(+0., -0.));
+  if (std::__constexpr_isinf(__x.real())) {
+    if (std::__constexpr_isnan(__x.imag()))
+      return __x;
+    if (std::__constexpr_isinf(__x.imag()))
+      return complex<_Tp>(__x.real(), std::copysign(__pi * _Tp(0.25), __x.imag()));
+    return complex<_Tp>(__x.real(), std::copysign(_Tp(0), __x.imag()));
+  }
+  if (std::__constexpr_isnan(__x.real())) {
+    if (std::__constexpr_isinf(__x.imag()))
+      return complex<_Tp>(__x.imag(), __x.real());
+    if (__x.imag() == 0)
+      return __x;
+    return complex<_Tp>(__x.real(), __x.real());
+  }
+  if (std::__constexpr_isinf(__x.imag()))
+    return complex<_Tp>(std::copysign(__x.imag(), __x.real()), std::copysign(__pi / _Tp(2), __x.imag()));
+  complex<_Tp> __z = std::log(__x + std::sqrt(std::__sqr(__x) + _Tp(1)));
+  return complex<_Tp>(std::copysign(__z.real(), __x.real()), std::copysign(__z.imag(), __x.imag()));
+}
+
+// acosh
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI complex<_Tp> acosh(const complex<_Tp>& __x) {
+  const _Tp __pi(atan2(+0., -0.));
+  if (std::__constexpr_isinf(__x.real())) {
+    if (std::__constexpr_isnan(__x.imag()))
+      return complex<_Tp>(std::abs(__x.real()), __x.imag());
+    if (std::__constexpr_isinf(__x.imag())) {
+      if (__x.real() > 0)
+        return complex<_Tp>(__x.real(), std::copysign(__pi * _Tp(0.25), __x.imag()));
+      else
+        return complex<_Tp>(-__x.real(), std::copysign(__pi * _Tp(0.75), __x.imag()));
+    }
+    if (__x.real() < 0)
+      return complex<_Tp>(-__x.real(), std::copysign(__pi, __x.imag()));
+    return complex<_Tp>(__x.real(), std::copysign(_Tp(0), __x.imag()));
+  }
+  if (std::__constexpr_isnan(__x.real())) {
+    if (std::__constexpr_isinf(__x.imag()))
+      return complex<_Tp>(std::abs(__x.imag()), __x.real());
+    return complex<_Tp>(__x.real(), __x.real());
+  }
+  if (std::__constexpr_isinf(__x.imag()))
+    return complex<_Tp>(std::abs(__x.imag()), std::copysign(__pi / _Tp(2), __x.imag()));
+  complex<_Tp> __z = std::log(__x + std::sqrt(std::__sqr(__x) - _Tp(1)));
+  return complex<_Tp>(std::copysign(__z.real(), _Tp(0)), std::copysign(__z.imag(), __x.imag()));
+}
+
+// atanh
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI complex<_Tp> atanh(const complex<_Tp>& __x) {
+  const _Tp __pi(atan2(+0., -0.));
+  if (std::__constexpr_isinf(__x.imag())) {
+    return complex<_Tp>(std::copysign(_Tp(0), __x.real()), std::copysign(__pi / _Tp(2), __x.imag()));
+  }
+  if (std::__constexpr_isnan(__x.imag())) {
+    if (std::__constexpr_isinf(__x.real()) || __x.real() == 0)
+      return complex<_Tp>(std::copysign(_Tp(0), __x.real()), __x.imag());
+    return complex<_Tp>(__x.imag(), __x.imag());
+  }
+  if (std::__constexpr_isnan(__x.real())) {
+    return complex<_Tp>(__x.real(), __x.real());
+  }
+  if (std::__constexpr_isinf(__x.real())) {
+    return complex<_Tp>(std::copysign(_Tp(0), __x.real()), std::copysign(__pi / _Tp(2), __x.imag()));
+  }
+  if (std::abs(__x.real()) == _Tp(1) && __x.imag() == _Tp(0)) {
+    return complex<_Tp>(std::copysign(_Tp(INFINITY), __x.real()), std::copysign(_Tp(0), __x.imag()));
+  }
+  complex<_Tp> __z = std::log((_Tp(1) + __x) / (_Tp(1) - __x)) / _Tp(2);
+  return complex<_Tp>(std::copysign(__z.real(), __x.real()), std::copysign(__z.imag(), __x.imag()));
+}
+
+// sinh
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI complex<_Tp> sinh(const complex<_Tp>& __x) {
+  if (std::__constexpr_isinf(__x.real()) && !std::__constexpr_isfinite(__x.imag()))
+    return complex<_Tp>(__x.real(), _Tp(NAN));
+  if (__x.real() == 0 && !std::__constexpr_isfinite(__x.imag()))
+    return complex<_Tp>(__x.real(), _Tp(NAN));
+  if (__x.imag() == 0 && !std::__constexpr_isfinite(__x.real()))
+    return __x;
+  return complex<_Tp>(std::sinh(__x.real()) * std::cos(__x.imag()), std::cosh(__x.real()) * std::sin(__x.imag()));
+}
+
+// cosh
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI complex<_Tp> cosh(const complex<_Tp>& __x) {
+  if (std::__constexpr_isinf(__x.real()) && !std::__constexpr_isfinite(__x.imag()))
+    return complex<_Tp>(std::abs(__x.real()), _Tp(NAN));
+  if (__x.real() == 0 && !std::__constexpr_isfinite(__x.imag()))
+    return complex<_Tp>(_Tp(NAN), __x.real());
+  if (__x.real() == 0 && __x.imag() == 0)
+    return complex<_Tp>(_Tp(1), __x.imag());
+  if (__x.imag() == 0 && !std::__constexpr_isfinite(__x.real()))
+    return complex<_Tp>(std::abs(__x.real()), __x.imag());
+  return complex<_Tp>(std::cosh(__x.real()) * std::cos(__x.imag()), std::sinh(__x.real()) * std::sin(__x.imag()));
+}
+
+// tanh
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI complex<_Tp> tanh(const complex<_Tp>& __x) {
+  if (std::__constexpr_isinf(__x.real())) {
+    if (!std::__constexpr_isfinite(__x.imag()))
+      return complex<_Tp>(std::copysign(_Tp(1), __x.real()), _Tp(0));
+    return complex<_Tp>(std::copysign(_Tp(1), __x.real()), std::copysign(_Tp(0), std::sin(_Tp(2) * __x.imag())));
+  }
+  if (std::__constexpr_isnan(__x.real()) && __x.imag() == 0)
+    return __x;
+  _Tp __2r(_Tp(2) * __x.real());
+  _Tp __2i(_Tp(2) * __x.imag());
+  _Tp __d(std::cosh(__2r) + std::cos(__2i));
+  _Tp __2rsh(std::sinh(__2r));
+  if (std::__constexpr_isinf(__2rsh) && std::__constexpr_isinf(__d))
+    return complex<_Tp>(__2rsh > _Tp(0) ? _Tp(1) : _Tp(-1), __2i > _Tp(0) ? _Tp(0) : _Tp(-0.));
+  return complex<_Tp>(__2rsh / __d, std::sin(__2i) / __d);
+}
+
+// asin
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI complex<_Tp> asin(const complex<_Tp>& __x) {
+  complex<_Tp> __z = std::asinh(complex<_Tp>(-__x.imag(), __x.real()));
+  return complex<_Tp>(__z.imag(), -__z.real());
+}
+
+// acos
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI complex<_Tp> acos(const complex<_Tp>& __x) {
+  const _Tp __pi(atan2(+0., -0.));
+  if (std::__constexpr_isinf(__x.real())) {
+    if (std::__constexpr_isnan(__x.imag()))
+      return complex<_Tp>(__x.imag(), __x.real());
+    if (std::__constexpr_isinf(__x.imag())) {
+      if (__x.real() < _Tp(0))
+        return complex<_Tp>(_Tp(0.75) * __pi, -__x.imag());
+      return complex<_Tp>(_Tp(0.25) * __pi, -__x.imag());
+    }
+    if (__x.real() < _Tp(0))
+      return complex<_Tp>(__pi, std::signbit(__x.imag()) ? -__x.real() : __x.real());
+    return complex<_Tp>(_Tp(0), std::signbit(__x.imag()) ? __x.real() : -__x.real());
+  }
+  if (std::__constexpr_isnan(__x.real())) {
+    if (std::__constexpr_isinf(__x.imag()))
+      return complex<_Tp>(__x.real(), -__x.imag());
+    return complex<_Tp>(__x.real(), __x.real());
+  }
+  if (std::__constexpr_isinf(__x.imag()))
+    return complex<_Tp>(__pi / _Tp(2), -__x.imag());
+  if (__x.real() == 0 && (__x.imag() == 0 || std::isnan(__x.imag())))
+    return complex<_Tp>(__pi / _Tp(2), -__x.imag());
+  complex<_Tp> __z = std::log(__x + std::sqrt(std::__sqr(__x) - _Tp(1)));
+  if (std::signbit(__x.imag()))
+    return complex<_Tp>(std::abs(__z.imag()), std::abs(__z.real()));
+  return complex<_Tp>(std::abs(__z.imag()), -std::abs(__z.real()));
+}
+
+// atan
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI complex<_Tp> atan(const complex<_Tp>& __x) {
+  complex<_Tp> __z = std::atanh(complex<_Tp>(-__x.imag(), __x.real()));
+  return complex<_Tp>(__z.imag(), -__z.real());
+}
+
+// sin
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI complex<_Tp> sin(const complex<_Tp>& __x) {
+  complex<_Tp> __z = std::sinh(complex<_Tp>(-__x.imag(), __x.real()));
+  return complex<_Tp>(__z.imag(), -__z.real());
+}
+
+// cos
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> cos(const complex<_Tp>& __x) {
+  return std::cosh(complex<_Tp>(-__x.imag(), __x.real()));
+}
+
+// tan
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI complex<_Tp> tan(const complex<_Tp>& __x) {
+  complex<_Tp> __z = std::tanh(complex<_Tp>(-__x.imag(), __x.real()));
+  return complex<_Tp>(__z.imag(), -__z.real());
+}
+
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+template <class _Tp, class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, complex<_Tp>& __x) {
+  if (__is.good()) {
+    std::ws(__is);
+    if (__is.peek() == _CharT('(')) {
+      __is.get();
+      _Tp __r;
+      __is >> __r;
+      if (!__is.fail()) {
+        std::ws(__is);
+        _CharT __c = __is.peek();
+        if (__c == _CharT(',')) {
+          __is.get();
+          _Tp __i;
+          __is >> __i;
+          if (!__is.fail()) {
+            std::ws(__is);
+            __c = __is.peek();
+            if (__c == _CharT(')')) {
+              __is.get();
+              __x = complex<_Tp>(__r, __i);
+            } else
+              __is.setstate(__is.failbit);
+          } else
+            __is.setstate(__is.failbit);
+        } else if (__c == _CharT(')')) {
+          __is.get();
+          __x = complex<_Tp>(__r, _Tp(0));
+        } else
+          __is.setstate(__is.failbit);
+      } else
+        __is.setstate(__is.failbit);
+    } else {
+      _Tp __r;
+      __is >> __r;
+      if (!__is.fail())
+        __x = complex<_Tp>(__r, _Tp(0));
+      else
+        __is.setstate(__is.failbit);
+    }
+  } else
+    __is.setstate(__is.failbit);
+  return __is;
+}
+
+template <class _Tp, class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const complex<_Tp>& __x) {
+  basic_ostringstream<_CharT, _Traits> __s;
+  __s.flags(__os.flags());
+  __s.imbue(__os.getloc());
+  __s.precision(__os.precision());
+  __s << '(' << __x.real() << ',' << __x.imag() << ')';
+  return __os << __s.str();
+}
+#endif // !_LIBCPP_HAS_NO_LOCALIZATION
+
+#if _LIBCPP_STD_VER >= 26
+
+// [complex.tuple], tuple interface
+
+template <class _Tp>
+struct tuple_size<complex<_Tp>> : integral_constant<size_t, 2> {};
+
+template <size_t _Ip, class _Tp>
+struct tuple_element<_Ip, complex<_Tp>> {
+  static_assert(_Ip < 2, "Index value is out of range.");
+  using type = _Tp;
+};
+
+template <size_t _Ip, class _Xp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Xp& get(complex<_Xp>& __z) noexcept {
+  static_assert(_Ip < 2, "Index value is out of range.");
+  if constexpr (_Ip == 0) {
+    return __z.__re_;
+  } else {
+    return __z.__im_;
+  }
+}
+
+template <size_t _Ip, class _Xp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Xp&& get(complex<_Xp>&& __z) noexcept {
+  static_assert(_Ip < 2, "Index value is out of range.");
+  if constexpr (_Ip == 0) {
+    return std::move(__z.__re_);
+  } else {
+    return std::move(__z.__im_);
+  }
+}
+
+template <size_t _Ip, class _Xp>
+_LIBCPP_HIDE_FROM_ABI constexpr const _Xp& get(const complex<_Xp>& __z) noexcept {
+  static_assert(_Ip < 2, "Index value is out of range.");
+  if constexpr (_Ip == 0) {
+    return __z.__re_;
+  } else {
+    return __z.__im_;
+  }
+}
+
+template <size_t _Ip, class _Xp>
+_LIBCPP_HIDE_FROM_ABI constexpr const _Xp&& get(const complex<_Xp>&& __z) noexcept {
+  static_assert(_Ip < 2, "Index value is out of range.");
+  if constexpr (_Ip == 0) {
+    return std::move(__z.__re_);
+  } else {
+    return std::move(__z.__im_);
+  }
+}
+
+#endif // _LIBCPP_STD_VER >= 26
+
+#if _LIBCPP_STD_VER >= 14
+// Literal suffix for complex number literals [complex.literals]
+inline namespace literals {
+inline namespace complex_literals {
+_LIBCPP_HIDE_FROM_ABI inline constexpr complex<long double> operator""il(long double __im) { return {0.0l, __im}; }
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr complex<long double> operator""il(unsigned long long __im) {
+  return {0.0l, static_cast<long double>(__im)};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr complex<double> operator""i(long double __im) {
+  return {0.0, static_cast<double>(__im)};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr complex<double> operator""i(unsigned long long __im) {
+  return {0.0, static_cast<double>(__im)};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr complex<float> operator""if(long double __im) {
+  return {0.0f, static_cast<float>(__im)};
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr complex<float> operator""if(unsigned long long __im) {
+  return {0.0f, static_cast<float>(__im)};
+}
+} // namespace complex_literals
+} // namespace literals
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <iosfwd>
+#  include <stdexcept>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_COMPLEX
diff --git a/libcxx/include/__cxx03/complex.h b/libcxx/include/__cxx03/complex.h
new file mode 100644
index 000000000000..a3da21c843f3
--- /dev/null
+++ b/libcxx/include/__cxx03/complex.h
@@ -0,0 +1,32 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_COMPLEX_H
+#define _LIBCPP_COMPLEX_H
+
+/*
+    complex.h synopsis
+
+#include <ccomplex>
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#ifdef __cplusplus
+#  include <ccomplex>
+#elif __has_include_next(<complex.h>)
+#  include_next <complex.h>
+#endif
+
+#endif // _LIBCPP_COMPLEX_H
diff --git a/libcxx/include/__cxx03/concepts b/libcxx/include/__cxx03/concepts
new file mode 100644
index 000000000000..e89d216a5937
--- /dev/null
+++ b/libcxx/include/__cxx03/concepts
@@ -0,0 +1,173 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CONCEPTS
+#define _LIBCPP_CONCEPTS
+
+/*
+    concepts synopsis
+namespace std {
+  // [concepts.lang], language-related concepts
+  // [concept.same], concept same_as
+  template<class T, class U>
+    concept same_as = see below;
+
+  // [concept.derived], concept derived_from
+  template<class Derived, class Base>
+    concept derived_from = see below;
+
+  // [concept.convertible], concept convertible_to
+  template<class From, class To>
+    concept convertible_to = see below;
+
+  // [concept.commonref], concept common_reference_with
+  template<class T, class U>
+    concept common_reference_with = see below;
+
+  // [concept.common], concept common_with
+  template<class T, class U>
+    concept common_with = see below;
+
+  // [concepts.arithmetic], arithmetic concepts
+  template<class T>
+    concept integral = see below;
+  template<class T>
+    concept signed_integral = see below;
+  template<class T>
+    concept unsigned_integral = see below;
+  template<class T>
+    concept floating_point = see below;
+
+  // [concept.assignable], concept assignable_from
+  template<class LHS, class RHS>
+    concept assignable_from = see below;
+
+  // [concept.swappable], concept swappable
+  namespace ranges {
+    inline namespace unspecified {
+      inline constexpr unspecified swap = unspecified;
+    }
+  }
+  template<class T>
+    concept swappable = see below;
+  template<class T, class U>
+    concept swappable_with = see below;
+
+  // [concept.destructible], concept destructible
+  template<class T>
+    concept destructible = see below;
+
+  // [concept.constructible], concept constructible_from
+  template<class T, class... Args>
+    concept constructible_from = see below;
+
+  // [concept.default.init], concept default_initializable
+  template<class T>
+    concept default_initializable = see below;
+
+  // [concept.moveconstructible], concept move_constructible
+  template<class T>
+    concept move_constructible = see below;
+
+  // [concept.copyconstructible], concept copy_constructible
+  template<class T>
+    concept copy_constructible = see below;
+
+  // [concept.equalitycomparable], concept equality_comparable
+  template<class T>
+    concept equality_comparable = see below;
+  template<class T, class U>
+    concept equality_comparable_with = see below;
+
+  // [concept.totallyordered], concept totally_ordered
+  template<class T>
+    concept totally_ordered = see below;
+  template<class T, class U>
+    concept totally_ordered_with = see below;
+
+  // [concepts.object], object concepts
+  template<class T>
+    concept movable = see below;
+  template<class T>
+    concept copyable = see below;
+  template<class T>
+    concept semiregular = see below;
+  template<class T>
+    concept regular = see below;
+
+  // [concepts.callable], callable concepts
+  // [concept.invocable], concept invocable
+  template<class F, class... Args>
+    concept invocable = see below;
+
+  // [concept.regularinvocable], concept regular_invocable
+  template<class F, class... Args>
+    concept regular_invocable = see below;
+
+  // [concept.predicate], concept predicate
+  template<class F, class... Args>
+    concept predicate = see below;
+
+  // [concept.relation], concept relation
+  template<class R, class T, class U>
+    concept relation = see below;
+
+  // [concept.equiv], concept equivalence_relation
+  template<class R, class T, class U>
+    concept equivalence_relation = see below;
+
+  // [concept.strictweakorder], concept strict_weak_order
+  template<class R, class T, class U>
+    concept strict_weak_order = see below;
+}
+
+*/
+
+#include <__config>
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__concepts/arithmetic.h>
+#  include <__concepts/assignable.h>
+#  include <__concepts/boolean_testable.h>
+#  include <__concepts/class_or_enum.h>
+#  include <__concepts/common_reference_with.h>
+#  include <__concepts/common_with.h>
+#  include <__concepts/constructible.h>
+#  include <__concepts/convertible_to.h>
+#  include <__concepts/copyable.h>
+#  include <__concepts/derived_from.h>
+#  include <__concepts/destructible.h>
+#  include <__concepts/different_from.h>
+#  include <__concepts/equality_comparable.h>
+#  include <__concepts/invocable.h>
+#  include <__concepts/movable.h>
+#  include <__concepts/predicate.h>
+#  include <__concepts/regular.h>
+#  include <__concepts/relation.h>
+#  include <__concepts/same_as.h>
+#  include <__concepts/semiregular.h>
+#  include <__concepts/swappable.h>
+#  include <__concepts/totally_ordered.h>
+#endif // _LIBCPP_STD_VER >= 20
+
+#include <version>
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#  include <cstddef>
+#endif
+
+#if _LIBCPP_STD_VER <= 20 && !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES)
+#  include <type_traits>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#endif // _LIBCPP_CONCEPTS
diff --git a/libcxx/include/__cxx03/condition_variable b/libcxx/include/__cxx03/condition_variable
new file mode 100644
index 000000000000..5195cd6057dd
--- /dev/null
+++ b/libcxx/include/__cxx03/condition_variable
@@ -0,0 +1,369 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CONDITION_VARIABLE
+#define _LIBCPP_CONDITION_VARIABLE
+
+/*
+    condition_variable synopsis
+
+namespace std
+{
+
+enum class cv_status { no_timeout, timeout };
+
+class condition_variable
+{
+public:
+    condition_variable();
+    ~condition_variable();
+
+    condition_variable(const condition_variable&) = delete;
+    condition_variable& operator=(const condition_variable&) = delete;
+
+    void notify_one() noexcept;
+    void notify_all() noexcept;
+
+    void wait(unique_lock<mutex>& lock);
+    template <class Predicate>
+        void wait(unique_lock<mutex>& lock, Predicate pred);
+
+    template <class Clock, class Duration>
+        cv_status
+        wait_until(unique_lock<mutex>& lock,
+                   const chrono::time_point<Clock, Duration>& abs_time);
+
+    template <class Clock, class Duration, class Predicate>
+        bool
+        wait_until(unique_lock<mutex>& lock,
+                   const chrono::time_point<Clock, Duration>& abs_time,
+                   Predicate pred);
+
+    template <class Rep, class Period>
+        cv_status
+        wait_for(unique_lock<mutex>& lock,
+                 const chrono::duration<Rep, Period>& rel_time);
+
+    template <class Rep, class Period, class Predicate>
+        bool
+        wait_for(unique_lock<mutex>& lock,
+                 const chrono::duration<Rep, Period>& rel_time,
+                 Predicate pred);
+
+    typedef pthread_cond_t* native_handle_type;
+    native_handle_type native_handle();
+};
+
+void notify_all_at_thread_exit(condition_variable& cond, unique_lock<mutex> lk);
+
+class condition_variable_any
+{
+public:
+    condition_variable_any();
+    ~condition_variable_any();
+
+    condition_variable_any(const condition_variable_any&) = delete;
+    condition_variable_any& operator=(const condition_variable_any&) = delete;
+
+    void notify_one() noexcept;
+    void notify_all() noexcept;
+
+    template <class Lock>
+        void wait(Lock& lock);
+    template <class Lock, class Predicate>
+        void wait(Lock& lock, Predicate pred);
+
+    template <class Lock, class Clock, class Duration>
+        cv_status
+        wait_until(Lock& lock,
+                   const chrono::time_point<Clock, Duration>& abs_time);
+
+    template <class Lock, class Clock, class Duration, class Predicate>
+        bool
+        wait_until(Lock& lock,
+                   const chrono::time_point<Clock, Duration>& abs_time,
+                   Predicate pred);
+
+    template <class Lock, class Rep, class Period>
+        cv_status
+        wait_for(Lock& lock,
+                 const chrono::duration<Rep, Period>& rel_time);
+
+    template <class Lock, class Rep, class Period, class Predicate>
+        bool
+        wait_for(Lock& lock,
+                 const chrono::duration<Rep, Period>& rel_time,
+                 Predicate pred);
+
+    // [thread.condvarany.intwait], interruptible waits
+    template <class Lock, class Predicate>
+      bool wait(Lock& lock, stop_token stoken, Predicate pred);                               // since C++20
+
+    template <class Lock, class Clock, class Duration, class Predicate>
+      bool wait_until(Lock& lock, stop_token stoken,
+                      const chrono::time_point<Clock, Duration>& abs_time, Predicate pred);   // since C++20
+
+    template <class Lock, class Rep, class Period, class Predicate>
+      bool wait_for(Lock& lock, stop_token stoken,
+                    const chrono::duration<Rep, Period>& rel_time, Predicate pred);           // since C++20
+};
+
+}  // std
+
+*/
+
+#include <__chrono/duration.h>
+#include <__chrono/steady_clock.h>
+#include <__chrono/time_point.h>
+#include <__condition_variable/condition_variable.h>
+#include <__config>
+#include <__memory/shared_ptr.h>
+#include <__mutex/lock_guard.h>
+#include <__mutex/mutex.h>
+#include <__mutex/tag_types.h>
+#include <__mutex/unique_lock.h>
+#include <__stop_token/stop_callback.h>
+#include <__stop_token/stop_token.h>
+#include <__utility/move.h>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#ifndef _LIBCPP_HAS_NO_THREADS
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+class _LIBCPP_EXPORTED_FROM_ABI condition_variable_any {
+  condition_variable __cv_;
+  shared_ptr<mutex> __mut_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI condition_variable_any();
+
+  _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT;
+
+  template <class _Lock>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS void wait(_Lock& __lock);
+  template <class _Lock, class _Predicate>
+  _LIBCPP_HIDE_FROM_ABI void wait(_Lock& __lock, _Predicate __pred);
+
+  template <class _Lock, class _Clock, class _Duration>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS cv_status
+  wait_until(_Lock& __lock, const chrono::time_point<_Clock, _Duration>& __t);
+
+  template <class _Lock, class _Clock, class _Duration, class _Predicate>
+  bool _LIBCPP_HIDE_FROM_ABI
+  wait_until(_Lock& __lock, const chrono::time_point<_Clock, _Duration>& __t, _Predicate __pred);
+
+  template <class _Lock, class _Rep, class _Period>
+  cv_status _LIBCPP_HIDE_FROM_ABI wait_for(_Lock& __lock, const chrono::duration<_Rep, _Period>& __d);
+
+  template <class _Lock, class _Rep, class _Period, class _Predicate>
+  bool _LIBCPP_HIDE_FROM_ABI wait_for(_Lock& __lock, const chrono::duration<_Rep, _Period>& __d, _Predicate __pred);
+
+#  if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN)
+
+  template <class _Lock, class _Predicate>
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool wait(_Lock& __lock, stop_token __stoken, _Predicate __pred);
+
+  template <class _Lock, class _Clock, class _Duration, class _Predicate>
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool wait_until(
+      _Lock& __lock, stop_token __stoken, const chrono::time_point<_Clock, _Duration>& __abs_time, _Predicate __pred);
+
+  template <class _Lock, class _Rep, class _Period, class _Predicate>
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool
+  wait_for(_Lock& __lock, stop_token __stoken, const chrono::duration<_Rep, _Period>& __rel_time, _Predicate __pred);
+
+#  endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN)
+};
+
+inline condition_variable_any::condition_variable_any() : __mut_(make_shared<mutex>()) {}
+
+inline void condition_variable_any::notify_one() _NOEXCEPT {
+  { lock_guard<mutex> __lx(*__mut_); }
+  __cv_.notify_one();
+}
+
+inline void condition_variable_any::notify_all() _NOEXCEPT {
+  { lock_guard<mutex> __lx(*__mut_); }
+  __cv_.notify_all();
+}
+
+template <class _Lock>
+struct __unlock_guard {
+  _Lock& __lock_;
+
+  _LIBCPP_HIDE_FROM_ABI __unlock_guard(_Lock& __lock) : __lock_(__lock) { __lock_.unlock(); }
+
+  _LIBCPP_HIDE_FROM_ABI ~__unlock_guard() _NOEXCEPT // turns exception to std::terminate
+  {
+    __lock_.lock();
+  }
+
+  __unlock_guard(const __unlock_guard&)            = delete;
+  __unlock_guard& operator=(const __unlock_guard&) = delete;
+};
+
+template <class _Lock>
+void condition_variable_any::wait(_Lock& __lock) {
+  shared_ptr<mutex> __mut = __mut_;
+  unique_lock<mutex> __lk(*__mut);
+  __unlock_guard<_Lock> __unlock(__lock);
+  lock_guard<unique_lock<mutex> > __lx(__lk, adopt_lock_t());
+  __cv_.wait(__lk);
+} // __mut_.unlock(), __lock.lock()
+
+template <class _Lock, class _Predicate>
+inline void condition_variable_any::wait(_Lock& __lock, _Predicate __pred) {
+  while (!__pred())
+    wait(__lock);
+}
+
+template <class _Lock, class _Clock, class _Duration>
+cv_status condition_variable_any::wait_until(_Lock& __lock, const chrono::time_point<_Clock, _Duration>& __t) {
+  shared_ptr<mutex> __mut = __mut_;
+  unique_lock<mutex> __lk(*__mut);
+  __unlock_guard<_Lock> __unlock(__lock);
+  lock_guard<unique_lock<mutex> > __lx(__lk, adopt_lock_t());
+  return __cv_.wait_until(__lk, __t);
+} // __mut_.unlock(), __lock.lock()
+
+template <class _Lock, class _Clock, class _Duration, class _Predicate>
+inline bool
+condition_variable_any::wait_until(_Lock& __lock, const chrono::time_point<_Clock, _Duration>& __t, _Predicate __pred) {
+  while (!__pred())
+    if (wait_until(__lock, __t) == cv_status::timeout)
+      return __pred();
+  return true;
+}
+
+template <class _Lock, class _Rep, class _Period>
+inline cv_status condition_variable_any::wait_for(_Lock& __lock, const chrono::duration<_Rep, _Period>& __d) {
+  return wait_until(__lock, chrono::steady_clock::now() + __d);
+}
+
+template <class _Lock, class _Rep, class _Period, class _Predicate>
+inline bool
+condition_variable_any::wait_for(_Lock& __lock, const chrono::duration<_Rep, _Period>& __d, _Predicate __pred) {
+  return wait_until(__lock, chrono::steady_clock::now() + __d, std::move(__pred));
+}
+
+#  if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN)
+
+template <class _Lock, class _Predicate>
+bool condition_variable_any::wait(_Lock& __user_lock, stop_token __stoken, _Predicate __pred) {
+  if (__stoken.stop_requested())
+    return __pred();
+
+  // Per https://eel.is/c++draft/thread.condition.condvarany#general-note-2,
+  // we do need to take a copy of the shared pointer __mut_
+  // This ensures that a thread can call the destructor immediately after calling
+  // notify_all, without waiting all the wait calls.
+  // A thread can also safely call the destructor immediately after calling
+  // request_stop, as the call to request_stop would evaluate the callback,
+  // which accesses the internal condition variable, immediately on the same thread.
+  // In this situation, it is OK even without copying a shared ownership the internal
+  // condition variable. However, this needs the evaluation of stop_callback to
+  // happen-before the destruction.
+  // The spec only says "Only the notification to unblock the wait needs to happen
+  // before destruction". To make this work, we need to copy the shared ownership of
+  // the internal condition variable inside this function, which is not possible
+  // with the current ABI.
+  shared_ptr<mutex> __mut = __mut_;
+
+  stop_callback __cb(__stoken, [this] { notify_all(); });
+
+  while (true) {
+    if (__pred())
+      return true;
+
+    // We need to take the internal lock before checking stop_requested,
+    // so that the notification cannot come in between the stop_requested
+    // check and entering the wait.
+    // Note that the stop_callback takes the same internal lock before notifying
+    unique_lock<mutex> __internal_lock(*__mut);
+    if (__stoken.stop_requested())
+      break;
+
+    __unlock_guard<_Lock> __unlock(__user_lock);
+    unique_lock<mutex> __internal_lock2(
+        std::move(__internal_lock)); // switch unlock order between __internal_lock and __user_lock
+    __cv_.wait(__internal_lock2);
+  } // __internal_lock2.unlock(), __user_lock.lock()
+  return __pred();
+}
+
+template <class _Lock, class _Clock, class _Duration, class _Predicate>
+bool condition_variable_any::wait_until(
+    _Lock& __user_lock,
+    stop_token __stoken,
+    const chrono::time_point<_Clock, _Duration>& __abs_time,
+    _Predicate __pred) {
+  if (__stoken.stop_requested())
+    return __pred();
+
+  shared_ptr<mutex> __mut = __mut_;
+  stop_callback __cb(__stoken, [this] { notify_all(); });
+
+  while (true) {
+    if (__pred())
+      return true;
+
+    unique_lock<mutex> __internal_lock(*__mut);
+    if (__stoken.stop_requested())
+      break;
+
+    __unlock_guard<_Lock> __unlock(__user_lock);
+    unique_lock<mutex> __internal_lock2(
+        std::move(__internal_lock)); // switch unlock order between __internal_lock and __user_lock
+
+    if (__cv_.wait_until(__internal_lock2, __abs_time) == cv_status::timeout)
+      break;
+  } // __internal_lock2.unlock(), __user_lock.lock()
+  return __pred();
+}
+
+template <class _Lock, class _Rep, class _Period, class _Predicate>
+bool condition_variable_any::wait_for(
+    _Lock& __lock, stop_token __stoken, const chrono::duration<_Rep, _Period>& __rel_time, _Predicate __pred) {
+  return wait_until(__lock, std::move(__stoken), chrono::steady_clock::now() + __rel_time, std::move(__pred));
+}
+
+#  endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN)
+
+_LIBCPP_EXPORTED_FROM_ABI void notify_all_at_thread_exit(condition_variable&, unique_lock<mutex>);
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // !_LIBCPP_HAS_NO_THREADS
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <concepts>
+#  include <cstdint>
+#  include <cstdlib>
+#  include <cstring>
+#  include <initializer_list>
+#  include <iosfwd>
+#  include <new>
+#  include <stdexcept>
+#  include <system_error>
+#  include <type_traits>
+#  include <typeinfo>
+#endif
+
+#endif // _LIBCPP_CONDITION_VARIABLE
diff --git a/libcxx/include/__cxx03/coroutine b/libcxx/include/__cxx03/coroutine
new file mode 100644
index 000000000000..ee54388ad5aa
--- /dev/null
+++ b/libcxx/include/__cxx03/coroutine
@@ -0,0 +1,67 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_COROUTINE
+#define _LIBCPP_COROUTINE
+
+/**
+    coroutine synopsis
+
+namespace std {
+// [coroutine.traits]
+template <class R, class... ArgTypes>
+  struct coroutine_traits;
+// [coroutine.handle]
+template <class Promise = void>
+  struct coroutine_handle;
+// [coroutine.handle.compare]
+constexpr bool operator==(coroutine_handle<> x, coroutine_handle<> y) noexcept;
+constexpr strong_ordering operator<=>(coroutine_handle<> x, coroutine_handle<> y) noexcept;
+// [coroutine.handle.hash]
+template <class T> struct hash;
+template <class P> struct hash<coroutine_handle<P>>;
+// [coroutine.noop]
+struct noop_coroutine_promise;
+template<> struct coroutine_handle<noop_coroutine_promise>;
+using noop_coroutine_handle = coroutine_handle<noop_coroutine_promise>;
+noop_coroutine_handle noop_coroutine() noexcept;
+// [coroutine.trivial.awaitables]
+struct suspend_never;
+struct suspend_always;
+} // namespace std
+
+ */
+
+#include <__config>
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__coroutine/coroutine_handle.h>
+#  include <__coroutine/coroutine_traits.h>
+#  include <__coroutine/noop_coroutine_handle.h>
+#  include <__coroutine/trivial_awaitables.h>
+#endif // _LIBCPP_STD_VER >= 20
+
+#include <version>
+
+// standard-mandated includes
+
+// [coroutine.syn]
+#include <compare>
+
+#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <iosfwd>
+#  include <limits>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_COROUTINE
diff --git a/libcxx/include/__cxx03/csetjmp b/libcxx/include/__cxx03/csetjmp
new file mode 100644
index 000000000000..7ba90068710a
--- /dev/null
+++ b/libcxx/include/__cxx03/csetjmp
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CSETJMP
+#define _LIBCPP_CSETJMP
+
+/*
+    csetjmp synopsis
+
+Macros:
+
+    setjmp
+
+namespace std
+{
+
+Types:
+
+    jmp_buf
+
+void longjmp(jmp_buf env, int val);
+
+}  // std
+
+*/
+
+#include <__config>
+
+// <setjmp.h> is not provided by libc++
+#if __has_include(<setjmp.h>)
+#  include <setjmp.h>
+#  ifdef _LIBCPP_SETJMP_H
+#    error "If libc++ starts defining <setjmp.h>, the __has_include check should move to libc++'s <setjmp.h>"
+#  endif
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::jmp_buf _LIBCPP_USING_IF_EXISTS;
+using ::longjmp _LIBCPP_USING_IF_EXISTS;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CSETJMP
diff --git a/libcxx/include/__cxx03/csignal b/libcxx/include/__cxx03/csignal
new file mode 100644
index 000000000000..804a7f95ae96
--- /dev/null
+++ b/libcxx/include/__cxx03/csignal
@@ -0,0 +1,64 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CSIGNAL
+#define _LIBCPP_CSIGNAL
+
+/*
+    csignal synopsis
+
+Macros:
+
+    SIG_DFL
+    SIG_ERR
+    SIG_IGN
+    SIGABRT
+    SIGFPE
+    SIGILL
+    SIGINT
+    SIGSEGV
+    SIGTERM
+
+namespace std
+{
+
+Types:
+
+    sig_atomic_t
+
+void (*signal(int sig, void (*func)(int)))(int);
+int raise(int sig);
+
+}  // std
+
+*/
+
+#include <__config>
+
+// <signal.h> is not provided by libc++
+#if __has_include(<signal.h>)
+#  include <signal.h>
+#  ifdef _LIBCPP_SIGNAL_H
+#    error "If libc++ starts defining <signal.h>, the __has_include check should move to libc++'s <signal.h>"
+#  endif
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::sig_atomic_t _LIBCPP_USING_IF_EXISTS;
+using ::signal _LIBCPP_USING_IF_EXISTS;
+using ::raise _LIBCPP_USING_IF_EXISTS;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CSIGNAL
diff --git a/libcxx/include/__cxx03/cstdarg b/libcxx/include/__cxx03/cstdarg
new file mode 100644
index 000000000000..4642eb7b5258
--- /dev/null
+++ b/libcxx/include/__cxx03/cstdarg
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CSTDARG
+#define _LIBCPP_CSTDARG
+
+/*
+    cstdarg synopsis
+
+Macros:
+
+    type va_arg(va_list ap, type);
+    void va_copy(va_list dest, va_list src);  // C99
+    void va_end(va_list ap);
+    void va_start(va_list ap, parmN);
+
+namespace std
+{
+
+Types:
+
+    va_list
+
+}  // std
+
+*/
+
+#include <__config>
+
+// <stdarg.h> is not provided by libc++
+#if __has_include(<stdarg.h>)
+#  include <stdarg.h>
+#  ifdef _LIBCPP_STDARG_H
+#    error "If libc++ starts defining <stdarg.h>, the __has_include check should move to libc++'s <stdarg.h>"
+#  endif
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::va_list _LIBCPP_USING_IF_EXISTS;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CSTDARG
diff --git a/libcxx/include/__cxx03/cstdbool b/libcxx/include/__cxx03/cstdbool
new file mode 100644
index 000000000000..ef731c021a4a
--- /dev/null
+++ b/libcxx/include/__cxx03/cstdbool
@@ -0,0 +1,31 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CSTDBOOL
+#define _LIBCPP_CSTDBOOL
+
+/*
+    cstdbool synopsis
+
+Macros:
+
+    __bool_true_false_are_defined
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#undef __bool_true_false_are_defined
+#define __bool_true_false_are_defined 1
+
+#endif // _LIBCPP_CSTDBOOL
diff --git a/libcxx/include/__cxx03/cstddef b/libcxx/include/__cxx03/cstddef
new file mode 100644
index 000000000000..1a4049e4d34f
--- /dev/null
+++ b/libcxx/include/__cxx03/cstddef
@@ -0,0 +1,133 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CSTDDEF
+#define _LIBCPP_CSTDDEF
+
+/*
+    cstddef synopsis
+
+Macros:
+
+    offsetof(type,member-designator)
+    NULL
+
+namespace std
+{
+
+Types:
+
+    ptrdiff_t
+    size_t
+    max_align_t // C++11
+    nullptr_t
+    byte // C++17
+
+}  // std
+
+*/
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_integral.h>
+#include <version>
+
+#include <stddef.h>
+
+#ifndef _LIBCPP_STDDEF_H
+#   error <cstddef> tried including <stddef.h> but didn't find libc++'s <stddef.h> header. \
+          This usually means that your header search paths are not configured properly. \
+          The header search paths should contain the C++ Standard Library headers before \
+          any C Standard Library, and you are probably using compiler flags that make that \
+          not be the case.
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::nullptr_t;
+using ::ptrdiff_t _LIBCPP_USING_IF_EXISTS;
+using ::size_t _LIBCPP_USING_IF_EXISTS;
+
+#if !defined(_LIBCPP_CXX03_LANG)
+using ::max_align_t _LIBCPP_USING_IF_EXISTS;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+namespace std // purposefully not versioned
+{
+enum class byte : unsigned char {};
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr byte operator|(byte __lhs, byte __rhs) noexcept {
+  return static_cast<byte>(
+      static_cast<unsigned char>(static_cast<unsigned int>(__lhs) | static_cast<unsigned int>(__rhs)));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr byte& operator|=(byte& __lhs, byte __rhs) noexcept {
+  return __lhs = __lhs | __rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr byte operator&(byte __lhs, byte __rhs) noexcept {
+  return static_cast<byte>(
+      static_cast<unsigned char>(static_cast<unsigned int>(__lhs) & static_cast<unsigned int>(__rhs)));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr byte& operator&=(byte& __lhs, byte __rhs) noexcept {
+  return __lhs = __lhs & __rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr byte operator^(byte __lhs, byte __rhs) noexcept {
+  return static_cast<byte>(
+      static_cast<unsigned char>(static_cast<unsigned int>(__lhs) ^ static_cast<unsigned int>(__rhs)));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr byte& operator^=(byte& __lhs, byte __rhs) noexcept {
+  return __lhs = __lhs ^ __rhs;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr byte operator~(byte __b) noexcept {
+  return static_cast<byte>(static_cast<unsigned char>(~static_cast<unsigned int>(__b)));
+}
+
+template <class _Integer, __enable_if_t<is_integral<_Integer>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr byte& operator<<=(byte& __lhs, _Integer __shift) noexcept {
+  return __lhs = __lhs << __shift;
+}
+
+template <class _Integer, __enable_if_t<is_integral<_Integer>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr byte operator<<(byte __lhs, _Integer __shift) noexcept {
+  return static_cast<byte>(static_cast<unsigned char>(static_cast<unsigned int>(__lhs) << __shift));
+}
+
+template <class _Integer, __enable_if_t<is_integral<_Integer>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr byte& operator>>=(byte& __lhs, _Integer __shift) noexcept {
+  return __lhs = __lhs >> __shift;
+}
+
+template <class _Integer, __enable_if_t<is_integral<_Integer>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr byte operator>>(byte __lhs, _Integer __shift) noexcept {
+  return static_cast<byte>(static_cast<unsigned char>(static_cast<unsigned int>(__lhs) >> __shift));
+}
+
+template <class _Integer, __enable_if_t<is_integral<_Integer>::value, int> = 0>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Integer to_integer(byte __b) noexcept {
+  return static_cast<_Integer>(__b);
+}
+
+} // namespace std
+
+#endif
+
+#endif // _LIBCPP_CSTDDEF
diff --git a/libcxx/include/__cxx03/cstdint b/libcxx/include/__cxx03/cstdint
new file mode 100644
index 000000000000..8c4782859426
--- /dev/null
+++ b/libcxx/include/__cxx03/cstdint
@@ -0,0 +1,199 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CSTDINT
+#define _LIBCPP_CSTDINT
+
+/*
+    cstdint synopsis
+
+Macros:
+
+    INT8_MIN
+    INT16_MIN
+    INT32_MIN
+    INT64_MIN
+
+    INT8_MAX
+    INT16_MAX
+    INT32_MAX
+    INT64_MAX
+
+    UINT8_MAX
+    UINT16_MAX
+    UINT32_MAX
+    UINT64_MAX
+
+    INT_LEAST8_MIN
+    INT_LEAST16_MIN
+    INT_LEAST32_MIN
+    INT_LEAST64_MIN
+
+    INT_LEAST8_MAX
+    INT_LEAST16_MAX
+    INT_LEAST32_MAX
+    INT_LEAST64_MAX
+
+    UINT_LEAST8_MAX
+    UINT_LEAST16_MAX
+    UINT_LEAST32_MAX
+    UINT_LEAST64_MAX
+
+    INT_FAST8_MIN
+    INT_FAST16_MIN
+    INT_FAST32_MIN
+    INT_FAST64_MIN
+
+    INT_FAST8_MAX
+    INT_FAST16_MAX
+    INT_FAST32_MAX
+    INT_FAST64_MAX
+
+    UINT_FAST8_MAX
+    UINT_FAST16_MAX
+    UINT_FAST32_MAX
+    UINT_FAST64_MAX
+
+    INTPTR_MIN
+    INTPTR_MAX
+    UINTPTR_MAX
+
+    INTMAX_MIN
+    INTMAX_MAX
+
+    UINTMAX_MAX
+
+    PTRDIFF_MIN
+    PTRDIFF_MAX
+
+    SIG_ATOMIC_MIN
+    SIG_ATOMIC_MAX
+
+    SIZE_MAX
+
+    WCHAR_MIN
+    WCHAR_MAX
+
+    WINT_MIN
+    WINT_MAX
+
+    INT8_C(value)
+    INT16_C(value)
+    INT32_C(value)
+    INT64_C(value)
+
+    UINT8_C(value)
+    UINT16_C(value)
+    UINT32_C(value)
+    UINT64_C(value)
+
+    INTMAX_C(value)
+    UINTMAX_C(value)
+
+namespace std
+{
+
+Types:
+
+    int8_t
+    int16_t
+    int32_t
+    int64_t
+
+    uint8_t
+    uint16_t
+    uint32_t
+    uint64_t
+
+    int_least8_t
+    int_least16_t
+    int_least32_t
+    int_least64_t
+
+    uint_least8_t
+    uint_least16_t
+    uint_least32_t
+    uint_least64_t
+
+    int_fast8_t
+    int_fast16_t
+    int_fast32_t
+    int_fast64_t
+
+    uint_fast8_t
+    uint_fast16_t
+    uint_fast32_t
+    uint_fast64_t
+
+    intptr_t
+    uintptr_t
+
+    intmax_t
+    uintmax_t
+
+}  // std
+*/
+
+#include <__config>
+
+#include <stdint.h>
+
+#ifndef _LIBCPP_STDINT_H
+#   error <cstdint> tried including <stdint.h> but didn't find libc++'s <stdint.h> header. \
+          This usually means that your header search paths are not configured properly. \
+          The header search paths should contain the C++ Standard Library headers before \
+          any C Standard Library, and you are probably using compiler flags that make that \
+          not be the case.
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::int8_t _LIBCPP_USING_IF_EXISTS;
+using ::int16_t _LIBCPP_USING_IF_EXISTS;
+using ::int32_t _LIBCPP_USING_IF_EXISTS;
+using ::int64_t _LIBCPP_USING_IF_EXISTS;
+
+using ::uint8_t _LIBCPP_USING_IF_EXISTS;
+using ::uint16_t _LIBCPP_USING_IF_EXISTS;
+using ::uint32_t _LIBCPP_USING_IF_EXISTS;
+using ::uint64_t _LIBCPP_USING_IF_EXISTS;
+
+using ::int_least8_t _LIBCPP_USING_IF_EXISTS;
+using ::int_least16_t _LIBCPP_USING_IF_EXISTS;
+using ::int_least32_t _LIBCPP_USING_IF_EXISTS;
+using ::int_least64_t _LIBCPP_USING_IF_EXISTS;
+
+using ::uint_least8_t _LIBCPP_USING_IF_EXISTS;
+using ::uint_least16_t _LIBCPP_USING_IF_EXISTS;
+using ::uint_least32_t _LIBCPP_USING_IF_EXISTS;
+using ::uint_least64_t _LIBCPP_USING_IF_EXISTS;
+
+using ::int_fast8_t _LIBCPP_USING_IF_EXISTS;
+using ::int_fast16_t _LIBCPP_USING_IF_EXISTS;
+using ::int_fast32_t _LIBCPP_USING_IF_EXISTS;
+using ::int_fast64_t _LIBCPP_USING_IF_EXISTS;
+
+using ::uint_fast8_t _LIBCPP_USING_IF_EXISTS;
+using ::uint_fast16_t _LIBCPP_USING_IF_EXISTS;
+using ::uint_fast32_t _LIBCPP_USING_IF_EXISTS;
+using ::uint_fast64_t _LIBCPP_USING_IF_EXISTS;
+
+using ::intptr_t _LIBCPP_USING_IF_EXISTS;
+using ::uintptr_t _LIBCPP_USING_IF_EXISTS;
+
+using ::intmax_t _LIBCPP_USING_IF_EXISTS;
+using ::uintmax_t _LIBCPP_USING_IF_EXISTS;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CSTDINT
diff --git a/libcxx/include/__cxx03/cstdio b/libcxx/include/__cxx03/cstdio
new file mode 100644
index 000000000000..7f94371081f8
--- /dev/null
+++ b/libcxx/include/__cxx03/cstdio
@@ -0,0 +1,174 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CSTDIO
+#define _LIBCPP_CSTDIO
+
+/*
+    cstdio synopsis
+
+Macros:
+
+    BUFSIZ
+    EOF
+    FILENAME_MAX
+    FOPEN_MAX
+    L_tmpnam
+    NULL
+    SEEK_CUR
+    SEEK_END
+    SEEK_SET
+    TMP_MAX
+    _IOFBF
+    _IOLBF
+    _IONBF
+    stderr
+    stdin
+    stdout
+
+namespace std
+{
+
+Types:
+
+FILE
+fpos_t
+size_t
+
+int remove(const char* filename);
+int rename(const char* old, const char* new);
+FILE* tmpfile(void);
+char* tmpnam(char* s);
+int fclose(FILE* stream);
+int fflush(FILE* stream);
+FILE* fopen(const char* restrict filename, const char* restrict mode);
+FILE* freopen(const char* restrict filename, const char * restrict mode,
+              FILE * restrict stream);
+void setbuf(FILE* restrict stream, char* restrict buf);
+int setvbuf(FILE* restrict stream, char* restrict buf, int mode, size_t size);
+int fprintf(FILE* restrict stream, const char* restrict format, ...);
+int fscanf(FILE* restrict stream, const char * restrict format, ...);
+int printf(const char* restrict format, ...);
+int scanf(const char* restrict format, ...);
+int snprintf(char* restrict s, size_t n, const char* restrict format, ...);    // C99
+int sprintf(char* restrict s, const char* restrict format, ...);
+int sscanf(const char* restrict s, const char* restrict format, ...);
+int vfprintf(FILE* restrict stream, const char* restrict format, va_list arg);
+int vfscanf(FILE* restrict stream, const char* restrict format, va_list arg);  // C99
+int vprintf(const char* restrict format, va_list arg);
+int vscanf(const char* restrict format, va_list arg);                          // C99
+int vsnprintf(char* restrict s, size_t n, const char* restrict format,         // C99
+              va_list arg);
+int vsprintf(char* restrict s, const char* restrict format, va_list arg);
+int vsscanf(const char* restrict s, const char* restrict format, va_list arg); // C99
+int fgetc(FILE* stream);
+char* fgets(char* restrict s, int n, FILE* restrict stream);
+int fputc(int c, FILE* stream);
+int fputs(const char* restrict s, FILE* restrict stream);
+int getc(FILE* stream);
+int getchar(void);
+char* gets(char* s);  // removed in C++14
+int putc(int c, FILE* stream);
+int putchar(int c);
+int puts(const char* s);
+int ungetc(int c, FILE* stream);
+size_t fread(void* restrict ptr, size_t size, size_t nmemb,
+             FILE* restrict stream);
+size_t fwrite(const void* restrict ptr, size_t size, size_t nmemb,
+              FILE* restrict stream);
+int fgetpos(FILE* restrict stream, fpos_t* restrict pos);
+int fseek(FILE* stream, long offset, int whence);
+int fsetpos(FILE*stream, const fpos_t* pos);
+long ftell(FILE* stream);
+void rewind(FILE* stream);
+void clearerr(FILE* stream);
+int feof(FILE* stream);
+int ferror(FILE* stream);
+void perror(const char* s);
+
+}  // std
+*/
+
+#include <__config>
+
+#include <stdio.h>
+
+#ifndef _LIBCPP_STDIO_H
+#   error <cstdio> tried including <stdio.h> but didn't find libc++'s <stdio.h> header. \
+          This usually means that your header search paths are not configured properly. \
+          The header search paths should contain the C++ Standard Library headers before \
+          any C Standard Library, and you are probably using compiler flags that make that \
+          not be the case.
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::FILE _LIBCPP_USING_IF_EXISTS;
+using ::fpos_t _LIBCPP_USING_IF_EXISTS;
+using ::size_t _LIBCPP_USING_IF_EXISTS;
+
+using ::fclose _LIBCPP_USING_IF_EXISTS;
+using ::fflush _LIBCPP_USING_IF_EXISTS;
+using ::setbuf _LIBCPP_USING_IF_EXISTS;
+using ::setvbuf _LIBCPP_USING_IF_EXISTS;
+using ::fprintf _LIBCPP_USING_IF_EXISTS;
+using ::fscanf _LIBCPP_USING_IF_EXISTS;
+using ::snprintf _LIBCPP_USING_IF_EXISTS;
+using ::sprintf _LIBCPP_USING_IF_EXISTS;
+using ::sscanf _LIBCPP_USING_IF_EXISTS;
+using ::vfprintf _LIBCPP_USING_IF_EXISTS;
+using ::vfscanf _LIBCPP_USING_IF_EXISTS;
+using ::vsscanf _LIBCPP_USING_IF_EXISTS;
+using ::vsnprintf _LIBCPP_USING_IF_EXISTS;
+using ::vsprintf _LIBCPP_USING_IF_EXISTS;
+using ::fgetc _LIBCPP_USING_IF_EXISTS;
+using ::fgets _LIBCPP_USING_IF_EXISTS;
+using ::fputc _LIBCPP_USING_IF_EXISTS;
+using ::fputs _LIBCPP_USING_IF_EXISTS;
+using ::getc _LIBCPP_USING_IF_EXISTS;
+using ::putc _LIBCPP_USING_IF_EXISTS;
+using ::ungetc _LIBCPP_USING_IF_EXISTS;
+using ::fread _LIBCPP_USING_IF_EXISTS;
+using ::fwrite _LIBCPP_USING_IF_EXISTS;
+using ::fgetpos _LIBCPP_USING_IF_EXISTS;
+using ::fseek _LIBCPP_USING_IF_EXISTS;
+using ::fsetpos _LIBCPP_USING_IF_EXISTS;
+using ::ftell _LIBCPP_USING_IF_EXISTS;
+using ::rewind _LIBCPP_USING_IF_EXISTS;
+using ::clearerr _LIBCPP_USING_IF_EXISTS;
+using ::feof _LIBCPP_USING_IF_EXISTS;
+using ::ferror _LIBCPP_USING_IF_EXISTS;
+using ::perror _LIBCPP_USING_IF_EXISTS;
+
+using ::fopen _LIBCPP_USING_IF_EXISTS;
+using ::freopen _LIBCPP_USING_IF_EXISTS;
+using ::remove _LIBCPP_USING_IF_EXISTS;
+using ::rename _LIBCPP_USING_IF_EXISTS;
+using ::tmpfile _LIBCPP_USING_IF_EXISTS;
+using ::tmpnam _LIBCPP_USING_IF_EXISTS;
+
+using ::getchar _LIBCPP_USING_IF_EXISTS;
+#if _LIBCPP_STD_VER <= 11
+using ::gets _LIBCPP_USING_IF_EXISTS;
+#endif
+using ::scanf _LIBCPP_USING_IF_EXISTS;
+using ::vscanf _LIBCPP_USING_IF_EXISTS;
+
+using ::printf _LIBCPP_USING_IF_EXISTS;
+using ::putchar _LIBCPP_USING_IF_EXISTS;
+using ::puts _LIBCPP_USING_IF_EXISTS;
+using ::vprintf _LIBCPP_USING_IF_EXISTS;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CSTDIO
diff --git a/libcxx/include/__cxx03/cstdlib b/libcxx/include/__cxx03/cstdlib
new file mode 100644
index 000000000000..c817fd8f4acc
--- /dev/null
+++ b/libcxx/include/__cxx03/cstdlib
@@ -0,0 +1,154 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CSTDLIB
+#define _LIBCPP_CSTDLIB
+
+/*
+    cstdlib synopsis
+
+Macros:
+
+    EXIT_FAILURE
+    EXIT_SUCCESS
+    MB_CUR_MAX
+    NULL
+    RAND_MAX
+
+namespace std
+{
+
+Types:
+
+    size_t
+    div_t
+    ldiv_t
+    lldiv_t                                                               // C99
+
+double    atof (const char* nptr);
+int       atoi (const char* nptr);
+long      atol (const char* nptr);
+long long atoll(const char* nptr);                                        // C99
+double             strtod  (const char* restrict nptr, char** restrict endptr);
+float              strtof  (const char* restrict nptr, char** restrict endptr); // C99
+long double        strtold (const char* restrict nptr, char** restrict endptr); // C99
+long               strtol  (const char* restrict nptr, char** restrict endptr, int base);
+long long          strtoll (const char* restrict nptr, char** restrict endptr, int base); // C99
+unsigned long      strtoul (const char* restrict nptr, char** restrict endptr, int base);
+unsigned long long strtoull(const char* restrict nptr, char** restrict endptr, int base); // C99
+int rand(void);
+void srand(unsigned int seed);
+void* calloc(size_t nmemb, size_t size);
+void free(void* ptr);
+void* malloc(size_t size);
+void* realloc(void* ptr, size_t size);
+void abort(void);
+int atexit(void (*func)(void));
+void exit(int status);
+void _Exit(int status);
+char* getenv(const char* name);
+int system(const char* string);
+void* bsearch(const void* key, const void* base, size_t nmemb, size_t size,
+              int (*compar)(const void *, const void *));
+void qsort(void* base, size_t nmemb, size_t size,
+           int (*compar)(const void *, const void *));
+int         abs(      int j);
+long        abs(     long j);
+long long   abs(long long j);                                             // C++0X
+long       labs(     long j);
+long long llabs(long long j);                                             // C99
+div_t     div(      int numer,       int denom);
+ldiv_t    div(     long numer,      long denom);
+lldiv_t   div(long long numer, long long denom);                          // C++0X
+ldiv_t   ldiv(     long numer,      long denom);
+lldiv_t lldiv(long long numer, long long denom);                          // C99
+int mblen(const char* s, size_t n);
+int mbtowc(wchar_t* restrict pwc, const char* restrict s, size_t n);
+int wctomb(char* s, wchar_t wchar);
+size_t mbstowcs(wchar_t* restrict pwcs, const char* restrict s, size_t n);
+size_t wcstombs(char* restrict s, const wchar_t* restrict pwcs, size_t n);
+int at_quick_exit(void (*func)(void))                                     // C++11
+void quick_exit(int status);                                              // C++11
+void *aligned_alloc(size_t alignment, size_t size);                       // C11
+
+}  // std
+
+*/
+
+#include <__config>
+
+#include <stdlib.h>
+
+#ifndef _LIBCPP_STDLIB_H
+#   error <cstdlib> tried including <stdlib.h> but didn't find libc++'s <stdlib.h> header. \
+          This usually means that your header search paths are not configured properly. \
+          The header search paths should contain the C++ Standard Library headers before \
+          any C Standard Library, and you are probably using compiler flags that make that \
+          not be the case.
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::size_t _LIBCPP_USING_IF_EXISTS;
+using ::div_t _LIBCPP_USING_IF_EXISTS;
+using ::ldiv_t _LIBCPP_USING_IF_EXISTS;
+using ::lldiv_t _LIBCPP_USING_IF_EXISTS;
+using ::atof _LIBCPP_USING_IF_EXISTS;
+using ::atoi _LIBCPP_USING_IF_EXISTS;
+using ::atol _LIBCPP_USING_IF_EXISTS;
+using ::atoll _LIBCPP_USING_IF_EXISTS;
+using ::strtod _LIBCPP_USING_IF_EXISTS;
+using ::strtof _LIBCPP_USING_IF_EXISTS;
+using ::strtold _LIBCPP_USING_IF_EXISTS;
+using ::strtol _LIBCPP_USING_IF_EXISTS;
+using ::strtoll _LIBCPP_USING_IF_EXISTS;
+using ::strtoul _LIBCPP_USING_IF_EXISTS;
+using ::strtoull _LIBCPP_USING_IF_EXISTS;
+using ::rand _LIBCPP_USING_IF_EXISTS;
+using ::srand _LIBCPP_USING_IF_EXISTS;
+using ::calloc _LIBCPP_USING_IF_EXISTS;
+using ::free _LIBCPP_USING_IF_EXISTS;
+using ::malloc _LIBCPP_USING_IF_EXISTS;
+using ::realloc _LIBCPP_USING_IF_EXISTS;
+using ::abort _LIBCPP_USING_IF_EXISTS;
+using ::atexit _LIBCPP_USING_IF_EXISTS;
+using ::exit _LIBCPP_USING_IF_EXISTS;
+using ::_Exit _LIBCPP_USING_IF_EXISTS;
+using ::getenv _LIBCPP_USING_IF_EXISTS;
+using ::system _LIBCPP_USING_IF_EXISTS;
+using ::bsearch _LIBCPP_USING_IF_EXISTS;
+using ::qsort _LIBCPP_USING_IF_EXISTS;
+using ::abs _LIBCPP_USING_IF_EXISTS;
+using ::labs _LIBCPP_USING_IF_EXISTS;
+using ::llabs _LIBCPP_USING_IF_EXISTS;
+using ::div _LIBCPP_USING_IF_EXISTS;
+using ::ldiv _LIBCPP_USING_IF_EXISTS;
+using ::lldiv _LIBCPP_USING_IF_EXISTS;
+using ::mblen _LIBCPP_USING_IF_EXISTS;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using ::mbtowc _LIBCPP_USING_IF_EXISTS;
+using ::wctomb _LIBCPP_USING_IF_EXISTS;
+using ::mbstowcs _LIBCPP_USING_IF_EXISTS;
+using ::wcstombs _LIBCPP_USING_IF_EXISTS;
+#endif
+#if !defined(_LIBCPP_CXX03_LANG)
+using ::at_quick_exit _LIBCPP_USING_IF_EXISTS;
+using ::quick_exit _LIBCPP_USING_IF_EXISTS;
+#endif
+#if _LIBCPP_STD_VER >= 17
+using ::aligned_alloc _LIBCPP_USING_IF_EXISTS;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CSTDLIB
diff --git a/libcxx/include/__cxx03/cstring b/libcxx/include/__cxx03/cstring
new file mode 100644
index 000000000000..c2c92b02e73c
--- /dev/null
+++ b/libcxx/include/__cxx03/cstring
@@ -0,0 +1,104 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CSTRING
+#define _LIBCPP_CSTRING
+
+/*
+    cstring synopsis
+
+Macros:
+
+    NULL
+
+namespace std
+{
+
+Types:
+
+    size_t
+
+void* memcpy(void* restrict s1, const void* restrict s2, size_t n);
+void* memmove(void* s1, const void* s2, size_t n);
+char* strcpy (char* restrict s1, const char* restrict s2);
+char* strncpy(char* restrict s1, const char* restrict s2, size_t n);
+char* strcat (char* restrict s1, const char* restrict s2);
+char* strncat(char* restrict s1, const char* restrict s2, size_t n);
+int memcmp(const void* s1, const void* s2, size_t n);
+int strcmp (const char* s1, const char* s2);
+int strncmp(const char* s1, const char* s2, size_t n);
+int strcoll(const char* s1, const char* s2);
+size_t strxfrm(char* restrict s1, const char* restrict s2, size_t n);
+const void* memchr(const void* s, int c, size_t n);
+      void* memchr(      void* s, int c, size_t n);
+const char* strchr(const char* s, int c);
+      char* strchr(      char* s, int c);
+size_t strcspn(const char* s1, const char* s2);
+const char* strpbrk(const char* s1, const char* s2);
+      char* strpbrk(      char* s1, const char* s2);
+const char* strrchr(const char* s, int c);
+      char* strrchr(      char* s, int c);
+size_t strspn(const char* s1, const char* s2);
+const char* strstr(const char* s1, const char* s2);
+      char* strstr(      char* s1, const char* s2);
+char* strtok(char* restrict s1, const char* restrict s2);
+void* memset(void* s, int c, size_t n);
+char* strerror(int errnum);
+size_t strlen(const char* s);
+
+}  // std
+
+*/
+
+#include <__config>
+#include <__type_traits/is_constant_evaluated.h>
+
+#include <string.h>
+
+#ifndef _LIBCPP_STRING_H
+#   error <cstring> tried including <string.h> but didn't find libc++'s <string.h> header. \
+          This usually means that your header search paths are not configured properly. \
+          The header search paths should contain the C++ Standard Library headers before \
+          any C Standard Library, and you are probably using compiler flags that make that \
+          not be the case.
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::size_t _LIBCPP_USING_IF_EXISTS;
+using ::memcpy _LIBCPP_USING_IF_EXISTS;
+using ::memmove _LIBCPP_USING_IF_EXISTS;
+using ::strcpy _LIBCPP_USING_IF_EXISTS;
+using ::strncpy _LIBCPP_USING_IF_EXISTS;
+using ::strcat _LIBCPP_USING_IF_EXISTS;
+using ::strncat _LIBCPP_USING_IF_EXISTS;
+using ::memcmp _LIBCPP_USING_IF_EXISTS;
+using ::strcmp _LIBCPP_USING_IF_EXISTS;
+using ::strncmp _LIBCPP_USING_IF_EXISTS;
+using ::strcoll _LIBCPP_USING_IF_EXISTS;
+using ::strxfrm _LIBCPP_USING_IF_EXISTS;
+using ::memchr _LIBCPP_USING_IF_EXISTS;
+using ::strchr _LIBCPP_USING_IF_EXISTS;
+using ::strcspn _LIBCPP_USING_IF_EXISTS;
+using ::strpbrk _LIBCPP_USING_IF_EXISTS;
+using ::strrchr _LIBCPP_USING_IF_EXISTS;
+using ::strspn _LIBCPP_USING_IF_EXISTS;
+using ::strstr _LIBCPP_USING_IF_EXISTS;
+using ::strtok _LIBCPP_USING_IF_EXISTS;
+using ::memset _LIBCPP_USING_IF_EXISTS;
+using ::strerror _LIBCPP_USING_IF_EXISTS;
+using ::strlen _LIBCPP_USING_IF_EXISTS;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CSTRING
diff --git a/libcxx/include/__cxx03/ctgmath b/libcxx/include/__cxx03/ctgmath
new file mode 100644
index 000000000000..6237979be490
--- /dev/null
+++ b/libcxx/include/__cxx03/ctgmath
@@ -0,0 +1,28 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CTGMATH
+#define _LIBCPP_CTGMATH
+
+/*
+    ctgmath synopsis
+
+#include <ccomplex>
+#include <cmath>
+
+*/
+
+#include <ccomplex>
+#include <cmath>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#endif // _LIBCPP_CTGMATH
diff --git a/libcxx/include/__cxx03/ctime b/libcxx/include/__cxx03/ctime
new file mode 100644
index 000000000000..f47b49a43e23
--- /dev/null
+++ b/libcxx/include/__cxx03/ctime
@@ -0,0 +1,86 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CTIME
+#define _LIBCPP_CTIME
+
+/*
+    ctime synopsis
+
+Macros:
+
+    NULL
+    CLOCKS_PER_SEC
+    TIME_UTC // C++17
+
+namespace std
+{
+
+Types:
+
+    clock_t
+    size_t
+    time_t
+    tm
+    timespec // C++17
+
+clock_t clock();
+double difftime(time_t time1, time_t time0);
+time_t mktime(tm* timeptr);
+time_t time(time_t* timer);
+char* asctime(const tm* timeptr);
+char* ctime(const time_t* timer);
+tm*    gmtime(const time_t* timer);
+tm* localtime(const time_t* timer);
+size_t strftime(char* restrict s, size_t maxsize, const char* restrict format,
+                const tm* restrict timeptr);
+int timespec_get( struct timespec *ts, int base); // C++17
+}  // std
+
+*/
+
+#include <__config>
+
+// <time.h> is not provided by libc++
+#if __has_include(<time.h>)
+#  include <time.h>
+#  ifdef _LIBCPP_TIME_H
+#    error "If libc++ starts defining <time.h>, the __has_include check should move to libc++'s <time.h>"
+#  endif
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::clock_t _LIBCPP_USING_IF_EXISTS;
+using ::size_t _LIBCPP_USING_IF_EXISTS;
+using ::time_t _LIBCPP_USING_IF_EXISTS;
+using ::tm _LIBCPP_USING_IF_EXISTS;
+#if _LIBCPP_STD_VER >= 17
+using ::timespec _LIBCPP_USING_IF_EXISTS;
+#endif
+using ::clock _LIBCPP_USING_IF_EXISTS;
+using ::difftime _LIBCPP_USING_IF_EXISTS;
+using ::mktime _LIBCPP_USING_IF_EXISTS;
+using ::time _LIBCPP_USING_IF_EXISTS;
+using ::asctime _LIBCPP_USING_IF_EXISTS;
+using ::ctime _LIBCPP_USING_IF_EXISTS;
+using ::gmtime _LIBCPP_USING_IF_EXISTS;
+using ::localtime _LIBCPP_USING_IF_EXISTS;
+using ::strftime _LIBCPP_USING_IF_EXISTS;
+#if _LIBCPP_STD_VER >= 17
+using ::timespec_get _LIBCPP_USING_IF_EXISTS;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CTIME
diff --git a/libcxx/include/__cxx03/ctype.h b/libcxx/include/__cxx03/ctype.h
new file mode 100644
index 000000000000..448e4410c554
--- /dev/null
+++ b/libcxx/include/__cxx03/ctype.h
@@ -0,0 +1,61 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CTYPE_H
+#define _LIBCPP_CTYPE_H
+
+/*
+    ctype.h synopsis
+
+int isalnum(int c);
+int isalpha(int c);
+int isblank(int c);  // C99
+int iscntrl(int c);
+int isdigit(int c);
+int isgraph(int c);
+int islower(int c);
+int isprint(int c);
+int ispunct(int c);
+int isspace(int c);
+int isupper(int c);
+int isxdigit(int c);
+int tolower(int c);
+int toupper(int c);
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if __has_include_next(<ctype.h>)
+#  include_next <ctype.h>
+#endif
+
+#ifdef __cplusplus
+
+#  undef isalnum
+#  undef isalpha
+#  undef isblank
+#  undef iscntrl
+#  undef isdigit
+#  undef isgraph
+#  undef islower
+#  undef isprint
+#  undef ispunct
+#  undef isspace
+#  undef isupper
+#  undef isxdigit
+#  undef tolower
+#  undef toupper
+
+#endif
+
+#endif // _LIBCPP_CTYPE_H
diff --git a/libcxx/include/__cxx03/cuchar b/libcxx/include/__cxx03/cuchar
new file mode 100644
index 000000000000..f0015be27536
--- /dev/null
+++ b/libcxx/include/__cxx03/cuchar
@@ -0,0 +1,75 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CUCHAR
+#define _LIBCPP_CUCHAR
+
+/*
+    cuchar synopsis // since C++11
+
+Macros:
+
+    __STDC_UTF_16__
+    __STDC_UTF_32__
+
+namespace std {
+
+Types:
+
+  mbstate_t
+  size_t
+
+size_t mbrtoc8(char8_t* pc8, const char* s, size_t n, mbstate_t* ps);   // since C++20
+size_t c8rtomb(char* s, char8_t c8, mbstate_t* ps);                     // since C++20
+size_t mbrtoc16(char16_t* pc16, const char* s, size_t n, mbstate_t* ps);
+size_t c16rtomb(char* s, char16_t c16, mbstate_t* ps);
+size_t mbrtoc32(char32_t* pc32, const char* s, size_t n, mbstate_t* ps);
+size_t c32rtomb(char* s, char32_t c32, mbstate_t* ps);
+
+} // std
+
+*/
+
+#include <__config>
+
+#include <uchar.h>
+
+#ifndef _LIBCPP_UCHAR_H
+#   error <cuchar> tried including <uchar.h> but didn't find libc++'s <uchar.h> header. \
+          This usually means that your header search paths are not configured properly. \
+          The header search paths should contain the C++ Standard Library headers before \
+          any C Standard Library, and you are probably using compiler flags that make that \
+          not be the case.
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if !defined(_LIBCPP_CXX03_LANG)
+
+using ::mbstate_t _LIBCPP_USING_IF_EXISTS;
+using ::size_t _LIBCPP_USING_IF_EXISTS;
+
+#  if !defined(_LIBCPP_HAS_NO_C8RTOMB_MBRTOC8)
+using ::mbrtoc8 _LIBCPP_USING_IF_EXISTS;
+using ::c8rtomb _LIBCPP_USING_IF_EXISTS;
+#  endif
+using ::mbrtoc16 _LIBCPP_USING_IF_EXISTS;
+using ::c16rtomb _LIBCPP_USING_IF_EXISTS;
+using ::mbrtoc32 _LIBCPP_USING_IF_EXISTS;
+using ::c32rtomb _LIBCPP_USING_IF_EXISTS;
+
+#endif // _LIBCPP_CXX03_LANG
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CUCHAR
diff --git a/libcxx/include/__cxx03/cwchar b/libcxx/include/__cxx03/cwchar
new file mode 100644
index 000000000000..08cfac58c846
--- /dev/null
+++ b/libcxx/include/__cxx03/cwchar
@@ -0,0 +1,261 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CWCHAR
+#define _LIBCPP_CWCHAR
+
+/*
+    cwchar synopsis
+
+Macros:
+
+    NULL
+    WCHAR_MAX
+    WCHAR_MIN
+    WEOF
+
+namespace std
+{
+
+Types:
+
+    mbstate_t
+    size_t
+    tm
+    wint_t
+
+int fwprintf(FILE* restrict stream, const wchar_t* restrict format, ...);
+int fwscanf(FILE* restrict stream, const wchar_t* restrict format, ...);
+int swprintf(wchar_t* restrict s, size_t n, const wchar_t* restrict format, ...);
+int swscanf(const wchar_t* restrict s, const wchar_t* restrict format, ...);
+int vfwprintf(FILE* restrict stream, const wchar_t* restrict format, va_list arg);
+int vfwscanf(FILE* restrict stream, const wchar_t* restrict format, va_list arg);  // C99
+int vswprintf(wchar_t* restrict s, size_t n, const wchar_t* restrict format, va_list arg);
+int vswscanf(const wchar_t* restrict s, const wchar_t* restrict format, va_list arg);  // C99
+int vwprintf(const wchar_t* restrict format, va_list arg);
+int vwscanf(const wchar_t* restrict format, va_list arg);  // C99
+int wprintf(const wchar_t* restrict format, ...);
+int wscanf(const wchar_t* restrict format, ...);
+wint_t fgetwc(FILE* stream);
+wchar_t* fgetws(wchar_t* restrict s, int n, FILE* restrict stream);
+wint_t fputwc(wchar_t c, FILE* stream);
+int fputws(const wchar_t* restrict s, FILE* restrict stream);
+int fwide(FILE* stream, int mode);
+wint_t getwc(FILE* stream);
+wint_t getwchar();
+wint_t putwc(wchar_t c, FILE* stream);
+wint_t putwchar(wchar_t c);
+wint_t ungetwc(wint_t c, FILE* stream);
+double wcstod(const wchar_t* restrict nptr, wchar_t** restrict endptr);
+float wcstof(const wchar_t* restrict nptr, wchar_t** restrict endptr);         // C99
+long double wcstold(const wchar_t* restrict nptr, wchar_t** restrict endptr);  // C99
+long wcstol(const wchar_t* restrict nptr, wchar_t** restrict endptr, int base);
+long long wcstoll(const wchar_t* restrict nptr, wchar_t** restrict endptr, int base);  // C99
+unsigned long wcstoul(const wchar_t* restrict nptr, wchar_t** restrict endptr, int base);
+unsigned long long wcstoull(const wchar_t* restrict nptr, wchar_t** restrict endptr, int base);  // C99
+wchar_t* wcscpy(wchar_t* restrict s1, const wchar_t* restrict s2);
+wchar_t* wcsncpy(wchar_t* restrict s1, const wchar_t* restrict s2, size_t n);
+wchar_t* wcscat(wchar_t* restrict s1, const wchar_t* restrict s2);
+wchar_t* wcsncat(wchar_t* restrict s1, const wchar_t* restrict s2, size_t n);
+int wcscmp(const wchar_t* s1, const wchar_t* s2);
+int wcscoll(const wchar_t* s1, const wchar_t* s2);
+int wcsncmp(const wchar_t* s1, const wchar_t* s2, size_t n);
+size_t wcsxfrm(wchar_t* restrict s1, const wchar_t* restrict s2, size_t n);
+const wchar_t* wcschr(const wchar_t* s, wchar_t c);
+      wchar_t* wcschr(      wchar_t* s, wchar_t c);
+size_t wcscspn(const wchar_t* s1, const wchar_t* s2);
+size_t wcslen(const wchar_t* s);
+const wchar_t* wcspbrk(const wchar_t* s1, const wchar_t* s2);
+      wchar_t* wcspbrk(      wchar_t* s1, const wchar_t* s2);
+const wchar_t* wcsrchr(const wchar_t* s, wchar_t c);
+      wchar_t* wcsrchr(      wchar_t* s, wchar_t c);
+size_t wcsspn(const wchar_t* s1, const wchar_t* s2);
+const wchar_t* wcsstr(const wchar_t* s1, const wchar_t* s2);
+      wchar_t* wcsstr(      wchar_t* s1, const wchar_t* s2);
+wchar_t* wcstok(wchar_t* restrict s1, const wchar_t* restrict s2, wchar_t** restrict ptr);
+const wchar_t* wmemchr(const wchar_t* s, wchar_t c, size_t n);
+      wchar_t* wmemchr(      wchar_t* s, wchar_t c, size_t n);
+int wmemcmp(wchar_t* restrict s1, const wchar_t* restrict s2, size_t n);
+wchar_t* wmemcpy(wchar_t* restrict s1, const wchar_t* restrict s2, size_t n);
+wchar_t* wmemmove(wchar_t* s1, const wchar_t* s2, size_t n);
+wchar_t* wmemset(wchar_t* s, wchar_t c, size_t n);
+size_t wcsftime(wchar_t* restrict s, size_t maxsize, const wchar_t* restrict format,
+                const tm* restrict timeptr);
+wint_t btowc(int c);
+int wctob(wint_t c);
+int mbsinit(const mbstate_t* ps);
+size_t mbrlen(const char* restrict s, size_t n, mbstate_t* restrict ps);
+size_t mbrtowc(wchar_t* restrict pwc, const char* restrict s, size_t n, mbstate_t* restrict ps);
+size_t wcrtomb(char* restrict s, wchar_t wc, mbstate_t* restrict ps);
+size_t mbsrtowcs(wchar_t* restrict dst, const char** restrict src, size_t len,
+                 mbstate_t* restrict ps);
+size_t wcsrtombs(char* restrict dst, const wchar_t** restrict src, size_t len,
+                 mbstate_t* restrict ps);
+
+}  // std
+
+*/
+
+#include <__config>
+#include <__type_traits/copy_cv.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_equality_comparable.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_cv.h>
+#include <cwctype>
+
+#include <wchar.h>
+
+#ifndef _LIBCPP_WCHAR_H
+#   error <cwchar> tried including <wchar.h> but didn't find libc++'s <wchar.h> header. \
+          This usually means that your header search paths are not configured properly. \
+          The header search paths should contain the C++ Standard Library headers before \
+          any C Standard Library, and you are probably using compiler flags that make that \
+          not be the case.
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::mbstate_t _LIBCPP_USING_IF_EXISTS;
+using ::size_t _LIBCPP_USING_IF_EXISTS;
+using ::tm _LIBCPP_USING_IF_EXISTS;
+using ::wint_t _LIBCPP_USING_IF_EXISTS;
+using ::FILE _LIBCPP_USING_IF_EXISTS;
+using ::fwprintf _LIBCPP_USING_IF_EXISTS;
+using ::fwscanf _LIBCPP_USING_IF_EXISTS;
+using ::swprintf _LIBCPP_USING_IF_EXISTS;
+using ::vfwprintf _LIBCPP_USING_IF_EXISTS;
+using ::vswprintf _LIBCPP_USING_IF_EXISTS;
+using ::swscanf _LIBCPP_USING_IF_EXISTS;
+using ::vfwscanf _LIBCPP_USING_IF_EXISTS;
+using ::vswscanf _LIBCPP_USING_IF_EXISTS;
+using ::fgetwc _LIBCPP_USING_IF_EXISTS;
+using ::fgetws _LIBCPP_USING_IF_EXISTS;
+using ::fputwc _LIBCPP_USING_IF_EXISTS;
+using ::fputws _LIBCPP_USING_IF_EXISTS;
+using ::fwide _LIBCPP_USING_IF_EXISTS;
+using ::getwc _LIBCPP_USING_IF_EXISTS;
+using ::putwc _LIBCPP_USING_IF_EXISTS;
+using ::ungetwc _LIBCPP_USING_IF_EXISTS;
+using ::wcstod _LIBCPP_USING_IF_EXISTS;
+using ::wcstof _LIBCPP_USING_IF_EXISTS;
+using ::wcstold _LIBCPP_USING_IF_EXISTS;
+using ::wcstol _LIBCPP_USING_IF_EXISTS;
+using ::wcstoll _LIBCPP_USING_IF_EXISTS;
+using ::wcstoul _LIBCPP_USING_IF_EXISTS;
+using ::wcstoull _LIBCPP_USING_IF_EXISTS;
+using ::wcscpy _LIBCPP_USING_IF_EXISTS;
+using ::wcsncpy _LIBCPP_USING_IF_EXISTS;
+using ::wcscat _LIBCPP_USING_IF_EXISTS;
+using ::wcsncat _LIBCPP_USING_IF_EXISTS;
+using ::wcscmp _LIBCPP_USING_IF_EXISTS;
+using ::wcscoll _LIBCPP_USING_IF_EXISTS;
+using ::wcsncmp _LIBCPP_USING_IF_EXISTS;
+using ::wcsxfrm _LIBCPP_USING_IF_EXISTS;
+using ::wcschr _LIBCPP_USING_IF_EXISTS;
+using ::wcspbrk _LIBCPP_USING_IF_EXISTS;
+using ::wcsrchr _LIBCPP_USING_IF_EXISTS;
+using ::wcsstr _LIBCPP_USING_IF_EXISTS;
+using ::wmemchr _LIBCPP_USING_IF_EXISTS;
+using ::wcscspn _LIBCPP_USING_IF_EXISTS;
+using ::wcslen _LIBCPP_USING_IF_EXISTS;
+using ::wcsspn _LIBCPP_USING_IF_EXISTS;
+using ::wcstok _LIBCPP_USING_IF_EXISTS;
+using ::wmemcmp _LIBCPP_USING_IF_EXISTS;
+using ::wmemcpy _LIBCPP_USING_IF_EXISTS;
+using ::wmemmove _LIBCPP_USING_IF_EXISTS;
+using ::wmemset _LIBCPP_USING_IF_EXISTS;
+using ::wcsftime _LIBCPP_USING_IF_EXISTS;
+using ::btowc _LIBCPP_USING_IF_EXISTS;
+using ::wctob _LIBCPP_USING_IF_EXISTS;
+using ::mbsinit _LIBCPP_USING_IF_EXISTS;
+using ::mbrlen _LIBCPP_USING_IF_EXISTS;
+using ::mbrtowc _LIBCPP_USING_IF_EXISTS;
+using ::wcrtomb _LIBCPP_USING_IF_EXISTS;
+using ::mbsrtowcs _LIBCPP_USING_IF_EXISTS;
+using ::wcsrtombs _LIBCPP_USING_IF_EXISTS;
+
+using ::getwchar _LIBCPP_USING_IF_EXISTS;
+using ::vwscanf _LIBCPP_USING_IF_EXISTS;
+using ::wscanf _LIBCPP_USING_IF_EXISTS;
+
+using ::putwchar _LIBCPP_USING_IF_EXISTS;
+using ::vwprintf _LIBCPP_USING_IF_EXISTS;
+using ::wprintf _LIBCPP_USING_IF_EXISTS;
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 size_t __constexpr_wcslen(const wchar_t* __str) {
+#if __has_builtin(__builtin_wcslen)
+  return __builtin_wcslen(__str);
+#else
+  if (!__libcpp_is_constant_evaluated())
+    return std::wcslen(__str);
+
+  size_t __len = 0;
+  for (; *__str != L'\0'; ++__str)
+    ++__len;
+  return __len;
+#endif
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int
+__constexpr_wmemcmp(const wchar_t* __lhs, const wchar_t* __rhs, size_t __count) {
+#if __has_builtin(__builtin_wmemcmp)
+  return __builtin_wmemcmp(__lhs, __rhs, __count);
+#else
+  if (!__libcpp_is_constant_evaluated())
+    return std::wmemcmp(__lhs, __rhs, __count);
+
+  for (; __count; --__count, ++__lhs, ++__rhs) {
+    if (*__lhs < *__rhs)
+      return -1;
+    if (*__rhs < *__lhs)
+      return 1;
+  }
+  return 0;
+#endif
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __constexpr_wmemchr(_Tp* __str, _Up __value, size_t __count) {
+  static_assert(sizeof(_Tp) == sizeof(wchar_t)&& _LIBCPP_ALIGNOF(_Tp) >= _LIBCPP_ALIGNOF(wchar_t) &&
+                    __libcpp_is_trivially_equality_comparable<_Tp, _Tp>::value,
+                "Calling wmemchr on non-trivially equality comparable types is unsafe.");
+
+#if __has_builtin(__builtin_wmemchr)
+  if (!__libcpp_is_constant_evaluated()) {
+    wchar_t __value_buffer = 0;
+    __builtin_memcpy(&__value_buffer, &__value, sizeof(wchar_t));
+    return reinterpret_cast<_Tp*>(
+        __builtin_wmemchr(reinterpret_cast<__copy_cv_t<_Tp, wchar_t>*>(__str), __value_buffer, __count));
+  }
+#  if _LIBCPP_STD_VER >= 17
+  else if constexpr (is_same_v<remove_cv_t<_Tp>, wchar_t>)
+    return __builtin_wmemchr(__str, __value, __count);
+#  endif
+#endif // __has_builtin(__builtin_wmemchr)
+
+  for (; __count; --__count) {
+    if (*__str == __value)
+      return __str;
+    ++__str;
+  }
+  return nullptr;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
+#endif
+
+#endif // _LIBCPP_CWCHAR
diff --git a/libcxx/include/__cxx03/cwctype b/libcxx/include/__cxx03/cwctype
new file mode 100644
index 000000000000..04abfabef579
--- /dev/null
+++ b/libcxx/include/__cxx03/cwctype
@@ -0,0 +1,97 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CWCTYPE
+#define _LIBCPP_CWCTYPE
+
+/*
+    cwctype synopsis
+
+Macros:
+
+    WEOF
+
+namespace std
+{
+
+Types:
+
+    wint_t
+    wctrans_t
+    wctype_t
+
+int iswalnum(wint_t wc);
+int iswalpha(wint_t wc);
+int iswblank(wint_t wc);  // C99
+int iswcntrl(wint_t wc);
+int iswdigit(wint_t wc);
+int iswgraph(wint_t wc);
+int iswlower(wint_t wc);
+int iswprint(wint_t wc);
+int iswpunct(wint_t wc);
+int iswspace(wint_t wc);
+int iswupper(wint_t wc);
+int iswxdigit(wint_t wc);
+int iswctype(wint_t wc, wctype_t desc);
+wctype_t wctype(const char* property);
+wint_t towlower(wint_t wc);
+wint_t towupper(wint_t wc);
+wint_t towctrans(wint_t wc, wctrans_t desc);
+wctrans_t wctrans(const char* property);
+
+}  // std
+
+*/
+
+#include <__config>
+#include <cctype>
+
+#include <wctype.h>
+
+#ifndef _LIBCPP_WCTYPE_H
+#   error <cwctype> tried including <wctype.h> but didn't find libc++'s <wctype.h> header. \
+          This usually means that your header search paths are not configured properly. \
+          The header search paths should contain the C++ Standard Library headers before \
+          any C Standard Library, and you are probably using compiler flags that make that \
+          not be the case.
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if defined(_LIBCPP_INCLUDED_C_LIBRARY_WCTYPE_H)
+using ::wint_t _LIBCPP_USING_IF_EXISTS;
+using ::wctrans_t _LIBCPP_USING_IF_EXISTS;
+using ::wctype_t _LIBCPP_USING_IF_EXISTS;
+using ::iswalnum _LIBCPP_USING_IF_EXISTS;
+using ::iswalpha _LIBCPP_USING_IF_EXISTS;
+using ::iswblank _LIBCPP_USING_IF_EXISTS;
+using ::iswcntrl _LIBCPP_USING_IF_EXISTS;
+using ::iswdigit _LIBCPP_USING_IF_EXISTS;
+using ::iswgraph _LIBCPP_USING_IF_EXISTS;
+using ::iswlower _LIBCPP_USING_IF_EXISTS;
+using ::iswprint _LIBCPP_USING_IF_EXISTS;
+using ::iswpunct _LIBCPP_USING_IF_EXISTS;
+using ::iswspace _LIBCPP_USING_IF_EXISTS;
+using ::iswupper _LIBCPP_USING_IF_EXISTS;
+using ::iswxdigit _LIBCPP_USING_IF_EXISTS;
+using ::iswctype _LIBCPP_USING_IF_EXISTS;
+using ::wctype _LIBCPP_USING_IF_EXISTS;
+using ::towlower _LIBCPP_USING_IF_EXISTS;
+using ::towupper _LIBCPP_USING_IF_EXISTS;
+using ::towctrans _LIBCPP_USING_IF_EXISTS;
+using ::wctrans _LIBCPP_USING_IF_EXISTS;
+#endif // _LIBCPP_INCLUDED_C_LIBRARY_WCTYPE_H
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_CWCTYPE
diff --git a/libcxx/include/__cxx03/deque b/libcxx/include/__cxx03/deque
new file mode 100644
index 000000000000..e73135a8647b
--- /dev/null
+++ b/libcxx/include/__cxx03/deque
@@ -0,0 +1,2621 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_DEQUE
+#define _LIBCPP_DEQUE
+
+/*
+    deque synopsis
+
+namespace std
+{
+
+template <class T, class Allocator = allocator<T> >
+class deque
+{
+public:
+    // types:
+    typedef T value_type;
+    typedef Allocator allocator_type;
+
+    typedef typename allocator_type::reference       reference;
+    typedef typename allocator_type::const_reference const_reference;
+    typedef implementation-defined                   iterator;
+    typedef implementation-defined                   const_iterator;
+    typedef typename allocator_type::size_type       size_type;
+    typedef typename allocator_type::difference_type difference_type;
+
+    typedef typename allocator_type::pointer         pointer;
+    typedef typename allocator_type::const_pointer   const_pointer;
+    typedef std::reverse_iterator<iterator>          reverse_iterator;
+    typedef std::reverse_iterator<const_iterator>    const_reverse_iterator;
+
+    // construct/copy/destroy:
+    deque() noexcept(is_nothrow_default_constructible<allocator_type>::value);
+    explicit deque(const allocator_type& a);
+    explicit deque(size_type n);
+    explicit deque(size_type n, const allocator_type& a); // C++14
+    deque(size_type n, const value_type& v);
+    deque(size_type n, const value_type& v, const allocator_type& a);
+    template <class InputIterator>
+        deque(InputIterator f, InputIterator l);
+    template <class InputIterator>
+        deque(InputIterator f, InputIterator l, const allocator_type& a);
+    template<container-compatible-range<T> R>
+        deque(from_range_t, R&& rg, const Allocator& = Allocator()); // C++23
+    deque(const deque& c);
+    deque(deque&& c)
+        noexcept(is_nothrow_move_constructible<allocator_type>::value);
+    deque(initializer_list<value_type> il, const Allocator& a = allocator_type());
+    deque(const deque& c, const allocator_type& a);
+    deque(deque&& c, const allocator_type& a);
+    ~deque();
+
+    deque& operator=(const deque& c);
+    deque& operator=(deque&& c)
+        noexcept(
+             allocator_type::propagate_on_container_move_assignment::value &&
+             is_nothrow_move_assignable<allocator_type>::value);
+    deque& operator=(initializer_list<value_type> il);
+
+    template <class InputIterator>
+        void assign(InputIterator f, InputIterator l);
+    template<container-compatible-range<T> R>
+      void assign_range(R&& rg); // C++23
+    void assign(size_type n, const value_type& v);
+    void assign(initializer_list<value_type> il);
+
+    allocator_type get_allocator() const noexcept;
+
+    // iterators:
+
+    iterator       begin() noexcept;
+    const_iterator begin() const noexcept;
+    iterator       end() noexcept;
+    const_iterator end() const noexcept;
+
+    reverse_iterator       rbegin() noexcept;
+    const_reverse_iterator rbegin() const noexcept;
+    reverse_iterator       rend() noexcept;
+    const_reverse_iterator rend() const noexcept;
+
+    const_iterator         cbegin() const noexcept;
+    const_iterator         cend() const noexcept;
+    const_reverse_iterator crbegin() const noexcept;
+    const_reverse_iterator crend() const noexcept;
+
+    // capacity:
+    size_type size() const noexcept;
+    size_type max_size() const noexcept;
+    void resize(size_type n);
+    void resize(size_type n, const value_type& v);
+    void shrink_to_fit();
+    bool empty() const noexcept;
+
+    // element access:
+    reference operator[](size_type i);
+    const_reference operator[](size_type i) const;
+    reference at(size_type i);
+    const_reference at(size_type i) const;
+    reference front();
+    const_reference front() const;
+    reference back();
+    const_reference back() const;
+
+    // modifiers:
+    void push_front(const value_type& v);
+    void push_front(value_type&& v);
+    template<container-compatible-range<T> R>
+      void prepend_range(R&& rg); // C++23
+    void push_back(const value_type& v);
+    void push_back(value_type&& v);
+    template<container-compatible-range<T> R>
+      void append_range(R&& rg); // C++23
+    template <class... Args> reference emplace_front(Args&&... args);  // reference in C++17
+    template <class... Args> reference emplace_back(Args&&... args);   // reference in C++17
+    template <class... Args> iterator emplace(const_iterator p, Args&&... args);
+    iterator insert(const_iterator p, const value_type& v);
+    iterator insert(const_iterator p, value_type&& v);
+    iterator insert(const_iterator p, size_type n, const value_type& v);
+    template <class InputIterator>
+        iterator insert(const_iterator p, InputIterator f, InputIterator l);
+    template<container-compatible-range<T> R>
+      iterator insert_range(const_iterator position, R&& rg); // C++23
+    iterator insert(const_iterator p, initializer_list<value_type> il);
+    void pop_front();
+    void pop_back();
+    iterator erase(const_iterator p);
+    iterator erase(const_iterator f, const_iterator l);
+    void swap(deque& c)
+        noexcept(allocator_traits<allocator_type>::is_always_equal::value);  // C++17
+    void clear() noexcept;
+};
+
+template <class InputIterator, class Allocator = allocator<typename iterator_traits<InputIterator>::value_type>>
+   deque(InputIterator, InputIterator, Allocator = Allocator())
+   -> deque<typename iterator_traits<InputIterator>::value_type, Allocator>; // C++17
+
+template<ranges::input_range R, class Allocator = allocator<ranges::range_value_t<R>>>
+  deque(from_range_t, R&&, Allocator = Allocator())
+    -> deque<ranges::range_value_t<R>, Allocator>; // C++23
+
+template <class T, class Allocator>
+    bool operator==(const deque<T,Allocator>& x, const deque<T,Allocator>& y);
+template <class T, class Allocator>
+    bool operator< (const deque<T,Allocator>& x, const deque<T,Allocator>& y); // removed in C++20
+template <class T, class Allocator>
+    bool operator!=(const deque<T,Allocator>& x, const deque<T,Allocator>& y); // removed in C++20
+template <class T, class Allocator>
+    bool operator> (const deque<T,Allocator>& x, const deque<T,Allocator>& y); // removed in C++20
+template <class T, class Allocator>
+    bool operator>=(const deque<T,Allocator>& x, const deque<T,Allocator>& y); // removed in C++20
+template <class T, class Allocator>
+    bool operator<=(const deque<T,Allocator>& x, const deque<T,Allocator>& y); // removed in C++20
+template<class T, class Allocator>
+    synth-three-way-result<T> operator<=>(const deque<T, Allocator>& x,
+                                          const deque<T, Allocator>& y);       // since C++20
+
+// specialized algorithms:
+template <class T, class Allocator>
+    void swap(deque<T,Allocator>& x, deque<T,Allocator>& y)
+         noexcept(noexcept(x.swap(y)));
+
+template <class T, class Allocator, class U>
+    typename deque<T, Allocator>::size_type
+    erase(deque<T, Allocator>& c, const U& value);       // C++20
+template <class T, class Allocator, class Predicate>
+    typename deque<T, Allocator>::size_type
+    erase_if(deque<T, Allocator>& c, Predicate pred);    // C++20
+
+}  // std
+
+*/
+
+#include <__algorithm/copy.h>
+#include <__algorithm/copy_backward.h>
+#include <__algorithm/copy_n.h>
+#include <__algorithm/equal.h>
+#include <__algorithm/fill_n.h>
+#include <__algorithm/lexicographical_compare.h>
+#include <__algorithm/lexicographical_compare_three_way.h>
+#include <__algorithm/min.h>
+#include <__algorithm/remove.h>
+#include <__algorithm/remove_if.h>
+#include <__algorithm/unwrap_iter.h>
+#include <__assert>
+#include <__config>
+#include <__debug_utils/sanitizers.h>
+#include <__format/enable_insertable.h>
+#include <__fwd/deque.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/prev.h>
+#include <__iterator/reverse_iterator.h>
+#include <__iterator/segmented_iterator.h>
+#include <__memory/addressof.h>
+#include <__memory/allocator_destructor.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/temp_value.h>
+#include <__memory/unique_ptr.h>
+#include <__memory_resource/polymorphic_allocator.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/container_compatible_range.h>
+#include <__ranges/from_range.h>
+#include <__ranges/size.h>
+#include <__split_buffer>
+#include <__type_traits/is_allocator.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_swappable.h>
+#include <__type_traits/type_identity.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <__utility/swap.h>
+#include <limits>
+#include <stdexcept>
+#include <version>
+
+// standard-mandated includes
+
+// [iterator.range]
+#include <__iterator/access.h>
+#include <__iterator/data.h>
+#include <__iterator/empty.h>
+#include <__iterator/reverse_access.h>
+#include <__iterator/size.h>
+
+// [deque.syn]
+#include <compare>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ValueType, class _DiffType>
+struct __deque_block_size {
+  static const _DiffType value = sizeof(_ValueType) < 256 ? 4096 / sizeof(_ValueType) : 16;
+};
+
+template <class _ValueType,
+          class _Pointer,
+          class _Reference,
+          class _MapPointer,
+          class _DiffType,
+          _DiffType _BS =
+#ifdef _LIBCPP_ABI_INCOMPLETE_TYPES_IN_DEQUE
+              // Keep template parameter to avoid changing all template declarations thoughout
+              // this file.
+          0
+#else
+              __deque_block_size<_ValueType, _DiffType>::value
+#endif
+          >
+class _LIBCPP_TEMPLATE_VIS __deque_iterator {
+  typedef _MapPointer __map_iterator;
+
+public:
+  typedef _Pointer pointer;
+  typedef _DiffType difference_type;
+
+private:
+  __map_iterator __m_iter_;
+  pointer __ptr_;
+
+  static const difference_type __block_size;
+
+public:
+  typedef _ValueType value_type;
+  typedef random_access_iterator_tag iterator_category;
+  typedef _Reference reference;
+
+  _LIBCPP_HIDE_FROM_ABI __deque_iterator() _NOEXCEPT
+#if _LIBCPP_STD_VER >= 14
+      : __m_iter_(nullptr),
+        __ptr_(nullptr)
+#endif
+  {
+  }
+
+  template <class _Pp, class _Rp, class _MP, __enable_if_t<is_convertible<_Pp, pointer>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  __deque_iterator(const __deque_iterator<value_type, _Pp, _Rp, _MP, difference_type, _BS>& __it) _NOEXCEPT
+      : __m_iter_(__it.__m_iter_),
+        __ptr_(__it.__ptr_) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return *__ptr_; }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return __ptr_; }
+
+  _LIBCPP_HIDE_FROM_ABI __deque_iterator& operator++() {
+    if (++__ptr_ - *__m_iter_ == __block_size) {
+      ++__m_iter_;
+      __ptr_ = *__m_iter_;
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __deque_iterator operator++(int) {
+    __deque_iterator __tmp = *this;
+    ++(*this);
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __deque_iterator& operator--() {
+    if (__ptr_ == *__m_iter_) {
+      --__m_iter_;
+      __ptr_ = *__m_iter_ + __block_size;
+    }
+    --__ptr_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __deque_iterator operator--(int) {
+    __deque_iterator __tmp = *this;
+    --(*this);
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __deque_iterator& operator+=(difference_type __n) {
+    if (__n != 0) {
+      __n += __ptr_ - *__m_iter_;
+      if (__n > 0) {
+        __m_iter_ += __n / __block_size;
+        __ptr_ = *__m_iter_ + __n % __block_size;
+      } else // (__n < 0)
+      {
+        difference_type __z = __block_size - 1 - __n;
+        __m_iter_ -= __z / __block_size;
+        __ptr_ = *__m_iter_ + (__block_size - 1 - __z % __block_size);
+      }
+    }
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __deque_iterator& operator-=(difference_type __n) { return *this += -__n; }
+
+  _LIBCPP_HIDE_FROM_ABI __deque_iterator operator+(difference_type __n) const {
+    __deque_iterator __t(*this);
+    __t += __n;
+    return __t;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __deque_iterator operator-(difference_type __n) const {
+    __deque_iterator __t(*this);
+    __t -= __n;
+    return __t;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend __deque_iterator operator+(difference_type __n, const __deque_iterator& __it) {
+    return __it + __n;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend difference_type operator-(const __deque_iterator& __x, const __deque_iterator& __y) {
+    if (__x != __y)
+      return (__x.__m_iter_ - __y.__m_iter_) * __block_size + (__x.__ptr_ - *__x.__m_iter_) -
+             (__y.__ptr_ - *__y.__m_iter_);
+    return 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI reference operator[](difference_type __n) const { return *(*this + __n); }
+
+  _LIBCPP_HIDE_FROM_ABI friend bool operator==(const __deque_iterator& __x, const __deque_iterator& __y) {
+    return __x.__ptr_ == __y.__ptr_;
+  }
+
+#if _LIBCPP_STD_VER <= 17
+  _LIBCPP_HIDE_FROM_ABI friend bool operator!=(const __deque_iterator& __x, const __deque_iterator& __y) {
+    return !(__x == __y);
+  }
+#endif
+
+  // TODO(mordante) disable these overloads in the LLVM 20 release.
+  _LIBCPP_HIDE_FROM_ABI friend bool operator<(const __deque_iterator& __x, const __deque_iterator& __y) {
+    return __x.__m_iter_ < __y.__m_iter_ || (__x.__m_iter_ == __y.__m_iter_ && __x.__ptr_ < __y.__ptr_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend bool operator>(const __deque_iterator& __x, const __deque_iterator& __y) {
+    return __y < __x;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend bool operator<=(const __deque_iterator& __x, const __deque_iterator& __y) {
+    return !(__y < __x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend bool operator>=(const __deque_iterator& __x, const __deque_iterator& __y) {
+    return !(__x < __y);
+  }
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI friend strong_ordering operator<=>(const __deque_iterator& __x, const __deque_iterator& __y) {
+    if (__x.__m_iter_ < __y.__m_iter_)
+      return strong_ordering::less;
+
+    if (__x.__m_iter_ == __y.__m_iter_) {
+      if constexpr (three_way_comparable<pointer, strong_ordering>) {
+        return __x.__ptr_ <=> __y.__ptr_;
+      } else {
+        if (__x.__ptr_ < __y.__ptr_)
+          return strong_ordering::less;
+
+        if (__x.__ptr_ == __y.__ptr_)
+          return strong_ordering::equal;
+
+        return strong_ordering::greater;
+      }
+    }
+
+    return strong_ordering::greater;
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+private:
+  _LIBCPP_HIDE_FROM_ABI explicit __deque_iterator(__map_iterator __m, pointer __p) _NOEXCEPT
+      : __m_iter_(__m),
+        __ptr_(__p) {}
+
+  template <class _Tp, class _Ap>
+  friend class _LIBCPP_TEMPLATE_VIS deque;
+  template <class _Vp, class _Pp, class _Rp, class _MP, class _Dp, _Dp>
+  friend class _LIBCPP_TEMPLATE_VIS __deque_iterator;
+
+  template <class>
+  friend struct __segmented_iterator_traits;
+};
+
+template <class _ValueType, class _Pointer, class _Reference, class _MapPointer, class _DiffType, _DiffType _BlockSize>
+struct __segmented_iterator_traits<
+    __deque_iterator<_ValueType, _Pointer, _Reference, _MapPointer, _DiffType, _BlockSize> > {
+private:
+  using _Iterator = __deque_iterator<_ValueType, _Pointer, _Reference, _MapPointer, _DiffType, _BlockSize>;
+
+public:
+  using __is_segmented_iterator = true_type;
+  using __segment_iterator      = _MapPointer;
+  using __local_iterator        = _Pointer;
+
+  static _LIBCPP_HIDE_FROM_ABI __segment_iterator __segment(_Iterator __iter) { return __iter.__m_iter_; }
+  static _LIBCPP_HIDE_FROM_ABI __local_iterator __local(_Iterator __iter) { return __iter.__ptr_; }
+  static _LIBCPP_HIDE_FROM_ABI __local_iterator __begin(__segment_iterator __iter) { return *__iter; }
+
+  static _LIBCPP_HIDE_FROM_ABI __local_iterator __end(__segment_iterator __iter) {
+    return *__iter + _Iterator::__block_size;
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI _Iterator __compose(__segment_iterator __segment, __local_iterator __local) {
+    if (__segment && __local == __end(__segment)) {
+      ++__segment;
+      return _Iterator(__segment, *__segment);
+    }
+    return _Iterator(__segment, __local);
+  }
+};
+
+template <class _ValueType, class _Pointer, class _Reference, class _MapPointer, class _DiffType, _DiffType _BlockSize>
+const _DiffType __deque_iterator<_ValueType, _Pointer, _Reference, _MapPointer, _DiffType, _BlockSize>::__block_size =
+    __deque_block_size<_ValueType, _DiffType>::value;
+
+template <class _Tp, class _Allocator /*= allocator<_Tp>*/>
+class _LIBCPP_TEMPLATE_VIS deque {
+public:
+  // types:
+
+  using value_type = _Tp;
+
+  using allocator_type = _Allocator;
+  using __alloc_traits = allocator_traits<allocator_type>;
+  static_assert(__check_valid_allocator<allocator_type>::value, "");
+  static_assert(is_same<typename allocator_type::value_type, value_type>::value,
+                "Allocator::value_type must be same type as value_type");
+
+  using size_type       = typename __alloc_traits::size_type;
+  using difference_type = typename __alloc_traits::difference_type;
+
+  using pointer       = typename __alloc_traits::pointer;
+  using const_pointer = typename __alloc_traits::const_pointer;
+
+  using __pointer_allocator       = __rebind_alloc<__alloc_traits, pointer>;
+  using __const_pointer_allocator = __rebind_alloc<__alloc_traits, const_pointer>;
+  using __map                     = __split_buffer<pointer, __pointer_allocator>;
+  using __map_alloc_traits        = allocator_traits<__pointer_allocator>;
+  using __map_pointer             = typename __map_alloc_traits::pointer;
+  using __map_const_pointer       = typename allocator_traits<__const_pointer_allocator>::const_pointer;
+  using __map_const_iterator      = typename __map::const_iterator;
+
+  using reference       = value_type&;
+  using const_reference = const value_type&;
+
+  using iterator = __deque_iterator<value_type, pointer, reference, __map_pointer, difference_type>;
+  using const_iterator =
+      __deque_iterator<value_type, const_pointer, const_reference, __map_const_pointer, difference_type>;
+  using reverse_iterator       = std::reverse_iterator<iterator>;
+  using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+
+  // A deque contains the following members which may be trivially relocatable:
+  // - __map: is a `__split_buffer`, see `__split_buffer` for more information on when it is trivially relocatable
+  // - size_type: is always trivially relocatable, since it is required to be an integral type
+  // - allocator_type: may not be trivially relocatable, so it's checked
+  // None of these are referencing the `deque` itself, so if all of them are trivially relocatable, `deque` is too.
+  using __trivially_relocatable = __conditional_t<
+      __libcpp_is_trivially_relocatable<__map>::value && __libcpp_is_trivially_relocatable<allocator_type>::value,
+      deque,
+      void>;
+
+  static_assert(is_nothrow_default_constructible<allocator_type>::value ==
+                    is_nothrow_default_constructible<__pointer_allocator>::value,
+                "rebinding an allocator should not change exception guarantees");
+  static_assert(is_nothrow_move_constructible<allocator_type>::value ==
+                    is_nothrow_move_constructible<typename __map::allocator_type>::value,
+                "rebinding an allocator should not change exception guarantees");
+
+private:
+  struct __deque_block_range {
+    explicit _LIBCPP_HIDE_FROM_ABI __deque_block_range(pointer __b, pointer __e) _NOEXCEPT
+        : __begin_(__b),
+          __end_(__e) {}
+    const pointer __begin_;
+    const pointer __end_;
+  };
+
+  struct __deque_range {
+    iterator __pos_;
+    const iterator __end_;
+
+    _LIBCPP_HIDE_FROM_ABI __deque_range(iterator __pos, iterator __e) _NOEXCEPT : __pos_(__pos), __end_(__e) {}
+
+    explicit _LIBCPP_HIDE_FROM_ABI operator bool() const _NOEXCEPT { return __pos_ != __end_; }
+
+    _LIBCPP_HIDE_FROM_ABI __deque_range begin() const { return *this; }
+
+    _LIBCPP_HIDE_FROM_ABI __deque_range end() const { return __deque_range(__end_, __end_); }
+    _LIBCPP_HIDE_FROM_ABI __deque_block_range operator*() const _NOEXCEPT {
+      if (__pos_.__m_iter_ == __end_.__m_iter_) {
+        return __deque_block_range(__pos_.__ptr_, __end_.__ptr_);
+      }
+      return __deque_block_range(__pos_.__ptr_, *__pos_.__m_iter_ + __block_size);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __deque_range& operator++() _NOEXCEPT {
+      if (__pos_.__m_iter_ == __end_.__m_iter_) {
+        __pos_ = __end_;
+      } else {
+        ++__pos_.__m_iter_;
+        __pos_.__ptr_ = *__pos_.__m_iter_;
+      }
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend bool operator==(__deque_range const& __lhs, __deque_range const& __rhs) {
+      return __lhs.__pos_ == __rhs.__pos_;
+    }
+    _LIBCPP_HIDE_FROM_ABI friend bool operator!=(__deque_range const& __lhs, __deque_range const& __rhs) {
+      return !(__lhs == __rhs);
+    }
+  };
+
+  struct _ConstructTransaction {
+    _LIBCPP_HIDE_FROM_ABI _ConstructTransaction(deque* __db, __deque_block_range& __r)
+        : __pos_(__r.__begin_), __end_(__r.__end_), __begin_(__r.__begin_), __base_(__db) {}
+
+    _LIBCPP_HIDE_FROM_ABI ~_ConstructTransaction() { __base_->__size() += (__pos_ - __begin_); }
+
+    pointer __pos_;
+    const pointer __end_;
+
+  private:
+    const pointer __begin_;
+    deque* const __base_;
+  };
+
+  static const difference_type __block_size;
+
+  __map __map_;
+  size_type __start_;
+  __compressed_pair<size_type, allocator_type> __size_;
+
+public:
+  // construct/copy/destroy:
+  _LIBCPP_HIDE_FROM_ABI deque() _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
+      : __start_(0), __size_(0, __default_init_tag()) {
+    __annotate_new(0);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~deque() {
+    clear();
+    __annotate_delete();
+    typename __map::iterator __i = __map_.begin();
+    typename __map::iterator __e = __map_.end();
+    for (; __i != __e; ++__i)
+      __alloc_traits::deallocate(__alloc(), *__i, __block_size);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI explicit deque(const allocator_type& __a)
+      : __map_(__pointer_allocator(__a)), __start_(0), __size_(0, __a) {
+    __annotate_new(0);
+  }
+
+  explicit _LIBCPP_HIDE_FROM_ABI deque(size_type __n);
+#if _LIBCPP_STD_VER >= 14
+  explicit _LIBCPP_HIDE_FROM_ABI deque(size_type __n, const _Allocator& __a);
+#endif
+  _LIBCPP_HIDE_FROM_ABI deque(size_type __n, const value_type& __v);
+
+  template <__enable_if_t<__is_allocator<_Allocator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI deque(size_type __n, const value_type& __v, const allocator_type& __a)
+      : __map_(__pointer_allocator(__a)), __start_(0), __size_(0, __a) {
+    __annotate_new(0);
+    if (__n > 0)
+      __append(__n, __v);
+  }
+
+  template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI deque(_InputIter __f, _InputIter __l);
+  template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI deque(_InputIter __f, _InputIter __l, const allocator_type& __a);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI deque(from_range_t, _Range&& __range, const allocator_type& __a = allocator_type())
+      : __map_(__pointer_allocator(__a)), __start_(0), __size_(0, __a) {
+    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      __append_with_size(ranges::begin(__range), ranges::distance(__range));
+
+    } else {
+      for (auto&& __e : __range) {
+        emplace_back(std::forward<decltype(__e)>(__e));
+      }
+    }
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI deque(const deque& __c);
+  _LIBCPP_HIDE_FROM_ABI deque(const deque& __c, const __type_identity_t<allocator_type>& __a);
+
+  _LIBCPP_HIDE_FROM_ABI deque& operator=(const deque& __c);
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI deque(initializer_list<value_type> __il);
+  _LIBCPP_HIDE_FROM_ABI deque(initializer_list<value_type> __il, const allocator_type& __a);
+
+  _LIBCPP_HIDE_FROM_ABI deque& operator=(initializer_list<value_type> __il) {
+    assign(__il);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI deque(deque&& __c) noexcept(is_nothrow_move_constructible<allocator_type>::value);
+  _LIBCPP_HIDE_FROM_ABI deque(deque&& __c, const __type_identity_t<allocator_type>& __a);
+  _LIBCPP_HIDE_FROM_ABI deque&
+  operator=(deque&& __c) noexcept(__alloc_traits::propagate_on_container_move_assignment::value &&
+                                  is_nothrow_move_assignable<allocator_type>::value);
+
+  _LIBCPP_HIDE_FROM_ABI void assign(initializer_list<value_type> __il) { assign(__il.begin(), __il.end()); }
+#endif // _LIBCPP_CXX03_LANG
+
+  template <class _InputIter,
+            __enable_if_t<__has_input_iterator_category<_InputIter>::value &&
+                              !__has_random_access_iterator_category<_InputIter>::value,
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void assign(_InputIter __f, _InputIter __l);
+  template <class _RAIter, __enable_if_t<__has_random_access_iterator_category<_RAIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void assign(_RAIter __f, _RAIter __l);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI void assign_range(_Range&& __range) {
+    if constexpr (ranges::random_access_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      __assign_with_size_random_access(ranges::begin(__range), __n);
+
+    } else if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      __assign_with_size(ranges::begin(__range), __n);
+
+    } else {
+      __assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
+    }
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI void assign(size_type __n, const value_type& __v);
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI allocator_type& __alloc() _NOEXCEPT { return __size_.second(); }
+  _LIBCPP_HIDE_FROM_ABI const allocator_type& __alloc() const _NOEXCEPT { return __size_.second(); }
+
+  // iterators:
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT {
+    __map_pointer __mp = __map_.begin() + __start_ / __block_size;
+    return iterator(__mp, __map_.empty() ? 0 : *__mp + __start_ % __block_size);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT {
+    __map_const_pointer __mp = static_cast<__map_const_pointer>(__map_.begin() + __start_ / __block_size);
+    return const_iterator(__mp, __map_.empty() ? 0 : *__mp + __start_ % __block_size);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT {
+    size_type __p      = size() + __start_;
+    __map_pointer __mp = __map_.begin() + __p / __block_size;
+    return iterator(__mp, __map_.empty() ? 0 : *__mp + __p % __block_size);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT {
+    size_type __p            = size() + __start_;
+    __map_const_pointer __mp = static_cast<__map_const_pointer>(__map_.begin() + __p / __block_size);
+    return const_iterator(__mp, __map_.empty() ? 0 : *__mp + __p % __block_size);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
+
+  // capacity:
+  _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __size(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type& __size() _NOEXCEPT { return __size_.first(); }
+  _LIBCPP_HIDE_FROM_ABI const size_type& __size() const _NOEXCEPT { return __size_.first(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
+    return std::min<size_type>(__alloc_traits::max_size(__alloc()), numeric_limits<difference_type>::max());
+  }
+  _LIBCPP_HIDE_FROM_ABI void resize(size_type __n);
+  _LIBCPP_HIDE_FROM_ABI void resize(size_type __n, const value_type& __v);
+  _LIBCPP_HIDE_FROM_ABI void shrink_to_fit() _NOEXCEPT;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return size() == 0; }
+
+  // element access:
+  _LIBCPP_HIDE_FROM_ABI reference operator[](size_type __i) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI const_reference operator[](size_type __i) const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI reference at(size_type __i);
+  _LIBCPP_HIDE_FROM_ABI const_reference at(size_type __i) const;
+  _LIBCPP_HIDE_FROM_ABI reference front() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI const_reference front() const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI reference back() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT;
+
+  // 23.2.2.3 modifiers:
+  _LIBCPP_HIDE_FROM_ABI void push_front(const value_type& __v);
+  _LIBCPP_HIDE_FROM_ABI void push_back(const value_type& __v);
+#ifndef _LIBCPP_CXX03_LANG
+#  if _LIBCPP_STD_VER >= 17
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI reference emplace_front(_Args&&... __args);
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI reference emplace_back(_Args&&... __args);
+#  else
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI void emplace_front(_Args&&... __args);
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI void emplace_back(_Args&&... __args);
+#  endif
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __p, _Args&&... __args);
+
+  _LIBCPP_HIDE_FROM_ABI void push_front(value_type&& __v);
+  _LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __v);
+
+#  if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI void prepend_range(_Range&& __range) {
+    insert_range(begin(), std::forward<_Range>(__range));
+  }
+
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI void append_range(_Range&& __range) {
+    insert_range(end(), std::forward<_Range>(__range));
+  }
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __v);
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, initializer_list<value_type> __il) {
+    return insert(__p, __il.begin(), __il.end());
+  }
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __v);
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, size_type __n, const value_type& __v);
+  template <class _InputIter, __enable_if_t<__has_exactly_input_iterator_category<_InputIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, _InputIter __f, _InputIter __l);
+  template <class _ForwardIterator,
+            __enable_if_t<__has_exactly_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, _ForwardIterator __f, _ForwardIterator __l);
+  template <class _BiIter, __enable_if_t<__has_bidirectional_iterator_category<_BiIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, _BiIter __f, _BiIter __l);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI iterator insert_range(const_iterator __position, _Range&& __range) {
+    if constexpr (ranges::bidirectional_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      return __insert_bidirectional(__position, ranges::begin(__range), ranges::end(__range), __n);
+
+    } else if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      return __insert_with_size(__position, ranges::begin(__range), __n);
+
+    } else {
+      return __insert_with_sentinel(__position, ranges::begin(__range), ranges::end(__range));
+    }
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI void pop_front();
+  _LIBCPP_HIDE_FROM_ABI void pop_back();
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p);
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __f, const_iterator __l);
+
+  _LIBCPP_HIDE_FROM_ABI void swap(deque& __c)
+#if _LIBCPP_STD_VER >= 14
+      _NOEXCEPT;
+#else
+      _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>);
+#endif
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI bool __invariants() const {
+    if (!__map_.__invariants())
+      return false;
+    if (__map_.size() >= size_type(-1) / __block_size)
+      return false;
+    for (__map_const_iterator __i = __map_.begin(), __e = __map_.end(); __i != __e; ++__i)
+      if (*__i == nullptr)
+        return false;
+    if (__map_.size() != 0) {
+      if (size() >= __map_.size() * __block_size)
+        return false;
+      if (__start_ >= __map_.size() * __block_size - size())
+        return false;
+    } else {
+      if (size() != 0)
+        return false;
+      if (__start_ != 0)
+        return false;
+    }
+    return true;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(deque& __c)
+      _NOEXCEPT_(!__alloc_traits::propagate_on_container_move_assignment::value ||
+                 is_nothrow_move_assignable<allocator_type>::value) {
+    __move_assign_alloc(__c, integral_constant<bool, __alloc_traits::propagate_on_container_move_assignment::value>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(deque& __c, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
+    __alloc() = std::move(__c.__alloc());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(deque&, false_type) _NOEXCEPT {}
+
+  _LIBCPP_HIDE_FROM_ABI void __move_assign(deque& __c)
+      _NOEXCEPT_(__alloc_traits::propagate_on_container_move_assignment::value&&
+                     is_nothrow_move_assignable<allocator_type>::value) {
+    __map_   = std::move(__c.__map_);
+    __start_ = __c.__start_;
+    __size() = __c.size();
+    __move_assign_alloc(__c);
+    __c.__start_ = __c.__size() = 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static size_type __recommend_blocks(size_type __n) {
+    return __n / __block_size + (__n % __block_size != 0);
+  }
+  _LIBCPP_HIDE_FROM_ABI size_type __capacity() const {
+    return __map_.size() == 0 ? 0 : __map_.size() * __block_size - 1;
+  }
+  _LIBCPP_HIDE_FROM_ABI size_type __block_count() const { return __map_.size(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type __front_spare() const { return __start_; }
+  _LIBCPP_HIDE_FROM_ABI size_type __front_spare_blocks() const { return __front_spare() / __block_size; }
+  _LIBCPP_HIDE_FROM_ABI size_type __back_spare() const { return __capacity() - (__start_ + size()); }
+  _LIBCPP_HIDE_FROM_ABI size_type __back_spare_blocks() const { return __back_spare() / __block_size; }
+
+private:
+  enum __asan_annotation_type { __asan_unposion, __asan_poison };
+
+  enum __asan_annotation_place {
+    __asan_front_moved,
+    __asan_back_moved,
+  };
+
+  _LIBCPP_HIDE_FROM_ABI void __annotate_from_to(
+      size_type __beg,
+      size_type __end,
+      __asan_annotation_type __annotation_type,
+      __asan_annotation_place __place) const _NOEXCEPT {
+    (void)__beg;
+    (void)__end;
+    (void)__annotation_type;
+    (void)__place;
+#ifndef _LIBCPP_HAS_NO_ASAN
+    // __beg - index of the first item to annotate
+    // __end - index behind the last item to annotate (so last item + 1)
+    // __annotation_type - __asan_unposion or __asan_poison
+    // __place - __asan_front_moved or __asan_back_moved
+    // Note: All indexes in __map_
+    if (__beg == __end)
+      return;
+    // __annotations_beg_map - first chunk which annotations we want to modify
+    // __annotations_end_map - last chunk which annotations we want to modify
+    // NOTE: if __end % __block_size == 0, __annotations_end_map points at the next block, which may not exist
+    __map_const_iterator __annotations_beg_map = __map_.begin() + __beg / __block_size;
+    __map_const_iterator __annotations_end_map = __map_.begin() + __end / __block_size;
+
+    bool const __poisoning = __annotation_type == __asan_poison;
+    // __old_c_beg_index - index of the first element in old container
+    // __old_c_end_index - index of the end of old container (last + 1)
+    // Note: may be outside the area we are annotating
+    size_t __old_c_beg_index = (__poisoning && __place == __asan_front_moved) ? __beg : __start_;
+    size_t __old_c_end_index = (__poisoning && __place == __asan_back_moved) ? __end : __start_ + size();
+    bool const __front       = __place == __asan_front_moved;
+
+    if (__poisoning && empty()) {
+      // Special case: we shouldn't trust __start_
+      __old_c_beg_index = __beg;
+      __old_c_end_index = __end;
+    }
+    // __old_c_beg_map - memory block (chunk) with first element
+    // __old_c_end_map - memory block (chunk) with end of old container
+    // Note: if __old_c_end_index % __block_size == 0, __old_c_end_map points at the next block,
+    // which may not exist
+    __map_const_iterator __old_c_beg_map = __map_.begin() + __old_c_beg_index / __block_size;
+    __map_const_iterator __old_c_end_map = __map_.begin() + __old_c_end_index / __block_size;
+
+    // One edge (front/end) of the container was moved and one was not modified.
+    // __new_edge_index - index of new edge
+    // __new_edge_map    - memory block (chunk) with new edge, it always equals to
+    //                    __annotations_beg_map or __annotations_end_map
+    // __old_edge_map    - memory block (chunk) with old edge, it always equals to
+    //                    __old_c_beg_map or __old_c_end_map
+    size_t __new_edge_index             = (__poisoning ^ __front) ? __beg : __end;
+    __map_const_iterator __new_edge_map = __map_.begin() + __new_edge_index / __block_size;
+    __map_const_iterator __old_edge_map = __front ? __old_c_end_map : __old_c_beg_map;
+
+    // We iterate over map pointers (chunks) and fully poison all memory blocks between the first and the last.
+    // First and last chunk may be partially poisoned.
+    // __annotate_end_map may point at not existing chunk, therefore we have to have a check for it.
+    for (__map_const_iterator __map_it = __annotations_beg_map; __map_it <= __annotations_end_map; ++__map_it) {
+      if (__map_it == __annotations_end_map && __end % __block_size == 0)
+        // Chunk may not exist, but nothing to do here anyway
+        break;
+
+      // The beginning and the end of the current memory block
+      const void* __mem_beg = std::__to_address(*__map_it);
+      const void* __mem_end = std::__to_address(*__map_it + __block_size);
+
+      // The beginning of memory-in-use in the memory block before container modification
+      const void* __old_beg =
+          (__map_it == __old_c_beg_map) ? std::__to_address(*__map_it + (__old_c_beg_index % __block_size)) : __mem_beg;
+
+      // The end of memory-in-use in the memory block before container modification
+      const void* __old_end;
+      if (__map_it < __old_c_beg_map || __map_it > __old_c_end_map || (!__poisoning && empty()))
+        __old_end = __old_beg;
+      else
+        __old_end = (__map_it == __old_c_end_map)
+                      ? std::__to_address(*__map_it + (__old_c_end_index % __block_size))
+                      : __mem_end;
+
+      // New edge of the container in current memory block
+      // If the edge is in a different chunk it points on corresponding end of the memory block
+      const void* __new_edge;
+      if (__map_it == __new_edge_map)
+        __new_edge = std::__to_address(*__map_it + (__new_edge_index % __block_size));
+      else
+        __new_edge = (__poisoning ^ __front) ? __mem_beg : __mem_end;
+
+      // Not modified edge of the container
+      // If the edge is in a different chunk it points on corresponding end of the memory block
+      const void* __old_edge;
+      if (__map_it == __old_edge_map)
+        __old_edge = __front ? __old_end : __old_beg;
+      else
+        __old_edge = __front ? __mem_end : __mem_beg;
+
+      // __new_beg - the beginning of memory-in-use in the memory block after container modification
+      // __new_end - the end of memory-in-use in the memory block after container modification
+      const void* __new_beg = __front ? __new_edge : __old_edge;
+      const void* __new_end = __front ? __old_edge : __new_edge;
+
+      std::__annotate_double_ended_contiguous_container<_Allocator>(
+          __mem_beg, __mem_end, __old_beg, __old_end, __new_beg, __new_end);
+    }
+#endif // !_LIBCPP_HAS_NO_ASAN
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __annotate_new(size_type __current_size) const _NOEXCEPT {
+    (void)__current_size;
+#ifndef _LIBCPP_HAS_NO_ASAN
+    if (__current_size == 0)
+      __annotate_from_to(0, __map_.size() * __block_size, __asan_poison, __asan_back_moved);
+    else {
+      __annotate_from_to(0, __start_, __asan_poison, __asan_front_moved);
+      __annotate_from_to(__start_ + __current_size, __map_.size() * __block_size, __asan_poison, __asan_back_moved);
+    }
+#endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __annotate_delete() const _NOEXCEPT {
+#ifndef _LIBCPP_HAS_NO_ASAN
+    if (empty()) {
+      for (size_t __i = 0; __i < __map_.size(); ++__i) {
+        __annotate_whole_block(__i, __asan_unposion);
+      }
+    } else {
+      __annotate_from_to(0, __start_, __asan_unposion, __asan_front_moved);
+      __annotate_from_to(__start_ + size(), __map_.size() * __block_size, __asan_unposion, __asan_back_moved);
+    }
+#endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __annotate_increase_front(size_type __n) const _NOEXCEPT {
+    (void)__n;
+#ifndef _LIBCPP_HAS_NO_ASAN
+    __annotate_from_to(__start_ - __n, __start_, __asan_unposion, __asan_front_moved);
+#endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __annotate_increase_back(size_type __n) const _NOEXCEPT {
+    (void)__n;
+#ifndef _LIBCPP_HAS_NO_ASAN
+    __annotate_from_to(__start_ + size(), __start_ + size() + __n, __asan_unposion, __asan_back_moved);
+#endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __annotate_shrink_front(size_type __old_size, size_type __old_start) const _NOEXCEPT {
+    (void)__old_size;
+    (void)__old_start;
+#ifndef _LIBCPP_HAS_NO_ASAN
+    __annotate_from_to(__old_start, __old_start + (__old_size - size()), __asan_poison, __asan_front_moved);
+#endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __annotate_shrink_back(size_type __old_size, size_type __old_start) const _NOEXCEPT {
+    (void)__old_size;
+    (void)__old_start;
+#ifndef _LIBCPP_HAS_NO_ASAN
+    __annotate_from_to(__old_start + size(), __old_start + __old_size, __asan_poison, __asan_back_moved);
+#endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __annotate_poison_block(const void* __beginning, const void* __end) const _NOEXCEPT {
+    std::__annotate_double_ended_contiguous_container<_Allocator>(__beginning, __end, __beginning, __end, __end, __end);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void
+  __annotate_whole_block(size_t __block_index, __asan_annotation_type __annotation_type) const _NOEXCEPT {
+    (void)__block_index;
+    (void)__annotation_type;
+#ifndef _LIBCPP_HAS_NO_ASAN
+    __map_const_iterator __block_it = __map_.begin() + __block_index;
+    const void* __block_start       = std::__to_address(*__block_it);
+    const void* __block_end         = std::__to_address(*__block_it + __block_size);
+
+    if (__annotation_type == __asan_poison)
+      __annotate_poison_block(__block_start, __block_end);
+    else {
+      std::__annotate_double_ended_contiguous_container<_Allocator>(
+          __block_start, __block_end, __block_start, __block_start, __block_start, __block_end);
+    }
+#endif
+  }
+#if !defined(_LIBCPP_HAS_NO_ASAN)
+
+public:
+  _LIBCPP_HIDE_FROM_ABI bool __verify_asan_annotations() const _NOEXCEPT {
+    // This function tests deque object annotations.
+    if (empty()) {
+      for (__map_const_iterator __it = __map_.begin(); __it != __map_.end(); ++__it) {
+        if (!__sanitizer_verify_double_ended_contiguous_container(
+                std::__to_address(*__it),
+                std::__to_address(*__it),
+                std::__to_address(*__it),
+                std::__to_address(*__it + __block_size)))
+          return false;
+      }
+
+      return true;
+    }
+
+    size_type __end                 = __start_ + size();
+    __map_const_iterator __first_mp = __map_.begin() + __start_ / __block_size;
+    __map_const_iterator __last_mp  = __map_.begin() + (__end - 1) / __block_size;
+
+    // Pointers to first and after last elements
+    // Those can be in different deque blocks
+    const void* __p_beg = std::__to_address(*__first_mp + (__start_ % __block_size));
+    const void* __p_end =
+        std::__to_address(*__last_mp + ((__end % __block_size == 0) ? __block_size : __end % __block_size));
+
+    for (__map_const_iterator __it = __map_.begin(); __it != __map_.end(); ++__it) {
+      // Go over all blocks, find the place we are in and verify its annotations
+      // Note that __p_end points *behind* the last item.
+
+      // - blocks before the first block with container elements
+      // - first block with items
+      // - last block with items
+      // - blocks after last block with ciontainer elements
+
+      // Is the block before or after deque blocks that contain elements?
+      if (__it < __first_mp || __it > __last_mp) {
+        if (!__sanitizer_verify_double_ended_contiguous_container(
+                std::__to_address(*__it),
+                std::__to_address(*__it),
+                std::__to_address(*__it),
+                std::__to_address(*__it + __block_size)))
+          return false;
+      } else {
+        const void* __containers_buffer_beg = (__it == __first_mp) ? __p_beg : (const void*)std::__to_address(*__it);
+        const void* __containers_buffer_end =
+            (__it == __last_mp) ? __p_end : (const void*)std::__to_address(*__it + __block_size);
+        if (!__sanitizer_verify_double_ended_contiguous_container(
+                std::__to_address(*__it),
+                __containers_buffer_beg,
+                __containers_buffer_end,
+                std::__to_address(*__it + __block_size))) {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+
+private:
+#endif // _LIBCPP_VERIFY_ASAN_DEQUE_ANNOTATIONS
+  _LIBCPP_HIDE_FROM_ABI bool __maybe_remove_front_spare(bool __keep_one = true) {
+    if (__front_spare_blocks() >= 2 || (!__keep_one && __front_spare_blocks())) {
+      __annotate_whole_block(0, __asan_unposion);
+      __alloc_traits::deallocate(__alloc(), __map_.front(), __block_size);
+      __map_.pop_front();
+      __start_ -= __block_size;
+      return true;
+    }
+    return false;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool __maybe_remove_back_spare(bool __keep_one = true) {
+    if (__back_spare_blocks() >= 2 || (!__keep_one && __back_spare_blocks())) {
+      __annotate_whole_block(__map_.size() - 1, __asan_unposion);
+      __alloc_traits::deallocate(__alloc(), __map_.back(), __block_size);
+      __map_.pop_back();
+      return true;
+    }
+    return false;
+  }
+
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __f, _Sentinel __l);
+
+  template <class _RandomAccessIterator>
+  _LIBCPP_HIDE_FROM_ABI void __assign_with_size_random_access(_RandomAccessIterator __f, difference_type __n);
+  template <class _Iterator>
+  _LIBCPP_HIDE_FROM_ABI void __assign_with_size(_Iterator __f, difference_type __n);
+
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI iterator __insert_with_sentinel(const_iterator __p, _Iterator __f, _Sentinel __l);
+
+  template <class _Iterator>
+  _LIBCPP_HIDE_FROM_ABI iterator __insert_with_size(const_iterator __p, _Iterator __f, size_type __n);
+
+  template <class _BiIter, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI iterator
+  __insert_bidirectional(const_iterator __p, _BiIter __f, _Sentinel __sent, size_type __n);
+  template <class _BiIter>
+  _LIBCPP_HIDE_FROM_ABI iterator __insert_bidirectional(const_iterator __p, _BiIter __f, _BiIter __l, size_type __n);
+
+  template <class _InpIter, __enable_if_t<__has_exactly_input_iterator_category<_InpIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void __append(_InpIter __f, _InpIter __l);
+  template <class _ForIter, __enable_if_t<__has_forward_iterator_category<_ForIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void __append(_ForIter __f, _ForIter __l);
+
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void __append_with_size(_InputIterator __from, size_type __n);
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI void __append_with_sentinel(_InputIterator __f, _Sentinel __l);
+
+  _LIBCPP_HIDE_FROM_ABI void __append(size_type __n);
+  _LIBCPP_HIDE_FROM_ABI void __append(size_type __n, const value_type& __v);
+  _LIBCPP_HIDE_FROM_ABI void __erase_to_end(const_iterator __f);
+  _LIBCPP_HIDE_FROM_ABI void __add_front_capacity();
+  _LIBCPP_HIDE_FROM_ABI void __add_front_capacity(size_type __n);
+  _LIBCPP_HIDE_FROM_ABI void __add_back_capacity();
+  _LIBCPP_HIDE_FROM_ABI void __add_back_capacity(size_type __n);
+  _LIBCPP_HIDE_FROM_ABI iterator __move_and_check(iterator __f, iterator __l, iterator __r, const_pointer& __vt);
+  _LIBCPP_HIDE_FROM_ABI iterator
+  __move_backward_and_check(iterator __f, iterator __l, iterator __r, const_pointer& __vt);
+  _LIBCPP_HIDE_FROM_ABI void __move_construct_and_check(iterator __f, iterator __l, iterator __r, const_pointer& __vt);
+  _LIBCPP_HIDE_FROM_ABI void
+  __move_construct_backward_and_check(iterator __f, iterator __l, iterator __r, const_pointer& __vt);
+
+  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const deque& __c) {
+    __copy_assign_alloc(__c, integral_constant<bool, __alloc_traits::propagate_on_container_copy_assignment::value>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const deque& __c, true_type) {
+    if (__alloc() != __c.__alloc()) {
+      clear();
+      shrink_to_fit();
+    }
+    __alloc()        = __c.__alloc();
+    __map_.__alloc() = __c.__map_.__alloc();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const deque&, false_type) {}
+
+  _LIBCPP_HIDE_FROM_ABI void __move_assign(deque& __c, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value);
+  _LIBCPP_HIDE_FROM_ABI void __move_assign(deque& __c, false_type);
+};
+
+template <class _Tp, class _Alloc>
+_LIBCPP_CONSTEXPR const typename allocator_traits<_Alloc>::difference_type deque<_Tp, _Alloc>::__block_size =
+    __deque_block_size<value_type, difference_type>::value;
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator,
+          class _Alloc = allocator<__iter_value_type<_InputIterator>>,
+          class        = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class        = enable_if_t<__is_allocator<_Alloc>::value> >
+deque(_InputIterator, _InputIterator) -> deque<__iter_value_type<_InputIterator>, _Alloc>;
+
+template <class _InputIterator,
+          class _Alloc,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<__is_allocator<_Alloc>::value> >
+deque(_InputIterator, _InputIterator, _Alloc) -> deque<__iter_value_type<_InputIterator>, _Alloc>;
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range,
+          class _Alloc = allocator<ranges::range_value_t<_Range>>,
+          class        = enable_if_t<__is_allocator<_Alloc>::value> >
+deque(from_range_t, _Range&&, _Alloc = _Alloc()) -> deque<ranges::range_value_t<_Range>, _Alloc>;
+#endif
+
+template <class _Tp, class _Allocator>
+deque<_Tp, _Allocator>::deque(size_type __n) : __start_(0), __size_(0, __default_init_tag()) {
+  __annotate_new(0);
+  if (__n > 0)
+    __append(__n);
+}
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp, class _Allocator>
+deque<_Tp, _Allocator>::deque(size_type __n, const _Allocator& __a)
+    : __map_(__pointer_allocator(__a)), __start_(0), __size_(0, __a) {
+  __annotate_new(0);
+  if (__n > 0)
+    __append(__n);
+}
+#endif
+
+template <class _Tp, class _Allocator>
+deque<_Tp, _Allocator>::deque(size_type __n, const value_type& __v) : __start_(0), __size_(0, __default_init_tag()) {
+  __annotate_new(0);
+  if (__n > 0)
+    __append(__n, __v);
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> >
+deque<_Tp, _Allocator>::deque(_InputIter __f, _InputIter __l) : __start_(0), __size_(0, __default_init_tag()) {
+  __annotate_new(0);
+  __append(__f, __l);
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> >
+deque<_Tp, _Allocator>::deque(_InputIter __f, _InputIter __l, const allocator_type& __a)
+    : __map_(__pointer_allocator(__a)), __start_(0), __size_(0, __a) {
+  __annotate_new(0);
+  __append(__f, __l);
+}
+
+template <class _Tp, class _Allocator>
+deque<_Tp, _Allocator>::deque(const deque& __c)
+    : __map_(__pointer_allocator(__alloc_traits::select_on_container_copy_construction(__c.__alloc()))),
+      __start_(0),
+      __size_(0, __map_.__alloc()) {
+  __annotate_new(0);
+  __append(__c.begin(), __c.end());
+}
+
+template <class _Tp, class _Allocator>
+deque<_Tp, _Allocator>::deque(const deque& __c, const __type_identity_t<allocator_type>& __a)
+    : __map_(__pointer_allocator(__a)), __start_(0), __size_(0, __a) {
+  __annotate_new(0);
+  __append(__c.begin(), __c.end());
+}
+
+template <class _Tp, class _Allocator>
+deque<_Tp, _Allocator>& deque<_Tp, _Allocator>::operator=(const deque& __c) {
+  if (this != std::addressof(__c)) {
+    __copy_assign_alloc(__c);
+    assign(__c.begin(), __c.end());
+  }
+  return *this;
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Allocator>
+deque<_Tp, _Allocator>::deque(initializer_list<value_type> __il) : __start_(0), __size_(0, __default_init_tag()) {
+  __annotate_new(0);
+  __append(__il.begin(), __il.end());
+}
+
+template <class _Tp, class _Allocator>
+deque<_Tp, _Allocator>::deque(initializer_list<value_type> __il, const allocator_type& __a)
+    : __map_(__pointer_allocator(__a)), __start_(0), __size_(0, __a) {
+  __annotate_new(0);
+  __append(__il.begin(), __il.end());
+}
+
+template <class _Tp, class _Allocator>
+inline deque<_Tp, _Allocator>::deque(deque&& __c) noexcept(is_nothrow_move_constructible<allocator_type>::value)
+    : __map_(std::move(__c.__map_)), __start_(std::move(__c.__start_)), __size_(std::move(__c.__size_)) {
+  __c.__start_ = 0;
+  __c.__size() = 0;
+}
+
+template <class _Tp, class _Allocator>
+inline deque<_Tp, _Allocator>::deque(deque&& __c, const __type_identity_t<allocator_type>& __a)
+    : __map_(std::move(__c.__map_), __pointer_allocator(__a)),
+      __start_(std::move(__c.__start_)),
+      __size_(std::move(__c.__size()), __a) {
+  if (__a == __c.__alloc()) {
+    __c.__start_ = 0;
+    __c.__size() = 0;
+  } else {
+    __map_.clear();
+    __start_ = 0;
+    __size() = 0;
+    typedef move_iterator<iterator> _Ip;
+    assign(_Ip(__c.begin()), _Ip(__c.end()));
+  }
+}
+
+template <class _Tp, class _Allocator>
+inline deque<_Tp, _Allocator>& deque<_Tp, _Allocator>::operator=(deque&& __c) noexcept(
+    __alloc_traits::propagate_on_container_move_assignment::value &&
+    is_nothrow_move_assignable<allocator_type>::value) {
+  __move_assign(__c, integral_constant<bool, __alloc_traits::propagate_on_container_move_assignment::value>());
+  return *this;
+}
+
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::__move_assign(deque& __c, false_type) {
+  if (__alloc() != __c.__alloc()) {
+    typedef move_iterator<iterator> _Ip;
+    assign(_Ip(__c.begin()), _Ip(__c.end()));
+  } else
+    __move_assign(__c, true_type());
+}
+
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::__move_assign(deque& __c,
+                                           true_type) noexcept(is_nothrow_move_assignable<allocator_type>::value) {
+  clear();
+  shrink_to_fit();
+  __move_assign(__c);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Allocator>
+template <class _InputIter,
+          __enable_if_t<__has_input_iterator_category<_InputIter>::value &&
+                            !__has_random_access_iterator_category<_InputIter>::value,
+                        int> >
+void deque<_Tp, _Allocator>::assign(_InputIter __f, _InputIter __l) {
+  __assign_with_sentinel(__f, __l);
+}
+
+template <class _Tp, class _Allocator>
+template <class _Iterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI void deque<_Tp, _Allocator>::__assign_with_sentinel(_Iterator __f, _Sentinel __l) {
+  iterator __i = begin();
+  iterator __e = end();
+  for (; __f != __l && __i != __e; ++__f, (void)++__i)
+    *__i = *__f;
+  if (__f != __l)
+    __append_with_sentinel(std::move(__f), std::move(__l));
+  else
+    __erase_to_end(__i);
+}
+
+template <class _Tp, class _Allocator>
+template <class _RAIter, __enable_if_t<__has_random_access_iterator_category<_RAIter>::value, int> >
+void deque<_Tp, _Allocator>::assign(_RAIter __f, _RAIter __l) {
+  __assign_with_size_random_access(__f, __l - __f);
+}
+
+template <class _Tp, class _Allocator>
+template <class _RandomAccessIterator>
+_LIBCPP_HIDE_FROM_ABI void
+deque<_Tp, _Allocator>::__assign_with_size_random_access(_RandomAccessIterator __f, difference_type __n) {
+  if (static_cast<size_type>(__n) > size()) {
+    auto __l = __f + size();
+    std::copy(__f, __l, begin());
+    __append_with_size(__l, __n - size());
+  } else
+    __erase_to_end(std::copy_n(__f, __n, begin()));
+}
+
+template <class _Tp, class _Allocator>
+template <class _Iterator>
+_LIBCPP_HIDE_FROM_ABI void deque<_Tp, _Allocator>::__assign_with_size(_Iterator __f, difference_type __n) {
+  if (static_cast<size_type>(__n) > size()) {
+    auto __added_size = __n - size();
+
+    auto __i = begin();
+    for (auto __count = size(); __count != 0; --__count) {
+      *__i++ = *__f++;
+    }
+
+    __append_with_size(__f, __added_size);
+
+  } else {
+    __erase_to_end(std::copy_n(__f, __n, begin()));
+  }
+}
+
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::assign(size_type __n, const value_type& __v) {
+  if (__n > size()) {
+    std::fill_n(begin(), size(), __v);
+    __n -= size();
+    __append(__n, __v);
+  } else
+    __erase_to_end(std::fill_n(begin(), __n, __v));
+}
+
+template <class _Tp, class _Allocator>
+inline _Allocator deque<_Tp, _Allocator>::get_allocator() const _NOEXCEPT {
+  return __alloc();
+}
+
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::resize(size_type __n) {
+  if (__n > size())
+    __append(__n - size());
+  else if (__n < size())
+    __erase_to_end(begin() + __n);
+}
+
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::resize(size_type __n, const value_type& __v) {
+  if (__n > size())
+    __append(__n - size(), __v);
+  else if (__n < size())
+    __erase_to_end(begin() + __n);
+}
+
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::shrink_to_fit() _NOEXCEPT {
+  allocator_type& __a = __alloc();
+  if (empty()) {
+    __annotate_delete();
+    while (__map_.size() > 0) {
+      __alloc_traits::deallocate(__a, __map_.back(), __block_size);
+      __map_.pop_back();
+    }
+    __start_ = 0;
+  } else {
+    __maybe_remove_front_spare(/*__keep_one=*/false);
+    __maybe_remove_back_spare(/*__keep_one=*/false);
+  }
+  __map_.shrink_to_fit();
+}
+
+template <class _Tp, class _Allocator>
+inline typename deque<_Tp, _Allocator>::reference deque<_Tp, _Allocator>::operator[](size_type __i) _NOEXCEPT {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__i < size(), "deque::operator[] index out of bounds");
+  size_type __p = __start_ + __i;
+  return *(*(__map_.begin() + __p / __block_size) + __p % __block_size);
+}
+
+template <class _Tp, class _Allocator>
+inline typename deque<_Tp, _Allocator>::const_reference
+deque<_Tp, _Allocator>::operator[](size_type __i) const _NOEXCEPT {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__i < size(), "deque::operator[] index out of bounds");
+  size_type __p = __start_ + __i;
+  return *(*(__map_.begin() + __p / __block_size) + __p % __block_size);
+}
+
+template <class _Tp, class _Allocator>
+inline typename deque<_Tp, _Allocator>::reference deque<_Tp, _Allocator>::at(size_type __i) {
+  if (__i >= size())
+    std::__throw_out_of_range("deque");
+  size_type __p = __start_ + __i;
+  return *(*(__map_.begin() + __p / __block_size) + __p % __block_size);
+}
+
+template <class _Tp, class _Allocator>
+inline typename deque<_Tp, _Allocator>::const_reference deque<_Tp, _Allocator>::at(size_type __i) const {
+  if (__i >= size())
+    std::__throw_out_of_range("deque");
+  size_type __p = __start_ + __i;
+  return *(*(__map_.begin() + __p / __block_size) + __p % __block_size);
+}
+
+template <class _Tp, class _Allocator>
+inline typename deque<_Tp, _Allocator>::reference deque<_Tp, _Allocator>::front() _NOEXCEPT {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "deque::front called on an empty deque");
+  return *(*(__map_.begin() + __start_ / __block_size) + __start_ % __block_size);
+}
+
+template <class _Tp, class _Allocator>
+inline typename deque<_Tp, _Allocator>::const_reference deque<_Tp, _Allocator>::front() const _NOEXCEPT {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "deque::front called on an empty deque");
+  return *(*(__map_.begin() + __start_ / __block_size) + __start_ % __block_size);
+}
+
+template <class _Tp, class _Allocator>
+inline typename deque<_Tp, _Allocator>::reference deque<_Tp, _Allocator>::back() _NOEXCEPT {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "deque::back called on an empty deque");
+  size_type __p = size() + __start_ - 1;
+  return *(*(__map_.begin() + __p / __block_size) + __p % __block_size);
+}
+
+template <class _Tp, class _Allocator>
+inline typename deque<_Tp, _Allocator>::const_reference deque<_Tp, _Allocator>::back() const _NOEXCEPT {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "deque::back called on an empty deque");
+  size_type __p = size() + __start_ - 1;
+  return *(*(__map_.begin() + __p / __block_size) + __p % __block_size);
+}
+
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::push_back(const value_type& __v) {
+  allocator_type& __a = __alloc();
+  if (__back_spare() == 0)
+    __add_back_capacity();
+  // __back_spare() >= 1
+  __annotate_increase_back(1);
+  __alloc_traits::construct(__a, std::addressof(*end()), __v);
+  ++__size();
+}
+
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::push_front(const value_type& __v) {
+  allocator_type& __a = __alloc();
+  if (__front_spare() == 0)
+    __add_front_capacity();
+  // __front_spare() >= 1
+  __annotate_increase_front(1);
+  __alloc_traits::construct(__a, std::addressof(*--begin()), __v);
+  --__start_;
+  ++__size();
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::push_back(value_type&& __v) {
+  allocator_type& __a = __alloc();
+  if (__back_spare() == 0)
+    __add_back_capacity();
+  // __back_spare() >= 1
+  __annotate_increase_back(1);
+  __alloc_traits::construct(__a, std::addressof(*end()), std::move(__v));
+  ++__size();
+}
+
+template <class _Tp, class _Allocator>
+template <class... _Args>
+#  if _LIBCPP_STD_VER >= 17
+typename deque<_Tp, _Allocator>::reference
+#  else
+void
+#  endif
+deque<_Tp, _Allocator>::emplace_back(_Args&&... __args) {
+  allocator_type& __a = __alloc();
+  if (__back_spare() == 0)
+    __add_back_capacity();
+  // __back_spare() >= 1
+  __annotate_increase_back(1);
+  __alloc_traits::construct(__a, std::addressof(*end()), std::forward<_Args>(__args)...);
+  ++__size();
+#  if _LIBCPP_STD_VER >= 17
+  return *--end();
+#  endif
+}
+
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::push_front(value_type&& __v) {
+  allocator_type& __a = __alloc();
+  if (__front_spare() == 0)
+    __add_front_capacity();
+  // __front_spare() >= 1
+  __annotate_increase_front(1);
+  __alloc_traits::construct(__a, std::addressof(*--begin()), std::move(__v));
+  --__start_;
+  ++__size();
+}
+
+template <class _Tp, class _Allocator>
+template <class... _Args>
+#  if _LIBCPP_STD_VER >= 17
+typename deque<_Tp, _Allocator>::reference
+#  else
+void
+#  endif
+deque<_Tp, _Allocator>::emplace_front(_Args&&... __args) {
+  allocator_type& __a = __alloc();
+  if (__front_spare() == 0)
+    __add_front_capacity();
+  // __front_spare() >= 1
+  __annotate_increase_front(1);
+  __alloc_traits::construct(__a, std::addressof(*--begin()), std::forward<_Args>(__args)...);
+  --__start_;
+  ++__size();
+#  if _LIBCPP_STD_VER >= 17
+  return *begin();
+#  endif
+}
+
+template <class _Tp, class _Allocator>
+typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::insert(const_iterator __p, value_type&& __v) {
+  size_type __pos     = __p - begin();
+  size_type __to_end  = size() - __pos;
+  allocator_type& __a = __alloc();
+  if (__pos < __to_end) { // insert by shifting things backward
+    if (__front_spare() == 0)
+      __add_front_capacity();
+    // __front_spare() >= 1
+    __annotate_increase_front(1);
+    if (__pos == 0) {
+      __alloc_traits::construct(__a, std::addressof(*--begin()), std::move(__v));
+      --__start_;
+      ++__size();
+    } else {
+      iterator __b   = begin();
+      iterator __bm1 = std::prev(__b);
+      __alloc_traits::construct(__a, std::addressof(*__bm1), std::move(*__b));
+      --__start_;
+      ++__size();
+      if (__pos > 1)
+        __b = std::move(std::next(__b), __b + __pos, __b);
+      *__b = std::move(__v);
+    }
+  } else { // insert by shifting things forward
+    if (__back_spare() == 0)
+      __add_back_capacity();
+    // __back_capacity >= 1
+    __annotate_increase_back(1);
+    size_type __de = size() - __pos;
+    if (__de == 0) {
+      __alloc_traits::construct(__a, std::addressof(*end()), std::move(__v));
+      ++__size();
+    } else {
+      iterator __e   = end();
+      iterator __em1 = std::prev(__e);
+      __alloc_traits::construct(__a, std::addressof(*__e), std::move(*__em1));
+      ++__size();
+      if (__de > 1)
+        __e = std::move_backward(__e - __de, __em1, __e);
+      *--__e = std::move(__v);
+    }
+  }
+  return begin() + __pos;
+}
+
+template <class _Tp, class _Allocator>
+template <class... _Args>
+typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::emplace(const_iterator __p, _Args&&... __args) {
+  size_type __pos     = __p - begin();
+  size_type __to_end  = size() - __pos;
+  allocator_type& __a = __alloc();
+  if (__pos < __to_end) { // insert by shifting things backward
+    if (__front_spare() == 0)
+      __add_front_capacity();
+    // __front_spare() >= 1
+    __annotate_increase_front(1);
+    if (__pos == 0) {
+      __alloc_traits::construct(__a, std::addressof(*--begin()), std::forward<_Args>(__args)...);
+      --__start_;
+      ++__size();
+    } else {
+      __temp_value<value_type, _Allocator> __tmp(__alloc(), std::forward<_Args>(__args)...);
+      iterator __b   = begin();
+      iterator __bm1 = std::prev(__b);
+      __alloc_traits::construct(__a, std::addressof(*__bm1), std::move(*__b));
+      --__start_;
+      ++__size();
+      if (__pos > 1)
+        __b = std::move(std::next(__b), __b + __pos, __b);
+      *__b = std::move(__tmp.get());
+    }
+  } else { // insert by shifting things forward
+    if (__back_spare() == 0)
+      __add_back_capacity();
+    // __back_capacity >= 1
+    __annotate_increase_back(1);
+    size_type __de = size() - __pos;
+    if (__de == 0) {
+      __alloc_traits::construct(__a, std::addressof(*end()), std::forward<_Args>(__args)...);
+      ++__size();
+    } else {
+      __temp_value<value_type, _Allocator> __tmp(__alloc(), std::forward<_Args>(__args)...);
+      iterator __e   = end();
+      iterator __em1 = std::prev(__e);
+      __alloc_traits::construct(__a, std::addressof(*__e), std::move(*__em1));
+      ++__size();
+      if (__de > 1)
+        __e = std::move_backward(__e - __de, __em1, __e);
+      *--__e = std::move(__tmp.get());
+    }
+  }
+  return begin() + __pos;
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Allocator>
+typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::insert(const_iterator __p, const value_type& __v) {
+  size_type __pos     = __p - begin();
+  size_type __to_end  = size() - __pos;
+  allocator_type& __a = __alloc();
+  if (__pos < __to_end) { // insert by shifting things backward
+    if (__front_spare() == 0)
+      __add_front_capacity();
+    // __front_spare() >= 1
+    __annotate_increase_front(1);
+    if (__pos == 0) {
+      __alloc_traits::construct(__a, std::addressof(*--begin()), __v);
+      --__start_;
+      ++__size();
+    } else {
+      const_pointer __vt = pointer_traits<const_pointer>::pointer_to(__v);
+      iterator __b       = begin();
+      iterator __bm1     = std::prev(__b);
+      if (__vt == pointer_traits<const_pointer>::pointer_to(*__b))
+        __vt = pointer_traits<const_pointer>::pointer_to(*__bm1);
+      __alloc_traits::construct(__a, std::addressof(*__bm1), std::move(*__b));
+      --__start_;
+      ++__size();
+      if (__pos > 1)
+        __b = __move_and_check(std::next(__b), __b + __pos, __b, __vt);
+      *__b = *__vt;
+    }
+  } else { // insert by shifting things forward
+    if (__back_spare() == 0)
+      __add_back_capacity();
+    // __back_capacity >= 1
+    __annotate_increase_back(1);
+    size_type __de = size() - __pos;
+    if (__de == 0) {
+      __alloc_traits::construct(__a, std::addressof(*end()), __v);
+      ++__size();
+    } else {
+      const_pointer __vt = pointer_traits<const_pointer>::pointer_to(__v);
+      iterator __e       = end();
+      iterator __em1     = std::prev(__e);
+      if (__vt == pointer_traits<const_pointer>::pointer_to(*__em1))
+        __vt = pointer_traits<const_pointer>::pointer_to(*__e);
+      __alloc_traits::construct(__a, std::addressof(*__e), std::move(*__em1));
+      ++__size();
+      if (__de > 1)
+        __e = __move_backward_and_check(__e - __de, __em1, __e, __vt);
+      *--__e = *__vt;
+    }
+  }
+  return begin() + __pos;
+}
+
+template <class _Tp, class _Allocator>
+typename deque<_Tp, _Allocator>::iterator
+deque<_Tp, _Allocator>::insert(const_iterator __p, size_type __n, const value_type& __v) {
+  size_type __pos     = __p - begin();
+  size_type __to_end  = __size() - __pos;
+  allocator_type& __a = __alloc();
+  if (__pos < __to_end) { // insert by shifting things backward
+    if (__n > __front_spare())
+      __add_front_capacity(__n - __front_spare());
+    // __n <= __front_spare()
+    __annotate_increase_front(__n);
+    iterator __old_begin = begin();
+    iterator __i         = __old_begin;
+    if (__n > __pos) {
+      for (size_type __m = __n - __pos; __m; --__m, --__start_, ++__size())
+        __alloc_traits::construct(__a, std::addressof(*--__i), __v);
+      __n = __pos;
+    }
+    if (__n > 0) {
+      const_pointer __vt = pointer_traits<const_pointer>::pointer_to(__v);
+      iterator __obn     = __old_begin + __n;
+      __move_construct_backward_and_check(__old_begin, __obn, __i, __vt);
+      if (__n < __pos)
+        __old_begin = __move_and_check(__obn, __old_begin + __pos, __old_begin, __vt);
+      std::fill_n(__old_begin, __n, *__vt);
+    }
+  } else { // insert by shifting things forward
+    size_type __back_capacity = __back_spare();
+    if (__n > __back_capacity)
+      __add_back_capacity(__n - __back_capacity);
+    // __n <= __back_capacity
+    __annotate_increase_back(__n);
+    iterator __old_end = end();
+    iterator __i       = __old_end;
+    size_type __de     = size() - __pos;
+    if (__n > __de) {
+      for (size_type __m = __n - __de; __m; --__m, (void)++__i, ++__size())
+        __alloc_traits::construct(__a, std::addressof(*__i), __v);
+      __n = __de;
+    }
+    if (__n > 0) {
+      const_pointer __vt = pointer_traits<const_pointer>::pointer_to(__v);
+      iterator __oen     = __old_end - __n;
+      __move_construct_and_check(__oen, __old_end, __i, __vt);
+      if (__n < __de)
+        __old_end = __move_backward_and_check(__old_end - __de, __oen, __old_end, __vt);
+      std::fill_n(__old_end - __n, __n, *__vt);
+    }
+  }
+  return begin() + __pos;
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIter, __enable_if_t<__has_exactly_input_iterator_category<_InputIter>::value, int> >
+typename deque<_Tp, _Allocator>::iterator
+deque<_Tp, _Allocator>::insert(const_iterator __p, _InputIter __f, _InputIter __l) {
+  return __insert_with_sentinel(__p, __f, __l);
+}
+
+template <class _Tp, class _Allocator>
+template <class _Iterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI typename deque<_Tp, _Allocator>::iterator
+deque<_Tp, _Allocator>::__insert_with_sentinel(const_iterator __p, _Iterator __f, _Sentinel __l) {
+  __split_buffer<value_type, allocator_type&> __buf(__alloc());
+  __buf.__construct_at_end_with_sentinel(std::move(__f), std::move(__l));
+  typedef typename __split_buffer<value_type, allocator_type&>::iterator __bi;
+  return insert(__p, move_iterator<__bi>(__buf.begin()), move_iterator<__bi>(__buf.end()));
+}
+
+template <class _Tp, class _Allocator>
+template <class _ForwardIterator, __enable_if_t<__has_exactly_forward_iterator_category<_ForwardIterator>::value, int> >
+typename deque<_Tp, _Allocator>::iterator
+deque<_Tp, _Allocator>::insert(const_iterator __p, _ForwardIterator __f, _ForwardIterator __l) {
+  return __insert_with_size(__p, __f, std::distance(__f, __l));
+}
+
+template <class _Tp, class _Allocator>
+template <class _Iterator>
+_LIBCPP_HIDE_FROM_ABI typename deque<_Tp, _Allocator>::iterator
+deque<_Tp, _Allocator>::__insert_with_size(const_iterator __p, _Iterator __f, size_type __n) {
+  __split_buffer<value_type, allocator_type&> __buf(__n, 0, __alloc());
+  __buf.__construct_at_end_with_size(__f, __n);
+  typedef typename __split_buffer<value_type, allocator_type&>::iterator __fwd;
+  return insert(__p, move_iterator<__fwd>(__buf.begin()), move_iterator<__fwd>(__buf.end()));
+}
+
+template <class _Tp, class _Allocator>
+template <class _BiIter, __enable_if_t<__has_bidirectional_iterator_category<_BiIter>::value, int> >
+typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::insert(const_iterator __p, _BiIter __f, _BiIter __l) {
+  return __insert_bidirectional(__p, __f, __l, std::distance(__f, __l));
+}
+
+template <class _Tp, class _Allocator>
+template <class _BiIter, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI typename deque<_Tp, _Allocator>::iterator
+deque<_Tp, _Allocator>::__insert_bidirectional(const_iterator __p, _BiIter __f, _Sentinel, size_type __n) {
+  return __insert_bidirectional(__p, __f, std::next(__f, __n), __n);
+}
+
+template <class _Tp, class _Allocator>
+template <class _BiIter>
+_LIBCPP_HIDE_FROM_ABI typename deque<_Tp, _Allocator>::iterator
+deque<_Tp, _Allocator>::__insert_bidirectional(const_iterator __p, _BiIter __f, _BiIter __l, size_type __n) {
+  size_type __pos     = __p - begin();
+  size_type __to_end  = size() - __pos;
+  allocator_type& __a = __alloc();
+  if (__pos < __to_end) { // insert by shifting things backward
+    if (__n > __front_spare())
+      __add_front_capacity(__n - __front_spare());
+    // __n <= __front_spare()
+    __annotate_increase_front(__n);
+    iterator __old_begin = begin();
+    iterator __i         = __old_begin;
+    _BiIter __m          = __f;
+    if (__n > __pos) {
+      __m = __pos < __n / 2 ? std::prev(__l, __pos) : std::next(__f, __n - __pos);
+      for (_BiIter __j = __m; __j != __f; --__start_, ++__size())
+        __alloc_traits::construct(__a, std::addressof(*--__i), *--__j);
+      __n = __pos;
+    }
+    if (__n > 0) {
+      iterator __obn = __old_begin + __n;
+      for (iterator __j = __obn; __j != __old_begin;) {
+        __alloc_traits::construct(__a, std::addressof(*--__i), std::move(*--__j));
+        --__start_;
+        ++__size();
+      }
+      if (__n < __pos)
+        __old_begin = std::move(__obn, __old_begin + __pos, __old_begin);
+      std::copy(__m, __l, __old_begin);
+    }
+  } else { // insert by shifting things forward
+    size_type __back_capacity = __back_spare();
+    if (__n > __back_capacity)
+      __add_back_capacity(__n - __back_capacity);
+    // __n <= __back_capacity
+    __annotate_increase_back(__n);
+    iterator __old_end = end();
+    iterator __i       = __old_end;
+    _BiIter __m        = __l;
+    size_type __de     = size() - __pos;
+    if (__n > __de) {
+      __m = __de < __n / 2 ? std::next(__f, __de) : std::prev(__l, __n - __de);
+      for (_BiIter __j = __m; __j != __l; ++__i, (void)++__j, ++__size())
+        __alloc_traits::construct(__a, std::addressof(*__i), *__j);
+      __n = __de;
+    }
+    if (__n > 0) {
+      iterator __oen = __old_end - __n;
+      for (iterator __j = __oen; __j != __old_end; ++__i, (void)++__j, ++__size())
+        __alloc_traits::construct(__a, std::addressof(*__i), std::move(*__j));
+      if (__n < __de)
+        __old_end = std::move_backward(__old_end - __de, __oen, __old_end);
+      std::copy_backward(__f, __m, __old_end);
+    }
+  }
+  return begin() + __pos;
+}
+
+template <class _Tp, class _Allocator>
+template <class _InpIter, __enable_if_t<__has_exactly_input_iterator_category<_InpIter>::value, int> >
+void deque<_Tp, _Allocator>::__append(_InpIter __f, _InpIter __l) {
+  __append_with_sentinel(__f, __l);
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI void deque<_Tp, _Allocator>::__append_with_sentinel(_InputIterator __f, _Sentinel __l) {
+  for (; __f != __l; ++__f)
+#ifdef _LIBCPP_CXX03_LANG
+    push_back(*__f);
+#else
+    emplace_back(*__f);
+#endif
+}
+
+template <class _Tp, class _Allocator>
+template <class _ForIter, __enable_if_t<__has_forward_iterator_category<_ForIter>::value, int> >
+void deque<_Tp, _Allocator>::__append(_ForIter __f, _ForIter __l) {
+  __append_with_size(__f, std::distance(__f, __l));
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIterator>
+_LIBCPP_HIDE_FROM_ABI void deque<_Tp, _Allocator>::__append_with_size(_InputIterator __f, size_type __n) {
+  allocator_type& __a       = __alloc();
+  size_type __back_capacity = __back_spare();
+  if (__n > __back_capacity)
+    __add_back_capacity(__n - __back_capacity);
+
+  // __n <= __back_capacity
+  __annotate_increase_back(__n);
+  for (__deque_block_range __br : __deque_range(end(), end() + __n)) {
+    _ConstructTransaction __tx(this, __br);
+    for (; __tx.__pos_ != __tx.__end_; ++__tx.__pos_, (void)++__f) {
+      __alloc_traits::construct(__a, std::__to_address(__tx.__pos_), *__f);
+    }
+  }
+}
+
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::__append(size_type __n) {
+  allocator_type& __a       = __alloc();
+  size_type __back_capacity = __back_spare();
+  if (__n > __back_capacity)
+    __add_back_capacity(__n - __back_capacity);
+  // __n <= __back_capacity
+  __annotate_increase_back(__n);
+  for (__deque_block_range __br : __deque_range(end(), end() + __n)) {
+    _ConstructTransaction __tx(this, __br);
+    for (; __tx.__pos_ != __tx.__end_; ++__tx.__pos_) {
+      __alloc_traits::construct(__a, std::__to_address(__tx.__pos_));
+    }
+  }
+}
+
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::__append(size_type __n, const value_type& __v) {
+  allocator_type& __a       = __alloc();
+  size_type __back_capacity = __back_spare();
+  if (__n > __back_capacity)
+    __add_back_capacity(__n - __back_capacity);
+  // __n <= __back_capacity
+  __annotate_increase_back(__n);
+  for (__deque_block_range __br : __deque_range(end(), end() + __n)) {
+    _ConstructTransaction __tx(this, __br);
+    for (; __tx.__pos_ != __tx.__end_; ++__tx.__pos_) {
+      __alloc_traits::construct(__a, std::__to_address(__tx.__pos_), __v);
+    }
+  }
+}
+
+// Create front capacity for one block of elements.
+// Strong guarantee.  Either do it or don't touch anything.
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::__add_front_capacity() {
+  allocator_type& __a = __alloc();
+  if (__back_spare() >= __block_size) {
+    __start_ += __block_size;
+    pointer __pt = __map_.back();
+    __map_.pop_back();
+    __map_.push_front(__pt);
+  }
+  // Else if __map_.size() < __map_.capacity() then we need to allocate 1 buffer
+  else if (__map_.size() < __map_.capacity()) { // we can put the new buffer into the map, but don't shift things around
+    // until all buffers are allocated.  If we throw, we don't need to fix
+    // anything up (any added buffers are undetectible)
+    if (__map_.__front_spare() > 0)
+      __map_.push_front(__alloc_traits::allocate(__a, __block_size));
+    else {
+      __map_.push_back(__alloc_traits::allocate(__a, __block_size));
+      // Done allocating, reorder capacity
+      pointer __pt = __map_.back();
+      __map_.pop_back();
+      __map_.push_front(__pt);
+    }
+    __start_ = __map_.size() == 1 ? __block_size / 2 : __start_ + __block_size;
+  }
+  // Else need to allocate 1 buffer, *and* we need to reallocate __map_.
+  else {
+    __split_buffer<pointer, __pointer_allocator&> __buf(
+        std::max<size_type>(2 * __map_.capacity(), 1), 0, __map_.__alloc());
+
+    typedef __allocator_destructor<_Allocator> _Dp;
+    unique_ptr<pointer, _Dp> __hold(__alloc_traits::allocate(__a, __block_size), _Dp(__a, __block_size));
+    __buf.push_back(__hold.get());
+    __hold.release();
+
+    for (__map_pointer __i = __map_.begin(); __i != __map_.end(); ++__i)
+      __buf.push_back(*__i);
+    std::swap(__map_.__first_, __buf.__first_);
+    std::swap(__map_.__begin_, __buf.__begin_);
+    std::swap(__map_.__end_, __buf.__end_);
+    std::swap(__map_.__end_cap(), __buf.__end_cap());
+    __start_ = __map_.size() == 1 ? __block_size / 2 : __start_ + __block_size;
+  }
+  __annotate_whole_block(0, __asan_poison);
+}
+
+// Create front capacity for __n elements.
+// Strong guarantee.  Either do it or don't touch anything.
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::__add_front_capacity(size_type __n) {
+  allocator_type& __a = __alloc();
+  size_type __nb      = __recommend_blocks(__n + __map_.empty());
+  // Number of unused blocks at back:
+  size_type __back_capacity = __back_spare() / __block_size;
+  __back_capacity           = std::min(__back_capacity, __nb); // don't take more than you need
+  __nb -= __back_capacity;                                     // number of blocks need to allocate
+  // If __nb == 0, then we have sufficient capacity.
+  if (__nb == 0) {
+    __start_ += __block_size * __back_capacity;
+    for (; __back_capacity > 0; --__back_capacity) {
+      pointer __pt = __map_.back();
+      __map_.pop_back();
+      __map_.push_front(__pt);
+    }
+  }
+  // Else if __nb <= __map_.capacity() - __map_.size() then we need to allocate __nb buffers
+  else if (__nb <= __map_.capacity() -
+                       __map_.size()) { // we can put the new buffers into the map, but don't shift things around
+    // until all buffers are allocated.  If we throw, we don't need to fix
+    // anything up (any added buffers are undetectible)
+    for (; __nb > 0; --__nb, __start_ += __block_size - (__map_.size() == 1)) {
+      if (__map_.__front_spare() == 0)
+        break;
+      __map_.push_front(__alloc_traits::allocate(__a, __block_size));
+      __annotate_whole_block(0, __asan_poison);
+    }
+    for (; __nb > 0; --__nb, ++__back_capacity)
+      __map_.push_back(__alloc_traits::allocate(__a, __block_size));
+    // Done allocating, reorder capacity
+    __start_ += __back_capacity * __block_size;
+    for (; __back_capacity > 0; --__back_capacity) {
+      pointer __pt = __map_.back();
+      __map_.pop_back();
+      __map_.push_front(__pt);
+      __annotate_whole_block(0, __asan_poison);
+    }
+  }
+  // Else need to allocate __nb buffers, *and* we need to reallocate __map_.
+  else {
+    size_type __ds = (__nb + __back_capacity) * __block_size - __map_.empty();
+    __split_buffer<pointer, __pointer_allocator&> __buf(
+        std::max<size_type>(2 * __map_.capacity(), __nb + __map_.size()), 0, __map_.__alloc());
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      for (; __nb > 0; --__nb) {
+        __buf.push_back(__alloc_traits::allocate(__a, __block_size));
+        // ASan: this is empty container, we have to poison whole block
+        __annotate_poison_block(std::__to_address(__buf.back()), std::__to_address(__buf.back() + __block_size));
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __annotate_delete();
+      for (__map_pointer __i = __buf.begin(); __i != __buf.end(); ++__i)
+        __alloc_traits::deallocate(__a, *__i, __block_size);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    for (; __back_capacity > 0; --__back_capacity) {
+      __buf.push_back(__map_.back());
+      __map_.pop_back();
+    }
+    for (__map_pointer __i = __map_.begin(); __i != __map_.end(); ++__i)
+      __buf.push_back(*__i);
+    std::swap(__map_.__first_, __buf.__first_);
+    std::swap(__map_.__begin_, __buf.__begin_);
+    std::swap(__map_.__end_, __buf.__end_);
+    std::swap(__map_.__end_cap(), __buf.__end_cap());
+    __start_ += __ds;
+  }
+}
+
+// Create back capacity for one block of elements.
+// Strong guarantee.  Either do it or don't touch anything.
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::__add_back_capacity() {
+  allocator_type& __a = __alloc();
+  if (__front_spare() >= __block_size) {
+    __start_ -= __block_size;
+    pointer __pt = __map_.front();
+    __map_.pop_front();
+    __map_.push_back(__pt);
+  }
+  // Else if __nb <= __map_.capacity() - __map_.size() then we need to allocate __nb buffers
+  else if (__map_.size() < __map_.capacity()) { // we can put the new buffer into the map, but don't shift things around
+    // until it is allocated.  If we throw, we don't need to fix
+    // anything up (any added buffers are undetectible)
+    if (__map_.__back_spare() != 0)
+      __map_.push_back(__alloc_traits::allocate(__a, __block_size));
+    else {
+      __map_.push_front(__alloc_traits::allocate(__a, __block_size));
+      // Done allocating, reorder capacity
+      pointer __pt = __map_.front();
+      __map_.pop_front();
+      __map_.push_back(__pt);
+    }
+    __annotate_whole_block(__map_.size() - 1, __asan_poison);
+  }
+  // Else need to allocate 1 buffer, *and* we need to reallocate __map_.
+  else {
+    __split_buffer<pointer, __pointer_allocator&> __buf(
+        std::max<size_type>(2 * __map_.capacity(), 1), __map_.size(), __map_.__alloc());
+
+    typedef __allocator_destructor<_Allocator> _Dp;
+    unique_ptr<pointer, _Dp> __hold(__alloc_traits::allocate(__a, __block_size), _Dp(__a, __block_size));
+    __buf.push_back(__hold.get());
+    __hold.release();
+
+    for (__map_pointer __i = __map_.end(); __i != __map_.begin();)
+      __buf.push_front(*--__i);
+    std::swap(__map_.__first_, __buf.__first_);
+    std::swap(__map_.__begin_, __buf.__begin_);
+    std::swap(__map_.__end_, __buf.__end_);
+    std::swap(__map_.__end_cap(), __buf.__end_cap());
+    __annotate_whole_block(__map_.size() - 1, __asan_poison);
+  }
+}
+
+// Create back capacity for __n elements.
+// Strong guarantee.  Either do it or don't touch anything.
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::__add_back_capacity(size_type __n) {
+  allocator_type& __a = __alloc();
+  size_type __nb      = __recommend_blocks(__n + __map_.empty());
+  // Number of unused blocks at front:
+  size_type __front_capacity = __front_spare() / __block_size;
+  __front_capacity           = std::min(__front_capacity, __nb); // don't take more than you need
+  __nb -= __front_capacity;                                      // number of blocks need to allocate
+  // If __nb == 0, then we have sufficient capacity.
+  if (__nb == 0) {
+    __start_ -= __block_size * __front_capacity;
+    for (; __front_capacity > 0; --__front_capacity) {
+      pointer __pt = __map_.front();
+      __map_.pop_front();
+      __map_.push_back(__pt);
+    }
+  }
+  // Else if __nb <= __map_.capacity() - __map_.size() then we need to allocate __nb buffers
+  else if (__nb <= __map_.capacity() -
+                       __map_.size()) { // we can put the new buffers into the map, but don't shift things around
+    // until all buffers are allocated.  If we throw, we don't need to fix
+    // anything up (any added buffers are undetectible)
+    for (; __nb > 0; --__nb) {
+      if (__map_.__back_spare() == 0)
+        break;
+      __map_.push_back(__alloc_traits::allocate(__a, __block_size));
+      __annotate_whole_block(__map_.size() - 1, __asan_poison);
+    }
+    for (; __nb > 0; --__nb, ++__front_capacity, __start_ += __block_size - (__map_.size() == 1)) {
+      __map_.push_front(__alloc_traits::allocate(__a, __block_size));
+      __annotate_whole_block(0, __asan_poison);
+    }
+    // Done allocating, reorder capacity
+    __start_ -= __block_size * __front_capacity;
+    for (; __front_capacity > 0; --__front_capacity) {
+      pointer __pt = __map_.front();
+      __map_.pop_front();
+      __map_.push_back(__pt);
+    }
+  }
+  // Else need to allocate __nb buffers, *and* we need to reallocate __map_.
+  else {
+    size_type __ds = __front_capacity * __block_size;
+    __split_buffer<pointer, __pointer_allocator&> __buf(
+        std::max<size_type>(2 * __map_.capacity(), __nb + __map_.size()),
+        __map_.size() - __front_capacity,
+        __map_.__alloc());
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      for (; __nb > 0; --__nb) {
+        __buf.push_back(__alloc_traits::allocate(__a, __block_size));
+        // ASan: this is an empty container, we have to poison the whole block
+        __annotate_poison_block(std::__to_address(__buf.back()), std::__to_address(__buf.back() + __block_size));
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __annotate_delete();
+      for (__map_pointer __i = __buf.begin(); __i != __buf.end(); ++__i)
+        __alloc_traits::deallocate(__a, *__i, __block_size);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    for (; __front_capacity > 0; --__front_capacity) {
+      __buf.push_back(__map_.front());
+      __map_.pop_front();
+    }
+    for (__map_pointer __i = __map_.end(); __i != __map_.begin();)
+      __buf.push_front(*--__i);
+    std::swap(__map_.__first_, __buf.__first_);
+    std::swap(__map_.__begin_, __buf.__begin_);
+    std::swap(__map_.__end_, __buf.__end_);
+    std::swap(__map_.__end_cap(), __buf.__end_cap());
+    __start_ -= __ds;
+  }
+}
+
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::pop_front() {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "deque::pop_front called on an empty deque");
+  size_type __old_sz    = size();
+  size_type __old_start = __start_;
+  allocator_type& __a   = __alloc();
+  __alloc_traits::destroy(
+      __a, std::__to_address(*(__map_.begin() + __start_ / __block_size) + __start_ % __block_size));
+  --__size();
+  ++__start_;
+  __annotate_shrink_front(__old_sz, __old_start);
+  __maybe_remove_front_spare();
+}
+
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::pop_back() {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "deque::pop_back called on an empty deque");
+  size_type __old_sz    = size();
+  size_type __old_start = __start_;
+  allocator_type& __a   = __alloc();
+  size_type __p         = size() + __start_ - 1;
+  __alloc_traits::destroy(__a, std::__to_address(*(__map_.begin() + __p / __block_size) + __p % __block_size));
+  --__size();
+  __annotate_shrink_back(__old_sz, __old_start);
+  __maybe_remove_back_spare();
+}
+
+// move assign [__f, __l) to [__r, __r + (__l-__f)).
+// If __vt points into [__f, __l), then subtract (__f - __r) from __vt.
+template <class _Tp, class _Allocator>
+typename deque<_Tp, _Allocator>::iterator
+deque<_Tp, _Allocator>::__move_and_check(iterator __f, iterator __l, iterator __r, const_pointer& __vt) {
+  // as if
+  //   for (; __f != __l; ++__f, ++__r)
+  //       *__r = std::move(*__f);
+  difference_type __n = __l - __f;
+  while (__n > 0) {
+    pointer __fb         = __f.__ptr_;
+    pointer __fe         = *__f.__m_iter_ + __block_size;
+    difference_type __bs = __fe - __fb;
+    if (__bs > __n) {
+      __bs = __n;
+      __fe = __fb + __bs;
+    }
+    if (__fb <= __vt && __vt < __fe)
+      __vt = (const_iterator(static_cast<__map_const_pointer>(__f.__m_iter_), __vt) -= __f - __r).__ptr_;
+    __r = std::move(__fb, __fe, __r);
+    __n -= __bs;
+    __f += __bs;
+  }
+  return __r;
+}
+
+// move assign [__f, __l) to [__r - (__l-__f), __r) backwards.
+// If __vt points into [__f, __l), then add (__r - __l) to __vt.
+template <class _Tp, class _Allocator>
+typename deque<_Tp, _Allocator>::iterator
+deque<_Tp, _Allocator>::__move_backward_and_check(iterator __f, iterator __l, iterator __r, const_pointer& __vt) {
+  // as if
+  //   while (__f != __l)
+  //       *--__r = std::move(*--__l);
+  difference_type __n = __l - __f;
+  while (__n > 0) {
+    --__l;
+    pointer __lb         = *__l.__m_iter_;
+    pointer __le         = __l.__ptr_ + 1;
+    difference_type __bs = __le - __lb;
+    if (__bs > __n) {
+      __bs = __n;
+      __lb = __le - __bs;
+    }
+    if (__lb <= __vt && __vt < __le)
+      __vt = (const_iterator(static_cast<__map_const_pointer>(__l.__m_iter_), __vt) += __r - __l - 1).__ptr_;
+    __r = std::move_backward(__lb, __le, __r);
+    __n -= __bs;
+    __l -= __bs - 1;
+  }
+  return __r;
+}
+
+// move construct [__f, __l) to [__r, __r + (__l-__f)).
+// If __vt points into [__f, __l), then add (__r - __f) to __vt.
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::__move_construct_and_check(iterator __f, iterator __l, iterator __r, const_pointer& __vt) {
+  allocator_type& __a = __alloc();
+  // as if
+  //   for (; __f != __l; ++__r, ++__f, ++__size())
+  //       __alloc_traits::construct(__a, std::addressof(*__r), std::move(*__f));
+  difference_type __n = __l - __f;
+  while (__n > 0) {
+    pointer __fb         = __f.__ptr_;
+    pointer __fe         = *__f.__m_iter_ + __block_size;
+    difference_type __bs = __fe - __fb;
+    if (__bs > __n) {
+      __bs = __n;
+      __fe = __fb + __bs;
+    }
+    if (__fb <= __vt && __vt < __fe)
+      __vt = (const_iterator(static_cast<__map_const_pointer>(__f.__m_iter_), __vt) += __r - __f).__ptr_;
+    for (; __fb != __fe; ++__fb, ++__r, ++__size())
+      __alloc_traits::construct(__a, std::addressof(*__r), std::move(*__fb));
+    __n -= __bs;
+    __f += __bs;
+  }
+}
+
+// move construct [__f, __l) to [__r - (__l-__f), __r) backwards.
+// If __vt points into [__f, __l), then subtract (__l - __r) from __vt.
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::__move_construct_backward_and_check(
+    iterator __f, iterator __l, iterator __r, const_pointer& __vt) {
+  allocator_type& __a = __alloc();
+  // as if
+  //   for (iterator __j = __l; __j != __f;)
+  //   {
+  //       __alloc_traitsconstruct(__a, std::addressof(*--__r), std::move(*--__j));
+  //       --__start_;
+  //       ++__size();
+  //   }
+  difference_type __n = __l - __f;
+  while (__n > 0) {
+    --__l;
+    pointer __lb         = *__l.__m_iter_;
+    pointer __le         = __l.__ptr_ + 1;
+    difference_type __bs = __le - __lb;
+    if (__bs > __n) {
+      __bs = __n;
+      __lb = __le - __bs;
+    }
+    if (__lb <= __vt && __vt < __le)
+      __vt = (const_iterator(static_cast<__map_const_pointer>(__l.__m_iter_), __vt) -= __l - __r + 1).__ptr_;
+    while (__le != __lb) {
+      __alloc_traits::construct(__a, std::addressof(*--__r), std::move(*--__le));
+      --__start_;
+      ++__size();
+    }
+    __n -= __bs;
+    __l -= __bs - 1;
+  }
+}
+
+template <class _Tp, class _Allocator>
+typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::erase(const_iterator __f) {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+      __f != end(), "deque::erase(iterator) called with a non-dereferenceable iterator");
+  size_type __old_sz    = size();
+  size_type __old_start = __start_;
+  iterator __b          = begin();
+  difference_type __pos = __f - __b;
+  iterator __p          = __b + __pos;
+  allocator_type& __a   = __alloc();
+  if (static_cast<size_t>(__pos) <= (size() - 1) / 2) { // erase from front
+    std::move_backward(__b, __p, std::next(__p));
+    __alloc_traits::destroy(__a, std::addressof(*__b));
+    --__size();
+    ++__start_;
+    __annotate_shrink_front(__old_sz, __old_start);
+    __maybe_remove_front_spare();
+  } else { // erase from back
+    iterator __i = std::move(std::next(__p), end(), __p);
+    __alloc_traits::destroy(__a, std::addressof(*__i));
+    --__size();
+    __annotate_shrink_back(__old_sz, __old_start);
+    __maybe_remove_back_spare();
+  }
+  return begin() + __pos;
+}
+
+template <class _Tp, class _Allocator>
+typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::erase(const_iterator __f, const_iterator __l) {
+  _LIBCPP_ASSERT_VALID_INPUT_RANGE(__f <= __l, "deque::erase(first, last) called with an invalid range");
+  size_type __old_sz    = size();
+  size_type __old_start = __start_;
+  difference_type __n   = __l - __f;
+  iterator __b          = begin();
+  difference_type __pos = __f - __b;
+  iterator __p          = __b + __pos;
+  if (__n > 0) {
+    allocator_type& __a = __alloc();
+    if (static_cast<size_t>(__pos) <= (size() - __n) / 2) { // erase from front
+      iterator __i = std::move_backward(__b, __p, __p + __n);
+      for (; __b != __i; ++__b)
+        __alloc_traits::destroy(__a, std::addressof(*__b));
+      __size() -= __n;
+      __start_ += __n;
+      __annotate_shrink_front(__old_sz, __old_start);
+      while (__maybe_remove_front_spare()) {
+      }
+    } else { // erase from back
+      iterator __i = std::move(__p + __n, end(), __p);
+      for (iterator __e = end(); __i != __e; ++__i)
+        __alloc_traits::destroy(__a, std::addressof(*__i));
+      __size() -= __n;
+      __annotate_shrink_back(__old_sz, __old_start);
+      while (__maybe_remove_back_spare()) {
+      }
+    }
+  }
+  return begin() + __pos;
+}
+
+template <class _Tp, class _Allocator>
+void deque<_Tp, _Allocator>::__erase_to_end(const_iterator __f) {
+  size_type __old_sz    = size();
+  size_type __old_start = __start_;
+  iterator __e          = end();
+  difference_type __n   = __e - __f;
+  if (__n > 0) {
+    allocator_type& __a   = __alloc();
+    iterator __b          = begin();
+    difference_type __pos = __f - __b;
+    for (iterator __p = __b + __pos; __p != __e; ++__p)
+      __alloc_traits::destroy(__a, std::addressof(*__p));
+    __size() -= __n;
+    __annotate_shrink_back(__old_sz, __old_start);
+    while (__maybe_remove_back_spare()) {
+    }
+  }
+}
+
+template <class _Tp, class _Allocator>
+inline void deque<_Tp, _Allocator>::swap(deque& __c)
+#if _LIBCPP_STD_VER >= 14
+    _NOEXCEPT
+#else
+    _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>)
+#endif
+{
+  __map_.swap(__c.__map_);
+  std::swap(__start_, __c.__start_);
+  std::swap(__size(), __c.__size());
+  std::__swap_allocator(__alloc(), __c.__alloc());
+}
+
+template <class _Tp, class _Allocator>
+inline void deque<_Tp, _Allocator>::clear() _NOEXCEPT {
+  __annotate_delete();
+  allocator_type& __a = __alloc();
+  for (iterator __i = begin(), __e = end(); __i != __e; ++__i)
+    __alloc_traits::destroy(__a, std::addressof(*__i));
+  __size() = 0;
+  while (__map_.size() > 2) {
+    __alloc_traits::deallocate(__a, __map_.front(), __block_size);
+    __map_.pop_front();
+  }
+  switch (__map_.size()) {
+  case 1:
+    __start_ = __block_size / 2;
+    break;
+  case 2:
+    __start_ = __block_size;
+    break;
+  }
+  __annotate_new(0);
+}
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(const deque<_Tp, _Allocator>& __x, const deque<_Tp, _Allocator>& __y) {
+  const typename deque<_Tp, _Allocator>::size_type __sz = __x.size();
+  return __sz == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const deque<_Tp, _Allocator>& __x, const deque<_Tp, _Allocator>& __y) {
+  return !(__x == __y);
+}
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(const deque<_Tp, _Allocator>& __x, const deque<_Tp, _Allocator>& __y) {
+  return std::lexicographical_compare(__x.begin(), __x.end(), __y.begin(), __y.end());
+}
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>(const deque<_Tp, _Allocator>& __x, const deque<_Tp, _Allocator>& __y) {
+  return __y < __x;
+}
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>=(const deque<_Tp, _Allocator>& __x, const deque<_Tp, _Allocator>& __y) {
+  return !(__x < __y);
+}
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const deque<_Tp, _Allocator>& __x, const deque<_Tp, _Allocator>& __y) {
+  return !(__y < __x);
+}
+
+#else // _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI __synth_three_way_result<_Tp>
+operator<=>(const deque<_Tp, _Allocator>& __x, const deque<_Tp, _Allocator>& __y) {
+  return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
+}
+
+#endif // _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI void swap(deque<_Tp, _Allocator>& __x, deque<_Tp, _Allocator>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp, class _Allocator, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI typename deque<_Tp, _Allocator>::size_type
+erase(deque<_Tp, _Allocator>& __c, const _Up& __v) {
+  auto __old_size = __c.size();
+  __c.erase(std::remove(__c.begin(), __c.end(), __v), __c.end());
+  return __old_size - __c.size();
+}
+
+template <class _Tp, class _Allocator, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI typename deque<_Tp, _Allocator>::size_type
+erase_if(deque<_Tp, _Allocator>& __c, _Predicate __pred) {
+  auto __old_size = __c.size();
+  __c.erase(std::remove_if(__c.begin(), __c.end(), __pred), __c.end());
+  return __old_size - __c.size();
+}
+
+template <>
+inline constexpr bool __format::__enable_insertable<std::deque<char>> = true;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+inline constexpr bool __format::__enable_insertable<std::deque<wchar_t>> = true;
+#  endif
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace pmr {
+template <class _ValueT>
+using deque _LIBCPP_AVAILABILITY_PMR = std::deque<_ValueT, polymorphic_allocator<_ValueT>>;
+} // namespace pmr
+_LIBCPP_END_NAMESPACE_STD
+#endif
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <algorithm>
+#  include <atomic>
+#  include <concepts>
+#  include <cstdlib>
+#  include <functional>
+#  include <iosfwd>
+#  include <iterator>
+#  include <type_traits>
+#  include <typeinfo>
+#endif
+
+#endif // _LIBCPP_DEQUE
diff --git a/libcxx/include/__cxx03/errno.h b/libcxx/include/__cxx03/errno.h
new file mode 100644
index 000000000000..e657ad84ff44
--- /dev/null
+++ b/libcxx/include/__cxx03/errno.h
@@ -0,0 +1,399 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_ERRNO_H
+#define _LIBCPP_ERRNO_H
+
+/*
+    errno.h synopsis
+
+Macros:
+
+    EDOM
+    EILSEQ  // C99
+    ERANGE
+    errno
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if __has_include_next(<errno.h>)
+#  include_next <errno.h>
+#endif
+
+#ifdef __cplusplus
+
+#  if !defined(EOWNERDEAD) || !defined(ENOTRECOVERABLE)
+
+#    ifdef ELAST
+
+static const int __elast1 = ELAST + 1;
+static const int __elast2 = ELAST + 2;
+
+#    else
+
+static const int __elast1 = 104;
+static const int __elast2 = 105;
+
+#    endif
+
+#    ifdef ENOTRECOVERABLE
+
+#      define EOWNERDEAD __elast1
+
+#      ifdef ELAST
+#        undef ELAST
+#        define ELAST EOWNERDEAD
+#      endif
+
+#    elif defined(EOWNERDEAD)
+
+#      define ENOTRECOVERABLE __elast1
+#      ifdef ELAST
+#        undef ELAST
+#        define ELAST ENOTRECOVERABLE
+#      endif
+
+#    else // defined(EOWNERDEAD)
+
+#      define EOWNERDEAD __elast1
+#      define ENOTRECOVERABLE __elast2
+#      ifdef ELAST
+#        undef ELAST
+#        define ELAST ENOTRECOVERABLE
+#      endif
+
+#    endif // defined(EOWNERDEAD)
+
+#  endif // !defined(EOWNERDEAD) || !defined(ENOTRECOVERABLE)
+
+//  supply errno values likely to be missing, particularly on Windows
+
+#  ifndef EAFNOSUPPORT
+#    define EAFNOSUPPORT 9901
+#  endif
+
+#  ifndef EADDRINUSE
+#    define EADDRINUSE 9902
+#  endif
+
+#  ifndef EADDRNOTAVAIL
+#    define EADDRNOTAVAIL 9903
+#  endif
+
+#  ifndef EISCONN
+#    define EISCONN 9904
+#  endif
+
+#  ifndef EBADMSG
+#    define EBADMSG 9905
+#  endif
+
+#  ifndef ECONNABORTED
+#    define ECONNABORTED 9906
+#  endif
+
+#  ifndef EALREADY
+#    define EALREADY 9907
+#  endif
+
+#  ifndef ECONNREFUSED
+#    define ECONNREFUSED 9908
+#  endif
+
+#  ifndef ECONNRESET
+#    define ECONNRESET 9909
+#  endif
+
+#  ifndef EDESTADDRREQ
+#    define EDESTADDRREQ 9910
+#  endif
+
+#  ifndef EHOSTUNREACH
+#    define EHOSTUNREACH 9911
+#  endif
+
+#  ifndef EIDRM
+#    define EIDRM 9912
+#  endif
+
+#  ifndef EMSGSIZE
+#    define EMSGSIZE 9913
+#  endif
+
+#  ifndef ENETDOWN
+#    define ENETDOWN 9914
+#  endif
+
+#  ifndef ENETRESET
+#    define ENETRESET 9915
+#  endif
+
+#  ifndef ENETUNREACH
+#    define ENETUNREACH 9916
+#  endif
+
+#  ifndef ENOBUFS
+#    define ENOBUFS 9917
+#  endif
+
+#  ifndef ENOLINK
+#    define ENOLINK 9918
+#  endif
+
+#  ifndef ENODATA
+#    define ENODATA 9919
+#  endif
+
+#  ifndef ENOMSG
+#    define ENOMSG 9920
+#  endif
+
+#  ifndef ENOPROTOOPT
+#    define ENOPROTOOPT 9921
+#  endif
+
+#  ifndef ENOSR
+#    define ENOSR 9922
+#  endif
+
+#  ifndef ENOTSOCK
+#    define ENOTSOCK 9923
+#  endif
+
+#  ifndef ENOSTR
+#    define ENOSTR 9924
+#  endif
+
+#  ifndef ENOTCONN
+#    define ENOTCONN 9925
+#  endif
+
+#  ifndef ENOTSUP
+#    define ENOTSUP 9926
+#  endif
+
+#  ifndef ECANCELED
+#    define ECANCELED 9927
+#  endif
+
+#  ifndef EINPROGRESS
+#    define EINPROGRESS 9928
+#  endif
+
+#  ifndef EOPNOTSUPP
+#    define EOPNOTSUPP 9929
+#  endif
+
+#  ifndef EWOULDBLOCK
+#    define EWOULDBLOCK 9930
+#  endif
+
+#  ifndef EOWNERDEAD
+#    define EOWNERDEAD 9931
+#  endif
+
+#  ifndef EPROTO
+#    define EPROTO 9932
+#  endif
+
+#  ifndef EPROTONOSUPPORT
+#    define EPROTONOSUPPORT 9933
+#  endif
+
+#  ifndef ENOTRECOVERABLE
+#    define ENOTRECOVERABLE 9934
+#  endif
+
+#  ifndef ETIME
+#    define ETIME 9935
+#  endif
+
+#  ifndef ETXTBSY
+#    define ETXTBSY 9936
+#  endif
+
+#  ifndef ETIMEDOUT
+#    define ETIMEDOUT 9938
+#  endif
+
+#  ifndef ELOOP
+#    define ELOOP 9939
+#  endif
+
+#  ifndef EOVERFLOW
+#    define EOVERFLOW 9940
+#  endif
+
+#  ifndef EPROTOTYPE
+#    define EPROTOTYPE 9941
+#  endif
+
+#  ifndef ENOSYS
+#    define ENOSYS 9942
+#  endif
+
+#  ifndef EINVAL
+#    define EINVAL 9943
+#  endif
+
+#  ifndef ERANGE
+#    define ERANGE 9944
+#  endif
+
+#  ifndef EILSEQ
+#    define EILSEQ 9945
+#  endif
+
+//  Windows Mobile doesn't appear to define these:
+
+#  ifndef E2BIG
+#    define E2BIG 9946
+#  endif
+
+#  ifndef EDOM
+#    define EDOM 9947
+#  endif
+
+#  ifndef EFAULT
+#    define EFAULT 9948
+#  endif
+
+#  ifndef EBADF
+#    define EBADF 9949
+#  endif
+
+#  ifndef EPIPE
+#    define EPIPE 9950
+#  endif
+
+#  ifndef EXDEV
+#    define EXDEV 9951
+#  endif
+
+#  ifndef EBUSY
+#    define EBUSY 9952
+#  endif
+
+#  ifndef ENOTEMPTY
+#    define ENOTEMPTY 9953
+#  endif
+
+#  ifndef ENOEXEC
+#    define ENOEXEC 9954
+#  endif
+
+#  ifndef EEXIST
+#    define EEXIST 9955
+#  endif
+
+#  ifndef EFBIG
+#    define EFBIG 9956
+#  endif
+
+#  ifndef ENAMETOOLONG
+#    define ENAMETOOLONG 9957
+#  endif
+
+#  ifndef ENOTTY
+#    define ENOTTY 9958
+#  endif
+
+#  ifndef EINTR
+#    define EINTR 9959
+#  endif
+
+#  ifndef ESPIPE
+#    define ESPIPE 9960
+#  endif
+
+#  ifndef EIO
+#    define EIO 9961
+#  endif
+
+#  ifndef EISDIR
+#    define EISDIR 9962
+#  endif
+
+#  ifndef ECHILD
+#    define ECHILD 9963
+#  endif
+
+#  ifndef ENOLCK
+#    define ENOLCK 9964
+#  endif
+
+#  ifndef ENOSPC
+#    define ENOSPC 9965
+#  endif
+
+#  ifndef ENXIO
+#    define ENXIO 9966
+#  endif
+
+#  ifndef ENODEV
+#    define ENODEV 9967
+#  endif
+
+#  ifndef ENOENT
+#    define ENOENT 9968
+#  endif
+
+#  ifndef ESRCH
+#    define ESRCH 9969
+#  endif
+
+#  ifndef ENOTDIR
+#    define ENOTDIR 9970
+#  endif
+
+#  ifndef ENOMEM
+#    define ENOMEM 9971
+#  endif
+
+#  ifndef EPERM
+#    define EPERM 9972
+#  endif
+
+#  ifndef EACCES
+#    define EACCES 9973
+#  endif
+
+#  ifndef EROFS
+#    define EROFS 9974
+#  endif
+
+#  ifndef EDEADLK
+#    define EDEADLK 9975
+#  endif
+
+#  ifndef EAGAIN
+#    define EAGAIN 9976
+#  endif
+
+#  ifndef ENFILE
+#    define ENFILE 9977
+#  endif
+
+#  ifndef EMFILE
+#    define EMFILE 9978
+#  endif
+
+#  ifndef EMLINK
+#    define EMLINK 9979
+#  endif
+
+#endif // __cplusplus
+
+#endif // _LIBCPP_ERRNO_H
diff --git a/libcxx/include/__cxx03/exception b/libcxx/include/__cxx03/exception
new file mode 100644
index 000000000000..5eff8e3f8a4b
--- /dev/null
+++ b/libcxx/include/__cxx03/exception
@@ -0,0 +1,96 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXCEPTION
+#define _LIBCPP_EXCEPTION
+
+/*
+    exception synopsis
+
+namespace std
+{
+
+class exception
+{
+public:
+    exception() noexcept;
+    exception(const exception&) noexcept;
+    exception& operator=(const exception&) noexcept;
+    virtual ~exception() noexcept;
+    virtual const char* what() const noexcept;
+};
+
+class bad_exception
+    : public exception
+{
+public:
+    bad_exception() noexcept;
+    bad_exception(const bad_exception&) noexcept;
+    bad_exception& operator=(const bad_exception&) noexcept;
+    virtual ~bad_exception() noexcept;
+    virtual const char* what() const noexcept;
+};
+
+typedef void (*unexpected_handler)();
+unexpected_handler set_unexpected(unexpected_handler  f ) noexcept;
+unexpected_handler get_unexpected() noexcept;
+[[noreturn]] void unexpected();
+
+typedef void (*terminate_handler)();
+terminate_handler set_terminate(terminate_handler  f ) noexcept;
+terminate_handler get_terminate() noexcept;
+[[noreturn]] void terminate() noexcept;
+
+bool uncaught_exception()  noexcept;
+int  uncaught_exceptions() noexcept;  // C++17
+
+typedef unspecified exception_ptr;
+
+exception_ptr current_exception() noexcept;
+void rethrow_exception [[noreturn]] (exception_ptr p);
+template<class E> exception_ptr make_exception_ptr(E e) noexcept;
+
+class nested_exception
+{
+public:
+    nested_exception() noexcept;
+    nested_exception(const nested_exception&) noexcept = default;
+    nested_exception& operator=(const nested_exception&) noexcept = default;
+    virtual ~nested_exception() = default;
+
+    // access functions
+    [[noreturn]] void rethrow_nested() const;
+    exception_ptr nested_ptr() const noexcept;
+};
+
+template <class T> [[noreturn]] void throw_with_nested(T&& t);
+template <class E> void rethrow_if_nested(const E& e);
+
+}  // std
+
+*/
+
+#include <__config>
+#include <__exception/exception.h>
+#include <__exception/exception_ptr.h>
+#include <__exception/nested_exception.h>
+#include <__exception/operations.h>
+#include <__exception/terminate.h>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstdlib>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_EXCEPTION
diff --git a/libcxx/include/__cxx03/execution b/libcxx/include/__cxx03/execution
new file mode 100644
index 000000000000..94d434b2e460
--- /dev/null
+++ b/libcxx/include/__cxx03/execution
@@ -0,0 +1,149 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXECUTION
+#define _LIBCPP_EXECUTION
+
+/*
+namespace std::execution {
+  struct sequenced_policy;
+  struct parallel_policy;
+  struct parallel_unsequenced_policy;
+  struct unsequenced_policy; // since C++20
+
+  inline constexpr sequenced_policy seq = implementation-defined;
+  inline constexpr parallel_policy par = implementation-defined;
+  inline constexpr parallel_unsequenced_policy par_unseq = implementation-defined;
+  inline constexpr unsequenced_policy unseq = implementation-defined; // since C++20
+}
+
+namespace std {
+  template <class T>
+  struct is_execution_policy;
+
+  template <class T>
+  inline constexpr bool is_execution_policy_v;
+}
+*/
+
+#include <__config>
+#include <__type_traits/is_execution_policy.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_cvref.h>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace execution {
+struct sequenced_policy {
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit sequenced_policy(__disable_user_instantiations_tag) {}
+  sequenced_policy(const sequenced_policy&)            = delete;
+  sequenced_policy& operator=(const sequenced_policy&) = delete;
+};
+
+inline constexpr sequenced_policy seq{__disable_user_instantiations_tag{}};
+
+struct parallel_policy {
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit parallel_policy(__disable_user_instantiations_tag) {}
+  parallel_policy(const parallel_policy&)            = delete;
+  parallel_policy& operator=(const parallel_policy&) = delete;
+};
+
+inline constexpr parallel_policy par{__disable_user_instantiations_tag{}};
+
+struct parallel_unsequenced_policy {
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit parallel_unsequenced_policy(__disable_user_instantiations_tag) {}
+  parallel_unsequenced_policy(const parallel_unsequenced_policy&)            = delete;
+  parallel_unsequenced_policy& operator=(const parallel_unsequenced_policy&) = delete;
+};
+
+inline constexpr parallel_unsequenced_policy par_unseq{__disable_user_instantiations_tag{}};
+
+struct __unsequenced_policy {
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __unsequenced_policy(__disable_user_instantiations_tag) {}
+  __unsequenced_policy(const __unsequenced_policy&)            = delete;
+  __unsequenced_policy& operator=(const __unsequenced_policy&) = delete;
+};
+
+constexpr __unsequenced_policy __unseq{__disable_user_instantiations_tag{}};
+
+#  if _LIBCPP_STD_VER >= 20
+
+struct unsequenced_policy {
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit unsequenced_policy(__disable_user_instantiations_tag) {}
+  unsequenced_policy(const unsequenced_policy&)            = delete;
+  unsequenced_policy& operator=(const unsequenced_policy&) = delete;
+};
+
+inline constexpr unsequenced_policy unseq{__disable_user_instantiations_tag{}};
+
+#  endif // _LIBCPP_STD_VER >= 20
+
+} // namespace execution
+
+template <>
+inline constexpr bool is_execution_policy_v<execution::sequenced_policy> = true;
+
+template <>
+inline constexpr bool is_execution_policy_v<execution::parallel_policy> = true;
+
+template <>
+inline constexpr bool is_execution_policy_v<execution::parallel_unsequenced_policy> = true;
+
+template <>
+inline constexpr bool is_execution_policy_v<execution::__unsequenced_policy> = true;
+
+template <>
+inline constexpr bool __is_parallel_execution_policy_impl<execution::parallel_policy> = true;
+
+template <>
+inline constexpr bool __is_parallel_execution_policy_impl<execution::parallel_unsequenced_policy> = true;
+
+template <>
+inline constexpr bool __is_unsequenced_execution_policy_impl<execution::__unsequenced_policy> = true;
+
+template <>
+inline constexpr bool __is_unsequenced_execution_policy_impl<execution::parallel_unsequenced_policy> = true;
+
+#  if _LIBCPP_STD_VER >= 20
+template <>
+inline constexpr bool is_execution_policy_v<execution::unsequenced_policy> = true;
+
+template <>
+inline constexpr bool __is_unsequenced_execution_policy_impl<execution::unsequenced_policy> = true;
+
+#  endif
+
+template <class _Tp>
+struct is_execution_policy : bool_constant<is_execution_policy_v<_Tp>> {};
+
+template <class _ExecutionPolicy>
+_LIBCPP_HIDE_FROM_ABI auto __remove_parallel_policy(const _ExecutionPolicy&) {
+  if constexpr (is_same_v<_ExecutionPolicy, execution::parallel_policy>) {
+    return execution::sequenced_policy(execution::__disable_user_instantiations_tag{});
+  } else if constexpr (is_same_v<_ExecutionPolicy, execution::parallel_unsequenced_policy>) {
+    return execution::__unsequenced_policy{execution::__disable_user_instantiations_tag{}};
+  }
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
+#endif
+
+#endif // _LIBCPP_EXECUTION
diff --git a/libcxx/include/__cxx03/expected b/libcxx/include/__cxx03/expected
new file mode 100644
index 000000000000..6a2f12f2bf3b
--- /dev/null
+++ b/libcxx/include/__cxx03/expected
@@ -0,0 +1,62 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPECTED
+#define _LIBCPP_EXPECTED
+
+/*
+  Header <expected> synopsis
+
+namespace std {
+  // [expected.unexpected], class template unexpected
+  template<class E> class unexpected;
+
+  // [expected.bad], class template bad_expected_access
+  template<class E> class bad_expected_access;
+
+  // [expected.bad.void], specialization for void
+  template<> class bad_expected_access<void>;
+
+  // in-place construction of unexpected values
+  struct unexpect_t {
+    explicit unexpect_t() = default;
+  };
+  inline constexpr unexpect_t unexpect{};
+
+  // [expected.expected], class template expected
+  template<class T, class E> class expected;
+
+  // [expected.void], partial specialization of expected for void types
+  template<class T, class E> requires is_void_v<T> class expected<T, E>;
+}
+
+*/
+
+#include <__config>
+
+#if _LIBCPP_STD_VER >= 23
+#  include <__expected/bad_expected_access.h>
+#  include <__expected/expected.h>
+#  include <__expected/unexpect.h>
+#  include <__expected/unexpected.h>
+#endif
+
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
+#  include <initializer_list>
+#  include <new>
+#endif
+
+#endif // _LIBCPP_EXPECTED
diff --git a/libcxx/include/__cxx03/experimental/__config b/libcxx/include/__cxx03/experimental/__config
new file mode 100644
index 000000000000..7b23791511ce
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/__config
@@ -0,0 +1,45 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL_CONFIG
+#define _LIBCPP_EXPERIMENTAL_CONFIG
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#define _LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL                                                                           \
+  namespace std {                                                                                                      \
+  namespace experimental {
+#define _LIBCPP_END_NAMESPACE_EXPERIMENTAL                                                                             \
+  }                                                                                                                    \
+  }
+
+#define _LIBCPP_BEGIN_NAMESPACE_LFTS _LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL inline namespace fundamentals_v1 {
+#define _LIBCPP_END_NAMESPACE_LFTS                                                                                     \
+  }                                                                                                                    \
+  }                                                                                                                    \
+  }
+
+#define _LIBCPP_BEGIN_NAMESPACE_LFTS_V2 _LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL inline namespace fundamentals_v2 {
+#define _LIBCPP_END_NAMESPACE_LFTS_V2                                                                                  \
+  }                                                                                                                    \
+  }                                                                                                                    \
+  }
+
+// TODO: support more targets
+#if defined(__AVX__)
+#  define _LIBCPP_NATIVE_SIMD_WIDTH_IN_BYTES 32
+#else
+#  define _LIBCPP_NATIVE_SIMD_WIDTH_IN_BYTES 16
+#endif
+
+#endif
diff --git a/libcxx/include/__cxx03/experimental/__simd/aligned_tag.h b/libcxx/include/__cxx03/experimental/__simd/aligned_tag.h
new file mode 100644
index 000000000000..31d2b50aa1dd
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/__simd/aligned_tag.h
@@ -0,0 +1,75 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H
+#define _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H
+
+#include <__memory/assume_aligned.h>
+#include <__type_traits/remove_const.h>
+#include <cstddef>
+#include <experimental/__config>
+#include <experimental/__simd/traits.h>
+
+#if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+
+_LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL
+inline namespace parallelism_v2 {
+// memory alignment
+struct element_aligned_tag {
+  template <class _Tp, class _Up = typename _Tp::value_type>
+  static constexpr size_t __alignment = alignof(_Up);
+
+  template <class _Tp, class _Up>
+  static _LIBCPP_HIDE_FROM_ABI constexpr _Up* __apply(_Up* __ptr) {
+    return __ptr;
+  }
+};
+
+template <>
+inline constexpr bool is_simd_flag_type_v<element_aligned_tag> = true;
+
+struct vector_aligned_tag {
+  template <class _Tp, class _Up = typename _Tp::value_type>
+  static constexpr size_t __alignment = memory_alignment_v<_Tp, remove_const_t<_Up>>;
+
+  template <class _Tp, class _Up>
+  static _LIBCPP_HIDE_FROM_ABI constexpr _Up* __apply(_Up* __ptr) {
+    return std::__assume_aligned<__alignment<_Tp, _Up>, _Up>(__ptr);
+  }
+};
+
+template <>
+inline constexpr bool is_simd_flag_type_v<vector_aligned_tag> = true;
+
+template <size_t _Np>
+struct overaligned_tag {
+  template <class _Tp, class _Up = typename _Tp::value_type>
+  static constexpr size_t __alignment = _Np;
+
+  template <class _Tp, class _Up>
+  static _LIBCPP_HIDE_FROM_ABI constexpr _Up* __apply(_Up* __ptr) {
+    return std::__assume_aligned<__alignment<_Tp, _Up>, _Up>(__ptr);
+  }
+};
+
+template <size_t _Np>
+inline constexpr bool is_simd_flag_type_v<overaligned_tag<_Np>> = true;
+
+inline constexpr element_aligned_tag element_aligned{};
+
+inline constexpr vector_aligned_tag vector_aligned{};
+
+template <size_t _Np>
+inline constexpr overaligned_tag<_Np> overaligned{};
+
+} // namespace parallelism_v2
+_LIBCPP_END_NAMESPACE_EXPERIMENTAL
+
+#endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+#endif // _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/declaration.h b/libcxx/include/__cxx03/experimental/__simd/declaration.h
new file mode 100644
index 000000000000..7b45d035c271
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/__simd/declaration.h
@@ -0,0 +1,81 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL___SIMD_DECLARATION_H
+#define _LIBCPP_EXPERIMENTAL___SIMD_DECLARATION_H
+
+#include <cstddef>
+#include <experimental/__config>
+
+#if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+
+_LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL
+inline namespace parallelism_v2 {
+namespace simd_abi {
+template <int>
+struct __vec_ext;
+struct __scalar;
+
+using scalar = __scalar;
+
+// TODO: make this platform dependent
+template <int _Np>
+using fixed_size = __vec_ext<_Np>;
+
+template <class _Tp>
+inline constexpr int max_fixed_size = 32;
+
+// TODO: make this platform dependent
+template <class _Tp>
+using compatible = __vec_ext<16 / sizeof(_Tp)>;
+
+// TODO: make this platform dependent
+template <class _Tp>
+using native = __vec_ext<_LIBCPP_NATIVE_SIMD_WIDTH_IN_BYTES / sizeof(_Tp)>;
+
+// TODO: make this platform dependent
+template <class _Tp, size_t _Np, class... _Abis>
+struct deduce {
+  using type = fixed_size<_Np>;
+};
+
+// TODO: make this platform dependent
+template <class _Tp, size_t _Np, class... _Abis>
+using deduce_t = typename deduce<_Tp, _Np, _Abis...>::type;
+
+} // namespace simd_abi
+
+template <class _Tp, class _Abi>
+struct __simd_storage;
+
+template <class _Tp, class _Abi>
+struct __mask_storage;
+
+template <class _Tp, class _Abi>
+struct __simd_operations;
+
+template <class _Tp, class _Abi>
+struct __mask_operations;
+
+struct element_aligned_tag;
+struct vector_aligned_tag;
+template <size_t>
+struct overaligned_tag;
+
+template <class _Tp, class _Abi = simd_abi::compatible<_Tp>>
+class simd;
+
+template <class _Tp, class _Abi = simd_abi::compatible<_Tp>>
+class simd_mask;
+
+} // namespace parallelism_v2
+_LIBCPP_END_NAMESPACE_EXPERIMENTAL
+
+#endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+#endif // _LIBCPP_EXPERIMENTAL___SIMD_DECLARATION_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/reference.h b/libcxx/include/__cxx03/experimental/__simd/reference.h
new file mode 100644
index 000000000000..af61dbcc2fe9
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/__simd/reference.h
@@ -0,0 +1,105 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL___SIMD_REFERENCE_H
+#define _LIBCPP_EXPERIMENTAL___SIMD_REFERENCE_H
+
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_same.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <cstddef>
+#include <experimental/__config>
+#include <experimental/__simd/utility.h>
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+
+_LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL
+inline namespace parallelism_v2 {
+template <class _Tp, class _Storage, class _Vp>
+class __simd_reference {
+  template <class, class>
+  friend class simd;
+  template <class, class>
+  friend class simd_mask;
+
+  _Storage& __s_;
+  size_t __idx_;
+
+  _LIBCPP_HIDE_FROM_ABI __simd_reference(_Storage& __s, size_t __idx) : __s_(__s), __idx_(__idx) {}
+
+  _LIBCPP_HIDE_FROM_ABI _Vp __get() const noexcept { return __s_.__get(__idx_); }
+
+  _LIBCPP_HIDE_FROM_ABI void __set(_Vp __v) {
+    if constexpr (is_same_v<_Vp, bool>)
+      __s_.__set(__idx_, experimental::__set_all_bits<_Tp>(__v));
+    else
+      __s_.__set(__idx_, __v);
+  }
+
+public:
+  using value_type = _Vp;
+
+  __simd_reference()                        = delete;
+  __simd_reference(const __simd_reference&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI operator value_type() const noexcept { return __get(); }
+
+  template <class _Up, enable_if_t<is_assignable_v<value_type&, _Up&&>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI __simd_reference operator=(_Up&& __v) && noexcept {
+    __set(static_cast<value_type>(std::forward<_Up>(__v)));
+    return {__s_, __idx_};
+  }
+
+  // Note: This approach might not fully align with the specification,
+  // which might be a wording defect. (https://wg21.link/N4808 section 9.6.3)
+  template <class _Tp1, class _Storage1, class _Vp1>
+  friend void
+  swap(__simd_reference<_Tp1, _Storage1, _Vp1>&& __a, __simd_reference<_Tp1, _Storage1, _Vp1>&& __b) noexcept;
+
+  template <class _Tp1, class _Storage1, class _Vp1>
+  friend void swap(_Vp1& __a, __simd_reference<_Tp1, _Storage1, _Vp1>&& __b) noexcept;
+
+  template <class _Tp1, class _Storage1, class _Vp1>
+  friend void swap(__simd_reference<_Tp1, _Storage1, _Vp1>&& __a, _Vp1& __b) noexcept;
+};
+
+template <class _Tp, class _Storage, class _Vp>
+_LIBCPP_HIDE_FROM_ABI void
+swap(__simd_reference<_Tp, _Storage, _Vp>&& __a, __simd_reference<_Tp, _Storage, _Vp>&& __b) noexcept {
+  _Vp __tmp(std::move(__a));
+  std::move(__a) = std::move(__b);
+  std::move(__b) = std::move(__tmp);
+}
+
+template <class _Tp, class _Storage, class _Vp>
+_LIBCPP_HIDE_FROM_ABI void swap(_Vp& __a, __simd_reference<_Tp, _Storage, _Vp>&& __b) noexcept {
+  _Vp __tmp(std::move(__a));
+  __a            = std::move(__b);
+  std::move(__b) = std::move(__tmp);
+}
+
+template <class _Tp, class _Storage, class _Vp>
+_LIBCPP_HIDE_FROM_ABI void swap(__simd_reference<_Tp, _Storage, _Vp>&& __a, _Vp& __b) noexcept {
+  _Vp __tmp(std::move(__a));
+  std::move(__a) = std::move(__b);
+  __b            = std::move(__tmp);
+}
+
+} // namespace parallelism_v2
+_LIBCPP_END_NAMESPACE_EXPERIMENTAL
+
+#endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP_EXPERIMENTAL___SIMD_REFERENCE_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/scalar.h b/libcxx/include/__cxx03/experimental/__simd/scalar.h
new file mode 100644
index 000000000000..1add4653209a
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/__simd/scalar.h
@@ -0,0 +1,87 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL___SIMD_SCALAR_H
+#define _LIBCPP_EXPERIMENTAL___SIMD_SCALAR_H
+
+#include <__assert>
+#include <cstddef>
+#include <experimental/__config>
+#include <experimental/__simd/declaration.h>
+#include <experimental/__simd/traits.h>
+
+#if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+
+_LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL
+inline namespace parallelism_v2 {
+namespace simd_abi {
+struct __scalar {
+  static constexpr size_t __simd_size = 1;
+};
+} // namespace simd_abi
+
+template <>
+inline constexpr bool is_abi_tag_v<simd_abi::__scalar> = true;
+
+template <class _Tp>
+struct __simd_storage<_Tp, simd_abi::__scalar> {
+  _Tp __data;
+
+  _LIBCPP_HIDE_FROM_ABI _Tp __get([[maybe_unused]] size_t __idx) const noexcept {
+    _LIBCPP_ASSERT_UNCATEGORIZED(__idx == 0, "Index is out of bounds");
+    return __data;
+  }
+  _LIBCPP_HIDE_FROM_ABI void __set([[maybe_unused]] size_t __idx, _Tp __v) noexcept {
+    _LIBCPP_ASSERT_UNCATEGORIZED(__idx == 0, "Index is out of bounds");
+    __data = __v;
+  }
+};
+
+template <class _Tp>
+struct __mask_storage<_Tp, simd_abi::__scalar> : __simd_storage<bool, simd_abi::__scalar> {};
+
+template <class _Tp>
+struct __simd_operations<_Tp, simd_abi::__scalar> {
+  using _SimdStorage = __simd_storage<_Tp, simd_abi::__scalar>;
+  using _MaskStorage = __mask_storage<_Tp, simd_abi::__scalar>;
+
+  static _LIBCPP_HIDE_FROM_ABI _SimdStorage __broadcast(_Tp __v) noexcept { return {__v}; }
+
+  template <class _Generator>
+  static _LIBCPP_HIDE_FROM_ABI _SimdStorage __generate(_Generator&& __g) noexcept {
+    return {__g(std::integral_constant<size_t, 0>())};
+  }
+
+  template <class _Up>
+  static _LIBCPP_HIDE_FROM_ABI void __load(_SimdStorage& __s, const _Up* __mem) noexcept {
+    __s.__data = static_cast<_Tp>(__mem[0]);
+  }
+
+  template <class _Up>
+  static _LIBCPP_HIDE_FROM_ABI void __store(_SimdStorage __s, _Up* __mem) noexcept {
+    *__mem = static_cast<_Up>(__s.__data);
+  }
+};
+
+template <class _Tp>
+struct __mask_operations<_Tp, simd_abi::__scalar> {
+  using _MaskStorage = __mask_storage<_Tp, simd_abi::__scalar>;
+
+  static _LIBCPP_HIDE_FROM_ABI _MaskStorage __broadcast(bool __v) noexcept { return {__v}; }
+
+  static _LIBCPP_HIDE_FROM_ABI void __load(_MaskStorage& __s, const bool* __mem) noexcept { __s.__data = __mem[0]; }
+
+  static _LIBCPP_HIDE_FROM_ABI void __store(_MaskStorage __s, bool* __mem) noexcept { __mem[0] = __s.__data; }
+};
+
+} // namespace parallelism_v2
+_LIBCPP_END_NAMESPACE_EXPERIMENTAL
+
+#endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+#endif // _LIBCPP_EXPERIMENTAL___SIMD_SCALAR_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/simd.h b/libcxx/include/__cxx03/experimental/__simd/simd.h
new file mode 100644
index 000000000000..37e334aad6da
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/__simd/simd.h
@@ -0,0 +1,102 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL___SIMD_SIMD_H
+#define _LIBCPP_EXPERIMENTAL___SIMD_SIMD_H
+
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/forward.h>
+#include <cstddef>
+#include <experimental/__config>
+#include <experimental/__simd/declaration.h>
+#include <experimental/__simd/reference.h>
+#include <experimental/__simd/traits.h>
+#include <experimental/__simd/utility.h>
+
+#if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+
+_LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL
+inline namespace parallelism_v2 {
+
+// class template simd [simd.class]
+// TODO: implement simd class
+template <class _Tp, class _Abi>
+class simd {
+  using _Impl    = __simd_operations<_Tp, _Abi>;
+  using _Storage = typename _Impl::_SimdStorage;
+
+  _Storage __s_;
+
+public:
+  using value_type = _Tp;
+  using reference  = __simd_reference<_Tp, _Storage, value_type>;
+  using mask_type  = simd_mask<_Tp, _Abi>;
+  using abi_type   = _Abi;
+
+  static _LIBCPP_HIDE_FROM_ABI constexpr size_t size() noexcept { return simd_size_v<value_type, abi_type>; }
+
+  _LIBCPP_HIDE_FROM_ABI simd() noexcept = default;
+
+  // broadcast constructor
+  template <class _Up, enable_if_t<__can_broadcast_v<value_type, __remove_cvref_t<_Up>>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI simd(_Up&& __v) noexcept : __s_(_Impl::__broadcast(static_cast<value_type>(__v))) {}
+
+  // implicit type conversion constructor
+  template <class _Up,
+            enable_if_t<!is_same_v<_Up, _Tp> && is_same_v<abi_type, simd_abi::fixed_size<size()>> &&
+                            __is_non_narrowing_convertible_v<_Up, value_type>,
+                        int> = 0>
+  _LIBCPP_HIDE_FROM_ABI simd(const simd<_Up, simd_abi::fixed_size<size()>>& __v) noexcept {
+    for (size_t __i = 0; __i < size(); __i++) {
+      (*this)[__i] = static_cast<value_type>(__v[__i]);
+    }
+  }
+
+  // generator constructor
+  template <class _Generator, enable_if_t<__can_generate_v<value_type, _Generator, size()>, int> = 0>
+  explicit _LIBCPP_HIDE_FROM_ABI simd(_Generator&& __g) noexcept
+      : __s_(_Impl::__generate(std::forward<_Generator>(__g))) {}
+
+  // load constructor
+  template <class _Up, class _Flags, enable_if_t<__is_vectorizable_v<_Up> && is_simd_flag_type_v<_Flags>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI simd(const _Up* __mem, _Flags) {
+    _Impl::__load(__s_, _Flags::template __apply<simd>(__mem));
+  }
+
+  // copy functions
+  template <class _Up, class _Flags, enable_if_t<__is_vectorizable_v<_Up> && is_simd_flag_type_v<_Flags>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void copy_from(const _Up* __mem, _Flags) {
+    _Impl::__load(__s_, _Flags::template __apply<simd>(__mem));
+  }
+
+  template <class _Up, class _Flags, enable_if_t<__is_vectorizable_v<_Up> && is_simd_flag_type_v<_Flags>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void copy_to(_Up* __mem, _Flags) const {
+    _Impl::__store(__s_, _Flags::template __apply<simd>(__mem));
+  }
+
+  // scalar access [simd.subscr]
+  _LIBCPP_HIDE_FROM_ABI reference operator[](size_t __i) noexcept { return reference(__s_, __i); }
+  _LIBCPP_HIDE_FROM_ABI value_type operator[](size_t __i) const noexcept { return __s_.__get(__i); }
+};
+
+template <class _Tp, class _Abi>
+inline constexpr bool is_simd_v<simd<_Tp, _Abi>> = true;
+
+template <class _Tp>
+using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
+
+template <class _Tp, int _Np>
+using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
+
+} // namespace parallelism_v2
+_LIBCPP_END_NAMESPACE_EXPERIMENTAL
+
+#endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+#endif // _LIBCPP_EXPERIMENTAL___SIMD_SIMD_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/simd_mask.h b/libcxx/include/__cxx03/experimental/__simd/simd_mask.h
new file mode 100644
index 000000000000..fd6dee2e28ee
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/__simd/simd_mask.h
@@ -0,0 +1,90 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL___SIMD_SIMD_MASK_H
+#define _LIBCPP_EXPERIMENTAL___SIMD_SIMD_MASK_H
+
+#include <__type_traits/is_same.h>
+#include <cstddef>
+#include <experimental/__config>
+#include <experimental/__simd/declaration.h>
+#include <experimental/__simd/reference.h>
+#include <experimental/__simd/traits.h>
+
+#if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+
+_LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL
+inline namespace parallelism_v2 {
+
+// class template simd_mask [simd.mask.class]
+// TODO: implement simd_mask class
+template <class _Tp, class _Abi>
+class simd_mask {
+  using _Impl    = __mask_operations<_Tp, _Abi>;
+  using _Storage = typename _Impl::_MaskStorage;
+
+  _Storage __s_;
+
+public:
+  using value_type = bool;
+  using reference  = __simd_reference<_Tp, _Storage, value_type>;
+  using simd_type  = simd<_Tp, _Abi>;
+  using abi_type   = _Abi;
+
+  static _LIBCPP_HIDE_FROM_ABI constexpr size_t size() noexcept { return simd_type::size(); }
+
+  _LIBCPP_HIDE_FROM_ABI simd_mask() noexcept = default;
+
+  // broadcast constructor
+  _LIBCPP_HIDE_FROM_ABI explicit simd_mask(value_type __v) noexcept : __s_(_Impl::__broadcast(__v)) {}
+
+  // implicit type conversion constructor
+  template <class _Up, enable_if_t<!is_same_v<_Up, _Tp> && is_same_v<abi_type, simd_abi::fixed_size<size()>>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI simd_mask(const simd_mask<_Up, simd_abi::fixed_size<size()>>& __v) noexcept {
+    for (size_t __i = 0; __i < size(); __i++) {
+      (*this)[__i] = __v[__i];
+    }
+  }
+
+  // load constructor
+  template <class _Flags, enable_if_t<is_simd_flag_type_v<_Flags>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI simd_mask(const value_type* __mem, _Flags) {
+    _Impl::__load(__s_, _Flags::template __apply<simd_mask>(__mem));
+  }
+
+  // copy functions
+  template <class _Flags, enable_if_t<is_simd_flag_type_v<_Flags>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void copy_from(const value_type* __mem, _Flags) {
+    _Impl::__load(__s_, _Flags::template __apply<simd_mask>(__mem));
+  }
+
+  template <class _Flags, enable_if_t<is_simd_flag_type_v<_Flags>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void copy_to(value_type* __mem, _Flags) const {
+    _Impl::__store(__s_, _Flags::template __apply<simd_mask>(__mem));
+  }
+
+  // scalar access [simd.mask.subscr]
+  _LIBCPP_HIDE_FROM_ABI reference operator[](size_t __i) noexcept { return reference(__s_, __i); }
+  _LIBCPP_HIDE_FROM_ABI value_type operator[](size_t __i) const noexcept { return __s_.__get(__i); }
+};
+
+template <class _Tp, class _Abi>
+inline constexpr bool is_simd_mask_v<simd_mask<_Tp, _Abi>> = true;
+
+template <class _Tp>
+using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
+
+template <class _Tp, int _Np>
+using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
+
+} // namespace parallelism_v2
+_LIBCPP_END_NAMESPACE_EXPERIMENTAL
+
+#endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+#endif // _LIBCPP_EXPERIMENTAL___SIMD_SIMD_MASK_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/traits.h b/libcxx/include/__cxx03/experimental/__simd/traits.h
new file mode 100644
index 000000000000..ec25b4bfa7f9
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/__simd/traits.h
@@ -0,0 +1,75 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL___SIMD_TRAITS_H
+#define _LIBCPP_EXPERIMENTAL___SIMD_TRAITS_H
+
+#include <__bit/bit_ceil.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_same.h>
+#include <cstddef>
+#include <experimental/__config>
+#include <experimental/__simd/declaration.h>
+#include <experimental/__simd/utility.h>
+
+#if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+
+_LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL
+inline namespace parallelism_v2 {
+
+// traits [simd.traits]
+template <class _Tp>
+inline constexpr bool is_abi_tag_v = false;
+
+template <class _Tp>
+struct is_abi_tag : bool_constant<is_abi_tag_v<_Tp>> {};
+
+template <class _Tp>
+inline constexpr bool is_simd_v = false;
+
+template <class _Tp>
+struct is_simd : bool_constant<is_simd_v<_Tp>> {};
+
+template <class _Tp>
+inline constexpr bool is_simd_mask_v = false;
+
+template <class _Tp>
+struct is_simd_mask : bool_constant<is_simd_mask_v<_Tp>> {};
+
+template <class _Tp>
+inline constexpr bool is_simd_flag_type_v = false;
+
+template <class _Tp>
+struct is_simd_flag_type : bool_constant<is_simd_flag_type_v<_Tp>> {};
+
+template <class _Tp, class _Abi = simd_abi::compatible<_Tp>, bool = (__is_vectorizable_v<_Tp> && is_abi_tag_v<_Abi>)>
+struct simd_size : integral_constant<size_t, _Abi::__simd_size> {};
+
+template <class _Tp, class _Abi>
+struct simd_size<_Tp, _Abi, false> {};
+
+template <class _Tp, class _Abi = simd_abi::compatible<_Tp>>
+inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
+
+template <class _Tp,
+          class _Up = typename _Tp::value_type,
+          bool      = (is_simd_v<_Tp> && __is_vectorizable_v<_Up>) || (is_simd_mask_v<_Tp> && is_same_v<_Up, bool>)>
+struct memory_alignment : integral_constant<size_t, std::__bit_ceil(sizeof(_Up) * _Tp::size())> {};
+
+template <class _Tp, class _Up>
+struct memory_alignment<_Tp, _Up, false> {};
+
+template <class _Tp, class _Up = typename _Tp::value_type>
+inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
+
+} // namespace parallelism_v2
+_LIBCPP_END_NAMESPACE_EXPERIMENTAL
+
+#endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+#endif // _LIBCPP_EXPERIMENTAL___SIMD_TRAITS_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/utility.h b/libcxx/include/__cxx03/experimental/__simd/utility.h
new file mode 100644
index 000000000000..708fa3d8f72c
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/__simd/utility.h
@@ -0,0 +1,103 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL___SIMD_UTILITY_H
+#define _LIBCPP_EXPERIMENTAL___SIMD_UTILITY_H
+
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_const.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_unsigned.h>
+#include <__type_traits/is_volatile.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+#include <__utility/integer_sequence.h>
+#include <cstddef>
+#include <cstdint>
+#include <experimental/__config>
+#include <limits>
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+
+_LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL
+inline namespace parallelism_v2 {
+template <class _Tp>
+inline constexpr bool __is_vectorizable_v =
+    is_arithmetic_v<_Tp> && !is_const_v<_Tp> && !is_volatile_v<_Tp> && !is_same_v<_Tp, bool>;
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI auto __choose_mask_type() {
+  if constexpr (sizeof(_Tp) == 1) {
+    return uint8_t{};
+  } else if constexpr (sizeof(_Tp) == 2) {
+    return uint16_t{};
+  } else if constexpr (sizeof(_Tp) == 4) {
+    return uint32_t{};
+  } else if constexpr (sizeof(_Tp) == 8) {
+    return uint64_t{};
+  }
+#  ifndef _LIBCPP_HAS_NO_INT128
+  else if constexpr (sizeof(_Tp) == 16) {
+    return __uint128_t{};
+  }
+#  endif
+  else
+    static_assert(sizeof(_Tp) == 0, "Unexpected size");
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI auto constexpr __set_all_bits(bool __v) {
+  return __v ? (numeric_limits<decltype(__choose_mask_type<_Tp>())>::max()) : 0;
+}
+
+template <class _From, class _To, class = void>
+inline constexpr bool __is_non_narrowing_convertible_v = false;
+
+template <class _From, class _To>
+inline constexpr bool __is_non_narrowing_convertible_v<_From, _To, std::void_t<decltype(_To{std::declval<_From>()})>> =
+    true;
+
+template <class _Tp, class _Up>
+inline constexpr bool __can_broadcast_v =
+    (__is_vectorizable_v<_Up> && __is_non_narrowing_convertible_v<_Up, _Tp>) ||
+    (!__is_vectorizable_v<_Up> && is_convertible_v<_Up, _Tp>) || is_same_v<_Up, int> ||
+    (is_same_v<_Up, unsigned int> && is_unsigned_v<_Tp>);
+
+template <class _Tp, class _Generator, std::size_t _Idx, class = void>
+inline constexpr bool __is_well_formed = false;
+
+template <class _Tp, class _Generator, std::size_t _Idx>
+inline constexpr bool
+    __is_well_formed<_Tp,
+                     _Generator,
+                     _Idx,
+                     std::void_t<decltype(std::declval<_Generator>()(integral_constant<size_t, _Idx>()))>> =
+        __can_broadcast_v<_Tp, decltype(std::declval<_Generator>()(integral_constant<size_t, _Idx>()))>;
+
+template <class _Tp, class _Generator, std::size_t... _Idxes>
+_LIBCPP_HIDE_FROM_ABI constexpr bool __can_generate(index_sequence<_Idxes...>) {
+  return (true && ... && __is_well_formed<_Tp, _Generator, _Idxes>);
+}
+
+template <class _Tp, class _Generator, std::size_t _Size>
+inline constexpr bool __can_generate_v = experimental::__can_generate<_Tp, _Generator>(make_index_sequence<_Size>());
+
+} // namespace parallelism_v2
+_LIBCPP_END_NAMESPACE_EXPERIMENTAL
+
+#endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP_EXPERIMENTAL___SIMD_UTILITY_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/vec_ext.h b/libcxx/include/__cxx03/experimental/__simd/vec_ext.h
new file mode 100644
index 000000000000..316866b84873
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/__simd/vec_ext.h
@@ -0,0 +1,119 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL___SIMD_VEC_EXT_H
+#define _LIBCPP_EXPERIMENTAL___SIMD_VEC_EXT_H
+
+#include <__assert>
+#include <__bit/bit_ceil.h>
+#include <__utility/forward.h>
+#include <__utility/integer_sequence.h>
+#include <cstddef>
+#include <experimental/__config>
+#include <experimental/__simd/declaration.h>
+#include <experimental/__simd/traits.h>
+#include <experimental/__simd/utility.h>
+
+#if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+
+_LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL
+inline namespace parallelism_v2 {
+namespace simd_abi {
+template <int _Np>
+struct __vec_ext {
+  static constexpr size_t __simd_size = _Np;
+};
+} // namespace simd_abi
+
+template <int _Np>
+inline constexpr bool is_abi_tag_v<simd_abi::__vec_ext<_Np>> = _Np > 0 && _Np <= 32;
+
+template <class _Tp, int _Np>
+struct __simd_storage<_Tp, simd_abi::__vec_ext<_Np>> {
+  _Tp __data __attribute__((__vector_size__(std::__bit_ceil((sizeof(_Tp) * _Np)))));
+
+  _LIBCPP_HIDE_FROM_ABI _Tp __get(size_t __idx) const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__idx >= 0 && __idx < _Np, "Index is out of bounds");
+    return __data[__idx];
+  }
+  _LIBCPP_HIDE_FROM_ABI void __set(size_t __idx, _Tp __v) noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__idx >= 0 && __idx < _Np, "Index is out of bounds");
+    __data[__idx] = __v;
+  }
+};
+
+template <class _Tp, int _Np>
+struct __mask_storage<_Tp, simd_abi::__vec_ext<_Np>>
+    : __simd_storage<decltype(experimental::__choose_mask_type<_Tp>()), simd_abi::__vec_ext<_Np>> {};
+
+template <class _Tp, int _Np>
+struct __simd_operations<_Tp, simd_abi::__vec_ext<_Np>> {
+  using _SimdStorage = __simd_storage<_Tp, simd_abi::__vec_ext<_Np>>;
+  using _MaskStorage = __mask_storage<_Tp, simd_abi::__vec_ext<_Np>>;
+
+  static _LIBCPP_HIDE_FROM_ABI _SimdStorage __broadcast(_Tp __v) noexcept {
+    _SimdStorage __result;
+    for (int __i = 0; __i < _Np; ++__i) {
+      __result.__set(__i, __v);
+    }
+    return __result;
+  }
+
+  template <class _Generator, size_t... _Is>
+  static _LIBCPP_HIDE_FROM_ABI _SimdStorage __generate_init(_Generator&& __g, std::index_sequence<_Is...>) {
+    return _SimdStorage{{__g(std::integral_constant<size_t, _Is>())...}};
+  }
+
+  template <class _Generator>
+  static _LIBCPP_HIDE_FROM_ABI _SimdStorage __generate(_Generator&& __g) noexcept {
+    return __generate_init(std::forward<_Generator>(__g), std::make_index_sequence<_Np>());
+  }
+
+  template <class _Up>
+  static _LIBCPP_HIDE_FROM_ABI void __load(_SimdStorage& __s, const _Up* __mem) noexcept {
+    for (size_t __i = 0; __i < _Np; __i++)
+      __s.__data[__i] = static_cast<_Tp>(__mem[__i]);
+  }
+
+  template <class _Up>
+  static _LIBCPP_HIDE_FROM_ABI void __store(_SimdStorage __s, _Up* __mem) noexcept {
+    for (size_t __i = 0; __i < _Np; __i++)
+      __mem[__i] = static_cast<_Up>(__s.__data[__i]);
+  }
+};
+
+template <class _Tp, int _Np>
+struct __mask_operations<_Tp, simd_abi::__vec_ext<_Np>> {
+  using _MaskStorage = __mask_storage<_Tp, simd_abi::__vec_ext<_Np>>;
+
+  static _LIBCPP_HIDE_FROM_ABI _MaskStorage __broadcast(bool __v) noexcept {
+    _MaskStorage __result;
+    auto __all_bits_v = experimental::__set_all_bits<_Tp>(__v);
+    for (int __i = 0; __i < _Np; ++__i) {
+      __result.__set(__i, __all_bits_v);
+    }
+    return __result;
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI void __load(_MaskStorage& __s, const bool* __mem) noexcept {
+    for (size_t __i = 0; __i < _Np; __i++)
+      __s.__data[__i] = experimental::__set_all_bits<_Tp>(__mem[__i]);
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI void __store(_MaskStorage __s, bool* __mem) noexcept {
+    for (size_t __i = 0; __i < _Np; __i++)
+      __mem[__i] = static_cast<bool>(__s.__data[__i]);
+  }
+};
+
+} // namespace parallelism_v2
+_LIBCPP_END_NAMESPACE_EXPERIMENTAL
+
+#endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
+#endif // _LIBCPP_EXPERIMENTAL___SIMD_VEC_EXT_H
diff --git a/libcxx/include/__cxx03/experimental/iterator b/libcxx/include/__cxx03/experimental/iterator
new file mode 100644
index 000000000000..de82da2d3d72
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/iterator
@@ -0,0 +1,127 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL_ITERATOR
+#define _LIBCPP_EXPERIMENTAL_ITERATOR
+
+/*
+namespace std {
+  namespace experimental {
+    inline namespace fundamentals_v2 {
+
+    template <class DelimT, class charT = char, class traits = char_traits<charT>>
+        class ostream_joiner {
+        public:
+         typedef charT                        char_type;
+         typedef traits                       traits_type;
+         typedef basic_ostream<charT, traits> ostream_type;
+         typedef output_iterator_tag          iterator_category;
+         typedef void                         value_type;
+         typedef void                         difference_type;
+         typedef void                         pointer;
+         typedef void                         reference;
+
+         ostream_joiner(ostream_type& s, const DelimT& delimiter);
+         ostream_joiner(ostream_type& s, DelimT&& delimiter);
+
+         template<typename T>
+         ostream_joiner& operator=(const T& value);
+
+         ostream_joiner& operator*() noexcept;
+         ostream_joiner& operator++() noexcept;
+         ostream_joiner& operator++(int) noexcept;
+   private:
+      ostream_type* out_stream;   // exposition only
+      DelimT delim;               // exposition only
+      bool first_element;         // exposition only
+   };
+
+  template <class charT, class traits, class DelimT>
+    ostream_joiner<decay_t<DelimT>, charT, traits>
+    make_ostream_joiner(basic_ostream<charT, traits>& os, DelimT&& delimiter);
+
+    } // inline namespace fundamentals_v2
+  } // namespace experimental
+} // namespace std
+
+*/
+
+#include <__memory/addressof.h>
+#include <__type_traits/decay.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <experimental/__config>
+#include <iterator>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 14
+
+_LIBCPP_BEGIN_NAMESPACE_LFTS
+
+template <class _Delim, class _CharT = char, class _Traits = char_traits<_CharT>>
+class ostream_joiner {
+public:
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef basic_ostream<char_type, traits_type> ostream_type;
+  typedef output_iterator_tag iterator_category;
+  typedef void value_type;
+  typedef void difference_type;
+  typedef void pointer;
+  typedef void reference;
+
+  _LIBCPP_HIDE_FROM_ABI ostream_joiner(ostream_type& __os, _Delim&& __d)
+      : __output_iter_(std::addressof(__os)), __delim_(std::move(__d)), __first_(true) {}
+
+  _LIBCPP_HIDE_FROM_ABI ostream_joiner(ostream_type& __os, const _Delim& __d)
+      : __output_iter_(std::addressof(__os)), __delim_(__d), __first_(true) {}
+
+  template <typename _Tp>
+  _LIBCPP_HIDE_FROM_ABI ostream_joiner& operator=(const _Tp& __v) {
+    if (!__first_)
+      *__output_iter_ << __delim_;
+    __first_ = false;
+    *__output_iter_ << __v;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ostream_joiner& operator*() _NOEXCEPT { return *this; }
+  _LIBCPP_HIDE_FROM_ABI ostream_joiner& operator++() _NOEXCEPT { return *this; }
+  _LIBCPP_HIDE_FROM_ABI ostream_joiner& operator++(int) _NOEXCEPT { return *this; }
+
+private:
+  ostream_type* __output_iter_;
+  _Delim __delim_;
+  bool __first_;
+};
+
+template <class _CharT, class _Traits, class _Delim>
+_LIBCPP_HIDE_FROM_ABI ostream_joiner<__decay_t<_Delim>, _CharT, _Traits>
+make_ostream_joiner(basic_ostream<_CharT, _Traits>& __os, _Delim&& __d) {
+  return ostream_joiner<__decay_t<_Delim>, _CharT, _Traits>(__os, std::forward<_Delim>(__d));
+}
+
+_LIBCPP_END_NAMESPACE_LFTS
+
+#endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <iosfwd>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_EXPERIMENTAL_ITERATOR
diff --git a/libcxx/include/__cxx03/experimental/memory b/libcxx/include/__cxx03/experimental/memory
new file mode 100644
index 000000000000..e9663d43a8ab
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/memory
@@ -0,0 +1,198 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL_MEMORY
+#define _LIBCPP_EXPERIMENTAL_MEMORY
+
+/*
+    experimental/memory synopsis
+
+namespace std::experimental::inline fundamentals_v2  {
+
+template <class W> class observer_ptr {
+public:
+    using element_type = W;
+    using pointer = add_pointer_t<W>; // exposition-only
+    using reference = add_lvalue_reference_t<W>; // exposition-only
+
+    // default ctor
+    constexpr observer_ptr() noexcept;
+
+    // pointer-accepting ctors
+    constexpr observer_ptr(nullptr_t) noexcept;
+    constexpr explicit observer_ptr(pointer) noexcept;
+
+    // copying ctors (in addition to compiler-generated copy ctor)
+    template <class W2> constexpr observer_ptr(observer_ptr<W2>) noexcept;
+
+    // observers
+    constexpr pointer get() const noexcept;
+    constexpr reference operator*() const;
+    constexpr pointer operator->() const noexcept;
+    constexpr explicit operator bool() const noexcept;
+
+    // conversions
+    constexpr explicit operator pointer() const noexcept;
+
+    // modifiers
+    constexpr pointer release() noexcept;
+    constexpr void reset(pointer = nullptr) noexcept;
+    constexpr void swap(observer_ptr&) noexcept;
+};
+
+}
+*/
+
+#include <__functional/hash.h>
+#include <__functional/operations.h>
+#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_pointer.h>
+#include <__type_traits/common_type.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_convertible.h>
+#include <cstddef>
+#include <experimental/__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#ifdef _LIBCPP_ENABLE_EXPERIMENTAL
+
+_LIBCPP_BEGIN_NAMESPACE_LFTS_V2
+
+#  if _LIBCPP_STD_VER >= 17
+
+template <class _Wp>
+class observer_ptr {
+public:
+  using element_type = _Wp;
+
+  // constructors
+  _LIBCPP_HIDE_FROM_ABI constexpr observer_ptr() noexcept : __ptr_(nullptr) {}
+  _LIBCPP_HIDE_FROM_ABI constexpr observer_ptr(nullptr_t) noexcept : __ptr_(nullptr) {}
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit observer_ptr(element_type* __p) noexcept : __ptr_(__p) {}
+
+  template <class _W2, __enable_if_t<is_convertible<_W2*, _Wp*>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI constexpr observer_ptr(observer_ptr<_W2> __other) noexcept : __ptr_(__other.get()) {}
+
+  // observers
+  _LIBCPP_HIDE_FROM_ABI constexpr element_type* get() const noexcept { return __ptr_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr add_lvalue_reference_t<_Wp> operator*() const { return *__ptr_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr element_type* operator->() const noexcept { return __ptr_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return __ptr_ != nullptr; }
+
+  // conversions
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit operator element_type*() const noexcept { return __ptr_; }
+
+  // modifiers
+  _LIBCPP_HIDE_FROM_ABI constexpr void reset(element_type* __p = nullptr) noexcept { __ptr_ = __p; }
+  _LIBCPP_HIDE_FROM_ABI constexpr void swap(observer_ptr& __other) noexcept {
+    observer_ptr __tmp = __other;
+    __other            = *this;
+    *this              = __tmp;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr element_type* release() noexcept {
+    observer_ptr __p;
+    __p.swap(*this);
+    return __p.get();
+  }
+
+private:
+  element_type* __ptr_;
+};
+
+// specializations
+
+template <class _Wp>
+_LIBCPP_HIDE_FROM_ABI constexpr void swap(observer_ptr<_Wp>& __a, observer_ptr<_Wp>& __b) noexcept {
+  __a.swap(__b);
+}
+
+template <class _Wp>
+_LIBCPP_HIDE_FROM_ABI observer_ptr<_Wp> make_observer(_Wp* __ptr) noexcept {
+  return observer_ptr<_Wp>{__ptr};
+}
+
+template <class _W1, class _W2>
+_LIBCPP_HIDE_FROM_ABI bool operator==(observer_ptr<_W1> __a, observer_ptr<_W2> __b) {
+  return __a.get() == __b.get();
+}
+
+template <class _W1, class _W2>
+_LIBCPP_HIDE_FROM_ABI bool operator!=(observer_ptr<_W1> __a, observer_ptr<_W2> __b) {
+  return !(__a == __b);
+}
+
+template <class _Wp>
+_LIBCPP_HIDE_FROM_ABI bool operator==(observer_ptr<_Wp> __p, nullptr_t) {
+  return !__p;
+}
+
+template <class _Wp>
+_LIBCPP_HIDE_FROM_ABI bool operator==(nullptr_t, observer_ptr<_Wp> __p) {
+  return !__p;
+}
+
+template <class _Wp>
+_LIBCPP_HIDE_FROM_ABI bool operator!=(observer_ptr<_Wp> __p, nullptr_t) {
+  return (bool)__p;
+}
+
+template <class _Wp>
+_LIBCPP_HIDE_FROM_ABI bool operator!=(nullptr_t, observer_ptr<_Wp> __p) {
+  return (bool)__p;
+}
+
+template <class _W1, class _W2>
+_LIBCPP_HIDE_FROM_ABI bool operator<(observer_ptr<_W1> __a, observer_ptr<_W2> __b) {
+  return std::less<typename std::common_type<_W1*, _W2*>::type>()(__a.get(), __b.get());
+}
+
+template <class _W1, class _W2>
+_LIBCPP_HIDE_FROM_ABI bool operator>(observer_ptr<_W1> __a, observer_ptr<_W2> __b) {
+  return __b < __a;
+}
+
+template <class _W1, class _W2>
+_LIBCPP_HIDE_FROM_ABI bool operator<=(observer_ptr<_W1> __a, observer_ptr<_W2> __b) {
+  return !(__a > __b);
+}
+
+template <class _W1, class _W2>
+_LIBCPP_HIDE_FROM_ABI bool operator>=(observer_ptr<_W1> __a, observer_ptr<_W2> __b) {
+  return !(__a < __b);
+}
+
+#  endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_LFTS_V2
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// hash
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+struct hash<experimental::observer_ptr<_Tp>> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const experimental::observer_ptr<_Tp>& __ptr) const noexcept {
+    return hash<_Tp*>()(__ptr.get());
+  }
+};
+#  endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_ENABLE_EXPERIMENTAL
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <limits>
+#endif
+
+#endif /* _LIBCPP_EXPERIMENTAL_MEMORY */
diff --git a/libcxx/include/__cxx03/experimental/propagate_const b/libcxx/include/__cxx03/experimental/propagate_const
new file mode 100644
index 000000000000..d7a695d83889
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/propagate_const
@@ -0,0 +1,490 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL_PROPAGATE_CONST
+#define _LIBCPP_EXPERIMENTAL_PROPAGATE_CONST
+
+/*
+    propagate_const synopsis
+
+    namespace std { namespace experimental { inline namespace fundamentals_v2 {
+
+    // [propagate_const]
+    template <class T> class propagate_const;
+
+    // [propagate_const.underlying], underlying pointer access
+    constexpr const _Tp& get_underlying(const propagate_const<T>& pt) noexcept;
+    constexpr T& get_underlying(propagate_const<T>& pt) noexcept;
+
+    // [propagate_const.relational], relational operators
+    template <class T> constexpr bool operator==(const propagate_const<T>& pt, nullptr_t);
+    template <class T> constexpr bool operator==(nullptr_t, const propagate_const<T>& pu);
+    template <class T> constexpr bool operator!=(const propagate_const<T>& pt, nullptr_t);
+    template <class T> constexpr bool operator!=(nullptr_t, const propagate_const<T>& pu);
+    template <class T, class U> constexpr bool operator==(const propagate_const<T>& pt, const propagate_const<_Up>& pu);
+    template <class T, class U> constexpr bool operator!=(const propagate_const<T>& pt, const propagate_const<_Up>& pu);
+    template <class T, class U> constexpr bool operator<(const propagate_const<T>& pt, const propagate_const<_Up>& pu);
+    template <class T, class U> constexpr bool operator>(const propagate_const<T>& pt, const propagate_const<_Up>& pu);
+    template <class T, class U> constexpr bool operator<=(const propagate_const<T>& pt, const propagate_const<_Up>& pu);
+    template <class T, class U> constexpr bool operator>=(const propagate_const<T>& pt, const propagate_const<_Up>& pu);
+    template <class T, class U> constexpr bool operator==(const propagate_const<T>& pt, const _Up& u);
+    template <class T, class U> constexpr bool operator!=(const propagate_const<T>& pt, const _Up& u);
+    template <class T, class U> constexpr bool operator<(const propagate_const<T>& pt, const _Up& u);
+    template <class T, class U> constexpr bool operator>(const propagate_const<T>& pt, const _Up& u);
+    template <class T, class U> constexpr bool operator<=(const propagate_const<T>& pt, const _Up& u);
+    template <class T, class U> constexpr bool operator>=(const propagate_const<T>& pt, const _Up& u);
+    template <class T, class U> constexpr bool operator==(const _Tp& t, const propagate_const<_Up>& pu);
+    template <class T, class U> constexpr bool operator!=(const _Tp& t, const propagate_const<_Up>& pu);
+    template <class T, class U> constexpr bool operator<(const _Tp& t, const propagate_const<_Up>& pu);
+    template <class T, class U> constexpr bool operator>(const _Tp& t, const propagate_const<_Up>& pu);
+    template <class T, class U> constexpr bool operator<=(const _Tp& t, const propagate_const<_Up>& pu);
+    template <class T, class U> constexpr bool operator>=(const _Tp& t, const propagate_const<_Up>& pu);
+
+    // [propagate_const.algorithms], specialized algorithms
+    template <class T> constexpr void swap(propagate_const<T>& pt, propagate_const<T>& pu) noexcept(see below);
+
+    template <class T>
+    class propagate_const
+    {
+
+    public:
+      typedef remove_reference_t<decltype(*declval<T&>())> element_type;
+
+      // [propagate_const.ctor], constructors
+      constexpr propagate_const() = default;
+      propagate_const(const propagate_const& p) = delete;
+      constexpr propagate_const(propagate_const&& p) = default;
+      template <class U> EXPLICIT constexpr propagate_const(propagate_const<_Up>&& pu); // see below
+      template <class U> EXPLICIT constexpr propagate_const(U&& u); // see below
+
+      // [propagate_const.assignment], assignment
+      propagate_const& operator=(const propagate_const& p) = delete;
+      constexpr propagate_const& operator=(propagate_const&& p) = default;
+      template <class U> constexpr propagate_const& operator=(propagate_const<_Up>&& pu);
+      template <class U> constexpr propagate_const& operator=(U&& u); // see below
+
+      // [propagate_const.const_observers], const observers
+      explicit constexpr operator bool() const;
+      constexpr const element_type* operator->() const;
+      constexpr operator const element_type*() const; // Not always defined
+      constexpr const element_type& operator*() const;
+      constexpr const element_type* get() const;
+
+      // [propagate_const.non_const_observers], non-const observers
+      constexpr element_type* operator->();
+      constexpr operator element_type*(); // Not always defined
+      constexpr element_type& operator*();
+      constexpr element_type* get();
+
+      // [propagate_const.modifiers], modifiers
+      constexpr void swap(propagate_const& pt) noexcept(see below)
+
+    private:
+      T t_; // exposition only
+    };
+
+  } // namespace fundamentals_v2
+  } // namespace experimental
+
+  // [propagate_const.hash], hash support
+  template <class T> struct hash<experimental::propagate_const<T>>;
+
+  // [propagate_const.comparison_function_objects], comparison function objects
+  template <class T> struct equal_to<experimental::propagate_const<T>>;
+  template <class T> struct not_equal_to<experimental::propagate_const<T>>;
+  template <class T> struct less<experimental::propagate_const<T>>;
+  template <class T> struct greater<experimental::propagate_const<T>>;
+  template <class T> struct less_equal<experimental::propagate_const<T>>;
+  template <class T> struct greater_equal<experimental::propagate_const<T>>;
+
+} // namespace std
+
+*/
+
+#include <__functional/operations.h>
+#include <__fwd/functional.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_function.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_swappable.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/remove_pointer.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+#include <cstddef>
+#include <experimental/__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 14
+
+_LIBCPP_BEGIN_NAMESPACE_LFTS_V2
+
+template <class _Tp>
+class propagate_const;
+
+template <class _Up>
+inline _LIBCPP_HIDE_FROM_ABI constexpr const _Up& get_underlying(const propagate_const<_Up>& __pu) _NOEXCEPT;
+
+template <class _Up>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _Up& get_underlying(propagate_const<_Up>& __pu) _NOEXCEPT;
+
+template <class _Tp>
+class propagate_const {
+public:
+  typedef remove_reference_t<decltype(*std::declval<_Tp&>())> element_type;
+
+  static_assert(!is_array<_Tp>::value, "Instantiation of propagate_const with an array type is ill-formed.");
+  static_assert(!is_reference<_Tp>::value, "Instantiation of propagate_const with a reference type is ill-formed.");
+  static_assert(!(is_pointer<_Tp>::value && is_function<__remove_pointer_t<_Tp> >::value),
+                "Instantiation of propagate_const with a function-pointer type is ill-formed.");
+  static_assert(!(is_pointer<_Tp>::value && is_same<__remove_cv_t<__remove_pointer_t<_Tp> >, void>::value),
+                "Instantiation of propagate_const with a pointer to (possibly cv-qualified) void is ill-formed.");
+
+private:
+  template <class _Up>
+  static _LIBCPP_HIDE_FROM_ABI constexpr element_type* __get_pointer(_Up* __u) {
+    return __u;
+  }
+
+  template <class _Up>
+  static _LIBCPP_HIDE_FROM_ABI constexpr element_type* __get_pointer(_Up& __u) {
+    return __get_pointer(__u.get());
+  }
+
+  template <class _Up>
+  static _LIBCPP_HIDE_FROM_ABI constexpr const element_type* __get_pointer(const _Up* __u) {
+    return __u;
+  }
+
+  template <class _Up>
+  static _LIBCPP_HIDE_FROM_ABI constexpr const element_type* __get_pointer(const _Up& __u) {
+    return __get_pointer(__u.get());
+  }
+
+  template <class _Up>
+  struct __is_propagate_const : false_type {};
+
+  template <class _Up>
+  struct __is_propagate_const<propagate_const<_Up>> : true_type {};
+
+  _Tp __t_;
+
+public:
+  template <class _Up>
+  friend constexpr const _Up& experimental::fundamentals_v2::get_underlying(const propagate_const<_Up>& __pu) _NOEXCEPT;
+  template <class _Up>
+  friend constexpr _Up& experimental::fundamentals_v2::get_underlying(propagate_const<_Up>& __pu) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr propagate_const() = default;
+
+  propagate_const(const propagate_const&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr propagate_const(propagate_const&&) = default;
+
+  template <class _Up,
+            enable_if_t<!is_convertible<_Up, _Tp>::value && is_constructible<_Tp, _Up&&>::value, bool> = true>
+  explicit _LIBCPP_HIDE_FROM_ABI constexpr propagate_const(propagate_const<_Up>&& __pu)
+      : __t_(std::move(experimental::get_underlying(__pu))) {}
+
+  template <class _Up,
+            enable_if_t<is_convertible<_Up&&, _Tp>::value && is_constructible<_Tp, _Up&&>::value, bool> = false>
+  _LIBCPP_HIDE_FROM_ABI constexpr propagate_const(propagate_const<_Up>&& __pu)
+      : __t_(std::move(experimental::get_underlying(__pu))) {}
+
+  template <class _Up,
+            enable_if_t<!is_convertible<_Up&&, _Tp>::value && is_constructible<_Tp, _Up&&>::value &&
+                            !__is_propagate_const<decay_t<_Up>>::value,
+                        bool> = true>
+  explicit _LIBCPP_HIDE_FROM_ABI constexpr propagate_const(_Up&& __u) : __t_(std::forward<_Up>(__u)) {}
+
+  template <class _Up,
+            enable_if_t<is_convertible<_Up&&, _Tp>::value && is_constructible<_Tp, _Up&&>::value &&
+                            !__is_propagate_const<decay_t<_Up>>::value,
+                        bool> = false>
+  _LIBCPP_HIDE_FROM_ABI constexpr propagate_const(_Up&& __u) : __t_(std::forward<_Up>(__u)) {}
+
+  propagate_const& operator=(const propagate_const&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr propagate_const& operator=(propagate_const&&) = default;
+
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI constexpr propagate_const& operator=(propagate_const<_Up>&& __pu) {
+    __t_ = std::move(experimental::get_underlying(__pu));
+    return *this;
+  }
+
+  template <class _Up, class _Vp = enable_if_t<!__is_propagate_const<decay_t<_Up>>::value>>
+  _LIBCPP_HIDE_FROM_ABI constexpr propagate_const& operator=(_Up&& __u) {
+    __t_ = std::forward<_Up>(__u);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const element_type* get() const { return __get_pointer(__t_); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr element_type* get() { return __get_pointer(__t_); }
+
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr operator bool() const { return get() != nullptr; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const element_type* operator->() const { return get(); }
+
+  template <class _Dummy = _Tp, class _Up = enable_if_t<is_convertible< const _Dummy, const element_type*>::value>>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator const element_type*() const {
+    return get();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const element_type& operator*() const { return *get(); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr element_type* operator->() { return get(); }
+
+  template <class _Dummy = _Tp, class _Up = enable_if_t< is_convertible<_Dummy, element_type*>::value>>
+  _LIBCPP_HIDE_FROM_ABI constexpr operator element_type*() {
+    return get();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr element_type& operator*() { return *get(); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr void swap(propagate_const& __pt) noexcept(__is_nothrow_swappable_v<_Tp>) {
+    using std::swap;
+    swap(__t_, __pt.__t_);
+  }
+};
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator==(const propagate_const<_Tp>& __pt, nullptr_t) {
+  return experimental::get_underlying(__pt) == nullptr;
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator==(nullptr_t, const propagate_const<_Tp>& __pt) {
+  return nullptr == experimental::get_underlying(__pt);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator!=(const propagate_const<_Tp>& __pt, nullptr_t) {
+  return experimental::get_underlying(__pt) != nullptr;
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator!=(nullptr_t, const propagate_const<_Tp>& __pt) {
+  return nullptr != experimental::get_underlying(__pt);
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator==(const propagate_const<_Tp>& __pt, const propagate_const<_Up>& __pu) {
+  return experimental::get_underlying(__pt) == experimental::get_underlying(__pu);
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator!=(const propagate_const<_Tp>& __pt, const propagate_const<_Up>& __pu) {
+  return experimental::get_underlying(__pt) != experimental::get_underlying(__pu);
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<(const propagate_const<_Tp>& __pt, const propagate_const<_Up>& __pu) {
+  return experimental::get_underlying(__pt) < experimental::get_underlying(__pu);
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>(const propagate_const<_Tp>& __pt, const propagate_const<_Up>& __pu) {
+  return experimental::get_underlying(__pt) > experimental::get_underlying(__pu);
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<=(const propagate_const<_Tp>& __pt, const propagate_const<_Up>& __pu) {
+  return experimental::get_underlying(__pt) <= experimental::get_underlying(__pu);
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>=(const propagate_const<_Tp>& __pt, const propagate_const<_Up>& __pu) {
+  return experimental::get_underlying(__pt) >= experimental::get_underlying(__pu);
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator==(const propagate_const<_Tp>& __pt, const _Up& __u) {
+  return experimental::get_underlying(__pt) == __u;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator!=(const propagate_const<_Tp>& __pt, const _Up& __u) {
+  return experimental::get_underlying(__pt) != __u;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<(const propagate_const<_Tp>& __pt, const _Up& __u) {
+  return experimental::get_underlying(__pt) < __u;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>(const propagate_const<_Tp>& __pt, const _Up& __u) {
+  return experimental::get_underlying(__pt) > __u;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<=(const propagate_const<_Tp>& __pt, const _Up& __u) {
+  return experimental::get_underlying(__pt) <= __u;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>=(const propagate_const<_Tp>& __pt, const _Up& __u) {
+  return experimental::get_underlying(__pt) >= __u;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator==(const _Tp& __t, const propagate_const<_Up>& __pu) {
+  return __t == experimental::get_underlying(__pu);
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator!=(const _Tp& __t, const propagate_const<_Up>& __pu) {
+  return __t != experimental::get_underlying(__pu);
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<(const _Tp& __t, const propagate_const<_Up>& __pu) {
+  return __t < experimental::get_underlying(__pu);
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>(const _Tp& __t, const propagate_const<_Up>& __pu) {
+  return __t > experimental::get_underlying(__pu);
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<=(const _Tp& __t, const propagate_const<_Up>& __pu) {
+  return __t <= experimental::get_underlying(__pu);
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>=(const _Tp& __t, const propagate_const<_Up>& __pu) {
+  return __t >= experimental::get_underlying(__pu);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr void
+swap(propagate_const<_Tp>& __pc1, propagate_const<_Tp>& __pc2) noexcept(__is_nothrow_swappable_v<_Tp>) {
+  __pc1.swap(__pc2);
+}
+
+template <class _Tp>
+constexpr const _Tp& get_underlying(const propagate_const<_Tp>& __pt) _NOEXCEPT {
+  return __pt.__t_;
+}
+
+template <class _Tp>
+constexpr _Tp& get_underlying(propagate_const<_Tp>& __pt) _NOEXCEPT {
+  return __pt.__t_;
+}
+
+_LIBCPP_END_NAMESPACE_LFTS_V2
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct hash<experimental::propagate_const<_Tp>> {
+  typedef size_t result_type;
+  typedef experimental::propagate_const<_Tp> argument_type;
+
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const experimental::propagate_const<_Tp>& __pc1) const {
+    return std::hash<_Tp>()(experimental::get_underlying(__pc1));
+  }
+};
+
+template <class _Tp>
+struct equal_to<experimental::propagate_const<_Tp>> {
+  typedef experimental::propagate_const<_Tp> first_argument_type;
+  typedef experimental::propagate_const<_Tp> second_argument_type;
+
+  _LIBCPP_HIDE_FROM_ABI bool
+  operator()(const experimental::propagate_const<_Tp>& __pc1, const experimental::propagate_const<_Tp>& __pc2) const {
+    return std::equal_to<_Tp>()(experimental::get_underlying(__pc1), experimental::get_underlying(__pc2));
+  }
+};
+
+template <class _Tp>
+struct not_equal_to<experimental::propagate_const<_Tp>> {
+  typedef experimental::propagate_const<_Tp> first_argument_type;
+  typedef experimental::propagate_const<_Tp> second_argument_type;
+
+  _LIBCPP_HIDE_FROM_ABI bool
+  operator()(const experimental::propagate_const<_Tp>& __pc1, const experimental::propagate_const<_Tp>& __pc2) const {
+    return std::not_equal_to<_Tp>()(experimental::get_underlying(__pc1), experimental::get_underlying(__pc2));
+  }
+};
+
+template <class _Tp>
+struct less<experimental::propagate_const<_Tp>> {
+  typedef experimental::propagate_const<_Tp> first_argument_type;
+  typedef experimental::propagate_const<_Tp> second_argument_type;
+
+  _LIBCPP_HIDE_FROM_ABI bool
+  operator()(const experimental::propagate_const<_Tp>& __pc1, const experimental::propagate_const<_Tp>& __pc2) const {
+    return std::less<_Tp>()(experimental::get_underlying(__pc1), experimental::get_underlying(__pc2));
+  }
+};
+
+template <class _Tp>
+struct greater<experimental::propagate_const<_Tp>> {
+  typedef experimental::propagate_const<_Tp> first_argument_type;
+  typedef experimental::propagate_const<_Tp> second_argument_type;
+
+  _LIBCPP_HIDE_FROM_ABI bool
+  operator()(const experimental::propagate_const<_Tp>& __pc1, const experimental::propagate_const<_Tp>& __pc2) const {
+    return std::greater<_Tp>()(experimental::get_underlying(__pc1), experimental::get_underlying(__pc2));
+  }
+};
+
+template <class _Tp>
+struct less_equal<experimental::propagate_const<_Tp>> {
+  typedef experimental::propagate_const<_Tp> first_argument_type;
+  typedef experimental::propagate_const<_Tp> second_argument_type;
+
+  _LIBCPP_HIDE_FROM_ABI bool
+  operator()(const experimental::propagate_const<_Tp>& __pc1, const experimental::propagate_const<_Tp>& __pc2) const {
+    return std::less_equal<_Tp>()(experimental::get_underlying(__pc1), experimental::get_underlying(__pc2));
+  }
+};
+
+template <class _Tp>
+struct greater_equal<experimental::propagate_const<_Tp>> {
+  typedef experimental::propagate_const<_Tp> first_argument_type;
+  typedef experimental::propagate_const<_Tp> second_argument_type;
+
+  _LIBCPP_HIDE_FROM_ABI bool
+  operator()(const experimental::propagate_const<_Tp>& __pc1, const experimental::propagate_const<_Tp>& __pc2) const {
+    return std::greater_equal<_Tp>()(experimental::get_underlying(__pc1), experimental::get_underlying(__pc2));
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_EXPERIMENTAL_PROPAGATE_CONST
diff --git a/libcxx/include/__cxx03/experimental/simd b/libcxx/include/__cxx03/experimental/simd
new file mode 100644
index 000000000000..484543b81daf
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/simd
@@ -0,0 +1,88 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL_SIMD
+#define _LIBCPP_EXPERIMENTAL_SIMD
+
+/*
+    experimental/simd synopsis
+
+namespace std::experimental {
+inline namespace parallelism_v2 {
+  namespace simd_abi {
+    using scalar = see below;
+    template<int N> using fixed_size = see below;
+    template<class T> inline constexpr int max_fixed_size = implementation-defined;
+    template<class T> using compatible = implementation-defined;
+    template<class T> using native = implementation-defined;
+
+    template<class T, size_t N, class... Abis> struct deduce { using type = see below; };
+    template<class T, size_t N, class... Abis> using deduce_t =
+      typename deduce<T, N, Abis...>::type;
+  } // namespace simd_abi
+
+  // class template simd [simd.class]
+  template <class T, class Abi = simd_abi::compatible<T>> class simd;
+  template <class T> using native_simd = simd<T, simd_abi::native<T>>;
+  template <class T, int N> using fixed_size_simd = simd<T, simd_abi::fixed_size<N>>;
+
+  // class template simd_mask [simd.mask.class]
+  template <class T, class Abi = simd_abi::compatible<T>> class simd_mask;
+  template <class T> using native_simd_mask = simd_mask<T, simd_abi::native<T>>;
+  template <class T, int N> using fixed_size_simd_mask = simd_mask<T, simd_abi::fixed_size<N>>;
+
+  // memory alignment
+  struct element_aligned_tag {};
+  struct vector_aligned_tag {};
+  template <size_t> struct overaligned_tag {};
+  inline constexpr element_aligned_tag element_aligned{};
+  inline constexpr vector_aligned_tag vector_aligned{};
+  template <size_t N> inline constexpr overaligned_tag<N> overaligned{};
+
+  // traits [simd.traits]
+  template<class T> struct is_abi_tag;
+  template<class T> inline constexpr bool is_abi_tag_v = is_abi_tag<T>::value;
+
+  template <class T> struct is_simd;
+  template <class T> inline constexpr bool is_simd_v = is_simd<T>::value;
+
+  template <class T> struct is_simd_mask;
+  template <class T> inline constexpr bool is_simd_mask_v = is_simd_mask<T>::value;
+
+  template<class T> struct is_simd_flag_type;
+  template<class T> inline constexpr bool is_simd_flag_type_v = is_simd_flag_type<T>::value;
+
+  template<class T, class Abi = simd_abi::compatible<T>> struct simd_size;
+  template<class T, class Abi = simd_abi::compatible<T>>
+    inline constexpr size_t simd_size_v = simd_size<T,Abi>::value;
+
+  template<class T, class U = typename T::value_type> struct memory_alignment;
+  template<class T, class U = typename T::value_type>
+    inline constexpr size_t memory_alignment_v = memory_alignment<T,U>::value;
+
+} // namespace parallelism_v2
+} // namespace std::experimental
+
+*/
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#include <experimental/__config>
+#include <experimental/__simd/aligned_tag.h>
+#include <experimental/__simd/declaration.h>
+#include <experimental/__simd/reference.h>
+#include <experimental/__simd/scalar.h>
+#include <experimental/__simd/simd.h>
+#include <experimental/__simd/simd_mask.h>
+#include <experimental/__simd/traits.h>
+#include <experimental/__simd/vec_ext.h>
+
+#endif /* _LIBCPP_EXPERIMENTAL_SIMD */
diff --git a/libcxx/include/__cxx03/experimental/type_traits b/libcxx/include/__cxx03/experimental/type_traits
new file mode 100644
index 000000000000..31b041bc94c4
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/type_traits
@@ -0,0 +1,153 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL_TYPE_TRAITS
+#define _LIBCPP_EXPERIMENTAL_TYPE_TRAITS
+
+/**
+    experimental/type_traits synopsis
+
+// C++1y
+#include <type_traits>
+
+namespace std {
+namespace experimental {
+inline namespace fundamentals_v1 {
+
+  // 3.3.2, Other type transformations
+  template <class> class invocation_type; // not defined
+  template <class F, class... ArgTypes> class invocation_type<F(ArgTypes...)>;
+  template <class> class raw_invocation_type; // not defined
+  template <class F, class... ArgTypes> class raw_invocation_type<F(ArgTypes...)>;
+
+  template <class T>
+    using invocation_type_t = typename invocation_type<T>::type;
+  template <class T>
+    using raw_invocation_type_t = typename raw_invocation_type<T>::type;
+
+  // 3.3.4, Detection idiom
+  template <class...> using void_t = void;
+
+  struct nonesuch {
+    nonesuch() = delete;
+    ~nonesuch() = delete;
+    nonesuch(nonesuch const&) = delete;
+    void operator=(nonesuch const&) = delete;
+  };
+
+  template <template<class...> class Op, class... Args>
+    using is_detected = see below;
+  template <template<class...> class Op, class... Args>
+    constexpr bool is_detected_v = is_detected<Op, Args...>::value;
+  template <template<class...> class Op, class... Args>
+    using detected_t = see below;
+  template <class Default, template<class...> class Op, class... Args>
+    using detected_or = see below;
+  template <class Default, template<class...> class Op, class... Args>
+    using detected_or_t = typename detected_or<Default, Op, Args...>::type;
+  template <class Expected, template<class...> class Op, class... Args>
+    using is_detected_exact = is_same<Expected, detected_t<Op, Args...>>;
+  template <class Expected, template<class...> class Op, class... Args>
+    constexpr bool is_detected_exact_v
+      = is_detected_exact<Expected, Op, Args...>::value;
+  template <class To, template<class...> class Op, class... Args>
+     using is_detected_convertible = is_convertible<detected_t<Op, Args...>, To>;
+  template <class To, template<class...> class Op, class... Args>
+     constexpr bool is_detected_convertible_v
+       = is_detected_convertible<To, Op, Args...>::value;
+
+} // namespace fundamentals_v1
+} // namespace experimental
+} // namespace std
+
+ */
+
+#include <experimental/__config>
+
+#if _LIBCPP_STD_VER >= 14
+
+#  include <initializer_list>
+#  include <type_traits>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_LFTS
+
+// 3.3.2, Other type transformations
+/*
+template <class>
+class _LIBCPP_TEMPLATE_VIS raw_invocation_type;
+
+template <class _Fn, class ..._Args>
+class _LIBCPP_TEMPLATE_VIS raw_invocation_type<_Fn(_Args...)>;
+
+template <class>
+class _LIBCPP_TEMPLATE_VIS invokation_type;
+
+template <class _Fn, class ..._Args>
+class _LIBCPP_TEMPLATE_VIS invokation_type<_Fn(_Args...)>;
+
+template <class _Tp>
+using invokation_type_t = typename invokation_type<_Tp>::type;
+
+template <class _Tp>
+using raw_invocation_type_t = typename raw_invocation_type<_Tp>::type;
+*/
+
+// 3.3.4, Detection idiom
+template <class...>
+using void_t = void;
+
+struct nonesuch : private __nat { // make nonesuch "not an aggregate"
+  ~nonesuch()                     = delete;
+  nonesuch(nonesuch const&)       = delete;
+  void operator=(nonesuch const&) = delete;
+};
+
+template <class _Default, class _AlwaysVoid, template <class...> class _Op, class... _Args>
+struct _DETECTOR {
+  using value_t = false_type;
+  using type    = _Default;
+};
+
+template <class _Default, template <class...> class _Op, class... _Args>
+struct _DETECTOR<_Default, void_t<_Op<_Args...>>, _Op, _Args...> {
+  using value_t = true_type;
+  using type    = _Op<_Args...>;
+};
+
+template <template <class...> class _Op, class... _Args>
+using is_detected = typename _DETECTOR<nonesuch, void, _Op, _Args...>::value_t;
+template <template <class...> class _Op, class... _Args>
+using detected_t = typename _DETECTOR<nonesuch, void, _Op, _Args...>::type;
+template <template <class...> class _Op, class... _Args>
+constexpr bool is_detected_v = is_detected<_Op, _Args...>::value;
+
+template <class _Default, template <class...> class _Op, class... _Args>
+using detected_or = _DETECTOR<_Default, void, _Op, _Args...>;
+template <class _Default, template <class...> class _Op, class... _Args>
+using detected_or_t = typename detected_or<_Default, _Op, _Args...>::type;
+
+template <class _Expected, template <class...> class _Op, class... _Args>
+using is_detected_exact = is_same<_Expected, detected_t<_Op, _Args...>>;
+template <class _Expected, template <class...> class _Op, class... _Args>
+constexpr bool is_detected_exact_v = is_detected_exact<_Expected, _Op, _Args...>::value;
+
+template <class _To, template <class...> class _Op, class... _Args>
+using is_detected_convertible = is_convertible<detected_t<_Op, _Args...>, _To>;
+template <class _To, template <class...> class _Op, class... _Args>
+constexpr bool is_detected_convertible_v = is_detected_convertible<_To, _Op, _Args...>::value;
+
+_LIBCPP_END_NAMESPACE_LFTS
+
+#endif /* _LIBCPP_STD_VER >= 14 */
+
+#endif /* _LIBCPP_EXPERIMENTAL_TYPE_TRAITS */
diff --git a/libcxx/include/__cxx03/experimental/utility b/libcxx/include/__cxx03/experimental/utility
new file mode 100644
index 000000000000..8bd0a055b778
--- /dev/null
+++ b/libcxx/include/__cxx03/experimental/utility
@@ -0,0 +1,46 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXPERIMENTAL_UTILITY
+#define _LIBCPP_EXPERIMENTAL_UTILITY
+
+/*
+    experimental/utility synopsis
+
+// C++1y
+
+#include <utility>
+
+namespace std {
+namespace experimental {
+inline namespace fundamentals_v1 {
+
+  3.1.2, erased-type placeholder
+  struct erased_type { };
+
+} // namespace fundamentals_v1
+} // namespace experimental
+} // namespace std
+
+ */
+
+#include <experimental/__config>
+#include <utility>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_LFTS
+
+struct _LIBCPP_TEMPLATE_VIS erased_type {};
+
+_LIBCPP_END_NAMESPACE_LFTS
+
+#endif /* _LIBCPP_EXPERIMENTAL_UTILITY */
diff --git a/libcxx/include/__cxx03/ext/__hash b/libcxx/include/__cxx03/ext/__hash
new file mode 100644
index 000000000000..67f7e351756f
--- /dev/null
+++ b/libcxx/include/__cxx03/ext/__hash
@@ -0,0 +1,85 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_EXT_HASH
+#define _LIBCPP_EXT_HASH
+
+#pragma GCC system_header
+
+#include <__config>
+#include <cstring>
+#include <stddef.h>
+#include <string>
+
+namespace __gnu_cxx {
+
+template <typename _Tp>
+struct _LIBCPP_TEMPLATE_VIS hash {};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<const char*> : public std::__unary_function<const char*, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const char* __c) const _NOEXCEPT {
+    return std::__do_string_hash(__c, __c + strlen(__c));
+  }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<char*> : public std::__unary_function<char*, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(char* __c) const _NOEXCEPT {
+    return std::__do_string_hash<const char*>(__c, __c + strlen(__c));
+  }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<char> : public std::__unary_function<char, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(char __c) const _NOEXCEPT { return __c; }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<signed char> : public std::__unary_function<signed char, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(signed char __c) const _NOEXCEPT { return __c; }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<unsigned char> : public std::__unary_function<unsigned char, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned char __c) const _NOEXCEPT { return __c; }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<short> : public std::__unary_function<short, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(short __c) const _NOEXCEPT { return __c; }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<unsigned short> : public std::__unary_function<unsigned short, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned short __c) const _NOEXCEPT { return __c; }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<int> : public std::__unary_function<int, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(int __c) const _NOEXCEPT { return __c; }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<unsigned int> : public std::__unary_function<unsigned int, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned int __c) const _NOEXCEPT { return __c; }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<long> : public std::__unary_function<long, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(long __c) const _NOEXCEPT { return __c; }
+};
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<unsigned long> : public std::__unary_function<unsigned long, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned long __c) const _NOEXCEPT { return __c; }
+};
+} // namespace __gnu_cxx
+
+#endif // _LIBCPP_EXT_HASH
diff --git a/libcxx/include/__cxx03/ext/hash_map b/libcxx/include/__cxx03/ext/hash_map
new file mode 100644
index 000000000000..7b5b31c40817
--- /dev/null
+++ b/libcxx/include/__cxx03/ext/hash_map
@@ -0,0 +1,872 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_HASH_MAP
+#define _LIBCPP_HASH_MAP
+
+/*
+
+    hash_map synopsis
+
+namespace __gnu_cxx
+{
+
+template <class Key, class T, class Hash = hash<Key>, class Pred = equal_to<Key>,
+          class Alloc = allocator<pair<const Key, T>>>
+class hash_map
+{
+public:
+    // types
+    typedef Key                                                        key_type;
+    typedef T                                                          mapped_type;
+    typedef Hash                                                       hasher;
+    typedef Pred                                                       key_equal;
+    typedef Alloc                                                      allocator_type;
+    typedef pair<const key_type, mapped_type>                          value_type;
+    typedef value_type&                                                reference;
+    typedef const value_type&                                          const_reference;
+    typedef typename allocator_traits<allocator_type>::pointer         pointer;
+    typedef typename allocator_traits<allocator_type>::const_pointer   const_pointer;
+    typedef typename allocator_traits<allocator_type>::size_type       size_type;
+    typedef typename allocator_traits<allocator_type>::difference_type difference_type;
+
+    typedef /unspecified/ iterator;
+    typedef /unspecified/ const_iterator;
+
+    hash_map();
+    explicit hash_map(size_type n, const hasher& hf = hasher(),
+                           const key_equal& eql = key_equal(),
+                           const allocator_type& a = allocator_type());
+    template <class InputIterator>
+    hash_map(InputIterator f, InputIterator l);
+    template <class InputIterator>
+    hash_map(InputIterator f, InputIterator l,
+                size_type n, const hasher& hf = hasher(),
+                const key_equal& eql = key_equal(),
+                const allocator_type& a = allocator_type());
+    hash_map(const hash_map&);
+    ~hash_map();
+    hash_map& operator=(const hash_map&);
+
+    allocator_type get_allocator() const;
+
+    bool      empty() const;
+    size_type size() const;
+    size_type max_size() const;
+
+    iterator       begin();
+    iterator       end();
+    const_iterator begin()  const;
+    const_iterator end()    const;
+
+    pair<iterator, bool> insert(const value_type& obj);
+    template <class InputIterator>
+        void insert(InputIterator first, InputIterator last);
+
+    void erase(const_iterator position);
+    size_type erase(const key_type& k);
+    void erase(const_iterator first, const_iterator last);
+    void clear();
+
+    void swap(hash_map&);
+
+    hasher hash_funct() const;
+    key_equal key_eq() const;
+
+    iterator       find(const key_type& k);
+    const_iterator find(const key_type& k) const;
+    size_type count(const key_type& k) const;
+    pair<iterator, iterator>             equal_range(const key_type& k);
+    pair<const_iterator, const_iterator> equal_range(const key_type& k) const;
+
+    mapped_type& operator[](const key_type& k);
+
+    size_type bucket_count() const;
+    size_type max_bucket_count() const;
+
+    size_type elems_in_bucket(size_type n) const;
+
+    void resize(size_type n);
+};
+
+template <class Key, class T, class Hash, class Pred, class Alloc>
+    void swap(hash_map<Key, T, Hash, Pred, Alloc>& x,
+              hash_map<Key, T, Hash, Pred, Alloc>& y);
+
+template <class Key, class T, class Hash, class Pred, class Alloc>
+    bool
+    operator==(const hash_map<Key, T, Hash, Pred, Alloc>& x,
+               const hash_map<Key, T, Hash, Pred, Alloc>& y);
+
+template <class Key, class T, class Hash, class Pred, class Alloc>
+    bool
+    operator!=(const hash_map<Key, T, Hash, Pred, Alloc>& x,
+               const hash_map<Key, T, Hash, Pred, Alloc>& y);
+
+template <class Key, class T, class Hash = hash<Key>, class Pred = equal_to<Key>,
+          class Alloc = allocator<pair<const Key, T>>>
+class hash_multimap
+{
+public:
+    // types
+    typedef Key                                                        key_type;
+    typedef T                                                          mapped_type;
+    typedef Hash                                                       hasher;
+    typedef Pred                                                       key_equal;
+    typedef Alloc                                                      allocator_type;
+    typedef pair<const key_type, mapped_type>                          value_type;
+    typedef value_type&                                                reference;
+    typedef const value_type&                                          const_reference;
+    typedef typename allocator_traits<allocator_type>::pointer         pointer;
+    typedef typename allocator_traits<allocator_type>::const_pointer   const_pointer;
+    typedef typename allocator_traits<allocator_type>::size_type       size_type;
+    typedef typename allocator_traits<allocator_type>::difference_type difference_type;
+
+    typedef /unspecified/ iterator;
+    typedef /unspecified/ const_iterator;
+
+    explicit hash_multimap(size_type n = 193, const hasher& hf = hasher(),
+                           const key_equal& eql = key_equal(),
+                           const allocator_type& a = allocator_type());
+    template <class InputIterator>
+        hash_multimap(InputIterator f, InputIterator l,
+                      size_type n = 193, const hasher& hf = hasher(),
+                      const key_equal& eql = key_equal(),
+                      const allocator_type& a = allocator_type());
+    explicit hash_multimap(const allocator_type&);
+    hash_multimap(const hash_multimap&);
+    ~hash_multimap();
+    hash_multimap& operator=(const hash_multimap&);
+
+    allocator_type get_allocator() const;
+
+    bool      empty() const;
+    size_type size() const;
+    size_type max_size() const;
+
+    iterator       begin();
+    iterator       end();
+    const_iterator begin()  const;
+    const_iterator end()    const;
+
+    iterator insert(const value_type& obj);
+    template <class InputIterator>
+        void insert(InputIterator first, InputIterator last);
+
+    void erase(const_iterator position);
+    size_type erase(const key_type& k);
+    void erase(const_iterator first, const_iterator last);
+    void clear();
+
+    void swap(hash_multimap&);
+
+    hasher hash_funct() const;
+    key_equal key_eq() const;
+
+    iterator       find(const key_type& k);
+    const_iterator find(const key_type& k) const;
+    size_type count(const key_type& k) const;
+    pair<iterator, iterator>             equal_range(const key_type& k);
+    pair<const_iterator, const_iterator> equal_range(const key_type& k) const;
+
+    size_type bucket_count() const;
+    size_type max_bucket_count() const;
+
+    size_type elems_in_bucket(size_type n) const;
+
+    void resize(size_type n);
+};
+
+template <class Key, class T, class Hash, class Pred, class Alloc>
+    void swap(hash_multimap<Key, T, Hash, Pred, Alloc>& x,
+              hash_multimap<Key, T, Hash, Pred, Alloc>& y);
+
+template <class Key, class T, class Hash, class Pred, class Alloc>
+    bool
+    operator==(const hash_multimap<Key, T, Hash, Pred, Alloc>& x,
+               const hash_multimap<Key, T, Hash, Pred, Alloc>& y);
+
+template <class Key, class T, class Hash, class Pred, class Alloc>
+    bool
+    operator!=(const hash_multimap<Key, T, Hash, Pred, Alloc>& x,
+               const hash_multimap<Key, T, Hash, Pred, Alloc>& y);
+
+}  // __gnu_cxx
+
+*/
+
+#include <__config>
+#include <__hash_table>
+#include <algorithm>
+#include <ext/__hash>
+#include <functional>
+
+#if defined(__DEPRECATED) && __DEPRECATED
+#  if defined(_LIBCPP_WARNING)
+_LIBCPP_WARNING("Use of the header <ext/hash_map> is deprecated.  Migrate to <unordered_map>")
+#  else
+#    warning Use of the header <ext/hash_map> is deprecated.  Migrate to <unordered_map>
+#  endif
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+namespace __gnu_cxx {
+
+template <class _Tp, class _Hash, bool = std::is_empty<_Hash>::value && !std::__libcpp_is_final<_Hash>::value >
+class __hash_map_hasher : private _Hash {
+public:
+  _LIBCPP_HIDE_FROM_ABI __hash_map_hasher() : _Hash() {}
+  _LIBCPP_HIDE_FROM_ABI __hash_map_hasher(const _Hash& __h) : _Hash(__h) {}
+  _LIBCPP_HIDE_FROM_ABI const _Hash& hash_function() const { return *this; }
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Tp& __x) const { return static_cast<const _Hash&>(*this)(__x.first); }
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const typename _Tp::first_type& __x) const {
+    return static_cast<const _Hash&>(*this)(__x);
+  }
+};
+
+template <class _Tp, class _Hash>
+class __hash_map_hasher<_Tp, _Hash, false> {
+  _Hash __hash_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __hash_map_hasher() : __hash_() {}
+  _LIBCPP_HIDE_FROM_ABI __hash_map_hasher(const _Hash& __h) : __hash_(__h) {}
+  _LIBCPP_HIDE_FROM_ABI const _Hash& hash_function() const { return __hash_; }
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Tp& __x) const { return __hash_(__x.first); }
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const typename _Tp::first_type& __x) const { return __hash_(__x); }
+};
+
+template <class _Tp, class _Pred, bool = std::is_empty<_Pred>::value && !std::__libcpp_is_final<_Pred>::value >
+class __hash_map_equal : private _Pred {
+public:
+  _LIBCPP_HIDE_FROM_ABI __hash_map_equal() : _Pred() {}
+  _LIBCPP_HIDE_FROM_ABI __hash_map_equal(const _Pred& __p) : _Pred(__p) {}
+  _LIBCPP_HIDE_FROM_ABI const _Pred& key_eq() const { return *this; }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const {
+    return static_cast<const _Pred&>(*this)(__x.first, __y.first);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const typename _Tp::first_type& __x, const _Tp& __y) const {
+    return static_cast<const _Pred&>(*this)(__x, __y.first);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const typename _Tp::first_type& __y) const {
+    return static_cast<const _Pred&>(*this)(__x.first, __y);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool
+  operator()(const typename _Tp::first_type& __x, const typename _Tp::first_type& __y) const {
+    return static_cast<const _Pred&>(*this)(__x, __y);
+  }
+};
+
+template <class _Tp, class _Pred>
+class __hash_map_equal<_Tp, _Pred, false> {
+  _Pred __pred_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __hash_map_equal() : __pred_() {}
+  _LIBCPP_HIDE_FROM_ABI __hash_map_equal(const _Pred& __p) : __pred_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI const _Pred& key_eq() const { return __pred_; }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const { return __pred_(__x.first, __y.first); }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const typename _Tp::first_type& __x, const _Tp& __y) const {
+    return __pred_(__x, __y.first);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const typename _Tp::first_type& __y) const {
+    return __pred_(__x.first, __y);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool
+  operator()(const typename _Tp::first_type& __x, const typename _Tp::first_type& __y) const {
+    return __pred_(__x, __y);
+  }
+};
+
+template <class _Alloc>
+class __hash_map_node_destructor {
+  typedef _Alloc allocator_type;
+  typedef std::allocator_traits<allocator_type> __alloc_traits;
+  typedef typename __alloc_traits::value_type::__node_value_type value_type;
+
+public:
+  typedef typename __alloc_traits::pointer pointer;
+
+private:
+  typedef typename value_type::first_type first_type;
+  typedef typename value_type::second_type second_type;
+
+  allocator_type& __na_;
+
+public:
+  bool __first_constructed;
+  bool __second_constructed;
+
+  _LIBCPP_HIDE_FROM_ABI __hash_map_node_destructor(__hash_map_node_destructor const&) = default;
+  __hash_map_node_destructor& operator=(const __hash_map_node_destructor&)            = delete;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __hash_map_node_destructor(allocator_type& __na)
+      : __na_(__na), __first_constructed(false), __second_constructed(false) {}
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI __hash_map_node_destructor(std::__hash_node_destructor<allocator_type>&& __x)
+      : __na_(__x.__na_), __first_constructed(__x.__value_constructed), __second_constructed(__x.__value_constructed) {
+    __x.__value_constructed = false;
+  }
+#else  // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI __hash_map_node_destructor(const std::__hash_node_destructor<allocator_type>& __x)
+      : __na_(__x.__na_), __first_constructed(__x.__value_constructed), __second_constructed(__x.__value_constructed) {
+    const_cast<bool&>(__x.__value_constructed) = false;
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) {
+    if (__second_constructed)
+      __alloc_traits::destroy(__na_, std::addressof(__p->__get_value().second));
+    if (__first_constructed)
+      __alloc_traits::destroy(__na_, std::addressof(__p->__get_value().first));
+    if (__p)
+      __alloc_traits::deallocate(__na_, __p, 1);
+  }
+};
+
+template <class _HashIterator>
+class _LIBCPP_TEMPLATE_VIS __hash_map_iterator {
+  _HashIterator __i_;
+
+  typedef const typename _HashIterator::value_type::first_type key_type;
+  typedef typename _HashIterator::value_type::second_type mapped_type;
+
+public:
+  typedef std::forward_iterator_tag iterator_category;
+  typedef std::pair<key_type, mapped_type> value_type;
+  typedef typename _HashIterator::difference_type difference_type;
+  typedef value_type& reference;
+  typedef std::__rebind_pointer_t<typename _HashIterator::pointer, value_type> pointer;
+
+  _LIBCPP_HIDE_FROM_ABI __hash_map_iterator() {}
+
+  _LIBCPP_HIDE_FROM_ABI __hash_map_iterator(_HashIterator __i) : __i_(__i) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return *operator->(); }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return (pointer)__i_.operator->(); }
+
+  _LIBCPP_HIDE_FROM_ABI __hash_map_iterator& operator++() {
+    ++__i_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __hash_map_iterator operator++(int) {
+    __hash_map_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __hash_map_iterator& __x, const __hash_map_iterator& __y) {
+    return __x.__i_ == __y.__i_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __hash_map_iterator& __x, const __hash_map_iterator& __y) {
+    return __x.__i_ != __y.__i_;
+  }
+
+  template <class, class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS hash_map;
+  template <class, class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS hash_multimap;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_const_iterator;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_const_local_iterator;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_map_const_iterator;
+};
+
+template <class _HashIterator>
+class _LIBCPP_TEMPLATE_VIS __hash_map_const_iterator {
+  _HashIterator __i_;
+
+  typedef const typename _HashIterator::value_type::first_type key_type;
+  typedef typename _HashIterator::value_type::second_type mapped_type;
+
+public:
+  typedef std::forward_iterator_tag iterator_category;
+  typedef std::pair<key_type, mapped_type> value_type;
+  typedef typename _HashIterator::difference_type difference_type;
+  typedef const value_type& reference;
+  typedef std::__rebind_pointer_t<typename _HashIterator::pointer, const value_type> pointer;
+
+  _LIBCPP_HIDE_FROM_ABI __hash_map_const_iterator() {}
+
+  _LIBCPP_HIDE_FROM_ABI __hash_map_const_iterator(_HashIterator __i) : __i_(__i) {}
+  _LIBCPP_HIDE_FROM_ABI __hash_map_const_iterator(__hash_map_iterator<typename _HashIterator::__non_const_iterator> __i)
+      : __i_(__i.__i_) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return *operator->(); }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return (pointer)__i_.operator->(); }
+
+  _LIBCPP_HIDE_FROM_ABI __hash_map_const_iterator& operator++() {
+    ++__i_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __hash_map_const_iterator operator++(int) {
+    __hash_map_const_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const __hash_map_const_iterator& __x, const __hash_map_const_iterator& __y) {
+    return __x.__i_ == __y.__i_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator!=(const __hash_map_const_iterator& __x, const __hash_map_const_iterator& __y) {
+    return __x.__i_ != __y.__i_;
+  }
+
+  template <class, class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS hash_map;
+  template <class, class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS hash_multimap;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_const_iterator;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_const_local_iterator;
+};
+
+template <class _Key,
+          class _Tp,
+          class _Hash  = hash<_Key>,
+          class _Pred  = std::equal_to<_Key>,
+          class _Alloc = std::allocator<std::pair<const _Key, _Tp> > >
+class _LIBCPP_TEMPLATE_VIS hash_map {
+public:
+  // types
+  typedef _Key key_type;
+  typedef _Tp mapped_type;
+  typedef _Tp data_type;
+  typedef _Hash hasher;
+  typedef _Pred key_equal;
+  typedef _Alloc allocator_type;
+  typedef std::pair<const key_type, mapped_type> value_type;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+
+private:
+  typedef std::pair<key_type, mapped_type> __value_type;
+  typedef __hash_map_hasher<__value_type, hasher> __hasher;
+  typedef __hash_map_equal<__value_type, key_equal> __key_equal;
+  typedef std::__rebind_alloc<std::allocator_traits<allocator_type>, __value_type> __allocator_type;
+
+  typedef std::__hash_table<__value_type, __hasher, __key_equal, __allocator_type> __table;
+
+  __table __table_;
+
+  typedef typename __table::__node_pointer __node_pointer;
+  typedef typename __table::__node_const_pointer __node_const_pointer;
+  typedef typename __table::__node_traits __node_traits;
+  typedef typename __table::__node_allocator __node_allocator;
+  typedef typename __table::__node __node;
+  typedef __hash_map_node_destructor<__node_allocator> _Dp;
+  typedef std::unique_ptr<__node, _Dp> __node_holder;
+  typedef std::allocator_traits<allocator_type> __alloc_traits;
+
+public:
+  typedef typename __alloc_traits::pointer pointer;
+  typedef typename __alloc_traits::const_pointer const_pointer;
+  typedef typename __alloc_traits::size_type size_type;
+  typedef typename __alloc_traits::difference_type difference_type;
+
+  typedef __hash_map_iterator<typename __table::iterator> iterator;
+  typedef __hash_map_const_iterator<typename __table::const_iterator> const_iterator;
+
+  _LIBCPP_HIDE_FROM_ABI hash_map() {}
+  explicit _LIBCPP_HIDE_FROM_ABI
+  hash_map(size_type __n, const hasher& __hf = hasher(), const key_equal& __eql = key_equal());
+  _LIBCPP_HIDE_FROM_ABI hash_map(size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI hash_map(_InputIterator __first, _InputIterator __last);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI
+  hash_map(_InputIterator __first,
+           _InputIterator __last,
+           size_type __n,
+           const hasher& __hf     = hasher(),
+           const key_equal& __eql = key_equal());
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI
+  hash_map(_InputIterator __first,
+           _InputIterator __last,
+           size_type __n,
+           const hasher& __hf,
+           const key_equal& __eql,
+           const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI hash_map(const hash_map& __u);
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const { return allocator_type(__table_.__node_alloc()); }
+
+  _LIBCPP_HIDE_FROM_ABI bool empty() const { return __table_.size() == 0; }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const { return __table_.size(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const { return __table_.max_size(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI iterator end() { return __table_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const { return __table_.end(); }
+
+  _LIBCPP_HIDE_FROM_ABI std::pair<iterator, bool> insert(const value_type& __x) {
+    return __table_.__insert_unique(__x);
+  }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator, const value_type& __x) { return insert(__x).first; }
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last);
+
+  _LIBCPP_HIDE_FROM_ABI void erase(const_iterator __p) { __table_.erase(__p.__i_); }
+  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __table_.__erase_unique(__k); }
+  _LIBCPP_HIDE_FROM_ABI void erase(const_iterator __first, const_iterator __last) {
+    __table_.erase(__first.__i_, __last.__i_);
+  }
+  _LIBCPP_HIDE_FROM_ABI void clear() { __table_.clear(); }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(hash_map& __u) { __table_.swap(__u.__table_); }
+
+  _LIBCPP_HIDE_FROM_ABI hasher hash_funct() const { return __table_.hash_function().hash_function(); }
+  _LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq().key_eq(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_unique(__k); }
+  _LIBCPP_HIDE_FROM_ABI std::pair<iterator, iterator> equal_range(const key_type& __k) {
+    return __table_.__equal_range_unique(__k);
+  }
+  _LIBCPP_HIDE_FROM_ABI std::pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
+    return __table_.__equal_range_unique(__k);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI mapped_type& operator[](const key_type& __k);
+
+  _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const { return __table_.bucket_count(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const { return __table_.max_bucket_count(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type elems_in_bucket(size_type __n) const { return __table_.bucket_size(__n); }
+
+  _LIBCPP_HIDE_FROM_ABI void resize(size_type __n) { __table_.__rehash_unique(__n); }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI __node_holder __construct_node(const key_type& __k);
+};
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_unique(__n);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(
+    size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
+    : __table_(__hf, __eql, __a) {
+  __table_.__rehash_unique(__n);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(_InputIterator __first, _InputIterator __last) {
+  insert(__first, __last);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(
+    _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_unique(__n);
+  insert(__first, __last);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(
+    _InputIterator __first,
+    _InputIterator __last,
+    size_type __n,
+    const hasher& __hf,
+    const key_equal& __eql,
+    const allocator_type& __a)
+    : __table_(__hf, __eql, __a) {
+  __table_.__rehash_unique(__n);
+  insert(__first, __last);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(const hash_map& __u) : __table_(__u.__table_) {
+  __table_.__rehash_unique(__u.bucket_count());
+  insert(__u.begin(), __u.end());
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+typename hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::__node_holder
+hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::__construct_node(const key_type& __k) {
+  __node_allocator& __na = __table_.__node_alloc();
+  __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
+  __node_traits::construct(__na, std::addressof(__h->__get_value().first), __k);
+  __h.get_deleter().__first_constructed = true;
+  __node_traits::construct(__na, std::addressof(__h->__get_value().second));
+  __h.get_deleter().__second_constructed = true;
+  return __h;
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+inline void hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::insert(_InputIterator __first, _InputIterator __last) {
+  for (; __first != __last; ++__first)
+    __table_.__insert_unique(*__first);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+_Tp& hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator[](const key_type& __k) {
+  iterator __i = find(__k);
+  if (__i != end())
+    return __i->second;
+  __node_holder __h             = __construct_node(__k);
+  std::pair<iterator, bool> __r = __table_.__node_insert_unique(__h.get());
+  __h.release();
+  return __r.first->second;
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __x, hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __y) {
+  __x.swap(__y);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+_LIBCPP_HIDE_FROM_ABI bool
+operator==(const hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __x, const hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __y) {
+  if (__x.size() != __y.size())
+    return false;
+  typedef typename hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::const_iterator const_iterator;
+  for (const_iterator __i = __x.begin(), __ex = __x.end(), __ey = __y.end(); __i != __ex; ++__i) {
+    const_iterator __j = __y.find(__i->first);
+    if (__j == __ey || !(*__i == *__j))
+      return false;
+  }
+  return true;
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __x, const hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __y) {
+  return !(__x == __y);
+}
+
+template <class _Key,
+          class _Tp,
+          class _Hash  = hash<_Key>,
+          class _Pred  = std::equal_to<_Key>,
+          class _Alloc = std::allocator<std::pair<const _Key, _Tp> > >
+class _LIBCPP_TEMPLATE_VIS hash_multimap {
+public:
+  // types
+  typedef _Key key_type;
+  typedef _Tp mapped_type;
+  typedef _Tp data_type;
+  typedef _Hash hasher;
+  typedef _Pred key_equal;
+  typedef _Alloc allocator_type;
+  typedef std::pair<const key_type, mapped_type> value_type;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+
+private:
+  typedef std::pair<key_type, mapped_type> __value_type;
+  typedef __hash_map_hasher<__value_type, hasher> __hasher;
+  typedef __hash_map_equal<__value_type, key_equal> __key_equal;
+  typedef std::__rebind_alloc<std::allocator_traits<allocator_type>, __value_type> __allocator_type;
+
+  typedef std::__hash_table<__value_type, __hasher, __key_equal, __allocator_type> __table;
+
+  __table __table_;
+
+  typedef typename __table::__node_traits __node_traits;
+  typedef typename __table::__node_allocator __node_allocator;
+  typedef typename __table::__node __node;
+  typedef __hash_map_node_destructor<__node_allocator> _Dp;
+  typedef std::unique_ptr<__node, _Dp> __node_holder;
+  typedef std::allocator_traits<allocator_type> __alloc_traits;
+
+public:
+  typedef typename __alloc_traits::pointer pointer;
+  typedef typename __alloc_traits::const_pointer const_pointer;
+  typedef typename __alloc_traits::size_type size_type;
+  typedef typename __alloc_traits::difference_type difference_type;
+
+  typedef __hash_map_iterator<typename __table::iterator> iterator;
+  typedef __hash_map_const_iterator<typename __table::const_iterator> const_iterator;
+
+  _LIBCPP_HIDE_FROM_ABI hash_multimap() {}
+  explicit _LIBCPP_HIDE_FROM_ABI
+  hash_multimap(size_type __n, const hasher& __hf = hasher(), const key_equal& __eql = key_equal());
+  _LIBCPP_HIDE_FROM_ABI
+  hash_multimap(size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI hash_multimap(_InputIterator __first, _InputIterator __last);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI
+  hash_multimap(_InputIterator __first,
+                _InputIterator __last,
+                size_type __n,
+                const hasher& __hf     = hasher(),
+                const key_equal& __eql = key_equal());
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI hash_multimap(
+      _InputIterator __first,
+      _InputIterator __last,
+      size_type __n,
+      const hasher& __hf,
+      const key_equal& __eql,
+      const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI hash_multimap(const hash_multimap& __u);
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const { return allocator_type(__table_.__node_alloc()); }
+
+  _LIBCPP_HIDE_FROM_ABI bool empty() const { return __table_.size() == 0; }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const { return __table_.size(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const { return __table_.max_size(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI iterator end() { return __table_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const { return __table_.end(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __x) { return __table_.__insert_multi(__x); }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator, const value_type& __x) { return insert(__x); }
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last);
+
+  _LIBCPP_HIDE_FROM_ABI void erase(const_iterator __p) { __table_.erase(__p.__i_); }
+  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __table_.__erase_multi(__k); }
+  _LIBCPP_HIDE_FROM_ABI void erase(const_iterator __first, const_iterator __last) {
+    __table_.erase(__first.__i_, __last.__i_);
+  }
+  _LIBCPP_HIDE_FROM_ABI void clear() { __table_.clear(); }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(hash_multimap& __u) { __table_.swap(__u.__table_); }
+
+  _LIBCPP_HIDE_FROM_ABI hasher hash_funct() const { return __table_.hash_function().hash_function(); }
+  _LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq().key_eq(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_multi(__k); }
+  _LIBCPP_HIDE_FROM_ABI std::pair<iterator, iterator> equal_range(const key_type& __k) {
+    return __table_.__equal_range_multi(__k);
+  }
+  _LIBCPP_HIDE_FROM_ABI std::pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
+    return __table_.__equal_range_multi(__k);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const { return __table_.bucket_count(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const { return __table_.max_bucket_count(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type elems_in_bucket(size_type __n) const { return __table_.bucket_size(__n); }
+
+  _LIBCPP_HIDE_FROM_ABI void resize(size_type __n) { __table_.__rehash_multi(__n); }
+};
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_multimap(size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_multi(__n);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_multimap(
+    size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
+    : __table_(__hf, __eql, __a) {
+  __table_.__rehash_multi(__n);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_multimap(_InputIterator __first, _InputIterator __last) {
+  insert(__first, __last);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_multimap(
+    _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_multi(__n);
+  insert(__first, __last);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_multimap(
+    _InputIterator __first,
+    _InputIterator __last,
+    size_type __n,
+    const hasher& __hf,
+    const key_equal& __eql,
+    const allocator_type& __a)
+    : __table_(__hf, __eql, __a) {
+  __table_.__rehash_multi(__n);
+  insert(__first, __last);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_multimap(const hash_multimap& __u) : __table_(__u.__table_) {
+  __table_.__rehash_multi(__u.bucket_count());
+  insert(__u.begin(), __u.end());
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+inline void hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::insert(_InputIterator __first, _InputIterator __last) {
+  for (; __first != __last; ++__first)
+    __table_.__insert_multi(*__first);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __x, hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __y) {
+  __x.swap(__y);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+_LIBCPP_HIDE_FROM_ABI bool operator==(const hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
+                                      const hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __y) {
+  if (__x.size() != __y.size())
+    return false;
+  typedef typename hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::const_iterator const_iterator;
+  typedef std::pair<const_iterator, const_iterator> _EqRng;
+  for (const_iterator __i = __x.begin(), __ex = __x.end(); __i != __ex;) {
+    _EqRng __xeq = __x.equal_range(__i->first);
+    _EqRng __yeq = __y.equal_range(__i->first);
+    if (std::distance(__xeq.first, __xeq.second) != std::distance(__yeq.first, __yeq.second) ||
+        !std::is_permutation(__xeq.first, __xeq.second, __yeq.first))
+      return false;
+    __i = __xeq.second;
+  }
+  return true;
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
+                                             const hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __y) {
+  return !(__x == __y);
+}
+
+} // namespace __gnu_cxx
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <concepts>
+#  include <iterator>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_HASH_MAP
diff --git a/libcxx/include/__cxx03/ext/hash_set b/libcxx/include/__cxx03/ext/hash_set
new file mode 100644
index 000000000000..1ab259b59979
--- /dev/null
+++ b/libcxx/include/__cxx03/ext/hash_set
@@ -0,0 +1,584 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_HASH_SET
+#define _LIBCPP_HASH_SET
+
+/*
+
+    hash_set synopsis
+
+namespace __gnu_cxx
+{
+
+template <class Value, class Hash = hash<Value>, class Pred = equal_to<Value>,
+          class Alloc = allocator<Value>>
+class hash_set
+{
+public:
+    // types
+    typedef Value                                                      key_type;
+    typedef key_type                                                   value_type;
+    typedef Hash                                                       hasher;
+    typedef Pred                                                       key_equal;
+    typedef Alloc                                                      allocator_type;
+    typedef value_type&                                                reference;
+    typedef const value_type&                                          const_reference;
+    typedef typename allocator_traits<allocator_type>::pointer         pointer;
+    typedef typename allocator_traits<allocator_type>::const_pointer   const_pointer;
+    typedef typename allocator_traits<allocator_type>::size_type       size_type;
+    typedef typename allocator_traits<allocator_type>::difference_type difference_type;
+
+    typedef /unspecified/ iterator;
+    typedef /unspecified/ const_iterator;
+
+    explicit hash_set(size_type n = 193, const hasher& hf = hasher(),
+                           const key_equal& eql = key_equal(),
+                           const allocator_type& a = allocator_type());
+    template <class InputIterator>
+        hash_set(InputIterator f, InputIterator l,
+                      size_type n = 193, const hasher& hf = hasher(),
+                      const key_equal& eql = key_equal(),
+                      const allocator_type& a = allocator_type());
+    hash_set(const hash_set&);
+    ~hash_set();
+    hash_set& operator=(const hash_set&);
+
+    allocator_type get_allocator() const;
+
+    bool      empty() const;
+    size_type size() const;
+    size_type max_size() const;
+
+    iterator       begin();
+    iterator       end();
+    const_iterator begin()  const;
+    const_iterator end()    const;
+
+    pair<iterator, bool> insert(const value_type& obj);
+    template <class InputIterator>
+        void insert(InputIterator first, InputIterator last);
+
+    void erase(const_iterator position);
+    size_type erase(const key_type& k);
+    void erase(const_iterator first, const_iterator last);
+    void clear();
+
+    void swap(hash_set&);
+
+    hasher hash_funct() const;
+    key_equal key_eq() const;
+
+    iterator       find(const key_type& k);
+    const_iterator find(const key_type& k) const;
+    size_type count(const key_type& k) const;
+    pair<iterator, iterator>             equal_range(const key_type& k);
+    pair<const_iterator, const_iterator> equal_range(const key_type& k) const;
+
+    size_type bucket_count() const;
+    size_type max_bucket_count() const;
+
+    size_type elems_in_bucket(size_type n) const;
+
+    void resize(size_type n);
+};
+
+template <class Value, class Hash, class Pred, class Alloc>
+    void swap(hash_set<Value, Hash, Pred, Alloc>& x,
+              hash_set<Value, Hash, Pred, Alloc>& y);
+
+template <class Value, class Hash, class Pred, class Alloc>
+    bool
+    operator==(const hash_set<Value, Hash, Pred, Alloc>& x,
+               const hash_set<Value, Hash, Pred, Alloc>& y);
+
+template <class Value, class Hash, class Pred, class Alloc>
+    bool
+    operator!=(const hash_set<Value, Hash, Pred, Alloc>& x,
+               const hash_set<Value, Hash, Pred, Alloc>& y);
+
+template <class Value, class Hash = hash<Value>, class Pred = equal_to<Value>,
+          class Alloc = allocator<Value>>
+class hash_multiset
+{
+public:
+    // types
+    typedef Value                                                      key_type;
+    typedef key_type                                                   value_type;
+    typedef Hash                                                       hasher;
+    typedef Pred                                                       key_equal;
+    typedef Alloc                                                      allocator_type;
+    typedef value_type&                                                reference;
+    typedef const value_type&                                          const_reference;
+    typedef typename allocator_traits<allocator_type>::pointer         pointer;
+    typedef typename allocator_traits<allocator_type>::const_pointer   const_pointer;
+    typedef typename allocator_traits<allocator_type>::size_type       size_type;
+    typedef typename allocator_traits<allocator_type>::difference_type difference_type;
+
+    typedef /unspecified/ iterator;
+    typedef /unspecified/ const_iterator;
+
+    explicit hash_multiset(size_type n = 193, const hasher& hf = hasher(),
+                           const key_equal& eql = key_equal(),
+                           const allocator_type& a = allocator_type());
+    template <class InputIterator>
+        hash_multiset(InputIterator f, InputIterator l,
+                      size_type n = 193, const hasher& hf = hasher(),
+                      const key_equal& eql = key_equal(),
+                      const allocator_type& a = allocator_type());
+    hash_multiset(const hash_multiset&);
+    ~hash_multiset();
+    hash_multiset& operator=(const hash_multiset&);
+
+    allocator_type get_allocator() const;
+
+    bool      empty() const;
+    size_type size() const;
+    size_type max_size() const;
+
+    iterator       begin();
+    iterator       end();
+    const_iterator begin()  const;
+    const_iterator end()    const;
+
+    iterator insert(const value_type& obj);
+    template <class InputIterator>
+        void insert(InputIterator first, InputIterator last);
+
+    void erase(const_iterator position);
+    size_type erase(const key_type& k);
+    void erase(const_iterator first, const_iterator last);
+    void clear();
+
+    void swap(hash_multiset&);
+
+    hasher hash_funct() const;
+    key_equal key_eq() const;
+
+    iterator       find(const key_type& k);
+    const_iterator find(const key_type& k) const;
+    size_type count(const key_type& k) const;
+    pair<iterator, iterator>             equal_range(const key_type& k);
+    pair<const_iterator, const_iterator> equal_range(const key_type& k) const;
+
+    size_type bucket_count() const;
+    size_type max_bucket_count() const;
+
+    size_type elems_in_bucket(size_type n) const;
+
+    void resize(size_type n);
+};
+
+template <class Value, class Hash, class Pred, class Alloc>
+    void swap(hash_multiset<Value, Hash, Pred, Alloc>& x,
+              hash_multiset<Value, Hash, Pred, Alloc>& y);
+
+template <class Value, class Hash, class Pred, class Alloc>
+    bool
+    operator==(const hash_multiset<Value, Hash, Pred, Alloc>& x,
+               const hash_multiset<Value, Hash, Pred, Alloc>& y);
+
+template <class Value, class Hash, class Pred, class Alloc>
+    bool
+    operator!=(const hash_multiset<Value, Hash, Pred, Alloc>& x,
+               const hash_multiset<Value, Hash, Pred, Alloc>& y);
+}  // __gnu_cxx
+
+*/
+
+#include <__config>
+#include <__hash_table>
+#include <algorithm>
+#include <ext/__hash>
+#include <functional>
+
+#if defined(__DEPRECATED) && __DEPRECATED
+#  if defined(_LIBCPP_WARNING)
+_LIBCPP_WARNING("Use of the header <ext/hash_set> is deprecated.  Migrate to <unordered_set>")
+#  else
+#    warning Use of the header <ext/hash_set> is deprecated.  Migrate to <unordered_set>
+#  endif
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+namespace __gnu_cxx {
+
+template <class _Value,
+          class _Hash  = hash<_Value>,
+          class _Pred  = std::equal_to<_Value>,
+          class _Alloc = std::allocator<_Value> >
+class _LIBCPP_TEMPLATE_VIS hash_set {
+public:
+  // types
+  typedef _Value key_type;
+  typedef key_type value_type;
+  typedef _Hash hasher;
+  typedef _Pred key_equal;
+  typedef _Alloc allocator_type;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+
+private:
+  typedef std::__hash_table<value_type, hasher, key_equal, allocator_type> __table;
+
+  __table __table_;
+
+public:
+  typedef typename __table::pointer pointer;
+  typedef typename __table::const_pointer const_pointer;
+  typedef typename __table::size_type size_type;
+  typedef typename __table::difference_type difference_type;
+
+  typedef typename __table::const_iterator iterator;
+  typedef typename __table::const_iterator const_iterator;
+
+  _LIBCPP_HIDE_FROM_ABI hash_set() {}
+  _LIBCPP_HIDE_FROM_ABI explicit hash_set(
+      size_type __n, const hasher& __hf = hasher(), const key_equal& __eql = key_equal());
+  _LIBCPP_HIDE_FROM_ABI hash_set(size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI hash_set(_InputIterator __first, _InputIterator __last);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI
+  hash_set(_InputIterator __first,
+           _InputIterator __last,
+           size_type __n,
+           const hasher& __hf     = hasher(),
+           const key_equal& __eql = key_equal());
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI
+  hash_set(_InputIterator __first,
+           _InputIterator __last,
+           size_type __n,
+           const hasher& __hf,
+           const key_equal& __eql,
+           const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI hash_set(const hash_set& __u);
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const { return allocator_type(__table_.__node_alloc()); }
+
+  _LIBCPP_HIDE_FROM_ABI bool empty() const { return __table_.size() == 0; }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const { return __table_.size(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const { return __table_.max_size(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI iterator end() { return __table_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const { return __table_.end(); }
+
+  _LIBCPP_HIDE_FROM_ABI std::pair<iterator, bool> insert(const value_type& __x) {
+    return __table_.__insert_unique(__x);
+  }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator, const value_type& __x) { return insert(__x).first; }
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last);
+
+  _LIBCPP_HIDE_FROM_ABI void erase(const_iterator __p) { __table_.erase(__p); }
+  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __table_.__erase_unique(__k); }
+  _LIBCPP_HIDE_FROM_ABI void erase(const_iterator __first, const_iterator __last) { __table_.erase(__first, __last); }
+  _LIBCPP_HIDE_FROM_ABI void clear() { __table_.clear(); }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(hash_set& __u) { __table_.swap(__u.__table_); }
+
+  _LIBCPP_HIDE_FROM_ABI hasher hash_funct() const { return __table_.hash_function(); }
+  _LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_unique(__k); }
+  _LIBCPP_HIDE_FROM_ABI std::pair<iterator, iterator> equal_range(const key_type& __k) {
+    return __table_.__equal_range_unique(__k);
+  }
+  _LIBCPP_HIDE_FROM_ABI std::pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
+    return __table_.__equal_range_unique(__k);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const { return __table_.bucket_count(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const { return __table_.max_bucket_count(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type elems_in_bucket(size_type __n) const { return __table_.bucket_size(__n); }
+
+  _LIBCPP_HIDE_FROM_ABI void resize(size_type __n) { __table_.__rehash_unique(__n); }
+};
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+hash_set<_Value, _Hash, _Pred, _Alloc>::hash_set(size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_unique(__n);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+hash_set<_Value, _Hash, _Pred, _Alloc>::hash_set(
+    size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
+    : __table_(__hf, __eql, __a) {
+  __table_.__rehash_unique(__n);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+hash_set<_Value, _Hash, _Pred, _Alloc>::hash_set(_InputIterator __first, _InputIterator __last) {
+  insert(__first, __last);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+hash_set<_Value, _Hash, _Pred, _Alloc>::hash_set(
+    _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_unique(__n);
+  insert(__first, __last);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+hash_set<_Value, _Hash, _Pred, _Alloc>::hash_set(
+    _InputIterator __first,
+    _InputIterator __last,
+    size_type __n,
+    const hasher& __hf,
+    const key_equal& __eql,
+    const allocator_type& __a)
+    : __table_(__hf, __eql, __a) {
+  __table_.__rehash_unique(__n);
+  insert(__first, __last);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+hash_set<_Value, _Hash, _Pred, _Alloc>::hash_set(const hash_set& __u) : __table_(__u.__table_) {
+  __table_.__rehash_unique(__u.bucket_count());
+  insert(__u.begin(), __u.end());
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+inline void hash_set<_Value, _Hash, _Pred, _Alloc>::insert(_InputIterator __first, _InputIterator __last) {
+  for (; __first != __last; ++__first)
+    __table_.__insert_unique(*__first);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(hash_set<_Value, _Hash, _Pred, _Alloc>& __x, hash_set<_Value, _Hash, _Pred, _Alloc>& __y) {
+  __x.swap(__y);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+_LIBCPP_HIDE_FROM_ABI bool
+operator==(const hash_set<_Value, _Hash, _Pred, _Alloc>& __x, const hash_set<_Value, _Hash, _Pred, _Alloc>& __y) {
+  if (__x.size() != __y.size())
+    return false;
+  typedef typename hash_set<_Value, _Hash, _Pred, _Alloc>::const_iterator const_iterator;
+  for (const_iterator __i = __x.begin(), __ex = __x.end(), __ey = __y.end(); __i != __ex; ++__i) {
+    const_iterator __j = __y.find(*__i);
+    if (__j == __ey || !(*__i == *__j))
+      return false;
+  }
+  return true;
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const hash_set<_Value, _Hash, _Pred, _Alloc>& __x, const hash_set<_Value, _Hash, _Pred, _Alloc>& __y) {
+  return !(__x == __y);
+}
+
+template <class _Value,
+          class _Hash  = hash<_Value>,
+          class _Pred  = std::equal_to<_Value>,
+          class _Alloc = std::allocator<_Value> >
+class _LIBCPP_TEMPLATE_VIS hash_multiset {
+public:
+  // types
+  typedef _Value key_type;
+  typedef key_type value_type;
+  typedef _Hash hasher;
+  typedef _Pred key_equal;
+  typedef _Alloc allocator_type;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+
+private:
+  typedef std::__hash_table<value_type, hasher, key_equal, allocator_type> __table;
+
+  __table __table_;
+
+public:
+  typedef typename __table::pointer pointer;
+  typedef typename __table::const_pointer const_pointer;
+  typedef typename __table::size_type size_type;
+  typedef typename __table::difference_type difference_type;
+
+  typedef typename __table::const_iterator iterator;
+  typedef typename __table::const_iterator const_iterator;
+
+  _LIBCPP_HIDE_FROM_ABI hash_multiset() {}
+  explicit _LIBCPP_HIDE_FROM_ABI
+  hash_multiset(size_type __n, const hasher& __hf = hasher(), const key_equal& __eql = key_equal());
+  _LIBCPP_HIDE_FROM_ABI
+  hash_multiset(size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI hash_multiset(_InputIterator __first, _InputIterator __last);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI
+  hash_multiset(_InputIterator __first,
+                _InputIterator __last,
+                size_type __n,
+                const hasher& __hf     = hasher(),
+                const key_equal& __eql = key_equal());
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI hash_multiset(
+      _InputIterator __first,
+      _InputIterator __last,
+      size_type __n,
+      const hasher& __hf,
+      const key_equal& __eql,
+      const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI hash_multiset(const hash_multiset& __u);
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const { return allocator_type(__table_.__node_alloc()); }
+
+  _LIBCPP_HIDE_FROM_ABI bool empty() const { return __table_.size() == 0; }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const { return __table_.size(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const { return __table_.max_size(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI iterator end() { return __table_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const { return __table_.end(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __x) { return __table_.__insert_multi(__x); }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator, const value_type& __x) { return insert(__x); }
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last);
+
+  _LIBCPP_HIDE_FROM_ABI void erase(const_iterator __p) { __table_.erase(__p); }
+  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __table_.__erase_multi(__k); }
+  _LIBCPP_HIDE_FROM_ABI void erase(const_iterator __first, const_iterator __last) { __table_.erase(__first, __last); }
+  _LIBCPP_HIDE_FROM_ABI void clear() { __table_.clear(); }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(hash_multiset& __u) { __table_.swap(__u.__table_); }
+
+  _LIBCPP_HIDE_FROM_ABI hasher hash_funct() const { return __table_.hash_function(); }
+  _LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_multi(__k); }
+  _LIBCPP_HIDE_FROM_ABI std::pair<iterator, iterator> equal_range(const key_type& __k) {
+    return __table_.__equal_range_multi(__k);
+  }
+  _LIBCPP_HIDE_FROM_ABI std::pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
+    return __table_.__equal_range_multi(__k);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const { return __table_.bucket_count(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const { return __table_.max_bucket_count(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type elems_in_bucket(size_type __n) const { return __table_.bucket_size(__n); }
+
+  _LIBCPP_HIDE_FROM_ABI void resize(size_type __n) { __table_.__rehash_multi(__n); }
+};
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+hash_multiset<_Value, _Hash, _Pred, _Alloc>::hash_multiset(size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_multi(__n);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+hash_multiset<_Value, _Hash, _Pred, _Alloc>::hash_multiset(
+    size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
+    : __table_(__hf, __eql, __a) {
+  __table_.__rehash_multi(__n);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+hash_multiset<_Value, _Hash, _Pred, _Alloc>::hash_multiset(_InputIterator __first, _InputIterator __last) {
+  insert(__first, __last);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+hash_multiset<_Value, _Hash, _Pred, _Alloc>::hash_multiset(
+    _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_multi(__n);
+  insert(__first, __last);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+hash_multiset<_Value, _Hash, _Pred, _Alloc>::hash_multiset(
+    _InputIterator __first,
+    _InputIterator __last,
+    size_type __n,
+    const hasher& __hf,
+    const key_equal& __eql,
+    const allocator_type& __a)
+    : __table_(__hf, __eql, __a) {
+  __table_.__rehash_multi(__n);
+  insert(__first, __last);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+hash_multiset<_Value, _Hash, _Pred, _Alloc>::hash_multiset(const hash_multiset& __u) : __table_(__u.__table_) {
+  __table_.__rehash_multi(__u.bucket_count());
+  insert(__u.begin(), __u.end());
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+inline void hash_multiset<_Value, _Hash, _Pred, _Alloc>::insert(_InputIterator __first, _InputIterator __last) {
+  for (; __first != __last; ++__first)
+    __table_.__insert_multi(*__first);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(hash_multiset<_Value, _Hash, _Pred, _Alloc>& __x, hash_multiset<_Value, _Hash, _Pred, _Alloc>& __y) {
+  __x.swap(__y);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+_LIBCPP_HIDE_FROM_ABI bool operator==(const hash_multiset<_Value, _Hash, _Pred, _Alloc>& __x,
+                                      const hash_multiset<_Value, _Hash, _Pred, _Alloc>& __y) {
+  if (__x.size() != __y.size())
+    return false;
+  typedef typename hash_multiset<_Value, _Hash, _Pred, _Alloc>::const_iterator const_iterator;
+  typedef std::pair<const_iterator, const_iterator> _EqRng;
+  for (const_iterator __i = __x.begin(), __ex = __x.end(); __i != __ex;) {
+    _EqRng __xeq = __x.equal_range(*__i);
+    _EqRng __yeq = __y.equal_range(*__i);
+    if (std::distance(__xeq.first, __xeq.second) != std::distance(__yeq.first, __yeq.second) ||
+        !std::is_permutation(__xeq.first, __xeq.second, __yeq.first))
+      return false;
+    __i = __xeq.second;
+  }
+  return true;
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const hash_multiset<_Value, _Hash, _Pred, _Alloc>& __x,
+                                             const hash_multiset<_Value, _Hash, _Pred, _Alloc>& __y) {
+  return !(__x == __y);
+}
+
+} // namespace __gnu_cxx
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <concepts>
+#  include <iterator>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_HASH_SET
diff --git a/libcxx/include/__cxx03/fenv.h b/libcxx/include/__cxx03/fenv.h
new file mode 100644
index 000000000000..5647f2b33955
--- /dev/null
+++ b/libcxx/include/__cxx03/fenv.h
@@ -0,0 +1,114 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_FENV_H
+#define _LIBCPP_FENV_H
+
+/*
+    fenv.h synopsis
+
+This entire header is C99 / C++0X
+
+Macros:
+
+    FE_DIVBYZERO
+    FE_INEXACT
+    FE_INVALID
+    FE_OVERFLOW
+    FE_UNDERFLOW
+    FE_ALL_EXCEPT
+    FE_DOWNWARD
+    FE_TONEAREST
+    FE_TOWARDZERO
+    FE_UPWARD
+    FE_DFL_ENV
+
+Types:
+
+    fenv_t
+    fexcept_t
+
+int feclearexcept(int excepts);
+int fegetexceptflag(fexcept_t* flagp, int excepts);
+int feraiseexcept(int excepts);
+int fesetexceptflag(const fexcept_t* flagp, int excepts);
+int fetestexcept(int excepts);
+int fegetround();
+int fesetround(int round);
+int fegetenv(fenv_t* envp);
+int feholdexcept(fenv_t* envp);
+int fesetenv(const fenv_t* envp);
+int feupdateenv(const fenv_t* envp);
+
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if __has_include_next(<fenv.h>)
+#  include_next <fenv.h>
+#endif
+
+#ifdef __cplusplus
+
+extern "C++" {
+
+#  ifdef feclearexcept
+#    undef feclearexcept
+#  endif
+
+#  ifdef fegetexceptflag
+#    undef fegetexceptflag
+#  endif
+
+#  ifdef feraiseexcept
+#    undef feraiseexcept
+#  endif
+
+#  ifdef fesetexceptflag
+#    undef fesetexceptflag
+#  endif
+
+#  ifdef fetestexcept
+#    undef fetestexcept
+#  endif
+
+#  ifdef fegetround
+#    undef fegetround
+#  endif
+
+#  ifdef fesetround
+#    undef fesetround
+#  endif
+
+#  ifdef fegetenv
+#    undef fegetenv
+#  endif
+
+#  ifdef feholdexcept
+#    undef feholdexcept
+#  endif
+
+#  ifdef fesetenv
+#    undef fesetenv
+#  endif
+
+#  ifdef feupdateenv
+#    undef feupdateenv
+#  endif
+
+} // extern "C++"
+
+#endif // defined(__cplusplus)
+
+#endif // _LIBCPP_FENV_H
diff --git a/libcxx/include/__cxx03/filesystem b/libcxx/include/__cxx03/filesystem
new file mode 100644
index 000000000000..6ea04df0a089
--- /dev/null
+++ b/libcxx/include/__cxx03/filesystem
@@ -0,0 +1,577 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_FILESYSTEM
+#define _LIBCPP_FILESYSTEM
+
+/*
+    filesystem synopsis
+
+    namespace std::filesystem {
+
+    // `class path` from http://eel.is/c++draft/fs.class.path.general#6
+    class path {
+      public:
+        using value_type  = see below;
+        using string_type = basic_string<value_type>;
+        static constexpr value_type preferred_separator = see below;
+
+        enum format;
+
+        path() noexcept;
+        path(const path& p);
+        path(path&& p) noexcept;
+        path(string_type&& source, format fmt = auto_format);
+        template<class Source>
+          path(const Source& source, format fmt = auto_format);
+        template<class InputIterator>
+          path(InputIterator first, InputIterator last, format fmt = auto_format);
+        template<class Source>
+          path(const Source& source, const locale& loc, format fmt = auto_format);
+        template<class InputIterator>
+          path(InputIterator first, InputIterator last, const locale& loc, format fmt = auto_format);
+        ~path();
+
+        path& operator=(const path& p);
+        path& operator=(path&& p) noexcept;
+        path& operator=(string_type&& source);
+        path& assign(string_type&& source);
+        template<class Source>
+          path& operator=(const Source& source);
+        template<class Source>
+          path& assign(const Source& source);
+        template<class InputIterator>
+          path& assign(InputIterator first, InputIterator last);
+
+        path& operator/=(const path& p);
+        template<class Source>
+          path& operator/=(const Source& source);
+        template<class Source>
+          path& append(const Source& source);
+        template<class InputIterator>
+          path& append(InputIterator first, InputIterator last);
+
+        path& operator+=(const path& x);
+        path& operator+=(const string_type& x);
+        path& operator+=(basic_string_view<value_type> x);
+        path& operator+=(const value_type* x);
+        path& operator+=(value_type x);
+        template<class Source>
+          path& operator+=(const Source& x);
+        template<class EcharT>
+          path& operator+=(EcharT x);
+        template<class Source>
+          path& concat(const Source& x);
+        template<class InputIterator>
+          path& concat(InputIterator first, InputIterator last);
+
+        void  clear() noexcept;
+        path& make_preferred();
+        path& remove_filename();
+        path& replace_filename(const path& replacement);
+        path& replace_extension(const path& replacement = path());
+        void  swap(path& rhs) noexcept;
+
+        friend bool operator==(const path& lhs, const path& rhs) noexcept;
+        friend bool operator!=(const path& lhs, const path& rhs) noexcept;             // removed in C++20
+        friend bool operator< (const path& lhs, const path& rhs) noexcept;             // removed in C++20
+        friend bool operator<=(const path& lhs, const path& rhs) noexcept;             // removed in C++20
+        friend bool operator> (const path& lhs, const path& rhs) noexcept;             // removed in C++20
+        friend bool operator>=(const path& lhs, const path& rhs) noexcept;             // removed in C++20
+        friend strong_ordering operator<=>(const path& lhs, const path& rhs) noexcept; // C++20
+
+        friend path operator/(const path& lhs, const path& rhs);
+
+        const string_type& native() const noexcept;
+        const value_type*  c_str() const noexcept;
+        operator string_type() const;
+
+        template<class EcharT, class traits = char_traits<EcharT>,
+                 class Allocator = allocator<EcharT>>
+          basic_string<EcharT, traits, Allocator>
+            string(const Allocator& a = Allocator()) const;
+        std::string    string() const;
+        std::wstring   wstring() const;
+        std::u8string  u8string() const;
+        std::u16string u16string() const;
+        std::u32string u32string() const;
+
+        template<class EcharT, class traits = char_traits<EcharT>,
+                 class Allocator = allocator<EcharT>>
+          basic_string<EcharT, traits, Allocator>
+            generic_string(const Allocator& a = Allocator()) const;
+        std::string    generic_string() const;
+        std::wstring   generic_wstring() const;
+        std::u8string  generic_u8string() const;
+        std::u16string generic_u16string() const;
+        std::u32string generic_u32string() const;
+
+        int compare(const path& p) const noexcept;
+        int compare(const string_type& s) const;
+        int compare(basic_string_view<value_type> s) const;
+        int compare(const value_type* s) const;
+
+        path root_name() const;
+        path root_directory() const;
+        path root_path() const;
+        path relative_path() const;
+        path parent_path() const;
+        path filename() const;
+        path stem() const;
+        path extension() const;
+
+        [[nodiscard]] bool empty() const noexcept;
+        bool has_root_name() const;
+        bool has_root_directory() const;
+        bool has_root_path() const;
+        bool has_relative_path() const;
+        bool has_parent_path() const;
+        bool has_filename() const;
+        bool has_stem() const;
+        bool has_extension() const;
+        bool is_absolute() const;
+        bool is_relative() const;
+
+        path lexically_normal() const;
+        path lexically_relative(const path& base) const;
+        path lexically_proximate(const path& base) const;
+
+        class iterator;
+        using const_iterator = iterator;
+
+        iterator begin() const;
+        iterator end() const;
+
+        template<class charT, class traits>
+          friend basic_ostream<charT, traits>&
+            operator<<(basic_ostream<charT, traits>& os, const path& p);
+        template<class charT, class traits>
+          friend basic_istream<charT, traits>&
+            operator>>(basic_istream<charT, traits>& is, path& p);
+    };
+
+    void swap(path& lhs, path& rhs) noexcept;
+    size_t hash_value(const path& p) noexcept;
+
+    // [fs.path.hash], hash support
+    template<> struct hash<filesystem::path>;
+
+    template <class Source>
+      path u8path(const Source& source);
+    template <class InputIterator>
+      path u8path(InputIterator first, InputIterator last);
+
+    class filesystem_error;
+
+    class directory_entry {
+    public:
+      directory_entry() noexcept = default;
+      directory_entry(const directory_entry&) = default;
+      directory_entry(directory_entry&&) noexcept = default;
+      explicit directory_entry(const filesystem::path& p);
+      directory_entry(const filesystem::path& p, error_code& ec);
+      ~directory_entry();
+
+      directory_entry& operator=(const directory_entry&) = default;
+      directory_entry& operator=(directory_entry&&) noexcept = default;
+
+      void assign(const filesystem::path& p);
+      void assign(const filesystem::path& p, error_code& ec);
+      void replace_filename(const filesystem::path& p);
+      void replace_filename(const filesystem::path& p, error_code& ec);
+      void refresh();
+      void refresh(error_code& ec) noexcept;
+
+      const filesystem::path& path() const noexcept;
+      operator const filesystem::path&() const noexcept;
+      bool exists() const;
+      bool exists(error_code& ec) const noexcept;
+      bool is_block_file() const;
+      bool is_block_file(error_code& ec) const noexcept;
+      bool is_character_file() const;
+      bool is_character_file(error_code& ec) const noexcept;
+      bool is_directory() const;
+      bool is_directory(error_code& ec) const noexcept;
+      bool is_fifo() const;
+      bool is_fifo(error_code& ec) const noexcept;
+      bool is_other() const;
+      bool is_other(error_code& ec) const noexcept;
+      bool is_regular_file() const;
+      bool is_regular_file(error_code& ec) const noexcept;
+      bool is_socket() const;
+      bool is_socket(error_code& ec) const noexcept;
+      bool is_symlink() const;
+      bool is_symlink(error_code& ec) const noexcept;
+      uintmax_t file_size() const;
+      uintmax_t file_size(error_code& ec) const noexcept;
+      uintmax_t hard_link_count() const;
+      uintmax_t hard_link_count(error_code& ec) const noexcept;
+      file_time_type last_write_time() const;
+      file_time_type last_write_time(error_code& ec) const noexcept;
+      file_status status() const;
+      file_status status(error_code& ec) const noexcept;
+      file_status symlink_status() const;
+      file_status symlink_status(error_code& ec) const noexcept;
+
+      bool operator==(const directory_entry& rhs) const noexcept;
+      bool operator!=(const directory_entry& rhs) const noexcept;             // removed  in C++20
+      bool operator< (const directory_entry& rhs) const noexcept;             // removed  in C++20
+      bool operator<=(const directory_entry& rhs) const noexcept;             // removed  in C++20
+      bool operator> (const directory_entry& rhs) const noexcept;             // removed  in C++20
+      bool operator>=(const directory_entry& rhs) const noexcept;             // removed  in C++20
+      strong_ordering operator<=>(const directory_entry& rhs) const noexcept; // since C++20
+
+      template<class charT, class traits>
+        friend basic_ostream<charT, traits>&
+          operator<<(basic_ostream<charT, traits>& os, const directory_entry& d);
+
+    private:
+      filesystem::path pathobject;        // exposition only
+      friend class directory_iterator;    // exposition only
+    };
+
+  class directory_iterator {
+  public:
+    using iterator_category = input_iterator_tag;
+    using value_type        = directory_entry;
+    using difference_type   = ptrdiff_t;
+    using pointer           = const directory_entry*;
+    using reference         = const directory_entry&;
+
+    // [fs.dir.itr.members], member functions
+    directory_iterator() noexcept;
+    explicit directory_iterator(const path& p);
+    directory_iterator(const path& p, directory_options options);
+    directory_iterator(const path& p, error_code& ec);
+    directory_iterator(const path& p, directory_options options,
+                       error_code& ec);
+    directory_iterator(const directory_iterator& rhs);
+    directory_iterator(directory_iterator&& rhs) noexcept;
+    ~directory_iterator();
+
+    directory_iterator& operator=(const directory_iterator& rhs);
+    directory_iterator& operator=(directory_iterator&& rhs) noexcept;
+
+    const directory_entry& operator*() const;
+    const directory_entry* operator->() const;
+    directory_iterator&    operator++();
+    directory_iterator&    increment(error_code& ec);
+
+    bool operator==(default_sentinel_t) const noexcept {          // since C++20
+      return *this == directory_iterator();
+    }
+
+    // other members as required by [input.iterators], input iterators
+  };
+
+    // enable directory_iterator range-based for statements
+    directory_iterator begin(directory_iterator iter) noexcept;
+    directory_iterator end(directory_iterator) noexcept;
+
+class recursive_directory_iterator {
+  public:
+    using iterator_category = input_iterator_tag;
+    using value_type        = directory_entry;
+    using difference_type   = ptrdiff_t;
+    using pointer           = const directory_entry*;
+    using reference         = const directory_entry&;
+
+    // [fs.rec.dir.itr.members], constructors and destructor
+    recursive_directory_iterator() noexcept;
+    explicit recursive_directory_iterator(const path& p);
+    recursive_directory_iterator(const path& p, directory_options options);
+    recursive_directory_iterator(const path& p, directory_options options,
+                                 error_code& ec);
+    recursive_directory_iterator(const path& p, error_code& ec);
+    recursive_directory_iterator(const recursive_directory_iterator& rhs);
+    recursive_directory_iterator(recursive_directory_iterator&& rhs) noexcept;
+    ~recursive_directory_iterator();
+
+    // [fs.rec.dir.itr.members], observers
+    directory_options  options() const;
+    int                depth() const;
+    bool               recursion_pending() const;
+
+    const directory_entry& operator*() const;
+    const directory_entry* operator->() const;
+
+    // [fs.rec.dir.itr.members], modifiers
+    recursive_directory_iterator&
+      operator=(const recursive_directory_iterator& rhs);
+    recursive_directory_iterator&
+      operator=(recursive_directory_iterator&& rhs) noexcept;
+
+    recursive_directory_iterator& operator++();
+    recursive_directory_iterator& increment(error_code& ec);
+
+    void pop();
+    void pop(error_code& ec);
+    void disable_recursion_pending();
+
+    bool operator==(default_sentinel_t) const noexcept {          // since C++20
+      return *this == recursive_directory_iterator();
+    }
+
+    // other members as required by [input.iterators], input iterators
+  };
+
+    // enable recursive_directory_iterator range-based for statements
+    recursive_directory_iterator begin(recursive_directory_iterator iter) noexcept;
+    recursive_directory_iterator end(recursive_directory_iterator) noexcept;
+
+  class file_status {
+  public:
+    // [fs.file.status.cons], constructors and destructor
+    file_status() noexcept : file_status(file_type::none) {}
+    explicit file_status(file_type ft,
+                         perms prms = perms::unknown) noexcept;
+    file_status(const file_status&) noexcept = default;
+    file_status(file_status&&) noexcept = default;
+    ~file_status();
+
+    // assignments
+    file_status& operator=(const file_status&) noexcept = default;
+    file_status& operator=(file_status&&) noexcept = default;
+
+    // [fs.file.status.mods], modifiers
+    void       type(file_type ft) noexcept;
+    void       permissions(perms prms) noexcept;
+
+    // [fs.file.status.obs], observers
+    file_type  type() const noexcept;
+    perms      permissions() const noexcept;
+
+    friend bool operator==(const file_status& lhs, const file_status& rhs) noexcept
+      { return lhs.type() == rhs.type() && lhs.permissions() == rhs.permissions(); } // C++20
+  };
+
+    struct space_info
+    {
+      uintmax_t capacity;
+      uintmax_t free;
+      uintmax_t available;
+
+      friend bool operator==(const space_info&, const space_info&) = default; // C++20
+    };
+
+    enum class file_type;
+    enum class perms;
+    enum class perm_options;
+    enum class copy_options;
+    enum class directory_options;
+
+    typedef chrono::time_point<trivial-clock>  file_time_type;
+
+    // operational functions
+
+    path absolute(const path& p);
+    path absolute(const path& p, error_code &ec);
+
+    path canonical(const path& p);
+    path canonical(const path& p, error_code& ec);
+
+    void copy(const path& from, const path& to);
+    void copy(const path& from, const path& to, error_code& ec);
+    void copy(const path& from, const path& to, copy_options options);
+    void copy(const path& from, const path& to, copy_options options,
+                   error_code& ec);
+
+    bool copy_file(const path& from, const path& to);
+    bool copy_file(const path& from, const path& to, error_code& ec);
+    bool copy_file(const path& from, const path& to, copy_options option);
+    bool copy_file(const path& from, const path& to, copy_options option,
+                           error_code& ec);
+
+    void copy_symlink(const path& existing_symlink, const path& new_symlink);
+    void copy_symlink(const path& existing_symlink, const path& new_symlink,
+                              error_code& ec) noexcept;
+
+    bool create_directories(const path& p);
+    bool create_directories(const path& p, error_code& ec);
+
+    bool create_directory(const path& p);
+    bool create_directory(const path& p, error_code& ec) noexcept;
+
+    bool create_directory(const path& p, const path& attributes);
+    bool create_directory(const path& p, const path& attributes,
+                                  error_code& ec) noexcept;
+
+    void create_directory_symlink(const path& to, const path& new_symlink);
+    void create_directory_symlink(const path& to, const path& new_symlink,
+                                          error_code& ec) noexcept;
+
+    void create_hard_link(const path& to, const path& new_hard_link);
+    void create_hard_link(const path& to, const path& new_hard_link,
+                                  error_code& ec) noexcept;
+
+    void create_symlink(const path& to, const path& new_symlink);
+    void create_symlink(const path& to, const path& new_symlink,
+                                error_code& ec) noexcept;
+
+    path current_path();
+    path current_path(error_code& ec);
+    void current_path(const path& p);
+    void current_path(const path& p, error_code& ec) noexcept;
+
+    bool exists(file_status s) noexcept;
+    bool exists(const path& p);
+    bool exists(const path& p, error_code& ec) noexcept;
+
+    bool equivalent(const path& p1, const path& p2);
+    bool equivalent(const path& p1, const path& p2, error_code& ec) noexcept;
+
+    uintmax_t    file_size(const path& p);
+    uintmax_t    file_size(const path& p, error_code& ec) noexcept;
+
+    uintmax_t    hard_link_count(const path& p);
+    uintmax_t    hard_link_count(const path& p, error_code& ec) noexcept;
+
+    bool is_block_file(file_status s) noexcept;
+    bool is_block_file(const path& p);
+    bool is_block_file(const path& p, error_code& ec) noexcept;
+
+    bool is_character_file(file_status s) noexcept;
+    bool is_character_file(const path& p);
+    bool is_character_file(const path& p, error_code& ec) noexcept;
+
+    bool is_directory(file_status s) noexcept;
+    bool is_directory(const path& p);
+    bool is_directory(const path& p, error_code& ec) noexcept;
+
+    bool is_empty(const path& p);
+    bool is_empty(const path& p, error_code& ec) noexcept;
+
+    bool is_fifo(file_status s) noexcept;
+    bool is_fifo(const path& p);
+    bool is_fifo(const path& p, error_code& ec) noexcept;
+
+    bool is_other(file_status s) noexcept;
+    bool is_other(const path& p);
+    bool is_other(const path& p, error_code& ec) noexcept;
+
+    bool is_regular_file(file_status s) noexcept;
+    bool is_regular_file(const path& p);
+    bool is_regular_file(const path& p, error_code& ec) noexcept;
+
+    bool is_socket(file_status s) noexcept;
+    bool is_socket(const path& p);
+    bool is_socket(const path& p, error_code& ec) noexcept;
+
+    bool is_symlink(file_status s) noexcept;
+    bool is_symlink(const path& p);
+    bool is_symlink(const path& p, error_code& ec) noexcept;
+
+    file_time_type  last_write_time(const path& p);
+    file_time_type  last_write_time(const path& p, error_code& ec) noexcept;
+    void last_write_time(const path& p, file_time_type new_time);
+    void last_write_time(const path& p, file_time_type new_time,
+                                 error_code& ec) noexcept;
+
+    void permissions(const path& p, perms prms,
+                     perm_options opts=perm_options::replace);
+    void permissions(const path& p, perms prms, error_code& ec) noexcept;
+    void permissions(const path& p, perms prms, perm_options opts,
+                     error_code& ec);
+
+    path proximate(const path& p, error_code& ec);
+    path proximate(const path& p, const path& base = current_path());
+    path proximate(const path& p, const path& base, error_code &ec);
+
+    path read_symlink(const path& p);
+    path read_symlink(const path& p, error_code& ec);
+
+    path relative(const path& p, error_code& ec);
+    path relative(const path& p, const path& base=current_path());
+    path relative(const path& p, const path& base, error_code& ec);
+
+    bool remove(const path& p);
+    bool remove(const path& p, error_code& ec) noexcept;
+
+    uintmax_t    remove_all(const path& p);
+    uintmax_t    remove_all(const path& p, error_code& ec);
+
+    void rename(const path& from, const path& to);
+    void rename(const path& from, const path& to, error_code& ec) noexcept;
+
+    void resize_file(const path& p, uintmax_t size);
+    void resize_file(const path& p, uintmax_t size, error_code& ec) noexcept;
+
+    space_info   space(const path& p);
+    space_info   space(const path& p, error_code& ec) noexcept;
+
+    file_status  status(const path& p);
+    file_status  status(const path& p, error_code& ec) noexcept;
+
+    bool status_known(file_status s) noexcept;
+
+    file_status  symlink_status(const path& p);
+    file_status  symlink_status(const path& p, error_code& ec) noexcept;
+
+    path temp_directory_path();
+    path temp_directory_path(error_code& ec);
+
+    path weakly_canonical(path const& p);
+    path weakly_canonical(path const& p, error_code& ec);
+
+} // namespace std::filesystem
+
+template <>
+inline constexpr bool std::ranges::enable_borrowed_range<std::filesystem::directory_iterator> = true;
+template <>
+inline constexpr bool std::ranges::enable_borrowed_range<std::filesystem::recursive_directory_iterator> = true;
+
+template <>
+inline constexpr bool std::ranges::enable_view<std::filesystem::directory_iterator> = true;
+template <>
+inline constexpr bool std::ranges::enable_view<std::filesystem::recursive_directory_iterator> = true;
+
+*/
+
+#include <__config>
+
+#if _LIBCPP_STD_VER >= 17
+#  include <__filesystem/copy_options.h>
+#  include <__filesystem/directory_entry.h>
+#  include <__filesystem/directory_iterator.h>
+#  include <__filesystem/directory_options.h>
+#  include <__filesystem/file_status.h>
+#  include <__filesystem/file_time_type.h>
+#  include <__filesystem/file_type.h>
+#  include <__filesystem/filesystem_error.h>
+#  include <__filesystem/operations.h>
+#  include <__filesystem/path.h>
+#  include <__filesystem/path_iterator.h>
+#  include <__filesystem/perm_options.h>
+#  include <__filesystem/perms.h>
+#  include <__filesystem/recursive_directory_iterator.h>
+#  include <__filesystem/space_info.h>
+#  include <__filesystem/u8path.h>
+#endif
+
+#include <version>
+
+// standard-mandated includes
+
+// [fs.filesystem.syn]
+#include <compare>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <concepts>
+#  include <cstdlib>
+#  include <cstring>
+#  include <iosfwd>
+#  include <new>
+#  include <system_error>
+#endif
+
+#endif // _LIBCPP_FILESYSTEM
diff --git a/libcxx/include/__cxx03/float.h b/libcxx/include/__cxx03/float.h
new file mode 100644
index 000000000000..d572866c1358
--- /dev/null
+++ b/libcxx/include/__cxx03/float.h
@@ -0,0 +1,95 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_FLOAT_H
+#define _LIBCPP_FLOAT_H
+
+/*
+    float.h synopsis
+
+Macros:
+
+    FLT_ROUNDS
+    FLT_EVAL_METHOD     // C99
+    FLT_RADIX
+
+    FLT_MANT_DIG
+    DBL_MANT_DIG
+    LDBL_MANT_DIG
+
+    FLT_HAS_SUBNORM     // C11
+    DBL_HAS_SUBNORM     // C11
+    LDBL_HAS_SUBNORM    // C11
+
+    DECIMAL_DIG         // C99
+    FLT_DECIMAL_DIG     // C11
+    DBL_DECIMAL_DIG     // C11
+    LDBL_DECIMAL_DIG    // C11
+
+    FLT_DIG
+    DBL_DIG
+    LDBL_DIG
+
+    FLT_MIN_EXP
+    DBL_MIN_EXP
+    LDBL_MIN_EXP
+
+    FLT_MIN_10_EXP
+    DBL_MIN_10_EXP
+    LDBL_MIN_10_EXP
+
+    FLT_MAX_EXP
+    DBL_MAX_EXP
+    LDBL_MAX_EXP
+
+    FLT_MAX_10_EXP
+    DBL_MAX_10_EXP
+    LDBL_MAX_10_EXP
+
+    FLT_MAX
+    DBL_MAX
+    LDBL_MAX
+
+    FLT_EPSILON
+    DBL_EPSILON
+    LDBL_EPSILON
+
+    FLT_MIN
+    DBL_MIN
+    LDBL_MIN
+
+    FLT_TRUE_MIN        // C11
+    DBL_TRUE_MIN        // C11
+    LDBL_TRUE_MIN       // C11
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if __has_include_next(<float.h>)
+#  include_next <float.h>
+#endif
+
+#ifdef __cplusplus
+
+#  ifndef FLT_EVAL_METHOD
+#    define FLT_EVAL_METHOD __FLT_EVAL_METHOD__
+#  endif
+
+#  ifndef DECIMAL_DIG
+#    define DECIMAL_DIG __DECIMAL_DIG__
+#  endif
+
+#endif // __cplusplus
+
+#endif // _LIBCPP_FLOAT_H
diff --git a/libcxx/include/__cxx03/format b/libcxx/include/__cxx03/format
new file mode 100644
index 000000000000..a88b3ef8528e
--- /dev/null
+++ b/libcxx/include/__cxx03/format
@@ -0,0 +1,257 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_FORMAT
+#define _LIBCPP_FORMAT
+
+/*
+
+namespace std {
+  // [format.context], class template basic_format_context
+  template<class Out, class charT> class basic_format_context;
+  using format_context = basic_format_context<unspecified, char>;
+  using wformat_context = basic_format_context<unspecified, wchar_t>;
+
+  // [format.args], class template basic_format_args
+  template<class Context> class basic_format_args;
+  using format_args = basic_format_args<format_context>;
+  using wformat_args = basic_format_args<wformat_context>;
+
+  // [format.fmt.string], class template basic_format_string
+  template<class charT, class... Args>
+    struct basic_format_string {                                // since C++23, exposition only before C++23
+    private:
+      basic_string_view<charT> str;                             // exposition only
+
+    public:
+      template<class T> consteval basic_format_string(const T& s);
+      basic_format_string(runtime-format-string<charT> s) noexcept : str(s.str) {}   // since C++26
+
+      constexpr basic_string_view<charT> get() const noexcept { return str; }
+    };
+  template<class... Args>
+    using format_string =                                       // since C++23, exposition only before C++23
+      basic_format_string<char, type_identity_t<Args>...>;
+  template<class... Args>
+    using wformat_string =                                      // since C++23, exposition only before C++23
+      basic_format_string<wchar_t, type_identity_t<Args>...>;
+
+  template<class charT> struct runtime-format-string {          // since C++26, exposition-only
+  private:
+    basic_string_view<charT> str;                               // exposition-only
+
+  public:
+    runtime-format-string(basic_string_view<charT> s) noexcept : str(s) {}
+
+    runtime-format-string(const runtime-format-string&) = delete;
+    runtime-format-string& operator=(const runtime-format-string&) = delete;
+  };
+
+  runtime-format-string<char> runtime_format(string_view fmt) noexcept {
+    return fmt;
+  }
+  runtime-format-string<wchar_t> runtime_format(wstring_view fmt) noexcept {
+    return fmt;
+  }
+
+  // [format.functions], formatting functions
+  template<class... Args>
+    string format(format-string<Args...> fmt, Args&&... args);
+  template<class... Args>
+    wstring format(wformat-string<Args...> fmt, Args&&... args);
+  template<class... Args>
+    string format(const locale& loc, format-string<Args...> fmt, Args&&... args);
+  template<class... Args>
+    wstring format(const locale& loc, wformat-string<Args...> fmt, Args&&... args);
+
+  string vformat(string_view fmt, format_args args);
+  wstring vformat(wstring_view fmt, wformat_args args);
+  string vformat(const locale& loc, string_view fmt, format_args args);
+  wstring vformat(const locale& loc, wstring_view fmt, wformat_args args);
+
+  template<class Out, class... Args>
+    Out format_to(Out out, format-string<Args...> fmt, Args&&... args);
+  template<class Out, class... Args>
+    Out format_to(Out out, wformat-string<Args...> fmt, Args&&... args);
+  template<class Out, class... Args>
+    Out format_to(Out out, const locale& loc, format-string<Args...> fmt, Args&&... args);
+  template<class Out, class... Args>
+    Out format_to(Out out, const locale& loc, wformat-string<Args...> fmt, Args&&... args);
+
+  template<class Out>
+    Out vformat_to(Out out, string_view fmt, format_args args);
+  template<class Out>
+    Out vformat_to(Out out, wstring_view fmt, wformat_args args);
+  template<class Out>
+    Out vformat_to(Out out, const locale& loc, string_view fmt,
+                   format_args char> args);
+  template<class Out>
+    Out vformat_to(Out out, const locale& loc, wstring_view fmt,
+                   wformat_args args);
+
+  template<class Out> struct format_to_n_result {
+    Out out;
+    iter_difference_t<Out> size;
+  };
+  template<class Out, class... Args>
+    format_to_n_result<Out> format_to_n(Out out, iter_difference_t<Out> n,
+                                        format-string<Args...> fmt, Args&&... args);
+  template<class Out, class... Args>
+    format_to_n_result<Out> format_to_n(Out out, iter_difference_t<Out> n,
+                                        wformat-string<Args...> fmt, Args&&... args);
+  template<class Out, class... Args>
+    format_to_n_result<Out> format_to_n(Out out, iter_difference_t<Out> n,
+                                        const locale& loc, format-string<Args...> fmt,
+                                        Args&&... args);
+  template<class Out, class... Args>
+    format_to_n_result<Out> format_to_n(Out out, iter_difference_t<Out> n,
+                                        const locale& loc, wformat-string<Args...> fmt,
+                                        Args&&... args);
+
+  template<class... Args>
+    size_t formatted_size(format-string<Args...> fmt, Args&&... args);
+  template<class... Args>
+    size_t formatted_size(wformat-string<Args...> fmt, Args&&... args);
+  template<class... Args>
+    size_t formatted_size(const locale& loc, format-string<Args...> fmt, Args&&... args);
+  template<class... Args>
+    size_t formatted_size(const locale& loc, wformat-string<Args...> fmt, Args&&... args);
+
+  // [format.formatter], formatter
+  template<class T, class charT = char> struct formatter;
+
+  // [format.parse.ctx], class template basic_format_parse_context
+  template<class charT> class basic_format_parse_context;
+  using format_parse_context = basic_format_parse_context<char>;
+  using wformat_parse_context = basic_format_parse_context<wchar_t>;
+
+  // [format.range], formatting of ranges
+  // [format.range.fmtkind], variable template format_kind
+  enum class range_format {                                     // since C++23
+    disabled,
+    map,
+    set,
+    sequence,
+    string,
+    debug_string
+  };
+
+  template<class R>
+    constexpr unspecified format_kind = unspecified;            // since C++23
+
+  template<ranges::input_range R>
+      requires same_as<R, remove_cvref_t<R>>
+    constexpr range_format format_kind<R> = see below;          // since C++23
+
+  // [format.range.formatter], class template range_formatter
+  template<class T, class charT = char>
+    requires same_as<remove_cvref_t<T>, T> && formattable<T, charT>
+  class range_formatter;                                        // since C++23
+
+  // [format.range.fmtdef], class template range-default-formatter
+  template<range_format K, ranges::input_range R, class charT>
+    struct range-default-formatter;                             // exposition only, since C++23
+
+  // [format.range.fmtmap], [format.range.fmtset], [format.range.fmtstr],
+  // specializations for maps, sets, and strings
+  template<ranges::input_range R, class charT>
+    requires (format_kind<R> != range_format::disabled) &&
+             formattable<ranges::range_reference_t<R>, charT>
+  struct formatter<R, charT> : range-default-formatter<format_kind<R>, R, charT> { }; // since C++23
+
+  // [format.arguments], arguments
+  // [format.arg], class template basic_format_arg
+  template<class Context> class basic_format_arg;
+
+  template<class Visitor, class Context>
+    see below visit_format_arg(Visitor&& vis, basic_format_arg<Context> arg); // Deprecated in C++26
+
+  // [format.arg.store], class template format-arg-store
+  template<class Context, class... Args> struct format-arg-store;      // exposition only
+
+  template<class Context = format_context, class... Args>
+    format-arg-store<Context, Args...>
+      make_format_args(Args&... args);
+  template<class... Args>
+    format-arg-store<wformat_context, Args...>
+      make_wformat_args(Args&... args);
+
+  // [format.error], class format_error
+  class format_error;
+}
+
+*/
+
+#include <__config>
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__format/buffer.h>
+#  include <__format/concepts.h>
+#  include <__format/container_adaptor.h>
+#  include <__format/enable_insertable.h>
+#  include <__format/escaped_output_table.h>
+#  include <__format/extended_grapheme_cluster_table.h>
+#  include <__format/format_arg.h>
+#  include <__format/format_arg_store.h>
+#  include <__format/format_args.h>
+#  include <__format/format_context.h>
+#  include <__format/format_error.h>
+#  include <__format/format_functions.h>
+#  include <__format/format_parse_context.h>
+#  include <__format/format_string.h>
+#  include <__format/format_to_n_result.h>
+#  include <__format/formatter.h>
+#  include <__format/formatter_bool.h>
+#  include <__format/formatter_char.h>
+#  include <__format/formatter_floating_point.h>
+#  include <__format/formatter_integer.h>
+#  include <__format/formatter_pointer.h>
+#  include <__format/formatter_string.h>
+#  include <__format/formatter_tuple.h>
+#  include <__format/parser_std_format_spec.h>
+#  include <__format/range_default_formatter.h>
+#  include <__format/range_formatter.h>
+#  include <__format/unicode.h>
+#  include <__fwd/format.h>
+#endif
+
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <array>
+#  include <cctype>
+#  include <cerrno>
+#  include <clocale>
+#  include <cmath>
+#  include <cstddef>
+#  include <cstdint>
+#  include <cstdlib>
+#  include <cstring>
+#  include <initializer_list>
+#  include <limits>
+#  include <locale>
+#  include <new>
+#  include <optional>
+#  include <queue>
+#  include <stack>
+#  include <stdexcept>
+#  include <string>
+#  include <string_view>
+#  include <tuple>
+
+#  if !defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS)
+#    include <cwchar>
+#  endif
+#endif
+
+#endif // _LIBCPP_FORMAT
diff --git a/libcxx/include/__cxx03/forward_list b/libcxx/include/__cxx03/forward_list
new file mode 100644
index 000000000000..7330a6619be7
--- /dev/null
+++ b/libcxx/include/__cxx03/forward_list
@@ -0,0 +1,1574 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_FORWARD_LIST
+#define _LIBCPP_FORWARD_LIST
+
+/*
+    forward_list synopsis
+
+namespace std
+{
+
+template <class T, class Allocator = allocator<T>>
+class forward_list
+{
+public:
+    typedef T         value_type;
+    typedef Allocator allocator_type;
+
+    typedef value_type&                                                reference;
+    typedef const value_type&                                          const_reference;
+    typedef typename allocator_traits<allocator_type>::pointer         pointer;
+    typedef typename allocator_traits<allocator_type>::const_pointer   const_pointer;
+    typedef typename allocator_traits<allocator_type>::size_type       size_type;
+    typedef typename allocator_traits<allocator_type>::difference_type difference_type;
+
+    typedef <details> iterator;
+    typedef <details> const_iterator;
+
+    forward_list()
+        noexcept(is_nothrow_default_constructible<allocator_type>::value);
+    explicit forward_list(const allocator_type& a);
+    explicit forward_list(size_type n);
+    explicit forward_list(size_type n, const allocator_type& a); // C++14
+    forward_list(size_type n, const value_type& v);
+    forward_list(size_type n, const value_type& v, const allocator_type& a);
+    template <class InputIterator>
+        forward_list(InputIterator first, InputIterator last);
+    template <class InputIterator>
+        forward_list(InputIterator first, InputIterator last, const allocator_type& a);
+    template<container-compatible-range<T> R>
+        forward_list(from_range_t, R&& rg, const Allocator& = Allocator()); // C++23
+    forward_list(const forward_list& x);
+    forward_list(const forward_list& x, const allocator_type& a);
+    forward_list(forward_list&& x)
+        noexcept(is_nothrow_move_constructible<allocator_type>::value);
+    forward_list(forward_list&& x, const allocator_type& a);
+    forward_list(initializer_list<value_type> il);
+    forward_list(initializer_list<value_type> il, const allocator_type& a);
+
+    ~forward_list();
+
+    forward_list& operator=(const forward_list& x);
+    forward_list& operator=(forward_list&& x)
+        noexcept(
+             allocator_type::propagate_on_container_move_assignment::value &&
+             is_nothrow_move_assignable<allocator_type>::value);
+    forward_list& operator=(initializer_list<value_type> il);
+
+    template <class InputIterator>
+        void assign(InputIterator first, InputIterator last);
+    template<container-compatible-range<T> R>
+      void assign_range(R&& rg); // C++23
+    void assign(size_type n, const value_type& v);
+    void assign(initializer_list<value_type> il);
+
+    allocator_type get_allocator() const noexcept;
+
+    iterator       begin() noexcept;
+    const_iterator begin() const noexcept;
+    iterator       end() noexcept;
+    const_iterator end() const noexcept;
+
+    const_iterator cbegin() const noexcept;
+    const_iterator cend() const noexcept;
+
+    iterator       before_begin() noexcept;
+    const_iterator before_begin() const noexcept;
+    const_iterator cbefore_begin() const noexcept;
+
+    bool empty() const noexcept;
+    size_type max_size() const noexcept;
+
+    reference       front();
+    const_reference front() const;
+
+    template <class... Args> reference emplace_front(Args&&... args);  // reference in C++17
+    void push_front(const value_type& v);
+    void push_front(value_type&& v);
+    template<container-compatible-range<T> R>
+      void prepend_range(R&& rg); // C++23
+
+    void pop_front();
+
+    template <class... Args>
+        iterator emplace_after(const_iterator p, Args&&... args);
+    iterator insert_after(const_iterator p, const value_type& v);
+    iterator insert_after(const_iterator p, value_type&& v);
+    iterator insert_after(const_iterator p, size_type n, const value_type& v);
+    template <class InputIterator>
+        iterator insert_after(const_iterator p,
+                              InputIterator first, InputIterator last);
+    template<container-compatible-range<T> R>
+      iterator insert_range_after(const_iterator position, R&& rg); // C++23
+    iterator insert_after(const_iterator p, initializer_list<value_type> il);
+
+    iterator erase_after(const_iterator p);
+    iterator erase_after(const_iterator first, const_iterator last);
+
+    void swap(forward_list& x)
+        noexcept(allocator_traits<allocator_type>::is_always_equal::value);  // C++17
+
+    void resize(size_type n);
+    void resize(size_type n, const value_type& v);
+    void clear() noexcept;
+
+    void splice_after(const_iterator p, forward_list& x);
+    void splice_after(const_iterator p, forward_list&& x);
+    void splice_after(const_iterator p, forward_list& x, const_iterator i);
+    void splice_after(const_iterator p, forward_list&& x, const_iterator i);
+    void splice_after(const_iterator p, forward_list& x,
+                      const_iterator first, const_iterator last);
+    void splice_after(const_iterator p, forward_list&& x,
+                      const_iterator first, const_iterator last);
+    size_type remove(const value_type& v);           // void before C++20
+    template <class Predicate>
+      size_type remove_if(Predicate pred);           // void before C++20
+    size_type unique();                              // void before C++20
+    template <class BinaryPredicate>
+      size_type unique(BinaryPredicate binary_pred); // void before C++20
+    void merge(forward_list& x);
+    void merge(forward_list&& x);
+    template <class Compare> void merge(forward_list& x, Compare comp);
+    template <class Compare> void merge(forward_list&& x, Compare comp);
+    void sort();
+    template <class Compare> void sort(Compare comp);
+    void reverse() noexcept;
+};
+
+
+template <class InputIterator, class Allocator = allocator<typename iterator_traits<InputIterator>::value_type>>
+    forward_list(InputIterator, InputIterator, Allocator = Allocator())
+    -> forward_list<typename iterator_traits<InputIterator>::value_type, Allocator>;  // C++17
+
+template<ranges::input_range R, class Allocator = allocator<ranges::range_value_t<R>>>
+  forward_list(from_range_t, R&&, Allocator = Allocator())
+      -> forward_list<ranges::range_value_t<R>, Allocator>; // C++23
+
+template <class T, class Allocator>
+    bool operator==(const forward_list<T, Allocator>& x,
+                    const forward_list<T, Allocator>& y);
+
+template <class T, class Allocator>
+    bool operator< (const forward_list<T, Allocator>& x,
+                    const forward_list<T, Allocator>& y); // removed in C++20
+
+template <class T, class Allocator>
+    bool operator!=(const forward_list<T, Allocator>& x,
+                    const forward_list<T, Allocator>& y); // removed in C++20
+
+template <class T, class Allocator>
+    bool operator> (const forward_list<T, Allocator>& x,
+                    const forward_list<T, Allocator>& y); // removed in C++20
+
+template <class T, class Allocator>
+    bool operator>=(const forward_list<T, Allocator>& x,
+                    const forward_list<T, Allocator>& y); // removed in C++20
+
+template <class T, class Allocator>
+    bool operator<=(const forward_list<T, Allocator>& x,
+                    const forward_list<T, Allocator>& y); // removed in C++20
+
+template<class T, class Allocator>
+    synth-three-way-result<T> operator<=>(const forward_list<T, Allocator>& x,
+                                          const forward_list<T, Allocator>& y); // since C++20
+
+template <class T, class Allocator>
+    void swap(forward_list<T, Allocator>& x, forward_list<T, Allocator>& y)
+         noexcept(noexcept(x.swap(y)));
+
+template <class T, class Allocator, class U>
+    typename forward_list<T, Allocator>::size_type
+    erase(forward_list<T, Allocator>& c, const U& value);       // C++20
+template <class T, class Allocator, class Predicate>
+    typename forward_list<T, Allocator>::size_type
+    erase_if(forward_list<T, Allocator>& c, Predicate pred);    // C++20
+
+}  // std
+
+*/
+
+#include <__algorithm/comp.h>
+#include <__algorithm/lexicographical_compare.h>
+#include <__algorithm/lexicographical_compare_three_way.h>
+#include <__algorithm/min.h>
+#include <__config>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/move_iterator.h>
+#include <__iterator/next.h>
+#include <__memory/addressof.h>
+#include <__memory/allocation_guard.h>
+#include <__memory/allocator.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/compressed_pair.h>
+#include <__memory/construct_at.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/swap_allocator.h>
+#include <__memory_resource/polymorphic_allocator.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/container_compatible_range.h>
+#include <__ranges/from_range.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/is_allocator.h>
+#include <__type_traits/is_const.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_swappable.h>
+#include <__type_traits/type_identity.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <limits>
+#include <new> // __launder
+#include <version>
+
+// standard-mandated includes
+
+// [iterator.range]
+#include <__iterator/access.h>
+#include <__iterator/data.h>
+#include <__iterator/empty.h>
+#include <__iterator/reverse_access.h>
+#include <__iterator/size.h>
+
+// [forward.list.syn]
+#include <compare>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _VoidPtr>
+struct __forward_list_node;
+template <class _NodePtr>
+struct __forward_begin_node;
+
+template <class>
+struct __forward_list_node_value_type;
+
+template <class _Tp, class _VoidPtr>
+struct __forward_list_node_value_type<__forward_list_node<_Tp, _VoidPtr> > {
+  typedef _Tp type;
+};
+
+template <class _NodePtr>
+struct __forward_node_traits {
+  typedef __remove_cv_t<typename pointer_traits<_NodePtr>::element_type> __node_type;
+  typedef typename __forward_list_node_value_type<__node_type>::type __node_value_type;
+  typedef _NodePtr __node_pointer;
+  typedef __forward_begin_node<_NodePtr> __begin_node;
+  typedef __rebind_pointer_t<_NodePtr, __begin_node> __begin_node_pointer;
+  typedef __rebind_pointer_t<_NodePtr, void> __void_pointer;
+
+#if defined(_LIBCPP_ABI_FORWARD_LIST_REMOVE_NODE_POINTER_UB)
+  typedef __begin_node_pointer __iter_node_pointer;
+#else
+  typedef __conditional_t<is_pointer<__void_pointer>::value, __begin_node_pointer, __node_pointer> __iter_node_pointer;
+#endif
+
+  typedef __conditional_t<is_same<__iter_node_pointer, __node_pointer>::value, __begin_node_pointer, __node_pointer>
+      __non_iter_node_pointer;
+
+  _LIBCPP_HIDE_FROM_ABI static __iter_node_pointer __as_iter_node(__iter_node_pointer __p) { return __p; }
+  _LIBCPP_HIDE_FROM_ABI static __iter_node_pointer __as_iter_node(__non_iter_node_pointer __p) {
+    return static_cast<__iter_node_pointer>(static_cast<__void_pointer>(__p));
+  }
+};
+
+template <class _NodePtr>
+struct __forward_begin_node {
+  typedef _NodePtr pointer;
+  typedef __rebind_pointer_t<_NodePtr, __forward_begin_node> __begin_node_pointer;
+
+  pointer __next_;
+
+  _LIBCPP_HIDE_FROM_ABI __forward_begin_node() : __next_(nullptr) {}
+  _LIBCPP_HIDE_FROM_ABI explicit __forward_begin_node(pointer __n) : __next_(__n) {}
+
+  _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __next_as_begin() const {
+    return static_cast<__begin_node_pointer>(__next_);
+  }
+};
+
+template <class _Tp, class _VoidPtr>
+using __begin_node_of = __forward_begin_node<__rebind_pointer_t<_VoidPtr, __forward_list_node<_Tp, _VoidPtr> > >;
+
+template <class _Tp, class _VoidPtr>
+struct __forward_list_node : public __begin_node_of<_Tp, _VoidPtr> {
+  typedef _Tp value_type;
+  typedef __begin_node_of<_Tp, _VoidPtr> _Base;
+  typedef typename _Base::pointer _NodePtr;
+
+  // We allow starting the lifetime of nodes without initializing the value held by the node,
+  // since that is handled by the list itself in order to be allocator-aware.
+#ifndef _LIBCPP_CXX03_LANG
+
+private:
+  union {
+    _Tp __value_;
+  };
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; }
+#else
+
+private:
+  _ALIGNAS_TYPE(_Tp) char __buffer_[sizeof(_Tp)];
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return *std::__launder(reinterpret_cast<_Tp*>(&__buffer_)); }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI explicit __forward_list_node(_NodePtr __next) : _Base(__next) {}
+  _LIBCPP_HIDE_FROM_ABI ~__forward_list_node() {}
+};
+
+template <class _Tp, class _Alloc = allocator<_Tp> >
+class _LIBCPP_TEMPLATE_VIS forward_list;
+template <class _NodeConstPtr>
+class _LIBCPP_TEMPLATE_VIS __forward_list_const_iterator;
+
+template <class _NodePtr>
+class _LIBCPP_TEMPLATE_VIS __forward_list_iterator {
+  typedef __forward_node_traits<_NodePtr> __traits;
+  typedef typename __traits::__node_pointer __node_pointer;
+  typedef typename __traits::__begin_node_pointer __begin_node_pointer;
+  typedef typename __traits::__iter_node_pointer __iter_node_pointer;
+  typedef typename __traits::__void_pointer __void_pointer;
+
+  __iter_node_pointer __ptr_;
+
+  _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __get_begin() const {
+    return static_cast<__begin_node_pointer>(static_cast<__void_pointer>(__ptr_));
+  }
+  _LIBCPP_HIDE_FROM_ABI __node_pointer __get_unsafe_node_pointer() const {
+    return static_cast<__node_pointer>(static_cast<__void_pointer>(__ptr_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI explicit __forward_list_iterator(nullptr_t) _NOEXCEPT : __ptr_(nullptr) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit __forward_list_iterator(__begin_node_pointer __p) _NOEXCEPT
+      : __ptr_(__traits::__as_iter_node(__p)) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit __forward_list_iterator(__node_pointer __p) _NOEXCEPT
+      : __ptr_(__traits::__as_iter_node(__p)) {}
+
+  template <class, class>
+  friend class _LIBCPP_TEMPLATE_VIS forward_list;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __forward_list_const_iterator;
+
+public:
+  typedef forward_iterator_tag iterator_category;
+  typedef typename __traits::__node_value_type value_type;
+  typedef value_type& reference;
+  typedef typename pointer_traits<__node_pointer>::difference_type difference_type;
+  typedef __rebind_pointer_t<__node_pointer, value_type> pointer;
+
+  _LIBCPP_HIDE_FROM_ABI __forward_list_iterator() _NOEXCEPT : __ptr_(nullptr) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __get_unsafe_node_pointer()->__get_value(); }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
+    return pointer_traits<pointer>::pointer_to(__get_unsafe_node_pointer()->__get_value());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __forward_list_iterator& operator++() {
+    __ptr_ = __traits::__as_iter_node(__ptr_->__next_);
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __forward_list_iterator operator++(int) {
+    __forward_list_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __forward_list_iterator& __x, const __forward_list_iterator& __y) {
+    return __x.__ptr_ == __y.__ptr_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __forward_list_iterator& __x, const __forward_list_iterator& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _NodeConstPtr>
+class _LIBCPP_TEMPLATE_VIS __forward_list_const_iterator {
+  static_assert(!is_const<typename pointer_traits<_NodeConstPtr>::element_type>::value, "");
+  typedef _NodeConstPtr _NodePtr;
+
+  typedef __forward_node_traits<_NodePtr> __traits;
+  typedef typename __traits::__node_type __node_type;
+  typedef typename __traits::__node_pointer __node_pointer;
+  typedef typename __traits::__begin_node_pointer __begin_node_pointer;
+  typedef typename __traits::__iter_node_pointer __iter_node_pointer;
+  typedef typename __traits::__void_pointer __void_pointer;
+
+  __iter_node_pointer __ptr_;
+
+  _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __get_begin() const {
+    return static_cast<__begin_node_pointer>(static_cast<__void_pointer>(__ptr_));
+  }
+  _LIBCPP_HIDE_FROM_ABI __node_pointer __get_unsafe_node_pointer() const {
+    return static_cast<__node_pointer>(static_cast<__void_pointer>(__ptr_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI explicit __forward_list_const_iterator(nullptr_t) _NOEXCEPT : __ptr_(nullptr) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit __forward_list_const_iterator(__begin_node_pointer __p) _NOEXCEPT
+      : __ptr_(__traits::__as_iter_node(__p)) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit __forward_list_const_iterator(__node_pointer __p) _NOEXCEPT
+      : __ptr_(__traits::__as_iter_node(__p)) {}
+
+  template <class, class>
+  friend class forward_list;
+
+public:
+  typedef forward_iterator_tag iterator_category;
+  typedef typename __traits::__node_value_type value_type;
+  typedef const value_type& reference;
+  typedef typename pointer_traits<__node_pointer>::difference_type difference_type;
+  typedef __rebind_pointer_t<__node_pointer, const value_type> pointer;
+
+  _LIBCPP_HIDE_FROM_ABI __forward_list_const_iterator() _NOEXCEPT : __ptr_(nullptr) {}
+  _LIBCPP_HIDE_FROM_ABI __forward_list_const_iterator(__forward_list_iterator<__node_pointer> __p) _NOEXCEPT
+      : __ptr_(__p.__ptr_) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __get_unsafe_node_pointer()->__get_value(); }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
+    return pointer_traits<pointer>::pointer_to(__get_unsafe_node_pointer()->__get_value());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __forward_list_const_iterator& operator++() {
+    __ptr_ = __traits::__as_iter_node(__ptr_->__next_);
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __forward_list_const_iterator operator++(int) {
+    __forward_list_const_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const __forward_list_const_iterator& __x, const __forward_list_const_iterator& __y) {
+    return __x.__ptr_ == __y.__ptr_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator!=(const __forward_list_const_iterator& __x, const __forward_list_const_iterator& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _Tp, class _Alloc>
+class __forward_list_base {
+protected:
+  typedef _Tp value_type;
+  typedef _Alloc allocator_type;
+
+  typedef typename allocator_traits<allocator_type>::void_pointer void_pointer;
+  typedef __forward_list_node<value_type, void_pointer> __node_type;
+  typedef __begin_node_of<value_type, void_pointer> __begin_node;
+  typedef __rebind_alloc<allocator_traits<allocator_type>, __node_type> __node_allocator;
+  typedef allocator_traits<__node_allocator> __node_traits;
+  typedef typename __node_traits::pointer __node_pointer;
+
+  typedef __rebind_alloc<allocator_traits<allocator_type>, __begin_node> __begin_node_allocator;
+  typedef typename allocator_traits<__begin_node_allocator>::pointer __begin_node_pointer;
+
+  __compressed_pair<__begin_node, __node_allocator> __before_begin_;
+
+  _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __before_begin() _NOEXCEPT {
+    return pointer_traits<__begin_node_pointer>::pointer_to(__before_begin_.first());
+  }
+  _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __before_begin() const _NOEXCEPT {
+    return pointer_traits<__begin_node_pointer>::pointer_to(const_cast<__begin_node&>(__before_begin_.first()));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __node_allocator& __alloc() _NOEXCEPT { return __before_begin_.second(); }
+  _LIBCPP_HIDE_FROM_ABI const __node_allocator& __alloc() const _NOEXCEPT { return __before_begin_.second(); }
+
+  typedef __forward_list_iterator<__node_pointer> iterator;
+  typedef __forward_list_const_iterator<__node_pointer> const_iterator;
+
+  _LIBCPP_HIDE_FROM_ABI __forward_list_base() _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value)
+      : __before_begin_(__begin_node(), __default_init_tag()) {}
+  _LIBCPP_HIDE_FROM_ABI explicit __forward_list_base(const allocator_type& __a)
+      : __before_begin_(__begin_node(), __node_allocator(__a)) {}
+  _LIBCPP_HIDE_FROM_ABI explicit __forward_list_base(const __node_allocator& __a)
+      : __before_begin_(__begin_node(), __a) {}
+
+public:
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI
+  __forward_list_base(__forward_list_base&& __x) noexcept(is_nothrow_move_constructible<__node_allocator>::value);
+  _LIBCPP_HIDE_FROM_ABI __forward_list_base(__forward_list_base&& __x, const allocator_type& __a);
+#endif // _LIBCPP_CXX03_LANG
+
+  __forward_list_base(const __forward_list_base&)            = delete;
+  __forward_list_base& operator=(const __forward_list_base&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI ~__forward_list_base();
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __forward_list_base& __x) {
+    __copy_assign_alloc(__x, integral_constant<bool, __node_traits::propagate_on_container_copy_assignment::value>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__forward_list_base& __x)
+      _NOEXCEPT_(!__node_traits::propagate_on_container_move_assignment::value ||
+                 is_nothrow_move_assignable<__node_allocator>::value) {
+    __move_assign_alloc(__x, integral_constant<bool, __node_traits::propagate_on_container_move_assignment::value>());
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI __node_pointer __create_node(__node_pointer __next, _Args&&... __args) {
+    __node_allocator& __a = __alloc();
+    __allocation_guard<__node_allocator> __guard(__a, 1);
+    // Begin the lifetime of the node itself. Note that this doesn't begin the lifetime of the value
+    // held inside the node, since we need to use the allocator's construct() method for that.
+    //
+    // We don't use the allocator's construct() method to construct the node itself since the
+    // Cpp17FooInsertable named requirements don't require the allocator's construct() method
+    // to work on anything other than the value_type.
+    std::__construct_at(std::addressof(*__guard.__get()), __next);
+
+    // Now construct the value_type using the allocator's construct() method.
+    __node_traits::construct(__a, std::addressof(__guard.__get()->__get_value()), std::forward<_Args>(__args)...);
+    return __guard.__release_ptr();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __delete_node(__node_pointer __node) {
+    // For the same reason as above, we use the allocator's destroy() method for the value_type,
+    // but not for the node itself.
+    __node_allocator& __a = __alloc();
+    __node_traits::destroy(__a, std::addressof(__node->__get_value()));
+    std::__destroy_at(std::addressof(*__node));
+    __node_traits::deallocate(__a, __node, 1);
+  }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI void swap(__forward_list_base& __x)
+#if _LIBCPP_STD_VER >= 14
+      _NOEXCEPT;
+#else
+      _NOEXCEPT_(!__node_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__node_allocator>);
+#endif
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT;
+
+private:
+  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __forward_list_base&, false_type) {}
+  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __forward_list_base& __x, true_type) {
+    if (__alloc() != __x.__alloc())
+      clear();
+    __alloc() = __x.__alloc();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__forward_list_base&, false_type) _NOEXCEPT {}
+  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__forward_list_base& __x, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<__node_allocator>::value) {
+    __alloc() = std::move(__x.__alloc());
+  }
+};
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Alloc>
+inline __forward_list_base<_Tp, _Alloc>::__forward_list_base(__forward_list_base&& __x) noexcept(
+    is_nothrow_move_constructible<__node_allocator>::value)
+    : __before_begin_(std::move(__x.__before_begin_)) {
+  __x.__before_begin()->__next_ = nullptr;
+}
+
+template <class _Tp, class _Alloc>
+inline __forward_list_base<_Tp, _Alloc>::__forward_list_base(__forward_list_base&& __x, const allocator_type& __a)
+    : __before_begin_(__begin_node(), __node_allocator(__a)) {
+  if (__alloc() == __x.__alloc()) {
+    __before_begin()->__next_     = __x.__before_begin()->__next_;
+    __x.__before_begin()->__next_ = nullptr;
+  }
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Alloc>
+__forward_list_base<_Tp, _Alloc>::~__forward_list_base() {
+  clear();
+}
+
+template <class _Tp, class _Alloc>
+inline void __forward_list_base<_Tp, _Alloc>::swap(__forward_list_base& __x)
+#if _LIBCPP_STD_VER >= 14
+    _NOEXCEPT
+#else
+    _NOEXCEPT_(!__node_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__node_allocator>)
+#endif
+{
+  std::__swap_allocator(
+      __alloc(), __x.__alloc(), integral_constant<bool, __node_traits::propagate_on_container_swap::value>());
+  using std::swap;
+  swap(__before_begin()->__next_, __x.__before_begin()->__next_);
+}
+
+template <class _Tp, class _Alloc>
+void __forward_list_base<_Tp, _Alloc>::clear() _NOEXCEPT {
+  for (__node_pointer __p = __before_begin()->__next_; __p != nullptr;) {
+    __node_pointer __next = __p->__next_;
+    __delete_node(__p);
+    __p = __next;
+  }
+  __before_begin()->__next_ = nullptr;
+}
+
+template <class _Tp, class _Alloc /*= allocator<_Tp>*/>
+class _LIBCPP_TEMPLATE_VIS forward_list : private __forward_list_base<_Tp, _Alloc> {
+  typedef __forward_list_base<_Tp, _Alloc> base;
+  typedef typename base::__node_allocator __node_allocator;
+  typedef typename base::__node_type __node_type;
+  typedef typename base::__node_traits __node_traits;
+  typedef typename base::__node_pointer __node_pointer;
+  typedef typename base::__begin_node_pointer __begin_node_pointer;
+
+public:
+  typedef _Tp value_type;
+  typedef _Alloc allocator_type;
+
+  static_assert(__check_valid_allocator<allocator_type>::value, "");
+
+  static_assert(is_same<value_type, typename allocator_type::value_type>::value,
+                "Allocator::value_type must be same type as value_type");
+
+  static_assert(!is_same<allocator_type, __node_allocator>::value,
+                "internal allocator type must differ from user-specified type; otherwise overload resolution breaks");
+
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+  typedef typename allocator_traits<allocator_type>::pointer pointer;
+  typedef typename allocator_traits<allocator_type>::const_pointer const_pointer;
+  typedef typename allocator_traits<allocator_type>::size_type size_type;
+  typedef typename allocator_traits<allocator_type>::difference_type difference_type;
+
+  typedef typename base::iterator iterator;
+  typedef typename base::const_iterator const_iterator;
+#if _LIBCPP_STD_VER >= 20
+  typedef size_type __remove_return_type;
+#else
+  typedef void __remove_return_type;
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI forward_list() _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value) {
+  } // = default;
+  _LIBCPP_HIDE_FROM_ABI explicit forward_list(const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI explicit forward_list(size_type __n);
+#if _LIBCPP_STD_VER >= 14
+  _LIBCPP_HIDE_FROM_ABI explicit forward_list(size_type __n, const allocator_type& __a);
+#endif
+  _LIBCPP_HIDE_FROM_ABI forward_list(size_type __n, const value_type& __v);
+
+  template <__enable_if_t<__is_allocator<_Alloc>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI forward_list(size_type __n, const value_type& __v, const allocator_type& __a) : base(__a) {
+    insert_after(cbefore_begin(), __n, __v);
+  }
+
+  template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI forward_list(_InputIterator __f, _InputIterator __l);
+
+  template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI forward_list(_InputIterator __f, _InputIterator __l, const allocator_type& __a);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI forward_list(from_range_t, _Range&& __range, const allocator_type& __a = allocator_type())
+      : base(__a) {
+    prepend_range(std::forward<_Range>(__range));
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI forward_list(const forward_list& __x);
+  _LIBCPP_HIDE_FROM_ABI forward_list(const forward_list& __x, const __type_identity_t<allocator_type>& __a);
+
+  _LIBCPP_HIDE_FROM_ABI forward_list& operator=(const forward_list& __x);
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI forward_list(forward_list&& __x) noexcept(is_nothrow_move_constructible<base>::value)
+      : base(std::move(__x)) {}
+  _LIBCPP_HIDE_FROM_ABI forward_list(forward_list&& __x, const __type_identity_t<allocator_type>& __a);
+
+  _LIBCPP_HIDE_FROM_ABI forward_list(initializer_list<value_type> __il);
+  _LIBCPP_HIDE_FROM_ABI forward_list(initializer_list<value_type> __il, const allocator_type& __a);
+
+  _LIBCPP_HIDE_FROM_ABI forward_list& operator=(forward_list&& __x) noexcept(
+      __node_traits::propagate_on_container_move_assignment::value &&
+      is_nothrow_move_assignable<allocator_type>::value);
+
+  _LIBCPP_HIDE_FROM_ABI forward_list& operator=(initializer_list<value_type> __il);
+
+  _LIBCPP_HIDE_FROM_ABI void assign(initializer_list<value_type> __il);
+#endif // _LIBCPP_CXX03_LANG
+
+  // ~forward_list() = default;
+
+  template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI assign(_InputIterator __f, _InputIterator __l);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI void assign_range(_Range&& __range) {
+    __assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI void assign(size_type __n, const value_type& __v);
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT { return allocator_type(base::__alloc()); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return iterator(base::__before_begin()->__next_); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT {
+    return const_iterator(base::__before_begin()->__next_);
+  }
+  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return iterator(nullptr); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return const_iterator(nullptr); }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT {
+    return const_iterator(base::__before_begin()->__next_);
+  }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return const_iterator(nullptr); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator before_begin() _NOEXCEPT { return iterator(base::__before_begin()); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator before_begin() const _NOEXCEPT { return const_iterator(base::__before_begin()); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cbefore_begin() const _NOEXCEPT {
+    return const_iterator(base::__before_begin());
+  }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT {
+    return base::__before_begin()->__next_ == nullptr;
+  }
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
+    return std::min<size_type>(__node_traits::max_size(base::__alloc()), numeric_limits<difference_type>::max());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI reference front() { return base::__before_begin()->__next_->__get_value(); }
+  _LIBCPP_HIDE_FROM_ABI const_reference front() const { return base::__before_begin()->__next_->__get_value(); }
+
+#ifndef _LIBCPP_CXX03_LANG
+#  if _LIBCPP_STD_VER >= 17
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI reference emplace_front(_Args&&... __args);
+#  else
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI void emplace_front(_Args&&... __args);
+#  endif
+  _LIBCPP_HIDE_FROM_ABI void push_front(value_type&& __v);
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI void push_front(const value_type& __v);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI void prepend_range(_Range&& __range) {
+    insert_range_after(cbefore_begin(), std::forward<_Range>(__range));
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI void pop_front();
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace_after(const_iterator __p, _Args&&... __args);
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, value_type&& __v);
+  _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, initializer_list<value_type> __il) {
+    return insert_after(__p, __il.begin(), __il.end());
+  }
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, const value_type& __v);
+  _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, size_type __n, const value_type& __v);
+  template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, _InputIterator __f, _InputIterator __l);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI iterator insert_range_after(const_iterator __position, _Range&& __range) {
+    return __insert_after_with_sentinel(__position, ranges::begin(__range), ranges::end(__range));
+  }
+#endif
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI iterator __insert_after_with_sentinel(const_iterator __p, _InputIterator __f, _Sentinel __l);
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase_after(const_iterator __p);
+  _LIBCPP_HIDE_FROM_ABI iterator erase_after(const_iterator __f, const_iterator __l);
+
+  _LIBCPP_HIDE_FROM_ABI void swap(forward_list& __x)
+#if _LIBCPP_STD_VER >= 14
+      _NOEXCEPT
+#else
+      _NOEXCEPT_(!__node_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__node_allocator>)
+#endif
+  {
+    base::swap(__x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void resize(size_type __n);
+  _LIBCPP_HIDE_FROM_ABI void resize(size_type __n, const value_type& __v);
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { base::clear(); }
+
+  _LIBCPP_HIDE_FROM_ABI void splice_after(const_iterator __p, forward_list&& __x);
+  _LIBCPP_HIDE_FROM_ABI void splice_after(const_iterator __p, forward_list&& __x, const_iterator __i);
+  _LIBCPP_HIDE_FROM_ABI void
+  splice_after(const_iterator __p, forward_list&& __x, const_iterator __f, const_iterator __l);
+  _LIBCPP_HIDE_FROM_ABI void splice_after(const_iterator __p, forward_list& __x);
+  _LIBCPP_HIDE_FROM_ABI void splice_after(const_iterator __p, forward_list& __x, const_iterator __i);
+  _LIBCPP_HIDE_FROM_ABI void
+  splice_after(const_iterator __p, forward_list& __x, const_iterator __f, const_iterator __l);
+  _LIBCPP_HIDE_FROM_ABI __remove_return_type remove(const value_type& __v);
+  template <class _Predicate>
+  _LIBCPP_HIDE_FROM_ABI __remove_return_type remove_if(_Predicate __pred);
+  _LIBCPP_HIDE_FROM_ABI __remove_return_type unique() { return unique(__equal_to()); }
+  template <class _BinaryPredicate>
+  _LIBCPP_HIDE_FROM_ABI __remove_return_type unique(_BinaryPredicate __binary_pred);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI void merge(forward_list&& __x) { merge(__x, __less<>()); }
+  template <class _Compare>
+  _LIBCPP_HIDE_FROM_ABI void merge(forward_list&& __x, _Compare __comp) {
+    merge(__x, std::move(__comp));
+  }
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI void merge(forward_list& __x) { merge(__x, __less<>()); }
+  template <class _Compare>
+  _LIBCPP_HIDE_FROM_ABI void merge(forward_list& __x, _Compare __comp);
+  _LIBCPP_HIDE_FROM_ABI void sort() { sort(__less<>()); }
+  template <class _Compare>
+  _LIBCPP_HIDE_FROM_ABI void sort(_Compare __comp);
+  _LIBCPP_HIDE_FROM_ABI void reverse() _NOEXCEPT;
+
+private:
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI void __move_assign(forward_list& __x, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value);
+  _LIBCPP_HIDE_FROM_ABI void __move_assign(forward_list& __x, false_type);
+#endif // _LIBCPP_CXX03_LANG
+
+  template <class _Iter, class _Sent>
+  _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iter __f, _Sent __l);
+
+  template <class _Compare>
+  static _LIBCPP_HIDE_FROM_ABI __node_pointer __merge(__node_pointer __f1, __node_pointer __f2, _Compare& __comp);
+
+  // TODO: Make this _LIBCPP_HIDE_FROM_ABI
+  template <class _Compare>
+  static _LIBCPP_HIDDEN __node_pointer __sort(__node_pointer __f, difference_type __sz, _Compare& __comp);
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator,
+          class _Alloc = allocator<__iter_value_type<_InputIterator>>,
+          class        = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class        = enable_if_t<__is_allocator<_Alloc>::value> >
+forward_list(_InputIterator, _InputIterator) -> forward_list<__iter_value_type<_InputIterator>, _Alloc>;
+
+template <class _InputIterator,
+          class _Alloc,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<__is_allocator<_Alloc>::value> >
+forward_list(_InputIterator, _InputIterator, _Alloc) -> forward_list<__iter_value_type<_InputIterator>, _Alloc>;
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range,
+          class _Alloc = allocator<ranges::range_value_t<_Range>>,
+          class        = enable_if_t<__is_allocator<_Alloc>::value> >
+forward_list(from_range_t, _Range&&, _Alloc = _Alloc()) -> forward_list<ranges::range_value_t<_Range>, _Alloc>;
+#endif
+
+template <class _Tp, class _Alloc>
+inline forward_list<_Tp, _Alloc>::forward_list(const allocator_type& __a) : base(__a) {}
+
+template <class _Tp, class _Alloc>
+forward_list<_Tp, _Alloc>::forward_list(size_type __n) {
+  if (__n > 0) {
+    for (__begin_node_pointer __p = base::__before_begin(); __n > 0; --__n, __p = __p->__next_as_begin()) {
+      __p->__next_ = this->__create_node(/* next = */ nullptr);
+    }
+  }
+}
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp, class _Alloc>
+forward_list<_Tp, _Alloc>::forward_list(size_type __n, const allocator_type& __base_alloc) : base(__base_alloc) {
+  if (__n > 0) {
+    for (__begin_node_pointer __p = base::__before_begin(); __n > 0; --__n, __p = __p->__next_as_begin()) {
+      __p->__next_ = this->__create_node(/* next = */ nullptr);
+    }
+  }
+}
+#endif
+
+template <class _Tp, class _Alloc>
+forward_list<_Tp, _Alloc>::forward_list(size_type __n, const value_type& __v) {
+  insert_after(cbefore_begin(), __n, __v);
+}
+
+template <class _Tp, class _Alloc>
+template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> >
+forward_list<_Tp, _Alloc>::forward_list(_InputIterator __f, _InputIterator __l) {
+  insert_after(cbefore_begin(), __f, __l);
+}
+
+template <class _Tp, class _Alloc>
+template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> >
+forward_list<_Tp, _Alloc>::forward_list(_InputIterator __f, _InputIterator __l, const allocator_type& __a) : base(__a) {
+  insert_after(cbefore_begin(), __f, __l);
+}
+
+template <class _Tp, class _Alloc>
+forward_list<_Tp, _Alloc>::forward_list(const forward_list& __x)
+    : base(__node_traits::select_on_container_copy_construction(__x.__alloc())) {
+  insert_after(cbefore_begin(), __x.begin(), __x.end());
+}
+
+template <class _Tp, class _Alloc>
+forward_list<_Tp, _Alloc>::forward_list(const forward_list& __x, const __type_identity_t<allocator_type>& __a)
+    : base(__a) {
+  insert_after(cbefore_begin(), __x.begin(), __x.end());
+}
+
+template <class _Tp, class _Alloc>
+forward_list<_Tp, _Alloc>& forward_list<_Tp, _Alloc>::operator=(const forward_list& __x) {
+  if (this != std::addressof(__x)) {
+    base::__copy_assign_alloc(__x);
+    assign(__x.begin(), __x.end());
+  }
+  return *this;
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Tp, class _Alloc>
+forward_list<_Tp, _Alloc>::forward_list(forward_list&& __x, const __type_identity_t<allocator_type>& __a)
+    : base(std::move(__x), __a) {
+  if (base::__alloc() != __x.__alloc()) {
+    typedef move_iterator<iterator> _Ip;
+    insert_after(cbefore_begin(), _Ip(__x.begin()), _Ip(__x.end()));
+  }
+}
+
+template <class _Tp, class _Alloc>
+forward_list<_Tp, _Alloc>::forward_list(initializer_list<value_type> __il) {
+  insert_after(cbefore_begin(), __il.begin(), __il.end());
+}
+
+template <class _Tp, class _Alloc>
+forward_list<_Tp, _Alloc>::forward_list(initializer_list<value_type> __il, const allocator_type& __a) : base(__a) {
+  insert_after(cbefore_begin(), __il.begin(), __il.end());
+}
+
+template <class _Tp, class _Alloc>
+void forward_list<_Tp, _Alloc>::__move_assign(forward_list& __x, true_type)
+    _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
+  clear();
+  base::__move_assign_alloc(__x);
+  base::__before_begin()->__next_ = __x.__before_begin()->__next_;
+  __x.__before_begin()->__next_   = nullptr;
+}
+
+template <class _Tp, class _Alloc>
+void forward_list<_Tp, _Alloc>::__move_assign(forward_list& __x, false_type) {
+  if (base::__alloc() == __x.__alloc())
+    __move_assign(__x, true_type());
+  else {
+    typedef move_iterator<iterator> _Ip;
+    assign(_Ip(__x.begin()), _Ip(__x.end()));
+  }
+}
+
+template <class _Tp, class _Alloc>
+inline forward_list<_Tp, _Alloc>& forward_list<_Tp, _Alloc>::operator=(forward_list&& __x) _NOEXCEPT_(
+    __node_traits::propagate_on_container_move_assignment::value&& is_nothrow_move_assignable<allocator_type>::value) {
+  __move_assign(__x, integral_constant<bool, __node_traits::propagate_on_container_move_assignment::value>());
+  return *this;
+}
+
+template <class _Tp, class _Alloc>
+inline forward_list<_Tp, _Alloc>& forward_list<_Tp, _Alloc>::operator=(initializer_list<value_type> __il) {
+  assign(__il.begin(), __il.end());
+  return *this;
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Alloc>
+template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> >
+void forward_list<_Tp, _Alloc>::assign(_InputIterator __f, _InputIterator __l) {
+  __assign_with_sentinel(__f, __l);
+}
+
+template <class _Tp, class _Alloc>
+template <class _Iter, class _Sent>
+_LIBCPP_HIDE_FROM_ABI void forward_list<_Tp, _Alloc>::__assign_with_sentinel(_Iter __f, _Sent __l) {
+  iterator __i = before_begin();
+  iterator __j = std::next(__i);
+  iterator __e = end();
+  for (; __j != __e && __f != __l; ++__i, (void)++__j, ++__f)
+    *__j = *__f;
+  if (__j == __e)
+    __insert_after_with_sentinel(__i, std::move(__f), std::move(__l));
+  else
+    erase_after(__i, __e);
+}
+
+template <class _Tp, class _Alloc>
+void forward_list<_Tp, _Alloc>::assign(size_type __n, const value_type& __v) {
+  iterator __i = before_begin();
+  iterator __j = std::next(__i);
+  iterator __e = end();
+  for (; __j != __e && __n > 0; --__n, ++__i, ++__j)
+    *__j = __v;
+  if (__j == __e)
+    insert_after(__i, __n, __v);
+  else
+    erase_after(__i, __e);
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Alloc>
+inline void forward_list<_Tp, _Alloc>::assign(initializer_list<value_type> __il) {
+  assign(__il.begin(), __il.end());
+}
+
+template <class _Tp, class _Alloc>
+template <class... _Args>
+#  if _LIBCPP_STD_VER >= 17
+typename forward_list<_Tp, _Alloc>::reference
+#  else
+void
+#  endif
+forward_list<_Tp, _Alloc>::emplace_front(_Args&&... __args) {
+  base::__before_begin()->__next_ =
+      this->__create_node(/* next = */ base::__before_begin()->__next_, std::forward<_Args>(__args)...);
+#  if _LIBCPP_STD_VER >= 17
+  return base::__before_begin()->__next_->__get_value();
+#  endif
+}
+
+template <class _Tp, class _Alloc>
+void forward_list<_Tp, _Alloc>::push_front(value_type&& __v) {
+  base::__before_begin()->__next_ = this->__create_node(/* next = */ base::__before_begin()->__next_, std::move(__v));
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Alloc>
+void forward_list<_Tp, _Alloc>::push_front(const value_type& __v) {
+  base::__before_begin()->__next_ = this->__create_node(/* next = */ base::__before_begin()->__next_, __v);
+}
+
+template <class _Tp, class _Alloc>
+void forward_list<_Tp, _Alloc>::pop_front() {
+  __node_pointer __p              = base::__before_begin()->__next_;
+  base::__before_begin()->__next_ = __p->__next_;
+  this->__delete_node(__p);
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Alloc>
+template <class... _Args>
+typename forward_list<_Tp, _Alloc>::iterator
+forward_list<_Tp, _Alloc>::emplace_after(const_iterator __p, _Args&&... __args) {
+  __begin_node_pointer const __r = __p.__get_begin();
+  __r->__next_                   = this->__create_node(/* next = */ __r->__next_, std::forward<_Args>(__args)...);
+  return iterator(__r->__next_);
+}
+
+template <class _Tp, class _Alloc>
+typename forward_list<_Tp, _Alloc>::iterator
+forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, value_type&& __v) {
+  __begin_node_pointer const __r = __p.__get_begin();
+  __r->__next_                   = this->__create_node(/* next = */ __r->__next_, std::move(__v));
+  return iterator(__r->__next_);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Alloc>
+typename forward_list<_Tp, _Alloc>::iterator
+forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, const value_type& __v) {
+  __begin_node_pointer const __r = __p.__get_begin();
+  __r->__next_                   = this->__create_node(/* next = */ __r->__next_, __v);
+  return iterator(__r->__next_);
+}
+
+template <class _Tp, class _Alloc>
+typename forward_list<_Tp, _Alloc>::iterator
+forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, size_type __n, const value_type& __v) {
+  __begin_node_pointer __r = __p.__get_begin();
+  if (__n > 0) {
+    __node_pointer __first = this->__create_node(/* next = */ nullptr, __v);
+    __node_pointer __last  = __first;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      for (--__n; __n != 0; --__n, __last = __last->__next_) {
+        __last->__next_ = this->__create_node(/* next = */ nullptr, __v);
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      while (__first != nullptr) {
+        __node_pointer __next = __first->__next_;
+        this->__delete_node(__first);
+        __first = __next;
+      }
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __last->__next_ = __r->__next_;
+    __r->__next_    = __first;
+    __r             = static_cast<__begin_node_pointer>(__last);
+  }
+  return iterator(__r);
+}
+
+template <class _Tp, class _Alloc>
+template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> >
+typename forward_list<_Tp, _Alloc>::iterator
+forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, _InputIterator __f, _InputIterator __l) {
+  return __insert_after_with_sentinel(__p, std::move(__f), std::move(__l));
+}
+
+template <class _Tp, class _Alloc>
+template <class _InputIterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI typename forward_list<_Tp, _Alloc>::iterator
+forward_list<_Tp, _Alloc>::__insert_after_with_sentinel(const_iterator __p, _InputIterator __f, _Sentinel __l) {
+  __begin_node_pointer __r = __p.__get_begin();
+
+  if (__f != __l) {
+    __node_pointer __first = this->__create_node(/* next = */ nullptr, *__f);
+    __node_pointer __last  = __first;
+
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      for (++__f; __f != __l; ++__f, ((void)(__last = __last->__next_))) {
+        __last->__next_ = this->__create_node(/* next = */ nullptr, *__f);
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      while (__first != nullptr) {
+        __node_pointer __next = __first->__next_;
+        this->__delete_node(__first);
+        __first = __next;
+      }
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+
+    __last->__next_ = __r->__next_;
+    __r->__next_    = __first;
+    __r             = static_cast<__begin_node_pointer>(__last);
+  }
+
+  return iterator(__r);
+}
+
+template <class _Tp, class _Alloc>
+typename forward_list<_Tp, _Alloc>::iterator forward_list<_Tp, _Alloc>::erase_after(const_iterator __f) {
+  __begin_node_pointer __p = __f.__get_begin();
+  __node_pointer __n       = __p->__next_;
+  __p->__next_             = __n->__next_;
+  this->__delete_node(__n);
+  return iterator(__p->__next_);
+}
+
+template <class _Tp, class _Alloc>
+typename forward_list<_Tp, _Alloc>::iterator
+forward_list<_Tp, _Alloc>::erase_after(const_iterator __f, const_iterator __l) {
+  __node_pointer __e = __l.__get_unsafe_node_pointer();
+  if (__f != __l) {
+    __begin_node_pointer __bp = __f.__get_begin();
+
+    __node_pointer __n = __bp->__next_;
+    if (__n != __e) {
+      __bp->__next_ = __e;
+      do {
+        __node_pointer __tmp = __n->__next_;
+        this->__delete_node(__n);
+        __n = __tmp;
+      } while (__n != __e);
+    }
+  }
+  return iterator(__e);
+}
+
+template <class _Tp, class _Alloc>
+void forward_list<_Tp, _Alloc>::resize(size_type __n) {
+  size_type __sz = 0;
+  iterator __p   = before_begin();
+  iterator __i   = begin();
+  iterator __e   = end();
+  for (; __i != __e && __sz < __n; ++__p, ++__i, ++__sz)
+    ;
+  if (__i != __e)
+    erase_after(__p, __e);
+  else {
+    __n -= __sz;
+    if (__n > 0) {
+      for (__begin_node_pointer __ptr = __p.__get_begin(); __n > 0; --__n, __ptr = __ptr->__next_as_begin()) {
+        __ptr->__next_ = this->__create_node(/* next = */ nullptr);
+      }
+    }
+  }
+}
+
+template <class _Tp, class _Alloc>
+void forward_list<_Tp, _Alloc>::resize(size_type __n, const value_type& __v) {
+  size_type __sz = 0;
+  iterator __p   = before_begin();
+  iterator __i   = begin();
+  iterator __e   = end();
+  for (; __i != __e && __sz < __n; ++__p, ++__i, ++__sz)
+    ;
+  if (__i != __e)
+    erase_after(__p, __e);
+  else {
+    __n -= __sz;
+    if (__n > 0) {
+      for (__begin_node_pointer __ptr = __p.__get_begin(); __n > 0; --__n, __ptr = __ptr->__next_as_begin()) {
+        __ptr->__next_ = this->__create_node(/* next = */ nullptr, __v);
+      }
+    }
+  }
+}
+
+template <class _Tp, class _Alloc>
+void forward_list<_Tp, _Alloc>::splice_after(const_iterator __p, forward_list& __x) {
+  if (!__x.empty()) {
+    if (__p.__get_begin()->__next_ != nullptr) {
+      const_iterator __lm1 = __x.before_begin();
+      while (__lm1.__get_begin()->__next_ != nullptr)
+        ++__lm1;
+      __lm1.__get_begin()->__next_ = __p.__get_begin()->__next_;
+    }
+    __p.__get_begin()->__next_    = __x.__before_begin()->__next_;
+    __x.__before_begin()->__next_ = nullptr;
+  }
+}
+
+template <class _Tp, class _Alloc>
+void forward_list<_Tp, _Alloc>::splice_after(const_iterator __p, forward_list& /*__other*/, const_iterator __i) {
+  const_iterator __lm1 = std::next(__i);
+  if (__p != __i && __p != __lm1) {
+    __i.__get_begin()->__next_   = __lm1.__get_begin()->__next_;
+    __lm1.__get_begin()->__next_ = __p.__get_begin()->__next_;
+    __p.__get_begin()->__next_   = __lm1.__get_unsafe_node_pointer();
+  }
+}
+
+template <class _Tp, class _Alloc>
+void forward_list<_Tp, _Alloc>::splice_after(
+    const_iterator __p, forward_list& /*__other*/, const_iterator __f, const_iterator __l) {
+  if (__f != __l && __p != __f) {
+    const_iterator __lm1 = __f;
+    while (__lm1.__get_begin()->__next_ != __l.__get_begin())
+      ++__lm1;
+    if (__f != __lm1) {
+      __lm1.__get_begin()->__next_ = __p.__get_begin()->__next_;
+      __p.__get_begin()->__next_   = __f.__get_begin()->__next_;
+      __f.__get_begin()->__next_   = __l.__get_unsafe_node_pointer();
+    }
+  }
+}
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI void forward_list<_Tp, _Alloc>::splice_after(const_iterator __p, forward_list&& __x) {
+  splice_after(__p, __x);
+}
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI void
+forward_list<_Tp, _Alloc>::splice_after(const_iterator __p, forward_list&& __x, const_iterator __i) {
+  splice_after(__p, __x, __i);
+}
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI void forward_list<_Tp, _Alloc>::splice_after(
+    const_iterator __p, forward_list&& __x, const_iterator __f, const_iterator __l) {
+  splice_after(__p, __x, __f, __l);
+}
+
+template <class _Tp, class _Alloc>
+typename forward_list<_Tp, _Alloc>::__remove_return_type forward_list<_Tp, _Alloc>::remove(const value_type& __v) {
+  forward_list<_Tp, _Alloc> __deleted_nodes(get_allocator()); // collect the nodes we're removing
+  typename forward_list<_Tp, _Alloc>::size_type __count_removed = 0;
+  const iterator __e                                            = end();
+  for (iterator __i = before_begin(); __i.__get_begin()->__next_ != nullptr;) {
+    if (__i.__get_begin()->__next_->__get_value() == __v) {
+      ++__count_removed;
+      iterator __j = std::next(__i, 2);
+      for (; __j != __e && *__j == __v; ++__j)
+        ++__count_removed;
+      __deleted_nodes.splice_after(__deleted_nodes.before_begin(), *this, __i, __j);
+      if (__j == __e)
+        break;
+      __i = __j;
+    } else
+      ++__i;
+  }
+
+  return (__remove_return_type)__count_removed;
+}
+
+template <class _Tp, class _Alloc>
+template <class _Predicate>
+typename forward_list<_Tp, _Alloc>::__remove_return_type forward_list<_Tp, _Alloc>::remove_if(_Predicate __pred) {
+  forward_list<_Tp, _Alloc> __deleted_nodes(get_allocator()); // collect the nodes we're removing
+  typename forward_list<_Tp, _Alloc>::size_type __count_removed = 0;
+  const iterator __e                                            = end();
+  for (iterator __i = before_begin(); __i.__get_begin()->__next_ != nullptr;) {
+    if (__pred(__i.__get_begin()->__next_->__get_value())) {
+      ++__count_removed;
+      iterator __j = std::next(__i, 2);
+      for (; __j != __e && __pred(*__j); ++__j)
+        ++__count_removed;
+      __deleted_nodes.splice_after(__deleted_nodes.before_begin(), *this, __i, __j);
+      if (__j == __e)
+        break;
+      __i = __j;
+    } else
+      ++__i;
+  }
+
+  return (__remove_return_type)__count_removed;
+}
+
+template <class _Tp, class _Alloc>
+template <class _BinaryPredicate>
+typename forward_list<_Tp, _Alloc>::__remove_return_type
+forward_list<_Tp, _Alloc>::unique(_BinaryPredicate __binary_pred) {
+  forward_list<_Tp, _Alloc> __deleted_nodes(get_allocator()); // collect the nodes we're removing
+  typename forward_list<_Tp, _Alloc>::size_type __count_removed = 0;
+  for (iterator __i = begin(), __e = end(); __i != __e;) {
+    iterator __j = std::next(__i);
+    for (; __j != __e && __binary_pred(*__i, *__j); ++__j)
+      ++__count_removed;
+    if (__i.__get_begin()->__next_ != __j.__get_unsafe_node_pointer())
+      __deleted_nodes.splice_after(__deleted_nodes.before_begin(), *this, __i, __j);
+    __i = __j;
+  }
+
+  return (__remove_return_type)__count_removed;
+}
+
+template <class _Tp, class _Alloc>
+template <class _Compare>
+void forward_list<_Tp, _Alloc>::merge(forward_list& __x, _Compare __comp) {
+  if (this != std::addressof(__x)) {
+    base::__before_begin()->__next_ = __merge(base::__before_begin()->__next_, __x.__before_begin()->__next_, __comp);
+    __x.__before_begin()->__next_   = nullptr;
+  }
+}
+
+template <class _Tp, class _Alloc>
+template <class _Compare>
+typename forward_list<_Tp, _Alloc>::__node_pointer
+forward_list<_Tp, _Alloc>::__merge(__node_pointer __f1, __node_pointer __f2, _Compare& __comp) {
+  if (__f1 == nullptr)
+    return __f2;
+  if (__f2 == nullptr)
+    return __f1;
+  __node_pointer __r;
+  if (__comp(__f2->__get_value(), __f1->__get_value())) {
+    __node_pointer __t = __f2;
+    while (__t->__next_ != nullptr && __comp(__t->__next_->__get_value(), __f1->__get_value()))
+      __t = __t->__next_;
+    __r          = __f2;
+    __f2         = __t->__next_;
+    __t->__next_ = __f1;
+  } else
+    __r = __f1;
+  __node_pointer __p = __f1;
+  __f1               = __f1->__next_;
+  while (__f1 != nullptr && __f2 != nullptr) {
+    if (__comp(__f2->__get_value(), __f1->__get_value())) {
+      __node_pointer __t = __f2;
+      while (__t->__next_ != nullptr && __comp(__t->__next_->__get_value(), __f1->__get_value()))
+        __t = __t->__next_;
+      __p->__next_ = __f2;
+      __f2         = __t->__next_;
+      __t->__next_ = __f1;
+    }
+    __p  = __f1;
+    __f1 = __f1->__next_;
+  }
+  if (__f2 != nullptr)
+    __p->__next_ = __f2;
+  return __r;
+}
+
+template <class _Tp, class _Alloc>
+template <class _Compare>
+inline void forward_list<_Tp, _Alloc>::sort(_Compare __comp) {
+  base::__before_begin()->__next_ = __sort(base::__before_begin()->__next_, std::distance(begin(), end()), __comp);
+}
+
+template <class _Tp, class _Alloc>
+template <class _Compare>
+typename forward_list<_Tp, _Alloc>::__node_pointer
+forward_list<_Tp, _Alloc>::__sort(__node_pointer __f1, difference_type __sz, _Compare& __comp) {
+  switch (__sz) {
+  case 0:
+  case 1:
+    return __f1;
+  case 2:
+    if (__comp(__f1->__next_->__get_value(), __f1->__get_value())) {
+      __node_pointer __t = __f1->__next_;
+      __t->__next_       = __f1;
+      __f1->__next_      = nullptr;
+      __f1               = __t;
+    }
+    return __f1;
+  }
+  difference_type __sz1 = __sz / 2;
+  difference_type __sz2 = __sz - __sz1;
+  __node_pointer __t    = std::next(iterator(__f1), __sz1 - 1).__get_unsafe_node_pointer();
+  __node_pointer __f2   = __t->__next_;
+  __t->__next_          = nullptr;
+  return __merge(__sort(__f1, __sz1, __comp), __sort(__f2, __sz2, __comp), __comp);
+}
+
+template <class _Tp, class _Alloc>
+void forward_list<_Tp, _Alloc>::reverse() _NOEXCEPT {
+  __node_pointer __p = base::__before_begin()->__next_;
+  if (__p != nullptr) {
+    __node_pointer __f = __p->__next_;
+    __p->__next_       = nullptr;
+    while (__f != nullptr) {
+      __node_pointer __t = __f->__next_;
+      __f->__next_       = __p;
+      __p                = __f;
+      __f                = __t;
+    }
+    base::__before_begin()->__next_ = __p;
+  }
+}
+
+template <class _Tp, class _Alloc>
+_LIBCPP_HIDE_FROM_ABI bool operator==(const forward_list<_Tp, _Alloc>& __x, const forward_list<_Tp, _Alloc>& __y) {
+  typedef forward_list<_Tp, _Alloc> _Cp;
+  typedef typename _Cp::const_iterator _Ip;
+  _Ip __ix = __x.begin();
+  _Ip __ex = __x.end();
+  _Ip __iy = __y.begin();
+  _Ip __ey = __y.end();
+  for (; __ix != __ex && __iy != __ey; ++__ix, ++__iy)
+    if (!(*__ix == *__iy))
+      return false;
+  return (__ix == __ex) == (__iy == __ey);
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const forward_list<_Tp, _Alloc>& __x, const forward_list<_Tp, _Alloc>& __y) {
+  return !(__x == __y);
+}
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<(const forward_list<_Tp, _Alloc>& __x, const forward_list<_Tp, _Alloc>& __y) {
+  return std::lexicographical_compare(__x.begin(), __x.end(), __y.begin(), __y.end());
+}
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>(const forward_list<_Tp, _Alloc>& __x, const forward_list<_Tp, _Alloc>& __y) {
+  return __y < __x;
+}
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>=(const forward_list<_Tp, _Alloc>& __x, const forward_list<_Tp, _Alloc>& __y) {
+  return !(__x < __y);
+}
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<=(const forward_list<_Tp, _Alloc>& __x, const forward_list<_Tp, _Alloc>& __y) {
+  return !(__y < __x);
+}
+
+#else // #if _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI __synth_three_way_result<_Tp>
+operator<=>(const forward_list<_Tp, _Allocator>& __x, const forward_list<_Tp, _Allocator>& __y) {
+  return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
+}
+
+#endif // #if _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI void swap(forward_list<_Tp, _Alloc>& __x, forward_list<_Tp, _Alloc>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp, class _Allocator, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI typename forward_list<_Tp, _Allocator>::size_type
+erase_if(forward_list<_Tp, _Allocator>& __c, _Predicate __pred) {
+  return __c.remove_if(__pred);
+}
+
+template <class _Tp, class _Allocator, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI typename forward_list<_Tp, _Allocator>::size_type
+erase(forward_list<_Tp, _Allocator>& __c, const _Up& __v) {
+  return std::erase_if(__c, [&](auto& __elem) { return __elem == __v; });
+}
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace pmr {
+template <class _ValueT>
+using forward_list _LIBCPP_AVAILABILITY_PMR = std::forward_list<_ValueT, polymorphic_allocator<_ValueT>>;
+} // namespace pmr
+_LIBCPP_END_NAMESPACE_STD
+#endif
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <algorithm>
+#  include <atomic>
+#  include <concepts>
+#  include <cstdint>
+#  include <cstdlib>
+#  include <cstring>
+#  include <functional>
+#  include <iosfwd>
+#  include <iterator>
+#  include <stdexcept>
+#  include <type_traits>
+#  include <typeinfo>
+#endif
+
+#endif // _LIBCPP_FORWARD_LIST
diff --git a/libcxx/include/__cxx03/fstream b/libcxx/include/__cxx03/fstream
new file mode 100644
index 000000000000..ab5ebf8e2c3d
--- /dev/null
+++ b/libcxx/include/__cxx03/fstream
@@ -0,0 +1,1572 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_FSTREAM
+#define _LIBCPP_FSTREAM
+
+/*
+    fstream synopsis
+
+template <class charT, class traits = char_traits<charT> >
+class basic_filebuf
+    : public basic_streambuf<charT, traits>
+{
+public:
+    typedef charT                          char_type;
+    typedef traits                         traits_type;
+    typedef typename traits_type::int_type int_type;
+    typedef typename traits_type::pos_type pos_type;
+    typedef typename traits_type::off_type off_type;
+
+    // 27.9.1.2 Constructors/destructor:
+    basic_filebuf();
+    basic_filebuf(basic_filebuf&& rhs);
+    virtual ~basic_filebuf();
+
+    // 27.9.1.3 Assign/swap:
+    basic_filebuf& operator=(basic_filebuf&& rhs);
+    void swap(basic_filebuf& rhs);
+
+    // 27.9.1.4 Members:
+    bool is_open() const;
+    basic_filebuf* open(const char* s, ios_base::openmode mode);
+    basic_filebuf* open(const string& s, ios_base::openmode mode);
+    basic_filebuf* open(const filesystem::path& p, ios_base::openmode mode); // C++17
+    basic_filebuf* close();
+
+protected:
+    // 27.9.1.5 Overridden virtual functions:
+    virtual streamsize showmanyc();
+    virtual int_type underflow();
+    virtual int_type uflow();
+    virtual int_type pbackfail(int_type c = traits_type::eof());
+    virtual int_type overflow (int_type c = traits_type::eof());
+    virtual basic_streambuf<char_type, traits_type>* setbuf(char_type* s, streamsize n);
+    virtual pos_type seekoff(off_type off, ios_base::seekdir way,
+                             ios_base::openmode which = ios_base::in | ios_base::out);
+    virtual pos_type seekpos(pos_type sp,
+                             ios_base::openmode which = ios_base::in | ios_base::out);
+    virtual int sync();
+    virtual void imbue(const locale& loc);
+};
+
+template <class charT, class traits>
+  void
+  swap(basic_filebuf<charT, traits>& x, basic_filebuf<charT, traits>& y);
+
+typedef basic_filebuf<char>    filebuf;
+typedef basic_filebuf<wchar_t> wfilebuf;
+
+template <class charT, class traits = char_traits<charT> >
+class basic_ifstream
+    : public basic_istream<charT,traits>
+{
+public:
+    typedef charT                          char_type;
+    typedef traits                         traits_type;
+    typedef typename traits_type::int_type int_type;
+    typedef typename traits_type::pos_type pos_type;
+    typedef typename traits_type::off_type off_type;
+    using native_handle_type = typename basic_filebuf<charT, traits>::native_handle_type; // Since C++26
+
+    basic_ifstream();
+    explicit basic_ifstream(const char* s, ios_base::openmode mode = ios_base::in);
+    explicit basic_ifstream(const string& s, ios_base::openmode mode = ios_base::in);
+    template<class T>
+    explicit basic_ifstream(const T& s, ios_base::openmode mode = ios_base::in); // Since C++17
+    basic_ifstream(basic_ifstream&& rhs);
+
+    basic_ifstream& operator=(basic_ifstream&& rhs);
+    void swap(basic_ifstream& rhs);
+
+    basic_filebuf<char_type, traits_type>* rdbuf() const;
+    native_handle_type native_handle() const noexcept; // Since C++26
+    bool is_open() const;
+    void open(const char* s, ios_base::openmode mode = ios_base::in);
+    void open(const string& s, ios_base::openmode mode = ios_base::in);
+    void open(const filesystem::path& s, ios_base::openmode mode = ios_base::in); // C++17
+
+    void close();
+};
+
+template <class charT, class traits>
+  void
+  swap(basic_ifstream<charT, traits>& x, basic_ifstream<charT, traits>& y);
+
+typedef basic_ifstream<char>    ifstream;
+typedef basic_ifstream<wchar_t> wifstream;
+
+template <class charT, class traits = char_traits<charT> >
+class basic_ofstream
+    : public basic_ostream<charT,traits>
+{
+public:
+    typedef charT                          char_type;
+    typedef traits                         traits_type;
+    typedef typename traits_type::int_type int_type;
+    typedef typename traits_type::pos_type pos_type;
+    typedef typename traits_type::off_type off_type;
+    using native_handle_type = typename basic_filebuf<charT, traits>::native_handle_type; // Since C++26
+
+    basic_ofstream();
+    explicit basic_ofstream(const char* s, ios_base::openmode mode = ios_base::out);
+    explicit basic_ofstream(const string& s, ios_base::openmode mode = ios_base::out);
+    template<class T>
+    explicit basic_ofstream(const T& s, ios_base::openmode mode = ios_base::out); // Since C++17
+    basic_ofstream(basic_ofstream&& rhs);
+
+    basic_ofstream& operator=(basic_ofstream&& rhs);
+    void swap(basic_ofstream& rhs);
+
+    basic_filebuf<char_type, traits_type>* rdbuf() const;
+    native_handle_type native_handle() const noexcept; // Since C++26
+
+    bool is_open() const;
+    void open(const char* s, ios_base::openmode mode = ios_base::out);
+    void open(const string& s, ios_base::openmode mode = ios_base::out);
+    void open(const filesystem::path& p,
+              ios_base::openmode mode = ios_base::out); // C++17
+
+    void close();
+};
+
+template <class charT, class traits>
+  void
+  swap(basic_ofstream<charT, traits>& x, basic_ofstream<charT, traits>& y);
+
+typedef basic_ofstream<char>    ofstream;
+typedef basic_ofstream<wchar_t> wofstream;
+
+template <class charT, class traits=char_traits<charT> >
+class basic_fstream
+    : public basic_iostream<charT,traits>
+{
+public:
+    typedef charT                          char_type;
+    typedef traits                         traits_type;
+    typedef typename traits_type::int_type int_type;
+    typedef typename traits_type::pos_type pos_type;
+    typedef typename traits_type::off_type off_type;
+    using native_handle_type = typename basic_filebuf<charT, traits>::native_handle_type; // Since C++26
+
+    basic_fstream();
+    explicit basic_fstream(const char* s, ios_base::openmode mode = ios_base::in|ios_base::out);
+    explicit basic_fstream(const string& s, ios_base::openmode mode = ios_base::in|ios_base::out);
+    template<class T>
+    explicit basic_fstream(const T& s, ios_base::openmode mode = ios_base::in | ios_base::out); // Since C++17
+    basic_fstream(basic_fstream&& rhs);
+
+    basic_fstream& operator=(basic_fstream&& rhs);
+    void swap(basic_fstream& rhs);
+
+    basic_filebuf<char_type, traits_type>* rdbuf() const;
+    native_handle_type native_handle() const noexcept; // Since C++26
+    bool is_open() const;
+    void open(const char* s, ios_base::openmode mode = ios_base::in|ios_base::out);
+    void open(const string& s, ios_base::openmode mode = ios_base::in|ios_base::out);
+    void open(const filesystem::path& s,
+              ios_base::openmode mode = ios_base::in|ios_base::out); // C++17
+
+    void close();
+};
+
+template <class charT, class traits>
+  void swap(basic_fstream<charT, traits>& x, basic_fstream<charT, traits>& y);
+
+typedef basic_fstream<char>    fstream;
+typedef basic_fstream<wchar_t> wfstream;
+
+}  // std
+
+*/
+
+#include <__algorithm/max.h>
+#include <__assert>
+#include <__config>
+#include <__fwd/fstream.h>
+#include <__locale>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_same.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+#include <__utility/unreachable.h>
+#include <cstdio>
+#include <filesystem>
+#include <istream>
+#include <ostream>
+#include <typeinfo>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if defined(_LIBCPP_MSVCRT) || defined(_NEWLIB_VERSION)
+#  define _LIBCPP_HAS_NO_OFF_T_FUNCTIONS
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_FILESYSTEM)
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_WIN32API)
+_LIBCPP_EXPORTED_FROM_ABI void* __filebuf_windows_native_handle(FILE* __file) noexcept;
+#  endif
+
+template <class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS basic_filebuf : public basic_streambuf<_CharT, _Traits> {
+public:
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+  typedef typename traits_type::state_type state_type;
+#  if _LIBCPP_STD_VER >= 26
+#    if defined(_LIBCPP_WIN32API)
+  using native_handle_type = void*; // HANDLE
+#    elif __has_include(<unistd.h>)
+  using native_handle_type = int; // POSIX file descriptor
+#    else
+#      error "Provide a native file handle!"
+#    endif
+#  endif
+
+  // 27.9.1.2 Constructors/destructor:
+  basic_filebuf();
+  basic_filebuf(basic_filebuf&& __rhs);
+  ~basic_filebuf() override;
+
+  // 27.9.1.3 Assign/swap:
+  _LIBCPP_HIDE_FROM_ABI basic_filebuf& operator=(basic_filebuf&& __rhs);
+  void swap(basic_filebuf& __rhs);
+
+  // 27.9.1.4 Members:
+  _LIBCPP_HIDE_FROM_ABI bool is_open() const;
+  basic_filebuf* open(const char* __s, ios_base::openmode __mode);
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+  basic_filebuf* open(const wchar_t* __s, ios_base::openmode __mode);
+#  endif
+  _LIBCPP_HIDE_FROM_ABI basic_filebuf* open(const string& __s, ios_base::openmode __mode);
+
+#  if _LIBCPP_STD_VER >= 17
+  _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_HIDE_FROM_ABI basic_filebuf*
+  open(const filesystem::path& __p, ios_base::openmode __mode) {
+    return open(__p.c_str(), __mode);
+  }
+#  endif
+  _LIBCPP_HIDE_FROM_ABI basic_filebuf* __open(int __fd, ios_base::openmode __mode);
+  basic_filebuf* close();
+#  if _LIBCPP_STD_VER >= 26
+  _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept {
+    _LIBCPP_ASSERT_UNCATEGORIZED(this->is_open(), "File must be opened");
+#    if defined(_LIBCPP_WIN32API)
+    return std::__filebuf_windows_native_handle(__file_);
+#    elif __has_include(<unistd.h>)
+    return fileno(__file_);
+#    else
+#      error "Provide a way to determine the file native handle!"
+#    endif
+  }
+#  endif //  _LIBCPP_STD_VER >= 26
+
+  _LIBCPP_HIDE_FROM_ABI inline static const char* __make_mdstring(ios_base::openmode __mode) _NOEXCEPT;
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+  _LIBCPP_HIDE_FROM_ABI inline static const wchar_t* __make_mdwstring(ios_base::openmode __mode) _NOEXCEPT;
+#  endif
+
+protected:
+  // 27.9.1.5 Overridden virtual functions:
+  int_type underflow() override;
+  int_type pbackfail(int_type __c = traits_type::eof()) override;
+  int_type overflow(int_type __c = traits_type::eof()) override;
+  basic_streambuf<char_type, traits_type>* setbuf(char_type* __s, streamsize __n) override;
+  pos_type
+  seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __wch = ios_base::in | ios_base::out) override;
+  pos_type seekpos(pos_type __sp, ios_base::openmode __wch = ios_base::in | ios_base::out) override;
+  int sync() override;
+  void imbue(const locale& __loc) override;
+
+private:
+  char* __extbuf_;
+  const char* __extbufnext_;
+  const char* __extbufend_;
+  char __extbuf_min_[8];
+  size_t __ebs_;
+  char_type* __intbuf_;
+  size_t __ibs_;
+  FILE* __file_;
+  const codecvt<char_type, char, state_type>* __cv_;
+  state_type __st_;
+  state_type __st_last_;
+  ios_base::openmode __om_;
+  // There have been no file operations yet, which allows setting unbuffered
+  // I/O mode.
+  static const ios_base::openmode __no_io_operations = ios_base::trunc;
+  // Unbuffered I/O mode has been requested.
+  static const ios_base::openmode __use_unbuffered_io = ios_base::ate;
+  // Used to track the currently used mode and track whether the output should
+  // be unbuffered.
+  // [filebuf.virtuals]/12
+  //   If setbuf(0, 0) is called on a stream before any I/O has occurred on
+  //   that stream, the stream becomes unbuffered. Otherwise the results are
+  //   implementation-defined.
+  // This allows calling setbuf(0, 0)
+  // - before opening a file,
+  // - after opening a file, before
+  //   - a read
+  //   - a write
+  //   - a seek.
+  // Note that opening a file with ios_base::ate does a seek operation.
+  // Normally underflow, overflow, and sync change this flag to ios_base::in,
+  // ios_base_out, or 0.
+  //
+  // The ios_base::trunc and ios_base::ate flags are not used in __cm_. They
+  // are used to track the state of the unbuffered request. For readability
+  // they have the aliases __no_io_operations and __use_unbuffered_io
+  // respectively.
+  //
+  // The __no_io_operations and __use_unbuffered_io flags are used in the
+  // following way:
+  // - __no_io_operations is set upon construction to indicate the unbuffered
+  //   state can be set.
+  // - When requesting unbuffered output:
+  //   - If the file is open it sets the mode.
+  //   - Else places a request by adding the __use_unbuffered_io flag.
+  // - When a file is opened it checks whether both __no_io_operations and
+  //   __use_unbuffered_io are set. If so switches to unbuffered mode.
+  // - All file I/O operations change the mode effectively clearing the
+  //   __no_io_operations and __use_unbuffered_io flags.
+  ios_base::openmode __cm_;
+  bool __owns_eb_;
+  bool __owns_ib_;
+  bool __always_noconv_;
+
+  bool __read_mode();
+  void __write_mode();
+
+  _LIBCPP_EXPORTED_FROM_ABI friend FILE* __get_ostream_file(ostream&);
+
+  // There are multiple (__)open function, they use different C-API open
+  // function. After that call these functions behave the same. This function
+  // does that part and determines the final return value.
+  _LIBCPP_HIDE_FROM_ABI basic_filebuf* __do_open(FILE* __file, ios_base::openmode __mode) {
+    __file_ = __file;
+    if (!__file_)
+      return nullptr;
+
+    __om_ = __mode;
+    if (__cm_ == (__no_io_operations | __use_unbuffered_io)) {
+      std::setbuf(__file_, nullptr);
+      __cm_ = 0;
+    }
+
+    if (__mode & ios_base::ate) {
+      __cm_ = 0;
+      if (fseek(__file_, 0, SEEK_END)) {
+        fclose(__file_);
+        __file_ = nullptr;
+        return nullptr;
+      }
+    }
+
+    return this;
+  }
+
+  // If the file is already open, switch to unbuffered mode. Otherwise, record
+  // the request to use unbuffered mode so that we use that mode when we
+  // eventually open the file.
+  _LIBCPP_HIDE_FROM_ABI void __request_unbuffered_mode(char_type* __s, streamsize __n) {
+    if (__cm_ == __no_io_operations && __s == nullptr && __n == 0) {
+      if (__file_) {
+        std::setbuf(__file_, nullptr);
+        __cm_ = 0;
+      } else {
+        __cm_ = __no_io_operations | __use_unbuffered_io;
+      }
+    }
+  }
+};
+
+template <class _CharT, class _Traits>
+basic_filebuf<_CharT, _Traits>::basic_filebuf()
+    : __extbuf_(nullptr),
+      __extbufnext_(nullptr),
+      __extbufend_(nullptr),
+      __ebs_(0),
+      __intbuf_(nullptr),
+      __ibs_(0),
+      __file_(nullptr),
+      __cv_(nullptr),
+      __st_(),
+      __st_last_(),
+      __om_(0),
+      __cm_(__no_io_operations),
+      __owns_eb_(false),
+      __owns_ib_(false),
+      __always_noconv_(false) {
+  if (std::has_facet<codecvt<char_type, char, state_type> >(this->getloc())) {
+    __cv_            = &std::use_facet<codecvt<char_type, char, state_type> >(this->getloc());
+    __always_noconv_ = __cv_->always_noconv();
+  }
+  setbuf(nullptr, 4096);
+}
+
+template <class _CharT, class _Traits>
+basic_filebuf<_CharT, _Traits>::basic_filebuf(basic_filebuf&& __rhs) : basic_streambuf<_CharT, _Traits>(__rhs) {
+  if (__rhs.__extbuf_ == __rhs.__extbuf_min_) {
+    __extbuf_     = __extbuf_min_;
+    __extbufnext_ = __extbuf_ + (__rhs.__extbufnext_ - __rhs.__extbuf_);
+    __extbufend_  = __extbuf_ + (__rhs.__extbufend_ - __rhs.__extbuf_);
+  } else {
+    __extbuf_     = __rhs.__extbuf_;
+    __extbufnext_ = __rhs.__extbufnext_;
+    __extbufend_  = __rhs.__extbufend_;
+  }
+  __ebs_           = __rhs.__ebs_;
+  __intbuf_        = __rhs.__intbuf_;
+  __ibs_           = __rhs.__ibs_;
+  __file_          = __rhs.__file_;
+  __cv_            = __rhs.__cv_;
+  __st_            = __rhs.__st_;
+  __st_last_       = __rhs.__st_last_;
+  __om_            = __rhs.__om_;
+  __cm_            = __rhs.__cm_;
+  __owns_eb_       = __rhs.__owns_eb_;
+  __owns_ib_       = __rhs.__owns_ib_;
+  __always_noconv_ = __rhs.__always_noconv_;
+  if (__rhs.pbase()) {
+    if (__rhs.pbase() == __rhs.__intbuf_)
+      this->setp(__intbuf_, __intbuf_ + (__rhs.epptr() - __rhs.pbase()));
+    else
+      this->setp((char_type*)__extbuf_, (char_type*)__extbuf_ + (__rhs.epptr() - __rhs.pbase()));
+    this->__pbump(__rhs.pptr() - __rhs.pbase());
+  } else if (__rhs.eback()) {
+    if (__rhs.eback() == __rhs.__intbuf_)
+      this->setg(__intbuf_, __intbuf_ + (__rhs.gptr() - __rhs.eback()), __intbuf_ + (__rhs.egptr() - __rhs.eback()));
+    else
+      this->setg((char_type*)__extbuf_,
+                 (char_type*)__extbuf_ + (__rhs.gptr() - __rhs.eback()),
+                 (char_type*)__extbuf_ + (__rhs.egptr() - __rhs.eback()));
+  }
+  __rhs.__extbuf_     = nullptr;
+  __rhs.__extbufnext_ = nullptr;
+  __rhs.__extbufend_  = nullptr;
+  __rhs.__ebs_        = 0;
+  __rhs.__intbuf_     = 0;
+  __rhs.__ibs_        = 0;
+  __rhs.__file_       = nullptr;
+  __rhs.__st_         = state_type();
+  __rhs.__st_last_    = state_type();
+  __rhs.__om_         = 0;
+  __rhs.__cm_         = 0;
+  __rhs.__owns_eb_    = false;
+  __rhs.__owns_ib_    = false;
+  __rhs.setg(0, 0, 0);
+  __rhs.setp(0, 0);
+}
+
+template <class _CharT, class _Traits>
+inline basic_filebuf<_CharT, _Traits>& basic_filebuf<_CharT, _Traits>::operator=(basic_filebuf&& __rhs) {
+  close();
+  swap(__rhs);
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_filebuf<_CharT, _Traits>::~basic_filebuf() {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    close();
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+  }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  if (__owns_eb_)
+    delete[] __extbuf_;
+  if (__owns_ib_)
+    delete[] __intbuf_;
+}
+
+template <class _CharT, class _Traits>
+void basic_filebuf<_CharT, _Traits>::swap(basic_filebuf& __rhs) {
+  basic_streambuf<char_type, traits_type>::swap(__rhs);
+  if (__extbuf_ != __extbuf_min_ && __rhs.__extbuf_ != __rhs.__extbuf_min_) {
+    // Neither *this nor __rhs uses the small buffer, so we can simply swap the pointers.
+    std::swap(__extbuf_, __rhs.__extbuf_);
+    std::swap(__extbufnext_, __rhs.__extbufnext_);
+    std::swap(__extbufend_, __rhs.__extbufend_);
+  } else {
+    ptrdiff_t __ln = __extbufnext_ ? __extbufnext_ - __extbuf_ : 0;
+    ptrdiff_t __le = __extbufend_ ? __extbufend_ - __extbuf_ : 0;
+    ptrdiff_t __rn = __rhs.__extbufnext_ ? __rhs.__extbufnext_ - __rhs.__extbuf_ : 0;
+    ptrdiff_t __re = __rhs.__extbufend_ ? __rhs.__extbufend_ - __rhs.__extbuf_ : 0;
+    if (__extbuf_ == __extbuf_min_ && __rhs.__extbuf_ != __rhs.__extbuf_min_) {
+      // *this uses the small buffer, but __rhs doesn't.
+      __extbuf_       = __rhs.__extbuf_;
+      __rhs.__extbuf_ = __rhs.__extbuf_min_;
+      std::memmove(__rhs.__extbuf_min_, __extbuf_min_, sizeof(__extbuf_min_));
+    } else if (__extbuf_ != __extbuf_min_ && __rhs.__extbuf_ == __rhs.__extbuf_min_) {
+      // *this doesn't use the small buffer, but __rhs does.
+      __rhs.__extbuf_ = __extbuf_;
+      __extbuf_       = __extbuf_min_;
+      std::memmove(__extbuf_min_, __rhs.__extbuf_min_, sizeof(__extbuf_min_));
+    } else {
+      // Both *this and __rhs use the small buffer.
+      char __tmp[sizeof(__extbuf_min_)];
+      std::memmove(__tmp, __extbuf_min_, sizeof(__extbuf_min_));
+      std::memmove(__extbuf_min_, __rhs.__extbuf_min_, sizeof(__extbuf_min_));
+      std::memmove(__rhs.__extbuf_min_, __tmp, sizeof(__extbuf_min_));
+    }
+    __extbufnext_       = __extbuf_ + __rn;
+    __extbufend_        = __extbuf_ + __re;
+    __rhs.__extbufnext_ = __rhs.__extbuf_ + __ln;
+    __rhs.__extbufend_  = __rhs.__extbuf_ + __le;
+  }
+  std::swap(__ebs_, __rhs.__ebs_);
+  std::swap(__intbuf_, __rhs.__intbuf_);
+  std::swap(__ibs_, __rhs.__ibs_);
+  std::swap(__file_, __rhs.__file_);
+  std::swap(__cv_, __rhs.__cv_);
+  std::swap(__st_, __rhs.__st_);
+  std::swap(__st_last_, __rhs.__st_last_);
+  std::swap(__om_, __rhs.__om_);
+  std::swap(__cm_, __rhs.__cm_);
+  std::swap(__owns_eb_, __rhs.__owns_eb_);
+  std::swap(__owns_ib_, __rhs.__owns_ib_);
+  std::swap(__always_noconv_, __rhs.__always_noconv_);
+  if (this->eback() == (char_type*)__rhs.__extbuf_min_) {
+    ptrdiff_t __n = this->gptr() - this->eback();
+    ptrdiff_t __e = this->egptr() - this->eback();
+    this->setg((char_type*)__extbuf_min_, (char_type*)__extbuf_min_ + __n, (char_type*)__extbuf_min_ + __e);
+  } else if (this->pbase() == (char_type*)__rhs.__extbuf_min_) {
+    ptrdiff_t __n = this->pptr() - this->pbase();
+    ptrdiff_t __e = this->epptr() - this->pbase();
+    this->setp((char_type*)__extbuf_min_, (char_type*)__extbuf_min_ + __e);
+    this->__pbump(__n);
+  }
+  if (__rhs.eback() == (char_type*)__extbuf_min_) {
+    ptrdiff_t __n = __rhs.gptr() - __rhs.eback();
+    ptrdiff_t __e = __rhs.egptr() - __rhs.eback();
+    __rhs.setg(
+        (char_type*)__rhs.__extbuf_min_, (char_type*)__rhs.__extbuf_min_ + __n, (char_type*)__rhs.__extbuf_min_ + __e);
+  } else if (__rhs.pbase() == (char_type*)__extbuf_min_) {
+    ptrdiff_t __n = __rhs.pptr() - __rhs.pbase();
+    ptrdiff_t __e = __rhs.epptr() - __rhs.pbase();
+    __rhs.setp((char_type*)__rhs.__extbuf_min_, (char_type*)__rhs.__extbuf_min_ + __e);
+    __rhs.__pbump(__n);
+  }
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI void swap(basic_filebuf<_CharT, _Traits>& __x, basic_filebuf<_CharT, _Traits>& __y) {
+  __x.swap(__y);
+}
+
+template <class _CharT, class _Traits>
+inline bool basic_filebuf<_CharT, _Traits>::is_open() const {
+  return __file_ != nullptr;
+}
+
+template <class _CharT, class _Traits>
+const char* basic_filebuf<_CharT, _Traits>::__make_mdstring(ios_base::openmode __mode) _NOEXCEPT {
+  switch (__mode & ~ios_base::ate) {
+  case ios_base::out:
+  case ios_base::out | ios_base::trunc:
+    return "w" _LIBCPP_FOPEN_CLOEXEC_MODE;
+  case ios_base::out | ios_base::app:
+  case ios_base::app:
+    return "a" _LIBCPP_FOPEN_CLOEXEC_MODE;
+  case ios_base::in:
+    return "r" _LIBCPP_FOPEN_CLOEXEC_MODE;
+  case ios_base::in | ios_base::out:
+    return "r+" _LIBCPP_FOPEN_CLOEXEC_MODE;
+  case ios_base::in | ios_base::out | ios_base::trunc:
+    return "w+" _LIBCPP_FOPEN_CLOEXEC_MODE;
+  case ios_base::in | ios_base::out | ios_base::app:
+  case ios_base::in | ios_base::app:
+    return "a+" _LIBCPP_FOPEN_CLOEXEC_MODE;
+  case ios_base::out | ios_base::binary:
+  case ios_base::out | ios_base::trunc | ios_base::binary:
+    return "wb" _LIBCPP_FOPEN_CLOEXEC_MODE;
+  case ios_base::out | ios_base::app | ios_base::binary:
+  case ios_base::app | ios_base::binary:
+    return "ab" _LIBCPP_FOPEN_CLOEXEC_MODE;
+  case ios_base::in | ios_base::binary:
+    return "rb" _LIBCPP_FOPEN_CLOEXEC_MODE;
+  case ios_base::in | ios_base::out | ios_base::binary:
+    return "r+b" _LIBCPP_FOPEN_CLOEXEC_MODE;
+  case ios_base::in | ios_base::out | ios_base::trunc | ios_base::binary:
+    return "w+b" _LIBCPP_FOPEN_CLOEXEC_MODE;
+  case ios_base::in | ios_base::out | ios_base::app | ios_base::binary:
+  case ios_base::in | ios_base::app | ios_base::binary:
+    return "a+b" _LIBCPP_FOPEN_CLOEXEC_MODE;
+#  if _LIBCPP_STD_VER >= 23
+  case ios_base::out | ios_base::noreplace:
+  case ios_base::out | ios_base::trunc | ios_base::noreplace:
+    return "wx" _LIBCPP_FOPEN_CLOEXEC_MODE;
+  case ios_base::in | ios_base::out | ios_base::trunc | ios_base::noreplace:
+    return "w+x" _LIBCPP_FOPEN_CLOEXEC_MODE;
+  case ios_base::out | ios_base::binary | ios_base::noreplace:
+  case ios_base::out | ios_base::trunc | ios_base::binary | ios_base::noreplace:
+    return "wbx" _LIBCPP_FOPEN_CLOEXEC_MODE;
+  case ios_base::in | ios_base::out | ios_base::trunc | ios_base::binary | ios_base::noreplace:
+    return "w+bx" _LIBCPP_FOPEN_CLOEXEC_MODE;
+#  endif // _LIBCPP_STD_VER >= 23
+  default:
+    return nullptr;
+  }
+  __libcpp_unreachable();
+}
+
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+template <class _CharT, class _Traits>
+const wchar_t* basic_filebuf<_CharT, _Traits>::__make_mdwstring(ios_base::openmode __mode) _NOEXCEPT {
+  switch (__mode & ~ios_base::ate) {
+  case ios_base::out:
+  case ios_base::out | ios_base::trunc:
+    return L"w";
+  case ios_base::out | ios_base::app:
+  case ios_base::app:
+    return L"a";
+  case ios_base::in:
+    return L"r";
+  case ios_base::in | ios_base::out:
+    return L"r+";
+  case ios_base::in | ios_base::out | ios_base::trunc:
+    return L"w+";
+  case ios_base::in | ios_base::out | ios_base::app:
+  case ios_base::in | ios_base::app:
+    return L"a+";
+  case ios_base::out | ios_base::binary:
+  case ios_base::out | ios_base::trunc | ios_base::binary:
+    return L"wb";
+  case ios_base::out | ios_base::app | ios_base::binary:
+  case ios_base::app | ios_base::binary:
+    return L"ab";
+  case ios_base::in | ios_base::binary:
+    return L"rb";
+  case ios_base::in | ios_base::out | ios_base::binary:
+    return L"r+b";
+  case ios_base::in | ios_base::out | ios_base::trunc | ios_base::binary:
+    return L"w+b";
+  case ios_base::in | ios_base::out | ios_base::app | ios_base::binary:
+  case ios_base::in | ios_base::app | ios_base::binary:
+    return L"a+b";
+#    if _LIBCPP_STD_VER >= 23
+  case ios_base::out | ios_base::noreplace:
+  case ios_base::out | ios_base::trunc | ios_base::noreplace:
+    return L"wx";
+  case ios_base::in | ios_base::out | ios_base::trunc | ios_base::noreplace:
+    return L"w+x";
+  case ios_base::out | ios_base::binary | ios_base::noreplace:
+  case ios_base::out | ios_base::trunc | ios_base::binary | ios_base::noreplace:
+    return L"wbx";
+  case ios_base::in | ios_base::out | ios_base::trunc | ios_base::binary | ios_base::noreplace:
+    return L"w+bx";
+#    endif // _LIBCPP_STD_VER >= 23
+  default:
+    return nullptr;
+  }
+  __libcpp_unreachable();
+}
+#  endif
+
+template <class _CharT, class _Traits>
+basic_filebuf<_CharT, _Traits>* basic_filebuf<_CharT, _Traits>::open(const char* __s, ios_base::openmode __mode) {
+  if (__file_)
+    return nullptr;
+  const char* __mdstr = __make_mdstring(__mode);
+  if (!__mdstr)
+    return nullptr;
+
+  return __do_open(fopen(__s, __mdstr), __mode);
+}
+
+template <class _CharT, class _Traits>
+inline basic_filebuf<_CharT, _Traits>* basic_filebuf<_CharT, _Traits>::__open(int __fd, ios_base::openmode __mode) {
+  if (__file_)
+    return nullptr;
+  const char* __mdstr = __make_mdstring(__mode);
+  if (!__mdstr)
+    return nullptr;
+
+  return __do_open(fdopen(__fd, __mdstr), __mode);
+}
+
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+// This is basically the same as the char* overload except that it uses _wfopen
+// and long mode strings.
+template <class _CharT, class _Traits>
+basic_filebuf<_CharT, _Traits>* basic_filebuf<_CharT, _Traits>::open(const wchar_t* __s, ios_base::openmode __mode) {
+  if (__file_)
+    return nullptr;
+  const wchar_t* __mdstr = __make_mdwstring(__mode);
+  if (!__mdstr)
+    return nullptr;
+
+  return __do_open(_wfopen(__s, __mdstr), __mode);
+}
+#  endif
+
+template <class _CharT, class _Traits>
+inline basic_filebuf<_CharT, _Traits>*
+basic_filebuf<_CharT, _Traits>::open(const string& __s, ios_base::openmode __mode) {
+  return open(__s.c_str(), __mode);
+}
+
+template <class _CharT, class _Traits>
+basic_filebuf<_CharT, _Traits>* basic_filebuf<_CharT, _Traits>::close() {
+  basic_filebuf<_CharT, _Traits>* __rt = nullptr;
+  if (__file_) {
+    __rt = this;
+    unique_ptr<FILE, int (*)(FILE*)> __h(__file_, fclose);
+    if (sync())
+      __rt = nullptr;
+    if (fclose(__h.release()))
+      __rt = nullptr;
+    __file_ = nullptr;
+    setbuf(0, 0);
+  }
+  return __rt;
+}
+
+template <class _CharT, class _Traits>
+typename basic_filebuf<_CharT, _Traits>::int_type basic_filebuf<_CharT, _Traits>::underflow() {
+  if (__file_ == nullptr)
+    return traits_type::eof();
+  bool __initial = __read_mode();
+  char_type __1buf;
+  if (this->gptr() == nullptr)
+    this->setg(&__1buf, &__1buf + 1, &__1buf + 1);
+  const size_t __unget_sz = __initial ? 0 : std::min<size_t>((this->egptr() - this->eback()) / 2, 4);
+  int_type __c            = traits_type::eof();
+  if (this->gptr() == this->egptr()) {
+    std::memmove(this->eback(), this->egptr() - __unget_sz, __unget_sz * sizeof(char_type));
+    if (__always_noconv_) {
+      size_t __nmemb = static_cast<size_t>(this->egptr() - this->eback() - __unget_sz);
+      __nmemb        = ::fread(this->eback() + __unget_sz, 1, __nmemb, __file_);
+      if (__nmemb != 0) {
+        this->setg(this->eback(), this->eback() + __unget_sz, this->eback() + __unget_sz + __nmemb);
+        __c = traits_type::to_int_type(*this->gptr());
+      }
+    } else {
+      if (__extbufend_ != __extbufnext_) {
+        _LIBCPP_ASSERT_NON_NULL(__extbufnext_ != nullptr, "underflow moving from nullptr");
+        _LIBCPP_ASSERT_NON_NULL(__extbuf_ != nullptr, "underflow moving into nullptr");
+        std::memmove(__extbuf_, __extbufnext_, __extbufend_ - __extbufnext_);
+      }
+      __extbufnext_ = __extbuf_ + (__extbufend_ - __extbufnext_);
+      __extbufend_  = __extbuf_ + (__extbuf_ == __extbuf_min_ ? sizeof(__extbuf_min_) : __ebs_);
+      size_t __nmemb =
+          std::min(static_cast<size_t>(__ibs_ - __unget_sz), static_cast<size_t>(__extbufend_ - __extbufnext_));
+      codecvt_base::result __r;
+      __st_last_  = __st_;
+      size_t __nr = fread((void*)const_cast<char*>(__extbufnext_), 1, __nmemb, __file_);
+      if (__nr != 0) {
+        if (!__cv_)
+          __throw_bad_cast();
+
+        __extbufend_ = __extbufnext_ + __nr;
+        char_type* __inext;
+        __r = __cv_->in(
+            __st_, __extbuf_, __extbufend_, __extbufnext_, this->eback() + __unget_sz, this->eback() + __ibs_, __inext);
+        if (__r == codecvt_base::noconv) {
+          this->setg((char_type*)__extbuf_, (char_type*)__extbuf_, (char_type*)const_cast<char*>(__extbufend_));
+          __c = traits_type::to_int_type(*this->gptr());
+        } else if (__inext != this->eback() + __unget_sz) {
+          this->setg(this->eback(), this->eback() + __unget_sz, __inext);
+          __c = traits_type::to_int_type(*this->gptr());
+        }
+      }
+    }
+  } else
+    __c = traits_type::to_int_type(*this->gptr());
+  if (this->eback() == &__1buf)
+    this->setg(nullptr, nullptr, nullptr);
+  return __c;
+}
+
+template <class _CharT, class _Traits>
+typename basic_filebuf<_CharT, _Traits>::int_type basic_filebuf<_CharT, _Traits>::pbackfail(int_type __c) {
+  if (__file_ && this->eback() < this->gptr()) {
+    if (traits_type::eq_int_type(__c, traits_type::eof())) {
+      this->gbump(-1);
+      return traits_type::not_eof(__c);
+    }
+    if ((__om_ & ios_base::out) || traits_type::eq(traits_type::to_char_type(__c), this->gptr()[-1])) {
+      this->gbump(-1);
+      *this->gptr() = traits_type::to_char_type(__c);
+      return __c;
+    }
+  }
+  return traits_type::eof();
+}
+
+template <class _CharT, class _Traits>
+typename basic_filebuf<_CharT, _Traits>::int_type basic_filebuf<_CharT, _Traits>::overflow(int_type __c) {
+  if (__file_ == nullptr)
+    return traits_type::eof();
+  __write_mode();
+  char_type __1buf;
+  char_type* __pb_save  = this->pbase();
+  char_type* __epb_save = this->epptr();
+  if (!traits_type::eq_int_type(__c, traits_type::eof())) {
+    if (this->pptr() == nullptr)
+      this->setp(&__1buf, &__1buf + 1);
+    *this->pptr() = traits_type::to_char_type(__c);
+    this->pbump(1);
+  }
+  if (this->pptr() != this->pbase()) {
+    if (__always_noconv_) {
+      size_t __nmemb = static_cast<size_t>(this->pptr() - this->pbase());
+      if (std::fwrite(this->pbase(), sizeof(char_type), __nmemb, __file_) != __nmemb)
+        return traits_type::eof();
+    } else {
+      char* __extbe = __extbuf_;
+      codecvt_base::result __r;
+      do {
+        if (!__cv_)
+          __throw_bad_cast();
+
+        const char_type* __e;
+        __r = __cv_->out(__st_, this->pbase(), this->pptr(), __e, __extbuf_, __extbuf_ + __ebs_, __extbe);
+        if (__e == this->pbase())
+          return traits_type::eof();
+        if (__r == codecvt_base::noconv) {
+          size_t __nmemb = static_cast<size_t>(this->pptr() - this->pbase());
+          if (std::fwrite(this->pbase(), 1, __nmemb, __file_) != __nmemb)
+            return traits_type::eof();
+        } else if (__r == codecvt_base::ok || __r == codecvt_base::partial) {
+          size_t __nmemb = static_cast<size_t>(__extbe - __extbuf_);
+          if (fwrite(__extbuf_, 1, __nmemb, __file_) != __nmemb)
+            return traits_type::eof();
+          if (__r == codecvt_base::partial) {
+            this->setp(const_cast<char_type*>(__e), this->pptr());
+            this->__pbump(this->epptr() - this->pbase());
+          }
+        } else
+          return traits_type::eof();
+      } while (__r == codecvt_base::partial);
+    }
+    this->setp(__pb_save, __epb_save);
+  }
+  return traits_type::not_eof(__c);
+}
+
+template <class _CharT, class _Traits>
+basic_streambuf<_CharT, _Traits>* basic_filebuf<_CharT, _Traits>::setbuf(char_type* __s, streamsize __n) {
+  this->setg(nullptr, nullptr, nullptr);
+  this->setp(nullptr, nullptr);
+  __request_unbuffered_mode(__s, __n);
+  if (__owns_eb_)
+    delete[] __extbuf_;
+  if (__owns_ib_)
+    delete[] __intbuf_;
+  __ebs_ = __n;
+  if (__ebs_ > sizeof(__extbuf_min_)) {
+    if (__always_noconv_ && __s) {
+      __extbuf_  = (char*)__s;
+      __owns_eb_ = false;
+    } else {
+      __extbuf_  = new char[__ebs_];
+      __owns_eb_ = true;
+    }
+  } else {
+    __extbuf_  = __extbuf_min_;
+    __ebs_     = sizeof(__extbuf_min_);
+    __owns_eb_ = false;
+  }
+  if (!__always_noconv_) {
+    __ibs_ = max<streamsize>(__n, sizeof(__extbuf_min_));
+    if (__s && __ibs_ > sizeof(__extbuf_min_)) {
+      __intbuf_  = __s;
+      __owns_ib_ = false;
+    } else {
+      __intbuf_  = new char_type[__ibs_];
+      __owns_ib_ = true;
+    }
+  } else {
+    __ibs_     = 0;
+    __intbuf_  = nullptr;
+    __owns_ib_ = false;
+  }
+  return this;
+}
+
+template <class _CharT, class _Traits>
+typename basic_filebuf<_CharT, _Traits>::pos_type
+basic_filebuf<_CharT, _Traits>::seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode) {
+  if (!__cv_)
+    __throw_bad_cast();
+
+  int __width = __cv_->encoding();
+  if (__file_ == nullptr || (__width <= 0 && __off != 0) || sync())
+    return pos_type(off_type(-1));
+  // __width > 0 || __off == 0
+  int __whence;
+  switch (__way) {
+  case ios_base::beg:
+    __whence = SEEK_SET;
+    break;
+  case ios_base::cur:
+    __whence = SEEK_CUR;
+    break;
+  case ios_base::end:
+    __whence = SEEK_END;
+    break;
+  default:
+    return pos_type(off_type(-1));
+  }
+#  if defined(_LIBCPP_HAS_NO_OFF_T_FUNCTIONS)
+  if (fseek(__file_, __width > 0 ? __width * __off : 0, __whence))
+    return pos_type(off_type(-1));
+  pos_type __r = ftell(__file_);
+#  else
+  if (::fseeko(__file_, __width > 0 ? __width * __off : 0, __whence))
+    return pos_type(off_type(-1));
+  pos_type __r = ftello(__file_);
+#  endif
+  __r.state(__st_);
+  return __r;
+}
+
+template <class _CharT, class _Traits>
+typename basic_filebuf<_CharT, _Traits>::pos_type
+basic_filebuf<_CharT, _Traits>::seekpos(pos_type __sp, ios_base::openmode) {
+  if (__file_ == nullptr || sync())
+    return pos_type(off_type(-1));
+#  if defined(_LIBCPP_HAS_NO_OFF_T_FUNCTIONS)
+  if (fseek(__file_, __sp, SEEK_SET))
+    return pos_type(off_type(-1));
+#  else
+  if (::fseeko(__file_, __sp, SEEK_SET))
+    return pos_type(off_type(-1));
+#  endif
+  __st_ = __sp.state();
+  return __sp;
+}
+
+template <class _CharT, class _Traits>
+int basic_filebuf<_CharT, _Traits>::sync() {
+  if (__file_ == nullptr)
+    return 0;
+  if (!__cv_)
+    __throw_bad_cast();
+
+  if (__cm_ & ios_base::out) {
+    if (this->pptr() != this->pbase())
+      if (overflow() == traits_type::eof())
+        return -1;
+    codecvt_base::result __r;
+    do {
+      char* __extbe;
+      __r            = __cv_->unshift(__st_, __extbuf_, __extbuf_ + __ebs_, __extbe);
+      size_t __nmemb = static_cast<size_t>(__extbe - __extbuf_);
+      if (fwrite(__extbuf_, 1, __nmemb, __file_) != __nmemb)
+        return -1;
+    } while (__r == codecvt_base::partial);
+    if (__r == codecvt_base::error)
+      return -1;
+    if (fflush(__file_))
+      return -1;
+  } else if (__cm_ & ios_base::in) {
+    off_type __c;
+    state_type __state = __st_last_;
+    bool __update_st   = false;
+    if (__always_noconv_)
+      __c = this->egptr() - this->gptr();
+    else {
+      int __width = __cv_->encoding();
+      __c         = __extbufend_ - __extbufnext_;
+      if (__width > 0)
+        __c += __width * (this->egptr() - this->gptr());
+      else {
+        if (this->gptr() != this->egptr()) {
+          const int __off = __cv_->length(__state, __extbuf_, __extbufnext_, this->gptr() - this->eback());
+          __c += __extbufnext_ - __extbuf_ - __off;
+          __update_st = true;
+        }
+      }
+    }
+#  if defined(_LIBCPP_HAS_NO_OFF_T_FUNCTIONS)
+    if (fseek(__file_, -__c, SEEK_CUR))
+      return -1;
+#  else
+    if (::fseeko(__file_, -__c, SEEK_CUR))
+      return -1;
+#  endif
+    if (__update_st)
+      __st_ = __state;
+    __extbufnext_ = __extbufend_ = __extbuf_;
+    this->setg(nullptr, nullptr, nullptr);
+    __cm_ = 0;
+  }
+  return 0;
+}
+
+template <class _CharT, class _Traits>
+void basic_filebuf<_CharT, _Traits>::imbue(const locale& __loc) {
+  sync();
+  __cv_            = &std::use_facet<codecvt<char_type, char, state_type> >(__loc);
+  bool __old_anc   = __always_noconv_;
+  __always_noconv_ = __cv_->always_noconv();
+  if (__old_anc != __always_noconv_) {
+    this->setg(nullptr, nullptr, nullptr);
+    this->setp(nullptr, nullptr);
+    // invariant, char_type is char, else we couldn't get here
+    if (__always_noconv_) // need to dump __intbuf_
+    {
+      if (__owns_eb_)
+        delete[] __extbuf_;
+      __owns_eb_ = __owns_ib_;
+      __ebs_     = __ibs_;
+      __extbuf_  = (char*)__intbuf_;
+      __ibs_     = 0;
+      __intbuf_  = nullptr;
+      __owns_ib_ = false;
+    } else // need to obtain an __intbuf_.
+    {      // If __extbuf_ is user-supplied, use it, else new __intbuf_
+      if (!__owns_eb_ && __extbuf_ != __extbuf_min_) {
+        __ibs_     = __ebs_;
+        __intbuf_  = (char_type*)__extbuf_;
+        __owns_ib_ = false;
+        __extbuf_  = new char[__ebs_];
+        __owns_eb_ = true;
+      } else {
+        __ibs_     = __ebs_;
+        __intbuf_  = new char_type[__ibs_];
+        __owns_ib_ = true;
+      }
+    }
+  }
+}
+
+template <class _CharT, class _Traits>
+bool basic_filebuf<_CharT, _Traits>::__read_mode() {
+  if (!(__cm_ & ios_base::in)) {
+    this->setp(nullptr, nullptr);
+    if (__always_noconv_)
+      this->setg((char_type*)__extbuf_, (char_type*)__extbuf_ + __ebs_, (char_type*)__extbuf_ + __ebs_);
+    else
+      this->setg(__intbuf_, __intbuf_ + __ibs_, __intbuf_ + __ibs_);
+    __cm_ = ios_base::in;
+    return true;
+  }
+  return false;
+}
+
+template <class _CharT, class _Traits>
+void basic_filebuf<_CharT, _Traits>::__write_mode() {
+  if (!(__cm_ & ios_base::out)) {
+    this->setg(nullptr, nullptr, nullptr);
+    if (__ebs_ > sizeof(__extbuf_min_)) {
+      if (__always_noconv_)
+        this->setp((char_type*)__extbuf_, (char_type*)__extbuf_ + (__ebs_ - 1));
+      else
+        this->setp(__intbuf_, __intbuf_ + (__ibs_ - 1));
+    } else
+      this->setp(nullptr, nullptr);
+    __cm_ = ios_base::out;
+  }
+}
+
+// basic_ifstream
+
+template <class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS basic_ifstream : public basic_istream<_CharT, _Traits> {
+public:
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+#  if _LIBCPP_STD_VER >= 26
+  using native_handle_type = typename basic_filebuf<_CharT, _Traits>::native_handle_type;
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI basic_ifstream();
+  _LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const char* __s, ios_base::openmode __mode = ios_base::in);
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+  _LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const wchar_t* __s, ios_base::openmode __mode = ios_base::in);
+#  endif
+  _LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const string& __s, ios_base::openmode __mode = ios_base::in);
+#  if _LIBCPP_STD_VER >= 17
+  template <class _Tp, class = enable_if_t<is_same_v<_Tp, filesystem::path>>>
+  _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY
+      _LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const _Tp& __p, ios_base::openmode __mode = ios_base::in)
+      : basic_ifstream(__p.c_str(), __mode) {}
+#  endif // _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI basic_ifstream(basic_ifstream&& __rhs);
+  _LIBCPP_HIDE_FROM_ABI basic_ifstream& operator=(basic_ifstream&& __rhs);
+  _LIBCPP_HIDE_FROM_ABI void swap(basic_ifstream& __rhs);
+
+  _LIBCPP_HIDE_FROM_ABI basic_filebuf<char_type, traits_type>* rdbuf() const;
+#  if _LIBCPP_STD_VER >= 26
+  _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { return rdbuf()->native_handle(); }
+#  endif
+  _LIBCPP_HIDE_FROM_ABI bool is_open() const;
+  void open(const char* __s, ios_base::openmode __mode = ios_base::in);
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+  void open(const wchar_t* __s, ios_base::openmode __mode = ios_base::in);
+#  endif
+  void open(const string& __s, ios_base::openmode __mode = ios_base::in);
+#  if _LIBCPP_STD_VER >= 17
+  _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_HIDE_FROM_ABI void
+  open(const filesystem::path& __p, ios_base::openmode __mode = ios_base::in) {
+    return open(__p.c_str(), __mode);
+  }
+#  endif // _LIBCPP_STD_VER >= 17
+
+  _LIBCPP_HIDE_FROM_ABI void __open(int __fd, ios_base::openmode __mode);
+  _LIBCPP_HIDE_FROM_ABI void close();
+
+private:
+  basic_filebuf<char_type, traits_type> __sb_;
+};
+
+template <class _CharT, class _Traits>
+inline basic_ifstream<_CharT, _Traits>::basic_ifstream() : basic_istream<char_type, traits_type>(&__sb_) {}
+
+template <class _CharT, class _Traits>
+inline basic_ifstream<_CharT, _Traits>::basic_ifstream(const char* __s, ios_base::openmode __mode)
+    : basic_istream<char_type, traits_type>(&__sb_) {
+  if (__sb_.open(__s, __mode | ios_base::in) == nullptr)
+    this->setstate(ios_base::failbit);
+}
+
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+template <class _CharT, class _Traits>
+inline basic_ifstream<_CharT, _Traits>::basic_ifstream(const wchar_t* __s, ios_base::openmode __mode)
+    : basic_istream<char_type, traits_type>(&__sb_) {
+  if (__sb_.open(__s, __mode | ios_base::in) == nullptr)
+    this->setstate(ios_base::failbit);
+}
+#  endif
+
+template <class _CharT, class _Traits>
+inline basic_ifstream<_CharT, _Traits>::basic_ifstream(const string& __s, ios_base::openmode __mode)
+    : basic_istream<char_type, traits_type>(&__sb_) {
+  if (__sb_.open(__s, __mode | ios_base::in) == nullptr)
+    this->setstate(ios_base::failbit);
+}
+
+template <class _CharT, class _Traits>
+inline basic_ifstream<_CharT, _Traits>::basic_ifstream(basic_ifstream&& __rhs)
+    : basic_istream<char_type, traits_type>(std::move(__rhs)), __sb_(std::move(__rhs.__sb_)) {
+  this->set_rdbuf(&__sb_);
+}
+
+template <class _CharT, class _Traits>
+inline basic_ifstream<_CharT, _Traits>& basic_ifstream<_CharT, _Traits>::operator=(basic_ifstream&& __rhs) {
+  basic_istream<char_type, traits_type>::operator=(std::move(__rhs));
+  __sb_ = std::move(__rhs.__sb_);
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+inline void basic_ifstream<_CharT, _Traits>::swap(basic_ifstream& __rhs) {
+  basic_istream<char_type, traits_type>::swap(__rhs);
+  __sb_.swap(__rhs.__sb_);
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI void swap(basic_ifstream<_CharT, _Traits>& __x, basic_ifstream<_CharT, _Traits>& __y) {
+  __x.swap(__y);
+}
+
+template <class _CharT, class _Traits>
+inline basic_filebuf<_CharT, _Traits>* basic_ifstream<_CharT, _Traits>::rdbuf() const {
+  return const_cast<basic_filebuf<char_type, traits_type>*>(&__sb_);
+}
+
+template <class _CharT, class _Traits>
+inline bool basic_ifstream<_CharT, _Traits>::is_open() const {
+  return __sb_.is_open();
+}
+
+template <class _CharT, class _Traits>
+void basic_ifstream<_CharT, _Traits>::open(const char* __s, ios_base::openmode __mode) {
+  if (__sb_.open(__s, __mode | ios_base::in))
+    this->clear();
+  else
+    this->setstate(ios_base::failbit);
+}
+
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+template <class _CharT, class _Traits>
+void basic_ifstream<_CharT, _Traits>::open(const wchar_t* __s, ios_base::openmode __mode) {
+  if (__sb_.open(__s, __mode | ios_base::in))
+    this->clear();
+  else
+    this->setstate(ios_base::failbit);
+}
+#  endif
+
+template <class _CharT, class _Traits>
+void basic_ifstream<_CharT, _Traits>::open(const string& __s, ios_base::openmode __mode) {
+  if (__sb_.open(__s, __mode | ios_base::in))
+    this->clear();
+  else
+    this->setstate(ios_base::failbit);
+}
+
+template <class _CharT, class _Traits>
+inline void basic_ifstream<_CharT, _Traits>::__open(int __fd, ios_base::openmode __mode) {
+  if (__sb_.__open(__fd, __mode | ios_base::in))
+    this->clear();
+  else
+    this->setstate(ios_base::failbit);
+}
+
+template <class _CharT, class _Traits>
+inline void basic_ifstream<_CharT, _Traits>::close() {
+  if (__sb_.close() == 0)
+    this->setstate(ios_base::failbit);
+}
+
+// basic_ofstream
+
+template <class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS basic_ofstream : public basic_ostream<_CharT, _Traits> {
+public:
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+#  if _LIBCPP_STD_VER >= 26
+  using native_handle_type = typename basic_filebuf<_CharT, _Traits>::native_handle_type;
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI basic_ofstream();
+  _LIBCPP_HIDE_FROM_ABI explicit basic_ofstream(const char* __s, ios_base::openmode __mode = ios_base::out);
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+  _LIBCPP_HIDE_FROM_ABI explicit basic_ofstream(const wchar_t* __s, ios_base::openmode __mode = ios_base::out);
+#  endif
+  _LIBCPP_HIDE_FROM_ABI explicit basic_ofstream(const string& __s, ios_base::openmode __mode = ios_base::out);
+
+#  if _LIBCPP_STD_VER >= 17
+  template <class _Tp, class = enable_if_t<is_same_v<_Tp, filesystem::path>>>
+  _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY
+      _LIBCPP_HIDE_FROM_ABI explicit basic_ofstream(const _Tp& __p, ios_base::openmode __mode = ios_base::out)
+      : basic_ofstream(__p.c_str(), __mode) {}
+#  endif // _LIBCPP_STD_VER >= 17
+
+  _LIBCPP_HIDE_FROM_ABI basic_ofstream(basic_ofstream&& __rhs);
+  _LIBCPP_HIDE_FROM_ABI basic_ofstream& operator=(basic_ofstream&& __rhs);
+  _LIBCPP_HIDE_FROM_ABI void swap(basic_ofstream& __rhs);
+
+  _LIBCPP_HIDE_FROM_ABI basic_filebuf<char_type, traits_type>* rdbuf() const;
+#  if _LIBCPP_STD_VER >= 26
+  _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { return rdbuf()->native_handle(); }
+#  endif
+  _LIBCPP_HIDE_FROM_ABI bool is_open() const;
+  void open(const char* __s, ios_base::openmode __mode = ios_base::out);
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+  void open(const wchar_t* __s, ios_base::openmode __mode = ios_base::out);
+#  endif
+  void open(const string& __s, ios_base::openmode __mode = ios_base::out);
+
+#  if _LIBCPP_STD_VER >= 17
+  _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_HIDE_FROM_ABI void
+  open(const filesystem::path& __p, ios_base::openmode __mode = ios_base::out) {
+    return open(__p.c_str(), __mode);
+  }
+#  endif // _LIBCPP_STD_VER >= 17
+
+  _LIBCPP_HIDE_FROM_ABI void __open(int __fd, ios_base::openmode __mode);
+  _LIBCPP_HIDE_FROM_ABI void close();
+
+private:
+  basic_filebuf<char_type, traits_type> __sb_;
+};
+
+template <class _CharT, class _Traits>
+inline basic_ofstream<_CharT, _Traits>::basic_ofstream() : basic_ostream<char_type, traits_type>(&__sb_) {}
+
+template <class _CharT, class _Traits>
+inline basic_ofstream<_CharT, _Traits>::basic_ofstream(const char* __s, ios_base::openmode __mode)
+    : basic_ostream<char_type, traits_type>(&__sb_) {
+  if (__sb_.open(__s, __mode | ios_base::out) == nullptr)
+    this->setstate(ios_base::failbit);
+}
+
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+template <class _CharT, class _Traits>
+inline basic_ofstream<_CharT, _Traits>::basic_ofstream(const wchar_t* __s, ios_base::openmode __mode)
+    : basic_ostream<char_type, traits_type>(&__sb_) {
+  if (__sb_.open(__s, __mode | ios_base::out) == nullptr)
+    this->setstate(ios_base::failbit);
+}
+#  endif
+
+template <class _CharT, class _Traits>
+inline basic_ofstream<_CharT, _Traits>::basic_ofstream(const string& __s, ios_base::openmode __mode)
+    : basic_ostream<char_type, traits_type>(&__sb_) {
+  if (__sb_.open(__s, __mode | ios_base::out) == nullptr)
+    this->setstate(ios_base::failbit);
+}
+
+template <class _CharT, class _Traits>
+inline basic_ofstream<_CharT, _Traits>::basic_ofstream(basic_ofstream&& __rhs)
+    : basic_ostream<char_type, traits_type>(std::move(__rhs)), __sb_(std::move(__rhs.__sb_)) {
+  this->set_rdbuf(&__sb_);
+}
+
+template <class _CharT, class _Traits>
+inline basic_ofstream<_CharT, _Traits>& basic_ofstream<_CharT, _Traits>::operator=(basic_ofstream&& __rhs) {
+  basic_ostream<char_type, traits_type>::operator=(std::move(__rhs));
+  __sb_ = std::move(__rhs.__sb_);
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+inline void basic_ofstream<_CharT, _Traits>::swap(basic_ofstream& __rhs) {
+  basic_ostream<char_type, traits_type>::swap(__rhs);
+  __sb_.swap(__rhs.__sb_);
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI void swap(basic_ofstream<_CharT, _Traits>& __x, basic_ofstream<_CharT, _Traits>& __y) {
+  __x.swap(__y);
+}
+
+template <class _CharT, class _Traits>
+inline basic_filebuf<_CharT, _Traits>* basic_ofstream<_CharT, _Traits>::rdbuf() const {
+  return const_cast<basic_filebuf<char_type, traits_type>*>(&__sb_);
+}
+
+template <class _CharT, class _Traits>
+inline bool basic_ofstream<_CharT, _Traits>::is_open() const {
+  return __sb_.is_open();
+}
+
+template <class _CharT, class _Traits>
+void basic_ofstream<_CharT, _Traits>::open(const char* __s, ios_base::openmode __mode) {
+  if (__sb_.open(__s, __mode | ios_base::out))
+    this->clear();
+  else
+    this->setstate(ios_base::failbit);
+}
+
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+template <class _CharT, class _Traits>
+void basic_ofstream<_CharT, _Traits>::open(const wchar_t* __s, ios_base::openmode __mode) {
+  if (__sb_.open(__s, __mode | ios_base::out))
+    this->clear();
+  else
+    this->setstate(ios_base::failbit);
+}
+#  endif
+
+template <class _CharT, class _Traits>
+void basic_ofstream<_CharT, _Traits>::open(const string& __s, ios_base::openmode __mode) {
+  if (__sb_.open(__s, __mode | ios_base::out))
+    this->clear();
+  else
+    this->setstate(ios_base::failbit);
+}
+
+template <class _CharT, class _Traits>
+inline void basic_ofstream<_CharT, _Traits>::__open(int __fd, ios_base::openmode __mode) {
+  if (__sb_.__open(__fd, __mode | ios_base::out))
+    this->clear();
+  else
+    this->setstate(ios_base::failbit);
+}
+
+template <class _CharT, class _Traits>
+inline void basic_ofstream<_CharT, _Traits>::close() {
+  if (__sb_.close() == nullptr)
+    this->setstate(ios_base::failbit);
+}
+
+// basic_fstream
+
+template <class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS basic_fstream : public basic_iostream<_CharT, _Traits> {
+public:
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+#  if _LIBCPP_STD_VER >= 26
+  using native_handle_type = typename basic_filebuf<_CharT, _Traits>::native_handle_type;
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI basic_fstream();
+  _LIBCPP_HIDE_FROM_ABI explicit basic_fstream(const char* __s,
+                                               ios_base::openmode __mode = ios_base::in | ios_base::out);
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+  _LIBCPP_HIDE_FROM_ABI explicit basic_fstream(const wchar_t* __s,
+                                               ios_base::openmode __mode = ios_base::in | ios_base::out);
+#  endif
+  _LIBCPP_HIDE_FROM_ABI explicit basic_fstream(const string& __s,
+                                               ios_base::openmode __mode = ios_base::in | ios_base::out);
+
+#  if _LIBCPP_STD_VER >= 17
+  template <class _Tp, class = enable_if_t<is_same_v<_Tp, filesystem::path>>>
+  _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_HIDE_FROM_ABI explicit basic_fstream(
+      const _Tp& __p, ios_base::openmode __mode = ios_base::in | ios_base::out)
+      : basic_fstream(__p.c_str(), __mode) {}
+#  endif // _LIBCPP_STD_VER >= 17
+
+  _LIBCPP_HIDE_FROM_ABI basic_fstream(basic_fstream&& __rhs);
+
+  _LIBCPP_HIDE_FROM_ABI basic_fstream& operator=(basic_fstream&& __rhs);
+
+  _LIBCPP_HIDE_FROM_ABI void swap(basic_fstream& __rhs);
+
+  _LIBCPP_HIDE_FROM_ABI basic_filebuf<char_type, traits_type>* rdbuf() const;
+#  if _LIBCPP_STD_VER >= 26
+  _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { return rdbuf()->native_handle(); }
+#  endif
+  _LIBCPP_HIDE_FROM_ABI bool is_open() const;
+  _LIBCPP_HIDE_FROM_ABI void open(const char* __s, ios_base::openmode __mode = ios_base::in | ios_base::out);
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+  void open(const wchar_t* __s, ios_base::openmode __mode = ios_base::in | ios_base::out);
+#  endif
+  _LIBCPP_HIDE_FROM_ABI void open(const string& __s, ios_base::openmode __mode = ios_base::in | ios_base::out);
+
+#  if _LIBCPP_STD_VER >= 17
+  _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_HIDE_FROM_ABI void
+  open(const filesystem::path& __p, ios_base::openmode __mode = ios_base::in | ios_base::out) {
+    return open(__p.c_str(), __mode);
+  }
+#  endif // _LIBCPP_STD_VER >= 17
+
+  _LIBCPP_HIDE_FROM_ABI void close();
+
+private:
+  basic_filebuf<char_type, traits_type> __sb_;
+};
+
+template <class _CharT, class _Traits>
+inline basic_fstream<_CharT, _Traits>::basic_fstream() : basic_iostream<char_type, traits_type>(&__sb_) {}
+
+template <class _CharT, class _Traits>
+inline basic_fstream<_CharT, _Traits>::basic_fstream(const char* __s, ios_base::openmode __mode)
+    : basic_iostream<char_type, traits_type>(&__sb_) {
+  if (__sb_.open(__s, __mode) == nullptr)
+    this->setstate(ios_base::failbit);
+}
+
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+template <class _CharT, class _Traits>
+inline basic_fstream<_CharT, _Traits>::basic_fstream(const wchar_t* __s, ios_base::openmode __mode)
+    : basic_iostream<char_type, traits_type>(&__sb_) {
+  if (__sb_.open(__s, __mode) == nullptr)
+    this->setstate(ios_base::failbit);
+}
+#  endif
+
+template <class _CharT, class _Traits>
+inline basic_fstream<_CharT, _Traits>::basic_fstream(const string& __s, ios_base::openmode __mode)
+    : basic_iostream<char_type, traits_type>(&__sb_) {
+  if (__sb_.open(__s, __mode) == nullptr)
+    this->setstate(ios_base::failbit);
+}
+
+template <class _CharT, class _Traits>
+inline basic_fstream<_CharT, _Traits>::basic_fstream(basic_fstream&& __rhs)
+    : basic_iostream<char_type, traits_type>(std::move(__rhs)), __sb_(std::move(__rhs.__sb_)) {
+  this->set_rdbuf(&__sb_);
+}
+
+template <class _CharT, class _Traits>
+inline basic_fstream<_CharT, _Traits>& basic_fstream<_CharT, _Traits>::operator=(basic_fstream&& __rhs) {
+  basic_iostream<char_type, traits_type>::operator=(std::move(__rhs));
+  __sb_ = std::move(__rhs.__sb_);
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+inline void basic_fstream<_CharT, _Traits>::swap(basic_fstream& __rhs) {
+  basic_iostream<char_type, traits_type>::swap(__rhs);
+  __sb_.swap(__rhs.__sb_);
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI void swap(basic_fstream<_CharT, _Traits>& __x, basic_fstream<_CharT, _Traits>& __y) {
+  __x.swap(__y);
+}
+
+template <class _CharT, class _Traits>
+inline basic_filebuf<_CharT, _Traits>* basic_fstream<_CharT, _Traits>::rdbuf() const {
+  return const_cast<basic_filebuf<char_type, traits_type>*>(&__sb_);
+}
+
+template <class _CharT, class _Traits>
+inline bool basic_fstream<_CharT, _Traits>::is_open() const {
+  return __sb_.is_open();
+}
+
+template <class _CharT, class _Traits>
+void basic_fstream<_CharT, _Traits>::open(const char* __s, ios_base::openmode __mode) {
+  if (__sb_.open(__s, __mode))
+    this->clear();
+  else
+    this->setstate(ios_base::failbit);
+}
+
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+template <class _CharT, class _Traits>
+void basic_fstream<_CharT, _Traits>::open(const wchar_t* __s, ios_base::openmode __mode) {
+  if (__sb_.open(__s, __mode))
+    this->clear();
+  else
+    this->setstate(ios_base::failbit);
+}
+#  endif
+
+template <class _CharT, class _Traits>
+void basic_fstream<_CharT, _Traits>::open(const string& __s, ios_base::openmode __mode) {
+  if (__sb_.open(__s, __mode))
+    this->clear();
+  else
+    this->setstate(ios_base::failbit);
+}
+
+template <class _CharT, class _Traits>
+inline void basic_fstream<_CharT, _Traits>::close() {
+  if (__sb_.close() == nullptr)
+    this->setstate(ios_base::failbit);
+}
+
+#  if _LIBCPP_AVAILABILITY_HAS_ADDITIONAL_IOSTREAM_EXPLICIT_INSTANTIATIONS_1
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_ifstream<char>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_ofstream<char>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_filebuf<char>;
+#  endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_NO_FILESYSTEM
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <concepts>
+#  include <cstdlib>
+#  include <iosfwd>
+#  include <limits>
+#  include <mutex>
+#  include <new>
+#  include <stdexcept>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_FSTREAM
diff --git a/libcxx/include/__cxx03/functional b/libcxx/include/__cxx03/functional
new file mode 100644
index 000000000000..3d39f654ddb0
--- /dev/null
+++ b/libcxx/include/__cxx03/functional
@@ -0,0 +1,598 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_FUNCTIONAL
+#define _LIBCPP_FUNCTIONAL
+
+/*
+    functional synopsis
+
+namespace std
+{
+
+template <class Arg, class Result>
+struct unary_function
+{
+    typedef Arg    argument_type;
+    typedef Result result_type;
+};
+
+template <class Arg1, class Arg2, class Result>
+struct binary_function
+{
+    typedef Arg1   first_argument_type;
+    typedef Arg2   second_argument_type;
+    typedef Result result_type;
+};
+
+template <class T>
+class reference_wrapper
+    : public unary_function<T1, R> // if wrapping a unary functor
+    : public binary_function<T1, T2, R> // if wrapping a binary functor
+{
+public:
+    // types
+    typedef T type;
+    typedef see below result_type; // Not always defined
+
+    // construct/copy/destroy
+    template<class U>
+      constexpr reference_wrapper(U&&);                                   // constexpr since C++20
+    constexpr reference_wrapper(const reference_wrapper<T>& x) noexcept;  // constexpr since C++20
+
+    // assignment
+    constexpr reference_wrapper&
+    operator=(const reference_wrapper<T>& x) noexcept;                    // constexpr since C++20
+
+    // access
+    constexpr operator T& () const noexcept;                              // constexpr since C++20
+    constexpr T& get() const noexcept;                                    // constexpr since C++20
+
+    // invoke
+    template <class... ArgTypes>
+      constexpr typename result_of<T&(ArgTypes&&...)>::type               // constexpr since C++20
+          operator() (ArgTypes&&...) const
+              noexcept(is_nothrow_invocable_v<T&, ArgTypes...>);          // noexcept since C++17
+};
+
+template <class T>
+  reference_wrapper(T&) -> reference_wrapper<T>;
+
+template <class T> reference_wrapper<T> ref(T& t) noexcept;
+template <class T> void ref(const T&& t) = delete;
+template <class T> reference_wrapper<T> ref(reference_wrapper<T>t) noexcept;
+
+template <class T> reference_wrapper<const T> cref(const T& t) noexcept;
+template <class T> void cref(const T&& t) = delete;
+template <class T> reference_wrapper<const T> cref(reference_wrapper<T> t) noexcept;
+
+template <class T> struct unwrap_reference;                                       // since C++20
+template <class T> struct unwrap_ref_decay : unwrap_reference<decay_t<T>> { };    // since C++20
+template <class T> using unwrap_reference_t = typename unwrap_reference<T>::type; // since C++20
+template <class T> using unwrap_ref_decay_t = typename unwrap_ref_decay<T>::type; // since C++20
+
+// [refwrap.comparisons], comparisons
+friend constexpr bool operator==(reference_wrapper, reference_wrapper);           // Since C++26
+friend constexpr bool operator==(reference_wrapper, const T&);                    // Since C++26
+friend constexpr bool operator==(reference_wrapper, reference_wrapper<const T>);  // Since C++26
+
+friend constexpr auto operator<=>(reference_wrapper, reference_wrapper);          // Since C++26
+friend constexpr auto operator<=>(reference_wrapper, const T&);                   // Since C++26
+friend constexpr auto operator<=>(reference_wrapper, reference_wrapper<const T>); // Since C++26
+
+template <class T> // <class T=void> in C++14
+struct plus {
+    T operator()(const T& x, const T& y) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct minus {
+    T operator()(const T& x, const T& y) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct multiplies {
+    T operator()(const T& x, const T& y) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct divides {
+    T operator()(const T& x, const T& y) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct modulus {
+    T operator()(const T& x, const T& y) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct negate {
+    T operator()(const T& x) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct equal_to {
+    bool operator()(const T& x, const T& y) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct not_equal_to {
+    bool operator()(const T& x, const T& y) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct greater {
+    bool operator()(const T& x, const T& y) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct less {
+    bool operator()(const T& x, const T& y) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct greater_equal {
+    bool operator()(const T& x, const T& y) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct less_equal {
+    bool operator()(const T& x, const T& y) const;
+};
+
+// [comparisons.three.way], class compare_three_way
+struct compare_three_way;
+
+template <class T> // <class T=void> in C++14
+struct logical_and {
+    bool operator()(const T& x, const T& y) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct logical_or {
+    bool operator()(const T& x, const T& y) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct logical_not {
+    bool operator()(const T& x) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct bit_and {
+    T operator()(const T& x, const T& y) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct bit_or {
+    T operator()(const T& x, const T& y) const;
+};
+
+template <class T> // <class T=void> in C++14
+struct bit_xor {
+    T operator()(const T& x, const T& y) const;
+};
+
+template <class T=void> // C++14
+struct bit_not {
+    T operator()(const T& x) const;
+};
+
+struct identity; // C++20
+
+template <class Predicate>
+class unary_negate // deprecated in C++17, removed in C++20
+    : public unary_function<typename Predicate::argument_type, bool>
+{
+public:
+    explicit unary_negate(const Predicate& pred);
+    bool operator()(const typename Predicate::argument_type& x) const;
+};
+
+template <class Predicate> // deprecated in C++17, removed in C++20
+unary_negate<Predicate> not1(const Predicate& pred);
+
+template <class Predicate>
+class binary_negate // deprecated in C++17, removed in C++20
+    : public binary_function<typename Predicate::first_argument_type,
+                             typename Predicate::second_argument_type,
+                             bool>
+{
+public:
+    explicit binary_negate(const Predicate& pred);
+    bool operator()(const typename Predicate::first_argument_type& x,
+                    const typename Predicate::second_argument_type& y) const;
+};
+
+template <class Predicate> // deprecated in C++17, removed in C++20
+binary_negate<Predicate> not2(const Predicate& pred);
+
+template <class F>
+constexpr unspecified not_fn(F&& f); // C++17, constexpr in C++20
+
+// [func.bind.partial], function templates bind_front and bind_back
+template<class F, class... Args>
+  constexpr unspecified bind_front(F&&, Args&&...); // C++20
+template<class F, class... Args>
+  constexpr unspecified bind_back(F&&, Args&&...);  // C++23
+
+template<class T> struct is_bind_expression;
+template<class T> struct is_placeholder;
+
+    // See C++14 20.9.9, Function object binders
+template <class T> inline constexpr bool is_bind_expression_v
+  = is_bind_expression<T>::value; // C++17
+template <class T> inline constexpr int is_placeholder_v
+  = is_placeholder<T>::value; // C++17
+
+
+template<class Fn, class... BoundArgs>
+  constexpr unspecified bind(Fn&&, BoundArgs&&...);  // constexpr in C++20
+template<class R, class Fn, class... BoundArgs>
+  constexpr unspecified bind(Fn&&, BoundArgs&&...);  // constexpr in C++20
+
+// [func.invoke]
+template<class F, class... Args>
+ constexpr // constexpr in C++20
+ invoke_result_t<F, Args...> invoke(F&& f, Args&&... args) // C++17
+    noexcept(is_nothrow_invocable_v<F, Args...>);
+
+template<class R, class F, class... Args>
+  constexpr R invoke_r(F&& f, Args&&... args)              // C++23
+    noexcept(is_nothrow_invocable_r_v<R, F, Args...>);
+
+namespace placeholders {
+  // M is the implementation-defined number of placeholders
+  extern unspecified _1;
+  extern unspecified _2;
+  .
+  .
+  .
+  extern unspecified _Mp;
+}
+
+template <class Operation>
+class binder1st     // deprecated in C++11, removed in C++17
+    : public unary_function<typename Operation::second_argument_type,
+                            typename Operation::result_type>
+{
+protected:
+    Operation                               op;
+    typename Operation::first_argument_type value;
+public:
+    binder1st(const Operation& x, const typename Operation::first_argument_type y);
+    typename Operation::result_type operator()(      typename Operation::second_argument_type& x) const;
+    typename Operation::result_type operator()(const typename Operation::second_argument_type& x) const;
+};
+
+template <class Operation, class T>
+binder1st<Operation> bind1st(const Operation& op, const T& x);                  // deprecated in C++11, removed in C++17
+
+template <class Operation>
+class binder2nd                                                                 // deprecated in C++11, removed in C++17
+    : public unary_function<typename Operation::first_argument_type,
+                            typename Operation::result_type>
+{
+protected:
+    Operation                                op;
+    typename Operation::second_argument_type value;
+public:
+    binder2nd(const Operation& x, const typename Operation::second_argument_type y);
+    typename Operation::result_type operator()(      typename Operation::first_argument_type& x) const;
+    typename Operation::result_type operator()(const typename Operation::first_argument_type& x) const;
+};
+
+template <class Operation, class T>
+binder2nd<Operation> bind2nd(const Operation& op, const T& x);                  // deprecated in C++11, removed in C++17
+
+template <class Arg, class Result>                                              // deprecated in C++11, removed in C++17
+class pointer_to_unary_function : public unary_function<Arg, Result>
+{
+public:
+    explicit pointer_to_unary_function(Result (*f)(Arg));
+    Result operator()(Arg x) const;
+};
+
+template <class Arg, class Result>
+pointer_to_unary_function<Arg,Result> ptr_fun(Result (*f)(Arg));                // deprecated in C++11, removed in C++17
+
+template <class Arg1, class Arg2, class Result>                                 // deprecated in C++11, removed in C++17
+class pointer_to_binary_function : public binary_function<Arg1, Arg2, Result>
+{
+public:
+    explicit pointer_to_binary_function(Result (*f)(Arg1, Arg2));
+    Result operator()(Arg1 x, Arg2 y) const;
+};
+
+template <class Arg1, class Arg2, class Result>
+pointer_to_binary_function<Arg1,Arg2,Result> ptr_fun(Result (*f)(Arg1,Arg2));   // deprecated in C++11, removed in C++17
+
+template<class S, class T>                                                      // deprecated in C++11, removed in C++17
+class mem_fun_t : public unary_function<T*, S>
+{
+public:
+    explicit mem_fun_t(S (T::*p)());
+    S operator()(T* p) const;
+};
+
+template<class S, class T, class A>
+class mem_fun1_t : public binary_function<T*, A, S>                             // deprecated in C++11, removed in C++17
+{
+public:
+    explicit mem_fun1_t(S (T::*p)(A));
+    S operator()(T* p, A x) const;
+};
+
+template<class S, class T>          mem_fun_t<S,T>    mem_fun(S (T::*f)());     // deprecated in C++11, removed in C++17
+template<class S, class T, class A> mem_fun1_t<S,T,A> mem_fun(S (T::*f)(A));    // deprecated in C++11, removed in C++17
+
+template<class S, class T>
+class mem_fun_ref_t : public unary_function<T, S>                               // deprecated in C++11, removed in C++17
+{
+public:
+    explicit mem_fun_ref_t(S (T::*p)());
+    S operator()(T& p) const;
+};
+
+template<class S, class T, class A>
+class mem_fun1_ref_t : public binary_function<T, A, S>                          // deprecated in C++11, removed in C++17
+{
+public:
+    explicit mem_fun1_ref_t(S (T::*p)(A));
+    S operator()(T& p, A x) const;
+};
+
+template<class S, class T>
+mem_fun_ref_t<S,T>    mem_fun_ref(S (T::*f)());                                 // deprecated in C++11, removed in C++17
+template<class S, class T, class A>
+mem_fun1_ref_t<S,T,A> mem_fun_ref(S (T::*f)(A));                                // deprecated in C++11, removed in C++17
+
+template <class S, class T>
+class const_mem_fun_t : public unary_function<const T*, S>                      // deprecated in C++11, removed in C++17
+{
+public:
+    explicit const_mem_fun_t(S (T::*p)() const);
+    S operator()(const T* p) const;
+};
+
+template <class S, class T, class A>
+class const_mem_fun1_t : public binary_function<const T*, A, S>                 // deprecated in C++11, removed in C++17
+{
+public:
+    explicit const_mem_fun1_t(S (T::*p)(A) const);
+    S operator()(const T* p, A x) const;
+};
+
+template <class S, class T>
+const_mem_fun_t<S,T>    mem_fun(S (T::*f)() const);                             // deprecated in C++11, removed in C++17
+template <class S, class T, class A>
+const_mem_fun1_t<S,T,A> mem_fun(S (T::*f)(A) const);                            // deprecated in C++11, removed in C++17
+
+template <class S, class T>
+class const_mem_fun_ref_t : public unary_function<T, S>                         // deprecated in C++11, removed in C++17
+{
+public:
+    explicit const_mem_fun_ref_t(S (T::*p)() const);
+    S operator()(const T& p) const;
+};
+
+template <class S, class T, class A>
+class const_mem_fun1_ref_t : public binary_function<T, A, S>                    // deprecated in C++11, removed in C++17
+{
+public:
+    explicit const_mem_fun1_ref_t(S (T::*p)(A) const);
+    S operator()(const T& p, A x) const;
+};
+
+template <class S, class T>
+const_mem_fun_ref_t<S,T>    mem_fun_ref(S (T::*f)() const);                     // deprecated in C++11, removed in C++17
+template <class S, class T, class A>
+const_mem_fun1_ref_t<S,T,A> mem_fun_ref(S (T::*f)(A) const);                    // deprecated in C++11, removed in C++17
+
+template<class R, class T> constexpr unspecified mem_fn(R T::*);                // constexpr in C++20
+
+class bad_function_call
+    : public exception
+{
+};
+
+template<class> class function; // undefined
+
+template<class R, class... ArgTypes>
+class function<R(ArgTypes...)>
+  : public unary_function<T1, R>      // iff sizeof...(ArgTypes) == 1 and
+                                      // ArgTypes contains T1
+  : public binary_function<T1, T2, R> // iff sizeof...(ArgTypes) == 2 and
+                                      // ArgTypes contains T1 and T2
+{
+public:
+    typedef R result_type;
+
+    // construct/copy/destroy:
+    function() noexcept;
+    function(nullptr_t) noexcept;
+    function(const function&);
+    function(function&&) noexcept;
+    template<class F>
+      function(F);
+    template<Allocator Alloc>
+      function(allocator_arg_t, const Alloc&) noexcept;            // removed in C++17
+    template<Allocator Alloc>
+      function(allocator_arg_t, const Alloc&, nullptr_t) noexcept; // removed in C++17
+    template<Allocator Alloc>
+      function(allocator_arg_t, const Alloc&, const function&);    // removed in C++17
+    template<Allocator Alloc>
+      function(allocator_arg_t, const Alloc&, function&&);         // removed in C++17
+    template<class F, Allocator Alloc>
+      function(allocator_arg_t, const Alloc&, F);                  // removed in C++17
+
+    function& operator=(const function&);
+    function& operator=(function&&) noexcept;
+    function& operator=(nullptr_t) noexcept;
+    template<class F>
+      function& operator=(F&&);
+    template<class F>
+      function& operator=(reference_wrapper<F>) noexcept;
+
+    ~function();
+
+    // function modifiers:
+    void swap(function&) noexcept;
+    template<class F, class Alloc>
+      void assign(F&&, const Alloc&);                 // Removed in C++17
+
+    // function capacity:
+    explicit operator bool() const noexcept;
+
+    // function invocation:
+    R operator()(ArgTypes...) const;
+
+    // function target access:
+    const std::type_info& target_type() const noexcept;
+    template <typename T>       T* target() noexcept;
+    template <typename T> const T* target() const noexcept;
+};
+
+// Deduction guides
+template<class R, class ...Args>
+function(R(*)(Args...)) -> function<R(Args...)>; // since C++17
+
+template<class F>
+function(F) -> function<see-below>; // since C++17
+
+// Null pointer comparisons:
+template <class R, class ... ArgTypes>
+  bool operator==(const function<R(ArgTypes...)>&, nullptr_t) noexcept;
+
+template <class R, class ... ArgTypes>
+  bool operator==(nullptr_t, const function<R(ArgTypes...)>&) noexcept; // removed in C++20
+
+template <class R, class ... ArgTypes>
+  bool operator!=(const function<R(ArgTypes...)>&, nullptr_t) noexcept; // removed in C++20
+
+template <class  R, class ... ArgTypes>
+  bool operator!=(nullptr_t, const function<R(ArgTypes...)>&) noexcept; // removed in C++20
+
+// specialized algorithms:
+template <class  R, class ... ArgTypes>
+  void swap(function<R(ArgTypes...)>&, function<R(ArgTypes...)>&) noexcept;
+
+template <class T> struct hash;
+
+template <> struct hash<bool>;
+template <> struct hash<char>;
+template <> struct hash<signed char>;
+template <> struct hash<unsigned char>;
+template <> struct hash<char8_t>; // since C++20
+template <> struct hash<char16_t>;
+template <> struct hash<char32_t>;
+template <> struct hash<wchar_t>;
+template <> struct hash<short>;
+template <> struct hash<unsigned short>;
+template <> struct hash<int>;
+template <> struct hash<unsigned int>;
+template <> struct hash<long>;
+template <> struct hash<long long>;
+template <> struct hash<unsigned long>;
+template <> struct hash<unsigned long long>;
+
+template <> struct hash<float>;
+template <> struct hash<double>;
+template <> struct hash<long double>;
+
+template<class T> struct hash<T*>;
+template <> struct hash<nullptr_t>;  // C++17
+
+namespace ranges {
+  // [range.cmp], concept-constrained comparisons
+  struct equal_to;
+  struct not_equal_to;
+  struct greater;
+  struct less;
+  struct greater_equal;
+  struct less_equal;
+}
+
+}  // std
+
+POLICY:  For non-variadic implementations, the number of arguments is limited
+         to 3.  It is hoped that the need for non-variadic implementations
+         will be minimal.
+
+*/
+
+#include <__config>
+
+#include <__functional/binary_function.h>
+#include <__functional/binary_negate.h>
+#include <__functional/bind.h>
+#include <__functional/binder1st.h>
+#include <__functional/binder2nd.h>
+#include <__functional/hash.h>
+#include <__functional/mem_fn.h> // TODO: deprecate
+#include <__functional/mem_fun_ref.h>
+#include <__functional/operations.h>
+#include <__functional/pointer_to_binary_function.h>
+#include <__functional/pointer_to_unary_function.h>
+#include <__functional/reference_wrapper.h>
+#include <__functional/unary_function.h>
+#include <__functional/unary_negate.h>
+
+#ifndef _LIBCPP_CXX03_LANG
+#  include <__functional/function.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+#  include <__functional/boyer_moore_searcher.h>
+#  include <__functional/default_searcher.h>
+#  include <__functional/invoke.h>
+#  include <__functional/not_fn.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__functional/bind_back.h>
+#  include <__functional/bind_front.h>
+#  include <__functional/identity.h>
+#  include <__functional/ranges_operations.h>
+#  include <__type_traits/unwrap_ref.h>
+#endif
+
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && defined(_LIBCPP_CXX03_LANG)
+#  include <limits>
+#  include <new>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 14
+#  include <array>
+#  include <initializer_list>
+#  include <unordered_map>
+#  include <vector>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <concepts>
+#  include <cstdlib>
+#  include <exception>
+#  include <iosfwd>
+#  include <memory>
+#  include <stdexcept>
+#  include <tuple>
+#  include <type_traits>
+#  include <typeinfo>
+#  include <utility>
+#endif
+
+#endif // _LIBCPP_FUNCTIONAL
diff --git a/libcxx/include/__cxx03/future b/libcxx/include/__cxx03/future
new file mode 100644
index 000000000000..0be32620139e
--- /dev/null
+++ b/libcxx/include/__cxx03/future
@@ -0,0 +1,2062 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_FUTURE
+#define _LIBCPP_FUTURE
+
+/*
+    future synopsis
+
+namespace std
+{
+
+enum class future_errc
+{
+    future_already_retrieved = 1,
+    promise_already_satisfied,
+    no_state,
+    broken_promise
+};
+
+enum class launch
+{
+    async = 1,
+    deferred = 2,
+    any = async | deferred
+};
+
+enum class future_status
+{
+    ready,
+    timeout,
+    deferred
+};
+
+template <> struct is_error_code_enum<future_errc> : public true_type { };
+error_code make_error_code(future_errc e) noexcept;
+error_condition make_error_condition(future_errc e) noexcept;
+
+const error_category& future_category() noexcept;
+
+class future_error : public logic_error {
+public:
+    explicit future_error(future_errc e); // since C++17
+
+    const error_code& code() const noexcept;
+    const char*       what() const noexcept;
+
+private:
+    error_code ec_;             // exposition only
+};
+
+template <class R>
+class promise
+{
+public:
+    promise();
+    template <class Allocator>
+        promise(allocator_arg_t, const Allocator& a);
+    promise(promise&& rhs) noexcept;
+    promise(const promise& rhs) = delete;
+    ~promise();
+
+    // assignment
+    promise& operator=(promise&& rhs) noexcept;
+    promise& operator=(const promise& rhs) = delete;
+    void swap(promise& other) noexcept;
+
+    // retrieving the result
+    future<R> get_future();
+
+    // setting the result
+    void set_value(const R& r);
+    void set_value(R&& r);
+    void set_exception(exception_ptr p);
+
+    // setting the result with deferred notification
+    void set_value_at_thread_exit(const R& r);
+    void set_value_at_thread_exit(R&& r);
+    void set_exception_at_thread_exit(exception_ptr p);
+};
+
+template <class R>
+class promise<R&>
+{
+public:
+    promise();
+    template <class Allocator>
+        promise(allocator_arg_t, const Allocator& a);
+    promise(promise&& rhs) noexcept;
+    promise(const promise& rhs) = delete;
+    ~promise();
+
+    // assignment
+    promise& operator=(promise&& rhs) noexcept;
+    promise& operator=(const promise& rhs) = delete;
+    void swap(promise& other) noexcept;
+
+    // retrieving the result
+    future<R&> get_future();
+
+    // setting the result
+    void set_value(R& r);
+    void set_exception(exception_ptr p);
+
+    // setting the result with deferred notification
+    void set_value_at_thread_exit(R&);
+    void set_exception_at_thread_exit(exception_ptr p);
+};
+
+template <>
+class promise<void>
+{
+public:
+    promise();
+    template <class Allocator>
+        promise(allocator_arg_t, const Allocator& a);
+    promise(promise&& rhs) noexcept;
+    promise(const promise& rhs) = delete;
+    ~promise();
+
+    // assignment
+    promise& operator=(promise&& rhs) noexcept;
+    promise& operator=(const promise& rhs) = delete;
+    void swap(promise& other) noexcept;
+
+    // retrieving the result
+    future<void> get_future();
+
+    // setting the result
+    void set_value();
+    void set_exception(exception_ptr p);
+
+    // setting the result with deferred notification
+    void set_value_at_thread_exit();
+    void set_exception_at_thread_exit(exception_ptr p);
+};
+
+template <class R> void swap(promise<R>& x, promise<R>& y) noexcept;
+
+template <class R, class Alloc>
+    struct uses_allocator<promise<R>, Alloc> : public true_type {};
+
+template <class R>
+class future
+{
+public:
+    future() noexcept;
+    future(future&&) noexcept;
+    future(const future& rhs) = delete;
+    ~future();
+    future& operator=(const future& rhs) = delete;
+    future& operator=(future&&) noexcept;
+    shared_future<R> share() noexcept;
+
+    // retrieving the value
+    R get();
+
+    // functions to check state
+    bool valid() const noexcept;
+
+    void wait() const;
+    template <class Rep, class Period>
+        future_status
+        wait_for(const chrono::duration<Rep, Period>& rel_time) const;
+    template <class Clock, class Duration>
+        future_status
+        wait_until(const chrono::time_point<Clock, Duration>& abs_time) const;
+};
+
+template <class R>
+class future<R&>
+{
+public:
+    future() noexcept;
+    future(future&&) noexcept;
+    future(const future& rhs) = delete;
+    ~future();
+    future& operator=(const future& rhs) = delete;
+    future& operator=(future&&) noexcept;
+    shared_future<R&> share() noexcept;
+
+    // retrieving the value
+    R& get();
+
+    // functions to check state
+    bool valid() const noexcept;
+
+    void wait() const;
+    template <class Rep, class Period>
+        future_status
+        wait_for(const chrono::duration<Rep, Period>& rel_time) const;
+    template <class Clock, class Duration>
+        future_status
+        wait_until(const chrono::time_point<Clock, Duration>& abs_time) const;
+};
+
+template <>
+class future<void>
+{
+public:
+    future() noexcept;
+    future(future&&) noexcept;
+    future(const future& rhs) = delete;
+    ~future();
+    future& operator=(const future& rhs) = delete;
+    future& operator=(future&&) noexcept;
+    shared_future<void> share() noexcept;
+
+    // retrieving the value
+    void get();
+
+    // functions to check state
+    bool valid() const noexcept;
+
+    void wait() const;
+    template <class Rep, class Period>
+        future_status
+        wait_for(const chrono::duration<Rep, Period>& rel_time) const;
+    template <class Clock, class Duration>
+        future_status
+        wait_until(const chrono::time_point<Clock, Duration>& abs_time) const;
+};
+
+template <class R>
+class shared_future
+{
+public:
+    shared_future() noexcept;
+    shared_future(const shared_future& rhs);
+    shared_future(future<R>&&) noexcept;
+    shared_future(shared_future&& rhs) noexcept;
+    ~shared_future();
+    shared_future& operator=(const shared_future& rhs);
+    shared_future& operator=(shared_future&& rhs) noexcept;
+
+    // retrieving the value
+    const R& get() const;
+
+    // functions to check state
+    bool valid() const noexcept;
+
+    void wait() const;
+    template <class Rep, class Period>
+        future_status
+        wait_for(const chrono::duration<Rep, Period>& rel_time) const;
+    template <class Clock, class Duration>
+        future_status
+        wait_until(const chrono::time_point<Clock, Duration>& abs_time) const;
+};
+
+template <class R>
+class shared_future<R&>
+{
+public:
+    shared_future() noexcept;
+    shared_future(const shared_future& rhs);
+    shared_future(future<R&>&&) noexcept;
+    shared_future(shared_future&& rhs) noexcept;
+    ~shared_future();
+    shared_future& operator=(const shared_future& rhs);
+    shared_future& operator=(shared_future&& rhs) noexcept;
+
+    // retrieving the value
+    R& get() const;
+
+    // functions to check state
+    bool valid() const noexcept;
+
+    void wait() const;
+    template <class Rep, class Period>
+        future_status
+        wait_for(const chrono::duration<Rep, Period>& rel_time) const;
+    template <class Clock, class Duration>
+        future_status
+        wait_until(const chrono::time_point<Clock, Duration>& abs_time) const;
+};
+
+template <>
+class shared_future<void>
+{
+public:
+    shared_future() noexcept;
+    shared_future(const shared_future& rhs);
+    shared_future(future<void>&&) noexcept;
+    shared_future(shared_future&& rhs) noexcept;
+    ~shared_future();
+    shared_future& operator=(const shared_future& rhs);
+    shared_future& operator=(shared_future&& rhs) noexcept;
+
+    // retrieving the value
+    void get() const;
+
+    // functions to check state
+    bool valid() const noexcept;
+
+    void wait() const;
+    template <class Rep, class Period>
+        future_status
+        wait_for(const chrono::duration<Rep, Period>& rel_time) const;
+    template <class Clock, class Duration>
+        future_status
+        wait_until(const chrono::time_point<Clock, Duration>& abs_time) const;
+};
+
+template <class F, class... Args>
+  future<typename result_of<typename decay<F>::type(typename decay<Args>::type...)>::type>
+  async(F&& f, Args&&... args);
+
+template <class F, class... Args>
+  future<typename result_of<typename decay<F>::type(typename decay<Args>::type...)>::type>
+  async(launch policy, F&& f, Args&&... args);
+
+template <class> class packaged_task; // undefined
+
+template <class R, class... ArgTypes>
+class packaged_task<R(ArgTypes...)>
+{
+public:
+    typedef R result_type; // extension
+
+    // construction and destruction
+    packaged_task() noexcept;
+    template <class F>
+        explicit packaged_task(F&& f);
+    template <class F, class Allocator>
+        packaged_task(allocator_arg_t, const Allocator& a, F&& f);
+    ~packaged_task();
+
+    // no copy
+    packaged_task(const packaged_task&) = delete;
+    packaged_task& operator=(const packaged_task&) = delete;
+
+    // move support
+    packaged_task(packaged_task&& other) noexcept;
+    packaged_task& operator=(packaged_task&& other) noexcept;
+    void swap(packaged_task& other) noexcept;
+
+    bool valid() const noexcept;
+
+    // result retrieval
+    future<R> get_future();
+
+    // execution
+    void operator()(ArgTypes... );
+    void make_ready_at_thread_exit(ArgTypes...);
+
+    void reset();
+};
+
+template <class R>
+  void swap(packaged_task<R(ArgTypes...)&, packaged_task<R(ArgTypes...)>&) noexcept;
+
+template <class R, class Alloc> struct uses_allocator<packaged_task<R>, Alloc>;
+
+}  // std
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+
+#  include <__assert>
+#  include <__chrono/duration.h>
+#  include <__chrono/time_point.h>
+#  include <__exception/exception_ptr.h>
+#  include <__memory/addressof.h>
+#  include <__memory/allocator.h>
+#  include <__memory/allocator_arg_t.h>
+#  include <__memory/allocator_destructor.h>
+#  include <__memory/allocator_traits.h>
+#  include <__memory/compressed_pair.h>
+#  include <__memory/pointer_traits.h>
+#  include <__memory/shared_ptr.h>
+#  include <__memory/unique_ptr.h>
+#  include <__memory/uses_allocator.h>
+#  include <__system_error/error_category.h>
+#  include <__system_error/error_code.h>
+#  include <__system_error/error_condition.h>
+#  include <__type_traits/aligned_storage.h>
+#  include <__type_traits/strip_signature.h>
+#  include <__utility/auto_cast.h>
+#  include <__utility/forward.h>
+#  include <__utility/move.h>
+#  include <mutex>
+#  include <new>
+#  include <stdexcept>
+#  include <thread>
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// enum class future_errc
+_LIBCPP_DECLARE_STRONG_ENUM(future_errc){
+    future_already_retrieved = 1, promise_already_satisfied, no_state, broken_promise};
+_LIBCPP_DECLARE_STRONG_ENUM_EPILOG(future_errc)
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS is_error_code_enum<future_errc> : public true_type {};
+
+#  ifdef _LIBCPP_CXX03_LANG
+template <>
+struct _LIBCPP_TEMPLATE_VIS is_error_code_enum<future_errc::__lx> : public true_type {};
+#  endif
+
+// enum class launch
+_LIBCPP_DECLARE_STRONG_ENUM(launch){async = 1, deferred = 2, any = async | deferred};
+_LIBCPP_DECLARE_STRONG_ENUM_EPILOG(launch)
+
+#  ifndef _LIBCPP_CXX03_LANG
+
+typedef underlying_type<launch>::type __launch_underlying_type;
+
+inline _LIBCPP_HIDE_FROM_ABI constexpr launch operator&(launch __x, launch __y) {
+  return static_cast<launch>(static_cast<__launch_underlying_type>(__x) & static_cast<__launch_underlying_type>(__y));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI constexpr launch operator|(launch __x, launch __y) {
+  return static_cast<launch>(static_cast<__launch_underlying_type>(__x) | static_cast<__launch_underlying_type>(__y));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI constexpr launch operator^(launch __x, launch __y) {
+  return static_cast<launch>(static_cast<__launch_underlying_type>(__x) ^ static_cast<__launch_underlying_type>(__y));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI constexpr launch operator~(launch __x) {
+  return static_cast<launch>(~static_cast<__launch_underlying_type>(__x) & 3);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI launch& operator&=(launch& __x, launch __y) {
+  __x = __x & __y;
+  return __x;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI launch& operator|=(launch& __x, launch __y) {
+  __x = __x | __y;
+  return __x;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI launch& operator^=(launch& __x, launch __y) {
+  __x = __x ^ __y;
+  return __x;
+}
+
+#  endif // !_LIBCPP_CXX03_LANG
+
+// enum class future_status
+_LIBCPP_DECLARE_STRONG_ENUM(future_status){ready, timeout, deferred};
+_LIBCPP_DECLARE_STRONG_ENUM_EPILOG(future_status)
+
+_LIBCPP_EXPORTED_FROM_ABI const error_category& future_category() _NOEXCEPT;
+
+inline _LIBCPP_HIDE_FROM_ABI error_code make_error_code(future_errc __e) _NOEXCEPT {
+  return error_code(static_cast<int>(__e), future_category());
+}
+
+inline _LIBCPP_HIDE_FROM_ABI error_condition make_error_condition(future_errc __e) _NOEXCEPT {
+  return error_condition(static_cast<int>(__e), future_category());
+}
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_future_error(future_errc __ev);
+
+class _LIBCPP_EXPORTED_FROM_ABI future_error : public logic_error {
+  error_code __ec_;
+
+  future_error(error_code);
+  friend void __throw_future_error(future_errc);
+  template <class>
+  friend class promise;
+
+public:
+#  if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI explicit future_error(future_errc __ec) : future_error(std::make_error_code(__ec)) {}
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI const error_code& code() const _NOEXCEPT { return __ec_; }
+
+  _LIBCPP_HIDE_FROM_ABI future_error(const future_error&) _NOEXCEPT = default;
+  ~future_error() _NOEXCEPT override;
+};
+
+// Declared above std::future_error
+void __throw_future_error(future_errc __ev) {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw future_error(make_error_code(__ev));
+#  else
+  (void)__ev;
+  _LIBCPP_VERBOSE_ABORT("future_error was thrown in -fno-exceptions mode");
+#  endif
+}
+
+class _LIBCPP_EXPORTED_FROM_ABI __assoc_sub_state : public __shared_count {
+protected:
+  exception_ptr __exception_;
+  mutable mutex __mut_;
+  mutable condition_variable __cv_;
+  unsigned __state_;
+
+  void __on_zero_shared() _NOEXCEPT override;
+  void __sub_wait(unique_lock<mutex>& __lk);
+
+public:
+  enum { __constructed = 1, __future_attached = 2, ready = 4, deferred = 8 };
+
+  _LIBCPP_HIDE_FROM_ABI __assoc_sub_state() : __state_(0) {}
+
+  _LIBCPP_HIDE_FROM_ABI bool __has_value() const { return (__state_ & __constructed) || (__exception_ != nullptr); }
+
+  _LIBCPP_HIDE_FROM_ABI void __attach_future() {
+    lock_guard<mutex> __lk(__mut_);
+    bool __has_future_attached = (__state_ & __future_attached) != 0;
+    if (__has_future_attached)
+      __throw_future_error(future_errc::future_already_retrieved);
+    this->__add_shared();
+    __state_ |= __future_attached;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __set_deferred() { __state_ |= deferred; }
+
+  void __make_ready();
+  _LIBCPP_HIDE_FROM_ABI bool __is_ready() const { return (__state_ & ready) != 0; }
+
+  void set_value();
+  void set_value_at_thread_exit();
+
+  void set_exception(exception_ptr __p);
+  void set_exception_at_thread_exit(exception_ptr __p);
+
+  void copy();
+
+  void wait();
+  template <class _Rep, class _Period>
+  future_status _LIBCPP_HIDE_FROM_ABI wait_for(const chrono::duration<_Rep, _Period>& __rel_time) const;
+  template <class _Clock, class _Duration>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS future_status
+  wait_until(const chrono::time_point<_Clock, _Duration>& __abs_time) const;
+
+  virtual void __execute();
+};
+
+template <class _Clock, class _Duration>
+future_status __assoc_sub_state::wait_until(const chrono::time_point<_Clock, _Duration>& __abs_time) const {
+  unique_lock<mutex> __lk(__mut_);
+  if (__state_ & deferred)
+    return future_status::deferred;
+  while (!(__state_ & ready) && _Clock::now() < __abs_time)
+    __cv_.wait_until(__lk, __abs_time);
+  if (__state_ & ready)
+    return future_status::ready;
+  return future_status::timeout;
+}
+
+template <class _Rep, class _Period>
+inline future_status __assoc_sub_state::wait_for(const chrono::duration<_Rep, _Period>& __rel_time) const {
+  return wait_until(chrono::steady_clock::now() + __rel_time);
+}
+
+template <class _Rp>
+class _LIBCPP_HIDDEN __assoc_state : public __assoc_sub_state {
+  typedef __assoc_sub_state base;
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  typedef typename aligned_storage<sizeof(_Rp), _LIBCPP_ALIGNOF(_Rp)>::type _Up;
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+
+protected:
+  _Up __value_;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared() _NOEXCEPT override;
+
+public:
+  template <class _Arg>
+  _LIBCPP_HIDE_FROM_ABI void set_value(_Arg&& __arg);
+
+  template <class _Arg>
+  _LIBCPP_HIDE_FROM_ABI void set_value_at_thread_exit(_Arg&& __arg);
+
+  _LIBCPP_HIDE_FROM_ABI _Rp move();
+  _LIBCPP_HIDE_FROM_ABI __add_lvalue_reference_t<_Rp> copy();
+};
+
+template <class _Rp>
+void __assoc_state<_Rp>::__on_zero_shared() _NOEXCEPT {
+  if (this->__state_ & base::__constructed)
+    reinterpret_cast<_Rp*>(&__value_)->~_Rp();
+  delete this;
+}
+
+template <class _Rp>
+template <class _Arg>
+void __assoc_state<_Rp>::set_value(_Arg&& __arg) {
+  unique_lock<mutex> __lk(this->__mut_);
+  if (this->__has_value())
+    __throw_future_error(future_errc::promise_already_satisfied);
+  ::new ((void*)&__value_) _Rp(std::forward<_Arg>(__arg));
+  this->__state_ |= base::__constructed | base::ready;
+  __cv_.notify_all();
+}
+
+template <class _Rp>
+template <class _Arg>
+void __assoc_state<_Rp>::set_value_at_thread_exit(_Arg&& __arg) {
+  unique_lock<mutex> __lk(this->__mut_);
+  if (this->__has_value())
+    __throw_future_error(future_errc::promise_already_satisfied);
+  ::new ((void*)&__value_) _Rp(std::forward<_Arg>(__arg));
+  this->__state_ |= base::__constructed;
+  __thread_local_data()->__make_ready_at_thread_exit(this);
+}
+
+template <class _Rp>
+_Rp __assoc_state<_Rp>::move() {
+  unique_lock<mutex> __lk(this->__mut_);
+  this->__sub_wait(__lk);
+  if (this->__exception_ != nullptr)
+    std::rethrow_exception(this->__exception_);
+  return std::move(*reinterpret_cast<_Rp*>(&__value_));
+}
+
+template <class _Rp>
+__add_lvalue_reference_t<_Rp> __assoc_state<_Rp>::copy() {
+  unique_lock<mutex> __lk(this->__mut_);
+  this->__sub_wait(__lk);
+  if (this->__exception_ != nullptr)
+    std::rethrow_exception(this->__exception_);
+  return *reinterpret_cast<_Rp*>(&__value_);
+}
+
+template <class _Rp>
+class __assoc_state<_Rp&> : public __assoc_sub_state {
+  typedef __assoc_sub_state base;
+  typedef _Rp* _Up;
+
+protected:
+  _Up __value_;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared() _NOEXCEPT override;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI void set_value(_Rp& __arg);
+  _LIBCPP_HIDE_FROM_ABI void set_value_at_thread_exit(_Rp& __arg);
+
+  _LIBCPP_HIDE_FROM_ABI _Rp& copy();
+};
+
+template <class _Rp>
+void __assoc_state<_Rp&>::__on_zero_shared() _NOEXCEPT {
+  delete this;
+}
+
+template <class _Rp>
+void __assoc_state<_Rp&>::set_value(_Rp& __arg) {
+  unique_lock<mutex> __lk(this->__mut_);
+  if (this->__has_value())
+    __throw_future_error(future_errc::promise_already_satisfied);
+  __value_ = std::addressof(__arg);
+  this->__state_ |= base::__constructed | base::ready;
+  __cv_.notify_all();
+}
+
+template <class _Rp>
+void __assoc_state<_Rp&>::set_value_at_thread_exit(_Rp& __arg) {
+  unique_lock<mutex> __lk(this->__mut_);
+  if (this->__has_value())
+    __throw_future_error(future_errc::promise_already_satisfied);
+  __value_ = std::addressof(__arg);
+  this->__state_ |= base::__constructed;
+  __thread_local_data()->__make_ready_at_thread_exit(this);
+}
+
+template <class _Rp>
+_Rp& __assoc_state<_Rp&>::copy() {
+  unique_lock<mutex> __lk(this->__mut_);
+  this->__sub_wait(__lk);
+  if (this->__exception_ != nullptr)
+    std::rethrow_exception(this->__exception_);
+  return *__value_;
+}
+
+template <class _Rp, class _Alloc>
+class __assoc_state_alloc : public __assoc_state<_Rp> {
+  typedef __assoc_state<_Rp> base;
+  _Alloc __alloc_;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __on_zero_shared() _NOEXCEPT;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __assoc_state_alloc(const _Alloc& __a) : __alloc_(__a) {}
+};
+
+template <class _Rp, class _Alloc>
+void __assoc_state_alloc<_Rp, _Alloc>::__on_zero_shared() _NOEXCEPT {
+  if (this->__state_ & base::__constructed)
+    reinterpret_cast<_Rp*>(std::addressof(this->__value_))->~_Rp();
+  typedef typename __allocator_traits_rebind<_Alloc, __assoc_state_alloc>::type _Al;
+  typedef allocator_traits<_Al> _ATraits;
+  typedef pointer_traits<typename _ATraits::pointer> _PTraits;
+  _Al __a(__alloc_);
+  this->~__assoc_state_alloc();
+  __a.deallocate(_PTraits::pointer_to(*this), 1);
+}
+
+template <class _Rp, class _Alloc>
+class __assoc_state_alloc<_Rp&, _Alloc> : public __assoc_state<_Rp&> {
+  typedef __assoc_state<_Rp&> base;
+  _Alloc __alloc_;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __on_zero_shared() _NOEXCEPT;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __assoc_state_alloc(const _Alloc& __a) : __alloc_(__a) {}
+};
+
+template <class _Rp, class _Alloc>
+void __assoc_state_alloc<_Rp&, _Alloc>::__on_zero_shared() _NOEXCEPT {
+  typedef typename __allocator_traits_rebind<_Alloc, __assoc_state_alloc>::type _Al;
+  typedef allocator_traits<_Al> _ATraits;
+  typedef pointer_traits<typename _ATraits::pointer> _PTraits;
+  _Al __a(__alloc_);
+  this->~__assoc_state_alloc();
+  __a.deallocate(_PTraits::pointer_to(*this), 1);
+}
+
+template <class _Alloc>
+class __assoc_sub_state_alloc : public __assoc_sub_state {
+  typedef __assoc_sub_state base;
+  _Alloc __alloc_;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared() _NOEXCEPT override;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __assoc_sub_state_alloc(const _Alloc& __a) : __alloc_(__a) {}
+};
+
+template <class _Alloc>
+void __assoc_sub_state_alloc<_Alloc>::__on_zero_shared() _NOEXCEPT {
+  typedef typename __allocator_traits_rebind<_Alloc, __assoc_sub_state_alloc>::type _Al;
+  typedef allocator_traits<_Al> _ATraits;
+  typedef pointer_traits<typename _ATraits::pointer> _PTraits;
+  _Al __a(__alloc_);
+  this->~__assoc_sub_state_alloc();
+  __a.deallocate(_PTraits::pointer_to(*this), 1);
+}
+
+template <class _Rp, class _Fp>
+class __deferred_assoc_state : public __assoc_state<_Rp> {
+  typedef __assoc_state<_Rp> base;
+
+  _Fp __func_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __deferred_assoc_state(_Fp&& __f);
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __execute();
+};
+
+template <class _Rp, class _Fp>
+inline __deferred_assoc_state<_Rp, _Fp>::__deferred_assoc_state(_Fp&& __f) : __func_(std::forward<_Fp>(__f)) {
+  this->__set_deferred();
+}
+
+template <class _Rp, class _Fp>
+void __deferred_assoc_state<_Rp, _Fp>::__execute() {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    this->set_value(__func_());
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->set_exception(current_exception());
+  }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+}
+
+template <class _Fp>
+class __deferred_assoc_state<void, _Fp> : public __assoc_sub_state {
+  typedef __assoc_sub_state base;
+
+  _Fp __func_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __deferred_assoc_state(_Fp&& __f);
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __execute() override;
+};
+
+template <class _Fp>
+inline __deferred_assoc_state<void, _Fp>::__deferred_assoc_state(_Fp&& __f) : __func_(std::forward<_Fp>(__f)) {
+  this->__set_deferred();
+}
+
+template <class _Fp>
+void __deferred_assoc_state<void, _Fp>::__execute() {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __func_();
+    this->set_value();
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->set_exception(current_exception());
+  }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+}
+
+template <class _Rp, class _Fp>
+class __async_assoc_state : public __assoc_state<_Rp> {
+  typedef __assoc_state<_Rp> base;
+
+  _Fp __func_;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __on_zero_shared() _NOEXCEPT;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __async_assoc_state(_Fp&& __f);
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __execute();
+};
+
+template <class _Rp, class _Fp>
+inline __async_assoc_state<_Rp, _Fp>::__async_assoc_state(_Fp&& __f) : __func_(std::forward<_Fp>(__f)) {}
+
+template <class _Rp, class _Fp>
+void __async_assoc_state<_Rp, _Fp>::__execute() {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    this->set_value(__func_());
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->set_exception(current_exception());
+  }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+}
+
+template <class _Rp, class _Fp>
+void __async_assoc_state<_Rp, _Fp>::__on_zero_shared() _NOEXCEPT {
+  this->wait();
+  base::__on_zero_shared();
+}
+
+template <class _Fp>
+class __async_assoc_state<void, _Fp> : public __assoc_sub_state {
+  typedef __assoc_sub_state base;
+
+  _Fp __func_;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared() _NOEXCEPT override;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __async_assoc_state(_Fp&& __f);
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __execute() override;
+};
+
+template <class _Fp>
+inline __async_assoc_state<void, _Fp>::__async_assoc_state(_Fp&& __f) : __func_(std::forward<_Fp>(__f)) {}
+
+template <class _Fp>
+void __async_assoc_state<void, _Fp>::__execute() {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __func_();
+    this->set_value();
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    this->set_exception(current_exception());
+  }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+}
+
+template <class _Fp>
+void __async_assoc_state<void, _Fp>::__on_zero_shared() _NOEXCEPT {
+  this->wait();
+  base::__on_zero_shared();
+}
+
+template <class _Rp>
+class _LIBCPP_TEMPLATE_VIS promise;
+template <class _Rp>
+class _LIBCPP_TEMPLATE_VIS shared_future;
+
+// future
+
+template <class _Rp>
+class _LIBCPP_TEMPLATE_VIS future;
+
+template <class _Rp, class _Fp>
+_LIBCPP_HIDE_FROM_ABI future<_Rp> __make_deferred_assoc_state(_Fp&& __f);
+
+template <class _Rp, class _Fp>
+_LIBCPP_HIDE_FROM_ABI future<_Rp> __make_async_assoc_state(_Fp&& __f);
+
+template <class _Rp>
+class _LIBCPP_TEMPLATE_VIS future {
+  __assoc_state<_Rp>* __state_;
+
+  explicit _LIBCPP_HIDE_FROM_ABI future(__assoc_state<_Rp>* __state);
+
+  template <class>
+  friend class promise;
+  template <class>
+  friend class shared_future;
+
+  template <class _R1, class _Fp>
+  friend future<_R1> __make_deferred_assoc_state(_Fp&& __f);
+  template <class _R1, class _Fp>
+  friend future<_R1> __make_async_assoc_state(_Fp&& __f);
+
+public:
+  _LIBCPP_HIDE_FROM_ABI future() _NOEXCEPT : __state_(nullptr) {}
+  _LIBCPP_HIDE_FROM_ABI future(future&& __rhs) _NOEXCEPT : __state_(__rhs.__state_) { __rhs.__state_ = nullptr; }
+  future(const future&)            = delete;
+  future& operator=(const future&) = delete;
+  _LIBCPP_HIDE_FROM_ABI future& operator=(future&& __rhs) _NOEXCEPT {
+    future(std::move(__rhs)).swap(*this);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~future();
+  _LIBCPP_HIDE_FROM_ABI shared_future<_Rp> share() _NOEXCEPT;
+
+  // retrieving the value
+  _LIBCPP_HIDE_FROM_ABI _Rp get();
+
+  _LIBCPP_HIDE_FROM_ABI void swap(future& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
+
+  // functions to check state
+  _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
+
+  _LIBCPP_HIDE_FROM_ABI void wait() const { __state_->wait(); }
+  template <class _Rep, class _Period>
+  _LIBCPP_HIDE_FROM_ABI future_status wait_for(const chrono::duration<_Rep, _Period>& __rel_time) const {
+    return __state_->wait_for(__rel_time);
+  }
+  template <class _Clock, class _Duration>
+  _LIBCPP_HIDE_FROM_ABI future_status wait_until(const chrono::time_point<_Clock, _Duration>& __abs_time) const {
+    return __state_->wait_until(__abs_time);
+  }
+};
+
+template <class _Rp>
+future<_Rp>::future(__assoc_state<_Rp>* __state) : __state_(__state) {
+  __state_->__attach_future();
+}
+
+struct __release_shared_count {
+  _LIBCPP_HIDE_FROM_ABI void operator()(__shared_count* __p) { __p->__release_shared(); }
+};
+
+template <class _Rp>
+future<_Rp>::~future() {
+  if (__state_)
+    __state_->__release_shared();
+}
+
+template <class _Rp>
+_Rp future<_Rp>::get() {
+  unique_ptr<__shared_count, __release_shared_count> __guard(__state_);
+  __assoc_state<_Rp>* __s = __state_;
+  __state_                = nullptr;
+  return __s->move();
+}
+
+template <class _Rp>
+class _LIBCPP_TEMPLATE_VIS future<_Rp&> {
+  __assoc_state<_Rp&>* __state_;
+
+  explicit _LIBCPP_HIDE_FROM_ABI future(__assoc_state<_Rp&>* __state);
+
+  template <class>
+  friend class promise;
+  template <class>
+  friend class shared_future;
+
+  template <class _R1, class _Fp>
+  friend future<_R1> __make_deferred_assoc_state(_Fp&& __f);
+  template <class _R1, class _Fp>
+  friend future<_R1> __make_async_assoc_state(_Fp&& __f);
+
+public:
+  _LIBCPP_HIDE_FROM_ABI future() _NOEXCEPT : __state_(nullptr) {}
+  _LIBCPP_HIDE_FROM_ABI future(future&& __rhs) _NOEXCEPT : __state_(__rhs.__state_) { __rhs.__state_ = nullptr; }
+  future(const future&)            = delete;
+  future& operator=(const future&) = delete;
+  _LIBCPP_HIDE_FROM_ABI future& operator=(future&& __rhs) _NOEXCEPT {
+    future(std::move(__rhs)).swap(*this);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~future();
+  _LIBCPP_HIDE_FROM_ABI shared_future<_Rp&> share() _NOEXCEPT;
+
+  // retrieving the value
+  _LIBCPP_HIDE_FROM_ABI _Rp& get();
+
+  _LIBCPP_HIDE_FROM_ABI void swap(future& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
+
+  // functions to check state
+  _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
+
+  _LIBCPP_HIDE_FROM_ABI void wait() const { __state_->wait(); }
+  template <class _Rep, class _Period>
+  _LIBCPP_HIDE_FROM_ABI future_status wait_for(const chrono::duration<_Rep, _Period>& __rel_time) const {
+    return __state_->wait_for(__rel_time);
+  }
+  template <class _Clock, class _Duration>
+  _LIBCPP_HIDE_FROM_ABI future_status wait_until(const chrono::time_point<_Clock, _Duration>& __abs_time) const {
+    return __state_->wait_until(__abs_time);
+  }
+};
+
+template <class _Rp>
+future<_Rp&>::future(__assoc_state<_Rp&>* __state) : __state_(__state) {
+  __state_->__attach_future();
+}
+
+template <class _Rp>
+future<_Rp&>::~future() {
+  if (__state_)
+    __state_->__release_shared();
+}
+
+template <class _Rp>
+_Rp& future<_Rp&>::get() {
+  unique_ptr<__shared_count, __release_shared_count> __guard(__state_);
+  __assoc_state<_Rp&>* __s = __state_;
+  __state_                 = nullptr;
+  return __s->copy();
+}
+
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI future<void> {
+  __assoc_sub_state* __state_;
+
+  explicit future(__assoc_sub_state* __state);
+
+  template <class>
+  friend class promise;
+  template <class>
+  friend class shared_future;
+
+  template <class _R1, class _Fp>
+  friend future<_R1> __make_deferred_assoc_state(_Fp&& __f);
+  template <class _R1, class _Fp>
+  friend future<_R1> __make_async_assoc_state(_Fp&& __f);
+
+public:
+  _LIBCPP_HIDE_FROM_ABI future() _NOEXCEPT : __state_(nullptr) {}
+  _LIBCPP_HIDE_FROM_ABI future(future&& __rhs) _NOEXCEPT : __state_(__rhs.__state_) { __rhs.__state_ = nullptr; }
+  future(const future&)            = delete;
+  future& operator=(const future&) = delete;
+  _LIBCPP_HIDE_FROM_ABI future& operator=(future&& __rhs) _NOEXCEPT {
+    future(std::move(__rhs)).swap(*this);
+    return *this;
+  }
+
+  ~future();
+  _LIBCPP_HIDE_FROM_ABI shared_future<void> share() _NOEXCEPT;
+
+  // retrieving the value
+  void get();
+
+  _LIBCPP_HIDE_FROM_ABI void swap(future& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
+
+  // functions to check state
+  _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
+
+  _LIBCPP_HIDE_FROM_ABI void wait() const { __state_->wait(); }
+  template <class _Rep, class _Period>
+  _LIBCPP_HIDE_FROM_ABI future_status wait_for(const chrono::duration<_Rep, _Period>& __rel_time) const {
+    return __state_->wait_for(__rel_time);
+  }
+  template <class _Clock, class _Duration>
+  _LIBCPP_HIDE_FROM_ABI future_status wait_until(const chrono::time_point<_Clock, _Duration>& __abs_time) const {
+    return __state_->wait_until(__abs_time);
+  }
+};
+
+template <class _Rp>
+inline _LIBCPP_HIDE_FROM_ABI void swap(future<_Rp>& __x, future<_Rp>& __y) _NOEXCEPT {
+  __x.swap(__y);
+}
+
+// promise<R>
+
+template <class _Callable>
+class packaged_task;
+
+template <class _Rp>
+class _LIBCPP_TEMPLATE_VIS promise {
+  __assoc_state<_Rp>* __state_;
+
+  _LIBCPP_HIDE_FROM_ABI explicit promise(nullptr_t) _NOEXCEPT : __state_(nullptr) {}
+
+  template <class>
+  friend class packaged_task;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI promise();
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI promise(allocator_arg_t, const _Alloc& __a);
+  _LIBCPP_HIDE_FROM_ABI promise(promise&& __rhs) _NOEXCEPT : __state_(__rhs.__state_) { __rhs.__state_ = nullptr; }
+  promise(const promise& __rhs) = delete;
+  _LIBCPP_HIDE_FROM_ABI ~promise();
+
+  // assignment
+  _LIBCPP_HIDE_FROM_ABI promise& operator=(promise&& __rhs) _NOEXCEPT {
+    promise(std::move(__rhs)).swap(*this);
+    return *this;
+  }
+  promise& operator=(const promise& __rhs) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI void swap(promise& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
+
+  // retrieving the result
+  _LIBCPP_HIDE_FROM_ABI future<_Rp> get_future();
+
+  // setting the result
+  _LIBCPP_HIDE_FROM_ABI void set_value(const _Rp& __r);
+  _LIBCPP_HIDE_FROM_ABI void set_value(_Rp&& __r);
+  _LIBCPP_HIDE_FROM_ABI void set_exception(exception_ptr __p);
+
+  // setting the result with deferred notification
+  _LIBCPP_HIDE_FROM_ABI void set_value_at_thread_exit(const _Rp& __r);
+  _LIBCPP_HIDE_FROM_ABI void set_value_at_thread_exit(_Rp&& __r);
+  _LIBCPP_HIDE_FROM_ABI void set_exception_at_thread_exit(exception_ptr __p);
+};
+
+template <class _Rp>
+promise<_Rp>::promise() : __state_(new __assoc_state<_Rp>) {}
+
+template <class _Rp>
+template <class _Alloc>
+promise<_Rp>::promise(allocator_arg_t, const _Alloc& __a0) {
+  typedef __assoc_state_alloc<_Rp, _Alloc> _State;
+  typedef typename __allocator_traits_rebind<_Alloc, _State>::type _A2;
+  typedef __allocator_destructor<_A2> _D2;
+  _A2 __a(__a0);
+  unique_ptr<_State, _D2> __hold(__a.allocate(1), _D2(__a, 1));
+  ::new ((void*)std::addressof(*__hold.get())) _State(__a0);
+  __state_ = std::addressof(*__hold.release());
+}
+
+template <class _Rp>
+promise<_Rp>::~promise() {
+  if (__state_) {
+    if (!__state_->__has_value() && __state_->use_count() > 1)
+      __state_->set_exception(make_exception_ptr(future_error(make_error_code(future_errc::broken_promise))));
+    __state_->__release_shared();
+  }
+}
+
+template <class _Rp>
+future<_Rp> promise<_Rp>::get_future() {
+  if (__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  return future<_Rp>(__state_);
+}
+
+template <class _Rp>
+void promise<_Rp>::set_value(const _Rp& __r) {
+  if (__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  __state_->set_value(__r);
+}
+
+template <class _Rp>
+void promise<_Rp>::set_value(_Rp&& __r) {
+  if (__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  __state_->set_value(std::move(__r));
+}
+
+template <class _Rp>
+void promise<_Rp>::set_exception(exception_ptr __p) {
+  _LIBCPP_ASSERT_NON_NULL(__p != nullptr, "promise::set_exception: received nullptr");
+  if (__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  __state_->set_exception(__p);
+}
+
+template <class _Rp>
+void promise<_Rp>::set_value_at_thread_exit(const _Rp& __r) {
+  if (__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  __state_->set_value_at_thread_exit(__r);
+}
+
+template <class _Rp>
+void promise<_Rp>::set_value_at_thread_exit(_Rp&& __r) {
+  if (__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  __state_->set_value_at_thread_exit(std::move(__r));
+}
+
+template <class _Rp>
+void promise<_Rp>::set_exception_at_thread_exit(exception_ptr __p) {
+  _LIBCPP_ASSERT_NON_NULL(__p != nullptr, "promise::set_exception_at_thread_exit: received nullptr");
+  if (__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  __state_->set_exception_at_thread_exit(__p);
+}
+
+// promise<R&>
+
+template <class _Rp>
+class _LIBCPP_TEMPLATE_VIS promise<_Rp&> {
+  __assoc_state<_Rp&>* __state_;
+
+  _LIBCPP_HIDE_FROM_ABI explicit promise(nullptr_t) _NOEXCEPT : __state_(nullptr) {}
+
+  template <class>
+  friend class packaged_task;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI promise();
+  template <class _Allocator>
+  _LIBCPP_HIDE_FROM_ABI promise(allocator_arg_t, const _Allocator& __a);
+  _LIBCPP_HIDE_FROM_ABI promise(promise&& __rhs) _NOEXCEPT : __state_(__rhs.__state_) { __rhs.__state_ = nullptr; }
+  promise(const promise& __rhs) = delete;
+  _LIBCPP_HIDE_FROM_ABI ~promise();
+
+  // assignment
+  _LIBCPP_HIDE_FROM_ABI promise& operator=(promise&& __rhs) _NOEXCEPT {
+    promise(std::move(__rhs)).swap(*this);
+    return *this;
+  }
+  promise& operator=(const promise& __rhs) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI void swap(promise& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
+
+  // retrieving the result
+  _LIBCPP_HIDE_FROM_ABI future<_Rp&> get_future();
+
+  // setting the result
+  _LIBCPP_HIDE_FROM_ABI void set_value(_Rp& __r);
+  _LIBCPP_HIDE_FROM_ABI void set_exception(exception_ptr __p);
+
+  // setting the result with deferred notification
+  _LIBCPP_HIDE_FROM_ABI void set_value_at_thread_exit(_Rp&);
+  _LIBCPP_HIDE_FROM_ABI void set_exception_at_thread_exit(exception_ptr __p);
+};
+
+template <class _Rp>
+promise<_Rp&>::promise() : __state_(new __assoc_state<_Rp&>) {}
+
+template <class _Rp>
+template <class _Alloc>
+promise<_Rp&>::promise(allocator_arg_t, const _Alloc& __a0) {
+  typedef __assoc_state_alloc<_Rp&, _Alloc> _State;
+  typedef typename __allocator_traits_rebind<_Alloc, _State>::type _A2;
+  typedef __allocator_destructor<_A2> _D2;
+  _A2 __a(__a0);
+  unique_ptr<_State, _D2> __hold(__a.allocate(1), _D2(__a, 1));
+  ::new ((void*)std::addressof(*__hold.get())) _State(__a0);
+  __state_ = std::addressof(*__hold.release());
+}
+
+template <class _Rp>
+promise<_Rp&>::~promise() {
+  if (__state_) {
+    if (!__state_->__has_value() && __state_->use_count() > 1)
+      __state_->set_exception(make_exception_ptr(future_error(make_error_code(future_errc::broken_promise))));
+    __state_->__release_shared();
+  }
+}
+
+template <class _Rp>
+future<_Rp&> promise<_Rp&>::get_future() {
+  if (__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  return future<_Rp&>(__state_);
+}
+
+template <class _Rp>
+void promise<_Rp&>::set_value(_Rp& __r) {
+  if (__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  __state_->set_value(__r);
+}
+
+template <class _Rp>
+void promise<_Rp&>::set_exception(exception_ptr __p) {
+  _LIBCPP_ASSERT_NON_NULL(__p != nullptr, "promise::set_exception: received nullptr");
+  if (__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  __state_->set_exception(__p);
+}
+
+template <class _Rp>
+void promise<_Rp&>::set_value_at_thread_exit(_Rp& __r) {
+  if (__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  __state_->set_value_at_thread_exit(__r);
+}
+
+template <class _Rp>
+void promise<_Rp&>::set_exception_at_thread_exit(exception_ptr __p) {
+  _LIBCPP_ASSERT_NON_NULL(__p != nullptr, "promise::set_exception_at_thread_exit: received nullptr");
+  if (__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  __state_->set_exception_at_thread_exit(__p);
+}
+
+// promise<void>
+
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI promise<void> {
+  __assoc_sub_state* __state_;
+
+  _LIBCPP_HIDE_FROM_ABI explicit promise(nullptr_t) _NOEXCEPT : __state_(nullptr) {}
+
+  template <class>
+  friend class packaged_task;
+
+public:
+  promise();
+  template <class _Allocator>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS promise(allocator_arg_t, const _Allocator& __a);
+  _LIBCPP_HIDE_FROM_ABI promise(promise&& __rhs) _NOEXCEPT : __state_(__rhs.__state_) { __rhs.__state_ = nullptr; }
+  promise(const promise& __rhs) = delete;
+  ~promise();
+
+  // assignment
+  _LIBCPP_HIDE_FROM_ABI promise& operator=(promise&& __rhs) _NOEXCEPT {
+    promise(std::move(__rhs)).swap(*this);
+    return *this;
+  }
+  promise& operator=(const promise& __rhs) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI void swap(promise& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
+
+  // retrieving the result
+  future<void> get_future();
+
+  // setting the result
+  void set_value();
+  void set_exception(exception_ptr __p);
+
+  // setting the result with deferred notification
+  void set_value_at_thread_exit();
+  void set_exception_at_thread_exit(exception_ptr __p);
+};
+
+template <class _Alloc>
+promise<void>::promise(allocator_arg_t, const _Alloc& __a0) {
+  typedef __assoc_sub_state_alloc<_Alloc> _State;
+  typedef typename __allocator_traits_rebind<_Alloc, _State>::type _A2;
+  typedef __allocator_destructor<_A2> _D2;
+  _A2 __a(__a0);
+  unique_ptr<_State, _D2> __hold(__a.allocate(1), _D2(__a, 1));
+  ::new ((void*)std::addressof(*__hold.get())) _State(__a0);
+  __state_ = std::addressof(*__hold.release());
+}
+
+template <class _Rp>
+inline _LIBCPP_HIDE_FROM_ABI void swap(promise<_Rp>& __x, promise<_Rp>& __y) _NOEXCEPT {
+  __x.swap(__y);
+}
+
+template <class _Rp, class _Alloc>
+struct _LIBCPP_TEMPLATE_VIS uses_allocator<promise<_Rp>, _Alloc> : public true_type {};
+
+// packaged_task
+
+template <class _Fp>
+class __packaged_task_base;
+
+template <class _Rp, class... _ArgTypes>
+class __packaged_task_base<_Rp(_ArgTypes...)> {
+public:
+  _LIBCPP_HIDE_FROM_ABI __packaged_task_base() {}
+  __packaged_task_base(const __packaged_task_base&)            = delete;
+  __packaged_task_base& operator=(const __packaged_task_base&) = delete;
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL
+  virtual ~__packaged_task_base() {}
+  virtual void __move_to(__packaged_task_base*) _NOEXCEPT = 0;
+  virtual void destroy()                                  = 0;
+  virtual void destroy_deallocate()                       = 0;
+  virtual _Rp operator()(_ArgTypes&&...)                  = 0;
+};
+
+template <class _FD, class _Alloc, class _FB>
+class __packaged_task_func;
+
+template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
+class __packaged_task_func<_Fp, _Alloc, _Rp(_ArgTypes...)> : public __packaged_task_base<_Rp(_ArgTypes...)> {
+  __compressed_pair<_Fp, _Alloc> __f_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __packaged_task_func(const _Fp& __f) : __f_(__f, __default_init_tag()) {}
+  _LIBCPP_HIDE_FROM_ABI explicit __packaged_task_func(_Fp&& __f) : __f_(std::move(__f), __default_init_tag()) {}
+  _LIBCPP_HIDE_FROM_ABI __packaged_task_func(const _Fp& __f, const _Alloc& __a) : __f_(__f, __a) {}
+  _LIBCPP_HIDE_FROM_ABI __packaged_task_func(_Fp&& __f, const _Alloc& __a) : __f_(std::move(__f), __a) {}
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __move_to(__packaged_task_base<_Rp(_ArgTypes...)>*) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy();
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy_deallocate();
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual _Rp operator()(_ArgTypes&&... __args);
+};
+
+template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
+void __packaged_task_func<_Fp, _Alloc, _Rp(_ArgTypes...)>::__move_to(
+    __packaged_task_base<_Rp(_ArgTypes...)>* __p) _NOEXCEPT {
+  ::new ((void*)__p) __packaged_task_func(std::move(__f_.first()), std::move(__f_.second()));
+}
+
+template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
+void __packaged_task_func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy() {
+  __f_.~__compressed_pair<_Fp, _Alloc>();
+}
+
+template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
+void __packaged_task_func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy_deallocate() {
+  typedef typename __allocator_traits_rebind<_Alloc, __packaged_task_func>::type _Ap;
+  typedef allocator_traits<_Ap> _ATraits;
+  typedef pointer_traits<typename _ATraits::pointer> _PTraits;
+  _Ap __a(__f_.second());
+  __f_.~__compressed_pair<_Fp, _Alloc>();
+  __a.deallocate(_PTraits::pointer_to(*this), 1);
+}
+
+template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
+_Rp __packaged_task_func<_Fp, _Alloc, _Rp(_ArgTypes...)>::operator()(_ArgTypes&&... __arg) {
+  return std::__invoke(__f_.first(), std::forward<_ArgTypes>(__arg)...);
+}
+
+template <class _Callable>
+class __packaged_task_function;
+
+template <class _Rp, class... _ArgTypes>
+class __packaged_task_function<_Rp(_ArgTypes...)> {
+  typedef __packaged_task_base<_Rp(_ArgTypes...)> __base;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI __base* __get_buf() { return (__base*)&__buf_; }
+
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  typename aligned_storage<3 * sizeof(void*)>::type __buf_;
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  __base* __f_;
+
+public:
+  typedef _Rp result_type;
+
+  // construct/copy/destroy:
+  _LIBCPP_HIDE_FROM_ABI __packaged_task_function() _NOEXCEPT : __f_(nullptr) {}
+  template <class _Fp>
+  _LIBCPP_HIDE_FROM_ABI __packaged_task_function(_Fp&& __f);
+  template <class _Fp, class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI __packaged_task_function(allocator_arg_t, const _Alloc& __a, _Fp&& __f);
+
+  _LIBCPP_HIDE_FROM_ABI __packaged_task_function(__packaged_task_function&&) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI __packaged_task_function& operator=(__packaged_task_function&&) _NOEXCEPT;
+
+  __packaged_task_function(const __packaged_task_function&)            = delete;
+  __packaged_task_function& operator=(const __packaged_task_function&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI ~__packaged_task_function();
+
+  _LIBCPP_HIDE_FROM_ABI void swap(__packaged_task_function&) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes...) const;
+};
+
+template <class _Rp, class... _ArgTypes>
+__packaged_task_function<_Rp(_ArgTypes...)>::__packaged_task_function(__packaged_task_function&& __f) _NOEXCEPT {
+  if (__f.__f_ == nullptr)
+    __f_ = nullptr;
+  else if (__f.__f_ == __f.__get_buf()) {
+    __f.__f_->__move_to(__get_buf());
+    __f_ = (__base*)&__buf_;
+  } else {
+    __f_     = __f.__f_;
+    __f.__f_ = nullptr;
+  }
+}
+
+template <class _Rp, class... _ArgTypes>
+template <class _Fp>
+__packaged_task_function<_Rp(_ArgTypes...)>::__packaged_task_function(_Fp&& __f) : __f_(nullptr) {
+  typedef __libcpp_remove_reference_t<__decay_t<_Fp> > _FR;
+  typedef __packaged_task_func<_FR, allocator<_FR>, _Rp(_ArgTypes...)> _FF;
+  if (sizeof(_FF) <= sizeof(__buf_)) {
+    ::new ((void*)&__buf_) _FF(std::forward<_Fp>(__f));
+    __f_ = (__base*)&__buf_;
+  } else {
+    typedef allocator<_FF> _Ap;
+    _Ap __a;
+    typedef __allocator_destructor<_Ap> _Dp;
+    unique_ptr<__base, _Dp> __hold(__a.allocate(1), _Dp(__a, 1));
+    ::new ((void*)__hold.get()) _FF(std::forward<_Fp>(__f), allocator<_FR>(__a));
+    __f_ = __hold.release();
+  }
+}
+
+template <class _Rp, class... _ArgTypes>
+template <class _Fp, class _Alloc>
+__packaged_task_function<_Rp(_ArgTypes...)>::__packaged_task_function(allocator_arg_t, const _Alloc& __a0, _Fp&& __f)
+    : __f_(nullptr) {
+  typedef __libcpp_remove_reference_t<__decay_t<_Fp> > _FR;
+  typedef __packaged_task_func<_FR, _Alloc, _Rp(_ArgTypes...)> _FF;
+  if (sizeof(_FF) <= sizeof(__buf_)) {
+    __f_ = (__base*)&__buf_;
+    ::new ((void*)__f_) _FF(std::forward<_Fp>(__f));
+  } else {
+    typedef typename __allocator_traits_rebind<_Alloc, _FF>::type _Ap;
+    _Ap __a(__a0);
+    typedef __allocator_destructor<_Ap> _Dp;
+    unique_ptr<__base, _Dp> __hold(__a.allocate(1), _Dp(__a, 1));
+    ::new ((void*)std::addressof(*__hold.get())) _FF(std::forward<_Fp>(__f), _Alloc(__a));
+    __f_ = std::addressof(*__hold.release());
+  }
+}
+
+template <class _Rp, class... _ArgTypes>
+__packaged_task_function<_Rp(_ArgTypes...)>&
+__packaged_task_function<_Rp(_ArgTypes...)>::operator=(__packaged_task_function&& __f) _NOEXCEPT {
+  if (__f_ == __get_buf())
+    __f_->destroy();
+  else if (__f_)
+    __f_->destroy_deallocate();
+  __f_ = nullptr;
+  if (__f.__f_ == nullptr)
+    __f_ = nullptr;
+  else if (__f.__f_ == __f.__get_buf()) {
+    __f.__f_->__move_to(__get_buf());
+    __f_ = __get_buf();
+  } else {
+    __f_     = __f.__f_;
+    __f.__f_ = nullptr;
+  }
+  return *this;
+}
+
+template <class _Rp, class... _ArgTypes>
+__packaged_task_function<_Rp(_ArgTypes...)>::~__packaged_task_function() {
+  if (__f_ == __get_buf())
+    __f_->destroy();
+  else if (__f_)
+    __f_->destroy_deallocate();
+}
+
+template <class _Rp, class... _ArgTypes>
+_LIBCPP_NO_CFI void __packaged_task_function<_Rp(_ArgTypes...)>::swap(__packaged_task_function& __f) _NOEXCEPT {
+  if (__f_ == (__base*)&__buf_ && __f.__f_ == (__base*)&__f.__buf_) {
+    _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+    typename aligned_storage<sizeof(__buf_)>::type __tempbuf;
+    _LIBCPP_SUPPRESS_DEPRECATED_POP
+    __base* __t = (__base*)&__tempbuf;
+    __f_->__move_to(__t);
+    __f_->destroy();
+    __f_ = nullptr;
+    __f.__f_->__move_to((__base*)&__buf_);
+    __f.__f_->destroy();
+    __f.__f_ = nullptr;
+    __f_     = (__base*)&__buf_;
+    __t->__move_to((__base*)&__f.__buf_);
+    __t->destroy();
+    __f.__f_ = (__base*)&__f.__buf_;
+  } else if (__f_ == (__base*)&__buf_) {
+    __f_->__move_to((__base*)&__f.__buf_);
+    __f_->destroy();
+    __f_     = __f.__f_;
+    __f.__f_ = (__base*)&__f.__buf_;
+  } else if (__f.__f_ == (__base*)&__f.__buf_) {
+    __f.__f_->__move_to((__base*)&__buf_);
+    __f.__f_->destroy();
+    __f.__f_ = __f_;
+    __f_     = (__base*)&__buf_;
+  } else
+    std::swap(__f_, __f.__f_);
+}
+
+template <class _Rp, class... _ArgTypes>
+inline _Rp __packaged_task_function<_Rp(_ArgTypes...)>::operator()(_ArgTypes... __arg) const {
+  return (*__f_)(std::forward<_ArgTypes>(__arg)...);
+}
+
+template <class _Rp, class... _ArgTypes>
+class _LIBCPP_TEMPLATE_VIS packaged_task<_Rp(_ArgTypes...)> {
+public:
+  typedef _Rp result_type; // extension
+
+private:
+  __packaged_task_function<result_type(_ArgTypes...)> __f_;
+  promise<result_type> __p_;
+
+public:
+  // construction and destruction
+  _LIBCPP_HIDE_FROM_ABI packaged_task() _NOEXCEPT : __p_(nullptr) {}
+
+  template <class _Fp, __enable_if_t<!is_same<__remove_cvref_t<_Fp>, packaged_task>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit packaged_task(_Fp&& __f) : __f_(std::forward<_Fp>(__f)) {}
+
+  template <class _Fp, class _Allocator, __enable_if_t<!is_same<__remove_cvref_t<_Fp>, packaged_task>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI packaged_task(allocator_arg_t, const _Allocator& __a, _Fp&& __f)
+      : __f_(allocator_arg_t(), __a, std::forward<_Fp>(__f)), __p_(allocator_arg_t(), __a) {}
+  // ~packaged_task() = default;
+
+  // no copy
+  packaged_task(const packaged_task&)            = delete;
+  packaged_task& operator=(const packaged_task&) = delete;
+
+  // move support
+  _LIBCPP_HIDE_FROM_ABI packaged_task(packaged_task&& __other) _NOEXCEPT
+      : __f_(std::move(__other.__f_)),
+        __p_(std::move(__other.__p_)) {}
+  _LIBCPP_HIDE_FROM_ABI packaged_task& operator=(packaged_task&& __other) _NOEXCEPT {
+    __f_ = std::move(__other.__f_);
+    __p_ = std::move(__other.__p_);
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI void swap(packaged_task& __other) _NOEXCEPT {
+    __f_.swap(__other.__f_);
+    __p_.swap(__other.__p_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __p_.__state_ != nullptr; }
+
+  // result retrieval
+  _LIBCPP_HIDE_FROM_ABI future<result_type> get_future() { return __p_.get_future(); }
+
+  // execution
+  _LIBCPP_HIDE_FROM_ABI void operator()(_ArgTypes... __args);
+  _LIBCPP_HIDE_FROM_ABI void make_ready_at_thread_exit(_ArgTypes... __args);
+
+  _LIBCPP_HIDE_FROM_ABI void reset();
+};
+
+template <class _Rp, class... _ArgTypes>
+void packaged_task<_Rp(_ArgTypes...)>::operator()(_ArgTypes... __args) {
+  if (__p_.__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  if (__p_.__state_->__has_value())
+    __throw_future_error(future_errc::promise_already_satisfied);
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __p_.set_value(__f_(std::forward<_ArgTypes>(__args)...));
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    __p_.set_exception(current_exception());
+  }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+}
+
+template <class _Rp, class... _ArgTypes>
+void packaged_task<_Rp(_ArgTypes...)>::make_ready_at_thread_exit(_ArgTypes... __args) {
+  if (__p_.__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  if (__p_.__state_->__has_value())
+    __throw_future_error(future_errc::promise_already_satisfied);
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __p_.set_value_at_thread_exit(__f_(std::forward<_ArgTypes>(__args)...));
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    __p_.set_exception_at_thread_exit(current_exception());
+  }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+}
+
+template <class _Rp, class... _ArgTypes>
+void packaged_task<_Rp(_ArgTypes...)>::reset() {
+  if (!valid())
+    __throw_future_error(future_errc::no_state);
+  __p_ = promise<result_type>();
+}
+
+template <class... _ArgTypes>
+class _LIBCPP_TEMPLATE_VIS packaged_task<void(_ArgTypes...)> {
+public:
+  typedef void result_type; // extension
+
+private:
+  __packaged_task_function<result_type(_ArgTypes...)> __f_;
+  promise<result_type> __p_;
+
+public:
+  // construction and destruction
+  _LIBCPP_HIDE_FROM_ABI packaged_task() _NOEXCEPT : __p_(nullptr) {}
+  template <class _Fp, __enable_if_t<!is_same<__remove_cvref_t<_Fp>, packaged_task>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit packaged_task(_Fp&& __f) : __f_(std::forward<_Fp>(__f)) {}
+  template <class _Fp, class _Allocator, __enable_if_t<!is_same<__remove_cvref_t<_Fp>, packaged_task>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI packaged_task(allocator_arg_t, const _Allocator& __a, _Fp&& __f)
+      : __f_(allocator_arg_t(), __a, std::forward<_Fp>(__f)), __p_(allocator_arg_t(), __a) {}
+  // ~packaged_task() = default;
+
+  // no copy
+  packaged_task(const packaged_task&)            = delete;
+  packaged_task& operator=(const packaged_task&) = delete;
+
+  // move support
+  _LIBCPP_HIDE_FROM_ABI packaged_task(packaged_task&& __other) _NOEXCEPT
+      : __f_(std::move(__other.__f_)),
+        __p_(std::move(__other.__p_)) {}
+  _LIBCPP_HIDE_FROM_ABI packaged_task& operator=(packaged_task&& __other) _NOEXCEPT {
+    __f_ = std::move(__other.__f_);
+    __p_ = std::move(__other.__p_);
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI void swap(packaged_task& __other) _NOEXCEPT {
+    __f_.swap(__other.__f_);
+    __p_.swap(__other.__p_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __p_.__state_ != nullptr; }
+
+  // result retrieval
+  _LIBCPP_HIDE_FROM_ABI future<result_type> get_future() { return __p_.get_future(); }
+
+  // execution
+  _LIBCPP_HIDE_FROM_ABI void operator()(_ArgTypes... __args);
+  _LIBCPP_HIDE_FROM_ABI void make_ready_at_thread_exit(_ArgTypes... __args);
+
+  _LIBCPP_HIDE_FROM_ABI void reset();
+};
+
+#  if _LIBCPP_STD_VER >= 17
+
+template <class _Rp, class... _Args>
+packaged_task(_Rp (*)(_Args...)) -> packaged_task<_Rp(_Args...)>;
+
+template <class _Fp, class _Stripped = typename __strip_signature<decltype(&_Fp::operator())>::type>
+packaged_task(_Fp) -> packaged_task<_Stripped>;
+
+#  endif
+
+template <class... _ArgTypes>
+void packaged_task<void(_ArgTypes...)>::operator()(_ArgTypes... __args) {
+  if (__p_.__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  if (__p_.__state_->__has_value())
+    __throw_future_error(future_errc::promise_already_satisfied);
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __f_(std::forward<_ArgTypes>(__args)...);
+    __p_.set_value();
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    __p_.set_exception(current_exception());
+  }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+}
+
+template <class... _ArgTypes>
+void packaged_task<void(_ArgTypes...)>::make_ready_at_thread_exit(_ArgTypes... __args) {
+  if (__p_.__state_ == nullptr)
+    __throw_future_error(future_errc::no_state);
+  if (__p_.__state_->__has_value())
+    __throw_future_error(future_errc::promise_already_satisfied);
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __f_(std::forward<_ArgTypes>(__args)...);
+    __p_.set_value_at_thread_exit();
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    __p_.set_exception_at_thread_exit(current_exception());
+  }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+}
+
+template <class... _ArgTypes>
+void packaged_task<void(_ArgTypes...)>::reset() {
+  if (!valid())
+    __throw_future_error(future_errc::no_state);
+  __p_ = promise<result_type>();
+}
+
+template <class _Rp, class... _ArgTypes>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(packaged_task<_Rp(_ArgTypes...)>& __x, packaged_task<_Rp(_ArgTypes...)>& __y) _NOEXCEPT {
+  __x.swap(__y);
+}
+
+template <class _Callable, class _Alloc>
+struct _LIBCPP_TEMPLATE_VIS uses_allocator<packaged_task<_Callable>, _Alloc> : public true_type {};
+
+template <class _Rp, class _Fp>
+_LIBCPP_HIDE_FROM_ABI future<_Rp> __make_deferred_assoc_state(_Fp&& __f) {
+  unique_ptr<__deferred_assoc_state<_Rp, _Fp>, __release_shared_count> __h(
+      new __deferred_assoc_state<_Rp, _Fp>(std::forward<_Fp>(__f)));
+  return future<_Rp>(__h.get());
+}
+
+template <class _Rp, class _Fp>
+_LIBCPP_HIDE_FROM_ABI future<_Rp> __make_async_assoc_state(_Fp&& __f) {
+  unique_ptr<__async_assoc_state<_Rp, _Fp>, __release_shared_count> __h(
+      new __async_assoc_state<_Rp, _Fp>(std::forward<_Fp>(__f)));
+  std::thread(&__async_assoc_state<_Rp, _Fp>::__execute, __h.get()).detach();
+  return future<_Rp>(__h.get());
+}
+
+#  ifndef _LIBCPP_CXX03_LANG
+
+template <class _Fp, class... _Args>
+class _LIBCPP_HIDDEN __async_func {
+  tuple<_Fp, _Args...> __f_;
+
+public:
+  typedef typename __invoke_of<_Fp, _Args...>::type _Rp;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __async_func(_Fp&& __f, _Args&&... __args)
+      : __f_(std::move(__f), std::move(__args)...) {}
+
+  _LIBCPP_HIDE_FROM_ABI __async_func(__async_func&& __f) : __f_(std::move(__f.__f_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI _Rp operator()() {
+    typedef typename __make_tuple_indices<1 + sizeof...(_Args), 1>::type _Index;
+    return __execute(_Index());
+  }
+
+private:
+  template <size_t... _Indices>
+  _LIBCPP_HIDE_FROM_ABI _Rp __execute(__tuple_indices<_Indices...>) {
+    return std::__invoke(std::move(std::get<0>(__f_)), std::move(std::get<_Indices>(__f_))...);
+  }
+};
+
+inline _LIBCPP_HIDE_FROM_ABI bool __does_policy_contain(launch __policy, launch __value) {
+  return (int(__policy) & int(__value)) != 0;
+}
+
+template <class _Fp, class... _Args>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI future<typename __invoke_of<__decay_t<_Fp>, __decay_t<_Args>...>::type>
+async(launch __policy, _Fp&& __f, _Args&&... __args) {
+  typedef __async_func<__decay_t<_Fp>, __decay_t<_Args>...> _BF;
+  typedef typename _BF::_Rp _Rp;
+
+#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#    endif
+    if (__does_policy_contain(__policy, launch::async))
+      return std::__make_async_assoc_state<_Rp>(
+          _BF(_LIBCPP_AUTO_CAST(std::forward<_Fp>(__f)), _LIBCPP_AUTO_CAST(std::forward<_Args>(__args))...));
+#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    if (__policy == launch::async)
+      throw;
+  }
+#    endif
+
+  if (__does_policy_contain(__policy, launch::deferred))
+    return std::__make_deferred_assoc_state<_Rp>(
+        _BF(_LIBCPP_AUTO_CAST(std::forward<_Fp>(__f)), _LIBCPP_AUTO_CAST(std::forward<_Args>(__args))...));
+  return future<_Rp>{};
+}
+
+template <class _Fp, class... _Args>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI future<typename __invoke_of<__decay_t<_Fp>, __decay_t<_Args>...>::type>
+async(_Fp&& __f, _Args&&... __args) {
+  return std::async(launch::any, std::forward<_Fp>(__f), std::forward<_Args>(__args)...);
+}
+
+#  endif // C++03
+
+// shared_future
+
+template <class _Rp>
+class _LIBCPP_TEMPLATE_VIS shared_future {
+  __assoc_state<_Rp>* __state_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI shared_future() _NOEXCEPT : __state_(nullptr) {}
+  _LIBCPP_HIDE_FROM_ABI shared_future(const shared_future& __rhs) _NOEXCEPT : __state_(__rhs.__state_) {
+    if (__state_)
+      __state_->__add_shared();
+  }
+  _LIBCPP_HIDE_FROM_ABI shared_future(future<_Rp>&& __f) _NOEXCEPT : __state_(__f.__state_) { __f.__state_ = nullptr; }
+  _LIBCPP_HIDE_FROM_ABI shared_future(shared_future&& __rhs) _NOEXCEPT : __state_(__rhs.__state_) {
+    __rhs.__state_ = nullptr;
+  }
+  _LIBCPP_HIDE_FROM_ABI ~shared_future();
+  _LIBCPP_HIDE_FROM_ABI shared_future& operator=(const shared_future& __rhs) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI shared_future& operator=(shared_future&& __rhs) _NOEXCEPT {
+    shared_future(std::move(__rhs)).swap(*this);
+    return *this;
+  }
+
+  // retrieving the value
+  _LIBCPP_HIDE_FROM_ABI const _Rp& get() const { return __state_->copy(); }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(shared_future& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
+
+  // functions to check state
+  _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
+
+  _LIBCPP_HIDE_FROM_ABI void wait() const { __state_->wait(); }
+  template <class _Rep, class _Period>
+  _LIBCPP_HIDE_FROM_ABI future_status wait_for(const chrono::duration<_Rep, _Period>& __rel_time) const {
+    return __state_->wait_for(__rel_time);
+  }
+  template <class _Clock, class _Duration>
+  _LIBCPP_HIDE_FROM_ABI future_status wait_until(const chrono::time_point<_Clock, _Duration>& __abs_time) const {
+    return __state_->wait_until(__abs_time);
+  }
+};
+
+template <class _Rp>
+shared_future<_Rp>::~shared_future() {
+  if (__state_)
+    __state_->__release_shared();
+}
+
+template <class _Rp>
+shared_future<_Rp>& shared_future<_Rp>::operator=(const shared_future& __rhs) _NOEXCEPT {
+  if (__rhs.__state_)
+    __rhs.__state_->__add_shared();
+  if (__state_)
+    __state_->__release_shared();
+  __state_ = __rhs.__state_;
+  return *this;
+}
+
+template <class _Rp>
+class _LIBCPP_TEMPLATE_VIS shared_future<_Rp&> {
+  __assoc_state<_Rp&>* __state_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI shared_future() _NOEXCEPT : __state_(nullptr) {}
+  _LIBCPP_HIDE_FROM_ABI shared_future(const shared_future& __rhs) : __state_(__rhs.__state_) {
+    if (__state_)
+      __state_->__add_shared();
+  }
+  _LIBCPP_HIDE_FROM_ABI shared_future(future<_Rp&>&& __f) _NOEXCEPT : __state_(__f.__state_) { __f.__state_ = nullptr; }
+  _LIBCPP_HIDE_FROM_ABI shared_future(shared_future&& __rhs) _NOEXCEPT : __state_(__rhs.__state_) {
+    __rhs.__state_ = nullptr;
+  }
+  _LIBCPP_HIDE_FROM_ABI ~shared_future();
+  _LIBCPP_HIDE_FROM_ABI shared_future& operator=(const shared_future& __rhs);
+  _LIBCPP_HIDE_FROM_ABI shared_future& operator=(shared_future&& __rhs) _NOEXCEPT {
+    shared_future(std::move(__rhs)).swap(*this);
+    return *this;
+  }
+
+  // retrieving the value
+  _LIBCPP_HIDE_FROM_ABI _Rp& get() const { return __state_->copy(); }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(shared_future& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
+
+  // functions to check state
+  _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
+
+  _LIBCPP_HIDE_FROM_ABI void wait() const { __state_->wait(); }
+  template <class _Rep, class _Period>
+  _LIBCPP_HIDE_FROM_ABI future_status wait_for(const chrono::duration<_Rep, _Period>& __rel_time) const {
+    return __state_->wait_for(__rel_time);
+  }
+  template <class _Clock, class _Duration>
+  _LIBCPP_HIDE_FROM_ABI future_status wait_until(const chrono::time_point<_Clock, _Duration>& __abs_time) const {
+    return __state_->wait_until(__abs_time);
+  }
+};
+
+template <class _Rp>
+shared_future<_Rp&>::~shared_future() {
+  if (__state_)
+    __state_->__release_shared();
+}
+
+template <class _Rp>
+shared_future<_Rp&>& shared_future<_Rp&>::operator=(const shared_future& __rhs) {
+  if (__rhs.__state_)
+    __rhs.__state_->__add_shared();
+  if (__state_)
+    __state_->__release_shared();
+  __state_ = __rhs.__state_;
+  return *this;
+}
+
+template <>
+class _LIBCPP_EXPORTED_FROM_ABI shared_future<void> {
+  __assoc_sub_state* __state_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI shared_future() _NOEXCEPT : __state_(nullptr) {}
+  _LIBCPP_HIDE_FROM_ABI shared_future(const shared_future& __rhs) : __state_(__rhs.__state_) {
+    if (__state_)
+      __state_->__add_shared();
+  }
+  _LIBCPP_HIDE_FROM_ABI shared_future(future<void>&& __f) _NOEXCEPT : __state_(__f.__state_) { __f.__state_ = nullptr; }
+  _LIBCPP_HIDE_FROM_ABI shared_future(shared_future&& __rhs) _NOEXCEPT : __state_(__rhs.__state_) {
+    __rhs.__state_ = nullptr;
+  }
+  ~shared_future();
+  shared_future& operator=(const shared_future& __rhs);
+  _LIBCPP_HIDE_FROM_ABI shared_future& operator=(shared_future&& __rhs) _NOEXCEPT {
+    shared_future(std::move(__rhs)).swap(*this);
+    return *this;
+  }
+
+  // retrieving the value
+  _LIBCPP_HIDE_FROM_ABI void get() const { __state_->copy(); }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(shared_future& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
+
+  // functions to check state
+  _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
+
+  _LIBCPP_HIDE_FROM_ABI void wait() const { __state_->wait(); }
+  template <class _Rep, class _Period>
+  _LIBCPP_HIDE_FROM_ABI future_status wait_for(const chrono::duration<_Rep, _Period>& __rel_time) const {
+    return __state_->wait_for(__rel_time);
+  }
+  template <class _Clock, class _Duration>
+  _LIBCPP_HIDE_FROM_ABI future_status wait_until(const chrono::time_point<_Clock, _Duration>& __abs_time) const {
+    return __state_->wait_until(__abs_time);
+  }
+};
+
+template <class _Rp>
+inline _LIBCPP_HIDE_FROM_ABI void swap(shared_future<_Rp>& __x, shared_future<_Rp>& __y) _NOEXCEPT {
+  __x.swap(__y);
+}
+
+template <class _Rp>
+inline shared_future<_Rp> future<_Rp>::share() _NOEXCEPT {
+  return shared_future<_Rp>(std::move(*this));
+}
+
+template <class _Rp>
+inline shared_future<_Rp&> future<_Rp&>::share() _NOEXCEPT {
+  return shared_future<_Rp&>(std::move(*this));
+}
+
+inline shared_future<void> future<void>::share() _NOEXCEPT { return shared_future<void>(std::move(*this)); }
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // !defined(_LIBCPP_HAS_NO_THREADS)
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#  include <chrono>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <cstdlib>
+#  include <exception>
+#  include <iosfwd>
+#  include <system_error>
+#endif
+
+#endif // _LIBCPP_FUTURE
diff --git a/libcxx/include/__cxx03/initializer_list b/libcxx/include/__cxx03/initializer_list
new file mode 100644
index 000000000000..680ca1cd20d5
--- /dev/null
+++ b/libcxx/include/__cxx03/initializer_list
@@ -0,0 +1,98 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_INITIALIZER_LIST
+#define _LIBCPP_INITIALIZER_LIST
+
+/*
+    initializer_list synopsis
+
+namespace std
+{
+
+template<class E>
+class initializer_list
+{
+public:
+    typedef E        value_type;
+    typedef const E& reference;
+    typedef const E& const_reference;
+    typedef size_t   size_type;
+
+    typedef const E* iterator;
+    typedef const E* const_iterator;
+
+    initializer_list() noexcept; // constexpr in C++14
+
+    size_t   size()  const noexcept; // constexpr in C++14
+    const E* begin() const noexcept; // constexpr in C++14
+    const E* end()   const noexcept; // constexpr in C++14
+};
+
+template<class E> const E* begin(initializer_list<E> il) noexcept; // constexpr in C++14
+template<class E> const E* end(initializer_list<E> il) noexcept; // constexpr in C++14
+
+}  // std
+
+*/
+
+#include <__config>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+namespace std // purposefully not versioned
+{
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Ep>
+class _LIBCPP_TEMPLATE_VIS initializer_list {
+  const _Ep* __begin_;
+  size_t __size_;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 initializer_list(const _Ep* __b, size_t __s) _NOEXCEPT
+      : __begin_(__b),
+        __size_(__s) {}
+
+public:
+  typedef _Ep value_type;
+  typedef const _Ep& reference;
+  typedef const _Ep& const_reference;
+  typedef size_t size_type;
+
+  typedef const _Ep* iterator;
+  typedef const _Ep* const_iterator;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 initializer_list() _NOEXCEPT : __begin_(nullptr), __size_(0) {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 size_t size() const _NOEXCEPT { return __size_; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Ep* begin() const _NOEXCEPT { return __begin_; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Ep* end() const _NOEXCEPT { return __begin_ + __size_; }
+};
+
+template <class _Ep>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Ep* begin(initializer_list<_Ep> __il) _NOEXCEPT {
+  return __il.begin();
+}
+
+template <class _Ep>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Ep* end(initializer_list<_Ep> __il) _NOEXCEPT {
+  return __il.end();
+}
+
+#endif // !defined(_LIBCPP_CXX03_LANG)
+
+} // namespace std
+
+#endif // _LIBCPP_INITIALIZER_LIST
diff --git a/libcxx/include/__cxx03/inttypes.h b/libcxx/include/__cxx03/inttypes.h
new file mode 100644
index 000000000000..8664412bd52f
--- /dev/null
+++ b/libcxx/include/__cxx03/inttypes.h
@@ -0,0 +1,264 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_INTTYPES_H
+// AIX system headers need inttypes.h to be re-enterable while _STD_TYPES_T
+// is defined until an inclusion of it without _STD_TYPES_T occurs, in which
+// case the header guard macro is defined.
+#if !defined(_AIX) || !defined(_STD_TYPES_T)
+#  define _LIBCPP_INTTYPES_H
+#endif // _STD_TYPES_T
+
+/*
+    inttypes.h synopsis
+
+This entire header is C99 / C++0X
+
+#include <stdint.h>  // <cinttypes> includes <cstdint>
+
+Macros:
+
+    PRId8
+    PRId16
+    PRId32
+    PRId64
+
+    PRIdLEAST8
+    PRIdLEAST16
+    PRIdLEAST32
+    PRIdLEAST64
+
+    PRIdFAST8
+    PRIdFAST16
+    PRIdFAST32
+    PRIdFAST64
+
+    PRIdMAX
+    PRIdPTR
+
+    PRIi8
+    PRIi16
+    PRIi32
+    PRIi64
+
+    PRIiLEAST8
+    PRIiLEAST16
+    PRIiLEAST32
+    PRIiLEAST64
+
+    PRIiFAST8
+    PRIiFAST16
+    PRIiFAST32
+    PRIiFAST64
+
+    PRIiMAX
+    PRIiPTR
+
+    PRIo8
+    PRIo16
+    PRIo32
+    PRIo64
+
+    PRIoLEAST8
+    PRIoLEAST16
+    PRIoLEAST32
+    PRIoLEAST64
+
+    PRIoFAST8
+    PRIoFAST16
+    PRIoFAST32
+    PRIoFAST64
+
+    PRIoMAX
+    PRIoPTR
+
+    PRIu8
+    PRIu16
+    PRIu32
+    PRIu64
+
+    PRIuLEAST8
+    PRIuLEAST16
+    PRIuLEAST32
+    PRIuLEAST64
+
+    PRIuFAST8
+    PRIuFAST16
+    PRIuFAST32
+    PRIuFAST64
+
+    PRIuMAX
+    PRIuPTR
+
+    PRIx8
+    PRIx16
+    PRIx32
+    PRIx64
+
+    PRIxLEAST8
+    PRIxLEAST16
+    PRIxLEAST32
+    PRIxLEAST64
+
+    PRIxFAST8
+    PRIxFAST16
+    PRIxFAST32
+    PRIxFAST64
+
+    PRIxMAX
+    PRIxPTR
+
+    PRIX8
+    PRIX16
+    PRIX32
+    PRIX64
+
+    PRIXLEAST8
+    PRIXLEAST16
+    PRIXLEAST32
+    PRIXLEAST64
+
+    PRIXFAST8
+    PRIXFAST16
+    PRIXFAST32
+    PRIXFAST64
+
+    PRIXMAX
+    PRIXPTR
+
+    SCNd8
+    SCNd16
+    SCNd32
+    SCNd64
+
+    SCNdLEAST8
+    SCNdLEAST16
+    SCNdLEAST32
+    SCNdLEAST64
+
+    SCNdFAST8
+    SCNdFAST16
+    SCNdFAST32
+    SCNdFAST64
+
+    SCNdMAX
+    SCNdPTR
+
+    SCNi8
+    SCNi16
+    SCNi32
+    SCNi64
+
+    SCNiLEAST8
+    SCNiLEAST16
+    SCNiLEAST32
+    SCNiLEAST64
+
+    SCNiFAST8
+    SCNiFAST16
+    SCNiFAST32
+    SCNiFAST64
+
+    SCNiMAX
+    SCNiPTR
+
+    SCNo8
+    SCNo16
+    SCNo32
+    SCNo64
+
+    SCNoLEAST8
+    SCNoLEAST16
+    SCNoLEAST32
+    SCNoLEAST64
+
+    SCNoFAST8
+    SCNoFAST16
+    SCNoFAST32
+    SCNoFAST64
+
+    SCNoMAX
+    SCNoPTR
+
+    SCNu8
+    SCNu16
+    SCNu32
+    SCNu64
+
+    SCNuLEAST8
+    SCNuLEAST16
+    SCNuLEAST32
+    SCNuLEAST64
+
+    SCNuFAST8
+    SCNuFAST16
+    SCNuFAST32
+    SCNuFAST64
+
+    SCNuMAX
+    SCNuPTR
+
+    SCNx8
+    SCNx16
+    SCNx32
+    SCNx64
+
+    SCNxLEAST8
+    SCNxLEAST16
+    SCNxLEAST32
+    SCNxLEAST64
+
+    SCNxFAST8
+    SCNxFAST16
+    SCNxFAST32
+    SCNxFAST64
+
+    SCNxMAX
+    SCNxPTR
+
+Types:
+
+    imaxdiv_t
+
+intmax_t  imaxabs(intmax_t j);
+imaxdiv_t imaxdiv(intmax_t numer, intmax_t denom);
+intmax_t  strtoimax(const char* restrict nptr, char** restrict endptr, int base);
+uintmax_t strtoumax(const char* restrict nptr, char** restrict endptr, int base);
+intmax_t  wcstoimax(const wchar_t* restrict nptr, wchar_t** restrict endptr, int base);
+uintmax_t wcstoumax(const wchar_t* restrict nptr, wchar_t** restrict endptr, int base);
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+/* C99 stdlib (e.g. glibc < 2.18) does not provide format macros needed
+   for C++11 unless __STDC_FORMAT_MACROS is defined
+*/
+#if defined(__cplusplus) && !defined(__STDC_FORMAT_MACROS)
+#  define __STDC_FORMAT_MACROS
+#endif
+
+#if __has_include_next(<inttypes.h>)
+#  include_next <inttypes.h>
+#endif
+
+#ifdef __cplusplus
+
+#  include <stdint.h>
+
+#  undef imaxabs
+#  undef imaxdiv
+
+#endif // __cplusplus
+
+#endif // _LIBCPP_INTTYPES_H
diff --git a/libcxx/include/__cxx03/iomanip b/libcxx/include/__cxx03/iomanip
new file mode 100644
index 000000000000..fb4f15b9a585
--- /dev/null
+++ b/libcxx/include/__cxx03/iomanip
@@ -0,0 +1,542 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_IOMANIP
+#define _LIBCPP_IOMANIP
+
+/*
+    iomanip synopsis
+
+namespace std {
+
+// types T1, T2, ... are unspecified implementation types
+T1 resetiosflags(ios_base::fmtflags mask);
+T2 setiosflags (ios_base::fmtflags mask);
+T3 setbase(int base);
+template<charT> T4 setfill(charT c);
+T5 setprecision(int n);
+T6 setw(int n);
+template <class moneyT> T7 get_money(moneyT& mon, bool intl = false);
+template <class charT, class moneyT> T8 put_money(const moneyT& mon, bool intl = false);
+template <class charT> T9 get_time(struct tm* tmb, const charT* fmt);
+template <class charT> T10 put_time(const struct tm* tmb, const charT* fmt);
+
+template <class charT>
+  T11 quoted(const charT* s, charT delim=charT('"'), charT escape=charT('\\')); // C++14
+
+template <class charT, class traits, class Allocator>
+  T12 quoted(const basic_string<charT, traits, Allocator>& s,
+             charT delim=charT('"'), charT escape=charT('\\')); // C++14
+
+template <class charT, class traits, class Allocator>
+  T13 quoted(basic_string<charT, traits, Allocator>& s,
+             charT delim=charT('"'), charT escape=charT('\\')); // C++14
+
+}  // std
+
+*/
+
+#include <__config>
+#include <istream>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// resetiosflags
+
+class __iom_t1 {
+  ios_base::fmtflags __mask_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __iom_t1(ios_base::fmtflags __m) : __mask_(__m) {}
+
+  template <class _CharT, class _Traits>
+  friend _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t1& __x) {
+    __is.unsetf(__x.__mask_);
+    return __is;
+  }
+
+  template <class _CharT, class _Traits>
+  friend _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t1& __x) {
+    __os.unsetf(__x.__mask_);
+    return __os;
+  }
+};
+
+inline _LIBCPP_HIDE_FROM_ABI __iom_t1 resetiosflags(ios_base::fmtflags __mask) { return __iom_t1(__mask); }
+
+// setiosflags
+
+class __iom_t2 {
+  ios_base::fmtflags __mask_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __iom_t2(ios_base::fmtflags __m) : __mask_(__m) {}
+
+  template <class _CharT, class _Traits>
+  friend _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t2& __x) {
+    __is.setf(__x.__mask_);
+    return __is;
+  }
+
+  template <class _CharT, class _Traits>
+  friend _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t2& __x) {
+    __os.setf(__x.__mask_);
+    return __os;
+  }
+};
+
+inline _LIBCPP_HIDE_FROM_ABI __iom_t2 setiosflags(ios_base::fmtflags __mask) { return __iom_t2(__mask); }
+
+// setbase
+
+class __iom_t3 {
+  int __base_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __iom_t3(int __b) : __base_(__b) {}
+
+  template <class _CharT, class _Traits>
+  friend _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t3& __x) {
+    __is.setf(__x.__base_ == 8    ? ios_base::oct
+              : __x.__base_ == 10 ? ios_base::dec
+              : __x.__base_ == 16 ? ios_base::hex
+                                  : ios_base::fmtflags(0),
+              ios_base::basefield);
+    return __is;
+  }
+
+  template <class _CharT, class _Traits>
+  friend _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t3& __x) {
+    __os.setf(__x.__base_ == 8    ? ios_base::oct
+              : __x.__base_ == 10 ? ios_base::dec
+              : __x.__base_ == 16 ? ios_base::hex
+                                  : ios_base::fmtflags(0),
+              ios_base::basefield);
+    return __os;
+  }
+};
+
+inline _LIBCPP_HIDE_FROM_ABI __iom_t3 setbase(int __base) { return __iom_t3(__base); }
+
+// setfill
+
+template <class _CharT>
+class __iom_t4 {
+  _CharT __fill_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __iom_t4(_CharT __c) : __fill_(__c) {}
+
+  template <class _Traits>
+  friend _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t4& __x) {
+    __os.fill(__x.__fill_);
+    return __os;
+  }
+};
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI __iom_t4<_CharT> setfill(_CharT __c) {
+  return __iom_t4<_CharT>(__c);
+}
+
+// setprecision
+
+class __iom_t5 {
+  int __n_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __iom_t5(int __n) : __n_(__n) {}
+
+  template <class _CharT, class _Traits>
+  friend _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t5& __x) {
+    __is.precision(__x.__n_);
+    return __is;
+  }
+
+  template <class _CharT, class _Traits>
+  friend _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t5& __x) {
+    __os.precision(__x.__n_);
+    return __os;
+  }
+};
+
+inline _LIBCPP_HIDE_FROM_ABI __iom_t5 setprecision(int __n) { return __iom_t5(__n); }
+
+// setw
+
+class __iom_t6 {
+  int __n_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __iom_t6(int __n) : __n_(__n) {}
+
+  template <class _CharT, class _Traits>
+  friend _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t6& __x) {
+    __is.width(__x.__n_);
+    return __is;
+  }
+
+  template <class _CharT, class _Traits>
+  friend _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t6& __x) {
+    __os.width(__x.__n_);
+    return __os;
+  }
+};
+
+inline _LIBCPP_HIDE_FROM_ABI __iom_t6 setw(int __n) { return __iom_t6(__n); }
+
+// get_money
+
+template <class _MoneyT>
+class __iom_t7;
+
+template <class _CharT, class _Traits, class _MoneyT>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t7<_MoneyT>& __x);
+
+template <class _MoneyT>
+class __iom_t7 {
+  _MoneyT& __mon_;
+  bool __intl_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __iom_t7(_MoneyT& __mon, bool __intl) : __mon_(__mon), __intl_(__intl) {}
+
+  template <class _CharT, class _Traits, class _Mp>
+  friend basic_istream<_CharT, _Traits>& operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t7<_Mp>& __x);
+};
+
+template <class _CharT, class _Traits, class _MoneyT>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t7<_MoneyT>& __x) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    typename basic_istream<_CharT, _Traits>::sentry __s(__is);
+    if (__s) {
+      typedef istreambuf_iterator<_CharT, _Traits> _Ip;
+      typedef money_get<_CharT, _Ip> _Fp;
+      ios_base::iostate __err = ios_base::goodbit;
+      const _Fp& __mf         = std::use_facet<_Fp>(__is.getloc());
+      __mf.get(_Ip(__is), _Ip(), __x.__intl_, __is, __err, __x.__mon_);
+      __is.setstate(__err);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    __is.__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return __is;
+}
+
+template <class _MoneyT>
+inline _LIBCPP_HIDE_FROM_ABI __iom_t7<_MoneyT> get_money(_MoneyT& __mon, bool __intl = false) {
+  return __iom_t7<_MoneyT>(__mon, __intl);
+}
+
+// put_money
+
+template <class _MoneyT>
+class __iom_t8;
+
+template <class _CharT, class _Traits, class _MoneyT>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t8<_MoneyT>& __x);
+
+template <class _MoneyT>
+class __iom_t8 {
+  const _MoneyT& __mon_;
+  bool __intl_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __iom_t8(const _MoneyT& __mon, bool __intl) : __mon_(__mon), __intl_(__intl) {}
+
+  template <class _CharT, class _Traits, class _Mp>
+  friend basic_ostream<_CharT, _Traits>& operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t8<_Mp>& __x);
+};
+
+template <class _CharT, class _Traits, class _MoneyT>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t8<_MoneyT>& __x) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    typename basic_ostream<_CharT, _Traits>::sentry __s(__os);
+    if (__s) {
+      typedef ostreambuf_iterator<_CharT, _Traits> _Op;
+      typedef money_put<_CharT, _Op> _Fp;
+      const _Fp& __mf = std::use_facet<_Fp>(__os.getloc());
+      if (__mf.put(_Op(__os), __x.__intl_, __os, __os.fill(), __x.__mon_).failed())
+        __os.setstate(ios_base::badbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    __os.__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return __os;
+}
+
+template <class _MoneyT>
+inline _LIBCPP_HIDE_FROM_ABI __iom_t8<_MoneyT> put_money(const _MoneyT& __mon, bool __intl = false) {
+  return __iom_t8<_MoneyT>(__mon, __intl);
+}
+
+// get_time
+
+template <class _CharT>
+class __iom_t9;
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t9<_CharT>& __x);
+
+template <class _CharT>
+class __iom_t9 {
+  tm* __tm_;
+  const _CharT* __fmt_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __iom_t9(tm* __tm, const _CharT* __fmt) : __tm_(__tm), __fmt_(__fmt) {}
+
+  template <class _Cp, class _Traits>
+  friend basic_istream<_Cp, _Traits>& operator>>(basic_istream<_Cp, _Traits>& __is, const __iom_t9<_Cp>& __x);
+};
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t9<_CharT>& __x) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    typename basic_istream<_CharT, _Traits>::sentry __s(__is);
+    if (__s) {
+      typedef istreambuf_iterator<_CharT, _Traits> _Ip;
+      typedef time_get<_CharT, _Ip> _Fp;
+      ios_base::iostate __err = ios_base::goodbit;
+      const _Fp& __tf         = std::use_facet<_Fp>(__is.getloc());
+      __tf.get(_Ip(__is), _Ip(), __is, __err, __x.__tm_, __x.__fmt_, __x.__fmt_ + _Traits::length(__x.__fmt_));
+      __is.setstate(__err);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    __is.__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return __is;
+}
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI __iom_t9<_CharT> get_time(tm* __tm, const _CharT* __fmt) {
+  return __iom_t9<_CharT>(__tm, __fmt);
+}
+
+// put_time
+
+template <class _CharT>
+class __iom_t10;
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t10<_CharT>& __x);
+
+template <class _CharT>
+class __iom_t10 {
+  const tm* __tm_;
+  const _CharT* __fmt_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __iom_t10(const tm* __tm, const _CharT* __fmt) : __tm_(__tm), __fmt_(__fmt) {}
+
+  template <class _Cp, class _Traits>
+  friend basic_ostream<_Cp, _Traits>& operator<<(basic_ostream<_Cp, _Traits>& __os, const __iom_t10<_Cp>& __x);
+};
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t10<_CharT>& __x) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    typename basic_ostream<_CharT, _Traits>::sentry __s(__os);
+    if (__s) {
+      typedef ostreambuf_iterator<_CharT, _Traits> _Op;
+      typedef time_put<_CharT, _Op> _Fp;
+      const _Fp& __tf = std::use_facet<_Fp>(__os.getloc());
+      if (__tf.put(_Op(__os), __os, __os.fill(), __x.__tm_, __x.__fmt_, __x.__fmt_ + _Traits::length(__x.__fmt_))
+              .failed())
+        __os.setstate(ios_base::badbit);
+    }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    __os.__set_badbit_and_consider_rethrow();
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return __os;
+}
+
+template <class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI __iom_t10<_CharT> put_time(const tm* __tm, const _CharT* __fmt) {
+  return __iom_t10<_CharT>(__tm, __fmt);
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>& __quoted_output(
+    basic_ostream<_CharT, _Traits>& __os,
+    const _CharT* __first,
+    const _CharT* __last,
+    _CharT __delim,
+    _CharT __escape) {
+  basic_string<_CharT, _Traits> __str;
+  __str.push_back(__delim);
+  for (; __first != __last; ++__first) {
+    if (_Traits::eq(*__first, __escape) || _Traits::eq(*__first, __delim))
+      __str.push_back(__escape);
+    __str.push_back(*__first);
+  }
+  __str.push_back(__delim);
+  return std::__put_character_sequence(__os, __str.data(), __str.size());
+}
+
+template <class _CharT, class _Traits, class _String>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+__quoted_input(basic_istream<_CharT, _Traits>& __is, _String& __string, _CharT __delim, _CharT __escape) {
+  __string.clear();
+  _CharT __c;
+  __is >> __c;
+  if (__is.fail())
+    return __is;
+
+  if (!_Traits::eq(__c, __delim)) {
+    // no delimiter, read the whole string
+    __is.unget();
+    __is >> __string;
+    return __is;
+  }
+
+  __save_flags<_CharT, _Traits> __sf(__is);
+  std::noskipws(__is);
+  while (true) {
+    __is >> __c;
+    if (__is.fail())
+      break;
+    if (_Traits::eq(__c, __escape)) {
+      __is >> __c;
+      if (__is.fail())
+        break;
+    } else if (_Traits::eq(__c, __delim))
+      break;
+    __string.push_back(__c);
+  }
+  return __is;
+}
+
+template <class _CharT, class _Traits>
+struct _LIBCPP_HIDDEN __quoted_output_proxy {
+  const _CharT* __first_;
+  const _CharT* __last_;
+  _CharT __delim_;
+  _CharT __escape_;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __quoted_output_proxy(const _CharT* __f, const _CharT* __l, _CharT __d, _CharT __e)
+      : __first_(__f), __last_(__l), __delim_(__d), __escape_(__e) {}
+
+  template <class _T2, __enable_if_t<_IsSame<_Traits, void>::value || _IsSame<_Traits, _T2>::value, int> = 0>
+  friend _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _T2>&
+  operator<<(basic_ostream<_CharT, _T2>& __os, const __quoted_output_proxy& __p) {
+    return std::__quoted_output(__os, __p.__first_, __p.__last_, __p.__delim_, __p.__escape_);
+  }
+};
+
+template <class _CharT, class _Traits, class _Allocator>
+struct _LIBCPP_HIDDEN __quoted_proxy {
+  basic_string<_CharT, _Traits, _Allocator>& __string_;
+  _CharT __delim_;
+  _CharT __escape_;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __quoted_proxy(basic_string<_CharT, _Traits, _Allocator>& __s, _CharT __d, _CharT __e)
+      : __string_(__s), __delim_(__d), __escape_(__e) {}
+
+  friend _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+  operator<<(basic_ostream<_CharT, _Traits>& __os, const __quoted_proxy& __p) {
+    return std::__quoted_output(
+        __os, __p.__string_.data(), __p.__string_.data() + __p.__string_.size(), __p.__delim_, __p.__escape_);
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, const __quoted_proxy& __p) {
+    return std::__quoted_input(__is, __p.__string_, __p.__delim_, __p.__escape_);
+  }
+};
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI __quoted_output_proxy<_CharT, _Traits>
+__quoted(const basic_string<_CharT, _Traits, _Allocator>& __s,
+         _CharT __delim  = _CharT('"'),
+         _CharT __escape = _CharT('\\')) {
+  return __quoted_output_proxy<_CharT, _Traits>(__s.data(), __s.data() + __s.size(), __delim, __escape);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI __quoted_proxy<_CharT, _Traits, _Allocator>
+__quoted(basic_string<_CharT, _Traits, _Allocator>& __s, _CharT __delim = _CharT('"'), _CharT __escape = _CharT('\\')) {
+  return __quoted_proxy<_CharT, _Traits, _Allocator>(__s, __delim, __escape);
+}
+
+#if _LIBCPP_STD_VER >= 14
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI auto quoted(const _CharT* __s, _CharT __delim = _CharT('"'), _CharT __escape = _CharT('\\')) {
+  const _CharT* __end = __s;
+  while (*__end)
+    ++__end;
+  return __quoted_output_proxy<_CharT, void>(__s, __end, __delim, __escape);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI auto
+quoted(const basic_string<_CharT, _Traits, _Allocator>& __s,
+       _CharT __delim  = _CharT('"'),
+       _CharT __escape = _CharT('\\')) {
+  return __quoted_output_proxy<_CharT, _Traits>(__s.data(), __s.data() + __s.size(), __delim, __escape);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI auto
+quoted(basic_string<_CharT, _Traits, _Allocator>& __s, _CharT __delim = _CharT('"'), _CharT __escape = _CharT('\\')) {
+  return __quoted_proxy<_CharT, _Traits, _Allocator>(__s, __delim, __escape);
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI auto
+quoted(basic_string_view<_CharT, _Traits> __sv, _CharT __delim = _CharT('"'), _CharT __escape = _CharT('\\')) {
+  return __quoted_output_proxy<_CharT, _Traits>(__sv.data(), __sv.data() + __sv.size(), __delim, __escape);
+}
+
+#endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_IOMANIP
diff --git a/libcxx/include/__cxx03/ios b/libcxx/include/__cxx03/ios
new file mode 100644
index 000000000000..fae71caa87ba
--- /dev/null
+++ b/libcxx/include/__cxx03/ios
@@ -0,0 +1,889 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_IOS
+#define _LIBCPP_IOS
+
+/*
+    ios synopsis
+
+#include <iosfwd>
+
+namespace std
+{
+
+typedef OFF_T streamoff;
+typedef SZ_T streamsize;
+template <class stateT> class fpos;
+
+class ios_base
+{
+public:
+    class failure;
+
+    typedef T1 fmtflags;
+    static constexpr fmtflags boolalpha;
+    static constexpr fmtflags dec;
+    static constexpr fmtflags fixed;
+    static constexpr fmtflags hex;
+    static constexpr fmtflags internal;
+    static constexpr fmtflags left;
+    static constexpr fmtflags oct;
+    static constexpr fmtflags right;
+    static constexpr fmtflags scientific;
+    static constexpr fmtflags showbase;
+    static constexpr fmtflags showpoint;
+    static constexpr fmtflags showpos;
+    static constexpr fmtflags skipws;
+    static constexpr fmtflags unitbuf;
+    static constexpr fmtflags uppercase;
+    static constexpr fmtflags adjustfield;
+    static constexpr fmtflags basefield;
+    static constexpr fmtflags floatfield;
+
+    typedef T2 iostate;
+    static constexpr iostate badbit;
+    static constexpr iostate eofbit;
+    static constexpr iostate failbit;
+    static constexpr iostate goodbit;
+
+    typedef T3 openmode;
+    static constexpr openmode app;
+    static constexpr openmode ate;
+    static constexpr openmode binary;
+    static constexpr openmode in;
+    static constexpr openmode noreplace; // since C++23
+    static constexpr openmode out;
+    static constexpr openmode trunc;
+
+    typedef T4 seekdir;
+    static constexpr seekdir beg;
+    static constexpr seekdir cur;
+    static constexpr seekdir end;
+
+    class Init;
+
+    // 27.5.2.2 fmtflags state:
+    fmtflags flags() const;
+    fmtflags flags(fmtflags fmtfl);
+    fmtflags setf(fmtflags fmtfl);
+    fmtflags setf(fmtflags fmtfl, fmtflags mask);
+    void unsetf(fmtflags mask);
+
+    streamsize precision() const;
+    streamsize precision(streamsize prec);
+    streamsize width() const;
+    streamsize width(streamsize wide);
+
+    // 27.5.2.3 locales:
+    locale imbue(const locale& loc);
+    locale getloc() const;
+
+    // 27.5.2.5 storage:
+    static int xalloc();
+    long& iword(int index);
+    void*& pword(int index);
+
+    // destructor
+    virtual ~ios_base();
+
+    // 27.5.2.6 callbacks;
+    enum event { erase_event, imbue_event, copyfmt_event };
+    typedef void (*event_callback)(event, ios_base&, int index);
+    void register_callback(event_callback fn, int index);
+
+    ios_base(const ios_base&) = delete;
+    ios_base& operator=(const ios_base&) = delete;
+
+    static bool sync_with_stdio(bool sync = true);
+
+protected:
+    ios_base();
+};
+
+template <class charT, class traits = char_traits<charT> >
+class basic_ios
+    : public ios_base
+{
+public:
+    // types:
+    typedef charT char_type;
+    typedef typename traits::int_type int_type;  // removed in C++17
+    typedef typename traits::pos_type pos_type;  // removed in C++17
+    typedef typename traits::off_type off_type;  // removed in C++17
+    typedef traits traits_type;
+
+    operator unspecified-bool-type() const;
+    bool operator!() const;
+    iostate rdstate() const;
+    void clear(iostate state = goodbit);
+    void setstate(iostate state);
+    bool good() const;
+    bool eof() const;
+    bool fail() const;
+    bool bad() const;
+
+    iostate exceptions() const;
+    void exceptions(iostate except);
+
+    // 27.5.4.1 Constructor/destructor:
+    explicit basic_ios(basic_streambuf<charT,traits>* sb);
+    virtual ~basic_ios();
+
+    // 27.5.4.2 Members:
+    basic_ostream<charT,traits>* tie() const;
+    basic_ostream<charT,traits>* tie(basic_ostream<charT,traits>* tiestr);
+
+    basic_streambuf<charT,traits>* rdbuf() const;
+    basic_streambuf<charT,traits>* rdbuf(basic_streambuf<charT,traits>* sb);
+
+    basic_ios& copyfmt(const basic_ios& rhs);
+
+    char_type fill() const;
+    char_type fill(char_type ch);
+
+    locale imbue(const locale& loc);
+
+    char narrow(char_type c, char dfault) const;
+    char_type widen(char c) const;
+
+    basic_ios(const basic_ios& ) = delete;
+    basic_ios& operator=(const basic_ios&) = delete;
+
+protected:
+    basic_ios();
+    void init(basic_streambuf<charT,traits>* sb);
+    void move(basic_ios& rhs);
+    void swap(basic_ios& rhs) noexcept;
+    void set_rdbuf(basic_streambuf<charT, traits>* sb);
+};
+
+// 27.5.5, manipulators:
+ios_base& boolalpha (ios_base& str);
+ios_base& noboolalpha(ios_base& str);
+ios_base& showbase (ios_base& str);
+ios_base& noshowbase (ios_base& str);
+ios_base& showpoint (ios_base& str);
+ios_base& noshowpoint(ios_base& str);
+ios_base& showpos (ios_base& str);
+ios_base& noshowpos (ios_base& str);
+ios_base& skipws (ios_base& str);
+ios_base& noskipws (ios_base& str);
+ios_base& uppercase (ios_base& str);
+ios_base& nouppercase(ios_base& str);
+ios_base& unitbuf (ios_base& str);
+ios_base& nounitbuf (ios_base& str);
+
+// 27.5.5.2 adjustfield:
+ios_base& internal (ios_base& str);
+ios_base& left (ios_base& str);
+ios_base& right (ios_base& str);
+
+// 27.5.5.3 basefield:
+ios_base& dec (ios_base& str);
+ios_base& hex (ios_base& str);
+ios_base& oct (ios_base& str);
+
+// 27.5.5.4 floatfield:
+ios_base& fixed (ios_base& str);
+ios_base& scientific (ios_base& str);
+ios_base& hexfloat (ios_base& str);
+ios_base& defaultfloat(ios_base& str);
+
+// 27.5.5.5 error reporting:
+enum class io_errc
+{
+    stream = 1
+};
+
+concept_map ErrorCodeEnum<io_errc> { };
+error_code make_error_code(io_errc e) noexcept;
+error_condition make_error_condition(io_errc e) noexcept;
+storage-class-specifier const error_category& iostream_category() noexcept;
+
+}  // std
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+
+#  include <__fwd/ios.h>
+#  include <__ios/fpos.h>
+#  include <__locale>
+#  include <__system_error/error_category.h>
+#  include <__system_error/error_code.h>
+#  include <__system_error/error_condition.h>
+#  include <__system_error/system_error.h>
+#  include <__utility/swap.h>
+#  include <__verbose_abort>
+#  include <version>
+
+// standard-mandated includes
+
+// [ios.syn]
+#  include <iosfwd>
+
+#  if !defined(_LIBCPP_HAS_NO_ATOMIC_HEADER)
+#    include <__atomic/atomic.h> // for __xindex_
+#  endif
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+typedef ptrdiff_t streamsize;
+
+class _LIBCPP_EXPORTED_FROM_ABI ios_base {
+public:
+  class _LIBCPP_EXPORTED_FROM_ABI failure;
+
+  typedef unsigned int fmtflags;
+  static const fmtflags boolalpha   = 0x0001;
+  static const fmtflags dec         = 0x0002;
+  static const fmtflags fixed       = 0x0004;
+  static const fmtflags hex         = 0x0008;
+  static const fmtflags internal    = 0x0010;
+  static const fmtflags left        = 0x0020;
+  static const fmtflags oct         = 0x0040;
+  static const fmtflags right       = 0x0080;
+  static const fmtflags scientific  = 0x0100;
+  static const fmtflags showbase    = 0x0200;
+  static const fmtflags showpoint   = 0x0400;
+  static const fmtflags showpos     = 0x0800;
+  static const fmtflags skipws      = 0x1000;
+  static const fmtflags unitbuf     = 0x2000;
+  static const fmtflags uppercase   = 0x4000;
+  static const fmtflags adjustfield = left | right | internal;
+  static const fmtflags basefield   = dec | oct | hex;
+  static const fmtflags floatfield  = scientific | fixed;
+
+  typedef unsigned int iostate;
+  static const iostate badbit  = 0x1;
+  static const iostate eofbit  = 0x2;
+  static const iostate failbit = 0x4;
+  static const iostate goodbit = 0x0;
+
+  typedef unsigned int openmode;
+  static const openmode app    = 0x01;
+  static const openmode ate    = 0x02;
+  static const openmode binary = 0x04;
+  static const openmode in     = 0x08;
+  static const openmode out    = 0x10;
+  static const openmode trunc  = 0x20;
+#  if _LIBCPP_STD_VER >= 23
+  static const openmode noreplace = 0x40;
+#  endif
+
+  enum seekdir { beg, cur, end };
+
+#  if _LIBCPP_STD_VER <= 14
+  typedef iostate io_state;
+  typedef openmode open_mode;
+  typedef seekdir seek_dir;
+
+  typedef std::streamoff streamoff;
+  typedef std::streampos streampos;
+#  endif
+
+  class _LIBCPP_EXPORTED_FROM_ABI Init;
+
+  // 27.5.2.2 fmtflags state:
+  _LIBCPP_HIDE_FROM_ABI fmtflags flags() const;
+  _LIBCPP_HIDE_FROM_ABI fmtflags flags(fmtflags __fmtfl);
+  _LIBCPP_HIDE_FROM_ABI fmtflags setf(fmtflags __fmtfl);
+  _LIBCPP_HIDE_FROM_ABI fmtflags setf(fmtflags __fmtfl, fmtflags __mask);
+  _LIBCPP_HIDE_FROM_ABI void unsetf(fmtflags __mask);
+
+  _LIBCPP_HIDE_FROM_ABI streamsize precision() const;
+  _LIBCPP_HIDE_FROM_ABI streamsize precision(streamsize __prec);
+  _LIBCPP_HIDE_FROM_ABI streamsize width() const;
+  _LIBCPP_HIDE_FROM_ABI streamsize width(streamsize __wide);
+
+  // 27.5.2.3 locales:
+  locale imbue(const locale& __loc);
+  locale getloc() const;
+
+  // 27.5.2.5 storage:
+  static int xalloc();
+  long& iword(int __index);
+  void*& pword(int __index);
+
+  // destructor
+  virtual ~ios_base();
+
+  // 27.5.2.6 callbacks;
+  enum event { erase_event, imbue_event, copyfmt_event };
+  typedef void (*event_callback)(event, ios_base&, int __index);
+  void register_callback(event_callback __fn, int __index);
+
+  ios_base(const ios_base&)            = delete;
+  ios_base& operator=(const ios_base&) = delete;
+
+  static bool sync_with_stdio(bool __sync = true);
+
+  _LIBCPP_HIDE_FROM_ABI iostate rdstate() const;
+  void clear(iostate __state = goodbit);
+  _LIBCPP_HIDE_FROM_ABI void setstate(iostate __state);
+
+  _LIBCPP_HIDE_FROM_ABI bool good() const;
+  _LIBCPP_HIDE_FROM_ABI bool eof() const;
+  _LIBCPP_HIDE_FROM_ABI bool fail() const;
+  _LIBCPP_HIDE_FROM_ABI bool bad() const;
+
+  _LIBCPP_HIDE_FROM_ABI iostate exceptions() const;
+  _LIBCPP_HIDE_FROM_ABI void exceptions(iostate __iostate);
+
+  void __set_badbit_and_consider_rethrow();
+  void __set_failbit_and_consider_rethrow();
+
+  _LIBCPP_HIDE_FROM_ABI void __setstate_nothrow(iostate __state) {
+    if (__rdbuf_)
+      __rdstate_ |= __state;
+    else
+      __rdstate_ |= __state | ios_base::badbit;
+  }
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI ios_base() : __loc_(nullptr) {
+    // Purposefully does no initialization
+    //
+    // Except for the locale, this is a sentinel to avoid destroying
+    // an uninitialized object. See
+    // test/libcxx/input.output/iostreams.base/ios.base/ios.base.cons/dtor.uninitialized.pass.cpp
+    // for the details.
+  }
+
+  void init(void* __sb);
+  _LIBCPP_HIDE_FROM_ABI void* rdbuf() const { return __rdbuf_; }
+
+  _LIBCPP_HIDE_FROM_ABI void rdbuf(void* __sb) {
+    __rdbuf_ = __sb;
+    clear();
+  }
+
+  void __call_callbacks(event);
+  void copyfmt(const ios_base&);
+  void move(ios_base&);
+  void swap(ios_base&) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI void set_rdbuf(void* __sb) { __rdbuf_ = __sb; }
+
+private:
+  // All data members must be scalars
+  fmtflags __fmtflags_;
+  streamsize __precision_;
+  streamsize __width_;
+  iostate __rdstate_;
+  iostate __exceptions_;
+  void* __rdbuf_;
+  void* __loc_;
+  event_callback* __fn_;
+  int* __index_;
+  size_t __event_size_;
+  size_t __event_cap_;
+// TODO(EricWF): Enable this for both Clang and GCC. Currently it is only
+// enabled with clang.
+#  if defined(_LIBCPP_HAS_C_ATOMIC_IMP) && !defined(_LIBCPP_HAS_NO_THREADS)
+  static atomic<int> __xindex_;
+#  else
+  static int __xindex_;
+#  endif
+  long* __iarray_;
+  size_t __iarray_size_;
+  size_t __iarray_cap_;
+  void** __parray_;
+  size_t __parray_size_;
+  size_t __parray_cap_;
+};
+
+// enum class io_errc
+_LIBCPP_DECLARE_STRONG_ENUM(io_errc){stream = 1};
+_LIBCPP_DECLARE_STRONG_ENUM_EPILOG(io_errc)
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS is_error_code_enum<io_errc> : public true_type {};
+
+#  ifdef _LIBCPP_CXX03_LANG
+template <>
+struct _LIBCPP_TEMPLATE_VIS is_error_code_enum<io_errc::__lx> : public true_type {};
+#  endif
+
+_LIBCPP_EXPORTED_FROM_ABI const error_category& iostream_category() _NOEXCEPT;
+
+inline _LIBCPP_HIDE_FROM_ABI error_code make_error_code(io_errc __e) _NOEXCEPT {
+  return error_code(static_cast<int>(__e), iostream_category());
+}
+
+inline _LIBCPP_HIDE_FROM_ABI error_condition make_error_condition(io_errc __e) _NOEXCEPT {
+  return error_condition(static_cast<int>(__e), iostream_category());
+}
+
+class _LIBCPP_EXPORTED_FROM_ABI ios_base::failure : public system_error {
+public:
+  explicit failure(const string& __msg, const error_code& __ec = io_errc::stream);
+  explicit failure(const char* __msg, const error_code& __ec = io_errc::stream);
+  _LIBCPP_HIDE_FROM_ABI failure(const failure&) _NOEXCEPT = default;
+  ~failure() _NOEXCEPT override;
+};
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_failure(char const* __msg) {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw ios_base::failure(__msg);
+#  else
+  _LIBCPP_VERBOSE_ABORT("ios_base::failure was thrown in -fno-exceptions mode with message \"%s\"", __msg);
+#  endif
+}
+
+class _LIBCPP_EXPORTED_FROM_ABI ios_base::Init {
+public:
+  Init();
+  ~Init();
+};
+
+// fmtflags
+
+inline _LIBCPP_HIDE_FROM_ABI ios_base::fmtflags ios_base::flags() const { return __fmtflags_; }
+
+inline _LIBCPP_HIDE_FROM_ABI ios_base::fmtflags ios_base::flags(fmtflags __fmtfl) {
+  fmtflags __r = __fmtflags_;
+  __fmtflags_  = __fmtfl;
+  return __r;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI ios_base::fmtflags ios_base::setf(fmtflags __fmtfl) {
+  fmtflags __r = __fmtflags_;
+  __fmtflags_ |= __fmtfl;
+  return __r;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI void ios_base::unsetf(fmtflags __mask) { __fmtflags_ &= ~__mask; }
+
+inline _LIBCPP_HIDE_FROM_ABI ios_base::fmtflags ios_base::setf(fmtflags __fmtfl, fmtflags __mask) {
+  fmtflags __r = __fmtflags_;
+  unsetf(__mask);
+  __fmtflags_ |= __fmtfl & __mask;
+  return __r;
+}
+
+// precision
+
+inline _LIBCPP_HIDE_FROM_ABI streamsize ios_base::precision() const { return __precision_; }
+
+inline _LIBCPP_HIDE_FROM_ABI streamsize ios_base::precision(streamsize __prec) {
+  streamsize __r = __precision_;
+  __precision_   = __prec;
+  return __r;
+}
+
+// width
+
+inline _LIBCPP_HIDE_FROM_ABI streamsize ios_base::width() const { return __width_; }
+
+inline _LIBCPP_HIDE_FROM_ABI streamsize ios_base::width(streamsize __wide) {
+  streamsize __r = __width_;
+  __width_       = __wide;
+  return __r;
+}
+
+// iostate
+
+inline _LIBCPP_HIDE_FROM_ABI ios_base::iostate ios_base::rdstate() const { return __rdstate_; }
+
+inline _LIBCPP_HIDE_FROM_ABI void ios_base::setstate(iostate __state) { clear(__rdstate_ | __state); }
+
+inline _LIBCPP_HIDE_FROM_ABI bool ios_base::good() const { return __rdstate_ == 0; }
+
+inline _LIBCPP_HIDE_FROM_ABI bool ios_base::eof() const { return (__rdstate_ & eofbit) != 0; }
+
+inline _LIBCPP_HIDE_FROM_ABI bool ios_base::fail() const { return (__rdstate_ & (failbit | badbit)) != 0; }
+
+inline _LIBCPP_HIDE_FROM_ABI bool ios_base::bad() const { return (__rdstate_ & badbit) != 0; }
+
+inline _LIBCPP_HIDE_FROM_ABI ios_base::iostate ios_base::exceptions() const { return __exceptions_; }
+
+inline _LIBCPP_HIDE_FROM_ABI void ios_base::exceptions(iostate __iostate) {
+  __exceptions_ = __iostate;
+  clear(__rdstate_);
+}
+
+template <class _Traits>
+// Attribute 'packed' is used to keep the layout compatible with the previous
+// definition of the '__fill_' and '_set_' pair in basic_ios on AIX & z/OS.
+struct _LIBCPP_PACKED _FillHelper {
+  _LIBCPP_HIDE_FROM_ABI void __init() { __set_ = false; }
+  _LIBCPP_HIDE_FROM_ABI _FillHelper& operator=(typename _Traits::int_type __x) {
+    __set_      = true;
+    __fill_val_ = __x;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI bool __is_set() const { return __set_; }
+  _LIBCPP_HIDE_FROM_ABI typename _Traits::int_type __get() const { return __fill_val_; }
+
+private:
+  typename _Traits::int_type __fill_val_;
+  bool __set_;
+};
+
+template <class _Traits>
+struct _LIBCPP_PACKED _SentinelValueFill {
+  _LIBCPP_HIDE_FROM_ABI void __init() { __fill_val_ = _Traits::eof(); }
+  _LIBCPP_HIDE_FROM_ABI _SentinelValueFill& operator=(typename _Traits::int_type __x) {
+    __fill_val_ = __x;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI bool __is_set() const { return __fill_val_ != _Traits::eof(); }
+  _LIBCPP_HIDE_FROM_ABI typename _Traits::int_type __get() const { return __fill_val_; }
+
+private:
+  typename _Traits::int_type __fill_val_;
+};
+
+template <class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS basic_ios : public ios_base {
+public:
+  // types:
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+
+  static_assert(is_same<_CharT, typename traits_type::char_type>::value,
+                "traits_type::char_type must be the same type as CharT");
+
+#  ifdef _LIBCPP_CXX03_LANG
+  // Preserve the ability to compare with literal 0,
+  // and implicitly convert to bool, but not implicitly convert to int.
+  _LIBCPP_HIDE_FROM_ABI operator void*() const { return fail() ? nullptr : (void*)this; }
+#  else
+  _LIBCPP_HIDE_FROM_ABI explicit operator bool() const { return !fail(); }
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI bool operator!() const { return fail(); }
+  _LIBCPP_HIDE_FROM_ABI iostate rdstate() const { return ios_base::rdstate(); }
+  _LIBCPP_HIDE_FROM_ABI void clear(iostate __state = goodbit) { ios_base::clear(__state); }
+  _LIBCPP_HIDE_FROM_ABI void setstate(iostate __state) { ios_base::setstate(__state); }
+  _LIBCPP_HIDE_FROM_ABI bool good() const { return ios_base::good(); }
+  _LIBCPP_HIDE_FROM_ABI bool eof() const { return ios_base::eof(); }
+  _LIBCPP_HIDE_FROM_ABI bool fail() const { return ios_base::fail(); }
+  _LIBCPP_HIDE_FROM_ABI bool bad() const { return ios_base::bad(); }
+
+  _LIBCPP_HIDE_FROM_ABI iostate exceptions() const { return ios_base::exceptions(); }
+  _LIBCPP_HIDE_FROM_ABI void exceptions(iostate __iostate) { ios_base::exceptions(__iostate); }
+
+  // 27.5.4.1 Constructor/destructor:
+  _LIBCPP_HIDE_FROM_ABI explicit basic_ios(basic_streambuf<char_type, traits_type>* __sb);
+  ~basic_ios() override;
+
+  // 27.5.4.2 Members:
+  _LIBCPP_HIDE_FROM_ABI basic_ostream<char_type, traits_type>* tie() const;
+  _LIBCPP_HIDE_FROM_ABI basic_ostream<char_type, traits_type>* tie(basic_ostream<char_type, traits_type>* __tiestr);
+
+  _LIBCPP_HIDE_FROM_ABI basic_streambuf<char_type, traits_type>* rdbuf() const;
+  _LIBCPP_HIDE_FROM_ABI basic_streambuf<char_type, traits_type>* rdbuf(basic_streambuf<char_type, traits_type>* __sb);
+
+  basic_ios& copyfmt(const basic_ios& __rhs);
+
+  _LIBCPP_HIDE_FROM_ABI char_type fill() const;
+  _LIBCPP_HIDE_FROM_ABI char_type fill(char_type __ch);
+
+  _LIBCPP_HIDE_FROM_ABI locale imbue(const locale& __loc);
+
+  _LIBCPP_HIDE_FROM_ABI char narrow(char_type __c, char __dfault) const;
+  _LIBCPP_HIDE_FROM_ABI char_type widen(char __c) const;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI basic_ios() {
+    // purposefully does no initialization
+    // since the destructor does nothing this does not have ios_base issues.
+  }
+  _LIBCPP_HIDE_FROM_ABI void init(basic_streambuf<char_type, traits_type>* __sb);
+
+  _LIBCPP_HIDE_FROM_ABI void move(basic_ios& __rhs);
+  _LIBCPP_HIDE_FROM_ABI void move(basic_ios&& __rhs) { move(__rhs); }
+  _LIBCPP_HIDE_FROM_ABI void swap(basic_ios& __rhs) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI void set_rdbuf(basic_streambuf<char_type, traits_type>* __sb);
+
+private:
+  basic_ostream<char_type, traits_type>* __tie_;
+
+#  if defined(_LIBCPP_ABI_IOS_ALLOW_ARBITRARY_FILL_VALUE)
+  using _FillType = _FillHelper<traits_type>;
+#  else
+  using _FillType = _SentinelValueFill<traits_type>;
+#  endif
+  mutable _FillType __fill_;
+};
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI basic_ios<_CharT, _Traits>::basic_ios(basic_streambuf<char_type, traits_type>* __sb) {
+  init(__sb);
+}
+
+template <class _CharT, class _Traits>
+basic_ios<_CharT, _Traits>::~basic_ios() {}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI void basic_ios<_CharT, _Traits>::init(basic_streambuf<char_type, traits_type>* __sb) {
+  ios_base::init(__sb);
+  __tie_ = nullptr;
+  __fill_.__init();
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>* basic_ios<_CharT, _Traits>::tie() const {
+  return __tie_;
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>*
+basic_ios<_CharT, _Traits>::tie(basic_ostream<char_type, traits_type>* __tiestr) {
+  basic_ostream<char_type, traits_type>* __r = __tie_;
+  __tie_                                     = __tiestr;
+  return __r;
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI basic_streambuf<_CharT, _Traits>* basic_ios<_CharT, _Traits>::rdbuf() const {
+  return static_cast<basic_streambuf<char_type, traits_type>*>(ios_base::rdbuf());
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI basic_streambuf<_CharT, _Traits>*
+basic_ios<_CharT, _Traits>::rdbuf(basic_streambuf<char_type, traits_type>* __sb) {
+  basic_streambuf<char_type, traits_type>* __r = rdbuf();
+  ios_base::rdbuf(__sb);
+  return __r;
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI locale basic_ios<_CharT, _Traits>::imbue(const locale& __loc) {
+  locale __r = getloc();
+  ios_base::imbue(__loc);
+  if (rdbuf())
+    rdbuf()->pubimbue(__loc);
+  return __r;
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI char basic_ios<_CharT, _Traits>::narrow(char_type __c, char __dfault) const {
+  return std::use_facet<ctype<char_type> >(getloc()).narrow(__c, __dfault);
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI _CharT basic_ios<_CharT, _Traits>::widen(char __c) const {
+  return std::use_facet<ctype<char_type> >(getloc()).widen(__c);
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI _CharT basic_ios<_CharT, _Traits>::fill() const {
+  if (!__fill_.__is_set())
+    __fill_ = widen(' ');
+  return __fill_.__get();
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI _CharT basic_ios<_CharT, _Traits>::fill(char_type __ch) {
+  if (!__fill_.__is_set())
+    __fill_ = widen(' ');
+  char_type __r = __fill_.__get();
+  __fill_       = __ch;
+  return __r;
+}
+
+template <class _CharT, class _Traits>
+basic_ios<_CharT, _Traits>& basic_ios<_CharT, _Traits>::copyfmt(const basic_ios& __rhs) {
+  if (this != &__rhs) {
+    __call_callbacks(erase_event);
+    ios_base::copyfmt(__rhs);
+    __tie_  = __rhs.__tie_;
+    __fill_ = __rhs.__fill_;
+    __call_callbacks(copyfmt_event);
+    exceptions(__rhs.exceptions());
+  }
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI void basic_ios<_CharT, _Traits>::move(basic_ios& __rhs) {
+  ios_base::move(__rhs);
+  __tie_       = __rhs.__tie_;
+  __rhs.__tie_ = nullptr;
+  __fill_      = __rhs.__fill_;
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI void basic_ios<_CharT, _Traits>::swap(basic_ios& __rhs) _NOEXCEPT {
+  ios_base::swap(__rhs);
+  std::swap(__tie_, __rhs.__tie_);
+  std::swap(__fill_, __rhs.__fill_);
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI void basic_ios<_CharT, _Traits>::set_rdbuf(basic_streambuf<char_type, traits_type>* __sb) {
+  ios_base::set_rdbuf(__sb);
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_ios<char>;
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_ios<wchar_t>;
+#  endif
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& boolalpha(ios_base& __str) {
+  __str.setf(ios_base::boolalpha);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& noboolalpha(ios_base& __str) {
+  __str.unsetf(ios_base::boolalpha);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& showbase(ios_base& __str) {
+  __str.setf(ios_base::showbase);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& noshowbase(ios_base& __str) {
+  __str.unsetf(ios_base::showbase);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& showpoint(ios_base& __str) {
+  __str.setf(ios_base::showpoint);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& noshowpoint(ios_base& __str) {
+  __str.unsetf(ios_base::showpoint);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& showpos(ios_base& __str) {
+  __str.setf(ios_base::showpos);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& noshowpos(ios_base& __str) {
+  __str.unsetf(ios_base::showpos);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& skipws(ios_base& __str) {
+  __str.setf(ios_base::skipws);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& noskipws(ios_base& __str) {
+  __str.unsetf(ios_base::skipws);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& uppercase(ios_base& __str) {
+  __str.setf(ios_base::uppercase);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& nouppercase(ios_base& __str) {
+  __str.unsetf(ios_base::uppercase);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& unitbuf(ios_base& __str) {
+  __str.setf(ios_base::unitbuf);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& nounitbuf(ios_base& __str) {
+  __str.unsetf(ios_base::unitbuf);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& internal(ios_base& __str) {
+  __str.setf(ios_base::internal, ios_base::adjustfield);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& left(ios_base& __str) {
+  __str.setf(ios_base::left, ios_base::adjustfield);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& right(ios_base& __str) {
+  __str.setf(ios_base::right, ios_base::adjustfield);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& dec(ios_base& __str) {
+  __str.setf(ios_base::dec, ios_base::basefield);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& hex(ios_base& __str) {
+  __str.setf(ios_base::hex, ios_base::basefield);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& oct(ios_base& __str) {
+  __str.setf(ios_base::oct, ios_base::basefield);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& fixed(ios_base& __str) {
+  __str.setf(ios_base::fixed, ios_base::floatfield);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& scientific(ios_base& __str) {
+  __str.setf(ios_base::scientific, ios_base::floatfield);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& hexfloat(ios_base& __str) {
+  __str.setf(ios_base::fixed | ios_base::scientific, ios_base::floatfield);
+  return __str;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline ios_base& defaultfloat(ios_base& __str) {
+  __str.unsetf(ios_base::floatfield);
+  return __str;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <concepts>
+#  include <cstddef>
+#  include <cstdlib>
+#  include <cstring>
+#  include <initializer_list>
+#  include <limits>
+#  include <mutex>
+#  include <new>
+#  include <stdexcept>
+#  include <system_error>
+#  include <type_traits>
+#  include <typeinfo>
+#endif
+
+#endif // _LIBCPP_IOS
diff --git a/libcxx/include/__cxx03/iosfwd b/libcxx/include/__cxx03/iosfwd
new file mode 100644
index 000000000000..eeafcc37c598
--- /dev/null
+++ b/libcxx/include/__cxx03/iosfwd
@@ -0,0 +1,186 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_IOSFWD
+#define _LIBCPP_IOSFWD
+
+/*
+    iosfwd synopsis
+
+namespace std
+{
+
+template<class charT> struct char_traits;
+template<>            struct char_traits<char>;
+template<>            struct char_traits<char8_t>;  // C++20
+template<>            struct char_traits<char16_t>;
+template<>            struct char_traits<char32_t>;
+template<>            struct char_traits<wchar_t>;
+
+template<class T>     class allocator;
+
+template <class charT, class traits = char_traits<charT> > class basic_ios;
+
+template <class charT, class traits = char_traits<charT> > class basic_streambuf;
+template <class charT, class traits = char_traits<charT> > class basic_istream;
+template <class charT, class traits = char_traits<charT> > class basic_ostream;
+template <class charT, class traits = char_traits<charT> > class basic_iostream;
+
+template <class charT, class traits = char_traits<charT>, class Allocator = allocator<charT> >
+    class basic_stringbuf;
+template <class charT, class traits = char_traits<charT>, class Allocator = allocator<charT> >
+    class basic_istringstream;
+template <class charT, class traits = char_traits<charT>, class Allocator = allocator<charT> >
+    class basic_ostringstream;
+template <class charT, class traits = char_traits<charT>, class Allocator = allocator<charT> >
+    class basic_stringstream;
+
+template <class charT, class traits = char_traits<charT> > class basic_filebuf;
+template <class charT, class traits = char_traits<charT> > class basic_ifstream;
+template <class charT, class traits = char_traits<charT> > class basic_ofstream;
+template <class charT, class traits = char_traits<charT> > class basic_fstream;
+
+template <class charT, class traits = char_traits<charT> > class istreambuf_iterator;
+template <class charT, class traits = char_traits<charT> > class ostreambuf_iterator;
+
+typedef basic_ios<char>              ios;
+typedef basic_ios<wchar_t>           wios;
+
+typedef basic_streambuf<char>        streambuf;
+typedef basic_istream<char>          istream;
+typedef basic_ostream<char>          ostream;
+typedef basic_iostream<char>         iostream;
+
+typedef basic_stringbuf<char>        stringbuf;
+typedef basic_istringstream<char>    istringstream;
+typedef basic_ostringstream<char>    ostringstream;
+typedef basic_stringstream<char>     stringstream;
+
+typedef basic_filebuf<char>          filebuf;
+typedef basic_ifstream<char>         ifstream;
+typedef basic_ofstream<char>         ofstream;
+typedef basic_fstream<char>          fstream;
+
+typedef basic_streambuf<wchar_t>     wstreambuf;
+typedef basic_istream<wchar_t>       wistream;
+typedef basic_ostream<wchar_t>       wostream;
+typedef basic_iostream<wchar_t>      wiostream;
+
+typedef basic_stringbuf<wchar_t>     wstringbuf;
+typedef basic_istringstream<wchar_t> wistringstream;
+typedef basic_ostringstream<wchar_t> wostringstream;
+typedef basic_stringstream<wchar_t>  wstringstream;
+
+typedef basic_filebuf<wchar_t>       wfilebuf;
+typedef basic_ifstream<wchar_t>      wifstream;
+typedef basic_ofstream<wchar_t>      wofstream;
+typedef basic_fstream<wchar_t>       wfstream;
+
+template <class state> class fpos;
+using streampos  = fpos<char_traits<char>::state_type>;
+using wstreampos = fpos<char_traits<wchar_t>::state_type>;
+using u8streampos = fpos<char_traits<char8_t>::state_type>; // C++20
+using u16streampos = fpos<char_traits<char16_t>::state_type>;
+using u32streampos = fpos<char_traits<char32_t>::state_type>;
+
+template <class charT, class traits = char_traits<charT>, class Allocator = allocator<charT>>
+    class basic_syncbuf;                          // C++20
+
+using syncbuf = basic_syncbuf<char>;              // C++20
+using wsyncbuf = basic_syncbuf<wchar_t>;          // C++20
+
+template <class charT, class traits = char_traits<charT>, class Allocator = allocator<charT>>
+    class basic_osyncstream;                      // C++20
+
+using osyncstream = basic_osyncstream<char>;      // C++20
+using wosyncstream = basic_osyncstream<wchar_t>;  // C++20
+
+}  // std
+
+*/
+
+#include <__config>
+#include <__fwd/fstream.h>
+#include <__fwd/ios.h>
+#include <__fwd/istream.h>
+#include <__fwd/memory.h>
+#include <__fwd/ostream.h>
+#include <__fwd/sstream.h>
+#include <__fwd/streambuf.h>
+#include <__fwd/string.h>
+#include <__std_mbstate_t.h>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS istreambuf_iterator;
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS ostreambuf_iterator;
+
+template <class _State>
+class _LIBCPP_TEMPLATE_VIS fpos;
+typedef fpos<mbstate_t> streampos;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+typedef fpos<mbstate_t> wstreampos;
+#endif
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+typedef fpos<mbstate_t> u8streampos;
+#endif
+typedef fpos<mbstate_t> u16streampos;
+typedef fpos<mbstate_t> u32streampos;
+
+#if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM)
+
+template <class _CharT, class _Traits = char_traits<_CharT>, class _Allocator = allocator<_CharT>>
+class basic_syncbuf;
+
+using syncbuf = basic_syncbuf<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using wsyncbuf = basic_syncbuf<wchar_t>;
+#  endif
+
+template <class _CharT, class _Traits = char_traits<_CharT>, class _Allocator = allocator<_CharT>>
+class basic_osyncstream;
+
+using osyncstream = basic_osyncstream<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using wosyncstream = basic_osyncstream<wchar_t>;
+#  endif
+
+#endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM)
+
+template <class _CharT, class _Traits>
+class __save_flags {
+  typedef basic_ios<_CharT, _Traits> __stream_type;
+  typedef typename __stream_type::fmtflags fmtflags;
+
+  __stream_type& __stream_;
+  fmtflags __fmtflags_;
+  _CharT __fill_;
+
+public:
+  __save_flags(const __save_flags&)            = delete;
+  __save_flags& operator=(const __save_flags&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __save_flags(__stream_type& __stream)
+      : __stream_(__stream), __fmtflags_(__stream.flags()), __fill_(__stream.fill()) {}
+  _LIBCPP_HIDE_FROM_ABI ~__save_flags() {
+    __stream_.flags(__fmtflags_);
+    __stream_.fill(__fill_);
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_IOSFWD
diff --git a/libcxx/include/__cxx03/iostream b/libcxx/include/__cxx03/iostream
new file mode 100644
index 000000000000..5df45c6d3f78
--- /dev/null
+++ b/libcxx/include/__cxx03/iostream
@@ -0,0 +1,67 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_IOSTREAM
+#define _LIBCPP_IOSTREAM
+
+/*
+    iostream synopsis
+
+#include <ios>
+#include <istream>
+#include <ostream>
+#include <streambuf>
+
+namespace std {
+
+extern istream cin;
+extern ostream cout;
+extern ostream cerr;
+extern ostream clog;
+extern wistream wcin;
+extern wostream wcout;
+extern wostream wcerr;
+extern wostream wclog;
+
+}  // std
+
+*/
+
+#include <__config>
+#include <version>
+
+// standard-mandated includes
+
+// [iostream.syn]
+#include <ios>
+#include <istream>
+#include <ostream>
+#include <streambuf>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+extern _LIBCPP_EXPORTED_FROM_ABI istream cin;
+extern _LIBCPP_EXPORTED_FROM_ABI ostream cout;
+extern _LIBCPP_EXPORTED_FROM_ABI ostream cerr;
+extern _LIBCPP_EXPORTED_FROM_ABI ostream clog;
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern _LIBCPP_EXPORTED_FROM_ABI wistream wcin;
+extern _LIBCPP_EXPORTED_FROM_ABI wostream wcout;
+extern _LIBCPP_EXPORTED_FROM_ABI wostream wcerr;
+extern _LIBCPP_EXPORTED_FROM_ABI wostream wclog;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_IOSTREAM
diff --git a/libcxx/include/__cxx03/istream b/libcxx/include/__cxx03/istream
new file mode 100644
index 000000000000..d2b577a9ad9e
--- /dev/null
+++ b/libcxx/include/__cxx03/istream
@@ -0,0 +1,1373 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_ISTREAM
+#define _LIBCPP_ISTREAM
+
+/*
+    istream synopsis
+
+template <class charT, class traits = char_traits<charT> >
+class basic_istream
+    : virtual public basic_ios<charT,traits>
+{
+public:
+    // types (inherited from basic_ios (27.5.4)):
+    typedef charT                          char_type;
+    typedef traits                         traits_type;
+    typedef typename traits_type::int_type int_type;
+    typedef typename traits_type::pos_type pos_type;
+    typedef typename traits_type::off_type off_type;
+
+    // 27.7.1.1.1 Constructor/destructor:
+    explicit basic_istream(basic_streambuf<char_type, traits_type>* sb);
+    basic_istream(basic_istream&& rhs);
+    virtual ~basic_istream();
+
+    // 27.7.1.1.2 Assign/swap:
+    basic_istream& operator=(basic_istream&& rhs);
+    void swap(basic_istream& rhs);
+
+    // 27.7.1.1.3 Prefix/suffix:
+    class sentry;
+
+    // 27.7.1.2 Formatted input:
+    basic_istream& operator>>(basic_istream& (*pf)(basic_istream&));
+    basic_istream& operator>>(basic_ios<char_type, traits_type>&
+                              (*pf)(basic_ios<char_type, traits_type>&));
+    basic_istream& operator>>(ios_base& (*pf)(ios_base&));
+    basic_istream& operator>>(basic_streambuf<char_type, traits_type>* sb);
+    basic_istream& operator>>(bool& n);
+    basic_istream& operator>>(short& n);
+    basic_istream& operator>>(unsigned short& n);
+    basic_istream& operator>>(int& n);
+    basic_istream& operator>>(unsigned int& n);
+    basic_istream& operator>>(long& n);
+    basic_istream& operator>>(unsigned long& n);
+    basic_istream& operator>>(long long& n);
+    basic_istream& operator>>(unsigned long long& n);
+    basic_istream& operator>>(float& f);
+    basic_istream& operator>>(double& f);
+    basic_istream& operator>>(long double& f);
+    basic_istream& operator>>(void*& p);
+
+    // 27.7.1.3 Unformatted input:
+    streamsize gcount() const;
+    int_type get();
+    basic_istream& get(char_type& c);
+    basic_istream& get(char_type* s, streamsize n);
+    basic_istream& get(char_type* s, streamsize n, char_type delim);
+    basic_istream& get(basic_streambuf<char_type,traits_type>& sb);
+    basic_istream& get(basic_streambuf<char_type,traits_type>& sb, char_type delim);
+
+    basic_istream& getline(char_type* s, streamsize n);
+    basic_istream& getline(char_type* s, streamsize n, char_type delim);
+
+    basic_istream& ignore(streamsize n = 1, int_type delim = traits_type::eof());
+    int_type peek();
+    basic_istream& read (char_type* s, streamsize n);
+    streamsize readsome(char_type* s, streamsize n);
+
+    basic_istream& putback(char_type c);
+    basic_istream& unget();
+    int sync();
+
+    pos_type tellg();
+    basic_istream& seekg(pos_type);
+    basic_istream& seekg(off_type, ios_base::seekdir);
+protected:
+    basic_istream(const basic_istream& rhs) = delete;
+    basic_istream(basic_istream&& rhs);
+    // 27.7.2.1.2 Assign/swap:
+    basic_istream& operator=(const basic_istream& rhs) = delete;
+    basic_istream& operator=(basic_istream&& rhs);
+    void swap(basic_istream& rhs);
+};
+
+// 27.7.1.2.3 character extraction templates:
+template<class charT, class traits>
+  basic_istream<charT,traits>& operator>>(basic_istream<charT,traits>&, charT&);
+
+template<class traits>
+  basic_istream<char,traits>& operator>>(basic_istream<char,traits>&, unsigned char&);
+
+template<class traits>
+  basic_istream<char,traits>& operator>>(basic_istream<char,traits>&, signed char&);
+
+template<class charT, class traits>
+  basic_istream<charT,traits>& operator>>(basic_istream<charT,traits>&, charT*);
+
+template<class traits>
+  basic_istream<char,traits>& operator>>(basic_istream<char,traits>&, unsigned char*);
+
+template<class traits>
+  basic_istream<char,traits>& operator>>(basic_istream<char,traits>&, signed char*);
+
+template <class charT, class traits>
+  void
+  swap(basic_istream<charT, traits>& x, basic_istream<charT, traits>& y);
+
+typedef basic_istream<char> istream;
+typedef basic_istream<wchar_t> wistream;
+
+template <class charT, class traits = char_traits<charT> >
+class basic_iostream :
+    public basic_istream<charT,traits>,
+    public basic_ostream<charT,traits>
+{
+public:
+    // types:
+    typedef charT                          char_type;
+    typedef traits                         traits_type;
+    typedef typename traits_type::int_type int_type;
+    typedef typename traits_type::pos_type pos_type;
+    typedef typename traits_type::off_type off_type;
+
+    // constructor/destructor
+    explicit basic_iostream(basic_streambuf<char_type, traits_type>* sb);
+    basic_iostream(basic_iostream&& rhs);
+    virtual ~basic_iostream();
+
+    // assign/swap
+    basic_iostream& operator=(basic_iostream&& rhs);
+    void swap(basic_iostream& rhs);
+};
+
+template <class charT, class traits>
+  void
+  swap(basic_iostream<charT, traits>& x, basic_iostream<charT, traits>& y);
+
+typedef basic_iostream<char> iostream;
+typedef basic_iostream<wchar_t> wiostream;
+
+template <class charT, class traits>
+  basic_istream<charT,traits>&
+  ws(basic_istream<charT,traits>& is);
+
+// rvalue stream extraction
+template <class Stream, class T>
+  Stream&& operator>>(Stream&& is, T&& x);
+
+}  // std
+
+*/
+
+#include <__config>
+#include <__fwd/istream.h>
+#include <__iterator/istreambuf_iterator.h>
+#include <__ostream/basic_ostream.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_base_of.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <bitset>
+#include <ios>
+#include <locale>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS basic_istream : virtual public basic_ios<_CharT, _Traits> {
+  streamsize __gc_;
+
+  _LIBCPP_HIDE_FROM_ABI void __inc_gcount() {
+    if (__gc_ < numeric_limits<streamsize>::max())
+      ++__gc_;
+  }
+
+public:
+  // types (inherited from basic_ios (27.5.4)):
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+
+  // 27.7.1.1.1 Constructor/destructor:
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 explicit basic_istream(basic_streambuf<char_type, traits_type>* __sb)
+      : __gc_(0) {
+    this->init(__sb);
+  }
+  ~basic_istream() override;
+
+protected:
+  inline _LIBCPP_HIDE_FROM_ABI basic_istream(basic_istream&& __rhs);
+
+  // 27.7.1.1.2 Assign/swap:
+  inline _LIBCPP_HIDE_FROM_ABI basic_istream& operator=(basic_istream&& __rhs);
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void swap(basic_istream& __rhs) {
+    std::swap(__gc_, __rhs.__gc_);
+    basic_ios<char_type, traits_type>::swap(__rhs);
+  }
+
+public:
+  basic_istream(const basic_istream& __rhs)            = delete;
+  basic_istream& operator=(const basic_istream& __rhs) = delete;
+
+  // 27.7.1.1.3 Prefix/suffix:
+  class _LIBCPP_TEMPLATE_VIS sentry;
+
+  // 27.7.1.2 Formatted input:
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& operator>>(basic_istream& (*__pf)(basic_istream&)) {
+    return __pf(*this);
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream&
+  operator>>(basic_ios<char_type, traits_type>& (*__pf)(basic_ios<char_type, traits_type>&)) {
+    __pf(*this);
+    return *this;
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& operator>>(ios_base& (*__pf)(ios_base&)) {
+    __pf(*this);
+    return *this;
+  }
+
+  basic_istream& operator>>(basic_streambuf<char_type, traits_type>* __sb);
+  basic_istream& operator>>(bool& __n);
+  basic_istream& operator>>(short& __n);
+  basic_istream& operator>>(unsigned short& __n);
+  basic_istream& operator>>(int& __n);
+  basic_istream& operator>>(unsigned int& __n);
+  basic_istream& operator>>(long& __n);
+  basic_istream& operator>>(unsigned long& __n);
+  basic_istream& operator>>(long long& __n);
+  basic_istream& operator>>(unsigned long long& __n);
+  basic_istream& operator>>(float& __f);
+  basic_istream& operator>>(double& __f);
+  basic_istream& operator>>(long double& __f);
+  basic_istream& operator>>(void*& __p);
+
+  // 27.7.1.3 Unformatted input:
+  _LIBCPP_HIDE_FROM_ABI streamsize gcount() const { return __gc_; }
+  int_type get();
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& get(char_type& __c) {
+    int_type __ch = get();
+    if (__ch != traits_type::eof())
+      __c = traits_type::to_char_type(__ch);
+    return *this;
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& get(char_type* __s, streamsize __n) {
+    return get(__s, __n, this->widen('\n'));
+  }
+
+  basic_istream& get(char_type* __s, streamsize __n, char_type __dlm);
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& get(basic_streambuf<char_type, traits_type>& __sb) {
+    return get(__sb, this->widen('\n'));
+  }
+
+  basic_istream& get(basic_streambuf<char_type, traits_type>& __sb, char_type __dlm);
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& getline(char_type* __s, streamsize __n) {
+    return getline(__s, __n, this->widen('\n'));
+  }
+
+  basic_istream& getline(char_type* __s, streamsize __n, char_type __dlm);
+
+  basic_istream& ignore(streamsize __n = 1, int_type __dlm = traits_type::eof());
+  int_type peek();
+  basic_istream& read(char_type* __s, streamsize __n);
+  streamsize readsome(char_type* __s, streamsize __n);
+
+  basic_istream& putback(char_type __c);
+  basic_istream& unget();
+  int sync();
+
+  pos_type tellg();
+  basic_istream& seekg(pos_type __pos);
+  basic_istream& seekg(off_type __off, ios_base::seekdir __dir);
+};
+
+template <class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS basic_istream<_CharT, _Traits>::sentry {
+  bool __ok_;
+
+public:
+  explicit sentry(basic_istream<_CharT, _Traits>& __is, bool __noskipws = false);
+  //    ~sentry() = default;
+
+  _LIBCPP_HIDE_FROM_ABI explicit operator bool() const { return __ok_; }
+
+  sentry(const sentry&)            = delete;
+  sentry& operator=(const sentry&) = delete;
+};
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>::sentry::sentry(basic_istream<_CharT, _Traits>& __is, bool __noskipws) : __ok_(false) {
+  if (__is.good()) {
+    if (__is.tie())
+      __is.tie()->flush();
+    if (!__noskipws && (__is.flags() & ios_base::skipws)) {
+      typedef istreambuf_iterator<_CharT, _Traits> _Ip;
+      const ctype<_CharT>& __ct = std::use_facet<ctype<_CharT> >(__is.getloc());
+      _Ip __i(__is);
+      _Ip __eof;
+      for (; __i != __eof; ++__i)
+        if (!__ct.is(__ct.space, *__i))
+          break;
+      if (__i == __eof)
+        __is.setstate(ios_base::failbit | ios_base::eofbit);
+    }
+    __ok_ = __is.good();
+  } else
+    __is.setstate(ios_base::failbit);
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>::basic_istream(basic_istream&& __rhs) : __gc_(__rhs.__gc_) {
+  __rhs.__gc_ = 0;
+  this->move(__rhs);
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::operator=(basic_istream&& __rhs) {
+  swap(__rhs);
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>::~basic_istream() {}
+
+template <class _Tp, class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+__input_arithmetic(basic_istream<_CharT, _Traits>& __is, _Tp& __n) {
+  ios_base::iostate __state = ios_base::goodbit;
+  typename basic_istream<_CharT, _Traits>::sentry __s(__is);
+  if (__s) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      typedef istreambuf_iterator<_CharT, _Traits> _Ip;
+      typedef num_get<_CharT, _Ip> _Fp;
+      std::use_facet<_Fp>(__is.getloc()).get(_Ip(__is), _Ip(), __is, __state, __n);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      __is.__setstate_nothrow(__state);
+      if (__is.exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif
+    __is.setstate(__state);
+  }
+  return __is;
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::operator>>(unsigned short& __n) {
+  return std::__input_arithmetic<unsigned short>(*this, __n);
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::operator>>(unsigned int& __n) {
+  return std::__input_arithmetic<unsigned int>(*this, __n);
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::operator>>(long& __n) {
+  return std::__input_arithmetic<long>(*this, __n);
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::operator>>(unsigned long& __n) {
+  return std::__input_arithmetic<unsigned long>(*this, __n);
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::operator>>(long long& __n) {
+  return std::__input_arithmetic<long long>(*this, __n);
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::operator>>(unsigned long long& __n) {
+  return std::__input_arithmetic<unsigned long long>(*this, __n);
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::operator>>(float& __n) {
+  return std::__input_arithmetic<float>(*this, __n);
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::operator>>(double& __n) {
+  return std::__input_arithmetic<double>(*this, __n);
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::operator>>(long double& __n) {
+  return std::__input_arithmetic<long double>(*this, __n);
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::operator>>(bool& __n) {
+  return std::__input_arithmetic<bool>(*this, __n);
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::operator>>(void*& __n) {
+  return std::__input_arithmetic<void*>(*this, __n);
+}
+
+template <class _Tp, class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+__input_arithmetic_with_numeric_limits(basic_istream<_CharT, _Traits>& __is, _Tp& __n) {
+  ios_base::iostate __state = ios_base::goodbit;
+  typename basic_istream<_CharT, _Traits>::sentry __s(__is);
+  if (__s) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      typedef istreambuf_iterator<_CharT, _Traits> _Ip;
+      typedef num_get<_CharT, _Ip> _Fp;
+      long __temp;
+      std::use_facet<_Fp>(__is.getloc()).get(_Ip(__is), _Ip(), __is, __state, __temp);
+      if (__temp < numeric_limits<_Tp>::min()) {
+        __state |= ios_base::failbit;
+        __n = numeric_limits<_Tp>::min();
+      } else if (__temp > numeric_limits<_Tp>::max()) {
+        __state |= ios_base::failbit;
+        __n = numeric_limits<_Tp>::max();
+      } else {
+        __n = static_cast<_Tp>(__temp);
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      __is.__setstate_nothrow(__state);
+      if (__is.exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __is.setstate(__state);
+  }
+  return __is;
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::operator>>(short& __n) {
+  return std::__input_arithmetic_with_numeric_limits<short>(*this, __n);
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::operator>>(int& __n) {
+  return std::__input_arithmetic_with_numeric_limits<int>(*this, __n);
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+__input_c_string(basic_istream<_CharT, _Traits>& __is, _CharT* __p, size_t __n) {
+  ios_base::iostate __state = ios_base::goodbit;
+  typename basic_istream<_CharT, _Traits>::sentry __sen(__is);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif
+      _CharT* __s               = __p;
+      const ctype<_CharT>& __ct = std::use_facet<ctype<_CharT> >(__is.getloc());
+      while (__s != __p + (__n - 1)) {
+        typename _Traits::int_type __i = __is.rdbuf()->sgetc();
+        if (_Traits::eq_int_type(__i, _Traits::eof())) {
+          __state |= ios_base::eofbit;
+          break;
+        }
+        _CharT __ch = _Traits::to_char_type(__i);
+        if (__ct.is(__ct.space, __ch))
+          break;
+        *__s++ = __ch;
+        __is.rdbuf()->sbumpc();
+      }
+      *__s = _CharT();
+      __is.width(0);
+      if (__s == __p)
+        __state |= ios_base::failbit;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      __is.__setstate_nothrow(__state);
+      if (__is.exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif
+    __is.setstate(__state);
+  }
+  return __is;
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _CharT, class _Traits, size_t _Np>
+inline _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, _CharT (&__buf)[_Np]) {
+  size_t __n = _Np;
+  if (__is.width() > 0)
+    __n = std::min(size_t(__is.width()), _Np);
+  return std::__input_c_string(__is, __buf, __n);
+}
+
+template <class _Traits, size_t _Np>
+inline _LIBCPP_HIDE_FROM_ABI basic_istream<char, _Traits>&
+operator>>(basic_istream<char, _Traits>& __is, unsigned char (&__buf)[_Np]) {
+  return __is >> (char(&)[_Np])__buf;
+}
+
+template <class _Traits, size_t _Np>
+inline _LIBCPP_HIDE_FROM_ABI basic_istream<char, _Traits>&
+operator>>(basic_istream<char, _Traits>& __is, signed char (&__buf)[_Np]) {
+  return __is >> (char(&)[_Np])__buf;
+}
+
+#else
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, _CharT* __s) {
+  streamsize __n = __is.width();
+  if (__n <= 0)
+    __n = numeric_limits<streamsize>::max() / sizeof(_CharT) - 1;
+  return std::__input_c_string(__is, __s, size_t(__n));
+}
+
+template <class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI basic_istream<char, _Traits>&
+operator>>(basic_istream<char, _Traits>& __is, unsigned char* __s) {
+  return __is >> (char*)__s;
+}
+
+template <class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI basic_istream<char, _Traits>&
+operator>>(basic_istream<char, _Traits>& __is, signed char* __s) {
+  return __is >> (char*)__s;
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>& operator>>(basic_istream<_CharT, _Traits>& __is, _CharT& __c) {
+  ios_base::iostate __state = ios_base::goodbit;
+  typename basic_istream<_CharT, _Traits>::sentry __sen(__is);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif
+      typename _Traits::int_type __i = __is.rdbuf()->sbumpc();
+      if (_Traits::eq_int_type(__i, _Traits::eof()))
+        __state |= ios_base::eofbit | ios_base::failbit;
+      else
+        __c = _Traits::to_char_type(__i);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      __is.__setstate_nothrow(__state);
+      if (__is.exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif
+    __is.setstate(__state);
+  }
+  return __is;
+}
+
+template <class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI basic_istream<char, _Traits>&
+operator>>(basic_istream<char, _Traits>& __is, unsigned char& __c) {
+  return __is >> (char&)__c;
+}
+
+template <class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI basic_istream<char, _Traits>&
+operator>>(basic_istream<char, _Traits>& __is, signed char& __c) {
+  return __is >> (char&)__c;
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>&
+basic_istream<_CharT, _Traits>::operator>>(basic_streambuf<char_type, traits_type>* __sb) {
+  ios_base::iostate __state = ios_base::goodbit;
+  __gc_                     = 0;
+  sentry __s(*this, true);
+  if (__s) {
+    if (__sb) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+        while (true) {
+          typename traits_type::int_type __i = this->rdbuf()->sgetc();
+          if (traits_type::eq_int_type(__i, _Traits::eof())) {
+            __state |= ios_base::eofbit;
+            break;
+          }
+          if (traits_type::eq_int_type(__sb->sputc(traits_type::to_char_type(__i)), traits_type::eof()))
+            break;
+          __inc_gcount();
+          this->rdbuf()->sbumpc();
+        }
+        if (__gc_ == 0)
+          __state |= ios_base::failbit;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      } catch (...) {
+        __state |= ios_base::badbit;
+        if (__gc_ == 0)
+          __state |= ios_base::failbit;
+
+        this->__setstate_nothrow(__state);
+        if (this->exceptions() & ios_base::failbit || this->exceptions() & ios_base::badbit) {
+          throw;
+        }
+      }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    } else {
+      __state |= ios_base::failbit;
+    }
+    this->setstate(__state);
+  }
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+typename basic_istream<_CharT, _Traits>::int_type basic_istream<_CharT, _Traits>::get() {
+  ios_base::iostate __state = ios_base::goodbit;
+  __gc_                     = 0;
+  int_type __r              = traits_type::eof();
+  sentry __s(*this, true);
+  if (__s) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif
+      __r = this->rdbuf()->sbumpc();
+      if (traits_type::eq_int_type(__r, traits_type::eof()))
+        __state |= ios_base::failbit | ios_base::eofbit;
+      else
+        __gc_ = 1;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      this->__setstate_nothrow(this->rdstate() | ios_base::badbit);
+      if (this->exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif
+    this->setstate(__state);
+  }
+  return __r;
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::get(char_type* __s, streamsize __n, char_type __dlm) {
+  ios_base::iostate __state = ios_base::goodbit;
+  __gc_                     = 0;
+  sentry __sen(*this, true);
+  if (__sen) {
+    if (__n > 0) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      try {
+#endif
+        while (__gc_ < __n - 1) {
+          int_type __i = this->rdbuf()->sgetc();
+          if (traits_type::eq_int_type(__i, traits_type::eof())) {
+            __state |= ios_base::eofbit;
+            break;
+          }
+          char_type __ch = traits_type::to_char_type(__i);
+          if (traits_type::eq(__ch, __dlm))
+            break;
+          *__s++ = __ch;
+          __inc_gcount();
+          this->rdbuf()->sbumpc();
+        }
+        if (__gc_ == 0)
+          __state |= ios_base::failbit;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      } catch (...) {
+        __state |= ios_base::badbit;
+        this->__setstate_nothrow(__state);
+        if (this->exceptions() & ios_base::badbit) {
+          if (__n > 0)
+            *__s = char_type();
+          throw;
+        }
+      }
+#endif
+    } else {
+      __state |= ios_base::failbit;
+    }
+
+    if (__n > 0)
+      *__s = char_type();
+    this->setstate(__state);
+  }
+  if (__n > 0)
+    *__s = char_type();
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>&
+basic_istream<_CharT, _Traits>::get(basic_streambuf<char_type, traits_type>& __sb, char_type __dlm) {
+  ios_base::iostate __state = ios_base::goodbit;
+  __gc_                     = 0;
+  sentry __sen(*this, true);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      while (true) {
+        typename traits_type::int_type __i = this->rdbuf()->sgetc();
+        if (traits_type::eq_int_type(__i, traits_type::eof())) {
+          __state |= ios_base::eofbit;
+          break;
+        }
+        char_type __ch = traits_type::to_char_type(__i);
+        if (traits_type::eq(__ch, __dlm))
+          break;
+        if (traits_type::eq_int_type(__sb.sputc(__ch), traits_type::eof()))
+          break;
+        __inc_gcount();
+        this->rdbuf()->sbumpc();
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      // according to the spec, exceptions here are caught but not rethrown
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    if (__gc_ == 0)
+      __state |= ios_base::failbit;
+    this->setstate(__state);
+  }
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>&
+basic_istream<_CharT, _Traits>::getline(char_type* __s, streamsize __n, char_type __dlm) {
+  ios_base::iostate __state = ios_base::goodbit;
+  __gc_                     = 0;
+  sentry __sen(*this, true);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      while (true) {
+        typename traits_type::int_type __i = this->rdbuf()->sgetc();
+        if (traits_type::eq_int_type(__i, traits_type::eof())) {
+          __state |= ios_base::eofbit;
+          break;
+        }
+        char_type __ch = traits_type::to_char_type(__i);
+        if (traits_type::eq(__ch, __dlm)) {
+          this->rdbuf()->sbumpc();
+          __inc_gcount();
+          break;
+        }
+        if (__gc_ >= __n - 1) {
+          __state |= ios_base::failbit;
+          break;
+        }
+        *__s++ = __ch;
+        this->rdbuf()->sbumpc();
+        __inc_gcount();
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      this->__setstate_nothrow(__state);
+      if (this->exceptions() & ios_base::badbit) {
+        if (__n > 0)
+          *__s = char_type();
+        if (__gc_ == 0)
+          __state |= ios_base::failbit;
+        throw;
+      }
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+  if (__n > 0)
+    *__s = char_type();
+  if (__gc_ == 0)
+    __state |= ios_base::failbit;
+  this->setstate(__state);
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::ignore(streamsize __n, int_type __dlm) {
+  ios_base::iostate __state = ios_base::goodbit;
+  __gc_                     = 0;
+  sentry __sen(*this, true);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      if (__n == numeric_limits<streamsize>::max()) {
+        while (true) {
+          typename traits_type::int_type __i = this->rdbuf()->sbumpc();
+          if (traits_type::eq_int_type(__i, traits_type::eof())) {
+            __state |= ios_base::eofbit;
+            break;
+          }
+          __inc_gcount();
+          if (traits_type::eq_int_type(__i, __dlm))
+            break;
+        }
+      } else {
+        while (__gc_ < __n) {
+          typename traits_type::int_type __i = this->rdbuf()->sbumpc();
+          if (traits_type::eq_int_type(__i, traits_type::eof())) {
+            __state |= ios_base::eofbit;
+            break;
+          }
+          __inc_gcount();
+          if (traits_type::eq_int_type(__i, __dlm))
+            break;
+        }
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      this->__setstate_nothrow(__state);
+      if (this->exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    this->setstate(__state);
+  }
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+typename basic_istream<_CharT, _Traits>::int_type basic_istream<_CharT, _Traits>::peek() {
+  ios_base::iostate __state = ios_base::goodbit;
+  __gc_                     = 0;
+  int_type __r              = traits_type::eof();
+  sentry __sen(*this, true);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      __r = this->rdbuf()->sgetc();
+      if (traits_type::eq_int_type(__r, traits_type::eof()))
+        __state |= ios_base::eofbit;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      this->__setstate_nothrow(__state);
+      if (this->exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    this->setstate(__state);
+  }
+  return __r;
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::read(char_type* __s, streamsize __n) {
+  ios_base::iostate __state = ios_base::goodbit;
+  __gc_                     = 0;
+  sentry __sen(*this, true);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      __gc_ = this->rdbuf()->sgetn(__s, __n);
+      if (__gc_ != __n)
+        __state |= ios_base::failbit | ios_base::eofbit;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      this->__setstate_nothrow(__state);
+      if (this->exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  } else {
+    __state |= ios_base::failbit;
+  }
+  this->setstate(__state);
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+streamsize basic_istream<_CharT, _Traits>::readsome(char_type* __s, streamsize __n) {
+  ios_base::iostate __state = ios_base::goodbit;
+  __gc_                     = 0;
+  sentry __sen(*this, true);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      streamsize __c = this->rdbuf()->in_avail();
+      switch (__c) {
+      case -1:
+        __state |= ios_base::eofbit;
+        break;
+      case 0:
+        break;
+      default:
+        __n   = std::min(__c, __n);
+        __gc_ = this->rdbuf()->sgetn(__s, __n);
+        if (__gc_ != __n)
+          __state |= ios_base::failbit | ios_base::eofbit;
+        break;
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      this->__setstate_nothrow(__state);
+      if (this->exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  } else {
+    __state |= ios_base::failbit;
+  }
+  this->setstate(__state);
+  return __gc_;
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::putback(char_type __c) {
+  ios_base::iostate __state = this->rdstate() & ~ios_base::eofbit;
+  __gc_                     = 0;
+  this->clear(__state);
+  sentry __sen(*this, true);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      if (this->rdbuf() == nullptr || this->rdbuf()->sputbackc(__c) == traits_type::eof())
+        __state |= ios_base::badbit;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      this->__setstate_nothrow(__state);
+      if (this->exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  } else {
+    __state |= ios_base::failbit;
+  }
+  this->setstate(__state);
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::unget() {
+  ios_base::iostate __state = this->rdstate() & ~ios_base::eofbit;
+  __gc_                     = 0;
+  this->clear(__state);
+  sentry __sen(*this, true);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      if (this->rdbuf() == nullptr || this->rdbuf()->sungetc() == traits_type::eof())
+        __state |= ios_base::badbit;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      this->__setstate_nothrow(__state);
+      if (this->exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  } else {
+    __state |= ios_base::failbit;
+  }
+  this->setstate(__state);
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+int basic_istream<_CharT, _Traits>::sync() {
+  ios_base::iostate __state = ios_base::goodbit;
+  sentry __sen(*this, true);
+  if (this->rdbuf() == nullptr)
+    return -1;
+
+  int __r = 0;
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      if (this->rdbuf()->pubsync() == -1) {
+        __state |= ios_base::badbit;
+        __r = -1;
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      this->__setstate_nothrow(__state);
+      if (this->exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    this->setstate(__state);
+  }
+  return __r;
+}
+
+template <class _CharT, class _Traits>
+typename basic_istream<_CharT, _Traits>::pos_type basic_istream<_CharT, _Traits>::tellg() {
+  ios_base::iostate __state = ios_base::goodbit;
+  pos_type __r(-1);
+  sentry __sen(*this, true);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      __r = this->rdbuf()->pubseekoff(0, ios_base::cur, ios_base::in);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      this->__setstate_nothrow(__state);
+      if (this->exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    this->setstate(__state);
+  }
+  return __r;
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::seekg(pos_type __pos) {
+  ios_base::iostate __state = this->rdstate() & ~ios_base::eofbit;
+  this->clear(__state);
+  sentry __sen(*this, true);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      if (this->rdbuf()->pubseekpos(__pos, ios_base::in) == pos_type(-1))
+        __state |= ios_base::failbit;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      this->__setstate_nothrow(__state);
+      if (this->exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    this->setstate(__state);
+  }
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::seekg(off_type __off, ios_base::seekdir __dir) {
+  ios_base::iostate __state = this->rdstate() & ~ios_base::eofbit;
+  this->clear(__state);
+  sentry __sen(*this, true);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      if (this->rdbuf()->pubseekoff(__off, __dir, ios_base::in) == pos_type(-1))
+        __state |= ios_base::failbit;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      this->__setstate_nothrow(__state);
+      if (this->exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    this->setstate(__state);
+  }
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>& ws(basic_istream<_CharT, _Traits>& __is) {
+  ios_base::iostate __state = ios_base::goodbit;
+  typename basic_istream<_CharT, _Traits>::sentry __sen(__is, true);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      const ctype<_CharT>& __ct = std::use_facet<ctype<_CharT> >(__is.getloc());
+      while (true) {
+        typename _Traits::int_type __i = __is.rdbuf()->sgetc();
+        if (_Traits::eq_int_type(__i, _Traits::eof())) {
+          __state |= ios_base::eofbit;
+          break;
+        }
+        if (!__ct.is(__ct.space, _Traits::to_char_type(__i)))
+          break;
+        __is.rdbuf()->sbumpc();
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      __is.__setstate_nothrow(__state);
+      if (__is.exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __is.setstate(__state);
+  }
+  return __is;
+}
+
+template <class _Stream, class _Tp, class = void>
+struct __is_istreamable : false_type {};
+
+template <class _Stream, class _Tp>
+struct __is_istreamable<_Stream, _Tp, decltype(std::declval<_Stream>() >> std::declval<_Tp>(), void())> : true_type {};
+
+template <class _Stream,
+          class _Tp,
+          __enable_if_t< _And<is_base_of<ios_base, _Stream>, __is_istreamable<_Stream&, _Tp&&> >::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _Stream&& operator>>(_Stream&& __is, _Tp&& __x) {
+  __is >> std::forward<_Tp>(__x);
+  return std::move(__is);
+}
+
+template <class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS basic_iostream
+    : public basic_istream<_CharT, _Traits>,
+      public basic_ostream<_CharT, _Traits> {
+public:
+  // types:
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+
+  // constructor/destructor
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 explicit basic_iostream(basic_streambuf<char_type, traits_type>* __sb)
+      : basic_istream<_CharT, _Traits>(__sb) {}
+
+  ~basic_iostream() override;
+
+protected:
+  inline _LIBCPP_HIDE_FROM_ABI basic_iostream(basic_iostream&& __rhs);
+
+  // assign/swap
+  inline _LIBCPP_HIDE_FROM_ABI basic_iostream& operator=(basic_iostream&& __rhs);
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void swap(basic_iostream& __rhs) {
+    basic_istream<char_type, traits_type>::swap(__rhs);
+  }
+};
+
+template <class _CharT, class _Traits>
+basic_iostream<_CharT, _Traits>::basic_iostream(basic_iostream&& __rhs)
+    : basic_istream<_CharT, _Traits>(std::move(__rhs)) {}
+
+template <class _CharT, class _Traits>
+basic_iostream<_CharT, _Traits>& basic_iostream<_CharT, _Traits>::operator=(basic_iostream&& __rhs) {
+  swap(__rhs);
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+basic_iostream<_CharT, _Traits>::~basic_iostream() {}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _Allocator>& __str) {
+  ios_base::iostate __state = ios_base::goodbit;
+  typename basic_istream<_CharT, _Traits>::sentry __sen(__is);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif
+      __str.clear();
+      streamsize __n = __is.width();
+      if (__n <= 0)
+        __n = __str.max_size();
+      if (__n <= 0)
+        __n = numeric_limits<streamsize>::max();
+      streamsize __c            = 0;
+      const ctype<_CharT>& __ct = std::use_facet<ctype<_CharT> >(__is.getloc());
+      while (__c < __n) {
+        typename _Traits::int_type __i = __is.rdbuf()->sgetc();
+        if (_Traits::eq_int_type(__i, _Traits::eof())) {
+          __state |= ios_base::eofbit;
+          break;
+        }
+        _CharT __ch = _Traits::to_char_type(__i);
+        if (__ct.is(__ct.space, __ch))
+          break;
+        __str.push_back(__ch);
+        ++__c;
+        __is.rdbuf()->sbumpc();
+      }
+      __is.width(0);
+      if (__c == 0)
+        __state |= ios_base::failbit;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      __is.__setstate_nothrow(__state);
+      if (__is.exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif
+    __is.setstate(__state);
+  }
+  return __is;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+getline(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _Allocator>& __str, _CharT __dlm) {
+  ios_base::iostate __state = ios_base::goodbit;
+  typename basic_istream<_CharT, _Traits>::sentry __sen(__is, true);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif
+      __str.clear();
+      streamsize __extr = 0;
+      while (true) {
+        typename _Traits::int_type __i = __is.rdbuf()->sbumpc();
+        if (_Traits::eq_int_type(__i, _Traits::eof())) {
+          __state |= ios_base::eofbit;
+          break;
+        }
+        ++__extr;
+        _CharT __ch = _Traits::to_char_type(__i);
+        if (_Traits::eq(__ch, __dlm))
+          break;
+        __str.push_back(__ch);
+        if (__str.size() == __str.max_size()) {
+          __state |= ios_base::failbit;
+          break;
+        }
+      }
+      if (__extr == 0)
+        __state |= ios_base::failbit;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      __is.__setstate_nothrow(__state);
+      if (__is.exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif
+    __is.setstate(__state);
+  }
+  return __is;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+getline(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _Allocator>& __str) {
+  return std::getline(__is, __str, __is.widen('\n'));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+getline(basic_istream<_CharT, _Traits>&& __is, basic_string<_CharT, _Traits, _Allocator>& __str, _CharT __dlm) {
+  return std::getline(__is, __str, __dlm);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+getline(basic_istream<_CharT, _Traits>&& __is, basic_string<_CharT, _Traits, _Allocator>& __str) {
+  return std::getline(__is, __str, __is.widen('\n'));
+}
+
+template <class _CharT, class _Traits, size_t _Size>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, bitset<_Size>& __x) {
+  ios_base::iostate __state = ios_base::goodbit;
+  typename basic_istream<_CharT, _Traits>::sentry __sen(__is);
+  if (__sen) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif
+      basic_string<_CharT, _Traits> __str;
+      const ctype<_CharT>& __ct = std::use_facet<ctype<_CharT> >(__is.getloc());
+      size_t __c                = 0;
+      _CharT __zero             = __ct.widen('0');
+      _CharT __one              = __ct.widen('1');
+      while (__c != _Size) {
+        typename _Traits::int_type __i = __is.rdbuf()->sgetc();
+        if (_Traits::eq_int_type(__i, _Traits::eof())) {
+          __state |= ios_base::eofbit;
+          break;
+        }
+        _CharT __ch = _Traits::to_char_type(__i);
+        if (!_Traits::eq(__ch, __zero) && !_Traits::eq(__ch, __one))
+          break;
+        __str.push_back(__ch);
+        ++__c;
+        __is.rdbuf()->sbumpc();
+      }
+      __x = bitset<_Size>(__str);
+      if (_Size > 0 && __c == 0)
+        __state |= ios_base::failbit;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __state |= ios_base::badbit;
+      __is.__setstate_nothrow(__state);
+      if (__is.exceptions() & ios_base::badbit) {
+        throw;
+      }
+    }
+#endif
+    __is.setstate(__state);
+  }
+  return __is;
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_istream<char>;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_istream<wchar_t>;
+#endif
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_iostream<char>;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <concepts>
+#  include <iosfwd>
+#  include <ostream>
+#  include <type_traits>
+#endif
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP_ISTREAM
diff --git a/libcxx/include/__cxx03/iterator b/libcxx/include/__cxx03/iterator
new file mode 100644
index 000000000000..fca75f0a19ed
--- /dev/null
+++ b/libcxx/include/__cxx03/iterator
@@ -0,0 +1,754 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_ITERATOR
+#define _LIBCPP_ITERATOR
+
+/*
+    iterator synopsis
+
+#include <concepts>
+
+namespace std
+{
+template<class> struct incrementable_traits;       // since C++20
+template<class T>
+  using iter_difference_t = see below;             // since C++20
+
+template<class> struct indirectly_readable_traits; // since C++20
+template<class T>
+  using iter_value_t = see below;                  // since C++20
+
+template<class Iterator>
+struct iterator_traits;
+
+template<class T>
+  requires is_object_v<T>                    // since C++20
+struct iterator_traits<T*>;
+
+template<dereferenceable T>
+  using iter_reference_t = decltype(*declval<T&>());
+
+namespace ranges::inline unspecified {
+    inline constexpr unspecified iter_move = unspecified; // since C++20, nodiscard as an extension
+}}
+
+template<dereferenceable T>
+  requires ...
+using iter_rvalue_reference_t = decltype(ranges::iter_move(declval<T&>())); // since C++20
+
+// [iterator.concepts], iterator concepts
+// [iterator.concept.readable], concept indirectly_readable
+template<class In>
+  concept indirectly_readable = see below;                      // since C++20
+
+template<indirectly_readable T>
+  using iter_common_reference_t =
+    common_reference_t<iter_reference_t<T>, iter_value_t<T>&>;  // since C++20
+
+// [iterator.concept.writable], concept indirectly_writable
+template<class Out, class T>
+  concept indirectly_writable = see below;                // since C++20
+
+// [iterator.concept.winc], concept weakly_incrementable
+template<class I>
+  concept weakly_incrementable = see below;                // since C++20
+
+// [iterator.concept.inc], concept incrementable
+template<class I>
+  concept incrementable = see below;                       // since C++20
+
+// [iterator.concept.iterator], concept input_or_output_iterator
+  template<class I>
+    concept input_or_output_iterator = see below;          // since C++20
+
+// [iterator.concept.sentinel], concept sentinel_for
+template<class S, class I>
+  concept sentinel_for = see below;                        // since C++20
+
+// [iterator.concept.sizedsentinel], concept sized_sentinel_for
+template<class S, class I>
+  inline constexpr bool disable_sized_sentinel_for = false;
+
+template<class S, class I>
+  concept sized_sentinel_for = see below;
+
+// [iterator.concept.input], concept input_iterator
+template<class I>
+  concept input_iterator = see below;                      // since C++20
+
+// [iterator.concept.output], concept output_iterator
+template<class I, class T>
+  concept output_iterator = see below;                     // since C++20
+
+// [iterator.concept.forward], concept forward_iterator
+template<class I>
+  concept forward_iterator = see below;                    // since C++20
+
+// [iterator.concept.bidir], concept bidirectional_iterator
+template<class I>
+  concept bidirectional_iterator = see below;              // since C++20
+
+// [iterator.concept.random.access], concept random_access_iterator
+template<class I>
+  concept random_access_iterator = see below;              // since C++20
+
+// [indirectcallable]
+// [indirectcallable.indirectinvocable]
+template<class F, class I>
+  concept indirectly_unary_invocable = see below;          // since C++20
+
+template<class F, class I>
+  concept indirectly_regular_unary_invocable = see below;  // since C++20
+
+template<class F, class I>
+  concept indirect_unary_predicate = see below;            // since C++20
+
+template<class F, class I1, class I2>
+  concept indirect_binary_predicate = see below;           // since C++20
+
+template<class F, class I1, class I2 = I1>
+  concept indirect_equivalence_relation = see below;       // since C++20
+
+template<class F, class I1, class I2 = I1>
+  concept indirect_strict_weak_order = see below;          // since C++20
+
+template<class F, class... Is>
+  using indirect_result_t = see below;                     // since C++20
+
+// [projected], projected
+template<indirectly_readable I, indirectly_regular_unary_invocable<I> Proj>
+  struct projected;                                        // since C++20
+
+template<weakly_incrementable I, indirectly_regular_unary_invocable<I> Proj>
+  struct incrementable_traits<projected<I, Proj>>;         // since C++20
+
+// [alg.req.ind.move], concept indirectly_movable
+template<class In, class Out>
+  concept indirectly_movable = see below;                  // since C++20
+
+template<class In, class Out>
+  concept indirectly_movable_storable = see below;         // since C++20
+
+// [alg.req.ind.copy], concept indirectly_copyable
+template<class In, class Out>
+  concept indirectly_copyable = see below;                 // since C++20
+
+template<class In, class Out>
+  concept indirectly_copyable_storable = see below;        // since C++20
+
+// [alg.req.ind.swap], concept indirectly_swappable
+template<class I1, class I2 = I1>
+  concept indirectly_swappable = see below;                // since C++20
+
+template<class I1, class I2, class R, class P1 = identity,
+         class P2 = identity>
+  concept indirectly_comparable =
+    indirect_binary_predicate<R, projected<I1, P1>, projected<I2, P2>>; // since C++20
+
+// [alg.req.permutable], concept permutable
+template<class I>
+  concept permutable = see below;                          // since C++20
+
+ // [alg.req.mergeable], concept mergeable
+template<class I1, class I2, class Out,
+    class R = ranges::less, class P1 = identity, class P2 = identity>
+  concept mergeable = see below;                           // since C++20
+
+// [alg.req.sortable], concept sortable
+template<class I, class R = ranges::less, class P = identity>
+  concept sortable = see below;                            // since C++20
+
+template<input_or_output_iterator I, sentinel_for<I> S>
+  requires (!same_as<I, S> && copyable<I>)
+class common_iterator;                                     // since C++20
+
+template<class Category, class T, class Distance = ptrdiff_t,
+         class Pointer = T*, class Reference = T&>
+struct iterator                                            // deprecated in C++17
+{
+    typedef T         value_type;
+    typedef Distance  difference_type;
+    typedef Pointer   pointer;
+    typedef Reference reference;
+    typedef Category  iterator_category;
+};
+
+struct input_iterator_tag  {};
+struct output_iterator_tag {};
+struct forward_iterator_tag       : public input_iterator_tag         {};
+struct bidirectional_iterator_tag : public forward_iterator_tag       {};
+struct random_access_iterator_tag : public bidirectional_iterator_tag {};
+struct contiguous_iterator_tag    : public random_access_iterator_tag {};
+
+// 27.4.3, iterator operations
+template <class InputIterator, class Distance>  // constexpr in C++17
+  constexpr void advance(InputIterator& i, Distance n);
+
+template <class InputIterator>  // constexpr in C++17
+  constexpr typename iterator_traits<InputIterator>::difference_type
+    distance(InputIterator first, InputIterator last);
+
+template <class InputIterator>  // constexpr in C++17
+  constexpr InputIterator next(InputIterator x,
+typename iterator_traits<InputIterator>::difference_type n = 1);
+
+template <class BidirectionalIterator>  // constexpr in C++17
+  constexpr BidirectionalIterator prev(BidirectionalIterator x,
+    typename iterator_traits<BidirectionalIterator>::difference_type n = 1);
+
+// [range.iter.ops], range iterator operations
+namespace ranges {
+  // [range.iter.op.advance], ranges::advance
+  template<input_or_output_iterator I>
+    constexpr void advance(I& i, iter_difference_t<I> n);                          // since C++20
+  template<input_or_output_iterator I, sentinel_for<I> S>
+    constexpr void advance(I& i, S bound);                                         // since C++20
+  template<input_or_output_iterator I, sentinel_for<I> S>
+    constexpr iter_difference_t<I> advance(I& i, iter_difference_t<I> n, S bound); // since C++20
+}
+
+template <class Iterator>
+class reverse_iterator
+    : public iterator<typename iterator_traits<Iterator>::iterator_category, // until C++17
+                      typename iterator_traits<Iterator>::value_type,
+                      typename iterator_traits<Iterator>::difference_type,
+                      typename iterator_traits<Iterator>::pointer,
+                      typename iterator_traits<Iterator>::reference>
+{
+protected:
+    Iterator current;
+public:
+    using iterator_type     = Iterator;
+    using iterator_concept  = see below; // since C++20
+    using iterator_category = typename iterator_traits<Iterator>::iterator_category; // since C++17, until C++20
+    using iterator_category = see below; // since C++20
+    using value_type        = typename iterator_traits<Iterator>::value_type; // since C++17, until C++20
+    using value_type        = iter_value_t<Iterator>; // since C++20
+    using difference_type   = typename iterator_traits<Iterator>::difference_type; // until C++20
+    using difference_type   = iter_difference_t<Iterator>; // since C++20
+    using pointer           = typename iterator_traits<Iterator>::pointer;
+    using reference         = typename iterator_traits<Iterator>::reference; // until C++20
+    using reference         = iter_reference_t<Iterator>; // since C++20
+
+    constexpr reverse_iterator();
+    constexpr explicit reverse_iterator(Iterator x);
+    template <class U> constexpr reverse_iterator(const reverse_iterator<U>& u);
+    template <class U> constexpr reverse_iterator& operator=(const reverse_iterator<U>& u);
+    constexpr Iterator base() const;
+    constexpr reference operator*() const;
+    constexpr pointer   operator->() const; // until C++20
+    constexpr pointer   operator->() const requires see below; // since C++20
+    constexpr reverse_iterator& operator++();
+    constexpr reverse_iterator  operator++(int);
+    constexpr reverse_iterator& operator--();
+    constexpr reverse_iterator  operator--(int);
+    constexpr reverse_iterator  operator+ (difference_type n) const;
+    constexpr reverse_iterator& operator+=(difference_type n);
+    constexpr reverse_iterator  operator- (difference_type n) const;
+    constexpr reverse_iterator& operator-=(difference_type n);
+    constexpr unspecified       operator[](difference_type n) const;
+
+    friend constexpr iter_rvalue_reference_t<Iterator>
+      iter_move(const reverse_iterator& i) noexcept(see below);
+    template<indirectly_swappable<Iterator> Iterator2>
+      friend constexpr void
+        iter_swap(const reverse_iterator& x,
+                  const reverse_iterator<Iterator2>& y) noexcept(see below);
+};
+
+template <class Iterator1, class Iterator2>
+constexpr bool                          // constexpr in C++17
+operator==(const reverse_iterator<Iterator1>& x, const reverse_iterator<Iterator2>& y);
+
+template <class Iterator1, class Iterator2>
+constexpr bool                          // constexpr in C++17
+operator!=(const reverse_iterator<Iterator1>& x, const reverse_iterator<Iterator2>& y);
+
+template <class Iterator1, class Iterator2>
+constexpr bool                          // constexpr in C++17
+operator<(const reverse_iterator<Iterator1>& x, const reverse_iterator<Iterator2>& y);
+
+template <class Iterator1, class Iterator2>
+constexpr bool                          // constexpr in C++17
+operator>(const reverse_iterator<Iterator1>& x, const reverse_iterator<Iterator2>& y);
+
+template <class Iterator1, class Iterator2>
+constexpr bool                          // constexpr in C++17
+operator<=(const reverse_iterator<Iterator1>& x, const reverse_iterator<Iterator2>& y);
+
+template <class Iterator1, class Iterator2>
+constexpr bool                          // constexpr in C++17
+operator>=(const reverse_iterator<Iterator1>& x, const reverse_iterator<Iterator2>& y);
+
+template<class Iterator1, three_way_comparable_with<Iterator1> Iterator2>
+  constexpr compare_three_way_result_t<Iterator1, Iterator2>
+    operator<=>(const reverse_iterator<Iterator1>& x,
+                const reverse_iterator<Iterator2>& y);
+
+template <class Iterator1, class Iterator2>
+constexpr auto
+operator-(const reverse_iterator<Iterator1>& x, const reverse_iterator<Iterator2>& y)
+-> decltype(__y.base() - __x.base());   // constexpr in C++17
+
+template <class Iterator>
+constexpr reverse_iterator<Iterator>
+operator+(typename reverse_iterator<Iterator>::difference_type n,
+          const reverse_iterator<Iterator>& x);   // constexpr in C++17
+
+template <class Iterator>
+constexpr reverse_iterator<Iterator> make_reverse_iterator(Iterator i); // C++14, constexpr in C++17
+
+template<class Iterator1, class Iterator2>
+    requires (!sized_sentinel_for<Iterator1, Iterator2>)
+  inline constexpr bool disable_sized_sentinel_for<reverse_iterator<Iterator1>,
+                                                   reverse_iterator<Iterator2>> = true;
+
+template <class Container>
+class back_insert_iterator
+    : public iterator<output_iterator_tag, void, void, void, void> // until C++17
+{
+protected:
+    Container* container;
+public:
+    typedef Container                   container_type;
+    typedef void                        value_type;
+    typedef void                        difference_type; // until C++20
+    typedef ptrdiff_t                   difference_type; // since C++20
+    typedef void                        reference;
+    typedef void                        pointer;
+
+    explicit back_insert_iterator(Container& x);  // constexpr in C++20
+    back_insert_iterator& operator=(const typename Container::value_type& value);  // constexpr in C++20
+    back_insert_iterator& operator*();  // constexpr in C++20
+    back_insert_iterator& operator++();  // constexpr in C++20
+    back_insert_iterator  operator++(int);  // constexpr in C++20
+};
+
+template <class Container> back_insert_iterator<Container> back_inserter(Container& x);  // constexpr in C++20
+
+template <class Container>
+class front_insert_iterator
+    : public iterator<output_iterator_tag, void, void, void, void> // until C++17
+{
+protected:
+    Container* container;
+public:
+    typedef Container                    container_type;
+    typedef void                         value_type;
+    typedef void                         difference_type; // until C++20
+    typedef ptrdiff_t                    difference_type; // since C++20
+    typedef void                         reference;
+    typedef void                         pointer;
+
+    explicit front_insert_iterator(Container& x);  // constexpr in C++20
+    front_insert_iterator& operator=(const typename Container::value_type& value);  // constexpr in C++20
+    front_insert_iterator& operator*();  // constexpr in C++20
+    front_insert_iterator& operator++();  // constexpr in C++20
+    front_insert_iterator  operator++(int);  // constexpr in C++20
+};
+
+template <class Container> front_insert_iterator<Container> front_inserter(Container& x);  // constexpr in C++20
+
+template <class Container>
+class insert_iterator
+    : public iterator<output_iterator_tag, void, void, void, void> // until C++17
+{
+protected:
+    Container* container;
+    typename Container::iterator iter;
+public:
+    typedef Container              container_type;
+    typedef void                   value_type;
+    typedef void                   difference_type; // until C++20
+    typedef ptrdiff_t              difference_type; // since C++20
+    typedef void                   reference;
+    typedef void                   pointer;
+
+    insert_iterator(Container& x, typename Container::iterator i);  // constexpr in C++20
+    insert_iterator& operator=(const typename Container::value_type& value);  // constexpr in C++20
+    insert_iterator& operator*();  // constexpr in C++20
+    insert_iterator& operator++();  // constexpr in C++20
+    insert_iterator& operator++(int);  // constexpr in C++20
+};
+
+template <class Container>
+insert_iterator<Container> inserter(Container& x, typename Container::iterator i);  // until C++20
+template <class Container>
+constexpr insert_iterator<Container> inserter(Container& x, ranges::iterator_t<Container> i);  // since C++20
+
+template <class Iterator>
+class move_iterator {
+public:
+    using iterator_type     = Iterator;
+    using iterator_concept  = see below; // From C++20
+    using iterator_category = see below; // not always present starting from C++20
+    using value_type        = iter_value_t<Iterator>; // Until C++20, iterator_traits<Iterator>::value_type
+    using difference_type   = iter_difference_t<Iterator>; // Until C++20, iterator_traits<Iterator>::difference_type;
+    using pointer           = Iterator;
+    using reference         = iter_rvalue_reference_t<Iterator>; // Until C++20, value_type&&
+
+    constexpr move_iterator();  // all the constexprs are in C++17
+    constexpr explicit move_iterator(Iterator i);
+    template <class U>
+      constexpr move_iterator(const move_iterator<U>& u);
+    template <class U>
+      constexpr move_iterator& operator=(const move_iterator<U>& u);
+
+    constexpr iterator_type base() const; // Until C++20
+    constexpr const Iterator& base() const & noexcept; // From C++20
+    constexpr Iterator base() &&; // From C++20
+
+    constexpr reference operator*() const;
+    constexpr pointer operator->() const; // Deprecated in C++20
+    constexpr move_iterator& operator++();
+    constexpr auto operator++(int); // Return type was move_iterator until C++20
+    constexpr move_iterator& operator--();
+    constexpr move_iterator operator--(int);
+    constexpr move_iterator operator+(difference_type n) const;
+    constexpr move_iterator& operator+=(difference_type n);
+    constexpr move_iterator operator-(difference_type n) const;
+    constexpr move_iterator& operator-=(difference_type n);
+    constexpr reference operator[](difference_type n) const; // Return type unspecified until C++20
+
+    template<sentinel_for<Iterator> S>
+      friend constexpr bool
+        operator==(const move_iterator& x, const move_sentinel<S>& y); // Since C++20
+    template<sized_sentinel_for<Iterator> S>
+      friend constexpr iter_difference_t<Iterator>
+        operator-(const move_sentinel<S>& x, const move_iterator& y); // Since C++20
+    template<sized_sentinel_for<Iterator> S>
+      friend constexpr iter_difference_t<Iterator>
+        operator-(const move_iterator& x, const move_sentinel<S>& y); // Since C++20
+    friend constexpr iter_rvalue_reference_t<Iterator>
+      iter_move(const move_iterator& i)
+        noexcept(noexcept(ranges::iter_move(i.current))); // Since C++20
+    template<indirectly_swappable<Iterator> Iterator2>
+      friend constexpr void
+        iter_swap(const move_iterator& x, const move_iterator<Iterator2>& y)
+          noexcept(noexcept(ranges::iter_swap(x.current, y.current))); // Since C++20
+
+private:
+    Iterator current; // exposition only
+};
+
+template <class Iterator1, class Iterator2>
+constexpr bool   // constexpr in C++17
+operator==(const move_iterator<Iterator1>& x, const move_iterator<Iterator2>& y);
+
+template <class Iterator1, class Iterator2>
+constexpr bool   // constexpr in C++17
+operator!=(const move_iterator<Iterator1>& x, const move_iterator<Iterator2>& y);
+
+template <class Iterator1, class Iterator2>
+constexpr bool   // constexpr in C++17
+operator<(const move_iterator<Iterator1>& x, const move_iterator<Iterator2>& y);
+
+template <class Iterator1, class Iterator2>
+constexpr bool   // constexpr in C++17
+operator<=(const move_iterator<Iterator1>& x, const move_iterator<Iterator2>& y);
+
+template <class Iterator1, class Iterator2>
+constexpr bool   // constexpr in C++17
+operator>(const move_iterator<Iterator1>& x, const move_iterator<Iterator2>& y);
+
+template <class Iterator1, class Iterator2>
+constexpr bool   // constexpr in C++17
+operator>=(const move_iterator<Iterator1>& x, const move_iterator<Iterator2>& y);
+
+template <class Iterator1, class Iterator2>
+constexpr auto   // constexpr in C++17
+operator-(const move_iterator<Iterator1>& x,
+          const move_iterator<Iterator2>& y) -> decltype(x.base() - y.base());
+
+template <class Iterator>
+constexpr move_iterator<Iterator> operator+(   // constexpr in C++17
+            typename move_iterator<Iterator>::difference_type n,
+            const move_iterator<Iterator>& x);
+
+template <class Iterator>   // constexpr in C++17
+constexpr  move_iterator<Iterator> make_move_iterator(const Iterator& i);
+
+template<class Iterator1, class Iterator2>
+    requires (!sized_sentinel_for<Iterator1, Iterator2>)
+  inline constexpr bool disable_sized_sentinel_for<move_iterator<Iterator1>,    // since C++20
+                                                    move_iterator<Iterator2>> = true;
+
+template<semiregular S>
+class move_sentinel {
+public:
+  constexpr move_sentinel();
+  constexpr explicit move_sentinel(S s);
+  template<class S2>
+    requires convertible_to<const S2&, S>
+      constexpr move_sentinel(const move_sentinel<S2>& s);
+  template<class S2>
+    requires assignable_from<S&, const S2&>
+      constexpr move_sentinel& operator=(const move_sentinel<S2>& s);
+
+  constexpr S base() const;
+private:
+  S last;     // exposition only
+};
+
+// [default.sentinel], default sentinel
+struct default_sentinel_t;
+inline constexpr default_sentinel_t default_sentinel{};
+
+// [iterators.counted], counted iterators
+template<input_or_output_iterator I> class counted_iterator;
+
+template<input_iterator I>
+  requires see below
+  struct iterator_traits<counted_iterator<I>>;
+
+// [unreachable.sentinel], unreachable sentinel
+struct unreachable_sentinel_t;
+inline constexpr unreachable_sentinel_t unreachable_sentinel{};
+
+template <class T, class charT = char, class traits = char_traits<charT>, class Distance = ptrdiff_t>
+class istream_iterator
+    : public iterator<input_iterator_tag, T, Distance, const T*, const T&> // until C++17
+{
+public:
+    typedef input_iterator_tag           iterator_category;
+    typedef T                            value_type;
+    typedef Distance                     difference_type;
+    typedef const T*                     pointer;
+    typedef const T&                     reference;
+
+    typedef charT                        char_type;
+    typedef traits                       traits_type;
+    typedef basic_istream<charT, traits> istream_type;
+
+    istream_iterator(); // constexpr since C++11
+    constexpr istream_iterator(default_sentinel_t); // since C++20
+    istream_iterator(istream_type& s);
+    istream_iterator(const istream_iterator& x);
+    ~istream_iterator();
+
+    const T& operator*() const;
+    const T* operator->() const;
+    istream_iterator& operator++();
+    istream_iterator  operator++(int);
+    friend bool operator==(const istream_iterator& i, default_sentinel_t); // since C++20
+};
+
+template <class T, class charT, class traits, class Distance>
+bool operator==(const istream_iterator<T,charT,traits,Distance>& x,
+                const istream_iterator<T,charT,traits,Distance>& y);
+template <class T, class charT, class traits, class Distance>
+bool operator!=(const istream_iterator<T,charT,traits,Distance>& x,
+                const istream_iterator<T,charT,traits,Distance>& y); // until C++20
+
+template <class T, class charT = char, class traits = char_traits<charT> >
+class ostream_iterator
+    : public iterator<output_iterator_tag, void, void, void, void> // until C++17
+{
+public:
+    typedef output_iterator_tag         iterator_category;
+    typedef void                        value_type;
+    typedef void                        difference_type; // until C++20
+    typedef ptrdiff_t                   difference_type; // since C++20
+    typedef void                        pointer;
+    typedef void                        reference;
+
+    typedef charT char_type;
+    typedef traits traits_type;
+    typedef basic_ostream<charT,traits> ostream_type;
+
+    ostream_iterator(ostream_type& s);
+    ostream_iterator(ostream_type& s, const charT* delimiter);
+    ostream_iterator(const ostream_iterator& x);
+    ~ostream_iterator();
+    ostream_iterator& operator=(const T& value);
+
+    ostream_iterator& operator*();
+    ostream_iterator& operator++();
+    ostream_iterator& operator++(int);
+};
+
+template<class charT, class traits = char_traits<charT> >
+class istreambuf_iterator
+    : public iterator<input_iterator_tag, charT, traits::off_type, unspecified, charT> // until C++17
+{
+public:
+    typedef input_iterator_tag              iterator_category;
+    typedef charT                           value_type;
+    typedef traits::off_type                difference_type;
+    typedef unspecified                     pointer;
+    typedef charT                           reference;
+
+    typedef charT                           char_type;
+    typedef traits                          traits_type;
+    typedef traits::int_type                int_type;
+    typedef basic_streambuf<charT, traits>  streambuf_type;
+    typedef basic_istream<charT, traits>    istream_type;
+
+    istreambuf_iterator() noexcept; // constexpr since C++11
+    constexpr istreambuf_iterator(default_sentinel_t) noexcept; // since C++20
+    istreambuf_iterator(istream_type& s) noexcept;
+    istreambuf_iterator(streambuf_type* s) noexcept;
+    istreambuf_iterator(a-private-type) noexcept;
+
+    charT                operator*() const;
+    pointer operator->() const;
+    istreambuf_iterator& operator++();
+    a-private-type       operator++(int);
+
+    bool equal(const istreambuf_iterator& b) const;
+    friend bool operator==(const istreambuf_iterator& i, default_sentinel_t s); // since C++20
+};
+
+template <class charT, class traits>
+bool operator==(const istreambuf_iterator<charT,traits>& a,
+                const istreambuf_iterator<charT,traits>& b);
+template <class charT, class traits>
+bool operator!=(const istreambuf_iterator<charT,traits>& a,
+                const istreambuf_iterator<charT,traits>& b); // until C++20
+
+template <class charT, class traits = char_traits<charT> >
+class ostreambuf_iterator
+    : public iterator<output_iterator_tag, void, void, void, void> // until C++17
+{
+public:
+    typedef output_iterator_tag            iterator_category;
+    typedef void                           value_type;
+    typedef void                           difference_type; // until C++20
+    typedef ptrdiff_t                      difference_type; // since C++20
+    typedef void                           pointer;
+    typedef void                           reference;
+
+    typedef charT                          char_type;
+    typedef traits                         traits_type;
+    typedef basic_streambuf<charT, traits> streambuf_type;
+    typedef basic_ostream<charT, traits>   ostream_type;
+
+    ostreambuf_iterator(ostream_type& s) noexcept;
+    ostreambuf_iterator(streambuf_type* s) noexcept;
+    ostreambuf_iterator& operator=(charT c);
+    ostreambuf_iterator& operator*();
+    ostreambuf_iterator& operator++();
+    ostreambuf_iterator& operator++(int);
+    bool failed() const noexcept;
+};
+
+template <class C> constexpr auto begin(C& c) -> decltype(c.begin());                   // constexpr since C++17
+template <class C> constexpr auto begin(const C& c) -> decltype(c.begin());             // constexpr since C++17
+template <class C> constexpr auto end(C& c) -> decltype(c.end());                       // constexpr since C++17
+template <class C> constexpr auto end(const C& c) -> decltype(c.end());                 // constexpr since C++17
+template <class T, size_t N> constexpr T* begin(T (&array)[N]) noexcept;
+template <class T, size_t N> constexpr T* end(T (&array)[N]) noexcept;
+
+template <class C> constexpr auto cbegin(const C& c) noexcept(see-below) -> decltype(std::begin(c)); // C++14
+template <class C> constexpr auto cend(const C& c) noexcept(see-below) -> decltype(std::end(c));     // C++14
+template <class C> constexpr auto rbegin(C& c) -> decltype(c.rbegin());                 // C++14, constexpr since C++17
+template <class C> constexpr auto rbegin(const C& c) -> decltype(c.rbegin());           // C++14, constexpr since C++17
+template <class C> constexpr auto rend(C& c) -> decltype(c.rend());                     // C++14, constexpr since C++17
+template <class C> constexpr auto rend(const C& c) -> decltype(c.rend());               // C++14, constexpr since C++17
+template <class E> constexpr reverse_iterator<const E*> rbegin(initializer_list<E> il); // C++14, constexpr since C++17
+template <class E> constexpr reverse_iterator<const E*> rend(initializer_list<E> il);   // C++14, constexpr since C++17
+template <class T, size_t N> constexpr reverse_iterator<T*> rbegin(T (&array)[N]);      // C++14, constexpr since C++17
+template <class T, size_t N> constexpr reverse_iterator<T*> rend(T (&array)[N]);        // C++14, constexpr since C++17
+template <class C> constexpr auto crbegin(const C& c) -> decltype(std::rbegin(c));      // C++14, constexpr since C++17
+template <class C> constexpr auto crend(const C& c) -> decltype(std::rend(c));          // C++14, constexpr since C++17
+
+// 24.8, container access:
+template <class C> constexpr auto size(const C& c) -> decltype(c.size());         // C++17
+template <class T, size_t N> constexpr size_t size(const T (&array)[N]) noexcept; // C++17
+
+template <class C> constexpr auto ssize(const C& c)
+    -> common_type_t<ptrdiff_t, make_signed_t<decltype(c.size())>>;                    // C++20
+template <class T, ptrdiff_t> constexpr ptrdiff_t ssize(const T (&array)[N]) noexcept; // C++20
+
+template <class C> constexpr auto empty(const C& c) -> decltype(c.empty());       // C++17
+template <class T, size_t N> constexpr bool empty(const T (&array)[N]) noexcept;  // C++17
+template <class E> constexpr bool empty(initializer_list<E> il) noexcept;         // C++17
+template <class C> constexpr auto data(C& c) -> decltype(c.data());               // C++17
+template <class C> constexpr auto data(const C& c) -> decltype(c.data());         // C++17
+template <class T, size_t N> constexpr T* data(T (&array)[N]) noexcept;           // C++17
+template <class E> constexpr const E* data(initializer_list<E> il) noexcept;      // C++17
+
+}  // std
+
+*/
+
+#include <__config>
+#include <__iterator/access.h>
+#include <__iterator/advance.h>
+#include <__iterator/back_insert_iterator.h>
+#include <__iterator/distance.h>
+#include <__iterator/front_insert_iterator.h>
+#include <__iterator/insert_iterator.h>
+#include <__iterator/istream_iterator.h>
+#include <__iterator/istreambuf_iterator.h>
+#include <__iterator/iterator.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/move_iterator.h>
+#include <__iterator/next.h>
+#include <__iterator/ostream_iterator.h>
+#include <__iterator/ostreambuf_iterator.h>
+#include <__iterator/prev.h>
+#include <__iterator/reverse_iterator.h>
+#include <__iterator/wrap_iter.h>
+
+#if _LIBCPP_STD_VER >= 14
+#  include <__iterator/reverse_access.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/size.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__iterator/common_iterator.h>
+#  include <__iterator/concepts.h>
+#  include <__iterator/counted_iterator.h>
+#  include <__iterator/default_sentinel.h>
+#  include <__iterator/incrementable_traits.h>
+#  include <__iterator/indirectly_comparable.h>
+#  include <__iterator/iter_move.h>
+#  include <__iterator/iter_swap.h>
+#  include <__iterator/mergeable.h>
+#  include <__iterator/move_sentinel.h>
+#  include <__iterator/permutable.h>
+#  include <__iterator/projected.h>
+#  include <__iterator/readable_traits.h>
+#  include <__iterator/sortable.h>
+#  include <__iterator/unreachable_sentinel.h>
+#endif
+
+#include <version>
+
+// standard-mandated includes
+
+// [iterator.synopsis]
+#include <compare>
+#include <concepts>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#  include <variant>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstdlib>
+#  include <exception>
+#  include <new>
+#  include <type_traits>
+#  include <typeinfo>
+#  include <utility>
+#endif
+
+#endif // _LIBCPP_ITERATOR
diff --git a/libcxx/include/__cxx03/latch b/libcxx/include/__cxx03/latch
new file mode 100644
index 000000000000..81d6028a9c2c
--- /dev/null
+++ b/libcxx/include/__cxx03/latch
@@ -0,0 +1,129 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_LATCH
+#define _LIBCPP_LATCH
+
+/*
+    latch synopsis
+
+namespace std
+{
+
+  class latch
+  {
+  public:
+    static constexpr ptrdiff_t max() noexcept;
+
+    constexpr explicit latch(ptrdiff_t __expected);
+    ~latch();
+
+    latch(const latch&) = delete;
+    latch& operator=(const latch&) = delete;
+
+    void count_down(ptrdiff_t __update = 1);
+    bool try_wait() const noexcept;
+    void wait() const;
+    void arrive_and_wait(ptrdiff_t __update = 1);
+
+  private:
+    ptrdiff_t __counter; // exposition only
+  };
+
+}
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+
+#  include <__assert>
+#  include <__atomic/atomic_base.h>
+#  include <__atomic/atomic_sync.h>
+#  include <__atomic/memory_order.h>
+#  include <cstddef>
+#  include <limits>
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+#  if _LIBCPP_STD_VER >= 14
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+class _LIBCPP_DEPRECATED_ATOMIC_SYNC latch {
+  __atomic_base<ptrdiff_t> __a_;
+
+public:
+  static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { return numeric_limits<ptrdiff_t>::max(); }
+
+  inline _LIBCPP_HIDE_FROM_ABI constexpr explicit latch(ptrdiff_t __expected) : __a_(__expected) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __expected >= 0,
+        "latch::latch(ptrdiff_t): latch cannot be "
+        "initialized with a negative value");
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __expected <= max(),
+        "latch::latch(ptrdiff_t): latch cannot be "
+        "initialized with a value greater than max()");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~latch() = default;
+  latch(const latch&)            = delete;
+  latch& operator=(const latch&) = delete;
+
+  inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void count_down(ptrdiff_t __update = 1) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__update >= 0, "latch::count_down called with a negative value");
+    auto const __old = __a_.fetch_sub(__update, memory_order_release);
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __update <= __old,
+        "latch::count_down called with a value greater "
+        "than the internal counter");
+    if (__old == __update)
+      __a_.notify_all();
+  }
+  inline _LIBCPP_HIDE_FROM_ABI bool try_wait() const noexcept {
+    auto __value = __a_.load(memory_order_acquire);
+    return try_wait_impl(__value);
+  }
+  inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait() const {
+    std::__atomic_wait_unless(
+        __a_, [this](ptrdiff_t& __value) -> bool { return try_wait_impl(__value); }, memory_order_acquire);
+  }
+  inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_wait(ptrdiff_t __update = 1) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__update >= 0, "latch::arrive_and_wait called with a negative value");
+    // other preconditions on __update are checked in count_down()
+
+    count_down(__update);
+    wait();
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI bool try_wait_impl(ptrdiff_t& __value) const noexcept { return __value == 0; }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#  endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_POP_MACROS
+
+#endif // !defined(_LIBCPP_HAS_NO_THREADS)
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#endif
+
+#endif //_LIBCPP_LATCH
diff --git a/libcxx/include/__cxx03/limits b/libcxx/include/__cxx03/limits
new file mode 100644
index 000000000000..d55c7cd75f34
--- /dev/null
+++ b/libcxx/include/__cxx03/limits
@@ -0,0 +1,584 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_LIMITS
+#define _LIBCPP_LIMITS
+
+/*
+    limits synopsis
+
+namespace std
+{
+
+template<class T>
+class numeric_limits
+{
+public:
+    static constexpr bool is_specialized = false;
+    static constexpr T min() noexcept;
+    static constexpr T max() noexcept;
+    static constexpr T lowest() noexcept;
+
+    static constexpr int  digits = 0;
+    static constexpr int  digits10 = 0;
+    static constexpr int  max_digits10 = 0;
+    static constexpr bool is_signed = false;
+    static constexpr bool is_integer = false;
+    static constexpr bool is_exact = false;
+    static constexpr int  radix = 0;
+    static constexpr T epsilon() noexcept;
+    static constexpr T round_error() noexcept;
+
+    static constexpr int  min_exponent = 0;
+    static constexpr int  min_exponent10 = 0;
+    static constexpr int  max_exponent = 0;
+    static constexpr int  max_exponent10 = 0;
+
+    static constexpr bool has_infinity = false;
+    static constexpr bool has_quiet_NaN = false;
+    static constexpr bool has_signaling_NaN = false;
+    static constexpr float_denorm_style has_denorm = denorm_absent; // deprecated in C++23
+    static constexpr bool has_denorm_loss = false;                  // deprecated in C++23
+    static constexpr T infinity() noexcept;
+    static constexpr T quiet_NaN() noexcept;
+    static constexpr T signaling_NaN() noexcept;
+    static constexpr T denorm_min() noexcept;
+
+    static constexpr bool is_iec559 = false;
+    static constexpr bool is_bounded = false;
+    static constexpr bool is_modulo = false;
+
+    static constexpr bool traps = false;
+    static constexpr bool tinyness_before = false;
+    static constexpr float_round_style round_style = round_toward_zero;
+};
+
+enum float_round_style
+{
+    round_indeterminate       = -1,
+    round_toward_zero         =  0,
+    round_to_nearest          =  1,
+    round_toward_infinity     =  2,
+    round_toward_neg_infinity =  3
+};
+
+enum float_denorm_style // deprecated in C++23
+{
+    denorm_indeterminate = -1,
+    denorm_absent = 0,
+    denorm_present = 1
+};
+
+template<> class numeric_limits<cv bool>;
+
+template<> class numeric_limits<cv char>;
+template<> class numeric_limits<cv signed char>;
+template<> class numeric_limits<cv unsigned char>;
+template<> class numeric_limits<cv wchar_t>;
+template<> class numeric_limits<cv char8_t>; // C++20
+template<> class numeric_limits<cv char16_t>;
+template<> class numeric_limits<cv char32_t>;
+
+template<> class numeric_limits<cv short>;
+template<> class numeric_limits<cv int>;
+template<> class numeric_limits<cv long>;
+template<> class numeric_limits<cv long long>;
+template<> class numeric_limits<cv unsigned short>;
+template<> class numeric_limits<cv unsigned int>;
+template<> class numeric_limits<cv unsigned long>;
+template<> class numeric_limits<cv unsigned long long>;
+
+template<> class numeric_limits<cv float>;
+template<> class numeric_limits<cv double>;
+template<> class numeric_limits<cv long double>;
+
+}  // std
+
+*/
+
+#include <__config>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_signed.h>
+#include <__type_traits/remove_cv.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+#include <version>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+enum float_round_style {
+  round_indeterminate       = -1,
+  round_toward_zero         = 0,
+  round_to_nearest          = 1,
+  round_toward_infinity     = 2,
+  round_toward_neg_infinity = 3
+};
+
+enum _LIBCPP_DEPRECATED_IN_CXX23 float_denorm_style {
+  denorm_indeterminate = -1,
+  denorm_absent        = 0,
+  denorm_present       = 1
+};
+
+template <class _Tp, bool = is_arithmetic<_Tp>::value>
+class __libcpp_numeric_limits {
+protected:
+  typedef _Tp type;
+
+  static _LIBCPP_CONSTEXPR const bool is_specialized = false;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type min() _NOEXCEPT { return type(); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type max() _NOEXCEPT { return type(); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type lowest() _NOEXCEPT { return type(); }
+
+  static _LIBCPP_CONSTEXPR const int digits       = 0;
+  static _LIBCPP_CONSTEXPR const int digits10     = 0;
+  static _LIBCPP_CONSTEXPR const int max_digits10 = 0;
+  static _LIBCPP_CONSTEXPR const bool is_signed   = false;
+  static _LIBCPP_CONSTEXPR const bool is_integer  = false;
+  static _LIBCPP_CONSTEXPR const bool is_exact    = false;
+  static _LIBCPP_CONSTEXPR const int radix        = 0;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type epsilon() _NOEXCEPT { return type(); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type round_error() _NOEXCEPT { return type(); }
+
+  static _LIBCPP_CONSTEXPR const int min_exponent   = 0;
+  static _LIBCPP_CONSTEXPR const int min_exponent10 = 0;
+  static _LIBCPP_CONSTEXPR const int max_exponent   = 0;
+  static _LIBCPP_CONSTEXPR const int max_exponent10 = 0;
+
+  static _LIBCPP_CONSTEXPR const bool has_infinity                                         = false;
+  static _LIBCPP_CONSTEXPR const bool has_quiet_NaN                                        = false;
+  static _LIBCPP_CONSTEXPR const bool has_signaling_NaN                                    = false;
+  static _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_CONSTEXPR const float_denorm_style has_denorm = denorm_absent;
+  static _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_CONSTEXPR const bool has_denorm_loss          = false;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type infinity() _NOEXCEPT { return type(); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type quiet_NaN() _NOEXCEPT { return type(); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type signaling_NaN() _NOEXCEPT { return type(); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type denorm_min() _NOEXCEPT { return type(); }
+
+  static _LIBCPP_CONSTEXPR const bool is_iec559  = false;
+  static _LIBCPP_CONSTEXPR const bool is_bounded = false;
+  static _LIBCPP_CONSTEXPR const bool is_modulo  = false;
+
+  static _LIBCPP_CONSTEXPR const bool traps                    = false;
+  static _LIBCPP_CONSTEXPR const bool tinyness_before          = false;
+  static _LIBCPP_CONSTEXPR const float_round_style round_style = round_toward_zero;
+};
+
+template <class _Tp, int __digits, bool _IsSigned>
+struct __libcpp_compute_min {
+  static _LIBCPP_CONSTEXPR const _Tp value = _Tp(_Tp(1) << __digits);
+};
+
+template <class _Tp, int __digits>
+struct __libcpp_compute_min<_Tp, __digits, false> {
+  static _LIBCPP_CONSTEXPR const _Tp value = _Tp(0);
+};
+
+template <class _Tp>
+class __libcpp_numeric_limits<_Tp, true> {
+protected:
+  typedef _Tp type;
+
+  static _LIBCPP_CONSTEXPR const bool is_specialized = true;
+
+  static _LIBCPP_CONSTEXPR const bool is_signed   = type(-1) < type(0);
+  static _LIBCPP_CONSTEXPR const int digits       = static_cast<int>(sizeof(type) * __CHAR_BIT__ - is_signed);
+  static _LIBCPP_CONSTEXPR const int digits10     = digits * 3 / 10;
+  static _LIBCPP_CONSTEXPR const int max_digits10 = 0;
+  static _LIBCPP_CONSTEXPR const type __min       = __libcpp_compute_min<type, digits, is_signed>::value;
+  static _LIBCPP_CONSTEXPR const type __max       = is_signed ? type(type(~0) ^ __min) : type(~0);
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type min() _NOEXCEPT { return __min; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type max() _NOEXCEPT { return __max; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type lowest() _NOEXCEPT { return min(); }
+
+  static _LIBCPP_CONSTEXPR const bool is_integer = true;
+  static _LIBCPP_CONSTEXPR const bool is_exact   = true;
+  static _LIBCPP_CONSTEXPR const int radix       = 2;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type epsilon() _NOEXCEPT { return type(0); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type round_error() _NOEXCEPT { return type(0); }
+
+  static _LIBCPP_CONSTEXPR const int min_exponent   = 0;
+  static _LIBCPP_CONSTEXPR const int min_exponent10 = 0;
+  static _LIBCPP_CONSTEXPR const int max_exponent   = 0;
+  static _LIBCPP_CONSTEXPR const int max_exponent10 = 0;
+
+  static _LIBCPP_CONSTEXPR const bool has_infinity                                         = false;
+  static _LIBCPP_CONSTEXPR const bool has_quiet_NaN                                        = false;
+  static _LIBCPP_CONSTEXPR const bool has_signaling_NaN                                    = false;
+  static _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_CONSTEXPR const float_denorm_style has_denorm = denorm_absent;
+  static _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_CONSTEXPR const bool has_denorm_loss          = false;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type infinity() _NOEXCEPT { return type(0); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type quiet_NaN() _NOEXCEPT { return type(0); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type signaling_NaN() _NOEXCEPT { return type(0); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type denorm_min() _NOEXCEPT { return type(0); }
+
+  static _LIBCPP_CONSTEXPR const bool is_iec559  = false;
+  static _LIBCPP_CONSTEXPR const bool is_bounded = true;
+  static _LIBCPP_CONSTEXPR const bool is_modulo  = !std::is_signed<_Tp>::value;
+
+#if defined(__i386__) || defined(__x86_64__) || defined(__pnacl__) || defined(__wasm__)
+  static _LIBCPP_CONSTEXPR const bool traps = true;
+#else
+  static _LIBCPP_CONSTEXPR const bool traps = false;
+#endif
+  static _LIBCPP_CONSTEXPR const bool tinyness_before          = false;
+  static _LIBCPP_CONSTEXPR const float_round_style round_style = round_toward_zero;
+};
+
+template <>
+class __libcpp_numeric_limits<bool, true> {
+protected:
+  typedef bool type;
+
+  static _LIBCPP_CONSTEXPR const bool is_specialized = true;
+
+  static _LIBCPP_CONSTEXPR const bool is_signed   = false;
+  static _LIBCPP_CONSTEXPR const int digits       = 1;
+  static _LIBCPP_CONSTEXPR const int digits10     = 0;
+  static _LIBCPP_CONSTEXPR const int max_digits10 = 0;
+  static _LIBCPP_CONSTEXPR const type __min       = false;
+  static _LIBCPP_CONSTEXPR const type __max       = true;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type min() _NOEXCEPT { return __min; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type max() _NOEXCEPT { return __max; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type lowest() _NOEXCEPT { return min(); }
+
+  static _LIBCPP_CONSTEXPR const bool is_integer = true;
+  static _LIBCPP_CONSTEXPR const bool is_exact   = true;
+  static _LIBCPP_CONSTEXPR const int radix       = 2;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type epsilon() _NOEXCEPT { return type(0); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type round_error() _NOEXCEPT { return type(0); }
+
+  static _LIBCPP_CONSTEXPR const int min_exponent   = 0;
+  static _LIBCPP_CONSTEXPR const int min_exponent10 = 0;
+  static _LIBCPP_CONSTEXPR const int max_exponent   = 0;
+  static _LIBCPP_CONSTEXPR const int max_exponent10 = 0;
+
+  static _LIBCPP_CONSTEXPR const bool has_infinity                                         = false;
+  static _LIBCPP_CONSTEXPR const bool has_quiet_NaN                                        = false;
+  static _LIBCPP_CONSTEXPR const bool has_signaling_NaN                                    = false;
+  static _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_CONSTEXPR const float_denorm_style has_denorm = denorm_absent;
+  static _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_CONSTEXPR const bool has_denorm_loss          = false;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type infinity() _NOEXCEPT { return type(0); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type quiet_NaN() _NOEXCEPT { return type(0); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type signaling_NaN() _NOEXCEPT { return type(0); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type denorm_min() _NOEXCEPT { return type(0); }
+
+  static _LIBCPP_CONSTEXPR const bool is_iec559  = false;
+  static _LIBCPP_CONSTEXPR const bool is_bounded = true;
+  static _LIBCPP_CONSTEXPR const bool is_modulo  = false;
+
+  static _LIBCPP_CONSTEXPR const bool traps                    = false;
+  static _LIBCPP_CONSTEXPR const bool tinyness_before          = false;
+  static _LIBCPP_CONSTEXPR const float_round_style round_style = round_toward_zero;
+};
+
+template <>
+class __libcpp_numeric_limits<float, true> {
+protected:
+  typedef float type;
+
+  static _LIBCPP_CONSTEXPR const bool is_specialized = true;
+
+  static _LIBCPP_CONSTEXPR const bool is_signed   = true;
+  static _LIBCPP_CONSTEXPR const int digits       = __FLT_MANT_DIG__;
+  static _LIBCPP_CONSTEXPR const int digits10     = __FLT_DIG__;
+  static _LIBCPP_CONSTEXPR const int max_digits10 = 2 + (digits * 30103l) / 100000l;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type min() _NOEXCEPT { return __FLT_MIN__; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type max() _NOEXCEPT { return __FLT_MAX__; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type lowest() _NOEXCEPT { return -max(); }
+
+  static _LIBCPP_CONSTEXPR const bool is_integer = false;
+  static _LIBCPP_CONSTEXPR const bool is_exact   = false;
+  static _LIBCPP_CONSTEXPR const int radix       = __FLT_RADIX__;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type epsilon() _NOEXCEPT { return __FLT_EPSILON__; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type round_error() _NOEXCEPT { return 0.5F; }
+
+  static _LIBCPP_CONSTEXPR const int min_exponent   = __FLT_MIN_EXP__;
+  static _LIBCPP_CONSTEXPR const int min_exponent10 = __FLT_MIN_10_EXP__;
+  static _LIBCPP_CONSTEXPR const int max_exponent   = __FLT_MAX_EXP__;
+  static _LIBCPP_CONSTEXPR const int max_exponent10 = __FLT_MAX_10_EXP__;
+
+  static _LIBCPP_CONSTEXPR const bool has_infinity                                         = true;
+  static _LIBCPP_CONSTEXPR const bool has_quiet_NaN                                        = true;
+  static _LIBCPP_CONSTEXPR const bool has_signaling_NaN                                    = true;
+  static _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_CONSTEXPR const float_denorm_style has_denorm = denorm_present;
+  static _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_CONSTEXPR const bool has_denorm_loss          = false;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type infinity() _NOEXCEPT {
+    return __builtin_huge_valf();
+  }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type quiet_NaN() _NOEXCEPT {
+    return __builtin_nanf("");
+  }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type signaling_NaN() _NOEXCEPT {
+    return __builtin_nansf("");
+  }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type denorm_min() _NOEXCEPT {
+    return __FLT_DENORM_MIN__;
+  }
+
+  static _LIBCPP_CONSTEXPR const bool is_iec559  = true;
+  static _LIBCPP_CONSTEXPR const bool is_bounded = true;
+  static _LIBCPP_CONSTEXPR const bool is_modulo  = false;
+
+  static _LIBCPP_CONSTEXPR const bool traps = false;
+#if (defined(__arm__) || defined(__aarch64__))
+  static _LIBCPP_CONSTEXPR const bool tinyness_before = true;
+#else
+  static _LIBCPP_CONSTEXPR const bool tinyness_before = false;
+#endif
+  static _LIBCPP_CONSTEXPR const float_round_style round_style = round_to_nearest;
+};
+
+template <>
+class __libcpp_numeric_limits<double, true> {
+protected:
+  typedef double type;
+
+  static _LIBCPP_CONSTEXPR const bool is_specialized = true;
+
+  static _LIBCPP_CONSTEXPR const bool is_signed   = true;
+  static _LIBCPP_CONSTEXPR const int digits       = __DBL_MANT_DIG__;
+  static _LIBCPP_CONSTEXPR const int digits10     = __DBL_DIG__;
+  static _LIBCPP_CONSTEXPR const int max_digits10 = 2 + (digits * 30103l) / 100000l;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type min() _NOEXCEPT { return __DBL_MIN__; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type max() _NOEXCEPT { return __DBL_MAX__; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type lowest() _NOEXCEPT { return -max(); }
+
+  static _LIBCPP_CONSTEXPR const bool is_integer = false;
+  static _LIBCPP_CONSTEXPR const bool is_exact   = false;
+  static _LIBCPP_CONSTEXPR const int radix       = __FLT_RADIX__;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type epsilon() _NOEXCEPT { return __DBL_EPSILON__; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type round_error() _NOEXCEPT { return 0.5; }
+
+  static _LIBCPP_CONSTEXPR const int min_exponent   = __DBL_MIN_EXP__;
+  static _LIBCPP_CONSTEXPR const int min_exponent10 = __DBL_MIN_10_EXP__;
+  static _LIBCPP_CONSTEXPR const int max_exponent   = __DBL_MAX_EXP__;
+  static _LIBCPP_CONSTEXPR const int max_exponent10 = __DBL_MAX_10_EXP__;
+
+  static _LIBCPP_CONSTEXPR const bool has_infinity                                         = true;
+  static _LIBCPP_CONSTEXPR const bool has_quiet_NaN                                        = true;
+  static _LIBCPP_CONSTEXPR const bool has_signaling_NaN                                    = true;
+  static _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_CONSTEXPR const float_denorm_style has_denorm = denorm_present;
+  static _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_CONSTEXPR const bool has_denorm_loss          = false;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type infinity() _NOEXCEPT {
+    return __builtin_huge_val();
+  }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type quiet_NaN() _NOEXCEPT {
+    return __builtin_nan("");
+  }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type signaling_NaN() _NOEXCEPT {
+    return __builtin_nans("");
+  }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type denorm_min() _NOEXCEPT {
+    return __DBL_DENORM_MIN__;
+  }
+
+  static _LIBCPP_CONSTEXPR const bool is_iec559  = true;
+  static _LIBCPP_CONSTEXPR const bool is_bounded = true;
+  static _LIBCPP_CONSTEXPR const bool is_modulo  = false;
+
+  static _LIBCPP_CONSTEXPR const bool traps = false;
+#if (defined(__arm__) || defined(__aarch64__))
+  static _LIBCPP_CONSTEXPR const bool tinyness_before = true;
+#else
+  static _LIBCPP_CONSTEXPR const bool tinyness_before = false;
+#endif
+  static _LIBCPP_CONSTEXPR const float_round_style round_style = round_to_nearest;
+};
+
+template <>
+class __libcpp_numeric_limits<long double, true> {
+protected:
+  typedef long double type;
+
+  static _LIBCPP_CONSTEXPR const bool is_specialized = true;
+
+  static _LIBCPP_CONSTEXPR const bool is_signed   = true;
+  static _LIBCPP_CONSTEXPR const int digits       = __LDBL_MANT_DIG__;
+  static _LIBCPP_CONSTEXPR const int digits10     = __LDBL_DIG__;
+  static _LIBCPP_CONSTEXPR const int max_digits10 = 2 + (digits * 30103l) / 100000l;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type min() _NOEXCEPT { return __LDBL_MIN__; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type max() _NOEXCEPT { return __LDBL_MAX__; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type lowest() _NOEXCEPT { return -max(); }
+
+  static _LIBCPP_CONSTEXPR const bool is_integer = false;
+  static _LIBCPP_CONSTEXPR const bool is_exact   = false;
+  static _LIBCPP_CONSTEXPR const int radix       = __FLT_RADIX__;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type epsilon() _NOEXCEPT { return __LDBL_EPSILON__; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type round_error() _NOEXCEPT { return 0.5L; }
+
+  static _LIBCPP_CONSTEXPR const int min_exponent   = __LDBL_MIN_EXP__;
+  static _LIBCPP_CONSTEXPR const int min_exponent10 = __LDBL_MIN_10_EXP__;
+  static _LIBCPP_CONSTEXPR const int max_exponent   = __LDBL_MAX_EXP__;
+  static _LIBCPP_CONSTEXPR const int max_exponent10 = __LDBL_MAX_10_EXP__;
+
+  static _LIBCPP_CONSTEXPR const bool has_infinity                                         = true;
+  static _LIBCPP_CONSTEXPR const bool has_quiet_NaN                                        = true;
+  static _LIBCPP_CONSTEXPR const bool has_signaling_NaN                                    = true;
+  static _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_CONSTEXPR const float_denorm_style has_denorm = denorm_present;
+  static _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_CONSTEXPR const bool has_denorm_loss          = false;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type infinity() _NOEXCEPT {
+    return __builtin_huge_vall();
+  }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type quiet_NaN() _NOEXCEPT {
+    return __builtin_nanl("");
+  }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type signaling_NaN() _NOEXCEPT {
+    return __builtin_nansl("");
+  }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type denorm_min() _NOEXCEPT {
+    return __LDBL_DENORM_MIN__;
+  }
+
+#if defined(__powerpc__) && defined(__LONG_DOUBLE_IBM128__)
+  static _LIBCPP_CONSTEXPR const bool is_iec559 = false;
+#else
+  static _LIBCPP_CONSTEXPR const bool is_iec559 = true;
+#endif
+  static _LIBCPP_CONSTEXPR const bool is_bounded = true;
+  static _LIBCPP_CONSTEXPR const bool is_modulo  = false;
+
+  static _LIBCPP_CONSTEXPR const bool traps = false;
+#if (defined(__arm__) || defined(__aarch64__))
+  static _LIBCPP_CONSTEXPR const bool tinyness_before = true;
+#else
+  static _LIBCPP_CONSTEXPR const bool tinyness_before = false;
+#endif
+  static _LIBCPP_CONSTEXPR const float_round_style round_style = round_to_nearest;
+};
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS numeric_limits : private __libcpp_numeric_limits<_Tp> {
+  typedef __libcpp_numeric_limits<_Tp> __base;
+  typedef typename __base::type type;
+
+public:
+  static _LIBCPP_CONSTEXPR const bool is_specialized = __base::is_specialized;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type min() _NOEXCEPT { return __base::min(); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type max() _NOEXCEPT { return __base::max(); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type lowest() _NOEXCEPT { return __base::lowest(); }
+
+  static _LIBCPP_CONSTEXPR const int digits       = __base::digits;
+  static _LIBCPP_CONSTEXPR const int digits10     = __base::digits10;
+  static _LIBCPP_CONSTEXPR const int max_digits10 = __base::max_digits10;
+  static _LIBCPP_CONSTEXPR const bool is_signed   = __base::is_signed;
+  static _LIBCPP_CONSTEXPR const bool is_integer  = __base::is_integer;
+  static _LIBCPP_CONSTEXPR const bool is_exact    = __base::is_exact;
+  static _LIBCPP_CONSTEXPR const int radix        = __base::radix;
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type epsilon() _NOEXCEPT {
+    return __base::epsilon();
+  }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type round_error() _NOEXCEPT {
+    return __base::round_error();
+  }
+
+  static _LIBCPP_CONSTEXPR const int min_exponent   = __base::min_exponent;
+  static _LIBCPP_CONSTEXPR const int min_exponent10 = __base::min_exponent10;
+  static _LIBCPP_CONSTEXPR const int max_exponent   = __base::max_exponent;
+  static _LIBCPP_CONSTEXPR const int max_exponent10 = __base::max_exponent10;
+
+  static _LIBCPP_CONSTEXPR const bool has_infinity      = __base::has_infinity;
+  static _LIBCPP_CONSTEXPR const bool has_quiet_NaN     = __base::has_quiet_NaN;
+  static _LIBCPP_CONSTEXPR const bool has_signaling_NaN = __base::has_signaling_NaN;
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  static _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_CONSTEXPR const float_denorm_style has_denorm = __base::has_denorm;
+  static _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_CONSTEXPR const bool has_denorm_loss          = __base::has_denorm_loss;
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type infinity() _NOEXCEPT {
+    return __base::infinity();
+  }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type quiet_NaN() _NOEXCEPT {
+    return __base::quiet_NaN();
+  }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type signaling_NaN() _NOEXCEPT {
+    return __base::signaling_NaN();
+  }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR type denorm_min() _NOEXCEPT {
+    return __base::denorm_min();
+  }
+
+  static _LIBCPP_CONSTEXPR const bool is_iec559  = __base::is_iec559;
+  static _LIBCPP_CONSTEXPR const bool is_bounded = __base::is_bounded;
+  static _LIBCPP_CONSTEXPR const bool is_modulo  = __base::is_modulo;
+
+  static _LIBCPP_CONSTEXPR const bool traps                    = __base::traps;
+  static _LIBCPP_CONSTEXPR const bool tinyness_before          = __base::tinyness_before;
+  static _LIBCPP_CONSTEXPR const float_round_style round_style = __base::round_style;
+};
+
+template <class _Tp>
+_LIBCPP_CONSTEXPR const bool numeric_limits<_Tp>::is_specialized;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const int numeric_limits<_Tp>::digits;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const int numeric_limits<_Tp>::digits10;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const int numeric_limits<_Tp>::max_digits10;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const bool numeric_limits<_Tp>::is_signed;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const bool numeric_limits<_Tp>::is_integer;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const bool numeric_limits<_Tp>::is_exact;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const int numeric_limits<_Tp>::radix;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const int numeric_limits<_Tp>::min_exponent;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const int numeric_limits<_Tp>::min_exponent10;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const int numeric_limits<_Tp>::max_exponent;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const int numeric_limits<_Tp>::max_exponent10;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const bool numeric_limits<_Tp>::has_infinity;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const bool numeric_limits<_Tp>::has_quiet_NaN;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const bool numeric_limits<_Tp>::has_signaling_NaN;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const float_denorm_style numeric_limits<_Tp>::has_denorm;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const bool numeric_limits<_Tp>::has_denorm_loss;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const bool numeric_limits<_Tp>::is_iec559;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const bool numeric_limits<_Tp>::is_bounded;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const bool numeric_limits<_Tp>::is_modulo;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const bool numeric_limits<_Tp>::traps;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const bool numeric_limits<_Tp>::tinyness_before;
+template <class _Tp>
+_LIBCPP_CONSTEXPR const float_round_style numeric_limits<_Tp>::round_style;
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS numeric_limits<const _Tp> : public numeric_limits<_Tp> {};
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS numeric_limits<volatile _Tp> : public numeric_limits<_Tp> {};
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS numeric_limits<const volatile _Tp> : public numeric_limits<_Tp> {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_LIMITS
diff --git a/libcxx/include/__cxx03/list b/libcxx/include/__cxx03/list
new file mode 100644
index 000000000000..76b1d9241b41
--- /dev/null
+++ b/libcxx/include/__cxx03/list
@@ -0,0 +1,1742 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_LIST
+#define _LIBCPP_LIST
+
+/*
+    list synopsis
+
+namespace std
+{
+
+template <class T, class Alloc = allocator<T> >
+class list
+{
+public:
+
+    // types:
+    typedef T value_type;
+    typedef Alloc allocator_type;
+    typedef typename allocator_type::reference reference;
+    typedef typename allocator_type::const_reference const_reference;
+    typedef typename allocator_type::pointer pointer;
+    typedef typename allocator_type::const_pointer const_pointer;
+    typedef implementation-defined iterator;
+    typedef implementation-defined const_iterator;
+    typedef implementation-defined size_type;
+    typedef implementation-defined difference_type;
+    typedef reverse_iterator<iterator> reverse_iterator;
+    typedef reverse_iterator<const_iterator> const_reverse_iterator;
+
+    list()
+        noexcept(is_nothrow_default_constructible<allocator_type>::value);
+    explicit list(const allocator_type& a);
+    explicit list(size_type n);
+    explicit list(size_type n, const allocator_type& a); // C++14
+    list(size_type n, const value_type& value);
+    list(size_type n, const value_type& value, const allocator_type& a);
+    template <class Iter>
+        list(Iter first, Iter last);
+    template <class Iter>
+        list(Iter first, Iter last, const allocator_type& a);
+    template<container-compatible-range<T> R>
+      list(from_range_t, R&& rg, const Allocator& = Allocator()); // C++23
+    list(const list& x);
+    list(const list&, const allocator_type& a);
+    list(list&& x)
+        noexcept(is_nothrow_move_constructible<allocator_type>::value);
+    list(list&&, const allocator_type& a);
+    list(initializer_list<value_type>);
+    list(initializer_list<value_type>, const allocator_type& a);
+
+    ~list();
+
+    list& operator=(const list& x);
+    list& operator=(list&& x)
+        noexcept(
+             allocator_type::propagate_on_container_move_assignment::value &&
+             is_nothrow_move_assignable<allocator_type>::value);
+    list& operator=(initializer_list<value_type>);
+    template <class Iter>
+        void assign(Iter first, Iter last);
+    template<container-compatible-range<T> R>
+      void assign_range(R&& rg); // C++23
+    void assign(size_type n, const value_type& t);
+    void assign(initializer_list<value_type>);
+
+    allocator_type get_allocator() const noexcept;
+
+    iterator begin() noexcept;
+    const_iterator begin() const noexcept;
+    iterator end() noexcept;
+    const_iterator end() const noexcept;
+    reverse_iterator rbegin() noexcept;
+    const_reverse_iterator rbegin() const noexcept;
+    reverse_iterator rend() noexcept;
+    const_reverse_iterator rend() const noexcept;
+    const_iterator cbegin() const noexcept;
+    const_iterator cend() const noexcept;
+    const_reverse_iterator crbegin() const noexcept;
+    const_reverse_iterator crend() const noexcept;
+
+    reference front();
+    const_reference front() const;
+    reference back();
+    const_reference back() const;
+
+    bool empty() const noexcept;
+    size_type size() const noexcept;
+    size_type max_size() const noexcept;
+
+    template <class... Args>
+        reference emplace_front(Args&&... args); // reference in C++17
+    void pop_front();
+    template <class... Args>
+        reference emplace_back(Args&&... args);  // reference in C++17
+    void pop_back();
+    void push_front(const value_type& x);
+    void push_front(value_type&& x);
+    template<container-compatible-range<T> R>
+      void prepend_range(R&& rg); // C++23
+    void push_back(const value_type& x);
+    void push_back(value_type&& x);
+    template<container-compatible-range<T> R>
+      void append_range(R&& rg); // C++23
+    template <class... Args>
+        iterator emplace(const_iterator position, Args&&... args);
+    iterator insert(const_iterator position, const value_type& x);
+    iterator insert(const_iterator position, value_type&& x);
+    iterator insert(const_iterator position, size_type n, const value_type& x);
+    template <class Iter>
+        iterator insert(const_iterator position, Iter first, Iter last);
+    template<container-compatible-range<T> R>
+      iterator insert_range(const_iterator position, R&& rg); // C++23
+    iterator insert(const_iterator position, initializer_list<value_type> il);
+
+    iterator erase(const_iterator position);
+    iterator erase(const_iterator position, const_iterator last);
+
+    void resize(size_type sz);
+    void resize(size_type sz, const value_type& c);
+
+    void swap(list&)
+        noexcept(allocator_traits<allocator_type>::is_always_equal::value);  // C++17
+    void clear() noexcept;
+
+    void splice(const_iterator position, list& x);
+    void splice(const_iterator position, list&& x);
+    void splice(const_iterator position, list& x, const_iterator i);
+    void splice(const_iterator position, list&& x, const_iterator i);
+    void splice(const_iterator position, list& x, const_iterator first,
+                                                  const_iterator last);
+    void splice(const_iterator position, list&& x, const_iterator first,
+                                                  const_iterator last);
+
+    size_type remove(const value_type& value);       // void before C++20
+    template <class Pred>
+      size_type remove_if(Pred pred);                // void before C++20
+    size_type unique();                              // void before C++20
+    template <class BinaryPredicate>
+      size_type unique(BinaryPredicate binary_pred); // void before C++20
+    void merge(list& x);
+    void merge(list&& x);
+    template <class Compare>
+        void merge(list& x, Compare comp);
+    template <class Compare>
+        void merge(list&& x, Compare comp);
+    void sort();
+    template <class Compare>
+        void sort(Compare comp);
+    void reverse() noexcept;
+};
+
+
+template <class InputIterator, class Allocator = allocator<typename iterator_traits<InputIterator>::value_type>>
+    list(InputIterator, InputIterator, Allocator = Allocator())
+    -> list<typename iterator_traits<InputIterator>::value_type, Allocator>;  // C++17
+
+template<ranges::input_range R, class Allocator = allocator<ranges::range_value_t<R>>>
+  list(from_range_t, R&&, Allocator = Allocator())
+    -> list<ranges::range_value_t<R>, Allocator>; // C++23
+
+template <class T, class Alloc>
+    bool operator==(const list<T,Alloc>& x, const list<T,Alloc>& y);
+template <class T, class Alloc>
+    bool operator< (const list<T,Alloc>& x, const list<T,Alloc>& y);     // removed in C++20
+template <class T, class Alloc>
+    bool operator!=(const list<T,Alloc>& x, const list<T,Alloc>& y);     // removed in C++20
+template <class T, class Alloc>
+    bool operator> (const list<T,Alloc>& x, const list<T,Alloc>& y);     // removed in C++20
+template <class T, class Alloc>
+    bool operator>=(const list<T,Alloc>& x, const list<T,Alloc>& y);     // removed in C++20
+template <class T, class Alloc>
+    bool operator<=(const list<T,Alloc>& x, const list<T,Alloc>& y);     // removed in C++20
+template<class T, class Allocator>
+  synth-three-way-result<T> operator<=>(const list<T, Allocator>& x,
+                                        const list<T, Allocator>& y);    // since C++20
+
+template <class T, class Alloc>
+    void swap(list<T,Alloc>& x, list<T,Alloc>& y)
+         noexcept(noexcept(x.swap(y)));
+
+template <class T, class Allocator, class U>
+    typename list<T, Allocator>::size_type
+    erase(list<T, Allocator>& c, const U& value);       // since C++20
+template <class T, class Allocator, class Predicate>
+    typename list<T, Allocator>::size_type
+    erase_if(list<T, Allocator>& c, Predicate pred);    // since C++20
+
+}  // std
+
+*/
+
+#include <__algorithm/comp.h>
+#include <__algorithm/equal.h>
+#include <__algorithm/lexicographical_compare.h>
+#include <__algorithm/lexicographical_compare_three_way.h>
+#include <__algorithm/min.h>
+#include <__assert>
+#include <__config>
+#include <__format/enable_insertable.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/move_iterator.h>
+#include <__iterator/next.h>
+#include <__iterator/prev.h>
+#include <__iterator/reverse_iterator.h>
+#include <__memory/addressof.h>
+#include <__memory/allocation_guard.h>
+#include <__memory/allocator.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/compressed_pair.h>
+#include <__memory/construct_at.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/swap_allocator.h>
+#include <__memory_resource/polymorphic_allocator.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/container_compatible_range.h>
+#include <__ranges/from_range.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/is_allocator.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/type_identity.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+#include <cstring>
+#include <limits>
+#include <new> // __launder
+#include <version>
+
+// standard-mandated includes
+
+// [iterator.range]
+#include <__iterator/access.h>
+#include <__iterator/data.h>
+#include <__iterator/empty.h>
+#include <__iterator/reverse_access.h>
+#include <__iterator/size.h>
+
+// [list.syn]
+#include <compare>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _VoidPtr>
+struct __list_node;
+template <class _Tp, class _VoidPtr>
+struct __list_node_base;
+
+template <class _Tp, class _VoidPtr>
+struct __list_node_pointer_traits {
+  typedef __rebind_pointer_t<_VoidPtr, __list_node<_Tp, _VoidPtr> > __node_pointer;
+  typedef __rebind_pointer_t<_VoidPtr, __list_node_base<_Tp, _VoidPtr> > __base_pointer;
+
+#if defined(_LIBCPP_ABI_LIST_REMOVE_NODE_POINTER_UB)
+  typedef __base_pointer __link_pointer;
+#else
+  typedef __conditional_t<is_pointer<_VoidPtr>::value, __base_pointer, __node_pointer> __link_pointer;
+#endif
+
+  typedef __conditional_t<is_same<__link_pointer, __node_pointer>::value, __base_pointer, __node_pointer>
+      __non_link_pointer;
+
+  static _LIBCPP_HIDE_FROM_ABI __link_pointer __unsafe_link_pointer_cast(__link_pointer __p) { return __p; }
+
+  static _LIBCPP_HIDE_FROM_ABI __link_pointer __unsafe_link_pointer_cast(__non_link_pointer __p) {
+    return static_cast<__link_pointer>(static_cast<_VoidPtr>(__p));
+  }
+};
+
+template <class _Tp, class _VoidPtr>
+struct __list_node_base {
+  typedef __list_node_pointer_traits<_Tp, _VoidPtr> _NodeTraits;
+  typedef typename _NodeTraits::__node_pointer __node_pointer;
+  typedef typename _NodeTraits::__base_pointer __base_pointer;
+  typedef typename _NodeTraits::__link_pointer __link_pointer;
+
+  __link_pointer __prev_;
+  __link_pointer __next_;
+
+  _LIBCPP_HIDE_FROM_ABI __list_node_base()
+      : __prev_(_NodeTraits::__unsafe_link_pointer_cast(__self())),
+        __next_(_NodeTraits::__unsafe_link_pointer_cast(__self())) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit __list_node_base(__link_pointer __prev, __link_pointer __next)
+      : __prev_(__prev), __next_(__next) {}
+
+  _LIBCPP_HIDE_FROM_ABI __base_pointer __self() { return pointer_traits<__base_pointer>::pointer_to(*this); }
+
+  _LIBCPP_HIDE_FROM_ABI __node_pointer __as_node() { return static_cast<__node_pointer>(__self()); }
+};
+
+template <class _Tp, class _VoidPtr>
+struct __list_node : public __list_node_base<_Tp, _VoidPtr> {
+  // We allow starting the lifetime of nodes without initializing the value held by the node,
+  // since that is handled by the list itself in order to be allocator-aware.
+#ifndef _LIBCPP_CXX03_LANG
+
+private:
+  union {
+    _Tp __value_;
+  };
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; }
+#else
+
+private:
+  _ALIGNAS_TYPE(_Tp) char __buffer_[sizeof(_Tp)];
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return *std::__launder(reinterpret_cast<_Tp*>(&__buffer_)); }
+#endif
+
+  typedef __list_node_base<_Tp, _VoidPtr> __base;
+  typedef typename __base::__link_pointer __link_pointer;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __list_node(__link_pointer __prev, __link_pointer __next) : __base(__prev, __next) {}
+  _LIBCPP_HIDE_FROM_ABI ~__list_node() {}
+
+  _LIBCPP_HIDE_FROM_ABI __link_pointer __as_link() { return static_cast<__link_pointer>(__base::__self()); }
+};
+
+template <class _Tp, class _Alloc = allocator<_Tp> >
+class _LIBCPP_TEMPLATE_VIS list;
+template <class _Tp, class _Alloc>
+class __list_imp;
+template <class _Tp, class _VoidPtr>
+class _LIBCPP_TEMPLATE_VIS __list_const_iterator;
+
+template <class _Tp, class _VoidPtr>
+class _LIBCPP_TEMPLATE_VIS __list_iterator {
+  typedef __list_node_pointer_traits<_Tp, _VoidPtr> _NodeTraits;
+  typedef typename _NodeTraits::__link_pointer __link_pointer;
+
+  __link_pointer __ptr_;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __list_iterator(__link_pointer __p) _NOEXCEPT : __ptr_(__p) {}
+
+  template <class, class>
+  friend class list;
+  template <class, class>
+  friend class __list_imp;
+  template <class, class>
+  friend class __list_const_iterator;
+
+public:
+  typedef bidirectional_iterator_tag iterator_category;
+  typedef _Tp value_type;
+  typedef value_type& reference;
+  typedef __rebind_pointer_t<_VoidPtr, value_type> pointer;
+  typedef typename pointer_traits<pointer>::difference_type difference_type;
+
+  _LIBCPP_HIDE_FROM_ABI __list_iterator() _NOEXCEPT : __ptr_(nullptr) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __ptr_->__as_node()->__get_value(); }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
+    return pointer_traits<pointer>::pointer_to(__ptr_->__as_node()->__get_value());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __list_iterator& operator++() {
+    __ptr_ = __ptr_->__next_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __list_iterator operator++(int) {
+    __list_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __list_iterator& operator--() {
+    __ptr_ = __ptr_->__prev_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __list_iterator operator--(int) {
+    __list_iterator __t(*this);
+    --(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __list_iterator& __x, const __list_iterator& __y) {
+    return __x.__ptr_ == __y.__ptr_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __list_iterator& __x, const __list_iterator& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _Tp, class _VoidPtr>
+class _LIBCPP_TEMPLATE_VIS __list_const_iterator {
+  typedef __list_node_pointer_traits<_Tp, _VoidPtr> _NodeTraits;
+  typedef typename _NodeTraits::__link_pointer __link_pointer;
+
+  __link_pointer __ptr_;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __list_const_iterator(__link_pointer __p) _NOEXCEPT : __ptr_(__p) {}
+
+  template <class, class>
+  friend class list;
+  template <class, class>
+  friend class __list_imp;
+
+public:
+  typedef bidirectional_iterator_tag iterator_category;
+  typedef _Tp value_type;
+  typedef const value_type& reference;
+  typedef __rebind_pointer_t<_VoidPtr, const value_type> pointer;
+  typedef typename pointer_traits<pointer>::difference_type difference_type;
+
+  _LIBCPP_HIDE_FROM_ABI __list_const_iterator() _NOEXCEPT : __ptr_(nullptr) {}
+  _LIBCPP_HIDE_FROM_ABI __list_const_iterator(const __list_iterator<_Tp, _VoidPtr>& __p) _NOEXCEPT
+      : __ptr_(__p.__ptr_) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __ptr_->__as_node()->__get_value(); }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
+    return pointer_traits<pointer>::pointer_to(__ptr_->__as_node()->__get_value());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __list_const_iterator& operator++() {
+    __ptr_ = __ptr_->__next_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __list_const_iterator operator++(int) {
+    __list_const_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __list_const_iterator& operator--() {
+    __ptr_ = __ptr_->__prev_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __list_const_iterator operator--(int) {
+    __list_const_iterator __t(*this);
+    --(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __list_const_iterator& __x, const __list_const_iterator& __y) {
+    return __x.__ptr_ == __y.__ptr_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __list_const_iterator& __x, const __list_const_iterator& __y) {
+    return !(__x == __y);
+  }
+};
+
+template <class _Tp, class _Alloc>
+class __list_imp {
+public:
+  __list_imp(const __list_imp&)            = delete;
+  __list_imp& operator=(const __list_imp&) = delete;
+
+  typedef _Alloc allocator_type;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+  typedef typename __alloc_traits::size_type size_type;
+
+protected:
+  typedef _Tp value_type;
+  typedef typename __alloc_traits::void_pointer __void_pointer;
+  typedef __list_iterator<value_type, __void_pointer> iterator;
+  typedef __list_const_iterator<value_type, __void_pointer> const_iterator;
+  typedef __list_node_base<value_type, __void_pointer> __node_base;
+  typedef __list_node<value_type, __void_pointer> __node_type;
+  typedef __rebind_alloc<__alloc_traits, __node_type> __node_allocator;
+  typedef allocator_traits<__node_allocator> __node_alloc_traits;
+  typedef typename __node_alloc_traits::pointer __node_pointer;
+  typedef typename __node_alloc_traits::pointer __node_const_pointer;
+  typedef __list_node_pointer_traits<value_type, __void_pointer> __node_pointer_traits;
+  typedef typename __node_pointer_traits::__link_pointer __link_pointer;
+  typedef __link_pointer __link_const_pointer;
+  typedef typename __alloc_traits::pointer pointer;
+  typedef typename __alloc_traits::const_pointer const_pointer;
+  typedef typename __alloc_traits::difference_type difference_type;
+
+  typedef __rebind_alloc<__alloc_traits, __node_base> __node_base_allocator;
+  typedef typename allocator_traits<__node_base_allocator>::pointer __node_base_pointer;
+  static_assert(!is_same<allocator_type, __node_allocator>::value,
+                "internal allocator type must differ from user-specified type; otherwise overload resolution breaks");
+
+  __node_base __end_;
+  __compressed_pair<size_type, __node_allocator> __size_alloc_;
+
+  _LIBCPP_HIDE_FROM_ABI __link_pointer __end_as_link() const _NOEXCEPT {
+    return __node_pointer_traits::__unsafe_link_pointer_cast(const_cast<__node_base&>(__end_).__self());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_type& __sz() _NOEXCEPT { return __size_alloc_.first(); }
+  _LIBCPP_HIDE_FROM_ABI const size_type& __sz() const _NOEXCEPT { return __size_alloc_.first(); }
+  _LIBCPP_HIDE_FROM_ABI __node_allocator& __node_alloc() _NOEXCEPT { return __size_alloc_.second(); }
+  _LIBCPP_HIDE_FROM_ABI const __node_allocator& __node_alloc() const _NOEXCEPT { return __size_alloc_.second(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type __node_alloc_max_size() const _NOEXCEPT {
+    return __node_alloc_traits::max_size(__node_alloc());
+  }
+  _LIBCPP_HIDE_FROM_ABI static void __unlink_nodes(__link_pointer __f, __link_pointer __l) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI __list_imp() _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value);
+  _LIBCPP_HIDE_FROM_ABI __list_imp(const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI __list_imp(const __node_allocator& __a);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI __list_imp(__node_allocator&& __a) _NOEXCEPT;
+#endif
+  _LIBCPP_HIDE_FROM_ABI ~__list_imp();
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __sz() == 0; }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return iterator(__end_.__next_); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return const_iterator(__end_.__next_); }
+  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return iterator(__end_as_link()); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return const_iterator(__end_as_link()); }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(__list_imp& __c)
+#if _LIBCPP_STD_VER >= 14
+      _NOEXCEPT;
+#else
+      _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>);
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __list_imp& __c) {
+    __copy_assign_alloc(
+        __c, integral_constant<bool, __node_alloc_traits::propagate_on_container_copy_assignment::value>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__list_imp& __c)
+      _NOEXCEPT_(!__node_alloc_traits::propagate_on_container_move_assignment::value ||
+                 is_nothrow_move_assignable<__node_allocator>::value) {
+    __move_assign_alloc(
+        __c, integral_constant<bool, __node_alloc_traits::propagate_on_container_move_assignment::value>());
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI __node_pointer __create_node(__link_pointer __prev, __link_pointer __next, _Args&&... __args) {
+    __node_allocator& __alloc = __node_alloc();
+    __allocation_guard<__node_allocator> __guard(__alloc, 1);
+    // Begin the lifetime of the node itself. Note that this doesn't begin the lifetime of the value
+    // held inside the node, since we need to use the allocator's construct() method for that.
+    //
+    // We don't use the allocator's construct() method to construct the node itself since the
+    // Cpp17FooInsertable named requirements don't require the allocator's construct() method
+    // to work on anything other than the value_type.
+    std::__construct_at(std::addressof(*__guard.__get()), __prev, __next);
+
+    // Now construct the value_type using the allocator's construct() method.
+    __node_alloc_traits::construct(
+        __alloc, std::addressof(__guard.__get()->__get_value()), std::forward<_Args>(__args)...);
+    return __guard.__release_ptr();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __delete_node(__node_pointer __node) {
+    // For the same reason as above, we use the allocator's destroy() method for the value_type,
+    // but not for the node itself.
+    __node_allocator& __alloc = __node_alloc();
+    __node_alloc_traits::destroy(__alloc, std::addressof(__node->__get_value()));
+    std::__destroy_at(std::addressof(*__node));
+    __node_alloc_traits::deallocate(__alloc, __node, 1);
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __list_imp& __c, true_type) {
+    if (__node_alloc() != __c.__node_alloc())
+      clear();
+    __node_alloc() = __c.__node_alloc();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __list_imp&, false_type) {}
+
+  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__list_imp& __c, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<__node_allocator>::value) {
+    __node_alloc() = std::move(__c.__node_alloc());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__list_imp&, false_type) _NOEXCEPT {}
+};
+
+// Unlink nodes [__f, __l]
+template <class _Tp, class _Alloc>
+inline void __list_imp<_Tp, _Alloc>::__unlink_nodes(__link_pointer __f, __link_pointer __l) _NOEXCEPT {
+  __f->__prev_->__next_ = __l->__next_;
+  __l->__next_->__prev_ = __f->__prev_;
+}
+
+template <class _Tp, class _Alloc>
+inline __list_imp<_Tp, _Alloc>::__list_imp() _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value)
+    : __size_alloc_(0, __default_init_tag()) {}
+
+template <class _Tp, class _Alloc>
+inline __list_imp<_Tp, _Alloc>::__list_imp(const allocator_type& __a) : __size_alloc_(0, __node_allocator(__a)) {}
+
+template <class _Tp, class _Alloc>
+inline __list_imp<_Tp, _Alloc>::__list_imp(const __node_allocator& __a) : __size_alloc_(0, __a) {}
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Tp, class _Alloc>
+inline __list_imp<_Tp, _Alloc>::__list_imp(__node_allocator&& __a) _NOEXCEPT : __size_alloc_(0, std::move(__a)) {}
+#endif
+
+template <class _Tp, class _Alloc>
+__list_imp<_Tp, _Alloc>::~__list_imp() {
+  clear();
+}
+
+template <class _Tp, class _Alloc>
+void __list_imp<_Tp, _Alloc>::clear() _NOEXCEPT {
+  if (!empty()) {
+    __link_pointer __f = __end_.__next_;
+    __link_pointer __l = __end_as_link();
+    __unlink_nodes(__f, __l->__prev_);
+    __sz() = 0;
+    while (__f != __l) {
+      __node_pointer __np = __f->__as_node();
+      __f                 = __f->__next_;
+      __delete_node(__np);
+    }
+  }
+}
+
+template <class _Tp, class _Alloc>
+void __list_imp<_Tp, _Alloc>::swap(__list_imp& __c)
+#if _LIBCPP_STD_VER >= 14
+    _NOEXCEPT
+#else
+    _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>)
+#endif
+{
+  _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+      __alloc_traits::propagate_on_container_swap::value || this->__node_alloc() == __c.__node_alloc(),
+      "list::swap: Either propagate_on_container_swap must be true"
+      " or the allocators must compare equal");
+  using std::swap;
+  std::__swap_allocator(__node_alloc(), __c.__node_alloc());
+  swap(__sz(), __c.__sz());
+  swap(__end_, __c.__end_);
+  if (__sz() == 0)
+    __end_.__next_ = __end_.__prev_ = __end_as_link();
+  else
+    __end_.__prev_->__next_ = __end_.__next_->__prev_ = __end_as_link();
+  if (__c.__sz() == 0)
+    __c.__end_.__next_ = __c.__end_.__prev_ = __c.__end_as_link();
+  else
+    __c.__end_.__prev_->__next_ = __c.__end_.__next_->__prev_ = __c.__end_as_link();
+}
+
+template <class _Tp, class _Alloc /*= allocator<_Tp>*/>
+class _LIBCPP_TEMPLATE_VIS list : private __list_imp<_Tp, _Alloc> {
+  typedef __list_imp<_Tp, _Alloc> base;
+  typedef typename base::__node_type __node_type;
+  typedef typename base::__node_allocator __node_allocator;
+  typedef typename base::__node_pointer __node_pointer;
+  typedef typename base::__node_alloc_traits __node_alloc_traits;
+  typedef typename base::__node_base __node_base;
+  typedef typename base::__node_base_pointer __node_base_pointer;
+  typedef typename base::__link_pointer __link_pointer;
+
+public:
+  typedef _Tp value_type;
+  typedef _Alloc allocator_type;
+  static_assert(__check_valid_allocator<allocator_type>::value);
+  static_assert(is_same<value_type, typename allocator_type::value_type>::value,
+                "Allocator::value_type must be same type as value_type");
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+  typedef typename base::pointer pointer;
+  typedef typename base::const_pointer const_pointer;
+  typedef typename base::size_type size_type;
+  typedef typename base::difference_type difference_type;
+  typedef typename base::iterator iterator;
+  typedef typename base::const_iterator const_iterator;
+  typedef std::reverse_iterator<iterator> reverse_iterator;
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+#if _LIBCPP_STD_VER >= 20
+  typedef size_type __remove_return_type;
+#else
+  typedef void __remove_return_type;
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI list() _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value) {}
+  _LIBCPP_HIDE_FROM_ABI explicit list(const allocator_type& __a) : base(__a) {}
+  _LIBCPP_HIDE_FROM_ABI explicit list(size_type __n);
+#if _LIBCPP_STD_VER >= 14
+  _LIBCPP_HIDE_FROM_ABI explicit list(size_type __n, const allocator_type& __a);
+#endif
+  _LIBCPP_HIDE_FROM_ABI list(size_type __n, const value_type& __x);
+  template <__enable_if_t<__is_allocator<_Alloc>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI list(size_type __n, const value_type& __x, const allocator_type& __a) : base(__a) {
+    for (; __n > 0; --__n)
+      push_back(__x);
+  }
+
+  template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI list(_InpIter __f, _InpIter __l);
+
+  template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI list(_InpIter __f, _InpIter __l, const allocator_type& __a);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI list(from_range_t, _Range&& __range, const allocator_type& __a = allocator_type()) : base(__a) {
+    prepend_range(std::forward<_Range>(__range));
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI list(const list& __c);
+  _LIBCPP_HIDE_FROM_ABI list(const list& __c, const __type_identity_t<allocator_type>& __a);
+  _LIBCPP_HIDE_FROM_ABI list& operator=(const list& __c);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI list(initializer_list<value_type> __il);
+  _LIBCPP_HIDE_FROM_ABI list(initializer_list<value_type> __il, const allocator_type& __a);
+
+  _LIBCPP_HIDE_FROM_ABI list(list&& __c) _NOEXCEPT_(is_nothrow_move_constructible<__node_allocator>::value);
+  _LIBCPP_HIDE_FROM_ABI list(list&& __c, const __type_identity_t<allocator_type>& __a);
+  _LIBCPP_HIDE_FROM_ABI list& operator=(list&& __c)
+      _NOEXCEPT_(__node_alloc_traits::propagate_on_container_move_assignment::value&&
+                     is_nothrow_move_assignable<__node_allocator>::value);
+
+  _LIBCPP_HIDE_FROM_ABI list& operator=(initializer_list<value_type> __il) {
+    assign(__il.begin(), __il.end());
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void assign(initializer_list<value_type> __il) { assign(__il.begin(), __il.end()); }
+#endif // _LIBCPP_CXX03_LANG
+
+  template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void assign(_InpIter __f, _InpIter __l);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI void assign_range(_Range&& __range) {
+    __assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI void assign(size_type __n, const value_type& __x);
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return base::__sz(); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return base::empty(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
+    return std::min<size_type>(base::__node_alloc_max_size(), numeric_limits<difference_type >::max());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return base::begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return base::begin(); }
+  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return base::end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return base::end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return base::begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return base::end(); }
+
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
+
+  _LIBCPP_HIDE_FROM_ABI reference front() {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::front called on empty list");
+    return base::__end_.__next_->__as_node()->__get_value();
+  }
+  _LIBCPP_HIDE_FROM_ABI const_reference front() const {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::front called on empty list");
+    return base::__end_.__next_->__as_node()->__get_value();
+  }
+  _LIBCPP_HIDE_FROM_ABI reference back() {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::back called on empty list");
+    return base::__end_.__prev_->__as_node()->__get_value();
+  }
+  _LIBCPP_HIDE_FROM_ABI const_reference back() const {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::back called on empty list");
+    return base::__end_.__prev_->__as_node()->__get_value();
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI void push_front(value_type&& __x);
+  _LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __x);
+
+#  if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI void prepend_range(_Range&& __range) {
+    insert_range(begin(), std::forward<_Range>(__range));
+  }
+
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI void append_range(_Range&& __range) {
+    insert_range(end(), std::forward<_Range>(__range));
+  }
+#  endif
+
+  template <class... _Args>
+#  if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI reference emplace_front(_Args&&... __args);
+#  else
+  _LIBCPP_HIDE_FROM_ABI void emplace_front(_Args&&... __args);
+#  endif
+  template <class... _Args>
+#  if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI reference emplace_back(_Args&&... __args);
+#  else
+  _LIBCPP_HIDE_FROM_ABI void emplace_back(_Args&&... __args);
+#  endif
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __p, _Args&&... __args);
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __x);
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, initializer_list<value_type> __il) {
+    return insert(__p, __il.begin(), __il.end());
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI void push_front(const value_type& __x);
+  _LIBCPP_HIDE_FROM_ABI void push_back(const value_type& __x);
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class _Arg>
+  _LIBCPP_HIDE_FROM_ABI void __emplace_back(_Arg&& __arg) {
+    emplace_back(std::forward<_Arg>(__arg));
+  }
+#else
+  _LIBCPP_HIDE_FROM_ABI void __emplace_back(value_type const& __arg) { push_back(__arg); }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __x);
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, size_type __n, const value_type& __x);
+
+  template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, _InpIter __f, _InpIter __l);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI iterator insert_range(const_iterator __position, _Range&& __range) {
+    return __insert_with_sentinel(__position, ranges::begin(__range), ranges::end(__range));
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI void swap(list& __c)
+#if _LIBCPP_STD_VER >= 14
+      _NOEXCEPT
+#else
+      _NOEXCEPT_(!__node_alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__node_allocator>)
+#endif
+  {
+    base::swap(__c);
+  }
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { base::clear(); }
+
+  _LIBCPP_HIDE_FROM_ABI void pop_front();
+  _LIBCPP_HIDE_FROM_ABI void pop_back();
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p);
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __f, const_iterator __l);
+
+  _LIBCPP_HIDE_FROM_ABI void resize(size_type __n);
+  _LIBCPP_HIDE_FROM_ABI void resize(size_type __n, const value_type& __x);
+
+  _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list& __c);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list&& __c) { splice(__p, __c); }
+  _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list&& __c, const_iterator __i) { splice(__p, __c, __i); }
+  _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list&& __c, const_iterator __f, const_iterator __l) {
+    splice(__p, __c, __f, __l);
+  }
+#endif
+  _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list& __c, const_iterator __i);
+  _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list& __c, const_iterator __f, const_iterator __l);
+
+  _LIBCPP_HIDE_FROM_ABI __remove_return_type remove(const value_type& __x);
+  template <class _Pred>
+  _LIBCPP_HIDE_FROM_ABI __remove_return_type remove_if(_Pred __pred);
+  _LIBCPP_HIDE_FROM_ABI __remove_return_type unique() { return unique(__equal_to()); }
+  template <class _BinaryPred>
+  _LIBCPP_HIDE_FROM_ABI __remove_return_type unique(_BinaryPred __binary_pred);
+  _LIBCPP_HIDE_FROM_ABI void merge(list& __c);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI void merge(list&& __c) { merge(__c); }
+
+  template <class _Comp>
+  _LIBCPP_HIDE_FROM_ABI void merge(list&& __c, _Comp __comp) {
+    merge(__c, __comp);
+  }
+#endif
+  template <class _Comp>
+  _LIBCPP_HIDE_FROM_ABI void merge(list& __c, _Comp __comp);
+
+  _LIBCPP_HIDE_FROM_ABI void sort();
+  template <class _Comp>
+  _LIBCPP_HIDE_FROM_ABI void sort(_Comp __comp);
+
+  _LIBCPP_HIDE_FROM_ABI void reverse() _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI bool __invariants() const;
+
+private:
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __f, _Sentinel __l);
+
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI iterator __insert_with_sentinel(const_iterator __p, _Iterator __f, _Sentinel __l);
+
+  _LIBCPP_HIDE_FROM_ABI static void __link_nodes(__link_pointer __p, __link_pointer __f, __link_pointer __l);
+  _LIBCPP_HIDE_FROM_ABI void __link_nodes_at_front(__link_pointer __f, __link_pointer __l);
+  _LIBCPP_HIDE_FROM_ABI void __link_nodes_at_back(__link_pointer __f, __link_pointer __l);
+  _LIBCPP_HIDE_FROM_ABI iterator __iterator(size_type __n);
+  // TODO: Make this _LIBCPP_HIDE_FROM_ABI
+  template <class _Comp>
+  _LIBCPP_HIDDEN static iterator __sort(iterator __f1, iterator __e2, size_type __n, _Comp& __comp);
+
+  _LIBCPP_HIDE_FROM_ABI void __move_assign(list& __c, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<__node_allocator>::value);
+  _LIBCPP_HIDE_FROM_ABI void __move_assign(list& __c, false_type);
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator,
+          class _Alloc = allocator<__iter_value_type<_InputIterator>>,
+          class        = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class        = enable_if_t<__is_allocator<_Alloc>::value> >
+list(_InputIterator, _InputIterator) -> list<__iter_value_type<_InputIterator>, _Alloc>;
+
+template <class _InputIterator,
+          class _Alloc,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<__is_allocator<_Alloc>::value> >
+list(_InputIterator, _InputIterator, _Alloc) -> list<__iter_value_type<_InputIterator>, _Alloc>;
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range,
+          class _Alloc = allocator<ranges::range_value_t<_Range>>,
+          class        = enable_if_t<__is_allocator<_Alloc>::value> >
+list(from_range_t, _Range&&, _Alloc = _Alloc()) -> list<ranges::range_value_t<_Range>, _Alloc>;
+#endif
+
+// Link in nodes [__f, __l] just prior to __p
+template <class _Tp, class _Alloc>
+inline void list<_Tp, _Alloc>::__link_nodes(__link_pointer __p, __link_pointer __f, __link_pointer __l) {
+  __p->__prev_->__next_ = __f;
+  __f->__prev_          = __p->__prev_;
+  __p->__prev_          = __l;
+  __l->__next_          = __p;
+}
+
+// Link in nodes [__f, __l] at the front of the list
+template <class _Tp, class _Alloc>
+inline void list<_Tp, _Alloc>::__link_nodes_at_front(__link_pointer __f, __link_pointer __l) {
+  __f->__prev_          = base::__end_as_link();
+  __l->__next_          = base::__end_.__next_;
+  __l->__next_->__prev_ = __l;
+  base::__end_.__next_  = __f;
+}
+
+// Link in nodes [__f, __l] at the back of the list
+template <class _Tp, class _Alloc>
+inline void list<_Tp, _Alloc>::__link_nodes_at_back(__link_pointer __f, __link_pointer __l) {
+  __l->__next_          = base::__end_as_link();
+  __f->__prev_          = base::__end_.__prev_;
+  __f->__prev_->__next_ = __f;
+  base::__end_.__prev_  = __l;
+}
+
+template <class _Tp, class _Alloc>
+inline typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::__iterator(size_type __n) {
+  return __n <= base::__sz() / 2 ? std::next(begin(), __n) : std::prev(end(), base::__sz() - __n);
+}
+
+template <class _Tp, class _Alloc>
+list<_Tp, _Alloc>::list(size_type __n) {
+  for (; __n > 0; --__n)
+#ifndef _LIBCPP_CXX03_LANG
+    emplace_back();
+#else
+    push_back(value_type());
+#endif
+}
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp, class _Alloc>
+list<_Tp, _Alloc>::list(size_type __n, const allocator_type& __a) : base(__a) {
+  for (; __n > 0; --__n)
+    emplace_back();
+}
+#endif
+
+template <class _Tp, class _Alloc>
+list<_Tp, _Alloc>::list(size_type __n, const value_type& __x) {
+  for (; __n > 0; --__n)
+    push_back(__x);
+}
+
+template <class _Tp, class _Alloc>
+template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> >
+list<_Tp, _Alloc>::list(_InpIter __f, _InpIter __l) {
+  for (; __f != __l; ++__f)
+    __emplace_back(*__f);
+}
+
+template <class _Tp, class _Alloc>
+template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> >
+list<_Tp, _Alloc>::list(_InpIter __f, _InpIter __l, const allocator_type& __a) : base(__a) {
+  for (; __f != __l; ++__f)
+    __emplace_back(*__f);
+}
+
+template <class _Tp, class _Alloc>
+list<_Tp, _Alloc>::list(const list& __c)
+    : base(__node_alloc_traits::select_on_container_copy_construction(__c.__node_alloc())) {
+  for (const_iterator __i = __c.begin(), __e = __c.end(); __i != __e; ++__i)
+    push_back(*__i);
+}
+
+template <class _Tp, class _Alloc>
+list<_Tp, _Alloc>::list(const list& __c, const __type_identity_t<allocator_type>& __a) : base(__a) {
+  for (const_iterator __i = __c.begin(), __e = __c.end(); __i != __e; ++__i)
+    push_back(*__i);
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Alloc>
+list<_Tp, _Alloc>::list(initializer_list<value_type> __il, const allocator_type& __a) : base(__a) {
+  for (typename initializer_list<value_type>::const_iterator __i = __il.begin(), __e = __il.end(); __i != __e; ++__i)
+    push_back(*__i);
+}
+
+template <class _Tp, class _Alloc>
+list<_Tp, _Alloc>::list(initializer_list<value_type> __il) {
+  for (typename initializer_list<value_type>::const_iterator __i = __il.begin(), __e = __il.end(); __i != __e; ++__i)
+    push_back(*__i);
+}
+
+template <class _Tp, class _Alloc>
+inline list<_Tp, _Alloc>::list(list&& __c) noexcept(is_nothrow_move_constructible<__node_allocator>::value)
+    : base(std::move(__c.__node_alloc())) {
+  splice(end(), __c);
+}
+
+template <class _Tp, class _Alloc>
+inline list<_Tp, _Alloc>::list(list&& __c, const __type_identity_t<allocator_type>& __a) : base(__a) {
+  if (__a == __c.get_allocator())
+    splice(end(), __c);
+  else {
+    typedef move_iterator<iterator> _Ip;
+    assign(_Ip(__c.begin()), _Ip(__c.end()));
+  }
+}
+
+template <class _Tp, class _Alloc>
+inline list<_Tp, _Alloc>& list<_Tp, _Alloc>::operator=(list&& __c) noexcept(
+    __node_alloc_traits::propagate_on_container_move_assignment::value &&
+    is_nothrow_move_assignable<__node_allocator>::value) {
+  __move_assign(__c, integral_constant<bool, __node_alloc_traits::propagate_on_container_move_assignment::value>());
+  return *this;
+}
+
+template <class _Tp, class _Alloc>
+void list<_Tp, _Alloc>::__move_assign(list& __c, false_type) {
+  if (base::__node_alloc() != __c.__node_alloc()) {
+    typedef move_iterator<iterator> _Ip;
+    assign(_Ip(__c.begin()), _Ip(__c.end()));
+  } else
+    __move_assign(__c, true_type());
+}
+
+template <class _Tp, class _Alloc>
+void list<_Tp, _Alloc>::__move_assign(list& __c,
+                                      true_type) noexcept(is_nothrow_move_assignable<__node_allocator>::value) {
+  clear();
+  base::__move_assign_alloc(__c);
+  splice(end(), __c);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Alloc>
+inline list<_Tp, _Alloc>& list<_Tp, _Alloc>::operator=(const list& __c) {
+  if (this != std::addressof(__c)) {
+    base::__copy_assign_alloc(__c);
+    assign(__c.begin(), __c.end());
+  }
+  return *this;
+}
+
+template <class _Tp, class _Alloc>
+template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> >
+void list<_Tp, _Alloc>::assign(_InpIter __f, _InpIter __l) {
+  __assign_with_sentinel(__f, __l);
+}
+
+template <class _Tp, class _Alloc>
+template <class _Iterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI void list<_Tp, _Alloc>::__assign_with_sentinel(_Iterator __f, _Sentinel __l) {
+  iterator __i = begin();
+  iterator __e = end();
+  for (; __f != __l && __i != __e; ++__f, (void)++__i)
+    *__i = *__f;
+  if (__i == __e)
+    __insert_with_sentinel(__e, std::move(__f), std::move(__l));
+  else
+    erase(__i, __e);
+}
+
+template <class _Tp, class _Alloc>
+void list<_Tp, _Alloc>::assign(size_type __n, const value_type& __x) {
+  iterator __i = begin();
+  iterator __e = end();
+  for (; __n > 0 && __i != __e; --__n, (void)++__i)
+    *__i = __x;
+  if (__i == __e)
+    insert(__e, __n, __x);
+  else
+    erase(__i, __e);
+}
+
+template <class _Tp, class _Alloc>
+inline _Alloc list<_Tp, _Alloc>::get_allocator() const _NOEXCEPT {
+  return allocator_type(base::__node_alloc());
+}
+
+template <class _Tp, class _Alloc>
+typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::insert(const_iterator __p, const value_type& __x) {
+  __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, __x);
+  __link_nodes(__p.__ptr_, __node->__as_link(), __node->__as_link());
+  ++base::__sz();
+  return iterator(__node->__as_link());
+}
+
+template <class _Tp, class _Alloc>
+typename list<_Tp, _Alloc>::iterator
+list<_Tp, _Alloc>::insert(const_iterator __p, size_type __n, const value_type& __x) {
+  iterator __r(__p.__ptr_);
+  if (__n > 0) {
+    size_type __ds        = 0;
+    __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, __x);
+    ++__ds;
+    __r          = iterator(__node->__as_link());
+    iterator __e = __r;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      for (--__n; __n != 0; --__n, (void)++__e, ++__ds) {
+        __e.__ptr_->__next_ = this->__create_node(/* prev = */ __e.__ptr_, /* next = */ nullptr, __x)->__as_link();
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      while (true) {
+        __link_pointer __prev    = __e.__ptr_->__prev_;
+        __node_pointer __current = __e.__ptr_->__as_node();
+        this->__delete_node(__current);
+        if (__prev == 0)
+          break;
+        __e = iterator(__prev);
+      }
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __link_nodes(__p.__ptr_, __r.__ptr_, __e.__ptr_);
+    base::__sz() += __ds;
+  }
+  return __r;
+}
+
+template <class _Tp, class _Alloc>
+template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> >
+typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::insert(const_iterator __p, _InpIter __f, _InpIter __l) {
+  return __insert_with_sentinel(__p, __f, __l);
+}
+
+template <class _Tp, class _Alloc>
+template <class _Iterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI typename list<_Tp, _Alloc>::iterator
+list<_Tp, _Alloc>::__insert_with_sentinel(const_iterator __p, _Iterator __f, _Sentinel __l) {
+  iterator __r(__p.__ptr_);
+  if (__f != __l) {
+    size_type __ds        = 0;
+    __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, *__f);
+    ++__ds;
+    __r          = iterator(__node->__as_link());
+    iterator __e = __r;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      for (++__f; __f != __l; ++__f, (void)++__e, ++__ds) {
+        __e.__ptr_->__next_ = this->__create_node(/* prev = */ __e.__ptr_, /* next = */ nullptr, *__f)->__as_link();
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      while (true) {
+        __link_pointer __prev    = __e.__ptr_->__prev_;
+        __node_pointer __current = __e.__ptr_->__as_node();
+        this->__delete_node(__current);
+        if (__prev == 0)
+          break;
+        __e = iterator(__prev);
+      }
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __link_nodes(__p.__ptr_, __r.__ptr_, __e.__ptr_);
+    base::__sz() += __ds;
+  }
+  return __r;
+}
+
+template <class _Tp, class _Alloc>
+void list<_Tp, _Alloc>::push_front(const value_type& __x) {
+  __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, __x);
+  __link_pointer __nl   = __node->__as_link();
+  __link_nodes_at_front(__nl, __nl);
+  ++base::__sz();
+}
+
+template <class _Tp, class _Alloc>
+void list<_Tp, _Alloc>::push_back(const value_type& __x) {
+  __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, __x);
+  __link_pointer __nl   = __node->__as_link();
+  __link_nodes_at_back(__nl, __nl);
+  ++base::__sz();
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Alloc>
+void list<_Tp, _Alloc>::push_front(value_type&& __x) {
+  __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, std::move(__x));
+  __link_pointer __nl   = __node->__as_link();
+  __link_nodes_at_front(__nl, __nl);
+  ++base::__sz();
+}
+
+template <class _Tp, class _Alloc>
+void list<_Tp, _Alloc>::push_back(value_type&& __x) {
+  __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, std::move(__x));
+  __link_pointer __nl   = __node->__as_link();
+  __link_nodes_at_back(__nl, __nl);
+  ++base::__sz();
+}
+
+template <class _Tp, class _Alloc>
+template <class... _Args>
+#  if _LIBCPP_STD_VER >= 17
+typename list<_Tp, _Alloc>::reference
+#  else
+void
+#  endif
+list<_Tp, _Alloc>::emplace_front(_Args&&... __args) {
+  __node_pointer __node =
+      this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, std::forward<_Args>(__args)...);
+  __link_pointer __nl = __node->__as_link();
+  __link_nodes_at_front(__nl, __nl);
+  ++base::__sz();
+#  if _LIBCPP_STD_VER >= 17
+  return __node->__get_value();
+#  endif
+}
+
+template <class _Tp, class _Alloc>
+template <class... _Args>
+#  if _LIBCPP_STD_VER >= 17
+typename list<_Tp, _Alloc>::reference
+#  else
+void
+#  endif
+list<_Tp, _Alloc>::emplace_back(_Args&&... __args) {
+  __node_pointer __node =
+      this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, std::forward<_Args>(__args)...);
+  __link_pointer __nl = __node->__as_link();
+  __link_nodes_at_back(__nl, __nl);
+  ++base::__sz();
+#  if _LIBCPP_STD_VER >= 17
+  return __node->__get_value();
+#  endif
+}
+
+template <class _Tp, class _Alloc>
+template <class... _Args>
+typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::emplace(const_iterator __p, _Args&&... __args) {
+  __node_pointer __node =
+      this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, std::forward<_Args>(__args)...);
+  __link_pointer __nl = __node->__as_link();
+  __link_nodes(__p.__ptr_, __nl, __nl);
+  ++base::__sz();
+  return iterator(__nl);
+}
+
+template <class _Tp, class _Alloc>
+typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::insert(const_iterator __p, value_type&& __x) {
+  __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, std::move(__x));
+  __link_pointer __nl   = __node->__as_link();
+  __link_nodes(__p.__ptr_, __nl, __nl);
+  ++base::__sz();
+  return iterator(__nl);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Alloc>
+void list<_Tp, _Alloc>::pop_front() {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::pop_front() called with empty list");
+  __link_pointer __n = base::__end_.__next_;
+  base::__unlink_nodes(__n, __n);
+  --base::__sz();
+  this->__delete_node(__n->__as_node());
+}
+
+template <class _Tp, class _Alloc>
+void list<_Tp, _Alloc>::pop_back() {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::pop_back() called on an empty list");
+  __link_pointer __n = base::__end_.__prev_;
+  base::__unlink_nodes(__n, __n);
+  --base::__sz();
+  this->__delete_node(__n->__as_node());
+}
+
+template <class _Tp, class _Alloc>
+typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::erase(const_iterator __p) {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__p != end(), "list::erase(iterator) called with a non-dereferenceable iterator");
+  __link_pointer __n = __p.__ptr_;
+  __link_pointer __r = __n->__next_;
+  base::__unlink_nodes(__n, __n);
+  --base::__sz();
+  this->__delete_node(__n->__as_node());
+  return iterator(__r);
+}
+
+template <class _Tp, class _Alloc>
+typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::erase(const_iterator __f, const_iterator __l) {
+  if (__f != __l) {
+    base::__unlink_nodes(__f.__ptr_, __l.__ptr_->__prev_);
+    while (__f != __l) {
+      __link_pointer __n = __f.__ptr_;
+      ++__f;
+      --base::__sz();
+      this->__delete_node(__n->__as_node());
+    }
+  }
+  return iterator(__l.__ptr_);
+}
+
+template <class _Tp, class _Alloc>
+void list<_Tp, _Alloc>::resize(size_type __n) {
+  if (__n < base::__sz())
+    erase(__iterator(__n), end());
+  else if (__n > base::__sz()) {
+    __n -= base::__sz();
+    size_type __ds        = 0;
+    __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr);
+    ++__ds;
+    iterator __r = iterator(__node->__as_link());
+    iterator __e = __r;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      for (--__n; __n != 0; --__n, (void)++__e, ++__ds) {
+        __e.__ptr_->__next_ = this->__create_node(/* prev = */ __e.__ptr_, /* next = */ nullptr)->__as_link();
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      while (true) {
+        __link_pointer __prev    = __e.__ptr_->__prev_;
+        __node_pointer __current = __e.__ptr_->__as_node();
+        this->__delete_node(__current);
+        if (__prev == 0)
+          break;
+        __e = iterator(__prev);
+      }
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __link_nodes_at_back(__r.__ptr_, __e.__ptr_);
+    base::__sz() += __ds;
+  }
+}
+
+template <class _Tp, class _Alloc>
+void list<_Tp, _Alloc>::resize(size_type __n, const value_type& __x) {
+  if (__n < base::__sz())
+    erase(__iterator(__n), end());
+  else if (__n > base::__sz()) {
+    __n -= base::__sz();
+    size_type __ds        = 0;
+    __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, __x);
+    ++__ds;
+    __link_pointer __nl = __node->__as_link();
+    iterator __r        = iterator(__nl);
+    iterator __e        = __r;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      for (--__n; __n != 0; --__n, (void)++__e, ++__ds) {
+        __e.__ptr_->__next_ = this->__create_node(/* prev = */ __e.__ptr_, /* next = */ nullptr, __x)->__as_link();
+      }
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      while (true) {
+        __link_pointer __prev    = __e.__ptr_->__prev_;
+        __node_pointer __current = __e.__ptr_->__as_node();
+        this->__delete_node(__current);
+        if (__prev == 0)
+          break;
+        __e = iterator(__prev);
+      }
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __link_nodes(base::__end_as_link(), __r.__ptr_, __e.__ptr_);
+    base::__sz() += __ds;
+  }
+}
+
+template <class _Tp, class _Alloc>
+void list<_Tp, _Alloc>::splice(const_iterator __p, list& __c) {
+  _LIBCPP_ASSERT_VALID_INPUT_RANGE(
+      this != std::addressof(__c), "list::splice(iterator, list) called with this == &list");
+  if (!__c.empty()) {
+    __link_pointer __f = __c.__end_.__next_;
+    __link_pointer __l = __c.__end_.__prev_;
+    base::__unlink_nodes(__f, __l);
+    __link_nodes(__p.__ptr_, __f, __l);
+    base::__sz() += __c.__sz();
+    __c.__sz() = 0;
+  }
+}
+
+template <class _Tp, class _Alloc>
+void list<_Tp, _Alloc>::splice(const_iterator __p, list& __c, const_iterator __i) {
+  if (__p.__ptr_ != __i.__ptr_ && __p.__ptr_ != __i.__ptr_->__next_) {
+    __link_pointer __f = __i.__ptr_;
+    base::__unlink_nodes(__f, __f);
+    __link_nodes(__p.__ptr_, __f, __f);
+    --__c.__sz();
+    ++base::__sz();
+  }
+}
+
+template <class _Tp, class _Alloc>
+void list<_Tp, _Alloc>::splice(const_iterator __p, list& __c, const_iterator __f, const_iterator __l) {
+  if (__f != __l) {
+    __link_pointer __first = __f.__ptr_;
+    --__l;
+    __link_pointer __last = __l.__ptr_;
+    if (this != std::addressof(__c)) {
+      size_type __s = std::distance(__f, __l) + 1;
+      __c.__sz() -= __s;
+      base::__sz() += __s;
+    }
+    base::__unlink_nodes(__first, __last);
+    __link_nodes(__p.__ptr_, __first, __last);
+  }
+}
+
+template <class _Tp, class _Alloc>
+typename list<_Tp, _Alloc>::__remove_return_type list<_Tp, _Alloc>::remove(const value_type& __x) {
+  list<_Tp, _Alloc> __deleted_nodes(get_allocator()); // collect the nodes we're removing
+  for (const_iterator __i = begin(), __e = end(); __i != __e;) {
+    if (*__i == __x) {
+      const_iterator __j = std::next(__i);
+      for (; __j != __e && *__j == __x; ++__j)
+        ;
+      __deleted_nodes.splice(__deleted_nodes.end(), *this, __i, __j);
+      __i = __j;
+      if (__i != __e)
+        ++__i;
+    } else
+      ++__i;
+  }
+
+  return (__remove_return_type)__deleted_nodes.size();
+}
+
+template <class _Tp, class _Alloc>
+template <class _Pred>
+typename list<_Tp, _Alloc>::__remove_return_type list<_Tp, _Alloc>::remove_if(_Pred __pred) {
+  list<_Tp, _Alloc> __deleted_nodes(get_allocator()); // collect the nodes we're removing
+  for (iterator __i = begin(), __e = end(); __i != __e;) {
+    if (__pred(*__i)) {
+      iterator __j = std::next(__i);
+      for (; __j != __e && __pred(*__j); ++__j)
+        ;
+      __deleted_nodes.splice(__deleted_nodes.end(), *this, __i, __j);
+      __i = __j;
+      if (__i != __e)
+        ++__i;
+    } else
+      ++__i;
+  }
+
+  return (__remove_return_type)__deleted_nodes.size();
+}
+
+template <class _Tp, class _Alloc>
+template <class _BinaryPred>
+typename list<_Tp, _Alloc>::__remove_return_type list<_Tp, _Alloc>::unique(_BinaryPred __binary_pred) {
+  list<_Tp, _Alloc> __deleted_nodes(get_allocator()); // collect the nodes we're removing
+  for (iterator __i = begin(), __e = end(); __i != __e;) {
+    iterator __j = std::next(__i);
+    for (; __j != __e && __binary_pred(*__i, *__j); ++__j)
+      ;
+    if (++__i != __j) {
+      __deleted_nodes.splice(__deleted_nodes.end(), *this, __i, __j);
+      __i = __j;
+    }
+  }
+
+  return (__remove_return_type)__deleted_nodes.size();
+}
+
+template <class _Tp, class _Alloc>
+inline void list<_Tp, _Alloc>::merge(list& __c) {
+  merge(__c, __less<>());
+}
+
+template <class _Tp, class _Alloc>
+template <class _Comp>
+void list<_Tp, _Alloc>::merge(list& __c, _Comp __comp) {
+  if (this != std::addressof(__c)) {
+    iterator __f1 = begin();
+    iterator __e1 = end();
+    iterator __f2 = __c.begin();
+    iterator __e2 = __c.end();
+    while (__f1 != __e1 && __f2 != __e2) {
+      if (__comp(*__f2, *__f1)) {
+        size_type __ds = 1;
+        iterator __m2  = std::next(__f2);
+        for (; __m2 != __e2 && __comp(*__m2, *__f1); ++__m2, (void)++__ds)
+          ;
+        base::__sz() += __ds;
+        __c.__sz() -= __ds;
+        __link_pointer __f = __f2.__ptr_;
+        __link_pointer __l = __m2.__ptr_->__prev_;
+        __f2               = __m2;
+        base::__unlink_nodes(__f, __l);
+        __m2 = std::next(__f1);
+        __link_nodes(__f1.__ptr_, __f, __l);
+        __f1 = __m2;
+      } else
+        ++__f1;
+    }
+    splice(__e1, __c);
+  }
+}
+
+template <class _Tp, class _Alloc>
+inline void list<_Tp, _Alloc>::sort() {
+  sort(__less<>());
+}
+
+template <class _Tp, class _Alloc>
+template <class _Comp>
+inline void list<_Tp, _Alloc>::sort(_Comp __comp) {
+  __sort(begin(), end(), base::__sz(), __comp);
+}
+
+template <class _Tp, class _Alloc>
+template <class _Comp>
+typename list<_Tp, _Alloc>::iterator
+list<_Tp, _Alloc>::__sort(iterator __f1, iterator __e2, size_type __n, _Comp& __comp) {
+  switch (__n) {
+  case 0:
+  case 1:
+    return __f1;
+  case 2:
+    if (__comp(*--__e2, *__f1)) {
+      __link_pointer __f = __e2.__ptr_;
+      base::__unlink_nodes(__f, __f);
+      __link_nodes(__f1.__ptr_, __f, __f);
+      return __e2;
+    }
+    return __f1;
+  }
+  size_type __n2 = __n / 2;
+  iterator __e1  = std::next(__f1, __n2);
+  iterator __r = __f1 = __sort(__f1, __e1, __n2, __comp);
+  iterator __f2 = __e1 = __sort(__e1, __e2, __n - __n2, __comp);
+  if (__comp(*__f2, *__f1)) {
+    iterator __m2 = std::next(__f2);
+    for (; __m2 != __e2 && __comp(*__m2, *__f1); ++__m2)
+      ;
+    __link_pointer __f = __f2.__ptr_;
+    __link_pointer __l = __m2.__ptr_->__prev_;
+    __r                = __f2;
+    __e1 = __f2 = __m2;
+    base::__unlink_nodes(__f, __l);
+    __m2 = std::next(__f1);
+    __link_nodes(__f1.__ptr_, __f, __l);
+    __f1 = __m2;
+  } else
+    ++__f1;
+  while (__f1 != __e1 && __f2 != __e2) {
+    if (__comp(*__f2, *__f1)) {
+      iterator __m2 = std::next(__f2);
+      for (; __m2 != __e2 && __comp(*__m2, *__f1); ++__m2)
+        ;
+      __link_pointer __f = __f2.__ptr_;
+      __link_pointer __l = __m2.__ptr_->__prev_;
+      if (__e1 == __f2)
+        __e1 = __m2;
+      __f2 = __m2;
+      base::__unlink_nodes(__f, __l);
+      __m2 = std::next(__f1);
+      __link_nodes(__f1.__ptr_, __f, __l);
+      __f1 = __m2;
+    } else
+      ++__f1;
+  }
+  return __r;
+}
+
+template <class _Tp, class _Alloc>
+void list<_Tp, _Alloc>::reverse() _NOEXCEPT {
+  if (base::__sz() > 1) {
+    iterator __e = end();
+    for (iterator __i = begin(); __i.__ptr_ != __e.__ptr_;) {
+      std::swap(__i.__ptr_->__prev_, __i.__ptr_->__next_);
+      __i.__ptr_ = __i.__ptr_->__prev_;
+    }
+    std::swap(__e.__ptr_->__prev_, __e.__ptr_->__next_);
+  }
+}
+
+template <class _Tp, class _Alloc>
+bool list<_Tp, _Alloc>::__invariants() const {
+  return size() == std::distance(begin(), end());
+}
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
+  return __x.size() == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
+  return std::lexicographical_compare(__x.begin(), __x.end(), __y.begin(), __y.end());
+}
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
+  return !(__x == __y);
+}
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
+  return __y < __x;
+}
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>=(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
+  return !(__x < __y);
+}
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
+  return !(__y < __x);
+}
+
+#else // _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI __synth_three_way_result<_Tp>
+operator<=>(const list<_Tp, _Allocator>& __x, const list<_Tp, _Allocator>& __y) {
+  return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
+}
+
+#endif // _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI void swap(list<_Tp, _Alloc>& __x, list<_Tp, _Alloc>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp, class _Allocator, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI typename list<_Tp, _Allocator>::size_type
+erase_if(list<_Tp, _Allocator>& __c, _Predicate __pred) {
+  return __c.remove_if(__pred);
+}
+
+template <class _Tp, class _Allocator, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI typename list<_Tp, _Allocator>::size_type
+erase(list<_Tp, _Allocator>& __c, const _Up& __v) {
+  return std::erase_if(__c, [&](auto& __elem) { return __elem == __v; });
+}
+
+template <>
+inline constexpr bool __format::__enable_insertable<std::list<char>> = true;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+inline constexpr bool __format::__enable_insertable<std::list<wchar_t>> = true;
+#  endif
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace pmr {
+template <class _ValueT>
+using list _LIBCPP_AVAILABILITY_PMR = std::list<_ValueT, polymorphic_allocator<_ValueT>>;
+} // namespace pmr
+_LIBCPP_END_NAMESPACE_STD
+#endif
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <algorithm>
+#  include <atomic>
+#  include <concepts>
+#  include <cstdint>
+#  include <cstdlib>
+#  include <functional>
+#  include <iosfwd>
+#  include <iterator>
+#  include <stdexcept>
+#  include <type_traits>
+#  include <typeinfo>
+#endif
+
+#endif // _LIBCPP_LIST
diff --git a/libcxx/include/__cxx03/locale b/libcxx/include/__cxx03/locale
new file mode 100644
index 000000000000..573910a85bef
--- /dev/null
+++ b/libcxx/include/__cxx03/locale
@@ -0,0 +1,3767 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_LOCALE
+#define _LIBCPP_LOCALE
+
+/*
+    locale synopsis
+
+namespace std
+{
+
+class locale
+{
+public:
+    // types:
+    class facet;
+    class id;
+
+    typedef int category;
+    static const category // values assigned here are for exposition only
+        none     = 0x000,
+        collate  = 0x010,
+        ctype    = 0x020,
+        monetary = 0x040,
+        numeric  = 0x080,
+        time     = 0x100,
+        messages = 0x200,
+        all = collate | ctype | monetary | numeric | time | messages;
+
+    // construct/copy/destroy:
+    locale() noexcept;
+    locale(const locale& other) noexcept;
+    explicit locale(const char* std_name);
+    explicit locale(const string& std_name);
+    locale(const locale& other, const char* std_name, category);
+    locale(const locale& other, const string& std_name, category);
+    template <class Facet> locale(const locale& other, Facet* f);
+    locale(const locale& other, const locale& one, category);
+
+    ~locale(); // not virtual
+
+    const locale& operator=(const locale& other) noexcept;
+
+    template <class Facet> locale combine(const locale& other) const;
+
+    // locale operations:
+    basic_string<char> name() const;
+    bool operator==(const locale& other) const;
+    bool operator!=(const locale& other) const;                              // removed C++20
+    template <class charT, class Traits, class Allocator>
+      bool operator()(const basic_string<charT,Traits,Allocator>& s1,
+                      const basic_string<charT,Traits,Allocator>& s2) const;
+
+    // global locale objects:
+    static locale global(const locale&);
+    static const locale& classic();
+};
+
+template <class Facet> const Facet& use_facet(const locale&);
+template <class Facet> bool has_facet(const locale&) noexcept;
+
+// 22.3.3, convenience interfaces:
+template <class charT> bool isspace (charT c, const locale& loc);
+template <class charT> bool isprint (charT c, const locale& loc);
+template <class charT> bool iscntrl (charT c, const locale& loc);
+template <class charT> bool isupper (charT c, const locale& loc);
+template <class charT> bool islower (charT c, const locale& loc);
+template <class charT> bool isalpha (charT c, const locale& loc);
+template <class charT> bool isdigit (charT c, const locale& loc);
+template <class charT> bool ispunct (charT c, const locale& loc);
+template <class charT> bool isxdigit(charT c, const locale& loc);
+template <class charT> bool isalnum (charT c, const locale& loc);
+template <class charT> bool isgraph (charT c, const locale& loc);
+template <class charT> charT toupper(charT c, const locale& loc);
+template <class charT> charT tolower(charT c, const locale& loc);
+
+template<class Codecvt, class Elem = wchar_t,
+         class Wide_alloc = allocator<Elem>,
+         class Byte_alloc = allocator<char>>
+class wstring_convert                                      // Removed in C++26
+{
+public:
+    typedef basic_string<char, char_traits<char>, Byte_alloc> byte_string;
+    typedef basic_string<Elem, char_traits<Elem>, Wide_alloc> wide_string;
+    typedef typename Codecvt::state_type                      state_type;
+    typedef typename wide_string::traits_type::int_type       int_type;
+
+    wstring_convert(Codecvt* pcvt = new Codecvt);          // before C++14
+    explicit wstring_convert(Codecvt* pcvt = new Codecvt); // before C++20
+    wstring_convert() : wstring_convert(new Codecvt) {}    // C++20
+    explicit wstring_convert(Codecvt* pcvt);               // C++20
+
+    wstring_convert(Codecvt* pcvt, state_type state);
+    explicit wstring_convert(const byte_string& byte_err,           // explicit in C++14
+                    const wide_string& wide_err = wide_string());
+    wstring_convert(const wstring_convert&) = delete;               // C++14
+    wstring_convert & operator=(const wstring_convert &) = delete;  // C++14
+    ~wstring_convert();
+
+    wide_string from_bytes(char byte);
+    wide_string from_bytes(const char* ptr);
+    wide_string from_bytes(const byte_string& str);
+    wide_string from_bytes(const char* first, const char* last);
+
+    byte_string to_bytes(Elem wchar);
+    byte_string to_bytes(const Elem* wptr);
+    byte_string to_bytes(const wide_string& wstr);
+    byte_string to_bytes(const Elem* first, const Elem* last);
+
+    size_t converted() const; // noexcept in C++14
+    state_type state() const;
+};
+
+template <class Codecvt, class Elem = wchar_t, class Tr = char_traits<Elem>>
+class wbuffer_convert                                               // Removed in C++26
+    : public basic_streambuf<Elem, Tr>
+{
+public:
+    typedef typename Tr::state_type state_type;
+
+    wbuffer_convert(streambuf* bytebuf = 0, Codecvt* pcvt = new Codecvt,
+                    state_type state = state_type());          // before C++14
+    explicit wbuffer_convert(streambuf* bytebuf = nullptr, Codecvt* pcvt = new Codecvt,
+                            state_type state = state_type()); // before C++20
+    wbuffer_convert() : wbuffer_convert(nullptr) {} // C++20
+    explicit wbuffer_convert(streambuf* bytebuf, Codecvt* pcvt = new Codecvt,
+                            state_type state = state_type()); // C++20
+
+    wbuffer_convert(const wbuffer_convert&) = delete;               // C++14
+    wbuffer_convert & operator=(const wbuffer_convert &) = delete;  // C++14
+    ~wbuffer_convert();                                             // C++14
+
+    streambuf* rdbuf() const;
+    streambuf* rdbuf(streambuf* bytebuf);
+
+    state_type state() const;
+};
+
+// 22.4.1 and 22.4.1.3, ctype:
+class ctype_base;
+template <class charT> class ctype;
+template <> class ctype<char>; // specialization
+template <class charT> class ctype_byname;
+template <> class ctype_byname<char>; // specialization
+
+class codecvt_base;
+template <class internT, class externT, class stateT> class codecvt;
+template <class internT, class externT, class stateT> class codecvt_byname;
+
+// 22.4.2 and 22.4.3, numeric:
+template <class charT, class InputIterator> class num_get;
+template <class charT, class OutputIterator> class num_put;
+template <class charT> class numpunct;
+template <class charT> class numpunct_byname;
+
+// 22.4.4, col lation:
+template <class charT> class collate;
+template <class charT> class collate_byname;
+
+// 22.4.5, date and time:
+class time_base;
+template <class charT, class InputIterator> class time_get;
+template <class charT, class InputIterator> class time_get_byname;
+template <class charT, class OutputIterator> class time_put;
+template <class charT, class OutputIterator> class time_put_byname;
+
+// 22.4.6, money:
+class money_base;
+template <class charT, class InputIterator> class money_get;
+template <class charT, class OutputIterator> class money_put;
+template <class charT, bool Intl> class moneypunct;
+template <class charT, bool Intl> class moneypunct_byname;
+
+// 22.4.7, message retrieval:
+class messages_base;
+template <class charT> class messages;
+template <class charT> class messages_byname;
+
+}  // std
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+
+#  include <__algorithm/copy.h>
+#  include <__algorithm/equal.h>
+#  include <__algorithm/find.h>
+#  include <__algorithm/max.h>
+#  include <__algorithm/reverse.h>
+#  include <__algorithm/unwrap_iter.h>
+#  include <__assert>
+#  include <__iterator/access.h>
+#  include <__iterator/back_insert_iterator.h>
+#  include <__iterator/istreambuf_iterator.h>
+#  include <__iterator/ostreambuf_iterator.h>
+#  include <__locale>
+#  include <__memory/unique_ptr.h>
+#  include <__type_traits/make_unsigned.h>
+#  include <cerrno>
+#  include <cstdio>
+#  include <cstdlib>
+#  include <ctime>
+#  include <ios>
+#  include <limits>
+#  include <new>
+#  include <streambuf>
+#  include <version>
+
+// TODO: Fix __bsd_locale_defaults.h
+// NOLINTBEGIN(libcpp-robust-against-adl)
+
+#  if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))
+// Most unix variants have catopen.  These are the specific ones that don't.
+#    if !defined(__BIONIC__) && !defined(_NEWLIB_VERSION) && !defined(__EMSCRIPTEN__)
+#      define _LIBCPP_HAS_CATOPEN 1
+#      include <nl_types.h>
+#    endif
+#  endif
+
+#  ifdef _LIBCPP_LOCALE__L_EXTENSIONS
+#    include <__locale_dir/locale_base_api/bsd_locale_defaults.h>
+#  else
+#    include <__locale_dir/locale_base_api/bsd_locale_fallbacks.h>
+#  endif
+
+#  if defined(__APPLE__) || defined(__FreeBSD__)
+#    include <xlocale.h>
+#  endif
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  if defined(__APPLE__) || defined(__FreeBSD__)
+#    define _LIBCPP_GET_C_LOCALE 0
+#  elif defined(__NetBSD__)
+#    define _LIBCPP_GET_C_LOCALE LC_C_LOCALE
+#  else
+#    define _LIBCPP_GET_C_LOCALE __cloc()
+// Get the C locale object
+_LIBCPP_EXPORTED_FROM_ABI locale_t __cloc();
+#    define __cloc_defined
+#  endif
+
+// __scan_keyword
+// Scans [__b, __e) until a match is found in the basic_strings range
+//  [__kb, __ke) or until it can be shown that there is no match in [__kb, __ke).
+//  __b will be incremented (visibly), consuming CharT until a match is found
+//  or proved to not exist.  A keyword may be "", in which will match anything.
+//  If one keyword is a prefix of another, and the next CharT in the input
+//  might match another keyword, the algorithm will attempt to find the longest
+//  matching keyword.  If the longer matching keyword ends up not matching, then
+//  no keyword match is found.  If no keyword match is found, __ke is returned
+//  and failbit is set in __err.
+//  Else an iterator pointing to the matching keyword is found.  If more than
+//  one keyword matches, an iterator to the first matching keyword is returned.
+//  If on exit __b == __e, eofbit is set in __err.  If __case_sensitive is false,
+//  __ct is used to force to lower case before comparing characters.
+//  Examples:
+//  Keywords:  "a", "abb"
+//  If the input is "a", the first keyword matches and eofbit is set.
+//  If the input is "abc", no match is found and "ab" are consumed.
+template <class _InputIterator, class _ForwardIterator, class _Ctype>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator __scan_keyword(
+    _InputIterator& __b,
+    _InputIterator __e,
+    _ForwardIterator __kb,
+    _ForwardIterator __ke,
+    const _Ctype& __ct,
+    ios_base::iostate& __err,
+    bool __case_sensitive = true) {
+  typedef typename iterator_traits<_InputIterator>::value_type _CharT;
+  size_t __nkw                       = static_cast<size_t>(std::distance(__kb, __ke));
+  const unsigned char __doesnt_match = '\0';
+  const unsigned char __might_match  = '\1';
+  const unsigned char __does_match   = '\2';
+  unsigned char __statbuf[100];
+  unsigned char* __status = __statbuf;
+  unique_ptr<unsigned char, void (*)(void*)> __stat_hold(nullptr, free);
+  if (__nkw > sizeof(__statbuf)) {
+    __status = (unsigned char*)malloc(__nkw);
+    if (__status == nullptr)
+      __throw_bad_alloc();
+    __stat_hold.reset(__status);
+  }
+  size_t __n_might_match = __nkw; // At this point, any keyword might match
+  size_t __n_does_match  = 0;     // but none of them definitely do
+  // Initialize all statuses to __might_match, except for "" keywords are __does_match
+  unsigned char* __st = __status;
+  for (_ForwardIterator __ky = __kb; __ky != __ke; ++__ky, (void)++__st) {
+    if (!__ky->empty())
+      *__st = __might_match;
+    else {
+      *__st = __does_match;
+      --__n_might_match;
+      ++__n_does_match;
+    }
+  }
+  // While there might be a match, test keywords against the next CharT
+  for (size_t __indx = 0; __b != __e && __n_might_match > 0; ++__indx) {
+    // Peek at the next CharT but don't consume it
+    _CharT __c = *__b;
+    if (!__case_sensitive)
+      __c = __ct.toupper(__c);
+    bool __consume = false;
+    // For each keyword which might match, see if the __indx character is __c
+    // If a match if found, consume __c
+    // If a match is found, and that is the last character in the keyword,
+    //    then that keyword matches.
+    // If the keyword doesn't match this character, then change the keyword
+    //    to doesn't match
+    __st = __status;
+    for (_ForwardIterator __ky = __kb; __ky != __ke; ++__ky, (void)++__st) {
+      if (*__st == __might_match) {
+        _CharT __kc = (*__ky)[__indx];
+        if (!__case_sensitive)
+          __kc = __ct.toupper(__kc);
+        if (__c == __kc) {
+          __consume = true;
+          if (__ky->size() == __indx + 1) {
+            *__st = __does_match;
+            --__n_might_match;
+            ++__n_does_match;
+          }
+        } else {
+          *__st = __doesnt_match;
+          --__n_might_match;
+        }
+      }
+    }
+    // consume if we matched a character
+    if (__consume) {
+      ++__b;
+      // If we consumed a character and there might be a matched keyword that
+      //   was marked matched on a previous iteration, then such keywords
+      //   which are now marked as not matching.
+      if (__n_might_match + __n_does_match > 1) {
+        __st = __status;
+        for (_ForwardIterator __ky = __kb; __ky != __ke; ++__ky, (void)++__st) {
+          if (*__st == __does_match && __ky->size() != __indx + 1) {
+            *__st = __doesnt_match;
+            --__n_does_match;
+          }
+        }
+      }
+    }
+  }
+  // We've exited the loop because we hit eof and/or we have no more "might matches".
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  // Return the first matching result
+  for (__st = __status; __kb != __ke; ++__kb, (void)++__st)
+    if (*__st == __does_match)
+      break;
+  if (__kb == __ke)
+    __err |= ios_base::failbit;
+  return __kb;
+}
+
+struct _LIBCPP_EXPORTED_FROM_ABI __num_get_base {
+  static const int __num_get_buf_sz = 40;
+
+  static int __get_base(ios_base&);
+  static const char __src[33]; // "0123456789abcdefABCDEFxX+-pPiInN"
+  // count of leading characters in __src used for parsing integers ("012..X+-")
+  static const size_t __int_chr_cnt = 26;
+  // count of leading characters in __src used for parsing floating-point values ("012..-pP")
+  static const size_t __fp_chr_cnt = 28;
+};
+
+_LIBCPP_EXPORTED_FROM_ABI void
+__check_grouping(const string& __grouping, unsigned* __g, unsigned* __g_end, ios_base::iostate& __err);
+
+template <class _CharT>
+struct __num_get : protected __num_get_base {
+  static string __stage2_float_prep(ios_base& __iob, _CharT* __atoms, _CharT& __decimal_point, _CharT& __thousands_sep);
+
+  static int __stage2_float_loop(
+      _CharT __ct,
+      bool& __in_units,
+      char& __exp,
+      char* __a,
+      char*& __a_end,
+      _CharT __decimal_point,
+      _CharT __thousands_sep,
+      const string& __grouping,
+      unsigned* __g,
+      unsigned*& __g_end,
+      unsigned& __dc,
+      _CharT* __atoms);
+#  ifndef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
+  static string __stage2_int_prep(ios_base& __iob, _CharT* __atoms, _CharT& __thousands_sep);
+  static int __stage2_int_loop(
+      _CharT __ct,
+      int __base,
+      char* __a,
+      char*& __a_end,
+      unsigned& __dc,
+      _CharT __thousands_sep,
+      const string& __grouping,
+      unsigned* __g,
+      unsigned*& __g_end,
+      _CharT* __atoms);
+
+#  else
+  static string __stage2_int_prep(ios_base& __iob, _CharT& __thousands_sep) {
+    locale __loc                 = __iob.getloc();
+    const numpunct<_CharT>& __np = use_facet<numpunct<_CharT> >(__loc);
+    __thousands_sep              = __np.thousands_sep();
+    return __np.grouping();
+  }
+
+  const _CharT* __do_widen(ios_base& __iob, _CharT* __atoms) const { return __do_widen_p(__iob, __atoms); }
+
+  static int __stage2_int_loop(
+      _CharT __ct,
+      int __base,
+      char* __a,
+      char*& __a_end,
+      unsigned& __dc,
+      _CharT __thousands_sep,
+      const string& __grouping,
+      unsigned* __g,
+      unsigned*& __g_end,
+      const _CharT* __atoms);
+
+private:
+  template <typename _Tp>
+  const _Tp* __do_widen_p(ios_base& __iob, _Tp* __atoms) const {
+    locale __loc = __iob.getloc();
+    use_facet<ctype<_Tp> >(__loc).widen(__src, __src + __int_chr_cnt, __atoms);
+    return __atoms;
+  }
+
+  const char* __do_widen_p(ios_base& __iob, char* __atoms) const {
+    (void)__iob;
+    (void)__atoms;
+    return __src;
+  }
+#  endif
+};
+
+#  ifndef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
+template <class _CharT>
+string __num_get<_CharT>::__stage2_int_prep(ios_base& __iob, _CharT* __atoms, _CharT& __thousands_sep) {
+  locale __loc = __iob.getloc();
+  std::use_facet<ctype<_CharT> >(__loc).widen(__src, __src + __int_chr_cnt, __atoms);
+  const numpunct<_CharT>& __np = std::use_facet<numpunct<_CharT> >(__loc);
+  __thousands_sep              = __np.thousands_sep();
+  return __np.grouping();
+}
+#  endif
+
+template <class _CharT>
+string __num_get<_CharT>::__stage2_float_prep(
+    ios_base& __iob, _CharT* __atoms, _CharT& __decimal_point, _CharT& __thousands_sep) {
+  locale __loc = __iob.getloc();
+  std::use_facet<ctype<_CharT> >(__loc).widen(__src, __src + __fp_chr_cnt, __atoms);
+  const numpunct<_CharT>& __np = std::use_facet<numpunct<_CharT> >(__loc);
+  __decimal_point              = __np.decimal_point();
+  __thousands_sep              = __np.thousands_sep();
+  return __np.grouping();
+}
+
+template <class _CharT>
+int
+#  ifndef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
+__num_get<_CharT>::__stage2_int_loop(_CharT __ct, int __base, char* __a, char*& __a_end,
+                  unsigned& __dc, _CharT __thousands_sep, const string& __grouping,
+                  unsigned* __g, unsigned*& __g_end, _CharT* __atoms)
+#  else
+__num_get<_CharT>::__stage2_int_loop(_CharT __ct, int __base, char* __a, char*& __a_end,
+                  unsigned& __dc, _CharT __thousands_sep, const string& __grouping,
+                  unsigned* __g, unsigned*& __g_end, const _CharT* __atoms)
+
+#  endif
+{
+  if (__a_end == __a && (__ct == __atoms[24] || __ct == __atoms[25])) {
+    *__a_end++ = __ct == __atoms[24] ? '+' : '-';
+    __dc       = 0;
+    return 0;
+  }
+  if (__grouping.size() != 0 && __ct == __thousands_sep) {
+    if (__g_end - __g < __num_get_buf_sz) {
+      *__g_end++ = __dc;
+      __dc       = 0;
+    }
+    return 0;
+  }
+  ptrdiff_t __f = std::find(__atoms, __atoms + __int_chr_cnt, __ct) - __atoms;
+  if (__f >= 24)
+    return -1;
+  switch (__base) {
+  case 8:
+  case 10:
+    if (__f >= __base)
+      return -1;
+    break;
+  case 16:
+    if (__f < 22)
+      break;
+    if (__a_end != __a && __a_end - __a <= 2 && __a_end[-1] == '0') {
+      __dc       = 0;
+      *__a_end++ = __src[__f];
+      return 0;
+    }
+    return -1;
+  }
+  *__a_end++ = __src[__f];
+  ++__dc;
+  return 0;
+}
+
+template <class _CharT>
+int __num_get<_CharT>::__stage2_float_loop(
+    _CharT __ct,
+    bool& __in_units,
+    char& __exp,
+    char* __a,
+    char*& __a_end,
+    _CharT __decimal_point,
+    _CharT __thousands_sep,
+    const string& __grouping,
+    unsigned* __g,
+    unsigned*& __g_end,
+    unsigned& __dc,
+    _CharT* __atoms) {
+  if (__ct == __decimal_point) {
+    if (!__in_units)
+      return -1;
+    __in_units = false;
+    *__a_end++ = '.';
+    if (__grouping.size() != 0 && __g_end - __g < __num_get_buf_sz)
+      *__g_end++ = __dc;
+    return 0;
+  }
+  if (__ct == __thousands_sep && __grouping.size() != 0) {
+    if (!__in_units)
+      return -1;
+    if (__g_end - __g < __num_get_buf_sz) {
+      *__g_end++ = __dc;
+      __dc       = 0;
+    }
+    return 0;
+  }
+  ptrdiff_t __f = std::find(__atoms, __atoms + __num_get_base::__fp_chr_cnt, __ct) - __atoms;
+  if (__f >= static_cast<ptrdiff_t>(__num_get_base::__fp_chr_cnt))
+    return -1;
+  char __x = __src[__f];
+  if (__x == '-' || __x == '+') {
+    if (__a_end == __a || (std::toupper(__a_end[-1]) == std::toupper(__exp))) {
+      *__a_end++ = __x;
+      return 0;
+    }
+    return -1;
+  }
+  if (__x == 'x' || __x == 'X')
+    __exp = 'P';
+  else if (std::toupper(__x) == __exp) {
+    __exp = std::tolower(__exp);
+    if (__in_units) {
+      __in_units = false;
+      if (__grouping.size() != 0 && __g_end - __g < __num_get_buf_sz)
+        *__g_end++ = __dc;
+    }
+  }
+  *__a_end++ = __x;
+  if (__f >= 22)
+    return 0;
+  ++__dc;
+  return 0;
+}
+
+extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_get<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_get<wchar_t>;
+#  endif
+
+template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS num_get : public locale::facet, private __num_get<_CharT> {
+public:
+  typedef _CharT char_type;
+  typedef _InputIterator iter_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit num_get(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long long& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned short& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned int& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long long& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, float& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, double& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~num_get() override {}
+
+  template <class _Fp>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS iter_type
+  __do_get_floating_point(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Fp& __v) const;
+
+  template <class _Signed>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS iter_type
+  __do_get_signed(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Signed& __v) const;
+
+  template <class _Unsigned>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS iter_type
+  __do_get_unsigned(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Unsigned& __v) const;
+
+  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const;
+
+  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long& __v) const {
+    return this->__do_get_signed(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long long& __v) const {
+    return this->__do_get_signed(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned short& __v) const {
+    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned int& __v) const {
+    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long& __v) const {
+    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long long& __v) const {
+    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, float& __v) const {
+    return this->__do_get_floating_point(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, double& __v) const {
+    return this->__do_get_floating_point(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
+    return this->__do_get_floating_point(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const;
+};
+
+template <class _CharT, class _InputIterator>
+locale::id num_get<_CharT, _InputIterator>::id;
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__num_get_signed_integral(const char* __a, const char* __a_end, ios_base::iostate& __err, int __base) {
+  if (__a != __a_end) {
+    __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
+    errno                                                     = 0;
+    char* __p2;
+    long long __ll                                               = strtoll_l(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
+    __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
+    if (__current_errno == 0)
+      errno = __save_errno;
+    if (__p2 != __a_end) {
+      __err = ios_base::failbit;
+      return 0;
+    } else if (__current_errno == ERANGE || __ll < numeric_limits<_Tp>::min() || numeric_limits<_Tp>::max() < __ll) {
+      __err = ios_base::failbit;
+      if (__ll > 0)
+        return numeric_limits<_Tp>::max();
+      else
+        return numeric_limits<_Tp>::min();
+    }
+    return static_cast<_Tp>(__ll);
+  }
+  __err = ios_base::failbit;
+  return 0;
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__num_get_unsigned_integral(const char* __a, const char* __a_end, ios_base::iostate& __err, int __base) {
+  if (__a != __a_end) {
+    const bool __negate = *__a == '-';
+    if (__negate && ++__a == __a_end) {
+      __err = ios_base::failbit;
+      return 0;
+    }
+    __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
+    errno                                                     = 0;
+    char* __p2;
+    unsigned long long __ll                                      = strtoull_l(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
+    __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
+    if (__current_errno == 0)
+      errno = __save_errno;
+    if (__p2 != __a_end) {
+      __err = ios_base::failbit;
+      return 0;
+    } else if (__current_errno == ERANGE || numeric_limits<_Tp>::max() < __ll) {
+      __err = ios_base::failbit;
+      return numeric_limits<_Tp>::max();
+    }
+    _Tp __res = static_cast<_Tp>(__ll);
+    if (__negate)
+      __res = -__res;
+    return __res;
+  }
+  __err = ios_base::failbit;
+  return 0;
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __do_strtod(const char* __a, char** __p2);
+
+template <>
+inline _LIBCPP_HIDE_FROM_ABI float __do_strtod<float>(const char* __a, char** __p2) {
+  return strtof_l(__a, __p2, _LIBCPP_GET_C_LOCALE);
+}
+
+template <>
+inline _LIBCPP_HIDE_FROM_ABI double __do_strtod<double>(const char* __a, char** __p2) {
+  return strtod_l(__a, __p2, _LIBCPP_GET_C_LOCALE);
+}
+
+template <>
+inline _LIBCPP_HIDE_FROM_ABI long double __do_strtod<long double>(const char* __a, char** __p2) {
+  return strtold_l(__a, __p2, _LIBCPP_GET_C_LOCALE);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __num_get_float(const char* __a, const char* __a_end, ios_base::iostate& __err) {
+  if (__a != __a_end) {
+    __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
+    errno                                                     = 0;
+    char* __p2;
+    _Tp __ld                                                     = std::__do_strtod<_Tp>(__a, &__p2);
+    __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
+    if (__current_errno == 0)
+      errno = __save_errno;
+    if (__p2 != __a_end) {
+      __err = ios_base::failbit;
+      return 0;
+    } else if (__current_errno == ERANGE)
+      __err = ios_base::failbit;
+    return __ld;
+  }
+  __err = ios_base::failbit;
+  return 0;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator num_get<_CharT, _InputIterator>::do_get(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const {
+  if ((__iob.flags() & ios_base::boolalpha) == 0) {
+    long __lv = -1;
+    __b       = do_get(__b, __e, __iob, __err, __lv);
+    switch (__lv) {
+    case 0:
+      __v = false;
+      break;
+    case 1:
+      __v = true;
+      break;
+    default:
+      __v   = true;
+      __err = ios_base::failbit;
+      break;
+    }
+    return __b;
+  }
+  const ctype<_CharT>& __ct    = std::use_facet<ctype<_CharT> >(__iob.getloc());
+  const numpunct<_CharT>& __np = std::use_facet<numpunct<_CharT> >(__iob.getloc());
+  typedef typename numpunct<_CharT>::string_type string_type;
+  const string_type __names[2] = {__np.truename(), __np.falsename()};
+  const string_type* __i       = std::__scan_keyword(__b, __e, __names, __names + 2, __ct, __err);
+  __v                          = __i == __names;
+  return __b;
+}
+
+// signed
+
+template <class _CharT, class _InputIterator>
+template <class _Signed>
+_InputIterator num_get<_CharT, _InputIterator>::__do_get_signed(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Signed& __v) const {
+  // Stage 1
+  int __base = this->__get_base(__iob);
+  // Stage 2
+  char_type __thousands_sep;
+  const int __atoms_size = __num_get_base::__int_chr_cnt;
+#  ifdef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
+  char_type __atoms1[__atoms_size];
+  const char_type* __atoms = this->__do_widen(__iob, __atoms1);
+  string __grouping        = this->__stage2_int_prep(__iob, __thousands_sep);
+#  else
+  char_type __atoms[__atoms_size];
+  string __grouping = this->__stage2_int_prep(__iob, __atoms, __thousands_sep);
+#  endif
+  string __buf;
+  __buf.resize(__buf.capacity());
+  char* __a     = &__buf[0];
+  char* __a_end = __a;
+  unsigned __g[__num_get_base::__num_get_buf_sz];
+  unsigned* __g_end = __g;
+  unsigned __dc     = 0;
+  for (; __b != __e; ++__b) {
+    if (__a_end == __a + __buf.size()) {
+      size_t __tmp = __buf.size();
+      __buf.resize(2 * __buf.size());
+      __buf.resize(__buf.capacity());
+      __a     = &__buf[0];
+      __a_end = __a + __tmp;
+    }
+    if (this->__stage2_int_loop(*__b, __base, __a, __a_end, __dc, __thousands_sep, __grouping, __g, __g_end, __atoms))
+      break;
+  }
+  if (__grouping.size() != 0 && __g_end - __g < __num_get_base::__num_get_buf_sz)
+    *__g_end++ = __dc;
+  // Stage 3
+  __v = std::__num_get_signed_integral<_Signed>(__a, __a_end, __err, __base);
+  // Digit grouping checked
+  __check_grouping(__grouping, __g, __g_end, __err);
+  // EOF checked
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __b;
+}
+
+// unsigned
+
+template <class _CharT, class _InputIterator>
+template <class _Unsigned>
+_InputIterator num_get<_CharT, _InputIterator>::__do_get_unsigned(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Unsigned& __v) const {
+  // Stage 1
+  int __base = this->__get_base(__iob);
+  // Stage 2
+  char_type __thousands_sep;
+  const int __atoms_size = __num_get_base::__int_chr_cnt;
+#  ifdef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
+  char_type __atoms1[__atoms_size];
+  const char_type* __atoms = this->__do_widen(__iob, __atoms1);
+  string __grouping        = this->__stage2_int_prep(__iob, __thousands_sep);
+#  else
+  char_type __atoms[__atoms_size];
+  string __grouping = this->__stage2_int_prep(__iob, __atoms, __thousands_sep);
+#  endif
+  string __buf;
+  __buf.resize(__buf.capacity());
+  char* __a     = &__buf[0];
+  char* __a_end = __a;
+  unsigned __g[__num_get_base::__num_get_buf_sz];
+  unsigned* __g_end = __g;
+  unsigned __dc     = 0;
+  for (; __b != __e; ++__b) {
+    if (__a_end == __a + __buf.size()) {
+      size_t __tmp = __buf.size();
+      __buf.resize(2 * __buf.size());
+      __buf.resize(__buf.capacity());
+      __a     = &__buf[0];
+      __a_end = __a + __tmp;
+    }
+    if (this->__stage2_int_loop(*__b, __base, __a, __a_end, __dc, __thousands_sep, __grouping, __g, __g_end, __atoms))
+      break;
+  }
+  if (__grouping.size() != 0 && __g_end - __g < __num_get_base::__num_get_buf_sz)
+    *__g_end++ = __dc;
+  // Stage 3
+  __v = std::__num_get_unsigned_integral<_Unsigned>(__a, __a_end, __err, __base);
+  // Digit grouping checked
+  __check_grouping(__grouping, __g, __g_end, __err);
+  // EOF checked
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __b;
+}
+
+// floating point
+
+template <class _CharT, class _InputIterator>
+template <class _Fp>
+_InputIterator num_get<_CharT, _InputIterator>::__do_get_floating_point(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Fp& __v) const {
+  // Stage 1, nothing to do
+  // Stage 2
+  char_type __atoms[__num_get_base::__fp_chr_cnt];
+  char_type __decimal_point;
+  char_type __thousands_sep;
+  string __grouping = this->__stage2_float_prep(__iob, __atoms, __decimal_point, __thousands_sep);
+  string __buf;
+  __buf.resize(__buf.capacity());
+  char* __a     = &__buf[0];
+  char* __a_end = __a;
+  unsigned __g[__num_get_base::__num_get_buf_sz];
+  unsigned* __g_end        = __g;
+  unsigned __dc            = 0;
+  bool __in_units          = true;
+  char __exp               = 'E';
+  bool __is_leading_parsed = false;
+  for (; __b != __e; ++__b) {
+    if (__a_end == __a + __buf.size()) {
+      size_t __tmp = __buf.size();
+      __buf.resize(2 * __buf.size());
+      __buf.resize(__buf.capacity());
+      __a     = &__buf[0];
+      __a_end = __a + __tmp;
+    }
+    if (this->__stage2_float_loop(
+            *__b,
+            __in_units,
+            __exp,
+            __a,
+            __a_end,
+            __decimal_point,
+            __thousands_sep,
+            __grouping,
+            __g,
+            __g_end,
+            __dc,
+            __atoms))
+      break;
+
+    // the leading character excluding the sign must be a decimal digit
+    if (!__is_leading_parsed) {
+      if (__a_end - __a >= 1 && __a[0] != '-' && __a[0] != '+') {
+        if (('0' <= __a[0] && __a[0] <= '9') || __a[0] == '.')
+          __is_leading_parsed = true;
+        else
+          break;
+      } else if (__a_end - __a >= 2 && (__a[0] == '-' || __a[0] == '+')) {
+        if (('0' <= __a[1] && __a[1] <= '9') || __a[1] == '.')
+          __is_leading_parsed = true;
+        else
+          break;
+      }
+    }
+  }
+  if (__grouping.size() != 0 && __in_units && __g_end - __g < __num_get_base::__num_get_buf_sz)
+    *__g_end++ = __dc;
+  // Stage 3
+  __v = std::__num_get_float<_Fp>(__a, __a_end, __err);
+  // Digit grouping checked
+  __check_grouping(__grouping, __g, __g_end, __err);
+  // EOF checked
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator num_get<_CharT, _InputIterator>::do_get(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const {
+  // Stage 1
+  int __base = 16;
+  // Stage 2
+  char_type __atoms[__num_get_base::__int_chr_cnt];
+  char_type __thousands_sep = char_type();
+  string __grouping;
+  std::use_facet<ctype<_CharT> >(__iob.getloc())
+      .widen(__num_get_base::__src, __num_get_base::__src + __num_get_base::__int_chr_cnt, __atoms);
+  string __buf;
+  __buf.resize(__buf.capacity());
+  char* __a     = &__buf[0];
+  char* __a_end = __a;
+  unsigned __g[__num_get_base::__num_get_buf_sz];
+  unsigned* __g_end = __g;
+  unsigned __dc     = 0;
+  for (; __b != __e; ++__b) {
+    if (__a_end == __a + __buf.size()) {
+      size_t __tmp = __buf.size();
+      __buf.resize(2 * __buf.size());
+      __buf.resize(__buf.capacity());
+      __a     = &__buf[0];
+      __a_end = __a + __tmp;
+    }
+    if (this->__stage2_int_loop(*__b, __base, __a, __a_end, __dc, __thousands_sep, __grouping, __g, __g_end, __atoms))
+      break;
+  }
+  // Stage 3
+  __buf.resize(__a_end - __a);
+  if (__libcpp_sscanf_l(__buf.c_str(), _LIBCPP_GET_C_LOCALE, "%p", &__v) != 1)
+    __err = ios_base::failbit;
+  // EOF checked
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __b;
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_get<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_get<wchar_t>;
+#  endif
+
+struct _LIBCPP_EXPORTED_FROM_ABI __num_put_base {
+protected:
+  static void __format_int(char* __fmt, const char* __len, bool __signd, ios_base::fmtflags __flags);
+  static bool __format_float(char* __fmt, const char* __len, ios_base::fmtflags __flags);
+  static char* __identify_padding(char* __nb, char* __ne, const ios_base& __iob);
+};
+
+template <class _CharT>
+struct __num_put : protected __num_put_base {
+  static void __widen_and_group_int(
+      char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc);
+  static void __widen_and_group_float(
+      char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc);
+};
+
+template <class _CharT>
+void __num_put<_CharT>::__widen_and_group_int(
+    char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc) {
+  const ctype<_CharT>& __ct     = std::use_facet<ctype<_CharT> >(__loc);
+  const numpunct<_CharT>& __npt = std::use_facet<numpunct<_CharT> >(__loc);
+  string __grouping             = __npt.grouping();
+  if (__grouping.empty()) {
+    __ct.widen(__nb, __ne, __ob);
+    __oe = __ob + (__ne - __nb);
+  } else {
+    __oe       = __ob;
+    char* __nf = __nb;
+    if (*__nf == '-' || *__nf == '+')
+      *__oe++ = __ct.widen(*__nf++);
+    if (__ne - __nf >= 2 && __nf[0] == '0' && (__nf[1] == 'x' || __nf[1] == 'X')) {
+      *__oe++ = __ct.widen(*__nf++);
+      *__oe++ = __ct.widen(*__nf++);
+    }
+    std::reverse(__nf, __ne);
+    _CharT __thousands_sep = __npt.thousands_sep();
+    unsigned __dc          = 0;
+    unsigned __dg          = 0;
+    for (char* __p = __nf; __p < __ne; ++__p) {
+      if (static_cast<unsigned>(__grouping[__dg]) > 0 && __dc == static_cast<unsigned>(__grouping[__dg])) {
+        *__oe++ = __thousands_sep;
+        __dc    = 0;
+        if (__dg < __grouping.size() - 1)
+          ++__dg;
+      }
+      *__oe++ = __ct.widen(*__p);
+      ++__dc;
+    }
+    std::reverse(__ob + (__nf - __nb), __oe);
+  }
+  if (__np == __ne)
+    __op = __oe;
+  else
+    __op = __ob + (__np - __nb);
+}
+
+template <class _CharT>
+void __num_put<_CharT>::__widen_and_group_float(
+    char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc) {
+  const ctype<_CharT>& __ct     = std::use_facet<ctype<_CharT> >(__loc);
+  const numpunct<_CharT>& __npt = std::use_facet<numpunct<_CharT> >(__loc);
+  string __grouping             = __npt.grouping();
+  __oe                          = __ob;
+  char* __nf                    = __nb;
+  if (*__nf == '-' || *__nf == '+')
+    *__oe++ = __ct.widen(*__nf++);
+  char* __ns;
+  if (__ne - __nf >= 2 && __nf[0] == '0' && (__nf[1] == 'x' || __nf[1] == 'X')) {
+    *__oe++ = __ct.widen(*__nf++);
+    *__oe++ = __ct.widen(*__nf++);
+    for (__ns = __nf; __ns < __ne; ++__ns)
+      if (!isxdigit_l(*__ns, _LIBCPP_GET_C_LOCALE))
+        break;
+  } else {
+    for (__ns = __nf; __ns < __ne; ++__ns)
+      if (!isdigit_l(*__ns, _LIBCPP_GET_C_LOCALE))
+        break;
+  }
+  if (__grouping.empty()) {
+    __ct.widen(__nf, __ns, __oe);
+    __oe += __ns - __nf;
+  } else {
+    std::reverse(__nf, __ns);
+    _CharT __thousands_sep = __npt.thousands_sep();
+    unsigned __dc          = 0;
+    unsigned __dg          = 0;
+    for (char* __p = __nf; __p < __ns; ++__p) {
+      if (__grouping[__dg] > 0 && __dc == static_cast<unsigned>(__grouping[__dg])) {
+        *__oe++ = __thousands_sep;
+        __dc    = 0;
+        if (__dg < __grouping.size() - 1)
+          ++__dg;
+      }
+      *__oe++ = __ct.widen(*__p);
+      ++__dc;
+    }
+    std::reverse(__ob + (__nf - __nb), __oe);
+  }
+  for (__nf = __ns; __nf < __ne; ++__nf) {
+    if (*__nf == '.') {
+      *__oe++ = __npt.decimal_point();
+      ++__nf;
+      break;
+    } else
+      *__oe++ = __ct.widen(*__nf);
+  }
+  __ct.widen(__nf, __ne, __oe);
+  __oe += __ne - __nf;
+  if (__np == __ne)
+    __op = __oe;
+  else
+    __op = __ob + (__np - __nb);
+}
+
+extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_put<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_put<wchar_t>;
+#  endif
+
+template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS num_put : public locale::facet, private __num_put<_CharT> {
+public:
+  typedef _CharT char_type;
+  typedef _OutputIterator iter_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit num_put(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, bool __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, long __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, long long __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long long __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, double __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, long double __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, const void* __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~num_put() override {}
+
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, bool __v) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, long __v) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, long long __v) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long long) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, double __v) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, long double __v) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, const void* __v) const;
+
+  template <class _Integral>
+  _LIBCPP_HIDE_FROM_ABI inline _OutputIterator
+  __do_put_integral(iter_type __s, ios_base& __iob, char_type __fl, _Integral __v, char const* __len) const;
+
+  template <class _Float>
+  _LIBCPP_HIDE_FROM_ABI inline _OutputIterator
+  __do_put_floating_point(iter_type __s, ios_base& __iob, char_type __fl, _Float __v, char const* __len) const;
+};
+
+template <class _CharT, class _OutputIterator>
+locale::id num_put<_CharT, _OutputIterator>::id;
+
+template <class _CharT, class _OutputIterator>
+_LIBCPP_HIDE_FROM_ABI _OutputIterator __pad_and_output(
+    _OutputIterator __s, const _CharT* __ob, const _CharT* __op, const _CharT* __oe, ios_base& __iob, _CharT __fl) {
+  streamsize __sz = __oe - __ob;
+  streamsize __ns = __iob.width();
+  if (__ns > __sz)
+    __ns -= __sz;
+  else
+    __ns = 0;
+  for (; __ob < __op; ++__ob, ++__s)
+    *__s = *__ob;
+  for (; __ns; --__ns, ++__s)
+    *__s = __fl;
+  for (; __ob < __oe; ++__ob, ++__s)
+    *__s = *__ob;
+  __iob.width(0);
+  return __s;
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI ostreambuf_iterator<_CharT, _Traits> __pad_and_output(
+    ostreambuf_iterator<_CharT, _Traits> __s,
+    const _CharT* __ob,
+    const _CharT* __op,
+    const _CharT* __oe,
+    ios_base& __iob,
+    _CharT __fl) {
+  if (__s.__sbuf_ == nullptr)
+    return __s;
+  streamsize __sz = __oe - __ob;
+  streamsize __ns = __iob.width();
+  if (__ns > __sz)
+    __ns -= __sz;
+  else
+    __ns = 0;
+  streamsize __np = __op - __ob;
+  if (__np > 0) {
+    if (__s.__sbuf_->sputn(__ob, __np) != __np) {
+      __s.__sbuf_ = nullptr;
+      return __s;
+    }
+  }
+  if (__ns > 0) {
+    basic_string<_CharT, _Traits> __sp(__ns, __fl);
+    if (__s.__sbuf_->sputn(__sp.data(), __ns) != __ns) {
+      __s.__sbuf_ = nullptr;
+      return __s;
+    }
+  }
+  __np = __oe - __op;
+  if (__np > 0) {
+    if (__s.__sbuf_->sputn(__op, __np) != __np) {
+      __s.__sbuf_ = nullptr;
+      return __s;
+    }
+  }
+  __iob.width(0);
+  return __s;
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, bool __v) const {
+  if ((__iob.flags() & ios_base::boolalpha) == 0)
+    return do_put(__s, __iob, __fl, (unsigned long)__v);
+  const numpunct<char_type>& __np = std::use_facet<numpunct<char_type> >(__iob.getloc());
+  typedef typename numpunct<char_type>::string_type string_type;
+  string_type __nm = __v ? __np.truename() : __np.falsename();
+  for (typename string_type::iterator __i = __nm.begin(); __i != __nm.end(); ++__i, ++__s)
+    *__s = *__i;
+  return __s;
+}
+
+template <class _CharT, class _OutputIterator>
+template <class _Integral>
+_LIBCPP_HIDE_FROM_ABI inline _OutputIterator num_put<_CharT, _OutputIterator>::__do_put_integral(
+    iter_type __s, ios_base& __iob, char_type __fl, _Integral __v, char const* __len) const {
+  // Stage 1 - Get number in narrow char
+  char __fmt[8] = {'%', 0};
+  this->__format_int(__fmt + 1, __len, is_signed<_Integral>::value, __iob.flags());
+  // Worst case is octal, with showbase enabled. Note that octal is always
+  // printed as an unsigned value.
+  using _Unsigned = typename make_unsigned<_Integral>::type;
+  _LIBCPP_CONSTEXPR const unsigned __nbuf =
+      (numeric_limits<_Unsigned>::digits / 3)          // 1 char per 3 bits
+      + ((numeric_limits<_Unsigned>::digits % 3) != 0) // round up
+      + 2;                                             // base prefix + terminating null character
+  char __nar[__nbuf];
+  _LIBCPP_DIAGNOSTIC_PUSH
+  _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wformat-nonliteral")
+  _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wformat-nonliteral")
+  int __nc = __libcpp_snprintf_l(__nar, sizeof(__nar), _LIBCPP_GET_C_LOCALE, __fmt, __v);
+  _LIBCPP_DIAGNOSTIC_POP
+  char* __ne = __nar + __nc;
+  char* __np = this->__identify_padding(__nar, __ne, __iob);
+  // Stage 2 - Widen __nar while adding thousands separators
+  char_type __o[2 * (__nbuf - 1) - 1];
+  char_type* __op; // pad here
+  char_type* __oe; // end of output
+  this->__widen_and_group_int(__nar, __np, __ne, __o, __op, __oe, __iob.getloc());
+  // [__o, __oe) contains thousands_sep'd wide number
+  // Stage 3 & 4
+  return std::__pad_and_output(__s, __o, __op, __oe, __iob, __fl);
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, long __v) const {
+  return this->__do_put_integral(__s, __iob, __fl, __v, "l");
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, long long __v) const {
+  return this->__do_put_integral(__s, __iob, __fl, __v, "ll");
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long __v) const {
+  return this->__do_put_integral(__s, __iob, __fl, __v, "l");
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long long __v) const {
+  return this->__do_put_integral(__s, __iob, __fl, __v, "ll");
+}
+
+template <class _CharT, class _OutputIterator>
+template <class _Float>
+_LIBCPP_HIDE_FROM_ABI inline _OutputIterator num_put<_CharT, _OutputIterator>::__do_put_floating_point(
+    iter_type __s, ios_base& __iob, char_type __fl, _Float __v, char const* __len) const {
+  // Stage 1 - Get number in narrow char
+  char __fmt[8]            = {'%', 0};
+  bool __specify_precision = this->__format_float(__fmt + 1, __len, __iob.flags());
+  const unsigned __nbuf    = 30;
+  char __nar[__nbuf];
+  char* __nb = __nar;
+  int __nc;
+  _LIBCPP_DIAGNOSTIC_PUSH
+  _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wformat-nonliteral")
+  _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wformat-nonliteral")
+  if (__specify_precision)
+    __nc = __libcpp_snprintf_l(__nb, __nbuf, _LIBCPP_GET_C_LOCALE, __fmt, (int)__iob.precision(), __v);
+  else
+    __nc = __libcpp_snprintf_l(__nb, __nbuf, _LIBCPP_GET_C_LOCALE, __fmt, __v);
+  unique_ptr<char, void (*)(void*)> __nbh(nullptr, free);
+  if (__nc > static_cast<int>(__nbuf - 1)) {
+    if (__specify_precision)
+      __nc = __libcpp_asprintf_l(&__nb, _LIBCPP_GET_C_LOCALE, __fmt, (int)__iob.precision(), __v);
+    else
+      __nc = __libcpp_asprintf_l(&__nb, _LIBCPP_GET_C_LOCALE, __fmt, __v);
+    if (__nc == -1)
+      __throw_bad_alloc();
+    __nbh.reset(__nb);
+  }
+  _LIBCPP_DIAGNOSTIC_POP
+  char* __ne = __nb + __nc;
+  char* __np = this->__identify_padding(__nb, __ne, __iob);
+  // Stage 2 - Widen __nar while adding thousands separators
+  char_type __o[2 * (__nbuf - 1) - 1];
+  char_type* __ob = __o;
+  unique_ptr<char_type, void (*)(void*)> __obh(0, free);
+  if (__nb != __nar) {
+    __ob = (char_type*)malloc(2 * static_cast<size_t>(__nc) * sizeof(char_type));
+    if (__ob == 0)
+      __throw_bad_alloc();
+    __obh.reset(__ob);
+  }
+  char_type* __op; // pad here
+  char_type* __oe; // end of output
+  this->__widen_and_group_float(__nb, __np, __ne, __ob, __op, __oe, __iob.getloc());
+  // [__o, __oe) contains thousands_sep'd wide number
+  // Stage 3 & 4
+  __s = std::__pad_and_output(__s, __ob, __op, __oe, __iob, __fl);
+  return __s;
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, double __v) const {
+  return this->__do_put_floating_point(__s, __iob, __fl, __v, "");
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, long double __v) const {
+  return this->__do_put_floating_point(__s, __iob, __fl, __v, "L");
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, const void* __v) const {
+  // Stage 1 - Get pointer in narrow char
+  const unsigned __nbuf = 20;
+  char __nar[__nbuf];
+  int __nc   = __libcpp_snprintf_l(__nar, sizeof(__nar), _LIBCPP_GET_C_LOCALE, "%p", __v);
+  char* __ne = __nar + __nc;
+  char* __np = this->__identify_padding(__nar, __ne, __iob);
+  // Stage 2 - Widen __nar
+  char_type __o[2 * (__nbuf - 1) - 1];
+  char_type* __op; // pad here
+  char_type* __oe; // end of output
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  __ct.widen(__nar, __ne, __o);
+  __oe = __o + (__ne - __nar);
+  if (__np == __ne)
+    __op = __oe;
+  else
+    __op = __o + (__np - __nar);
+  // [__o, __oe) contains wide number
+  // Stage 3 & 4
+  return std::__pad_and_output(__s, __o, __op, __oe, __iob, __fl);
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_put<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_put<wchar_t>;
+#  endif
+
+template <class _CharT, class _InputIterator>
+_LIBCPP_HIDE_FROM_ABI int __get_up_to_n_digits(
+    _InputIterator& __b, _InputIterator __e, ios_base::iostate& __err, const ctype<_CharT>& __ct, int __n) {
+  // Precondition:  __n >= 1
+  if (__b == __e) {
+    __err |= ios_base::eofbit | ios_base::failbit;
+    return 0;
+  }
+  // get first digit
+  _CharT __c = *__b;
+  if (!__ct.is(ctype_base::digit, __c)) {
+    __err |= ios_base::failbit;
+    return 0;
+  }
+  int __r = __ct.narrow(__c, 0) - '0';
+  for (++__b, (void)--__n; __b != __e && __n > 0; ++__b, (void)--__n) {
+    // get next digit
+    __c = *__b;
+    if (!__ct.is(ctype_base::digit, __c))
+      return __r;
+    __r = __r * 10 + __ct.narrow(__c, 0) - '0';
+  }
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __r;
+}
+
+class _LIBCPP_EXPORTED_FROM_ABI time_base {
+public:
+  enum dateorder { no_order, dmy, mdy, ymd, ydm };
+};
+
+template <class _CharT>
+class _LIBCPP_TEMPLATE_VIS __time_get_c_storage {
+protected:
+  typedef basic_string<_CharT> string_type;
+
+  virtual const string_type* __weeks() const;
+  virtual const string_type* __months() const;
+  virtual const string_type* __am_pm() const;
+  virtual const string_type& __c() const;
+  virtual const string_type& __r() const;
+  virtual const string_type& __x() const;
+  virtual const string_type& __X() const;
+
+  _LIBCPP_HIDE_FROM_ABI ~__time_get_c_storage() {}
+};
+
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string* __time_get_c_storage<char>::__weeks() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string* __time_get_c_storage<char>::__months() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string* __time_get_c_storage<char>::__am_pm() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__c() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__r() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__x() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__X() const;
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__weeks() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__months() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__am_pm() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__c() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__r() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__x() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__X() const;
+#  endif
+
+template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS time_get : public locale::facet, public time_base, private __time_get_c_storage<_CharT> {
+public:
+  typedef _CharT char_type;
+  typedef _InputIterator iter_type;
+  typedef time_base::dateorder dateorder;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit time_get(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI dateorder date_order() const { return this->do_date_order(); }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get_time(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+    return do_get_time(__b, __e, __iob, __err, __tm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get_date(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+    return do_get_date(__b, __e, __iob, __err, __tm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get_weekday(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+    return do_get_weekday(__b, __e, __iob, __err, __tm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get_monthname(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+    return do_get_monthname(__b, __e, __iob, __err, __tm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get_year(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+    return do_get_year(__b, __e, __iob, __err, __tm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm, char __fmt, char __mod = 0)
+      const {
+    return do_get(__b, __e, __iob, __err, __tm, __fmt, __mod);
+  }
+
+  iter_type
+  get(iter_type __b,
+      iter_type __e,
+      ios_base& __iob,
+      ios_base::iostate& __err,
+      tm* __tm,
+      const char_type* __fmtb,
+      const char_type* __fmte) const;
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_get() override {}
+
+  virtual dateorder do_date_order() const;
+  virtual iter_type
+  do_get_time(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
+  virtual iter_type
+  do_get_date(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
+  virtual iter_type
+  do_get_weekday(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
+  virtual iter_type
+  do_get_monthname(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
+  virtual iter_type
+  do_get_year(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
+  virtual iter_type do_get(
+      iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm, char __fmt, char __mod) const;
+
+private:
+  void __get_white_space(iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void __get_percent(iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+
+  void __get_weekdayname(
+      int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void __get_monthname(
+      int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void __get_day(int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_month(int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_year(int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_year4(int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_hour(int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_12_hour(int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_am_pm(int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_minute(int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_second(int& __s, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_weekday(int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void __get_day_year_num(
+      int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+};
+
+template <class _CharT, class _InputIterator>
+locale::id time_get<_CharT, _InputIterator>::id;
+
+// time_get primitives
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_weekdayname(
+    int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  // Note:  ignoring case comes from the POSIX strptime spec
+  const string_type* __wk = this->__weeks();
+  ptrdiff_t __i           = std::__scan_keyword(__b, __e, __wk, __wk + 14, __ct, __err, false) - __wk;
+  if (__i < 14)
+    __w = __i % 7;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_monthname(
+    int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  // Note:  ignoring case comes from the POSIX strptime spec
+  const string_type* __month = this->__months();
+  ptrdiff_t __i              = std::__scan_keyword(__b, __e, __month, __month + 24, __ct, __err, false) - __month;
+  if (__i < 24)
+    __m = __i % 12;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_day(
+    int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
+  if (!(__err & ios_base::failbit) && 1 <= __t && __t <= 31)
+    __d = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_month(
+    int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2) - 1;
+  if (!(__err & ios_base::failbit) && 0 <= __t && __t <= 11)
+    __m = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_year(
+    int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 4);
+  if (!(__err & ios_base::failbit)) {
+    if (__t < 69)
+      __t += 2000;
+    else if (69 <= __t && __t <= 99)
+      __t += 1900;
+    __y = __t - 1900;
+  }
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_year4(
+    int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 4);
+  if (!(__err & ios_base::failbit))
+    __y = __t - 1900;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_hour(
+    int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
+  if (!(__err & ios_base::failbit) && __t <= 23)
+    __h = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_12_hour(
+    int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
+  if (!(__err & ios_base::failbit) && 1 <= __t && __t <= 12)
+    __h = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_minute(
+    int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
+  if (!(__err & ios_base::failbit) && __t <= 59)
+    __m = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_second(
+    int& __s, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
+  if (!(__err & ios_base::failbit) && __t <= 60)
+    __s = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_weekday(
+    int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 1);
+  if (!(__err & ios_base::failbit) && __t <= 6)
+    __w = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_day_year_num(
+    int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 3);
+  if (!(__err & ios_base::failbit) && __t <= 365)
+    __d = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_white_space(
+    iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  for (; __b != __e && __ct.is(ctype_base::space, *__b); ++__b)
+    ;
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_am_pm(
+    int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  const string_type* __ap = this->__am_pm();
+  if (__ap[0].size() + __ap[1].size() == 0) {
+    __err |= ios_base::failbit;
+    return;
+  }
+  ptrdiff_t __i = std::__scan_keyword(__b, __e, __ap, __ap + 2, __ct, __err, false) - __ap;
+  if (__i == 0 && __h == 12)
+    __h = 0;
+  else if (__i == 1 && __h < 12)
+    __h += 12;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_percent(
+    iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  if (__b == __e) {
+    __err |= ios_base::eofbit | ios_base::failbit;
+    return;
+  }
+  if (__ct.narrow(*__b, 0) != '%')
+    __err |= ios_base::failbit;
+  else if (++__b == __e)
+    __err |= ios_base::eofbit;
+}
+
+// time_get end primitives
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::get(
+    iter_type __b,
+    iter_type __e,
+    ios_base& __iob,
+    ios_base::iostate& __err,
+    tm* __tm,
+    const char_type* __fmtb,
+    const char_type* __fmte) const {
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  __err                        = ios_base::goodbit;
+  while (__fmtb != __fmte && __err == ios_base::goodbit) {
+    if (__b == __e) {
+      __err = ios_base::failbit;
+      break;
+    }
+    if (__ct.narrow(*__fmtb, 0) == '%') {
+      if (++__fmtb == __fmte) {
+        __err = ios_base::failbit;
+        break;
+      }
+      char __cmd = __ct.narrow(*__fmtb, 0);
+      char __opt = '\0';
+      if (__cmd == 'E' || __cmd == '0') {
+        if (++__fmtb == __fmte) {
+          __err = ios_base::failbit;
+          break;
+        }
+        __opt = __cmd;
+        __cmd = __ct.narrow(*__fmtb, 0);
+      }
+      __b = do_get(__b, __e, __iob, __err, __tm, __cmd, __opt);
+      ++__fmtb;
+    } else if (__ct.is(ctype_base::space, *__fmtb)) {
+      for (++__fmtb; __fmtb != __fmte && __ct.is(ctype_base::space, *__fmtb); ++__fmtb)
+        ;
+      for (; __b != __e && __ct.is(ctype_base::space, *__b); ++__b)
+        ;
+    } else if (__ct.toupper(*__b) == __ct.toupper(*__fmtb)) {
+      ++__b;
+      ++__fmtb;
+    } else
+      __err = ios_base::failbit;
+  }
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+typename time_get<_CharT, _InputIterator>::dateorder time_get<_CharT, _InputIterator>::do_date_order() const {
+  return mdy;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get_time(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+  const char_type __fmt[] = {'%', 'H', ':', '%', 'M', ':', '%', 'S'};
+  return get(__b, __e, __iob, __err, __tm, __fmt, __fmt + sizeof(__fmt) / sizeof(__fmt[0]));
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get_date(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+  const string_type& __fmt = this->__x();
+  return get(__b, __e, __iob, __err, __tm, __fmt.data(), __fmt.data() + __fmt.size());
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get_weekday(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  __get_weekdayname(__tm->tm_wday, __b, __e, __err, __ct);
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get_monthname(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  __get_monthname(__tm->tm_mon, __b, __e, __err, __ct);
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get_year(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  __get_year(__tm->tm_year, __b, __e, __err, __ct);
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm, char __fmt, char) const {
+  __err                        = ios_base::goodbit;
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  switch (__fmt) {
+  case 'a':
+  case 'A':
+    __get_weekdayname(__tm->tm_wday, __b, __e, __err, __ct);
+    break;
+  case 'b':
+  case 'B':
+  case 'h':
+    __get_monthname(__tm->tm_mon, __b, __e, __err, __ct);
+    break;
+  case 'c': {
+    const string_type& __fm = this->__c();
+    __b                     = get(__b, __e, __iob, __err, __tm, __fm.data(), __fm.data() + __fm.size());
+  } break;
+  case 'd':
+  case 'e':
+    __get_day(__tm->tm_mday, __b, __e, __err, __ct);
+    break;
+  case 'D': {
+    const char_type __fm[] = {'%', 'm', '/', '%', 'd', '/', '%', 'y'};
+    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
+  } break;
+  case 'F': {
+    const char_type __fm[] = {'%', 'Y', '-', '%', 'm', '-', '%', 'd'};
+    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
+  } break;
+  case 'H':
+    __get_hour(__tm->tm_hour, __b, __e, __err, __ct);
+    break;
+  case 'I':
+    __get_12_hour(__tm->tm_hour, __b, __e, __err, __ct);
+    break;
+  case 'j':
+    __get_day_year_num(__tm->tm_yday, __b, __e, __err, __ct);
+    break;
+  case 'm':
+    __get_month(__tm->tm_mon, __b, __e, __err, __ct);
+    break;
+  case 'M':
+    __get_minute(__tm->tm_min, __b, __e, __err, __ct);
+    break;
+  case 'n':
+  case 't':
+    __get_white_space(__b, __e, __err, __ct);
+    break;
+  case 'p':
+    __get_am_pm(__tm->tm_hour, __b, __e, __err, __ct);
+    break;
+  case 'r': {
+    const char_type __fm[] = {'%', 'I', ':', '%', 'M', ':', '%', 'S', ' ', '%', 'p'};
+    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
+  } break;
+  case 'R': {
+    const char_type __fm[] = {'%', 'H', ':', '%', 'M'};
+    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
+  } break;
+  case 'S':
+    __get_second(__tm->tm_sec, __b, __e, __err, __ct);
+    break;
+  case 'T': {
+    const char_type __fm[] = {'%', 'H', ':', '%', 'M', ':', '%', 'S'};
+    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
+  } break;
+  case 'w':
+    __get_weekday(__tm->tm_wday, __b, __e, __err, __ct);
+    break;
+  case 'x':
+    return do_get_date(__b, __e, __iob, __err, __tm);
+  case 'X': {
+    const string_type& __fm = this->__X();
+    __b                     = get(__b, __e, __iob, __err, __tm, __fm.data(), __fm.data() + __fm.size());
+  } break;
+  case 'y':
+    __get_year(__tm->tm_year, __b, __e, __err, __ct);
+    break;
+  case 'Y':
+    __get_year4(__tm->tm_year, __b, __e, __err, __ct);
+    break;
+  case '%':
+    __get_percent(__b, __e, __err, __ct);
+    break;
+  default:
+    __err |= ios_base::failbit;
+  }
+  return __b;
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get<wchar_t>;
+#  endif
+
+class _LIBCPP_EXPORTED_FROM_ABI __time_get {
+protected:
+  locale_t __loc_;
+
+  __time_get(const char* __nm);
+  __time_get(const string& __nm);
+  ~__time_get();
+};
+
+template <class _CharT>
+class _LIBCPP_TEMPLATE_VIS __time_get_storage : public __time_get {
+protected:
+  typedef basic_string<_CharT> string_type;
+
+  string_type __weeks_[14];
+  string_type __months_[24];
+  string_type __am_pm_[2];
+  string_type __c_;
+  string_type __r_;
+  string_type __x_;
+  string_type __X_;
+
+  explicit __time_get_storage(const char* __nm);
+  explicit __time_get_storage(const string& __nm);
+
+  _LIBCPP_HIDE_FROM_ABI ~__time_get_storage() {}
+
+  time_base::dateorder __do_date_order() const;
+
+private:
+  void init(const ctype<_CharT>&);
+  string_type __analyze(char __fmt, const ctype<_CharT>&);
+};
+
+#  define _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(_CharT)                                                      \
+    template <>                                                                                                        \
+    _LIBCPP_EXPORTED_FROM_ABI time_base::dateorder __time_get_storage<_CharT>::__do_date_order() const;                \
+    template <>                                                                                                        \
+    _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const char*);                             \
+    template <>                                                                                                        \
+    _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&);                           \
+    template <>                                                                                                        \
+    _LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&);                             \
+    template <>                                                                                                        \
+    _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type __time_get_storage<_CharT>::__analyze(           \
+        char, const ctype<_CharT>&);                                                                                   \
+    extern template _LIBCPP_EXPORTED_FROM_ABI time_base::dateorder __time_get_storage<_CharT>::__do_date_order()       \
+        const;                                                                                                         \
+    extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const char*);             \
+    extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&);           \
+    extern template _LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&);             \
+    extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type                                  \
+    __time_get_storage<_CharT>::__analyze(char, const ctype<_CharT>&);                                                 \
+    /**/
+
+_LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(char)
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+_LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(wchar_t)
+#  endif
+#  undef _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION
+
+template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS time_get_byname
+    : public time_get<_CharT, _InputIterator>,
+      private __time_get_storage<_CharT> {
+public:
+  typedef time_base::dateorder dateorder;
+  typedef _InputIterator iter_type;
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit time_get_byname(const char* __nm, size_t __refs = 0)
+      : time_get<_CharT, _InputIterator>(__refs), __time_get_storage<_CharT>(__nm) {}
+  _LIBCPP_HIDE_FROM_ABI explicit time_get_byname(const string& __nm, size_t __refs = 0)
+      : time_get<_CharT, _InputIterator>(__refs), __time_get_storage<_CharT>(__nm) {}
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_get_byname() override {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL dateorder do_date_order() const override { return this->__do_date_order(); }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type* __weeks() const override { return this->__weeks_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type* __months() const override { return this->__months_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type* __am_pm() const override { return this->__am_pm_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __c() const override { return this->__c_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __r() const override { return this->__r_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __x() const override { return this->__x_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __X() const override { return this->__X_; }
+};
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get_byname<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get_byname<wchar_t>;
+#  endif
+
+class _LIBCPP_EXPORTED_FROM_ABI __time_put {
+  locale_t __loc_;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI __time_put() : __loc_(_LIBCPP_GET_C_LOCALE) {}
+  __time_put(const char* __nm);
+  __time_put(const string& __nm);
+  ~__time_put();
+  void __do_put(char* __nb, char*& __ne, const tm* __tm, char __fmt, char __mod) const;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+  void __do_put(wchar_t* __wb, wchar_t*& __we, const tm* __tm, char __fmt, char __mod) const;
+#  endif
+};
+
+template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS time_put : public locale::facet, private __time_put {
+public:
+  typedef _CharT char_type;
+  typedef _OutputIterator iter_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit time_put(size_t __refs = 0) : locale::facet(__refs) {}
+
+  iter_type
+  put(iter_type __s, ios_base& __iob, char_type __fl, const tm* __tm, const char_type* __pb, const char_type* __pe)
+      const;
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  put(iter_type __s, ios_base& __iob, char_type __fl, const tm* __tm, char __fmt, char __mod = 0) const {
+    return do_put(__s, __iob, __fl, __tm, __fmt, __mod);
+  }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_put() override {}
+  virtual iter_type do_put(iter_type __s, ios_base&, char_type, const tm* __tm, char __fmt, char __mod) const;
+
+  _LIBCPP_HIDE_FROM_ABI explicit time_put(const char* __nm, size_t __refs) : locale::facet(__refs), __time_put(__nm) {}
+  _LIBCPP_HIDE_FROM_ABI explicit time_put(const string& __nm, size_t __refs)
+      : locale::facet(__refs), __time_put(__nm) {}
+};
+
+template <class _CharT, class _OutputIterator>
+locale::id time_put<_CharT, _OutputIterator>::id;
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator time_put<_CharT, _OutputIterator>::put(
+    iter_type __s, ios_base& __iob, char_type __fl, const tm* __tm, const char_type* __pb, const char_type* __pe)
+    const {
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  for (; __pb != __pe; ++__pb) {
+    if (__ct.narrow(*__pb, 0) == '%') {
+      if (++__pb == __pe) {
+        *__s++ = __pb[-1];
+        break;
+      }
+      char __mod = 0;
+      char __fmt = __ct.narrow(*__pb, 0);
+      if (__fmt == 'E' || __fmt == 'O') {
+        if (++__pb == __pe) {
+          *__s++ = __pb[-2];
+          *__s++ = __pb[-1];
+          break;
+        }
+        __mod = __fmt;
+        __fmt = __ct.narrow(*__pb, 0);
+      }
+      __s = do_put(__s, __iob, __fl, __tm, __fmt, __mod);
+    } else
+      *__s++ = *__pb;
+  }
+  return __s;
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator time_put<_CharT, _OutputIterator>::do_put(
+    iter_type __s, ios_base&, char_type, const tm* __tm, char __fmt, char __mod) const {
+  char_type __nar[100];
+  char_type* __nb = __nar;
+  char_type* __ne = __nb + 100;
+  __do_put(__nb, __ne, __tm, __fmt, __mod);
+  return std::copy(__nb, __ne, __s);
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put<wchar_t>;
+#  endif
+
+template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS time_put_byname : public time_put<_CharT, _OutputIterator> {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit time_put_byname(const char* __nm, size_t __refs = 0)
+      : time_put<_CharT, _OutputIterator>(__nm, __refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit time_put_byname(const string& __nm, size_t __refs = 0)
+      : time_put<_CharT, _OutputIterator>(__nm, __refs) {}
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_put_byname() override {}
+};
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put_byname<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put_byname<wchar_t>;
+#  endif
+
+// money_base
+
+class _LIBCPP_EXPORTED_FROM_ABI money_base {
+public:
+  enum part { none, space, symbol, sign, value };
+  struct pattern {
+    char field[4];
+  };
+
+  _LIBCPP_HIDE_FROM_ABI money_base() {}
+};
+
+// moneypunct
+
+template <class _CharT, bool _International = false>
+class _LIBCPP_TEMPLATE_VIS moneypunct : public locale::facet, public money_base {
+public:
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit moneypunct(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI char_type decimal_point() const { return do_decimal_point(); }
+  _LIBCPP_HIDE_FROM_ABI char_type thousands_sep() const { return do_thousands_sep(); }
+  _LIBCPP_HIDE_FROM_ABI string grouping() const { return do_grouping(); }
+  _LIBCPP_HIDE_FROM_ABI string_type curr_symbol() const { return do_curr_symbol(); }
+  _LIBCPP_HIDE_FROM_ABI string_type positive_sign() const { return do_positive_sign(); }
+  _LIBCPP_HIDE_FROM_ABI string_type negative_sign() const { return do_negative_sign(); }
+  _LIBCPP_HIDE_FROM_ABI int frac_digits() const { return do_frac_digits(); }
+  _LIBCPP_HIDE_FROM_ABI pattern pos_format() const { return do_pos_format(); }
+  _LIBCPP_HIDE_FROM_ABI pattern neg_format() const { return do_neg_format(); }
+
+  static locale::id id;
+  static const bool intl = _International;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~moneypunct() override {}
+
+  virtual char_type do_decimal_point() const { return numeric_limits<char_type>::max(); }
+  virtual char_type do_thousands_sep() const { return numeric_limits<char_type>::max(); }
+  virtual string do_grouping() const { return string(); }
+  virtual string_type do_curr_symbol() const { return string_type(); }
+  virtual string_type do_positive_sign() const { return string_type(); }
+  virtual string_type do_negative_sign() const { return string_type(1, '-'); }
+  virtual int do_frac_digits() const { return 0; }
+  virtual pattern do_pos_format() const {
+    pattern __p = {{symbol, sign, none, value}};
+    return __p;
+  }
+  virtual pattern do_neg_format() const {
+    pattern __p = {{symbol, sign, none, value}};
+    return __p;
+  }
+};
+
+template <class _CharT, bool _International>
+locale::id moneypunct<_CharT, _International>::id;
+
+template <class _CharT, bool _International>
+const bool moneypunct<_CharT, _International>::intl;
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<char, false>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<char, true>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<wchar_t, false>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<wchar_t, true>;
+#  endif
+
+// moneypunct_byname
+
+template <class _CharT, bool _International = false>
+class _LIBCPP_TEMPLATE_VIS moneypunct_byname : public moneypunct<_CharT, _International> {
+public:
+  typedef money_base::pattern pattern;
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit moneypunct_byname(const char* __nm, size_t __refs = 0)
+      : moneypunct<_CharT, _International>(__refs) {
+    init(__nm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI explicit moneypunct_byname(const string& __nm, size_t __refs = 0)
+      : moneypunct<_CharT, _International>(__refs) {
+    init(__nm.c_str());
+  }
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~moneypunct_byname() override {}
+
+  char_type do_decimal_point() const override { return __decimal_point_; }
+  char_type do_thousands_sep() const override { return __thousands_sep_; }
+  string do_grouping() const override { return __grouping_; }
+  string_type do_curr_symbol() const override { return __curr_symbol_; }
+  string_type do_positive_sign() const override { return __positive_sign_; }
+  string_type do_negative_sign() const override { return __negative_sign_; }
+  int do_frac_digits() const override { return __frac_digits_; }
+  pattern do_pos_format() const override { return __pos_format_; }
+  pattern do_neg_format() const override { return __neg_format_; }
+
+private:
+  char_type __decimal_point_;
+  char_type __thousands_sep_;
+  string __grouping_;
+  string_type __curr_symbol_;
+  string_type __positive_sign_;
+  string_type __negative_sign_;
+  int __frac_digits_;
+  pattern __pos_format_;
+  pattern __neg_format_;
+
+  void init(const char*);
+};
+
+template <>
+_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<char, false>::init(const char*);
+template <>
+_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<char, true>::init(const char*);
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<char, false>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<char, true>;
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<wchar_t, false>::init(const char*);
+template <>
+_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<wchar_t, true>::init(const char*);
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<wchar_t, false>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<wchar_t, true>;
+#  endif
+
+// money_get
+
+template <class _CharT>
+class __money_get {
+protected:
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI __money_get() {}
+
+  static void __gather_info(
+      bool __intl,
+      const locale& __loc,
+      money_base::pattern& __pat,
+      char_type& __dp,
+      char_type& __ts,
+      string& __grp,
+      string_type& __sym,
+      string_type& __psn,
+      string_type& __nsn,
+      int& __fd);
+};
+
+template <class _CharT>
+void __money_get<_CharT>::__gather_info(
+    bool __intl,
+    const locale& __loc,
+    money_base::pattern& __pat,
+    char_type& __dp,
+    char_type& __ts,
+    string& __grp,
+    string_type& __sym,
+    string_type& __psn,
+    string_type& __nsn,
+    int& __fd) {
+  if (__intl) {
+    const moneypunct<char_type, true>& __mp = std::use_facet<moneypunct<char_type, true> >(__loc);
+    __pat                                   = __mp.neg_format();
+    __nsn                                   = __mp.negative_sign();
+    __psn                                   = __mp.positive_sign();
+    __dp                                    = __mp.decimal_point();
+    __ts                                    = __mp.thousands_sep();
+    __grp                                   = __mp.grouping();
+    __sym                                   = __mp.curr_symbol();
+    __fd                                    = __mp.frac_digits();
+  } else {
+    const moneypunct<char_type, false>& __mp = std::use_facet<moneypunct<char_type, false> >(__loc);
+    __pat                                    = __mp.neg_format();
+    __nsn                                    = __mp.negative_sign();
+    __psn                                    = __mp.positive_sign();
+    __dp                                     = __mp.decimal_point();
+    __ts                                     = __mp.thousands_sep();
+    __grp                                    = __mp.grouping();
+    __sym                                    = __mp.curr_symbol();
+    __fd                                     = __mp.frac_digits();
+  }
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_get<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_get<wchar_t>;
+#  endif
+
+template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS money_get : public locale::facet, private __money_get<_CharT> {
+public:
+  typedef _CharT char_type;
+  typedef _InputIterator iter_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit money_get(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
+    return do_get(__b, __e, __intl, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, string_type& __v) const {
+    return do_get(__b, __e, __intl, __iob, __err, __v);
+  }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~money_get() override {}
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, long double& __v) const;
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, string_type& __v) const;
+
+private:
+  static bool __do_get(
+      iter_type& __b,
+      iter_type __e,
+      bool __intl,
+      const locale& __loc,
+      ios_base::fmtflags __flags,
+      ios_base::iostate& __err,
+      bool& __neg,
+      const ctype<char_type>& __ct,
+      unique_ptr<char_type, void (*)(void*)>& __wb,
+      char_type*& __wn,
+      char_type* __we);
+};
+
+template <class _CharT, class _InputIterator>
+locale::id money_get<_CharT, _InputIterator>::id;
+
+_LIBCPP_EXPORTED_FROM_ABI void __do_nothing(void*);
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void __double_or_nothing(unique_ptr<_Tp, void (*)(void*)>& __b, _Tp*& __n, _Tp*& __e) {
+  bool __owns      = __b.get_deleter() != __do_nothing;
+  size_t __cur_cap = static_cast<size_t>(__e - __b.get()) * sizeof(_Tp);
+  size_t __new_cap = __cur_cap < numeric_limits<size_t>::max() / 2 ? 2 * __cur_cap : numeric_limits<size_t>::max();
+  if (__new_cap == 0)
+    __new_cap = sizeof(_Tp);
+  size_t __n_off = static_cast<size_t>(__n - __b.get());
+  _Tp* __t       = (_Tp*)std::realloc(__owns ? __b.get() : 0, __new_cap);
+  if (__t == 0)
+    __throw_bad_alloc();
+  if (__owns)
+    __b.release();
+  __b = unique_ptr<_Tp, void (*)(void*)>(__t, free);
+  __new_cap /= sizeof(_Tp);
+  __n = __b.get() + __n_off;
+  __e = __b.get() + __new_cap;
+}
+
+// true == success
+template <class _CharT, class _InputIterator>
+bool money_get<_CharT, _InputIterator>::__do_get(
+    iter_type& __b,
+    iter_type __e,
+    bool __intl,
+    const locale& __loc,
+    ios_base::fmtflags __flags,
+    ios_base::iostate& __err,
+    bool& __neg,
+    const ctype<char_type>& __ct,
+    unique_ptr<char_type, void (*)(void*)>& __wb,
+    char_type*& __wn,
+    char_type* __we) {
+  if (__b == __e) {
+    __err |= ios_base::failbit;
+    return false;
+  }
+  const unsigned __bz = 100;
+  unsigned __gbuf[__bz];
+  unique_ptr<unsigned, void (*)(void*)> __gb(__gbuf, __do_nothing);
+  unsigned* __gn = __gb.get();
+  unsigned* __ge = __gn + __bz;
+  money_base::pattern __pat;
+  char_type __dp;
+  char_type __ts;
+  string __grp;
+  string_type __sym;
+  string_type __psn;
+  string_type __nsn;
+  // Capture the spaces read into money_base::{space,none} so they
+  // can be compared to initial spaces in __sym.
+  string_type __spaces;
+  int __fd;
+  __money_get<_CharT>::__gather_info(__intl, __loc, __pat, __dp, __ts, __grp, __sym, __psn, __nsn, __fd);
+  const string_type* __trailing_sign = 0;
+  __wn                               = __wb.get();
+  for (unsigned __p = 0; __p < 4 && __b != __e; ++__p) {
+    switch (__pat.field[__p]) {
+    case money_base::space:
+      if (__p != 3) {
+        if (__ct.is(ctype_base::space, *__b))
+          __spaces.push_back(*__b++);
+        else {
+          __err |= ios_base::failbit;
+          return false;
+        }
+      }
+      _LIBCPP_FALLTHROUGH();
+    case money_base::none:
+      if (__p != 3) {
+        while (__b != __e && __ct.is(ctype_base::space, *__b))
+          __spaces.push_back(*__b++);
+      }
+      break;
+    case money_base::sign:
+      if (__psn.size() > 0 && *__b == __psn[0]) {
+        ++__b;
+        __neg = false;
+        if (__psn.size() > 1)
+          __trailing_sign = &__psn;
+        break;
+      }
+      if (__nsn.size() > 0 && *__b == __nsn[0]) {
+        ++__b;
+        __neg = true;
+        if (__nsn.size() > 1)
+          __trailing_sign = &__nsn;
+        break;
+      }
+      if (__psn.size() > 0 && __nsn.size() > 0) { // sign is required
+        __err |= ios_base::failbit;
+        return false;
+      }
+      if (__psn.size() == 0 && __nsn.size() == 0)
+        // locale has no way of specifying a sign. Use the initial value of __neg as a default
+        break;
+      __neg = (__nsn.size() == 0);
+      break;
+    case money_base::symbol: {
+      bool __more_needed =
+          __trailing_sign || (__p < 2) || (__p == 2 && __pat.field[3] != static_cast<char>(money_base::none));
+      bool __sb = (__flags & ios_base::showbase) != 0;
+      if (__sb || __more_needed) {
+        typename string_type::const_iterator __sym_space_end = __sym.begin();
+        if (__p > 0 && (__pat.field[__p - 1] == money_base::none || __pat.field[__p - 1] == money_base::space)) {
+          // Match spaces we've already read against spaces at
+          // the beginning of __sym.
+          while (__sym_space_end != __sym.end() && __ct.is(ctype_base::space, *__sym_space_end))
+            ++__sym_space_end;
+          const size_t __num_spaces = __sym_space_end - __sym.begin();
+          if (__num_spaces > __spaces.size() ||
+              !std::equal(__spaces.end() - __num_spaces, __spaces.end(), __sym.begin())) {
+            // No match. Put __sym_space_end back at the
+            // beginning of __sym, which will prevent a
+            // match in the next loop.
+            __sym_space_end = __sym.begin();
+          }
+        }
+        typename string_type::const_iterator __sym_curr_char = __sym_space_end;
+        while (__sym_curr_char != __sym.end() && __b != __e && *__b == *__sym_curr_char) {
+          ++__b;
+          ++__sym_curr_char;
+        }
+        if (__sb && __sym_curr_char != __sym.end()) {
+          __err |= ios_base::failbit;
+          return false;
+        }
+      }
+    } break;
+    case money_base::value: {
+      unsigned __ng = 0;
+      for (; __b != __e; ++__b) {
+        char_type __c = *__b;
+        if (__ct.is(ctype_base::digit, __c)) {
+          if (__wn == __we)
+            std::__double_or_nothing(__wb, __wn, __we);
+          *__wn++ = __c;
+          ++__ng;
+        } else if (__grp.size() > 0 && __ng > 0 && __c == __ts) {
+          if (__gn == __ge)
+            std::__double_or_nothing(__gb, __gn, __ge);
+          *__gn++ = __ng;
+          __ng    = 0;
+        } else
+          break;
+      }
+      if (__gb.get() != __gn && __ng > 0) {
+        if (__gn == __ge)
+          std::__double_or_nothing(__gb, __gn, __ge);
+        *__gn++ = __ng;
+      }
+      if (__fd > 0) {
+        if (__b == __e || *__b != __dp) {
+          __err |= ios_base::failbit;
+          return false;
+        }
+        for (++__b; __fd > 0; --__fd, ++__b) {
+          if (__b == __e || !__ct.is(ctype_base::digit, *__b)) {
+            __err |= ios_base::failbit;
+            return false;
+          }
+          if (__wn == __we)
+            std::__double_or_nothing(__wb, __wn, __we);
+          *__wn++ = *__b;
+        }
+      }
+      if (__wn == __wb.get()) {
+        __err |= ios_base::failbit;
+        return false;
+      }
+    } break;
+    }
+  }
+  if (__trailing_sign) {
+    for (unsigned __i = 1; __i < __trailing_sign->size(); ++__i, ++__b) {
+      if (__b == __e || *__b != (*__trailing_sign)[__i]) {
+        __err |= ios_base::failbit;
+        return false;
+      }
+    }
+  }
+  if (__gb.get() != __gn) {
+    ios_base::iostate __et = ios_base::goodbit;
+    __check_grouping(__grp, __gb.get(), __gn, __et);
+    if (__et) {
+      __err |= ios_base::failbit;
+      return false;
+    }
+  }
+  return true;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator money_get<_CharT, _InputIterator>::do_get(
+    iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
+  const int __bz = 100;
+  char_type __wbuf[__bz];
+  unique_ptr<char_type, void (*)(void*)> __wb(__wbuf, __do_nothing);
+  char_type* __wn;
+  char_type* __we              = __wbuf + __bz;
+  locale __loc                 = __iob.getloc();
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
+  bool __neg                   = false;
+  if (__do_get(__b, __e, __intl, __loc, __iob.flags(), __err, __neg, __ct, __wb, __wn, __we)) {
+    const char __src[] = "0123456789";
+    char_type __atoms[sizeof(__src) - 1];
+    __ct.widen(__src, __src + (sizeof(__src) - 1), __atoms);
+    char __nbuf[__bz];
+    char* __nc = __nbuf;
+    unique_ptr<char, void (*)(void*)> __h(nullptr, free);
+    if (__wn - __wb.get() > __bz - 2) {
+      __h.reset((char*)malloc(static_cast<size_t>(__wn - __wb.get() + 2)));
+      if (__h.get() == nullptr)
+        __throw_bad_alloc();
+      __nc = __h.get();
+    }
+    if (__neg)
+      *__nc++ = '-';
+    for (const char_type* __w = __wb.get(); __w < __wn; ++__w, ++__nc)
+      *__nc = __src[std::find(__atoms, std::end(__atoms), *__w) - __atoms];
+    *__nc = char();
+    if (sscanf(__nbuf, "%Lf", &__v) != 1)
+      __throw_runtime_error("money_get error");
+  }
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator money_get<_CharT, _InputIterator>::do_get(
+    iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, string_type& __v) const {
+  const int __bz = 100;
+  char_type __wbuf[__bz];
+  unique_ptr<char_type, void (*)(void*)> __wb(__wbuf, __do_nothing);
+  char_type* __wn;
+  char_type* __we              = __wbuf + __bz;
+  locale __loc                 = __iob.getloc();
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
+  bool __neg                   = false;
+  if (__do_get(__b, __e, __intl, __loc, __iob.flags(), __err, __neg, __ct, __wb, __wn, __we)) {
+    __v.clear();
+    if (__neg)
+      __v.push_back(__ct.widen('-'));
+    char_type __z = __ct.widen('0');
+    char_type* __w;
+    for (__w = __wb.get(); __w < __wn - 1; ++__w)
+      if (*__w != __z)
+        break;
+    __v.append(__w, __wn);
+  }
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __b;
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_get<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_get<wchar_t>;
+#  endif
+
+// money_put
+
+template <class _CharT>
+class __money_put {
+protected:
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI __money_put() {}
+
+  static void __gather_info(
+      bool __intl,
+      bool __neg,
+      const locale& __loc,
+      money_base::pattern& __pat,
+      char_type& __dp,
+      char_type& __ts,
+      string& __grp,
+      string_type& __sym,
+      string_type& __sn,
+      int& __fd);
+  static void __format(
+      char_type* __mb,
+      char_type*& __mi,
+      char_type*& __me,
+      ios_base::fmtflags __flags,
+      const char_type* __db,
+      const char_type* __de,
+      const ctype<char_type>& __ct,
+      bool __neg,
+      const money_base::pattern& __pat,
+      char_type __dp,
+      char_type __ts,
+      const string& __grp,
+      const string_type& __sym,
+      const string_type& __sn,
+      int __fd);
+};
+
+template <class _CharT>
+void __money_put<_CharT>::__gather_info(
+    bool __intl,
+    bool __neg,
+    const locale& __loc,
+    money_base::pattern& __pat,
+    char_type& __dp,
+    char_type& __ts,
+    string& __grp,
+    string_type& __sym,
+    string_type& __sn,
+    int& __fd) {
+  if (__intl) {
+    const moneypunct<char_type, true>& __mp = std::use_facet<moneypunct<char_type, true> >(__loc);
+    if (__neg) {
+      __pat = __mp.neg_format();
+      __sn  = __mp.negative_sign();
+    } else {
+      __pat = __mp.pos_format();
+      __sn  = __mp.positive_sign();
+    }
+    __dp  = __mp.decimal_point();
+    __ts  = __mp.thousands_sep();
+    __grp = __mp.grouping();
+    __sym = __mp.curr_symbol();
+    __fd  = __mp.frac_digits();
+  } else {
+    const moneypunct<char_type, false>& __mp = std::use_facet<moneypunct<char_type, false> >(__loc);
+    if (__neg) {
+      __pat = __mp.neg_format();
+      __sn  = __mp.negative_sign();
+    } else {
+      __pat = __mp.pos_format();
+      __sn  = __mp.positive_sign();
+    }
+    __dp  = __mp.decimal_point();
+    __ts  = __mp.thousands_sep();
+    __grp = __mp.grouping();
+    __sym = __mp.curr_symbol();
+    __fd  = __mp.frac_digits();
+  }
+}
+
+template <class _CharT>
+void __money_put<_CharT>::__format(
+    char_type* __mb,
+    char_type*& __mi,
+    char_type*& __me,
+    ios_base::fmtflags __flags,
+    const char_type* __db,
+    const char_type* __de,
+    const ctype<char_type>& __ct,
+    bool __neg,
+    const money_base::pattern& __pat,
+    char_type __dp,
+    char_type __ts,
+    const string& __grp,
+    const string_type& __sym,
+    const string_type& __sn,
+    int __fd) {
+  __me = __mb;
+  for (char __p : __pat.field) {
+    switch (__p) {
+    case money_base::none:
+      __mi = __me;
+      break;
+    case money_base::space:
+      __mi    = __me;
+      *__me++ = __ct.widen(' ');
+      break;
+    case money_base::sign:
+      if (!__sn.empty())
+        *__me++ = __sn[0];
+      break;
+    case money_base::symbol:
+      if (!__sym.empty() && (__flags & ios_base::showbase))
+        __me = std::copy(__sym.begin(), __sym.end(), __me);
+      break;
+    case money_base::value: {
+      // remember start of value so we can reverse it
+      char_type* __t = __me;
+      // find beginning of digits
+      if (__neg)
+        ++__db;
+      // find end of digits
+      const char_type* __d;
+      for (__d = __db; __d < __de; ++__d)
+        if (!__ct.is(ctype_base::digit, *__d))
+          break;
+      // print fractional part
+      if (__fd > 0) {
+        int __f;
+        for (__f = __fd; __d > __db && __f > 0; --__f)
+          *__me++ = *--__d;
+        char_type __z = __f > 0 ? __ct.widen('0') : char_type();
+        for (; __f > 0; --__f)
+          *__me++ = __z;
+        *__me++ = __dp;
+      }
+      // print units part
+      if (__d == __db) {
+        *__me++ = __ct.widen('0');
+      } else {
+        unsigned __ng = 0;
+        unsigned __ig = 0;
+        unsigned __gl = __grp.empty() ? numeric_limits<unsigned>::max() : static_cast<unsigned>(__grp[__ig]);
+        while (__d != __db) {
+          if (__ng == __gl) {
+            *__me++ = __ts;
+            __ng    = 0;
+            if (++__ig < __grp.size())
+              __gl = __grp[__ig] == numeric_limits<char>::max()
+                       ? numeric_limits<unsigned>::max()
+                       : static_cast<unsigned>(__grp[__ig]);
+          }
+          *__me++ = *--__d;
+          ++__ng;
+        }
+      }
+      // reverse it
+      std::reverse(__t, __me);
+    } break;
+    }
+  }
+  // print rest of sign, if any
+  if (__sn.size() > 1)
+    __me = std::copy(__sn.begin() + 1, __sn.end(), __me);
+  // set alignment
+  if ((__flags & ios_base::adjustfield) == ios_base::left)
+    __mi = __me;
+  else if ((__flags & ios_base::adjustfield) != ios_base::internal)
+    __mi = __mb;
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_put<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_put<wchar_t>;
+#  endif
+
+template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS money_put : public locale::facet, private __money_put<_CharT> {
+public:
+  typedef _CharT char_type;
+  typedef _OutputIterator iter_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit money_put(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, long double __units) const {
+    return do_put(__s, __intl, __iob, __fl, __units);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, const string_type& __digits) const {
+    return do_put(__s, __intl, __iob, __fl, __digits);
+  }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~money_put() override {}
+
+  virtual iter_type do_put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, long double __units) const;
+  virtual iter_type
+  do_put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, const string_type& __digits) const;
+};
+
+template <class _CharT, class _OutputIterator>
+locale::id money_put<_CharT, _OutputIterator>::id;
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator money_put<_CharT, _OutputIterator>::do_put(
+    iter_type __s, bool __intl, ios_base& __iob, char_type __fl, long double __units) const {
+  // convert to char
+  const size_t __bs = 100;
+  char __buf[__bs];
+  char* __bb = __buf;
+  char_type __digits[__bs];
+  char_type* __db = __digits;
+  int __n         = snprintf(__bb, __bs, "%.0Lf", __units);
+  unique_ptr<char, void (*)(void*)> __hn(nullptr, free);
+  unique_ptr<char_type, void (*)(void*)> __hd(0, free);
+  // secure memory for digit storage
+  if (static_cast<size_t>(__n) > __bs - 1) {
+    __n = __libcpp_asprintf_l(&__bb, _LIBCPP_GET_C_LOCALE, "%.0Lf", __units);
+    if (__n == -1)
+      __throw_bad_alloc();
+    __hn.reset(__bb);
+    __hd.reset((char_type*)malloc(static_cast<size_t>(__n) * sizeof(char_type)));
+    if (__hd == nullptr)
+      __throw_bad_alloc();
+    __db = __hd.get();
+  }
+  // gather info
+  locale __loc                 = __iob.getloc();
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
+  __ct.widen(__bb, __bb + __n, __db);
+  bool __neg = __n > 0 && __bb[0] == '-';
+  money_base::pattern __pat;
+  char_type __dp;
+  char_type __ts;
+  string __grp;
+  string_type __sym;
+  string_type __sn;
+  int __fd;
+  this->__gather_info(__intl, __neg, __loc, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
+  // secure memory for formatting
+  char_type __mbuf[__bs];
+  char_type* __mb = __mbuf;
+  unique_ptr<char_type, void (*)(void*)> __hw(0, free);
+  size_t __exn = __n > __fd ? (static_cast<size_t>(__n) - static_cast<size_t>(__fd)) * 2 + __sn.size() + __sym.size() +
+                                  static_cast<size_t>(__fd) + 1
+                            : __sn.size() + __sym.size() + static_cast<size_t>(__fd) + 2;
+  if (__exn > __bs) {
+    __hw.reset((char_type*)malloc(__exn * sizeof(char_type)));
+    __mb = __hw.get();
+    if (__mb == 0)
+      __throw_bad_alloc();
+  }
+  // format
+  char_type* __mi;
+  char_type* __me;
+  this->__format(
+      __mb, __mi, __me, __iob.flags(), __db, __db + __n, __ct, __neg, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
+  return std::__pad_and_output(__s, __mb, __mi, __me, __iob, __fl);
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator money_put<_CharT, _OutputIterator>::do_put(
+    iter_type __s, bool __intl, ios_base& __iob, char_type __fl, const string_type& __digits) const {
+  // gather info
+  locale __loc                 = __iob.getloc();
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
+  bool __neg                   = __digits.size() > 0 && __digits[0] == __ct.widen('-');
+  money_base::pattern __pat;
+  char_type __dp;
+  char_type __ts;
+  string __grp;
+  string_type __sym;
+  string_type __sn;
+  int __fd;
+  this->__gather_info(__intl, __neg, __loc, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
+  // secure memory for formatting
+  char_type __mbuf[100];
+  char_type* __mb = __mbuf;
+  unique_ptr<char_type, void (*)(void*)> __h(0, free);
+  size_t __exn =
+      static_cast<int>(__digits.size()) > __fd
+          ? (__digits.size() - static_cast<size_t>(__fd)) * 2 + __sn.size() + __sym.size() + static_cast<size_t>(__fd) +
+                1
+          : __sn.size() + __sym.size() + static_cast<size_t>(__fd) + 2;
+  if (__exn > 100) {
+    __h.reset((char_type*)malloc(__exn * sizeof(char_type)));
+    __mb = __h.get();
+    if (__mb == 0)
+      __throw_bad_alloc();
+  }
+  // format
+  char_type* __mi;
+  char_type* __me;
+  this->__format(
+      __mb,
+      __mi,
+      __me,
+      __iob.flags(),
+      __digits.data(),
+      __digits.data() + __digits.size(),
+      __ct,
+      __neg,
+      __pat,
+      __dp,
+      __ts,
+      __grp,
+      __sym,
+      __sn,
+      __fd);
+  return std::__pad_and_output(__s, __mb, __mi, __me, __iob, __fl);
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_put<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_put<wchar_t>;
+#  endif
+
+// messages
+
+class _LIBCPP_EXPORTED_FROM_ABI messages_base {
+public:
+  typedef intptr_t catalog;
+
+  _LIBCPP_HIDE_FROM_ABI messages_base() {}
+};
+
+template <class _CharT>
+class _LIBCPP_TEMPLATE_VIS messages : public locale::facet, public messages_base {
+public:
+  typedef _CharT char_type;
+  typedef basic_string<_CharT> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit messages(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI catalog open(const basic_string<char>& __nm, const locale& __loc) const {
+    return do_open(__nm, __loc);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI string_type get(catalog __c, int __set, int __msgid, const string_type& __dflt) const {
+    return do_get(__c, __set, __msgid, __dflt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void close(catalog __c) const { do_close(__c); }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~messages() override {}
+
+  virtual catalog do_open(const basic_string<char>&, const locale&) const;
+  virtual string_type do_get(catalog, int __set, int __msgid, const string_type& __dflt) const;
+  virtual void do_close(catalog) const;
+};
+
+template <class _CharT>
+locale::id messages<_CharT>::id;
+
+template <class _CharT>
+typename messages<_CharT>::catalog messages<_CharT>::do_open(const basic_string<char>& __nm, const locale&) const {
+#  ifdef _LIBCPP_HAS_CATOPEN
+  return (catalog)catopen(__nm.c_str(), NL_CAT_LOCALE);
+#  else  // !_LIBCPP_HAS_CATOPEN
+  (void)__nm;
+  return -1;
+#  endif // _LIBCPP_HAS_CATOPEN
+}
+
+template <class _CharT>
+typename messages<_CharT>::string_type
+messages<_CharT>::do_get(catalog __c, int __set, int __msgid, const string_type& __dflt) const {
+#  ifdef _LIBCPP_HAS_CATOPEN
+  string __ndflt;
+  __narrow_to_utf8<sizeof(char_type) * __CHAR_BIT__>()(
+      std::back_inserter(__ndflt), __dflt.c_str(), __dflt.c_str() + __dflt.size());
+  nl_catd __cat = (nl_catd)__c;
+  static_assert(sizeof(catalog) >= sizeof(nl_catd), "Unexpected nl_catd type");
+  char* __n = catgets(__cat, __set, __msgid, __ndflt.c_str());
+  string_type __w;
+  __widen_from_utf8<sizeof(char_type) * __CHAR_BIT__>()(std::back_inserter(__w), __n, __n + std::strlen(__n));
+  return __w;
+#  else  // !_LIBCPP_HAS_CATOPEN
+  (void)__c;
+  (void)__set;
+  (void)__msgid;
+  return __dflt;
+#  endif // _LIBCPP_HAS_CATOPEN
+}
+
+template <class _CharT>
+void messages<_CharT>::do_close(catalog __c) const {
+#  ifdef _LIBCPP_HAS_CATOPEN
+  catclose((nl_catd)__c);
+#  else  // !_LIBCPP_HAS_CATOPEN
+  (void)__c;
+#  endif // _LIBCPP_HAS_CATOPEN
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages<wchar_t>;
+#  endif
+
+template <class _CharT>
+class _LIBCPP_TEMPLATE_VIS messages_byname : public messages<_CharT> {
+public:
+  typedef messages_base::catalog catalog;
+  typedef basic_string<_CharT> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit messages_byname(const char*, size_t __refs = 0) : messages<_CharT>(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit messages_byname(const string&, size_t __refs = 0) : messages<_CharT>(__refs) {}
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~messages_byname() override {}
+};
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages_byname<char>;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages_byname<wchar_t>;
+#  endif
+
+#  if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
+
+template <class _Codecvt,
+          class _Elem      = wchar_t,
+          class _WideAlloc = allocator<_Elem>,
+          class _ByteAlloc = allocator<char> >
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 wstring_convert {
+public:
+  typedef basic_string<char, char_traits<char>, _ByteAlloc> byte_string;
+  typedef basic_string<_Elem, char_traits<_Elem>, _WideAlloc> wide_string;
+  typedef typename _Codecvt::state_type state_type;
+  typedef typename wide_string::traits_type::int_type int_type;
+
+private:
+  byte_string __byte_err_string_;
+  wide_string __wide_err_string_;
+  _Codecvt* __cvtptr_;
+  state_type __cvtstate_;
+  size_t __cvtcount_;
+
+public:
+#    ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI wstring_convert() : wstring_convert(new _Codecvt) {}
+  _LIBCPP_HIDE_FROM_ABI explicit wstring_convert(_Codecvt* __pcvt);
+#    else
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_EXPLICIT_SINCE_CXX14 wstring_convert(_Codecvt* __pcvt = new _Codecvt);
+#    endif
+
+  _LIBCPP_HIDE_FROM_ABI wstring_convert(_Codecvt* __pcvt, state_type __state);
+  _LIBCPP_EXPLICIT_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+  wstring_convert(const byte_string& __byte_err, const wide_string& __wide_err = wide_string());
+#    ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI wstring_convert(wstring_convert&& __wc);
+#    endif
+  _LIBCPP_HIDE_FROM_ABI ~wstring_convert();
+
+  wstring_convert(const wstring_convert& __wc)            = delete;
+  wstring_convert& operator=(const wstring_convert& __wc) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(char __byte) { return from_bytes(&__byte, &__byte + 1); }
+  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(const char* __ptr) {
+    return from_bytes(__ptr, __ptr + char_traits<char>::length(__ptr));
+  }
+  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(const byte_string& __str) {
+    return from_bytes(__str.data(), __str.data() + __str.size());
+  }
+  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(const char* __first, const char* __last);
+
+  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(_Elem __wchar) { return to_bytes(&__wchar, &__wchar + 1); }
+  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(const _Elem* __wptr) {
+    return to_bytes(__wptr, __wptr + char_traits<_Elem>::length(__wptr));
+  }
+  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(const wide_string& __wstr) {
+    return to_bytes(__wstr.data(), __wstr.data() + __wstr.size());
+  }
+  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(const _Elem* __first, const _Elem* __last);
+
+  _LIBCPP_HIDE_FROM_ABI size_t converted() const _NOEXCEPT { return __cvtcount_; }
+  _LIBCPP_HIDE_FROM_ABI state_type state() const { return __cvtstate_; }
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(_Codecvt* __pcvt)
+    : __cvtptr_(__pcvt), __cvtstate_(), __cvtcount_(0) {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(_Codecvt* __pcvt, state_type __state)
+    : __cvtptr_(__pcvt), __cvtstate_(__state), __cvtcount_(0) {}
+
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(
+    const byte_string& __byte_err, const wide_string& __wide_err)
+    : __byte_err_string_(__byte_err), __wide_err_string_(__wide_err), __cvtstate_(), __cvtcount_(0) {
+  __cvtptr_ = new _Codecvt;
+}
+
+#    ifndef _LIBCPP_CXX03_LANG
+
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(wstring_convert&& __wc)
+    : __byte_err_string_(std::move(__wc.__byte_err_string_)),
+      __wide_err_string_(std::move(__wc.__wide_err_string_)),
+      __cvtptr_(__wc.__cvtptr_),
+      __cvtstate_(__wc.__cvtstate_),
+      __cvtcount_(__wc.__cvtcount_) {
+  __wc.__cvtptr_ = nullptr;
+}
+
+#    endif // _LIBCPP_CXX03_LANG
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::~wstring_convert() {
+  delete __cvtptr_;
+}
+
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+typename wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wide_string
+wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::from_bytes(const char* __frm, const char* __frm_end) {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  __cvtcount_ = 0;
+  if (__cvtptr_ != nullptr) {
+    wide_string __ws(2 * (__frm_end - __frm), _Elem());
+    if (__frm != __frm_end)
+      __ws.resize(__ws.capacity());
+    codecvt_base::result __r = codecvt_base::ok;
+    state_type __st          = __cvtstate_;
+    if (__frm != __frm_end) {
+      _Elem* __to     = &__ws[0];
+      _Elem* __to_end = __to + __ws.size();
+      const char* __frm_nxt;
+      do {
+        _Elem* __to_nxt;
+        __r = __cvtptr_->in(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+        __cvtcount_ += __frm_nxt - __frm;
+        if (__frm_nxt == __frm) {
+          __r = codecvt_base::error;
+        } else if (__r == codecvt_base::noconv) {
+          __ws.resize(__to - &__ws[0]);
+          // This only gets executed if _Elem is char
+          __ws.append((const _Elem*)__frm, (const _Elem*)__frm_end);
+          __frm = __frm_nxt;
+          __r   = codecvt_base::ok;
+        } else if (__r == codecvt_base::ok) {
+          __ws.resize(__to_nxt - &__ws[0]);
+          __frm = __frm_nxt;
+        } else if (__r == codecvt_base::partial) {
+          ptrdiff_t __s = __to_nxt - &__ws[0];
+          __ws.resize(2 * __s);
+          __to     = &__ws[0] + __s;
+          __to_end = &__ws[0] + __ws.size();
+          __frm    = __frm_nxt;
+        }
+      } while (__r == codecvt_base::partial && __frm_nxt < __frm_end);
+    }
+    if (__r == codecvt_base::ok)
+      return __ws;
+  }
+
+  if (__wide_err_string_.empty())
+    __throw_range_error("wstring_convert: from_bytes error");
+
+  return __wide_err_string_;
+}
+
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+typename wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::byte_string
+wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::to_bytes(const _Elem* __frm, const _Elem* __frm_end) {
+  __cvtcount_ = 0;
+  if (__cvtptr_ != nullptr) {
+    byte_string __bs(2 * (__frm_end - __frm), char());
+    if (__frm != __frm_end)
+      __bs.resize(__bs.capacity());
+    codecvt_base::result __r = codecvt_base::ok;
+    state_type __st          = __cvtstate_;
+    if (__frm != __frm_end) {
+      char* __to     = &__bs[0];
+      char* __to_end = __to + __bs.size();
+      const _Elem* __frm_nxt;
+      do {
+        char* __to_nxt;
+        __r = __cvtptr_->out(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+        __cvtcount_ += __frm_nxt - __frm;
+        if (__frm_nxt == __frm) {
+          __r = codecvt_base::error;
+        } else if (__r == codecvt_base::noconv) {
+          __bs.resize(__to - &__bs[0]);
+          // This only gets executed if _Elem is char
+          __bs.append((const char*)__frm, (const char*)__frm_end);
+          __frm = __frm_nxt;
+          __r   = codecvt_base::ok;
+        } else if (__r == codecvt_base::ok) {
+          __bs.resize(__to_nxt - &__bs[0]);
+          __frm = __frm_nxt;
+        } else if (__r == codecvt_base::partial) {
+          ptrdiff_t __s = __to_nxt - &__bs[0];
+          __bs.resize(2 * __s);
+          __to     = &__bs[0] + __s;
+          __to_end = &__bs[0] + __bs.size();
+          __frm    = __frm_nxt;
+        }
+      } while (__r == codecvt_base::partial && __frm_nxt < __frm_end);
+    }
+    if (__r == codecvt_base::ok) {
+      size_t __s = __bs.size();
+      __bs.resize(__bs.capacity());
+      char* __to     = &__bs[0] + __s;
+      char* __to_end = __to + __bs.size();
+      do {
+        char* __to_nxt;
+        __r = __cvtptr_->unshift(__st, __to, __to_end, __to_nxt);
+        if (__r == codecvt_base::noconv) {
+          __bs.resize(__to - &__bs[0]);
+          __r = codecvt_base::ok;
+        } else if (__r == codecvt_base::ok) {
+          __bs.resize(__to_nxt - &__bs[0]);
+        } else if (__r == codecvt_base::partial) {
+          ptrdiff_t __sp = __to_nxt - &__bs[0];
+          __bs.resize(2 * __sp);
+          __to     = &__bs[0] + __sp;
+          __to_end = &__bs[0] + __bs.size();
+        }
+      } while (__r == codecvt_base::partial);
+      if (__r == codecvt_base::ok)
+        return __bs;
+    }
+  }
+
+  if (__byte_err_string_.empty())
+    __throw_range_error("wstring_convert: to_bytes error");
+
+  return __byte_err_string_;
+}
+
+template <class _Codecvt, class _Elem = wchar_t, class _Tr = char_traits<_Elem> >
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 wbuffer_convert : public basic_streambuf<_Elem, _Tr> {
+public:
+  // types:
+  typedef _Elem char_type;
+  typedef _Tr traits_type;
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+  typedef typename _Codecvt::state_type state_type;
+
+private:
+  char* __extbuf_;
+  const char* __extbufnext_;
+  const char* __extbufend_;
+  char __extbuf_min_[8];
+  size_t __ebs_;
+  char_type* __intbuf_;
+  size_t __ibs_;
+  streambuf* __bufptr_;
+  _Codecvt* __cv_;
+  state_type __st_;
+  ios_base::openmode __cm_;
+  bool __owns_eb_;
+  bool __owns_ib_;
+  bool __always_noconv_;
+
+public:
+#    ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI wbuffer_convert() : wbuffer_convert(nullptr) {}
+  explicit _LIBCPP_HIDE_FROM_ABI
+  wbuffer_convert(streambuf* __bytebuf, _Codecvt* __pcvt = new _Codecvt, state_type __state = state_type());
+#    else
+  _LIBCPP_EXPLICIT_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+  wbuffer_convert(streambuf* __bytebuf = nullptr, _Codecvt* __pcvt = new _Codecvt, state_type __state = state_type());
+#    endif
+
+  _LIBCPP_HIDE_FROM_ABI ~wbuffer_convert();
+
+  _LIBCPP_HIDE_FROM_ABI streambuf* rdbuf() const { return __bufptr_; }
+  _LIBCPP_HIDE_FROM_ABI streambuf* rdbuf(streambuf* __bytebuf) {
+    streambuf* __r = __bufptr_;
+    __bufptr_      = __bytebuf;
+    return __r;
+  }
+
+  wbuffer_convert(const wbuffer_convert&)            = delete;
+  wbuffer_convert& operator=(const wbuffer_convert&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI state_type state() const { return __st_; }
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int_type underflow();
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int_type pbackfail(int_type __c = traits_type::eof());
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int_type overflow(int_type __c = traits_type::eof());
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual basic_streambuf<char_type, traits_type>* setbuf(char_type* __s, streamsize __n);
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual pos_type
+  seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __wch = ios_base::in | ios_base::out);
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual pos_type
+  seekpos(pos_type __sp, ios_base::openmode __wch = ios_base::in | ios_base::out);
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int sync();
+
+private:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL bool __read_mode();
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __write_mode();
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL wbuffer_convert* __close();
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+wbuffer_convert<_Codecvt, _Elem, _Tr>::wbuffer_convert(streambuf* __bytebuf, _Codecvt* __pcvt, state_type __state)
+    : __extbuf_(nullptr),
+      __extbufnext_(nullptr),
+      __extbufend_(nullptr),
+      __ebs_(0),
+      __intbuf_(0),
+      __ibs_(0),
+      __bufptr_(__bytebuf),
+      __cv_(__pcvt),
+      __st_(__state),
+      __cm_(0),
+      __owns_eb_(false),
+      __owns_ib_(false),
+      __always_noconv_(__cv_ ? __cv_->always_noconv() : false) {
+  setbuf(0, 4096);
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+wbuffer_convert<_Codecvt, _Elem, _Tr>::~wbuffer_convert() {
+  __close();
+  delete __cv_;
+  if (__owns_eb_)
+    delete[] __extbuf_;
+  if (__owns_ib_)
+    delete[] __intbuf_;
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+typename wbuffer_convert<_Codecvt, _Elem, _Tr>::int_type wbuffer_convert<_Codecvt, _Elem, _Tr>::underflow() {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  if (__cv_ == 0 || __bufptr_ == nullptr)
+    return traits_type::eof();
+  bool __initial = __read_mode();
+  char_type __1buf;
+  if (this->gptr() == 0)
+    this->setg(&__1buf, &__1buf + 1, &__1buf + 1);
+  const size_t __unget_sz = __initial ? 0 : std::min<size_t>((this->egptr() - this->eback()) / 2, 4);
+  int_type __c            = traits_type::eof();
+  if (this->gptr() == this->egptr()) {
+    std::memmove(this->eback(), this->egptr() - __unget_sz, __unget_sz * sizeof(char_type));
+    if (__always_noconv_) {
+      streamsize __nmemb = static_cast<streamsize>(this->egptr() - this->eback() - __unget_sz);
+      __nmemb            = __bufptr_->sgetn((char*)this->eback() + __unget_sz, __nmemb);
+      if (__nmemb != 0) {
+        this->setg(this->eback(), this->eback() + __unget_sz, this->eback() + __unget_sz + __nmemb);
+        __c = *this->gptr();
+      }
+    } else {
+      if (__extbufend_ != __extbufnext_) {
+        _LIBCPP_ASSERT_NON_NULL(__extbufnext_ != nullptr, "underflow moving from nullptr");
+        _LIBCPP_ASSERT_NON_NULL(__extbuf_ != nullptr, "underflow moving into nullptr");
+        std::memmove(__extbuf_, __extbufnext_, __extbufend_ - __extbufnext_);
+      }
+      __extbufnext_      = __extbuf_ + (__extbufend_ - __extbufnext_);
+      __extbufend_       = __extbuf_ + (__extbuf_ == __extbuf_min_ ? sizeof(__extbuf_min_) : __ebs_);
+      streamsize __nmemb = std::min(static_cast<streamsize>(this->egptr() - this->eback() - __unget_sz),
+                                    static_cast<streamsize>(__extbufend_ - __extbufnext_));
+      codecvt_base::result __r;
+      // FIXME: Do we ever need to restore the state here?
+      // state_type __svs = __st_;
+      streamsize __nr = __bufptr_->sgetn(const_cast<char*>(__extbufnext_), __nmemb);
+      if (__nr != 0) {
+        __extbufend_ = __extbufnext_ + __nr;
+        char_type* __inext;
+        __r = __cv_->in(
+            __st_, __extbuf_, __extbufend_, __extbufnext_, this->eback() + __unget_sz, this->egptr(), __inext);
+        if (__r == codecvt_base::noconv) {
+          this->setg((char_type*)__extbuf_, (char_type*)__extbuf_, (char_type*)const_cast<char*>(__extbufend_));
+          __c = *this->gptr();
+        } else if (__inext != this->eback() + __unget_sz) {
+          this->setg(this->eback(), this->eback() + __unget_sz, __inext);
+          __c = *this->gptr();
+        }
+      }
+    }
+  } else
+    __c = *this->gptr();
+  if (this->eback() == &__1buf)
+    this->setg(0, 0, 0);
+  return __c;
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+typename wbuffer_convert<_Codecvt, _Elem, _Tr>::int_type
+wbuffer_convert<_Codecvt, _Elem, _Tr>::pbackfail(int_type __c) {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  if (__cv_ != 0 && __bufptr_ && this->eback() < this->gptr()) {
+    if (traits_type::eq_int_type(__c, traits_type::eof())) {
+      this->gbump(-1);
+      return traits_type::not_eof(__c);
+    }
+    if (traits_type::eq(traits_type::to_char_type(__c), this->gptr()[-1])) {
+      this->gbump(-1);
+      *this->gptr() = traits_type::to_char_type(__c);
+      return __c;
+    }
+  }
+  return traits_type::eof();
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+typename wbuffer_convert<_Codecvt, _Elem, _Tr>::int_type wbuffer_convert<_Codecvt, _Elem, _Tr>::overflow(int_type __c) {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  if (__cv_ == 0 || !__bufptr_)
+    return traits_type::eof();
+  __write_mode();
+  char_type __1buf;
+  char_type* __pb_save  = this->pbase();
+  char_type* __epb_save = this->epptr();
+  if (!traits_type::eq_int_type(__c, traits_type::eof())) {
+    if (this->pptr() == 0)
+      this->setp(&__1buf, &__1buf + 1);
+    *this->pptr() = traits_type::to_char_type(__c);
+    this->pbump(1);
+  }
+  if (this->pptr() != this->pbase()) {
+    if (__always_noconv_) {
+      streamsize __nmemb = static_cast<streamsize>(this->pptr() - this->pbase());
+      if (__bufptr_->sputn((const char*)this->pbase(), __nmemb) != __nmemb)
+        return traits_type::eof();
+    } else {
+      char* __extbe = __extbuf_;
+      codecvt_base::result __r;
+      do {
+        const char_type* __e;
+        __r = __cv_->out(__st_, this->pbase(), this->pptr(), __e, __extbuf_, __extbuf_ + __ebs_, __extbe);
+        if (__e == this->pbase())
+          return traits_type::eof();
+        if (__r == codecvt_base::noconv) {
+          streamsize __nmemb = static_cast<size_t>(this->pptr() - this->pbase());
+          if (__bufptr_->sputn((const char*)this->pbase(), __nmemb) != __nmemb)
+            return traits_type::eof();
+        } else if (__r == codecvt_base::ok || __r == codecvt_base::partial) {
+          streamsize __nmemb = static_cast<size_t>(__extbe - __extbuf_);
+          if (__bufptr_->sputn(__extbuf_, __nmemb) != __nmemb)
+            return traits_type::eof();
+          if (__r == codecvt_base::partial) {
+            this->setp(const_cast<char_type*>(__e), this->pptr());
+            this->__pbump(this->epptr() - this->pbase());
+          }
+        } else
+          return traits_type::eof();
+      } while (__r == codecvt_base::partial);
+    }
+    this->setp(__pb_save, __epb_save);
+  }
+  return traits_type::not_eof(__c);
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+basic_streambuf<_Elem, _Tr>* wbuffer_convert<_Codecvt, _Elem, _Tr>::setbuf(char_type* __s, streamsize __n) {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  this->setg(0, 0, 0);
+  this->setp(0, 0);
+  if (__owns_eb_)
+    delete[] __extbuf_;
+  if (__owns_ib_)
+    delete[] __intbuf_;
+  __ebs_ = __n;
+  if (__ebs_ > sizeof(__extbuf_min_)) {
+    if (__always_noconv_ && __s) {
+      __extbuf_  = (char*)__s;
+      __owns_eb_ = false;
+    } else {
+      __extbuf_  = new char[__ebs_];
+      __owns_eb_ = true;
+    }
+  } else {
+    __extbuf_  = __extbuf_min_;
+    __ebs_     = sizeof(__extbuf_min_);
+    __owns_eb_ = false;
+  }
+  if (!__always_noconv_) {
+    __ibs_ = max<streamsize>(__n, sizeof(__extbuf_min_));
+    if (__s && __ibs_ >= sizeof(__extbuf_min_)) {
+      __intbuf_  = __s;
+      __owns_ib_ = false;
+    } else {
+      __intbuf_  = new char_type[__ibs_];
+      __owns_ib_ = true;
+    }
+  } else {
+    __ibs_     = 0;
+    __intbuf_  = 0;
+    __owns_ib_ = false;
+  }
+  return this;
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+typename wbuffer_convert<_Codecvt, _Elem, _Tr>::pos_type
+wbuffer_convert<_Codecvt, _Elem, _Tr>::seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __om) {
+  int __width = __cv_->encoding();
+  if (__cv_ == 0 || !__bufptr_ || (__width <= 0 && __off != 0) || sync())
+    return pos_type(off_type(-1));
+  // __width > 0 || __off == 0, now check __way
+  if (__way != ios_base::beg && __way != ios_base::cur && __way != ios_base::end)
+    return pos_type(off_type(-1));
+  pos_type __r = __bufptr_->pubseekoff(__width * __off, __way, __om);
+  __r.state(__st_);
+  return __r;
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+typename wbuffer_convert<_Codecvt, _Elem, _Tr>::pos_type
+wbuffer_convert<_Codecvt, _Elem, _Tr>::seekpos(pos_type __sp, ios_base::openmode __wch) {
+  if (__cv_ == 0 || !__bufptr_ || sync())
+    return pos_type(off_type(-1));
+  if (__bufptr_->pubseekpos(__sp, __wch) == pos_type(off_type(-1)))
+    return pos_type(off_type(-1));
+  return __sp;
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+int wbuffer_convert<_Codecvt, _Elem, _Tr>::sync() {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  if (__cv_ == 0 || !__bufptr_)
+    return 0;
+  if (__cm_ & ios_base::out) {
+    if (this->pptr() != this->pbase())
+      if (overflow() == traits_type::eof())
+        return -1;
+    codecvt_base::result __r;
+    do {
+      char* __extbe;
+      __r                = __cv_->unshift(__st_, __extbuf_, __extbuf_ + __ebs_, __extbe);
+      streamsize __nmemb = static_cast<streamsize>(__extbe - __extbuf_);
+      if (__bufptr_->sputn(__extbuf_, __nmemb) != __nmemb)
+        return -1;
+    } while (__r == codecvt_base::partial);
+    if (__r == codecvt_base::error)
+      return -1;
+    if (__bufptr_->pubsync())
+      return -1;
+  } else if (__cm_ & ios_base::in) {
+    off_type __c;
+    if (__always_noconv_)
+      __c = this->egptr() - this->gptr();
+    else {
+      int __width = __cv_->encoding();
+      __c         = __extbufend_ - __extbufnext_;
+      if (__width > 0)
+        __c += __width * (this->egptr() - this->gptr());
+      else {
+        if (this->gptr() != this->egptr()) {
+          std::reverse(this->gptr(), this->egptr());
+          codecvt_base::result __r;
+          const char_type* __e = this->gptr();
+          char* __extbe;
+          do {
+            __r = __cv_->out(__st_, __e, this->egptr(), __e, __extbuf_, __extbuf_ + __ebs_, __extbe);
+            switch (__r) {
+            case codecvt_base::noconv:
+              __c += this->egptr() - this->gptr();
+              break;
+            case codecvt_base::ok:
+            case codecvt_base::partial:
+              __c += __extbe - __extbuf_;
+              break;
+            default:
+              return -1;
+            }
+          } while (__r == codecvt_base::partial);
+        }
+      }
+    }
+    if (__bufptr_->pubseekoff(-__c, ios_base::cur, __cm_) == pos_type(off_type(-1)))
+      return -1;
+    this->setg(0, 0, 0);
+    __cm_ = 0;
+  }
+  return 0;
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+bool wbuffer_convert<_Codecvt, _Elem, _Tr>::__read_mode() {
+  if (!(__cm_ & ios_base::in)) {
+    this->setp(0, 0);
+    if (__always_noconv_)
+      this->setg((char_type*)__extbuf_, (char_type*)__extbuf_ + __ebs_, (char_type*)__extbuf_ + __ebs_);
+    else
+      this->setg(__intbuf_, __intbuf_ + __ibs_, __intbuf_ + __ibs_);
+    __cm_ = ios_base::in;
+    return true;
+  }
+  return false;
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+void wbuffer_convert<_Codecvt, _Elem, _Tr>::__write_mode() {
+  if (!(__cm_ & ios_base::out)) {
+    this->setg(0, 0, 0);
+    if (__ebs_ > sizeof(__extbuf_min_)) {
+      if (__always_noconv_)
+        this->setp((char_type*)__extbuf_, (char_type*)__extbuf_ + (__ebs_ - 1));
+      else
+        this->setp(__intbuf_, __intbuf_ + (__ibs_ - 1));
+    } else
+      this->setp(0, 0);
+    __cm_ = ios_base::out;
+  }
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+wbuffer_convert<_Codecvt, _Elem, _Tr>* wbuffer_convert<_Codecvt, _Elem, _Tr>::__close() {
+  wbuffer_convert* __rt = nullptr;
+  if (__cv_ != nullptr && __bufptr_ != nullptr) {
+    __rt = this;
+    if ((__cm_ & ios_base::out) && sync())
+      __rt = nullptr;
+  }
+  return __rt;
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+#  endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+// NOLINTEND(libcpp-robust-against-adl)
+
+#endif // !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <concepts>
+#  include <cstdarg>
+#  include <iterator>
+#  include <mutex>
+#  include <stdexcept>
+#  include <type_traits>
+#  include <typeinfo>
+#endif
+
+#endif // _LIBCPP_LOCALE
diff --git a/libcxx/include/__cxx03/locale.h b/libcxx/include/__cxx03/locale.h
new file mode 100644
index 000000000000..425bf47d437a
--- /dev/null
+++ b/libcxx/include/__cxx03/locale.h
@@ -0,0 +1,46 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_LOCALE_H
+#define _LIBCPP_LOCALE_H
+
+/*
+    locale.h synopsis
+
+Macros:
+
+    LC_ALL
+    LC_COLLATE
+    LC_CTYPE
+    LC_MONETARY
+    LC_NUMERIC
+    LC_TIME
+
+Types:
+
+    lconv
+
+Functions:
+
+   setlocale
+   localeconv
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if __has_include_next(<locale.h>)
+#  include_next <locale.h>
+#endif
+
+#endif // _LIBCPP_LOCALE_H
diff --git a/libcxx/include/__cxx03/map b/libcxx/include/__cxx03/map
new file mode 100644
index 000000000000..02bd17ccb4e8
--- /dev/null
+++ b/libcxx/include/__cxx03/map
@@ -0,0 +1,2181 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_MAP
+#define _LIBCPP_MAP
+
+/*
+
+    map synopsis
+
+namespace std
+{
+
+template <class Key, class T, class Compare = less<Key>,
+          class Allocator = allocator<pair<const Key, T>>>
+class map
+{
+public:
+    // types:
+    typedef Key                                      key_type;
+    typedef T                                        mapped_type;
+    typedef pair<const key_type, mapped_type>        value_type;
+    typedef Compare                                  key_compare;
+    typedef Allocator                                allocator_type;
+    typedef typename allocator_type::reference       reference;
+    typedef typename allocator_type::const_reference const_reference;
+    typedef typename allocator_type::pointer         pointer;
+    typedef typename allocator_type::const_pointer   const_pointer;
+    typedef typename allocator_type::size_type       size_type;
+    typedef typename allocator_type::difference_type difference_type;
+
+    typedef implementation-defined                   iterator;
+    typedef implementation-defined                   const_iterator;
+    typedef std::reverse_iterator<iterator>          reverse_iterator;
+    typedef std::reverse_iterator<const_iterator>    const_reverse_iterator;
+    typedef unspecified                              node_type;              // C++17
+    typedef INSERT_RETURN_TYPE<iterator, node_type>  insert_return_type;     // C++17
+
+    class value_compare
+    {
+        friend class map;
+    protected:
+        key_compare comp;
+
+        value_compare(key_compare c);
+    public:
+        typedef bool result_type;  // deprecated in C++17, removed in C++20
+        typedef value_type first_argument_type;  // deprecated in C++17, removed in C++20
+        typedef value_type second_argument_type;  // deprecated in C++17, removed in C++20
+        bool operator()(const value_type& x, const value_type& y) const;
+    };
+
+    // construct/copy/destroy:
+    map()
+        noexcept(
+            is_nothrow_default_constructible<allocator_type>::value &&
+            is_nothrow_default_constructible<key_compare>::value &&
+            is_nothrow_copy_constructible<key_compare>::value);
+    explicit map(const key_compare& comp);
+    map(const key_compare& comp, const allocator_type& a);
+    template <class InputIterator>
+        map(InputIterator first, InputIterator last,
+            const key_compare& comp = key_compare());
+    template <class InputIterator>
+        map(InputIterator first, InputIterator last,
+            const key_compare& comp, const allocator_type& a);
+    template<container-compatible-range<value_type> R>
+      map(from_range_t, R&& rg, const Compare& comp = Compare(), const Allocator& = Allocator()); // C++23
+    map(const map& m);
+    map(map&& m)
+        noexcept(
+            is_nothrow_move_constructible<allocator_type>::value &&
+            is_nothrow_move_constructible<key_compare>::value);
+    explicit map(const allocator_type& a);
+    map(const map& m, const allocator_type& a);
+    map(map&& m, const allocator_type& a);
+    map(initializer_list<value_type> il, const key_compare& comp = key_compare());
+    map(initializer_list<value_type> il, const key_compare& comp, const allocator_type& a);
+    template <class InputIterator>
+        map(InputIterator first, InputIterator last, const allocator_type& a)
+            : map(first, last, Compare(), a) {}  // C++14
+    template<container-compatible-range<value_type> R>
+      map(from_range_t, R&& rg, const Allocator& a))
+        : map(from_range, std::forward<R>(rg), Compare(), a) { } // C++23
+    map(initializer_list<value_type> il, const allocator_type& a)
+        : map(il, Compare(), a) {}  // C++14
+   ~map();
+
+    map& operator=(const map& m);
+    map& operator=(map&& m)
+        noexcept(
+            allocator_type::propagate_on_container_move_assignment::value &&
+            is_nothrow_move_assignable<allocator_type>::value &&
+            is_nothrow_move_assignable<key_compare>::value);
+    map& operator=(initializer_list<value_type> il);
+
+    // iterators:
+          iterator begin() noexcept;
+    const_iterator begin() const noexcept;
+          iterator end() noexcept;
+    const_iterator end()   const noexcept;
+
+          reverse_iterator rbegin() noexcept;
+    const_reverse_iterator rbegin() const noexcept;
+          reverse_iterator rend() noexcept;
+    const_reverse_iterator rend()   const noexcept;
+
+    const_iterator         cbegin()  const noexcept;
+    const_iterator         cend()    const noexcept;
+    const_reverse_iterator crbegin() const noexcept;
+    const_reverse_iterator crend()   const noexcept;
+
+    // capacity:
+    bool      empty()    const noexcept;
+    size_type size()     const noexcept;
+    size_type max_size() const noexcept;
+
+    // element access:
+    mapped_type& operator[](const key_type& k);
+    mapped_type& operator[](key_type&& k);
+
+          mapped_type& at(const key_type& k);
+    const mapped_type& at(const key_type& k) const;
+
+    // modifiers:
+    template <class... Args>
+        pair<iterator, bool> emplace(Args&&... args);
+    template <class... Args>
+        iterator emplace_hint(const_iterator position, Args&&... args);
+    pair<iterator, bool> insert(const value_type& v);
+    pair<iterator, bool> insert(      value_type&& v);                                // C++17
+    template <class P>
+        pair<iterator, bool> insert(P&& p);
+    iterator insert(const_iterator position, const value_type& v);
+    iterator insert(const_iterator position,       value_type&& v);                   // C++17
+    template <class P>
+        iterator insert(const_iterator position, P&& p);
+    template <class InputIterator>
+        void insert(InputIterator first, InputIterator last);
+    template<container-compatible-range<value_type> R>
+      void insert_range(R&& rg);                                                      // C++23
+    void insert(initializer_list<value_type> il);
+
+    node_type extract(const_iterator position);                                       // C++17
+    node_type extract(const key_type& x);                                             // C++17
+    insert_return_type insert(node_type&& nh);                                        // C++17
+    iterator insert(const_iterator hint, node_type&& nh);                             // C++17
+
+    template <class... Args>
+        pair<iterator, bool> try_emplace(const key_type& k, Args&&... args);          // C++17
+    template <class... Args>
+        pair<iterator, bool> try_emplace(key_type&& k, Args&&... args);               // C++17
+    template <class... Args>
+        iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args); // C++17
+    template <class... Args>
+        iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args);      // C++17
+    template <class M>
+        pair<iterator, bool> insert_or_assign(const key_type& k, M&& obj);            // C++17
+    template <class M>
+        pair<iterator, bool> insert_or_assign(key_type&& k, M&& obj);                 // C++17
+    template <class M>
+        iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj);   // C++17
+    template <class M>
+        iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj);        // C++17
+
+    iterator  erase(const_iterator position);
+    iterator  erase(iterator position); // C++14
+    size_type erase(const key_type& k);
+    iterator  erase(const_iterator first, const_iterator last);
+    void clear() noexcept;
+
+    template<class C2>
+      void merge(map<Key, T, C2, Allocator>& source);         // C++17
+    template<class C2>
+      void merge(map<Key, T, C2, Allocator>&& source);        // C++17
+    template<class C2>
+      void merge(multimap<Key, T, C2, Allocator>& source);    // C++17
+    template<class C2>
+      void merge(multimap<Key, T, C2, Allocator>&& source);   // C++17
+
+    void swap(map& m)
+        noexcept(allocator_traits<allocator_type>::is_always_equal::value &&
+            is_nothrow_swappable<key_compare>::value); // C++17
+
+    // observers:
+    allocator_type get_allocator() const noexcept;
+    key_compare    key_comp()      const;
+    value_compare  value_comp()    const;
+
+    // map operations:
+          iterator find(const key_type& k);
+    const_iterator find(const key_type& k) const;
+    template<typename K>
+        iterator find(const K& x);              // C++14
+    template<typename K>
+        const_iterator find(const K& x) const;  // C++14
+
+    template<typename K>
+      size_type count(const K& x) const;        // C++14
+    size_type      count(const key_type& k) const;
+
+    bool           contains(const key_type& x) const;  // C++20
+    template<class K> bool contains(const K& x) const; // C++20
+
+          iterator lower_bound(const key_type& k);
+    const_iterator lower_bound(const key_type& k) const;
+    template<typename K>
+        iterator lower_bound(const K& x);              // C++14
+    template<typename K>
+        const_iterator lower_bound(const K& x) const;  // C++14
+
+          iterator upper_bound(const key_type& k);
+    const_iterator upper_bound(const key_type& k) const;
+    template<typename K>
+        iterator upper_bound(const K& x);              // C++14
+    template<typename K>
+        const_iterator upper_bound(const K& x) const;  // C++14
+
+    pair<iterator,iterator>             equal_range(const key_type& k);
+    pair<const_iterator,const_iterator> equal_range(const key_type& k) const;
+    template<typename K>
+        pair<iterator,iterator>             equal_range(const K& x);        // C++14
+    template<typename K>
+        pair<const_iterator,const_iterator> equal_range(const K& x) const;  // C++14
+};
+
+template <class InputIterator,
+      class Compare = less<iter_key_t<InputIterator>>,
+      class Allocator = allocator<iter_to_alloc_t<InputIterator>>>
+map(InputIterator, InputIterator, Compare = Compare(), Allocator = Allocator())
+  -> map<iter_key_t<InputIterator>, iter_val_t<InputIterator>, Compare, Allocator>; // C++17
+
+template<ranges::input_range R, class Compare = less<range-key-type<R>,
+         class Allocator = allocator<range-to-alloc-type<R>>>
+  map(from_range_t, R&&, Compare = Compare(), Allocator = Allocator())
+    -> map<range-key-type<R>, range-mapped-type<R>, Compare, Allocator>; // C++23
+
+template<class Key, class T, class Compare = less<Key>,
+    class Allocator = allocator<pair<const Key, T>>>
+map(initializer_list<pair<const Key, T>>, Compare = Compare(), Allocator = Allocator())
+  -> map<Key, T, Compare, Allocator>; // C++17
+
+template <class InputIterator, class Allocator>
+map(InputIterator, InputIterator, Allocator)
+  -> map<iter_key_t<InputIterator>, iter_val_t<InputIterator>, less<iter_key_t<InputIterator>>,
+    Allocator>; // C++17
+
+template<ranges::input_range R, class Allocator>
+  map(from_range_t, R&&, Allocator)
+    -> map<range-key-type<R>, range-mapped-type<R>, less<range-key-type<R>>, Allocator>; // C++23
+
+template<class Key, class T, class Allocator>
+map(initializer_list<pair<const Key, T>>, Allocator) -> map<Key, T, less<Key>, Allocator>; // C++17
+
+template <class Key, class T, class Compare, class Allocator>
+bool
+operator==(const map<Key, T, Compare, Allocator>& x,
+           const map<Key, T, Compare, Allocator>& y);
+
+template <class Key, class T, class Compare, class Allocator>
+bool
+operator< (const map<Key, T, Compare, Allocator>& x,
+           const map<Key, T, Compare, Allocator>& y);      // removed in C++20
+
+template <class Key, class T, class Compare, class Allocator>
+bool
+operator!=(const map<Key, T, Compare, Allocator>& x,
+           const map<Key, T, Compare, Allocator>& y);      // removed in C++20
+
+template <class Key, class T, class Compare, class Allocator>
+bool
+operator> (const map<Key, T, Compare, Allocator>& x,
+           const map<Key, T, Compare, Allocator>& y);      // removed in C++20
+
+template <class Key, class T, class Compare, class Allocator>
+bool
+operator>=(const map<Key, T, Compare, Allocator>& x,
+           const map<Key, T, Compare, Allocator>& y);      // removed in C++20
+
+template <class Key, class T, class Compare, class Allocator>
+bool
+operator<=(const map<Key, T, Compare, Allocator>& x,
+           const map<Key, T, Compare, Allocator>& y);      // removed in C++20
+
+template<class Key, class T, class Compare, class Allocator>
+  synth-three-way-result<pair<const Key, T>>
+    operator<=>(const map<Key, T, Compare, Allocator>& x,
+                const map<Key, T, Compare, Allocator>& y); // since C++20
+
+// specialized algorithms:
+template <class Key, class T, class Compare, class Allocator>
+void
+swap(map<Key, T, Compare, Allocator>& x, map<Key, T, Compare, Allocator>& y)
+    noexcept(noexcept(x.swap(y)));
+
+template <class Key, class T, class Compare, class Allocator, class Predicate>
+typename map<Key, T, Compare, Allocator>::size_type
+erase_if(map<Key, T, Compare, Allocator>& c, Predicate pred);  // C++20
+
+
+template <class Key, class T, class Compare = less<Key>,
+          class Allocator = allocator<pair<const Key, T>>>
+class multimap
+{
+public:
+    // types:
+    typedef Key                                      key_type;
+    typedef T                                        mapped_type;
+    typedef pair<const key_type,mapped_type>         value_type;
+    typedef Compare                                  key_compare;
+    typedef Allocator                                allocator_type;
+    typedef typename allocator_type::reference       reference;
+    typedef typename allocator_type::const_reference const_reference;
+    typedef typename allocator_type::size_type       size_type;
+    typedef typename allocator_type::difference_type difference_type;
+    typedef typename allocator_type::pointer         pointer;
+    typedef typename allocator_type::const_pointer   const_pointer;
+
+    typedef implementation-defined                   iterator;
+    typedef implementation-defined                   const_iterator;
+    typedef std::reverse_iterator<iterator>          reverse_iterator;
+    typedef std::reverse_iterator<const_iterator>    const_reverse_iterator;
+    typedef unspecified                              node_type;              // C++17
+
+    class value_compare
+    {
+        friend class multimap;
+    protected:
+        key_compare comp;
+        value_compare(key_compare c);
+    public:
+        typedef bool result_type;  // deprecated in C++17, removed in C++20
+        typedef value_type first_argument_type;  // deprecated in C++17, removed in C++20
+        typedef value_type second_argument_type;  // deprecated in C++17, removed in C++20
+        bool operator()(const value_type& x, const value_type& y) const;
+    };
+
+    // construct/copy/destroy:
+    multimap()
+        noexcept(
+            is_nothrow_default_constructible<allocator_type>::value &&
+            is_nothrow_default_constructible<key_compare>::value &&
+            is_nothrow_copy_constructible<key_compare>::value);
+    explicit multimap(const key_compare& comp);
+    multimap(const key_compare& comp, const allocator_type& a);
+    template <class InputIterator>
+        multimap(InputIterator first, InputIterator last, const key_compare& comp);
+    template <class InputIterator>
+        multimap(InputIterator first, InputIterator last, const key_compare& comp,
+                 const allocator_type& a);
+    template<container-compatible-range<value_type> R>
+      multimap(from_range_t, R&& rg,
+               const Compare& comp = Compare(), const Allocator& = Allocator()); // C++23
+    multimap(const multimap& m);
+    multimap(multimap&& m)
+        noexcept(
+            is_nothrow_move_constructible<allocator_type>::value &&
+            is_nothrow_move_constructible<key_compare>::value);
+    explicit multimap(const allocator_type& a);
+    multimap(const multimap& m, const allocator_type& a);
+    multimap(multimap&& m, const allocator_type& a);
+    multimap(initializer_list<value_type> il, const key_compare& comp = key_compare());
+    multimap(initializer_list<value_type> il, const key_compare& comp,
+             const allocator_type& a);
+    template <class InputIterator>
+        multimap(InputIterator first, InputIterator last, const allocator_type& a)
+            : multimap(first, last, Compare(), a) {} // C++14
+    template<container-compatible-range<value_type> R>
+      multimap(from_range_t, R&& rg, const Allocator& a))
+        : multimap(from_range, std::forward<R>(rg), Compare(), a) { } // C++23
+    multimap(initializer_list<value_type> il, const allocator_type& a)
+        : multimap(il, Compare(), a) {} // C++14
+    ~multimap();
+
+    multimap& operator=(const multimap& m);
+    multimap& operator=(multimap&& m)
+        noexcept(
+            allocator_type::propagate_on_container_move_assignment::value &&
+            is_nothrow_move_assignable<allocator_type>::value &&
+            is_nothrow_move_assignable<key_compare>::value);
+    multimap& operator=(initializer_list<value_type> il);
+
+    // iterators:
+          iterator begin() noexcept;
+    const_iterator begin() const noexcept;
+          iterator end() noexcept;
+    const_iterator end()   const noexcept;
+
+          reverse_iterator rbegin() noexcept;
+    const_reverse_iterator rbegin() const noexcept;
+          reverse_iterator rend() noexcept;
+    const_reverse_iterator rend()   const noexcept;
+
+    const_iterator         cbegin()  const noexcept;
+    const_iterator         cend()    const noexcept;
+    const_reverse_iterator crbegin() const noexcept;
+    const_reverse_iterator crend()   const noexcept;
+
+    // capacity:
+    bool      empty()    const noexcept;
+    size_type size()     const noexcept;
+    size_type max_size() const noexcept;
+
+    // modifiers:
+    template <class... Args>
+        iterator emplace(Args&&... args);
+    template <class... Args>
+        iterator emplace_hint(const_iterator position, Args&&... args);
+    iterator insert(const value_type& v);
+    iterator insert(      value_type&& v);                                            // C++17
+    template <class P>
+        iterator insert(P&& p);
+    iterator insert(const_iterator position, const value_type& v);
+    iterator insert(const_iterator position,       value_type&& v);                   // C++17
+    template <class P>
+        iterator insert(const_iterator position, P&& p);
+    template <class InputIterator>
+        void insert(InputIterator first, InputIterator last);
+    template<container-compatible-range<value_type> R>
+      void insert_range(R&& rg);                                                      // C++23
+    void insert(initializer_list<value_type> il);
+
+    node_type extract(const_iterator position);                                       // C++17
+    node_type extract(const key_type& x);                                             // C++17
+    iterator insert(node_type&& nh);                                                  // C++17
+    iterator insert(const_iterator hint, node_type&& nh);                             // C++17
+
+    iterator  erase(const_iterator position);
+    iterator  erase(iterator position); // C++14
+    size_type erase(const key_type& k);
+    iterator  erase(const_iterator first, const_iterator last);
+    void clear() noexcept;
+
+    template<class C2>
+      void merge(multimap<Key, T, C2, Allocator>& source);    // C++17
+    template<class C2>
+      void merge(multimap<Key, T, C2, Allocator>&& source);   // C++17
+    template<class C2>
+      void merge(map<Key, T, C2, Allocator>& source);         // C++17
+    template<class C2>
+      void merge(map<Key, T, C2, Allocator>&& source);        // C++17
+
+    void swap(multimap& m)
+        noexcept(allocator_traits<allocator_type>::is_always_equal::value &&
+            is_nothrow_swappable<key_compare>::value); // C++17
+
+    // observers:
+    allocator_type get_allocator() const noexcept;
+    key_compare    key_comp()      const;
+    value_compare  value_comp()    const;
+
+    // map operations:
+          iterator find(const key_type& k);
+    const_iterator find(const key_type& k) const;
+    template<typename K>
+        iterator find(const K& x);              // C++14
+    template<typename K>
+        const_iterator find(const K& x) const;  // C++14
+
+    template<typename K>
+      size_type count(const K& x) const;        // C++14
+    size_type      count(const key_type& k) const;
+
+    bool           contains(const key_type& x) const;  // C++20
+    template<class K> bool contains(const K& x) const; // C++20
+
+          iterator lower_bound(const key_type& k);
+    const_iterator lower_bound(const key_type& k) const;
+    template<typename K>
+        iterator lower_bound(const K& x);              // C++14
+    template<typename K>
+        const_iterator lower_bound(const K& x) const;  // C++14
+
+          iterator upper_bound(const key_type& k);
+    const_iterator upper_bound(const key_type& k) const;
+    template<typename K>
+        iterator upper_bound(const K& x);              // C++14
+    template<typename K>
+        const_iterator upper_bound(const K& x) const;  // C++14
+
+    pair<iterator,iterator>             equal_range(const key_type& k);
+    pair<const_iterator,const_iterator> equal_range(const key_type& k) const;
+    template<typename K>
+        pair<iterator,iterator>             equal_range(const K& x);        // C++14
+    template<typename K>
+        pair<const_iterator,const_iterator> equal_range(const K& x) const;  // C++14
+};
+
+template <class InputIterator,
+      class Compare = less<iter_key_t<InputIterator>>,
+      class Allocator = allocator<iter_to_alloc_t<InputIterator>>>
+multimap(InputIterator, InputIterator, Compare = Compare(), Allocator = Allocator())
+  -> multimap<iter_key_t<InputIterator>, iter_val_t<InputIterator>, Compare, Allocator>; // C++17
+
+template<ranges::input_range R, class Compare = less<range-key-type<R>>,
+          class Allocator = allocator<range-to-alloc-type<R>>>
+  multimap(from_range_t, R&&, Compare = Compare(), Allocator = Allocator())
+    -> multimap<range-key-type<R>, range-mapped-type<R>, Compare, Allocator>; // C++23
+
+template<class Key, class T, class Compare = less<Key>,
+    class Allocator = allocator<pair<const Key, T>>>
+multimap(initializer_list<pair<const Key, T>>, Compare = Compare(), Allocator = Allocator())
+  -> multimap<Key, T, Compare, Allocator>; // C++17
+
+template <class InputIterator, class Allocator>
+multimap(InputIterator, InputIterator, Allocator)
+  -> multimap<iter_key_t<InputIterator>, iter_val_t<InputIterator>,
+    less<iter_key_t<InputIterator>>, Allocator>; // C++17
+
+template<ranges::input_range R, class Allocator>
+  multimap(from_range_t, R&&, Allocator)
+    -> multimap<range-key-type<R>, range-mapped-type<R>, less<range-key-type<R>>, Allocator>; // C++23
+
+template<class Key, class T, class Allocator>
+multimap(initializer_list<pair<const Key, T>>, Allocator)
+  -> multimap<Key, T, less<Key>, Allocator>; // C++17
+
+template <class Key, class T, class Compare, class Allocator>
+bool
+operator==(const multimap<Key, T, Compare, Allocator>& x,
+           const multimap<Key, T, Compare, Allocator>& y);
+
+template <class Key, class T, class Compare, class Allocator>
+bool
+operator< (const multimap<Key, T, Compare, Allocator>& x,
+           const multimap<Key, T, Compare, Allocator>& y);      // removed in C++20
+
+template <class Key, class T, class Compare, class Allocator>
+bool
+operator!=(const multimap<Key, T, Compare, Allocator>& x,
+           const multimap<Key, T, Compare, Allocator>& y);      // removed in C++20
+
+template <class Key, class T, class Compare, class Allocator>
+bool
+operator> (const multimap<Key, T, Compare, Allocator>& x,
+           const multimap<Key, T, Compare, Allocator>& y);      // removed in C++20
+
+template <class Key, class T, class Compare, class Allocator>
+bool
+operator>=(const multimap<Key, T, Compare, Allocator>& x,
+           const multimap<Key, T, Compare, Allocator>& y);      // removed in C++20
+
+template <class Key, class T, class Compare, class Allocator>
+bool
+operator<=(const multimap<Key, T, Compare, Allocator>& x,
+           const multimap<Key, T, Compare, Allocator>& y);      // removed in C++20
+
+template<class Key, class T, class Compare, class Allocator>
+  synth-three-way-result<pair<const Key, T>>
+    operator<=>(const multimap<Key, T, Compare, Allocator>& x,
+                const multimap<Key, T, Compare, Allocator>& y); // since c++20
+
+// specialized algorithms:
+template <class Key, class T, class Compare, class Allocator>
+void
+swap(multimap<Key, T, Compare, Allocator>& x,
+     multimap<Key, T, Compare, Allocator>& y)
+    noexcept(noexcept(x.swap(y)));
+
+template <class Key, class T, class Compare, class Allocator, class Predicate>
+typename multimap<Key, T, Compare, Allocator>::size_type
+erase_if(multimap<Key, T, Compare, Allocator>& c, Predicate pred);  // C++20
+
+}  // std
+
+*/
+
+#include <__algorithm/equal.h>
+#include <__algorithm/lexicographical_compare.h>
+#include <__algorithm/lexicographical_compare_three_way.h>
+#include <__assert>
+#include <__config>
+#include <__functional/binary_function.h>
+#include <__functional/is_transparent.h>
+#include <__functional/operations.h>
+#include <__iterator/erase_if_container.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/ranges_iterator_traits.h>
+#include <__iterator/reverse_iterator.h>
+#include <__memory/addressof.h>
+#include <__memory/allocator.h>
+#include <__memory_resource/polymorphic_allocator.h>
+#include <__node_handle>
+#include <__ranges/concepts.h>
+#include <__ranges/container_compatible_range.h>
+#include <__ranges/from_range.h>
+#include <__tree>
+#include <__type_traits/is_allocator.h>
+#include <__utility/forward.h>
+#include <__utility/piecewise_construct.h>
+#include <__utility/swap.h>
+#include <stdexcept>
+#include <tuple>
+#include <version>
+
+// standard-mandated includes
+
+// [iterator.range]
+#include <__iterator/access.h>
+#include <__iterator/data.h>
+#include <__iterator/empty.h>
+#include <__iterator/reverse_access.h>
+#include <__iterator/size.h>
+
+// [associative.map.syn]
+#include <compare>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Key,
+          class _CP,
+          class _Compare,
+          bool = is_empty<_Compare>::value && !__libcpp_is_final<_Compare>::value>
+class __map_value_compare : private _Compare {
+public:
+  _LIBCPP_HIDE_FROM_ABI __map_value_compare() _NOEXCEPT_(is_nothrow_default_constructible<_Compare>::value)
+      : _Compare() {}
+  _LIBCPP_HIDE_FROM_ABI __map_value_compare(_Compare __c) _NOEXCEPT_(is_nothrow_copy_constructible<_Compare>::value)
+      : _Compare(__c) {}
+  _LIBCPP_HIDE_FROM_ABI const _Compare& key_comp() const _NOEXCEPT { return *this; }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _CP& __y) const {
+    return static_cast<const _Compare&>(*this)(__x.__get_value().first, __y.__get_value().first);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _Key& __y) const {
+    return static_cast<const _Compare&>(*this)(__x.__get_value().first, __y);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _CP& __y) const {
+    return static_cast<const _Compare&>(*this)(__x, __y.__get_value().first);
+  }
+  _LIBCPP_HIDE_FROM_ABI void swap(__map_value_compare& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Compare>) {
+    using std::swap;
+    swap(static_cast<_Compare&>(*this), static_cast<_Compare&>(__y));
+  }
+
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _CP& __y) const {
+    return static_cast<const _Compare&>(*this)(__x, __y.__get_value().first);
+  }
+
+  template <typename _K2>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _K2& __y) const {
+    return static_cast<const _Compare&>(*this)(__x.__get_value().first, __y);
+  }
+#endif
+};
+
+template <class _Key, class _CP, class _Compare>
+class __map_value_compare<_Key, _CP, _Compare, false> {
+  _Compare __comp_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __map_value_compare() _NOEXCEPT_(is_nothrow_default_constructible<_Compare>::value)
+      : __comp_() {}
+  _LIBCPP_HIDE_FROM_ABI __map_value_compare(_Compare __c) _NOEXCEPT_(is_nothrow_copy_constructible<_Compare>::value)
+      : __comp_(__c) {}
+  _LIBCPP_HIDE_FROM_ABI const _Compare& key_comp() const _NOEXCEPT { return __comp_; }
+
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _CP& __y) const {
+    return __comp_(__x.__get_value().first, __y.__get_value().first);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _Key& __y) const {
+    return __comp_(__x.__get_value().first, __y);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _CP& __y) const {
+    return __comp_(__x, __y.__get_value().first);
+  }
+  void swap(__map_value_compare& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Compare>) {
+    using std::swap;
+    swap(__comp_, __y.__comp_);
+  }
+
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _CP& __y) const {
+    return __comp_(__x, __y.__get_value().first);
+  }
+
+  template <typename _K2>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _K2& __y) const {
+    return __comp_(__x.__get_value().first, __y);
+  }
+#endif
+};
+
+template <class _Key, class _CP, class _Compare, bool __b>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(__map_value_compare<_Key, _CP, _Compare, __b>& __x, __map_value_compare<_Key, _CP, _Compare, __b>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+template <class _Allocator>
+class __map_node_destructor {
+  typedef _Allocator allocator_type;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+
+public:
+  typedef typename __alloc_traits::pointer pointer;
+
+private:
+  allocator_type& __na_;
+
+public:
+  bool __first_constructed;
+  bool __second_constructed;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __map_node_destructor(allocator_type& __na) _NOEXCEPT
+      : __na_(__na),
+        __first_constructed(false),
+        __second_constructed(false) {}
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI __map_node_destructor(__tree_node_destructor<allocator_type>&& __x) _NOEXCEPT
+      : __na_(__x.__na_),
+        __first_constructed(__x.__value_constructed),
+        __second_constructed(__x.__value_constructed) {
+    __x.__value_constructed = false;
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  __map_node_destructor& operator=(const __map_node_destructor&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) _NOEXCEPT {
+    if (__second_constructed)
+      __alloc_traits::destroy(__na_, std::addressof(__p->__value_.__get_value().second));
+    if (__first_constructed)
+      __alloc_traits::destroy(__na_, std::addressof(__p->__value_.__get_value().first));
+    if (__p)
+      __alloc_traits::deallocate(__na_, __p, 1);
+  }
+};
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+class map;
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+class multimap;
+template <class _TreeIterator>
+class __map_const_iterator;
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Key, class _Tp>
+struct _LIBCPP_STANDALONE_DEBUG __value_type {
+  typedef _Key key_type;
+  typedef _Tp mapped_type;
+  typedef pair<const key_type, mapped_type> value_type;
+  typedef pair<key_type&, mapped_type&> __nc_ref_pair_type;
+  typedef pair<key_type&&, mapped_type&&> __nc_rref_pair_type;
+
+private:
+  value_type __cc_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI value_type& __get_value() {
+#  if _LIBCPP_STD_VER >= 17
+    return *std::launder(std::addressof(__cc_));
+#  else
+    return __cc_;
+#  endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const value_type& __get_value() const {
+#  if _LIBCPP_STD_VER >= 17
+    return *std::launder(std::addressof(__cc_));
+#  else
+    return __cc_;
+#  endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __nc_ref_pair_type __ref() {
+    value_type& __v = __get_value();
+    return __nc_ref_pair_type(const_cast<key_type&>(__v.first), __v.second);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __nc_rref_pair_type __move() {
+    value_type& __v = __get_value();
+    return __nc_rref_pair_type(std::move(const_cast<key_type&>(__v.first)), std::move(__v.second));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __value_type& operator=(const __value_type& __v) {
+    __ref() = __v.__get_value();
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __value_type& operator=(__value_type&& __v) {
+    __ref() = __v.__move();
+    return *this;
+  }
+
+  template <class _ValueTp, __enable_if_t<__is_same_uncvref<_ValueTp, value_type>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI __value_type& operator=(_ValueTp&& __v) {
+    __ref() = std::forward<_ValueTp>(__v);
+    return *this;
+  }
+
+  __value_type()                    = delete;
+  ~__value_type()                   = delete;
+  __value_type(const __value_type&) = delete;
+  __value_type(__value_type&&)      = delete;
+};
+
+#else
+
+template <class _Key, class _Tp>
+struct __value_type {
+  typedef _Key key_type;
+  typedef _Tp mapped_type;
+  typedef pair<const key_type, mapped_type> value_type;
+
+private:
+  value_type __cc_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI value_type& __get_value() { return __cc_; }
+  _LIBCPP_HIDE_FROM_ABI const value_type& __get_value() const { return __cc_; }
+
+  __value_type()                               = delete;
+  __value_type(__value_type const&)            = delete;
+  __value_type& operator=(__value_type const&) = delete;
+  ~__value_type()                              = delete;
+};
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp>
+struct __extract_key_value_types;
+
+template <class _Key, class _Tp>
+struct __extract_key_value_types<__value_type<_Key, _Tp> > {
+  typedef _Key const __key_type;
+  typedef _Tp __mapped_type;
+};
+
+template <class _TreeIterator>
+class _LIBCPP_TEMPLATE_VIS __map_iterator {
+  typedef typename _TreeIterator::_NodeTypes _NodeTypes;
+  typedef typename _TreeIterator::__pointer_traits __pointer_traits;
+
+  _TreeIterator __i_;
+
+public:
+  typedef bidirectional_iterator_tag iterator_category;
+  typedef typename _NodeTypes::__map_value_type value_type;
+  typedef typename _TreeIterator::difference_type difference_type;
+  typedef value_type& reference;
+  typedef typename _NodeTypes::__map_value_type_pointer pointer;
+
+  _LIBCPP_HIDE_FROM_ABI __map_iterator() _NOEXCEPT {}
+
+  _LIBCPP_HIDE_FROM_ABI __map_iterator(_TreeIterator __i) _NOEXCEPT : __i_(__i) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __i_->__get_value(); }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return pointer_traits<pointer>::pointer_to(__i_->__get_value()); }
+
+  _LIBCPP_HIDE_FROM_ABI __map_iterator& operator++() {
+    ++__i_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __map_iterator operator++(int) {
+    __map_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __map_iterator& operator--() {
+    --__i_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __map_iterator operator--(int) {
+    __map_iterator __t(*this);
+    --(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __map_iterator& __x, const __map_iterator& __y) {
+    return __x.__i_ == __y.__i_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __map_iterator& __x, const __map_iterator& __y) {
+    return __x.__i_ != __y.__i_;
+  }
+
+  template <class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS map;
+  template <class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS multimap;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __map_const_iterator;
+};
+
+template <class _TreeIterator>
+class _LIBCPP_TEMPLATE_VIS __map_const_iterator {
+  typedef typename _TreeIterator::_NodeTypes _NodeTypes;
+  typedef typename _TreeIterator::__pointer_traits __pointer_traits;
+
+  _TreeIterator __i_;
+
+public:
+  typedef bidirectional_iterator_tag iterator_category;
+  typedef typename _NodeTypes::__map_value_type value_type;
+  typedef typename _TreeIterator::difference_type difference_type;
+  typedef const value_type& reference;
+  typedef typename _NodeTypes::__const_map_value_type_pointer pointer;
+
+  _LIBCPP_HIDE_FROM_ABI __map_const_iterator() _NOEXCEPT {}
+
+  _LIBCPP_HIDE_FROM_ABI __map_const_iterator(_TreeIterator __i) _NOEXCEPT : __i_(__i) {}
+  _LIBCPP_HIDE_FROM_ABI
+  __map_const_iterator(__map_iterator< typename _TreeIterator::__non_const_iterator> __i) _NOEXCEPT : __i_(__i.__i_) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __i_->__get_value(); }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return pointer_traits<pointer>::pointer_to(__i_->__get_value()); }
+
+  _LIBCPP_HIDE_FROM_ABI __map_const_iterator& operator++() {
+    ++__i_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __map_const_iterator operator++(int) {
+    __map_const_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __map_const_iterator& operator--() {
+    --__i_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __map_const_iterator operator--(int) {
+    __map_const_iterator __t(*this);
+    --(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __map_const_iterator& __x, const __map_const_iterator& __y) {
+    return __x.__i_ == __y.__i_;
+  }
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __map_const_iterator& __x, const __map_const_iterator& __y) {
+    return __x.__i_ != __y.__i_;
+  }
+
+  template <class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS map;
+  template <class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS multimap;
+  template <class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS __tree_const_iterator;
+};
+
+template <class _Key, class _Tp, class _Compare = less<_Key>, class _Allocator = allocator<pair<const _Key, _Tp> > >
+class _LIBCPP_TEMPLATE_VIS map {
+public:
+  // types:
+  typedef _Key key_type;
+  typedef _Tp mapped_type;
+  typedef pair<const key_type, mapped_type> value_type;
+  typedef __type_identity_t<_Compare> key_compare;
+  typedef __type_identity_t<_Allocator> allocator_type;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+
+  static_assert(is_same<typename allocator_type::value_type, value_type>::value,
+                "Allocator::value_type must be same type as value_type");
+
+  class _LIBCPP_TEMPLATE_VIS value_compare : public __binary_function<value_type, value_type, bool> {
+    friend class map;
+
+  protected:
+    key_compare comp;
+
+    _LIBCPP_HIDE_FROM_ABI value_compare(key_compare __c) : comp(__c) {}
+
+  public:
+    _LIBCPP_HIDE_FROM_ABI bool operator()(const value_type& __x, const value_type& __y) const {
+      return comp(__x.first, __y.first);
+    }
+  };
+
+private:
+  typedef std::__value_type<key_type, mapped_type> __value_type;
+  typedef __map_value_compare<key_type, __value_type, key_compare> __vc;
+  typedef __rebind_alloc<allocator_traits<allocator_type>, __value_type> __allocator_type;
+  typedef __tree<__value_type, __vc, __allocator_type> __base;
+  typedef typename __base::__node_traits __node_traits;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+
+  static_assert(__check_valid_allocator<allocator_type>::value, "");
+
+  __base __tree_;
+
+public:
+  typedef typename __alloc_traits::pointer pointer;
+  typedef typename __alloc_traits::const_pointer const_pointer;
+  typedef typename __alloc_traits::size_type size_type;
+  typedef typename __alloc_traits::difference_type difference_type;
+  typedef __map_iterator<typename __base::iterator> iterator;
+  typedef __map_const_iterator<typename __base::const_iterator> const_iterator;
+  typedef std::reverse_iterator<iterator> reverse_iterator;
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+
+#if _LIBCPP_STD_VER >= 17
+  typedef __map_node_handle<typename __base::__node, allocator_type> node_type;
+  typedef __insert_return_type<iterator, node_type> insert_return_type;
+#endif
+
+  template <class _Key2, class _Value2, class _Comp2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS map;
+  template <class _Key2, class _Value2, class _Comp2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS multimap;
+
+  _LIBCPP_HIDE_FROM_ABI map() _NOEXCEPT_(
+      is_nothrow_default_constructible<allocator_type>::value&& is_nothrow_default_constructible<key_compare>::value&&
+          is_nothrow_copy_constructible<key_compare>::value)
+      : __tree_(__vc(key_compare())) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit map(const key_compare& __comp) _NOEXCEPT_(
+      is_nothrow_default_constructible<allocator_type>::value&& is_nothrow_copy_constructible<key_compare>::value)
+      : __tree_(__vc(__comp)) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit map(const key_compare& __comp, const allocator_type& __a)
+      : __tree_(__vc(__comp), typename __base::allocator_type(__a)) {}
+
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI map(_InputIterator __f, _InputIterator __l, const key_compare& __comp = key_compare())
+      : __tree_(__vc(__comp)) {
+    insert(__f, __l);
+  }
+
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI
+  map(_InputIterator __f, _InputIterator __l, const key_compare& __comp, const allocator_type& __a)
+      : __tree_(__vc(__comp), typename __base::allocator_type(__a)) {
+    insert(__f, __l);
+  }
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI
+  map(from_range_t,
+      _Range&& __range,
+      const key_compare& __comp = key_compare(),
+      const allocator_type& __a = allocator_type())
+      : __tree_(__vc(__comp), typename __base::allocator_type(__a)) {
+    insert_range(std::forward<_Range>(__range));
+  }
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI map(_InputIterator __f, _InputIterator __l, const allocator_type& __a)
+      : map(__f, __l, key_compare(), __a) {}
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI map(from_range_t, _Range&& __range, const allocator_type& __a)
+      : map(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI map(const map& __m) : __tree_(__m.__tree_) { insert(__m.begin(), __m.end()); }
+
+  _LIBCPP_HIDE_FROM_ABI map& operator=(const map& __m) {
+#ifndef _LIBCPP_CXX03_LANG
+    __tree_ = __m.__tree_;
+#else
+    if (this != std::addressof(__m)) {
+      __tree_.clear();
+      __tree_.value_comp() = __m.__tree_.value_comp();
+      __tree_.__copy_assign_alloc(__m.__tree_);
+      insert(__m.begin(), __m.end());
+    }
+#endif
+    return *this;
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI map(map&& __m) noexcept(is_nothrow_move_constructible<__base>::value)
+      : __tree_(std::move(__m.__tree_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI map(map&& __m, const allocator_type& __a);
+
+  _LIBCPP_HIDE_FROM_ABI map& operator=(map&& __m) noexcept(is_nothrow_move_assignable<__base>::value) {
+    __tree_ = std::move(__m.__tree_);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI map(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
+      : __tree_(__vc(__comp)) {
+    insert(__il.begin(), __il.end());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI map(initializer_list<value_type> __il, const key_compare& __comp, const allocator_type& __a)
+      : __tree_(__vc(__comp), typename __base::allocator_type(__a)) {
+    insert(__il.begin(), __il.end());
+  }
+
+#  if _LIBCPP_STD_VER >= 14
+  _LIBCPP_HIDE_FROM_ABI map(initializer_list<value_type> __il, const allocator_type& __a)
+      : map(__il, key_compare(), __a) {}
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI map& operator=(initializer_list<value_type> __il) {
+    __tree_.__assign_unique(__il.begin(), __il.end());
+    return *this;
+  }
+
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI explicit map(const allocator_type& __a) : __tree_(typename __base::allocator_type(__a)) {}
+
+  _LIBCPP_HIDE_FROM_ABI map(const map& __m, const allocator_type& __a)
+      : __tree_(__m.__tree_.value_comp(), typename __base::allocator_type(__a)) {
+    insert(__m.begin(), __m.end());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~map() { static_assert(sizeof(__diagnose_non_const_comparator<_Key, _Compare>()), ""); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __tree_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __tree_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __tree_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __tree_.end(); }
+
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return rbegin(); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __tree_.size() == 0; }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __tree_.size(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __tree_.max_size(); }
+
+  _LIBCPP_HIDE_FROM_ABI mapped_type& operator[](const key_type& __k);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI mapped_type& operator[](key_type&& __k);
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI mapped_type& at(const key_type& __k);
+  _LIBCPP_HIDE_FROM_ABI const mapped_type& at(const key_type& __k) const;
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT { return allocator_type(__tree_.__alloc()); }
+  _LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __tree_.value_comp().key_comp(); }
+  _LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return value_compare(__tree_.value_comp().key_comp()); }
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> emplace(_Args&&... __args) {
+    return __tree_.__emplace_unique(std::forward<_Args>(__args)...);
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __p, _Args&&... __args) {
+    return __tree_.__emplace_hint_unique(__p.__i_, std::forward<_Args>(__args)...);
+  }
+
+  template <class _Pp, __enable_if_t<is_constructible<value_type, _Pp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(_Pp&& __p) {
+    return __tree_.__insert_unique(std::forward<_Pp>(__p));
+  }
+
+  template <class _Pp, __enable_if_t<is_constructible<value_type, _Pp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __pos, _Pp&& __p) {
+    return __tree_.__insert_unique(__pos.__i_, std::forward<_Pp>(__p));
+  }
+
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(const value_type& __v) { return __tree_.__insert_unique(__v); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __v) {
+    return __tree_.__insert_unique(__p.__i_, __v);
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(value_type&& __v) {
+    return __tree_.__insert_unique(std::move(__v));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __v) {
+    return __tree_.__insert_unique(__p.__i_, std::move(__v));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
+#endif
+
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __f, _InputIterator __l) {
+    for (const_iterator __e = cend(); __f != __l; ++__f)
+      insert(__e.__i_, *__f);
+  }
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
+    const_iterator __end = cend();
+    for (auto&& __element : __range) {
+      insert(__end.__i_, std::forward<decltype(__element)>(__element));
+    }
+  }
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> try_emplace(const key_type& __k, _Args&&... __args) {
+    return __tree_.__emplace_unique_key_args(
+        __k,
+        std::piecewise_construct,
+        std::forward_as_tuple(__k),
+        std::forward_as_tuple(std::forward<_Args>(__args)...));
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> try_emplace(key_type&& __k, _Args&&... __args) {
+    return __tree_.__emplace_unique_key_args(
+        __k,
+        std::piecewise_construct,
+        std::forward_as_tuple(std::move(__k)),
+        std::forward_as_tuple(std::forward<_Args>(__args)...));
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator try_emplace(const_iterator __h, const key_type& __k, _Args&&... __args) {
+    return __tree_
+        .__emplace_hint_unique_key_args(
+            __h.__i_,
+            __k,
+            std::piecewise_construct,
+            std::forward_as_tuple(__k),
+            std::forward_as_tuple(std::forward<_Args>(__args)...))
+        .first;
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator try_emplace(const_iterator __h, key_type&& __k, _Args&&... __args) {
+    return __tree_
+        .__emplace_hint_unique_key_args(
+            __h.__i_,
+            __k,
+            std::piecewise_construct,
+            std::forward_as_tuple(std::move(__k)),
+            std::forward_as_tuple(std::forward<_Args>(__args)...))
+        .first;
+  }
+
+  template <class _Vp>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert_or_assign(const key_type& __k, _Vp&& __v) {
+    iterator __p = lower_bound(__k);
+    if (__p != end() && !key_comp()(__k, __p->first)) {
+      __p->second = std::forward<_Vp>(__v);
+      return std::make_pair(__p, false);
+    }
+    return std::make_pair(emplace_hint(__p, __k, std::forward<_Vp>(__v)), true);
+  }
+
+  template <class _Vp>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert_or_assign(key_type&& __k, _Vp&& __v) {
+    iterator __p = lower_bound(__k);
+    if (__p != end() && !key_comp()(__k, __p->first)) {
+      __p->second = std::forward<_Vp>(__v);
+      return std::make_pair(__p, false);
+    }
+    return std::make_pair(emplace_hint(__p, std::move(__k), std::forward<_Vp>(__v)), true);
+  }
+
+  template <class _Vp>
+  _LIBCPP_HIDE_FROM_ABI iterator insert_or_assign(const_iterator __h, const key_type& __k, _Vp&& __v) {
+    auto [__r, __inserted] = __tree_.__emplace_hint_unique_key_args(__h.__i_, __k, __k, std::forward<_Vp>(__v));
+
+    if (!__inserted)
+      __r->__get_value().second = std::forward<_Vp>(__v);
+
+    return __r;
+  }
+
+  template <class _Vp>
+  _LIBCPP_HIDE_FROM_ABI iterator insert_or_assign(const_iterator __h, key_type&& __k, _Vp&& __v) {
+    auto [__r, __inserted] =
+        __tree_.__emplace_hint_unique_key_args(__h.__i_, __k, std::move(__k), std::forward<_Vp>(__v));
+
+    if (!__inserted)
+      __r->__get_value().second = std::forward<_Vp>(__v);
+
+    return __r;
+  }
+
+#endif // _LIBCPP_STD_VER >= 17
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __tree_.erase(__p.__i_); }
+  _LIBCPP_HIDE_FROM_ABI iterator erase(iterator __p) { return __tree_.erase(__p.__i_); }
+  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __tree_.__erase_unique(__k); }
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __f, const_iterator __l) {
+    return __tree_.erase(__f.__i_, __l.__i_);
+  }
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __tree_.clear(); }
+
+#if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI insert_return_type insert(node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to map::insert()");
+    return __tree_.template __node_handle_insert_unique< node_type, insert_return_type>(std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to map::insert()");
+    return __tree_.template __node_handle_insert_unique<node_type>(__hint.__i_, std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
+    return __tree_.template __node_handle_extract<node_type>(__key);
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
+    return __tree_.template __node_handle_extract<node_type>(__it.__i_);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(map<key_type, mapped_type, _Compare2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __tree_.__node_handle_merge_unique(__source.__tree_);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(map<key_type, mapped_type, _Compare2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __tree_.__node_handle_merge_unique(__source.__tree_);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(multimap<key_type, mapped_type, _Compare2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __tree_.__node_handle_merge_unique(__source.__tree_);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(multimap<key_type, mapped_type, _Compare2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __tree_.__node_handle_merge_unique(__source.__tree_);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI void swap(map& __m) _NOEXCEPT_(__is_nothrow_swappable_v<__base>) { __tree_.swap(__m.__tree_); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
+    return __tree_.find(__k);
+  }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
+    return __tree_.find(__k);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_unique(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
+    return __tree_.__count_multi(__k);
+  }
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
+    return find(__k) != end();
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
+    return __tree_.lower_bound(__k);
+  }
+
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
+    return __tree_.lower_bound(__k);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
+    return __tree_.upper_bound(__k);
+  }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
+    return __tree_.upper_bound(__k);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
+    return __tree_.__equal_range_unique(__k);
+  }
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
+    return __tree_.__equal_range_unique(__k);
+  }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
+    return __tree_.__equal_range_multi(__k);
+  }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
+    return __tree_.__equal_range_multi(__k);
+  }
+#endif
+
+private:
+  typedef typename __base::__node __node;
+  typedef typename __base::__node_allocator __node_allocator;
+  typedef typename __base::__node_pointer __node_pointer;
+  typedef typename __base::__node_base_pointer __node_base_pointer;
+  typedef typename __base::__parent_pointer __parent_pointer;
+
+  typedef __map_node_destructor<__node_allocator> _Dp;
+  typedef unique_ptr<__node, _Dp> __node_holder;
+
+#ifdef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI __node_holder __construct_node_with_key(const key_type& __k);
+#endif
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator,
+          class _Compare   = less<__iter_key_type<_InputIterator>>,
+          class _Allocator = allocator<__iter_to_alloc_type<_InputIterator>>,
+          class            = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
+          class            = enable_if_t<!__is_allocator<_Compare>::value, void>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value, void>>
+map(_InputIterator, _InputIterator, _Compare = _Compare(), _Allocator = _Allocator())
+    -> map<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Compare, _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range,
+          class _Compare   = less<__range_key_type<_Range>>,
+          class _Allocator = allocator<__range_to_alloc_type<_Range>>,
+          class            = enable_if_t<!__is_allocator<_Compare>::value, void>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value, void>>
+map(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator())
+    -> map<__range_key_type<_Range>, __range_mapped_type<_Range>, _Compare, _Allocator>;
+#  endif
+
+template <class _Key,
+          class _Tp,
+          class _Compare   = less<remove_const_t<_Key>>,
+          class _Allocator = allocator<pair<const _Key, _Tp>>,
+          class            = enable_if_t<!__is_allocator<_Compare>::value, void>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value, void>>
+map(initializer_list<pair<_Key, _Tp>>,
+    _Compare   = _Compare(),
+    _Allocator = _Allocator()) -> map<remove_const_t<_Key>, _Tp, _Compare, _Allocator>;
+
+template <class _InputIterator,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
+          class = enable_if_t<__is_allocator<_Allocator>::value, void>>
+map(_InputIterator, _InputIterator, _Allocator)
+    -> map<__iter_key_type<_InputIterator>,
+           __iter_mapped_type<_InputIterator>,
+           less<__iter_key_type<_InputIterator>>,
+           _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
+map(from_range_t, _Range&&, _Allocator)
+    -> map<__range_key_type<_Range>, __range_mapped_type<_Range>, less<__range_key_type<_Range>>, _Allocator>;
+#  endif
+
+template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
+map(initializer_list<pair<_Key, _Tp>>,
+    _Allocator) -> map<remove_const_t<_Key>, _Tp, less<remove_const_t<_Key>>, _Allocator>;
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+map<_Key, _Tp, _Compare, _Allocator>::map(map&& __m, const allocator_type& __a)
+    : __tree_(std::move(__m.__tree_), typename __base::allocator_type(__a)) {
+  if (__a != __m.get_allocator()) {
+    const_iterator __e = cend();
+    while (!__m.empty())
+      __tree_.__insert_unique(__e.__i_, __m.__tree_.remove(__m.begin().__i_)->__value_.__move());
+  }
+}
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+_Tp& map<_Key, _Tp, _Compare, _Allocator>::operator[](const key_type& __k) {
+  return __tree_
+      .__emplace_unique_key_args(__k, std::piecewise_construct, std::forward_as_tuple(__k), std::forward_as_tuple())
+      .first->__get_value()
+      .second;
+}
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+_Tp& map<_Key, _Tp, _Compare, _Allocator>::operator[](key_type&& __k) {
+  // TODO investigate this clang-tidy warning.
+  // NOLINTBEGIN(bugprone-use-after-move)
+  return __tree_
+      .__emplace_unique_key_args(
+          __k, std::piecewise_construct, std::forward_as_tuple(std::move(__k)), std::forward_as_tuple())
+      .first->__get_value()
+      .second;
+  // NOLINTEND(bugprone-use-after-move)
+}
+
+#else // _LIBCPP_CXX03_LANG
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+typename map<_Key, _Tp, _Compare, _Allocator>::__node_holder
+map<_Key, _Tp, _Compare, _Allocator>::__construct_node_with_key(const key_type& __k) {
+  __node_allocator& __na = __tree_.__node_alloc();
+  __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
+  __node_traits::construct(__na, std::addressof(__h->__value_.__get_value().first), __k);
+  __h.get_deleter().__first_constructed = true;
+  __node_traits::construct(__na, std::addressof(__h->__value_.__get_value().second));
+  __h.get_deleter().__second_constructed = true;
+  return __h;
+}
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+_Tp& map<_Key, _Tp, _Compare, _Allocator>::operator[](const key_type& __k) {
+  __parent_pointer __parent;
+  __node_base_pointer& __child = __tree_.__find_equal(__parent, __k);
+  __node_pointer __r           = static_cast<__node_pointer>(__child);
+  if (__child == nullptr) {
+    __node_holder __h = __construct_node_with_key(__k);
+    __tree_.__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
+    __r = __h.release();
+  }
+  return __r->__value_.__get_value().second;
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+_Tp& map<_Key, _Tp, _Compare, _Allocator>::at(const key_type& __k) {
+  __parent_pointer __parent;
+  __node_base_pointer& __child = __tree_.__find_equal(__parent, __k);
+  if (__child == nullptr)
+    __throw_out_of_range("map::at:  key not found");
+  return static_cast<__node_pointer>(__child)->__value_.__get_value().second;
+}
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+const _Tp& map<_Key, _Tp, _Compare, _Allocator>::at(const key_type& __k) const {
+  __parent_pointer __parent;
+  __node_base_pointer __child = __tree_.__find_equal(__parent, __k);
+  if (__child == nullptr)
+    __throw_out_of_range("map::at:  key not found");
+  return static_cast<__node_pointer>(__child)->__value_.__get_value().second;
+}
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const map<_Key, _Tp, _Compare, _Allocator>& __x, const map<_Key, _Tp, _Compare, _Allocator>& __y) {
+  return __x.size() == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<(const map<_Key, _Tp, _Compare, _Allocator>& __x, const map<_Key, _Tp, _Compare, _Allocator>& __y) {
+  return std::lexicographical_compare(__x.begin(), __x.end(), __y.begin(), __y.end());
+}
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const map<_Key, _Tp, _Compare, _Allocator>& __x, const map<_Key, _Tp, _Compare, _Allocator>& __y) {
+  return !(__x == __y);
+}
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>(const map<_Key, _Tp, _Compare, _Allocator>& __x, const map<_Key, _Tp, _Compare, _Allocator>& __y) {
+  return __y < __x;
+}
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>=(const map<_Key, _Tp, _Compare, _Allocator>& __x, const map<_Key, _Tp, _Compare, _Allocator>& __y) {
+  return !(__x < __y);
+}
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<=(const map<_Key, _Tp, _Compare, _Allocator>& __x, const map<_Key, _Tp, _Compare, _Allocator>& __y) {
+  return !(__y < __x);
+}
+
+#else // #if _LIBCPP_STD_VER <= 17
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI __synth_three_way_result<pair<const _Key, _Tp>>
+operator<=>(const map<_Key, _Tp, _Compare, _Allocator>& __x, const map<_Key, _Tp, _Compare, _Allocator>& __y) {
+  return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
+}
+
+#endif // #if _LIBCPP_STD_VER <= 17
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(map<_Key, _Tp, _Compare, _Allocator>& __x, map<_Key, _Tp, _Compare, _Allocator>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Key, class _Tp, class _Compare, class _Allocator, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI typename map<_Key, _Tp, _Compare, _Allocator>::size_type
+erase_if(map<_Key, _Tp, _Compare, _Allocator>& __c, _Predicate __pred) {
+  return std::__libcpp_erase_if_container(__c, __pred);
+}
+#endif
+
+template <class _Key, class _Tp, class _Compare = less<_Key>, class _Allocator = allocator<pair<const _Key, _Tp> > >
+class _LIBCPP_TEMPLATE_VIS multimap {
+public:
+  // types:
+  typedef _Key key_type;
+  typedef _Tp mapped_type;
+  typedef pair<const key_type, mapped_type> value_type;
+  typedef __type_identity_t<_Compare> key_compare;
+  typedef __type_identity_t<_Allocator> allocator_type;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+
+  static_assert(__check_valid_allocator<allocator_type>::value, "");
+  static_assert(is_same<typename allocator_type::value_type, value_type>::value,
+                "Allocator::value_type must be same type as value_type");
+
+  class _LIBCPP_TEMPLATE_VIS value_compare : public __binary_function<value_type, value_type, bool> {
+    friend class multimap;
+
+  protected:
+    key_compare comp;
+
+    _LIBCPP_HIDE_FROM_ABI value_compare(key_compare __c) : comp(__c) {}
+
+  public:
+    _LIBCPP_HIDE_FROM_ABI bool operator()(const value_type& __x, const value_type& __y) const {
+      return comp(__x.first, __y.first);
+    }
+  };
+
+private:
+  typedef std::__value_type<key_type, mapped_type> __value_type;
+  typedef __map_value_compare<key_type, __value_type, key_compare> __vc;
+  typedef __rebind_alloc<allocator_traits<allocator_type>, __value_type> __allocator_type;
+  typedef __tree<__value_type, __vc, __allocator_type> __base;
+  typedef typename __base::__node_traits __node_traits;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+
+  __base __tree_;
+
+public:
+  typedef typename __alloc_traits::pointer pointer;
+  typedef typename __alloc_traits::const_pointer const_pointer;
+  typedef typename __alloc_traits::size_type size_type;
+  typedef typename __alloc_traits::difference_type difference_type;
+  typedef __map_iterator<typename __base::iterator> iterator;
+  typedef __map_const_iterator<typename __base::const_iterator> const_iterator;
+  typedef std::reverse_iterator<iterator> reverse_iterator;
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+
+#if _LIBCPP_STD_VER >= 17
+  typedef __map_node_handle<typename __base::__node, allocator_type> node_type;
+#endif
+
+  template <class _Key2, class _Value2, class _Comp2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS map;
+  template <class _Key2, class _Value2, class _Comp2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS multimap;
+
+  _LIBCPP_HIDE_FROM_ABI multimap() _NOEXCEPT_(
+      is_nothrow_default_constructible<allocator_type>::value&& is_nothrow_default_constructible<key_compare>::value&&
+          is_nothrow_copy_constructible<key_compare>::value)
+      : __tree_(__vc(key_compare())) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit multimap(const key_compare& __comp) _NOEXCEPT_(
+      is_nothrow_default_constructible<allocator_type>::value&& is_nothrow_copy_constructible<key_compare>::value)
+      : __tree_(__vc(__comp)) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit multimap(const key_compare& __comp, const allocator_type& __a)
+      : __tree_(__vc(__comp), typename __base::allocator_type(__a)) {}
+
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI multimap(_InputIterator __f, _InputIterator __l, const key_compare& __comp = key_compare())
+      : __tree_(__vc(__comp)) {
+    insert(__f, __l);
+  }
+
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI
+  multimap(_InputIterator __f, _InputIterator __l, const key_compare& __comp, const allocator_type& __a)
+      : __tree_(__vc(__comp), typename __base::allocator_type(__a)) {
+    insert(__f, __l);
+  }
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI
+  multimap(from_range_t,
+           _Range&& __range,
+           const key_compare& __comp = key_compare(),
+           const allocator_type& __a = allocator_type())
+      : __tree_(__vc(__comp), typename __base::allocator_type(__a)) {
+    insert_range(std::forward<_Range>(__range));
+  }
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI multimap(_InputIterator __f, _InputIterator __l, const allocator_type& __a)
+      : multimap(__f, __l, key_compare(), __a) {}
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI multimap(from_range_t, _Range&& __range, const allocator_type& __a)
+      : multimap(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m)
+      : __tree_(__m.__tree_.value_comp(),
+                __alloc_traits::select_on_container_copy_construction(__m.__tree_.__alloc())) {
+    insert(__m.begin(), __m.end());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI multimap& operator=(const multimap& __m) {
+#ifndef _LIBCPP_CXX03_LANG
+    __tree_ = __m.__tree_;
+#else
+    if (this != std::addressof(__m)) {
+      __tree_.clear();
+      __tree_.value_comp() = __m.__tree_.value_comp();
+      __tree_.__copy_assign_alloc(__m.__tree_);
+      insert(__m.begin(), __m.end());
+    }
+#endif
+    return *this;
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m) noexcept(is_nothrow_move_constructible<__base>::value)
+      : __tree_(std::move(__m.__tree_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m, const allocator_type& __a);
+
+  _LIBCPP_HIDE_FROM_ABI multimap& operator=(multimap&& __m) noexcept(is_nothrow_move_assignable<__base>::value) {
+    __tree_ = std::move(__m.__tree_);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI multimap(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
+      : __tree_(__vc(__comp)) {
+    insert(__il.begin(), __il.end());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI
+  multimap(initializer_list<value_type> __il, const key_compare& __comp, const allocator_type& __a)
+      : __tree_(__vc(__comp), typename __base::allocator_type(__a)) {
+    insert(__il.begin(), __il.end());
+  }
+
+#  if _LIBCPP_STD_VER >= 14
+  _LIBCPP_HIDE_FROM_ABI multimap(initializer_list<value_type> __il, const allocator_type& __a)
+      : multimap(__il, key_compare(), __a) {}
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI multimap& operator=(initializer_list<value_type> __il) {
+    __tree_.__assign_multi(__il.begin(), __il.end());
+    return *this;
+  }
+
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI explicit multimap(const allocator_type& __a) : __tree_(typename __base::allocator_type(__a)) {}
+
+  _LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m, const allocator_type& __a)
+      : __tree_(__m.__tree_.value_comp(), typename __base::allocator_type(__a)) {
+    insert(__m.begin(), __m.end());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~multimap() { static_assert(sizeof(__diagnose_non_const_comparator<_Key, _Compare>()), ""); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __tree_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __tree_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __tree_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __tree_.end(); }
+
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return rbegin(); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __tree_.size() == 0; }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __tree_.size(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __tree_.max_size(); }
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT { return allocator_type(__tree_.__alloc()); }
+  _LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __tree_.value_comp().key_comp(); }
+  _LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return value_compare(__tree_.value_comp().key_comp()); }
+
+#ifndef _LIBCPP_CXX03_LANG
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace(_Args&&... __args) {
+    return __tree_.__emplace_multi(std::forward<_Args>(__args)...);
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __p, _Args&&... __args) {
+    return __tree_.__emplace_hint_multi(__p.__i_, std::forward<_Args>(__args)...);
+  }
+
+  template <class _Pp, __enable_if_t<is_constructible<value_type, _Pp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator insert(_Pp&& __p) {
+    return __tree_.__insert_multi(std::forward<_Pp>(__p));
+  }
+
+  template <class _Pp, __enable_if_t<is_constructible<value_type, _Pp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __pos, _Pp&& __p) {
+    return __tree_.__insert_multi(__pos.__i_, std::forward<_Pp>(__p));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(value_type&& __v) { return __tree_.__insert_multi(std::move(__v)); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __v) {
+    return __tree_.__insert_multi(__p.__i_, std::move(__v));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
+
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __v) { return __tree_.__insert_multi(__v); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __v) {
+    return __tree_.__insert_multi(__p.__i_, __v);
+  }
+
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __f, _InputIterator __l) {
+    for (const_iterator __e = cend(); __f != __l; ++__f)
+      __tree_.__insert_multi(__e.__i_, *__f);
+  }
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
+    const_iterator __end = cend();
+    for (auto&& __element : __range) {
+      __tree_.__insert_multi(__end.__i_, std::forward<decltype(__element)>(__element));
+    }
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __tree_.erase(__p.__i_); }
+  _LIBCPP_HIDE_FROM_ABI iterator erase(iterator __p) { return __tree_.erase(__p.__i_); }
+  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __tree_.__erase_multi(__k); }
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __f, const_iterator __l) {
+    return __tree_.erase(__f.__i_, __l.__i_);
+  }
+
+#if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI iterator insert(node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to multimap::insert()");
+    return __tree_.template __node_handle_insert_multi<node_type>(std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to multimap::insert()");
+    return __tree_.template __node_handle_insert_multi<node_type>(__hint.__i_, std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
+    return __tree_.template __node_handle_extract<node_type>(__key);
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
+    return __tree_.template __node_handle_extract<node_type>(__it.__i_);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(multimap<key_type, mapped_type, _Compare2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __tree_.__node_handle_merge_multi(__source.__tree_);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(multimap<key_type, mapped_type, _Compare2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __tree_.__node_handle_merge_multi(__source.__tree_);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(map<key_type, mapped_type, _Compare2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __tree_.__node_handle_merge_multi(__source.__tree_);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(map<key_type, mapped_type, _Compare2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __tree_.__node_handle_merge_multi(__source.__tree_);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __tree_.clear(); }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(multimap& __m) _NOEXCEPT_(__is_nothrow_swappable_v<__base>) {
+    __tree_.swap(__m.__tree_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
+    return __tree_.find(__k);
+  }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
+    return __tree_.find(__k);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_multi(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
+    return __tree_.__count_multi(__k);
+  }
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
+    return find(__k) != end();
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
+    return __tree_.lower_bound(__k);
+  }
+
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
+    return __tree_.lower_bound(__k);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
+    return __tree_.upper_bound(__k);
+  }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
+    return __tree_.upper_bound(__k);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
+    return __tree_.__equal_range_multi(__k);
+  }
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
+    return __tree_.__equal_range_multi(__k);
+  }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
+    return __tree_.__equal_range_multi(__k);
+  }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
+    return __tree_.__equal_range_multi(__k);
+  }
+#endif
+
+private:
+  typedef typename __base::__node __node;
+  typedef typename __base::__node_allocator __node_allocator;
+  typedef typename __base::__node_pointer __node_pointer;
+
+  typedef __map_node_destructor<__node_allocator> _Dp;
+  typedef unique_ptr<__node, _Dp> __node_holder;
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator,
+          class _Compare   = less<__iter_key_type<_InputIterator>>,
+          class _Allocator = allocator<__iter_to_alloc_type<_InputIterator>>,
+          class            = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
+          class            = enable_if_t<!__is_allocator<_Compare>::value, void>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value, void>>
+multimap(_InputIterator, _InputIterator, _Compare = _Compare(), _Allocator = _Allocator())
+    -> multimap<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Compare, _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range,
+          class _Compare   = less<__range_key_type<_Range>>,
+          class _Allocator = allocator<__range_to_alloc_type<_Range>>,
+          class            = enable_if_t<!__is_allocator<_Compare>::value, void>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value, void>>
+multimap(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator())
+    -> multimap<__range_key_type<_Range>, __range_mapped_type<_Range>, _Compare, _Allocator>;
+#  endif
+
+template <class _Key,
+          class _Tp,
+          class _Compare   = less<remove_const_t<_Key>>,
+          class _Allocator = allocator<pair<const _Key, _Tp>>,
+          class            = enable_if_t<!__is_allocator<_Compare>::value, void>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value, void>>
+multimap(initializer_list<pair<_Key, _Tp>>,
+         _Compare   = _Compare(),
+         _Allocator = _Allocator()) -> multimap<remove_const_t<_Key>, _Tp, _Compare, _Allocator>;
+
+template <class _InputIterator,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
+          class = enable_if_t<__is_allocator<_Allocator>::value, void>>
+multimap(_InputIterator, _InputIterator, _Allocator)
+    -> multimap<__iter_key_type<_InputIterator>,
+                __iter_mapped_type<_InputIterator>,
+                less<__iter_key_type<_InputIterator>>,
+                _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
+multimap(from_range_t, _Range&&, _Allocator)
+    -> multimap<__range_key_type<_Range>, __range_mapped_type<_Range>, less<__range_key_type<_Range>>, _Allocator>;
+#  endif
+
+template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
+multimap(initializer_list<pair<_Key, _Tp>>,
+         _Allocator) -> multimap<remove_const_t<_Key>, _Tp, less<remove_const_t<_Key>>, _Allocator>;
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+multimap<_Key, _Tp, _Compare, _Allocator>::multimap(multimap&& __m, const allocator_type& __a)
+    : __tree_(std::move(__m.__tree_), typename __base::allocator_type(__a)) {
+  if (__a != __m.get_allocator()) {
+    const_iterator __e = cend();
+    while (!__m.empty())
+      __tree_.__insert_multi(__e.__i_, std::move(__m.__tree_.remove(__m.begin().__i_)->__value_.__move()));
+  }
+}
+#endif
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const multimap<_Key, _Tp, _Compare, _Allocator>& __x, const multimap<_Key, _Tp, _Compare, _Allocator>& __y) {
+  return __x.size() == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<(const multimap<_Key, _Tp, _Compare, _Allocator>& __x, const multimap<_Key, _Tp, _Compare, _Allocator>& __y) {
+  return std::lexicographical_compare(__x.begin(), __x.end(), __y.begin(), __y.end());
+}
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const multimap<_Key, _Tp, _Compare, _Allocator>& __x, const multimap<_Key, _Tp, _Compare, _Allocator>& __y) {
+  return !(__x == __y);
+}
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>(const multimap<_Key, _Tp, _Compare, _Allocator>& __x, const multimap<_Key, _Tp, _Compare, _Allocator>& __y) {
+  return __y < __x;
+}
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>=(const multimap<_Key, _Tp, _Compare, _Allocator>& __x, const multimap<_Key, _Tp, _Compare, _Allocator>& __y) {
+  return !(__x < __y);
+}
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<=(const multimap<_Key, _Tp, _Compare, _Allocator>& __x, const multimap<_Key, _Tp, _Compare, _Allocator>& __y) {
+  return !(__y < __x);
+}
+
+#else // #if _LIBCPP_STD_VER <= 17
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI __synth_three_way_result<pair<const _Key, _Tp>>
+operator<=>(const multimap<_Key, _Tp, _Compare, _Allocator>& __x,
+            const multimap<_Key, _Tp, _Compare, _Allocator>& __y) {
+  return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), __synth_three_way);
+}
+
+#endif // #if _LIBCPP_STD_VER <= 17
+
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(multimap<_Key, _Tp, _Compare, _Allocator>& __x, multimap<_Key, _Tp, _Compare, _Allocator>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Key, class _Tp, class _Compare, class _Allocator, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI typename multimap<_Key, _Tp, _Compare, _Allocator>::size_type
+erase_if(multimap<_Key, _Tp, _Compare, _Allocator>& __c, _Predicate __pred) {
+  return std::__libcpp_erase_if_container(__c, __pred);
+}
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace pmr {
+template <class _KeyT, class _ValueT, class _CompareT = std::less<_KeyT>>
+using map _LIBCPP_AVAILABILITY_PMR =
+    std::map<_KeyT, _ValueT, _CompareT, polymorphic_allocator<std::pair<const _KeyT, _ValueT>>>;
+
+template <class _KeyT, class _ValueT, class _CompareT = std::less<_KeyT>>
+using multimap _LIBCPP_AVAILABILITY_PMR =
+    std::multimap<_KeyT, _ValueT, _CompareT, polymorphic_allocator<std::pair<const _KeyT, _ValueT>>>;
+} // namespace pmr
+_LIBCPP_END_NAMESPACE_STD
+#endif
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <concepts>
+#  include <cstdlib>
+#  include <functional>
+#  include <iterator>
+#  include <type_traits>
+#  include <utility>
+#endif
+
+#endif // _LIBCPP_MAP
diff --git a/libcxx/include/__cxx03/math.h b/libcxx/include/__cxx03/math.h
new file mode 100644
index 000000000000..4e6304a75398
--- /dev/null
+++ b/libcxx/include/__cxx03/math.h
@@ -0,0 +1,520 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_MATH_H
+#  define _LIBCPP_MATH_H
+
+/*
+    math.h synopsis
+
+Macros:
+
+    HUGE_VAL
+    HUGE_VALF               // C99
+    HUGE_VALL               // C99
+    INFINITY                // C99
+    NAN                     // C99
+    FP_INFINITE             // C99
+    FP_NAN                  // C99
+    FP_NORMAL               // C99
+    FP_SUBNORMAL            // C99
+    FP_ZERO                 // C99
+    FP_FAST_FMA             // C99
+    FP_FAST_FMAF            // C99
+    FP_FAST_FMAL            // C99
+    FP_ILOGB0               // C99
+    FP_ILOGBNAN             // C99
+    MATH_ERRNO              // C99
+    MATH_ERREXCEPT          // C99
+    math_errhandling        // C99
+
+Types:
+
+    float_t                 // C99
+    double_t                // C99
+
+// C90
+
+floating_point abs(floating_point x);
+
+floating_point acos (arithmetic x);
+float          acosf(float x);
+long double    acosl(long double x);
+
+floating_point asin (arithmetic x);
+float          asinf(float x);
+long double    asinl(long double x);
+
+floating_point atan (arithmetic x);
+float          atanf(float x);
+long double    atanl(long double x);
+
+floating_point atan2 (arithmetic y, arithmetic x);
+float          atan2f(float y, float x);
+long double    atan2l(long double y, long double x);
+
+floating_point ceil (arithmetic x);
+float          ceilf(float x);
+long double    ceill(long double x);
+
+floating_point cos (arithmetic x);
+float          cosf(float x);
+long double    cosl(long double x);
+
+floating_point cosh (arithmetic x);
+float          coshf(float x);
+long double    coshl(long double x);
+
+floating_point exp (arithmetic x);
+float          expf(float x);
+long double    expl(long double x);
+
+floating_point fabs (arithmetic x);
+float          fabsf(float x);
+long double    fabsl(long double x);
+
+floating_point floor (arithmetic x);
+float          floorf(float x);
+long double    floorl(long double x);
+
+floating_point fmod (arithmetic x, arithmetic y);
+float          fmodf(float x, float y);
+long double    fmodl(long double x, long double y);
+
+floating_point frexp (arithmetic value, int* exp);
+float          frexpf(float value, int* exp);
+long double    frexpl(long double value, int* exp);
+
+floating_point ldexp (arithmetic value, int exp);
+float          ldexpf(float value, int exp);
+long double    ldexpl(long double value, int exp);
+
+floating_point log (arithmetic x);
+float          logf(float x);
+long double    logl(long double x);
+
+floating_point log10 (arithmetic x);
+float          log10f(float x);
+long double    log10l(long double x);
+
+floating_point modf (floating_point value, floating_point* iptr);
+float          modff(float value, float* iptr);
+long double    modfl(long double value, long double* iptr);
+
+floating_point pow (arithmetic x, arithmetic y);
+float          powf(float x, float y);
+long double    powl(long double x, long double y);
+
+floating_point sin (arithmetic x);
+float          sinf(float x);
+long double    sinl(long double x);
+
+floating_point sinh (arithmetic x);
+float          sinhf(float x);
+long double    sinhl(long double x);
+
+floating_point sqrt (arithmetic x);
+float          sqrtf(float x);
+long double    sqrtl(long double x);
+
+floating_point tan (arithmetic x);
+float          tanf(float x);
+long double    tanl(long double x);
+
+floating_point tanh (arithmetic x);
+float          tanhf(float x);
+long double    tanhl(long double x);
+
+//  C99
+
+bool signbit(arithmetic x);
+
+int fpclassify(arithmetic x);
+
+bool isfinite(arithmetic x);
+bool isinf(arithmetic x);
+bool isnan(arithmetic x);
+bool isnormal(arithmetic x);
+
+bool isgreater(arithmetic x, arithmetic y);
+bool isgreaterequal(arithmetic x, arithmetic y);
+bool isless(arithmetic x, arithmetic y);
+bool islessequal(arithmetic x, arithmetic y);
+bool islessgreater(arithmetic x, arithmetic y);
+bool isunordered(arithmetic x, arithmetic y);
+
+floating_point acosh (arithmetic x);
+float          acoshf(float x);
+long double    acoshl(long double x);
+
+floating_point asinh (arithmetic x);
+float          asinhf(float x);
+long double    asinhl(long double x);
+
+floating_point atanh (arithmetic x);
+float          atanhf(float x);
+long double    atanhl(long double x);
+
+floating_point cbrt (arithmetic x);
+float          cbrtf(float x);
+long double    cbrtl(long double x);
+
+floating_point copysign (arithmetic x, arithmetic y);
+float          copysignf(float x, float y);
+long double    copysignl(long double x, long double y);
+
+floating_point erf (arithmetic x);
+float          erff(float x);
+long double    erfl(long double x);
+
+floating_point erfc (arithmetic x);
+float          erfcf(float x);
+long double    erfcl(long double x);
+
+floating_point exp2 (arithmetic x);
+float          exp2f(float x);
+long double    exp2l(long double x);
+
+floating_point expm1 (arithmetic x);
+float          expm1f(float x);
+long double    expm1l(long double x);
+
+floating_point fdim (arithmetic x, arithmetic y);
+float          fdimf(float x, float y);
+long double    fdiml(long double x, long double y);
+
+floating_point fma (arithmetic x, arithmetic y, arithmetic z);
+float          fmaf(float x, float y, float z);
+long double    fmal(long double x, long double y, long double z);
+
+floating_point fmax (arithmetic x, arithmetic y);
+float          fmaxf(float x, float y);
+long double    fmaxl(long double x, long double y);
+
+floating_point fmin (arithmetic x, arithmetic y);
+float          fminf(float x, float y);
+long double    fminl(long double x, long double y);
+
+floating_point hypot (arithmetic x, arithmetic y);
+float          hypotf(float x, float y);
+long double    hypotl(long double x, long double y);
+
+int ilogb (arithmetic x);
+int ilogbf(float x);
+int ilogbl(long double x);
+
+floating_point lgamma (arithmetic x);
+float          lgammaf(float x);
+long double    lgammal(long double x);
+
+long long llrint (arithmetic x);
+long long llrintf(float x);
+long long llrintl(long double x);
+
+long long llround (arithmetic x);
+long long llroundf(float x);
+long long llroundl(long double x);
+
+floating_point log1p (arithmetic x);
+float          log1pf(float x);
+long double    log1pl(long double x);
+
+floating_point log2 (arithmetic x);
+float          log2f(float x);
+long double    log2l(long double x);
+
+floating_point logb (arithmetic x);
+float          logbf(float x);
+long double    logbl(long double x);
+
+long lrint (arithmetic x);
+long lrintf(float x);
+long lrintl(long double x);
+
+long lround (arithmetic x);
+long lroundf(float x);
+long lroundl(long double x);
+
+double      nan (const char* str);
+float       nanf(const char* str);
+long double nanl(const char* str);
+
+floating_point nearbyint (arithmetic x);
+float          nearbyintf(float x);
+long double    nearbyintl(long double x);
+
+floating_point nextafter (arithmetic x, arithmetic y);
+float          nextafterf(float x, float y);
+long double    nextafterl(long double x, long double y);
+
+floating_point nexttoward (arithmetic x, long double y);
+float          nexttowardf(float x, long double y);
+long double    nexttowardl(long double x, long double y);
+
+floating_point remainder (arithmetic x, arithmetic y);
+float          remainderf(float x, float y);
+long double    remainderl(long double x, long double y);
+
+floating_point remquo (arithmetic x, arithmetic y, int* pquo);
+float          remquof(float x, float y, int* pquo);
+long double    remquol(long double x, long double y, int* pquo);
+
+floating_point rint (arithmetic x);
+float          rintf(float x);
+long double    rintl(long double x);
+
+floating_point round (arithmetic x);
+float          roundf(float x);
+long double    roundl(long double x);
+
+floating_point scalbln (arithmetic x, long ex);
+float          scalblnf(float x, long ex);
+long double    scalblnl(long double x, long ex);
+
+floating_point scalbn (arithmetic x, int ex);
+float          scalbnf(float x, int ex);
+long double    scalbnl(long double x, int ex);
+
+floating_point tgamma (arithmetic x);
+float          tgammaf(float x);
+long double    tgammal(long double x);
+
+floating_point trunc (arithmetic x);
+float          truncf(float x);
+long double    truncl(long double x);
+
+*/
+
+#  include <__config>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if __has_include_next(<math.h>)
+#    include_next <math.h>
+#  endif
+
+#  ifdef __cplusplus
+
+// We support including .h headers inside 'extern "C"' contexts, so switch
+// back to C++ linkage before including these C++ headers.
+extern "C++" {
+
+#    ifdef fpclassify
+#      undef fpclassify
+#    endif
+
+#    ifdef signbit
+#      undef signbit
+#    endif
+
+#    ifdef isfinite
+#      undef isfinite
+#    endif
+
+#    ifdef isinf
+#      undef isinf
+#    endif
+
+#    ifdef isnan
+#      undef isnan
+#    endif
+
+#    ifdef isnormal
+#      undef isnormal
+#    endif
+
+#    ifdef isgreater
+#      undef isgreater
+#    endif
+
+#    ifdef isgreaterequal
+#      undef isgreaterequal
+#    endif
+
+#    ifdef isless
+#      undef isless
+#    endif
+
+#    ifdef islessequal
+#      undef islessequal
+#    endif
+
+#    ifdef islessgreater
+#      undef islessgreater
+#    endif
+
+#    ifdef isunordered
+#      undef isunordered
+#    endif
+
+#    include <__math/abs.h>
+#    include <__math/copysign.h>
+#    include <__math/error_functions.h>
+#    include <__math/exponential_functions.h>
+#    include <__math/fdim.h>
+#    include <__math/fma.h>
+#    include <__math/gamma.h>
+#    include <__math/hyperbolic_functions.h>
+#    include <__math/hypot.h>
+#    include <__math/inverse_hyperbolic_functions.h>
+#    include <__math/inverse_trigonometric_functions.h>
+#    include <__math/logarithms.h>
+#    include <__math/min_max.h>
+#    include <__math/modulo.h>
+#    include <__math/remainder.h>
+#    include <__math/roots.h>
+#    include <__math/rounding_functions.h>
+#    include <__math/traits.h>
+#    include <__math/trigonometric_functions.h>
+#    include <__type_traits/enable_if.h>
+#    include <__type_traits/is_floating_point.h>
+#    include <__type_traits/is_integral.h>
+#    include <stdlib.h>
+
+// fpclassify relies on implementation-defined constants, so we can't move it to a detail header
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __math {
+
+// fpclassify
+
+// template on non-double overloads to make them weaker than same overloads from MSVC runtime
+template <class = int>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI int fpclassify(float __x) _NOEXCEPT {
+  return __builtin_fpclassify(FP_NAN, FP_INFINITE, FP_NORMAL, FP_SUBNORMAL, FP_ZERO, __x);
+}
+
+template <class = int>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI int fpclassify(double __x) _NOEXCEPT {
+  return __builtin_fpclassify(FP_NAN, FP_INFINITE, FP_NORMAL, FP_SUBNORMAL, FP_ZERO, __x);
+}
+
+template <class = int>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI int fpclassify(long double __x) _NOEXCEPT {
+  return __builtin_fpclassify(FP_NAN, FP_INFINITE, FP_NORMAL, FP_SUBNORMAL, FP_ZERO, __x);
+}
+
+template <class _A1, std::__enable_if_t<std::is_integral<_A1>::value, int> = 0>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI int fpclassify(_A1 __x) _NOEXCEPT {
+  return __x == 0 ? FP_ZERO : FP_NORMAL;
+}
+
+} // namespace __math
+
+_LIBCPP_END_NAMESPACE_STD
+
+using std::__math::fpclassify;
+using std::__math::signbit;
+
+// The MSVC runtime already provides these functions as templates
+#    ifndef _LIBCPP_MSVCRT
+using std::__math::isfinite;
+using std::__math::isgreater;
+using std::__math::isgreaterequal;
+using std::__math::isinf;
+using std::__math::isless;
+using std::__math::islessequal;
+using std::__math::islessgreater;
+using std::__math::isnan;
+using std::__math::isnormal;
+using std::__math::isunordered;
+#    endif // _LIBCPP_MSVCRT
+
+// abs
+//
+// handled in stdlib.h
+
+// div
+//
+// handled in stdlib.h
+
+// We have to provide double overloads for <math.h> to work on platforms that don't provide the full set of math
+// functions. To make the overload set work with multiple functions that take the same arguments, we make our overloads
+// templates. Functions are preferred over function templates during overload resolution, which means that our overload
+// will only be selected when the C library doesn't provide one.
+
+using std::__math::acos;
+using std::__math::acosh;
+using std::__math::asin;
+using std::__math::asinh;
+using std::__math::atan;
+using std::__math::atan2;
+using std::__math::atanh;
+using std::__math::cbrt;
+using std::__math::ceil;
+using std::__math::copysign;
+using std::__math::cos;
+using std::__math::cosh;
+using std::__math::erf;
+using std::__math::erfc;
+using std::__math::exp;
+using std::__math::exp2;
+using std::__math::expm1;
+using std::__math::fabs;
+using std::__math::fdim;
+using std::__math::floor;
+using std::__math::fma;
+using std::__math::fmax;
+using std::__math::fmin;
+using std::__math::fmod;
+using std::__math::frexp;
+using std::__math::hypot;
+using std::__math::ilogb;
+using std::__math::ldexp;
+using std::__math::lgamma;
+using std::__math::llrint;
+using std::__math::llround;
+using std::__math::log;
+using std::__math::log10;
+using std::__math::log1p;
+using std::__math::log2;
+using std::__math::logb;
+using std::__math::lrint;
+using std::__math::lround;
+using std::__math::modf;
+using std::__math::nearbyint;
+using std::__math::nextafter;
+using std::__math::nexttoward;
+using std::__math::pow;
+using std::__math::remainder;
+using std::__math::remquo;
+using std::__math::rint;
+using std::__math::round;
+using std::__math::scalbln;
+using std::__math::scalbn;
+using std::__math::signbit;
+using std::__math::sin;
+using std::__math::sinh;
+using std::__math::sqrt;
+using std::__math::tan;
+using std::__math::tanh;
+using std::__math::tgamma;
+using std::__math::trunc;
+
+} // extern "C++"
+
+#  endif // __cplusplus
+
+#else // _LIBCPP_MATH_H
+
+// This include lives outside the header guard in order to support an MSVC
+// extension which allows users to do:
+//
+// #define _USE_MATH_DEFINES
+// #include <math.h>
+//
+// and receive the definitions of mathematical constants, even if <math.h>
+// has previously been included.
+#  if defined(_LIBCPP_MSVCRT) && defined(_USE_MATH_DEFINES)
+#    include_next <math.h>
+#  endif
+
+#endif // _LIBCPP_MATH_H
diff --git a/libcxx/include/__cxx03/mdspan b/libcxx/include/__cxx03/mdspan
new file mode 100644
index 000000000000..29190e4a9953
--- /dev/null
+++ b/libcxx/include/__cxx03/mdspan
@@ -0,0 +1,438 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+/*
+
+// Overall mdspan synopsis
+
+namespace std {
+  // [mdspan.extents], class template extents
+  template<class IndexType, size_t... Extents>
+    class extents;
+
+  // [mdspan.extents.dextents], alias template dextents
+  template<class IndexType, size_t Rank>
+    using dextents = see below;
+
+  // [mdspan.extents.dims], alias template dims
+  template<size_t Rank, class IndexType = size_t>
+    using dims = see below; // since C++26
+
+  // [mdspan.layout], layout mapping
+  struct layout_left;
+  struct layout_right;
+  struct layout_stride;
+
+  // [mdspan.accessor.default], class template default_accessor
+  template<class ElementType>
+    class default_accessor;
+
+  // [mdspan.mdspan], class template mdspan
+  template<class ElementType, class Extents, class LayoutPolicy = layout_right,
+           class AccessorPolicy = default_accessor<ElementType>>
+    class mdspan; // not implemented yet
+}
+
+// extents synopsis
+
+namespace std {
+  template<class _IndexType, size_t... _Extents>
+  class extents {
+  public:
+    using index_type = _IndexType;
+    using size_type = make_unsigned_t<index_type>;
+    using rank_type = size_t;
+
+    // [mdspan.extents.obs], observers of the multidimensional index space
+    static constexpr rank_type rank() noexcept { return sizeof...(_Extents); }
+    static constexpr rank_type rank_dynamic() noexcept { return dynamic-index(rank()); }
+    static constexpr size_t static_extent(rank_type) noexcept;
+    constexpr index_type extent(rank_type) const noexcept;
+
+    // [mdspan.extents.cons], constructors
+    constexpr extents() noexcept = default;
+
+    template<class _OtherIndexType, size_t... _OtherExtents>
+      constexpr explicit(see below)
+        extents(const extents<_OtherIndexType, _OtherExtents...>&) noexcept;
+    template<class... _OtherIndexTypes>
+      constexpr explicit extents(_OtherIndexTypes...) noexcept;
+    template<class _OtherIndexType, size_t N>
+      constexpr explicit(N != rank_dynamic())
+        extents(span<_OtherIndexType, N>) noexcept;
+    template<class _OtherIndexType, size_t N>
+      constexpr explicit(N != rank_dynamic())
+        extents(const array<_OtherIndexType, N>&) noexcept;
+
+    // [mdspan.extents.cmp], comparison operators
+    template<class _OtherIndexType, size_t... _OtherExtents>
+      friend constexpr bool operator==(const extents&,
+                                       const extents<_OtherIndexType, _OtherExtents...>&) noexcept;
+
+  private:
+    // libcxx note: we do not use an array here, but we need to preserve the as-if behavior
+    // for example the default constructor must zero initialize dynamic extents
+    array<index_type, rank_dynamic()> dynamic-extents{};                // exposition only
+  };
+
+  template<class... Integrals>
+    explicit extents(Integrals...)
+      -> see below;
+}
+
+// layout_left synopsis
+
+namespace std {
+  template<class Extents>
+  class layout_left::mapping {
+  public:
+    using extents_type = Extents;
+    using index_type = typename extents_type::index_type;
+    using size_type = typename extents_type::size_type;
+    using rank_type = typename extents_type::rank_type;
+    using layout_type = layout_left;
+
+    // [mdspan.layout.right.cons], constructors
+    constexpr mapping() noexcept = default;
+    constexpr mapping(const mapping&) noexcept = default;
+    constexpr mapping(const extents_type&) noexcept;
+    template<class OtherExtents>
+      constexpr explicit(!is_convertible_v<OtherExtents, extents_type>)
+        mapping(const mapping<OtherExtents>&) noexcept;
+    template<class OtherExtents>
+      constexpr explicit(!is_convertible_v<OtherExtents, extents_type>)
+        mapping(const layout_right::mapping<OtherExtents>&) noexcept;
+    template<class OtherExtents>
+      constexpr explicit(extents_type::rank() > 0)
+        mapping(const layout_stride::mapping<OtherExtents>&) noexcept;
+
+    constexpr mapping& operator=(const mapping&) noexcept = default;
+
+    // [mdspan.layout.right.obs], observers
+    constexpr const extents_type& extents() const noexcept { return extents_; }
+
+    constexpr index_type required_span_size() const noexcept;
+
+    template<class... Indices>
+      constexpr index_type operator()(Indices...) const noexcept;
+
+    static constexpr bool is_always_unique() noexcept { return true; }
+    static constexpr bool is_always_exhaustive() noexcept { return true; }
+    static constexpr bool is_always_strided() noexcept { return true; }
+
+    static constexpr bool is_unique() noexcept { return true; }
+    static constexpr bool is_exhaustive() noexcept { return true; }
+    static constexpr bool is_strided() noexcept { return true; }
+
+    constexpr index_type stride(rank_type) const noexcept;
+
+    template<class OtherExtents>
+      friend constexpr bool operator==(const mapping&, const mapping<OtherExtents>&) noexcept;
+
+  private:
+    extents_type extents_{};    // exposition only
+  };
+}
+
+// layout_right synopsis
+
+namespace std {
+  template<class Extents>
+  class layout_right::mapping {
+  public:
+    using extents_type = Extents;
+    using index_type = typename extents_type::index_type;
+    using size_type = typename extents_type::size_type;
+    using rank_type = typename extents_type::rank_type;
+    using layout_type = layout_right;
+
+    // [mdspan.layout.right.cons], constructors
+    constexpr mapping() noexcept = default;
+    constexpr mapping(const mapping&) noexcept = default;
+    constexpr mapping(const extents_type&) noexcept;
+    template<class OtherExtents>
+      constexpr explicit(!is_convertible_v<OtherExtents, extents_type>)
+        mapping(const mapping<OtherExtents>&) noexcept;
+    template<class OtherExtents>
+      constexpr explicit(!is_convertible_v<OtherExtents, extents_type>)
+        mapping(const layout_left::mapping<OtherExtents>&) noexcept;
+    template<class OtherExtents>
+      constexpr explicit(extents_type::rank() > 0)
+        mapping(const layout_stride::mapping<OtherExtents>&) noexcept;
+
+    constexpr mapping& operator=(const mapping&) noexcept = default;
+
+    // [mdspan.layout.right.obs], observers
+    constexpr const extents_type& extents() const noexcept { return extents_; }
+
+    constexpr index_type required_span_size() const noexcept;
+
+    template<class... Indices>
+      constexpr index_type operator()(Indices...) const noexcept;
+
+    static constexpr bool is_always_unique() noexcept { return true; }
+    static constexpr bool is_always_exhaustive() noexcept { return true; }
+    static constexpr bool is_always_strided() noexcept { return true; }
+
+    static constexpr bool is_unique() noexcept { return true; }
+    static constexpr bool is_exhaustive() noexcept { return true; }
+    static constexpr bool is_strided() noexcept { return true; }
+
+    constexpr index_type stride(rank_type) const noexcept;
+
+    template<class OtherExtents>
+      friend constexpr bool operator==(const mapping&, const mapping<OtherExtents>&) noexcept;
+
+  private:
+    extents_type extents_{};    // exposition only
+  };
+}
+
+// layout_stride synopsis
+
+namespace std {
+  template<class Extents>
+  class layout_stride::mapping {
+  public:
+    using extents_type = Extents;
+    using index_type = typename extents_type::index_type;
+    using size_type = typename extents_type::size_type;
+    using rank_type = typename extents_type::rank_type;
+    using layout_type = layout_stride;
+
+  private:
+    static constexpr rank_type rank_ = extents_type::rank();    // exposition only
+
+  public:
+    // [mdspan.layout.stride.cons], constructors
+    constexpr mapping() noexcept;
+    constexpr mapping(const mapping&) noexcept = default;
+    template<class OtherIndexType>
+      constexpr mapping(const extents_type&, span<OtherIndexType, rank_>) noexcept;
+    template<class OtherIndexType>
+      constexpr mapping(const extents_type&, const array<OtherIndexType, rank_>&) noexcept;
+
+    template<class StridedLayoutMapping>
+      constexpr explicit(see below) mapping(const StridedLayoutMapping&) noexcept;
+
+    constexpr mapping& operator=(const mapping&) noexcept = default;
+
+    // [mdspan.layout.stride.obs], observers
+    constexpr const extents_type& extents() const noexcept { return extents_; }
+    constexpr array<index_type, rank_> strides() const noexcept { return strides_; }
+
+    constexpr index_type required_span_size() const noexcept;
+
+    template<class... Indices>
+      constexpr index_type operator()(Indices...) const noexcept;
+
+    static constexpr bool is_always_unique() noexcept { return true; }
+    static constexpr bool is_always_exhaustive() noexcept { return false; }
+    static constexpr bool is_always_strided() noexcept { return true; }
+
+    static constexpr bool is_unique() noexcept { return true; }
+    constexpr bool is_exhaustive() const noexcept;
+    static constexpr bool is_strided() noexcept { return true; }
+
+    constexpr index_type stride(rank_type i) const noexcept { return strides_[i]; }
+
+    template<class OtherMapping>
+      friend constexpr bool operator==(const mapping&, const OtherMapping&) noexcept;
+
+  private:
+    extents_type extents_{};                    // exposition only
+    array<index_type, rank_> strides_{};        // exposition only
+  };
+}
+
+// default_accessor synopsis
+
+namespace std {
+  template<class ElementType>
+  struct default_accessor {
+    using offset_policy = default_accessor;
+    using element_type = ElementType;
+    using reference = ElementType&;
+    using data_handle_type = ElementType*;
+
+    constexpr default_accessor() noexcept = default;
+    template<class OtherElementType>
+      constexpr default_accessor(default_accessor<OtherElementType>) noexcept;
+    constexpr reference access(data_handle_type p, size_t i) const noexcept;
+    constexpr data_handle_type offset(data_handle_type p, size_t i) const noexcept;
+  };
+}
+
+// mdspan synopsis
+
+namespace std {
+  template<class ElementType, class Extents, class LayoutPolicy = layout_right,
+           class AccessorPolicy = default_accessor<ElementType>>
+  class mdspan {
+  public:
+    using extents_type = Extents;
+    using layout_type = LayoutPolicy;
+    using accessor_type = AccessorPolicy;
+    using mapping_type = typename layout_type::template mapping<extents_type>;
+    using element_type = ElementType;
+    using value_type = remove_cv_t<element_type>;
+    using index_type = typename extents_type::index_type;
+    using size_type = typename extents_type::size_type;
+    using rank_type = typename extents_type::rank_type;
+    using data_handle_type = typename accessor_type::data_handle_type;
+    using reference = typename accessor_type::reference;
+
+    static constexpr rank_type rank() noexcept { return extents_type::rank(); }
+    static constexpr rank_type rank_dynamic() noexcept { return extents_type::rank_dynamic(); }
+    static constexpr size_t static_extent(rank_type r) noexcept
+      { return extents_type::static_extent(r); }
+    constexpr index_type extent(rank_type r) const noexcept { return extents().extent(r); }
+
+    // [mdspan.mdspan.cons], constructors
+    constexpr mdspan();
+    constexpr mdspan(const mdspan& rhs) = default;
+    constexpr mdspan(mdspan&& rhs) = default;
+
+    template<class... OtherIndexTypes>
+      constexpr explicit mdspan(data_handle_type ptr, OtherIndexTypes... exts);
+    template<class OtherIndexType, size_t N>
+      constexpr explicit(N != rank_dynamic())
+        mdspan(data_handle_type p, span<OtherIndexType, N> exts);
+    template<class OtherIndexType, size_t N>
+      constexpr explicit(N != rank_dynamic())
+        mdspan(data_handle_type p, const array<OtherIndexType, N>& exts);
+    constexpr mdspan(data_handle_type p, const extents_type& ext);
+    constexpr mdspan(data_handle_type p, const mapping_type& m);
+    constexpr mdspan(data_handle_type p, const mapping_type& m, const accessor_type& a);
+
+    template<class OtherElementType, class OtherExtents,
+             class OtherLayoutPolicy, class OtherAccessorPolicy>
+      constexpr explicit(see below)
+        mdspan(const mdspan<OtherElementType, OtherExtents,
+                            OtherLayoutPolicy, OtherAccessorPolicy>& other);
+
+    constexpr mdspan& operator=(const mdspan& rhs) = default;
+    constexpr mdspan& operator=(mdspan&& rhs) = default;
+
+    // [mdspan.mdspan.members], members
+    template<class... OtherIndexTypes>
+      constexpr reference operator[](OtherIndexTypes... indices) const;
+    template<class OtherIndexType>
+      constexpr reference operator[](span<OtherIndexType, rank()> indices) const;
+    template<class OtherIndexType>
+      constexpr reference operator[](const array<OtherIndexType, rank()>& indices) const;
+
+    constexpr size_type size() const noexcept;
+    [[nodiscard]] constexpr bool empty() const noexcept;
+
+    friend constexpr void swap(mdspan& x, mdspan& y) noexcept;
+
+    constexpr const extents_type& extents() const noexcept { return map_.extents(); }
+    constexpr const data_handle_type& data_handle() const noexcept { return ptr_; }
+    constexpr const mapping_type& mapping() const noexcept { return map_; }
+    constexpr const accessor_type& accessor() const noexcept { return acc_; }
+
+    // per LWG-4021 "mdspan::is_always_meow() should be noexcept"
+    static constexpr bool is_always_unique() noexcept
+      { return mapping_type::is_always_unique(); }
+    static constexpr bool is_always_exhaustive() noexcept
+      { return mapping_type::is_always_exhaustive(); }
+    static constexpr bool is_always_strided() noexcept
+      { return mapping_type::is_always_strided(); }
+
+    constexpr bool is_unique() const
+      { return map_.is_unique(); }
+    constexpr bool is_exhaustive() const
+      { return map_.is_exhaustive(); }
+    constexpr bool is_strided() const
+      { return map_.is_strided(); }
+    constexpr index_type stride(rank_type r) const
+      { return map_.stride(r); }
+
+  private:
+    accessor_type acc_;         // exposition only
+    mapping_type map_;          // exposition only
+    data_handle_type ptr_;      // exposition only
+  };
+
+  template<class CArray>
+    requires(is_array_v<CArray> && rank_v<CArray> == 1)
+    mdspan(CArray&)
+      -> mdspan<remove_all_extents_t<CArray>, extents<size_t, extent_v<CArray, 0>>>;
+
+  template<class Pointer>
+    requires(is_pointer_v<remove_reference_t<Pointer>>)
+    mdspan(Pointer&&)
+      -> mdspan<remove_pointer_t<remove_reference_t<Pointer>>, extents<size_t>>;
+
+  template<class ElementType, class... Integrals>
+    requires((is_convertible_v<Integrals, size_t> && ...) && sizeof...(Integrals) > 0)
+    explicit mdspan(ElementType*, Integrals...)
+      -> mdspan<ElementType, dextents<size_t, sizeof...(Integrals)>>;            // until C++26
+  template<class ElementType, class... Integrals>
+    requires((is_convertible_v<Integrals, size_t> && ...) && sizeof...(Integrals) > 0)
+    explicit mdspan(ElementType*, Integrals...)
+      -> mdspan<ElementType, extents<size_t, maybe-static-ext<Integrals>...>>;  // since C++26
+
+  template<class ElementType, class OtherIndexType, size_t N>
+    mdspan(ElementType*, span<OtherIndexType, N>)
+      -> mdspan<ElementType, dextents<size_t, N>>;
+
+  template<class ElementType, class OtherIndexType, size_t N>
+    mdspan(ElementType*, const array<OtherIndexType, N>&)
+      -> mdspan<ElementType, dextents<size_t, N>>;
+
+  template<class ElementType, class IndexType, size_t... ExtentsPack>
+    mdspan(ElementType*, const extents<IndexType, ExtentsPack...>&)
+      -> mdspan<ElementType, extents<IndexType, ExtentsPack...>>;
+
+  template<class ElementType, class MappingType>
+    mdspan(ElementType*, const MappingType&)
+      -> mdspan<ElementType, typename MappingType::extents_type,
+                typename MappingType::layout_type>;
+
+  template<class MappingType, class AccessorType>
+    mdspan(const typename AccessorType::data_handle_type&, const MappingType&,
+           const AccessorType&)
+      -> mdspan<typename AccessorType::element_type, typename MappingType::extents_type,
+                typename MappingType::layout_type, AccessorType>;
+}
+*/
+
+#ifndef _LIBCPP_MDSPAN
+#define _LIBCPP_MDSPAN
+
+#include <__config>
+
+#if _LIBCPP_STD_VER >= 23
+#  include <__fwd/mdspan.h>
+#  include <__mdspan/default_accessor.h>
+#  include <__mdspan/extents.h>
+#  include <__mdspan/layout_left.h>
+#  include <__mdspan/layout_right.h>
+#  include <__mdspan/layout_stride.h>
+#  include <__mdspan/mdspan.h>
+#endif
+
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <array>
+#  include <cinttypes>
+#  include <concepts>
+#  include <cstddef>
+#  include <limits>
+#  include <span>
+#endif
+
+#endif // _LIBCPP_MDSPAN
diff --git a/libcxx/include/__cxx03/memory b/libcxx/include/__cxx03/memory
new file mode 100644
index 000000000000..b940a32c3ebe
--- /dev/null
+++ b/libcxx/include/__cxx03/memory
@@ -0,0 +1,998 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_MEMORY
+#define _LIBCPP_MEMORY
+
+// clang-format off
+
+/*
+    memory synopsis
+
+namespace std
+{
+
+struct allocator_arg_t { };
+inline constexpr allocator_arg_t allocator_arg = allocator_arg_t();
+
+template <class T, class Alloc> struct uses_allocator;
+
+template <class Ptr>
+struct pointer_traits
+{
+    typedef Ptr pointer;
+    typedef <details> element_type;
+    typedef <details> difference_type;
+
+    template <class U> using rebind = <details>;
+
+    static pointer pointer_to(<details>);
+};
+
+template <class T>
+struct pointer_traits<T*>
+{
+    typedef T* pointer;
+    typedef T element_type;
+    typedef ptrdiff_t difference_type;
+
+    template <class U> using rebind = U*;
+
+    static pointer pointer_to(<details>) noexcept; // constexpr in C++20
+};
+
+template <class T> constexpr T* to_address(T* p) noexcept; // C++20
+template <class Ptr> constexpr auto to_address(const Ptr& p) noexcept; // C++20
+
+template <class Alloc>
+struct allocator_traits
+{
+    typedef Alloc                        allocator_type;
+    typedef typename allocator_type::value_type
+                                         value_type;
+
+    typedef Alloc::pointer | value_type* pointer;
+    typedef Alloc::const_pointer
+          | pointer_traits<pointer>::rebind<const value_type>
+                                         const_pointer;
+    typedef Alloc::void_pointer
+          | pointer_traits<pointer>::rebind<void>
+                                         void_pointer;
+    typedef Alloc::const_void_pointer
+          | pointer_traits<pointer>::rebind<const void>
+                                         const_void_pointer;
+    typedef Alloc::difference_type
+          | pointer_traits<pointer>::difference_type
+                                         difference_type;
+    typedef Alloc::size_type
+          | make_unsigned<difference_type>::type
+                                         size_type;
+    typedef Alloc::propagate_on_container_copy_assignment
+          | false_type                   propagate_on_container_copy_assignment;
+    typedef Alloc::propagate_on_container_move_assignment
+          | false_type                   propagate_on_container_move_assignment;
+    typedef Alloc::propagate_on_container_swap
+          | false_type                   propagate_on_container_swap;
+    typedef Alloc::is_always_equal
+          | is_empty                     is_always_equal;
+
+    template <class T> using rebind_alloc  = Alloc::rebind<T>::other | Alloc<T, Args...>;
+    template <class T> using rebind_traits = allocator_traits<rebind_alloc<T>>;
+
+    static pointer allocate(allocator_type& a, size_type n);                          // constexpr and [[nodiscard]] in C++20
+    static pointer allocate(allocator_type& a, size_type n, const_void_pointer hint); // constexpr and [[nodiscard]] in C++20
+
+    [[nodiscard]] static constexpr allocation_result<pointer, size_type>
+      allocate_at_least(Alloc& a, size_type n);                                 // Since C++23
+
+    static void deallocate(allocator_type& a, pointer p, size_type n) noexcept; // constexpr in C++20
+
+    template <class T, class... Args>
+    static void construct(allocator_type& a, T* p, Args&&... args); // constexpr in C++20
+
+    template <class T>
+    static void destroy(allocator_type& a, T* p); // constexpr in C++20
+
+    static size_type max_size(const allocator_type& a); // noexcept in C++14, constexpr in C++20
+    static allocator_type select_on_container_copy_construction(const allocator_type& a); // constexpr in C++20
+};
+
+template<class Pointer, class SizeType = size_t>
+struct allocation_result {
+    Pointer ptr;
+    SizeType count;
+}; // Since C++23
+
+template <>
+class allocator<void> // removed in C++20
+{
+public:
+    typedef void*                                 pointer;
+    typedef const void*                           const_pointer;
+    typedef void                                  value_type;
+
+    template <class _Up> struct rebind {typedef allocator<_Up> other;};
+};
+
+template <class T>
+class allocator
+{
+public:
+    typedef size_t    size_type;
+    typedef ptrdiff_t difference_type;
+    typedef T*        pointer;                           // deprecated in C++17, removed in C++20
+    typedef const T*  const_pointer;                     // deprecated in C++17, removed in C++20
+    typedef typename add_lvalue_reference<T>::type
+                      reference;                         // deprecated in C++17, removed in C++20
+    typedef typename add_lvalue_reference<const T>::type
+                      const_reference;                   // deprecated in C++17, removed in C++20
+
+    typedef T         value_type;
+
+    template <class U> struct rebind {typedef allocator<U> other;}; // deprecated in C++17, removed in C++20
+
+    typedef true_type propagate_on_container_move_assignment;
+    typedef true_type is_always_equal;                   // Deprecated in C++23, removed in C++26
+
+    constexpr allocator() noexcept;                      // constexpr in C++20
+    constexpr allocator(const allocator&) noexcept;      // constexpr in C++20
+    template <class U>
+      constexpr allocator(const allocator<U>&) noexcept; // constexpr in C++20
+    ~allocator();                                        // constexpr in C++20
+    pointer address(reference x) const noexcept;             // deprecated in C++17, removed in C++20
+    const_pointer address(const_reference x) const noexcept; // deprecated in C++17, removed in C++20
+    T* allocate(size_t n, const void* hint);          // deprecated in C++17, removed in C++20
+    T* allocate(size_t n);                              // constexpr in C++20
+    void deallocate(T* p, size_t n) noexcept;           // constexpr in C++20
+    size_type max_size() const noexcept;              // deprecated in C++17, removed in C++20
+    template<class U, class... Args>
+        void construct(U* p, Args&&... args);         // deprecated in C++17, removed in C++20
+    template <class U>
+        void destroy(U* p);                           // deprecated in C++17, removed in C++20
+};
+
+template <class T, class U>
+bool operator==(const allocator<T>&, const allocator<U>&) noexcept; // constexpr in C++20
+
+template <class T, class U>
+bool operator!=(const allocator<T>&, const allocator<U>&) noexcept; // removed in C++20
+
+template <class OutputIterator, class T>
+class raw_storage_iterator // deprecated in C++17, removed in C++20
+    : public iterator<output_iterator_tag, void, void, void, void> // until C++17
+{
+public:
+    typedef output_iterator_tag iterator_category;
+    typedef void                value_type;
+    typedef void                difference_type; // until C++20
+    typedef ptrdiff_t           difference_type; // since C++20
+    typedef void                pointer;
+    typedef void                reference;
+
+    explicit raw_storage_iterator(OutputIterator x);
+    raw_storage_iterator& operator*();
+    raw_storage_iterator& operator=(const T& element);
+    raw_storage_iterator& operator++();
+    raw_storage_iterator  operator++(int);
+};
+
+template <class T> pair<T*,ptrdiff_t> get_temporary_buffer(ptrdiff_t n) noexcept;
+template <class T> void               return_temporary_buffer(T* p) noexcept;
+
+template <class T> T* addressof(T& r) noexcept;
+template <class T> T* addressof(const T&& r) noexcept = delete;
+
+template <class InputIterator, class ForwardIterator>
+ForwardIterator
+uninitialized_copy(InputIterator first, InputIterator last, ForwardIterator result);
+
+namespace ranges {
+
+template<class InputIterator, class OutputIterator>
+using uninitialized_copy_result = in_out_result<InputIterator, OutputIterator>; // since C++20
+
+template<input_iterator InputIterator, sentinel-for<InputIterator> Sentinel1, nothrow-forward-iterator OutputIterator, nothrow-sentinel-for<OutputIterator> Sentinel2>
+  requires constructible_from<iter_value_t<OutputIterator>, iter_reference_t<InputIterator>>
+uninitialized_copy_result<InputIterator, OutputIterator>
+uninitialized_copy(InputIterator ifirst, Sentinel1 ilast, OutputIterator ofirst, Sentinel2 olast); // since C++20
+
+template<input_range InputRange, nothrow-forward-range OutputRange>
+  requires constructible_from<range_value_t<OutputRange>, range_reference_t<InputRange>>
+uninitialized_copy_result<borrowed_iterator_t<InputRange>, borrowed_iterator_t<OutputRange>>
+uninitialized_copy(InputRange&& in_range, OutputRange&& out_range); // since C++20
+
+}
+
+template <class InputIterator, class Size, class ForwardIterator>
+ForwardIterator
+uninitialized_copy_n(InputIterator first, Size n, ForwardIterator result);
+
+namespace ranges {
+
+template<class InputIterator, class OutputIterator>
+using uninitialized_copy_n_result = in_out_result<InputIterator, OutputIterator>; // since C++20
+
+template<input_iterator InputIterator, nothrow-forward-iterator OutputIterator, nothrow-sentinel-for<OutputIterator> Sentinel>
+  requires constructible_from<iter_value_t<OutputIterator>, iter_reference_t<InputIterator>>
+uninitialized_copy_n_result<InputIterator, OutputIterator>
+uninitialized_copy_n(InputIterator ifirst, iter_difference_t<InputIterator> n, OutputIterator ofirst, Sentinel olast); // since C++20
+
+}
+
+template <class ForwardIterator, class T>
+void uninitialized_fill(ForwardIterator first, ForwardIterator last, const T& x);
+
+namespace ranges {
+
+template <nothrow-forward-iterator ForwardIterator, nothrow-sentinel-for<ForwardIterator> Sentinel, class T>
+  requires constructible_from<iter_value_t<ForwardIterator>, const T&>
+ForwardIterator uninitialized_fill(ForwardIterator first, Sentinel last, const T& x); // since C++20
+
+template <nothrow-forward-range ForwardRange, class T>
+  requires constructible_from<range_value_t<ForwardRange>, const T&>
+borrowed_iterator_t<ForwardRange> uninitialized_fill(ForwardRange&& range, const T& x); // since C++20
+
+}
+
+template <class ForwardIterator, class Size, class T>
+ForwardIterator
+uninitialized_fill_n(ForwardIterator first, Size n, const T& x);
+
+namespace ranges {
+
+template <nothrow-forward-iterator ForwardIterator, class T>
+  requires constructible_from<iter_value_t<ForwardIterator>, const T&>
+ForwardIterator uninitialized_fill_n(ForwardIterator first, iter_difference_t<ForwardIterator> n); // since C++20
+
+}
+
+template <class T, class ...Args>
+constexpr T* construct_at(T* location, Args&& ...args); // since C++20
+
+namespace ranges {
+  template<class T, class... Args>
+    constexpr T* construct_at(T* location, Args&&... args); // since C++20
+}
+
+template <class T>
+void destroy_at(T* location); // constexpr in C++20
+
+namespace ranges {
+  template<destructible T>
+    constexpr void destroy_at(T* location) noexcept; // since C++20
+}
+
+template <class ForwardIterator>
+void destroy(ForwardIterator first, ForwardIterator last); // constexpr in C++20
+
+namespace ranges {
+  template<nothrow-input-iterator InputIterator, nothrow-sentinel-for<InputIterator> Sentinel>
+    requires destructible<iter_value_t<InputIterator>>
+    constexpr InputIterator destroy(InputIterator first, Sentinel last) noexcept; // since C++20
+  template<nothrow-input-range InputRange>
+    requires destructible<range_value_t<InputRange>>
+    constexpr borrowed_iterator_t<InputRange> destroy(InputRange&& range) noexcept; // since C++20
+}
+
+template <class ForwardIterator, class Size>
+ForwardIterator destroy_n(ForwardIterator first, Size n); // constexpr in C++20
+
+namespace ranges {
+  template<nothrow-input-iterator InputIterator>
+    requires destructible<iter_value_t<InputIterator>>
+    constexpr InputIterator destroy_n(InputIterator first, iter_difference_t<InputIterator> n) noexcept; // since C++20
+}
+
+template <class InputIterator, class ForwardIterator>
+ ForwardIterator uninitialized_move(InputIterator first, InputIterator last, ForwardIterator result);
+
+namespace ranges {
+
+template<class InputIterator, class OutputIterator>
+using uninitialized_move_result = in_out_result<InputIterator, OutputIterator>; // since C++20
+
+template <input_iterator InputIterator, sentinel_for<InputIterator> Sentinel1, nothrow-forward-iterator OutputIterator, nothrow-sentinel-for<O> Sentinel2>
+  requires constructible_from<iter_value_t<OutputIterator>, iter_rvalue_reference_t<InputIterator>>
+uninitialized_move_result<InputIterator, OutputIterator>
+uninitialized_move(InputIterator ifirst, Sentinel1 ilast, OutputIterator ofirst, Sentinel2 olast); // since C++20
+
+template<input_range InputRange, nothrow-forward-range OutputRange>
+  requires constructible_from<range_value_t<OutputRange>, range_rvalue_reference_t<InputRange>>
+uninitialized_move_result<borrowed_iterator_t<InputRange>, borrowed_iterator_t<OutputRange>>
+uninitialized_move(InputRange&& in_range, OutputRange&& out_range); // since C++20
+
+}
+
+template <class InputIterator, class Size, class ForwardIterator>
+ pair<InputIterator,ForwardIterator> uninitialized_move_n(InputIterator first, Size n, ForwardIterator result);
+
+namespace ranges {
+
+template<class InputIterator, class OutputIterator>
+using uninitialized_move_n_result = in_out_result<InputIterator, OutputIterator>; // since C++20
+
+template<input_iterator InputIterator, nothrow-forward-iterator OutputIterator, nothrow-sentinel-for<OutputIterator> Sentinel>
+  requires constructible_from<iter_value_t<OutputIterator>, iter_rvalue_reference_t<InputIterator>>
+uninitialized_move_n_result<InputIterator, OutputIterator>
+uninitialized_move_n(InputIterator ifirst, iter_difference_t<InputIterator> n, OutputIterator ofirst, Sentinel olast); // since C++20
+
+}
+
+template <class ForwardIterator>
+ void uninitialized_value_construct(ForwardIterator first, ForwardIterator last);
+
+namespace ranges {
+
+template <nothrow-forward-iterator ForwardIterator, nothrow-sentinel-for<ForwardIterator> Sentinel>
+  requires default_initializable<iter_value_t<ForwardIterator>>
+ ForwardIterator uninitialized_value_construct(ForwardIterator first, Sentinel last); // since C++20
+
+template <nothrow-forward-range ForwardRange>
+  requires default_initializable<range_value_t<ForwardRange>>
+ borrowed_iterator_t<ForwardRange> uninitialized_value_construct(ForwardRange&& r); // since C++20
+
+}
+
+template <class ForwardIterator, class Size>
+ ForwardIterator uninitialized_value_construct_n(ForwardIterator first, Size n);
+
+namespace ranges {
+
+template <nothrow-forward-iterator ForwardIterator>
+  requires default_initializable<iter_value_t<ForwardIterator>>
+ ForwardIterator uninitialized_value_construct_n(ForwardIterator first, iter_difference_t<ForwardIterator> n); // since C++20
+
+}
+
+template <class ForwardIterator>
+ void uninitialized_default_construct(ForwardIterator first, ForwardIterator last);
+
+namespace ranges {
+
+template <nothrow-forward-iterator ForwardIterator, nothrow-sentinel-for<ForwardIterator> Sentinel>
+  requires default_initializable<iter_value_t<ForwardIterator>>
+ ForwardIterator uninitialized_default_construct(ForwardIterator first, Sentinel last); // since C++20
+
+template <nothrow-forward-range ForwardRange>
+  requires default_initializable<range_value_t<ForwardRange>>
+ borrowed_iterator_t<ForwardRange> uninitialized_default_construct(ForwardRange&& r); // since C++20
+
+}
+
+template <class ForwardIterator, class Size>
+ ForwardIterator uninitialized_default_construct_n(ForwardIterator first, Size n);
+
+namespace ranges {
+
+template <nothrow-forward-iterator ForwardIterator>
+  requires default_initializable<iter_value_t<ForwardIterator>>
+ ForwardIterator uninitialized_default_construct_n(ForwardIterator first, iter_difference_t<ForwardIterator> n); // since C++20
+
+}
+
+template <class Y> struct auto_ptr_ref {};      // deprecated in C++11, removed in C++17
+
+template<class X>
+class auto_ptr                                  // deprecated in C++11, removed in C++17
+{
+public:
+    typedef X element_type;
+
+    explicit auto_ptr(X* p =0) throw();
+    auto_ptr(auto_ptr&) throw();
+    template<class Y> auto_ptr(auto_ptr<Y>&) throw();
+    auto_ptr& operator=(auto_ptr&) throw();
+    template<class Y> auto_ptr& operator=(auto_ptr<Y>&) throw();
+    auto_ptr& operator=(auto_ptr_ref<X> r) throw();
+    ~auto_ptr() throw();
+
+    typename add_lvalue_reference<X>::type operator*() const throw();
+    X* operator->() const throw();
+    X* get() const throw();
+    X* release() throw();
+    void reset(X* p =0) throw();
+
+    auto_ptr(auto_ptr_ref<X>) throw();
+    template<class Y> operator auto_ptr_ref<Y>() throw();
+    template<class Y> operator auto_ptr<Y>() throw();
+};
+
+template <class T>
+struct default_delete
+{
+    constexpr default_delete() noexcept = default;
+    template <class U> constexpr default_delete(const default_delete<U>&) noexcept; // constexpr since C++23
+
+    constexpr void operator()(T*) const noexcept;                                   // constexpr since C++23
+};
+
+template <class T>
+struct default_delete<T[]>
+{
+    constexpr default_delete() noexcept = default;
+    template <class U> constexpr default_delete(const default_delete <U[]>&) noexcept; // constexpr since C++23
+    constexpr void operator()(T*) const noexcept;                                      // constexpr since C++23
+    template <class U> void operator()(U*) const = delete;
+};
+
+template <class T, class D = default_delete<T>>
+class unique_ptr
+{
+public:
+    typedef see below pointer;
+    typedef T element_type;
+    typedef D deleter_type;
+
+    // constructors
+    constexpr unique_ptr() noexcept;
+    constexpr explicit unique_ptr(pointer p) noexcept;           // constexpr since C++23
+    constexpr unique_ptr(pointer p, see below d1) noexcept;      // constexpr since C++23
+    constexpr unique_ptr(pointer p, see below d2) noexcept;      // constexpr since C++23
+    constexpr unique_ptr(unique_ptr&& u) noexcept;               // constexpr since C++23
+    constexpr unique_ptr(nullptr_t) noexcept : unique_ptr() { }
+    template <class U, class E>
+        constexpr unique_ptr(unique_ptr<U, E>&& u) noexcept;     // constexpr since C++23
+    template <class U>
+        unique_ptr(auto_ptr<U>&& u) noexcept;                    // removed in C++17
+
+    // destructor
+    constexpr ~unique_ptr();                                     // constexpr since C++23
+
+    // assignment
+    constexpr unique_ptr& operator=(unique_ptr&& u) noexcept;                         // constexpr since C++23
+    template <class U, class E>
+    constexpr unique_ptr& operator=(unique_ptr<U, E>&& u) noexcept;                   // constexpr since C++23
+    constexpr unique_ptr& operator=(nullptr_t) noexcept;                              // constexpr since C++23
+
+    // observers
+    constexpr
+    add_lvalue_reference<T>::type operator*() const noexcept(see below);              // constexpr since C++23
+    constexpr pointer operator->() const noexcept;                                    // constexpr since C++23
+    constexpr pointer get() const noexcept;                                           // constexpr since C++23
+    constexpr deleter_type& get_deleter() noexcept;                                   // constexpr since C++23
+    constexpr const deleter_type& get_deleter() const noexcept;                       // constexpr since C++23
+    constexpr explicit operator bool() const noexcept;                                // constexpr since C++23
+
+    // modifiers
+    constexpr pointer release() noexcept;                                             // constexpr since C++23
+    constexpr void reset(pointer p = pointer()) noexcept;                             // constexpr since C++23
+    constexpr void swap(unique_ptr& u) noexcept;                                      // constexpr since C++23
+};
+
+template <class T, class D>
+class unique_ptr<T[], D>
+{
+public:
+    typedef implementation-defined pointer;
+    typedef T element_type;
+    typedef D deleter_type;
+
+    // constructors
+    constexpr unique_ptr() noexcept;
+    constexpr explicit unique_ptr(pointer p) noexcept;          // constexpr since C++23
+    constexpr unique_ptr(pointer p, see below d) noexcept;      // constexpr since C++23
+    constexpr unique_ptr(pointer p, see below d) noexcept;      // constexpr since C++23
+    constexpr unique_ptr(unique_ptr&& u) noexcept;              // constexpr since C++23
+    template <class U, class E>
+    constexpr unique_ptr(unique_ptr <U, E>&& u) noexcept;       // constexpr since C++23
+    constexpr unique_ptr(nullptr_t) noexcept : unique_ptr() { }
+
+    // destructor
+    constexpr ~unique_ptr();                                    // constexpr since C++23
+
+    // assignment
+    constexpr unique_ptr& operator=(unique_ptr&& u) noexcept;        // constexpr since C++23
+    template <class U, class E>
+    constexpr unique_ptr& operator=(unique_ptr <U, E>&& u) noexcept; // constexpr since C++23
+    constexpr unique_ptr& operator=(nullptr_t) noexcept;             // constexpr since C++23
+
+    // observers
+    constexpr T& operator[](size_t i) const;                    // constexpr since C++23
+    constexpr pointer get() const noexcept;                     // constexpr since C++23
+    constexpr deleter_type& get_deleter() noexcept;             // constexpr since C++23
+    constexpr const deleter_type& get_deleter() const noexcept; // constexpr since C++23
+    constexpr explicit operator bool() const noexcept;          // constexpr since C++23
+
+    // modifiers
+    constexpr pointer release() noexcept;                       // constexpr since C++23
+    constexpr void reset(pointer p = pointer()) noexcept;       // constexpr since C++23
+    constexpr void reset(nullptr_t) noexcept;                   // constexpr since C++23
+  template <class U> void reset(U) = delete;
+    constexpr void swap(unique_ptr& u) noexcept;                // constexpr since C++23
+};
+
+template <class T, class D>
+    constexpr void swap(unique_ptr<T, D>& x, unique_ptr<T, D>& y) noexcept;                 // constexpr since C++23
+
+template <class T1, class D1, class T2, class D2>
+    constexpr bool operator==(const unique_ptr<T1, D1>& x, const unique_ptr<T2, D2>& y);    // constexpr since C++23
+template <class T1, class D1, class T2, class D2>
+    bool operator!=(const unique_ptr<T1, D1>& x, const unique_ptr<T2, D2>& y);              // removed in C++20
+template <class T1, class D1, class T2, class D2>
+    bool operator<(const unique_ptr<T1, D1>& x, const unique_ptr<T2, D2>& y);
+template <class T1, class D1, class T2, class D2>
+    bool operator<=(const unique_ptr<T1, D1>& x, const unique_ptr<T2, D2>& y);
+template <class T1, class D1, class T2, class D2>
+    bool operator>(const unique_ptr<T1, D1>& x, const unique_ptr<T2, D2>& y);
+template <class T1, class D1, class T2, class D2>
+    bool operator>=(const unique_ptr<T1, D1>& x, const unique_ptr<T2, D2>& y);
+template<class T1, class D1, class T2, class D2>
+  requires three_way_comparable_with<typename unique_ptr<T1, D1>::pointer,
+                                     typename unique_ptr<T2, D2>::pointer>
+  compare_three_way_result_t<typename unique_ptr<T1, D1>::pointer,
+                             typename unique_ptr<T2, D2>::pointer>
+    operator<=>(const unique_ptr<T1, D1>& x, const unique_ptr<T2, D2>& y);      // C++20
+
+template <class T, class D>
+    constexpr bool operator==(const unique_ptr<T, D>& x, nullptr_t) noexcept;   // constexpr since C++23
+template <class T, class D>
+    bool operator==(nullptr_t, const unique_ptr<T, D>& y) noexcept;             // removed in C++20
+template <class T, class D>
+    bool operator!=(const unique_ptr<T, D>& x, nullptr_t) noexcept;             // removed in C++20
+template <class T, class D>
+    bool operator!=(nullptr_t, const unique_ptr<T, D>& y) noexcept;             // removed in C++20
+
+template <class T, class D>
+    constexpr bool operator<(const unique_ptr<T, D>& x, nullptr_t);     // constexpr since C++23
+template <class T, class D>
+    constexpr bool operator<(nullptr_t, const unique_ptr<T, D>& y);     // constexpr since C++23
+template <class T, class D>
+    constexpr bool operator<=(const unique_ptr<T, D>& x, nullptr_t);    // constexpr since C++23
+template <class T, class D>
+    constexpr bool operator<=(nullptr_t, const unique_ptr<T, D>& y);    // constexpr since C++23
+template <class T, class D>
+    constexpr bool operator>(const unique_ptr<T, D>& x, nullptr_t);     // constexpr since C++23
+template <class T, class D>
+    constexpr bool operator>(nullptr_t, const unique_ptr<T, D>& y);     // constexpr since C++23
+template <class T, class D>
+    constexpr bool operator>=(const unique_ptr<T, D>& x, nullptr_t);    // constexpr since C++23
+template <class T, class D>
+    constexpr bool operator>=(nullptr_t, const unique_ptr<T, D>& y);    // constexpr since C++23
+template<class T, class D>
+  requires three_way_comparable<typename unique_ptr<T, D>::pointer>
+  compare_three_way_result_t<typename unique_ptr<T, D>::pointer>
+    constexpr operator<=>(const unique_ptr<T, D>& x, nullptr_t);        // C++20, constexpr since C++23
+
+class bad_weak_ptr
+    : public std::exception
+{
+    bad_weak_ptr() noexcept;
+};
+
+template<class T, class... Args>
+constexpr unique_ptr<T> make_unique(Args&&... args);                            // C++14, constexpr since C++23
+template<class T>
+constexpr unique_ptr<T> make_unique(size_t n);                                  // C++14, constexpr since C++23
+template<class T, class... Args> unspecified   make_unique(Args&&...) = delete; // C++14, T == U[N]
+
+template<class T>
+  constexpr unique_ptr<T> make_unique_for_overwrite();                        // T is not array, C++20, constexpr since C++23
+template<class T>
+  constexpr unique_ptr<T> make_unique_for_overwrite(size_t n);                // T is U[], C++20, constexpr since C++23
+template<class T, class... Args>
+  unspecified make_unique_for_overwrite(Args&&...) = delete;                  // T is U[N], C++20
+
+template<class E, class T, class Y, class D>
+    basic_ostream<E, T>& operator<< (basic_ostream<E, T>& os, unique_ptr<Y, D> const& p);
+
+template<class T>
+class shared_ptr
+{
+public:
+    typedef T element_type; // until C++17
+    typedef remove_extent_t<T> element_type; // since C++17
+    typedef weak_ptr<T> weak_type; // C++17
+
+    // constructors:
+    constexpr shared_ptr() noexcept;
+    template<class Y> explicit shared_ptr(Y* p);
+    template<class Y, class D> shared_ptr(Y* p, D d);
+    template<class Y, class D, class A> shared_ptr(Y* p, D d, A a);
+    template <class D> shared_ptr(nullptr_t p, D d);
+    template <class D, class A> shared_ptr(nullptr_t p, D d, A a);
+    template<class Y> shared_ptr(const shared_ptr<Y>& r, T *p) noexcept;
+    shared_ptr(const shared_ptr& r) noexcept;
+    template<class Y> shared_ptr(const shared_ptr<Y>& r) noexcept;
+    shared_ptr(shared_ptr&& r) noexcept;
+    template<class Y> shared_ptr(shared_ptr<Y>&& r) noexcept;
+    template<class Y> explicit shared_ptr(const weak_ptr<Y>& r);
+    template<class Y> shared_ptr(auto_ptr<Y>&& r);          // removed in C++17
+    template <class Y, class D> shared_ptr(unique_ptr<Y, D>&& r);
+    shared_ptr(nullptr_t) : shared_ptr() { }
+
+    // destructor:
+    ~shared_ptr();
+
+    // assignment:
+    shared_ptr& operator=(const shared_ptr& r) noexcept;
+    template<class Y> shared_ptr& operator=(const shared_ptr<Y>& r) noexcept;
+    shared_ptr& operator=(shared_ptr&& r) noexcept;
+    template<class Y> shared_ptr& operator=(shared_ptr<Y>&& r);
+    template<class Y> shared_ptr& operator=(auto_ptr<Y>&& r); // removed in C++17
+    template <class Y, class D> shared_ptr& operator=(unique_ptr<Y, D>&& r);
+
+    // modifiers:
+    void swap(shared_ptr& r) noexcept;
+    void reset() noexcept;
+    template<class Y> void reset(Y* p);
+    template<class Y, class D> void reset(Y* p, D d);
+    template<class Y, class D, class A> void reset(Y* p, D d, A a);
+
+    // observers:
+    T* get() const noexcept;
+    T& operator*() const noexcept;
+    T* operator->() const noexcept;
+    long use_count() const noexcept;
+    bool unique() const noexcept;  // deprected in C++17, removed in C++20
+    explicit operator bool() const noexcept;
+    template<class U> bool owner_before(shared_ptr<U> const& b) const noexcept;
+    template<class U> bool owner_before(weak_ptr<U> const& b) const noexcept;
+};
+
+template<class T>
+shared_ptr(weak_ptr<T>) -> shared_ptr<T>;
+template<class T, class D>
+shared_ptr(unique_ptr<T, D>) -> shared_ptr<T>;
+
+// shared_ptr comparisons:
+template<class T, class U>
+    bool operator==(shared_ptr<T> const& a, shared_ptr<U> const& b) noexcept;
+template<class T, class U>
+    bool operator!=(shared_ptr<T> const& a, shared_ptr<U> const& b) noexcept;               // removed in C++20
+template<class T, class U>
+    bool operator<(shared_ptr<T> const& a, shared_ptr<U> const& b) noexcept;                // removed in C++20
+template<class T, class U>
+    bool operator>(shared_ptr<T> const& a, shared_ptr<U> const& b) noexcept;                // removed in C++20
+template<class T, class U>
+    bool operator<=(shared_ptr<T> const& a, shared_ptr<U> const& b) noexcept;               // removed in C++20
+template<class T, class U>
+    bool operator>=(shared_ptr<T> const& a, shared_ptr<U> const& b) noexcept;               // removed in C++20
+template<class T, class U>
+    strong_ordering operator<=>(shared_ptr<T> const& a, shared_ptr<U> const& b) noexcept;   // C++20
+
+template <class T>
+    bool operator==(const shared_ptr<T>& x, nullptr_t) noexcept;
+template <class T>
+    bool operator==(nullptr_t, const shared_ptr<T>& y) noexcept;               // removed in C++20
+template <class T>
+    bool operator!=(const shared_ptr<T>& x, nullptr_t) noexcept;               // removed in C++20
+template <class T>
+    bool operator!=(nullptr_t, const shared_ptr<T>& y) noexcept;               // removed in C++20
+template <class T>
+    bool operator<(const shared_ptr<T>& x, nullptr_t) noexcept;                // removed in C++20
+template <class T>
+    bool operator<(nullptr_t, const shared_ptr<T>& y) noexcept;                // removed in C++20
+template <class T>
+    bool operator<=(const shared_ptr<T>& x, nullptr_t) noexcept;               // removed in C++20
+template <class T>
+    bool operator<=(nullptr_t, const shared_ptr<T>& y) noexcept;               // removed in C++20
+template <class T>
+    bool operator>(const shared_ptr<T>& x, nullptr_t) noexcept;                // removed in C++20
+template <class T>
+    bool operator>(nullptr_t, const shared_ptr<T>& y) noexcept;                // removed in C++20
+template <class T>
+    bool operator>=(const shared_ptr<T>& x, nullptr_t) noexcept;               // removed in C++20
+template <class T>
+    bool operator>=(nullptr_t, const shared_ptr<T>& y) noexcept;               // removed in C++20
+template<class T>
+    strong_ordering operator<=>(shared_ptr<T> const& x, nullptr_t) noexcept;   // C++20
+
+// shared_ptr specialized algorithms:
+template<class T> void swap(shared_ptr<T>& a, shared_ptr<T>& b) noexcept;
+
+// shared_ptr casts:
+template<class T, class U>
+    shared_ptr<T> static_pointer_cast(shared_ptr<U> const& r) noexcept;
+template<class T, class U>
+    shared_ptr<T> dynamic_pointer_cast(shared_ptr<U> const& r) noexcept;
+template<class T, class U>
+    shared_ptr<T> const_pointer_cast(shared_ptr<U> const& r) noexcept;
+
+// shared_ptr I/O:
+template<class E, class T, class Y>
+    basic_ostream<E, T>& operator<< (basic_ostream<E, T>& os, shared_ptr<Y> const& p);
+
+// shared_ptr get_deleter:
+template<class D, class T> D* get_deleter(shared_ptr<T> const& p) noexcept;
+
+template<class T, class... Args>
+    shared_ptr<T> make_shared(Args&&... args); // T is not an array
+template<class T, class A, class... Args>
+    shared_ptr<T> allocate_shared(const A& a, Args&&... args); // T is not an array
+
+template<class T>
+    shared_ptr<T> make_shared(size_t N); // T is U[] (since C++20)
+template<class T, class A>
+    shared_ptr<T> allocate_shared(const A& a, size_t N); // T is U[] (since C++20)
+
+template<class T>
+    shared_ptr<T> make_shared(); // T is U[N] (since C++20)
+template<class T, class A>
+    shared_ptr<T> allocate_shared(const A& a); // T is U[N] (since C++20)
+
+template<class T>
+    shared_ptr<T> make_shared(size_t N, const remove_extent_t<T>& u); // T is U[] (since C++20)
+template<class T, class A>
+    shared_ptr<T> allocate_shared(const A& a, size_t N, const remove_extent_t<T>& u); // T is U[] (since C++20)
+
+template<class T> shared_ptr<T>
+    make_shared(const remove_extent_t<T>& u); // T is U[N] (since C++20)
+template<class T, class A>
+    shared_ptr<T> allocate_shared(const A& a, const remove_extent_t<T>& u); // T is U[N] (since C++20)
+
+template<class T>
+  shared_ptr<T> make_shared_for_overwrite();                                  // T is not U[], C++20
+template<class T, class A>
+  shared_ptr<T> allocate_shared_for_overwrite(const A& a);                    // T is not U[], C++20
+
+template<class T>
+  shared_ptr<T> make_shared_for_overwrite(size_t N);                          // T is U[], C++20
+template<class T, class A>
+  shared_ptr<T> allocate_shared_for_overwrite(const A& a, size_t N);          // T is U[], C++20
+
+template<class T>
+class weak_ptr
+{
+public:
+    typedef T element_type; // until C++17
+    typedef remove_extent_t<T> element_type; // since C++17
+
+    // constructors
+    constexpr weak_ptr() noexcept;
+    template<class Y> weak_ptr(shared_ptr<Y> const& r) noexcept;
+    weak_ptr(weak_ptr const& r) noexcept;
+    template<class Y> weak_ptr(weak_ptr<Y> const& r) noexcept;
+    weak_ptr(weak_ptr&& r) noexcept;                      // C++14
+    template<class Y> weak_ptr(weak_ptr<Y>&& r) noexcept; // C++14
+
+    // destructor
+    ~weak_ptr();
+
+    // assignment
+    weak_ptr& operator=(weak_ptr const& r) noexcept;
+    template<class Y> weak_ptr& operator=(weak_ptr<Y> const& r) noexcept;
+    template<class Y> weak_ptr& operator=(shared_ptr<Y> const& r) noexcept;
+    weak_ptr& operator=(weak_ptr&& r) noexcept;                      // C++14
+    template<class Y> weak_ptr& operator=(weak_ptr<Y>&& r) noexcept; // C++14
+
+    // modifiers
+    void swap(weak_ptr& r) noexcept;
+    void reset() noexcept;
+
+    // observers
+    long use_count() const noexcept;
+    bool expired() const noexcept;
+    shared_ptr<T> lock() const noexcept;
+    template<class U> bool owner_before(shared_ptr<U> const& b) const noexcept;
+    template<class U> bool owner_before(weak_ptr<U> const& b) const noexcept;
+};
+
+template<class T>
+weak_ptr(shared_ptr<T>) -> weak_ptr<T>;
+
+// weak_ptr specialized algorithms:
+template<class T> void swap(weak_ptr<T>& a, weak_ptr<T>& b) noexcept;
+
+// class owner_less:
+template<class T> struct owner_less;
+
+template<class T>
+struct owner_less<shared_ptr<T> >
+    : binary_function<shared_ptr<T>, shared_ptr<T>, bool>
+{
+    typedef bool result_type;
+    bool operator()(shared_ptr<T> const&, shared_ptr<T> const&) const noexcept;
+    bool operator()(shared_ptr<T> const&, weak_ptr<T> const&) const noexcept;
+    bool operator()(weak_ptr<T> const&, shared_ptr<T> const&) const noexcept;
+};
+
+template<class T>
+struct owner_less<weak_ptr<T> >
+    : binary_function<weak_ptr<T>, weak_ptr<T>, bool>
+{
+    typedef bool result_type;
+    bool operator()(weak_ptr<T> const&, weak_ptr<T> const&) const noexcept;
+    bool operator()(shared_ptr<T> const&, weak_ptr<T> const&) const noexcept;
+    bool operator()(weak_ptr<T> const&, shared_ptr<T> const&) const noexcept;
+};
+
+template <>  // Added in C++14
+struct owner_less<void>
+{
+    template <class _Tp, class _Up>
+    bool operator()( shared_ptr<_Tp> const& __x, shared_ptr<_Up> const& __y) const noexcept;
+    template <class _Tp, class _Up>
+    bool operator()( shared_ptr<_Tp> const& __x,   weak_ptr<_Up> const& __y) const noexcept;
+    template <class _Tp, class _Up>
+    bool operator()(   weak_ptr<_Tp> const& __x, shared_ptr<_Up> const& __y) const noexcept;
+    template <class _Tp, class _Up>
+    bool operator()(   weak_ptr<_Tp> const& __x,   weak_ptr<_Up> const& __y) const noexcept;
+
+    typedef void is_transparent;
+};
+
+template<class T>
+class enable_shared_from_this
+{
+protected:
+    constexpr enable_shared_from_this() noexcept;
+    enable_shared_from_this(enable_shared_from_this const&) noexcept;
+    enable_shared_from_this& operator=(enable_shared_from_this const&) noexcept;
+    ~enable_shared_from_this();
+public:
+    shared_ptr<T> shared_from_this();
+    shared_ptr<T const> shared_from_this() const;
+};
+
+template<class T>
+    bool atomic_is_lock_free(const shared_ptr<T>* p);
+template<class T>
+    shared_ptr<T> atomic_load(const shared_ptr<T>* p);
+template<class T>
+    shared_ptr<T> atomic_load_explicit(const shared_ptr<T>* p, memory_order mo);
+template<class T>
+    void atomic_store(shared_ptr<T>* p, shared_ptr<T> r);
+template<class T>
+    void atomic_store_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo);
+template<class T>
+    shared_ptr<T> atomic_exchange(shared_ptr<T>* p, shared_ptr<T> r);
+template<class T>
+    shared_ptr<T>
+    atomic_exchange_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo);
+template<class T>
+    bool
+    atomic_compare_exchange_weak(shared_ptr<T>* p, shared_ptr<T>* v, shared_ptr<T> w);
+template<class T>
+    bool
+    atomic_compare_exchange_strong( shared_ptr<T>* p, shared_ptr<T>* v, shared_ptr<T> w);
+template<class T>
+    bool
+    atomic_compare_exchange_weak_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
+                                          shared_ptr<T> w, memory_order success,
+                                          memory_order failure);
+template<class T>
+    bool
+    atomic_compare_exchange_strong_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
+                                            shared_ptr<T> w, memory_order success,
+                                            memory_order failure);
+// Hash support
+template <class T> struct hash;
+template <class T, class D> struct hash<unique_ptr<T, D> >;
+template <class T> struct hash<shared_ptr<T> >;
+
+template <class T, class Alloc>
+  inline constexpr bool uses_allocator_v = uses_allocator<T, Alloc>::value;
+
+// [allocator.uses.construction], uses-allocator construction
+template<class T, class Alloc, class... Args>
+  constexpr auto uses_allocator_construction_args(const Alloc& alloc,             // since C++20
+                                                  Args&&... args) noexcept;
+template<class T, class Alloc, class Tuple1, class Tuple2>
+  constexpr auto uses_allocator_construction_args(const Alloc& alloc,             // since C++20
+                                                  piecewise_construct_t,
+                                                  Tuple1&& x, Tuple2&& y) noexcept;
+template<class T, class Alloc>
+  constexpr auto uses_allocator_construction_args(const Alloc& alloc) noexcept;   // since C++20
+template<class T, class Alloc, class U, class V>
+  constexpr auto uses_allocator_construction_args(const Alloc& alloc,             // since C++20
+                                                  U&& u, V&& v) noexcept;
+template<class T, class Alloc, class U, class V>
+  constexpr auto uses_allocator_construction_args(const Alloc& alloc,             // since C++23
+                                                  pair<U, V>& pr) noexcept;
+template<class T, class Alloc, class U, class V>
+  constexpr auto uses_allocator_construction_args(const Alloc& alloc,             // since C++20
+                                                  const pair<U, V>& pr) noexcept;
+template<class T, class Alloc, class U, class V>
+  constexpr auto uses_allocator_construction_args(const Alloc& alloc,             // since C++20
+                                                  pair<U, V>&& pr) noexcept;
+template<class T, class Alloc, class U, class V>
+  constexpr auto uses_allocator_construction_args(const Alloc& alloc,             // since C++23
+                                                  const pair<U, V>&& pr) noexcept;
+template<class T, class Alloc, pair-like P>
+  constexpr auto uses_allocator_construction_args(const Alloc& alloc,             // since C++20
+                                                  P&& p) noexcept;
+template<class T, class Alloc, class U>
+  constexpr auto uses_allocator_construction_args(const Alloc& alloc,             // since C++20
+                                                  U&& u) noexcept;
+template<class T, class Alloc, class... Args>
+  constexpr T make_obj_using_allocator(const Alloc& alloc, Args&&... args);       // since C++20
+template<class T, class Alloc, class... Args>
+  constexpr T* uninitialized_construct_using_allocator(T* p,                      // since C++20
+                                                         const Alloc& alloc, Args&&... args);
+
+// [ptr.align]
+void* align(size_t alignment, size_t size, void*& ptr, size_t& space);
+
+template<size_t N, class T>
+[[nodiscard]] constexpr T* assume_aligned(T* ptr); // since C++20
+
+// [out.ptr.t], class template out_ptr_t
+template<class Smart, class Pointer, class... Args>
+  class out_ptr_t;                                          // since c++23
+
+// [out.ptr], function template out_ptr
+template<class Pointer = void, class Smart, class... Args>
+  auto out_ptr(Smart& s, Args&&... args);                   // since c++23
+
+// [inout.ptr.t], class template inout_ptr_t
+template<class Smart, class Pointer, class... Args>
+  class inout_ptr_t;                                        // since c++23
+
+// [inout.ptr], function template inout_ptr
+template<class Pointer = void, class Smart, class... Args>
+  auto inout_ptr(Smart& s, Args&&... args);                 // since c++23
+
+}  // std
+
+*/
+
+// clang-format on
+
+#include <__config>
+#include <__memory/addressof.h>
+#include <__memory/align.h>
+#include <__memory/allocator.h>
+#include <__memory/allocator_arg_t.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/auto_ptr.h>
+#include <__memory/inout_ptr.h>
+#include <__memory/out_ptr.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/raw_storage_iterator.h>
+#include <__memory/shared_ptr.h>
+#include <__memory/temporary_buffer.h>
+#include <__memory/uninitialized_algorithms.h>
+#include <__memory/unique_ptr.h>
+#include <__memory/uses_allocator.h>
+
+// standard-mandated includes
+
+#if _LIBCPP_STD_VER >= 17
+#  include <__memory/construct_at.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__memory/assume_aligned.h>
+#  include <__memory/concepts.h>
+#  include <__memory/ranges_construct_at.h>
+#  include <__memory/ranges_uninitialized_algorithms.h>
+#  include <__memory/uses_allocator_construction.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+#  include <__memory/allocate_at_least.h>
+#endif
+
+#include <version>
+
+// [memory.syn]
+#include <compare>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <concepts>
+#  include <cstddef>
+#  include <cstdint>
+#  include <cstdlib>
+#  include <cstring>
+#  include <iosfwd>
+#  include <iterator>
+#  include <new>
+#  include <stdexcept>
+#  include <tuple>
+#  include <type_traits>
+#  include <typeinfo>
+#  include <utility>
+#endif
+
+#endif // _LIBCPP_MEMORY
diff --git a/libcxx/include/__cxx03/memory_resource b/libcxx/include/__cxx03/memory_resource
new file mode 100644
index 000000000000..67411054820a
--- /dev/null
+++ b/libcxx/include/__cxx03/memory_resource
@@ -0,0 +1,83 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_MEMORY_RESOURCE
+#define _LIBCPP_MEMORY_RESOURCE
+
+/**
+    memory_resource synopsis
+
+// C++17
+
+namespace std::pmr {
+
+  class memory_resource;
+
+  bool operator==(const memory_resource& a,
+                  const memory_resource& b) noexcept;
+  bool operator!=(const memory_resource& a,
+                  const memory_resource& b) noexcept;           // removed in C++20
+
+  template <class Tp> class polymorphic_allocator;
+
+  template <class T1, class T2>
+  bool operator==(const polymorphic_allocator<T1>& a,
+                  const polymorphic_allocator<T2>& b) noexcept;
+  template <class T1, class T2>
+  bool operator!=(const polymorphic_allocator<T1>& a,
+                  const polymorphic_allocator<T2>& b) noexcept; // removed in C++20
+
+  // Global memory resources
+  memory_resource* set_default_resource(memory_resource* r) noexcept;
+  memory_resource* get_default_resource() noexcept;
+  memory_resource* new_delete_resource() noexcept;
+  memory_resource* null_memory_resource() noexcept;
+
+  // Pool resource classes
+  struct pool_options;
+  class synchronized_pool_resource;
+  class unsynchronized_pool_resource;
+  class monotonic_buffer_resource;
+
+} // namespace std::pmr
+
+ */
+
+#include <__config>
+
+#if _LIBCPP_STD_VER >= 17
+#  include <__memory_resource/memory_resource.h>
+#  include <__memory_resource/monotonic_buffer_resource.h>
+#  include <__memory_resource/polymorphic_allocator.h>
+#  include <__memory_resource/pool_options.h>
+#  include <__memory_resource/synchronized_pool_resource.h>
+#  include <__memory_resource/unsynchronized_pool_resource.h>
+#endif
+
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 14
+#  include <cstddef>
+#  include <cstdint>
+#  include <limits>
+#  include <mutex>
+#  include <new>
+#  include <stdexcept>
+#  include <tuple>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <stdexcept>
+#endif
+
+#endif /* _LIBCPP_MEMORY_RESOURCE */
diff --git a/libcxx/include/__cxx03/module.modulemap b/libcxx/include/__cxx03/module.modulemap
new file mode 100644
index 000000000000..13d0dce34d97
--- /dev/null
+++ b/libcxx/include/__cxx03/module.modulemap
@@ -0,0 +1,2136 @@
+// Main C++ standard library interfaces
+module std_algorithm [system] {
+  header "algorithm"
+  export *
+}
+module std_any [system] {
+  header "any"
+  export *
+}
+module std_array [system] {
+  header "array"
+  export *
+}
+module std_atomic [system] {
+  header "atomic"
+  export *
+}
+module std_barrier [system] {
+  header "barrier"
+  export *
+}
+module std_bit [system] {
+  header "bit"
+  export *
+}
+module std_bitset [system] {
+  header "bitset"
+  export *
+}
+module std_charconv [system] {
+  header "charconv"
+  export *
+}
+module std_chrono [system] {
+  header "chrono"
+  export *
+}
+module std_codecvt [system] {
+  header "codecvt"
+  export *
+}
+module std_compare [system] {
+  header "compare"
+  export *
+}
+module std_complex [system] {
+  header "complex"
+  export *
+}
+module std_concepts [system] {
+  header "concepts"
+  export *
+}
+module std_condition_variable [system] {
+  header "condition_variable"
+  export *
+}
+module std_coroutine [system] {
+  header "coroutine"
+  export *
+}
+module std_deque [system] {
+  header "deque"
+  export *
+}
+module std_exception [system] {
+  header "exception"
+  export *
+}
+module std_execution [system] {
+  header "execution"
+  export *
+}
+module std_expected [system] {
+  header "expected"
+  export *
+}
+module std_filesystem [system] {
+  header "filesystem"
+  export *
+}
+module std_format [system] {
+  header "format"
+  export *
+}
+module std_forward_list [system] {
+  header "forward_list"
+  export *
+}
+module std_fstream [system] {
+  header "fstream"
+  export *
+}
+module std_functional [system] {
+  header "functional"
+  export *
+}
+module std_future [system] {
+  header "future"
+  export *
+}
+module std_initializer_list [system] {
+  header "initializer_list"
+  export *
+}
+module std_iomanip [system] {
+  header "iomanip"
+  export *
+}
+module std_ios [system] {
+  header "ios"
+  export *
+}
+module std_iosfwd [system] {
+  header "iosfwd"
+  export *
+}
+module std_iostream [system] {
+  header "iostream"
+  export *
+}
+module std_istream [system] {
+  header "istream"
+  export *
+}
+module std_iterator [system] {
+  header "iterator"
+  export *
+}
+module std_latch [system] {
+  header "latch"
+  export *
+}
+module std_limits [system] {
+  header "limits"
+  export *
+}
+module std_list [system] {
+  header "list"
+  export *
+}
+module std_locale [system] {
+  header "locale"
+  export *
+}
+module std_map [system] {
+  header "map"
+  export *
+}
+module std_mdspan [system] {
+  header "mdspan"
+  export *
+}
+module std_memory [system] {
+  header "memory"
+  export *
+}
+module std_memory_resource [system] {
+  header "memory_resource"
+  export *
+}
+module std_mutex [system] {
+  header "mutex"
+  export *
+}
+module std_new [system] {
+  header "new"
+  export *
+}
+module std_numbers [system] {
+  header "numbers"
+  export *
+}
+module std_numeric [system] {
+  header "numeric"
+  export *
+}
+module std_optional [system] {
+  header "optional"
+  export *
+}
+module std_ostream [system] {
+  header "ostream"
+  export *
+}
+module std_print [system] {
+  header "print"
+  export *
+}
+module std_queue [system] {
+  header "queue"
+  export *
+}
+module std_random [system] {
+  header "random"
+  export *
+}
+module std_ranges [system] {
+  header "ranges"
+  export *
+}
+module std_ratio [system] {
+  header "ratio"
+  export *
+}
+module std_regex [system] {
+  header "regex"
+  export *
+}
+module std_scoped_allocator [system] {
+  header "scoped_allocator"
+  export *
+}
+module std_semaphore [system] {
+  header "semaphore"
+  export *
+}
+module std_set [system] {
+  header "set"
+  export *
+}
+module std_shared_mutex [system] {
+  header "shared_mutex"
+  export std_version
+}
+module std_source_location [system] {
+  header "source_location"
+  export *
+}
+module std_span [system] {
+  header "span"
+  export std_private_ranges_enable_borrowed_range
+  export std_version
+  export std_private_span_span_fwd
+}
+module std_sstream [system] {
+  header "sstream"
+  export *
+}
+module std_stack [system] {
+  header "stack"
+  export *
+}
+module std_stdexcept [system] {
+  header "stdexcept"
+  export *
+}
+module std_stop_token {
+  header "stop_token"
+  export *
+}
+module std_streambuf [system] {
+  header "streambuf"
+  export *
+}
+module std_string [system] {
+  header "string"
+  export *
+}
+module std_string_view [system] {
+  header "string_view"
+  export *
+}
+module std_strstream [system] {
+  header "strstream"
+  export *
+}
+module std_syncstream [system] {
+  header "syncstream"
+  export *
+}
+module std_system_error [system] {
+  header "system_error"
+  export *
+}
+module std_thread [system] {
+  header "thread"
+  export *
+}
+module std_tuple [system] {
+  header "tuple"
+  export *
+}
+module std_type_traits [system] {
+  header "type_traits"
+  export *
+}
+module std_typeindex [system] {
+  header "typeindex"
+  export *
+}
+module std_typeinfo [system] {
+  header "typeinfo"
+  export *
+}
+module std_unordered_map [system] {
+  header "unordered_map"
+  export *
+}
+module std_unordered_set [system] {
+  header "unordered_set"
+  export *
+}
+module std_utility [system] {
+  header "utility"
+  export *
+}
+module std_valarray [system] {
+  header "valarray"
+  export *
+}
+module std_variant [system] {
+  header "variant"
+  export *
+}
+module std_vector [system] {
+  header "vector"
+  export *
+}
+module std_version [system] {
+  header "version"
+  export *
+}
+
+// C standard library interface wrappers
+module std_cassert [system] {
+  // <cassert>'s use of NDEBUG requires textual inclusion.
+  textual header "cassert"
+}
+module std_ccomplex [system] {
+  header "ccomplex"
+  export *
+}
+module std_cctype [system] {
+  header "cctype"
+  export *
+}
+module std_cerrno [system] {
+  header "cerrno"
+  export *
+}
+module std_cfenv [system] {
+  header "cfenv"
+  export *
+}
+module std_cfloat [system] {
+  header "cfloat"
+  export *
+}
+module std_cinttypes [system] {
+  header "cinttypes"
+  export *
+}
+module std_ciso646 [system] {
+  header "ciso646"
+  export *
+}
+module std_climits [system] {
+  header "climits"
+  export *
+}
+module std_clocale [system] {
+  header "clocale"
+  export *
+}
+module std_cmath [system] {
+  header "cmath"
+  export *
+}
+module std_csetjmp [system] {
+  header "csetjmp"
+  export *
+}
+module std_csignal [system] {
+  header "csignal"
+  export *
+}
+// FIXME: <cstdalign> is missing.
+module std_cstdarg [system] {
+  header "cstdarg"
+  export *
+}
+module std_cstdbool [system] {
+  header "cstdbool"
+  export *
+}
+module std_cstddef [system] {
+  header "cstddef"
+  export *
+}
+module std_cstdint [system] {
+  header "cstdint"
+  export *
+}
+module std_cstdio [system] {
+  header "cstdio"
+  export *
+}
+module std_cstdlib [system] {
+  header "cstdlib"
+  export *
+}
+module std_cstring [system] {
+  header "cstring"
+  export *
+}
+module std_ctgmath [system] {
+  header "ctgmath"
+  export *
+}
+module std_ctime [system] {
+  header "ctime"
+  export *
+}
+module std_cuchar [system] {
+  header "cuchar"
+  export *
+}
+module std_cwchar [system] {
+  header "cwchar"
+  export *
+}
+module std_cwctype [system] {
+  header "cwctype"
+  export *
+}
+
+// C standard library interfaces augmented/replaced in C++
+// <assert.h> provided by C library.
+module std_complex_h [system] {
+  header "complex.h"
+  export *
+}
+module std_ctype_h [system] {
+  header "ctype.h"
+  export *
+}
+module std_errno_h [system] {
+  header "errno.h"
+  export *
+}
+module std_fenv_h [system] {
+  header "fenv.h"
+  export *
+}
+module std_float_h [system] {
+  header "float.h"
+  export *
+}
+module std_inttypes_h [system] {
+  header "inttypes.h"
+  export *
+}
+// <iso646.h> provided by compiler.
+module std_locale_h [system] {
+  header "locale.h"
+  export *
+}
+module std_math_h [system] {
+  header "math.h"
+  export *
+}
+// <setjmp.h> provided by C library.
+// <signal.h> provided by C library.
+// FIXME: <stdalign.h> is missing.
+// <stdarg.h> provided by compiler.
+module std_stdatomic_h [system] {
+  header "stdatomic.h"
+  export *
+}
+module std_stdbool_h [system] {
+  // <stdbool.h>'s __bool_true_false_are_defined macro requires textual inclusion.
+  textual header "stdbool.h"
+  export *
+}
+module std_stddef_h [system] {
+  // <stddef.h>'s __need_* macros require textual inclusion.
+  textual header "stddef.h"
+  export *
+}
+module std_stdint_h [system] {
+  header "stdint.h"
+  export *
+}
+module std_stdio_h [system] {
+  // <stdio.h>'s __need_* macros require textual inclusion.
+  textual header "stdio.h"
+  export *
+}
+module std_stdlib_h [system] {
+  // <stdlib.h>'s __need_* macros require textual inclusion.
+  textual header "stdlib.h"
+  export *
+}
+module std_string_h [system] {
+  header "string.h"
+  export *
+}
+module std_tgmath_h [system] {
+  header "tgmath.h"
+  export *
+}
+module std_uchar_h [system] {
+  header "uchar.h"
+  export *
+}
+// <time.h> provided by C library.
+module std_wchar_h [system] {
+  // <wchar.h>'s __need_* macros require textual inclusion.
+  textual header "wchar.h"
+  export *
+}
+module std_wctype_h [system] {
+  header "wctype.h"
+  export *
+}
+
+// Experimental C++ standard library interfaces
+module std_experimental [system] {
+  module iterator {
+    header "experimental/iterator"
+    export *
+  }
+  module memory {
+    header "experimental/memory"
+    export *
+  }
+  module propagate_const {
+    header "experimental/propagate_const"
+    export *
+  }
+  module simd {
+    module aligned_tag          { private header "experimental/__simd/aligned_tag.h" }
+    module declaration          { private header "experimental/__simd/declaration.h" }
+    module reference            { private header "experimental/__simd/reference.h" }
+    module scalar               { private header "experimental/__simd/scalar.h" }
+    module simd                 { private header "experimental/__simd/simd.h" }
+    module simd_mask            { private header "experimental/__simd/simd_mask.h" }
+    module traits               { private header "experimental/__simd/traits.h" }
+    module utility              { private header "experimental/__simd/utility.h" }
+    module vec_ext              { private header "experimental/__simd/vec_ext.h" }
+
+    header "experimental/simd"
+    export *
+  }
+  module type_traits {
+    header "experimental/type_traits"
+    export *
+  }
+  module utility {
+    header "experimental/utility"
+    export *
+  }
+  module __config {
+    textual header "experimental/__config"
+    export *
+  }
+}
+
+// Convenience method to get all of the above modules in a single import statement.
+// Importing only the needed modules is likely to be more performant.
+module std [system] {
+  header "__std_clang_module"
+  export *
+}
+
+// Implementation detail headers that are private to libc++. These modules
+// must not be directly imported.
+module std_private_assert            [system] {
+  header "__assert"
+  export *
+}
+module std_private_bit_reference     [system] {
+  header "__bit_reference"
+  export *
+}
+module std_private_fwd_bit_reference [system] {
+  header "__fwd/bit_reference.h"
+}
+module std_private_config            [system] {
+  textual header "__config"
+  textual header "__configuration/abi.h"
+  textual header "__configuration/availability.h"
+  textual header "__configuration/compiler.h"
+  textual header "__configuration/language.h"
+  textual header "__configuration/platform.h"
+  export *
+}
+module std_private_hash_table        [system] {
+  header "__hash_table"
+  export *
+}
+module std_private_locale            [system] {
+  header "__locale"
+  export *
+}
+module std_private_mbstate_t         [system] {
+  header "__mbstate_t.h"
+  export *
+}
+module std_private_node_handle       [system] {
+  header "__node_handle"
+  export *
+}
+module std_private_split_buffer      [system] {
+  header "__split_buffer"
+  export *
+}
+module std_private_std_mbstate_t     [system] {
+  header "__std_mbstate_t.h"
+  export *
+}
+module std_private_tree              [system] {
+  header "__tree"
+  export *
+}
+module std_private_undef_macros      [system] {
+  textual header "__undef_macros"
+  export *
+}
+module std_private_verbose_abort     [system] {
+  header "__verbose_abort"
+  export *
+}
+
+module std_private_algorithm_adjacent_find                               [system] { header "__algorithm/adjacent_find.h" }
+module std_private_algorithm_all_of                                      [system] { header "__algorithm/all_of.h" }
+module std_private_algorithm_any_of                                      [system] { header "__algorithm/any_of.h" }
+module std_private_algorithm_binary_search                               [system] { header "__algorithm/binary_search.h" }
+module std_private_algorithm_clamp                                       [system] { header "__algorithm/clamp.h" }
+module std_private_algorithm_comp                                        [system] { header "__algorithm/comp.h" }
+module std_private_algorithm_comp_ref_type                               [system] { header "__algorithm/comp_ref_type.h" }
+module std_private_algorithm_copy                                        [system] {
+  header "__algorithm/copy.h"
+  export std_private_algorithm_copy_move_common
+}
+module std_private_algorithm_copy_backward                               [system] { header "__algorithm/copy_backward.h" }
+module std_private_algorithm_copy_if                                     [system] { header "__algorithm/copy_if.h" }
+module std_private_algorithm_copy_move_common                            [system] {
+  header "__algorithm/copy_move_common.h"
+  export std_private_type_traits_is_trivially_copyable
+}
+module std_private_algorithm_copy_n                                      [system] { header "__algorithm/copy_n.h" }
+module std_private_algorithm_count                                       [system] { header "__algorithm/count.h" }
+module std_private_algorithm_count_if                                    [system] { header "__algorithm/count_if.h" }
+module std_private_algorithm_equal                                       [system] { header "__algorithm/equal.h" }
+module std_private_algorithm_equal_range                                 [system] { header "__algorithm/equal_range.h" }
+module std_private_algorithm_fill                                        [system] { header "__algorithm/fill.h" }
+module std_private_algorithm_fill_n                                      [system] { header "__algorithm/fill_n.h" }
+module std_private_algorithm_find                                        [system] {
+  header "__algorithm/find.h"
+  export std_private_algorithm_unwrap_iter
+}
+module std_private_algorithm_find_end                                    [system] { header "__algorithm/find_end.h" }
+module std_private_algorithm_find_first_of                               [system] { header "__algorithm/find_first_of.h" }
+module std_private_algorithm_find_if                                     [system] { header "__algorithm/find_if.h" }
+module std_private_algorithm_find_if_not                                 [system] { header "__algorithm/find_if_not.h" }
+module std_private_algorithm_find_segment_if                             [system] { header "__algorithm/find_segment_if.h" }
+module std_private_algorithm_fold                                        [system] { header "__algorithm/fold.h" }
+module std_private_algorithm_for_each                                    [system] { header "__algorithm/for_each.h" }
+module std_private_algorithm_for_each_n                                  [system] { header "__algorithm/for_each_n.h" }
+module std_private_algorithm_for_each_segment                            [system] { header "__algorithm/for_each_segment.h" }
+module std_private_algorithm_generate                                    [system] { header "__algorithm/generate.h" }
+module std_private_algorithm_generate_n                                  [system] { header "__algorithm/generate_n.h" }
+module std_private_algorithm_half_positive                               [system] { header "__algorithm/half_positive.h" }
+module std_private_algorithm_in_found_result                             [system] { header "__algorithm/in_found_result.h" }
+module std_private_algorithm_in_fun_result                               [system] { header "__algorithm/in_fun_result.h" }
+module std_private_algorithm_in_in_out_result                            [system] { header "__algorithm/in_in_out_result.h" }
+module std_private_algorithm_in_in_result                                [system] { header "__algorithm/in_in_result.h" }
+module std_private_algorithm_in_out_out_result                           [system] { header "__algorithm/in_out_out_result.h" }
+module std_private_algorithm_in_out_result                               [system] { header "__algorithm/in_out_result.h" }
+module std_private_algorithm_includes                                    [system] { header "__algorithm/includes.h" }
+module std_private_algorithm_inplace_merge                               [system] { header "__algorithm/inplace_merge.h" }
+module std_private_algorithm_is_heap                                     [system] { header "__algorithm/is_heap.h" }
+module std_private_algorithm_is_heap_until                               [system] { header "__algorithm/is_heap_until.h" }
+module std_private_algorithm_is_partitioned                              [system] { header "__algorithm/is_partitioned.h" }
+module std_private_algorithm_is_permutation                              [system] { header "__algorithm/is_permutation.h" }
+module std_private_algorithm_is_sorted                                   [system] { header "__algorithm/is_sorted.h" }
+module std_private_algorithm_is_sorted_until                             [system] { header "__algorithm/is_sorted_until.h" }
+module std_private_algorithm_iter_swap                                   [system] { header "__algorithm/iter_swap.h" }
+module std_private_algorithm_iterator_operations                         [system] {
+  header "__algorithm/iterator_operations.h"
+  export *
+}
+module std_private_algorithm_lexicographical_compare                     [system] { header "__algorithm/lexicographical_compare.h" }
+module std_private_algorithm_lexicographical_compare_three_way           [system] { header "__algorithm/lexicographical_compare_three_way.h" }
+module std_private_algorithm_lower_bound                                 [system] { header "__algorithm/lower_bound.h" }
+module std_private_algorithm_make_heap                                   [system] { header "__algorithm/make_heap.h" }
+module std_private_algorithm_make_projected                              [system] { header "__algorithm/make_projected.h" }
+module std_private_algorithm_max                                         [system] { header "__algorithm/max.h" }
+module std_private_algorithm_max_element                                 [system] { header "__algorithm/max_element.h" }
+module std_private_algorithm_merge                                       [system] { header "__algorithm/merge.h" }
+module std_private_algorithm_min                                         [system] { header "__algorithm/min.h" }
+module std_private_algorithm_min_element                                 [system] { header "__algorithm/min_element.h" }
+module std_private_algorithm_min_max_result                              [system] { header "__algorithm/min_max_result.h" }
+module std_private_algorithm_minmax                                      [system] {
+  header "__algorithm/minmax.h"
+  export *
+}
+module std_private_algorithm_minmax_element                              [system] { header "__algorithm/minmax_element.h" }
+module std_private_algorithm_mismatch                                    [system] {
+  header "__algorithm/mismatch.h"
+  export std_private_algorithm_simd_utils
+  export std_private_iterator_aliasing_iterator
+}
+module std_private_algorithm_move                                        [system] { header "__algorithm/move.h" }
+module std_private_algorithm_move_backward                               [system] { header "__algorithm/move_backward.h" }
+module std_private_algorithm_next_permutation                            [system] { header "__algorithm/next_permutation.h" }
+module std_private_algorithm_none_of                                     [system] { header "__algorithm/none_of.h" }
+module std_private_algorithm_nth_element                                 [system] { header "__algorithm/nth_element.h" }
+module std_private_algorithm_partial_sort                                [system] { header "__algorithm/partial_sort.h" }
+module std_private_algorithm_partial_sort_copy                           [system] { header "__algorithm/partial_sort_copy.h" }
+module std_private_algorithm_partition                                   [system] { header "__algorithm/partition.h" }
+module std_private_algorithm_partition_copy                              [system] { header "__algorithm/partition_copy.h" }
+module std_private_algorithm_partition_point                             [system] { header "__algorithm/partition_point.h" }
+module std_private_algorithm_pop_heap                                    [system] { header "__algorithm/pop_heap.h" }
+module std_private_algorithm_prev_permutation                            [system] { header "__algorithm/prev_permutation.h" }
+module std_private_algorithm_pstl                                        [system] {
+  header "__algorithm/pstl.h"
+  export *
+}
+module std_private_algorithm_push_heap                                   [system] { header "__algorithm/push_heap.h" }
+module std_private_algorithm_ranges_adjacent_find                        [system] { header "__algorithm/ranges_adjacent_find.h" }
+module std_private_algorithm_ranges_all_of                               [system] { header "__algorithm/ranges_all_of.h" }
+module std_private_algorithm_ranges_any_of                               [system] { header "__algorithm/ranges_any_of.h" }
+module std_private_algorithm_ranges_binary_search                        [system] {
+  header "__algorithm/ranges_binary_search.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_clamp                                [system] {
+  header "__algorithm/ranges_clamp.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_contains                             [system] { header "__algorithm/ranges_contains.h" }
+module std_private_algorithm_ranges_contains_subrange                    [system] { header "__algorithm/ranges_contains_subrange.h" }
+module std_private_algorithm_ranges_copy                                 [system] {
+  header "__algorithm/ranges_copy.h"
+  export std_private_algorithm_in_out_result
+}
+module std_private_algorithm_ranges_copy_backward                        [system] {
+  header "__algorithm/ranges_copy_backward.h"
+  export std_private_algorithm_in_out_result
+}
+module std_private_algorithm_ranges_copy_if                              [system] {
+  header "__algorithm/ranges_copy_if.h"
+  export std_private_algorithm_in_out_result
+}
+module std_private_algorithm_ranges_copy_n                               [system] {
+  header "__algorithm/ranges_copy_n.h"
+  export std_private_algorithm_in_out_result
+}
+module std_private_algorithm_ranges_count                                [system] { header "__algorithm/ranges_count.h" }
+module std_private_algorithm_ranges_count_if                             [system] { header "__algorithm/ranges_count_if.h" }
+module std_private_algorithm_ranges_ends_with                            [system] { header "__algorithm/ranges_ends_with.h" }
+module std_private_algorithm_ranges_equal                                [system] { header "__algorithm/ranges_equal.h" }
+module std_private_algorithm_ranges_equal_range                          [system] {
+  header "__algorithm/ranges_equal_range.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_fill                                 [system] { header "__algorithm/ranges_fill.h" }
+module std_private_algorithm_ranges_fill_n                               [system] { header "__algorithm/ranges_fill_n.h" }
+module std_private_algorithm_ranges_find                                 [system] { header "__algorithm/ranges_find.h" }
+module std_private_algorithm_ranges_find_end                             [system] { header "__algorithm/ranges_find_end.h" }
+module std_private_algorithm_ranges_find_first_of                        [system] { header "__algorithm/ranges_find_first_of.h" }
+module std_private_algorithm_ranges_find_if                              [system] { header "__algorithm/ranges_find_if.h" }
+module std_private_algorithm_ranges_find_if_not                          [system] { header "__algorithm/ranges_find_if_not.h" }
+module std_private_algorithm_ranges_find_last                            [system] { header "__algorithm/ranges_find_last.h" }
+module std_private_algorithm_ranges_for_each                             [system] {
+  header "__algorithm/ranges_for_each.h"
+  export std_private_algorithm_in_fun_result
+}
+module std_private_algorithm_ranges_for_each_n                           [system] {
+  header "__algorithm/ranges_for_each_n.h"
+  export std_private_algorithm_in_fun_result
+}
+module std_private_algorithm_ranges_generate                             [system] { header "__algorithm/ranges_generate.h" }
+module std_private_algorithm_ranges_generate_n                           [system] { header "__algorithm/ranges_generate_n.h" }
+module std_private_algorithm_ranges_includes                             [system] {
+  header "__algorithm/ranges_includes.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_inplace_merge                        [system] {
+  header "__algorithm/ranges_inplace_merge.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_is_heap                              [system] {
+  header "__algorithm/ranges_is_heap.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_is_heap_until                        [system] {
+  header "__algorithm/ranges_is_heap_until.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_is_partitioned                       [system] { header "__algorithm/ranges_is_partitioned.h" }
+module std_private_algorithm_ranges_is_permutation                       [system] { header "__algorithm/ranges_is_permutation.h" }
+module std_private_algorithm_ranges_is_sorted                            [system] {
+  header "__algorithm/ranges_is_sorted.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_is_sorted_until                      [system] {
+  header "__algorithm/ranges_is_sorted_until.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_iterator_concept                     [system] { header "__algorithm/ranges_iterator_concept.h" }
+module std_private_algorithm_ranges_lexicographical_compare              [system] {
+  header "__algorithm/ranges_lexicographical_compare.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_lower_bound                          [system] {
+  header "__algorithm/ranges_lower_bound.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_make_heap                            [system] {
+  header "__algorithm/ranges_make_heap.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_max                                  [system] {
+  header "__algorithm/ranges_max.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_max_element                          [system] {
+  header "__algorithm/ranges_max_element.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_merge                                [system] {
+  header "__algorithm/ranges_merge.h"
+  export std_private_algorithm_in_in_out_result
+}
+module std_private_algorithm_ranges_min                                  [system] {
+  header "__algorithm/ranges_min.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_min_element                          [system] {
+  header "__algorithm/ranges_min_element.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_minmax                               [system] {
+  header "__algorithm/ranges_minmax.h"
+  export std_private_functional_ranges_operations
+  export std_private_algorithm_min_max_result
+}
+module std_private_algorithm_ranges_minmax_element                       [system] {
+  header "__algorithm/ranges_minmax_element.h"
+  export std_private_functional_ranges_operations
+  export std_private_algorithm_min_max_result
+}
+module std_private_algorithm_ranges_mismatch                             [system] {
+  header "__algorithm/ranges_mismatch.h"
+  export std_private_algorithm_in_in_result
+}
+module std_private_algorithm_ranges_move                                 [system] {
+  header "__algorithm/ranges_move.h"
+  export std_private_algorithm_in_out_result
+}
+module std_private_algorithm_ranges_move_backward                        [system] {
+  header "__algorithm/ranges_move_backward.h"
+  export std_private_algorithm_in_out_result
+}
+module std_private_algorithm_ranges_next_permutation                     [system] {
+  header "__algorithm/ranges_next_permutation.h"
+  export std_private_algorithm_in_found_result
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_none_of                              [system] { header "__algorithm/ranges_none_of.h" }
+module std_private_algorithm_ranges_nth_element                          [system] {
+  header "__algorithm/ranges_nth_element.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_partial_sort                         [system] {
+  header "__algorithm/ranges_partial_sort.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_partial_sort_copy                    [system] {
+  header "__algorithm/ranges_partial_sort_copy.h"
+  export std_private_algorithm_in_out_result
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_partition                            [system] { header "__algorithm/ranges_partition.h" }
+module std_private_algorithm_ranges_partition_copy                       [system] { header "__algorithm/ranges_partition_copy.h" }
+module std_private_algorithm_ranges_partition_point                      [system] { header "__algorithm/ranges_partition_point.h" }
+module std_private_algorithm_ranges_pop_heap                             [system] {
+  header "__algorithm/ranges_pop_heap.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_prev_permutation                     [system] {
+  header "__algorithm/ranges_prev_permutation.h"
+  export std_private_algorithm_in_found_result
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_push_heap                            [system] {
+  header "__algorithm/ranges_push_heap.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_remove                               [system] { header "__algorithm/ranges_remove.h" }
+module std_private_algorithm_ranges_remove_copy                          [system] {
+  header "__algorithm/ranges_remove_copy.h"
+  export std_private_algorithm_in_out_result
+}
+module std_private_algorithm_ranges_remove_copy_if                       [system] {
+  header "__algorithm/ranges_remove_copy_if.h"
+  export std_private_algorithm_in_out_result
+}
+module std_private_algorithm_ranges_remove_if                            [system] { header "__algorithm/ranges_remove_if.h" }
+module std_private_algorithm_ranges_replace                              [system] { header "__algorithm/ranges_replace.h" }
+module std_private_algorithm_ranges_replace_copy                         [system] {
+  header "__algorithm/ranges_replace_copy.h"
+  export std_private_algorithm_in_out_result
+}
+module std_private_algorithm_ranges_replace_copy_if                      [system] {
+  header "__algorithm/ranges_replace_copy_if.h"
+  export std_private_algorithm_in_out_result
+}
+module std_private_algorithm_ranges_replace_if                           [system] { header "__algorithm/ranges_replace_if.h" }
+module std_private_algorithm_ranges_reverse                              [system] { header "__algorithm/ranges_reverse.h" }
+module std_private_algorithm_ranges_reverse_copy                         [system] {
+  header "__algorithm/ranges_reverse_copy.h"
+  export std_private_algorithm_in_out_result
+}
+module std_private_algorithm_ranges_rotate                               [system] { header "__algorithm/ranges_rotate.h" }
+module std_private_algorithm_ranges_rotate_copy                          [system] {
+  header "__algorithm/ranges_rotate_copy.h"
+  export std_private_algorithm_in_out_result
+}
+module std_private_algorithm_ranges_sample                               [system] { header "__algorithm/ranges_sample.h" }
+module std_private_algorithm_ranges_search                               [system] { header "__algorithm/ranges_search.h" }
+module std_private_algorithm_ranges_search_n                             [system] { header "__algorithm/ranges_search_n.h" }
+module std_private_algorithm_ranges_set_difference                       [system] {
+  header "__algorithm/ranges_set_difference.h"
+  export std_private_algorithm_in_out_result
+}
+module std_private_algorithm_ranges_set_intersection                     [system] {
+  header "__algorithm/ranges_set_intersection.h"
+  export std_private_algorithm_in_in_out_result
+}
+module std_private_algorithm_ranges_set_symmetric_difference             [system] {
+  header "__algorithm/ranges_set_symmetric_difference.h"
+  export std_private_algorithm_in_in_out_result
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_set_union                            [system] {
+  header "__algorithm/ranges_set_union.h"
+  export std_private_algorithm_in_in_out_result
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_shuffle                              [system] { header "__algorithm/ranges_shuffle.h" }
+module std_private_algorithm_ranges_sort                                 [system] {
+  header "__algorithm/ranges_sort.h"
+  export std_private_algorithm_make_projected
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_sort_heap                            [system] {
+  header "__algorithm/ranges_sort_heap.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_stable_partition                     [system] { header "__algorithm/ranges_stable_partition.h" }
+module std_private_algorithm_ranges_stable_sort                          [system] {
+  header "__algorithm/ranges_stable_sort.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_ranges_starts_with                          [system] { header "__algorithm/ranges_starts_with.h" }
+module std_private_algorithm_ranges_swap_ranges                          [system] {
+  header "__algorithm/ranges_swap_ranges.h"
+  export std_private_algorithm_in_in_result
+}
+module std_private_algorithm_ranges_transform                            [system] {
+  header "__algorithm/ranges_transform.h"
+  export std_private_algorithm_in_in_out_result
+  export std_private_algorithm_in_out_result
+}
+module std_private_algorithm_ranges_unique                               [system] { header "__algorithm/ranges_unique.h" }
+module std_private_algorithm_ranges_unique_copy                          [system] {
+  header "__algorithm/ranges_unique_copy.h"
+  export std_private_algorithm_in_out_result
+}
+module std_private_algorithm_ranges_upper_bound                          [system] {
+  header "__algorithm/ranges_upper_bound.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_algorithm_remove                                      [system] { header "__algorithm/remove.h" }
+module std_private_algorithm_remove_copy                                 [system] { header "__algorithm/remove_copy.h" }
+module std_private_algorithm_remove_copy_if                              [system] { header "__algorithm/remove_copy_if.h" }
+module std_private_algorithm_remove_if                                   [system] { header "__algorithm/remove_if.h" }
+module std_private_algorithm_replace                                     [system] { header "__algorithm/replace.h" }
+module std_private_algorithm_replace_copy                                [system] { header "__algorithm/replace_copy.h" }
+module std_private_algorithm_replace_copy_if                             [system] { header "__algorithm/replace_copy_if.h" }
+module std_private_algorithm_replace_if                                  [system] { header "__algorithm/replace_if.h" }
+module std_private_algorithm_reverse                                     [system] { header "__algorithm/reverse.h" }
+module std_private_algorithm_reverse_copy                                [system] { header "__algorithm/reverse_copy.h" }
+module std_private_algorithm_rotate                                      [system] { header "__algorithm/rotate.h" }
+module std_private_algorithm_rotate_copy                                 [system] { header "__algorithm/rotate_copy.h" }
+module std_private_algorithm_sample                                      [system] { header "__algorithm/sample.h" }
+module std_private_algorithm_search                                      [system] { header "__algorithm/search.h" }
+module std_private_algorithm_search_n                                    [system] { header "__algorithm/search_n.h" }
+module std_private_algorithm_set_difference                              [system] { header "__algorithm/set_difference.h" }
+module std_private_algorithm_set_intersection                            [system] { header "__algorithm/set_intersection.h" }
+module std_private_algorithm_set_symmetric_difference                    [system] { header "__algorithm/set_symmetric_difference.h" }
+module std_private_algorithm_set_union                                   [system] { header "__algorithm/set_union.h" }
+module std_private_algorithm_shift_left                                  [system] { header "__algorithm/shift_left.h" }
+module std_private_algorithm_shift_right                                 [system] { header "__algorithm/shift_right.h" }
+module std_private_algorithm_shuffle                                     [system] { header "__algorithm/shuffle.h" }
+module std_private_algorithm_sift_down                                   [system] { header "__algorithm/sift_down.h" }
+module std_private_algorithm_sort                                        [system] {
+  header "__algorithm/sort.h"
+  export std_private_debug_utils_strict_weak_ordering_check
+}
+module std_private_algorithm_simd_utils                                  [system] { header "__algorithm/simd_utils.h" }
+module std_private_algorithm_sort_heap                                   [system] { header "__algorithm/sort_heap.h" }
+module std_private_algorithm_stable_partition                            [system] { header "__algorithm/stable_partition.h" }
+module std_private_algorithm_stable_sort                                 [system] { header "__algorithm/stable_sort.h" }
+module std_private_algorithm_swap_ranges                                 [system] {
+  header "__algorithm/swap_ranges.h"
+  export std_private_algorithm_iterator_operations
+}
+module std_private_algorithm_three_way_comp_ref_type                     [system] { header "__algorithm/three_way_comp_ref_type.h" }
+module std_private_algorithm_transform                                   [system] { header "__algorithm/transform.h" }
+module std_private_algorithm_uniform_random_bit_generator_adaptor        [system] { header "__algorithm/uniform_random_bit_generator_adaptor.h" }
+module std_private_algorithm_unique                                      [system] { header "__algorithm/unique.h" }
+module std_private_algorithm_unique_copy                                 [system] { header "__algorithm/unique_copy.h" }
+module std_private_algorithm_unwrap_iter                                 [system] {
+  header "__algorithm/unwrap_iter.h"
+  export std_private_iterator_iterator_traits
+}
+module std_private_algorithm_unwrap_range                                [system] {
+  header "__algorithm/unwrap_range.h"
+  export std_private_utility_pair
+}
+module std_private_algorithm_upper_bound                                 [system] { header "__algorithm/upper_bound.h" }
+
+module std_private_array_array_fwd [system] { header "__fwd/array.h" }
+
+module std_private_atomic_aliases             [system] {
+  header "__atomic/aliases.h"
+  export std_private_atomic_atomic
+}
+module std_private_atomic_atomic              [system] {
+  header "__atomic/atomic.h"
+  export std_private_atomic_atomic_base
+}
+module std_private_atomic_atomic_base         [system] { header "__atomic/atomic_base.h" }
+module std_private_atomic_atomic_flag         [system] {
+  header "__atomic/atomic_flag.h"
+  export *
+}
+module std_private_atomic_atomic_init         [system] { header "__atomic/atomic_init.h" }
+module std_private_atomic_atomic_lock_free    [system] { header "__atomic/atomic_lock_free.h" }
+module std_private_atomic_atomic_ref          [system] { header "__atomic/atomic_ref.h" }
+module std_private_atomic_atomic_sync         [system] {
+  header "__atomic/atomic_sync.h"
+  export std_private_atomic_to_gcc_order
+}
+module std_private_atomic_check_memory_order  [system] { header "__atomic/check_memory_order.h" }
+module std_private_atomic_contention_t        [system] { header "__atomic/contention_t.h" }
+module std_private_atomic_cxx_atomic_impl     [system] { header "__atomic/cxx_atomic_impl.h" }
+module std_private_atomic_fence               [system] { header "__atomic/fence.h" }
+module std_private_atomic_is_always_lock_free [system] { header "__atomic/is_always_lock_free.h" }
+module std_private_atomic_kill_dependency     [system] { header "__atomic/kill_dependency.h" }
+module std_private_atomic_memory_order        [system] { header "__atomic/memory_order.h" }
+module std_private_atomic_to_gcc_order        [system] {
+  header "__atomic/to_gcc_order.h"
+  export std_private_atomic_memory_order
+}
+
+module std_private_bit_bit_cast       [system] { header "__bit/bit_cast.h" }
+module std_private_bit_bit_ceil       [system] { header "__bit/bit_ceil.h" }
+module std_private_bit_bit_floor      [system] { header "__bit/bit_floor.h" }
+module std_private_bit_bit_log2       [system] { header "__bit/bit_log2.h" }
+module std_private_bit_bit_width      [system] { header "__bit/bit_width.h" }
+module std_private_bit_blsr           [system] { header "__bit/blsr.h" }
+module std_private_bit_byteswap       [system] { header "__bit/byteswap.h" }
+module std_private_bit_countl         [system] { header "__bit/countl.h" }
+module std_private_bit_countr         [system] { header "__bit/countr.h" }
+module std_private_bit_endian         [system] { header "__bit/endian.h" }
+module std_private_bit_has_single_bit [system] { header "__bit/has_single_bit.h" }
+module std_private_bit_invert_if      [system] { header "__bit/invert_if.h" }
+module std_private_bit_popcount       [system] { header "__bit/popcount.h" }
+module std_private_bit_rotate         [system] { header "__bit/rotate.h" }
+
+module std_private_charconv_chars_format            [system] { header "__charconv/chars_format.h" }
+module std_private_charconv_from_chars_integral     [system] { header "__charconv/from_chars_integral.h" }
+module std_private_charconv_from_chars_result       [system] { header "__charconv/from_chars_result.h" }
+module std_private_charconv_tables                  [system] { header "__charconv/tables.h" }
+module std_private_charconv_to_chars                [system] { header "__charconv/to_chars.h" }
+module std_private_charconv_to_chars_base_10        [system] { header "__charconv/to_chars_base_10.h" }
+module std_private_charconv_to_chars_floating_point [system] { header "__charconv/to_chars_floating_point.h" }
+module std_private_charconv_to_chars_integral       [system] {
+  header "__charconv/to_chars_integral.h"
+  export std_private_charconv_traits
+}
+module std_private_charconv_to_chars_result         [system] {
+  header "__charconv/to_chars_result.h"
+  export *
+}
+module std_private_charconv_traits                  [system] { header "__charconv/traits.h" }
+
+module std_private_chrono_calendar               [system] { header "__chrono/calendar.h" }
+module std_private_chrono_concepts               [system] { header "__chrono/concepts.h" }
+module std_private_chrono_convert_to_timespec    [system] { header "__chrono/convert_to_timespec.h" }
+module std_private_chrono_convert_to_tm          [system] { header "__chrono/convert_to_tm.h" }
+module std_private_chrono_day                    [system] { header "__chrono/day.h" }
+module std_private_chrono_duration               [system] {
+  header "__chrono/duration.h"
+  export std_private_type_traits_is_convertible
+}
+module std_private_chrono_exception              [system] { header "__chrono/exception.h" }
+module std_private_chrono_file_clock             [system] { header "__chrono/file_clock.h" }
+module std_private_chrono_formatter              [system] {
+  header "__chrono/formatter.h"
+}
+module std_private_chrono_hh_mm_ss               [system] { header "__chrono/hh_mm_ss.h" }
+module std_private_chrono_high_resolution_clock  [system] {
+  header "__chrono/high_resolution_clock.h"
+  export std_private_chrono_steady_clock
+  export std_private_chrono_system_clock
+}
+module std_private_chrono_leap_second            [system] { header "__chrono/leap_second.h" }
+module std_private_chrono_literals               [system] { header "__chrono/literals.h" }
+module std_private_chrono_local_info             [system] {
+  header "__chrono/local_info.h"
+  export std_private_chrono_sys_info
+}
+module std_private_chrono_month                  [system] { header "__chrono/month.h" }
+module std_private_chrono_month_weekday          [system] { header "__chrono/month_weekday.h" }
+module std_private_chrono_monthday               [system] { header "__chrono/monthday.h" }
+module std_private_chrono_ostream                [system] {
+  header "__chrono/ostream.h"
+}
+module std_private_chrono_parser_std_format_spec [system] {
+  header "__chrono/parser_std_format_spec.h"
+}
+module std_private_chrono_statically_widen       [system] { header "__chrono/statically_widen.h" }
+module std_private_chrono_steady_clock           [system] {
+  header "__chrono/steady_clock.h"
+  export std_private_chrono_time_point
+}
+module std_private_chrono_time_zone              [system] {
+  header "__chrono/time_zone.h"
+  export std_private_memory_unique_ptr
+}
+module std_private_chrono_time_zone_link         [system] {
+  header "__chrono/time_zone_link.h"
+}
+module std_private_chrono_sys_info               [system] {
+  header "__chrono/sys_info.h"
+}
+module std_private_chrono_system_clock           [system] {
+  header "__chrono/system_clock.h"
+  export std_private_chrono_time_point
+}
+module std_private_chrono_tzdb                   [system] {
+  header "__chrono/tzdb.h"
+  export *
+}
+module std_private_chrono_tzdb_list              [system] {
+  header "__chrono/tzdb_list.h"
+  export *
+}
+module std_private_chrono_time_point             [system] { header "__chrono/time_point.h" }
+module std_private_chrono_weekday                [system] { header "__chrono/weekday.h" }
+module std_private_chrono_year                   [system] { header "__chrono/year.h" }
+module std_private_chrono_year_month             [system] { header "__chrono/year_month.h" }
+module std_private_chrono_year_month_day         [system] { header "__chrono/year_month_day.h" }
+module std_private_chrono_year_month_weekday     [system] { header "__chrono/year_month_weekday.h" }
+module std_private_chrono_zoned_time             [system] { header "__chrono/zoned_time.h" }
+
+module std_private_compare_common_comparison_category     [system] { header "__compare/common_comparison_category.h" }
+module std_private_compare_compare_partial_order_fallback [system] { header "__compare/compare_partial_order_fallback.h" }
+module std_private_compare_compare_strong_order_fallback  [system] { header "__compare/compare_strong_order_fallback.h" }
+module std_private_compare_compare_three_way              [system] { header "__compare/compare_three_way.h" }
+module std_private_compare_compare_three_way_result       [system] { header "__compare/compare_three_way_result.h" }
+module std_private_compare_compare_weak_order_fallback    [system] { header "__compare/compare_weak_order_fallback.h" }
+module std_private_compare_is_eq                          [system] { header "__compare/is_eq.h" }
+module std_private_compare_ordering                       [system] { header "__compare/ordering.h" }
+module std_private_compare_partial_order                  [system] { header "__compare/partial_order.h" }
+module std_private_compare_strong_order                   [system] { header "__compare/strong_order.h" }
+module std_private_compare_synth_three_way                [system] { header "__compare/synth_three_way.h" }
+module std_private_compare_three_way_comparable           [system] { header "__compare/three_way_comparable.h" }
+module std_private_compare_weak_order                     [system] { header "__compare/weak_order.h" }
+
+module std_private_complex_complex_fwd            [system] { header "__fwd/complex.h" }
+
+module std_private_concepts_arithmetic            [system] { header "__concepts/arithmetic.h" }
+module std_private_concepts_assignable            [system] { header "__concepts/assignable.h" }
+module std_private_concepts_boolean_testable      [system] { header "__concepts/boolean_testable.h" }
+module std_private_concepts_class_or_enum         [system] { header "__concepts/class_or_enum.h" }
+module std_private_concepts_common_reference_with [system] { header "__concepts/common_reference_with.h" }
+module std_private_concepts_common_with           [system] { header "__concepts/common_with.h" }
+module std_private_concepts_constructible         [system] {
+  header "__concepts/constructible.h"
+  export std_private_concepts_destructible
+}
+module std_private_concepts_convertible_to        [system] { header "__concepts/convertible_to.h" }
+module std_private_concepts_copyable              [system] { header "__concepts/copyable.h" }
+module std_private_concepts_derived_from          [system] { header "__concepts/derived_from.h" }
+module std_private_concepts_destructible          [system] {
+  header "__concepts/destructible.h"
+  export std_private_type_traits_is_nothrow_destructible
+}
+module std_private_concepts_different_from        [system] { header "__concepts/different_from.h" }
+module std_private_concepts_equality_comparable   [system] {
+  header "__concepts/equality_comparable.h"
+  export std_private_type_traits_common_reference
+}
+module std_private_concepts_invocable             [system] { header "__concepts/invocable.h" }
+module std_private_concepts_movable               [system] {
+  header "__concepts/movable.h"
+  export std_private_type_traits_is_object
+}
+module std_private_concepts_predicate             [system] { header "__concepts/predicate.h" }
+module std_private_concepts_regular               [system] { header "__concepts/regular.h" }
+module std_private_concepts_relation              [system] { header "__concepts/relation.h" }
+module std_private_concepts_same_as               [system] {
+  header "__concepts/same_as.h"
+  export std_private_type_traits_is_same
+}
+module std_private_concepts_semiregular           [system] { header "__concepts/semiregular.h" }
+module std_private_concepts_swappable             [system] { header "__concepts/swappable.h" }
+module std_private_concepts_totally_ordered       [system] { header "__concepts/totally_ordered.h" }
+
+module std_private_condition_variable_condition_variable [system] {
+  header "__condition_variable/condition_variable.h"
+  export *
+}
+
+module std_private_coroutine_coroutine_handle      [system] { header "__coroutine/coroutine_handle.h" }
+module std_private_coroutine_coroutine_traits      [system] { header "__coroutine/coroutine_traits.h" }
+module std_private_coroutine_noop_coroutine_handle [system] { header "__coroutine/noop_coroutine_handle.h" }
+module std_private_coroutine_trivial_awaitables    [system] { header "__coroutine/trivial_awaitables.h" }
+
+module std_private_debug_utils_randomize_range            [system] { header "__debug_utils/randomize_range.h" }
+module std_private_debug_utils_sanitizers                 [system] { header "__debug_utils/sanitizers.h" }
+module std_private_debug_utils_strict_weak_ordering_check [system] {
+  header "__debug_utils/strict_weak_ordering_check.h"
+  export std_private_type_traits_is_constant_evaluated
+}
+
+module std_private_deque_fwd [system] { header "__fwd/deque.h" }
+
+module std_private_exception_exception        [system] { header "__exception/exception.h" }
+module std_private_exception_exception_ptr    [system] {
+  header "__exception/exception_ptr.h"
+  export std_private_exception_operations
+}
+module std_private_exception_nested_exception [system] { header "__exception/nested_exception.h" }
+module std_private_exception_operations       [system] { header "__exception/operations.h" }
+module std_private_exception_terminate        [system] { header "__exception/terminate.h" }
+
+module std_private_expected_bad_expected_access [system] { header "__expected/bad_expected_access.h" }
+module std_private_expected_expected            [system] { header "__expected/expected.h" }
+module std_private_expected_unexpect            [system] { header "__expected/unexpect.h" }
+module std_private_expected_unexpected          [system] { header "__expected/unexpected.h" }
+
+module std_private_filesystem_copy_options                 [system] { header "__filesystem/copy_options.h" }
+module std_private_filesystem_directory_entry              [system] {
+  header "__filesystem/directory_entry.h"
+  export *
+}
+module std_private_filesystem_directory_iterator           [system] {
+  header "__filesystem/directory_iterator.h"
+  export *
+}
+module std_private_filesystem_directory_options            [system] { header "__filesystem/directory_options.h" }
+module std_private_filesystem_file_status                  [system] { header "__filesystem/file_status.h" }
+module std_private_filesystem_file_time_type               [system] { header "__filesystem/file_time_type.h" }
+module std_private_filesystem_file_type                    [system] { header "__filesystem/file_type.h" }
+module std_private_filesystem_filesystem_error             [system] {
+  header "__filesystem/filesystem_error.h"
+  export *
+}
+module std_private_filesystem_operations                   [system] { header "__filesystem/operations.h" }
+module std_private_filesystem_path                         [system] {
+  header "__filesystem/path.h"
+  export *
+}
+module std_private_filesystem_path_iterator                [system] { header "__filesystem/path_iterator.h" }
+module std_private_filesystem_perm_options                 [system] { header "__filesystem/perm_options.h" }
+module std_private_filesystem_perms                        [system] { header "__filesystem/perms.h" }
+module std_private_filesystem_recursive_directory_iterator [system] {
+  header "__filesystem/recursive_directory_iterator.h"
+  export *
+}
+module std_private_filesystem_space_info                   [system] { header "__filesystem/space_info.h" }
+module std_private_filesystem_u8path                       [system] { header "__filesystem/u8path.h" }
+
+module std_private_format_buffer                          [system] { header "__format/buffer.h" }
+module std_private_format_concepts                        [system] { header "__format/concepts.h" }
+module std_private_format_container_adaptor               [system] { header "__format/container_adaptor.h" }
+module std_private_format_enable_insertable               [system] { header "__format/enable_insertable.h" }
+module std_private_format_escaped_output_table            [system] { header "__format/escaped_output_table.h" }
+module std_private_format_extended_grapheme_cluster_table [system] { header "__format/extended_grapheme_cluster_table.h" }
+module std_private_format_format_arg                      [system] { header "__format/format_arg.h" }
+module std_private_format_format_arg_store                [system] { header "__format/format_arg_store.h" }
+module std_private_format_format_args                     [system] { header "__format/format_args.h" }
+module std_private_format_format_context                  [system] {
+  header "__format/format_context.h"
+  export *
+}
+module std_private_format_format_error                    [system] { header "__format/format_error.h" }
+module std_private_format_format_functions                [system] {
+  header "__format/format_functions.h"
+  export std_string
+}
+module std_private_format_fwd                             [system] { header "__fwd/format.h" }
+module std_private_format_format_parse_context            [system] { header "__format/format_parse_context.h" }
+module std_private_format_format_string                   [system] { header "__format/format_string.h" }
+module std_private_format_format_to_n_result              [system] {
+  header "__format/format_to_n_result.h"
+  export std_private_iterator_incrementable_traits
+}
+module std_private_format_formatter                       [system] { header "__format/formatter.h" }
+module std_private_format_formatter_bool                  [system] { header "__format/formatter_bool.h" }
+module std_private_format_formatter_char                  [system] { header "__format/formatter_char.h" }
+module std_private_format_formatter_floating_point        [system] { header "__format/formatter_floating_point.h" }
+module std_private_format_formatter_integer               [system] { header "__format/formatter_integer.h" }
+module std_private_format_formatter_integral              [system] { header "__format/formatter_integral.h" }
+module std_private_format_formatter_output                [system] { header "__format/formatter_output.h" }
+module std_private_format_formatter_pointer               [system] { header "__format/formatter_pointer.h" }
+module std_private_format_formatter_string                [system] { header "__format/formatter_string.h" }
+module std_private_format_formatter_tuple                 [system] { header "__format/formatter_tuple.h" }
+module std_private_format_indic_conjunct_break_table      [system] { header "__format/indic_conjunct_break_table.h" }
+module std_private_format_parser_std_format_spec          [system] { header "__format/parser_std_format_spec.h" }
+module std_private_format_range_default_formatter         [system] { header "__format/range_default_formatter.h" }
+module std_private_format_range_formatter                 [system] { header "__format/range_formatter.h" }
+module std_private_format_unicode                         [system] {
+  header "__format/unicode.h"
+  export std_private_format_extended_grapheme_cluster_table
+  export std_private_format_indic_conjunct_break_table
+}
+module std_private_format_width_estimation_table          [system] { header "__format/width_estimation_table.h" }
+module std_private_format_write_escaped                   [system] { header "__format/write_escaped.h" }
+
+module std_private_functional_binary_function            [system] { header "__functional/binary_function.h" }
+module std_private_functional_binary_negate              [system] { header "__functional/binary_negate.h" }
+module std_private_functional_bind                       [system] { header "__functional/bind.h" }
+module std_private_functional_bind_back                  [system] { header "__functional/bind_back.h" }
+module std_private_functional_bind_front                 [system] { header "__functional/bind_front.h" }
+module std_private_functional_binder1st                  [system] { header "__functional/binder1st.h" }
+module std_private_functional_binder2nd                  [system] { header "__functional/binder2nd.h" }
+module std_private_functional_boyer_moore_searcher       [system] {
+  header "__functional/boyer_moore_searcher.h"
+  export std_private_memory_shared_ptr
+}
+module std_private_functional_compose                    [system] {
+  header "__functional/compose.h"
+  export std_private_functional_perfect_forward
+}
+module std_private_functional_default_searcher           [system] { header "__functional/default_searcher.h" }
+module std_private_functional_function                   [system] { header "__functional/function.h" }
+module std_private_functional_hash                       [system] {
+  header "__functional/hash.h"
+  export std_cstdint
+  export std_private_type_traits_underlying_type
+  export std_private_utility_pair
+}
+module std_private_functional_fwd                        [system] { header "__fwd/functional.h" }
+module std_private_functional_identity                   [system] { header "__functional/identity.h" }
+module std_private_functional_invoke                     [system] {
+  header "__functional/invoke.h"
+  export *
+}
+module std_private_functional_is_transparent             [system] { header "__functional/is_transparent.h" }
+module std_private_functional_mem_fn                     [system] { header "__functional/mem_fn.h" }
+module std_private_functional_mem_fun_ref                [system] { header "__functional/mem_fun_ref.h" }
+module std_private_functional_not_fn                     [system] {
+  header "__functional/not_fn.h"
+  export std_private_functional_perfect_forward
+}
+module std_private_functional_operations                 [system] { header "__functional/operations.h" }
+module std_private_functional_perfect_forward            [system] {
+  header "__functional/perfect_forward.h"
+  export *
+}
+module std_private_functional_pointer_to_binary_function [system] { header "__functional/pointer_to_binary_function.h" }
+module std_private_functional_pointer_to_unary_function  [system] { header "__functional/pointer_to_unary_function.h" }
+module std_private_functional_ranges_operations          [system] { header "__functional/ranges_operations.h" }
+module std_private_functional_reference_wrapper          [system] { header "__functional/reference_wrapper.h" }
+module std_private_functional_unary_function             [system] { header "__functional/unary_function.h" }
+module std_private_functional_unary_negate               [system] { header "__functional/unary_negate.h" }
+module std_private_functional_weak_result_type           [system] { header "__functional/weak_result_type.h" }
+
+module std_private_ios_fpos [system] { header "__ios/fpos.h" }
+
+module std_private_iosfwd_fstream_fwd   [system] { header "__fwd/fstream.h" }
+module std_private_iosfwd_ios_fwd       [system] { header "__fwd/ios.h" }
+module std_private_iosfwd_istream_fwd   [system] { header "__fwd/istream.h" }
+module std_private_iosfwd_ostream_fwd   [system] { header "__fwd/ostream.h" }
+module std_private_iosfwd_sstream_fwd   [system] { header "__fwd/sstream.h" }
+module std_private_iosfwd_streambuf_fwd [system] { header "__fwd/streambuf.h" }
+
+module std_private_iterator_access                  [system] { header "__iterator/access.h" }
+module std_private_iterator_advance                 [system] { header "__iterator/advance.h" }
+module std_private_iterator_aliasing_iterator       [system] { header "__iterator/aliasing_iterator.h" }
+module std_private_iterator_back_insert_iterator    [system] { header "__iterator/back_insert_iterator.h" }
+module std_private_iterator_bounded_iter            [system] { header "__iterator/bounded_iter.h" }
+module std_private_iterator_common_iterator         [system] { header "__iterator/common_iterator.h" }
+module std_private_iterator_concepts                [system] {
+  header "__iterator/concepts.h"
+  export std_private_concepts_constructible
+  export std_private_concepts_equality_comparable
+  export std_private_concepts_movable
+  export std_private_type_traits_common_reference
+  export std_private_type_traits_is_reference
+  export std_private_type_traits_remove_cvref
+}
+module std_private_iterator_counted_iterator        [system] { header "__iterator/counted_iterator.h" }
+module std_private_iterator_cpp17_iterator_concepts [system] { header "__iterator/cpp17_iterator_concepts.h" }
+module std_private_iterator_data                    [system] { header "__iterator/data.h" }
+module std_private_iterator_default_sentinel        [system] { header "__iterator/default_sentinel.h" }
+module std_private_iterator_distance                [system] {
+  header "__iterator/distance.h"
+  export std_private_ranges_size
+}
+module std_private_iterator_empty                   [system] { header "__iterator/empty.h" }
+module std_private_iterator_erase_if_container      [system] { header "__iterator/erase_if_container.h" }
+module std_private_iterator_front_insert_iterator   [system] { header "__iterator/front_insert_iterator.h" }
+module std_private_iterator_incrementable_traits    [system] { header "__iterator/incrementable_traits.h" }
+module std_private_iterator_indirectly_comparable   [system] { header "__iterator/indirectly_comparable.h" }
+module std_private_iterator_insert_iterator         [system] { header "__iterator/insert_iterator.h" }
+module std_private_iterator_istream_iterator        [system] { header "__iterator/istream_iterator.h" }
+module std_private_iterator_istreambuf_iterator     [system] { header "__iterator/istreambuf_iterator.h" }
+module std_private_iterator_iter_move               [system] { header "__iterator/iter_move.h" }
+module std_private_iterator_iter_swap               [system] { header "__iterator/iter_swap.h" }
+module std_private_iterator_iterator                [system] { header "__iterator/iterator.h" }
+module std_private_iterator_iterator_traits         [system] {
+  header "__iterator/iterator_traits.h"
+  export std_private_type_traits_is_primary_template
+}
+module std_private_iterator_iterator_with_data      [system] { header "__iterator/iterator_with_data.h" }
+module std_private_iterator_mergeable               [system] {
+  header "__iterator/mergeable.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_iterator_move_iterator           [system] { header "__iterator/move_iterator.h" }
+module std_private_iterator_move_sentinel           [system] { header "__iterator/move_sentinel.h" }
+module std_private_iterator_next                    [system] { header "__iterator/next.h" }
+module std_private_iterator_ostream_iterator        [system] { header "__iterator/ostream_iterator.h" }
+module std_private_iterator_ostreambuf_iterator     [system] {
+  header "__iterator/ostreambuf_iterator.h"
+  export *
+}
+module std_private_iterator_permutable              [system] { header "__iterator/permutable.h" }
+module std_private_iterator_prev                    [system] { header "__iterator/prev.h" }
+module std_private_iterator_projected               [system] { header "__iterator/projected.h" }
+module std_private_iterator_ranges_iterator_traits  [system] { header "__iterator/ranges_iterator_traits.h" }
+module std_private_iterator_readable_traits         [system] { header "__iterator/readable_traits.h" }
+module std_private_iterator_reverse_access          [system] { header "__iterator/reverse_access.h" }
+module std_private_iterator_reverse_iterator        [system] { header "__iterator/reverse_iterator.h" }
+module std_private_iterator_segmented_iterator      [system] { header "__iterator/segmented_iterator.h" }
+module std_private_iterator_size                    [system] { header "__iterator/size.h" }
+module std_private_iterator_sortable                [system] {
+  header "__iterator/sortable.h"
+  export std_private_functional_ranges_operations
+}
+module std_private_iterator_unreachable_sentinel    [system] { header "__iterator/unreachable_sentinel.h" }
+module std_private_iterator_wrap_iter               [system] { header "__iterator/wrap_iter.h" }
+
+module std_private_locale_locale_base_api_android              [system] { textual header "__locale_dir/locale_base_api/android.h" }
+module std_private_locale_locale_base_api_bsd_locale_defaults  [system] { textual header "__locale_dir/locale_base_api/bsd_locale_defaults.h" }
+module std_private_locale_locale_base_api_bsd_locale_fallbacks [system] { textual header "__locale_dir/locale_base_api/bsd_locale_fallbacks.h" }
+module std_private_locale_locale_base_api_fuchsia              [system] { textual header "__locale_dir/locale_base_api/fuchsia.h" }
+module std_private_locale_locale_base_api_ibm                  [system] { textual header "__locale_dir/locale_base_api/ibm.h" }
+module std_private_locale_locale_base_api_locale_guard         [system] { header "__locale_dir/locale_base_api/locale_guard.h" }
+module std_private_locale_locale_base_api_musl                 [system] { textual header "__locale_dir/locale_base_api/musl.h" }
+module std_private_locale_locale_base_api_newlib               [system] { textual header "__locale_dir/locale_base_api/newlib.h" }
+module std_private_locale_locale_base_api_openbsd              [system] { textual header "__locale_dir/locale_base_api/openbsd.h" }
+module std_private_locale_locale_base_api_win32                [system] { textual header "__locale_dir/locale_base_api/win32.h" }
+module std_private_locale_locale_base_api                      [system] {
+  header "__locale_dir/locale_base_api.h"
+  export *
+}
+
+module std_private_math_abs                             [system] { header "__math/abs.h" }
+module std_private_math_copysign                        [system] { header "__math/copysign.h" }
+module std_private_math_error_functions                 [system] { header "__math/error_functions.h" }
+module std_private_math_exponential_functions           [system] { header "__math/exponential_functions.h" }
+module std_private_math_fdim                            [system] { header "__math/fdim.h" }
+module std_private_math_fma                             [system] { header "__math/fma.h" }
+module std_private_math_gamma                           [system] { header "__math/gamma.h" }
+module std_private_math_hyperbolic_functions            [system] { header "__math/hyperbolic_functions.h" }
+module std_private_math_hypot                           [system] { header "__math/hypot.h" }
+module std_private_math_inverse_hyperbolic_functions    [system] { header "__math/inverse_hyperbolic_functions.h" }
+module std_private_math_inverse_trigonometric_functions [system] { header "__math/inverse_trigonometric_functions.h" }
+module std_private_math_logarithms                      [system] { header "__math/logarithms.h" }
+module std_private_math_min_max                         [system] { header "__math/min_max.h" }
+module std_private_math_modulo                          [system] { header "__math/modulo.h" }
+module std_private_math_remainder                       [system] { header "__math/remainder.h" }
+module std_private_math_roots                           [system] { header "__math/roots.h" }
+module std_private_math_rounding_functions              [system] { header "__math/rounding_functions.h" }
+module std_private_math_special_functions               [system] { header "__math/special_functions.h" }
+module std_private_math_traits                          [system] { header "__math/traits.h" }
+module std_private_math_trigonometric_functions         [system] { header "__math/trigonometric_functions.h" }
+
+module std_private_mdspan_default_accessor [system] { header "__mdspan/default_accessor.h" }
+module std_private_mdspan_extents          [system] {
+  header "__mdspan/extents.h"
+  export *
+}
+module std_private_mdspan_layout_left      [system] { header "__mdspan/layout_left.h" }
+module std_private_mdspan_layout_right     [system] { header "__mdspan/layout_right.h" }
+module std_private_mdspan_layout_stride    [system] { header "__mdspan/layout_stride.h" }
+module std_private_mdspan_mdspan           [system] { header "__mdspan/mdspan.h" }
+module std_private_mdspan_mdspan_fwd       [system] { header "__fwd/mdspan.h" }
+
+module std_private_memory_addressof                       [system] { header "__memory/addressof.h" }
+module std_private_memory_align                           [system] { header "__memory/align.h" }
+module std_private_memory_aligned_alloc                   [system] { header "__memory/aligned_alloc.h" }
+module std_private_memory_allocate_at_least               [system] { header "__memory/allocate_at_least.h" }
+module std_private_memory_allocation_guard                [system] { header "__memory/allocation_guard.h" }
+module std_private_memory_allocator                       [system] { header "__memory/allocator.h" }
+module std_private_memory_allocator_arg_t                 [system] { header "__memory/allocator_arg_t.h" }
+module std_private_memory_allocator_destructor            [system] { header "__memory/allocator_destructor.h" }
+module std_private_memory_allocator_traits                [system] { header "__memory/allocator_traits.h" }
+module std_private_memory_assume_aligned                  [system] { header "__memory/assume_aligned.h" }
+module std_private_memory_auto_ptr                        [system] { header "__memory/auto_ptr.h" }
+module std_private_memory_builtin_new_allocator           [system] {
+  header "__memory/builtin_new_allocator.h"
+  export *
+}
+module std_private_memory_compressed_pair                 [system] { header "__memory/compressed_pair.h" }
+module std_private_memory_concepts                        [system] {
+  header "__memory/concepts.h"
+  export std_private_type_traits_remove_reference
+}
+module std_private_memory_construct_at                    [system] { header "__memory/construct_at.h" }
+module std_private_memory_destruct_n                      [system] { header "__memory/destruct_n.h" }
+module std_private_memory_fwd                             [system] { header "__fwd/memory.h" }
+module std_private_memory_inout_ptr                       [system] { header "__memory/inout_ptr.h" }
+module std_private_memory_out_ptr                         [system] { header "__memory/out_ptr.h" }
+module std_private_memory_pointer_traits                  [system] { header "__memory/pointer_traits.h" }
+module std_private_memory_ranges_construct_at             [system] { header "__memory/ranges_construct_at.h" }
+module std_private_memory_ranges_uninitialized_algorithms [system] {
+  header "__memory/ranges_uninitialized_algorithms.h"
+  export std_private_algorithm_in_out_result
+}
+module std_private_memory_raw_storage_iterator            [system] { header "__memory/raw_storage_iterator.h" }
+module std_private_memory_shared_ptr                      [system] {
+  header "__memory/shared_ptr.h"
+  export std_private_memory_uninitialized_algorithms
+}
+module std_private_memory_swap_allocator                  [system] { header "__memory/swap_allocator.h" }
+module std_private_memory_temp_value                      [system] { header "__memory/temp_value.h" }
+module std_private_memory_temporary_buffer                [system] {
+  header "__memory/temporary_buffer.h"
+  export std_private_utility_pair
+}
+module std_private_memory_uninitialized_algorithms        [system] {
+  header "__memory/uninitialized_algorithms.h"
+  export std_private_algorithm_copy
+}
+module std_private_memory_unique_ptr                      [system] {
+  header "__memory/unique_ptr.h"
+  export std_private_type_traits_add_lvalue_reference
+  export std_private_type_traits_is_pointer
+  export std_private_type_traits_type_identity
+}
+module std_private_memory_uses_allocator                  [system] { header "__memory/uses_allocator.h" }
+module std_private_memory_uses_allocator_construction     [system] { header "__memory/uses_allocator_construction.h" }
+module std_private_memory_voidify                         [system] { header "__memory/voidify.h" }
+
+module std_private_memory_resource_memory_resource              [system] { header "__memory_resource/memory_resource.h" }
+module std_private_memory_resource_memory_resource_fwd          [system] { header "__fwd/memory_resource.h" }
+module std_private_memory_resource_monotonic_buffer_resource    [system] { header "__memory_resource/monotonic_buffer_resource.h" }
+module std_private_memory_resource_polymorphic_allocator        [system] { header "__memory_resource/polymorphic_allocator.h" }
+module std_private_memory_resource_pool_options                 [system] { header "__memory_resource/pool_options.h" }
+module std_private_memory_resource_synchronized_pool_resource   [system] {
+  header "__memory_resource/synchronized_pool_resource.h"
+  export *
+}
+module std_private_memory_resource_unsynchronized_pool_resource [system] { header "__memory_resource/unsynchronized_pool_resource.h" }
+
+module std_private_mutex_lock_guard  [system] { header "__mutex/lock_guard.h" }
+module std_private_mutex_mutex       [system] { header "__mutex/mutex.h" }
+module std_private_mutex_once_flag  [system]  { header "__mutex/once_flag.h" }
+module std_private_mutex_tag_types   [system] { header "__mutex/tag_types.h" }
+module std_private_mutex_unique_lock [system] { header "__mutex/unique_lock.h" }
+
+module std_private_numeric_accumulate               [system] { header "__numeric/accumulate.h" }
+module std_private_numeric_adjacent_difference      [system] { header "__numeric/adjacent_difference.h" }
+module std_private_numeric_exclusive_scan           [system] { header "__numeric/exclusive_scan.h" }
+module std_private_numeric_gcd_lcm                  [system] { header "__numeric/gcd_lcm.h" }
+module std_private_numeric_inclusive_scan           [system] { header "__numeric/inclusive_scan.h" }
+module std_private_numeric_inner_product            [system] { header "__numeric/inner_product.h" }
+module std_private_numeric_iota                     [system] { header "__numeric/iota.h" }
+module std_private_numeric_midpoint                 [system] { header "__numeric/midpoint.h" }
+module std_private_numeric_partial_sum              [system] { header "__numeric/partial_sum.h" }
+module std_private_numeric_pstl                     [system] {
+  header "__numeric/pstl.h"
+  export *
+}
+module std_private_numeric_reduce                   [system] { header "__numeric/reduce.h" }
+module std_private_numeric_saturation_arithmetic    [system] { header "__numeric/saturation_arithmetic.h" }
+module std_private_numeric_transform_exclusive_scan [system] { header "__numeric/transform_exclusive_scan.h" }
+module std_private_numeric_transform_inclusive_scan [system] { header "__numeric/transform_inclusive_scan.h" }
+module std_private_numeric_transform_reduce         [system] { header "__numeric/transform_reduce.h" }
+
+module std_private_pstl_backend                    [system] {
+  header "__pstl/backend.h"
+  export *
+}
+module std_private_pstl_backend_fwd                [system] {
+  header "__pstl/backend_fwd.h"
+  export *
+}
+module std_private_pstl_backends_default           [system] {
+  header "__pstl/backends/default.h"
+  export *
+}
+module std_private_pstl_backends_libdispatch       [system] {
+  header "__pstl/backends/libdispatch.h"
+  export *
+}
+module std_private_pstl_backends_serial            [system] {
+  header "__pstl/backends/serial.h"
+  export *
+}
+module std_private_pstl_backends_std_thread        [system] {
+  header "__pstl/backends/std_thread.h"
+  export *
+}
+module std_private_pstl_cpu_algos_any_of           [system] { header "__pstl/cpu_algos/any_of.h" }
+module std_private_pstl_cpu_algos_cpu_traits       [system] { header "__pstl/cpu_algos/cpu_traits.h" }
+module std_private_pstl_cpu_algos_fill             [system] { header "__pstl/cpu_algos/fill.h" }
+module std_private_pstl_cpu_algos_find_if          [system] { header "__pstl/cpu_algos/find_if.h" }
+module std_private_pstl_cpu_algos_for_each         [system] { header "__pstl/cpu_algos/for_each.h" }
+module std_private_pstl_cpu_algos_merge            [system] { header "__pstl/cpu_algos/merge.h" }
+module std_private_pstl_cpu_algos_stable_sort      [system] { header "__pstl/cpu_algos/stable_sort.h" }
+module std_private_pstl_cpu_algos_transform        [system] { header "__pstl/cpu_algos/transform.h" }
+module std_private_pstl_cpu_algos_transform_reduce [system] { header "__pstl/cpu_algos/transform_reduce.h" }
+module std_private_pstl_dispatch                   [system] { header "__pstl/dispatch.h" }
+module std_private_pstl_handle_exception           [system] { header "__pstl/handle_exception.h" }
+
+module std_private_queue_fwd [system] { header "__fwd/queue.h" }
+
+module std_private_ostream_basic_ostream [system] {
+  header "__ostream/basic_ostream.h"
+  export std_streambuf
+}
+module std_private_ostream_print         [system] {
+  header "__ostream/print.h"
+  export std_print
+}
+
+module std_private_random_bernoulli_distribution          [system] { header "__random/bernoulli_distribution.h" }
+module std_private_random_binomial_distribution           [system] { header "__random/binomial_distribution.h" }
+module std_private_random_cauchy_distribution             [system] { header "__random/cauchy_distribution.h" }
+module std_private_random_chi_squared_distribution        [system] { header "__random/chi_squared_distribution.h" }
+module std_private_random_clamp_to_integral               [system] { header "__random/clamp_to_integral.h" }
+module std_private_random_default_random_engine           [system] { header "__random/default_random_engine.h" }
+module std_private_random_discard_block_engine            [system] { header "__random/discard_block_engine.h" }
+module std_private_random_discrete_distribution           [system] {
+  header "__random/discrete_distribution.h"
+  export *
+}
+module std_private_random_exponential_distribution        [system] { header "__random/exponential_distribution.h" }
+module std_private_random_extreme_value_distribution      [system] { header "__random/extreme_value_distribution.h" }
+module std_private_random_fisher_f_distribution           [system] { header "__random/fisher_f_distribution.h" }
+module std_private_random_gamma_distribution              [system] { header "__random/gamma_distribution.h" }
+module std_private_random_generate_canonical              [system] { header "__random/generate_canonical.h" }
+module std_private_random_geometric_distribution          [system] { header "__random/geometric_distribution.h" }
+module std_private_random_independent_bits_engine         [system] { header "__random/independent_bits_engine.h" }
+module std_private_random_is_seed_sequence                [system] { header "__random/is_seed_sequence.h" }
+module std_private_random_is_valid                        [system] { header "__random/is_valid.h" }
+module std_private_random_knuth_b                         [system] { header "__random/knuth_b.h" }
+module std_private_random_linear_congruential_engine      [system] { header "__random/linear_congruential_engine.h" }
+module std_private_random_log2                            [system] { header "__random/log2.h" }
+module std_private_random_lognormal_distribution          [system] { header "__random/lognormal_distribution.h" }
+module std_private_random_mersenne_twister_engine         [system] { header "__random/mersenne_twister_engine.h" }
+module std_private_random_negative_binomial_distribution  [system] { header "__random/negative_binomial_distribution.h" }
+module std_private_random_normal_distribution             [system] { header "__random/normal_distribution.h" }
+module std_private_random_piecewise_constant_distribution [system] {
+  header "__random/piecewise_constant_distribution.h"
+  export *
+}
+module std_private_random_piecewise_linear_distribution   [system] {
+  header "__random/piecewise_linear_distribution.h"
+  export *
+}
+module std_private_random_poisson_distribution            [system] { header "__random/poisson_distribution.h" }
+module std_private_random_random_device                   [system] {
+  header "__random/random_device.h"
+  export *
+}
+module std_private_random_ranlux                          [system] { header "__random/ranlux.h" }
+module std_private_random_seed_seq                        [system] {
+  header "__random/seed_seq.h"
+  export *
+}
+module std_private_random_shuffle_order_engine            [system] { header "__random/shuffle_order_engine.h" }
+module std_private_random_student_t_distribution          [system] { header "__random/student_t_distribution.h" }
+module std_private_random_subtract_with_carry_engine      [system] { header "__random/subtract_with_carry_engine.h" }
+module std_private_random_uniform_int_distribution        [system] { header "__random/uniform_int_distribution.h" }
+module std_private_random_uniform_random_bit_generator    [system] { header "__random/uniform_random_bit_generator.h" }
+module std_private_random_uniform_real_distribution       [system] { header "__random/uniform_real_distribution.h" }
+module std_private_random_weibull_distribution            [system] { header "__random/weibull_distribution.h" }
+
+module std_private_ranges_access                     [system] { header "__ranges/access.h" }
+module std_private_ranges_all                        [system] {
+  header "__ranges/all.h"
+  export std_private_functional_compose
+  export std_private_functional_perfect_forward
+  export std_private_ranges_owning_view
+}
+module std_private_ranges_as_rvalue_view             [system] { header "__ranges/as_rvalue_view.h" }
+module std_private_ranges_chunk_by_view              [system] { header "__ranges/chunk_by_view.h" }
+module std_private_ranges_common_view                [system] { header "__ranges/common_view.h" }
+module std_private_ranges_concepts                   [system] {
+  header "__ranges/concepts.h"
+  export std_private_iterator_concepts
+}
+module std_private_ranges_container_compatible_range [system] { header "__ranges/container_compatible_range.h" }
+module std_private_ranges_counted                    [system] {
+  header "__ranges/counted.h"
+  export std_span
+}
+module std_private_ranges_dangling                   [system] { header "__ranges/dangling.h" }
+module std_private_ranges_data                       [system] { header "__ranges/data.h" }
+module std_private_ranges_drop_view                  [system] { header "__ranges/drop_view.h" }
+module std_private_ranges_drop_while_view            [system] { header "__ranges/drop_while_view.h" }
+module std_private_ranges_elements_view              [system] { header "__ranges/elements_view.h" }
+module std_private_ranges_empty                      [system] { header "__ranges/empty.h" }
+module std_private_ranges_empty_view                 [system] { header "__ranges/empty_view.h" }
+module std_private_ranges_enable_borrowed_range      [system] { header "__ranges/enable_borrowed_range.h" }
+module std_private_ranges_enable_view                [system] { header "__ranges/enable_view.h" }
+module std_private_ranges_filter_view                [system] {
+  header "__ranges/filter_view.h"
+  export std_private_ranges_range_adaptor
+}
+module std_private_ranges_from_range                 [system] { header "__ranges/from_range.h" }
+module std_private_ranges_iota_view                  [system] { header "__ranges/iota_view.h" }
+module std_private_ranges_istream_view               [system] {
+  header "__ranges/istream_view.h"
+}
+module std_private_ranges_join_view                  [system] {
+  header "__ranges/join_view.h"
+  export std_private_iterator_iterator_with_data
+  export std_private_iterator_segmented_iterator
+}
+module std_private_ranges_lazy_split_view            [system] {
+  header "__ranges/lazy_split_view.h"
+  export std_private_ranges_non_propagating_cache
+}
+module std_private_ranges_movable_box                [system] { header "__ranges/movable_box.h" }
+module std_private_ranges_non_propagating_cache      [system] { header "__ranges/non_propagating_cache.h" }
+module std_private_ranges_owning_view                [system] { header "__ranges/owning_view.h" }
+module std_private_ranges_range_adaptor              [system] { header "__ranges/range_adaptor.h" }
+module std_private_ranges_rbegin                     [system] { header "__ranges/rbegin.h" }
+module std_private_ranges_ref_view                   [system] { header "__ranges/ref_view.h" }
+module std_private_ranges_rend                       [system] { header "__ranges/rend.h" }
+module std_private_ranges_repeat_view                [system] { header "__ranges/repeat_view.h" }
+module std_private_ranges_reverse_view               [system] { header "__ranges/reverse_view.h" }
+module std_private_ranges_single_view                [system] { header "__ranges/single_view.h" }
+module std_private_ranges_size                       [system] {
+  header "__ranges/size.h"
+  export std_private_type_traits_make_unsigned
+}
+module std_private_ranges_split_view                 [system] { header "__ranges/split_view.h" }
+module std_private_ranges_subrange                   [system] {
+  header "__ranges/subrange.h"
+  export std_private_ranges_subrange_fwd
+}
+module std_private_ranges_subrange_fwd               [system] {
+  header "__fwd/subrange.h"
+  export std_private_iterator_concepts
+}
+module std_private_ranges_take_view                  [system] { header "__ranges/take_view.h" }
+module std_private_ranges_take_while_view            [system] { header "__ranges/take_while_view.h" }
+module std_private_ranges_to                         [system] { header "__ranges/to.h" }
+module std_private_ranges_transform_view             [system] {
+  header "__ranges/transform_view.h"
+  export std_private_functional_bind_back
+  export std_private_functional_perfect_forward
+  export std_private_ranges_movable_box
+}
+module std_private_ranges_view_interface             [system] { header "__ranges/view_interface.h" }
+module std_private_ranges_views                      [system] { header "__ranges/views.h" }
+module std_private_ranges_zip_view                   [system] {
+  header "__ranges/zip_view.h"
+  export std_private_utility_pair
+}
+
+module std_private_span_span_fwd [system] { header "__fwd/span.h" }
+
+module std_private_stack_fwd [system] { header "__fwd/stack.h" }
+
+module std_private_stop_token_atomic_unique_lock   [system] { header "__stop_token/atomic_unique_lock.h" }
+module std_private_stop_token_intrusive_list_view  [system] { header "__stop_token/intrusive_list_view.h" }
+module std_private_stop_token_intrusive_shared_ptr [system] { header "__stop_token/intrusive_shared_ptr.h" }
+module std_private_stop_token_stop_callback        [system] { header "__stop_token/stop_callback.h" }
+module std_private_stop_token_stop_source          [system] {
+  header "__stop_token/stop_source.h"
+  export *
+}
+module std_private_stop_token_stop_state           [system] {
+  header "__stop_token/stop_state.h"
+  export *
+}
+module std_private_stop_token_stop_token           [system] {
+  header "__stop_token/stop_token.h"
+  export *
+}
+
+module std_private_string_char_traits           [system] {
+  header "__string/char_traits.h"
+  export *
+}
+module std_private_string_constexpr_c_functions [system] {
+  header "__string/constexpr_c_functions.h"
+  export std_private_type_traits_is_equality_comparable
+}
+module std_private_string_extern_template_lists [system] { header "__string/extern_template_lists.h" }
+module std_private_string_string_fwd            [system] { header "__fwd/string.h" }
+
+module std_private_string_view_string_view_fwd [system] { header "__fwd/string_view.h" }
+
+module std_private_system_error_errc            [system] { header "__system_error/errc.h" }
+module std_private_system_error_error_category  [system] { header "__system_error/error_category.h" }
+module std_private_system_error_error_code      [system] {
+  header "__system_error/error_code.h"
+  export std_private_functional_hash
+  export std_private_functional_unary_function
+}
+module std_private_system_error_error_condition [system] {
+  header "__system_error/error_condition.h"
+  export std_private_functional_hash
+  export std_private_functional_unary_function
+}
+module std_private_system_error_system_error    [system] { header "__system_error/system_error.h" }
+
+module std_private_thread_formatter            [system] { header "__thread/formatter.h" }
+module std_private_thread_id                   [system] { header "__thread/id.h" }
+module std_private_thread_jthread              [system] {
+  header "__thread/jthread.h"
+  export *
+}
+module std_private_thread_poll_with_backoff    [system] { header "__thread/poll_with_backoff.h" }
+module std_private_thread_support              [system] {
+  header "__thread/support.h"
+  export *
+}
+module std_private_thread_support_c11          [system] { textual header "__thread/support/c11.h" }
+module std_private_thread_support_external     [system] { textual header "__thread/support/external.h" }
+module std_private_thread_support_pthread      [system] { textual header "__thread/support/pthread.h" }
+module std_private_thread_support_windows      [system] { textual header "__thread/support/windows.h" }
+module std_private_thread_this_thread          [system] { header "__thread/this_thread.h" }
+module std_private_thread_thread               [system] {
+  header "__thread/thread.h"
+  export *
+}
+module std_private_thread_timed_backoff_policy [system] { header "__thread/timed_backoff_policy.h" }
+
+module std_private_tuple_find_index             [system] { header "__tuple/find_index.h" }
+module std_private_tuple_ignore                 [system] { header "__tuple/ignore.h" }
+module std_private_tuple_make_tuple_types       [system] { header "__tuple/make_tuple_types.h" }
+module std_private_tuple_tuple_like_no_subrange [system] {
+  header "__tuple/tuple_like_no_subrange.h"
+}
+module std_private_tuple_sfinae_helpers         [system] { header "__tuple/sfinae_helpers.h" }
+module std_private_tuple_tuple_element          [system] { header "__tuple/tuple_element.h" }
+module std_private_tuple_tuple_fwd              [system] { header "__fwd/tuple.h" }
+module std_private_tuple_tuple_indices          [system] { header "__tuple/tuple_indices.h" }
+module std_private_tuple_tuple_like             [system] {
+  header "__tuple/tuple_like.h"
+  export *
+}
+module std_private_tuple_tuple_like_ext         [system] { header "__tuple/tuple_like_ext.h" }
+module std_private_tuple_tuple_size             [system] { header "__tuple/tuple_size.h" }
+module std_private_tuple_tuple_types            [system] { header "__tuple/tuple_types.h" }
+
+module std_private_type_traits_add_const                                 [system] { header "__type_traits/add_const.h" }
+module std_private_type_traits_add_cv                                    [system] { header "__type_traits/add_cv.h" }
+module std_private_type_traits_add_lvalue_reference                      [system] {
+  header "__type_traits/add_lvalue_reference.h"
+  export std_private_type_traits_is_referenceable
+}
+module std_private_type_traits_add_pointer                               [system] { header "__type_traits/add_pointer.h" }
+module std_private_type_traits_add_rvalue_reference                      [system] { header "__type_traits/add_rvalue_reference.h" }
+module std_private_type_traits_add_volatile                              [system] { header "__type_traits/add_volatile.h" }
+module std_private_type_traits_aligned_storage                           [system] { header "__type_traits/aligned_storage.h" }
+module std_private_type_traits_aligned_union                             [system] { header "__type_traits/aligned_union.h" }
+module std_private_type_traits_alignment_of                              [system] { header "__type_traits/alignment_of.h" }
+module std_private_type_traits_can_extract_key                           [system] { header "__type_traits/can_extract_key.h" }
+module std_private_type_traits_common_reference                          [system] {
+  header "__type_traits/common_reference.h"
+  export std_private_type_traits_remove_cvref
+}
+module std_private_type_traits_common_type                               [system] {
+  header "__type_traits/common_type.h"
+  export std_private_utility_declval
+}
+module std_private_type_traits_conditional                               [system] { header "__type_traits/conditional.h" }
+module std_private_type_traits_conjunction                               [system] { header "__type_traits/conjunction.h" }
+module std_private_type_traits_copy_cv                                   [system] { header "__type_traits/copy_cv.h" }
+module std_private_type_traits_copy_cvref                                [system] { header "__type_traits/copy_cvref.h" }
+module std_private_type_traits_datasizeof                                [system] { header "__type_traits/datasizeof.h" }
+module std_private_type_traits_decay                                     [system] {
+  header "__type_traits/decay.h"
+  export std_private_type_traits_add_pointer
+}
+module std_private_type_traits_dependent_type                            [system] { header "__type_traits/dependent_type.h" }
+module std_private_type_traits_desugars_to                               [system] { header "__type_traits/desugars_to.h" }
+module std_private_type_traits_disjunction                               [system] { header "__type_traits/disjunction.h" }
+module std_private_type_traits_enable_if                                 [system] { header "__type_traits/enable_if.h" }
+module std_private_type_traits_extent                                    [system] { header "__type_traits/extent.h" }
+module std_private_type_traits_has_unique_object_representation          [system] { header "__type_traits/has_unique_object_representation.h" }
+module std_private_type_traits_has_virtual_destructor                    [system] { header "__type_traits/has_virtual_destructor.h" }
+module std_private_type_traits_integral_constant                         [system] { header "__type_traits/integral_constant.h" }
+module std_private_type_traits_invoke                                    [system] {
+  header "__type_traits/invoke.h"
+  export std_private_type_traits_conditional
+  export std_private_type_traits_decay
+  export std_private_type_traits_decay
+  export std_private_type_traits_enable_if
+  export std_private_type_traits_is_base_of
+  export std_private_type_traits_is_core_convertible
+  export std_private_type_traits_is_reference_wrapper
+  export std_private_type_traits_is_same
+  export std_private_type_traits_is_void
+  export std_private_type_traits_nat
+  export std_private_type_traits_remove_cv
+}
+module std_private_type_traits_is_abstract                               [system] { header "__type_traits/is_abstract.h" }
+module std_private_type_traits_is_aggregate                              [system] { header "__type_traits/is_aggregate.h" }
+module std_private_type_traits_is_allocator                              [system] { header "__type_traits/is_allocator.h" }
+module std_private_type_traits_is_always_bitcastable                     [system] { header "__type_traits/is_always_bitcastable.h" }
+module std_private_type_traits_is_arithmetic                             [system] {
+  header "__type_traits/is_arithmetic.h"
+  export std_private_type_traits_integral_constant
+}
+module std_private_type_traits_is_array                                  [system] {
+  header "__type_traits/is_array.h"
+  export std_private_type_traits_integral_constant
+}
+module std_private_type_traits_is_assignable                             [system] { header "__type_traits/is_assignable.h" }
+module std_private_type_traits_is_base_of                                [system] { header "__type_traits/is_base_of.h" }
+module std_private_type_traits_is_bounded_array                          [system] { header "__type_traits/is_bounded_array.h" }
+module std_private_type_traits_is_callable                               [system] {
+  header "__type_traits/is_callable.h"
+  export std_private_type_traits_integral_constant
+}
+module std_private_type_traits_is_char_like_type                         [system] { header "__type_traits/is_char_like_type.h" }
+module std_private_type_traits_is_class                                  [system] { header "__type_traits/is_class.h" }
+module std_private_type_traits_is_compound                               [system] { header "__type_traits/is_compound.h" }
+module std_private_type_traits_is_const                                  [system] { header "__type_traits/is_const.h" }
+module std_private_type_traits_is_constant_evaluated                     [system] { header "__type_traits/is_constant_evaluated.h" }
+module std_private_type_traits_is_constructible                          [system] { header "__type_traits/is_constructible.h" }
+module std_private_type_traits_is_convertible                            [system] {
+  header "__type_traits/is_convertible.h"
+  export std_private_type_traits_is_array
+}
+module std_private_type_traits_is_copy_assignable                        [system] { header "__type_traits/is_copy_assignable.h" }
+module std_private_type_traits_is_copy_constructible                     [system] { header "__type_traits/is_copy_constructible.h" }
+module std_private_type_traits_is_core_convertible                       [system] {
+  header "__type_traits/is_core_convertible.h"
+  export std_private_type_traits_integral_constant
+}
+module std_private_type_traits_is_destructible                           [system] { header "__type_traits/is_destructible.h" }
+module std_private_type_traits_is_empty                                  [system] { header "__type_traits/is_empty.h" }
+module std_private_type_traits_is_enum                                   [system] {
+  header "__type_traits/is_enum.h"
+  export std_private_type_traits_integral_constant
+}
+module std_private_type_traits_is_equality_comparable                    [system] {
+  header "__type_traits/is_equality_comparable.h"
+  export std_private_type_traits_integral_constant
+}
+module std_private_type_traits_is_execution_policy                       [system] {
+  header "__type_traits/is_execution_policy.h"
+  export std_private_type_traits_remove_cvref
+}
+module std_private_type_traits_is_final                                  [system] { header "__type_traits/is_final.h" }
+module std_private_type_traits_is_floating_point                         [system] { header "__type_traits/is_floating_point.h" }
+module std_private_type_traits_is_function                               [system] { header "__type_traits/is_function.h" }
+module std_private_type_traits_is_fundamental                            [system] { header "__type_traits/is_fundamental.h" }
+module std_private_type_traits_is_implicitly_default_constructible       [system] {
+  header "__type_traits/is_implicitly_default_constructible.h"
+  export std_private_type_traits_integral_constant
+}
+module std_private_type_traits_is_integral                               [system] { header "__type_traits/is_integral.h" }
+module std_private_type_traits_is_literal_type                           [system] { header "__type_traits/is_literal_type.h" }
+module std_private_type_traits_is_member_pointer                         [system] { header "__type_traits/is_member_pointer.h" }
+module std_private_type_traits_is_nothrow_assignable                     [system] { header "__type_traits/is_nothrow_assignable.h" }
+module std_private_type_traits_is_nothrow_constructible                  [system] {
+  header "__type_traits/is_nothrow_constructible.h"
+  export std_private_type_traits_integral_constant
+}
+module std_private_type_traits_is_nothrow_convertible                    [system] { header "__type_traits/is_nothrow_convertible.h" }
+module std_private_type_traits_is_nothrow_destructible                   [system] {
+  header "__type_traits/is_nothrow_destructible.h"
+  export std_private_type_traits_is_destructible
+}
+module std_private_type_traits_is_null_pointer                           [system] {
+  header "__type_traits/is_null_pointer.h"
+  export std_cstddef
+}
+module std_private_type_traits_is_object                                 [system] {
+  header "__type_traits/is_object.h"
+  export std_private_type_traits_is_scalar
+}
+module std_private_type_traits_is_pod                                    [system] { header "__type_traits/is_pod.h" }
+module std_private_type_traits_is_pointer                                [system] { header "__type_traits/is_pointer.h" }
+module std_private_type_traits_is_polymorphic                            [system] { header "__type_traits/is_polymorphic.h" }
+module std_private_type_traits_is_primary_template                       [system] {
+  header "__type_traits/is_primary_template.h"
+  export std_private_type_traits_enable_if
+}
+module std_private_type_traits_is_reference                              [system] { header "__type_traits/is_reference.h" }
+module std_private_type_traits_is_reference_wrapper                      [system] { header "__type_traits/is_reference_wrapper.h" }
+module std_private_type_traits_is_referenceable                          [system] { header "__type_traits/is_referenceable.h" }
+module std_private_type_traits_is_same                                   [system] {
+  header "__type_traits/is_same.h"
+  export std_private_type_traits_integral_constant
+}
+module std_private_type_traits_is_scalar                                 [system] {
+  header "__type_traits/is_scalar.h"
+  export std_private_type_traits_is_null_pointer
+}
+module std_private_type_traits_is_signed                                 [system] { header "__type_traits/is_signed.h" }
+module std_private_type_traits_is_signed_integer                         [system] { header "__type_traits/is_signed_integer.h" }
+module std_private_type_traits_is_specialization                         [system] { header "__type_traits/is_specialization.h" }
+module std_private_type_traits_is_standard_layout                        [system] { header "__type_traits/is_standard_layout.h" }
+module std_private_type_traits_is_swappable                              [system] {
+  header "__type_traits/is_swappable.h"
+  export std_private_type_traits_is_move_constructible
+}
+module std_private_type_traits_is_trivial                                [system] { header "__type_traits/is_trivial.h" }
+module std_private_type_traits_is_trivially_assignable                   [system] { header "__type_traits/is_trivially_assignable.h" }
+module std_private_type_traits_is_trivially_constructible                [system] { header "__type_traits/is_trivially_constructible.h" }
+module std_private_type_traits_is_trivially_copyable                     [system] { header "__type_traits/is_trivially_copyable.h" }
+module std_private_type_traits_is_trivially_destructible                 [system] { header "__type_traits/is_trivially_destructible.h" }
+module std_private_type_traits_is_trivially_lexicographically_comparable [system] { header "__type_traits/is_trivially_lexicographically_comparable.h" }
+module std_private_type_traits_is_trivially_relocatable                  [system] { header "__type_traits/is_trivially_relocatable.h" }
+module std_private_type_traits_is_unbounded_array                        [system] { header "__type_traits/is_unbounded_array.h" }
+module std_private_type_traits_is_union                                  [system] { header "__type_traits/is_union.h" }
+module std_private_type_traits_is_unsigned                               [system] { header "__type_traits/is_unsigned.h" }
+module std_private_type_traits_is_unsigned_integer                       [system] { header "__type_traits/is_unsigned_integer.h" }
+module std_private_type_traits_is_valid_expansion                        [system] { header "__type_traits/is_valid_expansion.h" }
+module std_private_type_traits_is_void                                   [system] {
+  header "__type_traits/is_void.h"
+  export std_private_type_traits_integral_constant
+}
+module std_private_type_traits_is_volatile                               [system] { header "__type_traits/is_volatile.h" }
+module std_private_type_traits_lazy                                      [system] { header "__type_traits/lazy.h" }
+module std_private_type_traits_make_32_64_or_128_bit                     [system] { header "__type_traits/make_32_64_or_128_bit.h" }
+module std_private_type_traits_make_const_lvalue_ref                     [system] { header "__type_traits/make_const_lvalue_ref.h" }
+module std_private_type_traits_make_signed                               [system] { header "__type_traits/make_signed.h" }
+module std_private_type_traits_make_unsigned                             [system] {
+  header "__type_traits/make_unsigned.h"
+  export std_private_type_traits_is_unsigned
+}
+module std_private_type_traits_maybe_const                               [system] { header "__type_traits/maybe_const.h" }
+module std_private_type_traits_nat                                       [system] { header "__type_traits/nat.h" }
+module std_private_type_traits_negation                                  [system] { header "__type_traits/negation.h" }
+module std_private_type_traits_noexcept_move_assign_container            [system] { header "__type_traits/noexcept_move_assign_container.h" }
+module std_private_type_traits_promote                                   [system] { header "__type_traits/promote.h" }
+module std_private_type_traits_rank                                      [system] { header "__type_traits/rank.h" }
+module std_private_type_traits_remove_all_extents                        [system] { header "__type_traits/remove_all_extents.h" }
+module std_private_type_traits_remove_const                              [system] { header "__type_traits/remove_const.h" }
+module std_private_type_traits_remove_const_ref                          [system] { header "__type_traits/remove_const_ref.h" }
+module std_private_type_traits_remove_cv                                 [system] {
+  header "__type_traits/remove_cv.h"
+  export std_private_type_traits_remove_const
+  export std_private_type_traits_remove_volatile
+}
+module std_private_type_traits_remove_cvref                              [system] { header "__type_traits/remove_cvref.h" }
+module std_private_type_traits_remove_extent                             [system] { header "__type_traits/remove_extent.h" }
+module std_private_type_traits_remove_pointer                            [system] { header "__type_traits/remove_pointer.h" }
+module std_private_type_traits_remove_reference                          [system] { header "__type_traits/remove_reference.h" }
+module std_private_type_traits_remove_volatile                           [system] { header "__type_traits/remove_volatile.h" }
+module std_private_type_traits_result_of                                 [system] { header "__type_traits/result_of.h" }
+module std_private_type_traits_strip_signature                           [system] { header "__type_traits/strip_signature.h" }
+module std_private_type_traits_type_identity                             [system] { header "__type_traits/type_identity.h" }
+module std_private_type_traits_type_list                                 [system] { header "__type_traits/type_list.h" }
+module std_private_type_traits_underlying_type                           [system] {
+  header "__type_traits/underlying_type.h"
+  export std_private_type_traits_is_enum
+}
+module std_private_type_traits_unwrap_ref                                [system] { header "__type_traits/unwrap_ref.h" }
+module std_private_type_traits_void_t                                    [system] { header "__type_traits/void_t.h" }
+
+module std_private_utility_as_const               [system] { header "__utility/as_const.h" }
+module std_private_utility_as_lvalue              [system] { header "__utility/as_lvalue.h" }
+module std_private_utility_auto_cast              [system] {
+  header "__utility/auto_cast.h"
+  export std_private_type_traits_decay
+}
+module std_private_utility_cmp                    [system] {
+  header "__utility/cmp.h"
+  export std_private_type_traits_make_unsigned
+}
+module std_private_utility_convert_to_integral    [system] { header "__utility/convert_to_integral.h" }
+module std_private_utility_declval                [system] { header "__utility/declval.h" }
+module std_private_utility_empty                  [system] { header "__utility/empty.h" }
+module std_private_utility_exception_guard        [system] { header "__utility/exception_guard.h" }
+module std_private_utility_exchange               [system] { header "__utility/exchange.h" }
+module std_private_utility_forward                [system] { header "__utility/forward.h" }
+module std_private_utility_forward_like           [system] { header "__utility/forward_like.h" }
+module std_private_utility_in_place               [system] { header "__utility/in_place.h" }
+module std_private_utility_integer_sequence       [system] { header "__utility/integer_sequence.h" }
+module std_private_utility_is_pointer_in_range    [system] { header "__utility/is_pointer_in_range.h" }
+module std_private_utility_is_valid_range         [system] { header "__utility/is_valid_range.h" }
+module std_private_utility_move                   [system] {
+  header "__utility/move.h"
+  export std_private_type_traits_is_copy_constructible
+  export std_private_type_traits_is_nothrow_move_constructible
+  export std_private_type_traits_remove_reference
+}
+module std_private_utility_no_destroy             [system] { header "__utility/no_destroy.h" }
+module std_private_utility_pair                   [system] {
+  header "__utility/pair.h"
+  export std_private_ranges_subrange_fwd
+  export std_private_tuple_pair_like
+  export std_private_type_traits_is_assignable
+  export std_private_type_traits_is_constructible
+  export std_private_type_traits_is_convertible
+  export std_private_type_traits_is_copy_assignable
+  export std_private_type_traits_is_move_assignable
+  export std_private_type_traits_is_nothrow_copy_constructible
+  export std_private_type_traits_is_nothrow_default_constructible
+  export std_private_type_traits_is_nothrow_move_assignable
+  export std_private_utility_pair_fwd
+}
+module std_private_utility_pair_fwd                [system] { header "__fwd/pair.h" }
+module std_private_utility_piecewise_construct     [system] { header "__utility/piecewise_construct.h" }
+module std_private_utility_priority_tag            [system] { header "__utility/priority_tag.h" }
+module std_private_utility_private_constructor_tag [system] { header "__utility/private_constructor_tag.h" }
+module std_private_utility_rel_ops                 [system] { header "__utility/rel_ops.h" }
+module std_private_utility_small_buffer            [system] { header "__utility/small_buffer.h" }
+module std_private_utility_swap                    [system] {
+  header "__utility/swap.h"
+  export std_private_type_traits_is_swappable
+}
+module std_private_utility_to_underlying           [system] { header "__utility/to_underlying.h" }
+module std_private_utility_unreachable             [system] { header "__utility/unreachable.h" }
+
+module std_private_variant_monostate               [system] { header "__variant/monostate.h" }
+
+module std_private_vector_fwd                      [system] { header "__fwd/vector.h" }
diff --git a/libcxx/include/__cxx03/mutex b/libcxx/include/__cxx03/mutex
new file mode 100644
index 000000000000..02c52dd72f02
--- /dev/null
+++ b/libcxx/include/__cxx03/mutex
@@ -0,0 +1,516 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_MUTEX
+#define _LIBCPP_MUTEX
+
+/*
+    mutex synopsis
+
+namespace std
+{
+
+class mutex
+{
+public:
+     constexpr mutex() noexcept;
+     ~mutex();
+
+    mutex(const mutex&) = delete;
+    mutex& operator=(const mutex&) = delete;
+
+    void lock();
+    bool try_lock();
+    void unlock();
+
+    typedef pthread_mutex_t* native_handle_type;
+    native_handle_type native_handle();
+};
+
+class recursive_mutex
+{
+public:
+     recursive_mutex();
+     ~recursive_mutex();
+
+    recursive_mutex(const recursive_mutex&) = delete;
+    recursive_mutex& operator=(const recursive_mutex&) = delete;
+
+    void lock();
+    bool try_lock() noexcept;
+    void unlock();
+
+    typedef pthread_mutex_t* native_handle_type;
+    native_handle_type native_handle();
+};
+
+class timed_mutex
+{
+public:
+     timed_mutex();
+     ~timed_mutex();
+
+    timed_mutex(const timed_mutex&) = delete;
+    timed_mutex& operator=(const timed_mutex&) = delete;
+
+    void lock();
+    bool try_lock();
+    template <class Rep, class Period>
+        bool try_lock_for(const chrono::duration<Rep, Period>& rel_time);
+    template <class Clock, class Duration>
+        bool try_lock_until(const chrono::time_point<Clock, Duration>& abs_time);
+    void unlock();
+};
+
+class recursive_timed_mutex
+{
+public:
+     recursive_timed_mutex();
+     ~recursive_timed_mutex();
+
+    recursive_timed_mutex(const recursive_timed_mutex&) = delete;
+    recursive_timed_mutex& operator=(const recursive_timed_mutex&) = delete;
+
+    void lock();
+    bool try_lock() noexcept;
+    template <class Rep, class Period>
+        bool try_lock_for(const chrono::duration<Rep, Period>& rel_time);
+    template <class Clock, class Duration>
+        bool try_lock_until(const chrono::time_point<Clock, Duration>& abs_time);
+    void unlock();
+};
+
+struct defer_lock_t { explicit defer_lock_t() = default; };
+struct try_to_lock_t { explicit try_to_lock_t() = default; };
+struct adopt_lock_t { explicit adopt_lock_t() = default; };
+
+inline constexpr defer_lock_t  defer_lock{};
+inline constexpr try_to_lock_t try_to_lock{};
+inline constexpr adopt_lock_t  adopt_lock{};
+
+template <class Mutex>
+class lock_guard
+{
+public:
+    typedef Mutex mutex_type;
+
+    explicit lock_guard(mutex_type& m);
+    lock_guard(mutex_type& m, adopt_lock_t);
+    ~lock_guard();
+
+    lock_guard(lock_guard const&) = delete;
+    lock_guard& operator=(lock_guard const&) = delete;
+};
+
+template <class... MutexTypes>
+class scoped_lock // C++17
+{
+public:
+    using mutex_type = Mutex;  // Only if sizeof...(MutexTypes) == 1
+
+    explicit scoped_lock(MutexTypes&... m);
+    scoped_lock(adopt_lock_t, MutexTypes&... m);
+    ~scoped_lock();
+    scoped_lock(scoped_lock const&) = delete;
+    scoped_lock& operator=(scoped_lock const&) = delete;
+private:
+    tuple<MutexTypes&...> pm; // exposition only
+};
+
+template <class Mutex>
+class unique_lock
+{
+public:
+    typedef Mutex mutex_type;
+    unique_lock() noexcept;
+    explicit unique_lock(mutex_type& m);
+    unique_lock(mutex_type& m, defer_lock_t) noexcept;
+    unique_lock(mutex_type& m, try_to_lock_t);
+    unique_lock(mutex_type& m, adopt_lock_t);
+    template <class Clock, class Duration>
+        unique_lock(mutex_type& m, const chrono::time_point<Clock, Duration>& abs_time);
+    template <class Rep, class Period>
+        unique_lock(mutex_type& m, const chrono::duration<Rep, Period>& rel_time);
+    ~unique_lock();
+
+    unique_lock(unique_lock const&) = delete;
+    unique_lock& operator=(unique_lock const&) = delete;
+
+    unique_lock(unique_lock&& u) noexcept;
+    unique_lock& operator=(unique_lock&& u) noexcept;
+
+    void lock();
+    bool try_lock();
+
+    template <class Rep, class Period>
+        bool try_lock_for(const chrono::duration<Rep, Period>& rel_time);
+    template <class Clock, class Duration>
+        bool try_lock_until(const chrono::time_point<Clock, Duration>& abs_time);
+
+    void unlock();
+
+    void swap(unique_lock& u) noexcept;
+    mutex_type* release() noexcept;
+
+    bool owns_lock() const noexcept;
+    explicit operator bool () const noexcept;
+    mutex_type* mutex() const noexcept;
+};
+
+template <class Mutex>
+  void swap(unique_lock<Mutex>& x, unique_lock<Mutex>& y) noexcept;
+
+template <class L1, class L2, class... L3>
+  int try_lock(L1&, L2&, L3&...);
+template <class L1, class L2, class... L3>
+  void lock(L1&, L2&, L3&...);
+
+struct once_flag
+{
+    constexpr once_flag() noexcept;
+
+    once_flag(const once_flag&) = delete;
+    once_flag& operator=(const once_flag&) = delete;
+};
+
+template<class Callable, class ...Args>
+  void call_once(once_flag& flag, Callable&& func, Args&&... args);
+
+}  // std
+
+*/
+
+#include <__chrono/steady_clock.h>
+#include <__chrono/time_point.h>
+#include <__condition_variable/condition_variable.h>
+#include <__config>
+#include <__memory/shared_ptr.h>
+#include <__mutex/lock_guard.h>
+#include <__mutex/mutex.h>
+#include <__mutex/once_flag.h>
+#include <__mutex/tag_types.h>
+#include <__mutex/unique_lock.h>
+#include <__thread/id.h>
+#include <__thread/support.h>
+#include <__utility/forward.h>
+#include <cstddef>
+#include <limits>
+#ifndef _LIBCPP_CXX03_LANG
+#  include <tuple>
+#endif
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifndef _LIBCPP_HAS_NO_THREADS
+
+class _LIBCPP_EXPORTED_FROM_ABI recursive_mutex {
+  __libcpp_recursive_mutex_t __m_;
+
+public:
+  recursive_mutex();
+  ~recursive_mutex();
+
+  recursive_mutex(const recursive_mutex&)            = delete;
+  recursive_mutex& operator=(const recursive_mutex&) = delete;
+
+  void lock();
+  bool try_lock() _NOEXCEPT;
+  void unlock() _NOEXCEPT;
+
+  typedef __libcpp_recursive_mutex_t* native_handle_type;
+
+  _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; }
+};
+
+class _LIBCPP_EXPORTED_FROM_ABI timed_mutex {
+  mutex __m_;
+  condition_variable __cv_;
+  bool __locked_;
+
+public:
+  timed_mutex();
+  ~timed_mutex();
+
+  timed_mutex(const timed_mutex&)            = delete;
+  timed_mutex& operator=(const timed_mutex&) = delete;
+
+public:
+  void lock();
+  bool try_lock() _NOEXCEPT;
+  template <class _Rep, class _Period>
+  _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __d) {
+    return try_lock_until(chrono::steady_clock::now() + __d);
+  }
+  template <class _Clock, class _Duration>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS bool
+  try_lock_until(const chrono::time_point<_Clock, _Duration>& __t);
+  void unlock() _NOEXCEPT;
+};
+
+template <class _Clock, class _Duration>
+bool timed_mutex::try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) {
+  using namespace chrono;
+  unique_lock<mutex> __lk(__m_);
+  bool __no_timeout = _Clock::now() < __t;
+  while (__no_timeout && __locked_)
+    __no_timeout = __cv_.wait_until(__lk, __t) == cv_status::no_timeout;
+  if (!__locked_) {
+    __locked_ = true;
+    return true;
+  }
+  return false;
+}
+
+class _LIBCPP_EXPORTED_FROM_ABI recursive_timed_mutex {
+  mutex __m_;
+  condition_variable __cv_;
+  size_t __count_;
+  __thread_id __id_;
+
+public:
+  recursive_timed_mutex();
+  ~recursive_timed_mutex();
+
+  recursive_timed_mutex(const recursive_timed_mutex&)            = delete;
+  recursive_timed_mutex& operator=(const recursive_timed_mutex&) = delete;
+
+  void lock();
+  bool try_lock() _NOEXCEPT;
+  template <class _Rep, class _Period>
+  _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __d) {
+    return try_lock_until(chrono::steady_clock::now() + __d);
+  }
+  template <class _Clock, class _Duration>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS bool
+  try_lock_until(const chrono::time_point<_Clock, _Duration>& __t);
+  void unlock() _NOEXCEPT;
+};
+
+template <class _Clock, class _Duration>
+bool recursive_timed_mutex::try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) {
+  using namespace chrono;
+  __thread_id __id = this_thread::get_id();
+  unique_lock<mutex> __lk(__m_);
+  if (__id == __id_) {
+    if (__count_ == numeric_limits<size_t>::max())
+      return false;
+    ++__count_;
+    return true;
+  }
+  bool __no_timeout = _Clock::now() < __t;
+  while (__no_timeout && __count_ != 0)
+    __no_timeout = __cv_.wait_until(__lk, __t) == cv_status::no_timeout;
+  if (__count_ == 0) {
+    __count_ = 1;
+    __id_    = __id;
+    return true;
+  }
+  return false;
+}
+
+template <class _L0, class _L1>
+_LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1) {
+  unique_lock<_L0> __u0(__l0, try_to_lock_t());
+  if (__u0.owns_lock()) {
+    if (__l1.try_lock()) {
+      __u0.release();
+      return -1;
+    } else
+      return 1;
+  }
+  return 0;
+}
+
+#  ifndef _LIBCPP_CXX03_LANG
+
+template <class _L0, class _L1, class _L2, class... _L3>
+_LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3) {
+  int __r = 0;
+  unique_lock<_L0> __u0(__l0, try_to_lock);
+  if (__u0.owns_lock()) {
+    __r = std::try_lock(__l1, __l2, __l3...);
+    if (__r == -1)
+      __u0.release();
+    else
+      ++__r;
+  }
+  return __r;
+}
+
+#  endif // _LIBCPP_CXX03_LANG
+
+template <class _L0, class _L1>
+_LIBCPP_HIDE_FROM_ABI void lock(_L0& __l0, _L1& __l1) {
+  while (true) {
+    {
+      unique_lock<_L0> __u0(__l0);
+      if (__l1.try_lock()) {
+        __u0.release();
+        break;
+      }
+    }
+    __libcpp_thread_yield();
+    {
+      unique_lock<_L1> __u1(__l1);
+      if (__l0.try_lock()) {
+        __u1.release();
+        break;
+      }
+    }
+    __libcpp_thread_yield();
+  }
+}
+
+#  ifndef _LIBCPP_CXX03_LANG
+
+template <class _L0, class _L1, class _L2, class... _L3>
+void __lock_first(int __i, _L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3) {
+  while (true) {
+    switch (__i) {
+    case 0: {
+      unique_lock<_L0> __u0(__l0);
+      __i = std::try_lock(__l1, __l2, __l3...);
+      if (__i == -1) {
+        __u0.release();
+        return;
+      }
+    }
+      ++__i;
+      __libcpp_thread_yield();
+      break;
+    case 1: {
+      unique_lock<_L1> __u1(__l1);
+      __i = std::try_lock(__l2, __l3..., __l0);
+      if (__i == -1) {
+        __u1.release();
+        return;
+      }
+    }
+      if (__i == sizeof...(_L3) + 1)
+        __i = 0;
+      else
+        __i += 2;
+      __libcpp_thread_yield();
+      break;
+    default:
+      std::__lock_first(__i - 2, __l2, __l3..., __l0, __l1);
+      return;
+    }
+  }
+}
+
+template <class _L0, class _L1, class _L2, class... _L3>
+inline _LIBCPP_HIDE_FROM_ABI void lock(_L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3) {
+  std::__lock_first(0, __l0, __l1, __l2, __l3...);
+}
+
+#  endif // _LIBCPP_CXX03_LANG
+
+#  if _LIBCPP_STD_VER >= 17
+template <class... _Mutexes>
+class _LIBCPP_TEMPLATE_VIS scoped_lock;
+
+template <>
+class _LIBCPP_TEMPLATE_VIS scoped_lock<> {
+public:
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI explicit scoped_lock() {}
+  ~scoped_lock() = default;
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI explicit scoped_lock(adopt_lock_t) {}
+
+  scoped_lock(scoped_lock const&)            = delete;
+  scoped_lock& operator=(scoped_lock const&) = delete;
+};
+
+template <class _Mutex>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_THREAD_SAFETY_ANNOTATION(scoped_lockable) scoped_lock<_Mutex> {
+public:
+  typedef _Mutex mutex_type;
+
+private:
+  mutex_type& __m_;
+
+public:
+  [[nodiscard]]
+  _LIBCPP_HIDE_FROM_ABI explicit scoped_lock(mutex_type& __m) _LIBCPP_THREAD_SAFETY_ANNOTATION(acquire_capability(__m))
+      : __m_(__m) {
+    __m_.lock();
+  }
+
+  ~scoped_lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(release_capability()) { __m_.unlock(); }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI explicit scoped_lock(adopt_lock_t, mutex_type& __m)
+      _LIBCPP_THREAD_SAFETY_ANNOTATION(requires_capability(__m))
+      : __m_(__m) {}
+
+  scoped_lock(scoped_lock const&)            = delete;
+  scoped_lock& operator=(scoped_lock const&) = delete;
+};
+
+template <class... _MArgs>
+class _LIBCPP_TEMPLATE_VIS scoped_lock {
+  static_assert(sizeof...(_MArgs) > 1, "At least 2 lock types required");
+  typedef tuple<_MArgs&...> _MutexTuple;
+
+public:
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI explicit scoped_lock(_MArgs&... __margs) : __t_(__margs...) {
+    std::lock(__margs...);
+  }
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI scoped_lock(adopt_lock_t, _MArgs&... __margs) : __t_(__margs...) {}
+
+  _LIBCPP_HIDE_FROM_ABI ~scoped_lock() {
+    typedef typename __make_tuple_indices<sizeof...(_MArgs)>::type _Indices;
+    __unlock_unpack(_Indices{}, __t_);
+  }
+
+  scoped_lock(scoped_lock const&)            = delete;
+  scoped_lock& operator=(scoped_lock const&) = delete;
+
+private:
+  template <size_t... _Indx>
+  _LIBCPP_HIDE_FROM_ABI static void __unlock_unpack(__tuple_indices<_Indx...>, _MutexTuple& __mt) {
+    (std::get<_Indx>(__mt).unlock(), ...);
+  }
+
+  _MutexTuple __t_;
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(scoped_lock);
+
+#  endif // _LIBCPP_STD_VER >= 17
+#endif   // !_LIBCPP_HAS_NO_THREADS
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <concepts>
+#  include <cstdlib>
+#  include <cstring>
+#  include <ctime>
+#  include <initializer_list>
+#  include <iosfwd>
+#  include <new>
+#  include <stdexcept>
+#  include <system_error>
+#  include <type_traits>
+#  include <typeinfo>
+#endif
+
+#endif // _LIBCPP_MUTEX
diff --git a/libcxx/include/__cxx03/new b/libcxx/include/__cxx03/new
new file mode 100644
index 000000000000..214dbc398530
--- /dev/null
+++ b/libcxx/include/__cxx03/new
@@ -0,0 +1,362 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_NEW
+#define _LIBCPP_NEW
+
+/*
+    new synopsis
+
+namespace std
+{
+
+class bad_alloc
+    : public exception
+{
+public:
+    bad_alloc() noexcept;
+    bad_alloc(const bad_alloc&) noexcept;
+    bad_alloc& operator=(const bad_alloc&) noexcept;
+    virtual const char* what() const noexcept;
+};
+
+class bad_array_new_length : public bad_alloc // C++14
+{
+public:
+    bad_array_new_length() noexcept;
+};
+
+enum class align_val_t : size_t {}; // C++17
+
+struct destroying_delete_t { // C++20
+  explicit destroying_delete_t() = default;
+};
+inline constexpr destroying_delete_t destroying_delete{}; // C++20
+
+struct nothrow_t { explicit nothrow_t() = default; };
+extern const nothrow_t nothrow;
+typedef void (*new_handler)();
+new_handler set_new_handler(new_handler new_p) noexcept;
+new_handler get_new_handler() noexcept;
+
+// 21.6.4, pointer optimization barrier
+template <class T> [[nodiscard]] constexpr T* launder(T* p) noexcept;   // C++17, nodiscard since C++20
+}  // std
+
+void* operator new(std::size_t size);                                   // replaceable, nodiscard in C++20
+void* operator new(std::size_t size, std::align_val_t alignment);       // replaceable, C++17, nodiscard in C++20
+void* operator new(std::size_t size, const std::nothrow_t&) noexcept;   // replaceable, nodiscard in C++20
+void* operator new(std::size_t size, std::align_val_t alignment,
+                   const std::nothrow_t&) noexcept;                     // replaceable, C++17, nodiscard in C++20
+void  operator delete(void* ptr) noexcept;                              // replaceable
+void  operator delete(void* ptr, std::size_t size) noexcept;            // replaceable, C++14
+void  operator delete(void* ptr, std::align_val_t alignment) noexcept;  // replaceable, C++17
+void  operator delete(void* ptr, std::size_t size,
+                      std::align_val_t alignment) noexcept;             // replaceable, C++17
+void  operator delete(void* ptr, const std::nothrow_t&) noexcept;       // replaceable
+void  operator delete(void* ptr, std:align_val_t alignment,
+                      const std::nothrow_t&) noexcept;                  // replaceable, C++17
+
+void* operator new[](std::size_t size);                                 // replaceable, nodiscard in C++20
+void* operator new[](std::size_t size,
+                     std::align_val_t alignment) noexcept;              // replaceable, C++17, nodiscard in C++20
+void* operator new[](std::size_t size, const std::nothrow_t&) noexcept; // replaceable, nodiscard in C++20
+void* operator new[](std::size_t size, std::align_val_t alignment,
+                     const std::nothrow_t&) noexcept;                   // replaceable, C++17, nodiscard in C++20
+void  operator delete[](void* ptr) noexcept;                            // replaceable
+void  operator delete[](void* ptr, std::size_t size) noexcept;          // replaceable, C++14
+void  operator delete[](void* ptr,
+                        std::align_val_t alignment) noexcept;           // replaceable, C++17
+void  operator delete[](void* ptr, std::size_t size,
+                        std::align_val_t alignment) noexcept;           // replaceable, C++17
+void  operator delete[](void* ptr, const std::nothrow_t&) noexcept;     // replaceable
+void  operator delete[](void* ptr, std::align_val_t alignment,
+                        const std::nothrow_t&) noexcept;                // replaceable, C++17
+
+void* operator new  (std::size_t size, void* ptr) noexcept;             // nodiscard in C++20
+void* operator new[](std::size_t size, void* ptr) noexcept;             // nodiscard in C++20
+void  operator delete  (void* ptr, void*) noexcept;
+void  operator delete[](void* ptr, void*) noexcept;
+
+*/
+
+#include <__config>
+#include <__exception/exception.h>
+#include <__type_traits/is_function.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_cv.h>
+#include <__verbose_abort>
+#include <cstddef>
+#include <version>
+
+#if defined(_LIBCPP_ABI_VCRUNTIME)
+#  include <new.h>
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(__cpp_sized_deallocation) || __cpp_sized_deallocation < 201309L
+#  define _LIBCPP_HAS_NO_LANGUAGE_SIZED_DEALLOCATION
+#endif
+
+#if !defined(_LIBCPP_BUILDING_LIBRARY) && _LIBCPP_STD_VER < 14 && defined(_LIBCPP_HAS_NO_LANGUAGE_SIZED_DEALLOCATION)
+#  define _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION
+#endif
+
+#if defined(_LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION) || defined(_LIBCPP_HAS_NO_LANGUAGE_SIZED_DEALLOCATION)
+#  define _LIBCPP_HAS_NO_SIZED_DEALLOCATION
+#endif
+
+namespace std // purposefully not using versioning namespace
+{
+
+#if !defined(_LIBCPP_ABI_VCRUNTIME)
+struct _LIBCPP_EXPORTED_FROM_ABI nothrow_t {
+  explicit nothrow_t() = default;
+};
+extern _LIBCPP_EXPORTED_FROM_ABI const nothrow_t nothrow;
+
+class _LIBCPP_EXPORTED_FROM_ABI bad_alloc : public exception {
+public:
+  bad_alloc() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI bad_alloc(const bad_alloc&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI bad_alloc& operator=(const bad_alloc&) _NOEXCEPT = default;
+  ~bad_alloc() _NOEXCEPT override;
+  const char* what() const _NOEXCEPT override;
+};
+
+class _LIBCPP_EXPORTED_FROM_ABI bad_array_new_length : public bad_alloc {
+public:
+  bad_array_new_length() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI bad_array_new_length(const bad_array_new_length&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI bad_array_new_length& operator=(const bad_array_new_length&) _NOEXCEPT = default;
+  ~bad_array_new_length() _NOEXCEPT override;
+  const char* what() const _NOEXCEPT override;
+};
+
+typedef void (*new_handler)();
+_LIBCPP_EXPORTED_FROM_ABI new_handler set_new_handler(new_handler) _NOEXCEPT;
+_LIBCPP_EXPORTED_FROM_ABI new_handler get_new_handler() _NOEXCEPT;
+
+#elif defined(_HAS_EXCEPTIONS) && _HAS_EXCEPTIONS == 0 // !_LIBCPP_ABI_VCRUNTIME
+
+// When _HAS_EXCEPTIONS == 0, these complete definitions are needed,
+// since they would normally be provided in vcruntime_exception.h
+class bad_alloc : public exception {
+public:
+  bad_alloc() noexcept : exception("bad allocation") {}
+
+private:
+  friend class bad_array_new_length;
+
+  bad_alloc(char const* const __message) noexcept : exception(__message) {}
+};
+
+class bad_array_new_length : public bad_alloc {
+public:
+  bad_array_new_length() noexcept : bad_alloc("bad array new length") {}
+};
+#endif // defined(_LIBCPP_ABI_VCRUNTIME) && defined(_HAS_EXCEPTIONS) && _HAS_EXCEPTIONS == 0
+
+_LIBCPP_NORETURN _LIBCPP_EXPORTED_FROM_ABI void __throw_bad_alloc(); // not in C++ spec
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_bad_array_new_length() {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw bad_array_new_length();
+#else
+  _LIBCPP_VERBOSE_ABORT("bad_array_new_length was thrown in -fno-exceptions mode");
+#endif
+}
+
+#if !defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION) && !defined(_LIBCPP_ABI_VCRUNTIME)
+#  ifndef _LIBCPP_CXX03_LANG
+enum class align_val_t : size_t {};
+#  else
+enum align_val_t { __zero = 0, __max = (size_t)-1 };
+#  endif
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+// Enable the declaration even if the compiler doesn't support the language
+// feature.
+struct destroying_delete_t {
+  explicit destroying_delete_t() = default;
+};
+inline constexpr destroying_delete_t destroying_delete{};
+#endif // _LIBCPP_STD_VER >= 20
+
+} // namespace std
+
+#if defined(_LIBCPP_CXX03_LANG)
+#  define _THROW_BAD_ALLOC throw(std::bad_alloc)
+#else
+#  define _THROW_BAD_ALLOC
+#endif
+
+#if !defined(_LIBCPP_ABI_VCRUNTIME)
+
+_LIBCPP_NODISCARD _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz) _THROW_BAD_ALLOC;
+_LIBCPP_NODISCARD _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, const std::nothrow_t&) _NOEXCEPT
+    _LIBCPP_NOALIAS;
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p) _NOEXCEPT;
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, const std::nothrow_t&) _NOEXCEPT;
+#  ifndef _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::size_t __sz) _NOEXCEPT;
+#  endif
+
+_LIBCPP_NODISCARD _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz) _THROW_BAD_ALLOC;
+_LIBCPP_NODISCARD _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz, const std::nothrow_t&) _NOEXCEPT
+    _LIBCPP_NOALIAS;
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p) _NOEXCEPT;
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, const std::nothrow_t&) _NOEXCEPT;
+#  ifndef _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::size_t __sz) _NOEXCEPT;
+#  endif
+
+#  ifndef _LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+_LIBCPP_NODISCARD _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, std::align_val_t) _THROW_BAD_ALLOC;
+_LIBCPP_NODISCARD _LIBCPP_OVERRIDABLE_FUNC_VIS void*
+operator new(std::size_t __sz, std::align_val_t, const std::nothrow_t&) _NOEXCEPT _LIBCPP_NOALIAS;
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::align_val_t) _NOEXCEPT;
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::align_val_t, const std::nothrow_t&) _NOEXCEPT;
+#    ifndef _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::size_t __sz, std::align_val_t) _NOEXCEPT;
+#    endif
+
+_LIBCPP_NODISCARD _LIBCPP_OVERRIDABLE_FUNC_VIS void*
+operator new[](std::size_t __sz, std::align_val_t) _THROW_BAD_ALLOC;
+_LIBCPP_NODISCARD _LIBCPP_OVERRIDABLE_FUNC_VIS void*
+operator new[](std::size_t __sz, std::align_val_t, const std::nothrow_t&) _NOEXCEPT _LIBCPP_NOALIAS;
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::align_val_t) _NOEXCEPT;
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::align_val_t, const std::nothrow_t&) _NOEXCEPT;
+#    ifndef _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::size_t __sz, std::align_val_t) _NOEXCEPT;
+#    endif
+#  endif
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI void* operator new(std::size_t, void* __p) _NOEXCEPT { return __p; }
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI void* operator new[](std::size_t, void* __p) _NOEXCEPT { return __p; }
+inline _LIBCPP_HIDE_FROM_ABI void operator delete(void*, void*) _NOEXCEPT {}
+inline _LIBCPP_HIDE_FROM_ABI void operator delete[](void*, void*) _NOEXCEPT {}
+
+#endif // !_LIBCPP_ABI_VCRUNTIME
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_CONSTEXPR inline _LIBCPP_HIDE_FROM_ABI bool __is_overaligned_for_new(size_t __align) _NOEXCEPT {
+#ifdef __STDCPP_DEFAULT_NEW_ALIGNMENT__
+  return __align > __STDCPP_DEFAULT_NEW_ALIGNMENT__;
+#else
+  return __align > _LIBCPP_ALIGNOF(max_align_t);
+#endif
+}
+
+template <class... _Args>
+_LIBCPP_HIDE_FROM_ABI void* __libcpp_operator_new(_Args... __args) {
+#if __has_builtin(__builtin_operator_new) && __has_builtin(__builtin_operator_delete)
+  return __builtin_operator_new(__args...);
+#else
+  return ::operator new(__args...);
+#endif
+}
+
+template <class... _Args>
+_LIBCPP_HIDE_FROM_ABI void __libcpp_operator_delete(_Args... __args) {
+#if __has_builtin(__builtin_operator_new) && __has_builtin(__builtin_operator_delete)
+  __builtin_operator_delete(__args...);
+#else
+  ::operator delete(__args...);
+#endif
+}
+
+inline _LIBCPP_HIDE_FROM_ABI void* __libcpp_allocate(size_t __size, size_t __align) {
+#ifndef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION
+  if (__is_overaligned_for_new(__align)) {
+    const align_val_t __align_val = static_cast<align_val_t>(__align);
+    return __libcpp_operator_new(__size, __align_val);
+  }
+#endif
+
+  (void)__align;
+  return __libcpp_operator_new(__size);
+}
+
+template <class... _Args>
+_LIBCPP_HIDE_FROM_ABI void __do_deallocate_handle_size(void* __ptr, size_t __size, _Args... __args) {
+#ifdef _LIBCPP_HAS_NO_SIZED_DEALLOCATION
+  (void)__size;
+  return std::__libcpp_operator_delete(__ptr, __args...);
+#else
+  return std::__libcpp_operator_delete(__ptr, __size, __args...);
+#endif
+}
+
+inline _LIBCPP_HIDE_FROM_ABI void __libcpp_deallocate(void* __ptr, size_t __size, size_t __align) {
+#if defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION)
+  (void)__align;
+  return __do_deallocate_handle_size(__ptr, __size);
+#else
+  if (__is_overaligned_for_new(__align)) {
+    const align_val_t __align_val = static_cast<align_val_t>(__align);
+    return __do_deallocate_handle_size(__ptr, __size, __align_val);
+  } else {
+    return __do_deallocate_handle_size(__ptr, __size);
+  }
+#endif
+}
+
+inline _LIBCPP_HIDE_FROM_ABI void __libcpp_deallocate_unsized(void* __ptr, size_t __align) {
+#if defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION)
+  (void)__align;
+  return __libcpp_operator_delete(__ptr);
+#else
+  if (__is_overaligned_for_new(__align)) {
+    const align_val_t __align_val = static_cast<align_val_t>(__align);
+    return __libcpp_operator_delete(__ptr, __align_val);
+  } else {
+    return __libcpp_operator_delete(__ptr);
+  }
+#endif
+}
+
+template <class _Tp>
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp* __launder(_Tp* __p) _NOEXCEPT {
+  static_assert(!(is_function<_Tp>::value), "can't launder functions");
+  static_assert(!(is_same<void, __remove_cv_t<_Tp> >::value), "can't launder cv-void");
+  return __builtin_launder(__p);
+}
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp* launder(_Tp* __p) noexcept {
+  return std::__launder(__p);
+}
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+#  if defined(__GCC_DESTRUCTIVE_SIZE) && defined(__GCC_CONSTRUCTIVE_SIZE)
+
+inline constexpr size_t hardware_destructive_interference_size  = __GCC_DESTRUCTIVE_SIZE;
+inline constexpr size_t hardware_constructive_interference_size = __GCC_CONSTRUCTIVE_SIZE;
+
+#  endif // defined(__GCC_DESTRUCTIVE_SIZE) && defined(__GCC_CONSTRUCTIVE_SIZE)
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstdlib>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_NEW
diff --git a/libcxx/include/__cxx03/numbers b/libcxx/include/__cxx03/numbers
new file mode 100644
index 000000000000..f48ba4baf38f
--- /dev/null
+++ b/libcxx/include/__cxx03/numbers
@@ -0,0 +1,164 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_NUMBERS
+#define _LIBCPP_NUMBERS
+
+/*
+    numbers synopsis
+
+namespace std::numbers {
+  template<class T> inline constexpr T e_v          = unspecified;
+  template<class T> inline constexpr T log2e_v      = unspecified;
+  template<class T> inline constexpr T log10e_v     = unspecified;
+  template<class T> inline constexpr T pi_v         = unspecified;
+  template<class T> inline constexpr T inv_pi_v     = unspecified;
+  template<class T> inline constexpr T inv_sqrtpi_v = unspecified;
+  template<class T> inline constexpr T ln2_v        = unspecified;
+  template<class T> inline constexpr T ln10_v       = unspecified;
+  template<class T> inline constexpr T sqrt2_v      = unspecified;
+  template<class T> inline constexpr T sqrt3_v      = unspecified;
+  template<class T> inline constexpr T inv_sqrt3_v  = unspecified;
+  template<class T> inline constexpr T egamma_v     = unspecified;
+  template<class T> inline constexpr T phi_v        = unspecified;
+
+  template<floating_point T> inline constexpr T e_v<T>          = see below;
+  template<floating_point T> inline constexpr T log2e_v<T>      = see below;
+  template<floating_point T> inline constexpr T log10e_v<T>     = see below;
+  template<floating_point T> inline constexpr T pi_v<T>         = see below;
+  template<floating_point T> inline constexpr T inv_pi_v<T>     = see below;
+  template<floating_point T> inline constexpr T inv_sqrtpi_v<T> = see below;
+  template<floating_point T> inline constexpr T ln2_v<T>        = see below;
+  template<floating_point T> inline constexpr T ln10_v<T>       = see below;
+  template<floating_point T> inline constexpr T sqrt2_v<T>      = see below;
+  template<floating_point T> inline constexpr T sqrt3_v<T>      = see below;
+  template<floating_point T> inline constexpr T inv_sqrt3_v<T>  = see below;
+  template<floating_point T> inline constexpr T egamma_v<T>     = see below;
+  template<floating_point T> inline constexpr T phi_v<T>        = see below;
+
+  inline constexpr double e          = e_v<double>;
+  inline constexpr double log2e      = log2e_v<double>;
+  inline constexpr double log10e     = log10e_v<double>;
+  inline constexpr double pi         = pi_v<double>;
+  inline constexpr double inv_pi     = inv_pi_v<double>;
+  inline constexpr double inv_sqrtpi = inv_sqrtpi_v<double>;
+  inline constexpr double ln2        = ln2_v<double>;
+  inline constexpr double ln10       = ln10_v<double>;
+  inline constexpr double sqrt2      = sqrt2_v<double>;
+  inline constexpr double sqrt3      = sqrt3_v<double>;
+  inline constexpr double inv_sqrt3  = inv_sqrt3_v<double>;
+  inline constexpr double egamma     = egamma_v<double>;
+  inline constexpr double phi        = phi_v<double>;
+}
+*/
+
+#include <__concepts/arithmetic.h>
+#include <__config>
+#include <version>
+
+#if _LIBCPP_STD_VER >= 20
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace numbers {
+
+template <class _Tp>
+inline constexpr bool __false = false;
+
+template <class _Tp>
+struct __illformed {
+  static_assert(
+      __false<_Tp>,
+      "A program that instantiates a primary template of a mathematical constant variable template is ill-formed.");
+};
+
+template <class _Tp>
+inline constexpr _Tp e_v = __illformed<_Tp>{};
+template <class _Tp>
+inline constexpr _Tp log2e_v = __illformed<_Tp>{};
+template <class _Tp>
+inline constexpr _Tp log10e_v = __illformed<_Tp>{};
+template <class _Tp>
+inline constexpr _Tp pi_v = __illformed<_Tp>{};
+template <class _Tp>
+inline constexpr _Tp inv_pi_v = __illformed<_Tp>{};
+template <class _Tp>
+inline constexpr _Tp inv_sqrtpi_v = __illformed<_Tp>{};
+template <class _Tp>
+inline constexpr _Tp ln2_v = __illformed<_Tp>{};
+template <class _Tp>
+inline constexpr _Tp ln10_v = __illformed<_Tp>{};
+template <class _Tp>
+inline constexpr _Tp sqrt2_v = __illformed<_Tp>{};
+template <class _Tp>
+inline constexpr _Tp sqrt3_v = __illformed<_Tp>{};
+template <class _Tp>
+inline constexpr _Tp inv_sqrt3_v = __illformed<_Tp>{};
+template <class _Tp>
+inline constexpr _Tp egamma_v = __illformed<_Tp>{};
+template <class _Tp>
+inline constexpr _Tp phi_v = __illformed<_Tp>{};
+
+template <floating_point _Tp>
+inline constexpr _Tp e_v<_Tp> = 2.718281828459045235360287471352662;
+template <floating_point _Tp>
+inline constexpr _Tp log2e_v<_Tp> = 1.442695040888963407359924681001892;
+template <floating_point _Tp>
+inline constexpr _Tp log10e_v<_Tp> = 0.434294481903251827651128918916605;
+template <floating_point _Tp>
+inline constexpr _Tp pi_v<_Tp> = 3.141592653589793238462643383279502;
+template <floating_point _Tp>
+inline constexpr _Tp inv_pi_v<_Tp> = 0.318309886183790671537767526745028;
+template <floating_point _Tp>
+inline constexpr _Tp inv_sqrtpi_v<_Tp> = 0.564189583547756286948079451560772;
+template <floating_point _Tp>
+inline constexpr _Tp ln2_v<_Tp> = 0.693147180559945309417232121458176;
+template <floating_point _Tp>
+inline constexpr _Tp ln10_v<_Tp> = 2.302585092994045684017991454684364;
+template <floating_point _Tp>
+inline constexpr _Tp sqrt2_v<_Tp> = 1.414213562373095048801688724209698;
+template <floating_point _Tp>
+inline constexpr _Tp sqrt3_v<_Tp> = 1.732050807568877293527446341505872;
+template <floating_point _Tp>
+inline constexpr _Tp inv_sqrt3_v<_Tp> = 0.577350269189625764509148780501957;
+template <floating_point _Tp>
+inline constexpr _Tp egamma_v<_Tp> = 0.577215664901532860606512090082402;
+template <floating_point _Tp>
+inline constexpr _Tp phi_v<_Tp> = 1.618033988749894848204586834365638;
+
+inline constexpr double e          = e_v<double>;
+inline constexpr double log2e      = log2e_v<double>;
+inline constexpr double log10e     = log10e_v<double>;
+inline constexpr double pi         = pi_v<double>;
+inline constexpr double inv_pi     = inv_pi_v<double>;
+inline constexpr double inv_sqrtpi = inv_sqrtpi_v<double>;
+inline constexpr double ln2        = ln2_v<double>;
+inline constexpr double ln10       = ln10_v<double>;
+inline constexpr double sqrt2      = sqrt2_v<double>;
+inline constexpr double sqrt3      = sqrt3_v<double>;
+inline constexpr double inv_sqrt3  = inv_sqrt3_v<double>;
+inline constexpr double egamma     = egamma_v<double>;
+inline constexpr double phi        = phi_v<double>;
+
+} // namespace numbers
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <concepts>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_NUMBERS
diff --git a/libcxx/include/__cxx03/numeric b/libcxx/include/__cxx03/numeric
new file mode 100644
index 000000000000..6b92ce3a0712
--- /dev/null
+++ b/libcxx/include/__cxx03/numeric
@@ -0,0 +1,207 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_NUMERIC
+#define _LIBCPP_NUMERIC
+
+/*
+    numeric synopsis
+
+namespace std
+{
+
+template <class InputIterator, class T>
+    constexpr T  // constexpr since C++20
+    accumulate(InputIterator first, InputIterator last, T init);
+
+template <class InputIterator, class T, class BinaryOperation>
+    constexpr T  // constexpr since C++20
+    accumulate(InputIterator first, InputIterator last, T init, BinaryOperation binary_op);
+
+template<class InputIterator>
+    constexpr typename iterator_traits<InputIterator>::value_type  // constexpr since C++20
+    reduce(InputIterator first, InputIterator last);  // C++17
+
+template<class InputIterator, class T>
+    constexpr T  // constexpr since C++20
+    reduce(InputIterator first, InputIterator last, T init);  // C++17
+
+template<class InputIterator, class T, class BinaryOperation>
+    constexpr T  // constexpr since C++20
+    reduce(InputIterator first, InputIterator last, T init, BinaryOperation binary_op);  // C++17
+
+template <class InputIterator1, class InputIterator2, class T>
+    constexpr T  // constexpr since C++20
+    inner_product(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, T init);
+
+template <class InputIterator1, class InputIterator2, class T, class BinaryOperation1, class BinaryOperation2>
+    constexpr T  // constexpr since C++20
+    inner_product(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2,
+                  T init, BinaryOperation1 binary_op1, BinaryOperation2 binary_op2);
+
+
+template<class InputIterator1, class InputIterator2, class T>
+    constexpr T  // constexpr since C++20
+    transform_reduce(InputIterator1 first1, InputIterator1 last1,
+                     InputIterator2 first2, T init);  // C++17
+
+template<class InputIterator1, class InputIterator2, class T, class BinaryOperation1, class BinaryOperation2>
+    constexpr T  // constexpr since C++20
+    transform_reduce(InputIterator1 first1, InputIterator1 last1,
+                     InputIterator2 first2, T init,
+                     BinaryOperation1 binary_op1, BinaryOperation2 binary_op2);  // C++17
+
+template<class InputIterator, class T, class BinaryOperation, class UnaryOperation>
+    constexpr T  // constexpr since C++20
+    transform_reduce(InputIterator first, InputIterator last, T init,
+                     BinaryOperation binary_op, UnaryOperation unary_op);  // C++17
+
+template <class InputIterator, class OutputIterator>
+    constexpr OutputIterator  // constexpr since C++20
+    partial_sum(InputIterator first, InputIterator last, OutputIterator result);
+
+template <class InputIterator, class OutputIterator, class BinaryOperation>
+    constexpr OutputIterator  // constexpr since C++20
+    partial_sum(InputIterator first, InputIterator last, OutputIterator result, BinaryOperation binary_op);
+
+template<class InputIterator, class OutputIterator, class T>
+    constexpr OutputIterator  // constexpr since C++20
+    exclusive_scan(InputIterator first, InputIterator last,
+                   OutputIterator result, T init); // C++17
+
+template<class InputIterator, class OutputIterator, class T, class BinaryOperation>
+    constexpr OutputIterator  // constexpr since C++20
+    exclusive_scan(InputIterator first, InputIterator last,
+                   OutputIterator result, T init, BinaryOperation binary_op); // C++17
+
+template<class InputIterator, class OutputIterator>
+    constexpr OutputIterator  // constexpr since C++20
+    inclusive_scan(InputIterator first, InputIterator last, OutputIterator result);  // C++17
+
+template<class InputIterator, class OutputIterator, class BinaryOperation>
+    constexpr OutputIterator  // constexpr since C++20
+    inclusive_scan(InputIterator first, InputIterator last,
+                   OutputIterator result, BinaryOperation binary_op);  // C++17
+
+template<class InputIterator, class OutputIterator, class BinaryOperation, class T>
+    constexpr OutputIterator  // constexpr since C++20
+    inclusive_scan(InputIterator first, InputIterator last,
+                   OutputIterator result, BinaryOperation binary_op, T init);  // C++17
+
+template<class InputIterator, class OutputIterator, class T,
+         class BinaryOperation, class UnaryOperation>
+    constexpr OutputIterator  // constexpr since C++20
+    transform_exclusive_scan(InputIterator first, InputIterator last,
+                             OutputIterator result, T init,
+                             BinaryOperation binary_op, UnaryOperation unary_op);  // C++17
+
+template<class InputIterator, class OutputIterator,
+         class BinaryOperation, class UnaryOperation>
+    constexpr OutputIterator  // constexpr since C++20
+    transform_inclusive_scan(InputIterator first, InputIterator last,
+                             OutputIterator result,
+                             BinaryOperation binary_op, UnaryOperation unary_op);  // C++17
+
+template<class InputIterator, class OutputIterator,
+         class BinaryOperation, class UnaryOperation, class T>
+    constexpr OutputIterator  // constexpr since C++20
+    transform_inclusive_scan(InputIterator first, InputIterator last,
+                             OutputIterator result,
+                             BinaryOperation binary_op, UnaryOperation unary_op,
+                             T init);  // C++17
+
+template <class InputIterator, class OutputIterator>
+    constexpr OutputIterator  // constexpr since C++20
+    adjacent_difference(InputIterator first, InputIterator last, OutputIterator result);
+
+template <class InputIterator, class OutputIterator, class BinaryOperation>
+    constexpr OutputIterator  // constexpr since C++20
+    adjacent_difference(InputIterator first, InputIterator last, OutputIterator result, BinaryOperation binary_op);
+
+template <class ForwardIterator, class T>
+    constexpr void  // constexpr since C++20
+    iota(ForwardIterator first, ForwardIterator last, T value);
+
+template <class M, class N>
+    constexpr common_type_t<M,N> gcd(M m, N n);    // C++17
+
+template <class M, class N>
+    constexpr common_type_t<M,N> lcm(M m, N n);    // C++17
+
+template<class T>
+    constexpr T midpoint(T a, T b) noexcept;  // C++20
+
+template<class T>
+    constexpr T* midpoint(T* a, T* b);        // C++20
+
+// [numeric.sat], saturation arithmetic
+template<class T>
+constexpr T add_sat(T x, T y) noexcept;                     // freestanding, Since C++26
+template<class T>
+constexpr T sub_sat(T x, T y) noexcept;                     // freestanding, Since C++26
+template<class T>
+constexpr T mul_sat(T x, T y) noexcept;                     // freestanding, Since C++26
+template<class T>
+constexpr T div_sat(T x, T y) noexcept;                     // freestanding, Since C++26
+template<class T, class U>
+constexpr T saturate_cast(U x) noexcept;                    // freestanding, Since C++26
+
+}  // std
+
+*/
+
+#include <__config>
+
+#include <__numeric/accumulate.h>
+#include <__numeric/adjacent_difference.h>
+#include <__numeric/inner_product.h>
+#include <__numeric/iota.h>
+#include <__numeric/partial_sum.h>
+
+#if _LIBCPP_STD_VER >= 17
+#  include <__numeric/exclusive_scan.h>
+#  include <__numeric/gcd_lcm.h>
+#  include <__numeric/inclusive_scan.h>
+#  include <__numeric/pstl.h>
+#  include <__numeric/reduce.h>
+#  include <__numeric/transform_exclusive_scan.h>
+#  include <__numeric/transform_inclusive_scan.h>
+#  include <__numeric/transform_reduce.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__numeric/midpoint.h>
+#  include <__numeric/saturation_arithmetic.h>
+#endif
+
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 14
+#  include <initializer_list>
+#  include <limits>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <climits>
+#  include <cmath>
+#  include <concepts>
+#  include <cstdint>
+#  include <execution>
+#  include <functional>
+#  include <iterator>
+#  include <new>
+#  include <optional>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_NUMERIC
diff --git a/libcxx/include/__cxx03/optional b/libcxx/include/__cxx03/optional
new file mode 100644
index 000000000000..41d7515a2b68
--- /dev/null
+++ b/libcxx/include/__cxx03/optional
@@ -0,0 +1,1304 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_OPTIONAL
+#define _LIBCPP_OPTIONAL
+
+/*
+    optional synopsis
+
+// C++1z
+
+namespace std {
+  // [optional.optional], class template optional
+  template <class T>
+    class optional;
+
+  template<class T>
+    concept is-derived-from-optional = requires(const T& t) {       // exposition only
+      []<class U>(const optional<U>&){ }(t);
+    };
+
+  // [optional.nullopt], no-value state indicator
+  struct nullopt_t{see below };
+  inline constexpr nullopt_t nullopt(unspecified );
+
+  // [optional.bad.access], class bad_optional_access
+  class bad_optional_access;
+
+  // [optional.relops], relational operators
+  template <class T, class U>
+    constexpr bool operator==(const optional<T>&, const optional<U>&);
+  template <class T, class U>
+    constexpr bool operator!=(const optional<T>&, const optional<U>&);
+  template <class T, class U>
+    constexpr bool operator<(const optional<T>&, const optional<U>&);
+  template <class T, class U>
+    constexpr bool operator>(const optional<T>&, const optional<U>&);
+  template <class T, class U>
+    constexpr bool operator<=(const optional<T>&, const optional<U>&);
+  template <class T, class U>
+    constexpr bool operator>=(const optional<T>&, const optional<U>&);
+  template<class T, three_way_comparable_with<T> U>
+    constexpr compare_three_way_result_t<T, U>
+      operator<=>(const optional<T>&, const optional<U>&); // since C++20
+
+  // [optional.nullops], comparison with nullopt
+  template<class T> constexpr bool operator==(const optional<T>&, nullopt_t) noexcept;
+  template<class T> constexpr bool operator==(nullopt_t, const optional<T>&) noexcept; // until C++17
+  template<class T> constexpr bool operator!=(const optional<T>&, nullopt_t) noexcept; // until C++17
+  template<class T> constexpr bool operator!=(nullopt_t, const optional<T>&) noexcept; // until C++17
+  template<class T> constexpr bool operator<(const optional<T>&, nullopt_t) noexcept;  // until C++17
+  template<class T> constexpr bool operator<(nullopt_t, const optional<T>&) noexcept;  // until C++17
+  template<class T> constexpr bool operator<=(const optional<T>&, nullopt_t) noexcept; // until C++17
+  template<class T> constexpr bool operator<=(nullopt_t, const optional<T>&) noexcept; // until C++17
+  template<class T> constexpr bool operator>(const optional<T>&, nullopt_t) noexcept;  // until C++17
+  template<class T> constexpr bool operator>(nullopt_t, const optional<T>&) noexcept;  // until C++17
+  template<class T> constexpr bool operator>=(const optional<T>&, nullopt_t) noexcept; // until C++17
+  template<class T> constexpr bool operator>=(nullopt_t, const optional<T>&) noexcept; // until C++17
+  template<class T>
+    constexpr strong_ordering operator<=>(const optional<T>&, nullopt_t) noexcept;     // since C++20
+
+  // [optional.comp.with.t], comparison with T
+  template<class T, class U> constexpr bool operator==(const optional<T>&, const U&);
+  template<class T, class U> constexpr bool operator==(const T&, const optional<U>&);
+  template<class T, class U> constexpr bool operator!=(const optional<T>&, const U&);
+  template<class T, class U> constexpr bool operator!=(const T&, const optional<U>&);
+  template<class T, class U> constexpr bool operator<(const optional<T>&, const U&);
+  template<class T, class U> constexpr bool operator<(const T&, const optional<U>&);
+  template<class T, class U> constexpr bool operator<=(const optional<T>&, const U&);
+  template<class T, class U> constexpr bool operator<=(const T&, const optional<U>&);
+  template<class T, class U> constexpr bool operator>(const optional<T>&, const U&);
+  template<class T, class U> constexpr bool operator>(const T&, const optional<U>&);
+  template<class T, class U> constexpr bool operator>=(const optional<T>&, const U&);
+  template<class T, class U> constexpr bool operator>=(const T&, const optional<U>&);
+  template<class T, class U>
+      requires (!is-derived-from-optional<U>) && three_way_comparable_with<T, U>
+    constexpr compare_three_way_result_t<T, U>
+      operator<=>(const optional<T>&, const U&);                                       // since C++20
+
+  // [optional.specalg], specialized algorithms
+  template<class T>
+    void swap(optional<T>&, optional<T>&) noexcept(see below ); // constexpr in C++20
+
+  template<class T>
+    constexpr optional<see below > make_optional(T&&);
+  template<class T, class... Args>
+    constexpr optional<T> make_optional(Args&&... args);
+  template<class T, class U, class... Args>
+    constexpr optional<T> make_optional(initializer_list<U> il, Args&&... args);
+
+  // [optional.hash], hash support
+  template<class T> struct hash;
+  template<class T> struct hash<optional<T>>;
+
+  template<class T>
+  class optional {
+  public:
+    using value_type = T;
+
+    // [optional.ctor], constructors
+    constexpr optional() noexcept;
+    constexpr optional(nullopt_t) noexcept;
+    constexpr optional(const optional &);
+    constexpr optional(optional &&) noexcept(see below);
+    template<class... Args>
+      constexpr explicit optional(in_place_t, Args &&...);
+    template<class U, class... Args>
+      constexpr explicit optional(in_place_t, initializer_list<U>, Args &&...);
+    template<class U = T>
+      constexpr explicit(see-below) optional(U &&);
+    template<class U>
+      explicit(see-below) optional(const optional<U> &);                          // constexpr in C++20
+    template<class U>
+      explicit(see-below) optional(optional<U> &&);                               // constexpr in C++20
+
+    // [optional.dtor], destructor
+    ~optional(); // constexpr in C++20
+
+    // [optional.assign], assignment
+    optional &operator=(nullopt_t) noexcept;                                      // constexpr in C++20
+    constexpr optional &operator=(const optional &);
+    constexpr optional &operator=(optional &&) noexcept(see below);
+    template<class U = T> optional &operator=(U &&);                              // constexpr in C++20
+    template<class U> optional &operator=(const optional<U> &);                   // constexpr in C++20
+    template<class U> optional &operator=(optional<U> &&);                        // constexpr in C++20
+    template<class... Args> T& emplace(Args &&...);                               // constexpr in C++20
+    template<class U, class... Args> T& emplace(initializer_list<U>, Args &&...); // constexpr in C++20
+
+    // [optional.swap], swap
+    void swap(optional &) noexcept(see below ); // constexpr in C++20
+
+    // [optional.observe], observers
+    constexpr T const *operator->() const noexcept;
+    constexpr T *operator->() noexcept;
+    constexpr T const &operator*() const & noexcept;
+    constexpr T &operator*() & noexcept;
+    constexpr T &&operator*() && noexcept;
+    constexpr const T &&operator*() const && noexcept;
+    constexpr explicit operator bool() const noexcept;
+    constexpr bool has_value() const noexcept;
+    constexpr T const &value() const &;
+    constexpr T &value() &;
+    constexpr T &&value() &&;
+    constexpr const T &&value() const &&;
+    template<class U> constexpr T value_or(U &&) const &;
+    template<class U> constexpr T value_or(U &&) &&;
+
+    // [optional.monadic], monadic operations
+    template<class F> constexpr auto and_then(F&& f) &;         // since C++23
+    template<class F> constexpr auto and_then(F&& f) &&;        // since C++23
+    template<class F> constexpr auto and_then(F&& f) const&;    // since C++23
+    template<class F> constexpr auto and_then(F&& f) const&&;   // since C++23
+    template<class F> constexpr auto transform(F&& f) &;        // since C++23
+    template<class F> constexpr auto transform(F&& f) &&;       // since C++23
+    template<class F> constexpr auto transform(F&& f) const&;   // since C++23
+    template<class F> constexpr auto transform(F&& f) const&&;  // since C++23
+    template<class F> constexpr optional or_else(F&& f) &&;     // since C++23
+    template<class F> constexpr optional or_else(F&& f) const&; // since C++23
+
+    // [optional.mod], modifiers
+    void reset() noexcept;                                      // constexpr in C++20
+
+  private:
+    T *val;         // exposition only
+  };
+
+  template<class T>
+    optional(T) -> optional<T>;
+
+} // namespace std
+
+*/
+
+#include <__assert>
+#include <__compare/compare_three_way_result.h>
+#include <__compare/three_way_comparable.h>
+#include <__concepts/invocable.h>
+#include <__config>
+#include <__exception/exception.h>
+#include <__functional/hash.h>
+#include <__functional/invoke.h>
+#include <__functional/unary_function.h>
+#include <__fwd/functional.h>
+#include <__memory/addressof.h>
+#include <__memory/construct_at.h>
+#include <__tuple/sfinae_helpers.h>
+#include <__type_traits/add_pointer.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/disjunction.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_destructible.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_object.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/is_scalar.h>
+#include <__type_traits/is_swappable.h>
+#include <__type_traits/is_trivially_assignable.h>
+#include <__type_traits/is_trivially_constructible.h>
+#include <__type_traits/is_trivially_destructible.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__type_traits/negation.h>
+#include <__type_traits/remove_const.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/in_place.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+#include <__verbose_abort>
+#include <initializer_list>
+#include <new>
+#include <version>
+
+// standard-mandated includes
+
+// [optional.syn]
+#include <compare>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+namespace std // purposefully not using versioning namespace
+{
+
+class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS bad_optional_access : public exception {
+public:
+  _LIBCPP_HIDE_FROM_ABI bad_optional_access() _NOEXCEPT                                      = default;
+  _LIBCPP_HIDE_FROM_ABI bad_optional_access(const bad_optional_access&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI bad_optional_access& operator=(const bad_optional_access&) _NOEXCEPT = default;
+  // Get the key function ~bad_optional_access() into the dylib
+  ~bad_optional_access() _NOEXCEPT override;
+  const char* what() const _NOEXCEPT override;
+};
+
+} // namespace std
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS void
+__throw_bad_optional_access() {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw bad_optional_access();
+#  else
+  _LIBCPP_VERBOSE_ABORT("bad_optional_access was thrown in -fno-exceptions mode");
+#  endif
+}
+
+struct nullopt_t {
+  struct __secret_tag {
+    explicit __secret_tag() = default;
+  };
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit nullopt_t(__secret_tag, __secret_tag) noexcept {}
+};
+
+inline constexpr nullopt_t nullopt{nullopt_t::__secret_tag{}, nullopt_t::__secret_tag{}};
+
+struct __optional_construct_from_invoke_tag {};
+
+template <class _Tp, bool = is_trivially_destructible<_Tp>::value>
+struct __optional_destruct_base;
+
+template <class _Tp>
+struct __optional_destruct_base<_Tp, false> {
+  typedef _Tp value_type;
+  static_assert(is_object_v<value_type>, "instantiation of optional with a non-object type is undefined behavior");
+  union {
+    char __null_state_;
+    value_type __val_;
+  };
+  bool __engaged_;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__optional_destruct_base() {
+    if (__engaged_)
+      __val_.~value_type();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __optional_destruct_base() noexcept : __null_state_(), __engaged_(false) {}
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __optional_destruct_base(in_place_t, _Args&&... __args)
+      : __val_(std::forward<_Args>(__args)...), __engaged_(true) {}
+
+#  if _LIBCPP_STD_VER >= 23
+  template <class _Fp, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __optional_destruct_base(
+      __optional_construct_from_invoke_tag, _Fp&& __f, _Args&&... __args)
+      : __val_(std::invoke(std::forward<_Fp>(__f), std::forward<_Args>(__args)...)), __engaged_(true) {}
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void reset() noexcept {
+    if (__engaged_) {
+      __val_.~value_type();
+      __engaged_ = false;
+    }
+  }
+};
+
+template <class _Tp>
+struct __optional_destruct_base<_Tp, true> {
+  typedef _Tp value_type;
+  static_assert(is_object_v<value_type>, "instantiation of optional with a non-object type is undefined behavior");
+  union {
+    char __null_state_;
+    value_type __val_;
+  };
+  bool __engaged_;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __optional_destruct_base() noexcept : __null_state_(), __engaged_(false) {}
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __optional_destruct_base(in_place_t, _Args&&... __args)
+      : __val_(std::forward<_Args>(__args)...), __engaged_(true) {}
+
+#  if _LIBCPP_STD_VER >= 23
+  template <class _Fp, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr __optional_destruct_base(
+      __optional_construct_from_invoke_tag, _Fp&& __f, _Args&&... __args)
+      : __val_(std::invoke(std::forward<_Fp>(__f), std::forward<_Args>(__args)...)), __engaged_(true) {}
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void reset() noexcept {
+    if (__engaged_) {
+      __engaged_ = false;
+    }
+  }
+};
+
+template <class _Tp, bool = is_reference<_Tp>::value>
+struct __optional_storage_base : __optional_destruct_base<_Tp> {
+  using __base     = __optional_destruct_base<_Tp>;
+  using value_type = _Tp;
+  using __base::__base;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool has_value() const noexcept { return this->__engaged_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr value_type& __get() & noexcept { return this->__val_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr const value_type& __get() const& noexcept { return this->__val_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr value_type&& __get() && noexcept { return std::move(this->__val_); }
+  _LIBCPP_HIDE_FROM_ABI constexpr const value_type&& __get() const&& noexcept { return std::move(this->__val_); }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __construct(_Args&&... __args) {
+    _LIBCPP_ASSERT_INTERNAL(!has_value(), "__construct called for engaged __optional_storage");
+    std::__construct_at(std::addressof(this->__val_), std::forward<_Args>(__args)...);
+    this->__engaged_ = true;
+  }
+
+  template <class _That>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __construct_from(_That&& __opt) {
+    if (__opt.has_value())
+      __construct(std::forward<_That>(__opt).__get());
+  }
+
+  template <class _That>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __assign_from(_That&& __opt) {
+    if (this->__engaged_ == __opt.has_value()) {
+      if (this->__engaged_)
+        this->__val_ = std::forward<_That>(__opt).__get();
+    } else {
+      if (this->__engaged_)
+        this->reset();
+      else
+        __construct(std::forward<_That>(__opt).__get());
+    }
+  }
+};
+
+// optional<T&> is currently required to be ill-formed. However, it may
+// be allowed in the future. For this reason, it has already been implemented
+// to ensure we can make the change in an ABI-compatible manner.
+template <class _Tp>
+struct __optional_storage_base<_Tp, true> {
+  using value_type = _Tp;
+  using __raw_type = remove_reference_t<_Tp>;
+  __raw_type* __value_;
+
+  template <class _Up>
+  static _LIBCPP_HIDE_FROM_ABI constexpr bool __can_bind_reference() {
+    using _RawUp = __libcpp_remove_reference_t<_Up>;
+    using _UpPtr = _RawUp*;
+    using _RawTp = __libcpp_remove_reference_t<_Tp>;
+    using _TpPtr = _RawTp*;
+    using _CheckLValueArg =
+        integral_constant<bool,
+                          (is_lvalue_reference<_Up>::value && is_convertible<_UpPtr, _TpPtr>::value) ||
+                              is_same<_RawUp, reference_wrapper<_RawTp>>::value ||
+                              is_same<_RawUp, reference_wrapper<__remove_const_t<_RawTp>>>::value >;
+    return (is_lvalue_reference<_Tp>::value && _CheckLValueArg::value) ||
+           (is_rvalue_reference<_Tp>::value && !is_lvalue_reference<_Up>::value &&
+            is_convertible<_UpPtr, _TpPtr>::value);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __optional_storage_base() noexcept : __value_(nullptr) {}
+
+  template <class _UArg>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __optional_storage_base(in_place_t, _UArg&& __uarg)
+      : __value_(std::addressof(__uarg)) {
+    static_assert(__can_bind_reference<_UArg>(),
+                  "Attempted to construct a reference element in tuple from a "
+                  "possible temporary");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void reset() noexcept { __value_ = nullptr; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool has_value() const noexcept { return __value_ != nullptr; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr value_type& __get() const& noexcept { return *__value_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr value_type&& __get() const&& noexcept { return std::forward<value_type>(*__value_); }
+
+  template <class _UArg>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __construct(_UArg&& __val) {
+    _LIBCPP_ASSERT_INTERNAL(!has_value(), "__construct called for engaged __optional_storage");
+    static_assert(__can_bind_reference<_UArg>(),
+                  "Attempted to construct a reference element in tuple from a "
+                  "possible temporary");
+    __value_ = std::addressof(__val);
+  }
+
+  template <class _That>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __construct_from(_That&& __opt) {
+    if (__opt.has_value())
+      __construct(std::forward<_That>(__opt).__get());
+  }
+
+  template <class _That>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __assign_from(_That&& __opt) {
+    if (has_value() == __opt.has_value()) {
+      if (has_value())
+        *__value_ = std::forward<_That>(__opt).__get();
+    } else {
+      if (has_value())
+        reset();
+      else
+        __construct(std::forward<_That>(__opt).__get());
+    }
+  }
+};
+
+template <class _Tp, bool = is_trivially_copy_constructible<_Tp>::value>
+struct __optional_copy_base : __optional_storage_base<_Tp> {
+  using __optional_storage_base<_Tp>::__optional_storage_base;
+};
+
+template <class _Tp>
+struct __optional_copy_base<_Tp, false> : __optional_storage_base<_Tp> {
+  using __optional_storage_base<_Tp>::__optional_storage_base;
+
+  _LIBCPP_HIDE_FROM_ABI __optional_copy_base() = default;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __optional_copy_base(const __optional_copy_base& __opt) {
+    this->__construct_from(__opt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __optional_copy_base(__optional_copy_base&&)                 = default;
+  _LIBCPP_HIDE_FROM_ABI __optional_copy_base& operator=(const __optional_copy_base&) = default;
+  _LIBCPP_HIDE_FROM_ABI __optional_copy_base& operator=(__optional_copy_base&&)      = default;
+};
+
+template <class _Tp, bool = is_trivially_move_constructible<_Tp>::value>
+struct __optional_move_base : __optional_copy_base<_Tp> {
+  using __optional_copy_base<_Tp>::__optional_copy_base;
+};
+
+template <class _Tp>
+struct __optional_move_base<_Tp, false> : __optional_copy_base<_Tp> {
+  using value_type = _Tp;
+  using __optional_copy_base<_Tp>::__optional_copy_base;
+
+  _LIBCPP_HIDE_FROM_ABI __optional_move_base()                            = default;
+  _LIBCPP_HIDE_FROM_ABI __optional_move_base(const __optional_move_base&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  __optional_move_base(__optional_move_base&& __opt) noexcept(is_nothrow_move_constructible_v<value_type>) {
+    this->__construct_from(std::move(__opt));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __optional_move_base& operator=(const __optional_move_base&) = default;
+  _LIBCPP_HIDE_FROM_ABI __optional_move_base& operator=(__optional_move_base&&)      = default;
+};
+
+template <class _Tp,
+          bool = is_trivially_destructible<_Tp>::value && is_trivially_copy_constructible<_Tp>::value &&
+                 is_trivially_copy_assignable<_Tp>::value>
+struct __optional_copy_assign_base : __optional_move_base<_Tp> {
+  using __optional_move_base<_Tp>::__optional_move_base;
+};
+
+template <class _Tp>
+struct __optional_copy_assign_base<_Tp, false> : __optional_move_base<_Tp> {
+  using __optional_move_base<_Tp>::__optional_move_base;
+
+  _LIBCPP_HIDE_FROM_ABI __optional_copy_assign_base()                                   = default;
+  _LIBCPP_HIDE_FROM_ABI __optional_copy_assign_base(const __optional_copy_assign_base&) = default;
+  _LIBCPP_HIDE_FROM_ABI __optional_copy_assign_base(__optional_copy_assign_base&&)      = default;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __optional_copy_assign_base&
+  operator=(const __optional_copy_assign_base& __opt) {
+    this->__assign_from(__opt);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __optional_copy_assign_base& operator=(__optional_copy_assign_base&&) = default;
+};
+
+template <class _Tp,
+          bool = is_trivially_destructible<_Tp>::value && is_trivially_move_constructible<_Tp>::value &&
+                 is_trivially_move_assignable<_Tp>::value>
+struct __optional_move_assign_base : __optional_copy_assign_base<_Tp> {
+  using __optional_copy_assign_base<_Tp>::__optional_copy_assign_base;
+};
+
+template <class _Tp>
+struct __optional_move_assign_base<_Tp, false> : __optional_copy_assign_base<_Tp> {
+  using value_type = _Tp;
+  using __optional_copy_assign_base<_Tp>::__optional_copy_assign_base;
+
+  _LIBCPP_HIDE_FROM_ABI __optional_move_assign_base()                                              = default;
+  _LIBCPP_HIDE_FROM_ABI __optional_move_assign_base(const __optional_move_assign_base& __opt)      = default;
+  _LIBCPP_HIDE_FROM_ABI __optional_move_assign_base(__optional_move_assign_base&&)                 = default;
+  _LIBCPP_HIDE_FROM_ABI __optional_move_assign_base& operator=(const __optional_move_assign_base&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __optional_move_assign_base&
+  operator=(__optional_move_assign_base&& __opt) noexcept(
+      is_nothrow_move_assignable_v<value_type> && is_nothrow_move_constructible_v<value_type>) {
+    this->__assign_from(std::move(__opt));
+    return *this;
+  }
+};
+
+template <class _Tp>
+using __optional_sfinae_ctor_base_t =
+    __sfinae_ctor_base< is_copy_constructible<_Tp>::value, is_move_constructible<_Tp>::value >;
+
+template <class _Tp>
+using __optional_sfinae_assign_base_t =
+    __sfinae_assign_base< (is_copy_constructible<_Tp>::value && is_copy_assignable<_Tp>::value),
+                          (is_move_constructible<_Tp>::value && is_move_assignable<_Tp>::value) >;
+
+template <class _Tp>
+class optional;
+
+#  if _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+concept __is_derived_from_optional = requires(const _Tp& __t) { []<class _Up>(const optional<_Up>&) {}(__t); };
+
+#  endif // _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+struct __is_std_optional : false_type {};
+template <class _Tp>
+struct __is_std_optional<optional<_Tp>> : true_type {};
+
+template <class _Tp>
+class _LIBCPP_DECLSPEC_EMPTY_BASES optional
+    : private __optional_move_assign_base<_Tp>,
+      private __optional_sfinae_ctor_base_t<_Tp>,
+      private __optional_sfinae_assign_base_t<_Tp> {
+  using __base = __optional_move_assign_base<_Tp>;
+
+public:
+  using value_type = _Tp;
+
+  using __trivially_relocatable = conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value, optional, void>;
+
+private:
+  // Disable the reference extension using this static assert.
+  static_assert(!is_same_v<__remove_cvref_t<value_type>, in_place_t>,
+                "instantiation of optional with in_place_t is ill-formed");
+  static_assert(!is_same_v<__remove_cvref_t<value_type>, nullopt_t>,
+                "instantiation of optional with nullopt_t is ill-formed");
+  static_assert(!is_reference_v<value_type>, "instantiation of optional with a reference type is ill-formed");
+  static_assert(is_destructible_v<value_type>, "instantiation of optional with a non-destructible type is ill-formed");
+  static_assert(!is_array_v<value_type>, "instantiation of optional with an array type is ill-formed");
+
+  // LWG2756: conditionally explicit conversion from _Up
+  struct _CheckOptionalArgsConstructor {
+    template <class _Up>
+    _LIBCPP_HIDE_FROM_ABI static constexpr bool __enable_implicit() {
+      return is_constructible_v<_Tp, _Up&&> && is_convertible_v<_Up&&, _Tp>;
+    }
+
+    template <class _Up>
+    _LIBCPP_HIDE_FROM_ABI static constexpr bool __enable_explicit() {
+      return is_constructible_v<_Tp, _Up&&> && !is_convertible_v<_Up&&, _Tp>;
+    }
+  };
+  template <class _Up>
+  using _CheckOptionalArgsCtor =
+      _If< _IsNotSame<__remove_cvref_t<_Up>, in_place_t>::value && _IsNotSame<__remove_cvref_t<_Up>, optional>::value &&
+               (!is_same_v<remove_cv_t<_Tp>, bool> || !__is_std_optional<__remove_cvref_t<_Up>>::value),
+           _CheckOptionalArgsConstructor,
+           __check_tuple_constructor_fail >;
+  template <class _QualUp>
+  struct _CheckOptionalLikeConstructor {
+    template <class _Up, class _Opt = optional<_Up>>
+    using __check_constructible_from_opt =
+        _Or< is_constructible<_Tp, _Opt&>,
+             is_constructible<_Tp, _Opt const&>,
+             is_constructible<_Tp, _Opt&&>,
+             is_constructible<_Tp, _Opt const&&>,
+             is_convertible<_Opt&, _Tp>,
+             is_convertible<_Opt const&, _Tp>,
+             is_convertible<_Opt&&, _Tp>,
+             is_convertible<_Opt const&&, _Tp> >;
+    template <class _Up, class _Opt = optional<_Up>>
+    using __check_assignable_from_opt =
+        _Or< is_assignable<_Tp&, _Opt&>,
+             is_assignable<_Tp&, _Opt const&>,
+             is_assignable<_Tp&, _Opt&&>,
+             is_assignable<_Tp&, _Opt const&&> >;
+    template <class _Up, class _QUp = _QualUp>
+    _LIBCPP_HIDE_FROM_ABI static constexpr bool __enable_implicit() {
+      return is_convertible<_QUp, _Tp>::value &&
+             (is_same_v<remove_cv_t<_Tp>, bool> || !__check_constructible_from_opt<_Up>::value);
+    }
+    template <class _Up, class _QUp = _QualUp>
+    _LIBCPP_HIDE_FROM_ABI static constexpr bool __enable_explicit() {
+      return !is_convertible<_QUp, _Tp>::value &&
+             (is_same_v<remove_cv_t<_Tp>, bool> || !__check_constructible_from_opt<_Up>::value);
+    }
+    template <class _Up, class _QUp = _QualUp>
+    _LIBCPP_HIDE_FROM_ABI static constexpr bool __enable_assign() {
+      // Construction and assignability of _QUp to _Tp has already been
+      // checked.
+      return !__check_constructible_from_opt<_Up>::value && !__check_assignable_from_opt<_Up>::value;
+    }
+  };
+
+  template <class _Up, class _QualUp>
+  using _CheckOptionalLikeCtor =
+      _If< _And< _IsNotSame<_Up, _Tp>, is_constructible<_Tp, _QualUp> >::value,
+           _CheckOptionalLikeConstructor<_QualUp>,
+           __check_tuple_constructor_fail >;
+  template <class _Up, class _QualUp>
+  using _CheckOptionalLikeAssign =
+      _If< _And< _IsNotSame<_Up, _Tp>, is_constructible<_Tp, _QualUp>, is_assignable<_Tp&, _QualUp> >::value,
+           _CheckOptionalLikeConstructor<_QualUp>,
+           __check_tuple_constructor_fail >;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr optional() noexcept {}
+  _LIBCPP_HIDE_FROM_ABI constexpr optional(const optional&) = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr optional(optional&&)      = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr optional(nullopt_t) noexcept {}
+
+  template <
+      class _InPlaceT,
+      class... _Args,
+      class = enable_if_t< _And< _IsSame<_InPlaceT, in_place_t>, is_constructible<value_type, _Args...> >::value > >
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit optional(_InPlaceT, _Args&&... __args)
+      : __base(in_place, std::forward<_Args>(__args)...) {}
+
+  template <class _Up,
+            class... _Args,
+            class = enable_if_t< is_constructible_v<value_type, initializer_list<_Up>&, _Args...>> >
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit optional(in_place_t, initializer_list<_Up> __il, _Args&&... __args)
+      : __base(in_place, __il, std::forward<_Args>(__args)...) {}
+
+  template <class _Up                                                                         = value_type,
+            enable_if_t< _CheckOptionalArgsCtor<_Up>::template __enable_implicit<_Up>(), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI constexpr optional(_Up&& __v) : __base(in_place, std::forward<_Up>(__v)) {}
+
+  template <class _Up, enable_if_t< _CheckOptionalArgsCtor<_Up>::template __enable_explicit<_Up>(), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit optional(_Up&& __v) : __base(in_place, std::forward<_Up>(__v)) {}
+
+  // LWG2756: conditionally explicit conversion from const optional<_Up>&
+  template <class _Up,
+            enable_if_t< _CheckOptionalLikeCtor<_Up, _Up const&>::template __enable_implicit<_Up>(), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 optional(const optional<_Up>& __v) {
+    this->__construct_from(__v);
+  }
+  template <class _Up,
+            enable_if_t< _CheckOptionalLikeCtor<_Up, _Up const&>::template __enable_explicit<_Up>(), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit optional(const optional<_Up>& __v) {
+    this->__construct_from(__v);
+  }
+
+  // LWG2756: conditionally explicit conversion from optional<_Up>&&
+  template <class _Up, enable_if_t< _CheckOptionalLikeCtor<_Up, _Up&&>::template __enable_implicit<_Up>(), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 optional(optional<_Up>&& __v) {
+    this->__construct_from(std::move(__v));
+  }
+  template <class _Up, enable_if_t< _CheckOptionalLikeCtor<_Up, _Up&&>::template __enable_explicit<_Up>(), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit optional(optional<_Up>&& __v) {
+    this->__construct_from(std::move(__v));
+  }
+
+#  if _LIBCPP_STD_VER >= 23
+  template <class _Tag,
+            class _Fp,
+            class... _Args,
+            __enable_if_t<_IsSame<_Tag, __optional_construct_from_invoke_tag>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit optional(_Tag, _Fp&& __f, _Args&&... __args)
+      : __base(__optional_construct_from_invoke_tag{}, std::forward<_Fp>(__f), std::forward<_Args>(__args)...) {}
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 optional& operator=(nullopt_t) noexcept {
+    reset();
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr optional& operator=(const optional&) = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr optional& operator=(optional&&)      = default;
+
+  // LWG2756
+  template <
+      class _Up = value_type,
+      class     = enable_if_t< _And< _IsNotSame<__remove_cvref_t<_Up>, optional>,
+                                     _Or< _IsNotSame<__remove_cvref_t<_Up>, value_type>, _Not<is_scalar<value_type>> >,
+                                     is_constructible<value_type, _Up>,
+                                     is_assignable<value_type&, _Up> >::value> >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 optional& operator=(_Up&& __v) {
+    if (this->has_value())
+      this->__get() = std::forward<_Up>(__v);
+    else
+      this->__construct(std::forward<_Up>(__v));
+    return *this;
+  }
+
+  // LWG2756
+  template <class _Up,
+            enable_if_t< _CheckOptionalLikeAssign<_Up, _Up const&>::template __enable_assign<_Up>(), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 optional& operator=(const optional<_Up>& __v) {
+    this->__assign_from(__v);
+    return *this;
+  }
+
+  // LWG2756
+  template <class _Up, enable_if_t< _CheckOptionalLikeCtor<_Up, _Up&&>::template __enable_assign<_Up>(), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 optional& operator=(optional<_Up>&& __v) {
+    this->__assign_from(std::move(__v));
+    return *this;
+  }
+
+  template <class... _Args, class = enable_if_t< is_constructible_v<value_type, _Args...> > >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& emplace(_Args&&... __args) {
+    reset();
+    this->__construct(std::forward<_Args>(__args)...);
+    return this->__get();
+  }
+
+  template <class _Up,
+            class... _Args,
+            class = enable_if_t< is_constructible_v<value_type, initializer_list<_Up>&, _Args...> > >
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& emplace(initializer_list<_Up> __il, _Args&&... __args) {
+    reset();
+    this->__construct(__il, std::forward<_Args>(__args)...);
+    return this->__get();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+  swap(optional& __opt) noexcept(is_nothrow_move_constructible_v<value_type> && is_nothrow_swappable_v<value_type>) {
+    if (this->has_value() == __opt.has_value()) {
+      using std::swap;
+      if (this->has_value())
+        swap(this->__get(), __opt.__get());
+    } else {
+      if (this->has_value()) {
+        __opt.__construct(std::move(this->__get()));
+        reset();
+      } else {
+        this->__construct(std::move(__opt.__get()));
+        __opt.reset();
+      }
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<value_type const> operator->() const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator-> called on a disengaged value");
+    return std::addressof(this->__get());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<value_type> operator->() noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator-> called on a disengaged value");
+    return std::addressof(this->__get());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const value_type& operator*() const& noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator* called on a disengaged value");
+    return this->__get();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr value_type& operator*() & noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator* called on a disengaged value");
+    return this->__get();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr value_type&& operator*() && noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator* called on a disengaged value");
+    return std::move(this->__get());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const value_type&& operator*() const&& noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator* called on a disengaged value");
+    return std::move(this->__get());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return has_value(); }
+
+  using __base::__get;
+  using __base::has_value;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr value_type const& value() const& {
+    if (!this->has_value())
+      __throw_bad_optional_access();
+    return this->__get();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr value_type& value() & {
+    if (!this->has_value())
+      __throw_bad_optional_access();
+    return this->__get();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr value_type&& value() && {
+    if (!this->has_value())
+      __throw_bad_optional_access();
+    return std::move(this->__get());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr value_type const&& value() const&& {
+    if (!this->has_value())
+      __throw_bad_optional_access();
+    return std::move(this->__get());
+  }
+
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI constexpr value_type value_or(_Up&& __v) const& {
+    static_assert(is_copy_constructible_v<value_type>, "optional<T>::value_or: T must be copy constructible");
+    static_assert(is_convertible_v<_Up, value_type>, "optional<T>::value_or: U must be convertible to T");
+    return this->has_value() ? this->__get() : static_cast<value_type>(std::forward<_Up>(__v));
+  }
+
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI constexpr value_type value_or(_Up&& __v) && {
+    static_assert(is_move_constructible_v<value_type>, "optional<T>::value_or: T must be move constructible");
+    static_assert(is_convertible_v<_Up, value_type>, "optional<T>::value_or: U must be convertible to T");
+    return this->has_value() ? std::move(this->__get()) : static_cast<value_type>(std::forward<_Up>(__v));
+  }
+
+#  if _LIBCPP_STD_VER >= 23
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr auto and_then(_Func&& __f) & {
+    using _Up = invoke_result_t<_Func, value_type&>;
+    static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
+                  "Result of f(value()) must be a specialization of std::optional");
+    if (*this)
+      return std::invoke(std::forward<_Func>(__f), value());
+    return remove_cvref_t<_Up>();
+  }
+
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr auto and_then(_Func&& __f) const& {
+    using _Up = invoke_result_t<_Func, const value_type&>;
+    static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
+                  "Result of f(value()) must be a specialization of std::optional");
+    if (*this)
+      return std::invoke(std::forward<_Func>(__f), value());
+    return remove_cvref_t<_Up>();
+  }
+
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr auto and_then(_Func&& __f) && {
+    using _Up = invoke_result_t<_Func, value_type&&>;
+    static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
+                  "Result of f(std::move(value())) must be a specialization of std::optional");
+    if (*this)
+      return std::invoke(std::forward<_Func>(__f), std::move(value()));
+    return remove_cvref_t<_Up>();
+  }
+
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const&& {
+    using _Up = invoke_result_t<_Func, const value_type&&>;
+    static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
+                  "Result of f(std::move(value())) must be a specialization of std::optional");
+    if (*this)
+      return std::invoke(std::forward<_Func>(__f), std::move(value()));
+    return remove_cvref_t<_Up>();
+  }
+
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr auto transform(_Func&& __f) & {
+    using _Up = remove_cv_t<invoke_result_t<_Func, value_type&>>;
+    static_assert(!is_array_v<_Up>, "Result of f(value()) should not be an Array");
+    static_assert(!is_same_v<_Up, in_place_t>, "Result of f(value()) should not be std::in_place_t");
+    static_assert(!is_same_v<_Up, nullopt_t>, "Result of f(value()) should not be std::nullopt_t");
+    static_assert(is_object_v<_Up>, "Result of f(value()) should be an object type");
+    if (*this)
+      return optional<_Up>(__optional_construct_from_invoke_tag{}, std::forward<_Func>(__f), value());
+    return optional<_Up>();
+  }
+
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr auto transform(_Func&& __f) const& {
+    using _Up = remove_cv_t<invoke_result_t<_Func, const value_type&>>;
+    static_assert(!is_array_v<_Up>, "Result of f(value()) should not be an Array");
+    static_assert(!is_same_v<_Up, in_place_t>, "Result of f(value()) should not be std::in_place_t");
+    static_assert(!is_same_v<_Up, nullopt_t>, "Result of f(value()) should not be std::nullopt_t");
+    static_assert(is_object_v<_Up>, "Result of f(value()) should be an object type");
+    if (*this)
+      return optional<_Up>(__optional_construct_from_invoke_tag{}, std::forward<_Func>(__f), value());
+    return optional<_Up>();
+  }
+
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr auto transform(_Func&& __f) && {
+    using _Up = remove_cv_t<invoke_result_t<_Func, value_type&&>>;
+    static_assert(!is_array_v<_Up>, "Result of f(std::move(value())) should not be an Array");
+    static_assert(!is_same_v<_Up, in_place_t>, "Result of f(std::move(value())) should not be std::in_place_t");
+    static_assert(!is_same_v<_Up, nullopt_t>, "Result of f(std::move(value())) should not be std::nullopt_t");
+    static_assert(is_object_v<_Up>, "Result of f(std::move(value())) should be an object type");
+    if (*this)
+      return optional<_Up>(__optional_construct_from_invoke_tag{}, std::forward<_Func>(__f), std::move(value()));
+    return optional<_Up>();
+  }
+
+  template <class _Func>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr auto transform(_Func&& __f) const&& {
+    using _Up = remove_cvref_t<invoke_result_t<_Func, const value_type&&>>;
+    static_assert(!is_array_v<_Up>, "Result of f(std::move(value())) should not be an Array");
+    static_assert(!is_same_v<_Up, in_place_t>, "Result of f(std::move(value())) should not be std::in_place_t");
+    static_assert(!is_same_v<_Up, nullopt_t>, "Result of f(std::move(value())) should not be std::nullopt_t");
+    static_assert(is_object_v<_Up>, "Result of f(std::move(value())) should be an object type");
+    if (*this)
+      return optional<_Up>(__optional_construct_from_invoke_tag{}, std::forward<_Func>(__f), std::move(value()));
+    return optional<_Up>();
+  }
+
+  template <invocable _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr optional or_else(_Func&& __f) const&
+    requires is_copy_constructible_v<value_type>
+  {
+    static_assert(is_same_v<remove_cvref_t<invoke_result_t<_Func>>, optional>,
+                  "Result of f() should be the same type as this optional");
+    if (*this)
+      return *this;
+    return std::forward<_Func>(__f)();
+  }
+
+  template <invocable _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr optional or_else(_Func&& __f) &&
+    requires is_move_constructible_v<value_type>
+  {
+    static_assert(is_same_v<remove_cvref_t<invoke_result_t<_Func>>, optional>,
+                  "Result of f() should be the same type as this optional");
+    if (*this)
+      return std::move(*this);
+    return std::forward<_Func>(__f)();
+  }
+#  endif // _LIBCPP_STD_VER >= 23
+
+  using __base::reset;
+};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+optional(_Tp) -> optional<_Tp>;
+#  endif
+
+// Comparisons between optionals
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() == std::declval<const _Up&>()), bool>,
+    bool >
+operator==(const optional<_Tp>& __x, const optional<_Up>& __y) {
+  if (static_cast<bool>(__x) != static_cast<bool>(__y))
+    return false;
+  if (!static_cast<bool>(__x))
+    return true;
+  return *__x == *__y;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() != std::declval<const _Up&>()), bool>,
+    bool >
+operator!=(const optional<_Tp>& __x, const optional<_Up>& __y) {
+  if (static_cast<bool>(__x) != static_cast<bool>(__y))
+    return true;
+  if (!static_cast<bool>(__x))
+    return false;
+  return *__x != *__y;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() < std::declval<const _Up&>()), bool>,
+    bool >
+operator<(const optional<_Tp>& __x, const optional<_Up>& __y) {
+  if (!static_cast<bool>(__y))
+    return false;
+  if (!static_cast<bool>(__x))
+    return true;
+  return *__x < *__y;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() > std::declval<const _Up&>()), bool>,
+    bool >
+operator>(const optional<_Tp>& __x, const optional<_Up>& __y) {
+  if (!static_cast<bool>(__x))
+    return false;
+  if (!static_cast<bool>(__y))
+    return true;
+  return *__x > *__y;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() <= std::declval<const _Up&>()), bool>,
+    bool >
+operator<=(const optional<_Tp>& __x, const optional<_Up>& __y) {
+  if (!static_cast<bool>(__x))
+    return true;
+  if (!static_cast<bool>(__y))
+    return false;
+  return *__x <= *__y;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() >= std::declval<const _Up&>()), bool>,
+    bool >
+operator>=(const optional<_Tp>& __x, const optional<_Up>& __y) {
+  if (!static_cast<bool>(__y))
+    return true;
+  if (!static_cast<bool>(__x))
+    return false;
+  return *__x >= *__y;
+}
+
+#  if _LIBCPP_STD_VER >= 20
+
+template <class _Tp, three_way_comparable_with<_Tp> _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr compare_three_way_result_t<_Tp, _Up>
+operator<=>(const optional<_Tp>& __x, const optional<_Up>& __y) {
+  if (__x && __y)
+    return *__x <=> *__y;
+  return __x.has_value() <=> __y.has_value();
+}
+
+#  endif // _LIBCPP_STD_VER >= 20
+
+// Comparisons with nullopt
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator==(const optional<_Tp>& __x, nullopt_t) noexcept {
+  return !static_cast<bool>(__x);
+}
+
+#  if _LIBCPP_STD_VER <= 17
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator==(nullopt_t, const optional<_Tp>& __x) noexcept {
+  return !static_cast<bool>(__x);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator!=(const optional<_Tp>& __x, nullopt_t) noexcept {
+  return static_cast<bool>(__x);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator!=(nullopt_t, const optional<_Tp>& __x) noexcept {
+  return static_cast<bool>(__x);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<(const optional<_Tp>&, nullopt_t) noexcept {
+  return false;
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<(nullopt_t, const optional<_Tp>& __x) noexcept {
+  return static_cast<bool>(__x);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<=(const optional<_Tp>& __x, nullopt_t) noexcept {
+  return !static_cast<bool>(__x);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<=(nullopt_t, const optional<_Tp>&) noexcept {
+  return true;
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>(const optional<_Tp>& __x, nullopt_t) noexcept {
+  return static_cast<bool>(__x);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>(nullopt_t, const optional<_Tp>&) noexcept {
+  return false;
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>=(const optional<_Tp>&, nullopt_t) noexcept {
+  return true;
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>=(nullopt_t, const optional<_Tp>& __x) noexcept {
+  return !static_cast<bool>(__x);
+}
+
+#  else // _LIBCPP_STD_VER <= 17
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr strong_ordering operator<=>(const optional<_Tp>& __x, nullopt_t) noexcept {
+  return __x.has_value() <=> false;
+}
+
+#  endif // _LIBCPP_STD_VER <= 17
+
+// Comparisons with T
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() == std::declval<const _Up&>()), bool>,
+    bool >
+operator==(const optional<_Tp>& __x, const _Up& __v) {
+  return static_cast<bool>(__x) ? *__x == __v : false;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() == std::declval<const _Up&>()), bool>,
+    bool >
+operator==(const _Tp& __v, const optional<_Up>& __x) {
+  return static_cast<bool>(__x) ? __v == *__x : false;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() != std::declval<const _Up&>()), bool>,
+    bool >
+operator!=(const optional<_Tp>& __x, const _Up& __v) {
+  return static_cast<bool>(__x) ? *__x != __v : true;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() != std::declval<const _Up&>()), bool>,
+    bool >
+operator!=(const _Tp& __v, const optional<_Up>& __x) {
+  return static_cast<bool>(__x) ? __v != *__x : true;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() < std::declval<const _Up&>()), bool>,
+    bool >
+operator<(const optional<_Tp>& __x, const _Up& __v) {
+  return static_cast<bool>(__x) ? *__x < __v : true;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() < std::declval<const _Up&>()), bool>,
+    bool >
+operator<(const _Tp& __v, const optional<_Up>& __x) {
+  return static_cast<bool>(__x) ? __v < *__x : false;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() <= std::declval<const _Up&>()), bool>,
+    bool >
+operator<=(const optional<_Tp>& __x, const _Up& __v) {
+  return static_cast<bool>(__x) ? *__x <= __v : true;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() <= std::declval<const _Up&>()), bool>,
+    bool >
+operator<=(const _Tp& __v, const optional<_Up>& __x) {
+  return static_cast<bool>(__x) ? __v <= *__x : false;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() > std::declval<const _Up&>()), bool>,
+    bool >
+operator>(const optional<_Tp>& __x, const _Up& __v) {
+  return static_cast<bool>(__x) ? *__x > __v : false;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() > std::declval<const _Up&>()), bool>,
+    bool >
+operator>(const _Tp& __v, const optional<_Up>& __x) {
+  return static_cast<bool>(__x) ? __v > *__x : true;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() >= std::declval<const _Up&>()), bool>,
+    bool >
+operator>=(const optional<_Tp>& __x, const _Up& __v) {
+  return static_cast<bool>(__x) ? *__x >= __v : false;
+}
+
+template <class _Tp, class _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<
+    is_convertible_v<decltype(std::declval<const _Tp&>() >= std::declval<const _Up&>()), bool>,
+    bool >
+operator>=(const _Tp& __v, const optional<_Up>& __x) {
+  return static_cast<bool>(__x) ? __v >= *__x : true;
+}
+
+#  if _LIBCPP_STD_VER >= 20
+
+template <class _Tp, class _Up>
+  requires(!__is_derived_from_optional<_Up>) && three_way_comparable_with<_Tp, _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr compare_three_way_result_t<_Tp, _Up>
+operator<=>(const optional<_Tp>& __x, const _Up& __v) {
+  return __x.has_value() ? *__x <=> __v : strong_ordering::less;
+}
+
+#  endif // _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI
+_LIBCPP_CONSTEXPR_SINCE_CXX20 enable_if_t< is_move_constructible_v<_Tp> && is_swappable_v<_Tp>, void >
+swap(optional<_Tp>& __x, optional<_Tp>& __y) noexcept(noexcept(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr optional<decay_t<_Tp>> make_optional(_Tp&& __v) {
+  return optional<decay_t<_Tp>>(std::forward<_Tp>(__v));
+}
+
+template <class _Tp, class... _Args>
+_LIBCPP_HIDE_FROM_ABI constexpr optional<_Tp> make_optional(_Args&&... __args) {
+  return optional<_Tp>(in_place, std::forward<_Args>(__args)...);
+}
+
+template <class _Tp, class _Up, class... _Args>
+_LIBCPP_HIDE_FROM_ABI constexpr optional<_Tp> make_optional(initializer_list<_Up> __il, _Args&&... __args) {
+  return optional<_Tp>(in_place, __il, std::forward<_Args>(__args)...);
+}
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS hash< __enable_hash_helper<optional<_Tp>, remove_const_t<_Tp>> > {
+#  if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef optional<_Tp> argument_type;
+  _LIBCPP_DEPRECATED_IN_CXX17 typedef size_t result_type;
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const optional<_Tp>& __opt) const {
+    return static_cast<bool>(__opt) ? hash<remove_const_t<_Tp>>()(*__opt) : 0;
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <climits>
+#  include <concepts>
+#  include <ctime>
+#  include <iterator>
+#  include <limits>
+#  include <memory>
+#  include <ratio>
+#  include <stdexcept>
+#  include <tuple>
+#  include <type_traits>
+#  include <typeinfo>
+#  include <utility>
+#  include <variant>
+#endif
+
+#endif // _LIBCPP_OPTIONAL
diff --git a/libcxx/include/__cxx03/ostream b/libcxx/include/__cxx03/ostream
new file mode 100644
index 000000000000..359d3c0e19c4
--- /dev/null
+++ b/libcxx/include/__cxx03/ostream
@@ -0,0 +1,202 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_OSTREAM
+#define _LIBCPP_OSTREAM
+
+/*
+    ostream synopsis
+
+template <class charT, class traits = char_traits<charT> >
+class basic_ostream
+    : virtual public basic_ios<charT,traits>
+{
+public:
+    // types (inherited from basic_ios (27.5.4)):
+    typedef charT                          char_type;
+    typedef traits                         traits_type;
+    typedef typename traits_type::int_type int_type;
+    typedef typename traits_type::pos_type pos_type;
+    typedef typename traits_type::off_type off_type;
+
+    // 27.7.2.2 Constructor/destructor:
+    explicit basic_ostream(basic_streambuf<char_type,traits>* sb);
+    basic_ostream(basic_ostream&& rhs);
+    virtual ~basic_ostream();
+
+    // 27.7.2.3 Assign/swap
+    basic_ostream& operator=(const basic_ostream& rhs) = delete; // C++14
+    basic_ostream& operator=(basic_ostream&& rhs);
+    void swap(basic_ostream& rhs);
+
+    // 27.7.2.4 Prefix/suffix:
+    class sentry;
+
+    // 27.7.2.6 Formatted output:
+    basic_ostream& operator<<(basic_ostream& (*pf)(basic_ostream&));
+    basic_ostream& operator<<(basic_ios<charT, traits>& (*pf)(basic_ios<charT,traits>&));
+    basic_ostream& operator<<(ios_base& (*pf)(ios_base&));
+    basic_ostream& operator<<(bool n);
+    basic_ostream& operator<<(short n);
+    basic_ostream& operator<<(unsigned short n);
+    basic_ostream& operator<<(int n);
+    basic_ostream& operator<<(unsigned int n);
+    basic_ostream& operator<<(long n);
+    basic_ostream& operator<<(unsigned long n);
+    basic_ostream& operator<<(long long n);
+    basic_ostream& operator<<(unsigned long long n);
+    basic_ostream& operator<<(float f);
+    basic_ostream& operator<<(double f);
+    basic_ostream& operator<<(long double f);
+    basic_ostream& operator<<(const void* p);
+    basic_ostream& operator<<(const volatile void* val); // C++23
+    basic_ostream& operator<<(basic_streambuf<char_type,traits>* sb);
+    basic_ostream& operator<<(nullptr_t);
+
+    // 27.7.2.7 Unformatted output:
+    basic_ostream& put(char_type c);
+    basic_ostream& write(const char_type* s, streamsize n);
+    basic_ostream& flush();
+
+    // 27.7.2.5 seeks:
+    pos_type tellp();
+    basic_ostream& seekp(pos_type);
+    basic_ostream& seekp(off_type, ios_base::seekdir);
+protected:
+    basic_ostream(const basic_ostream& rhs) = delete;
+    basic_ostream(basic_ostream&& rhs);
+    // 27.7.3.3 Assign/swap
+    basic_ostream& operator=(basic_ostream& rhs) = delete;
+    basic_ostream& operator=(const basic_ostream&& rhs);
+    void swap(basic_ostream& rhs);
+};
+
+// 27.7.2.6.4 character inserters
+
+template<class charT, class traits>
+  basic_ostream<charT,traits>& operator<<(basic_ostream<charT,traits>&, charT);
+
+template<class charT, class traits>
+  basic_ostream<charT,traits>& operator<<(basic_ostream<charT,traits>&, char);
+
+template<class traits>
+  basic_ostream<char,traits>& operator<<(basic_ostream<char,traits>&, char);
+
+// signed and unsigned
+
+template<class traits>
+  basic_ostream<char,traits>& operator<<(basic_ostream<char,traits>&, signed char);
+
+template<class traits>
+  basic_ostream<char,traits>& operator<<(basic_ostream<char,traits>&, unsigned char);
+
+// NTBS
+template<class charT, class traits>
+  basic_ostream<charT,traits>& operator<<(basic_ostream<charT,traits>&, const charT*);
+
+template<class charT, class traits>
+  basic_ostream<charT,traits>& operator<<(basic_ostream<charT,traits>&, const char*);
+
+template<class traits>
+  basic_ostream<char,traits>& operator<<(basic_ostream<char,traits>&, const char*);
+
+// signed and unsigned
+template<class traits>
+basic_ostream<char,traits>& operator<<(basic_ostream<char,traits>&, const signed char*);
+
+template<class traits>
+  basic_ostream<char,traits>& operator<<(basic_ostream<char,traits>&, const unsigned char*);
+
+// swap:
+template <class charT, class traits>
+  void swap(basic_ostream<charT, traits>& x, basic_ostream<charT, traits>& y);
+
+template <class charT, class traits>
+  basic_ostream<charT,traits>& endl(basic_ostream<charT,traits>& os);
+
+template <class charT, class traits>
+  basic_ostream<charT,traits>& ends(basic_ostream<charT,traits>& os);
+
+template <class charT, class traits>
+  basic_ostream<charT,traits>& flush(basic_ostream<charT,traits>& os);
+
+// rvalue stream insertion
+template <class Stream, class T>
+  Stream&& operator<<(Stream&& os, const T& x);
+
+template<class traits>
+basic_ostream<char, traits>& operator<<(basic_ostream<char, traits>&, wchar_t) = delete;               // since C++20
+template<class traits>
+basic_ostream<char, traits>& operator<<(basic_ostream<char, traits>&, char8_t) = delete;               // since C++20
+template<class traits>
+basic_ostream<char, traits>& operator<<(basic_ostream<char, traits>&, char16_t) = delete;              // since C++20
+template<class traits>
+basic_ostream<char, traits>& operator<<(basic_ostream<char, traits>&, char32_t) = delete;              // since C++20
+template<class traits>
+basic_ostream<wchar_t, traits>& operator<<(basic_ostream<wchar_t, traits>&, char8_t) = delete;         // since C++20
+template<class traits>
+basic_ostream<wchar_t, traits>& operator<<(basic_ostream<wchar_t, traits>&, char16_t) = delete;        // since C++20
+template<class traits>
+basic_ostream<wchar_t, traits>& operator<<(basic_ostream<wchar_t, traits>&, char32_t) = delete;        // since C++20
+template<class traits>
+basic_ostream<char, traits>& operator<<(basic_ostream<char, traits>&, const wchar_t*) = delete;        // since C++20
+template<class traits>
+basic_ostream<char, traits>& operator<<(basic_ostream<char, traits>&, const char8_t*) = delete;        // since C++20
+template<class traits>
+basic_ostream<char, traits>& operator<<(basic_ostream<char, traits>&, const char16_t*) = delete;       // since C++20
+template<class traits>
+basic_ostream<char, traits>& operator<<(basic_ostream<char, traits>&, const char32_t*) = delete;       // since C++20
+template<class traits>
+basic_ostream<wchar_t, traits>& operator<<(basic_ostream<wchar_t, traits>&, const char8_t*) = delete;  // since C++20
+template<class traits>
+basic_ostream<wchar_t, traits>& operator<<(basic_ostream<wchar_t, traits>&, const char16_t*) = delete; // since C++20
+template<class traits>
+basic_ostream<wchar_t, traits>& operator<<(basic_ostream<wchar_t, traits>&, const char32_t*) = delete; // since C++20
+
+// [ostream.formatted.print], print functions
+template<class... Args>                                                                                // since C++23
+  void print(ostream& os, format_string<Args...> fmt, Args&&... args);
+template<class... Args>                                                                                // since C++23
+  void println(ostream& os, format_string<Args...> fmt, Args&&... args);
+void println(ostream& os);                                                                             // since C++26
+
+void vprint_unicode(ostream& os, string_view fmt, format_args args);                                   // since C++23
+void vprint_nonunicode(ostream& os, string_view fmt, format_args args);                                // since C++23
+}  // std
+
+*/
+
+#include <__config>
+
+#include <__ostream/basic_ostream.h>
+
+#if _LIBCPP_STD_VER >= 23
+#  include <__ostream/print.h>
+#endif
+
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <concepts>
+#  include <cstdio>
+#  include <cstdlib>
+#  include <format>
+#  include <iosfwd>
+#  include <iterator>
+#  include <print>
+#  include <stdexcept>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_OSTREAM
diff --git a/libcxx/include/__cxx03/print b/libcxx/include/__cxx03/print
new file mode 100644
index 000000000000..1a579daff270
--- /dev/null
+++ b/libcxx/include/__cxx03/print
@@ -0,0 +1,402 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_PRINT
+#define _LIBCPP_PRINT
+
+/*
+namespace std {
+  // [print.fun], print functions
+  template<class... Args>
+    void print(format_string<Args...> fmt, Args&&... args);
+  void println();                                                          // Since C++26
+  template<class... Args>
+    void print(FILE* stream, format_string<Args...> fmt, Args&&... args);
+  void println(FILE* stream);                                              // Since C++26
+
+  template<class... Args>
+    void println(format_string<Args...> fmt, Args&&... args);
+  template<class... Args>
+    void println(FILE* stream, format_string<Args...> fmt, Args&&... args);
+
+  void vprint_unicode(string_view fmt, format_args args);
+  void vprint_unicode(FILE* stream, string_view fmt, format_args args);
+
+  void vprint_nonunicode(string_view fmt, format_args args);
+  void vprint_nonunicode(FILE* stream, string_view fmt, format_args args);
+}
+*/
+
+#include <__assert>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__system_error/system_error.h>
+#include <__utility/forward.h>
+#include <cerrno>
+#include <cstdio>
+#include <format>
+#include <string>
+#include <string_view>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifdef _LIBCPP_WIN32API
+_LIBCPP_EXPORTED_FROM_ABI bool __is_windows_terminal(FILE* __stream);
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+// A wrapper for WriteConsoleW which is used to write to the Windows
+// console. This function is in the dylib to avoid pulling in windows.h
+// in the library headers. The function itself uses some private parts
+// of the dylib too.
+//
+// The function does not depend on the language standard used. Guarding
+// it with C++23 would fail since the dylib is currently built using C++20.
+//
+// Note the function is only implemented on the Windows platform.
+_LIBCPP_EXPORTED_FROM_ABI void __write_to_windows_console(FILE* __stream, wstring_view __view);
+#  endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+#elif __has_include(<unistd.h>)
+_LIBCPP_EXPORTED_FROM_ABI bool __is_posix_terminal(FILE* __stream);
+#endif // _LIBCPP_WIN32API
+
+#if _LIBCPP_STD_VER >= 23
+
+#  ifndef _LIBCPP_HAS_NO_UNICODE
+// This is the code to transcode UTF-8 to UTF-16. This is used on
+// Windows for the native Unicode API. The code is modeled to make it
+// easier to extend to
+//
+//  P2728R0 Unicode in the Library, Part 1: UTF Transcoding
+//
+// This paper is still under heavy development so it makes no sense yet
+// to strictly follow the paper.
+namespace __unicode {
+
+// The names of these concepts are modelled after P2728R0, but the
+// implementation is not. char16_t may contain 32-bits so depending on the
+// number of bits is an issue.
+#    ifdef _LIBCPP_SHORT_WCHAR
+template <class _Tp>
+concept __utf16_code_unit =
+    same_as<_Tp, char16_t>
+#      ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+    || same_as<_Tp, wchar_t>
+#      endif
+    ;
+template <class _Tp>
+concept __utf32_code_unit = same_as<_Tp, char32_t>;
+#    else // _LIBCPP_SHORT_WCHAR
+template <class _Tp>
+concept __utf16_code_unit = same_as<_Tp, char16_t>;
+template <class _Tp>
+concept __utf32_code_unit =
+    same_as<_Tp, char32_t>
+#      ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+    || same_as<_Tp, wchar_t>
+#      endif
+    ;
+#    endif // _LIBCPP_SHORT_WCHAR
+
+// Pass by reference since an output_iterator may not be copyable.
+template <class _OutIt>
+_LIBCPP_HIDE_FROM_ABI constexpr void __encode(_OutIt&, char32_t) = delete;
+
+template <class _OutIt>
+  requires __utf16_code_unit<iter_value_t<_OutIt>>
+_LIBCPP_HIDE_FROM_ABI constexpr void __encode(_OutIt& __out_it, char32_t __value) {
+  // [print.fun]/7 : "if `out` contains invalid code units, the behavior is undefined and implementations are encouraged
+  // to diagnose it".
+  _LIBCPP_ASSERT_UNCATEGORIZED(__is_scalar_value(__value), "an invalid unicode scalar value results in invalid UTF-16");
+
+  if (__value < 0x10000) {
+    *__out_it++ = __value;
+    return;
+  }
+
+  __value -= 0x10000;
+  *__out_it++ = 0xd800 + (__value >> 10);
+  *__out_it++ = 0xdc00 + (__value & 0x3FF);
+}
+
+template <class _OutIt>
+  requires __utf32_code_unit<iter_value_t<_OutIt>>
+_LIBCPP_HIDE_FROM_ABI constexpr void __encode(_OutIt& __out_it, char32_t __value) {
+  // [print.fun]/7 : "if `out` contains invalid code units, the behavior is undefined and implementations are encouraged
+  // to diagnose it".
+  _LIBCPP_ASSERT_UNCATEGORIZED(__is_scalar_value(__value), "an invalid unicode scalar value results in invalid UTF-32");
+  *__out_it++ = __value;
+}
+
+template <class _OutIt, input_iterator _InIt>
+  requires output_iterator<_OutIt, const iter_value_t<_OutIt>&> && (!same_as<iter_value_t<_OutIt>, iter_value_t<_InIt>>)
+_LIBCPP_HIDE_FROM_ABI constexpr _OutIt __transcode(_InIt __first, _InIt __last, _OutIt __out_it) {
+  // The __code_point_view has a basic_string_view interface.
+  // When transcoding becomes part of the standard we probably want to
+  // look at smarter algorithms.
+  // For example, when processing a code point that is encoded in
+  // 1 to 3 code units in UTF-8, the result will always be encoded
+  // in 1 code unit in UTF-16 (code points that require 4 code
+  // units in UTF-8 will require 2 code units in UTF-16).
+  //
+  // Note if P2728 is accepted types like int may become valid. In that case
+  // the __code_point_view should use a span. Libc++ will remove support for
+  // char_traits<int>.
+
+  // TODO PRINT Validate with clang-tidy
+  // NOLINTNEXTLINE(bugprone-dangling-handle)
+  basic_string_view<iter_value_t<_InIt>> __data{__first, __last};
+  __code_point_view<iter_value_t<_InIt>> __view{__data.begin(), __data.end()};
+  while (!__view.__at_end())
+    __unicode::__encode(__out_it, __view.__consume().__code_point);
+  return __out_it;
+}
+
+} // namespace __unicode
+
+#  endif //  _LIBCPP_HAS_NO_UNICODE
+
+namespace __print {
+
+// [print.fun]/2
+//   Effects: If the ordinary literal encoding ([lex.charset]) is UTF-8, equivalent to:
+//     vprint_unicode(stream, fmt.str, make_format_args(args...));
+//   Otherwise, equivalent to:
+//     vprint_nonunicode(stream, fmt.str, make_format_args(args...));
+//
+// Based on the compiler and its compilation flags this value is or is
+// not true. As mentioned in P2093R14 this only affects Windows. The
+// test below could also be done for
+// - GCC using __GNUC_EXECUTION_CHARSET_NAME
+//   https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html
+// - Clang using __clang_literal_encoding__
+//   https://clang.llvm.org/docs/LanguageExtensions.html#builtin-macros
+//   (note at the time of writing Clang is hard-coded to UTF-8.)
+//
+
+#  ifdef _LIBCPP_HAS_NO_UNICODE
+inline constexpr bool __use_unicode_execution_charset = false;
+#  elif defined(_MSVC_EXECUTION_CHARACTER_SET)
+// This is the same test MSVC STL uses in their implementation of <print>
+// See: https://learn.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
+inline constexpr bool __use_unicode_execution_charset = _MSVC_EXECUTION_CHARACTER_SET == 65001;
+#  else
+inline constexpr bool __use_unicode_execution_charset = true;
+#  endif
+
+_LIBCPP_HIDE_FROM_ABI inline bool __is_terminal([[maybe_unused]] FILE* __stream) {
+  // The macro _LIBCPP_TESTING_PRINT_IS_TERMINAL is used to change
+  // the behavior in the test. This is not part of the public API.
+#  ifdef _LIBCPP_TESTING_PRINT_IS_TERMINAL
+  return _LIBCPP_TESTING_PRINT_IS_TERMINAL(__stream);
+#  elif _LIBCPP_AVAILABILITY_HAS_PRINT == 0
+  return false;
+#  elif defined(_LIBCPP_WIN32API)
+  return std::__is_windows_terminal(__stream);
+#  elif __has_include(<unistd.h>)
+  return std::__is_posix_terminal(__stream);
+#  else
+#    error "Provide a way to determine whether a FILE* is a terminal"
+#  endif
+}
+
+template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
+_LIBCPP_HIDE_FROM_ABI inline void
+__vprint_nonunicode(FILE* __stream, string_view __fmt, format_args __args, bool __write_nl) {
+  _LIBCPP_ASSERT_NON_NULL(__stream, "__stream must be a valid pointer to an output C stream");
+  string __str = std::vformat(__fmt, __args);
+  if (__write_nl)
+    __str.push_back('\n');
+
+  size_t __size = fwrite(__str.data(), 1, __str.size(), __stream);
+  if (__size < __str.size()) {
+    if (std::feof(__stream))
+      std::__throw_system_error(EIO, "EOF while writing the formatted output");
+    std::__throw_system_error(std::ferror(__stream), "failed to write formatted output");
+  }
+}
+
+#  ifndef _LIBCPP_HAS_NO_UNICODE
+
+// Note these helper functions are mainly used to aid testing.
+// On POSIX systems and Windows the output is no longer considered a
+// terminal when the output is redirected. Typically during testing the
+// output is redirected to be able to capture it. This makes it hard to
+// test this code path.
+template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
+_LIBCPP_HIDE_FROM_ABI inline void
+__vprint_unicode_posix(FILE* __stream, string_view __fmt, format_args __args, bool __write_nl, bool __is_terminal) {
+  // TODO PRINT Should flush errors throw too?
+  if (__is_terminal)
+    std::fflush(__stream);
+
+  __print::__vprint_nonunicode(__stream, __fmt, __args, __write_nl);
+}
+
+#    ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
+_LIBCPP_HIDE_FROM_ABI inline void
+__vprint_unicode_windows(FILE* __stream, string_view __fmt, format_args __args, bool __write_nl, bool __is_terminal) {
+  if (!__is_terminal)
+    return __print::__vprint_nonunicode(__stream, __fmt, __args, __write_nl);
+
+  // TODO PRINT Should flush errors throw too?
+  std::fflush(__stream);
+
+  string __str = std::vformat(__fmt, __args);
+  // UTF-16 uses the same number or less code units than UTF-8.
+  // However the size of the code unit is 16 bits instead of 8 bits.
+  //
+  // The buffer uses the worst-case estimate and should never resize.
+  // However when the string is large this could lead to OOM. Using a
+  // smaller size might work, but since the buffer uses a grow factor
+  // the final size might be larger when the estimate is wrong.
+  //
+  // TODO PRINT profile and improve the speed of this code.
+  __format::__retarget_buffer<wchar_t> __buffer{__str.size()};
+  __unicode::__transcode(__str.begin(), __str.end(), __buffer.__make_output_iterator());
+  if (__write_nl)
+    __buffer.push_back(L'\n');
+
+  [[maybe_unused]] wstring_view __view = __buffer.__view();
+
+  // The macro _LIBCPP_TESTING_PRINT_WRITE_TO_WINDOWS_CONSOLE_FUNCTION is used to change
+  // the behavior in the test. This is not part of the public API.
+#      ifdef _LIBCPP_TESTING_PRINT_WRITE_TO_WINDOWS_CONSOLE_FUNCTION
+  _LIBCPP_TESTING_PRINT_WRITE_TO_WINDOWS_CONSOLE_FUNCTION(__stream, __view);
+#      elif defined(_LIBCPP_WIN32API)
+  std::__write_to_windows_console(__stream, __view);
+#      else
+  std::__throw_runtime_error("No defintion of _LIBCPP_TESTING_PRINT_WRITE_TO_WINDOWS_CONSOLE_FUNCTION and "
+                             "__write_to_windows_console is not available.");
+#      endif
+}
+#    endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
+_LIBCPP_HIDE_FROM_ABI inline void
+__vprint_unicode([[maybe_unused]] FILE* __stream,
+                 [[maybe_unused]] string_view __fmt,
+                 [[maybe_unused]] format_args __args,
+                 [[maybe_unused]] bool __write_nl) {
+  _LIBCPP_ASSERT_NON_NULL(__stream, "__stream must be a valid pointer to an output C stream");
+
+  // [print.fun]
+  //   7 - Effects: If stream refers to a terminal capable of displaying
+  //       Unicode, writes out to the terminal using the native Unicode
+  //       API; if out contains invalid code units, the behavior is
+  //       undefined and implementations are encouraged to diagnose it.
+  //       Otherwise writes out to stream unchanged. If the native
+  //       Unicode API is used, the function flushes stream before
+  //       writing out.
+  //   8 - Throws: Any exception thrown by the call to vformat
+  //       ([format.err.report]). system_error if writing to the terminal
+  //       or stream fails. May throw bad_alloc.
+  //   9 - Recommended practice: If invoking the native Unicode API
+  //       requires transcoding, implementations should substitute
+  //       invalid code units with U+FFFD replacement character per the
+  //       Unicode Standard, Chapter 3.9 U+FFFD Substitution in
+  //       Conversion.
+
+  // On non-Windows platforms the Unicode API is the normal file I/O API
+  // so there the call can be forwarded to the non_unicode API. On
+  // Windows there is a different API. This API requires transcoding.
+
+#    ifndef _LIBCPP_WIN32API
+  __print::__vprint_unicode_posix(__stream, __fmt, __args, __write_nl, __print::__is_terminal(__stream));
+#    elif !defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS)
+  __print::__vprint_unicode_windows(__stream, __fmt, __args, __write_nl, __print::__is_terminal(__stream));
+#    else
+#      error "Windows builds with wchar_t disabled are not supported."
+#    endif
+}
+
+#  endif // _LIBCPP_HAS_NO_UNICODE
+
+} // namespace __print
+
+template <class... _Args>
+_LIBCPP_HIDE_FROM_ABI void print(FILE* __stream, format_string<_Args...> __fmt, _Args&&... __args) {
+#  ifndef _LIBCPP_HAS_NO_UNICODE
+  if constexpr (__print::__use_unicode_execution_charset)
+    __print::__vprint_unicode(__stream, __fmt.get(), std::make_format_args(__args...), false);
+  else
+    __print::__vprint_nonunicode(__stream, __fmt.get(), std::make_format_args(__args...), false);
+#  else  // _LIBCPP_HAS_NO_UNICODE
+  __print::__vprint_nonunicode(__stream, __fmt.get(), std::make_format_args(__args...), false);
+#  endif // _LIBCPP_HAS_NO_UNICODE
+}
+
+template <class... _Args>
+_LIBCPP_HIDE_FROM_ABI void print(format_string<_Args...> __fmt, _Args&&... __args) {
+  std::print(stdout, __fmt, std::forward<_Args>(__args)...);
+}
+
+template <class... _Args>
+_LIBCPP_HIDE_FROM_ABI void println(FILE* __stream, format_string<_Args...> __fmt, _Args&&... __args) {
+#  ifndef _LIBCPP_HAS_NO_UNICODE
+  // Note the wording in the Standard is inefficient. The output of
+  // std::format is a std::string which is then copied. This solution
+  // just appends a newline at the end of the output.
+  if constexpr (__print::__use_unicode_execution_charset)
+    __print::__vprint_unicode(__stream, __fmt.get(), std::make_format_args(__args...), true);
+  else
+    __print::__vprint_nonunicode(__stream, __fmt.get(), std::make_format_args(__args...), true);
+#  else  // _LIBCPP_HAS_NO_UNICODE
+  __print::__vprint_nonunicode(__stream, __fmt.get(), std::make_format_args(__args...), true);
+#  endif // _LIBCPP_HAS_NO_UNICODE
+}
+
+template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
+_LIBCPP_HIDE_FROM_ABI inline void println(FILE* __stream) {
+  std::print(__stream, "\n");
+}
+
+template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
+_LIBCPP_HIDE_FROM_ABI inline void println() {
+  println(stdout);
+}
+
+template <class... _Args>
+_LIBCPP_HIDE_FROM_ABI void println(format_string<_Args...> __fmt, _Args&&... __args) {
+  std::println(stdout, __fmt, std::forward<_Args>(__args)...);
+}
+
+#  ifndef _LIBCPP_HAS_NO_UNICODE
+template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
+_LIBCPP_HIDE_FROM_ABI inline void vprint_unicode(FILE* __stream, string_view __fmt, format_args __args) {
+  __print::__vprint_unicode(__stream, __fmt, __args, false);
+}
+
+template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
+_LIBCPP_HIDE_FROM_ABI inline void vprint_unicode(string_view __fmt, format_args __args) {
+  std::vprint_unicode(stdout, __fmt, __args);
+}
+
+#  endif // _LIBCPP_HAS_NO_UNICODE
+
+template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
+_LIBCPP_HIDE_FROM_ABI inline void vprint_nonunicode(FILE* __stream, string_view __fmt, format_args __args) {
+  __print::__vprint_nonunicode(__stream, __fmt, __args, false);
+}
+
+template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
+_LIBCPP_HIDE_FROM_ABI inline void vprint_nonunicode(string_view __fmt, format_args __args) {
+  std::vprint_nonunicode(stdout, __fmt, __args);
+}
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_PRINT
diff --git a/libcxx/include/__cxx03/queue b/libcxx/include/__cxx03/queue
new file mode 100644
index 000000000000..9508de9f9eff
--- /dev/null
+++ b/libcxx/include/__cxx03/queue
@@ -0,0 +1,956 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_QUEUE
+#define _LIBCPP_QUEUE
+
+/*
+    queue synopsis
+
+namespace std
+{
+
+template <class T, class Container = deque<T>>
+class queue
+{
+public:
+    typedef Container                                container_type;
+    typedef typename container_type::value_type      value_type;
+    typedef typename container_type::reference       reference;
+    typedef typename container_type::const_reference const_reference;
+    typedef typename container_type::size_type       size_type;
+
+protected:
+    container_type c;
+
+public:
+    queue() = default;
+    ~queue() = default;
+
+    queue(const queue& q) = default;
+    queue(queue&& q) = default;
+
+    queue& operator=(const queue& q) = default;
+    queue& operator=(queue&& q) = default;
+
+    explicit queue(const container_type& c);
+    explicit queue(container_type&& c)
+    template<class InputIterator>
+        queue(InputIterator first, InputIterator last); // since C++23
+    template<container-compatible-range<T> R> queue(from_range_t, R&& rg); // since C++23
+    template <class Alloc>
+        explicit queue(const Alloc& a);
+    template <class Alloc>
+        queue(const container_type& c, const Alloc& a);
+    template <class Alloc>
+        queue(container_type&& c, const Alloc& a);
+    template <class Alloc>
+        queue(const queue& q, const Alloc& a);
+    template <class Alloc>
+        queue(queue&& q, const Alloc& a);
+    template <class InputIterator, class Alloc>
+        queue(InputIterator first, InputIterator last, const Alloc&); // since C++23
+    template<container-compatible-range<T> R, class Alloc>
+        queue(from_range_t, R&& rg, const Alloc&); // since C++23
+
+    bool      empty() const;
+    size_type size() const;
+
+    reference       front();
+    const_reference front() const;
+    reference       back();
+    const_reference back() const;
+
+    void push(const value_type& v);
+    void push(value_type&& v);
+    template<container-compatible-range<T> R>
+      void push_range(R&& rg); // C++23
+    template <class... Args> reference emplace(Args&&... args); // reference in C++17
+    void pop();
+
+    void swap(queue& q) noexcept(is_nothrow_swappable_v<Container>)
+};
+
+template<class Container>
+  queue(Container) -> queue<typename Container::value_type, Container>; // C++17
+
+template<class InputIterator>
+  queue(InputIterator, InputIterator) -> queue<iter-value-type<InputIterator>>; // since C++23
+
+template<ranges::input_range R>
+  queue(from_range_t, R&&) -> queue<ranges::range_value_t<R>>; // since C++23
+
+template<class Container, class Allocator>
+  queue(Container, Allocator) -> queue<typename Container::value_type, Container>; // C++17
+
+template<class InputIterator, class Allocator>
+  queue(InputIterator, InputIterator, Allocator)
+  -> queue<iter-value-type<InputIterator>,
+           deque<iter-value-type<InputIterator>, Allocator>>; // since C++23
+
+template<ranges::input_range R, class Allocator>
+    queue(from_range_t, R&&, Allocator)
+      -> queue<ranges::range_value_t<R>, deque<ranges::range_value_t<R>, Allocator>>; // since C++23
+
+template <class T, class Container>
+  bool operator==(const queue<T, Container>& x,const queue<T, Container>& y);
+
+template <class T, class Container>
+  bool operator< (const queue<T, Container>& x,const queue<T, Container>& y);
+
+template <class T, class Container>
+  bool operator!=(const queue<T, Container>& x,const queue<T, Container>& y);
+
+template <class T, class Container>
+  bool operator> (const queue<T, Container>& x,const queue<T, Container>& y);
+
+template <class T, class Container>
+  bool operator>=(const queue<T, Container>& x,const queue<T, Container>& y);
+
+template <class T, class Container>
+  bool operator<=(const queue<T, Container>& x,const queue<T, Container>& y);
+
+template<class T, three_way_comparable Container>
+  compare_three_way_result_t<Container>
+    operator<=>(const queue<T, Container>& x, const queue<T, Container>& y);  // since C++20
+
+template <class T, class Container>
+  void swap(queue<T, Container>& x, queue<T, Container>& y)
+  noexcept(noexcept(x.swap(y)));
+
+template <class T, class Container = vector<T>,
+          class Compare = less<typename Container::value_type>>
+class priority_queue
+{
+public:
+    typedef Container                                container_type;
+    typedef typename container_type::value_type      value_type;
+    typedef typename container_type::reference       reference;
+    typedef typename container_type::const_reference const_reference;
+    typedef typename container_type::size_type       size_type;
+
+protected:
+    container_type c;
+    Compare comp;
+
+public:
+    priority_queue() : priority_queue(Compare()) {} // C++20
+    explicit priority_queue(const Compare& x) : priority_queue(x, Container()) {}
+    priority_queue(const Compare& x, const Container&);
+    explicit priority_queue(const Compare& x = Compare(), Container&& = Container()); // before C++20
+    priority_queue(const Compare& x, Container&&); // C++20
+    template <class InputIterator>
+        priority_queue(InputIterator first, InputIterator last,
+                       const Compare& comp = Compare());
+    template <class InputIterator>
+        priority_queue(InputIterator first, InputIterator last,
+                       const Compare& comp, const Container& c);
+    template <class InputIterator>
+        priority_queue(InputIterator first, InputIterator last,
+                       const Compare& comp, Container&& c);
+    template <container-compatible-range<T> R>
+        priority_queue(from_range_t, R&& rg, const Compare& x = Compare()); // since C++23
+    template <class Alloc>
+        explicit priority_queue(const Alloc& a);
+    template <class Alloc>
+        priority_queue(const Compare& comp, const Alloc& a);
+    template <class Alloc>
+        priority_queue(const Compare& comp, const Container& c,
+                       const Alloc& a);
+    template <class Alloc>
+        priority_queue(const Compare& comp, Container&& c,
+                       const Alloc& a);
+    template <class InputIterator>
+        priority_queue(InputIterator first, InputIterator last,
+                       const Alloc& a);
+    template <class InputIterator>
+        priority_queue(InputIterator first, InputIterator last,
+                       const Compare& comp, const Alloc& a);
+    template <class InputIterator>
+        priority_queue(InputIterator first, InputIterator last,
+                       const Compare& comp, const Container& c, const Alloc& a);
+    template <class InputIterator>
+        priority_queue(InputIterator first, InputIterator last,
+                       const Compare& comp, Container&& c, const Alloc& a);
+    template <container-compatible-range<T> R, class Alloc>
+      priority_queue(from_range_t, R&& rg, const Compare&, const Alloc&); // since C++23
+    template <container-compatible-range<T> R, class Alloc>
+      priority_queue(from_range_t, R&& rg, const Alloc&); // since C++23
+    template <class Alloc>
+        priority_queue(const priority_queue& q, const Alloc& a);
+    template <class Alloc>
+        priority_queue(priority_queue&& q, const Alloc& a);
+
+    bool            empty() const;
+    size_type       size() const;
+    const_reference top() const;
+
+    void push(const value_type& v);
+    void push(value_type&& v);
+    template<container-compatible-range<T> R>
+      void push_range(R&& rg); // C++23
+    template <class... Args> void emplace(Args&&... args);
+    void pop();
+
+    void swap(priority_queue& q)
+        noexcept(is_nothrow_swappable_v<Container> &&
+                 is_nothrow_swappable_v<Comp>)
+};
+
+template <class Compare, class Container>
+priority_queue(Compare, Container)
+    -> priority_queue<typename Container::value_type, Container, Compare>; // C++17
+
+template<class InputIterator,
+         class Compare = less<iter-value-type<InputIterator>>,
+         class Container = vector<iter-value-type<InputIterator>>>
+priority_queue(InputIterator, InputIterator, Compare = Compare(), Container = Container())
+    -> priority_queue<iter-value-type<InputIterator>, Container, Compare>; // C++17
+
+template<ranges::input_range R, class Compare = less<ranges::range_value_t<R>>>
+  priority_queue(from_range_t, R&&, Compare = Compare())
+    -> priority_queue<ranges::range_value_t<R>, vector<ranges::range_value_t<R>>, Compare>; // C++23
+
+template<class Compare, class Container, class Allocator>
+priority_queue(Compare, Container, Allocator)
+    -> priority_queue<typename Container::value_type, Container, Compare>; // C++17
+
+template<class InputIterator, class Allocator>
+priority_queue(InputIterator, InputIterator, Allocator)
+    -> priority_queue<iter-value-type<InputIterator>,
+                      vector<iter-value-type<InputIterator>, Allocator>,
+                      less<iter-value-type<InputIterator>>>; // C++17
+
+template<class InputIterator, class Compare, class Allocator>
+priority_queue(InputIterator, InputIterator, Compare, Allocator)
+    -> priority_queue<iter-value-type<InputIterator>,
+                      vector<iter-value-type<InputIterator>, Allocator>, Compare>; // C++17
+
+template<class InputIterator, class Compare, class Container, class Allocator>
+priority_queue(InputIterator, InputIterator, Compare, Container, Allocator)
+    -> priority_queue<typename Container::value_type, Container, Compare>; // C++17
+
+template<ranges::input_range R, class Compare, class Allocator>
+  priority_queue(from_range_t, R&&, Compare, Allocator)
+    -> priority_queue<ranges::range_value_t<R>, vector<ranges::range_value_t<R>, Allocator>,
+                        Compare>; // C++23
+
+template<ranges::input_range R, class Allocator>
+  priority_queue(from_range_t, R&&, Allocator)
+    -> priority_queue<ranges::range_value_t<R>, vector<ranges::range_value_t<R>, Allocator>>; // C++23
+
+template <class T, class Container, class Compare>
+  void swap(priority_queue<T, Container, Compare>& x,
+            priority_queue<T, Container, Compare>& y)
+            noexcept(noexcept(x.swap(y)));
+
+}  // std
+
+*/
+
+#include <__algorithm/make_heap.h>
+#include <__algorithm/pop_heap.h>
+#include <__algorithm/push_heap.h>
+#include <__algorithm/ranges_copy.h>
+#include <__config>
+#include <__functional/operations.h>
+#include <__fwd/deque.h>
+#include <__fwd/queue.h>
+#include <__iterator/back_insert_iterator.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/uses_allocator.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/container_compatible_range.h>
+#include <__ranges/from_range.h>
+#include <__utility/forward.h>
+#include <deque>
+#include <vector>
+#include <version>
+
+// standard-mandated includes
+
+// [queue.syn]
+#include <compare>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Container>
+_LIBCPP_HIDE_FROM_ABI bool operator==(const queue<_Tp, _Container>& __x, const queue<_Tp, _Container>& __y);
+
+template <class _Tp, class _Container>
+_LIBCPP_HIDE_FROM_ABI bool operator<(const queue<_Tp, _Container>& __x, const queue<_Tp, _Container>& __y);
+
+template <class _Tp, class _Container /*= deque<_Tp>*/>
+class _LIBCPP_TEMPLATE_VIS queue {
+public:
+  typedef _Container container_type;
+  typedef typename container_type::value_type value_type;
+  typedef typename container_type::reference reference;
+  typedef typename container_type::const_reference const_reference;
+  typedef typename container_type::size_type size_type;
+  static_assert(is_same<_Tp, value_type>::value, "");
+
+protected:
+  container_type c;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI queue() _NOEXCEPT_(is_nothrow_default_constructible<container_type>::value) : c() {}
+
+  _LIBCPP_HIDE_FROM_ABI queue(const queue& __q) : c(__q.c) {}
+
+#if _LIBCPP_STD_VER >= 23
+  template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI queue(_InputIterator __first, _InputIterator __last) : c(__first, __last) {}
+
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI queue(from_range_t, _Range&& __range) : c(from_range, std::forward<_Range>(__range)) {}
+
+  template <class _InputIterator,
+            class _Alloc,
+            __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0,
+            __enable_if_t<uses_allocator<container_type, _Alloc>::value, int>        = 0>
+  _LIBCPP_HIDE_FROM_ABI queue(_InputIterator __first, _InputIterator __second, const _Alloc& __alloc)
+      : c(__first, __second, __alloc) {}
+
+  template <_ContainerCompatibleRange<_Tp> _Range,
+            class _Alloc,
+            __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI queue(from_range_t, _Range&& __range, const _Alloc& __alloc)
+      : c(from_range, std::forward<_Range>(__range), __alloc) {}
+
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI queue& operator=(const queue& __q) {
+    c = __q.c;
+    return *this;
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI queue(queue&& __q) noexcept(is_nothrow_move_constructible<container_type>::value)
+      : c(std::move(__q.c)) {}
+
+  _LIBCPP_HIDE_FROM_ABI queue& operator=(queue&& __q) noexcept(is_nothrow_move_assignable<container_type>::value) {
+    c = std::move(__q.c);
+    return *this;
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI explicit queue(const container_type& __c) : c(__c) {}
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI explicit queue(container_type&& __c) : c(std::move(__c)) {}
+#endif // _LIBCPP_CXX03_LANG
+
+  template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit queue(const _Alloc& __a) : c(__a) {}
+
+  template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI queue(const queue& __q, const _Alloc& __a) : c(__q.c, __a) {}
+
+  template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI queue(const container_type& __c, const _Alloc& __a) : c(__c, __a) {}
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI queue(container_type&& __c, const _Alloc& __a) : c(std::move(__c), __a) {}
+
+  template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI queue(queue&& __q, const _Alloc& __a) : c(std::move(__q.c), __a) {}
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const { return c.empty(); }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const { return c.size(); }
+
+  _LIBCPP_HIDE_FROM_ABI reference front() { return c.front(); }
+  _LIBCPP_HIDE_FROM_ABI const_reference front() const { return c.front(); }
+  _LIBCPP_HIDE_FROM_ABI reference back() { return c.back(); }
+  _LIBCPP_HIDE_FROM_ABI const_reference back() const { return c.back(); }
+
+  _LIBCPP_HIDE_FROM_ABI void push(const value_type& __v) { c.push_back(__v); }
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI void push(value_type&& __v) { c.push_back(std::move(__v)); }
+
+#  if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI void push_range(_Range&& __range) {
+    if constexpr (requires(container_type& __c) { __c.append_range(std::forward<_Range>(__range)); }) {
+      c.append_range(std::forward<_Range>(__range));
+    } else {
+      ranges::copy(std::forward<_Range>(__range), std::back_inserter(c));
+    }
+  }
+#  endif
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI
+#  if _LIBCPP_STD_VER >= 17
+  decltype(auto)
+  emplace(_Args&&... __args) {
+    return c.emplace_back(std::forward<_Args>(__args)...);
+  }
+#  else
+  void
+  emplace(_Args&&... __args) {
+    c.emplace_back(std::forward<_Args>(__args)...);
+  }
+#  endif
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI void pop() { c.pop_front(); }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(queue& __q) _NOEXCEPT_(__is_nothrow_swappable_v<container_type>) {
+    using std::swap;
+    swap(c, __q.c);
+  }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI const _Container& __get_container() const { return c; }
+
+  template <class _T1, class _OtherContainer>
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const queue<_T1, _OtherContainer>& __x, const queue<_T1, _OtherContainer>& __y);
+
+  template <class _T1, class _OtherContainer>
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator<(const queue<_T1, _OtherContainer>& __x, const queue<_T1, _OtherContainer>& __y);
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Container, class = enable_if_t<!__is_allocator<_Container>::value> >
+queue(_Container) -> queue<typename _Container::value_type, _Container>;
+
+template <class _Container,
+          class _Alloc,
+          class = enable_if_t<!__is_allocator<_Container>::value>,
+          class = enable_if_t<uses_allocator<_Container, _Alloc>::value> >
+queue(_Container, _Alloc) -> queue<typename _Container::value_type, _Container>;
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
+queue(_InputIterator, _InputIterator) -> queue<__iter_value_type<_InputIterator>>;
+
+template <ranges::input_range _Range>
+queue(from_range_t, _Range&&) -> queue<ranges::range_value_t<_Range>>;
+
+template <class _InputIterator,
+          class _Alloc,
+          __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0,
+          __enable_if_t<__is_allocator<_Alloc>::value, int>                        = 0>
+queue(_InputIterator,
+      _InputIterator,
+      _Alloc) -> queue<__iter_value_type<_InputIterator>, deque<__iter_value_type<_InputIterator>, _Alloc>>;
+
+template <ranges::input_range _Range, class _Alloc, __enable_if_t<__is_allocator<_Alloc>::value, int> = 0>
+queue(from_range_t,
+      _Range&&,
+      _Alloc) -> queue<ranges::range_value_t<_Range>, deque<ranges::range_value_t<_Range>, _Alloc>>;
+#endif
+
+template <class _Tp, class _Container>
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(const queue<_Tp, _Container>& __x, const queue<_Tp, _Container>& __y) {
+  return __x.c == __y.c;
+}
+
+template <class _Tp, class _Container>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(const queue<_Tp, _Container>& __x, const queue<_Tp, _Container>& __y) {
+  return __x.c < __y.c;
+}
+
+template <class _Tp, class _Container>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const queue<_Tp, _Container>& __x, const queue<_Tp, _Container>& __y) {
+  return !(__x == __y);
+}
+
+template <class _Tp, class _Container>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>(const queue<_Tp, _Container>& __x, const queue<_Tp, _Container>& __y) {
+  return __y < __x;
+}
+
+template <class _Tp, class _Container>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>=(const queue<_Tp, _Container>& __x, const queue<_Tp, _Container>& __y) {
+  return !(__x < __y);
+}
+
+template <class _Tp, class _Container>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const queue<_Tp, _Container>& __x, const queue<_Tp, _Container>& __y) {
+  return !(__y < __x);
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp, three_way_comparable _Container>
+_LIBCPP_HIDE_FROM_ABI compare_three_way_result_t<_Container>
+operator<=>(const queue<_Tp, _Container>& __x, const queue<_Tp, _Container>& __y) {
+  // clang 16 bug: declaring `friend operator<=>` causes "use of overloaded operator '*' is ambiguous" errors
+  return __x.__get_container() <=> __y.__get_container();
+}
+
+#endif
+
+template <class _Tp, class _Container, __enable_if_t<__is_swappable_v<_Container>, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI void swap(queue<_Tp, _Container>& __x, queue<_Tp, _Container>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+template <class _Tp, class _Container, class _Alloc>
+struct _LIBCPP_TEMPLATE_VIS uses_allocator<queue<_Tp, _Container>, _Alloc> : public uses_allocator<_Container, _Alloc> {
+};
+
+template <class _Tp, class _Container, class _Compare>
+class _LIBCPP_TEMPLATE_VIS priority_queue {
+public:
+  typedef _Container container_type;
+  typedef _Compare value_compare;
+  typedef typename container_type::value_type value_type;
+  typedef typename container_type::reference reference;
+  typedef typename container_type::const_reference const_reference;
+  typedef typename container_type::size_type size_type;
+  static_assert(is_same<_Tp, value_type>::value, "");
+
+protected:
+  container_type c;
+  value_compare comp;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI priority_queue() _NOEXCEPT_(
+      is_nothrow_default_constructible<container_type>::value&& is_nothrow_default_constructible<value_compare>::value)
+      : c(), comp() {}
+
+  _LIBCPP_HIDE_FROM_ABI priority_queue(const priority_queue& __q) : c(__q.c), comp(__q.comp) {}
+
+  _LIBCPP_HIDE_FROM_ABI priority_queue& operator=(const priority_queue& __q) {
+    c    = __q.c;
+    comp = __q.comp;
+    return *this;
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI priority_queue(priority_queue&& __q) noexcept(
+      is_nothrow_move_constructible<container_type>::value && is_nothrow_move_constructible<value_compare>::value)
+      : c(std::move(__q.c)), comp(std::move(__q.comp)) {}
+
+  _LIBCPP_HIDE_FROM_ABI priority_queue& operator=(priority_queue&& __q) noexcept(
+      is_nothrow_move_assignable<container_type>::value && is_nothrow_move_assignable<value_compare>::value) {
+    c    = std::move(__q.c);
+    comp = std::move(__q.comp);
+    return *this;
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI explicit priority_queue(const value_compare& __comp) : c(), comp(__comp) {}
+  _LIBCPP_HIDE_FROM_ABI priority_queue(const value_compare& __comp, const container_type& __c);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI priority_queue(const value_compare& __comp, container_type&& __c);
+#endif
+  template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp = value_compare());
+
+  template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp, const container_type& __c);
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp, container_type&& __c);
+#endif // _LIBCPP_CXX03_LANG
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI priority_queue(from_range_t, _Range&& __range, const value_compare& __comp = value_compare())
+      : c(from_range, std::forward<_Range>(__range)), comp(__comp) {
+    std::make_heap(c.begin(), c.end(), comp);
+  }
+#endif
+
+  template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit priority_queue(const _Alloc& __a);
+
+  template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI priority_queue(const value_compare& __comp, const _Alloc& __a);
+
+  template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI priority_queue(const value_compare& __comp, const container_type& __c, const _Alloc& __a);
+
+  template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI priority_queue(const priority_queue& __q, const _Alloc& __a);
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI priority_queue(const value_compare& __comp, container_type&& __c, const _Alloc& __a);
+
+  template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI priority_queue(priority_queue&& __q, const _Alloc& __a);
+#endif // _LIBCPP_CXX03_LANG
+
+  template <
+      class _InputIter,
+      class _Alloc,
+      __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<container_type, _Alloc>::value,
+                    int> = 0>
+  _LIBCPP_HIDE_FROM_ABI priority_queue(_InputIter __f, _InputIter __l, const _Alloc& __a);
+
+  template <
+      class _InputIter,
+      class _Alloc,
+      __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<container_type, _Alloc>::value,
+                    int> = 0>
+  _LIBCPP_HIDE_FROM_ABI priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp, const _Alloc& __a);
+
+  template <
+      class _InputIter,
+      class _Alloc,
+      __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<container_type, _Alloc>::value,
+                    int> = 0>
+  _LIBCPP_HIDE_FROM_ABI priority_queue(
+      _InputIter __f, _InputIter __l, const value_compare& __comp, const container_type& __c, const _Alloc& __a);
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <
+      class _InputIter,
+      class _Alloc,
+      __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<container_type, _Alloc>::value,
+                    int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp, container_type&& __c, const _Alloc& __a);
+#endif // _LIBCPP_CXX03_LANG
+
+#if _LIBCPP_STD_VER >= 23
+
+  template <_ContainerCompatibleRange<_Tp> _Range,
+            class _Alloc,
+            class = enable_if_t<uses_allocator<_Container, _Alloc>::value>>
+  _LIBCPP_HIDE_FROM_ABI priority_queue(from_range_t, _Range&& __range, const value_compare& __comp, const _Alloc& __a)
+      : c(from_range, std::forward<_Range>(__range), __a), comp(__comp) {
+    std::make_heap(c.begin(), c.end(), comp);
+  }
+
+  template <_ContainerCompatibleRange<_Tp> _Range,
+            class _Alloc,
+            class = enable_if_t<uses_allocator<_Container, _Alloc>::value>>
+  _LIBCPP_HIDE_FROM_ABI priority_queue(from_range_t, _Range&& __range, const _Alloc& __a)
+      : c(from_range, std::forward<_Range>(__range), __a), comp() {
+    std::make_heap(c.begin(), c.end(), comp);
+  }
+
+#endif
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const { return c.empty(); }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const { return c.size(); }
+  _LIBCPP_HIDE_FROM_ABI const_reference top() const { return c.front(); }
+
+  _LIBCPP_HIDE_FROM_ABI void push(const value_type& __v);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI void push(value_type&& __v);
+
+#  if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI void push_range(_Range&& __range) {
+    if constexpr (requires(container_type& __c) { __c.append_range(std::forward<_Range>(__range)); }) {
+      c.append_range(std::forward<_Range>(__range));
+    } else {
+      ranges::copy(std::forward<_Range>(__range), std::back_inserter(c));
+    }
+
+    std::make_heap(c.begin(), c.end(), comp);
+  }
+#  endif
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI void emplace(_Args&&... __args);
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI void pop();
+
+  _LIBCPP_HIDE_FROM_ABI void swap(priority_queue& __q)
+      _NOEXCEPT_(__is_nothrow_swappable_v<container_type>&& __is_nothrow_swappable_v<value_compare>);
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI const _Container& __get_container() const { return c; }
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Compare,
+          class _Container,
+          class = enable_if_t<!__is_allocator<_Compare>::value>,
+          class = enable_if_t<!__is_allocator<_Container>::value> >
+priority_queue(_Compare, _Container) -> priority_queue<typename _Container::value_type, _Container, _Compare>;
+
+template <class _InputIterator,
+          class _Compare   = less<__iter_value_type<_InputIterator>>,
+          class _Container = vector<__iter_value_type<_InputIterator>>,
+          class            = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class            = enable_if_t<!__is_allocator<_Compare>::value>,
+          class            = enable_if_t<!__is_allocator<_Container>::value> >
+priority_queue(_InputIterator, _InputIterator, _Compare = _Compare(), _Container = _Container())
+    -> priority_queue<__iter_value_type<_InputIterator>, _Container, _Compare>;
+
+template <class _Compare,
+          class _Container,
+          class _Alloc,
+          class = enable_if_t<!__is_allocator<_Compare>::value>,
+          class = enable_if_t<!__is_allocator<_Container>::value>,
+          class = enable_if_t<uses_allocator<_Container, _Alloc>::value> >
+priority_queue(_Compare, _Container, _Alloc) -> priority_queue<typename _Container::value_type, _Container, _Compare>;
+
+template <class _InputIterator,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value> >
+priority_queue(_InputIterator, _InputIterator, _Allocator)
+    -> priority_queue<__iter_value_type<_InputIterator>,
+                      vector<__iter_value_type<_InputIterator>, _Allocator>,
+                      less<__iter_value_type<_InputIterator>>>;
+
+template <class _InputIterator,
+          class _Compare,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<!__is_allocator<_Compare>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value> >
+priority_queue(_InputIterator, _InputIterator, _Compare, _Allocator)
+    -> priority_queue<__iter_value_type<_InputIterator>,
+                      vector<__iter_value_type<_InputIterator>, _Allocator>,
+                      _Compare>;
+
+template <class _InputIterator,
+          class _Compare,
+          class _Container,
+          class _Alloc,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<!__is_allocator<_Compare>::value>,
+          class = enable_if_t<!__is_allocator<_Container>::value>,
+          class = enable_if_t<uses_allocator<_Container, _Alloc>::value> >
+priority_queue(_InputIterator, _InputIterator, _Compare, _Container, _Alloc)
+    -> priority_queue<typename _Container::value_type, _Container, _Compare>;
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+
+template <ranges::input_range _Range,
+          class _Compare = less<ranges::range_value_t<_Range>>,
+          class          = enable_if_t<!__is_allocator<_Compare>::value>>
+priority_queue(from_range_t, _Range&&, _Compare = _Compare())
+    -> priority_queue<ranges::range_value_t<_Range>, vector<ranges::range_value_t<_Range>>, _Compare>;
+
+template <ranges::input_range _Range,
+          class _Compare,
+          class _Alloc,
+          class = enable_if_t<!__is_allocator<_Compare>::value>,
+          class = enable_if_t<__is_allocator<_Alloc>::value>>
+priority_queue(from_range_t, _Range&&, _Compare, _Alloc)
+    -> priority_queue<ranges::range_value_t<_Range>, vector<ranges::range_value_t<_Range>, _Alloc>, _Compare>;
+
+template <ranges::input_range _Range, class _Alloc, class = enable_if_t<__is_allocator<_Alloc>::value>>
+priority_queue(from_range_t, _Range&&, _Alloc)
+    -> priority_queue<ranges::range_value_t<_Range>, vector<ranges::range_value_t<_Range>, _Alloc>>;
+
+#endif
+
+template <class _Tp, class _Container, class _Compare>
+inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const _Compare& __comp, const container_type& __c)
+    : c(__c), comp(__comp) {
+  std::make_heap(c.begin(), c.end(), comp);
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Container, class _Compare>
+inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const value_compare& __comp, container_type&& __c)
+    : c(std::move(__c)), comp(__comp) {
+  std::make_heap(c.begin(), c.end(), comp);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Container, class _Compare>
+template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> >
+inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+    _InputIter __f, _InputIter __l, const value_compare& __comp)
+    : c(__f, __l), comp(__comp) {
+  std::make_heap(c.begin(), c.end(), comp);
+}
+
+template <class _Tp, class _Container, class _Compare>
+template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> >
+inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+    _InputIter __f, _InputIter __l, const value_compare& __comp, const container_type& __c)
+    : c(__c), comp(__comp) {
+  c.insert(c.end(), __f, __l);
+  std::make_heap(c.begin(), c.end(), comp);
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Container, class _Compare>
+template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> >
+inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+    _InputIter __f, _InputIter __l, const value_compare& __comp, container_type&& __c)
+    : c(std::move(__c)), comp(__comp) {
+  c.insert(c.end(), __f, __l);
+  std::make_heap(c.begin(), c.end(), comp);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Container, class _Compare>
+template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value, int> >
+inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const _Alloc& __a) : c(__a) {}
+
+template <class _Tp, class _Container, class _Compare>
+template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value, int> >
+inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const value_compare& __comp, const _Alloc& __a)
+    : c(__a), comp(__comp) {}
+
+template <class _Tp, class _Container, class _Compare>
+template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value, int> >
+inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+    const value_compare& __comp, const container_type& __c, const _Alloc& __a)
+    : c(__c, __a), comp(__comp) {
+  std::make_heap(c.begin(), c.end(), comp);
+}
+
+template <class _Tp, class _Container, class _Compare>
+template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value, int> >
+inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const priority_queue& __q, const _Alloc& __a)
+    : c(__q.c, __a), comp(__q.comp) {}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Container, class _Compare>
+template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value, int> >
+inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+    const value_compare& __comp, container_type&& __c, const _Alloc& __a)
+    : c(std::move(__c), __a), comp(__comp) {
+  std::make_heap(c.begin(), c.end(), comp);
+}
+
+template <class _Tp, class _Container, class _Compare>
+template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value, int> >
+inline priority_queue<_Tp, _Container, _Compare>::priority_queue(priority_queue&& __q, const _Alloc& __a)
+    : c(std::move(__q.c), __a), comp(std::move(__q.comp)) {}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Container, class _Compare>
+template <
+    class _InputIter,
+    class _Alloc,
+    __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<_Container, _Alloc>::value, int> >
+inline priority_queue<_Tp, _Container, _Compare>::priority_queue(_InputIter __f, _InputIter __l, const _Alloc& __a)
+    : c(__f, __l, __a), comp() {
+  std::make_heap(c.begin(), c.end(), comp);
+}
+
+template <class _Tp, class _Container, class _Compare>
+template <
+    class _InputIter,
+    class _Alloc,
+    __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<_Container, _Alloc>::value, int> >
+inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+    _InputIter __f, _InputIter __l, const value_compare& __comp, const _Alloc& __a)
+    : c(__f, __l, __a), comp(__comp) {
+  std::make_heap(c.begin(), c.end(), comp);
+}
+
+template <class _Tp, class _Container, class _Compare>
+template <
+    class _InputIter,
+    class _Alloc,
+    __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<_Container, _Alloc>::value, int> >
+inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+    _InputIter __f, _InputIter __l, const value_compare& __comp, const container_type& __c, const _Alloc& __a)
+    : c(__c, __a), comp(__comp) {
+  c.insert(c.end(), __f, __l);
+  std::make_heap(c.begin(), c.end(), comp);
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Tp, class _Container, class _Compare>
+template <
+    class _InputIter,
+    class _Alloc,
+    __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<_Container, _Alloc>::value, int> >
+inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+    _InputIter __f, _InputIter __l, const value_compare& __comp, container_type&& __c, const _Alloc& __a)
+    : c(std::move(__c), __a), comp(__comp) {
+  c.insert(c.end(), __f, __l);
+  std::make_heap(c.begin(), c.end(), comp);
+}
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Container, class _Compare>
+inline void priority_queue<_Tp, _Container, _Compare>::push(const value_type& __v) {
+  c.push_back(__v);
+  std::push_heap(c.begin(), c.end(), comp);
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Container, class _Compare>
+inline void priority_queue<_Tp, _Container, _Compare>::push(value_type&& __v) {
+  c.push_back(std::move(__v));
+  std::push_heap(c.begin(), c.end(), comp);
+}
+
+template <class _Tp, class _Container, class _Compare>
+template <class... _Args>
+inline void priority_queue<_Tp, _Container, _Compare>::emplace(_Args&&... __args) {
+  c.emplace_back(std::forward<_Args>(__args)...);
+  std::push_heap(c.begin(), c.end(), comp);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Container, class _Compare>
+inline void priority_queue<_Tp, _Container, _Compare>::pop() {
+  std::pop_heap(c.begin(), c.end(), comp);
+  c.pop_back();
+}
+
+template <class _Tp, class _Container, class _Compare>
+inline void priority_queue<_Tp, _Container, _Compare>::swap(priority_queue& __q)
+    _NOEXCEPT_(__is_nothrow_swappable_v<container_type>&& __is_nothrow_swappable_v<value_compare>) {
+  using std::swap;
+  swap(c, __q.c);
+  swap(comp, __q.comp);
+}
+
+template <class _Tp,
+          class _Container,
+          class _Compare,
+          __enable_if_t<__is_swappable_v<_Container> && __is_swappable_v<_Compare>, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(priority_queue<_Tp, _Container, _Compare>& __x, priority_queue<_Tp, _Container, _Compare>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+template <class _Tp, class _Container, class _Compare, class _Alloc>
+struct _LIBCPP_TEMPLATE_VIS uses_allocator<priority_queue<_Tp, _Container, _Compare>, _Alloc>
+    : public uses_allocator<_Container, _Alloc> {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <concepts>
+#  include <cstdlib>
+#  include <functional>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_QUEUE
diff --git a/libcxx/include/__cxx03/random b/libcxx/include/__cxx03/random
new file mode 100644
index 000000000000..6cc3760c20e1
--- /dev/null
+++ b/libcxx/include/__cxx03/random
@@ -0,0 +1,1742 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_RANDOM
+#define _LIBCPP_RANDOM
+
+/*
+    random synopsis
+
+#include <initializer_list>
+
+namespace std
+{
+// [rand.req.urng], uniform random bit generator requirements
+template<class G>
+concept uniform_random_bit_generator = see below; // C++20
+
+// Engines
+
+template <class UIntType, UIntType a, UIntType c, UIntType m>
+class linear_congruential_engine
+{
+public:
+    // types
+    typedef UIntType result_type;
+
+    // engine characteristics
+    static constexpr result_type multiplier = a;
+    static constexpr result_type increment = c;
+    static constexpr result_type modulus = m;
+    static constexpr result_type min() { return c == 0u ? 1u: 0u;}
+    static constexpr result_type max() { return m - 1u;}
+    static constexpr result_type default_seed = 1u;
+
+    // constructors and seeding functions
+    explicit linear_congruential_engine(result_type s = default_seed);         // before C++20
+    linear_congruential_engine() : linear_congruential_engine(default_seed) {} // C++20
+    explicit linear_congruential_engine(result_type s);                        // C++20
+    template<class Sseq> explicit linear_congruential_engine(Sseq& q);
+    void seed(result_type s = default_seed);
+    template<class Sseq> void seed(Sseq& q);
+
+    // generating functions
+    result_type operator()();
+    void discard(unsigned long long z);
+};
+
+template <class UIntType, UIntType a, UIntType c, UIntType m>
+bool
+operator==(const linear_congruential_engine<UIntType, a, c, m>& x,
+           const linear_congruential_engine<UIntType, a, c, m>& y);
+
+template <class UIntType, UIntType a, UIntType c, UIntType m>
+bool
+operator!=(const linear_congruential_engine<UIntType, a, c, m>& x,
+           const linear_congruential_engine<UIntType, a, c, m>& y);
+
+template <class charT, class traits,
+          class UIntType, UIntType a, UIntType c, UIntType m>
+basic_ostream<charT, traits>&
+operator<<(basic_ostream<charT, traits>& os,
+           const linear_congruential_engine<UIntType, a, c, m>& x);
+
+template <class charT, class traits,
+          class UIntType, UIntType a, UIntType c, UIntType m>
+basic_istream<charT, traits>&
+operator>>(basic_istream<charT, traits>& is,
+           linear_congruential_engine<UIntType, a, c, m>& x);
+
+template <class UIntType, size_t w, size_t n, size_t m, size_t r,
+          UIntType a, size_t u, UIntType d, size_t s,
+          UIntType b, size_t t, UIntType c, size_t l, UIntType f>
+class mersenne_twister_engine
+{
+public:
+    // types
+    typedef UIntType result_type;
+
+    // engine characteristics
+    static constexpr size_t word_size = w;
+    static constexpr size_t state_size = n;
+    static constexpr size_t shift_size = m;
+    static constexpr size_t mask_bits = r;
+    static constexpr result_type xor_mask = a;
+    static constexpr size_t tempering_u = u;
+    static constexpr result_type tempering_d = d;
+    static constexpr size_t tempering_s = s;
+    static constexpr result_type tempering_b = b;
+    static constexpr size_t tempering_t = t;
+    static constexpr result_type tempering_c = c;
+    static constexpr size_t tempering_l = l;
+    static constexpr result_type initialization_multiplier = f;
+    static constexpr result_type min () { return 0; }
+    static constexpr result_type max() { return 2^w - 1; }
+    static constexpr result_type default_seed = 5489u;
+
+    // constructors and seeding functions
+    explicit mersenne_twister_engine(result_type s = default_seed);      // before C++20
+    mersenne_twister_engine() : mersenne_twister_engine(default_seed) {} // C++20
+    explicit mersenne_twister_engine(result_type s);                     // C++20
+    template<class Sseq> explicit mersenne_twister_engine(Sseq& q);
+    void seed(result_type value = default_seed);
+    template<class Sseq> void seed(Sseq& q);
+
+    // generating functions
+    result_type operator()();
+    void discard(unsigned long long z);
+};
+
+template <class UIntType, size_t w, size_t n, size_t m, size_t r,
+          UIntType a, size_t u, UIntType d, size_t s,
+          UIntType b, size_t t, UIntType c, size_t l, UIntType f>
+bool
+operator==(
+    const mersenne_twister_engine<UIntType, w, n, m, r, a, u, d, s, b, t, c, l, f>& x,
+    const mersenne_twister_engine<UIntType, w, n, m, r, a, u, d, s, b, t, c, l, f>& y);
+
+template <class UIntType, size_t w, size_t n, size_t m, size_t r,
+          UIntType a, size_t u, UIntType d, size_t s,
+          UIntType b, size_t t, UIntType c, size_t l, UIntType f>
+bool
+operator!=(
+    const mersenne_twister_engine<UIntType, w, n, m, r, a, u, d, s, b, t, c, l, f>& x,
+    const mersenne_twister_engine<UIntType, w, n, m, r, a, u, d, s, b, t, c, l, f>& y);
+
+template <class charT, class traits,
+          class UIntType, size_t w, size_t n, size_t m, size_t r,
+          UIntType a, size_t u, UIntType d, size_t s,
+          UIntType b, size_t t, UIntType c, size_t l, UIntType f>
+basic_ostream<charT, traits>&
+operator<<(basic_ostream<charT, traits>& os,
+           const mersenne_twister_engine<UIntType, w, n, m, r, a, u, d, s, b, t, c, l, f>& x);
+
+template <class charT, class traits,
+          class UIntType, size_t w, size_t n, size_t m, size_t r,
+          UIntType a, size_t u, UIntType d, size_t s,
+          UIntType b, size_t t, UIntType c, size_t l, UIntType f>
+basic_istream<charT, traits>&
+operator>>(basic_istream<charT, traits>& is,
+           mersenne_twister_engine<UIntType, w, n, m, r, a, u, d, s, b, t, c, l, f>& x);
+
+template<class UIntType, size_t w, size_t s, size_t r>
+class subtract_with_carry_engine
+{
+public:
+    // types
+    typedef UIntType result_type;
+
+    // engine characteristics
+    static constexpr size_t word_size = w;
+    static constexpr size_t short_lag = s;
+    static constexpr size_t long_lag = r;
+    static constexpr result_type min() { return 0; }
+    static constexpr result_type max() { return m-1; }
+    static constexpr result_type default_seed = 19780503u;
+
+    // constructors and seeding functions
+    explicit subtract_with_carry_engine(result_type value = default_seed);     // before C++20
+    subtract_with_carry_engine() : subtract_with_carry_engine(default_seed) {} // C++20
+    explicit subtract_with_carry_engine(result_type value);                    // C++20
+    template<class Sseq> explicit subtract_with_carry_engine(Sseq& q);
+    void seed(result_type value = default_seed);
+    template<class Sseq> void seed(Sseq& q);
+
+    // generating functions
+    result_type operator()();
+    void discard(unsigned long long z);
+};
+
+template<class UIntType, size_t w, size_t s, size_t r>
+bool
+operator==(
+    const subtract_with_carry_engine<UIntType, w, s, r>& x,
+    const subtract_with_carry_engine<UIntType, w, s, r>& y);
+
+template<class UIntType, size_t w, size_t s, size_t r>
+bool
+operator!=(
+    const subtract_with_carry_engine<UIntType, w, s, r>& x,
+    const subtract_with_carry_engine<UIntType, w, s, r>& y);
+
+template <class charT, class traits,
+          class UIntType, size_t w, size_t s, size_t r>
+basic_ostream<charT, traits>&
+operator<<(basic_ostream<charT, traits>& os,
+           const subtract_with_carry_engine<UIntType, w, s, r>& x);
+
+template <class charT, class traits,
+          class UIntType, size_t w, size_t s, size_t r>
+basic_istream<charT, traits>&
+operator>>(basic_istream<charT, traits>& is,
+           subtract_with_carry_engine<UIntType, w, s, r>& x);
+
+template<class Engine, size_t p, size_t r>
+class discard_block_engine
+{
+public:
+    // types
+    typedef typename Engine::result_type result_type;
+
+    // engine characteristics
+    static constexpr size_t block_size = p;
+    static constexpr size_t used_block = r;
+    static constexpr result_type min() { return Engine::min(); }
+    static constexpr result_type max() { return Engine::max(); }
+
+    // constructors and seeding functions
+    discard_block_engine();
+    explicit discard_block_engine(const Engine& e);
+    explicit discard_block_engine(Engine&& e);
+    explicit discard_block_engine(result_type s);
+    template<class Sseq> explicit discard_block_engine(Sseq& q);
+    void seed();
+    void seed(result_type s);
+    template<class Sseq> void seed(Sseq& q);
+
+    // generating functions
+    result_type operator()();
+    void discard(unsigned long long z);
+
+    // property functions
+    const Engine& base() const noexcept;
+};
+
+template<class Engine, size_t p, size_t r>
+bool
+operator==(
+    const discard_block_engine<Engine, p, r>& x,
+    const discard_block_engine<Engine, p, r>& y);
+
+template<class Engine, size_t p, size_t r>
+bool
+operator!=(
+    const discard_block_engine<Engine, p, r>& x,
+    const discard_block_engine<Engine, p, r>& y);
+
+template <class charT, class traits,
+          class Engine, size_t p, size_t r>
+basic_ostream<charT, traits>&
+operator<<(basic_ostream<charT, traits>& os,
+           const discard_block_engine<Engine, p, r>& x);
+
+template <class charT, class traits,
+          class Engine, size_t p, size_t r>
+basic_istream<charT, traits>&
+operator>>(basic_istream<charT, traits>& is,
+           discard_block_engine<Engine, p, r>& x);
+
+template<class Engine, size_t w, class UIntType>
+class independent_bits_engine
+{
+public:
+    // types
+    typedef UIntType result_type;
+
+    // engine characteristics
+    static constexpr result_type min() { return 0; }
+    static constexpr result_type max() { return 2^w - 1; }
+
+    // constructors and seeding functions
+    independent_bits_engine();
+    explicit independent_bits_engine(const Engine& e);
+    explicit independent_bits_engine(Engine&& e);
+    explicit independent_bits_engine(result_type s);
+    template<class Sseq> explicit independent_bits_engine(Sseq& q);
+    void seed();
+    void seed(result_type s);
+    template<class Sseq> void seed(Sseq& q);
+
+    // generating functions
+    result_type operator()(); void discard(unsigned long long z);
+
+    // property functions
+    const Engine& base() const noexcept;
+};
+
+template<class Engine, size_t w, class UIntType>
+bool
+operator==(
+    const independent_bits_engine<Engine, w, UIntType>& x,
+    const independent_bits_engine<Engine, w, UIntType>& y);
+
+template<class Engine, size_t w, class UIntType>
+bool
+operator!=(
+    const independent_bits_engine<Engine, w, UIntType>& x,
+    const independent_bits_engine<Engine, w, UIntType>& y);
+
+template <class charT, class traits,
+          class Engine, size_t w, class UIntType>
+basic_ostream<charT, traits>&
+operator<<(basic_ostream<charT, traits>& os,
+           const independent_bits_engine<Engine, w, UIntType>& x);
+
+template <class charT, class traits,
+          class Engine, size_t w, class UIntType>
+basic_istream<charT, traits>&
+operator>>(basic_istream<charT, traits>& is,
+           independent_bits_engine<Engine, w, UIntType>& x);
+
+template<class Engine, size_t k>
+class shuffle_order_engine
+{
+public:
+    // types
+    typedef typename Engine::result_type result_type;
+
+    // engine characteristics
+    static constexpr size_t table_size = k;
+    static constexpr result_type min() { return Engine::min; }
+    static constexpr result_type max() { return Engine::max; }
+
+    // constructors and seeding functions
+    shuffle_order_engine();
+    explicit shuffle_order_engine(const Engine& e);
+    explicit shuffle_order_engine(Engine&& e);
+    explicit shuffle_order_engine(result_type s);
+    template<class Sseq> explicit shuffle_order_engine(Sseq& q);
+    void seed();
+    void seed(result_type s);
+    template<class Sseq> void seed(Sseq& q);
+
+    // generating functions
+    result_type operator()();
+    void discard(unsigned long long z);
+
+    // property functions
+    const Engine& base() const noexcept;
+};
+
+template<class Engine, size_t k>
+bool
+operator==(
+    const shuffle_order_engine<Engine, k>& x,
+    const shuffle_order_engine<Engine, k>& y);
+
+template<class Engine, size_t k>
+bool
+operator!=(
+    const shuffle_order_engine<Engine, k>& x,
+    const shuffle_order_engine<Engine, k>& y);
+
+template <class charT, class traits,
+          class Engine, size_t k>
+basic_ostream<charT, traits>&
+operator<<(basic_ostream<charT, traits>& os,
+           const shuffle_order_engine<Engine, k>& x);
+
+template <class charT, class traits,
+          class Engine, size_t k>
+basic_istream<charT, traits>&
+operator>>(basic_istream<charT, traits>& is,
+           shuffle_order_engine<Engine, k>& x);
+
+typedef linear_congruential_engine<uint_fast32_t, 16807, 0, 2147483647>
+                                                                   minstd_rand0;
+typedef linear_congruential_engine<uint_fast32_t, 48271, 0, 2147483647>
+                                                                    minstd_rand;
+typedef mersenne_twister_engine<uint_fast32_t, 32, 624, 397, 31,
+                                0x9908b0df,
+                                11, 0xffffffff,
+                                7,  0x9d2c5680,
+                                15, 0xefc60000,
+                                18, 1812433253>                         mt19937;
+typedef mersenne_twister_engine<uint_fast64_t, 64, 312, 156, 31,
+                                0xb5026f5aa96619e9,
+                                29, 0x5555555555555555,
+                                17, 0x71d67fffeda60000,
+                                37, 0xfff7eee000000000,
+                                43, 6364136223846793005>             mt19937_64;
+typedef subtract_with_carry_engine<uint_fast32_t, 24, 10, 24>     ranlux24_base;
+typedef subtract_with_carry_engine<uint_fast64_t, 48,  5, 12>     ranlux48_base;
+typedef discard_block_engine<ranlux24_base, 223, 23>                   ranlux24;
+typedef discard_block_engine<ranlux48_base, 389, 11>                   ranlux48;
+typedef shuffle_order_engine<minstd_rand0, 256>                         knuth_b;
+typedef minstd_rand                                       default_random_engine;
+
+// Generators
+
+class random_device
+{
+public:
+    // types
+    typedef unsigned int result_type;
+
+    // generator characteristics
+    static constexpr result_type min() { return numeric_limits<result_type>::min(); }
+    static constexpr result_type max() { return numeric_limits<result_type>::max(); }
+
+    // constructors
+    explicit random_device(const string& token = implementation-defined); // before C++20
+    random_device() : random_device(implementation-defined) {}            // C++20
+    explicit random_device(const string& token);                          // C++20
+
+    // generating functions
+    result_type operator()();
+
+    // property functions
+    double entropy() const noexcept;
+
+    // no copy functions
+    random_device(const random_device& ) = delete;
+    void operator=(const random_device& ) = delete;
+};
+
+// Utilities
+
+class seed_seq
+{
+public:
+    // types
+    typedef uint_least32_t result_type;
+
+    // constructors
+    seed_seq();
+    template<class T>
+        seed_seq(initializer_list<T> il);
+    template<class InputIterator>
+        seed_seq(InputIterator begin, InputIterator end);
+
+    // generating functions
+    template<class RandomAccessIterator>
+        void generate(RandomAccessIterator begin, RandomAccessIterator end);
+
+    // property functions
+    size_t size() const;
+    template<class OutputIterator>
+        void param(OutputIterator dest) const;
+
+    // no copy functions
+    seed_seq(const seed_seq&) = delete;
+    void operator=(const seed_seq& ) = delete;
+};
+
+template<class RealType, size_t bits, class URNG>
+    RealType generate_canonical(URNG& g);
+
+// Distributions
+
+template<class IntType = int>
+class uniform_int_distribution
+{
+public:
+    // types
+    typedef IntType result_type;
+
+    class param_type
+    {
+    public:
+        typedef uniform_int_distribution distribution_type;
+
+        explicit param_type(IntType a = 0,
+                                    IntType b = numeric_limits<IntType>::max());
+
+        result_type a() const;
+        result_type b() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructors and reset functions
+    explicit uniform_int_distribution(IntType a = 0,
+                                      IntType b = numeric_limits<IntType>::max()); // before C++20
+    uniform_int_distribution() : uniform_int_distribution(0) {}                    // C++20
+    explicit uniform_int_distribution(IntType a,
+                                      IntType b = numeric_limits<IntType>::max()); // C++20
+    explicit uniform_int_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    result_type a() const;
+    result_type b() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const uniform_int_distribution& x,
+                           const uniform_int_distribution& y);
+    friend bool operator!=(const uniform_int_distribution& x,
+                           const uniform_int_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const uniform_int_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               uniform_int_distribution& x);
+};
+
+template<class RealType = double>
+class uniform_real_distribution
+{
+public:
+    // types
+    typedef RealType result_type;
+
+    class param_type
+    {
+    public:
+        typedef uniform_real_distribution distribution_type;
+
+        explicit param_type(RealType a = 0,
+                            RealType b = 1);
+
+        result_type a() const;
+        result_type b() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructors and reset functions
+    explicit uniform_real_distribution(RealType a = 0.0, RealType b = 1.0); // before C++20
+    uniform_real_distribution() : uniform_real_distribution(0.0) {}         // C++20
+    explicit uniform_real_distribution(RealType a, RealType b = 1.0);       // C++20
+    explicit uniform_real_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    result_type a() const;
+    result_type b() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const uniform_real_distribution& x,
+                           const uniform_real_distribution& y);
+    friend bool operator!=(const uniform_real_distribution& x,
+                           const uniform_real_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const uniform_real_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               uniform_real_distribution& x);
+};
+
+class bernoulli_distribution
+{
+public:
+    // types
+    typedef bool result_type;
+
+    class param_type
+    {
+    public:
+        typedef bernoulli_distribution distribution_type;
+
+        explicit param_type(double p = 0.5);
+
+        double p() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructors and reset functions
+    explicit bernoulli_distribution(double p = 0.5);          // before C++20
+    bernoulli_distribution() : bernoulli_distribution(0.5) {} // C++20
+    explicit bernoulli_distribution(double p);                // C++20
+    explicit bernoulli_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    double p() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const bernoulli_distribution& x,
+                           const bernoulli_distribution& y);
+    friend bool operator!=(const bernoulli_distribution& x,
+                           const bernoulli_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const bernoulli_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               bernoulli_distribution& x);
+};
+
+template<class IntType = int>
+class binomial_distribution
+{
+public:
+    // types
+    typedef IntType result_type;
+
+    class param_type
+    {
+    public:
+        typedef binomial_distribution distribution_type;
+
+        explicit param_type(IntType t = 1, double p = 0.5);
+
+        IntType t() const;
+        double p() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructors and reset functions
+    explicit binomial_distribution(IntType t = 1, double p = 0.5); // before C++20
+    binomial_distribution() : binomial_distribution(1) {}          // C++20
+    explicit binomial_distribution(IntType t, double p = 0.5);     // C++20
+    explicit binomial_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    IntType t() const;
+    double p() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const binomial_distribution& x,
+                           const binomial_distribution& y);
+    friend bool operator!=(const binomial_distribution& x,
+                           const binomial_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const binomial_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               binomial_distribution& x);
+};
+
+template<class IntType = int>
+class geometric_distribution
+{
+public:
+    // types
+    typedef IntType result_type;
+
+    class param_type
+    {
+    public:
+        typedef geometric_distribution distribution_type;
+
+        explicit param_type(double p = 0.5);
+
+        double p() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructors and reset functions
+    explicit geometric_distribution(double p = 0.5);          // before C++20
+    geometric_distribution() : geometric_distribution(0.5) {} // C++20
+    explicit geometric_distribution(double p);                // C++20
+    explicit geometric_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    double p() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const geometric_distribution& x,
+                           const geometric_distribution& y);
+    friend bool operator!=(const geometric_distribution& x,
+                           const geometric_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const geometric_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               geometric_distribution& x);
+};
+
+template<class IntType = int>
+class negative_binomial_distribution
+{
+public:
+    // types
+    typedef IntType result_type;
+
+    class param_type
+    {
+    public:
+        typedef negative_binomial_distribution distribution_type;
+
+        explicit param_type(result_type k = 1, double p = 0.5);
+
+        result_type k() const;
+        double p() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructor and reset functions
+    explicit negative_binomial_distribution(IntType k = 1, double p = 0.5); // before C++20
+    negative_binomial_distribution() : negative_binomial_distribution(1) {} // C++20
+    explicit negative_binomial_distribution(IntType k, double p = 0.5);     // C++20
+    explicit negative_binomial_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    result_type k() const;
+    double p() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const negative_binomial_distribution& x,
+                           const negative_binomial_distribution& y);
+    friend bool operator!=(const negative_binomial_distribution& x,
+                           const negative_binomial_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const negative_binomial_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               negative_binomial_distribution& x);
+};
+
+template<class IntType = int>
+class poisson_distribution
+{
+public:
+    // types
+    typedef IntType result_type;
+
+    class param_type
+    {
+    public:
+        typedef poisson_distribution distribution_type;
+
+        explicit param_type(double mean = 1.0);
+
+        double mean() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructors and reset functions
+    explicit poisson_distribution(double mean = 1.0);     // before C++20
+    poisson_distribution() : poisson_distribution(1.0) {} // C++20
+    explicit poisson_distribution(double mean);           // C++20
+    explicit poisson_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    double mean() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const poisson_distribution& x,
+                           const poisson_distribution& y);
+    friend bool operator!=(const poisson_distribution& x,
+                           const poisson_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const poisson_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               poisson_distribution& x);
+};
+
+template<class RealType = double>
+class exponential_distribution
+{
+public:
+    // types
+    typedef RealType result_type;
+
+    class param_type
+    {
+    public:
+        typedef exponential_distribution distribution_type;
+
+        explicit param_type(result_type lambda = 1.0);
+
+        result_type lambda() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructors and reset functions
+    explicit exponential_distribution(RealType lambda = 1.0);     // before C++20
+    exponential_distribution() : exponential_distribution(1.0) {} // C++20
+    explicit exponential_distribution(RealType lambda);           // C++20
+    explicit exponential_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    result_type lambda() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const exponential_distribution& x,
+                           const exponential_distribution& y);
+    friend bool operator!=(const exponential_distribution& x,
+                           const exponential_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const exponential_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               exponential_distribution& x);
+};
+
+template<class RealType = double>
+class gamma_distribution
+{
+public:
+    // types
+    typedef RealType result_type;
+
+    class param_type
+    {
+    public:
+        typedef gamma_distribution distribution_type;
+
+        explicit param_type(result_type alpha = 1, result_type beta = 1);
+
+        result_type alpha() const;
+        result_type beta() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructors and reset functions
+    explicit gamma_distribution(RealType alpha = 0.0, RealType beta = 1.0); // before C++20
+    gamma_distribution() : gamma_distribution(0.0) {}                       // C++20
+    explicit gamma_distribution(RealType alpha, RealType beta = 1.0);       // C++20
+    explicit gamma_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    result_type alpha() const;
+    result_type beta() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const gamma_distribution& x,
+                           const gamma_distribution& y);
+    friend bool operator!=(const gamma_distribution& x,
+                           const gamma_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const gamma_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               gamma_distribution& x);
+};
+
+template<class RealType = double>
+class weibull_distribution
+{
+public:
+    // types
+    typedef RealType result_type;
+
+    class param_type
+    {
+    public:
+        typedef weibull_distribution distribution_type;
+
+        explicit param_type(result_type alpha = 1, result_type beta = 1);
+
+        result_type a() const;
+        result_type b() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructor and reset functions
+    explicit weibull_distribution(RealType a = 1.0, RealType b = 1.0); // before C++20
+    weibull_distribution() : weibull_distribution(1.0) {}              // C++20
+    explicit weibull_distribution(RealType a, RealType b = 1.0);       // C++20
+    explicit weibull_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    result_type a() const;
+    result_type b() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const weibull_distribution& x,
+                           const weibull_distribution& y);
+    friend bool operator!=(const weibull_distribution& x,
+                           const weibull_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const weibull_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               weibull_distribution& x);
+};
+
+template<class RealType = double>
+class extreme_value_distribution
+{
+public:
+    // types
+    typedef RealType result_type;
+
+    class param_type
+    {
+    public:
+        typedef extreme_value_distribution distribution_type;
+
+        explicit param_type(result_type a = 0, result_type b = 1);
+
+        result_type a() const;
+        result_type b() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructor and reset functions
+    explicit extreme_value_distribution(RealType a = 0.0, RealType b = 1.0); // before C++20
+    extreme_value_distribution() : extreme_value_distribution(0.0) {}        // C++20
+    explicit extreme_value_distribution(RealType a, RealType b = 1.0);       // C++20
+    explicit extreme_value_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    result_type a() const;
+    result_type b() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const extreme_value_distribution& x,
+                           const extreme_value_distribution& y);
+    friend bool operator!=(const extreme_value_distribution& x,
+                           const extreme_value_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const extreme_value_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               extreme_value_distribution& x);
+};
+
+template<class RealType = double>
+class normal_distribution
+{
+public:
+    // types
+    typedef RealType result_type;
+
+    class param_type
+    {
+    public:
+        typedef normal_distribution distribution_type;
+
+        explicit param_type(result_type mean = 0, result_type stddev = 1);
+
+        result_type mean() const;
+        result_type stddev() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructors and reset functions
+    explicit normal_distribution(RealType mean = 0.0, RealType stddev = 1.0); // before C++20
+    normal_distribution() : normal_distribution(0.0) {}                       // C++20
+    explicit normal_distribution(RealType mean, RealType stddev = 1.0);       // C++20
+    explicit normal_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    result_type mean() const;
+    result_type stddev() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const normal_distribution& x,
+                           const normal_distribution& y);
+    friend bool operator!=(const normal_distribution& x,
+                           const normal_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const normal_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               normal_distribution& x);
+};
+
+template<class RealType = double>
+class lognormal_distribution
+{
+public:
+    // types
+    typedef RealType result_type;
+
+    class param_type
+    {
+    public:
+        typedef lognormal_distribution distribution_type;
+
+        explicit param_type(result_type m = 0, result_type s = 1);
+
+        result_type m() const;
+        result_type s() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructor and reset functions
+    explicit lognormal_distribution(RealType mean = 0.0, RealType stddev = 1.0); // before C++20
+    lognormal_distribution() : lognormal_distribution(0.0) {}                    // C++20
+    explicit lognormal_distribution(RealType mean, RealType stddev = 1.0);       // C++20
+    explicit lognormal_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    result_type m() const;
+    result_type s() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const lognormal_distribution& x,
+                           const lognormal_distribution& y);
+    friend bool operator!=(const lognormal_distribution& x,
+                           const lognormal_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const lognormal_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               lognormal_distribution& x);
+};
+
+template<class RealType = double>
+class chi_squared_distribution
+{
+public:
+    // types
+    typedef RealType result_type;
+
+    class param_type
+    {
+    public:
+        typedef chi_squared_distribution distribution_type;
+
+        explicit param_type(result_type n = 1);
+
+        result_type n() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructor and reset functions
+    explicit chi_squared_distribution(RealType n = 1.0);          // before C++20
+    chi_squared_distribution() : chi_squared_distribution(1.0) {} // C++20
+    explicit chi_squared_distribution(RealType n);                // C++20
+    explicit chi_squared_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    result_type n() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const chi_squared_distribution& x,
+                           const chi_squared_distribution& y);
+    friend bool operator!=(const chi_squared_distribution& x,
+                           const chi_squared_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const chi_squared_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               chi_squared_distribution& x);
+};
+
+template<class RealType = double>
+class cauchy_distribution
+{
+public:
+    // types
+    typedef RealType result_type;
+
+    class param_type
+    {
+    public:
+        typedef cauchy_distribution distribution_type;
+
+        explicit param_type(result_type a = 0, result_type b = 1);
+
+        result_type a() const;
+        result_type b() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructor and reset functions
+    explicit cauchy_distribution(RealType a = 0.0, RealType b = 1.0); // before C++20
+    cauchy_distribution() : cauchy_distribution(0.0) {}               // C++20
+    explicit cauchy_distribution(RealType a, RealType b = 1.0);       // C++20
+    explicit cauchy_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    result_type a() const;
+    result_type b() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const cauchy_distribution& x,
+                           const cauchy_distribution& y);
+    friend bool operator!=(const cauchy_distribution& x,
+                           const cauchy_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const cauchy_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               cauchy_distribution& x);
+};
+
+template<class RealType = double>
+class fisher_f_distribution
+{
+public:
+    // types
+    typedef RealType result_type;
+
+    class param_type
+    {
+    public:
+        typedef fisher_f_distribution distribution_type;
+
+        explicit param_type(result_type m = 1, result_type n = 1);
+
+        result_type m() const;
+        result_type n() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructor and reset functions
+    explicit fisher_f_distribution(RealType m = 1.0, RealType n = 1.0); // before C++20
+    fisher_f_distribution() : fisher_f_distribution(1.0) {}             // C++20
+    explicit fisher_f_distribution(RealType m, RealType n = 1.0);       // C++20
+    explicit fisher_f_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    result_type m() const;
+    result_type n() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const fisher_f_distribution& x,
+                           const fisher_f_distribution& y);
+    friend bool operator!=(const fisher_f_distribution& x,
+                           const fisher_f_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const fisher_f_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               fisher_f_distribution& x);
+};
+
+template<class RealType = double>
+class student_t_distribution
+{
+public:
+    // types
+    typedef RealType result_type;
+
+    class param_type
+    {
+    public:
+        typedef student_t_distribution distribution_type;
+
+        explicit param_type(result_type n = 1);
+
+        result_type n() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructor and reset functions
+    explicit student_t_distribution(RealType n = 1.0);        // before C++20
+    student_t_distribution() : student_t_distribution(1.0) {} // C++20
+    explicit student_t_distribution(RealType n);              // C++20
+    explicit student_t_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    result_type n() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const student_t_distribution& x,
+                           const student_t_distribution& y);
+    friend bool operator!=(const student_t_distribution& x,
+                           const student_t_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const student_t_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               student_t_distribution& x);
+};
+
+template<class IntType = int>
+class discrete_distribution
+{
+public:
+    // types
+    typedef IntType result_type;
+
+    class param_type
+    {
+    public:
+        typedef discrete_distribution distribution_type;
+
+        param_type();
+        template<class InputIterator>
+            param_type(InputIterator firstW, InputIterator lastW);
+        param_type(initializer_list<double> wl);
+        template<class UnaryOperation>
+            param_type(size_t nw, double xmin, double xmax, UnaryOperation fw);
+
+        vector<double> probabilities() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructor and reset functions
+    discrete_distribution();
+    template<class InputIterator>
+        discrete_distribution(InputIterator firstW, InputIterator lastW);
+    discrete_distribution(initializer_list<double> wl);
+    template<class UnaryOperation>
+        discrete_distribution(size_t nw, double xmin, double xmax,
+                              UnaryOperation fw);
+    explicit discrete_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    vector<double> probabilities() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const discrete_distribution& x,
+                           const discrete_distribution& y);
+    friend bool operator!=(const discrete_distribution& x,
+                           const discrete_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const discrete_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               discrete_distribution& x);
+};
+
+template<class RealType = double>
+class piecewise_constant_distribution
+{
+    // types
+    typedef RealType result_type;
+
+    class param_type
+    {
+    public:
+        typedef piecewise_constant_distribution distribution_type;
+
+        param_type();
+        template<class InputIteratorB, class InputIteratorW>
+            param_type(InputIteratorB firstB, InputIteratorB lastB,
+                       InputIteratorW firstW);
+        template<class UnaryOperation>
+            param_type(initializer_list<result_type> bl, UnaryOperation fw);
+        template<class UnaryOperation>
+            param_type(size_t nw, result_type xmin, result_type xmax,
+                       UnaryOperation fw);
+
+        vector<result_type> intervals() const;
+        vector<result_type> densities() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructor and reset functions
+    piecewise_constant_distribution();
+    template<class InputIteratorB, class InputIteratorW>
+        piecewise_constant_distribution(InputIteratorB firstB,
+                                        InputIteratorB lastB,
+                                        InputIteratorW firstW);
+    template<class UnaryOperation>
+        piecewise_constant_distribution(initializer_list<result_type> bl,
+                                        UnaryOperation fw);
+    template<class UnaryOperation>
+        piecewise_constant_distribution(size_t nw, result_type xmin,
+                                        result_type xmax, UnaryOperation fw);
+    explicit piecewise_constant_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    vector<result_type> intervals() const;
+    vector<result_type> densities() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const piecewise_constant_distribution& x,
+                           const piecewise_constant_distribution& y);
+    friend bool operator!=(const piecewise_constant_distribution& x,
+                           const piecewise_constant_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const piecewise_constant_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               piecewise_constant_distribution& x);
+};
+
+template<class RealType = double>
+class piecewise_linear_distribution
+{
+    // types
+    typedef RealType result_type;
+
+    class param_type
+    {
+    public:
+        typedef piecewise_linear_distribution distribution_type;
+
+        param_type();
+        template<class InputIteratorB, class InputIteratorW>
+            param_type(InputIteratorB firstB, InputIteratorB lastB,
+                       InputIteratorW firstW);
+        template<class UnaryOperation>
+            param_type(initializer_list<result_type> bl, UnaryOperation fw);
+        template<class UnaryOperation>
+            param_type(size_t nw, result_type xmin, result_type xmax,
+                       UnaryOperation fw);
+
+        vector<result_type> intervals() const;
+        vector<result_type> densities() const;
+
+        friend bool operator==(const param_type& x, const param_type& y);
+        friend bool operator!=(const param_type& x, const param_type& y);
+    };
+
+    // constructor and reset functions
+    piecewise_linear_distribution();
+    template<class InputIteratorB, class InputIteratorW>
+        piecewise_linear_distribution(InputIteratorB firstB,
+                                      InputIteratorB lastB,
+                                      InputIteratorW firstW);
+
+    template<class UnaryOperation>
+        piecewise_linear_distribution(initializer_list<result_type> bl,
+                                      UnaryOperation fw);
+
+    template<class UnaryOperation>
+        piecewise_linear_distribution(size_t nw, result_type xmin,
+                                      result_type xmax, UnaryOperation fw);
+
+    explicit piecewise_linear_distribution(const param_type& parm);
+    void reset();
+
+    // generating functions
+    template<class URNG> result_type operator()(URNG& g);
+    template<class URNG> result_type operator()(URNG& g, const param_type& parm);
+
+    // property functions
+    vector<result_type> intervals() const;
+    vector<result_type> densities() const;
+
+    param_type param() const;
+    void param(const param_type& parm);
+
+    result_type min() const;
+    result_type max() const;
+
+    friend bool operator==(const piecewise_linear_distribution& x,
+                           const piecewise_linear_distribution& y);
+    friend bool operator!=(const piecewise_linear_distribution& x,
+                           const piecewise_linear_distribution& y);
+
+    template <class charT, class traits>
+    friend
+    basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os,
+               const piecewise_linear_distribution& x);
+
+    template <class charT, class traits>
+    friend
+    basic_istream<charT, traits>&
+    operator>>(basic_istream<charT, traits>& is,
+               piecewise_linear_distribution& x);
+};
+
+} // std
+*/
+
+#include <__config>
+#include <__random/bernoulli_distribution.h>
+#include <__random/binomial_distribution.h>
+#include <__random/cauchy_distribution.h>
+#include <__random/chi_squared_distribution.h>
+#include <__random/default_random_engine.h>
+#include <__random/discard_block_engine.h>
+#include <__random/discrete_distribution.h>
+#include <__random/exponential_distribution.h>
+#include <__random/extreme_value_distribution.h>
+#include <__random/fisher_f_distribution.h>
+#include <__random/gamma_distribution.h>
+#include <__random/generate_canonical.h>
+#include <__random/geometric_distribution.h>
+#include <__random/independent_bits_engine.h>
+#include <__random/is_seed_sequence.h>
+#include <__random/knuth_b.h>
+#include <__random/linear_congruential_engine.h>
+#include <__random/lognormal_distribution.h>
+#include <__random/mersenne_twister_engine.h>
+#include <__random/negative_binomial_distribution.h>
+#include <__random/normal_distribution.h>
+#include <__random/piecewise_constant_distribution.h>
+#include <__random/piecewise_linear_distribution.h>
+#include <__random/poisson_distribution.h>
+#include <__random/random_device.h>
+#include <__random/ranlux.h>
+#include <__random/seed_seq.h>
+#include <__random/shuffle_order_engine.h>
+#include <__random/student_t_distribution.h>
+#include <__random/subtract_with_carry_engine.h>
+#include <__random/uniform_int_distribution.h>
+#include <__random/uniform_random_bit_generator.h>
+#include <__random/uniform_real_distribution.h>
+#include <__random/weibull_distribution.h>
+#include <version>
+
+// standard-mandated includes
+
+// [rand.synopsis]
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <algorithm>
+#  include <climits>
+#  include <cmath>
+#  include <concepts>
+#  include <cstddef>
+#  include <cstdint>
+#  include <cstdlib>
+#  include <iosfwd>
+#  include <limits>
+#  include <numeric>
+#  include <string>
+#  include <type_traits>
+#  include <vector>
+#endif
+
+#endif // _LIBCPP_RANDOM
diff --git a/libcxx/include/__cxx03/ranges b/libcxx/include/__cxx03/ranges
new file mode 100644
index 000000000000..fa35874265de
--- /dev/null
+++ b/libcxx/include/__cxx03/ranges
@@ -0,0 +1,463 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_RANGES
+#define _LIBCPP_RANGES
+
+/*
+
+#include <compare>              // see [compare.syn]
+#include <initializer_list>     // see [initializer.list.syn]
+#include <iterator>             // see [iterator.synopsis]
+
+namespace std::ranges {
+  inline namespace unspecified {
+    // [range.access], range access
+    inline constexpr unspecified begin = unspecified;
+    inline constexpr unspecified end = unspecified;
+    inline constexpr unspecified cbegin = unspecified;
+    inline constexpr unspecified cend = unspecified;
+
+    inline constexpr unspecified size = unspecified;
+    inline constexpr unspecified ssize = unspecified;
+  }
+
+  // [range.range], ranges
+  template<class T>
+    concept range = see below;
+
+  template<class T>
+    inline constexpr bool enable_borrowed_range = false;
+
+  template<class T>
+    using iterator_t = decltype(ranges::begin(declval<T&>()));
+  template<range R>
+    using sentinel_t = decltype(ranges::end(declval<R&>()));
+  template<range R>
+    using range_difference_t = iter_difference_t<iterator_t<R>>;
+  template<sized_range R>
+    using range_size_t = decltype(ranges::size(declval<R&>()));
+  template<range R>
+    using range_value_t = iter_value_t<iterator_t<R>>;
+  template<range R>
+    using range_reference_t = iter_reference_t<iterator_t<R>>;
+  template<range R>
+    using range_rvalue_reference_t = iter_rvalue_reference_t<iterator_t<R>>;
+  template <range R>
+    using range_common_reference_t = iter_common_reference_t<iterator_t<R>>;
+
+  // [range.sized], sized ranges
+  template<class>
+    inline constexpr bool disable_sized_range = false;
+
+  template<class T>
+    concept sized_range = ...;
+
+  // [range.view], views
+  template<class T>
+    inline constexpr bool enable_view = ...;
+
+  struct view_base { };
+
+  template<class T>
+    concept view = ...;
+
+  // [range.refinements], other range refinements
+  template<class R, class T>
+    concept output_range = see below;
+
+  template<class T>
+    concept input_range = see below;
+
+  template<class T>
+    concept forward_range = see below;
+
+  template<class T>
+  concept bidirectional_range = see below;
+
+  template<class T>
+  concept random_access_range = see below;
+
+  template<class T>
+  concept contiguous_range = see below;
+
+  template <class _Tp>
+    concept common_range = see below;
+
+  template<class T>
+  concept viewable_range = see below;
+
+  // [range.adaptor.object], range adaptor objects
+  template<class D>
+    requires is_class_v<D> && same_as<D, remove_cv_t<D>>
+  class range_adaptor_closure { };  // Since c++23
+
+  // [view.interface], class template view_interface
+  template<class D>
+    requires is_class_v<D> && same_as<D, remove_cv_t<D>>
+  class view_interface;
+
+  // [range.subrange], sub-ranges
+  enum class subrange_kind : bool { unsized, sized };
+
+  template<input_or_output_iterator I, sentinel_for<I> S = I, subrange_kind K = see below>
+    requires (K == subrange_kind::sized || !sized_sentinel_for<S, I>)
+  class subrange;
+
+  template<class I, class S, subrange_kind K>
+    inline constexpr bool enable_borrowed_range<subrange<I, S, K>> = true;
+
+  // [range.dangling], dangling iterator handling
+  struct dangling;
+
+  template<range R>
+    using borrowed_iterator_t = see below;
+
+  template<range R>
+    using borrowed_subrange_t = see below;
+
+  // [range.elements], elements view
+  template<input_range V, size_t N>
+    requires see below
+  class elements_view;
+
+  template<class T, size_t N>
+    inline constexpr bool enable_borrowed_range<elements_view<T, N>> =
+      enable_borrowed_range<T>;
+
+  template<class R>
+    using keys_view = elements_view<R, 0>;
+  template<class R>
+    using values_view = elements_view<R, 1>;
+
+  namespace views {
+    template<size_t N>
+      inline constexpr unspecified elements = unspecified;
+    inline constexpr auto keys = elements<0>;
+    inline constexpr auto values = elements<1>;
+  }
+
+  // [range.utility.conv], range conversions
+  template<class C, input_range R, class... Args> requires (!view<C>)
+    constexpr C to(R&& r, Args&&... args);     // Since C++23
+  template<template<class...> class C, input_range R, class... Args>
+    constexpr auto to(R&& r, Args&&... args);  // Since C++23
+  template<class C, class... Args> requires (!view<C>)
+    constexpr auto to(Args&&... args);         // Since C++23
+  template<template<class...> class C, class... Args>
+    constexpr auto to(Args&&... args);         // Since C++23
+
+  // [range.empty], empty view
+  template<class T>
+    requires is_object_v<T>
+  class empty_view;
+
+  template<class T>
+    inline constexpr bool enable_borrowed_range<empty_view<T>> = true;
+
+  namespace views {
+    template<class T>
+      inline constexpr empty_view<T> empty{};
+  }
+
+  // [range.all], all view
+  namespace views {
+    inline constexpr unspecified all = unspecified;
+
+    template<viewable_range R>
+      using all_t = decltype(all(declval<R>()));
+  }
+
+  template<range R>
+    requires is_object_v<R>
+  class ref_view;
+
+  template<class T>
+    inline constexpr bool enable_borrowed_range<ref_view<T>> = true;
+
+  template<range R>
+    requires see below
+  class owning_view;
+
+  template<class T>
+    inline constexpr bool enable_borrowed_range<owning_view<T>> = enable_borrowed_range<T>;
+
+  // [range.filter], filter view
+  template<input_range V, indirect_unary_predicate<iterator_t<V>> Pred>
+    requires view<V> && is_object_v<Pred>
+  class filter_view;
+
+  namespace views {
+    inline constexpr unspecified filter = unspecified;
+  }
+
+  // [range.drop], drop view
+  template<view V>
+    class drop_view;
+
+  template<class T>
+    inline constexpr bool enable_borrowed_range<drop_view<T>> = enable_borrowed_range<T>;
+
+  // [range.drop.while], drop while view
+  template<view V, class Pred>
+    requires input_range<V> && is_object_v<Pred> &&
+             indirect_unary_predicate<const Pred, iterator_t<V>>
+    class drop_while_view;
+
+  template<class T, class Pred>
+    inline constexpr bool enable_borrowed_range<drop_while_view<T, Pred>> =
+      enable_borrowed_range<T>;
+
+  namespace views { inline constexpr unspecified drop_while = unspecified; }
+
+  // [range.transform], transform view
+  template<input_range V, copy_constructible F>
+    requires view<V> && is_object_v<F> &&
+             regular_invocable<F&, range_reference_t<V>> &&
+             can-reference<invoke_result_t<F&, range_reference_t<V>>>
+  class transform_view;
+
+  // [range.counted], counted view
+  namespace views { inline constexpr unspecified counted = unspecified; }
+
+  // [range.common], common view
+  template<view V>
+    requires (!common_range<V> && copyable<iterator_t<V>>)
+  class common_view;
+
+ // [range.reverse], reverse view
+  template<view V>
+    requires bidirectional_range<V>
+  class reverse_view;
+
+  template<class T>
+    inline constexpr bool enable_borrowed_range<reverse_view<T>> = enable_borrowed_range<T>;
+
+  template<class T>
+  inline constexpr bool enable_borrowed_range<common_view<T>> = enable_borrowed_range<T>;
+
+   // [range.take], take view
+  template<view> class take_view;
+
+  template<class T>
+  inline constexpr bool enable_borrowed_range<take_view<T>> = enable_borrowed_range<T>;
+
+    // [range.take.while], take while view
+  template<view V, class Pred>
+    requires input_range<V> && is_object_v<Pred> &&
+             indirect_unary_predicate<const Pred, iterator_t<V>>
+    class take_while_view;
+
+  namespace views { inline constexpr unspecified take_while = unspecified; }
+
+  template<copy_constructible T>
+    requires is_object_v<T>
+  class single_view;
+
+  template<weakly_incrementable W, semiregular Bound = unreachable_sentinel_t>
+    requires weakly-equality-comparable-with<W, Bound> && copyable<W>
+  class iota_view;
+
+  template<class W, class Bound>
+    inline constexpr bool enable_borrowed_range<iota_view<W, Bound>> = true;
+
+  // [range.repeat], repeat view
+  template<class T>
+    concept integer-like-with-usable-difference-type =  // exposition only
+      is-signed-integer-like<T> || (is-integer-like<T> && weakly_incrementable<T>);
+
+  template<move_constructible T, semiregular Bound = unreachable_sentinel_t>
+    requires (is_object_v<T> && same_as<T, remove_cv_t<T>> &&
+              (integer-like-with-usable-difference-type<Bound> ||
+               same_as<Bound, unreachable_sentinel_t>))
+  class repeat_view;
+
+  namespace views { inline constexpr unspecified repeat = unspecified; }
+
+  // [range.join], join view
+  template<input_range V>
+    requires view<V> && input_range<range_reference_t<V>>
+  class join_view;
+
+  // [range.lazy.split], lazy split view
+  template<class R>
+    concept tiny-range = see below;   // exposition only
+
+  template<input_range V, forward_range Pattern>
+    requires view<V> && view<Pattern> &&
+             indirectly_comparable<iterator_t<V>, iterator_t<Pattern>, ranges::equal_to> &&
+             (forward_range<V> || tiny-range<Pattern>)
+  class lazy_split_view;
+
+  // [range.split], split view
+  template<forward_range V, forward_range Pattern>
+    requires view<V> && view<Pattern> &&
+             indirectly_comparable<iterator_t<V>, iterator_t<Pattern>, ranges::equal_to>
+  class split_view;
+
+  namespace views {
+    inline constexpr unspecified lazy_split = unspecified;
+    inline constexpr unspecified split = unspecified;
+  }
+
+  // [range.istream], istream view
+  template<movable Val, class CharT, class Traits = char_traits<CharT>>
+    requires see below
+  class basic_istream_view;
+
+  template<class Val>
+    using istream_view = basic_istream_view<Val, char>;
+
+  template<class Val>
+    using wistream_view = basic_istream_view<Val, wchar_t>;
+
+  namespace views { template<class T> inline constexpr unspecified istream = unspecified; }
+
+  // [range.zip], zip view
+  template<input_range... Views>
+    requires (view<Views> && ...) && (sizeof...(Views) > 0)
+  class zip_view;        // C++23
+
+  template<class... Views>
+    inline constexpr bool enable_borrowed_range<zip_view<Views...>> =       // C++23
+      (enable_borrowed_range<Views> && ...);
+
+  namespace views { inline constexpr unspecified zip = unspecified; }       // C++23
+
+  // [range.as.rvalue]
+  template <view V>
+    requires input_range<V>
+  class as_rvalue_view;                                                     // C++23
+
+  namespace views { inline constexpr unspecified as_rvalue ) unspecified; } // C++23
+
+  [range.chunk.by]
+  template<forward_range V, indirect_binary_predicate<iterator_t<V>, iterator_t<V>> Pred>
+    requires view<V> && is_object_v<Pred>
+  class chunk_by_view;                                                      // C++23
+
+  namespace views { inline constexpr unspecified chunk_by = unspecified; }  // C++23
+}
+
+namespace std {
+  namespace views = ranges::views;
+
+  template<class T> struct tuple_size;
+  template<size_t I, class T> struct tuple_element;
+
+  template<class I, class S, ranges::subrange_kind K>
+  struct tuple_size<ranges::subrange<I, S, K>>
+    : integral_constant<size_t, 2> {};
+
+  template<class I, class S, ranges::subrange_kind K>
+  struct tuple_element<0, ranges::subrange<I, S, K>> {
+    using type = I;
+  };
+
+  template<class I, class S, ranges::subrange_kind K>
+  struct tuple_element<1, ranges::subrange<I, S, K>> {
+    using type = S;
+  };
+
+  template<class I, class S, ranges::subrange_kind K>
+  struct tuple_element<0, const ranges::subrange<I, S, K>> {
+    using type = I;
+  };
+
+  template<class I, class S, ranges::subrange_kind K>
+  struct tuple_element<1, const ranges::subrange<I, S, K>> {
+    using type = S;
+  };
+
+  struct from_range_t { explicit from_range_t() = default; };  // Since C++23
+  inline constexpr from_range_t from_range{};                  // Since C++23
+}
+*/
+
+#include <__config>
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__ranges/access.h>
+#  include <__ranges/all.h>
+#  include <__ranges/common_view.h>
+#  include <__ranges/concepts.h>
+#  include <__ranges/counted.h>
+#  include <__ranges/dangling.h>
+#  include <__ranges/data.h>
+#  include <__ranges/drop_view.h>
+#  include <__ranges/drop_while_view.h>
+#  include <__ranges/elements_view.h>
+#  include <__ranges/empty.h>
+#  include <__ranges/empty_view.h>
+#  include <__ranges/enable_borrowed_range.h>
+#  include <__ranges/enable_view.h>
+#  include <__ranges/filter_view.h>
+#  include <__ranges/iota_view.h>
+#  include <__ranges/join_view.h>
+#  include <__ranges/lazy_split_view.h>
+#  include <__ranges/rbegin.h>
+#  include <__ranges/ref_view.h>
+#  include <__ranges/rend.h>
+#  include <__ranges/reverse_view.h>
+#  include <__ranges/single_view.h>
+#  include <__ranges/size.h>
+#  include <__ranges/split_view.h>
+#  include <__ranges/subrange.h>
+#  include <__ranges/take_view.h>
+#  include <__ranges/take_while_view.h>
+#  include <__ranges/transform_view.h>
+#  include <__ranges/view_interface.h>
+#  include <__ranges/views.h>
+
+#  if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#    include <__ranges/istream_view.h>
+#  endif
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+#  include <__ranges/as_rvalue_view.h>
+#  include <__ranges/chunk_by_view.h>
+#  include <__ranges/from_range.h>
+#  include <__ranges/repeat_view.h>
+#  include <__ranges/to.h>
+#  include <__ranges/zip_view.h>
+#endif
+
+#include <version>
+
+// standard-mandated includes
+
+// [ranges.syn]
+#include <compare>
+#include <initializer_list>
+#include <iterator>
+
+// [tuple.helper]
+#include <__tuple/tuple_element.h>
+#include <__tuple/tuple_size.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#  include <cstddef>
+#  include <limits>
+#  include <optional>
+#  include <span>
+#  include <tuple>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstdlib>
+#  include <iosfwd>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_RANGES
diff --git a/libcxx/include/__cxx03/ratio b/libcxx/include/__cxx03/ratio
new file mode 100644
index 000000000000..b989c272aaee
--- /dev/null
+++ b/libcxx/include/__cxx03/ratio
@@ -0,0 +1,517 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_RATIO
+#define _LIBCPP_RATIO
+
+/*
+    ratio synopsis
+
+namespace std
+{
+
+template <intmax_t N, intmax_t D = 1>
+class ratio
+{
+public:
+    static constexpr intmax_t num;
+    static constexpr intmax_t den;
+    typedef ratio<num, den> type;
+};
+
+// ratio arithmetic
+template <class R1, class R2> using ratio_add = ...;
+template <class R1, class R2> using ratio_subtract = ...;
+template <class R1, class R2> using ratio_multiply = ...;
+template <class R1, class R2> using ratio_divide = ...;
+
+// ratio comparison
+template <class R1, class R2> struct ratio_equal;
+template <class R1, class R2> struct ratio_not_equal;
+template <class R1, class R2> struct ratio_less;
+template <class R1, class R2> struct ratio_less_equal;
+template <class R1, class R2> struct ratio_greater;
+template <class R1, class R2> struct ratio_greater_equal;
+
+// convenience SI typedefs
+using quecto = ratio <1, 1'000'000'000'000'000'000'000'000'000'000>; // Since C++26; not supported
+using ronto = ratio <1, 1'000'000'000'000'000'000'000'000'000>;      // Since C++26; not supported
+typedef ratio<1, 1000000000000000000000000> yocto;  // not supported
+typedef ratio<1,    1000000000000000000000> zepto;  // not supported
+typedef ratio<1,       1000000000000000000> atto;
+typedef ratio<1,          1000000000000000> femto;
+typedef ratio<1,             1000000000000> pico;
+typedef ratio<1,                1000000000> nano;
+typedef ratio<1,                   1000000> micro;
+typedef ratio<1,                      1000> milli;
+typedef ratio<1,                       100> centi;
+typedef ratio<1,                        10> deci;
+typedef ratio<                       10, 1> deca;
+typedef ratio<                      100, 1> hecto;
+typedef ratio<                     1000, 1> kilo;
+typedef ratio<                  1000000, 1> mega;
+typedef ratio<               1000000000, 1> giga;
+typedef ratio<            1000000000000, 1> tera;
+typedef ratio<         1000000000000000, 1> peta;
+typedef ratio<      1000000000000000000, 1> exa;
+typedef ratio<   1000000000000000000000, 1> zetta;  // not supported
+typedef ratio<1000000000000000000000000, 1> yotta;  // not supported
+using ronna = ratio <1'000'000'000'000'000'000'000'000'000, 1>;      // Since C++26; not supported
+using quetta = ratio <1'000'000'000'000'000'000'000'000'000'000, 1>; // Since C++26; not supported
+
+  // 20.11.5, ratio comparison
+  template <class R1, class R2> inline constexpr bool ratio_equal_v
+    = ratio_equal<R1, R2>::value;                                       // C++17
+  template <class R1, class R2> inline constexpr bool ratio_not_equal_v
+    = ratio_not_equal<R1, R2>::value;                                   // C++17
+  template <class R1, class R2> inline constexpr bool ratio_less_v
+    = ratio_less<R1, R2>::value;                                        // C++17
+  template <class R1, class R2> inline constexpr bool ratio_less_equal_v
+    = ratio_less_equal<R1, R2>::value;                                  // C++17
+  template <class R1, class R2> inline constexpr bool ratio_greater_v
+    = ratio_greater<R1, R2>::value;                                     // C++17
+  template <class R1, class R2> inline constexpr bool ratio_greater_equal_v
+    = ratio_greater_equal<R1, R2>::value;                               // C++17
+}
+*/
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <climits>
+#include <cstdint>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// __static_gcd
+
+template <intmax_t _Xp, intmax_t _Yp>
+struct __static_gcd {
+  static const intmax_t value = __static_gcd<_Yp, _Xp % _Yp>::value;
+};
+
+template <intmax_t _Xp>
+struct __static_gcd<_Xp, 0> {
+  static const intmax_t value = _Xp;
+};
+
+template <>
+struct __static_gcd<0, 0> {
+  static const intmax_t value = 1;
+};
+
+// __static_lcm
+
+template <intmax_t _Xp, intmax_t _Yp>
+struct __static_lcm {
+  static const intmax_t value = _Xp / __static_gcd<_Xp, _Yp>::value * _Yp;
+};
+
+template <intmax_t _Xp>
+struct __static_abs {
+  static const intmax_t value = _Xp < 0 ? -_Xp : _Xp;
+};
+
+template <intmax_t _Xp>
+struct __static_sign {
+  static const intmax_t value = _Xp == 0 ? 0 : (_Xp < 0 ? -1 : 1);
+};
+
+template <intmax_t _Xp, intmax_t _Yp, intmax_t = __static_sign<_Yp>::value>
+class __ll_add;
+
+template <intmax_t _Xp, intmax_t _Yp>
+class __ll_add<_Xp, _Yp, 1> {
+  static const intmax_t min = (1LL << (sizeof(intmax_t) * CHAR_BIT - 1)) + 1;
+  static const intmax_t max = -min;
+
+  static_assert(_Xp <= max - _Yp, "overflow in __ll_add");
+
+public:
+  static const intmax_t value = _Xp + _Yp;
+};
+
+template <intmax_t _Xp, intmax_t _Yp>
+class __ll_add<_Xp, _Yp, 0> {
+public:
+  static const intmax_t value = _Xp;
+};
+
+template <intmax_t _Xp, intmax_t _Yp>
+class __ll_add<_Xp, _Yp, -1> {
+  static const intmax_t min = (1LL << (sizeof(intmax_t) * CHAR_BIT - 1)) + 1;
+  static const intmax_t max = -min;
+
+  static_assert(min - _Yp <= _Xp, "overflow in __ll_add");
+
+public:
+  static const intmax_t value = _Xp + _Yp;
+};
+
+template <intmax_t _Xp, intmax_t _Yp, intmax_t = __static_sign<_Yp>::value>
+class __ll_sub;
+
+template <intmax_t _Xp, intmax_t _Yp>
+class __ll_sub<_Xp, _Yp, 1> {
+  static const intmax_t min = (1LL << (sizeof(intmax_t) * CHAR_BIT - 1)) + 1;
+  static const intmax_t max = -min;
+
+  static_assert(min + _Yp <= _Xp, "overflow in __ll_sub");
+
+public:
+  static const intmax_t value = _Xp - _Yp;
+};
+
+template <intmax_t _Xp, intmax_t _Yp>
+class __ll_sub<_Xp, _Yp, 0> {
+public:
+  static const intmax_t value = _Xp;
+};
+
+template <intmax_t _Xp, intmax_t _Yp>
+class __ll_sub<_Xp, _Yp, -1> {
+  static const intmax_t min = (1LL << (sizeof(intmax_t) * CHAR_BIT - 1)) + 1;
+  static const intmax_t max = -min;
+
+  static_assert(_Xp <= max + _Yp, "overflow in __ll_sub");
+
+public:
+  static const intmax_t value = _Xp - _Yp;
+};
+
+template <intmax_t _Xp, intmax_t _Yp>
+class __ll_mul {
+  static const intmax_t nan   = (1LL << (sizeof(intmax_t) * CHAR_BIT - 1));
+  static const intmax_t min   = nan + 1;
+  static const intmax_t max   = -min;
+  static const intmax_t __a_x = __static_abs<_Xp>::value;
+  static const intmax_t __a_y = __static_abs<_Yp>::value;
+
+  static_assert(_Xp != nan && _Yp != nan && __a_x <= max / __a_y, "overflow in __ll_mul");
+
+public:
+  static const intmax_t value = _Xp * _Yp;
+};
+
+template <intmax_t _Yp>
+class __ll_mul<0, _Yp> {
+public:
+  static const intmax_t value = 0;
+};
+
+template <intmax_t _Xp>
+class __ll_mul<_Xp, 0> {
+public:
+  static const intmax_t value = 0;
+};
+
+template <>
+class __ll_mul<0, 0> {
+public:
+  static const intmax_t value = 0;
+};
+
+// Not actually used but left here in case needed in future maintenance
+template <intmax_t _Xp, intmax_t _Yp>
+class __ll_div {
+  static const intmax_t nan = (1LL << (sizeof(intmax_t) * CHAR_BIT - 1));
+  static const intmax_t min = nan + 1;
+  static const intmax_t max = -min;
+
+  static_assert(_Xp != nan && _Yp != nan && _Yp != 0, "overflow in __ll_div");
+
+public:
+  static const intmax_t value = _Xp / _Yp;
+};
+
+template <intmax_t _Num, intmax_t _Den = 1>
+class _LIBCPP_TEMPLATE_VIS ratio {
+  static_assert(__static_abs<_Num>::value >= 0, "ratio numerator is out of range");
+  static_assert(_Den != 0, "ratio divide by 0");
+  static_assert(__static_abs<_Den>::value > 0, "ratio denominator is out of range");
+  static _LIBCPP_CONSTEXPR const intmax_t __na  = __static_abs<_Num>::value;
+  static _LIBCPP_CONSTEXPR const intmax_t __da  = __static_abs<_Den>::value;
+  static _LIBCPP_CONSTEXPR const intmax_t __s   = __static_sign<_Num>::value * __static_sign<_Den>::value;
+  static _LIBCPP_CONSTEXPR const intmax_t __gcd = __static_gcd<__na, __da>::value;
+
+public:
+  static _LIBCPP_CONSTEXPR const intmax_t num = __s * __na / __gcd;
+  static _LIBCPP_CONSTEXPR const intmax_t den = __da / __gcd;
+
+  typedef ratio<num, den> type;
+};
+
+template <intmax_t _Num, intmax_t _Den>
+_LIBCPP_CONSTEXPR const intmax_t ratio<_Num, _Den>::num;
+
+template <intmax_t _Num, intmax_t _Den>
+_LIBCPP_CONSTEXPR const intmax_t ratio<_Num, _Den>::den;
+
+template <class _Tp>
+struct __is_ratio : false_type {};
+template <intmax_t _Num, intmax_t _Den>
+struct __is_ratio<ratio<_Num, _Den> > : true_type {};
+
+typedef ratio<1LL, 1000000000000000000LL> atto;
+typedef ratio<1LL, 1000000000000000LL> femto;
+typedef ratio<1LL, 1000000000000LL> pico;
+typedef ratio<1LL, 1000000000LL> nano;
+typedef ratio<1LL, 1000000LL> micro;
+typedef ratio<1LL, 1000LL> milli;
+typedef ratio<1LL, 100LL> centi;
+typedef ratio<1LL, 10LL> deci;
+typedef ratio< 10LL, 1LL> deca;
+typedef ratio< 100LL, 1LL> hecto;
+typedef ratio< 1000LL, 1LL> kilo;
+typedef ratio< 1000000LL, 1LL> mega;
+typedef ratio< 1000000000LL, 1LL> giga;
+typedef ratio< 1000000000000LL, 1LL> tera;
+typedef ratio< 1000000000000000LL, 1LL> peta;
+typedef ratio<1000000000000000000LL, 1LL> exa;
+
+template <class _R1, class _R2>
+struct __ratio_multiply {
+private:
+  static const intmax_t __gcd_n1_d2 = __static_gcd<_R1::num, _R2::den>::value;
+  static const intmax_t __gcd_d1_n2 = __static_gcd<_R1::den, _R2::num>::value;
+
+  static_assert(__is_ratio<_R1>::value, "[ratio.general]/2 requires R1 to be a specialisation of the ratio template");
+  static_assert(__is_ratio<_R2>::value, "[ratio.general]/2 requires R2 to be a specialisation of the ratio template");
+
+public:
+  typedef typename ratio< __ll_mul<_R1::num / __gcd_n1_d2, _R2::num / __gcd_d1_n2>::value,
+                          __ll_mul<_R2::den / __gcd_n1_d2, _R1::den / __gcd_d1_n2>::value >::type type;
+};
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _R1, class _R2>
+using ratio_multiply = typename __ratio_multiply<_R1, _R2>::type;
+
+#else // _LIBCPP_CXX03_LANG
+
+template <class _R1, class _R2>
+struct _LIBCPP_TEMPLATE_VIS ratio_multiply : public __ratio_multiply<_R1, _R2>::type {};
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _R1, class _R2>
+struct __ratio_divide {
+private:
+  static const intmax_t __gcd_n1_n2 = __static_gcd<_R1::num, _R2::num>::value;
+  static const intmax_t __gcd_d1_d2 = __static_gcd<_R1::den, _R2::den>::value;
+
+  static_assert(__is_ratio<_R1>::value, "[ratio.general]/2 requires R1 to be a specialisation of the ratio template");
+  static_assert(__is_ratio<_R2>::value, "[ratio.general]/2 requires R2 to be a specialisation of the ratio template");
+
+public:
+  typedef typename ratio< __ll_mul<_R1::num / __gcd_n1_n2, _R2::den / __gcd_d1_d2>::value,
+                          __ll_mul<_R2::num / __gcd_n1_n2, _R1::den / __gcd_d1_d2>::value >::type type;
+};
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _R1, class _R2>
+using ratio_divide = typename __ratio_divide<_R1, _R2>::type;
+
+#else // _LIBCPP_CXX03_LANG
+
+template <class _R1, class _R2>
+struct _LIBCPP_TEMPLATE_VIS ratio_divide : public __ratio_divide<_R1, _R2>::type {};
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _R1, class _R2>
+struct __ratio_add {
+private:
+  static const intmax_t __gcd_n1_n2 = __static_gcd<_R1::num, _R2::num>::value;
+  static const intmax_t __gcd_d1_d2 = __static_gcd<_R1::den, _R2::den>::value;
+
+  static_assert(__is_ratio<_R1>::value, "[ratio.general]/2 requires R1 to be a specialisation of the ratio template");
+  static_assert(__is_ratio<_R2>::value, "[ratio.general]/2 requires R2 to be a specialisation of the ratio template");
+
+public:
+  typedef typename ratio_multiply<
+      ratio<__gcd_n1_n2, _R1::den / __gcd_d1_d2>,
+      ratio< __ll_add< __ll_mul<_R1::num / __gcd_n1_n2, _R2::den / __gcd_d1_d2>::value,
+                       __ll_mul<_R2::num / __gcd_n1_n2, _R1::den / __gcd_d1_d2>::value >::value,
+             _R2::den > >::type type;
+};
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _R1, class _R2>
+using ratio_add = typename __ratio_add<_R1, _R2>::type;
+
+#else // _LIBCPP_CXX03_LANG
+
+template <class _R1, class _R2>
+struct _LIBCPP_TEMPLATE_VIS ratio_add : public __ratio_add<_R1, _R2>::type {};
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _R1, class _R2>
+struct __ratio_subtract {
+private:
+  static const intmax_t __gcd_n1_n2 = __static_gcd<_R1::num, _R2::num>::value;
+  static const intmax_t __gcd_d1_d2 = __static_gcd<_R1::den, _R2::den>::value;
+
+  static_assert(__is_ratio<_R1>::value, "[ratio.general]/2 requires R1 to be a specialisation of the ratio template");
+  static_assert(__is_ratio<_R2>::value, "[ratio.general]/2 requires R2 to be a specialisation of the ratio template");
+
+public:
+  typedef typename ratio_multiply<
+      ratio<__gcd_n1_n2, _R1::den / __gcd_d1_d2>,
+      ratio< __ll_sub< __ll_mul<_R1::num / __gcd_n1_n2, _R2::den / __gcd_d1_d2>::value,
+                       __ll_mul<_R2::num / __gcd_n1_n2, _R1::den / __gcd_d1_d2>::value >::value,
+             _R2::den > >::type type;
+};
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _R1, class _R2>
+using ratio_subtract = typename __ratio_subtract<_R1, _R2>::type;
+
+#else // _LIBCPP_CXX03_LANG
+
+template <class _R1, class _R2>
+struct _LIBCPP_TEMPLATE_VIS ratio_subtract : public __ratio_subtract<_R1, _R2>::type {};
+
+#endif // _LIBCPP_CXX03_LANG
+
+// ratio_equal
+
+template <class _R1, class _R2>
+struct _LIBCPP_TEMPLATE_VIS ratio_equal : _BoolConstant<(_R1::num == _R2::num && _R1::den == _R2::den)> {
+  static_assert(__is_ratio<_R1>::value, "[ratio.general]/2 requires R1 to be a specialisation of the ratio template");
+  static_assert(__is_ratio<_R2>::value, "[ratio.general]/2 requires R2 to be a specialisation of the ratio template");
+};
+
+template <class _R1, class _R2>
+struct _LIBCPP_TEMPLATE_VIS ratio_not_equal : _BoolConstant<!ratio_equal<_R1, _R2>::value> {
+  static_assert(__is_ratio<_R1>::value, "[ratio.general]/2 requires R1 to be a specialisation of the ratio template");
+  static_assert(__is_ratio<_R2>::value, "[ratio.general]/2 requires R2 to be a specialisation of the ratio template");
+};
+
+// ratio_less
+
+template <class _R1,
+          class _R2,
+          bool _Odd    = false,
+          intmax_t _Q1 = _R1::num / _R1::den,
+          intmax_t _M1 = _R1::num % _R1::den,
+          intmax_t _Q2 = _R2::num / _R2::den,
+          intmax_t _M2 = _R2::num % _R2::den>
+struct __ratio_less1 {
+  static const bool value = _Odd ? _Q2 < _Q1 : _Q1 < _Q2;
+};
+
+template <class _R1, class _R2, bool _Odd, intmax_t _Qp>
+struct __ratio_less1<_R1, _R2, _Odd, _Qp, 0, _Qp, 0> {
+  static const bool value = false;
+};
+
+template <class _R1, class _R2, bool _Odd, intmax_t _Qp, intmax_t _M2>
+struct __ratio_less1<_R1, _R2, _Odd, _Qp, 0, _Qp, _M2> {
+  static const bool value = !_Odd;
+};
+
+template <class _R1, class _R2, bool _Odd, intmax_t _Qp, intmax_t _M1>
+struct __ratio_less1<_R1, _R2, _Odd, _Qp, _M1, _Qp, 0> {
+  static const bool value = _Odd;
+};
+
+template <class _R1, class _R2, bool _Odd, intmax_t _Qp, intmax_t _M1, intmax_t _M2>
+struct __ratio_less1<_R1, _R2, _Odd, _Qp, _M1, _Qp, _M2> {
+  static const bool value = __ratio_less1<ratio<_R1::den, _M1>, ratio<_R2::den, _M2>, !_Odd>::value;
+};
+
+template <class _R1,
+          class _R2,
+          intmax_t _S1 = __static_sign<_R1::num>::value,
+          intmax_t _S2 = __static_sign<_R2::num>::value>
+struct __ratio_less {
+  static const bool value = _S1 < _S2;
+};
+
+template <class _R1, class _R2>
+struct __ratio_less<_R1, _R2, 1LL, 1LL> {
+  static const bool value = __ratio_less1<_R1, _R2>::value;
+};
+
+template <class _R1, class _R2>
+struct __ratio_less<_R1, _R2, -1LL, -1LL> {
+  static const bool value = __ratio_less1<ratio<-_R2::num, _R2::den>, ratio<-_R1::num, _R1::den> >::value;
+};
+
+template <class _R1, class _R2>
+struct _LIBCPP_TEMPLATE_VIS ratio_less : _BoolConstant<__ratio_less<_R1, _R2>::value> {
+  static_assert(__is_ratio<_R1>::value, "[ratio.general]/2 requires R1 to be a specialisation of the ratio template");
+  static_assert(__is_ratio<_R2>::value, "[ratio.general]/2 requires R2 to be a specialisation of the ratio template");
+};
+
+template <class _R1, class _R2>
+struct _LIBCPP_TEMPLATE_VIS ratio_less_equal : _BoolConstant<!ratio_less<_R2, _R1>::value> {
+  static_assert(__is_ratio<_R1>::value, "[ratio.general]/2 requires R1 to be a specialisation of the ratio template");
+  static_assert(__is_ratio<_R2>::value, "[ratio.general]/2 requires R2 to be a specialisation of the ratio template");
+};
+
+template <class _R1, class _R2>
+struct _LIBCPP_TEMPLATE_VIS ratio_greater : _BoolConstant<ratio_less<_R2, _R1>::value> {
+  static_assert(__is_ratio<_R1>::value, "[ratio.general]/2 requires R1 to be a specialisation of the ratio template");
+  static_assert(__is_ratio<_R2>::value, "[ratio.general]/2 requires R2 to be a specialisation of the ratio template");
+};
+
+template <class _R1, class _R2>
+struct _LIBCPP_TEMPLATE_VIS ratio_greater_equal : _BoolConstant<!ratio_less<_R1, _R2>::value> {
+  static_assert(__is_ratio<_R1>::value, "[ratio.general]/2 requires R1 to be a specialisation of the ratio template");
+  static_assert(__is_ratio<_R2>::value, "[ratio.general]/2 requires R2 to be a specialisation of the ratio template");
+};
+
+template <class _R1, class _R2>
+struct __ratio_gcd {
+  typedef ratio<__static_gcd<_R1::num, _R2::num>::value, __static_lcm<_R1::den, _R2::den>::value> type;
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _R1, class _R2>
+inline constexpr bool ratio_equal_v = ratio_equal<_R1, _R2>::value;
+
+template <class _R1, class _R2>
+inline constexpr bool ratio_not_equal_v = ratio_not_equal<_R1, _R2>::value;
+
+template <class _R1, class _R2>
+inline constexpr bool ratio_less_v = ratio_less<_R1, _R2>::value;
+
+template <class _R1, class _R2>
+inline constexpr bool ratio_less_equal_v = ratio_less_equal<_R1, _R2>::value;
+
+template <class _R1, class _R2>
+inline constexpr bool ratio_greater_v = ratio_greater<_R1, _R2>::value;
+
+template <class _R1, class _R2>
+inline constexpr bool ratio_greater_equal_v = ratio_greater_equal<_R1, _R2>::value;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_RATIO
diff --git a/libcxx/include/__cxx03/regex b/libcxx/include/__cxx03/regex
new file mode 100644
index 000000000000..b81413512132
--- /dev/null
+++ b/libcxx/include/__cxx03/regex
@@ -0,0 +1,5836 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_REGEX
+#define _LIBCPP_REGEX
+
+/*
+    regex synopsis
+
+#include <compare>
+#include <initializer_list>
+
+namespace std
+{
+
+namespace regex_constants
+{
+
+enum syntax_option_type
+{
+    icase      = unspecified,
+    nosubs     = unspecified,
+    optimize   = unspecified,
+    collate    = unspecified,
+    ECMAScript = unspecified,
+    basic      = unspecified,
+    extended   = unspecified,
+    awk        = unspecified,
+    grep       = unspecified,
+    egrep      = unspecified,
+    multiline  = unspecified
+};
+
+constexpr syntax_option_type operator~(syntax_option_type f);
+constexpr syntax_option_type operator&(syntax_option_type lhs, syntax_option_type rhs);
+constexpr syntax_option_type operator|(syntax_option_type lhs, syntax_option_type rhs);
+
+enum match_flag_type
+{
+    match_default     = 0,
+    match_not_bol     = unspecified,
+    match_not_eol     = unspecified,
+    match_not_bow     = unspecified,
+    match_not_eow     = unspecified,
+    match_any         = unspecified,
+    match_not_null    = unspecified,
+    match_continuous  = unspecified,
+    match_prev_avail  = unspecified,
+    format_default    = 0,
+    format_sed        = unspecified,
+    format_no_copy    = unspecified,
+    format_first_only = unspecified
+};
+
+constexpr match_flag_type operator~(match_flag_type f);
+constexpr match_flag_type operator&(match_flag_type lhs, match_flag_type rhs);
+constexpr match_flag_type operator|(match_flag_type lhs, match_flag_type rhs);
+
+enum error_type
+{
+    error_collate    = unspecified,
+    error_ctype      = unspecified,
+    error_escape     = unspecified,
+    error_backref    = unspecified,
+    error_brack      = unspecified,
+    error_paren      = unspecified,
+    error_brace      = unspecified,
+    error_badbrace   = unspecified,
+    error_range      = unspecified,
+    error_space      = unspecified,
+    error_badrepeat  = unspecified,
+    error_complexity = unspecified,
+    error_stack      = unspecified
+};
+
+}  // regex_constants
+
+class regex_error
+    : public runtime_error
+{
+public:
+    explicit regex_error(regex_constants::error_type ecode);
+    regex_constants::error_type code() const;
+};
+
+template <class charT>
+struct regex_traits
+{
+public:
+    typedef charT                   char_type;
+    typedef basic_string<char_type> string_type;
+    typedef locale                  locale_type;
+    typedef /bitmask_type/          char_class_type;
+
+    regex_traits();
+
+    static size_t length(const char_type* p);
+    charT translate(charT c) const;
+    charT translate_nocase(charT c) const;
+    template <class ForwardIterator>
+        string_type
+        transform(ForwardIterator first, ForwardIterator last) const;
+    template <class ForwardIterator>
+        string_type
+        transform_primary( ForwardIterator first, ForwardIterator last) const;
+    template <class ForwardIterator>
+        string_type
+        lookup_collatename(ForwardIterator first, ForwardIterator last) const;
+    template <class ForwardIterator>
+        char_class_type
+        lookup_classname(ForwardIterator first, ForwardIterator last,
+                         bool icase = false) const;
+    bool isctype(charT c, char_class_type f) const;
+    int value(charT ch, int radix) const;
+    locale_type imbue(locale_type l);
+    locale_type getloc()const;
+};
+
+template <class charT, class traits = regex_traits<charT>>
+class basic_regex
+{
+public:
+    // types:
+    typedef charT                               value_type;
+    typedef traits                              traits_type;
+    typedef typename traits::string_type        string_type;
+    typedef regex_constants::syntax_option_type flag_type;
+    typedef typename traits::locale_type        locale_type;
+
+    // constants:
+    static constexpr regex_constants::syntax_option_type icase = regex_constants::icase;
+    static constexpr regex_constants::syntax_option_type nosubs = regex_constants::nosubs;
+    static constexpr regex_constants::syntax_option_type optimize = regex_constants::optimize;
+    static constexpr regex_constants::syntax_option_type collate = regex_constants::collate;
+    static constexpr regex_constants::syntax_option_type ECMAScript = regex_constants::ECMAScript;
+    static constexpr regex_constants::syntax_option_type basic = regex_constants::basic;
+    static constexpr regex_constants::syntax_option_type extended = regex_constants::extended;
+    static constexpr regex_constants::syntax_option_type awk = regex_constants::awk;
+    static constexpr regex_constants::syntax_option_type grep = regex_constants::grep;
+    static constexpr regex_constants::syntax_option_type egrep = regex_constants::egrep;
+    static constexpr regex_constants::syntax_option_type multiline = regex_constants::multiline;
+
+    // construct/copy/destroy:
+    basic_regex();
+    explicit basic_regex(const charT* p, flag_type f = regex_constants::ECMAScript);
+    basic_regex(const charT* p, size_t len, flag_type f = regex_constants::ECMAScript);
+    basic_regex(const basic_regex&);
+    basic_regex(basic_regex&&) noexcept;
+    template <class ST, class SA>
+        explicit basic_regex(const basic_string<charT, ST, SA>& p,
+                             flag_type f = regex_constants::ECMAScript);
+    template <class ForwardIterator>
+        basic_regex(ForwardIterator first, ForwardIterator last,
+                    flag_type f = regex_constants::ECMAScript);
+    basic_regex(initializer_list<charT>, flag_type = regex_constants::ECMAScript);
+
+    ~basic_regex();
+
+    basic_regex& operator=(const basic_regex&);
+    basic_regex& operator=(basic_regex&&) noexcept;
+    basic_regex& operator=(const charT* ptr);
+    basic_regex& operator=(initializer_list<charT> il);
+    template <class ST, class SA>
+        basic_regex& operator=(const basic_string<charT, ST, SA>& p);
+
+    // assign:
+    basic_regex& assign(const basic_regex& that);
+    basic_regex& assign(basic_regex&& that) noexcept;
+    basic_regex& assign(const charT* ptr,           flag_type f = regex_constants::ECMAScript);
+    basic_regex& assign(const charT* p, size_t len, flag_type f = regex_constants::ECMAScript);
+    template <class string_traits, class A>
+        basic_regex& assign(const basic_string<charT, string_traits, A>& s,
+                                                    flag_type f = regex_constants::ECMAScript);
+    template <class InputIterator>
+        basic_regex& assign(InputIterator first, InputIterator last,
+                                                    flag_type f = regex_constants::ECMAScript);
+    basic_regex& assign(initializer_list<charT>,    flag_type f = regex_constants::ECMAScript);
+
+    // const operations:
+    unsigned mark_count() const;
+    flag_type flags() const;
+
+    // locale:
+    locale_type imbue(locale_type loc);
+    locale_type getloc() const;
+
+    // swap:
+    void swap(basic_regex&);
+};
+
+template<class ForwardIterator>
+basic_regex(ForwardIterator, ForwardIterator,
+            regex_constants::syntax_option_type = regex_constants::ECMAScript)
+    -> basic_regex<typename iterator_traits<ForwardIterator>::value_type>; // C++17
+
+typedef basic_regex<char>    regex;
+typedef basic_regex<wchar_t> wregex;
+
+template <class charT, class traits>
+    void swap(basic_regex<charT, traits>& e1, basic_regex<charT, traits>& e2);
+
+template <class BidirectionalIterator>
+class sub_match
+    : public pair<BidirectionalIterator, BidirectionalIterator>
+{
+public:
+    typedef typename iterator_traits<BidirectionalIterator>::value_type value_type;
+    typedef typename iterator_traits<BidirectionalIterator>::difference_type difference_type;
+    typedef BidirectionalIterator                                      iterator;
+    typedef basic_string<value_type>                                string_type;
+
+    bool matched;
+
+    constexpr sub_match();
+
+    difference_type length() const;
+    operator string_type() const;
+    string_type str() const;
+
+    int compare(const sub_match& s) const;
+    int compare(const string_type& s) const;
+    int compare(const value_type* s) const;
+
+    void swap(sub_match& s) noexcept(see below);
+};
+
+typedef sub_match<const char*>             csub_match;
+typedef sub_match<const wchar_t*>          wcsub_match;
+typedef sub_match<string::const_iterator>  ssub_match;
+typedef sub_match<wstring::const_iterator> wssub_match;
+
+template <class BiIter>
+    bool
+    operator==(const sub_match<BiIter>& lhs, const sub_match<BiIter>& rhs);
+
+template <class BiIter>
+    auto
+    operator<=>(const sub_match<BiIter>& lhs, const sub_match<BiIter>& rhs); // Since C++20
+
+ template <class BiIter>                                                     // Removed in C++20
+    bool
+    operator!=(const sub_match<BiIter>& lhs, const sub_match<BiIter>& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator<(const sub_match<BiIter>& lhs, const sub_match<BiIter>& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator<=(const sub_match<BiIter>& lhs, const sub_match<BiIter>& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator>=(const sub_match<BiIter>& lhs, const sub_match<BiIter>& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator>(const sub_match<BiIter>& lhs, const sub_match<BiIter>& rhs);
+
+template <class BiIter, class ST, class SA>                                  // Removed in C++20
+    bool
+    operator==(const basic_string<typename iterator_traits<BiIter>::value_type, ST, SA>& lhs,
+               const sub_match<BiIter>& rhs);
+
+template <class BiIter, class ST, class SA>                                  // Removed in C++20
+    bool
+    operator!=(const basic_string<typename iterator_traits<BiIter>::value_type, ST, SA>& lhs,
+               const sub_match<BiIter>& rhs);
+
+template <class BiIter, class ST, class SA>                                  // Removed in C++20
+    bool
+    operator<(const basic_string<typename iterator_traits<BiIter>::value_type, ST, SA>& lhs,
+              const sub_match<BiIter>& rhs);
+
+template <class BiIter, class ST, class SA>                                  // Removed in C++20
+    bool
+    operator>(const basic_string<typename iterator_traits<BiIter>::value_type, ST, SA>& lhs,
+              const sub_match<BiIter>& rhs);
+
+template <class BiIter, class ST, class SA>                                  // Removed in C++20
+    bool operator>=(const basic_string<typename iterator_traits<BiIter>::value_type, ST, SA>& lhs,
+                    const sub_match<BiIter>& rhs);
+
+template <class BiIter, class ST, class SA>                                  // Removed in C++20
+    bool
+    operator<=(const basic_string<typename iterator_traits<BiIter>::value_type, ST, SA>& lhs,
+               const sub_match<BiIter>& rhs);
+
+template <class BiIter, class ST, class SA>
+    bool
+    operator==(const sub_match<BiIter>& lhs,
+               const basic_string<typename iterator_traits<BiIter>::value_type, ST, SA>& rhs);
+
+template <class BiIter, class ST, class SA>                                  // Since C++20
+    auto
+    operator<=>(const sub_match<BiIter>& lhs,
+                const basic_string<typename iterator_traits<BiIter>::value_type, ST, SA>& rhs);
+
+template <class BiIter, class ST, class SA>                                  // Removed in C++20
+    bool
+    operator!=(const sub_match<BiIter>& lhs,
+               const basic_string<typename iterator_traits<BiIter>::value_type, ST, SA>& rhs);
+
+template <class BiIter, class ST, class SA>                                  // Removed in C++20
+    bool
+    operator<(const sub_match<BiIter>& lhs,
+              const basic_string<typename iterator_traits<BiIter>::value_type, ST, SA>& rhs);
+
+template <class BiIter, class ST, class SA>                                  // Removed in C++20
+    bool
+    operator>(const sub_match<BiIter>& lhs,
+                   const basic_string<typename iterator_traits<BiIter>::value_type, ST, SA>& rhs);
+
+template <class BiIter, class ST, class SA>                                  // Removed in C++20
+    bool
+    operator>=(const sub_match<BiIter>& lhs,
+               const basic_string<typename iterator_traits<BiIter>::value_type, ST, SA>& rhs);
+
+template <class BiIter, class ST, class SA>                                  // Removed in C++20
+    bool
+    operator<=(const sub_match<BiIter>& lhs,
+               const basic_string<typename iterator_traits<BiIter>::value_type, ST, SA>& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator==(typename iterator_traits<BiIter>::value_type const* lhs,
+               const sub_match<BiIter>& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator!=(typename iterator_traits<BiIter>::value_type const* lhs,
+               const sub_match<BiIter>& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator<(typename iterator_traits<BiIter>::value_type const* lhs,
+              const sub_match<BiIter>& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator>(typename iterator_traits<BiIter>::value_type const* lhs,
+              const sub_match<BiIter>& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator>=(typename iterator_traits<BiIter>::value_type const* lhs,
+               const sub_match<BiIter>& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator<=(typename iterator_traits<BiIter>::value_type const* lhs,
+               const sub_match<BiIter>& rhs);
+
+template <class BiIter>
+    bool
+    operator==(const sub_match<BiIter>& lhs,
+               typename iterator_traits<BiIter>::value_type const* rhs);
+
+template <class BiIter>                                                      // Since C++20
+    auto
+    operator<=>(const sub_match<BiIter>& lhs,
+                typename iterator_traits<BiIter>::value_type const* rhs);
+
+template <class BiIter, class ST, class SA>                                  // Removed in C++20
+    bool
+    operator!=(const sub_match<BiIter>& lhs,
+               typename iterator_traits<BiIter>::value_type const* rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator<(const sub_match<BiIter>& lhs,
+              typename iterator_traits<BiIter>::value_type const* rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator>(const sub_match<BiIter>& lhs,
+              typename iterator_traits<BiIter>::value_type const* rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator>=(const sub_match<BiIter>& lhs,
+               typename iterator_traits<BiIter>::value_type const* rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator<=(const sub_match<BiIter>& lhs,
+               typename iterator_traits<BiIter>::value_type const* rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator==(typename iterator_traits<BiIter>::value_type const& lhs,
+               const sub_match<BiIter>& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator!=(typename iterator_traits<BiIter>::value_type const& lhs,
+               const sub_match<BiIter>& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator<(typename iterator_traits<BiIter>::value_type const& lhs,
+              const sub_match<BiIter>& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator>(typename iterator_traits<BiIter>::value_type const& lhs,
+              const sub_match<BiIter>& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator>=(typename iterator_traits<BiIter>::value_type const& lhs,
+               const sub_match<BiIter>& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator<=(typename iterator_traits<BiIter>::value_type const& lhs,
+               const sub_match<BiIter>& rhs);
+
+template <class BiIter>
+    bool
+    operator==(const sub_match<BiIter>& lhs,
+               typename iterator_traits<BiIter>::value_type const& rhs);
+
+template <class BiIter>                                                      // Since C++20
+    auto
+    operator<=>(const sub_match<BiIter>& lhs,
+                typename iterator_traits<BiIter>::value_type const& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator!=(const sub_match<BiIter>& lhs,
+               typename iterator_traits<BiIter>::value_type const& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator<(const sub_match<BiIter>& lhs,
+              typename iterator_traits<BiIter>::value_type const& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator>(const sub_match<BiIter>& lhs,
+              typename iterator_traits<BiIter>::value_type const& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator>=(const sub_match<BiIter>& lhs,
+               typename iterator_traits<BiIter>::value_type const& rhs);
+
+template <class BiIter>                                                      // Removed in C++20
+    bool
+    operator<=(const sub_match<BiIter>& lhs,
+               typename iterator_traits<BiIter>::value_type const& rhs);
+
+template <class charT, class ST, class BiIter>
+    basic_ostream<charT, ST>&
+    operator<<(basic_ostream<charT, ST>& os, const sub_match<BiIter>& m);
+
+template <class BidirectionalIterator,
+          class Allocator = allocator<sub_match<BidirectionalIterator>>>
+class match_results
+{
+public:
+    typedef sub_match<BidirectionalIterator>                  value_type;
+    typedef const value_type&                                 const_reference;
+    typedef value_type&                                       reference;
+    typedef /implementation-defined/                          const_iterator;
+    typedef const_iterator                                    iterator;
+    typedef typename iterator_traits<BidirectionalIterator>::difference_type difference_type;
+    typedef typename allocator_traits<Allocator>::size_type   size_type;
+    typedef Allocator                                         allocator_type;
+    typedef typename iterator_traits<BidirectionalIterator>::value_type char_type;
+    typedef basic_string<char_type>                           string_type;
+
+    // construct/copy/destroy:
+    explicit match_results(const Allocator& a = Allocator()); // before C++20
+    match_results() : match_results(Allocator()) {}           // C++20
+    explicit match_results(const Allocator& a);               // C++20
+    match_results(const match_results& m);
+    match_results(match_results&& m) noexcept;
+    match_results& operator=(const match_results& m);
+    match_results& operator=(match_results&& m);
+    ~match_results();
+
+    bool ready() const;
+
+    // size:
+    size_type size() const;
+    size_type max_size() const;
+    bool empty() const;
+
+    // element access:
+    difference_type length(size_type sub = 0) const;
+    difference_type position(size_type sub = 0) const;
+    string_type str(size_type sub = 0) const;
+    const_reference operator[](size_type n) const;
+
+    const_reference prefix() const;
+    const_reference suffix() const;
+
+    const_iterator begin() const;
+    const_iterator end() const;
+    const_iterator cbegin() const;
+    const_iterator cend() const;
+
+    // format:
+    template <class OutputIter>
+        OutputIter
+        format(OutputIter out, const char_type* fmt_first,
+               const char_type* fmt_last,
+               regex_constants::match_flag_type flags = regex_constants::format_default) const;
+    template <class OutputIter, class ST, class SA>
+        OutputIter
+        format(OutputIter out, const basic_string<char_type, ST, SA>& fmt,
+               regex_constants::match_flag_type flags = regex_constants::format_default) const;
+    template <class ST, class SA>
+        basic_string<char_type, ST, SA>
+        format(const basic_string<char_type, ST, SA>& fmt,
+               regex_constants::match_flag_type flags = regex_constants::format_default) const;
+    string_type
+        format(const char_type* fmt,
+               regex_constants::match_flag_type flags = regex_constants::format_default) const;
+
+    // allocator:
+    allocator_type get_allocator() const;
+
+    // swap:
+    void swap(match_results& that);
+};
+
+typedef match_results<const char*>             cmatch;
+typedef match_results<const wchar_t*>          wcmatch;
+typedef match_results<string::const_iterator>  smatch;
+typedef match_results<wstring::const_iterator> wsmatch;
+
+template <class BidirectionalIterator, class Allocator>
+    bool
+    operator==(const match_results<BidirectionalIterator, Allocator>& m1,
+               const match_results<BidirectionalIterator, Allocator>& m2);
+
+template <class BidirectionalIterator, class Allocator>                    // Removed in C++20
+    bool
+    operator!=(const match_results<BidirectionalIterator, Allocator>& m1,
+               const match_results<BidirectionalIterator, Allocator>& m2);
+
+template <class BidirectionalIterator, class Allocator>
+    void
+    swap(match_results<BidirectionalIterator, Allocator>& m1,
+         match_results<BidirectionalIterator, Allocator>& m2);
+
+template <class BidirectionalIterator, class Allocator, class charT, class traits>
+    bool
+    regex_match(BidirectionalIterator first, BidirectionalIterator last,
+                match_results<BidirectionalIterator, Allocator>& m,
+                const basic_regex<charT, traits>& e,
+                regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class BidirectionalIterator, class charT, class traits>
+    bool
+    regex_match(BidirectionalIterator first, BidirectionalIterator last,
+                const basic_regex<charT, traits>& e,
+                regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class charT, class Allocator, class traits>
+    bool
+    regex_match(const charT* str, match_results<const charT*, Allocator>& m,
+                const basic_regex<charT, traits>& e,
+                regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class ST, class SA, class Allocator, class charT, class traits>
+    bool
+    regex_match(const basic_string<charT, ST, SA>& s,
+                match_results<typename basic_string<charT, ST, SA>::const_iterator, Allocator>& m,
+                const basic_regex<charT, traits>& e,
+                regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class ST, class SA, class Allocator, class charT, class traits>
+    bool
+    regex_match(const basic_string<charT, ST, SA>&& s,
+                match_results<typename basic_string<charT, ST, SA>::const_iterator, Allocator>& m,
+                const basic_regex<charT, traits>& e,
+                regex_constants::match_flag_type flags = regex_constants::match_default) = delete; // C++14
+
+template <class charT, class traits>
+    bool
+    regex_match(const charT* str, const basic_regex<charT, traits>& e,
+                regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class ST, class SA, class charT, class traits>
+    bool
+    regex_match(const basic_string<charT, ST, SA>& s,
+                const basic_regex<charT, traits>& e,
+                regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class BidirectionalIterator, class Allocator, class charT, class traits>
+    bool
+    regex_search(BidirectionalIterator first, BidirectionalIterator last,
+                 match_results<BidirectionalIterator, Allocator>& m,
+                 const basic_regex<charT, traits>& e,
+                 regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class BidirectionalIterator, class charT, class traits>
+    bool
+    regex_search(BidirectionalIterator first, BidirectionalIterator last,
+                 const basic_regex<charT, traits>& e,
+                 regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class charT, class Allocator, class traits>
+    bool
+    regex_search(const charT* str, match_results<const charT*, Allocator>& m,
+                 const basic_regex<charT, traits>& e,
+                 regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class charT, class traits>
+    bool
+    regex_search(const charT* str, const basic_regex<charT, traits>& e,
+                 regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class ST, class SA, class charT, class traits>
+    bool
+    regex_search(const basic_string<charT, ST, SA>& s,
+                 const basic_regex<charT, traits>& e,
+                 regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class ST, class SA, class Allocator, class charT, class traits>
+    bool
+    regex_search(const basic_string<charT, ST, SA>& s,
+                 match_results<typename basic_string<charT, ST, SA>::const_iterator, Allocator>& m,
+                 const basic_regex<charT, traits>& e,
+                 regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class ST, class SA, class Allocator, class charT, class traits>
+    bool
+    regex_search(const basic_string<charT, ST, SA>&& s,
+                 match_results<typename basic_string<charT, ST, SA>::const_iterator, Allocator>& m,
+                 const basic_regex<charT, traits>& e,
+                 regex_constants::match_flag_type flags = regex_constants::match_default) = delete; // C++14
+
+template <class OutputIterator, class BidirectionalIterator,
+          class traits, class charT, class ST, class SA>
+    OutputIterator
+    regex_replace(OutputIterator out,
+                  BidirectionalIterator first, BidirectionalIterator last,
+                  const basic_regex<charT, traits>& e,
+                  const basic_string<charT, ST, SA>& fmt,
+                  regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class OutputIterator, class BidirectionalIterator,
+          class traits, class charT>
+    OutputIterator
+    regex_replace(OutputIterator out,
+                  BidirectionalIterator first, BidirectionalIterator last,
+                  const basic_regex<charT, traits>& e, const charT* fmt,
+                  regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class traits, class charT, class ST, class SA, class FST, class FSA>
+    basic_string<charT, ST, SA>
+    regex_replace(const basic_string<charT, ST, SA>& s,
+                  const basic_regex<charT, traits>& e,
+                  const basic_string<charT, FST, FSA>& fmt,
+                  regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class traits, class charT, class ST, class SA>
+    basic_string<charT, ST, SA>
+    regex_replace(const basic_string<charT, ST, SA>& s,
+                  const basic_regex<charT, traits>& e, const charT* fmt,
+                  regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class traits, class charT, class ST, class SA>
+    basic_string<charT>
+    regex_replace(const charT* s,
+                  const basic_regex<charT, traits>& e,
+                  const basic_string<charT, ST, SA>& fmt,
+                  regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class traits, class charT>
+    basic_string<charT>
+    regex_replace(const charT* s,
+                  const basic_regex<charT, traits>& e,
+                  const charT* fmt,
+                  regex_constants::match_flag_type flags = regex_constants::match_default);
+
+template <class BidirectionalIterator,
+          class charT = typename iterator_traits< BidirectionalIterator>::value_type,
+          class traits = regex_traits<charT>>
+class regex_iterator
+{
+public:
+    typedef basic_regex<charT, traits>           regex_type;
+    typedef match_results<BidirectionalIterator> value_type;
+    typedef ptrdiff_t                            difference_type;
+    typedef const value_type*                    pointer;
+    typedef const value_type&                    reference;
+    typedef forward_iterator_tag                 iterator_category;
+    typedef input_iterator_tag                   iterator_concept; // since C++20
+
+    regex_iterator();
+    regex_iterator(BidirectionalIterator a, BidirectionalIterator b,
+                   const regex_type& re,
+                   regex_constants::match_flag_type m = regex_constants::match_default);
+    regex_iterator(BidirectionalIterator a, BidirectionalIterator b,
+                   const regex_type&& re,
+                   regex_constants::match_flag_type m
+                                     = regex_constants::match_default) = delete; // C++14
+    regex_iterator(const regex_iterator&);
+    regex_iterator& operator=(const regex_iterator&);
+
+    bool operator==(const regex_iterator&) const;
+    bool operator==(default_sentinel_t) const { return *this == regex_iterator(); } // since C++20
+    bool operator!=(const regex_iterator&) const;                                   // Removed in C++20
+
+    const value_type& operator*() const;
+    const value_type* operator->() const;
+
+    regex_iterator& operator++();
+    regex_iterator operator++(int);
+};
+
+typedef regex_iterator<const char*>             cregex_iterator;
+typedef regex_iterator<const wchar_t*>          wcregex_iterator;
+typedef regex_iterator<string::const_iterator>  sregex_iterator;
+typedef regex_iterator<wstring::const_iterator> wsregex_iterator;
+
+template <class BidirectionalIterator,
+          class charT = typename iterator_traits<BidirectionalIterator>::value_type,
+          class traits = regex_traits<charT>>
+class regex_token_iterator
+{
+public:
+    typedef basic_regex<charT, traits>       regex_type;
+    typedef sub_match<BidirectionalIterator> value_type;
+    typedef ptrdiff_t                        difference_type;
+    typedef const value_type*                pointer;
+    typedef const value_type&                reference;
+    typedef forward_iterator_tag             iterator_category;
+    typedef input_iterator_tag               iterator_concept; // since C++20
+
+    regex_token_iterator();
+    regex_token_iterator(BidirectionalIterator a, BidirectionalIterator b,
+                         const regex_type& re, int submatch = 0,
+                         regex_constants::match_flag_type m = regex_constants::match_default);
+    regex_token_iterator(BidirectionalIterator a, BidirectionalIterator b,
+                         const regex_type&& re, int submatch = 0,
+                         regex_constants::match_flag_type m = regex_constants::match_default) = delete; // C++14
+    regex_token_iterator(BidirectionalIterator a, BidirectionalIterator b,
+                         const regex_type& re, const vector<int>& submatches,
+                         regex_constants::match_flag_type m = regex_constants::match_default);
+    regex_token_iterator(BidirectionalIterator a, BidirectionalIterator b,
+                         const regex_type&& re, const vector<int>& submatches,
+                         regex_constants::match_flag_type m = regex_constants::match_default) = delete; // C++14
+    regex_token_iterator(BidirectionalIterator a, BidirectionalIterator b,
+                         const regex_type& re, initializer_list<int> submatches,
+                         regex_constants::match_flag_type m = regex_constants::match_default);
+    regex_token_iterator(BidirectionalIterator a, BidirectionalIterator b,
+                         const regex_type&& re, initializer_list<int> submatches,
+                         regex_constants::match_flag_type m = regex_constants::match_default) = delete; // C++14
+    template <size_t N>
+        regex_token_iterator(BidirectionalIterator a, BidirectionalIterator b,
+                             const regex_type& re, const int (&submatches)[N],
+                             regex_constants::match_flag_type m = regex_constants::match_default);
+    template <size_t N>
+        regex_token_iterator(BidirectionalIterator a, BidirectionalIterator b,
+                             const regex_type&& re, const int (&submatches)[N],
+                             regex_constants::match_flag_type m = regex_constants::match_default) = delete; // C++14
+    regex_token_iterator(const regex_token_iterator&);
+    regex_token_iterator& operator=(const regex_token_iterator&);
+
+    bool operator==(const regex_token_iterator&) const;
+    bool operator==(default_sentinel_t) const { return *this == regex_token_iterator(); } // since C++20
+    bool operator!=(const regex_token_iterator&) const;                                   // Removed in C++20
+
+    const value_type& operator*() const;
+    const value_type* operator->() const;
+
+    regex_token_iterator& operator++();
+    regex_token_iterator operator++(int);
+};
+
+typedef regex_token_iterator<const char*>             cregex_token_iterator;
+typedef regex_token_iterator<const wchar_t*>          wcregex_token_iterator;
+typedef regex_token_iterator<string::const_iterator>  sregex_token_iterator;
+typedef regex_token_iterator<wstring::const_iterator> wsregex_token_iterator;
+
+} // std
+*/
+
+#include <__algorithm/find.h>
+#include <__algorithm/search.h>
+#include <__assert>
+#include <__config>
+#include <__iterator/back_insert_iterator.h>
+#include <__iterator/default_sentinel.h>
+#include <__iterator/wrap_iter.h>
+#include <__locale>
+#include <__memory/shared_ptr.h>
+#include <__memory_resource/polymorphic_allocator.h>
+#include <__type_traits/is_swappable.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <__utility/swap.h>
+#include <__verbose_abort>
+#include <deque>
+#include <stdexcept>
+#include <string>
+#include <vector>
+#include <version>
+
+// standard-mandated includes
+
+// [iterator.range]
+#include <__iterator/access.h>
+#include <__iterator/data.h>
+#include <__iterator/empty.h>
+#include <__iterator/reverse_access.h>
+#include <__iterator/size.h>
+
+// [re.syn]
+#include <compare>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#define _LIBCPP_REGEX_COMPLEXITY_FACTOR 4096
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace regex_constants {
+
+// syntax_option_type
+
+enum syntax_option_type {
+  icase    = 1 << 0,
+  nosubs   = 1 << 1,
+  optimize = 1 << 2,
+  collate  = 1 << 3,
+#ifdef _LIBCPP_ABI_REGEX_CONSTANTS_NONZERO
+  ECMAScript = 1 << 9,
+#else
+  ECMAScript = 0,
+#endif
+  basic    = 1 << 4,
+  extended = 1 << 5,
+  awk      = 1 << 6,
+  grep     = 1 << 7,
+  egrep    = 1 << 8,
+  // 1 << 9 may be used by ECMAScript
+  multiline = 1 << 10
+};
+
+_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR syntax_option_type __get_grammar(syntax_option_type __g) {
+#ifdef _LIBCPP_ABI_REGEX_CONSTANTS_NONZERO
+  return static_cast<syntax_option_type>(__g & 0x3F0);
+#else
+  return static_cast<syntax_option_type>(__g & 0x1F0);
+#endif
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR syntax_option_type operator~(syntax_option_type __x) {
+  return syntax_option_type(~int(__x) & 0x1FF);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR syntax_option_type
+operator&(syntax_option_type __x, syntax_option_type __y) {
+  return syntax_option_type(int(__x) & int(__y));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR syntax_option_type
+operator|(syntax_option_type __x, syntax_option_type __y) {
+  return syntax_option_type(int(__x) | int(__y));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR syntax_option_type
+operator^(syntax_option_type __x, syntax_option_type __y) {
+  return syntax_option_type(int(__x) ^ int(__y));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI syntax_option_type& operator&=(syntax_option_type& __x, syntax_option_type __y) {
+  __x = __x & __y;
+  return __x;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI syntax_option_type& operator|=(syntax_option_type& __x, syntax_option_type __y) {
+  __x = __x | __y;
+  return __x;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI syntax_option_type& operator^=(syntax_option_type& __x, syntax_option_type __y) {
+  __x = __x ^ __y;
+  return __x;
+}
+
+// match_flag_type
+
+enum match_flag_type {
+  match_default     = 0,
+  match_not_bol     = 1 << 0,
+  match_not_eol     = 1 << 1,
+  match_not_bow     = 1 << 2,
+  match_not_eow     = 1 << 3,
+  match_any         = 1 << 4,
+  match_not_null    = 1 << 5,
+  match_continuous  = 1 << 6,
+  match_prev_avail  = 1 << 7,
+  format_default    = 0,
+  format_sed        = 1 << 8,
+  format_no_copy    = 1 << 9,
+  format_first_only = 1 << 10,
+  __no_update_pos   = 1 << 11,
+  __full_match      = 1 << 12
+};
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR match_flag_type operator~(match_flag_type __x) {
+  return match_flag_type(~int(__x) & 0x0FFF);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR match_flag_type operator&(match_flag_type __x, match_flag_type __y) {
+  return match_flag_type(int(__x) & int(__y));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR match_flag_type operator|(match_flag_type __x, match_flag_type __y) {
+  return match_flag_type(int(__x) | int(__y));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR match_flag_type operator^(match_flag_type __x, match_flag_type __y) {
+  return match_flag_type(int(__x) ^ int(__y));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI match_flag_type& operator&=(match_flag_type& __x, match_flag_type __y) {
+  __x = __x & __y;
+  return __x;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI match_flag_type& operator|=(match_flag_type& __x, match_flag_type __y) {
+  __x = __x | __y;
+  return __x;
+}
+
+inline _LIBCPP_HIDE_FROM_ABI match_flag_type& operator^=(match_flag_type& __x, match_flag_type __y) {
+  __x = __x ^ __y;
+  return __x;
+}
+
+enum error_type {
+  error_collate = 1,
+  error_ctype,
+  error_escape,
+  error_backref,
+  error_brack,
+  error_paren,
+  error_brace,
+  error_badbrace,
+  error_range,
+  error_space,
+  error_badrepeat,
+  error_complexity,
+  error_stack,
+  __re_err_grammar,
+  __re_err_empty,
+  __re_err_unknown,
+  __re_err_parse
+};
+
+} // namespace regex_constants
+
+class _LIBCPP_EXPORTED_FROM_ABI regex_error : public runtime_error {
+  regex_constants::error_type __code_;
+
+public:
+  explicit regex_error(regex_constants::error_type __ecode);
+  _LIBCPP_HIDE_FROM_ABI regex_error(const regex_error&) _NOEXCEPT = default;
+  ~regex_error() _NOEXCEPT override;
+  _LIBCPP_HIDE_FROM_ABI regex_constants::error_type code() const { return __code_; }
+};
+
+template <regex_constants::error_type _Ev>
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_regex_error() {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw regex_error(_Ev);
+#else
+  _LIBCPP_VERBOSE_ABORT("regex_error was thrown in -fno-exceptions mode");
+#endif
+}
+
+template <class _CharT>
+struct _LIBCPP_TEMPLATE_VIS regex_traits {
+public:
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+  typedef locale locale_type;
+#if defined(__BIONIC__) || defined(_NEWLIB_VERSION)
+  // Originally bionic's ctype_base used its own ctype masks because the
+  // builtin ctype implementation wasn't in libc++ yet. Bionic's ctype mask
+  // was only 8 bits wide and already saturated, so it used a wider type here
+  // to make room for __regex_word (then a part of this class rather than
+  // ctype_base). Bionic has since moved to the builtin ctype_base
+  // implementation, but this was not updated to match. Since then Android has
+  // needed to maintain a stable libc++ ABI, and this can't be changed without
+  // an ABI break.
+  // We also need this workaround for newlib since _NEWLIB_VERSION is not
+  // defined yet inside __config, so we can't set the
+  // _LIBCPP_PROVIDES_DEFAULT_RUNE_TABLE macro. Additionally, newlib is
+  // often used for space constrained environments, so it makes sense not to
+  // duplicate the ctype table.
+  typedef uint16_t char_class_type;
+#else
+  typedef ctype_base::mask char_class_type;
+#endif
+
+  static const char_class_type __regex_word = ctype_base::__regex_word;
+
+private:
+  locale __loc_;
+  const ctype<char_type>* __ct_;
+  const collate<char_type>* __col_;
+
+public:
+  regex_traits();
+
+  _LIBCPP_HIDE_FROM_ABI static size_t length(const char_type* __p) { return char_traits<char_type>::length(__p); }
+  _LIBCPP_HIDE_FROM_ABI char_type translate(char_type __c) const { return __c; }
+  char_type translate_nocase(char_type __c) const;
+  template <class _ForwardIterator>
+  string_type transform(_ForwardIterator __f, _ForwardIterator __l) const;
+  template <class _ForwardIterator>
+  _LIBCPP_HIDE_FROM_ABI string_type transform_primary(_ForwardIterator __f, _ForwardIterator __l) const {
+    return __transform_primary(__f, __l, char_type());
+  }
+  template <class _ForwardIterator>
+  _LIBCPP_HIDE_FROM_ABI string_type lookup_collatename(_ForwardIterator __f, _ForwardIterator __l) const {
+    return __lookup_collatename(__f, __l, char_type());
+  }
+  template <class _ForwardIterator>
+  _LIBCPP_HIDE_FROM_ABI char_class_type
+  lookup_classname(_ForwardIterator __f, _ForwardIterator __l, bool __icase = false) const {
+    return __lookup_classname(__f, __l, __icase, char_type());
+  }
+  bool isctype(char_type __c, char_class_type __m) const;
+  _LIBCPP_HIDE_FROM_ABI int value(char_type __ch, int __radix) const { return __regex_traits_value(__ch, __radix); }
+  locale_type imbue(locale_type __l);
+  _LIBCPP_HIDE_FROM_ABI locale_type getloc() const { return __loc_; }
+
+private:
+  void __init();
+
+  template <class _ForwardIterator>
+  string_type __transform_primary(_ForwardIterator __f, _ForwardIterator __l, char) const;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+  template <class _ForwardIterator>
+  string_type __transform_primary(_ForwardIterator __f, _ForwardIterator __l, wchar_t) const;
+#endif
+  template <class _ForwardIterator>
+  string_type __lookup_collatename(_ForwardIterator __f, _ForwardIterator __l, char) const;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+  template <class _ForwardIterator>
+  string_type __lookup_collatename(_ForwardIterator __f, _ForwardIterator __l, wchar_t) const;
+#endif
+  template <class _ForwardIterator>
+  char_class_type __lookup_classname(_ForwardIterator __f, _ForwardIterator __l, bool __icase, char) const;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+  template <class _ForwardIterator>
+  char_class_type __lookup_classname(_ForwardIterator __f, _ForwardIterator __l, bool __icase, wchar_t) const;
+#endif
+
+  static int __regex_traits_value(unsigned char __ch, int __radix);
+  _LIBCPP_HIDE_FROM_ABI int __regex_traits_value(char __ch, int __radix) const {
+    return __regex_traits_value(static_cast<unsigned char>(__ch), __radix);
+  }
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+  _LIBCPP_HIDE_FROM_ABI int __regex_traits_value(wchar_t __ch, int __radix) const;
+#endif
+};
+
+template <class _CharT>
+const typename regex_traits<_CharT>::char_class_type regex_traits<_CharT>::__regex_word;
+
+template <class _CharT>
+regex_traits<_CharT>::regex_traits() {
+  __init();
+}
+
+template <class _CharT>
+typename regex_traits<_CharT>::char_type regex_traits<_CharT>::translate_nocase(char_type __c) const {
+  return __ct_->tolower(__c);
+}
+
+template <class _CharT>
+template <class _ForwardIterator>
+typename regex_traits<_CharT>::string_type
+regex_traits<_CharT>::transform(_ForwardIterator __f, _ForwardIterator __l) const {
+  string_type __s(__f, __l);
+  return __col_->transform(__s.data(), __s.data() + __s.size());
+}
+
+template <class _CharT>
+void regex_traits<_CharT>::__init() {
+  __ct_  = &std::use_facet<ctype<char_type> >(__loc_);
+  __col_ = &std::use_facet<collate<char_type> >(__loc_);
+}
+
+template <class _CharT>
+typename regex_traits<_CharT>::locale_type regex_traits<_CharT>::imbue(locale_type __l) {
+  locale __r = __loc_;
+  __loc_     = __l;
+  __init();
+  return __r;
+}
+
+// transform_primary is very FreeBSD-specific
+
+template <class _CharT>
+template <class _ForwardIterator>
+typename regex_traits<_CharT>::string_type
+regex_traits<_CharT>::__transform_primary(_ForwardIterator __f, _ForwardIterator __l, char) const {
+  const string_type __s(__f, __l);
+  string_type __d = __col_->transform(__s.data(), __s.data() + __s.size());
+  switch (__d.size()) {
+  case 1:
+    break;
+  case 12:
+    __d[11] = __d[3];
+    break;
+  default:
+    __d.clear();
+    break;
+  }
+  return __d;
+}
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class _CharT>
+template <class _ForwardIterator>
+typename regex_traits<_CharT>::string_type
+regex_traits<_CharT>::__transform_primary(_ForwardIterator __f, _ForwardIterator __l, wchar_t) const {
+  const string_type __s(__f, __l);
+  string_type __d = __col_->transform(__s.data(), __s.data() + __s.size());
+  switch (__d.size()) {
+  case 1:
+    break;
+  case 3:
+    __d[2] = __d[0];
+    break;
+  default:
+    __d.clear();
+    break;
+  }
+  return __d;
+}
+#endif
+
+// lookup_collatename is very FreeBSD-specific
+
+_LIBCPP_EXPORTED_FROM_ABI string __get_collation_name(const char* __s);
+
+template <class _CharT>
+template <class _ForwardIterator>
+typename regex_traits<_CharT>::string_type
+regex_traits<_CharT>::__lookup_collatename(_ForwardIterator __f, _ForwardIterator __l, char) const {
+  string_type __s(__f, __l);
+  string_type __r;
+  if (!__s.empty()) {
+    __r = std::__get_collation_name(__s.c_str());
+    if (__r.empty() && __s.size() <= 2) {
+      __r = __col_->transform(__s.data(), __s.data() + __s.size());
+      if (__r.size() == 1 || __r.size() == 12)
+        __r = __s;
+      else
+        __r.clear();
+    }
+  }
+  return __r;
+}
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class _CharT>
+template <class _ForwardIterator>
+typename regex_traits<_CharT>::string_type
+regex_traits<_CharT>::__lookup_collatename(_ForwardIterator __f, _ForwardIterator __l, wchar_t) const {
+  string_type __s(__f, __l);
+  string __n;
+  __n.reserve(__s.size());
+  for (typename string_type::const_iterator __i = __s.begin(), __e = __s.end(); __i != __e; ++__i) {
+    if (static_cast<unsigned>(*__i) >= 127)
+      return string_type();
+    __n.push_back(char(*__i));
+  }
+  string_type __r;
+  if (!__s.empty()) {
+    __n = __get_collation_name(__n.c_str());
+    if (!__n.empty())
+      __r.assign(__n.begin(), __n.end());
+    else if (__s.size() <= 2) {
+      __r = __col_->transform(__s.data(), __s.data() + __s.size());
+      if (__r.size() == 1 || __r.size() == 3)
+        __r = __s;
+      else
+        __r.clear();
+    }
+  }
+  return __r;
+}
+#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+// lookup_classname
+
+regex_traits<char>::char_class_type _LIBCPP_EXPORTED_FROM_ABI __get_classname(const char* __s, bool __icase);
+
+template <class _CharT>
+template <class _ForwardIterator>
+typename regex_traits<_CharT>::char_class_type
+regex_traits<_CharT>::__lookup_classname(_ForwardIterator __f, _ForwardIterator __l, bool __icase, char) const {
+  string_type __s(__f, __l);
+  __ct_->tolower(&__s[0], &__s[0] + __s.size());
+  return std::__get_classname(__s.c_str(), __icase);
+}
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class _CharT>
+template <class _ForwardIterator>
+typename regex_traits<_CharT>::char_class_type
+regex_traits<_CharT>::__lookup_classname(_ForwardIterator __f, _ForwardIterator __l, bool __icase, wchar_t) const {
+  string_type __s(__f, __l);
+  __ct_->tolower(&__s[0], &__s[0] + __s.size());
+  string __n;
+  __n.reserve(__s.size());
+  for (typename string_type::const_iterator __i = __s.begin(), __e = __s.end(); __i != __e; ++__i) {
+    if (static_cast<unsigned>(*__i) >= 127)
+      return char_class_type();
+    __n.push_back(char(*__i));
+  }
+  return __get_classname(__n.c_str(), __icase);
+}
+#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+template <class _CharT>
+bool regex_traits<_CharT>::isctype(char_type __c, char_class_type __m) const {
+  if (__ct_->is(__m, __c))
+    return true;
+  return (__c == '_' && (__m & __regex_word));
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool __is_07(unsigned char __c) {
+  return (__c & 0xF8u) ==
+#if defined(__MVS__) && !defined(__NATIVE_ASCII_F)
+         0xF0;
+#else
+         0x30;
+#endif
+}
+
+inline _LIBCPP_HIDE_FROM_ABI bool __is_89(unsigned char __c) {
+  return (__c & 0xFEu) ==
+#if defined(__MVS__) && !defined(__NATIVE_ASCII_F)
+         0xF8;
+#else
+         0x38;
+#endif
+}
+
+inline _LIBCPP_HIDE_FROM_ABI unsigned char __to_lower(unsigned char __c) {
+#if defined(__MVS__) && !defined(__NATIVE_ASCII_F)
+  return __c & 0xBF;
+#else
+  return __c | 0x20;
+#endif
+}
+
+template <class _CharT>
+int regex_traits<_CharT>::__regex_traits_value(unsigned char __ch, int __radix) {
+  if (__is_07(__ch)) // '0' <= __ch && __ch <= '7'
+    return __ch - '0';
+  if (__radix != 8) {
+    if (__is_89(__ch)) // '8' <= __ch && __ch <= '9'
+      return __ch - '0';
+    if (__radix == 16) {
+      __ch = __to_lower(__ch); // tolower
+      if ('a' <= __ch && __ch <= 'f')
+        return __ch - ('a' - 10);
+    }
+  }
+  return -1;
+}
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class _CharT>
+inline int regex_traits<_CharT>::__regex_traits_value(wchar_t __ch, int __radix) const {
+  return __regex_traits_value(static_cast<unsigned char>(__ct_->narrow(__ch, char_type())), __radix);
+}
+#endif
+
+template <class _CharT>
+class __node;
+
+template <class _BidirectionalIterator>
+class _LIBCPP_TEMPLATE_VIS sub_match;
+
+template <class _BidirectionalIterator, class _Allocator = allocator<sub_match<_BidirectionalIterator> > >
+class _LIBCPP_TEMPLATE_VIS match_results;
+
+template <class _CharT>
+struct __state {
+  enum {
+    __end_state = -1000,
+    __consume_input,          // -999
+    __begin_marked_expr,      // -998
+    __end_marked_expr,        // -997
+    __pop_state,              // -996
+    __accept_and_consume,     // -995
+    __accept_but_not_consume, // -994
+    __reject,                 // -993
+    __split,
+    __repeat
+  };
+
+  int __do_;
+  const _CharT* __first_;
+  const _CharT* __current_;
+  const _CharT* __last_;
+  vector<sub_match<const _CharT*> > __sub_matches_;
+  vector<pair<size_t, const _CharT*> > __loop_data_;
+  const __node<_CharT>* __node_;
+  regex_constants::match_flag_type __flags_;
+  bool __at_first_;
+
+  _LIBCPP_HIDE_FROM_ABI __state()
+      : __do_(0),
+        __first_(nullptr),
+        __current_(nullptr),
+        __last_(nullptr),
+        __node_(nullptr),
+        __flags_(),
+        __at_first_(false) {}
+};
+
+// __node
+
+template <class _CharT>
+class __node {
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI __node() {}
+  __node(const __node&)            = delete;
+  __node& operator=(const __node&) = delete;
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL
+  virtual ~__node() {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL
+  virtual void __exec(__state&) const {}
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL
+  virtual void __exec_split(bool, __state&) const {}
+};
+
+// __end_state
+
+template <class _CharT>
+class __end_state : public __node<_CharT> {
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI __end_state() {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT>
+void __end_state<_CharT>::__exec(__state& __s) const {
+  __s.__do_ = __state::__end_state;
+}
+
+// __has_one_state
+
+template <class _CharT>
+class __has_one_state : public __node<_CharT> {
+  __node<_CharT>* __first_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __has_one_state(__node<_CharT>* __s) : __first_(__s) {}
+
+  _LIBCPP_HIDE_FROM_ABI __node<_CharT>* first() const { return __first_; }
+  _LIBCPP_HIDE_FROM_ABI __node<_CharT>*& first() { return __first_; }
+};
+
+// __owns_one_state
+
+template <class _CharT>
+class __owns_one_state : public __has_one_state<_CharT> {
+  typedef __has_one_state<_CharT> base;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __owns_one_state(__node<_CharT>* __s) : base(__s) {}
+
+  ~__owns_one_state() override;
+};
+
+template <class _CharT>
+__owns_one_state<_CharT>::~__owns_one_state() {
+  delete this->first();
+}
+
+// __empty_state
+
+template <class _CharT>
+class __empty_state : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __empty_state(__node<_CharT>* __s) : base(__s) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT>
+void __empty_state<_CharT>::__exec(__state& __s) const {
+  __s.__do_   = __state::__accept_but_not_consume;
+  __s.__node_ = this->first();
+}
+
+// __empty_non_own_state
+
+template <class _CharT>
+class __empty_non_own_state : public __has_one_state<_CharT> {
+  typedef __has_one_state<_CharT> base;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __empty_non_own_state(__node<_CharT>* __s) : base(__s) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT>
+void __empty_non_own_state<_CharT>::__exec(__state& __s) const {
+  __s.__do_   = __state::__accept_but_not_consume;
+  __s.__node_ = this->first();
+}
+
+// __repeat_one_loop
+
+template <class _CharT>
+class __repeat_one_loop : public __has_one_state<_CharT> {
+  typedef __has_one_state<_CharT> base;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __repeat_one_loop(__node<_CharT>* __s) : base(__s) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT>
+void __repeat_one_loop<_CharT>::__exec(__state& __s) const {
+  __s.__do_   = __state::__repeat;
+  __s.__node_ = this->first();
+}
+
+// __owns_two_states
+
+template <class _CharT>
+class __owns_two_states : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+  base* __second_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __owns_two_states(__node<_CharT>* __s1, base* __s2) : base(__s1), __second_(__s2) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual ~__owns_two_states();
+
+  _LIBCPP_HIDE_FROM_ABI base* second() const { return __second_; }
+  _LIBCPP_HIDE_FROM_ABI base*& second() { return __second_; }
+};
+
+template <class _CharT>
+__owns_two_states<_CharT>::~__owns_two_states() {
+  delete __second_;
+}
+
+// __loop
+
+template <class _CharT>
+class __loop : public __owns_two_states<_CharT> {
+  typedef __owns_two_states<_CharT> base;
+
+  size_t __min_;
+  size_t __max_;
+  unsigned __loop_id_;
+  unsigned __mexp_begin_;
+  unsigned __mexp_end_;
+  bool __greedy_;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __loop(
+      unsigned __loop_id,
+      __node<_CharT>* __s1,
+      __owns_one_state<_CharT>* __s2,
+      unsigned __mexp_begin,
+      unsigned __mexp_end,
+      bool __greedy = true,
+      size_t __min  = 0,
+      size_t __max  = numeric_limits<size_t>::max())
+      : base(__s1, __s2),
+        __min_(__min),
+        __max_(__max),
+        __loop_id_(__loop_id),
+        __mexp_begin_(__mexp_begin),
+        __mexp_end_(__mexp_end),
+        __greedy_(__greedy) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state& __s) const;
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec_split(bool __second, __state& __s) const;
+
+private:
+  _LIBCPP_HIDE_FROM_ABI void __init_repeat(__state& __s) const {
+    __s.__loop_data_[__loop_id_].second = __s.__current_;
+    for (size_t __i = __mexp_begin_ - 1; __i != __mexp_end_ - 1; ++__i) {
+      __s.__sub_matches_[__i].first   = __s.__last_;
+      __s.__sub_matches_[__i].second  = __s.__last_;
+      __s.__sub_matches_[__i].matched = false;
+    }
+  }
+};
+
+template <class _CharT>
+void __loop<_CharT>::__exec(__state& __s) const {
+  if (__s.__do_ == __state::__repeat) {
+    bool __do_repeat = ++__s.__loop_data_[__loop_id_].first < __max_;
+    bool __do_alt    = __s.__loop_data_[__loop_id_].first >= __min_;
+    if (__do_repeat && __do_alt && __s.__loop_data_[__loop_id_].second == __s.__current_)
+      __do_repeat = false;
+    if (__do_repeat && __do_alt)
+      __s.__do_ = __state::__split;
+    else if (__do_repeat) {
+      __s.__do_   = __state::__accept_but_not_consume;
+      __s.__node_ = this->first();
+      __init_repeat(__s);
+    } else {
+      __s.__do_   = __state::__accept_but_not_consume;
+      __s.__node_ = this->second();
+    }
+  } else {
+    __s.__loop_data_[__loop_id_].first = 0;
+    bool __do_repeat                   = 0 < __max_;
+    bool __do_alt                      = 0 >= __min_;
+    if (__do_repeat && __do_alt)
+      __s.__do_ = __state::__split;
+    else if (__do_repeat) {
+      __s.__do_   = __state::__accept_but_not_consume;
+      __s.__node_ = this->first();
+      __init_repeat(__s);
+    } else {
+      __s.__do_   = __state::__accept_but_not_consume;
+      __s.__node_ = this->second();
+    }
+  }
+}
+
+template <class _CharT>
+void __loop<_CharT>::__exec_split(bool __second, __state& __s) const {
+  __s.__do_ = __state::__accept_but_not_consume;
+  if (__greedy_ != __second) {
+    __s.__node_ = this->first();
+    __init_repeat(__s);
+  } else
+    __s.__node_ = this->second();
+}
+
+// __alternate
+
+template <class _CharT>
+class __alternate : public __owns_two_states<_CharT> {
+  typedef __owns_two_states<_CharT> base;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __alternate(__owns_one_state<_CharT>* __s1, __owns_one_state<_CharT>* __s2)
+      : base(__s1, __s2) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state& __s) const;
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec_split(bool __second, __state& __s) const;
+};
+
+template <class _CharT>
+void __alternate<_CharT>::__exec(__state& __s) const {
+  __s.__do_ = __state::__split;
+}
+
+template <class _CharT>
+void __alternate<_CharT>::__exec_split(bool __second, __state& __s) const {
+  __s.__do_ = __state::__accept_but_not_consume;
+  if (__second)
+    __s.__node_ = this->second();
+  else
+    __s.__node_ = this->first();
+}
+
+// __begin_marked_subexpression
+
+template <class _CharT>
+class __begin_marked_subexpression : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+  unsigned __mexp_;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __begin_marked_subexpression(unsigned __mexp, __node<_CharT>* __s)
+      : base(__s), __mexp_(__mexp) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT>
+void __begin_marked_subexpression<_CharT>::__exec(__state& __s) const {
+  __s.__do_                             = __state::__accept_but_not_consume;
+  __s.__sub_matches_[__mexp_ - 1].first = __s.__current_;
+  __s.__node_                           = this->first();
+}
+
+// __end_marked_subexpression
+
+template <class _CharT>
+class __end_marked_subexpression : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+  unsigned __mexp_;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __end_marked_subexpression(unsigned __mexp, __node<_CharT>* __s)
+      : base(__s), __mexp_(__mexp) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT>
+void __end_marked_subexpression<_CharT>::__exec(__state& __s) const {
+  __s.__do_                               = __state::__accept_but_not_consume;
+  __s.__sub_matches_[__mexp_ - 1].second  = __s.__current_;
+  __s.__sub_matches_[__mexp_ - 1].matched = true;
+  __s.__node_                             = this->first();
+}
+
+// __back_ref
+
+template <class _CharT>
+class __back_ref : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+  unsigned __mexp_;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __back_ref(unsigned __mexp, __node<_CharT>* __s) : base(__s), __mexp_(__mexp) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT>
+void __back_ref<_CharT>::__exec(__state& __s) const {
+  if (__mexp_ > __s.__sub_matches_.size())
+    __throw_regex_error<regex_constants::error_backref>();
+  sub_match<const _CharT*>& __sm = __s.__sub_matches_[__mexp_ - 1];
+  if (__sm.matched) {
+    ptrdiff_t __len = __sm.second - __sm.first;
+    if (__s.__last_ - __s.__current_ >= __len && std::equal(__sm.first, __sm.second, __s.__current_)) {
+      __s.__do_ = __state::__accept_but_not_consume;
+      __s.__current_ += __len;
+      __s.__node_ = this->first();
+    } else {
+      __s.__do_   = __state::__reject;
+      __s.__node_ = nullptr;
+    }
+  } else {
+    __s.__do_   = __state::__reject;
+    __s.__node_ = nullptr;
+  }
+}
+
+// __back_ref_icase
+
+template <class _CharT, class _Traits>
+class __back_ref_icase : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+  _Traits __traits_;
+  unsigned __mexp_;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __back_ref_icase(const _Traits& __traits, unsigned __mexp, __node<_CharT>* __s)
+      : base(__s), __traits_(__traits), __mexp_(__mexp) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT, class _Traits>
+void __back_ref_icase<_CharT, _Traits>::__exec(__state& __s) const {
+  sub_match<const _CharT*>& __sm = __s.__sub_matches_[__mexp_ - 1];
+  if (__sm.matched) {
+    ptrdiff_t __len = __sm.second - __sm.first;
+    if (__s.__last_ - __s.__current_ >= __len) {
+      for (ptrdiff_t __i = 0; __i < __len; ++__i) {
+        if (__traits_.translate_nocase(__sm.first[__i]) != __traits_.translate_nocase(__s.__current_[__i]))
+          goto __not_equal;
+      }
+      __s.__do_ = __state::__accept_but_not_consume;
+      __s.__current_ += __len;
+      __s.__node_ = this->first();
+    } else {
+      __s.__do_   = __state::__reject;
+      __s.__node_ = nullptr;
+    }
+  } else {
+  __not_equal:
+    __s.__do_   = __state::__reject;
+    __s.__node_ = nullptr;
+  }
+}
+
+// __back_ref_collate
+
+template <class _CharT, class _Traits>
+class __back_ref_collate : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+  _Traits __traits_;
+  unsigned __mexp_;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __back_ref_collate(const _Traits& __traits, unsigned __mexp, __node<_CharT>* __s)
+      : base(__s), __traits_(__traits), __mexp_(__mexp) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT, class _Traits>
+void __back_ref_collate<_CharT, _Traits>::__exec(__state& __s) const {
+  sub_match<const _CharT*>& __sm = __s.__sub_matches_[__mexp_ - 1];
+  if (__sm.matched) {
+    ptrdiff_t __len = __sm.second - __sm.first;
+    if (__s.__last_ - __s.__current_ >= __len) {
+      for (ptrdiff_t __i = 0; __i < __len; ++__i) {
+        if (__traits_.translate(__sm.first[__i]) != __traits_.translate(__s.__current_[__i]))
+          goto __not_equal;
+      }
+      __s.__do_ = __state::__accept_but_not_consume;
+      __s.__current_ += __len;
+      __s.__node_ = this->first();
+    } else {
+      __s.__do_   = __state::__reject;
+      __s.__node_ = nullptr;
+    }
+  } else {
+  __not_equal:
+    __s.__do_   = __state::__reject;
+    __s.__node_ = nullptr;
+  }
+}
+
+// __word_boundary
+
+template <class _CharT, class _Traits>
+class __word_boundary : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+  _Traits __traits_;
+  bool __invert_;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __word_boundary(const _Traits& __traits, bool __invert, __node<_CharT>* __s)
+      : base(__s), __traits_(__traits), __invert_(__invert) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT, class _Traits>
+void __word_boundary<_CharT, _Traits>::__exec(__state& __s) const {
+  bool __is_word_b = false;
+  if (__s.__first_ != __s.__last_) {
+    if (__s.__current_ == __s.__last_) {
+      if (!(__s.__flags_ & regex_constants::match_not_eow)) {
+        _CharT __c  = __s.__current_[-1];
+        __is_word_b = __c == '_' || __traits_.isctype(__c, ctype_base::alnum);
+      }
+    } else if (__s.__current_ == __s.__first_ && !(__s.__flags_ & regex_constants::match_prev_avail)) {
+      if (!(__s.__flags_ & regex_constants::match_not_bow)) {
+        _CharT __c  = *__s.__current_;
+        __is_word_b = __c == '_' || __traits_.isctype(__c, ctype_base::alnum);
+      }
+    } else {
+      _CharT __c1    = __s.__current_[-1];
+      _CharT __c2    = *__s.__current_;
+      bool __is_c1_b = __c1 == '_' || __traits_.isctype(__c1, ctype_base::alnum);
+      bool __is_c2_b = __c2 == '_' || __traits_.isctype(__c2, ctype_base::alnum);
+      __is_word_b    = __is_c1_b != __is_c2_b;
+    }
+  }
+  if (__is_word_b != __invert_) {
+    __s.__do_   = __state::__accept_but_not_consume;
+    __s.__node_ = this->first();
+  } else {
+    __s.__do_   = __state::__reject;
+    __s.__node_ = nullptr;
+  }
+}
+
+// __l_anchor
+
+template <class _CharT>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __is_eol(_CharT __c) {
+  return __c == '\r' || __c == '\n';
+}
+
+template <class _CharT>
+class __l_anchor_multiline : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+  bool __multiline_;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI __l_anchor_multiline(bool __multiline, __node<_CharT>* __s)
+      : base(__s), __multiline_(__multiline) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT>
+void __l_anchor_multiline<_CharT>::__exec(__state& __s) const {
+  if (__s.__at_first_ && __s.__current_ == __s.__first_ && !(__s.__flags_ & regex_constants::match_not_bol)) {
+    __s.__do_   = __state::__accept_but_not_consume;
+    __s.__node_ = this->first();
+  } else if (__multiline_ && !__s.__at_first_ && std::__is_eol(*std::prev(__s.__current_))) {
+    __s.__do_   = __state::__accept_but_not_consume;
+    __s.__node_ = this->first();
+  } else {
+    __s.__do_   = __state::__reject;
+    __s.__node_ = nullptr;
+  }
+}
+
+// __r_anchor
+
+template <class _CharT>
+class __r_anchor_multiline : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+  bool __multiline_;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI __r_anchor_multiline(bool __multiline, __node<_CharT>* __s)
+      : base(__s), __multiline_(__multiline) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT>
+void __r_anchor_multiline<_CharT>::__exec(__state& __s) const {
+  if (__s.__current_ == __s.__last_ && !(__s.__flags_ & regex_constants::match_not_eol)) {
+    __s.__do_   = __state::__accept_but_not_consume;
+    __s.__node_ = this->first();
+  } else if (__multiline_ && std::__is_eol(*__s.__current_)) {
+    __s.__do_   = __state::__accept_but_not_consume;
+    __s.__node_ = this->first();
+  } else {
+    __s.__do_   = __state::__reject;
+    __s.__node_ = nullptr;
+  }
+}
+
+// __match_any
+
+template <class _CharT>
+class __match_any : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI __match_any(__node<_CharT>* __s) : base(__s) {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT>
+void __match_any<_CharT>::__exec(__state& __s) const {
+  if (__s.__current_ != __s.__last_ && *__s.__current_ != 0) {
+    __s.__do_ = __state::__accept_and_consume;
+    ++__s.__current_;
+    __s.__node_ = this->first();
+  } else {
+    __s.__do_   = __state::__reject;
+    __s.__node_ = nullptr;
+  }
+}
+
+// __match_any_but_newline
+
+template <class _CharT>
+class __match_any_but_newline : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI __match_any_but_newline(__node<_CharT>* __s) : base(__s) {}
+
+  void __exec(__state&) const override;
+};
+
+template <>
+_LIBCPP_EXPORTED_FROM_ABI void __match_any_but_newline<char>::__exec(__state&) const;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+_LIBCPP_EXPORTED_FROM_ABI void __match_any_but_newline<wchar_t>::__exec(__state&) const;
+#endif
+
+// __match_char
+
+template <class _CharT>
+class __match_char : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+  _CharT __c_;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI __match_char(_CharT __c, __node<_CharT>* __s) : base(__s), __c_(__c) {}
+
+  __match_char(const __match_char&)            = delete;
+  __match_char& operator=(const __match_char&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT>
+void __match_char<_CharT>::__exec(__state& __s) const {
+  if (__s.__current_ != __s.__last_ && *__s.__current_ == __c_) {
+    __s.__do_ = __state::__accept_and_consume;
+    ++__s.__current_;
+    __s.__node_ = this->first();
+  } else {
+    __s.__do_   = __state::__reject;
+    __s.__node_ = nullptr;
+  }
+}
+
+// __match_char_icase
+
+template <class _CharT, class _Traits>
+class __match_char_icase : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+  _Traits __traits_;
+  _CharT __c_;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI __match_char_icase(const _Traits& __traits, _CharT __c, __node<_CharT>* __s)
+      : base(__s), __traits_(__traits), __c_(__traits.translate_nocase(__c)) {}
+
+  __match_char_icase(const __match_char_icase&)            = delete;
+  __match_char_icase& operator=(const __match_char_icase&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT, class _Traits>
+void __match_char_icase<_CharT, _Traits>::__exec(__state& __s) const {
+  if (__s.__current_ != __s.__last_ && __traits_.translate_nocase(*__s.__current_) == __c_) {
+    __s.__do_ = __state::__accept_and_consume;
+    ++__s.__current_;
+    __s.__node_ = this->first();
+  } else {
+    __s.__do_   = __state::__reject;
+    __s.__node_ = nullptr;
+  }
+}
+
+// __match_char_collate
+
+template <class _CharT, class _Traits>
+class __match_char_collate : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+  _Traits __traits_;
+  _CharT __c_;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI __match_char_collate(const _Traits& __traits, _CharT __c, __node<_CharT>* __s)
+      : base(__s), __traits_(__traits), __c_(__traits.translate(__c)) {}
+
+  __match_char_collate(const __match_char_collate&)            = delete;
+  __match_char_collate& operator=(const __match_char_collate&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT, class _Traits>
+void __match_char_collate<_CharT, _Traits>::__exec(__state& __s) const {
+  if (__s.__current_ != __s.__last_ && __traits_.translate(*__s.__current_) == __c_) {
+    __s.__do_ = __state::__accept_and_consume;
+    ++__s.__current_;
+    __s.__node_ = this->first();
+  } else {
+    __s.__do_   = __state::__reject;
+    __s.__node_ = nullptr;
+  }
+}
+
+// __bracket_expression
+
+template <class _CharT, class _Traits>
+class __bracket_expression : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+  typedef typename _Traits::string_type string_type;
+
+  _Traits __traits_;
+  vector<_CharT> __chars_;
+  vector<_CharT> __neg_chars_;
+  vector<pair<string_type, string_type> > __ranges_;
+  vector<pair<_CharT, _CharT> > __digraphs_;
+  vector<string_type> __equivalences_;
+  typename regex_traits<_CharT>::char_class_type __mask_;
+  typename regex_traits<_CharT>::char_class_type __neg_mask_;
+  bool __negate_;
+  bool __icase_;
+  bool __collate_;
+  bool __might_have_digraph_;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI
+  __bracket_expression(const _Traits& __traits, __node<_CharT>* __s, bool __negate, bool __icase, bool __collate)
+      : base(__s),
+        __traits_(__traits),
+        __mask_(),
+        __neg_mask_(),
+        __negate_(__negate),
+        __icase_(__icase),
+        __collate_(__collate),
+        __might_have_digraph_(__traits_.getloc().name() != "C") {}
+
+  __bracket_expression(const __bracket_expression&)            = delete;
+  __bracket_expression& operator=(const __bracket_expression&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+
+  _LIBCPP_HIDE_FROM_ABI bool __negated() const { return __negate_; }
+
+  _LIBCPP_HIDE_FROM_ABI void __add_char(_CharT __c) {
+    if (__icase_)
+      __chars_.push_back(__traits_.translate_nocase(__c));
+    else if (__collate_)
+      __chars_.push_back(__traits_.translate(__c));
+    else
+      __chars_.push_back(__c);
+  }
+  _LIBCPP_HIDE_FROM_ABI void __add_neg_char(_CharT __c) {
+    if (__icase_)
+      __neg_chars_.push_back(__traits_.translate_nocase(__c));
+    else if (__collate_)
+      __neg_chars_.push_back(__traits_.translate(__c));
+    else
+      __neg_chars_.push_back(__c);
+  }
+  _LIBCPP_HIDE_FROM_ABI void __add_range(string_type __b, string_type __e) {
+    if (__collate_) {
+      if (__icase_) {
+        for (size_t __i = 0; __i < __b.size(); ++__i)
+          __b[__i] = __traits_.translate_nocase(__b[__i]);
+        for (size_t __i = 0; __i < __e.size(); ++__i)
+          __e[__i] = __traits_.translate_nocase(__e[__i]);
+      } else {
+        for (size_t __i = 0; __i < __b.size(); ++__i)
+          __b[__i] = __traits_.translate(__b[__i]);
+        for (size_t __i = 0; __i < __e.size(); ++__i)
+          __e[__i] = __traits_.translate(__e[__i]);
+      }
+      __ranges_.push_back(
+          std::make_pair(__traits_.transform(__b.begin(), __b.end()), __traits_.transform(__e.begin(), __e.end())));
+    } else {
+      if (__b.size() != 1 || __e.size() != 1)
+        __throw_regex_error<regex_constants::error_range>();
+      if (__icase_) {
+        __b[0] = __traits_.translate_nocase(__b[0]);
+        __e[0] = __traits_.translate_nocase(__e[0]);
+      }
+      __ranges_.push_back(std::make_pair(std::move(__b), std::move(__e)));
+    }
+  }
+  _LIBCPP_HIDE_FROM_ABI void __add_digraph(_CharT __c1, _CharT __c2) {
+    if (__icase_)
+      __digraphs_.push_back(std::make_pair(__traits_.translate_nocase(__c1), __traits_.translate_nocase(__c2)));
+    else if (__collate_)
+      __digraphs_.push_back(std::make_pair(__traits_.translate(__c1), __traits_.translate(__c2)));
+    else
+      __digraphs_.push_back(std::make_pair(__c1, __c2));
+  }
+  _LIBCPP_HIDE_FROM_ABI void __add_equivalence(const string_type& __s) { __equivalences_.push_back(__s); }
+  _LIBCPP_HIDE_FROM_ABI void __add_class(typename regex_traits<_CharT>::char_class_type __mask) { __mask_ |= __mask; }
+  _LIBCPP_HIDE_FROM_ABI void __add_neg_class(typename regex_traits<_CharT>::char_class_type __mask) {
+    __neg_mask_ |= __mask;
+  }
+};
+
+template <class _CharT, class _Traits>
+void __bracket_expression<_CharT, _Traits>::__exec(__state& __s) const {
+  bool __found        = false;
+  unsigned __consumed = 0;
+  if (__s.__current_ != __s.__last_) {
+    ++__consumed;
+    if (__might_have_digraph_) {
+      const _CharT* __next = std::next(__s.__current_);
+      if (__next != __s.__last_) {
+        pair<_CharT, _CharT> __ch2(*__s.__current_, *__next);
+        if (__icase_) {
+          __ch2.first  = __traits_.translate_nocase(__ch2.first);
+          __ch2.second = __traits_.translate_nocase(__ch2.second);
+        } else if (__collate_) {
+          __ch2.first  = __traits_.translate(__ch2.first);
+          __ch2.second = __traits_.translate(__ch2.second);
+        }
+        if (!__traits_.lookup_collatename(&__ch2.first, &__ch2.first + 2).empty()) {
+          // __ch2 is a digraph in this locale
+          ++__consumed;
+          for (size_t __i = 0; __i < __digraphs_.size(); ++__i) {
+            if (__ch2 == __digraphs_[__i]) {
+              __found = true;
+              goto __exit;
+            }
+          }
+          if (__collate_ && !__ranges_.empty()) {
+            string_type __s2 = __traits_.transform(&__ch2.first, &__ch2.first + 2);
+            for (size_t __i = 0; __i < __ranges_.size(); ++__i) {
+              if (__ranges_[__i].first <= __s2 && __s2 <= __ranges_[__i].second) {
+                __found = true;
+                goto __exit;
+              }
+            }
+          }
+          if (!__equivalences_.empty()) {
+            string_type __s2 = __traits_.transform_primary(&__ch2.first, &__ch2.first + 2);
+            for (size_t __i = 0; __i < __equivalences_.size(); ++__i) {
+              if (__s2 == __equivalences_[__i]) {
+                __found = true;
+                goto __exit;
+              }
+            }
+          }
+          if (__traits_.isctype(__ch2.first, __mask_) && __traits_.isctype(__ch2.second, __mask_)) {
+            __found = true;
+            goto __exit;
+          }
+          if (!__traits_.isctype(__ch2.first, __neg_mask_) && !__traits_.isctype(__ch2.second, __neg_mask_)) {
+            __found = true;
+            goto __exit;
+          }
+          goto __exit;
+        }
+      }
+    }
+    // test *__s.__current_ as not a digraph
+    _CharT __ch = *__s.__current_;
+    if (__icase_)
+      __ch = __traits_.translate_nocase(__ch);
+    else if (__collate_)
+      __ch = __traits_.translate(__ch);
+    for (size_t __i = 0; __i < __chars_.size(); ++__i) {
+      if (__ch == __chars_[__i]) {
+        __found = true;
+        goto __exit;
+      }
+    }
+    // When there's at least one of __neg_chars_ and __neg_mask_, the set
+    // of "__found" chars is
+    //   union(complement(union(__neg_chars_, __neg_mask_)),
+    //         other cases...)
+    //
+    // It doesn't make sense to check this when there are no __neg_chars_
+    // and no __neg_mask_.
+    if (!(__neg_mask_ == 0 && __neg_chars_.empty())) {
+      const bool __in_neg_mask  = __traits_.isctype(__ch, __neg_mask_);
+      const bool __in_neg_chars = std::find(__neg_chars_.begin(), __neg_chars_.end(), __ch) != __neg_chars_.end();
+      if (!(__in_neg_mask || __in_neg_chars)) {
+        __found = true;
+        goto __exit;
+      }
+    }
+    if (!__ranges_.empty()) {
+      string_type __s2 = __collate_ ? __traits_.transform(&__ch, &__ch + 1) : string_type(1, __ch);
+      for (size_t __i = 0; __i < __ranges_.size(); ++__i) {
+        if (__ranges_[__i].first <= __s2 && __s2 <= __ranges_[__i].second) {
+          __found = true;
+          goto __exit;
+        }
+      }
+    }
+    if (!__equivalences_.empty()) {
+      string_type __s2 = __traits_.transform_primary(&__ch, &__ch + 1);
+      for (size_t __i = 0; __i < __equivalences_.size(); ++__i) {
+        if (__s2 == __equivalences_[__i]) {
+          __found = true;
+          goto __exit;
+        }
+      }
+    }
+    if (__traits_.isctype(__ch, __mask_)) {
+      __found = true;
+      goto __exit;
+    }
+  } else
+    __found = __negate_; // force reject
+__exit:
+  if (__found != __negate_) {
+    __s.__do_ = __state::__accept_and_consume;
+    __s.__current_ += __consumed;
+    __s.__node_ = this->first();
+  } else {
+    __s.__do_   = __state::__reject;
+    __s.__node_ = nullptr;
+  }
+}
+
+template <class _CharT, class _Traits>
+class __lookahead;
+
+template <class _CharT, class _Traits = regex_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_regex;
+
+typedef basic_regex<char> regex;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+typedef basic_regex<wchar_t> wregex;
+#endif
+
+template <class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_PREFERRED_NAME(regex)
+    _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wregex)) basic_regex {
+public:
+  // types:
+  typedef _CharT value_type;
+  typedef _Traits traits_type;
+  typedef typename _Traits::string_type string_type;
+  typedef regex_constants::syntax_option_type flag_type;
+  typedef typename _Traits::locale_type locale_type;
+
+private:
+  _Traits __traits_;
+  flag_type __flags_;
+  unsigned __marked_count_;
+  unsigned __loop_count_;
+  int __open_count_;
+  shared_ptr<__empty_state<_CharT> > __start_;
+  __owns_one_state<_CharT>* __end_;
+
+  typedef std::__state<_CharT> __state;
+  typedef std::__node<_CharT> __node;
+
+public:
+  // constants:
+  static const regex_constants::syntax_option_type icase      = regex_constants::icase;
+  static const regex_constants::syntax_option_type nosubs     = regex_constants::nosubs;
+  static const regex_constants::syntax_option_type optimize   = regex_constants::optimize;
+  static const regex_constants::syntax_option_type collate    = regex_constants::collate;
+  static const regex_constants::syntax_option_type ECMAScript = regex_constants::ECMAScript;
+  static const regex_constants::syntax_option_type basic      = regex_constants::basic;
+  static const regex_constants::syntax_option_type extended   = regex_constants::extended;
+  static const regex_constants::syntax_option_type awk        = regex_constants::awk;
+  static const regex_constants::syntax_option_type grep       = regex_constants::grep;
+  static const regex_constants::syntax_option_type egrep      = regex_constants::egrep;
+  static const regex_constants::syntax_option_type multiline  = regex_constants::multiline;
+
+  // construct/copy/destroy:
+  _LIBCPP_HIDE_FROM_ABI basic_regex()
+      : __flags_(regex_constants::ECMAScript),
+        __marked_count_(0),
+        __loop_count_(0),
+        __open_count_(0),
+        __end_(nullptr) {}
+  _LIBCPP_HIDE_FROM_ABI explicit basic_regex(const value_type* __p, flag_type __f = regex_constants::ECMAScript)
+      : __flags_(__f), __marked_count_(0), __loop_count_(0), __open_count_(0), __end_(nullptr) {
+    __init(__p, __p + __traits_.length(__p));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI basic_regex(const value_type* __p, size_t __len, flag_type __f = regex_constants::ECMAScript)
+      : __flags_(__f), __marked_count_(0), __loop_count_(0), __open_count_(0), __end_(nullptr) {
+    __init(__p, __p + __len);
+  }
+
+  //     basic_regex(const basic_regex&) = default;
+  //     basic_regex(basic_regex&&) = default;
+  template <class _ST, class _SA>
+  _LIBCPP_HIDE_FROM_ABI explicit basic_regex(const basic_string<value_type, _ST, _SA>& __p,
+                                             flag_type __f = regex_constants::ECMAScript)
+      : __flags_(__f), __marked_count_(0), __loop_count_(0), __open_count_(0), __end_(nullptr) {
+    __init(__p.begin(), __p.end());
+  }
+
+  template <class _ForwardIterator>
+  _LIBCPP_HIDE_FROM_ABI
+  basic_regex(_ForwardIterator __first, _ForwardIterator __last, flag_type __f = regex_constants::ECMAScript)
+      : __flags_(__f), __marked_count_(0), __loop_count_(0), __open_count_(0), __end_(nullptr) {
+    __init(__first, __last);
+  }
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI basic_regex(initializer_list<value_type> __il, flag_type __f = regex_constants::ECMAScript)
+      : __flags_(__f), __marked_count_(0), __loop_count_(0), __open_count_(0), __end_(nullptr) {
+    __init(__il.begin(), __il.end());
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  //    ~basic_regex() = default;
+
+  //     basic_regex& operator=(const basic_regex&) = default;
+  //     basic_regex& operator=(basic_regex&&) = default;
+  _LIBCPP_HIDE_FROM_ABI basic_regex& operator=(const value_type* __p) { return assign(__p); }
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI basic_regex& operator=(initializer_list<value_type> __il) { return assign(__il); }
+#endif // _LIBCPP_CXX03_LANG
+  template <class _ST, class _SA>
+  _LIBCPP_HIDE_FROM_ABI basic_regex& operator=(const basic_string<value_type, _ST, _SA>& __p) {
+    return assign(__p);
+  }
+
+  // assign:
+  _LIBCPP_HIDE_FROM_ABI basic_regex& assign(const basic_regex& __that) { return *this = __that; }
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI basic_regex& assign(basic_regex&& __that) _NOEXCEPT { return *this = std::move(__that); }
+#endif
+  _LIBCPP_HIDE_FROM_ABI basic_regex& assign(const value_type* __p, flag_type __f = regex_constants::ECMAScript) {
+    return assign(__p, __p + __traits_.length(__p), __f);
+  }
+  _LIBCPP_HIDE_FROM_ABI basic_regex&
+  assign(const value_type* __p, size_t __len, flag_type __f = regex_constants::ECMAScript) {
+    return assign(__p, __p + __len, __f);
+  }
+  template <class _ST, class _SA>
+  _LIBCPP_HIDE_FROM_ABI basic_regex&
+  assign(const basic_string<value_type, _ST, _SA>& __s, flag_type __f = regex_constants::ECMAScript) {
+    return assign(__s.begin(), __s.end(), __f);
+  }
+
+  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI basic_regex&
+  assign(_InputIterator __first, _InputIterator __last, flag_type __f = regex_constants::ECMAScript) {
+    basic_string<_CharT> __t(__first, __last);
+    return assign(__t.begin(), __t.end(), __f);
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI void __member_init(flag_type __f) {
+    __flags_        = __f;
+    __marked_count_ = 0;
+    __loop_count_   = 0;
+    __open_count_   = 0;
+    __end_          = nullptr;
+  }
+
+public:
+  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI basic_regex&
+  assign(_ForwardIterator __first, _ForwardIterator __last, flag_type __f = regex_constants::ECMAScript) {
+    return assign(basic_regex(__first, __last, __f));
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI basic_regex&
+  assign(initializer_list<value_type> __il, flag_type __f = regex_constants::ECMAScript) {
+    return assign(__il.begin(), __il.end(), __f);
+  }
+
+#endif // _LIBCPP_CXX03_LANG
+
+  // const operations:
+  _LIBCPP_HIDE_FROM_ABI unsigned mark_count() const { return __marked_count_; }
+  _LIBCPP_HIDE_FROM_ABI flag_type flags() const { return __flags_; }
+
+  // locale:
+  _LIBCPP_HIDE_FROM_ABI locale_type imbue(locale_type __loc) {
+    __member_init(ECMAScript);
+    __start_.reset();
+    return __traits_.imbue(__loc);
+  }
+  _LIBCPP_HIDE_FROM_ABI locale_type getloc() const { return __traits_.getloc(); }
+
+  // swap:
+  void swap(basic_regex& __r);
+
+private:
+  _LIBCPP_HIDE_FROM_ABI unsigned __loop_count() const { return __loop_count_; }
+
+  _LIBCPP_HIDE_FROM_ABI bool __use_multiline() const {
+    return __get_grammar(__flags_) == ECMAScript && (__flags_ & multiline);
+  }
+
+  template <class _ForwardIterator>
+  void __init(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_basic_reg_exp(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_RE_expression(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_simple_RE(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_nondupl_RE(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_one_char_or_coll_elem_RE(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_Back_open_paren(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_Back_close_paren(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_Back_open_brace(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_Back_close_brace(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_BACKREF(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_ORD_CHAR(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_QUOTED_CHAR(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_RE_dupl_symbol(
+      _ForwardIterator __first,
+      _ForwardIterator __last,
+      __owns_one_state<_CharT>* __s,
+      unsigned __mexp_begin,
+      unsigned __mexp_end);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_ERE_dupl_symbol(
+      _ForwardIterator __first,
+      _ForwardIterator __last,
+      __owns_one_state<_CharT>* __s,
+      unsigned __mexp_begin,
+      unsigned __mexp_end);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_bracket_expression(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator
+  __parse_follow_list(_ForwardIterator __first, _ForwardIterator __last, __bracket_expression<_CharT, _Traits>* __ml);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_expression_term(
+      _ForwardIterator __first, _ForwardIterator __last, __bracket_expression<_CharT, _Traits>* __ml);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_equivalence_class(
+      _ForwardIterator __first, _ForwardIterator __last, __bracket_expression<_CharT, _Traits>* __ml);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_character_class(
+      _ForwardIterator __first, _ForwardIterator __last, __bracket_expression<_CharT, _Traits>* __ml);
+  template <class _ForwardIterator>
+  _ForwardIterator
+  __parse_collating_symbol(_ForwardIterator __first, _ForwardIterator __last, basic_string<_CharT>& __col_sym);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_DUP_COUNT(_ForwardIterator __first, _ForwardIterator __last, int& __c);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_extended_reg_exp(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_ERE_branch(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_ERE_expression(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_one_char_or_coll_elem_ERE(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_ORD_CHAR_ERE(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_QUOTED_CHAR_ERE(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_ecma_exp(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_alternative(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_term(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_assertion(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_atom(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_atom_escape(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_decimal_escape(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_character_class_escape(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator
+  __parse_character_escape(_ForwardIterator __first, _ForwardIterator __last, basic_string<_CharT>* __str = nullptr);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_pattern_character(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_grep(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_egrep(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator>
+  _ForwardIterator __parse_class_escape(
+      _ForwardIterator __first,
+      _ForwardIterator __last,
+      basic_string<_CharT>& __str,
+      __bracket_expression<_CharT, _Traits>* __ml);
+  template <class _ForwardIterator>
+  _ForwardIterator
+  __parse_awk_escape(_ForwardIterator __first, _ForwardIterator __last, basic_string<_CharT>* __str = nullptr);
+
+  bool __test_back_ref(_CharT);
+
+  _LIBCPP_HIDE_FROM_ABI void __push_l_anchor();
+  void __push_r_anchor();
+  void __push_match_any();
+  void __push_match_any_but_newline();
+  _LIBCPP_HIDE_FROM_ABI void __push_greedy_inf_repeat(
+      size_t __min, __owns_one_state<_CharT>* __s, unsigned __mexp_begin = 0, unsigned __mexp_end = 0) {
+    __push_loop(__min, numeric_limits<size_t>::max(), __s, __mexp_begin, __mexp_end);
+  }
+  _LIBCPP_HIDE_FROM_ABI void __push_nongreedy_inf_repeat(
+      size_t __min, __owns_one_state<_CharT>* __s, unsigned __mexp_begin = 0, unsigned __mexp_end = 0) {
+    __push_loop(__min, numeric_limits<size_t>::max(), __s, __mexp_begin, __mexp_end, false);
+  }
+  void __push_loop(size_t __min,
+                   size_t __max,
+                   __owns_one_state<_CharT>* __s,
+                   size_t __mexp_begin = 0,
+                   size_t __mexp_end   = 0,
+                   bool __greedy       = true);
+  __bracket_expression<_CharT, _Traits>* __start_matching_list(bool __negate);
+  void __push_char(value_type __c);
+  void __push_back_ref(int __i);
+  void __push_alternation(__owns_one_state<_CharT>* __sa, __owns_one_state<_CharT>* __sb);
+  void __push_begin_marked_subexpression();
+  void __push_end_marked_subexpression(unsigned);
+  void __push_empty();
+  void __push_word_boundary(bool);
+  void __push_lookahead(const basic_regex&, bool, unsigned);
+
+  template <class _Allocator>
+  bool __search(const _CharT* __first,
+                const _CharT* __last,
+                match_results<const _CharT*, _Allocator>& __m,
+                regex_constants::match_flag_type __flags) const;
+
+  template <class _Allocator>
+  bool __match_at_start(const _CharT* __first,
+                        const _CharT* __last,
+                        match_results<const _CharT*, _Allocator>& __m,
+                        regex_constants::match_flag_type __flags,
+                        bool) const;
+  template <class _Allocator>
+  bool __match_at_start_ecma(
+      const _CharT* __first,
+      const _CharT* __last,
+      match_results<const _CharT*, _Allocator>& __m,
+      regex_constants::match_flag_type __flags,
+      bool) const;
+  template <class _Allocator>
+  bool __match_at_start_posix_nosubs(
+      const _CharT* __first,
+      const _CharT* __last,
+      match_results<const _CharT*, _Allocator>& __m,
+      regex_constants::match_flag_type __flags,
+      bool) const;
+  template <class _Allocator>
+  bool __match_at_start_posix_subs(
+      const _CharT* __first,
+      const _CharT* __last,
+      match_results<const _CharT*, _Allocator>& __m,
+      regex_constants::match_flag_type __flags,
+      bool) const;
+
+  template <class _Bp, class _Ap, class _Cp, class _Tp>
+  friend bool
+  regex_search(_Bp, _Bp, match_results<_Bp, _Ap>&, const basic_regex<_Cp, _Tp>&, regex_constants::match_flag_type);
+
+  template <class _Ap, class _Cp, class _Tp>
+  friend bool
+  regex_search(const _Cp*,
+               const _Cp*,
+               match_results<const _Cp*, _Ap>&,
+               const basic_regex<_Cp, _Tp>&,
+               regex_constants::match_flag_type);
+
+  template <class _Bp, class _Cp, class _Tp>
+  friend bool regex_search(_Bp, _Bp, const basic_regex<_Cp, _Tp>&, regex_constants::match_flag_type);
+
+  template <class _Cp, class _Tp>
+  friend bool regex_search(const _Cp*, const _Cp*, const basic_regex<_Cp, _Tp>&, regex_constants::match_flag_type);
+
+  template <class _Cp, class _Ap, class _Tp>
+  friend bool regex_search(
+      const _Cp*, match_results<const _Cp*, _Ap>&, const basic_regex<_Cp, _Tp>&, regex_constants::match_flag_type);
+
+  template <class _ST, class _SA, class _Cp, class _Tp>
+  friend bool regex_search(const basic_string<_Cp, _ST, _SA>& __s,
+                           const basic_regex<_Cp, _Tp>& __e,
+                           regex_constants::match_flag_type __flags);
+
+  template <class _ST, class _SA, class _Ap, class _Cp, class _Tp>
+  friend bool regex_search(const basic_string<_Cp, _ST, _SA>& __s,
+                           match_results<typename basic_string<_Cp, _ST, _SA>::const_iterator, _Ap>&,
+                           const basic_regex<_Cp, _Tp>& __e,
+                           regex_constants::match_flag_type __flags);
+
+  template <class _Iter, class _Ap, class _Cp, class _Tp>
+  friend bool
+  regex_search(__wrap_iter<_Iter> __first,
+               __wrap_iter<_Iter> __last,
+               match_results<__wrap_iter<_Iter>, _Ap>& __m,
+               const basic_regex<_Cp, _Tp>& __e,
+               regex_constants::match_flag_type __flags);
+
+  template <class, class>
+  friend class __lookahead;
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+basic_regex(_ForwardIterator, _ForwardIterator, regex_constants::syntax_option_type = regex_constants::ECMAScript)
+    -> basic_regex<typename iterator_traits<_ForwardIterator>::value_type>;
+#endif
+
+template <class _CharT, class _Traits>
+const regex_constants::syntax_option_type basic_regex<_CharT, _Traits>::icase;
+template <class _CharT, class _Traits>
+const regex_constants::syntax_option_type basic_regex<_CharT, _Traits>::nosubs;
+template <class _CharT, class _Traits>
+const regex_constants::syntax_option_type basic_regex<_CharT, _Traits>::optimize;
+template <class _CharT, class _Traits>
+const regex_constants::syntax_option_type basic_regex<_CharT, _Traits>::collate;
+template <class _CharT, class _Traits>
+const regex_constants::syntax_option_type basic_regex<_CharT, _Traits>::ECMAScript;
+template <class _CharT, class _Traits>
+const regex_constants::syntax_option_type basic_regex<_CharT, _Traits>::basic;
+template <class _CharT, class _Traits>
+const regex_constants::syntax_option_type basic_regex<_CharT, _Traits>::extended;
+template <class _CharT, class _Traits>
+const regex_constants::syntax_option_type basic_regex<_CharT, _Traits>::awk;
+template <class _CharT, class _Traits>
+const regex_constants::syntax_option_type basic_regex<_CharT, _Traits>::grep;
+template <class _CharT, class _Traits>
+const regex_constants::syntax_option_type basic_regex<_CharT, _Traits>::egrep;
+
+template <class _CharT, class _Traits>
+void basic_regex<_CharT, _Traits>::swap(basic_regex& __r) {
+  using std::swap;
+  swap(__traits_, __r.__traits_);
+  swap(__flags_, __r.__flags_);
+  swap(__marked_count_, __r.__marked_count_);
+  swap(__loop_count_, __r.__loop_count_);
+  swap(__open_count_, __r.__open_count_);
+  swap(__start_, __r.__start_);
+  swap(__end_, __r.__end_);
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI void swap(basic_regex<_CharT, _Traits>& __x, basic_regex<_CharT, _Traits>& __y) {
+  return __x.swap(__y);
+}
+
+// __lookahead
+
+template <class _CharT, class _Traits>
+class __lookahead : public __owns_one_state<_CharT> {
+  typedef __owns_one_state<_CharT> base;
+
+  basic_regex<_CharT, _Traits> __exp_;
+  unsigned __mexp_;
+  bool __invert_;
+
+public:
+  typedef std::__state<_CharT> __state;
+
+  _LIBCPP_HIDE_FROM_ABI
+  __lookahead(const basic_regex<_CharT, _Traits>& __exp, bool __invert, __node<_CharT>* __s, unsigned __mexp)
+      : base(__s), __exp_(__exp), __mexp_(__mexp), __invert_(__invert) {}
+
+  __lookahead(const __lookahead&)            = delete;
+  __lookahead& operator=(const __lookahead&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __exec(__state&) const;
+};
+
+template <class _CharT, class _Traits>
+void __lookahead<_CharT, _Traits>::__exec(__state& __s) const {
+  match_results<const _CharT*> __m;
+  __m.__init(1 + __exp_.mark_count(), __s.__current_, __s.__last_);
+  bool __matched = __exp_.__match_at_start_ecma(
+      __s.__current_,
+      __s.__last_,
+      __m,
+      (__s.__flags_ | regex_constants::match_continuous) & ~regex_constants::__full_match,
+      __s.__at_first_ && __s.__current_ == __s.__first_);
+  if (__matched != __invert_) {
+    __s.__do_   = __state::__accept_but_not_consume;
+    __s.__node_ = this->first();
+    for (unsigned __i = 1; __i < __m.size(); ++__i) {
+      __s.__sub_matches_[__mexp_ + __i - 1] = __m.__matches_[__i];
+    }
+  } else {
+    __s.__do_   = __state::__reject;
+    __s.__node_ = nullptr;
+  }
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+void basic_regex<_CharT, _Traits>::__init(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__get_grammar(__flags_) == 0)
+    __flags_ |= regex_constants::ECMAScript;
+  _ForwardIterator __temp = __parse(__first, __last);
+  if (__temp != __last)
+    __throw_regex_error<regex_constants::__re_err_parse>();
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse(_ForwardIterator __first, _ForwardIterator __last) {
+  {
+    unique_ptr<__node> __h(new __end_state<_CharT>);
+    __start_.reset(new __empty_state<_CharT>(__h.get()));
+    __h.release();
+    __end_ = __start_.get();
+  }
+  switch (__get_grammar(__flags_)) {
+  case ECMAScript:
+    __first = __parse_ecma_exp(__first, __last);
+    break;
+  case basic:
+    __first = __parse_basic_reg_exp(__first, __last);
+    break;
+  case extended:
+  case awk:
+    __first = __parse_extended_reg_exp(__first, __last);
+    break;
+  case grep:
+    __first = __parse_grep(__first, __last);
+    break;
+  case egrep:
+    __first = __parse_egrep(__first, __last);
+    break;
+  default:
+    __throw_regex_error<regex_constants::__re_err_grammar>();
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_basic_reg_exp(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    if (*__first == '^') {
+      __push_l_anchor();
+      ++__first;
+    }
+    if (__first != __last) {
+      __first = __parse_RE_expression(__first, __last);
+      if (__first != __last) {
+        _ForwardIterator __temp = std::next(__first);
+        if (__temp == __last && *__first == '$') {
+          __push_r_anchor();
+          ++__first;
+        }
+      }
+    }
+    if (__first != __last)
+      __throw_regex_error<regex_constants::__re_err_empty>();
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_extended_reg_exp(_ForwardIterator __first, _ForwardIterator __last) {
+  __owns_one_state<_CharT>* __sa = __end_;
+  _ForwardIterator __temp        = __parse_ERE_branch(__first, __last);
+  if (__temp == __first)
+    __throw_regex_error<regex_constants::__re_err_empty>();
+  __first = __temp;
+  while (__first != __last && *__first == '|') {
+    __owns_one_state<_CharT>* __sb = __end_;
+    __temp                         = __parse_ERE_branch(++__first, __last);
+    if (__temp == __first)
+      __throw_regex_error<regex_constants::__re_err_empty>();
+    __push_alternation(__sa, __sb);
+    __first = __temp;
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_ERE_branch(_ForwardIterator __first, _ForwardIterator __last) {
+  _ForwardIterator __temp = __parse_ERE_expression(__first, __last);
+  if (__temp == __first)
+    __throw_regex_error<regex_constants::__re_err_empty>();
+  do {
+    __first = __temp;
+    __temp  = __parse_ERE_expression(__first, __last);
+  } while (__temp != __first);
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_ERE_expression(_ForwardIterator __first, _ForwardIterator __last) {
+  __owns_one_state<_CharT>* __e = __end_;
+  unsigned __mexp_begin         = __marked_count_;
+  _ForwardIterator __temp       = __parse_one_char_or_coll_elem_ERE(__first, __last);
+  if (__temp == __first && __temp != __last) {
+    switch (*__temp) {
+    case '^':
+      __push_l_anchor();
+      ++__temp;
+      break;
+    case '$':
+      __push_r_anchor();
+      ++__temp;
+      break;
+    case '(':
+      __push_begin_marked_subexpression();
+      unsigned __temp_count = __marked_count_;
+      ++__open_count_;
+      __temp = __parse_extended_reg_exp(++__temp, __last);
+      if (__temp == __last || *__temp != ')')
+        __throw_regex_error<regex_constants::error_paren>();
+      __push_end_marked_subexpression(__temp_count);
+      --__open_count_;
+      ++__temp;
+      break;
+    }
+  }
+  if (__temp != __first)
+    __temp = __parse_ERE_dupl_symbol(__temp, __last, __e, __mexp_begin + 1, __marked_count_ + 1);
+  __first = __temp;
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_RE_expression(_ForwardIterator __first, _ForwardIterator __last) {
+  while (true) {
+    _ForwardIterator __temp = __parse_simple_RE(__first, __last);
+    if (__temp == __first)
+      break;
+    __first = __temp;
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_simple_RE(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    __owns_one_state<_CharT>* __e = __end_;
+    unsigned __mexp_begin         = __marked_count_;
+    _ForwardIterator __temp       = __parse_nondupl_RE(__first, __last);
+    if (__temp != __first)
+      __first = __parse_RE_dupl_symbol(__temp, __last, __e, __mexp_begin + 1, __marked_count_ + 1);
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_nondupl_RE(_ForwardIterator __first, _ForwardIterator __last) {
+  _ForwardIterator __temp = __first;
+  __first                 = __parse_one_char_or_coll_elem_RE(__first, __last);
+  if (__temp == __first) {
+    __temp = __parse_Back_open_paren(__first, __last);
+    if (__temp != __first) {
+      __push_begin_marked_subexpression();
+      unsigned __temp_count = __marked_count_;
+      __first               = __parse_RE_expression(__temp, __last);
+      __temp                = __parse_Back_close_paren(__first, __last);
+      if (__temp == __first)
+        __throw_regex_error<regex_constants::error_paren>();
+      __push_end_marked_subexpression(__temp_count);
+      __first = __temp;
+    } else
+      __first = __parse_BACKREF(__first, __last);
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_one_char_or_coll_elem_RE(_ForwardIterator __first, _ForwardIterator __last) {
+  _ForwardIterator __temp = __parse_ORD_CHAR(__first, __last);
+  if (__temp == __first) {
+    __temp = __parse_QUOTED_CHAR(__first, __last);
+    if (__temp == __first) {
+      if (__temp != __last && *__temp == '.') {
+        __push_match_any();
+        ++__temp;
+      } else
+        __temp = __parse_bracket_expression(__first, __last);
+    }
+  }
+  __first = __temp;
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_one_char_or_coll_elem_ERE(_ForwardIterator __first, _ForwardIterator __last) {
+  _ForwardIterator __temp = __parse_ORD_CHAR_ERE(__first, __last);
+  if (__temp == __first) {
+    __temp = __parse_QUOTED_CHAR_ERE(__first, __last);
+    if (__temp == __first) {
+      if (__temp != __last && *__temp == '.') {
+        __push_match_any();
+        ++__temp;
+      } else
+        __temp = __parse_bracket_expression(__first, __last);
+    }
+  }
+  __first = __temp;
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_Back_open_paren(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    _ForwardIterator __temp = std::next(__first);
+    if (__temp != __last) {
+      if (*__first == '\\' && *__temp == '(')
+        __first = ++__temp;
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_Back_close_paren(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    _ForwardIterator __temp = std::next(__first);
+    if (__temp != __last) {
+      if (*__first == '\\' && *__temp == ')')
+        __first = ++__temp;
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_Back_open_brace(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    _ForwardIterator __temp = std::next(__first);
+    if (__temp != __last) {
+      if (*__first == '\\' && *__temp == '{')
+        __first = ++__temp;
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_Back_close_brace(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    _ForwardIterator __temp = std::next(__first);
+    if (__temp != __last) {
+      if (*__first == '\\' && *__temp == '}')
+        __first = ++__temp;
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_BACKREF(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    _ForwardIterator __temp = std::next(__first);
+    if (__temp != __last && *__first == '\\' && __test_back_ref(*__temp))
+      __first = ++__temp;
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_ORD_CHAR(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    _ForwardIterator __temp = std::next(__first);
+    if (__temp == __last && *__first == '$')
+      return __first;
+    // Not called inside a bracket
+    if (*__first == '.' || *__first == '\\' || *__first == '[')
+      return __first;
+    __push_char(*__first);
+    ++__first;
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_ORD_CHAR_ERE(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    switch (*__first) {
+    case '^':
+    case '.':
+    case '[':
+    case '$':
+    case '(':
+    case '|':
+    case '*':
+    case '+':
+    case '?':
+    case '{':
+    case '\\':
+      break;
+    case ')':
+      if (__open_count_ == 0) {
+        __push_char(*__first);
+        ++__first;
+      }
+      break;
+    default:
+      __push_char(*__first);
+      ++__first;
+      break;
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_QUOTED_CHAR(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    _ForwardIterator __temp = std::next(__first);
+    if (__temp != __last) {
+      if (*__first == '\\') {
+        switch (*__temp) {
+        case '^':
+        case '.':
+        case '*':
+        case '[':
+        case '$':
+        case '\\':
+          __push_char(*__temp);
+          __first = ++__temp;
+          break;
+        }
+      }
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_QUOTED_CHAR_ERE(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    _ForwardIterator __temp = std::next(__first);
+    if (__temp != __last) {
+      if (*__first == '\\') {
+        switch (*__temp) {
+        case '^':
+        case '.':
+        case '*':
+        case '[':
+        case '$':
+        case '\\':
+        case '(':
+        case ')':
+        case '|':
+        case '+':
+        case '?':
+        case '{':
+        case '}':
+          __push_char(*__temp);
+          __first = ++__temp;
+          break;
+        default:
+          if (__get_grammar(__flags_) == awk)
+            __first = __parse_awk_escape(++__first, __last);
+          else if (__test_back_ref(*__temp))
+            __first = ++__temp;
+          break;
+        }
+      }
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_RE_dupl_symbol(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    __owns_one_state<_CharT>* __s,
+    unsigned __mexp_begin,
+    unsigned __mexp_end) {
+  if (__first != __last) {
+    if (*__first == '*') {
+      __push_greedy_inf_repeat(0, __s, __mexp_begin, __mexp_end);
+      ++__first;
+    } else {
+      _ForwardIterator __temp = __parse_Back_open_brace(__first, __last);
+      if (__temp != __first) {
+        int __min = 0;
+        __first   = __temp;
+        __temp    = __parse_DUP_COUNT(__first, __last, __min);
+        if (__temp == __first)
+          __throw_regex_error<regex_constants::error_badbrace>();
+        __first = __temp;
+        if (__first == __last)
+          __throw_regex_error<regex_constants::error_brace>();
+        if (*__first != ',') {
+          __temp = __parse_Back_close_brace(__first, __last);
+          if (__temp == __first)
+            __throw_regex_error<regex_constants::error_brace>();
+          __push_loop(__min, __min, __s, __mexp_begin, __mexp_end, true);
+          __first = __temp;
+        } else {
+          ++__first; // consume ','
+          int __max = -1;
+          __first   = __parse_DUP_COUNT(__first, __last, __max);
+          __temp    = __parse_Back_close_brace(__first, __last);
+          if (__temp == __first)
+            __throw_regex_error<regex_constants::error_brace>();
+          if (__max == -1)
+            __push_greedy_inf_repeat(__min, __s, __mexp_begin, __mexp_end);
+          else {
+            if (__max < __min)
+              __throw_regex_error<regex_constants::error_badbrace>();
+            __push_loop(__min, __max, __s, __mexp_begin, __mexp_end, true);
+          }
+          __first = __temp;
+        }
+      }
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_ERE_dupl_symbol(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    __owns_one_state<_CharT>* __s,
+    unsigned __mexp_begin,
+    unsigned __mexp_end) {
+  if (__first != __last) {
+    unsigned __grammar = __get_grammar(__flags_);
+    switch (*__first) {
+    case '*':
+      ++__first;
+      if (__grammar == ECMAScript && __first != __last && *__first == '?') {
+        ++__first;
+        __push_nongreedy_inf_repeat(0, __s, __mexp_begin, __mexp_end);
+      } else
+        __push_greedy_inf_repeat(0, __s, __mexp_begin, __mexp_end);
+      break;
+    case '+':
+      ++__first;
+      if (__grammar == ECMAScript && __first != __last && *__first == '?') {
+        ++__first;
+        __push_nongreedy_inf_repeat(1, __s, __mexp_begin, __mexp_end);
+      } else
+        __push_greedy_inf_repeat(1, __s, __mexp_begin, __mexp_end);
+      break;
+    case '?':
+      ++__first;
+      if (__grammar == ECMAScript && __first != __last && *__first == '?') {
+        ++__first;
+        __push_loop(0, 1, __s, __mexp_begin, __mexp_end, false);
+      } else
+        __push_loop(0, 1, __s, __mexp_begin, __mexp_end);
+      break;
+    case '{': {
+      int __min;
+      _ForwardIterator __temp = __parse_DUP_COUNT(++__first, __last, __min);
+      if (__temp == __first)
+        __throw_regex_error<regex_constants::error_badbrace>();
+      __first = __temp;
+      if (__first == __last)
+        __throw_regex_error<regex_constants::error_brace>();
+      switch (*__first) {
+      case '}':
+        ++__first;
+        if (__grammar == ECMAScript && __first != __last && *__first == '?') {
+          ++__first;
+          __push_loop(__min, __min, __s, __mexp_begin, __mexp_end, false);
+        } else
+          __push_loop(__min, __min, __s, __mexp_begin, __mexp_end);
+        break;
+      case ',':
+        ++__first;
+        if (__first == __last)
+          __throw_regex_error<regex_constants::error_badbrace>();
+        if (*__first == '}') {
+          ++__first;
+          if (__grammar == ECMAScript && __first != __last && *__first == '?') {
+            ++__first;
+            __push_nongreedy_inf_repeat(__min, __s, __mexp_begin, __mexp_end);
+          } else
+            __push_greedy_inf_repeat(__min, __s, __mexp_begin, __mexp_end);
+        } else {
+          int __max = -1;
+          __temp    = __parse_DUP_COUNT(__first, __last, __max);
+          if (__temp == __first)
+            __throw_regex_error<regex_constants::error_brace>();
+          __first = __temp;
+          if (__first == __last || *__first != '}')
+            __throw_regex_error<regex_constants::error_brace>();
+          ++__first;
+          if (__max < __min)
+            __throw_regex_error<regex_constants::error_badbrace>();
+          if (__grammar == ECMAScript && __first != __last && *__first == '?') {
+            ++__first;
+            __push_loop(__min, __max, __s, __mexp_begin, __mexp_end, false);
+          } else
+            __push_loop(__min, __max, __s, __mexp_begin, __mexp_end);
+        }
+        break;
+      default:
+        __throw_regex_error<regex_constants::error_badbrace>();
+      }
+    } break;
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_bracket_expression(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last && *__first == '[') {
+    ++__first;
+    if (__first == __last)
+      __throw_regex_error<regex_constants::error_brack>();
+    bool __negate = false;
+    if (*__first == '^') {
+      ++__first;
+      __negate = true;
+    }
+    __bracket_expression<_CharT, _Traits>* __ml = __start_matching_list(__negate);
+    // __ml owned by *this
+    if (__first == __last)
+      __throw_regex_error<regex_constants::error_brack>();
+    if (__get_grammar(__flags_) != ECMAScript && *__first == ']') {
+      __ml->__add_char(']');
+      ++__first;
+    }
+    __first = __parse_follow_list(__first, __last, __ml);
+    if (__first == __last)
+      __throw_regex_error<regex_constants::error_brack>();
+    if (*__first == '-') {
+      __ml->__add_char('-');
+      ++__first;
+    }
+    if (__first == __last || *__first != ']')
+      __throw_regex_error<regex_constants::error_brack>();
+    ++__first;
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_follow_list(
+    _ForwardIterator __first, _ForwardIterator __last, __bracket_expression<_CharT, _Traits>* __ml) {
+  if (__first != __last) {
+    while (true) {
+      _ForwardIterator __temp = __parse_expression_term(__first, __last, __ml);
+      if (__temp == __first)
+        break;
+      __first = __temp;
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_expression_term(
+    _ForwardIterator __first, _ForwardIterator __last, __bracket_expression<_CharT, _Traits>* __ml) {
+  if (__first != __last && *__first != ']') {
+    _ForwardIterator __temp = std::next(__first);
+    basic_string<_CharT> __start_range;
+    if (__temp != __last && *__first == '[') {
+      if (*__temp == '=')
+        return __parse_equivalence_class(++__temp, __last, __ml);
+      else if (*__temp == ':')
+        return __parse_character_class(++__temp, __last, __ml);
+      else if (*__temp == '.')
+        __first = __parse_collating_symbol(++__temp, __last, __start_range);
+    }
+    unsigned __grammar = __get_grammar(__flags_);
+    if (__start_range.empty()) {
+      if ((__grammar == ECMAScript || __grammar == awk) && *__first == '\\') {
+        if (__grammar == ECMAScript)
+          __first = __parse_class_escape(++__first, __last, __start_range, __ml);
+        else
+          __first = __parse_awk_escape(++__first, __last, &__start_range);
+      } else {
+        __start_range = *__first;
+        ++__first;
+      }
+    }
+    if (__first != __last && *__first != ']') {
+      __temp = std::next(__first);
+      if (__temp != __last && *__first == '-' && *__temp != ']') {
+        // parse a range
+        basic_string<_CharT> __end_range;
+        __first = __temp;
+        ++__temp;
+        if (__temp != __last && *__first == '[' && *__temp == '.')
+          __first = __parse_collating_symbol(++__temp, __last, __end_range);
+        else {
+          if ((__grammar == ECMAScript || __grammar == awk) && *__first == '\\') {
+            if (__grammar == ECMAScript)
+              __first = __parse_class_escape(++__first, __last, __end_range, __ml);
+            else
+              __first = __parse_awk_escape(++__first, __last, &__end_range);
+          } else {
+            __end_range = *__first;
+            ++__first;
+          }
+        }
+        __ml->__add_range(std::move(__start_range), std::move(__end_range));
+      } else if (!__start_range.empty()) {
+        if (__start_range.size() == 1)
+          __ml->__add_char(__start_range[0]);
+        else
+          __ml->__add_digraph(__start_range[0], __start_range[1]);
+      }
+    } else if (!__start_range.empty()) {
+      if (__start_range.size() == 1)
+        __ml->__add_char(__start_range[0]);
+      else
+        __ml->__add_digraph(__start_range[0], __start_range[1]);
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_class_escape(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    basic_string<_CharT>& __str,
+    __bracket_expression<_CharT, _Traits>* __ml) {
+  if (__first == __last)
+    __throw_regex_error<regex_constants::error_escape>();
+  switch (*__first) {
+  case 0:
+    __str = *__first;
+    return ++__first;
+  case 'b':
+    __str = _CharT(8);
+    return ++__first;
+  case 'd':
+    __ml->__add_class(ctype_base::digit);
+    return ++__first;
+  case 'D':
+    __ml->__add_neg_class(ctype_base::digit);
+    return ++__first;
+  case 's':
+    __ml->__add_class(ctype_base::space);
+    return ++__first;
+  case 'S':
+    __ml->__add_neg_class(ctype_base::space);
+    return ++__first;
+  case 'w':
+    __ml->__add_class(ctype_base::alnum);
+    __ml->__add_char('_');
+    return ++__first;
+  case 'W':
+    __ml->__add_neg_class(ctype_base::alnum);
+    __ml->__add_neg_char('_');
+    return ++__first;
+  }
+  __first = __parse_character_escape(__first, __last, &__str);
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_awk_escape(
+    _ForwardIterator __first, _ForwardIterator __last, basic_string<_CharT>* __str) {
+  if (__first == __last)
+    __throw_regex_error<regex_constants::error_escape>();
+  switch (*__first) {
+  case '\\':
+  case '"':
+  case '/':
+    if (__str)
+      *__str = *__first;
+    else
+      __push_char(*__first);
+    return ++__first;
+  case 'a':
+    if (__str)
+      *__str = _CharT(7);
+    else
+      __push_char(_CharT(7));
+    return ++__first;
+  case 'b':
+    if (__str)
+      *__str = _CharT(8);
+    else
+      __push_char(_CharT(8));
+    return ++__first;
+  case 'f':
+    if (__str)
+      *__str = _CharT(0xC);
+    else
+      __push_char(_CharT(0xC));
+    return ++__first;
+  case 'n':
+    if (__str)
+      *__str = _CharT(0xA);
+    else
+      __push_char(_CharT(0xA));
+    return ++__first;
+  case 'r':
+    if (__str)
+      *__str = _CharT(0xD);
+    else
+      __push_char(_CharT(0xD));
+    return ++__first;
+  case 't':
+    if (__str)
+      *__str = _CharT(0x9);
+    else
+      __push_char(_CharT(0x9));
+    return ++__first;
+  case 'v':
+    if (__str)
+      *__str = _CharT(0xB);
+    else
+      __push_char(_CharT(0xB));
+    return ++__first;
+  }
+  if ('0' <= *__first && *__first <= '7') {
+    unsigned __val = *__first - '0';
+    if (++__first != __last && ('0' <= *__first && *__first <= '7')) {
+      __val = 8 * __val + *__first - '0';
+      if (++__first != __last && ('0' <= *__first && *__first <= '7'))
+        __val = 8 * __val + *__first++ - '0';
+    }
+    if (__str)
+      *__str = _CharT(__val);
+    else
+      __push_char(_CharT(__val));
+  } else
+    __throw_regex_error<regex_constants::error_escape>();
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_equivalence_class(
+    _ForwardIterator __first, _ForwardIterator __last, __bracket_expression<_CharT, _Traits>* __ml) {
+  // Found [=
+  //   This means =] must exist
+  value_type __equal_close[2] = {'=', ']'};
+  _ForwardIterator __temp     = std::search(__first, __last, __equal_close, __equal_close + 2);
+  if (__temp == __last)
+    __throw_regex_error<regex_constants::error_brack>();
+  // [__first, __temp) contains all text in [= ... =]
+  string_type __collate_name = __traits_.lookup_collatename(__first, __temp);
+  if (__collate_name.empty())
+    __throw_regex_error<regex_constants::error_collate>();
+  string_type __equiv_name = __traits_.transform_primary(__collate_name.begin(), __collate_name.end());
+  if (!__equiv_name.empty())
+    __ml->__add_equivalence(__equiv_name);
+  else {
+    switch (__collate_name.size()) {
+    case 1:
+      __ml->__add_char(__collate_name[0]);
+      break;
+    case 2:
+      __ml->__add_digraph(__collate_name[0], __collate_name[1]);
+      break;
+    default:
+      __throw_regex_error<regex_constants::error_collate>();
+    }
+  }
+  __first = std::next(__temp, 2);
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_character_class(
+    _ForwardIterator __first, _ForwardIterator __last, __bracket_expression<_CharT, _Traits>* __ml) {
+  // Found [:
+  //   This means :] must exist
+  value_type __colon_close[2] = {':', ']'};
+  _ForwardIterator __temp     = std::search(__first, __last, __colon_close, __colon_close + 2);
+  if (__temp == __last)
+    __throw_regex_error<regex_constants::error_brack>();
+  // [__first, __temp) contains all text in [: ... :]
+  typedef typename _Traits::char_class_type char_class_type;
+  char_class_type __class_type = __traits_.lookup_classname(__first, __temp, __flags_ & icase);
+  if (__class_type == 0)
+    __throw_regex_error<regex_constants::error_ctype>();
+  __ml->__add_class(__class_type);
+  __first = std::next(__temp, 2);
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_collating_symbol(
+    _ForwardIterator __first, _ForwardIterator __last, basic_string<_CharT>& __col_sym) {
+  // Found [.
+  //   This means .] must exist
+  value_type __dot_close[2] = {'.', ']'};
+  _ForwardIterator __temp   = std::search(__first, __last, __dot_close, __dot_close + 2);
+  if (__temp == __last)
+    __throw_regex_error<regex_constants::error_brack>();
+  // [__first, __temp) contains all text in [. ... .]
+  __col_sym = __traits_.lookup_collatename(__first, __temp);
+  switch (__col_sym.size()) {
+  case 1:
+  case 2:
+    break;
+  default:
+    __throw_regex_error<regex_constants::error_collate>();
+  }
+  __first = std::next(__temp, 2);
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_DUP_COUNT(_ForwardIterator __first, _ForwardIterator __last, int& __c) {
+  if (__first != __last) {
+    int __val = __traits_.value(*__first, 10);
+    if (__val != -1) {
+      __c = __val;
+      for (++__first; __first != __last && (__val = __traits_.value(*__first, 10)) != -1; ++__first) {
+        if (__c >= numeric_limits<int>::max() / 10)
+          __throw_regex_error<regex_constants::error_badbrace>();
+        __c *= 10;
+        __c += __val;
+      }
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_ecma_exp(_ForwardIterator __first, _ForwardIterator __last) {
+  __owns_one_state<_CharT>* __sa = __end_;
+  _ForwardIterator __temp        = __parse_alternative(__first, __last);
+  if (__temp == __first)
+    __push_empty();
+  __first = __temp;
+  while (__first != __last && *__first == '|') {
+    __owns_one_state<_CharT>* __sb = __end_;
+    __temp                         = __parse_alternative(++__first, __last);
+    if (__temp == __first)
+      __push_empty();
+    __push_alternation(__sa, __sb);
+    __first = __temp;
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_alternative(_ForwardIterator __first, _ForwardIterator __last) {
+  while (true) {
+    _ForwardIterator __temp = __parse_term(__first, __last);
+    if (__temp == __first)
+      break;
+    __first = __temp;
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_term(_ForwardIterator __first, _ForwardIterator __last) {
+  _ForwardIterator __temp = __parse_assertion(__first, __last);
+  if (__temp == __first) {
+    __owns_one_state<_CharT>* __e = __end_;
+    unsigned __mexp_begin         = __marked_count_;
+    __temp                        = __parse_atom(__first, __last);
+    if (__temp != __first)
+      __first = __parse_ERE_dupl_symbol(__temp, __last, __e, __mexp_begin + 1, __marked_count_ + 1);
+  } else
+    __first = __temp;
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_assertion(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    switch (*__first) {
+    case '^':
+      __push_l_anchor();
+      ++__first;
+      break;
+    case '$':
+      __push_r_anchor();
+      ++__first;
+      break;
+    case '\\': {
+      _ForwardIterator __temp = std::next(__first);
+      if (__temp != __last) {
+        if (*__temp == 'b') {
+          __push_word_boundary(false);
+          __first = ++__temp;
+        } else if (*__temp == 'B') {
+          __push_word_boundary(true);
+          __first = ++__temp;
+        }
+      }
+    } break;
+    case '(': {
+      _ForwardIterator __temp = std::next(__first);
+      if (__temp != __last && *__temp == '?') {
+        if (++__temp != __last) {
+          switch (*__temp) {
+          case '=': {
+            basic_regex __exp;
+            __exp.__flags_  = __flags_;
+            __temp          = __exp.__parse(++__temp, __last);
+            unsigned __mexp = __exp.__marked_count_;
+            __push_lookahead(std::move(__exp), false, __marked_count_);
+            __marked_count_ += __mexp;
+            if (__temp == __last || *__temp != ')')
+              __throw_regex_error<regex_constants::error_paren>();
+            __first = ++__temp;
+          } break;
+          case '!': {
+            basic_regex __exp;
+            __exp.__flags_  = __flags_;
+            __temp          = __exp.__parse(++__temp, __last);
+            unsigned __mexp = __exp.__marked_count_;
+            __push_lookahead(std::move(__exp), true, __marked_count_);
+            __marked_count_ += __mexp;
+            if (__temp == __last || *__temp != ')')
+              __throw_regex_error<regex_constants::error_paren>();
+            __first = ++__temp;
+          } break;
+          }
+        }
+      }
+    } break;
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_atom(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    switch (*__first) {
+    case '.':
+      __push_match_any_but_newline();
+      ++__first;
+      break;
+    case '\\':
+      __first = __parse_atom_escape(__first, __last);
+      break;
+    case '[':
+      __first = __parse_bracket_expression(__first, __last);
+      break;
+    case '(': {
+      ++__first;
+      if (__first == __last)
+        __throw_regex_error<regex_constants::error_paren>();
+      _ForwardIterator __temp = std::next(__first);
+      if (__temp != __last && *__first == '?' && *__temp == ':') {
+        ++__open_count_;
+        __first = __parse_ecma_exp(++__temp, __last);
+        if (__first == __last || *__first != ')')
+          __throw_regex_error<regex_constants::error_paren>();
+        --__open_count_;
+        ++__first;
+      } else {
+        __push_begin_marked_subexpression();
+        unsigned __temp_count = __marked_count_;
+        ++__open_count_;
+        __first = __parse_ecma_exp(__first, __last);
+        if (__first == __last || *__first != ')')
+          __throw_regex_error<regex_constants::error_paren>();
+        __push_end_marked_subexpression(__temp_count);
+        --__open_count_;
+        ++__first;
+      }
+    } break;
+    case '*':
+    case '+':
+    case '?':
+    case '{':
+      __throw_regex_error<regex_constants::error_badrepeat>();
+      break;
+    default:
+      __first = __parse_pattern_character(__first, __last);
+      break;
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_atom_escape(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last && *__first == '\\') {
+    _ForwardIterator __t1 = std::next(__first);
+    if (__t1 == __last)
+      __throw_regex_error<regex_constants::error_escape>();
+
+    _ForwardIterator __t2 = __parse_decimal_escape(__t1, __last);
+    if (__t2 != __t1)
+      __first = __t2;
+    else {
+      __t2 = __parse_character_class_escape(__t1, __last);
+      if (__t2 != __t1)
+        __first = __t2;
+      else {
+        __t2 = __parse_character_escape(__t1, __last);
+        if (__t2 != __t1)
+          __first = __t2;
+      }
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_decimal_escape(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    if (*__first == '0') {
+      __push_char(_CharT());
+      ++__first;
+    } else if ('1' <= *__first && *__first <= '9') {
+      unsigned __v = *__first - '0';
+      for (++__first; __first != __last && '0' <= *__first && *__first <= '9'; ++__first) {
+        if (__v >= numeric_limits<unsigned>::max() / 10)
+          __throw_regex_error<regex_constants::error_backref>();
+        __v = 10 * __v + *__first - '0';
+      }
+      if (__v == 0 || __v > mark_count())
+        __throw_regex_error<regex_constants::error_backref>();
+      __push_back_ref(__v);
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_character_class_escape(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    __bracket_expression<_CharT, _Traits>* __ml;
+    switch (*__first) {
+    case 'd':
+      __ml = __start_matching_list(false);
+      __ml->__add_class(ctype_base::digit);
+      ++__first;
+      break;
+    case 'D':
+      __ml = __start_matching_list(true);
+      __ml->__add_class(ctype_base::digit);
+      ++__first;
+      break;
+    case 's':
+      __ml = __start_matching_list(false);
+      __ml->__add_class(ctype_base::space);
+      ++__first;
+      break;
+    case 'S':
+      __ml = __start_matching_list(true);
+      __ml->__add_class(ctype_base::space);
+      ++__first;
+      break;
+    case 'w':
+      __ml = __start_matching_list(false);
+      __ml->__add_class(ctype_base::alnum);
+      __ml->__add_char('_');
+      ++__first;
+      break;
+    case 'W':
+      __ml = __start_matching_list(true);
+      __ml->__add_class(ctype_base::alnum);
+      __ml->__add_char('_');
+      ++__first;
+      break;
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_character_escape(
+    _ForwardIterator __first, _ForwardIterator __last, basic_string<_CharT>* __str) {
+  if (__first != __last) {
+    _ForwardIterator __t;
+    unsigned __sum = 0;
+    int __hd;
+    switch (*__first) {
+    case 'f':
+      if (__str)
+        *__str = _CharT(0xC);
+      else
+        __push_char(_CharT(0xC));
+      ++__first;
+      break;
+    case 'n':
+      if (__str)
+        *__str = _CharT(0xA);
+      else
+        __push_char(_CharT(0xA));
+      ++__first;
+      break;
+    case 'r':
+      if (__str)
+        *__str = _CharT(0xD);
+      else
+        __push_char(_CharT(0xD));
+      ++__first;
+      break;
+    case 't':
+      if (__str)
+        *__str = _CharT(0x9);
+      else
+        __push_char(_CharT(0x9));
+      ++__first;
+      break;
+    case 'v':
+      if (__str)
+        *__str = _CharT(0xB);
+      else
+        __push_char(_CharT(0xB));
+      ++__first;
+      break;
+    case 'c':
+      if ((__t = std::next(__first)) != __last) {
+        if (('A' <= *__t && *__t <= 'Z') || ('a' <= *__t && *__t <= 'z')) {
+          if (__str)
+            *__str = _CharT(*__t % 32);
+          else
+            __push_char(_CharT(*__t % 32));
+          __first = ++__t;
+        } else
+          __throw_regex_error<regex_constants::error_escape>();
+      } else
+        __throw_regex_error<regex_constants::error_escape>();
+      break;
+    case 'u':
+      ++__first;
+      if (__first == __last)
+        __throw_regex_error<regex_constants::error_escape>();
+      __hd = __traits_.value(*__first, 16);
+      if (__hd == -1)
+        __throw_regex_error<regex_constants::error_escape>();
+      __sum = 16 * __sum + static_cast<unsigned>(__hd);
+      ++__first;
+      if (__first == __last)
+        __throw_regex_error<regex_constants::error_escape>();
+      __hd = __traits_.value(*__first, 16);
+      if (__hd == -1)
+        __throw_regex_error<regex_constants::error_escape>();
+      __sum = 16 * __sum + static_cast<unsigned>(__hd);
+      // fallthrough
+    case 'x':
+      ++__first;
+      if (__first == __last)
+        __throw_regex_error<regex_constants::error_escape>();
+      __hd = __traits_.value(*__first, 16);
+      if (__hd == -1)
+        __throw_regex_error<regex_constants::error_escape>();
+      __sum = 16 * __sum + static_cast<unsigned>(__hd);
+      ++__first;
+      if (__first == __last)
+        __throw_regex_error<regex_constants::error_escape>();
+      __hd = __traits_.value(*__first, 16);
+      if (__hd == -1)
+        __throw_regex_error<regex_constants::error_escape>();
+      __sum = 16 * __sum + static_cast<unsigned>(__hd);
+      if (__str)
+        *__str = _CharT(__sum);
+      else
+        __push_char(_CharT(__sum));
+      ++__first;
+      break;
+    case '0':
+      if (__str)
+        *__str = _CharT(0);
+      else
+        __push_char(_CharT(0));
+      ++__first;
+      break;
+    default:
+      if (*__first != '_' && !__traits_.isctype(*__first, ctype_base::alnum)) {
+        if (__str)
+          *__str = *__first;
+        else
+          __push_char(*__first);
+        ++__first;
+      } else
+        __throw_regex_error<regex_constants::error_escape>();
+      break;
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator
+basic_regex<_CharT, _Traits>::__parse_pattern_character(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__first != __last) {
+    switch (*__first) {
+    case '^':
+    case '$':
+    case '\\':
+    case '.':
+    case '*':
+    case '+':
+    case '?':
+    case '(':
+    case ')':
+    case '[':
+    case ']':
+    case '{':
+    case '}':
+    case '|':
+      break;
+    default:
+      __push_char(*__first);
+      ++__first;
+      break;
+    }
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_grep(_ForwardIterator __first, _ForwardIterator __last) {
+  __owns_one_state<_CharT>* __sa = __end_;
+  _ForwardIterator __t1          = std::find(__first, __last, _CharT('\n'));
+  if (__t1 != __first)
+    __parse_basic_reg_exp(__first, __t1);
+  else
+    __push_empty();
+  __first = __t1;
+  if (__first != __last)
+    ++__first;
+  while (__first != __last) {
+    __t1                           = std::find(__first, __last, _CharT('\n'));
+    __owns_one_state<_CharT>* __sb = __end_;
+    if (__t1 != __first)
+      __parse_basic_reg_exp(__first, __t1);
+    else
+      __push_empty();
+    __push_alternation(__sa, __sb);
+    __first = __t1;
+    if (__first != __last)
+      ++__first;
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+template <class _ForwardIterator>
+_ForwardIterator basic_regex<_CharT, _Traits>::__parse_egrep(_ForwardIterator __first, _ForwardIterator __last) {
+  __owns_one_state<_CharT>* __sa = __end_;
+  _ForwardIterator __t1          = std::find(__first, __last, _CharT('\n'));
+  if (__t1 != __first)
+    __parse_extended_reg_exp(__first, __t1);
+  else
+    __push_empty();
+  __first = __t1;
+  if (__first != __last)
+    ++__first;
+  while (__first != __last) {
+    __t1                           = std::find(__first, __last, _CharT('\n'));
+    __owns_one_state<_CharT>* __sb = __end_;
+    if (__t1 != __first)
+      __parse_extended_reg_exp(__first, __t1);
+    else
+      __push_empty();
+    __push_alternation(__sa, __sb);
+    __first = __t1;
+    if (__first != __last)
+      ++__first;
+  }
+  return __first;
+}
+
+template <class _CharT, class _Traits>
+bool basic_regex<_CharT, _Traits>::__test_back_ref(_CharT __c) {
+  unsigned __val = __traits_.value(__c, 10);
+  if (__val >= 1 && __val <= 9) {
+    if (__val > mark_count())
+      __throw_regex_error<regex_constants::error_backref>();
+    __push_back_ref(__val);
+    return true;
+  }
+
+  return false;
+}
+
+template <class _CharT, class _Traits>
+void basic_regex<_CharT, _Traits>::__push_loop(
+    size_t __min, size_t __max, __owns_one_state<_CharT>* __s, size_t __mexp_begin, size_t __mexp_end, bool __greedy) {
+  unique_ptr<__empty_state<_CharT> > __e1(new __empty_state<_CharT>(__end_->first()));
+  __end_->first() = nullptr;
+  unique_ptr<__loop<_CharT> > __e2(
+      new __loop<_CharT>(__loop_count_, __s->first(), __e1.get(), __mexp_begin, __mexp_end, __greedy, __min, __max));
+  __s->first() = nullptr;
+  __e1.release();
+  __end_->first() = new __repeat_one_loop<_CharT>(__e2.get());
+  __end_          = __e2->second();
+  __s->first()    = __e2.release();
+  ++__loop_count_;
+}
+
+template <class _CharT, class _Traits>
+void basic_regex<_CharT, _Traits>::__push_char(value_type __c) {
+  if (flags() & icase)
+    __end_->first() = new __match_char_icase<_CharT, _Traits>(__traits_, __c, __end_->first());
+  else if (flags() & collate)
+    __end_->first() = new __match_char_collate<_CharT, _Traits>(__traits_, __c, __end_->first());
+  else
+    __end_->first() = new __match_char<_CharT>(__c, __end_->first());
+  __end_ = static_cast<__owns_one_state<_CharT>*>(__end_->first());
+}
+
+template <class _CharT, class _Traits>
+void basic_regex<_CharT, _Traits>::__push_begin_marked_subexpression() {
+  if (!(__flags_ & nosubs)) {
+    __end_->first() = new __begin_marked_subexpression<_CharT>(++__marked_count_, __end_->first());
+    __end_          = static_cast<__owns_one_state<_CharT>*>(__end_->first());
+  }
+}
+
+template <class _CharT, class _Traits>
+void basic_regex<_CharT, _Traits>::__push_end_marked_subexpression(unsigned __sub) {
+  if (!(__flags_ & nosubs)) {
+    __end_->first() = new __end_marked_subexpression<_CharT>(__sub, __end_->first());
+    __end_          = static_cast<__owns_one_state<_CharT>*>(__end_->first());
+  }
+}
+
+template <class _CharT, class _Traits>
+void basic_regex<_CharT, _Traits>::__push_l_anchor() {
+  __end_->first() = new __l_anchor_multiline<_CharT>(__use_multiline(), __end_->first());
+  __end_          = static_cast<__owns_one_state<_CharT>*>(__end_->first());
+}
+
+template <class _CharT, class _Traits>
+void basic_regex<_CharT, _Traits>::__push_r_anchor() {
+  __end_->first() = new __r_anchor_multiline<_CharT>(__use_multiline(), __end_->first());
+  __end_          = static_cast<__owns_one_state<_CharT>*>(__end_->first());
+}
+
+template <class _CharT, class _Traits>
+void basic_regex<_CharT, _Traits>::__push_match_any() {
+  __end_->first() = new __match_any<_CharT>(__end_->first());
+  __end_          = static_cast<__owns_one_state<_CharT>*>(__end_->first());
+}
+
+template <class _CharT, class _Traits>
+void basic_regex<_CharT, _Traits>::__push_match_any_but_newline() {
+  __end_->first() = new __match_any_but_newline<_CharT>(__end_->first());
+  __end_          = static_cast<__owns_one_state<_CharT>*>(__end_->first());
+}
+
+template <class _CharT, class _Traits>
+void basic_regex<_CharT, _Traits>::__push_empty() {
+  __end_->first() = new __empty_state<_CharT>(__end_->first());
+  __end_          = static_cast<__owns_one_state<_CharT>*>(__end_->first());
+}
+
+template <class _CharT, class _Traits>
+void basic_regex<_CharT, _Traits>::__push_word_boundary(bool __invert) {
+  __end_->first() = new __word_boundary<_CharT, _Traits>(__traits_, __invert, __end_->first());
+  __end_          = static_cast<__owns_one_state<_CharT>*>(__end_->first());
+}
+
+template <class _CharT, class _Traits>
+void basic_regex<_CharT, _Traits>::__push_back_ref(int __i) {
+  if (flags() & icase)
+    __end_->first() = new __back_ref_icase<_CharT, _Traits>(__traits_, __i, __end_->first());
+  else if (flags() & collate)
+    __end_->first() = new __back_ref_collate<_CharT, _Traits>(__traits_, __i, __end_->first());
+  else
+    __end_->first() = new __back_ref<_CharT>(__i, __end_->first());
+  __end_ = static_cast<__owns_one_state<_CharT>*>(__end_->first());
+}
+
+template <class _CharT, class _Traits>
+void basic_regex<_CharT, _Traits>::__push_alternation(__owns_one_state<_CharT>* __sa, __owns_one_state<_CharT>* __ea) {
+  __sa->first() = new __alternate<_CharT>(
+      static_cast<__owns_one_state<_CharT>*>(__sa->first()), static_cast<__owns_one_state<_CharT>*>(__ea->first()));
+  __ea->first()   = nullptr;
+  __ea->first()   = new __empty_state<_CharT>(__end_->first());
+  __end_->first() = nullptr;
+  __end_->first() = new __empty_non_own_state<_CharT>(__ea->first());
+  __end_          = static_cast<__owns_one_state<_CharT>*>(__ea->first());
+}
+
+template <class _CharT, class _Traits>
+__bracket_expression<_CharT, _Traits>* basic_regex<_CharT, _Traits>::__start_matching_list(bool __negate) {
+  __bracket_expression<_CharT, _Traits>* __r = new __bracket_expression<_CharT, _Traits>(
+      __traits_, __end_->first(), __negate, __flags_ & icase, __flags_ & collate);
+  __end_->first() = __r;
+  __end_          = __r;
+  return __r;
+}
+
+template <class _CharT, class _Traits>
+void basic_regex<_CharT, _Traits>::__push_lookahead(const basic_regex& __exp, bool __invert, unsigned __mexp) {
+  __end_->first() = new __lookahead<_CharT, _Traits>(__exp, __invert, __end_->first(), __mexp);
+  __end_          = static_cast<__owns_one_state<_CharT>*>(__end_->first());
+}
+
+// sub_match
+
+typedef sub_match<const char*> csub_match;
+typedef sub_match<string::const_iterator> ssub_match;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+typedef sub_match<const wchar_t*> wcsub_match;
+typedef sub_match<wstring::const_iterator> wssub_match;
+#endif
+
+template <class _BidirectionalIterator>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_PREFERRED_NAME(csub_match)
+    _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wcsub_match)) _LIBCPP_PREFERRED_NAME(ssub_match)
+        _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wssub_match)) sub_match
+    : public pair<_BidirectionalIterator, _BidirectionalIterator> {
+public:
+  typedef _BidirectionalIterator iterator;
+  typedef typename iterator_traits<iterator>::value_type value_type;
+  typedef typename iterator_traits<iterator>::difference_type difference_type;
+  typedef basic_string<value_type> string_type;
+
+  bool matched;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR sub_match() : matched() {}
+
+  _LIBCPP_HIDE_FROM_ABI difference_type length() const {
+    return matched ? std::distance(this->first, this->second) : 0;
+  }
+  _LIBCPP_HIDE_FROM_ABI string_type str() const {
+    return matched ? string_type(this->first, this->second) : string_type();
+  }
+  _LIBCPP_HIDE_FROM_ABI operator string_type() const { return str(); }
+
+  _LIBCPP_HIDE_FROM_ABI int compare(const sub_match& __s) const { return str().compare(__s.str()); }
+  _LIBCPP_HIDE_FROM_ABI int compare(const string_type& __s) const { return str().compare(__s); }
+  _LIBCPP_HIDE_FROM_ABI int compare(const value_type* __s) const { return str().compare(__s); }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(sub_match& __s) _NOEXCEPT_(__is_nothrow_swappable_v<_BidirectionalIterator>) {
+    this->pair<_BidirectionalIterator, _BidirectionalIterator>::swap(__s);
+    std::swap(matched, __s.matched);
+  }
+};
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(const sub_match<_BiIter>& __x, const sub_match<_BiIter>& __y) {
+  return __x.compare(__y) == 0;
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _BiIter>
+using __sub_match_cat = compare_three_way_result_t<basic_string<typename iterator_traits<_BiIter>::value_type>>;
+
+template <class _BiIter>
+_LIBCPP_HIDE_FROM_ABI auto operator<=>(const sub_match<_BiIter>& __x, const sub_match<_BiIter>& __y) {
+  return static_cast<__sub_match_cat<_BiIter>>(__x.compare(__y) <=> 0);
+}
+#else  // _LIBCPP_STD_VER >= 20
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const sub_match<_BiIter>& __x, const sub_match<_BiIter>& __y) {
+  return !(__x == __y);
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(const sub_match<_BiIter>& __x, const sub_match<_BiIter>& __y) {
+  return __x.compare(__y) < 0;
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const sub_match<_BiIter>& __x, const sub_match<_BiIter>& __y) {
+  return !(__y < __x);
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>=(const sub_match<_BiIter>& __x, const sub_match<_BiIter>& __y) {
+  return !(__x < __y);
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>(const sub_match<_BiIter>& __x, const sub_match<_BiIter>& __y) {
+  return __y < __x;
+}
+
+template <class _BiIter, class _ST, class _SA>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const basic_string<typename iterator_traits<_BiIter>::value_type, _ST, _SA>& __x,
+           const sub_match<_BiIter>& __y) {
+  return __y.compare(typename sub_match<_BiIter>::string_type(__x.data(), __x.size())) == 0;
+}
+
+template <class _BiIter, class _ST, class _SA>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const basic_string<typename iterator_traits<_BiIter>::value_type, _ST, _SA>& __x,
+           const sub_match<_BiIter>& __y) {
+  return !(__x == __y);
+}
+
+template <class _BiIter, class _ST, class _SA>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<(const basic_string<typename iterator_traits<_BiIter>::value_type, _ST, _SA>& __x,
+          const sub_match<_BiIter>& __y) {
+  return __y.compare(typename sub_match<_BiIter>::string_type(__x.data(), __x.size())) > 0;
+}
+
+template <class _BiIter, class _ST, class _SA>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>(const basic_string<typename iterator_traits<_BiIter>::value_type, _ST, _SA>& __x,
+          const sub_match<_BiIter>& __y) {
+  return __y < __x;
+}
+
+template <class _BiIter, class _ST, class _SA>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>=(const basic_string<typename iterator_traits<_BiIter>::value_type, _ST, _SA>& __x,
+           const sub_match<_BiIter>& __y) {
+  return !(__x < __y);
+}
+
+template <class _BiIter, class _ST, class _SA>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<=(const basic_string<typename iterator_traits<_BiIter>::value_type, _ST, _SA>& __x,
+           const sub_match<_BiIter>& __y) {
+  return !(__y < __x);
+}
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _BiIter, class _ST, class _SA>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const sub_match<_BiIter>& __x,
+           const basic_string<typename iterator_traits<_BiIter>::value_type, _ST, _SA>& __y) {
+  return __x.compare(typename sub_match<_BiIter>::string_type(__y.data(), __y.size())) == 0;
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _BiIter, class _ST, class _SA>
+_LIBCPP_HIDE_FROM_ABI auto
+operator<=>(const sub_match<_BiIter>& __x,
+            const basic_string<typename iterator_traits<_BiIter>::value_type, _ST, _SA>& __y) {
+  return static_cast<__sub_match_cat<_BiIter>>(
+      __x.compare(typename sub_match<_BiIter>::string_type(__y.data(), __y.size())) <=> 0);
+}
+#else  // _LIBCPP_STD_VER >= 20
+template <class _BiIter, class _ST, class _SA>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const sub_match<_BiIter>& __x,
+           const basic_string<typename iterator_traits<_BiIter>::value_type, _ST, _SA>& __y) {
+  return !(__x == __y);
+}
+
+template <class _BiIter, class _ST, class _SA>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<(const sub_match<_BiIter>& __x,
+          const basic_string<typename iterator_traits<_BiIter>::value_type, _ST, _SA>& __y) {
+  return __x.compare(typename sub_match<_BiIter>::string_type(__y.data(), __y.size())) < 0;
+}
+
+template <class _BiIter, class _ST, class _SA>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>(const sub_match<_BiIter>& __x,
+          const basic_string<typename iterator_traits<_BiIter>::value_type, _ST, _SA>& __y) {
+  return __y < __x;
+}
+
+template <class _BiIter, class _ST, class _SA>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>=(const sub_match<_BiIter>& __x,
+           const basic_string<typename iterator_traits<_BiIter>::value_type, _ST, _SA>& __y) {
+  return !(__x < __y);
+}
+
+template <class _BiIter, class _ST, class _SA>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<=(const sub_match<_BiIter>& __x,
+           const basic_string<typename iterator_traits<_BiIter>::value_type, _ST, _SA>& __y) {
+  return !(__y < __x);
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(typename iterator_traits<_BiIter>::value_type const* __x, const sub_match<_BiIter>& __y) {
+  return __y.compare(__x) == 0;
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(typename iterator_traits<_BiIter>::value_type const* __x, const sub_match<_BiIter>& __y) {
+  return !(__x == __y);
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<(typename iterator_traits<_BiIter>::value_type const* __x, const sub_match<_BiIter>& __y) {
+  return __y.compare(__x) > 0;
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>(typename iterator_traits<_BiIter>::value_type const* __x, const sub_match<_BiIter>& __y) {
+  return __y < __x;
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>=(typename iterator_traits<_BiIter>::value_type const* __x, const sub_match<_BiIter>& __y) {
+  return !(__x < __y);
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<=(typename iterator_traits<_BiIter>::value_type const* __x, const sub_match<_BiIter>& __y) {
+  return !(__y < __x);
+}
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const* __y) {
+  return __x.compare(__y) == 0;
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _BiIter>
+_LIBCPP_HIDE_FROM_ABI auto
+operator<=>(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const* __y) {
+  return static_cast<__sub_match_cat<_BiIter>>(__x.compare(__y) <=> 0);
+}
+#else  // _LIBCPP_STD_VER >= 20
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const* __y) {
+  return !(__x == __y);
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const* __y) {
+  return __x.compare(__y) < 0;
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const* __y) {
+  return __y < __x;
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>=(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const* __y) {
+  return !(__x < __y);
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<=(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const* __y) {
+  return !(__y < __x);
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(typename iterator_traits<_BiIter>::value_type const& __x, const sub_match<_BiIter>& __y) {
+  typedef basic_string<typename iterator_traits<_BiIter>::value_type> string_type;
+  return __y.compare(string_type(1, __x)) == 0;
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(typename iterator_traits<_BiIter>::value_type const& __x, const sub_match<_BiIter>& __y) {
+  return !(__x == __y);
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<(typename iterator_traits<_BiIter>::value_type const& __x, const sub_match<_BiIter>& __y) {
+  typedef basic_string<typename iterator_traits<_BiIter>::value_type> string_type;
+  return __y.compare(string_type(1, __x)) > 0;
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>(typename iterator_traits<_BiIter>::value_type const& __x, const sub_match<_BiIter>& __y) {
+  return __y < __x;
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>=(typename iterator_traits<_BiIter>::value_type const& __x, const sub_match<_BiIter>& __y) {
+  return !(__x < __y);
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<=(typename iterator_traits<_BiIter>::value_type const& __x, const sub_match<_BiIter>& __y) {
+  return !(__y < __x);
+}
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const& __y) {
+  typedef basic_string<typename iterator_traits<_BiIter>::value_type> string_type;
+  return __x.compare(string_type(1, __y)) == 0;
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _BiIter>
+_LIBCPP_HIDE_FROM_ABI auto
+operator<=>(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const& __y) {
+  using string_type = basic_string<typename iterator_traits<_BiIter>::value_type>;
+  return static_cast<__sub_match_cat<_BiIter>>(__x.compare(string_type(1, __y)) <=> 0);
+}
+#else  // _LIBCPP_STD_VER >= 20
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const& __y) {
+  return !(__x == __y);
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const& __y) {
+  typedef basic_string<typename iterator_traits<_BiIter>::value_type> string_type;
+  return __x.compare(string_type(1, __y)) < 0;
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const& __y) {
+  return __y < __x;
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>=(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const& __y) {
+  return !(__x < __y);
+}
+
+template <class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<=(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const& __y) {
+  return !(__y < __x);
+}
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _CharT, class _ST, class _BiIter>
+inline _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _ST>&
+operator<<(basic_ostream<_CharT, _ST>& __os, const sub_match<_BiIter>& __m) {
+  return __os << __m.str();
+}
+
+typedef match_results<const char*> cmatch;
+typedef match_results<string::const_iterator> smatch;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+typedef match_results<const wchar_t*> wcmatch;
+typedef match_results<wstring::const_iterator> wsmatch;
+#endif
+
+template <class _BidirectionalIterator, class _Allocator>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_PREFERRED_NAME(cmatch) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wcmatch))
+    _LIBCPP_PREFERRED_NAME(smatch) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wsmatch)) match_results {
+public:
+  typedef _Allocator allocator_type;
+  typedef sub_match<_BidirectionalIterator> value_type;
+
+private:
+  typedef vector<value_type, allocator_type> __container_type;
+
+  __container_type __matches_;
+  value_type __unmatched_;
+  value_type __prefix_;
+  value_type __suffix_;
+  bool __ready_;
+
+public:
+  _BidirectionalIterator __position_start_;
+  typedef const value_type& const_reference;
+  typedef value_type& reference;
+  typedef typename __container_type::const_iterator const_iterator;
+  typedef const_iterator iterator;
+  typedef typename iterator_traits<_BidirectionalIterator>::difference_type difference_type;
+  typedef typename allocator_traits<allocator_type>::size_type size_type;
+  typedef typename iterator_traits<_BidirectionalIterator>::value_type char_type;
+  typedef basic_string<char_type> string_type;
+
+  // construct/copy/destroy:
+#ifndef _LIBCPP_CXX03_LANG
+  match_results() : match_results(allocator_type()) {}
+  explicit match_results(const allocator_type& __a);
+#else
+  explicit match_results(const allocator_type& __a = allocator_type());
+#endif
+
+  //    match_results(const match_results&) = default;
+  //    match_results& operator=(const match_results&) = default;
+  //    match_results(match_results&& __m) = default;
+  //    match_results& operator=(match_results&& __m) = default;
+  //    ~match_results() = default;
+
+  _LIBCPP_HIDE_FROM_ABI bool ready() const { return __ready_; }
+
+  // size:
+  _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __matches_.size(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __matches_.max_size(); }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return size() == 0; }
+
+  // element access:
+  _LIBCPP_HIDE_FROM_ABI difference_type length(size_type __sub = 0) const {
+    // If the match results are not ready, this will return `0`.
+    _LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::length() called when not ready");
+    return (*this)[__sub].length();
+  }
+  _LIBCPP_HIDE_FROM_ABI difference_type position(size_type __sub = 0) const {
+    // If the match results are not ready, this will return the result of subtracting two default-constructed iterators
+    // (which is typically a well-defined operation).
+    _LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::position() called when not ready");
+    return std::distance(__position_start_, (*this)[__sub].first);
+  }
+  _LIBCPP_HIDE_FROM_ABI string_type str(size_type __sub = 0) const {
+    // If the match results are not ready, this will return an empty string.
+    _LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::str() called when not ready");
+    return (*this)[__sub].str();
+  }
+  _LIBCPP_HIDE_FROM_ABI const_reference operator[](size_type __n) const {
+    // If the match results are not ready, this call will be equivalent to calling this function with `__n >= size()`,
+    // returning an empty subrange.
+    _LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::operator[]() called when not ready");
+    return __n < __matches_.size() ? __matches_[__n] : __unmatched_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const_reference prefix() const {
+    // If the match results are not ready, this will return a default-constructed empty `__suffix_`.
+    _LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::prefix() called when not ready");
+    return __prefix_;
+  }
+  _LIBCPP_HIDE_FROM_ABI const_reference suffix() const {
+    // If the match results are not ready, this will return a default-constructed empty `__suffix_`.
+    _LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::suffix() called when not ready");
+    return __suffix_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const { return empty() ? __matches_.end() : __matches_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const { return __matches_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const { return empty() ? __matches_.end() : __matches_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const { return __matches_.end(); }
+
+  // format:
+  template <class _OutputIter>
+  _OutputIter format(_OutputIter __output_iter,
+                     const char_type* __fmt_first,
+                     const char_type* __fmt_last,
+                     regex_constants::match_flag_type __flags = regex_constants::format_default) const;
+  template <class _OutputIter, class _ST, class _SA>
+  _LIBCPP_HIDE_FROM_ABI _OutputIter
+  format(_OutputIter __output_iter,
+         const basic_string<char_type, _ST, _SA>& __fmt,
+         regex_constants::match_flag_type __flags = regex_constants::format_default) const {
+    return format(__output_iter, __fmt.data(), __fmt.data() + __fmt.size(), __flags);
+  }
+  template <class _ST, class _SA>
+  _LIBCPP_HIDE_FROM_ABI basic_string<char_type, _ST, _SA>
+  format(const basic_string<char_type, _ST, _SA>& __fmt,
+         regex_constants::match_flag_type __flags = regex_constants::format_default) const {
+    basic_string<char_type, _ST, _SA> __r;
+    format(std::back_inserter(__r), __fmt.data(), __fmt.data() + __fmt.size(), __flags);
+    return __r;
+  }
+  _LIBCPP_HIDE_FROM_ABI string_type
+  format(const char_type* __fmt, regex_constants::match_flag_type __flags = regex_constants::format_default) const {
+    string_type __r;
+    format(std::back_inserter(__r), __fmt, __fmt + char_traits<char_type>::length(__fmt), __flags);
+    return __r;
+  }
+
+  // allocator:
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const { return __matches_.get_allocator(); }
+
+  // swap:
+  void swap(match_results& __m);
+
+  template <class _Bp, class _Ap>
+  _LIBCPP_HIDE_FROM_ABI void
+  __assign(_BidirectionalIterator __f,
+           _BidirectionalIterator __l,
+           const match_results<_Bp, _Ap>& __m,
+           bool __no_update_pos) {
+    _Bp __mf = __m.prefix().first;
+    __matches_.resize(__m.size());
+    for (size_type __i = 0; __i < __matches_.size(); ++__i) {
+      __matches_[__i].first   = std::next(__f, std::distance(__mf, __m[__i].first));
+      __matches_[__i].second  = std::next(__f, std::distance(__mf, __m[__i].second));
+      __matches_[__i].matched = __m[__i].matched;
+    }
+    __unmatched_.first   = __l;
+    __unmatched_.second  = __l;
+    __unmatched_.matched = false;
+    __prefix_.first      = std::next(__f, std::distance(__mf, __m.prefix().first));
+    __prefix_.second     = std::next(__f, std::distance(__mf, __m.prefix().second));
+    __prefix_.matched    = __m.prefix().matched;
+    __suffix_.first      = std::next(__f, std::distance(__mf, __m.suffix().first));
+    __suffix_.second     = std::next(__f, std::distance(__mf, __m.suffix().second));
+    __suffix_.matched    = __m.suffix().matched;
+    if (!__no_update_pos)
+      __position_start_ = __prefix_.first;
+    __ready_ = __m.ready();
+  }
+
+private:
+  void __init(unsigned __s, _BidirectionalIterator __f, _BidirectionalIterator __l, bool __no_update_pos = false);
+
+  template <class, class>
+  friend class basic_regex;
+
+  template <class _Bp, class _Ap, class _Cp, class _Tp>
+  friend bool
+  regex_match(_Bp, _Bp, match_results<_Bp, _Ap>&, const basic_regex<_Cp, _Tp>&, regex_constants::match_flag_type);
+
+  template <class _Bp, class _Ap>
+  friend bool operator==(const match_results<_Bp, _Ap>&, const match_results<_Bp, _Ap>&);
+
+  template <class, class>
+  friend class __lookahead;
+
+  template <class, class, class>
+  friend class regex_iterator;
+};
+
+template <class _BidirectionalIterator, class _Allocator>
+match_results<_BidirectionalIterator, _Allocator>::match_results(const allocator_type& __a)
+    : __matches_(__a), __unmatched_(), __prefix_(), __suffix_(), __ready_(false), __position_start_() {}
+
+template <class _BidirectionalIterator, class _Allocator>
+void match_results<_BidirectionalIterator, _Allocator>::__init(
+    unsigned __s, _BidirectionalIterator __f, _BidirectionalIterator __l, bool __no_update_pos) {
+  __unmatched_.first   = __l;
+  __unmatched_.second  = __l;
+  __unmatched_.matched = false;
+  __matches_.assign(__s, __unmatched_);
+  __prefix_.first   = __f;
+  __prefix_.second  = __f;
+  __prefix_.matched = false;
+  __suffix_         = __unmatched_;
+  if (!__no_update_pos)
+    __position_start_ = __prefix_.first;
+  __ready_ = true;
+}
+
+template <class _BidirectionalIterator, class _Allocator>
+template <class _OutputIter>
+_OutputIter match_results<_BidirectionalIterator, _Allocator>::format(
+    _OutputIter __output_iter,
+    const char_type* __fmt_first,
+    const char_type* __fmt_last,
+    regex_constants::match_flag_type __flags) const {
+  // Note: this duplicates a check in `vector::operator[]` but provides a better error message.
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(ready(), "match_results::format() called when not ready");
+  if (__flags & regex_constants::format_sed) {
+    for (; __fmt_first != __fmt_last; ++__fmt_first) {
+      if (*__fmt_first == '&')
+        __output_iter = std::copy(__matches_[0].first, __matches_[0].second, __output_iter);
+      else if (*__fmt_first == '\\' && __fmt_first + 1 != __fmt_last) {
+        ++__fmt_first;
+        if ('0' <= *__fmt_first && *__fmt_first <= '9') {
+          size_t __i    = *__fmt_first - '0';
+          __output_iter = std::copy((*this)[__i].first, (*this)[__i].second, __output_iter);
+        } else {
+          *__output_iter = *__fmt_first;
+          ++__output_iter;
+        }
+      } else {
+        *__output_iter = *__fmt_first;
+        ++__output_iter;
+      }
+    }
+  } else {
+    for (; __fmt_first != __fmt_last; ++__fmt_first) {
+      if (*__fmt_first == '$' && __fmt_first + 1 != __fmt_last) {
+        switch (__fmt_first[1]) {
+        case '$':
+          *__output_iter = *++__fmt_first;
+          ++__output_iter;
+          break;
+        case '&':
+          ++__fmt_first;
+          __output_iter = std::copy(__matches_[0].first, __matches_[0].second, __output_iter);
+          break;
+        case '`':
+          ++__fmt_first;
+          __output_iter = std::copy(__prefix_.first, __prefix_.second, __output_iter);
+          break;
+        case '\'':
+          ++__fmt_first;
+          __output_iter = std::copy(__suffix_.first, __suffix_.second, __output_iter);
+          break;
+        default:
+          if ('0' <= __fmt_first[1] && __fmt_first[1] <= '9') {
+            ++__fmt_first;
+            size_t __idx = *__fmt_first - '0';
+            if (__fmt_first + 1 != __fmt_last && '0' <= __fmt_first[1] && __fmt_first[1] <= '9') {
+              ++__fmt_first;
+              if (__idx >= numeric_limits<size_t>::max() / 10)
+                __throw_regex_error<regex_constants::error_escape>();
+              __idx = 10 * __idx + *__fmt_first - '0';
+            }
+            __output_iter = std::copy((*this)[__idx].first, (*this)[__idx].second, __output_iter);
+          } else {
+            *__output_iter = *__fmt_first;
+            ++__output_iter;
+          }
+          break;
+        }
+      } else {
+        *__output_iter = *__fmt_first;
+        ++__output_iter;
+      }
+    }
+  }
+  return __output_iter;
+}
+
+template <class _BidirectionalIterator, class _Allocator>
+void match_results<_BidirectionalIterator, _Allocator>::swap(match_results& __m) {
+  using std::swap;
+  swap(__matches_, __m.__matches_);
+  swap(__unmatched_, __m.__unmatched_);
+  swap(__prefix_, __m.__prefix_);
+  swap(__suffix_, __m.__suffix_);
+  swap(__position_start_, __m.__position_start_);
+  swap(__ready_, __m.__ready_);
+}
+
+template <class _BidirectionalIterator, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI bool operator==(const match_results<_BidirectionalIterator, _Allocator>& __x,
+                                      const match_results<_BidirectionalIterator, _Allocator>& __y) {
+  if (__x.__ready_ != __y.__ready_)
+    return false;
+  if (!__x.__ready_)
+    return true;
+  return __x.__matches_ == __y.__matches_ && __x.__prefix_ == __y.__prefix_ && __x.__suffix_ == __y.__suffix_;
+}
+
+#if _LIBCPP_STD_VER < 20
+template <class _BidirectionalIterator, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const match_results<_BidirectionalIterator, _Allocator>& __x,
+                                             const match_results<_BidirectionalIterator, _Allocator>& __y) {
+  return !(__x == __y);
+}
+#endif
+
+template <class _BidirectionalIterator, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(match_results<_BidirectionalIterator, _Allocator>& __x, match_results<_BidirectionalIterator, _Allocator>& __y) {
+  __x.swap(__y);
+}
+
+// regex_search
+
+template <class _CharT, class _Traits>
+template <class _Allocator>
+bool basic_regex<_CharT, _Traits>::__match_at_start_ecma(
+    const _CharT* __first,
+    const _CharT* __last,
+    match_results<const _CharT*, _Allocator>& __m,
+    regex_constants::match_flag_type __flags,
+    bool __at_first) const {
+  vector<__state> __states;
+  __node* __st = __start_.get();
+  if (__st) {
+    sub_match<const _CharT*> __unmatched;
+    __unmatched.first   = __last;
+    __unmatched.second  = __last;
+    __unmatched.matched = false;
+
+    __states.push_back(__state());
+    __states.back().__do_      = 0;
+    __states.back().__first_   = __first;
+    __states.back().__current_ = __first;
+    __states.back().__last_    = __last;
+    __states.back().__sub_matches_.resize(mark_count(), __unmatched);
+    __states.back().__loop_data_.resize(__loop_count());
+    __states.back().__node_     = __st;
+    __states.back().__flags_    = __flags;
+    __states.back().__at_first_ = __at_first;
+    int __counter               = 0;
+    int __length                = __last - __first;
+    do {
+      ++__counter;
+      if (__counter % _LIBCPP_REGEX_COMPLEXITY_FACTOR == 0 && __counter / _LIBCPP_REGEX_COMPLEXITY_FACTOR >= __length)
+        __throw_regex_error<regex_constants::error_complexity>();
+      __state& __s = __states.back();
+      if (__s.__node_)
+        __s.__node_->__exec(__s);
+      switch (__s.__do_) {
+      case __state::__end_state:
+        if ((__flags & regex_constants::match_not_null) && __s.__current_ == __first) {
+          __states.pop_back();
+          break;
+        }
+        if ((__flags & regex_constants::__full_match) && __s.__current_ != __last) {
+          __states.pop_back();
+          break;
+        }
+        __m.__matches_[0].first   = __first;
+        __m.__matches_[0].second  = std::next(__first, __s.__current_ - __first);
+        __m.__matches_[0].matched = true;
+        for (unsigned __i = 0; __i < __s.__sub_matches_.size(); ++__i)
+          __m.__matches_[__i + 1] = __s.__sub_matches_[__i];
+        return true;
+      case __state::__accept_and_consume:
+      case __state::__repeat:
+      case __state::__accept_but_not_consume:
+        break;
+      case __state::__split: {
+        __state __snext = __s;
+        __s.__node_->__exec_split(true, __s);
+        __snext.__node_->__exec_split(false, __snext);
+        __states.push_back(std::move(__snext));
+      } break;
+      case __state::__reject:
+        __states.pop_back();
+        break;
+      default:
+        __throw_regex_error<regex_constants::__re_err_unknown>();
+        break;
+      }
+    } while (!__states.empty());
+  }
+  return false;
+}
+
+template <class _CharT, class _Traits>
+template <class _Allocator>
+bool basic_regex<_CharT, _Traits>::__match_at_start_posix_nosubs(
+    const _CharT* __first,
+    const _CharT* __last,
+    match_results<const _CharT*, _Allocator>& __m,
+    regex_constants::match_flag_type __flags,
+    bool __at_first) const {
+  deque<__state> __states;
+  ptrdiff_t __highest_j = 0;
+  ptrdiff_t __np        = std::distance(__first, __last);
+  __node* __st          = __start_.get();
+  if (__st) {
+    __states.push_back(__state());
+    __states.back().__do_      = 0;
+    __states.back().__first_   = __first;
+    __states.back().__current_ = __first;
+    __states.back().__last_    = __last;
+    __states.back().__loop_data_.resize(__loop_count());
+    __states.back().__node_     = __st;
+    __states.back().__flags_    = __flags;
+    __states.back().__at_first_ = __at_first;
+    bool __matched              = false;
+    int __counter               = 0;
+    int __length                = __last - __first;
+    do {
+      ++__counter;
+      if (__counter % _LIBCPP_REGEX_COMPLEXITY_FACTOR == 0 && __counter / _LIBCPP_REGEX_COMPLEXITY_FACTOR >= __length)
+        __throw_regex_error<regex_constants::error_complexity>();
+      __state& __s = __states.back();
+      if (__s.__node_)
+        __s.__node_->__exec(__s);
+      switch (__s.__do_) {
+      case __state::__end_state:
+        if ((__flags & regex_constants::match_not_null) && __s.__current_ == __first) {
+          __states.pop_back();
+          break;
+        }
+        if ((__flags & regex_constants::__full_match) && __s.__current_ != __last) {
+          __states.pop_back();
+          break;
+        }
+        if (!__matched || __highest_j < __s.__current_ - __s.__first_)
+          __highest_j = __s.__current_ - __s.__first_;
+        __matched = true;
+        if (__highest_j == __np)
+          __states.clear();
+        else
+          __states.pop_back();
+        break;
+      case __state::__consume_input:
+        break;
+      case __state::__accept_and_consume:
+        __states.push_front(std::move(__s));
+        __states.pop_back();
+        break;
+      case __state::__repeat:
+      case __state::__accept_but_not_consume:
+        break;
+      case __state::__split: {
+        __state __snext = __s;
+        __s.__node_->__exec_split(true, __s);
+        __snext.__node_->__exec_split(false, __snext);
+        __states.push_back(std::move(__snext));
+      } break;
+      case __state::__reject:
+        __states.pop_back();
+        break;
+      default:
+        __throw_regex_error<regex_constants::__re_err_unknown>();
+        break;
+      }
+    } while (!__states.empty());
+    if (__matched) {
+      __m.__matches_[0].first   = __first;
+      __m.__matches_[0].second  = std::next(__first, __highest_j);
+      __m.__matches_[0].matched = true;
+      return true;
+    }
+  }
+  return false;
+}
+
+template <class _CharT, class _Traits>
+template <class _Allocator>
+bool basic_regex<_CharT, _Traits>::__match_at_start_posix_subs(
+    const _CharT* __first,
+    const _CharT* __last,
+    match_results<const _CharT*, _Allocator>& __m,
+    regex_constants::match_flag_type __flags,
+    bool __at_first) const {
+  vector<__state> __states;
+  __state __best_state;
+  ptrdiff_t __highest_j = 0;
+  ptrdiff_t __np        = std::distance(__first, __last);
+  __node* __st          = __start_.get();
+  if (__st) {
+    sub_match<const _CharT*> __unmatched;
+    __unmatched.first   = __last;
+    __unmatched.second  = __last;
+    __unmatched.matched = false;
+
+    __states.push_back(__state());
+    __states.back().__do_      = 0;
+    __states.back().__first_   = __first;
+    __states.back().__current_ = __first;
+    __states.back().__last_    = __last;
+    __states.back().__sub_matches_.resize(mark_count(), __unmatched);
+    __states.back().__loop_data_.resize(__loop_count());
+    __states.back().__node_     = __st;
+    __states.back().__flags_    = __flags;
+    __states.back().__at_first_ = __at_first;
+    bool __matched              = false;
+    int __counter               = 0;
+    int __length                = __last - __first;
+    do {
+      ++__counter;
+      if (__counter % _LIBCPP_REGEX_COMPLEXITY_FACTOR == 0 && __counter / _LIBCPP_REGEX_COMPLEXITY_FACTOR >= __length)
+        __throw_regex_error<regex_constants::error_complexity>();
+      __state& __s = __states.back();
+      if (__s.__node_)
+        __s.__node_->__exec(__s);
+      switch (__s.__do_) {
+      case __state::__end_state:
+        if ((__flags & regex_constants::match_not_null) && __s.__current_ == __first) {
+          __states.pop_back();
+          break;
+        }
+        if ((__flags & regex_constants::__full_match) && __s.__current_ != __last) {
+          __states.pop_back();
+          break;
+        }
+        if (!__matched || __highest_j < __s.__current_ - __s.__first_) {
+          __highest_j  = __s.__current_ - __s.__first_;
+          __best_state = __s;
+        }
+        __matched = true;
+        if (__highest_j == __np)
+          __states.clear();
+        else
+          __states.pop_back();
+        break;
+      case __state::__accept_and_consume:
+      case __state::__repeat:
+      case __state::__accept_but_not_consume:
+        break;
+      case __state::__split: {
+        __state __snext = __s;
+        __s.__node_->__exec_split(true, __s);
+        __snext.__node_->__exec_split(false, __snext);
+        __states.push_back(std::move(__snext));
+      } break;
+      case __state::__reject:
+        __states.pop_back();
+        break;
+      default:
+        __throw_regex_error<regex_constants::__re_err_unknown>();
+        break;
+      }
+    } while (!__states.empty());
+    if (__matched) {
+      __m.__matches_[0].first   = __first;
+      __m.__matches_[0].second  = std::next(__first, __highest_j);
+      __m.__matches_[0].matched = true;
+      for (unsigned __i = 0; __i < __best_state.__sub_matches_.size(); ++__i)
+        __m.__matches_[__i + 1] = __best_state.__sub_matches_[__i];
+      return true;
+    }
+  }
+  return false;
+}
+
+template <class _CharT, class _Traits>
+template <class _Allocator>
+bool basic_regex<_CharT, _Traits>::__match_at_start(
+    const _CharT* __first,
+    const _CharT* __last,
+    match_results<const _CharT*, _Allocator>& __m,
+    regex_constants::match_flag_type __flags,
+    bool __at_first) const {
+  if (__get_grammar(__flags_) == ECMAScript)
+    return __match_at_start_ecma(__first, __last, __m, __flags, __at_first);
+  if (mark_count() == 0)
+    return __match_at_start_posix_nosubs(__first, __last, __m, __flags, __at_first);
+  return __match_at_start_posix_subs(__first, __last, __m, __flags, __at_first);
+}
+
+template <class _CharT, class _Traits>
+template <class _Allocator>
+bool basic_regex<_CharT, _Traits>::__search(
+    const _CharT* __first,
+    const _CharT* __last,
+    match_results<const _CharT*, _Allocator>& __m,
+    regex_constants::match_flag_type __flags) const {
+  if (__flags & regex_constants::match_prev_avail)
+    __flags &= ~(regex_constants::match_not_bol | regex_constants::match_not_bow);
+
+  __m.__init(1 + mark_count(), __first, __last, __flags & regex_constants::__no_update_pos);
+  if (__match_at_start(__first, __last, __m, __flags, !(__flags & regex_constants::__no_update_pos))) {
+    __m.__prefix_.second  = __m[0].first;
+    __m.__prefix_.matched = __m.__prefix_.first != __m.__prefix_.second;
+    __m.__suffix_.first   = __m[0].second;
+    __m.__suffix_.matched = __m.__suffix_.first != __m.__suffix_.second;
+    return true;
+  }
+  if (__first != __last && !(__flags & regex_constants::match_continuous)) {
+    __flags |= regex_constants::match_prev_avail;
+    for (++__first; __first != __last; ++__first) {
+      __m.__matches_.assign(__m.size(), __m.__unmatched_);
+      if (__match_at_start(__first, __last, __m, __flags, false)) {
+        __m.__prefix_.second  = __m[0].first;
+        __m.__prefix_.matched = __m.__prefix_.first != __m.__prefix_.second;
+        __m.__suffix_.first   = __m[0].second;
+        __m.__suffix_.matched = __m.__suffix_.first != __m.__suffix_.second;
+        return true;
+      }
+      __m.__matches_.assign(__m.size(), __m.__unmatched_);
+    }
+    __m.__matches_.assign(__m.size(), __m.__unmatched_);
+    if (__match_at_start(__first, __last, __m, __flags, false)) {
+      __m.__prefix_.second  = __m[0].first;
+      __m.__prefix_.matched = __m.__prefix_.first != __m.__prefix_.second;
+      __m.__suffix_.first   = __m[0].second;
+      __m.__suffix_.matched = __m.__suffix_.first != __m.__suffix_.second;
+      return true;
+    }
+  }
+  __m.__matches_.clear();
+  return false;
+}
+
+template <class _BidirectionalIterator, class _Allocator, class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+regex_search(_BidirectionalIterator __first,
+             _BidirectionalIterator __last,
+             match_results<_BidirectionalIterator, _Allocator>& __m,
+             const basic_regex<_CharT, _Traits>& __e,
+             regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  int __offset = (__flags & regex_constants::match_prev_avail) ? 1 : 0;
+  basic_string<_CharT> __s(std::prev(__first, __offset), __last);
+  match_results<const _CharT*> __mc;
+  bool __r = __e.__search(__s.data() + __offset, __s.data() + __s.size(), __mc, __flags);
+  __m.__assign(__first, __last, __mc, __flags & regex_constants::__no_update_pos);
+  return __r;
+}
+
+template <class _Iter, class _Allocator, class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+regex_search(__wrap_iter<_Iter> __first,
+             __wrap_iter<_Iter> __last,
+             match_results<__wrap_iter<_Iter>, _Allocator>& __m,
+             const basic_regex<_CharT, _Traits>& __e,
+             regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  match_results<const _CharT*> __mc;
+  bool __r = __e.__search(__first.base(), __last.base(), __mc, __flags);
+  __m.__assign(__first, __last, __mc, __flags & regex_constants::__no_update_pos);
+  return __r;
+}
+
+template <class _Allocator, class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+regex_search(const _CharT* __first,
+             const _CharT* __last,
+             match_results<const _CharT*, _Allocator>& __m,
+             const basic_regex<_CharT, _Traits>& __e,
+             regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  return __e.__search(__first, __last, __m, __flags);
+}
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+regex_search(_BidirectionalIterator __first,
+             _BidirectionalIterator __last,
+             const basic_regex<_CharT, _Traits>& __e,
+             regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  basic_string<_CharT> __s(__first, __last);
+  match_results<const _CharT*> __mc;
+  return __e.__search(__s.data(), __s.data() + __s.size(), __mc, __flags);
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+regex_search(const _CharT* __first,
+             const _CharT* __last,
+             const basic_regex<_CharT, _Traits>& __e,
+             regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  match_results<const _CharT*> __mc;
+  return __e.__search(__first, __last, __mc, __flags);
+}
+
+template <class _CharT, class _Allocator, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+regex_search(const _CharT* __str,
+             match_results<const _CharT*, _Allocator>& __m,
+             const basic_regex<_CharT, _Traits>& __e,
+             regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  return __e.__search(__str, __str + _Traits::length(__str), __m, __flags);
+}
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+regex_search(const _CharT* __str,
+             const basic_regex<_CharT, _Traits>& __e,
+             regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  match_results<const _CharT*> __m;
+  return std::regex_search(__str, __m, __e, __flags);
+}
+
+template <class _ST, class _SA, class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+regex_search(const basic_string<_CharT, _ST, _SA>& __s,
+             const basic_regex<_CharT, _Traits>& __e,
+             regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  match_results<const _CharT*> __mc;
+  return __e.__search(__s.data(), __s.data() + __s.size(), __mc, __flags);
+}
+
+template <class _ST, class _SA, class _Allocator, class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+regex_search(const basic_string<_CharT, _ST, _SA>& __s,
+             match_results<typename basic_string<_CharT, _ST, _SA>::const_iterator, _Allocator>& __m,
+             const basic_regex<_CharT, _Traits>& __e,
+             regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  match_results<const _CharT*> __mc;
+  bool __r = __e.__search(__s.data(), __s.data() + __s.size(), __mc, __flags);
+  __m.__assign(__s.begin(), __s.end(), __mc, __flags & regex_constants::__no_update_pos);
+  return __r;
+}
+
+#if _LIBCPP_STD_VER >= 14
+template <class _ST, class _SA, class _Ap, class _Cp, class _Tp>
+bool regex_search(const basic_string<_Cp, _ST, _SA>&& __s,
+                  match_results<typename basic_string<_Cp, _ST, _SA>::const_iterator, _Ap>&,
+                  const basic_regex<_Cp, _Tp>& __e,
+                  regex_constants::match_flag_type __flags = regex_constants::match_default) = delete;
+#endif
+
+// regex_match
+
+template <class _BidirectionalIterator, class _Allocator, class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI bool
+regex_match(_BidirectionalIterator __first,
+            _BidirectionalIterator __last,
+            match_results<_BidirectionalIterator, _Allocator>& __m,
+            const basic_regex<_CharT, _Traits>& __e,
+            regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  bool __r = std::regex_search(
+      __first, __last, __m, __e, __flags | regex_constants::match_continuous | regex_constants::__full_match);
+  if (__r) {
+    __r = !__m.suffix().matched;
+    if (!__r)
+      __m.__matches_.clear();
+  }
+  return __r;
+}
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+regex_match(_BidirectionalIterator __first,
+            _BidirectionalIterator __last,
+            const basic_regex<_CharT, _Traits>& __e,
+            regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  match_results<_BidirectionalIterator> __m;
+  return std::regex_match(__first, __last, __m, __e, __flags);
+}
+
+template <class _CharT, class _Allocator, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+regex_match(const _CharT* __str,
+            match_results<const _CharT*, _Allocator>& __m,
+            const basic_regex<_CharT, _Traits>& __e,
+            regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  return std::regex_match(__str, __str + _Traits::length(__str), __m, __e, __flags);
+}
+
+template <class _ST, class _SA, class _Allocator, class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+regex_match(const basic_string<_CharT, _ST, _SA>& __s,
+            match_results<typename basic_string<_CharT, _ST, _SA>::const_iterator, _Allocator>& __m,
+            const basic_regex<_CharT, _Traits>& __e,
+            regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  return std::regex_match(__s.begin(), __s.end(), __m, __e, __flags);
+}
+
+#if _LIBCPP_STD_VER >= 14
+template <class _ST, class _SA, class _Allocator, class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+regex_match(const basic_string<_CharT, _ST, _SA>&& __s,
+            match_results<typename basic_string<_CharT, _ST, _SA>::const_iterator, _Allocator>& __m,
+            const basic_regex<_CharT, _Traits>& __e,
+            regex_constants::match_flag_type __flags = regex_constants::match_default) = delete;
+#endif
+
+template <class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+regex_match(const _CharT* __str,
+            const basic_regex<_CharT, _Traits>& __e,
+            regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  return std::regex_match(__str, __str + _Traits::length(__str), __e, __flags);
+}
+
+template <class _ST, class _SA, class _CharT, class _Traits>
+inline _LIBCPP_HIDE_FROM_ABI bool
+regex_match(const basic_string<_CharT, _ST, _SA>& __s,
+            const basic_regex<_CharT, _Traits>& __e,
+            regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  return std::regex_match(__s.begin(), __s.end(), __e, __flags);
+}
+
+// regex_iterator
+
+template <class _BidirectionalIterator,
+          class _CharT  = typename iterator_traits<_BidirectionalIterator>::value_type,
+          class _Traits = regex_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS regex_iterator;
+
+typedef regex_iterator<const char*> cregex_iterator;
+typedef regex_iterator<string::const_iterator> sregex_iterator;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+typedef regex_iterator<const wchar_t*> wcregex_iterator;
+typedef regex_iterator<wstring::const_iterator> wsregex_iterator;
+#endif
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_PREFERRED_NAME(cregex_iterator)
+    _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wcregex_iterator)) _LIBCPP_PREFERRED_NAME(sregex_iterator)
+        _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wsregex_iterator)) regex_iterator {
+public:
+  typedef basic_regex<_CharT, _Traits> regex_type;
+  typedef match_results<_BidirectionalIterator> value_type;
+  typedef ptrdiff_t difference_type;
+  typedef const value_type* pointer;
+  typedef const value_type& reference;
+  typedef forward_iterator_tag iterator_category;
+#if _LIBCPP_STD_VER >= 20
+  typedef input_iterator_tag iterator_concept;
+#endif
+
+private:
+  _BidirectionalIterator __begin_;
+  _BidirectionalIterator __end_;
+  const regex_type* __pregex_;
+  regex_constants::match_flag_type __flags_;
+  value_type __match_;
+
+public:
+  regex_iterator();
+  regex_iterator(_BidirectionalIterator __a,
+                 _BidirectionalIterator __b,
+                 const regex_type& __re,
+                 regex_constants::match_flag_type __m = regex_constants::match_default);
+#if _LIBCPP_STD_VER >= 14
+  regex_iterator(_BidirectionalIterator __a,
+                 _BidirectionalIterator __b,
+                 const regex_type&& __re,
+                 regex_constants::match_flag_type __m = regex_constants::match_default) = delete;
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI bool operator==(const regex_iterator& __x) const;
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI bool operator==(default_sentinel_t) const { return *this == regex_iterator(); }
+#endif
+#if _LIBCPP_STD_VER < 20
+  _LIBCPP_HIDE_FROM_ABI bool operator!=(const regex_iterator& __x) const { return !(*this == __x); }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __match_; }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return std::addressof(__match_); }
+
+  regex_iterator& operator++();
+  _LIBCPP_HIDE_FROM_ABI regex_iterator operator++(int) {
+    regex_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+};
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+regex_iterator<_BidirectionalIterator, _CharT, _Traits>::regex_iterator()
+    : __begin_(), __end_(), __pregex_(nullptr), __flags_(), __match_() {}
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+regex_iterator<_BidirectionalIterator, _CharT, _Traits>::regex_iterator(
+    _BidirectionalIterator __a,
+    _BidirectionalIterator __b,
+    const regex_type& __re,
+    regex_constants::match_flag_type __m)
+    : __begin_(__a), __end_(__b), __pregex_(std::addressof(__re)), __flags_(__m) {
+  std::regex_search(__begin_, __end_, __match_, *__pregex_, __flags_);
+}
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+bool regex_iterator<_BidirectionalIterator, _CharT, _Traits>::operator==(const regex_iterator& __x) const {
+  if (__match_.empty() && __x.__match_.empty())
+    return true;
+  if (__match_.empty() || __x.__match_.empty())
+    return false;
+  return __begin_ == __x.__begin_ && __end_ == __x.__end_ && __pregex_ == __x.__pregex_ && __flags_ == __x.__flags_ &&
+         __match_[0] == __x.__match_[0];
+}
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+regex_iterator<_BidirectionalIterator, _CharT, _Traits>&
+regex_iterator<_BidirectionalIterator, _CharT, _Traits>::operator++() {
+  __flags_ |= regex_constants::__no_update_pos;
+  _BidirectionalIterator __start        = __match_[0].second;
+  _BidirectionalIterator __prefix_start = __start;
+
+  if (__match_[0].first == __match_[0].second) {
+    if (__start == __end_) {
+      __match_ = value_type();
+      return *this;
+    } else if (std::regex_search(__start,
+                                 __end_,
+                                 __match_,
+                                 *__pregex_,
+                                 __flags_ | regex_constants::match_not_null | regex_constants::match_continuous))
+      return *this;
+    else
+      ++__start;
+  }
+
+  __flags_ |= regex_constants::match_prev_avail;
+  if (!std::regex_search(__start, __end_, __match_, *__pregex_, __flags_)) {
+    __match_ = value_type();
+
+  } else {
+    // The Standard mandates that if `regex_search` returns true ([re.regiter.incr]), "`match.prefix().first` shall be
+    // equal to the previous value of `match[0].second`... It is unspecified how the implementation makes these
+    // adjustments." The adjustment is necessary if we incremented `__start` above (the branch that deals with
+    // zero-length matches).
+    auto& __prefix   = __match_.__prefix_;
+    __prefix.first   = __prefix_start;
+    __prefix.matched = __prefix.first != __prefix.second;
+  }
+
+  return *this;
+}
+
+// regex_token_iterator
+
+template <class _BidirectionalIterator,
+          class _CharT  = typename iterator_traits<_BidirectionalIterator>::value_type,
+          class _Traits = regex_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS regex_token_iterator;
+
+typedef regex_token_iterator<const char*> cregex_token_iterator;
+typedef regex_token_iterator<string::const_iterator> sregex_token_iterator;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+typedef regex_token_iterator<const wchar_t*> wcregex_token_iterator;
+typedef regex_token_iterator<wstring::const_iterator> wsregex_token_iterator;
+#endif
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_PREFERRED_NAME(cregex_token_iterator)
+    _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wcregex_token_iterator))
+        _LIBCPP_PREFERRED_NAME(sregex_token_iterator)
+            _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wsregex_token_iterator)) regex_token_iterator {
+public:
+  typedef basic_regex<_CharT, _Traits> regex_type;
+  typedef sub_match<_BidirectionalIterator> value_type;
+  typedef ptrdiff_t difference_type;
+  typedef const value_type* pointer;
+  typedef const value_type& reference;
+  typedef forward_iterator_tag iterator_category;
+#if _LIBCPP_STD_VER >= 20
+  typedef input_iterator_tag iterator_concept;
+#endif
+
+private:
+  typedef regex_iterator<_BidirectionalIterator, _CharT, _Traits> _Position;
+
+  _Position __position_;
+  const value_type* __result_;
+  value_type __suffix_;
+  ptrdiff_t __n_;
+  vector<int> __subs_;
+
+public:
+  regex_token_iterator();
+  regex_token_iterator(_BidirectionalIterator __a,
+                       _BidirectionalIterator __b,
+                       const regex_type& __re,
+                       int __submatch                       = 0,
+                       regex_constants::match_flag_type __m = regex_constants::match_default);
+#if _LIBCPP_STD_VER >= 14
+  regex_token_iterator(_BidirectionalIterator __a,
+                       _BidirectionalIterator __b,
+                       const regex_type&& __re,
+                       int __submatch                       = 0,
+                       regex_constants::match_flag_type __m = regex_constants::match_default) = delete;
+#endif
+
+  regex_token_iterator(_BidirectionalIterator __a,
+                       _BidirectionalIterator __b,
+                       const regex_type& __re,
+                       const vector<int>& __submatches,
+                       regex_constants::match_flag_type __m = regex_constants::match_default);
+#if _LIBCPP_STD_VER >= 14
+  regex_token_iterator(_BidirectionalIterator __a,
+                       _BidirectionalIterator __b,
+                       const regex_type&& __re,
+                       const vector<int>& __submatches,
+                       regex_constants::match_flag_type __m = regex_constants::match_default) = delete;
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+  regex_token_iterator(_BidirectionalIterator __a,
+                       _BidirectionalIterator __b,
+                       const regex_type& __re,
+                       initializer_list<int> __submatches,
+                       regex_constants::match_flag_type __m = regex_constants::match_default);
+
+#  if _LIBCPP_STD_VER >= 14
+  regex_token_iterator(_BidirectionalIterator __a,
+                       _BidirectionalIterator __b,
+                       const regex_type&& __re,
+                       initializer_list<int> __submatches,
+                       regex_constants::match_flag_type __m = regex_constants::match_default) = delete;
+#  endif
+#endif // _LIBCPP_CXX03_LANG
+  template <size_t _Np>
+  regex_token_iterator(_BidirectionalIterator __a,
+                       _BidirectionalIterator __b,
+                       const regex_type& __re,
+                       const int (&__submatches)[_Np],
+                       regex_constants::match_flag_type __m = regex_constants::match_default);
+#if _LIBCPP_STD_VER >= 14
+  template <size_t _Np>
+  regex_token_iterator(_BidirectionalIterator __a,
+                       _BidirectionalIterator __b,
+                       const regex_type&& __re,
+                       const int (&__submatches)[_Np],
+                       regex_constants::match_flag_type __m = regex_constants::match_default) = delete;
+#endif
+
+  regex_token_iterator(const regex_token_iterator&);
+  regex_token_iterator& operator=(const regex_token_iterator&);
+
+  _LIBCPP_HIDE_FROM_ABI bool operator==(const regex_token_iterator& __x) const;
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_HIDE_FROM_ABI bool operator==(default_sentinel_t) const {
+    return *this == regex_token_iterator();
+  }
+#endif
+#if _LIBCPP_STD_VER < 20
+  _LIBCPP_HIDE_FROM_ABI bool operator!=(const regex_token_iterator& __x) const { return !(*this == __x); }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI const value_type& operator*() const { return *__result_; }
+  _LIBCPP_HIDE_FROM_ABI const value_type* operator->() const { return __result_; }
+
+  regex_token_iterator& operator++();
+  _LIBCPP_HIDE_FROM_ABI regex_token_iterator operator++(int) {
+    regex_token_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+private:
+  void __init(_BidirectionalIterator __a, _BidirectionalIterator __b);
+  void __establish_result() {
+    if (__subs_[__n_] == -1)
+      __result_ = &__position_->prefix();
+    else
+      __result_ = &(*__position_)[__subs_[__n_]];
+  }
+};
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>::regex_token_iterator()
+    : __result_(nullptr), __suffix_(), __n_(0) {}
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+void regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>::__init(
+    _BidirectionalIterator __a, _BidirectionalIterator __b) {
+  if (__position_ != _Position())
+    __establish_result();
+  else if (__subs_[__n_] == -1) {
+    __suffix_.matched = true;
+    __suffix_.first   = __a;
+    __suffix_.second  = __b;
+    __result_         = &__suffix_;
+  } else
+    __result_ = nullptr;
+}
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>::regex_token_iterator(
+    _BidirectionalIterator __a,
+    _BidirectionalIterator __b,
+    const regex_type& __re,
+    int __submatch,
+    regex_constants::match_flag_type __m)
+    : __position_(__a, __b, __re, __m), __n_(0), __subs_(1, __submatch) {
+  __init(__a, __b);
+}
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>::regex_token_iterator(
+    _BidirectionalIterator __a,
+    _BidirectionalIterator __b,
+    const regex_type& __re,
+    const vector<int>& __submatches,
+    regex_constants::match_flag_type __m)
+    : __position_(__a, __b, __re, __m), __n_(0), __subs_(__submatches) {
+  __init(__a, __b);
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>::regex_token_iterator(
+    _BidirectionalIterator __a,
+    _BidirectionalIterator __b,
+    const regex_type& __re,
+    initializer_list<int> __submatches,
+    regex_constants::match_flag_type __m)
+    : __position_(__a, __b, __re, __m), __n_(0), __subs_(__submatches) {
+  __init(__a, __b);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+template <size_t _Np>
+regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>::regex_token_iterator(
+    _BidirectionalIterator __a,
+    _BidirectionalIterator __b,
+    const regex_type& __re,
+    const int (&__submatches)[_Np],
+    regex_constants::match_flag_type __m)
+    : __position_(__a, __b, __re, __m), __n_(0), __subs_(begin(__submatches), end(__submatches)) {
+  __init(__a, __b);
+}
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>::regex_token_iterator(const regex_token_iterator& __x)
+    : __position_(__x.__position_),
+      __result_(__x.__result_),
+      __suffix_(__x.__suffix_),
+      __n_(__x.__n_),
+      __subs_(__x.__subs_) {
+  if (__x.__result_ == &__x.__suffix_)
+    __result_ = &__suffix_;
+  else if (__result_ != nullptr)
+    __establish_result();
+}
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>&
+regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>::operator=(const regex_token_iterator& __x) {
+  if (this != &__x) {
+    __position_ = __x.__position_;
+    if (__x.__result_ == &__x.__suffix_)
+      __result_ = &__suffix_;
+    else
+      __result_ = __x.__result_;
+    __suffix_ = __x.__suffix_;
+    __n_      = __x.__n_;
+    __subs_   = __x.__subs_;
+
+    if (__result_ != nullptr && __result_ != &__suffix_)
+      __establish_result();
+  }
+  return *this;
+}
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+bool regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>::operator==(const regex_token_iterator& __x) const {
+  if (__result_ == nullptr && __x.__result_ == nullptr)
+    return true;
+  if (__result_ == &__suffix_ && __x.__result_ == &__x.__suffix_ && __suffix_ == __x.__suffix_)
+    return true;
+  if (__result_ == nullptr || __x.__result_ == nullptr)
+    return false;
+  if (__result_ == &__suffix_ || __x.__result_ == &__x.__suffix_)
+    return false;
+  return __position_ == __x.__position_ && __n_ == __x.__n_ && __subs_ == __x.__subs_;
+}
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>&
+regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>::operator++() {
+  _Position __prev = __position_;
+  if (__result_ == &__suffix_)
+    __result_ = nullptr;
+  else if (static_cast<size_t>(__n_ + 1) < __subs_.size()) {
+    ++__n_;
+    __establish_result();
+  } else {
+    __n_ = 0;
+    ++__position_;
+    if (__position_ != _Position())
+      __establish_result();
+    else {
+      if (std::find(__subs_.begin(), __subs_.end(), -1) != __subs_.end() && __prev->suffix().length() != 0) {
+        __suffix_.matched = true;
+        __suffix_.first   = __prev->suffix().first;
+        __suffix_.second  = __prev->suffix().second;
+        __result_         = &__suffix_;
+      } else
+        __result_ = nullptr;
+    }
+  }
+  return *this;
+}
+
+// regex_replace
+
+template <class _OutputIterator, class _BidirectionalIterator, class _Traits, class _CharT>
+_LIBCPP_HIDE_FROM_ABI _OutputIterator regex_replace(
+    _OutputIterator __output_iter,
+    _BidirectionalIterator __first,
+    _BidirectionalIterator __last,
+    const basic_regex<_CharT, _Traits>& __e,
+    const _CharT* __fmt,
+    regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  typedef regex_iterator<_BidirectionalIterator, _CharT, _Traits> _Iter;
+  _Iter __i(__first, __last, __e, __flags);
+  _Iter __eof;
+  if (__i == __eof) {
+    if (!(__flags & regex_constants::format_no_copy))
+      __output_iter = std::copy(__first, __last, __output_iter);
+  } else {
+    sub_match<_BidirectionalIterator> __lm;
+    for (size_t __len = char_traits<_CharT>::length(__fmt); __i != __eof; ++__i) {
+      if (!(__flags & regex_constants::format_no_copy))
+        __output_iter = std::copy(__i->prefix().first, __i->prefix().second, __output_iter);
+      __output_iter = __i->format(__output_iter, __fmt, __fmt + __len, __flags);
+      __lm          = __i->suffix();
+      if (__flags & regex_constants::format_first_only)
+        break;
+    }
+    if (!(__flags & regex_constants::format_no_copy))
+      __output_iter = std::copy(__lm.first, __lm.second, __output_iter);
+  }
+  return __output_iter;
+}
+
+template <class _OutputIterator, class _BidirectionalIterator, class _Traits, class _CharT, class _ST, class _SA>
+inline _LIBCPP_HIDE_FROM_ABI _OutputIterator regex_replace(
+    _OutputIterator __output_iter,
+    _BidirectionalIterator __first,
+    _BidirectionalIterator __last,
+    const basic_regex<_CharT, _Traits>& __e,
+    const basic_string<_CharT, _ST, _SA>& __fmt,
+    regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  return std::regex_replace(__output_iter, __first, __last, __e, __fmt.c_str(), __flags);
+}
+
+template <class _Traits, class _CharT, class _ST, class _SA, class _FST, class _FSA>
+inline _LIBCPP_HIDE_FROM_ABI basic_string<_CharT, _ST, _SA>
+regex_replace(const basic_string<_CharT, _ST, _SA>& __s,
+              const basic_regex<_CharT, _Traits>& __e,
+              const basic_string<_CharT, _FST, _FSA>& __fmt,
+              regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  basic_string<_CharT, _ST, _SA> __r;
+  std::regex_replace(std::back_inserter(__r), __s.begin(), __s.end(), __e, __fmt.c_str(), __flags);
+  return __r;
+}
+
+template <class _Traits, class _CharT, class _ST, class _SA>
+inline _LIBCPP_HIDE_FROM_ABI basic_string<_CharT, _ST, _SA>
+regex_replace(const basic_string<_CharT, _ST, _SA>& __s,
+              const basic_regex<_CharT, _Traits>& __e,
+              const _CharT* __fmt,
+              regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  basic_string<_CharT, _ST, _SA> __r;
+  std::regex_replace(std::back_inserter(__r), __s.begin(), __s.end(), __e, __fmt, __flags);
+  return __r;
+}
+
+template <class _Traits, class _CharT, class _ST, class _SA>
+inline _LIBCPP_HIDE_FROM_ABI basic_string<_CharT>
+regex_replace(const _CharT* __s,
+              const basic_regex<_CharT, _Traits>& __e,
+              const basic_string<_CharT, _ST, _SA>& __fmt,
+              regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  basic_string<_CharT> __r;
+  std::regex_replace(std::back_inserter(__r), __s, __s + char_traits<_CharT>::length(__s), __e, __fmt.c_str(), __flags);
+  return __r;
+}
+
+template <class _Traits, class _CharT>
+inline _LIBCPP_HIDE_FROM_ABI basic_string<_CharT>
+regex_replace(const _CharT* __s,
+              const basic_regex<_CharT, _Traits>& __e,
+              const _CharT* __fmt,
+              regex_constants::match_flag_type __flags = regex_constants::match_default) {
+  basic_string<_CharT> __r;
+  std::regex_replace(std::back_inserter(__r), __s, __s + char_traits<_CharT>::length(__s), __e, __fmt, __flags);
+  return __r;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace pmr {
+template <class _BidirT>
+using match_results _LIBCPP_AVAILABILITY_PMR =
+    std::match_results<_BidirT, polymorphic_allocator<std::sub_match<_BidirT>>>;
+
+using cmatch _LIBCPP_AVAILABILITY_PMR = match_results<const char*>;
+using smatch _LIBCPP_AVAILABILITY_PMR = match_results<std::pmr::string::const_iterator>;
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using wcmatch _LIBCPP_AVAILABILITY_PMR = match_results<const wchar_t*>;
+using wsmatch _LIBCPP_AVAILABILITY_PMR = match_results<std::pmr::wstring::const_iterator>;
+#  endif
+} // namespace pmr
+_LIBCPP_END_NAMESPACE_STD
+#endif
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <concepts>
+#  include <cstdlib>
+#  include <iosfwd>
+#  include <iterator>
+#  include <mutex>
+#  include <new>
+#  include <type_traits>
+#  include <typeinfo>
+#  include <utility>
+#endif
+
+#endif // _LIBCPP_REGEX
diff --git a/libcxx/include/__cxx03/scoped_allocator b/libcxx/include/__cxx03/scoped_allocator
new file mode 100644
index 000000000000..a49ff465b1d5
--- /dev/null
+++ b/libcxx/include/__cxx03/scoped_allocator
@@ -0,0 +1,573 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_SCOPED_ALLOCATOR
+#define _LIBCPP_SCOPED_ALLOCATOR
+
+/*
+    scoped_allocator synopsis
+
+namespace std
+{
+
+template <class OuterAlloc, class... InnerAllocs>
+class scoped_allocator_adaptor : public OuterAlloc
+{
+    typedef allocator_traits<OuterAlloc> OuterTraits; // exposition only
+    scoped_allocator_adaptor<InnerAllocs...> inner;   // exposition only
+public:
+
+    typedef OuterAlloc outer_allocator_type;
+    typedef see below inner_allocator_type;
+
+    typedef typename OuterTraits::value_type value_type;
+    typedef typename OuterTraits::size_type size_type;
+    typedef typename OuterTraits::difference_type difference_type;
+    typedef typename OuterTraits::pointer pointer;
+    typedef typename OuterTraits::const_pointer const_pointer;
+    typedef typename OuterTraits::void_pointer void_pointer;
+    typedef typename OuterTraits::const_void_pointer const_void_pointer;
+
+    typedef see below propagate_on_container_copy_assignment;
+    typedef see below propagate_on_container_move_assignment;
+    typedef see below propagate_on_container_swap;
+    typedef see below is_always_equal;
+
+    template <class Tp>
+        struct rebind
+        {
+            typedef scoped_allocator_adaptor<
+                OuterTraits::template rebind_alloc<Tp>, InnerAllocs...> other;
+        };
+
+    scoped_allocator_adaptor();
+    template <class OuterA2>
+        scoped_allocator_adaptor(OuterA2&& outerAlloc,
+                                 const InnerAllocs&... innerAllocs) noexcept;
+    scoped_allocator_adaptor(const scoped_allocator_adaptor& other) noexcept;
+    scoped_allocator_adaptor(scoped_allocator_adaptor&& other) noexcept;
+    template <class OuterA2>
+        scoped_allocator_adaptor(const scoped_allocator_adaptor<OuterA2, InnerAllocs...>& other) noexcept;
+    template <class OuterA2>
+        scoped_allocator_adaptor(const scoped_allocator_adaptor<OuterA2, InnerAllocs...>&& other) noexcept;
+
+    scoped_allocator_adaptor& operator=(const scoped_allocator_adaptor&) = default;
+    scoped_allocator_adaptor& operator=(scoped_allocator_adaptor&&) = default;
+    ~scoped_allocator_adaptor();
+
+    inner_allocator_type& inner_allocator() noexcept;
+    const inner_allocator_type& inner_allocator() const noexcept;
+
+    outer_allocator_type& outer_allocator() noexcept;
+    const outer_allocator_type& outer_allocator() const noexcept;
+
+    pointer allocate(size_type n);                           // [[nodiscard]] in C++20
+    pointer allocate(size_type n, const_void_pointer hint);  // [[nodiscard]] in C++20
+    void deallocate(pointer p, size_type n) noexcept;
+
+    size_type max_size() const;
+    template <class T, class... Args> void construct(T* p, Args&& args);
+    template <class T1, class T2, class... Args1, class... Args2>
+        void construct(pair<T1, T2>* p, piecewise_construct t, tuple<Args1...> x,
+                       tuple<Args2...> y);
+    template <class T1, class T2>
+        void construct(pair<T1, T2>* p);
+    template <class T1, class T2, class U, class V>
+        void construct(pair<T1, T2>* p, U&& x, V&& y);
+    template <class T1, class T2, class U, class V>
+        void construct(pair<T1, T2>* p, const pair<U, V>& x);
+    template <class T1, class T2, class U, class V>
+        void construct(pair<T1, T2>* p, pair<U, V>&& x);
+    template <class T> void destroy(T* p);
+
+    template <class T> void destroy(T* p) noexcept;
+
+    scoped_allocator_adaptor select_on_container_copy_construction() const noexcept;
+};
+
+template<class OuterAlloc, class... InnerAllocs>
+    scoped_allocator_adaptor(OuterAlloc, InnerAllocs...)
+        -> scoped_allocator_adaptor<OuterAlloc, InnerAllocs...>;
+
+template <class OuterA1, class OuterA2, class... InnerAllocs>
+    bool
+    operator==(const scoped_allocator_adaptor<OuterA1, InnerAllocs...>& a,
+               const scoped_allocator_adaptor<OuterA2, InnerAllocs...>& b) noexcept;
+
+template <class OuterA1, class OuterA2, class... InnerAllocs>
+    bool
+    operator!=(const scoped_allocator_adaptor<OuterA1, InnerAllocs...>& a,
+               const scoped_allocator_adaptor<OuterA2, InnerAllocs...>& b) noexcept; // removed in C++20
+
+}  // std
+
+*/
+
+#include <__config>
+#include <__memory/allocator_traits.h>
+#include <__memory/uses_allocator_construction.h>
+#include <__type_traits/common_type.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <__utility/piecewise_construct.h>
+#include <tuple>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if !defined(_LIBCPP_CXX03_LANG)
+
+// scoped_allocator_adaptor
+
+template <class... _Allocs>
+class scoped_allocator_adaptor;
+
+template <class... _Allocs>
+struct __get_poc_copy_assignment;
+
+template <class _A0>
+struct __get_poc_copy_assignment<_A0> {
+  static const bool value = allocator_traits<_A0>::propagate_on_container_copy_assignment::value;
+};
+
+template <class _A0, class... _Allocs>
+struct __get_poc_copy_assignment<_A0, _Allocs...> {
+  static const bool value = allocator_traits<_A0>::propagate_on_container_copy_assignment::value ||
+                            __get_poc_copy_assignment<_Allocs...>::value;
+};
+
+template <class... _Allocs>
+struct __get_poc_move_assignment;
+
+template <class _A0>
+struct __get_poc_move_assignment<_A0> {
+  static const bool value = allocator_traits<_A0>::propagate_on_container_move_assignment::value;
+};
+
+template <class _A0, class... _Allocs>
+struct __get_poc_move_assignment<_A0, _Allocs...> {
+  static const bool value = allocator_traits<_A0>::propagate_on_container_move_assignment::value ||
+                            __get_poc_move_assignment<_Allocs...>::value;
+};
+
+template <class... _Allocs>
+struct __get_poc_swap;
+
+template <class _A0>
+struct __get_poc_swap<_A0> {
+  static const bool value = allocator_traits<_A0>::propagate_on_container_swap::value;
+};
+
+template <class _A0, class... _Allocs>
+struct __get_poc_swap<_A0, _Allocs...> {
+  static const bool value =
+      allocator_traits<_A0>::propagate_on_container_swap::value || __get_poc_swap<_Allocs...>::value;
+};
+
+template <class... _Allocs>
+struct __get_is_always_equal;
+
+template <class _A0>
+struct __get_is_always_equal<_A0> {
+  static const bool value = allocator_traits<_A0>::is_always_equal::value;
+};
+
+template <class _A0, class... _Allocs>
+struct __get_is_always_equal<_A0, _Allocs...> {
+  static const bool value = allocator_traits<_A0>::is_always_equal::value && __get_is_always_equal<_Allocs...>::value;
+};
+
+template <class... _Allocs>
+class __scoped_allocator_storage;
+
+template <class _OuterAlloc, class... _InnerAllocs>
+class __scoped_allocator_storage<_OuterAlloc, _InnerAllocs...> : public _OuterAlloc {
+  typedef _OuterAlloc outer_allocator_type;
+
+protected:
+  typedef scoped_allocator_adaptor<_InnerAllocs...> inner_allocator_type;
+
+private:
+  inner_allocator_type __inner_;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI __scoped_allocator_storage() _NOEXCEPT {}
+
+  template <class _OuterA2, __enable_if_t<is_constructible<outer_allocator_type, _OuterA2>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  __scoped_allocator_storage(_OuterA2&& __outer_alloc, const _InnerAllocs&... __inner_allocs) _NOEXCEPT
+      : outer_allocator_type(std::forward<_OuterA2>(__outer_alloc)),
+        __inner_(__inner_allocs...) {}
+
+  template <class _OuterA2, __enable_if_t<is_constructible<outer_allocator_type, const _OuterA2&>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  __scoped_allocator_storage(const __scoped_allocator_storage<_OuterA2, _InnerAllocs...>& __other) _NOEXCEPT
+      : outer_allocator_type(__other.outer_allocator()),
+        __inner_(__other.inner_allocator()) {}
+
+  template <class _OuterA2, __enable_if_t<is_constructible<outer_allocator_type, _OuterA2>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  __scoped_allocator_storage(__scoped_allocator_storage<_OuterA2, _InnerAllocs...>&& __other) _NOEXCEPT
+      : outer_allocator_type(std::move(__other.outer_allocator())),
+        __inner_(std::move(__other.inner_allocator())) {}
+
+  template <class _OuterA2, __enable_if_t<is_constructible<outer_allocator_type, _OuterA2>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI __scoped_allocator_storage(_OuterA2&& __o, const inner_allocator_type& __i) _NOEXCEPT
+      : outer_allocator_type(std::forward<_OuterA2>(__o)),
+        __inner_(__i) {}
+
+  _LIBCPP_HIDE_FROM_ABI inner_allocator_type& inner_allocator() _NOEXCEPT { return __inner_; }
+  _LIBCPP_HIDE_FROM_ABI const inner_allocator_type& inner_allocator() const _NOEXCEPT { return __inner_; }
+
+  _LIBCPP_HIDE_FROM_ABI outer_allocator_type& outer_allocator() _NOEXCEPT {
+    return static_cast<outer_allocator_type&>(*this);
+  }
+  _LIBCPP_HIDE_FROM_ABI const outer_allocator_type& outer_allocator() const _NOEXCEPT {
+    return static_cast<const outer_allocator_type&>(*this);
+  }
+
+  scoped_allocator_adaptor<outer_allocator_type, _InnerAllocs...> _LIBCPP_HIDE_FROM_ABI
+  select_on_container_copy_construction() const _NOEXCEPT {
+    return scoped_allocator_adaptor<outer_allocator_type, _InnerAllocs...>(
+        allocator_traits<outer_allocator_type>::select_on_container_copy_construction(outer_allocator()),
+        allocator_traits<inner_allocator_type>::select_on_container_copy_construction(inner_allocator()));
+  }
+
+  template <class...>
+  friend class __scoped_allocator_storage;
+};
+
+template <class _OuterAlloc>
+class __scoped_allocator_storage<_OuterAlloc> : public _OuterAlloc {
+  typedef _OuterAlloc outer_allocator_type;
+
+protected:
+  typedef scoped_allocator_adaptor<_OuterAlloc> inner_allocator_type;
+
+  _LIBCPP_HIDE_FROM_ABI __scoped_allocator_storage() _NOEXCEPT {}
+
+  template <class _OuterA2, __enable_if_t<is_constructible<outer_allocator_type, _OuterA2>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI __scoped_allocator_storage(_OuterA2&& __outer_alloc) _NOEXCEPT
+      : outer_allocator_type(std::forward<_OuterA2>(__outer_alloc)) {}
+
+  template <class _OuterA2, __enable_if_t<is_constructible<outer_allocator_type, const _OuterA2&>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI __scoped_allocator_storage(const __scoped_allocator_storage<_OuterA2>& __other) _NOEXCEPT
+      : outer_allocator_type(__other.outer_allocator()) {}
+
+  template <class _OuterA2, __enable_if_t<is_constructible<outer_allocator_type, _OuterA2>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI __scoped_allocator_storage(__scoped_allocator_storage<_OuterA2>&& __other) _NOEXCEPT
+      : outer_allocator_type(std::move(__other.outer_allocator())) {}
+
+  _LIBCPP_HIDE_FROM_ABI inner_allocator_type& inner_allocator() _NOEXCEPT {
+    return static_cast<inner_allocator_type&>(*this);
+  }
+  _LIBCPP_HIDE_FROM_ABI const inner_allocator_type& inner_allocator() const _NOEXCEPT {
+    return static_cast<const inner_allocator_type&>(*this);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI outer_allocator_type& outer_allocator() _NOEXCEPT {
+    return static_cast<outer_allocator_type&>(*this);
+  }
+  _LIBCPP_HIDE_FROM_ABI const outer_allocator_type& outer_allocator() const _NOEXCEPT {
+    return static_cast<const outer_allocator_type&>(*this);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI scoped_allocator_adaptor<outer_allocator_type>
+  select_on_container_copy_construction() const _NOEXCEPT {
+    return scoped_allocator_adaptor<outer_allocator_type>(
+        allocator_traits<outer_allocator_type>::select_on_container_copy_construction(outer_allocator()));
+  }
+
+  __scoped_allocator_storage(const outer_allocator_type& __o, const inner_allocator_type& __i) _NOEXCEPT;
+
+  template <class...>
+  friend class __scoped_allocator_storage;
+};
+
+// __outermost
+
+template <class _Alloc>
+decltype(std::declval<_Alloc>().outer_allocator(), true_type()) __has_outer_allocator_test(_Alloc&& __a);
+
+template <class _Alloc>
+false_type __has_outer_allocator_test(const volatile _Alloc& __a);
+
+template <class _Alloc>
+struct __has_outer_allocator
+    : public common_type< decltype(std::__has_outer_allocator_test(std::declval<_Alloc&>()))>::type {};
+
+template <class _Alloc, bool = __has_outer_allocator<_Alloc>::value>
+struct __outermost {
+  typedef _Alloc type;
+  _LIBCPP_HIDE_FROM_ABI type& operator()(type& __a) const _NOEXCEPT { return __a; }
+};
+
+template <class _Alloc>
+struct __outermost<_Alloc, true> {
+  typedef __libcpp_remove_reference_t< decltype(std::declval<_Alloc>().outer_allocator()) > _OuterAlloc;
+  typedef typename __outermost<_OuterAlloc>::type type;
+  _LIBCPP_HIDE_FROM_ABI type& operator()(_Alloc& __a) const _NOEXCEPT {
+    return __outermost<_OuterAlloc>()(__a.outer_allocator());
+  }
+};
+
+template <class _OuterAlloc, class... _InnerAllocs>
+class _LIBCPP_TEMPLATE_VIS scoped_allocator_adaptor<_OuterAlloc, _InnerAllocs...>
+    : public __scoped_allocator_storage<_OuterAlloc, _InnerAllocs...> {
+  typedef __scoped_allocator_storage<_OuterAlloc, _InnerAllocs...> _Base;
+  typedef allocator_traits<_OuterAlloc> _OuterTraits;
+
+public:
+  typedef _OuterAlloc outer_allocator_type;
+  typedef typename _Base::inner_allocator_type inner_allocator_type;
+  typedef typename _OuterTraits::size_type size_type;
+  typedef typename _OuterTraits::difference_type difference_type;
+  typedef typename _OuterTraits::pointer pointer;
+  typedef typename _OuterTraits::const_pointer const_pointer;
+  typedef typename _OuterTraits::void_pointer void_pointer;
+  typedef typename _OuterTraits::const_void_pointer const_void_pointer;
+
+  typedef integral_constant< bool, __get_poc_copy_assignment<outer_allocator_type, _InnerAllocs...>::value >
+      propagate_on_container_copy_assignment;
+  typedef integral_constant< bool, __get_poc_move_assignment<outer_allocator_type, _InnerAllocs...>::value >
+      propagate_on_container_move_assignment;
+  typedef integral_constant< bool, __get_poc_swap<outer_allocator_type, _InnerAllocs...>::value >
+      propagate_on_container_swap;
+  typedef integral_constant< bool, __get_is_always_equal<outer_allocator_type, _InnerAllocs...>::value >
+      is_always_equal;
+
+  template <class _Tp>
+  struct rebind {
+    typedef scoped_allocator_adaptor< typename _OuterTraits::template rebind_alloc<_Tp>, _InnerAllocs... > other;
+  };
+
+  _LIBCPP_HIDE_FROM_ABI scoped_allocator_adaptor() _NOEXCEPT {}
+  template <class _OuterA2, __enable_if_t<is_constructible<outer_allocator_type, _OuterA2>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  scoped_allocator_adaptor(_OuterA2&& __outer_alloc, const _InnerAllocs&... __inner_allocs) _NOEXCEPT
+      : _Base(std::forward<_OuterA2>(__outer_alloc), __inner_allocs...) {}
+  // scoped_allocator_adaptor(const scoped_allocator_adaptor& __other) = default;
+  template <class _OuterA2, __enable_if_t<is_constructible<outer_allocator_type, const _OuterA2&>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  scoped_allocator_adaptor(const scoped_allocator_adaptor<_OuterA2, _InnerAllocs...>& __other) _NOEXCEPT
+      : _Base(__other) {}
+  template <class _OuterA2, __enable_if_t<is_constructible<outer_allocator_type, _OuterA2>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  scoped_allocator_adaptor(scoped_allocator_adaptor<_OuterA2, _InnerAllocs...>&& __other) _NOEXCEPT
+      : _Base(std::move(__other)) {}
+
+  // scoped_allocator_adaptor& operator=(const scoped_allocator_adaptor&) = default;
+  // scoped_allocator_adaptor& operator=(scoped_allocator_adaptor&&) = default;
+  // ~scoped_allocator_adaptor() = default;
+
+  _LIBCPP_HIDE_FROM_ABI inner_allocator_type& inner_allocator() _NOEXCEPT { return _Base::inner_allocator(); }
+  _LIBCPP_HIDE_FROM_ABI const inner_allocator_type& inner_allocator() const _NOEXCEPT {
+    return _Base::inner_allocator();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI outer_allocator_type& outer_allocator() _NOEXCEPT { return _Base::outer_allocator(); }
+  _LIBCPP_HIDE_FROM_ABI const outer_allocator_type& outer_allocator() const _NOEXCEPT {
+    return _Base::outer_allocator();
+  }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI pointer allocate(size_type __n) {
+    return allocator_traits<outer_allocator_type>::allocate(outer_allocator(), __n);
+  }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI pointer allocate(size_type __n, const_void_pointer __hint) {
+    return allocator_traits<outer_allocator_type>::allocate(outer_allocator(), __n, __hint);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void deallocate(pointer __p, size_type __n) _NOEXCEPT {
+    allocator_traits<outer_allocator_type>::deallocate(outer_allocator(), __p, __n);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const {
+    return allocator_traits<outer_allocator_type>::max_size(outer_allocator());
+  }
+
+#  if _LIBCPP_STD_VER >= 20
+  template <class _Type, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI void construct(_Type* __ptr, _Args&&... __args) {
+    using _OM = __outermost<outer_allocator_type>;
+    std::apply(
+        [__ptr, this](auto&&... __newargs) {
+          allocator_traits<typename _OM::type>::construct(
+              _OM()(outer_allocator()), __ptr, std::forward<decltype(__newargs)>(__newargs)...);
+        },
+        std::uses_allocator_construction_args<_Type>(inner_allocator(), std::forward<_Args>(__args)...));
+  }
+#  else
+  template <class _Tp, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI void construct(_Tp* __p, _Args&&... __args) {
+    __construct(__uses_alloc_ctor<_Tp, inner_allocator_type&, _Args...>(), __p, std::forward<_Args>(__args)...);
+  }
+
+  template <class _T1, class _T2, class... _Args1, class... _Args2>
+  _LIBCPP_HIDE_FROM_ABI void
+  construct(pair<_T1, _T2>* __p, piecewise_construct_t, tuple<_Args1...> __x, tuple<_Args2...> __y) {
+    typedef __outermost<outer_allocator_type> _OM;
+    allocator_traits<typename _OM::type>::construct(
+        _OM()(outer_allocator()),
+        __p,
+        piecewise_construct,
+        __transform_tuple(typename __uses_alloc_ctor< _T1, inner_allocator_type&, _Args1... >::type(),
+                          std::move(__x),
+                          typename __make_tuple_indices<sizeof...(_Args1)>::type{}),
+        __transform_tuple(typename __uses_alloc_ctor< _T2, inner_allocator_type&, _Args2... >::type(),
+                          std::move(__y),
+                          typename __make_tuple_indices<sizeof...(_Args2)>::type{}));
+  }
+
+  template <class _T1, class _T2>
+  _LIBCPP_HIDE_FROM_ABI void construct(pair<_T1, _T2>* __p) {
+    construct(__p, piecewise_construct, tuple<>{}, tuple<>{});
+  }
+
+  template <class _T1, class _T2, class _Up, class _Vp>
+  _LIBCPP_HIDE_FROM_ABI void construct(pair<_T1, _T2>* __p, _Up&& __x, _Vp&& __y) {
+    construct(__p,
+              piecewise_construct,
+              std::forward_as_tuple(std::forward<_Up>(__x)),
+              std::forward_as_tuple(std::forward<_Vp>(__y)));
+  }
+
+  template <class _T1, class _T2, class _Up, class _Vp>
+  _LIBCPP_HIDE_FROM_ABI void construct(pair<_T1, _T2>* __p, const pair<_Up, _Vp>& __x) {
+    construct(__p, piecewise_construct, std::forward_as_tuple(__x.first), std::forward_as_tuple(__x.second));
+  }
+
+  template <class _T1, class _T2, class _Up, class _Vp>
+  _LIBCPP_HIDE_FROM_ABI void construct(pair<_T1, _T2>* __p, pair<_Up, _Vp>&& __x) {
+    construct(__p,
+              piecewise_construct,
+              std::forward_as_tuple(std::forward<_Up>(__x.first)),
+              std::forward_as_tuple(std::forward<_Vp>(__x.second)));
+  }
+#  endif
+
+  template <class _Tp>
+  _LIBCPP_HIDE_FROM_ABI void destroy(_Tp* __p) {
+    typedef __outermost<outer_allocator_type> _OM;
+    allocator_traits<typename _OM::type>::destroy(_OM()(outer_allocator()), __p);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI scoped_allocator_adaptor select_on_container_copy_construction() const _NOEXCEPT {
+    return _Base::select_on_container_copy_construction();
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI explicit scoped_allocator_adaptor(
+      outer_allocator_type&& __o, inner_allocator_type&& __i) _NOEXCEPT : _Base(std::move(__o), std::move(__i)) {}
+
+  template <class _Tp, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI void __construct(integral_constant<int, 0>, _Tp* __p, _Args&&... __args) {
+    typedef __outermost<outer_allocator_type> _OM;
+    allocator_traits<typename _OM::type>::construct(_OM()(outer_allocator()), __p, std::forward<_Args>(__args)...);
+  }
+
+  template <class _Tp, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI void __construct(integral_constant<int, 1>, _Tp* __p, _Args&&... __args) {
+    typedef __outermost<outer_allocator_type> _OM;
+    allocator_traits<typename _OM::type>::construct(
+        _OM()(outer_allocator()), __p, allocator_arg, inner_allocator(), std::forward<_Args>(__args)...);
+  }
+
+  template <class _Tp, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI void __construct(integral_constant<int, 2>, _Tp* __p, _Args&&... __args) {
+    typedef __outermost<outer_allocator_type> _OM;
+    allocator_traits<typename _OM::type>::construct(
+        _OM()(outer_allocator()), __p, std::forward<_Args>(__args)..., inner_allocator());
+  }
+
+  template <class... _Args, size_t... _Idx>
+  _LIBCPP_HIDE_FROM_ABI tuple<_Args&&...>
+  __transform_tuple(integral_constant<int, 0>, tuple<_Args...>&& __t, __tuple_indices<_Idx...>) {
+    return std::forward_as_tuple(std::get<_Idx>(std::move(__t))...);
+  }
+
+  template <class... _Args, size_t... _Idx>
+  _LIBCPP_HIDE_FROM_ABI tuple<allocator_arg_t, inner_allocator_type&, _Args&&...>
+  __transform_tuple(integral_constant<int, 1>, tuple<_Args...>&& __t, __tuple_indices<_Idx...>) {
+    using _Tup = tuple<allocator_arg_t, inner_allocator_type&, _Args&&...>;
+    return _Tup(allocator_arg, inner_allocator(), std::get<_Idx>(std::move(__t))...);
+  }
+
+  template <class... _Args, size_t... _Idx>
+  _LIBCPP_HIDE_FROM_ABI tuple<_Args&&..., inner_allocator_type&>
+  __transform_tuple(integral_constant<int, 2>, tuple<_Args...>&& __t, __tuple_indices<_Idx...>) {
+    using _Tup = tuple<_Args&&..., inner_allocator_type&>;
+    return _Tup(std::get<_Idx>(std::move(__t))..., inner_allocator());
+  }
+
+  template <class...>
+  friend class __scoped_allocator_storage;
+};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _OuterAlloc, class... _InnerAllocs>
+scoped_allocator_adaptor(_OuterAlloc, _InnerAllocs...) -> scoped_allocator_adaptor<_OuterAlloc, _InnerAllocs...>;
+#  endif
+
+template <class _OuterA1, class _OuterA2>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const scoped_allocator_adaptor<_OuterA1>& __a, const scoped_allocator_adaptor<_OuterA2>& __b) _NOEXCEPT {
+  return __a.outer_allocator() == __b.outer_allocator();
+}
+
+template <class _OuterA1, class _OuterA2, class _InnerA0, class... _InnerAllocs>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const scoped_allocator_adaptor<_OuterA1, _InnerA0, _InnerAllocs...>& __a,
+           const scoped_allocator_adaptor<_OuterA2, _InnerA0, _InnerAllocs...>& __b) _NOEXCEPT {
+  return __a.outer_allocator() == __b.outer_allocator() && __a.inner_allocator() == __b.inner_allocator();
+}
+
+#  if _LIBCPP_STD_VER <= 17
+
+template <class _OuterA1, class _OuterA2, class... _InnerAllocs>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const scoped_allocator_adaptor<_OuterA1, _InnerAllocs...>& __a,
+                                             const scoped_allocator_adaptor<_OuterA2, _InnerAllocs...>& __b) _NOEXCEPT {
+  return !(__a == __b);
+}
+
+#  endif // _LIBCPP_STD_VER <= 17
+
+#endif // !defined(_LIBCPP_CXX03_LANG)
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#  include <climits>
+#  include <concepts>
+#  include <cstring>
+#  include <ctime>
+#  include <iterator>
+#  include <memory>
+#  include <ratio>
+#  include <stdexcept>
+#  include <type_traits>
+#  include <variant>
+#endif
+
+#endif // _LIBCPP_SCOPED_ALLOCATOR
diff --git a/libcxx/include/__cxx03/semaphore b/libcxx/include/__cxx03/semaphore
new file mode 100644
index 000000000000..95a4375f21c1
--- /dev/null
+++ b/libcxx/include/__cxx03/semaphore
@@ -0,0 +1,188 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_SEMAPHORE
+#define _LIBCPP_SEMAPHORE
+
+/*
+    semaphore synopsis
+
+namespace std {
+
+template<ptrdiff_t least_max_value = implementation-defined>
+class counting_semaphore
+{
+public:
+static constexpr ptrdiff_t max() noexcept;
+
+constexpr explicit counting_semaphore(ptrdiff_t desired);
+~counting_semaphore();
+
+counting_semaphore(const counting_semaphore&) = delete;
+counting_semaphore& operator=(const counting_semaphore&) = delete;
+
+void release(ptrdiff_t update = 1);
+void acquire();
+bool try_acquire() noexcept;
+template<class Rep, class Period>
+    bool try_acquire_for(const chrono::duration<Rep, Period>& rel_time);
+template<class Clock, class Duration>
+    bool try_acquire_until(const chrono::time_point<Clock, Duration>& abs_time);
+
+private:
+ptrdiff_t counter; // exposition only
+};
+
+using binary_semaphore = counting_semaphore<1>;
+
+}
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+
+#  include <__assert>
+#  include <__atomic/atomic_base.h>
+#  include <__atomic/atomic_sync.h>
+#  include <__atomic/memory_order.h>
+#  include <__chrono/time_point.h>
+#  include <__thread/poll_with_backoff.h>
+#  include <__thread/support.h>
+#  include <__thread/timed_backoff_policy.h>
+#  include <cstddef>
+#  include <limits>
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+#  if _LIBCPP_STD_VER >= 14
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+/*
+
+__atomic_semaphore_base is the general-case implementation.
+It is a typical Dijkstra semaphore algorithm over atomics, wait and notify
+functions. It avoids contention against users' own use of those facilities.
+
+*/
+
+#    define _LIBCPP_SEMAPHORE_MAX (numeric_limits<ptrdiff_t>::max())
+
+class __atomic_semaphore_base {
+  __atomic_base<ptrdiff_t> __a_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __atomic_semaphore_base(ptrdiff_t __count) : __a_(__count) {}
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void release(ptrdiff_t __update = 1) {
+    auto __old = __a_.fetch_add(__update, memory_order_release);
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __update <= _LIBCPP_SEMAPHORE_MAX - __old, "update is greater than the expected value");
+    if (__old == 0) {
+      __a_.notify_all();
+    }
+  }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void acquire() {
+    std::__atomic_wait_unless(
+        __a_, [this](ptrdiff_t& __old) { return __try_acquire_impl(__old); }, memory_order_relaxed);
+  }
+  template <class _Rep, class _Period>
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool
+  try_acquire_for(chrono::duration<_Rep, _Period> const& __rel_time) {
+    if (__rel_time == chrono::duration<_Rep, _Period>::zero())
+      return try_acquire();
+    auto const __poll_fn = [this]() { return try_acquire(); };
+    return std::__libcpp_thread_poll_with_backoff(__poll_fn, __libcpp_timed_backoff_policy(), __rel_time);
+  }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool try_acquire() {
+    auto __old = __a_.load(memory_order_relaxed);
+    return __try_acquire_impl(__old);
+  }
+
+private:
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool __try_acquire_impl(ptrdiff_t& __old) {
+    while (true) {
+      if (__old == 0)
+        return false;
+      if (__a_.compare_exchange_weak(__old, __old - 1, memory_order_acquire, memory_order_relaxed))
+        return true;
+    }
+  }
+};
+
+template <ptrdiff_t __least_max_value = _LIBCPP_SEMAPHORE_MAX>
+class _LIBCPP_DEPRECATED_ATOMIC_SYNC counting_semaphore {
+  __atomic_semaphore_base __semaphore_;
+
+public:
+  static_assert(__least_max_value >= 0, "The least maximum value must be a positive number");
+
+  static constexpr ptrdiff_t max() noexcept { return __least_max_value; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit counting_semaphore(ptrdiff_t __count) : __semaphore_(__count) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __count >= 0,
+        "counting_semaphore::counting_semaphore(ptrdiff_t): counting_semaphore cannot be "
+        "initialized with a negative value");
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __count <= max(),
+        "counting_semaphore::counting_semaphore(ptrdiff_t): counting_semaphore cannot be "
+        "initialized with a value greater than max()");
+  }
+  ~counting_semaphore() = default;
+
+  counting_semaphore(const counting_semaphore&)            = delete;
+  counting_semaphore& operator=(const counting_semaphore&) = delete;
+
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void release(ptrdiff_t __update = 1) {
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__update >= 0, "counting_semaphore:release called with a negative value");
+    __semaphore_.release(__update);
+  }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void acquire() { __semaphore_.acquire(); }
+  template <class _Rep, class _Period>
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool
+  try_acquire_for(chrono::duration<_Rep, _Period> const& __rel_time) {
+    return __semaphore_.try_acquire_for(chrono::duration_cast<chrono::nanoseconds>(__rel_time));
+  }
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool try_acquire() { return __semaphore_.try_acquire(); }
+  template <class _Clock, class _Duration>
+  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool
+  try_acquire_until(chrono::time_point<_Clock, _Duration> const& __abs_time) {
+    auto const __current = _Clock::now();
+    if (__current >= __abs_time)
+      return try_acquire();
+    else
+      return try_acquire_for(__abs_time - __current);
+  }
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+using binary_semaphore _LIBCPP_DEPRECATED_ATOMIC_SYNC = counting_semaphore<1>;
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+_LIBCPP_END_NAMESPACE_STD
+
+#  endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_POP_MACROS
+
+#endif // !defined(_LIBCPP_HAS_NO_THREADS)
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <atomic>
+#endif
+
+#endif //_LIBCPP_SEMAPHORE
diff --git a/libcxx/include/__cxx03/set b/libcxx/include/__cxx03/set
new file mode 100644
index 000000000000..7e9661a0149a
--- /dev/null
+++ b/libcxx/include/__cxx03/set
@@ -0,0 +1,1500 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_SET
+#define _LIBCPP_SET
+
+/*
+
+    set synopsis
+
+namespace std
+{
+
+template <class Key, class Compare = less<Key>,
+          class Allocator = allocator<Key>>
+class set
+{
+public:
+    // types:
+    typedef Key                                      key_type;
+    typedef key_type                                 value_type;
+    typedef Compare                                  key_compare;
+    typedef key_compare                              value_compare;
+    typedef Allocator                                allocator_type;
+    typedef typename allocator_type::reference       reference;
+    typedef typename allocator_type::const_reference const_reference;
+    typedef typename allocator_type::size_type       size_type;
+    typedef typename allocator_type::difference_type difference_type;
+    typedef typename allocator_type::pointer         pointer;
+    typedef typename allocator_type::const_pointer   const_pointer;
+
+    typedef implementation-defined                   iterator;
+    typedef implementation-defined                   const_iterator;
+    typedef std::reverse_iterator<iterator>          reverse_iterator;
+    typedef std::reverse_iterator<const_iterator>    const_reverse_iterator;
+    typedef unspecified                              node_type;               // C++17
+    typedef INSERT_RETURN_TYPE<iterator, node_type>  insert_return_type;      // C++17
+
+    // construct/copy/destroy:
+    set()
+        noexcept(
+            is_nothrow_default_constructible<allocator_type>::value &&
+            is_nothrow_default_constructible<key_compare>::value &&
+            is_nothrow_copy_constructible<key_compare>::value);
+    explicit set(const value_compare& comp);
+    set(const value_compare& comp, const allocator_type& a);
+    template <class InputIterator>
+        set(InputIterator first, InputIterator last,
+            const value_compare& comp = value_compare());
+    template <class InputIterator>
+        set(InputIterator first, InputIterator last, const value_compare& comp,
+            const allocator_type& a);
+    template<container-compatible-range<value_type> R>
+      set(from_range_t, R&& rg, const Compare& comp = Compare(), const Allocator& = Allocator()); // C++23
+    set(const set& s);
+    set(set&& s)
+        noexcept(
+            is_nothrow_move_constructible<allocator_type>::value &&
+            is_nothrow_move_constructible<key_compare>::value);
+    explicit set(const allocator_type& a);
+    set(const set& s, const allocator_type& a);
+    set(set&& s, const allocator_type& a);
+    set(initializer_list<value_type> il, const value_compare& comp = value_compare());
+    set(initializer_list<value_type> il, const value_compare& comp,
+        const allocator_type& a);
+    template <class InputIterator>
+        set(InputIterator first, InputIterator last, const allocator_type& a)
+            : set(first, last, Compare(), a) {}  // C++14
+    template<container-compatible-range<value_type> R>
+      set(from_range_t, R&& rg, const Allocator& a))
+        : set(from_range, std::forward<R>(rg), Compare(), a) { } // C++23
+    set(initializer_list<value_type> il, const allocator_type& a)
+        : set(il, Compare(), a) {}  // C++14
+    ~set();
+
+    set& operator=(const set& s);
+    set& operator=(set&& s)
+        noexcept(
+            allocator_type::propagate_on_container_move_assignment::value &&
+            is_nothrow_move_assignable<allocator_type>::value &&
+            is_nothrow_move_assignable<key_compare>::value);
+    set& operator=(initializer_list<value_type> il);
+
+    // iterators:
+          iterator begin() noexcept;
+    const_iterator begin() const noexcept;
+          iterator end() noexcept;
+    const_iterator end()   const noexcept;
+
+          reverse_iterator rbegin() noexcept;
+    const_reverse_iterator rbegin() const noexcept;
+          reverse_iterator rend() noexcept;
+    const_reverse_iterator rend()   const noexcept;
+
+    const_iterator         cbegin()  const noexcept;
+    const_iterator         cend()    const noexcept;
+    const_reverse_iterator crbegin() const noexcept;
+    const_reverse_iterator crend()   const noexcept;
+
+    // capacity:
+    bool      empty()    const noexcept;
+    size_type size()     const noexcept;
+    size_type max_size() const noexcept;
+
+    // modifiers:
+    template <class... Args>
+        pair<iterator, bool> emplace(Args&&... args);
+    template <class... Args>
+        iterator emplace_hint(const_iterator position, Args&&... args);
+    pair<iterator,bool> insert(const value_type& v);
+    pair<iterator,bool> insert(value_type&& v);
+    iterator insert(const_iterator position, const value_type& v);
+    iterator insert(const_iterator position, value_type&& v);
+    template <class InputIterator>
+        void insert(InputIterator first, InputIterator last);
+    template<container-compatible-range<value_type> R>
+      void insert_range(R&& rg);                                                      // C++23
+    void insert(initializer_list<value_type> il);
+
+    node_type extract(const_iterator position);                                       // C++17
+    node_type extract(const key_type& x);                                             // C++17
+    insert_return_type insert(node_type&& nh);                                        // C++17
+    iterator insert(const_iterator hint, node_type&& nh);                             // C++17
+
+    iterator  erase(const_iterator position);
+    iterator  erase(iterator position);  // C++14
+    size_type erase(const key_type& k);
+    iterator  erase(const_iterator first, const_iterator last);
+    void clear() noexcept;
+
+    template<class C2>
+      void merge(set<Key, C2, Allocator>& source);         // C++17
+    template<class C2>
+      void merge(set<Key, C2, Allocator>&& source);        // C++17
+    template<class C2>
+      void merge(multiset<Key, C2, Allocator>& source);    // C++17
+    template<class C2>
+      void merge(multiset<Key, C2, Allocator>&& source);   // C++17
+
+    void swap(set& s)
+        noexcept(
+            __is_nothrow_swappable<key_compare>::value &&
+            (!allocator_type::propagate_on_container_swap::value ||
+             __is_nothrow_swappable<allocator_type>::value));
+
+    // observers:
+    allocator_type get_allocator() const noexcept;
+    key_compare    key_comp()      const;
+    value_compare  value_comp()    const;
+
+    // set operations:
+          iterator find(const key_type& k);
+    const_iterator find(const key_type& k) const;
+    template<typename K>
+        iterator find(const K& x);
+    template<typename K>
+        const_iterator find(const K& x) const;  // C++14
+
+    template<typename K>
+        size_type count(const K& x) const;        // C++14
+    size_type      count(const key_type& k) const;
+
+    bool           contains(const key_type& x) const;  // C++20
+    template<class K> bool contains(const K& x) const; // C++20
+
+          iterator lower_bound(const key_type& k);
+    const_iterator lower_bound(const key_type& k) const;
+    template<typename K>
+        iterator lower_bound(const K& x);              // C++14
+    template<typename K>
+        const_iterator lower_bound(const K& x) const;  // C++14
+
+          iterator upper_bound(const key_type& k);
+    const_iterator upper_bound(const key_type& k) const;
+    template<typename K>
+        iterator upper_bound(const K& x);              // C++14
+    template<typename K>
+        const_iterator upper_bound(const K& x) const;  // C++14
+    pair<iterator,iterator>             equal_range(const key_type& k);
+    pair<const_iterator,const_iterator> equal_range(const key_type& k) const;
+    template<typename K>
+        pair<iterator,iterator>             equal_range(const K& x);        // C++14
+    template<typename K>
+        pair<const_iterator,const_iterator> equal_range(const K& x) const;  // C++14
+};
+
+template <class InputIterator,
+      class Compare = less<typename iterator_traits<InputIterator>::value_type>,
+      class Allocator = allocator<typename iterator_traits<InputIterator>::value_type>>
+set(InputIterator, InputIterator,
+    Compare = Compare(), Allocator = Allocator())
+  -> set<typename iterator_traits<InputIterator>::value_type, Compare, Allocator>; // C++17
+
+template<ranges::input_range R, class Compare = less<ranges::range_value_t<R>>,
+         class Allocator = allocator<ranges::range_value_t<R>>>
+  set(from_range_t, R&&, Compare = Compare(), Allocator = Allocator())
+    -> set<ranges::range_value_t<R>, Compare, Allocator>; // C++23
+
+template<class Key, class Compare = less<Key>, class Allocator = allocator<Key>>
+set(initializer_list<Key>, Compare = Compare(), Allocator = Allocator())
+  -> set<Key, Compare, Allocator>; // C++17
+
+template<class InputIterator, class Allocator>
+set(InputIterator, InputIterator, Allocator)
+  -> set<typename iterator_traits<InputIterator>::value_type,
+          less<typename iterator_traits<InputIterator>::value_type>, Allocator>; // C++17
+
+template<ranges::input_range R, class Allocator>
+  set(from_range_t, R&&, Allocator)
+    -> set<ranges::range_value_t<R>, less<ranges::range_value_t<R>>, Allocator>; // C++23
+
+template<class Key, class Allocator>
+set(initializer_list<Key>, Allocator) -> set<Key, less<Key>, Allocator>; // C++17
+
+template <class Key, class Compare, class Allocator>
+bool
+operator==(const set<Key, Compare, Allocator>& x,
+           const set<Key, Compare, Allocator>& y);
+
+template <class Key, class Compare, class Allocator>
+bool
+operator< (const set<Key, Compare, Allocator>& x,
+           const set<Key, Compare, Allocator>& y);                                // removed in C++20
+
+template <class Key, class Compare, class Allocator>
+bool
+operator!=(const set<Key, Compare, Allocator>& x,
+           const set<Key, Compare, Allocator>& y);                                // removed in C++20
+
+template <class Key, class Compare, class Allocator>
+bool
+operator> (const set<Key, Compare, Allocator>& x,
+           const set<Key, Compare, Allocator>& y);                                // removed in C++20
+
+template <class Key, class Compare, class Allocator>
+bool
+operator>=(const set<Key, Compare, Allocator>& x,
+           const set<Key, Compare, Allocator>& y);                                // removed in C++20
+
+template <class Key, class Compare, class Allocator>
+bool
+operator<=(const set<Key, Compare, Allocator>& x,
+           const set<Key, Compare, Allocator>& y);                                // removed in C++20
+
+template<class Key, class Compare, class Allocator>
+  synth-three-way-result<Key> operator<=>(const set<Key, Compare, Allocator>& x,
+                                          const set<Key, Compare, Allocator>& y); // since C++20
+
+// specialized algorithms:
+template <class Key, class Compare, class Allocator>
+void
+swap(set<Key, Compare, Allocator>& x, set<Key, Compare, Allocator>& y)
+    noexcept(noexcept(x.swap(y)));
+
+template <class Key, class Compare, class Allocator, class Predicate>
+typename set<Key, Compare, Allocator>::size_type
+erase_if(set<Key, Compare, Allocator>& c, Predicate pred);  // C++20
+
+template <class Key, class Compare = less<Key>,
+          class Allocator = allocator<Key>>
+class multiset
+{
+public:
+    // types:
+    typedef Key                                      key_type;
+    typedef key_type                                 value_type;
+    typedef Compare                                  key_compare;
+    typedef key_compare                              value_compare;
+    typedef Allocator                                allocator_type;
+    typedef typename allocator_type::reference       reference;
+    typedef typename allocator_type::const_reference const_reference;
+    typedef typename allocator_type::size_type       size_type;
+    typedef typename allocator_type::difference_type difference_type;
+    typedef typename allocator_type::pointer         pointer;
+    typedef typename allocator_type::const_pointer   const_pointer;
+
+    typedef implementation-defined                   iterator;
+    typedef implementation-defined                   const_iterator;
+    typedef std::reverse_iterator<iterator>          reverse_iterator;
+    typedef std::reverse_iterator<const_iterator>    const_reverse_iterator;
+    typedef unspecified                              node_type;               // C++17
+
+    // construct/copy/destroy:
+    multiset()
+        noexcept(
+            is_nothrow_default_constructible<allocator_type>::value &&
+            is_nothrow_default_constructible<key_compare>::value &&
+            is_nothrow_copy_constructible<key_compare>::value);
+    explicit multiset(const value_compare& comp);
+    multiset(const value_compare& comp, const allocator_type& a);
+    template <class InputIterator>
+        multiset(InputIterator first, InputIterator last,
+                 const value_compare& comp = value_compare());
+    template <class InputIterator>
+        multiset(InputIterator first, InputIterator last,
+                 const value_compare& comp, const allocator_type& a);
+    template<container-compatible-range<value_type> R>
+      multiset(from_range_t, R&& rg,
+               const Compare& comp = Compare(), const Allocator& = Allocator()); // C++23
+    multiset(const multiset& s);
+    multiset(multiset&& s)
+        noexcept(
+            is_nothrow_move_constructible<allocator_type>::value &&
+            is_nothrow_move_constructible<key_compare>::value);
+    explicit multiset(const allocator_type& a);
+    multiset(const multiset& s, const allocator_type& a);
+    multiset(multiset&& s, const allocator_type& a);
+    multiset(initializer_list<value_type> il, const value_compare& comp = value_compare());
+    multiset(initializer_list<value_type> il, const value_compare& comp,
+             const allocator_type& a);
+    template <class InputIterator>
+        multiset(InputIterator first, InputIterator last, const allocator_type& a)
+            : set(first, last, Compare(), a) {}  // C++14
+    template<container-compatible-range<value_type> R>
+      multiset(from_range_t, R&& rg, const Allocator& a))
+        : multiset(from_range, std::forward<R>(rg), Compare(), a) { } // C++23
+    multiset(initializer_list<value_type> il, const allocator_type& a)
+        : set(il, Compare(), a) {}  // C++14
+    ~multiset();
+
+    multiset& operator=(const multiset& s);
+    multiset& operator=(multiset&& s)
+        noexcept(
+            allocator_type::propagate_on_container_move_assignment::value &&
+            is_nothrow_move_assignable<allocator_type>::value &&
+            is_nothrow_move_assignable<key_compare>::value);
+    multiset& operator=(initializer_list<value_type> il);
+
+    // iterators:
+          iterator begin() noexcept;
+    const_iterator begin() const noexcept;
+          iterator end() noexcept;
+    const_iterator end()   const noexcept;
+
+          reverse_iterator rbegin() noexcept;
+    const_reverse_iterator rbegin() const noexcept;
+          reverse_iterator rend() noexcept;
+    const_reverse_iterator rend()   const noexcept;
+
+    const_iterator         cbegin()  const noexcept;
+    const_iterator         cend()    const noexcept;
+    const_reverse_iterator crbegin() const noexcept;
+    const_reverse_iterator crend()   const noexcept;
+
+    // capacity:
+    bool      empty()    const noexcept;
+    size_type size()     const noexcept;
+    size_type max_size() const noexcept;
+
+    // modifiers:
+    template <class... Args>
+        iterator emplace(Args&&... args);
+    template <class... Args>
+        iterator emplace_hint(const_iterator position, Args&&... args);
+    iterator insert(const value_type& v);
+    iterator insert(value_type&& v);
+    iterator insert(const_iterator position, const value_type& v);
+    iterator insert(const_iterator position, value_type&& v);
+    template <class InputIterator>
+        void insert(InputIterator first, InputIterator last);
+    template<container-compatible-range<value_type> R>
+      void insert_range(R&& rg);                                                      // C++23
+    void insert(initializer_list<value_type> il);
+
+    node_type extract(const_iterator position);                                       // C++17
+    node_type extract(const key_type& x);                                             // C++17
+    iterator insert(node_type&& nh);                                                  // C++17
+    iterator insert(const_iterator hint, node_type&& nh);                             // C++17
+
+    iterator  erase(const_iterator position);
+    iterator  erase(iterator position);  // C++14
+    size_type erase(const key_type& k);
+    iterator  erase(const_iterator first, const_iterator last);
+    void clear() noexcept;
+
+    template<class C2>
+      void merge(multiset<Key, C2, Allocator>& source);    // C++17
+    template<class C2>
+      void merge(multiset<Key, C2, Allocator>&& source);   // C++17
+    template<class C2>
+      void merge(set<Key, C2, Allocator>& source);         // C++17
+    template<class C2>
+      void merge(set<Key, C2, Allocator>&& source);        // C++17
+
+    void swap(multiset& s)
+        noexcept(
+            __is_nothrow_swappable<key_compare>::value &&
+            (!allocator_type::propagate_on_container_swap::value ||
+             __is_nothrow_swappable<allocator_type>::value));
+
+    // observers:
+    allocator_type get_allocator() const noexcept;
+    key_compare    key_comp()      const;
+    value_compare  value_comp()    const;
+
+    // set operations:
+          iterator find(const key_type& k);
+    const_iterator find(const key_type& k) const;
+    template<typename K>
+        iterator find(const K& x);
+    template<typename K>
+        const_iterator find(const K& x) const;  // C++14
+
+    template<typename K>
+        size_type count(const K& x) const;      // C++14
+    size_type      count(const key_type& k) const;
+
+    bool           contains(const key_type& x) const;  // C++20
+    template<class K> bool contains(const K& x) const; // C++20
+
+          iterator lower_bound(const key_type& k);
+    const_iterator lower_bound(const key_type& k) const;
+    template<typename K>
+        iterator lower_bound(const K& x);              // C++14
+    template<typename K>
+        const_iterator lower_bound(const K& x) const;  // C++14
+
+          iterator upper_bound(const key_type& k);
+    const_iterator upper_bound(const key_type& k) const;
+    template<typename K>
+        iterator upper_bound(const K& x);              // C++14
+    template<typename K>
+        const_iterator upper_bound(const K& x) const;  // C++14
+
+    pair<iterator,iterator>             equal_range(const key_type& k);
+    pair<const_iterator,const_iterator> equal_range(const key_type& k) const;
+    template<typename K>
+        pair<iterator,iterator>             equal_range(const K& x);        // C++14
+    template<typename K>
+        pair<const_iterator,const_iterator> equal_range(const K& x) const;  // C++14
+};
+
+template <class InputIterator,
+      class Compare = less<typename iterator_traits<InputIterator>::value_type>,
+      class Allocator = allocator<typename iterator_traits<InputIterator>::value_type>>
+multiset(InputIterator, InputIterator,
+    Compare = Compare(), Allocator = Allocator())
+  -> multiset<typename iterator_traits<InputIterator>::value_type, Compare, Allocator>; // C++17
+
+template<ranges::input_range R, class Compare = less<ranges::range_value_t<R>>,
+          class Allocator = allocator<ranges::range_value_t<R>>>
+  multiset(from_range_t, R&&, Compare = Compare(), Allocator = Allocator())
+    -> multiset<ranges::range_value_t<R>, Compare, Allocator>;
+
+template<class Key, class Compare = less<Key>, class Allocator = allocator<Key>>
+multiset(initializer_list<Key>, Compare = Compare(), Allocator = Allocator())
+  -> multiset<Key, Compare, Allocator>; // C++17
+
+template<class InputIterator, class Allocator>
+multiset(InputIterator, InputIterator, Allocator)
+  -> multiset<typename iterator_traits<InputIterator>::value_type,
+          less<typename iterator_traits<InputIterator>::value_type>, Allocator>; // C++17
+
+template<ranges::input_range R, class Allocator>
+  multiset(from_range_t, R&&, Allocator)
+    -> multiset<ranges::range_value_t<R>, less<ranges::range_value_t<R>>, Allocator>;
+
+template<class Key, class Allocator>
+multiset(initializer_list<Key>, Allocator) -> multiset<Key, less<Key>, Allocator>; // C++17
+
+template <class Key, class Compare, class Allocator>
+bool
+operator==(const multiset<Key, Compare, Allocator>& x,
+           const multiset<Key, Compare, Allocator>& y);
+
+template <class Key, class Compare, class Allocator>
+bool
+operator< (const multiset<Key, Compare, Allocator>& x,
+           const multiset<Key, Compare, Allocator>& y);                                // removed in C++20
+
+template <class Key, class Compare, class Allocator>
+bool
+operator!=(const multiset<Key, Compare, Allocator>& x,
+           const multiset<Key, Compare, Allocator>& y);                                // removed in C++20
+
+template <class Key, class Compare, class Allocator>
+bool
+operator> (const multiset<Key, Compare, Allocator>& x,
+           const multiset<Key, Compare, Allocator>& y);                                // removed in C++20
+
+template <class Key, class Compare, class Allocator>
+bool
+operator>=(const multiset<Key, Compare, Allocator>& x,
+           const multiset<Key, Compare, Allocator>& y);                                // removed in C++20
+
+template <class Key, class Compare, class Allocator>
+bool
+operator<=(const multiset<Key, Compare, Allocator>& x,
+           const multiset<Key, Compare, Allocator>& y);                                // removed in C++20
+
+template<class Key, class Compare, class Allocator>
+  synth-three-way-result<Key> operator<=>(const multiset<Key, Compare, Allocator>& x,
+                                          const multiset<Key, Compare, Allocator>& y); // since C++20
+
+// specialized algorithms:
+template <class Key, class Compare, class Allocator>
+void
+swap(multiset<Key, Compare, Allocator>& x, multiset<Key, Compare, Allocator>& y)
+    noexcept(noexcept(x.swap(y)));
+
+template <class Key, class Compare, class Allocator, class Predicate>
+typename multiset<Key, Compare, Allocator>::size_type
+erase_if(multiset<Key, Compare, Allocator>& c, Predicate pred);  // C++20
+
+}  // std
+
+*/
+
+#include <__algorithm/equal.h>
+#include <__algorithm/lexicographical_compare.h>
+#include <__algorithm/lexicographical_compare_three_way.h>
+#include <__assert>
+#include <__config>
+#include <__functional/is_transparent.h>
+#include <__functional/operations.h>
+#include <__iterator/erase_if_container.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/ranges_iterator_traits.h>
+#include <__iterator/reverse_iterator.h>
+#include <__memory/allocator.h>
+#include <__memory_resource/polymorphic_allocator.h>
+#include <__node_handle>
+#include <__ranges/concepts.h>
+#include <__ranges/container_compatible_range.h>
+#include <__ranges/from_range.h>
+#include <__tree>
+#include <__type_traits/is_allocator.h>
+#include <__utility/forward.h>
+#include <version>
+
+// standard-mandated includes
+
+// [iterator.range]
+#include <__iterator/access.h>
+#include <__iterator/data.h>
+#include <__iterator/empty.h>
+#include <__iterator/reverse_access.h>
+#include <__iterator/size.h>
+
+// [associative.set.syn]
+#include <compare>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Key, class _Compare, class _Allocator>
+class multiset;
+
+template <class _Key, class _Compare = less<_Key>, class _Allocator = allocator<_Key> >
+class _LIBCPP_TEMPLATE_VIS set {
+public:
+  // types:
+  typedef _Key key_type;
+  typedef key_type value_type;
+  typedef __type_identity_t<_Compare> key_compare;
+  typedef key_compare value_compare;
+  typedef __type_identity_t<_Allocator> allocator_type;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+
+  static_assert(is_same<typename allocator_type::value_type, value_type>::value,
+                "Allocator::value_type must be same type as value_type");
+
+private:
+  typedef __tree<value_type, value_compare, allocator_type> __base;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+
+  static_assert(__check_valid_allocator<allocator_type>::value, "");
+
+  __base __tree_;
+
+public:
+  typedef typename __base::pointer pointer;
+  typedef typename __base::const_pointer const_pointer;
+  typedef typename __base::size_type size_type;
+  typedef typename __base::difference_type difference_type;
+  typedef typename __base::const_iterator iterator;
+  typedef typename __base::const_iterator const_iterator;
+  typedef std::reverse_iterator<iterator> reverse_iterator;
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+
+#if _LIBCPP_STD_VER >= 17
+  typedef __set_node_handle<typename __base::__node, allocator_type> node_type;
+  typedef __insert_return_type<iterator, node_type> insert_return_type;
+#endif
+
+  template <class _Key2, class _Compare2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS set;
+  template <class _Key2, class _Compare2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS multiset;
+
+  _LIBCPP_HIDE_FROM_ABI set() _NOEXCEPT_(
+      is_nothrow_default_constructible<allocator_type>::value&& is_nothrow_default_constructible<key_compare>::value&&
+          is_nothrow_copy_constructible<key_compare>::value)
+      : __tree_(value_compare()) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit set(const value_compare& __comp) _NOEXCEPT_(
+      is_nothrow_default_constructible<allocator_type>::value&& is_nothrow_copy_constructible<key_compare>::value)
+      : __tree_(__comp) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit set(const value_compare& __comp, const allocator_type& __a) : __tree_(__comp, __a) {}
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI set(_InputIterator __f, _InputIterator __l, const value_compare& __comp = value_compare())
+      : __tree_(__comp) {
+    insert(__f, __l);
+  }
+
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI
+  set(_InputIterator __f, _InputIterator __l, const value_compare& __comp, const allocator_type& __a)
+      : __tree_(__comp, __a) {
+    insert(__f, __l);
+  }
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI
+  set(from_range_t,
+      _Range&& __range,
+      const key_compare& __comp = key_compare(),
+      const allocator_type& __a = allocator_type())
+      : __tree_(__comp, __a) {
+    insert_range(std::forward<_Range>(__range));
+  }
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI set(_InputIterator __f, _InputIterator __l, const allocator_type& __a)
+      : set(__f, __l, key_compare(), __a) {}
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI set(from_range_t, _Range&& __range, const allocator_type& __a)
+      : set(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI set(const set& __s) : __tree_(__s.__tree_) { insert(__s.begin(), __s.end()); }
+
+  _LIBCPP_HIDE_FROM_ABI set& operator=(const set& __s) {
+    __tree_ = __s.__tree_;
+    return *this;
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI set(set&& __s) noexcept(is_nothrow_move_constructible<__base>::value)
+      : __tree_(std::move(__s.__tree_)) {}
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI explicit set(const allocator_type& __a) : __tree_(__a) {}
+
+  _LIBCPP_HIDE_FROM_ABI set(const set& __s, const allocator_type& __a) : __tree_(__s.__tree_.value_comp(), __a) {
+    insert(__s.begin(), __s.end());
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI set(set&& __s, const allocator_type& __a);
+
+  _LIBCPP_HIDE_FROM_ABI set(initializer_list<value_type> __il, const value_compare& __comp = value_compare())
+      : __tree_(__comp) {
+    insert(__il.begin(), __il.end());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI set(initializer_list<value_type> __il, const value_compare& __comp, const allocator_type& __a)
+      : __tree_(__comp, __a) {
+    insert(__il.begin(), __il.end());
+  }
+
+#  if _LIBCPP_STD_VER >= 14
+  _LIBCPP_HIDE_FROM_ABI set(initializer_list<value_type> __il, const allocator_type& __a)
+      : set(__il, key_compare(), __a) {}
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI set& operator=(initializer_list<value_type> __il) {
+    __tree_.__assign_unique(__il.begin(), __il.end());
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI set& operator=(set&& __s) noexcept(is_nothrow_move_assignable<__base>::value) {
+    __tree_ = std::move(__s.__tree_);
+    return *this;
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI ~set() { static_assert(sizeof(__diagnose_non_const_comparator<_Key, _Compare>()), ""); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __tree_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __tree_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __tree_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __tree_.end(); }
+
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return rbegin(); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __tree_.size() == 0; }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __tree_.size(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __tree_.max_size(); }
+
+  // modifiers:
+#ifndef _LIBCPP_CXX03_LANG
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> emplace(_Args&&... __args) {
+    return __tree_.__emplace_unique(std::forward<_Args>(__args)...);
+  }
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __p, _Args&&... __args) {
+    return __tree_.__emplace_hint_unique(__p, std::forward<_Args>(__args)...);
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(const value_type& __v) { return __tree_.__insert_unique(__v); }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __v) {
+    return __tree_.__insert_unique(__p, __v);
+  }
+
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __f, _InputIterator __l) {
+    for (const_iterator __e = cend(); __f != __l; ++__f)
+      __tree_.__insert_unique(__e, *__f);
+  }
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
+    const_iterator __end = cend();
+    for (auto&& __element : __range) {
+      __tree_.__insert_unique(__end, std::forward<decltype(__element)>(__element));
+    }
+  }
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(value_type&& __v) {
+    return __tree_.__insert_unique(std::move(__v));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __v) {
+    return __tree_.__insert_unique(__p, std::move(__v));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __tree_.erase(__p); }
+  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __tree_.__erase_unique(__k); }
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __f, const_iterator __l) { return __tree_.erase(__f, __l); }
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __tree_.clear(); }
+
+#if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI insert_return_type insert(node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to set::insert()");
+    return __tree_.template __node_handle_insert_unique< node_type, insert_return_type>(std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to set::insert()");
+    return __tree_.template __node_handle_insert_unique<node_type>(__hint, std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
+    return __tree_.template __node_handle_extract<node_type>(__key);
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
+    return __tree_.template __node_handle_extract<node_type>(__it);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(set<key_type, _Compare2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __tree_.__node_handle_merge_unique(__source.__tree_);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(set<key_type, _Compare2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __tree_.__node_handle_merge_unique(__source.__tree_);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(multiset<key_type, _Compare2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __tree_.__node_handle_merge_unique(__source.__tree_);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(multiset<key_type, _Compare2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __tree_.__node_handle_merge_unique(__source.__tree_);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI void swap(set& __s) _NOEXCEPT_(__is_nothrow_swappable_v<__base>) { __tree_.swap(__s.__tree_); }
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT { return __tree_.__alloc(); }
+  _LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __tree_.value_comp(); }
+  _LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return __tree_.value_comp(); }
+
+  // set operations:
+  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
+    return __tree_.find(__k);
+  }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
+    return __tree_.find(__k);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_unique(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
+    return __tree_.__count_multi(__k);
+  }
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
+    return find(__k) != end();
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
+    return __tree_.lower_bound(__k);
+  }
+
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
+    return __tree_.lower_bound(__k);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
+    return __tree_.upper_bound(__k);
+  }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
+    return __tree_.upper_bound(__k);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
+    return __tree_.__equal_range_unique(__k);
+  }
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
+    return __tree_.__equal_range_unique(__k);
+  }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
+    return __tree_.__equal_range_multi(__k);
+  }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
+    return __tree_.__equal_range_multi(__k);
+  }
+#endif
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator,
+          class _Compare   = less<__iter_value_type<_InputIterator>>,
+          class _Allocator = allocator<__iter_value_type<_InputIterator>>,
+          class            = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value, void>,
+          class            = enable_if_t<!__is_allocator<_Compare>::value, void>>
+set(_InputIterator, _InputIterator, _Compare = _Compare(), _Allocator = _Allocator())
+    -> set<__iter_value_type<_InputIterator>, _Compare, _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range,
+          class _Compare   = less<ranges::range_value_t<_Range>>,
+          class _Allocator = allocator<ranges::range_value_t<_Range>>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value, void>,
+          class            = enable_if_t<!__is_allocator<_Compare>::value, void>>
+set(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator())
+    -> set<ranges::range_value_t<_Range>, _Compare, _Allocator>;
+#  endif
+
+template <class _Key,
+          class _Compare   = less<_Key>,
+          class _Allocator = allocator<_Key>,
+          class            = enable_if_t<!__is_allocator<_Compare>::value, void>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value, void>>
+set(initializer_list<_Key>, _Compare = _Compare(), _Allocator = _Allocator()) -> set<_Key, _Compare, _Allocator>;
+
+template <class _InputIterator,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
+          class = enable_if_t<__is_allocator<_Allocator>::value, void>>
+set(_InputIterator,
+    _InputIterator,
+    _Allocator) -> set<__iter_value_type<_InputIterator>, less<__iter_value_type<_InputIterator>>, _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
+set(from_range_t,
+    _Range&&,
+    _Allocator) -> set<ranges::range_value_t<_Range>, less<ranges::range_value_t<_Range>>, _Allocator>;
+#  endif
+
+template <class _Key, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
+set(initializer_list<_Key>, _Allocator) -> set<_Key, less<_Key>, _Allocator>;
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Key, class _Compare, class _Allocator>
+set<_Key, _Compare, _Allocator>::set(set&& __s, const allocator_type& __a) : __tree_(std::move(__s.__tree_), __a) {
+  if (__a != __s.get_allocator()) {
+    const_iterator __e = cend();
+    while (!__s.empty())
+      insert(__e, std::move(__s.__tree_.remove(__s.begin())->__value_));
+  }
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Key, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const set<_Key, _Compare, _Allocator>& __x, const set<_Key, _Compare, _Allocator>& __y) {
+  return __x.size() == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Key, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<(const set<_Key, _Compare, _Allocator>& __x, const set<_Key, _Compare, _Allocator>& __y) {
+  return std::lexicographical_compare(__x.begin(), __x.end(), __y.begin(), __y.end());
+}
+
+template <class _Key, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const set<_Key, _Compare, _Allocator>& __x, const set<_Key, _Compare, _Allocator>& __y) {
+  return !(__x == __y);
+}
+
+template <class _Key, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>(const set<_Key, _Compare, _Allocator>& __x, const set<_Key, _Compare, _Allocator>& __y) {
+  return __y < __x;
+}
+
+template <class _Key, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>=(const set<_Key, _Compare, _Allocator>& __x, const set<_Key, _Compare, _Allocator>& __y) {
+  return !(__x < __y);
+}
+
+template <class _Key, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<=(const set<_Key, _Compare, _Allocator>& __x, const set<_Key, _Compare, _Allocator>& __y) {
+  return !(__y < __x);
+}
+
+#else // _LIBCPP_STD_VER <= 17
+
+template <class _Key, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI __synth_three_way_result<_Key>
+operator<=>(const set<_Key, _Allocator>& __x, const set<_Key, _Allocator>& __y) {
+  return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
+}
+
+#endif // _LIBCPP_STD_VER <= 17
+
+// specialized algorithms:
+template <class _Key, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI void swap(set<_Key, _Compare, _Allocator>& __x, set<_Key, _Compare, _Allocator>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Key, class _Compare, class _Allocator, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI typename set<_Key, _Compare, _Allocator>::size_type
+erase_if(set<_Key, _Compare, _Allocator>& __c, _Predicate __pred) {
+  return std::__libcpp_erase_if_container(__c, __pred);
+}
+#endif
+
+template <class _Key, class _Compare = less<_Key>, class _Allocator = allocator<_Key> >
+class _LIBCPP_TEMPLATE_VIS multiset {
+public:
+  // types:
+  typedef _Key key_type;
+  typedef key_type value_type;
+  typedef __type_identity_t<_Compare> key_compare;
+  typedef key_compare value_compare;
+  typedef __type_identity_t<_Allocator> allocator_type;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+
+  static_assert(is_same<typename allocator_type::value_type, value_type>::value,
+                "Allocator::value_type must be same type as value_type");
+
+private:
+  typedef __tree<value_type, value_compare, allocator_type> __base;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+
+  static_assert(__check_valid_allocator<allocator_type>::value, "");
+
+  __base __tree_;
+
+public:
+  typedef typename __base::pointer pointer;
+  typedef typename __base::const_pointer const_pointer;
+  typedef typename __base::size_type size_type;
+  typedef typename __base::difference_type difference_type;
+  typedef typename __base::const_iterator iterator;
+  typedef typename __base::const_iterator const_iterator;
+  typedef std::reverse_iterator<iterator> reverse_iterator;
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+
+#if _LIBCPP_STD_VER >= 17
+  typedef __set_node_handle<typename __base::__node, allocator_type> node_type;
+#endif
+
+  template <class _Key2, class _Compare2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS set;
+  template <class _Key2, class _Compare2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS multiset;
+
+  // construct/copy/destroy:
+  _LIBCPP_HIDE_FROM_ABI multiset() _NOEXCEPT_(
+      is_nothrow_default_constructible<allocator_type>::value&& is_nothrow_default_constructible<key_compare>::value&&
+          is_nothrow_copy_constructible<key_compare>::value)
+      : __tree_(value_compare()) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit multiset(const value_compare& __comp) _NOEXCEPT_(
+      is_nothrow_default_constructible<allocator_type>::value&& is_nothrow_copy_constructible<key_compare>::value)
+      : __tree_(__comp) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit multiset(const value_compare& __comp, const allocator_type& __a)
+      : __tree_(__comp, __a) {}
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI multiset(_InputIterator __f, _InputIterator __l, const value_compare& __comp = value_compare())
+      : __tree_(__comp) {
+    insert(__f, __l);
+  }
+
+#if _LIBCPP_STD_VER >= 14
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI multiset(_InputIterator __f, _InputIterator __l, const allocator_type& __a)
+      : multiset(__f, __l, key_compare(), __a) {}
+#endif
+
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI
+  multiset(_InputIterator __f, _InputIterator __l, const value_compare& __comp, const allocator_type& __a)
+      : __tree_(__comp, __a) {
+    insert(__f, __l);
+  }
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI
+  multiset(from_range_t,
+           _Range&& __range,
+           const key_compare& __comp = key_compare(),
+           const allocator_type& __a = allocator_type())
+      : __tree_(__comp, __a) {
+    insert_range(std::forward<_Range>(__range));
+  }
+
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI multiset(from_range_t, _Range&& __range, const allocator_type& __a)
+      : multiset(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s)
+      : __tree_(__s.__tree_.value_comp(),
+                __alloc_traits::select_on_container_copy_construction(__s.__tree_.__alloc())) {
+    insert(__s.begin(), __s.end());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI multiset& operator=(const multiset& __s) {
+    __tree_ = __s.__tree_;
+    return *this;
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s) noexcept(is_nothrow_move_constructible<__base>::value)
+      : __tree_(std::move(__s.__tree_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s, const allocator_type& __a);
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI explicit multiset(const allocator_type& __a) : __tree_(__a) {}
+  _LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s, const allocator_type& __a)
+      : __tree_(__s.__tree_.value_comp(), __a) {
+    insert(__s.begin(), __s.end());
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI multiset(initializer_list<value_type> __il, const value_compare& __comp = value_compare())
+      : __tree_(__comp) {
+    insert(__il.begin(), __il.end());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI
+  multiset(initializer_list<value_type> __il, const value_compare& __comp, const allocator_type& __a)
+      : __tree_(__comp, __a) {
+    insert(__il.begin(), __il.end());
+  }
+
+#  if _LIBCPP_STD_VER >= 14
+  _LIBCPP_HIDE_FROM_ABI multiset(initializer_list<value_type> __il, const allocator_type& __a)
+      : multiset(__il, key_compare(), __a) {}
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI multiset& operator=(initializer_list<value_type> __il) {
+    __tree_.__assign_multi(__il.begin(), __il.end());
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI multiset& operator=(multiset&& __s) _NOEXCEPT_(is_nothrow_move_assignable<__base>::value) {
+    __tree_ = std::move(__s.__tree_);
+    return *this;
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI ~multiset() { static_assert(sizeof(__diagnose_non_const_comparator<_Key, _Compare>()), ""); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __tree_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __tree_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __tree_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __tree_.end(); }
+
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return rbegin(); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __tree_.size() == 0; }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __tree_.size(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __tree_.max_size(); }
+
+  // modifiers:
+#ifndef _LIBCPP_CXX03_LANG
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace(_Args&&... __args) {
+    return __tree_.__emplace_multi(std::forward<_Args>(__args)...);
+  }
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __p, _Args&&... __args) {
+    return __tree_.__emplace_hint_multi(__p, std::forward<_Args>(__args)...);
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __v) { return __tree_.__insert_multi(__v); }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __v) {
+    return __tree_.__insert_multi(__p, __v);
+  }
+
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __f, _InputIterator __l) {
+    for (const_iterator __e = cend(); __f != __l; ++__f)
+      __tree_.__insert_multi(__e, *__f);
+  }
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
+    const_iterator __end = cend();
+    for (auto&& __element : __range) {
+      __tree_.__insert_multi(__end, std::forward<decltype(__element)>(__element));
+    }
+  }
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI iterator insert(value_type&& __v) { return __tree_.__insert_multi(std::move(__v)); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __v) {
+    return __tree_.__insert_multi(__p, std::move(__v));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __tree_.erase(__p); }
+  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __tree_.__erase_multi(__k); }
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __f, const_iterator __l) { return __tree_.erase(__f, __l); }
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __tree_.clear(); }
+
+#if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI iterator insert(node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to multiset::insert()");
+    return __tree_.template __node_handle_insert_multi<node_type>(std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to multiset::insert()");
+    return __tree_.template __node_handle_insert_multi<node_type>(__hint, std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
+    return __tree_.template __node_handle_extract<node_type>(__key);
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
+    return __tree_.template __node_handle_extract<node_type>(__it);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(multiset<key_type, _Compare2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __tree_.__node_handle_merge_multi(__source.__tree_);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(multiset<key_type, _Compare2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __tree_.__node_handle_merge_multi(__source.__tree_);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(set<key_type, _Compare2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __tree_.__node_handle_merge_multi(__source.__tree_);
+  }
+  template <class _Compare2>
+  _LIBCPP_HIDE_FROM_ABI void merge(set<key_type, _Compare2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __tree_.__node_handle_merge_multi(__source.__tree_);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI void swap(multiset& __s) _NOEXCEPT_(__is_nothrow_swappable_v<__base>) {
+    __tree_.swap(__s.__tree_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT { return __tree_.__alloc(); }
+  _LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __tree_.value_comp(); }
+  _LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return __tree_.value_comp(); }
+
+  // set operations:
+  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
+    return __tree_.find(__k);
+  }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
+    return __tree_.find(__k);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_multi(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
+    return __tree_.__count_multi(__k);
+  }
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
+    return find(__k) != end();
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
+    return __tree_.lower_bound(__k);
+  }
+
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
+    return __tree_.lower_bound(__k);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
+    return __tree_.upper_bound(__k);
+  }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
+    return __tree_.upper_bound(__k);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
+    return __tree_.__equal_range_multi(__k);
+  }
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
+    return __tree_.__equal_range_multi(__k);
+  }
+#if _LIBCPP_STD_VER >= 14
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
+    return __tree_.__equal_range_multi(__k);
+  }
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
+    return __tree_.__equal_range_multi(__k);
+  }
+#endif
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator,
+          class _Compare   = less<__iter_value_type<_InputIterator>>,
+          class _Allocator = allocator<__iter_value_type<_InputIterator>>,
+          class            = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value, void>,
+          class            = enable_if_t<!__is_allocator<_Compare>::value, void>>
+multiset(_InputIterator, _InputIterator, _Compare = _Compare(), _Allocator = _Allocator())
+    -> multiset<__iter_value_type<_InputIterator>, _Compare, _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range,
+          class _Compare   = less<ranges::range_value_t<_Range>>,
+          class _Allocator = allocator<ranges::range_value_t<_Range>>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value, void>,
+          class            = enable_if_t<!__is_allocator<_Compare>::value, void>>
+multiset(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator())
+    -> multiset<ranges::range_value_t<_Range>, _Compare, _Allocator>;
+#  endif
+
+template <class _Key,
+          class _Compare   = less<_Key>,
+          class _Allocator = allocator<_Key>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value, void>,
+          class            = enable_if_t<!__is_allocator<_Compare>::value, void>>
+multiset(initializer_list<_Key>,
+         _Compare   = _Compare(),
+         _Allocator = _Allocator()) -> multiset<_Key, _Compare, _Allocator>;
+
+template <class _InputIterator,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
+          class = enable_if_t<__is_allocator<_Allocator>::value, void>>
+multiset(_InputIterator, _InputIterator, _Allocator)
+    -> multiset<__iter_value_type<_InputIterator>, less<__iter_value_type<_InputIterator>>, _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
+multiset(from_range_t,
+         _Range&&,
+         _Allocator) -> multiset<ranges::range_value_t<_Range>, less<ranges::range_value_t<_Range>>, _Allocator>;
+#  endif
+
+template <class _Key, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
+multiset(initializer_list<_Key>, _Allocator) -> multiset<_Key, less<_Key>, _Allocator>;
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Key, class _Compare, class _Allocator>
+multiset<_Key, _Compare, _Allocator>::multiset(multiset&& __s, const allocator_type& __a)
+    : __tree_(std::move(__s.__tree_), __a) {
+  if (__a != __s.get_allocator()) {
+    const_iterator __e = cend();
+    while (!__s.empty())
+      insert(__e, std::move(__s.__tree_.remove(__s.begin())->__value_));
+  }
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Key, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const multiset<_Key, _Compare, _Allocator>& __x, const multiset<_Key, _Compare, _Allocator>& __y) {
+  return __x.size() == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Key, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<(const multiset<_Key, _Compare, _Allocator>& __x, const multiset<_Key, _Compare, _Allocator>& __y) {
+  return std::lexicographical_compare(__x.begin(), __x.end(), __y.begin(), __y.end());
+}
+
+template <class _Key, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const multiset<_Key, _Compare, _Allocator>& __x, const multiset<_Key, _Compare, _Allocator>& __y) {
+  return !(__x == __y);
+}
+
+template <class _Key, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>(const multiset<_Key, _Compare, _Allocator>& __x, const multiset<_Key, _Compare, _Allocator>& __y) {
+  return __y < __x;
+}
+
+template <class _Key, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>=(const multiset<_Key, _Compare, _Allocator>& __x, const multiset<_Key, _Compare, _Allocator>& __y) {
+  return !(__x < __y);
+}
+
+template <class _Key, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<=(const multiset<_Key, _Compare, _Allocator>& __x, const multiset<_Key, _Compare, _Allocator>& __y) {
+  return !(__y < __x);
+}
+
+#else // _LIBCPP_STD_VER <= 17
+
+template <class _Key, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI __synth_three_way_result<_Key>
+operator<=>(const multiset<_Key, _Allocator>& __x, const multiset<_Key, _Allocator>& __y) {
+  return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), __synth_three_way);
+}
+
+#endif // _LIBCPP_STD_VER <= 17
+
+template <class _Key, class _Compare, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(multiset<_Key, _Compare, _Allocator>& __x, multiset<_Key, _Compare, _Allocator>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Key, class _Compare, class _Allocator, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI typename multiset<_Key, _Compare, _Allocator>::size_type
+erase_if(multiset<_Key, _Compare, _Allocator>& __c, _Predicate __pred) {
+  return std::__libcpp_erase_if_container(__c, __pred);
+}
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace pmr {
+template <class _KeyT, class _CompareT = std::less<_KeyT>>
+using set _LIBCPP_AVAILABILITY_PMR = std::set<_KeyT, _CompareT, polymorphic_allocator<_KeyT>>;
+
+template <class _KeyT, class _CompareT = std::less<_KeyT>>
+using multiset _LIBCPP_AVAILABILITY_PMR = std::multiset<_KeyT, _CompareT, polymorphic_allocator<_KeyT>>;
+} // namespace pmr
+_LIBCPP_END_NAMESPACE_STD
+#endif
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <concepts>
+#  include <cstdlib>
+#  include <functional>
+#  include <iterator>
+#  include <stdexcept>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_SET
diff --git a/libcxx/include/__cxx03/shared_mutex b/libcxx/include/__cxx03/shared_mutex
new file mode 100644
index 000000000000..f63bd2549387
--- /dev/null
+++ b/libcxx/include/__cxx03/shared_mutex
@@ -0,0 +1,464 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_SHARED_MUTEX
+#define _LIBCPP_SHARED_MUTEX
+
+/*
+    shared_mutex synopsis
+
+// C++1y
+
+namespace std
+{
+
+class shared_mutex      // C++17
+{
+public:
+    shared_mutex();
+    ~shared_mutex();
+
+    shared_mutex(const shared_mutex&) = delete;
+    shared_mutex& operator=(const shared_mutex&) = delete;
+
+    // Exclusive ownership
+    void lock(); // blocking
+    bool try_lock();
+    void unlock();
+
+    // Shared ownership
+    void lock_shared(); // blocking
+    bool try_lock_shared();
+    void unlock_shared();
+
+    typedef implementation-defined native_handle_type; // See 30.2.3
+    native_handle_type native_handle(); // See 30.2.3
+};
+
+class shared_timed_mutex
+{
+public:
+    shared_timed_mutex();
+    ~shared_timed_mutex();
+
+    shared_timed_mutex(const shared_timed_mutex&) = delete;
+    shared_timed_mutex& operator=(const shared_timed_mutex&) = delete;
+
+    // Exclusive ownership
+    void lock(); // blocking
+    bool try_lock();
+    template <class Rep, class Period>
+        bool try_lock_for(const chrono::duration<Rep, Period>& rel_time);
+    template <class Clock, class Duration>
+        bool try_lock_until(const chrono::time_point<Clock, Duration>& abs_time);
+    void unlock();
+
+    // Shared ownership
+    void lock_shared(); // blocking
+    bool try_lock_shared();
+    template <class Rep, class Period>
+        bool
+        try_lock_shared_for(const chrono::duration<Rep, Period>& rel_time);
+    template <class Clock, class Duration>
+        bool
+        try_lock_shared_until(const chrono::time_point<Clock, Duration>& abs_time);
+    void unlock_shared();
+};
+
+template <class Mutex>
+class shared_lock
+{
+public:
+    typedef Mutex mutex_type;
+
+    // Shared locking
+    shared_lock() noexcept;
+    explicit shared_lock(mutex_type& m); // blocking
+    shared_lock(mutex_type& m, defer_lock_t) noexcept;
+    shared_lock(mutex_type& m, try_to_lock_t);
+    shared_lock(mutex_type& m, adopt_lock_t);
+    template <class Clock, class Duration>
+        shared_lock(mutex_type& m,
+                    const chrono::time_point<Clock, Duration>& abs_time);
+    template <class Rep, class Period>
+        shared_lock(mutex_type& m,
+                    const chrono::duration<Rep, Period>& rel_time);
+    ~shared_lock();
+
+    shared_lock(shared_lock const&) = delete;
+    shared_lock& operator=(shared_lock const&) = delete;
+
+    shared_lock(shared_lock&& u) noexcept;
+    shared_lock& operator=(shared_lock&& u) noexcept;
+
+    void lock(); // blocking
+    bool try_lock();
+    template <class Rep, class Period>
+        bool try_lock_for(const chrono::duration<Rep, Period>& rel_time);
+    template <class Clock, class Duration>
+        bool try_lock_until(const chrono::time_point<Clock, Duration>& abs_time);
+    void unlock();
+
+    // Setters
+    void swap(shared_lock& u) noexcept;
+    mutex_type* release() noexcept;
+
+    // Getters
+    bool owns_lock() const noexcept;
+    explicit operator bool () const noexcept;
+    mutex_type* mutex() const noexcept;
+};
+
+template <class Mutex>
+    void swap(shared_lock<Mutex>& x, shared_lock<Mutex>& y) noexcept;
+
+}  // std
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+
+#  include <__chrono/duration.h>
+#  include <__chrono/steady_clock.h>
+#  include <__chrono/time_point.h>
+#  include <__condition_variable/condition_variable.h>
+#  include <__memory/addressof.h>
+#  include <__mutex/mutex.h>
+#  include <__mutex/tag_types.h>
+#  include <__mutex/unique_lock.h>
+#  include <__system_error/system_error.h>
+#  include <__utility/swap.h>
+#  include <cerrno>
+#  include <version>
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+#  if _LIBCPP_STD_VER >= 14
+
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct _LIBCPP_EXPORTED_FROM_ABI __shared_mutex_base {
+  mutex __mut_;
+  condition_variable __gate1_;
+  condition_variable __gate2_;
+  unsigned __state_;
+
+  static const unsigned __write_entered_ = 1U << (sizeof(unsigned) * __CHAR_BIT__ - 1);
+  static const unsigned __n_readers_     = ~__write_entered_;
+
+  __shared_mutex_base();
+  _LIBCPP_HIDE_FROM_ABI ~__shared_mutex_base() = default;
+
+  __shared_mutex_base(const __shared_mutex_base&)            = delete;
+  __shared_mutex_base& operator=(const __shared_mutex_base&) = delete;
+
+  // Exclusive ownership
+  void lock(); // blocking
+  bool try_lock();
+  void unlock();
+
+  // Shared ownership
+  void lock_shared(); // blocking
+  bool try_lock_shared();
+  void unlock_shared();
+
+  //     typedef implementation-defined native_handle_type; // See 30.2.3
+  //     native_handle_type native_handle(); // See 30.2.3
+};
+
+#    if _LIBCPP_STD_VER >= 17
+class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_THREAD_SAFETY_ANNOTATION(__capability__("shared_mutex")) shared_mutex {
+  __shared_mutex_base __base_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI shared_mutex() : __base_() {}
+  _LIBCPP_HIDE_FROM_ABI ~shared_mutex() = default;
+
+  shared_mutex(const shared_mutex&)            = delete;
+  shared_mutex& operator=(const shared_mutex&) = delete;
+
+  // Exclusive ownership
+  _LIBCPP_HIDE_FROM_ABI void lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__acquire_capability__()) {
+    return __base_.lock();
+  }
+  _LIBCPP_HIDE_FROM_ABI bool try_lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_capability__(true)) {
+    return __base_.try_lock();
+  }
+  _LIBCPP_HIDE_FROM_ABI void unlock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__release_capability__()) {
+    return __base_.unlock();
+  }
+
+  // Shared ownership
+  _LIBCPP_HIDE_FROM_ABI void lock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(__acquire_shared_capability__()) {
+    return __base_.lock_shared();
+  }
+  _LIBCPP_HIDE_FROM_ABI bool try_lock_shared()
+      _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_shared_capability__(true)) {
+    return __base_.try_lock_shared();
+  }
+  _LIBCPP_HIDE_FROM_ABI void unlock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(__release_shared_capability__()) {
+    return __base_.unlock_shared();
+  }
+
+  //     typedef __shared_mutex_base::native_handle_type native_handle_type;
+  //     _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return __base::unlock_shared(); }
+};
+#    endif
+
+class _LIBCPP_EXPORTED_FROM_ABI
+_LIBCPP_THREAD_SAFETY_ANNOTATION(__capability__("shared_timed_mutex")) shared_timed_mutex {
+  __shared_mutex_base __base_;
+
+public:
+  shared_timed_mutex();
+  _LIBCPP_HIDE_FROM_ABI ~shared_timed_mutex() = default;
+
+  shared_timed_mutex(const shared_timed_mutex&)            = delete;
+  shared_timed_mutex& operator=(const shared_timed_mutex&) = delete;
+
+  // Exclusive ownership
+  void lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__acquire_capability__());
+  bool try_lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_capability__(true));
+  template <class _Rep, class _Period>
+  _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __rel_time)
+      _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_capability__(true)) {
+    return try_lock_until(chrono::steady_clock::now() + __rel_time);
+  }
+  template <class _Clock, class _Duration>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS bool
+  try_lock_until(const chrono::time_point<_Clock, _Duration>& __abs_time)
+      _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_capability__(true));
+  void unlock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__release_capability__());
+
+  // Shared ownership
+  void lock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(__acquire_shared_capability__());
+  bool try_lock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_shared_capability__(true));
+  template <class _Rep, class _Period>
+  _LIBCPP_HIDE_FROM_ABI bool try_lock_shared_for(const chrono::duration<_Rep, _Period>& __rel_time)
+      _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_shared_capability__(true)) {
+    return try_lock_shared_until(chrono::steady_clock::now() + __rel_time);
+  }
+  template <class _Clock, class _Duration>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS bool
+  try_lock_shared_until(const chrono::time_point<_Clock, _Duration>& __abs_time)
+      _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_shared_capability__(true));
+  void unlock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(__release_shared_capability__());
+};
+
+template <class _Clock, class _Duration>
+bool shared_timed_mutex::try_lock_until(const chrono::time_point<_Clock, _Duration>& __abs_time) {
+  unique_lock<mutex> __lk(__base_.__mut_);
+  if (__base_.__state_ & __base_.__write_entered_) {
+    while (true) {
+      cv_status __status = __base_.__gate1_.wait_until(__lk, __abs_time);
+      if ((__base_.__state_ & __base_.__write_entered_) == 0)
+        break;
+      if (__status == cv_status::timeout)
+        return false;
+    }
+  }
+  __base_.__state_ |= __base_.__write_entered_;
+  if (__base_.__state_ & __base_.__n_readers_) {
+    while (true) {
+      cv_status __status = __base_.__gate2_.wait_until(__lk, __abs_time);
+      if ((__base_.__state_ & __base_.__n_readers_) == 0)
+        break;
+      if (__status == cv_status::timeout) {
+        __base_.__state_ &= ~__base_.__write_entered_;
+        __base_.__gate1_.notify_all();
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+template <class _Clock, class _Duration>
+bool shared_timed_mutex::try_lock_shared_until(const chrono::time_point<_Clock, _Duration>& __abs_time) {
+  unique_lock<mutex> __lk(__base_.__mut_);
+  if ((__base_.__state_ & __base_.__write_entered_) ||
+      (__base_.__state_ & __base_.__n_readers_) == __base_.__n_readers_) {
+    while (true) {
+      cv_status __status = __base_.__gate1_.wait_until(__lk, __abs_time);
+      if ((__base_.__state_ & __base_.__write_entered_) == 0 &&
+          (__base_.__state_ & __base_.__n_readers_) < __base_.__n_readers_)
+        break;
+      if (__status == cv_status::timeout)
+        return false;
+    }
+  }
+  unsigned __num_readers = (__base_.__state_ & __base_.__n_readers_) + 1;
+  __base_.__state_ &= ~__base_.__n_readers_;
+  __base_.__state_ |= __num_readers;
+  return true;
+}
+
+template <class _Mutex>
+class shared_lock {
+public:
+  typedef _Mutex mutex_type;
+
+private:
+  mutex_type* __m_;
+  bool __owns_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI shared_lock() _NOEXCEPT : __m_(nullptr), __owns_(false) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit shared_lock(mutex_type& __m) : __m_(std::addressof(__m)), __owns_(true) {
+    __m_->lock_shared();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI shared_lock(mutex_type& __m, defer_lock_t) _NOEXCEPT
+      : __m_(std::addressof(__m)),
+        __owns_(false) {}
+
+  _LIBCPP_HIDE_FROM_ABI shared_lock(mutex_type& __m, try_to_lock_t)
+      : __m_(std::addressof(__m)), __owns_(__m.try_lock_shared()) {}
+
+  _LIBCPP_HIDE_FROM_ABI shared_lock(mutex_type& __m, adopt_lock_t) : __m_(std::addressof(__m)), __owns_(true) {}
+
+  template <class _Clock, class _Duration>
+  _LIBCPP_HIDE_FROM_ABI shared_lock(mutex_type& __m, const chrono::time_point<_Clock, _Duration>& __abs_time)
+      : __m_(std::addressof(__m)), __owns_(__m.try_lock_shared_until(__abs_time)) {}
+
+  template <class _Rep, class _Period>
+  _LIBCPP_HIDE_FROM_ABI shared_lock(mutex_type& __m, const chrono::duration<_Rep, _Period>& __rel_time)
+      : __m_(std::addressof(__m)), __owns_(__m.try_lock_shared_for(__rel_time)) {}
+
+  _LIBCPP_HIDE_FROM_ABI ~shared_lock() {
+    if (__owns_)
+      __m_->unlock_shared();
+  }
+
+  shared_lock(shared_lock const&)            = delete;
+  shared_lock& operator=(shared_lock const&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI shared_lock(shared_lock&& __u) _NOEXCEPT : __m_(__u.__m_), __owns_(__u.__owns_) {
+    __u.__m_    = nullptr;
+    __u.__owns_ = false;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI shared_lock& operator=(shared_lock&& __u) _NOEXCEPT {
+    if (__owns_)
+      __m_->unlock_shared();
+    __m_        = nullptr;
+    __owns_     = false;
+    __m_        = __u.__m_;
+    __owns_     = __u.__owns_;
+    __u.__m_    = nullptr;
+    __u.__owns_ = false;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void lock();
+  _LIBCPP_HIDE_FROM_ABI bool try_lock();
+  template <class _Rep, class _Period>
+  _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __rel_time);
+  template <class _Clock, class _Duration>
+  _LIBCPP_HIDE_FROM_ABI bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __abs_time);
+  _LIBCPP_HIDE_FROM_ABI void unlock();
+
+  // Setters
+  _LIBCPP_HIDE_FROM_ABI void swap(shared_lock& __u) _NOEXCEPT {
+    std::swap(__m_, __u.__m_);
+    std::swap(__owns_, __u.__owns_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI mutex_type* release() _NOEXCEPT {
+    mutex_type* __m = __m_;
+    __m_            = nullptr;
+    __owns_         = false;
+    return __m;
+  }
+
+  // Getters
+  _LIBCPP_HIDE_FROM_ABI bool owns_lock() const _NOEXCEPT { return __owns_; }
+
+  _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return __owns_; }
+
+  _LIBCPP_HIDE_FROM_ABI mutex_type* mutex() const _NOEXCEPT { return __m_; }
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(shared_lock);
+
+template <class _Mutex>
+void shared_lock<_Mutex>::lock() {
+  if (__m_ == nullptr)
+    __throw_system_error(EPERM, "shared_lock::lock: references null mutex");
+  if (__owns_)
+    __throw_system_error(EDEADLK, "shared_lock::lock: already locked");
+  __m_->lock_shared();
+  __owns_ = true;
+}
+
+template <class _Mutex>
+bool shared_lock<_Mutex>::try_lock() {
+  if (__m_ == nullptr)
+    __throw_system_error(EPERM, "shared_lock::try_lock: references null mutex");
+  if (__owns_)
+    __throw_system_error(EDEADLK, "shared_lock::try_lock: already locked");
+  __owns_ = __m_->try_lock_shared();
+  return __owns_;
+}
+
+template <class _Mutex>
+template <class _Rep, class _Period>
+bool shared_lock<_Mutex>::try_lock_for(const chrono::duration<_Rep, _Period>& __d) {
+  if (__m_ == nullptr)
+    __throw_system_error(EPERM, "shared_lock::try_lock_for: references null mutex");
+  if (__owns_)
+    __throw_system_error(EDEADLK, "shared_lock::try_lock_for: already locked");
+  __owns_ = __m_->try_lock_shared_for(__d);
+  return __owns_;
+}
+
+template <class _Mutex>
+template <class _Clock, class _Duration>
+bool shared_lock<_Mutex>::try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) {
+  if (__m_ == nullptr)
+    __throw_system_error(EPERM, "shared_lock::try_lock_until: references null mutex");
+  if (__owns_)
+    __throw_system_error(EDEADLK, "shared_lock::try_lock_until: already locked");
+  __owns_ = __m_->try_lock_shared_until(__t);
+  return __owns_;
+}
+
+template <class _Mutex>
+void shared_lock<_Mutex>::unlock() {
+  if (!__owns_)
+    __throw_system_error(EPERM, "shared_lock::unlock: not locked");
+  __m_->unlock_shared();
+  __owns_ = false;
+}
+
+template <class _Mutex>
+inline _LIBCPP_HIDE_FROM_ABI void swap(shared_lock<_Mutex>& __x, shared_lock<_Mutex>& __y) _NOEXCEPT {
+  __x.swap(__y);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#  endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_POP_MACROS
+
+#endif // !defined(_LIBCPP_HAS_NO_THREADS)
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <system_error>
+#endif
+
+#endif // _LIBCPP_SHARED_MUTEX
diff --git a/libcxx/include/__cxx03/source_location b/libcxx/include/__cxx03/source_location
new file mode 100644
index 000000000000..d16e3c46fce5
--- /dev/null
+++ b/libcxx/include/__cxx03/source_location
@@ -0,0 +1,85 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_SOURCE_LOCATION
+#define _LIBCPP_SOURCE_LOCATION
+
+/* source_location synopsis
+
+namespace std {
+  struct source_location {
+    static consteval source_location current() noexcept;
+    constexpr source_location() noexcept;
+
+    constexpr uint_least32_t line() const noexcept;
+    constexpr uint_least32_t column() const noexcept;
+    constexpr const char* file_name() const noexcept;
+    constexpr const char* function_name() const noexcept;
+  };
+}
+*/
+
+#include <__config>
+#include <cstdint>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+class source_location {
+  // The names source_location::__impl, _M_file_name, _M_function_name, _M_line, and _M_column
+  // are hard-coded in the compiler and must not be changed here.
+  struct __impl {
+    const char* _M_file_name;
+    const char* _M_function_name;
+    unsigned _M_line;
+    unsigned _M_column;
+  };
+  const __impl* __ptr_ = nullptr;
+  // GCC returns the type 'const void*' from the builtin, while clang returns
+  // `const __impl*`. Per C++ [expr.const], casts from void* are not permitted
+  // in constant evaluation, so we don't want to use `void*` as the argument
+  // type unless the builtin returned that, anyhow, and the invalid cast is
+  // unavoidable.
+  using __bsl_ty = decltype(__builtin_source_location());
+
+public:
+  // The defaulted __ptr argument is necessary so that the builtin is evaluated
+  // in the context of the caller. An explicit value should never be provided.
+  static consteval source_location current(__bsl_ty __ptr = __builtin_source_location()) noexcept {
+    source_location __sl;
+    __sl.__ptr_ = static_cast<const __impl*>(__ptr);
+    return __sl;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr source_location() noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr uint_least32_t line() const noexcept {
+    return __ptr_ != nullptr ? __ptr_->_M_line : 0;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr uint_least32_t column() const noexcept {
+    return __ptr_ != nullptr ? __ptr_->_M_column : 0;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr const char* file_name() const noexcept {
+    return __ptr_ != nullptr ? __ptr_->_M_file_name : "";
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr const char* function_name() const noexcept {
+    return __ptr_ != nullptr ? __ptr_->_M_function_name : "";
+  }
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_SOURCE_LOCATION
diff --git a/libcxx/include/__cxx03/span b/libcxx/include/__cxx03/span
new file mode 100644
index 000000000000..da631cdc3f90
--- /dev/null
+++ b/libcxx/include/__cxx03/span
@@ -0,0 +1,636 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_SPAN
+#define _LIBCPP_SPAN
+
+/*
+    span synopsis
+
+namespace std {
+
+// constants
+inline constexpr size_t dynamic_extent = numeric_limits<size_t>::max();
+
+template<class T>
+  concept integral-constant-like =                           // exposition only, since C++26
+    is_integral_v<decltype(T::value)> &&
+    !is_same_v<bool, remove_const_t<decltype(T::value)>> &&
+    convertible_to<T, decltype(T::value)> &&
+    equality_comparable_with<T, decltype(T::value)> &&
+    bool_constant<T() == T::value>::value &&
+    bool_constant<static_cast<decltype(T::value)>(T()) == T::value>::value;
+
+template<class T>
+  constexpr size_t maybe-static-ext = dynamic_extent;        // exposition only, since C++26
+template<integral-constant-like T>
+  constexpr size_t maybe-static-ext<T> = {T::value};
+
+// [views.span], class template span
+template <class ElementType, size_t Extent = dynamic_extent>
+    class span;
+
+template<class ElementType, size_t Extent>
+  inline constexpr bool ranges::enable_view<span<ElementType, Extent>> = true;
+
+template<class ElementType, size_t Extent>
+    inline constexpr bool ranges::enable_borrowed_range<span<ElementType, Extent>> = true;
+
+// [span.objectrep], views of object representation
+template <class ElementType, size_t Extent>
+    span<const byte, ((Extent == dynamic_extent) ? dynamic_extent :
+        (sizeof(ElementType) * Extent))> as_bytes(span<ElementType, Extent> s) noexcept;
+
+template <class ElementType, size_t Extent>
+    span<      byte, ((Extent == dynamic_extent) ? dynamic_extent :
+        (sizeof(ElementType) * Extent))> as_writable_bytes(span<ElementType, Extent> s) noexcept;
+
+
+template <class ElementType, size_t Extent = dynamic_extent>
+class span {
+public:
+    // constants and types
+    using element_type = ElementType;
+    using value_type = remove_cv_t<ElementType>;
+    using size_type = size_t;
+    using difference_type = ptrdiff_t;
+    using pointer = element_type*;
+    using const_pointer = const element_type*;
+    using reference = element_type&;
+    using const_reference = const element_type&;
+    using iterator = implementation-defined;
+    using reverse_iterator = std::reverse_iterator<iterator>;
+    static constexpr size_type extent = Extent;
+
+    // [span.cons], span constructors, copy, assignment, and destructor
+    constexpr span() noexcept;
+    template <class It>
+    constexpr explicit(Extent != dynamic_extent) span(It first, size_type count);
+    template <class It, class End>
+    constexpr explicit(Extent != dynamic_extent) span(It first, End last);
+    template <size_t N>
+        constexpr span(type_identity_t<element_type> (&arr)[N]) noexcept;
+    template <size_t N>
+        constexpr span(array<value_type, N>& arr) noexcept;
+    template <size_t N>
+        constexpr span(const array<value_type, N>& arr) noexcept;
+    template<class R>
+      constexpr explicit(Extent != dynamic_extent) span(R&& r);
+    constexpr explicit(extent != dynamic_extent) span(std::initializer_list<value_type> il); // Since C++26
+    constexpr span(const span& other) noexcept = default;
+    template <class OtherElementType, size_t OtherExtent>
+        constexpr explicit(Extent != dynamic_extent) span(const span<OtherElementType, OtherExtent>& s) noexcept;
+    constexpr span& operator=(const span& other) noexcept = default;
+
+    // [span.sub], span subviews
+    template <size_t Count>
+        constexpr span<element_type, Count> first() const;
+    template <size_t Count>
+        constexpr span<element_type, Count> last() const;
+    template <size_t Offset, size_t Count = dynamic_extent>
+        constexpr span<element_type, see below> subspan() const;
+
+    constexpr span<element_type, dynamic_extent> first(size_type count) const;
+    constexpr span<element_type, dynamic_extent> last(size_type count) const;
+    constexpr span<element_type, dynamic_extent> subspan(size_type offset, size_type count = dynamic_extent) const;
+
+    // [span.obs], span observers
+    constexpr size_type size() const noexcept;
+    constexpr size_type size_bytes() const noexcept;
+    [[nodiscard]] constexpr bool empty() const noexcept;
+
+    // [span.elem], span element access
+    constexpr reference operator[](size_type idx) const;
+    constexpr reference at(size_type idx) const; // since C++26
+    constexpr reference front() const;
+    constexpr reference back() const;
+    constexpr pointer data() const noexcept;
+
+    // [span.iterators], span iterator support
+    constexpr iterator begin() const noexcept;
+    constexpr iterator end() const noexcept;
+    constexpr reverse_iterator rbegin() const noexcept;
+    constexpr reverse_iterator rend() const noexcept;
+
+private:
+    pointer data_;    // exposition only
+    size_type size_;  // exposition only
+};
+
+template<class It, class EndOrSize>
+    span(It, EndOrSize) -> span<remove_reference_t<iter_reference_t<_It>>>;                             // until C++26
+template<class It, class EndOrSize>
+    span(It, EndOrSize) -> span<remove_reference_t<iter_reference_t<It>>, maybe-static-ext<EndOrSize>>; // since C++26
+
+template<class T, size_t N>
+    span(T (&)[N]) -> span<T, N>;
+
+template<class T, size_t N>
+    span(array<T, N>&) -> span<T, N>;
+
+template<class T, size_t N>
+    span(const array<T, N>&) -> span<const T, N>;
+
+template<class R>
+    span(R&&) -> span<remove_reference_t<ranges::range_reference_t<R>>>;
+
+} // namespace std
+
+*/
+
+#include <__assert>
+#include <__concepts/convertible_to.h>
+#include <__concepts/equality_comparable.h>
+#include <__config>
+#include <__fwd/array.h>
+#include <__fwd/span.h>
+#include <__iterator/bounded_iter.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/reverse_iterator.h>
+#include <__iterator/wrap_iter.h>
+#include <__memory/pointer_traits.h>
+#include <__ranges/concepts.h>
+#include <__ranges/data.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/enable_view.h>
+#include <__ranges/size.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_const.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_const.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/remove_reference.h>
+#include <__type_traits/type_identity.h>
+#include <__utility/forward.h>
+#include <cstddef> // for byte
+#include <initializer_list>
+#include <stdexcept>
+#include <version>
+
+// standard-mandated includes
+
+// [iterator.range]
+#include <__iterator/access.h>
+#include <__iterator/data.h>
+#include <__iterator/empty.h>
+#include <__iterator/reverse_access.h>
+#include <__iterator/size.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+struct __is_std_span : false_type {};
+
+template <class _Tp, size_t _Sz>
+struct __is_std_span<span<_Tp, _Sz>> : true_type {};
+
+template <class _Range, class _ElementType>
+concept __span_compatible_range =
+    !__is_std_span<remove_cvref_t<_Range>>::value &&                //
+    ranges::contiguous_range<_Range> &&                             //
+    ranges::sized_range<_Range> &&                                  //
+    (ranges::borrowed_range<_Range> || is_const_v<_ElementType>) && //
+    !__is_std_array<remove_cvref_t<_Range>>::value &&               //
+    !is_array_v<remove_cvref_t<_Range>> &&                          //
+    is_convertible_v<remove_reference_t<ranges::range_reference_t<_Range>> (*)[], _ElementType (*)[]>;
+
+template <class _From, class _To>
+concept __span_array_convertible = is_convertible_v<_From (*)[], _To (*)[]>;
+
+template <class _It, class _Tp>
+concept __span_compatible_iterator =
+    contiguous_iterator<_It> && __span_array_convertible<remove_reference_t<iter_reference_t<_It>>, _Tp>;
+
+template <class _Sentinel, class _It>
+concept __span_compatible_sentinel_for = sized_sentinel_for<_Sentinel, _It> && !is_convertible_v<_Sentinel, size_t>;
+
+template <typename _Tp, size_t _Extent>
+class _LIBCPP_TEMPLATE_VIS span {
+public:
+  //  constants and types
+  using element_type    = _Tp;
+  using value_type      = remove_cv_t<_Tp>;
+  using size_type       = size_t;
+  using difference_type = ptrdiff_t;
+  using pointer         = _Tp*;
+  using const_pointer   = const _Tp*;
+  using reference       = _Tp&;
+  using const_reference = const _Tp&;
+#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+  using iterator = __bounded_iter<pointer>;
+#  else
+  using iterator = __wrap_iter<pointer>;
+#  endif
+  using reverse_iterator = std::reverse_iterator<iterator>;
+
+  static constexpr size_type extent = _Extent;
+
+  // [span.cons], span constructors, copy, assignment, and destructor
+  template <size_t _Sz = _Extent>
+    requires(_Sz == 0)
+  _LIBCPP_HIDE_FROM_ABI constexpr span() noexcept : __data_{nullptr} {}
+
+#  if _LIBCPP_STD_VER >= 26
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit span(std::initializer_list<value_type> __il)
+    requires is_const_v<element_type>
+      : __data_{__il.begin()} {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(
+        _Extent == __il.size(), "Size mismatch in span's constructor _Extent != __il.size().");
+  }
+#  endif
+
+  constexpr span(const span&) noexcept            = default;
+  constexpr span& operator=(const span&) noexcept = default;
+
+  template <__span_compatible_iterator<element_type> _It>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit span(_It __first, size_type __count) : __data_{std::to_address(__first)} {
+    (void)__count;
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(_Extent == __count, "size mismatch in span's constructor (iterator, len)");
+  }
+
+  template <__span_compatible_iterator<element_type> _It, __span_compatible_sentinel_for<_It> _End>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit span(_It __first, _End __last) : __data_{std::to_address(__first)} {
+    // [span.cons]/10
+    // Throws: When and what last - first throws.
+    [[maybe_unused]] auto __dist = __last - __first;
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__dist >= 0, "invalid range in span's constructor (iterator, sentinel)");
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __dist == _Extent, "invalid range in span's constructor (iterator, sentinel): last - first != extent");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr span(type_identity_t<element_type> (&__arr)[_Extent]) noexcept : __data_{__arr} {}
+
+  template <__span_array_convertible<element_type> _OtherElementType>
+  _LIBCPP_HIDE_FROM_ABI constexpr span(array<_OtherElementType, _Extent>& __arr) noexcept : __data_{__arr.data()} {}
+
+  template <class _OtherElementType>
+    requires __span_array_convertible<const _OtherElementType, element_type>
+  _LIBCPP_HIDE_FROM_ABI constexpr span(const array<_OtherElementType, _Extent>& __arr) noexcept
+      : __data_{__arr.data()} {}
+
+  template <__span_compatible_range<element_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit span(_Range&& __r) : __data_{ranges::data(__r)} {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(ranges::size(__r) == _Extent, "size mismatch in span's constructor (range)");
+  }
+
+  template <__span_array_convertible<element_type> _OtherElementType>
+  _LIBCPP_HIDE_FROM_ABI constexpr span(const span<_OtherElementType, _Extent>& __other) noexcept
+      : __data_{__other.data()} {}
+
+  template <__span_array_convertible<element_type> _OtherElementType>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit span(const span<_OtherElementType, dynamic_extent>& __other) noexcept
+      : __data_{__other.data()} {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(_Extent == __other.size(), "size mismatch in span's constructor (other span)");
+  }
+
+  template <size_t _Count>
+  _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, _Count> first() const noexcept {
+    static_assert(_Count <= _Extent, "span<T, N>::first<Count>(): Count out of range");
+    return span<element_type, _Count>{data(), _Count};
+  }
+
+  template <size_t _Count>
+  _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, _Count> last() const noexcept {
+    static_assert(_Count <= _Extent, "span<T, N>::last<Count>(): Count out of range");
+    return span<element_type, _Count>{data() + size() - _Count, _Count};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, dynamic_extent> first(size_type __count) const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__count <= size(), "span<T, N>::first(count): count out of range");
+    return {data(), __count};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, dynamic_extent> last(size_type __count) const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__count <= size(), "span<T, N>::last(count): count out of range");
+    return {data() + size() - __count, __count};
+  }
+
+  template <size_t _Offset, size_t _Count = dynamic_extent>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto
+  subspan() const noexcept -> span<element_type, _Count != dynamic_extent ? _Count : _Extent - _Offset> {
+    static_assert(_Offset <= _Extent, "span<T, N>::subspan<Offset, Count>(): Offset out of range");
+    static_assert(_Count == dynamic_extent || _Count <= _Extent - _Offset,
+                  "span<T, N>::subspan<Offset, Count>(): Offset + Count out of range");
+
+    using _ReturnType = span<element_type, _Count != dynamic_extent ? _Count : _Extent - _Offset>;
+    return _ReturnType{data() + _Offset, _Count == dynamic_extent ? size() - _Offset : _Count};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, dynamic_extent>
+  subspan(size_type __offset, size_type __count = dynamic_extent) const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__offset <= size(), "span<T, N>::subspan(offset, count): offset out of range");
+    if (__count == dynamic_extent)
+      return {data() + __offset, size() - __offset};
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __count <= size() - __offset, "span<T, N>::subspan(offset, count): offset + count out of range");
+    return {data() + __offset, __count};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr size_type size() const noexcept { return _Extent; }
+  _LIBCPP_HIDE_FROM_ABI constexpr size_type size_bytes() const noexcept { return _Extent * sizeof(element_type); }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool empty() const noexcept { return _Extent == 0; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr reference operator[](size_type __idx) const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__idx < size(), "span<T, N>::operator[](index): index out of range");
+    return __data_[__idx];
+  }
+
+#  if _LIBCPP_STD_VER >= 26
+  _LIBCPP_HIDE_FROM_ABI constexpr reference at(size_type __index) const {
+    if (__index >= size())
+      std::__throw_out_of_range("span");
+    return __data_[__index];
+  }
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI constexpr reference front() const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "span<T, N>::front() on empty span");
+    return __data_[0];
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr reference back() const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "span<T, N>::back() on empty span");
+    return __data_[size() - 1];
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr pointer data() const noexcept { return __data_; }
+
+  // [span.iter], span iterator support
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator begin() const noexcept {
+#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+    return std::__make_bounded_iter(data(), data(), data() + size());
+#  else
+    return iterator(data());
+#  endif
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator end() const noexcept {
+#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+    return std::__make_bounded_iter(data() + size(), data(), data() + size());
+#  else
+    return iterator(data() + size());
+#  endif
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rbegin() const noexcept { return reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rend() const noexcept { return reverse_iterator(begin()); }
+
+  _LIBCPP_HIDE_FROM_ABI span<const byte, _Extent * sizeof(element_type)> __as_bytes() const noexcept {
+    return span<const byte, _Extent * sizeof(element_type)>{reinterpret_cast<const byte*>(data()), size_bytes()};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI span<byte, _Extent * sizeof(element_type)> __as_writable_bytes() const noexcept {
+    return span<byte, _Extent * sizeof(element_type)>{reinterpret_cast<byte*>(data()), size_bytes()};
+  }
+
+private:
+  pointer __data_;
+};
+
+template <typename _Tp>
+class _LIBCPP_TEMPLATE_VIS span<_Tp, dynamic_extent> {
+public:
+  //  constants and types
+  using element_type    = _Tp;
+  using value_type      = remove_cv_t<_Tp>;
+  using size_type       = size_t;
+  using difference_type = ptrdiff_t;
+  using pointer         = _Tp*;
+  using const_pointer   = const _Tp*;
+  using reference       = _Tp&;
+  using const_reference = const _Tp&;
+#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+  using iterator = __bounded_iter<pointer>;
+#  else
+  using iterator = __wrap_iter<pointer>;
+#  endif
+  using reverse_iterator = std::reverse_iterator<iterator>;
+
+  static constexpr size_type extent = dynamic_extent;
+
+  // [span.cons], span constructors, copy, assignment, and destructor
+  _LIBCPP_HIDE_FROM_ABI constexpr span() noexcept : __data_{nullptr}, __size_{0} {}
+
+#  if _LIBCPP_STD_VER >= 26
+  _LIBCPP_HIDE_FROM_ABI constexpr span(std::initializer_list<value_type> __il)
+    requires is_const_v<element_type>
+      : __data_{__il.begin()}, __size_{__il.size()} {}
+#  endif
+
+  constexpr span(const span&) noexcept            = default;
+  constexpr span& operator=(const span&) noexcept = default;
+
+  template <__span_compatible_iterator<element_type> _It>
+  _LIBCPP_HIDE_FROM_ABI constexpr span(_It __first, size_type __count)
+      : __data_{std::to_address(__first)}, __size_{__count} {}
+
+  template <__span_compatible_iterator<element_type> _It, __span_compatible_sentinel_for<_It> _End>
+  _LIBCPP_HIDE_FROM_ABI constexpr span(_It __first, _End __last)
+      : __data_(std::to_address(__first)), __size_(__last - __first) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__last - __first >= 0, "invalid range in span's constructor (iterator, sentinel)");
+  }
+
+  template <size_t _Sz>
+  _LIBCPP_HIDE_FROM_ABI constexpr span(type_identity_t<element_type> (&__arr)[_Sz]) noexcept
+      : __data_{__arr}, __size_{_Sz} {}
+
+  template <__span_array_convertible<element_type> _OtherElementType, size_t _Sz>
+  _LIBCPP_HIDE_FROM_ABI constexpr span(array<_OtherElementType, _Sz>& __arr) noexcept
+      : __data_{__arr.data()}, __size_{_Sz} {}
+
+  template <class _OtherElementType, size_t _Sz>
+    requires __span_array_convertible<const _OtherElementType, element_type>
+  _LIBCPP_HIDE_FROM_ABI constexpr span(const array<_OtherElementType, _Sz>& __arr) noexcept
+      : __data_{__arr.data()}, __size_{_Sz} {}
+
+  template <__span_compatible_range<element_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr span(_Range&& __r) : __data_(ranges::data(__r)), __size_{ranges::size(__r)} {}
+
+  template <__span_array_convertible<element_type> _OtherElementType, size_t _OtherExtent>
+  _LIBCPP_HIDE_FROM_ABI constexpr span(const span<_OtherElementType, _OtherExtent>& __other) noexcept
+      : __data_{__other.data()}, __size_{__other.size()} {}
+
+  template <size_t _Count>
+  _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, _Count> first() const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(_Count <= size(), "span<T>::first<Count>(): Count out of range");
+    return span<element_type, _Count>{data(), _Count};
+  }
+
+  template <size_t _Count>
+  _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, _Count> last() const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(_Count <= size(), "span<T>::last<Count>(): Count out of range");
+    return span<element_type, _Count>{data() + size() - _Count, _Count};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, dynamic_extent> first(size_type __count) const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__count <= size(), "span<T>::first(count): count out of range");
+    return {data(), __count};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, dynamic_extent> last(size_type __count) const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__count <= size(), "span<T>::last(count): count out of range");
+    return {data() + size() - __count, __count};
+  }
+
+  template <size_t _Offset, size_t _Count = dynamic_extent>
+  _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, _Count> subspan() const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(_Offset <= size(), "span<T>::subspan<Offset, Count>(): Offset out of range");
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(_Count == dynamic_extent || _Count <= size() - _Offset,
+                                        "span<T>::subspan<Offset, Count>(): Offset + Count out of range");
+    return span<element_type, _Count>{data() + _Offset, _Count == dynamic_extent ? size() - _Offset : _Count};
+  }
+
+  constexpr span<element_type, dynamic_extent> _LIBCPP_HIDE_FROM_ABI
+  subspan(size_type __offset, size_type __count = dynamic_extent) const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__offset <= size(), "span<T>::subspan(offset, count): offset out of range");
+    if (__count == dynamic_extent)
+      return {data() + __offset, size() - __offset};
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __count <= size() - __offset, "span<T>::subspan(offset, count): offset + count out of range");
+    return {data() + __offset, __count};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr size_type size() const noexcept { return __size_; }
+  _LIBCPP_HIDE_FROM_ABI constexpr size_type size_bytes() const noexcept { return __size_ * sizeof(element_type); }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool empty() const noexcept { return __size_ == 0; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr reference operator[](size_type __idx) const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__idx < size(), "span<T>::operator[](index): index out of range");
+    return __data_[__idx];
+  }
+
+#  if _LIBCPP_STD_VER >= 26
+  _LIBCPP_HIDE_FROM_ABI constexpr reference at(size_type __index) const {
+    if (__index >= size())
+      std::__throw_out_of_range("span");
+    return __data_[__index];
+  }
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI constexpr reference front() const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "span<T>::front() on empty span");
+    return __data_[0];
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr reference back() const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "span<T>::back() on empty span");
+    return __data_[size() - 1];
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr pointer data() const noexcept { return __data_; }
+
+  // [span.iter], span iterator support
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator begin() const noexcept {
+#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+    return std::__make_bounded_iter(data(), data(), data() + size());
+#  else
+    return iterator(data());
+#  endif
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator end() const noexcept {
+#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+    return std::__make_bounded_iter(data() + size(), data(), data() + size());
+#  else
+    return iterator(data() + size());
+#  endif
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rbegin() const noexcept { return reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rend() const noexcept { return reverse_iterator(begin()); }
+
+  _LIBCPP_HIDE_FROM_ABI span<const byte, dynamic_extent> __as_bytes() const noexcept {
+    return {reinterpret_cast<const byte*>(data()), size_bytes()};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI span<byte, dynamic_extent> __as_writable_bytes() const noexcept {
+    return {reinterpret_cast<byte*>(data()), size_bytes()};
+  }
+
+private:
+  pointer __data_;
+  size_type __size_;
+};
+
+template <class _Tp, size_t _Extent>
+inline constexpr bool ranges::enable_borrowed_range<span<_Tp, _Extent> > = true;
+
+template <class _ElementType, size_t _Extent>
+inline constexpr bool ranges::enable_view<span<_ElementType, _Extent>> = true;
+
+//  as_bytes & as_writable_bytes
+template <class _Tp, size_t _Extent>
+_LIBCPP_HIDE_FROM_ABI auto as_bytes(span<_Tp, _Extent> __s) noexcept {
+  return __s.__as_bytes();
+}
+
+template <class _Tp, size_t _Extent>
+  requires(!is_const_v<_Tp>)
+_LIBCPP_HIDE_FROM_ABI auto as_writable_bytes(span<_Tp, _Extent> __s) noexcept {
+  return __s.__as_writable_bytes();
+}
+
+#  if _LIBCPP_STD_VER >= 26
+template <class _Tp>
+concept __integral_constant_like =
+    is_integral_v<decltype(_Tp::value)> && !is_same_v<bool, remove_const_t<decltype(_Tp::value)>> &&
+    convertible_to<_Tp, decltype(_Tp::value)> && equality_comparable_with<_Tp, decltype(_Tp::value)> &&
+    bool_constant<_Tp() == _Tp::value>::value &&
+    bool_constant<static_cast<decltype(_Tp::value)>(_Tp()) == _Tp::value>::value;
+
+template <class _Tp>
+inline constexpr size_t __maybe_static_ext = dynamic_extent;
+
+template <__integral_constant_like _Tp>
+inline constexpr size_t __maybe_static_ext<_Tp> = {_Tp::value};
+
+template <contiguous_iterator _It, class _EndOrSize>
+span(_It, _EndOrSize) -> span<remove_reference_t<iter_reference_t<_It>>, __maybe_static_ext<_EndOrSize>>;
+#  else
+template <contiguous_iterator _It, class _EndOrSize>
+span(_It, _EndOrSize) -> span<remove_reference_t<iter_reference_t<_It>>>;
+#  endif
+
+template <class _Tp, size_t _Sz>
+span(_Tp (&)[_Sz]) -> span<_Tp, _Sz>;
+
+template <class _Tp, size_t _Sz>
+span(array<_Tp, _Sz>&) -> span<_Tp, _Sz>;
+
+template <class _Tp, size_t _Sz>
+span(const array<_Tp, _Sz>&) -> span<const _Tp, _Sz>;
+
+template <ranges::contiguous_range _Range>
+span(_Range&&) -> span<remove_reference_t<ranges::range_reference_t<_Range>>>;
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <array>
+#  include <concepts>
+#  include <functional>
+#  include <iterator>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_SPAN
diff --git a/libcxx/include/__cxx03/sstream b/libcxx/include/__cxx03/sstream
new file mode 100644
index 000000000000..9ba43ffeb850
--- /dev/null
+++ b/libcxx/include/__cxx03/sstream
@@ -0,0 +1,1276 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_SSTREAM
+#define _LIBCPP_SSTREAM
+
+// clang-format off
+
+/*
+    sstream synopsis [sstream.syn]
+
+// Class template basic_stringbuf [stringbuf]
+template <class charT, class traits = char_traits<charT>, class Allocator = allocator<charT> >
+class basic_stringbuf
+    : public basic_streambuf<charT, traits>
+{
+public:
+    typedef charT                          char_type;
+    typedef traits                         traits_type;
+    typedef typename traits_type::int_type int_type;
+    typedef typename traits_type::pos_type pos_type;
+    typedef typename traits_type::off_type off_type;
+    typedef Allocator                      allocator_type;
+
+    // [stringbuf.cons] constructors:
+    explicit basic_stringbuf(ios_base::openmode which = ios_base::in | ios_base::out); // before C++20
+    basic_stringbuf() : basic_stringbuf(ios_base::in | ios_base::out) {}               // C++20
+    explicit basic_stringbuf(ios_base::openmode which);                                // C++20
+    explicit basic_stringbuf(const basic_string<char_type, traits_type, allocator_type>& s,
+                             ios_base::openmode which = ios_base::in | ios_base::out);
+    explicit basic_stringbuf(const allocator_type& a)
+        : basic_stringbuf(ios_base::in | ios_base::out, a) {}                          // C++20
+    basic_stringbuf(ios_base::openmode which, const allocator_type& a);                // C++20
+    explicit basic_stringbuf(basic_string<char_type, traits_type, allocator_type>&& s,
+                             ios_base::openmode which = ios_base::in | ios_base::out); // C++20
+    template <class SAlloc>
+    basic_stringbuf(const basic_string<char_type, traits_type, SAlloc>& s, const allocator_type& a)
+        : basic_stringbuf(s, ios_base::in | ios_base::out, a) {}                       // C++20
+    template <class SAlloc>
+    basic_stringbuf(const basic_string<char_type, traits_type, SAlloc>& s,
+                    ios_base::openmode which, const allocator_type& a);                // C++20
+    template <class SAlloc>
+    explicit basic_stringbuf(const basic_string<char_type, traits_type, SAlloc>& s,
+                             ios_base::openmode which = ios_base::in | ios_base::out); // C++20
+    template<class T>
+      explicit basic_stringbuf(const T& t,
+                               ios_base::openmode which = ios_base::in | ios_base::out); // Since C++26
+    template<class T>
+      basic_stringbuf(const T& t, const Allocator& a);                                   // Since C++26
+    template<class T>
+      basic_stringbuf(const T& t, ios_base::openmode which, const Allocator& a);         // Since C++26
+    basic_stringbuf(const basic_stringbuf&) = delete;
+    basic_stringbuf(basic_stringbuf&& rhs);
+    basic_stringbuf(basic_stringbuf&& rhs, const allocator_type& a);                   // C++20
+
+    // [stringbuf.assign] Assign and swap:
+    basic_stringbuf& operator=(const basic_stringbuf&) = delete;
+    basic_stringbuf& operator=(basic_stringbuf&& rhs);
+    void swap(basic_stringbuf& rhs) noexcept(see below);                               // conditionally noexcept since C++20
+
+    // [stringbuf.members] Member functions:
+    allocator_type get_allocator() const noexcept;                                     // C++20
+    basic_string<char_type, traits_type, allocator_type> str() const;                  // before C++20
+    basic_string<char_type, traits_type, allocator_type> str() const &;                // C++20
+    template <class SAlloc>
+    basic_string<char_type, traits_type, SAlloc> str(const SAlloc& sa) const;          // C++20
+    basic_string<char_type, traits_type, allocator_type> str() &&;                     // C++20
+    basic_string_view<char_type, traits_type> view() const noexcept;                   // C++20
+    void str(const basic_string<char_type, traits_type, allocator_type>& s);
+    template <class SAlloc>
+    void str(const basic_string<char_type, traits_type, SAlloc>& s);                   // C++20
+    void str(basic_string<char_type, traits_type, allocator_type>&& s);                // C++20
+    template<class T>
+      void str(const T& t);                                                            // Since C++26
+
+protected:
+    // [stringbuf.virtuals] Overridden virtual functions:
+    virtual int_type underflow();
+    virtual int_type pbackfail(int_type c = traits_type::eof());
+    virtual int_type overflow (int_type c = traits_type::eof());
+    virtual basic_streambuf<char_type, traits_type>* setbuf(char_type*, streamsize);
+    virtual pos_type seekoff(off_type off, ios_base::seekdir way,
+                             ios_base::openmode which = ios_base::in | ios_base::out);
+    virtual pos_type seekpos(pos_type sp,
+                             ios_base::openmode which = ios_base::in | ios_base::out);
+};
+
+// [stringbuf.assign] non member swap
+template <class charT, class traits, class Allocator>
+void swap(basic_stringbuf<charT, traits, Allocator>& x,
+          basic_stringbuf<charT, traits, Allocator>& y); // conditionally noexcept since C++20
+
+typedef basic_stringbuf<char>    stringbuf;
+typedef basic_stringbuf<wchar_t> wstringbuf;
+
+// Class template basic_istringstream [istringstream]
+template <class charT, class traits = char_traits<charT>, class Allocator = allocator<charT> >
+class basic_istringstream
+    : public basic_istream<charT, traits>
+{
+public:
+    typedef charT                          char_type;
+    typedef traits                         traits_type;
+    typedef typename traits_type::int_type int_type;
+    typedef typename traits_type::pos_type pos_type;
+    typedef typename traits_type::off_type off_type;
+    typedef Allocator                      allocator_type;
+
+    // [istringstream.cons] Constructors:
+    explicit basic_istringstream(ios_base::openmode which = ios_base::in);             // before C++20
+    basic_istringstream() : basic_istringstream(ios_base::in) {}                       // C++20
+    explicit basic_istringstream(ios_base::openmode which);                            // C++20
+    explicit basic_istringstream(const basic_string<char_type, traits_type, allocator_type>& s,
+                                 ios_base::openmode which = ios_base::in);
+    basic_istringstream(ios_base::openmode which, const allocator_type& a);            // C++20
+    explicit basic_istringstream(basic_string<char_type, traits_type, allocator_type>&& s,
+                                 ios_base::openmode which = ios_base::in);             // C++20
+    template <class SAlloc>
+    basic_istringstream(const basic_string<char_type, traits_type, SAlloc>& s, const allocator_type& a)
+        : basic_istringstream(s, ios_base::in, a) {}                                   // C++20
+    template <class SAlloc>
+    basic_istringstream(const basic_string<char_type, traits_type, SAlloc>& s,
+                        ios_base::openmode which, const allocator_type& a);            // C++20
+    template <class SAlloc>
+    explicit basic_istringstream(const basic_string<char_type, traits_type, SAlloc>& s,
+                                 ios_base::openmode which = ios_base::in);             // C++20
+    template<class T>
+      explicit basic_istringstream(const T& t, ios_base::openmode which = ios_base::in); // Since C++26
+    template<class T>
+      basic_istringstream(const T& t, const Allocator& a);                               // Since C++26
+    template<class T>
+      basic_istringstream(const T& t, ios_base::openmode which, const Allocator& a);     // Since C++26
+    basic_istringstream(const basic_istringstream&) = delete;
+    basic_istringstream(basic_istringstream&& rhs);
+
+    // [istringstream.assign] Assign and swap:
+    basic_istringstream& operator=(const basic_istringstream&) = delete;
+    basic_istringstream& operator=(basic_istringstream&& rhs);
+    void swap(basic_istringstream& rhs);
+
+    // [istringstream.members] Member functions:
+    basic_stringbuf<char_type, traits_type, allocator_type>* rdbuf() const;
+    basic_string<char_type, traits_type, allocator_type> str() const;                  // before C++20
+    basic_string<char_type, traits_type, allocator_type> str() const &;                // C++20
+    template <class SAlloc>
+    basic_string<char_type, traits_type, SAlloc> str(const SAlloc& sa) const;          // C++20
+    basic_string<char_type, traits_type, allocator_type> str() &&;                     // C++20
+    basic_string_view<char_type, traits_type> view() const noexcept;                   // C++20
+    void str(const basic_string<char_type, traits_type, allocator_type>& s);
+    template <class SAlloc>
+    void str(const basic_string<char_type, traits_type, SAlloc>& s);                   // C++20
+    void str(basic_string<char_type, traits_type, allocator_type>&& s);                // C++20
+    template<class T>
+      void str(const T& t);                                                            // Since C++26
+};
+
+template <class charT, class traits, class Allocator>
+void swap(basic_istringstream<charT, traits, Allocator>& x,
+          basic_istringstream<charT, traits, Allocator>& y);
+
+typedef basic_istringstream<char>    istringstream;
+typedef basic_istringstream<wchar_t> wistringstream;
+
+// Class template basic_ostringstream [ostringstream]
+template <class charT, class traits = char_traits<charT>, class Allocator = allocator<charT> >
+class basic_ostringstream
+    : public basic_ostream<charT, traits>
+{
+public:
+    // types:
+    typedef charT                          char_type;
+    typedef traits                         traits_type;
+    typedef typename traits_type::int_type int_type;
+    typedef typename traits_type::pos_type pos_type;
+    typedef typename traits_type::off_type off_type;
+    typedef Allocator                      allocator_type;
+
+    // [ostringstream.cons] Constructors:
+    explicit basic_ostringstream(ios_base::openmode which = ios_base::out);            // before C++20
+    basic_ostringstream() : basic_ostringstream(ios_base::out) {}                      // C++20
+    explicit basic_ostringstream(ios_base::openmode which);                            // C++20
+    explicit basic_ostringstream(const basic_string<char_type, traits_type, allocator_type>& s,
+                                 ios_base::openmode which = ios_base::out);
+    basic_ostringstream(ios_base::openmode which, const allocator_type& a);            // C++20
+    explicit basic_ostringstream(basic_string<char_type, traits_type, allocator_type>&& s,
+                                 ios_base::openmode which = ios_base::out);            // C++20
+    template <class SAlloc>
+    basic_ostringstream(const basic_string<char_type, traits_type, SAlloc>& s, const allocator_type& a)
+        : basic_ostringstream(s, ios_base::out, a) {}                                  // C++20
+    template <class SAlloc>
+    basic_ostringstream(const basic_string<char_type, traits_type, SAlloc>& s,
+                        ios_base::openmode which, const allocator_type& a);            // C++20
+    template <class SAlloc>
+    explicit basic_ostringstream(const basic_string<char_type, traits_type, SAlloc>& s,
+                                 ios_base::openmode which = ios_base::out);            // C++20
+    template<class T>
+      explicit basic_ostringstream(const T& t, ios_base::openmode which = ios_base::out); // Since C++26
+    template<class T>
+      basic_ostringstream(const T& t, const Allocator& a);                                // Since C++26
+    template<class T>
+      basic_ostringstream(const T& t, ios_base::openmode which, const Allocator& a);      // Since C++26
+    basic_ostringstream(const basic_ostringstream&) = delete;
+    basic_ostringstream(basic_ostringstream&& rhs);
+
+    // [ostringstream.assign] Assign and swap:
+    basic_ostringstream& operator=(const basic_ostringstream&) = delete;
+    basic_ostringstream& operator=(basic_ostringstream&& rhs);
+    void swap(basic_ostringstream& rhs);
+
+    // [ostringstream.members] Member functions:
+    basic_stringbuf<char_type, traits_type, allocator_type>* rdbuf() const;
+    basic_string<char_type, traits_type, allocator_type> str() const;                  // before C++20
+    basic_string<char_type, traits_type, allocator_type> str() const &;                // C++20
+    template <class SAlloc>
+    basic_string<char_type, traits_type, SAlloc> str(const SAlloc& sa) const;          // C++20
+    basic_string<char_type, traits_type, allocator_type> str() &&;                     // C++20
+    basic_string_view<char_type, traits_type> view() const noexcept;                   // C++20
+    void str(const basic_string<char_type, traits_type, allocator_type>& s);
+    template <class SAlloc>
+    void str(const basic_string<char_type, traits_type, SAlloc>& s);                   // C++20
+    void str(basic_string<char_type, traits_type, allocator_type>&& s);                // C++20
+    template<class T>
+      void str(const T& t);                                                            // Since C++26
+};
+
+template <class charT, class traits, class Allocator>
+void swap(basic_ostringstream<charT, traits, Allocator>& x,
+          basic_ostringstream<charT, traits, Allocator>& y);
+
+typedef basic_ostringstream<char>    ostringstream;
+typedef basic_ostringstream<wchar_t> wostringstream;
+
+// Class template basic_stringstream [stringstream]
+template <class charT, class traits = char_traits<charT>, class Allocator = allocator<charT> >
+class basic_stringstream
+    : public basic_iostream<charT, traits>
+{
+public:
+    // types:
+    typedef charT                          char_type;
+    typedef traits                         traits_type;
+    typedef typename traits_type::int_type int_type;
+    typedef typename traits_type::pos_type pos_type;
+    typedef typename traits_type::off_type off_type;
+    typedef Allocator                      allocator_type;
+
+    // [stringstream.cons] constructors
+    explicit basic_stringstream(ios_base::openmode which = ios_base::out | ios_base::in); // before C++20
+    basic_stringstream() : basic_stringstream(ios_base::out | ios_base::in) {}            // C++20
+    explicit basic_stringstream(ios_base::openmode which);                                // C++20
+    explicit basic_stringstream(const basic_string<char_type, traits_type, allocator_type>& s,
+                                ios_base::openmode which = ios_base::out | ios_base::in);
+    basic_stringstream(ios_base::openmode which, const allocator_type& a);                // C++20
+    explicit basic_stringstream(basic_string<char_type, traits_type, allocator_type>&& s,
+                                ios_base::openmode which = ios_base::out | ios_base::in); // C++20
+    template <class SAlloc>
+    basic_stringstream(const basic_string<char_type, traits_type, SAlloc>& s, const allocator_type& a)
+        : basic_stringstream(s, ios_base::out | ios_base::in, a) {}                       // C++20
+    template <class SAlloc>
+    basic_stringstream(const basic_string<char_type, traits_type, SAlloc>& s,
+                       ios_base::openmode which, const allocator_type& a);                // C++20
+    template <class SAlloc>
+    explicit basic_stringstream(const basic_string<char_type, traits_type, SAlloc>& s,
+                                ios_base::openmode which = ios_base::out | ios_base::in); // C++20
+    template<class T>
+      explicit basic_stringstream(const T& t,
+                                  ios_base::openmode which = ios_base::out | ios_base::in); // Since C++26
+    template<class T>
+      basic_stringstream(const T& t, const Allocator& a);                                   // Since C++26
+    template<class T>
+      basic_stringstream(const T& t, ios_base::openmode which, const Allocator& a);         // Since C++26
+    basic_stringstream(const basic_stringstream&) = delete;
+    basic_stringstream(basic_stringstream&& rhs);
+
+    // [stringstream.assign] Assign and swap:
+    basic_stringstream& operator=(const basic_stringstream&) = delete;
+    basic_stringstream& operator=(basic_stringstream&& rhs);
+    void swap(basic_stringstream& rhs);
+
+    // [stringstream.members] Member functions:
+    basic_stringbuf<char_type, traits_type, allocator_type>* rdbuf() const;
+    basic_string<char_type, traits_type, allocator_type> str() const;                     // before C++20
+    basic_string<char_type, traits_type, allocator_type> str() const &;                   // C++20
+    template <class SAlloc>
+    basic_string<char_type, traits_type, SAlloc> str(const SAlloc& sa) const;             // C++20
+    basic_string<char_type, traits_type, allocator_type> str() &&;                        // C++20
+    basic_string_view<char_type, traits_type> view() const noexcept;                      // C++20
+    void str(const basic_string<char_type, traits_type, allocator_type>& s);
+    template <class SAlloc>
+    void str(const basic_string<char_type, traits_type, SAlloc>& s);                      // C++20
+    void str(basic_string<char_type, traits_type, allocator_type>&& s);                   // C++20
+    template<class T>
+      void str(const T& t);                                                               // Since C++26
+};
+
+template <class charT, class traits, class Allocator>
+void swap(basic_stringstream<charT, traits, Allocator>& x,
+          basic_stringstream<charT, traits, Allocator>& y);
+
+typedef basic_stringstream<char>    stringstream;
+typedef basic_stringstream<wchar_t> wstringstream;
+
+}  // std
+
+*/
+
+// clang-format on
+
+#include <__config>
+#include <__fwd/sstream.h>
+#include <__ostream/basic_ostream.h>
+#include <__type_traits/is_convertible.h>
+#include <__utility/swap.h>
+#include <istream>
+#include <string>
+#include <string_view>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Class template basic_stringbuf [stringbuf]
+
+template <class _CharT, class _Traits, class _Allocator>
+class _LIBCPP_TEMPLATE_VIS basic_stringbuf : public basic_streambuf<_CharT, _Traits> {
+public:
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+  typedef _Allocator allocator_type;
+
+  typedef basic_string<char_type, traits_type, allocator_type> string_type;
+
+private:
+  string_type __str_;
+  mutable char_type* __hm_;
+  ios_base::openmode __mode_;
+  _LIBCPP_HIDE_FROM_ABI void __init_buf_ptrs();
+  _LIBCPP_HIDE_FROM_ABI void __move_init(basic_stringbuf&& __rhs);
+
+public:
+  // [stringbuf.cons] constructors:
+  _LIBCPP_HIDE_FROM_ABI basic_stringbuf() : __hm_(nullptr), __mode_(ios_base::in | ios_base::out) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_stringbuf(ios_base::openmode __wch) : __hm_(nullptr), __mode_(__wch) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_stringbuf(const string_type& __s,
+                                                 ios_base::openmode __wch = ios_base::in | ios_base::out)
+      : __str_(__s.get_allocator()), __hm_(nullptr), __mode_(__wch) {
+    str(__s);
+  }
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI explicit basic_stringbuf(const allocator_type& __a)
+      : basic_stringbuf(ios_base::in | ios_base::out, __a) {}
+
+  _LIBCPP_HIDE_FROM_ABI basic_stringbuf(ios_base::openmode __wch, const allocator_type& __a)
+      : __str_(__a), __hm_(nullptr), __mode_(__wch) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_stringbuf(string_type&& __s,
+                                                 ios_base::openmode __wch = ios_base::in | ios_base::out)
+      : __str_(std::move(__s)), __hm_(nullptr), __mode_(__wch) {
+    __init_buf_ptrs();
+  }
+
+  template <class _SAlloc>
+  _LIBCPP_HIDE_FROM_ABI
+  basic_stringbuf(const basic_string<char_type, traits_type, _SAlloc>& __s, const allocator_type& __a)
+      : basic_stringbuf(__s, ios_base::in | ios_base::out, __a) {}
+
+  template <class _SAlloc>
+  _LIBCPP_HIDE_FROM_ABI basic_stringbuf(
+      const basic_string<char_type, traits_type, _SAlloc>& __s, ios_base::openmode __wch, const allocator_type& __a)
+      : __str_(__s, __a), __hm_(nullptr), __mode_(__wch) {
+    __init_buf_ptrs();
+  }
+
+  template <class _SAlloc>
+    requires(!is_same_v<_SAlloc, allocator_type>)
+  _LIBCPP_HIDE_FROM_ABI explicit basic_stringbuf(const basic_string<char_type, traits_type, _SAlloc>& __s,
+                                                 ios_base::openmode __wch = ios_base::in | ios_base::out)
+      : __str_(__s), __hm_(nullptr), __mode_(__wch) {
+    __init_buf_ptrs();
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 26
+
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI explicit basic_stringbuf(const _Tp& __t,
+                                                 ios_base::openmode __which = ios_base::in | ios_base::out)
+      : basic_stringbuf(__t, __which, _Allocator()) {}
+
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI basic_stringbuf(const _Tp& __t, const _Allocator& __a)
+      : basic_stringbuf(__t, ios_base::in | ios_base::out, __a) {}
+
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI basic_stringbuf(const _Tp& __t, ios_base::openmode __which, const _Allocator& __a)
+      : __hm_(nullptr), __mode_(__which) {
+    basic_string_view<_CharT, _Traits> __sv = __t;
+    __str_                                  = string_type(__sv, __a);
+    __init_buf_ptrs();
+  }
+
+#endif //  _LIBCPP_STD_VER >= 26
+
+  basic_stringbuf(const basic_stringbuf&) = delete;
+  basic_stringbuf(basic_stringbuf&& __rhs) : __mode_(__rhs.__mode_) { __move_init(std::move(__rhs)); }
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI basic_stringbuf(basic_stringbuf&& __rhs, const allocator_type& __a)
+      : basic_stringbuf(__rhs.__mode_, __a) {
+    __move_init(std::move(__rhs));
+  }
+#endif
+
+  // [stringbuf.assign] Assign and swap:
+  basic_stringbuf& operator=(const basic_stringbuf&) = delete;
+  basic_stringbuf& operator=(basic_stringbuf&& __rhs);
+  void swap(basic_stringbuf& __rhs)
+#if _LIBCPP_STD_VER >= 20
+      noexcept(allocator_traits<allocator_type>::propagate_on_container_swap::value ||
+               allocator_traits<allocator_type>::is_always_equal::value)
+#endif
+          ;
+
+  // [stringbuf.members] Member functions:
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const noexcept { return __str_.get_allocator(); }
+#endif
+
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
+  string_type str() const;
+#else
+  _LIBCPP_HIDE_FROM_ABI string_type str() const& { return str(__str_.get_allocator()); }
+
+  _LIBCPP_HIDE_FROM_ABI string_type str() && {
+    const basic_string_view<_CharT, _Traits> __view = view();
+    typename string_type::size_type __pos           = __view.empty() ? 0 : __view.data() - __str_.data();
+    // In C++23, this is just string_type(std::move(__str_), __pos, __view.size(), __str_.get_allocator());
+    // But we need something that works in C++20 also.
+    string_type __result(std::move(__str_), __str_.get_allocator());
+    __result.resize(__pos + __view.size());
+    __result.erase(0, __pos);
+    __init_buf_ptrs();
+    return __result;
+  }
+#endif // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
+
+#if _LIBCPP_STD_VER >= 20
+  template <class _SAlloc>
+    requires __is_allocator<_SAlloc>::value
+  _LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
+    return basic_string<_CharT, _Traits, _SAlloc>(view(), __sa);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept;
+#endif // _LIBCPP_STD_VER >= 20
+
+  void str(const string_type& __s) {
+    __str_ = __s;
+    __init_buf_ptrs();
+  }
+
+#if _LIBCPP_STD_VER >= 20
+  template <class _SAlloc>
+    requires(!is_same_v<_SAlloc, allocator_type>)
+  _LIBCPP_HIDE_FROM_ABI void str(const basic_string<char_type, traits_type, _SAlloc>& __s) {
+    __str_ = __s;
+    __init_buf_ptrs();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void str(string_type&& __s) {
+    __str_ = std::move(__s);
+    __init_buf_ptrs();
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 26
+
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI void str(const _Tp& __t) {
+    basic_string_view<_CharT, _Traits> __sv = __t;
+    __str_                                  = __sv;
+    __init_buf_ptrs();
+  }
+
+#endif //  _LIBCPP_STD_VER >= 26
+
+protected:
+  // [stringbuf.virtuals] Overridden virtual functions:
+  int_type underflow() override;
+  int_type pbackfail(int_type __c = traits_type::eof()) override;
+  int_type overflow(int_type __c = traits_type::eof()) override;
+  pos_type
+  seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __wch = ios_base::in | ios_base::out) override;
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL
+  pos_type seekpos(pos_type __sp, ios_base::openmode __wch = ios_base::in | ios_base::out) override {
+    return seekoff(__sp, ios_base::beg, __wch);
+  }
+};
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI void basic_stringbuf<_CharT, _Traits, _Allocator>::__move_init(basic_stringbuf&& __rhs) {
+  char_type* __p   = const_cast<char_type*>(__rhs.__str_.data());
+  ptrdiff_t __binp = -1;
+  ptrdiff_t __ninp = -1;
+  ptrdiff_t __einp = -1;
+  if (__rhs.eback() != nullptr) {
+    __binp = __rhs.eback() - __p;
+    __ninp = __rhs.gptr() - __p;
+    __einp = __rhs.egptr() - __p;
+  }
+  ptrdiff_t __bout = -1;
+  ptrdiff_t __nout = -1;
+  ptrdiff_t __eout = -1;
+  if (__rhs.pbase() != nullptr) {
+    __bout = __rhs.pbase() - __p;
+    __nout = __rhs.pptr() - __p;
+    __eout = __rhs.epptr() - __p;
+  }
+  ptrdiff_t __hm = __rhs.__hm_ == nullptr ? -1 : __rhs.__hm_ - __p;
+  __str_         = std::move(__rhs.__str_);
+  __p            = const_cast<char_type*>(__str_.data());
+  if (__binp != -1)
+    this->setg(__p + __binp, __p + __ninp, __p + __einp);
+  if (__bout != -1) {
+    this->setp(__p + __bout, __p + __eout);
+    this->__pbump(__nout);
+  }
+  __hm_ = __hm == -1 ? nullptr : __p + __hm;
+  __p   = const_cast<char_type*>(__rhs.__str_.data());
+  __rhs.setg(__p, __p, __p);
+  __rhs.setp(__p, __p);
+  __rhs.__hm_ = __p;
+  this->pubimbue(__rhs.getloc());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+basic_stringbuf<_CharT, _Traits, _Allocator>&
+basic_stringbuf<_CharT, _Traits, _Allocator>::operator=(basic_stringbuf&& __rhs) {
+  char_type* __p   = const_cast<char_type*>(__rhs.__str_.data());
+  ptrdiff_t __binp = -1;
+  ptrdiff_t __ninp = -1;
+  ptrdiff_t __einp = -1;
+  if (__rhs.eback() != nullptr) {
+    __binp = __rhs.eback() - __p;
+    __ninp = __rhs.gptr() - __p;
+    __einp = __rhs.egptr() - __p;
+  }
+  ptrdiff_t __bout = -1;
+  ptrdiff_t __nout = -1;
+  ptrdiff_t __eout = -1;
+  if (__rhs.pbase() != nullptr) {
+    __bout = __rhs.pbase() - __p;
+    __nout = __rhs.pptr() - __p;
+    __eout = __rhs.epptr() - __p;
+  }
+  ptrdiff_t __hm = __rhs.__hm_ == nullptr ? -1 : __rhs.__hm_ - __p;
+  __str_         = std::move(__rhs.__str_);
+  __p            = const_cast<char_type*>(__str_.data());
+  if (__binp != -1)
+    this->setg(__p + __binp, __p + __ninp, __p + __einp);
+  else
+    this->setg(nullptr, nullptr, nullptr);
+  if (__bout != -1) {
+    this->setp(__p + __bout, __p + __eout);
+    this->__pbump(__nout);
+  } else
+    this->setp(nullptr, nullptr);
+
+  __hm_   = __hm == -1 ? nullptr : __p + __hm;
+  __mode_ = __rhs.__mode_;
+  __p     = const_cast<char_type*>(__rhs.__str_.data());
+  __rhs.setg(__p, __p, __p);
+  __rhs.setp(__p, __p);
+  __rhs.__hm_ = __p;
+  this->pubimbue(__rhs.getloc());
+  return *this;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+void basic_stringbuf<_CharT, _Traits, _Allocator>::swap(basic_stringbuf& __rhs)
+#if _LIBCPP_STD_VER >= 20
+    noexcept(allocator_traits<_Allocator>::propagate_on_container_swap::value ||
+             allocator_traits<_Allocator>::is_always_equal::value)
+#endif
+{
+  char_type* __p    = const_cast<char_type*>(__rhs.__str_.data());
+  ptrdiff_t __rbinp = -1;
+  ptrdiff_t __rninp = -1;
+  ptrdiff_t __reinp = -1;
+  if (__rhs.eback() != nullptr) {
+    __rbinp = __rhs.eback() - __p;
+    __rninp = __rhs.gptr() - __p;
+    __reinp = __rhs.egptr() - __p;
+  }
+  ptrdiff_t __rbout = -1;
+  ptrdiff_t __rnout = -1;
+  ptrdiff_t __reout = -1;
+  if (__rhs.pbase() != nullptr) {
+    __rbout = __rhs.pbase() - __p;
+    __rnout = __rhs.pptr() - __p;
+    __reout = __rhs.epptr() - __p;
+  }
+  ptrdiff_t __rhm   = __rhs.__hm_ == nullptr ? -1 : __rhs.__hm_ - __p;
+  __p               = const_cast<char_type*>(__str_.data());
+  ptrdiff_t __lbinp = -1;
+  ptrdiff_t __lninp = -1;
+  ptrdiff_t __leinp = -1;
+  if (this->eback() != nullptr) {
+    __lbinp = this->eback() - __p;
+    __lninp = this->gptr() - __p;
+    __leinp = this->egptr() - __p;
+  }
+  ptrdiff_t __lbout = -1;
+  ptrdiff_t __lnout = -1;
+  ptrdiff_t __leout = -1;
+  if (this->pbase() != nullptr) {
+    __lbout = this->pbase() - __p;
+    __lnout = this->pptr() - __p;
+    __leout = this->epptr() - __p;
+  }
+  ptrdiff_t __lhm = __hm_ == nullptr ? -1 : __hm_ - __p;
+  std::swap(__mode_, __rhs.__mode_);
+  __str_.swap(__rhs.__str_);
+  __p = const_cast<char_type*>(__str_.data());
+  if (__rbinp != -1)
+    this->setg(__p + __rbinp, __p + __rninp, __p + __reinp);
+  else
+    this->setg(nullptr, nullptr, nullptr);
+  if (__rbout != -1) {
+    this->setp(__p + __rbout, __p + __reout);
+    this->__pbump(__rnout);
+  } else
+    this->setp(nullptr, nullptr);
+  __hm_ = __rhm == -1 ? nullptr : __p + __rhm;
+  __p   = const_cast<char_type*>(__rhs.__str_.data());
+  if (__lbinp != -1)
+    __rhs.setg(__p + __lbinp, __p + __lninp, __p + __leinp);
+  else
+    __rhs.setg(nullptr, nullptr, nullptr);
+  if (__lbout != -1) {
+    __rhs.setp(__p + __lbout, __p + __leout);
+    __rhs.__pbump(__lnout);
+  } else
+    __rhs.setp(nullptr, nullptr);
+  __rhs.__hm_ = __lhm == -1 ? nullptr : __p + __lhm;
+  locale __tl = __rhs.getloc();
+  __rhs.pubimbue(this->getloc());
+  this->pubimbue(__tl);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(basic_stringbuf<_CharT, _Traits, _Allocator>& __x, basic_stringbuf<_CharT, _Traits, _Allocator>& __y)
+#if _LIBCPP_STD_VER >= 20
+    noexcept(noexcept(__x.swap(__y)))
+#endif
+{
+  __x.swap(__y);
+}
+
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
+template <class _CharT, class _Traits, class _Allocator>
+basic_string<_CharT, _Traits, _Allocator> basic_stringbuf<_CharT, _Traits, _Allocator>::str() const {
+  if (__mode_ & ios_base::out) {
+    if (__hm_ < this->pptr())
+      __hm_ = this->pptr();
+    return string_type(this->pbase(), __hm_, __str_.get_allocator());
+  } else if (__mode_ & ios_base::in)
+    return string_type(this->eback(), this->egptr(), __str_.get_allocator());
+  return string_type(__str_.get_allocator());
+}
+#endif // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI void basic_stringbuf<_CharT, _Traits, _Allocator>::__init_buf_ptrs() {
+  __hm_                                = nullptr;
+  char_type* __data                    = const_cast<char_type*>(__str_.data());
+  typename string_type::size_type __sz = __str_.size();
+  if (__mode_ & ios_base::in) {
+    __hm_ = __data + __sz;
+    this->setg(__data, __data, __hm_);
+  }
+  if (__mode_ & ios_base::out) {
+    __hm_ = __data + __sz;
+    __str_.resize(__str_.capacity());
+    this->setp(__data, __data + __str_.size());
+    if (__mode_ & (ios_base::app | ios_base::ate)) {
+      while (__sz > INT_MAX) {
+        this->pbump(INT_MAX);
+        __sz -= INT_MAX;
+      }
+      if (__sz > 0)
+        this->pbump(__sz);
+    }
+  }
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI basic_string_view<_CharT, _Traits>
+basic_stringbuf<_CharT, _Traits, _Allocator>::view() const noexcept {
+  if (__mode_ & ios_base::out) {
+    if (__hm_ < this->pptr())
+      __hm_ = this->pptr();
+    return basic_string_view<_CharT, _Traits>(this->pbase(), __hm_);
+  } else if (__mode_ & ios_base::in)
+    return basic_string_view<_CharT, _Traits>(this->eback(), this->egptr());
+  return basic_string_view<_CharT, _Traits>();
+}
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _CharT, class _Traits, class _Allocator>
+typename basic_stringbuf<_CharT, _Traits, _Allocator>::int_type
+basic_stringbuf<_CharT, _Traits, _Allocator>::underflow() {
+  if (__hm_ < this->pptr())
+    __hm_ = this->pptr();
+  if (__mode_ & ios_base::in) {
+    if (this->egptr() < __hm_)
+      this->setg(this->eback(), this->gptr(), __hm_);
+    if (this->gptr() < this->egptr())
+      return traits_type::to_int_type(*this->gptr());
+  }
+  return traits_type::eof();
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+typename basic_stringbuf<_CharT, _Traits, _Allocator>::int_type
+basic_stringbuf<_CharT, _Traits, _Allocator>::pbackfail(int_type __c) {
+  if (__hm_ < this->pptr())
+    __hm_ = this->pptr();
+  if (this->eback() < this->gptr()) {
+    if (traits_type::eq_int_type(__c, traits_type::eof())) {
+      this->setg(this->eback(), this->gptr() - 1, __hm_);
+      return traits_type::not_eof(__c);
+    }
+    if ((__mode_ & ios_base::out) || traits_type::eq(traits_type::to_char_type(__c), this->gptr()[-1])) {
+      this->setg(this->eback(), this->gptr() - 1, __hm_);
+      *this->gptr() = traits_type::to_char_type(__c);
+      return __c;
+    }
+  }
+  return traits_type::eof();
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+typename basic_stringbuf<_CharT, _Traits, _Allocator>::int_type
+basic_stringbuf<_CharT, _Traits, _Allocator>::overflow(int_type __c) {
+  if (!traits_type::eq_int_type(__c, traits_type::eof())) {
+    ptrdiff_t __ninp = this->gptr() - this->eback();
+    if (this->pptr() == this->epptr()) {
+      if (!(__mode_ & ios_base::out))
+        return traits_type::eof();
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+        ptrdiff_t __nout = this->pptr() - this->pbase();
+        ptrdiff_t __hm   = __hm_ - this->pbase();
+        __str_.push_back(char_type());
+        __str_.resize(__str_.capacity());
+        char_type* __p = const_cast<char_type*>(__str_.data());
+        this->setp(__p, __p + __str_.size());
+        this->__pbump(__nout);
+        __hm_ = this->pbase() + __hm;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      } catch (...) {
+        return traits_type::eof();
+      }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    }
+    __hm_ = std::max(this->pptr() + 1, __hm_);
+    if (__mode_ & ios_base::in) {
+      char_type* __p = const_cast<char_type*>(__str_.data());
+      this->setg(__p, __p + __ninp, __hm_);
+    }
+    return this->sputc(traits_type::to_char_type(__c));
+  }
+  return traits_type::not_eof(__c);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+typename basic_stringbuf<_CharT, _Traits, _Allocator>::pos_type basic_stringbuf<_CharT, _Traits, _Allocator>::seekoff(
+    off_type __off, ios_base::seekdir __way, ios_base::openmode __wch) {
+  if (__hm_ < this->pptr())
+    __hm_ = this->pptr();
+  if ((__wch & (ios_base::in | ios_base::out)) == 0)
+    return pos_type(-1);
+  if ((__wch & (ios_base::in | ios_base::out)) == (ios_base::in | ios_base::out) && __way == ios_base::cur)
+    return pos_type(-1);
+  const ptrdiff_t __hm = __hm_ == nullptr ? 0 : __hm_ - __str_.data();
+  off_type __noff;
+  switch (__way) {
+  case ios_base::beg:
+    __noff = 0;
+    break;
+  case ios_base::cur:
+    if (__wch & ios_base::in)
+      __noff = this->gptr() - this->eback();
+    else
+      __noff = this->pptr() - this->pbase();
+    break;
+  case ios_base::end:
+    __noff = __hm;
+    break;
+  default:
+    return pos_type(-1);
+  }
+  __noff += __off;
+  if (__noff < 0 || __hm < __noff)
+    return pos_type(-1);
+  if (__noff != 0) {
+    if ((__wch & ios_base::in) && this->gptr() == nullptr)
+      return pos_type(-1);
+    if ((__wch & ios_base::out) && this->pptr() == nullptr)
+      return pos_type(-1);
+  }
+  if (__wch & ios_base::in)
+    this->setg(this->eback(), this->eback() + __noff, __hm_);
+  if (__wch & ios_base::out) {
+    this->setp(this->pbase(), this->epptr());
+    this->__pbump(__noff);
+  }
+  return pos_type(__noff);
+}
+
+// Class template basic_istringstream [istringstream]
+
+template <class _CharT, class _Traits, class _Allocator>
+class _LIBCPP_TEMPLATE_VIS basic_istringstream : public basic_istream<_CharT, _Traits> {
+public:
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+  typedef _Allocator allocator_type;
+
+  typedef basic_string<char_type, traits_type, allocator_type> string_type;
+
+private:
+  basic_stringbuf<char_type, traits_type, allocator_type> __sb_;
+
+public:
+  // [istringstream.cons] Constructors:
+  _LIBCPP_HIDE_FROM_ABI basic_istringstream() : basic_istream<_CharT, _Traits>(&__sb_), __sb_(ios_base::in) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_istringstream(ios_base::openmode __wch)
+      : basic_istream<_CharT, _Traits>(&__sb_), __sb_(__wch | ios_base::in) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_istringstream(const string_type& __s, ios_base::openmode __wch = ios_base::in)
+      : basic_istream<_CharT, _Traits>(&__sb_), __sb_(__s, __wch | ios_base::in) {}
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI basic_istringstream(ios_base::openmode __wch, const _Allocator& __a)
+      : basic_istream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__wch | ios_base::in, __a) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_istringstream(string_type&& __s, ios_base::openmode __wch = ios_base::in)
+      : basic_istream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(std::move(__s), __wch | ios_base::in) {}
+
+  template <class _SAlloc>
+  _LIBCPP_HIDE_FROM_ABI basic_istringstream(const basic_string<_CharT, _Traits, _SAlloc>& __s, const _Allocator& __a)
+      : basic_istringstream(__s, ios_base::in, __a) {}
+
+  template <class _SAlloc>
+  _LIBCPP_HIDE_FROM_ABI basic_istringstream(
+      const basic_string<_CharT, _Traits, _SAlloc>& __s, ios_base::openmode __wch, const _Allocator& __a)
+      : basic_istream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch | ios_base::in, __a) {}
+
+  template <class _SAlloc>
+  _LIBCPP_HIDE_FROM_ABI explicit basic_istringstream(const basic_string<_CharT, _Traits, _SAlloc>& __s,
+                                                     ios_base::openmode __wch = ios_base::in)
+      : basic_istream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch | ios_base::in) {}
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 26
+
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI explicit basic_istringstream(const _Tp& __t, ios_base::openmode __which = ios_base::in)
+      : basic_istringstream(__t, __which, _Allocator()) {}
+
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI basic_istringstream(const _Tp& __t, const _Allocator& __a)
+      : basic_istringstream(__t, ios_base::in, __a) {}
+
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI basic_istringstream(const _Tp& __t, ios_base::openmode __which, const _Allocator& __a)
+      : basic_istream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__t, __which | ios_base::in, __a) {}
+
+#endif //  _LIBCPP_STD_VER >= 26
+
+  basic_istringstream(const basic_istringstream&) = delete;
+  _LIBCPP_HIDE_FROM_ABI basic_istringstream(basic_istringstream&& __rhs)
+      : basic_istream<_CharT, _Traits>(std::move(__rhs)), __sb_(std::move(__rhs.__sb_)) {
+    basic_istream<_CharT, _Traits>::set_rdbuf(&__sb_);
+  }
+
+  // [istringstream.assign] Assign and swap:
+  basic_istringstream& operator=(const basic_istringstream&) = delete;
+  basic_istringstream& operator=(basic_istringstream&& __rhs) {
+    basic_istream<char_type, traits_type>::operator=(std::move(__rhs));
+    __sb_ = std::move(__rhs.__sb_);
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI void swap(basic_istringstream& __rhs) {
+    basic_istream<char_type, traits_type>::swap(__rhs);
+    __sb_.swap(__rhs.__sb_);
+  }
+
+  // [istringstream.members] Member functions:
+  _LIBCPP_HIDE_FROM_ABI basic_stringbuf<char_type, traits_type, allocator_type>* rdbuf() const {
+    return const_cast<basic_stringbuf<char_type, traits_type, allocator_type>*>(&__sb_);
+  }
+
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
+  _LIBCPP_HIDE_FROM_ABI string_type str() const { return __sb_.str(); }
+#else
+  _LIBCPP_HIDE_FROM_ABI string_type str() const& { return __sb_.str(); }
+
+  _LIBCPP_HIDE_FROM_ABI string_type str() && { return std::move(__sb_).str(); }
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+  template <class _SAlloc>
+    requires __is_allocator<_SAlloc>::value
+  _LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
+    return __sb_.str(__sa);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept { return __sb_.view(); }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI void str(const string_type& __s) { __sb_.str(__s); }
+
+#if _LIBCPP_STD_VER >= 20
+  template <class _SAlloc>
+  _LIBCPP_HIDE_FROM_ABI void str(const basic_string<char_type, traits_type, _SAlloc>& __s) {
+    __sb_.str(__s);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void str(string_type&& __s) { __sb_.str(std::move(__s)); }
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 26
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI void str(const _Tp& __t) {
+    rdbuf()->str(__t);
+  }
+#endif //  _LIBCPP_STD_VER >= 26
+};
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(basic_istringstream<_CharT, _Traits, _Allocator>& __x, basic_istringstream<_CharT, _Traits, _Allocator>& __y) {
+  __x.swap(__y);
+}
+
+// Class template basic_ostringstream [ostringstream]
+
+template <class _CharT, class _Traits, class _Allocator>
+class _LIBCPP_TEMPLATE_VIS basic_ostringstream : public basic_ostream<_CharT, _Traits> {
+public:
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+  typedef _Allocator allocator_type;
+
+  typedef basic_string<char_type, traits_type, allocator_type> string_type;
+
+private:
+  basic_stringbuf<char_type, traits_type, allocator_type> __sb_;
+
+public:
+  // [ostringstream.cons] Constructors:
+  _LIBCPP_HIDE_FROM_ABI basic_ostringstream() : basic_ostream<_CharT, _Traits>(&__sb_), __sb_(ios_base::out) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_ostringstream(ios_base::openmode __wch)
+      : basic_ostream<_CharT, _Traits>(&__sb_), __sb_(__wch | ios_base::out) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_ostringstream(const string_type& __s, ios_base::openmode __wch = ios_base::out)
+      : basic_ostream<_CharT, _Traits>(&__sb_), __sb_(__s, __wch | ios_base::out) {}
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI basic_ostringstream(ios_base::openmode __wch, const _Allocator& __a)
+      : basic_ostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__wch | ios_base::out, __a) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_ostringstream(string_type&& __s, ios_base::openmode __wch = ios_base::out)
+      : basic_ostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(std::move(__s), __wch | ios_base::out) {}
+
+  template <class _SAlloc>
+  _LIBCPP_HIDE_FROM_ABI basic_ostringstream(const basic_string<_CharT, _Traits, _SAlloc>& __s, const _Allocator& __a)
+      : basic_ostringstream(__s, ios_base::out, __a) {}
+
+  template <class _SAlloc>
+  _LIBCPP_HIDE_FROM_ABI basic_ostringstream(
+      const basic_string<_CharT, _Traits, _SAlloc>& __s, ios_base::openmode __wch, const _Allocator& __a)
+      : basic_ostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch | ios_base::out, __a) {}
+
+  template <class _SAlloc>
+    requires(!is_same_v<_SAlloc, allocator_type>)
+  _LIBCPP_HIDE_FROM_ABI explicit basic_ostringstream(const basic_string<_CharT, _Traits, _SAlloc>& __s,
+                                                     ios_base::openmode __wch = ios_base::out)
+      : basic_ostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch | ios_base::out) {}
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 26
+
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI explicit basic_ostringstream(const _Tp& __t, ios_base::openmode __which = ios_base::out)
+      : basic_ostringstream(__t, __which | ios_base::out, _Allocator()) {}
+
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI basic_ostringstream(const _Tp& __t, const _Allocator& __a)
+      : basic_ostringstream(__t, ios_base::out, __a) {}
+
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI basic_ostringstream(const _Tp& __t, ios_base::openmode __which, const _Allocator& __a)
+      : basic_ostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__t, __which | ios_base::out, __a) {}
+
+#endif //  _LIBCPP_STD_VER >= 26
+
+  basic_ostringstream(const basic_ostringstream&) = delete;
+  _LIBCPP_HIDE_FROM_ABI basic_ostringstream(basic_ostringstream&& __rhs)
+      : basic_ostream<_CharT, _Traits>(std::move(__rhs)), __sb_(std::move(__rhs.__sb_)) {
+    basic_ostream<_CharT, _Traits>::set_rdbuf(&__sb_);
+  }
+
+  // [ostringstream.assign] Assign and swap:
+  basic_ostringstream& operator=(const basic_ostringstream&) = delete;
+  basic_ostringstream& operator=(basic_ostringstream&& __rhs) {
+    basic_ostream<char_type, traits_type>::operator=(std::move(__rhs));
+    __sb_ = std::move(__rhs.__sb_);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(basic_ostringstream& __rhs) {
+    basic_ostream<char_type, traits_type>::swap(__rhs);
+    __sb_.swap(__rhs.__sb_);
+  }
+
+  // [ostringstream.members] Member functions:
+  _LIBCPP_HIDE_FROM_ABI basic_stringbuf<char_type, traits_type, allocator_type>* rdbuf() const {
+    return const_cast<basic_stringbuf<char_type, traits_type, allocator_type>*>(&__sb_);
+  }
+
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
+  _LIBCPP_HIDE_FROM_ABI string_type str() const { return __sb_.str(); }
+#else
+  _LIBCPP_HIDE_FROM_ABI string_type str() const& { return __sb_.str(); }
+
+  _LIBCPP_HIDE_FROM_ABI string_type str() && { return std::move(__sb_).str(); }
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+  template <class _SAlloc>
+    requires __is_allocator<_SAlloc>::value
+  _LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
+    return __sb_.str(__sa);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept { return __sb_.view(); }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI void str(const string_type& __s) { __sb_.str(__s); }
+
+#if _LIBCPP_STD_VER >= 20
+  template <class _SAlloc>
+  _LIBCPP_HIDE_FROM_ABI void str(const basic_string<char_type, traits_type, _SAlloc>& __s) {
+    __sb_.str(__s);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void str(string_type&& __s) { __sb_.str(std::move(__s)); }
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 26
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI void str(const _Tp& __t) {
+    rdbuf()->str(__t);
+  }
+#endif //  _LIBCPP_STD_VER >= 26
+};
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(basic_ostringstream<_CharT, _Traits, _Allocator>& __x, basic_ostringstream<_CharT, _Traits, _Allocator>& __y) {
+  __x.swap(__y);
+}
+
+// Class template basic_stringstream [stringstream]
+
+template <class _CharT, class _Traits, class _Allocator>
+class _LIBCPP_TEMPLATE_VIS basic_stringstream : public basic_iostream<_CharT, _Traits> {
+public:
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+  typedef _Allocator allocator_type;
+
+  typedef basic_string<char_type, traits_type, allocator_type> string_type;
+
+private:
+  basic_stringbuf<char_type, traits_type, allocator_type> __sb_;
+
+public:
+  // [stringstream.cons] constructors
+  _LIBCPP_HIDE_FROM_ABI basic_stringstream()
+      : basic_iostream<_CharT, _Traits>(&__sb_), __sb_(ios_base::in | ios_base::out) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_stringstream(ios_base::openmode __wch)
+      : basic_iostream<_CharT, _Traits>(&__sb_), __sb_(__wch) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_stringstream(const string_type& __s,
+                                                    ios_base::openmode __wch = ios_base::in | ios_base::out)
+      : basic_iostream<_CharT, _Traits>(&__sb_), __sb_(__s, __wch) {}
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI basic_stringstream(ios_base::openmode __wch, const _Allocator& __a)
+      : basic_iostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__wch, __a) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_stringstream(string_type&& __s,
+                                                    ios_base::openmode __wch = ios_base::out | ios_base::in)
+      : basic_iostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(std::move(__s), __wch) {}
+
+  template <class _SAlloc>
+  _LIBCPP_HIDE_FROM_ABI basic_stringstream(const basic_string<_CharT, _Traits, _SAlloc>& __s, const _Allocator& __a)
+      : basic_stringstream(__s, ios_base::out | ios_base::in, __a) {}
+
+  template <class _SAlloc>
+  _LIBCPP_HIDE_FROM_ABI
+  basic_stringstream(const basic_string<_CharT, _Traits, _SAlloc>& __s, ios_base::openmode __wch, const _Allocator& __a)
+      : basic_iostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch, __a) {}
+
+  template <class _SAlloc>
+    requires(!is_same_v<_SAlloc, allocator_type>)
+  _LIBCPP_HIDE_FROM_ABI explicit basic_stringstream(const basic_string<_CharT, _Traits, _SAlloc>& __s,
+                                                    ios_base::openmode __wch = ios_base::out | ios_base::in)
+      : basic_iostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch) {}
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 26
+
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI explicit basic_stringstream(const _Tp& __t,
+                                                    ios_base::openmode __which = ios_base::out | ios_base::in)
+      : basic_stringstream(__t, __which, _Allocator()) {}
+
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI basic_stringstream(const _Tp& __t, const _Allocator& __a)
+      : basic_stringstream(__t, ios_base::out | ios_base::in, __a) {}
+
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI basic_stringstream(const _Tp& __t, ios_base::openmode __which, const _Allocator& __a)
+      : basic_iostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__t, __which, __a) {}
+
+#endif //  _LIBCPP_STD_VER >= 26
+
+  basic_stringstream(const basic_stringstream&) = delete;
+  _LIBCPP_HIDE_FROM_ABI basic_stringstream(basic_stringstream&& __rhs)
+      : basic_iostream<_CharT, _Traits>(std::move(__rhs)), __sb_(std::move(__rhs.__sb_)) {
+    basic_istream<_CharT, _Traits>::set_rdbuf(&__sb_);
+  }
+
+  // [stringstream.assign] Assign and swap:
+  basic_stringstream& operator=(const basic_stringstream&) = delete;
+  basic_stringstream& operator=(basic_stringstream&& __rhs) {
+    basic_iostream<char_type, traits_type>::operator=(std::move(__rhs));
+    __sb_ = std::move(__rhs.__sb_);
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI void swap(basic_stringstream& __rhs) {
+    basic_iostream<char_type, traits_type>::swap(__rhs);
+    __sb_.swap(__rhs.__sb_);
+  }
+
+  // [stringstream.members] Member functions:
+  _LIBCPP_HIDE_FROM_ABI basic_stringbuf<char_type, traits_type, allocator_type>* rdbuf() const {
+    return const_cast<basic_stringbuf<char_type, traits_type, allocator_type>*>(&__sb_);
+  }
+
+#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
+  _LIBCPP_HIDE_FROM_ABI string_type str() const { return __sb_.str(); }
+#else
+  _LIBCPP_HIDE_FROM_ABI string_type str() const& { return __sb_.str(); }
+
+  _LIBCPP_HIDE_FROM_ABI string_type str() && { return std::move(__sb_).str(); }
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+  template <class _SAlloc>
+    requires __is_allocator<_SAlloc>::value
+  _LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
+    return __sb_.str(__sa);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept { return __sb_.view(); }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI void str(const string_type& __s) { __sb_.str(__s); }
+
+#if _LIBCPP_STD_VER >= 20
+  template <class _SAlloc>
+  _LIBCPP_HIDE_FROM_ABI void str(const basic_string<char_type, traits_type, _SAlloc>& __s) {
+    __sb_.str(__s);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void str(string_type&& __s) { __sb_.str(std::move(__s)); }
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 26
+  template <class _Tp>
+    requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
+  _LIBCPP_HIDE_FROM_ABI void str(const _Tp& __t) {
+    rdbuf()->str(__t);
+  }
+#endif //  _LIBCPP_STD_VER >= 26
+};
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(basic_stringstream<_CharT, _Traits, _Allocator>& __x, basic_stringstream<_CharT, _Traits, _Allocator>& __y) {
+  __x.swap(__y);
+}
+
+#if _LIBCPP_AVAILABILITY_HAS_ADDITIONAL_IOSTREAM_EXPLICIT_INSTANTIATIONS_1
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_stringbuf<char>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_stringstream<char>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_ostringstream<char>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_istringstream<char>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if _LIBCPP_STD_VER <= 20 && !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES)
+#  include <ostream>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_SSTREAM
diff --git a/libcxx/include/__cxx03/stack b/libcxx/include/__cxx03/stack
new file mode 100644
index 000000000000..90f8933cca30
--- /dev/null
+++ b/libcxx/include/__cxx03/stack
@@ -0,0 +1,379 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_STACK
+#define _LIBCPP_STACK
+
+/*
+    stack synopsis
+
+namespace std
+{
+
+template <class T, class Container = deque<T>>
+class stack
+{
+public:
+    typedef Container                                container_type;
+    typedef typename container_type::value_type      value_type;
+    typedef typename container_type::reference       reference;
+    typedef typename container_type::const_reference const_reference;
+    typedef typename container_type::size_type       size_type;
+
+protected:
+    container_type c;
+
+public:
+    stack() = default;
+    ~stack() = default;
+
+    stack(const stack& q) = default;
+    stack(stack&& q) = default;
+
+    stack& operator=(const stack& q) = default;
+    stack& operator=(stack&& q) = default;
+
+    explicit stack(const container_type& c);
+    explicit stack(container_type&& c);
+    template <class InputIterator> stack(InputIterator first, InputIterator last); // since C++23
+    template<container-compatible-range<T> R> stack(from_range_t, R&& rg); // since C++23
+    template <class Alloc> explicit stack(const Alloc& a);
+    template <class Alloc> stack(const container_type& c, const Alloc& a);
+    template <class Alloc> stack(container_type&& c, const Alloc& a);
+    template <class Alloc> stack(const stack& c, const Alloc& a);
+    template <class Alloc> stack(stack&& c, const Alloc& a);
+    template<class InputIterator, class Alloc>
+    stack(InputIterator first, InputIterator last, const Alloc&); // since C++23
+    template<container-compatible-range<T> R, class Alloc>
+      stack(from_range_t, R&& rg, const Alloc&); // since C++23
+
+    bool empty() const;
+    size_type size() const;
+    reference top();
+    const_reference top() const;
+
+    void push(const value_type& x);
+    void push(value_type&& x);
+    template<container-compatible-range<T> R>
+      void push_range(R&& rg); // C++23
+    template <class... Args> reference emplace(Args&&... args); // reference in C++17
+    void pop();
+
+    void swap(stack& c) noexcept(is_nothrow_swappable_v<Container>)
+};
+
+template<class Container>
+  stack(Container) -> stack<typename Container::value_type, Container>;  // C++17
+
+template<class InputIterator>
+  stack(InputIterator, InputIterator) -> stack<iter-value-type<InputIterator>>; // since C++23
+
+template<ranges::input_range R>
+  stack(from_range_t, R&&) -> stack<ranges::range_value_t<R>>; // since C++23
+
+template<class Container, class Allocator>
+  stack(Container, Allocator) -> stack<typename Container::value_type, Container>; // C++17
+
+template<class InputIterator, class Allocator>
+  stack(InputIterator, InputIterator, Allocator)
+    -> stack<iter-value-type<InputIterator>,
+             deque<iter-value-type<InputIterator>, Allocator>>; // since C++23
+
+template<ranges::input_range R, class Allocator>
+  stack(from_range_t, R&&, Allocator)
+    -> stack<ranges::range_value_t<R>, deque<ranges::range_value_t<R>, Allocator>>; // since C++23
+
+template <class T, class Container>
+  bool operator==(const stack<T, Container>& x, const stack<T, Container>& y);
+template <class T, class Container>
+  bool operator< (const stack<T, Container>& x, const stack<T, Container>& y);
+template <class T, class Container>
+  bool operator!=(const stack<T, Container>& x, const stack<T, Container>& y);
+template <class T, class Container>
+  bool operator> (const stack<T, Container>& x, const stack<T, Container>& y);
+template <class T, class Container>
+  bool operator>=(const stack<T, Container>& x, const stack<T, Container>& y);
+template <class T, class Container>
+  bool operator<=(const stack<T, Container>& x, const stack<T, Container>& y);
+template<class T, three_way_comparable Container>
+  compare_three_way_result_t<Container>
+    operator<=>(const stack<T, Container>& x, const stack<T, Container>& y); // since C++20
+
+template <class T, class Container>
+  void swap(stack<T, Container>& x, stack<T, Container>& y)
+  noexcept(noexcept(x.swap(y)));
+
+}  // std
+
+*/
+
+#include <__algorithm/ranges_copy.h>
+#include <__config>
+#include <__fwd/stack.h>
+#include <__iterator/back_insert_iterator.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/uses_allocator.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/container_compatible_range.h>
+#include <__ranges/from_range.h>
+#include <__type_traits/is_same.h>
+#include <__utility/forward.h>
+#include <deque>
+#include <version>
+
+// standard-mandated includes
+
+// [stack.syn]
+#include <compare>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Container>
+_LIBCPP_HIDE_FROM_ABI bool operator==(const stack<_Tp, _Container>& __x, const stack<_Tp, _Container>& __y);
+
+template <class _Tp, class _Container>
+_LIBCPP_HIDE_FROM_ABI bool operator<(const stack<_Tp, _Container>& __x, const stack<_Tp, _Container>& __y);
+
+template <class _Tp, class _Container /*= deque<_Tp>*/>
+class _LIBCPP_TEMPLATE_VIS stack {
+public:
+  typedef _Container container_type;
+  typedef typename container_type::value_type value_type;
+  typedef typename container_type::reference reference;
+  typedef typename container_type::const_reference const_reference;
+  typedef typename container_type::size_type size_type;
+  static_assert(is_same<_Tp, value_type>::value, "");
+
+protected:
+  container_type c;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI stack() _NOEXCEPT_(is_nothrow_default_constructible<container_type>::value) : c() {}
+
+  _LIBCPP_HIDE_FROM_ABI stack(const stack& __q) : c(__q.c) {}
+
+  _LIBCPP_HIDE_FROM_ABI stack& operator=(const stack& __q) {
+    c = __q.c;
+    return *this;
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI stack(stack&& __q) noexcept(is_nothrow_move_constructible<container_type>::value)
+      : c(std::move(__q.c)) {}
+
+  _LIBCPP_HIDE_FROM_ABI stack& operator=(stack&& __q) noexcept(is_nothrow_move_assignable<container_type>::value) {
+    c = std::move(__q.c);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI explicit stack(container_type&& __c) : c(std::move(__c)) {}
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI explicit stack(const container_type& __c) : c(__c) {}
+
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI explicit stack(const _Alloc& __a,
+                                       __enable_if_t<uses_allocator<container_type, _Alloc>::value>* = 0)
+      : c(__a) {}
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI
+  stack(const container_type& __c, const _Alloc& __a, __enable_if_t<uses_allocator<container_type, _Alloc>::value>* = 0)
+      : c(__c, __a) {}
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI
+  stack(const stack& __s, const _Alloc& __a, __enable_if_t<uses_allocator<container_type, _Alloc>::value>* = 0)
+      : c(__s.c, __a) {}
+#ifndef _LIBCPP_CXX03_LANG
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI
+  stack(container_type&& __c, const _Alloc& __a, __enable_if_t<uses_allocator<container_type, _Alloc>::value>* = 0)
+      : c(std::move(__c), __a) {}
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI
+  stack(stack&& __s, const _Alloc& __a, __enable_if_t<uses_allocator<container_type, _Alloc>::value>* = 0)
+      : c(std::move(__s.c), __a) {}
+#endif // _LIBCPP_CXX03_LANG
+
+#if _LIBCPP_STD_VER >= 23
+  template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI stack(_InputIterator __first, _InputIterator __last) : c(__first, __last) {}
+
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI stack(from_range_t, _Range&& __range) : c(from_range, std::forward<_Range>(__range)) {}
+
+  template <class _InputIterator,
+            class _Alloc,
+            __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0,
+            __enable_if_t<uses_allocator<container_type, _Alloc>::value, int>        = 0>
+  _LIBCPP_HIDE_FROM_ABI stack(_InputIterator __first, _InputIterator __last, const _Alloc& __alloc)
+      : c(__first, __last, __alloc) {}
+
+  template <_ContainerCompatibleRange<_Tp> _Range,
+            class _Alloc,
+            __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI stack(from_range_t, _Range&& __range, const _Alloc& __alloc)
+      : c(from_range, std::forward<_Range>(__range), __alloc) {}
+
+#endif
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const { return c.empty(); }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const { return c.size(); }
+  _LIBCPP_HIDE_FROM_ABI reference top() { return c.back(); }
+  _LIBCPP_HIDE_FROM_ABI const_reference top() const { return c.back(); }
+
+  _LIBCPP_HIDE_FROM_ABI void push(const value_type& __v) { c.push_back(__v); }
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI void push(value_type&& __v) { c.push_back(std::move(__v)); }
+
+#  if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI void push_range(_Range&& __range) {
+    if constexpr (requires(container_type& __c) { __c.append_range(std::forward<_Range>(__range)); }) {
+      c.append_range(std::forward<_Range>(__range));
+    } else {
+      ranges::copy(std::forward<_Range>(__range), std::back_inserter(c));
+    }
+  }
+#  endif
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI
+#  if _LIBCPP_STD_VER >= 17
+  decltype(auto)
+  emplace(_Args&&... __args) {
+    return c.emplace_back(std::forward<_Args>(__args)...);
+  }
+#  else
+  void
+  emplace(_Args&&... __args) {
+    c.emplace_back(std::forward<_Args>(__args)...);
+  }
+#  endif
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI void pop() { c.pop_back(); }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(stack& __s) _NOEXCEPT_(__is_nothrow_swappable_v<container_type>) {
+    using std::swap;
+    swap(c, __s.c);
+  }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI const _Container& __get_container() const { return c; }
+
+  template <class _T1, class _OtherContainer>
+  friend bool operator==(const stack<_T1, _OtherContainer>& __x, const stack<_T1, _OtherContainer>& __y);
+
+  template <class _T1, class _OtherContainer>
+  friend bool operator<(const stack<_T1, _OtherContainer>& __x, const stack<_T1, _OtherContainer>& __y);
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Container, class = enable_if_t<!__is_allocator<_Container>::value> >
+stack(_Container) -> stack<typename _Container::value_type, _Container>;
+
+template <class _Container,
+          class _Alloc,
+          class = enable_if_t<!__is_allocator<_Container>::value>,
+          class = enable_if_t<uses_allocator<_Container, _Alloc>::value> >
+stack(_Container, _Alloc) -> stack<typename _Container::value_type, _Container>;
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
+stack(_InputIterator, _InputIterator) -> stack<__iter_value_type<_InputIterator>>;
+
+template <ranges::input_range _Range>
+stack(from_range_t, _Range&&) -> stack<ranges::range_value_t<_Range>>;
+
+template <class _InputIterator,
+          class _Alloc,
+          __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0,
+          __enable_if_t<__is_allocator<_Alloc>::value, int>                        = 0>
+stack(_InputIterator,
+      _InputIterator,
+      _Alloc) -> stack<__iter_value_type<_InputIterator>, deque<__iter_value_type<_InputIterator>, _Alloc>>;
+
+template <ranges::input_range _Range, class _Alloc, __enable_if_t<__is_allocator<_Alloc>::value, int> = 0>
+stack(from_range_t,
+      _Range&&,
+      _Alloc) -> stack<ranges::range_value_t<_Range>, deque<ranges::range_value_t<_Range>, _Alloc>>;
+
+#endif
+
+template <class _Tp, class _Container>
+inline _LIBCPP_HIDE_FROM_ABI bool operator==(const stack<_Tp, _Container>& __x, const stack<_Tp, _Container>& __y) {
+  return __x.c == __y.c;
+}
+
+template <class _Tp, class _Container>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(const stack<_Tp, _Container>& __x, const stack<_Tp, _Container>& __y) {
+  return __x.c < __y.c;
+}
+
+template <class _Tp, class _Container>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const stack<_Tp, _Container>& __x, const stack<_Tp, _Container>& __y) {
+  return !(__x == __y);
+}
+
+template <class _Tp, class _Container>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>(const stack<_Tp, _Container>& __x, const stack<_Tp, _Container>& __y) {
+  return __y < __x;
+}
+
+template <class _Tp, class _Container>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>=(const stack<_Tp, _Container>& __x, const stack<_Tp, _Container>& __y) {
+  return !(__x < __y);
+}
+
+template <class _Tp, class _Container>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const stack<_Tp, _Container>& __x, const stack<_Tp, _Container>& __y) {
+  return !(__y < __x);
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp, three_way_comparable _Container>
+_LIBCPP_HIDE_FROM_ABI compare_three_way_result_t<_Container>
+operator<=>(const stack<_Tp, _Container>& __x, const stack<_Tp, _Container>& __y) {
+  // clang 16 bug: declaring `friend operator<=>` causes "use of overloaded operator '*' is ambiguous" errors
+  return __x.__get_container() <=> __y.__get_container();
+}
+
+#endif
+
+template <class _Tp, class _Container, __enable_if_t<__is_swappable_v<_Container>, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI void swap(stack<_Tp, _Container>& __x, stack<_Tp, _Container>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+template <class _Tp, class _Container, class _Alloc>
+struct _LIBCPP_TEMPLATE_VIS uses_allocator<stack<_Tp, _Container>, _Alloc> : public uses_allocator<_Container, _Alloc> {
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <concepts>
+#  include <functional>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_STACK
diff --git a/libcxx/include/__cxx03/stdatomic.h b/libcxx/include/__cxx03/stdatomic.h
new file mode 100644
index 000000000000..79772eb7fce1
--- /dev/null
+++ b/libcxx/include/__cxx03/stdatomic.h
@@ -0,0 +1,235 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_STDATOMIC_H
+#define _LIBCPP_STDATOMIC_H
+
+/*
+    stdatomic.h synopsis
+
+template<class T>
+  using std-atomic = std::atomic<T>;        // exposition only
+
+#define _Atomic(T) std-atomic<T>
+
+#define ATOMIC_BOOL_LOCK_FREE see below
+#define ATOMIC_CHAR_LOCK_FREE see below
+#define ATOMIC_CHAR16_T_LOCK_FREE see below
+#define ATOMIC_CHAR32_T_LOCK_FREE see below
+#define ATOMIC_WCHAR_T_LOCK_FREE see below
+#define ATOMIC_SHORT_LOCK_FREE see below
+#define ATOMIC_INT_LOCK_FREE see below
+#define ATOMIC_LONG_LOCK_FREE see below
+#define ATOMIC_LLONG_LOCK_FREE see below
+#define ATOMIC_POINTER_LOCK_FREE see below
+
+using std::memory_order                // see below
+using std::memory_order_relaxed        // see below
+using std::memory_order_consume        // see below
+using std::memory_order_acquire        // see below
+using std::memory_order_release        // see below
+using std::memory_order_acq_rel        // see below
+using std::memory_order_seq_cst        // see below
+
+using std::atomic_flag                 // see below
+
+using std::atomic_bool                 // see below
+using std::atomic_char                 // see below
+using std::atomic_schar                // see below
+using std::atomic_uchar                // see below
+using std::atomic_short                // see below
+using std::atomic_ushort               // see below
+using std::atomic_int                  // see below
+using std::atomic_uint                 // see below
+using std::atomic_long                 // see below
+using std::atomic_ulong                // see below
+using std::atomic_llong                // see below
+using std::atomic_ullong               // see below
+using std::atomic_char8_t              // see below
+using std::atomic_char16_t             // see below
+using std::atomic_char32_t             // see below
+using std::atomic_wchar_t              // see below
+using std::atomic_int8_t               // see below
+using std::atomic_uint8_t              // see below
+using std::atomic_int16_t              // see below
+using std::atomic_uint16_t             // see below
+using std::atomic_int32_t              // see below
+using std::atomic_uint32_t             // see below
+using std::atomic_int64_t              // see below
+using std::atomic_uint64_t             // see below
+using std::atomic_int_least8_t         // see below
+using std::atomic_uint_least8_t        // see below
+using std::atomic_int_least16_t        // see below
+using std::atomic_uint_least16_t       // see below
+using std::atomic_int_least32_t        // see below
+using std::atomic_uint_least32_t       // see below
+using std::atomic_int_least64_t        // see below
+using std::atomic_uint_least64_t       // see below
+using std::atomic_int_fast8_t          // see below
+using std::atomic_uint_fast8_t         // see below
+using std::atomic_int_fast16_t         // see below
+using std::atomic_uint_fast16_t        // see below
+using std::atomic_int_fast32_t         // see below
+using std::atomic_uint_fast32_t        // see below
+using std::atomic_int_fast64_t         // see below
+using std::atomic_uint_fast64_t        // see below
+using std::atomic_intptr_t             // see below
+using std::atomic_uintptr_t            // see below
+using std::atomic_size_t               // see below
+using std::atomic_ptrdiff_t            // see below
+using std::atomic_intmax_t             // see below
+using std::atomic_uintmax_t            // see below
+
+using std::atomic_is_lock_free                         // see below
+using std::atomic_load                                 // see below
+using std::atomic_load_explicit                        // see below
+using std::atomic_store                                // see below
+using std::atomic_store_explicit                       // see below
+using std::atomic_exchange                             // see below
+using std::atomic_exchange_explicit                    // see below
+using std::atomic_compare_exchange_strong              // see below
+using std::atomic_compare_exchange_strong_explicit     // see below
+using std::atomic_compare_exchange_weak                // see below
+using std::atomic_compare_exchange_weak_explicit       // see below
+using std::atomic_fetch_add                            // see below
+using std::atomic_fetch_add_explicit                   // see below
+using std::atomic_fetch_sub                            // see below
+using std::atomic_fetch_sub_explicit                   // see below
+using std::atomic_fetch_or                             // see below
+using std::atomic_fetch_or_explicit                    // see below
+using std::atomic_fetch_and                            // see below
+using std::atomic_fetch_and_explicit                   // see below
+using std::atomic_flag_test_and_set                    // see below
+using std::atomic_flag_test_and_set_explicit           // see below
+using std::atomic_flag_clear                           // see below
+using std::atomic_flag_clear_explicit                  // see below
+
+using std::atomic_thread_fence                         // see below
+using std::atomic_signal_fence                         // see below
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if defined(__cplusplus) && _LIBCPP_STD_VER >= 23
+
+#  include <atomic>
+#  include <version>
+
+#  ifdef _Atomic
+#    undef _Atomic
+#  endif
+
+#  define _Atomic(_Tp) ::std::atomic<_Tp>
+
+using std::memory_order _LIBCPP_USING_IF_EXISTS;
+using std::memory_order_relaxed _LIBCPP_USING_IF_EXISTS;
+using std::memory_order_consume _LIBCPP_USING_IF_EXISTS;
+using std::memory_order_acquire _LIBCPP_USING_IF_EXISTS;
+using std::memory_order_release _LIBCPP_USING_IF_EXISTS;
+using std::memory_order_acq_rel _LIBCPP_USING_IF_EXISTS;
+using std::memory_order_seq_cst _LIBCPP_USING_IF_EXISTS;
+
+using std::atomic_flag _LIBCPP_USING_IF_EXISTS;
+
+using std::atomic_bool _LIBCPP_USING_IF_EXISTS;
+using std::atomic_char _LIBCPP_USING_IF_EXISTS;
+using std::atomic_schar _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uchar _LIBCPP_USING_IF_EXISTS;
+using std::atomic_short _LIBCPP_USING_IF_EXISTS;
+using std::atomic_ushort _LIBCPP_USING_IF_EXISTS;
+using std::atomic_int _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uint _LIBCPP_USING_IF_EXISTS;
+using std::atomic_long _LIBCPP_USING_IF_EXISTS;
+using std::atomic_ulong _LIBCPP_USING_IF_EXISTS;
+using std::atomic_llong _LIBCPP_USING_IF_EXISTS;
+using std::atomic_ullong _LIBCPP_USING_IF_EXISTS;
+using std::atomic_char8_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_char16_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_char32_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_wchar_t _LIBCPP_USING_IF_EXISTS;
+
+using std::atomic_int8_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uint8_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_int16_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uint16_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_int32_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uint32_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_int64_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uint64_t _LIBCPP_USING_IF_EXISTS;
+
+using std::atomic_int_least8_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uint_least8_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_int_least16_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uint_least16_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_int_least32_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uint_least32_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_int_least64_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uint_least64_t _LIBCPP_USING_IF_EXISTS;
+
+using std::atomic_int_fast8_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uint_fast8_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_int_fast16_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uint_fast16_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_int_fast32_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uint_fast32_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_int_fast64_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uint_fast64_t _LIBCPP_USING_IF_EXISTS;
+
+using std::atomic_intptr_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uintptr_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_size_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_ptrdiff_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_intmax_t _LIBCPP_USING_IF_EXISTS;
+using std::atomic_uintmax_t _LIBCPP_USING_IF_EXISTS;
+
+using std::atomic_compare_exchange_strong _LIBCPP_USING_IF_EXISTS;
+using std::atomic_compare_exchange_strong_explicit _LIBCPP_USING_IF_EXISTS;
+using std::atomic_compare_exchange_weak _LIBCPP_USING_IF_EXISTS;
+using std::atomic_compare_exchange_weak_explicit _LIBCPP_USING_IF_EXISTS;
+using std::atomic_exchange _LIBCPP_USING_IF_EXISTS;
+using std::atomic_exchange_explicit _LIBCPP_USING_IF_EXISTS;
+using std::atomic_fetch_add _LIBCPP_USING_IF_EXISTS;
+using std::atomic_fetch_add_explicit _LIBCPP_USING_IF_EXISTS;
+using std::atomic_fetch_and _LIBCPP_USING_IF_EXISTS;
+using std::atomic_fetch_and_explicit _LIBCPP_USING_IF_EXISTS;
+using std::atomic_fetch_or _LIBCPP_USING_IF_EXISTS;
+using std::atomic_fetch_or_explicit _LIBCPP_USING_IF_EXISTS;
+using std::atomic_fetch_sub _LIBCPP_USING_IF_EXISTS;
+using std::atomic_fetch_sub_explicit _LIBCPP_USING_IF_EXISTS;
+using std::atomic_flag_clear _LIBCPP_USING_IF_EXISTS;
+using std::atomic_flag_clear_explicit _LIBCPP_USING_IF_EXISTS;
+using std::atomic_flag_test_and_set _LIBCPP_USING_IF_EXISTS;
+using std::atomic_flag_test_and_set_explicit _LIBCPP_USING_IF_EXISTS;
+using std::atomic_is_lock_free _LIBCPP_USING_IF_EXISTS;
+using std::atomic_load _LIBCPP_USING_IF_EXISTS;
+using std::atomic_load_explicit _LIBCPP_USING_IF_EXISTS;
+using std::atomic_store _LIBCPP_USING_IF_EXISTS;
+using std::atomic_store_explicit _LIBCPP_USING_IF_EXISTS;
+
+using std::atomic_signal_fence _LIBCPP_USING_IF_EXISTS;
+using std::atomic_thread_fence _LIBCPP_USING_IF_EXISTS;
+
+#elif defined(_LIBCPP_COMPILER_CLANG_BASED)
+
+// Before C++23, we include the next <stdatomic.h> on the path to avoid hijacking
+// the header. We do this because Clang has historically shipped a <stdatomic.h>
+// header that would be available in all Standard modes, and we don't want to
+// break that use case.
+#  if __has_include_next(<stdatomic.h>)
+#    include_next <stdatomic.h>
+#  endif
+
+#endif // defined(__cplusplus) && _LIBCPP_STD_VER >= 23
+
+#endif // _LIBCPP_STDATOMIC_H
diff --git a/libcxx/include/__cxx03/stdbool.h b/libcxx/include/__cxx03/stdbool.h
new file mode 100644
index 000000000000..e74d91f45945
--- /dev/null
+++ b/libcxx/include/__cxx03/stdbool.h
@@ -0,0 +1,40 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_STDBOOL_H
+#define _LIBCPP_STDBOOL_H
+
+/*
+    stdbool.h synopsis
+
+Macros:
+
+    __bool_true_false_are_defined
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if __has_include_next(<stdbool.h>)
+#  include_next <stdbool.h>
+#endif
+
+#ifdef __cplusplus
+#  undef bool
+#  undef true
+#  undef false
+#  undef __bool_true_false_are_defined
+#  define __bool_true_false_are_defined 1
+#endif
+
+#endif // _LIBCPP_STDBOOL_H
diff --git a/libcxx/include/__cxx03/stddef.h b/libcxx/include/__cxx03/stddef.h
new file mode 100644
index 000000000000..1583e78e3739
--- /dev/null
+++ b/libcxx/include/__cxx03/stddef.h
@@ -0,0 +1,44 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+/*
+    stddef.h synopsis
+
+Macros:
+
+    offsetof(type,member-designator)
+    NULL
+
+Types:
+
+    ptrdiff_t
+    size_t
+    max_align_t // C++11
+    nullptr_t
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+// Note: This include is outside of header guards because we sometimes get included multiple times
+//       with different defines and the underlying <stddef.h> will know how to deal with that.
+#include_next <stddef.h>
+
+#ifndef _LIBCPP_STDDEF_H
+#  define _LIBCPP_STDDEF_H
+
+#  ifdef __cplusplus
+typedef decltype(nullptr) nullptr_t;
+#  endif
+
+#endif // _LIBCPP_STDDEF_H
diff --git a/libcxx/include/__cxx03/stdexcept b/libcxx/include/__cxx03/stdexcept
new file mode 100644
index 000000000000..853c185187c7
--- /dev/null
+++ b/libcxx/include/__cxx03/stdexcept
@@ -0,0 +1,286 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_STDEXCEPT
+#define _LIBCPP_STDEXCEPT
+
+/*
+    stdexcept synopsis
+
+namespace std
+{
+
+class logic_error;
+class domain_error;
+class invalid_argument;
+class length_error;
+class out_of_range;
+class runtime_error;
+class range_error;
+class overflow_error;
+class underflow_error;
+
+for each class xxx_error:
+
+class xxx_error : public exception // at least indirectly
+{
+public:
+    explicit xxx_error(const string& what_arg);
+    explicit xxx_error(const char*   what_arg);
+
+    virtual const char* what() const noexcept // returns what_arg
+};
+
+}  // std
+
+*/
+
+#include <__config>
+#include <__exception/exception.h>
+#include <__fwd/string.h>
+#include <__verbose_abort>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifndef _LIBCPP_ABI_VCRUNTIME
+class _LIBCPP_HIDDEN __libcpp_refstring {
+  const char* __imp_;
+
+  bool __uses_refcount() const;
+
+public:
+  explicit __libcpp_refstring(const char* __msg);
+  __libcpp_refstring(const __libcpp_refstring& __s) _NOEXCEPT;
+  __libcpp_refstring& operator=(const __libcpp_refstring& __s) _NOEXCEPT;
+  ~__libcpp_refstring();
+
+  _LIBCPP_HIDE_FROM_ABI const char* c_str() const _NOEXCEPT { return __imp_; }
+};
+#endif // !_LIBCPP_ABI_VCRUNTIME
+
+_LIBCPP_END_NAMESPACE_STD
+
+namespace std // purposefully not using versioning namespace
+{
+
+class _LIBCPP_EXPORTED_FROM_ABI logic_error : public exception {
+#ifndef _LIBCPP_ABI_VCRUNTIME
+
+private:
+  std::__libcpp_refstring __imp_;
+
+public:
+  explicit logic_error(const string&);
+  explicit logic_error(const char*);
+
+  logic_error(const logic_error&) _NOEXCEPT;
+  logic_error& operator=(const logic_error&) _NOEXCEPT;
+
+  ~logic_error() _NOEXCEPT override;
+
+  const char* what() const _NOEXCEPT override;
+#else
+
+public:
+  explicit logic_error(const std::string&); // Symbol uses versioned std::string
+  _LIBCPP_HIDE_FROM_ABI explicit logic_error(const char* __s) : exception(__s) {}
+#endif
+};
+
+class _LIBCPP_EXPORTED_FROM_ABI runtime_error : public exception {
+#ifndef _LIBCPP_ABI_VCRUNTIME
+
+private:
+  std::__libcpp_refstring __imp_;
+
+public:
+  explicit runtime_error(const string&);
+  explicit runtime_error(const char*);
+
+  runtime_error(const runtime_error&) _NOEXCEPT;
+  runtime_error& operator=(const runtime_error&) _NOEXCEPT;
+
+  ~runtime_error() _NOEXCEPT override;
+
+  const char* what() const _NOEXCEPT override;
+#else
+
+public:
+  explicit runtime_error(const std::string&); // Symbol uses versioned std::string
+  _LIBCPP_HIDE_FROM_ABI explicit runtime_error(const char* __s) : exception(__s) {}
+#endif // _LIBCPP_ABI_VCRUNTIME
+};
+
+class _LIBCPP_EXPORTED_FROM_ABI domain_error : public logic_error {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit domain_error(const string& __s) : logic_error(__s) {}
+  _LIBCPP_HIDE_FROM_ABI explicit domain_error(const char* __s) : logic_error(__s) {}
+
+#ifndef _LIBCPP_ABI_VCRUNTIME
+  _LIBCPP_HIDE_FROM_ABI domain_error(const domain_error&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI domain_error& operator=(const domain_error&) _NOEXCEPT = default;
+  ~domain_error() _NOEXCEPT override;
+#endif
+};
+
+class _LIBCPP_EXPORTED_FROM_ABI invalid_argument : public logic_error {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit invalid_argument(const string& __s) : logic_error(__s) {}
+  _LIBCPP_HIDE_FROM_ABI explicit invalid_argument(const char* __s) : logic_error(__s) {}
+
+#ifndef _LIBCPP_ABI_VCRUNTIME
+  _LIBCPP_HIDE_FROM_ABI invalid_argument(const invalid_argument&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI invalid_argument& operator=(const invalid_argument&) _NOEXCEPT = default;
+  ~invalid_argument() _NOEXCEPT override;
+#endif
+};
+
+class _LIBCPP_EXPORTED_FROM_ABI length_error : public logic_error {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit length_error(const string& __s) : logic_error(__s) {}
+  _LIBCPP_HIDE_FROM_ABI explicit length_error(const char* __s) : logic_error(__s) {}
+#ifndef _LIBCPP_ABI_VCRUNTIME
+  _LIBCPP_HIDE_FROM_ABI length_error(const length_error&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI length_error& operator=(const length_error&) _NOEXCEPT = default;
+  ~length_error() _NOEXCEPT override;
+#endif
+};
+
+class _LIBCPP_EXPORTED_FROM_ABI out_of_range : public logic_error {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit out_of_range(const string& __s) : logic_error(__s) {}
+  _LIBCPP_HIDE_FROM_ABI explicit out_of_range(const char* __s) : logic_error(__s) {}
+
+#ifndef _LIBCPP_ABI_VCRUNTIME
+  _LIBCPP_HIDE_FROM_ABI out_of_range(const out_of_range&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI out_of_range& operator=(const out_of_range&) _NOEXCEPT = default;
+  ~out_of_range() _NOEXCEPT override;
+#endif
+};
+
+class _LIBCPP_EXPORTED_FROM_ABI range_error : public runtime_error {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit range_error(const string& __s) : runtime_error(__s) {}
+  _LIBCPP_HIDE_FROM_ABI explicit range_error(const char* __s) : runtime_error(__s) {}
+
+#ifndef _LIBCPP_ABI_VCRUNTIME
+  _LIBCPP_HIDE_FROM_ABI range_error(const range_error&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI range_error& operator=(const range_error&) _NOEXCEPT = default;
+  ~range_error() _NOEXCEPT override;
+#endif
+};
+
+class _LIBCPP_EXPORTED_FROM_ABI overflow_error : public runtime_error {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit overflow_error(const string& __s) : runtime_error(__s) {}
+  _LIBCPP_HIDE_FROM_ABI explicit overflow_error(const char* __s) : runtime_error(__s) {}
+
+#ifndef _LIBCPP_ABI_VCRUNTIME
+  _LIBCPP_HIDE_FROM_ABI overflow_error(const overflow_error&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI overflow_error& operator=(const overflow_error&) _NOEXCEPT = default;
+  ~overflow_error() _NOEXCEPT override;
+#endif
+};
+
+class _LIBCPP_EXPORTED_FROM_ABI underflow_error : public runtime_error {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit underflow_error(const string& __s) : runtime_error(__s) {}
+  _LIBCPP_HIDE_FROM_ABI explicit underflow_error(const char* __s) : runtime_error(__s) {}
+
+#ifndef _LIBCPP_ABI_VCRUNTIME
+  _LIBCPP_HIDE_FROM_ABI underflow_error(const underflow_error&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI underflow_error& operator=(const underflow_error&) _NOEXCEPT = default;
+  ~underflow_error() _NOEXCEPT override;
+#endif
+};
+
+} // namespace std
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// in the dylib
+_LIBCPP_NORETURN _LIBCPP_EXPORTED_FROM_ABI void __throw_runtime_error(const char*);
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_logic_error(const char* __msg) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw logic_error(__msg);
+#else
+  _LIBCPP_VERBOSE_ABORT("logic_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
+#endif
+}
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_domain_error(const char* __msg) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw domain_error(__msg);
+#else
+  _LIBCPP_VERBOSE_ABORT("domain_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
+#endif
+}
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_invalid_argument(const char* __msg) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw invalid_argument(__msg);
+#else
+  _LIBCPP_VERBOSE_ABORT("invalid_argument was thrown in -fno-exceptions mode with message \"%s\"", __msg);
+#endif
+}
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_length_error(const char* __msg) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw length_error(__msg);
+#else
+  _LIBCPP_VERBOSE_ABORT("length_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
+#endif
+}
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_out_of_range(const char* __msg) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw out_of_range(__msg);
+#else
+  _LIBCPP_VERBOSE_ABORT("out_of_range was thrown in -fno-exceptions mode with message \"%s\"", __msg);
+#endif
+}
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_range_error(const char* __msg) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw range_error(__msg);
+#else
+  _LIBCPP_VERBOSE_ABORT("range_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
+#endif
+}
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_overflow_error(const char* __msg) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw overflow_error(__msg);
+#else
+  _LIBCPP_VERBOSE_ABORT("overflow_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
+#endif
+}
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_underflow_error(const char* __msg) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw underflow_error(__msg);
+#else
+  _LIBCPP_VERBOSE_ABORT("underflow_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
+#endif
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstdlib>
+#  include <exception>
+#  include <iosfwd>
+#endif
+
+#endif // _LIBCPP_STDEXCEPT
diff --git a/libcxx/include/__cxx03/stdint.h b/libcxx/include/__cxx03/stdint.h
new file mode 100644
index 000000000000..35e5b8cbdad2
--- /dev/null
+++ b/libcxx/include/__cxx03/stdint.h
@@ -0,0 +1,127 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_STDINT_H
+// AIX system headers need stdint.h to be re-enterable while _STD_TYPES_T
+// is defined until an inclusion of it without _STD_TYPES_T occurs, in which
+// case the header guard macro is defined.
+#if !defined(_AIX) || !defined(_STD_TYPES_T)
+#  define _LIBCPP_STDINT_H
+#endif // _STD_TYPES_T
+
+/*
+    stdint.h synopsis
+
+Macros:
+
+    INT8_MIN
+    INT16_MIN
+    INT32_MIN
+    INT64_MIN
+
+    INT8_MAX
+    INT16_MAX
+    INT32_MAX
+    INT64_MAX
+
+    UINT8_MAX
+    UINT16_MAX
+    UINT32_MAX
+    UINT64_MAX
+
+    INT_LEAST8_MIN
+    INT_LEAST16_MIN
+    INT_LEAST32_MIN
+    INT_LEAST64_MIN
+
+    INT_LEAST8_MAX
+    INT_LEAST16_MAX
+    INT_LEAST32_MAX
+    INT_LEAST64_MAX
+
+    UINT_LEAST8_MAX
+    UINT_LEAST16_MAX
+    UINT_LEAST32_MAX
+    UINT_LEAST64_MAX
+
+    INT_FAST8_MIN
+    INT_FAST16_MIN
+    INT_FAST32_MIN
+    INT_FAST64_MIN
+
+    INT_FAST8_MAX
+    INT_FAST16_MAX
+    INT_FAST32_MAX
+    INT_FAST64_MAX
+
+    UINT_FAST8_MAX
+    UINT_FAST16_MAX
+    UINT_FAST32_MAX
+    UINT_FAST64_MAX
+
+    INTPTR_MIN
+    INTPTR_MAX
+    UINTPTR_MAX
+
+    INTMAX_MIN
+    INTMAX_MAX
+
+    UINTMAX_MAX
+
+    PTRDIFF_MIN
+    PTRDIFF_MAX
+
+    SIG_ATOMIC_MIN
+    SIG_ATOMIC_MAX
+
+    SIZE_MAX
+
+    WCHAR_MIN
+    WCHAR_MAX
+
+    WINT_MIN
+    WINT_MAX
+
+    INT8_C(value)
+    INT16_C(value)
+    INT32_C(value)
+    INT64_C(value)
+
+    UINT8_C(value)
+    UINT16_C(value)
+    UINT32_C(value)
+    UINT64_C(value)
+
+    INTMAX_C(value)
+    UINTMAX_C(value)
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+/* C99 stdlib (e.g. glibc < 2.18) does not provide macros needed
+   for C++11 unless __STDC_LIMIT_MACROS and __STDC_CONSTANT_MACROS
+   are defined
+*/
+#if defined(__cplusplus) && !defined(__STDC_LIMIT_MACROS)
+#  define __STDC_LIMIT_MACROS
+#endif
+#if defined(__cplusplus) && !defined(__STDC_CONSTANT_MACROS)
+#  define __STDC_CONSTANT_MACROS
+#endif
+
+#if __has_include_next(<stdint.h>)
+#  include_next <stdint.h>
+#endif
+
+#endif // _LIBCPP_STDINT_H
diff --git a/libcxx/include/__cxx03/stdio.h b/libcxx/include/__cxx03/stdio.h
new file mode 100644
index 000000000000..3aa559393f18
--- /dev/null
+++ b/libcxx/include/__cxx03/stdio.h
@@ -0,0 +1,123 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__need_FILE) || defined(__need___FILE)
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  include_next <stdio.h>
+
+#elif !defined(_LIBCPP_STDIO_H)
+#  define _LIBCPP_STDIO_H
+
+/*
+    stdio.h synopsis
+
+Macros:
+
+    BUFSIZ
+    EOF
+    FILENAME_MAX
+    FOPEN_MAX
+    L_tmpnam
+    NULL
+    SEEK_CUR
+    SEEK_END
+    SEEK_SET
+    TMP_MAX
+    _IOFBF
+    _IOLBF
+    _IONBF
+    stderr
+    stdin
+    stdout
+
+Types:
+
+FILE
+fpos_t
+size_t
+
+int remove(const char* filename);
+int rename(const char* old, const char* new);
+FILE* tmpfile(void);
+char* tmpnam(char* s);
+int fclose(FILE* stream);
+int fflush(FILE* stream);
+FILE* fopen(const char* restrict filename, const char* restrict mode);
+FILE* freopen(const char* restrict filename, const char * restrict mode,
+              FILE * restrict stream);
+void setbuf(FILE* restrict stream, char* restrict buf);
+int setvbuf(FILE* restrict stream, char* restrict buf, int mode, size_t size);
+int fprintf(FILE* restrict stream, const char* restrict format, ...);
+int fscanf(FILE* restrict stream, const char * restrict format, ...);
+int printf(const char* restrict format, ...);
+int scanf(const char* restrict format, ...);
+int snprintf(char* restrict s, size_t n, const char* restrict format, ...);    // C99
+int sprintf(char* restrict s, const char* restrict format, ...);
+int sscanf(const char* restrict s, const char* restrict format, ...);
+int vfprintf(FILE* restrict stream, const char* restrict format, va_list arg);
+int vfscanf(FILE* restrict stream, const char* restrict format, va_list arg);  // C99
+int vprintf(const char* restrict format, va_list arg);
+int vscanf(const char* restrict format, va_list arg);                          // C99
+int vsnprintf(char* restrict s, size_t n, const char* restrict format,         // C99
+              va_list arg);
+int vsprintf(char* restrict s, const char* restrict format, va_list arg);
+int vsscanf(const char* restrict s, const char* restrict format, va_list arg); // C99
+int fgetc(FILE* stream);
+char* fgets(char* restrict s, int n, FILE* restrict stream);
+int fputc(int c, FILE* stream);
+int fputs(const char* restrict s, FILE* restrict stream);
+int getc(FILE* stream);
+int getchar(void);
+char* gets(char* s);  // removed in C++14
+int putc(int c, FILE* stream);
+int putchar(int c);
+int puts(const char* s);
+int ungetc(int c, FILE* stream);
+size_t fread(void* restrict ptr, size_t size, size_t nmemb,
+             FILE* restrict stream);
+size_t fwrite(const void* restrict ptr, size_t size, size_t nmemb,
+              FILE* restrict stream);
+int fgetpos(FILE* restrict stream, fpos_t* restrict pos);
+int fseek(FILE* stream, long offset, int whence);
+int fsetpos(FILE*stream, const fpos_t* pos);
+long ftell(FILE* stream);
+void rewind(FILE* stream);
+void clearerr(FILE* stream);
+int feof(FILE* stream);
+int ferror(FILE* stream);
+void perror(const char* s);
+*/
+
+#  include <__config>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if __has_include_next(<stdio.h>)
+#    include_next <stdio.h>
+#  endif
+
+#  ifdef __cplusplus
+
+#    undef getc
+#    undef putc
+#    undef clearerr
+#    undef feof
+#    undef ferror
+#    undef putchar
+#    undef getchar
+
+#  endif
+
+#endif // _LIBCPP_STDIO_H
diff --git a/libcxx/include/__cxx03/stdlib.h b/libcxx/include/__cxx03/stdlib.h
new file mode 100644
index 000000000000..a74344d49150
--- /dev/null
+++ b/libcxx/include/__cxx03/stdlib.h
@@ -0,0 +1,151 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__need_malloc_and_calloc)
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  include_next <stdlib.h>
+
+#elif !defined(_LIBCPP_STDLIB_H)
+#  define _LIBCPP_STDLIB_H
+
+/*
+    stdlib.h synopsis
+
+Macros:
+
+    EXIT_FAILURE
+    EXIT_SUCCESS
+    MB_CUR_MAX
+    NULL
+    RAND_MAX
+
+Types:
+
+    size_t
+    div_t
+    ldiv_t
+    lldiv_t                                                               // C99
+
+double    atof (const char* nptr);
+int       atoi (const char* nptr);
+long      atol (const char* nptr);
+long long atoll(const char* nptr);                                        // C99
+double             strtod  (const char* restrict nptr, char** restrict endptr);
+float              strtof  (const char* restrict nptr, char** restrict endptr); // C99
+long double        strtold (const char* restrict nptr, char** restrict endptr); // C99
+long               strtol  (const char* restrict nptr, char** restrict endptr, int base);
+long long          strtoll (const char* restrict nptr, char** restrict endptr, int base); // C99
+unsigned long      strtoul (const char* restrict nptr, char** restrict endptr, int base);
+unsigned long long strtoull(const char* restrict nptr, char** restrict endptr, int base); // C99
+int rand(void);
+void srand(unsigned int seed);
+void* calloc(size_t nmemb, size_t size);
+void free(void* ptr);
+void* malloc(size_t size);
+void* realloc(void* ptr, size_t size);
+void abort(void);
+int atexit(void (*func)(void));
+void exit(int status);
+void _Exit(int status);
+char* getenv(const char* name);
+int system(const char* string);
+void* bsearch(const void* key, const void* base, size_t nmemb, size_t size,
+              int (*compar)(const void *, const void *));
+void qsort(void* base, size_t nmemb, size_t size,
+           int (*compar)(const void *, const void *));
+int         abs(      int j);
+long        abs(     long j);
+long long   abs(long long j);                                             // C++0X
+long       labs(     long j);
+long long llabs(long long j);                                             // C99
+div_t     div(      int numer,       int denom);
+ldiv_t    div(     long numer,      long denom);
+lldiv_t   div(long long numer, long long denom);                          // C++0X
+ldiv_t   ldiv(     long numer,      long denom);
+lldiv_t lldiv(long long numer, long long denom);                          // C99
+int mblen(const char* s, size_t n);
+int mbtowc(wchar_t* restrict pwc, const char* restrict s, size_t n);
+int wctomb(char* s, wchar_t wchar);
+size_t mbstowcs(wchar_t* restrict pwcs, const char* restrict s, size_t n);
+size_t wcstombs(char* restrict s, const wchar_t* restrict pwcs, size_t n);
+int at_quick_exit(void (*func)(void))                                     // C++11
+void quick_exit(int status);                                              // C++11
+void *aligned_alloc(size_t alignment, size_t size);                       // C11
+
+*/
+
+#  include <__config>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if __has_include_next(<stdlib.h>)
+#    include_next <stdlib.h>
+#  endif
+
+#  ifdef __cplusplus
+extern "C++" {
+// abs
+
+#    ifdef abs
+#      undef abs
+#    endif
+#    ifdef labs
+#      undef labs
+#    endif
+#    ifdef llabs
+#      undef llabs
+#    endif
+
+// MSVCRT already has the correct prototype in <stdlib.h> if __cplusplus is defined
+#    if !defined(_LIBCPP_MSVCRT)
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI long abs(long __x) _NOEXCEPT { return __builtin_labs(__x); }
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI long long abs(long long __x) _NOEXCEPT { return __builtin_llabs(__x); }
+#    endif // !defined(_LIBCPP_MSVCRT)
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI float abs(float __lcpp_x) _NOEXCEPT {
+  return __builtin_fabsf(__lcpp_x); // Use builtins to prevent needing math.h
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI double abs(double __lcpp_x) _NOEXCEPT {
+  return __builtin_fabs(__lcpp_x);
+}
+
+_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI long double abs(long double __lcpp_x) _NOEXCEPT {
+  return __builtin_fabsl(__lcpp_x);
+}
+
+// div
+
+#    ifdef div
+#      undef div
+#    endif
+#    ifdef ldiv
+#      undef ldiv
+#    endif
+#    ifdef lldiv
+#      undef lldiv
+#    endif
+
+// MSVCRT already has the correct prototype in <stdlib.h> if __cplusplus is defined
+#    if !defined(_LIBCPP_MSVCRT)
+inline _LIBCPP_HIDE_FROM_ABI ldiv_t div(long __x, long __y) _NOEXCEPT { return ::ldiv(__x, __y); }
+#      if !(defined(__FreeBSD__) && !defined(__LONG_LONG_SUPPORTED))
+inline _LIBCPP_HIDE_FROM_ABI lldiv_t div(long long __x, long long __y) _NOEXCEPT { return ::lldiv(__x, __y); }
+#      endif
+#    endif // _LIBCPP_MSVCRT
+} // extern "C++"
+#  endif   // __cplusplus
+
+#endif // _LIBCPP_STDLIB_H
diff --git a/libcxx/include/__cxx03/stop_token b/libcxx/include/__cxx03/stop_token
new file mode 100644
index 000000000000..d4e651d9541f
--- /dev/null
+++ b/libcxx/include/__cxx03/stop_token
@@ -0,0 +1,56 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_STOP_TOKEN
+#define _LIBCPP_STOP_TOKEN
+
+/*
+
+namespace std {
+  // [stoptoken], class stop_token
+  class stop_token;
+
+  // [stopsource], class stop_source
+  class stop_source;
+
+  // no-shared-stop-state indicator
+  struct nostopstate_t {
+    explicit nostopstate_t() = default;
+  };
+  inline constexpr nostopstate_t nostopstate{};
+
+  // [stopcallback], class template stop_callback
+  template<class Callback>
+    class stop_callback;
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+
+#  if _LIBCPP_STD_VER >= 20
+#    include <__stop_token/stop_callback.h>
+#    include <__stop_token/stop_source.h>
+#    include <__stop_token/stop_token.h>
+#  endif
+
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#endif // !defined(_LIBCPP_HAS_NO_THREADS)
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <iosfwd>
+#endif
+
+#endif // _LIBCPP_STOP_TOKEN
diff --git a/libcxx/include/__cxx03/streambuf b/libcxx/include/__cxx03/streambuf
new file mode 100644
index 000000000000..5a3c17ef7c99
--- /dev/null
+++ b/libcxx/include/__cxx03/streambuf
@@ -0,0 +1,445 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_STREAMBUF
+#define _LIBCPP_STREAMBUF
+
+/*
+    streambuf synopsis
+
+namespace std
+{
+
+template <class charT, class traits = char_traits<charT> >
+class basic_streambuf
+{
+public:
+    // types:
+    typedef charT char_type;
+    typedef traits traits_type;
+    typedef typename traits_type::int_type int_type;
+    typedef typename traits_type::pos_type pos_type;
+    typedef typename traits_type::off_type off_type;
+
+    virtual ~basic_streambuf();
+
+    // 27.6.2.2.1 locales:
+    locale pubimbue(const locale& loc);
+    locale getloc() const;
+
+    // 27.6.2.2.2 buffer and positioning:
+    basic_streambuf* pubsetbuf(char_type* s, streamsize n);
+    pos_type pubseekoff(off_type off, ios_base::seekdir way,
+                        ios_base::openmode which = ios_base::in | ios_base::out);
+    pos_type pubseekpos(pos_type sp,
+                        ios_base::openmode which = ios_base::in | ios_base::out);
+    int pubsync();
+
+    // Get and put areas:
+    // 27.6.2.2.3 Get area:
+    streamsize in_avail();
+    int_type snextc();
+    int_type sbumpc();
+    int_type sgetc();
+    streamsize sgetn(char_type* s, streamsize n);
+
+    // 27.6.2.2.4 Putback:
+    int_type sputbackc(char_type c);
+    int_type sungetc();
+
+    // 27.6.2.2.5 Put area:
+    int_type sputc(char_type c);
+    streamsize sputn(const char_type* s, streamsize n);
+
+protected:
+    basic_streambuf();
+    basic_streambuf(const basic_streambuf& rhs);
+    basic_streambuf& operator=(const basic_streambuf& rhs);
+    void swap(basic_streambuf& rhs);
+
+    // 27.6.2.3.2 Get area:
+    char_type* eback() const;
+    char_type* gptr() const;
+    char_type* egptr() const;
+    void gbump(int n);
+    void setg(char_type* gbeg, char_type* gnext, char_type* gend);
+
+    // 27.6.2.3.3 Put area:
+    char_type* pbase() const;
+    char_type* pptr() const;
+    char_type* epptr() const;
+    void pbump(int n);
+    void setp(char_type* pbeg, char_type* pend);
+
+    // 27.6.2.4 virtual functions:
+    // 27.6.2.4.1 Locales:
+    virtual void imbue(const locale& loc);
+
+    // 27.6.2.4.2 Buffer management and positioning:
+    virtual basic_streambuf* setbuf(char_type* s, streamsize n);
+    virtual pos_type seekoff(off_type off, ios_base::seekdir way,
+                             ios_base::openmode which = ios_base::in | ios_base::out);
+    virtual pos_type seekpos(pos_type sp,
+                             ios_base::openmode which = ios_base::in | ios_base::out);
+    virtual int sync();
+
+    // 27.6.2.4.3 Get area:
+    virtual streamsize showmanyc();
+    virtual streamsize xsgetn(char_type* s, streamsize n);
+    virtual int_type underflow();
+    virtual int_type uflow();
+
+    // 27.6.2.4.4 Putback:
+    virtual int_type pbackfail(int_type c = traits_type::eof());
+
+    // 27.6.2.4.5 Put area:
+    virtual streamsize xsputn(const char_type* s, streamsize n);
+    virtual int_type overflow (int_type c = traits_type::eof());
+};
+
+}  // std
+
+*/
+
+#include <__assert>
+#include <__config>
+#include <__fwd/streambuf.h>
+#include <__locale>
+#include <__type_traits/is_same.h>
+#include <__utility/is_valid_range.h>
+#include <climits>
+#include <ios>
+#include <iosfwd>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits>
+class _LIBCPP_TEMPLATE_VIS basic_streambuf {
+public:
+  // types:
+  typedef _CharT char_type;
+  typedef _Traits traits_type;
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+
+  static_assert(is_same<_CharT, typename traits_type::char_type>::value,
+                "traits_type::char_type must be the same type as CharT");
+
+  virtual ~basic_streambuf();
+
+  // 27.6.2.2.1 locales:
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 locale pubimbue(const locale& __loc) {
+    imbue(__loc);
+    locale __r = __loc_;
+    __loc_     = __loc;
+    return __r;
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 locale getloc() const { return __loc_; }
+
+  // 27.6.2.2.2 buffer and positioning:
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_streambuf* pubsetbuf(char_type* __s, streamsize __n) {
+    return setbuf(__s, __n);
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 pos_type
+  pubseekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __which = ios_base::in | ios_base::out) {
+    return seekoff(__off, __way, __which);
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 pos_type
+  pubseekpos(pos_type __sp, ios_base::openmode __which = ios_base::in | ios_base::out) {
+    return seekpos(__sp, __which);
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int pubsync() { return sync(); }
+
+  // Get and put areas:
+  // 27.6.2.2.3 Get area:
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 streamsize in_avail() {
+    if (__ninp_ < __einp_)
+      return static_cast<streamsize>(__einp_ - __ninp_);
+    return showmanyc();
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type snextc() {
+    if (sbumpc() == traits_type::eof())
+      return traits_type::eof();
+    return sgetc();
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sbumpc() {
+    if (__ninp_ == __einp_)
+      return uflow();
+    return traits_type::to_int_type(*__ninp_++);
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sgetc() {
+    if (__ninp_ == __einp_)
+      return underflow();
+    return traits_type::to_int_type(*__ninp_);
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 streamsize sgetn(char_type* __s, streamsize __n) { return xsgetn(__s, __n); }
+
+  // 27.6.2.2.4 Putback:
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sputbackc(char_type __c) {
+    if (__binp_ == __ninp_ || !traits_type::eq(__c, __ninp_[-1]))
+      return pbackfail(traits_type::to_int_type(__c));
+    return traits_type::to_int_type(*--__ninp_);
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sungetc() {
+    if (__binp_ == __ninp_)
+      return pbackfail();
+    return traits_type::to_int_type(*--__ninp_);
+  }
+
+  // 27.6.2.2.5 Put area:
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sputc(char_type __c) {
+    if (__nout_ == __eout_)
+      return overflow(traits_type::to_int_type(__c));
+    *__nout_++ = __c;
+    return traits_type::to_int_type(__c);
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 streamsize sputn(const char_type* __s, streamsize __n) {
+    return xsputn(__s, __n);
+  }
+
+protected:
+  basic_streambuf();
+  basic_streambuf(const basic_streambuf& __rhs);
+  basic_streambuf& operator=(const basic_streambuf& __rhs);
+  void swap(basic_streambuf& __rhs);
+
+  // 27.6.2.3.2 Get area:
+  _LIBCPP_HIDE_FROM_ABI char_type* eback() const { return __binp_; }
+  _LIBCPP_HIDE_FROM_ABI char_type* gptr() const { return __ninp_; }
+  _LIBCPP_HIDE_FROM_ABI char_type* egptr() const { return __einp_; }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void gbump(int __n) { __ninp_ += __n; }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void setg(char_type* __gbeg, char_type* __gnext, char_type* __gend) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(std::__is_valid_range(__gbeg, __gnext), "[gbeg, gnext) must be a valid range");
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(std::__is_valid_range(__gbeg, __gend), "[gbeg, gend) must be a valid range");
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(std::__is_valid_range(__gnext, __gend), "[gnext, gend) must be a valid range");
+    __binp_ = __gbeg;
+    __ninp_ = __gnext;
+    __einp_ = __gend;
+  }
+
+  // 27.6.2.3.3 Put area:
+  _LIBCPP_HIDE_FROM_ABI char_type* pbase() const { return __bout_; }
+  _LIBCPP_HIDE_FROM_ABI char_type* pptr() const { return __nout_; }
+  _LIBCPP_HIDE_FROM_ABI char_type* epptr() const { return __eout_; }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void pbump(int __n) { __nout_ += __n; }
+
+  _LIBCPP_HIDE_FROM_ABI void __pbump(streamsize __n) { __nout_ += __n; }
+
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void setp(char_type* __pbeg, char_type* __pend) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(std::__is_valid_range(__pbeg, __pend), "[pbeg, pend) must be a valid range");
+    __bout_ = __nout_ = __pbeg;
+    __eout_           = __pend;
+  }
+
+  // 27.6.2.4 virtual functions:
+  // 27.6.2.4.1 Locales:
+  virtual void imbue(const locale& __loc);
+
+  // 27.6.2.4.2 Buffer management and positioning:
+  virtual basic_streambuf* setbuf(char_type* __s, streamsize __n);
+  virtual pos_type
+  seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __which = ios_base::in | ios_base::out);
+  virtual pos_type seekpos(pos_type __sp, ios_base::openmode __which = ios_base::in | ios_base::out);
+  virtual int sync();
+
+  // 27.6.2.4.3 Get area:
+  virtual streamsize showmanyc();
+  virtual streamsize xsgetn(char_type* __s, streamsize __n);
+  virtual int_type underflow();
+  virtual int_type uflow();
+
+  // 27.6.2.4.4 Putback:
+  virtual int_type pbackfail(int_type __c = traits_type::eof());
+
+  // 27.6.2.4.5 Put area:
+  virtual streamsize xsputn(const char_type* __s, streamsize __n);
+  virtual int_type overflow(int_type __c = traits_type::eof());
+
+private:
+  locale __loc_;
+  char_type* __binp_;
+  char_type* __ninp_;
+  char_type* __einp_;
+  char_type* __bout_;
+  char_type* __nout_;
+  char_type* __eout_;
+};
+
+template <class _CharT, class _Traits>
+basic_streambuf<_CharT, _Traits>::~basic_streambuf() {}
+
+template <class _CharT, class _Traits>
+basic_streambuf<_CharT, _Traits>::basic_streambuf()
+    : __binp_(nullptr), __ninp_(nullptr), __einp_(nullptr), __bout_(nullptr), __nout_(nullptr), __eout_(nullptr) {}
+
+template <class _CharT, class _Traits>
+basic_streambuf<_CharT, _Traits>::basic_streambuf(const basic_streambuf& __sb)
+    : __loc_(__sb.__loc_),
+      __binp_(__sb.__binp_),
+      __ninp_(__sb.__ninp_),
+      __einp_(__sb.__einp_),
+      __bout_(__sb.__bout_),
+      __nout_(__sb.__nout_),
+      __eout_(__sb.__eout_) {}
+
+template <class _CharT, class _Traits>
+basic_streambuf<_CharT, _Traits>& basic_streambuf<_CharT, _Traits>::operator=(const basic_streambuf& __sb) {
+  __loc_  = __sb.__loc_;
+  __binp_ = __sb.__binp_;
+  __ninp_ = __sb.__ninp_;
+  __einp_ = __sb.__einp_;
+  __bout_ = __sb.__bout_;
+  __nout_ = __sb.__nout_;
+  __eout_ = __sb.__eout_;
+  return *this;
+}
+
+template <class _CharT, class _Traits>
+void basic_streambuf<_CharT, _Traits>::swap(basic_streambuf& __sb) {
+  std::swap(__loc_, __sb.__loc_);
+  std::swap(__binp_, __sb.__binp_);
+  std::swap(__ninp_, __sb.__ninp_);
+  std::swap(__einp_, __sb.__einp_);
+  std::swap(__bout_, __sb.__bout_);
+  std::swap(__nout_, __sb.__nout_);
+  std::swap(__eout_, __sb.__eout_);
+}
+
+template <class _CharT, class _Traits>
+void basic_streambuf<_CharT, _Traits>::imbue(const locale&) {}
+
+template <class _CharT, class _Traits>
+basic_streambuf<_CharT, _Traits>* basic_streambuf<_CharT, _Traits>::setbuf(char_type*, streamsize) {
+  return this;
+}
+
+template <class _CharT, class _Traits>
+typename basic_streambuf<_CharT, _Traits>::pos_type
+basic_streambuf<_CharT, _Traits>::seekoff(off_type, ios_base::seekdir, ios_base::openmode) {
+  return pos_type(off_type(-1));
+}
+
+template <class _CharT, class _Traits>
+typename basic_streambuf<_CharT, _Traits>::pos_type
+basic_streambuf<_CharT, _Traits>::seekpos(pos_type, ios_base::openmode) {
+  return pos_type(off_type(-1));
+}
+
+template <class _CharT, class _Traits>
+int basic_streambuf<_CharT, _Traits>::sync() {
+  return 0;
+}
+
+template <class _CharT, class _Traits>
+streamsize basic_streambuf<_CharT, _Traits>::showmanyc() {
+  return 0;
+}
+
+template <class _CharT, class _Traits>
+streamsize basic_streambuf<_CharT, _Traits>::xsgetn(char_type* __s, streamsize __n) {
+  const int_type __eof = traits_type::eof();
+  int_type __c;
+  streamsize __i = 0;
+  while (__i < __n) {
+    if (__ninp_ < __einp_) {
+      const streamsize __len = std::min(static_cast<streamsize>(INT_MAX), std::min(__einp_ - __ninp_, __n - __i));
+      traits_type::copy(__s, __ninp_, __len);
+      __s += __len;
+      __i += __len;
+      this->gbump(__len);
+    } else if ((__c = uflow()) != __eof) {
+      *__s = traits_type::to_char_type(__c);
+      ++__s;
+      ++__i;
+    } else
+      break;
+  }
+  return __i;
+}
+
+template <class _CharT, class _Traits>
+typename basic_streambuf<_CharT, _Traits>::int_type basic_streambuf<_CharT, _Traits>::underflow() {
+  return traits_type::eof();
+}
+
+template <class _CharT, class _Traits>
+typename basic_streambuf<_CharT, _Traits>::int_type basic_streambuf<_CharT, _Traits>::uflow() {
+  if (underflow() == traits_type::eof())
+    return traits_type::eof();
+  return traits_type::to_int_type(*__ninp_++);
+}
+
+template <class _CharT, class _Traits>
+typename basic_streambuf<_CharT, _Traits>::int_type basic_streambuf<_CharT, _Traits>::pbackfail(int_type) {
+  return traits_type::eof();
+}
+
+template <class _CharT, class _Traits>
+streamsize basic_streambuf<_CharT, _Traits>::xsputn(const char_type* __s, streamsize __n) {
+  streamsize __i = 0;
+  int_type __eof = traits_type::eof();
+  while (__i < __n) {
+    if (__nout_ >= __eout_) {
+      if (overflow(traits_type::to_int_type(*__s)) == __eof)
+        break;
+      ++__s;
+      ++__i;
+    } else {
+      streamsize __chunk_size = std::min(__eout_ - __nout_, __n - __i);
+      traits_type::copy(__nout_, __s, __chunk_size);
+      __nout_ += __chunk_size;
+      __s += __chunk_size;
+      __i += __chunk_size;
+    }
+  }
+  return __i;
+}
+
+template <class _CharT, class _Traits>
+typename basic_streambuf<_CharT, _Traits>::int_type basic_streambuf<_CharT, _Traits>::overflow(int_type) {
+  return traits_type::eof();
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_streambuf<char>;
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_streambuf<wchar_t>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstdint>
+#endif
+
+#endif // _LIBCPP_STREAMBUF
diff --git a/libcxx/include/__cxx03/string b/libcxx/include/__cxx03/string
new file mode 100644
index 000000000000..08d66803f8a1
--- /dev/null
+++ b/libcxx/include/__cxx03/string
@@ -0,0 +1,4352 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_STRING
+#define _LIBCPP_STRING
+
+// clang-format off
+
+/*
+    string synopsis
+
+#include <compare>
+#include <initializer_list>
+
+namespace std
+{
+
+template <class stateT>
+class fpos
+{
+private:
+    stateT st;
+public:
+    fpos(streamoff = streamoff());
+
+    operator streamoff() const;
+
+    stateT state() const;
+    void state(stateT);
+
+    fpos& operator+=(streamoff);
+    fpos  operator+ (streamoff) const;
+    fpos& operator-=(streamoff);
+    fpos  operator- (streamoff) const;
+};
+
+template <class stateT> streamoff operator-(const fpos<stateT>& x, const fpos<stateT>& y);
+
+template <class stateT> bool operator==(const fpos<stateT>& x, const fpos<stateT>& y);
+template <class stateT> bool operator!=(const fpos<stateT>& x, const fpos<stateT>& y);
+
+template <class charT>
+struct char_traits
+{
+    using char_type           = charT;
+    using int_type            = ...;
+    using off_type            = streamoff;
+    using pos_type            = streampos;
+    using state_type          = mbstate_t;
+    using comparison_category = strong_ordering; // Since C++20 only for the specializations
+                                                 // char, wchar_t, char8_t, char16_t, and char32_t.
+
+    static void assign(char_type& c1, const char_type& c2) noexcept;
+    static constexpr bool eq(char_type c1, char_type c2) noexcept;
+    static constexpr bool lt(char_type c1, char_type c2) noexcept;
+
+    static int              compare(const char_type* s1, const char_type* s2, size_t n);
+    static size_t           length(const char_type* s);
+    static const char_type* find(const char_type* s, size_t n, const char_type& a);
+    static char_type*       move(char_type* s1, const char_type* s2, size_t n);
+    static char_type*       copy(char_type* s1, const char_type* s2, size_t n);
+    static char_type*       assign(char_type* s, size_t n, char_type a);
+
+    static constexpr int_type  not_eof(int_type c) noexcept;
+    static constexpr char_type to_char_type(int_type c) noexcept;
+    static constexpr int_type  to_int_type(char_type c) noexcept;
+    static constexpr bool      eq_int_type(int_type c1, int_type c2) noexcept;
+    static constexpr int_type  eof() noexcept;
+};
+
+template <> struct char_traits<char>;
+template <> struct char_traits<wchar_t>;
+template <> struct char_traits<char8_t>;  // C++20
+template <> struct char_traits<char16_t>;
+template <> struct char_traits<char32_t>;
+
+template<class charT, class traits = char_traits<charT>, class Allocator = allocator<charT> >
+class basic_string
+{
+public:
+// types:
+    typedef traits traits_type;
+    typedef typename traits_type::char_type value_type;
+    typedef Allocator allocator_type;
+    typedef typename allocator_type::size_type size_type;
+    typedef typename allocator_type::difference_type difference_type;
+    typedef typename allocator_type::reference reference;
+    typedef typename allocator_type::const_reference const_reference;
+    typedef typename allocator_type::pointer pointer;
+    typedef typename allocator_type::const_pointer const_pointer;
+    typedef implementation-defined iterator;
+    typedef implementation-defined const_iterator;
+    typedef std::reverse_iterator<iterator> reverse_iterator;
+    typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+
+    static const size_type npos = -1;
+
+    basic_string()
+        noexcept(is_nothrow_default_constructible<allocator_type>::value);                      // constexpr since C++20
+    explicit basic_string(const allocator_type& a);                                             // constexpr since C++20
+    basic_string(const basic_string& str);                                                      // constexpr since C++20
+    basic_string(basic_string&& str)
+        noexcept(is_nothrow_move_constructible<allocator_type>::value);                         // constexpr since C++20
+    basic_string(const basic_string& str, size_type pos,
+                 const allocator_type& a = allocator_type());                                   // constexpr since C++20
+    basic_string(const basic_string& str, size_type pos, size_type n,
+                 const Allocator& a = Allocator());                                             // constexpr since C++20
+    constexpr basic_string(
+        basic_string&& str, size_type pos, const Allocator& a = Allocator());                   // since C++23
+    constexpr basic_string(
+        basic_string&& str, size_type pos, size_type n, const Allocator& a = Allocator());      // since C++23
+    template<class T>
+        basic_string(const T& t, size_type pos, size_type n, const Allocator& a = Allocator()); // C++17, constexpr since C++20
+    template <class T>
+        explicit basic_string(const T& t, const Allocator& a = Allocator());                    // C++17, constexpr since C++20
+    basic_string(const value_type* s, const allocator_type& a = allocator_type());              // constexpr since C++20
+    basic_string(const value_type* s, size_type n, const allocator_type& a = allocator_type()); // constexpr since C++20
+    basic_string(nullptr_t) = delete; // C++23
+    basic_string(size_type n, value_type c, const allocator_type& a = allocator_type());        // constexpr since C++20
+    template<class InputIterator>
+        basic_string(InputIterator begin, InputIterator end,
+                     const allocator_type& a = allocator_type());                               // constexpr since C++20
+    template<container-compatible-range<charT> R>
+      constexpr basic_string(from_range_t, R&& rg, const Allocator& a = Allocator());           // since C++23
+    basic_string(initializer_list<value_type>, const Allocator& = Allocator());                 // constexpr since C++20
+    basic_string(const basic_string&, const Allocator&);                                        // constexpr since C++20
+    basic_string(basic_string&&, const Allocator&);                                             // constexpr since C++20
+
+    ~basic_string();                                                                            // constexpr since C++20
+
+    operator basic_string_view<charT, traits>() const noexcept;                                 // constexpr since C++20
+
+    basic_string& operator=(const basic_string& str);                                           // constexpr since C++20
+    template <class T>
+        basic_string& operator=(const T& t);                                                    // C++17, constexpr since C++20
+    basic_string& operator=(basic_string&& str)
+        noexcept(
+             allocator_type::propagate_on_container_move_assignment::value ||
+             allocator_type::is_always_equal::value );                                          // C++17, constexpr since C++20
+    basic_string& operator=(const value_type* s);                                               // constexpr since C++20
+    basic_string& operator=(nullptr_t) = delete; // C++23
+    basic_string& operator=(value_type c);                                                      // constexpr since C++20
+    basic_string& operator=(initializer_list<value_type>);                                      // constexpr since C++20
+
+    iterator       begin() noexcept;                                                            // constexpr since C++20
+    const_iterator begin() const noexcept;                                                      // constexpr since C++20
+    iterator       end() noexcept;                                                              // constexpr since C++20
+    const_iterator end() const noexcept;                                                        // constexpr since C++20
+
+    reverse_iterator       rbegin() noexcept;                                                   // constexpr since C++20
+    const_reverse_iterator rbegin() const noexcept;                                             // constexpr since C++20
+    reverse_iterator       rend() noexcept;                                                     // constexpr since C++20
+    const_reverse_iterator rend() const noexcept;                                               // constexpr since C++20
+
+    const_iterator         cbegin() const noexcept;                                             // constexpr since C++20
+    const_iterator         cend() const noexcept;                                               // constexpr since C++20
+    const_reverse_iterator crbegin() const noexcept;                                            // constexpr since C++20
+    const_reverse_iterator crend() const noexcept;                                              // constexpr since C++20
+
+    size_type size() const noexcept;                                                            // constexpr since C++20
+    size_type length() const noexcept;                                                          // constexpr since C++20
+    size_type max_size() const noexcept;                                                        // constexpr since C++20
+    size_type capacity() const noexcept;                                                        // constexpr since C++20
+
+    void resize(size_type n, value_type c);                                                     // constexpr since C++20
+    void resize(size_type n);                                                                   // constexpr since C++20
+
+    template<class Operation>
+    constexpr void resize_and_overwrite(size_type n, Operation op); // since C++23
+
+    void reserve(size_type res_arg);                                                            // constexpr since C++20
+    void reserve();                                                                             // deprecated in C++20, removed in C++26
+    void shrink_to_fit();                                                                       // constexpr since C++20
+    void clear() noexcept;                                                                      // constexpr since C++20
+    bool empty() const noexcept;                                                                // constexpr since C++20
+
+    const_reference operator[](size_type pos) const;                                            // constexpr since C++20
+    reference       operator[](size_type pos);                                                  // constexpr since C++20
+
+    const_reference at(size_type n) const;                                                      // constexpr since C++20
+    reference       at(size_type n);                                                            // constexpr since C++20
+
+    basic_string& operator+=(const basic_string& str);                                          // constexpr since C++20
+    template <class T>
+        basic_string& operator+=(const T& t);                                                   // C++17, constexpr since C++20
+    basic_string& operator+=(const value_type* s);                                              // constexpr since C++20
+    basic_string& operator+=(value_type c);                                                     // constexpr since C++20
+    basic_string& operator+=(initializer_list<value_type>);                                     // constexpr since C++20
+
+    basic_string& append(const basic_string& str);                                              // constexpr since C++20
+    template <class T>
+        basic_string& append(const T& t);                                                       // C++17, constexpr since C++20
+    basic_string& append(const basic_string& str, size_type pos, size_type n=npos);             // C++14, constexpr since C++20
+    template <class T>
+        basic_string& append(const T& t, size_type pos, size_type n=npos);                      // C++17, constexpr since C++20
+    basic_string& append(const value_type* s, size_type n);                                     // constexpr since C++20
+    basic_string& append(const value_type* s);                                                  // constexpr since C++20
+    basic_string& append(size_type n, value_type c);                                            // constexpr since C++20
+    template<class InputIterator>
+        basic_string& append(InputIterator first, InputIterator last);                          // constexpr since C++20
+    template<container-compatible-range<charT> R>
+      constexpr basic_string& append_range(R&& rg);                                             // C++23
+    basic_string& append(initializer_list<value_type>);                                         // constexpr since C++20
+
+    void push_back(value_type c);                                                               // constexpr since C++20
+    void pop_back();                                                                            // constexpr since C++20
+    reference       front();                                                                    // constexpr since C++20
+    const_reference front() const;                                                              // constexpr since C++20
+    reference       back();                                                                     // constexpr since C++20
+    const_reference back() const;                                                               // constexpr since C++20
+
+    basic_string& assign(const basic_string& str);                                              // constexpr since C++20
+    template <class T>
+        basic_string& assign(const T& t);                                                       // C++17, constexpr since C++20
+    basic_string& assign(basic_string&& str);                                                   // constexpr since C++20
+    basic_string& assign(const basic_string& str, size_type pos, size_type n=npos);             // C++14, constexpr since C++20
+    template <class T>
+        basic_string& assign(const T& t, size_type pos, size_type n=npos);                      // C++17, constexpr since C++20
+    basic_string& assign(const value_type* s, size_type n);                                     // constexpr since C++20
+    basic_string& assign(const value_type* s);                                                  // constexpr since C++20
+    basic_string& assign(size_type n, value_type c);                                            // constexpr since C++20
+    template<class InputIterator>
+        basic_string& assign(InputIterator first, InputIterator last);                          // constexpr since C++20
+    template<container-compatible-range<charT> R>
+      constexpr basic_string& assign_range(R&& rg);                                             // C++23
+    basic_string& assign(initializer_list<value_type>);                                         // constexpr since C++20
+
+    basic_string& insert(size_type pos1, const basic_string& str);                              // constexpr since C++20
+    template <class T>
+        basic_string& insert(size_type pos1, const T& t);                                       // constexpr since C++20
+    basic_string& insert(size_type pos1, const basic_string& str,
+                         size_type pos2, size_type n);                                          // constexpr since C++20
+    template <class T>
+        basic_string& insert(size_type pos1, const T& t, size_type pos2, size_type n);          // C++17, constexpr since C++20
+    basic_string& insert(size_type pos, const value_type* s, size_type n=npos);                 // C++14, constexpr since C++20
+    basic_string& insert(size_type pos, const value_type* s);                                   // constexpr since C++20
+    basic_string& insert(size_type pos, size_type n, value_type c);                             // constexpr since C++20
+    iterator      insert(const_iterator p, value_type c);                                       // constexpr since C++20
+    iterator      insert(const_iterator p, size_type n, value_type c);                          // constexpr since C++20
+    template<class InputIterator>
+        iterator insert(const_iterator p, InputIterator first, InputIterator last);             // constexpr since C++20
+    template<container-compatible-range<charT> R>
+      constexpr iterator insert_range(const_iterator p, R&& rg);                                // C++23
+    iterator      insert(const_iterator p, initializer_list<value_type>);                       // constexpr since C++20
+
+    basic_string& erase(size_type pos = 0, size_type n = npos);                                 // constexpr since C++20
+    iterator      erase(const_iterator position);                                               // constexpr since C++20
+    iterator      erase(const_iterator first, const_iterator last);                             // constexpr since C++20
+
+    basic_string& replace(size_type pos1, size_type n1, const basic_string& str);               // constexpr since C++20
+    template <class T>
+    basic_string& replace(size_type pos1, size_type n1, const T& t);                            // C++17, constexpr since C++20
+    basic_string& replace(size_type pos1, size_type n1, const basic_string& str,
+                          size_type pos2, size_type n2=npos);                                   // C++14, constexpr since C++20
+    template <class T>
+        basic_string& replace(size_type pos1, size_type n1, const T& t,
+                              size_type pos2, size_type n);                                     // C++17, constexpr since C++20
+    basic_string& replace(size_type pos, size_type n1, const value_type* s, size_type n2);      // constexpr since C++20
+    basic_string& replace(size_type pos, size_type n1, const value_type* s);                    // constexpr since C++20
+    basic_string& replace(size_type pos, size_type n1, size_type n2, value_type c);             // constexpr since C++20
+    basic_string& replace(const_iterator i1, const_iterator i2, const basic_string& str);       // constexpr since C++20
+    template <class T>
+        basic_string& replace(const_iterator i1, const_iterator i2, const T& t);                // C++17, constexpr since C++20
+    basic_string& replace(const_iterator i1, const_iterator i2, const value_type* s, size_type n); // constexpr since C++20
+    basic_string& replace(const_iterator i1, const_iterator i2, const value_type* s);           // constexpr since C++20
+    basic_string& replace(const_iterator i1, const_iterator i2, size_type n, value_type c);     // constexpr since C++20
+    template<class InputIterator>
+        basic_string& replace(const_iterator i1, const_iterator i2, InputIterator j1, InputIterator j2); // constexpr since C++20
+    template<container-compatible-range<charT> R>
+      constexpr basic_string& replace_with_range(const_iterator i1, const_iterator i2, R&& rg); // C++23
+    basic_string& replace(const_iterator i1, const_iterator i2, initializer_list<value_type>);  // constexpr since C++20
+
+    size_type copy(value_type* s, size_type n, size_type pos = 0) const;                        // constexpr since C++20
+    basic_string substr(size_type pos = 0, size_type n = npos) const;                           // constexpr in C++20, removed in C++23
+    basic_string substr(size_type pos = 0, size_type n = npos) const&;                          // since C++23
+    constexpr basic_string substr(size_type pos = 0, size_type n = npos) &&;                    // since C++23
+    void swap(basic_string& str)
+        noexcept(allocator_traits<allocator_type>::propagate_on_container_swap::value ||
+                 allocator_traits<allocator_type>::is_always_equal::value);                     // C++17, constexpr since C++20
+
+    const value_type* c_str() const noexcept;                                                   // constexpr since C++20
+    const value_type* data() const noexcept;                                                    // constexpr since C++20
+          value_type* data()       noexcept;                                                    // C++17, constexpr since C++20
+
+    allocator_type get_allocator() const noexcept;                                              // constexpr since C++20
+
+    size_type find(const basic_string& str, size_type pos = 0) const noexcept;                  // constexpr since C++20
+    template <class T>
+        size_type find(const T& t, size_type pos = 0) const noexcept;                           // C++17, noexcept as an extension, constexpr since C++20
+    size_type find(const value_type* s, size_type pos, size_type n) const noexcept;             // constexpr since C++20
+    size_type find(const value_type* s, size_type pos = 0) const noexcept;                      // constexpr since C++20
+    size_type find(value_type c, size_type pos = 0) const noexcept;                             // constexpr since C++20
+
+    size_type rfind(const basic_string& str, size_type pos = npos) const noexcept;              // constexpr since C++20
+    template <class T>
+        size_type rfind(const T& t, size_type pos = npos) const noexcept;                       // C++17, noexcept as an extension, constexpr since C++20
+    size_type rfind(const value_type* s, size_type pos, size_type n) const noexcept;            // constexpr since C++20
+    size_type rfind(const value_type* s, size_type pos = npos) const noexcept;                  // constexpr since C++20
+    size_type rfind(value_type c, size_type pos = npos) const noexcept;                         // constexpr since C++20
+
+    size_type find_first_of(const basic_string& str, size_type pos = 0) const noexcept;         // constexpr since C++20
+    template <class T>
+        size_type find_first_of(const T& t, size_type pos = 0) const noexcept;                  // C++17, noexcept as an extension, constexpr since C++20
+    size_type find_first_of(const value_type* s, size_type pos, size_type n) const noexcept;    // constexpr since C++20
+    size_type find_first_of(const value_type* s, size_type pos = 0) const noexcept;             // constexpr since C++20
+    size_type find_first_of(value_type c, size_type pos = 0) const noexcept;                    // constexpr since C++20
+
+    size_type find_last_of(const basic_string& str, size_type pos = npos) const noexcept;       // constexpr since C++20
+    template <class T>
+        size_type find_last_of(const T& t, size_type pos = npos) const noexcept noexcept;       // C++17, noexcept as an extension, constexpr since C++20
+    size_type find_last_of(const value_type* s, size_type pos, size_type n) const noexcept;     // constexpr since C++20
+    size_type find_last_of(const value_type* s, size_type pos = npos) const noexcept;           // constexpr since C++20
+    size_type find_last_of(value_type c, size_type pos = npos) const noexcept;                  // constexpr since C++20
+
+    size_type find_first_not_of(const basic_string& str, size_type pos = 0) const noexcept;     // constexpr since C++20
+    template <class T>
+        size_type find_first_not_of(const T& t, size_type pos = 0) const noexcept;              // C++17, noexcept as an extension, constexpr since C++20
+    size_type find_first_not_of(const value_type* s, size_type pos, size_type n) const noexcept; // constexpr since C++20
+    size_type find_first_not_of(const value_type* s, size_type pos = 0) const noexcept;         // constexpr since C++20
+    size_type find_first_not_of(value_type c, size_type pos = 0) const noexcept;                // constexpr since C++20
+
+    size_type find_last_not_of(const basic_string& str, size_type pos = npos) const noexcept;   // constexpr since C++20
+    template <class T>
+        size_type find_last_not_of(const T& t, size_type pos = npos) const noexcept;            // C++17, noexcept as an extension, constexpr since C++20
+    size_type find_last_not_of(const value_type* s, size_type pos, size_type n) const noexcept; // constexpr since C++20
+    size_type find_last_not_of(const value_type* s, size_type pos = npos) const noexcept;       // constexpr since C++20
+    size_type find_last_not_of(value_type c, size_type pos = npos) const noexcept;              // constexpr since C++20
+
+    int compare(const basic_string& str) const noexcept;                                        // constexpr since C++20
+    template <class T>
+        int compare(const T& t) const noexcept;                                                 // C++17, noexcept as an extension, constexpr since C++20
+    int compare(size_type pos1, size_type n1, const basic_string& str) const;                   // constexpr since C++20
+    template <class T>
+        int compare(size_type pos1, size_type n1, const T& t) const;                            // C++17, constexpr since C++20
+    int compare(size_type pos1, size_type n1, const basic_string& str,
+                size_type pos2, size_type n2=npos) const;                                       // C++14, constexpr since C++20
+    template <class T>
+        int compare(size_type pos1, size_type n1, const T& t,
+                    size_type pos2, size_type n2=npos) const;                                   // C++17, constexpr since C++20
+    int compare(const value_type* s) const noexcept;                                            // constexpr since C++20
+    int compare(size_type pos1, size_type n1, const value_type* s) const;                       // constexpr since C++20
+    int compare(size_type pos1, size_type n1, const value_type* s, size_type n2) const;         // constexpr since C++20
+
+    constexpr bool starts_with(basic_string_view<charT, traits> sv) const noexcept;             // C++20
+    constexpr bool starts_with(charT c) const noexcept;                                         // C++20
+    constexpr bool starts_with(const charT* s) const;                                           // C++20
+    constexpr bool ends_with(basic_string_view<charT, traits> sv) const noexcept;               // C++20
+    constexpr bool ends_with(charT c) const noexcept;                                           // C++20
+    constexpr bool ends_with(const charT* s) const;                                             // C++20
+
+    constexpr bool contains(basic_string_view<charT, traits> sv) const noexcept;                // C++23
+    constexpr bool contains(charT c) const noexcept;                                            // C++23
+    constexpr bool contains(const charT* s) const;                                              // C++23
+};
+
+template<class InputIterator,
+         class Allocator = allocator<typename iterator_traits<InputIterator>::value_type>>
+basic_string(InputIterator, InputIterator, Allocator = Allocator())
+   -> basic_string<typename iterator_traits<InputIterator>::value_type,
+                  char_traits<typename iterator_traits<InputIterator>::value_type>,
+                  Allocator>;   // C++17
+
+template<ranges::input_range R,
+         class Allocator = allocator<ranges::range_value_t<R>>>
+  basic_string(from_range_t, R&&, Allocator = Allocator())
+    -> basic_string<ranges::range_value_t<R>, char_traits<ranges::range_value_t<R>>,
+                    Allocator>; // C++23
+
+template<class charT,
+         class traits,
+         class Allocator = allocator<charT>>
+  explicit basic_string(basic_string_view<charT, traits>, const Allocator& = Allocator())
+    -> basic_string<charT, traits, Allocator>; // C++17
+
+template<class charT,
+         class traits,
+         class Allocator = allocator<charT>>
+  basic_string(basic_string_view<charT, traits>,
+                typename see below::size_type, typename see below::size_type,
+                const Allocator& = Allocator())
+    -> basic_string<charT, traits, Allocator>; // C++17
+
+template<class charT, class traits, class Allocator>
+basic_string<charT, traits, Allocator>
+operator+(const basic_string<charT, traits, Allocator>& lhs,
+          const basic_string<charT, traits, Allocator>& rhs);                                   // constexpr since C++20
+
+template<class charT, class traits, class Allocator>
+basic_string<charT, traits, Allocator>
+operator+(const charT* lhs , const basic_string<charT,traits,Allocator>&rhs);                   // constexpr since C++20
+
+template<class charT, class traits, class Allocator>
+basic_string<charT, traits, Allocator>
+operator+(charT lhs, const basic_string<charT,traits,Allocator>& rhs);                          // constexpr since C++20
+
+template<class charT, class traits, class Allocator>
+basic_string<charT, traits, Allocator>
+operator+(const basic_string<charT, traits, Allocator>& lhs, const charT* rhs);                 // constexpr since C++20
+
+template<class charT, class traits, class Allocator>
+basic_string<charT, traits, Allocator>
+operator+(const basic_string<charT, traits, Allocator>& lhs, charT rhs);                        // constexpr since C++20
+
+template<class charT, class traits, class Allocator>
+  constexpr basic_string<charT, traits, Allocator>
+    operator+(const basic_string<charT, traits, Allocator>& lhs,
+              type_identity_t<basic_string_view<charT, traits>> rhs);                           // Since C++26
+template<class charT, class traits, class Allocator>
+  constexpr basic_string<charT, traits, Allocator>
+    operator+(basic_string<charT, traits, Allocator>&& lhs,
+              type_identity_t<basic_string_view<charT, traits>> rhs);                           // Since C++26
+template<class charT, class traits, class Allocator>
+  constexpr basic_string<charT, traits, Allocator>
+    operator+(type_identity_t<basic_string_view<charT, traits>> lhs,
+              const basic_string<charT, traits, Allocator>& rhs);                               // Since C++26
+template<class charT, class traits, class Allocator>
+  constexpr basic_string<charT, traits, Allocator>
+    operator+(type_identity_t<basic_string_view<charT, traits>> lhs,
+              basic_string<charT, traits, Allocator>&& rhs);                                    // Since C++26
+
+
+template<class charT, class traits, class Allocator>
+bool operator==(const basic_string<charT, traits, Allocator>& lhs,
+                const basic_string<charT, traits, Allocator>& rhs) noexcept;                    // constexpr since C++20
+
+template<class charT, class traits, class Allocator>
+bool operator==(const charT* lhs, const basic_string<charT, traits, Allocator>& rhs) noexcept;  // removed in C++20
+
+template<class charT, class traits, class Allocator>
+bool operator==(const basic_string<charT,traits,Allocator>& lhs, const charT* rhs) noexcept;    // constexpr since C++20
+
+template<class charT, class traits, class Allocator>
+bool operator!=(const basic_string<charT,traits,Allocator>& lhs,
+                const basic_string<charT, traits, Allocator>& rhs) noexcept;                    // removed in C++20
+
+template<class charT, class traits, class Allocator>
+bool operator!=(const charT* lhs, const basic_string<charT, traits, Allocator>& rhs) noexcept;  // removed in C++20
+
+template<class charT, class traits, class Allocator>
+bool operator!=(const basic_string<charT, traits, Allocator>& lhs, const charT* rhs) noexcept;  // removed in C++20
+
+template<class charT, class traits, class Allocator>
+bool operator< (const basic_string<charT, traits, Allocator>& lhs,
+                const basic_string<charT, traits, Allocator>& rhs) noexcept;                    // removed in C++20
+
+template<class charT, class traits, class Allocator>
+bool operator< (const basic_string<charT, traits, Allocator>& lhs, const charT* rhs) noexcept;  // removed in C++20
+
+template<class charT, class traits, class Allocator>
+bool operator< (const charT* lhs, const basic_string<charT, traits, Allocator>& rhs) noexcept;  // removed in C++20
+
+template<class charT, class traits, class Allocator>
+bool operator> (const basic_string<charT, traits, Allocator>& lhs,
+                const basic_string<charT, traits, Allocator>& rhs) noexcept;                    // removed in C++20
+
+template<class charT, class traits, class Allocator>
+bool operator> (const basic_string<charT, traits, Allocator>& lhs, const charT* rhs) noexcept;  // removed in C++20
+
+template<class charT, class traits, class Allocator>
+bool operator> (const charT* lhs, const basic_string<charT, traits, Allocator>& rhs) noexcept;  // removed in C++20
+
+template<class charT, class traits, class Allocator>
+bool operator<=(const basic_string<charT, traits, Allocator>& lhs,
+                const basic_string<charT, traits, Allocator>& rhs) noexcept;                    // removed in C++20
+
+template<class charT, class traits, class Allocator>
+bool operator<=(const basic_string<charT, traits, Allocator>& lhs, const charT* rhs) noexcept;  // removed in C++20
+
+template<class charT, class traits, class Allocator>
+bool operator<=(const charT* lhs, const basic_string<charT, traits, Allocator>& rhs) noexcept;  // removed in C++20
+
+template<class charT, class traits, class Allocator>
+bool operator>=(const basic_string<charT, traits, Allocator>& lhs,
+                const basic_string<charT, traits, Allocator>& rhs) noexcept;                    // removed in C++20
+
+template<class charT, class traits, class Allocator>
+bool operator>=(const basic_string<charT, traits, Allocator>& lhs, const charT* rhs) noexcept;  // removed in C++20
+
+template<class charT, class traits, class Allocator>
+bool operator>=(const charT* lhs, const basic_string<charT, traits, Allocator>& rhs) noexcept;  // removed in C++20
+
+template<class charT, class traits, class Allocator>                                            // since C++20
+constexpr see below operator<=>(const basic_string<charT, traits, Allocator>& lhs,
+                                const basic_string<charT, traits, Allocator>& rhs) noexcept;
+
+template<class charT, class traits, class Allocator>                                            // since C++20
+constexpr see below operator<=>(const basic_string<charT, traits, Allocator>& lhs,
+                                const charT* rhs) noexcept;
+
+template<class charT, class traits, class Allocator>
+void swap(basic_string<charT, traits, Allocator>& lhs,
+          basic_string<charT, traits, Allocator>& rhs)
+            noexcept(noexcept(lhs.swap(rhs)));                                                  // constexpr since C++20
+
+template<class charT, class traits, class Allocator>
+basic_istream<charT, traits>&
+operator>>(basic_istream<charT, traits>& is, basic_string<charT, traits, Allocator>& str);
+
+template<class charT, class traits, class Allocator>
+basic_ostream<charT, traits>&
+operator<<(basic_ostream<charT, traits>& os, const basic_string<charT, traits, Allocator>& str);
+
+template<class charT, class traits, class Allocator>
+basic_istream<charT, traits>&
+getline(basic_istream<charT, traits>& is, basic_string<charT, traits, Allocator>& str,
+        charT delim);
+
+template<class charT, class traits, class Allocator>
+basic_istream<charT, traits>&
+getline(basic_istream<charT, traits>& is, basic_string<charT, traits, Allocator>& str);
+
+template<class charT, class traits, class Allocator, class U>
+typename basic_string<charT, traits, Allocator>::size_type
+erase(basic_string<charT, traits, Allocator>& c, const U& value);    // C++20
+template<class charT, class traits, class Allocator, class Predicate>
+typename basic_string<charT, traits, Allocator>::size_type
+erase_if(basic_string<charT, traits, Allocator>& c, Predicate pred); // C++20
+
+typedef basic_string<char>    string;
+typedef basic_string<wchar_t> wstring;
+typedef basic_string<char8_t> u8string; // C++20
+typedef basic_string<char16_t> u16string;
+typedef basic_string<char32_t> u32string;
+
+int                stoi  (const string& str, size_t* idx = nullptr, int base = 10);
+long               stol  (const string& str, size_t* idx = nullptr, int base = 10);
+unsigned long      stoul (const string& str, size_t* idx = nullptr, int base = 10);
+long long          stoll (const string& str, size_t* idx = nullptr, int base = 10);
+unsigned long long stoull(const string& str, size_t* idx = nullptr, int base = 10);
+
+float       stof (const string& str, size_t* idx = nullptr);
+double      stod (const string& str, size_t* idx = nullptr);
+long double stold(const string& str, size_t* idx = nullptr);
+
+string to_string(int val);
+string to_string(unsigned val);
+string to_string(long val);
+string to_string(unsigned long val);
+string to_string(long long val);
+string to_string(unsigned long long val);
+string to_string(float val);
+string to_string(double val);
+string to_string(long double val);
+
+int                stoi  (const wstring& str, size_t* idx = nullptr, int base = 10);
+long               stol  (const wstring& str, size_t* idx = nullptr, int base = 10);
+unsigned long      stoul (const wstring& str, size_t* idx = nullptr, int base = 10);
+long long          stoll (const wstring& str, size_t* idx = nullptr, int base = 10);
+unsigned long long stoull(const wstring& str, size_t* idx = nullptr, int base = 10);
+
+float       stof (const wstring& str, size_t* idx = nullptr);
+double      stod (const wstring& str, size_t* idx = nullptr);
+long double stold(const wstring& str, size_t* idx = nullptr);
+
+wstring to_wstring(int val);
+wstring to_wstring(unsigned val);
+wstring to_wstring(long val);
+wstring to_wstring(unsigned long val);
+wstring to_wstring(long long val);
+wstring to_wstring(unsigned long long val);
+wstring to_wstring(float val);
+wstring to_wstring(double val);
+wstring to_wstring(long double val);
+
+template <> struct hash<string>;
+template <> struct hash<u8string>; // C++20
+template <> struct hash<u16string>;
+template <> struct hash<u32string>;
+template <> struct hash<wstring>;
+
+basic_string<char>     operator""s( const char *str,     size_t len );           // C++14, constexpr since C++20
+basic_string<wchar_t>  operator""s( const wchar_t *str,  size_t len );           // C++14, constexpr since C++20
+constexpr basic_string<char8_t>  operator""s( const char8_t *str,  size_t len ); // C++20
+basic_string<char16_t> operator""s( const char16_t *str, size_t len );           // C++14, constexpr since C++20
+basic_string<char32_t> operator""s( const char32_t *str, size_t len );           // C++14, constexpr since C++20
+
+}  // std
+
+*/
+
+// clang-format on
+
+#include <__algorithm/max.h>
+#include <__algorithm/min.h>
+#include <__algorithm/remove.h>
+#include <__algorithm/remove_if.h>
+#include <__assert>
+#include <__config>
+#include <__debug_utils/sanitizers.h>
+#include <__format/enable_insertable.h>
+#include <__functional/hash.h>
+#include <__functional/unary_function.h>
+#include <__fwd/string.h>
+#include <__ios/fpos.h>
+#include <__iterator/bounded_iter.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/reverse_iterator.h>
+#include <__iterator/wrap_iter.h>
+#include <__memory/addressof.h>
+#include <__memory/allocate_at_least.h>
+#include <__memory/allocator.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/compressed_pair.h>
+#include <__memory/construct_at.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/swap_allocator.h>
+#include <__memory_resource/polymorphic_allocator.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/container_compatible_range.h>
+#include <__ranges/from_range.h>
+#include <__ranges/size.h>
+#include <__string/char_traits.h>
+#include <__string/extern_template_lists.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/is_allocator.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_standard_layout.h>
+#include <__type_traits/is_trivial.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__type_traits/noexcept_move_assign_container.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/void_t.h>
+#include <__utility/auto_cast.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/is_pointer_in_range.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+#include <__utility/unreachable.h>
+#include <climits>
+#include <cstdio> // EOF
+#include <cstring>
+#include <limits>
+#include <stdexcept>
+#include <string_view>
+#include <version>
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+#  include <cwchar>
+#endif
+
+// standard-mandated includes
+
+// [iterator.range]
+#include <__iterator/access.h>
+#include <__iterator/data.h>
+#include <__iterator/empty.h>
+#include <__iterator/reverse_access.h>
+#include <__iterator/size.h>
+
+// [string.syn]
+#include <compare>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+#  define _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS __attribute__((__no_sanitize__("address")))
+// This macro disables AddressSanitizer (ASan) instrumentation for a specific function,
+// allowing memory accesses that would normally trigger ASan errors to proceed without crashing.
+// This is useful for accessing parts of objects memory, which should not be accessed,
+// such as unused bytes in short strings, that should never be accessed
+// by other parts of the program.
+#else
+#  define _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// basic_string
+
+template <class _CharT, class _Traits, class _Allocator>
+basic_string<_CharT, _Traits, _Allocator> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+operator+(const basic_string<_CharT, _Traits, _Allocator>& __x, const basic_string<_CharT, _Traits, _Allocator>& __y);
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDDEN _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(const _CharT* __x, const basic_string<_CharT, _Traits, _Allocator>& __y);
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(_CharT __x, const basic_string<_CharT, _Traits, _Allocator>& __y);
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(const basic_string<_CharT, _Traits, _Allocator>& __x, const _CharT* __y);
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(const basic_string<_CharT, _Traits, _Allocator>& __x, _CharT __y);
+
+#if _LIBCPP_STD_VER >= 26
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI constexpr basic_string<_CharT, _Traits, _Allocator>
+operator+(const basic_string<_CharT, _Traits, _Allocator>& __lhs,
+          type_identity_t<basic_string_view<_CharT, _Traits>> __rhs);
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI constexpr basic_string<_CharT, _Traits, _Allocator>
+operator+(basic_string<_CharT, _Traits, _Allocator>&& __lhs, type_identity_t<basic_string_view<_CharT, _Traits>> __rhs);
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI constexpr basic_string<_CharT, _Traits, _Allocator>
+operator+(type_identity_t<basic_string_view<_CharT, _Traits>> __lhs,
+          const basic_string<_CharT, _Traits, _Allocator>& __rhs);
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI constexpr basic_string<_CharT, _Traits, _Allocator>
+operator+(type_identity_t<basic_string_view<_CharT, _Traits>> __lhs, basic_string<_CharT, _Traits, _Allocator>&& __rhs);
+
+#endif
+
+extern template _LIBCPP_EXPORTED_FROM_ABI string operator+
+    <char, char_traits<char>, allocator<char> >(char const*, string const&);
+
+template <class _Iter>
+struct __string_is_trivial_iterator : public false_type {};
+
+template <class _Tp>
+struct __string_is_trivial_iterator<_Tp*> : public is_arithmetic<_Tp> {};
+
+template <class _Iter>
+struct __string_is_trivial_iterator<__wrap_iter<_Iter> > : public __string_is_trivial_iterator<_Iter> {};
+
+template <class _CharT, class _Traits, class _Tp>
+struct __can_be_converted_to_string_view
+    : public _BoolConstant< is_convertible<const _Tp&, basic_string_view<_CharT, _Traits> >::value &&
+                            !is_convertible<const _Tp&, const _CharT*>::value > {};
+
+struct __uninitialized_size_tag {};
+struct __init_with_sentinel_tag {};
+
+template <class _CharT, class _Traits, class _Allocator>
+class basic_string {
+private:
+  using __default_allocator_type = allocator<_CharT>;
+
+public:
+  typedef basic_string __self;
+  typedef basic_string_view<_CharT, _Traits> __self_view;
+  typedef _Traits traits_type;
+  typedef _CharT value_type;
+  typedef _Allocator allocator_type;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+  typedef typename __alloc_traits::size_type size_type;
+  typedef typename __alloc_traits::difference_type difference_type;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+  typedef typename __alloc_traits::pointer pointer;
+  typedef typename __alloc_traits::const_pointer const_pointer;
+
+  // A basic_string contains the following members which may be trivially relocatable:
+  // - pointer: is currently assumed to be trivially relocatable, but is still checked in case that changes
+  // - size_type: is always trivially relocatable, since it has to be an integral type
+  // - value_type: is always trivially relocatable, since it has to be trivial
+  // - unsigned char: is a fundamental type, so it's trivially relocatable
+  // - allocator_type: may or may not be trivially relocatable, so it's checked
+  //
+  // This string implementation doesn't contain any references into itself. It only contains a bit that says whether
+  // it is in small or large string mode, so the entire structure is trivially relocatable if its members are.
+#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+  // When compiling with AddressSanitizer (ASan), basic_string cannot be trivially
+  // relocatable. Because the object's memory might be poisoned when its content
+  // is kept inside objects memory (short string optimization), instead of in allocated
+  // external memory. In such cases, the destructor is responsible for unpoisoning
+  // the memory to avoid triggering false positives.
+  // Therefore it's crucial to ensure the destructor is called.
+  using __trivially_relocatable = void;
+#else
+  using __trivially_relocatable = __conditional_t<
+      __libcpp_is_trivially_relocatable<allocator_type>::value && __libcpp_is_trivially_relocatable<pointer>::value,
+      basic_string,
+      void>;
+#endif
+#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pointer __asan_volatile_wrapper(pointer const& __ptr) const {
+    if (__libcpp_is_constant_evaluated())
+      return __ptr;
+
+    pointer volatile __copy_ptr = __ptr;
+
+    return const_cast<pointer&>(__copy_ptr);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_pointer
+  __asan_volatile_wrapper(const_pointer const& __ptr) const {
+    if (__libcpp_is_constant_evaluated())
+      return __ptr;
+
+    const_pointer volatile __copy_ptr = __ptr;
+
+    return const_cast<const_pointer&>(__copy_ptr);
+  }
+#  define _LIBCPP_ASAN_VOLATILE_WRAPPER(PTR) __asan_volatile_wrapper(PTR)
+#else
+#  define _LIBCPP_ASAN_VOLATILE_WRAPPER(PTR) PTR
+#endif
+
+  static_assert(!is_array<value_type>::value, "Character type of basic_string must not be an array");
+  static_assert(is_standard_layout<value_type>::value, "Character type of basic_string must be standard-layout");
+  static_assert(is_trivial<value_type>::value, "Character type of basic_string must be trivial");
+  static_assert(is_same<_CharT, typename traits_type::char_type>::value,
+                "traits_type::char_type must be the same type as CharT");
+  static_assert(is_same<typename allocator_type::value_type, value_type>::value,
+                "Allocator::value_type must be same type as value_type");
+  static_assert(__check_valid_allocator<allocator_type>::value, "");
+
+#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
+  // Users might provide custom allocators, and prior to C++20 we have no existing way to detect whether the allocator's
+  // pointer type is contiguous (though it has to be by the Standard). Using the wrapper type ensures the iterator is
+  // considered contiguous.
+  typedef __bounded_iter<__wrap_iter<pointer>> iterator;
+  typedef __bounded_iter<__wrap_iter<const_pointer>> const_iterator;
+#else
+  typedef __wrap_iter<pointer> iterator;
+  typedef __wrap_iter<const_pointer> const_iterator;
+#endif
+  typedef std::reverse_iterator<iterator> reverse_iterator;
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+
+private:
+  static_assert(CHAR_BIT == 8, "This implementation assumes that one byte contains 8 bits");
+
+#ifdef _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
+
+  struct __long {
+    pointer __data_;
+    size_type __size_;
+    size_type __cap_ : sizeof(size_type) * CHAR_BIT - 1;
+    size_type __is_long_ : 1;
+  };
+
+  enum { __min_cap = (sizeof(__long) - 1) / sizeof(value_type) > 2 ? (sizeof(__long) - 1) / sizeof(value_type) : 2 };
+
+  struct __short {
+    value_type __data_[__min_cap];
+    unsigned char __padding_[sizeof(value_type) - 1];
+    unsigned char __size_    : 7;
+    unsigned char __is_long_ : 1;
+  };
+
+  // The __endian_factor is required because the field we use to store the size
+  // has one fewer bit than it would if it were not a bitfield.
+  //
+  // If the LSB is used to store the short-flag in the short string representation,
+  // we have to multiply the size by two when it is stored and divide it by two when
+  // it is loaded to make sure that we always store an even number. In the long string
+  // representation, we can ignore this because we can assume that we always allocate
+  // an even amount of value_types.
+  //
+  // If the MSB is used for the short-flag, the max_size() is numeric_limits<size_type>::max() / 2.
+  // This does not impact the short string representation, since we never need the MSB
+  // for representing the size of a short string anyway.
+
+#  ifdef _LIBCPP_BIG_ENDIAN
+  static const size_type __endian_factor = 2;
+#  else
+  static const size_type __endian_factor = 1;
+#  endif
+
+#else // _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
+
+#  ifdef _LIBCPP_BIG_ENDIAN
+  static const size_type __endian_factor = 1;
+#  else
+  static const size_type __endian_factor = 2;
+#  endif
+
+  // Attribute 'packed' is used to keep the layout compatible with the
+  // previous definition that did not use bit fields. This is because on
+  // some platforms bit fields have a default size rather than the actual
+  // size used, e.g., it is 4 bytes on AIX. See D128285 for details.
+  struct __long {
+    struct _LIBCPP_PACKED {
+      size_type __is_long_ : 1;
+      size_type __cap_ : sizeof(size_type) * CHAR_BIT - 1;
+    };
+    size_type __size_;
+    pointer __data_;
+  };
+
+  enum { __min_cap = (sizeof(__long) - 1) / sizeof(value_type) > 2 ? (sizeof(__long) - 1) / sizeof(value_type) : 2 };
+
+  struct __short {
+    struct _LIBCPP_PACKED {
+      unsigned char __is_long_ : 1;
+      unsigned char __size_    : 7;
+    };
+    char __padding_[sizeof(value_type) - 1];
+    value_type __data_[__min_cap];
+  };
+
+#endif // _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
+
+  static_assert(sizeof(__short) == (sizeof(value_type) * (__min_cap + 1)), "__short has an unexpected size.");
+
+  union __rep {
+    __short __s;
+    __long __l;
+  };
+
+  __compressed_pair<__rep, allocator_type> __r_;
+
+  // Construct a string with the given allocator and enough storage to hold `__size` characters, but
+  // don't initialize the characters. The contents of the string, including the null terminator, must be
+  // initialized separately.
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit basic_string(
+      __uninitialized_size_tag, size_type __size, const allocator_type& __a)
+      : __r_(__default_init_tag(), __a) {
+    if (__size > max_size())
+      __throw_length_error();
+    if (__fits_in_sso(__size)) {
+      __r_.first() = __rep();
+      __set_short_size(__size);
+    } else {
+      auto __capacity   = __recommend(__size) + 1;
+      auto __allocation = __alloc_traits::allocate(__alloc(), __capacity);
+      __begin_lifetime(__allocation, __capacity);
+      __set_long_cap(__capacity);
+      __set_long_pointer(__allocation);
+      __set_long_size(__size);
+    }
+    __annotate_new(__size);
+  }
+
+  template <class _Iter, class _Sent>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  basic_string(__init_with_sentinel_tag, _Iter __first, _Sent __last, const allocator_type& __a)
+      : __r_(__default_init_tag(), __a) {
+    __init_with_sentinel(std::move(__first), std::move(__last));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator __make_iterator(pointer __p) {
+#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
+    // Bound the iterator according to the size (and not the capacity, unlike vector).
+    //
+    // By the Standard, string iterators are generally not guaranteed to stay valid when the container is modified,
+    // regardless of whether reallocation occurs. This allows us to check for out-of-bounds accesses using logical size,
+    // a stricter check, since correct code can never rely on being able to access newly-added elements via an existing
+    // iterator.
+    return std::__make_bounded_iter(
+        std::__wrap_iter<pointer>(__p),
+        std::__wrap_iter<pointer>(__get_pointer()),
+        std::__wrap_iter<pointer>(__get_pointer() + size()));
+#else
+    return iterator(__p);
+#endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator __make_const_iterator(const_pointer __p) const {
+#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
+    // Bound the iterator according to the size (and not the capacity, unlike vector).
+    return std::__make_bounded_iter(
+        std::__wrap_iter<const_pointer>(__p),
+        std::__wrap_iter<const_pointer>(__get_pointer()),
+        std::__wrap_iter<const_pointer>(__get_pointer() + size()));
+#else
+    return const_iterator(__p);
+#endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
+  }
+
+public:
+  _LIBCPP_TEMPLATE_DATA_VIS static const size_type npos = -1;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string()
+      _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
+      : __r_(__value_init_tag(), __default_init_tag()) {
+    __annotate_new(0);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit basic_string(const allocator_type& __a)
+#if _LIBCPP_STD_VER <= 14
+      _NOEXCEPT_(is_nothrow_copy_constructible<allocator_type>::value)
+#else
+      _NOEXCEPT
+#endif
+      : __r_(__value_init_tag(), __a) {
+    __annotate_new(0);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS basic_string(const basic_string& __str)
+      : __r_(__default_init_tag(), __alloc_traits::select_on_container_copy_construction(__str.__alloc())) {
+    if (!__str.__is_long()) {
+      __r_.first() = __str.__r_.first();
+      __annotate_new(__get_short_size());
+    } else
+      __init_copy_ctor_external(std::__to_address(__str.__get_long_pointer()), __str.__get_long_size());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
+  basic_string(const basic_string& __str, const allocator_type& __a)
+      : __r_(__default_init_tag(), __a) {
+    if (!__str.__is_long()) {
+      __r_.first() = __str.__r_.first();
+      __annotate_new(__get_short_size());
+    } else
+      __init_copy_ctor_external(std::__to_address(__str.__get_long_pointer()), __str.__get_long_size());
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(basic_string&& __str)
+#  if _LIBCPP_STD_VER <= 14
+      _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value)
+#  else
+      _NOEXCEPT
+#  endif
+      // Turning off ASan instrumentation for variable initialization with _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
+      // does not work consistently during initialization of __r_, so we instead unpoison __str's memory manually first.
+      // __str's memory needs to be unpoisoned only in the case where it's a short string.
+      : __r_([](basic_string& __s) -> decltype(__s.__r_)&& {
+          if (!__s.__is_long())
+            __s.__annotate_delete();
+          return std::move(__s.__r_);
+        }(__str)) {
+    __str.__r_.first() = __rep();
+    __str.__annotate_new(0);
+    if (!__is_long())
+      __annotate_new(size());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(basic_string&& __str, const allocator_type& __a)
+      : __r_(__default_init_tag(), __a) {
+    if (__str.__is_long() && __a != __str.__alloc()) // copy, not move
+      __init(std::__to_address(__str.__get_long_pointer()), __str.__get_long_size());
+    else {
+      if (__libcpp_is_constant_evaluated())
+        __r_.first() = __rep();
+      if (!__str.__is_long())
+        __str.__annotate_delete();
+      __r_.first()       = __str.__r_.first();
+      __str.__r_.first() = __rep();
+      __str.__annotate_new(0);
+      if (!__is_long() && this != &__str)
+        __annotate_new(size());
+    }
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  template <__enable_if_t<__is_allocator<_Allocator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(const _CharT* __s)
+      : __r_(__default_init_tag(), __default_init_tag()) {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "basic_string(const char*) detected nullptr");
+    __init(__s, traits_type::length(__s));
+  }
+
+  template <__enable_if_t<__is_allocator<_Allocator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(const _CharT* __s, const _Allocator& __a)
+      : __r_(__default_init_tag(), __a) {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "basic_string(const char*, allocator) detected nullptr");
+    __init(__s, traits_type::length(__s));
+  }
+
+#if _LIBCPP_STD_VER >= 23
+  basic_string(nullptr_t) = delete;
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(const _CharT* __s, size_type __n)
+      : __r_(__default_init_tag(), __default_init_tag()) {
+    _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "basic_string(const char*, n) detected nullptr");
+    __init(__s, __n);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  basic_string(const _CharT* __s, size_type __n, const _Allocator& __a)
+      : __r_(__default_init_tag(), __a) {
+    _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "basic_string(const char*, n, allocator) detected nullptr");
+    __init(__s, __n);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(size_type __n, _CharT __c)
+      : __r_(__default_init_tag(), __default_init_tag()) {
+    __init(__n, __c);
+  }
+
+#if _LIBCPP_STD_VER >= 23
+  _LIBCPP_HIDE_FROM_ABI constexpr basic_string(
+      basic_string&& __str, size_type __pos, const _Allocator& __alloc = _Allocator())
+      : basic_string(std::move(__str), __pos, npos, __alloc) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr basic_string(
+      basic_string&& __str, size_type __pos, size_type __n, const _Allocator& __alloc = _Allocator())
+      : __r_(__default_init_tag(), __alloc) {
+    if (__pos > __str.size())
+      __throw_out_of_range();
+
+    auto __len = std::min<size_type>(__n, __str.size() - __pos);
+    if (__alloc_traits::is_always_equal::value || __alloc == __str.__alloc()) {
+      __move_assign(std::move(__str), __pos, __len);
+    } else {
+      // Perform a copy because the allocators are not compatible.
+      __init(__str.data() + __pos, __len);
+    }
+  }
+#endif
+
+  template <__enable_if_t<__is_allocator<_Allocator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(size_type __n, _CharT __c, const _Allocator& __a)
+      : __r_(__default_init_tag(), __a) {
+    __init(__n, __c);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20
+  basic_string(const basic_string& __str, size_type __pos, size_type __n, const _Allocator& __a = _Allocator())
+      : __r_(__default_init_tag(), __a) {
+    size_type __str_sz = __str.size();
+    if (__pos > __str_sz)
+      __throw_out_of_range();
+    __init(__str.data() + __pos, std::min(__n, __str_sz - __pos));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  basic_string(const basic_string& __str, size_type __pos, const _Allocator& __a = _Allocator())
+      : __r_(__default_init_tag(), __a) {
+    size_type __str_sz = __str.size();
+    if (__pos > __str_sz)
+      __throw_out_of_range();
+    __init(__str.data() + __pos, __str_sz - __pos);
+  }
+
+  template <class _Tp,
+            __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                              !__is_same_uncvref<_Tp, basic_string>::value,
+                          int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20
+  basic_string(const _Tp& __t, size_type __pos, size_type __n, const allocator_type& __a = allocator_type())
+      : __r_(__default_init_tag(), __a) {
+    __self_view __sv0 = __t;
+    __self_view __sv  = __sv0.substr(__pos, __n);
+    __init(__sv.data(), __sv.size());
+  }
+
+  template <class _Tp,
+            __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                              !__is_same_uncvref<_Tp, basic_string>::value,
+                          int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit basic_string(const _Tp& __t)
+      : __r_(__default_init_tag(), __default_init_tag()) {
+    __self_view __sv = __t;
+    __init(__sv.data(), __sv.size());
+  }
+
+  template <class _Tp,
+            __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                              !__is_same_uncvref<_Tp, basic_string>::value,
+                          int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit basic_string(const _Tp& __t, const allocator_type& __a)
+      : __r_(__default_init_tag(), __a) {
+    __self_view __sv = __t;
+    __init(__sv.data(), __sv.size());
+  }
+
+  template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(_InputIterator __first, _InputIterator __last)
+      : __r_(__default_init_tag(), __default_init_tag()) {
+    __init(__first, __last);
+  }
+
+  template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  basic_string(_InputIterator __first, _InputIterator __last, const allocator_type& __a)
+      : __r_(__default_init_tag(), __a) {
+    __init(__first, __last);
+  }
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_CharT> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr basic_string(
+      from_range_t, _Range&& __range, const allocator_type& __a = allocator_type())
+      : __r_(__default_init_tag(), __a) {
+    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      __init_with_size(ranges::begin(__range), ranges::end(__range), ranges::distance(__range));
+    } else {
+      __init_with_sentinel(ranges::begin(__range), ranges::end(__range));
+    }
+  }
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(initializer_list<_CharT> __il)
+      : __r_(__default_init_tag(), __default_init_tag()) {
+    __init(__il.begin(), __il.end());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(initializer_list<_CharT> __il, const _Allocator& __a)
+      : __r_(__default_init_tag(), __a) {
+    __init(__il.begin(), __il.end());
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  inline _LIBCPP_CONSTEXPR_SINCE_CXX20 ~basic_string() {
+    __annotate_delete();
+    if (__is_long())
+      __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 operator __self_view() const _NOEXCEPT {
+    return __self_view(data(), size());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS basic_string&
+  operator=(const basic_string& __str);
+
+  template <class _Tp,
+            __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                              !__is_same_uncvref<_Tp, basic_string>::value,
+                          int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(const _Tp& __t) {
+    __self_view __sv = __t;
+    return assign(__sv);
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  operator=(basic_string&& __str) noexcept(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value) {
+    __move_assign(__str, integral_constant<bool, __alloc_traits::propagate_on_container_move_assignment::value>());
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(initializer_list<value_type> __il) {
+    return assign(__il.begin(), __il.size());
+  }
+#endif
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(const value_type* __s) {
+    return assign(__s);
+  }
+#if _LIBCPP_STD_VER >= 23
+  basic_string& operator=(nullptr_t) = delete;
+#endif
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(value_type __c);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator begin() _NOEXCEPT {
+    return __make_iterator(__get_pointer());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator begin() const _NOEXCEPT {
+    return __make_const_iterator(__get_pointer());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator end() _NOEXCEPT {
+    return __make_iterator(__get_pointer() + size());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator end() const _NOEXCEPT {
+    return __make_const_iterator(__get_pointer() + size());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rbegin() _NOEXCEPT {
+    return reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator rbegin() const _NOEXCEPT {
+    return const_reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rend() _NOEXCEPT {
+    return reverse_iterator(begin());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator rend() const _NOEXCEPT {
+    return const_reverse_iterator(begin());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cbegin() const _NOEXCEPT { return begin(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cend() const _NOEXCEPT { return end(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator crbegin() const _NOEXCEPT {
+    return rbegin();
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type size() const _NOEXCEPT {
+    return __is_long() ? __get_long_size() : __get_short_size();
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type length() const _NOEXCEPT { return size(); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type max_size() const _NOEXCEPT {
+    size_type __m = __alloc_traits::max_size(__alloc());
+    if (__m <= std::numeric_limits<size_type>::max() / 2) {
+      return __m - __alignment;
+    } else {
+      bool __uses_lsb = __endian_factor == 2;
+      return __uses_lsb ? __m - __alignment : (__m / 2) - __alignment;
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type capacity() const _NOEXCEPT {
+    return (__is_long() ? __get_long_cap() : static_cast<size_type>(__min_cap)) - 1;
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 void resize(size_type __n, value_type __c);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void resize(size_type __n) { resize(__n, value_type()); }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 void reserve(size_type __requested_capacity);
+
+#if _LIBCPP_STD_VER >= 23
+  template <class _Op>
+  _LIBCPP_HIDE_FROM_ABI constexpr void resize_and_overwrite(size_type __n, _Op __op) {
+    __resize_default_init(__n);
+    __erase_to_end(std::move(__op)(data(), _LIBCPP_AUTO_CAST(__n)));
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __resize_default_init(size_type __n);
+
+#if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_STRING_RESERVE)
+  _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void reserve() _NOEXCEPT { shrink_to_fit(); }
+#endif
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void shrink_to_fit() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void clear() _NOEXCEPT;
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool empty() const _NOEXCEPT {
+    return size() == 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference operator[](size_type __pos) const _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__pos <= size(), "string index out of bounds");
+    if (__builtin_constant_p(__pos) && !__fits_in_sso(__pos)) {
+      return *(__get_long_pointer() + __pos);
+    }
+    return *(data() + __pos);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference operator[](size_type __pos) _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__pos <= size(), "string index out of bounds");
+    if (__builtin_constant_p(__pos) && !__fits_in_sso(__pos)) {
+      return *(__get_long_pointer() + __pos);
+    }
+    return *(__get_pointer() + __pos);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference at(size_type __n) const;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 reference at(size_type __n);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator+=(const basic_string& __str) {
+    return append(__str);
+  }
+
+  template <class _Tp,
+            __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                              !__is_same_uncvref<_Tp, basic_string >::value,
+                          int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  operator+=(const _Tp& __t) {
+    __self_view __sv = __t;
+    return append(__sv);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator+=(const value_type* __s) {
+    return append(__s);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator+=(value_type __c) {
+    push_back(__c);
+    return *this;
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator+=(initializer_list<value_type> __il) {
+    return append(__il);
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(const basic_string& __str) {
+    return append(__str.data(), __str.size());
+  }
+
+  template <class _Tp,
+            __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                              !__is_same_uncvref<_Tp, basic_string>::value,
+                          int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  append(const _Tp& __t) {
+    __self_view __sv = __t;
+    return append(__sv.data(), __sv.size());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(const basic_string& __str, size_type __pos, size_type __n = npos);
+
+  template <class _Tp,
+            __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                              !__is_same_uncvref<_Tp, basic_string>::value,
+                          int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20
+
+  basic_string&
+  append(const _Tp& __t, size_type __pos, size_type __n = npos);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(const value_type* __s, size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(const value_type* __s);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(size_type __n, value_type __c);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __append_default_init(size_type __n);
+
+  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  append(_InputIterator __first, _InputIterator __last) {
+    const basic_string __temp(__first, __last, __alloc());
+    append(__temp.data(), __temp.size());
+    return *this;
+  }
+
+  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  append(_ForwardIterator __first, _ForwardIterator __last);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_CharT> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr basic_string& append_range(_Range&& __range) {
+    insert_range(end(), std::forward<_Range>(__range));
+    return *this;
+  }
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(initializer_list<value_type> __il) {
+    return append(__il.begin(), __il.size());
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 void push_back(value_type __c);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void pop_back();
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference front() _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "string::front(): string is empty");
+    return *__get_pointer();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference front() const _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "string::front(): string is empty");
+    return *data();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference back() _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "string::back(): string is empty");
+    return *(__get_pointer() + size() - 1);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference back() const _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "string::back(): string is empty");
+    return *(data() + size() - 1);
+  }
+
+  template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  assign(const _Tp& __t) {
+    __self_view __sv = __t;
+    return assign(__sv.data(), __sv.size());
+  }
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI constexpr void __move_assign(basic_string&& __str, size_type __pos, size_type __len) {
+    // Pilfer the allocation from __str.
+    _LIBCPP_ASSERT_INTERNAL(__alloc() == __str.__alloc(), "__move_assign called with wrong allocator");
+    size_type __old_sz = __str.size();
+    if (!__str.__is_long())
+      __str.__annotate_delete();
+    __r_.first()       = __str.__r_.first();
+    __str.__r_.first() = __rep();
+    __str.__annotate_new(0);
+
+    _Traits::move(data(), data() + __pos, __len);
+    __set_size(__len);
+    _Traits::assign(data()[__len], value_type());
+
+    if (!__is_long()) {
+      __annotate_new(__len);
+    } else if (__old_sz > __len) {
+      __annotate_shrink(__old_sz);
+    }
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(const basic_string& __str) {
+    return *this = __str;
+  }
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  assign(basic_string&& __str) noexcept(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value) {
+    *this = std::move(__str);
+    return *this;
+  }
+#endif
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(const basic_string& __str, size_type __pos, size_type __n = npos);
+
+  template <class _Tp,
+            __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                              !__is_same_uncvref<_Tp, basic_string>::value,
+                          int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  assign(const _Tp& __t, size_type __pos, size_type __n = npos);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(const value_type* __s, size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(const value_type* __s);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(size_type __n, value_type __c);
+  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  assign(_InputIterator __first, _InputIterator __last);
+
+  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  assign(_ForwardIterator __first, _ForwardIterator __last);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_CharT> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr basic_string& assign_range(_Range&& __range) {
+    if constexpr (__string_is_trivial_iterator<ranges::iterator_t<_Range>>::value &&
+                  (ranges::forward_range<_Range> || ranges::sized_range<_Range>)) {
+      size_type __n = static_cast<size_type>(ranges::distance(__range));
+      __assign_trivial(ranges::begin(__range), ranges::end(__range), __n);
+
+    } else {
+      __assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
+    }
+
+    return *this;
+  }
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(initializer_list<value_type> __il) {
+    return assign(__il.begin(), __il.size());
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  insert(size_type __pos1, const basic_string& __str) {
+    return insert(__pos1, __str.data(), __str.size());
+  }
+
+  template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  insert(size_type __pos1, const _Tp& __t) {
+    __self_view __sv = __t;
+    return insert(__pos1, __sv.data(), __sv.size());
+  }
+
+  template <class _Tp,
+            __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                              !__is_same_uncvref<_Tp, basic_string>::value,
+                          int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  insert(size_type __pos1, const _Tp& __t, size_type __pos2, size_type __n = npos);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  insert(size_type __pos1, const basic_string& __str, size_type __pos2, size_type __n = npos);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& insert(size_type __pos, const value_type* __s, size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& insert(size_type __pos, const value_type* __s);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& insert(size_type __pos, size_type __n, value_type __c);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator insert(const_iterator __pos, value_type __c);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_CharT> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator insert_range(const_iterator __position, _Range&& __range) {
+    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      return __insert_with_size(__position, ranges::begin(__range), ranges::end(__range), __n);
+
+    } else {
+      basic_string __temp(from_range, std::forward<_Range>(__range), __alloc());
+      return insert(__position, __temp.data(), __temp.data() + __temp.size());
+    }
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
+  insert(const_iterator __pos, size_type __n, value_type __c) {
+    difference_type __p = __pos - begin();
+    insert(static_cast<size_type>(__p), __n, __c);
+    return begin() + __p;
+  }
+
+  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
+  insert(const_iterator __pos, _InputIterator __first, _InputIterator __last);
+
+  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
+  insert(const_iterator __pos, _ForwardIterator __first, _ForwardIterator __last);
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
+  insert(const_iterator __pos, initializer_list<value_type> __il) {
+    return insert(__pos, __il.begin(), __il.end());
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& erase(size_type __pos = 0, size_type __n = npos);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator erase(const_iterator __pos);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator erase(const_iterator __first, const_iterator __last);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  replace(size_type __pos1, size_type __n1, const basic_string& __str) {
+    return replace(__pos1, __n1, __str.data(), __str.size());
+  }
+
+  template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  replace(size_type __pos1, size_type __n1, const _Tp& __t) {
+    __self_view __sv = __t;
+    return replace(__pos1, __n1, __sv.data(), __sv.size());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  replace(size_type __pos1, size_type __n1, const basic_string& __str, size_type __pos2, size_type __n2 = npos);
+
+  template <class _Tp,
+            __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                              !__is_same_uncvref<_Tp, basic_string>::value,
+                          int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  replace(size_type __pos1, size_type __n1, const _Tp& __t, size_type __pos2, size_type __n2 = npos);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  replace(size_type __pos, size_type __n1, const value_type* __s, size_type __n2);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& replace(size_type __pos, size_type __n1, const value_type* __s);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& replace(size_type __pos, size_type __n1, size_type __n2, value_type __c);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  replace(const_iterator __i1, const_iterator __i2, const basic_string& __str) {
+    return replace(
+        static_cast<size_type>(__i1 - begin()), static_cast<size_type>(__i2 - __i1), __str.data(), __str.size());
+  }
+
+  template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  replace(const_iterator __i1, const_iterator __i2, const _Tp& __t) {
+    __self_view __sv = __t;
+    return replace(__i1 - begin(), __i2 - __i1, __sv);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  replace(const_iterator __i1, const_iterator __i2, const value_type* __s, size_type __n) {
+    return replace(static_cast<size_type>(__i1 - begin()), static_cast<size_type>(__i2 - __i1), __s, __n);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  replace(const_iterator __i1, const_iterator __i2, const value_type* __s) {
+    return replace(static_cast<size_type>(__i1 - begin()), static_cast<size_type>(__i2 - __i1), __s);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  replace(const_iterator __i1, const_iterator __i2, size_type __n, value_type __c) {
+    return replace(static_cast<size_type>(__i1 - begin()), static_cast<size_type>(__i2 - __i1), __n, __c);
+  }
+
+  template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  replace(const_iterator __i1, const_iterator __i2, _InputIterator __j1, _InputIterator __j2);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_CharT> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr basic_string&
+  replace_with_range(const_iterator __i1, const_iterator __i2, _Range&& __range) {
+    basic_string __temp(from_range, std::forward<_Range>(__range), __alloc());
+    return replace(__i1, __i2, __temp);
+  }
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  replace(const_iterator __i1, const_iterator __i2, initializer_list<value_type> __il) {
+    return replace(__i1, __i2, __il.begin(), __il.end());
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type copy(value_type* __s, size_type __n, size_type __pos = 0) const;
+
+#if _LIBCPP_STD_VER <= 20
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string
+  substr(size_type __pos = 0, size_type __n = npos) const {
+    return basic_string(*this, __pos, __n);
+  }
+#else
+  _LIBCPP_HIDE_FROM_ABI constexpr basic_string substr(size_type __pos = 0, size_type __n = npos) const& {
+    return basic_string(*this, __pos, __n);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr basic_string substr(size_type __pos = 0, size_type __n = npos) && {
+    return basic_string(std::move(*this), __pos, __n);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(basic_string& __str)
+#if _LIBCPP_STD_VER >= 14
+      _NOEXCEPT;
+#else
+      _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>);
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const value_type* c_str() const _NOEXCEPT { return data(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const value_type* data() const _NOEXCEPT {
+    return std::__to_address(__get_pointer());
+  }
+#if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 value_type* data() _NOEXCEPT {
+    return std::__to_address(__get_pointer());
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator_type get_allocator() const _NOEXCEPT {
+    return __alloc();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find(const basic_string& __str, size_type __pos = 0) const _NOEXCEPT;
+
+  template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find(const _Tp& __t, size_type __pos = 0) const _NOEXCEPT;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type find(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find(const value_type* __s, size_type __pos = 0) const _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type find(value_type __c, size_type __pos = 0) const _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  rfind(const basic_string& __str, size_type __pos = npos) const _NOEXCEPT;
+
+  template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  rfind(const _Tp& __t, size_type __pos = npos) const _NOEXCEPT;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type rfind(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  rfind(const value_type* __s, size_type __pos = npos) const _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type rfind(value_type __c, size_type __pos = npos) const _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_first_of(const basic_string& __str, size_type __pos = 0) const _NOEXCEPT;
+
+  template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_first_of(const _Tp& __t, size_type __pos = 0) const _NOEXCEPT;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_first_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_first_of(const value_type* __s, size_type __pos = 0) const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_first_of(value_type __c, size_type __pos = 0) const _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_last_of(const basic_string& __str, size_type __pos = npos) const _NOEXCEPT;
+
+  template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_last_of(const _Tp& __t, size_type __pos = npos) const _NOEXCEPT;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_last_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_last_of(const value_type* __s, size_type __pos = npos) const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_last_of(value_type __c, size_type __pos = npos) const _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_first_not_of(const basic_string& __str, size_type __pos = 0) const _NOEXCEPT;
+
+  template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_first_not_of(const _Tp& __t, size_type __pos = 0) const _NOEXCEPT;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_first_not_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_first_not_of(const value_type* __s, size_type __pos = 0) const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_first_not_of(value_type __c, size_type __pos = 0) const _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_last_not_of(const basic_string& __str, size_type __pos = npos) const _NOEXCEPT;
+
+  template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_last_not_of(const _Tp& __t, size_type __pos = npos) const _NOEXCEPT;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_last_not_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_last_not_of(const value_type* __s, size_type __pos = npos) const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  find_last_not_of(value_type __c, size_type __pos = npos) const _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int compare(const basic_string& __str) const _NOEXCEPT;
+
+  template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 int
+  compare(const _Tp& __t) const _NOEXCEPT;
+
+  template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
+  _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 int
+  compare(size_type __pos1, size_type __n1, const _Tp& __t) const;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int
+  compare(size_type __pos1, size_type __n1, const basic_string& __str) const;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 int
+  compare(size_type __pos1, size_type __n1, const basic_string& __str, size_type __pos2, size_type __n2 = npos) const;
+
+  template <class _Tp,
+            __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                              !__is_same_uncvref<_Tp, basic_string>::value,
+                          int> = 0>
+  inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int
+  compare(size_type __pos1, size_type __n1, const _Tp& __t, size_type __pos2, size_type __n2 = npos) const;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 int compare(const value_type* __s) const _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 int compare(size_type __pos1, size_type __n1, const value_type* __s) const;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 int
+  compare(size_type __pos1, size_type __n1, const value_type* __s, size_type __n2) const;
+
+#if _LIBCPP_STD_VER >= 20
+  constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(__self_view __sv) const noexcept {
+    return __self_view(data(), size()).starts_with(__sv);
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(value_type __c) const noexcept {
+    return !empty() && _Traits::eq(front(), __c);
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(const value_type* __s) const noexcept {
+    return starts_with(__self_view(__s));
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(__self_view __sv) const noexcept {
+    return __self_view(data(), size()).ends_with(__sv);
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(value_type __c) const noexcept {
+    return !empty() && _Traits::eq(back(), __c);
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(const value_type* __s) const noexcept {
+    return ends_with(__self_view(__s));
+  }
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+  constexpr _LIBCPP_HIDE_FROM_ABI bool contains(__self_view __sv) const noexcept {
+    return __self_view(data(), size()).contains(__sv);
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI bool contains(value_type __c) const noexcept {
+    return __self_view(data(), size()).contains(__c);
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI bool contains(const value_type* __s) const {
+    return __self_view(data(), size()).contains(__s);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __invariants() const;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __clear_and_shrink() _NOEXCEPT;
+
+private:
+  template <class _Alloc>
+  inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool friend
+  operator==(const basic_string<char, char_traits<char>, _Alloc>& __lhs,
+             const basic_string<char, char_traits<char>, _Alloc>& __rhs) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __shrink_or_extend(size_type __target_capacity);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS bool
+  __is_long() const _NOEXCEPT {
+    if (__libcpp_is_constant_evaluated() && __builtin_constant_p(__r_.first().__l.__is_long_)) {
+      return __r_.first().__l.__is_long_;
+    }
+    return __r_.first().__s.__is_long_;
+  }
+
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __begin_lifetime(pointer __begin, size_type __n) {
+#if _LIBCPP_STD_VER >= 20
+    if (__libcpp_is_constant_evaluated()) {
+      for (size_type __i = 0; __i != __n; ++__i)
+        std::construct_at(std::addressof(__begin[__i]));
+    }
+#else
+    (void)__begin;
+    (void)__n;
+#endif // _LIBCPP_STD_VER >= 20
+  }
+
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI static bool __fits_in_sso(size_type __sz) { return __sz < __min_cap; }
+
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __assign_trivial(_Iterator __first, _Sentinel __last, size_type __n);
+
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __first, _Sentinel __last);
+
+  // Copy [__first, __last) into [__dest, __dest + (__last - __first)). Assumes that the ranges don't overlap.
+  template <class _ForwardIter, class _Sent>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static value_type*
+  __copy_non_overlapping_range(_ForwardIter __first, _Sent __last, value_type* __dest) {
+#ifndef _LIBCPP_CXX03_LANG
+    if constexpr (__libcpp_is_contiguous_iterator<_ForwardIter>::value &&
+                  is_same<value_type, __iter_value_type<_ForwardIter>>::value && is_same<_ForwardIter, _Sent>::value) {
+      traits_type::copy(__dest, std::__to_address(__first), __last - __first);
+      return __dest + (__last - __first);
+    }
+#endif
+
+    for (; __first != __last; ++__first)
+      traits_type::assign(*__dest++, *__first);
+    return __dest;
+  }
+
+  template <class _ForwardIterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 iterator
+  __insert_from_safe_copy(size_type __n, size_type __ip, _ForwardIterator __first, _Sentinel __last) {
+    size_type __sz  = size();
+    size_type __cap = capacity();
+    value_type* __p;
+    if (__cap - __sz >= __n) {
+      __annotate_increase(__n);
+      __p                = std::__to_address(__get_pointer());
+      size_type __n_move = __sz - __ip;
+      if (__n_move != 0)
+        traits_type::move(__p + __ip + __n, __p + __ip, __n_move);
+    } else {
+      __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __ip, 0, __n);
+      __p = std::__to_address(__get_long_pointer());
+    }
+    __sz += __n;
+    __set_size(__sz);
+    traits_type::assign(__p[__sz], value_type());
+    __copy_non_overlapping_range(__first, __last, __p + __ip);
+
+    return begin() + __ip;
+  }
+
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
+  __insert_with_size(const_iterator __pos, _Iterator __first, _Sentinel __last, size_type __n);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 allocator_type& __alloc() _NOEXCEPT { return __r_.second(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR const allocator_type& __alloc() const _NOEXCEPT { return __r_.second(); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS void
+  __set_short_size(size_type __s) _NOEXCEPT {
+    _LIBCPP_ASSERT_INTERNAL(__s < __min_cap, "__s should never be greater than or equal to the short string capacity");
+    __r_.first().__s.__size_    = __s;
+    __r_.first().__s.__is_long_ = false;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS size_type
+  __get_short_size() const _NOEXCEPT {
+    _LIBCPP_ASSERT_INTERNAL(!__r_.first().__s.__is_long_, "String has to be short when trying to get the short size");
+    return __r_.first().__s.__size_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __set_long_size(size_type __s) _NOEXCEPT {
+    __r_.first().__l.__size_ = __s;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type __get_long_size() const _NOEXCEPT {
+    return __r_.first().__l.__size_;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __set_size(size_type __s) _NOEXCEPT {
+    if (__is_long())
+      __set_long_size(__s);
+    else
+      __set_short_size(__s);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __set_long_cap(size_type __s) _NOEXCEPT {
+    __r_.first().__l.__cap_     = __s / __endian_factor;
+    __r_.first().__l.__is_long_ = true;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type __get_long_cap() const _NOEXCEPT {
+    return __r_.first().__l.__cap_ * __endian_factor;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __set_long_pointer(pointer __p) _NOEXCEPT {
+    __r_.first().__l.__data_ = __p;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pointer __get_long_pointer() _NOEXCEPT {
+    return _LIBCPP_ASAN_VOLATILE_WRAPPER(__r_.first().__l.__data_);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_pointer __get_long_pointer() const _NOEXCEPT {
+    return _LIBCPP_ASAN_VOLATILE_WRAPPER(__r_.first().__l.__data_);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS pointer
+  __get_short_pointer() _NOEXCEPT {
+    return _LIBCPP_ASAN_VOLATILE_WRAPPER(pointer_traits<pointer>::pointer_to(__r_.first().__s.__data_[0]));
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS const_pointer
+  __get_short_pointer() const _NOEXCEPT {
+    return _LIBCPP_ASAN_VOLATILE_WRAPPER(pointer_traits<const_pointer>::pointer_to(__r_.first().__s.__data_[0]));
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pointer __get_pointer() _NOEXCEPT {
+    return __is_long() ? __get_long_pointer() : __get_short_pointer();
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_pointer __get_pointer() const _NOEXCEPT {
+    return __is_long() ? __get_long_pointer() : __get_short_pointer();
+  }
+
+  // The following functions are no-ops outside of AddressSanitizer mode.
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+  __annotate_contiguous_container(const void* __old_mid, const void* __new_mid) const {
+    (void)__old_mid;
+    (void)__new_mid;
+#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+#  if defined(__APPLE__)
+    // TODO: remove after addressing issue #96099 (https://github.com/llvm/llvm-project/issues/96099)
+    if (!__is_long())
+      return;
+#  endif
+    std::__annotate_contiguous_container<_Allocator>(data(), data() + capacity() + 1, __old_mid, __new_mid);
+#endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_new(size_type __current_size) const _NOEXCEPT {
+    (void)__current_size;
+#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+    if (!__libcpp_is_constant_evaluated())
+      __annotate_contiguous_container(data() + capacity() + 1, data() + __current_size + 1);
+#endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_delete() const _NOEXCEPT {
+#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+    if (!__libcpp_is_constant_evaluated())
+      __annotate_contiguous_container(data() + size() + 1, data() + capacity() + 1);
+#endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_increase(size_type __n) const _NOEXCEPT {
+    (void)__n;
+#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+    if (!__libcpp_is_constant_evaluated())
+      __annotate_contiguous_container(data() + size() + 1, data() + size() + 1 + __n);
+#endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_shrink(size_type __old_size) const _NOEXCEPT {
+    (void)__old_size;
+#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+    if (!__libcpp_is_constant_evaluated())
+      __annotate_contiguous_container(data() + __old_size + 1, data() + size() + 1);
+#endif
+  }
+
+  template <size_type __a>
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type __align_it(size_type __s) _NOEXCEPT {
+    return (__s + (__a - 1)) & ~(__a - 1);
+  }
+  enum { __alignment = 8 };
+  static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type __recommend(size_type __s) _NOEXCEPT {
+    if (__s < __min_cap) {
+      return static_cast<size_type>(__min_cap) - 1;
+    }
+    const size_type __boundary = sizeof(value_type) < __alignment ? __alignment / sizeof(value_type) : __endian_factor;
+    size_type __guess          = __align_it<__boundary>(__s + 1) - 1;
+    if (__guess == __min_cap)
+      __guess += __endian_factor;
+    return __guess;
+  }
+
+  inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void __init(const value_type* __s, size_type __sz, size_type __reserve);
+  inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void __init(const value_type* __s, size_type __sz);
+  inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void __init(size_type __n, value_type __c);
+
+  // Slow path for the (inlined) copy constructor for 'long' strings.
+  // Always externally instantiated and not inlined.
+  // Requires that __s is zero terminated.
+  // The main reason for this function to exist is because for unstable, we
+  // want to allow inlining of the copy constructor. However, we don't want
+  // to call the __init() functions as those are marked as inline which may
+  // result in over-aggressive inlining by the compiler, where our aim is
+  // to only inline the fast path code directly in the ctor.
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE void __init_copy_ctor_external(const value_type* __s, size_type __sz);
+
+  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
+  inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void __init(_InputIterator __first, _InputIterator __last);
+
+  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+  inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void __init(_ForwardIterator __first, _ForwardIterator __last);
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+  __init_with_sentinel(_InputIterator __first, _Sentinel __last);
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+  __init_with_size(_InputIterator __first, _Sentinel __last, size_type __sz);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20
+#if _LIBCPP_ABI_VERSION >= 2 //  We want to use the function in the dylib in ABIv1
+  _LIBCPP_HIDE_FROM_ABI
+#endif
+  _LIBCPP_DEPRECATED_("use __grow_by_without_replace") void __grow_by(
+      size_type __old_cap,
+      size_type __delta_cap,
+      size_type __old_sz,
+      size_type __n_copy,
+      size_type __n_del,
+      size_type __n_add = 0);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __grow_by_without_replace(
+      size_type __old_cap,
+      size_type __delta_cap,
+      size_type __old_sz,
+      size_type __n_copy,
+      size_type __n_del,
+      size_type __n_add = 0);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 void __grow_by_and_replace(
+      size_type __old_cap,
+      size_type __delta_cap,
+      size_type __old_sz,
+      size_type __n_copy,
+      size_type __n_del,
+      size_type __n_add,
+      const value_type* __p_new_stuff);
+
+  // __assign_no_alias is invoked for assignment operations where we
+  // have proof that the input does not alias the current instance.
+  // For example, operator=(basic_string) performs a 'self' check.
+  template <bool __is_short>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE basic_string& __assign_no_alias(const value_type* __s, size_type __n);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __erase_to_end(size_type __pos) {
+    __null_terminate_at(std::__to_address(__get_pointer()), __pos);
+  }
+
+  // __erase_external_with_move is invoked for erase() invocations where
+  // `n ~= npos`, likely requiring memory moves on the string data.
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE void __erase_external_with_move(size_type __pos, size_type __n);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __copy_assign_alloc(const basic_string& __str) {
+    __copy_assign_alloc(
+        __str, integral_constant<bool, __alloc_traits::propagate_on_container_copy_assignment::value>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __copy_assign_alloc(const basic_string& __str, true_type) {
+    if (__alloc() == __str.__alloc())
+      __alloc() = __str.__alloc();
+    else {
+      if (!__str.__is_long()) {
+        __clear_and_shrink();
+        __alloc() = __str.__alloc();
+      } else {
+        __annotate_delete();
+        allocator_type __a = __str.__alloc();
+        auto __allocation  = std::__allocate_at_least(__a, __str.__get_long_cap());
+        __begin_lifetime(__allocation.ptr, __allocation.count);
+        if (__is_long())
+          __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap());
+        __alloc() = std::move(__a);
+        __set_long_pointer(__allocation.ptr);
+        __set_long_cap(__allocation.count);
+        __set_long_size(__str.size());
+        __annotate_new(__get_long_size());
+      }
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+  __copy_assign_alloc(const basic_string&, false_type) _NOEXCEPT {}
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+  __move_assign(basic_string& __str, false_type) noexcept(__alloc_traits::is_always_equal::value);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS void
+  __move_assign(basic_string& __str, true_type)
+#  if _LIBCPP_STD_VER >= 17
+      noexcept;
+#  else
+      noexcept(is_nothrow_move_assignable<allocator_type>::value);
+#  endif
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign_alloc(basic_string& __str)
+      _NOEXCEPT_(!__alloc_traits::propagate_on_container_move_assignment::value ||
+                 is_nothrow_move_assignable<allocator_type>::value) {
+    __move_assign_alloc(
+        __str, integral_constant<bool, __alloc_traits::propagate_on_container_move_assignment::value>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign_alloc(basic_string& __c, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
+    __alloc() = std::move(__c.__alloc());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign_alloc(basic_string&, false_type) _NOEXCEPT {}
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE basic_string& __assign_external(const value_type* __s);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE basic_string& __assign_external(const value_type* __s, size_type __n);
+
+  // Assigns the value in __s, guaranteed to be __n < __min_cap in length.
+  inline _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& __assign_short(const value_type* __s, size_type __n) {
+    size_type __old_size = size();
+    if (__n > __old_size)
+      __annotate_increase(__n - __old_size);
+    pointer __p =
+        __is_long() ? (__set_long_size(__n), __get_long_pointer()) : (__set_short_size(__n), __get_short_pointer());
+    traits_type::move(std::__to_address(__p), __s, __n);
+    traits_type::assign(__p[__n], value_type());
+    if (__old_size > __n)
+      __annotate_shrink(__old_size);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+  __null_terminate_at(value_type* __p, size_type __newsz) {
+    size_type __old_size = size();
+    if (__newsz > __old_size)
+      __annotate_increase(__newsz - __old_size);
+    __set_size(__newsz);
+    traits_type::assign(__p[__newsz], value_type());
+    if (__old_size > __newsz)
+      __annotate_shrink(__old_size);
+    return *this;
+  }
+
+  template <class _Tp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __addr_in_range(const _Tp& __v) const {
+    return std::__is_pointer_in_range(data(), data() + size() + 1, std::addressof(__v));
+  }
+
+  _LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI void __throw_length_error() const {
+    std::__throw_length_error("basic_string");
+  }
+
+  _LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI void __throw_out_of_range() const {
+    std::__throw_out_of_range("basic_string");
+  }
+
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string operator+ <>(const basic_string&, const basic_string&);
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string operator+ <>(const value_type*, const basic_string&);
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string operator+ <>(value_type, const basic_string&);
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string operator+ <>(const basic_string&, const value_type*);
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string operator+ <>(const basic_string&, value_type);
+#if _LIBCPP_STD_VER >= 26
+  friend constexpr basic_string operator+ <>(const basic_string&, type_identity_t<__self_view>);
+  friend constexpr basic_string operator+ <>(type_identity_t<__self_view>, const basic_string&);
+#endif
+};
+
+// These declarations must appear before any functions are implicitly used
+// so that they have the correct visibility specifier.
+#define _LIBCPP_DECLARE(...) extern template __VA_ARGS__;
+#ifdef _LIBCPP_ABI_STRING_OPTIMIZED_EXTERNAL_INSTANTIATION
+_LIBCPP_STRING_UNSTABLE_EXTERN_TEMPLATE_LIST(_LIBCPP_DECLARE, char)
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+_LIBCPP_STRING_UNSTABLE_EXTERN_TEMPLATE_LIST(_LIBCPP_DECLARE, wchar_t)
+#  endif
+#else
+_LIBCPP_STRING_V1_EXTERN_TEMPLATE_LIST(_LIBCPP_DECLARE, char)
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+_LIBCPP_STRING_V1_EXTERN_TEMPLATE_LIST(_LIBCPP_DECLARE, wchar_t)
+#  endif
+#endif
+#undef _LIBCPP_DECLARE
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator,
+          class _CharT     = __iter_value_type<_InputIterator>,
+          class _Allocator = allocator<_CharT>,
+          class            = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value> >
+basic_string(_InputIterator, _InputIterator, _Allocator = _Allocator())
+    -> basic_string<_CharT, char_traits<_CharT>, _Allocator>;
+
+template <class _CharT,
+          class _Traits,
+          class _Allocator = allocator<_CharT>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value> >
+explicit basic_string(basic_string_view<_CharT, _Traits>,
+                      const _Allocator& = _Allocator()) -> basic_string<_CharT, _Traits, _Allocator>;
+
+template <class _CharT,
+          class _Traits,
+          class _Allocator = allocator<_CharT>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value>,
+          class _Sz        = typename allocator_traits<_Allocator>::size_type >
+basic_string(basic_string_view<_CharT, _Traits>, _Sz, _Sz, const _Allocator& = _Allocator())
+    -> basic_string<_CharT, _Traits, _Allocator>;
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range,
+          class _Allocator = allocator<ranges::range_value_t<_Range>>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value> >
+basic_string(from_range_t, _Range&&, _Allocator = _Allocator())
+    -> basic_string<ranges::range_value_t<_Range>, char_traits<ranges::range_value_t<_Range>>, _Allocator>;
+#endif
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+basic_string<_CharT, _Traits, _Allocator>::__init(const value_type* __s, size_type __sz, size_type __reserve) {
+  if (__libcpp_is_constant_evaluated())
+    __r_.first() = __rep();
+  if (__reserve > max_size())
+    __throw_length_error();
+  pointer __p;
+  if (__fits_in_sso(__reserve)) {
+    __set_short_size(__sz);
+    __p = __get_short_pointer();
+  } else {
+    auto __allocation = std::__allocate_at_least(__alloc(), __recommend(__reserve) + 1);
+    __p               = __allocation.ptr;
+    __begin_lifetime(__p, __allocation.count);
+    __set_long_pointer(__p);
+    __set_long_cap(__allocation.count);
+    __set_long_size(__sz);
+  }
+  traits_type::copy(std::__to_address(__p), __s, __sz);
+  traits_type::assign(__p[__sz], value_type());
+  __annotate_new(__sz);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+basic_string<_CharT, _Traits, _Allocator>::__init(const value_type* __s, size_type __sz) {
+  if (__libcpp_is_constant_evaluated())
+    __r_.first() = __rep();
+  if (__sz > max_size())
+    __throw_length_error();
+  pointer __p;
+  if (__fits_in_sso(__sz)) {
+    __set_short_size(__sz);
+    __p = __get_short_pointer();
+  } else {
+    auto __allocation = std::__allocate_at_least(__alloc(), __recommend(__sz) + 1);
+    __p               = __allocation.ptr;
+    __begin_lifetime(__p, __allocation.count);
+    __set_long_pointer(__p);
+    __set_long_cap(__allocation.count);
+    __set_long_size(__sz);
+  }
+  traits_type::copy(std::__to_address(__p), __s, __sz);
+  traits_type::assign(__p[__sz], value_type());
+  __annotate_new(__sz);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE void
+basic_string<_CharT, _Traits, _Allocator>::__init_copy_ctor_external(const value_type* __s, size_type __sz) {
+  if (__libcpp_is_constant_evaluated())
+    __r_.first() = __rep();
+
+  pointer __p;
+  if (__fits_in_sso(__sz)) {
+    __p = __get_short_pointer();
+    __set_short_size(__sz);
+  } else {
+    if (__sz > max_size())
+      __throw_length_error();
+    auto __allocation = std::__allocate_at_least(__alloc(), __recommend(__sz) + 1);
+    __p               = __allocation.ptr;
+    __begin_lifetime(__p, __allocation.count);
+    __set_long_pointer(__p);
+    __set_long_cap(__allocation.count);
+    __set_long_size(__sz);
+  }
+  traits_type::copy(std::__to_address(__p), __s, __sz + 1);
+  __annotate_new(__sz);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__init(size_type __n, value_type __c) {
+  if (__libcpp_is_constant_evaluated())
+    __r_.first() = __rep();
+
+  if (__n > max_size())
+    __throw_length_error();
+  pointer __p;
+  if (__fits_in_sso(__n)) {
+    __set_short_size(__n);
+    __p = __get_short_pointer();
+  } else {
+    auto __allocation = std::__allocate_at_least(__alloc(), __recommend(__n) + 1);
+    __p               = __allocation.ptr;
+    __begin_lifetime(__p, __allocation.count);
+    __set_long_pointer(__p);
+    __set_long_cap(__allocation.count);
+    __set_long_size(__n);
+  }
+  traits_type::assign(std::__to_address(__p), __n, __c);
+  traits_type::assign(__p[__n], value_type());
+  __annotate_new(__n);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+basic_string<_CharT, _Traits, _Allocator>::__init(_InputIterator __first, _InputIterator __last) {
+  __init_with_sentinel(std::move(__first), std::move(__last));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _InputIterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+basic_string<_CharT, _Traits, _Allocator>::__init_with_sentinel(_InputIterator __first, _Sentinel __last) {
+  __r_.first() = __rep();
+  __annotate_new(0);
+
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    for (; __first != __last; ++__first)
+      push_back(*__first);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    __annotate_delete();
+    if (__is_long())
+      __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap());
+    throw;
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+basic_string<_CharT, _Traits, _Allocator>::__init(_ForwardIterator __first, _ForwardIterator __last) {
+  size_type __sz = static_cast<size_type>(std::distance(__first, __last));
+  __init_with_size(__first, __last, __sz);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _InputIterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+basic_string<_CharT, _Traits, _Allocator>::__init_with_size(_InputIterator __first, _Sentinel __last, size_type __sz) {
+  if (__libcpp_is_constant_evaluated())
+    __r_.first() = __rep();
+
+  if (__sz > max_size())
+    __throw_length_error();
+
+  pointer __p;
+  if (__fits_in_sso(__sz)) {
+    __set_short_size(__sz);
+    __p = __get_short_pointer();
+
+  } else {
+    auto __allocation = std::__allocate_at_least(__alloc(), __recommend(__sz) + 1);
+    __p               = __allocation.ptr;
+    __begin_lifetime(__p, __allocation.count);
+    __set_long_pointer(__p);
+    __set_long_cap(__allocation.count);
+    __set_long_size(__sz);
+  }
+
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    auto __end = __copy_non_overlapping_range(__first, __last, std::__to_address(__p));
+    traits_type::assign(*__end, value_type());
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    if (__is_long())
+      __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap());
+    throw;
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  __annotate_new(__sz);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__grow_by_and_replace(
+    size_type __old_cap,
+    size_type __delta_cap,
+    size_type __old_sz,
+    size_type __n_copy,
+    size_type __n_del,
+    size_type __n_add,
+    const value_type* __p_new_stuff) {
+  size_type __ms = max_size();
+  if (__delta_cap > __ms - __old_cap - 1)
+    __throw_length_error();
+  pointer __old_p = __get_pointer();
+  size_type __cap =
+      __old_cap < __ms / 2 - __alignment ? __recommend(std::max(__old_cap + __delta_cap, 2 * __old_cap)) : __ms - 1;
+  __annotate_delete();
+  auto __allocation = std::__allocate_at_least(__alloc(), __cap + 1);
+  pointer __p       = __allocation.ptr;
+  __begin_lifetime(__p, __allocation.count);
+  if (__n_copy != 0)
+    traits_type::copy(std::__to_address(__p), std::__to_address(__old_p), __n_copy);
+  if (__n_add != 0)
+    traits_type::copy(std::__to_address(__p) + __n_copy, __p_new_stuff, __n_add);
+  size_type __sec_cp_sz = __old_sz - __n_del - __n_copy;
+  if (__sec_cp_sz != 0)
+    traits_type::copy(
+        std::__to_address(__p) + __n_copy + __n_add, std::__to_address(__old_p) + __n_copy + __n_del, __sec_cp_sz);
+  if (__old_cap + 1 != __min_cap)
+    __alloc_traits::deallocate(__alloc(), __old_p, __old_cap + 1);
+  __set_long_pointer(__p);
+  __set_long_cap(__allocation.count);
+  __old_sz = __n_copy + __n_add + __sec_cp_sz;
+  __set_long_size(__old_sz);
+  traits_type::assign(__p[__old_sz], value_type());
+  __annotate_new(__old_sz);
+}
+
+// __grow_by is deprecated because it does not set the size. It may not update the size when the size is changed, and it
+// may also not set the size at all when the string was short initially. This leads to unpredictable size value. It is
+// not removed or changed to avoid breaking the ABI.
+template <class _CharT, class _Traits, class _Allocator>
+void _LIBCPP_CONSTEXPR_SINCE_CXX20
+#if _LIBCPP_ABI_VERSION >= 2 // We want to use the function in the dylib in ABIv1
+_LIBCPP_HIDE_FROM_ABI
+#endif
+_LIBCPP_DEPRECATED_("use __grow_by_without_replace") basic_string<_CharT, _Traits, _Allocator>::__grow_by(
+    size_type __old_cap,
+    size_type __delta_cap,
+    size_type __old_sz,
+    size_type __n_copy,
+    size_type __n_del,
+    size_type __n_add) {
+  size_type __ms = max_size();
+  if (__delta_cap > __ms - __old_cap)
+    __throw_length_error();
+  pointer __old_p = __get_pointer();
+  size_type __cap =
+      __old_cap < __ms / 2 - __alignment ? __recommend(std::max(__old_cap + __delta_cap, 2 * __old_cap)) : __ms - 1;
+  __annotate_delete();
+  auto __allocation = std::__allocate_at_least(__alloc(), __cap + 1);
+  pointer __p       = __allocation.ptr;
+  __begin_lifetime(__p, __allocation.count);
+  if (__n_copy != 0)
+    traits_type::copy(std::__to_address(__p), std::__to_address(__old_p), __n_copy);
+  size_type __sec_cp_sz = __old_sz - __n_del - __n_copy;
+  if (__sec_cp_sz != 0)
+    traits_type::copy(
+        std::__to_address(__p) + __n_copy + __n_add, std::__to_address(__old_p) + __n_copy + __n_del, __sec_cp_sz);
+  if (__old_cap + 1 != __min_cap)
+    __alloc_traits::deallocate(__alloc(), __old_p, __old_cap + 1);
+  __set_long_pointer(__p);
+  __set_long_cap(__allocation.count);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+void _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+basic_string<_CharT, _Traits, _Allocator>::__grow_by_without_replace(
+    size_type __old_cap,
+    size_type __delta_cap,
+    size_type __old_sz,
+    size_type __n_copy,
+    size_type __n_del,
+    size_type __n_add) {
+  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+  __grow_by(__old_cap, __delta_cap, __old_sz, __n_copy, __n_del, __n_add);
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  __set_long_size(__old_sz - __n_del + __n_add);
+  __annotate_new(__old_sz - __n_del + __n_add);
+}
+
+// assign
+
+template <class _CharT, class _Traits, class _Allocator>
+template <bool __is_short>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::__assign_no_alias(const value_type* __s, size_type __n) {
+  size_type __cap = __is_short ? static_cast<size_type>(__min_cap) : __get_long_cap();
+  if (__n < __cap) {
+    size_type __old_size = __is_short ? __get_short_size() : __get_long_size();
+    if (__n > __old_size)
+      __annotate_increase(__n - __old_size);
+    pointer __p = __is_short ? __get_short_pointer() : __get_long_pointer();
+    __is_short ? __set_short_size(__n) : __set_long_size(__n);
+    traits_type::copy(std::__to_address(__p), __s, __n);
+    traits_type::assign(__p[__n], value_type());
+    if (__old_size > __n)
+      __annotate_shrink(__old_size);
+  } else {
+    size_type __sz = __is_short ? __get_short_size() : __get_long_size();
+    __grow_by_and_replace(__cap - 1, __n - __cap + 1, __sz, 0, __sz, __n, __s);
+  }
+  return *this;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::__assign_external(const value_type* __s, size_type __n) {
+  size_type __cap = capacity();
+  if (__cap >= __n) {
+    size_type __old_size = size();
+    if (__n > __old_size)
+      __annotate_increase(__n - __old_size);
+    value_type* __p = std::__to_address(__get_pointer());
+    traits_type::move(__p, __s, __n);
+    return __null_terminate_at(__p, __n);
+  } else {
+    size_type __sz = size();
+    __grow_by_and_replace(__cap, __n - __cap, __sz, 0, __sz, __n, __s);
+    return *this;
+  }
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::assign(const value_type* __s, size_type __n) {
+  _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::assign received nullptr");
+  return (__builtin_constant_p(__n) && __fits_in_sso(__n)) ? __assign_short(__s, __n) : __assign_external(__s, __n);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::assign(size_type __n, value_type __c) {
+  size_type __cap      = capacity();
+  size_type __old_size = size();
+  if (__cap < __n) {
+    size_type __sz = size();
+    __grow_by_without_replace(__cap, __n - __cap, __sz, 0, __sz);
+    __annotate_increase(__n);
+  } else if (__n > __old_size)
+    __annotate_increase(__n - __old_size);
+  value_type* __p = std::__to_address(__get_pointer());
+  traits_type::assign(__p, __n, __c);
+  return __null_terminate_at(__p, __n);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::operator=(value_type __c) {
+  pointer __p;
+  size_type __old_size = size();
+  if (__old_size == 0)
+    __annotate_increase(1);
+  if (__is_long()) {
+    __p = __get_long_pointer();
+    __set_long_size(1);
+  } else {
+    __p = __get_short_pointer();
+    __set_short_size(1);
+  }
+  traits_type::assign(*__p, __c);
+  traits_type::assign(*++__p, value_type());
+  if (__old_size > 1)
+    __annotate_shrink(__old_size);
+  return *this;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::operator=(const basic_string& __str) {
+  if (this != std::addressof(__str)) {
+    __copy_assign_alloc(__str);
+    if (!__is_long()) {
+      if (!__str.__is_long()) {
+        size_type __old_size = __get_short_size();
+        if (__get_short_size() < __str.__get_short_size())
+          __annotate_increase(__str.__get_short_size() - __get_short_size());
+        __r_.first() = __str.__r_.first();
+        if (__old_size > __get_short_size())
+          __annotate_shrink(__old_size);
+      } else {
+        return __assign_no_alias<true>(__str.data(), __str.size());
+      }
+    } else {
+      return __assign_no_alias<false>(__str.data(), __str.size());
+    }
+  }
+  return *this;
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__move_assign(
+    basic_string& __str, false_type) noexcept(__alloc_traits::is_always_equal::value) {
+  if (__alloc() != __str.__alloc())
+    assign(__str);
+  else
+    __move_assign(__str, true_type());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS void
+basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, true_type)
+#  if _LIBCPP_STD_VER >= 17
+    noexcept
+#  else
+    noexcept(is_nothrow_move_assignable<allocator_type>::value)
+#  endif
+{
+  __annotate_delete();
+  if (__is_long()) {
+    __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap());
+#  if _LIBCPP_STD_VER <= 14
+    if (!is_nothrow_move_assignable<allocator_type>::value) {
+      __set_short_size(0);
+      traits_type::assign(__get_short_pointer()[0], value_type());
+      __annotate_new(0);
+    }
+#  endif
+  }
+  size_type __str_old_size = __str.size();
+  bool __str_was_short     = !__str.__is_long();
+
+  __move_assign_alloc(__str);
+  __r_.first() = __str.__r_.first();
+  __str.__set_short_size(0);
+  traits_type::assign(__str.__get_short_pointer()[0], value_type());
+
+  if (__str_was_short && this != &__str)
+    __str.__annotate_shrink(__str_old_size);
+  else
+    // ASan annotations: was long, so object memory is unpoisoned as new.
+    // Or is same as *this, and __annotate_delete() was called.
+    __str.__annotate_new(0);
+
+  // ASan annotations: Guard against `std::string s; s = std::move(s);`
+  // You can find more here: https://en.cppreference.com/w/cpp/utility/move
+  // Quote: "Unless otherwise specified, all standard library objects that have been moved
+  // from are placed in a "valid but unspecified state", meaning the object's class
+  // invariants hold (so functions without preconditions, such as the assignment operator,
+  // can be safely used on the object after it was moved from):"
+  // Quote: "v = std::move(v); // the value of v is unspecified"
+  if (!__is_long() && &__str != this)
+    // If it is long string, delete was never called on original __str's buffer.
+    __annotate_new(__get_short_size());
+}
+
+#endif
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::assign(_InputIterator __first, _InputIterator __last) {
+  __assign_with_sentinel(__first, __last);
+  return *this;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _InputIterator, class _Sentinel>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+basic_string<_CharT, _Traits, _Allocator>::__assign_with_sentinel(_InputIterator __first, _Sentinel __last) {
+  const basic_string __temp(__init_with_sentinel_tag(), std::move(__first), std::move(__last), __alloc());
+  assign(__temp.data(), __temp.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::assign(_ForwardIterator __first, _ForwardIterator __last) {
+  if (__string_is_trivial_iterator<_ForwardIterator>::value) {
+    size_type __n = static_cast<size_type>(std::distance(__first, __last));
+    __assign_trivial(__first, __last, __n);
+  } else {
+    __assign_with_sentinel(__first, __last);
+  }
+
+  return *this;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _Iterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+basic_string<_CharT, _Traits, _Allocator>::__assign_trivial(_Iterator __first, _Sentinel __last, size_type __n) {
+  _LIBCPP_ASSERT_INTERNAL(
+      __string_is_trivial_iterator<_Iterator>::value, "The iterator type given to `__assign_trivial` must be trivial");
+
+  size_type __old_size = size();
+  size_type __cap      = capacity();
+  if (__cap < __n) {
+    // Unlike `append` functions, if the input range points into the string itself, there is no case that the input
+    // range could get invalidated by reallocation:
+    // 1. If the input range is a subset of the string itself, its size cannot exceed the capacity of the string,
+    //    thus no reallocation would happen.
+    // 2. In the exotic case where the input range is the byte representation of the string itself, the string
+    //    object itself stays valid even if reallocation happens.
+    size_type __sz = size();
+    __grow_by_without_replace(__cap, __n - __cap, __sz, 0, __sz);
+    __annotate_increase(__n);
+  } else if (__n > __old_size)
+    __annotate_increase(__n - __old_size);
+  pointer __p = __get_pointer();
+  for (; __first != __last; ++__p, (void)++__first)
+    traits_type::assign(*__p, *__first);
+  traits_type::assign(*__p, value_type());
+  __set_size(__n);
+  if (__n < __old_size)
+    __annotate_shrink(__old_size);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::assign(const basic_string& __str, size_type __pos, size_type __n) {
+  size_type __sz = __str.size();
+  if (__pos > __sz)
+    __throw_out_of_range();
+  return assign(__str.data() + __pos, std::min(__n, __sz - __pos));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _Tp,
+          __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                            !__is_same_uncvref<_Tp, basic_string<_CharT, _Traits, _Allocator> >::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::assign(const _Tp& __t, size_type __pos, size_type __n) {
+  __self_view __sv = __t;
+  size_type __sz   = __sv.size();
+  if (__pos > __sz)
+    __throw_out_of_range();
+  return assign(__sv.data() + __pos, std::min(__n, __sz - __pos));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::__assign_external(const value_type* __s) {
+  return __assign_external(__s, traits_type::length(__s));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::assign(const value_type* __s) {
+  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::assign received nullptr");
+  return __builtin_constant_p(*__s)
+           ? (__fits_in_sso(traits_type::length(__s)) ? __assign_short(__s, traits_type::length(__s))
+                                                      : __assign_external(__s, traits_type::length(__s)))
+           : __assign_external(__s);
+}
+// append
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::append(const value_type* __s, size_type __n) {
+  _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::append received nullptr");
+  size_type __cap = capacity();
+  size_type __sz  = size();
+  if (__cap - __sz >= __n) {
+    if (__n) {
+      __annotate_increase(__n);
+      value_type* __p = std::__to_address(__get_pointer());
+      traits_type::copy(__p + __sz, __s, __n);
+      __sz += __n;
+      __set_size(__sz);
+      traits_type::assign(__p[__sz], value_type());
+    }
+  } else
+    __grow_by_and_replace(__cap, __sz + __n - __cap, __sz, __sz, 0, __n, __s);
+  return *this;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::append(size_type __n, value_type __c) {
+  if (__n) {
+    size_type __cap = capacity();
+    size_type __sz  = size();
+    if (__cap - __sz < __n)
+      __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
+    __annotate_increase(__n);
+    pointer __p = __get_pointer();
+    traits_type::assign(std::__to_address(__p) + __sz, __n, __c);
+    __sz += __n;
+    __set_size(__sz);
+    traits_type::assign(__p[__sz], value_type());
+  }
+  return *this;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void
+basic_string<_CharT, _Traits, _Allocator>::__append_default_init(size_type __n) {
+  if (__n) {
+    size_type __cap = capacity();
+    size_type __sz  = size();
+    if (__cap - __sz < __n)
+      __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
+    __annotate_increase(__n);
+    pointer __p = __get_pointer();
+    __sz += __n;
+    __set_size(__sz);
+    traits_type::assign(__p[__sz], value_type());
+  }
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::push_back(value_type __c) {
+  bool __is_short = !__is_long();
+  size_type __cap;
+  size_type __sz;
+  if (__is_short) {
+    __cap = __min_cap - 1;
+    __sz  = __get_short_size();
+  } else {
+    __cap = __get_long_cap() - 1;
+    __sz  = __get_long_size();
+  }
+  if (__sz == __cap) {
+    __grow_by_without_replace(__cap, 1, __sz, __sz, 0);
+    __annotate_increase(1);
+    __is_short = false; // the string is always long after __grow_by
+  } else
+    __annotate_increase(1);
+  pointer __p = __get_pointer();
+  if (__is_short) {
+    __p = __get_short_pointer() + __sz;
+    __set_short_size(__sz + 1);
+  } else {
+    __p = __get_long_pointer() + __sz;
+    __set_long_size(__sz + 1);
+  }
+  traits_type::assign(*__p, __c);
+  traits_type::assign(*++__p, value_type());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::append(_ForwardIterator __first, _ForwardIterator __last) {
+  size_type __sz  = size();
+  size_type __cap = capacity();
+  size_type __n   = static_cast<size_type>(std::distance(__first, __last));
+  if (__n) {
+    if (__string_is_trivial_iterator<_ForwardIterator>::value && !__addr_in_range(*__first)) {
+      if (__cap - __sz < __n)
+        __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
+      __annotate_increase(__n);
+      auto __end = __copy_non_overlapping_range(__first, __last, std::__to_address(__get_pointer() + __sz));
+      traits_type::assign(*__end, value_type());
+      __set_size(__sz + __n);
+    } else {
+      const basic_string __temp(__first, __last, __alloc());
+      append(__temp.data(), __temp.size());
+    }
+  }
+  return *this;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::append(const basic_string& __str, size_type __pos, size_type __n) {
+  size_type __sz = __str.size();
+  if (__pos > __sz)
+    __throw_out_of_range();
+  return append(__str.data() + __pos, std::min(__n, __sz - __pos));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _Tp,
+          __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                            !__is_same_uncvref<_Tp, basic_string<_CharT, _Traits, _Allocator> >::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::append(const _Tp& __t, size_type __pos, size_type __n) {
+  __self_view __sv = __t;
+  size_type __sz   = __sv.size();
+  if (__pos > __sz)
+    __throw_out_of_range();
+  return append(__sv.data() + __pos, std::min(__n, __sz - __pos));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::append(const value_type* __s) {
+  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::append received nullptr");
+  return append(__s, traits_type::length(__s));
+}
+
+// insert
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::insert(size_type __pos, const value_type* __s, size_type __n) {
+  _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::insert received nullptr");
+  size_type __sz = size();
+  if (__pos > __sz)
+    __throw_out_of_range();
+  size_type __cap = capacity();
+  if (__cap - __sz >= __n) {
+    if (__n) {
+      __annotate_increase(__n);
+      value_type* __p    = std::__to_address(__get_pointer());
+      size_type __n_move = __sz - __pos;
+      if (__n_move != 0) {
+        if (std::__is_pointer_in_range(__p + __pos, __p + __sz, __s))
+          __s += __n;
+        traits_type::move(__p + __pos + __n, __p + __pos, __n_move);
+      }
+      traits_type::move(__p + __pos, __s, __n);
+      __sz += __n;
+      __set_size(__sz);
+      traits_type::assign(__p[__sz], value_type());
+    }
+  } else
+    __grow_by_and_replace(__cap, __sz + __n - __cap, __sz, __pos, 0, __n, __s);
+  return *this;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::insert(size_type __pos, size_type __n, value_type __c) {
+  size_type __sz = size();
+  if (__pos > __sz)
+    __throw_out_of_range();
+  if (__n) {
+    size_type __cap = capacity();
+    value_type* __p;
+    if (__cap - __sz >= __n) {
+      __annotate_increase(__n);
+      __p                = std::__to_address(__get_pointer());
+      size_type __n_move = __sz - __pos;
+      if (__n_move != 0)
+        traits_type::move(__p + __pos + __n, __p + __pos, __n_move);
+    } else {
+      __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __pos, 0, __n);
+      __p = std::__to_address(__get_long_pointer());
+    }
+    traits_type::assign(__p + __pos, __n, __c);
+    __sz += __n;
+    __set_size(__sz);
+    traits_type::assign(__p[__sz], value_type());
+  }
+  return *this;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::iterator
+basic_string<_CharT, _Traits, _Allocator>::insert(const_iterator __pos, _InputIterator __first, _InputIterator __last) {
+  const basic_string __temp(__first, __last, __alloc());
+  return insert(__pos, __temp.data(), __temp.data() + __temp.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::iterator
+basic_string<_CharT, _Traits, _Allocator>::insert(
+    const_iterator __pos, _ForwardIterator __first, _ForwardIterator __last) {
+  auto __n = static_cast<size_type>(std::distance(__first, __last));
+  return __insert_with_size(__pos, __first, __last, __n);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _Iterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::iterator
+basic_string<_CharT, _Traits, _Allocator>::__insert_with_size(
+    const_iterator __pos, _Iterator __first, _Sentinel __last, size_type __n) {
+  size_type __ip = static_cast<size_type>(__pos - begin());
+  if (__n == 0)
+    return begin() + __ip;
+
+  if (__string_is_trivial_iterator<_Iterator>::value && !__addr_in_range(*__first)) {
+    return __insert_from_safe_copy(__n, __ip, __first, __last);
+  } else {
+    const basic_string __temp(__init_with_sentinel_tag(), __first, __last, __alloc());
+    return __insert_from_safe_copy(__n, __ip, __temp.begin(), __temp.end());
+  }
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::insert(
+    size_type __pos1, const basic_string& __str, size_type __pos2, size_type __n) {
+  size_type __str_sz = __str.size();
+  if (__pos2 > __str_sz)
+    __throw_out_of_range();
+  return insert(__pos1, __str.data() + __pos2, std::min(__n, __str_sz - __pos2));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _Tp,
+          __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                            !__is_same_uncvref<_Tp, basic_string<_CharT, _Traits, _Allocator> >::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::insert(size_type __pos1, const _Tp& __t, size_type __pos2, size_type __n) {
+  __self_view __sv   = __t;
+  size_type __str_sz = __sv.size();
+  if (__pos2 > __str_sz)
+    __throw_out_of_range();
+  return insert(__pos1, __sv.data() + __pos2, std::min(__n, __str_sz - __pos2));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::insert(size_type __pos, const value_type* __s) {
+  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::insert received nullptr");
+  return insert(__pos, __s, traits_type::length(__s));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::iterator
+basic_string<_CharT, _Traits, _Allocator>::insert(const_iterator __pos, value_type __c) {
+  size_type __ip  = static_cast<size_type>(__pos - begin());
+  size_type __sz  = size();
+  size_type __cap = capacity();
+  value_type* __p;
+  if (__cap == __sz) {
+    __grow_by_without_replace(__cap, 1, __sz, __ip, 0, 1);
+    __p = std::__to_address(__get_long_pointer());
+  } else {
+    __annotate_increase(1);
+    __p                = std::__to_address(__get_pointer());
+    size_type __n_move = __sz - __ip;
+    if (__n_move != 0)
+      traits_type::move(__p + __ip + 1, __p + __ip, __n_move);
+  }
+  traits_type::assign(__p[__ip], __c);
+  traits_type::assign(__p[++__sz], value_type());
+  __set_size(__sz);
+  return begin() + static_cast<difference_type>(__ip);
+}
+
+// replace
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::replace(
+    size_type __pos, size_type __n1, const value_type* __s, size_type __n2)
+    _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK {
+  _LIBCPP_ASSERT_NON_NULL(__n2 == 0 || __s != nullptr, "string::replace received nullptr");
+  size_type __sz = size();
+  if (__pos > __sz)
+    __throw_out_of_range();
+  __n1            = std::min(__n1, __sz - __pos);
+  size_type __cap = capacity();
+  if (__cap - __sz + __n1 >= __n2) {
+    value_type* __p = std::__to_address(__get_pointer());
+    if (__n1 != __n2) {
+      if (__n2 > __n1)
+        __annotate_increase(__n2 - __n1);
+      size_type __n_move = __sz - __pos - __n1;
+      if (__n_move != 0) {
+        if (__n1 > __n2) {
+          traits_type::move(__p + __pos, __s, __n2);
+          traits_type::move(__p + __pos + __n2, __p + __pos + __n1, __n_move);
+          return __null_terminate_at(__p, __sz + (__n2 - __n1));
+        }
+        if (std::__is_pointer_in_range(__p + __pos + 1, __p + __sz, __s)) {
+          if (__p + __pos + __n1 <= __s)
+            __s += __n2 - __n1;
+          else // __p + __pos < __s < __p + __pos + __n1
+          {
+            traits_type::move(__p + __pos, __s, __n1);
+            __pos += __n1;
+            __s += __n2;
+            __n2 -= __n1;
+            __n1 = 0;
+          }
+        }
+        traits_type::move(__p + __pos + __n2, __p + __pos + __n1, __n_move);
+      }
+    }
+    traits_type::move(__p + __pos, __s, __n2);
+    return __null_terminate_at(__p, __sz + (__n2 - __n1));
+  } else
+    __grow_by_and_replace(__cap, __sz - __n1 + __n2 - __cap, __sz, __pos, __n1, __n2, __s);
+  return *this;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::replace(size_type __pos, size_type __n1, size_type __n2, value_type __c) {
+  size_type __sz = size();
+  if (__pos > __sz)
+    __throw_out_of_range();
+  __n1            = std::min(__n1, __sz - __pos);
+  size_type __cap = capacity();
+  value_type* __p;
+  if (__cap - __sz + __n1 >= __n2) {
+    __p = std::__to_address(__get_pointer());
+    if (__n1 != __n2) {
+      if (__n2 > __n1)
+        __annotate_increase(__n2 - __n1);
+      size_type __n_move = __sz - __pos - __n1;
+      if (__n_move != 0)
+        traits_type::move(__p + __pos + __n2, __p + __pos + __n1, __n_move);
+    }
+  } else {
+    __grow_by_without_replace(__cap, __sz - __n1 + __n2 - __cap, __sz, __pos, __n1, __n2);
+    __p = std::__to_address(__get_long_pointer());
+  }
+  traits_type::assign(__p + __pos, __n2, __c);
+  return __null_terminate_at(__p, __sz - (__n1 - __n2));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::replace(
+    const_iterator __i1, const_iterator __i2, _InputIterator __j1, _InputIterator __j2) {
+  const basic_string __temp(__j1, __j2, __alloc());
+  return replace(__i1, __i2, __temp);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::replace(
+    size_type __pos1, size_type __n1, const basic_string& __str, size_type __pos2, size_type __n2) {
+  size_type __str_sz = __str.size();
+  if (__pos2 > __str_sz)
+    __throw_out_of_range();
+  return replace(__pos1, __n1, __str.data() + __pos2, std::min(__n2, __str_sz - __pos2));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _Tp,
+          __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                            !__is_same_uncvref<_Tp, basic_string<_CharT, _Traits, _Allocator> >::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::replace(
+    size_type __pos1, size_type __n1, const _Tp& __t, size_type __pos2, size_type __n2) {
+  __self_view __sv   = __t;
+  size_type __str_sz = __sv.size();
+  if (__pos2 > __str_sz)
+    __throw_out_of_range();
+  return replace(__pos1, __n1, __sv.data() + __pos2, std::min(__n2, __str_sz - __pos2));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::replace(size_type __pos, size_type __n1, const value_type* __s) {
+  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::replace received nullptr");
+  return replace(__pos, __n1, __s, traits_type::length(__s));
+}
+
+// erase
+
+// 'externally instantiated' erase() implementation, called when __n != npos.
+// Does not check __pos against size()
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE void
+basic_string<_CharT, _Traits, _Allocator>::__erase_external_with_move(size_type __pos, size_type __n) {
+  if (__n) {
+    size_type __sz     = size();
+    value_type* __p    = std::__to_address(__get_pointer());
+    __n                = std::min(__n, __sz - __pos);
+    size_type __n_move = __sz - __pos - __n;
+    if (__n_move != 0)
+      traits_type::move(__p + __pos, __p + __pos + __n, __n_move);
+    __null_terminate_at(__p, __sz - __n);
+  }
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::erase(size_type __pos, size_type __n) {
+  if (__pos > size())
+    __throw_out_of_range();
+  if (__n == npos) {
+    __erase_to_end(__pos);
+  } else {
+    __erase_external_with_move(__pos, __n);
+  }
+  return *this;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::iterator
+basic_string<_CharT, _Traits, _Allocator>::erase(const_iterator __pos) {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+      __pos != end(), "string::erase(iterator) called with a non-dereferenceable iterator");
+  iterator __b  = begin();
+  size_type __r = static_cast<size_type>(__pos - __b);
+  erase(__r, 1);
+  return __b + static_cast<difference_type>(__r);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::iterator
+basic_string<_CharT, _Traits, _Allocator>::erase(const_iterator __first, const_iterator __last) {
+  _LIBCPP_ASSERT_VALID_INPUT_RANGE(__first <= __last, "string::erase(first, last) called with invalid range");
+  iterator __b  = begin();
+  size_type __r = static_cast<size_type>(__first - __b);
+  erase(__r, static_cast<size_type>(__last - __first));
+  return __b + static_cast<difference_type>(__r);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::pop_back() {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "string::pop_back(): string is already empty");
+  __erase_to_end(size() - 1);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::clear() _NOEXCEPT {
+  size_type __old_size = size();
+  if (__is_long()) {
+    traits_type::assign(*__get_long_pointer(), value_type());
+    __set_long_size(0);
+  } else {
+    traits_type::assign(*__get_short_pointer(), value_type());
+    __set_short_size(0);
+  }
+  __annotate_shrink(__old_size);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::resize(size_type __n, value_type __c) {
+  size_type __sz = size();
+  if (__n > __sz)
+    append(__n - __sz, __c);
+  else
+    __erase_to_end(__n);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void
+basic_string<_CharT, _Traits, _Allocator>::__resize_default_init(size_type __n) {
+  size_type __sz = size();
+  if (__n > __sz) {
+    __append_default_init(__n - __sz);
+  } else
+    __erase_to_end(__n);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::reserve(size_type __requested_capacity) {
+  if (__requested_capacity > max_size())
+    __throw_length_error();
+
+  // Make sure reserve(n) never shrinks. This is technically only required in C++20
+  // and later (since P0966R1), however we provide consistent behavior in all Standard
+  // modes because this function is instantiated in the shared library.
+  if (__requested_capacity <= capacity())
+    return;
+
+  size_type __target_capacity = std::max(__requested_capacity, size());
+  __target_capacity           = __recommend(__target_capacity);
+  if (__target_capacity == capacity())
+    return;
+
+  __shrink_or_extend(__target_capacity);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::shrink_to_fit() _NOEXCEPT {
+  size_type __target_capacity = __recommend(size());
+  if (__target_capacity == capacity())
+    return;
+
+  __shrink_or_extend(__target_capacity);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+basic_string<_CharT, _Traits, _Allocator>::__shrink_or_extend(size_type __target_capacity) {
+  __annotate_delete();
+  size_type __cap = capacity();
+  size_type __sz  = size();
+
+  pointer __new_data, __p;
+  bool __was_long, __now_long;
+  if (__fits_in_sso(__target_capacity)) {
+    __was_long = true;
+    __now_long = false;
+    __new_data = __get_short_pointer();
+    __p        = __get_long_pointer();
+  } else {
+    if (__target_capacity > __cap) {
+      // Extend
+      // - called from reserve should propagate the exception thrown.
+      auto __allocation = std::__allocate_at_least(__alloc(), __target_capacity + 1);
+      __new_data        = __allocation.ptr;
+      __target_capacity = __allocation.count - 1;
+    } else {
+      // Shrink
+      // - called from shrink_to_fit should not throw.
+      // - called from reserve may throw but is not required to.
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+        auto __allocation = std::__allocate_at_least(__alloc(), __target_capacity + 1);
+
+        // The Standard mandates shrink_to_fit() does not increase the capacity.
+        // With equal capacity keep the existing buffer. This avoids extra work
+        // due to swapping the elements.
+        if (__allocation.count - 1 > __target_capacity) {
+          __alloc_traits::deallocate(__alloc(), __allocation.ptr, __allocation.count);
+          __annotate_new(__sz); // Undoes the __annotate_delete()
+          return;
+        }
+        __new_data        = __allocation.ptr;
+        __target_capacity = __allocation.count - 1;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      } catch (...) {
+        return;
+      }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    }
+    __begin_lifetime(__new_data, __target_capacity + 1);
+    __now_long = true;
+    __was_long = __is_long();
+    __p        = __get_pointer();
+  }
+  traits_type::copy(std::__to_address(__new_data), std::__to_address(__p), size() + 1);
+  if (__was_long)
+    __alloc_traits::deallocate(__alloc(), __p, __cap + 1);
+  if (__now_long) {
+    __set_long_cap(__target_capacity + 1);
+    __set_long_size(__sz);
+    __set_long_pointer(__new_data);
+  } else
+    __set_short_size(__sz);
+  __annotate_new(__sz);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::const_reference
+basic_string<_CharT, _Traits, _Allocator>::at(size_type __n) const {
+  if (__n >= size())
+    __throw_out_of_range();
+  return (*this)[__n];
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::reference
+basic_string<_CharT, _Traits, _Allocator>::at(size_type __n) {
+  if (__n >= size())
+    __throw_out_of_range();
+  return (*this)[__n];
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::copy(value_type* __s, size_type __n, size_type __pos) const {
+  size_type __sz = size();
+  if (__pos > __sz)
+    __throw_out_of_range();
+  size_type __rlen = std::min(__n, __sz - __pos);
+  traits_type::copy(__s, data() + __pos, __rlen);
+  return __rlen;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::swap(basic_string& __str)
+#if _LIBCPP_STD_VER >= 14
+    _NOEXCEPT
+#else
+    _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>)
+#endif
+{
+  _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+      __alloc_traits::propagate_on_container_swap::value || __alloc_traits::is_always_equal::value ||
+          __alloc() == __str.__alloc(),
+      "swapping non-equal allocators");
+  if (!__is_long())
+    __annotate_delete();
+  if (this != &__str && !__str.__is_long())
+    __str.__annotate_delete();
+  std::swap(__r_.first(), __str.__r_.first());
+  std::__swap_allocator(__alloc(), __str.__alloc());
+  if (!__is_long())
+    __annotate_new(__get_short_size());
+  if (this != &__str && !__str.__is_long())
+    __str.__annotate_new(__str.__get_short_size());
+}
+
+// find
+
+template <class _Traits>
+struct _LIBCPP_HIDDEN __traits_eq {
+  typedef typename _Traits::char_type char_type;
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const char_type& __x, const char_type& __y) _NOEXCEPT {
+    return _Traits::eq(__x, __y);
+  }
+};
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+  _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::find(): received nullptr");
+  return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find(const basic_string& __str, size_type __pos) const _NOEXCEPT {
+  return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __str.data(), __pos, __str.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find(const _Tp& __t, size_type __pos) const _NOEXCEPT {
+  __self_view __sv = __t;
+  return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __sv.data(), __pos, __sv.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find(const value_type* __s, size_type __pos) const _NOEXCEPT {
+  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find(): received nullptr");
+  return std::__str_find<value_type, size_type, traits_type, npos>(
+      data(), size(), __s, __pos, traits_type::length(__s));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find(value_type __c, size_type __pos) const _NOEXCEPT {
+  return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
+}
+
+// rfind
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::rfind(
+    const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+  _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::rfind(): received nullptr");
+  return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::rfind(const basic_string& __str, size_type __pos) const _NOEXCEPT {
+  return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __str.data(), __pos, __str.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::rfind(const _Tp& __t, size_type __pos) const _NOEXCEPT {
+  __self_view __sv = __t;
+  return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __sv.data(), __pos, __sv.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::rfind(const value_type* __s, size_type __pos) const _NOEXCEPT {
+  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::rfind(): received nullptr");
+  return std::__str_rfind<value_type, size_type, traits_type, npos>(
+      data(), size(), __s, __pos, traits_type::length(__s));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::rfind(value_type __c, size_type __pos) const _NOEXCEPT {
+  return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
+}
+
+// find_first_of
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_first_of(
+    const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+  _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::find_first_of(): received nullptr");
+  return std::__str_find_first_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_first_of(const basic_string& __str, size_type __pos) const _NOEXCEPT {
+  return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
+      data(), size(), __str.data(), __pos, __str.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_first_of(const _Tp& __t, size_type __pos) const _NOEXCEPT {
+  __self_view __sv = __t;
+  return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
+      data(), size(), __sv.data(), __pos, __sv.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_first_of(const value_type* __s, size_type __pos) const _NOEXCEPT {
+  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_first_of(): received nullptr");
+  return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
+      data(), size(), __s, __pos, traits_type::length(__s));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_first_of(value_type __c, size_type __pos) const _NOEXCEPT {
+  return find(__c, __pos);
+}
+
+// find_last_of
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_last_of(
+    const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+  _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::find_last_of(): received nullptr");
+  return std::__str_find_last_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_last_of(const basic_string& __str, size_type __pos) const _NOEXCEPT {
+  return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
+      data(), size(), __str.data(), __pos, __str.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_last_of(const _Tp& __t, size_type __pos) const _NOEXCEPT {
+  __self_view __sv = __t;
+  return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
+      data(), size(), __sv.data(), __pos, __sv.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_last_of(const value_type* __s, size_type __pos) const _NOEXCEPT {
+  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_last_of(): received nullptr");
+  return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
+      data(), size(), __s, __pos, traits_type::length(__s));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_last_of(value_type __c, size_type __pos) const _NOEXCEPT {
+  return rfind(__c, __pos);
+}
+
+// find_first_not_of
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(
+    const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+  _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::find_first_not_of(): received nullptr");
+  return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(
+    const basic_string& __str, size_type __pos) const _NOEXCEPT {
+  return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
+      data(), size(), __str.data(), __pos, __str.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(const _Tp& __t, size_type __pos) const _NOEXCEPT {
+  __self_view __sv = __t;
+  return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
+      data(), size(), __sv.data(), __pos, __sv.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(const value_type* __s, size_type __pos) const _NOEXCEPT {
+  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_first_not_of(): received nullptr");
+  return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
+      data(), size(), __s, __pos, traits_type::length(__s));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(value_type __c, size_type __pos) const _NOEXCEPT {
+  return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
+}
+
+// find_last_not_of
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(
+    const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+  _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::find_last_not_of(): received nullptr");
+  return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(
+    const basic_string& __str, size_type __pos) const _NOEXCEPT {
+  return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
+      data(), size(), __str.data(), __pos, __str.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(const _Tp& __t, size_type __pos) const _NOEXCEPT {
+  __self_view __sv = __t;
+  return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
+      data(), size(), __sv.data(), __pos, __sv.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(const value_type* __s, size_type __pos) const _NOEXCEPT {
+  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_last_not_of(): received nullptr");
+  return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
+      data(), size(), __s, __pos, traits_type::length(__s));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::size_type
+basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(value_type __c, size_type __pos) const _NOEXCEPT {
+  return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
+}
+
+// compare
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 int basic_string<_CharT, _Traits, _Allocator>::compare(const _Tp& __t) const _NOEXCEPT {
+  __self_view __sv = __t;
+  size_t __lhs_sz  = size();
+  size_t __rhs_sz  = __sv.size();
+  int __result     = traits_type::compare(data(), __sv.data(), std::min(__lhs_sz, __rhs_sz));
+  if (__result != 0)
+    return __result;
+  if (__lhs_sz < __rhs_sz)
+    return -1;
+  if (__lhs_sz > __rhs_sz)
+    return 1;
+  return 0;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 int
+basic_string<_CharT, _Traits, _Allocator>::compare(const basic_string& __str) const _NOEXCEPT {
+  return compare(__self_view(__str));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 int basic_string<_CharT, _Traits, _Allocator>::compare(
+    size_type __pos1, size_type __n1, const value_type* __s, size_type __n2) const {
+  _LIBCPP_ASSERT_NON_NULL(__n2 == 0 || __s != nullptr, "string::compare(): received nullptr");
+  size_type __sz = size();
+  if (__pos1 > __sz || __n2 == npos)
+    __throw_out_of_range();
+  size_type __rlen = std::min(__n1, __sz - __pos1);
+  int __r          = traits_type::compare(data() + __pos1, __s, std::min(__rlen, __n2));
+  if (__r == 0) {
+    if (__rlen < __n2)
+      __r = -1;
+    else if (__rlen > __n2)
+      __r = 1;
+  }
+  return __r;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 int
+basic_string<_CharT, _Traits, _Allocator>::compare(size_type __pos1, size_type __n1, const _Tp& __t) const {
+  __self_view __sv = __t;
+  return compare(__pos1, __n1, __sv.data(), __sv.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 int
+basic_string<_CharT, _Traits, _Allocator>::compare(size_type __pos1, size_type __n1, const basic_string& __str) const {
+  return compare(__pos1, __n1, __str.data(), __str.size());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+template <class _Tp,
+          __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
+                            !__is_same_uncvref<_Tp, basic_string<_CharT, _Traits, _Allocator> >::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 int basic_string<_CharT, _Traits, _Allocator>::compare(
+    size_type __pos1, size_type __n1, const _Tp& __t, size_type __pos2, size_type __n2) const {
+  __self_view __sv = __t;
+  return __self_view(*this).substr(__pos1, __n1).compare(__sv.substr(__pos2, __n2));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 int basic_string<_CharT, _Traits, _Allocator>::compare(
+    size_type __pos1, size_type __n1, const basic_string& __str, size_type __pos2, size_type __n2) const {
+  return compare(__pos1, __n1, __self_view(__str), __pos2, __n2);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 int
+basic_string<_CharT, _Traits, _Allocator>::compare(const value_type* __s) const _NOEXCEPT {
+  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::compare(): received nullptr");
+  return compare(0, npos, __s, traits_type::length(__s));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 int
+basic_string<_CharT, _Traits, _Allocator>::compare(size_type __pos1, size_type __n1, const value_type* __s) const {
+  _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::compare(): received nullptr");
+  return compare(__pos1, __n1, __s, traits_type::length(__s));
+}
+
+// __invariants
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 bool basic_string<_CharT, _Traits, _Allocator>::__invariants() const {
+  if (size() > capacity())
+    return false;
+  if (capacity() < __min_cap - 1)
+    return false;
+  if (data() == nullptr)
+    return false;
+  if (!_Traits::eq(data()[size()], value_type()))
+    return false;
+  return true;
+}
+
+// __clear_and_shrink
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__clear_and_shrink() _NOEXCEPT {
+  clear();
+  if (__is_long()) {
+    __annotate_delete();
+    __alloc_traits::deallocate(__alloc(), __get_long_pointer(), capacity() + 1);
+    __r_.first() = __rep();
+  }
+}
+
+// operator==
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool
+operator==(const basic_string<_CharT, _Traits, _Allocator>& __lhs,
+           const basic_string<_CharT, _Traits, _Allocator>& __rhs) _NOEXCEPT {
+#if _LIBCPP_STD_VER >= 20
+  return basic_string_view<_CharT, _Traits>(__lhs) == basic_string_view<_CharT, _Traits>(__rhs);
+#else
+  size_t __lhs_sz = __lhs.size();
+  return __lhs_sz == __rhs.size() && _Traits::compare(__lhs.data(), __rhs.data(), __lhs_sz) == 0;
+#endif
+}
+
+template <class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool
+operator==(const basic_string<char, char_traits<char>, _Allocator>& __lhs,
+           const basic_string<char, char_traits<char>, _Allocator>& __rhs) _NOEXCEPT {
+  size_t __sz = __lhs.size();
+  if (__sz != __rhs.size())
+    return false;
+  return char_traits<char>::compare(__lhs.data(), __rhs.data(), __sz) == 0;
+}
+
+#if _LIBCPP_STD_VER <= 17
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const _CharT* __lhs, const basic_string<_CharT, _Traits, _Allocator>& __rhs) _NOEXCEPT {
+  typedef basic_string<_CharT, _Traits, _Allocator> _String;
+  _LIBCPP_ASSERT_NON_NULL(__lhs != nullptr, "operator==(char*, basic_string): received nullptr");
+  size_t __lhs_len = _Traits::length(__lhs);
+  if (__lhs_len != __rhs.size())
+    return false;
+  return __rhs.compare(0, _String::npos, __lhs, __lhs_len) == 0;
+}
+#endif // _LIBCPP_STD_VER <= 17
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool
+operator==(const basic_string<_CharT, _Traits, _Allocator>& __lhs, const _CharT* __rhs) _NOEXCEPT {
+#if _LIBCPP_STD_VER >= 20
+  return basic_string_view<_CharT, _Traits>(__lhs) == basic_string_view<_CharT, _Traits>(__rhs);
+#else
+  typedef basic_string<_CharT, _Traits, _Allocator> _String;
+  _LIBCPP_ASSERT_NON_NULL(__rhs != nullptr, "operator==(basic_string, char*): received nullptr");
+  size_t __rhs_len = _Traits::length(__rhs);
+  if (__rhs_len != __lhs.size())
+    return false;
+  return __lhs.compare(0, _String::npos, __rhs, __rhs_len) == 0;
+#endif
+}
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI constexpr auto operator<=>(const basic_string<_CharT, _Traits, _Allocator>& __lhs,
+                                                 const basic_string<_CharT, _Traits, _Allocator>& __rhs) noexcept {
+  return basic_string_view<_CharT, _Traits>(__lhs) <=> basic_string_view<_CharT, _Traits>(__rhs);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+operator<=>(const basic_string<_CharT, _Traits, _Allocator>& __lhs, const _CharT* __rhs) {
+  return basic_string_view<_CharT, _Traits>(__lhs) <=> basic_string_view<_CharT, _Traits>(__rhs);
+}
+
+#else  // _LIBCPP_STD_VER >= 20
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const basic_string<_CharT, _Traits, _Allocator>& __lhs,
+                                             const basic_string<_CharT, _Traits, _Allocator>& __rhs) _NOEXCEPT {
+  return !(__lhs == __rhs);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const _CharT* __lhs, const basic_string<_CharT, _Traits, _Allocator>& __rhs) _NOEXCEPT {
+  return !(__lhs == __rhs);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const basic_string<_CharT, _Traits, _Allocator>& __lhs, const _CharT* __rhs) _NOEXCEPT {
+  return !(__lhs == __rhs);
+}
+
+// operator<
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(const basic_string<_CharT, _Traits, _Allocator>& __lhs,
+                                            const basic_string<_CharT, _Traits, _Allocator>& __rhs) _NOEXCEPT {
+  return __lhs.compare(__rhs) < 0;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<(const basic_string<_CharT, _Traits, _Allocator>& __lhs, const _CharT* __rhs) _NOEXCEPT {
+  return __lhs.compare(__rhs) < 0;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<(const _CharT* __lhs, const basic_string<_CharT, _Traits, _Allocator>& __rhs) _NOEXCEPT {
+  return __rhs.compare(__lhs) > 0;
+}
+
+// operator>
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>(const basic_string<_CharT, _Traits, _Allocator>& __lhs,
+                                            const basic_string<_CharT, _Traits, _Allocator>& __rhs) _NOEXCEPT {
+  return __rhs < __lhs;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>(const basic_string<_CharT, _Traits, _Allocator>& __lhs, const _CharT* __rhs) _NOEXCEPT {
+  return __rhs < __lhs;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>(const _CharT* __lhs, const basic_string<_CharT, _Traits, _Allocator>& __rhs) _NOEXCEPT {
+  return __rhs < __lhs;
+}
+
+// operator<=
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const basic_string<_CharT, _Traits, _Allocator>& __lhs,
+                                             const basic_string<_CharT, _Traits, _Allocator>& __rhs) _NOEXCEPT {
+  return !(__rhs < __lhs);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<=(const basic_string<_CharT, _Traits, _Allocator>& __lhs, const _CharT* __rhs) _NOEXCEPT {
+  return !(__rhs < __lhs);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator<=(const _CharT* __lhs, const basic_string<_CharT, _Traits, _Allocator>& __rhs) _NOEXCEPT {
+  return !(__rhs < __lhs);
+}
+
+// operator>=
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>=(const basic_string<_CharT, _Traits, _Allocator>& __lhs,
+                                             const basic_string<_CharT, _Traits, _Allocator>& __rhs) _NOEXCEPT {
+  return !(__lhs < __rhs);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>=(const basic_string<_CharT, _Traits, _Allocator>& __lhs, const _CharT* __rhs) _NOEXCEPT {
+  return !(__lhs < __rhs);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool
+operator>=(const _CharT* __lhs, const basic_string<_CharT, _Traits, _Allocator>& __rhs) _NOEXCEPT {
+  return !(__lhs < __rhs);
+}
+#endif // _LIBCPP_STD_VER >= 20
+
+// operator +
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(const basic_string<_CharT, _Traits, _Allocator>& __lhs,
+          const basic_string<_CharT, _Traits, _Allocator>& __rhs) {
+  using _String = basic_string<_CharT, _Traits, _Allocator>;
+  auto __lhs_sz = __lhs.size();
+  auto __rhs_sz = __rhs.size();
+  _String __r(__uninitialized_size_tag(),
+              __lhs_sz + __rhs_sz,
+              _String::__alloc_traits::select_on_container_copy_construction(__lhs.get_allocator()));
+  auto __ptr = std::__to_address(__r.__get_pointer());
+  _Traits::copy(__ptr, __lhs.data(), __lhs_sz);
+  _Traits::copy(__ptr + __lhs_sz, __rhs.data(), __rhs_sz);
+  _Traits::assign(__ptr + __lhs_sz + __rhs_sz, 1, _CharT());
+  return __r;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDDEN _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(const _CharT* __lhs, const basic_string<_CharT, _Traits, _Allocator>& __rhs) {
+  using _String = basic_string<_CharT, _Traits, _Allocator>;
+  auto __lhs_sz = _Traits::length(__lhs);
+  auto __rhs_sz = __rhs.size();
+  _String __r(__uninitialized_size_tag(),
+              __lhs_sz + __rhs_sz,
+              _String::__alloc_traits::select_on_container_copy_construction(__rhs.get_allocator()));
+  auto __ptr = std::__to_address(__r.__get_pointer());
+  _Traits::copy(__ptr, __lhs, __lhs_sz);
+  _Traits::copy(__ptr + __lhs_sz, __rhs.data(), __rhs_sz);
+  _Traits::assign(__ptr + __lhs_sz + __rhs_sz, 1, _CharT());
+  return __r;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(_CharT __lhs, const basic_string<_CharT, _Traits, _Allocator>& __rhs) {
+  using _String                        = basic_string<_CharT, _Traits, _Allocator>;
+  typename _String::size_type __rhs_sz = __rhs.size();
+  _String __r(__uninitialized_size_tag(),
+              __rhs_sz + 1,
+              _String::__alloc_traits::select_on_container_copy_construction(__rhs.get_allocator()));
+  auto __ptr = std::__to_address(__r.__get_pointer());
+  _Traits::assign(__ptr, 1, __lhs);
+  _Traits::copy(__ptr + 1, __rhs.data(), __rhs_sz);
+  _Traits::assign(__ptr + 1 + __rhs_sz, 1, _CharT());
+  return __r;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(const basic_string<_CharT, _Traits, _Allocator>& __lhs, const _CharT* __rhs) {
+  using _String                        = basic_string<_CharT, _Traits, _Allocator>;
+  typename _String::size_type __lhs_sz = __lhs.size();
+  typename _String::size_type __rhs_sz = _Traits::length(__rhs);
+  _String __r(__uninitialized_size_tag(),
+              __lhs_sz + __rhs_sz,
+              _String::__alloc_traits::select_on_container_copy_construction(__lhs.get_allocator()));
+  auto __ptr = std::__to_address(__r.__get_pointer());
+  _Traits::copy(__ptr, __lhs.data(), __lhs_sz);
+  _Traits::copy(__ptr + __lhs_sz, __rhs, __rhs_sz);
+  _Traits::assign(__ptr + __lhs_sz + __rhs_sz, 1, _CharT());
+  return __r;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(const basic_string<_CharT, _Traits, _Allocator>& __lhs, _CharT __rhs) {
+  using _String                        = basic_string<_CharT, _Traits, _Allocator>;
+  typename _String::size_type __lhs_sz = __lhs.size();
+  _String __r(__uninitialized_size_tag(),
+              __lhs_sz + 1,
+              _String::__alloc_traits::select_on_container_copy_construction(__lhs.get_allocator()));
+  auto __ptr = std::__to_address(__r.__get_pointer());
+  _Traits::copy(__ptr, __lhs.data(), __lhs_sz);
+  _Traits::assign(__ptr + __lhs_sz, 1, __rhs);
+  _Traits::assign(__ptr + 1 + __lhs_sz, 1, _CharT());
+  return __r;
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(basic_string<_CharT, _Traits, _Allocator>&& __lhs, const basic_string<_CharT, _Traits, _Allocator>& __rhs) {
+  return std::move(__lhs.append(__rhs));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(const basic_string<_CharT, _Traits, _Allocator>& __lhs, basic_string<_CharT, _Traits, _Allocator>&& __rhs) {
+  return std::move(__rhs.insert(0, __lhs));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(basic_string<_CharT, _Traits, _Allocator>&& __lhs, basic_string<_CharT, _Traits, _Allocator>&& __rhs) {
+  return std::move(__lhs.append(__rhs));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(const _CharT* __lhs, basic_string<_CharT, _Traits, _Allocator>&& __rhs) {
+  return std::move(__rhs.insert(0, __lhs));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(_CharT __lhs, basic_string<_CharT, _Traits, _Allocator>&& __rhs) {
+  __rhs.insert(__rhs.begin(), __lhs);
+  return std::move(__rhs);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(basic_string<_CharT, _Traits, _Allocator>&& __lhs, const _CharT* __rhs) {
+  return std::move(__lhs.append(__rhs));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
+operator+(basic_string<_CharT, _Traits, _Allocator>&& __lhs, _CharT __rhs) {
+  __lhs.push_back(__rhs);
+  return std::move(__lhs);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+#if _LIBCPP_STD_VER >= 26
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI constexpr basic_string<_CharT, _Traits, _Allocator>
+operator+(const basic_string<_CharT, _Traits, _Allocator>& __lhs,
+          type_identity_t<basic_string_view<_CharT, _Traits>> __rhs) {
+  using _String                        = basic_string<_CharT, _Traits, _Allocator>;
+  typename _String::size_type __lhs_sz = __lhs.size();
+  typename _String::size_type __rhs_sz = __rhs.size();
+  _String __r(__uninitialized_size_tag(),
+              __lhs_sz + __rhs_sz,
+              _String::__alloc_traits::select_on_container_copy_construction(__lhs.get_allocator()));
+  auto __ptr = std::__to_address(__r.__get_pointer());
+  _Traits::copy(__ptr, __lhs.data(), __lhs_sz);
+  _Traits::copy(__ptr + __lhs_sz, __rhs.data(), __rhs_sz);
+  _Traits::assign(__ptr + __lhs_sz + __rhs_sz, 1, _CharT());
+  return __r;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI constexpr basic_string<_CharT, _Traits, _Allocator>
+operator+(basic_string<_CharT, _Traits, _Allocator>&& __lhs,
+          type_identity_t<basic_string_view<_CharT, _Traits>> __rhs) {
+  __lhs.append(__rhs);
+  return std::move(__lhs);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI constexpr basic_string<_CharT, _Traits, _Allocator>
+operator+(type_identity_t<basic_string_view<_CharT, _Traits>> __lhs,
+          const basic_string<_CharT, _Traits, _Allocator>& __rhs) {
+  using _String                        = basic_string<_CharT, _Traits, _Allocator>;
+  typename _String::size_type __lhs_sz = __lhs.size();
+  typename _String::size_type __rhs_sz = __rhs.size();
+  _String __r(__uninitialized_size_tag(),
+              __lhs_sz + __rhs_sz,
+              _String::__alloc_traits::select_on_container_copy_construction(__rhs.get_allocator()));
+  auto __ptr = std::__to_address(__r.__get_pointer());
+  _Traits::copy(__ptr, __lhs.data(), __lhs_sz);
+  _Traits::copy(__ptr + __lhs_sz, __rhs.data(), __rhs_sz);
+  _Traits::assign(__ptr + __lhs_sz + __rhs_sz, 1, _CharT());
+  return __r;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI constexpr basic_string<_CharT, _Traits, _Allocator>
+operator+(type_identity_t<basic_string_view<_CharT, _Traits>> __lhs,
+          basic_string<_CharT, _Traits, _Allocator>&& __rhs) {
+  __rhs.insert(0, __lhs);
+  return std::move(__rhs);
+}
+
+#endif // _LIBCPP_STD_VER >= 26
+
+// swap
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+swap(basic_string<_CharT, _Traits, _Allocator>& __lhs, basic_string<_CharT, _Traits, _Allocator>& __rhs)
+    _NOEXCEPT_(_NOEXCEPT_(__lhs.swap(__rhs))) {
+  __lhs.swap(__rhs);
+}
+
+_LIBCPP_EXPORTED_FROM_ABI int stoi(const string& __str, size_t* __idx = nullptr, int __base = 10);
+_LIBCPP_EXPORTED_FROM_ABI long stol(const string& __str, size_t* __idx = nullptr, int __base = 10);
+_LIBCPP_EXPORTED_FROM_ABI unsigned long stoul(const string& __str, size_t* __idx = nullptr, int __base = 10);
+_LIBCPP_EXPORTED_FROM_ABI long long stoll(const string& __str, size_t* __idx = nullptr, int __base = 10);
+_LIBCPP_EXPORTED_FROM_ABI unsigned long long stoull(const string& __str, size_t* __idx = nullptr, int __base = 10);
+
+_LIBCPP_EXPORTED_FROM_ABI float stof(const string& __str, size_t* __idx = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI double stod(const string& __str, size_t* __idx = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI long double stold(const string& __str, size_t* __idx = nullptr);
+
+_LIBCPP_EXPORTED_FROM_ABI string to_string(int __val);
+_LIBCPP_EXPORTED_FROM_ABI string to_string(unsigned __val);
+_LIBCPP_EXPORTED_FROM_ABI string to_string(long __val);
+_LIBCPP_EXPORTED_FROM_ABI string to_string(unsigned long __val);
+_LIBCPP_EXPORTED_FROM_ABI string to_string(long long __val);
+_LIBCPP_EXPORTED_FROM_ABI string to_string(unsigned long long __val);
+_LIBCPP_EXPORTED_FROM_ABI string to_string(float __val);
+_LIBCPP_EXPORTED_FROM_ABI string to_string(double __val);
+_LIBCPP_EXPORTED_FROM_ABI string to_string(long double __val);
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+_LIBCPP_EXPORTED_FROM_ABI int stoi(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
+_LIBCPP_EXPORTED_FROM_ABI long stol(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
+_LIBCPP_EXPORTED_FROM_ABI unsigned long stoul(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
+_LIBCPP_EXPORTED_FROM_ABI long long stoll(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
+_LIBCPP_EXPORTED_FROM_ABI unsigned long long stoull(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
+
+_LIBCPP_EXPORTED_FROM_ABI float stof(const wstring& __str, size_t* __idx = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI double stod(const wstring& __str, size_t* __idx = nullptr);
+_LIBCPP_EXPORTED_FROM_ABI long double stold(const wstring& __str, size_t* __idx = nullptr);
+
+_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(int __val);
+_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(unsigned __val);
+_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(long __val);
+_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(unsigned long __val);
+_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(long long __val);
+_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(unsigned long long __val);
+_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(float __val);
+_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(double __val);
+_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(long double __val);
+#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_TEMPLATE_DATA_VIS const typename basic_string<_CharT, _Traits, _Allocator>::size_type
+    basic_string<_CharT, _Traits, _Allocator>::npos;
+
+template <class _CharT, class _Allocator>
+struct __string_hash : public __unary_function<basic_string<_CharT, char_traits<_CharT>, _Allocator>, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t
+  operator()(const basic_string<_CharT, char_traits<_CharT>, _Allocator>& __val) const _NOEXCEPT {
+    return std::__do_string_hash(__val.data(), __val.data() + __val.size());
+  }
+};
+
+template <class _Allocator>
+struct hash<basic_string<char, char_traits<char>, _Allocator> > : __string_hash<char, _Allocator> {};
+
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+template <class _Allocator>
+struct hash<basic_string<char8_t, char_traits<char8_t>, _Allocator> > : __string_hash<char8_t, _Allocator> {};
+#endif
+
+template <class _Allocator>
+struct hash<basic_string<char16_t, char_traits<char16_t>, _Allocator> > : __string_hash<char16_t, _Allocator> {};
+
+template <class _Allocator>
+struct hash<basic_string<char32_t, char_traits<char32_t>, _Allocator> > : __string_hash<char32_t, _Allocator> {};
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <class _Allocator>
+struct hash<basic_string<wchar_t, char_traits<wchar_t>, _Allocator> > : __string_hash<wchar_t, _Allocator> {};
+#endif
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const basic_string<_CharT, _Traits, _Allocator>& __str);
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+operator>>(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _Allocator>& __str);
+
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+getline(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _Allocator>& __str, _CharT __dlm);
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+getline(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _Allocator>& __str);
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+getline(basic_istream<_CharT, _Traits>&& __is, basic_string<_CharT, _Traits, _Allocator>& __str, _CharT __dlm);
+
+template <class _CharT, class _Traits, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
+getline(basic_istream<_CharT, _Traits>&& __is, basic_string<_CharT, _Traits, _Allocator>& __str);
+
+#if _LIBCPP_STD_VER >= 20
+template <class _CharT, class _Traits, class _Allocator, class _Up>
+inline _LIBCPP_HIDE_FROM_ABI typename basic_string<_CharT, _Traits, _Allocator>::size_type
+erase(basic_string<_CharT, _Traits, _Allocator>& __str, const _Up& __v) {
+  auto __old_size = __str.size();
+  __str.erase(std::remove(__str.begin(), __str.end(), __v), __str.end());
+  return __old_size - __str.size();
+}
+
+template <class _CharT, class _Traits, class _Allocator, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI typename basic_string<_CharT, _Traits, _Allocator>::size_type
+erase_if(basic_string<_CharT, _Traits, _Allocator>& __str, _Predicate __pred) {
+  auto __old_size = __str.size();
+  __str.erase(std::remove_if(__str.begin(), __str.end(), __pred), __str.end());
+  return __old_size - __str.size();
+}
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+// Literal suffixes for basic_string [basic.string.literals]
+inline namespace literals {
+inline namespace string_literals {
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<char>
+operator""s(const char* __str, size_t __len) {
+  return basic_string<char>(__str, __len);
+}
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<wchar_t>
+operator""s(const wchar_t* __str, size_t __len) {
+  return basic_string<wchar_t>(__str, __len);
+}
+#  endif
+
+#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string<char8_t> operator""s(const char8_t* __str, size_t __len) {
+  return basic_string<char8_t>(__str, __len);
+}
+#  endif
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<char16_t>
+operator""s(const char16_t* __str, size_t __len) {
+  return basic_string<char16_t>(__str, __len);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<char32_t>
+operator""s(const char32_t* __str, size_t __len) {
+  return basic_string<char32_t>(__str, __len);
+}
+} // namespace string_literals
+} // namespace literals
+
+#  if _LIBCPP_STD_VER >= 20
+template <>
+inline constexpr bool __format::__enable_insertable<std::basic_string<char>> = true;
+#    ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+inline constexpr bool __format::__enable_insertable<std::basic_string<wchar_t>> = true;
+#    endif
+#  endif
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <algorithm>
+#  include <concepts>
+#  include <cstdlib>
+#  include <iterator>
+#  include <new>
+#  include <type_traits>
+#  include <typeinfo>
+#  include <utility>
+#endif
+
+#endif // _LIBCPP_STRING
diff --git a/libcxx/include/__cxx03/string.h b/libcxx/include/__cxx03/string.h
new file mode 100644
index 000000000000..ae10d5ad2c76
--- /dev/null
+++ b/libcxx/include/__cxx03/string.h
@@ -0,0 +1,110 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_STRING_H
+#define _LIBCPP_STRING_H
+
+/*
+    string.h synopsis
+
+Macros:
+
+    NULL
+
+Types:
+
+    size_t
+
+void* memcpy(void* restrict s1, const void* restrict s2, size_t n);
+void* memmove(void* s1, const void* s2, size_t n);
+char* strcpy (char* restrict s1, const char* restrict s2);
+char* strncpy(char* restrict s1, const char* restrict s2, size_t n);
+char* strcat (char* restrict s1, const char* restrict s2);
+char* strncat(char* restrict s1, const char* restrict s2, size_t n);
+int memcmp(const void* s1, const void* s2, size_t n);
+int strcmp (const char* s1, const char* s2);
+int strncmp(const char* s1, const char* s2, size_t n);
+int strcoll(const char* s1, const char* s2);
+size_t strxfrm(char* restrict s1, const char* restrict s2, size_t n);
+const void* memchr(const void* s, int c, size_t n);
+      void* memchr(      void* s, int c, size_t n);
+const char* strchr(const char* s, int c);
+      char* strchr(      char* s, int c);
+size_t strcspn(const char* s1, const char* s2);
+const char* strpbrk(const char* s1, const char* s2);
+      char* strpbrk(      char* s1, const char* s2);
+const char* strrchr(const char* s, int c);
+      char* strrchr(      char* s, int c);
+size_t strspn(const char* s1, const char* s2);
+const char* strstr(const char* s1, const char* s2);
+      char* strstr(      char* s1, const char* s2);
+char* strtok(char* restrict s1, const char* restrict s2);
+void* memset(void* s, int c, size_t n);
+char* strerror(int errnum);
+size_t strlen(const char* s);
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if __has_include_next(<string.h>)
+#  include_next <string.h>
+#endif
+
+// MSVCRT, GNU libc and its derivates may already have the correct prototype in
+// <string.h>. This macro can be defined by users if their C library provides
+// the right signature.
+#if defined(__CORRECT_ISO_CPP_STRING_H_PROTO) || defined(_LIBCPP_MSVCRT) || defined(_STRING_H_CPLUSPLUS_98_CONFORMANCE_)
+#  define _LIBCPP_STRING_H_HAS_CONST_OVERLOADS
+#endif
+
+#if defined(__cplusplus) && !defined(_LIBCPP_STRING_H_HAS_CONST_OVERLOADS) && defined(_LIBCPP_PREFERRED_OVERLOAD)
+extern "C++" {
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD const char* strchr(const char* __s, int __c) {
+  return __builtin_strchr(__s, __c);
+}
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD char* strchr(char* __s, int __c) {
+  return __builtin_strchr(__s, __c);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD const char* strpbrk(const char* __s1, const char* __s2) {
+  return __builtin_strpbrk(__s1, __s2);
+}
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD char* strpbrk(char* __s1, const char* __s2) {
+  return __builtin_strpbrk(__s1, __s2);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD const char* strrchr(const char* __s, int __c) {
+  return __builtin_strrchr(__s, __c);
+}
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD char* strrchr(char* __s, int __c) {
+  return __builtin_strrchr(__s, __c);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD const void* memchr(const void* __s, int __c, size_t __n) {
+  return __builtin_memchr(__s, __c, __n);
+}
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD void* memchr(void* __s, int __c, size_t __n) {
+  return __builtin_memchr(__s, __c, __n);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD const char* strstr(const char* __s1, const char* __s2) {
+  return __builtin_strstr(__s1, __s2);
+}
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD char* strstr(char* __s1, const char* __s2) {
+  return __builtin_strstr(__s1, __s2);
+}
+} // extern "C++"
+#endif
+
+#endif // _LIBCPP_STRING_H
diff --git a/libcxx/include/__cxx03/string_view b/libcxx/include/__cxx03/string_view
new file mode 100644
index 000000000000..72dbf0bfa8e5
--- /dev/null
+++ b/libcxx/include/__cxx03/string_view
@@ -0,0 +1,948 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_STRING_VIEW
+#define _LIBCPP_STRING_VIEW
+
+// clang-format off
+
+/*
+
+    string_view synopsis
+
+#include <compare>
+
+namespace std {
+
+    // 7.2, Class template basic_string_view
+    template<class charT, class traits = char_traits<charT>>
+        class basic_string_view;
+
+    template<class charT, class traits>
+    inline constexpr bool ranges::enable_view<basic_string_view<charT, traits>> = true;
+
+    template<class charT, class traits>
+    inline constexpr bool ranges::enable_borrowed_range<basic_string_view<charT, traits>> = true;  // C++20
+
+    // 7.9, basic_string_view non-member comparison functions
+    template<class charT, class traits>
+    constexpr bool operator==(basic_string_view<charT, traits> x,
+                              basic_string_view<charT, traits> y) noexcept;
+    template<class charT, class traits>                                                            // Removed in C++20
+    constexpr bool operator!=(basic_string_view<charT, traits> x,
+                              basic_string_view<charT, traits> y) noexcept;
+    template<class charT, class traits>                                                            // Removed in C++20
+    constexpr bool operator< (basic_string_view<charT, traits> x,
+                                 basic_string_view<charT, traits> y) noexcept;
+    template<class charT, class traits>                                                            // Removed in C++20
+    constexpr bool operator> (basic_string_view<charT, traits> x,
+                              basic_string_view<charT, traits> y) noexcept;
+    template<class charT, class traits>                                                            // Removed in C++20
+    constexpr bool operator<=(basic_string_view<charT, traits> x,
+                                 basic_string_view<charT, traits> y) noexcept;
+    template<class charT, class traits>                                                            // Removed in C++20
+    constexpr bool operator>=(basic_string_view<charT, traits> x,
+                              basic_string_view<charT, traits> y) noexcept;
+    template<class charT, class traits>                                                            // Since C++20
+    constexpr see below operator<=>(basic_string_view<charT, traits> x,
+                                    basic_string_view<charT, traits> y) noexcept;
+
+    // see below, sufficient additional overloads of comparison functions
+
+    // 7.10, Inserters and extractors
+    template<class charT, class traits>
+      basic_ostream<charT, traits>&
+        operator<<(basic_ostream<charT, traits>& os,
+                   basic_string_view<charT, traits> str);
+
+    // basic_string_view typedef names
+    typedef basic_string_view<char> string_view;
+    typedef basic_string_view<char8_t> u8string_view; // C++20
+    typedef basic_string_view<char16_t> u16string_view;
+    typedef basic_string_view<char32_t> u32string_view;
+    typedef basic_string_view<wchar_t> wstring_view;
+
+    template<class charT, class traits = char_traits<charT>>
+    class basic_string_view {
+      public:
+      // types
+      typedef traits traits_type;
+      typedef charT value_type;
+      typedef charT* pointer;
+      typedef const charT* const_pointer;
+      typedef charT& reference;
+      typedef const charT& const_reference;
+      typedef implementation-defined const_iterator;
+      typedef const_iterator iterator;
+      typedef reverse_iterator<const_iterator> const_reverse_iterator;
+      typedef const_reverse_iterator reverse_iterator;
+      typedef size_t size_type;
+      typedef ptrdiff_t difference_type;
+      static constexpr size_type npos = size_type(-1);
+
+      // 7.3, basic_string_view constructors and assignment operators
+      constexpr basic_string_view() noexcept;
+      constexpr basic_string_view(const basic_string_view&) noexcept = default;
+      basic_string_view& operator=(const basic_string_view&) noexcept = default;
+      template<class Allocator>
+      constexpr basic_string_view(const charT* str);
+      basic_string_view(nullptr_t) = delete; // C++23
+      constexpr basic_string_view(const charT* str, size_type len);
+      template <class It, class End>
+      constexpr basic_string_view(It begin, End end); // C++20
+      template <class Range>
+      constexpr basic_string_view(Range&& r); // C++23
+
+      // 7.4, basic_string_view iterator support
+      constexpr const_iterator begin() const noexcept;
+      constexpr const_iterator end() const noexcept;
+      constexpr const_iterator cbegin() const noexcept;
+      constexpr const_iterator cend() const noexcept;
+      const_reverse_iterator rbegin() const noexcept;
+      const_reverse_iterator rend() const noexcept;
+      const_reverse_iterator crbegin() const noexcept;
+      const_reverse_iterator crend() const noexcept;
+
+      // 7.5, basic_string_view capacity
+      constexpr size_type size() const noexcept;
+      constexpr size_type length() const noexcept;
+      constexpr size_type max_size() const noexcept;
+      constexpr bool empty() const noexcept;
+
+      // 7.6, basic_string_view element access
+      constexpr const_reference operator[](size_type pos) const;
+      constexpr const_reference at(size_type pos) const;
+      constexpr const_reference front() const;
+      constexpr const_reference back() const;
+      constexpr const_pointer data() const noexcept;
+
+      // 7.7, basic_string_view modifiers
+      constexpr void remove_prefix(size_type n);
+      constexpr void remove_suffix(size_type n);
+      constexpr void swap(basic_string_view& s) noexcept;
+
+      size_type copy(charT* s, size_type n, size_type pos = 0) const;  // constexpr in C++20
+
+      constexpr basic_string_view substr(size_type pos = 0, size_type n = npos) const;
+      constexpr int compare(basic_string_view s) const noexcept;
+      constexpr int compare(size_type pos1, size_type n1, basic_string_view s) const;
+      constexpr int compare(size_type pos1, size_type n1,
+                            basic_string_view s, size_type pos2, size_type n2) const;
+      constexpr int compare(const charT* s) const;
+      constexpr int compare(size_type pos1, size_type n1, const charT* s) const;
+      constexpr int compare(size_type pos1, size_type n1,
+                            const charT* s, size_type n2) const;
+      constexpr size_type find(basic_string_view s, size_type pos = 0) const noexcept;
+      constexpr size_type find(charT c, size_type pos = 0) const noexcept;
+      constexpr size_type find(const charT* s, size_type pos, size_type n) const noexcept; // noexcept as an extension
+      constexpr size_type find(const charT* s, size_type pos = 0) const noexcept; // noexcept as an extension
+      constexpr size_type rfind(basic_string_view s, size_type pos = npos) const noexcept;
+      constexpr size_type rfind(charT c, size_type pos = npos) const noexcept;
+      constexpr size_type rfind(const charT* s, size_type pos, size_type n) const noexcept; // noexcept as an extension
+      constexpr size_type rfind(const charT* s, size_type pos = npos) const noexcept; // noexcept as an extension
+      constexpr size_type find_first_of(basic_string_view s, size_type pos = 0) const noexcept;
+      constexpr size_type find_first_of(charT c, size_type pos = 0) const noexcept;
+      constexpr size_type find_first_of(const charT* s, size_type pos, size_type n) const noexcept; // noexcept as an extension
+      constexpr size_type find_first_of(const charT* s, size_type pos = 0) const noexcept; // noexcept as an extension
+      constexpr size_type find_last_of(basic_string_view s, size_type pos = npos) const noexcept;
+      constexpr size_type find_last_of(charT c, size_type pos = npos) const noexcept;
+      constexpr size_type find_last_of(const charT* s, size_type pos, size_type n) const noexcept; // noexcept as an extension
+      constexpr size_type find_last_of(const charT* s, size_type pos = npos) const noexcept; // noexcept as an extension
+      constexpr size_type find_first_not_of(basic_string_view s, size_type pos = 0) const noexcept;
+      constexpr size_type find_first_not_of(charT c, size_type pos = 0) const noexcept;
+      constexpr size_type find_first_not_of(const charT* s, size_type pos, size_type n) const noexcept; // noexcept as an extension
+      constexpr size_type find_first_not_of(const charT* s, size_type pos = 0) const noexcept; // noexcept as an extension
+      constexpr size_type find_last_not_of(basic_string_view s, size_type pos = npos) const noexcept;
+      constexpr size_type find_last_not_of(charT c, size_type pos = npos) const noexcept;
+      constexpr size_type find_last_not_of(const charT* s, size_type pos, size_type n) const noexcept; // noexcept as an extension
+      constexpr size_type find_last_not_of(const charT* s, size_type pos = npos) const noexcept; // noexcept as an extension
+
+      constexpr bool starts_with(basic_string_view s) const noexcept; // C++20
+      constexpr bool starts_with(charT c) const noexcept;             // C++20
+      constexpr bool starts_with(const charT* s) const;               // C++20
+      constexpr bool ends_with(basic_string_view s) const noexcept;   // C++20
+      constexpr bool ends_with(charT c) const noexcept;               // C++20
+      constexpr bool ends_with(const charT* s) const;                 // C++20
+
+      constexpr bool contains(basic_string_view s) const noexcept; // C++23
+      constexpr bool contains(charT c) const noexcept;             // C++23
+      constexpr bool contains(const charT* s) const;               // C++23
+
+     private:
+      const_pointer data_;  // exposition only
+      size_type     size_;  // exposition only
+    };
+
+  // basic_string_view deduction guides
+  template<class It, class End>
+    basic_string_view(It, End) -> basic_string_view<iter_value_t<It>>; // C++20
+  template<class Range>
+    basic_string_view(Range&&) -> basic_string_view<ranges::range_value_t<Range>>; // C++23
+
+  // 7.11, Hash support
+  template <class T> struct hash;
+  template <> struct hash<string_view>;
+  template <> struct hash<u8string_view>; // C++20
+  template <> struct hash<u16string_view>;
+  template <> struct hash<u32string_view>;
+  template <> struct hash<wstring_view>;
+
+  constexpr basic_string_view<char>     operator""sv(const char *str,     size_t len) noexcept;
+  constexpr basic_string_view<wchar_t>  operator""sv(const wchar_t *str,  size_t len) noexcept;
+  constexpr basic_string_view<char8_t>  operator""sv(const char8_t *str,  size_t len) noexcept; // C++20
+  constexpr basic_string_view<char16_t> operator""sv(const char16_t *str, size_t len) noexcept;
+  constexpr basic_string_view<char32_t> operator""sv(const char32_t *str, size_t len) noexcept;
+
+}  // namespace std
+
+*/
+
+// clang-format on
+
+#include <__algorithm/min.h>
+#include <__assert>
+#include <__config>
+#include <__functional/hash.h>
+#include <__functional/unary_function.h>
+#include <__fwd/ostream.h>
+#include <__fwd/string_view.h>
+#include <__iterator/bounded_iter.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/reverse_iterator.h>
+#include <__iterator/wrap_iter.h>
+#include <__memory/pointer_traits.h>
+#include <__ranges/concepts.h>
+#include <__ranges/data.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/enable_view.h>
+#include <__ranges/size.h>
+#include <__string/char_traits.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_standard_layout.h>
+#include <__type_traits/is_trivial.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/remove_reference.h>
+#include <__type_traits/type_identity.h>
+#include <cstddef>
+#include <iosfwd>
+#include <limits>
+#include <stdexcept>
+#include <version>
+
+// standard-mandated includes
+
+// [iterator.range]
+#include <__iterator/access.h>
+#include <__iterator/data.h>
+#include <__iterator/empty.h>
+#include <__iterator/reverse_access.h>
+#include <__iterator/size.h>
+
+// [string.view.synop]
+#include <compare>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// TODO: This is a workaround for some vendors to carry a downstream diff to accept `nullptr` in
+//       string_view constructors. This can be refactored when this exact form isn't needed anymore.
+template <class _Traits>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR inline size_t
+__char_traits_length_checked(const typename _Traits::char_type* __s) _NOEXCEPT {
+  // This needs to be a single statement for C++11 constexpr
+  return _LIBCPP_ASSERT_NON_NULL(
+             __s != nullptr, "null pointer passed to non-null argument of char_traits<...>::length"),
+         _Traits::length(__s);
+}
+
+template <class _CharT, class _Traits>
+class basic_string_view {
+public:
+  // types
+  using traits_type     = _Traits;
+  using value_type      = _CharT;
+  using pointer         = _CharT*;
+  using const_pointer   = const _CharT*;
+  using reference       = _CharT&;
+  using const_reference = const _CharT&;
+#if defined(_LIBCPP_ABI_BOUNDED_ITERATORS)
+  using const_iterator = __bounded_iter<const_pointer>;
+#elif defined(_LIBCPP_ABI_USE_WRAP_ITER_IN_STD_STRING_VIEW)
+  using const_iterator = __wrap_iter<const_pointer>;
+#else
+  using const_iterator = const_pointer;
+#endif
+  using iterator                                = const_iterator;
+  using const_reverse_iterator                  = std::reverse_iterator<const_iterator>;
+  using reverse_iterator                        = const_reverse_iterator;
+  using size_type                               = size_t;
+  using difference_type                         = ptrdiff_t;
+  static _LIBCPP_CONSTEXPR const size_type npos = -1; // size_type(-1);
+
+  static_assert(!is_array<value_type>::value, "Character type of basic_string_view must not be an array");
+  static_assert(is_standard_layout<value_type>::value, "Character type of basic_string_view must be standard-layout");
+  static_assert(is_trivial<value_type>::value, "Character type of basic_string_view must be trivial");
+  static_assert(is_same<_CharT, typename traits_type::char_type>::value,
+                "traits_type::char_type must be the same type as CharT");
+
+  // [string.view.cons], construct/copy
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI basic_string_view() _NOEXCEPT : __data_(nullptr), __size_(0) {}
+
+  _LIBCPP_HIDE_FROM_ABI basic_string_view(const basic_string_view&) _NOEXCEPT = default;
+
+  _LIBCPP_HIDE_FROM_ABI basic_string_view& operator=(const basic_string_view&) _NOEXCEPT = default;
+
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI basic_string_view(const _CharT* __s, size_type __len) _NOEXCEPT
+      : __data_(__s),
+        __size_(__len) {
+#if _LIBCPP_STD_VER >= 14
+    // Allocations must fit in `ptrdiff_t` for pointer arithmetic to work. If `__len` exceeds it, the input
+    // range could not have been valid. Most likely the caller underflowed some arithmetic and inadvertently
+    // passed in a negative length.
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(
+        __len <= static_cast<size_type>(numeric_limits<difference_type>::max()),
+        "string_view::string_view(_CharT *, size_t): length does not fit in difference_type");
+    _LIBCPP_ASSERT_NON_NULL(
+        __len == 0 || __s != nullptr, "string_view::string_view(_CharT *, size_t): received nullptr");
+#endif
+  }
+
+#if _LIBCPP_STD_VER >= 20
+  template <contiguous_iterator _It, sized_sentinel_for<_It> _End>
+    requires(is_same_v<iter_value_t<_It>, _CharT> && !is_convertible_v<_End, size_type>)
+  constexpr _LIBCPP_HIDE_FROM_ABI basic_string_view(_It __begin, _End __end)
+      : __data_(std::to_address(__begin)), __size_(__end - __begin) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(
+        (__end - __begin) >= 0, "std::string_view::string_view(iterator, sentinel) received invalid range");
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 23
+  template <class _Range>
+    requires(!is_same_v<remove_cvref_t<_Range>, basic_string_view> && ranges::contiguous_range<_Range> &&
+             ranges::sized_range<_Range> && is_same_v<ranges::range_value_t<_Range>, _CharT> &&
+             !is_convertible_v<_Range, const _CharT*> &&
+             (!requires(remove_cvref_t<_Range>& __d) { __d.operator std::basic_string_view<_CharT, _Traits>(); }))
+  constexpr explicit _LIBCPP_HIDE_FROM_ABI basic_string_view(_Range&& __r)
+      : __data_(ranges::data(__r)), __size_(ranges::size(__r)) {}
+#endif // _LIBCPP_STD_VER >= 23
+
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI basic_string_view(const _CharT* __s)
+      : __data_(__s), __size_(std::__char_traits_length_checked<_Traits>(__s)) {}
+
+#if _LIBCPP_STD_VER >= 23
+  basic_string_view(nullptr_t) = delete;
+#endif
+
+  // [string.view.iterators], iterators
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return cbegin(); }
+
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return cend(); }
+
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT {
+#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+    return std::__make_bounded_iter(data(), data(), data() + size());
+#else
+    return const_iterator(__data_);
+#endif
+  }
+
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT {
+#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+    return std::__make_bounded_iter(data() + size(), data(), data() + size());
+#else
+    return const_iterator(__data_ + __size_);
+#endif
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT {
+    return const_reverse_iterator(cend());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
+    return const_reverse_iterator(cbegin());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT {
+    return const_reverse_iterator(cend());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT {
+    return const_reverse_iterator(cbegin());
+  }
+
+  // [string.view.capacity], capacity
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __size_; }
+
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI size_type length() const _NOEXCEPT { return __size_; }
+
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
+    return numeric_limits<size_type>::max() / sizeof(value_type);
+  }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool empty() const _NOEXCEPT { return __size_ == 0; }
+
+  // [string.view.access], element access
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_reference operator[](size_type __pos) const _NOEXCEPT {
+    return _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__pos < size(), "string_view[] index out of bounds"), __data_[__pos];
+  }
+
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_reference at(size_type __pos) const {
+    return __pos >= size() ? (__throw_out_of_range("string_view::at"), __data_[0]) : __data_[__pos];
+  }
+
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_reference front() const _NOEXCEPT {
+    return _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "string_view::front(): string is empty"), __data_[0];
+  }
+
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT {
+    return _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "string_view::back(): string is empty"), __data_[__size_ - 1];
+  }
+
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_pointer data() const _NOEXCEPT { return __data_; }
+
+  // [string.view.modifiers], modifiers:
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI void remove_prefix(size_type __n) _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n <= size(), "remove_prefix() can't remove more than size()");
+    __data_ += __n;
+    __size_ -= __n;
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI void remove_suffix(size_type __n) _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n <= size(), "remove_suffix() can't remove more than size()");
+    __size_ -= __n;
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI void swap(basic_string_view& __other) _NOEXCEPT {
+    const value_type* __p = __data_;
+    __data_               = __other.__data_;
+    __other.__data_       = __p;
+
+    size_type __sz  = __size_;
+    __size_         = __other.__size_;
+    __other.__size_ = __sz;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
+  copy(_CharT* __s, size_type __n, size_type __pos = 0) const {
+    if (__pos > size())
+      __throw_out_of_range("string_view::copy");
+    size_type __rlen = std::min(__n, size() - __pos);
+    _Traits::copy(__s, data() + __pos, __rlen);
+    return __rlen;
+  }
+
+  _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI basic_string_view substr(size_type __pos = 0, size_type __n = npos) const {
+    return __pos > size() ? (__throw_out_of_range("string_view::substr"), basic_string_view())
+                          : basic_string_view(data() + __pos, std::min(__n, size() - __pos));
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 int compare(basic_string_view __sv) const _NOEXCEPT {
+    size_type __rlen = std::min(size(), __sv.size());
+    int __retval     = _Traits::compare(data(), __sv.data(), __rlen);
+    if (__retval == 0) // first __rlen chars matched
+      __retval = size() == __sv.size() ? 0 : (size() < __sv.size() ? -1 : 1);
+    return __retval;
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI int
+  compare(size_type __pos1, size_type __n1, basic_string_view __sv) const {
+    return substr(__pos1, __n1).compare(__sv);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI int
+  compare(size_type __pos1, size_type __n1, basic_string_view __sv, size_type __pos2, size_type __n2) const {
+    return substr(__pos1, __n1).compare(__sv.substr(__pos2, __n2));
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI int compare(const _CharT* __s) const _NOEXCEPT {
+    return compare(basic_string_view(__s));
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI int
+  compare(size_type __pos1, size_type __n1, const _CharT* __s) const {
+    return substr(__pos1, __n1).compare(basic_string_view(__s));
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI int
+  compare(size_type __pos1, size_type __n1, const _CharT* __s, size_type __n2) const {
+    return substr(__pos1, __n1).compare(basic_string_view(__s, __n2));
+  }
+
+  // find
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find(basic_string_view __s, size_type __pos = 0) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s.size() == 0 || __s.data() != nullptr, "string_view::find(): received nullptr");
+    return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __s.data(), __pos, __s.size());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type find(_CharT __c, size_type __pos = 0) const _NOEXCEPT {
+    return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string_view::find(): received nullptr");
+    return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find(const _CharT* __s, size_type __pos = 0) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string_view::find(): received nullptr");
+    return std::__str_find<value_type, size_type, traits_type, npos>(
+        data(), size(), __s, __pos, traits_type::length(__s));
+  }
+
+  // rfind
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  rfind(basic_string_view __s, size_type __pos = npos) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s.size() == 0 || __s.data() != nullptr, "string_view::find(): received nullptr");
+    return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __s.data(), __pos, __s.size());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  rfind(_CharT __c, size_type __pos = npos) const _NOEXCEPT {
+    return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  rfind(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string_view::rfind(): received nullptr");
+    return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  rfind(const _CharT* __s, size_type __pos = npos) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string_view::rfind(): received nullptr");
+    return std::__str_rfind<value_type, size_type, traits_type, npos>(
+        data(), size(), __s, __pos, traits_type::length(__s));
+  }
+
+  // find_first_of
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_first_of(basic_string_view __s, size_type __pos = 0) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s.size() == 0 || __s.data() != nullptr, "string_view::find_first_of(): received nullptr");
+    return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __s.data(), __pos, __s.size());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_first_of(_CharT __c, size_type __pos = 0) const _NOEXCEPT {
+    return find(__c, __pos);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_first_of(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string_view::find_first_of(): received nullptr");
+    return std::__str_find_first_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_first_of(const _CharT* __s, size_type __pos = 0) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string_view::find_first_of(): received nullptr");
+    return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __s, __pos, traits_type::length(__s));
+  }
+
+  // find_last_of
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_last_of(basic_string_view __s, size_type __pos = npos) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s.size() == 0 || __s.data() != nullptr, "string_view::find_last_of(): received nullptr");
+    return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __s.data(), __pos, __s.size());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_last_of(_CharT __c, size_type __pos = npos) const _NOEXCEPT {
+    return rfind(__c, __pos);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_last_of(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string_view::find_last_of(): received nullptr");
+    return std::__str_find_last_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_last_of(const _CharT* __s, size_type __pos = npos) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string_view::find_last_of(): received nullptr");
+    return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __s, __pos, traits_type::length(__s));
+  }
+
+  // find_first_not_of
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_first_not_of(basic_string_view __s, size_type __pos = 0) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(
+        __s.size() == 0 || __s.data() != nullptr, "string_view::find_first_not_of(): received nullptr");
+    return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __s.data(), __pos, __s.size());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_first_not_of(_CharT __c, size_type __pos = 0) const _NOEXCEPT {
+    return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_first_not_of(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string_view::find_first_not_of(): received nullptr");
+    return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_first_not_of(const _CharT* __s, size_type __pos = 0) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string_view::find_first_not_of(): received nullptr");
+    return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __s, __pos, traits_type::length(__s));
+  }
+
+  // find_last_not_of
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_last_not_of(basic_string_view __s, size_type __pos = npos) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(
+        __s.size() == 0 || __s.data() != nullptr, "string_view::find_last_not_of(): received nullptr");
+    return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __s.data(), __pos, __s.size());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_last_not_of(_CharT __c, size_type __pos = npos) const _NOEXCEPT {
+    return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_last_not_of(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string_view::find_last_not_of(): received nullptr");
+    return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
+  find_last_not_of(const _CharT* __s, size_type __pos = npos) const _NOEXCEPT {
+    _LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string_view::find_last_not_of(): received nullptr");
+    return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
+        data(), size(), __s, __pos, traits_type::length(__s));
+  }
+
+#if _LIBCPP_STD_VER >= 20
+  constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(basic_string_view __s) const noexcept {
+    return size() >= __s.size() && compare(0, __s.size(), __s) == 0;
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(value_type __c) const noexcept {
+    return !empty() && _Traits::eq(front(), __c);
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(const value_type* __s) const noexcept {
+    return starts_with(basic_string_view(__s));
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(basic_string_view __s) const noexcept {
+    return size() >= __s.size() && compare(size() - __s.size(), npos, __s) == 0;
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(value_type __c) const noexcept {
+    return !empty() && _Traits::eq(back(), __c);
+  }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(const value_type* __s) const noexcept {
+    return ends_with(basic_string_view(__s));
+  }
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+  constexpr _LIBCPP_HIDE_FROM_ABI bool contains(basic_string_view __sv) const noexcept { return find(__sv) != npos; }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI bool contains(value_type __c) const noexcept { return find(__c) != npos; }
+
+  constexpr _LIBCPP_HIDE_FROM_ABI bool contains(const value_type* __s) const { return find(__s) != npos; }
+#endif
+
+private:
+  const value_type* __data_;
+  size_type __size_;
+};
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(basic_string_view);
+
+#if _LIBCPP_STD_VER >= 20
+template <class _CharT, class _Traits>
+inline constexpr bool ranges::enable_view<basic_string_view<_CharT, _Traits>> = true;
+
+template <class _CharT, class _Traits>
+inline constexpr bool ranges::enable_borrowed_range<basic_string_view<_CharT, _Traits> > = true;
+#endif // _LIBCPP_STD_VER >= 20
+
+// [string.view.deduct]
+
+#if _LIBCPP_STD_VER >= 20
+template <contiguous_iterator _It, sized_sentinel_for<_It> _End>
+basic_string_view(_It, _End) -> basic_string_view<iter_value_t<_It>>;
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 23
+template <ranges::contiguous_range _Range>
+basic_string_view(_Range) -> basic_string_view<ranges::range_value_t<_Range>>;
+#endif
+
+// [string.view.comparison]
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator==(basic_string_view<_CharT, _Traits> __lhs,
+                                                type_identity_t<basic_string_view<_CharT, _Traits>> __rhs) noexcept {
+  if (__lhs.size() != __rhs.size())
+    return false;
+  return __lhs.compare(__rhs) == 0;
+}
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI constexpr auto operator<=>(basic_string_view<_CharT, _Traits> __lhs,
+                                                 type_identity_t<basic_string_view<_CharT, _Traits>> __rhs) noexcept {
+  if constexpr (requires { typename _Traits::comparison_category; }) {
+    // [string.view]/4
+    static_assert(
+        __comparison_category<typename _Traits::comparison_category>, "return type is not a comparison category type");
+    return static_cast<typename _Traits::comparison_category>(__lhs.compare(__rhs) <=> 0);
+  } else {
+    return static_cast<weak_ordering>(__lhs.compare(__rhs) <=> 0);
+  }
+}
+
+#else
+
+// operator ==
+
+template <class _CharT, class _Traits>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator==(basic_string_view<_CharT, _Traits> __lhs, basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT {
+  if (__lhs.size() != __rhs.size())
+    return false;
+  return __lhs.compare(__rhs) == 0;
+}
+
+// The dummy default template parameters are used to work around a MSVC issue with mangling, see VSO-409326 for details.
+// This applies to the other sufficient overloads below for the other comparison operators.
+template <class _CharT, class _Traits, int = 1>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator==(basic_string_view<_CharT, _Traits> __lhs,
+           __type_identity_t<basic_string_view<_CharT, _Traits> > __rhs) _NOEXCEPT {
+  if (__lhs.size() != __rhs.size())
+    return false;
+  return __lhs.compare(__rhs) == 0;
+}
+
+template <class _CharT, class _Traits, int = 2>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator==(__type_identity_t<basic_string_view<_CharT, _Traits> > __lhs,
+           basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT {
+  if (__lhs.size() != __rhs.size())
+    return false;
+  return __lhs.compare(__rhs) == 0;
+}
+
+// operator !=
+template <class _CharT, class _Traits>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator!=(basic_string_view<_CharT, _Traits> __lhs, basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT {
+  if (__lhs.size() != __rhs.size())
+    return true;
+  return __lhs.compare(__rhs) != 0;
+}
+
+template <class _CharT, class _Traits, int = 1>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator!=(basic_string_view<_CharT, _Traits> __lhs,
+           __type_identity_t<basic_string_view<_CharT, _Traits> > __rhs) _NOEXCEPT {
+  if (__lhs.size() != __rhs.size())
+    return true;
+  return __lhs.compare(__rhs) != 0;
+}
+
+template <class _CharT, class _Traits, int = 2>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator!=(__type_identity_t<basic_string_view<_CharT, _Traits> > __lhs,
+           basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT {
+  if (__lhs.size() != __rhs.size())
+    return true;
+  return __lhs.compare(__rhs) != 0;
+}
+
+// operator <
+template <class _CharT, class _Traits>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator<(basic_string_view<_CharT, _Traits> __lhs, basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT {
+  return __lhs.compare(__rhs) < 0;
+}
+
+template <class _CharT, class _Traits, int = 1>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator<(basic_string_view<_CharT, _Traits> __lhs,
+          __type_identity_t<basic_string_view<_CharT, _Traits> > __rhs) _NOEXCEPT {
+  return __lhs.compare(__rhs) < 0;
+}
+
+template <class _CharT, class _Traits, int = 2>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator<(__type_identity_t<basic_string_view<_CharT, _Traits> > __lhs,
+          basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT {
+  return __lhs.compare(__rhs) < 0;
+}
+
+// operator >
+template <class _CharT, class _Traits>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator>(basic_string_view<_CharT, _Traits> __lhs, basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT {
+  return __lhs.compare(__rhs) > 0;
+}
+
+template <class _CharT, class _Traits, int = 1>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator>(basic_string_view<_CharT, _Traits> __lhs,
+          __type_identity_t<basic_string_view<_CharT, _Traits> > __rhs) _NOEXCEPT {
+  return __lhs.compare(__rhs) > 0;
+}
+
+template <class _CharT, class _Traits, int = 2>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator>(__type_identity_t<basic_string_view<_CharT, _Traits> > __lhs,
+          basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT {
+  return __lhs.compare(__rhs) > 0;
+}
+
+// operator <=
+template <class _CharT, class _Traits>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator<=(basic_string_view<_CharT, _Traits> __lhs, basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT {
+  return __lhs.compare(__rhs) <= 0;
+}
+
+template <class _CharT, class _Traits, int = 1>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator<=(basic_string_view<_CharT, _Traits> __lhs,
+           __type_identity_t<basic_string_view<_CharT, _Traits> > __rhs) _NOEXCEPT {
+  return __lhs.compare(__rhs) <= 0;
+}
+
+template <class _CharT, class _Traits, int = 2>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator<=(__type_identity_t<basic_string_view<_CharT, _Traits> > __lhs,
+           basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT {
+  return __lhs.compare(__rhs) <= 0;
+}
+
+// operator >=
+template <class _CharT, class _Traits>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator>=(basic_string_view<_CharT, _Traits> __lhs, basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT {
+  return __lhs.compare(__rhs) >= 0;
+}
+
+template <class _CharT, class _Traits, int = 1>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator>=(basic_string_view<_CharT, _Traits> __lhs,
+           __type_identity_t<basic_string_view<_CharT, _Traits> > __rhs) _NOEXCEPT {
+  return __lhs.compare(__rhs) >= 0;
+}
+
+template <class _CharT, class _Traits, int = 2>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool
+operator>=(__type_identity_t<basic_string_view<_CharT, _Traits> > __lhs,
+           basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT {
+  return __lhs.compare(__rhs) >= 0;
+}
+
+#endif //  _LIBCPP_STD_VER >= 20
+
+template <class _CharT, class _Traits>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, basic_string_view<_CharT, _Traits> __str);
+
+// [string.view.hash]
+template <class _CharT>
+struct __string_view_hash : public __unary_function<basic_string_view<_CharT, char_traits<_CharT> >, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const basic_string_view<_CharT, char_traits<_CharT> > __val) const _NOEXCEPT {
+    return std::__do_string_hash(__val.data(), __val.data() + __val.size());
+  }
+};
+
+template <>
+struct hash<basic_string_view<char, char_traits<char> > > : __string_view_hash<char> {};
+
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+template <>
+struct hash<basic_string_view<char8_t, char_traits<char8_t> > > : __string_view_hash<char8_t> {};
+#endif
+
+template <>
+struct hash<basic_string_view<char16_t, char_traits<char16_t> > > : __string_view_hash<char16_t> {};
+
+template <>
+struct hash<basic_string_view<char32_t, char_traits<char32_t> > > : __string_view_hash<char32_t> {};
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+struct hash<basic_string_view<wchar_t, char_traits<wchar_t> > > : __string_view_hash<wchar_t> {};
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+inline namespace literals {
+inline namespace string_view_literals {
+inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char> operator""sv(const char* __str, size_t __len) noexcept {
+  return basic_string_view<char>(__str, __len);
+}
+
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<wchar_t>
+operator""sv(const wchar_t* __str, size_t __len) noexcept {
+  return basic_string_view<wchar_t>(__str, __len);
+}
+#  endif
+
+#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char8_t>
+operator""sv(const char8_t* __str, size_t __len) noexcept {
+  return basic_string_view<char8_t>(__str, __len);
+}
+#  endif
+
+inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char16_t>
+operator""sv(const char16_t* __str, size_t __len) noexcept {
+  return basic_string_view<char16_t>(__str, __len);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char32_t>
+operator""sv(const char32_t* __str, size_t __len) noexcept {
+  return basic_string_view<char32_t>(__str, __len);
+}
+} // namespace string_view_literals
+} // namespace literals
+#endif
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <algorithm>
+#  include <concepts>
+#  include <cstdlib>
+#  include <iterator>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_STRING_VIEW
diff --git a/libcxx/include/__cxx03/strstream b/libcxx/include/__cxx03/strstream
new file mode 100644
index 000000000000..9ff4024a7c7e
--- /dev/null
+++ b/libcxx/include/__cxx03/strstream
@@ -0,0 +1,351 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_STRSTREAM
+#define _LIBCPP_STRSTREAM
+
+/*
+    strstream synopsis
+
+class strstreambuf                                    // Removed in C++26
+    : public basic_streambuf<char>
+{
+public:
+    explicit strstreambuf(streamsize alsize_arg = 0); // before C++20
+    strstreambuf() : strstreambuf(0) {}               // C++20
+    explicit strstreambuf(streamsize alsize_arg);     // C++20
+
+    strstreambuf(void* (*palloc_arg)(size_t), void (*pfree_arg)(void*));
+    strstreambuf(char* gnext_arg, streamsize n, char* pbeg_arg = nullptr);
+    strstreambuf(const char* gnext_arg, streamsize n);
+
+    strstreambuf(signed char* gnext_arg, streamsize n, signed char* pbeg_arg = nullptr);
+    strstreambuf(const signed char* gnext_arg, streamsize n);
+    strstreambuf(unsigned char* gnext_arg, streamsize n, unsigned char* pbeg_arg = nullptr);
+    strstreambuf(const unsigned char* gnext_arg, streamsize n);
+
+    strstreambuf(strstreambuf&& rhs);
+    strstreambuf& operator=(strstreambuf&& rhs);
+
+    virtual ~strstreambuf();
+
+    void swap(strstreambuf& rhs);
+
+    void freeze(bool freezefl = true);
+    char* str();
+    int pcount() const;
+
+protected:
+    virtual int_type overflow (int_type c = EOF);
+    virtual int_type pbackfail(int_type c = EOF);
+    virtual int_type underflow();
+    virtual pos_type seekoff(off_type off, ios_base::seekdir way,
+                             ios_base::openmode which = ios_base::in | ios_base::out);
+    virtual pos_type seekpos(pos_type sp,
+                             ios_base::openmode which = ios_base::in | ios_base::out);
+    virtual streambuf* setbuf(char* s, streamsize n);
+
+private:
+    typedef T1 strstate;                // exposition only
+    static const strstate allocated;    // exposition only
+    static const strstate constant;     // exposition only
+    static const strstate dynamic;      // exposition only
+    static const strstate frozen;       // exposition only
+    strstate strmode;                   // exposition only
+    streamsize alsize;                  // exposition only
+    void* (*palloc)(size_t);            // exposition only
+    void (*pfree)(void*);               // exposition only
+};
+
+class istrstream                                      // Removed in C++26
+    : public basic_istream<char>
+{
+public:
+    explicit istrstream(const char* s);
+    explicit istrstream(char* s);
+    istrstream(const char* s, streamsize n);
+    istrstream(char* s, streamsize n);
+
+    virtual ~istrstream();
+
+    strstreambuf* rdbuf() const;
+    char *str();
+
+private:
+    strstreambuf sb; // exposition only
+};
+
+class ostrstream                                      // Removed in C++26
+    : public basic_ostream<char>
+{
+public:
+    ostrstream();
+    ostrstream(char* s, int n, ios_base::openmode mode = ios_base::out);
+
+    virtual ~ostrstream();
+
+    strstreambuf* rdbuf() const;
+    void freeze(bool freezefl = true);
+    char* str();
+    int pcount() const;
+
+private:
+    strstreambuf sb; // exposition only
+};
+
+class strstream                                       // Removed in C++26
+    : public basic_iostream<char>
+{
+public:
+    // Types
+    typedef char                        char_type;
+    typedef char_traits<char>::int_type int_type;
+    typedef char_traits<char>::pos_type pos_type;
+    typedef char_traits<char>::off_type off_type;
+
+    // constructors/destructor
+    strstream();
+    strstream(char* s, int n, ios_base::openmode mode = ios_base::in | ios_base::out);
+
+    virtual ~strstream();
+
+    // Members:
+    strstreambuf* rdbuf() const;
+    void freeze(bool freezefl = true);
+    int pcount() const;
+    char* str();
+
+private:
+    strstreambuf sb; // exposition only
+};
+
+}  // std
+
+*/
+
+#include <__config>
+#include <istream>
+#include <ostream>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_STRSTREAM) || defined(_LIBCPP_BUILDING_LIBRARY)
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+class _LIBCPP_DEPRECATED _LIBCPP_EXPORTED_FROM_ABI strstreambuf : public streambuf {
+public:
+#  ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI strstreambuf() : strstreambuf(0) {}
+  explicit strstreambuf(streamsize __alsize);
+#  else
+  explicit strstreambuf(streamsize __alsize = 0);
+#  endif
+  strstreambuf(void* (*__palloc)(size_t), void (*__pfree)(void*));
+  strstreambuf(char* __gnext, streamsize __n, char* __pbeg = nullptr);
+  strstreambuf(const char* __gnext, streamsize __n);
+
+  strstreambuf(signed char* __gnext, streamsize __n, signed char* __pbeg = nullptr);
+  strstreambuf(const signed char* __gnext, streamsize __n);
+  strstreambuf(unsigned char* __gnext, streamsize __n, unsigned char* __pbeg = nullptr);
+  strstreambuf(const unsigned char* __gnext, streamsize __n);
+
+#  ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI strstreambuf(strstreambuf&& __rhs);
+  _LIBCPP_HIDE_FROM_ABI strstreambuf& operator=(strstreambuf&& __rhs);
+#  endif // _LIBCPP_CXX03_LANG
+
+  ~strstreambuf() override;
+
+  void swap(strstreambuf& __rhs);
+
+  void freeze(bool __freezefl = true);
+  char* str();
+  int pcount() const;
+
+protected:
+  int_type overflow(int_type __c = EOF) override;
+  int_type pbackfail(int_type __c = EOF) override;
+  int_type underflow() override;
+  pos_type
+  seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __which = ios_base::in | ios_base::out) override;
+  pos_type seekpos(pos_type __sp, ios_base::openmode __which = ios_base::in | ios_base::out) override;
+
+private:
+  typedef unsigned __mode_type;
+  static const __mode_type __allocated     = 0x01;
+  static const __mode_type __constant      = 0x02;
+  static const __mode_type __dynamic       = 0x04;
+  static const __mode_type __frozen        = 0x08;
+  static const streamsize __default_alsize = 4096;
+
+  __mode_type __strmode_;
+  streamsize __alsize_;
+  void* (*__palloc_)(size_t);
+  void (*__pfree_)(void*);
+
+  void __init(char* __gnext, streamsize __n, char* __pbeg);
+};
+
+#  ifndef _LIBCPP_CXX03_LANG
+
+inline _LIBCPP_HIDE_FROM_ABI strstreambuf::strstreambuf(strstreambuf&& __rhs)
+    : streambuf(__rhs),
+      __strmode_(__rhs.__strmode_),
+      __alsize_(__rhs.__alsize_),
+      __palloc_(__rhs.__palloc_),
+      __pfree_(__rhs.__pfree_) {
+  __rhs.setg(nullptr, nullptr, nullptr);
+  __rhs.setp(nullptr, nullptr);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI strstreambuf& strstreambuf::operator=(strstreambuf&& __rhs) {
+  if (eback() && (__strmode_ & __allocated) != 0 && (__strmode_ & __frozen) == 0) {
+    if (__pfree_)
+      __pfree_(eback());
+    else
+      delete[] eback();
+  }
+  streambuf::operator=(__rhs);
+  __strmode_ = __rhs.__strmode_;
+  __alsize_  = __rhs.__alsize_;
+  __palloc_  = __rhs.__palloc_;
+  __pfree_   = __rhs.__pfree_;
+  __rhs.setg(nullptr, nullptr, nullptr);
+  __rhs.setp(nullptr, nullptr);
+  return *this;
+}
+
+#  endif // _LIBCPP_CXX03_LANG
+
+class _LIBCPP_DEPRECATED _LIBCPP_EXPORTED_FROM_ABI istrstream : public istream {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit istrstream(const char* __s) : istream(&__sb_), __sb_(__s, 0) {}
+  _LIBCPP_HIDE_FROM_ABI explicit istrstream(char* __s) : istream(&__sb_), __sb_(__s, 0) {}
+  _LIBCPP_HIDE_FROM_ABI istrstream(const char* __s, streamsize __n) : istream(&__sb_), __sb_(__s, __n) {}
+  _LIBCPP_HIDE_FROM_ABI istrstream(char* __s, streamsize __n) : istream(&__sb_), __sb_(__s, __n) {}
+
+#  ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI istrstream(istrstream&& __rhs) // extension
+      : istream(std::move(static_cast<istream&>(__rhs))), __sb_(std::move(__rhs.__sb_)) {
+    istream::set_rdbuf(&__sb_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI istrstream& operator=(istrstream&& __rhs) {
+    __sb_ = std::move(__rhs.__sb_);
+    istream::operator=(std::move(__rhs));
+    return *this;
+  }
+#  endif // _LIBCPP_CXX03_LANG
+
+  ~istrstream() override;
+
+  _LIBCPP_HIDE_FROM_ABI void swap(istrstream& __rhs) {
+    istream::swap(__rhs);
+    __sb_.swap(__rhs.__sb_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI strstreambuf* rdbuf() const { return const_cast<strstreambuf*>(&__sb_); }
+  _LIBCPP_HIDE_FROM_ABI char* str() { return __sb_.str(); }
+
+private:
+  strstreambuf __sb_;
+};
+
+class _LIBCPP_DEPRECATED _LIBCPP_EXPORTED_FROM_ABI ostrstream : public ostream {
+public:
+  _LIBCPP_HIDE_FROM_ABI ostrstream() : ostream(&__sb_) {}
+  _LIBCPP_HIDE_FROM_ABI ostrstream(char* __s, int __n, ios_base::openmode __mode = ios_base::out)
+      : ostream(&__sb_), __sb_(__s, __n, __s + (__mode & ios::app ? std::strlen(__s) : 0)) {}
+
+#  ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI ostrstream(ostrstream&& __rhs) // extension
+      : ostream(std::move(static_cast<ostream&>(__rhs))), __sb_(std::move(__rhs.__sb_)) {
+    ostream::set_rdbuf(&__sb_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ostrstream& operator=(ostrstream&& __rhs) {
+    __sb_ = std::move(__rhs.__sb_);
+    ostream::operator=(std::move(__rhs));
+    return *this;
+  }
+#  endif // _LIBCPP_CXX03_LANG
+
+  ~ostrstream() override;
+
+  _LIBCPP_HIDE_FROM_ABI void swap(ostrstream& __rhs) {
+    ostream::swap(__rhs);
+    __sb_.swap(__rhs.__sb_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI strstreambuf* rdbuf() const { return const_cast<strstreambuf*>(&__sb_); }
+  _LIBCPP_HIDE_FROM_ABI void freeze(bool __freezefl = true) { __sb_.freeze(__freezefl); }
+  _LIBCPP_HIDE_FROM_ABI char* str() { return __sb_.str(); }
+  _LIBCPP_HIDE_FROM_ABI int pcount() const { return __sb_.pcount(); }
+
+private:
+  strstreambuf __sb_; // exposition only
+};
+
+class _LIBCPP_DEPRECATED _LIBCPP_EXPORTED_FROM_ABI strstream : public iostream {
+public:
+  // Types
+  typedef char char_type;
+  typedef char_traits<char>::int_type int_type;
+  typedef char_traits<char>::pos_type pos_type;
+  typedef char_traits<char>::off_type off_type;
+
+  // constructors/destructor
+  _LIBCPP_HIDE_FROM_ABI strstream() : iostream(&__sb_) {}
+  _LIBCPP_HIDE_FROM_ABI strstream(char* __s, int __n, ios_base::openmode __mode = ios_base::in | ios_base::out)
+      : iostream(&__sb_), __sb_(__s, __n, __s + (__mode & ios::app ? std::strlen(__s) : 0)) {}
+
+#  ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI strstream(strstream&& __rhs) // extension
+      : iostream(std::move(static_cast<iostream&>(__rhs))), __sb_(std::move(__rhs.__sb_)) {
+    iostream::set_rdbuf(&__sb_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI strstream& operator=(strstream&& __rhs) {
+    __sb_ = std::move(__rhs.__sb_);
+    iostream::operator=(std::move(__rhs));
+    return *this;
+  }
+#  endif // _LIBCPP_CXX03_LANG
+
+  ~strstream() override;
+
+  _LIBCPP_HIDE_FROM_ABI void swap(strstream& __rhs) {
+    iostream::swap(__rhs);
+    __sb_.swap(__rhs.__sb_);
+  }
+
+  // Members:
+  _LIBCPP_HIDE_FROM_ABI strstreambuf* rdbuf() const { return const_cast<strstreambuf*>(&__sb_); }
+  _LIBCPP_HIDE_FROM_ABI void freeze(bool __freezefl = true) { __sb_.freeze(__freezefl); }
+  _LIBCPP_HIDE_FROM_ABI int pcount() const { return __sb_.pcount(); }
+  _LIBCPP_HIDE_FROM_ABI char* str() { return __sb_.str(); }
+
+private:
+  strstreambuf __sb_; // exposition only
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_STRSTREAM) || defined(_LIBCPP_BUILDING_LIBRARY)
+
+#endif // _LIBCPP_STRSTREAM
diff --git a/libcxx/include/__cxx03/syncstream b/libcxx/include/__cxx03/syncstream
new file mode 100644
index 000000000000..e6f35b6f428e
--- /dev/null
+++ b/libcxx/include/__cxx03/syncstream
@@ -0,0 +1,512 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_SYNCSTREAM
+#define _LIBCPP_SYNCSTREAM
+
+/*
+    syncstream synopsis
+
+#include <ostream>  // see [ostream.syn]
+
+namespace std {
+    template<class charT, class traits, class Allocator>
+    class basic_syncbuf;
+
+    // [syncstream.syncbuf.special], specialized algorithms
+    template<class charT, class traits, class Allocator>
+      void swap(basic_syncbuf<charT, traits, Allocator>&,
+                basic_syncbuf<charT, traits, Allocator>&);
+
+    using syncbuf = basic_syncbuf<char>;
+    using wsyncbuf = basic_syncbuf<wchar_t>;
+
+    template<class charT, class traits, class Allocator>
+    class basic_osyncstream;
+
+    using osyncstream = basic_osyncstream<char>;
+    using wosyncstream = basic_osyncstream<wchar_t>;
+
+    template<class charT, class traits, class Allocator>
+    class basic_syncbuf : public basic_streambuf<charT, traits> {
+    public:
+        using char_type      = charT;
+        using int_type       = typename traits::int_type;
+        using pos_type       = typename traits::pos_type;
+        using off_type       = typename traits::off_type;
+        using traits_type    = traits;
+        using allocator_type = Allocator;
+
+        using streambuf_type = basic_streambuf<charT, traits>;
+
+        // [syncstream.syncbuf.cons], construction and destruction
+        explicit basic_syncbuf(streambuf_type* obuf = nullptr)
+          : basic_syncbuf(obuf, Allocator()) {}
+        basic_syncbuf(streambuf_type*, const Allocator&);
+        basic_syncbuf(basic_syncbuf&&);
+        ~basic_syncbuf();
+
+        // [syncstream.syncbuf.assign], assignment and swap
+        basic_syncbuf& operator=(basic_syncbuf&&);
+        void swap(basic_syncbuf&);
+
+        // [syncstream.syncbuf.members], member functions
+        bool emit();
+        streambuf_type* get_wrapped() const noexcept;
+        allocator_type get_allocator() const noexcept;
+        void set_emit_on_sync(bool) noexcept;
+
+    protected:
+        // [syncstream.syncbuf.virtuals], overridden virtual functions
+        int sync() override;
+
+    private:
+        streambuf_type* wrapped;    // exposition only
+        bool emit_on_sync{};        // exposition only
+    };
+
+    // [syncstream.syncbuf.special], specialized algorithms
+    template<class charT, class traits, class Allocator>
+    void swap(basic_syncbuf<charT, traits, Allocator>&,
+              basic_syncbuf<charT, traits, Allocator>&);
+
+    template<class charT, class traits, class Allocator>
+    class basic_osyncstream : public basic_ostream<charT, traits> {
+    public:
+        using char_type   = charT;
+        using int_type    = typename traits::int_type;
+        using pos_type    = typename traits::pos_type;
+        using off_type    = typename traits::off_type;
+        using traits_type = traits;
+
+        using allocator_type = Allocator;
+        using streambuf_type = basic_streambuf<charT, traits>;
+        using syncbuf_type   = basic_syncbuf<charT, traits, Allocator>;
+
+        // [syncstream.osyncstream.cons], construction and destruction
+        basic_osyncstream(streambuf_type*, const Allocator&);
+        explicit basic_osyncstream(streambuf_type* obuf)
+          : basic_osyncstream(obuf, Allocator()) {}
+        basic_osyncstream(basic_ostream<charT, traits>& os, const Allocator& allocator)
+          : basic_osyncstream(os.rdbuf(), allocator) {}
+        explicit basic_osyncstream(basic_ostream<charT, traits>& os)
+          : basic_osyncstream(os, Allocator()) {}
+        basic_osyncstream(basic_osyncstream&&) noexcept;
+        ~basic_osyncstream();
+
+        // [syncstream.osyncstream.assign], assignment
+        basic_osyncstream& operator=(basic_osyncstream&&);
+
+        // [syncstream.osyncstream.members], member functions
+        void emit();
+        streambuf_type* get_wrapped() const noexcept;
+        syncbuf_type* rdbuf() const noexcept { return const_cast<syncbuf_type*>(addressof(sb)); }
+
+    private:
+        syncbuf_type sb;    // exposition only
+    };
+}
+
+*/
+
+#include <__config>
+#include <__utility/move.h>
+#include <ios>
+#include <iosfwd> // required for declaration of default arguments
+#include <streambuf>
+#include <string>
+
+#ifndef _LIBCPP_HAS_NO_THREADS
+#  include <map>
+#  include <mutex>
+#  include <shared_mutex>
+#endif
+
+// standard-mandated includes
+
+// [syncstream.syn]
+#include <ostream>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM)
+
+// [syncstream.syncbuf.overview]/1
+//   Class template basic_syncbuf stores character data written to it,
+//   known as the associated output, into internal buffers allocated
+//   using the object's allocator. The associated output is transferred
+//   to the wrapped stream buffer object *wrapped when emit() is called
+//   or when the basic_syncbuf object is destroyed. Such transfers are
+//   atomic with respect to transfers by other basic_syncbuf objects
+//   with the same wrapped stream buffer object.
+//
+// This helper singleton is used to implement the required
+// synchronisation guarantees.
+#  ifndef _LIBCPP_HAS_NO_THREADS
+class __wrapped_streambuf_mutex {
+  _LIBCPP_HIDE_FROM_ABI __wrapped_streambuf_mutex() = default;
+
+public:
+  __wrapped_streambuf_mutex(const __wrapped_streambuf_mutex&)            = delete;
+  __wrapped_streambuf_mutex& operator=(const __wrapped_streambuf_mutex&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI void __inc_reference([[maybe_unused]] void* __ptr) {
+    _LIBCPP_ASSERT_INTERNAL(__ptr != nullptr, "non-wrapped streambufs are never written to");
+    unique_lock __lock{__mutex_};
+    ++__lut_[reinterpret_cast<uintptr_t>(__ptr)].__count;
+  }
+
+  // pre: __ptr is in __lut_
+  _LIBCPP_HIDE_FROM_ABI void __dec_reference([[maybe_unused]] void* __ptr) noexcept {
+    unique_lock __lock{__mutex_};
+
+    auto __it = __get_it(__ptr);
+    if (__it->second.__count == 1)
+      __lut_.erase(__it);
+    else
+      --__it->second.__count;
+  }
+
+  // TODO
+  // This function causes emit() aquire two mutexes:
+  // - __mutex_ shared
+  // _ __get_it(__ptr)->second.__mutex exclusive
+  //
+  // Instead store a pointer to __get_it(__ptr)->second.__mutex when
+  // calling __inc_reference.
+  //
+  // pre: __ptr is in __lut_
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI lock_guard<mutex> __get_lock([[maybe_unused]] void* __ptr) noexcept {
+    shared_lock __lock{__mutex_};
+    return lock_guard{__get_it(__ptr)->second.__mutex};
+  }
+
+  // This function is used for testing.
+  //
+  // It is allowed to call this function with a non-registered pointer.
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI size_t __get_count([[maybe_unused]] void* __ptr) noexcept {
+    _LIBCPP_ASSERT_INTERNAL(__ptr != nullptr, "non-wrapped streambufs are never written to");
+    shared_lock __lock{__mutex_};
+
+    auto __it = __lut_.find(reinterpret_cast<uintptr_t>(__ptr));
+    return __it != __lut_.end() ? __it->second.__count : 0;
+  }
+
+  [[nodiscard]] static _LIBCPP_HIDE_FROM_ABI __wrapped_streambuf_mutex& __instance() noexcept {
+    static __wrapped_streambuf_mutex __result;
+    return __result;
+  }
+
+private:
+  struct __value {
+    mutex __mutex;
+    size_t __count{0};
+  };
+
+  shared_mutex __mutex_;
+  map<uintptr_t, __value> __lut_;
+
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI map<uintptr_t, __value>::iterator __get_it(void* __ptr) noexcept {
+    _LIBCPP_ASSERT_INTERNAL(__ptr != nullptr, "non-wrapped streambufs are never written to");
+
+    auto __it = __lut_.find(reinterpret_cast<uintptr_t>(__ptr));
+    _LIBCPP_ASSERT_INTERNAL(__it != __lut_.end(), "using a wrapped streambuf that has not been registered");
+    _LIBCPP_ASSERT_INTERNAL(__it->second.__count >= 1, "found an inactive streambuf wrapper");
+    return __it;
+  }
+};
+#  endif // _LIBCPP_HAS_NO_THREADS
+
+// basic_syncbuf
+
+// The class uses a basic_string<_CharT, _Traits, _Allocator> as
+// internal buffer. Per [syncstream.syncbuf.cons]/4
+//   Remarks: A copy of allocator is used to allocate memory for
+//   internal buffers holding the associated output.
+//
+// Therefore the allocator used in the constructor is passed to the
+// basic_string. The class does not keep a copy of this allocator.
+template <class _CharT, class _Traits, class _Allocator>
+class _LIBCPP_TEMPLATE_VIS basic_syncbuf : public basic_streambuf<_CharT, _Traits> {
+public:
+  using char_type      = _CharT;
+  using traits_type    = _Traits;
+  using int_type       = typename traits_type::int_type;
+  using pos_type       = typename traits_type::pos_type;
+  using off_type       = typename traits_type::off_type;
+  using allocator_type = _Allocator;
+
+  using streambuf_type = basic_streambuf<_CharT, _Traits>;
+
+  // [syncstream.syncbuf.cons], construction and destruction
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_syncbuf(streambuf_type* __obuf = nullptr)
+      : basic_syncbuf(__obuf, _Allocator()) {}
+
+  _LIBCPP_HIDE_FROM_ABI basic_syncbuf(streambuf_type* __obuf, _Allocator const& __alloc)
+      : __wrapped_(__obuf), __str_(__alloc) {
+    __inc_reference();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI basic_syncbuf(basic_syncbuf&& __other)
+      : __wrapped_(__other.get_wrapped()), __str_(std::move(__other.__str_)), __emit_on_sync_(__other.__emit_on_sync_) {
+    __move_common(__other);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~basic_syncbuf() {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      emit();
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+    }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    __dec_reference();
+  }
+
+  // [syncstream.syncbuf.assign], assignment and swap
+
+  _LIBCPP_HIDE_FROM_ABI basic_syncbuf& operator=(basic_syncbuf&& __other) {
+    // The function is specified to call emit. This call should
+    // propagate the exception thrown.
+    emit();
+    __dec_reference();
+
+    __wrapped_      = __other.get_wrapped();
+    __str_          = std::move(__other.__str_);
+    __emit_on_sync_ = __other.__emit_on_sync_;
+
+    __move_common(__other);
+
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(basic_syncbuf& __other) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        allocator_traits<_Allocator>::propagate_on_container_swap::value || get_allocator() == __other.get_allocator(),
+        "violates the mandated swap precondition");
+
+    basic_syncbuf __tmp(std::move(__other));
+    __other = std::move(*this);
+    *this   = std::move(__tmp);
+  }
+
+  // [syncstream.syncbuf.members], member functions
+
+  _LIBCPP_HIDE_FROM_ABI bool emit() { return emit(false); }
+
+  _LIBCPP_HIDE_FROM_ABI streambuf_type* get_wrapped() const noexcept { return __wrapped_; }
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const noexcept { return __str_.get_allocator(); }
+
+  _LIBCPP_HIDE_FROM_ABI void set_emit_on_sync(bool __b) noexcept { __emit_on_sync_ = __b; }
+
+protected:
+  // [syncstream.syncbuf.virtuals], overridden virtual functions
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL
+  int sync() override {
+    if (__emit_on_sync_ && !emit(true))
+      return -1;
+    return 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL
+  int_type overflow(int_type __c = traits_type::eof()) override {
+    if (traits_type::eq_int_type(__c, traits_type::eof()))
+      return traits_type::not_eof(__c);
+
+    if (this->pptr() == this->epptr()) {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      try {
+#  endif
+        size_t __size = __str_.size();
+        __str_.resize(__str_.capacity() + 1);
+        _LIBCPP_ASSERT_INTERNAL(__str_.size() > __size, "the buffer hasn't grown");
+
+        char_type* __p = static_cast<char_type*>(__str_.data());
+        this->setp(__p, __p + __str_.size());
+        this->pbump(__size);
+
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      } catch (...) {
+        return traits_type::eof();
+      }
+#  endif
+    }
+
+    return this->sputc(traits_type::to_char_type(__c));
+  }
+
+private:
+  streambuf_type* __wrapped_;
+
+  // TODO Use a more generic buffer.
+  // That buffer should be light with almost no additional headers. Then
+  // it can be use here, the __retarget_buffer, and place that use
+  // the now deprecated get_temporary_buffer
+
+  basic_string<_CharT, _Traits, _Allocator> __str_;
+  bool __emit_on_sync_{false};
+
+  _LIBCPP_HIDE_FROM_ABI bool emit(bool __flush) {
+    if (!__wrapped_)
+      return false;
+
+#  ifndef _LIBCPP_HAS_NO_THREADS
+    lock_guard<mutex> __lock = __wrapped_streambuf_mutex::__instance().__get_lock(__wrapped_);
+#  endif
+
+    bool __result = true;
+    if (this->pptr() != this->pbase()) {
+      _LIBCPP_ASSERT_INTERNAL(this->pbase() && this->pptr() && this->epptr(), "all put area pointers shold be valid");
+
+      // The __str_ does not know how much of its buffer is used. This
+      // information is extracted from the information of the base class.
+      __result &= (__wrapped_->sputn(this->pbase(), this->pptr() - this->pbase()) != -1);
+      // Clears the buffer, but keeps the contents (and) size of the
+      // internal buffer.
+      this->setp(this->pbase(), this->epptr());
+    }
+
+    if (__flush)
+      __result &= (__wrapped_->pubsync() != -1);
+
+    return __result;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __move_common(basic_syncbuf& __other) {
+    // Adjust the put area pointers to our buffer.
+    char_type* __p = static_cast<char_type*>(__str_.data());
+    this->setp(__p, __p + __str_.size());
+    this->pbump(__other.pptr() - __other.pbase());
+
+    // Clear __other_ so the destructor will act as a NOP.
+    __other.setp(nullptr, nullptr);
+    __other.__wrapped_ = nullptr;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __inc_reference() {
+#  ifndef _LIBCPP_HAS_NO_THREADS
+    if (__wrapped_)
+      __wrapped_streambuf_mutex::__instance().__inc_reference(__wrapped_);
+#  endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __dec_reference() noexcept {
+#  ifndef _LIBCPP_HAS_NO_THREADS
+    if (__wrapped_)
+      __wrapped_streambuf_mutex::__instance().__dec_reference(__wrapped_);
+#  endif
+  }
+};
+
+using std::syncbuf;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using std::wsyncbuf;
+#  endif
+
+// [syncstream.syncbuf.special], specialized algorithms
+template <class _CharT, class _Traits, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI void
+swap(basic_syncbuf<_CharT, _Traits, _Allocator>& __lhs, basic_syncbuf<_CharT, _Traits, _Allocator>& __rhs) {
+  __lhs.swap(__rhs);
+}
+
+// basic_osyncstream
+
+template <class _CharT, class _Traits, class _Allocator>
+class _LIBCPP_TEMPLATE_VIS basic_osyncstream : public basic_ostream<_CharT, _Traits> {
+public:
+  using char_type   = _CharT;
+  using traits_type = _Traits;
+  using int_type    = typename traits_type::int_type;
+  using pos_type    = typename traits_type::pos_type;
+  using off_type    = typename traits_type::off_type;
+
+  using allocator_type = _Allocator;
+  using streambuf_type = basic_streambuf<char_type, traits_type>;
+  using syncbuf_type   = basic_syncbuf<char_type, traits_type, allocator_type>;
+
+  // [syncstream.osyncstream.cons], construction and destruction
+
+  _LIBCPP_HIDE_FROM_ABI basic_osyncstream(streambuf_type* __obuf, allocator_type const& __alloc)
+      : basic_ostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__obuf, __alloc) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_osyncstream(streambuf_type* __obuf)
+      : basic_osyncstream(__obuf, allocator_type()) {}
+
+  _LIBCPP_HIDE_FROM_ABI basic_osyncstream(basic_ostream<char_type, traits_type>& __os, allocator_type const& __alloc)
+      : basic_osyncstream(__os.rdbuf(), __alloc) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit basic_osyncstream(basic_ostream<char_type, traits_type>& __os)
+      : basic_osyncstream(__os, allocator_type()) {}
+
+  _LIBCPP_HIDE_FROM_ABI basic_osyncstream(basic_osyncstream&& __other) noexcept
+      : basic_ostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(std::move(__other.__sb_)) {
+    this->set_rdbuf(std::addressof(__sb_));
+  }
+
+  // [syncstream.osyncstream.assign], assignment
+
+  _LIBCPP_HIDE_FROM_ABI basic_osyncstream& operator=(basic_osyncstream&& __other) = default;
+
+  // [syncstream.osyncstream.members], member functions
+
+  _LIBCPP_HIDE_FROM_ABI void emit() {
+    // The basic_ostream::put places the sentry in a try
+    // catch, this does not match the wording of the standard
+    // [ostream.unformatted]
+    // TODO validate other unformatted output functions.
+    typename basic_ostream<char_type, traits_type>::sentry __s(*this);
+    if (__s) {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      try {
+#  endif
+
+        if (__sb_.emit() == false)
+          this->setstate(ios::badbit);
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      } catch (...) {
+        this->__set_badbit_and_consider_rethrow();
+      }
+#  endif
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI streambuf_type* get_wrapped() const noexcept { return __sb_.get_wrapped(); }
+
+  _LIBCPP_HIDE_FROM_ABI syncbuf_type* rdbuf() const noexcept {
+    return const_cast<syncbuf_type*>(std::addressof(__sb_));
+  }
+
+private:
+  syncbuf_type __sb_;
+};
+
+using std::osyncstream;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+using std::wosyncstream;
+#  endif
+
+#endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM)
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP_SYNCSTREAM
diff --git a/libcxx/include/__cxx03/system_error b/libcxx/include/__cxx03/system_error
new file mode 100644
index 000000000000..eeab347788a9
--- /dev/null
+++ b/libcxx/include/__cxx03/system_error
@@ -0,0 +1,171 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_SYSTEM_ERROR
+#define _LIBCPP_SYSTEM_ERROR
+
+/*
+    system_error synopsis
+
+namespace std
+{
+
+class error_category
+{
+public:
+    virtual ~error_category() noexcept;
+
+    constexpr error_category();
+    error_category(const error_category&) = delete;
+    error_category& operator=(const error_category&) = delete;
+
+    virtual const char* name() const noexcept = 0;
+    virtual error_condition default_error_condition(int ev) const noexcept;
+    virtual bool equivalent(int code, const error_condition& condition) const noexcept;
+    virtual bool equivalent(const error_code& code, int condition) const noexcept;
+    virtual string message(int ev) const = 0;
+
+    bool operator==(const error_category& rhs) const noexcept;
+    bool operator!=(const error_category& rhs) const noexcept;              // removed in C++20
+    bool operator<(const error_category& rhs) const noexcept;               // removed in C++20
+    strong_ordering operator<=>(const error_category& rhs) const noexcept;  // C++20
+};
+
+const error_category& generic_category() noexcept;
+const error_category& system_category() noexcept;
+
+template <class T> struct is_error_code_enum
+    : public false_type {};
+
+template <class T> struct is_error_condition_enum
+    : public false_type {};
+
+template <class _Tp>
+inline constexpr bool is_error_condition_enum_v = is_error_condition_enum<_Tp>::value; // C++17
+
+template <class _Tp>
+inline constexpr bool is_error_code_enum_v = is_error_code_enum<_Tp>::value; // C++17
+
+class error_code
+{
+public:
+    // constructors:
+    error_code() noexcept;
+    error_code(int val, const error_category& cat) noexcept;
+    template <class ErrorCodeEnum>
+        error_code(ErrorCodeEnum e) noexcept;
+
+    // modifiers:
+    void assign(int val, const error_category& cat) noexcept;
+    template <class ErrorCodeEnum>
+        error_code& operator=(ErrorCodeEnum e) noexcept;
+    void clear() noexcept;
+
+    // observers:
+    int value() const noexcept;
+    const error_category& category() const noexcept;
+    error_condition default_error_condition() const noexcept;
+    string message() const;
+    explicit operator bool() const noexcept;
+};
+
+// non-member functions:
+template <class charT, class traits>
+    basic_ostream<charT,traits>&
+    operator<<(basic_ostream<charT,traits>& os, const error_code& ec);
+
+class error_condition
+{
+public:
+    // constructors:
+    error_condition() noexcept;
+    error_condition(int val, const error_category& cat) noexcept;
+    template <class ErrorConditionEnum>
+        error_condition(ErrorConditionEnum e) noexcept;
+
+    // modifiers:
+    void assign(int val, const error_category& cat) noexcept;
+    template <class ErrorConditionEnum>
+        error_condition& operator=(ErrorConditionEnum e) noexcept;
+    void clear() noexcept;
+
+    // observers:
+    int value() const noexcept;
+    const error_category& category() const noexcept;
+    string message() const noexcept;
+    explicit operator bool() const noexcept;
+};
+
+class system_error
+    : public runtime_error
+{
+public:
+    system_error(error_code ec, const string& what_arg);
+    system_error(error_code ec, const char* what_arg);
+    system_error(error_code ec);
+    system_error(int ev, const error_category& ecat, const string& what_arg);
+    system_error(int ev, const error_category& ecat, const char* what_arg);
+    system_error(int ev, const error_category& ecat);
+
+    const error_code& code() const noexcept;
+    const char* what() const noexcept;
+};
+
+template <> struct is_error_condition_enum<errc>
+    : true_type { }
+
+error_code make_error_code(errc e) noexcept;
+error_condition make_error_condition(errc e) noexcept;
+
+// Comparison operators:
+bool operator==(const error_code& lhs, const error_code& rhs) noexcept;
+bool operator==(const error_code& lhs, const error_condition& rhs) noexcept;
+bool operator==(const error_condition& lhs, const error_code& rhs) noexcept;                  // removed in C++20
+bool operator==(const error_condition& lhs, const error_condition& rhs) noexcept;
+bool operator!=(const error_code& lhs, const error_code& rhs) noexcept;                       // removed in C++20
+bool operator!=(const error_code& lhs, const error_condition& rhs) noexcept;                  // removed in C++20
+bool operator!=(const error_condition& lhs, const error_code& rhs) noexcept;                  // removed in C++20
+bool operator!=(const error_condition& lhs, const error_condition& rhs) noexcept;             // removed in C++20
+bool operator<(const error_condition& lhs, const error_condition& rhs) noexcept;              // removed in C++20
+bool operator<(const error_code& lhs, const error_code& rhs) noexcept;                        // removed in C++20
+strong_ordering operator<=>(const error_code& lhs, const error_code& rhs) noexcept;           // C++20
+strong_ordering operator<=>(const error_condition& lhs, const error_condition& rhs) noexcept; // C++20
+
+template <> struct hash<std::error_code>;
+template <> struct hash<std::error_condition>;
+
+}  // std
+
+*/
+
+#include <__config>
+#include <__system_error/errc.h>
+#include <__system_error/error_category.h>
+#include <__system_error/error_code.h>
+#include <__system_error/error_condition.h>
+#include <__system_error/system_error.h>
+#include <version>
+
+// standard-mandated includes
+
+// [system.error.syn]
+#include <compare>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstdint>
+#  include <cstring>
+#  include <limits>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_SYSTEM_ERROR
diff --git a/libcxx/include/__cxx03/tgmath.h b/libcxx/include/__cxx03/tgmath.h
new file mode 100644
index 000000000000..e6f0a4ab2611
--- /dev/null
+++ b/libcxx/include/__cxx03/tgmath.h
@@ -0,0 +1,34 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_TGMATH_H
+#define _LIBCPP_TGMATH_H
+
+/*
+    tgmath.h synopsis
+
+#include <ctgmath>
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#ifdef __cplusplus
+#  include <ctgmath>
+#else
+#  if __has_include_next(<tgmath.h>)
+#    include_next <tgmath.h>
+#  endif
+#endif
+
+#endif // _LIBCPP_TGMATH_H
diff --git a/libcxx/include/__cxx03/thread b/libcxx/include/__cxx03/thread
new file mode 100644
index 000000000000..25cb7ce6d723
--- /dev/null
+++ b/libcxx/include/__cxx03/thread
@@ -0,0 +1,130 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_THREAD
+#define _LIBCPP_THREAD
+
+/*
+
+    thread synopsis
+
+namespace std
+{
+
+class thread
+{
+public:
+    class id;
+    typedef pthread_t native_handle_type;
+
+    thread() noexcept;
+    template <class F, class ...Args> explicit thread(F&& f, Args&&... args);
+    ~thread();
+
+    thread(const thread&) = delete;
+    thread(thread&& t) noexcept;
+
+    thread& operator=(const thread&) = delete;
+    thread& operator=(thread&& t) noexcept;
+
+    void swap(thread& t) noexcept;
+
+    bool joinable() const noexcept;
+    void join();
+    void detach();
+    id get_id() const noexcept;
+    native_handle_type native_handle();
+
+    static unsigned hardware_concurrency() noexcept;
+};
+
+void swap(thread& x, thread& y) noexcept;
+
+class thread::id
+{
+public:
+    id() noexcept;
+};
+
+bool operator==(thread::id x, thread::id y) noexcept;
+bool operator!=(thread::id x, thread::id y) noexcept;             // removed in C++20
+bool operator< (thread::id x, thread::id y) noexcept;             // removed in C++20
+bool operator<=(thread::id x, thread::id y) noexcept;             // removed in C++20
+bool operator> (thread::id x, thread::id y) noexcept;             // removed in C++20
+bool operator>=(thread::id x, thread::id y) noexcept;             // removed in C++20
+strong_ordering operator<=>(thread::id x, thread::id y) noexcept; // C++20
+
+template<class charT, class traits>
+basic_ostream<charT, traits>&
+operator<<(basic_ostream<charT, traits>& out, thread::id id);
+
+template<class charT>
+struct formatter<thread::id, charT>;
+
+namespace this_thread
+{
+
+thread::id get_id() noexcept;
+
+void yield() noexcept;
+
+template <class Clock, class Duration>
+void sleep_until(const chrono::time_point<Clock, Duration>& abs_time);
+
+template <class Rep, class Period>
+void sleep_for(const chrono::duration<Rep, Period>& rel_time);
+
+}  // this_thread
+
+}  // std
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+
+#  include <__thread/formatter.h>
+#  include <__thread/jthread.h>
+#  include <__thread/support.h>
+#  include <__thread/this_thread.h>
+#  include <__thread/thread.h>
+#  include <version>
+
+// standard-mandated includes
+
+// [thread.syn]
+#  include <compare>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#endif // !defined(_LIBCPP_HAS_NO_THREADS)
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES)
+#  include <cstddef>
+#  include <ctime>
+#  include <iosfwd>
+#  include <ratio>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#  include <chrono>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstring>
+#  include <functional>
+#  include <new>
+#  include <system_error>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_THREAD
diff --git a/libcxx/include/__cxx03/tuple b/libcxx/include/__cxx03/tuple
new file mode 100644
index 000000000000..5161c2aa97c2
--- /dev/null
+++ b/libcxx/include/__cxx03/tuple
@@ -0,0 +1,1419 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_TUPLE
+#define _LIBCPP_TUPLE
+
+// clang-format off
+
+/*
+    tuple synopsis
+
+namespace std
+{
+
+template <class... T>
+class tuple {
+public:
+    explicit(see-below) constexpr tuple();
+    explicit(see-below) tuple(const T&...);  // constexpr in C++14
+    template <class... U>
+        explicit(see-below) tuple(U&&...);  // constexpr in C++14
+    tuple(const tuple&) = default;
+    tuple(tuple&&) = default;
+
+    template<class... UTypes>
+        constexpr explicit(see-below) tuple(tuple<UTypes...>&);  // C++23
+    template <class... U>
+        explicit(see-below) tuple(const tuple<U...>&);  // constexpr in C++14
+    template <class... U>
+        explicit(see-below) tuple(tuple<U...>&&);  // constexpr in C++14
+    template<class... UTypes>
+        constexpr explicit(see-below) tuple(const tuple<UTypes...>&&); // C++23
+
+    template<class U1, class U2>
+        constexpr explicit(see-below) tuple(pair<U1, U2>&);  // iff sizeof...(Types) == 2 // C++23
+    template <class U1, class U2>
+        explicit(see-below) tuple(const pair<U1, U2>&); // iff sizeof...(T) == 2 // constexpr in C++14
+    template <class U1, class U2>
+        explicit(see-below) tuple(pair<U1, U2>&&); // iff sizeof...(T) == 2  // constexpr in C++14
+    template<class U1, class U2>
+        constexpr explicit(see-below) tuple(const pair<U1, U2>&&);  // iff sizeof...(Types) == 2 // C++23
+
+    // allocator-extended constructors
+    template <class Alloc>
+        tuple(allocator_arg_t, const Alloc& a);
+    template <class Alloc>
+        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, const T&...);          // constexpr in C++20
+    template <class Alloc, class... U>
+        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, U&&...);               // constexpr in C++20
+    template <class Alloc>
+        tuple(allocator_arg_t, const Alloc& a, const tuple&);                             // constexpr in C++20
+    template <class Alloc>
+        tuple(allocator_arg_t, const Alloc& a, tuple&&);                                  // constexpr in C++20
+    template<class Alloc, class... UTypes>
+        constexpr explicit(see-below)
+          tuple(allocator_arg_t, const Alloc& a, tuple<UTypes...>&);                      // C++23
+    template <class Alloc, class... U>
+        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, const tuple<U...>&);   // constexpr in C++20
+    template <class Alloc, class... U>
+        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, tuple<U...>&&);        // constexpr in C++20
+    template<class Alloc, class... UTypes>
+        constexpr explicit(see-below)
+          tuple(allocator_arg_t, const Alloc& a, const tuple<UTypes...>&&);               // C++23
+    template<class Alloc, class U1, class U2>
+        constexpr explicit(see-below)
+          tuple(allocator_arg_t, const Alloc& a, pair<U1, U2>&);                          // C++23
+    template <class Alloc, class U1, class U2>
+        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, const pair<U1, U2>&);  // constexpr in C++20
+    template <class Alloc, class U1, class U2>
+        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, pair<U1, U2>&&);       // constexpr in C++20
+    template<class Alloc, class U1, class U2>
+        constexpr explicit(see-below)
+          tuple(allocator_arg_t, const Alloc& a, const pair<U1, U2>&&);                   // C++23
+
+    tuple& operator=(const tuple&);                                                       // constexpr in C++20
+    constexpr const tuple& operator=(const tuple&) const;                                 // C++23
+    tuple& operator=(tuple&&) noexcept(is_nothrow_move_assignable_v<T> && ...);           // constexpr in C++20
+    constexpr const tuple& operator=(tuple&&) const;                                      // C++23
+    template <class... U>
+        tuple& operator=(const tuple<U...>&);                                             // constexpr in C++20
+    template<class... UTypes>
+        constexpr const tuple& operator=(const tuple<UTypes...>&) const;                  // C++23
+    template <class... U>
+        tuple& operator=(tuple<U...>&&);                                                  // constexpr in C++20
+    template<class... UTypes>
+        constexpr const tuple& operator=(tuple<UTypes...>&&) const;                       // C++23
+    template <class U1, class U2>
+        tuple& operator=(const pair<U1, U2>&); // iff sizeof...(T) == 2                   // constexpr in C++20
+    template<class U1, class U2>
+        constexpr const tuple& operator=(const pair<U1, U2>&) const;   // iff sizeof...(Types) == 2 // C++23
+    template <class U1, class U2>
+        tuple& operator=(pair<U1, U2>&&); // iff sizeof...(T) == 2                        // constexpr in C++20
+    template<class U1, class U2>
+        constexpr const tuple& operator=(pair<U1, U2>&&) const;  // iff sizeof...(Types) == 2 // C++23
+
+    template<class U, size_t N>
+        tuple& operator=(array<U, N> const&) // iff sizeof...(T) == N, EXTENSION
+    template<class U, size_t N>
+        tuple& operator=(array<U, N>&&) // iff sizeof...(T) == N, EXTENSION
+
+    void swap(tuple&) noexcept(AND(swap(declval<T&>(), declval<T&>())...));               // constexpr in C++20
+    constexpr void swap(const tuple&) const noexcept(see-below);                          // C++23
+};
+
+
+template<class... TTypes, class... UTypes, template<class> class TQual, template<class> class UQual> // since C++23
+  requires requires { typename tuple<common_reference_t<TQual<TTypes>, UQual<UTypes>>...>; }
+struct basic_common_reference<tuple<TTypes...>, tuple<UTypes...>, TQual, UQual> {
+  using type = tuple<common_reference_t<TQual<TTypes>, UQual<UTypes>>...>;
+};
+
+template<class... TTypes, class... UTypes>                                // since C++23
+  requires requires { typename tuple<common_type_t<TTypes, UTypes>...>; }
+struct common_type<tuple<TTypes...>, tuple<UTypes...>> {
+  using type = tuple<common_type_t<TTypes, UTypes>...>;
+};
+
+template <class ...T>
+tuple(T...) -> tuple<T...>;                                         // since C++17
+template <class T1, class T2>
+tuple(pair<T1, T2>) -> tuple<T1, T2>;                               // since C++17
+template <class Alloc, class ...T>
+tuple(allocator_arg_t, Alloc, T...) -> tuple<T...>;                 // since C++17
+template <class Alloc, class T1, class T2>
+tuple(allocator_arg_t, Alloc, pair<T1, T2>) -> tuple<T1, T2>;       // since C++17
+template <class Alloc, class ...T>
+tuple(allocator_arg_t, Alloc, tuple<T...>) -> tuple<T...>;          // since C++17
+
+struct ignore-type { // exposition only                             // Since C++26
+  constexpr const ignore-type&
+    operator=(const auto &) const noexcept
+      { return *this; }
+};
+inline constexpr ignore-type ignore;
+
+template <class... T> tuple<V...>  make_tuple(T&&...); // constexpr in C++14
+template <class... T> tuple<ATypes...> forward_as_tuple(T&&...) noexcept; // constexpr in C++14
+template <class... T> tuple<T&...> tie(T&...) noexcept; // constexpr in C++14
+template <class... Tuples> tuple<CTypes...> tuple_cat(Tuples&&... tpls); // constexpr in C++14
+
+// [tuple.apply], calling a function with a tuple of arguments:
+template <class F, class Tuple>
+  constexpr decltype(auto) apply(F&& f, Tuple&& t) noexcept(see below);  // C++17 noexcept since C++23
+template <class T, class Tuple>
+  constexpr T make_from_tuple(Tuple&& t); // C++17
+
+// 20.4.1.4, tuple helper classes:
+template <class T> struct tuple_size; // undefined
+template <class... T> struct tuple_size<tuple<T...>>;
+template <class T>
+ inline constexpr size_t tuple_size_v = tuple_size<T>::value; // C++17
+template <size_t I, class T> struct tuple_element; // undefined
+template <size_t I, class... T> struct tuple_element<I, tuple<T...>>;
+template <size_t I, class T>
+  using tuple_element_t = typename tuple_element <I, T>::type; // C++14
+
+// 20.4.1.5, element access:
+template <size_t I, class... T>
+    typename tuple_element<I, tuple<T...>>::type&
+    get(tuple<T...>&) noexcept; // constexpr in C++14
+template <size_t I, class... T>
+    const typename tuple_element<I, tuple<T...>>::type&
+    get(const tuple<T...>&) noexcept; // constexpr in C++14
+template <size_t I, class... T>
+    typename tuple_element<I, tuple<T...>>::type&&
+    get(tuple<T...>&&) noexcept; // constexpr in C++14
+template <size_t I, class... T>
+    const typename tuple_element<I, tuple<T...>>::type&&
+    get(const tuple<T...>&&) noexcept; // constexpr in C++14
+
+template <class T1, class... T>
+    constexpr T1& get(tuple<T...>&) noexcept;  // C++14
+template <class T1, class... T>
+    constexpr const T1& get(const tuple<T...>&) noexcept;   // C++14
+template <class T1, class... T>
+    constexpr T1&& get(tuple<T...>&&) noexcept;   // C++14
+template <class T1, class... T>
+    constexpr const T1&& get(const tuple<T...>&&) noexcept;   // C++14
+
+// 20.4.1.6, relational operators:
+template<class... T, class... U> bool operator==(const tuple<T...>&, const tuple<U...>&); // constexpr in C++14
+template<class... T, class... U> bool operator<(const tuple<T...>&, const tuple<U...>&);  // constexpr in C++14, removed in C++20
+template<class... T, class... U> bool operator!=(const tuple<T...>&, const tuple<U...>&); // constexpr in C++14, removed in C++20
+template<class... T, class... U> bool operator>(const tuple<T...>&, const tuple<U...>&);  // constexpr in C++14, removed in C++20
+template<class... T, class... U> bool operator<=(const tuple<T...>&, const tuple<U...>&); // constexpr in C++14, removed in C++20
+template<class... T, class... U> bool operator>=(const tuple<T...>&, const tuple<U...>&); // constexpr in C++14, removed in C++20
+template<class... T, class... U>
+  constexpr common_comparison_category_t<synth-three-way-result<T, U>...>
+    operator<=>(const tuple<T...>&, const tuple<U...>&);                                  // since C++20
+
+template <class... Types, class Alloc>
+  struct uses_allocator<tuple<Types...>, Alloc>;
+
+template <class... Types>
+  void
+  swap(tuple<Types...>& x, tuple<Types...>& y) noexcept(noexcept(x.swap(y)));
+
+template <class... Types>
+  constexpr void swap(const tuple<Types...>& x, const tuple<Types...>& y) noexcept(see-below);   // C++23
+
+}  // std
+
+*/
+
+// clang-format on
+
+#include <__compare/common_comparison_category.h>
+#include <__compare/synth_three_way.h>
+#include <__config>
+#include <__functional/invoke.h>
+#include <__fwd/array.h>
+#include <__fwd/pair.h>
+#include <__fwd/tuple.h>
+#include <__memory/allocator_arg_t.h>
+#include <__memory/uses_allocator.h>
+#include <__tuple/find_index.h>
+#include <__tuple/ignore.h>
+#include <__tuple/make_tuple_types.h>
+#include <__tuple/sfinae_helpers.h>
+#include <__tuple/tuple_element.h>
+#include <__tuple/tuple_indices.h>
+#include <__tuple/tuple_like_ext.h>
+#include <__tuple/tuple_size.h>
+#include <__tuple/tuple_types.h>
+#include <__type_traits/common_reference.h>
+#include <__type_traits/common_type.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/copy_cvref.h>
+#include <__type_traits/disjunction.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_empty.h>
+#include <__type_traits/is_final.h>
+#include <__type_traits/is_implicitly_default_constructible.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_swappable.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__type_traits/lazy.h>
+#include <__type_traits/maybe_const.h>
+#include <__type_traits/nat.h>
+#include <__type_traits/negation.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/remove_reference.h>
+#include <__type_traits/unwrap_ref.h>
+#include <__utility/forward.h>
+#include <__utility/integer_sequence.h>
+#include <__utility/move.h>
+#include <__utility/piecewise_construct.h>
+#include <__utility/swap.h>
+#include <cstddef>
+#include <version>
+
+// standard-mandated includes
+
+// [tuple.syn]
+#include <compare>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifndef _LIBCPP_CXX03_LANG
+
+// __tuple_leaf
+
+template <size_t _Ip, class _Hp, bool = is_empty<_Hp>::value && !__libcpp_is_final<_Hp>::value >
+class __tuple_leaf;
+
+template <size_t _Ip, class _Hp, bool _Ep>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+swap(__tuple_leaf<_Ip, _Hp, _Ep>& __x, __tuple_leaf<_Ip, _Hp, _Ep>& __y) noexcept(__is_nothrow_swappable_v<_Hp>) {
+  swap(__x.get(), __y.get());
+}
+
+template <size_t _Ip, class _Hp, bool _Ep>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+swap(const __tuple_leaf<_Ip, _Hp, _Ep>& __x,
+     const __tuple_leaf<_Ip, _Hp, _Ep>& __y) noexcept(__is_nothrow_swappable_v<const _Hp>) {
+  swap(__x.get(), __y.get());
+}
+
+template <size_t _Ip, class _Hp, bool>
+class __tuple_leaf {
+  _Hp __value_;
+
+  template <class _Tp>
+  static _LIBCPP_HIDE_FROM_ABI constexpr bool __can_bind_reference() {
+#  if __has_keyword(__reference_binds_to_temporary)
+    return !__reference_binds_to_temporary(_Hp, _Tp);
+#  else
+    return true;
+#  endif
+  }
+
+public:
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 __tuple_leaf& operator=(const __tuple_leaf&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __tuple_leaf() noexcept(is_nothrow_default_constructible<_Hp>::value) : __value_() {
+    static_assert(!is_reference<_Hp>::value, "Attempted to default construct a reference element in a tuple");
+  }
+
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI constexpr __tuple_leaf(integral_constant<int, 0>, const _Alloc&) : __value_() {
+    static_assert(!is_reference<_Hp>::value, "Attempted to default construct a reference element in a tuple");
+  }
+
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI constexpr __tuple_leaf(integral_constant<int, 1>, const _Alloc& __a)
+      : __value_(allocator_arg_t(), __a) {
+    static_assert(!is_reference<_Hp>::value, "Attempted to default construct a reference element in a tuple");
+  }
+
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI constexpr __tuple_leaf(integral_constant<int, 2>, const _Alloc& __a) : __value_(__a) {
+    static_assert(!is_reference<_Hp>::value, "Attempted to default construct a reference element in a tuple");
+  }
+
+  template <
+      class _Tp,
+      __enable_if_t<_And<_IsNotSame<__remove_cvref_t<_Tp>, __tuple_leaf>, is_constructible<_Hp, _Tp> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __tuple_leaf(_Tp&& __t) noexcept(is_nothrow_constructible<_Hp, _Tp>::value)
+      : __value_(std::forward<_Tp>(__t)) {
+    static_assert(__can_bind_reference<_Tp&&>(),
+                  "Attempted construction of reference element binds to a temporary whose lifetime has ended");
+  }
+
+  template <class _Tp, class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __tuple_leaf(integral_constant<int, 0>, const _Alloc&, _Tp&& __t)
+      : __value_(std::forward<_Tp>(__t)) {
+    static_assert(__can_bind_reference<_Tp&&>(),
+                  "Attempted construction of reference element binds to a temporary whose lifetime has ended");
+  }
+
+  template <class _Tp, class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __tuple_leaf(integral_constant<int, 1>, const _Alloc& __a, _Tp&& __t)
+      : __value_(allocator_arg_t(), __a, std::forward<_Tp>(__t)) {
+    static_assert(!is_reference<_Hp>::value, "Attempted to uses-allocator construct a reference element in a tuple");
+  }
+
+  template <class _Tp, class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __tuple_leaf(integral_constant<int, 2>, const _Alloc& __a, _Tp&& __t)
+      : __value_(std::forward<_Tp>(__t), __a) {
+    static_assert(!is_reference<_Hp>::value, "Attempted to uses-allocator construct a reference element in a tuple");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __tuple_leaf(const __tuple_leaf& __t) = default;
+  _LIBCPP_HIDE_FROM_ABI __tuple_leaf(__tuple_leaf&& __t)      = default;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int
+  swap(__tuple_leaf& __t) noexcept(__is_nothrow_swappable_v<__tuple_leaf>) {
+    std::swap(*this, __t);
+    return 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int swap(const __tuple_leaf& __t) const
+      noexcept(__is_nothrow_swappable_v<const __tuple_leaf>) {
+    std::swap(*this, __t);
+    return 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Hp& get() _NOEXCEPT { return __value_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Hp& get() const _NOEXCEPT { return __value_; }
+};
+
+template <size_t _Ip, class _Hp>
+class __tuple_leaf<_Ip, _Hp, true> : private _Hp {
+public:
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 __tuple_leaf& operator=(const __tuple_leaf&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr __tuple_leaf() noexcept(is_nothrow_default_constructible<_Hp>::value) {}
+
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI constexpr __tuple_leaf(integral_constant<int, 0>, const _Alloc&) {}
+
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI constexpr __tuple_leaf(integral_constant<int, 1>, const _Alloc& __a)
+      : _Hp(allocator_arg_t(), __a) {}
+
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI constexpr __tuple_leaf(integral_constant<int, 2>, const _Alloc& __a) : _Hp(__a) {}
+
+  template <class _Tp,
+            __enable_if_t< _And< _IsNotSame<__remove_cvref_t<_Tp>, __tuple_leaf>, is_constructible<_Hp, _Tp> >::value,
+                           int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __tuple_leaf(_Tp&& __t) noexcept(is_nothrow_constructible<_Hp, _Tp>::value)
+      : _Hp(std::forward<_Tp>(__t)) {}
+
+  template <class _Tp, class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __tuple_leaf(integral_constant<int, 0>, const _Alloc&, _Tp&& __t)
+      : _Hp(std::forward<_Tp>(__t)) {}
+
+  template <class _Tp, class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __tuple_leaf(integral_constant<int, 1>, const _Alloc& __a, _Tp&& __t)
+      : _Hp(allocator_arg_t(), __a, std::forward<_Tp>(__t)) {}
+
+  template <class _Tp, class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __tuple_leaf(integral_constant<int, 2>, const _Alloc& __a, _Tp&& __t)
+      : _Hp(std::forward<_Tp>(__t), __a) {}
+
+  __tuple_leaf(__tuple_leaf const&) = default;
+  __tuple_leaf(__tuple_leaf&&)      = default;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int
+  swap(__tuple_leaf& __t) noexcept(__is_nothrow_swappable_v<__tuple_leaf>) {
+    std::swap(*this, __t);
+    return 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int swap(const __tuple_leaf& __rhs) const
+      noexcept(__is_nothrow_swappable_v<const __tuple_leaf>) {
+    std::swap(*this, __rhs);
+    return 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Hp& get() _NOEXCEPT { return static_cast<_Hp&>(*this); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Hp& get() const _NOEXCEPT {
+    return static_cast<const _Hp&>(*this);
+  }
+};
+
+template <class... _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __swallow(_Tp&&...) _NOEXCEPT {}
+
+template <class _Tp>
+struct __all_default_constructible;
+
+template <class... _Tp>
+struct __all_default_constructible<__tuple_types<_Tp...>> : __all<is_default_constructible<_Tp>::value...> {};
+
+// __tuple_impl
+
+template <class _Indx, class... _Tp>
+struct __tuple_impl;
+
+template <size_t... _Indx, class... _Tp>
+struct _LIBCPP_DECLSPEC_EMPTY_BASES __tuple_impl<__tuple_indices<_Indx...>, _Tp...>
+    : public __tuple_leaf<_Indx, _Tp>... {
+  _LIBCPP_HIDE_FROM_ABI constexpr __tuple_impl() noexcept(
+      __all<is_nothrow_default_constructible<_Tp>::value...>::value) {}
+
+  template <size_t... _Uf, class... _Tf, size_t... _Ul, class... _Tl, class... _Up>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __tuple_impl(
+      __tuple_indices<_Uf...>,
+      __tuple_types<_Tf...>,
+      __tuple_indices<_Ul...>,
+      __tuple_types<_Tl...>,
+      _Up&&... __u) noexcept(__all<is_nothrow_constructible<_Tf, _Up>::value...>::value &&
+                             __all<is_nothrow_default_constructible<_Tl>::value...>::value)
+      : __tuple_leaf<_Uf, _Tf>(std::forward<_Up>(__u))..., __tuple_leaf<_Ul, _Tl>()... {}
+
+  template <class _Alloc, size_t... _Uf, class... _Tf, size_t... _Ul, class... _Tl, class... _Up>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __tuple_impl(
+      allocator_arg_t,
+      const _Alloc& __a,
+      __tuple_indices<_Uf...>,
+      __tuple_types<_Tf...>,
+      __tuple_indices<_Ul...>,
+      __tuple_types<_Tl...>,
+      _Up&&... __u)
+      : __tuple_leaf<_Uf, _Tf>(__uses_alloc_ctor<_Tf, _Alloc, _Up>(), __a, std::forward<_Up>(__u))...,
+        __tuple_leaf<_Ul, _Tl>(__uses_alloc_ctor<_Tl, _Alloc>(), __a)... {}
+
+  template <class _Tuple, __enable_if_t<__tuple_constructible<_Tuple, tuple<_Tp...> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __tuple_impl(_Tuple&& __t) noexcept(
+      (__all<is_nothrow_constructible<
+           _Tp,
+           typename tuple_element<_Indx, typename __make_tuple_types<_Tuple>::type>::type>::value...>::value))
+      : __tuple_leaf<_Indx, _Tp>(
+            std::forward<typename tuple_element<_Indx, typename __make_tuple_types<_Tuple>::type>::type>(
+                std::get<_Indx>(__t)))... {}
+
+  template <class _Alloc, class _Tuple, __enable_if_t<__tuple_constructible<_Tuple, tuple<_Tp...> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __tuple_impl(allocator_arg_t, const _Alloc& __a, _Tuple&& __t)
+      : __tuple_leaf<_Indx, _Tp>(
+            __uses_alloc_ctor<_Tp,
+                              _Alloc,
+                              typename tuple_element<_Indx, typename __make_tuple_types<_Tuple>::type>::type>(),
+            __a,
+            std::forward<typename tuple_element<_Indx, typename __make_tuple_types<_Tuple>::type>::type>(
+                std::get<_Indx>(__t)))... {}
+
+  __tuple_impl(const __tuple_impl&) = default;
+  __tuple_impl(__tuple_impl&&)      = default;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+  swap(__tuple_impl& __t) noexcept(__all<__is_nothrow_swappable_v<_Tp>...>::value) {
+    std::__swallow(__tuple_leaf<_Indx, _Tp>::swap(static_cast<__tuple_leaf<_Indx, _Tp>&>(__t))...);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void swap(const __tuple_impl& __t) const
+      noexcept(__all<__is_nothrow_swappable_v<const _Tp>...>::value) {
+    std::__swallow(__tuple_leaf<_Indx, _Tp>::swap(static_cast<const __tuple_leaf<_Indx, _Tp>&>(__t))...);
+  }
+};
+
+template <class _Dest, class _Source, size_t... _Np>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+__memberwise_copy_assign(_Dest& __dest, _Source const& __source, __tuple_indices<_Np...>) {
+  std::__swallow(((std::get<_Np>(__dest) = std::get<_Np>(__source)), void(), 0)...);
+}
+
+template <class _Dest, class _Source, class... _Up, size_t... _Np>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+__memberwise_forward_assign(_Dest& __dest, _Source&& __source, __tuple_types<_Up...>, __tuple_indices<_Np...>) {
+  std::__swallow(((std::get<_Np>(__dest) = std::forward<_Up>(std::get<_Np>(__source))), void(), 0)...);
+}
+
+template <class... _Tp>
+class _LIBCPP_TEMPLATE_VIS tuple {
+  typedef __tuple_impl<typename __make_tuple_indices<sizeof...(_Tp)>::type, _Tp...> _BaseT;
+
+  _BaseT __base_;
+
+  template <size_t _Jp, class... _Up>
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Jp, tuple<_Up...> >::type& get(tuple<_Up...>&) _NOEXCEPT;
+  template <size_t _Jp, class... _Up>
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Jp, tuple<_Up...> >::type&
+  get(const tuple<_Up...>&) _NOEXCEPT;
+  template <size_t _Jp, class... _Up>
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Jp, tuple<_Up...> >::type&&
+  get(tuple<_Up...>&&) _NOEXCEPT;
+  template <size_t _Jp, class... _Up>
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Jp, tuple<_Up...> >::type&&
+  get(const tuple<_Up...>&&) _NOEXCEPT;
+
+public:
+  using __trivially_relocatable = __conditional_t<_And<__libcpp_is_trivially_relocatable<_Tp>...>::value, tuple, void>;
+
+  // [tuple.cnstr]
+
+  // tuple() constructors (including allocator_arg_t variants)
+  template <template <class...> class _IsImpDefault                = __is_implicitly_default_constructible,
+            template <class...> class _IsDefault                   = is_default_constructible,
+            __enable_if_t< _And< _IsDefault<_Tp>... >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(_Not<_Lazy<_And, _IsImpDefault<_Tp>...> >::value)
+      tuple() noexcept(_And<is_nothrow_default_constructible<_Tp>...>::value) {}
+
+  template <class _Alloc,
+            template <class...> class _IsImpDefault                = __is_implicitly_default_constructible,
+            template <class...> class _IsDefault                   = is_default_constructible,
+            __enable_if_t< _And< _IsDefault<_Tp>... >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_Lazy<_And, _IsImpDefault<_Tp>...> >::value)
+      tuple(allocator_arg_t, _Alloc const& __a)
+      : __base_(allocator_arg_t(),
+                __a,
+                __tuple_indices<>(),
+                __tuple_types<>(),
+                typename __make_tuple_indices<sizeof...(_Tp), 0>::type(),
+                __tuple_types<_Tp...>()) {}
+
+  // tuple(const T&...) constructors (including allocator_arg_t variants)
+  template <template <class...> class _And = _And,
+            __enable_if_t< _And< _BoolConstant<sizeof...(_Tp) >= 1>, is_copy_constructible<_Tp>... >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_Lazy<_And, is_convertible<const _Tp&, _Tp>...> >::value)
+      tuple(const _Tp&... __t) noexcept(_And<is_nothrow_copy_constructible<_Tp>...>::value)
+      : __base_(typename __make_tuple_indices<sizeof...(_Tp)>::type(),
+                typename __make_tuple_types<tuple, sizeof...(_Tp)>::type(),
+                typename __make_tuple_indices<0>::type(),
+                typename __make_tuple_types<tuple, 0>::type(),
+                __t...) {}
+
+  template <class _Alloc,
+            template <class...> class _And = _And,
+            __enable_if_t< _And< _BoolConstant<sizeof...(_Tp) >= 1>, is_copy_constructible<_Tp>... >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_Lazy<_And, is_convertible<const _Tp&, _Tp>...> >::value)
+      tuple(allocator_arg_t, const _Alloc& __a, const _Tp&... __t)
+      : __base_(allocator_arg_t(),
+                __a,
+                typename __make_tuple_indices<sizeof...(_Tp)>::type(),
+                typename __make_tuple_types<tuple, sizeof...(_Tp)>::type(),
+                typename __make_tuple_indices<0>::type(),
+                typename __make_tuple_types<tuple, 0>::type(),
+                __t...) {}
+
+  // tuple(U&& ...) constructors (including allocator_arg_t variants)
+  template <class... _Up>
+  struct _IsThisTuple : false_type {};
+  template <class _Up>
+  struct _IsThisTuple<_Up> : is_same<__remove_cvref_t<_Up>, tuple> {};
+
+  template <class... _Up>
+  struct _EnableUTypesCtor
+      : _And< _BoolConstant<sizeof...(_Tp) >= 1>,
+              _Not<_IsThisTuple<_Up...> >, // extension to allow mis-behaved user constructors
+              is_constructible<_Tp, _Up>... > {};
+
+  template <class... _Up,
+            __enable_if_t< _And< _BoolConstant<sizeof...(_Up) == sizeof...(_Tp)>, _EnableUTypesCtor<_Up...> >::value,
+                           int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_Lazy<_And, is_convertible<_Up, _Tp>...> >::value)
+      tuple(_Up&&... __u) noexcept(_And<is_nothrow_constructible<_Tp, _Up>...>::value)
+      : __base_(typename __make_tuple_indices<sizeof...(_Up)>::type(),
+                typename __make_tuple_types<tuple, sizeof...(_Up)>::type(),
+                typename __make_tuple_indices<sizeof...(_Tp), sizeof...(_Up)>::type(),
+                typename __make_tuple_types<tuple, sizeof...(_Tp), sizeof...(_Up)>::type(),
+                std::forward<_Up>(__u)...) {}
+
+  template <class _Alloc,
+            class... _Up,
+            __enable_if_t< _And< _BoolConstant<sizeof...(_Up) == sizeof...(_Tp)>, _EnableUTypesCtor<_Up...> >::value,
+                           int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_Lazy<_And, is_convertible<_Up, _Tp>...> >::value)
+      tuple(allocator_arg_t, const _Alloc& __a, _Up&&... __u)
+      : __base_(allocator_arg_t(),
+                __a,
+                typename __make_tuple_indices<sizeof...(_Up)>::type(),
+                typename __make_tuple_types<tuple, sizeof...(_Up)>::type(),
+                typename __make_tuple_indices<sizeof...(_Tp), sizeof...(_Up)>::type(),
+                typename __make_tuple_types<tuple, sizeof...(_Tp), sizeof...(_Up)>::type(),
+                std::forward<_Up>(__u)...) {}
+
+  // Copy and move constructors (including the allocator_arg_t variants)
+  tuple(const tuple&) = default;
+  tuple(tuple&&)      = default;
+
+  template <class _Alloc,
+            template <class...> class _And                                  = _And,
+            __enable_if_t< _And<is_copy_constructible<_Tp>...>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple(allocator_arg_t, const _Alloc& __alloc, const tuple& __t)
+      : __base_(allocator_arg_t(), __alloc, __t) {}
+
+  template <class _Alloc,
+            template <class...> class _And                                  = _And,
+            __enable_if_t< _And<is_move_constructible<_Tp>...>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple(allocator_arg_t, const _Alloc& __alloc, tuple&& __t)
+      : __base_(allocator_arg_t(), __alloc, std::move(__t)) {}
+
+  // tuple(const tuple<U...>&) constructors (including allocator_arg_t variants)
+
+  template <class _OtherTuple, class _DecayedOtherTuple = __remove_cvref_t<_OtherTuple>, class = void>
+  struct _EnableCtorFromUTypesTuple : false_type {};
+
+  template <class _OtherTuple, class... _Up>
+  struct _EnableCtorFromUTypesTuple<
+      _OtherTuple,
+      tuple<_Up...>,
+      // the length of the packs needs to checked first otherwise the 2 packs cannot be expanded simultaneously below
+      __enable_if_t<sizeof...(_Up) == sizeof...(_Tp)>>
+      : _And<
+            // the two conditions below are not in spec. The purpose is to disable the UTypes Ctor when copy/move Ctor
+            // can work. Otherwise, is_constructible can trigger hard error in those cases
+            // https://godbolt.org/z/M94cGdKcE
+            _Not<is_same<_OtherTuple, const tuple&> >,
+            _Not<is_same<_OtherTuple, tuple&&> >,
+            is_constructible<_Tp, __copy_cvref_t<_OtherTuple, _Up> >...,
+            _Lazy<_Or,
+                  _BoolConstant<sizeof...(_Tp) != 1>,
+                  // _Tp and _Up are 1-element packs - the pack expansions look
+                  // weird to avoid tripping up the type traits in degenerate cases
+                  _Lazy<_And,
+                        _Not<is_same<_Tp, _Up> >...,
+                        _Not<is_convertible<_OtherTuple, _Tp> >...,
+                        _Not<is_constructible<_Tp, _OtherTuple> >... > > > {};
+
+  template <class... _Up, __enable_if_t< _And< _EnableCtorFromUTypesTuple<const tuple<_Up...>&> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_Lazy<_And, is_convertible<const _Up&, _Tp>...> >::value)
+      tuple(const tuple<_Up...>& __t) noexcept(_And<is_nothrow_constructible<_Tp, const _Up&>...>::value)
+      : __base_(__t) {}
+
+  template <class... _Up,
+            class _Alloc,
+            __enable_if_t< _And< _EnableCtorFromUTypesTuple<const tuple<_Up...>&> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_Lazy<_And, is_convertible<const _Up&, _Tp>...> >::value)
+      tuple(allocator_arg_t, const _Alloc& __a, const tuple<_Up...>& __t)
+      : __base_(allocator_arg_t(), __a, __t) {}
+
+#  if _LIBCPP_STD_VER >= 23
+  // tuple(tuple<U...>&) constructors (including allocator_arg_t variants)
+
+  template <class... _Up, enable_if_t< _EnableCtorFromUTypesTuple<tuple<_Up...>&>::value>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_Lazy<_And, is_convertible<_Up&, _Tp>...>::value) tuple(tuple<_Up...>& __t)
+      : __base_(__t) {}
+
+  template <class _Alloc, class... _Up, enable_if_t< _EnableCtorFromUTypesTuple<tuple<_Up...>&>::value>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_Lazy<_And, is_convertible<_Up&, _Tp>...>::value)
+      tuple(allocator_arg_t, const _Alloc& __alloc, tuple<_Up...>& __t)
+      : __base_(allocator_arg_t(), __alloc, __t) {}
+#  endif // _LIBCPP_STD_VER >= 23
+
+  // tuple(tuple<U...>&&) constructors (including allocator_arg_t variants)
+  template <class... _Up, __enable_if_t< _And< _EnableCtorFromUTypesTuple<tuple<_Up...>&&> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_Lazy<_And, is_convertible<_Up, _Tp>...> >::value)
+      tuple(tuple<_Up...>&& __t) noexcept(_And<is_nothrow_constructible<_Tp, _Up>...>::value)
+      : __base_(std::move(__t)) {}
+
+  template <class _Alloc,
+            class... _Up,
+            __enable_if_t< _And< _EnableCtorFromUTypesTuple<tuple<_Up...>&&> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_Lazy<_And, is_convertible<_Up, _Tp>...> >::value)
+      tuple(allocator_arg_t, const _Alloc& __a, tuple<_Up...>&& __t)
+      : __base_(allocator_arg_t(), __a, std::move(__t)) {}
+
+#  if _LIBCPP_STD_VER >= 23
+  // tuple(const tuple<U...>&&) constructors (including allocator_arg_t variants)
+
+  template <class... _Up, enable_if_t< _EnableCtorFromUTypesTuple<const tuple<_Up...>&&>::value>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_Lazy<_And, is_convertible<const _Up&&, _Tp>...>::value)
+      tuple(const tuple<_Up...>&& __t)
+      : __base_(std::move(__t)) {}
+
+  template <class _Alloc,
+            class... _Up,
+            enable_if_t< _EnableCtorFromUTypesTuple<const tuple<_Up...>&&>::value>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_Lazy<_And, is_convertible<const _Up&&, _Tp>...>::value)
+      tuple(allocator_arg_t, const _Alloc& __alloc, const tuple<_Up...>&& __t)
+      : __base_(allocator_arg_t(), __alloc, std::move(__t)) {}
+#  endif // _LIBCPP_STD_VER >= 23
+
+  // tuple(const pair<U1, U2>&) constructors (including allocator_arg_t variants)
+
+  template <template <class...> class _Pred,
+            class _Pair,
+            class _DecayedPair = __remove_cvref_t<_Pair>,
+            class _Tuple       = tuple>
+  struct _CtorPredicateFromPair : false_type {};
+
+  template <template <class...> class _Pred, class _Pair, class _Up1, class _Up2, class _Tp1, class _Tp2>
+  struct _CtorPredicateFromPair<_Pred, _Pair, pair<_Up1, _Up2>, tuple<_Tp1, _Tp2> >
+      : _And< _Pred<_Tp1, __copy_cvref_t<_Pair, _Up1> >, _Pred<_Tp2, __copy_cvref_t<_Pair, _Up2> > > {};
+
+  template <class _Pair>
+  struct _EnableCtorFromPair : _CtorPredicateFromPair<is_constructible, _Pair> {};
+
+  template <class _Pair>
+  struct _NothrowConstructibleFromPair : _CtorPredicateFromPair<is_nothrow_constructible, _Pair> {};
+
+  template <class _Pair, class _DecayedPair = __remove_cvref_t<_Pair>, class _Tuple = tuple>
+  struct _BothImplicitlyConvertible : false_type {};
+
+  template <class _Pair, class _Up1, class _Up2, class _Tp1, class _Tp2>
+  struct _BothImplicitlyConvertible<_Pair, pair<_Up1, _Up2>, tuple<_Tp1, _Tp2> >
+      : _And< is_convertible<__copy_cvref_t<_Pair, _Up1>, _Tp1>, is_convertible<__copy_cvref_t<_Pair, _Up2>, _Tp2> > {};
+
+  template <class _Up1,
+            class _Up2,
+            template <class...> class _And                                                   = _And,
+            __enable_if_t< _And< _EnableCtorFromPair<const pair<_Up1, _Up2>&> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_BothImplicitlyConvertible<const pair<_Up1, _Up2>&> >::value)
+      tuple(const pair<_Up1, _Up2>& __p) noexcept(_NothrowConstructibleFromPair<const pair<_Up1, _Up2>&>::value)
+      : __base_(__p) {}
+
+  template <class _Alloc,
+            class _Up1,
+            class _Up2,
+            template <class...> class _And                                                   = _And,
+            __enable_if_t< _And< _EnableCtorFromPair<const pair<_Up1, _Up2>&> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_BothImplicitlyConvertible<const pair<_Up1, _Up2>&> >::value)
+      tuple(allocator_arg_t, const _Alloc& __a, const pair<_Up1, _Up2>& __p)
+      : __base_(allocator_arg_t(), __a, __p) {}
+
+#  if _LIBCPP_STD_VER >= 23
+  // tuple(pair<U1, U2>&) constructors (including allocator_arg_t variants)
+
+  template <class _U1, class _U2, enable_if_t< _EnableCtorFromPair<pair<_U1, _U2>&>::value>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_BothImplicitlyConvertible<pair<_U1, _U2>&>::value)
+      tuple(pair<_U1, _U2>& __p)
+      : __base_(__p) {}
+
+  template <class _Alloc,
+            class _U1,
+            class _U2,
+            enable_if_t< _EnableCtorFromPair<std::pair<_U1, _U2>&>::value>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_BothImplicitlyConvertible<pair<_U1, _U2>&>::value)
+      tuple(allocator_arg_t, const _Alloc& __alloc, pair<_U1, _U2>& __p)
+      : __base_(allocator_arg_t(), __alloc, __p) {}
+#  endif
+
+  // tuple(pair<U1, U2>&&) constructors (including allocator_arg_t variants)
+
+  template <class _Up1,
+            class _Up2,
+            template <class...> class _And                                              = _And,
+            __enable_if_t< _And< _EnableCtorFromPair<pair<_Up1, _Up2>&&> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_BothImplicitlyConvertible<pair<_Up1, _Up2>&&> >::value)
+      tuple(pair<_Up1, _Up2>&& __p) noexcept(_NothrowConstructibleFromPair<pair<_Up1, _Up2>&&>::value)
+      : __base_(std::move(__p)) {}
+
+  template <class _Alloc,
+            class _Up1,
+            class _Up2,
+            template <class...> class _And                                              = _And,
+            __enable_if_t< _And< _EnableCtorFromPair<pair<_Up1, _Up2>&&> >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_BothImplicitlyConvertible<pair<_Up1, _Up2>&&> >::value)
+      tuple(allocator_arg_t, const _Alloc& __a, pair<_Up1, _Up2>&& __p)
+      : __base_(allocator_arg_t(), __a, std::move(__p)) {}
+
+#  if _LIBCPP_STD_VER >= 23
+  // tuple(const pair<U1, U2>&&) constructors (including allocator_arg_t variants)
+
+  template <class _U1, class _U2, enable_if_t< _EnableCtorFromPair<const pair<_U1, _U2>&&>::value>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_BothImplicitlyConvertible<const pair<_U1, _U2>&&>::value)
+      tuple(const pair<_U1, _U2>&& __p)
+      : __base_(std::move(__p)) {}
+
+  template <class _Alloc,
+            class _U1,
+            class _U2,
+            enable_if_t< _EnableCtorFromPair<const pair<_U1, _U2>&&>::value>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_BothImplicitlyConvertible<const pair<_U1, _U2>&&>::value)
+      tuple(allocator_arg_t, const _Alloc& __alloc, const pair<_U1, _U2>&& __p)
+      : __base_(allocator_arg_t(), __alloc, std::move(__p)) {}
+#  endif // _LIBCPP_STD_VER >= 23
+
+  // [tuple.assign]
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
+  operator=(_If<_And<is_copy_assignable<_Tp>...>::value, tuple, __nat> const& __tuple) noexcept(
+      _And<is_nothrow_copy_assignable<_Tp>...>::value) {
+    std::__memberwise_copy_assign(*this, __tuple, typename __make_tuple_indices<sizeof...(_Tp)>::type());
+    return *this;
+  }
+
+#  if _LIBCPP_STD_VER >= 23
+  _LIBCPP_HIDE_FROM_ABI constexpr const tuple& operator=(tuple const& __tuple) const
+    requires(_And<is_copy_assignable<const _Tp>...>::value)
+  {
+    std::__memberwise_copy_assign(*this, __tuple, typename __make_tuple_indices<sizeof...(_Tp)>::type());
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const tuple& operator=(tuple&& __tuple) const
+    requires(_And<is_assignable<const _Tp&, _Tp>...>::value)
+  {
+    std::__memberwise_forward_assign(
+        *this, std::move(__tuple), __tuple_types<_Tp...>(), typename __make_tuple_indices<sizeof...(_Tp)>::type());
+    return *this;
+  }
+#  endif // _LIBCPP_STD_VER >= 23
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
+  operator=(_If<_And<is_move_assignable<_Tp>...>::value, tuple, __nat>&& __tuple) noexcept(
+      _And<is_nothrow_move_assignable<_Tp>...>::value) {
+    std::__memberwise_forward_assign(
+        *this, std::move(__tuple), __tuple_types<_Tp...>(), typename __make_tuple_indices<sizeof...(_Tp)>::type());
+    return *this;
+  }
+
+  template <
+      class... _Up,
+      __enable_if_t< _And< _BoolConstant<sizeof...(_Tp) == sizeof...(_Up)>, is_assignable<_Tp&, _Up const&>... >::value,
+                     int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
+  operator=(tuple<_Up...> const& __tuple) noexcept(_And<is_nothrow_assignable<_Tp&, _Up const&>...>::value) {
+    std::__memberwise_copy_assign(*this, __tuple, typename __make_tuple_indices<sizeof...(_Tp)>::type());
+    return *this;
+  }
+
+  template <class... _Up,
+            __enable_if_t< _And< _BoolConstant<sizeof...(_Tp) == sizeof...(_Up)>, is_assignable<_Tp&, _Up>... >::value,
+                           int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
+  operator=(tuple<_Up...>&& __tuple) noexcept(_And<is_nothrow_assignable<_Tp&, _Up>...>::value) {
+    std::__memberwise_forward_assign(
+        *this, std::move(__tuple), __tuple_types<_Up...>(), typename __make_tuple_indices<sizeof...(_Tp)>::type());
+    return *this;
+  }
+
+#  if _LIBCPP_STD_VER >= 23
+  template <class... _UTypes,
+            enable_if_t< _And<_BoolConstant<sizeof...(_Tp) == sizeof...(_UTypes)>,
+                              is_assignable<const _Tp&, const _UTypes&>...>::value>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI constexpr const tuple& operator=(const tuple<_UTypes...>& __u) const {
+    std::__memberwise_copy_assign(*this, __u, typename __make_tuple_indices<sizeof...(_Tp)>::type());
+    return *this;
+  }
+
+  template <class... _UTypes,
+            enable_if_t< _And<_BoolConstant<sizeof...(_Tp) == sizeof...(_UTypes)>,
+                              is_assignable<const _Tp&, _UTypes>...>::value>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI constexpr const tuple& operator=(tuple<_UTypes...>&& __u) const {
+    std::__memberwise_forward_assign(
+        *this, __u, __tuple_types<_UTypes...>(), typename __make_tuple_indices<sizeof...(_Tp)>::type());
+    return *this;
+  }
+#  endif // _LIBCPP_STD_VER >= 23
+
+  template <template <class...> class _Pred,
+            bool _Const,
+            class _Pair,
+            class _DecayedPair = __remove_cvref_t<_Pair>,
+            class _Tuple       = tuple>
+  struct _AssignPredicateFromPair : false_type {};
+
+  template <template <class...> class _Pred, bool _Const, class _Pair, class _Up1, class _Up2, class _Tp1, class _Tp2>
+  struct _AssignPredicateFromPair<_Pred, _Const, _Pair, pair<_Up1, _Up2>, tuple<_Tp1, _Tp2> >
+      : _And<_Pred<__maybe_const<_Const, _Tp1>&, __copy_cvref_t<_Pair, _Up1> >,
+             _Pred<__maybe_const<_Const, _Tp2>&, __copy_cvref_t<_Pair, _Up2> > > {};
+
+  template <bool _Const, class _Pair>
+  struct _EnableAssignFromPair : _AssignPredicateFromPair<is_assignable, _Const, _Pair> {};
+
+  template <bool _Const, class _Pair>
+  struct _NothrowAssignFromPair : _AssignPredicateFromPair<is_nothrow_assignable, _Const, _Pair> {};
+
+#  if _LIBCPP_STD_VER >= 23
+  template <class _U1, class _U2, enable_if_t< _EnableAssignFromPair<true, const pair<_U1, _U2>&>::value>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI constexpr const tuple& operator=(const pair<_U1, _U2>& __pair) const
+      noexcept(_NothrowAssignFromPair<true, const pair<_U1, _U2>&>::value) {
+    std::get<0>(*this) = __pair.first;
+    std::get<1>(*this) = __pair.second;
+    return *this;
+  }
+
+  template <class _U1, class _U2, enable_if_t< _EnableAssignFromPair<true, pair<_U1, _U2>&&>::value>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI constexpr const tuple& operator=(pair<_U1, _U2>&& __pair) const
+      noexcept(_NothrowAssignFromPair<true, pair<_U1, _U2>&&>::value) {
+    std::get<0>(*this) = std::move(__pair.first);
+    std::get<1>(*this) = std::move(__pair.second);
+    return *this;
+  }
+#  endif // _LIBCPP_STD_VER >= 23
+
+  template <class _Up1,
+            class _Up2,
+            __enable_if_t< _EnableAssignFromPair<false, pair<_Up1, _Up2> const&>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
+  operator=(pair<_Up1, _Up2> const& __pair) noexcept(_NothrowAssignFromPair<false, pair<_Up1, _Up2> const&>::value) {
+    std::get<0>(*this) = __pair.first;
+    std::get<1>(*this) = __pair.second;
+    return *this;
+  }
+
+  template <class _Up1, class _Up2, __enable_if_t< _EnableAssignFromPair<false, pair<_Up1, _Up2>&&>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
+  operator=(pair<_Up1, _Up2>&& __pair) noexcept(_NothrowAssignFromPair<false, pair<_Up1, _Up2>&&>::value) {
+    std::get<0>(*this) = std::forward<_Up1>(__pair.first);
+    std::get<1>(*this) = std::forward<_Up2>(__pair.second);
+    return *this;
+  }
+
+  // EXTENSION
+  template <
+      class _Up,
+      size_t _Np,
+      __enable_if_t< _And< _BoolConstant<_Np == sizeof...(_Tp)>, is_assignable<_Tp&, _Up const&>... >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
+  operator=(array<_Up, _Np> const& __array) noexcept(_And<is_nothrow_assignable<_Tp&, _Up const&>...>::value) {
+    std::__memberwise_copy_assign(*this, __array, typename __make_tuple_indices<sizeof...(_Tp)>::type());
+    return *this;
+  }
+
+  // EXTENSION
+  template <class _Up,
+            size_t _Np,
+            class = void,
+            __enable_if_t< _And< _BoolConstant<_Np == sizeof...(_Tp)>, is_assignable<_Tp&, _Up>... >::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
+  operator=(array<_Up, _Np>&& __array) noexcept(_And<is_nothrow_assignable<_Tp&, _Up>...>::value) {
+    std::__memberwise_forward_assign(
+        *this,
+        std::move(__array),
+        __tuple_types<_If<true, _Up, _Tp>...>(),
+        typename __make_tuple_indices<sizeof...(_Tp)>::type());
+    return *this;
+  }
+
+  // [tuple.swap]
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+  swap(tuple& __t) noexcept(__all<__is_nothrow_swappable_v<_Tp>...>::value) {
+    __base_.swap(__t.__base_);
+  }
+
+#  if _LIBCPP_STD_VER >= 23
+  _LIBCPP_HIDE_FROM_ABI constexpr void swap(const tuple& __t) const
+      noexcept(__all<is_nothrow_swappable_v<const _Tp&>...>::value) {
+    __base_.swap(__t.__base_);
+  }
+#  endif // _LIBCPP_STD_VER >= 23
+};
+
+template <>
+class _LIBCPP_TEMPLATE_VIS tuple<> {
+public:
+  constexpr tuple() _NOEXCEPT = default;
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple(allocator_arg_t, const _Alloc&) _NOEXCEPT {}
+  template <class _Alloc>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple(allocator_arg_t, const _Alloc&, const tuple&) _NOEXCEPT {}
+  template <class _Up>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple(array<_Up, 0>) _NOEXCEPT {}
+  template <class _Alloc, class _Up>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple(allocator_arg_t, const _Alloc&, array<_Up, 0>) _NOEXCEPT {}
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(tuple&) _NOEXCEPT {}
+#  if _LIBCPP_STD_VER >= 23
+  _LIBCPP_HIDE_FROM_ABI constexpr void swap(const tuple&) const noexcept {}
+#  endif
+};
+
+#  if _LIBCPP_STD_VER >= 23
+template <class... _TTypes, class... _UTypes, template <class> class _TQual, template <class> class _UQual>
+  requires requires { typename tuple<common_reference_t<_TQual<_TTypes>, _UQual<_UTypes>>...>; }
+struct basic_common_reference<tuple<_TTypes...>, tuple<_UTypes...>, _TQual, _UQual> {
+  using type = tuple<common_reference_t<_TQual<_TTypes>, _UQual<_UTypes>>...>;
+};
+
+template <class... _TTypes, class... _UTypes>
+  requires requires { typename tuple<common_type_t<_TTypes, _UTypes>...>; }
+struct common_type<tuple<_TTypes...>, tuple<_UTypes...>> {
+  using type = tuple<common_type_t<_TTypes, _UTypes>...>;
+};
+#  endif // _LIBCPP_STD_VER >= 23
+
+#  if _LIBCPP_STD_VER >= 17
+template <class... _Tp>
+tuple(_Tp...) -> tuple<_Tp...>;
+template <class _Tp1, class _Tp2>
+tuple(pair<_Tp1, _Tp2>) -> tuple<_Tp1, _Tp2>;
+template <class _Alloc, class... _Tp>
+tuple(allocator_arg_t, _Alloc, _Tp...) -> tuple<_Tp...>;
+template <class _Alloc, class _Tp1, class _Tp2>
+tuple(allocator_arg_t, _Alloc, pair<_Tp1, _Tp2>) -> tuple<_Tp1, _Tp2>;
+template <class _Alloc, class... _Tp>
+tuple(allocator_arg_t, _Alloc, tuple<_Tp...>) -> tuple<_Tp...>;
+#  endif
+
+template <class... _Tp, __enable_if_t<__all<__is_swappable_v<_Tp>...>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+swap(tuple<_Tp...>& __t, tuple<_Tp...>& __u) noexcept(__all<__is_nothrow_swappable_v<_Tp>...>::value) {
+  __t.swap(__u);
+}
+
+#  if _LIBCPP_STD_VER >= 23
+template <class... _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<__all<is_swappable_v<const _Tp>...>::value, void>
+swap(const tuple<_Tp...>& __lhs,
+     const tuple<_Tp...>& __rhs) noexcept(__all<is_nothrow_swappable_v<const _Tp>...>::value) {
+  __lhs.swap(__rhs);
+}
+#  endif
+
+// get
+
+template <size_t _Ip, class... _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, tuple<_Tp...> >::type&
+get(tuple<_Tp...>& __t) _NOEXCEPT {
+  typedef _LIBCPP_NODEBUG typename tuple_element<_Ip, tuple<_Tp...> >::type type;
+  return static_cast<__tuple_leaf<_Ip, type>&>(__t.__base_).get();
+}
+
+template <size_t _Ip, class... _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, tuple<_Tp...> >::type&
+get(const tuple<_Tp...>& __t) _NOEXCEPT {
+  typedef _LIBCPP_NODEBUG typename tuple_element<_Ip, tuple<_Tp...> >::type type;
+  return static_cast<const __tuple_leaf<_Ip, type>&>(__t.__base_).get();
+}
+
+template <size_t _Ip, class... _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, tuple<_Tp...> >::type&&
+get(tuple<_Tp...>&& __t) _NOEXCEPT {
+  typedef _LIBCPP_NODEBUG typename tuple_element<_Ip, tuple<_Tp...> >::type type;
+  return static_cast<type&&>(static_cast<__tuple_leaf<_Ip, type>&&>(__t.__base_).get());
+}
+
+template <size_t _Ip, class... _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, tuple<_Tp...> >::type&&
+get(const tuple<_Tp...>&& __t) _NOEXCEPT {
+  typedef _LIBCPP_NODEBUG typename tuple_element<_Ip, tuple<_Tp...> >::type type;
+  return static_cast<const type&&>(static_cast<const __tuple_leaf<_Ip, type>&&>(__t.__base_).get());
+}
+
+#  if _LIBCPP_STD_VER >= 14
+
+template <class _T1, class... _Args>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _T1& get(tuple<_Args...>& __tup) noexcept {
+  return std::get<__find_exactly_one_t<_T1, _Args...>::value>(__tup);
+}
+
+template <class _T1, class... _Args>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const& get(tuple<_Args...> const& __tup) noexcept {
+  return std::get<__find_exactly_one_t<_T1, _Args...>::value>(__tup);
+}
+
+template <class _T1, class... _Args>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _T1&& get(tuple<_Args...>&& __tup) noexcept {
+  return std::get<__find_exactly_one_t<_T1, _Args...>::value>(std::move(__tup));
+}
+
+template <class _T1, class... _Args>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const&& get(tuple<_Args...> const&& __tup) noexcept {
+  return std::get<__find_exactly_one_t<_T1, _Args...>::value>(std::move(__tup));
+}
+
+#  endif
+
+// tie
+
+template <class... _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 tuple<_Tp&...> tie(_Tp&... __t) _NOEXCEPT {
+  return tuple<_Tp&...>(__t...);
+}
+
+template <class... _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 tuple<typename __unwrap_ref_decay<_Tp>::type...>
+make_tuple(_Tp&&... __t) {
+  return tuple<typename __unwrap_ref_decay<_Tp>::type...>(std::forward<_Tp>(__t)...);
+}
+
+template <class... _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 tuple<_Tp&&...> forward_as_tuple(_Tp&&... __t) _NOEXCEPT {
+  return tuple<_Tp&&...>(std::forward<_Tp>(__t)...);
+}
+
+template <size_t _Ip>
+struct __tuple_equal {
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator()(const _Tp& __x, const _Up& __y) {
+    return __tuple_equal<_Ip - 1>()(__x, __y) && std::get<_Ip - 1>(__x) == std::get<_Ip - 1>(__y);
+  }
+};
+
+template <>
+struct __tuple_equal<0> {
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator()(const _Tp&, const _Up&) {
+    return true;
+  }
+};
+
+template <class... _Tp, class... _Up>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator==(const tuple<_Tp...>& __x, const tuple<_Up...>& __y) {
+  static_assert(sizeof...(_Tp) == sizeof...(_Up), "Can't compare tuples of different sizes");
+  return __tuple_equal<sizeof...(_Tp)>()(__x, __y);
+}
+
+#  if _LIBCPP_STD_VER >= 20
+
+// operator<=>
+
+template <class... _Tp, class... _Up, size_t... _Is>
+_LIBCPP_HIDE_FROM_ABI constexpr auto
+__tuple_compare_three_way(const tuple<_Tp...>& __x, const tuple<_Up...>& __y, index_sequence<_Is...>) {
+  common_comparison_category_t<__synth_three_way_result<_Tp, _Up>...> __result = strong_ordering::equal;
+  static_cast<void>(
+      ((__result = std::__synth_three_way(std::get<_Is>(__x), std::get<_Is>(__y)), __result != 0) || ...));
+  return __result;
+}
+
+template <class... _Tp, class... _Up>
+  requires(sizeof...(_Tp) == sizeof...(_Up))
+_LIBCPP_HIDE_FROM_ABI constexpr common_comparison_category_t<__synth_three_way_result<_Tp, _Up>...>
+operator<=>(const tuple<_Tp...>& __x, const tuple<_Up...>& __y) {
+  return std::__tuple_compare_three_way(__x, __y, index_sequence_for<_Tp...>{});
+}
+
+#  else // _LIBCPP_STD_VER >= 20
+
+template <class... _Tp, class... _Up>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator!=(const tuple<_Tp...>& __x, const tuple<_Up...>& __y) {
+  return !(__x == __y);
+}
+
+template <size_t _Ip>
+struct __tuple_less {
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator()(const _Tp& __x, const _Up& __y) {
+    const size_t __idx = tuple_size<_Tp>::value - _Ip;
+    if (std::get<__idx>(__x) < std::get<__idx>(__y))
+      return true;
+    if (std::get<__idx>(__y) < std::get<__idx>(__x))
+      return false;
+    return __tuple_less<_Ip - 1>()(__x, __y);
+  }
+};
+
+template <>
+struct __tuple_less<0> {
+  template <class _Tp, class _Up>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator()(const _Tp&, const _Up&) {
+    return false;
+  }
+};
+
+template <class... _Tp, class... _Up>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator<(const tuple<_Tp...>& __x, const tuple<_Up...>& __y) {
+  static_assert(sizeof...(_Tp) == sizeof...(_Up), "Can't compare tuples of different sizes");
+  return __tuple_less<sizeof...(_Tp)>()(__x, __y);
+}
+
+template <class... _Tp, class... _Up>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator>(const tuple<_Tp...>& __x, const tuple<_Up...>& __y) {
+  return __y < __x;
+}
+
+template <class... _Tp, class... _Up>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator>=(const tuple<_Tp...>& __x, const tuple<_Up...>& __y) {
+  return !(__x < __y);
+}
+
+template <class... _Tp, class... _Up>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+operator<=(const tuple<_Tp...>& __x, const tuple<_Up...>& __y) {
+  return !(__y < __x);
+}
+
+#  endif // _LIBCPP_STD_VER >= 20
+
+// tuple_cat
+
+template <class _Tp, class _Up>
+struct __tuple_cat_type;
+
+template <class... _Ttypes, class... _Utypes>
+struct __tuple_cat_type<tuple<_Ttypes...>, __tuple_types<_Utypes...> > {
+  typedef _LIBCPP_NODEBUG tuple<_Ttypes..., _Utypes...> type;
+};
+
+template <class _ResultTuple, bool _Is_Tuple0TupleLike, class... _Tuples>
+struct __tuple_cat_return_1 {};
+
+template <class... _Types, class _Tuple0>
+struct __tuple_cat_return_1<tuple<_Types...>, true, _Tuple0> {
+  using type _LIBCPP_NODEBUG =
+      typename __tuple_cat_type< tuple<_Types...>,
+                                 typename __make_tuple_types<__remove_cvref_t<_Tuple0> >::type >::type;
+};
+
+template <class... _Types, class _Tuple0, class _Tuple1, class... _Tuples>
+struct __tuple_cat_return_1<tuple<_Types...>, true, _Tuple0, _Tuple1, _Tuples...>
+    : public __tuple_cat_return_1<
+          typename __tuple_cat_type< tuple<_Types...>,
+                                     typename __make_tuple_types<__remove_cvref_t<_Tuple0> >::type >::type,
+          __tuple_like_ext<__libcpp_remove_reference_t<_Tuple1> >::value,
+          _Tuple1,
+          _Tuples...> {};
+
+template <class... _Tuples>
+struct __tuple_cat_return;
+
+template <class _Tuple0, class... _Tuples>
+struct __tuple_cat_return<_Tuple0, _Tuples...>
+    : public __tuple_cat_return_1<tuple<>,
+                                  __tuple_like_ext<__libcpp_remove_reference_t<_Tuple0> >::value,
+                                  _Tuple0,
+                                  _Tuples...> {};
+
+template <>
+struct __tuple_cat_return<> {
+  typedef _LIBCPP_NODEBUG tuple<> type;
+};
+
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 tuple<> tuple_cat() { return tuple<>(); }
+
+template <class _Rp, class _Indices, class _Tuple0, class... _Tuples>
+struct __tuple_cat_return_ref_imp;
+
+template <class... _Types, size_t... _I0, class _Tuple0>
+struct __tuple_cat_return_ref_imp<tuple<_Types...>, __tuple_indices<_I0...>, _Tuple0> {
+  typedef _LIBCPP_NODEBUG __libcpp_remove_reference_t<_Tuple0> _T0;
+  typedef tuple<_Types..., __copy_cvref_t<_Tuple0, typename tuple_element<_I0, _T0>::type>&&...> type;
+};
+
+template <class... _Types, size_t... _I0, class _Tuple0, class _Tuple1, class... _Tuples>
+struct __tuple_cat_return_ref_imp<tuple<_Types...>, __tuple_indices<_I0...>, _Tuple0, _Tuple1, _Tuples...>
+    : public __tuple_cat_return_ref_imp<
+          tuple<_Types...,
+                __copy_cvref_t<_Tuple0, typename tuple_element<_I0, __libcpp_remove_reference_t<_Tuple0>>::type>&&...>,
+          typename __make_tuple_indices<tuple_size<__libcpp_remove_reference_t<_Tuple1> >::value>::type,
+          _Tuple1,
+          _Tuples...> {};
+
+template <class _Tuple0, class... _Tuples>
+struct __tuple_cat_return_ref
+    : public __tuple_cat_return_ref_imp<
+          tuple<>,
+          typename __make_tuple_indices< tuple_size<__libcpp_remove_reference_t<_Tuple0> >::value >::type,
+          _Tuple0,
+          _Tuples...> {};
+
+template <class _Types, class _I0, class _J0>
+struct __tuple_cat;
+
+template <class... _Types, size_t... _I0, size_t... _J0>
+struct __tuple_cat<tuple<_Types...>, __tuple_indices<_I0...>, __tuple_indices<_J0...> > {
+  template <class _Tuple0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+  typename __tuple_cat_return_ref<tuple<_Types...>&&, _Tuple0&&>::type
+  operator()(tuple<_Types...> __t, _Tuple0&& __t0) {
+    (void)__t; // avoid unused parameter warning on GCC when _I0 is empty
+    return std::forward_as_tuple(
+        std::forward<_Types>(std::get<_I0>(__t))..., std::get<_J0>(std::forward<_Tuple0>(__t0))...);
+  }
+
+  template <class _Tuple0, class _Tuple1, class... _Tuples>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+  typename __tuple_cat_return_ref<tuple<_Types...>&&, _Tuple0&&, _Tuple1&&, _Tuples&&...>::type
+  operator()(tuple<_Types...> __t, _Tuple0&& __t0, _Tuple1&& __t1, _Tuples&&... __tpls) {
+    (void)__t; // avoid unused parameter warning on GCC when _I0 is empty
+    typedef _LIBCPP_NODEBUG __libcpp_remove_reference_t<_Tuple0> _T0;
+    typedef _LIBCPP_NODEBUG __libcpp_remove_reference_t<_Tuple1> _T1;
+    return __tuple_cat<tuple<_Types..., __copy_cvref_t<_Tuple0, typename tuple_element<_J0, _T0>::type>&&...>,
+                       typename __make_tuple_indices<sizeof...(_Types) + tuple_size<_T0>::value>::type,
+                       typename __make_tuple_indices<tuple_size<_T1>::value>::type>()(
+        std::forward_as_tuple(
+            std::forward<_Types>(std::get<_I0>(__t))..., std::get<_J0>(std::forward<_Tuple0>(__t0))...),
+        std::forward<_Tuple1>(__t1),
+        std::forward<_Tuples>(__tpls)...);
+  }
+};
+
+template <class _Tuple0, class... _Tuples>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename __tuple_cat_return<_Tuple0, _Tuples...>::type
+tuple_cat(_Tuple0&& __t0, _Tuples&&... __tpls) {
+  typedef _LIBCPP_NODEBUG __libcpp_remove_reference_t<_Tuple0> _T0;
+  return __tuple_cat<tuple<>, __tuple_indices<>, typename __make_tuple_indices<tuple_size<_T0>::value>::type>()(
+      tuple<>(), std::forward<_Tuple0>(__t0), std::forward<_Tuples>(__tpls)...);
+}
+
+template <class... _Tp, class _Alloc>
+struct _LIBCPP_TEMPLATE_VIS uses_allocator<tuple<_Tp...>, _Alloc> : true_type {};
+
+#  if _LIBCPP_STD_VER >= 17
+#    define _LIBCPP_NOEXCEPT_RETURN(...)                                                                               \
+      noexcept(noexcept(__VA_ARGS__)) { return __VA_ARGS__; }
+
+// The _LIBCPP_NOEXCEPT_RETURN macro breaks formatting.
+// clang-format off
+template <class _Fn, class _Tuple, size_t... _Id>
+inline _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto)
+__apply_tuple_impl(_Fn&& __f, _Tuple&& __t, __tuple_indices<_Id...>)
+    _LIBCPP_NOEXCEPT_RETURN(std::__invoke(std::forward<_Fn>(__f), std::get<_Id>(std::forward<_Tuple>(__t))...))
+
+template <class _Fn, class _Tuple>
+inline _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) apply(_Fn&& __f, _Tuple&& __t)
+    _LIBCPP_NOEXCEPT_RETURN(std::__apply_tuple_impl(
+        std::forward<_Fn>(__f),
+        std::forward<_Tuple>(__t),
+        typename __make_tuple_indices<tuple_size_v<remove_reference_t<_Tuple>>>::type{}))
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp, class _Tuple, size_t... _Idx>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp __make_from_tuple_impl(_Tuple&& __t, __tuple_indices<_Idx...>)
+  noexcept(noexcept(_Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...)))
+  requires is_constructible_v<_Tp, decltype(std::get<_Idx>(std::forward<_Tuple>(__t)))...> {
+  return _Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...);
+}
+#else
+template <class _Tp, class _Tuple, size_t... _Idx>
+inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp __make_from_tuple_impl(_Tuple&& __t, __tuple_indices<_Idx...>,
+    enable_if_t<is_constructible_v<_Tp, decltype(std::get<_Idx>(std::forward<_Tuple>(__t)))...>> * = nullptr)
+    _LIBCPP_NOEXCEPT_RETURN(_Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...))
+#endif // _LIBCPP_STD_VER >= 20
+
+template <class _Tp, class _Tuple,
+          class _Seq = typename __make_tuple_indices<tuple_size_v<remove_reference_t<_Tuple>>>::type, class = void>
+inline constexpr bool __can_make_from_tuple = false;
+
+template <class _Tp, class _Tuple, size_t... _Idx>
+inline constexpr bool __can_make_from_tuple<_Tp, _Tuple, __tuple_indices<_Idx...>,
+    enable_if_t<is_constructible_v<_Tp, decltype(std::get<_Idx>(std::declval<_Tuple>()))...>>> = true;
+
+// Based on LWG3528(https://wg21.link/LWG3528) and http://eel.is/c++draft/description#structure.requirements-9,
+// the standard allows to impose requirements, we constraint std::make_from_tuple to make std::make_from_tuple
+// SFINAE friendly and also avoid worse diagnostic messages. We still keep the constraints of std::__make_from_tuple_impl
+// so that std::__make_from_tuple_impl will have the same advantages when used alone.
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp, class _Tuple>
+  requires __can_make_from_tuple<_Tp, _Tuple> // strengthen
+#else
+template <class _Tp, class _Tuple, class = enable_if_t<__can_make_from_tuple<_Tp, _Tuple>>> // strengthen
+#endif // _LIBCPP_STD_VER >= 20
+inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp make_from_tuple(_Tuple&& __t)
+    _LIBCPP_NOEXCEPT_RETURN(std::__make_from_tuple_impl<_Tp>(
+        std::forward<_Tuple>(__t), typename __make_tuple_indices<tuple_size_v<remove_reference_t<_Tuple>>>::type{}))
+#    undef _LIBCPP_NOEXCEPT_RETURN
+
+#  endif // _LIBCPP_STD_VER >= 17
+
+#endif // !defined(_LIBCPP_CXX03_LANG)
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+// clang-format on
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <exception>
+#  include <iosfwd>
+#  include <new>
+#  include <type_traits>
+#  include <typeinfo>
+#  include <utility>
+#endif
+
+#endif // _LIBCPP_TUPLE
diff --git a/libcxx/include/__cxx03/type_traits b/libcxx/include/__cxx03/type_traits
new file mode 100644
index 000000000000..7f231cd09df5
--- /dev/null
+++ b/libcxx/include/__cxx03/type_traits
@@ -0,0 +1,523 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_TYPE_TRAITS
+#define _LIBCPP_TYPE_TRAITS
+
+/*
+    type_traits synopsis
+
+namespace std
+{
+
+    // helper class:
+    template <class T, T v> struct integral_constant;
+    typedef integral_constant<bool, true>  true_type;   // C++11
+    typedef integral_constant<bool, false> false_type;  // C++11
+
+    template <bool B>                                   // C++14
+    using bool_constant = integral_constant<bool, B>;   // C++14
+    typedef bool_constant<true> true_type;              // C++14
+    typedef bool_constant<false> false_type;            // C++14
+
+    // helper traits
+    template <bool, class T = void> struct enable_if;
+    template <bool, class T, class F> struct conditional;
+
+    // Primary classification traits:
+    template <class T> struct is_void;
+    template <class T> struct is_null_pointer;  // C++14
+    template <class T> struct is_integral;
+    template <class T> struct is_floating_point;
+    template <class T> struct is_array;
+    template <class T> struct is_pointer;
+    template <class T> struct is_lvalue_reference;
+    template <class T> struct is_rvalue_reference;
+    template <class T> struct is_member_object_pointer;
+    template <class T> struct is_member_function_pointer;
+    template <class T> struct is_enum;
+    template <class T> struct is_union;
+    template <class T> struct is_class;
+    template <class T> struct is_function;
+
+    // Secondary classification traits:
+    template <class T> struct is_reference;
+    template <class T> struct is_arithmetic;
+    template <class T> struct is_fundamental;
+    template <class T> struct is_member_pointer;
+    template <class T> struct is_scoped_enum; // C++23
+    template <class T> struct is_scalar;
+    template <class T> struct is_object;
+    template <class T> struct is_compound;
+
+    // Const-volatile properties and transformations:
+    template <class T> struct is_const;
+    template <class T> struct is_volatile;
+    template <class T> struct remove_const;
+    template <class T> struct remove_volatile;
+    template <class T> struct remove_cv;
+    template <class T> struct add_const;
+    template <class T> struct add_volatile;
+    template <class T> struct add_cv;
+
+    // Reference transformations:
+    template <class T> struct remove_reference;
+    template <class T> struct add_lvalue_reference;
+    template <class T> struct add_rvalue_reference;
+
+    // Pointer transformations:
+    template <class T> struct remove_pointer;
+    template <class T> struct add_pointer;
+
+    template<class T> struct type_identity;                     // C++20
+    template<class T>
+      using type_identity_t = typename type_identity<T>::type;  // C++20
+
+    // Integral properties:
+    template <class T> struct is_signed;
+    template <class T> struct is_unsigned;
+    template <class T> struct make_signed;
+    template <class T> struct make_unsigned;
+
+    // Array properties and transformations:
+    template <class T> struct rank;
+    template <class T, unsigned I = 0> struct extent;
+    template <class T> struct remove_extent;
+    template <class T> struct remove_all_extents;
+
+    template <class T> struct is_bounded_array;                 // C++20
+    template <class T> struct is_unbounded_array;               // C++20
+
+    // Member introspection:
+    template <class T> struct is_pod;
+    template <class T> struct is_trivial;
+    template <class T> struct is_trivially_copyable;
+    template <class T> struct is_standard_layout;
+    template <class T> struct is_literal_type; // Deprecated in C++17; removed in C++20
+    template <class T> struct is_empty;
+    template <class T> struct is_polymorphic;
+    template <class T> struct is_abstract;
+    template <class T> struct is_final; // C++14
+    template <class T> struct is_aggregate; // C++17
+
+    template <class T, class... Args> struct is_constructible;
+    template <class T>                struct is_default_constructible;
+    template <class T>                struct is_copy_constructible;
+    template <class T>                struct is_move_constructible;
+    template <class T, class U>       struct is_assignable;
+    template <class T>                struct is_copy_assignable;
+    template <class T>                struct is_move_assignable;
+    template <class T, class U>       struct is_swappable_with;       // C++17
+    template <class T>                struct is_swappable;            // C++17
+    template <class T>                struct is_destructible;
+
+    template <class T, class... Args> struct is_trivially_constructible;
+    template <class T>                struct is_trivially_default_constructible;
+    template <class T>                struct is_trivially_copy_constructible;
+    template <class T>                struct is_trivially_move_constructible;
+    template <class T, class U>       struct is_trivially_assignable;
+    template <class T>                struct is_trivially_copy_assignable;
+    template <class T>                struct is_trivially_move_assignable;
+    template <class T>                struct is_trivially_destructible;
+
+    template <class T, class... Args> struct is_nothrow_constructible;
+    template <class T>                struct is_nothrow_default_constructible;
+    template <class T>                struct is_nothrow_copy_constructible;
+    template <class T>                struct is_nothrow_move_constructible;
+    template <class T, class U>       struct is_nothrow_assignable;
+    template <class T>                struct is_nothrow_copy_assignable;
+    template <class T>                struct is_nothrow_move_assignable;
+    template <class T, class U>       struct is_nothrow_swappable_with; // C++17
+    template <class T>                struct is_nothrow_swappable;      // C++17
+    template <class T>                struct is_nothrow_destructible;
+
+    template <class T> struct has_virtual_destructor;
+
+    template<class T> struct has_unique_object_representations;         // C++17
+
+    // Relationships between types:
+    template <class T, class U> struct is_same;
+    template <class Base, class Derived> struct is_base_of;
+
+    template <class From, class To> struct is_convertible;
+    template <typename From, typename To> struct is_nothrow_convertible;                  // C++20
+    template <typename From, typename To> inline constexpr bool is_nothrow_convertible_v; // C++20
+
+    template <class Fn, class... ArgTypes> struct is_invocable;
+    template <class R, class Fn, class... ArgTypes> struct is_invocable_r;
+
+    template <class Fn, class... ArgTypes> struct is_nothrow_invocable;
+    template <class R, class Fn, class... ArgTypes> struct is_nothrow_invocable_r;
+
+    // Alignment properties and transformations:
+    template <class T> struct alignment_of;
+    template <size_t Len, size_t Align = most_stringent_alignment_requirement>
+        struct aligned_storage;                                 // deprecated in C++23
+    template <size_t Len, class... Types> struct aligned_union; // deprecated in C++23
+    template <class T> struct remove_cvref; // C++20
+
+    template <class T> struct decay;
+    template <class... T> struct common_type;
+    template <class T> struct underlying_type;
+    template <class> class result_of; // undefined; deprecated in C++17; removed in C++20
+    template <class Fn, class... ArgTypes> class result_of<Fn(ArgTypes...)>; // deprecated in C++17; removed in C++20
+    template <class Fn, class... ArgTypes> struct invoke_result;  // C++17
+
+    // const-volatile modifications:
+    template <class T>
+      using remove_const_t    = typename remove_const<T>::type;  // C++14
+    template <class T>
+      using remove_volatile_t = typename remove_volatile<T>::type;  // C++14
+    template <class T>
+      using remove_cv_t       = typename remove_cv<T>::type;  // C++14
+    template <class T>
+      using add_const_t       = typename add_const<T>::type;  // C++14
+    template <class T>
+      using add_volatile_t    = typename add_volatile<T>::type;  // C++14
+    template <class T>
+      using add_cv_t          = typename add_cv<T>::type;  // C++14
+
+    // reference modifications:
+    template <class T>
+      using remove_reference_t     = typename remove_reference<T>::type;  // C++14
+    template <class T>
+      using add_lvalue_reference_t = typename add_lvalue_reference<T>::type;  // C++14
+    template <class T>
+      using add_rvalue_reference_t = typename add_rvalue_reference<T>::type;  // C++14
+
+    // sign modifications:
+    template <class T>
+      using make_signed_t   = typename make_signed<T>::type;  // C++14
+    template <class T>
+      using make_unsigned_t = typename make_unsigned<T>::type;  // C++14
+
+    // array modifications:
+    template <class T>
+      using remove_extent_t      = typename remove_extent<T>::type;  // C++14
+    template <class T>
+      using remove_all_extents_t = typename remove_all_extents<T>::type;  // C++14
+
+    template <class T>
+      inline constexpr bool is_bounded_array_v
+        = is_bounded_array<T>::value;                                     // C++20
+      inline constexpr bool is_unbounded_array_v
+        = is_unbounded_array<T>::value;                                   // C++20
+
+    // pointer modifications:
+    template <class T>
+      using remove_pointer_t = typename remove_pointer<T>::type;  // C++14
+    template <class T>
+      using add_pointer_t    = typename add_pointer<T>::type;  // C++14
+
+    // other transformations:
+    template <size_t Len, size_t Align=default-alignment>
+      using aligned_storage_t = typename aligned_storage<Len,Align>::type;  // C++14
+    template <size_t Len, class... Types>
+      using aligned_union_t   = typename aligned_union<Len,Types...>::type;  // C++14
+    template <class T>
+      using remove_cvref_t    = typename remove_cvref<T>::type;  // C++20
+    template <class T>
+      using decay_t           = typename decay<T>::type;  // C++14
+    template <bool b, class T=void>
+      using enable_if_t       = typename enable_if<b,T>::type;  // C++14
+    template <bool b, class T, class F>
+      using conditional_t     = typename conditional<b,T,F>::type;  // C++14
+    template <class... T>
+      using common_type_t     = typename common_type<T...>::type;  // C++14
+    template <class T>
+      using underlying_type_t = typename underlying_type<T>::type;  // C++14
+    template <class T>
+      using result_of_t       = typename result_of<T>::type;  // C++14; deprecated in C++17; removed in C++20
+    template <class Fn, class... ArgTypes>
+      using invoke_result_t   = typename invoke_result<Fn, ArgTypes...>::type;  // C++17
+
+    template <class...>
+      using void_t = void;   // C++17
+
+      // See C++14 20.10.4.1, primary type categories
+      template <class T> inline constexpr bool is_void_v
+        = is_void<T>::value;                                             // C++17
+      template <class T> inline constexpr bool is_null_pointer_v
+        = is_null_pointer<T>::value;                                     // C++17
+      template <class T> inline constexpr bool is_integral_v
+        = is_integral<T>::value;                                         // C++17
+      template <class T> inline constexpr bool is_floating_point_v
+        = is_floating_point<T>::value;                                   // C++17
+      template <class T> inline constexpr bool is_array_v
+        = is_array<T>::value;                                            // C++17
+      template <class T> inline constexpr bool is_pointer_v
+        = is_pointer<T>::value;                                          // C++17
+      template <class T> inline constexpr bool is_lvalue_reference_v
+        = is_lvalue_reference<T>::value;                                 // C++17
+      template <class T> inline constexpr bool is_rvalue_reference_v
+        = is_rvalue_reference<T>::value;                                 // C++17
+      template <class T> inline constexpr bool is_member_object_pointer_v
+        = is_member_object_pointer<T>::value;                            // C++17
+      template <class T> inline constexpr bool is_member_function_pointer_v
+        = is_member_function_pointer<T>::value;                          // C++17
+      template <class T> inline constexpr bool is_enum_v
+        = is_enum<T>::value;                                             // C++17
+      template <class T> inline constexpr bool is_union_v
+        = is_union<T>::value;                                            // C++17
+      template <class T> inline constexpr bool is_class_v
+        = is_class<T>::value;                                            // C++17
+      template <class T> inline constexpr bool is_function_v
+        = is_function<T>::value;                                         // C++17
+
+      // See C++14 20.10.4.2, composite type categories
+      template <class T> inline constexpr bool is_reference_v
+        = is_reference<T>::value;                                        // C++17
+      template <class T> inline constexpr bool is_arithmetic_v
+        = is_arithmetic<T>::value;                                       // C++17
+      template <class T> inline constexpr bool is_fundamental_v
+        = is_fundamental<T>::value;                                      // C++17
+      template <class T> inline constexpr bool is_object_v
+        = is_object<T>::value;                                           // C++17
+      template <class T> inline constexpr bool is_scalar_v
+        = is_scalar<T>::value;                                           // C++17
+      template <class T> inline constexpr bool is_compound_v
+        = is_compound<T>::value;                                         // C++17
+      template <class T> inline constexpr bool is_member_pointer_v
+        = is_member_pointer<T>::value;                                   // C++17
+      template <class T> inline constexpr bool is_scoped_enum_v
+        = is_scoped_enum<T>::value;                                      // C++23
+
+      // See C++14 20.10.4.3, type properties
+      template <class T> inline constexpr bool is_const_v
+        = is_const<T>::value;                                            // C++17
+      template <class T> inline constexpr bool is_volatile_v
+        = is_volatile<T>::value;                                         // C++17
+      template <class T> inline constexpr bool is_trivial_v
+        = is_trivial<T>::value;                                          // C++17
+      template <class T> inline constexpr bool is_trivially_copyable_v
+        = is_trivially_copyable<T>::value;                               // C++17
+      template <class T> inline constexpr bool is_standard_layout_v
+        = is_standard_layout<T>::value;                                  // C++17
+      template <class T> inline constexpr bool is_pod_v
+        = is_pod<T>::value;                                              // C++17
+      template <class T> inline constexpr bool is_literal_type_v
+        = is_literal_type<T>::value;                                     // C++17; deprecated in C++17; removed in C++20
+      template <class T> inline constexpr bool is_empty_v
+        = is_empty<T>::value;                                            // C++17
+      template <class T> inline constexpr bool is_polymorphic_v
+        = is_polymorphic<T>::value;                                      // C++17
+      template <class T> inline constexpr bool is_abstract_v
+        = is_abstract<T>::value;                                         // C++17
+      template <class T> inline constexpr bool is_final_v
+        = is_final<T>::value;                                            // C++17
+      template <class T> inline constexpr bool is_aggregate_v
+        = is_aggregate<T>::value;                                        // C++17
+      template <class T> inline constexpr bool is_signed_v
+        = is_signed<T>::value;                                           // C++17
+      template <class T> inline constexpr bool is_unsigned_v
+        = is_unsigned<T>::value;                                         // C++17
+      template <class T, class... Args> inline constexpr bool is_constructible_v
+        = is_constructible<T, Args...>::value;                           // C++17
+      template <class T> inline constexpr bool is_default_constructible_v
+        = is_default_constructible<T>::value;                            // C++17
+      template <class T> inline constexpr bool is_copy_constructible_v
+        = is_copy_constructible<T>::value;                               // C++17
+      template <class T> inline constexpr bool is_move_constructible_v
+        = is_move_constructible<T>::value;                               // C++17
+      template <class T, class U> inline constexpr bool is_assignable_v
+        = is_assignable<T, U>::value;                                    // C++17
+      template <class T> inline constexpr bool is_copy_assignable_v
+        = is_copy_assignable<T>::value;                                  // C++17
+      template <class T> inline constexpr bool is_move_assignable_v
+        = is_move_assignable<T>::value;                                  // C++17
+      template <class T, class U> inline constexpr bool is_swappable_with_v
+        = is_swappable_with<T, U>::value;                                // C++17
+      template <class T> inline constexpr bool is_swappable_v
+        = is_swappable<T>::value;                                        // C++17
+      template <class T> inline constexpr bool is_destructible_v
+        = is_destructible<T>::value;                                     // C++17
+      template <class T, class... Args> inline constexpr bool is_trivially_constructible_v
+        = is_trivially_constructible<T, Args...>::value;                 // C++17
+      template <class T> inline constexpr bool is_trivially_default_constructible_v
+        = is_trivially_default_constructible<T>::value;                  // C++17
+      template <class T> inline constexpr bool is_trivially_copy_constructible_v
+        = is_trivially_copy_constructible<T>::value;                     // C++17
+      template <class T> inline constexpr bool is_trivially_move_constructible_v
+        = is_trivially_move_constructible<T>::value;                     // C++17
+      template <class T, class U> inline constexpr bool is_trivially_assignable_v
+        = is_trivially_assignable<T, U>::value;                          // C++17
+      template <class T> inline constexpr bool is_trivially_copy_assignable_v
+        = is_trivially_copy_assignable<T>::value;                        // C++17
+      template <class T> inline constexpr bool is_trivially_move_assignable_v
+        = is_trivially_move_assignable<T>::value;                        // C++17
+      template <class T> inline constexpr bool is_trivially_destructible_v
+        = is_trivially_destructible<T>::value;                           // C++17
+      template <class T, class... Args> inline constexpr bool is_nothrow_constructible_v
+        = is_nothrow_constructible<T, Args...>::value;                   // C++17
+      template <class T> inline constexpr bool is_nothrow_default_constructible_v
+        = is_nothrow_default_constructible<T>::value;                    // C++17
+      template <class T> inline constexpr bool is_nothrow_copy_constructible_v
+        = is_nothrow_copy_constructible<T>::value;                       // C++17
+      template <class T> inline constexpr bool is_nothrow_move_constructible_v
+        = is_nothrow_move_constructible<T>::value;                       // C++17
+      template <class T, class U> inline constexpr bool is_nothrow_assignable_v
+        = is_nothrow_assignable<T, U>::value;                            // C++17
+      template <class T> inline constexpr bool is_nothrow_copy_assignable_v
+        = is_nothrow_copy_assignable<T>::value;                          // C++17
+      template <class T> inline constexpr bool is_nothrow_move_assignable_v
+        = is_nothrow_move_assignable<T>::value;                          // C++17
+      template <class T, class U> inline constexpr bool is_nothrow_swappable_with_v
+        = is_nothrow_swappable_with<T, U>::value;                       // C++17
+      template <class T> inline constexpr bool is_nothrow_swappable_v
+        = is_nothrow_swappable<T>::value;                               // C++17
+      template <class T> inline constexpr bool is_nothrow_destructible_v
+        = is_nothrow_destructible<T>::value;                             // C++17
+      template <class T> inline constexpr bool has_virtual_destructor_v
+        = has_virtual_destructor<T>::value;                              // C++17
+      template<class T> inline constexpr bool has_unique_object_representations_v // C++17
+        = has_unique_object_representations<T>::value;
+
+      // See C++14 20.10.5, type property queries
+      template <class T> inline constexpr size_t alignment_of_v
+        = alignment_of<T>::value;                                        // C++17
+      template <class T> inline constexpr size_t rank_v
+        = rank<T>::value;                                                // C++17
+      template <class T, unsigned I = 0> inline constexpr size_t extent_v
+        = extent<T, I>::value;                                           // C++17
+
+      // See C++14 20.10.6, type relations
+      template <class T, class U> inline constexpr bool is_same_v
+        = is_same<T, U>::value;                                          // C++17
+      template <class Base, class Derived> inline constexpr bool is_base_of_v
+        = is_base_of<Base, Derived>::value;                              // C++17
+      template <class From, class To> inline constexpr bool is_convertible_v
+        = is_convertible<From, To>::value;                               // C++17
+      template <class Fn, class... ArgTypes> inline constexpr bool is_invocable_v
+        = is_invocable<Fn, ArgTypes...>::value;                          // C++17
+      template <class R, class Fn, class... ArgTypes> inline constexpr bool is_invocable_r_v
+        = is_invocable_r<R, Fn, ArgTypes...>::value;                     // C++17
+      template <class Fn, class... ArgTypes> inline constexpr bool is_nothrow_invocable_v
+        = is_nothrow_invocable<Fn, ArgTypes...>::value;                  // C++17
+      template <class R, class Fn, class... ArgTypes> inline constexpr bool is_nothrow_invocable_r_v
+        = is_nothrow_invocable_r<R, Fn, ArgTypes...>::value;             // C++17
+
+      // [meta.logical], logical operator traits:
+      template<class... B> struct conjunction;                           // C++17
+      template<class... B>
+        inline constexpr bool conjunction_v = conjunction<B...>::value;  // C++17
+      template<class... B> struct disjunction;                           // C++17
+      template<class... B>
+        inline constexpr bool disjunction_v = disjunction<B...>::value;  // C++17
+      template<class B> struct negation;                                 // C++17
+      template<class B>
+        inline constexpr bool negation_v = negation<B>::value;           // C++17
+
+}
+
+*/
+
+#include <__config>
+#include <__fwd/functional.h> // This is https://llvm.org/PR56938
+#include <__type_traits/add_const.h>
+#include <__type_traits/add_cv.h>
+#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_pointer.h>
+#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/add_volatile.h>
+#include <__type_traits/aligned_storage.h>
+#include <__type_traits/aligned_union.h>
+#include <__type_traits/alignment_of.h>
+#include <__type_traits/common_type.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/extent.h>
+#include <__type_traits/has_virtual_destructor.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_abstract.h>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_base_of.h>
+#include <__type_traits/is_class.h>
+#include <__type_traits/is_compound.h>
+#include <__type_traits/is_const.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_destructible.h>
+#include <__type_traits/is_empty.h>
+#include <__type_traits/is_enum.h>
+#include <__type_traits/is_floating_point.h>
+#include <__type_traits/is_function.h>
+#include <__type_traits/is_fundamental.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_literal_type.h>
+#include <__type_traits/is_member_pointer.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_nothrow_destructible.h>
+#include <__type_traits/is_object.h>
+#include <__type_traits/is_pod.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/is_polymorphic.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_scalar.h>
+#include <__type_traits/is_signed.h>
+#include <__type_traits/is_standard_layout.h>
+#include <__type_traits/is_trivial.h>
+#include <__type_traits/is_trivially_assignable.h>
+#include <__type_traits/is_trivially_constructible.h>
+#include <__type_traits/is_trivially_copyable.h>
+#include <__type_traits/is_trivially_destructible.h>
+#include <__type_traits/is_union.h>
+#include <__type_traits/is_unsigned.h>
+#include <__type_traits/is_void.h>
+#include <__type_traits/is_volatile.h>
+#include <__type_traits/make_signed.h>
+#include <__type_traits/make_unsigned.h>
+#include <__type_traits/rank.h>
+#include <__type_traits/remove_all_extents.h>
+#include <__type_traits/remove_const.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/remove_extent.h>
+#include <__type_traits/remove_pointer.h>
+#include <__type_traits/remove_reference.h>
+#include <__type_traits/remove_volatile.h>
+#include <__type_traits/result_of.h>
+#include <__type_traits/underlying_type.h>
+
+#if _LIBCPP_STD_VER >= 14
+#  include <__type_traits/is_final.h>
+#  include <__type_traits/is_null_pointer.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+#  include <__type_traits/conjunction.h>
+#  include <__type_traits/disjunction.h>
+#  include <__type_traits/has_unique_object_representation.h>
+#  include <__type_traits/invoke.h>
+#  include <__type_traits/is_aggregate.h>
+#  include <__type_traits/is_swappable.h>
+#  include <__type_traits/negation.h>
+#  include <__type_traits/void_t.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__type_traits/common_reference.h>
+#  include <__type_traits/is_bounded_array.h>
+#  include <__type_traits/is_constant_evaluated.h>
+#  include <__type_traits/is_nothrow_convertible.h>
+#  include <__type_traits/is_unbounded_array.h>
+#  include <__type_traits/type_identity.h>
+#  include <__type_traits/unwrap_ref.h>
+#endif
+
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#endif // _LIBCPP_TYPE_TRAITS
diff --git a/libcxx/include/__cxx03/typeindex b/libcxx/include/__cxx03/typeindex
new file mode 100644
index 000000000000..6398aa40d616
--- /dev/null
+++ b/libcxx/include/__cxx03/typeindex
@@ -0,0 +1,106 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_TYPEINDEX
+#define _LIBCPP_TYPEINDEX
+
+/*
+
+    typeindex synopsis
+
+namespace std
+{
+
+class type_index
+{
+public:
+    type_index(const type_info& rhs) noexcept;
+
+    bool operator==(const type_index& rhs) const noexcept;
+    bool operator!=(const type_index& rhs) const noexcept; // removed in C++20
+    bool operator< (const type_index& rhs) const noexcept;
+    bool operator<=(const type_index& rhs) const noexcept;
+    bool operator> (const type_index& rhs) const noexcept;
+    bool operator>=(const type_index& rhs) const noexcept;
+    strong_ordering operator<=>(const type_index& rhs) const noexcept; // C++20
+
+    size_t hash_code() const noexcept;
+    const char* name() const noexcept;
+};
+
+template <>
+struct hash<type_index>
+    : public unary_function<type_index, size_t>
+{
+    size_t operator()(type_index index) const noexcept;
+};
+
+}  // std
+
+*/
+
+#include <__config>
+#include <__functional/unary_function.h>
+#include <typeinfo>
+#include <version>
+
+// standard-mandated includes
+#include <compare>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+class _LIBCPP_TEMPLATE_VIS type_index {
+  const type_info* __t_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI type_index(const type_info& __y) _NOEXCEPT : __t_(&__y) {}
+
+  _LIBCPP_HIDE_FROM_ABI bool operator==(const type_index& __y) const _NOEXCEPT { return *__t_ == *__y.__t_; }
+#if _LIBCPP_STD_VER <= 17
+  _LIBCPP_HIDE_FROM_ABI bool operator!=(const type_index& __y) const _NOEXCEPT { return *__t_ != *__y.__t_; }
+#endif
+  _LIBCPP_HIDE_FROM_ABI bool operator<(const type_index& __y) const _NOEXCEPT { return __t_->before(*__y.__t_); }
+  _LIBCPP_HIDE_FROM_ABI bool operator<=(const type_index& __y) const _NOEXCEPT { return !__y.__t_->before(*__t_); }
+  _LIBCPP_HIDE_FROM_ABI bool operator>(const type_index& __y) const _NOEXCEPT { return __y.__t_->before(*__t_); }
+  _LIBCPP_HIDE_FROM_ABI bool operator>=(const type_index& __y) const _NOEXCEPT { return !__t_->before(*__y.__t_); }
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI strong_ordering operator<=>(const type_index& __y) const noexcept {
+    if (*__t_ == *__y.__t_)
+      return strong_ordering::equal;
+    if (__t_->before(*__y.__t_))
+      return strong_ordering::less;
+    return strong_ordering::greater;
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI size_t hash_code() const _NOEXCEPT { return __t_->hash_code(); }
+  _LIBCPP_HIDE_FROM_ABI const char* name() const _NOEXCEPT { return __t_->name(); }
+};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS hash;
+
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<type_index> : public __unary_function<type_index, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(type_index __index) const _NOEXCEPT { return __index.hash_code(); }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <iosfwd>
+#  include <new>
+#  include <utility>
+#endif
+
+#endif // _LIBCPP_TYPEINDEX
diff --git a/libcxx/include/__cxx03/typeinfo b/libcxx/include/__cxx03/typeinfo
new file mode 100644
index 000000000000..2727cad02fa9
--- /dev/null
+++ b/libcxx/include/__cxx03/typeinfo
@@ -0,0 +1,389 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_TYPEINFO
+#define _LIBCPP_TYPEINFO
+
+/*
+
+    typeinfo synopsis
+
+namespace std {
+
+class type_info
+{
+public:
+    virtual ~type_info();
+
+    bool operator==(const type_info& rhs) const noexcept; // constexpr since C++23
+    bool operator!=(const type_info& rhs) const noexcept; // removed in C++20
+
+    bool before(const type_info& rhs) const noexcept;
+    size_t hash_code() const noexcept;
+    const char* name() const noexcept;
+
+    type_info(const type_info& rhs) = delete;
+    type_info& operator=(const type_info& rhs) = delete;
+};
+
+class bad_cast
+    : public exception
+{
+public:
+    bad_cast() noexcept;
+    bad_cast(const bad_cast&) noexcept;
+    bad_cast& operator=(const bad_cast&) noexcept;
+    virtual const char* what() const noexcept;
+};
+
+class bad_typeid
+    : public exception
+{
+public:
+    bad_typeid() noexcept;
+    bad_typeid(const bad_typeid&) noexcept;
+    bad_typeid& operator=(const bad_typeid&) noexcept;
+    virtual const char* what() const noexcept;
+};
+
+}  // std
+
+*/
+
+#include <__config>
+#include <__exception/exception.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__verbose_abort>
+#include <cstddef>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if defined(_LIBCPP_ABI_VCRUNTIME)
+#  include <vcruntime_typeinfo.h>
+#else
+
+namespace std // purposefully not using versioning namespace
+{
+
+#  if defined(_LIBCPP_ABI_MICROSOFT)
+
+class _LIBCPP_EXPORTED_FROM_ABI type_info {
+  type_info& operator=(const type_info&);
+  type_info(const type_info&);
+
+  mutable struct {
+    const char* __undecorated_name;
+    const char __decorated_name[1];
+  } __data;
+
+  int __compare(const type_info& __rhs) const _NOEXCEPT;
+
+public:
+  virtual ~type_info();
+
+  const char* name() const _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI bool before(const type_info& __arg) const _NOEXCEPT { return __compare(__arg) < 0; }
+
+  size_t hash_code() const _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool operator==(const type_info& __arg) const _NOEXCEPT {
+    // When evaluated in a constant expression, both type infos simply can't come
+    // from different translation units, so it is sufficient to compare their addresses.
+    if (__libcpp_is_constant_evaluated()) {
+      return this == &__arg;
+    }
+    return __compare(__arg) == 0;
+  }
+
+#    if _LIBCPP_STD_VER <= 17
+  _LIBCPP_HIDE_FROM_ABI bool operator!=(const type_info& __arg) const _NOEXCEPT { return !operator==(__arg); }
+#    endif
+};
+
+#  else // !defined(_LIBCPP_ABI_MICROSOFT)
+
+// ========================================================================== //
+//                           Implementations
+// ========================================================================== //
+// ------------------------------------------------------------------------- //
+//                               Unique
+//               (_LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION = 1)
+// ------------------------------------------------------------------------- //
+// This implementation of type_info assumes a unique copy of the RTTI for a
+// given type inside a program. This is a valid assumption when abiding to the
+// Itanium ABI (http://itanium-cxx-abi.github.io/cxx-abi/abi.html#vtable-components).
+// Under this assumption, we can always compare the addresses of the type names
+// to implement equality-comparison of type_infos instead of having to perform
+// a deep string comparison.
+// -------------------------------------------------------------------------- //
+//                             NonUnique
+//               (_LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION = 2)
+// -------------------------------------------------------------------------- //
+// This implementation of type_info does not assume there is always a unique
+// copy of the RTTI for a given type inside a program. For various reasons
+// the linker may have failed to merge every copy of a types RTTI
+// (For example: -Bsymbolic or llvm.org/PR37398). Under this assumption, two
+// type_infos are equal if their addresses are equal or if a deep string
+// comparison is equal.
+// -------------------------------------------------------------------------- //
+//                          NonUniqueARMRTTIBit
+//               (_LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION = 3)
+// -------------------------------------------------------------------------- //
+// This implementation is specific to ARM64 on Apple platforms.
+//
+// This implementation of type_info does not assume always a unique copy of
+// the RTTI for a given type inside a program. When constructing the type_info,
+// the compiler packs the pointer to the type name into a uintptr_t and reserves
+// the high bit of that pointer, which is assumed to be free for use under that
+// ABI. If that high bit is set, that specific copy of the RTTI can't be assumed
+// to be unique within the program. If the high bit is unset, then the RTTI can
+// be assumed to be unique within the program.
+//
+// When comparing type_infos, if both RTTIs can be assumed to be unique, it
+// suffices to compare their addresses. If both the RTTIs can't be assumed to
+// be unique, we must perform a deep string comparison of the type names.
+// However, if one of the RTTIs is guaranteed unique and the other one isn't,
+// then both RTTIs are necessarily not to be considered equal.
+//
+// The intent of this design is to remove the need for weak symbols. Specifically,
+// if a type would normally have a default-visibility RTTI emitted as a weak
+// symbol, it is given hidden visibility instead and the non-unique bit is set.
+// Otherwise, types declared with hidden visibility are always considered to have
+// a unique RTTI: the RTTI is emitted with linkonce_odr linkage and is assumed
+// to be deduplicated by the linker within the linked image. Across linked image
+// boundaries, such types are thus considered different types.
+
+// This value can be overriden in the __config_site. When it's not overriden,
+// we pick a default implementation based on the platform here.
+#    ifndef _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION
+
+// Windows and AIX binaries can't merge typeinfos, so use the NonUnique implementation.
+#      if defined(_LIBCPP_OBJECT_FORMAT_COFF) || defined(_LIBCPP_OBJECT_FORMAT_XCOFF)
+#        define _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION 2
+
+// On arm64 on Apple platforms, use the special NonUniqueARMRTTIBit implementation.
+#      elif defined(__APPLE__) && defined(__LP64__) && !defined(__x86_64__)
+#        define _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION 3
+
+// On all other platforms, assume the Itanium C++ ABI and use the Unique implementation.
+#      else
+#        define _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION 1
+#      endif
+#    endif
+
+struct __type_info_implementations {
+  struct __string_impl_base {
+    typedef const char* __type_name_t;
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE _LIBCPP_CONSTEXPR static const char*
+    __type_name_to_string(__type_name_t __v) _NOEXCEPT {
+      return __v;
+    }
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE _LIBCPP_CONSTEXPR static __type_name_t
+    __string_to_type_name(const char* __v) _NOEXCEPT {
+      return __v;
+    }
+  };
+
+  struct __unique_impl : __string_impl_base {
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static size_t __hash(__type_name_t __v) _NOEXCEPT {
+      return reinterpret_cast<size_t>(__v);
+    }
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __eq(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
+      return __lhs == __rhs;
+    }
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __lt(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
+      return __lhs < __rhs;
+    }
+  };
+
+  struct __non_unique_impl : __string_impl_base {
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static size_t __hash(__type_name_t __ptr) _NOEXCEPT {
+      size_t __hash = 5381;
+      while (unsigned char __c = static_cast<unsigned char>(*__ptr++))
+        __hash = (__hash * 33) ^ __c;
+      return __hash;
+    }
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __eq(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
+      return __lhs == __rhs || __builtin_strcmp(__lhs, __rhs) == 0;
+    }
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __lt(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
+      return __builtin_strcmp(__lhs, __rhs) < 0;
+    }
+  };
+
+  struct __non_unique_arm_rtti_bit_impl {
+    typedef uintptr_t __type_name_t;
+
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static const char* __type_name_to_string(__type_name_t __v) _NOEXCEPT {
+      return reinterpret_cast<const char*>(__v & ~__non_unique_rtti_bit::value);
+    }
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static __type_name_t __string_to_type_name(const char* __v) _NOEXCEPT {
+      return reinterpret_cast<__type_name_t>(__v);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static size_t __hash(__type_name_t __v) _NOEXCEPT {
+      if (__is_type_name_unique(__v))
+        return __v;
+      return __non_unique_impl::__hash(__type_name_to_string(__v));
+    }
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __eq(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
+      if (__lhs == __rhs)
+        return true;
+      if (__is_type_name_unique(__lhs) || __is_type_name_unique(__rhs))
+        // Either both are unique and have a different address, or one of them
+        // is unique and the other one isn't. In both cases they are unequal.
+        return false;
+      return __builtin_strcmp(__type_name_to_string(__lhs), __type_name_to_string(__rhs)) == 0;
+    }
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __lt(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
+      if (__is_type_name_unique(__lhs) || __is_type_name_unique(__rhs))
+        return __lhs < __rhs;
+      return __builtin_strcmp(__type_name_to_string(__lhs), __type_name_to_string(__rhs)) < 0;
+    }
+
+  private:
+    // The unique bit is the top bit. It is expected that __type_name_t is 64 bits when
+    // this implementation is actually used.
+    typedef integral_constant<__type_name_t, (1ULL << ((__CHAR_BIT__ * sizeof(__type_name_t)) - 1))>
+        __non_unique_rtti_bit;
+
+    _LIBCPP_HIDE_FROM_ABI static bool __is_type_name_unique(__type_name_t __lhs) _NOEXCEPT {
+      return !(__lhs & __non_unique_rtti_bit::value);
+    }
+  };
+
+  typedef
+#    if _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION == 1
+      __unique_impl
+#    elif _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION == 2
+      __non_unique_impl
+#    elif _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION == 3
+      __non_unique_arm_rtti_bit_impl
+#    else
+#      error invalid configuration for _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION
+#    endif
+          __impl;
+};
+
+#    if __has_cpp_attribute(_Clang::__ptrauth_vtable_pointer__)
+#      if __has_feature(ptrauth_type_info_vtable_pointer_discrimination)
+#        define _LIBCPP_TYPE_INFO_VTABLE_POINTER_AUTH                                                                  \
+          [[_Clang::__ptrauth_vtable_pointer__(process_independent, address_discrimination, type_discrimination)]]
+#      else
+#        define _LIBCPP_TYPE_INFO_VTABLE_POINTER_AUTH                                                                  \
+          [[_Clang::__ptrauth_vtable_pointer__(                                                                        \
+              process_independent, no_address_discrimination, no_extra_discrimination)]]
+#      endif
+#    else
+#      define _LIBCPP_TYPE_INFO_VTABLE_POINTER_AUTH
+#    endif
+
+class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_TYPE_INFO_VTABLE_POINTER_AUTH type_info {
+  type_info& operator=(const type_info&);
+  type_info(const type_info&);
+
+protected:
+  typedef __type_info_implementations::__impl __impl;
+
+  __impl::__type_name_t __type_name;
+
+  _LIBCPP_HIDE_FROM_ABI explicit type_info(const char* __n) : __type_name(__impl::__string_to_type_name(__n)) {}
+
+public:
+  virtual ~type_info();
+
+  _LIBCPP_HIDE_FROM_ABI const char* name() const _NOEXCEPT { return __impl::__type_name_to_string(__type_name); }
+
+  _LIBCPP_HIDE_FROM_ABI bool before(const type_info& __arg) const _NOEXCEPT {
+    return __impl::__lt(__type_name, __arg.__type_name);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_t hash_code() const _NOEXCEPT { return __impl::__hash(__type_name); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool operator==(const type_info& __arg) const _NOEXCEPT {
+    // When evaluated in a constant expression, both type infos simply can't come
+    // from different translation units, so it is sufficient to compare their addresses.
+    if (__libcpp_is_constant_evaluated()) {
+      return this == &__arg;
+    }
+    return __impl::__eq(__type_name, __arg.__type_name);
+  }
+
+#    if _LIBCPP_STD_VER <= 17
+  _LIBCPP_HIDE_FROM_ABI bool operator!=(const type_info& __arg) const _NOEXCEPT { return !operator==(__arg); }
+#    endif
+};
+#  endif // defined(_LIBCPP_ABI_MICROSOFT)
+
+class _LIBCPP_EXPORTED_FROM_ABI bad_cast : public exception {
+public:
+  bad_cast() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI bad_cast(const bad_cast&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI bad_cast& operator=(const bad_cast&) _NOEXCEPT = default;
+  ~bad_cast() _NOEXCEPT override;
+  const char* what() const _NOEXCEPT override;
+};
+
+class _LIBCPP_EXPORTED_FROM_ABI bad_typeid : public exception {
+public:
+  bad_typeid() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI bad_typeid(const bad_typeid&) _NOEXCEPT            = default;
+  _LIBCPP_HIDE_FROM_ABI bad_typeid& operator=(const bad_typeid&) _NOEXCEPT = default;
+  ~bad_typeid() _NOEXCEPT override;
+  const char* what() const _NOEXCEPT override;
+};
+
+} // namespace std
+
+#endif // defined(_LIBCPP_ABI_VCRUNTIME)
+
+#if defined(_LIBCPP_ABI_VCRUNTIME) && _HAS_EXCEPTIONS == 0
+
+namespace std {
+
+class bad_cast : public exception {
+public:
+  bad_cast() _NOEXCEPT : exception("bad cast") {}
+
+private:
+  bad_cast(const char* const __message) _NOEXCEPT : exception(__message) {}
+};
+
+class bad_typeid : public exception {
+public:
+  bad_typeid() _NOEXCEPT : exception("bad typeid") {}
+
+private:
+  bad_typeid(const char* const __message) _NOEXCEPT : exception(__message) {}
+};
+
+} // namespace std
+
+#endif // defined(_LIBCPP_ABI_VCRUNTIME) && _HAS_EXCEPTIONS == 0
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_bad_cast() {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw bad_cast();
+#else
+  _LIBCPP_VERBOSE_ABORT("bad_cast was thrown in -fno-exceptions mode");
+#endif
+}
+_LIBCPP_END_NAMESPACE_STD
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstdlib>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_TYPEINFO
diff --git a/libcxx/include/__cxx03/uchar.h b/libcxx/include/__cxx03/uchar.h
new file mode 100644
index 000000000000..07b78611406d
--- /dev/null
+++ b/libcxx/include/__cxx03/uchar.h
@@ -0,0 +1,56 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_UCHAR_H
+#define _LIBCPP_UCHAR_H
+
+/*
+    uchar.h synopsis // since C++11
+
+Macros:
+
+    __STDC_UTF_16__
+    __STDC_UTF_32__
+
+Types:
+
+  mbstate_t
+  size_t
+
+size_t mbrtoc8(char8_t* pc8, const char* s, size_t n, mbstate_t* ps);     // since C++20
+size_t c8rtomb(char* s, char8_t c8, mbstate_t* ps);                       // since C++20
+size_t mbrtoc16(char16_t* pc16, const char* s, size_t n, mbstate_t* ps);
+size_t c16rtomb(char* s, char16_t c16, mbstate_t* ps);
+size_t mbrtoc32(char32_t* pc32, const char* s, size_t n, mbstate_t* ps);
+size_t c32rtomb(char* s, char32_t c32, mbstate_t* ps);
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_CXX03_LANG)
+
+// Some platforms don't implement <uchar.h> and we don't want to give a hard
+// error on those platforms. When the platform doesn't provide <uchar.h>, at
+// least include <stddef.h> so we get the declaration for size_t, and try to
+// get the declaration of mbstate_t too.
+#  if __has_include_next(<uchar.h>)
+#    include_next <uchar.h>
+#  else
+#    include <__mbstate_t.h>
+#    include <stddef.h>
+#  endif
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP_UCHAR_H
diff --git a/libcxx/include/__cxx03/unordered_map b/libcxx/include/__cxx03/unordered_map
new file mode 100644
index 000000000000..7c31c4fce26b
--- /dev/null
+++ b/libcxx/include/__cxx03/unordered_map
@@ -0,0 +1,2541 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_UNORDERED_MAP
+#define _LIBCPP_UNORDERED_MAP
+
+/*
+
+    unordered_map synopsis
+
+#include <initializer_list>
+
+namespace std
+{
+
+template <class Key, class T, class Hash = hash<Key>, class Pred = equal_to<Key>,
+          class Alloc = allocator<pair<const Key, T>>>
+class unordered_map
+{
+public:
+    // types
+    typedef Key                                                        key_type;
+    typedef T                                                          mapped_type;
+    typedef Hash                                                       hasher;
+    typedef Pred                                                       key_equal;
+    typedef Alloc                                                      allocator_type;
+    typedef pair<const key_type, mapped_type>                          value_type;
+    typedef value_type&                                                reference;
+    typedef const value_type&                                          const_reference;
+    typedef typename allocator_traits<allocator_type>::pointer         pointer;
+    typedef typename allocator_traits<allocator_type>::const_pointer   const_pointer;
+    typedef typename allocator_traits<allocator_type>::size_type       size_type;
+    typedef typename allocator_traits<allocator_type>::difference_type difference_type;
+
+    typedef /unspecified/ iterator;
+    typedef /unspecified/ const_iterator;
+    typedef /unspecified/ local_iterator;
+    typedef /unspecified/ const_local_iterator;
+
+    typedef unspecified                             node_type;            // C++17
+    typedef INSERT_RETURN_TYPE<iterator, node_type> insert_return_type;   // C++17
+
+    unordered_map()
+        noexcept(
+            is_nothrow_default_constructible<hasher>::value &&
+            is_nothrow_default_constructible<key_equal>::value &&
+            is_nothrow_default_constructible<allocator_type>::value);
+    explicit unordered_map(size_type n, const hasher& hf = hasher(),
+                           const key_equal& eql = key_equal(),
+                           const allocator_type& a = allocator_type());
+    template <class InputIterator>
+        unordered_map(InputIterator f, InputIterator l,
+                      size_type n = 0, const hasher& hf = hasher(),
+                      const key_equal& eql = key_equal(),
+                      const allocator_type& a = allocator_type());
+    template<container-compatible-range<value_type> R>
+      unordered_map(from_range_t, R&& rg, size_type n = see below,
+        const hasher& hf = hasher(), const key_equal& eql = key_equal(),
+        const allocator_type& a = allocator_type()); // C++23
+
+    explicit unordered_map(const allocator_type&);
+    unordered_map(const unordered_map&);
+    unordered_map(const unordered_map&, const Allocator&);
+    unordered_map(unordered_map&&)
+        noexcept(
+            is_nothrow_move_constructible<hasher>::value &&
+            is_nothrow_move_constructible<key_equal>::value &&
+            is_nothrow_move_constructible<allocator_type>::value);
+    unordered_map(unordered_map&&, const Allocator&);
+    unordered_map(initializer_list<value_type>, size_type n = 0,
+                  const hasher& hf = hasher(), const key_equal& eql = key_equal(),
+                  const allocator_type& a = allocator_type());
+    unordered_map(size_type n, const allocator_type& a)
+      : unordered_map(n, hasher(), key_equal(), a) {}  // C++14
+    unordered_map(size_type n, const hasher& hf, const allocator_type& a)
+      : unordered_map(n, hf, key_equal(), a) {}  // C++14
+    template <class InputIterator>
+      unordered_map(InputIterator f, InputIterator l, size_type n, const allocator_type& a)
+      : unordered_map(f, l, n, hasher(), key_equal(), a) {}  // C++14
+    template <class InputIterator>
+      unordered_map(InputIterator f, InputIterator l, size_type n, const hasher& hf,
+        const allocator_type& a)
+      : unordered_map(f, l, n, hf, key_equal(), a) {}  // C++14
+    template<container-compatible-range<value_type> R>
+      unordered_map(from_range_t, R&& rg, size_type n, const allocator_type& a)
+        : unordered_map(from_range, std::forward<R>(rg), n, hasher(), key_equal(), a) { } // C++23
+    template<container-compatible-range<value_type> R>
+      unordered_map(from_range_t, R&& rg, size_type n, const hasher& hf, const allocator_type& a)
+        : unordered_map(from_range, std::forward<R>(rg), n, hf, key_equal(), a) { }       // C++23
+    unordered_map(initializer_list<value_type> il, size_type n, const allocator_type& a)
+      : unordered_map(il, n, hasher(), key_equal(), a) {}  // C++14
+    unordered_map(initializer_list<value_type> il, size_type n, const hasher& hf,
+      const allocator_type& a)
+      : unordered_map(il, n, hf, key_equal(), a) {}  // C++14
+    ~unordered_map();
+    unordered_map& operator=(const unordered_map&);
+    unordered_map& operator=(unordered_map&&)
+        noexcept(
+            allocator_type::propagate_on_container_move_assignment::value &&
+            is_nothrow_move_assignable<allocator_type>::value &&
+            is_nothrow_move_assignable<hasher>::value &&
+            is_nothrow_move_assignable<key_equal>::value);
+    unordered_map& operator=(initializer_list<value_type>);
+
+    allocator_type get_allocator() const noexcept;
+
+    bool      empty() const noexcept;
+    size_type size() const noexcept;
+    size_type max_size() const noexcept;
+
+    iterator       begin() noexcept;
+    iterator       end() noexcept;
+    const_iterator begin()  const noexcept;
+    const_iterator end()    const noexcept;
+    const_iterator cbegin() const noexcept;
+    const_iterator cend()   const noexcept;
+
+    template <class... Args>
+        pair<iterator, bool> emplace(Args&&... args);
+    template <class... Args>
+        iterator emplace_hint(const_iterator position, Args&&... args);
+    pair<iterator, bool> insert(const value_type& obj);
+    template <class P>
+        pair<iterator, bool> insert(P&& obj);
+    iterator insert(const_iterator hint, const value_type& obj);
+    template <class P>
+        iterator insert(const_iterator hint, P&& obj);
+    template <class InputIterator>
+        void insert(InputIterator first, InputIterator last);
+    template<container-compatible-range<value_type> R>
+      void insert_range(R&& rg);                                                      // C++23
+    void insert(initializer_list<value_type>);
+
+    node_type extract(const_iterator position);                                       // C++17
+    node_type extract(const key_type& x);                                             // C++17
+    insert_return_type insert(node_type&& nh);                                        // C++17
+    iterator           insert(const_iterator hint, node_type&& nh);                   // C++17
+
+    template <class... Args>
+        pair<iterator, bool> try_emplace(const key_type& k, Args&&... args);          // C++17
+    template <class... Args>
+        pair<iterator, bool> try_emplace(key_type&& k, Args&&... args);               // C++17
+    template <class... Args>
+        iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args); // C++17
+    template <class... Args>
+        iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args);      // C++17
+    template <class M>
+        pair<iterator, bool> insert_or_assign(const key_type& k, M&& obj);            // C++17
+    template <class M>
+        pair<iterator, bool> insert_or_assign(key_type&& k, M&& obj);                 // C++17
+    template <class M>
+        iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj);   // C++17
+    template <class M>
+        iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj);        // C++17
+
+    iterator erase(const_iterator position);
+    iterator erase(iterator position);  // C++14
+    size_type erase(const key_type& k);
+    iterator erase(const_iterator first, const_iterator last);
+    void clear() noexcept;
+
+    template<class H2, class P2>
+      void merge(unordered_map<Key, T, H2, P2, Allocator>& source);         // C++17
+    template<class H2, class P2>
+      void merge(unordered_map<Key, T, H2, P2, Allocator>&& source);        // C++17
+    template<class H2, class P2>
+      void merge(unordered_multimap<Key, T, H2, P2, Allocator>& source);    // C++17
+    template<class H2, class P2>
+      void merge(unordered_multimap<Key, T, H2, P2, Allocator>&& source);   // C++17
+
+    void swap(unordered_map&)
+        noexcept(
+            (!allocator_type::propagate_on_container_swap::value ||
+             __is_nothrow_swappable<allocator_type>::value) &&
+            __is_nothrow_swappable<hasher>::value &&
+            __is_nothrow_swappable<key_equal>::value);
+
+    hasher hash_function() const;
+    key_equal key_eq() const;
+
+    iterator       find(const key_type& k);
+    const_iterator find(const key_type& k) const;
+    template<typename K>
+        iterator find(const K& x);              // C++20
+    template<typename K>
+        const_iterator find(const K& x) const;  // C++20
+    size_type count(const key_type& k) const;
+    template<typename K>
+        size_type count(const K& k) const; // C++20
+    bool contains(const key_type& k) const; // C++20
+    template<typename K>
+        bool contains(const K& k) const; // C++20
+    pair<iterator, iterator>             equal_range(const key_type& k);
+    pair<const_iterator, const_iterator> equal_range(const key_type& k) const;
+    template<typename K>
+        pair<iterator, iterator>             equal_range(const K& k); // C++20
+    template<typename K>
+        pair<const_iterator, const_iterator> equal_range(const K& k) const; // C++20
+
+    mapped_type& operator[](const key_type& k);
+    mapped_type& operator[](key_type&& k);
+
+    mapped_type&       at(const key_type& k);
+    const mapped_type& at(const key_type& k) const;
+
+    size_type bucket_count() const noexcept;
+    size_type max_bucket_count() const noexcept;
+
+    size_type bucket_size(size_type n) const;
+    size_type bucket(const key_type& k) const;
+
+    local_iterator       begin(size_type n);
+    local_iterator       end(size_type n);
+    const_local_iterator begin(size_type n) const;
+    const_local_iterator end(size_type n) const;
+    const_local_iterator cbegin(size_type n) const;
+    const_local_iterator cend(size_type n) const;
+
+    float load_factor() const noexcept;
+    float max_load_factor() const noexcept;
+    void max_load_factor(float z);
+    void rehash(size_type n);
+    void reserve(size_type n);
+};
+
+template<class InputIterator,
+    class Hash = hash<iter_key_t<InputIterator>>, class Pred = equal_to<iter_key_t<InputIterator>>,
+    class Allocator = allocator<iter_to_alloc_t<InputIterator>>>
+unordered_map(InputIterator, InputIterator, typename see below::size_type = see below,
+    Hash = Hash(), Pred = Pred(), Allocator = Allocator())
+  -> unordered_map<iter_key_t<InputIterator>, iter_value_t<InputIterator>, Hash, Pred,
+    Allocator>; // C++17
+
+template<ranges::input_range R, class Hash = hash<range-key-type<R>>,
+         class Pred = equal_to<range-key-type<R>>,
+         class Allocator = allocator<range-to-alloc-type<R>>>
+  unordered_map(from_range_t, R&&, typename see below::size_type = see below,
+                Hash = Hash(), Pred = Pred(), Allocator = Allocator())
+    -> unordered_map<range-key-type<R>, range-mapped-type<R>, Hash, Pred, Allocator>; // C++23
+
+template<class Key, class T, class Hash = hash<Key>,
+    class Pred = equal_to<Key>, class Allocator = allocator<pair<const Key, T>>>
+unordered_map(initializer_list<pair<const Key, T>>, typename see below::size_type = see below,
+    Hash = Hash(), Pred = Pred(), Allocator = Allocator())
+  -> unordered_map<Key, T, Hash, Pred, Allocator>; // C++17
+
+template<class InputIterator, class Allocator>
+unordered_map(InputIterator, InputIterator, typename see below::size_type, Allocator)
+  -> unordered_map<iter_key_t<InputIterator>, iter_val_t<InputIterator>,
+        hash<iter_key_t<InputIterator>>, equal_to<iter_key_t<InputIterator>>, Allocator>; // C++17
+
+template<class InputIterator, class Allocator>
+unordered_map(InputIterator, InputIterator, Allocator)
+  -> unordered_map<iter_key_t<InputIterator>, iter_val_t<InputIterator>,
+        hash<iter_key_t<InputIterator>>, equal_to<iter_key_t<InputIterator>>, Allocator>; // C++17
+
+template<class InputIterator, class Hash, class Allocator>
+unordered_map(InputIterator, InputIterator, typename see below::size_type, Hash, Allocator)
+  -> unordered_map<iter_key_t<InputIterator>, iter_val_t<InputIterator>, Hash,
+          equal_to<iter_key_t<InputIterator>>, Allocator>; // C++17
+
+template<ranges::input_range R, class Allocator>
+  unordered_map(from_range_t, R&&, typename see below::size_type, Allocator)
+    -> unordered_map<range-key-type<R>, range-mapped-type<R>, hash<range-key-type<R>>,
+                      equal_to<range-key-type<R>>, Allocator>;   // C++23
+
+template<ranges::input_range R, class Allocator>
+  unordered_map(from_range_t, R&&, Allocator)
+    -> unordered_map<range-key-type<R>, range-mapped-type<R>, hash<range-key-type<R>>,
+                      equal_to<range-key-type<R>>, Allocator>;   // C++23
+
+template<ranges::input_range R, class Hash, class Allocator>
+  unordered_map(from_range_t, R&&, typename see below::size_type, Hash, Allocator)
+    -> unordered_map<range-key-type<R>, range-mapped-type<R>, Hash,
+                      equal_to<range-key-type<R>>, Allocator>;   // C++23
+
+template<class Key, class T, typename Allocator>
+unordered_map(initializer_list<pair<const Key, T>>, typename see below::size_type, Allocator)
+  -> unordered_map<Key, T, hash<Key>, equal_to<Key>, Allocator>; // C++17
+
+template<class Key, class T, typename Allocator>
+unordered_map(initializer_list<pair<const Key, T>>, Allocator)
+  -> unordered_map<Key, T, hash<Key>, equal_to<Key>, Allocator>; // C++17
+
+template<class Key, class T, class Hash, class Allocator>
+unordered_map(initializer_list<pair<const Key, T>>, typename see below::size_type, Hash, Allocator)
+  -> unordered_map<Key, T, Hash, equal_to<Key>, Allocator>; // C++17
+
+template <class Key, class T, class Hash, class Pred, class Alloc>
+    void swap(unordered_map<Key, T, Hash, Pred, Alloc>& x,
+              unordered_map<Key, T, Hash, Pred, Alloc>& y)
+              noexcept(noexcept(x.swap(y)));
+
+template <class Key, class T, class Hash, class Pred, class Alloc>
+    bool
+    operator==(const unordered_map<Key, T, Hash, Pred, Alloc>& x,
+               const unordered_map<Key, T, Hash, Pred, Alloc>& y);
+
+template <class Key, class T, class Hash, class Pred, class Alloc>
+    bool
+    operator!=(const unordered_map<Key, T, Hash, Pred, Alloc>& x,
+               const unordered_map<Key, T, Hash, Pred, Alloc>& y); // Removed in C++20
+
+template <class Key, class T, class Hash = hash<Key>, class Pred = equal_to<Key>,
+          class Alloc = allocator<pair<const Key, T>>>
+class unordered_multimap
+{
+public:
+    // types
+    typedef Key                                                        key_type;
+    typedef T                                                          mapped_type;
+    typedef Hash                                                       hasher;
+    typedef Pred                                                       key_equal;
+    typedef Alloc                                                      allocator_type;
+    typedef pair<const key_type, mapped_type>                          value_type;
+    typedef value_type&                                                reference;
+    typedef const value_type&                                          const_reference;
+    typedef typename allocator_traits<allocator_type>::pointer         pointer;
+    typedef typename allocator_traits<allocator_type>::const_pointer   const_pointer;
+    typedef typename allocator_traits<allocator_type>::size_type       size_type;
+    typedef typename allocator_traits<allocator_type>::difference_type difference_type;
+
+    typedef /unspecified/ iterator;
+    typedef /unspecified/ const_iterator;
+    typedef /unspecified/ local_iterator;
+    typedef /unspecified/ const_local_iterator;
+
+    typedef unspecified node_type;    // C++17
+
+    unordered_multimap()
+        noexcept(
+            is_nothrow_default_constructible<hasher>::value &&
+            is_nothrow_default_constructible<key_equal>::value &&
+            is_nothrow_default_constructible<allocator_type>::value);
+    explicit unordered_multimap(size_type n, const hasher& hf = hasher(),
+                           const key_equal& eql = key_equal(),
+                           const allocator_type& a = allocator_type());
+    template <class InputIterator>
+        unordered_multimap(InputIterator f, InputIterator l,
+                      size_type n = 0, const hasher& hf = hasher(),
+                      const key_equal& eql = key_equal(),
+                      const allocator_type& a = allocator_type());
+    template<container-compatible-range<value_type> R>
+      unordered_multimap(from_range_t, R&& rg, size_type n = see below,
+        const hasher& hf = hasher(), const key_equal& eql = key_equal(),
+        const allocator_type& a = allocator_type()); // C++23
+    explicit unordered_multimap(const allocator_type&);
+    unordered_multimap(const unordered_multimap&);
+    unordered_multimap(const unordered_multimap&, const Allocator&);
+    unordered_multimap(unordered_multimap&&)
+        noexcept(
+            is_nothrow_move_constructible<hasher>::value &&
+            is_nothrow_move_constructible<key_equal>::value &&
+            is_nothrow_move_constructible<allocator_type>::value);
+    unordered_multimap(unordered_multimap&&, const Allocator&);
+    unordered_multimap(initializer_list<value_type>, size_type n = 0,
+                  const hasher& hf = hasher(), const key_equal& eql = key_equal(),
+                  const allocator_type& a = allocator_type());
+    unordered_multimap(size_type n, const allocator_type& a)
+      : unordered_multimap(n, hasher(), key_equal(), a) {}  // C++14
+    unordered_multimap(size_type n, const hasher& hf, const allocator_type& a)
+      : unordered_multimap(n, hf, key_equal(), a) {}  // C++14
+    template <class InputIterator>
+      unordered_multimap(InputIterator f, InputIterator l, size_type n, const allocator_type& a)
+      : unordered_multimap(f, l, n, hasher(), key_equal(), a) {}  // C++14
+    template <class InputIterator>
+      unordered_multimap(InputIterator f, InputIterator l, size_type n, const hasher& hf,
+        const allocator_type& a)
+      : unordered_multimap(f, l, n, hf, key_equal(), a) {}  // C++14
+    template<container-compatible-range<value_type> R>
+      unordered_multimap(from_range_t, R&& rg, size_type n, const allocator_type& a)
+        : unordered_multimap(from_range, std::forward<R>(rg), n, hasher(), key_equal(), a) { } // C++23
+    template<container-compatible-range<value_type> R>
+      unordered_multimap(from_range_t, R&& rg, size_type n, const hasher& hf, const allocator_type& a)
+        : unordered_multimap(from_range, std::forward<R>(rg), n, hf, key_equal(), a) { }       // C++23
+    unordered_multimap(initializer_list<value_type> il, size_type n, const allocator_type& a)
+      : unordered_multimap(il, n, hasher(), key_equal(), a) {}  // C++14
+    unordered_multimap(initializer_list<value_type> il, size_type n, const hasher& hf,
+      const allocator_type& a)
+      : unordered_multimap(il, n, hf, key_equal(), a) {}  // C++14
+    ~unordered_multimap();
+    unordered_multimap& operator=(const unordered_multimap&);
+    unordered_multimap& operator=(unordered_multimap&&)
+        noexcept(
+            allocator_type::propagate_on_container_move_assignment::value &&
+            is_nothrow_move_assignable<allocator_type>::value &&
+            is_nothrow_move_assignable<hasher>::value &&
+            is_nothrow_move_assignable<key_equal>::value);
+    unordered_multimap& operator=(initializer_list<value_type>);
+
+    allocator_type get_allocator() const noexcept;
+
+    bool      empty() const noexcept;
+    size_type size() const noexcept;
+    size_type max_size() const noexcept;
+
+    iterator       begin() noexcept;
+    iterator       end() noexcept;
+    const_iterator begin()  const noexcept;
+    const_iterator end()    const noexcept;
+    const_iterator cbegin() const noexcept;
+    const_iterator cend()   const noexcept;
+
+    template <class... Args>
+        iterator emplace(Args&&... args);
+    template <class... Args>
+        iterator emplace_hint(const_iterator position, Args&&... args);
+    iterator insert(const value_type& obj);
+    template <class P>
+        iterator insert(P&& obj);
+    iterator insert(const_iterator hint, const value_type& obj);
+    template <class P>
+        iterator insert(const_iterator hint, P&& obj);
+    template <class InputIterator>
+        void insert(InputIterator first, InputIterator last);
+    template<container-compatible-range<value_type> R>
+      void insert_range(R&& rg);                               // C++23
+    void insert(initializer_list<value_type>);
+
+    node_type extract(const_iterator position);                // C++17
+    node_type extract(const key_type& x);                      // C++17
+    iterator insert(node_type&& nh);                           // C++17
+    iterator insert(const_iterator hint, node_type&& nh);      // C++17
+
+    iterator erase(const_iterator position);
+    iterator erase(iterator position);  // C++14
+    size_type erase(const key_type& k);
+    iterator erase(const_iterator first, const_iterator last);
+    void clear() noexcept;
+
+    template<class H2, class P2>
+      void merge(unordered_multimap<Key, T, H2, P2, Allocator>& source);    // C++17
+    template<class H2, class P2>
+      void merge(unordered_multimap<Key, T, H2, P2, Allocator>&& source);   // C++17
+    template<class H2, class P2>
+      void merge(unordered_map<Key, T, H2, P2, Allocator>& source);         // C++17
+    template<class H2, class P2>
+      void merge(unordered_map<Key, T, H2, P2, Allocator>&& source);        // C++17
+
+    void swap(unordered_multimap&)
+        noexcept(
+            (!allocator_type::propagate_on_container_swap::value ||
+             __is_nothrow_swappable<allocator_type>::value) &&
+            __is_nothrow_swappable<hasher>::value &&
+            __is_nothrow_swappable<key_equal>::value);
+
+    hasher hash_function() const;
+    key_equal key_eq() const;
+
+    iterator       find(const key_type& k);
+    const_iterator find(const key_type& k) const;
+    template<typename K>
+        iterator find(const K& x);              // C++20
+    template<typename K>
+        const_iterator find(const K& x) const;  // C++20
+    size_type count(const key_type& k) const;
+    template<typename K>
+        size_type count(const K& k) const; // C++20
+    bool contains(const key_type& k) const; // C++20
+    template<typename K>
+        bool contains(const K& k) const; // C++20
+    pair<iterator, iterator>             equal_range(const key_type& k);
+    pair<const_iterator, const_iterator> equal_range(const key_type& k) const;
+    template<typename K>
+        pair<iterator, iterator>             equal_range(const K& k); // C++20
+    template<typename K>
+        pair<const_iterator, const_iterator> equal_range(const K& k) const; // C++20
+
+    size_type bucket_count() const noexcept;
+    size_type max_bucket_count() const noexcept;
+
+    size_type bucket_size(size_type n) const;
+    size_type bucket(const key_type& k) const;
+
+    local_iterator       begin(size_type n);
+    local_iterator       end(size_type n);
+    const_local_iterator begin(size_type n) const;
+    const_local_iterator end(size_type n) const;
+    const_local_iterator cbegin(size_type n) const;
+    const_local_iterator cend(size_type n) const;
+
+    float load_factor() const noexcept;
+    float max_load_factor() const noexcept;
+    void max_load_factor(float z);
+    void rehash(size_type n);
+    void reserve(size_type n);
+};
+
+template<class InputIterator,
+    class Hash = hash<iter_key_t<InputIterator>>, class Pred = equal_to<iter_key_t<InputIterator>>,
+    class Allocator = allocator<iter_to_alloc_t<InputIterator>>>
+unordered_multimap(InputIterator, InputIterator, typename see below::size_type = see below,
+    Hash = Hash(), Pred = Pred(), Allocator = Allocator())
+  -> unordered_multimap<iter_key_t<InputIterator>, iter_value_t<InputIterator>, Hash, Pred,
+    Allocator>; // C++17
+
+template<ranges::input_range R, class Hash = hash<range-key-type<R>>,
+         class Pred = equal_to<range-key-type<R>>,
+         class Allocator = allocator<range-to-alloc-type<R>>>
+  unordered_multimap(from_range_t, R&&, typename see below::size_type = see below,
+                Hash = Hash(), Pred = Pred(), Allocator = Allocator())
+    -> unordered_multimap<range-key-type<R>, range-mapped-type<R>, Hash, Pred, Allocator>; // C++23
+
+template<class Key, class T, class Hash = hash<Key>,
+    class Pred = equal_to<Key>, class Allocator = allocator<pair<const Key, T>>>
+unordered_multimap(initializer_list<pair<const Key, T>>, typename see below::size_type = see below,
+    Hash = Hash(), Pred = Pred(), Allocator = Allocator())
+  -> unordered_multimap<Key, T, Hash, Pred, Allocator>; // C++17
+
+template<class InputIterator, class Allocator>
+unordered_multimap(InputIterator, InputIterator, typename see below::size_type, Allocator)
+  -> unordered_multimap<iter_key_t<InputIterator>, iter_val_t<InputIterator>,
+        hash<iter_key_t<InputIterator>>, equal_to<iter_key_t<InputIterator>>, Allocator>; // C++17
+
+template<class InputIterator, class Allocator>
+unordered_multimap(InputIterator, InputIterator, Allocator)
+  -> unordered_multimap<iter_key_t<InputIterator>, iter_val_t<InputIterator>,
+        hash<iter_key_t<InputIterator>>, equal_to<iter_key_t<InputIterator>>, Allocator>; // C++17
+
+template<class InputIterator, class Hash, class Allocator>
+unordered_multimap(InputIterator, InputIterator, typename see below::size_type, Hash, Allocator)
+  -> unordered_multimap<iter_key_t<InputIterator>, iter_val_t<InputIterator>, Hash,
+          equal_to<iter_key_t<InputIterator>>, Allocator>; // C++17
+
+template<ranges::input_range R, class Allocator>
+  unordered_multimap(from_range_t, R&&, typename see below::size_type, Allocator)
+    -> unordered_multimap<range-key-type<R>, range-mapped-type<R>, hash<range-key-type<R>>,
+                      equal_to<range-key-type<R>>, Allocator>;   // C++23
+
+template<ranges::input_range R, class Allocator>
+  unordered_multimap(from_range_t, R&&, Allocator)
+    -> unordered_multimap<range-key-type<R>, range-mapped-type<R>, hash<range-key-type<R>>,
+                      equal_to<range-key-type<R>>, Allocator>;   // C++23
+
+template<ranges::input_range R, class Hash, class Allocator>
+  unordered_multimap(from_range_t, R&&, typename see below::size_type, Hash, Allocator)
+    -> unordered_multimap<range-key-type<R>, range-mapped-type<R>, Hash,
+                      equal_to<range-key-type<R>>, Allocator>;   // C++23
+
+template<class Key, class T, typename Allocator>
+unordered_multimap(initializer_list<pair<const Key, T>>, typename see below::size_type, Allocator)
+  -> unordered_multimap<Key, T, hash<Key>, equal_to<Key>, Allocator>; // C++17
+
+template<class Key, class T, typename Allocator>
+unordered_multimap(initializer_list<pair<const Key, T>>, Allocator)
+  -> unordered_multimap<Key, T, hash<Key>, equal_to<Key>, Allocator>; // C++17
+
+template<class Key, class T, class Hash, class Allocator>
+unordered_multimap(initializer_list<pair<const Key, T>>, typename see below::size_type, Hash,
+    Allocator)
+  -> unordered_multimap<Key, T, Hash, equal_to<Key>, Allocator>; // C++17
+
+template <class Key, class T, class Hash, class Pred, class Alloc>
+    void swap(unordered_multimap<Key, T, Hash, Pred, Alloc>& x,
+              unordered_multimap<Key, T, Hash, Pred, Alloc>& y)
+              noexcept(noexcept(x.swap(y)));
+
+template <class K, class T, class H, class P, class A, class Predicate>
+    typename unordered_map<K, T, H, P, A>::size_type
+    erase_if(unordered_map<K, T, H, P, A>& c, Predicate pred);       // C++20
+
+template <class K, class T, class H, class P, class A, class Predicate>
+    typename unordered_multimap<K, T, H, P, A>::size_type
+    erase_if(unordered_multimap<K, T, H, P, A>& c, Predicate pred);  // C++20
+
+template <class Key, class T, class Hash, class Pred, class Alloc>
+    bool
+    operator==(const unordered_multimap<Key, T, Hash, Pred, Alloc>& x,
+               const unordered_multimap<Key, T, Hash, Pred, Alloc>& y);
+
+template <class Key, class T, class Hash, class Pred, class Alloc>
+    bool
+    operator!=(const unordered_multimap<Key, T, Hash, Pred, Alloc>& x,
+               const unordered_multimap<Key, T, Hash, Pred, Alloc>& y); // Removed in C++20
+
+}  // std
+
+*/
+
+#include <__algorithm/is_permutation.h>
+#include <__assert>
+#include <__config>
+#include <__functional/is_transparent.h>
+#include <__functional/operations.h>
+#include <__hash_table>
+#include <__iterator/distance.h>
+#include <__iterator/erase_if_container.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/ranges_iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__memory/allocator.h>
+#include <__memory_resource/polymorphic_allocator.h>
+#include <__node_handle>
+#include <__ranges/concepts.h>
+#include <__ranges/container_compatible_range.h>
+#include <__ranges/from_range.h>
+#include <__type_traits/is_allocator.h>
+#include <__type_traits/type_identity.h>
+#include <__utility/forward.h>
+#include <stdexcept>
+#include <tuple>
+#include <version>
+
+// standard-mandated includes
+
+// [iterator.range]
+#include <__iterator/access.h>
+#include <__iterator/data.h>
+#include <__iterator/empty.h>
+#include <__iterator/reverse_access.h>
+#include <__iterator/size.h>
+
+// [unord.map.syn]
+#include <compare>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Key,
+          class _Cp,
+          class _Hash,
+          class _Pred,
+          bool = is_empty<_Hash>::value && !__libcpp_is_final<_Hash>::value>
+class __unordered_map_hasher : private _Hash {
+public:
+  _LIBCPP_HIDE_FROM_ABI __unordered_map_hasher() _NOEXCEPT_(is_nothrow_default_constructible<_Hash>::value) : _Hash() {}
+  _LIBCPP_HIDE_FROM_ABI __unordered_map_hasher(const _Hash& __h) _NOEXCEPT_(is_nothrow_copy_constructible<_Hash>::value)
+      : _Hash(__h) {}
+  _LIBCPP_HIDE_FROM_ABI const _Hash& hash_function() const _NOEXCEPT { return *this; }
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Cp& __x) const {
+    return static_cast<const _Hash&>(*this)(__x.__get_value().first);
+  }
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Key& __x) const { return static_cast<const _Hash&>(*this)(__x); }
+#if _LIBCPP_STD_VER >= 20
+  template <typename _K2>
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _K2& __x) const {
+    return static_cast<const _Hash&>(*this)(__x);
+  }
+#endif
+  _LIBCPP_HIDE_FROM_ABI void swap(__unordered_map_hasher& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Hash>) {
+    using std::swap;
+    swap(static_cast<_Hash&>(*this), static_cast<_Hash&>(__y));
+  }
+};
+
+template <class _Key, class _Cp, class _Hash, class _Pred>
+class __unordered_map_hasher<_Key, _Cp, _Hash, _Pred, false> {
+  _Hash __hash_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __unordered_map_hasher() _NOEXCEPT_(is_nothrow_default_constructible<_Hash>::value)
+      : __hash_() {}
+  _LIBCPP_HIDE_FROM_ABI __unordered_map_hasher(const _Hash& __h) _NOEXCEPT_(is_nothrow_copy_constructible<_Hash>::value)
+      : __hash_(__h) {}
+  _LIBCPP_HIDE_FROM_ABI const _Hash& hash_function() const _NOEXCEPT { return __hash_; }
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Cp& __x) const { return __hash_(__x.__get_value().first); }
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Key& __x) const { return __hash_(__x); }
+#if _LIBCPP_STD_VER >= 20
+  template <typename _K2>
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _K2& __x) const {
+    return __hash_(__x);
+  }
+#endif
+  _LIBCPP_HIDE_FROM_ABI void swap(__unordered_map_hasher& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Hash>) {
+    using std::swap;
+    swap(__hash_, __y.__hash_);
+  }
+};
+
+template <class _Key, class _Cp, class _Hash, class _Pred, bool __b>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(__unordered_map_hasher<_Key, _Cp, _Hash, _Pred, __b>& __x,
+     __unordered_map_hasher<_Key, _Cp, _Hash, _Pred, __b>& __y) _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+template <class _Key,
+          class _Cp,
+          class _Pred,
+          class _Hash,
+          bool = is_empty<_Pred>::value && !__libcpp_is_final<_Pred>::value>
+class __unordered_map_equal : private _Pred {
+public:
+  _LIBCPP_HIDE_FROM_ABI __unordered_map_equal() _NOEXCEPT_(is_nothrow_default_constructible<_Pred>::value) : _Pred() {}
+  _LIBCPP_HIDE_FROM_ABI __unordered_map_equal(const _Pred& __p) _NOEXCEPT_(is_nothrow_copy_constructible<_Pred>::value)
+      : _Pred(__p) {}
+  _LIBCPP_HIDE_FROM_ABI const _Pred& key_eq() const _NOEXCEPT { return *this; }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _Cp& __y) const {
+    return static_cast<const _Pred&>(*this)(__x.__get_value().first, __y.__get_value().first);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _Key& __y) const {
+    return static_cast<const _Pred&>(*this)(__x.__get_value().first, __y);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _Cp& __y) const {
+    return static_cast<const _Pred&>(*this)(__x, __y.__get_value().first);
+  }
+#if _LIBCPP_STD_VER >= 20
+  template <typename _K2>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _K2& __y) const {
+    return static_cast<const _Pred&>(*this)(__x.__get_value().first, __y);
+  }
+  template <typename _K2>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _Cp& __y) const {
+    return static_cast<const _Pred&>(*this)(__x, __y.__get_value().first);
+  }
+  template <typename _K2>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _K2& __y) const {
+    return static_cast<const _Pred&>(*this)(__x, __y);
+  }
+  template <typename _K2>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _Key& __y) const {
+    return static_cast<const _Pred&>(*this)(__x, __y);
+  }
+#endif
+  _LIBCPP_HIDE_FROM_ABI void swap(__unordered_map_equal& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Pred>) {
+    using std::swap;
+    swap(static_cast<_Pred&>(*this), static_cast<_Pred&>(__y));
+  }
+};
+
+template <class _Key, class _Cp, class _Pred, class _Hash>
+class __unordered_map_equal<_Key, _Cp, _Pred, _Hash, false> {
+  _Pred __pred_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __unordered_map_equal() _NOEXCEPT_(is_nothrow_default_constructible<_Pred>::value)
+      : __pred_() {}
+  _LIBCPP_HIDE_FROM_ABI __unordered_map_equal(const _Pred& __p) _NOEXCEPT_(is_nothrow_copy_constructible<_Pred>::value)
+      : __pred_(__p) {}
+  _LIBCPP_HIDE_FROM_ABI const _Pred& key_eq() const _NOEXCEPT { return __pred_; }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _Cp& __y) const {
+    return __pred_(__x.__get_value().first, __y.__get_value().first);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _Key& __y) const {
+    return __pred_(__x.__get_value().first, __y);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _Cp& __y) const {
+    return __pred_(__x, __y.__get_value().first);
+  }
+#if _LIBCPP_STD_VER >= 20
+  template <typename _K2>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _K2& __y) const {
+    return __pred_(__x.__get_value().first, __y);
+  }
+  template <typename _K2>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _Cp& __y) const {
+    return __pred_(__x, __y.__get_value().first);
+  }
+  template <typename _K2>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _K2& __y) const {
+    return __pred_(__x, __y);
+  }
+  template <typename _K2>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _Key& __y) const {
+    return __pred_(__x, __y);
+  }
+#endif
+  _LIBCPP_HIDE_FROM_ABI void swap(__unordered_map_equal& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Pred>) {
+    using std::swap;
+    swap(__pred_, __y.__pred_);
+  }
+};
+
+template <class _Key, class _Cp, class _Pred, class _Hash, bool __b>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(__unordered_map_equal<_Key, _Cp, _Pred, _Hash, __b>& __x, __unordered_map_equal<_Key, _Cp, _Pred, _Hash, __b>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+template <class _Alloc>
+class __hash_map_node_destructor {
+  typedef _Alloc allocator_type;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+
+public:
+  typedef typename __alloc_traits::pointer pointer;
+
+private:
+  allocator_type& __na_;
+
+public:
+  bool __first_constructed;
+  bool __second_constructed;
+
+  __hash_map_node_destructor& operator=(const __hash_map_node_destructor&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __hash_map_node_destructor(allocator_type& __na) _NOEXCEPT
+      : __na_(__na),
+        __first_constructed(false),
+        __second_constructed(false) {}
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI __hash_map_node_destructor(__hash_node_destructor<allocator_type>&& __x) _NOEXCEPT
+      : __na_(__x.__na_),
+        __first_constructed(__x.__value_constructed),
+        __second_constructed(__x.__value_constructed) {
+    __x.__value_constructed = false;
+  }
+#else  // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI __hash_map_node_destructor(const __hash_node_destructor<allocator_type>& __x)
+      : __na_(__x.__na_), __first_constructed(__x.__value_constructed), __second_constructed(__x.__value_constructed) {
+    const_cast<bool&>(__x.__value_constructed) = false;
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) _NOEXCEPT {
+    if (__second_constructed)
+      __alloc_traits::destroy(__na_, std::addressof(__p->__get_value().__get_value().second));
+    if (__first_constructed)
+      __alloc_traits::destroy(__na_, std::addressof(__p->__get_value().__get_value().first));
+    if (__p)
+      __alloc_traits::deallocate(__na_, __p, 1);
+  }
+};
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Key, class _Tp>
+struct _LIBCPP_STANDALONE_DEBUG __hash_value_type {
+  typedef _Key key_type;
+  typedef _Tp mapped_type;
+  typedef pair<const key_type, mapped_type> value_type;
+  typedef pair<key_type&, mapped_type&> __nc_ref_pair_type;
+  typedef pair<key_type&&, mapped_type&&> __nc_rref_pair_type;
+
+private:
+  value_type __cc_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI value_type& __get_value() {
+#  if _LIBCPP_STD_VER >= 17
+    return *std::launder(std::addressof(__cc_));
+#  else
+    return __cc_;
+#  endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const value_type& __get_value() const {
+#  if _LIBCPP_STD_VER >= 17
+    return *std::launder(std::addressof(__cc_));
+#  else
+    return __cc_;
+#  endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __nc_ref_pair_type __ref() {
+    value_type& __v = __get_value();
+    return __nc_ref_pair_type(const_cast<key_type&>(__v.first), __v.second);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __nc_rref_pair_type __move() {
+    value_type& __v = __get_value();
+    return __nc_rref_pair_type(std::move(const_cast<key_type&>(__v.first)), std::move(__v.second));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __hash_value_type& operator=(const __hash_value_type& __v) {
+    __ref() = __v.__get_value();
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __hash_value_type& operator=(__hash_value_type&& __v) {
+    __ref() = __v.__move();
+    return *this;
+  }
+
+  template <class _ValueTp, __enable_if_t<__is_same_uncvref<_ValueTp, value_type>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI __hash_value_type& operator=(_ValueTp&& __v) {
+    __ref() = std::forward<_ValueTp>(__v);
+    return *this;
+  }
+
+  __hash_value_type(const __hash_value_type& __v) = delete;
+  __hash_value_type(__hash_value_type&& __v)      = delete;
+  template <class... _Args>
+  explicit __hash_value_type(_Args&&... __args) = delete;
+
+  ~__hash_value_type() = delete;
+};
+
+#else
+
+template <class _Key, class _Tp>
+struct __hash_value_type {
+  typedef _Key key_type;
+  typedef _Tp mapped_type;
+  typedef pair<const key_type, mapped_type> value_type;
+
+private:
+  value_type __cc_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI value_type& __get_value() { return __cc_; }
+  _LIBCPP_HIDE_FROM_ABI const value_type& __get_value() const { return __cc_; }
+
+  ~__hash_value_type() = delete;
+};
+
+#endif
+
+template <class _HashIterator>
+class _LIBCPP_TEMPLATE_VIS __hash_map_iterator {
+  _HashIterator __i_;
+
+  typedef __hash_node_types_from_iterator<_HashIterator> _NodeTypes;
+
+public:
+  typedef forward_iterator_tag iterator_category;
+  typedef typename _NodeTypes::__map_value_type value_type;
+  typedef typename _NodeTypes::difference_type difference_type;
+  typedef value_type& reference;
+  typedef typename _NodeTypes::__map_value_type_pointer pointer;
+
+  _LIBCPP_HIDE_FROM_ABI __hash_map_iterator() _NOEXCEPT {}
+
+  _LIBCPP_HIDE_FROM_ABI __hash_map_iterator(_HashIterator __i) _NOEXCEPT : __i_(__i) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __i_->__get_value(); }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return pointer_traits<pointer>::pointer_to(__i_->__get_value()); }
+
+  _LIBCPP_HIDE_FROM_ABI __hash_map_iterator& operator++() {
+    ++__i_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __hash_map_iterator operator++(int) {
+    __hash_map_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __hash_map_iterator& __x, const __hash_map_iterator& __y) {
+    return __x.__i_ == __y.__i_;
+  }
+#if _LIBCPP_STD_VER <= 17
+  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __hash_map_iterator& __x, const __hash_map_iterator& __y) {
+    return __x.__i_ != __y.__i_;
+  }
+#endif
+
+  template <class, class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_map;
+  template <class, class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_multimap;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_const_iterator;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_const_local_iterator;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_map_const_iterator;
+};
+
+template <class _HashIterator>
+class _LIBCPP_TEMPLATE_VIS __hash_map_const_iterator {
+  _HashIterator __i_;
+
+  typedef __hash_node_types_from_iterator<_HashIterator> _NodeTypes;
+
+public:
+  typedef forward_iterator_tag iterator_category;
+  typedef typename _NodeTypes::__map_value_type value_type;
+  typedef typename _NodeTypes::difference_type difference_type;
+  typedef const value_type& reference;
+  typedef typename _NodeTypes::__const_map_value_type_pointer pointer;
+
+  _LIBCPP_HIDE_FROM_ABI __hash_map_const_iterator() _NOEXCEPT {}
+
+  _LIBCPP_HIDE_FROM_ABI __hash_map_const_iterator(_HashIterator __i) _NOEXCEPT : __i_(__i) {}
+  _LIBCPP_HIDE_FROM_ABI
+  __hash_map_const_iterator(__hash_map_iterator<typename _HashIterator::__non_const_iterator> __i) _NOEXCEPT
+      : __i_(__i.__i_) {}
+
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __i_->__get_value(); }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return pointer_traits<pointer>::pointer_to(__i_->__get_value()); }
+
+  _LIBCPP_HIDE_FROM_ABI __hash_map_const_iterator& operator++() {
+    ++__i_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI __hash_map_const_iterator operator++(int) {
+    __hash_map_const_iterator __t(*this);
+    ++(*this);
+    return __t;
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const __hash_map_const_iterator& __x, const __hash_map_const_iterator& __y) {
+    return __x.__i_ == __y.__i_;
+  }
+#if _LIBCPP_STD_VER <= 17
+  friend _LIBCPP_HIDE_FROM_ABI bool
+  operator!=(const __hash_map_const_iterator& __x, const __hash_map_const_iterator& __y) {
+    return __x.__i_ != __y.__i_;
+  }
+#endif
+
+  template <class, class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_map;
+  template <class, class, class, class, class>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_multimap;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_const_iterator;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS __hash_const_local_iterator;
+};
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+class unordered_multimap;
+
+template <class _Key,
+          class _Tp,
+          class _Hash  = hash<_Key>,
+          class _Pred  = equal_to<_Key>,
+          class _Alloc = allocator<pair<const _Key, _Tp> > >
+class _LIBCPP_TEMPLATE_VIS unordered_map {
+public:
+  // types
+  typedef _Key key_type;
+  typedef _Tp mapped_type;
+  typedef __type_identity_t<_Hash> hasher;
+  typedef __type_identity_t<_Pred> key_equal;
+  typedef __type_identity_t<_Alloc> allocator_type;
+  typedef pair<const key_type, mapped_type> value_type;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+  static_assert(is_same<value_type, typename allocator_type::value_type>::value,
+                "Allocator::value_type must be same type as value_type");
+
+private:
+  typedef __hash_value_type<key_type, mapped_type> __value_type;
+  typedef __unordered_map_hasher<key_type, __value_type, hasher, key_equal> __hasher;
+  typedef __unordered_map_equal<key_type, __value_type, key_equal, hasher> __key_equal;
+  typedef __rebind_alloc<allocator_traits<allocator_type>, __value_type> __allocator_type;
+
+  typedef __hash_table<__value_type, __hasher, __key_equal, __allocator_type> __table;
+
+  __table __table_;
+
+  typedef typename __table::_NodeTypes _NodeTypes;
+  typedef typename __table::__node_pointer __node_pointer;
+  typedef typename __table::__node_const_pointer __node_const_pointer;
+  typedef typename __table::__node_traits __node_traits;
+  typedef typename __table::__node_allocator __node_allocator;
+  typedef typename __table::__node __node;
+  typedef __hash_map_node_destructor<__node_allocator> _Dp;
+  typedef unique_ptr<__node, _Dp> __node_holder;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+
+  static_assert(__check_valid_allocator<allocator_type>::value, "");
+
+  static_assert(is_same<typename __table::__container_value_type, value_type>::value, "");
+  static_assert(is_same<typename __table::__node_value_type, __value_type>::value, "");
+
+public:
+  typedef typename __alloc_traits::pointer pointer;
+  typedef typename __alloc_traits::const_pointer const_pointer;
+  typedef typename __table::size_type size_type;
+  typedef typename __table::difference_type difference_type;
+
+  typedef __hash_map_iterator<typename __table::iterator> iterator;
+  typedef __hash_map_const_iterator<typename __table::const_iterator> const_iterator;
+  typedef __hash_map_iterator<typename __table::local_iterator> local_iterator;
+  typedef __hash_map_const_iterator<typename __table::const_local_iterator> const_local_iterator;
+
+#if _LIBCPP_STD_VER >= 17
+  typedef __map_node_handle<__node, allocator_type> node_type;
+  typedef __insert_return_type<iterator, node_type> insert_return_type;
+#endif
+
+  template <class _Key2, class _Tp2, class _Hash2, class _Pred2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_map;
+  template <class _Key2, class _Tp2, class _Hash2, class _Pred2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_multimap;
+
+  _LIBCPP_HIDE_FROM_ABI unordered_map() _NOEXCEPT_(is_nothrow_default_constructible<__table>::value) {}
+  explicit _LIBCPP_HIDE_FROM_ABI
+  unordered_map(size_type __n, const hasher& __hf = hasher(), const key_equal& __eql = key_equal());
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_map(size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI unordered_map(_InputIterator __first, _InputIterator __last);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_map(_InputIterator __first,
+                _InputIterator __last,
+                size_type __n,
+                const hasher& __hf     = hasher(),
+                const key_equal& __eql = key_equal());
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI unordered_map(
+      _InputIterator __first,
+      _InputIterator __last,
+      size_type __n,
+      const hasher& __hf,
+      const key_equal& __eql,
+      const allocator_type& __a);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI unordered_map(
+      from_range_t,
+      _Range&& __range,
+      size_type __n             = /*implementation-defined*/ 0,
+      const hasher& __hf        = hasher(),
+      const key_equal& __eql    = key_equal(),
+      const allocator_type& __a = allocator_type())
+      : __table_(__hf, __eql, typename __table::allocator_type(__a)) {
+    if (__n > 0) {
+      __table_.__rehash_unique(__n);
+    }
+    insert_range(std::forward<_Range>(__range));
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI explicit unordered_map(const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u);
+  _LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u, const allocator_type& __a);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u) _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u, const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI unordered_map(initializer_list<value_type> __il);
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_map(initializer_list<value_type> __il,
+                size_type __n,
+                const hasher& __hf     = hasher(),
+                const key_equal& __eql = key_equal());
+  _LIBCPP_HIDE_FROM_ABI unordered_map(
+      initializer_list<value_type> __il,
+      size_type __n,
+      const hasher& __hf,
+      const key_equal& __eql,
+      const allocator_type& __a);
+#endif // _LIBCPP_CXX03_LANG
+#if _LIBCPP_STD_VER >= 14
+  _LIBCPP_HIDE_FROM_ABI unordered_map(size_type __n, const allocator_type& __a)
+      : unordered_map(__n, hasher(), key_equal(), __a) {}
+  _LIBCPP_HIDE_FROM_ABI unordered_map(size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_map(__n, __hf, key_equal(), __a) {}
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_map(_InputIterator __first, _InputIterator __last, size_type __n, const allocator_type& __a)
+      : unordered_map(__first, __last, __n, hasher(), key_equal(), __a) {}
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI unordered_map(
+      _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_map(__first, __last, __n, __hf, key_equal(), __a) {}
+
+#  if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI unordered_map(from_range_t, _Range&& __range, size_type __n, const allocator_type& __a)
+      : unordered_map(from_range, std::forward<_Range>(__range), __n, hasher(), key_equal(), __a) {}
+
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_map(from_range_t, _Range&& __range, size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_map(from_range, std::forward<_Range>(__range), __n, __hf, key_equal(), __a) {}
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI unordered_map(initializer_list<value_type> __il, size_type __n, const allocator_type& __a)
+      : unordered_map(__il, __n, hasher(), key_equal(), __a) {}
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_map(initializer_list<value_type> __il, size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_map(__il, __n, __hf, key_equal(), __a) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI ~unordered_map() {
+    static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Key, _Hash, _Pred>(0)), "");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI unordered_map& operator=(const unordered_map& __u) {
+#ifndef _LIBCPP_CXX03_LANG
+    __table_ = __u.__table_;
+#else
+    if (this != std::addressof(__u)) {
+      __table_.clear();
+      __table_.hash_function()   = __u.__table_.hash_function();
+      __table_.key_eq()          = __u.__table_.key_eq();
+      __table_.max_load_factor() = __u.__table_.max_load_factor();
+      __table_.__copy_assign_alloc(__u.__table_);
+      insert(__u.begin(), __u.end());
+    }
+#endif
+    return *this;
+  }
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI unordered_map& operator=(unordered_map&& __u)
+      _NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_map& operator=(initializer_list<value_type> __il);
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
+    return allocator_type(__table_.__node_alloc());
+  }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __table_.size() == 0; }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __table_.size(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __table_.max_size(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __table_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __table_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __table_.end(); }
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(const value_type& __x) { return __table_.__insert_unique(__x); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator, const value_type& __x) { return insert(__x).first; }
+
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
+    for (auto&& __element : __range) {
+      __table_.__insert_unique(std::forward<decltype(__element)>(__element));
+    }
+  }
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(value_type&& __x) {
+    return __table_.__insert_unique(std::move(__x));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator, value_type&& __x) {
+    return __table_.__insert_unique(std::move(__x)).first;
+  }
+
+  template <class _Pp, __enable_if_t<is_constructible<value_type, _Pp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(_Pp&& __x) {
+    return __table_.__insert_unique(std::forward<_Pp>(__x));
+  }
+
+  template <class _Pp, __enable_if_t<is_constructible<value_type, _Pp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator, _Pp&& __x) {
+    return insert(std::forward<_Pp>(__x)).first;
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> emplace(_Args&&... __args) {
+    return __table_.__emplace_unique(std::forward<_Args>(__args)...);
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator, _Args&&... __args) {
+    return __table_.__emplace_unique(std::forward<_Args>(__args)...).first;
+  }
+
+#endif // _LIBCPP_CXX03_LANG
+
+#if _LIBCPP_STD_VER >= 17
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> try_emplace(const key_type& __k, _Args&&... __args) {
+    return __table_.__emplace_unique_key_args(
+        __k, piecewise_construct, std::forward_as_tuple(__k), std::forward_as_tuple(std::forward<_Args>(__args)...));
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> try_emplace(key_type&& __k, _Args&&... __args) {
+    return __table_.__emplace_unique_key_args(
+        __k,
+        piecewise_construct,
+        std::forward_as_tuple(std::move(__k)),
+        std::forward_as_tuple(std::forward<_Args>(__args)...));
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator try_emplace(const_iterator, const key_type& __k, _Args&&... __args) {
+    return try_emplace(__k, std::forward<_Args>(__args)...).first;
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator try_emplace(const_iterator, key_type&& __k, _Args&&... __args) {
+    return try_emplace(std::move(__k), std::forward<_Args>(__args)...).first;
+  }
+
+  template <class _Vp>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert_or_assign(const key_type& __k, _Vp&& __v) {
+    pair<iterator, bool> __res = __table_.__emplace_unique_key_args(__k, __k, std::forward<_Vp>(__v));
+    if (!__res.second) {
+      __res.first->second = std::forward<_Vp>(__v);
+    }
+    return __res;
+  }
+
+  template <class _Vp>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert_or_assign(key_type&& __k, _Vp&& __v) {
+    pair<iterator, bool> __res = __table_.__emplace_unique_key_args(__k, std::move(__k), std::forward<_Vp>(__v));
+    if (!__res.second) {
+      __res.first->second = std::forward<_Vp>(__v);
+    }
+    return __res;
+  }
+
+  template <class _Vp>
+  _LIBCPP_HIDE_FROM_ABI iterator insert_or_assign(const_iterator, const key_type& __k, _Vp&& __v) {
+    return insert_or_assign(__k, std::forward<_Vp>(__v)).first;
+  }
+
+  template <class _Vp>
+  _LIBCPP_HIDE_FROM_ABI iterator insert_or_assign(const_iterator, key_type&& __k, _Vp&& __v) {
+    return insert_or_assign(std::move(__k), std::forward<_Vp>(__v)).first;
+  }
+#endif // _LIBCPP_STD_VER >= 17
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __table_.erase(__p.__i_); }
+  _LIBCPP_HIDE_FROM_ABI iterator erase(iterator __p) { return __table_.erase(__p.__i_); }
+  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __table_.__erase_unique(__k); }
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last) {
+    return __table_.erase(__first.__i_, __last.__i_);
+  }
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __table_.clear(); }
+
+#if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI insert_return_type insert(node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to unordered_map::insert()");
+    return __table_.template __node_handle_insert_unique< node_type, insert_return_type>(std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to unordered_map::insert()");
+    return __table_.template __node_handle_insert_unique<node_type>(__hint.__i_, std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
+    return __table_.template __node_handle_extract<node_type>(__key);
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
+    return __table_.template __node_handle_extract<node_type>(__it.__i_);
+  }
+
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_map<key_type, mapped_type, _H2, _P2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __table_.__node_handle_merge_unique(__source.__table_);
+  }
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_map<key_type, mapped_type, _H2, _P2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __table_.__node_handle_merge_unique(__source.__table_);
+  }
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_multimap<key_type, mapped_type, _H2, _P2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __table_.__node_handle_merge_unique(__source.__table_);
+  }
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_multimap<key_type, mapped_type, _H2, _P2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __table_.__node_handle_merge_unique(__source.__table_);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI void swap(unordered_map& __u) _NOEXCEPT_(__is_nothrow_swappable_v<__table>) {
+    __table_.swap(__u.__table_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI hasher hash_function() const { return __table_.hash_function().hash_function(); }
+  _LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq().key_eq(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
+#if _LIBCPP_STD_VER >= 20
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
+    return __table_.find(__k);
+  }
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
+    return __table_.find(__k);
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_unique(__k); }
+#if _LIBCPP_STD_VER >= 20
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
+    return __table_.__count_unique(__k);
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
+
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
+    return find(__k) != end();
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
+    return __table_.__equal_range_unique(__k);
+  }
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
+    return __table_.__equal_range_unique(__k);
+  }
+#if _LIBCPP_STD_VER >= 20
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
+    return __table_.__equal_range_unique(__k);
+  }
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
+    return __table_.__equal_range_unique(__k);
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI mapped_type& operator[](const key_type& __k);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI mapped_type& operator[](key_type&& __k);
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI mapped_type& at(const key_type& __k);
+  _LIBCPP_HIDE_FROM_ABI const mapped_type& at(const key_type& __k) const;
+
+  _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __table_.bucket_count(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT { return __table_.max_bucket_count(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type bucket_size(size_type __n) const { return __table_.bucket_size(__n); }
+  _LIBCPP_HIDE_FROM_ABI size_type bucket(const key_type& __k) const { return __table_.bucket(__k); }
+
+  _LIBCPP_HIDE_FROM_ABI local_iterator begin(size_type __n) { return __table_.begin(__n); }
+  _LIBCPP_HIDE_FROM_ABI local_iterator end(size_type __n) { return __table_.end(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator begin(size_type __n) const { return __table_.cbegin(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator end(size_type __n) const { return __table_.cend(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator cbegin(size_type __n) const { return __table_.cbegin(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator cend(size_type __n) const { return __table_.cend(__n); }
+
+  _LIBCPP_HIDE_FROM_ABI float load_factor() const _NOEXCEPT { return __table_.load_factor(); }
+  _LIBCPP_HIDE_FROM_ABI float max_load_factor() const _NOEXCEPT { return __table_.max_load_factor(); }
+  _LIBCPP_HIDE_FROM_ABI void max_load_factor(float __mlf) { __table_.max_load_factor(__mlf); }
+  _LIBCPP_HIDE_FROM_ABI void rehash(size_type __n) { __table_.__rehash_unique(__n); }
+  _LIBCPP_HIDE_FROM_ABI void reserve(size_type __n) { __table_.__reserve_unique(__n); }
+
+private:
+#ifdef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI __node_holder __construct_node_with_key(const key_type& __k);
+#endif
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator,
+          class _Hash      = hash<__iter_key_type<_InputIterator>>,
+          class _Pred      = equal_to<__iter_key_type<_InputIterator>>,
+          class _Allocator = allocator<__iter_to_alloc_type<_InputIterator>>,
+          class            = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class            = enable_if_t<!__is_allocator<_Hash>::value>,
+          class            = enable_if_t<!is_integral<_Hash>::value>,
+          class            = enable_if_t<!__is_allocator<_Pred>::value>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_map(_InputIterator,
+              _InputIterator,
+              typename allocator_traits<_Allocator>::size_type = 0,
+              _Hash                                            = _Hash(),
+              _Pred                                            = _Pred(),
+              _Allocator                                       = _Allocator())
+    -> unordered_map<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Hash, _Pred, _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range,
+          class _Hash      = hash<__range_key_type<_Range>>,
+          class _Pred      = equal_to<__range_key_type<_Range>>,
+          class _Allocator = allocator<__range_to_alloc_type<_Range>>,
+          class            = enable_if_t<!__is_allocator<_Hash>::value>,
+          class            = enable_if_t<!is_integral<_Hash>::value>,
+          class            = enable_if_t<!__is_allocator<_Pred>::value>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_map(from_range_t,
+              _Range&&,
+              typename allocator_traits<_Allocator>::size_type = 0,
+              _Hash                                            = _Hash(),
+              _Pred                                            = _Pred(),
+              _Allocator                                       = _Allocator())
+    -> unordered_map<__range_key_type<_Range>, __range_mapped_type<_Range>, _Hash, _Pred, _Allocator>; // C++23
+#  endif
+
+template <class _Key,
+          class _Tp,
+          class _Hash      = hash<remove_const_t<_Key>>,
+          class _Pred      = equal_to<remove_const_t<_Key>>,
+          class _Allocator = allocator<pair<const _Key, _Tp>>,
+          class            = enable_if_t<!__is_allocator<_Hash>::value>,
+          class            = enable_if_t<!is_integral<_Hash>::value>,
+          class            = enable_if_t<!__is_allocator<_Pred>::value>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_map(initializer_list<pair<_Key, _Tp>>,
+              typename allocator_traits<_Allocator>::size_type = 0,
+              _Hash                                            = _Hash(),
+              _Pred                                            = _Pred(),
+              _Allocator = _Allocator()) -> unordered_map<remove_const_t<_Key>, _Tp, _Hash, _Pred, _Allocator>;
+
+template <class _InputIterator,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_map(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Allocator)
+    -> unordered_map<__iter_key_type<_InputIterator>,
+                     __iter_mapped_type<_InputIterator>,
+                     hash<__iter_key_type<_InputIterator>>,
+                     equal_to<__iter_key_type<_InputIterator>>,
+                     _Allocator>;
+
+template <class _InputIterator,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_map(_InputIterator, _InputIterator, _Allocator)
+    -> unordered_map<__iter_key_type<_InputIterator>,
+                     __iter_mapped_type<_InputIterator>,
+                     hash<__iter_key_type<_InputIterator>>,
+                     equal_to<__iter_key_type<_InputIterator>>,
+                     _Allocator>;
+
+template <class _InputIterator,
+          class _Hash,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<!__is_allocator<_Hash>::value>,
+          class = enable_if_t<!is_integral<_Hash>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_map(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
+    -> unordered_map<__iter_key_type<_InputIterator>,
+                     __iter_mapped_type<_InputIterator>,
+                     _Hash,
+                     equal_to<__iter_key_type<_InputIterator>>,
+                     _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+
+template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_map(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Allocator)
+    -> unordered_map<__range_key_type<_Range>,
+                     __range_mapped_type<_Range>,
+                     hash<__range_key_type<_Range>>,
+                     equal_to<__range_key_type<_Range>>,
+                     _Allocator>;
+
+template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_map(from_range_t, _Range&&, _Allocator)
+    -> unordered_map<__range_key_type<_Range>,
+                     __range_mapped_type<_Range>,
+                     hash<__range_key_type<_Range>>,
+                     equal_to<__range_key_type<_Range>>,
+                     _Allocator>;
+
+template <ranges::input_range _Range,
+          class _Hash,
+          class _Allocator,
+          class = enable_if_t<!__is_allocator<_Hash>::value>,
+          class = enable_if_t<!is_integral<_Hash>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_map(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
+    -> unordered_map<__range_key_type<_Range>,
+                     __range_mapped_type<_Range>,
+                     _Hash,
+                     equal_to<__range_key_type<_Range>>,
+                     _Allocator>;
+
+#  endif
+
+template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_map(initializer_list<pair<_Key, _Tp>>, typename allocator_traits<_Allocator>::size_type, _Allocator)
+    -> unordered_map<remove_const_t<_Key>, _Tp, hash<remove_const_t<_Key>>, equal_to<remove_const_t<_Key>>, _Allocator>;
+
+template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_map(initializer_list<pair<_Key, _Tp>>, _Allocator)
+    -> unordered_map<remove_const_t<_Key>, _Tp, hash<remove_const_t<_Key>>, equal_to<remove_const_t<_Key>>, _Allocator>;
+
+template <class _Key,
+          class _Tp,
+          class _Hash,
+          class _Allocator,
+          class = enable_if_t<!__is_allocator<_Hash>::value>,
+          class = enable_if_t<!is_integral<_Hash>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_map(initializer_list<pair<_Key, _Tp>>, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
+    -> unordered_map<remove_const_t<_Key>, _Tp, _Hash, equal_to<remove_const_t<_Key>>, _Allocator>;
+#endif
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_unique(__n);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(
+    size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
+    : __table_(__hf, __eql, typename __table::allocator_type(__a)) {
+  __table_.__rehash_unique(__n);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const allocator_type& __a)
+    : __table_(typename __table::allocator_type(__a)) {}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(_InputIterator __first, _InputIterator __last) {
+  insert(__first, __last);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(
+    _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_unique(__n);
+  insert(__first, __last);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(
+    _InputIterator __first,
+    _InputIterator __last,
+    size_type __n,
+    const hasher& __hf,
+    const key_equal& __eql,
+    const allocator_type& __a)
+    : __table_(__hf, __eql, typename __table::allocator_type(__a)) {
+  __table_.__rehash_unique(__n);
+  insert(__first, __last);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const unordered_map& __u) : __table_(__u.__table_) {
+  __table_.__rehash_unique(__u.bucket_count());
+  insert(__u.begin(), __u.end());
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const unordered_map& __u, const allocator_type& __a)
+    : __table_(__u.__table_, typename __table::allocator_type(__a)) {
+  __table_.__rehash_unique(__u.bucket_count());
+  insert(__u.begin(), __u.end());
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(unordered_map&& __u)
+    _NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
+    : __table_(std::move(__u.__table_)) {}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(unordered_map&& __u, const allocator_type& __a)
+    : __table_(std::move(__u.__table_), typename __table::allocator_type(__a)) {
+  if (__a != __u.get_allocator()) {
+    iterator __i = __u.begin();
+    while (__u.size() != 0) {
+      __table_.__emplace_unique(__u.__table_.remove((__i++).__i_)->__get_value().__move());
+    }
+  }
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(initializer_list<value_type> __il) {
+  insert(__il.begin(), __il.end());
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(
+    initializer_list<value_type> __il, size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_unique(__n);
+  insert(__il.begin(), __il.end());
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(
+    initializer_list<value_type> __il,
+    size_type __n,
+    const hasher& __hf,
+    const key_equal& __eql,
+    const allocator_type& __a)
+    : __table_(__hf, __eql, typename __table::allocator_type(__a)) {
+  __table_.__rehash_unique(__n);
+  insert(__il.begin(), __il.end());
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>&
+unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(unordered_map&& __u)
+    _NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
+  __table_ = std::move(__u.__table_);
+  return *this;
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>&
+unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {
+  __table_.__assign_unique(__il.begin(), __il.end());
+  return *this;
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+inline void unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::insert(_InputIterator __first, _InputIterator __last) {
+  for (; __first != __last; ++__first)
+    __table_.__insert_unique(*__first);
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+_Tp& unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator[](const key_type& __k) {
+  return __table_
+      .__emplace_unique_key_args(__k, piecewise_construct, std::forward_as_tuple(__k), std::forward_as_tuple())
+      .first->__get_value()
+      .second;
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+_Tp& unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator[](key_type&& __k) {
+  return __table_
+      .__emplace_unique_key_args(
+          __k, piecewise_construct, std::forward_as_tuple(std::move(__k)), std::forward_as_tuple())
+      .first->__get_value()
+      .second;
+}
+#else // _LIBCPP_CXX03_LANG
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+typename unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::__node_holder
+unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::__construct_node_with_key(const key_type& __k) {
+  __node_allocator& __na = __table_.__node_alloc();
+  __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
+  __node_traits::construct(__na, std::addressof(__h->__get_value().__get_value().first), __k);
+  __h.get_deleter().__first_constructed = true;
+  __node_traits::construct(__na, std::addressof(__h->__get_value().__get_value().second));
+  __h.get_deleter().__second_constructed = true;
+  return __h;
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+_Tp& unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator[](const key_type& __k) {
+  iterator __i = find(__k);
+  if (__i != end())
+    return __i->second;
+  __node_holder __h        = __construct_node_with_key(__k);
+  pair<iterator, bool> __r = __table_.__node_insert_unique(__h.get());
+  __h.release();
+  return __r.first->second;
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+_Tp& unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::at(const key_type& __k) {
+  iterator __i = find(__k);
+  if (__i == end())
+    __throw_out_of_range("unordered_map::at: key not found");
+  return __i->second;
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+const _Tp& unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::at(const key_type& __k) const {
+  const_iterator __i = find(__k);
+  if (__i == end())
+    __throw_out_of_range("unordered_map::at: key not found");
+  return __i->second;
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __x, unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI typename unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::size_type
+erase_if(unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __c, _Predicate __pred) {
+  return std::__libcpp_erase_if_container(__c, __pred);
+}
+#endif
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+_LIBCPP_HIDE_FROM_ABI bool operator==(const unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
+                                      const unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __y) {
+  if (__x.size() != __y.size())
+    return false;
+  typedef typename unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::const_iterator const_iterator;
+  for (const_iterator __i = __x.begin(), __ex = __x.end(), __ey = __y.end(); __i != __ex; ++__i) {
+    const_iterator __j = __y.find(__i->first);
+    if (__j == __ey || !(*__i == *__j))
+      return false;
+  }
+  return true;
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
+                                             const unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __y) {
+  return !(__x == __y);
+}
+
+#endif
+
+template <class _Key,
+          class _Tp,
+          class _Hash  = hash<_Key>,
+          class _Pred  = equal_to<_Key>,
+          class _Alloc = allocator<pair<const _Key, _Tp> > >
+class _LIBCPP_TEMPLATE_VIS unordered_multimap {
+public:
+  // types
+  typedef _Key key_type;
+  typedef _Tp mapped_type;
+  typedef __type_identity_t<_Hash> hasher;
+  typedef __type_identity_t<_Pred> key_equal;
+  typedef __type_identity_t<_Alloc> allocator_type;
+  typedef pair<const key_type, mapped_type> value_type;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+  static_assert(__check_valid_allocator<allocator_type>::value, "");
+  static_assert(is_same<value_type, typename allocator_type::value_type>::value,
+                "Allocator::value_type must be same type as value_type");
+
+private:
+  typedef __hash_value_type<key_type, mapped_type> __value_type;
+  typedef __unordered_map_hasher<key_type, __value_type, hasher, key_equal> __hasher;
+  typedef __unordered_map_equal<key_type, __value_type, key_equal, hasher> __key_equal;
+  typedef __rebind_alloc<allocator_traits<allocator_type>, __value_type> __allocator_type;
+
+  typedef __hash_table<__value_type, __hasher, __key_equal, __allocator_type> __table;
+
+  __table __table_;
+
+  typedef typename __table::_NodeTypes _NodeTypes;
+  typedef typename __table::__node_traits __node_traits;
+  typedef typename __table::__node_allocator __node_allocator;
+  typedef typename __table::__node __node;
+  typedef __hash_map_node_destructor<__node_allocator> _Dp;
+  typedef unique_ptr<__node, _Dp> __node_holder;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+  static_assert(is_same<typename __node_traits::size_type, typename __alloc_traits::size_type>::value,
+                "Allocator uses different size_type for different types");
+
+public:
+  typedef typename __alloc_traits::pointer pointer;
+  typedef typename __alloc_traits::const_pointer const_pointer;
+  typedef typename __table::size_type size_type;
+  typedef typename __table::difference_type difference_type;
+
+  typedef __hash_map_iterator<typename __table::iterator> iterator;
+  typedef __hash_map_const_iterator<typename __table::const_iterator> const_iterator;
+  typedef __hash_map_iterator<typename __table::local_iterator> local_iterator;
+  typedef __hash_map_const_iterator<typename __table::const_local_iterator> const_local_iterator;
+
+#if _LIBCPP_STD_VER >= 17
+  typedef __map_node_handle<__node, allocator_type> node_type;
+#endif
+
+  template <class _Key2, class _Tp2, class _Hash2, class _Pred2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_map;
+  template <class _Key2, class _Tp2, class _Hash2, class _Pred2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_multimap;
+
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap() _NOEXCEPT_(is_nothrow_default_constructible<__table>::value) {}
+  explicit _LIBCPP_HIDE_FROM_ABI
+  unordered_multimap(size_type __n, const hasher& __hf = hasher(), const key_equal& __eql = key_equal());
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_multimap(size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(_InputIterator __first, _InputIterator __last);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(
+      _InputIterator __first,
+      _InputIterator __last,
+      size_type __n,
+      const hasher& __hf     = hasher(),
+      const key_equal& __eql = key_equal());
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(
+      _InputIterator __first,
+      _InputIterator __last,
+      size_type __n,
+      const hasher& __hf,
+      const key_equal& __eql,
+      const allocator_type& __a);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(
+      from_range_t,
+      _Range&& __range,
+      size_type __n             = /*implementation-defined*/ 0,
+      const hasher& __hf        = hasher(),
+      const key_equal& __eql    = key_equal(),
+      const allocator_type& __a = allocator_type())
+      : __table_(__hf, __eql, typename __table::allocator_type(__a)) {
+    if (__n > 0) {
+      __table_.__rehash_multi(__n);
+    }
+    insert_range(std::forward<_Range>(__range));
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI explicit unordered_multimap(const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u);
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u, const allocator_type& __a);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(unordered_multimap&& __u)
+      _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(unordered_multimap&& __u, const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(initializer_list<value_type> __il);
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(
+      initializer_list<value_type> __il,
+      size_type __n,
+      const hasher& __hf     = hasher(),
+      const key_equal& __eql = key_equal());
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(
+      initializer_list<value_type> __il,
+      size_type __n,
+      const hasher& __hf,
+      const key_equal& __eql,
+      const allocator_type& __a);
+#endif // _LIBCPP_CXX03_LANG
+#if _LIBCPP_STD_VER >= 14
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(size_type __n, const allocator_type& __a)
+      : unordered_multimap(__n, hasher(), key_equal(), __a) {}
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_multimap(__n, __hf, key_equal(), __a) {}
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_multimap(_InputIterator __first, _InputIterator __last, size_type __n, const allocator_type& __a)
+      : unordered_multimap(__first, __last, __n, hasher(), key_equal(), __a) {}
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(
+      _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_multimap(__first, __last, __n, __hf, key_equal(), __a) {}
+
+#  if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(from_range_t, _Range&& __range, size_type __n, const allocator_type& __a)
+      : unordered_multimap(from_range, std::forward<_Range>(__range), __n, hasher(), key_equal(), __a) {}
+
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_multimap(from_range_t, _Range&& __range, size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_multimap(from_range, std::forward<_Range>(__range), __n, __hf, key_equal(), __a) {}
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(initializer_list<value_type> __il, size_type __n, const allocator_type& __a)
+      : unordered_multimap(__il, __n, hasher(), key_equal(), __a) {}
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_multimap(initializer_list<value_type> __il, size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_multimap(__il, __n, __hf, key_equal(), __a) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI ~unordered_multimap() {
+    static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Key, _Hash, _Pred>(0)), "");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(const unordered_multimap& __u) {
+#ifndef _LIBCPP_CXX03_LANG
+    __table_ = __u.__table_;
+#else
+    if (this != std::addressof(__u)) {
+      __table_.clear();
+      __table_.hash_function()   = __u.__table_.hash_function();
+      __table_.key_eq()          = __u.__table_.key_eq();
+      __table_.max_load_factor() = __u.__table_.max_load_factor();
+      __table_.__copy_assign_alloc(__u.__table_);
+      insert(__u.begin(), __u.end());
+    }
+#endif
+    return *this;
+  }
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(unordered_multimap&& __u)
+      _NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(initializer_list<value_type> __il);
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
+    return allocator_type(__table_.__node_alloc());
+  }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __table_.size() == 0; }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __table_.size(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __table_.max_size(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __table_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __table_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __table_.end(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __x) { return __table_.__insert_multi(__x); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __x) {
+    return __table_.__insert_multi(__p.__i_, __x);
+  }
+
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
+    for (auto&& __element : __range) {
+      __table_.__insert_multi(std::forward<decltype(__element)>(__element));
+    }
+  }
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(value_type&& __x) { return __table_.__insert_multi(std::move(__x)); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __x) {
+    return __table_.__insert_multi(__p.__i_, std::move(__x));
+  }
+
+  template <class _Pp, __enable_if_t<is_constructible<value_type, _Pp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator insert(_Pp&& __x) {
+    return __table_.__insert_multi(std::forward<_Pp>(__x));
+  }
+
+  template <class _Pp, __enable_if_t<is_constructible<value_type, _Pp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, _Pp&& __x) {
+    return __table_.__insert_multi(__p.__i_, std::forward<_Pp>(__x));
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace(_Args&&... __args) {
+    return __table_.__emplace_multi(std::forward<_Args>(__args)...);
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __p, _Args&&... __args) {
+    return __table_.__emplace_hint_multi(__p.__i_, std::forward<_Args>(__args)...);
+  }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __table_.erase(__p.__i_); }
+  _LIBCPP_HIDE_FROM_ABI iterator erase(iterator __p) { return __table_.erase(__p.__i_); }
+  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __table_.__erase_multi(__k); }
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last) {
+    return __table_.erase(__first.__i_, __last.__i_);
+  }
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __table_.clear(); }
+
+#if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI iterator insert(node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to unordered_multimap::insert()");
+    return __table_.template __node_handle_insert_multi<node_type>(std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to unordered_multimap::insert()");
+    return __table_.template __node_handle_insert_multi<node_type>(__hint.__i_, std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
+    return __table_.template __node_handle_extract<node_type>(__key);
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
+    return __table_.template __node_handle_extract<node_type>(__it.__i_);
+  }
+
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_multimap<key_type, mapped_type, _H2, _P2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __table_.__node_handle_merge_multi(__source.__table_);
+  }
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_multimap<key_type, mapped_type, _H2, _P2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __table_.__node_handle_merge_multi(__source.__table_);
+  }
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_map<key_type, mapped_type, _H2, _P2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __table_.__node_handle_merge_multi(__source.__table_);
+  }
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_map<key_type, mapped_type, _H2, _P2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __table_.__node_handle_merge_multi(__source.__table_);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI void swap(unordered_multimap& __u) _NOEXCEPT_(__is_nothrow_swappable_v<__table>) {
+    __table_.swap(__u.__table_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI hasher hash_function() const { return __table_.hash_function().hash_function(); }
+  _LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq().key_eq(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
+#if _LIBCPP_STD_VER >= 20
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
+    return __table_.find(__k);
+  }
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
+    return __table_.find(__k);
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_multi(__k); }
+#if _LIBCPP_STD_VER >= 20
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
+    return __table_.__count_multi(__k);
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
+
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
+    return find(__k) != end();
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
+    return __table_.__equal_range_multi(__k);
+  }
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
+    return __table_.__equal_range_multi(__k);
+  }
+#if _LIBCPP_STD_VER >= 20
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
+    return __table_.__equal_range_multi(__k);
+  }
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
+    return __table_.__equal_range_multi(__k);
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __table_.bucket_count(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT { return __table_.max_bucket_count(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type bucket_size(size_type __n) const { return __table_.bucket_size(__n); }
+  _LIBCPP_HIDE_FROM_ABI size_type bucket(const key_type& __k) const { return __table_.bucket(__k); }
+
+  _LIBCPP_HIDE_FROM_ABI local_iterator begin(size_type __n) { return __table_.begin(__n); }
+  _LIBCPP_HIDE_FROM_ABI local_iterator end(size_type __n) { return __table_.end(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator begin(size_type __n) const { return __table_.cbegin(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator end(size_type __n) const { return __table_.cend(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator cbegin(size_type __n) const { return __table_.cbegin(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator cend(size_type __n) const { return __table_.cend(__n); }
+
+  _LIBCPP_HIDE_FROM_ABI float load_factor() const _NOEXCEPT { return __table_.load_factor(); }
+  _LIBCPP_HIDE_FROM_ABI float max_load_factor() const _NOEXCEPT { return __table_.max_load_factor(); }
+  _LIBCPP_HIDE_FROM_ABI void max_load_factor(float __mlf) { __table_.max_load_factor(__mlf); }
+  _LIBCPP_HIDE_FROM_ABI void rehash(size_type __n) { __table_.__rehash_multi(__n); }
+  _LIBCPP_HIDE_FROM_ABI void reserve(size_type __n) { __table_.__reserve_multi(__n); }
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator,
+          class _Hash      = hash<__iter_key_type<_InputIterator>>,
+          class _Pred      = equal_to<__iter_key_type<_InputIterator>>,
+          class _Allocator = allocator<__iter_to_alloc_type<_InputIterator>>,
+          class            = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class            = enable_if_t<!__is_allocator<_Hash>::value>,
+          class            = enable_if_t<!is_integral<_Hash>::value>,
+          class            = enable_if_t<!__is_allocator<_Pred>::value>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multimap(_InputIterator,
+                   _InputIterator,
+                   typename allocator_traits<_Allocator>::size_type = 0,
+                   _Hash                                            = _Hash(),
+                   _Pred                                            = _Pred(),
+                   _Allocator                                       = _Allocator())
+    -> unordered_multimap<__iter_key_type<_InputIterator>,
+                          __iter_mapped_type<_InputIterator>,
+                          _Hash,
+                          _Pred,
+                          _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range,
+          class _Hash      = hash<__range_key_type<_Range>>,
+          class _Pred      = equal_to<__range_key_type<_Range>>,
+          class _Allocator = allocator<__range_to_alloc_type<_Range>>,
+          class            = enable_if_t<!__is_allocator<_Hash>::value>,
+          class            = enable_if_t<!is_integral<_Hash>::value>,
+          class            = enable_if_t<!__is_allocator<_Pred>::value>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multimap(from_range_t,
+                   _Range&&,
+                   typename allocator_traits<_Allocator>::size_type = 0,
+                   _Hash                                            = _Hash(),
+                   _Pred                                            = _Pred(),
+                   _Allocator                                       = _Allocator())
+    -> unordered_multimap<__range_key_type<_Range>, __range_mapped_type<_Range>, _Hash, _Pred, _Allocator>;
+#  endif
+
+template <class _Key,
+          class _Tp,
+          class _Hash      = hash<remove_const_t<_Key>>,
+          class _Pred      = equal_to<remove_const_t<_Key>>,
+          class _Allocator = allocator<pair<const _Key, _Tp>>,
+          class            = enable_if_t<!__is_allocator<_Hash>::value>,
+          class            = enable_if_t<!is_integral<_Hash>::value>,
+          class            = enable_if_t<!__is_allocator<_Pred>::value>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multimap(
+    initializer_list<pair<_Key, _Tp>>,
+    typename allocator_traits<_Allocator>::size_type = 0,
+    _Hash                                            = _Hash(),
+    _Pred                                            = _Pred(),
+    _Allocator = _Allocator()) -> unordered_multimap<remove_const_t<_Key>, _Tp, _Hash, _Pred, _Allocator>;
+
+template <class _InputIterator,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multimap(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Allocator)
+    -> unordered_multimap<__iter_key_type<_InputIterator>,
+                          __iter_mapped_type<_InputIterator>,
+                          hash<__iter_key_type<_InputIterator>>,
+                          equal_to<__iter_key_type<_InputIterator>>,
+                          _Allocator>;
+
+template <class _InputIterator,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multimap(_InputIterator, _InputIterator, _Allocator)
+    -> unordered_multimap<__iter_key_type<_InputIterator>,
+                          __iter_mapped_type<_InputIterator>,
+                          hash<__iter_key_type<_InputIterator>>,
+                          equal_to<__iter_key_type<_InputIterator>>,
+                          _Allocator>;
+
+template <class _InputIterator,
+          class _Hash,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<!__is_allocator<_Hash>::value>,
+          class = enable_if_t<!is_integral<_Hash>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multimap(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
+    -> unordered_multimap<__iter_key_type<_InputIterator>,
+                          __iter_mapped_type<_InputIterator>,
+                          _Hash,
+                          equal_to<__iter_key_type<_InputIterator>>,
+                          _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+
+template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multimap(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Allocator)
+    -> unordered_multimap<__range_key_type<_Range>,
+                          __range_mapped_type<_Range>,
+                          hash<__range_key_type<_Range>>,
+                          equal_to<__range_key_type<_Range>>,
+                          _Allocator>;
+
+template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multimap(from_range_t, _Range&&, _Allocator)
+    -> unordered_multimap<__range_key_type<_Range>,
+                          __range_mapped_type<_Range>,
+                          hash<__range_key_type<_Range>>,
+                          equal_to<__range_key_type<_Range>>,
+                          _Allocator>;
+
+template <ranges::input_range _Range,
+          class _Hash,
+          class _Allocator,
+          class = enable_if_t<!__is_allocator<_Hash>::value>,
+          class = enable_if_t<!is_integral<_Hash>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multimap(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
+    -> unordered_multimap<__range_key_type<_Range>,
+                          __range_mapped_type<_Range>,
+                          _Hash,
+                          equal_to<__range_key_type<_Range>>,
+                          _Allocator>;
+
+#  endif
+
+template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multimap(initializer_list<pair<_Key, _Tp>>, typename allocator_traits<_Allocator>::size_type, _Allocator)
+    -> unordered_multimap<remove_const_t<_Key>,
+                          _Tp,
+                          hash<remove_const_t<_Key>>,
+                          equal_to<remove_const_t<_Key>>,
+                          _Allocator>;
+
+template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multimap(initializer_list<pair<_Key, _Tp>>, _Allocator)
+    -> unordered_multimap<remove_const_t<_Key>,
+                          _Tp,
+                          hash<remove_const_t<_Key>>,
+                          equal_to<remove_const_t<_Key>>,
+                          _Allocator>;
+
+template <class _Key,
+          class _Tp,
+          class _Hash,
+          class _Allocator,
+          class = enable_if_t<!__is_allocator<_Hash>::value>,
+          class = enable_if_t<!is_integral<_Hash>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multimap(
+    initializer_list<pair<_Key, _Tp>>, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
+    -> unordered_multimap<remove_const_t<_Key>, _Tp, _Hash, equal_to<remove_const_t<_Key>>, _Allocator>;
+#endif
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
+    size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_multi(__n);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
+    size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
+    : __table_(__hf, __eql, typename __table::allocator_type(__a)) {
+  __table_.__rehash_multi(__n);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(_InputIterator __first, _InputIterator __last) {
+  insert(__first, __last);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
+    _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_multi(__n);
+  insert(__first, __last);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
+    _InputIterator __first,
+    _InputIterator __last,
+    size_type __n,
+    const hasher& __hf,
+    const key_equal& __eql,
+    const allocator_type& __a)
+    : __table_(__hf, __eql, typename __table::allocator_type(__a)) {
+  __table_.__rehash_multi(__n);
+  insert(__first, __last);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(const allocator_type& __a)
+    : __table_(typename __table::allocator_type(__a)) {}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(const unordered_multimap& __u)
+    : __table_(__u.__table_) {
+  __table_.__rehash_multi(__u.bucket_count());
+  insert(__u.begin(), __u.end());
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
+    const unordered_multimap& __u, const allocator_type& __a)
+    : __table_(__u.__table_, typename __table::allocator_type(__a)) {
+  __table_.__rehash_multi(__u.bucket_count());
+  insert(__u.begin(), __u.end());
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(unordered_multimap&& __u)
+    _NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
+    : __table_(std::move(__u.__table_)) {}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
+    unordered_multimap&& __u, const allocator_type& __a)
+    : __table_(std::move(__u.__table_), typename __table::allocator_type(__a)) {
+  if (__a != __u.get_allocator()) {
+    iterator __i = __u.begin();
+    while (__u.size() != 0) {
+      __table_.__insert_multi(__u.__table_.remove((__i++).__i_)->__get_value().__move());
+    }
+  }
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(initializer_list<value_type> __il) {
+  insert(__il.begin(), __il.end());
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
+    initializer_list<value_type> __il, size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_multi(__n);
+  insert(__il.begin(), __il.end());
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
+    initializer_list<value_type> __il,
+    size_type __n,
+    const hasher& __hf,
+    const key_equal& __eql,
+    const allocator_type& __a)
+    : __table_(__hf, __eql, typename __table::allocator_type(__a)) {
+  __table_.__rehash_multi(__n);
+  insert(__il.begin(), __il.end());
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>&
+unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(unordered_multimap&& __u)
+    _NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
+  __table_ = std::move(__u.__table_);
+  return *this;
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>&
+unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {
+  __table_.__assign_multi(__il.begin(), __il.end());
+  return *this;
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+inline void unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::insert(_InputIterator __first, _InputIterator __last) {
+  for (; __first != __last; ++__first)
+    __table_.__insert_multi(*__first);
+}
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __x, unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI typename unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::size_type
+erase_if(unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __c, _Predicate __pred) {
+  return std::__libcpp_erase_if_container(__c, __pred);
+}
+#endif
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+_LIBCPP_HIDE_FROM_ABI bool operator==(const unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
+                                      const unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __y) {
+  if (__x.size() != __y.size())
+    return false;
+  typedef typename unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::const_iterator const_iterator;
+  typedef pair<const_iterator, const_iterator> _EqRng;
+  for (const_iterator __i = __x.begin(), __ex = __x.end(); __i != __ex;) {
+    _EqRng __xeq = __x.equal_range(__i->first);
+    _EqRng __yeq = __y.equal_range(__i->first);
+    if (std::distance(__xeq.first, __xeq.second) != std::distance(__yeq.first, __yeq.second) ||
+        !std::is_permutation(__xeq.first, __xeq.second, __yeq.first))
+      return false;
+    __i = __xeq.second;
+  }
+  return true;
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
+                                             const unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __y) {
+  return !(__x == __y);
+}
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace pmr {
+template <class _KeyT, class _ValueT, class _HashT = std::hash<_KeyT>, class _PredT = std::equal_to<_KeyT>>
+using unordered_map _LIBCPP_AVAILABILITY_PMR =
+    std::unordered_map<_KeyT, _ValueT, _HashT, _PredT, polymorphic_allocator<std::pair<const _KeyT, _ValueT>>>;
+
+template <class _KeyT, class _ValueT, class _HashT = std::hash<_KeyT>, class _PredT = std::equal_to<_KeyT>>
+using unordered_multimap _LIBCPP_AVAILABILITY_PMR =
+    std::unordered_multimap<_KeyT, _ValueT, _HashT, _PredT, polymorphic_allocator<std::pair<const _KeyT, _ValueT>>>;
+} // namespace pmr
+_LIBCPP_END_NAMESPACE_STD
+#endif
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <algorithm>
+#  include <bit>
+#  include <concepts>
+#  include <cstdlib>
+#  include <iterator>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_UNORDERED_MAP
diff --git a/libcxx/include/__cxx03/unordered_set b/libcxx/include/__cxx03/unordered_set
new file mode 100644
index 000000000000..3297294a893f
--- /dev/null
+++ b/libcxx/include/__cxx03/unordered_set
@@ -0,0 +1,1811 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_UNORDERED_SET
+#define _LIBCPP_UNORDERED_SET
+
+// clang-format off
+
+/*
+
+    unordered_set synopsis
+
+#include <initializer_list>
+
+namespace std
+{
+
+template <class Value, class Hash = hash<Value>, class Pred = equal_to<Value>,
+          class Alloc = allocator<Value>>
+class unordered_set
+{
+public:
+    // types
+    typedef Value                                                      key_type;
+    typedef key_type                                                   value_type;
+    typedef Hash                                                       hasher;
+    typedef Pred                                                       key_equal;
+    typedef Alloc                                                      allocator_type;
+    typedef value_type&                                                reference;
+    typedef const value_type&                                          const_reference;
+    typedef typename allocator_traits<allocator_type>::pointer         pointer;
+    typedef typename allocator_traits<allocator_type>::const_pointer   const_pointer;
+    typedef typename allocator_traits<allocator_type>::size_type       size_type;
+    typedef typename allocator_traits<allocator_type>::difference_type difference_type;
+
+    typedef /unspecified/ iterator;
+    typedef /unspecified/ const_iterator;
+    typedef /unspecified/ local_iterator;
+    typedef /unspecified/ const_local_iterator;
+
+    typedef unspecified node_type unspecified;                            // C++17
+    typedef INSERT_RETURN_TYPE<iterator, node_type> insert_return_type;   // C++17
+
+    unordered_set()
+        noexcept(
+            is_nothrow_default_constructible<hasher>::value &&
+            is_nothrow_default_constructible<key_equal>::value &&
+            is_nothrow_default_constructible<allocator_type>::value);
+    explicit unordered_set(size_type n, const hasher& hf = hasher(),
+                           const key_equal& eql = key_equal(),
+                           const allocator_type& a = allocator_type());
+    template <class InputIterator>
+        unordered_set(InputIterator f, InputIterator l,
+                      size_type n = 0, const hasher& hf = hasher(),
+                      const key_equal& eql = key_equal(),
+                      const allocator_type& a = allocator_type());
+    template<container-compatible-range<value_type> R>
+      unordered_set(from_range_t, R&& rg, size_type n = see below,
+        const hasher& hf = hasher(), const key_equal& eql = key_equal(),
+        const allocator_type& a = allocator_type()); // C++23
+    explicit unordered_set(const allocator_type&);
+    unordered_set(const unordered_set&);
+    unordered_set(const unordered_set&, const Allocator&);
+    unordered_set(unordered_set&&)
+        noexcept(
+            is_nothrow_move_constructible<hasher>::value &&
+            is_nothrow_move_constructible<key_equal>::value &&
+            is_nothrow_move_constructible<allocator_type>::value);
+    unordered_set(unordered_set&&, const Allocator&);
+    unordered_set(initializer_list<value_type>, size_type n = 0,
+                  const hasher& hf = hasher(), const key_equal& eql = key_equal(),
+                  const allocator_type& a = allocator_type());
+    unordered_set(size_type n, const allocator_type& a); // C++14
+    unordered_set(size_type n, const hasher& hf, const allocator_type& a); // C++14
+    template <class InputIterator>
+      unordered_set(InputIterator f, InputIterator l, size_type n, const allocator_type& a); // C++14
+    template <class InputIterator>
+      unordered_set(InputIterator f, InputIterator l, size_type n,
+                    const hasher& hf,  const allocator_type& a); // C++14
+    template<container-compatible-range<value_type> R>
+      unordered_set(from_range_t, R&& rg, size_type n, const allocator_type& a)
+        : unordered_set(from_range, std::forward<R>(rg), n, hasher(), key_equal(), a) { } // C++23
+    template<container-compatible-range<value_type> R>
+      unordered_set(from_range_t, R&& rg, size_type n, const hasher& hf, const allocator_type& a)
+        : unordered_set(from_range, std::forward<R>(rg), n, hf, key_equal(), a) { }       // C++23
+    unordered_set(initializer_list<value_type> il, size_type n, const allocator_type& a); // C++14
+    unordered_set(initializer_list<value_type> il, size_type n,
+                  const hasher& hf,  const allocator_type& a); // C++14
+    ~unordered_set();
+    unordered_set& operator=(const unordered_set&);
+    unordered_set& operator=(unordered_set&&)
+        noexcept(
+            allocator_type::propagate_on_container_move_assignment::value &&
+            is_nothrow_move_assignable<allocator_type>::value &&
+            is_nothrow_move_assignable<hasher>::value &&
+            is_nothrow_move_assignable<key_equal>::value);
+    unordered_set& operator=(initializer_list<value_type>);
+
+    allocator_type get_allocator() const noexcept;
+
+    bool      empty() const noexcept;
+    size_type size() const noexcept;
+    size_type max_size() const noexcept;
+
+    iterator       begin() noexcept;
+    iterator       end() noexcept;
+    const_iterator begin()  const noexcept;
+    const_iterator end()    const noexcept;
+    const_iterator cbegin() const noexcept;
+    const_iterator cend()   const noexcept;
+
+    template <class... Args>
+        pair<iterator, bool> emplace(Args&&... args);
+    template <class... Args>
+        iterator emplace_hint(const_iterator position, Args&&... args);
+    pair<iterator, bool> insert(const value_type& obj);
+    pair<iterator, bool> insert(value_type&& obj);
+    iterator insert(const_iterator hint, const value_type& obj);
+    iterator insert(const_iterator hint, value_type&& obj);
+    template <class InputIterator>
+        void insert(InputIterator first, InputIterator last);
+    template<container-compatible-range<value_type> R>
+      void insert_range(R&& rg);                                      // C++23
+    void insert(initializer_list<value_type>);
+
+    node_type extract(const_iterator position);                       // C++17
+    node_type extract(const key_type& x);                             // C++17
+    insert_return_type insert(node_type&& nh);                        // C++17
+    iterator           insert(const_iterator hint, node_type&& nh);   // C++17
+
+    iterator erase(const_iterator position);
+    iterator erase(iterator position);  // C++14
+    size_type erase(const key_type& k);
+    iterator erase(const_iterator first, const_iterator last);
+    void clear() noexcept;
+
+    template<class H2, class P2>
+      void merge(unordered_set<Key, H2, P2, Allocator>& source);         // C++17
+    template<class H2, class P2>
+      void merge(unordered_set<Key, H2, P2, Allocator>&& source);        // C++17
+    template<class H2, class P2>
+      void merge(unordered_multiset<Key, H2, P2, Allocator>& source);    // C++17
+    template<class H2, class P2>
+      void merge(unordered_multiset<Key, H2, P2, Allocator>&& source);   // C++17
+
+    void swap(unordered_set&)
+       noexcept(allocator_traits<Allocator>::is_always_equal::value &&
+                 noexcept(swap(declval<hasher&>(), declval<hasher&>())) &&
+                 noexcept(swap(declval<key_equal&>(), declval<key_equal&>()))); // C++17
+
+    hasher hash_function() const;
+    key_equal key_eq() const;
+
+    iterator       find(const key_type& k);
+    const_iterator find(const key_type& k) const;
+    template<typename K>
+        iterator find(const K& x);              // C++20
+    template<typename K>
+        const_iterator find(const K& x) const;  // C++20
+    size_type count(const key_type& k) const;
+    template<typename K>
+        size_type count(const K& k) const; // C++20
+    bool contains(const key_type& k) const; // C++20
+    template<typename K>
+        bool contains(const K& k) const; // C++20
+    pair<iterator, iterator>             equal_range(const key_type& k);
+    pair<const_iterator, const_iterator> equal_range(const key_type& k) const;
+    template<typename K>
+        pair<iterator, iterator>             equal_range(const K& k); // C++20
+    template<typename K>
+        pair<const_iterator, const_iterator> equal_range(const K& k) const; // C++20
+
+    size_type bucket_count() const noexcept;
+    size_type max_bucket_count() const noexcept;
+
+    size_type bucket_size(size_type n) const;
+    size_type bucket(const key_type& k) const;
+
+    local_iterator       begin(size_type n);
+    local_iterator       end(size_type n);
+    const_local_iterator begin(size_type n) const;
+    const_local_iterator end(size_type n) const;
+    const_local_iterator cbegin(size_type n) const;
+    const_local_iterator cend(size_type n) const;
+
+    float load_factor() const noexcept;
+    float max_load_factor() const noexcept;
+    void max_load_factor(float z);
+    void rehash(size_type n);
+    void reserve(size_type n);
+};
+
+template<class InputIterator,
+    class Hash = hash<typename iterator_traits<InputIterator>::value_type>,
+    class Pred = equal_to<typename iterator_traits<InputIterator>::value_type>,
+    class Allocator = allocator<typename iterator_traits<InputIterator>::value_type>>
+unordered_set(InputIterator, InputIterator, typename see below::size_type = see below,
+    Hash = Hash(), Pred = Pred(), Allocator = Allocator())
+  -> unordered_set<typename iterator_traits<InputIterator>::value_type,
+        Hash, Pred, Allocator>; // C++17
+
+template<ranges::input_range R,
+         class Hash = hash<ranges::range_value_t<R>>,
+         class Pred = equal_to<ranges::range_value_t<R>>,
+         class Allocator = allocator<ranges::range_value_t<R>>>
+  unordered_set(from_range_t, R&&, typename see below::size_type = see below, Hash = Hash(), Pred = Pred(), Allocator = Allocator())
+    -> unordered_set<ranges::range_value_t<R>, Hash, Pred, Allocator>; // C++23
+
+template<class T, class Hash = hash<T>,
+          class Pred = equal_to<T>, class Allocator = allocator<T>>
+unordered_set(initializer_list<T>, typename see below::size_type = see below,
+    Hash = Hash(), Pred = Pred(), Allocator = Allocator())
+  -> unordered_set<T, Hash, Pred, Allocator>; // C++17
+
+template<class InputIterator,  class Allocator>
+unordered_set(InputIterator, InputIterator, typename see below::size_type, Allocator)
+  -> unordered_set<typename iterator_traits<InputIterator>::value_type,
+        hash<typename iterator_traits<InputIterator>::value_type>,
+        equal_to<typename iterator_traits<InputIterator>::value_type>,
+        Allocator>; // C++17
+
+template<class InputIterator, class Hash, class Allocator>
+unordered_set(InputIterator, InputIterator, typename see below::size_type,
+    Hash, Allocator)
+  -> unordered_set<typename iterator_traits<InputIterator>::value_type, Hash,
+        equal_to<typename iterator_traits<InputIterator>::value_type>,
+        Allocator>; // C++17
+
+template<ranges::input_range R, class Allocator>
+  unordered_set(from_range_t, R&&, typename see below::size_type, Allocator)
+    -> unordered_set<ranges::range_value_t<R>, hash<ranges::range_value_t<R>>,
+                      equal_to<ranges::range_value_t<R>>, Allocator>; // C++23
+
+template<ranges::input_range R, class Allocator>
+  unordered_set(from_range_t, R&&, Allocator)
+    -> unordered_set<ranges::range_value_t<R>, hash<ranges::range_value_t<R>>,
+                      equal_to<ranges::range_value_t<R>>, Allocator>; // C++23
+
+template<ranges::input_range R, class Hash, class Allocator>
+  unordered_set(from_range_t, R&&, typename see below::size_type, Hash, Allocator)
+    -> unordered_set<ranges::range_value_t<R>, Hash,
+                      equal_to<ranges::range_value_t<R>>, Allocator>; // C++23
+
+template<class T, class Allocator>
+unordered_set(initializer_list<T>, typename see below::size_type, Allocator)
+  -> unordered_set<T, hash<T>, equal_to<T>, Allocator>; // C++17
+
+template<class T, class Hash, class Allocator>
+unordered_set(initializer_list<T>, typename see below::size_type, Hash, Allocator)
+  -> unordered_set<T, Hash, equal_to<T>, Allocator>; // C++17
+
+template <class Value, class Hash, class Pred, class Alloc>
+    void swap(unordered_set<Value, Hash, Pred, Alloc>& x,
+              unordered_set<Value, Hash, Pred, Alloc>& y)
+              noexcept(noexcept(x.swap(y)));
+
+template <class Value, class Hash, class Pred, class Alloc>
+    bool
+    operator==(const unordered_set<Value, Hash, Pred, Alloc>& x,
+               const unordered_set<Value, Hash, Pred, Alloc>& y);
+
+template <class Value, class Hash, class Pred, class Alloc>
+    bool
+    operator!=(const unordered_set<Value, Hash, Pred, Alloc>& x,
+               const unordered_set<Value, Hash, Pred, Alloc>& y); // removed in C++20
+
+template <class Value, class Hash = hash<Value>, class Pred = equal_to<Value>,
+          class Alloc = allocator<Value>>
+class unordered_multiset
+{
+public:
+    // types
+    typedef Value                                                      key_type;
+    typedef key_type                                                   value_type;
+    typedef Hash                                                       hasher;
+    typedef Pred                                                       key_equal;
+    typedef Alloc                                                      allocator_type;
+    typedef value_type&                                                reference;
+    typedef const value_type&                                          const_reference;
+    typedef typename allocator_traits<allocator_type>::pointer         pointer;
+    typedef typename allocator_traits<allocator_type>::const_pointer   const_pointer;
+    typedef typename allocator_traits<allocator_type>::size_type       size_type;
+    typedef typename allocator_traits<allocator_type>::difference_type difference_type;
+
+    typedef /unspecified/ iterator;
+    typedef /unspecified/ const_iterator;
+    typedef /unspecified/ local_iterator;
+    typedef /unspecified/ const_local_iterator;
+
+    typedef unspecified node_type unspecified;   // C++17
+
+    unordered_multiset()
+        noexcept(
+            is_nothrow_default_constructible<hasher>::value &&
+            is_nothrow_default_constructible<key_equal>::value &&
+            is_nothrow_default_constructible<allocator_type>::value);
+    explicit unordered_multiset(size_type n, const hasher& hf = hasher(),
+                           const key_equal& eql = key_equal(),
+                           const allocator_type& a = allocator_type());
+    template <class InputIterator>
+        unordered_multiset(InputIterator f, InputIterator l,
+                      size_type n = 0, const hasher& hf = hasher(),
+                      const key_equal& eql = key_equal(),
+                      const allocator_type& a = allocator_type());
+    template<container-compatible-range<value_type> R>
+      unordered_multiset(from_range_t, R&& rg, size_type n = see below,
+        const hasher& hf = hasher(), const key_equal& eql = key_equal(),
+        const allocator_type& a = allocator_type()); // C++23
+    explicit unordered_multiset(const allocator_type&);
+    unordered_multiset(const unordered_multiset&);
+    unordered_multiset(const unordered_multiset&, const Allocator&);
+    unordered_multiset(unordered_multiset&&)
+        noexcept(
+            is_nothrow_move_constructible<hasher>::value &&
+            is_nothrow_move_constructible<key_equal>::value &&
+            is_nothrow_move_constructible<allocator_type>::value);
+    unordered_multiset(unordered_multiset&&, const Allocator&);
+    unordered_multiset(initializer_list<value_type>, size_type n = /see below/,
+                  const hasher& hf = hasher(), const key_equal& eql = key_equal(),
+                  const allocator_type& a = allocator_type());
+    unordered_multiset(size_type n, const allocator_type& a); // C++14
+    unordered_multiset(size_type n, const hasher& hf, const allocator_type& a); // C++14
+    template <class InputIterator>
+      unordered_multiset(InputIterator f, InputIterator l, size_type n, const allocator_type& a); // C++14
+    template <class InputIterator>
+      unordered_multiset(InputIterator f, InputIterator l, size_type n,
+                         const hasher& hf, const allocator_type& a); // C++14
+    template<container-compatible-range<value_type> R>
+      unordered_multiset(from_range_t, R&& rg, size_type n, const allocator_type& a)
+        : unordered_multiset(from_range, std::forward<R>(rg), n, hasher(), key_equal(), a) { } // C++23
+    template<container-compatible-range<value_type> R>
+      unordered_multiset(from_range_t, R&& rg, size_type n, const hasher& hf, const allocator_type& a)
+        : unordered_multiset(from_range, std::forward<R>(rg), n, hf, key_equal(), a) { }       // C++23
+    unordered_multiset(initializer_list<value_type> il, size_type n, const allocator_type& a); // C++14
+    unordered_multiset(initializer_list<value_type> il, size_type n,
+                       const hasher& hf,  const allocator_type& a); // C++14
+    ~unordered_multiset();
+    unordered_multiset& operator=(const unordered_multiset&);
+    unordered_multiset& operator=(unordered_multiset&&)
+        noexcept(
+            allocator_type::propagate_on_container_move_assignment::value &&
+            is_nothrow_move_assignable<allocator_type>::value &&
+            is_nothrow_move_assignable<hasher>::value &&
+            is_nothrow_move_assignable<key_equal>::value);
+    unordered_multiset& operator=(initializer_list<value_type>);
+
+    allocator_type get_allocator() const noexcept;
+
+    bool      empty() const noexcept;
+    size_type size() const noexcept;
+    size_type max_size() const noexcept;
+
+    iterator       begin() noexcept;
+    iterator       end() noexcept;
+    const_iterator begin()  const noexcept;
+    const_iterator end()    const noexcept;
+    const_iterator cbegin() const noexcept;
+    const_iterator cend()   const noexcept;
+
+    template <class... Args>
+        iterator emplace(Args&&... args);
+    template <class... Args>
+        iterator emplace_hint(const_iterator position, Args&&... args);
+    iterator insert(const value_type& obj);
+    iterator insert(value_type&& obj);
+    iterator insert(const_iterator hint, const value_type& obj);
+    iterator insert(const_iterator hint, value_type&& obj);
+    template <class InputIterator>
+        void insert(InputIterator first, InputIterator last);
+    template<container-compatible-range<value_type> R>
+      void insert_range(R&& rg);                            // C++23
+    void insert(initializer_list<value_type>);
+
+    node_type extract(const_iterator position);             // C++17
+    node_type extract(const key_type& x);                   // C++17
+    iterator insert(node_type&& nh);                        // C++17
+    iterator insert(const_iterator hint, node_type&& nh);   // C++17
+
+    iterator erase(const_iterator position);
+    iterator erase(iterator position);  // C++14
+    size_type erase(const key_type& k);
+    iterator erase(const_iterator first, const_iterator last);
+    void clear() noexcept;
+
+    template<class H2, class P2>
+      void merge(unordered_multiset<Key, H2, P2, Allocator>& source);    // C++17
+    template<class H2, class P2>
+      void merge(unordered_multiset<Key, H2, P2, Allocator>&& source);   // C++17
+    template<class H2, class P2>
+      void merge(unordered_set<Key, H2, P2, Allocator>& source);         // C++17
+    template<class H2, class P2>
+      void merge(unordered_set<Key, H2, P2, Allocator>&& source);        // C++17
+
+    void swap(unordered_multiset&)
+       noexcept(allocator_traits<Allocator>::is_always_equal::value &&
+                 noexcept(swap(declval<hasher&>(), declval<hasher&>())) &&
+                 noexcept(swap(declval<key_equal&>(), declval<key_equal&>()))); // C++17
+
+    hasher hash_function() const;
+    key_equal key_eq() const;
+
+    iterator       find(const key_type& k);
+    const_iterator find(const key_type& k) const;
+    template<typename K>
+        iterator find(const K& x);              // C++20
+    template<typename K>
+        const_iterator find(const K& x) const;  // C++20
+    size_type count(const key_type& k) const;
+    template<typename K>
+        size_type count(const K& k) const; // C++20
+    bool contains(const key_type& k) const; // C++20
+    template<typename K>
+        bool contains(const K& k) const; // C++20
+    pair<iterator, iterator>             equal_range(const key_type& k);
+    pair<const_iterator, const_iterator> equal_range(const key_type& k) const;
+    template<typename K>
+        pair<iterator, iterator>             equal_range(const K& k); // C++20
+    template<typename K>
+        pair<const_iterator, const_iterator> equal_range(const K& k) const; // C++20
+
+    size_type bucket_count() const noexcept;
+    size_type max_bucket_count() const noexcept;
+
+    size_type bucket_size(size_type n) const;
+    size_type bucket(const key_type& k) const;
+
+    local_iterator       begin(size_type n);
+    local_iterator       end(size_type n);
+    const_local_iterator begin(size_type n) const;
+    const_local_iterator end(size_type n) const;
+    const_local_iterator cbegin(size_type n) const;
+    const_local_iterator cend(size_type n) const;
+
+    float load_factor() const noexcept;
+    float max_load_factor() const noexcept;
+    void max_load_factor(float z);
+    void rehash(size_type n);
+    void reserve(size_type n);
+};
+
+template<class InputIterator,
+    class Hash = hash<typename iterator_traits<InputIterator>::value_type>,
+    class Pred = equal_to<typename iterator_traits<InputIterator>::value_type>,
+    class Allocator = allocator<typename iterator_traits<InputIterator>::value_type>>
+unordered_multiset(InputIterator, InputIterator, see below::size_type = see below,
+    Hash = Hash(), Pred = Pred(), Allocator = Allocator())
+  -> unordered_multiset<typename iterator_traits<InputIterator>::value_type,
+        Hash, Pred, Allocator>; // C++17
+
+template<ranges::input_range R,
+         class Hash = hash<ranges::range_value_t<R>>,
+         class Pred = equal_to<ranges::range_value_t<R>>,
+         class Allocator = allocator<ranges::range_value_t<R>>>
+  unordered_multiset(from_range_t, R&&, typename see below::size_type = see below, Hash = Hash(), Pred = Pred(), Allocator = Allocator())
+    -> unordered_multiset<ranges::range_value_t<R>, Hash, Pred, Allocator>; // C++23
+
+template<class T, class Hash = hash<T>,
+          class Pred = equal_to<T>, class Allocator = allocator<T>>
+unordered_multiset(initializer_list<T>, typename see below::size_type = see below,
+    Hash = Hash(), Pred = Pred(), Allocator = Allocator())
+  -> unordered_multiset<T, Hash, Pred, Allocator>; // C++17
+
+template<class InputIterator,  class Allocator>
+unordered_multiset(InputIterator, InputIterator, typename see below::size_type, Allocator)
+  -> unordered_multiset<typename iterator_traits<InputIterator>::value_type,
+        hash<typename iterator_traits<InputIterator>::value_type>,
+        equal_to<typename iterator_traits<InputIterator>::value_type>,
+        Allocator>; // C++17
+
+template<class InputIterator,  class Hash, class Allocator>
+unordered_multiset(InputIterator, InputIterator, typename see below::size_type,
+    Hash, Allocator)
+  -> unordered_multiset<typename iterator_traits<InputIterator>::value_type, Hash,
+        equal_to<typename iterator_traits<InputIterator>::value_type>, Allocator>; // C++17
+
+template<ranges::input_range R, class Allocator>
+  unordered_multiset(from_range_t, R&&, typename see below::size_type, Allocator)
+    -> unordered_multiset<ranges::range_value_t<R>, hash<ranges::range_value_t<R>>,
+                      equal_to<ranges::range_value_t<R>>, Allocator>; // C++23
+
+template<ranges::input_range R, class Allocator>
+  unordered_multiset(from_range_t, R&&, Allocator)
+    -> unordered_multiset<ranges::range_value_t<R>, hash<ranges::range_value_t<R>>,
+                      equal_to<ranges::range_value_t<R>>, Allocator>; // C++23
+
+template<ranges::input_range R, class Hash, class Allocator>
+  unordered_multiset(from_range_t, R&&, typename see below::size_type, Hash, Allocator)
+    -> unordered_multiset<ranges::range_value_t<R>, Hash,
+                      equal_to<ranges::range_value_t<R>>, Allocator>; // C++23
+
+template<class T, class Allocator>
+unordered_multiset(initializer_list<T>, typename see below::size_type, Allocator)
+  -> unordered_multiset<T, hash<T>, equal_to<T>, Allocator>; // C++17
+
+template<class T, class Hash, class Allocator>
+unordered_multiset(initializer_list<T>, typename see below::size_type, Hash, Allocator)
+  -> unordered_multiset<T, Hash, equal_to<T>, Allocator>; // C++17
+
+template <class Value, class Hash, class Pred, class Alloc>
+    void swap(unordered_multiset<Value, Hash, Pred, Alloc>& x,
+              unordered_multiset<Value, Hash, Pred, Alloc>& y)
+              noexcept(noexcept(x.swap(y)));
+
+template <class K, class T, class H, class P, class A, class Predicate>
+    typename unordered_set<K, T, H, P, A>::size_type
+    erase_if(unordered_set<K, T, H, P, A>& c, Predicate pred);       // C++20
+
+template <class K, class T, class H, class P, class A, class Predicate>
+    typename unordered_multiset<K, T, H, P, A>::size_type
+    erase_if(unordered_multiset<K, T, H, P, A>& c, Predicate pred);  // C++20
+
+
+template <class Value, class Hash, class Pred, class Alloc>
+    bool
+    operator==(const unordered_multiset<Value, Hash, Pred, Alloc>& x,
+               const unordered_multiset<Value, Hash, Pred, Alloc>& y);
+
+template <class Value, class Hash, class Pred, class Alloc>
+    bool
+    operator!=(const unordered_multiset<Value, Hash, Pred, Alloc>& x,
+               const unordered_multiset<Value, Hash, Pred, Alloc>& y); // removed in C++20
+}  // std
+
+*/
+
+// clang-format on
+
+#include <__algorithm/is_permutation.h>
+#include <__assert>
+#include <__config>
+#include <__functional/is_transparent.h>
+#include <__functional/operations.h>
+#include <__hash_table>
+#include <__iterator/distance.h>
+#include <__iterator/erase_if_container.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/ranges_iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__memory/allocator.h>
+#include <__memory_resource/polymorphic_allocator.h>
+#include <__node_handle>
+#include <__ranges/concepts.h>
+#include <__ranges/container_compatible_range.h>
+#include <__ranges/from_range.h>
+#include <__type_traits/is_allocator.h>
+#include <__utility/forward.h>
+#include <version>
+
+// standard-mandated includes
+
+// [iterator.range]
+#include <__iterator/access.h>
+#include <__iterator/data.h>
+#include <__iterator/empty.h>
+#include <__iterator/reverse_access.h>
+#include <__iterator/size.h>
+
+// [unord.set.syn]
+#include <compare>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+class unordered_multiset;
+
+template <class _Value, class _Hash = hash<_Value>, class _Pred = equal_to<_Value>, class _Alloc = allocator<_Value> >
+class _LIBCPP_TEMPLATE_VIS unordered_set {
+public:
+  // types
+  typedef _Value key_type;
+  typedef key_type value_type;
+  typedef __type_identity_t<_Hash> hasher;
+  typedef __type_identity_t<_Pred> key_equal;
+  typedef __type_identity_t<_Alloc> allocator_type;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+  static_assert(__check_valid_allocator<allocator_type>::value, "");
+  static_assert(is_same<value_type, typename allocator_type::value_type>::value,
+                "Allocator::value_type must be same type as value_type");
+
+private:
+  typedef __hash_table<value_type, hasher, key_equal, allocator_type> __table;
+
+  __table __table_;
+
+public:
+  typedef typename __table::pointer pointer;
+  typedef typename __table::const_pointer const_pointer;
+  typedef typename __table::size_type size_type;
+  typedef typename __table::difference_type difference_type;
+
+  typedef typename __table::const_iterator iterator;
+  typedef typename __table::const_iterator const_iterator;
+  typedef typename __table::const_local_iterator local_iterator;
+  typedef typename __table::const_local_iterator const_local_iterator;
+
+#if _LIBCPP_STD_VER >= 17
+  typedef __set_node_handle<typename __table::__node, allocator_type> node_type;
+  typedef __insert_return_type<iterator, node_type> insert_return_type;
+#endif
+
+  template <class _Value2, class _Hash2, class _Pred2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_set;
+  template <class _Value2, class _Hash2, class _Pred2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_multiset;
+
+  _LIBCPP_HIDE_FROM_ABI unordered_set() _NOEXCEPT_(is_nothrow_default_constructible<__table>::value) {}
+  explicit _LIBCPP_HIDE_FROM_ABI
+  unordered_set(size_type __n, const hasher& __hf = hasher(), const key_equal& __eql = key_equal());
+#if _LIBCPP_STD_VER >= 14
+  inline _LIBCPP_HIDE_FROM_ABI unordered_set(size_type __n, const allocator_type& __a)
+      : unordered_set(__n, hasher(), key_equal(), __a) {}
+  inline _LIBCPP_HIDE_FROM_ABI unordered_set(size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_set(__n, __hf, key_equal(), __a) {}
+#endif
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_set(size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI unordered_set(_InputIterator __first, _InputIterator __last);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_set(_InputIterator __first,
+                _InputIterator __last,
+                size_type __n,
+                const hasher& __hf     = hasher(),
+                const key_equal& __eql = key_equal());
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI unordered_set(
+      _InputIterator __first,
+      _InputIterator __last,
+      size_type __n,
+      const hasher& __hf,
+      const key_equal& __eql,
+      const allocator_type& __a);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI unordered_set(
+      from_range_t,
+      _Range&& __range,
+      size_type __n             = /*implementation-defined*/ 0,
+      const hasher& __hf        = hasher(),
+      const key_equal& __eql    = key_equal(),
+      const allocator_type& __a = allocator_type())
+      : __table_(__hf, __eql, __a) {
+    if (__n > 0) {
+      __table_.__rehash_unique(__n);
+    }
+    insert_range(std::forward<_Range>(__range));
+  }
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+  template <class _InputIterator>
+  inline _LIBCPP_HIDE_FROM_ABI
+  unordered_set(_InputIterator __first, _InputIterator __last, size_type __n, const allocator_type& __a)
+      : unordered_set(__first, __last, __n, hasher(), key_equal(), __a) {}
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI unordered_set(
+      _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_set(__first, __last, __n, __hf, key_equal(), __a) {}
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI unordered_set(from_range_t, _Range&& __range, size_type __n, const allocator_type& __a)
+      : unordered_set(from_range, std::forward<_Range>(__range), __n, hasher(), key_equal(), __a) {}
+
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_set(from_range_t, _Range&& __range, size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_set(from_range, std::forward<_Range>(__range), __n, __hf, key_equal(), __a) {}
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI explicit unordered_set(const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI unordered_set(const unordered_set& __u);
+  _LIBCPP_HIDE_FROM_ABI unordered_set(const unordered_set& __u, const allocator_type& __a);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI unordered_set(unordered_set&& __u) _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_set(unordered_set&& __u, const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI unordered_set(initializer_list<value_type> __il);
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_set(initializer_list<value_type> __il,
+                size_type __n,
+                const hasher& __hf     = hasher(),
+                const key_equal& __eql = key_equal());
+  _LIBCPP_HIDE_FROM_ABI unordered_set(
+      initializer_list<value_type> __il,
+      size_type __n,
+      const hasher& __hf,
+      const key_equal& __eql,
+      const allocator_type& __a);
+#  if _LIBCPP_STD_VER >= 14
+  inline _LIBCPP_HIDE_FROM_ABI
+  unordered_set(initializer_list<value_type> __il, size_type __n, const allocator_type& __a)
+      : unordered_set(__il, __n, hasher(), key_equal(), __a) {}
+  inline _LIBCPP_HIDE_FROM_ABI
+  unordered_set(initializer_list<value_type> __il, size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_set(__il, __n, __hf, key_equal(), __a) {}
+#  endif
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI ~unordered_set() {
+    static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Value, _Hash, _Pred>(0)), "");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI unordered_set& operator=(const unordered_set& __u) {
+    __table_ = __u.__table_;
+    return *this;
+  }
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI unordered_set& operator=(unordered_set&& __u)
+      _NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_set& operator=(initializer_list<value_type> __il);
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
+    return allocator_type(__table_.__node_alloc());
+  }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __table_.size() == 0; }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __table_.size(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __table_.max_size(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __table_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __table_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __table_.end(); }
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> emplace(_Args&&... __args) {
+    return __table_.__emplace_unique(std::forward<_Args>(__args)...);
+  }
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator, _Args&&... __args) {
+    return __table_.__emplace_unique(std::forward<_Args>(__args)...).first;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(value_type&& __x) {
+    return __table_.__insert_unique(std::move(__x));
+  }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator, value_type&& __x) { return insert(std::move(__x)).first; }
+
+  _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(const value_type& __x) { return __table_.__insert_unique(__x); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator, const value_type& __x) { return insert(__x).first; }
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
+    for (auto&& __element : __range) {
+      __table_.__insert_unique(std::forward<decltype(__element)>(__element));
+    }
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __table_.erase(__p); }
+  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __table_.__erase_unique(__k); }
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last) {
+    return __table_.erase(__first, __last);
+  }
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __table_.clear(); }
+
+#if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI insert_return_type insert(node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to unordered_set::insert()");
+    return __table_.template __node_handle_insert_unique< node_type, insert_return_type>(std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __h, node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to unordered_set::insert()");
+    return __table_.template __node_handle_insert_unique<node_type>(__h, std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
+    return __table_.template __node_handle_extract<node_type>(__key);
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
+    return __table_.template __node_handle_extract<node_type>(__it);
+  }
+
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_set<key_type, _H2, _P2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __table_.__node_handle_merge_unique(__source.__table_);
+  }
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_set<key_type, _H2, _P2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __table_.__node_handle_merge_unique(__source.__table_);
+  }
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_multiset<key_type, _H2, _P2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __table_.__node_handle_merge_unique(__source.__table_);
+  }
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_multiset<key_type, _H2, _P2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    __table_.__node_handle_merge_unique(__source.__table_);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI void swap(unordered_set& __u) _NOEXCEPT_(__is_nothrow_swappable_v<__table>) {
+    __table_.swap(__u.__table_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI hasher hash_function() const { return __table_.hash_function(); }
+  _LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
+#if _LIBCPP_STD_VER >= 20
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
+    return __table_.find(__k);
+  }
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
+    return __table_.find(__k);
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_unique(__k); }
+#if _LIBCPP_STD_VER >= 20
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
+    return __table_.__count_unique(__k);
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
+
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
+    return find(__k) != end();
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
+    return __table_.__equal_range_unique(__k);
+  }
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
+    return __table_.__equal_range_unique(__k);
+  }
+#if _LIBCPP_STD_VER >= 20
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
+    return __table_.__equal_range_unique(__k);
+  }
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
+    return __table_.__equal_range_unique(__k);
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __table_.bucket_count(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT { return __table_.max_bucket_count(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type bucket_size(size_type __n) const { return __table_.bucket_size(__n); }
+  _LIBCPP_HIDE_FROM_ABI size_type bucket(const key_type& __k) const { return __table_.bucket(__k); }
+
+  _LIBCPP_HIDE_FROM_ABI local_iterator begin(size_type __n) { return __table_.begin(__n); }
+  _LIBCPP_HIDE_FROM_ABI local_iterator end(size_type __n) { return __table_.end(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator begin(size_type __n) const { return __table_.cbegin(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator end(size_type __n) const { return __table_.cend(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator cbegin(size_type __n) const { return __table_.cbegin(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator cend(size_type __n) const { return __table_.cend(__n); }
+
+  _LIBCPP_HIDE_FROM_ABI float load_factor() const _NOEXCEPT { return __table_.load_factor(); }
+  _LIBCPP_HIDE_FROM_ABI float max_load_factor() const _NOEXCEPT { return __table_.max_load_factor(); }
+  _LIBCPP_HIDE_FROM_ABI void max_load_factor(float __mlf) { __table_.max_load_factor(__mlf); }
+  _LIBCPP_HIDE_FROM_ABI void rehash(size_type __n) { __table_.__rehash_unique(__n); }
+  _LIBCPP_HIDE_FROM_ABI void reserve(size_type __n) { __table_.__reserve_unique(__n); }
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator,
+          class _Hash      = hash<__iter_value_type<_InputIterator>>,
+          class _Pred      = equal_to<__iter_value_type<_InputIterator>>,
+          class _Allocator = allocator<__iter_value_type<_InputIterator>>,
+          class            = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class            = enable_if_t<!__is_allocator<_Hash>::value>,
+          class            = enable_if_t<!is_integral<_Hash>::value>,
+          class            = enable_if_t<!__is_allocator<_Pred>::value>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_set(_InputIterator,
+              _InputIterator,
+              typename allocator_traits<_Allocator>::size_type = 0,
+              _Hash                                            = _Hash(),
+              _Pred                                            = _Pred(),
+              _Allocator = _Allocator()) -> unordered_set<__iter_value_type<_InputIterator>, _Hash, _Pred, _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range,
+          class _Hash      = hash<ranges::range_value_t<_Range>>,
+          class _Pred      = equal_to<ranges::range_value_t<_Range>>,
+          class _Allocator = allocator<ranges::range_value_t<_Range>>,
+          class            = enable_if_t<!__is_allocator<_Hash>::value>,
+          class            = enable_if_t<!is_integral<_Hash>::value>,
+          class            = enable_if_t<!__is_allocator<_Pred>::value>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_set(
+    from_range_t,
+    _Range&&,
+    typename allocator_traits<_Allocator>::size_type = 0,
+    _Hash                                            = _Hash(),
+    _Pred                                            = _Pred(),
+    _Allocator = _Allocator()) -> unordered_set<ranges::range_value_t<_Range>, _Hash, _Pred, _Allocator>; // C++23
+#  endif
+
+template <class _Tp,
+          class _Hash      = hash<_Tp>,
+          class _Pred      = equal_to<_Tp>,
+          class _Allocator = allocator<_Tp>,
+          class            = enable_if_t<!__is_allocator<_Hash>::value>,
+          class            = enable_if_t<!is_integral<_Hash>::value>,
+          class            = enable_if_t<!__is_allocator<_Pred>::value>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_set(initializer_list<_Tp>,
+              typename allocator_traits<_Allocator>::size_type = 0,
+              _Hash                                            = _Hash(),
+              _Pred                                            = _Pred(),
+              _Allocator = _Allocator()) -> unordered_set<_Tp, _Hash, _Pred, _Allocator>;
+
+template <class _InputIterator,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_set(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Allocator)
+    -> unordered_set<__iter_value_type<_InputIterator>,
+                     hash<__iter_value_type<_InputIterator>>,
+                     equal_to<__iter_value_type<_InputIterator>>,
+                     _Allocator>;
+
+template <class _InputIterator,
+          class _Hash,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<!__is_allocator<_Hash>::value>,
+          class = enable_if_t<!is_integral<_Hash>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_set(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
+    -> unordered_set<__iter_value_type<_InputIterator>, _Hash, equal_to<__iter_value_type<_InputIterator>>, _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+
+template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_set(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Allocator)
+    -> unordered_set<ranges::range_value_t<_Range>,
+                     hash<ranges::range_value_t<_Range>>,
+                     equal_to<ranges::range_value_t<_Range>>,
+                     _Allocator>;
+
+template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_set(from_range_t, _Range&&, _Allocator)
+    -> unordered_set<ranges::range_value_t<_Range>,
+                     hash<ranges::range_value_t<_Range>>,
+                     equal_to<ranges::range_value_t<_Range>>,
+                     _Allocator>;
+
+template <ranges::input_range _Range,
+          class _Hash,
+          class _Allocator,
+          class = enable_if_t<!__is_allocator<_Hash>::value>,
+          class = enable_if_t<!is_integral<_Hash>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_set(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
+    -> unordered_set<ranges::range_value_t<_Range>, _Hash, equal_to<ranges::range_value_t<_Range>>, _Allocator>;
+
+#  endif
+
+template <class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_set(initializer_list<_Tp>, typename allocator_traits<_Allocator>::size_type, _Allocator)
+    -> unordered_set<_Tp, hash<_Tp>, equal_to<_Tp>, _Allocator>;
+
+template <class _Tp,
+          class _Hash,
+          class _Allocator,
+          class = enable_if_t<!__is_allocator<_Hash>::value>,
+          class = enable_if_t<!is_integral<_Hash>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_set(initializer_list<_Tp>, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
+    -> unordered_set<_Tp, _Hash, equal_to<_Tp>, _Allocator>;
+#endif
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_unique(__n);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(
+    size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
+    : __table_(__hf, __eql, __a) {
+  __table_.__rehash_unique(__n);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(_InputIterator __first, _InputIterator __last) {
+  insert(__first, __last);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(
+    _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_unique(__n);
+  insert(__first, __last);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(
+    _InputIterator __first,
+    _InputIterator __last,
+    size_type __n,
+    const hasher& __hf,
+    const key_equal& __eql,
+    const allocator_type& __a)
+    : __table_(__hf, __eql, __a) {
+  __table_.__rehash_unique(__n);
+  insert(__first, __last);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(const allocator_type& __a) : __table_(__a) {}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(const unordered_set& __u) : __table_(__u.__table_) {
+  __table_.__rehash_unique(__u.bucket_count());
+  insert(__u.begin(), __u.end());
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(const unordered_set& __u, const allocator_type& __a)
+    : __table_(__u.__table_, __a) {
+  __table_.__rehash_unique(__u.bucket_count());
+  insert(__u.begin(), __u.end());
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(unordered_set&& __u)
+    _NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
+    : __table_(std::move(__u.__table_)) {}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(unordered_set&& __u, const allocator_type& __a)
+    : __table_(std::move(__u.__table_), __a) {
+  if (__a != __u.get_allocator()) {
+    iterator __i = __u.begin();
+    while (__u.size() != 0)
+      __table_.__insert_unique(std::move(__u.__table_.remove(__i++)->__get_value()));
+  }
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(initializer_list<value_type> __il) {
+  insert(__il.begin(), __il.end());
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(
+    initializer_list<value_type> __il, size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_unique(__n);
+  insert(__il.begin(), __il.end());
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(
+    initializer_list<value_type> __il,
+    size_type __n,
+    const hasher& __hf,
+    const key_equal& __eql,
+    const allocator_type& __a)
+    : __table_(__hf, __eql, __a) {
+  __table_.__rehash_unique(__n);
+  insert(__il.begin(), __il.end());
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline unordered_set<_Value, _Hash, _Pred, _Alloc>&
+unordered_set<_Value, _Hash, _Pred, _Alloc>::operator=(unordered_set&& __u)
+    _NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
+  __table_ = std::move(__u.__table_);
+  return *this;
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline unordered_set<_Value, _Hash, _Pred, _Alloc>&
+unordered_set<_Value, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {
+  __table_.__assign_unique(__il.begin(), __il.end());
+  return *this;
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+inline void unordered_set<_Value, _Hash, _Pred, _Alloc>::insert(_InputIterator __first, _InputIterator __last) {
+  for (; __first != __last; ++__first)
+    __table_.__insert_unique(*__first);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(unordered_set<_Value, _Hash, _Pred, _Alloc>& __x, unordered_set<_Value, _Hash, _Pred, _Alloc>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Value, class _Hash, class _Pred, class _Alloc, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI typename unordered_set<_Value, _Hash, _Pred, _Alloc>::size_type
+erase_if(unordered_set<_Value, _Hash, _Pred, _Alloc>& __c, _Predicate __pred) {
+  return std::__libcpp_erase_if_container(__c, __pred);
+}
+#endif
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+_LIBCPP_HIDE_FROM_ABI bool operator==(const unordered_set<_Value, _Hash, _Pred, _Alloc>& __x,
+                                      const unordered_set<_Value, _Hash, _Pred, _Alloc>& __y) {
+  if (__x.size() != __y.size())
+    return false;
+  typedef typename unordered_set<_Value, _Hash, _Pred, _Alloc>::const_iterator const_iterator;
+  for (const_iterator __i = __x.begin(), __ex = __x.end(), __ey = __y.end(); __i != __ex; ++__i) {
+    const_iterator __j = __y.find(*__i);
+    if (__j == __ey || !(*__i == *__j))
+      return false;
+  }
+  return true;
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_set<_Value, _Hash, _Pred, _Alloc>& __x,
+                                             const unordered_set<_Value, _Hash, _Pred, _Alloc>& __y) {
+  return !(__x == __y);
+}
+
+#endif
+
+template <class _Value, class _Hash = hash<_Value>, class _Pred = equal_to<_Value>, class _Alloc = allocator<_Value> >
+class _LIBCPP_TEMPLATE_VIS unordered_multiset {
+public:
+  // types
+  typedef _Value key_type;
+  typedef key_type value_type;
+  typedef __type_identity_t<_Hash> hasher;
+  typedef __type_identity_t<_Pred> key_equal;
+  typedef __type_identity_t<_Alloc> allocator_type;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+  static_assert(is_same<value_type, typename allocator_type::value_type>::value,
+                "Allocator::value_type must be same type as value_type");
+
+private:
+  typedef __hash_table<value_type, hasher, key_equal, allocator_type> __table;
+
+  __table __table_;
+
+public:
+  typedef typename __table::pointer pointer;
+  typedef typename __table::const_pointer const_pointer;
+  typedef typename __table::size_type size_type;
+  typedef typename __table::difference_type difference_type;
+
+  typedef typename __table::const_iterator iterator;
+  typedef typename __table::const_iterator const_iterator;
+  typedef typename __table::const_local_iterator local_iterator;
+  typedef typename __table::const_local_iterator const_local_iterator;
+
+#if _LIBCPP_STD_VER >= 17
+  typedef __set_node_handle<typename __table::__node, allocator_type> node_type;
+#endif
+
+  template <class _Value2, class _Hash2, class _Pred2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_set;
+  template <class _Value2, class _Hash2, class _Pred2, class _Alloc2>
+  friend class _LIBCPP_TEMPLATE_VIS unordered_multiset;
+
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset() _NOEXCEPT_(is_nothrow_default_constructible<__table>::value) {}
+  explicit _LIBCPP_HIDE_FROM_ABI
+  unordered_multiset(size_type __n, const hasher& __hf = hasher(), const key_equal& __eql = key_equal());
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_multiset(size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a);
+#if _LIBCPP_STD_VER >= 14
+  inline _LIBCPP_HIDE_FROM_ABI unordered_multiset(size_type __n, const allocator_type& __a)
+      : unordered_multiset(__n, hasher(), key_equal(), __a) {}
+  inline _LIBCPP_HIDE_FROM_ABI unordered_multiset(size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_multiset(__n, __hf, key_equal(), __a) {}
+#endif
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset(_InputIterator __first, _InputIterator __last);
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset(
+      _InputIterator __first,
+      _InputIterator __last,
+      size_type __n,
+      const hasher& __hf     = hasher(),
+      const key_equal& __eql = key_equal());
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset(
+      _InputIterator __first,
+      _InputIterator __last,
+      size_type __n,
+      const hasher& __hf,
+      const key_equal& __eql,
+      const allocator_type& __a);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset(
+      from_range_t,
+      _Range&& __range,
+      size_type __n             = /*implementation-defined*/ 0,
+      const hasher& __hf        = hasher(),
+      const key_equal& __eql    = key_equal(),
+      const allocator_type& __a = allocator_type())
+      : __table_(__hf, __eql, __a) {
+    if (__n > 0) {
+      __table_.__rehash_multi(__n);
+    }
+    insert_range(std::forward<_Range>(__range));
+  }
+#endif
+
+#if _LIBCPP_STD_VER >= 14
+  template <class _InputIterator>
+  inline _LIBCPP_HIDE_FROM_ABI
+  unordered_multiset(_InputIterator __first, _InputIterator __last, size_type __n, const allocator_type& __a)
+      : unordered_multiset(__first, __last, __n, hasher(), key_equal(), __a) {}
+  template <class _InputIterator>
+  inline _LIBCPP_HIDE_FROM_ABI unordered_multiset(
+      _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_multiset(__first, __last, __n, __hf, key_equal(), __a) {}
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset(from_range_t, _Range&& __range, size_type __n, const allocator_type& __a)
+      : unordered_multiset(from_range, std::forward<_Range>(__range), __n, hasher(), key_equal(), __a) {}
+
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI
+  unordered_multiset(from_range_t, _Range&& __range, size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_multiset(from_range, std::forward<_Range>(__range), __n, __hf, key_equal(), __a) {}
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI explicit unordered_multiset(const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset(const unordered_multiset& __u);
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset(const unordered_multiset& __u, const allocator_type& __a);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset(unordered_multiset&& __u)
+      _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset(unordered_multiset&& __u, const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset(initializer_list<value_type> __il);
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset(
+      initializer_list<value_type> __il,
+      size_type __n,
+      const hasher& __hf     = hasher(),
+      const key_equal& __eql = key_equal());
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset(
+      initializer_list<value_type> __il,
+      size_type __n,
+      const hasher& __hf,
+      const key_equal& __eql,
+      const allocator_type& __a);
+#  if _LIBCPP_STD_VER >= 14
+  inline _LIBCPP_HIDE_FROM_ABI
+  unordered_multiset(initializer_list<value_type> __il, size_type __n, const allocator_type& __a)
+      : unordered_multiset(__il, __n, hasher(), key_equal(), __a) {}
+  inline _LIBCPP_HIDE_FROM_ABI
+  unordered_multiset(initializer_list<value_type> __il, size_type __n, const hasher& __hf, const allocator_type& __a)
+      : unordered_multiset(__il, __n, __hf, key_equal(), __a) {}
+#  endif
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI ~unordered_multiset() {
+    static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Value, _Hash, _Pred>(0)), "");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset& operator=(const unordered_multiset& __u) {
+    __table_ = __u.__table_;
+    return *this;
+  }
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset& operator=(unordered_multiset&& __u)
+      _NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset& operator=(initializer_list<value_type> __il);
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
+    return allocator_type(__table_.__node_alloc());
+  }
+
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __table_.size() == 0; }
+  _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __table_.size(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __table_.max_size(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __table_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __table_.end(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return __table_.begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __table_.end(); }
+
+#ifndef _LIBCPP_CXX03_LANG
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace(_Args&&... __args) {
+    return __table_.__emplace_multi(std::forward<_Args>(__args)...);
+  }
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __p, _Args&&... __args) {
+    return __table_.__emplace_hint_multi(__p, std::forward<_Args>(__args)...);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(value_type&& __x) { return __table_.__insert_multi(std::move(__x)); }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __x) {
+    return __table_.__insert_multi(__p, std::move(__x));
+  }
+  _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __x) { return __table_.__insert_multi(__x); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __x) {
+    return __table_.__insert_multi(__p, __x);
+  }
+
+  template <class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
+    for (auto&& __element : __range) {
+      __table_.__insert_multi(std::forward<decltype(__element)>(__element));
+    }
+  }
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI iterator insert(node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to unordered_multiset::insert()");
+    return __table_.template __node_handle_insert_multi<node_type>(std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, node_type&& __nh) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
+                                        "node_type with incompatible allocator passed to unordered_multiset::insert()");
+    return __table_.template __node_handle_insert_multi<node_type>(__hint, std::move(__nh));
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __position) {
+    return __table_.template __node_handle_extract<node_type>(__position);
+  }
+  _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
+    return __table_.template __node_handle_extract<node_type>(__key);
+  }
+
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_multiset<key_type, _H2, _P2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __table_.__node_handle_merge_multi(__source.__table_);
+  }
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_multiset<key_type, _H2, _P2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __table_.__node_handle_merge_multi(__source.__table_);
+  }
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_set<key_type, _H2, _P2, allocator_type>& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __table_.__node_handle_merge_multi(__source.__table_);
+  }
+  template <class _H2, class _P2>
+  _LIBCPP_HIDE_FROM_ABI void merge(unordered_set<key_type, _H2, _P2, allocator_type>&& __source) {
+    _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+        __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
+    return __table_.__node_handle_merge_multi(__source.__table_);
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __table_.erase(__p); }
+  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __table_.__erase_multi(__k); }
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last) {
+    return __table_.erase(__first, __last);
+  }
+  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __table_.clear(); }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(unordered_multiset& __u) _NOEXCEPT_(__is_nothrow_swappable_v<__table>) {
+    __table_.swap(__u.__table_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI hasher hash_function() const { return __table_.hash_function(); }
+  _LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq(); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
+#if _LIBCPP_STD_VER >= 20
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
+    return __table_.find(__k);
+  }
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
+    return __table_.find(__k);
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_multi(__k); }
+#if _LIBCPP_STD_VER >= 20
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
+    return __table_.__count_multi(__k);
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
+
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
+    return find(__k) != end();
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
+    return __table_.__equal_range_multi(__k);
+  }
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
+    return __table_.__equal_range_multi(__k);
+  }
+#if _LIBCPP_STD_VER >= 20
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
+    return __table_.__equal_range_multi(__k);
+  }
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
+    return __table_.__equal_range_multi(__k);
+  }
+#endif // _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __table_.bucket_count(); }
+  _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT { return __table_.max_bucket_count(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type bucket_size(size_type __n) const { return __table_.bucket_size(__n); }
+  _LIBCPP_HIDE_FROM_ABI size_type bucket(const key_type& __k) const { return __table_.bucket(__k); }
+
+  _LIBCPP_HIDE_FROM_ABI local_iterator begin(size_type __n) { return __table_.begin(__n); }
+  _LIBCPP_HIDE_FROM_ABI local_iterator end(size_type __n) { return __table_.end(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator begin(size_type __n) const { return __table_.cbegin(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator end(size_type __n) const { return __table_.cend(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator cbegin(size_type __n) const { return __table_.cbegin(__n); }
+  _LIBCPP_HIDE_FROM_ABI const_local_iterator cend(size_type __n) const { return __table_.cend(__n); }
+
+  _LIBCPP_HIDE_FROM_ABI float load_factor() const _NOEXCEPT { return __table_.load_factor(); }
+  _LIBCPP_HIDE_FROM_ABI float max_load_factor() const _NOEXCEPT { return __table_.max_load_factor(); }
+  _LIBCPP_HIDE_FROM_ABI void max_load_factor(float __mlf) { __table_.max_load_factor(__mlf); }
+  _LIBCPP_HIDE_FROM_ABI void rehash(size_type __n) { __table_.__rehash_multi(__n); }
+  _LIBCPP_HIDE_FROM_ABI void reserve(size_type __n) { __table_.__reserve_multi(__n); }
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator,
+          class _Hash      = hash<__iter_value_type<_InputIterator>>,
+          class _Pred      = equal_to<__iter_value_type<_InputIterator>>,
+          class _Allocator = allocator<__iter_value_type<_InputIterator>>,
+          class            = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class            = enable_if_t<!__is_allocator<_Hash>::value>,
+          class            = enable_if_t<!is_integral<_Hash>::value>,
+          class            = enable_if_t<!__is_allocator<_Pred>::value>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multiset(
+    _InputIterator,
+    _InputIterator,
+    typename allocator_traits<_Allocator>::size_type = 0,
+    _Hash                                            = _Hash(),
+    _Pred                                            = _Pred(),
+    _Allocator = _Allocator()) -> unordered_multiset<__iter_value_type<_InputIterator>, _Hash, _Pred, _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range,
+          class _Hash      = hash<ranges::range_value_t<_Range>>,
+          class _Pred      = equal_to<ranges::range_value_t<_Range>>,
+          class _Allocator = allocator<ranges::range_value_t<_Range>>,
+          class            = enable_if_t<!__is_allocator<_Hash>::value>,
+          class            = enable_if_t<!is_integral<_Hash>::value>,
+          class            = enable_if_t<!__is_allocator<_Pred>::value>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multiset(
+    from_range_t,
+    _Range&&,
+    typename allocator_traits<_Allocator>::size_type = 0,
+    _Hash                                            = _Hash(),
+    _Pred                                            = _Pred(),
+    _Allocator = _Allocator()) -> unordered_multiset<ranges::range_value_t<_Range>, _Hash, _Pred, _Allocator>; // C++23
+#  endif
+
+template <class _Tp,
+          class _Hash      = hash<_Tp>,
+          class _Pred      = equal_to<_Tp>,
+          class _Allocator = allocator<_Tp>,
+          class            = enable_if_t<!__is_allocator<_Hash>::value>,
+          class            = enable_if_t<!is_integral<_Hash>::value>,
+          class            = enable_if_t<!__is_allocator<_Pred>::value>,
+          class            = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multiset(initializer_list<_Tp>,
+                   typename allocator_traits<_Allocator>::size_type = 0,
+                   _Hash                                            = _Hash(),
+                   _Pred                                            = _Pred(),
+                   _Allocator = _Allocator()) -> unordered_multiset<_Tp, _Hash, _Pred, _Allocator>;
+
+template <class _InputIterator,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multiset(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Allocator)
+    -> unordered_multiset<__iter_value_type<_InputIterator>,
+                          hash<__iter_value_type<_InputIterator>>,
+                          equal_to<__iter_value_type<_InputIterator>>,
+                          _Allocator>;
+
+template <class _InputIterator,
+          class _Hash,
+          class _Allocator,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<!__is_allocator<_Hash>::value>,
+          class = enable_if_t<!is_integral<_Hash>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multiset(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
+    -> unordered_multiset<__iter_value_type<_InputIterator>,
+                          _Hash,
+                          equal_to<__iter_value_type<_InputIterator>>,
+                          _Allocator>;
+
+#  if _LIBCPP_STD_VER >= 23
+
+template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multiset(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Allocator)
+    -> unordered_multiset<ranges::range_value_t<_Range>,
+                          hash<ranges::range_value_t<_Range>>,
+                          equal_to<ranges::range_value_t<_Range>>,
+                          _Allocator>;
+
+template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multiset(from_range_t, _Range&&, _Allocator)
+    -> unordered_multiset<ranges::range_value_t<_Range>,
+                          hash<ranges::range_value_t<_Range>>,
+                          equal_to<ranges::range_value_t<_Range>>,
+                          _Allocator>;
+
+template <ranges::input_range _Range,
+          class _Hash,
+          class _Allocator,
+          class = enable_if_t<!__is_allocator<_Hash>::value>,
+          class = enable_if_t<!is_integral<_Hash>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multiset(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
+    -> unordered_multiset<ranges::range_value_t<_Range>, _Hash, equal_to<ranges::range_value_t<_Range>>, _Allocator>;
+
+#  endif
+
+template <class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multiset(initializer_list<_Tp>, typename allocator_traits<_Allocator>::size_type, _Allocator)
+    -> unordered_multiset<_Tp, hash<_Tp>, equal_to<_Tp>, _Allocator>;
+
+template <class _Tp,
+          class _Hash,
+          class _Allocator,
+          class = enable_if_t<!__is_allocator<_Hash>::value>,
+          class = enable_if_t<!is_integral<_Hash>::value>,
+          class = enable_if_t<__is_allocator<_Allocator>::value>>
+unordered_multiset(initializer_list<_Tp>, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
+    -> unordered_multiset<_Tp, _Hash, equal_to<_Tp>, _Allocator>;
+#endif
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
+    size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_multi(__n);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
+    size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
+    : __table_(__hf, __eql, __a) {
+  __table_.__rehash_multi(__n);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(_InputIterator __first, _InputIterator __last) {
+  insert(__first, __last);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
+    _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_multi(__n);
+  insert(__first, __last);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
+    _InputIterator __first,
+    _InputIterator __last,
+    size_type __n,
+    const hasher& __hf,
+    const key_equal& __eql,
+    const allocator_type& __a)
+    : __table_(__hf, __eql, __a) {
+  __table_.__rehash_multi(__n);
+  insert(__first, __last);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(const allocator_type& __a)
+    : __table_(__a) {}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(const unordered_multiset& __u)
+    : __table_(__u.__table_) {
+  __table_.__rehash_multi(__u.bucket_count());
+  insert(__u.begin(), __u.end());
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
+    const unordered_multiset& __u, const allocator_type& __a)
+    : __table_(__u.__table_, __a) {
+  __table_.__rehash_multi(__u.bucket_count());
+  insert(__u.begin(), __u.end());
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(unordered_multiset&& __u)
+    _NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
+    : __table_(std::move(__u.__table_)) {}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
+    unordered_multiset&& __u, const allocator_type& __a)
+    : __table_(std::move(__u.__table_), __a) {
+  if (__a != __u.get_allocator()) {
+    iterator __i = __u.begin();
+    while (__u.size() != 0)
+      __table_.__insert_multi(std::move(__u.__table_.remove(__i++)->__get_value()));
+  }
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(initializer_list<value_type> __il) {
+  insert(__il.begin(), __il.end());
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
+    initializer_list<value_type> __il, size_type __n, const hasher& __hf, const key_equal& __eql)
+    : __table_(__hf, __eql) {
+  __table_.__rehash_multi(__n);
+  insert(__il.begin(), __il.end());
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
+    initializer_list<value_type> __il,
+    size_type __n,
+    const hasher& __hf,
+    const key_equal& __eql,
+    const allocator_type& __a)
+    : __table_(__hf, __eql, __a) {
+  __table_.__rehash_multi(__n);
+  insert(__il.begin(), __il.end());
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline unordered_multiset<_Value, _Hash, _Pred, _Alloc>&
+unordered_multiset<_Value, _Hash, _Pred, _Alloc>::operator=(unordered_multiset&& __u)
+    _NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
+  __table_ = std::move(__u.__table_);
+  return *this;
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline unordered_multiset<_Value, _Hash, _Pred, _Alloc>&
+unordered_multiset<_Value, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {
+  __table_.__assign_multi(__il.begin(), __il.end());
+  return *this;
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+template <class _InputIterator>
+inline void unordered_multiset<_Value, _Hash, _Pred, _Alloc>::insert(_InputIterator __first, _InputIterator __last) {
+  for (; __first != __last; ++__first)
+    __table_.__insert_multi(*__first);
+}
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI void
+swap(unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __x, unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Value, class _Hash, class _Pred, class _Alloc, class _Predicate>
+inline _LIBCPP_HIDE_FROM_ABI typename unordered_multiset<_Value, _Hash, _Pred, _Alloc>::size_type
+erase_if(unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __c, _Predicate __pred) {
+  return std::__libcpp_erase_if_container(__c, __pred);
+}
+#endif
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+_LIBCPP_HIDE_FROM_ABI bool operator==(const unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __x,
+                                      const unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __y) {
+  if (__x.size() != __y.size())
+    return false;
+  typedef typename unordered_multiset<_Value, _Hash, _Pred, _Alloc>::const_iterator const_iterator;
+  typedef pair<const_iterator, const_iterator> _EqRng;
+  for (const_iterator __i = __x.begin(), __ex = __x.end(); __i != __ex;) {
+    _EqRng __xeq = __x.equal_range(*__i);
+    _EqRng __yeq = __y.equal_range(*__i);
+    if (std::distance(__xeq.first, __xeq.second) != std::distance(__yeq.first, __yeq.second) ||
+        !std::is_permutation(__xeq.first, __xeq.second, __yeq.first))
+      return false;
+    __i = __xeq.second;
+  }
+  return true;
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __x,
+                                             const unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __y) {
+  return !(__x == __y);
+}
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace pmr {
+template <class _KeyT, class _HashT = std::hash<_KeyT>, class _PredT = std::equal_to<_KeyT>>
+using unordered_set _LIBCPP_AVAILABILITY_PMR = std::unordered_set<_KeyT, _HashT, _PredT, polymorphic_allocator<_KeyT>>;
+
+template <class _KeyT, class _HashT = std::hash<_KeyT>, class _PredT = std::equal_to<_KeyT>>
+using unordered_multiset _LIBCPP_AVAILABILITY_PMR =
+    std::unordered_multiset<_KeyT, _HashT, _PredT, polymorphic_allocator<_KeyT>>;
+} // namespace pmr
+_LIBCPP_END_NAMESPACE_STD
+#endif
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <concepts>
+#  include <cstdlib>
+#  include <functional>
+#  include <iterator>
+#  include <stdexcept>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_UNORDERED_SET
diff --git a/libcxx/include/__cxx03/utility b/libcxx/include/__cxx03/utility
new file mode 100644
index 000000000000..f97907fbf72e
--- /dev/null
+++ b/libcxx/include/__cxx03/utility
@@ -0,0 +1,309 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_UTILITY
+#define _LIBCPP_UTILITY
+
+/*
+    utility synopsis
+
+#include <initializer_list>
+
+namespace std
+{
+
+template <class T>
+    void
+    swap(T& a, T& b);
+
+namespace rel_ops
+{
+    template<class T> bool operator!=(const T&, const T&);
+    template<class T> bool operator> (const T&, const T&);
+    template<class T> bool operator<=(const T&, const T&);
+    template<class T> bool operator>=(const T&, const T&);
+}
+
+template<class T>
+void
+swap(T& a, T& b) noexcept(is_nothrow_move_constructible<T>::value &&
+                          is_nothrow_move_assignable<T>::value);
+
+template <class T, size_t N>
+void
+swap(T (&a)[N], T (&b)[N]) noexcept(noexcept(swap(*a, *b)));
+
+template <class T> T&& forward(typename remove_reference<T>::type& t) noexcept;  // constexpr in C++14
+template <class T> T&& forward(typename remove_reference<T>::type&& t) noexcept; // constexpr in C++14
+
+template <typename T>
+[[nodiscard]] constexpr
+auto forward_like(auto&& x) noexcept -> see below;                               // since C++23
+
+template <class T> typename remove_reference<T>::type&& move(T&&) noexcept;      // constexpr in C++14
+
+template <class T>
+    typename conditional
+    <
+        !is_nothrow_move_constructible<T>::value && is_copy_constructible<T>::value,
+        const T&,
+        T&&
+    >::type
+    move_if_noexcept(T& x) noexcept; // constexpr in C++14
+
+template <class T> constexpr add_const_t<T>& as_const(T& t) noexcept;      // C++17
+template <class T>                      void as_const(const T&&) = delete; // C++17
+
+template <class T> typename add_rvalue_reference<T>::type declval() noexcept;
+
+template<class T, class U> constexpr bool cmp_equal(T t, U u) noexcept;         // C++20
+template<class T, class U> constexpr bool cmp_not_equal(T t, U u) noexcept;     // C++20
+template<class T, class U> constexpr bool cmp_less(T t, U u) noexcept;          // C++20
+template<class T, class U> constexpr bool cmp_greater(T t, U u) noexcept;       // C++20
+template<class T, class U> constexpr bool cmp_less_equal(T t, U u) noexcept;    // C++20
+template<class T, class U> constexpr bool cmp_greater_equal(T t, U u) noexcept; // C++20
+template<class R, class T> constexpr bool in_range(T t) noexcept;               // C++20
+
+template <class T1, class T2>
+struct pair
+{
+    typedef T1 first_type;
+    typedef T2 second_type;
+
+    T1 first;
+    T2 second;
+
+    pair(const pair&) = default;
+    pair(pair&&) = default;
+    explicit(see-below) constexpr pair();
+    explicit(see-below) pair(const T1& x, const T2& y);                          // constexpr in C++14
+    template <class U = T1, class V = T2> explicit(see-below) pair(U&&, V&&);    // constexpr in C++14
+    template <class U, class V> constexpr explicit(see-below) pair(pair<U, V>&); // since C++23
+    template <class U, class V> explicit(see-below) pair(const pair<U, V>& p);   // constexpr in C++14
+    template <class U, class V> explicit(see-below) pair(pair<U, V>&& p);        // constexpr in C++14
+    template <class U, class V>
+    constexpr explicit(see-below) pair(const pair<U, V>&&);                      // since C++23
+    template <pair-like P> constexpr explicit(see-below) pair(P&&);              // since C++23
+    template <class... Args1, class... Args2>
+        pair(piecewise_construct_t, tuple<Args1...> first_args,                  // constexpr in C++20
+                                    tuple<Args2...> second_args);
+
+    constexpr const pair& operator=(const pair& p) const;                        // since C++23
+    template <class U, class V> pair& operator=(const pair<U, V>& p);            // constexpr in C++20
+    template <class U, class V>
+    constexpr const pair& operator=(const pair<U, V>& p) const;                  // since C++23
+    pair& operator=(pair&& p) noexcept(is_nothrow_move_assignable<T1>::value &&
+                                       is_nothrow_move_assignable<T2>::value);   // constexpr in C++20
+    constexpr const pair& operator=(pair&& p) const;                             // since C++23
+    template <class U, class V> pair& operator=(pair<U, V>&& p);                 // constexpr in C++20
+    template <class U, class V>
+    constexpr const pair& operator=(pair<U, V>&& p) const;                       // since C++23
+    template <pair-like P> constexpr pair& operator=(P&&);                       // since C++23
+    template <pair-like P> constexpr const pair& operator=(P&&) const;           // since C++23
+
+    void swap(pair& p) noexcept(is_nothrow_swappable_v<T1> &&
+                                is_nothrow_swappable_v<T2>);                     // constexpr in C++20
+    constexpr void swap(const pair& p) const noexcept(see below);                // since C++23
+};
+
+template<class T1, class T2, class U1, class U2, template<class> class TQual, template<class> class UQual>
+struct basic_common_reference<pair<T1, T2>, pair<U1, U2>, TQual, UQual>;         // since C++23
+
+template<class T1, class T2, class U1, class U2>
+struct common_type<pair<T1, T2>, pair<U1, U2>>;                                  // since C++23
+
+template<class T1, class T2> pair(T1, T2) -> pair<T1, T2>;
+
+template <class T1, class T2, class U1, class U2>
+bool operator==(const pair<T1,T2>&, const pair<U1,U2>&);                         // constexpr in C++14
+template <class T1, class T2, class U1, class U2>
+bool operator!=(const pair<T1,T2>&, const pair<U1,U2>&);                         // constexpr in C++14, removed in C++20
+template <class T1, class T2, class U1, class U2>
+bool operator< (const pair<T1,T2>&, const pair<U1,U2>&);                         // constexpr in C++14, removed in C++20
+template <class T1, class T2, class U1, class U2>
+bool operator> (const pair<T1,T2>&, const pair<U1,U2>&);                         // constexpr in C++14, removed in C++20
+template <class T1, class T2, class U1, class U2>
+bool operator>=(const pair<T1,T2>&, const pair<U1,U2>&);                         // constexpr in C++14, removed in C++20
+template <class T1, class T2, class U1, class U2>
+bool operator<=(const pair<T1,T2>&, const pair<U1,U2>&);                         // constexpr in C++14, removed in C++20
+template <class T1, class T2, class U1, class U2>
+  constexpr common_comparison_type_t<synth-three-way-result<T1,U1>,
+                                     synth-three-way-result<T2,U2>>
+    operator<=>(const pair<T1,T2>&, const pair<U1,U2>&);                         // C++20
+
+template <class T1, class T2> pair<V1, V2> make_pair(T1&&, T2&&);                // constexpr in C++14
+template <class T1, class T2>
+void
+swap(pair<T1, T2>& x, pair<T1, T2>& y) noexcept(noexcept(x.swap(y)));            // constexpr in C++20
+
+template<class T1, class T2>                                                     // since C++23
+constexpr void swap(const pair<T1, T2>& x, const pair<T1, T2>& y) noexcept(noexcept(x.swap(y)));
+
+struct piecewise_construct_t { explicit piecewise_construct_t() = default; };
+inline constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t();
+
+template <class T> struct tuple_size;
+template <size_t I, class T> struct tuple_element;
+
+template <class T1, class T2> struct tuple_size<pair<T1, T2> >;
+template <class T1, class T2> struct tuple_element<0, pair<T1, T2> >;
+template <class T1, class T2> struct tuple_element<1, pair<T1, T2> >;
+
+template<size_t I, class T1, class T2>
+    typename tuple_element<I, pair<T1, T2> >::type&
+    get(pair<T1, T2>&) noexcept; // constexpr in C++14
+
+template<size_t I, class T1, class T2>
+    const typename tuple_element<I, pair<T1, T2> >::type&
+    get(const pair<T1, T2>&) noexcept; // constexpr in C++14
+
+template<size_t I, class T1, class T2>
+    typename tuple_element<I, pair<T1, T2> >::type&&
+    get(pair<T1, T2>&&) noexcept; // constexpr in C++14
+
+template<size_t I, class T1, class T2>
+    const typename tuple_element<I, pair<T1, T2> >::type&&
+    get(const pair<T1, T2>&&) noexcept; // constexpr in C++14
+
+template<class T1, class T2>
+    constexpr T1& get(pair<T1, T2>&) noexcept; // C++14
+
+template<class T1, class T2>
+    constexpr const T1& get(const pair<T1, T2>&) noexcept; // C++14
+
+template<class T1, class T2>
+    constexpr T1&& get(pair<T1, T2>&&) noexcept; // C++14
+
+template<class T1, class T2>
+    constexpr const T1&& get(const pair<T1, T2>&&) noexcept; // C++14
+
+template<class T1, class T2>
+    constexpr T1& get(pair<T2, T1>&) noexcept; // C++14
+
+template<class T1, class T2>
+    constexpr const T1& get(const pair<T2, T1>&) noexcept; // C++14
+
+template<class T1, class T2>
+    constexpr T1&& get(pair<T2, T1>&&) noexcept; // C++14
+
+template<class T1, class T2>
+    constexpr const T1&& get(const pair<T2, T1>&&) noexcept; // C++14
+
+// C++14
+
+template<class T, T... I>
+struct integer_sequence
+{
+    typedef T value_type;
+
+    static constexpr size_t size() noexcept;
+};
+
+template<size_t... I>
+  using index_sequence = integer_sequence<size_t, I...>;
+
+template<class T, T N>
+  using make_integer_sequence = integer_sequence<T, 0, 1, ..., N-1>;
+template<size_t N>
+  using make_index_sequence = make_integer_sequence<size_t, N>;
+
+template<class... T>
+  using index_sequence_for = make_index_sequence<sizeof...(T)>;
+
+template<class T, class U=T>
+    constexpr T exchange(T& obj, U&& new_value)                                 // constexpr in C++17, noexcept in C++23
+      noexcept(is_nothrow_move_constructible<T>::value && is_nothrow_assignable<T&, U>::value);
+
+// 20.2.7, in-place construction // C++17
+struct in_place_t {
+  explicit in_place_t() = default;
+};
+inline constexpr in_place_t in_place{};
+template <class T>
+  struct in_place_type_t {
+    explicit in_place_type_t() = default;
+  };
+template <class T>
+  inline constexpr in_place_type_t<T> in_place_type{};
+template <size_t I>
+  struct in_place_index_t {
+    explicit in_place_index_t() = default;
+  };
+template <size_t I>
+  inline constexpr in_place_index_t<I> in_place_index{};
+
+// [utility.underlying], to_underlying
+template <class T>
+    constexpr underlying_type_t<T> to_underlying( T value ) noexcept; // C++23
+
+}  // std
+
+*/
+
+#include <__config>
+
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <__utility/piecewise_construct.h>
+#include <__utility/rel_ops.h>
+#include <__utility/swap.h>
+
+#if _LIBCPP_STD_VER >= 14
+#  include <__utility/exchange.h>
+#  include <__utility/integer_sequence.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+#  include <__utility/as_const.h>
+#  include <__utility/in_place.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__utility/cmp.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+#  include <__utility/forward_like.h>
+#  include <__utility/to_underlying.h>
+#  include <__utility/unreachable.h>
+#endif
+
+#include <version>
+
+// standard-mandated includes
+
+// [utility.syn]
+#include <compare>
+#include <initializer_list>
+
+// [tuple.creation]
+
+#include <__tuple/ignore.h>
+
+// [tuple.helper]
+#include <__tuple/tuple_element.h>
+#include <__tuple/tuple_size.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <limits>
+#endif
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstdlib>
+#  include <iosfwd>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_UTILITY
diff --git a/libcxx/include/__cxx03/valarray b/libcxx/include/__cxx03/valarray
new file mode 100644
index 000000000000..44341eb2ba6c
--- /dev/null
+++ b/libcxx/include/__cxx03/valarray
@@ -0,0 +1,3364 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_VALARRAY
+#define _LIBCPP_VALARRAY
+
+/*
+    valarray synopsis
+
+namespace std
+{
+
+template<class T>
+class valarray
+{
+public:
+    typedef T value_type;
+
+    // construct/destroy:
+    valarray();
+    explicit valarray(size_t n);
+    valarray(const value_type& x, size_t n);
+    valarray(const value_type* px, size_t n);
+    valarray(const valarray& v);
+    valarray(valarray&& v) noexcept;
+    valarray(const slice_array<value_type>& sa);
+    valarray(const gslice_array<value_type>& ga);
+    valarray(const mask_array<value_type>& ma);
+    valarray(const indirect_array<value_type>& ia);
+    valarray(initializer_list<value_type> il);
+    ~valarray();
+
+    // assignment:
+    valarray& operator=(const valarray& v);
+    valarray& operator=(valarray&& v) noexcept;
+    valarray& operator=(initializer_list<value_type> il);
+    valarray& operator=(const value_type& x);
+    valarray& operator=(const slice_array<value_type>& sa);
+    valarray& operator=(const gslice_array<value_type>& ga);
+    valarray& operator=(const mask_array<value_type>& ma);
+    valarray& operator=(const indirect_array<value_type>& ia);
+
+    // element access:
+    const value_type& operator[](size_t i) const;
+    value_type&       operator[](size_t i);
+
+    // subset operations:
+    valarray                   operator[](slice s) const;
+    slice_array<value_type>    operator[](slice s);
+    valarray                   operator[](const gslice& gs) const;
+    gslice_array<value_type>   operator[](const gslice& gs);
+    valarray                   operator[](const valarray<bool>& vb) const;
+    mask_array<value_type>     operator[](const valarray<bool>& vb);
+    valarray                   operator[](const valarray<size_t>& vs) const;
+    indirect_array<value_type> operator[](const valarray<size_t>& vs);
+
+    // unary operators:
+    valarray       operator+() const;
+    valarray       operator-() const;
+    valarray       operator~() const;
+    valarray<bool> operator!() const;
+
+    // computed assignment:
+    valarray& operator*= (const value_type& x);
+    valarray& operator/= (const value_type& x);
+    valarray& operator%= (const value_type& x);
+    valarray& operator+= (const value_type& x);
+    valarray& operator-= (const value_type& x);
+    valarray& operator^= (const value_type& x);
+    valarray& operator&= (const value_type& x);
+    valarray& operator|= (const value_type& x);
+    valarray& operator<<=(const value_type& x);
+    valarray& operator>>=(const value_type& x);
+
+    valarray& operator*= (const valarray& v);
+    valarray& operator/= (const valarray& v);
+    valarray& operator%= (const valarray& v);
+    valarray& operator+= (const valarray& v);
+    valarray& operator-= (const valarray& v);
+    valarray& operator^= (const valarray& v);
+    valarray& operator|= (const valarray& v);
+    valarray& operator&= (const valarray& v);
+    valarray& operator<<=(const valarray& v);
+    valarray& operator>>=(const valarray& v);
+
+    // member functions:
+    void swap(valarray& v) noexcept;
+
+    size_t size() const;
+
+    value_type sum() const;
+    value_type min() const;
+    value_type max() const;
+
+    valarray shift (int i) const;
+    valarray cshift(int i) const;
+    valarray apply(value_type f(value_type)) const;
+    valarray apply(value_type f(const value_type&)) const;
+    void resize(size_t n, value_type x = value_type());
+};
+
+template<class T, size_t cnt> valarray(const T(&)[cnt], size_t) -> valarray<T>;
+
+class slice
+{
+public:
+    slice();
+    slice(size_t start, size_t size, size_t stride);
+
+    size_t start()  const;
+    size_t size()   const;
+    size_t stride() const;
+
+    friend bool operator==(const slice& x, const slice& y); // since C++20
+};
+
+template <class T>
+class slice_array
+{
+public:
+    typedef T value_type;
+
+    const slice_array& operator=(const slice_array& sa) const;
+    void operator=  (const valarray<value_type>& v) const;
+    void operator*= (const valarray<value_type>& v) const;
+    void operator/= (const valarray<value_type>& v) const;
+    void operator%= (const valarray<value_type>& v) const;
+    void operator+= (const valarray<value_type>& v) const;
+    void operator-= (const valarray<value_type>& v) const;
+    void operator^= (const valarray<value_type>& v) const;
+    void operator&= (const valarray<value_type>& v) const;
+    void operator|= (const valarray<value_type>& v) const;
+    void operator<<=(const valarray<value_type>& v) const;
+    void operator>>=(const valarray<value_type>& v) const;
+
+    void operator=(const value_type& x) const;
+    void operator=(const valarray<T>& val_arr) const;
+
+    slice_array() = delete;
+};
+
+class gslice
+{
+public:
+    gslice();
+    gslice(size_t start, const valarray<size_t>& size,
+                         const valarray<size_t>& stride);
+
+    size_t           start()  const;
+    valarray<size_t> size()   const;
+    valarray<size_t> stride() const;
+};
+
+template <class T>
+class gslice_array
+{
+public:
+    typedef T value_type;
+
+    void operator=  (const valarray<value_type>& v) const;
+    void operator*= (const valarray<value_type>& v) const;
+    void operator/= (const valarray<value_type>& v) const;
+    void operator%= (const valarray<value_type>& v) const;
+    void operator+= (const valarray<value_type>& v) const;
+    void operator-= (const valarray<value_type>& v) const;
+    void operator^= (const valarray<value_type>& v) const;
+    void operator&= (const valarray<value_type>& v) const;
+    void operator|= (const valarray<value_type>& v) const;
+    void operator<<=(const valarray<value_type>& v) const;
+    void operator>>=(const valarray<value_type>& v) const;
+
+    gslice_array(const gslice_array& ga);
+    ~gslice_array();
+    const gslice_array& operator=(const gslice_array& ga) const;
+    void operator=(const value_type& x) const;
+
+    gslice_array() = delete;
+};
+
+template <class T>
+class mask_array
+{
+public:
+    typedef T value_type;
+
+    void operator=  (const valarray<value_type>& v) const;
+    void operator*= (const valarray<value_type>& v) const;
+    void operator/= (const valarray<value_type>& v) const;
+    void operator%= (const valarray<value_type>& v) const;
+    void operator+= (const valarray<value_type>& v) const;
+    void operator-= (const valarray<value_type>& v) const;
+    void operator^= (const valarray<value_type>& v) const;
+    void operator&= (const valarray<value_type>& v) const;
+    void operator|= (const valarray<value_type>& v) const;
+    void operator<<=(const valarray<value_type>& v) const;
+    void operator>>=(const valarray<value_type>& v) const;
+
+    mask_array(const mask_array& ma);
+    ~mask_array();
+    const mask_array& operator=(const mask_array& ma) const;
+    void operator=(const value_type& x) const;
+
+    mask_array() = delete;
+};
+
+template <class T>
+class indirect_array
+{
+public:
+    typedef T value_type;
+
+    void operator=  (const valarray<value_type>& v) const;
+    void operator*= (const valarray<value_type>& v) const;
+    void operator/= (const valarray<value_type>& v) const;
+    void operator%= (const valarray<value_type>& v) const;
+    void operator+= (const valarray<value_type>& v) const;
+    void operator-= (const valarray<value_type>& v) const;
+    void operator^= (const valarray<value_type>& v) const;
+    void operator&= (const valarray<value_type>& v) const;
+    void operator|= (const valarray<value_type>& v) const;
+    void operator<<=(const valarray<value_type>& v) const;
+    void operator>>=(const valarray<value_type>& v) const;
+
+    indirect_array(const indirect_array& ia);
+    ~indirect_array();
+    const indirect_array& operator=(const indirect_array& ia) const;
+    void operator=(const value_type& x) const;
+
+    indirect_array() = delete;
+};
+
+template<class T> void swap(valarray<T>& x, valarray<T>& y) noexcept;
+
+template<class T> valarray<T> operator* (const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<T> operator* (const valarray<T>& x, const T& y);
+template<class T> valarray<T> operator* (const T& x, const valarray<T>& y);
+
+template<class T> valarray<T> operator/ (const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<T> operator/ (const valarray<T>& x, const T& y);
+template<class T> valarray<T> operator/ (const T& x, const valarray<T>& y);
+
+template<class T> valarray<T> operator% (const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<T> operator% (const valarray<T>& x, const T& y);
+template<class T> valarray<T> operator% (const T& x, const valarray<T>& y);
+
+template<class T> valarray<T> operator+ (const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<T> operator+ (const valarray<T>& x, const T& y);
+template<class T> valarray<T> operator+ (const T& x, const valarray<T>& y);
+
+template<class T> valarray<T> operator- (const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<T> operator- (const valarray<T>& x, const T& y);
+template<class T> valarray<T> operator- (const T& x, const valarray<T>& y);
+
+template<class T> valarray<T> operator^ (const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<T> operator^ (const valarray<T>& x, const T& y);
+template<class T> valarray<T> operator^ (const T& x, const valarray<T>& y);
+
+template<class T> valarray<T> operator& (const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<T> operator& (const valarray<T>& x, const T& y);
+template<class T> valarray<T> operator& (const T& x, const valarray<T>& y);
+
+template<class T> valarray<T> operator| (const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<T> operator| (const valarray<T>& x, const T& y);
+template<class T> valarray<T> operator| (const T& x, const valarray<T>& y);
+
+template<class T> valarray<T> operator<<(const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<T> operator<<(const valarray<T>& x, const T& y);
+template<class T> valarray<T> operator<<(const T& x, const valarray<T>& y);
+
+template<class T> valarray<T> operator>>(const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<T> operator>>(const valarray<T>& x, const T& y);
+template<class T> valarray<T> operator>>(const T& x, const valarray<T>& y);
+
+template<class T> valarray<bool> operator&&(const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<bool> operator&&(const valarray<T>& x, const T& y);
+template<class T> valarray<bool> operator&&(const T& x, const valarray<T>& y);
+
+template<class T> valarray<bool> operator||(const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<bool> operator||(const valarray<T>& x, const T& y);
+template<class T> valarray<bool> operator||(const T& x, const valarray<T>& y);
+
+template<class T> valarray<bool> operator==(const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<bool> operator==(const valarray<T>& x, const T& y);
+template<class T> valarray<bool> operator==(const T& x, const valarray<T>& y);
+
+template<class T> valarray<bool> operator!=(const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<bool> operator!=(const valarray<T>& x, const T& y);
+template<class T> valarray<bool> operator!=(const T& x, const valarray<T>& y);
+
+template<class T> valarray<bool> operator< (const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<bool> operator< (const valarray<T>& x, const T& y);
+template<class T> valarray<bool> operator< (const T& x, const valarray<T>& y);
+
+template<class T> valarray<bool> operator> (const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<bool> operator> (const valarray<T>& x, const T& y);
+template<class T> valarray<bool> operator> (const T& x, const valarray<T>& y);
+
+template<class T> valarray<bool> operator<=(const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<bool> operator<=(const valarray<T>& x, const T& y);
+template<class T> valarray<bool> operator<=(const T& x, const valarray<T>& y);
+
+template<class T> valarray<bool> operator>=(const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<bool> operator>=(const valarray<T>& x, const T& y);
+template<class T> valarray<bool> operator>=(const T& x, const valarray<T>& y);
+
+template<class T> valarray<T> abs (const valarray<T>& x);
+template<class T> valarray<T> acos (const valarray<T>& x);
+template<class T> valarray<T> asin (const valarray<T>& x);
+template<class T> valarray<T> atan (const valarray<T>& x);
+
+template<class T> valarray<T> atan2(const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<T> atan2(const valarray<T>& x, const T& y);
+template<class T> valarray<T> atan2(const T& x, const valarray<T>& y);
+
+template<class T> valarray<T> cos (const valarray<T>& x);
+template<class T> valarray<T> cosh (const valarray<T>& x);
+template<class T> valarray<T> exp (const valarray<T>& x);
+template<class T> valarray<T> log (const valarray<T>& x);
+template<class T> valarray<T> log10(const valarray<T>& x);
+
+template<class T> valarray<T> pow(const valarray<T>& x, const valarray<T>& y);
+template<class T> valarray<T> pow(const valarray<T>& x, const T& y);
+template<class T> valarray<T> pow(const T& x, const valarray<T>& y);
+
+template<class T> valarray<T> sin (const valarray<T>& x);
+template<class T> valarray<T> sinh (const valarray<T>& x);
+template<class T> valarray<T> sqrt (const valarray<T>& x);
+template<class T> valarray<T> tan (const valarray<T>& x);
+template<class T> valarray<T> tanh (const valarray<T>& x);
+
+template <class T> unspecified1 begin(valarray<T>& v);
+template <class T> unspecified2 begin(const valarray<T>& v);
+template <class T> unspecified1 end(valarray<T>& v);
+template <class T> unspecified2 end(const valarray<T>& v);
+
+}  // std
+
+*/
+
+#include <__algorithm/copy.h>
+#include <__algorithm/count.h>
+#include <__algorithm/fill.h>
+#include <__algorithm/max_element.h>
+#include <__algorithm/min.h>
+#include <__algorithm/min_element.h>
+#include <__algorithm/unwrap_iter.h>
+#include <__assert>
+#include <__config>
+#include <__functional/operations.h>
+#include <__memory/addressof.h>
+#include <__memory/allocator.h>
+#include <__memory/uninitialized_algorithms.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/remove_reference.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+#include <cmath>
+#include <cstddef>
+#include <new>
+#include <version>
+
+// standard-mandated includes
+
+// [valarray.syn]
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS valarray;
+
+class _LIBCPP_TEMPLATE_VIS slice {
+  size_t __start_;
+  size_t __size_;
+  size_t __stride_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI slice() : __start_(0), __size_(0), __stride_(0) {}
+
+  _LIBCPP_HIDE_FROM_ABI slice(size_t __start, size_t __size, size_t __stride)
+      : __start_(__start), __size_(__size), __stride_(__stride) {}
+
+  _LIBCPP_HIDE_FROM_ABI size_t start() const { return __start_; }
+  _LIBCPP_HIDE_FROM_ABI size_t size() const { return __size_; }
+  _LIBCPP_HIDE_FROM_ABI size_t stride() const { return __stride_; }
+
+#if _LIBCPP_STD_VER >= 20
+
+  _LIBCPP_HIDE_FROM_ABI friend bool operator==(const slice& __x, const slice& __y) {
+    return __x.start() == __y.start() && __x.size() == __y.size() && __x.stride() == __y.stride();
+  }
+
+#endif
+};
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS slice_array;
+class _LIBCPP_EXPORTED_FROM_ABI gslice;
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS gslice_array;
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS mask_array;
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS indirect_array;
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp* begin(valarray<_Tp>& __v);
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI const _Tp* begin(const valarray<_Tp>& __v);
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp* end(valarray<_Tp>& __v);
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI const _Tp* end(const valarray<_Tp>& __v);
+
+template <class _Op, class _A0>
+struct _UnaryOp {
+  typedef typename _Op::__result_type __result_type;
+  using value_type = __decay_t<__result_type>;
+
+  _Op __op_;
+  _A0 __a0_;
+
+  _LIBCPP_HIDE_FROM_ABI _UnaryOp(const _Op& __op, const _A0& __a0) : __op_(__op), __a0_(__a0) {}
+
+  _LIBCPP_HIDE_FROM_ABI __result_type operator[](size_t __i) const { return __op_(__a0_[__i]); }
+
+  _LIBCPP_HIDE_FROM_ABI size_t size() const { return __a0_.size(); }
+};
+
+template <class _Op, class _A0, class _A1>
+struct _BinaryOp {
+  typedef typename _Op::__result_type __result_type;
+  using value_type = __decay_t<__result_type>;
+
+  _Op __op_;
+  _A0 __a0_;
+  _A1 __a1_;
+
+  _LIBCPP_HIDE_FROM_ABI _BinaryOp(const _Op& __op, const _A0& __a0, const _A1& __a1)
+      : __op_(__op), __a0_(__a0), __a1_(__a1) {}
+
+  _LIBCPP_HIDE_FROM_ABI __result_type operator[](size_t __i) const { return __op_(__a0_[__i], __a1_[__i]); }
+
+  _LIBCPP_HIDE_FROM_ABI size_t size() const { return __a0_.size(); }
+};
+
+template <class _Tp>
+class __scalar_expr {
+public:
+  typedef _Tp value_type;
+  typedef const _Tp& __result_type;
+
+private:
+  const value_type& __t_;
+  size_t __s_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __scalar_expr(const value_type& __t, size_t __s) : __t_(__t), __s_(__s) {}
+
+  _LIBCPP_HIDE_FROM_ABI __result_type operator[](size_t) const { return __t_; }
+
+  _LIBCPP_HIDE_FROM_ABI size_t size() const { return __s_; }
+};
+
+template <class _Tp>
+struct __unary_plus {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return +__x; }
+};
+
+template <class _Tp>
+struct __bit_not {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return ~__x; }
+};
+
+template <class _Tp>
+struct __bit_shift_left {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const { return __x << __y; }
+};
+
+template <class _Tp>
+struct __bit_shift_right {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const { return __x >> __y; }
+};
+
+template <class _Tp, class _Fp>
+struct __apply_expr {
+private:
+  _Fp __f_;
+
+public:
+  typedef _Tp __result_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __apply_expr(_Fp __f) : __f_(__f) {}
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return __f_(__x); }
+};
+
+template <class _Tp>
+struct __abs_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return std::abs(__x); }
+};
+
+template <class _Tp>
+struct __acos_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return std::acos(__x); }
+};
+
+template <class _Tp>
+struct __asin_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return std::asin(__x); }
+};
+
+template <class _Tp>
+struct __atan_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return std::atan(__x); }
+};
+
+template <class _Tp>
+struct __atan2_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const { return std::atan2(__x, __y); }
+};
+
+template <class _Tp>
+struct __cos_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return std::cos(__x); }
+};
+
+template <class _Tp>
+struct __cosh_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return std::cosh(__x); }
+};
+
+template <class _Tp>
+struct __exp_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return std::exp(__x); }
+};
+
+template <class _Tp>
+struct __log_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return std::log(__x); }
+};
+
+template <class _Tp>
+struct __log10_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return std::log10(__x); }
+};
+
+template <class _Tp>
+struct __pow_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const { return std::pow(__x, __y); }
+};
+
+template <class _Tp>
+struct __sin_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return std::sin(__x); }
+};
+
+template <class _Tp>
+struct __sinh_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return std::sinh(__x); }
+};
+
+template <class _Tp>
+struct __sqrt_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return std::sqrt(__x); }
+};
+
+template <class _Tp>
+struct __tan_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return std::tan(__x); }
+};
+
+template <class _Tp>
+struct __tanh_expr {
+  typedef _Tp __result_type;
+  _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return std::tanh(__x); }
+};
+
+template <class _ValExpr>
+class __slice_expr {
+  typedef __libcpp_remove_reference_t<_ValExpr> _RmExpr;
+
+public:
+  typedef typename _RmExpr::value_type value_type;
+  typedef value_type __result_type;
+
+private:
+  _ValExpr __expr_;
+  size_t __start_;
+  size_t __size_;
+  size_t __stride_;
+
+  _LIBCPP_HIDE_FROM_ABI __slice_expr(const slice& __sl, const _RmExpr& __e)
+      : __expr_(__e), __start_(__sl.start()), __size_(__sl.size()), __stride_(__sl.stride()) {}
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __result_type operator[](size_t __i) const { return __expr_[__start_ + __i * __stride_]; }
+
+  _LIBCPP_HIDE_FROM_ABI size_t size() const { return __size_; }
+
+  template <class>
+  friend class __val_expr;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS valarray;
+};
+
+template <class _ValExpr>
+class __mask_expr;
+
+template <class _ValExpr>
+class __indirect_expr;
+
+template <class _ValExpr>
+class __shift_expr {
+  typedef __libcpp_remove_reference_t<_ValExpr> _RmExpr;
+
+public:
+  typedef typename _RmExpr::value_type value_type;
+  typedef value_type __result_type;
+
+private:
+  _ValExpr __expr_;
+  size_t __size_;
+  ptrdiff_t __ul_;
+  ptrdiff_t __sn_;
+  ptrdiff_t __n_;
+  static const ptrdiff_t _Np = static_cast<ptrdiff_t>(sizeof(ptrdiff_t) * __CHAR_BIT__ - 1);
+
+  _LIBCPP_HIDE_FROM_ABI __shift_expr(int __n, const _RmExpr& __e) : __expr_(__e), __size_(__e.size()), __n_(__n) {
+    ptrdiff_t __neg_n = static_cast<ptrdiff_t>(__n_ >> _Np);
+    __sn_             = __neg_n | static_cast<ptrdiff_t>(static_cast<size_t>(-__n_) >> _Np);
+    __ul_             = ((__size_ - __n_) & ~__neg_n) | ((__n_ + 1) & __neg_n);
+  }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __result_type operator[](size_t __j) const {
+    ptrdiff_t __i = static_cast<ptrdiff_t>(__j);
+    ptrdiff_t __m = (__sn_ * __i - __ul_) >> _Np;
+    return (__expr_[(__i + __n_) & __m] & __m) | (value_type() & ~__m);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_t size() const { return __size_; }
+
+  template <class>
+  friend class __val_expr;
+};
+
+template <class _ValExpr>
+class __cshift_expr {
+  typedef __libcpp_remove_reference_t<_ValExpr> _RmExpr;
+
+public:
+  typedef typename _RmExpr::value_type value_type;
+  typedef value_type __result_type;
+
+private:
+  _ValExpr __expr_;
+  size_t __size_;
+  size_t __m_;
+  size_t __o1_;
+  size_t __o2_;
+
+  _LIBCPP_HIDE_FROM_ABI __cshift_expr(int __n, const _RmExpr& __e) : __expr_(__e), __size_(__e.size()) {
+    __n %= static_cast<int>(__size_);
+    if (__n >= 0) {
+      __m_  = __size_ - __n;
+      __o1_ = __n;
+      __o2_ = __n - __size_;
+    } else {
+      __m_  = -__n;
+      __o1_ = __n + __size_;
+      __o2_ = __n;
+    }
+  }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __result_type operator[](size_t __i) const {
+    if (__i < __m_)
+      return __expr_[__i + __o1_];
+    return __expr_[__i + __o2_];
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_t size() const { return __size_; }
+
+  template <class>
+  friend class __val_expr;
+};
+
+template <class _ValExpr>
+class __val_expr;
+
+template <class _ValExpr>
+struct __is_val_expr : false_type {};
+
+template <class _ValExpr>
+struct __is_val_expr<__val_expr<_ValExpr> > : true_type {};
+
+template <class _Tp>
+struct __is_val_expr<valarray<_Tp> > : true_type {};
+
+template <class _Tp>
+struct __is_val_expr<slice_array<_Tp> > : true_type {};
+
+template <class _Tp>
+struct __is_val_expr<gslice_array<_Tp> > : true_type {};
+
+template <class _Tp>
+struct __is_val_expr<mask_array<_Tp> > : true_type {};
+
+template <class _Tp>
+struct __is_val_expr<indirect_array<_Tp> > : true_type {};
+
+// The functions using a __val_expr access the elements by their index.
+// valarray and the libc++ lazy proxies have an operator[]. The
+// Standard proxy array's don't have this operator, instead they have a
+// implementation specific accessor
+//   __get(size_t)
+//
+// The functions use the non-member function
+//   __get(__val_expr, size_t)
+//
+// If the __val_expr is a specialization of __val_expr_use_member_functions it
+// uses the __val_expr's member function
+//   __get(size_t)
+// else it uses the __val_expr's member function
+//   operator[](size_t)
+template <class _ValExpr>
+struct __val_expr_use_member_functions;
+
+template <class>
+struct __val_expr_use_member_functions : false_type {};
+
+template <class _Tp>
+struct __val_expr_use_member_functions<slice_array<_Tp> > : true_type {};
+
+template <class _Tp>
+struct __val_expr_use_member_functions<gslice_array<_Tp> > : true_type {};
+
+template <class _Tp>
+struct __val_expr_use_member_functions<mask_array<_Tp> > : true_type {};
+
+template <class _Tp>
+struct __val_expr_use_member_functions<indirect_array<_Tp> > : true_type {};
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS valarray {
+public:
+  typedef _Tp value_type;
+  typedef _Tp __result_type;
+
+private:
+  value_type* __begin_;
+  value_type* __end_;
+
+public:
+  // construct/destroy:
+  _LIBCPP_HIDE_FROM_ABI valarray() : __begin_(nullptr), __end_(nullptr) {}
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 explicit valarray(size_t __n);
+  _LIBCPP_HIDE_FROM_ABI valarray(const value_type& __x, size_t __n);
+  valarray(const value_type* __p, size_t __n);
+  valarray(const valarray& __v);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI valarray(valarray&& __v) _NOEXCEPT;
+  valarray(initializer_list<value_type> __il);
+#endif // _LIBCPP_CXX03_LANG
+  valarray(const slice_array<value_type>& __sa);
+  valarray(const gslice_array<value_type>& __ga);
+  valarray(const mask_array<value_type>& __ma);
+  valarray(const indirect_array<value_type>& __ia);
+  inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 ~valarray();
+
+  // assignment:
+  valarray& operator=(const valarray& __v);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI valarray& operator=(valarray&& __v) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI valarray& operator=(initializer_list<value_type>);
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI valarray& operator=(const value_type& __x);
+  _LIBCPP_HIDE_FROM_ABI valarray& operator=(const slice_array<value_type>& __sa);
+  _LIBCPP_HIDE_FROM_ABI valarray& operator=(const gslice_array<value_type>& __ga);
+  _LIBCPP_HIDE_FROM_ABI valarray& operator=(const mask_array<value_type>& __ma);
+  _LIBCPP_HIDE_FROM_ABI valarray& operator=(const indirect_array<value_type>& __ia);
+  template <class _ValExpr>
+  _LIBCPP_HIDE_FROM_ABI valarray& operator=(const __val_expr<_ValExpr>& __v);
+
+  // element access:
+  _LIBCPP_HIDE_FROM_ABI const value_type& operator[](size_t __i) const { return __begin_[__i]; }
+
+  _LIBCPP_HIDE_FROM_ABI value_type& operator[](size_t __i) { return __begin_[__i]; }
+
+  // subset operations:
+  _LIBCPP_HIDE_FROM_ABI __val_expr<__slice_expr<const valarray&> > operator[](slice __s) const;
+  _LIBCPP_HIDE_FROM_ABI slice_array<value_type> operator[](slice __s);
+  _LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<const valarray&> > operator[](const gslice& __gs) const;
+  _LIBCPP_HIDE_FROM_ABI gslice_array<value_type> operator[](const gslice& __gs);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<const valarray&> > operator[](gslice&& __gs) const;
+  _LIBCPP_HIDE_FROM_ABI gslice_array<value_type> operator[](gslice&& __gs);
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI __val_expr<__mask_expr<const valarray&> > operator[](const valarray<bool>& __vb) const;
+  _LIBCPP_HIDE_FROM_ABI mask_array<value_type> operator[](const valarray<bool>& __vb);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI __val_expr<__mask_expr<const valarray&> > operator[](valarray<bool>&& __vb) const;
+  _LIBCPP_HIDE_FROM_ABI mask_array<value_type> operator[](valarray<bool>&& __vb);
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<const valarray&> > operator[](const valarray<size_t>& __vs) const;
+  _LIBCPP_HIDE_FROM_ABI indirect_array<value_type> operator[](const valarray<size_t>& __vs);
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<const valarray&> > operator[](valarray<size_t>&& __vs) const;
+  _LIBCPP_HIDE_FROM_ABI indirect_array<value_type> operator[](valarray<size_t>&& __vs);
+#endif // _LIBCPP_CXX03_LANG
+
+  // unary operators:
+  _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__unary_plus<_Tp>, const valarray&> > operator+() const;
+  _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<negate<_Tp>, const valarray&> > operator-() const;
+  _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__bit_not<_Tp>, const valarray&> > operator~() const;
+  _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<logical_not<_Tp>, const valarray&> > operator!() const;
+
+  // computed assignment:
+  _LIBCPP_HIDE_FROM_ABI valarray& operator*=(const value_type& __x);
+  _LIBCPP_HIDE_FROM_ABI valarray& operator/=(const value_type& __x);
+  _LIBCPP_HIDE_FROM_ABI valarray& operator%=(const value_type& __x);
+  _LIBCPP_HIDE_FROM_ABI valarray& operator+=(const value_type& __x);
+  _LIBCPP_HIDE_FROM_ABI valarray& operator-=(const value_type& __x);
+  _LIBCPP_HIDE_FROM_ABI valarray& operator^=(const value_type& __x);
+  _LIBCPP_HIDE_FROM_ABI valarray& operator&=(const value_type& __x);
+  _LIBCPP_HIDE_FROM_ABI valarray& operator|=(const value_type& __x);
+  _LIBCPP_HIDE_FROM_ABI valarray& operator<<=(const value_type& __x);
+  _LIBCPP_HIDE_FROM_ABI valarray& operator>>=(const value_type& __x);
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI valarray& operator*=(const _Expr& __v);
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI valarray& operator/=(const _Expr& __v);
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI valarray& operator%=(const _Expr& __v);
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI valarray& operator+=(const _Expr& __v);
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI valarray& operator-=(const _Expr& __v);
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI valarray& operator^=(const _Expr& __v);
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI valarray& operator|=(const _Expr& __v);
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI valarray& operator&=(const _Expr& __v);
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI valarray& operator<<=(const _Expr& __v);
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI valarray& operator>>=(const _Expr& __v);
+
+  // member functions:
+  _LIBCPP_HIDE_FROM_ABI void swap(valarray& __v) _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI size_t size() const { return static_cast<size_t>(__end_ - __begin_); }
+
+  _LIBCPP_HIDE_FROM_ABI value_type sum() const;
+  _LIBCPP_HIDE_FROM_ABI value_type min() const;
+  _LIBCPP_HIDE_FROM_ABI value_type max() const;
+
+  valarray shift(int __i) const;
+  valarray cshift(int __i) const;
+  valarray apply(value_type __f(value_type)) const;
+  valarray apply(value_type __f(const value_type&)) const;
+  void resize(size_t __n, value_type __x = value_type());
+
+private:
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS valarray;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS slice_array;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS gslice_array;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS mask_array;
+  template <class>
+  friend class __mask_expr;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS indirect_array;
+  template <class>
+  friend class __indirect_expr;
+  template <class>
+  friend class __val_expr;
+
+  template <class _Up>
+  friend _Up* begin(valarray<_Up>& __v);
+
+  template <class _Up>
+  friend const _Up* begin(const valarray<_Up>& __v);
+
+  template <class _Up>
+  friend _Up* end(valarray<_Up>& __v);
+
+  template <class _Up>
+  friend const _Up* end(const valarray<_Up>& __v);
+
+  _LIBCPP_HIDE_FROM_ABI void __clear(size_t __capacity);
+  valarray& __assign_range(const value_type* __f, const value_type* __l);
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp, size_t _Size>
+valarray(const _Tp (&)[_Size], size_t) -> valarray<_Tp>;
+#endif
+
+template <class _Expr,
+          __enable_if_t<__is_val_expr<_Expr>::value && __val_expr_use_member_functions<_Expr>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI typename _Expr::value_type __get(const _Expr& __v, size_t __i) {
+  return __v.__get(__i);
+}
+
+template <class _Expr,
+          __enable_if_t<__is_val_expr<_Expr>::value && !__val_expr_use_member_functions<_Expr>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI typename _Expr::value_type __get(const _Expr& __v, size_t __i) {
+  return __v[__i];
+}
+
+extern template _LIBCPP_EXPORTED_FROM_ABI void valarray<size_t>::resize(size_t, size_t);
+
+template <class _Op, class _Tp>
+struct _UnaryOp<_Op, valarray<_Tp> > {
+  typedef typename _Op::__result_type __result_type;
+  using value_type = __decay_t<__result_type>;
+
+  _Op __op_;
+  const valarray<_Tp>& __a0_;
+
+  _LIBCPP_HIDE_FROM_ABI _UnaryOp(const _Op& __op, const valarray<_Tp>& __a0) : __op_(__op), __a0_(__a0) {}
+
+  _LIBCPP_HIDE_FROM_ABI __result_type operator[](size_t __i) const { return __op_(__a0_[__i]); }
+
+  _LIBCPP_HIDE_FROM_ABI size_t size() const { return __a0_.size(); }
+};
+
+template <class _Op, class _Tp, class _A1>
+struct _BinaryOp<_Op, valarray<_Tp>, _A1> {
+  typedef typename _Op::__result_type __result_type;
+  using value_type = __decay_t<__result_type>;
+
+  _Op __op_;
+  const valarray<_Tp>& __a0_;
+  _A1 __a1_;
+
+  _LIBCPP_HIDE_FROM_ABI _BinaryOp(const _Op& __op, const valarray<_Tp>& __a0, const _A1& __a1)
+      : __op_(__op), __a0_(__a0), __a1_(__a1) {}
+
+  _LIBCPP_HIDE_FROM_ABI __result_type operator[](size_t __i) const { return __op_(__a0_[__i], __a1_[__i]); }
+
+  _LIBCPP_HIDE_FROM_ABI size_t size() const { return __a0_.size(); }
+};
+
+template <class _Op, class _A0, class _Tp>
+struct _BinaryOp<_Op, _A0, valarray<_Tp> > {
+  typedef typename _Op::__result_type __result_type;
+  using value_type = __decay_t<__result_type>;
+
+  _Op __op_;
+  _A0 __a0_;
+  const valarray<_Tp>& __a1_;
+
+  _LIBCPP_HIDE_FROM_ABI _BinaryOp(const _Op& __op, const _A0& __a0, const valarray<_Tp>& __a1)
+      : __op_(__op), __a0_(__a0), __a1_(__a1) {}
+
+  _LIBCPP_HIDE_FROM_ABI __result_type operator[](size_t __i) const { return __op_(__a0_[__i], __a1_[__i]); }
+
+  _LIBCPP_HIDE_FROM_ABI size_t size() const { return __a0_.size(); }
+};
+
+template <class _Op, class _Tp>
+struct _BinaryOp<_Op, valarray<_Tp>, valarray<_Tp> > {
+  typedef typename _Op::__result_type __result_type;
+  using value_type = __decay_t<__result_type>;
+
+  _Op __op_;
+  const valarray<_Tp>& __a0_;
+  const valarray<_Tp>& __a1_;
+
+  _LIBCPP_HIDE_FROM_ABI _BinaryOp(const _Op& __op, const valarray<_Tp>& __a0, const valarray<_Tp>& __a1)
+      : __op_(__op), __a0_(__a0), __a1_(__a1) {}
+
+  _LIBCPP_HIDE_FROM_ABI __result_type operator[](size_t __i) const { return __op_(__a0_[__i], __a1_[__i]); }
+
+  _LIBCPP_HIDE_FROM_ABI size_t size() const { return __a0_.size(); }
+};
+
+// slice_array
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS slice_array {
+public:
+  typedef _Tp value_type;
+
+private:
+  value_type* __vp_;
+  size_t __size_;
+  size_t __stride_;
+
+public:
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator*=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator/=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator%=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator+=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator-=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator^=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator&=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator|=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator<<=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator>>=(const _Expr& __v) const;
+
+  slice_array(slice_array const&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI const slice_array& operator=(const slice_array& __sa) const;
+
+  _LIBCPP_HIDE_FROM_ABI void operator=(const value_type& __x) const;
+
+  _LIBCPP_HIDE_FROM_ABI void operator=(const valarray<value_type>& __va) const;
+
+  // Behaves like __val_expr::operator[], which returns by value.
+  _LIBCPP_HIDE_FROM_ABI value_type __get(size_t __i) const {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__i < __size_, "slice_array.__get() index out of bounds");
+    return __vp_[__i * __stride_];
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI slice_array(const slice& __sl, const valarray<value_type>& __v)
+      : __vp_(const_cast<value_type*>(__v.__begin_ + __sl.start())), __size_(__sl.size()), __stride_(__sl.stride()) {}
+
+  template <class>
+  friend class valarray;
+};
+
+template <class _Tp>
+inline const slice_array<_Tp>& slice_array<_Tp>::operator=(const slice_array& __sa) const {
+  value_type* __t       = __vp_;
+  const value_type* __s = __sa.__vp_;
+  for (size_t __n = __size_; __n; --__n, __t += __stride_, __s += __sa.__stride_)
+    *__t = *__s;
+  return *this;
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void slice_array<_Tp>::operator=(const _Expr& __v) const {
+  value_type* __t = __vp_;
+  for (size_t __i = 0; __i < __size_; ++__i, __t += __stride_)
+    *__t = __v[__i];
+}
+
+template <class _Tp>
+inline void slice_array<_Tp>::operator=(const valarray<value_type>& __va) const {
+  value_type* __t = __vp_;
+  for (size_t __i = 0; __i < __va.size(); ++__i, __t += __stride_)
+    *__t = __va[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void slice_array<_Tp>::operator*=(const _Expr& __v) const {
+  value_type* __t = __vp_;
+  for (size_t __i = 0; __i < __size_; ++__i, __t += __stride_)
+    *__t *= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void slice_array<_Tp>::operator/=(const _Expr& __v) const {
+  value_type* __t = __vp_;
+  for (size_t __i = 0; __i < __size_; ++__i, __t += __stride_)
+    *__t /= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void slice_array<_Tp>::operator%=(const _Expr& __v) const {
+  value_type* __t = __vp_;
+  for (size_t __i = 0; __i < __size_; ++__i, __t += __stride_)
+    *__t %= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void slice_array<_Tp>::operator+=(const _Expr& __v) const {
+  value_type* __t = __vp_;
+  for (size_t __i = 0; __i < __size_; ++__i, __t += __stride_)
+    *__t += __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void slice_array<_Tp>::operator-=(const _Expr& __v) const {
+  value_type* __t = __vp_;
+  for (size_t __i = 0; __i < __size_; ++__i, __t += __stride_)
+    *__t -= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void slice_array<_Tp>::operator^=(const _Expr& __v) const {
+  value_type* __t = __vp_;
+  for (size_t __i = 0; __i < __size_; ++__i, __t += __stride_)
+    *__t ^= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void slice_array<_Tp>::operator&=(const _Expr& __v) const {
+  value_type* __t = __vp_;
+  for (size_t __i = 0; __i < __size_; ++__i, __t += __stride_)
+    *__t &= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void slice_array<_Tp>::operator|=(const _Expr& __v) const {
+  value_type* __t = __vp_;
+  for (size_t __i = 0; __i < __size_; ++__i, __t += __stride_)
+    *__t |= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void slice_array<_Tp>::operator<<=(const _Expr& __v) const {
+  value_type* __t = __vp_;
+  for (size_t __i = 0; __i < __size_; ++__i, __t += __stride_)
+    *__t <<= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void slice_array<_Tp>::operator>>=(const _Expr& __v) const {
+  value_type* __t = __vp_;
+  for (size_t __i = 0; __i < __size_; ++__i, __t += __stride_)
+    *__t >>= __v[__i];
+}
+
+template <class _Tp>
+inline void slice_array<_Tp>::operator=(const value_type& __x) const {
+  value_type* __t = __vp_;
+  for (size_t __n = __size_; __n; --__n, __t += __stride_)
+    *__t = __x;
+}
+
+// gslice
+
+class _LIBCPP_EXPORTED_FROM_ABI gslice {
+  valarray<size_t> __size_;
+  valarray<size_t> __stride_;
+  valarray<size_t> __1d_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI gslice() {}
+
+  _LIBCPP_HIDE_FROM_ABI gslice(size_t __start, const valarray<size_t>& __size, const valarray<size_t>& __stride)
+      : __size_(__size), __stride_(__stride) {
+    __init(__start);
+  }
+
+#ifndef _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI gslice(size_t __start, const valarray<size_t>& __size, valarray<size_t>&& __stride)
+      : __size_(__size), __stride_(std::move(__stride)) {
+    __init(__start);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI gslice(size_t __start, valarray<size_t>&& __size, const valarray<size_t>& __stride)
+      : __size_(std::move(__size)), __stride_(__stride) {
+    __init(__start);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI gslice(size_t __start, valarray<size_t>&& __size, valarray<size_t>&& __stride)
+      : __size_(std::move(__size)), __stride_(std::move(__stride)) {
+    __init(__start);
+  }
+
+#endif // _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI size_t start() const { return __1d_.size() ? __1d_[0] : 0; }
+
+  _LIBCPP_HIDE_FROM_ABI valarray<size_t> size() const { return __size_; }
+
+  _LIBCPP_HIDE_FROM_ABI valarray<size_t> stride() const { return __stride_; }
+
+private:
+  void __init(size_t __start);
+
+  template <class>
+  friend class gslice_array;
+  template <class>
+  friend class valarray;
+  template <class>
+  friend class __val_expr;
+};
+
+// gslice_array
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS gslice_array {
+public:
+  typedef _Tp value_type;
+
+private:
+  value_type* __vp_;
+  valarray<size_t> __1d_;
+
+public:
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator*=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator/=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator%=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator+=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator-=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator^=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator&=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator|=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator<<=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator>>=(const _Expr& __v) const;
+
+  _LIBCPP_HIDE_FROM_ABI const gslice_array& operator=(const gslice_array& __ga) const;
+
+  _LIBCPP_HIDE_FROM_ABI void operator=(const value_type& __x) const;
+
+  gslice_array(const gslice_array&) = default;
+
+  // Behaves like __val_expr::operator[], which returns by value.
+  _LIBCPP_HIDE_FROM_ABI value_type __get(size_t __i) const {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__i < __1d_.size(), "gslice_array.__get() index out of bounds");
+    return __vp_[__1d_[__i]];
+  }
+
+private:
+  gslice_array(const gslice& __gs, const valarray<value_type>& __v)
+      : __vp_(const_cast<value_type*>(__v.__begin_)), __1d_(__gs.__1d_) {}
+
+#ifndef _LIBCPP_CXX03_LANG
+  gslice_array(gslice&& __gs, const valarray<value_type>& __v)
+      : __vp_(const_cast<value_type*>(__v.__begin_)), __1d_(std::move(__gs.__1d_)) {}
+#endif // _LIBCPP_CXX03_LANG
+
+  template <class>
+  friend class valarray;
+};
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void gslice_array<_Tp>::operator=(const _Expr& __v) const {
+  typedef const size_t* _Ip;
+  size_t __j = 0;
+  for (_Ip __i = __1d_.__begin_, __e = __1d_.__end_; __i != __e; ++__i, ++__j)
+    __vp_[*__i] = __v[__j];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void gslice_array<_Tp>::operator*=(const _Expr& __v) const {
+  typedef const size_t* _Ip;
+  size_t __j = 0;
+  for (_Ip __i = __1d_.__begin_, __e = __1d_.__end_; __i != __e; ++__i, ++__j)
+    __vp_[*__i] *= __v[__j];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void gslice_array<_Tp>::operator/=(const _Expr& __v) const {
+  typedef const size_t* _Ip;
+  size_t __j = 0;
+  for (_Ip __i = __1d_.__begin_, __e = __1d_.__end_; __i != __e; ++__i, ++__j)
+    __vp_[*__i] /= __v[__j];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void gslice_array<_Tp>::operator%=(const _Expr& __v) const {
+  typedef const size_t* _Ip;
+  size_t __j = 0;
+  for (_Ip __i = __1d_.__begin_, __e = __1d_.__end_; __i != __e; ++__i, ++__j)
+    __vp_[*__i] %= __v[__j];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void gslice_array<_Tp>::operator+=(const _Expr& __v) const {
+  typedef const size_t* _Ip;
+  size_t __j = 0;
+  for (_Ip __i = __1d_.__begin_, __e = __1d_.__end_; __i != __e; ++__i, ++__j)
+    __vp_[*__i] += __v[__j];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void gslice_array<_Tp>::operator-=(const _Expr& __v) const {
+  typedef const size_t* _Ip;
+  size_t __j = 0;
+  for (_Ip __i = __1d_.__begin_, __e = __1d_.__end_; __i != __e; ++__i, ++__j)
+    __vp_[*__i] -= __v[__j];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void gslice_array<_Tp>::operator^=(const _Expr& __v) const {
+  typedef const size_t* _Ip;
+  size_t __j = 0;
+  for (_Ip __i = __1d_.__begin_, __e = __1d_.__end_; __i != __e; ++__i, ++__j)
+    __vp_[*__i] ^= __v[__j];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void gslice_array<_Tp>::operator&=(const _Expr& __v) const {
+  typedef const size_t* _Ip;
+  size_t __j = 0;
+  for (_Ip __i = __1d_.__begin_, __e = __1d_.__end_; __i != __e; ++__i, ++__j)
+    __vp_[*__i] &= __v[__j];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void gslice_array<_Tp>::operator|=(const _Expr& __v) const {
+  typedef const size_t* _Ip;
+  size_t __j = 0;
+  for (_Ip __i = __1d_.__begin_, __e = __1d_.__end_; __i != __e; ++__i, ++__j)
+    __vp_[*__i] |= __v[__j];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void gslice_array<_Tp>::operator<<=(const _Expr& __v) const {
+  typedef const size_t* _Ip;
+  size_t __j = 0;
+  for (_Ip __i = __1d_.__begin_, __e = __1d_.__end_; __i != __e; ++__i, ++__j)
+    __vp_[*__i] <<= __v[__j];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void gslice_array<_Tp>::operator>>=(const _Expr& __v) const {
+  typedef const size_t* _Ip;
+  size_t __j = 0;
+  for (_Ip __i = __1d_.__begin_, __e = __1d_.__end_; __i != __e; ++__i, ++__j)
+    __vp_[*__i] >>= __v[__j];
+}
+
+template <class _Tp>
+inline const gslice_array<_Tp>& gslice_array<_Tp>::operator=(const gslice_array& __ga) const {
+  typedef const size_t* _Ip;
+  const value_type* __s = __ga.__vp_;
+  for (_Ip __i = __1d_.__begin_, __e = __1d_.__end_, __j = __ga.__1d_.__begin_; __i != __e; ++__i, ++__j)
+    __vp_[*__i] = __s[*__j];
+  return *this;
+}
+
+template <class _Tp>
+inline void gslice_array<_Tp>::operator=(const value_type& __x) const {
+  typedef const size_t* _Ip;
+  for (_Ip __i = __1d_.__begin_, __e = __1d_.__end_; __i != __e; ++__i)
+    __vp_[*__i] = __x;
+}
+
+// mask_array
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS mask_array {
+public:
+  typedef _Tp value_type;
+
+private:
+  value_type* __vp_;
+  valarray<size_t> __1d_;
+
+public:
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator*=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator/=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator%=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator+=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator-=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator^=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator&=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator|=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator<<=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator>>=(const _Expr& __v) const;
+
+  mask_array(const mask_array&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI const mask_array& operator=(const mask_array& __ma) const;
+
+  _LIBCPP_HIDE_FROM_ABI void operator=(const value_type& __x) const;
+
+  // Behaves like __val_expr::operator[], which returns by value.
+  _LIBCPP_HIDE_FROM_ABI value_type __get(size_t __i) const {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__i < __1d_.size(), "mask_array.__get() index out of bounds");
+    return __vp_[__1d_[__i]];
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI mask_array(const valarray<bool>& __vb, const valarray<value_type>& __v)
+      : __vp_(const_cast<value_type*>(__v.__begin_)),
+        __1d_(static_cast<size_t>(count(__vb.__begin_, __vb.__end_, true))) {
+    size_t __j = 0;
+    for (size_t __i = 0; __i < __vb.size(); ++__i)
+      if (__vb[__i])
+        __1d_[__j++] = __i;
+  }
+
+  template <class>
+  friend class valarray;
+};
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void mask_array<_Tp>::operator=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] = __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void mask_array<_Tp>::operator*=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] *= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void mask_array<_Tp>::operator/=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] /= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void mask_array<_Tp>::operator%=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] %= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void mask_array<_Tp>::operator+=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] += __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void mask_array<_Tp>::operator-=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] -= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void mask_array<_Tp>::operator^=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] ^= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void mask_array<_Tp>::operator&=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] &= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void mask_array<_Tp>::operator|=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] |= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void mask_array<_Tp>::operator<<=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] <<= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void mask_array<_Tp>::operator>>=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] >>= __v[__i];
+}
+
+template <class _Tp>
+inline const mask_array<_Tp>& mask_array<_Tp>::operator=(const mask_array& __ma) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] = __ma.__vp_[__1d_[__i]];
+  return *this;
+}
+
+template <class _Tp>
+inline void mask_array<_Tp>::operator=(const value_type& __x) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] = __x;
+}
+
+template <class _ValExpr>
+class __mask_expr {
+  typedef __libcpp_remove_reference_t<_ValExpr> _RmExpr;
+
+public:
+  typedef typename _RmExpr::value_type value_type;
+  typedef value_type __result_type;
+
+private:
+  _ValExpr __expr_;
+  valarray<size_t> __1d_;
+
+  _LIBCPP_HIDE_FROM_ABI __mask_expr(const valarray<bool>& __vb, const _RmExpr& __e)
+      : __expr_(__e), __1d_(static_cast<size_t>(count(__vb.__begin_, __vb.__end_, true))) {
+    size_t __j = 0;
+    for (size_t __i = 0; __i < __vb.size(); ++__i)
+      if (__vb[__i])
+        __1d_[__j++] = __i;
+  }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __result_type operator[](size_t __i) const { return __expr_[__1d_[__i]]; }
+
+  _LIBCPP_HIDE_FROM_ABI size_t size() const { return __1d_.size(); }
+
+  template <class>
+  friend class __val_expr;
+  template <class>
+  friend class valarray;
+};
+
+// indirect_array
+
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS indirect_array {
+public:
+  typedef _Tp value_type;
+
+private:
+  value_type* __vp_;
+  valarray<size_t> __1d_;
+
+public:
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator*=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator/=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator%=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator+=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator-=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator^=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator&=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator|=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator<<=(const _Expr& __v) const;
+
+  template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI operator>>=(const _Expr& __v) const;
+
+  indirect_array(const indirect_array&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI const indirect_array& operator=(const indirect_array& __ia) const;
+
+  _LIBCPP_HIDE_FROM_ABI void operator=(const value_type& __x) const;
+
+  // Behaves like __val_expr::operator[], which returns by value.
+  _LIBCPP_HIDE_FROM_ABI value_type __get(size_t __i) const {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__i < __1d_.size(), "indirect_array.__get() index out of bounds");
+    return __vp_[__1d_[__i]];
+  }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI indirect_array(const valarray<size_t>& __ia, const valarray<value_type>& __v)
+      : __vp_(const_cast<value_type*>(__v.__begin_)), __1d_(__ia) {}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI indirect_array(valarray<size_t>&& __ia, const valarray<value_type>& __v)
+      : __vp_(const_cast<value_type*>(__v.__begin_)), __1d_(std::move(__ia)) {}
+
+#endif // _LIBCPP_CXX03_LANG
+
+  template <class>
+  friend class valarray;
+};
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void indirect_array<_Tp>::operator=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] = __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void indirect_array<_Tp>::operator*=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] *= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void indirect_array<_Tp>::operator/=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] /= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void indirect_array<_Tp>::operator%=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] %= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void indirect_array<_Tp>::operator+=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] += __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void indirect_array<_Tp>::operator-=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] -= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void indirect_array<_Tp>::operator^=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] ^= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void indirect_array<_Tp>::operator&=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] &= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void indirect_array<_Tp>::operator|=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] |= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void indirect_array<_Tp>::operator<<=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] <<= __v[__i];
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline void indirect_array<_Tp>::operator>>=(const _Expr& __v) const {
+  size_t __n = __1d_.size();
+  for (size_t __i = 0; __i < __n; ++__i)
+    __vp_[__1d_[__i]] >>= __v[__i];
+}
+
+template <class _Tp>
+inline const indirect_array<_Tp>& indirect_array<_Tp>::operator=(const indirect_array& __ia) const {
+  typedef const size_t* _Ip;
+  const value_type* __s = __ia.__vp_;
+  for (_Ip __i = __1d_.__begin_, __e = __1d_.__end_, __j = __ia.__1d_.__begin_; __i != __e; ++__i, ++__j)
+    __vp_[*__i] = __s[*__j];
+  return *this;
+}
+
+template <class _Tp>
+inline void indirect_array<_Tp>::operator=(const value_type& __x) const {
+  typedef const size_t* _Ip;
+  for (_Ip __i = __1d_.__begin_, __e = __1d_.__end_; __i != __e; ++__i)
+    __vp_[*__i] = __x;
+}
+
+template <class _ValExpr>
+class __indirect_expr {
+  typedef __libcpp_remove_reference_t<_ValExpr> _RmExpr;
+
+public:
+  typedef typename _RmExpr::value_type value_type;
+  typedef value_type __result_type;
+
+private:
+  _ValExpr __expr_;
+  valarray<size_t> __1d_;
+
+  _LIBCPP_HIDE_FROM_ABI __indirect_expr(const valarray<size_t>& __ia, const _RmExpr& __e) : __expr_(__e), __1d_(__ia) {}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI __indirect_expr(valarray<size_t>&& __ia, const _RmExpr& __e)
+      : __expr_(__e), __1d_(std::move(__ia)) {}
+
+#endif // _LIBCPP_CXX03_LANG
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __result_type operator[](size_t __i) const { return __expr_[__1d_[__i]]; }
+
+  _LIBCPP_HIDE_FROM_ABI size_t size() const { return __1d_.size(); }
+
+  template <class>
+  friend class __val_expr;
+  template <class>
+  friend class _LIBCPP_TEMPLATE_VIS valarray;
+};
+
+template <class _ValExpr>
+class __val_expr {
+  typedef __libcpp_remove_reference_t<_ValExpr> _RmExpr;
+
+  _ValExpr __expr_;
+
+public:
+  typedef typename _RmExpr::value_type value_type;
+  typedef typename _RmExpr::__result_type __result_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __val_expr(const _RmExpr& __e) : __expr_(__e) {}
+
+  _LIBCPP_HIDE_FROM_ABI __result_type operator[](size_t __i) const { return __expr_[__i]; }
+
+  _LIBCPP_HIDE_FROM_ABI __val_expr<__slice_expr<_ValExpr> > operator[](slice __s) const {
+    typedef __slice_expr<_ValExpr> _NewExpr;
+    return __val_expr< _NewExpr >(_NewExpr(__s, __expr_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<_ValExpr> > operator[](const gslice& __gs) const {
+    typedef __indirect_expr<_ValExpr> _NewExpr;
+    return __val_expr<_NewExpr >(_NewExpr(__gs.__1d_, __expr_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __val_expr<__mask_expr<_ValExpr> > operator[](const valarray<bool>& __vb) const {
+    typedef __mask_expr<_ValExpr> _NewExpr;
+    return __val_expr< _NewExpr >(_NewExpr(__vb, __expr_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<_ValExpr> > operator[](const valarray<size_t>& __vs) const {
+    typedef __indirect_expr<_ValExpr> _NewExpr;
+    return __val_expr< _NewExpr >(_NewExpr(__vs, __expr_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__unary_plus<value_type>, _ValExpr> > operator+() const {
+    typedef _UnaryOp<__unary_plus<value_type>, _ValExpr> _NewExpr;
+    return __val_expr<_NewExpr>(_NewExpr(__unary_plus<value_type>(), __expr_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<negate<value_type>, _ValExpr> > operator-() const {
+    typedef _UnaryOp<negate<value_type>, _ValExpr> _NewExpr;
+    return __val_expr<_NewExpr>(_NewExpr(negate<value_type>(), __expr_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__bit_not<value_type>, _ValExpr> > operator~() const {
+    typedef _UnaryOp<__bit_not<value_type>, _ValExpr> _NewExpr;
+    return __val_expr<_NewExpr>(_NewExpr(__bit_not<value_type>(), __expr_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<logical_not<value_type>, _ValExpr> > operator!() const {
+    typedef _UnaryOp<logical_not<value_type>, _ValExpr> _NewExpr;
+    return __val_expr<_NewExpr>(_NewExpr(logical_not<value_type>(), __expr_));
+  }
+
+  operator valarray<__result_type>() const;
+
+  _LIBCPP_HIDE_FROM_ABI size_t size() const { return __expr_.size(); }
+
+  _LIBCPP_HIDE_FROM_ABI __result_type sum() const {
+    size_t __n        = __expr_.size();
+    __result_type __r = __n ? __expr_[0] : __result_type();
+    for (size_t __i = 1; __i < __n; ++__i)
+      __r += __expr_[__i];
+    return __r;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __result_type min() const {
+    size_t __n        = size();
+    __result_type __r = __n ? (*this)[0] : __result_type();
+    for (size_t __i = 1; __i < __n; ++__i) {
+      __result_type __x = __expr_[__i];
+      if (__x < __r)
+        __r = __x;
+    }
+    return __r;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __result_type max() const {
+    size_t __n        = size();
+    __result_type __r = __n ? (*this)[0] : __result_type();
+    for (size_t __i = 1; __i < __n; ++__i) {
+      __result_type __x = __expr_[__i];
+      if (__r < __x)
+        __r = __x;
+    }
+    return __r;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __val_expr<__shift_expr<_ValExpr> > shift(int __i) const {
+    return __val_expr<__shift_expr<_ValExpr> >(__shift_expr<_ValExpr>(__i, __expr_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __val_expr<__cshift_expr<_ValExpr> > cshift(int __i) const {
+    return __val_expr<__cshift_expr<_ValExpr> >(__cshift_expr<_ValExpr>(__i, __expr_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__apply_expr<value_type, value_type (*)(value_type)>, _ValExpr> >
+  apply(value_type __f(value_type)) const {
+    typedef __apply_expr<value_type, value_type (*)(value_type)> _Op;
+    typedef _UnaryOp<_Op, _ValExpr> _NewExpr;
+    return __val_expr<_NewExpr>(_NewExpr(_Op(__f), __expr_));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__apply_expr<value_type, value_type (*)(const value_type&)>, _ValExpr> >
+  apply(value_type __f(const value_type&)) const {
+    typedef __apply_expr<value_type, value_type (*)(const value_type&)> _Op;
+    typedef _UnaryOp<_Op, _ValExpr> _NewExpr;
+    return __val_expr<_NewExpr>(_NewExpr(_Op(__f), __expr_));
+  }
+};
+
+template <class _ValExpr>
+__val_expr<_ValExpr>::operator valarray<__val_expr::__result_type>() const {
+  valarray<__result_type> __r;
+  size_t __n = __expr_.size();
+  if (__n) {
+    __r.__begin_ = __r.__end_ = allocator<__result_type>().allocate(__n);
+    for (size_t __i = 0; __i != __n; ++__r.__end_, ++__i)
+      ::new ((void*)__r.__end_) __result_type(__expr_[__i]);
+  }
+  return __r;
+}
+
+// valarray
+
+template <class _Tp>
+inline valarray<_Tp>::valarray(size_t __n) : __begin_(nullptr), __end_(nullptr) {
+  if (__n) {
+    __begin_ = __end_ = allocator<value_type>().allocate(__n);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      for (size_t __n_left = __n; __n_left; --__n_left, ++__end_)
+        ::new ((void*)__end_) value_type();
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __clear(__n);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+}
+
+template <class _Tp>
+inline valarray<_Tp>::valarray(const value_type& __x, size_t __n) : __begin_(nullptr), __end_(nullptr) {
+  resize(__n, __x);
+}
+
+template <class _Tp>
+valarray<_Tp>::valarray(const value_type* __p, size_t __n) : __begin_(nullptr), __end_(nullptr) {
+  if (__n) {
+    __begin_ = __end_ = allocator<value_type>().allocate(__n);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      for (size_t __n_left = __n; __n_left; ++__end_, ++__p, --__n_left)
+        ::new ((void*)__end_) value_type(*__p);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __clear(__n);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+}
+
+template <class _Tp>
+valarray<_Tp>::valarray(const valarray& __v) : __begin_(nullptr), __end_(nullptr) {
+  if (__v.size()) {
+    __begin_ = __end_ = allocator<value_type>().allocate(__v.size());
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      for (value_type* __p = __v.__begin_; __p != __v.__end_; ++__end_, ++__p)
+        ::new ((void*)__end_) value_type(*__p);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __clear(__v.size());
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp>
+inline valarray<_Tp>::valarray(valarray&& __v) _NOEXCEPT : __begin_(__v.__begin_), __end_(__v.__end_) {
+  __v.__begin_ = __v.__end_ = nullptr;
+}
+
+template <class _Tp>
+valarray<_Tp>::valarray(initializer_list<value_type> __il) : __begin_(nullptr), __end_(nullptr) {
+  const size_t __n = __il.size();
+  if (__n) {
+    __begin_ = __end_ = allocator<value_type>().allocate(__n);
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      size_t __n_left = __n;
+      for (const value_type* __p = __il.begin(); __n_left; ++__end_, ++__p, --__n_left)
+        ::new ((void*)__end_) value_type(*__p);
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __clear(__n);
+      throw;
+    }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp>
+valarray<_Tp>::valarray(const slice_array<value_type>& __sa) : __begin_(nullptr), __end_(nullptr) {
+  const size_t __n = __sa.__size_;
+  if (__n) {
+    __begin_ = __end_ = allocator<value_type>().allocate(__n);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      size_t __n_left = __n;
+      for (const value_type* __p = __sa.__vp_; __n_left; ++__end_, __p += __sa.__stride_, --__n_left)
+        ::new ((void*)__end_) value_type(*__p);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __clear(__n);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+}
+
+template <class _Tp>
+valarray<_Tp>::valarray(const gslice_array<value_type>& __ga) : __begin_(nullptr), __end_(nullptr) {
+  const size_t __n = __ga.__1d_.size();
+  if (__n) {
+    __begin_ = __end_ = allocator<value_type>().allocate(__n);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      typedef const size_t* _Ip;
+      const value_type* __s = __ga.__vp_;
+      for (_Ip __i = __ga.__1d_.__begin_, __e = __ga.__1d_.__end_; __i != __e; ++__i, ++__end_)
+        ::new ((void*)__end_) value_type(__s[*__i]);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __clear(__n);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+}
+
+template <class _Tp>
+valarray<_Tp>::valarray(const mask_array<value_type>& __ma) : __begin_(nullptr), __end_(nullptr) {
+  const size_t __n = __ma.__1d_.size();
+  if (__n) {
+    __begin_ = __end_ = allocator<value_type>().allocate(__n);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      typedef const size_t* _Ip;
+      const value_type* __s = __ma.__vp_;
+      for (_Ip __i = __ma.__1d_.__begin_, __e = __ma.__1d_.__end_; __i != __e; ++__i, ++__end_)
+        ::new ((void*)__end_) value_type(__s[*__i]);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __clear(__n);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+}
+
+template <class _Tp>
+valarray<_Tp>::valarray(const indirect_array<value_type>& __ia) : __begin_(nullptr), __end_(nullptr) {
+  const size_t __n = __ia.__1d_.size();
+  if (__n) {
+    __begin_ = __end_ = allocator<value_type>().allocate(__n);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      typedef const size_t* _Ip;
+      const value_type* __s = __ia.__vp_;
+      for (_Ip __i = __ia.__1d_.__begin_, __e = __ia.__1d_.__end_; __i != __e; ++__i, ++__end_)
+        ::new ((void*)__end_) value_type(__s[*__i]);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __clear(__n);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+}
+
+template <class _Tp>
+inline valarray<_Tp>::~valarray() {
+  __clear(size());
+}
+
+template <class _Tp>
+valarray<_Tp>& valarray<_Tp>::__assign_range(const value_type* __f, const value_type* __l) {
+  size_t __n = __l - __f;
+  if (size() != __n) {
+    __clear(size());
+    __begin_ = allocator<value_type>().allocate(__n);
+    __end_   = __begin_ + __n;
+    std::uninitialized_copy(__f, __l, __begin_);
+  } else {
+    std::copy(__f, __l, __begin_);
+  }
+  return *this;
+}
+
+template <class _Tp>
+valarray<_Tp>& valarray<_Tp>::operator=(const valarray& __v) {
+  if (this != std::addressof(__v))
+    return __assign_range(__v.__begin_, __v.__end_);
+  return *this;
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator=(valarray&& __v) _NOEXCEPT {
+  __clear(size());
+  __begin_     = __v.__begin_;
+  __end_       = __v.__end_;
+  __v.__begin_ = nullptr;
+  __v.__end_   = nullptr;
+  return *this;
+}
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator=(initializer_list<value_type> __il) {
+  return __assign_range(__il.begin(), __il.end());
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator=(const value_type& __x) {
+  std::fill(__begin_, __end_, __x);
+  return *this;
+}
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator=(const slice_array<value_type>& __sa) {
+  value_type* __t       = __begin_;
+  const value_type* __s = __sa.__vp_;
+  for (size_t __n = __sa.__size_; __n; --__n, __s += __sa.__stride_, ++__t)
+    *__t = *__s;
+  return *this;
+}
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator=(const gslice_array<value_type>& __ga) {
+  typedef const size_t* _Ip;
+  value_type* __t       = __begin_;
+  const value_type* __s = __ga.__vp_;
+  for (_Ip __i = __ga.__1d_.__begin_, __e = __ga.__1d_.__end_; __i != __e; ++__i, ++__t)
+    *__t = __s[*__i];
+  return *this;
+}
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator=(const mask_array<value_type>& __ma) {
+  typedef const size_t* _Ip;
+  value_type* __t       = __begin_;
+  const value_type* __s = __ma.__vp_;
+  for (_Ip __i = __ma.__1d_.__begin_, __e = __ma.__1d_.__end_; __i != __e; ++__i, ++__t)
+    *__t = __s[*__i];
+  return *this;
+}
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator=(const indirect_array<value_type>& __ia) {
+  typedef const size_t* _Ip;
+  value_type* __t       = __begin_;
+  const value_type* __s = __ia.__vp_;
+  for (_Ip __i = __ia.__1d_.__begin_, __e = __ia.__1d_.__end_; __i != __e; ++__i, ++__t)
+    *__t = __s[*__i];
+  return *this;
+}
+
+template <class _Tp>
+template <class _ValExpr>
+inline valarray<_Tp>& valarray<_Tp>::operator=(const __val_expr<_ValExpr>& __v) {
+  size_t __n = __v.size();
+  if (size() != __n)
+    resize(__n);
+  value_type* __t = __begin_;
+  for (size_t __i = 0; __i != __n; ++__t, ++__i)
+    *__t = __result_type(__v[__i]);
+  return *this;
+}
+
+template <class _Tp>
+inline __val_expr<__slice_expr<const valarray<_Tp>&> > valarray<_Tp>::operator[](slice __s) const {
+  return __val_expr<__slice_expr<const valarray&> >(__slice_expr<const valarray&>(__s, *this));
+}
+
+template <class _Tp>
+inline slice_array<_Tp> valarray<_Tp>::operator[](slice __s) {
+  return slice_array<value_type>(__s, *this);
+}
+
+template <class _Tp>
+inline __val_expr<__indirect_expr<const valarray<_Tp>&> > valarray<_Tp>::operator[](const gslice& __gs) const {
+  return __val_expr<__indirect_expr<const valarray&> >(__indirect_expr<const valarray&>(__gs.__1d_, *this));
+}
+
+template <class _Tp>
+inline gslice_array<_Tp> valarray<_Tp>::operator[](const gslice& __gs) {
+  return gslice_array<value_type>(__gs, *this);
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp>
+inline __val_expr<__indirect_expr<const valarray<_Tp>&> > valarray<_Tp>::operator[](gslice&& __gs) const {
+  return __val_expr<__indirect_expr<const valarray&> >(__indirect_expr<const valarray&>(std::move(__gs.__1d_), *this));
+}
+
+template <class _Tp>
+inline gslice_array<_Tp> valarray<_Tp>::operator[](gslice&& __gs) {
+  return gslice_array<value_type>(std::move(__gs), *this);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp>
+inline __val_expr<__mask_expr<const valarray<_Tp>&> > valarray<_Tp>::operator[](const valarray<bool>& __vb) const {
+  return __val_expr<__mask_expr<const valarray&> >(__mask_expr<const valarray&>(__vb, *this));
+}
+
+template <class _Tp>
+inline mask_array<_Tp> valarray<_Tp>::operator[](const valarray<bool>& __vb) {
+  return mask_array<value_type>(__vb, *this);
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp>
+inline __val_expr<__mask_expr<const valarray<_Tp>&> > valarray<_Tp>::operator[](valarray<bool>&& __vb) const {
+  return __val_expr<__mask_expr<const valarray&> >(__mask_expr<const valarray&>(std::move(__vb), *this));
+}
+
+template <class _Tp>
+inline mask_array<_Tp> valarray<_Tp>::operator[](valarray<bool>&& __vb) {
+  return mask_array<value_type>(std::move(__vb), *this);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp>
+inline __val_expr<__indirect_expr<const valarray<_Tp>&> >
+valarray<_Tp>::operator[](const valarray<size_t>& __vs) const {
+  return __val_expr<__indirect_expr<const valarray&> >(__indirect_expr<const valarray&>(__vs, *this));
+}
+
+template <class _Tp>
+inline indirect_array<_Tp> valarray<_Tp>::operator[](const valarray<size_t>& __vs) {
+  return indirect_array<value_type>(__vs, *this);
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp>
+inline __val_expr<__indirect_expr<const valarray<_Tp>&> > valarray<_Tp>::operator[](valarray<size_t>&& __vs) const {
+  return __val_expr<__indirect_expr<const valarray&> >(__indirect_expr<const valarray&>(std::move(__vs), *this));
+}
+
+template <class _Tp>
+inline indirect_array<_Tp> valarray<_Tp>::operator[](valarray<size_t>&& __vs) {
+  return indirect_array<value_type>(std::move(__vs), *this);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp>
+inline __val_expr<_UnaryOp<__unary_plus<_Tp>, const valarray<_Tp>&> > valarray<_Tp>::operator+() const {
+  using _Op = _UnaryOp<__unary_plus<_Tp>, const valarray<_Tp>&>;
+  return __val_expr<_Op>(_Op(__unary_plus<_Tp>(), *this));
+}
+
+template <class _Tp>
+inline __val_expr<_UnaryOp<negate<_Tp>, const valarray<_Tp>&> > valarray<_Tp>::operator-() const {
+  using _Op = _UnaryOp<negate<_Tp>, const valarray<_Tp>&>;
+  return __val_expr<_Op>(_Op(negate<_Tp>(), *this));
+}
+
+template <class _Tp>
+inline __val_expr<_UnaryOp<__bit_not<_Tp>, const valarray<_Tp>&> > valarray<_Tp>::operator~() const {
+  using _Op = _UnaryOp<__bit_not<_Tp>, const valarray<_Tp>&>;
+  return __val_expr<_Op>(_Op(__bit_not<_Tp>(), *this));
+}
+
+template <class _Tp>
+inline __val_expr<_UnaryOp<logical_not<_Tp>, const valarray<_Tp>&> > valarray<_Tp>::operator!() const {
+  using _Op = _UnaryOp<logical_not<_Tp>, const valarray<_Tp>&>;
+  return __val_expr<_Op>(_Op(logical_not<_Tp>(), *this));
+}
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator*=(const value_type& __x) {
+  for (value_type* __p = __begin_; __p != __end_; ++__p)
+    *__p *= __x;
+  return *this;
+}
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator/=(const value_type& __x) {
+  for (value_type* __p = __begin_; __p != __end_; ++__p)
+    *__p /= __x;
+  return *this;
+}
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator%=(const value_type& __x) {
+  for (value_type* __p = __begin_; __p != __end_; ++__p)
+    *__p %= __x;
+  return *this;
+}
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator+=(const value_type& __x) {
+  for (value_type* __p = __begin_; __p != __end_; ++__p)
+    *__p += __x;
+  return *this;
+}
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator-=(const value_type& __x) {
+  for (value_type* __p = __begin_; __p != __end_; ++__p)
+    *__p -= __x;
+  return *this;
+}
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator^=(const value_type& __x) {
+  for (value_type* __p = __begin_; __p != __end_; ++__p)
+    *__p ^= __x;
+  return *this;
+}
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator&=(const value_type& __x) {
+  for (value_type* __p = __begin_; __p != __end_; ++__p)
+    *__p &= __x;
+  return *this;
+}
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator|=(const value_type& __x) {
+  for (value_type* __p = __begin_; __p != __end_; ++__p)
+    *__p |= __x;
+  return *this;
+}
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator<<=(const value_type& __x) {
+  for (value_type* __p = __begin_; __p != __end_; ++__p)
+    *__p <<= __x;
+  return *this;
+}
+
+template <class _Tp>
+inline valarray<_Tp>& valarray<_Tp>::operator>>=(const value_type& __x) {
+  for (value_type* __p = __begin_; __p != __end_; ++__p)
+    *__p >>= __x;
+  return *this;
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline valarray<_Tp>& valarray<_Tp>::operator*=(const _Expr& __v) {
+  size_t __i = 0;
+  for (value_type* __t = __begin_; __t != __end_; ++__t, ++__i)
+    *__t *= std::__get(__v, __i);
+  return *this;
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline valarray<_Tp>& valarray<_Tp>::operator/=(const _Expr& __v) {
+  size_t __i = 0;
+  for (value_type* __t = __begin_; __t != __end_; ++__t, ++__i)
+    *__t /= std::__get(__v, __i);
+  return *this;
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline valarray<_Tp>& valarray<_Tp>::operator%=(const _Expr& __v) {
+  size_t __i = 0;
+  for (value_type* __t = __begin_; __t != __end_; ++__t, ++__i)
+    *__t %= std::__get(__v, __i);
+  return *this;
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline valarray<_Tp>& valarray<_Tp>::operator+=(const _Expr& __v) {
+  size_t __i = 0;
+  for (value_type* __t = __begin_; __t != __end_; ++__t, ++__i)
+    *__t += std::__get(__v, __i);
+  return *this;
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline valarray<_Tp>& valarray<_Tp>::operator-=(const _Expr& __v) {
+  size_t __i = 0;
+  for (value_type* __t = __begin_; __t != __end_; ++__t, ++__i)
+    *__t -= std::__get(__v, __i);
+  return *this;
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline valarray<_Tp>& valarray<_Tp>::operator^=(const _Expr& __v) {
+  size_t __i = 0;
+  for (value_type* __t = __begin_; __t != __end_; ++__t, ++__i)
+    *__t ^= std::__get(__v, __i);
+  return *this;
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline valarray<_Tp>& valarray<_Tp>::operator|=(const _Expr& __v) {
+  size_t __i = 0;
+  for (value_type* __t = __begin_; __t != __end_; ++__t, ++__i)
+    *__t |= std::__get(__v, __i);
+  return *this;
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline valarray<_Tp>& valarray<_Tp>::operator&=(const _Expr& __v) {
+  size_t __i = 0;
+  for (value_type* __t = __begin_; __t != __end_; ++__t, ++__i)
+    *__t &= std::__get(__v, __i);
+  return *this;
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline valarray<_Tp>& valarray<_Tp>::operator<<=(const _Expr& __v) {
+  size_t __i = 0;
+  for (value_type* __t = __begin_; __t != __end_; ++__t, ++__i)
+    *__t <<= std::__get(__v, __i);
+  return *this;
+}
+
+template <class _Tp>
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> >
+inline valarray<_Tp>& valarray<_Tp>::operator>>=(const _Expr& __v) {
+  size_t __i = 0;
+  for (value_type* __t = __begin_; __t != __end_; ++__t, ++__i)
+    *__t >>= std::__get(__v, __i);
+  return *this;
+}
+
+template <class _Tp>
+inline void valarray<_Tp>::swap(valarray& __v) _NOEXCEPT {
+  std::swap(__begin_, __v.__begin_);
+  std::swap(__end_, __v.__end_);
+}
+
+template <class _Tp>
+inline _Tp valarray<_Tp>::sum() const {
+  if (__begin_ == __end_)
+    return value_type();
+  const value_type* __p = __begin_;
+  _Tp __r               = *__p;
+  for (++__p; __p != __end_; ++__p)
+    __r += *__p;
+  return __r;
+}
+
+template <class _Tp>
+inline _Tp valarray<_Tp>::min() const {
+  if (__begin_ == __end_)
+    return value_type();
+  return *std::min_element(__begin_, __end_);
+}
+
+template <class _Tp>
+inline _Tp valarray<_Tp>::max() const {
+  if (__begin_ == __end_)
+    return value_type();
+  return *std::max_element(__begin_, __end_);
+}
+
+template <class _Tp>
+valarray<_Tp> valarray<_Tp>::shift(int __i) const {
+  valarray<value_type> __r;
+  size_t __n = size();
+  if (__n) {
+    __r.__begin_ = __r.__end_ = allocator<value_type>().allocate(__n);
+    const value_type* __sb;
+    value_type* __tb;
+    value_type* __te;
+    if (__i >= 0) {
+      __i  = std::min(__i, static_cast<int>(__n));
+      __sb = __begin_ + __i;
+      __tb = __r.__begin_;
+      __te = __r.__begin_ + (__n - __i);
+    } else {
+      __i  = std::min(-__i, static_cast<int>(__n));
+      __sb = __begin_;
+      __tb = __r.__begin_ + __i;
+      __te = __r.__begin_ + __n;
+    }
+    for (; __r.__end_ != __tb; ++__r.__end_)
+      ::new ((void*)__r.__end_) value_type();
+    for (; __r.__end_ != __te; ++__r.__end_, ++__sb)
+      ::new ((void*)__r.__end_) value_type(*__sb);
+    for (__te = __r.__begin_ + __n; __r.__end_ != __te; ++__r.__end_)
+      ::new ((void*)__r.__end_) value_type();
+  }
+  return __r;
+}
+
+template <class _Tp>
+valarray<_Tp> valarray<_Tp>::cshift(int __i) const {
+  valarray<value_type> __r;
+  size_t __n = size();
+  if (__n) {
+    __r.__begin_ = __r.__end_ = allocator<value_type>().allocate(__n);
+    __i %= static_cast<int>(__n);
+    const value_type* __m = __i >= 0 ? __begin_ + __i : __end_ + __i;
+    for (const value_type* __s = __m; __s != __end_; ++__r.__end_, ++__s)
+      ::new ((void*)__r.__end_) value_type(*__s);
+    for (const value_type* __s = __begin_; __s != __m; ++__r.__end_, ++__s)
+      ::new ((void*)__r.__end_) value_type(*__s);
+  }
+  return __r;
+}
+
+template <class _Tp>
+valarray<_Tp> valarray<_Tp>::apply(value_type __f(value_type)) const {
+  valarray<value_type> __r;
+  size_t __n = size();
+  if (__n) {
+    __r.__begin_ = __r.__end_ = allocator<value_type>().allocate(__n);
+    for (const value_type* __p = __begin_; __n; ++__r.__end_, ++__p, --__n)
+      ::new ((void*)__r.__end_) value_type(__f(*__p));
+  }
+  return __r;
+}
+
+template <class _Tp>
+valarray<_Tp> valarray<_Tp>::apply(value_type __f(const value_type&)) const {
+  valarray<value_type> __r;
+  size_t __n = size();
+  if (__n) {
+    __r.__begin_ = __r.__end_ = allocator<value_type>().allocate(__n);
+    for (const value_type* __p = __begin_; __n; ++__r.__end_, ++__p, --__n)
+      ::new ((void*)__r.__end_) value_type(__f(*__p));
+  }
+  return __r;
+}
+
+template <class _Tp>
+inline void valarray<_Tp>::__clear(size_t __capacity) {
+  if (__begin_ != nullptr) {
+    while (__end_ != __begin_)
+      (--__end_)->~value_type();
+    allocator<value_type>().deallocate(__begin_, __capacity);
+    __begin_ = __end_ = nullptr;
+  }
+}
+
+template <class _Tp>
+void valarray<_Tp>::resize(size_t __n, value_type __x) {
+  __clear(size());
+  if (__n) {
+    __begin_ = __end_ = allocator<value_type>().allocate(__n);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      for (size_t __n_left = __n; __n_left; --__n_left, ++__end_)
+        ::new ((void*)__end_) value_type(__x);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      __clear(__n);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI void swap(valarray<_Tp>& __x, valarray<_Tp>& __y) _NOEXCEPT {
+  __x.swap(__y);
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<multiplies<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator*(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<multiplies<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(multiplies<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<multiplies<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator*(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<multiplies<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(multiplies<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<multiplies<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator*(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<multiplies<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(multiplies<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<divides<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator/(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<divides<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(divides<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<divides<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator/(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<divides<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(divides<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<divides<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator/(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<divides<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(divides<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<modulus<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator%(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<modulus<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(modulus<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<modulus<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator%(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<modulus<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(modulus<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<modulus<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator%(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<modulus<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(modulus<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<plus<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator+(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<plus<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(plus<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<plus<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator+(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<plus<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(plus<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<plus<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator+(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<plus<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(plus<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<minus<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator-(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<minus<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(minus<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<minus<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator-(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<minus<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(minus<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<minus<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator-(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<minus<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(minus<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<bit_xor<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator^(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<bit_xor<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(bit_xor<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<bit_xor<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator^(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<bit_xor<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(bit_xor<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<bit_xor<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator^(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<bit_xor<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(bit_xor<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<bit_and<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator&(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<bit_and<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(bit_and<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<bit_and<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator&(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<bit_and<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(bit_and<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<bit_and<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator&(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<bit_and<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(bit_and<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<bit_or<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator|(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<bit_or<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(bit_or<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<bit_or<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator|(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<bit_or<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(bit_or<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<bit_or<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator|(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<bit_or<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(bit_or<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<__bit_shift_left<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator<<(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<__bit_shift_left<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(__bit_shift_left<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr< _BinaryOp<__bit_shift_left<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator<<(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<__bit_shift_left<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(__bit_shift_left<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr< _BinaryOp<__bit_shift_left<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator<<(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<__bit_shift_left<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__bit_shift_left<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<__bit_shift_right<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator>>(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<__bit_shift_right<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(__bit_shift_right<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<
+    _BinaryOp<__bit_shift_right<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator>>(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<__bit_shift_right<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(__bit_shift_right<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr< _BinaryOp<__bit_shift_right<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator>>(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<__bit_shift_right<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__bit_shift_right<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<logical_and<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator&&(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<logical_and<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(logical_and<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<logical_and<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator&&(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<logical_and<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(logical_and<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<logical_and<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator&&(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<logical_and<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(logical_and<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<logical_or<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator||(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<logical_or<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(logical_or<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<logical_or<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator||(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<logical_or<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(logical_or<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<logical_or<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator||(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<logical_or<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(logical_or<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<equal_to<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator==(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<equal_to<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(equal_to<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<equal_to<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator==(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<equal_to<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(equal_to<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<equal_to<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator==(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<equal_to<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(equal_to<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<not_equal_to<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator!=(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<not_equal_to<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(not_equal_to<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<not_equal_to<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator!=(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<not_equal_to<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(not_equal_to<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<not_equal_to<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator!=(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<not_equal_to<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(not_equal_to<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<less<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator<(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<less<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(less<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<less<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator<(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<less<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(less<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<less<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator<(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<less<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(less<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<greater<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator>(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<greater<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(greater<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<greater<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator>(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<greater<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(greater<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<greater<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator>(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<greater<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(greater<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<less_equal<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator<=(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<less_equal<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(less_equal<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<less_equal<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator<=(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<less_equal<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(less_equal<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<less_equal<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator<=(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<less_equal<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(less_equal<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<greater_equal<typename _Expr1::value_type>, _Expr1, _Expr2> >
+operator>=(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<greater_equal<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(greater_equal<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<greater_equal<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+operator>=(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<greater_equal<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(greater_equal<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<greater_equal<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+operator>=(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<greater_equal<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(greater_equal<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__abs_expr<typename _Expr::value_type>, _Expr> >
+abs(const _Expr& __x) {
+  typedef typename _Expr::value_type value_type;
+  typedef _UnaryOp<__abs_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__abs_expr<value_type>(), __x));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__acos_expr<typename _Expr::value_type>, _Expr> >
+acos(const _Expr& __x) {
+  typedef typename _Expr::value_type value_type;
+  typedef _UnaryOp<__acos_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__acos_expr<value_type>(), __x));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__asin_expr<typename _Expr::value_type>, _Expr> >
+asin(const _Expr& __x) {
+  typedef typename _Expr::value_type value_type;
+  typedef _UnaryOp<__asin_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__asin_expr<value_type>(), __x));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__atan_expr<typename _Expr::value_type>, _Expr> >
+atan(const _Expr& __x) {
+  typedef typename _Expr::value_type value_type;
+  typedef _UnaryOp<__atan_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__atan_expr<value_type>(), __x));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<__atan2_expr<typename _Expr1::value_type>, _Expr1, _Expr2> >
+atan2(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<__atan2_expr<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(__atan2_expr<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<__atan2_expr<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+atan2(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<__atan2_expr<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(__atan2_expr<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<__atan2_expr<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+atan2(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<__atan2_expr<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__atan2_expr<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__cos_expr<typename _Expr::value_type>, _Expr> >
+cos(const _Expr& __x) {
+  typedef typename _Expr::value_type value_type;
+  typedef _UnaryOp<__cos_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__cos_expr<value_type>(), __x));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__cosh_expr<typename _Expr::value_type>, _Expr> >
+cosh(const _Expr& __x) {
+  typedef typename _Expr::value_type value_type;
+  typedef _UnaryOp<__cosh_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__cosh_expr<value_type>(), __x));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__exp_expr<typename _Expr::value_type>, _Expr> >
+exp(const _Expr& __x) {
+  typedef typename _Expr::value_type value_type;
+  typedef _UnaryOp<__exp_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__exp_expr<value_type>(), __x));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__log_expr<typename _Expr::value_type>, _Expr> >
+log(const _Expr& __x) {
+  typedef typename _Expr::value_type value_type;
+  typedef _UnaryOp<__log_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__log_expr<value_type>(), __x));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__log10_expr<typename _Expr::value_type>, _Expr> >
+log10(const _Expr& __x) {
+  typedef typename _Expr::value_type value_type;
+  typedef _UnaryOp<__log10_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__log10_expr<value_type>(), __x));
+}
+
+template <class _Expr1,
+          class _Expr2,
+          __enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<__pow_expr<typename _Expr1::value_type>, _Expr1, _Expr2> >
+pow(const _Expr1& __x, const _Expr2& __y) {
+  typedef typename _Expr1::value_type value_type;
+  typedef _BinaryOp<__pow_expr<value_type>, _Expr1, _Expr2> _Op;
+  return __val_expr<_Op>(_Op(__pow_expr<value_type>(), __x, __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<__pow_expr<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
+pow(const _Expr& __x, const typename _Expr::value_type& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<__pow_expr<value_type>, _Expr, __scalar_expr<value_type> > _Op;
+  return __val_expr<_Op>(_Op(__pow_expr<value_type>(), __x, __scalar_expr<value_type>(__y, __x.size())));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI
+__val_expr<_BinaryOp<__pow_expr<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
+pow(const typename _Expr::value_type& __x, const _Expr& __y) {
+  typedef typename _Expr::value_type value_type;
+  typedef _BinaryOp<__pow_expr<value_type>, __scalar_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__pow_expr<value_type>(), __scalar_expr<value_type>(__x, __y.size()), __y));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__sin_expr<typename _Expr::value_type>, _Expr> >
+sin(const _Expr& __x) {
+  typedef typename _Expr::value_type value_type;
+  typedef _UnaryOp<__sin_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__sin_expr<value_type>(), __x));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__sinh_expr<typename _Expr::value_type>, _Expr> >
+sinh(const _Expr& __x) {
+  typedef typename _Expr::value_type value_type;
+  typedef _UnaryOp<__sinh_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__sinh_expr<value_type>(), __x));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__sqrt_expr<typename _Expr::value_type>, _Expr> >
+sqrt(const _Expr& __x) {
+  typedef typename _Expr::value_type value_type;
+  typedef _UnaryOp<__sqrt_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__sqrt_expr<value_type>(), __x));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__tan_expr<typename _Expr::value_type>, _Expr> >
+tan(const _Expr& __x) {
+  typedef typename _Expr::value_type value_type;
+  typedef _UnaryOp<__tan_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__tan_expr<value_type>(), __x));
+}
+
+template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__tanh_expr<typename _Expr::value_type>, _Expr> >
+tanh(const _Expr& __x) {
+  typedef typename _Expr::value_type value_type;
+  typedef _UnaryOp<__tanh_expr<value_type>, _Expr> _Op;
+  return __val_expr<_Op>(_Op(__tanh_expr<value_type>(), __x));
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _Tp* begin(valarray<_Tp>& __v) {
+  return __v.__begin_;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI const _Tp* begin(const valarray<_Tp>& __v) {
+  return __v.__begin_;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI _Tp* end(valarray<_Tp>& __v) {
+  return __v.__end_;
+}
+
+template <class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI const _Tp* end(const valarray<_Tp>& __v) {
+  return __v.__end_;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <algorithm>
+#  include <concepts>
+#  include <cstdlib>
+#  include <cstring>
+#  include <functional>
+#  include <stdexcept>
+#  include <type_traits>
+#endif
+
+#endif // _LIBCPP_VALARRAY
diff --git a/libcxx/include/__cxx03/variant b/libcxx/include/__cxx03/variant
new file mode 100644
index 000000000000..5f2d03b7227b
--- /dev/null
+++ b/libcxx/include/__cxx03/variant
@@ -0,0 +1,1650 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_VARIANT
+#define _LIBCPP_VARIANT
+
+/*
+   variant synopsis
+
+namespace std {
+
+  // 20.7.2, class template variant
+  template <class... Types>
+  class variant {
+  public:
+
+    // 20.7.2.1, constructors
+    constexpr variant() noexcept(see below);
+    constexpr variant(const variant&);
+    constexpr variant(variant&&) noexcept(see below);
+
+    template <class T> constexpr variant(T&&) noexcept(see below);
+
+    template <class T, class... Args>
+    constexpr explicit variant(in_place_type_t<T>, Args&&...);
+
+    template <class T, class U, class... Args>
+    constexpr explicit variant(
+        in_place_type_t<T>, initializer_list<U>, Args&&...);
+
+    template <size_t I, class... Args>
+    constexpr explicit variant(in_place_index_t<I>, Args&&...);
+
+    template <size_t I, class U, class... Args>
+    constexpr explicit variant(
+        in_place_index_t<I>, initializer_list<U>, Args&&...);
+
+    // 20.7.2.2, destructor
+    constexpr ~variant();                                             // constexpr since c++20
+
+    // 20.7.2.3, assignment
+    constexpr variant& operator=(const variant&);
+    constexpr variant& operator=(variant&&) noexcept(see below);
+
+    template <class T>
+    constexpr variant& operator=(T&&) noexcept(see below);            // constexpr since c++20
+
+    // 20.7.2.4, modifiers
+    template <class T, class... Args>
+    constexpr T& emplace(Args&&...);                                  // constexpr since c++20
+
+    template <class T, class U, class... Args>
+    constexpr T& emplace(initializer_list<U>, Args&&...);             // constexpr since c++20
+
+    template <size_t I, class... Args>
+    constexpr variant_alternative_t<I, variant>& emplace(Args&&...);  // constexpr since c++20
+
+    template <size_t I, class U, class...  Args>
+    constexpr variant_alternative_t<I, variant>&
+        emplace(initializer_list<U>, Args&&...);                      // constexpr since c++20
+
+    // 20.7.2.5, value status
+    constexpr bool valueless_by_exception() const noexcept;
+    constexpr size_t index() const noexcept;
+
+    // 20.7.2.6, swap
+    void swap(variant&) noexcept(see below);
+
+    // [variant.visit], visitation
+    template<class Self, class Visitor>
+      constexpr decltype(auto) visit(this Self&&, Visitor&&); // Since C++26
+    template<class R, class Self, class Visitor>
+      constexpr R visit(this Self&&, Visitor&&);              // Since C++26
+  };
+
+  // 20.7.3, variant helper classes
+  template <class T> struct variant_size; // undefined
+
+  template <class T>
+  inline constexpr size_t variant_size_v = variant_size<T>::value;
+
+  template <class T> struct variant_size<const T>;
+  template <class T> struct variant_size<volatile T>;
+  template <class T> struct variant_size<const volatile T>;
+
+  template <class... Types>
+  struct variant_size<variant<Types...>>;
+
+  template <size_t I, class T> struct variant_alternative; // undefined
+
+  template <size_t I, class T>
+  using variant_alternative_t = typename variant_alternative<I, T>::type;
+
+  template <size_t I, class T> struct variant_alternative<I, const T>;
+  template <size_t I, class T> struct variant_alternative<I, volatile T>;
+  template <size_t I, class T> struct variant_alternative<I, const volatile T>;
+
+  template <size_t I, class... Types>
+  struct variant_alternative<I, variant<Types...>>;
+
+  inline constexpr size_t variant_npos = -1;
+
+  // 20.7.4, value access
+  template <class T, class... Types>
+  constexpr bool holds_alternative(const variant<Types...>&) noexcept;
+
+  template <size_t I, class... Types>
+  constexpr variant_alternative_t<I, variant<Types...>>&
+  get(variant<Types...>&);
+
+  template <size_t I, class... Types>
+  constexpr variant_alternative_t<I, variant<Types...>>&&
+  get(variant<Types...>&&);
+
+  template <size_t I, class... Types>
+  constexpr variant_alternative_t<I, variant<Types...>> const&
+  get(const variant<Types...>&);
+
+  template <size_t I, class... Types>
+  constexpr variant_alternative_t<I, variant<Types...>> const&&
+  get(const variant<Types...>&&);
+
+  template <class T, class...  Types>
+  constexpr T& get(variant<Types...>&);
+
+  template <class T, class... Types>
+  constexpr T&& get(variant<Types...>&&);
+
+  template <class T, class... Types>
+  constexpr const T& get(const variant<Types...>&);
+
+  template <class T, class... Types>
+  constexpr const T&& get(const variant<Types...>&&);
+
+  template <size_t I, class... Types>
+  constexpr add_pointer_t<variant_alternative_t<I, variant<Types...>>>
+  get_if(variant<Types...>*) noexcept;
+
+  template <size_t I, class... Types>
+  constexpr add_pointer_t<const variant_alternative_t<I, variant<Types...>>>
+  get_if(const variant<Types...>*) noexcept;
+
+  template <class T, class... Types>
+  constexpr add_pointer_t<T>
+  get_if(variant<Types...>*) noexcept;
+
+  template <class T, class... Types>
+  constexpr add_pointer_t<const T>
+  get_if(const variant<Types...>*) noexcept;
+
+  // 20.7.5, relational operators
+  template <class... Types>
+  constexpr bool operator==(const variant<Types...>&, const variant<Types...>&);
+
+  template <class... Types>
+  constexpr bool operator!=(const variant<Types...>&, const variant<Types...>&);
+
+  template <class... Types>
+  constexpr bool operator<(const variant<Types...>&, const variant<Types...>&);
+
+  template <class... Types>
+  constexpr bool operator>(const variant<Types...>&, const variant<Types...>&);
+
+  template <class... Types>
+  constexpr bool operator<=(const variant<Types...>&, const variant<Types...>&);
+
+  template <class... Types>
+  constexpr bool operator>=(const variant<Types...>&, const variant<Types...>&);
+
+  template <class... Types> requires (three_way_comparable<Types> && ...)
+  constexpr common_comparison_category_t<compare_three_way_result_t<Types>...>
+    operator<=>(const variant<Types...>&, const variant<Types...>&);           // since C++20
+
+  // 20.7.6, visitation
+  template <class Visitor, class... Variants>
+  constexpr see below visit(Visitor&&, Variants&&...);
+
+  template <class R, class Visitor, class... Variants>
+  constexpr R visit(Visitor&&, Variants&&...); // since C++20
+
+  // 20.7.7, class monostate
+  struct monostate;
+
+  // 20.7.8, monostate relational operators
+  constexpr bool operator==(monostate, monostate) noexcept;
+  constexpr bool operator!=(monostate, monostate) noexcept;             // until C++20
+  constexpr bool operator<(monostate, monostate) noexcept;              // until C++20
+  constexpr bool operator>(monostate, monostate) noexcept;              // until C++20
+  constexpr bool operator<=(monostate, monostate) noexcept;             // until C++20
+  constexpr bool operator>=(monostate, monostate) noexcept;             // until C++20
+  constexpr strong_ordering operator<=>(monostate, monostate) noexcept; // since C++20
+
+  // 20.7.9, specialized algorithms
+  template <class... Types>
+  void swap(variant<Types...>&, variant<Types...>&) noexcept(see below);
+
+  // 20.7.10, class bad_variant_access
+  class bad_variant_access;
+
+  // 20.7.11, hash support
+  template <class T> struct hash;
+  template <class... Types> struct hash<variant<Types...>>;
+  template <> struct hash<monostate>;
+
+} // namespace std
+
+*/
+
+#include <__compare/common_comparison_category.h>
+#include <__compare/compare_three_way_result.h>
+#include <__compare/three_way_comparable.h>
+#include <__config>
+#include <__exception/exception.h>
+#include <__functional/hash.h>
+#include <__functional/invoke.h>
+#include <__functional/operations.h>
+#include <__functional/unary_function.h>
+#include <__memory/addressof.h>
+#include <__memory/construct_at.h>
+#include <__tuple/find_index.h>
+#include <__tuple/sfinae_helpers.h>
+#include <__type_traits/add_const.h>
+#include <__type_traits/add_cv.h>
+#include <__type_traits/add_pointer.h>
+#include <__type_traits/add_volatile.h>
+#include <__type_traits/common_type.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/dependent_type.h>
+#include <__type_traits/is_array.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_destructible.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_reference.h>
+#include <__type_traits/is_trivially_assignable.h>
+#include <__type_traits/is_trivially_constructible.h>
+#include <__type_traits/is_trivially_destructible.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__type_traits/is_void.h>
+#include <__type_traits/remove_const.h>
+#include <__type_traits/remove_cvref.h>
+#include <__type_traits/type_identity.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+#include <__utility/forward.h>
+#include <__utility/forward_like.h>
+#include <__utility/in_place.h>
+#include <__utility/integer_sequence.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+#include <__variant/monostate.h>
+#include <__verbose_abort>
+#include <initializer_list>
+#include <limits>
+#include <new>
+#include <version>
+
+// standard-mandated includes
+
+// [variant.syn]
+#include <compare>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+namespace std { // explicitly not using versioning namespace
+
+class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_AVAILABILITY_BAD_VARIANT_ACCESS bad_variant_access : public exception {
+public:
+  const char* what() const _NOEXCEPT override;
+};
+
+} // namespace std
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+
+// Light N-dimensional array of function pointers. Used in place of std::array to avoid
+// adding a dependency.
+template <class _Tp, size_t _Size>
+struct __farray {
+  static_assert(_Size > 0, "N-dimensional array should never be empty in std::visit");
+  _Tp __buf_[_Size] = {};
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& operator[](size_t __n) const noexcept { return __buf_[__n]; }
+};
+
+_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS void
+__throw_bad_variant_access() {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw bad_variant_access();
+#  else
+  _LIBCPP_VERBOSE_ABORT("bad_variant_access was thrown in -fno-exceptions mode");
+#  endif
+}
+
+template <class... _Types>
+class _LIBCPP_TEMPLATE_VIS variant;
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS variant_size;
+
+template <class _Tp>
+inline constexpr size_t variant_size_v = variant_size<_Tp>::value;
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS variant_size<const _Tp> : variant_size<_Tp> {};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS variant_size<volatile _Tp> : variant_size<_Tp> {};
+
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS variant_size<const volatile _Tp> : variant_size<_Tp> {};
+
+template <class... _Types>
+struct _LIBCPP_TEMPLATE_VIS variant_size<variant<_Types...>> : integral_constant<size_t, sizeof...(_Types)> {};
+
+template <size_t _Ip, class _Tp>
+struct _LIBCPP_TEMPLATE_VIS variant_alternative;
+
+template <size_t _Ip, class _Tp>
+using variant_alternative_t = typename variant_alternative<_Ip, _Tp>::type;
+
+template <size_t _Ip, class _Tp>
+struct _LIBCPP_TEMPLATE_VIS variant_alternative<_Ip, const _Tp> : add_const<variant_alternative_t<_Ip, _Tp>> {};
+
+template <size_t _Ip, class _Tp>
+struct _LIBCPP_TEMPLATE_VIS variant_alternative<_Ip, volatile _Tp> : add_volatile<variant_alternative_t<_Ip, _Tp>> {};
+
+template <size_t _Ip, class _Tp>
+struct _LIBCPP_TEMPLATE_VIS variant_alternative<_Ip, const volatile _Tp> : add_cv<variant_alternative_t<_Ip, _Tp>> {};
+
+template <size_t _Ip, class... _Types>
+struct _LIBCPP_TEMPLATE_VIS variant_alternative<_Ip, variant<_Types...>> {
+  static_assert(_Ip < sizeof...(_Types), "Index out of bounds in std::variant_alternative<>");
+  using type = __type_pack_element<_Ip, _Types...>;
+};
+
+inline constexpr size_t variant_npos = static_cast<size_t>(-1);
+
+template <size_t _NumAlternatives>
+_LIBCPP_HIDE_FROM_ABI constexpr auto __choose_index_type() {
+#  ifdef _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION
+  if constexpr (_NumAlternatives < numeric_limits<unsigned char>::max())
+    return static_cast<unsigned char>(0);
+  else if constexpr (_NumAlternatives < numeric_limits<unsigned short>::max())
+    return static_cast<unsigned short>(0);
+  else
+#  endif // _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION
+    return static_cast<unsigned int>(0);
+}
+
+template <size_t _NumAlts>
+using __variant_index_t = decltype(std::__choose_index_type<_NumAlts>());
+
+template <class _IndexType>
+constexpr _IndexType __variant_npos = static_cast<_IndexType>(-1);
+
+template <class... _Types>
+class _LIBCPP_TEMPLATE_VIS variant;
+
+template <class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr variant<_Types...>& __as_variant(variant<_Types...>& __vs) noexcept {
+  return __vs;
+}
+
+template <class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr const variant<_Types...>& __as_variant(const variant<_Types...>& __vs) noexcept {
+  return __vs;
+}
+
+template <class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr variant<_Types...>&& __as_variant(variant<_Types...>&& __vs) noexcept {
+  return std::move(__vs);
+}
+
+template <class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr const variant<_Types...>&& __as_variant(const variant<_Types...>&& __vs) noexcept {
+  return std::move(__vs);
+}
+
+namespace __find_detail {
+
+template <class _Tp, class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr size_t __find_index() {
+  constexpr bool __matches[] = {is_same_v<_Tp, _Types>...};
+  size_t __result            = __not_found;
+  for (size_t __i = 0; __i < sizeof...(_Types); ++__i) {
+    if (__matches[__i]) {
+      if (__result != __not_found) {
+        return __ambiguous;
+      }
+      __result = __i;
+    }
+  }
+  return __result;
+}
+
+template <size_t _Index>
+struct __find_unambiguous_index_sfinae_impl : integral_constant<size_t, _Index> {};
+
+template <>
+struct __find_unambiguous_index_sfinae_impl<__not_found> {};
+
+template <>
+struct __find_unambiguous_index_sfinae_impl<__ambiguous> {};
+
+template <class _Tp, class... _Types>
+struct __find_unambiguous_index_sfinae : __find_unambiguous_index_sfinae_impl<__find_index<_Tp, _Types...>()> {};
+
+} // namespace __find_detail
+
+namespace __variant_detail {
+
+struct __valueless_t {};
+
+enum class _Trait { _TriviallyAvailable, _Available, _Unavailable };
+
+template <typename _Tp, template <typename> class _IsTriviallyAvailable, template <typename> class _IsAvailable>
+constexpr _Trait __trait =
+    _IsTriviallyAvailable<_Tp>::value ? _Trait::_TriviallyAvailable
+    : _IsAvailable<_Tp>::value
+        ? _Trait::_Available
+        : _Trait::_Unavailable;
+
+_LIBCPP_HIDE_FROM_ABI constexpr _Trait __common_trait(initializer_list<_Trait> __traits) {
+  _Trait __result = _Trait::_TriviallyAvailable;
+  for (_Trait __t : __traits) {
+    if (static_cast<int>(__t) > static_cast<int>(__result)) {
+      __result = __t;
+    }
+  }
+  return __result;
+}
+
+template <typename... _Types>
+struct __traits {
+  static constexpr _Trait __copy_constructible_trait =
+      __variant_detail::__common_trait({__trait<_Types, is_trivially_copy_constructible, is_copy_constructible>...});
+
+  static constexpr _Trait __move_constructible_trait =
+      __variant_detail::__common_trait({__trait<_Types, is_trivially_move_constructible, is_move_constructible>...});
+
+  static constexpr _Trait __copy_assignable_trait = __variant_detail::__common_trait(
+      {__copy_constructible_trait, __trait<_Types, is_trivially_copy_assignable, is_copy_assignable>...});
+
+  static constexpr _Trait __move_assignable_trait = __variant_detail::__common_trait(
+      {__move_constructible_trait, __trait<_Types, is_trivially_move_assignable, is_move_assignable>...});
+
+  static constexpr _Trait __destructible_trait =
+      __variant_detail::__common_trait({__trait<_Types, is_trivially_destructible, is_destructible>...});
+};
+
+namespace __access {
+
+struct __union {
+  template <class _Vp>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto&& __get_alt(_Vp&& __v, in_place_index_t<0>) {
+    return std::forward<_Vp>(__v).__head;
+  }
+
+  template <class _Vp, size_t _Ip>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto&& __get_alt(_Vp&& __v, in_place_index_t<_Ip>) {
+    return __get_alt(std::forward<_Vp>(__v).__tail, in_place_index<_Ip - 1>);
+  }
+};
+
+struct __base {
+  template <size_t _Ip, class _Vp>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto&& __get_alt(_Vp&& __v) {
+    return __union::__get_alt(std::forward<_Vp>(__v).__data, in_place_index<_Ip>);
+  }
+};
+
+struct __variant {
+  template <size_t _Ip, class _Vp>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto&& __get_alt(_Vp&& __v) {
+    return __base::__get_alt<_Ip>(std::forward<_Vp>(__v).__impl_);
+  }
+};
+
+} // namespace __access
+
+namespace __visitation {
+
+struct __base {
+  template <class _Visitor, class... _Vs>
+  _LIBCPP_HIDE_FROM_ABI static constexpr decltype(auto)
+  __visit_alt_at(size_t __index, _Visitor&& __visitor, _Vs&&... __vs) {
+    constexpr auto __fdiagonal = __make_fdiagonal<_Visitor&&, decltype(std::forward<_Vs>(__vs).__as_base())...>();
+    return __fdiagonal[__index](std::forward<_Visitor>(__visitor), std::forward<_Vs>(__vs).__as_base()...);
+  }
+
+  template <class _Visitor, class... _Vs>
+  _LIBCPP_HIDE_FROM_ABI static constexpr decltype(auto) __visit_alt(_Visitor&& __visitor, _Vs&&... __vs) {
+    constexpr auto __fmatrix = __make_fmatrix<_Visitor&&, decltype(std::forward<_Vs>(__vs).__as_base())...>();
+    return __at(__fmatrix, __vs.index()...)(std::forward<_Visitor>(__visitor), std::forward<_Vs>(__vs).__as_base()...);
+  }
+
+private:
+  template <class _Tp>
+  _LIBCPP_HIDE_FROM_ABI static constexpr const _Tp& __at(const _Tp& __elem) {
+    return __elem;
+  }
+
+  template <class _Tp, size_t _Np, typename... _Indices>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto&&
+  __at(const __farray<_Tp, _Np>& __elems, size_t __index, _Indices... __indices) {
+    return __at(__elems[__index], __indices...);
+  }
+
+  template <class _Fp, class... _Fs>
+  static _LIBCPP_HIDE_FROM_ABI constexpr void __std_visit_visitor_return_type_check() {
+    static_assert(
+        __all<is_same_v<_Fp, _Fs>...>::value, "`std::visit` requires the visitor to have a single return type.");
+  }
+
+  template <class... _Fs>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __make_farray(_Fs&&... __fs) {
+    __std_visit_visitor_return_type_check<__remove_cvref_t<_Fs>...>();
+    using __result = __farray<common_type_t<__remove_cvref_t<_Fs>...>, sizeof...(_Fs)>;
+    return __result{{std::forward<_Fs>(__fs)...}};
+  }
+
+  template <size_t... _Is>
+  struct __dispatcher {
+    template <class _Fp, class... _Vs>
+    _LIBCPP_HIDE_FROM_ABI static constexpr decltype(auto) __dispatch(_Fp __f, _Vs... __vs) {
+      return std::__invoke(static_cast<_Fp>(__f), __access::__base::__get_alt<_Is>(static_cast<_Vs>(__vs))...);
+    }
+  };
+
+  template <class _Fp, class... _Vs, size_t... _Is>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __make_dispatch(index_sequence<_Is...>) {
+    return __dispatcher<_Is...>::template __dispatch<_Fp, _Vs...>;
+  }
+
+  template <size_t _Ip, class _Fp, class... _Vs>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __make_fdiagonal_impl() {
+    return __make_dispatch<_Fp, _Vs...>(index_sequence<((void)__type_identity<_Vs>{}, _Ip)...>{});
+  }
+
+  template <class _Fp, class... _Vs, size_t... _Is>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __make_fdiagonal_impl(index_sequence<_Is...>) {
+    return __base::__make_farray(__make_fdiagonal_impl<_Is, _Fp, _Vs...>()...);
+  }
+
+  template <class _Fp, class _Vp, class... _Vs>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __make_fdiagonal() {
+    constexpr size_t __np = __remove_cvref_t<_Vp>::__size();
+    static_assert(__all<(__np == __remove_cvref_t<_Vs>::__size())...>::value);
+    return __make_fdiagonal_impl<_Fp, _Vp, _Vs...>(make_index_sequence<__np>{});
+  }
+
+  template <class _Fp, class... _Vs, size_t... _Is>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __make_fmatrix_impl(index_sequence<_Is...> __is) {
+    return __make_dispatch<_Fp, _Vs...>(__is);
+  }
+
+  template <class _Fp, class... _Vs, size_t... _Is, size_t... _Js, class... _Ls>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto
+  __make_fmatrix_impl(index_sequence<_Is...>, index_sequence<_Js...>, _Ls... __ls) {
+    return __base::__make_farray(__make_fmatrix_impl<_Fp, _Vs...>(index_sequence<_Is..., _Js>{}, __ls...)...);
+  }
+
+  template <class _Fp, class... _Vs>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __make_fmatrix() {
+    return __make_fmatrix_impl<_Fp, _Vs...>(
+        index_sequence<>{}, make_index_sequence<__remove_cvref_t<_Vs>::__size()>{}...);
+  }
+};
+
+struct __variant {
+  template <class _Visitor, class... _Vs>
+  _LIBCPP_HIDE_FROM_ABI static constexpr decltype(auto)
+  __visit_alt_at(size_t __index, _Visitor&& __visitor, _Vs&&... __vs) {
+    return __base::__visit_alt_at(__index, std::forward<_Visitor>(__visitor), std::forward<_Vs>(__vs).__impl_...);
+  }
+
+  template <class _Visitor, class... _Vs>
+  _LIBCPP_HIDE_FROM_ABI static constexpr decltype(auto) __visit_alt(_Visitor&& __visitor, _Vs&&... __vs) {
+    return __base::__visit_alt(
+        std::forward<_Visitor>(__visitor), std::__as_variant(std::forward<_Vs>(__vs)).__impl_...);
+  }
+
+  template <class _Visitor, class... _Vs>
+  _LIBCPP_HIDE_FROM_ABI static constexpr decltype(auto)
+  __visit_value_at(size_t __index, _Visitor&& __visitor, _Vs&&... __vs) {
+    return __visit_alt_at(__index, __make_value_visitor(std::forward<_Visitor>(__visitor)), std::forward<_Vs>(__vs)...);
+  }
+
+  template <class _Visitor, class... _Vs>
+  _LIBCPP_HIDE_FROM_ABI static constexpr decltype(auto) __visit_value(_Visitor&& __visitor, _Vs&&... __vs) {
+    return __visit_alt(__make_value_visitor(std::forward<_Visitor>(__visitor)), std::forward<_Vs>(__vs)...);
+  }
+
+#  if _LIBCPP_STD_VER >= 20
+  template <class _Rp, class _Visitor, class... _Vs>
+  _LIBCPP_HIDE_FROM_ABI static constexpr _Rp __visit_value(_Visitor&& __visitor, _Vs&&... __vs) {
+    return __visit_alt(__make_value_visitor<_Rp>(std::forward<_Visitor>(__visitor)), std::forward<_Vs>(__vs)...);
+  }
+#  endif
+
+private:
+  template <class _Visitor, class... _Values>
+  static _LIBCPP_HIDE_FROM_ABI constexpr void __std_visit_exhaustive_visitor_check() {
+    static_assert(is_invocable_v<_Visitor, _Values...>, "`std::visit` requires the visitor to be exhaustive.");
+  }
+
+  template <class _Visitor>
+  struct __value_visitor {
+    template <class... _Alts>
+    _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator()(_Alts&&... __alts) const {
+      __std_visit_exhaustive_visitor_check< _Visitor, decltype((std::forward<_Alts>(__alts).__value))...>();
+      return std::__invoke(std::forward<_Visitor>(__visitor), std::forward<_Alts>(__alts).__value...);
+    }
+    _Visitor&& __visitor;
+  };
+
+#  if _LIBCPP_STD_VER >= 20
+  template <class _Rp, class _Visitor>
+  struct __value_visitor_return_type {
+    template <class... _Alts>
+    _LIBCPP_HIDE_FROM_ABI constexpr _Rp operator()(_Alts&&... __alts) const {
+      __std_visit_exhaustive_visitor_check< _Visitor, decltype((std::forward<_Alts>(__alts).__value))...>();
+      if constexpr (is_void_v<_Rp>) {
+        std::__invoke(std::forward<_Visitor>(__visitor), std::forward<_Alts>(__alts).__value...);
+      } else {
+        return std::__invoke(std::forward<_Visitor>(__visitor), std::forward<_Alts>(__alts).__value...);
+      }
+    }
+
+    _Visitor&& __visitor;
+  };
+#  endif
+
+  template <class _Visitor>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __make_value_visitor(_Visitor&& __visitor) {
+    return __value_visitor<_Visitor>{std::forward<_Visitor>(__visitor)};
+  }
+
+#  if _LIBCPP_STD_VER >= 20
+  template <class _Rp, class _Visitor>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto __make_value_visitor(_Visitor&& __visitor) {
+    return __value_visitor_return_type<_Rp, _Visitor>{std::forward<_Visitor>(__visitor)};
+  }
+#  endif
+};
+
+} // namespace __visitation
+
+// Adding semi-colons in macro expansions helps clang-format to do a better job.
+// This macro is used to avoid compilation errors due to "stray" semi-colons.
+#  define _LIBCPP_EAT_SEMICOLON static_assert(true, "")
+
+template <size_t _Index, class _Tp>
+struct _LIBCPP_TEMPLATE_VIS __alt {
+  using __value_type              = _Tp;
+  static constexpr size_t __index = _Index;
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr __alt(in_place_t, _Args&&... __args)
+      : __value(std::forward<_Args>(__args)...) {}
+
+  __value_type __value;
+};
+
+template <_Trait _DestructibleTrait, size_t _Index, class... _Types>
+union _LIBCPP_TEMPLATE_VIS __union;
+
+template <_Trait _DestructibleTrait, size_t _Index>
+union _LIBCPP_TEMPLATE_VIS __union<_DestructibleTrait, _Index> {};
+
+#  define _LIBCPP_VARIANT_UNION(destructible_trait, destructor_definition)                                             \
+    template <size_t _Index, class _Tp, class... _Types>                                                               \
+    union _LIBCPP_TEMPLATE_VIS __union<destructible_trait, _Index, _Tp, _Types...> {                                   \
+    public:                                                                                                            \
+      _LIBCPP_HIDE_FROM_ABI explicit constexpr __union(__valueless_t) noexcept : __dummy{} {}                          \
+                                                                                                                       \
+      template <class... _Args>                                                                                        \
+      _LIBCPP_HIDE_FROM_ABI explicit constexpr __union(in_place_index_t<0>, _Args&&... __args)                         \
+          : __head(in_place, std::forward<_Args>(__args)...) {}                                                        \
+                                                                                                                       \
+      template <size_t _Ip, class... _Args>                                                                            \
+      _LIBCPP_HIDE_FROM_ABI explicit constexpr __union(in_place_index_t<_Ip>, _Args&&... __args)                       \
+          : __tail(in_place_index<_Ip - 1>, std::forward<_Args>(__args)...) {}                                         \
+                                                                                                                       \
+      _LIBCPP_HIDE_FROM_ABI __union(const __union&)            = default;                                              \
+      _LIBCPP_HIDE_FROM_ABI __union(__union&&)                 = default;                                              \
+      _LIBCPP_HIDE_FROM_ABI __union& operator=(const __union&) = default;                                              \
+      _LIBCPP_HIDE_FROM_ABI __union& operator=(__union&&)      = default;                                              \
+      destructor_definition;                                                                                           \
+                                                                                                                       \
+    private:                                                                                                           \
+      char __dummy;                                                                                                    \
+      __alt<_Index, _Tp> __head;                                                                                       \
+      __union<destructible_trait, _Index + 1, _Types...> __tail;                                                       \
+                                                                                                                       \
+      friend struct __access::__union;                                                                                 \
+    }
+
+_LIBCPP_VARIANT_UNION(_Trait::_TriviallyAvailable,
+                      _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__union() = default);
+_LIBCPP_VARIANT_UNION(
+    _Trait::_Available, _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__union() {} _LIBCPP_EAT_SEMICOLON);
+_LIBCPP_VARIANT_UNION(_Trait::_Unavailable, _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__union() = delete);
+
+#  undef _LIBCPP_VARIANT_UNION
+
+template <_Trait _DestructibleTrait, class... _Types>
+class _LIBCPP_TEMPLATE_VIS __base {
+public:
+  using __index_t = __variant_index_t<sizeof...(_Types)>;
+
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr __base(__valueless_t __tag) noexcept
+      : __data(__tag), __index(__variant_npos<__index_t>) {}
+
+  template <size_t _Ip, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr __base(in_place_index_t<_Ip>, _Args&&... __args)
+      : __data(in_place_index<_Ip>, std::forward<_Args>(__args)...), __index(_Ip) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool valueless_by_exception() const noexcept { return index() == variant_npos; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr size_t index() const noexcept {
+    return __index == __variant_npos<__index_t> ? variant_npos : __index;
+  }
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI constexpr auto&& __as_base() & { return *this; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto&& __as_base() && { return std::move(*this); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto&& __as_base() const& { return *this; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto&& __as_base() const&& { return std::move(*this); }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr size_t __size() { return sizeof...(_Types); }
+
+  __union<_DestructibleTrait, 0, _Types...> __data;
+  __index_t __index;
+
+  friend struct __access::__base;
+  friend struct __visitation::__base;
+};
+
+template <class _Traits, _Trait = _Traits::__destructible_trait>
+class _LIBCPP_TEMPLATE_VIS __dtor;
+
+#  define _LIBCPP_VARIANT_DESTRUCTOR(destructible_trait, destructor_definition, destroy)                               \
+    template <class... _Types>                                                                                         \
+    class _LIBCPP_TEMPLATE_VIS __dtor<__traits<_Types...>, destructible_trait>                                         \
+        : public __base<destructible_trait, _Types...> {                                                               \
+      using __base_type = __base<destructible_trait, _Types...>;                                                       \
+      using __index_t   = typename __base_type::__index_t;                                                             \
+                                                                                                                       \
+    public:                                                                                                            \
+      using __base_type::__base_type;                                                                                  \
+      using __base_type::operator=;                                                                                    \
+      _LIBCPP_HIDE_FROM_ABI __dtor(const __dtor&)            = default;                                                \
+      _LIBCPP_HIDE_FROM_ABI __dtor(__dtor&&)                 = default;                                                \
+      _LIBCPP_HIDE_FROM_ABI __dtor& operator=(const __dtor&) = default;                                                \
+      _LIBCPP_HIDE_FROM_ABI __dtor& operator=(__dtor&&)      = default;                                                \
+      destructor_definition;                                                                                           \
+                                                                                                                       \
+    protected:                                                                                                         \
+      destroy;                                                                                                         \
+    }
+
+_LIBCPP_VARIANT_DESTRUCTOR(
+    _Trait::_TriviallyAvailable,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__dtor() = default,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __destroy() noexcept {
+      this->__index = __variant_npos<__index_t>;
+    } _LIBCPP_EAT_SEMICOLON);
+
+_LIBCPP_VARIANT_DESTRUCTOR(
+    _Trait::_Available,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__dtor() { __destroy(); } _LIBCPP_EAT_SEMICOLON,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __destroy() noexcept {
+      if (!this->valueless_by_exception()) {
+        __visitation::__base::__visit_alt(
+            [](auto& __alt) noexcept {
+              using __alt_type = __remove_cvref_t<decltype(__alt)>;
+              __alt.~__alt_type();
+            },
+            *this);
+      }
+      this->__index = __variant_npos<__index_t>;
+    } _LIBCPP_EAT_SEMICOLON);
+
+_LIBCPP_VARIANT_DESTRUCTOR(_Trait::_Unavailable,
+                           _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__dtor()                 = delete,
+                           _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __destroy() noexcept = delete);
+
+#  undef _LIBCPP_VARIANT_DESTRUCTOR
+
+template <class _Traits>
+class _LIBCPP_TEMPLATE_VIS __ctor : public __dtor<_Traits> {
+  using __base_type = __dtor<_Traits>;
+
+public:
+  using __base_type::__base_type;
+  using __base_type::operator=;
+
+protected:
+  template <class _Rhs>
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX20 void __generic_construct(__ctor& __lhs, _Rhs&& __rhs) {
+    __lhs.__destroy();
+    if (!__rhs.valueless_by_exception()) {
+      auto __rhs_index = __rhs.index();
+      __visitation::__base::__visit_alt_at(
+          __rhs_index,
+          [&__lhs](auto&& __rhs_alt) {
+            std::__construct_at(std::addressof(__lhs.__data),
+                                in_place_index<__decay_t<decltype(__rhs_alt)>::__index>,
+                                std::forward<decltype(__rhs_alt)>(__rhs_alt).__value);
+          },
+          std::forward<_Rhs>(__rhs));
+      __lhs.__index = __rhs_index;
+    }
+  }
+};
+
+template <class _Traits, _Trait = _Traits::__move_constructible_trait>
+class _LIBCPP_TEMPLATE_VIS __move_constructor;
+
+#  define _LIBCPP_VARIANT_MOVE_CONSTRUCTOR(move_constructible_trait, move_constructor_definition)                      \
+    template <class... _Types>                                                                                         \
+    class _LIBCPP_TEMPLATE_VIS __move_constructor<__traits<_Types...>, move_constructible_trait>                       \
+        : public __ctor<__traits<_Types...>> {                                                                         \
+      using __base_type = __ctor<__traits<_Types...>>;                                                                 \
+                                                                                                                       \
+    public:                                                                                                            \
+      using __base_type::__base_type;                                                                                  \
+      using __base_type::operator=;                                                                                    \
+                                                                                                                       \
+      _LIBCPP_HIDE_FROM_ABI __move_constructor(const __move_constructor&)            = default;                        \
+      _LIBCPP_HIDE_FROM_ABI ~__move_constructor()                                    = default;                        \
+      _LIBCPP_HIDE_FROM_ABI __move_constructor& operator=(const __move_constructor&) = default;                        \
+      _LIBCPP_HIDE_FROM_ABI __move_constructor& operator=(__move_constructor&&)      = default;                        \
+      move_constructor_definition;                                                                                     \
+    }
+
+_LIBCPP_VARIANT_MOVE_CONSTRUCTOR(
+    _Trait::_TriviallyAvailable,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __move_constructor(__move_constructor&& __that) = default);
+
+_LIBCPP_VARIANT_MOVE_CONSTRUCTOR(
+    _Trait::_Available,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __move_constructor(__move_constructor&& __that) noexcept(
+        __all<is_nothrow_move_constructible_v<_Types>...>::value)
+    : __move_constructor(__valueless_t{}) {
+      this->__generic_construct(*this, std::move(__that));
+    } _LIBCPP_EAT_SEMICOLON);
+
+_LIBCPP_VARIANT_MOVE_CONSTRUCTOR(
+    _Trait::_Unavailable,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __move_constructor(__move_constructor&&) = delete);
+
+#  undef _LIBCPP_VARIANT_MOVE_CONSTRUCTOR
+
+template <class _Traits, _Trait = _Traits::__copy_constructible_trait>
+class _LIBCPP_TEMPLATE_VIS __copy_constructor;
+
+#  define _LIBCPP_VARIANT_COPY_CONSTRUCTOR(copy_constructible_trait, copy_constructor_definition)                      \
+    template <class... _Types>                                                                                         \
+    class _LIBCPP_TEMPLATE_VIS __copy_constructor<__traits<_Types...>, copy_constructible_trait>                       \
+        : public __move_constructor<__traits<_Types...>> {                                                             \
+      using __base_type = __move_constructor<__traits<_Types...>>;                                                     \
+                                                                                                                       \
+    public:                                                                                                            \
+      using __base_type::__base_type;                                                                                  \
+      using __base_type::operator=;                                                                                    \
+                                                                                                                       \
+      _LIBCPP_HIDE_FROM_ABI __copy_constructor(__copy_constructor&&)                 = default;                        \
+      _LIBCPP_HIDE_FROM_ABI ~__copy_constructor()                                    = default;                        \
+      _LIBCPP_HIDE_FROM_ABI __copy_constructor& operator=(const __copy_constructor&) = default;                        \
+      _LIBCPP_HIDE_FROM_ABI __copy_constructor& operator=(__copy_constructor&&)      = default;                        \
+      copy_constructor_definition;                                                                                     \
+    }
+
+_LIBCPP_VARIANT_COPY_CONSTRUCTOR(
+    _Trait::_TriviallyAvailable,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __copy_constructor(const __copy_constructor& __that) = default);
+
+_LIBCPP_VARIANT_COPY_CONSTRUCTOR(
+    _Trait::_Available,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __copy_constructor(const __copy_constructor& __that)
+    : __copy_constructor(__valueless_t{}) { this->__generic_construct(*this, __that); } _LIBCPP_EAT_SEMICOLON);
+
+_LIBCPP_VARIANT_COPY_CONSTRUCTOR(
+    _Trait::_Unavailable,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __copy_constructor(const __copy_constructor&) = delete);
+
+#  undef _LIBCPP_VARIANT_COPY_CONSTRUCTOR
+
+template <class _Traits>
+class _LIBCPP_TEMPLATE_VIS __assignment : public __copy_constructor<_Traits> {
+  using __base_type = __copy_constructor<_Traits>;
+
+public:
+  using __base_type::__base_type;
+  using __base_type::operator=;
+
+  template <size_t _Ip, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 auto& __emplace(_Args&&... __args) {
+    this->__destroy();
+    std::__construct_at(std::addressof(this->__data), in_place_index<_Ip>, std::forward<_Args>(__args)...);
+    this->__index = _Ip;
+    return __access::__base::__get_alt<_Ip>(*this).__value;
+  }
+
+protected:
+  template <size_t _Ip, class _Tp, class _Arg>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __assign_alt(__alt<_Ip, _Tp>& __a, _Arg&& __arg) {
+    if (this->index() == _Ip) {
+      __a.__value = std::forward<_Arg>(__arg);
+    } else {
+      struct {
+        _LIBCPP_HIDDEN _LIBCPP_CONSTEXPR_SINCE_CXX20 void operator()(true_type) const {
+          __this->__emplace<_Ip>(std::forward<_Arg>(__arg));
+        }
+        _LIBCPP_HIDDEN _LIBCPP_CONSTEXPR_SINCE_CXX20 void operator()(false_type) const {
+          __this->__emplace<_Ip>(_Tp(std::forward<_Arg>(__arg)));
+        }
+        __assignment* __this;
+        _Arg&& __arg;
+      } __impl{this, std::forward<_Arg>(__arg)};
+      __impl(bool_constant < is_nothrow_constructible_v<_Tp, _Arg> || !is_nothrow_move_constructible_v < _Tp >> {});
+    }
+  }
+
+  template <class _That>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __generic_assign(_That&& __that) {
+    if (this->valueless_by_exception() && __that.valueless_by_exception()) {
+      // do nothing.
+    } else if (__that.valueless_by_exception()) {
+      this->__destroy();
+    } else {
+      __visitation::__base::__visit_alt_at(
+          __that.index(),
+          [this](auto& __this_alt, auto&& __that_alt) {
+            this->__assign_alt(__this_alt, std::forward<decltype(__that_alt)>(__that_alt).__value);
+          },
+          *this,
+          std::forward<_That>(__that));
+    }
+  }
+};
+
+template <class _Traits, _Trait = _Traits::__move_assignable_trait>
+class _LIBCPP_TEMPLATE_VIS __move_assignment;
+
+#  define _LIBCPP_VARIANT_MOVE_ASSIGNMENT(move_assignable_trait, move_assignment_definition)                           \
+    template <class... _Types>                                                                                         \
+    class _LIBCPP_TEMPLATE_VIS __move_assignment<__traits<_Types...>, move_assignable_trait>                           \
+        : public __assignment<__traits<_Types...>> {                                                                   \
+      using __base_type = __assignment<__traits<_Types...>>;                                                           \
+                                                                                                                       \
+    public:                                                                                                            \
+      using __base_type::__base_type;                                                                                  \
+      using __base_type::operator=;                                                                                    \
+                                                                                                                       \
+      _LIBCPP_HIDE_FROM_ABI __move_assignment(const __move_assignment&)            = default;                          \
+      _LIBCPP_HIDE_FROM_ABI __move_assignment(__move_assignment&&)                 = default;                          \
+      _LIBCPP_HIDE_FROM_ABI ~__move_assignment()                                   = default;                          \
+      _LIBCPP_HIDE_FROM_ABI __move_assignment& operator=(const __move_assignment&) = default;                          \
+      move_assignment_definition;                                                                                      \
+    }
+
+_LIBCPP_VARIANT_MOVE_ASSIGNMENT(_Trait::_TriviallyAvailable,
+                                _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __move_assignment& operator=(
+                                    __move_assignment&& __that) = default);
+
+_LIBCPP_VARIANT_MOVE_ASSIGNMENT(
+    _Trait::_Available,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __move_assignment&
+    operator=(__move_assignment&& __that) noexcept(
+        __all<(is_nothrow_move_constructible_v<_Types> && is_nothrow_move_assignable_v<_Types>)...>::value) {
+      this->__generic_assign(std::move(__that));
+      return *this;
+    } _LIBCPP_EAT_SEMICOLON);
+
+_LIBCPP_VARIANT_MOVE_ASSIGNMENT(
+    _Trait::_Unavailable,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __move_assignment& operator=(__move_assignment&&) = delete);
+
+#  undef _LIBCPP_VARIANT_MOVE_ASSIGNMENT
+
+template <class _Traits, _Trait = _Traits::__copy_assignable_trait>
+class _LIBCPP_TEMPLATE_VIS __copy_assignment;
+
+#  define _LIBCPP_VARIANT_COPY_ASSIGNMENT(copy_assignable_trait, copy_assignment_definition)                           \
+    template <class... _Types>                                                                                         \
+    class _LIBCPP_TEMPLATE_VIS __copy_assignment<__traits<_Types...>, copy_assignable_trait>                           \
+        : public __move_assignment<__traits<_Types...>> {                                                              \
+      using __base_type = __move_assignment<__traits<_Types...>>;                                                      \
+                                                                                                                       \
+    public:                                                                                                            \
+      using __base_type::__base_type;                                                                                  \
+      using __base_type::operator=;                                                                                    \
+                                                                                                                       \
+      _LIBCPP_HIDE_FROM_ABI __copy_assignment(const __copy_assignment&)       = default;                               \
+      _LIBCPP_HIDE_FROM_ABI __copy_assignment(__copy_assignment&&)            = default;                               \
+      _LIBCPP_HIDE_FROM_ABI ~__copy_assignment()                              = default;                               \
+      _LIBCPP_HIDE_FROM_ABI __copy_assignment& operator=(__copy_assignment&&) = default;                               \
+      copy_assignment_definition;                                                                                      \
+    }
+
+_LIBCPP_VARIANT_COPY_ASSIGNMENT(_Trait::_TriviallyAvailable,
+                                _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __copy_assignment& operator=(
+                                    const __copy_assignment& __that) = default);
+
+_LIBCPP_VARIANT_COPY_ASSIGNMENT(
+    _Trait::_Available,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __copy_assignment&
+    operator=(const __copy_assignment& __that) {
+      this->__generic_assign(__that);
+      return *this;
+    } _LIBCPP_EAT_SEMICOLON);
+
+_LIBCPP_VARIANT_COPY_ASSIGNMENT(_Trait::_Unavailable,
+                                _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __copy_assignment& operator=(
+                                    const __copy_assignment&) = delete);
+
+#  undef _LIBCPP_VARIANT_COPY_ASSIGNMENT
+
+template <class... _Types>
+class _LIBCPP_TEMPLATE_VIS __impl : public __copy_assignment<__traits<_Types...>> {
+  using __base_type = __copy_assignment<__traits<_Types...>>;
+
+public:
+  using __base_type::__base_type; // get in_place_index_t constructor & friends
+  _LIBCPP_HIDE_FROM_ABI __impl(__impl const&)            = default;
+  _LIBCPP_HIDE_FROM_ABI __impl(__impl&&)                 = default;
+  _LIBCPP_HIDE_FROM_ABI __impl& operator=(__impl const&) = default;
+  _LIBCPP_HIDE_FROM_ABI __impl& operator=(__impl&&)      = default;
+
+  template <size_t _Ip, class _Arg>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __assign(_Arg&& __arg) {
+    this->__assign_alt(__access::__base::__get_alt<_Ip>(*this), std::forward<_Arg>(__arg));
+  }
+
+  inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __swap(__impl& __that) {
+    if (this->valueless_by_exception() && __that.valueless_by_exception()) {
+      // do nothing.
+    } else if (this->index() == __that.index()) {
+      __visitation::__base::__visit_alt_at(
+          this->index(),
+          [](auto& __this_alt, auto& __that_alt) {
+            using std::swap;
+            swap(__this_alt.__value, __that_alt.__value);
+          },
+          *this,
+          __that);
+    } else {
+      __impl* __lhs = this;
+      __impl* __rhs = std::addressof(__that);
+      if (__lhs->__move_nothrow() && !__rhs->__move_nothrow()) {
+        std::swap(__lhs, __rhs);
+      }
+      __impl __tmp(std::move(*__rhs));
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+      if constexpr (__all<is_nothrow_move_constructible_v<_Types>...>::value) {
+        this->__generic_construct(*__rhs, std::move(*__lhs));
+      } else {
+        // EXTENSION: When the move construction of `__lhs` into `__rhs` throws
+        // and `__tmp` is nothrow move constructible then we move `__tmp` back
+        // into `__rhs` and provide the strong exception safety guarantee.
+        try {
+          this->__generic_construct(*__rhs, std::move(*__lhs));
+        } catch (...) {
+          if (__tmp.__move_nothrow()) {
+            this->__generic_construct(*__rhs, std::move(__tmp));
+          }
+          throw;
+        }
+      }
+#  else
+      // this isn't consolidated with the `if constexpr` branch above due to
+      // `throw` being ill-formed with exceptions disabled even when discarded.
+      this->__generic_construct(*__rhs, std::move(*__lhs));
+#  endif
+      this->__generic_construct(*__lhs, std::move(__tmp));
+    }
+  }
+
+private:
+  constexpr inline _LIBCPP_HIDE_FROM_ABI bool __move_nothrow() const {
+    constexpr bool __results[] = {is_nothrow_move_constructible_v<_Types>...};
+    return this->valueless_by_exception() || __results[this->index()];
+  }
+};
+
+struct __no_narrowing_check {
+  template <class _Dest, class _Source>
+  using _Apply = __type_identity<_Dest>;
+};
+
+struct __narrowing_check {
+  template <class _Dest>
+  static auto __test_impl(_Dest (&&)[1]) -> __type_identity<_Dest>;
+  template <class _Dest, class _Source>
+  using _Apply _LIBCPP_NODEBUG = decltype(__test_impl<_Dest>({std::declval<_Source>()}));
+};
+
+template <class _Dest, class _Source>
+using __check_for_narrowing _LIBCPP_NODEBUG =
+    typename _If< is_arithmetic<_Dest>::value, __narrowing_check, __no_narrowing_check >::template _Apply<_Dest,
+                                                                                                          _Source>;
+
+template <class _Tp, size_t _Idx>
+struct __overload {
+  template <class _Up>
+  auto operator()(_Tp, _Up&&) const -> __check_for_narrowing<_Tp, _Up>;
+};
+
+template <class... _Bases>
+struct __all_overloads : _Bases... {
+  void operator()() const;
+  using _Bases::operator()...;
+};
+
+template <class _IdxSeq>
+struct __make_overloads_imp;
+
+template <size_t... _Idx>
+struct __make_overloads_imp<__tuple_indices<_Idx...> > {
+  template <class... _Types>
+  using _Apply _LIBCPP_NODEBUG = __all_overloads<__overload<_Types, _Idx>...>;
+};
+
+template <class... _Types>
+using _MakeOverloads _LIBCPP_NODEBUG =
+    typename __make_overloads_imp< __make_indices_imp<sizeof...(_Types), 0> >::template _Apply<_Types...>;
+
+template <class _Tp, class... _Types>
+using __best_match_t = typename invoke_result_t<_MakeOverloads<_Types...>, _Tp, _Tp>::type;
+
+} // namespace __variant_detail
+
+template <class _Visitor, class... _Vs, typename = void_t<decltype(std::__as_variant(std::declval<_Vs>()))...>>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr decltype(auto)
+visit(_Visitor&& __visitor, _Vs&&... __vs);
+
+#  if _LIBCPP_STD_VER >= 20
+template <class _Rp,
+          class _Visitor,
+          class... _Vs,
+          typename = void_t<decltype(std::__as_variant(std::declval<_Vs>()))...>>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr _Rp
+visit(_Visitor&& __visitor, _Vs&&... __vs);
+#  endif
+
+template <class... _Types>
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DECLSPEC_EMPTY_BASES variant
+    : private __sfinae_ctor_base< __all<is_copy_constructible_v<_Types>...>::value,
+                                  __all<is_move_constructible_v<_Types>...>::value>,
+      private __sfinae_assign_base<
+          __all<(is_copy_constructible_v<_Types> && is_copy_assignable_v<_Types>)...>::value,
+          __all<(is_move_constructible_v<_Types> && is_move_assignable_v<_Types>)...>::value> {
+  static_assert(0 < sizeof...(_Types), "variant must consist of at least one alternative.");
+
+  static_assert(__all<!is_array_v<_Types>...>::value, "variant can not have an array type as an alternative.");
+
+  static_assert(__all<!is_reference_v<_Types>...>::value, "variant can not have a reference type as an alternative.");
+
+  static_assert(__all<!is_void_v<_Types>...>::value, "variant can not have a void type as an alternative.");
+
+  using __first_type = variant_alternative_t<0, variant>;
+
+public:
+  using __trivially_relocatable =
+      conditional_t<_And<__libcpp_is_trivially_relocatable<_Types>...>::value, variant, void>;
+
+  template <bool _Dummy                                                                               = true,
+            enable_if_t<__dependent_type<is_default_constructible<__first_type>, _Dummy>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI constexpr variant() noexcept(is_nothrow_default_constructible_v<__first_type>)
+      : __impl_(in_place_index<0>) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr variant(const variant&) = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr variant(variant&&)      = default;
+
+  template < class _Arg,
+             enable_if_t<!is_same_v<__remove_cvref_t<_Arg>, variant>, int>        = 0,
+             enable_if_t<!__is_inplace_type<__remove_cvref_t<_Arg>>::value, int>  = 0,
+             enable_if_t<!__is_inplace_index<__remove_cvref_t<_Arg>>::value, int> = 0,
+             class _Tp  = __variant_detail::__best_match_t<_Arg, _Types...>,
+             size_t _Ip = __find_detail::__find_unambiguous_index_sfinae<_Tp, _Types...>::value,
+             enable_if_t<is_constructible_v<_Tp, _Arg>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI constexpr variant(_Arg&& __arg) noexcept(is_nothrow_constructible_v<_Tp, _Arg>)
+      : __impl_(in_place_index<_Ip>, std::forward<_Arg>(__arg)) {}
+
+  template <size_t _Ip,
+            class... _Args,
+            class                                               = enable_if_t<(_Ip < sizeof...(_Types)), int>,
+            class _Tp                                           = variant_alternative_t<_Ip, variant<_Types...>>,
+            enable_if_t<is_constructible_v<_Tp, _Args...>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr variant(in_place_index_t<_Ip>, _Args&&... __args) noexcept(
+      is_nothrow_constructible_v<_Tp, _Args...>)
+      : __impl_(in_place_index<_Ip>, std::forward<_Args>(__args)...) {}
+
+  template < size_t _Ip,
+             class _Up,
+             class... _Args,
+             enable_if_t<(_Ip < sizeof...(_Types)), int> = 0,
+             class _Tp                                   = variant_alternative_t<_Ip, variant<_Types...>>,
+             enable_if_t<is_constructible_v<_Tp, initializer_list<_Up>&, _Args...>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr variant(
+      in_place_index_t<_Ip>,
+      initializer_list<_Up> __il,
+      _Args&&... __args) noexcept(is_nothrow_constructible_v<_Tp, initializer_list<_Up>&, _Args...>)
+      : __impl_(in_place_index<_Ip>, __il, std::forward<_Args>(__args)...) {}
+
+  template < class _Tp,
+             class... _Args,
+             size_t _Ip = __find_detail::__find_unambiguous_index_sfinae<_Tp, _Types...>::value,
+             enable_if_t<is_constructible_v<_Tp, _Args...>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr variant(in_place_type_t<_Tp>, _Args&&... __args) noexcept(
+      is_nothrow_constructible_v<_Tp, _Args...>)
+      : __impl_(in_place_index<_Ip>, std::forward<_Args>(__args)...) {}
+
+  template < class _Tp,
+             class _Up,
+             class... _Args,
+             size_t _Ip = __find_detail::__find_unambiguous_index_sfinae<_Tp, _Types...>::value,
+             enable_if_t<is_constructible_v<_Tp, initializer_list<_Up>&, _Args...>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit constexpr variant(
+      in_place_type_t<_Tp>,
+      initializer_list<_Up> __il,
+      _Args&&... __args) noexcept(is_nothrow_constructible_v<_Tp, initializer_list< _Up>&, _Args...>)
+      : __impl_(in_place_index<_Ip>, __il, std::forward<_Args>(__args)...) {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~variant() = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr variant& operator=(const variant&) = default;
+  _LIBCPP_HIDE_FROM_ABI constexpr variant& operator=(variant&&)      = default;
+
+  template < class _Arg,
+             enable_if_t<!is_same_v<__remove_cvref_t<_Arg>, variant>, int> = 0,
+             class _Tp  = __variant_detail::__best_match_t<_Arg, _Types...>,
+             size_t _Ip = __find_detail::__find_unambiguous_index_sfinae<_Tp, _Types...>::value,
+             enable_if_t<is_assignable_v<_Tp&, _Arg> && is_constructible_v<_Tp, _Arg>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 variant&
+  operator=(_Arg&& __arg) noexcept(is_nothrow_assignable_v<_Tp&, _Arg> && is_nothrow_constructible_v<_Tp, _Arg>) {
+    __impl_.template __assign<_Ip>(std::forward<_Arg>(__arg));
+    return *this;
+  }
+
+  template < size_t _Ip,
+             class... _Args,
+             enable_if_t<(_Ip < sizeof...(_Types)), int>         = 0,
+             class _Tp                                           = variant_alternative_t<_Ip, variant<_Types...>>,
+             enable_if_t<is_constructible_v<_Tp, _Args...>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& emplace(_Args&&... __args) {
+    return __impl_.template __emplace<_Ip>(std::forward<_Args>(__args)...);
+  }
+
+  template < size_t _Ip,
+             class _Up,
+             class... _Args,
+             enable_if_t<(_Ip < sizeof...(_Types)), int> = 0,
+             class _Tp                                   = variant_alternative_t<_Ip, variant<_Types...>>,
+             enable_if_t<is_constructible_v<_Tp, initializer_list<_Up>&, _Args...>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& emplace(initializer_list<_Up> __il, _Args&&... __args) {
+    return __impl_.template __emplace<_Ip>(__il, std::forward<_Args>(__args)...);
+  }
+
+  template < class _Tp,
+             class... _Args,
+             size_t _Ip = __find_detail::__find_unambiguous_index_sfinae<_Tp, _Types...>::value,
+             enable_if_t<is_constructible_v<_Tp, _Args...>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& emplace(_Args&&... __args) {
+    return __impl_.template __emplace<_Ip>(std::forward<_Args>(__args)...);
+  }
+
+  template < class _Tp,
+             class _Up,
+             class... _Args,
+             size_t _Ip = __find_detail::__find_unambiguous_index_sfinae<_Tp, _Types...>::value,
+             enable_if_t<is_constructible_v<_Tp, initializer_list<_Up>&, _Args...>, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& emplace(initializer_list<_Up> __il, _Args&&... __args) {
+    return __impl_.template __emplace<_Ip>(__il, std::forward<_Args>(__args)...);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool valueless_by_exception() const noexcept {
+    return __impl_.valueless_by_exception();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr size_t index() const noexcept { return __impl_.index(); }
+
+  template < bool _Dummy       = true,
+             enable_if_t< __all<(__dependent_type<is_move_constructible<_Types>, _Dummy>::value &&
+                                 __dependent_type<is_swappable<_Types>, _Dummy>::value)...>::value,
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(variant& __that) noexcept(
+      __all<(is_nothrow_move_constructible_v<_Types> && is_nothrow_swappable_v<_Types>)...>::value) {
+    __impl_.__swap(__that.__impl_);
+  }
+
+#  if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+  // Helper class to implement [variant.visit]/10
+  //   Constraints: The call to visit does not use an explicit template-argument-list
+  //   that begins with a type template-argument.
+  struct __variant_visit_barrier_tag {
+    _LIBCPP_HIDE_FROM_ABI explicit __variant_visit_barrier_tag() = default;
+  };
+
+  template <__variant_visit_barrier_tag = __variant_visit_barrier_tag{}, class _Self, class _Visitor>
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) visit(this _Self&& __self, _Visitor&& __visitor) {
+    using _VariantT = _OverrideRef<_Self&&, _CopyConst<remove_reference_t<_Self>, variant>>;
+    return std::visit(std::forward<_Visitor>(__visitor), (_VariantT)__self);
+  }
+
+  template <class _Rp, class _Self, class _Visitor>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Rp visit(this _Self&& __self, _Visitor&& __visitor) {
+    using _VariantT = _OverrideRef<_Self&&, _CopyConst<remove_reference_t<_Self>, variant>>;
+    return std::visit<_Rp>(std::forward<_Visitor>(__visitor), (_VariantT)__self);
+  }
+#  endif
+
+private:
+  __variant_detail::__impl<_Types...> __impl_;
+
+  friend struct __variant_detail::__access::__variant;
+  friend struct __variant_detail::__visitation::__variant;
+};
+
+template <size_t _Ip, class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr bool __holds_alternative(const variant<_Types...>& __v) noexcept {
+  return __v.index() == _Ip;
+}
+
+template <class _Tp, class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr bool holds_alternative(const variant<_Types...>& __v) noexcept {
+  return std::__holds_alternative<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
+}
+
+template <size_t _Ip, class _Vp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr auto&& __generic_get(_Vp&& __v) {
+  using __variant_detail::__access::__variant;
+  if (!std::__holds_alternative<_Ip>(__v)) {
+    __throw_bad_variant_access();
+  }
+  return __variant::__get_alt<_Ip>(std::forward<_Vp>(__v)).__value;
+}
+
+template <size_t _Ip, class... _Types>
+_LIBCPP_HIDE_FROM_ABI
+_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr variant_alternative_t<_Ip, variant<_Types...>>&
+get(variant<_Types...>& __v) {
+  static_assert(_Ip < sizeof...(_Types));
+  static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
+  return std::__generic_get<_Ip>(__v);
+}
+
+template <size_t _Ip, class... _Types>
+_LIBCPP_HIDE_FROM_ABI
+_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr variant_alternative_t<_Ip, variant<_Types...>>&&
+get(variant<_Types...>&& __v) {
+  static_assert(_Ip < sizeof...(_Types));
+  static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
+  return std::__generic_get<_Ip>(std::move(__v));
+}
+
+template <size_t _Ip, class... _Types>
+_LIBCPP_HIDE_FROM_ABI
+_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const variant_alternative_t<_Ip, variant<_Types...>>&
+get(const variant<_Types...>& __v) {
+  static_assert(_Ip < sizeof...(_Types));
+  static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
+  return std::__generic_get<_Ip>(__v);
+}
+
+template <size_t _Ip, class... _Types>
+_LIBCPP_HIDE_FROM_ABI
+_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const variant_alternative_t<_Ip, variant<_Types...>>&&
+get(const variant<_Types...>&& __v) {
+  static_assert(_Ip < sizeof...(_Types));
+  static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
+  return std::__generic_get<_Ip>(std::move(__v));
+}
+
+template <class _Tp, class... _Types>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr _Tp& get(variant<_Types...>& __v) {
+  static_assert(!is_void_v<_Tp>);
+  return std::get<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
+}
+
+template <class _Tp, class... _Types>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr _Tp&& get(variant<_Types...>&& __v) {
+  static_assert(!is_void_v<_Tp>);
+  return std::get<__find_exactly_one_t<_Tp, _Types...>::value>(std::move(__v));
+}
+
+template <class _Tp, class... _Types>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const _Tp&
+get(const variant<_Types...>& __v) {
+  static_assert(!is_void_v<_Tp>);
+  return std::get<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
+}
+
+template <class _Tp, class... _Types>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const _Tp&&
+get(const variant<_Types...>&& __v) {
+  static_assert(!is_void_v<_Tp>);
+  return std::get<__find_exactly_one_t<_Tp, _Types...>::value>(std::move(__v));
+}
+
+template <size_t _Ip, class _Vp>
+_LIBCPP_HIDE_FROM_ABI constexpr auto* __generic_get_if(_Vp* __v) noexcept {
+  using __variant_detail::__access::__variant;
+  return __v && std::__holds_alternative<_Ip>(*__v) ? std::addressof(__variant::__get_alt<_Ip>(*__v).__value) : nullptr;
+}
+
+template <size_t _Ip, class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<variant_alternative_t<_Ip, variant<_Types...>>>
+get_if(variant<_Types...>* __v) noexcept {
+  static_assert(_Ip < sizeof...(_Types));
+  static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
+  return std::__generic_get_if<_Ip>(__v);
+}
+
+template <size_t _Ip, class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<const variant_alternative_t<_Ip, variant<_Types...>>>
+get_if(const variant<_Types...>* __v) noexcept {
+  static_assert(_Ip < sizeof...(_Types));
+  static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
+  return std::__generic_get_if<_Ip>(__v);
+}
+
+template <class _Tp, class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<_Tp> get_if(variant<_Types...>* __v) noexcept {
+  static_assert(!is_void_v<_Tp>);
+  return std::get_if<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
+}
+
+template <class _Tp, class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<const _Tp> get_if(const variant<_Types...>* __v) noexcept {
+  static_assert(!is_void_v<_Tp>);
+  return std::get_if<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
+}
+
+template <class _Operator>
+struct __convert_to_bool {
+  template <class _T1, class _T2>
+  _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_T1&& __t1, _T2&& __t2) const {
+    static_assert(is_convertible<decltype(_Operator{}(std::forward<_T1>(__t1), std::forward<_T2>(__t2))), bool>::value,
+                  "the relational operator does not return a type which is implicitly convertible to bool");
+    return _Operator{}(std::forward<_T1>(__t1), std::forward<_T2>(__t2));
+  }
+};
+
+template <class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator==(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
+  using __variant_detail::__visitation::__variant;
+  if (__lhs.index() != __rhs.index())
+    return false;
+  if (__lhs.valueless_by_exception())
+    return true;
+  return __variant::__visit_value_at(__lhs.index(), __convert_to_bool<equal_to<>>{}, __lhs, __rhs);
+}
+
+#  if _LIBCPP_STD_VER >= 20
+
+template <class... _Types>
+  requires(three_way_comparable<_Types> && ...)
+_LIBCPP_HIDE_FROM_ABI constexpr common_comparison_category_t<compare_three_way_result_t<_Types>...>
+operator<=>(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
+  using __variant_detail::__visitation::__variant;
+  using __result_t = common_comparison_category_t<compare_three_way_result_t<_Types>...>;
+  if (__lhs.valueless_by_exception() && __rhs.valueless_by_exception())
+    return strong_ordering::equal;
+  if (__lhs.valueless_by_exception())
+    return strong_ordering::less;
+  if (__rhs.valueless_by_exception())
+    return strong_ordering::greater;
+  if (auto __c = __lhs.index() <=> __rhs.index(); __c != 0)
+    return __c;
+  auto __three_way = []<class _Type>(const _Type& __v, const _Type& __w) -> __result_t { return __v <=> __w; };
+  return __variant::__visit_value_at(__lhs.index(), __three_way, __lhs, __rhs);
+}
+
+#  endif // _LIBCPP_STD_VER >= 20
+
+template <class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator!=(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
+  using __variant_detail::__visitation::__variant;
+  if (__lhs.index() != __rhs.index())
+    return true;
+  if (__lhs.valueless_by_exception())
+    return false;
+  return __variant::__visit_value_at(__lhs.index(), __convert_to_bool<not_equal_to<>>{}, __lhs, __rhs);
+}
+
+template <class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
+  using __variant_detail::__visitation::__variant;
+  if (__rhs.valueless_by_exception())
+    return false;
+  if (__lhs.valueless_by_exception())
+    return true;
+  if (__lhs.index() < __rhs.index())
+    return true;
+  if (__lhs.index() > __rhs.index())
+    return false;
+  return __variant::__visit_value_at(__lhs.index(), __convert_to_bool<less<>>{}, __lhs, __rhs);
+}
+
+template <class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
+  using __variant_detail::__visitation::__variant;
+  if (__lhs.valueless_by_exception())
+    return false;
+  if (__rhs.valueless_by_exception())
+    return true;
+  if (__lhs.index() > __rhs.index())
+    return true;
+  if (__lhs.index() < __rhs.index())
+    return false;
+  return __variant::__visit_value_at(__lhs.index(), __convert_to_bool<greater<>>{}, __lhs, __rhs);
+}
+
+template <class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator<=(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
+  using __variant_detail::__visitation::__variant;
+  if (__lhs.valueless_by_exception())
+    return true;
+  if (__rhs.valueless_by_exception())
+    return false;
+  if (__lhs.index() < __rhs.index())
+    return true;
+  if (__lhs.index() > __rhs.index())
+    return false;
+  return __variant::__visit_value_at(__lhs.index(), __convert_to_bool<less_equal<>>{}, __lhs, __rhs);
+}
+
+template <class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr bool operator>=(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
+  using __variant_detail::__visitation::__variant;
+  if (__rhs.valueless_by_exception())
+    return true;
+  if (__lhs.valueless_by_exception())
+    return false;
+  if (__lhs.index() > __rhs.index())
+    return true;
+  if (__lhs.index() < __rhs.index())
+    return false;
+  return __variant::__visit_value_at(__lhs.index(), __convert_to_bool<greater_equal<>>{}, __lhs, __rhs);
+}
+
+template <class... _Vs>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr void __throw_if_valueless(_Vs&&... __vs) {
+  const bool __valueless = (... || std::__as_variant(__vs).valueless_by_exception());
+  if (__valueless) {
+    __throw_bad_variant_access();
+  }
+}
+
+template < class _Visitor, class... _Vs, typename>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr decltype(auto)
+visit(_Visitor&& __visitor, _Vs&&... __vs) {
+  using __variant_detail::__visitation::__variant;
+  std::__throw_if_valueless(std::forward<_Vs>(__vs)...);
+  return __variant::__visit_value(std::forward<_Visitor>(__visitor), std::forward<_Vs>(__vs)...);
+}
+
+#  if _LIBCPP_STD_VER >= 20
+template < class _Rp, class _Visitor, class... _Vs, typename>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr _Rp
+visit(_Visitor&& __visitor, _Vs&&... __vs) {
+  using __variant_detail::__visitation::__variant;
+  std::__throw_if_valueless(std::forward<_Vs>(__vs)...);
+  return __variant::__visit_value<_Rp>(std::forward<_Visitor>(__visitor), std::forward<_Vs>(__vs)...);
+}
+#  endif
+
+template <class... _Types>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 auto
+swap(variant<_Types...>& __lhs,
+     variant<_Types...>& __rhs) noexcept(noexcept(__lhs.swap(__rhs))) -> decltype(__lhs.swap(__rhs)) {
+  return __lhs.swap(__rhs);
+}
+
+template <class... _Types>
+struct _LIBCPP_TEMPLATE_VIS hash< __enable_hash_helper<variant<_Types...>, remove_const_t<_Types>...>> {
+  using argument_type = variant<_Types...>;
+  using result_type   = size_t;
+
+  _LIBCPP_HIDE_FROM_ABI result_type operator()(const argument_type& __v) const {
+    using __variant_detail::__visitation::__variant;
+    size_t __res =
+        __v.valueless_by_exception()
+            ? 299792458 // Random value chosen by the universe upon creation
+            : __variant::__visit_alt(
+                  [](const auto& __alt) {
+                    using __alt_type   = __remove_cvref_t<decltype(__alt)>;
+                    using __value_type = remove_const_t< typename __alt_type::__value_type>;
+                    return hash<__value_type>{}(__alt.__value);
+                  },
+                  __v);
+    return std::__hash_combine(__res, hash<size_t>{}(__v.index()));
+  }
+};
+
+// __unchecked_get is the same as std::get, except, it is UB to use it with the wrong
+// type whereas std::get will throw or returning nullptr. This makes it faster than
+// std::get.
+template <size_t _Ip, class _Vp>
+_LIBCPP_HIDE_FROM_ABI constexpr auto&& __unchecked_get(_Vp&& __v) noexcept {
+  using __variant_detail::__access::__variant;
+  return __variant::__get_alt<_Ip>(std::forward<_Vp>(__v)).__value;
+}
+
+template <class _Tp, class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr auto&& __unchecked_get(const variant<_Types...>& __v) noexcept {
+  return std::__unchecked_get<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
+}
+
+template <class _Tp, class... _Types>
+_LIBCPP_HIDE_FROM_ABI constexpr auto&& __unchecked_get(variant<_Types...>& __v) noexcept {
+  return std::__unchecked_get<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
+}
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <exception>
+#  include <tuple>
+#  include <type_traits>
+#  include <typeinfo>
+#  include <utility>
+#endif
+
+#endif // _LIBCPP_VARIANT
diff --git a/libcxx/include/__cxx03/vector b/libcxx/include/__cxx03/vector
new file mode 100644
index 000000000000..0e4804a69ff8
--- /dev/null
+++ b/libcxx/include/__cxx03/vector
@@ -0,0 +1,3030 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_VECTOR
+#define _LIBCPP_VECTOR
+
+// clang-format off
+
+/*
+    vector synopsis
+
+namespace std
+{
+
+template <class T, class Allocator = allocator<T> >
+class vector
+{
+public:
+    typedef T                                        value_type;
+    typedef Allocator                                allocator_type;
+    typedef typename allocator_type::reference       reference;
+    typedef typename allocator_type::const_reference const_reference;
+    typedef implementation-defined                   iterator;
+    typedef implementation-defined                   const_iterator;
+    typedef typename allocator_type::size_type       size_type;
+    typedef typename allocator_type::difference_type difference_type;
+    typedef typename allocator_type::pointer         pointer;
+    typedef typename allocator_type::const_pointer   const_pointer;
+    typedef std::reverse_iterator<iterator>          reverse_iterator;
+    typedef std::reverse_iterator<const_iterator>    const_reverse_iterator;
+
+    vector()
+        noexcept(is_nothrow_default_constructible<allocator_type>::value);
+    explicit vector(const allocator_type&);
+    explicit vector(size_type n);
+    explicit vector(size_type n, const allocator_type&); // C++14
+    vector(size_type n, const value_type& value, const allocator_type& = allocator_type());
+    template <class InputIterator>
+        vector(InputIterator first, InputIterator last, const allocator_type& = allocator_type());
+    template<container-compatible-range<T> R>
+      constexpr vector(from_range_t, R&& rg, const Allocator& = Allocator()); // C++23
+    vector(const vector& x);
+    vector(vector&& x)
+        noexcept(is_nothrow_move_constructible<allocator_type>::value);
+    vector(initializer_list<value_type> il);
+    vector(initializer_list<value_type> il, const allocator_type& a);
+    ~vector();
+    vector& operator=(const vector& x);
+    vector& operator=(vector&& x)
+        noexcept(
+             allocator_type::propagate_on_container_move_assignment::value ||
+             allocator_type::is_always_equal::value); // C++17
+    vector& operator=(initializer_list<value_type> il);
+    template <class InputIterator>
+        void assign(InputIterator first, InputIterator last);
+    template<container-compatible-range<T> R>
+      constexpr void assign_range(R&& rg); // C++23
+    void assign(size_type n, const value_type& u);
+    void assign(initializer_list<value_type> il);
+
+    allocator_type get_allocator() const noexcept;
+
+    iterator               begin() noexcept;
+    const_iterator         begin()   const noexcept;
+    iterator               end() noexcept;
+    const_iterator         end()     const noexcept;
+
+    reverse_iterator       rbegin() noexcept;
+    const_reverse_iterator rbegin()  const noexcept;
+    reverse_iterator       rend() noexcept;
+    const_reverse_iterator rend()    const noexcept;
+
+    const_iterator         cbegin()  const noexcept;
+    const_iterator         cend()    const noexcept;
+    const_reverse_iterator crbegin() const noexcept;
+    const_reverse_iterator crend()   const noexcept;
+
+    size_type size() const noexcept;
+    size_type max_size() const noexcept;
+    size_type capacity() const noexcept;
+    bool empty() const noexcept;
+    void reserve(size_type n);
+    void shrink_to_fit() noexcept;
+
+    reference       operator[](size_type n);
+    const_reference operator[](size_type n) const;
+    reference       at(size_type n);
+    const_reference at(size_type n) const;
+
+    reference       front();
+    const_reference front() const;
+    reference       back();
+    const_reference back() const;
+
+    value_type*       data() noexcept;
+    const value_type* data() const noexcept;
+
+    void push_back(const value_type& x);
+    void push_back(value_type&& x);
+    template <class... Args>
+        reference emplace_back(Args&&... args); // reference in C++17
+    template<container-compatible-range<T> R>
+      constexpr void append_range(R&& rg); // C++23
+    void pop_back();
+
+    template <class... Args> iterator emplace(const_iterator position, Args&&... args);
+    iterator insert(const_iterator position, const value_type& x);
+    iterator insert(const_iterator position, value_type&& x);
+    iterator insert(const_iterator position, size_type n, const value_type& x);
+    template <class InputIterator>
+        iterator insert(const_iterator position, InputIterator first, InputIterator last);
+    template<container-compatible-range<T> R>
+      constexpr iterator insert_range(const_iterator position, R&& rg); // C++23
+    iterator insert(const_iterator position, initializer_list<value_type> il);
+
+    iterator erase(const_iterator position);
+    iterator erase(const_iterator first, const_iterator last);
+
+    void clear() noexcept;
+
+    void resize(size_type sz);
+    void resize(size_type sz, const value_type& c);
+
+    void swap(vector&)
+        noexcept(allocator_traits<allocator_type>::propagate_on_container_swap::value ||
+                 allocator_traits<allocator_type>::is_always_equal::value);  // C++17
+
+    bool __invariants() const;
+};
+
+template <class Allocator = allocator<T> >
+class vector<bool, Allocator>
+{
+public:
+    typedef bool                                     value_type;
+    typedef Allocator                                allocator_type;
+    typedef implementation-defined                   iterator;
+    typedef implementation-defined                   const_iterator;
+    typedef typename allocator_type::size_type       size_type;
+    typedef typename allocator_type::difference_type difference_type;
+    typedef iterator                                 pointer;
+    typedef const_iterator                           const_pointer;
+    typedef std::reverse_iterator<iterator>          reverse_iterator;
+    typedef std::reverse_iterator<const_iterator>    const_reverse_iterator;
+
+    class reference
+    {
+    public:
+        reference(const reference&) noexcept;
+        operator bool() const noexcept;
+        reference& operator=(bool x) noexcept;
+        reference& operator=(const reference& x) noexcept;
+        iterator operator&() const noexcept;
+        void flip() noexcept;
+    };
+
+    class const_reference
+    {
+    public:
+        const_reference(const reference&) noexcept;
+        operator bool() const noexcept;
+        const_iterator operator&() const noexcept;
+    };
+
+    vector()
+        noexcept(is_nothrow_default_constructible<allocator_type>::value);
+    explicit vector(const allocator_type&);
+    explicit vector(size_type n, const allocator_type& a = allocator_type()); // C++14
+    vector(size_type n, const value_type& value, const allocator_type& = allocator_type());
+    template <class InputIterator>
+        vector(InputIterator first, InputIterator last, const allocator_type& = allocator_type());
+    template<container-compatible-range<bool> R>
+      constexpr vector(from_range_t, R&& rg, const Allocator& = Allocator());
+    vector(const vector& x);
+    vector(vector&& x)
+        noexcept(is_nothrow_move_constructible<allocator_type>::value);
+    vector(initializer_list<value_type> il);
+    vector(initializer_list<value_type> il, const allocator_type& a);
+    ~vector();
+    vector& operator=(const vector& x);
+    vector& operator=(vector&& x)
+        noexcept(
+             allocator_type::propagate_on_container_move_assignment::value ||
+             allocator_type::is_always_equal::value); // C++17
+    vector& operator=(initializer_list<value_type> il);
+    template <class InputIterator>
+        void assign(InputIterator first, InputIterator last);
+    template<container-compatible-range<T> R>
+      constexpr void assign_range(R&& rg); // C++23
+    void assign(size_type n, const value_type& u);
+    void assign(initializer_list<value_type> il);
+
+    allocator_type get_allocator() const noexcept;
+
+    iterator               begin() noexcept;
+    const_iterator         begin()   const noexcept;
+    iterator               end() noexcept;
+    const_iterator         end()     const noexcept;
+
+    reverse_iterator       rbegin() noexcept;
+    const_reverse_iterator rbegin()  const noexcept;
+    reverse_iterator       rend() noexcept;
+    const_reverse_iterator rend()    const noexcept;
+
+    const_iterator         cbegin()  const noexcept;
+    const_iterator         cend()    const noexcept;
+    const_reverse_iterator crbegin() const noexcept;
+    const_reverse_iterator crend()   const noexcept;
+
+    size_type size() const noexcept;
+    size_type max_size() const noexcept;
+    size_type capacity() const noexcept;
+    bool empty() const noexcept;
+    void reserve(size_type n);
+    void shrink_to_fit() noexcept;
+
+    reference       operator[](size_type n);
+    const_reference operator[](size_type n) const;
+    reference       at(size_type n);
+    const_reference at(size_type n) const;
+
+    reference       front();
+    const_reference front() const;
+    reference       back();
+    const_reference back() const;
+
+    void push_back(const value_type& x);
+    template <class... Args> reference emplace_back(Args&&... args);  // C++14; reference in C++17
+    template<container-compatible-range<T> R>
+      constexpr void append_range(R&& rg); // C++23
+    void pop_back();
+
+    template <class... Args> iterator emplace(const_iterator position, Args&&... args);  // C++14
+    iterator insert(const_iterator position, const value_type& x);
+    iterator insert(const_iterator position, size_type n, const value_type& x);
+    template <class InputIterator>
+        iterator insert(const_iterator position, InputIterator first, InputIterator last);
+    template<container-compatible-range<T> R>
+      constexpr iterator insert_range(const_iterator position, R&& rg); // C++23
+    iterator insert(const_iterator position, initializer_list<value_type> il);
+
+    iterator erase(const_iterator position);
+    iterator erase(const_iterator first, const_iterator last);
+
+    void clear() noexcept;
+
+    void resize(size_type sz);
+    void resize(size_type sz, value_type x);
+
+    void swap(vector&)
+        noexcept(allocator_traits<allocator_type>::propagate_on_container_swap::value ||
+                 allocator_traits<allocator_type>::is_always_equal::value);  // C++17
+    void flip() noexcept;
+
+    bool __invariants() const;
+};
+
+template <class InputIterator, class Allocator = allocator<typename iterator_traits<InputIterator>::value_type>>
+   vector(InputIterator, InputIterator, Allocator = Allocator())
+   -> vector<typename iterator_traits<InputIterator>::value_type, Allocator>; // C++17
+
+template<ranges::input_range R, class Allocator = allocator<ranges::range_value_t<R>>>
+  vector(from_range_t, R&&, Allocator = Allocator())
+    -> vector<ranges::range_value_t<R>, Allocator>; // C++23
+
+template <class Allocator> struct hash<std::vector<bool, Allocator>>;
+
+template <class T, class Allocator> bool operator==(const vector<T,Allocator>& x, const vector<T,Allocator>& y);   // constexpr since C++20
+template <class T, class Allocator> bool operator!=(const vector<T,Allocator>& x, const vector<T,Allocator>& y);   // removed in C++20
+template <class T, class Allocator> bool operator< (const vector<T,Allocator>& x, const vector<T,Allocator>& y);   // removed in C++20
+template <class T, class Allocator> bool operator> (const vector<T,Allocator>& x, const vector<T,Allocator>& y);   // removed in C++20
+template <class T, class Allocator> bool operator>=(const vector<T,Allocator>& x, const vector<T,Allocator>& y);   // removed in C++20
+template <class T, class Allocator> bool operator<=(const vector<T,Allocator>& x, const vector<T,Allocator>& y);   // removed in C++20
+template <class T, class Allocator> constexpr
+  constexpr synth-three-way-result<T> operator<=>(const vector<T, Allocator>& x,
+                                                  const vector<T, Allocator>& y);                                  // since C++20
+
+template <class T, class Allocator>
+void swap(vector<T,Allocator>& x, vector<T,Allocator>& y)
+    noexcept(noexcept(x.swap(y)));
+
+template <class T, class Allocator, class U>
+typename vector<T, Allocator>::size_type
+erase(vector<T, Allocator>& c, const U& value);       // since C++20
+template <class T, class Allocator, class Predicate>
+typename vector<T, Allocator>::size_type
+erase_if(vector<T, Allocator>& c, Predicate pred);    // since C++20
+
+
+template<class T>
+ inline constexpr bool is-vector-bool-reference = see below;        // exposition only, since C++23
+
+template<class T, class charT> requires is-vector-bool-reference<T> // Since C++23
+ struct formatter<T, charT>;
+
+}  // std
+
+*/
+
+// clang-format on
+
+#include <__algorithm/copy.h>
+#include <__algorithm/equal.h>
+#include <__algorithm/fill_n.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/lexicographical_compare.h>
+#include <__algorithm/lexicographical_compare_three_way.h>
+#include <__algorithm/remove.h>
+#include <__algorithm/remove_if.h>
+#include <__algorithm/rotate.h>
+#include <__algorithm/unwrap_iter.h>
+#include <__assert>
+#include <__bit_reference>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__debug_utils/sanitizers.h>
+#include <__format/enable_insertable.h>
+#include <__format/formatter.h>
+#include <__format/formatter_bool.h>
+#include <__functional/hash.h>
+#include <__functional/unary_function.h>
+#include <__fwd/vector.h>
+#include <__iterator/advance.h>
+#include <__iterator/bounded_iter.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/reverse_iterator.h>
+#include <__iterator/wrap_iter.h>
+#include <__memory/addressof.h>
+#include <__memory/allocate_at_least.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/swap_allocator.h>
+#include <__memory/temp_value.h>
+#include <__memory/uninitialized_algorithms.h>
+#include <__memory_resource/polymorphic_allocator.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/container_compatible_range.h>
+#include <__ranges/from_range.h>
+#include <__ranges/size.h>
+#include <__split_buffer>
+#include <__type_traits/is_allocator.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/noexcept_move_assign_container.h>
+#include <__type_traits/type_identity.h>
+#include <__utility/exception_guard.h>
+#include <__utility/forward.h>
+#include <__utility/is_pointer_in_range.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <__utility/swap.h>
+#include <climits>
+#include <cstring>
+#include <limits>
+#include <stdexcept>
+#include <version>
+
+// standard-mandated includes
+
+// [iterator.range]
+#include <__iterator/access.h>
+#include <__iterator/data.h>
+#include <__iterator/empty.h>
+#include <__iterator/reverse_access.h>
+#include <__iterator/size.h>
+
+// [vector.syn]
+#include <compare>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Allocator /* = allocator<_Tp> */>
+class _LIBCPP_TEMPLATE_VIS vector {
+private:
+  typedef allocator<_Tp> __default_allocator_type;
+
+public:
+  typedef vector __self;
+  typedef _Tp value_type;
+  typedef _Allocator allocator_type;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+  typedef typename __alloc_traits::size_type size_type;
+  typedef typename __alloc_traits::difference_type difference_type;
+  typedef typename __alloc_traits::pointer pointer;
+  typedef typename __alloc_traits::const_pointer const_pointer;
+#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
+  // Users might provide custom allocators, and prior to C++20 we have no existing way to detect whether the allocator's
+  // pointer type is contiguous (though it has to be by the Standard). Using the wrapper type ensures the iterator is
+  // considered contiguous.
+  typedef __bounded_iter<__wrap_iter<pointer>> iterator;
+  typedef __bounded_iter<__wrap_iter<const_pointer>> const_iterator;
+#else
+  typedef __wrap_iter<pointer> iterator;
+  typedef __wrap_iter<const_pointer> const_iterator;
+#endif
+  typedef std::reverse_iterator<iterator> reverse_iterator;
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+
+  // A vector containers the following members which may be trivially relocatable:
+  // - pointer: may be trivially relocatable, so it's checked
+  // - allocator_type: may be trivially relocatable, so it's checked
+  // vector doesn't contain any self-references, so it's trivially relocatable if its members are.
+  using __trivially_relocatable = __conditional_t<
+      __libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<allocator_type>::value,
+      vector,
+      void>;
+
+  static_assert(__check_valid_allocator<allocator_type>::value, "");
+  static_assert(is_same<typename allocator_type::value_type, value_type>::value,
+                "Allocator::value_type must be same type as value_type");
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector()
+      _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit vector(const allocator_type& __a)
+#if _LIBCPP_STD_VER <= 14
+      _NOEXCEPT_(is_nothrow_copy_constructible<allocator_type>::value)
+#else
+      _NOEXCEPT
+#endif
+      : __end_cap_(nullptr, __a) {
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit vector(size_type __n) {
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+    if (__n > 0) {
+      __vallocate(__n);
+      __construct_at_end(__n);
+    }
+    __guard.__complete();
+  }
+
+#if _LIBCPP_STD_VER >= 14
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit vector(size_type __n, const allocator_type& __a)
+      : __end_cap_(nullptr, __a) {
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+    if (__n > 0) {
+      __vallocate(__n);
+      __construct_at_end(__n);
+    }
+    __guard.__complete();
+  }
+#endif
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(size_type __n, const value_type& __x) {
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+    if (__n > 0) {
+      __vallocate(__n);
+      __construct_at_end(__n, __x);
+    }
+    __guard.__complete();
+  }
+
+  template <__enable_if_t<__is_allocator<_Allocator>::value, int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+  vector(size_type __n, const value_type& __x, const allocator_type& __a)
+      : __end_cap_(nullptr, __a) {
+    if (__n > 0) {
+      __vallocate(__n);
+      __construct_at_end(__n, __x);
+    }
+  }
+
+  template <class _InputIterator,
+            __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                              is_constructible<value_type, typename iterator_traits<_InputIterator>::reference>::value,
+                          int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(_InputIterator __first, _InputIterator __last);
+  template <class _InputIterator,
+            __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                              is_constructible<value_type, typename iterator_traits<_InputIterator>::reference>::value,
+                          int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+  vector(_InputIterator __first, _InputIterator __last, const allocator_type& __a);
+
+  template <
+      class _ForwardIterator,
+      __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                        is_constructible<value_type, typename iterator_traits<_ForwardIterator>::reference>::value,
+                    int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(_ForwardIterator __first, _ForwardIterator __last);
+
+  template <
+      class _ForwardIterator,
+      __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                        is_constructible<value_type, typename iterator_traits<_ForwardIterator>::reference>::value,
+                    int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+  vector(_ForwardIterator __first, _ForwardIterator __last, const allocator_type& __a);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr vector(
+      from_range_t, _Range&& __range, const allocator_type& __alloc = allocator_type())
+      : __end_cap_(nullptr, __alloc) {
+    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      __init_with_size(ranges::begin(__range), ranges::end(__range), __n);
+
+    } else {
+      __init_with_sentinel(ranges::begin(__range), ranges::end(__range));
+    }
+  }
+#endif
+
+private:
+  class __destroy_vector {
+  public:
+    _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI __destroy_vector(vector& __vec) : __vec_(__vec) {}
+
+    _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void operator()() {
+      if (__vec_.__begin_ != nullptr) {
+        __vec_.__clear();
+        __vec_.__annotate_delete();
+        __alloc_traits::deallocate(__vec_.__alloc(), __vec_.__begin_, __vec_.capacity());
+      }
+    }
+
+  private:
+    vector& __vec_;
+  };
+
+public:
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~vector() { __destroy_vector (*this)(); }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(const vector& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+  vector(const vector& __x, const __type_identity_t<allocator_type>& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector& operator=(const vector& __x);
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(initializer_list<value_type> __il);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+  vector(initializer_list<value_type> __il, const allocator_type& __a);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector& operator=(initializer_list<value_type> __il) {
+    assign(__il.begin(), __il.end());
+    return *this;
+  }
+#endif // !_LIBCPP_CXX03_LANG
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(vector&& __x)
+#if _LIBCPP_STD_VER >= 17
+      noexcept;
+#else
+      _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value);
+#endif
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+  vector(vector&& __x, const __type_identity_t<allocator_type>& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector& operator=(vector&& __x)
+      _NOEXCEPT_(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value);
+
+  template <class _InputIterator,
+            __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                              is_constructible<value_type, typename iterator_traits<_InputIterator>::reference>::value,
+                          int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void assign(_InputIterator __first, _InputIterator __last);
+  template <
+      class _ForwardIterator,
+      __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                        is_constructible<value_type, typename iterator_traits<_ForwardIterator>::reference>::value,
+                    int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void assign(_ForwardIterator __first, _ForwardIterator __last);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr void assign_range(_Range&& __range) {
+    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      __assign_with_size(ranges::begin(__range), ranges::end(__range), __n);
+
+    } else {
+      __assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
+    }
+  }
+#endif
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void assign(size_type __n, const_reference __u);
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void assign(initializer_list<value_type> __il) {
+    assign(__il.begin(), __il.end());
+  }
+#endif
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
+    return this->__alloc();
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT {
+    return reverse_iterator(end());
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT {
+    return const_reverse_iterator(end());
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT {
+    return reverse_iterator(begin());
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
+    return const_reverse_iterator(begin());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT {
+    return rbegin();
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT {
+    return static_cast<size_type>(this->__end_ - this->__begin_);
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const _NOEXCEPT {
+    return static_cast<size_type>(__end_cap() - this->__begin_);
+  }
+  _LIBCPP_NODISCARD _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT {
+    return this->__begin_ == this->__end_;
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void reserve(size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void shrink_to_fit() _NOEXCEPT;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference operator[](size_type __n) _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference operator[](size_type __n) const _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference at(size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference at(size_type __n) const;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference front() _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "front() called on an empty vector");
+    return *this->__begin_;
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference front() const _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "front() called on an empty vector");
+    return *this->__begin_;
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "back() called on an empty vector");
+    return *(this->__end_ - 1);
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "back() called on an empty vector");
+    return *(this->__end_ - 1);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI value_type* data() _NOEXCEPT {
+    return std::__to_address(this->__begin_);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const value_type* data() const _NOEXCEPT {
+    return std::__to_address(this->__begin_);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_back(const_reference __x);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __x);
+
+  template <class... _Args>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+#if _LIBCPP_STD_VER >= 17
+  reference
+  emplace_back(_Args&&... __args);
+#else
+  void
+  emplace_back(_Args&&... __args);
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr void append_range(_Range&& __range) {
+    insert_range(end(), std::forward<_Range>(__range));
+  }
+#endif
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_back();
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, const_reference __x);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, value_type&& __x);
+  template <class... _Args>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __position, _Args&&... __args);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  insert(const_iterator __position, size_type __n, const_reference __x);
+
+  template <class _InputIterator,
+            __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                              is_constructible< value_type, typename iterator_traits<_InputIterator>::reference>::value,
+                          int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  insert(const_iterator __position, _InputIterator __first, _InputIterator __last);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator insert_range(const_iterator __position, _Range&& __range) {
+    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      return __insert_with_size(__position, ranges::begin(__range), ranges::end(__range), __n);
+
+    } else {
+      return __insert_with_sentinel(__position, ranges::begin(__range), ranges::end(__range));
+    }
+  }
+#endif
+
+  template <
+      class _ForwardIterator,
+      __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                        is_constructible< value_type, typename iterator_traits<_ForwardIterator>::reference>::value,
+                    int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  insert(const_iterator __position, _ForwardIterator __first, _ForwardIterator __last);
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  insert(const_iterator __position, initializer_list<value_type> __il) {
+    return insert(__position, __il.begin(), __il.end());
+  }
+#endif
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __position);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT {
+    size_type __old_size = size();
+    __clear();
+    __annotate_shrink(__old_size);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void resize(size_type __sz);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void resize(size_type __sz, const_reference __x);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void swap(vector&)
+#if _LIBCPP_STD_VER >= 14
+      _NOEXCEPT;
+#else
+      _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>);
+#endif
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __invariants() const;
+
+private:
+  pointer __begin_ = nullptr;
+  pointer __end_   = nullptr;
+  __compressed_pair<pointer, allocator_type> __end_cap_ =
+      __compressed_pair<pointer, allocator_type>(nullptr, __default_init_tag());
+
+  //  Allocate space for __n objects
+  //  throws length_error if __n > max_size()
+  //  throws (probably bad_alloc) if memory run out
+  //  Precondition:  __begin_ == __end_ == __end_cap() == 0
+  //  Precondition:  __n > 0
+  //  Postcondition:  capacity() >= __n
+  //  Postcondition:  size() == 0
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __vallocate(size_type __n) {
+    if (__n > max_size())
+      __throw_length_error();
+    auto __allocation = std::__allocate_at_least(__alloc(), __n);
+    __begin_          = __allocation.ptr;
+    __end_            = __allocation.ptr;
+    __end_cap()       = __begin_ + __allocation.count;
+    __annotate_new(0);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __vdeallocate() _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __recommend(size_type __new_size) const;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_at_end(size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_at_end(size_type __n, const_reference __x);
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __init_with_size(_InputIterator __first, _Sentinel __last, size_type __n) {
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+
+    if (__n > 0) {
+      __vallocate(__n);
+      __construct_at_end(__first, __last, __n);
+    }
+
+    __guard.__complete();
+  }
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __init_with_sentinel(_InputIterator __first, _Sentinel __last) {
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+
+    for (; __first != __last; ++__first)
+      emplace_back(*__first);
+
+    __guard.__complete();
+  }
+
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __first, _Sentinel __last);
+
+  template <class _ForwardIterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __assign_with_size(_ForwardIterator __first, _Sentinel __last, difference_type __n);
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  __insert_with_sentinel(const_iterator __position, _InputIterator __first, _Sentinel __last);
+
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  __insert_with_size(const_iterator __position, _Iterator __first, _Sentinel __last, difference_type __n);
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __append(size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __append(size_type __n, const_reference __x);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator __make_iter(pointer __p) _NOEXCEPT {
+#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
+    // Bound the iterator according to the capacity, rather than the size.
+    //
+    // Vector guarantees that iterators stay valid as long as no reallocation occurs even if new elements are inserted
+    // into the container; for these cases, we need to make sure that the newly-inserted elements can be accessed
+    // through the bounded iterator without failing checks. The downside is that the bounded iterator won't catch
+    // access that is logically out-of-bounds, i.e., goes beyond the size, but is still within the capacity. With the
+    // current implementation, there is no connection between a bounded iterator and its associated container, so we
+    // don't have a way to update existing valid iterators when the container is resized and thus have to go with
+    // a laxer approach.
+    return std::__make_bounded_iter(
+        std::__wrap_iter<pointer>(__p),
+        std::__wrap_iter<pointer>(this->__begin_),
+        std::__wrap_iter<pointer>(this->__end_cap()));
+#else
+    return iterator(__p);
+#endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator __make_iter(const_pointer __p) const _NOEXCEPT {
+#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
+    // Bound the iterator according to the capacity, rather than the size.
+    return std::__make_bounded_iter(
+        std::__wrap_iter<const_pointer>(__p),
+        std::__wrap_iter<const_pointer>(this->__begin_),
+        std::__wrap_iter<const_pointer>(this->__end_cap()));
+#else
+    return const_iterator(__p);
+#endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer
+  __swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v, pointer __p);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __move_range(pointer __from_s, pointer __from_e, pointer __to);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign(vector& __c, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign(vector& __c, false_type)
+      _NOEXCEPT_(__alloc_traits::is_always_equal::value);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __destruct_at_end(pointer __new_last) _NOEXCEPT {
+    size_type __old_size = size();
+    __base_destruct_at_end(__new_last);
+    __annotate_shrink(__old_size);
+  }
+
+  template <class _Up>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI inline pointer __push_back_slow_path(_Up&& __x);
+
+  template <class... _Args>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI inline pointer __emplace_back_slow_path(_Args&&... __args);
+
+  // The following functions are no-ops outside of AddressSanitizer mode.
+  // We call annotations for every allocator, unless explicitly disabled.
+  //
+  // To disable annotations for a particular allocator, change value of
+  // __asan_annotate_container_with_allocator to false.
+  // For more details, see the "Using libc++" documentation page or
+  // the documentation for __sanitizer_annotate_contiguous_container.
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __annotate_contiguous_container(const void* __old_mid, const void* __new_mid) const {
+    std::__annotate_contiguous_container<_Allocator>(data(), data() + capacity(), __old_mid, __new_mid);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_new(size_type __current_size) const _NOEXCEPT {
+    (void)__current_size;
+#ifndef _LIBCPP_HAS_NO_ASAN
+    __annotate_contiguous_container(data() + capacity(), data() + __current_size);
+#endif
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_delete() const _NOEXCEPT {
+#ifndef _LIBCPP_HAS_NO_ASAN
+    __annotate_contiguous_container(data() + size(), data() + capacity());
+#endif
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_increase(size_type __n) const _NOEXCEPT {
+    (void)__n;
+#ifndef _LIBCPP_HAS_NO_ASAN
+    __annotate_contiguous_container(data() + size(), data() + size() + __n);
+#endif
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_shrink(size_type __old_size) const _NOEXCEPT {
+    (void)__old_size;
+#ifndef _LIBCPP_HAS_NO_ASAN
+    __annotate_contiguous_container(data() + __old_size, data() + size());
+#endif
+  }
+
+  struct _ConstructTransaction {
+    _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit _ConstructTransaction(vector& __v, size_type __n)
+        : __v_(__v), __pos_(__v.__end_), __new_end_(__v.__end_ + __n) {
+#ifndef _LIBCPP_HAS_NO_ASAN
+      __v_.__annotate_increase(__n);
+#endif
+    }
+
+    _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~_ConstructTransaction() {
+      __v_.__end_ = __pos_;
+#ifndef _LIBCPP_HAS_NO_ASAN
+      if (__pos_ != __new_end_) {
+        __v_.__annotate_shrink(__new_end_ - __v_.__begin_);
+      }
+#endif
+    }
+
+    vector& __v_;
+    pointer __pos_;
+    const_pointer const __new_end_;
+
+    _ConstructTransaction(_ConstructTransaction const&)            = delete;
+    _ConstructTransaction& operator=(_ConstructTransaction const&) = delete;
+  };
+
+  template <class... _Args>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_one_at_end(_Args&&... __args) {
+    _ConstructTransaction __tx(*this, 1);
+    __alloc_traits::construct(this->__alloc(), std::__to_address(__tx.__pos_), std::forward<_Args>(__args)...);
+    ++__tx.__pos_;
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type& __alloc() _NOEXCEPT {
+    return this->__end_cap_.second();
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const allocator_type& __alloc() const _NOEXCEPT {
+    return this->__end_cap_.second();
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer& __end_cap() _NOEXCEPT {
+    return this->__end_cap_.first();
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const pointer& __end_cap() const _NOEXCEPT {
+    return this->__end_cap_.first();
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __clear() _NOEXCEPT {
+    __base_destruct_at_end(this->__begin_);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __base_destruct_at_end(pointer __new_last) _NOEXCEPT {
+    pointer __soon_to_be_end = this->__end_;
+    while (__new_last != __soon_to_be_end)
+      __alloc_traits::destroy(__alloc(), std::__to_address(--__soon_to_be_end));
+    this->__end_ = __new_last;
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const vector& __c) {
+    __copy_assign_alloc(__c, integral_constant<bool, __alloc_traits::propagate_on_container_copy_assignment::value>());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(vector& __c)
+      _NOEXCEPT_(!__alloc_traits::propagate_on_container_move_assignment::value ||
+                 is_nothrow_move_assignable<allocator_type>::value) {
+    __move_assign_alloc(__c, integral_constant<bool, __alloc_traits::propagate_on_container_move_assignment::value>());
+  }
+
+  _LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI void __throw_length_error() const { std::__throw_length_error("vector"); }
+
+  _LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI void __throw_out_of_range() const { std::__throw_out_of_range("vector"); }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const vector& __c, true_type) {
+    if (__alloc() != __c.__alloc()) {
+      __clear();
+      __annotate_delete();
+      __alloc_traits::deallocate(__alloc(), this->__begin_, capacity());
+      this->__begin_ = this->__end_ = __end_cap() = nullptr;
+    }
+    __alloc() = __c.__alloc();
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const vector&, false_type) {}
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(vector& __c, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
+    __alloc() = std::move(__c.__alloc());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(vector&, false_type) _NOEXCEPT {}
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator,
+          class _Alloc = allocator<__iter_value_type<_InputIterator>>,
+          class        = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class        = enable_if_t<__is_allocator<_Alloc>::value> >
+vector(_InputIterator, _InputIterator) -> vector<__iter_value_type<_InputIterator>, _Alloc>;
+
+template <class _InputIterator,
+          class _Alloc,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<__is_allocator<_Alloc>::value> >
+vector(_InputIterator, _InputIterator, _Alloc) -> vector<__iter_value_type<_InputIterator>, _Alloc>;
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range,
+          class _Alloc = allocator<ranges::range_value_t<_Range>>,
+          class        = enable_if_t<__is_allocator<_Alloc>::value> >
+vector(from_range_t, _Range&&, _Alloc = _Alloc()) -> vector<ranges::range_value_t<_Range>, _Alloc>;
+#endif
+
+// __swap_out_circular_buffer relocates the objects in [__begin_, __end_) into the front of __v and swaps the buffers of
+// *this and __v. It is assumed that __v provides space for exactly (__end_ - __begin_) objects in the front. This
+// function has a strong exception guarantee.
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v) {
+  __annotate_delete();
+  auto __new_begin = __v.__begin_ - (__end_ - __begin_);
+  std::__uninitialized_allocator_relocate(
+      __alloc(), std::__to_address(__begin_), std::__to_address(__end_), std::__to_address(__new_begin));
+  __v.__begin_ = __new_begin;
+  __end_       = __begin_; // All the objects have been destroyed by relocating them.
+  std::swap(this->__begin_, __v.__begin_);
+  std::swap(this->__end_, __v.__end_);
+  std::swap(this->__end_cap(), __v.__end_cap());
+  __v.__first_ = __v.__begin_;
+  __annotate_new(size());
+}
+
+// __swap_out_circular_buffer relocates the objects in [__begin_, __p) into the front of __v, the objects in
+// [__p, __end_) into the back of __v and swaps the buffers of *this and __v. It is assumed that __v provides space for
+// exactly (__p - __begin_) objects in the front and space for at least (__end_ - __p) objects in the back. This
+// function has a strong exception guarantee if __begin_ == __p || __end_ == __p.
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::pointer
+vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v, pointer __p) {
+  __annotate_delete();
+  pointer __ret = __v.__begin_;
+
+  // Relocate [__p, __end_) first to avoid having a hole in [__begin_, __end_)
+  // in case something in [__begin_, __p) throws.
+  std::__uninitialized_allocator_relocate(
+      __alloc(), std::__to_address(__p), std::__to_address(__end_), std::__to_address(__v.__end_));
+  __v.__end_ += (__end_ - __p);
+  __end_           = __p; // The objects in [__p, __end_) have been destroyed by relocating them.
+  auto __new_begin = __v.__begin_ - (__p - __begin_);
+
+  std::__uninitialized_allocator_relocate(
+      __alloc(), std::__to_address(__begin_), std::__to_address(__p), std::__to_address(__new_begin));
+  __v.__begin_ = __new_begin;
+  __end_       = __begin_; // All the objects have been destroyed by relocating them.
+
+  std::swap(this->__begin_, __v.__begin_);
+  std::swap(this->__end_, __v.__end_);
+  std::swap(this->__end_cap(), __v.__end_cap());
+  __v.__first_ = __v.__begin_;
+  __annotate_new(size());
+  return __ret;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__vdeallocate() _NOEXCEPT {
+  if (this->__begin_ != nullptr) {
+    clear();
+    __annotate_delete();
+    __alloc_traits::deallocate(this->__alloc(), this->__begin_, capacity());
+    this->__begin_ = this->__end_ = this->__end_cap() = nullptr;
+  }
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::size_type
+vector<_Tp, _Allocator>::max_size() const _NOEXCEPT {
+  return std::min<size_type>(__alloc_traits::max_size(this->__alloc()), numeric_limits<difference_type>::max());
+}
+
+//  Precondition:  __new_size > capacity()
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::size_type
+vector<_Tp, _Allocator>::__recommend(size_type __new_size) const {
+  const size_type __ms = max_size();
+  if (__new_size > __ms)
+    this->__throw_length_error();
+  const size_type __cap = capacity();
+  if (__cap >= __ms / 2)
+    return __ms;
+  return std::max<size_type>(2 * __cap, __new_size);
+}
+
+//  Default constructs __n objects starting at __end_
+//  throws if construction throws
+//  Precondition:  __n > 0
+//  Precondition:  size() + __n <= capacity()
+//  Postcondition:  size() == size() + __n
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__construct_at_end(size_type __n) {
+  _ConstructTransaction __tx(*this, __n);
+  const_pointer __new_end = __tx.__new_end_;
+  for (pointer __pos = __tx.__pos_; __pos != __new_end; __tx.__pos_ = ++__pos) {
+    __alloc_traits::construct(this->__alloc(), std::__to_address(__pos));
+  }
+}
+
+//  Copy constructs __n objects starting at __end_ from __x
+//  throws if construction throws
+//  Precondition:  __n > 0
+//  Precondition:  size() + __n <= capacity()
+//  Postcondition:  size() == old size() + __n
+//  Postcondition:  [i] == __x for all i in [size() - __n, __n)
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void
+vector<_Tp, _Allocator>::__construct_at_end(size_type __n, const_reference __x) {
+  _ConstructTransaction __tx(*this, __n);
+  const_pointer __new_end = __tx.__new_end_;
+  for (pointer __pos = __tx.__pos_; __pos != __new_end; __tx.__pos_ = ++__pos) {
+    __alloc_traits::construct(this->__alloc(), std::__to_address(__pos), __x);
+  }
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+vector<_Tp, _Allocator>::__construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n) {
+  _ConstructTransaction __tx(*this, __n);
+  __tx.__pos_ = std::__uninitialized_allocator_copy(__alloc(), __first, __last, __tx.__pos_);
+}
+
+//  Default constructs __n objects starting at __end_
+//  throws if construction throws
+//  Postcondition:  size() == size() + __n
+//  Exception safety: strong.
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__append(size_type __n) {
+  if (static_cast<size_type>(this->__end_cap() - this->__end_) >= __n)
+    this->__construct_at_end(__n);
+  else {
+    allocator_type& __a = this->__alloc();
+    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), size(), __a);
+    __v.__construct_at_end(__n);
+    __swap_out_circular_buffer(__v);
+  }
+}
+
+//  Default constructs __n objects starting at __end_
+//  throws if construction throws
+//  Postcondition:  size() == size() + __n
+//  Exception safety: strong.
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__append(size_type __n, const_reference __x) {
+  if (static_cast<size_type>(this->__end_cap() - this->__end_) >= __n)
+    this->__construct_at_end(__n, __x);
+  else {
+    allocator_type& __a = this->__alloc();
+    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), size(), __a);
+    __v.__construct_at_end(__n, __x);
+    __swap_out_circular_buffer(__v);
+  }
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIterator,
+          __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_InputIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<_Tp, _Allocator>::vector(_InputIterator __first, _InputIterator __last) {
+  __init_with_sentinel(__first, __last);
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIterator,
+          __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_InputIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<_Tp, _Allocator>::vector(_InputIterator __first, _InputIterator __last, const allocator_type& __a)
+    : __end_cap_(nullptr, __a) {
+  __init_with_sentinel(__first, __last);
+}
+
+template <class _Tp, class _Allocator>
+template <class _ForwardIterator,
+          __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_ForwardIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<_Tp, _Allocator>::vector(_ForwardIterator __first, _ForwardIterator __last) {
+  size_type __n = static_cast<size_type>(std::distance(__first, __last));
+  __init_with_size(__first, __last, __n);
+}
+
+template <class _Tp, class _Allocator>
+template <class _ForwardIterator,
+          __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_ForwardIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<_Tp, _Allocator>::vector(_ForwardIterator __first, _ForwardIterator __last, const allocator_type& __a)
+    : __end_cap_(nullptr, __a) {
+  size_type __n = static_cast<size_type>(std::distance(__first, __last));
+  __init_with_size(__first, __last, __n);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<_Tp, _Allocator>::vector(const vector& __x)
+    : __end_cap_(nullptr, __alloc_traits::select_on_container_copy_construction(__x.__alloc())) {
+  __init_with_size(__x.__begin_, __x.__end_, __x.size());
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<_Tp, _Allocator>::vector(const vector& __x, const __type_identity_t<allocator_type>& __a)
+    : __end_cap_(nullptr, __a) {
+  __init_with_size(__x.__begin_, __x.__end_, __x.size());
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI vector<_Tp, _Allocator>::vector(vector&& __x)
+#if _LIBCPP_STD_VER >= 17
+    noexcept
+#else
+    _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value)
+#endif
+    : __end_cap_(nullptr, std::move(__x.__alloc())) {
+  this->__begin_    = __x.__begin_;
+  this->__end_      = __x.__end_;
+  this->__end_cap() = __x.__end_cap();
+  __x.__begin_ = __x.__end_ = __x.__end_cap() = nullptr;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI
+vector<_Tp, _Allocator>::vector(vector&& __x, const __type_identity_t<allocator_type>& __a)
+    : __end_cap_(nullptr, __a) {
+  if (__a == __x.__alloc()) {
+    this->__begin_    = __x.__begin_;
+    this->__end_      = __x.__end_;
+    this->__end_cap() = __x.__end_cap();
+    __x.__begin_ = __x.__end_ = __x.__end_cap() = nullptr;
+  } else {
+    typedef move_iterator<iterator> _Ip;
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+    assign(_Ip(__x.begin()), _Ip(__x.end()));
+    __guard.__complete();
+  }
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI
+vector<_Tp, _Allocator>::vector(initializer_list<value_type> __il) {
+  auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+  if (__il.size() > 0) {
+    __vallocate(__il.size());
+    __construct_at_end(__il.begin(), __il.end(), __il.size());
+  }
+  __guard.__complete();
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI
+vector<_Tp, _Allocator>::vector(initializer_list<value_type> __il, const allocator_type& __a)
+    : __end_cap_(nullptr, __a) {
+  auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+  if (__il.size() > 0) {
+    __vallocate(__il.size());
+    __construct_at_end(__il.begin(), __il.end(), __il.size());
+  }
+  __guard.__complete();
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI vector<_Tp, _Allocator>&
+vector<_Tp, _Allocator>::operator=(vector&& __x)
+    _NOEXCEPT_(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value) {
+  __move_assign(__x, integral_constant<bool, __alloc_traits::propagate_on_container_move_assignment::value>());
+  return *this;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__move_assign(vector& __c, false_type)
+    _NOEXCEPT_(__alloc_traits::is_always_equal::value) {
+  if (__alloc() != __c.__alloc()) {
+    typedef move_iterator<iterator> _Ip;
+    assign(_Ip(__c.begin()), _Ip(__c.end()));
+  } else
+    __move_assign(__c, true_type());
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__move_assign(vector& __c, true_type)
+    _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
+  __vdeallocate();
+  __move_assign_alloc(__c); // this can throw
+  this->__begin_    = __c.__begin_;
+  this->__end_      = __c.__end_;
+  this->__end_cap() = __c.__end_cap();
+  __c.__begin_ = __c.__end_ = __c.__end_cap() = nullptr;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI vector<_Tp, _Allocator>&
+vector<_Tp, _Allocator>::operator=(const vector& __x) {
+  if (this != std::addressof(__x)) {
+    __copy_assign_alloc(__x);
+    assign(__x.__begin_, __x.__end_);
+  }
+  return *this;
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIterator,
+          __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_InputIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::assign(_InputIterator __first, _InputIterator __last) {
+  __assign_with_sentinel(__first, __last);
+}
+
+template <class _Tp, class _Allocator>
+template <class _Iterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+vector<_Tp, _Allocator>::__assign_with_sentinel(_Iterator __first, _Sentinel __last) {
+  clear();
+  for (; __first != __last; ++__first)
+    emplace_back(*__first);
+}
+
+template <class _Tp, class _Allocator>
+template <class _ForwardIterator,
+          __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_ForwardIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::assign(_ForwardIterator __first, _ForwardIterator __last) {
+  __assign_with_size(__first, __last, std::distance(__first, __last));
+}
+
+template <class _Tp, class _Allocator>
+template <class _ForwardIterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+vector<_Tp, _Allocator>::__assign_with_size(_ForwardIterator __first, _Sentinel __last, difference_type __n) {
+  size_type __new_size = static_cast<size_type>(__n);
+  if (__new_size <= capacity()) {
+    if (__new_size > size()) {
+      _ForwardIterator __mid = std::next(__first, size());
+      std::copy(__first, __mid, this->__begin_);
+      __construct_at_end(__mid, __last, __new_size - size());
+    } else {
+      pointer __m = std::__copy<_ClassicAlgPolicy>(__first, __last, this->__begin_).second;
+      this->__destruct_at_end(__m);
+    }
+  } else {
+    __vdeallocate();
+    __vallocate(__recommend(__new_size));
+    __construct_at_end(__first, __last, __new_size);
+  }
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::assign(size_type __n, const_reference __u) {
+  if (__n <= capacity()) {
+    size_type __s = size();
+    std::fill_n(this->__begin_, std::min(__n, __s), __u);
+    if (__n > __s)
+      __construct_at_end(__n - __s, __u);
+    else
+      this->__destruct_at_end(this->__begin_ + __n);
+  } else {
+    __vdeallocate();
+    __vallocate(__recommend(static_cast<size_type>(__n)));
+    __construct_at_end(__n, __u);
+  }
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::begin() _NOEXCEPT {
+  return __make_iter(this->__begin_);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::const_iterator
+vector<_Tp, _Allocator>::begin() const _NOEXCEPT {
+  return __make_iter(this->__begin_);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::end() _NOEXCEPT {
+  return __make_iter(this->__end_);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::const_iterator
+vector<_Tp, _Allocator>::end() const _NOEXCEPT {
+  return __make_iter(this->__end_);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::reference
+vector<_Tp, _Allocator>::operator[](size_type __n) _NOEXCEPT {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < size(), "vector[] index out of bounds");
+  return this->__begin_[__n];
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::const_reference
+vector<_Tp, _Allocator>::operator[](size_type __n) const _NOEXCEPT {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < size(), "vector[] index out of bounds");
+  return this->__begin_[__n];
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::reference vector<_Tp, _Allocator>::at(size_type __n) {
+  if (__n >= size())
+    this->__throw_out_of_range();
+  return this->__begin_[__n];
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::const_reference
+vector<_Tp, _Allocator>::at(size_type __n) const {
+  if (__n >= size())
+    this->__throw_out_of_range();
+  return this->__begin_[__n];
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::reserve(size_type __n) {
+  if (__n > capacity()) {
+    if (__n > max_size())
+      this->__throw_length_error();
+    allocator_type& __a = this->__alloc();
+    __split_buffer<value_type, allocator_type&> __v(__n, size(), __a);
+    __swap_out_circular_buffer(__v);
+  }
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::shrink_to_fit() _NOEXCEPT {
+  if (capacity() > size()) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      allocator_type& __a = this->__alloc();
+      __split_buffer<value_type, allocator_type&> __v(size(), size(), __a);
+      // The Standard mandates shrink_to_fit() does not increase the capacity.
+      // With equal capacity keep the existing buffer. This avoids extra work
+      // due to swapping the elements.
+      if (__v.capacity() < capacity())
+        __swap_out_circular_buffer(__v);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+}
+
+template <class _Tp, class _Allocator>
+template <class _Up>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::pointer
+vector<_Tp, _Allocator>::__push_back_slow_path(_Up&& __x) {
+  allocator_type& __a = this->__alloc();
+  __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), size(), __a);
+  // __v.push_back(std::forward<_Up>(__x));
+  __alloc_traits::construct(__a, std::__to_address(__v.__end_), std::forward<_Up>(__x));
+  __v.__end_++;
+  __swap_out_circular_buffer(__v);
+  return this->__end_;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void
+vector<_Tp, _Allocator>::push_back(const_reference __x) {
+  pointer __end = this->__end_;
+  if (__end < this->__end_cap()) {
+    __construct_one_at_end(__x);
+    ++__end;
+  } else {
+    __end = __push_back_slow_path(__x);
+  }
+  this->__end_ = __end;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void vector<_Tp, _Allocator>::push_back(value_type&& __x) {
+  pointer __end = this->__end_;
+  if (__end < this->__end_cap()) {
+    __construct_one_at_end(std::move(__x));
+    ++__end;
+  } else {
+    __end = __push_back_slow_path(std::move(__x));
+  }
+  this->__end_ = __end;
+}
+
+template <class _Tp, class _Allocator>
+template <class... _Args>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::pointer
+vector<_Tp, _Allocator>::__emplace_back_slow_path(_Args&&... __args) {
+  allocator_type& __a = this->__alloc();
+  __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), size(), __a);
+  //    __v.emplace_back(std::forward<_Args>(__args)...);
+  __alloc_traits::construct(__a, std::__to_address(__v.__end_), std::forward<_Args>(__args)...);
+  __v.__end_++;
+  __swap_out_circular_buffer(__v);
+  return this->__end_;
+}
+
+template <class _Tp, class _Allocator>
+template <class... _Args>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline
+#if _LIBCPP_STD_VER >= 17
+    typename vector<_Tp, _Allocator>::reference
+#else
+    void
+#endif
+    vector<_Tp, _Allocator>::emplace_back(_Args&&... __args) {
+  pointer __end = this->__end_;
+  if (__end < this->__end_cap()) {
+    __construct_one_at_end(std::forward<_Args>(__args)...);
+    ++__end;
+  } else {
+    __end = __emplace_back_slow_path(std::forward<_Args>(__args)...);
+  }
+  this->__end_ = __end;
+#if _LIBCPP_STD_VER >= 17
+  return *(__end - 1);
+#endif
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void vector<_Tp, _Allocator>::pop_back() {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "vector::pop_back called on an empty vector");
+  this->__destruct_at_end(this->__end_ - 1);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::erase(const_iterator __position) {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+      __position != end(), "vector::erase(iterator) called with a non-dereferenceable iterator");
+  difference_type __ps = __position - cbegin();
+  pointer __p          = this->__begin_ + __ps;
+  this->__destruct_at_end(std::move(__p + 1, this->__end_, __p));
+  return __make_iter(__p);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::erase(const_iterator __first, const_iterator __last) {
+  _LIBCPP_ASSERT_VALID_INPUT_RANGE(__first <= __last, "vector::erase(first, last) called with invalid range");
+  pointer __p = this->__begin_ + (__first - begin());
+  if (__first != __last) {
+    this->__destruct_at_end(std::move(__p + (__last - __first), this->__end_, __p));
+  }
+  return __make_iter(__p);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+vector<_Tp, _Allocator>::__move_range(pointer __from_s, pointer __from_e, pointer __to) {
+  pointer __old_last  = this->__end_;
+  difference_type __n = __old_last - __to;
+  {
+    pointer __i = __from_s + __n;
+    _ConstructTransaction __tx(*this, __from_e - __i);
+    for (pointer __pos = __tx.__pos_; __i < __from_e; ++__i, (void)++__pos, __tx.__pos_ = __pos) {
+      __alloc_traits::construct(this->__alloc(), std::__to_address(__pos), std::move(*__i));
+    }
+  }
+  std::move_backward(__from_s, __from_s + __n, __old_last);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::insert(const_iterator __position, const_reference __x) {
+  pointer __p = this->__begin_ + (__position - begin());
+  if (this->__end_ < this->__end_cap()) {
+    if (__p == this->__end_) {
+      __construct_one_at_end(__x);
+    } else {
+      __move_range(__p, this->__end_, __p + 1);
+      const_pointer __xr = pointer_traits<const_pointer>::pointer_to(__x);
+      if (std::__is_pointer_in_range(std::__to_address(__p), std::__to_address(__end_), std::addressof(__x)))
+        ++__xr;
+      *__p = *__xr;
+    }
+  } else {
+    allocator_type& __a = this->__alloc();
+    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, __a);
+    __v.push_back(__x);
+    __p = __swap_out_circular_buffer(__v, __p);
+  }
+  return __make_iter(__p);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::insert(const_iterator __position, value_type&& __x) {
+  pointer __p = this->__begin_ + (__position - begin());
+  if (this->__end_ < this->__end_cap()) {
+    if (__p == this->__end_) {
+      __construct_one_at_end(std::move(__x));
+    } else {
+      __move_range(__p, this->__end_, __p + 1);
+      *__p = std::move(__x);
+    }
+  } else {
+    allocator_type& __a = this->__alloc();
+    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, __a);
+    __v.push_back(std::move(__x));
+    __p = __swap_out_circular_buffer(__v, __p);
+  }
+  return __make_iter(__p);
+}
+
+template <class _Tp, class _Allocator>
+template <class... _Args>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::emplace(const_iterator __position, _Args&&... __args) {
+  pointer __p = this->__begin_ + (__position - begin());
+  if (this->__end_ < this->__end_cap()) {
+    if (__p == this->__end_) {
+      __construct_one_at_end(std::forward<_Args>(__args)...);
+    } else {
+      __temp_value<value_type, _Allocator> __tmp(this->__alloc(), std::forward<_Args>(__args)...);
+      __move_range(__p, this->__end_, __p + 1);
+      *__p = std::move(__tmp.get());
+    }
+  } else {
+    allocator_type& __a = this->__alloc();
+    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, __a);
+    __v.emplace_back(std::forward<_Args>(__args)...);
+    __p = __swap_out_circular_buffer(__v, __p);
+  }
+  return __make_iter(__p);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::insert(const_iterator __position, size_type __n, const_reference __x) {
+  pointer __p = this->__begin_ + (__position - begin());
+  if (__n > 0) {
+    // We can't compare unrelated pointers inside constant expressions
+    if (!__libcpp_is_constant_evaluated() && __n <= static_cast<size_type>(this->__end_cap() - this->__end_)) {
+      size_type __old_n  = __n;
+      pointer __old_last = this->__end_;
+      if (__n > static_cast<size_type>(this->__end_ - __p)) {
+        size_type __cx = __n - (this->__end_ - __p);
+        __construct_at_end(__cx, __x);
+        __n -= __cx;
+      }
+      if (__n > 0) {
+        __move_range(__p, __old_last, __p + __old_n);
+        const_pointer __xr = pointer_traits<const_pointer>::pointer_to(__x);
+        if (__p <= __xr && __xr < this->__end_)
+          __xr += __old_n;
+        std::fill_n(__p, __n, *__xr);
+      }
+    } else {
+      allocator_type& __a = this->__alloc();
+      __split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), __p - this->__begin_, __a);
+      __v.__construct_at_end(__n, __x);
+      __p = __swap_out_circular_buffer(__v, __p);
+    }
+  }
+  return __make_iter(__p);
+}
+template <class _Tp, class _Allocator>
+template <class _InputIterator,
+          __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_InputIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::insert(const_iterator __position, _InputIterator __first, _InputIterator __last) {
+  return __insert_with_sentinel(__position, __first, __last);
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::__insert_with_sentinel(const_iterator __position, _InputIterator __first, _Sentinel __last) {
+  difference_type __off = __position - begin();
+  pointer __p           = this->__begin_ + __off;
+  allocator_type& __a   = this->__alloc();
+  pointer __old_last    = this->__end_;
+  for (; this->__end_ != this->__end_cap() && __first != __last; ++__first) {
+    __construct_one_at_end(*__first);
+  }
+  __split_buffer<value_type, allocator_type&> __v(__a);
+  if (__first != __last) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      __v.__construct_at_end_with_sentinel(std::move(__first), std::move(__last));
+      difference_type __old_size = __old_last - this->__begin_;
+      difference_type __old_p    = __p - this->__begin_;
+      reserve(__recommend(size() + __v.size()));
+      __p        = this->__begin_ + __old_p;
+      __old_last = this->__begin_ + __old_size;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      erase(__make_iter(__old_last), end());
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+  __p = std::rotate(__p, __old_last, this->__end_);
+  insert(__make_iter(__p), std::make_move_iterator(__v.begin()), std::make_move_iterator(__v.end()));
+  return begin() + __off;
+}
+
+template <class _Tp, class _Allocator>
+template <class _ForwardIterator,
+          __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_ForwardIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::insert(const_iterator __position, _ForwardIterator __first, _ForwardIterator __last) {
+  return __insert_with_size(__position, __first, __last, std::distance(__first, __last));
+}
+
+template <class _Tp, class _Allocator>
+template <class _Iterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::__insert_with_size(
+    const_iterator __position, _Iterator __first, _Sentinel __last, difference_type __n) {
+  auto __insertion_size = __n;
+  pointer __p           = this->__begin_ + (__position - begin());
+  if (__n > 0) {
+    if (__n <= this->__end_cap() - this->__end_) {
+      size_type __old_n    = __n;
+      pointer __old_last   = this->__end_;
+      _Iterator __m        = std::next(__first, __n);
+      difference_type __dx = this->__end_ - __p;
+      if (__n > __dx) {
+        __m                    = __first;
+        difference_type __diff = this->__end_ - __p;
+        std::advance(__m, __diff);
+        __construct_at_end(__m, __last, __n - __diff);
+        __n = __dx;
+      }
+      if (__n > 0) {
+        __move_range(__p, __old_last, __p + __old_n);
+        std::copy(__first, __m, __p);
+      }
+    } else {
+      allocator_type& __a = this->__alloc();
+      __split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), __p - this->__begin_, __a);
+      __v.__construct_at_end_with_size(__first, __insertion_size);
+      __p = __swap_out_circular_buffer(__v, __p);
+    }
+  }
+  return __make_iter(__p);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::resize(size_type __sz) {
+  size_type __cs = size();
+  if (__cs < __sz)
+    this->__append(__sz - __cs);
+  else if (__cs > __sz)
+    this->__destruct_at_end(this->__begin_ + __sz);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::resize(size_type __sz, const_reference __x) {
+  size_type __cs = size();
+  if (__cs < __sz)
+    this->__append(__sz - __cs, __x);
+  else if (__cs > __sz)
+    this->__destruct_at_end(this->__begin_ + __sz);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::swap(vector& __x)
+#if _LIBCPP_STD_VER >= 14
+    _NOEXCEPT
+#else
+    _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>)
+#endif
+{
+  _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+      __alloc_traits::propagate_on_container_swap::value || this->__alloc() == __x.__alloc(),
+      "vector::swap: Either propagate_on_container_swap must be true"
+      " or the allocators must compare equal");
+  std::swap(this->__begin_, __x.__begin_);
+  std::swap(this->__end_, __x.__end_);
+  std::swap(this->__end_cap(), __x.__end_cap());
+  std::__swap_allocator(
+      this->__alloc(), __x.__alloc(), integral_constant<bool, __alloc_traits::propagate_on_container_swap::value>());
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 bool vector<_Tp, _Allocator>::__invariants() const {
+  if (this->__begin_ == nullptr) {
+    if (this->__end_ != nullptr || this->__end_cap() != nullptr)
+      return false;
+  } else {
+    if (this->__begin_ > this->__end_)
+      return false;
+    if (this->__begin_ == this->__end_cap())
+      return false;
+    if (this->__end_ > this->__end_cap())
+      return false;
+  }
+  return true;
+}
+
+// vector<bool>
+
+template <class _Allocator>
+class vector<bool, _Allocator>;
+
+template <class _Allocator>
+struct hash<vector<bool, _Allocator> >;
+
+template <class _Allocator>
+struct __has_storage_type<vector<bool, _Allocator> > {
+  static const bool value = true;
+};
+
+template <class _Allocator>
+class _LIBCPP_TEMPLATE_VIS vector<bool, _Allocator> {
+public:
+  typedef vector __self;
+  typedef bool value_type;
+  typedef _Allocator allocator_type;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+  typedef typename __alloc_traits::size_type size_type;
+  typedef typename __alloc_traits::difference_type difference_type;
+  typedef size_type __storage_type;
+  typedef __bit_iterator<vector, false> pointer;
+  typedef __bit_iterator<vector, true> const_pointer;
+  typedef pointer iterator;
+  typedef const_pointer const_iterator;
+  typedef std::reverse_iterator<iterator> reverse_iterator;
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+
+private:
+  typedef __rebind_alloc<__alloc_traits, __storage_type> __storage_allocator;
+  typedef allocator_traits<__storage_allocator> __storage_traits;
+  typedef typename __storage_traits::pointer __storage_pointer;
+  typedef typename __storage_traits::const_pointer __const_storage_pointer;
+
+  __storage_pointer __begin_;
+  size_type __size_;
+  __compressed_pair<size_type, __storage_allocator> __cap_alloc_;
+
+public:
+  typedef __bit_reference<vector> reference;
+#ifdef _LIBCPP_ABI_BITSET_VECTOR_BOOL_CONST_SUBSCRIPT_RETURN_BOOL
+  using const_reference = bool;
+#else
+  typedef __bit_const_reference<vector> const_reference;
+#endif
+
+private:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type& __cap() _NOEXCEPT { return __cap_alloc_.first(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const size_type& __cap() const _NOEXCEPT {
+    return __cap_alloc_.first();
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __storage_allocator& __alloc() _NOEXCEPT {
+    return __cap_alloc_.second();
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const __storage_allocator& __alloc() const _NOEXCEPT {
+    return __cap_alloc_.second();
+  }
+
+  static const unsigned __bits_per_word = static_cast<unsigned>(sizeof(__storage_type) * CHAR_BIT);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static size_type
+  __internal_cap_to_external(size_type __n) _NOEXCEPT {
+    return __n * __bits_per_word;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static size_type
+  __external_cap_to_internal(size_type __n) _NOEXCEPT {
+    return (__n - 1) / __bits_per_word + 1;
+  }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector()
+      _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit vector(const allocator_type& __a)
+#if _LIBCPP_STD_VER <= 14
+      _NOEXCEPT_(is_nothrow_copy_constructible<allocator_type>::value);
+#else
+      _NOEXCEPT;
+#endif
+
+private:
+  class __destroy_vector {
+  public:
+    _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI __destroy_vector(vector& __vec) : __vec_(__vec) {}
+
+    _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void operator()() {
+      if (__vec_.__begin_ != nullptr)
+        __storage_traits::deallocate(__vec_.__alloc(), __vec_.__begin_, __vec_.__cap());
+    }
+
+  private:
+    vector& __vec_;
+  };
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~vector() { __destroy_vector (*this)(); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit vector(size_type __n);
+#if _LIBCPP_STD_VER >= 14
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit vector(size_type __n, const allocator_type& __a);
+#endif
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(size_type __n, const value_type& __v);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  vector(size_type __n, const value_type& __v, const allocator_type& __a);
+  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(_InputIterator __first, _InputIterator __last);
+  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  vector(_InputIterator __first, _InputIterator __last, const allocator_type& __a);
+  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  vector(_ForwardIterator __first, _ForwardIterator __last, const allocator_type& __a);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<bool> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr vector(from_range_t, _Range&& __range, const allocator_type& __a = allocator_type())
+      : __begin_(nullptr), __size_(0), __cap_alloc_(0, static_cast<__storage_allocator>(__a)) {
+    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      __init_with_size(ranges::begin(__range), ranges::end(__range), __n);
+
+    } else {
+      __init_with_sentinel(ranges::begin(__range), ranges::end(__range));
+    }
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(const vector& __v);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(const vector& __v, const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector& operator=(const vector& __v);
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(initializer_list<value_type> __il);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  vector(initializer_list<value_type> __il, const allocator_type& __a);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector& operator=(initializer_list<value_type> __il) {
+    assign(__il.begin(), __il.end());
+    return *this;
+  }
+
+#endif // !_LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(vector&& __v)
+#if _LIBCPP_STD_VER >= 17
+      noexcept;
+#else
+      _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value);
+#endif
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  vector(vector&& __v, const __type_identity_t<allocator_type>& __a);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector& operator=(vector&& __v)
+      _NOEXCEPT_(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value);
+
+  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 assign(_InputIterator __first, _InputIterator __last);
+  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 assign(_ForwardIterator __first, _ForwardIterator __last);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<bool> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr void assign_range(_Range&& __range) {
+    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      __assign_with_size(ranges::begin(__range), ranges::end(__range), __n);
+
+    } else {
+      __assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
+    }
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void assign(size_type __n, const value_type& __x);
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void assign(initializer_list<value_type> __il) {
+    assign(__il.begin(), __il.end());
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator_type get_allocator() const _NOEXCEPT {
+    return allocator_type(this->__alloc());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type max_size() const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type capacity() const _NOEXCEPT {
+    return __internal_cap_to_external(__cap());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type size() const _NOEXCEPT { return __size_; }
+  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool empty() const _NOEXCEPT {
+    return __size_ == 0;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void reserve(size_type __n);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void shrink_to_fit() _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator begin() _NOEXCEPT { return __make_iter(0); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator begin() const _NOEXCEPT { return __make_iter(0); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator end() _NOEXCEPT { return __make_iter(__size_); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator end() const _NOEXCEPT {
+    return __make_iter(__size_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rbegin() _NOEXCEPT {
+    return reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator rbegin() const _NOEXCEPT {
+    return const_reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rend() _NOEXCEPT {
+    return reverse_iterator(begin());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator rend() const _NOEXCEPT {
+    return const_reverse_iterator(begin());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cbegin() const _NOEXCEPT { return __make_iter(0); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cend() const _NOEXCEPT {
+    return __make_iter(__size_);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator crbegin() const _NOEXCEPT {
+    return rbegin();
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference operator[](size_type __n) { return __make_ref(__n); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference operator[](size_type __n) const {
+    return __make_ref(__n);
+  }
+  _LIBCPP_HIDE_FROM_ABI reference at(size_type __n);
+  _LIBCPP_HIDE_FROM_ABI const_reference at(size_type __n) const;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference front() { return __make_ref(0); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference front() const { return __make_ref(0); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference back() { return __make_ref(__size_ - 1); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference back() const { return __make_ref(__size_ - 1); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void push_back(const value_type& __x);
+#if _LIBCPP_STD_VER >= 14
+  template <class... _Args>
+#  if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference emplace_back(_Args&&... __args)
+#  else
+  _LIBCPP_HIDE_FROM_ABI void emplace_back(_Args&&... __args)
+#  endif
+  {
+    push_back(value_type(std::forward<_Args>(__args)...));
+#  if _LIBCPP_STD_VER >= 17
+    return this->back();
+#  endif
+  }
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<bool> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr void append_range(_Range&& __range) {
+    insert_range(end(), std::forward<_Range>(__range));
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void pop_back() { --__size_; }
+
+#if _LIBCPP_STD_VER >= 14
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator emplace(const_iterator __position, _Args&&... __args) {
+    return insert(__position, value_type(std::forward<_Args>(__args)...));
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator insert(const_iterator __position, const value_type& __x);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
+  insert(const_iterator __position, size_type __n, const value_type& __x);
+  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
+  iterator _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  insert(const_iterator __position, _InputIterator __first, _InputIterator __last);
+  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+  iterator _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  insert(const_iterator __position, _ForwardIterator __first, _ForwardIterator __last);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<bool> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator insert_range(const_iterator __position, _Range&& __range) {
+    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      return __insert_with_size(__position, ranges::begin(__range), ranges::end(__range), __n);
+
+    } else {
+      return __insert_with_sentinel(__position, ranges::begin(__range), ranges::end(__range));
+    }
+  }
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
+  insert(const_iterator __position, initializer_list<value_type> __il) {
+    return insert(__position, __il.begin(), __il.end());
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator erase(const_iterator __position);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator erase(const_iterator __first, const_iterator __last);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void clear() _NOEXCEPT { __size_ = 0; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(vector&)
+#if _LIBCPP_STD_VER >= 14
+      _NOEXCEPT;
+#else
+      _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>);
+#endif
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void swap(reference __x, reference __y) _NOEXCEPT {
+    std::swap(__x, __y);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void resize(size_type __sz, value_type __x = false);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void flip() _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __invariants() const;
+
+private:
+  _LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI void __throw_length_error() const { std::__throw_length_error("vector"); }
+
+  _LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI void __throw_out_of_range() const { std::__throw_out_of_range("vector"); }
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+  __init_with_size(_InputIterator __first, _Sentinel __last, size_type __n) {
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+
+    if (__n > 0) {
+      __vallocate(__n);
+      __construct_at_end(std::move(__first), std::move(__last), __n);
+    }
+
+    __guard.__complete();
+  }
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+  __init_with_sentinel(_InputIterator __first, _Sentinel __last) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      for (; __first != __last; ++__first)
+        push_back(*__first);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      if (__begin_ != nullptr)
+        __storage_traits::deallocate(__alloc(), __begin_, __cap());
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __first, _Sentinel __last);
+
+  template <class _ForwardIterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __assign_with_size(_ForwardIterator __first, _Sentinel __last, difference_type __ns);
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  __insert_with_sentinel(const_iterator __position, _InputIterator __first, _Sentinel __last);
+
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  __insert_with_size(const_iterator __position, _Iterator __first, _Sentinel __last, difference_type __n);
+
+  //  Allocate space for __n objects
+  //  throws length_error if __n > max_size()
+  //  throws (probably bad_alloc) if memory run out
+  //  Precondition:  __begin_ == __end_ == __cap() == 0
+  //  Precondition:  __n > 0
+  //  Postcondition:  capacity() >= __n
+  //  Postcondition:  size() == 0
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __vallocate(size_type __n) {
+    if (__n > max_size())
+      __throw_length_error();
+    auto __allocation = std::__allocate_at_least(__alloc(), __external_cap_to_internal(__n));
+    __begin_          = __allocation.ptr;
+    __size_           = 0;
+    __cap()           = __allocation.count;
+    if (__libcpp_is_constant_evaluated()) {
+      for (size_type __i = 0; __i != __cap(); ++__i)
+        std::__construct_at(std::__to_address(__begin_) + __i);
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __vdeallocate() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static size_type __align_it(size_type __new_size) _NOEXCEPT {
+    return (__new_size + (__bits_per_word - 1)) & ~((size_type)__bits_per_word - 1);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type __recommend(size_type __new_size) const;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __construct_at_end(size_type __n, bool __x);
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+  __construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __append(size_type __n, const_reference __x);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference __make_ref(size_type __pos) _NOEXCEPT {
+    return reference(__begin_ + __pos / __bits_per_word, __storage_type(1) << __pos % __bits_per_word);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference __make_ref(size_type __pos) const _NOEXCEPT {
+    return __bit_const_reference<vector>(
+        __begin_ + __pos / __bits_per_word, __storage_type(1) << __pos % __bits_per_word);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator __make_iter(size_type __pos) _NOEXCEPT {
+    return iterator(__begin_ + __pos / __bits_per_word, static_cast<unsigned>(__pos % __bits_per_word));
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator __make_iter(size_type __pos) const _NOEXCEPT {
+    return const_iterator(__begin_ + __pos / __bits_per_word, static_cast<unsigned>(__pos % __bits_per_word));
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator __const_iterator_cast(const_iterator __p) _NOEXCEPT {
+    return begin() + (__p - cbegin());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __copy_assign_alloc(const vector& __v) {
+    __copy_assign_alloc(
+        __v, integral_constant<bool, __storage_traits::propagate_on_container_copy_assignment::value>());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __copy_assign_alloc(const vector& __c, true_type) {
+    if (__alloc() != __c.__alloc())
+      __vdeallocate();
+    __alloc() = __c.__alloc();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __copy_assign_alloc(const vector&, false_type) {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign(vector& __c, false_type);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign(vector& __c, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign_alloc(vector& __c)
+      _NOEXCEPT_(!__storage_traits::propagate_on_container_move_assignment::value ||
+                 is_nothrow_move_assignable<allocator_type>::value) {
+    __move_assign_alloc(
+        __c, integral_constant<bool, __storage_traits::propagate_on_container_move_assignment::value>());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign_alloc(vector& __c, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
+    __alloc() = std::move(__c.__alloc());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign_alloc(vector&, false_type) _NOEXCEPT {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_t __hash_code() const _NOEXCEPT;
+
+  friend class __bit_reference<vector>;
+  friend class __bit_const_reference<vector>;
+  friend class __bit_iterator<vector, false>;
+  friend class __bit_iterator<vector, true>;
+  friend struct __bit_array<vector>;
+  friend struct _LIBCPP_TEMPLATE_VIS hash<vector>;
+};
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::__vdeallocate() _NOEXCEPT {
+  if (this->__begin_ != nullptr) {
+    __storage_traits::deallocate(this->__alloc(), this->__begin_, __cap());
+    this->__begin_ = nullptr;
+    this->__size_ = this->__cap() = 0;
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::size_type
+vector<bool, _Allocator>::max_size() const _NOEXCEPT {
+  size_type __amax = __storage_traits::max_size(__alloc());
+  size_type __nmax = numeric_limits<size_type>::max() / 2; // end() >= begin(), always
+  if (__nmax / __bits_per_word <= __amax)
+    return __nmax;
+  return __internal_cap_to_external(__amax);
+}
+
+//  Precondition:  __new_size > capacity()
+template <class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::size_type
+vector<bool, _Allocator>::__recommend(size_type __new_size) const {
+  const size_type __ms = max_size();
+  if (__new_size > __ms)
+    this->__throw_length_error();
+  const size_type __cap = capacity();
+  if (__cap >= __ms / 2)
+    return __ms;
+  return std::max(2 * __cap, __align_it(__new_size));
+}
+
+//  Default constructs __n objects starting at __end_
+//  Precondition:  __n > 0
+//  Precondition:  size() + __n <= capacity()
+//  Postcondition:  size() == size() + __n
+template <class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+vector<bool, _Allocator>::__construct_at_end(size_type __n, bool __x) {
+  size_type __old_size = this->__size_;
+  this->__size_ += __n;
+  if (__old_size == 0 || ((__old_size - 1) / __bits_per_word) != ((this->__size_ - 1) / __bits_per_word)) {
+    if (this->__size_ <= __bits_per_word)
+      this->__begin_[0] = __storage_type(0);
+    else
+      this->__begin_[(this->__size_ - 1) / __bits_per_word] = __storage_type(0);
+  }
+  std::fill_n(__make_iter(__old_size), __n, __x);
+}
+
+template <class _Allocator>
+template <class _InputIterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+vector<bool, _Allocator>::__construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n) {
+  size_type __old_size = this->__size_;
+  this->__size_ += __n;
+  if (__old_size == 0 || ((__old_size - 1) / __bits_per_word) != ((this->__size_ - 1) / __bits_per_word)) {
+    if (this->__size_ <= __bits_per_word)
+      this->__begin_[0] = __storage_type(0);
+    else
+      this->__begin_[(this->__size_ - 1) / __bits_per_word] = __storage_type(0);
+  }
+  std::__copy<_ClassicAlgPolicy>(__first, __last, __make_iter(__old_size));
+}
+
+template <class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector()
+    _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
+    : __begin_(nullptr), __size_(0), __cap_alloc_(0, __default_init_tag()) {}
+
+template <class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const allocator_type& __a)
+#if _LIBCPP_STD_VER <= 14
+    _NOEXCEPT_(is_nothrow_copy_constructible<allocator_type>::value)
+#else
+        _NOEXCEPT
+#endif
+    : __begin_(nullptr), __size_(0), __cap_alloc_(0, static_cast<__storage_allocator>(__a)) {
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(size_type __n)
+    : __begin_(nullptr), __size_(0), __cap_alloc_(0, __default_init_tag()) {
+  if (__n > 0) {
+    __vallocate(__n);
+    __construct_at_end(__n, false);
+  }
+}
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(size_type __n, const allocator_type& __a)
+    : __begin_(nullptr), __size_(0), __cap_alloc_(0, static_cast<__storage_allocator>(__a)) {
+  if (__n > 0) {
+    __vallocate(__n);
+    __construct_at_end(__n, false);
+  }
+}
+#endif
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(size_type __n, const value_type& __x)
+    : __begin_(nullptr), __size_(0), __cap_alloc_(0, __default_init_tag()) {
+  if (__n > 0) {
+    __vallocate(__n);
+    __construct_at_end(__n, __x);
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<bool, _Allocator>::vector(size_type __n, const value_type& __x, const allocator_type& __a)
+    : __begin_(nullptr), __size_(0), __cap_alloc_(0, static_cast<__storage_allocator>(__a)) {
+  if (__n > 0) {
+    __vallocate(__n);
+    __construct_at_end(__n, __x);
+  }
+}
+
+template <class _Allocator>
+template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(_InputIterator __first, _InputIterator __last)
+    : __begin_(nullptr), __size_(0), __cap_alloc_(0, __default_init_tag()) {
+  __init_with_sentinel(__first, __last);
+}
+
+template <class _Allocator>
+template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<bool, _Allocator>::vector(_InputIterator __first, _InputIterator __last, const allocator_type& __a)
+    : __begin_(nullptr), __size_(0), __cap_alloc_(0, static_cast<__storage_allocator>(__a)) {
+  __init_with_sentinel(__first, __last);
+}
+
+template <class _Allocator>
+template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(_ForwardIterator __first, _ForwardIterator __last)
+    : __begin_(nullptr), __size_(0), __cap_alloc_(0, __default_init_tag()) {
+  auto __n = static_cast<size_type>(std::distance(__first, __last));
+  __init_with_size(__first, __last, __n);
+}
+
+template <class _Allocator>
+template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<bool, _Allocator>::vector(_ForwardIterator __first, _ForwardIterator __last, const allocator_type& __a)
+    : __begin_(nullptr), __size_(0), __cap_alloc_(0, static_cast<__storage_allocator>(__a)) {
+  auto __n = static_cast<size_type>(std::distance(__first, __last));
+  __init_with_size(__first, __last, __n);
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(initializer_list<value_type> __il)
+    : __begin_(nullptr), __size_(0), __cap_alloc_(0, __default_init_tag()) {
+  size_type __n = static_cast<size_type>(__il.size());
+  if (__n > 0) {
+    __vallocate(__n);
+    __construct_at_end(__il.begin(), __il.end(), __n);
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<bool, _Allocator>::vector(initializer_list<value_type> __il, const allocator_type& __a)
+    : __begin_(nullptr), __size_(0), __cap_alloc_(0, static_cast<__storage_allocator>(__a)) {
+  size_type __n = static_cast<size_type>(__il.size());
+  if (__n > 0) {
+    __vallocate(__n);
+    __construct_at_end(__il.begin(), __il.end(), __n);
+  }
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const vector& __v)
+    : __begin_(nullptr),
+      __size_(0),
+      __cap_alloc_(0, __storage_traits::select_on_container_copy_construction(__v.__alloc())) {
+  if (__v.size() > 0) {
+    __vallocate(__v.size());
+    __construct_at_end(__v.begin(), __v.end(), __v.size());
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const vector& __v, const allocator_type& __a)
+    : __begin_(nullptr), __size_(0), __cap_alloc_(0, __a) {
+  if (__v.size() > 0) {
+    __vallocate(__v.size());
+    __construct_at_end(__v.begin(), __v.end(), __v.size());
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>& vector<bool, _Allocator>::operator=(const vector& __v) {
+  if (this != std::addressof(__v)) {
+    __copy_assign_alloc(__v);
+    if (__v.__size_) {
+      if (__v.__size_ > capacity()) {
+        __vdeallocate();
+        __vallocate(__v.__size_);
+      }
+      std::copy(__v.__begin_, __v.__begin_ + __external_cap_to_internal(__v.__size_), __begin_);
+    }
+    __size_ = __v.__size_;
+  }
+  return *this;
+}
+
+template <class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(vector&& __v)
+#if _LIBCPP_STD_VER >= 17
+    _NOEXCEPT
+#else
+    _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value)
+#endif
+    : __begin_(__v.__begin_),
+      __size_(__v.__size_),
+      __cap_alloc_(std::move(__v.__cap_alloc_)) {
+  __v.__begin_ = nullptr;
+  __v.__size_  = 0;
+  __v.__cap()  = 0;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<bool, _Allocator>::vector(vector&& __v, const __type_identity_t<allocator_type>& __a)
+    : __begin_(nullptr), __size_(0), __cap_alloc_(0, __a) {
+  if (__a == allocator_type(__v.__alloc())) {
+    this->__begin_ = __v.__begin_;
+    this->__size_  = __v.__size_;
+    this->__cap()  = __v.__cap();
+    __v.__begin_   = nullptr;
+    __v.__cap() = __v.__size_ = 0;
+  } else if (__v.size() > 0) {
+    __vallocate(__v.size());
+    __construct_at_end(__v.begin(), __v.end(), __v.size());
+  }
+}
+
+template <class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>&
+vector<bool, _Allocator>::operator=(vector&& __v)
+    _NOEXCEPT_(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value) {
+  __move_assign(__v, integral_constant<bool, __storage_traits::propagate_on_container_move_assignment::value>());
+  return *this;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::__move_assign(vector& __c, false_type) {
+  if (__alloc() != __c.__alloc())
+    assign(__c.begin(), __c.end());
+  else
+    __move_assign(__c, true_type());
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::__move_assign(vector& __c, true_type)
+    _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
+  __vdeallocate();
+  __move_assign_alloc(__c);
+  this->__begin_ = __c.__begin_;
+  this->__size_  = __c.__size_;
+  this->__cap()  = __c.__cap();
+  __c.__begin_   = nullptr;
+  __c.__cap() = __c.__size_ = 0;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::assign(size_type __n, const value_type& __x) {
+  __size_ = 0;
+  if (__n > 0) {
+    size_type __c = capacity();
+    if (__n <= __c)
+      __size_ = __n;
+    else {
+      vector __v(get_allocator());
+      __v.reserve(__recommend(__n));
+      __v.__size_ = __n;
+      swap(__v);
+    }
+    std::fill_n(begin(), __n, __x);
+  }
+}
+
+template <class _Allocator>
+template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::assign(_InputIterator __first, _InputIterator __last) {
+  __assign_with_sentinel(__first, __last);
+}
+
+template <class _Allocator>
+template <class _Iterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+vector<bool, _Allocator>::__assign_with_sentinel(_Iterator __first, _Sentinel __last) {
+  clear();
+  for (; __first != __last; ++__first)
+    push_back(*__first);
+}
+
+template <class _Allocator>
+template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::assign(_ForwardIterator __first, _ForwardIterator __last) {
+  __assign_with_size(__first, __last, std::distance(__first, __last));
+}
+
+template <class _Allocator>
+template <class _ForwardIterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+vector<bool, _Allocator>::__assign_with_size(_ForwardIterator __first, _Sentinel __last, difference_type __ns) {
+  _LIBCPP_ASSERT_VALID_INPUT_RANGE(__ns >= 0, "invalid range specified");
+
+  clear();
+
+  const size_t __n = static_cast<size_type>(__ns);
+  if (__n) {
+    if (__n > capacity()) {
+      __vdeallocate();
+      __vallocate(__n);
+    }
+    __construct_at_end(__first, __last, __n);
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::reserve(size_type __n) {
+  if (__n > capacity()) {
+    if (__n > max_size())
+      this->__throw_length_error();
+    vector __v(this->get_allocator());
+    __v.__vallocate(__n);
+    __v.__construct_at_end(this->begin(), this->end(), this->size());
+    swap(__v);
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::shrink_to_fit() _NOEXCEPT {
+  if (__external_cap_to_internal(size()) > __cap()) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      vector(*this, allocator_type(__alloc())).swap(*this);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+}
+
+template <class _Allocator>
+typename vector<bool, _Allocator>::reference vector<bool, _Allocator>::at(size_type __n) {
+  if (__n >= size())
+    this->__throw_out_of_range();
+  return (*this)[__n];
+}
+
+template <class _Allocator>
+typename vector<bool, _Allocator>::const_reference vector<bool, _Allocator>::at(size_type __n) const {
+  if (__n >= size())
+    this->__throw_out_of_range();
+  return (*this)[__n];
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::push_back(const value_type& __x) {
+  if (this->__size_ == this->capacity())
+    reserve(__recommend(this->__size_ + 1));
+  ++this->__size_;
+  back() = __x;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::insert(const_iterator __position, const value_type& __x) {
+  iterator __r;
+  if (size() < capacity()) {
+    const_iterator __old_end = end();
+    ++__size_;
+    std::copy_backward(__position, __old_end, end());
+    __r = __const_iterator_cast(__position);
+  } else {
+    vector __v(get_allocator());
+    __v.reserve(__recommend(__size_ + 1));
+    __v.__size_ = __size_ + 1;
+    __r         = std::copy(cbegin(), __position, __v.begin());
+    std::copy_backward(__position, cend(), __v.end());
+    swap(__v);
+  }
+  *__r = __x;
+  return __r;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::insert(const_iterator __position, size_type __n, const value_type& __x) {
+  iterator __r;
+  size_type __c = capacity();
+  if (__n <= __c && size() <= __c - __n) {
+    const_iterator __old_end = end();
+    __size_ += __n;
+    std::copy_backward(__position, __old_end, end());
+    __r = __const_iterator_cast(__position);
+  } else {
+    vector __v(get_allocator());
+    __v.reserve(__recommend(__size_ + __n));
+    __v.__size_ = __size_ + __n;
+    __r         = std::copy(cbegin(), __position, __v.begin());
+    std::copy_backward(__position, cend(), __v.end());
+    swap(__v);
+  }
+  std::fill_n(__r, __n, __x);
+  return __r;
+}
+
+template <class _Allocator>
+template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::insert(const_iterator __position, _InputIterator __first, _InputIterator __last) {
+  return __insert_with_sentinel(__position, __first, __last);
+}
+
+template <class _Allocator>
+template <class _InputIterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::__insert_with_sentinel(const_iterator __position, _InputIterator __first, _Sentinel __last) {
+  difference_type __off = __position - begin();
+  iterator __p          = __const_iterator_cast(__position);
+  iterator __old_end    = end();
+  for (; size() != capacity() && __first != __last; ++__first) {
+    ++this->__size_;
+    back() = *__first;
+  }
+  vector __v(get_allocator());
+  if (__first != __last) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      __v.__assign_with_sentinel(std::move(__first), std::move(__last));
+      difference_type __old_size = static_cast<difference_type>(__old_end - begin());
+      difference_type __old_p    = __p - begin();
+      reserve(__recommend(size() + __v.size()));
+      __p       = begin() + __old_p;
+      __old_end = begin() + __old_size;
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      erase(__old_end, end());
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  }
+  __p = std::rotate(__p, __old_end, end());
+  insert(__p, __v.begin(), __v.end());
+  return begin() + __off;
+}
+
+template <class _Allocator>
+template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::insert(const_iterator __position, _ForwardIterator __first, _ForwardIterator __last) {
+  return __insert_with_size(__position, __first, __last, std::distance(__first, __last));
+}
+
+template <class _Allocator>
+template <class _ForwardIterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::__insert_with_size(
+    const_iterator __position, _ForwardIterator __first, _Sentinel __last, difference_type __n_signed) {
+  _LIBCPP_ASSERT_VALID_INPUT_RANGE(__n_signed >= 0, "invalid range specified");
+  const size_type __n = static_cast<size_type>(__n_signed);
+  iterator __r;
+  size_type __c = capacity();
+  if (__n <= __c && size() <= __c - __n) {
+    const_iterator __old_end = end();
+    __size_ += __n;
+    std::copy_backward(__position, __old_end, end());
+    __r = __const_iterator_cast(__position);
+  } else {
+    vector __v(get_allocator());
+    __v.reserve(__recommend(__size_ + __n));
+    __v.__size_ = __size_ + __n;
+    __r         = std::copy(cbegin(), __position, __v.begin());
+    std::copy_backward(__position, cend(), __v.end());
+    swap(__v);
+  }
+  std::__copy<_ClassicAlgPolicy>(__first, __last, __r);
+  return __r;
+}
+
+template <class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::erase(const_iterator __position) {
+  iterator __r = __const_iterator_cast(__position);
+  std::copy(__position + 1, this->cend(), __r);
+  --__size_;
+  return __r;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::erase(const_iterator __first, const_iterator __last) {
+  iterator __r        = __const_iterator_cast(__first);
+  difference_type __d = __last - __first;
+  std::copy(__last, this->cend(), __r);
+  __size_ -= __d;
+  return __r;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::swap(vector& __x)
+#if _LIBCPP_STD_VER >= 14
+    _NOEXCEPT
+#else
+    _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>)
+#endif
+{
+  std::swap(this->__begin_, __x.__begin_);
+  std::swap(this->__size_, __x.__size_);
+  std::swap(this->__cap(), __x.__cap());
+  std::__swap_allocator(
+      this->__alloc(), __x.__alloc(), integral_constant<bool, __alloc_traits::propagate_on_container_swap::value>());
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::resize(size_type __sz, value_type __x) {
+  size_type __cs = size();
+  if (__cs < __sz) {
+    iterator __r;
+    size_type __c = capacity();
+    size_type __n = __sz - __cs;
+    if (__n <= __c && __cs <= __c - __n) {
+      __r = end();
+      __size_ += __n;
+    } else {
+      vector __v(get_allocator());
+      __v.reserve(__recommend(__size_ + __n));
+      __v.__size_ = __size_ + __n;
+      __r         = std::copy(cbegin(), cend(), __v.begin());
+      swap(__v);
+    }
+    std::fill_n(__r, __n, __x);
+  } else
+    __size_ = __sz;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::flip() _NOEXCEPT {
+  // do middle whole words
+  size_type __n         = __size_;
+  __storage_pointer __p = __begin_;
+  for (; __n >= __bits_per_word; ++__p, __n -= __bits_per_word)
+    *__p = ~*__p;
+  // do last partial word
+  if (__n > 0) {
+    __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+    __storage_type __b = *__p & __m;
+    *__p &= ~__m;
+    *__p |= ~__b & __m;
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 bool vector<bool, _Allocator>::__invariants() const {
+  if (this->__begin_ == nullptr) {
+    if (this->__size_ != 0 || this->__cap() != 0)
+      return false;
+  } else {
+    if (this->__cap() == 0)
+      return false;
+    if (this->__size_ > this->capacity())
+      return false;
+  }
+  return true;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 size_t vector<bool, _Allocator>::__hash_code() const _NOEXCEPT {
+  size_t __h = 0;
+  // do middle whole words
+  size_type __n         = __size_;
+  __storage_pointer __p = __begin_;
+  for (; __n >= __bits_per_word; ++__p, __n -= __bits_per_word)
+    __h ^= *__p;
+  // do last partial word
+  if (__n > 0) {
+    const __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+    __h ^= *__p & __m;
+  }
+  return __h;
+}
+
+template <class _Allocator>
+struct _LIBCPP_TEMPLATE_VIS hash<vector<bool, _Allocator> >
+    : public __unary_function<vector<bool, _Allocator>, size_t> {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_t
+  operator()(const vector<bool, _Allocator>& __vec) const _NOEXCEPT {
+    return __vec.__hash_code();
+  }
+};
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
+  const typename vector<_Tp, _Allocator>::size_type __sz = __x.size();
+  return __sz == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
+  return !(__x == __y);
+}
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
+  return std::lexicographical_compare(__x.begin(), __x.end(), __y.begin(), __y.end());
+}
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
+  return __y < __x;
+}
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>=(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
+  return !(__x < __y);
+}
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
+  return !(__y < __x);
+}
+
+#else // _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI constexpr __synth_three_way_result<_Tp>
+operator<=>(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
+  return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
+}
+
+#endif // _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void
+swap(vector<_Tp, _Allocator>& __x, vector<_Tp, _Allocator>& __y) _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp, class _Allocator, class _Up>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::size_type
+erase(vector<_Tp, _Allocator>& __c, const _Up& __v) {
+  auto __old_size = __c.size();
+  __c.erase(std::remove(__c.begin(), __c.end(), __v), __c.end());
+  return __old_size - __c.size();
+}
+
+template <class _Tp, class _Allocator, class _Predicate>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::size_type
+erase_if(vector<_Tp, _Allocator>& __c, _Predicate __pred) {
+  auto __old_size = __c.size();
+  __c.erase(std::remove_if(__c.begin(), __c.end(), __pred), __c.end());
+  return __old_size - __c.size();
+}
+
+template <>
+inline constexpr bool __format::__enable_insertable<vector<char>> = true;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+inline constexpr bool __format::__enable_insertable<vector<wchar_t>> = true;
+#  endif
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#if _LIBCPP_STD_VER >= 23
+template <class _Tp, class _CharT>
+// Since is-vector-bool-reference is only used once it's inlined here.
+  requires same_as<typename _Tp::__container, vector<bool, typename _Tp::__container::allocator_type>>
+struct _LIBCPP_TEMPLATE_VIS formatter<_Tp, _CharT> {
+private:
+  formatter<bool, _CharT> __underlying_;
+
+public:
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return __underlying_.parse(__ctx);
+  }
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(const _Tp& __ref, _FormatContext& __ctx) const {
+    return __underlying_.format(__ref, __ctx);
+  }
+};
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 17
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace pmr {
+template <class _ValueT>
+using vector _LIBCPP_AVAILABILITY_PMR = std::vector<_ValueT, polymorphic_allocator<_ValueT>>;
+} // namespace pmr
+_LIBCPP_END_NAMESPACE_STD
+#endif
+
+_LIBCPP_POP_MACROS
+
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <algorithm>
+#  include <atomic>
+#  include <concepts>
+#  include <cstdlib>
+#  include <iosfwd>
+#  if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#    include <locale>
+#  endif
+#  include <tuple>
+#  include <type_traits>
+#  include <typeinfo>
+#  include <utility>
+#endif
+
+#endif // _LIBCPP_VECTOR
diff --git a/libcxx/include/__cxx03/version b/libcxx/include/__cxx03/version
new file mode 100644
index 000000000000..fe64343eafbc
--- /dev/null
+++ b/libcxx/include/__cxx03/version
@@ -0,0 +1,566 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_VERSIONH
+#define _LIBCPP_VERSIONH
+
+/*
+  version synopsis
+
+Macro name                                              Value   Headers
+__cpp_lib_adaptor_iterator_pair_constructor             202106L <queue> <stack>
+__cpp_lib_addressof_constexpr                           201603L <memory>
+__cpp_lib_allocate_at_least                             202302L <memory>
+__cpp_lib_allocator_traits_is_always_equal              201411L <deque> <forward_list> <list>
+                                                                <map> <memory> <scoped_allocator>
+                                                                <set> <string> <unordered_map>
+                                                                <unordered_set> <vector>
+__cpp_lib_any                                           201606L <any>
+__cpp_lib_apply                                         201603L <tuple>
+__cpp_lib_array_constexpr                               201811L <array> <iterator>
+                                                        201603L // C++17
+__cpp_lib_as_const                                      201510L <utility>
+__cpp_lib_associative_heterogeneous_erasure             202110L <map> <set> <unordered_map>
+                                                                <unordered_set>
+__cpp_lib_associative_heterogeneous_insertion           202306L <map> <set> <unordered_map>
+                                                                <unordered_set>
+__cpp_lib_assume_aligned                                201811L <memory>
+__cpp_lib_atomic_flag_test                              201907L <atomic>
+__cpp_lib_atomic_float                                  201711L <atomic>
+__cpp_lib_atomic_is_always_lock_free                    201603L <atomic>
+__cpp_lib_atomic_lock_free_type_aliases                 201907L <atomic>
+__cpp_lib_atomic_min_max                                202403L <atomic>
+__cpp_lib_atomic_ref                                    201806L <atomic>
+__cpp_lib_atomic_shared_ptr                             201711L <atomic>
+__cpp_lib_atomic_value_initialization                   201911L <atomic> <memory>
+__cpp_lib_atomic_wait                                   201907L <atomic>
+__cpp_lib_barrier                                       201907L <barrier>
+__cpp_lib_bind_back                                     202202L <functional>
+__cpp_lib_bind_front                                    202306L <functional>
+                                                        201907L // C++20
+__cpp_lib_bit_cast                                      201806L <bit>
+__cpp_lib_bitops                                        201907L <bit>
+__cpp_lib_bitset                                        202306L <bitset>
+__cpp_lib_bool_constant                                 201505L <type_traits>
+__cpp_lib_bounded_array_traits                          201902L <type_traits>
+__cpp_lib_boyer_moore_searcher                          201603L <functional>
+__cpp_lib_byte                                          201603L <cstddef>
+__cpp_lib_byteswap                                      202110L <bit>
+__cpp_lib_char8_t                                       201907L <atomic> <filesystem> <istream>
+                                                                <limits> <locale> <ostream>
+                                                                <string> <string_view>
+__cpp_lib_chrono                                        201611L <chrono>
+__cpp_lib_chrono_udls                                   201304L <chrono>
+__cpp_lib_clamp                                         201603L <algorithm>
+__cpp_lib_complex_udls                                  201309L <complex>
+__cpp_lib_concepts                                      202002L <concepts>
+__cpp_lib_constexpr_algorithms                          201806L <algorithm> <utility>
+__cpp_lib_constexpr_bitset                              202207L <bitset>
+__cpp_lib_constexpr_charconv                            202207L <charconv>
+__cpp_lib_constexpr_cmath                               202202L <cmath> <cstdlib>
+__cpp_lib_constexpr_complex                             201711L <complex>
+__cpp_lib_constexpr_dynamic_alloc                       201907L <memory>
+__cpp_lib_constexpr_functional                          201907L <functional>
+__cpp_lib_constexpr_iterator                            201811L <iterator>
+__cpp_lib_constexpr_memory                              202202L <memory>
+                                                        201811L // C++20
+__cpp_lib_constexpr_new                                 202406L <new>
+__cpp_lib_constexpr_numeric                             201911L <numeric>
+__cpp_lib_constexpr_string                              201907L <string>
+__cpp_lib_constexpr_string_view                         201811L <string_view>
+__cpp_lib_constexpr_tuple                               201811L <tuple>
+__cpp_lib_constexpr_typeinfo                            202106L <typeinfo>
+__cpp_lib_constexpr_utility                             201811L <utility>
+__cpp_lib_constexpr_vector                              201907L <vector>
+__cpp_lib_constrained_equality                          202403L <optional> <tuple> <utility>
+                                                                <variant>
+__cpp_lib_containers_ranges                             202202L <deque> <forward_list> <list>
+                                                                <map> <queue> <set>
+                                                                <stack> <string> <unordered_map>
+                                                                <unordered_set> <vector>
+__cpp_lib_copyable_function                             202306L <functional>
+__cpp_lib_coroutine                                     201902L <coroutine>
+__cpp_lib_debugging                                     202311L <debugging>
+__cpp_lib_default_template_type_for_algorithm_values    202403L <algorithm> <deque> <forward_list>
+                                                                <list> <ranges> <string>
+                                                                <vector>
+__cpp_lib_destroying_delete                             201806L <new>
+__cpp_lib_enable_shared_from_this                       201603L <memory>
+__cpp_lib_endian                                        201907L <bit>
+__cpp_lib_erase_if                                      202002L <deque> <forward_list> <list>
+                                                                <map> <set> <string>
+                                                                <unordered_map> <unordered_set> <vector>
+__cpp_lib_exchange_function                             201304L <utility>
+__cpp_lib_execution                                     201902L <execution>
+                                                        201603L // C++17
+__cpp_lib_expected                                      202211L <expected>
+__cpp_lib_filesystem                                    201703L <filesystem>
+__cpp_lib_format                                        202110L <format>
+__cpp_lib_format_path                                   202403L <filesystem>
+__cpp_lib_format_ranges                                 202207L <format>
+__cpp_lib_format_uchar                                  202311L <format>
+__cpp_lib_formatters                                    202302L <stacktrace> <thread>
+__cpp_lib_forward_like                                  202207L <utility>
+__cpp_lib_freestanding_algorithm                        202311L <algorithm>
+__cpp_lib_freestanding_array                            202311L <array>
+__cpp_lib_freestanding_cstring                          202306L <cstring>
+__cpp_lib_freestanding_expected                         202311L <expected>
+__cpp_lib_freestanding_mdspan                           202311L <mdspan>
+__cpp_lib_freestanding_optional                         202311L <optional>
+__cpp_lib_freestanding_string_view                      202311L <string_view>
+__cpp_lib_freestanding_variant                          202311L <variant>
+__cpp_lib_fstream_native_handle                         202306L <fstream>
+__cpp_lib_function_ref                                  202306L <functional>
+__cpp_lib_gcd_lcm                                       201606L <numeric>
+__cpp_lib_generate_random                               202403L <random>
+__cpp_lib_generic_associative_lookup                    201304L <map> <set>
+__cpp_lib_generic_unordered_lookup                      201811L <unordered_map> <unordered_set>
+__cpp_lib_hardware_interference_size                    201703L <new>
+__cpp_lib_has_unique_object_representations             201606L <type_traits>
+__cpp_lib_hazard_pointer                                202306L <hazard_pointer>
+__cpp_lib_hypot                                         201603L <cmath>
+__cpp_lib_incomplete_container_elements                 201505L <forward_list> <list> <vector>
+__cpp_lib_inplace_vector                                202406L <inplace_vector>
+__cpp_lib_int_pow2                                      202002L <bit>
+__cpp_lib_integer_comparison_functions                  202002L <utility>
+__cpp_lib_integer_sequence                              201304L <utility>
+__cpp_lib_integral_constant_callable                    201304L <type_traits>
+__cpp_lib_interpolate                                   201902L <cmath> <numeric>
+__cpp_lib_invoke                                        201411L <functional>
+__cpp_lib_invoke_r                                      202106L <functional>
+__cpp_lib_ios_noreplace                                 202207L <ios>
+__cpp_lib_is_aggregate                                  201703L <type_traits>
+__cpp_lib_is_constant_evaluated                         201811L <type_traits>
+__cpp_lib_is_final                                      201402L <type_traits>
+__cpp_lib_is_invocable                                  201703L <type_traits>
+__cpp_lib_is_layout_compatible                          201907L <type_traits>
+__cpp_lib_is_nothrow_convertible                        201806L <type_traits>
+__cpp_lib_is_null_pointer                               201309L <type_traits>
+__cpp_lib_is_pointer_interconvertible                   201907L <type_traits>
+__cpp_lib_is_scoped_enum                                202011L <type_traits>
+__cpp_lib_is_swappable                                  201603L <type_traits>
+__cpp_lib_is_virtual_base_of                            202406L <type_traits>
+__cpp_lib_is_within_lifetime                            202306L <type_traits>
+__cpp_lib_jthread                                       201911L <stop_token> <thread>
+__cpp_lib_latch                                         201907L <latch>
+__cpp_lib_launder                                       201606L <new>
+__cpp_lib_linalg                                        202311L <linalg>
+__cpp_lib_list_remove_return_type                       201806L <forward_list> <list>
+__cpp_lib_logical_traits                                201510L <type_traits>
+__cpp_lib_make_from_tuple                               201606L <tuple>
+__cpp_lib_make_reverse_iterator                         201402L <iterator>
+__cpp_lib_make_unique                                   201304L <memory>
+__cpp_lib_map_try_emplace                               201411L <map>
+__cpp_lib_math_constants                                201907L <numbers>
+__cpp_lib_math_special_functions                        201603L <cmath>
+__cpp_lib_mdspan                                        202406L <mdspan>
+                                                        202207L // C++23
+__cpp_lib_memory_resource                               201603L <memory_resource>
+__cpp_lib_move_iterator_concept                         202207L <iterator>
+__cpp_lib_move_only_function                            202110L <functional>
+__cpp_lib_node_extract                                  201606L <map> <set> <unordered_map>
+                                                                <unordered_set>
+__cpp_lib_nonmember_container_access                    201411L <array> <deque> <forward_list>
+                                                                <iterator> <list> <map>
+                                                                <regex> <set> <string>
+                                                                <unordered_map> <unordered_set> <vector>
+__cpp_lib_not_fn                                        201603L <functional>
+__cpp_lib_null_iterators                                201304L <iterator>
+__cpp_lib_optional                                      202110L <optional>
+                                                        201606L // C++17
+__cpp_lib_optional_range_support                        202406L <optional>
+__cpp_lib_out_ptr                                       202311L <memory>
+                                                        202106L // C++23
+__cpp_lib_parallel_algorithm                            201603L <algorithm> <numeric>
+__cpp_lib_philox_engine                                 202406L <random>
+__cpp_lib_polymorphic_allocator                         201902L <memory_resource>
+__cpp_lib_print                                         202207L <ostream> <print>
+__cpp_lib_quoted_string_io                              201304L <iomanip>
+__cpp_lib_ranges                                        202207L <algorithm> <functional> <iterator>
+                                                                <memory> <ranges>
+__cpp_lib_ranges_as_const                               202207L <ranges>
+__cpp_lib_ranges_as_rvalue                              202207L <ranges>
+__cpp_lib_ranges_chunk                                  202202L <ranges>
+__cpp_lib_ranges_chunk_by                               202202L <ranges>
+__cpp_lib_ranges_concat                                 202403L <ranges>
+__cpp_lib_ranges_contains                               202207L <algorithm>
+__cpp_lib_ranges_find_last                              202207L <algorithm>
+__cpp_lib_ranges_iota                                   202202L <numeric>
+__cpp_lib_ranges_join_with                              202202L <ranges>
+__cpp_lib_ranges_repeat                                 202207L <ranges>
+__cpp_lib_ranges_slide                                  202202L <ranges>
+__cpp_lib_ranges_starts_ends_with                       202106L <algorithm>
+__cpp_lib_ranges_to_container                           202202L <ranges>
+__cpp_lib_ranges_zip                                    202110L <ranges> <tuple> <utility>
+__cpp_lib_ratio                                         202306L <ratio>
+__cpp_lib_raw_memory_algorithms                         201606L <memory>
+__cpp_lib_rcu                                           202306L <rcu>
+__cpp_lib_reference_from_temporary                      202202L <type_traits>
+__cpp_lib_reference_wrapper                             202403L <functional>
+__cpp_lib_remove_cvref                                  201711L <type_traits>
+__cpp_lib_result_of_sfinae                              201210L <functional> <type_traits>
+__cpp_lib_robust_nonmodifying_seq_ops                   201304L <algorithm>
+__cpp_lib_sample                                        201603L <algorithm>
+__cpp_lib_saturation_arithmetic                         202311L <numeric>
+__cpp_lib_scoped_lock                                   201703L <mutex>
+__cpp_lib_semaphore                                     201907L <semaphore>
+__cpp_lib_senders                                       202406L <execution>
+__cpp_lib_shared_mutex                                  201505L <shared_mutex>
+__cpp_lib_shared_ptr_arrays                             201707L <memory>
+                                                        201611L // C++17
+__cpp_lib_shared_ptr_weak_type                          201606L <memory>
+__cpp_lib_shared_timed_mutex                            201402L <shared_mutex>
+__cpp_lib_shift                                         201806L <algorithm>
+__cpp_lib_smart_ptr_for_overwrite                       202002L <memory>
+__cpp_lib_smart_ptr_owner_equality                      202306L <memory>
+__cpp_lib_source_location                               201907L <source_location>
+__cpp_lib_span                                          202002L <span>
+__cpp_lib_span_at                                       202311L <span>
+__cpp_lib_span_initializer_list                         202311L <span>
+__cpp_lib_spanstream                                    202106L <spanstream>
+__cpp_lib_ssize                                         201902L <iterator>
+__cpp_lib_sstream_from_string_view                      202306L <sstream>
+__cpp_lib_stacktrace                                    202011L <stacktrace>
+__cpp_lib_starts_ends_with                              201711L <string> <string_view>
+__cpp_lib_stdatomic_h                                   202011L <stdatomic.h>
+__cpp_lib_string_contains                               202011L <string> <string_view>
+__cpp_lib_string_resize_and_overwrite                   202110L <string>
+__cpp_lib_string_udls                                   201304L <string>
+__cpp_lib_string_view                                   202403L <string> <string_view>
+                                                        201803L // C++20
+                                                        201606L // C++17
+__cpp_lib_submdspan                                     202306L <mdspan>
+__cpp_lib_syncbuf                                       201803L <syncstream>
+__cpp_lib_text_encoding                                 202306L <text_encoding>
+__cpp_lib_three_way_comparison                          201907L <compare>
+__cpp_lib_to_address                                    201711L <memory>
+__cpp_lib_to_array                                      201907L <array>
+__cpp_lib_to_chars                                      202306L <charconv>
+                                                        201611L // C++17
+__cpp_lib_to_string                                     202306L <string>
+__cpp_lib_to_underlying                                 202102L <utility>
+__cpp_lib_transformation_trait_aliases                  201304L <type_traits>
+__cpp_lib_transparent_operators                         201510L <functional> <memory>
+                                                        201210L // C++14
+__cpp_lib_tuple_element_t                               201402L <tuple>
+__cpp_lib_tuple_like                                    202311L <map> <tuple> <unordered_map>
+                                                                <utility>
+                                                        202207L // C++23
+__cpp_lib_tuples_by_type                                201304L <tuple> <utility>
+__cpp_lib_type_identity                                 201806L <type_traits>
+__cpp_lib_type_trait_variable_templates                 201510L <type_traits>
+__cpp_lib_uncaught_exceptions                           201411L <exception>
+__cpp_lib_unordered_map_try_emplace                     201411L <unordered_map>
+__cpp_lib_unreachable                                   202202L <utility>
+__cpp_lib_unwrap_ref                                    201811L <functional>
+__cpp_lib_variant                                       202102L <variant>
+__cpp_lib_void_t                                        201411L <type_traits>
+
+*/
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+// clang-format off
+
+#if _LIBCPP_STD_VER >= 14
+# define __cpp_lib_chrono_udls                          201304L
+# define __cpp_lib_complex_udls                         201309L
+# define __cpp_lib_exchange_function                    201304L
+# define __cpp_lib_generic_associative_lookup           201304L
+# define __cpp_lib_integer_sequence                     201304L
+# define __cpp_lib_integral_constant_callable           201304L
+# define __cpp_lib_is_final                             201402L
+# define __cpp_lib_is_null_pointer                      201309L
+# define __cpp_lib_make_reverse_iterator                201402L
+# define __cpp_lib_make_unique                          201304L
+# define __cpp_lib_null_iterators                       201304L
+# if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#   define __cpp_lib_quoted_string_io                   201304L
+# endif
+# define __cpp_lib_result_of_sfinae                     201210L
+# define __cpp_lib_robust_nonmodifying_seq_ops          201304L
+# if !defined(_LIBCPP_HAS_NO_THREADS)
+#   define __cpp_lib_shared_timed_mutex                 201402L
+# endif
+# define __cpp_lib_string_udls                          201304L
+# define __cpp_lib_transformation_trait_aliases         201304L
+# define __cpp_lib_transparent_operators                201210L
+# define __cpp_lib_tuple_element_t                      201402L
+# define __cpp_lib_tuples_by_type                       201304L
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+# define __cpp_lib_addressof_constexpr                  201603L
+# define __cpp_lib_allocator_traits_is_always_equal     201411L
+# define __cpp_lib_any                                  201606L
+# define __cpp_lib_apply                                201603L
+# define __cpp_lib_array_constexpr                      201603L
+# define __cpp_lib_as_const                             201510L
+# define __cpp_lib_atomic_is_always_lock_free           201603L
+# define __cpp_lib_bool_constant                        201505L
+# define __cpp_lib_boyer_moore_searcher                 201603L
+# define __cpp_lib_byte                                 201603L
+# define __cpp_lib_chrono                               201611L
+# define __cpp_lib_clamp                                201603L
+# define __cpp_lib_enable_shared_from_this              201603L
+// # define __cpp_lib_execution                            201603L
+# if !defined(_LIBCPP_HAS_NO_FILESYSTEM) && _LIBCPP_AVAILABILITY_HAS_FILESYSTEM_LIBRARY
+#   define __cpp_lib_filesystem                         201703L
+# endif
+# define __cpp_lib_gcd_lcm                              201606L
+# if defined(__GCC_DESTRUCTIVE_SIZE) && defined(__GCC_CONSTRUCTIVE_SIZE)
+#   define __cpp_lib_hardware_interference_size         201703L
+# endif
+# define __cpp_lib_has_unique_object_representations    201606L
+# define __cpp_lib_hypot                                201603L
+# define __cpp_lib_incomplete_container_elements        201505L
+# define __cpp_lib_invoke                               201411L
+# define __cpp_lib_is_aggregate                         201703L
+# define __cpp_lib_is_invocable                         201703L
+# define __cpp_lib_is_swappable                         201603L
+# define __cpp_lib_launder                              201606L
+# define __cpp_lib_logical_traits                       201510L
+# define __cpp_lib_make_from_tuple                      201606L
+# define __cpp_lib_map_try_emplace                      201411L
+// # define __cpp_lib_math_special_functions               201603L
+# if _LIBCPP_AVAILABILITY_HAS_PMR
+#   define __cpp_lib_memory_resource                    201603L
+# endif
+# define __cpp_lib_node_extract                         201606L
+# define __cpp_lib_nonmember_container_access           201411L
+# define __cpp_lib_not_fn                               201603L
+# define __cpp_lib_optional                             201606L
+// # define __cpp_lib_parallel_algorithm                   201603L
+# define __cpp_lib_raw_memory_algorithms                201606L
+# define __cpp_lib_sample                               201603L
+# if !defined(_LIBCPP_HAS_NO_THREADS)
+#   define __cpp_lib_scoped_lock                        201703L
+# endif
+# if !defined(_LIBCPP_HAS_NO_THREADS)
+#   define __cpp_lib_shared_mutex                       201505L
+# endif
+# define __cpp_lib_shared_ptr_arrays                    201611L
+# define __cpp_lib_shared_ptr_weak_type                 201606L
+# define __cpp_lib_string_view                          201606L
+// # define __cpp_lib_to_chars                             201611L
+# undef  __cpp_lib_transparent_operators
+# define __cpp_lib_transparent_operators                201510L
+# define __cpp_lib_type_trait_variable_templates        201510L
+# define __cpp_lib_uncaught_exceptions                  201411L
+# define __cpp_lib_unordered_map_try_emplace            201411L
+# define __cpp_lib_variant                              202102L
+# define __cpp_lib_void_t                               201411L
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+# undef  __cpp_lib_array_constexpr
+# define __cpp_lib_array_constexpr                      201811L
+# define __cpp_lib_assume_aligned                       201811L
+# define __cpp_lib_atomic_flag_test                     201907L
+// # define __cpp_lib_atomic_float                         201711L
+# define __cpp_lib_atomic_lock_free_type_aliases        201907L
+# define __cpp_lib_atomic_ref                           201806L
+// # define __cpp_lib_atomic_shared_ptr                    201711L
+# define __cpp_lib_atomic_value_initialization          201911L
+# if _LIBCPP_AVAILABILITY_HAS_SYNC
+#   define __cpp_lib_atomic_wait                        201907L
+# endif
+# if !defined(_LIBCPP_HAS_NO_THREADS) && _LIBCPP_AVAILABILITY_HAS_SYNC
+#   define __cpp_lib_barrier                            201907L
+# endif
+# define __cpp_lib_bind_front                           201907L
+# define __cpp_lib_bit_cast                             201806L
+# define __cpp_lib_bitops                               201907L
+# define __cpp_lib_bounded_array_traits                 201902L
+# if !defined(_LIBCPP_HAS_NO_CHAR8_T)
+#   define __cpp_lib_char8_t                            201907L
+# endif
+# define __cpp_lib_concepts                             202002L
+# define __cpp_lib_constexpr_algorithms                 201806L
+# define __cpp_lib_constexpr_complex                    201711L
+# define __cpp_lib_constexpr_dynamic_alloc              201907L
+# define __cpp_lib_constexpr_functional                 201907L
+# define __cpp_lib_constexpr_iterator                   201811L
+# define __cpp_lib_constexpr_memory                     201811L
+# define __cpp_lib_constexpr_numeric                    201911L
+# define __cpp_lib_constexpr_string                     201907L
+# define __cpp_lib_constexpr_string_view                201811L
+# define __cpp_lib_constexpr_tuple                      201811L
+# define __cpp_lib_constexpr_utility                    201811L
+# define __cpp_lib_constexpr_vector                     201907L
+# define __cpp_lib_coroutine                            201902L
+# if _LIBCPP_STD_VER >= 20 && defined(__cpp_impl_destroying_delete) && __cpp_impl_destroying_delete >= 201806L
+#   define __cpp_lib_destroying_delete                  201806L
+# endif
+# define __cpp_lib_endian                               201907L
+# define __cpp_lib_erase_if                             202002L
+# undef  __cpp_lib_execution
+// # define __cpp_lib_execution                            201902L
+# define __cpp_lib_format                               202110L
+# define __cpp_lib_format_uchar                         202311L
+# define __cpp_lib_generic_unordered_lookup             201811L
+# define __cpp_lib_int_pow2                             202002L
+# define __cpp_lib_integer_comparison_functions         202002L
+# define __cpp_lib_interpolate                          201902L
+# define __cpp_lib_is_constant_evaluated                201811L
+// # define __cpp_lib_is_layout_compatible                 201907L
+# define __cpp_lib_is_nothrow_convertible               201806L
+// # define __cpp_lib_is_pointer_interconvertible          201907L
+# if !defined(_LIBCPP_HAS_NO_THREADS) && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN) && _LIBCPP_AVAILABILITY_HAS_SYNC
+#   define __cpp_lib_jthread                            201911L
+# endif
+# if !defined(_LIBCPP_HAS_NO_THREADS) && _LIBCPP_AVAILABILITY_HAS_SYNC
+#   define __cpp_lib_latch                              201907L
+# endif
+# define __cpp_lib_list_remove_return_type              201806L
+# define __cpp_lib_math_constants                       201907L
+# define __cpp_lib_move_iterator_concept                202207L
+# if _LIBCPP_AVAILABILITY_HAS_PMR
+#   define __cpp_lib_polymorphic_allocator              201902L
+# endif
+# define __cpp_lib_ranges                               202207L
+# define __cpp_lib_remove_cvref                         201711L
+# if !defined(_LIBCPP_HAS_NO_THREADS) && _LIBCPP_AVAILABILITY_HAS_SYNC
+#   define __cpp_lib_semaphore                          201907L
+# endif
+# undef  __cpp_lib_shared_ptr_arrays
+# define __cpp_lib_shared_ptr_arrays                    201707L
+# define __cpp_lib_shift                                201806L
+// # define __cpp_lib_smart_ptr_for_overwrite              202002L
+# define __cpp_lib_source_location                      201907L
+# define __cpp_lib_span                                 202002L
+# define __cpp_lib_ssize                                201902L
+# define __cpp_lib_starts_ends_with                     201711L
+# undef  __cpp_lib_string_view
+# define __cpp_lib_string_view                          201803L
+# if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM)
+#   define __cpp_lib_syncbuf                            201803L
+# endif
+# define __cpp_lib_three_way_comparison                 201907L
+# define __cpp_lib_to_address                           201711L
+# define __cpp_lib_to_array                             201907L
+# define __cpp_lib_type_identity                        201806L
+# define __cpp_lib_unwrap_ref                           201811L
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+# define __cpp_lib_adaptor_iterator_pair_constructor    202106L
+# define __cpp_lib_allocate_at_least                    202302L
+// # define __cpp_lib_associative_heterogeneous_erasure    202110L
+# define __cpp_lib_bind_back                            202202L
+# define __cpp_lib_byteswap                             202110L
+# define __cpp_lib_constexpr_bitset                     202207L
+# define __cpp_lib_constexpr_charconv                   202207L
+// # define __cpp_lib_constexpr_cmath                      202202L
+# undef  __cpp_lib_constexpr_memory
+# define __cpp_lib_constexpr_memory                     202202L
+# define __cpp_lib_constexpr_typeinfo                   202106L
+# define __cpp_lib_containers_ranges                    202202L
+# define __cpp_lib_expected                             202211L
+# define __cpp_lib_format_ranges                        202207L
+// # define __cpp_lib_formatters                           202302L
+# define __cpp_lib_forward_like                         202207L
+# define __cpp_lib_invoke_r                             202106L
+# define __cpp_lib_ios_noreplace                        202207L
+# define __cpp_lib_is_scoped_enum                       202011L
+# define __cpp_lib_mdspan                               202207L
+# define __cpp_lib_modules                              202207L
+// # define __cpp_lib_move_only_function                   202110L
+# undef  __cpp_lib_optional
+# define __cpp_lib_optional                             202110L
+# define __cpp_lib_out_ptr                              202106L
+# define __cpp_lib_print                                202207L
+// # define __cpp_lib_ranges_as_const                      202207L
+# define __cpp_lib_ranges_as_rvalue                     202207L
+// # define __cpp_lib_ranges_chunk                         202202L
+# define __cpp_lib_ranges_chunk_by                      202202L
+# define __cpp_lib_ranges_contains                      202207L
+# define __cpp_lib_ranges_find_last                     202207L
+// # define __cpp_lib_ranges_iota                          202202L
+// # define __cpp_lib_ranges_join_with                     202202L
+# define __cpp_lib_ranges_repeat                        202207L
+// # define __cpp_lib_ranges_slide                         202202L
+# define __cpp_lib_ranges_starts_ends_with              202106L
+# define __cpp_lib_ranges_to_container                  202202L
+// # define __cpp_lib_ranges_zip                           202110L
+// # define __cpp_lib_reference_from_temporary             202202L
+// # define __cpp_lib_spanstream                           202106L
+// # define __cpp_lib_stacktrace                           202011L
+# define __cpp_lib_stdatomic_h                          202011L
+# define __cpp_lib_string_contains                      202011L
+# define __cpp_lib_string_resize_and_overwrite          202110L
+# define __cpp_lib_to_underlying                        202102L
+// # define __cpp_lib_tuple_like                           202207L
+# define __cpp_lib_unreachable                          202202L
+#endif
+
+#if _LIBCPP_STD_VER >= 26
+// # define __cpp_lib_associative_heterogeneous_insertion  202306L
+// # define __cpp_lib_atomic_min_max                       202403L
+# undef  __cpp_lib_bind_front
+# define __cpp_lib_bind_front                           202306L
+# define __cpp_lib_bitset                               202306L
+// # define __cpp_lib_constexpr_new                        202406L
+// # define __cpp_lib_constrained_equality                 202403L
+// # define __cpp_lib_copyable_function                    202306L
+// # define __cpp_lib_debugging                            202311L
+// # define __cpp_lib_default_template_type_for_algorithm_values 202403L
+// # define __cpp_lib_format_path                          202403L
+// # define __cpp_lib_freestanding_algorithm               202311L
+// # define __cpp_lib_freestanding_array                   202311L
+// # define __cpp_lib_freestanding_cstring                 202306L
+// # define __cpp_lib_freestanding_expected                202311L
+// # define __cpp_lib_freestanding_mdspan                  202311L
+// # define __cpp_lib_freestanding_optional                202311L
+// # define __cpp_lib_freestanding_string_view             202311L
+// # define __cpp_lib_freestanding_variant                 202311L
+# if !defined(_LIBCPP_HAS_NO_FILESYSTEM) && !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#   define __cpp_lib_fstream_native_handle              202306L
+# endif
+// # define __cpp_lib_function_ref                         202306L
+// # define __cpp_lib_generate_random                      202403L
+// # define __cpp_lib_hazard_pointer                       202306L
+// # define __cpp_lib_inplace_vector                       202406L
+// # define __cpp_lib_is_virtual_base_of                   202406L
+// # define __cpp_lib_is_within_lifetime                   202306L
+// # define __cpp_lib_linalg                               202311L
+# undef  __cpp_lib_mdspan
+# define __cpp_lib_mdspan                               202406L
+// # define __cpp_lib_optional_range_support               202406L
+# undef  __cpp_lib_out_ptr
+# define __cpp_lib_out_ptr                              202311L
+// # define __cpp_lib_philox_engine                        202406L
+// # define __cpp_lib_ranges_concat                        202403L
+# define __cpp_lib_ratio                                202306L
+// # define __cpp_lib_rcu                                  202306L
+# define __cpp_lib_reference_wrapper                    202403L
+# define __cpp_lib_saturation_arithmetic                202311L
+// # define __cpp_lib_senders                              202406L
+// # define __cpp_lib_smart_ptr_owner_equality             202306L
+# define __cpp_lib_span_at                              202311L
+# define __cpp_lib_span_initializer_list                202311L
+# define __cpp_lib_sstream_from_string_view             202306L
+# undef  __cpp_lib_string_view
+# define __cpp_lib_string_view                          202403L
+// # define __cpp_lib_submdspan                            202306L
+// # define __cpp_lib_text_encoding                        202306L
+# undef  __cpp_lib_to_chars
+// # define __cpp_lib_to_chars                             202306L
+// # define __cpp_lib_to_string                            202306L
+# undef  __cpp_lib_tuple_like
+// # define __cpp_lib_tuple_like                           202311L
+#endif
+
+// clang-format on
+
+#endif // _LIBCPP_VERSIONH
diff --git a/libcxx/include/__cxx03/wchar.h b/libcxx/include/__cxx03/wchar.h
new file mode 100644
index 000000000000..d4268c6d2c24
--- /dev/null
+++ b/libcxx/include/__cxx03/wchar.h
@@ -0,0 +1,211 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__need_wint_t) || defined(__need_mbstate_t)
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  include_next <wchar.h>
+
+#elif !defined(_LIBCPP_WCHAR_H)
+#  define _LIBCPP_WCHAR_H
+
+/*
+    wchar.h synopsis
+
+Macros:
+
+    NULL
+    WCHAR_MAX
+    WCHAR_MIN
+    WEOF
+
+Types:
+
+    mbstate_t
+    size_t
+    tm
+    wint_t
+
+int fwprintf(FILE* restrict stream, const wchar_t* restrict format, ...);
+int fwscanf(FILE* restrict stream, const wchar_t* restrict format, ...);
+int swprintf(wchar_t* restrict s, size_t n, const wchar_t* restrict format, ...);
+int swscanf(const wchar_t* restrict s, const wchar_t* restrict format, ...);
+int vfwprintf(FILE* restrict stream, const wchar_t* restrict format, va_list arg);
+int vfwscanf(FILE* restrict stream, const wchar_t* restrict format, va_list arg);  // C99
+int vswprintf(wchar_t* restrict s, size_t n, const wchar_t* restrict format, va_list arg);
+int vswscanf(const wchar_t* restrict s, const wchar_t* restrict format, va_list arg);  // C99
+int vwprintf(const wchar_t* restrict format, va_list arg);
+int vwscanf(const wchar_t* restrict format, va_list arg);  // C99
+int wprintf(const wchar_t* restrict format, ...);
+int wscanf(const wchar_t* restrict format, ...);
+wint_t fgetwc(FILE* stream);
+wchar_t* fgetws(wchar_t* restrict s, int n, FILE* restrict stream);
+wint_t fputwc(wchar_t c, FILE* stream);
+int fputws(const wchar_t* restrict s, FILE* restrict stream);
+int fwide(FILE* stream, int mode);
+wint_t getwc(FILE* stream);
+wint_t getwchar();
+wint_t putwc(wchar_t c, FILE* stream);
+wint_t putwchar(wchar_t c);
+wint_t ungetwc(wint_t c, FILE* stream);
+double wcstod(const wchar_t* restrict nptr, wchar_t** restrict endptr);
+float wcstof(const wchar_t* restrict nptr, wchar_t** restrict endptr);         // C99
+long double wcstold(const wchar_t* restrict nptr, wchar_t** restrict endptr);  // C99
+long wcstol(const wchar_t* restrict nptr, wchar_t** restrict endptr, int base);
+long long wcstoll(const wchar_t* restrict nptr, wchar_t** restrict endptr, int base);  // C99
+unsigned long wcstoul(const wchar_t* restrict nptr, wchar_t** restrict endptr, int base);
+unsigned long long wcstoull(const wchar_t* restrict nptr, wchar_t** restrict endptr, int base);  // C99
+wchar_t* wcscpy(wchar_t* restrict s1, const wchar_t* restrict s2);
+wchar_t* wcsncpy(wchar_t* restrict s1, const wchar_t* restrict s2, size_t n);
+wchar_t* wcscat(wchar_t* restrict s1, const wchar_t* restrict s2);
+wchar_t* wcsncat(wchar_t* restrict s1, const wchar_t* restrict s2, size_t n);
+int wcscmp(const wchar_t* s1, const wchar_t* s2);
+int wcscoll(const wchar_t* s1, const wchar_t* s2);
+int wcsncmp(const wchar_t* s1, const wchar_t* s2, size_t n);
+size_t wcsxfrm(wchar_t* restrict s1, const wchar_t* restrict s2, size_t n);
+const wchar_t* wcschr(const wchar_t* s, wchar_t c);
+      wchar_t* wcschr(      wchar_t* s, wchar_t c);
+size_t wcscspn(const wchar_t* s1, const wchar_t* s2);
+size_t wcslen(const wchar_t* s);
+const wchar_t* wcspbrk(const wchar_t* s1, const wchar_t* s2);
+      wchar_t* wcspbrk(      wchar_t* s1, const wchar_t* s2);
+const wchar_t* wcsrchr(const wchar_t* s, wchar_t c);
+      wchar_t* wcsrchr(      wchar_t* s, wchar_t c);
+size_t wcsspn(const wchar_t* s1, const wchar_t* s2);
+const wchar_t* wcsstr(const wchar_t* s1, const wchar_t* s2);
+      wchar_t* wcsstr(      wchar_t* s1, const wchar_t* s2);
+wchar_t* wcstok(wchar_t* restrict s1, const wchar_t* restrict s2, wchar_t** restrict ptr);
+const wchar_t* wmemchr(const wchar_t* s, wchar_t c, size_t n);
+      wchar_t* wmemchr(      wchar_t* s, wchar_t c, size_t n);
+int wmemcmp(wchar_t* restrict s1, const wchar_t* restrict s2, size_t n);
+wchar_t* wmemcpy(wchar_t* restrict s1, const wchar_t* restrict s2, size_t n);
+wchar_t* wmemmove(wchar_t* s1, const wchar_t* s2, size_t n);
+wchar_t* wmemset(wchar_t* s, wchar_t c, size_t n);
+size_t wcsftime(wchar_t* restrict s, size_t maxsize, const wchar_t* restrict format,
+                const tm* restrict timeptr);
+wint_t btowc(int c);
+int wctob(wint_t c);
+int mbsinit(const mbstate_t* ps);
+size_t mbrlen(const char* restrict s, size_t n, mbstate_t* restrict ps);
+size_t mbrtowc(wchar_t* restrict pwc, const char* restrict s, size_t n, mbstate_t* restrict ps);
+size_t wcrtomb(char* restrict s, wchar_t wc, mbstate_t* restrict ps);
+size_t mbsrtowcs(wchar_t* restrict dst, const char** restrict src, size_t len,
+                 mbstate_t* restrict ps);
+size_t wcsrtombs(char* restrict dst, const wchar_t** restrict src, size_t len,
+                 mbstate_t* restrict ps);
+
+*/
+
+#  include <__config>
+#  include <stddef.h>
+
+#  if defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS)
+#    error                                                                                                             \
+        "The <wchar.h> header is not supported since libc++ has been configured with LIBCXX_ENABLE_WIDE_CHARACTERS disabled"
+#  endif
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+// We define this here to support older versions of glibc <wchar.h> that do
+// not define this for clang.
+#  ifdef __cplusplus
+#    define __CORRECT_ISO_CPP_WCHAR_H_PROTO
+#  endif
+
+#  if __has_include_next(<wchar.h>)
+#    include_next <wchar.h>
+#  else
+#    include <__mbstate_t.h> // make sure we have mbstate_t regardless of the existence of <wchar.h>
+#  endif
+
+// Determine whether we have const-correct overloads for wcschr and friends.
+#  if defined(_WCHAR_H_CPLUSPLUS_98_CONFORMANCE_)
+#    define _LIBCPP_WCHAR_H_HAS_CONST_OVERLOADS 1
+#  elif defined(__GLIBC_PREREQ)
+#    if __GLIBC_PREREQ(2, 10)
+#      define _LIBCPP_WCHAR_H_HAS_CONST_OVERLOADS 1
+#    endif
+#  elif defined(_LIBCPP_MSVCRT)
+#    if defined(_CRT_CONST_CORRECT_OVERLOADS)
+#      define _LIBCPP_WCHAR_H_HAS_CONST_OVERLOADS 1
+#    endif
+#  endif
+
+#  if defined(__cplusplus) && !defined(_LIBCPP_WCHAR_H_HAS_CONST_OVERLOADS) && defined(_LIBCPP_PREFERRED_OVERLOAD)
+extern "C++" {
+inline _LIBCPP_HIDE_FROM_ABI wchar_t* __libcpp_wcschr(const wchar_t* __s, wchar_t __c) {
+  return (wchar_t*)wcschr(__s, __c);
+}
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD const wchar_t* wcschr(const wchar_t* __s, wchar_t __c) {
+  return __libcpp_wcschr(__s, __c);
+}
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD wchar_t* wcschr(wchar_t* __s, wchar_t __c) {
+  return __libcpp_wcschr(__s, __c);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI wchar_t* __libcpp_wcspbrk(const wchar_t* __s1, const wchar_t* __s2) {
+  return (wchar_t*)wcspbrk(__s1, __s2);
+}
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD const wchar_t*
+wcspbrk(const wchar_t* __s1, const wchar_t* __s2) {
+  return __libcpp_wcspbrk(__s1, __s2);
+}
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD wchar_t* wcspbrk(wchar_t* __s1, const wchar_t* __s2) {
+  return __libcpp_wcspbrk(__s1, __s2);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI wchar_t* __libcpp_wcsrchr(const wchar_t* __s, wchar_t __c) {
+  return (wchar_t*)wcsrchr(__s, __c);
+}
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD const wchar_t* wcsrchr(const wchar_t* __s, wchar_t __c) {
+  return __libcpp_wcsrchr(__s, __c);
+}
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD wchar_t* wcsrchr(wchar_t* __s, wchar_t __c) {
+  return __libcpp_wcsrchr(__s, __c);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI wchar_t* __libcpp_wcsstr(const wchar_t* __s1, const wchar_t* __s2) {
+  return (wchar_t*)wcsstr(__s1, __s2);
+}
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD const wchar_t*
+wcsstr(const wchar_t* __s1, const wchar_t* __s2) {
+  return __libcpp_wcsstr(__s1, __s2);
+}
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD wchar_t* wcsstr(wchar_t* __s1, const wchar_t* __s2) {
+  return __libcpp_wcsstr(__s1, __s2);
+}
+
+inline _LIBCPP_HIDE_FROM_ABI wchar_t* __libcpp_wmemchr(const wchar_t* __s, wchar_t __c, size_t __n) {
+  return (wchar_t*)wmemchr(__s, __c, __n);
+}
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD const wchar_t*
+wmemchr(const wchar_t* __s, wchar_t __c, size_t __n) {
+  return __libcpp_wmemchr(__s, __c, __n);
+}
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD wchar_t* wmemchr(wchar_t* __s, wchar_t __c, size_t __n) {
+  return __libcpp_wmemchr(__s, __c, __n);
+}
+}
+#  endif
+
+#  if defined(__cplusplus) && (defined(_LIBCPP_MSVCRT_LIKE) || defined(__MVS__))
+extern "C" {
+size_t mbsnrtowcs(
+    wchar_t* __restrict __dst, const char** __restrict __src, size_t __nmc, size_t __len, mbstate_t* __restrict __ps);
+size_t wcsnrtombs(
+    char* __restrict __dst, const wchar_t** __restrict __src, size_t __nwc, size_t __len, mbstate_t* __restrict __ps);
+} // extern "C"
+#  endif // __cplusplus && (_LIBCPP_MSVCRT || __MVS__)
+
+#endif // _LIBCPP_WCHAR_H
diff --git a/libcxx/include/__cxx03/wctype.h b/libcxx/include/__cxx03/wctype.h
new file mode 100644
index 000000000000..c76ec5a3f060
--- /dev/null
+++ b/libcxx/include/__cxx03/wctype.h
@@ -0,0 +1,95 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_WCTYPE_H
+#define _LIBCPP_WCTYPE_H
+
+/*
+    wctype.h synopsis
+
+Macros:
+
+    WEOF
+
+Types:
+
+    wint_t
+    wctrans_t
+    wctype_t
+
+int iswalnum(wint_t wc);
+int iswalpha(wint_t wc);
+int iswblank(wint_t wc);  // C99
+int iswcntrl(wint_t wc);
+int iswdigit(wint_t wc);
+int iswgraph(wint_t wc);
+int iswlower(wint_t wc);
+int iswprint(wint_t wc);
+int iswpunct(wint_t wc);
+int iswspace(wint_t wc);
+int iswupper(wint_t wc);
+int iswxdigit(wint_t wc);
+int iswctype(wint_t wc, wctype_t desc);
+wctype_t wctype(const char* property);
+wint_t towlower(wint_t wc);
+wint_t towupper(wint_t wc);
+wint_t towctrans(wint_t wc, wctrans_t desc);
+wctrans_t wctrans(const char* property);
+
+*/
+
+#include <__config>
+
+#if defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS)
+#  error                                                                                                               \
+      "The <wctype.h> header is not supported since libc++ has been configured with LIBCXX_ENABLE_WIDE_CHARACTERS disabled"
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+// TODO:
+// In the future, we should unconditionally include_next <wctype.h> here and instead
+// have a mode under which the library does not need libc++'s <wctype.h> or <cwctype>
+// at all (i.e. a mode without wchar_t). As it stands, we need to do that to completely
+// bypass the using declarations in <cwctype> when we did not include <wctype.h>.
+// Otherwise, a using declaration like `using ::wint_t` in <cwctype> will refer to
+// nothing (with using_if_exists), and if we include another header that defines one
+// of these declarations (e.g. <wchar.h>), the second `using ::wint_t` with using_if_exists
+// will fail because it does not refer to the same declaration.
+#if __has_include_next(<wctype.h>)
+#  include_next <wctype.h>
+#  define _LIBCPP_INCLUDED_C_LIBRARY_WCTYPE_H
+#endif
+
+#ifdef __cplusplus
+
+#  undef iswalnum
+#  undef iswalpha
+#  undef iswblank
+#  undef iswcntrl
+#  undef iswdigit
+#  undef iswgraph
+#  undef iswlower
+#  undef iswprint
+#  undef iswpunct
+#  undef iswspace
+#  undef iswupper
+#  undef iswxdigit
+#  undef iswctype
+#  undef wctype
+#  undef towlower
+#  undef towupper
+#  undef towctrans
+#  undef wctrans
+
+#endif // __cplusplus
+
+#endif // _LIBCPP_WCTYPE_H
diff --git a/libcxx/utils/generate_iwyu_mapping.py b/libcxx/utils/generate_iwyu_mapping.py
index 7976f4afc24f..ab3e3f14e14d 100644
--- a/libcxx/utils/generate_iwyu_mapping.py
+++ b/libcxx/utils/generate_iwyu_mapping.py
@@ -10,6 +10,7 @@ import typing
 
 def IWYU_mapping(header: str) -> typing.Optional[typing.List[str]]:
     ignore = [
+        "__cxx03/.+",
         "__debug_utils/.+",
         "__fwd/get[.]h",
         "__pstl/.+",
-- 
GitLab


From 76bdc608980b307a354580dfbbed844fee0b75a8 Mon Sep 17 00:00:00 2001
From: Hubert Tong <hubert.reinterpretcast@gmail.com>
Date: Wed, 23 Oct 2024 18:28:43 -0400
Subject: [PATCH 020/693] [Legacy ThinLTO] NFC: Move helper class to an "Impl"
 namespace (#112846)

`::llvm::TargetMachineBuilder` is only a helper class for
`ThinLTOCodeGenerator` and not a real facility. Move it into an "Impl"
namespace.
---
 llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h | 11 +++++++++--
 llvm/lib/LTO/ThinLTOCodeGenerator.cpp               |  1 +
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h b/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h
index 7eb30d56e10c..676d4e179bed 100644
--- a/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h
+++ b/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h
@@ -31,8 +31,15 @@ namespace llvm {
 class StringRef;
 class TargetMachine;
 
+/// ThinLTOCodeGeneratorImpl - Namespace used for ThinLTOCodeGenerator
+/// implementation details. It should be considered private to the
+/// implementation.
+namespace ThinLTOCodeGeneratorImpl {
+struct TargetMachineBuilder;
+}
+
 /// Helper to gather options relevant to the target machine creation
-struct TargetMachineBuilder {
+struct ThinLTOCodeGeneratorImpl::TargetMachineBuilder {
   Triple TheTriple;
   std::string MCpu;
   std::string MAttr;
@@ -301,7 +308,7 @@ public:
 
 private:
   /// Helper factory to build a TargetMachine
-  TargetMachineBuilder TMBuilder;
+  ThinLTOCodeGeneratorImpl::TargetMachineBuilder TMBuilder;
 
   /// Vector holding the in-memory buffer containing the produced binaries, when
   /// SavedObjectsDirectoryPath isn't set.
diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
index 8074f8690cc1..311510ed9fab 100644
--- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -70,6 +70,7 @@
 #endif
 
 using namespace llvm;
+using namespace ThinLTOCodeGeneratorImpl;
 
 #define DEBUG_TYPE "thinlto"
 
-- 
GitLab


From 4e40b71c519a53de2c23f6796a816d10337d328c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Wed, 23 Oct 2024 15:28:51 -0700
Subject: [PATCH 021/693] [flang][cuda] Add specialized gpu.launch_func
 conversion (#113493)

---
 .../Transforms/CUFGPUToLLVMConversion.h       |  28 +++
 .../flang/Optimizer/Transforms/Passes.h       |   1 +
 .../flang/Optimizer/Transforms/Passes.td      |   7 +
 flang/lib/Optimizer/Transforms/CMakeLists.txt |   1 +
 .../Transforms/CUFGPUToLLVMConversion.cpp     | 180 ++++++++++++++++++
 .../Optimizer/Transforms/CUFOpConversion.cpp  |   1 +
 flang/runtime/CUDA/registration.cpp           |   6 +-
 flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir | 104 ++++++++++
 flang/tools/fir-opt/fir-opt.cpp               |   1 +
 9 files changed, 328 insertions(+), 1 deletion(-)
 create mode 100644 flang/include/flang/Optimizer/Transforms/CUFGPUToLLVMConversion.h
 create mode 100644 flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
 create mode 100644 flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir

diff --git a/flang/include/flang/Optimizer/Transforms/CUFGPUToLLVMConversion.h b/flang/include/flang/Optimizer/Transforms/CUFGPUToLLVMConversion.h
new file mode 100644
index 000000000000..7d76c1f4e521
--- /dev/null
+++ b/flang/include/flang/Optimizer/Transforms/CUFGPUToLLVMConversion.h
@@ -0,0 +1,28 @@
+//===------- Optimizer/Transforms/CUFGPUToLLVMConversion.h ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_OPTIMIZER_TRANSFORMS_CUFGPUTOLLVMCONVERSION_H_
+#define FORTRAN_OPTIMIZER_TRANSFORMS_CUFGPUTOLLVMCONVERSION_H_
+
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassRegistry.h"
+#include "mlir/Transforms/DialectConversion.h"
+
+namespace fir {
+class LLVMTypeConverter;
+}
+
+namespace cuf {
+
+void populateCUFGPUToLLVMConversionPatterns(
+    const fir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns,
+    mlir::PatternBenefit benefit = 1);
+
+} // namespace cuf
+
+#endif // FORTRAN_OPTIMIZER_TRANSFORMS_CUFGPUTOLLVMCONVERSION_H_
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
index 5d3067aa3598..e8f0a8444a31 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.h
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -41,6 +41,7 @@ namespace fir {
 #define GEN_PASS_DECL_CFGCONVERSION
 #define GEN_PASS_DECL_CUFADDCONSTRUCTOR
 #define GEN_PASS_DECL_CUFDEVICEGLOBAL
+#define GEN_PASS_DECL_CUFGPUTOLLVMCONVERSION
 #define GEN_PASS_DECL_CUFOPCONVERSION
 #define GEN_PASS_DECL_EXTERNALNAMECONVERSION
 #define GEN_PASS_DECL_MEMREFDATAFLOWOPT
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
index 2efa543ca071..a41f0f348f27 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -443,4 +443,11 @@ def CUFAddConstructor : Pass<"cuf-add-constructor", "mlir::ModuleOp"> {
   ];
 }
 
+def CUFGPUToLLVMConversion : Pass<"cuf-gpu-convert-to-llvm", "mlir::ModuleOp"> {
+  let summary = "Convert some GPU operations lowered from CUF to runtime calls";
+  let dependentDialects = [
+    "mlir::LLVM::LLVMDialect"
+  ];
+}
+
 #endif // FLANG_OPTIMIZER_TRANSFORMS_PASSES
diff --git a/flang/lib/Optimizer/Transforms/CMakeLists.txt b/flang/lib/Optimizer/Transforms/CMakeLists.txt
index 8f4f731e0092..d20d3bc4108c 100644
--- a/flang/lib/Optimizer/Transforms/CMakeLists.txt
+++ b/flang/lib/Optimizer/Transforms/CMakeLists.txt
@@ -12,6 +12,7 @@ add_flang_library(FIRTransforms
   CUFAddConstructor.cpp
   CUFDeviceGlobal.cpp
   CUFOpConversion.cpp
+  CUFGPUToLLVMConversion.cpp
   ArrayValueCopy.cpp
   ExternalNameConversion.cpp
   MemoryUtils.cpp
diff --git a/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp b/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
new file mode 100644
index 000000000000..5645ce6e6858
--- /dev/null
+++ b/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
@@ -0,0 +1,180 @@
+//===-- CUFGPUToLLVMConversion.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Transforms/CUFGPUToLLVMConversion.h"
+#include "flang/Common/Fortran.h"
+#include "flang/Optimizer/CodeGen/TypeConverter.h"
+#include "flang/Optimizer/Support/DataLayout.h"
+#include "flang/Runtime/CUDA/common.h"
+#include "mlir/Conversion/LLVMCommon/Pattern.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "llvm/Support/FormatVariadic.h"
+
+namespace fir {
+#define GEN_PASS_DEF_CUFGPUTOLLVMCONVERSION
+#include "flang/Optimizer/Transforms/Passes.h.inc"
+} // namespace fir
+
+using namespace fir;
+using namespace mlir;
+using namespace Fortran::runtime;
+
+namespace {
+
+static mlir::Value createKernelArgArray(mlir::Location loc,
+                                        mlir::ValueRange operands,
+                                        mlir::PatternRewriter &rewriter) {
+
+  auto *ctx = rewriter.getContext();
+  llvm::SmallVector<mlir::Type> structTypes(operands.size(), nullptr);
+
+  for (auto [i, arg] : llvm::enumerate(operands))
+    structTypes[i] = arg.getType();
+
+  auto structTy = mlir::LLVM::LLVMStructType::getLiteral(ctx, structTypes);
+  auto ptrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
+  mlir::Type i32Ty = rewriter.getI32Type();
+  auto one = rewriter.create<mlir::LLVM::ConstantOp>(
+      loc, i32Ty, rewriter.getIntegerAttr(i32Ty, 1));
+  mlir::Value argStruct =
+      rewriter.create<mlir::LLVM::AllocaOp>(loc, ptrTy, structTy, one);
+  auto size = rewriter.create<mlir::LLVM::ConstantOp>(
+      loc, i32Ty, rewriter.getIntegerAttr(i32Ty, structTypes.size()));
+  mlir::Value argArray =
+      rewriter.create<mlir::LLVM::AllocaOp>(loc, ptrTy, ptrTy, size);
+
+  for (auto [i, arg] : llvm::enumerate(operands)) {
+    auto indice = rewriter.create<mlir::LLVM::ConstantOp>(
+        loc, i32Ty, rewriter.getIntegerAttr(i32Ty, i));
+    mlir::Value structMember = rewriter.create<LLVM::GEPOp>(
+        loc, ptrTy, structTy, argStruct, mlir::ArrayRef<mlir::Value>({indice}));
+    rewriter.create<LLVM::StoreOp>(loc, arg, structMember);
+    mlir::Value arrayMember = rewriter.create<LLVM::GEPOp>(
+        loc, ptrTy, structTy, argArray, mlir::ArrayRef<mlir::Value>({indice}));
+    rewriter.create<LLVM::StoreOp>(loc, structMember, arrayMember);
+  }
+  return argArray;
+}
+
+struct GPULaunchKernelConversion
+    : public mlir::ConvertOpToLLVMPattern<mlir::gpu::LaunchFuncOp> {
+  explicit GPULaunchKernelConversion(
+      const fir::LLVMTypeConverter &typeConverter, mlir::PatternBenefit benefit)
+      : mlir::ConvertOpToLLVMPattern<mlir::gpu::LaunchFuncOp>(typeConverter,
+                                                              benefit) {}
+
+  using OpAdaptor = typename mlir::gpu::LaunchFuncOp::Adaptor;
+
+  mlir::LogicalResult
+  matchAndRewrite(mlir::gpu::LaunchFuncOp op, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+
+    if (op.hasClusterSize()) {
+      return mlir::failure();
+    }
+
+    mlir::Location loc = op.getLoc();
+    auto *ctx = rewriter.getContext();
+    mlir::ModuleOp mod = op->getParentOfType<mlir::ModuleOp>();
+    mlir::Value dynamicMemorySize = op.getDynamicSharedMemorySize();
+    mlir::Type i32Ty = rewriter.getI32Type();
+    if (!dynamicMemorySize)
+      dynamicMemorySize = rewriter.create<mlir::LLVM::ConstantOp>(
+          loc, i32Ty, rewriter.getIntegerAttr(i32Ty, 0));
+
+    mlir::Value kernelArgs =
+        createKernelArgArray(loc, adaptor.getKernelOperands(), rewriter);
+
+    auto ptrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
+    auto kernel = mod.lookupSymbol<mlir::LLVM::LLVMFuncOp>(op.getKernelName());
+    mlir::Value kernelPtr;
+    if (!kernel) {
+      auto funcOp = mod.lookupSymbol<mlir::func::FuncOp>(op.getKernelName());
+      if (!funcOp)
+        return mlir::failure();
+      kernelPtr =
+          rewriter.create<LLVM::AddressOfOp>(loc, ptrTy, funcOp.getName());
+    } else {
+      kernelPtr =
+          rewriter.create<LLVM::AddressOfOp>(loc, ptrTy, kernel.getName());
+    }
+
+    auto funcOp = mod.lookupSymbol<mlir::LLVM::LLVMFuncOp>(
+        RTNAME_STRING(CUFLaunchKernel));
+
+    auto llvmIntPtrType = mlir::IntegerType::get(
+        ctx, this->getTypeConverter()->getPointerBitwidth(0));
+    auto voidTy = mlir::LLVM::LLVMVoidType::get(ctx);
+    auto funcTy = mlir::LLVM::LLVMFunctionType::get(
+        voidTy,
+        {ptrTy, llvmIntPtrType, llvmIntPtrType, llvmIntPtrType, llvmIntPtrType,
+         llvmIntPtrType, llvmIntPtrType, i32Ty, ptrTy, ptrTy},
+        /*isVarArg=*/false);
+
+    auto cufLaunchKernel = mlir::SymbolRefAttr::get(
+        mod.getContext(), RTNAME_STRING(CUFLaunchKernel));
+    if (!funcOp) {
+      mlir::OpBuilder::InsertionGuard insertGuard(rewriter);
+      rewriter.setInsertionPointToStart(mod.getBody());
+      auto launchKernelFuncOp = rewriter.create<mlir::LLVM::LLVMFuncOp>(
+          loc, RTNAME_STRING(CUFLaunchKernel), funcTy);
+      launchKernelFuncOp.setVisibility(mlir::SymbolTable::Visibility::Private);
+    }
+
+    mlir::Value nullPtr = rewriter.create<LLVM::ZeroOp>(loc, ptrTy);
+
+    rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
+        op, funcTy, cufLaunchKernel,
+        mlir::ValueRange{kernelPtr, adaptor.getGridSizeX(),
+                         adaptor.getGridSizeY(), adaptor.getGridSizeZ(),
+                         adaptor.getBlockSizeX(), adaptor.getBlockSizeY(),
+                         adaptor.getBlockSizeZ(), dynamicMemorySize, kernelArgs,
+                         nullPtr});
+
+    return mlir::success();
+  }
+};
+
+class CUFGPUToLLVMConversion
+    : public fir::impl::CUFGPUToLLVMConversionBase<CUFGPUToLLVMConversion> {
+public:
+  void runOnOperation() override {
+    auto *ctx = &getContext();
+    mlir::RewritePatternSet patterns(ctx);
+    mlir::ConversionTarget target(*ctx);
+
+    mlir::Operation *op = getOperation();
+    mlir::ModuleOp module = mlir::dyn_cast<mlir::ModuleOp>(op);
+    if (!module)
+      return signalPassFailure();
+
+    std::optional<mlir::DataLayout> dl =
+        fir::support::getOrSetDataLayout(module, /*allowDefaultLayout=*/false);
+    fir::LLVMTypeConverter typeConverter(module, /*applyTBAA=*/false,
+                                         /*forceUnifiedTBAATree=*/false, *dl);
+    cuf::populateCUFGPUToLLVMConversionPatterns(typeConverter, patterns);
+    target.addIllegalOp<mlir::gpu::LaunchFuncOp>();
+    target.addLegalDialect<mlir::LLVM::LLVMDialect>();
+    if (mlir::failed(mlir::applyPartialConversion(getOperation(), target,
+                                                  std::move(patterns)))) {
+      mlir::emitError(mlir::UnknownLoc::get(ctx),
+                      "error in CUF GPU op conversion\n");
+      signalPassFailure();
+    }
+  }
+};
+} // namespace
+
+void cuf::populateCUFGPUToLLVMConversionPatterns(
+    const fir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns,
+    mlir::PatternBenefit benefit) {
+  patterns.add<GPULaunchKernelConversion>(converter, benefit);
+}
diff --git a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
index 069d88e0afca..9c2b882c7f46 100644
--- a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
@@ -20,6 +20,7 @@
 #include "flang/Runtime/CUDA/descriptor.h"
 #include "flang/Runtime/CUDA/memory.h"
 #include "flang/Runtime/allocatable.h"
+#include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
diff --git a/flang/runtime/CUDA/registration.cpp b/flang/runtime/CUDA/registration.cpp
index 22d43a7dc57a..20d274c4d8d1 100644
--- a/flang/runtime/CUDA/registration.cpp
+++ b/flang/runtime/CUDA/registration.cpp
@@ -7,6 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Runtime/CUDA/registration.h"
+#include "../terminator.h"
+#include "flang/Runtime/CUDA/common.h"
 
 #include "cuda_runtime.h"
 
@@ -31,5 +33,7 @@ void RTDEF(CUFRegisterFunction)(
   __cudaRegisterFunction(module, fctSym, fctName, fctName, -1, (uint3 *)0,
       (uint3 *)0, (dim3 *)0, (dim3 *)0, (int *)0);
 }
-}
+
+} // extern "C"
+
 } // namespace Fortran::runtime::cuda
diff --git a/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir b/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
new file mode 100644
index 000000000000..f10bd82f978d
--- /dev/null
+++ b/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
@@ -0,0 +1,104 @@
+// RUN: fir-opt --cuf-gpu-convert-to-llvm %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 20.0.0 (git@github.com:clementval/llvm-project.git ddcfd4d2dc17bf66cee8c3ef6284118684a2b0e6)", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
+  llvm.func @_QMmod1Phost_sub() {
+    %0 = llvm.mlir.constant(1 : i32) : i32
+    %1 = llvm.alloca %0 x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
+    %2 = llvm.mlir.constant(40 : i64) : i64
+    %3 = llvm.mlir.constant(16 : i32) : i32
+    %4 = llvm.mlir.constant(25 : i32) : i32
+    %5 = llvm.mlir.constant(21 : i32) : i32
+    %6 = llvm.mlir.constant(17 : i32) : i32
+    %7 = llvm.mlir.constant(1 : index) : i64
+    %8 = llvm.mlir.constant(27 : i32) : i32
+    %9 = llvm.mlir.constant(6 : i32) : i32
+    %10 = llvm.mlir.constant(1 : i32) : i32
+    %11 = llvm.mlir.constant(0 : i32) : i32
+    %12 = llvm.mlir.constant(10 : index) : i64
+    %13 = llvm.mlir.addressof @_QQclX91d13f6e74caa2f03965d7a7c6a8fdd5 : !llvm.ptr
+    %14 = llvm.call @_FortranACUFMemAlloc(%2, %11, %13, %6) : (i64, i32, !llvm.ptr, i32) -> !llvm.ptr
+    %15 = llvm.mlir.constant(10 : index) : i64
+    %16 = llvm.mlir.constant(1 : index) : i64
+    %17 = llvm.alloca %15 x i32 : (i64) -> !llvm.ptr
+    %18 = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+    %19 = llvm.insertvalue %17, %18[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> 
+    %20 = llvm.insertvalue %17, %19[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> 
+    %21 = llvm.mlir.constant(0 : index) : i64
+    %22 = llvm.insertvalue %21, %20[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> 
+    %23 = llvm.insertvalue %15, %22[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> 
+    %24 = llvm.insertvalue %16, %23[4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> 
+    %25 = llvm.extractvalue %24[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> 
+    %26 = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+    %27 = llvm.insertvalue %25, %26[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> 
+    %28 = llvm.insertvalue %25, %27[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> 
+    %29 = llvm.mlir.constant(0 : index) : i64
+    %30 = llvm.insertvalue %29, %28[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> 
+    %31 = llvm.mlir.constant(10 : index) : i64
+    %32 = llvm.insertvalue %31, %30[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> 
+    %33 = llvm.mlir.constant(1 : index) : i64
+    %34 = llvm.insertvalue %33, %32[4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> 
+    %35 = llvm.mlir.constant(1 : index) : i64
+    %36 = llvm.mlir.constant(11 : index) : i64
+    %37 = llvm.mlir.constant(1 : index) : i64
+    llvm.br ^bb1(%35 : i64)
+  ^bb1(%38: i64):  // 2 preds: ^bb0, ^bb2
+    %39 = llvm.icmp "slt" %38, %36 : i64
+    llvm.cond_br %39, ^bb2, ^bb3
+  ^bb2:  // pred: ^bb1
+    %40 = llvm.mlir.constant(-1 : index) : i64
+    %41 = llvm.add %38, %40 : i64
+    %42 = llvm.extractvalue %34[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> 
+    %43 = llvm.getelementptr %42[%41] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+    llvm.store %11, %43 : i32, !llvm.ptr
+    %44 = llvm.add %38, %37 : i64
+    llvm.br ^bb1(%44 : i64)
+  ^bb3:  // pred: ^bb1
+    %45 = llvm.call @_FortranACUFDataTransferPtrPtr(%14, %25, %2, %11, %13, %5) : (!llvm.ptr, !llvm.ptr, i64, i32, !llvm.ptr, i32) -> !llvm.struct<()>
+    gpu.launch_func  @cuda_device_mod::@_QMmod1Psub1 blocks in (%7, %7, %7) threads in (%12, %7, %7) : i64 dynamic_shared_memory_size %11 args(%14 : !llvm.ptr)
+    %46 = llvm.call @_FortranACUFDataTransferPtrPtr(%25, %14, %2, %10, %13, %4) : (!llvm.ptr, !llvm.ptr, i64, i32, !llvm.ptr, i32) -> !llvm.struct<()>
+    %47 = llvm.call @_FortranAioBeginExternalListOutput(%9, %13, %8) {fastmathFlags = #llvm.fastmath<contract>} : (i32, !llvm.ptr, i32) -> !llvm.ptr
+    %48 = llvm.mlir.constant(9 : i32) : i32
+    %49 = llvm.mlir.zero : !llvm.ptr
+    %50 = llvm.getelementptr %49[1] : (!llvm.ptr) -> !llvm.ptr, i32
+    %51 = llvm.ptrtoint %50 : !llvm.ptr to i64
+    %52 = llvm.mlir.undef : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>
+    %53 = llvm.insertvalue %51, %52[1] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> 
+    %54 = llvm.mlir.constant(20240719 : i32) : i32
+    %55 = llvm.insertvalue %54, %53[2] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> 
+    %56 = llvm.mlir.constant(1 : i32) : i32
+    %57 = llvm.trunc %56 : i32 to i8
+    %58 = llvm.insertvalue %57, %55[3] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> 
+    %59 = llvm.trunc %48 : i32 to i8
+    %60 = llvm.insertvalue %59, %58[4] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> 
+    %61 = llvm.mlir.constant(0 : i32) : i32
+    %62 = llvm.trunc %61 : i32 to i8
+    %63 = llvm.insertvalue %62, %60[5] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> 
+    %64 = llvm.mlir.constant(0 : i32) : i32
+    %65 = llvm.trunc %64 : i32 to i8
+    %66 = llvm.insertvalue %65, %63[6] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> 
+    %67 = llvm.mlir.constant(0 : i64) : i64
+    %68 = llvm.mlir.constant(1 : i64) : i64
+    %69 = llvm.insertvalue %68, %66[7, 0, 0] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> 
+    %70 = llvm.insertvalue %12, %69[7, 0, 1] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> 
+    %71 = llvm.insertvalue %51, %70[7, 0, 2] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> 
+    %72 = llvm.mul %51, %12 : i64
+    %73 = llvm.insertvalue %25, %71[0] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> 
+    llvm.store %73, %1 : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>, !llvm.ptr
+    llvm.return
+  }
+  llvm.func @_QMmod1Psub1(!llvm.ptr) -> ()
+  llvm.mlir.global linkonce constant @_QQclX91d13f6e74caa2f03965d7a7c6a8fdd5()  {addr_space = 0 : i32} : !llvm.array<2 x i8> {
+    %0 = llvm.mlir.constant("a\00") : !llvm.array<2 x i8>
+    llvm.return %0 : !llvm.array<2 x i8>
+  }
+  llvm.func @_FortranAioBeginExternalListOutput(i32, !llvm.ptr, i32) -> !llvm.ptr attributes {fir.io, fir.runtime, sym_visibility = "private"}
+  llvm.func @_FortranACUFMemAlloc(i64, i32, !llvm.ptr, i32) -> !llvm.ptr attributes {fir.runtime, sym_visibility = "private"}
+  llvm.func @_FortranACUFDataTransferPtrPtr(!llvm.ptr, !llvm.ptr, i64, i32, !llvm.ptr, i32) -> !llvm.struct<()> attributes {fir.runtime, sym_visibility = "private"}
+  llvm.func @_FortranACUFMemFree(!llvm.ptr, i32, !llvm.ptr, i32) -> !llvm.struct<()> attributes {fir.runtime, sym_visibility = "private"}
+  gpu.binary @cuda_device_mod  [#gpu.object<#nvvm.target, "">]
+}
+
+// CHECK-LABEL: _QMmod1Phost_sub
+
+// CHECK: %[[KERNEL_PTR:.*]] = llvm.mlir.addressof @_QMmod1Psub1 : !llvm.ptr
+// CHECK: llvm.call @_FortranACUFLaunchKernel(%[[KERNEL_PTR]], {{.*}})
diff --git a/flang/tools/fir-opt/fir-opt.cpp b/flang/tools/fir-opt/fir-opt.cpp
index 84a74770cf03..5f6a856116bc 100644
--- a/flang/tools/fir-opt/fir-opt.cpp
+++ b/flang/tools/fir-opt/fir-opt.cpp
@@ -43,6 +43,7 @@ int main(int argc, char **argv) {
   DialectRegistry registry;
   fir::support::registerDialects(registry);
   registry.insert<mlir::gpu::GPUDialect>();
+  registry.insert<mlir::NVVM::NVVMDialect>();
   fir::support::addFIRExtensions(registry);
   return failed(MlirOptMain(argc, argv, "FIR modular optimizer driver\n",
       registry));
-- 
GitLab


From 9ed6f7f99b556faa6dd24ff3a97bab1ab3c4fece Mon Sep 17 00:00:00 2001
From: Chris Apple <cja-private@pm.me>
Date: Wed, 23 Oct 2024 15:40:05 -0700
Subject: [PATCH 022/693] [rtsan] Add include guards around posix interceptors,
 tests (#113188)

---
 compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp          | 6 +++++-
 .../lib/rtsan/tests/rtsan_test_interceptors_posix.cpp       | 6 +++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp b/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
index 63b0ca28a1f4..890d6c11c407 100644
--- a/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
@@ -8,12 +8,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_POSIX
+
 #include "rtsan/rtsan_interceptors.h"
 
 #include "interception/interception.h"
 #include "sanitizer_common/sanitizer_allocator_dlsym.h"
 #include "sanitizer_common/sanitizer_allocator_internal.h"
-#include "sanitizer_common/sanitizer_platform.h"
 #include "sanitizer_common/sanitizer_platform_interceptors.h"
 
 #include "interception/interception.h"
@@ -608,3 +610,5 @@ void __rtsan::InitializeInterceptors() {
   INTERCEPT_FUNCTION(recvfrom);
   INTERCEPT_FUNCTION(shutdown);
 }
+
+#endif // SANITIZER_POSIX
diff --git a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
index f7a281f13e2f..6233c3e91800 100644
--- a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
+++ b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
@@ -8,9 +8,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_POSIX
+
 #include "gtest/gtest.h"
 
-#include "sanitizer_common/sanitizer_platform.h"
 #include "sanitizer_common/sanitizer_platform_interceptors.h"
 
 #include "rtsan_test_utilities.h"
@@ -704,3 +706,5 @@ TEST(TestRtsanInterceptors, ShutdownOnASocketDiesWhenRealtime) {
   ExpectRealtimeDeath(Func, "shutdown");
   ExpectNonRealtimeSurvival(Func);
 }
+
+#endif // SANITIZER_POSIX
-- 
GitLab


From 33363521ca24f912cc25530f6cecbca53acce8a3 Mon Sep 17 00:00:00 2001
From: Jordan Rupprecht <rupprecht@google.com>
Date: Wed, 23 Oct 2024 18:51:25 -0500
Subject: [PATCH 023/693] [NFC][WebAssembly] Inline var only used in assertion
 (#113507)

---
 llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 8611c05be89a..fb9f980cfff2 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -1635,8 +1635,7 @@ SDValue WebAssemblyTargetLowering::LowerMUL_LOHI(SDValue Op,
 SDValue WebAssemblyTargetLowering::Replace128Op(SDNode *N,
                                                 SelectionDAG &DAG) const {
   assert(Subtarget->hasWideArithmetic());
-  auto ValTy = N->getValueType(0);
-  assert(ValTy == MVT::i128);
+  assert(N->getValueType(0) == MVT::i128);
   SDLoc DL(N);
   unsigned Opcode;
   switch (N->getOpcode()) {
-- 
GitLab


From 786db636b9b6967087685a62af665046621b4957 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Wed, 23 Oct 2024 19:55:32 -0700
Subject: [PATCH 024/693] [clang-format] Add KeepFormFeed option (#113268)

Closes #113170.
---
 clang/docs/ClangFormatStyleOptions.rst     |  8 +++
 clang/docs/ReleaseNotes.rst                |  1 +
 clang/include/clang/Format/Format.h        | 10 +++-
 clang/lib/Format/Format.cpp                |  3 ++
 clang/lib/Format/FormatToken.h             |  3 ++
 clang/lib/Format/FormatTokenLexer.cpp      |  8 +++
 clang/lib/Format/WhitespaceManager.cpp     | 23 ++++----
 clang/lib/Format/WhitespaceManager.h       |  2 +-
 clang/unittests/Format/ConfigParseTest.cpp |  7 +--
 clang/unittests/Format/FormatTest.cpp      | 61 +++++++++++++++++++++-
 10 files changed, 110 insertions(+), 16 deletions(-)

diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index f36a5472b7e1..9ef1fd5f36d1 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -4663,6 +4663,14 @@ the configuration (without a prefix: ``Auto``).
 **KeepEmptyLinesAtTheStartOfBlocks** (``Boolean``) :versionbadge:`clang-format 3.7` :ref:`¶ <KeepEmptyLinesAtTheStartOfBlocks>`
   This option is deprecated. See ``AtStartOfBlock`` of ``KeepEmptyLines``.
 
+.. _KeepFormFeed:
+
+**KeepFormFeed** (``Boolean``) :versionbadge:`clang-format 20` :ref:`¶ <KeepFormFeed>`
+  Keep the form feed character if it's immediately preceded and followed by
+  a newline. Multiple form feeds and newlines within a whitespace range are
+  replaced with a single newline and form feed followed by the remaining
+  newlines.
+
 .. _LambdaBodyIndentation:
 
 **LambdaBodyIndentation** (``LambdaBodyIndentationKind``) :versionbadge:`clang-format 13` :ref:`¶ <LambdaBodyIndentation>`
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 28bb83a1c9d6..ce046a305c89 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -710,6 +710,7 @@ clang-format
   multi-line comments without touching their contents, renames ``false`` to
   ``Never``, and ``true`` to ``Always``.
 - Adds ``RemoveEmptyLinesInUnwrappedLines`` option.
+- Adds ``KeepFormFeed`` option and set it to ``true`` for ``GNU`` style.
 
 libclang
 --------
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index debba1c78228..a5fd7750c20b 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -3207,6 +3207,13 @@ struct FormatStyle {
   /// \version 3.7
   // bool KeepEmptyLinesAtTheStartOfBlocks;
 
+  /// Keep the form feed character if it's immediately preceded and followed by
+  /// a newline. Multiple form feeds and newlines within a whitespace range are
+  /// replaced with a single newline and form feed followed by the remaining
+  /// newlines.
+  /// \version 20
+  bool KeepFormFeed;
+
   /// Indentation logic for lambda bodies.
   enum LambdaBodyIndentationKind : int8_t {
     /// Align lambda body relative to the lambda signature. This is the default.
@@ -5222,7 +5229,8 @@ struct FormatStyle {
            JavaImportGroups == R.JavaImportGroups &&
            JavaScriptQuotes == R.JavaScriptQuotes &&
            JavaScriptWrapImports == R.JavaScriptWrapImports &&
-           KeepEmptyLines == R.KeepEmptyLines && Language == R.Language &&
+           KeepEmptyLines == R.KeepEmptyLines &&
+           KeepFormFeed == R.KeepFormFeed && Language == R.Language &&
            LambdaBodyIndentation == R.LambdaBodyIndentation &&
            LineEnding == R.LineEnding && MacroBlockBegin == R.MacroBlockBegin &&
            MacroBlockEnd == R.MacroBlockEnd && Macros == R.Macros &&
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index f85fe440f50a..0cf4cdbeab31 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -1052,6 +1052,7 @@ template <> struct MappingTraits<FormatStyle> {
     IO.mapOptional("JavaScriptQuotes", Style.JavaScriptQuotes);
     IO.mapOptional("JavaScriptWrapImports", Style.JavaScriptWrapImports);
     IO.mapOptional("KeepEmptyLines", Style.KeepEmptyLines);
+    IO.mapOptional("KeepFormFeed", Style.KeepFormFeed);
     IO.mapOptional("LambdaBodyIndentation", Style.LambdaBodyIndentation);
     IO.mapOptional("LineEnding", Style.LineEnding);
     IO.mapOptional("MacroBlockBegin", Style.MacroBlockBegin);
@@ -1567,6 +1568,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
       /*AtStartOfBlock=*/true,
       /*AtStartOfFile=*/true,
   };
+  LLVMStyle.KeepFormFeed = false;
   LLVMStyle.LambdaBodyIndentation = FormatStyle::LBI_Signature;
   LLVMStyle.Language = Language;
   LLVMStyle.LineEnding = FormatStyle::LE_DeriveLF;
@@ -1927,6 +1929,7 @@ FormatStyle getGNUStyle() {
   Style.ColumnLimit = 79;
   Style.Cpp11BracedListStyle = false;
   Style.FixNamespaceComments = false;
+  Style.KeepFormFeed = true;
   Style.SpaceBeforeParens = FormatStyle::SBPO_Always;
   return Style;
 }
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index 7d342a7dcca0..f6bb860a1fea 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -586,6 +586,9 @@ public:
   /// Might be function declaration open/closing paren.
   bool MightBeFunctionDeclParen = false;
 
+  /// Has "\n\f\n" or "\n\f\r\n" before TokenText.
+  bool HasFormFeedBefore = false;
+
   /// Number of optional braces to be inserted after this token:
   ///   -1: a single left brace
   ///    0: no braces
diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp
index 2cdf6cd286b2..7a264bddcdfe 100644
--- a/clang/lib/Format/FormatTokenLexer.cpp
+++ b/clang/lib/Format/FormatTokenLexer.cpp
@@ -1186,6 +1186,14 @@ FormatToken *FormatTokenLexer::getNextToken() {
         Column = 0;
         break;
       case '\f':
+        if (Style.KeepFormFeed && !FormatTok->HasFormFeedBefore &&
+            // The form feed is immediately preceded and followed by a newline.
+            i > 0 && Text[i - 1] == '\n' &&
+            ((i + 1 < e && Text[i + 1] == '\n') ||
+             (i + 2 < e && Text[i + 1] == '\r' && Text[i + 2] == '\n'))) {
+          FormatTok->HasFormFeedBefore = true;
+        }
+        [[fallthrough]];
       case '\v':
         Column = 0;
         break;
diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp
index b6b24672f6a3..b1e43f0313db 100644
--- a/clang/lib/Format/WhitespaceManager.cpp
+++ b/clang/lib/Format/WhitespaceManager.cpp
@@ -1674,7 +1674,7 @@ void WhitespaceManager::generateChanges() {
                                  C.PreviousEndOfTokenColumn,
                                  C.EscapedNewlineColumn);
       } else {
-        appendNewlineText(ReplacementText, C.NewlinesBefore);
+        appendNewlineText(ReplacementText, C);
       }
       // FIXME: This assert should hold if we computed the column correctly.
       // assert((int)C.StartOfTokenColumn >= C.Spaces);
@@ -1706,15 +1706,18 @@ void WhitespaceManager::storeReplacement(SourceRange Range, StringRef Text) {
   }
 }
 
-void WhitespaceManager::appendNewlineText(std::string &Text,
-                                          unsigned Newlines) {
-  if (UseCRLF) {
-    Text.reserve(Text.size() + 2 * Newlines);
-    for (unsigned i = 0; i < Newlines; ++i)
-      Text.append("\r\n");
-  } else {
-    Text.append(Newlines, '\n');
-  }
+void WhitespaceManager::appendNewlineText(std::string &Text, const Change &C) {
+  if (C.NewlinesBefore <= 0)
+    return;
+
+  StringRef Newline = UseCRLF ? "\r\n" : "\n";
+  Text.append(Newline);
+
+  if (C.Tok->HasFormFeedBefore)
+    Text.append("\f");
+
+  for (unsigned I = 1; I < C.NewlinesBefore; ++I)
+    Text.append(Newline);
 }
 
 void WhitespaceManager::appendEscapedNewlineText(
diff --git a/clang/lib/Format/WhitespaceManager.h b/clang/lib/Format/WhitespaceManager.h
index 7b91d8bf4db7..5dd667fb4ba3 100644
--- a/clang/lib/Format/WhitespaceManager.h
+++ b/clang/lib/Format/WhitespaceManager.h
@@ -349,7 +349,7 @@ private:
 
   /// Stores \p Text as the replacement for the whitespace in \p Range.
   void storeReplacement(SourceRange Range, StringRef Text);
-  void appendNewlineText(std::string &Text, unsigned Newlines);
+  void appendNewlineText(std::string &Text, const Change &C);
   void appendEscapedNewlineText(std::string &Text, unsigned Newlines,
                                 unsigned PreviousEndOfTokenColumn,
                                 unsigned EscapedNewlineColumn);
diff --git a/clang/unittests/Format/ConfigParseTest.cpp b/clang/unittests/Format/ConfigParseTest.cpp
index 9e8529050ed8..7fc749227166 100644
--- a/clang/unittests/Format/ConfigParseTest.cpp
+++ b/clang/unittests/Format/ConfigParseTest.cpp
@@ -165,24 +165,25 @@ TEST(ConfigParseTest, ParsesConfigurationBools) {
   CHECK_PARSE_BOOL(BreakBeforeTernaryOperators);
   CHECK_PARSE_BOOL(BreakStringLiterals);
   CHECK_PARSE_BOOL(CompactNamespaces);
+  CHECK_PARSE_BOOL(Cpp11BracedListStyle);
   CHECK_PARSE_BOOL(DerivePointerAlignment);
   CHECK_PARSE_BOOL_FIELD(DerivePointerAlignment, "DerivePointerBinding");
   CHECK_PARSE_BOOL(DisableFormat);
   CHECK_PARSE_BOOL(IndentAccessModifiers);
-  CHECK_PARSE_BOOL(IndentCaseLabels);
   CHECK_PARSE_BOOL(IndentCaseBlocks);
+  CHECK_PARSE_BOOL(IndentCaseLabels);
   CHECK_PARSE_BOOL(IndentGotoLabels);
-  CHECK_PARSE_BOOL_FIELD(IndentRequiresClause, "IndentRequires");
   CHECK_PARSE_BOOL(IndentRequiresClause);
+  CHECK_PARSE_BOOL_FIELD(IndentRequiresClause, "IndentRequires");
   CHECK_PARSE_BOOL(IndentWrappedFunctionNames);
   CHECK_PARSE_BOOL(InsertBraces);
   CHECK_PARSE_BOOL(InsertNewlineAtEOF);
   CHECK_PARSE_BOOL_FIELD(KeepEmptyLines.AtEndOfFile, "KeepEmptyLinesAtEOF");
   CHECK_PARSE_BOOL_FIELD(KeepEmptyLines.AtStartOfBlock,
                          "KeepEmptyLinesAtTheStartOfBlocks");
+  CHECK_PARSE_BOOL(KeepFormFeed);
   CHECK_PARSE_BOOL(ObjCSpaceAfterProperty);
   CHECK_PARSE_BOOL(ObjCSpaceBeforeProtocolList);
-  CHECK_PARSE_BOOL(Cpp11BracedListStyle);
   CHECK_PARSE_BOOL(RemoveBracesLLVM);
   CHECK_PARSE_BOOL(RemoveEmptyLinesInUnwrappedLines);
   CHECK_PARSE_BOOL(RemoveSemicolon);
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index 8f4c92148ada..cdb68b74ca79 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -28135,7 +28135,7 @@ TEST_F(FormatTest, BreakBinaryOperations) {
                Style);
 }
 
-TEST_F(FormatTest, RemovesEmptyLinesInUnwrappedLines) {
+TEST_F(FormatTest, RemoveEmptyLinesInUnwrappedLines) {
   auto Style = getLLVMStyle();
   Style.RemoveEmptyLinesInUnwrappedLines = true;
 
@@ -28212,6 +28212,65 @@ TEST_F(FormatTest, RemovesEmptyLinesInUnwrappedLines) {
                Style);
 }
 
+TEST_F(FormatTest, KeepFormFeed) {
+  auto Style = getLLVMStyle();
+  Style.KeepFormFeed = true;
+
+  constexpr StringRef NoFormFeed{"int i;\n"
+                                 "\n"
+                                 "void f();"};
+  verifyFormat(NoFormFeed,
+               "int i;\n"
+               " \f\n"
+               "void f();",
+               Style);
+  verifyFormat(NoFormFeed,
+               "int i;\n"
+               "\n"
+               "\fvoid f();",
+               Style);
+  verifyFormat(NoFormFeed,
+               "\fint i;\n"
+               "\n"
+               "void f();",
+               Style);
+  verifyFormat(NoFormFeed,
+               "int i;\n"
+               "\n"
+               "void f();\f",
+               Style);
+
+  constexpr StringRef FormFeed{"int i;\n"
+                               "\f\n"
+                               "void f();"};
+  verifyNoChange(FormFeed, Style);
+
+  Style.LineEnding = FormatStyle::LE_LF;
+  verifyFormat(FormFeed,
+               "int i;\r\n"
+               "\f\r\n"
+               "void f();",
+               Style);
+
+  constexpr StringRef FormFeedBeforeEmptyLine{"int i;\n"
+                                              "\f\n"
+                                              "\n"
+                                              "void f();"};
+  Style.MaxEmptyLinesToKeep = 2;
+  verifyFormat(FormFeedBeforeEmptyLine,
+               "int i;\n"
+               "\n"
+               "\f\n"
+               "void f();",
+               Style);
+  verifyFormat(FormFeedBeforeEmptyLine,
+               "int i;\n"
+               "\f\n"
+               "\f\n"
+               "void f();",
+               Style);
+}
+
 } // namespace
 } // namespace test
 } // namespace format
-- 
GitLab


From bf8f5cc9b50b411ab0d8ee6532c6189992d790db Mon Sep 17 00:00:00 2001
From: Yuta Saito <kateinoigakukun@gmail.com>
Date: Thu, 24 Oct 2024 12:52:50 +0900
Subject: [PATCH 025/693] Reland: [llvm-cov][WebAssembly] Read
 `__llvm_prf_names` from data segments (#112569)

On WebAssembly, most coverage metadata contents read by llvm-cov (like
`__llvm_covmap` and `__llvm_covfun`) are stored in custom sections
because they are not referenced at runtime. However, `__llvm_prf_names`
is referenced at runtime by the profile runtime library and is read by
llvm-cov post-processing tools, so it needs to be stored in a data
segment, which is allocatable at runtime and accessible by tools as long
as "name" section is present in the binary.

This patch changes the way llvm-cov reads `__llvm_prf_names` on
WebAssembly. Instead of looking for a section, it looks for a data
segment with the same name.

This reverts commit 157f10ddf2d851125a85a71e530dc9d50cb032a2 and fixes
PE/COFF `.lprfn$A` section handling.
---
 .../Coverage/CoverageMappingReader.cpp        |  92 ++++++++++++++----
 .../llvm-cov/Inputs/binary-formats.v6.wasm32  | Bin 0 -> 87781 bytes
 .../Inputs/binary-formats.wasm.proftext       |   4 +
 llvm/test/tools/llvm-cov/binary-formats.c     |   7 ++
 4 files changed, 85 insertions(+), 18 deletions(-)
 create mode 100755 llvm/test/tools/llvm-cov/Inputs/binary-formats.v6.wasm32
 create mode 100644 llvm/test/tools/llvm-cov/Inputs/binary-formats.wasm.proftext

diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
index 8881bffe41c5..e507b241d982 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
@@ -18,12 +18,14 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/BinaryFormat/Wasm.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/COFF.h"
 #include "llvm/Object/Error.h"
 #include "llvm/Object/MachOUniversal.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Object/Wasm.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compression.h"
@@ -492,22 +494,29 @@ Expected<bool> RawCoverageMappingDummyChecker::isDummy() {
   return Tag == Counter::Zero;
 }
 
-Error InstrProfSymtab::create(SectionRef &Section) {
-  Expected<StringRef> DataOrErr = Section.getContents();
-  if (!DataOrErr)
-    return DataOrErr.takeError();
-  Data = *DataOrErr;
-  Address = Section.getAddress();
-
+/// Determine if we should skip the first byte of the section content
+static bool shouldSkipSectionFirstByte(SectionRef &Section) {
+  const ObjectFile *Obj = Section.getObject();
   // If this is a linked PE/COFF file, then we have to skip over the null byte
   // that is allocated in the .lprfn$A section in the LLVM profiling runtime.
   // If the name section is .lprfcovnames, it doesn't have the null byte at the
   // beginning.
-  const ObjectFile *Obj = Section.getObject();
   if (isa<COFFObjectFile>(Obj) && !Obj->isRelocatableObject())
     if (Expected<StringRef> NameOrErr = Section.getName())
       if (*NameOrErr != getInstrProfSectionName(IPSK_covname, Triple::COFF))
-        Data = Data.drop_front(1);
+        return true;
+  return false;
+}
+
+Error InstrProfSymtab::create(SectionRef &Section) {
+  Expected<StringRef> DataOrErr = Section.getContents();
+  if (!DataOrErr)
+    return DataOrErr.takeError();
+  Data = *DataOrErr;
+  Address = Section.getAddress();
+
+  if (shouldSkipSectionFirstByte(Section))
+    Data = Data.substr(1);
 
   return Error::success();
 }
@@ -1077,6 +1086,56 @@ lookupSections(ObjectFile &OF, InstrProfSectKind IPSK) {
   return Sections;
 }
 
+/// Find a section that matches \p Name and is allocatable at runtime.
+///
+/// Returns the contents of the section and its start offset in the object file.
+static Expected<std::pair<StringRef, uint64_t>>
+lookupAllocatableSection(ObjectFile &OF, InstrProfSectKind IPSK) {
+  // On Wasm, allocatable sections can live only in data segments.
+  if (auto *WOF = dyn_cast<WasmObjectFile>(&OF)) {
+    std::vector<const WasmSegment *> Segments;
+    auto ObjFormat = OF.getTripleObjectFormat();
+    auto Name =
+        getInstrProfSectionName(IPSK, ObjFormat, /*AddSegmentInfo=*/false);
+    for (const auto &DebugName : WOF->debugNames()) {
+      if (DebugName.Type != wasm::NameType::DATA_SEGMENT ||
+          DebugName.Name != Name)
+        continue;
+      if (DebugName.Index >= WOF->dataSegments().size())
+        return make_error<CoverageMapError>(coveragemap_error::malformed);
+      auto &Segment = WOF->dataSegments()[DebugName.Index];
+      Segments.push_back(&Segment);
+    }
+    if (Segments.empty())
+      return make_error<CoverageMapError>(coveragemap_error::no_data_found);
+    if (Segments.size() != 1)
+      return make_error<CoverageMapError>(coveragemap_error::malformed);
+
+    const auto &Segment = *Segments.front();
+    auto &Data = Segment.Data;
+    StringRef Content(reinterpret_cast<const char *>(Data.Content.data()),
+                      Data.Content.size());
+    return std::make_pair(Content, Segment.SectionOffset);
+  }
+
+  // On other object file types, delegate to lookupSections to find the section.
+  auto Sections = lookupSections(OF, IPSK);
+  if (!Sections)
+    return Sections.takeError();
+  if (Sections->size() != 1)
+    return make_error<CoverageMapError>(
+        coveragemap_error::malformed,
+        "the size of coverage mapping section is not one");
+  auto &Section = Sections->front();
+  auto ContentsOrErr = Section.getContents();
+  if (!ContentsOrErr)
+    return ContentsOrErr.takeError();
+  auto Content = *ContentsOrErr;
+  if (shouldSkipSectionFirstByte(Section))
+    Content = Content.drop_front(1);
+  return std::make_pair(Content, Section.getAddress());
+}
+
 static Expected<std::unique_ptr<BinaryCoverageReader>>
 loadBinaryFormat(std::unique_ptr<Binary> Bin, StringRef Arch,
                  StringRef CompilationDir = "",
@@ -1107,23 +1166,20 @@ loadBinaryFormat(std::unique_ptr<Binary> Bin, StringRef Arch,
 
   // Look for the sections that we are interested in.
   auto ProfileNames = std::make_unique<InstrProfSymtab>();
-  std::vector<SectionRef> NamesSectionRefs;
   // If IPSK_name is not found, fallback to search for IPK_covname, which is
   // used when binary correlation is enabled.
-  auto NamesSection = lookupSections(*OF, IPSK_name);
+  auto NamesSection = lookupAllocatableSection(*OF, IPSK_name);
   if (auto E = NamesSection.takeError()) {
     consumeError(std::move(E));
-    NamesSection = lookupSections(*OF, IPSK_covname);
+    NamesSection = lookupAllocatableSection(*OF, IPSK_covname);
     if (auto E = NamesSection.takeError())
       return std::move(E);
   }
-  NamesSectionRefs = *NamesSection;
 
-  if (NamesSectionRefs.size() != 1)
-    return make_error<CoverageMapError>(
-        coveragemap_error::malformed,
-        "the size of coverage mapping section is not one");
-  if (Error E = ProfileNames->create(NamesSectionRefs.back()))
+  uint64_t NamesAddress;
+  StringRef NamesContent;
+  std::tie(NamesContent, NamesAddress) = *NamesSection;
+  if (Error E = ProfileNames->create(NamesContent, NamesAddress))
     return std::move(E);
 
   auto CoverageSection = lookupSections(*OF, IPSK_covmap);
diff --git a/llvm/test/tools/llvm-cov/Inputs/binary-formats.v6.wasm32 b/llvm/test/tools/llvm-cov/Inputs/binary-formats.v6.wasm32
new file mode 100755
index 0000000000000000000000000000000000000000..5a606d5a2f69fd150690bd17ebb9d922cc17c383
GIT binary patch
literal 87781
zcmd4437B0~mG^zmxwod=O6nx3kV;Y{dsiw@AqfNsNeBUQLa0hem?D#k5{5vy6(E;X
z7&Ihd6cuDp5fu~^2?9}^XGKNDc4$<z>27Ut=oZ_d(bj&Qx_N*9wfDJIl>vR<=X<{I
z(<0}dbN0CQ+H0---)pZ^I(PFWNvTvy{wbNhF4<mMSKhvTd$OIM{FB<x62F>wYBxM8
ztt%<nw(ZF_Qf!NFx!u055>r)ptNS*~wk_P0NS`Qqvpo&j^>6bwrJ!8iZVP3UV%DNO
z-ft_#j4DE{jExAj>S3!=$I8;J+mo%a*`@Thrt$k-dG6+m&)IzGxtlg`xNPX0O_#5~
z;^OsJ9@={D<ri%}=c4sPrSd*fjI$J*FMief7f)ARf9Vw$Uw+x8FIw36BGF$jul2$U
z&N+YMWt-QR(tWgSEGaI$VDr$qLoZtA#NlL{*AJa@;l^_>+Pu%+b}I438`oQ@O0I^M
z+V;@`wGm35dqHW;K2nSeDgLHLQj(>+;Nr{Ax%Avi)|Zlf6xu?eR%`V|5^P?-ez4TO
zj|6J?m6u;Uw7#^jA|`D*cWA>o=Tptl`g3T}`tye_yZq{XHmS{$U$$xerKPd^sIzU;
z<(Hj*&ibn^9x9c}KS_R+v^2GpE5-lP%D9ekm2uhl@e{^Z##bgbCuwIXNs~#{q%y9p
zElta*{@0RL_N$c3Nu{Y=Zk}9jPPi$zRJzJ7X$$|Wv{cecIc=#_%2UWfMLa51l8`xV
zuC$~n56jJMQ%lNOPRr%CG3BJ%UM){6ku0IKmU1<rv@}s0TGCPr84~JGlBBgYNlI!*
zIUz}!R4K5Ul)KAGQ`i12WFrklP@S%oressH-+oPH@-%OnxIM{l|4XI1y_A>q&tvoT
z@3SNR{jL3e$o>s)EusDM%)v^X=$~I6)cE<tf7;6j{%DtRdj5EL)KP7jm^NRs{t`wl
zEj6D5XI(y2n%LHpsFX>EoN&x>t4}=X<Wtt1KD7SSo0E?xUrN4`d@*@E`AIV7=U=V-
zD5;dPtEy12x;9C+r_0i1Gq2sTV@E09@d5s=^`)h;rBP4vWMNv$?ogWSc9NESDf@)3
zt~563s8WzCW%uhrsru4#WqWyBx_u_8NtouDFT2#02Ytm=Txnt2&V}+?^OA3JO|#1n
zJ*ITc%M;(?nqA9QuQzS3&9yEH@3<Y~#&lHoOVaK9rqaIjvEM6G;;6lYKFqw%D&tkW
zzkav-8+N<Daku-McDsMgZud9ucK_Pp`&mh)=xjn~Q<u1OkUp2Kvox4~cT|-szd`i%
zk+y2RE7i!5LUCp2s7_3hZOgBH&s+ce;2ZAw=yr9rqx#Bnd3(#Y^1kX=RBm<U+2vMW
zS>6IcE3WB!h}jA;A?G!|m8X!>wOubF?)26Cg~YeJs%x+Nv2JWfwJSv*ZkzeKqC>OG
zk*-%K<G1HGzgUYy-+WWL%$1KHTgq>-#La<qLIY~P)wQa7Z6qG!+I-cG3Dpm%df!xa
zO@3US_zpMDb<}<4vX1Jvl5z)aYVc4gcMz-Ol0gxxmzKhZcUgmtwHBt`*<0+k0;g3L
zrd`?F?6!&9riE!|_IA5%=C*lZnq_y}Z40+8@MHE?yKUvR6~5GPbVD;PT{M|XC#~j^
z(Qq#9G@MJT9|w<)5hdMNH!gdR71BXdGYZOTe!LqWn(8__s_!eewON<ct;)7>+s6)f
zuQ`9J6nKA7)QXu}#Xf5oIp)erF6qK_1()(dsNJegF6Bb!7K^@%HO77EQjyq=8Dx}8
zfyzTh<#BG@E-2jUIu@1ZE8!X%$z>&t%-s0wL2LU25=>ZB?$etUexfeRe5ae}I<uXY
zU=j%?Eh;<LI=kHG_tVAs$-2z<U3r(E!uu&TKh;h3)0hd}+;-Re{%(IiJ#Y0hxSdh+
z&N=Vher~esa?{*&$n}s4r@=k0CwqrgG&6su<a^yr*IV<m+$?{9J7AYTFjU*;4s?B6
z{cPQJ`GeeSchFWpC$wd*o8#tg_49O>`Geg&ckot!i0<b5`R))mf2%)Kcdh;~cc?pT
zt3O<%UqHha)cg_dh)DXwJE|MYP5YK&G1(|-<|^V0=ASsD*)_T5?7h~PmgRknYKv>#
z#vI5Ppo$w)gQ2Ril(vrQHBC*%8|g4lj24@JV}*}|3bTjp3M;`CZM&$l#f{kpPl$ml
zuDvFnh(%UAs?VhzP1{ohj11T$UMu`g?Z1ZLY0l^Gl}n9dTw;9T(naMi$O)Hra)C?X
zGjVBq_NX<gP1?q_jV+10I{ldCB}oKVT~y|Y*s0yMXZKsuv658dG`H&TZpW^`vYF<U
z%VPR2Kfz6i=_e|^v1{E=a+7w6K{Y0mVYU58-=0)eMW63l4tGs~{ibXcuV%Zfx@n<B
z-LBhBYqSR5jjyNcbp{j9aMK%=!^`ot4|T!9&hxqot!C2mnKjtBH_*DHdSjAGuKiWb
z*x)5m4($<Gsv_A{(jC$wk4#mAsglO5*UL(n_=A2dX^ecAihPo+yF|N=>OEL7+ncww
z78<1lW9%Vb-(^H8FXv+a%TQxwSGnFdF=X(!SRwHB+TkYDebvUdDs3P&le%`s9?CQS
zk(zEScP2~Q5&9YJN?k`yO^fU}uA}<!Zo>cnU?%LFT#YHgJbW>d#dLd+jkZo;>P)C1
z7AMBGLdhzo2ZGDg^v{xJO`#^EY*R7F3Ho59MX;!xNn1nBs4bhKwrs}sX{tfh7LD(S
zxr?~CjyY1+9EqB~ogo<OAR!9W^yV2pgWM*|>wba_!GvLjJTX?$>6jfg%?(wtpW9DU
zW0&s==7UX$YnTvaH<d}um{H3#?wAc-)Gukv<mjlLnzR(0GkR=B8h4R}XTBchF@9+<
zuPhVTv|W9a`o%VSZ6#S&6j_E~#%Y5H)22x3CKcW7nxqT6xS$I=U6a&wb6ycAx)vE4
zP14`(3`iS4+mJo!vhp&eli1}VJ#N;AsOQG6+RgAder%#tT@cdpOrY$EZi1UQgjgBx
zIwR>Skj|1#4#~PmX2ctUc$3}K8dRI6Dcg;f>)wd|YjXR`uCPqgL#7#Ia>x=lgfc`4
z)};M3EHi07n$OR2y{Nre?tlS*AY*G<ukL5t*v@8bRZKhT?;vHG6Uv?&%AQBrru&A_
zeRJI*HFV#6B|B8_rTgYMv|si!8SGOf7OiE6hujOueS}-!ju`R_-QjLw%`b9`RM%nz
z_hM?UTKZOhq|&$hC2onzKFS^Cj;#5mZi!neF~z&gA5F`TaYwsjYW`TLa4bX8PJX($
zjA71sl-K-nx4c&(rsS4&RBua@{R>tYZB55C(W-uvnVOZz*>H-h)DTO}YDx=C3uCqB
zcUvOk3nsm5)5|fu93!b(7Ac~gRLmwfZoqfY=FD}ttnSCt<MBwc`>dP^u`(op1VJYS
zb*h#fkC&KJWjBdZnJ1#jexdZqx>PWY?)weh^HX90=p30za7i$OGK$9llEGAt22<?p
zw2taql4NW_kP2zc!XH#u!*px5wG?;E2{eQ>Z%Jg(G~?2kX7;eXFv9~0cwb>kSTgjk
ztBOmxlXS6%*o`K#!(9jCW|UzVJF1VQi7y)VA|n+DVoI!HD}^QrrrV98`&cQ2EUZ@4
z5)38RHs1IHtr~{CJrWW#M%E0iRLa;qi9v+*csF6dPh>0$i-575G+gz5u?RB`YRJGY
z6)`1NFmfK3B=y{s*aS$hKjX%HE=Bg5&YYiKGs8enWo0?W(bbsPK8#~W^|O^EFNjsf
zHrTs_>w}GvaJ^g@O{Y2Jq-*=qK1LJ8kRjxnO@oDpMx%CU)F!1Lq}u9HQqAN=7Ue}B
zv>Bg$SllKp;Y>$tN5(a|i3`)Y?P-|Wz0j;lye*|NiJx$`iCZVh=k`MeLk6Uun`}~9
ztwbuTl~ddl@!V9`1$$40a?@P*YQI0_On3Xc>2*KD8Z@KPAm^M252<wN38l<*Gj-Ps
zDQ8j3teR&55XE4x!~AzW<B?rEN+k(ICvli>M;11@gVc?3HOjO;1mhe{1rv;YtPAFe
zh6f`Q53V5_o7^Gj3)R`~=OgxzkM7VK^6@Z#IHGTU_P#hV7w{si5d1>YE#h`j%`bL~
z{gHHSi96CQbW5`LEA><r*NQkadMtHIb$2wv@#sNBp*=j-9qW#Px+VuHY8j0-nL5|!
z@Wyf;EwB0G+;P1WxTECqj_T{uWNN`bqfLsK#>66;T<Xgv8<N6Et?3pXn{<>|FzHyL
z;g}QBm?jImf`F8!Gyy3P#;gue$cjZmXjGVVx{M1Cu`G3mQRBx`;sh-xQJLe4@tMf@
zObp`_1!gyF7V=J_iv8RqF_h5)acLZ7ue%YO4t^@f6=m#aS4Z`~lVnmsi+#@}hFo(=
zQ;NCN8{q<5m@%~?T+qT>G(~rW3rtf8JqU0?v?4kqT!0B7v6)4<fDx0xH1?Ux^kZlp
z#NpBnakzA0n{nxcI9xRMxJWc{Y1gFdOEuFBJ&?XJ!fCPnQH{Ev1S{BdtI1634XiM!
zqx$1<2g-V{usloS5>3|>+f)Cwf-bc2Ur3{8VHr1rcg-XItMa?Te-jH*w&hQy!wZRG
zbqcVzFnuyb2t&*#bcLFa>x!cJw63i2KCUZEl@II6BIPk%q3ff%!e9@jSy)cAH<lAx
z4$$-K{kY{wZZ}SfMUvZ%U+X8pNM?7{Ax~9;=m84q<6l3?cv^CtY-Yh~s`KuN6cNPr
z@if?C5z-J8F}dm{BaNyuWbZBt>b8QqLqUd63>P#b7G(BWT*V`x{Pe1uj)<zdo<-%`
zt)Ne*ex?$CJeAGndLvFUi!`$qm2WQ691znS=nilPW?_}yN1DDx<sC(XhAlg01!kR}
z6C3nsiuj_A$5Q0iTsLnJ`E{_LQ+0EYV28-GdlbP{b%!o0{~>YJ+4K<!#`WeDDaQ2=
ziC>W4onX{e-4Qj6x`lpGetUxWs=CEB%(x@{lKkcbkyUj^$%0#oygCf|x4<pJnman+
zGb(9EiU_N^V`#S3bbE?fS9QlODqoe|XH2_{+hvQ&_a$+_bMA9s+=NbH=S6r2(dJjU
z6_5Jkd42q%@?%!oRlc9w{zc`Fhi)W(fct?(<;U%QN8(p<zj9IeNxP>9tGHjasQi@O
z)6EmOKVeb%8SeXFPQO}LIAe`l?bZzX6WxjaBzMv-e=_M#UQ~Y8(m~f#+$sK4cj_*`
zmKSRmm7hxhqhG@Bmn<qjZ;4>N(~xAR1%5f5SEnmKOn3&r&rp_XeqZ9x<nGLxKg*rv
zU&_yyE=<=6%6b{UU$!v40LFefKVQBu-2g$(=I7Z9(~WB7lc|4&dxb>lE931s`LiXz
zjwI`9{#<viKab4kyK~%m?tJ&kLcCsJY<xiwu<J>@eqs9cG;$7QUg$22G`|Q9xxroJ
zFLoPz&0Xwj{eF-VH@ZQ$vF<OSbC-ltE~S)98>RTm)Ev6M$!gl9G$mhmb$|KrgxTyi
z`^&5DazErQb3-zX62FC?Tcle8M_%Es*y*q2*_ASxOa3Z;UZura=C9`G)iwVrp1zu&
zuMT#{R(@`k-J$%6zs6m&({JP1Hd!B7p8VX7<R(A7_$s#}srhS*albC%?R5w&N6PDU
zbG?~W6vd>tA#pb(JN=D%exp<b72l+ro4A=vfv-s%MZJa+Zx-LaR_|WRyZMy%I^DdE
zn+0kZJox&=y`Iu;(eqn)zL-R}>gHB%mQvaq5_far-au)$S^FMJ{q2c}3CV9yP|ktA
zcO>qPgcYcaf;$s;XX4+OL<+;9Z%W*o^rY(EnD{p*j-uX7`6BdPc6S$dMr)?QTN3vc
znsm3lxSJOu_*?Dnt=wsW{5HFL8+Wqt-fnkq=g!EDEVw6e_fW*W_TpY%i01FGyLWKc
z9eMPfiF;=vroGR)2E*OY5C5(tZ}#s_95?Su96#<$3Wj}vuDb{5s4TYkkl4M4@{DD(
zcbf=#P$?d?6eYh?DRveqVyzEZWgkxcd+o;)sejmhAp0Ll9B+BflSh*5V^-p$D&$cs
z#B{~iQ@<;bwtJsmypNH2f8ySsL>0S>Ev46&lLHF9SjpXpMORE@i?1$haZ2svf>O-5
zpc6%110-xYCiO4sgU(C0Vo`y04)}JcY5K43#~CAxvuV{q5gkxh<K8R-0$?m-cWg%q
z)!6LD$K{!9h-No|3Jl*MrEm%^?dZ#iSnQp_tXkzKQ%@J`ldiffbKx7LYQ;R~(uq#A
ztlca}G1Ie;TSfar{r$u6OsAIVYEIdjQ}HuE6iq2=ZPV;LEzoj46e{#SEG=eI#LQ4c
zFUfj`d(`Sd^Ro}CB8YMTFAw0Q70``+&mw?FeKo-S*^$r(vI3rvT-(?RDhse5UOj1b
zG;&B*lAj@2!G$4NHar%K`Dtq>CS^gjr3G-QVZnl9C+9>^(~+QaB-t|XH5bg2nuxER
zmd4t02#QZ|lWMX@4oO%TQB_y|!V6_+KG{vF`>A#_P5T{nmPAus_rPA3L`VWF*Fj{X
zu0cY6W-i;px!xgYH`CTaly8;<^FMOR3JBLHHkhq4mSI0wQFG{vZHWwFIvnKY)%?M9
z;}Can#Dvt^L*1cn_W0{GBqe`Xeib$Yy1&MzNV8j@wJ0U^`XllV42L5O4azrRIxKRF
z>q3B7{IL8P#CSN0S<)B_5ojqLIhs<Au3<GC<Bz2y%iL18Y{=&<fN}$V+);EQH1;@>
zYur}UFbj_N{Ujf7$GL$a)PBEPSwq{elBhW$v}iR{SmRD`==SV61G0>bmiiOv^hxeS
zcT&xtOmQbO;02Sekfv7|r>cx)el1nLgfuUy`P1BK{&ct2o$gK@Z+pkm?9J{B1iDSo
zGi-vM$)hvbm1~e4MK7EFrS7HUZwMrO8A(v}^HsxQw0*O*J;c;z+u4}(ui(imYH0aa
z`f~uq*SWLN>Fcr&+2lKynR9NKIp>k?ypdI%@6OlUy?`>-^KN}j%H3b&E^-$X!?%HV
z7xV7o8Y;eqes2x>z2+}L>tE_FL7QLdE*tQhpi$jza&^==RbC#d+)UQZp-PB5#2vbP
z#BA9@t}EOY)c6(X?<?I^C;F>-^eT6?dsQ9X{c2_lRc<9Cnj3j28eap)ZFASSZ8dZ@
zPnM#u$#iXk%0|r&d6e~blyyyt`uapHd4sg>jfu?~6!Q%Unz@cvwg)#SC~Y)qU3wW!
z{n}&zrA&z^VwCgid2tIGgkIf(a*YGE)T5G{O(i3()vPxpv<oeYcD{|eZ#T)f)Zd|!
zP`_y7J89b+6I3)Rw-ywz8u#YV474t4_%3Qg+cu+bQITPgQM=TTp>Xp~>DgxYHu9rl
z-)?G^*C^MwCUL~?mA1P#>O&N4vy*;Z(+-kIDJy)`uoY=oYxw<YI9m07>DHRGERUD^
z2h?!%=>rK0b;v(xYWG1Bph0&URUZlsM>jqs4Oy4c<PX&xCqKH=8jiZ;=cCCWT9Wqc
zqCM?;vr_ufDtmvz#Cc3FAESm3B<=%=|6sy|{gCvi`w$KIaN<6gxDWICBZ>Qn*$5xC
z-0l8j2|Cwt_pyQq9~Tim9*OV?mHUZE1Q_9yy8k5iu8`Z#f69tl=07cBd?s<9PTXgx
z>a+HEssEga@p$4sm$=7i%jXlvo6nQr3yJ%J5#x)27*8bbi-~&zVtgrapGy3f6ZfUW
ze<gAJ_)4<MKdFMgnz$zu_f-=8UE=<ZR*F5prdEB8R+;o#;lG}^uhYwR|Bcv&Z`#%Q
zzfT-L`S%-1Ay}SL-lxc05jiXVTXY`+d@JxPI~Mlz+e!W`_*2P$Cy7|)(~0BnJM`nb
ziTke6^m~c>cH;jbao<b)Gl}EJGs$ZIk81WmCGH;+_fItYpF{coYy=Kve!nR52Z`hF
z`;_@FiTf8T^M{H1Vd9@P>(g5Dqs0A45326j#Q#{0|1ph6n~FhylDMCct<(QBcKlys
z$N$Z)CI2(Kw)*Fi{PEwYKR++({`bW3_jBt0MdHXt4t|jSxy1c68GmCCsJ|q?`(;7t
zmHt<;yk93EasQFHUnlNA806=R;(t>V|J%gz_Zy1;&%}|7f1fWn?st&H{VsCc?^VO^
zsbNifYcyv5%POh(|CKoY{)@Tr2POK$@Sy!MaerjMg7xG4pG5LMC+<&)`!hBEMXCQn
zYA4+jHPe49&Hp6szZ3UAq<KMUULcL>CwK}hpxN!fZdqd)*7$2vSv%M1@pUNzGIiId
z(b~B-?Wk5t`9akmmOHfVSinDmbOiGx+b06=wDCOdHkZ&1i7Sovp$Vvr`OvV7A5pMM
zguiU3A)*zSQ?8jE{s^PBMEuPB8X`i1!;EWVt1!Z2V<Kv17VjX|mE2~=J_<CJx^e7+
zX7`%L>&Wi0tL-Zcf+1>09|Z9dEZT{fvz?D(U8V9mX^nO7$1R8<o<7<}oxGg=d|9|R
zVQulm`KimN3ww6cZo<}@;<~p2*z=;xPhZ{%N<ZDrxXC-fXt&jSy=e74%f|rY^thQf
zVQG2SyA?~zavZSSEG=!mVOjOL*=P8J7{WQ&QgiBluFcxH=uWu<RfI0fcS`8YA?O|Q
zg3b6ts(>_yx>@eft-vxA+ZQaXt$snoOl-4lE5P!6faNp-%W;?&vRgl9UMmBtfvte+
zHSwRWx=(RY8F`iCUGXmAcS5@0#-X%LcpZr`LLOZw7tE1T_pp;*th*xUug_J2YX@7l
zePJDh#jXw9KK5RLHX56+Jd)Q`n?@ygnz>e2tugYroJc#8pbH+bbZpsZNI;KwVp<Cu
z1^L^^(2os!>=kWDbh?TLv{UXHaZv*lis#U2*w;zG`LM2S7^cdENf`Kwo2ESggq2mq
z-i-!j|43_gA;beP09&tXy(dc#L*7&OGp(4J45sD7YeTFtE8<w}#LDItC%C?~em42c
z^QaCp%qjBBb#v9Jd67{Mj+4af?Kny1yLm9fd^mv(*tPy}%2^<!tuF7Qq8t!9vB{!1
z9~Z~@c%)nGj*RnhiCcupy#%oCD7SR&@IGrDK<?P0oMmp=?DArt#~Hai&dB53a(CRI
z05}=ikAk}8Sla!V*#mCnTEB`mo#0lv6YAh+tBbPMxHYrOr}z^A#xSn2tWUz0KG~hJ
z)}KoHweD26wvH|R60~EL2d6O(rxoL%K|DQ>^$d50_~T3s=~?bfch**HXCwH_+{;9~
zmoM)F7kjxodmEPWOWiBBVkz7EbKE)V%sTf<w{ELHH#|JgohM42AKf-Ca2H@3uiu4b
ze7?JI8^-au?xL;0UZ)SkUZ-Ie?+JS)SjQE?UIpiLVJcVLWtg#XG&ga(2}{U~5L%RB
zKy&HzmxG0EHtcJ&8(QU~k?gLhV<%q;Ig~2HUcO3vc(v}@F_~YbyH~rbv6f#AxV6<?
zv(|5;VcXp{x4n*yZ0xuL4HNQTD`Qz4d7Ty59;{_OxFNxM7MgWKA{dJwHzjLjEhoTN
zvXG;#{MtrYual*$+4}le)-4IfvL4(j16qjHtx3Z;9>6%JEG%M7;X5!VFpuw)6{K;-
zK(1gPi*A4f*3&o1IK(yvvU)RG?k=SeI+ca4zeTSxg5Q$3yLk;PRlypDu+cH#ZF(SJ
ziY+n9n~UX)aKdD+U^mx<NZp%60rhr}ns<$#qMeShNVX=Ekjb_dlZ>fJMybZtyq|3E
z({1y(b`MLvXXSQV_$H4OduLG{9&M+@>EASB-@>+#uJ<j-chp~j?6@&&y$E6`0-m)t
zd}9$xGjB2d*4U%I^9{0rvS`woFPk6*e2WC<{rpb7orng+QKyq#A!K0oO<Uel_#~3C
z7khS;ZfAiwRHO@+ic(9pO*Ff_40YJG6X95KQ%rpKrDuUaFyBlm)=-L;m^}~8#OZGO
zP9$P?@XV|@hxMTZYfz4s4p>L?k}cVtY{>R`{8HSkjs5_n<^eTiV#OUO|5VbhL^@Vn
zpFC2zTa3oE0@%yY9Gw&AC_+-2au<4Xjyrgp_Aqv$C}+F*H6&xj9jetb85bZUEk8T4
zn)nOcf}IG-!@?GZy}>U<Z`g*-?l&SIDGxyH#xAKN9T$e(jEekG=7%DW)@3np#L>vf
zV}K)$b<0+<>6)-Pk-Oz}e;j7vaY2==pfM|kiM<A4S#igMEk%LQ&u#xg(o^Jo1Zc$#
zz_f*USb;d@#Y#|%l@h1O(i7b3RelY1od{@gVqIGiQ9LX|o|1SOA{BTd(_TdO8SBwn
zWbE1+g0<pa0vLml7$vI<Ju$Jm0;u8)<vUXk`+{H{M6jOjURq=Cp&~#c>g<;@geG4H
zk*^i^ipU}>1V~5%oWsL&8rpgt-CJjh^<18xD{6#k-tEr|4LIMO=Pz*QBX%!v>sR>;
z>E1=|LU&Q!Z?Gw_p-><fEByj(J@`Se3EOu#*<V6>a0z#59eI0Mk#<w$6_c=ClCbXb
zjR@IIZgUM>!V21g=TrnE+?A{SRTOl!yUJZ%mvD{J>D4rv?F2O%%5CLNI|{`Rbs~2u
zs!Q;M<Ss%NBw`24zH24$5VQ#0>kxR?3z(qDC=6~$8qy#Oau=a{J+c>xfVjOW336BX
z1OoWA2;+U>6EYoXfyrI2ok8*zdj+>8^GYQN->!!69SXu1F}%e@ub?^PtmPdga__Rd
z$l9)ktR0f9<?n9tB4qh{BYgvxKtjJgSyD2r#9AX^tJd6`<Yj^W$lP}%LxLiB^G<~D
zeTlmdWmNL_TgqPH5tifK%JG0<4-b&zJ%<0kha3+kK>EtDQ*Z>}#a9229)U&3TQ=l=
zugLyzgzsy@@+%+_jF_p0$Ny8J@zG*+w<~e{JxbZ{6P_Sc4_KlkSmOPOFp0;6Nq|g%
zLks{+Q1pidO@K+%!6b}1KY~g`mXhnEs_UcFl@;8Pfm4v@<AN#tClcTZArbueWO5?-
z1Ft?U$N=b}E(GE;Vi0v>1t>)BfFOi9e2(TlZjUoi2=aVEI0P8PkpE%=HUauj^G^tY
z5VZ8A07`sWAOr|R9Uy|HJt-(6fDi&8KqLN+4t-6feLVq!0C%Vhb)atNfg1Sx2Bm^H
zRDc~=f$B}>pHhZzCEyMK4K@F5z5cd3Y~ZM6{ti?DUI?fePv-ikt=O}KH(-jN;=d=L
zVW;p0p8NyFfH^#!fHwd;GysQ_0UW67`-%H!Aq#c?1C{#&TWlwS87cq`nw6C*2bwgf
zKP(3I*#w9I^r7Z|lz=&CP=9Pqm@AxtCM^@pz)$}DaWeiU$w-?*bN#=@bpIwCK^cE$
zn8ME}>p4~V990IBVg(2UQqZ6W00=`LDj*NqHn1vwsn@?s03Co0YW~;C{%c+@izW)7
zLj}x%E*RsUE$rdhlK+h!3vK{)_-zuciRZCV`d;*akbGg<<+IaK-IR<gY?IL*b-~py
zB!newWj@6EOWr)u_tiXkaOe?6+?Ka!x5>^K<X1Ip5`1a!zm91zNII&QaON!H1EXDe
zN7_Qgc`Xq~Cmfox&zf#()&p7q3}ZOJ5VlaWYpeM&dB%<kyOCP_wa213zja<Rrl@Un
z_hORr$ZiI*P@9(4xGu#?@f|i_U#e)MKdQ}*iTPVQs{foM)AnA6l@0A|?<j1EHc@~a
z>}|?JIMrh`HeWmSs(IMFTQ$nObC`BXNIZ^FFehyJv0Ev&n3YjCVO|MiPDV-Lgs23q
z+9=X~QS^7R0g;zzT+-?Q4L;b#=9=ZeSG1#g{Rmgkzkz_2?DIBGv71&+`yO4i;@eoN
zw`HHR<YO2_9Z-S7IDYMMTIBGg`9%mENK%0$#-rwv%XzLenK_(kq(l73k?$r~935Q6
zq!w-T<vm`OS5e03Md!7yV#vDKT}_HXg6Y^@J#p0Tsv8#vs#{=#h;M_WQzTRo!6ug#
z(CkF20(mfxH+EA^OLPW-Nb+_paBCO?s5|rT%`mvFDDwe#lZ&FJ;@LJe(iAs(DO5R<
zPSb&?vK&Y|qZ{0GdSJLz&70#W)|bg-%i?nfLQ!@-HIE4%$>lq$KTngn1xfdIegj&?
zqjBx|Pg#P%_;JYrQKPSP_LvqH&@>!_3uoK@>Tb0tYKT|6C@K!muriG2hs5)4{1#z%
zjYMF0*-x#aN%>C=(B9McR;!v){-h}b+caekKowiD@TM9wcE>KTc9EYnYZnRn8Jb`K
zI-Y+5nzP!^B)55F)%~n=nVTh(K2&}{EaO0s92PKr%52!4)sq41r0$@aV4Xnhj_QY+
z+O&+^AHb{(pe7Kl0h}fIzPPz-d6YF^o%s%2LlN7tYWZ$Aanyq`GQm0A_d<Y1*JoN@
zZ8Us$r)M?X<c>AmTrgc$02J^}dvwSg&uB~t0_a4X3KOIcE|+z!7UR_DLiWhjk^@mD
zdz5z?3C(+V2zH(7_BU$FNP*hyY#V-7hvI&h*i$P>*wgiFh3aM3Yo}aP%q)trQw&-v
zmbFy0b$-av2Z<oPkhFlQp5f<`-n`uE{$QJY2W#XKe+d8C0lqqm#6yKW{D2d=Y*<H<
z%3b#`Hm&m?aE`UaI`ME@C+45E_4Fcc<*%Fndx<^k#jFgEbPKYNh$*o>t@UhU@AOBZ
z$_!D<US|w@G)+Fbp)rqPyyB{nS-H&R1AaNpGB~a7S6Fjb(Ap9!#j;k4=Eq^8p`RH7
z#O4NSEDL$k3d*LP?3FAES7m=Txx1Rt7Tjj9f!ng1)drHCNNFdA(x~nv?oQ&)gh#hO
zId`lY%kGq#R*g|UoZL~pcf^sJy~>BclI2K0%nw(tH3Y8prP(OoLem4xA82AQzlM=9
zc*>^pm{^89^-&zCe9g+S<g;T`H5~JwwuBi8!ns%_n1I7)gELo;;!G$3Z<g|}n?UG_
z<>SHPy6kkRvNsr1@)QQuBpQQAm^O^>uG|gZHK&w_$AIsulv(*_jgNMh3J3v!j8gu3
zb|=K72fEoc_;eP?e;V-^EgR;LZq6uT%#Do&;&5~2Z_yR}EwaDZAkQbmd~uW=AeQe1
zr*jUA^f{b7=6SI|O#|zY%LRJ^3*DmCFmGbKdt}`&fj&z_ADy~M<wBv+2t_XC0eDAy
zJY;t?TKpIu*#VqmBYPa(QT=%PEXg$O8p&Z;@R&9kv?-wHVbdUNGE9h@4LTbd{!3%p
zknXTT4OL_Wo7%VIDY3F<SJIKt=G{G|beOGK$dVmX!AnZpi8AVVC|@be+C}BFxZpp0
z8W)apox)|zqVh_mJPl>vEdOg<MzrXX+Z7&FV!0p{2ljF#E+}#;Sm`G&FG(gA`wDKq
zwWFM+pzxHKTBYMgEMs9Bl^_&hxbASCVE}Pxy2k+mdop<@u(e0tP+ztopUGpk73^?@
zy)xG8^0R3FEEW}%86BaP^7BBRA|)5Alw|;FTux!TA(fDH2Mxh-eQr+8gQAH?=8<|H
zfZ6ixz-i6yU^ek6;}CuxvZQpN8A%kdfWr-P^s#M+u=_NCsFX)A$Gqxr*+OXo=yDd^
zW*NZLv|2xsZ3gf(Z8f-~PQ{}q!J}uXJG$<Vp;yPigUYoMv1qw$!=V-7h=b2oAhl&~
zIl~;?#}*(dkJH#PVgdw@XaAueM<0`mr}>qXy$XT2s_svq>=UePwVh1}<yu3h*666Q
z)pR0v$i&6AQMv%3*es#AG`u|7oh()oyPRUaH}gsd4sTNkO~d5}`k0l~te2-n<KXl-
zd%@G(={4}QGezaI+-ZPmXMv=>)V=Iv|8iPt5L(^8f|kA_wsf%|G_~}cK$CUu9DlA`
z2T*pdJMS!iKDhwO+y!;N-o)$rM!pMa&xO`djle~r6B``fx;kiV1jef;pl9G`A}Q0|
zo)7BzV$igWs`(O-w83it)Hb?HNu^^j*;kASFQW=JGt3WZ0q#F?;oVehUAWByeu(A)
zsktq6e+8YoqA^v@VjIKiW(UJv*=W$!;h}A0w95C#UA+~A?Nx3o6;u2*RCtZR2l6fh
zVdHX&-wq$|2q4=KARAj9*Rou`PAHq`e7)d~8w9rPww*Ds=XQpX(6%(Jg4e;ms^RtW
zLSZ}O_1e)G^0%5#(5+PR21~GjT@C8E-TZ#|_qHS|-#d5>s;2#VkydE>ol4362A;S;
zY{Bi8A8!VHxl8y48yI(K8>0@w2F30cz7cW)vb~iKWx#9{0mAk+^Zdd82P6#$?Ouv_
zhl&7EdxsFUdO*~Q{P$~d_b#DiZrH_Vfn0n_g4EtE7atsbV$&Z~(;rmR3tt2Pww(#;
zA*lF=lE~C}{_yt@#CTYqKce^}a`g$-E&y5sP778?Rib>C3f@KX_X%JN;=H3eJt=R`
zpEG^FXB?A`>hD^U+X%-3)9onv(c8*efzQRxpsA8lrYQ`U)*{@thv#acFF|XoeA@sP
z3e}oFWrjdo><BA<E&OF~pd1YU>BH>6eG|n;$?P!;=;$^12S(9boQNAfWOCx@LnfV)
zf25p?8KOgO7|2?lTi(XHeB|+7S;8RduKTEjy_-Xl34VVno^H1B3|1#I>fYIn9qz6B
z9y~aE2K-D0eRx>{*USp@jEcJa0lYN7PF5upjs0On=|ueNM2OkQ2!0}54ETAZwIjZD
ze~3+mL&SX)(}|hPrQ09MM6g}vAte6%Xjk|nU~fv(8IU754T8A0J}xG)Z7&aT6eJM)
zs7P#%gJ>V+Xm_+*f+HsuGB&U~w&oG`Zjo%H5(2;Cau~26xI^|EQI0bjtiX+PMIC{^
zd|2SG8YS>0?#11!DQGo=7<#gXC+w9J8_w?J0e=epKGmJ#POba3*6+2k-@X1N^x-u3
z5_j5=KRx#0jMxX{xI4p6Wk{;Ols=d!ui>8ga!rG?RiE>(poCYtSGZRWA;WRYtoe0r
zo!+0T;}?N)cwzeUs<!D-aTj386@6~?>uK(VZoT$DJhIzu81NU<Tsv%0XVYUaGFiLd
z$m2^8(wEeVZD;FotM))7H{xD%YXQx+`?^+pqjs8CkJ@R5mDp=8Db-cbP$w%4n{3BJ
zGmPh}A}u7UUsXd?Z`JzDc0hXBc}~O5^N{58HEe>|X^Lx2NFbk$Jl9K3Bb;lJ(fnP_
zo<VqXqlDQ_aqqc?u%-#GF<~nIDC2~i1GZi4iPYH>QC{1FMp(a&QW4t~q_(YXj4N+d
zu?TLYcsxE)L1Htp3%YbklHU%(dd2?@j!7LvYKi1&iR7`M1U1Z19sX<1zHV}CUx4N<
zNs<+^u8hn!^vdEE1SGRbvQCEz@#!KilL#IUAlmtOK`$KY)e&vp3AT*~@P?3W(RU^u
zxil2CSWXqF3xjR23n3rega9+=HyF~er)18au%1|usc)B?W?{@X^Irr0Z9?u@+XmGr
zHNTq+&F|tu^E<h4x*_Ajfxq^67#WTSI>cIkk&_LYpmuZ_w8&>tuiSM5PNXy&Pp=_A
zF^)lJ90LMWDA<BfQg*>7GhNNYs|G~D>EaUBI3>s$Pi2j}?vW6M-V>w6XDl_nqqS^k
zS7@f^r!S@}+41^PDeWH6!pC?EZq}+{AzViY_i4tGN`CdTWe*>uyH1R7D`74zGq7SP
zY-a07@geRY*+`ZN_b*vxhlWgtg-nN&$y|Phw8D!A@^pBQMYd|PU<(<W+AVIexW;NK
zP8lPCkKzR@z91Vc2NJrYdg<`i!Du8uaH475R?Rp=@I|F&M!d0mU~{33>1NvoBl%C7
zjz~OUKE{3)il?Lc(Q>l5;QW+=hP^0MA9~A{d(b^<d2{~gg2Qlm)s(RM8r;ua=ZkPZ
z#l&Tu6u8U!3uMbpsQZbg2JDEjR&*5|7ZvbArsyuwUWjPLWw45YzC?jhsF$^&0V<4S
zslA|H`@ub4tm9^`!Yd;2<`q%*98-@%zSRRTirT1h_3lNMaJ+91%G?-eUJ1uR333)}
z?$CO0gUC?g0pX&D9Lq3zwsR~a5k?S#bX25{1H49&Vxfvy2&9PHZuI++9o4&<n&va@
z<s#EK_&eH4>owDjtb4WA%px%}E&HMglxV35n`cyIkBu<9+Envgc#2343N$eyO2OV@
zuUM-Uq2O^VIy%L4QuIWM7A()tIK(4t4Tf3WHB7#O)iz1uBEXrehGAr#yy3X7X`Kvf
ztu?IR6}W-y4q1Umpv7#^0Q68{&k}&=5~EykGa+NaMcuyFRpyndU`Gs+ftz&(yO)uU
zUZSf;d5JnaM6K3NgauVG)`$Re3LL^%n-?5k4wZN@j~DrelB^3U!KD)^VVOeQ0id8E
z2KI<}a984G2~tF7b(dsMifkeEQ80!fnX=p}0x`|P=*ECch7GuiLJxEgYtx*U<_M$c
z0~L2%v_PF-!3);w?)VxOoVmTYB3H%&ejxjTjo3=wt&GgsJM2hUE9^Q`-1&Ug@)ftH
zhHZCZ;5Oo8q%jbsE9X0|`&Oxr(kSQUdQWp+?$-iES_Bnahn5UV-p!ftKhoQLdq>p-
zK&k|3UfFvBr{&P@xg6dRwH|<}lkFw2+=wF4{Wl4J?GXx!@AQb5tcuW!9B*+=lG=9a
z53wdKNkb{qH(Qg+xl=cqeOg{-4-}k`2t(N$tl)CK{RGOkI*~^Cw0=xrx}#8vCQcO@
zGI8ZWPaap^wN<ea=|=#P@{&u2&plP=mC|LdlwYOToi2f5yY-fL8)^ljTwRy64GZsm
zh35>6)Rc$7kj*o1wIP;q(A<c%WZb4GbH<{!E<<OrJJnilLd4wMSPAYsPD3A^gPbE{
zp5R%mWuO8(S$GC*rO1jR9;oC@49mcTv?+;c)QyPY;t(-*j!#J7KEpfpVl-KveS~)+
zt^k$bXFM!M21bby8%sqFP%*uLfmYgdCbjM24IxVKE;mskEn_iFdJ!X*te0e_EQb66
zY?!bEWqP85^sv%qhhztl%-qa}Sn<skk_@0MtL0$57pB|;5Xd>c>}lJnm>+Tzz(mwI
zjNEpNZwMiBXpCH<+zY_~!IP07oZlO01TT@h9Q<<>ePW_7h15XEbK9GWI4Zpe5i7W{
z{2OBVceGnpW9gSmq%Ak1EN>9yxR8DY4KcYfgxomJ_17dUG_os0dsl^ICy>ni<Ax9o
ztD<O7drxvJ-ANnqj%(66zM#lyNIUC59pXDhj>o`xYu!s$gW)B38K34(uge`yGS^a`
ziKGw5@y`sazSX+{cy!z#JWKsb$FV$F53QcHpFgK+AYPMP-&!&7&!Zvd)i{QKzP~`l
zv*^>MEBd`%z^ZROC+@?TYzQeXc6Q)iUT<!2V0i6U_m@)jrGe&`Q84)2Tpd8MayRM1
zx)0a4O@plbn%w14xyypKIydXmi$5HfZdQX8`NO#@h5+n_+?6#Jf>(*uSBGS;3dvqg
zGF*<`)*;q@uX5Maz~;88BilK}w|yhaKgf7(;x@Ae1f9$N-q`CpBopu)>%Wos9Byy@
zEc?j7ijU>rYXGutPUzIl(btF7UK0**ftOebT0=)U!`Z5@DXjXWQ5e=+l}U%^@zx>3
zTvmXnjN9}yp47WN=|vXpD6uN+sJ_se+};o~d)?`J*1)5E>~v8f0uAvJqvQ_Yy^=_d
zxN6<QaK_i|8l+In*tkZ)ea4MtYpV!<lgVlrPiYfm6_^7r(M$B(4hPpUPdeRx1EV$>
z*kX_}V?C3~-NUxjLM_u`Ee(f({rgg)OH7Bh(;H2h5fcF|$$Qs3PoRbbdDGKKGBYM&
z%fZd8c@AHTRSuxec$h)^uWt6hZU##u|Gb!=Gc7tj%<`r$eMmd8^BYMHjY$Zaq}WNA
zsf6CpsfGoOBuB&~piWxv*b1kF3|gH1%DC%D`hVo8{x69+6jY8ik3~@`34@9`j!{@8
z2y$%Rjv(Spu?eF1a(TWD^Mx}GHMSa#bt^U^f{qEM3%Q`sK<*GgOXHd1nCk?#V#%@E
z4Y<`C5kvh!43VSTp9I0;iHCqb4RCZ=5ke3=bp-}tTof84^2pwH5xOOWVpT6CuK`el
z0-(Iyxv7BgA0CIYvw@&abg!%dL8bD<ZH<MlizESyQbZ6=KG;y4Pu1s3?AnIn1$kHS
z#MR;8RGSK|D*7URUxZ>deQ9TvHpEgdj-_&3Sf_>osciH%BC`Off@|(7e;Ezl<SuiY
z>Jms2tybIR`87KE5S(&T9Tv8I%?h$8u<;eKbPhteD@-7{Ej19SN8_R4nl=tJakkQK
zI|MN#apbPq;<vMzyMwI)$)X*@&bTLOlOPHqFsM2k1rXpy!``h;h@<Tdaijnqbp2+8
z33Ams{#s=RdqOk`a{_uw*(`uIX3ts?x9FL0sA3Zp9LmCYqy-FhTM`A(tq7o<lghu*
zzUbk@Ce`!N`Nl$?$E3EP+JNCp!52jO?Cubv#eQQQwKCl8Ep9elC>G->yaxq3hPiK#
zqXiRGC>}W?v=&p7B$Z4Y;RK@JUEnPiA%bWHqfH&x9VFELw3{9FjpEcWwA16zg0(36
z9U{mkq^B*uF%f!VB6AI~NVqot5JQb5du+TD#-Q8(6~x;e(#StQR-vtQP7ol4Y+4|O
z#+I^oo5}juL=n;oS;Q2=h`qocNe_-}6u&P)WJ{`znBtKf$HZoE`bvB4VB;b>#!(@Y
zAmH+Nv|<C&$zGegsm>JukGtbzI8L=<gH~BOgHia5FlwtXg;qtxbpx&=6?ej@0a+be
zfai!?&Dsjd*ocB~a%{yZ<YGtkEacy*2$@rB*iUQYyLiOlG<Tu~;52H)n8K1eon6o~
z+?fOZEVzzcQ1{Zhf0^;z%Z7RG<+1X!fz4lDlPFUm7*nxu4sZE&LCgtDUq?_QBv?fO
z4rIxl&(HG}Dyf87Ww83$#cF#c@(fa}6SD3utRc!S5+83M>BZvRn%m%NpzHd*(Oqn@
zjNG8RWWZladoA2S9oa?PB3py645zl%2}^gE*AQQuRooEr$96w;rT{C77zg<*45%x0
zrXU13==ZOtKr@}{{+c-Lu94)Wc`FfH*2?Xqk=R0J8G>HHhN5H0w8l=S>O+Wi1xpGc
zHl*E7{%*iAAMu6oFmp<>>orN^6hZ%AJ4LWTmJ^LQdpF2j=YKdv5T@-N#b)50+Sp{r
zv&I%DtYSNy9NDThf(wDGkKo9&>>QU0XWP?uK5IHyP+{9sp4^zEMg?|0jSO$IlLn2r
z2kP=Y>T<FFIizC+{M}2J-<jmkmHd6$2le+S4z2#Kxb@lZ-z{nVKs--SXY<n-?>#2&
zsc3`F6<C*d+F^p7wErP{a+>EbLFn?sMVBA3xCRfC=Fud_aZTHuF79uVi(St5j_mSj
zoF@o{Kc>S3?)_x^08cnlFtW=Zv_$JTVZcEH&KG3Qn9%ufXzGX6)L~x!NKE`u#YEt|
zL5)KOoG%c4J}&xvLgx%PWH6+&2K;>j`f%3Z52bL}KnD$wR-e^z1CAFAaLNGsJRSob
z)H!0{K2L?Qk}v4x7j@de#bX8)4jRCXQgw|X`%+B(<#_6##;F4i9H>b|Kj>;iKNt$p
z4>*6YMTZZ5R|=;Obn@WqiOwDH<D0Ap{$8~{Wf2daLO(ZR8^oEi!A>5Ck-nqQ1s2WV
z>0+q9TMX6rbk2Y?1~nZu_y=~Ox^>ckkv>Id4W8C<1A-fj;Li;@ePEoxnFCAr{doMK
z#_<E9AaMRbArA6GIBdZ21NXyZNy(<}DFim4#Xm~ivk6BIYVp9q+5RVX(BQ{B`6)$m
z;$Y;c|0_){`G3=ke<Q`u6t+O;4E{CYph5PJ#sWWw1&D3{V$)L1K4Fjkz3AyLik|+`
zjv4%dGJd7g2By4zZE+EPoorHggK*NI3x<7OH_wxjg9g7zIAxH1(hB`go@)*IJH7uM
z%@h#(pE_5Qf7iSre_vGgUkYVFq=Oo94e}`pY4F>Gm<9@QP|2P#Tl|kI<c}1>?zYHI
zT!ZZ)u7P42{5kReoe<Z65C;83GJtL`D3*bHfsx*k+T`4k29emU=mlL0UT|%y$OYFC
zcunyO+zsgo{>GH7H>K{z)ZLU4+9h?b0l>q!=<+wGG~i~iw18W0@UP{8_~mt}<LB#A
zjJ6bN+@f^1aCd9!ZcY6gELFF^4Ra1VEp@l0BiN(c6R*)qxg&M_-A=wcRluE8bT$DR
zhUdc@E#W%<rquECO(cAC>fW4cKHO!mHu$&1vhG$d-kLgsO1_nJZ&QJ9Q-O{Yck|@!
zspIdi6jswdsMOd!%5!h(?n&Lfq<Dw&yn{S8DV1b{e`hTIzSQyaK2qJEqVC?6y8Bc2
zE)^Eiy<2(TP2P=G_D26eD&_GWJ$?_l9!%YXX|z5cAY8;3%E{5gHtI;&Z={JT=3)48
z2o);8_Kh$E+ZnXr^rAy6!(=B0h6L@qwk@oetJ(9$jJ9NU?HEuEKMZqDw9>|f(i#!A
z^^KHqZv2R&5?$c?gg{kDE%pGn0L3BuMqzRN0gHo*7D=lP{x;1P#`_1p(|1O!&5k4<
zyVNU=){N*tujcpC{00OEe!oQ!Tt;Y;9VOhs<?T7!AH>vU_s=NZ3KC}}&Sk^V_WSVb
zbR1qCv0c*@c58-YfVxAs_`}t1w)sHs7la*}eyw}`K~;B<tj!Qhw6U$H<Ej;N<g}q|
z$J37B={yRtv*P%$&398JSw1|5f5W~T{fV1yj7)i%I%4iM+F4+)TUM%}w~E;i4h#YC
zhJ!nzmATTe58^7?@fZi?%;kKDjX1XAwt#s2N}jwUe9Ze<Uuby|e9R%EJb6{E&Pj0T
z9NxtxO0=S&Pf^4MOVaMUO@<h_O4@b?mv6_&aMTfr8G&jJy0EKjafGc|o@VtpqBsnJ
zva%_+g(04=AqHZ#unq1&&L&fYfgN=Xu2pl88m4~Dp@6xJ=v>7~V}!S`MRTOKXh1p?
zyn}tX6983{0IJpnZb({19ja}(F=3E5GRRiqVoEF=YVe&rati~z$j&3vGIOC+H0X-L
zK-<Pl=8vU#JC7_V!`x}KSFY9ja&DIw^01r5!*Lzeuck?F!3TQ<z2XYgmgL&FugrXd
zF*x&D<Jv^~Y*G)|c7OAhzZRv?5w(R;%Kq6(vF~=M=`5JrakN!@LJPXiI+O1u46x1=
z@YU%i0el&)_d6b9Tk6SqIi7XW7l5W@-&ctuC2zsPEVZI}#0F2JX85C(x!284HhwDN
z2ZdozwI*$)rs@17P$4n1Yym$*mx^~J^4isNK2)M3O+6&-8F7}Tqx#!q_=^L3CFNV#
z`GA~pi30(nr+hp<W1(@3&JvjMP>1AWm{9|M_pkuW2?YiSZ3Rcvcy!R#Olz3&uEUMT
zp+mnXGRGz=2J!^gIWTHkxyfN&-bHV^0&*6LBL<F+{TIl^RC;U#)Up5WC`P6dL%Joo
ztYDvgZ_Vr$LOa^i&5;m|ouDX0?28*tBIR+p5m>BjKG~rI=@I$GMNd030lQd5pOdhk
zU)P5gpvritB1SOOcLEfN#{&W<4WJAX=>j*o?z>3dg_3!|++Ftku!2}g-wT0V0*vf}
zgWkySF9Y6Fpa~8O|Ket9RH)o}cB6zmo0iCaVTKV~`{8Y$KQL0FFH!=9p`eunI*{x3
z;}$r}^3S0U$Q3CKGzK4|8W?qu#X5axtaL7_gG{|a#$-RRc@_mt_Ak0K!t^0*!n$T*
ztR1zY{eP5-J3_UG9vn&4(Q`X&1P}Pe$2bTQV^tF<Ccn;@ds!?Cz(QE6A~lDzVaHJn
z?t5Sq=B(}d2S85~&{IOi)cq<(VHNF<XDWbDMfcUQfoo!+Z1W0BCES7DvX>*4Aksnh
zL+j(I4AZG{FtR<KEdLE(V=MW!F*6}$6ae8g82NNk;J$Z;Tg@)d8O13K34_pyGh-Hj
zR>fc@CS&Zx%jm?*-OF(EdpXd=+3pns{*`p%9QR6hPTjAg6YGXM0Y0H-veP3*IA51;
zP>Bm-YuCrlT^K7<{6%01ytS3<;?OzJ)R=sbHV;ZrHo1!>BbpSTxxxjwqm=Kou5OG)
z$$NXFJOa;im$FM7qJC7X*|Q>3#Ii1<v6s=<>_^5C*^ljlPGP^uZK?^L?d8!9J{$Pe
zsmXU6(ZAB1<M6{yIw1{`3?~mQm}T*qdvSEvxlhmV%4N|Wm~Hn-4`g5#ppM`2fue;e
zQD3zEIx^zZ?Xi$sxZ1jg_+JqZ!Vr`?xgn9b;P@;9ggH?tk$ru+BQ#~t(}OAk*RY1`
zvhDXwja1cFR;HTx(*T*5?q3xGvD3Y=2=GdMhz<!LwSuw80K_DFiU8HZ;la_3+2d5u
z6FSnS0AUAXTpSWN@<pZx@cRjJ0cjJ+c37<I@Yo!Lg4}rt808LwizG2aLiSg(Uz)I5
z6tgUjS@8Xl$B1IJn#-<&y>p^NDw9?$Wi;6El^c7o5bxG856ZkDe!VCAtyl^Zr6gq2
z=P=>1&>&WeebPtf;XRiX(rCwHVQ{>|^$&UWd|m$*9DWVO1iOsUPzKm$S>tme#198;
zt^1QH{p8`&!7QzG^ZGTwQl-Cy(hFx_!knD8#h(FS2U5v4%b5YD?DsEYk~8n!%j^DZ
zqv_ePSQ`5Z-kGnjHd-v?$=$}LyjmB#ey&^R&W&9^k4Bw`b}ARN2C}qf*^!PKAmw^B
z>q44k;ZKGn5!^*v{KZ<5jS70wKPu>nwM#Qv<$--QBD2<jC@b3Zy-zU0rjURb;F}|q
zKo}J5xMcaaY>mGqj<ms-HGd`Y@+$r#-ju~wXFpdFTVEAd`LAXuV&Ex|OM<hm@gtDS
zI-4%kd9ARkDB7<ZKBJ+AvIofAtpHg{jJ<HXxZ}nGWw^;UgKq?70LQ%ELQpkg7;P7-
z$@-prk0xw(7dA8nvLh2nI%;?3^@c><A|&coo#|j_ct`*zf471}ahwD25J_`ej8LTj
zDR&6l>=&>Zk8S{L)&Vw+)^9Gr!+R7uofL*<f@zKbH1ARO?~>0D>wBenTf{RJFNIfc
zA3>RWbfiO*{$6W89Rb&5O-~P9e!o%XodsnOtNbZ~%KZ|T?^42e!h_uR3+F6?J#824
zS#b{pLbZ(~)O)NV81+Fr2+|1hDh7N=gnDn{5bE!RP!C&(*Y`rGM+~xiR1j!DKK=U=
zw@avJKR72{d`t)@IA<M@(+Kqebu#~mogM#R(F0IU13GI8rt;w=D%g({YT~2zx`Ar~
zatcWLm;rbIoTZV0Q~)*waefLIlQ6DHz%~Ik>)@Nv__L5dGT-L}aEio_ThAf!=S7>(
zQ~3P+gSUzKzECvqix$!K3y}B;xea_t3BSbMmlK6sc|vhjhDi+4Spn-bUThn|iceZS
zQ247!2>n_M;HM}I_Sw}4yfP&ClfSP+;cq7SPf7~A^38~UHlnTcgL%@y0`RO8R)r2e
zE&4{a{#`xz?g*~>o&ZpF8T>QMw`bJWXK3rP{8>I<TLK7;Q~sX}-~2~9N!*pkOA5SF
zA?}I+pJWFR{lSPngMEhhE6l&PYNRlj=nt(ru+T4=Mfjtl^dBqIiUFfPQO=*xf>ZLF
z`22Cn|I}Xnl$wA;pBfE^J{zO23={h~WmgmyoA|L;Kj)#z(SJ{XN{Pl2;qzZW5J>V1
z<H4Z!e_6EXR~F6nmsAT-`fHjEB>o@TqJ3VGV1<r8FLV@)6nL|r;IGggY!sOEcQDuQ
zMe5&=AoYI<6cwq3f|_Up5v^FL*LC^V_^fcr|FNk0PZr?yk5v6<SiU4kv;rJj6Dmsf
zKMNQg?lHh<1$5MUJf=__0H&eGJ5pd%!o9LjB#czNb)68?>xGp9lnx0mHOO>}@Y4AW
zymU}_DF-Y-0@=O}h25OeN#Ie?)7MfWSZM{QR60HO2~aeo1UKb(12|Q+Z&0-Wr-qsi
z2`c69wsecXBNdDVP<lt|0YmTPI}{NW9RL-jMi5cZ(Yt_f-;zSsx1^@X=Fu*Hw+b!5
z(IJ7O{JoVz-=2b&0Dppv-ji+=78<B7DAZU1(39Wqq#O4M{!|O^S78MJI^^G-f_nmm
zZV}`;Ux?=eDUj!&AWzDBkgPjJft@2s^pG0--qbymy7!XcVI_FjNMz_1O@1VeX49i-
zoLak5$KRtQe_!g}2dV(8y<flF`+4?Qn&Zjvfz&;gx({&w!PN2h2$hzK#2-?nd=}|L
zsrxV~K4N)4Lam>+RbuG*N7Ecx^D#B(V<h5hN_<b~6RG=X>OPUi&4f?t?I+2&!$x#R
z;y)#de=5Z&X*S-vg@{yo^l1%3oY$X8-DeC|`7GIuw?C(HK1Vs5^0%58dA#WB=Tpbu
z<22w4h9iGbDZj|=6R9Kn7gF~`arB)nfiLOpmng@?2%Y@09(<Vx!Q7{TUn$7>WKqRe
zQ^((vl=*k5<L@gZe+ZCE|BHkFH4*IVsry>$zD~MtsMK$e>~7J^5@95LvncuRi;|y8
z9e;mMj&G%ozi*P|KBE+B{@eEG+dQ&0&UaMicer~rSk8>$)3!ePu3mhXGQO9(@1=2#
z^fbu%bBZbyd1KF`HIf}=bQ`#k@bZAG4S)pc18aP5w(i>)mNseAAj$R?Frfl6B2*|Q
z85bE4|1vkuK9H+g<Qs2MAquyH-9Jg0v#tnVRm0iLO^WM^h>}bOS)V-osYB2?kfIW;
zoXSu9&mPB_<-e0;L~N7H%LAEt#jznR6=UeA3gd4JMLeAW^Xw4G>Zuag#s6LNxJoHW
z%PcQ`VXC4v_DAQcTD54jZ_KqT1lS#BzgA;{b_YtN4^&&Ms&+;0dxs_1gCYc@7V?!6
zM9gfI@v#I8L8I^VhDbib2i|FI*yDInS+O&L%80mOUy1Ih{(8i*mQwC~;z}`KwieHU
zBeL(=Vu;b!NvIf2s0sB`Y?9=g8sqMCpbKDqkfm`IVirtQR^);hHYi3eU~T*lRxlsd
z)7qGCfeLx(fU#b5X$)&kGT1DCuPN?5m&egan?gx^cT_=jfW|mBHTokY)H*{zg%QJ1
zs)HLvyilx<)Wy3{J1R3_8y#fTH_tw#B_|=P)Lebuh2{5-f{za%`2o?#8N02Ia<#2O
zxgq`R{1J=4aZoItm9wJnn9p<e4O`33qh-W6u#ixl{$P4B3YZ@l1<X@EeuuGmot-!$
z`>Hjs%4ja6B#<%yvoMk!)!UQdkE`P3KVrP&SO*WHviRDSt>7qGZGTKS`lpPycE?9t
zYDbe}Jl8hFfJG_;6de&iWIkNMsqGO_((L=UzCdR8ibgkp{dQZeBi#ue7p2DD{K?Q*
zZ0QZ-(R+(yW`&H4g>=NBupJ)l(6L9y4xLn;f%}R}+bN|y37u^FJVT(e;IdnQ^~2|$
zr^LeojSt=O<(uq!je}5Xc6s0)VrYb@LI&N3=)d~BdKx0px?+et2Sc)d5=%u}xx`SE
zu$+IYXH)rp{TGs?ah7SXBzxfZq}&UDFOk}8L#yK@Fjf%>3;xNzWLzAw?k>G(zIg~s
zd?I@&ygl|9As)X-n|;i<x7aDMtqE<FDDqeu_dG)%9Us#jQ_B9$`qNSUNpsU2A{I9h
zilt|m6ELs$CX8rw6hw=&cUnuDf(&*|jl~6rDYRP^8P^WoX%Ba&G438`>P<A2@<-lg
z-GX1+C5>C@SGllk?c<-4QENn~APzg2TOEqVf*4p4yGclQG0vvIH`>VH1CDGAb`HWf
z`;8KZ7Q~pW1Tm&dcHA?HL;C_Ks~#1+jJ;nX{{#URIB0-F_c450-J95I7(ZSP(IwxL
z?=lgyhc9$l;aBG8e|fWddH^2M5>yC*KDuJxpz3ji{V6)F@5`3;eOc@DyNuRzmcJ1H
zXE?u`d%d5xTxT@q6*9`rU&~4S1Z<yp?{$AT{Wu(6i(>(jrxqR&1IsQPQJh8nxn8LL
zPBs$D@{jJ!b=a%ymI(FvkGs_B-4E%*UB{5ZqWv7<j&{cmw;lrOw3oI124l^fC%Hn+
z7-fa||IB)I9M3uWWr3<z`V};J#c-2}7_TPxhfD)AS$)%vm2Rb;@`a}C!&deQl!1R@
zFP8xUAzw8j@8P59*cBk0mA=%}&o05qvAL!k7IVnM4*L%u^5`FR$b*J)%%haus2VNH
zN)W)ZeW1y1*Wk0rym=|lEDrL^;(R-+qxz?E((u<!@_oi|8=hY6Gl6pGY17O*?8>AP
zKwiq8v)dLEBP}#Xgi%yN0pvw|N~8tAc=mHkYC#nl!l<P5NrbW4zuT**`Z>EyA=%%m
z`?1FO-neWTJNFtstvxwPae{%%=$kV0POAX}K&D<P`-N^pHPiA620`)VaW+#4qLn?Z
z_t{r1(+s6+_l|U)bf|x<059dw=4$p4OQVtQjaoYMv$9{>)42c6l;X1q{kZiGe>TAg
zHYXA+L*viQ{!RlqHBxjQ!#l6>nga;g4ts4djgg1b2!(Q20~PtAtl(NjXtYBa*TZ;3
zTsYi;<JceCQ9Yld(oxs#&6vkbDbr3AyuP3;t)KnMI@1g{?0#0njnS#RCO>kI*NyF{
ze!fz6_=Ha>1b9kY_P+hHm$TB5c+qdej{0VQ%f@opR%V@-vBf$rdqJ(9DmLSheQP6_
zX5m(!7QEKx>UdT?d|D8$v#fu0;}xQ4>y@vilaVEaxGrR!s*AoW$k$gglrHlHH!^L)
znv(<W95(9FQ>Rm`Cb2Pt5@!T}JM#{f`kJ-NFGqMg+&Z#b?H-0hjfI*N3Eg&Qu{XmJ
zr307-qYkz7?{=tVgbuW`box?@5$2F6#5mQW*5FAVg&4<iwZozv|3jqe-}ol=Xpe1}
zgV~5F0PfH_ZNvn#lV;)(qD_}!D(!wQ8`fAt8avy6RAf_*YJV|FIt%s@x;4BMi)q4=
zEHX>90=Q_8s)>4$PX)WW)&bu*++X)q8-;3|TooTBlKE+ooli6858IMzk2Tj=m?l`|
zgijUI6sxBRG|*ODW*x=`5|iRhDbCO_8qi*Cvte^Kf6D5tM8Wm{Wth^tKcIo$b0uYA
zN?n(@WC|j}glP;FIW2#aW$MNfCN!xcl*+3~E+^)lqI2dm0$n<)vj_;CKg0wn`MpUX
z(WD$xJ^I&D-PhDI1xd4?Md_<-8(&acz}Vg;P9Mw$DA_As<&!>Ql{|%2h<D;z;%sVV
zHboZ}loyBL`bS08H3m`~*l5TU$Kc{jV$C2Lu@a9nDW2ai<=4JZo!(<5o`F^rQ;3=;
z)56I*H5jIQ9|mDp_LK<$_IaWE)JG9AG`|=5!diAw*|yL6itSxA*!0+u85GCyI24x)
z!ld8NWazYZ?H!H{8@pjUyqt}T-5>ynx`BH~?8ZT~3kxvIzh>Q-6Im7ArI>v47;8jI
z@C!W9%|kuKFL!K5HC3dViXQb!%FHfv;(s{;#T|x6nYjwOBhKJEPzf@FgZ_2D7{Rz$
z)6}}kCa$t9(YJHrw|{~w*=fV4`s>=D9Uhllr*rWj%Wt<`lH+27`Sg#@x*boWj+b?I
z$D1`O{XFgGgl?|WaUA5YVb3jh1K4zpZp33J>IQ7Hnp8w<!oi02&E?Y;^EuLAaVMIu
zN0b3;2nkkhQB27u{+q;q6ABL9x8o6Kp~FX~EOHKhJo<tV7S~R3shAee#i%6gcoY<?
zLdbR)73W3%-{#u0Z=3!Z8yq<7xF+*4+!gE!Z;bL=>)*TITI7x*YDnacMlhPGkvpc*
zYR&olo2DoB2v*z$TNZ8|)I5XaATGud=468A(_s0~-JtnCWu2G_9~90>3O~`SbhFeK
z$HHZnJ0Sax>H64EJQS3mFMkGG75<H*F8?H=n9iZzIbz~4>T`Lj<K@P@O@Vp)c4ywE
zf_WKr&Y0^XEy1k{uO&wPa2|!grM<LcN68(=vA86eT8#8)_2o)SN-P)Wl{x?}x&o3&
zFnbx)TQGAD6?#?(A2;SCY%7MAAs)-Xj^%^Nc#w5K5DP<XrR*jCH13<DqI<3pMBxdp
z&=GwP@jT?yVGduo|Lca%|9`ome@c?+1w}{4_=!}n2&2W)l5_|2k<AWFD}_GF7Fe@)
zTL20NDib<ER33-1mH)|>&bvcG`4iwglJA*)tu4Qa#RXEGTf#O?%D>ZOxX?s|`uIka
zWor(CUCoK{S%Lyx;~P<TaOO6BS<#WghQ4(BcT{gkhn+M>bCd9o-TEIo-V}X}q8S|t
z0{oF(XPTrHwi>14cK<AQ;4o`&2hFn3L*QZH85XyS&_e+JU}0|JpkNIt?M6e0XKn~#
z1xk$SIaF>0+QyTS2=ULWHD)<iP-@e%KWYZaZZOl?Oo>ozNA>W1eMlnKYZjsKWa`d7
zsd7uzFIUEPU>ivKw)0;b|5f-;CstdAxykLrRwz~{>I73Y7b_9##Et^M{>0?Sey#Sx
zQrm&LEesfPm{5%USdpWTsfI(U+aY8{(0wq76~SgavU`*d`Pxy=uFUnxw%$-<@OCXM
z=`Jlmlz%D<Q~U6rxp`rGvBq`#Xa8Xhn{J|Zx{pSH^Zg(~iMQBIb>B<jz3594&7yPw
zp`!E(lGRGLkc`=<t*`>Wj)E4;K!Wz5{6n@yvAaD$fdJRVN4*ZlO0XjX2N5K!Fm^=O
zimU_m1DjwcE7tk$uqe-I6|sQa;lr&`RG<Y>^&OEtV2!c>P+^S}*@z{F%8TI9BEA|J
z^n+^a^h@Y|bpO&SE_#0zVWN)C9T*ax={3(`*2q{yjGhDUW&g`6Eai`xRa)$dv#;UU
z1CN_TxE_udX8RMRrTmiPNn0uVFWnQbVZ?_EM$hC}S7c(Xz-@=6SkiC@;L>WSH>;HR
zB?~;RWwt6JO{+lZHZ+QzWKaw<EY~e?BjFp80AT=PO8(g4X$#(<ENtPpY)Ogct5o|g
zL~EGh+C_@Trn{KpxU&y2pTWGh08<_^T8#CN5@W?WR~D=lJ3Z7~eCGkWz>zHcN(RET
z`=0DiMqA^{Y-f-OXt%QO4F${!1(>?G4Hef*W-GlFRUZ#G$P3v}N{bJN5Sha*wkalR
zAaAkwc)up?i!44KNs`7=q6|yeU^GG-b~y~R;Iqp<Y9>oW=LrbfBZ$O;_Tcg=av~$d
zbE5?9KGqUS3Fs9Hv)u<x9=XIfX{o7+3E3uaH-S6zhVIq?byP1J{;<qw#ke<xNG)0%
zBgLYO77Lk-hK2M_dz3KL7%)2=Tg~`Z@G^E9v~G)w+0m?P_0Txv?%$ef{csGX(Pj0*
z2Rq=3&~{@UiSvMojQp01WN~x$-^MmA3^lv6+FPahv51zA>i<)<{LhQl^2L-7S2hSU
ze;-gqUzD1r7`1iw=e=*8(F+ayBZhE~+Bz$CAT$*?RgizQ%<L{J>@T0#xNirZ0b`+q
zp{SbOn2W9>dynQ)nB=T=l+}emEpEiArIovB9Mpn+fk1l^Whosq(=G3me3t<gdpH>L
zt>Nr*R$2VOi_P3#*8gRlPQhR%|6C%*&u8kFV}z(UR5LoN=l-8C1k%TL;Ip4wh4BkH
zaesba%y2+bE=FnJ)1;$XMS8=_Cb!E->@obxUZZ-dj~vzrw3=swJxOT2gx!n(KQHyA
zUzg74Np&1rx3^&DlzKo{^V0I|i_*Jw%g-G<Zh+MuFD*)M4^Lc@x7PAa_57KKzx#00
z<{r*6Sylp^VeQ@*66xpjf1reFzGKJD6m<JvZrD*;HC8T%>BvWe`9JcQtSmiZs)gzA
zlyms@2{kll1};pW2|x4g3)83L&k^P0*#8G*EiZGG{CT~v@#ClVL+#WguH@ej_u4Gr
z?zxaG-u<D-^ve;6nByVs4pgQU`MdBo78&#YCFG51U9G2*???y6LIUf2epR>*<{kB(
zO09?3c(u`#SXt|#eS_pna5EMG7?yiP(1qz!DjXW%;qZ9;iJ-S3bmdTRO$8EPvJ~LN
zQvS<?&zGdjRLX~yhuD3YE9Db4dR(ikruDSs<tuvls}R}`tIM<`)z@A5@;uqx)0CHo
zdddaGP&tv9NaY^pPHRsK{i9XQ`8L;dWqo0&$50~IqBTzlPRm4tXG~>TiTpzcrNNri
zWl9QNuCz}e7_Hj-X=`umiDMHcex=beS02jwI_wbbg%pFA*Ei5QyDbK_G2Ut-N~E=?
zG~C4UmBcR`(a0u=^i~H5VZbS|U1P)4Q}w(?6I%v*!T_U#dPr)6Q=wU{G|RQ-SJfkZ
zv?I`$KB<=b;Goc)bDjbU5B0PWcu;h5ZFJlQ!?C0mp@=X;ygnf$Fb-jSb&=wLd=jO6
zWrIjLe7_6wQ2_0=3iwH*j7rc<^?uwaHb$1!m<@S4l>Zdlu%z&VJ-n^ZOnQPzh@6+j
zM*F4Rdn)vyDets&&^a~eD5mZ)<>tz(dYYz@lhv%_2ZZ-c7MzE2yM$dkezf<rxe{Rx
z^}AKW)0f(`T{V@S!L+RQ)R*@5@s@!ad!+ZRe$vvA<Hu4#&gW>Fs%097&Z_jZjY`*M
zZ(Un#H#_(eGo|*die279=Fe@{_j<(Pw1C)1HXwm80@o2TSgQ>3j#MIN1`4``K)Nrq
zC|66)@*;1&;-a3);1!wyu;kESj@7M*O-8#yq%x+X=-XsP(`Oq|i84wn<x@zPTrtSM
zGy10uc~#$55Wb)=dV*k=t{L(+VJI|PN#S>suh{8^#_$**L+|##f~cA`Xd(JF^=qNk
zJdHX_G?>w$HC53VdS7nE5yjfIHl_?=)I@8Bb2OKD4r}r}5%IMK3JlXj16O(Ef}UkP
zlnq^sTeQt4EmsPx!RSw;12j4`z})MYMU1U+DFjqU8M`i@QXD_pnN4kC8Gi4^GSm`S
zhVgB0S42^_8!I$eCNM={8H~uD(kPY*3?-&CR;S5<WyVotqisEmm+7F<vzMn#{XFB9
z=HrOT&uyh<yUltLWQHv)AWk9Kknm<#MajnCivgO%OU2CPXEC3llx8@r9NJ|vp83~u
zB9-$sI2Fd-AmUEC!Ngw_|CEBrunbBW8;}l{7fLi8ZIhrceUo&GFPr`T*K~b*ZCKa8
zsHR7kReYKLc?n0+<twFHPb*|o3kQ3e>xc+VHKqp0yM<Y8^!Jxi+v3ZpuYPLx+OF?G
zd58#bC)&V@A6-CRnpZMYgIO^a*=9N{RZq=~@D-bLurDghrJCxD9t>X7O*lRT_i57P
zrEJ%=Y6s{;|5#+ToRP#OX89mewo6-;q$m+;r9lcR)#zMhu#V%UrUS#pEY1HX2$h(7
zxNj<}=vgxYP(9P?tg?D+3)y4s%;bTwI*a3Al|cvH8YL<Mer%%pK-KC<za%H~7qZiY
z{K-;HQ;p^_xp+oI+(FfBqmE9q;i$t&)=bg1;!9MgQ46J8PI?Ed#FVm^WNID^Z>JK6
zR8LjIz}RM7MoZW@5`OKvl-hBvhk^qr6Gn=P<LJP*!6alZM+zl;<kI@1$q|1^kkN8{
zg~Nq<PSza+M;f){i)(=o8pNcTJ%|e<EPXKD^PKpHuF-5>*&Y20|5wIf`<6#>i6gCv
zRHJH78+1hnnQX$*uq>e(*ufN=F{`rBIQprXY|ljkO;79$vqi(J!lE5vx6%rmlfy<W
zMeNYbEUOd$|1n<~kE<YlG+W_{977eREeut9E0XTpP(?fiLlu5D3$=oiuqKb#6lEs?
z3!U5Z)(~a-D#V3q!#K2(gKT4zQ|1E_*(9L!8zXPBa4*#(V+%VOvPL`ErLp=%%x-0e
zv23FyBgT-~ghj>}VvH10q~IP6j%Hqyv1q{^sa)abmDC04&?3{Y=wYr)^guSvSWW$g
z@ImZbG#arGgs>E$XTy|N-wV;+#Bel;K#G-Zx&gzkLWfYt!Pdogv>u0n3FMPLD%1p4
zc_3n#_2|T?`5M`yk*OC!eTVE(l^5A%-m}7z4TiK*glL)L^*li=pwX_Xd)ZrMLMN#2
zyl+*<wd82SypCpbPpKbQb-iszx;9x8RIgddtSj1v*A)m!xTarU<7Iq>cFShgaSL(>
zQ`Cx~F4+}UAvy#b6h2c1aw|iVhQGOCD6r;L+N=k8fGst`2E`~JVNfXDNP~iV?Dk+#
zU_FhRCTW-^*uITvLL=G&|Ec}?qBw|oUPCcb6ZNtpYs+Wui>Hh6gHs!8Kbb#yX?FS-
z3A{z{5AB&97SM|b#i9FFCW*b~Ztqg55IiL2STzch0fvWQMwXPaJeZ-jLNRMi_F5(1
z#%fPfkcJQT80SLV#@a|9zR{RNlg9E07EPBmMppBf*IFDYxIArZa;3)NNJ|VBN0Pqg
z&t(J6+=!zHCA2sa6Uc~}VeC(x(Uvde2T=6hY<Y$W?GIY%%F7`aJ0HWQycR}kURW^T
zp5rP*SS|Fz_)JYi;?P7(5ULj#72Cw6U2P^Erau!5)4vi-e<+B^(5Vwh+opw&nb)kB
zq~-R%0>fFW4cX2NK``!n;K))>n+VTb(qLGBV}<Iy>Q#jlhXJEoMK#b<c!?2FQYfUP
z^jNEyz917xEW1+oQ4h+%L^R33Y}l*HRESWDDMV-?h04Lev~af401cC_MH-M+YsIAz
zr}=cHoAGWenuHx_Sx(E0b4tR68l*3<1S7LF1$%FKr>inUnb*^5F^}XdQ%dJHR7sq_
zu$)MsEd-GO;)Ugk=C;_elO=s1W_ZoI6pp8j7^;*Z3s&@w?A?%Wl5`E`u0-b6AVwh;
z7X0c$Kw8AytOa0hj8eMBz*HK{T}I;5b^$21>MGXy5izlWq`n4o113;d!Q7R+Sg4l6
zLbY5hRB5`w2Ye_fFn4*F<fECJi7FIDs01z3RHuoscZ0dHO2t5I&WfGvV$7|P#sS=z
zyBwGsCKGcr*5zf61!-jNikQ1B=4L>|+@yp4k+~5enAt(9n4}q5p->RjJA%2n6?2Ey
z1m?CJN@~mv7GMjgRH9whzp)<TX?NC>@)ql1Pl^wW_24kWuX<pn2G@be!gU;v$XKQB
z&UMINaUG=_av|c;3~i(aw$f@<UDS#ua2>B>rCl_Uyyd(@5)%1o%e%O4LlQ{h%1VeT
z8cxynM~{(_0g})yCLzxPKU;Z0<Vx=rT)7vKEA<PqRdA)2s{xOK^+j+Kxo}cYm7zm|
z$Y_5%7l{{=MUjhYk0Q4WS0WPj0-s1TDtEUl?N#JLe3P^-ff+R_Fe9woU`BXU%m^XH
zjCL_*Ebmj&;{6RXwltWLPnJYUYx6ir+LnSDP13dmNgJ4AM1|_K>K(z1+=>}PYXUP`
z6P1*XVhTy=+9Xex4a-5w**7zSjlm0&gKo@=3BZzAF(ftt*j5NY%8smPc$-*}WC%dX
zyap?ZFV#k|qP-r;ij3|^RwT`ytOycj)@xu+I$W?Kny0~vFNTZ;mTcgnEN}mJac2VN
z)^*+Y_W;Za4he!Y9Fjwk50EHR3m3<VB+DBgMN*;`k|ibCO&4T1z>t^)7y_6fX-nor
za<(RET4%M>#Im~Bv1Lcj-o{Q8H%-%4X`R+-y11#G)=t_cjpDD1zb|p!-~ZhE9)Ovl
zM8{4dn}fI9<=nHMbC+lO(OK5(WJZnmoS*}@=>h}aGIQ`ok`jeE%|q3oMF2j}f_roe
z^B1cO#?702ry~6P3uEEXHHZ^9`Mfe$4yp5`Au59al>?^Fd7mK@h>&J1Jsw8<PpHf=
z(;{8(13{)W@WZKLN{3qZ(tvUic*YwHD$7#S+6v~d>j4j%2h7_|xcRwt9&k@)oyZ%{
z5)YVi5(XNB841A!80QG8b2T$)-<;9)u$sgJCR9)bYO6w4YBGcwlp|+AjFGa1GMba6
z1M+}#N~z(7MH$i$z2(e5#$sA=2@MNYVJHjf&|1uE5CIT~kf&#)<|e|_6mh|WH~B)h
z#c^BYF5tmytA;V7;dnQGNY`CJ37gI~;BjKtx(hjl2&v*`whhHiUcCSkkQN3^9g`OF
z2TpM(gXM+rQgC=dT7(nBh!{fy*J~I+t?0GO*J{RsfriPPi2yfl$-pgKWW^okF&Qy6
zHNP$(BRK|ATyPd9*yv#$VkkL99&hf_c*@eD)`Q)qJ$NZl*ACUL2Y21m1Dq;yFp&s-
zm=<u1%eA*3ugIi~{A*xRYC9&iX7IBVP;d>Crt-8guV(vrkSd;66*c|ocr{_p2-G-)
z=*k;i%VYL(rOi@(owzc4J+4fKwT>$_`d8%2btC9wB-khwK;xc`!n2H6SX6KxEba%`
z=n2noZBkz$LzWL;0l6HOtB>-KNknr<#e}Al_<_lW5%0a0@ZTsp%Qy!XL2@%K`1)3P
zcN98TA=6a?3h^9#Cg><7iaJLIEl9^2Bpj7`4H$fR4X6_blt2{Fl*5IkPvWzTb73vb
zhZZugVNfcu8=qz5!lESP!cq>M;9XdVWgjDxtEBGfsoUWu->*89M^*MH@=HN65Dqr8
zY@uNaJt*vdTJw0yG;_{#6taBT9^~`$8Lbc$4z$@8otd%0d?X4#^3HPh1HeWzEhXUz
z)~Sm6JsN$-2jbv?sgiSw-I9T_H+fIa9SESWC9`RcUC_TL%eV^!<-a)0ssh(h6uN50
zHeq~3jdz2+7$w4-a_HxjL(M4zi7ctbcxZ8wK*ZGsw<ej*m}>q_pATXRyydUvHFnET
zgXl-c+~D|p8kTH;#f<<%aXQzJtS_@NW?&SSU24S{$Zk@jL|N#4X`+bKjTUnjzFQ;I
z>NlDjQioj??L%WtNYeuuW?8S^#*Mo&kR?^+4iwBoqQl`4z(7vA9dtExqLf_+jM`Ar
z9gUo?z7A!X+-O{@LIkiGtFaeHBQxVFHXy>+l;uDd5tI#O(tQ*Gzrb@af+*c}m}tzK
z3XFk}Rc2~aG}5%T`Wtab=V!h&Y7h^5Gog8Fy3q7?Ip5>U44W^*+f4#?oiCH`X7iAC
zmaF2+*r230aOF%yslCH}>q$UuDieXqP`EBdH*>r&6jXA;%>pIWGzf=eDio8JLszL#
z=5!&x8!J=cCznw)47Il$UlZf47MxLBZVjk{r)U(s;nM4qQ?7Zz(Vxx$&*q<i4zc0z
zTK~ixxYR${Wc~@?5$4;yvIMS<`6t6#I;FspE(~fK(-09c6~h71;B08}Pc(u=_t<M{
zj=HeYbHqrOBbZ9-<p^=<#w^DI0xv*Jfr$fA^PY_WW3*3-ympWSDO~^uBl|H4e8Gr|
zdx<?5u#z#Z=zMa@9RgnpS;Azlcodoi$$x%RICw2WP4bmR!!u-7I@$bhWtBxjo@o}3
zGA2H*oFyQ|6((%X<4$tUJhj}|1|GK>Q^py8JL6QZ8NMX=vK!5GQOx-)R(`N&l|#0d
z6)`<xMW#o@iZ&T%Q_<#MTjp3KB@2}WMZG>tw#UD=OtiiA39TOgnwVUK042`9mb&nH
zDug<fWHZ7jIkcenuLYCk%zeb5{E|*->36%SBDU^zyG29<*o_J8x|Xf4>vmg^x{=#W
z7ls3~)GkZs{OTk~wJe>v-9YJkcKH&~I&&mJf%mf^T(1(B9bM;Vho-Zgj!v-x4XEk0
z^DFt;)<ai|s|UMt9OHd!>%lA11n;7N6HdvQ)T!fIuAQGP=ZRp@rbE^TgS_?338aE`
zW9Hn5v;>3Hc06m%U`V7st;1K~S&IoXQi>_hQpM<n4){~?EGj-_(k?=Dbv-}Zw9VIv
z4KvsCvy)-1V?&MJR7@KE^|IkcBj{r!2$c$=cbFWlL+BrWMxLFe6*5OsiXw1mW%IM8
z>1Y5KG4->vB|DW2K^!h&=23viF7n8a5@C{L0mYs9*~(aiX-Y1PZ{<!YxeXG<Ryj?{
z^(KmQ4!W+FO`_O*0?Ta*Lij25vVm8gEZW0?BmiM}oQ?H)VzJdaC!25M;%hl48*Q&V
zagUP?rM#1kWsyec?N4J@xAbQ|wnvY_33{B5Z50wM<@2u#OpOp<{AIAc+r=@Uu<*n3
z4V98MpVk_-B`Wc?{7;1}z;&ys<HO95++qnOYbSg3+T*Iy^ZjGjq$OtVbdkw^$O0Jg
z{<&CgWsXU<!h5Vcm#9X#UXr5aubgKJ49^vQPz8AQtE94`+<~BgV<t}?{NuoR(0pIo
zcZ<i8ix1B^*zn==459FTH4@~1GfWp6Yn5=&9e~%WN|%AKO>>2JyKZF+SiOc~^JaIG
z|N62|5arx1YrV!C6FmHFs|V_-2R7Moxf+xp2x`NF#9?Nv2r7K91Br2Ma7jaUoLbM@
z<PnX!SKT(YGDIF5HQ@T}rM!Pa7psalyHB28_vD%GlPyBkySn$jm2X}5<lELg`S$LU
zNrOiEj?Dq=(pv)oOY4&-+kD9ao0O7g6KU;&9ogid?|I4+_YKN&!dUWjr)wc}yX*?~
zbqUwXzNGDpdnX7Zx4GO-i*colVeCwmApf3BCUY(GPS5iZ%g*{)29BlUm%~f?S(=M?
zY6bBK(;2a)@^kUi1DD3aqaXkB|9eWy;$tS7#JRmvqBS5#fAPy-dCE$;2BS+;<$<HW
z{Pb$(UBG93tcGyC&DW!`Qn|2s73s>ZBxp*FWuQ5ewM=}i9q+JZoG@p+X(j&3Ue+|<
zmffLv@)>C`_k{WF!n5K?Wj>MPRJJUt!rf95IU7d_6X)_bqd<)9-e=Va6}-Yg3nvQW
zH9_IW^;RVdKVaoHR4nfg!{k7C$z>0cWAM7_B6YfF4c-TBUH$X+H!QqcZ`~}S(9-`N
zm09?(It6wx0u;buYqg}P?o5{6DKp`kr2=B~oK-=MD5d7ejT%`4Vy4Rny&!_(jX|_2
zC9t~YeHTO$Z3&23tr=SjVs=9iGrb_buOEwlfgl$CZi2%;0CNF4__JZUSE)}2Yx&~O
zJF=Sn@4LhPi+Wj6FpQ+73-47I#;ss=*u^g4eXa=~b{q(Xare-~c#!DuuI4sX%_6@a
zQ5)n>Fg6ro{fKtGAt;3p*$W~If2t*RTwbfzCvB{|fhl2@{10Uqp)p$!XXZ9%!VqS{
zKm?g1a(7<PAj=tDeq5JEYy4VWYpw+|m%;wWSk9?(VqXSeh-=t)W|w%I?fB@CDF?~E
zwZkQO18at%xzjiu%7nCX_COHCX9Z56Sxq+{rA+KfyWT@LOPh-ivFwuP89!pGXWjAr
zwkrlOq#vkuw;PN3aDv_-Q*@0pg7;<%45Sna9~3%+{9o(34@kLpA5g(^--NHYN7zG%
zl9RLqUdX)!@x1W7n$rS>T#EE4oBu?H&32JehNY}iJ6Qk+OgHpHxc-Xua@JEyhXGS}
z3Hk>S!Xc`m9F=-hIKt9h`CcQ=WccLIkXso#>ek|_#b{IkqHxS7=M00OmKo~N!WWE}
zSm7zi*!Zc04qdS7+gu7g-4qdCTOQJ&R86~O&6OUI=rpI-raOTq$0GkxDP*`pnGn>4
zvNNelGfu&_&8h8xzA#5jvH%=%U_FTA<1gzFhA69(A)=xDsenPCo!<f{cg$&bo`brN
zuRUVZIlNjqf^(QCsAR2+b?E9PwBAW`VvOgtpVP`0enN=I=U=?86Glgy<bEOD-)~4W
zkR-mcE-#vZgqTaiw<J<-7vSM)b!8g?juWW~;p0SvdtE6#{XX>|3itwOekU`%0#0L`
z#;YL!Rx04{kqRq(R1FAc@tX?&UDslK6J`mBk~fi*39EIpBCRyKM~9a)v6T%#tINcf
zrruO1Tlk-D{JLh8RFB`pDqQpLgOaB3ZM6<)0Ue4mlr190%zsq6$AJb6(k6YK>X~8$
za#F&a)g$WH!&2zG*V^iC_ZDp6gA{&J5J{R%rl<j$0nOTi4fU95h(U<gljo-MUlG~L
zN`T7QDtwn=YJhd!N_%>;0Za(GEa{=;SA+fwiHs~E3Zsrq!C8Cm(h&xgNYAz+-BKlh
zWf@BvbiRz$^aG$pMv!>d*YZCrQaUWXWPrLVq(qF{P_};Eyi4R6fsszQBEY3FhA~+|
z5>3VXcc9c%D^votQYE&q9*{(E(Zt$EVS+Oe)8j=LU~DKvd(UMC(vr*(G_=hsQ+1us
ze=Njf${e9VKaG*_AezH8hE#~>Jdi8xz+q?JFB8PVj`~ud<<8-y6b~7j({yFl2xk(T
z3G^o8#5FMI?-~ucVp#E;WHSlgfy{vQq*Fm!{+Nn7)_ff8?=bzp{@7T8Hb0{)(Z^k}
z<53TQmg))vALay(PW-7K=i%I;Fo=KpM|Guq{&?ZzYC8Yub!-Z;p$;4YlUf`A(pXDm
zk#qclX{6u#5o)KW9VA-oXF>u*2jr)W9fVzx>(j;#VfF~1F(E}Zm<cI8%RtgNd~ht#
ztNj_PWn`sD{H{=06mE0+-D>8I=F$eGNxMxk;@}}OQobEx**G%nM7}p6MiPfqx}H!g
z98vy$#|<d}rZ4}U1DWpZ%GZF#-R?7Sgk19EU9eda$vRRSXFwPa3QVVwCyj7T4H(mK
zV3d~#h8nTUPe?n&7DZWA3&Xcslo;eU2Zf&%@e03TB%JKKg=u_foMW*0O=5+x@C#P*
z-I)wLbp3G5*jnV-oEr$jj7?sL-WiOt-Kfb4p@w4;q3vbFL@XOR`eN`5fm-%ZI9duJ
zZSLsPtKl<e%=iuP1Fa$AvtbKgkd(OHMY^#(celi<mC$Mwm47i%5p`m$087#|7=)6F
zz6w92!5Phv1<E;0JJ9+>S22R=l{5|u^RwO1iTE_^#wEgAcgV6oZz%Byr#RMbTwN97
zTjXnpHis&T(=)7PJqz}<n@s*&vna}L9GvO)6Msy*&ZF?oK<-7Z@Oh0!Qe!)r6Rl&D
zQt_7(LoJ0u0XM6p$T;A0L~iEQLf0_9!|foK*@%a6hO_Y(NvGQuROzWO{(Rs{AfaML
zWzmD!C1?wu5DdFCL~NU>HFL{8C;Log=6mrgtq?rxRyNV1@{J%sp0{rSgNr8!HC?|_
z<vQ_~4b4n5DXRsxbZTOb&ru)^N^o%$jV2C*!Y>L20hfzMsU&X|hijD@w{U5ZtG4~;
zVvcOnQ%=>1EnQ@qtMFT@E+;S3B||Q+miK4wE&M&b&0p%$z8f~fBRQwv<7=oBFxy9i
z2vR9IgR52eJ8ETZ#IO23>gND&+4aQnVd>aXpSsODE)ZV}XtRs(wlGXJcE{ts$hh_P
zqwt0?y#z5+gXHG$F|j$Bik=QL`9BH6frRh6V{~1aiQ(h|ZRoIUFo!RQP?~*G^bkD2
zPAQv+24$K%2QVF|&05)PoGJXW!?4C`o6icL^eBfjSpq>}ngD|gMoKr1nBmV&iARi?
zA-f<h+pbwYuAp|T<bK~NQG`~K!xtv+0II%88=K~Y$(RX2S4D$xshyaT(fY5R(GTXU
z8=>1HI0TEt8dShv{?Q&v;_0qM@lo`K(Nw4Jm<CB{8d|at1c@yQzoK@&X1@Bd9%#}`
z(kt@SFJxGU=+#RfnR^%uszQeyT6y3pKh8H0M0_UeoYQH-VZ{Aa0|8H2iwA5MOxAmH
z*as7fQi!M{cO>muWRpdEcru$7V3qy}Qo67aG@f_r8$ZHsAvu%g3Jq;twFHV)6w&b1
ze0d0JOX4@7M24sEkIdVK3-oxTglj91nLHK3*(MwfvyrRdfkiptM|&P(l$uW9XIww3
zaVL$lLT?=baR`4{VuI4KlZrJmmEW}0(!J>Q%#ldppHp=~;n#$m!f)!Yt?s1kVIP#J
z-d%TR?vm2fr3)0$(40taiWEsmykJBv;Fxtaqq1k5KMWKukP?Hp^1qicmnw25A<P_+
zPw^$Wc}PQgod8Z3diLxK-S>9w-qRj~&m3`}n<GRc*&_~C@C199_lwdqkfN;sw;BX&
ztny{(^O~s}chBlY)fz6_3Ovb8cK-Po{4tfV)sok&1VSHBs-IW1E=$VUfu!<x**g9N
zAA}jN4wb5n3!<cOu54Yi4A438h~#>3HUuRO^5wwwcyE~&o#w|%d7oeNA_1rFWUs|w
z<XPvr=yH`FVGZmc_O+auqIGY2r!eQdZb|O2Z_e6Il|gL08nu+2;$k(^ieS(9|Dw<N
z`CM?KMK9**cm0B1uOPKXE?u0&$%U7`1ws}6f&S8NfJd!%-MKZkE;n#SPY3N=sC&;w
zx>1^6#y}F2Df}n32JL!uYDS@x){mh*vu+VrVjeBR37=2aJIhBh*02~I+orqR{PRya
zk3yu=lj|^C4{7`9MbFY+#&!vdxk)(X_iSr`qAXet;wH;^!A1VLkZFkUoqBi+@j%!W
zXr=`uMj?(*7a+gTjkd&}o{h`dqro|OKZz*7WM+>r(0=w$%9Upfl<8kFj450inScIC
zrQ5O`g0s3hOe@IOn^M+NO5g+eXR;>eU%hC0Ik`dBPvtCgM2g~1q)2QjpYZz1Bmuj!
z*G9)zmN3`0DXBlBwoR#h-P_LpRb;|3&l?eFd<(g>Z*-`Uh-+6ayoI3Z1D>XM`dDti
zjO&JG2QK7eQv)vN;MNU@4X|jL!oQZ7!8UsA-ECcumj~vfP<xwU&d6RT`q(mL_*g0i
z9T9e#!YV3cC}p!pC~aF+vbaj6B!hyxR31WA#iKHmzR~PFMKGu|lr&z(%4#)b76g=J
znrQ<V0LX2j1ePR8fhQ`SQ6xJq)2t=bc)352(HbmdXTiJDIC^&Ihe4T52du|#QjVBW
z10I^!K6vv^oO_tSWbPrgop(%GMtzEcet#zy`eYUF`8SX~0%iz2=6nHV$`}jl2-*8R
z(!~JcX~l#AeBK@mu#L~wlQR8BI6LAnAP7rr9Uh|EK!O2V&&?;U4Fl{X)&UOmj_WG@
zm%)sNQwTV?Y2=5|_kRMZ^O*g62tBa$=0jK(5TAj1mJc+&0?p+zKuLas#7oG(zE5lC
zd;sC*0DucFA;I)Jrng1+)J@Iie>OwaICKsRY^>hmuLKVRCiQTjW3#!!?EZ`;mVBAj
zYfR7nk_<d*i_ZeMz#63XR#RKH4~>WMTc{AW?hAmOyTO(CMcoly`pg;D^930XUalUY
z?2BykFz>%x)mUB=uAjB4NkeJNG3Tgfj;MjvC+C=Wev2PR+P8=s#>JKR<qaE1+wZo>
z34c8URIWfeJHJDiGVe?Fg*Jo2zZZV<-y3E^j$75+Pn0u9?_!bkS2%5eby-`$6^~hk
zr<cb9|KE8VvB+tQCX*L0-~SvE^&gF9v~N5RA2CN*>0$}u(^u>^lYB?!+gGnWW5xt~
z^732W^-In>xy7`XpJ-v~8s=k|v3MdZd|K^4FqQvq*%==xd<tJW+Z%(vq|TlttcvF3
z(<6xS&xh&d3+bg^evq>IeU@K24Zjv<j)-{-;9nz;mE{{P`b&F--&XG?vY^%0{KI6Z
zE8&Na1I0d0%*7R%Y$)sgr<Zq%L1{_}1Dfa!IP94F2q;M%5S~f)RimJUEnX_}<yET@
z4p24(OYTrEjxLoLKxBD+x?vo&l&Bvz{;OvI&>Z3Z6b{pgN&ExGb*_~O26dSuF!O<x
zfB*w@z?oMtRQ#!M&f`4D|L+q6yU56ePYz!TpY&9Dd61aKml%kDg`;8kQWMIJHeL?q
z=9#=2h=2c^bucQM|8*>+E5zr<M!5v%(%Vvbx*Tqm*kfpyBP~wac4bHBI81SXBT(F@
zXh7Un{uLe4vqvKLMX;^N727h~4`B>A#+M_K6S{PN?HJW^QD$*lDU{~b|DvKoYxlSq
z@6#r(kQ0@|QqgpCk;&3{pIr2!iwwu5VaFq$>qAmS66a;gMiW-ZP0^Ib^L8S08Av=_
z<?)Bcm<;qOeChQ_sUv;NEGYaNaU8o4iet*e(R>k>*iLB@ZDDybJ~LOIu%#x-a%JLM
zq^?;%M>&<qat>_B*^(2b?P4n}rJjtxSsB{BD4IBo!VcHp`5QTN2#RW7pJ&`5+(kQD
z94<LgE-ItvYehPG#uw|Utbo;;zMbwJ;($)Dg$_1h#KA`JtZ5qM!a$n<1yL%T?J$uL
zuRf&|xqZFDK0s99<P6uJ5M=&A0RVUEkPEZg0bS%wtH|b1F(~n&n4OAza3Z3e^Cgb6
zMm3Xr-9dU*bq!Jnn@;;FmL~iQBkIohrT6^Q7i3Rh_?_|SLU{s*0CY@vay>jnlLEQq
zP*I)`uH9fqWp07t!LiLs47*Ygj0#4(o^rvaF-_UQLRKBj+qwb<DO470s9QNg$aA9d
z50)FITT9`P?g6Mw;XjK(sNt6UQH{iaD^OThV3W0wO$zK-SHRZ#b6T};AzD{p+*%k&
z3hZB3fV>Pa3?>B*uPd<I3JfI$?p;?PvI69%+$dhuL{BnKjq(=W!&XXvh?OL*xIZ?<
zqQ!S`11zy$z9FnH3`$}4He!!;yp_q)q!1iUNbC+ulqZO+_z)$X>#-BEl&!hX^sqIx
zvKz4VgkmWeMuq#>dQw9w0Of?Os-OeM()Qf5vnoqACdAH@dE}L~TYFawBI=1xsAqyd
z6_4oL3O@7##A<^d_KKAzvPa-K1{Uq^W#-*t<}hN5g+g?v6Jh2_;c1r*l~c+mX+gc9
zxF(`0jdd!9s}XR}jHtK-e=+{nYjg1lal0ucDb|VjzJ<B?Cq4u_&VkJ`iVD50s=_e9
z6yqt=m8_BEZo3Qab6lVgqN2vEWi8v4QR6!z_hj!yV^YweXzQlUvIVe9F<}Nx9dDPQ
z+v0tc<?%5#rtDy3@3>Un&3DC2?lD)W%UAm!!?-plMOVah(?yZ<gkm-(YFtz0n>b2w
zZu7l`;#jdzC=|Q@ZPH~nxi1ul2lRe`2P0uFQ!I`a1HBhvreY3$T@5&Gm#>A~TiT0$
z;HkG&33wn8gR^-!VAd*k)4<?Oj9Dqdtzc)9J5t);OeJrg$G_QxfpF#C5C)reCO7!1
zo7f}MZ2@+2H&$u8m}^-7#08)dBfDK6)V`??UPvH_%X`nrI&LX#Lt92S$<dh59c-~p
zQOUMw?+1jm7eDG5AexY~9PK^2<q0(0-lMti;FAeO*g-vEZ?ZI!TXtBP<12}o(v&90
zXtLX*>#(njp-b^$-|&!YIGZ%w2e$7e!#51J@C5`2ix!`ZiVrbX$5?JuwqiLNI0~ds
zmG{DRNJC01rv<jhs|A-|;mx-=DoWa()}<Q&z2PWKO_lZ-w=-PjNDAM?0QQz5!y2OB
zk1h(0u9l;tqwjp`1L>F~mhTHSu*jn;A4MN97o>{@@#km*!#}z(?`sw@v@+LJ<l|>%
zj(3S(lM*wIk=<@$!P)S;rb@e01Ro8~qCj+rG+!_fc>F6$Wjdrqdy)U5$J!2ihs|QQ
z9sKad;GSNt25URWY`X?V^RY!&sIcS%OVWJeJ?QWiTX0mIDs9W-<nYaBr;Czb_C)=W
zY+8AToSk05K@~p40f9IZi24xr(^{m@No~aea1?^=zSnUSg7MAF0da+?)cl-&OYo{R
z9;I>g#xqAi>NsGQC*0bT0kkjQrTz~F0Itx3Ww6O7L1CPEg=XVY?VPjEcSVF46d;|V
zbjH9Lz&<!ODwPGpyqLRWRk8dmnxb!f-Z<jR55@WNc)XgSCANK*C#KwN=PqjTnPl@o
zEJbfFpsPjUt{+IXch1p*ZKTjNfecr-FurA4mL}!UGS+diPBKbh=^Z1VL@DF4K5R>G
zdRtSWZes{JBi}~*MY9GpZt7M3=3n3#)wlj3)kSxDTbJuOvu@xHRBee76ZI^Z5h{Jc
z34H0jr4l?_20J&$f5T+@mUYtZ-hNN|HvU^C@!zGtPU6c`rCXx%@zTvAJ8esR?Y|%m
zY`^psC0wJjd|epMm_tW=0rt3O4c17AHO8f-ptk<p0BcaIk2R<UYba!aQ`fQv)a(f-
z0K-DNqGKI~kW$)Sl!_XUdmI0pj=_&VZqxhH`{m+P{5|@hNc@sLD^5kFsC0fR&ZdoY
zg+Q*i<B`9x!NRlPHrko;h+_n};`f6&m#`=^fM?64V}bu~S|Tg$vH(7>p*<F4@ml45
z%xw6>NYnchV?*&E3PPH83P(z{Fp|=DU~mS@NcFxFNk1>d&y`A2S<(#wPAWo%#<_J0
zkEr~bx$+(ft^?LX_HsNECAEWK)}pr3hhcs5{ZR9WryMD-rb<!q=ihHtYrp>M@pC-`
zB!>fypz*diR047weYl4}2&nlnOLG(NZem0-$K%?;jw_^O*S3$8qN71+FNLr}-$-xK
zw_(b}^;@2Kvdm|Zj;?<H|M*OHwd^n6`P7>!RP;@d7tP};xS)DM4^gJT3^=qnu3og=
z+=@#xqrsfb0j;v*!X}O=Up0ztnUq)!rLGD3UGdJjc%&g8c|R^Npi0+{0R0H#@Q%Le
zFXvs43vC8#G`UjRO}sbRQL1zbrLZlUFty3|lcYE?kSozPbIkcx_@!t!BsMHdcS<4p
z7Nc!4y4V&`p(vFJQ5+}nnOg_^9(`i^xV&D5zAyfQ*0YR9`<)w1-SPQ2vmEUgyK0bR
z#VuG-@D8e!SEdD(iANS>@&(^EbJF8cgL#KAumj6FE{}g)7#Eg^5^#!Lhx597fSccC
zwr}bZc%tJyzR~#dobg1wO58-#Ba6iUu;gu?Rm1q71(D;@Tjx+hByor>Ou5OLk1Whn
zen?Y~c$f3#{qYdrqVff>;d8N$muA46g)r$s#;kqvw+<~p)=8amfHwlsg-h`SqcD2Q
z%d{7}h1mVZzq@Vp8f&AxFB(_ZYC9^jkIr%UKAZ0L0(9j5Qi%qQ=vv-Rup@5U4Ij#o
zY8oxf9}7(lsBGTX`|Mj12}~}&^cL%7%0K6Jl7_>T@_wR*$!I?(Kim$|Zw8uM)bs&q
z!Q0=<_ito+^7IU@jGY72=k3o%B`F;}l|hDmsO$k1wuYly`8XaH#1Mk&$D&!0XhIy}
z1j!ys35aK50`XfwD^F4nR{F~KA(pZYJOY8l6MqUu0#`)XVi!1-z(JoVz#EEQGoQFq
zNJf$PYvNHz5z?*oE)188fMxP*q?-b6g#sldJ@F=pbFaacAucLfV8=r|ddEYT%HsrK
zbA-6p*ow~eP5R=Ps+IS{V3!p^;s1d-IuI}T&!XU*#iL7<fG-he5_X6q2}OCcPArxT
z65w-!B-4Bo!5_<JpcGa1^N}uwIfpc01SHMDLjr79RFs}~Dp!SeK(XtW%QK`eqZ^0Z
zA<vZG-zwic_0ZTbmp3DXERUKi?K6xd{g-cLKoV6cRV6M*!39qQfhl_{EzfiiWDl04
z$SmI7yId}|aWNUDf5W)P6`mQv6f}UAG-y!ZCTb~mQ0QNbpo!^<L3C#vN~w-Vo4KZe
z_%OU_dDr=PWzM^;VkMl8*F-ld)>BXMf%pzV`rwqF48j-StUD5%GHzu|%IWBac~qV&
zO+f+gId7-qozYakd&Ih$9`|Ux$8%=!#Cp&9Yjlq!v*uU#I!98YfVgV2VtoJ}7QBCM
z07kbGKFf|wM0*~dD(@&wI!D3yVp3vYp8v3^3rm|Dk3#7+VWTL9ZZHmz$;09}{ebeA
zfZF8Vkr_rk8QBVlaS~T?=D{)h7Q?tUT-mYtBjf@oVbfm&@!|V|w?&(bQC$-e=20Yf
zkQ6!Aw+J^S?ZdALH_=g$BEA@n@v7@nBGG7`$!HHrFA?Hq1h`w=?PEcE^u#|DLZ8T%
z?eby+l)I%)-F<?z$f1nqV<2P9`9b{XcAWrW$x;ED5c)96F8JOC`V@R4Gd5Bk%${5m
zOvxEZN4@co$ANc5{~ny`j)VKlrnubjcPj@^k%vS#{7sl&&OS{82L2Bdd!JSKs-(7C
z^~1^*ok)#NOA{N4<)iK7`X)<ch;|@=_)1VPnncI(b@Ws=2gKLWE^))oXu=dA(-i|?
zcDD<1cY`(F3CT&2iwoBm<ZkZ~RU~`8sN(+iMKWJqklW>$J%HLneu3yjwGTiLwFbbc
z9stpRUvB`xYZpC-QTy0^hM434cJ%ZP`+U87@1h5Xql+G+AEqwnsHX+aE=92xIbXLF
zh3^81NN5C$!bo_c__%V#U~v85Rq!{_X<j%Hb{JUVnCLUjfcQ*UuT1u;G@Rg6CZ#RX
zsv`)LjAmC^IcIS)ny?(TD=F8|sz|upRU)_UF3~Do9Ft~Dho_Vfwq5EIUxjh9L4-cj
z!s}I*X*_eJ(?WENR?xqGoj{m?@Ozc$b%LTGuM2j?crLgsP8W3hj9z0lxd%p>Tm`kx
z;A+4x`+UTokv;QWFtC&lz{)LnU%NW-UZNU4AF>CWO~!egRhhJSlSHOS98pHwd5`<>
zk<#rV;bhza$2|{W5kK2@d;0V_{BGw5fuR^@4i`zhHyOXfg(FnzDc!C2wOr9>QPGT&
z-s)_ROq+`X2m+$p-~TL;fQDfH|LyN5EHDFZd50b1SIuE`eY%wYnM}u?$N7~1do#hk
z{NHQR9sm2POvQKk|8z7eH@+NaS_qy`?HA?O@tN9q%bXlJo6YHS?aXzTM~IK$%%q#m
zh)VDANc(I&qfSd7G6xl1yt;Zt^WSjvx@iKUvY&IQiXb?AguDp&5*2Y=tp~hO+;Se`
zpFlNRXwl7tOiQkc@j<QC(X_jI>AqF6!};Lw{;qj~`P~p@KAL=x34J14km)<2H~F)?
znc~^x6tfz{9zBjJIa((tX6-hVlgS=+PGGWLoqxR?(>>_QM1sU#6N$BZ702|?4M4AL
z`+y|8l_YDcoYyq+ThgTGpPN^&uq{QNP?w}B|H6U}d>6S|q$$CVRt1vvasC@?Y+>MQ
zrLC4L9GtsFk`zQ+^FNg{^(dwB%Ku|}^YYy0;?{s!$?U-3(D2A;e$&|IEd|7W%`%E0
z3h&A=`|6IbV_jsBuCU-0>Avf|y75}^&>XoRn$^5a5Yep7_io(dH-y*i@padKbvU?g
zt=IOy`l^6gGyGSr`>F+car?C(@Q!TodXcQ~Il2CXAnU0LCB;Yi9}Bx{FIPD{bilC_
z3WPP6in!U<#_W~kGYa*L#dQ);{W)3AdF^Iv&)L4WM!jWJvMX2A!HTeyaNU#y?e&yn
zb6v)>b$)jcfBwA;!tKy4d`1J>mF&#@$*{ZZZ&kZ~ye#D4qFns{w9Lac2X+t_J%P6f
z(O4MF%GHP2X?8`=aCBY3$-ZU3k?#j2QMM0{0{=^!_>Qrtgs)8a{d{02*)WtYd3XM!
z9Tt)kp8P%okPilX7RRx<T$^(CXO7t+u(n!|MYuT@*74Bo9w@U|PP@;B^bt}SO>Gex
z>RX-5el<1{NnmWTu6><^ReWyCDgCoMpUSrIkL56Hmi9u-{>lSvMbaW?ci0I+s7-S$
zK-8*le)!Tu{)Ldep7=$9)<uMIi?am?Gcn0oP#eaP9A#Z*;<qcQh<B1pv=FZ!pWB?x
z1R>bSDoD=iHFKqLXqyRXk_9|G%NS<6E5)+%m;=3_=M14X8>Q5*Rj}@@bF1LirBDhO
zSaCg-|DrY->HYyN4|e~fjyu&?%aQ3!jy2%1%t8rSfQr~OriBNrd(=@Ms_TZS^Le!m
zLdOzBxwM)-gT_JbJ!M(u_ljko>M5Ij#j<QVW$ii5lPq<hh{*s@fPzUkN!<&Jtf-DS
zc0-E4<>_|~FO&js$4V)N*DmYqCmQ_9m;d<k9BY&*aD~&SAiVW7;CiyGFxExcZ`YFW
z54Nl;+b7JjVk$u#g>k}^8K@wC5|l$O@Eth})3<QU@_QDyv3gXYg<m8j?ty%U>5M<L
zg68-RD`U%lVaN_)5ZNda!Bf<8B%tLm;+6*GjH&@oCBRYRcDSkz>jy<}i%N!a21O_<
ziBG7KkV?m}_Lt1u7l@=!xspuNsw9M3Uov}Nz#?IAW^Eb<wcrc}vgNY$_a~#kXO>Jw
z)ROHC$bd@v)ea^bA6t=>w$CPe(6aH8P6{YU?ShfpFtd=)u7>$=6_s^HDwwxZc{4&7
zrA2p|5Vbd|qh|&Cs*|pE+=FoESQ2gR@Poq8TF{K0kZS=-#@1`~G-gXCv@hE}AHg6v
z!@wpT>Cf2{v7ef9{3JIt!^wQd-7pRX!5csa`!Zdp{IDuA<EB~Eji)F05?0**qHG*7
zFhi3Fy*Zl=LiQ`TF0DN}&CKl&QP?mtH-mGx;R<Bftw7X^wiwV)7;k1T$w0hWB}@GB
zK^M^Gnh_VNCd&o`nnAv~8JN2b-yoyGuu53X@Ds*k*36hn(##f`$r!i*YD8qWa^zP;
z(uyCFx(UL&18e!VVBnsBW}|^a!PpyuF<l)9w%iwNp(`k=uJ}RrvP^7n+II?P8Qv76
z5PL;js;gnH9KcxzJv$_5V8}twcR@IQ+?wPk1YRLwi-&$poP%v~pKFB{_+E^eK4@39
z<doUwN<eX>=9zG0MIg0`@T18!ia1j9%##}8yCVG9aBO3mx^Z|R`;q^SY$j`Gx6@}y
zER47!IC+yBT9q2nGn4d)8Aurre`}ld+)ER4VIqs<y4Rfln-;V=Mr#FgiNq!ij98B`
zJ<LlW$tdII$Q(G`>As7OXz-=F_(_~p2?({Zi8K~lZI&r&ce(wDAXohB44H_e;7)a8
zC3d(ULQK6YxSvf_TS}`fq*oWyt7_5|)XQCpceLDRRwNox_9I8}idGBn6m27692LMf
zIqu2sa6k3|k9)HAeG~0}$Vlu(Ipp1~{Z^VNf0N<aA6pa%Cs4^@<HS_{b8gG7mrBD?
zvRBYFD5Nq)&;Ag9dbT&Kq1zXNfA<G+-sar4R%z)hC>Q*0??jf~wBK0FY*m@FRio0<
zH&_xa{puxY5stMv16%b1KegeBev>F9IqNB8qUGD-yY=N59IchF+YroaDL7duHDtrq
zK$-cZt{W(>2Zgfv3)(&<(MY#)x(~C-$Rr*L4U|-kpO6M!X^(kpvbTT$S515~b>MF1
z^9Siim~{jZNMY|$Std%}*+eJgUkc?%1tq9q^bolNT>+_xjqGR>k+oyi6!>`Mpn-VK
zZ1+BdPU|0nMm=;s2R2m<`VK!z(=mWg8H+(TqJ>Z`j2LR7jL@hNgDCs}LjVY$&%eVB
z9@DG6#%@TIb(q#5K(BceMtEV*MpM>F==uh@d;?BX(ho!xkRpX4!`uLC!1|LPBpseK
zF`xg#Z0gfk$5d;3j8WE{J9(VuIMc96hkF6KffynrC*B#z=tN$a#A&QJ9vD;`gGQb>
zury~-at@b~&TR+caBQ`aGmCP344+cUTJ;iV*42Z6uT+oynOe%GdWmD}>OtdIsy9SE
z#4rnFvL95($py2FUiOYH16WZ`0PJ33K>Sl|7Ie_47y($(!10G+57ePpKxqm#Y_#d4
zU#u<8w4iCD&b{9&_fnh$Y{HOyO!`VXGLk}ML2i^cL_-`~mIMB?F{<80l>W$FmZ<-s
z#jPmWU15kUQ6zL{;^?S>`H1mP<@4Wr+W-rQhx4Dh^}rob6dius;VZ#`!*9Fe@Y}dP
zaQMm{hp$9Ds@7(wmlrw*ms-u)*7W7zM17&wm|m<!jb<lm*E+%3<wm34xDYj$J5h5s
zYImkP^_gg<x!mZ~TJ2zZVWBxQ-Kjk>y|7$6LnG9`KY2NGu{JY*l)Gv>c;t~M9*ul~
z=&jQ&HF{sPuN^$zm~S*MH)yF<o337sF1PBPPOT9%rW?Ubv(c$HmYd7%Xt7zXMfG;H
z+-NT^Ej3%6S~Y4mqRvI0EiFuUW}B_WWHbVtdKZF)>6!URAVokquP61!EUhjsPB*I2
zLcLK7YK@h8tJzqrH9FDCbgMpnexVkeY^+Q#)T@!OqT1Ez&UDmiMvJx9g<7z0p~}C7
z<;uQlaOh%lv36)44Ac)@xOnN%<z{RCpxqv<)>aO+FV|-~2inc$)=cft!otep0h*kv
z&2$a{#1c)n4zxOl7V75@`K}I~Y_vNqhbQ1Qv>u%Kv+hPzr<dvTLF*3wU7ijed-Tk)
zlV^id@u{E%sq1h=eCA|u^3(~VWaWu7m7|ZJIC1=J<=n|{I-WjzG=5v<)Z>p<o`@eg
zS2<&EAM<a|oIQQw<Rix`PaHpc^z^wk5A`OfRKz5eq~O`dDko2!e5`W%)cwbU`;8NN
z8DVC*)q-dXS78JGbh!dfh#sAuxp?68ozd|}PDCR0{ijcyh{Ph(o%X@tSnd4sg$Pz?
z!pO7&`_$p3YO~gMeD63cCBMiHF{0^C<dJbO5}n}JI~`9%$WBrm9&M|1=OS$9Nj~d2
zsJU=3aNLlTcI*igcbd(py@)&=ypF6q-Vkq!4KLI>*1987v=Iy68MSJ&H4rp|oYWf#
z+QI1Y6f7_G(b9D1;*E88vDxmJUJ!OA6Lv#=w7~URSCJcQeF<6nsvEuz9;LV>T+kGB
z9Srt2mKPRo3r<$AL~ndkbgt6^_UP7Iqw>i{wRS~>TAFU7>pCr6t8m1a7VC}aPP0`G
zPAxA!ajxEhGu|8>wtC$sGt-UNbfWXM=<T&uv)oU%<E>V+B?ULS;qxvrO{b*XAU?Sh
zCD%*pcKz+Ouf3&K?Q*?Qy?#4R=hUJ{J+q_iP513<N3+uq{1vLc@?cQ5OV@8AH3hHK
z$n~K<VW3GUWH)LK<nq3D`Kzq5LGctabdOgyuKDSinOeIoV`N4$HAfq?;kf`KYARH!
zs4fM+EPLa$&I)E!+Gnwj^mk@1M)yUvD@&LN1a}|$<vfO00>7`ihLckG_U*e8wW(2`
zt=C%7#pyOI36Cd_s-{}7cvm9<0pKRE*Kchxt=fuUai?G5@@|751Y`Uv&CH|~zWQcP
zH9N5UX>il3WA~d2U0W=j6^X*pN_`sn6773#53U)PYv~*a^cL39&|0~Xnu%DL0vEl#
zf1++BA0j$^p7SQdw3yi$IUu+l{HZpVrMPh^mWBOx)HPkzdRvlSJvjPU6FqKHU+>%*
zO;@YYfrS~og?gjYh6o4L(gEKRU7K;KH|C}4<jep<+-}!eX2`no8tvcL7KHq(-nIra
zDgSgOuhWvl)Tmwdz}n!V2mxqAhL$&M%_ZyQz@n^MbYNxaK;jM_Si+%cBZqH#W8=-j
zP~slqzO_6CQaU7+WLCX5WDI&R5;{%Bf|CmiwF}b=(fO;$cDr_ISq3dQ)?A#1`(Tb{
zD>z$g<G`9IwrWeQT3eC@P6H?(;k9+H24|$t+a_hzTBBa82B!hkiJwTy&7z#nDs^xq
zkU`UYSh{Mxg(NmxSJO9ek)+=oWdya}?z98*(oD)0rg0F>lsSWcbxCj_2-676LTwfw
z8b719QfCyWokzV`12<PS4D%9^_x4pBP;5|Zc}Ys3eHBk*5lLKF=$7IMPB8u|t`F8?
z1vO}cPCVCktu`Or-)gqo2V8gZcIT--T|Hp_l8w+c9UMzWX;`FXa0bmA3|*Q@IK|9e
z1y8kB59*{%aKe{6{e@?j+hFoR90upF<JQkyG~nWDm9|U)Q6@OrYR;plmg*qs<e}4^
zo(Ty&*bo6>P_>i<9)FI$T?(CSL+<YFW3?+CM@Pfx@hi0%?*aOb5>!DlQ^!H<FF+Nr
z%O8`_1?R3Vo^LJ?^az7IYA$u7q_ogulr|OHTw+K<t_{;8m@yf?tF`t_tG)z70c%RL
zqtkdl)POOP*Ba~0#kx~_#&0T=AmW4yTi0T@AnjWK8k!XC^!AW4R~7o}Wpuf@OykWN
zR8epaOXLDRn7xV4HDaA<!KQM!3C<AxYQNZ^l4fcsuHeLS!*PiOslK?hP?KqMqJ*(&
zPa_Yw5)2I{!xTv(P9lXJXRTUr{|coa5pKJts|FN2MfrA`GtC80G2y_s34$cuZN@^7
zdaTw)-8zBxNP=rGEvw7mQ4d8IVQI6MiTrTXfyrdnx0b%vkh)~*dz^+F21xfVVe#4N
zMTw?bNMD)=`4I(4!Tvr@)jEnGB{)?D#mW70qbo%lNoK><>lMW5^A;qFlg=;C5({`O
z)E&?{L!KEVC+r~0KND0l>BFwJOoDW@?^3II!Njurw)ZJa1(O<EU&KD4kN?tglPJ1&
zWu{iEiY#dD`o-oF>R-xCEOY<#()9WILcLQr>1hLYeWnht1pWUlXHEtGHvFC3Kg{h2
z|K`9SWVYwN82nH+%-%cj6XDb0>FlB0p5W61&jrQop6n;X-_8yP&kqD)HS<SdZQ!oV
zcW3@p=0Nt5+<zH(U*=1>-2=k|AIm&EFp;?uoXxy3`$yUT8vNtzQ@NLd4`r_o+>&X8
zi`jDS!?|O@ABP_ep2>c5@cD2k_x-uQp1UJ>Tlo6$-N7dZek8m<b0Yjd13~b@WDx8O
zMuVL@cM4>l=)V#D$?K2)+Zk*f<$ZQ^C<v!UI6WyC8RTU?7*ri!(@Oe3o0A(_9@G0V
zl1Rab9*+e4P2VTw(%((#RS;~`wfz}!kNMmCzNaXcg`V3jT<7#B7;%sJ+xxz!D3`^a
z+pW5`KO^ojfBF9|p6=SEzZ?J8_ui1Vak=#AczP8CP{98gagX`i`@W|rm)m=8Cv<Ip
zM%-im^8drH{f5`w^^JGmbMNcpqxT;>euATx4(2}{CKZSMg)jmwp-d2D`IqCL=s(E6
zA^xetr(Vw={!<-|Yg9Kq|J_8TKcJvC=Xl<+w;RDPgbKBr-rUmwm+D{tsSS5^V}HfE
zLHxnP`D8%cy>K{x_^1EMwjbOx`Qg>1l`pMnH$_BJApY#(e6c__M^0J=3pJxZE5Yl=
zDiwK*6>QAxawBQCm%h&(dh*h#OYi$fVHkvAZhvrla9>X);z;#jlEtqOo7o=Z4s{ln
z3g=P(t*Zx;6wizb{`SF~QV_w;9c$jM+<ox!bbIl2-w^)H{x62X?Uf3K%d};M=k-dv
zF}>8j*z8nrCs*pV%dgb{+7+U(@bC?mRm^TRG6!D0hEL$AgY4^Wym;D2?gmCMTdmA2
zG~2a-L$BTeWoN6(^a`TEZ~SU1;3-yS7p5<?hwi?yYVwrSlPYRJ!#6e-C^KChxdCc0
z76k=<Frzm%n4|{YWJN5LzwL$wBbYYbSW)@sV>dQbOKfbu=hY!lOk0@&h)%7N`oUZ7
zxWS5YTnjf+m;k6k;M3W<wIXpro{3B<ijvyJJIVFeX348pE0=52^Y>W^Ec*GG3QwyQ
za;*lX#f94jE6}CY8P7>mOiX$8fMX%Iq2g1m6+(s9_MK~9p0AOSttiNvoLuv&Mn8J7
zW7nD&u8!#0-nnDVJ5{nbyKY+Z#??xK)v2{ms$%kT_u7}P?n;BXi}p=xhoGA7)t+Ps
zM(T>wR+W0Sy|=g^Y~XncBgd8(mvEuif{u)$YA@7hO1G>5U2hu8i<KpOVzLnxMbYi@
zTC}K?PeR=c^O}lIXi#r6xMU8dqB)p~%TiQtDX%H2W+cEBba2Iaocmy>zUDS4Qz5}Y
z3VC|r*4-Nv79FkC*F+n<P4I3d_M>Rmb(W0WsY*eX)VXaftrBFZ%iBw9+VZs+e5ZBQ
zy1QfVnj%U4q=P%RHE?v>_q)lkGmZoY_984Z%S<@bj*1wk%ol#Fd4@14UZ`Ja94you
z_0Caq*x#y*X6w*a<AL5shqu|I`%To{i<75cThQoFrnOI={svd<7;zfiS(7JEzs|8#
za%+9u)&K0o!gBlKH*U5*?ENuNa<?m!-k&7YyQg?rAX@rEJgRc{PS~^4jsC~49|S+7
z`R|*uW*?|^9%)WjA8l5b7i#Ai_jG5uReQr)p0Wu<NanKniVUv4aV;ZQ{=72VYA$-N
zd(-AN4HEsIoPBG(ara$s-c8pjjAsdPTh#~A)V0<**!3;j*T2>CBU9_2TO%4|yl4GO
zO$#W)r}sxUX!H~i-9Jga6ZKYmEj-7@dmB^}g5mh2A&cbY@=`hg6hZ66?j*6+``EKu
z`++UVTc~gfnSJoqHN4u*L03?~bX{2y=qJY$;yiIiFq}0kJw&FLkc<h@w|8mkGujTD
z58p%+z2iU81h<{qTU*mh<klY9vGMD(k3HIl<Yd^s+NrIo2G&lccs*V0X~||~&OUa=
ziuEA>>|=k);Y~9cU5IT!IZZ~K8(ds&@cY<i4NS=t<LcJqqtdeO`Uw%_{^ge5$}T)o
zYh37DeCy`^r*FGS+7aW`K8`^=-7rpGIG5z&|8jxi2F00Xb3qjSrakK@k)Rd>xlsEn
z6VyohJTu)n0b3t$to+q&OVh1(EpAldh2-^L8@GqaEv2PM|N3^koo=DWn#bG>e`SQ0
z&zw5P1lTukV*;X~S$hVkIaD<<UG7rZ+Z(7zuE9^hBdWFYJH<FKmqzQx`k=-3t@g}X
z>(*_py4I+)u-&9ITNTq3Htq_*&}MDBJrX=`t#>YxT09Hyw;Ho+TkqDqFp^feI8tr;
z*{ga{?u^=D5dPWRn8)qu^O}L59|KLxOD9h|Rk$G1CIhG5;$*$;Z;gUym3#*3YD4Oz
zr=ZB%?W8#&4ohCAeJ^dUw@)=2-^4uOvHFF2=h9HUZ5OS4&mxI-!Fg}K`muVevsGy6
zL-lf@+Ux-gyS_3}smxrtQn|3uJU_jFD7P@G_0Hw3XFa05u<pO@%Kpj&kDNXlKT@f*
zm(N#DR!;V<Ht^*R&MYlmUCXGX2A7$rTB%hQ=W#CIzLEKqyDT5!9R<@PM`8N=7nj~P
zbMeBH?#Y=>>qLz?E|OnQ<*m#k<e*Me-#KPCk2jQ?JW;*o441uw6pqJM!-?KP>GN+1
zx7c&iP)6|MSDp@c*^?7^oTh9+<hdHcq8Z(1!kzX~BZrgwQ2VaXb$o7_sV-!#e&ONz
z!os)urzBCSFKeZz-xdyPvc5U<?cs2B!Cihwn44|YYTp?SRu^>fH^M>r#?6`U3Wu=O
zy87;Lj5(hr0<p@>#pTBQ_k<&}i8TJraKOTv?+phklCZxO=6w*o(yUkicDTiP(`hjJ
z?hwLTz-r-X8%DH{^ZUZ0f8G;Vet(GHXoZu9Y55;WTTX(+9}LHR;l5DtheCWuYoq%Z
zA#wrV4~MWs`V#V3%%#`r=smrC*eKtdLf;>v{YY;oYXh;r6K?Ky0!pht+WSC3)sOXd
zWUAqPA)ciHwqbzuY*^@hExqyn?xQ5wc`igC)QH70jpoedYDGnfTpLaC{ND|62-g%a
zNl8omy>P%H@V_7CRP!H%+j}bLiErrx;eb{AU^rko+#gTix2~6(OXfZ*s#T@-hr-t+
zE;Qkl*R1{}VHn6=HnFNzKO7Qgq#*XjW6!UDGku;(tIoxbgt_+l*8E39G;ms#Uh&p&
zhHdH8PlSX_>ASApPu{-}%J-x2$HL)CWocQLKN)VS=zXQNSiv*?sc;ZR;r^$?VP?l|
zV((`{Qa{r+r&-a2=Ff_-<c((9m1e^N(4Pwjo!R*LFef?thhbktT2(~)kHVZZ=r4o=
z)#b&-s~-<XFu*g+b2i$a2nS|oSU~iP0*iTAW(t2vm$Oyj^+k2x?o{hdmpA$4a6tUF
z^2so#-+!zDxYxS<m2il5#ee^#`$D(B8V(v0uKd$(d-mYhJVgI29Go?__~%^A%Jcpg
zTp&J;=1Xd7R?KA?m0t&5LG~}hp;?<3p8Z!mQ750`w^LhK_>FGg8uY)`U^`9g?%(Ko
z)-JDnnhxyhH^b3Nr8&D$K{3t$TUFMLW0&7j1>HiPe+N|>uFHQP4qcvUb()Lk{{tf<
z=wCelAL+ftl7-Lk3w7K7DI8w$0RDEEch&IWTebfj4#B-rU!M)LL}s5u6*K=*akJWR
zKdK^GOz}J6$V$@Ze^K|&S@^GE4kcdyJj%DWICHV}1$VKy^t<k2>FV#<1>E%C;HgfF
zGXGs_)LpW==YP1zi%Y+67uDsZKd=iB{y)PZiZx~ym;O+LZ+Dg&GfP)r_D`3t{*k+)
z<j}_lwgs>wDsE=JveYyI4EH{8`muvm{N{s+KpsSY+w^Yb{B*mPxox8Yw9)ERQnzr{
zau?bC8y5wAYf9#B-?-$OMlH87a2){pTeYHt*D1QDS>&fZbVIPJ<nWC(y25(VksE4M
zB}ebv2myVW02b#Z8IJtTDlIkAC6f6A&I^32h1{EVtZx!oM=g!*S^owZ(}by!c=LuM
z>;uMRn}}k|<oc%Qq0uZ@8ykRVerOu-DW2JS)A}}^IAi4=Y&PeM+x5D?CHW;I#|MI;
zgRQ1`H5@RO&A`nb%-M=)mF<0?f;r4&TEciFxOHTSMwVw-!4z)dQWMp+>VflDW$C6H
z7Y1K@&%wh7?+%{W;-k!2a+6q&_P`za^UDkK2b^%+F}kR^Knp?d*s{26=se)O+4db<
V(uIr%G*R2WW4KLte}K5~{{w2iz+3<T

literal 0
HcmV?d00001

diff --git a/llvm/test/tools/llvm-cov/Inputs/binary-formats.wasm.proftext b/llvm/test/tools/llvm-cov/Inputs/binary-formats.wasm.proftext
new file mode 100644
index 000000000000..20fc3816c225
--- /dev/null
+++ b/llvm/test/tools/llvm-cov/Inputs/binary-formats.wasm.proftext
@@ -0,0 +1,4 @@
+__main_argc_argv
+0x0
+1
+100
diff --git a/llvm/test/tools/llvm-cov/binary-formats.c b/llvm/test/tools/llvm-cov/binary-formats.c
index a5bfc012860e..bb61b288cfc6 100644
--- a/llvm/test/tools/llvm-cov/binary-formats.c
+++ b/llvm/test/tools/llvm-cov/binary-formats.c
@@ -10,4 +10,11 @@ int main(int argc, const char *argv[]) {}
 // RUN: llvm-cov show %S/Inputs/binary-formats.v3.macho64l -instr-profile %t.profdata -path-equivalence=/tmp,%S %s | FileCheck %s
 // RUN: llvm-cov show %S/Inputs/binary-formats.v6.linux64l -instr-profile %t.profdata -path-equivalence=/tmp,%S %s | FileCheck %s
 
+// RUN: llvm-profdata merge %S/Inputs/binary-formats.wasm.proftext -o %t.wasm.profdata
+// NOTE: The wasm binary is built with the following command:
+//   clang -target wasm32-unknown-wasi %s -o %S/Inputs/binary-formats.v6.wasm32 \
+//     -mllvm -enable-name-compression=false \
+//     -fprofile-instr-generate -fcoverage-mapping -lwasi-emulated-getpid -lwasi-emulated-mman
+// RUN: llvm-cov show %S/Inputs/binary-formats.v6.wasm32 -instr-profile %t.wasm.profdata -path-equivalence=/tmp,%S %s | FileCheck %s
+
 // RUN: llvm-cov export %S/Inputs/binary-formats.macho64l -instr-profile %t.profdata | FileCheck %S/Inputs/binary-formats.canonical.json
-- 
GitLab


From ef217a0f6b37a7f5e7ca5e9333df8c2ab8c84c30 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 23 Oct 2024 21:01:51 -0700
Subject: [PATCH 026/693] [VPlan] Introduce and use getVectorPreheader (NFC).

Introduce a dedicated function to retrieve the vector preheader. This
ensures the correct block is used, even if the skeleton is exetended.
---
 llvm/lib/Transforms/Vectorize/VPlan.cpp           | 4 ++--
 llvm/lib/Transforms/Vectorize/VPlan.h             | 5 +++++
 llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 9 ++++-----
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 6ab8fb45c351..44ffcb954a28 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -275,8 +275,8 @@ Value *VPTransformState::get(VPValue *Def, bool NeedsScalar) {
     // Place the code for broadcasting invariant variables in the new preheader.
     IRBuilder<>::InsertPointGuard Guard(Builder);
     if (SafeToHoist) {
-      BasicBlock *LoopVectorPreHeader = CFG.VPBB2IRBB[cast<VPBasicBlock>(
-          Plan->getVectorLoopRegion()->getSinglePredecessor())];
+      BasicBlock *LoopVectorPreHeader =
+          CFG.VPBB2IRBB[Plan->getVectorPreheader()];
       if (LoopVectorPreHeader)
         Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
     }
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 6a192bdf01c4..f2e6729a2e26 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3812,6 +3812,11 @@ public:
     return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
   }
 
+  /// Returns the preheader of the vector loop region.
+  VPBasicBlock *getVectorPreheader() {
+    return cast<VPBasicBlock>(getVectorLoopRegion()->getSinglePredecessor());
+  }
+
   /// Returns the canonical induction recipe of the vector loop.
   VPCanonicalIVPHIRecipe *getCanonicalIV() {
     VPBasicBlock *EntryVPBB = getVectorLoopRegion()->getEntryBasicBlock();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index c6e09c4f2e6e..03c4110761ac 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1073,9 +1073,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
 
 /// Move loop-invariant recipes out of the vector loop region in \p Plan.
 static void licm(VPlan &Plan) {
-  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
-  VPBasicBlock *Preheader =
-      cast<VPBasicBlock>(LoopRegion->getSinglePredecessor());
+  VPBasicBlock *Preheader = Plan.getVectorPreheader();
 
   // Return true if we do not know how to (mechanically) hoist a given recipe
   // out of a loop region. Does not address legality concerns such as aliasing
@@ -1089,6 +1087,7 @@ static void licm(VPlan &Plan) {
   // Hoist any loop invariant recipes from the vector loop region to the
   // preheader. Preform a shallow traversal of the vector loop region, to
   // exclude recipes in replicate regions.
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
            vp_depth_first_shallow(LoopRegion->getEntry()))) {
     for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
@@ -1133,7 +1132,7 @@ void VPlanTransforms::truncateToMinimalBitwidths(
   DenseMap<VPValue *, VPWidenCastRecipe *> ProcessedTruncs;
   Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
   VPTypeAnalysis TypeInfo(CanonicalIVType);
-  VPBasicBlock *PH = Plan.getEntry();
+  VPBasicBlock *PH = Plan.getVectorPreheader();
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
            vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
     for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
@@ -1317,7 +1316,7 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
   // We can't use StartV directly in the ActiveLaneMask VPInstruction, since
   // we have to take unrolling into account. Each part needs to start at
   //   Part * VF
-  auto *VecPreheader = cast<VPBasicBlock>(TopRegion->getSinglePredecessor());
+  auto *VecPreheader = Plan.getVectorPreheader();
   VPBuilder Builder(VecPreheader);
 
   // Create the ActiveLaneMask instruction using the correct start values.
-- 
GitLab


From 04f14e4d652a344fedea6de8df532e88a652bcc4 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Wed, 23 Oct 2024 21:44:44 -0700
Subject: [PATCH 027/693] [clang-format] Remove repeated `TabWidth` in Format.h

---
 clang/include/clang/Format/Format.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index a5fd7750c20b..c9b72e65cb23 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -5308,8 +5308,8 @@ struct FormatStyle {
                R.TableGenBreakingDAGArgOperators &&
            TableGenBreakInsideDAGArg == R.TableGenBreakInsideDAGArg &&
            TabWidth == R.TabWidth && TemplateNames == R.TemplateNames &&
-           TabWidth == R.TabWidth && TypeNames == R.TypeNames &&
-           TypenameMacros == R.TypenameMacros && UseTab == R.UseTab &&
+           TypeNames == R.TypeNames && TypenameMacros == R.TypenameMacros &&
+           UseTab == R.UseTab &&
            VerilogBreakBetweenInstancePorts ==
                R.VerilogBreakBetweenInstancePorts &&
            WhitespaceSensitiveMacros == R.WhitespaceSensitiveMacros;
-- 
GitLab


From b8fddca7bdb354d51e340c60aafe3dff1b35a195 Mon Sep 17 00:00:00 2001
From: Thomas Fransham <tfransham@gmail.com>
Date: Thu, 24 Oct 2024 06:07:13 +0100
Subject: [PATCH 028/693] [llvm] Support llvm::Any across shared libraries on
 windows (#108051)

This is part of the effort to support for enabling plugins on windows by
adding better support for building llvm as a DLL. The export macros used
here were added in #96630

Since shared library symbols aren't deduplicated across multiple
libraries on windows like Linux we have to manually explicitly import
and export `Any::TypeId` template instantiations for the uses of
`llvm::Any` in the LLVM codebase to support LLVM Windows shared library
builds.
This change ensures that external code, including LLVM's own tests, can
use PassManager callbacks when LLVM is built as a DLL.

I also removed the only use of llvm::Any for LoopNest that only existed
in debug code and there also doesn't seem to be any code creating
`Any<LoopNest>`
---
 .../clang/Analysis/FlowSensitive/NoopLattice.h    | 10 ++++++++++
 .../FlowSensitive/TypeErasedDataflowAnalysis.cpp  | 15 +++++++++++++++
 llvm/include/llvm/Analysis/LazyCallGraph.h        |  3 +++
 llvm/include/llvm/IR/PassInstrumentation.h        | 10 +++++++++-
 llvm/lib/Analysis/LazyCallGraph.cpp               |  2 ++
 llvm/lib/IR/PassInstrumentation.cpp               |  4 ++++
 llvm/lib/Transforms/Scalar/LoopPassManager.cpp    |  5 +----
 llvm/unittests/IR/PassBuilderCallbacksTest.cpp    |  2 --
 8 files changed, 44 insertions(+), 7 deletions(-)

diff --git a/clang/include/clang/Analysis/FlowSensitive/NoopLattice.h b/clang/include/clang/Analysis/FlowSensitive/NoopLattice.h
index 019219328111..6cc00680eab7 100644
--- a/clang/include/clang/Analysis/FlowSensitive/NoopLattice.h
+++ b/clang/include/clang/Analysis/FlowSensitive/NoopLattice.h
@@ -14,6 +14,7 @@
 #define LLVM_CLANG_ANALYSIS_FLOWSENSITIVE_NOOP_LATTICE_H
 
 #include "clang/Analysis/FlowSensitive/DataflowLattice.h"
+#include "llvm/ADT/Any.h"
 #include <ostream>
 
 namespace clang {
@@ -38,4 +39,13 @@ inline std::ostream &operator<<(std::ostream &OS, const NoopLattice &) {
 } // namespace dataflow
 } // namespace clang
 
+namespace llvm {
+// This needs to be exported for ClangAnalysisFlowSensitiveTests so any_cast
+// uses the correct address of Any::TypeId from the clang shared library instead
+// of creating one in the test executable. when building with
+// CLANG_LINK_CLANG_DYLIB
+extern template struct CLANG_TEMPLATE_ABI
+    Any::TypeId<clang::dataflow::NoopLattice>;
+}; // namespace llvm
+
 #endif // LLVM_CLANG_ANALYSIS_FLOWSENSITIVE_NOOP_LATTICE_H
diff --git a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
index 8afd18b315d2..32b388643949 100644
--- a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
+++ b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
@@ -30,6 +30,7 @@
 #include "clang/Analysis/FlowSensitive/Transfer.h"
 #include "clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h"
 #include "clang/Analysis/FlowSensitive/Value.h"
+#include "clang/Support/Compiler.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Debug.h"
@@ -37,6 +38,20 @@
 
 #define DEBUG_TYPE "clang-dataflow"
 
+namespace clang {
+namespace dataflow {
+class NoopLattice;
+}
+} // namespace clang
+
+namespace llvm {
+// This needs to be exported for ClangAnalysisFlowSensitiveTests so any_cast
+// uses the correct address of Any::TypeId from the clang shared library instead
+// of creating one in the test executable. when building with
+// CLANG_LINK_CLANG_DYLIB
+template struct CLANG_EXPORT_TEMPLATE Any::TypeId<clang::dataflow::NoopLattice>;
+} // namespace llvm
+
 namespace clang {
 namespace dataflow {
 
diff --git a/llvm/include/llvm/Analysis/LazyCallGraph.h b/llvm/include/llvm/Analysis/LazyCallGraph.h
index e7fd18967d9b..289e9c3990bc 100644
--- a/llvm/include/llvm/Analysis/LazyCallGraph.h
+++ b/llvm/include/llvm/Analysis/LazyCallGraph.h
@@ -34,6 +34,7 @@
 #ifndef LLVM_ANALYSIS_LAZYCALLGRAPH_H
 #define LLVM_ANALYSIS_LAZYCALLGRAPH_H
 
+#include "llvm/ADT/Any.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/PointerIntPair.h"
@@ -1308,6 +1309,8 @@ public:
   static bool isRequired() { return true; }
 };
 
+extern template struct LLVM_TEMPLATE_ABI
+    Any::TypeId<const LazyCallGraph::SCC *>;
 } // end namespace llvm
 
 #endif // LLVM_ANALYSIS_LAZYCALLGRAPH_H
diff --git a/llvm/include/llvm/IR/PassInstrumentation.h b/llvm/include/llvm/IR/PassInstrumentation.h
index 9fcc2d5957a3..45ee372e7959 100644
--- a/llvm/include/llvm/IR/PassInstrumentation.h
+++ b/llvm/include/llvm/IR/PassInstrumentation.h
@@ -50,10 +50,11 @@
 #define LLVM_IR_PASSINSTRUMENTATION_H
 
 #include "llvm/ADT/Any.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/FunctionExtras.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/DenseMap.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include <type_traits>
 #include <vector>
 
@@ -61,6 +62,13 @@ namespace llvm {
 
 class PreservedAnalyses;
 class StringRef;
+class Module;
+class Loop;
+class Function;
+
+extern template struct LLVM_TEMPLATE_ABI Any::TypeId<const Module *>;
+extern template struct LLVM_TEMPLATE_ABI Any::TypeId<const Function *>;
+extern template struct LLVM_TEMPLATE_ABI Any::TypeId<const Loop *>;
 
 /// This class manages callbacks registration, as well as provides a way for
 /// PassInstrumentation to pass control to the registered callbacks.
diff --git a/llvm/lib/Analysis/LazyCallGraph.cpp b/llvm/lib/Analysis/LazyCallGraph.cpp
index e6bf8c9cbb28..5aa36bfc36d4 100644
--- a/llvm/lib/Analysis/LazyCallGraph.cpp
+++ b/llvm/lib/Analysis/LazyCallGraph.cpp
@@ -37,6 +37,8 @@ using namespace llvm;
 
 #define DEBUG_TYPE "lcg"
 
+template struct LLVM_EXPORT_TEMPLATE Any::TypeId<const LazyCallGraph::SCC *>;
+
 void LazyCallGraph::EdgeSequence::insertEdgeInternal(Node &TargetN,
                                                      Edge::Kind EK) {
   EdgeIndexMap.try_emplace(&TargetN, Edges.size());
diff --git a/llvm/lib/IR/PassInstrumentation.cpp b/llvm/lib/IR/PassInstrumentation.cpp
index 0c4e7698d9fa..94ad124a6c77 100644
--- a/llvm/lib/IR/PassInstrumentation.cpp
+++ b/llvm/lib/IR/PassInstrumentation.cpp
@@ -17,6 +17,10 @@
 
 namespace llvm {
 
+template struct LLVM_EXPORT_TEMPLATE Any::TypeId<const Module *>;
+template struct LLVM_EXPORT_TEMPLATE Any::TypeId<const Function *>;
+template struct LLVM_EXPORT_TEMPLATE Any::TypeId<const Loop *>;
+
 void PassInstrumentationCallbacks::addClassToPassName(StringRef ClassName,
                                                       StringRef PassName) {
   ClassToPassName.try_emplace(ClassName, PassName.str());
diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
index 5ef25c21162f..d3bcfb8cc640 100644
--- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
@@ -269,12 +269,9 @@ PreservedAnalyses FunctionToLoopPassAdaptor::run(Function &F,
   PI.pushBeforeNonSkippedPassCallback([&LAR, &LI](StringRef PassID, Any IR) {
     if (isSpecialPass(PassID, {"PassManager"}))
       return;
-    assert(llvm::any_cast<const Loop *>(&IR) ||
-           llvm::any_cast<const LoopNest *>(&IR));
+    assert(llvm::any_cast<const Loop *>(&IR));
     const Loop **LPtr = llvm::any_cast<const Loop *>(&IR);
     const Loop *L = LPtr ? *LPtr : nullptr;
-    if (!L)
-      L = &llvm::any_cast<const LoopNest *>(IR)->getOutermostLoop();
     assert(L && "Loop should be valid for printing");
 
     // Verify the loop structure and LCSSA form before visiting the loop.
diff --git a/llvm/unittests/IR/PassBuilderCallbacksTest.cpp b/llvm/unittests/IR/PassBuilderCallbacksTest.cpp
index 6230aed7b711..9aad6e3ca912 100644
--- a/llvm/unittests/IR/PassBuilderCallbacksTest.cpp
+++ b/llvm/unittests/IR/PassBuilderCallbacksTest.cpp
@@ -298,8 +298,6 @@ template <> std::string getName(const Any &WrappedIR) {
     return (*F)->getName().str();
   if (const auto *const *L = llvm::any_cast<const Loop *>(&WrappedIR))
     return (*L)->getName().str();
-  if (const auto *const *L = llvm::any_cast<const LoopNest *>(&WrappedIR))
-    return (*L)->getName().str();
   if (const auto *const *C =
           llvm::any_cast<const LazyCallGraph::SCC *>(&WrappedIR))
     return (*C)->getName();
-- 
GitLab


From a9050525954cbe11d45b415e2248d9e25e004bfe Mon Sep 17 00:00:00 2001
From: Thomas Fransham <tfransham@gmail.com>
Date: Thu, 24 Oct 2024 06:08:32 +0100
Subject: [PATCH 029/693] [SandboxIR] Add extern templates for
 GlobalWithNodeAPI (#111940)

These symbols will need to be explicitly exported for SandboxIRTests
when LLVM is built as shared library on window with explicitly
visibility macros are enabled.

This is part of the work to enable LLVM_BUILD_LLVM_DYLIB and plugins on
windows.
---
 llvm/include/llvm/SandboxIR/Constant.h | 20 ++++++++++++++++++++
 llvm/lib/SandboxIR/Constant.cpp        | 22 ++++++++++++++++++++++
 2 files changed, 42 insertions(+)

diff --git a/llvm/include/llvm/SandboxIR/Constant.h b/llvm/include/llvm/SandboxIR/Constant.h
index 6eafdefde602..17f55e973cd7 100644
--- a/llvm/include/llvm/SandboxIR/Constant.h
+++ b/llvm/include/llvm/SandboxIR/Constant.h
@@ -22,6 +22,7 @@
 #include "llvm/SandboxIR/Context.h"
 #include "llvm/SandboxIR/Type.h"
 #include "llvm/SandboxIR/User.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm::sandboxir {
 
@@ -797,6 +798,25 @@ public:
   }
 };
 
+// These are needed for SandboxIRTest when building with LLVM_BUILD_LLVM_DYLIB
+extern template LLVM_TEMPLATE_ABI GlobalIFunc &
+GlobalWithNodeAPI<GlobalIFunc, llvm::GlobalIFunc, GlobalObject,
+                  llvm::GlobalObject>::LLVMGVToGV::operator()(llvm::GlobalIFunc
+                                                                  &LLVMGV)
+    const;
+extern template LLVM_TEMPLATE_ABI Function &
+GlobalWithNodeAPI<Function, llvm::Function, GlobalObject, llvm::GlobalObject>::
+    LLVMGVToGV::operator()(llvm::Function &LLVMGV) const;
+
+extern template LLVM_TEMPLATE_ABI GlobalVariable &GlobalWithNodeAPI<
+    GlobalVariable, llvm::GlobalVariable, GlobalObject,
+    llvm::GlobalObject>::LLVMGVToGV::operator()(llvm::GlobalVariable &LLVMGV)
+    const;
+extern template LLVM_TEMPLATE_ABI GlobalAlias &
+GlobalWithNodeAPI<GlobalAlias, llvm::GlobalAlias, GlobalValue,
+                  llvm::GlobalValue>::LLVMGVToGV::operator()(llvm::GlobalAlias
+                                                                 &LLVMGV) const;
+
 class GlobalIFunc final
     : public GlobalWithNodeAPI<GlobalIFunc, llvm::GlobalIFunc, GlobalObject,
                                llvm::GlobalObject> {
diff --git a/llvm/lib/SandboxIR/Constant.cpp b/llvm/lib/SandboxIR/Constant.cpp
index f26c004f6309..3e13c935c428 100644
--- a/llvm/lib/SandboxIR/Constant.cpp
+++ b/llvm/lib/SandboxIR/Constant.cpp
@@ -11,6 +11,7 @@
 #include "llvm/SandboxIR/BasicBlock.h"
 #include "llvm/SandboxIR/Context.h"
 #include "llvm/SandboxIR/Function.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm::sandboxir {
 
@@ -301,6 +302,27 @@ template class GlobalWithNodeAPI<GlobalVariable, llvm::GlobalVariable,
 template class GlobalWithNodeAPI<GlobalAlias, llvm::GlobalAlias, GlobalValue,
                                  llvm::GlobalValue>;
 
+#ifdef _MSC_VER
+// These are needed for SandboxIRTest when building with LLVM_BUILD_LLVM_DYLIB
+template LLVM_EXPORT_TEMPLATE GlobalIFunc &
+GlobalWithNodeAPI<GlobalIFunc, llvm::GlobalIFunc, GlobalObject,
+                  llvm::GlobalObject>::LLVMGVToGV::operator()(llvm::GlobalIFunc
+                                                                  &LLVMGV)
+    const;
+template LLVM_EXPORT_TEMPLATE Function &
+GlobalWithNodeAPI<Function, llvm::Function, GlobalObject, llvm::GlobalObject>::
+    LLVMGVToGV::operator()(llvm::Function &LLVMGV) const;
+
+template LLVM_EXPORT_TEMPLATE GlobalVariable &GlobalWithNodeAPI<
+    GlobalVariable, llvm::GlobalVariable, GlobalObject,
+    llvm::GlobalObject>::LLVMGVToGV::operator()(llvm::GlobalVariable &LLVMGV)
+    const;
+template LLVM_EXPORT_TEMPLATE GlobalAlias &
+GlobalWithNodeAPI<GlobalAlias, llvm::GlobalAlias, GlobalValue,
+                  llvm::GlobalValue>::LLVMGVToGV::operator()(llvm::GlobalAlias
+                                                                 &LLVMGV) const;
+#endif
+
 void GlobalIFunc::setResolver(Constant *Resolver) {
   Ctx.getTracker()
       .emplaceIfTracking<
-- 
GitLab


From 089237c0d0b544b4a823891b9ea650325b9dee6d Mon Sep 17 00:00:00 2001
From: Haopeng Liu <153236845+haopliu@users.noreply.github.com>
Date: Wed, 23 Oct 2024 22:18:59 -0700
Subject: [PATCH 030/693] [DSE] Apply initializes attribute to DSE (#107282)

Apply the initializes attribute to DSE and guard with a flag,
"enable-dse-initializes-attr-improvement".

The attribute support has been landed in:
https://github.com/llvm/llvm-project/pull/84803
The attribute inference will be landed after this PR:
https://github.com/llvm/llvm-project/pull/97373
---
 .../Scalar/DeadStoreElimination.cpp           | 250 ++++++++++++---
 .../DeadStoreElimination/inter-procedural.ll  | 301 ++++++++++++++++++
 2 files changed, 509 insertions(+), 42 deletions(-)
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll

diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 6fce46a624c9..411f7cc34823 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -52,6 +52,7 @@
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
+#include "llvm/IR/ConstantRangeList.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfo.h"
@@ -164,6 +165,11 @@ static cl::opt<bool>
     OptimizeMemorySSA("dse-optimize-memoryssa", cl::init(true), cl::Hidden,
                       cl::desc("Allow DSE to optimize memory accesses."));
 
+// TODO: turn on and remove this flag.
+static cl::opt<bool> EnableInitializesImprovement(
+    "enable-dse-initializes-attr-improvement", cl::init(false), cl::Hidden,
+    cl::desc("Enable the initializes attr improvement in DSE"));
+
 //===----------------------------------------------------------------------===//
 // Helper functions
 //===----------------------------------------------------------------------===//
@@ -809,8 +815,10 @@ bool canSkipDef(MemoryDef *D, bool DefVisibleToCaller) {
 // A memory location wrapper that represents a MemoryLocation, `MemLoc`,
 // defined by `MemDef`.
 struct MemoryLocationWrapper {
-  MemoryLocationWrapper(MemoryLocation MemLoc, MemoryDef *MemDef)
-      : MemLoc(MemLoc), MemDef(MemDef) {
+  MemoryLocationWrapper(MemoryLocation MemLoc, MemoryDef *MemDef,
+                        bool DefByInitializesAttr)
+      : MemLoc(MemLoc), MemDef(MemDef),
+        DefByInitializesAttr(DefByInitializesAttr) {
     assert(MemLoc.Ptr && "MemLoc should be not null");
     UnderlyingObject = getUnderlyingObject(MemLoc.Ptr);
     DefInst = MemDef->getMemoryInst();
@@ -820,20 +828,59 @@ struct MemoryLocationWrapper {
   const Value *UnderlyingObject;
   MemoryDef *MemDef;
   Instruction *DefInst;
+  bool DefByInitializesAttr = false;
 };
 
 // A memory def wrapper that represents a MemoryDef and the MemoryLocation(s)
 // defined by this MemoryDef.
 struct MemoryDefWrapper {
-  MemoryDefWrapper(MemoryDef *MemDef, std::optional<MemoryLocation> MemLoc) {
+  MemoryDefWrapper(MemoryDef *MemDef,
+                   ArrayRef<std::pair<MemoryLocation, bool>> MemLocations) {
     DefInst = MemDef->getMemoryInst();
-    if (MemLoc.has_value())
-      DefinedLocation = MemoryLocationWrapper(*MemLoc, MemDef);
+    for (auto &[MemLoc, DefByInitializesAttr] : MemLocations)
+      DefinedLocations.push_back(
+          MemoryLocationWrapper(MemLoc, MemDef, DefByInitializesAttr));
   }
   Instruction *DefInst;
-  std::optional<MemoryLocationWrapper> DefinedLocation = std::nullopt;
+  SmallVector<MemoryLocationWrapper, 1> DefinedLocations;
+};
+
+bool hasInitializesAttr(Instruction *I) {
+  CallBase *CB = dyn_cast<CallBase>(I);
+  return CB && CB->getArgOperandWithAttribute(Attribute::Initializes);
+}
+
+struct ArgumentInitInfo {
+  unsigned Idx;
+  bool IsDeadOrInvisibleOnUnwind;
+  ConstantRangeList Inits;
 };
 
+// Return the intersected range list of the initializes attributes of "Args".
+// "Args" are call arguments that alias to each other.
+// If any argument in "Args" doesn't have dead_on_unwind attr and
+// "CallHasNoUnwindAttr" is false, return empty.
+ConstantRangeList getIntersectedInitRangeList(ArrayRef<ArgumentInitInfo> Args,
+                                              bool CallHasNoUnwindAttr) {
+  if (Args.empty())
+    return {};
+
+  // To address unwind, the function should have nounwind attribute or the
+  // arguments have dead or invisible on unwind. Otherwise, return empty.
+  for (const auto &Arg : Args) {
+    if (!CallHasNoUnwindAttr && !Arg.IsDeadOrInvisibleOnUnwind)
+      return {};
+    if (Arg.Inits.empty())
+      return {};
+  }
+
+  ConstantRangeList IntersectedIntervals = Args.front().Inits;
+  for (auto &Arg : Args.drop_front())
+    IntersectedIntervals = IntersectedIntervals.intersectWith(Arg.Inits);
+
+  return IntersectedIntervals;
+}
+
 struct DSEState {
   Function &F;
   AliasAnalysis &AA;
@@ -911,7 +958,8 @@ struct DSEState {
 
         auto *MD = dyn_cast_or_null<MemoryDef>(MA);
         if (MD && MemDefs.size() < MemorySSADefsPerBlockLimit &&
-            (getLocForWrite(&I) || isMemTerminatorInst(&I)))
+            (getLocForWrite(&I) || isMemTerminatorInst(&I) ||
+             (EnableInitializesImprovement && hasInitializesAttr(&I))))
           MemDefs.push_back(MD);
       }
     }
@@ -1147,13 +1195,26 @@ struct DSEState {
     return MemoryLocation::getOrNone(I);
   }
 
-  std::optional<MemoryLocation> getLocForInst(Instruction *I) {
+  // Returns a list of <MemoryLocation, bool> pairs written by I.
+  // The bool means whether the write is from Initializes attr.
+  SmallVector<std::pair<MemoryLocation, bool>, 1>
+  getLocForInst(Instruction *I, bool ConsiderInitializesAttr) {
+    SmallVector<std::pair<MemoryLocation, bool>, 1> Locations;
     if (isMemTerminatorInst(I)) {
-      if (auto Loc = getLocForTerminator(I)) {
-        return Loc->first;
+      if (auto Loc = getLocForTerminator(I))
+        Locations.push_back(std::make_pair(Loc->first, false));
+      return Locations;
+    }
+
+    if (auto Loc = getLocForWrite(I))
+      Locations.push_back(std::make_pair(*Loc, false));
+
+    if (ConsiderInitializesAttr) {
+      for (auto &MemLoc : getInitializesArgMemLoc(I)) {
+        Locations.push_back(std::make_pair(MemLoc, true));
       }
     }
-    return getLocForWrite(I);
+    return Locations;
   }
 
   /// Assuming this instruction has a dead analyzable write, can we delete
@@ -1365,7 +1426,8 @@ struct DSEState {
   getDomMemoryDef(MemoryDef *KillingDef, MemoryAccess *StartAccess,
                   const MemoryLocation &KillingLoc, const Value *KillingUndObj,
                   unsigned &ScanLimit, unsigned &WalkerStepLimit,
-                  bool IsMemTerm, unsigned &PartialLimit) {
+                  bool IsMemTerm, unsigned &PartialLimit,
+                  bool IsInitializesAttrMemLoc) {
     if (ScanLimit == 0 || WalkerStepLimit == 0) {
       LLVM_DEBUG(dbgs() << "\n    ...  hit scan limit\n");
       return std::nullopt;
@@ -1602,7 +1664,16 @@ struct DSEState {
 
       // Uses which may read the original MemoryDef mean we cannot eliminate the
       // original MD. Stop walk.
-      if (isReadClobber(MaybeDeadLoc, UseInst)) {
+      // If KillingDef is a CallInst with "initializes" attribute, the reads in
+      // the callee would be dominated by initializations, so it should be safe.
+      bool IsKillingDefFromInitAttr = false;
+      if (IsInitializesAttrMemLoc) {
+        if (KillingI == UseInst &&
+            KillingUndObj == getUnderlyingObject(MaybeDeadLoc.Ptr))
+          IsKillingDefFromInitAttr = true;
+      }
+
+      if (isReadClobber(MaybeDeadLoc, UseInst) && !IsKillingDefFromInitAttr) {
         LLVM_DEBUG(dbgs() << "    ... found read clobber\n");
         return std::nullopt;
       }
@@ -2171,6 +2242,16 @@ struct DSEState {
     return MadeChange;
   }
 
+  // Return the locations written by the initializes attribute.
+  // Note that this function considers:
+  // 1. Unwind edge: use "initializes" attribute only if the callee has
+  //    "nounwind" attribute, or the argument has "dead_on_unwind" attribute,
+  //    or the argument is invisible to caller on unwind. That is, we don't
+  //    perform incorrect DSE on unwind edges in the current function.
+  // 2. Argument alias: for aliasing arguments, the "initializes" attribute is
+  //    the intersected range list of their "initializes" attributes.
+  SmallVector<MemoryLocation, 1> getInitializesArgMemLoc(const Instruction *I);
+
   // Try to eliminate dead defs that access `KillingLocWrapper.MemLoc` and are
   // killed by `KillingLocWrapper.MemDef`. Return whether
   // any changes were made, and whether `KillingLocWrapper.DefInst` was deleted.
@@ -2182,6 +2263,75 @@ struct DSEState {
   bool eliminateDeadDefs(const MemoryDefWrapper &KillingDefWrapper);
 };
 
+SmallVector<MemoryLocation, 1>
+DSEState::getInitializesArgMemLoc(const Instruction *I) {
+  const CallBase *CB = dyn_cast<CallBase>(I);
+  if (!CB)
+    return {};
+
+  // Collect aliasing arguments and their initializes ranges.
+  SmallMapVector<Value *, SmallVector<ArgumentInitInfo, 2>, 2> Arguments;
+  for (unsigned Idx = 0, Count = CB->arg_size(); Idx < Count; ++Idx) {
+    ConstantRangeList Inits;
+    Attribute InitializesAttr = CB->getParamAttr(Idx, Attribute::Initializes);
+    if (InitializesAttr.isValid())
+      Inits = InitializesAttr.getValueAsConstantRangeList();
+
+    Value *CurArg = CB->getArgOperand(Idx);
+    // We don't perform incorrect DSE on unwind edges in the current function,
+    // and use the "initializes" attribute to kill dead stores if:
+    // - The call does not throw exceptions, "CB->doesNotThrow()".
+    // - Or the callee parameter has "dead_on_unwind" attribute.
+    // - Or the argument is invisible to caller on unwind, and there are no
+    //   unwind edges from this call in the current function (e.g. `CallInst`).
+    bool IsDeadOrInvisibleOnUnwind =
+        CB->paramHasAttr(Idx, Attribute::DeadOnUnwind) ||
+        (isa<CallInst>(CB) && isInvisibleToCallerOnUnwind(CurArg));
+    ArgumentInitInfo InitInfo{Idx, IsDeadOrInvisibleOnUnwind, Inits};
+    bool FoundAliasing = false;
+    for (auto &[Arg, AliasList] : Arguments) {
+      auto AAR = BatchAA.alias(MemoryLocation::getBeforeOrAfter(Arg),
+                               MemoryLocation::getBeforeOrAfter(CurArg));
+      if (AAR == AliasResult::NoAlias) {
+        continue;
+      } else if (AAR == AliasResult::MustAlias) {
+        FoundAliasing = true;
+        AliasList.push_back(InitInfo);
+      } else {
+        // For PartialAlias and MayAlias, there is an offset or may be an
+        // unknown offset between the arguments and we insert an empty init
+        // range to discard the entire initializes info while intersecting.
+        FoundAliasing = true;
+        AliasList.push_back(ArgumentInitInfo{Idx, IsDeadOrInvisibleOnUnwind,
+                                             ConstantRangeList()});
+      }
+    }
+    if (!FoundAliasing)
+      Arguments[CurArg] = {InitInfo};
+  }
+
+  SmallVector<MemoryLocation, 1> Locations;
+  for (const auto &[_, Args] : Arguments) {
+    auto IntersectedRanges =
+        getIntersectedInitRangeList(Args, CB->doesNotThrow());
+    if (IntersectedRanges.empty())
+      continue;
+
+    for (const auto &Arg : Args) {
+      for (const auto &Range : IntersectedRanges) {
+        int64_t Start = Range.getLower().getSExtValue();
+        int64_t End = Range.getUpper().getSExtValue();
+        // For now, we only handle locations starting at offset 0.
+        if (Start == 0)
+          Locations.push_back(MemoryLocation(CB->getArgOperand(Arg.Idx),
+                                             LocationSize::precise(End - Start),
+                                             CB->getAAMetadata()));
+      }
+    }
+  }
+  return Locations;
+}
+
 std::pair<bool, bool>
 DSEState::eliminateDeadDefs(const MemoryLocationWrapper &KillingLocWrapper) {
   bool Changed = false;
@@ -2208,7 +2358,8 @@ DSEState::eliminateDeadDefs(const MemoryLocationWrapper &KillingLocWrapper) {
     std::optional<MemoryAccess *> MaybeDeadAccess = getDomMemoryDef(
         KillingLocWrapper.MemDef, Current, KillingLocWrapper.MemLoc,
         KillingLocWrapper.UnderlyingObject, ScanLimit, WalkerStepLimit,
-        isMemTerminatorInst(KillingLocWrapper.DefInst), PartialLimit);
+        isMemTerminatorInst(KillingLocWrapper.DefInst), PartialLimit,
+        KillingLocWrapper.DefByInitializesAttr);
 
     if (!MaybeDeadAccess) {
       LLVM_DEBUG(dbgs() << "  finished walk\n");
@@ -2231,10 +2382,20 @@ DSEState::eliminateDeadDefs(const MemoryLocationWrapper &KillingLocWrapper) {
       }
       continue;
     }
+    // We cannot apply the initializes attribute to DeadAccess/DeadDef.
+    // It would incorrectly consider a call instruction as redundant store
+    // and remove this call instruction.
+    // TODO: this conflates the existence of a MemoryLocation with being able
+    // to delete the instruction. Fix isRemovable() to consider calls with
+    // side effects that cannot be removed, e.g. calls with the initializes
+    // attribute, and remove getLocForInst(ConsiderInitializesAttr = false).
     MemoryDefWrapper DeadDefWrapper(
         cast<MemoryDef>(DeadAccess),
-        getLocForInst(cast<MemoryDef>(DeadAccess)->getMemoryInst()));
-    MemoryLocationWrapper &DeadLocWrapper = *DeadDefWrapper.DefinedLocation;
+        getLocForInst(cast<MemoryDef>(DeadAccess)->getMemoryInst(),
+                      /*ConsiderInitializesAttr=*/false));
+    assert(DeadDefWrapper.DefinedLocations.size() == 1);
+    MemoryLocationWrapper &DeadLocWrapper =
+        DeadDefWrapper.DefinedLocations.front();
     LLVM_DEBUG(dbgs() << " (" << *DeadLocWrapper.DefInst << ")\n");
     ToCheck.insert(DeadLocWrapper.MemDef->getDefiningAccess());
     NumGetDomMemoryDefPassed++;
@@ -2309,37 +2470,41 @@ DSEState::eliminateDeadDefs(const MemoryLocationWrapper &KillingLocWrapper) {
 }
 
 bool DSEState::eliminateDeadDefs(const MemoryDefWrapper &KillingDefWrapper) {
-  if (!KillingDefWrapper.DefinedLocation.has_value()) {
+  if (KillingDefWrapper.DefinedLocations.empty()) {
     LLVM_DEBUG(dbgs() << "Failed to find analyzable write location for "
                       << *KillingDefWrapper.DefInst << "\n");
     return false;
   }
 
-  auto &KillingLocWrapper = *KillingDefWrapper.DefinedLocation;
-  LLVM_DEBUG(dbgs() << "Trying to eliminate MemoryDefs killed by "
-                    << *KillingLocWrapper.MemDef << " ("
-                    << *KillingLocWrapper.DefInst << ")\n");
-  auto [Changed, DeletedKillingLoc] = eliminateDeadDefs(KillingLocWrapper);
-
-  // Check if the store is a no-op.
-  if (!DeletedKillingLoc && storeIsNoop(KillingLocWrapper.MemDef,
-                                        KillingLocWrapper.UnderlyingObject)) {
-    LLVM_DEBUG(dbgs() << "DSE: Remove No-Op Store:\n  DEAD: "
-                      << *KillingLocWrapper.DefInst << '\n');
-    deleteDeadInstruction(KillingLocWrapper.DefInst);
-    NumRedundantStores++;
-    return true;
-  }
-  // Can we form a calloc from a memset/malloc pair?
-  if (!DeletedKillingLoc &&
-      tryFoldIntoCalloc(KillingLocWrapper.MemDef,
-                        KillingLocWrapper.UnderlyingObject)) {
-    LLVM_DEBUG(dbgs() << "DSE: Remove memset after forming calloc:\n"
-                      << "  DEAD: " << *KillingLocWrapper.DefInst << '\n');
-    deleteDeadInstruction(KillingLocWrapper.DefInst);
-    return true;
+  bool MadeChange = false;
+  for (auto &KillingLocWrapper : KillingDefWrapper.DefinedLocations) {
+    LLVM_DEBUG(dbgs() << "Trying to eliminate MemoryDefs killed by "
+                      << *KillingLocWrapper.MemDef << " ("
+                      << *KillingLocWrapper.DefInst << ")\n");
+    auto [Changed, DeletedKillingLoc] = eliminateDeadDefs(KillingLocWrapper);
+
+    // Check if the store is a no-op.
+    if (!DeletedKillingLoc && storeIsNoop(KillingLocWrapper.MemDef,
+                                          KillingLocWrapper.UnderlyingObject)) {
+      LLVM_DEBUG(dbgs() << "DSE: Remove No-Op Store:\n  DEAD: "
+                        << *KillingLocWrapper.DefInst << '\n');
+      deleteDeadInstruction(KillingLocWrapper.DefInst);
+      NumRedundantStores++;
+      MadeChange = true;
+      continue;
+    }
+    // Can we form a calloc from a memset/malloc pair?
+    if (!DeletedKillingLoc &&
+        tryFoldIntoCalloc(KillingLocWrapper.MemDef,
+                          KillingLocWrapper.UnderlyingObject)) {
+      LLVM_DEBUG(dbgs() << "DSE: Remove memset after forming calloc:\n"
+                        << "  DEAD: " << *KillingLocWrapper.DefInst << '\n');
+      deleteDeadInstruction(KillingLocWrapper.DefInst);
+      MadeChange = true;
+      continue;
+    }
   }
-  return Changed;
+  return MadeChange;
 }
 
 static bool eliminateDeadStores(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
@@ -2355,7 +2520,8 @@ static bool eliminateDeadStores(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
       continue;
 
     MemoryDefWrapper KillingDefWrapper(
-        KillingDef, State.getLocForInst(KillingDef->getMemoryInst()));
+        KillingDef, State.getLocForInst(KillingDef->getMemoryInst(),
+                                        EnableInitializesImprovement));
     MadeChange |= State.eliminateDeadDefs(KillingDefWrapper);
   }
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll b/llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll
new file mode 100644
index 000000000000..d93da9b6612b
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll
@@ -0,0 +1,301 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=dse -enable-dse-initializes-attr-improvement -S | FileCheck %s
+
+declare void @p1_write_only(ptr nocapture noundef writeonly initializes((0, 2)) dead_on_unwind)
+declare void @p1_write_then_read(ptr nocapture noundef initializes((0, 2)) dead_on_unwind)
+declare void @p1_clobber(ptr nocapture noundef)
+declare void @p2_same_range(ptr nocapture noundef initializes((0, 2)) dead_on_unwind, ptr nocapture noundef initializes((0, 2)) dead_on_unwind)
+declare void @p2_no_init(ptr nocapture noundef initializes((0, 2)) dead_on_unwind, ptr nocapture noundef dead_on_unwind)
+declare void @p2_no_dead_on_unwind(ptr nocapture noundef initializes((0, 2)) dead_on_unwind, ptr nocapture noundef initializes((0, 2)))
+declare void @p2_no_dead_on_unwind_but_nounwind(ptr nocapture noundef initializes((0, 2)) dead_on_unwind, ptr nocapture noundef initializes((0, 2))) nounwind
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p1_write_only_caller() {
+; CHECK-LABEL: @p1_write_only_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    call void @p1_write_only(ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  call void @p1_write_only(ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p1_write_then_read_caller() {
+; CHECK-LABEL: @p1_write_then_read_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    call void @p1_write_then_read(ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  call void @p1_write_then_read(ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p1_write_then_read_caller_with_clobber() {
+; CHECK-LABEL: @p1_write_then_read_caller_with_clobber(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    store i16 0, ptr [[PTR]], align 2
+; CHECK-NEXT:    call void @p1_clobber(ptr [[PTR]])
+; CHECK-NEXT:    call void @p1_write_then_read(ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  call void @p1_clobber(ptr %ptr)
+  call void @p1_write_then_read(ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+declare void @p1_write_then_read_raw(ptr nocapture noundef initializes((0, 2)))
+define i16 @p1_initializes_invoke() personality ptr undef {
+; CHECK-LABEL: @p1_initializes_invoke(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    store i16 0, ptr [[PTR]], align 2
+; CHECK-NEXT:    invoke void @p1_write_then_read_raw(ptr [[PTR]])
+; CHECK-NEXT:            to label [[BB1:%.*]] unwind label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    ret i16 0
+; CHECK:       bb2:
+; CHECK-NEXT:    [[TMP:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT:            cleanup
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+entry:
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  invoke void @p1_write_then_read_raw(ptr %ptr) to label %bb1 unwind label %bb2
+bb1:
+  ret i16 0
+bb2:
+  %tmp = landingpad { ptr, i32 }
+  cleanup
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_same_range_noalias_caller() {
+; CHECK-LABEL: @p2_same_range_noalias_caller(
+; CHECK-NEXT:    [[PTR1:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    [[PTR2:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    call void @p2_same_range(ptr [[PTR1]], ptr [[PTR2]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR1]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr1 = alloca i16
+  %ptr2 = alloca i16
+  store i16 0, ptr %ptr1
+  store i16 0, ptr %ptr2
+  call void @p2_same_range(ptr %ptr1, ptr %ptr2)
+  %l = load i16, ptr %ptr1
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_same_range_must_alias_caller() {
+; CHECK-LABEL: @p2_same_range_must_alias_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    call void @p2_same_range(ptr [[PTR]], ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  call void @p2_same_range(ptr %ptr, ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_same_range_may_or_partial_alias_caller1(ptr %base, i1 %x) {
+; CHECK-LABEL: @p2_same_range_may_or_partial_alias_caller1(
+; CHECK-NEXT:    [[BASEPLUS:%.*]] = getelementptr i8, ptr [[BASE:%.*]], i64 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[X:%.*]], ptr [[BASEPLUS]], ptr [[BASE]]
+; CHECK-NEXT:    store i32 0, ptr [[BASE]], align 4
+; CHECK-NEXT:    call void @p2_same_range(ptr [[BASE]], ptr [[SEL]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[BASE]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %baseplus = getelementptr i8, ptr %base, i64 1
+  %sel = select i1 %x, ptr %baseplus, ptr %base
+  store i32 0, ptr %base
+  call void @p2_same_range(ptr %base, ptr %sel)
+  %l = load i16, ptr %base
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_same_range_may_or_partial_alias_caller2(ptr %base1, ptr %base2) {
+; CHECK-LABEL: @p2_same_range_may_or_partial_alias_caller2(
+; CHECK-NEXT:    store i32 0, ptr [[BASE1:%.*]], align 4
+; CHECK-NEXT:    call void @p2_same_range(ptr [[BASE1]], ptr [[BASE2:%.*]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[BASE1]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  store i32 0, ptr %base1
+  call void @p2_same_range(ptr %base1, ptr %base2)
+  %l = load i16, ptr %base1
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_no_init_alias_caller() {
+; CHECK-LABEL: @p2_no_init_alias_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    store i16 0, ptr [[PTR]], align 2
+; CHECK-NEXT:    call void @p2_no_init(ptr [[PTR]], ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  call void @p2_no_init(ptr %ptr, ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Althrough the 2nd parameter of `p2_no_dead_on_unwind` doesn't have
+; the 'dead_on_unwind' attribute, it's invisble to caller on unwind.
+; DSE still uses the 'initializes' attribute and kill the dead store.
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_no_dead_on_unwind_but_invisible_to_caller_alias_caller() {
+; CHECK-LABEL: @p2_no_dead_on_unwind_but_invisible_to_caller_alias_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    call void @p2_no_dead_on_unwind(ptr [[PTR]], ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  call void @p2_no_dead_on_unwind(ptr %ptr, ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_no_dead_on_unwind_alias_caller(ptr %ptr) {
+; CHECK-LABEL: @p2_no_dead_on_unwind_alias_caller(
+; CHECK-NEXT:    store i16 0, ptr [[PTR:%.*]], align 2
+; CHECK-NEXT:    call void @p2_no_dead_on_unwind(ptr [[PTR]], ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  store i16 0, ptr %ptr
+  call void @p2_no_dead_on_unwind(ptr %ptr, ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_no_dead_on_unwind_but_nounwind_alias_caller() {
+; CHECK-LABEL: @p2_no_dead_on_unwind_but_nounwind_alias_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    call void @p2_no_dead_on_unwind_but_nounwind(ptr [[PTR]], ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  call void @p2_no_dead_on_unwind_but_nounwind(ptr %ptr, ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i1) nounwind
+declare void @large_p1(ptr nocapture noundef initializes((0, 200))) nounwind
+declare void @large_p2(ptr nocapture noundef initializes((0, 200)), ptr nocapture noundef initializes((0, 100))) nounwind
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @large_p1_caller() {
+; CHECK-LABEL: @large_p1_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca [200 x i8], align 1
+; CHECK-NEXT:    call void @large_p1(ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca [200 x i8]
+  call void @llvm.memset.p0.i64(ptr %ptr, i8 42, i64 100, i1 false)
+  call void @large_p1(ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @large_p2_nonalias_caller() {
+; CHECK-LABEL: @large_p2_nonalias_caller(
+; CHECK-NEXT:    [[PTR1:%.*]] = alloca [200 x i8], align 1
+; CHECK-NEXT:    [[PTR2:%.*]] = alloca [100 x i8], align 1
+; CHECK-NEXT:    call void @large_p2(ptr [[PTR1]], ptr [[PTR2]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR1]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr1 = alloca [200 x i8]
+  %ptr2 = alloca [100 x i8]
+  call void @llvm.memset.p0.i64(ptr %ptr1, i8 42, i64 200, i1 false)
+  call void @llvm.memset.p0.i64(ptr %ptr2, i8 42, i64 100, i1 false)
+  call void @large_p2(ptr %ptr1, ptr %ptr2)
+  %l = load i16, ptr %ptr1
+  ret i16 %l
+}
+
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @large_p2_must_alias_caller() {
+; CHECK-LABEL: @large_p2_must_alias_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca [300 x i8], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 100
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP1]], i8 42, i64 200, i1 false)
+; CHECK-NEXT:    call void @large_p2(ptr [[PTR]], ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca [300 x i8]
+  call void @llvm.memset.p0.i64(ptr %ptr, i8 42, i64 300, i1 false)
+  call void @large_p2(ptr %ptr, ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @large_p2_may_or_partial_alias_caller1(ptr %base) {
+; CHECK-LABEL: @large_p2_may_or_partial_alias_caller1(
+; CHECK-NEXT:    [[BASEPLUS:%.*]] = getelementptr i8, ptr [[BASE:%.*]], i64 100
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[BASE]], i8 42, i64 300, i1 false)
+; CHECK-NEXT:    call void @large_p2(ptr [[BASE]], ptr [[BASEPLUS]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[BASE]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %baseplus = getelementptr i8, ptr %base, i64 100
+  call void @llvm.memset.p0.i64(ptr %base, i8 42, i64 300, i1 false)
+  call void @large_p2(ptr %base, ptr %baseplus)
+  %l = load i16, ptr %base
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @large_p2_may_or_partial_alias_caller2(ptr %base1, ptr %base2) {
+; CHECK-LABEL: @large_p2_may_or_partial_alias_caller2(
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[BASE1:%.*]], i8 42, i64 300, i1 false)
+; CHECK-NEXT:    call void @large_p2(ptr [[BASE1]], ptr [[BASE2:%.*]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[BASE1]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  call void @llvm.memset.p0.i64(ptr %base1, i8 42, i64 300, i1 false)
+  call void @large_p2(ptr %base1, ptr %base2)
+  %l = load i16, ptr %base1
+  ret i16 %l
+}
+
-- 
GitLab


From d1fae5996e66c2a9f0b1c5b9776f86962ab8e9ea Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Thu, 24 Oct 2024 14:01:25 +0800
Subject: [PATCH 031/693] [clang-tidy] Fix build error (NFC)

/llvm-project/clang/include/clang/Analysis/FlowSensitive/NoopLattice.h:49:2:
error: extra ';' outside of a function is incompatible with C++98 [-Werror,-Wc++98-compat-extra-semi]
}; // namespace llvm
---
 clang/include/clang/Analysis/FlowSensitive/NoopLattice.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/include/clang/Analysis/FlowSensitive/NoopLattice.h b/clang/include/clang/Analysis/FlowSensitive/NoopLattice.h
index 6cc00680eab7..0938091cd689 100644
--- a/clang/include/clang/Analysis/FlowSensitive/NoopLattice.h
+++ b/clang/include/clang/Analysis/FlowSensitive/NoopLattice.h
@@ -46,6 +46,6 @@ namespace llvm {
 // CLANG_LINK_CLANG_DYLIB
 extern template struct CLANG_TEMPLATE_ABI
     Any::TypeId<clang::dataflow::NoopLattice>;
-}; // namespace llvm
+} // namespace llvm
 
 #endif // LLVM_CLANG_ANALYSIS_FLOWSENSITIVE_NOOP_LATTICE_H
-- 
GitLab


From 60944177b8fa43d0d315af6b4827f94226e0aa01 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl@google.com>
Date: Wed, 23 Oct 2024 23:47:27 -0700
Subject: [PATCH 032/693] [WPD][ThinLTO]Add cutoff option for WPD (#113383)

This option applies for _import_ WPD (i.e., when `DevirtModule` pass
de-virtualizes according to an imported summary, in ThinLTO backend
pipeline). It's meant for debugging (e.g., bisection).
---
 llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp  | 17 +++++++++++++++++
 .../Transforms/WholeProgramDevirt/import.ll     |  6 ++++++
 2 files changed, 23 insertions(+)

diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
index 45d32218f362..789d01540c29 100644
--- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -168,6 +168,14 @@ static cl::list<std::string>
                       cl::desc("Prevent function(s) from being devirtualized"),
                       cl::Hidden, cl::CommaSeparated);
 
+/// If explicitly specified, the devirt module pass will stop transformation
+/// once the total number of devirtualizations reach the cutoff value. Setting
+/// this option to 0 explicitly will do 0 devirtualization.
+static cl::opt<unsigned> WholeProgramDevirtCutoff(
+    "wholeprogramdevirt-cutoff",
+    cl::desc("Max number of devirtualizations for devirt module pass"),
+    cl::init(0));
+
 /// Mechanism to add runtime checking of devirtualization decisions, optionally
 /// trapping or falling back to indirect call on any that are not correct.
 /// Trapping mode is useful for debugging undefined behavior leading to failures
@@ -316,6 +324,9 @@ VirtualCallTarget::VirtualCallTarget(GlobalValue *Fn, const TypeMemberInfo *TM)
 
 namespace {
 
+// Tracks the number of devirted calls in the IR transformation.
+static unsigned NumDevirtCalls = 0;
+
 // A slot in a set of virtual tables. The TypeID identifies the set of virtual
 // tables, and the ByteOffset is the offset in bytes from the address point to
 // the virtual function pointer.
@@ -1169,10 +1180,16 @@ void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo,
       if (!OptimizedCalls.insert(&VCallSite.CB).second)
         continue;
 
+      // Stop when the number of devirted calls reaches the cutoff.
+      if (WholeProgramDevirtCutoff.getNumOccurrences() > 0 &&
+          NumDevirtCalls >= WholeProgramDevirtCutoff)
+        return;
+
       if (RemarksEnabled)
         VCallSite.emitRemark("single-impl",
                              TheFn->stripPointerCasts()->getName(), OREGetter);
       NumSingleImpl++;
+      NumDevirtCalls++;
       auto &CB = VCallSite.CB;
       assert(!CB.getCalledFunction() && "devirtualizing direct call?");
       IRBuilder<> Builder(&CB);
diff --git a/llvm/test/Transforms/WholeProgramDevirt/import.ll b/llvm/test/Transforms/WholeProgramDevirt/import.ll
index 4e4df5f35a77..de25bc10a7c1 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/import.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/import.ll
@@ -8,6 +8,12 @@
 ; RUN: opt -S -passes=wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-vcp-branch-funnel.yaml < %s | FileCheck --check-prefixes=CHECK,VCP,VCP-X86,VCP64,BRANCH-FUNNEL %s
 ; RUN: opt -S -passes=wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-branch-funnel.yaml < %s | FileCheck --check-prefixes=CHECK,BRANCH-FUNNEL,BRANCH-FUNNEL-NOVCP %s
 
+; Cutoff value is not explicitly set. Expect 3 remark messages.
+; RUN: opt -S -passes=wholeprogramdevirt -wholeprogramdevirt-summary-action=import -pass-remarks=wholeprogramdevirt -wholeprogramdevirt-read-summary=%S/Inputs/import-single-impl.yaml < %s  2>&1 | grep "single-impl" | count 3
+; Cutoff value is set to 1. Expect one remark messages.
+; RUN: opt -S -passes=wholeprogramdevirt -wholeprogramdevirt-summary-action=import -pass-remarks=wholeprogramdevirt -wholeprogramdevirt-cutoff=1  -wholeprogramdevirt-read-summary=%S/Inputs/import-single-impl.yaml < %s  2>&1 | grep "single-impl" | count 1
+; Cutoff value is explicitly set to zero. Expect no remark message.
+; RUN: opt -S -passes=wholeprogramdevirt -wholeprogramdevirt-summary-action=import -pass-remarks=wholeprogramdevirt -wholeprogramdevirt-cutoff=0  -wholeprogramdevirt-read-summary=%S/Inputs/import-single-impl.yaml < %s 2>&1  | FileCheck -implicit-check-not="remark" %s
 target datalayout = "e-p:64:64"
 target triple = "x86_64-unknown-linux-gnu"
 
-- 
GitLab


From 0b7e8c25ece1ff0e0dc0b7d283600b49498e8162 Mon Sep 17 00:00:00 2001
From: Dmitry Polukhin <34227995+dmpolukhin@users.noreply.github.com>
Date: Thu, 24 Oct 2024 08:20:43 +0100
Subject: [PATCH 033/693] [C++20][Modules] Quote header unit name in
 preprocessor output (-E) (#112883)

Summary:
Before this change clang produced output with header unit names that may
conaint path separators, dots and other non-identifier characters. This
diff prints header unit name in quotes and -E output can be compiled
again. Also remove unnecessary space between header unit name and semi.

Test Plan: check-clang
---
 .../lib/Frontend/PrintPreprocessedOutput.cpp  | 10 ++++++----
 clang/lib/Lex/TokenConcatenation.cpp          |  4 ++++
 .../header_unit_preprocessed_output.cpp       | 20 +++++++++++++++++++
 .../Modules/cxx20-include-translation.cpp     |  4 ++--
 4 files changed, 32 insertions(+), 6 deletions(-)
 create mode 100644 clang/test/Headers/header_unit_preprocessed_output.cpp

diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
index 383d43560849..1005825441b3 100644
--- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -952,13 +952,15 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
       continue;
     } else if (Tok.is(tok::annot_header_unit)) {
       // This is a header-name that has been (effectively) converted into a
-      // module-name.
+      // module-name, print them inside quote.
       // FIXME: The module name could contain non-identifier module name
-      // components. We don't have a good way to round-trip those.
+      // components and OS specific file paths components. We don't have a good
+      // way to round-trip those.
       Module *M = reinterpret_cast<Module *>(Tok.getAnnotationValue());
       std::string Name = M->getFullModuleName();
-      Callbacks->OS->write(Name.data(), Name.size());
-      Callbacks->HandleNewlinesInToken(Name.data(), Name.size());
+      *Callbacks->OS << '"';
+      Callbacks->OS->write_escaped(Name);
+      *Callbacks->OS << '"';
     } else if (Tok.is(tok::annot_embed)) {
       // Manually explode the binary data out to a stream of comma-delimited
       // integer values. If the user passed -dE, that is handled by the
diff --git a/clang/lib/Lex/TokenConcatenation.cpp b/clang/lib/Lex/TokenConcatenation.cpp
index 865879d18053..05f4203bd722 100644
--- a/clang/lib/Lex/TokenConcatenation.cpp
+++ b/clang/lib/Lex/TokenConcatenation.cpp
@@ -160,6 +160,10 @@ static char GetFirstChar(const Preprocessor &PP, const Token &Tok) {
 bool TokenConcatenation::AvoidConcat(const Token &PrevPrevTok,
                                      const Token &PrevTok,
                                      const Token &Tok) const {
+  // No space is required between header unit name in quote and semi.
+  if (PrevTok.is(tok::annot_header_unit) && Tok.is(tok::semi))
+    return false;
+
   // Conservatively assume that every annotation token that has a printable
   // form requires whitespace.
   if (PrevTok.isAnnotation())
diff --git a/clang/test/Headers/header_unit_preprocessed_output.cpp b/clang/test/Headers/header_unit_preprocessed_output.cpp
new file mode 100644
index 000000000000..12b4ab6159f3
--- /dev/null
+++ b/clang/test/Headers/header_unit_preprocessed_output.cpp
@@ -0,0 +1,20 @@
+// RUN: rm -fR %t
+// RUN: split-file %s %t
+// RUN: cd %t
+// RUN: %clang_cc1 -verify -std=c++20 -emit-header-unit -xc++-user-header bz0.h
+// RUN: %clang_cc1 -verify -std=c++20 -fmodule-file=bz0.pcm -xc++-user-header bz1.h -E -o bz1.output.h
+// RUN: FileCheck %s < bz1.output.h
+// RUN: %clang_cc1 -std=c++20 -fmodule-file=bz0.pcm -emit-header-unit -xc++-user-header bz1.output.h
+
+//--- bz0.h
+// expected-no-diagnostics
+#pragma once
+
+void foo();
+
+//--- bz1.h
+// expected-no-diagnostics
+import "bz0.h";
+
+// CHECK: # 1 ".{{/|\\\\?}}bz1.h"
+// CHECK: import ".{{/|\\\\?}}bz0.h";
diff --git a/clang/test/Modules/cxx20-include-translation.cpp b/clang/test/Modules/cxx20-include-translation.cpp
index 7bf318432582..a607e7c0373f 100644
--- a/clang/test/Modules/cxx20-include-translation.cpp
+++ b/clang/test/Modules/cxx20-include-translation.cpp
@@ -105,6 +105,6 @@ export void charlie() {
 }
 
 // CHECK: #pragma clang module import ".{{/|\\\\?}}h1.h"
-// CHECK: import .{{/|\\\\?}}h2.h
-// CHECK: import .{{/|\\\\?}}h3.h
+// CHECK: import ".{{/|\\\\?}}h2.h"
+// CHECK: import ".{{/|\\\\?}}h3.h"
 // CHECK-NOT: #pragma clang module import ".{{/|\\\\?}}h4.h"
-- 
GitLab


From 509af087cccca3bf8a90bc9871335224226dc6fe Mon Sep 17 00:00:00 2001
From: Nuno Lopes <nuno.lopes@tecnico.ulisboa.pt>
Date: Thu, 24 Oct 2024 09:01:25 +0100
Subject: [PATCH 034/693] replace 2 placeholder uses of undef with poison [NFC]

---
 llvm/lib/CodeGen/CodeGenPrepare.cpp | 2 +-
 llvm/lib/CodeGen/SjLjEHPrepare.cpp  | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 86f28293ba9f..67a359015114 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -3340,7 +3340,7 @@ class TypePromotionTransaction {
         // Set a dummy one.
         // We could use OperandSetter here, but that would imply an overhead
         // that we are not willing to pay.
-        Inst->setOperand(It, UndefValue::get(Val->getType()));
+        Inst->setOperand(It, PoisonValue::get(Val->getType()));
       }
     }
 
diff --git a/llvm/lib/CodeGen/SjLjEHPrepare.cpp b/llvm/lib/CodeGen/SjLjEHPrepare.cpp
index c4ad9f0b2172..c10c74011955 100644
--- a/llvm/lib/CodeGen/SjLjEHPrepare.cpp
+++ b/llvm/lib/CodeGen/SjLjEHPrepare.cpp
@@ -267,11 +267,11 @@ void SjLjEHPrepareImpl::lowerIncomingArguments(Function &F) {
 
     Type *Ty = AI.getType();
 
-    // Use 'select i8 true, %arg, undef' to simulate a 'no-op' instruction.
+    // Use 'select i8 true, %arg, poison' to simulate a 'no-op' instruction.
     Value *TrueValue = ConstantInt::getTrue(F.getContext());
-    Value *UndefValue = UndefValue::get(Ty);
+    Value *PoisonValue = PoisonValue::get(Ty);
     Instruction *SI = SelectInst::Create(
-        TrueValue, &AI, UndefValue, AI.getName() + ".tmp", AfterAllocaInsPt);
+        TrueValue, &AI, PoisonValue, AI.getName() + ".tmp", AfterAllocaInsPt);
     AI.replaceAllUsesWith(SI);
 
     // Reset the operand, because it  was clobbered by the RAUW above.
-- 
GitLab


From 17bfd21391d080afbf2697a0a1a631a1be76a2e4 Mon Sep 17 00:00:00 2001
From: SpencerAbson <Spencer.Abson@arm.com>
Date: Thu, 24 Oct 2024 09:05:58 +0100
Subject: [PATCH 035/693] [AArch64] Add assembly/disassembly for multi-vector
 AES instructions (#113307)

This patch adds assembly/disassembly for the following multi-vector SVE
instructions

      - AESE (two/four registers)
      - AESD (two/four registers)
      - AESDIMC (two/four registers)
      - AESEMC (two/four registers)

- Introduce assembler extension tests for the new Armv9.6 sve-aes2 and
ssve-aes features
- In accordance with:
https://developer.arm.com/documentation/ddi0602/latest/
---
 llvm/lib/Target/AArch64/AArch64.td            |  2 +-
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 13 +++
 llvm/lib/Target/AArch64/SVEInstrFormats.td    | 48 +++++++++++
 .../AArch64/SME2p1/directive-arch-negative.s  |  6 ++
 llvm/test/MC/AArch64/SME2p1/directive-arch.s  |  4 +
 .../directive-arch_extension-negative.s       |  9 +-
 .../AArch64/SME2p1/directive-arch_extension.s |  7 +-
 .../test/MC/AArch64/SVE2p1/aesd-diagnostics.s | 83 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p1/aesd.s            | 53 ++++++++++++
 .../MC/AArch64/SVE2p1/aesdimc-diagnostics.s   | 83 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p1/aesdimc.s         | 54 ++++++++++++
 .../test/MC/AArch64/SVE2p1/aese-diagnostics.s | 83 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p1/aese.s            | 53 ++++++++++++
 .../MC/AArch64/SVE2p1/aesemc-diagnostics.s    | 83 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p1/aesemc.s          | 53 ++++++++++++
 .../AArch64/SVE2p1/directive-arch-negative.s  |  6 ++
 llvm/test/MC/AArch64/SVE2p1/directive-arch.s  |  6 +-
 .../directive-arch_extension-negative.s       |  8 +-
 .../AArch64/SVE2p1/directive-arch_extension.s |  6 +-
 19 files changed, 654 insertions(+), 6 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/SVE2p1/aesd-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p1/aesd.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p1/aesdimc-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p1/aesdimc.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p1/aese-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p1/aese.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p1/aesemc-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p1/aesemc.s

diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index cdfbd3db6473..2262ad1dfd0c 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -73,7 +73,7 @@ def SVEUnsupported : AArch64Unsupported {
                       SVE2Unsupported.F);
 }
 
-let F = [HasSME2p1, HasSVE2p1_or_HasSME2p1] in
+let F = [HasSME2p1, HasSVE2p1_or_HasSME2p1, HasSVE2p1orSSVE_AES] in
 def SME2p1Unsupported : AArch64Unsupported;
 
 def SME2Unsupported : AArch64Unsupported {
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index c650d1b8a7fd..a097c138c5d6 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -3917,6 +3917,19 @@ let Predicates = [HasSVE2BitPerm] in {
   defm BGRP_ZZZ : sve2_misc_bitwise<0b1110, "bgrp", int_aarch64_sve_bgrp_x>;
 } // End HasSVE2BitPerm
 
+let Predicates = [HasSVEAES2, HasSVE2p1orSSVE_AES] in {
+  // SVE_AES2 multi-vector instructions (x2)
+  def AESE_2ZZI_B    : sve_crypto_binary_multi2<0b000, "aese">;
+  def AESD_2ZZI_B    : sve_crypto_binary_multi2<0b010, "aesd">;
+  def AESEMC_2ZZI_B  : sve_crypto_binary_multi2<0b100, "aesemc">;
+  def AESDMIC_2ZZI_B : sve_crypto_binary_multi2<0b110, "aesdimc">;
+  // SVE_AES2 multi-vector instructions (x4)
+  def AESE_4ZZI_B    : sve_crypto_binary_multi4<0b0000, "aese">;
+  def AESD_4ZZI_B    : sve_crypto_binary_multi4<0b0100, "aesd">;
+  def AESEMC_4ZZI_B  : sve_crypto_binary_multi4<0b1000, "aesemc">;
+  def AESDMIC_4ZZI_B : sve_crypto_binary_multi4<0b1100, "aesdimc">;
+} // End HasSVEAES2, HasSVE2p1orSSVE_AES
+
 //===----------------------------------------------------------------------===//
 // SME or SVE2.1 instructions
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 7acfff47cf65..42e00b534008 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -8734,6 +8734,54 @@ multiclass sve2_crypto_unary_op<bit opc, string asm, SDPatternOperator op> {
   def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME)>;
 }
 
+class sve_crypto_binary_multi2<bits<3> opc, string asm>
+: I<(outs ZZ_b_mul_r:$Zdn),
+    (ins ZZ_b_mul_r:$_Zdn, ZPR128:$Zm, VectorIndexS32b_timm:$imm2),
+  asm,
+  "\t$Zdn, $_Zdn, $Zm$imm2",
+  "",
+  []>, Sched<[]> {
+  bits<5> Zm;
+  bits<4> Zdn;
+  bits<2> imm2;
+  let Inst{31-21} = 0b01000101001;
+  let Inst{20-19} = imm2;
+  let Inst{18-17} = 0b01;
+  let Inst{16}    = opc{2};
+  let Inst{15-11} = 0b11101;
+  let Inst{10}    = opc{1};
+  let Inst{9-5}   = Zm;
+  let Inst{4-1}   = Zdn;
+  let Inst{0}     = opc{0};
+
+  let Constraints = "$Zdn = $_Zdn";
+  let hasSideEffects = 0;
+}
+
+class sve_crypto_binary_multi4<bits<4> opc, string asm>
+: I<(outs ZZZZ_b_mul_r:$Zdn),
+    (ins ZZZZ_b_mul_r:$_Zdn, ZPR128:$Zm, VectorIndexS32b_timm:$imm2),
+  asm,
+  "\t$Zdn, $_Zdn, $Zm$imm2",
+  "",
+  []>, Sched<[]> {
+  bits<5> Zm;
+  bits<3> Zdn;
+  bits<2> imm2;
+  let Inst{31-21} = 0b01000101001;
+  let Inst{20-19} = imm2;
+  let Inst{18-17} = 0b11;
+  let Inst{16}    = opc{3};
+  let Inst{15-11} = 0b11101;
+  let Inst{10}    = opc{2};
+  let Inst{9-5}   = Zm;
+  let Inst{4-2}   = Zdn;
+  let Inst{1-0}   = opc{1-0};
+
+  let Constraints = "$Zdn = $_Zdn";
+  let hasSideEffects = 0;
+}
+
 //===----------------------------------------------------------------------===//
 // SVE BFloat16 Group
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AArch64/SME2p1/directive-arch-negative.s b/llvm/test/MC/AArch64/SME2p1/directive-arch-negative.s
index 87ac6e3f34ae..06bc5ec23397 100644
--- a/llvm/test/MC/AArch64/SME2p1/directive-arch-negative.s
+++ b/llvm/test/MC/AArch64/SME2p1/directive-arch-negative.s
@@ -17,3 +17,9 @@ bfclamp { z0.h, z1.h }, z0.h, z0.h
 bfadd za.h[w8, 3], {z20.h-z21.h}
 // CHECK: error: instruction requires: sme-b16b16
 // CHECK: bfadd za.h[w8, 3], {z20.h-z21.h} 
+
+.arch armv9-a+sve-aes2+ssve-aes
+.arch armv9-a+nossve-aes
+aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
+// CHECK: error: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK: aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SME2p1/directive-arch.s b/llvm/test/MC/AArch64/SME2p1/directive-arch.s
index 085a38cef007..51d259af30e8 100644
--- a/llvm/test/MC/AArch64/SME2p1/directive-arch.s
+++ b/llvm/test/MC/AArch64/SME2p1/directive-arch.s
@@ -8,3 +8,7 @@ sqcvt z0.h, {z0.s, z1.s}
 .arch armv9-a+sme2+sve-b16b16
 bfclamp { z0.h, z1.h }, z0.h, z0.h
 // CHECK: bfclamp { z0.h, z1.h }, z0.h, z0.h
+
+.arch armv9-a+sve-aes2+ssve-aes
+aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
+// CHECK: aesdimc { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SME2p1/directive-arch_extension-negative.s b/llvm/test/MC/AArch64/SME2p1/directive-arch_extension-negative.s
index 84705d69ca36..6e185a7ff8cc 100644
--- a/llvm/test/MC/AArch64/SME2p1/directive-arch_extension-negative.s
+++ b/llvm/test/MC/AArch64/SME2p1/directive-arch_extension-negative.s
@@ -17,4 +17,11 @@ bfclamp { z0.h, z1.h }, z0.h, z0.h
 .arch_extension nosme-b16b16
 bfadd za.h[w8, 3], {z20.h-z21.h}
 // CHECK: error: instruction requires: sme-b16b16
-// CHECK: bfadd za.h[w8, 3], {z20.h-z21.h}
\ No newline at end of file
+// CHECK: bfadd za.h[w8, 3], {z20.h-z21.h}
+
+.arch_extension sve-aes2
+.arch_extension ssve-aes
+.arch_extension nossve-aes
+aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
+// CHECK: error: instruction requires: sve2p1 or ssve-aes
+// CHECK: aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SME2p1/directive-arch_extension.s b/llvm/test/MC/AArch64/SME2p1/directive-arch_extension.s
index 297e8b776975..3700325e6a5f 100644
--- a/llvm/test/MC/AArch64/SME2p1/directive-arch_extension.s
+++ b/llvm/test/MC/AArch64/SME2p1/directive-arch_extension.s
@@ -11,4 +11,9 @@ bfclamp { z0.h, z1.h }, z0.h, z0.h
 
 .arch_extension sme-b16b16
 bfadd za.h[w8, 3], {z20.h-z21.h}
-// CHECK: bfadd za.h[w8, 3, vgx2], { z20.h, z21.h }
\ No newline at end of file
+// CHECK: bfadd za.h[w8, 3, vgx2], { z20.h, z21.h }
+
+.arch_extension sve-aes2
+.arch_extension ssve-aes
+aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
+// CHECK: aesdimc { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p1/aesd-diagnostics.s b/llvm/test/MC/AArch64/SVE2p1/aesd-diagnostics.s
new file mode 100644
index 000000000000..6c7daadb371f
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p1/aesd-diagnostics.s
@@ -0,0 +1,83 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 2>&1 < %s| FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid vector list
+
+aesd {z0.b-z2.b}, {z0.b-z2.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aesd {z0.b-z2.b}, {z0.b-z2.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesd {z0.d-z1.d}, {z0.d-z1.d}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aesd {z0.d-z1.d}, {z0.d-z1.d}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesd {z0.s-z3.s}, {z0.s-z3.s}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aesd {z0.s-z3.s}, {z0.s-z3.s}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesd {z0.b-z0.b}, {z0.b-z0.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+// CHECK-NEXT: aesd {z0.b-z0.b}, {z0.b-z0.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesd {z3.b-z7.b}, {z3.b-z7.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+// CHECK-NEXT: aesd {z3.b-z7.b}, {z3.b-z7.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesd {z3.b-z4.b}, {z3.b-z4.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+// CHECK-NEXT: aesd {z3.b-z4.b}, {z3.b-z4.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesd {z5.b-z8.b}, {z5.b-z8.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 4 consecutive SVE vectors, where the first vector is a multiple of 4 and with matching element types
+// CHECK-NEXT: aesd {z5.b-z8.b}, {z5.b-z8.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid second source vector width
+
+aesd {z0.b-z1.b}, {z0.b-z1.b}, z0.d[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: aesd {z0.b-z1.b}, {z0.b-z1.b}, z0.d[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesd {z0.b-z3.b}, {z0.b-z3.b}, z0.s[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: aesd {z0.b-z3.b}, {z0.b-z3.b}, z0.s[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid immediate index
+
+aesd {z0.b-z1.b}, {z0.b-z1.b}, z0.q[4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: aesd {z0.b-z1.b}, {z0.b-z1.b}, z0.q[4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesd {z0.b-z3.b}, {z0.b-z3.b}, z0.q[-1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: aesd {z0.b-z3.b}, {z0.b-z3.b}, z0.q[-1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Source and Destination Registers must match
+
+aesd {z0.b-z1.b}, {z2.b-z3.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register list
+// CHECK-NEXT: aesd {z0.b-z1.b}, {z2.b-z3.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesd {z0.b-z3.b}, {z4.b-z7.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register list
+// CHECK-NEXT: aesd {z0.b-z3.b}, {z4.b-z7.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesd {z0.b-z3.b}, {z0.h-z3.h}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aesd {z0.b-z3.b}, {z0.h-z3.h}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p1/aesd.s b/llvm/test/MC/AArch64/SVE2p1/aesd.s
new file mode 100644
index 000000000000..e2731d221a84
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p1/aesd.s
@@ -0,0 +1,53 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+ssve-aes < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve-aes2,+sve2p1 --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve-aes2,+sve2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// x2
+aesd    {z0.b-z1.b}, {z0.b-z1.b}, z0.q[0]  // 01000101-00100010-11101100-00000000
+// CHECK-INST: aesd    { z0.b, z1.b }, { z0.b, z1.b }, z0.q[0]
+// CHECK-ENCODING: [0x00,0xec,0x22,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4522ec00 <unknown>
+
+aesd    {z20.b-z21.b}, {z20.b-z21.b}, z10.q[2]  // 01000101-00110010-11101101-01010100
+// CHECK-INST: aesd    { z20.b, z21.b }, { z20.b, z21.b }, z10.q[2]
+// CHECK-ENCODING: [0x54,0xed,0x32,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4532ed54 <unknown>
+
+aesd    {z30.b-z31.b}, {z30.b-z31.b}, z31.q[3]  // 01000101-00111010-11101111-11111110
+// CHECK-INST: aesd    { z30.b, z31.b }, { z30.b, z31.b }, z31.q[3]
+// CHECK-ENCODING: [0xfe,0xef,0x3a,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453aeffe <unknown>
+
+// x4
+aesd    {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]  // 01000101-00100110-11101100-00000000
+// CHECK-INST: aesd    { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
+// CHECK-ENCODING: [0x00,0xec,0x26,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4526ec00 <unknown>
+
+aesd    {z20.b-z23.b}, {z20.b-z23.b}, z13.q[1]  // 01000101-00101110-11101101-10110100
+// CHECK-INST: aesd    { z20.b - z23.b }, { z20.b - z23.b }, z13.q[1]
+// CHECK-ENCODING: [0xb4,0xed,0x2e,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 452eedb4 <unknown>
+
+aesd    {z28.b-z31.b}, {z28.b-z31.b}, z31.q[3]  // 01000101-00111110-11101111-11111100
+// CHECK-INST: aesd    { z28.b - z31.b }, { z28.b - z31.b }, z31.q[3]
+// CHECK-ENCODING: [0xfc,0xef,0x3e,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453eeffc <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p1/aesdimc-diagnostics.s b/llvm/test/MC/AArch64/SVE2p1/aesdimc-diagnostics.s
new file mode 100644
index 000000000000..b4b311e7c969
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p1/aesdimc-diagnostics.s
@@ -0,0 +1,83 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 2>&1 < %s| FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid vector list
+
+aesdimc {z0.b-z2.b}, {z0.b-z2.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aesdimc {z0.b-z2.b}, {z0.b-z2.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesdimc {z0.d-z1.d}, {z0.d-z1.d}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aesdimc {z0.d-z1.d}, {z0.d-z1.d}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesdimc {z0.s-z3.s}, {z0.s-z3.s}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aesdimc {z0.s-z3.s}, {z0.s-z3.s}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesdimc {z0.b-z0.b}, {z0.b-z0.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+// CHECK-NEXT: aesdimc {z0.b-z0.b}, {z0.b-z0.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesdimc {z3.b-z7.b}, {z3.b-z7.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+// CHECK-NEXT: aesdimc {z3.b-z7.b}, {z3.b-z7.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesdimc {z3.b-z4.b}, {z3.b-z4.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+// CHECK-NEXT: aesdimc {z3.b-z4.b}, {z3.b-z4.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesdimc {z5.b-z8.b}, {z5.b-z8.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 4 consecutive SVE vectors, where the first vector is a multiple of 4 and with matching element types
+// CHECK-NEXT: aesdimc {z5.b-z8.b}, {z5.b-z8.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid second source vector width
+
+aesdimc {z0.b-z1.b}, {z0.b-z1.b}, z0.d[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: aesdimc {z0.b-z1.b}, {z0.b-z1.b}, z0.d[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.s[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.s[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid immediate index
+
+aesdimc {z0.b-z1.b}, {z0.b-z1.b}, z0.q[4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: aesdimc {z0.b-z1.b}, {z0.b-z1.b}, z0.q[4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[-1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[-1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Source and Destination Registers must match
+
+aesdimc {z0.b-z1.b}, {z2.b-z3.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register list
+// CHECK-NEXT: aesdimc {z0.b-z1.b}, {z2.b-z3.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesdimc {z0.b-z3.b}, {z4.b-z7.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register list
+// CHECK-NEXT: aesdimc {z0.b-z3.b}, {z4.b-z7.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesdimc {z0.b-z3.b}, {z0.h-z3.h}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aesdimc {z0.b-z3.b}, {z0.h-z3.h}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p1/aesdimc.s b/llvm/test/MC/AArch64/SVE2p1/aesdimc.s
new file mode 100644
index 000000000000..6aa0e0d3d2ba
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p1/aesdimc.s
@@ -0,0 +1,54 @@
+
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+ssve-aes < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve-aes2,+sve2p1 --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve-aes2,+sve2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// x2
+aesdimc {z0.b-z1.b}, {z0.b-z1.b}, z0.q[0]  // 01000101-00100011-11101100-00000000
+// CHECK-INST: aesdimc { z0.b, z1.b }, { z0.b, z1.b }, z0.q[0]
+// CHECK-ENCODING: [0x00,0xec,0x23,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4523ec00 <unknown>
+
+aesdimc {z20.b-z21.b}, {z20.b-z21.b}, z10.q[2]  // 01000101-00110011-11101101-01010100
+// CHECK-INST: aesdimc { z20.b, z21.b }, { z20.b, z21.b }, z10.q[2]
+// CHECK-ENCODING: [0x54,0xed,0x33,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4533ed54 <unknown>
+
+aesdimc {z30.b-z31.b}, {z30.b-z31.b}, z31.q[3]  // 01000101-00111011-11101111-11111110
+// CHECK-INST: aesdimc { z30.b, z31.b }, { z30.b, z31.b }, z31.q[3]
+// CHECK-ENCODING: [0xfe,0xef,0x3b,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453beffe <unknown>
+
+// x4
+aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]  // 01000101-00100111-11101100-00000000
+// CHECK-INST: aesdimc { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
+// CHECK-ENCODING: [0x00,0xec,0x27,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4527ec00 <unknown>
+
+aesdimc {z20.b-z23.b}, {z20.b-z23.b}, z13.q[1]  // 01000101-00101111-11101101-10110100
+// CHECK-INST: aesdimc { z20.b - z23.b }, { z20.b - z23.b }, z13.q[1]
+// CHECK-ENCODING: [0xb4,0xed,0x2f,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 452fedb4 <unknown>
+
+aesdimc {z28.b-z31.b}, {z28.b-z31.b}, z31.q[3]  // 01000101-00111111-11101111-11111100
+// CHECK-INST: aesdimc { z28.b - z31.b }, { z28.b - z31.b }, z31.q[3]
+// CHECK-ENCODING: [0xfc,0xef,0x3f,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453feffc <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p1/aese-diagnostics.s b/llvm/test/MC/AArch64/SVE2p1/aese-diagnostics.s
new file mode 100644
index 000000000000..101920d9434b
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p1/aese-diagnostics.s
@@ -0,0 +1,83 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 2>&1 < %s| FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid vector list
+
+aese {z0.b-z2.b}, {z0.b-z2.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aese {z0.b-z2.b}, {z0.b-z2.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aese {z0.d-z1.d}, {z0.d-z1.d}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aese {z0.d-z1.d}, {z0.d-z1.d}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aese {z0.s-z3.s}, {z0.s-z3.s}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aese {z0.s-z3.s}, {z0.s-z3.s}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aese {z0.b-z0.b}, {z0.b-z0.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+// CHECK-NEXT: aese {z0.b-z0.b}, {z0.b-z0.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aese {z3.b-z7.b}, {z3.b-z7.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+// CHECK-NEXT: aese {z3.b-z7.b}, {z3.b-z7.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aese {z3.b-z4.b}, {z3.b-z4.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+// CHECK-NEXT: aese {z3.b-z4.b}, {z3.b-z4.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aese {z5.b-z8.b}, {z5.b-z8.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 4 consecutive SVE vectors, where the first vector is a multiple of 4 and with matching element types
+// CHECK-NEXT: aese {z5.b-z8.b}, {z5.b-z8.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid second source vector width
+
+aese {z0.b-z1.b}, {z0.b-z1.b}, z0.d[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: aese {z0.b-z1.b}, {z0.b-z1.b}, z0.d[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aese {z0.b-z3.b}, {z0.b-z3.b}, z0.s[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: aese {z0.b-z3.b}, {z0.b-z3.b}, z0.s[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid immediate index
+
+aese {z0.b-z1.b}, {z0.b-z1.b}, z0.q[4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: aese {z0.b-z1.b}, {z0.b-z1.b}, z0.q[4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aese {z0.b-z3.b}, {z0.b-z3.b}, z0.q[-1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: aese {z0.b-z3.b}, {z0.b-z3.b}, z0.q[-1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Source and Destination Registers must match
+
+aese {z0.b-z1.b}, {z2.b-z3.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register list
+// CHECK-NEXT: aese {z0.b-z1.b}, {z2.b-z3.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aese {z0.b-z3.b}, {z4.b-z7.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register list
+// CHECK-NEXT: aese {z0.b-z3.b}, {z4.b-z7.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aese {z0.b-z3.b}, {z0.h-z3.h}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aese {z0.b-z3.b}, {z0.h-z3.h}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p1/aese.s b/llvm/test/MC/AArch64/SVE2p1/aese.s
new file mode 100644
index 000000000000..a5272e62e245
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p1/aese.s
@@ -0,0 +1,53 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+ssve-aes < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve-aes2,+sve2p1 --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve-aes2,+sve2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// x2
+aese    {z0.b-z1.b}, {z0.b-z1.b}, z0.q[0]  // 01000101-00100010-11101000-00000000
+// CHECK-INST: aese    { z0.b, z1.b }, { z0.b, z1.b }, z0.q[0]
+// CHECK-ENCODING: [0x00,0xe8,0x22,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4522e800 <unknown>
+
+aese    {z20.b-z21.b}, {z20.b-z21.b}, z10.q[2]  // 01000101-00110010-11101001-01010100
+// CHECK-INST: aese    { z20.b, z21.b }, { z20.b, z21.b }, z10.q[2]
+// CHECK-ENCODING: [0x54,0xe9,0x32,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4532e954 <unknown>
+
+aese    {z30.b-z31.b}, {z30.b-z31.b}, z31.q[3]  // 01000101-00111010-11101011-11111110
+// CHECK-INST: aese    { z30.b, z31.b }, { z30.b, z31.b }, z31.q[3]
+// CHECK-ENCODING: [0xfe,0xeb,0x3a,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453aebfe <unknown>
+
+// x4
+aese    {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]  // 01000101-00100110-11101000-00000000
+// CHECK-INST: aese    { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
+// CHECK-ENCODING: [0x00,0xe8,0x26,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4526e800 <unknown>
+
+aese    {z20.b-z23.b}, {z20.b-z23.b}, z13.q[1]  // 01000101-00101110-11101001-10110100
+// CHECK-INST: aese    { z20.b - z23.b }, { z20.b - z23.b }, z13.q[1]
+// CHECK-ENCODING: [0xb4,0xe9,0x2e,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 452ee9b4 <unknown>
+
+aese    {z28.b-z31.b}, {z28.b-z31.b}, z31.q[3]  // 01000101-00111110-11101011-11111100
+// CHECK-INST: aese    { z28.b - z31.b }, { z28.b - z31.b }, z31.q[3]
+// CHECK-ENCODING: [0xfc,0xeb,0x3e,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453eebfc <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p1/aesemc-diagnostics.s b/llvm/test/MC/AArch64/SVE2p1/aesemc-diagnostics.s
new file mode 100644
index 000000000000..815b542c3448
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p1/aesemc-diagnostics.s
@@ -0,0 +1,83 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 2>&1 < %s| FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid vector list
+
+aesemc {z0.b-z2.b}, {z0.b-z2.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aesemc {z0.b-z2.b}, {z0.b-z2.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesemc {z0.d-z1.d}, {z0.d-z1.d}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aesemc {z0.d-z1.d}, {z0.d-z1.d}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesemc {z0.s-z3.s}, {z0.s-z3.s}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aesemc {z0.s-z3.s}, {z0.s-z3.s}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesemc {z0.b-z0.b}, {z0.b-z0.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+// CHECK-NEXT: aesemc {z0.b-z0.b}, {z0.b-z0.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesemc {z3.b-z7.b}, {z3.b-z7.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+// CHECK-NEXT: aesemc {z3.b-z7.b}, {z3.b-z7.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesemc {z3.b-z4.b}, {z3.b-z4.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+// CHECK-NEXT: aesemc {z3.b-z4.b}, {z3.b-z4.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesemc {z5.b-z8.b}, {z5.b-z8.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 4 consecutive SVE vectors, where the first vector is a multiple of 4 and with matching element types
+// CHECK-NEXT: aesemc {z5.b-z8.b}, {z5.b-z8.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid second source vector width
+
+aesemc {z0.b-z1.b}, {z0.b-z1.b}, z0.d[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: aesemc {z0.b-z1.b}, {z0.b-z1.b}, z0.d[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesemc {z0.b-z3.b}, {z0.b-z3.b}, z0.s[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: aesemc {z0.b-z3.b}, {z0.b-z3.b}, z0.s[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid immediate index
+
+aesemc {z0.b-z1.b}, {z0.b-z1.b}, z0.q[4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: aesemc {z0.b-z1.b}, {z0.b-z1.b}, z0.q[4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesemc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[-1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: aesemc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[-1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Source and Destination Registers must match
+
+aesemc {z0.b-z1.b}, {z2.b-z3.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register list
+// CHECK-NEXT: aesemc {z0.b-z1.b}, {z2.b-z3.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesemc {z0.b-z3.b}, {z4.b-z7.b}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register list
+// CHECK-NEXT: aesemc {z0.b-z3.b}, {z4.b-z7.b}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+aesemc {z0.b-z3.b}, {z0.h-z3.h}, z0.q[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: aesemc {z0.b-z3.b}, {z0.h-z3.h}, z0.q[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p1/aesemc.s b/llvm/test/MC/AArch64/SVE2p1/aesemc.s
new file mode 100644
index 000000000000..888c82811562
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p1/aesemc.s
@@ -0,0 +1,53 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+ssve-aes < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve-aes2,+sve2p1 --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve-aes2,+sve2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// x2
+aesemc  {z0.b-z1.b}, {z0.b-z1.b}, z0.q[0]  // 01000101-00100011-11101000-00000000
+// CHECK-INST: aesemc  { z0.b, z1.b }, { z0.b, z1.b }, z0.q[0]
+// CHECK-ENCODING: [0x00,0xe8,0x23,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4523e800 <unknown>
+
+aesemc  {z22.b-z23.b}, {z22.b-z23.b}, z13.q[1]  // 01000101-00101011-11101001-10110110
+// CHECK-INST: aesemc  { z22.b, z23.b }, { z22.b, z23.b }, z13.q[1]
+// CHECK-ENCODING: [0xb6,0xe9,0x2b,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 452be9b6 <unknown>
+
+aesemc  {z30.b-z31.b}, {z30.b-z31.b}, z31.q[3]  // 01000101-00111011-11101011-11111110
+// CHECK-INST: aesemc  { z30.b, z31.b }, { z30.b, z31.b }, z31.q[3]
+// CHECK-ENCODING: [0xfe,0xeb,0x3b,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453bebfe <unknown>
+
+// x4
+aesemc  {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]  // 01000101-00100111-11101000-00000000
+// CHECK-INST: aesemc  { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
+// CHECK-ENCODING: [0x00,0xe8,0x27,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4527e800 <unknown>
+
+aesemc  {z20.b-z23.b}, {z20.b-z23.b}, z10.q[2]  // 01000101-00110111-11101001-01010100
+// CHECK-INST: aesemc  { z20.b - z23.b }, { z20.b - z23.b }, z10.q[2]
+// CHECK-ENCODING: [0x54,0xe9,0x37,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4537e954 <unknown>
+
+aesemc  {z28.b-z31.b}, {z28.b-z31.b}, z31.q[3]  // 01000101-00111111-11101011-11111100
+// CHECK-INST: aesemc  { z28.b - z31.b }, { z28.b - z31.b }, z31.q[3]
+// CHECK-ENCODING: [0xfc,0xeb,0x3f,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453febfc <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p1/directive-arch-negative.s b/llvm/test/MC/AArch64/SVE2p1/directive-arch-negative.s
index 8e8ec9238049..e82c92df451d 100644
--- a/llvm/test/MC/AArch64/SVE2p1/directive-arch-negative.s
+++ b/llvm/test/MC/AArch64/SVE2p1/directive-arch-negative.s
@@ -11,3 +11,9 @@ sclamp z0.s, z1.s, z2.s
 bfadd   z23.h, p3/m, z23.h, z13.h
 // CHECK: error: instruction requires: sve-b16b16
 // CHECK: bfadd   z23.h, p3/m, z23.h, z13.h
+
+.arch armv9-a+sve2p1+sve-aes2
+.arch armv9-a+nosve-aes2
+aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
+// CHECK: error: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK: aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p1/directive-arch.s b/llvm/test/MC/AArch64/SVE2p1/directive-arch.s
index b8f25e59a5df..bf035d2accdb 100644
--- a/llvm/test/MC/AArch64/SVE2p1/directive-arch.s
+++ b/llvm/test/MC/AArch64/SVE2p1/directive-arch.s
@@ -6,4 +6,8 @@ sclamp z0.s, z1.s, z2.s
 
 .arch armv9-a+sve2p1+sve-b16b16
 bfadd   z23.h, p3/m, z23.h, z13.h
-// CHECK: bfadd   z23.h, p3/m, z23.h, z13.h
\ No newline at end of file
+// CHECK: bfadd   z23.h, p3/m, z23.h, z13.h
+
+.arch armv9-a+sve2p1+sve-aes2
+aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
+// CHECK: aesdimc { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p1/directive-arch_extension-negative.s b/llvm/test/MC/AArch64/SVE2p1/directive-arch_extension-negative.s
index 97c2c1fa3f06..4ab6373ee8a2 100644
--- a/llvm/test/MC/AArch64/SVE2p1/directive-arch_extension-negative.s
+++ b/llvm/test/MC/AArch64/SVE2p1/directive-arch_extension-negative.s
@@ -11,4 +11,10 @@ sclamp z0.s, z1.s, z2.s
 .arch_extension nosve-b16b16
 bfadd   z23.h, p3/m, z23.h, z13.h
 // CHECK: error: instruction requires: sve-b16b16
-// CHECK: bfadd   z23.h, p3/m, z23.h, z13.h
\ No newline at end of file
+// CHECK: bfadd   z23.h, p3/m, z23.h, z13.h
+
+.arch_extension sve-aes2
+.arch_extension nosve-aes2
+aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
+// CHECK: error: instruction requires: sve-aes2
+// CHECK: {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p1/directive-arch_extension.s b/llvm/test/MC/AArch64/SVE2p1/directive-arch_extension.s
index 867cf8829d31..d33e0812a0e0 100644
--- a/llvm/test/MC/AArch64/SVE2p1/directive-arch_extension.s
+++ b/llvm/test/MC/AArch64/SVE2p1/directive-arch_extension.s
@@ -7,4 +7,8 @@ sclamp z0.s, z1.s, z2.s
 .arch_extension sve2p1
 .arch_extension sve-b16b16
 bfadd   z23.h, p3/m, z23.h, z13.h
-// CHECK: bfadd   z23.h, p3/m, z23.h, z13.h
\ No newline at end of file
+// CHECK: bfadd   z23.h, p3/m, z23.h, z13.h
+
+.arch_extension sve-aes2
+aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
+// CHECK: aesdimc { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
\ No newline at end of file
-- 
GitLab


From 61a456bd5ae88eeccc39b28a30be4eb03289446d Mon Sep 17 00:00:00 2001
From: "Oleksandr T." <oleksandr.tarasiuk@outlook.com>
Date: Thu, 24 Oct 2024 11:34:58 +0300
Subject: [PATCH 036/693] [Clang] prevent assertion failure in value-dependent
 initializer expressions (#112612)

Fixes #112140

---

```
CXXConstructExpr 0x14209e580 'const S':'const struct S' contains-errors 'void (const int &)' list
`-CXXDefaultArgExpr 0x14209e500 'const int' contains-errors
  `-RecoveryExpr 0x14209daf0 'const int' contains-errors
```

This change resolves an issue with evaluating `ArrayFiller` initializers
in _dependent_ contexts, especially when they involve a `RecoveryExpr`.
In certain cases, `ArrayFiller` initializers containing a `RecoveryExpr`
from earlier errors are incorrectly passed to `EvaluateInPlace`, causing
evaluation failures when they are value-dependent.

When this is the case, the initializer is processed through
`EvaluateDependentExpr`, which prevents unnecessary evaluation attempts
and ensures proper handling of value-dependent initializers in
`ArrayFillers`.
---
 clang/docs/ReleaseNotes.rst                      |  1 +
 clang/lib/AST/ExprConstant.cpp                   |  3 +++
 clang/test/SemaCXX/constant-expression-cxx11.cpp | 10 ++++++++++
 3 files changed, 14 insertions(+)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index ce046a305c89..46021c9c17fe 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -544,6 +544,7 @@ Bug Fixes to C++ Support
 - Clang incorrectly considered a class with an anonymous union member to not be
   const-default-constructible even if a union member has a default member initializer.
   (#GH95854).
+- Fixed an assertion failure when evaluating an invalid expression in an array initializer (#GH112140)
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 8e36cad2d2c6..d664c503655b 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -11579,6 +11579,9 @@ bool ArrayExprEvaluator::VisitCXXParenListOrInitListExpr(
   LValue Subobject = This;
   Subobject.addArray(Info, ExprToVisit, CAT);
   auto Eval = [&](const Expr *Init, unsigned ArrayIndex) {
+    if (Init->isValueDependent())
+      return EvaluateDependentExpr(Init, Info);
+
     if (!EvaluateInPlace(Result.getArrayInitializedElt(ArrayIndex), Info,
                          Subobject, Init) ||
         !HandleLValueArrayAdjustment(Info, Init, Subobject,
diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp
index e2ea984b37cd..0c349333d89d 100644
--- a/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -2564,3 +2564,13 @@ GH50055::E2 GlobalInitNotCE2 = GH50055::testDefaultArgForParam(); // ok, not a c
 constexpr GH50055::E2 GlobalInitCE = (GH50055::E2)-1;
 // expected-error@-1 {{constexpr variable 'GlobalInitCE' must be initialized by a constant expression}}
 // expected-note@-2 {{integer value -1 is outside the valid range of values [0, 7] for the enumeration type 'E2'}}
+
+namespace GH112140 {
+struct S {
+  constexpr S(const int &a = ) { } // expected-error {{expected expression}}
+};
+
+void foo() {
+  constexpr S s[2] = { }; // expected-error {{constexpr variable 's' must be initialized by a constant expression}}
+}
+}
-- 
GitLab


From e37d736def5b95a2710f92881b5fc8b0494d8a05 Mon Sep 17 00:00:00 2001
From: Nashe Mncube <nashe.mncube@arm.com>
Date: Thu, 24 Oct 2024 10:12:01 +0100
Subject: [PATCH 037/693] Recommit: [llvm][ARM][GlobalOpt]Add widen global
 arrays pass  (#113289)

This is a recommit of #107120 . The original PR was approved but failed
buildbot. The newly added tests should only be run for compilers that
support the ARM target. This has been resolved by adding a config file
for these tests.

- Pass optimizes memcpy's by padding out destinations and sources to a
  full word to make ARM backend generate full word loads instead of
  loading a single byte (ldrb) and/or half word (ldrh). Only pads
  destination when it's a stack allocated constant size array and source
  when it's constant string. Heuristic to decide whether to pad or not
  is very basic and could be improved to allow more examples to be
  padded.
- Pass works at the midend level
---
 .../llvm/Analysis/TargetTransformInfo.h       |  11 ++
 .../llvm/Analysis/TargetTransformInfoImpl.h   |   4 +
 llvm/lib/Analysis/TargetTransformInfo.cpp     |   6 +
 .../lib/Target/ARM/ARMTargetTransformInfo.cpp |  33 ++++
 llvm/lib/Target/ARM/ARMTargetTransformInfo.h  |   3 +
 llvm/lib/Transforms/IPO/GlobalOpt.cpp         | 165 ++++++++++++++++++
 .../GlobalOpt/ARM/arm-widen-dest-non-array.ll |  39 +++++
 .../GlobalOpt/ARM/arm-widen-global-dest.ll    |  28 +++
 .../GlobalOpt/ARM/arm-widen-non-byte-array.ll |  22 +++
 .../ARM/arm-widen-non-const-global.ll         |  21 +++
 .../ARM/arm-widen-string-multi-use.ll         |  33 ++++
 .../GlobalOpt/ARM/arm-widen-strings-1.ll      |  21 +++
 .../GlobalOpt/ARM/arm-widen-strings-2.ll      |  21 +++
 .../arm-widen-strings-lengths-dont-match.ll   |  26 +++
 .../arm-widen-strings-more-than-64-bytes.ll   |  28 +++
 .../ARM/arm-widen-strings-ptrtoint.ll         |  54 ++++++
 .../ARM/arm-widen-strings-struct-test.ll      |  45 +++++
 .../ARM/arm-widen-strings-volatile.ll         |  29 +++
 .../Transforms/GlobalOpt/ARM/lit.local.cfg    |   2 +
 19 files changed, 591 insertions(+)
 create mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-dest-non-array.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-global-dest.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-byte-array.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-const-global.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-string-multi-use.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-1.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-2.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-lengths-dont-match.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-more-than-64-bytes.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-ptrtoint.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-struct-test.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-volatile.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/ARM/lit.local.cfg

diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 0459941fe05c..0dc513d8e65b 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1819,6 +1819,10 @@ public:
   /// \return The maximum number of function arguments the target supports.
   unsigned getMaxNumArgs() const;
 
+  /// \return For an array of given Size, return alignment boundary to
+  /// pad to. Default is no padding.
+  unsigned getNumBytesToPadGlobalArray(unsigned Size, Type *ArrayType) const;
+
   /// @}
 
 private:
@@ -2225,6 +2229,8 @@ public:
   getVPLegalizationStrategy(const VPIntrinsic &PI) const = 0;
   virtual bool hasArmWideBranch(bool Thumb) const = 0;
   virtual unsigned getMaxNumArgs() const = 0;
+  virtual unsigned getNumBytesToPadGlobalArray(unsigned Size,
+                                               Type *ArrayType) const = 0;
 };
 
 template <typename T>
@@ -3026,6 +3032,11 @@ public:
   unsigned getMaxNumArgs() const override {
     return Impl.getMaxNumArgs();
   }
+
+  unsigned getNumBytesToPadGlobalArray(unsigned Size,
+                                       Type *ArrayType) const override {
+    return Impl.getNumBytesToPadGlobalArray(Size, ArrayType);
+  }
 };
 
 template <typename T>
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index dbdfb4d8cdfa..0b7792f89a05 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -1006,6 +1006,10 @@ public:
 
   unsigned getMaxNumArgs() const { return UINT_MAX; }
 
+  unsigned getNumBytesToPadGlobalArray(unsigned Size, Type *ArrayType) const {
+    return 0;
+  }
+
 protected:
   // Obtain the minimum required size to hold the value (without the sign)
   // In case of a vector it returns the min required size for one element.
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index a47462b61e03..607047336376 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1383,6 +1383,12 @@ bool TargetTransformInfo::isVectorShiftByScalarCheap(Type *Ty) const {
   return TTIImpl->isVectorShiftByScalarCheap(Ty);
 }
 
+unsigned
+TargetTransformInfo::getNumBytesToPadGlobalArray(unsigned Size,
+                                                 Type *ArrayType) const {
+  return TTIImpl->getNumBytesToPadGlobalArray(Size, ArrayType);
+}
+
 TargetTransformInfo::Concept::~Concept() = default;
 
 TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 835ae98efb85..9f6e5e5ab142 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -56,6 +56,10 @@ static cl::opt<bool>
     AllowWLSLoops("allow-arm-wlsloops", cl::Hidden, cl::init(true),
                   cl::desc("Enable the generation of WLS loops"));
 
+static cl::opt<bool> UseWidenGlobalArrays(
+    "widen-global-strings", cl::Hidden, cl::init(true),
+    cl::desc("Enable the widening of global strings to alignment boundaries"));
+
 extern cl::opt<TailPredication::Mode> EnableTailPredication;
 
 extern cl::opt<bool> EnableMaskedGatherScatters;
@@ -2805,3 +2809,32 @@ bool ARMTTIImpl::isProfitableToSinkOperands(Instruction *I,
   }
   return true;
 }
+
+unsigned ARMTTIImpl::getNumBytesToPadGlobalArray(unsigned Size,
+                                                 Type *ArrayType) const {
+  if (!UseWidenGlobalArrays) {
+    LLVM_DEBUG(dbgs() << "Padding global arrays disabled\n");
+    return false;
+  }
+
+  // Don't modify none integer array types
+  if (!ArrayType || !ArrayType->isArrayTy() ||
+      !ArrayType->getArrayElementType()->isIntegerTy())
+    return 0;
+
+  // We pad to 4 byte boundaries
+  if (Size % 4 == 0)
+    return 0;
+
+  unsigned NumBytesToPad = 4 - (Size % 4);
+  unsigned NewSize = Size + NumBytesToPad;
+
+  // Max number of bytes that memcpy allows for lowering to load/stores before
+  // it uses library function (__aeabi_memcpy).
+  unsigned MaxMemIntrinsicSize = getMaxMemIntrinsicInlineSizeThreshold();
+
+  if (NewSize > MaxMemIntrinsicSize)
+    return 0;
+
+  return NumBytesToPad;
+}
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index b0a75134ee02..3a4f940088b2 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -337,6 +337,9 @@ public:
 
   bool isProfitableToSinkOperands(Instruction *I,
                                   SmallVectorImpl<Use *> &Ops) const;
+
+  unsigned getNumBytesToPadGlobalArray(unsigned Size, Type *ArrayType) const;
+
   /// @}
 };
 
diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index aae4926e027f..4647c65a5c85 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -92,6 +92,8 @@ STATISTIC(NumInternalFunc, "Number of internal functions");
 STATISTIC(NumColdCC, "Number of functions marked coldcc");
 STATISTIC(NumIFuncsResolved, "Number of statically resolved IFuncs");
 STATISTIC(NumIFuncsDeleted, "Number of IFuncs removed");
+STATISTIC(NumGlobalArraysPadded,
+          "Number of global arrays padded to alignment boundary");
 
 static cl::opt<bool>
     EnableColdCCStressTest("enable-coldcc-stress-test",
@@ -2029,6 +2031,165 @@ OptimizeFunctions(Module &M,
   return Changed;
 }
 
+static bool callInstIsMemcpy(CallInst *CI) {
+  if (!CI)
+    return false;
+
+  Function *F = CI->getCalledFunction();
+  if (!F || !F->isIntrinsic() || F->getIntrinsicID() != Intrinsic::memcpy)
+    return false;
+
+  return true;
+}
+
+static bool destArrayCanBeWidened(CallInst *CI) {
+  auto *IsVolatile = dyn_cast<ConstantInt>(CI->getArgOperand(3));
+  auto *Alloca = dyn_cast<AllocaInst>(CI->getArgOperand(0));
+
+  if (!Alloca || !IsVolatile || IsVolatile->isOne())
+    return false;
+
+  if (!Alloca->isStaticAlloca())
+    return false;
+
+  if (!Alloca->getAllocatedType()->isArrayTy())
+    return false;
+
+  return true;
+}
+
+static GlobalVariable *widenGlobalVariable(GlobalVariable *OldVar, Function *F,
+                                           unsigned NumBytesToPad,
+                                           unsigned NumBytesToCopy) {
+  if (!OldVar->hasInitializer())
+    return nullptr;
+
+  ConstantDataArray *DataArray =
+      dyn_cast<ConstantDataArray>(OldVar->getInitializer());
+  if (!DataArray)
+    return nullptr;
+
+  // Update to be word aligned (memcpy(...,X,...))
+  // create replacement with padded null bytes.
+  StringRef Data = DataArray->getRawDataValues();
+  std::vector<uint8_t> StrData(Data.begin(), Data.end());
+  for (unsigned int p = 0; p < NumBytesToPad; p++)
+    StrData.push_back('\0');
+  auto Arr = ArrayRef(StrData.data(), NumBytesToCopy + NumBytesToPad);
+  // Create new padded version of global variable.
+  Constant *SourceReplace = ConstantDataArray::get(F->getContext(), Arr);
+  GlobalVariable *NewGV = new GlobalVariable(
+      *(F->getParent()), SourceReplace->getType(), true, OldVar->getLinkage(),
+      SourceReplace, SourceReplace->getName());
+  // Copy any other attributes from original global variable
+  // e.g. unamed_addr
+  NewGV->copyAttributesFrom(OldVar);
+  NewGV->takeName(OldVar);
+  return NewGV;
+}
+
+static void widenDestArray(CallInst *CI, const unsigned NumBytesToPad,
+                           const unsigned NumBytesToCopy,
+                           ConstantDataArray *SourceDataArray) {
+
+  auto *Alloca = dyn_cast<AllocaInst>(CI->getArgOperand(0));
+  if (Alloca) {
+    unsigned ElementByteWidth = SourceDataArray->getElementByteSize();
+    unsigned int TotalBytes = NumBytesToCopy + NumBytesToPad;
+    unsigned NumElementsToCopy = divideCeil(TotalBytes, ElementByteWidth);
+    // Update destination array to be word aligned (memcpy(X,...,...))
+    IRBuilder<> BuildAlloca(Alloca);
+    AllocaInst *NewAlloca = BuildAlloca.CreateAlloca(ArrayType::get(
+        Alloca->getAllocatedType()->getArrayElementType(), NumElementsToCopy));
+    NewAlloca->takeName(Alloca);
+    NewAlloca->setAlignment(Alloca->getAlign());
+    Alloca->replaceAllUsesWith(NewAlloca);
+    Alloca->eraseFromParent();
+  }
+}
+
+static bool tryWidenGlobalArrayAndDests(Function *F, GlobalVariable *SourceVar,
+                                        const unsigned NumBytesToPad,
+                                        const unsigned NumBytesToCopy,
+                                        ConstantInt *BytesToCopyOp,
+                                        ConstantDataArray *SourceDataArray) {
+  auto *NewSourceGV =
+      widenGlobalVariable(SourceVar, F, NumBytesToPad, NumBytesToCopy);
+  if (!NewSourceGV)
+    return false;
+
+  // Update arguments of remaining uses  that
+  // are memcpys.
+  for (auto *User : SourceVar->users()) {
+    auto *CI = dyn_cast<CallInst>(User);
+    if (!callInstIsMemcpy(CI) || !destArrayCanBeWidened(CI))
+      continue;
+
+    if (CI->getArgOperand(1) != SourceVar)
+      continue;
+
+    widenDestArray(CI, NumBytesToPad, NumBytesToCopy, SourceDataArray);
+
+    CI->setArgOperand(2, ConstantInt::get(BytesToCopyOp->getType(),
+                                          NumBytesToCopy + NumBytesToPad));
+  }
+  SourceVar->replaceAllUsesWith(NewSourceGV);
+
+  NumGlobalArraysPadded++;
+  return true;
+}
+
+static bool tryWidenGlobalArraysUsedByMemcpy(
+    GlobalVariable *GV,
+    function_ref<TargetTransformInfo &(Function &)> GetTTI) {
+
+  if (!GV->hasInitializer() || !GV->isConstant() || !GV->hasLocalLinkage() ||
+      !GV->hasGlobalUnnamedAddr())
+    return false;
+
+  for (auto *User : GV->users()) {
+    CallInst *CI = dyn_cast<CallInst>(User);
+    if (!callInstIsMemcpy(CI) || !destArrayCanBeWidened(CI))
+      continue;
+
+    Function *F = CI->getCalledFunction();
+
+    auto *BytesToCopyOp = dyn_cast<ConstantInt>(CI->getArgOperand(2));
+    if (!BytesToCopyOp)
+      continue;
+
+    ConstantDataArray *SourceDataArray =
+        dyn_cast<ConstantDataArray>(GV->getInitializer());
+    if (!SourceDataArray)
+      continue;
+
+    unsigned NumBytesToCopy = BytesToCopyOp->getZExtValue();
+
+    auto *Alloca = dyn_cast<AllocaInst>(CI->getArgOperand(0));
+    uint64_t DZSize = Alloca->getAllocatedType()->getArrayNumElements();
+    uint64_t SZSize = SourceDataArray->getType()->getNumElements();
+    unsigned ElementByteWidth = SourceDataArray->getElementByteSize();
+    // Calculate the number of elements to copy while avoiding floored
+    // division of integers returning wrong values i.e. copying one byte
+    // from an array of i16 would yield 0 elements to copy as supposed to 1.
+    unsigned NumElementsToCopy = divideCeil(NumBytesToCopy, ElementByteWidth);
+
+    // For safety purposes lets add a constraint and only pad when
+    // NumElementsToCopy == destination array size ==
+    // source which is a constant
+    if (NumElementsToCopy != DZSize || DZSize != SZSize)
+      continue;
+
+    unsigned NumBytesToPad = GetTTI(*F).getNumBytesToPadGlobalArray(
+        NumBytesToCopy, SourceDataArray->getType());
+    if (NumBytesToPad) {
+      return tryWidenGlobalArrayAndDests(F, GV, NumBytesToPad, NumBytesToCopy,
+                                         BytesToCopyOp, SourceDataArray);
+    }
+  }
+  return false;
+}
+
 static bool
 OptimizeGlobalVars(Module &M,
                    function_ref<TargetTransformInfo &(Function &)> GetTTI,
@@ -2058,6 +2219,10 @@ OptimizeGlobalVars(Module &M,
       continue;
     }
 
+    // For global variable arrays called in a memcpy
+    // we try to pad to nearest valid alignment boundary
+    Changed |= tryWidenGlobalArraysUsedByMemcpy(&GV, GetTTI);
+
     Changed |= processGlobal(GV, GetTTI, GetTLI, LookupDomTree);
   }
   return Changed;
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-dest-non-array.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-dest-non-array.ll
new file mode 100644
index 000000000000..ab04e0a5bc69
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-dest-non-array.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
+
+@.i8 = private unnamed_addr constant [3 x i8] [i8 1, i8 2, i8 3] , align 1
+
+define  void @memcpy_struct()  {
+; CHECK-LABEL: define void @memcpy_struct() local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca { i8, i8, i8 }, align 1
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %something = alloca {i8, i8, i8}, align 1
+  %call1 = call i32 @bar(ptr nonnull %something)
+  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
+  ret void
+}
+
+
+@.i8_multi = private unnamed_addr constant [2 x [3 x i8]] [[3 x i8] [i8 1, i8 2, i8 3], [3 x i8] [i8 4, i8 5, i8 6]] , align 1
+
+define  void @memcpy_array_multidimensional()  {
+; CHECK-LABEL: define void @memcpy_array_multidimensional() local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [2 x [3 x i8]], align 1
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(3) @.i8_multi, i32 3, i1 false)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %something = alloca [2 x [3 x i8]], align 1
+  %call1 = call i32 @bar(ptr nonnull %something)
+  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something, ptr noundef nonnull align 1 dereferenceable(3) @.i8_multi, i32 3, i1 false)
+  ret void
+}
+
+declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-global-dest.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-global-dest.ll
new file mode 100644
index 000000000000..f435ffdeed2c
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-global-dest.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
+
+; CHECK: [3 x i8]
+@other = private unnamed_addr global [3 x i8] [i8 1, i8 2, i8 3] , align 1
+; CHECK: [4 x i8]
+@.i8 = private unnamed_addr constant [3 x i8] [i8 1, i8 2, i8 3] , align 1
+
+define  void @memcpy_multiple()  {
+; CHECK-LABEL: define void @memcpy_multiple() local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [4 x i8], align 1
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
+; CHECK-NEXT:    [[CALL3:%.*]] = call i32 @bar(ptr nonnull @other)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) @other, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 4, i1 false)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %something = alloca [3 x i8], align 1
+  %call1 = call i32 @bar(ptr nonnull %something)
+  %call2 = call i32 @bar(ptr nonnull @other)
+  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) @other, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
+  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
+  ret void
+}
+
+declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-byte-array.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-byte-array.ll
new file mode 100644
index 000000000000..c7ca7271fd3d
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-byte-array.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
+
+@.i16 = private unnamed_addr constant [5 x i16] [i16 1, i16 2, i16 3, i16 4, i16 5] , align 1
+
+define  void @memcpy_i16_array()  {
+; CHECK-LABEL: define void @memcpy_i16_array() local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SOMETHING1:%.*]] = alloca [6 x i16], align 1
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(10) [[SOMETHING1]], ptr noundef nonnull align 1 dereferenceable(10) @.i16, i32 12, i1 false)
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING1]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %something = alloca [5 x i16], align 1
+  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(10) %something, ptr noundef nonnull align 1 dereferenceable(10) @.i16, i32 10, i1 false)
+  %call2 = call i32 @bar(ptr nonnull %something)
+  ret void
+}
+
+
+declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-const-global.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-const-global.ll
new file mode 100644
index 000000000000..3d9c42fe1f3d
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-const-global.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
+
+@.str = unnamed_addr global [3 x i8] c"12\00", align 1
+
+define  void @foo()  {
+; CHECK-LABEL: define void @foo() local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [3 x i8], align 1
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(3) @.str, i32 3, i1 false)
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %something = alloca [3 x i8], align 1
+  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something, ptr noundef nonnull align 1 dereferenceable(3) @.str, i32 3, i1 false)
+  %call1 = call i32 @bar(ptr nonnull %something)
+  ret void
+}
+
+declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-string-multi-use.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-string-multi-use.ll
new file mode 100644
index 000000000000..e37925a78d2c
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-string-multi-use.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
+
+@.i8 = private unnamed_addr constant [3 x i8] [i8 1, i8 2, i8 3] , align 1
+
+define  void @memcpy_multiple()  {
+; CHECK-LABEL: define void @memcpy_multiple() local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SOMETHING2:%.*]] = alloca [4 x i8], align 1
+; CHECK-NEXT:    [[SOMETHING1:%.*]] = alloca [4 x i8], align 1
+; CHECK-NEXT:    [[SOMETHING3:%.*]] = alloca [4 x i8], align 1
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING2]], ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 4, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING1]], ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 4, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING3]], ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 4, i1 false)
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING2]])
+; CHECK-NEXT:    [[CALL3:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING1]])
+; CHECK-NEXT:    [[CALL4:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING3]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %something = alloca [3 x i8], align 1
+  %something1 = alloca [3 x i8], align 1
+  %something2 = alloca [3 x i8], align 1
+  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
+  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something1, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
+  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something2, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
+  %call2 = call i32 @bar(ptr nonnull %something)
+  %call3 = call i32 @bar(ptr nonnull %something1)
+  %call4 = call i32 @bar(ptr nonnull %something2)
+  ret void
+}
+
+declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-1.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-1.ll
new file mode 100644
index 000000000000..8ea9e2804370
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-1.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
+
+@.str = private unnamed_addr constant [10 x i8] c"123456789\00", align 1
+
+define  void @foo()  {
+; CHECK-LABEL: define void @foo() local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [12 x i8], align 1
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(10) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(10) @.str, i32 12, i1 false)
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %something = alloca [10 x i8], align 1
+  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(10) %something, ptr noundef nonnull align 1 dereferenceable(10) @.str, i32 10, i1 false)
+  %call2 = call i32 @bar(ptr nonnull %something)
+  ret void
+}
+
+declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-2.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-2.ll
new file mode 100644
index 000000000000..ad3620b14ea2
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-2.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
+
+@.str = private unnamed_addr constant [62 x i8] c"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\00", align 1
+
+define  void @foo()  {
+; CHECK-LABEL: define void @foo() local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [64 x i8], align 1
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(62) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(62) @.str, i32 64, i1 false)
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %something = alloca [62 x i8], align 1
+  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(62) %something, ptr noundef nonnull align 1 dereferenceable(62) @.str, i32 62, i1 false)
+  %call2 = call i32 @bar(ptr nonnull %something)
+  ret void
+}
+
+declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-lengths-dont-match.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-lengths-dont-match.ll
new file mode 100644
index 000000000000..b8e02c3f996d
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-lengths-dont-match.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
+; CHECK: [17 x i8]
+@.str = private unnamed_addr constant [17 x i8] c"aaaaaaaaaaaaaaaa\00", align 1
+
+; Function Attrs: nounwind
+define  void @foo()   {
+; CHECK-LABEL: define void @foo() local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [20 x i8], align 1
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr nonnull [[SOMETHING]])
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 1 [[SOMETHING]], ptr align 1 @.str, i32 17, i1 false)
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr nonnull [[SOMETHING]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %something = alloca [20 x i8], align 1
+  call void @llvm.lifetime.start(i64 20, ptr nonnull %something) #3
+  call void @llvm.memcpy.p0i8.p0i8.i32(ptr align 1 nonnull %something, ptr align 1 @.str, i32 17, i1 false)
+  %call2 = call i32 @bar(ptr nonnull %something) #3
+  call void @llvm.lifetime.end(i64 20, ptr nonnull %something) #3
+  ret void
+}
+
+declare i32 @bar(...)  #2
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-more-than-64-bytes.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-more-than-64-bytes.ll
new file mode 100644
index 000000000000..4ac31aa2f976
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-more-than-64-bytes.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
+
+; CHECK: [65 x i8]
+; CHECK-NOT: [68 x i8]
+@.str = private unnamed_addr constant [65 x i8] c"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaazzz\00", align 1
+
+; Function Attrs: nounwind
+define  void @foo()   {
+; CHECK-LABEL: define void @foo() local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [65 x i8], align 1
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 65, ptr nonnull [[SOMETHING]])
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 1 [[SOMETHING]], ptr align 1 @.str, i32 65, i1 false)
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 65, ptr nonnull [[SOMETHING]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %something = alloca [65 x i8], align 1
+  call void @llvm.lifetime.start(i64 65, ptr nonnull %something) #3
+  call void @llvm.memcpy.p0i8.p0i8.i32(ptr align 1 nonnull %something, ptr align 1 @.str, i32 65, i1 false)
+  %call2 = call i32 @bar(ptr nonnull %something) #3
+  call void @llvm.lifetime.end(i64 65, ptr nonnull %something) #3
+  ret void
+}
+
+declare i32 @bar(...)  #2
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-ptrtoint.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-ptrtoint.ll
new file mode 100644
index 000000000000..64f57884cd39
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-ptrtoint.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
+
+@f.string1 = private unnamed_addr constant [45 x i8] c"The quick brown dog jumps over the lazy fox.\00", align 1
+
+; Function Attrs: nounwind
+define  i32 @f() {
+; CHECK-LABEL: define i32 @f() local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[STRING1:%.*]] = alloca [48 x i8], align 1
+; CHECK-NEXT:    [[POS:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TOKEN:%.*]] = alloca ptr, align 4
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 45, ptr [[STRING1]])
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[STRING1]], ptr align 1 @f.string1, i32 48, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[POS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TOKEN]])
+; CHECK-NEXT:    [[CALL:%.*]] = call ptr @strchr(ptr [[STRING1]], i32 101)
+; CHECK-NEXT:    store ptr [[CALL]], ptr [[TOKEN]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TOKEN]], align 4
+; CHECK-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP1]] to i32
+; CHECK-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[STRING1]] to i32
+; CHECK-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i32 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUB_PTR_SUB]], 1
+; CHECK-NEXT:    store i32 [[ADD]], ptr [[POS]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[POS]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TOKEN]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[POS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 45, ptr [[STRING1]])
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+entry:
+  %string1 = alloca [45 x i8], align 1
+  %pos = alloca i32, align 4
+  %token = alloca ptr, align 4
+  call void @llvm.lifetime.start.p0i8(i64 45, ptr %string1)
+  call void @llvm.memcpy.p0i8.p0i8.i32(ptr align 1 %string1, ptr align 1 @f.string1, i32 45, i1 false)
+  call void @llvm.lifetime.start.p0i8(i64 4, ptr %pos)
+  call void @llvm.lifetime.start.p0i8(i64 4, ptr %token)
+  %call = call ptr @strchr(ptr %string1, i32 101)
+  store ptr %call, ptr %token, align 4
+  %0 = load ptr, ptr %token, align 4
+  %sub.ptr.lhs.cast = ptrtoint ptr %0 to i32
+  %sub.ptr.rhs.cast = ptrtoint ptr %string1 to i32
+  %sub.ptr.sub = sub i32 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast
+  %add = add nsw i32 %sub.ptr.sub, 1
+  store i32 %add, ptr %pos, align 4
+  %1 = load i32, ptr %pos, align 4
+  call void @llvm.lifetime.end.p0i8(i64 4, ptr %token)
+  call void @llvm.lifetime.end.p0i8(i64 4, ptr %pos)
+  call void @llvm.lifetime.end.p0i8(i64 45, ptr %string1)
+  ret i32 %1
+}
+
+declare ptr @strchr(ptr, i32)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-struct-test.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-struct-test.ll
new file mode 100644
index 000000000000..5367572704b1
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-struct-test.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
+%struct.P = type { i32, [13 x i8] }
+
+; CHECK-NOT: [16 x i8]
+@.str = private unnamed_addr constant [13 x i8] c"hello world\0A\00", align 1
+
+; Function Attrs: nounwind
+define  i32 @main()   {
+; CHECK-LABEL: define i32 @main() local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[P:%.*]] = alloca [[STRUCT_P:%.*]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr nonnull [[P]])
+; CHECK-NEXT:    store i32 10, ptr [[P]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [[STRUCT_P]], ptr [[P]], i32 0, i32 1, i32 0
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[ARRAYDECAY]], ptr align 1 @.str, i32 13, i1 false)
+; CHECK-NEXT:    [[PUTS:%.*]] = call i32 @puts(ptr [[ARRAYDECAY]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr nonnull [[P]])
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %p = alloca %struct.P, align 4
+  call void @llvm.lifetime.start(i64 20, ptr nonnull %p) #2
+  store i32 10, ptr %p, align 4, !tbaa !1
+  %arraydecay = getelementptr inbounds %struct.P, ptr %p, i32 0, i32 1, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(ptr align 1 %arraydecay, ptr align 1 @.str, i32 13, i1 false)
+  %puts = call i32 @puts(ptr %arraydecay)
+  call void @llvm.lifetime.end(i64 20, ptr nonnull %p) #2
+  ret i32 0
+}
+
+declare i32 @puts(ptr nocapture readonly) #2
+
+!1 = !{!2, !3, i64 0}
+!2 = !{!"P", !3, i64 0, !4, i64 4}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+;.
+; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META2:![0-9]+]], i64 0}
+; CHECK: [[META1]] = !{!"P", [[META2]], i64 0, [[META3:![0-9]+]], i64 4}
+; CHECK: [[META2]] = !{!"int", [[META3]], i64 0}
+; CHECK: [[META3]] = !{!"omnipotent char", [[META4:![0-9]+]], i64 0}
+; CHECK: [[META4]] = !{!"Simple C/C++ TBAA"}
+;.
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-volatile.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-volatile.ll
new file mode 100644
index 000000000000..b735a7788742
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-volatile.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
+
+; CHECK-NOT: [64 x i8]
+@.str = private unnamed_addr constant [62 x i8] c"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\00", align 1
+
+; Function Attrs: nounwind
+define  void @foo()   {
+; CHECK-LABEL: define void @foo() local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [62 x i8], align 1
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [62 x i8], ptr [[SOMETHING]], i32 0, i32 0
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 62, ptr nonnull [[TMP0]])
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 1 [[TMP0]], ptr align 1 @.str, i32 62, i1 true)
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[TMP0]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 62, ptr nonnull [[TMP0]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %something = alloca [62 x i8], align 1
+  %0 = getelementptr inbounds [62 x i8], ptr %something, i32 0, i32 0
+  call void @llvm.lifetime.start(i64 62, ptr nonnull %0) #3
+  call void @llvm.memcpy.p0i8.p0i8.i32(ptr align 1 nonnull %0, ptr align 1 @.str, i32 62, i1 true)
+  %call2 = call i32 @bar(ptr nonnull %0) #3
+  call void @llvm.lifetime.end(i64 62, ptr nonnull %0) #3
+  ret void
+}
+
+declare i32 @bar(...)  #2
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/lit.local.cfg b/llvm/test/Transforms/GlobalOpt/ARM/lit.local.cfg
new file mode 100644
index 000000000000..7c23e4f98013
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ARM/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "ARM" in config.root.targets:
+    config.unsupported = True
-- 
GitLab


From 4dd55c567aaed30c6842812e0798a70fee324c98 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Thu, 24 Oct 2024 10:23:40 +0100
Subject: [PATCH 038/693] [clang] Use {} instead of std::nullopt to initialize
 empty ArrayRef (#109399)

Follow up to #109133.
---
 clang/include/clang/AST/CommentSema.h         |  2 +-
 clang/include/clang/AST/DeclFriend.h          |  2 +-
 clang/include/clang/AST/DeclObjC.h            |  2 +-
 clang/include/clang/AST/DeclOpenMP.h          |  4 +-
 clang/include/clang/AST/ExprCXX.h             | 10 +--
 clang/include/clang/AST/OpenMPClause.h        |  6 +-
 clang/include/clang/AST/StmtOpenMP.h          |  4 +-
 clang/include/clang/AST/TemplateBase.h        |  2 +-
 .../clang/ASTMatchers/ASTMatchersInternal.h   | 12 +--
 .../clang/Analysis/Analyses/ThreadSafetyTIL.h |  4 +-
 clang/include/clang/Analysis/AnyCall.h        |  4 +-
 clang/include/clang/Basic/TargetInfo.h        |  6 +-
 clang/include/clang/Driver/Job.h              |  6 +-
 clang/include/clang/Frontend/ASTUnit.h        |  4 +-
 clang/include/clang/Lex/Preprocessor.h        |  6 +-
 .../include/clang/Sema/CodeCompleteConsumer.h |  7 +-
 clang/include/clang/Sema/Overload.h           |  5 +-
 clang/include/clang/Sema/Sema.h               | 74 ++++++++++---------
 clang/include/clang/Sema/SemaObjC.h           |  4 +-
 clang/include/clang/Sema/SemaOpenACC.h        |  2 +-
 clang/include/clang/Sema/SemaOpenMP.h         | 12 +--
 .../Core/BugReporter/BugReporter.h            |  8 +-
 clang/lib/ARCMigrate/Internals.h              |  2 +-
 clang/lib/AST/ASTContext.cpp                  |  4 +-
 clang/lib/AST/Comment.cpp                     |  2 +-
 clang/lib/AST/CommentParser.cpp               |  4 +-
 clang/lib/AST/Decl.cpp                        |  7 +-
 clang/lib/AST/DeclCXX.cpp                     |  5 +-
 clang/lib/AST/DeclObjC.cpp                    |  4 +-
 clang/lib/AST/DeclOpenMP.cpp                  |  2 +-
 clang/lib/AST/DeclTemplate.cpp                |  5 +-
 clang/lib/AST/Expr.cpp                        |  3 +-
 clang/lib/AST/MicrosoftMangle.cpp             |  2 +-
 clang/lib/AST/StmtOpenMP.cpp                  |  7 +-
 clang/lib/Analysis/BodyFarm.cpp               |  4 +-
 clang/lib/Analysis/UnsafeBufferUsage.cpp      |  2 +-
 clang/lib/Basic/Targets/AMDGPU.h              |  2 +-
 clang/lib/Basic/Targets/ARC.h                 |  6 +-
 clang/lib/Basic/Targets/AVR.h                 |  6 +-
 clang/lib/Basic/Targets/BPF.h                 |  6 +-
 clang/lib/Basic/Targets/DirectX.h             | 10 +--
 clang/lib/Basic/Targets/Lanai.h               |  4 +-
 clang/lib/Basic/Targets/M68k.cpp              |  2 +-
 clang/lib/Basic/Targets/MSP430.h              |  2 +-
 clang/lib/Basic/Targets/NVPTX.h               |  2 +-
 clang/lib/Basic/Targets/PNaCl.cpp             |  6 +-
 clang/lib/Basic/Targets/PNaCl.h               |  4 +-
 clang/lib/Basic/Targets/SPIR.h                | 10 +--
 clang/lib/Basic/Targets/Sparc.h               |  2 +-
 clang/lib/Basic/Targets/SystemZ.h             |  2 +-
 clang/lib/Basic/Targets/TCE.h                 | 10 +--
 clang/lib/Basic/Targets/WebAssembly.h         |  4 +-
 clang/lib/Basic/Targets/X86.h                 |  2 +-
 clang/lib/Basic/Targets/XCore.h               |  2 +-
 clang/lib/CodeGen/CGBuiltin.cpp               |  8 +-
 clang/lib/CodeGen/CGCall.cpp                  | 16 ++--
 clang/lib/CodeGen/CGCleanup.cpp               |  3 +-
 clang/lib/CodeGen/CGDebugInfo.cpp             |  5 +-
 clang/lib/CodeGen/CGExpr.cpp                  | 10 +--
 clang/lib/CodeGen/CGLoopInfo.cpp              |  2 +-
 clang/lib/CodeGen/CGObjC.cpp                  | 11 ++-
 clang/lib/CodeGen/CGObjCGNU.cpp               |  2 +-
 clang/lib/CodeGen/CGObjCMac.cpp               |  4 +-
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         | 31 ++++----
 clang/lib/CodeGen/CGOpenMPRuntime.h           |  4 +-
 clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp      | 14 ++--
 clang/lib/CodeGen/CGOpenMPRuntimeGPU.h        |  7 +-
 clang/lib/CodeGen/CGStmtOpenMP.cpp            | 16 ++--
 clang/lib/CodeGen/CodeGenFunction.cpp         |  2 +-
 clang/lib/CodeGen/CodeGenFunction.h           | 12 ++-
 clang/lib/CodeGen/CodeGenModule.h             |  6 +-
 clang/lib/CodeGen/CoverageMappingGen.cpp      |  2 +-
 clang/lib/CodeGen/ItaniumCXXABI.cpp           |  2 +-
 clang/lib/Frontend/DiagnosticRenderer.cpp     |  4 +-
 clang/lib/Index/IndexingContext.h             |  9 +--
 clang/lib/Lex/ModuleMap.cpp                   |  4 +-
 clang/lib/Lex/PPMacroExpansion.cpp            |  4 +-
 clang/lib/Parse/ParseDecl.cpp                 |  8 +-
 clang/lib/Parse/ParseDeclCXX.cpp              |  4 +-
 clang/lib/Parse/ParseExpr.cpp                 |  4 +-
 clang/lib/Parse/ParseExprCXX.cpp              |  5 +-
 clang/lib/Parse/ParseInit.cpp                 |  2 +-
 clang/lib/Parse/ParseObjc.cpp                 |  6 +-
 clang/lib/Parse/ParseOpenMP.cpp               |  2 +-
 clang/lib/Parse/ParseStmt.cpp                 |  6 +-
 clang/lib/Sema/Sema.cpp                       |  6 +-
 clang/lib/Sema/SemaChecking.cpp               |  4 +-
 clang/lib/Sema/SemaCodeComplete.cpp           | 16 ++--
 clang/lib/Sema/SemaCoroutine.cpp              | 22 +++---
 clang/lib/Sema/SemaDecl.cpp                   |  8 +-
 clang/lib/Sema/SemaDeclCXX.cpp                | 32 ++++----
 clang/lib/Sema/SemaDeclObjC.cpp               |  5 +-
 clang/lib/Sema/SemaExpr.cpp                   | 24 +++---
 clang/lib/Sema/SemaExprObjC.cpp               | 12 +--
 clang/lib/Sema/SemaInit.cpp                   | 21 +++---
 clang/lib/Sema/SemaLambda.cpp                 | 10 +--
 clang/lib/Sema/SemaLookup.cpp                 |  2 +-
 clang/lib/Sema/SemaObjCProperty.cpp           |  2 +-
 clang/lib/Sema/SemaOpenACC.cpp                |  3 +-
 clang/lib/Sema/SemaOpenMP.cpp                 |  8 +-
 clang/lib/Sema/SemaOverload.cpp               | 24 +++---
 clang/lib/Sema/SemaPseudoObject.cpp           |  8 +-
 clang/lib/Sema/SemaStmt.cpp                   |  2 +-
 clang/lib/Sema/SemaTemplate.cpp               |  2 +-
 clang/lib/Sema/SemaTemplateDeductionGuide.cpp |  2 +-
 clang/lib/Sema/SemaTemplateInstantiate.cpp    |  8 +-
 clang/lib/Sema/SemaType.cpp                   |  2 +-
 clang/lib/Sema/TreeTransform.h                | 19 +++--
 clang/lib/Serialization/ASTWriterDecl.cpp     |  2 +-
 clang/lib/StaticAnalyzer/Core/CallEvent.cpp   |  6 +-
 clang/lib/StaticAnalyzer/Core/MemRegion.cpp   |  2 +-
 clang/tools/libclang/CIndex.cpp               |  3 +-
 .../UncheckedOptionalAccessModelTest.cpp      |  6 +-
 clang/unittests/Frontend/ASTUnitTest.cpp      | 12 +--
 clang/utils/TableGen/NeonEmitter.cpp          |  2 +-
 115 files changed, 376 insertions(+), 435 deletions(-)

diff --git a/clang/include/clang/AST/CommentSema.h b/clang/include/clang/AST/CommentSema.h
index 03f13283ac0d..916d7945329c 100644
--- a/clang/include/clang/AST/CommentSema.h
+++ b/clang/include/clang/AST/CommentSema.h
@@ -80,7 +80,7 @@ public:
   ArrayRef<T> copyArray(ArrayRef<T> Source) {
     if (!Source.empty())
       return Source.copy(Allocator);
-    return std::nullopt;
+    return {};
   }
 
   ParagraphComment *actOnParagraphComment(
diff --git a/clang/include/clang/AST/DeclFriend.h b/clang/include/clang/AST/DeclFriend.h
index 095f14a81fd5..1578580c89cd 100644
--- a/clang/include/clang/AST/DeclFriend.h
+++ b/clang/include/clang/AST/DeclFriend.h
@@ -115,7 +115,7 @@ public:
   static FriendDecl *
   Create(ASTContext &C, DeclContext *DC, SourceLocation L, FriendUnion Friend_,
          SourceLocation FriendL, SourceLocation EllipsisLoc = {},
-         ArrayRef<TemplateParameterList *> FriendTypeTPLists = std::nullopt);
+         ArrayRef<TemplateParameterList *> FriendTypeTPLists = {});
   static FriendDecl *CreateDeserialized(ASTContext &C, GlobalDeclID ID,
                                         unsigned FriendTypeNumTPLists);
 
diff --git a/clang/include/clang/AST/DeclObjC.h b/clang/include/clang/AST/DeclObjC.h
index 1cda70530d7d..4663603f7975 100644
--- a/clang/include/clang/AST/DeclObjC.h
+++ b/clang/include/clang/AST/DeclObjC.h
@@ -386,7 +386,7 @@ public:
   /// If the method is implicit (not coming from source) \p SelLocs is
   /// ignored.
   void setMethodParams(ASTContext &C, ArrayRef<ParmVarDecl *> Params,
-                       ArrayRef<SourceLocation> SelLocs = std::nullopt);
+                       ArrayRef<SourceLocation> SelLocs = {});
 
   // Iterator access to parameter types.
   struct GetTypeFn {
diff --git a/clang/include/clang/AST/DeclOpenMP.h b/clang/include/clang/AST/DeclOpenMP.h
index 868662208efa..cf383889c0ad 100644
--- a/clang/include/clang/AST/DeclOpenMP.h
+++ b/clang/include/clang/AST/DeclOpenMP.h
@@ -34,7 +34,7 @@ template <typename U> class OMPDeclarativeDirective : public U {
   /// Get the clauses storage.
   MutableArrayRef<OMPClause *> getClauses() {
     if (!Data)
-      return std::nullopt;
+      return {};
     return Data->getClauses();
   }
 
@@ -90,7 +90,7 @@ public:
 
   ArrayRef<OMPClause *> clauses() const {
     if (!Data)
-      return std::nullopt;
+      return {};
     return Data->getClauses();
   }
 };
diff --git a/clang/include/clang/AST/ExprCXX.h b/clang/include/clang/AST/ExprCXX.h
index cfe3938f8384..57ab94bcb201 100644
--- a/clang/include/clang/AST/ExprCXX.h
+++ b/clang/include/clang/AST/ExprCXX.h
@@ -4306,11 +4306,11 @@ class SizeOfPackExpr final
       : Expr(SizeOfPackExprClass, Empty), Length(NumPartialArgs) {}
 
 public:
-  static SizeOfPackExpr *
-  Create(ASTContext &Context, SourceLocation OperatorLoc, NamedDecl *Pack,
-         SourceLocation PackLoc, SourceLocation RParenLoc,
-         std::optional<unsigned> Length = std::nullopt,
-         ArrayRef<TemplateArgument> PartialArgs = std::nullopt);
+  static SizeOfPackExpr *Create(ASTContext &Context, SourceLocation OperatorLoc,
+                                NamedDecl *Pack, SourceLocation PackLoc,
+                                SourceLocation RParenLoc,
+                                std::optional<unsigned> Length = std::nullopt,
+                                ArrayRef<TemplateArgument> PartialArgs = {});
   static SizeOfPackExpr *CreateDeserialized(ASTContext &Context,
                                             unsigned NumPartialArgs);
 
diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 2e48c1c3c72c..9cf46f73f6e4 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -6108,14 +6108,14 @@ public:
     return const_component_lists_iterator(
         getUniqueDeclsRef(), getDeclNumListsRef(), getComponentListSizesRef(),
         getComponentsRef(), SupportsMapper,
-        SupportsMapper ? getUDMapperRefs() : std::nullopt);
+        SupportsMapper ? getUDMapperRefs() : ArrayRef<Expr *>());
   }
   const_component_lists_iterator component_lists_end() const {
     return const_component_lists_iterator(
         ArrayRef<ValueDecl *>(), ArrayRef<unsigned>(), ArrayRef<unsigned>(),
         MappableExprComponentListRef(getComponentsRef().end(),
                                      getComponentsRef().end()),
-        SupportsMapper, std::nullopt);
+        SupportsMapper, {});
   }
   const_component_lists_range component_lists() const {
     return {component_lists_begin(), component_lists_end()};
@@ -6128,7 +6128,7 @@ public:
     return const_component_lists_iterator(
         VD, getUniqueDeclsRef(), getDeclNumListsRef(),
         getComponentListSizesRef(), getComponentsRef(), SupportsMapper,
-        SupportsMapper ? getUDMapperRefs() : std::nullopt);
+        SupportsMapper ? getUDMapperRefs() : ArrayRef<Expr *>());
   }
   const_component_lists_iterator decl_component_lists_end() const {
     return component_lists_end();
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index 930670d17f2d..9ec49b8683dc 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -277,7 +277,7 @@ class OMPExecutableDirective : public Stmt {
   /// Get the clauses storage.
   MutableArrayRef<OMPClause *> getClauses() {
     if (!Data)
-      return std::nullopt;
+      return {};
     return Data->getClauses();
   }
 
@@ -572,7 +572,7 @@ public:
 
   ArrayRef<OMPClause *> clauses() const {
     if (!Data)
-      return std::nullopt;
+      return {};
     return Data->getClauses();
   }
 
diff --git a/clang/include/clang/AST/TemplateBase.h b/clang/include/clang/AST/TemplateBase.h
index 0eaa4b0eedb3..a8f0263d5505 100644
--- a/clang/include/clang/AST/TemplateBase.h
+++ b/clang/include/clang/AST/TemplateBase.h
@@ -283,7 +283,7 @@ public:
   }
 
   static TemplateArgument getEmptyPack() {
-    return TemplateArgument(std::nullopt);
+    return TemplateArgument(ArrayRef<TemplateArgument>());
   }
 
   /// Create a new template argument pack by copying the given set of
diff --git a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
index c1cc63fdb743..ab8b146453e7 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
@@ -121,7 +121,7 @@ template <typename T> struct TypeListContainsSuperOf<EmptyTypeList, T> {
 template <typename ResultT, typename ArgT,
           ResultT (*Func)(ArrayRef<const ArgT *>)>
 struct VariadicFunction {
-  ResultT operator()() const { return Func(std::nullopt); }
+  ResultT operator()() const { return Func({}); }
 
   template <typename... ArgsT>
   ResultT operator()(const ArgT &Arg1, const ArgsT &... Args) const {
@@ -1949,35 +1949,35 @@ inline ArrayRef<TemplateArgument>
 getTemplateSpecializationArgs(const FunctionDecl &FD) {
   if (const auto* TemplateArgs = FD.getTemplateSpecializationArgs())
     return TemplateArgs->asArray();
-  return std::nullopt;
+  return {};
 }
 
 inline ArrayRef<TemplateArgumentLoc>
 getTemplateArgsWritten(const ClassTemplateSpecializationDecl &D) {
   if (const ASTTemplateArgumentListInfo *Args = D.getTemplateArgsAsWritten())
     return Args->arguments();
-  return std::nullopt;
+  return {};
 }
 
 inline ArrayRef<TemplateArgumentLoc>
 getTemplateArgsWritten(const VarTemplateSpecializationDecl &D) {
   if (const ASTTemplateArgumentListInfo *Args = D.getTemplateArgsAsWritten())
     return Args->arguments();
-  return std::nullopt;
+  return {};
 }
 
 inline ArrayRef<TemplateArgumentLoc>
 getTemplateArgsWritten(const FunctionDecl &FD) {
   if (const auto *Args = FD.getTemplateSpecializationArgsAsWritten())
     return Args->arguments();
-  return std::nullopt;
+  return {};
 }
 
 inline ArrayRef<TemplateArgumentLoc>
 getTemplateArgsWritten(const DeclRefExpr &DRE) {
   if (const auto *Args = DRE.getTemplateArgs())
     return {Args, DRE.getNumTemplateArgs()};
-  return std::nullopt;
+  return {};
 }
 
 inline SmallVector<TemplateArgumentLoc>
diff --git a/clang/include/clang/Analysis/Analyses/ThreadSafetyTIL.h b/clang/include/clang/Analysis/Analyses/ThreadSafetyTIL.h
index 65dd66ee093f..2f202607bd3f 100644
--- a/clang/include/clang/Analysis/Analyses/ThreadSafetyTIL.h
+++ b/clang/include/clang/Analysis/Analyses/ThreadSafetyTIL.h
@@ -1470,7 +1470,7 @@ public:
   static bool classof(const SExpr *E) { return E->opcode() == COP_Return; }
 
   /// Return an empty list.
-  ArrayRef<BasicBlock *> successors() { return std::nullopt; }
+  ArrayRef<BasicBlock *> successors() { return {}; }
 
   SExpr *returnValue() { return Retval; }
   const SExpr *returnValue() const { return Retval; }
@@ -1496,7 +1496,7 @@ inline ArrayRef<BasicBlock*> Terminator::successors() {
     case COP_Branch: return cast<Branch>(this)->successors();
     case COP_Return: return cast<Return>(this)->successors();
     default:
-      return std::nullopt;
+      return {};
   }
 }
 
diff --git a/clang/include/clang/Analysis/AnyCall.h b/clang/include/clang/Analysis/AnyCall.h
index 48abce062d13..3e95366c9859 100644
--- a/clang/include/clang/Analysis/AnyCall.h
+++ b/clang/include/clang/Analysis/AnyCall.h
@@ -143,7 +143,7 @@ public:
   /// \returns formal parameters for direct calls (including virtual calls)
   ArrayRef<ParmVarDecl *> parameters() const {
     if (!D)
-      return std::nullopt;
+      return {};
 
     if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
       return FD->parameters();
@@ -152,7 +152,7 @@ public:
     } else if (const auto *BD = dyn_cast<BlockDecl>(D)) {
       return BD->parameters();
     } else {
-      return std::nullopt;
+      return {};
     }
   }
 
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index e7469e1e9891..17262d5968b1 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -1858,11 +1858,9 @@ protected:
   }
   virtual ArrayRef<const char *> getGCCRegNames() const = 0;
   virtual ArrayRef<GCCRegAlias> getGCCRegAliases() const = 0;
-  virtual ArrayRef<AddlRegName> getGCCAddlRegNames() const {
-    return std::nullopt;
-  }
+  virtual ArrayRef<AddlRegName> getGCCAddlRegNames() const { return {}; }
 
- private:
+private:
   // Assert the values for the fractional and integral bits for each fixed point
   // type follow the restrictions given in clause 6.2.6.3 of N1169.
   void CheckFixedPointBits() const;
diff --git a/clang/include/clang/Driver/Job.h b/clang/include/clang/Driver/Job.h
index df9449463c53..561866197b78 100644
--- a/clang/include/clang/Driver/Job.h
+++ b/clang/include/clang/Driver/Job.h
@@ -172,8 +172,7 @@ public:
   Command(const Action &Source, const Tool &Creator,
           ResponseFileSupport ResponseSupport, const char *Executable,
           const llvm::opt::ArgStringList &Arguments, ArrayRef<InputInfo> Inputs,
-          ArrayRef<InputInfo> Outputs = std::nullopt,
-          const char *PrependArg = nullptr);
+          ArrayRef<InputInfo> Outputs = {}, const char *PrependArg = nullptr);
   // FIXME: This really shouldn't be copyable, but is currently copied in some
   // error handling in Driver::generateCompilationDiagnostics.
   Command(const Command &) = default;
@@ -245,8 +244,7 @@ public:
   CC1Command(const Action &Source, const Tool &Creator,
              ResponseFileSupport ResponseSupport, const char *Executable,
              const llvm::opt::ArgStringList &Arguments,
-             ArrayRef<InputInfo> Inputs,
-             ArrayRef<InputInfo> Outputs = std::nullopt,
+             ArrayRef<InputInfo> Inputs, ArrayRef<InputInfo> Outputs = {},
              const char *PrependArg = nullptr);
 
   void Print(llvm::raw_ostream &OS, const char *Terminator, bool Quote,
diff --git a/clang/include/clang/Frontend/ASTUnit.h b/clang/include/clang/Frontend/ASTUnit.h
index 8cefae8587aa..1f98c6ab328b 100644
--- a/clang/include/clang/Frontend/ASTUnit.h
+++ b/clang/include/clang/Frontend/ASTUnit.h
@@ -836,7 +836,7 @@ public:
       bool StorePreamblesInMemory = false,
       StringRef PreambleStoragePath = StringRef(), bool OnlyLocalDecls = false,
       CaptureDiagsKind CaptureDiagnostics = CaptureDiagsKind::None,
-      ArrayRef<RemappedFile> RemappedFiles = std::nullopt,
+      ArrayRef<RemappedFile> RemappedFiles = {},
       bool RemappedFilesKeepOriginalName = true,
       unsigned PrecompilePreambleAfterNParses = 0,
       TranslationUnitKind TUKind = TU_Complete,
@@ -864,7 +864,7 @@ public:
   /// \returns True if a failure occurred that causes the ASTUnit not to
   /// contain any translation-unit information, false otherwise.
   bool Reparse(std::shared_ptr<PCHContainerOperations> PCHContainerOps,
-               ArrayRef<RemappedFile> RemappedFiles = std::nullopt,
+               ArrayRef<RemappedFile> RemappedFiles = {},
                IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = nullptr);
 
   /// Free data that will be re-generated on the next parse.
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 92749e4de44b..f3f4de044fc4 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -911,7 +911,7 @@ private:
     getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
       if (auto *Info = getModuleInfo(PP, II))
         return Info->ActiveModuleMacros;
-      return std::nullopt;
+      return {};
     }
 
     MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
@@ -935,7 +935,7 @@ private:
     ArrayRef<ModuleMacro*> getOverriddenMacros() const {
       if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
         return Info->OverriddenMacros;
-      return std::nullopt;
+      return {};
     }
 
     void setOverriddenMacros(Preprocessor &PP,
@@ -1444,7 +1444,7 @@ public:
     auto I = LeafModuleMacros.find(II);
     if (I != LeafModuleMacros.end())
       return I->second;
-    return std::nullopt;
+    return {};
   }
 
   /// Get the list of submodules that we're currently building.
diff --git a/clang/include/clang/Sema/CodeCompleteConsumer.h b/clang/include/clang/Sema/CodeCompleteConsumer.h
index 0924dc27af82..2dd27593778c 100644
--- a/clang/include/clang/Sema/CodeCompleteConsumer.h
+++ b/clang/include/clang/Sema/CodeCompleteConsumer.h
@@ -375,12 +375,11 @@ private:
 public:
   /// Construct a new code-completion context of the given kind.
   CodeCompletionContext(Kind CCKind)
-      : CCKind(CCKind), IsUsingDeclaration(false), SelIdents(std::nullopt) {}
+      : CCKind(CCKind), IsUsingDeclaration(false), SelIdents() {}
 
   /// Construct a new code-completion context of the given kind.
-  CodeCompletionContext(
-      Kind CCKind, QualType T,
-      ArrayRef<const IdentifierInfo *> SelIdents = std::nullopt)
+  CodeCompletionContext(Kind CCKind, QualType T,
+                        ArrayRef<const IdentifierInfo *> SelIdents = {})
       : CCKind(CCKind), IsUsingDeclaration(false), SelIdents(SelIdents) {
     if (CCKind == CCC_DotMemberAccess || CCKind == CCC_ArrowMemberAccess ||
         CCKind == CCC_ObjCPropertyAccess || CCKind == CCC_ObjCClassMessage ||
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index c716a25bb673..58fa64c80a16 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -1206,9 +1206,8 @@ class Sema;
 
     /// Add a new candidate with NumConversions conversion sequence slots
     /// to the overload set.
-    OverloadCandidate &
-    addCandidate(unsigned NumConversions = 0,
-                 ConversionSequenceList Conversions = std::nullopt) {
+    OverloadCandidate &addCandidate(unsigned NumConversions = 0,
+                                    ConversionSequenceList Conversions = {}) {
       assert((Conversions.empty() || Conversions.size() == NumConversions) &&
              "preallocated conversion sequence has wrong length");
 
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 9e6b04bc3f8f..93d98e1cbb9c 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3478,10 +3478,12 @@ public:
   /// a C++0x [dcl.typedef]p2 alias-declaration: 'using T = A;'.
   NamedDecl *ActOnTypedefNameDecl(Scope *S, DeclContext *DC, TypedefNameDecl *D,
                                   LookupResult &Previous, bool &Redeclaration);
-  NamedDecl *ActOnVariableDeclarator(
-      Scope *S, Declarator &D, DeclContext *DC, TypeSourceInfo *TInfo,
-      LookupResult &Previous, MultiTemplateParamsArg TemplateParamLists,
-      bool &AddToScope, ArrayRef<BindingDecl *> Bindings = std::nullopt);
+  NamedDecl *ActOnVariableDeclarator(Scope *S, Declarator &D, DeclContext *DC,
+                                     TypeSourceInfo *TInfo,
+                                     LookupResult &Previous,
+                                     MultiTemplateParamsArg TemplateParamLists,
+                                     bool &AddToScope,
+                                     ArrayRef<BindingDecl *> Bindings = {});
 
   /// Perform semantic checking on a newly-created variable
   /// declaration.
@@ -5325,9 +5327,8 @@ public:
   bool SetDelegatingInitializer(CXXConstructorDecl *Constructor,
                                 CXXCtorInitializer *Initializer);
 
-  bool SetCtorInitializers(
-      CXXConstructorDecl *Constructor, bool AnyErrors,
-      ArrayRef<CXXCtorInitializer *> Initializers = std::nullopt);
+  bool SetCtorInitializers(CXXConstructorDecl *Constructor, bool AnyErrors,
+                           ArrayRef<CXXCtorInitializer *> Initializers = {});
 
   /// MarkBaseAndMemberDestructorsReferenced - Given a record decl,
   /// mark all the non-trivial destructors of its members and bases as
@@ -6623,9 +6624,9 @@ public:
   /// \param SkipLocalVariables If true, don't mark local variables as
   /// 'referenced'.
   /// \param StopAt Subexpressions that we shouldn't recurse into.
-  void MarkDeclarationsReferencedInExpr(
-      Expr *E, bool SkipLocalVariables = false,
-      ArrayRef<const Expr *> StopAt = std::nullopt);
+  void MarkDeclarationsReferencedInExpr(Expr *E,
+                                        bool SkipLocalVariables = false,
+                                        ArrayRef<const Expr *> StopAt = {});
 
   /// Try to convert an expression \p E to type \p Ty. Returns the result of the
   /// conversion.
@@ -6696,7 +6697,7 @@ public:
   DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
                       CorrectionCandidateCallback &CCC,
                       TemplateArgumentListInfo *ExplicitTemplateArgs = nullptr,
-                      ArrayRef<Expr *> Args = std::nullopt,
+                      ArrayRef<Expr *> Args = {},
                       DeclContext *LookupCtx = nullptr,
                       TypoExpr **Out = nullptr);
 
@@ -10124,15 +10125,17 @@ public:
   /// \param PartialOverloading true if we are performing "partial" overloading
   /// based on an incomplete set of function arguments. This feature is used by
   /// code completion.
-  void AddOverloadCandidate(
-      FunctionDecl *Function, DeclAccessPair FoundDecl, ArrayRef<Expr *> Args,
-      OverloadCandidateSet &CandidateSet, bool SuppressUserConversions = false,
-      bool PartialOverloading = false, bool AllowExplicit = true,
-      bool AllowExplicitConversion = false,
-      ADLCallKind IsADLCandidate = ADLCallKind::NotADL,
-      ConversionSequenceList EarlyConversions = std::nullopt,
-      OverloadCandidateParamOrder PO = {},
-      bool AggregateCandidateDeduction = false);
+  void AddOverloadCandidate(FunctionDecl *Function, DeclAccessPair FoundDecl,
+                            ArrayRef<Expr *> Args,
+                            OverloadCandidateSet &CandidateSet,
+                            bool SuppressUserConversions = false,
+                            bool PartialOverloading = false,
+                            bool AllowExplicit = true,
+                            bool AllowExplicitConversion = false,
+                            ADLCallKind IsADLCandidate = ADLCallKind::NotADL,
+                            ConversionSequenceList EarlyConversions = {},
+                            OverloadCandidateParamOrder PO = {},
+                            bool AggregateCandidateDeduction = false);
 
   /// Add all of the function declarations in the given function set to
   /// the overload candidate set.
@@ -10159,15 +10162,15 @@ public:
   /// both @c a1 and @c a2. If @p SuppressUserConversions, then don't
   /// allow user-defined conversions via constructors or conversion
   /// operators.
-  void
-  AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl,
-                     CXXRecordDecl *ActingContext, QualType ObjectType,
-                     Expr::Classification ObjectClassification,
-                     ArrayRef<Expr *> Args, OverloadCandidateSet &CandidateSet,
-                     bool SuppressUserConversions = false,
-                     bool PartialOverloading = false,
-                     ConversionSequenceList EarlyConversions = std::nullopt,
-                     OverloadCandidateParamOrder PO = {});
+  void AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl,
+                          CXXRecordDecl *ActingContext, QualType ObjectType,
+                          Expr::Classification ObjectClassification,
+                          ArrayRef<Expr *> Args,
+                          OverloadCandidateSet &CandidateSet,
+                          bool SuppressUserConversions = false,
+                          bool PartialOverloading = false,
+                          ConversionSequenceList EarlyConversions = {},
+                          OverloadCandidateParamOrder PO = {});
 
   /// Add a C++ member function template as a candidate to the candidate
   /// set, using template argument deduction to produce an appropriate member
@@ -12968,12 +12971,13 @@ public:
     bool CheckInstantiationDepth(SourceLocation PointOfInstantiation,
                                  SourceRange InstantiationRange);
 
-    InstantiatingTemplate(
-        Sema &SemaRef, CodeSynthesisContext::SynthesisKind Kind,
-        SourceLocation PointOfInstantiation, SourceRange InstantiationRange,
-        Decl *Entity, NamedDecl *Template = nullptr,
-        ArrayRef<TemplateArgument> TemplateArgs = std::nullopt,
-        sema::TemplateDeductionInfo *DeductionInfo = nullptr);
+    InstantiatingTemplate(Sema &SemaRef,
+                          CodeSynthesisContext::SynthesisKind Kind,
+                          SourceLocation PointOfInstantiation,
+                          SourceRange InstantiationRange, Decl *Entity,
+                          NamedDecl *Template = nullptr,
+                          ArrayRef<TemplateArgument> TemplateArgs = {},
+                          sema::TemplateDeductionInfo *DeductionInfo = nullptr);
 
     InstantiatingTemplate(const InstantiatingTemplate &) = delete;
 
diff --git a/clang/include/clang/Sema/SemaObjC.h b/clang/include/clang/Sema/SemaObjC.h
index 9367c680953f..1332eb4f4d42 100644
--- a/clang/include/clang/Sema/SemaObjC.h
+++ b/clang/include/clang/Sema/SemaObjC.h
@@ -336,8 +336,8 @@ public:
                                         ObjCInterfaceDecl *ID);
 
   Decl *ActOnAtEnd(Scope *S, SourceRange AtEnd,
-                   ArrayRef<Decl *> allMethods = std::nullopt,
-                   ArrayRef<DeclGroupPtrTy> allTUVars = std::nullopt);
+                   ArrayRef<Decl *> allMethods = {},
+                   ArrayRef<DeclGroupPtrTy> allTUVars = {});
 
   struct ObjCArgInfo {
     IdentifierInfo *Name;
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index d6e56f85e2d5..a34551f6b5a1 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -274,7 +274,7 @@ public:
              "Parsed clause kind does not have a queue id expr list");
 
       if (std::holds_alternative<std::monostate>(Details))
-        return ArrayRef<Expr *>{std::nullopt};
+        return ArrayRef<Expr *>();
 
       return std::get<WaitDetails>(Details).QueueIdExprs;
     }
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 80ad30b0f99e..1bf71b13cbb0 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -1196,21 +1196,21 @@ public:
       SourceLocation ModifierLoc, SourceLocation ColonLoc,
       SourceLocation EndLoc, CXXScopeSpec &ReductionIdScopeSpec,
       const DeclarationNameInfo &ReductionId,
-      ArrayRef<Expr *> UnresolvedReductions = std::nullopt);
+      ArrayRef<Expr *> UnresolvedReductions = {});
   /// Called on well-formed 'task_reduction' clause.
   OMPClause *ActOnOpenMPTaskReductionClause(
       ArrayRef<Expr *> VarList, SourceLocation StartLoc,
       SourceLocation LParenLoc, SourceLocation ColonLoc, SourceLocation EndLoc,
       CXXScopeSpec &ReductionIdScopeSpec,
       const DeclarationNameInfo &ReductionId,
-      ArrayRef<Expr *> UnresolvedReductions = std::nullopt);
+      ArrayRef<Expr *> UnresolvedReductions = {});
   /// Called on well-formed 'in_reduction' clause.
   OMPClause *ActOnOpenMPInReductionClause(
       ArrayRef<Expr *> VarList, SourceLocation StartLoc,
       SourceLocation LParenLoc, SourceLocation ColonLoc, SourceLocation EndLoc,
       CXXScopeSpec &ReductionIdScopeSpec,
       const DeclarationNameInfo &ReductionId,
-      ArrayRef<Expr *> UnresolvedReductions = std::nullopt);
+      ArrayRef<Expr *> UnresolvedReductions = {});
   /// Called on well-formed 'linear' clause.
   OMPClause *ActOnOpenMPLinearClause(
       ArrayRef<Expr *> VarList, Expr *Step, SourceLocation StartLoc,
@@ -1263,7 +1263,7 @@ public:
       OpenMPMapClauseKind MapType, bool IsMapTypeImplicit,
       SourceLocation MapLoc, SourceLocation ColonLoc, ArrayRef<Expr *> VarList,
       const OMPVarListLocTy &Locs, bool NoDiagnose = false,
-      ArrayRef<Expr *> UnresolvedMappers = std::nullopt);
+      ArrayRef<Expr *> UnresolvedMappers = {});
   /// Called on well-formed 'num_teams' clause.
   OMPClause *ActOnOpenMPNumTeamsClause(ArrayRef<Expr *> VarList,
                                        SourceLocation StartLoc,
@@ -1295,7 +1295,7 @@ public:
                       CXXScopeSpec &MapperIdScopeSpec,
                       DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
                       ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
-                      ArrayRef<Expr *> UnresolvedMappers = std::nullopt);
+                      ArrayRef<Expr *> UnresolvedMappers = {});
   /// Called on well-formed 'from' clause.
   OMPClause *
   ActOnOpenMPFromClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
@@ -1303,7 +1303,7 @@ public:
                         CXXScopeSpec &MapperIdScopeSpec,
                         DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
                         ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
-                        ArrayRef<Expr *> UnresolvedMappers = std::nullopt);
+                        ArrayRef<Expr *> UnresolvedMappers = {});
   /// Called on well-formed 'use_device_ptr' clause.
   OMPClause *ActOnOpenMPUseDevicePtrClause(ArrayRef<Expr *> VarList,
                                            const OMPVarListLocTy &Locs);
diff --git a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
index ead96ce6891c..8974342166fa 100644
--- a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
+++ b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
@@ -644,14 +644,14 @@ public:
   void EmitBasicReport(const Decl *DeclWithIssue, const CheckerBase *Checker,
                        StringRef BugName, StringRef BugCategory,
                        StringRef BugStr, PathDiagnosticLocation Loc,
-                       ArrayRef<SourceRange> Ranges = std::nullopt,
-                       ArrayRef<FixItHint> Fixits = std::nullopt);
+                       ArrayRef<SourceRange> Ranges = {},
+                       ArrayRef<FixItHint> Fixits = {});
 
   void EmitBasicReport(const Decl *DeclWithIssue, CheckerNameRef CheckerName,
                        StringRef BugName, StringRef BugCategory,
                        StringRef BugStr, PathDiagnosticLocation Loc,
-                       ArrayRef<SourceRange> Ranges = std::nullopt,
-                       ArrayRef<FixItHint> Fixits = std::nullopt);
+                       ArrayRef<SourceRange> Ranges = {},
+                       ArrayRef<FixItHint> Fixits = {});
 
 private:
   llvm::StringMap<std::unique_ptr<BugType>> StrBugTypes;
diff --git a/clang/lib/ARCMigrate/Internals.h b/clang/lib/ARCMigrate/Internals.h
index d790c7c02189..de6ebdce1bea 100644
--- a/clang/lib/ARCMigrate/Internals.h
+++ b/clang/lib/ARCMigrate/Internals.h
@@ -74,7 +74,7 @@ public:
 
   bool clearDiagnostic(ArrayRef<unsigned> IDs, SourceRange range);
   bool clearAllDiagnostics(SourceRange range) {
-    return clearDiagnostic(std::nullopt, range);
+    return clearDiagnostic({}, range);
   }
   bool clearDiagnostic(unsigned ID1, unsigned ID2, SourceRange range) {
     unsigned IDs[] = { ID1, ID2 };
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index a4e8d95035b9..69892bda42b2 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -1052,7 +1052,7 @@ ASTContext::getModulesWithMergedDefinition(const NamedDecl *Def) {
   auto MergedIt =
       MergedDefModules.find(cast<NamedDecl>(Def->getCanonicalDecl()));
   if (MergedIt == MergedDefModules.end())
-    return std::nullopt;
+    return {};
   return MergedIt->second;
 }
 
@@ -1111,7 +1111,7 @@ void ASTContext::addLazyModuleInitializers(Module *M,
 ArrayRef<Decl *> ASTContext::getModuleInitializers(Module *M) {
   auto It = ModuleInitializers.find(M);
   if (It == ModuleInitializers.end())
-    return std::nullopt;
+    return {};
 
   auto *Inits = It->second;
   Inits->resolve(*this);
diff --git a/clang/lib/AST/Comment.cpp b/clang/lib/AST/Comment.cpp
index cce8b12170f2..d022c5c845ae 100644
--- a/clang/lib/AST/Comment.cpp
+++ b/clang/lib/AST/Comment.cpp
@@ -210,7 +210,7 @@ void DeclInfo::fill() {
   IsInstanceMethod = false;
   IsClassMethod = false;
   IsVariadic = false;
-  ParamVars = std::nullopt;
+  ParamVars = {};
   TemplateParameters = nullptr;
 
   if (!CommentDecl) {
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index d5e5bb27ceba..61508fe886ef 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -499,7 +499,7 @@ BlockCommandComment *Parser::parseBlockCommand() {
   if (isTokBlockCommand()) {
     // Block command ahead.  We can't nest block commands, so pretend that this
     // command has an empty argument.
-    ParagraphComment *Paragraph = S.actOnParagraphComment(std::nullopt);
+    ParagraphComment *Paragraph = S.actOnParagraphComment({});
     if (PC) {
       S.actOnParamCommandFinish(PC, Paragraph);
       return PC;
@@ -547,7 +547,7 @@ BlockCommandComment *Parser::parseBlockCommand() {
 
   ParagraphComment *Paragraph;
   if (EmptyParagraph)
-    Paragraph = S.actOnParagraphComment(std::nullopt);
+    Paragraph = S.actOnParagraphComment({});
   else {
     BlockContentComment *Block = parseParagraphOrBlockCommand();
     // Since we have checked for a block command, we should have parsed a
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 8321cee0e0bc..86913763ef9f 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -5506,9 +5506,8 @@ IndirectFieldDecl::Create(ASTContext &C, DeclContext *DC, SourceLocation L,
 
 IndirectFieldDecl *IndirectFieldDecl::CreateDeserialized(ASTContext &C,
                                                          GlobalDeclID ID) {
-  return new (C, ID)
-      IndirectFieldDecl(C, nullptr, SourceLocation(), DeclarationName(),
-                        QualType(), std::nullopt);
+  return new (C, ID) IndirectFieldDecl(C, nullptr, SourceLocation(),
+                                       DeclarationName(), QualType(), {});
 }
 
 SourceRange EnumConstantDecl::getSourceRange() const {
@@ -5748,7 +5747,7 @@ ImportDecl *ImportDecl::CreateDeserialized(ASTContext &C, GlobalDeclID ID,
 
 ArrayRef<SourceLocation> ImportDecl::getIdentifierLocs() const {
   if (!isImportComplete())
-    return std::nullopt;
+    return {};
 
   const auto *StoredLocs = getTrailingObjects<SourceLocation>();
   return llvm::ArrayRef(StoredLocs,
diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp
index 17ebee0d6b22..db0ea62a2323 100644
--- a/clang/lib/AST/DeclCXX.cpp
+++ b/clang/lib/AST/DeclCXX.cpp
@@ -3293,8 +3293,7 @@ UsingPackDecl *UsingPackDecl::Create(ASTContext &C, DeclContext *DC,
 UsingPackDecl *UsingPackDecl::CreateDeserialized(ASTContext &C, GlobalDeclID ID,
                                                  unsigned NumExpansions) {
   size_t Extra = additionalSizeToAlloc<NamedDecl *>(NumExpansions);
-  auto *Result =
-      new (C, ID, Extra) UsingPackDecl(nullptr, nullptr, std::nullopt);
+  auto *Result = new (C, ID, Extra) UsingPackDecl(nullptr, nullptr, {});
   Result->NumExpansions = NumExpansions;
   auto *Trail = Result->getTrailingObjects<NamedDecl *>();
   for (unsigned I = 0; I != NumExpansions; ++I)
@@ -3443,7 +3442,7 @@ DecompositionDecl *DecompositionDecl::CreateDeserialized(ASTContext &C,
   size_t Extra = additionalSizeToAlloc<BindingDecl *>(NumBindings);
   auto *Result = new (C, ID, Extra)
       DecompositionDecl(C, nullptr, SourceLocation(), SourceLocation(),
-                        QualType(), nullptr, StorageClass(), std::nullopt);
+                        QualType(), nullptr, StorageClass(), {});
   // Set up and clean out the bindings array.
   Result->NumBindings = NumBindings;
   auto *Trail = Result->getTrailingObjects<BindingDecl *>();
diff --git a/clang/lib/AST/DeclObjC.cpp b/clang/lib/AST/DeclObjC.cpp
index 83062b0e6887..ae8a31076926 100644
--- a/clang/lib/AST/DeclObjC.cpp
+++ b/clang/lib/AST/DeclObjC.cpp
@@ -947,12 +947,12 @@ void ObjCMethodDecl::setMethodParams(ASTContext &C,
   assert((!SelLocs.empty() || isImplicit()) &&
          "No selector locs for non-implicit method");
   if (isImplicit())
-    return setParamsAndSelLocs(C, Params, std::nullopt);
+    return setParamsAndSelLocs(C, Params, {});
 
   setSelLocsKind(hasStandardSelectorLocs(getSelector(), SelLocs, Params,
                                         DeclEndLoc));
   if (getSelLocsKind() != SelLoc_NonStandard)
-    return setParamsAndSelLocs(C, Params, std::nullopt);
+    return setParamsAndSelLocs(C, Params, {});
 
   setParamsAndSelLocs(C, Params, SelLocs);
 }
diff --git a/clang/lib/AST/DeclOpenMP.cpp b/clang/lib/AST/DeclOpenMP.cpp
index 81ca48e60942..32c82f614d6f 100644
--- a/clang/lib/AST/DeclOpenMP.cpp
+++ b/clang/lib/AST/DeclOpenMP.cpp
@@ -30,7 +30,7 @@ OMPThreadPrivateDecl *OMPThreadPrivateDecl::Create(ASTContext &C,
                                                    SourceLocation L,
                                                    ArrayRef<Expr *> VL) {
   auto *D = OMPDeclarativeDirective::createDirective<OMPThreadPrivateDecl>(
-      C, DC, std::nullopt, VL.size(), L);
+      C, DC, {}, VL.size(), L);
   D->setVars(VL);
   return D;
 }
diff --git a/clang/lib/AST/DeclTemplate.cpp b/clang/lib/AST/DeclTemplate.cpp
index d2d8907b884e..4a506b7be456 100644
--- a/clang/lib/AST/DeclTemplate.cpp
+++ b/clang/lib/AST/DeclTemplate.cpp
@@ -794,8 +794,7 @@ NonTypeTemplateParmDecl::CreateDeserialized(ASTContext &C, GlobalDeclID ID,
            additionalSizeToAlloc<std::pair<QualType, TypeSourceInfo *>, Expr *>(
                NumExpandedTypes, HasTypeConstraint ? 1 : 0))
           NonTypeTemplateParmDecl(nullptr, SourceLocation(), SourceLocation(),
-                                  0, 0, nullptr, QualType(), nullptr,
-                                  std::nullopt, std::nullopt);
+                                  0, 0, nullptr, QualType(), nullptr, {}, {});
   NTTP->NumExpandedTypes = NumExpandedTypes;
   return NTTP;
 }
@@ -870,7 +869,7 @@ TemplateTemplateParmDecl::CreateDeserialized(ASTContext &C, GlobalDeclID ID,
   auto *TTP =
       new (C, ID, additionalSizeToAlloc<TemplateParameterList *>(NumExpansions))
           TemplateTemplateParmDecl(nullptr, SourceLocation(), 0, 0, nullptr,
-                                   false, nullptr, std::nullopt);
+                                   false, nullptr, {});
   TTP->NumExpandedParams = NumExpansions;
   return TTP;
 }
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 66db6263cb1b..bf2c1b92fa6b 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -4720,8 +4720,7 @@ DesignatedInitUpdateExpr::DesignatedInitUpdateExpr(const ASTContext &C,
            OK_Ordinary) {
   BaseAndUpdaterExprs[0] = baseExpr;
 
-  InitListExpr *ILE =
-      new (C) InitListExpr(C, lBraceLoc, std::nullopt, rBraceLoc);
+  InitListExpr *ILE = new (C) InitListExpr(C, lBraceLoc, {}, rBraceLoc);
   ILE->setType(baseExpr->getType());
   BaseAndUpdaterExprs[1] = ILE;
 
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index 3931fcaa3529..dbc161347025 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -397,7 +397,7 @@ public:
   void mangleBits(llvm::APInt Number);
   void mangleTagTypeKind(TagTypeKind TK);
   void mangleArtificialTagType(TagTypeKind TK, StringRef UnqualifiedName,
-                               ArrayRef<StringRef> NestedNames = std::nullopt);
+                               ArrayRef<StringRef> NestedNames = {});
   void mangleAddressSpaceType(QualType T, Qualifiers Quals, SourceRange Range);
   void mangleType(QualType T, SourceRange Range,
                   QualifierMangleMode QMM = QMM_Mangle);
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 5adfd9195746..4f441c2f92dc 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -452,8 +452,7 @@ OMPReverseDirective::Create(const ASTContext &C, SourceLocation StartLoc,
                             SourceLocation EndLoc, Stmt *AssociatedStmt,
                             Stmt *TransformedStmt, Stmt *PreInits) {
   OMPReverseDirective *Dir = createDirective<OMPReverseDirective>(
-      C, std::nullopt, AssociatedStmt, TransformedStmtOffset + 1, StartLoc,
-      EndLoc);
+      C, {}, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc);
   Dir->setTransformedStmt(TransformedStmt);
   Dir->setPreInits(PreInits);
   return Dir;
@@ -555,7 +554,7 @@ OMPSectionDirective *OMPSectionDirective::Create(const ASTContext &C,
                                                  Stmt *AssociatedStmt,
                                                  bool HasCancel) {
   auto *Dir =
-      createDirective<OMPSectionDirective>(C, std::nullopt, AssociatedStmt,
+      createDirective<OMPSectionDirective>(C, {}, AssociatedStmt,
                                            /*NumChildren=*/0, StartLoc, EndLoc);
   Dir->setHasCancel(HasCancel);
   return Dir;
@@ -605,7 +604,7 @@ OMPMasterDirective *OMPMasterDirective::Create(const ASTContext &C,
                                                SourceLocation StartLoc,
                                                SourceLocation EndLoc,
                                                Stmt *AssociatedStmt) {
-  return createDirective<OMPMasterDirective>(C, std::nullopt, AssociatedStmt,
+  return createDirective<OMPMasterDirective>(C, {}, AssociatedStmt,
                                              /*NumChildren=*/0, StartLoc,
                                              EndLoc);
 }
diff --git a/clang/lib/Analysis/BodyFarm.cpp b/clang/lib/Analysis/BodyFarm.cpp
index 127e843d4ead..c5f35b35ad35 100644
--- a/clang/lib/Analysis/BodyFarm.cpp
+++ b/clang/lib/Analysis/BodyFarm.cpp
@@ -542,7 +542,7 @@ static Stmt *create_dispatch_once(ASTContext &C, const FunctionDecl *D) {
   CallExpr *CE = CallExpr::Create(
       /*ASTContext=*/C,
       /*StmtClass=*/M.makeLvalueToRvalue(/*Expr=*/Block),
-      /*Args=*/std::nullopt,
+      /*Args=*/{},
       /*QualType=*/C.VoidTy,
       /*ExprValueType=*/VK_PRValue,
       /*SourceLocation=*/SourceLocation(), FPOptionsOverride());
@@ -610,7 +610,7 @@ static Stmt *create_dispatch_sync(ASTContext &C, const FunctionDecl *D) {
   ASTMaker M(C);
   DeclRefExpr *DR = M.makeDeclRefExpr(PV);
   ImplicitCastExpr *ICE = M.makeLvalueToRvalue(DR, Ty);
-  CallExpr *CE = CallExpr::Create(C, ICE, std::nullopt, C.VoidTy, VK_PRValue,
+  CallExpr *CE = CallExpr::Create(C, ICE, {}, C.VoidTy, VK_PRValue,
                                   SourceLocation(), FPOptionsOverride());
   return CE;
 }
diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index 5e0ec9ecc92e..fad2f52e89ef 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -3605,7 +3605,7 @@ public:
     auto It = VarGrpMap.find(Var);
 
     if (It == VarGrpMap.end())
-      return std::nullopt;
+      return {};
     return Groups[It->second];
   }
 
diff --git a/clang/lib/Basic/Targets/AMDGPU.h b/clang/lib/Basic/Targets/AMDGPU.h
index 94d9ba93ed22..6edd3474d4ed 100644
--- a/clang/lib/Basic/Targets/AMDGPU.h
+++ b/clang/lib/Basic/Targets/AMDGPU.h
@@ -122,7 +122,7 @@ public:
   ArrayRef<const char *> getGCCRegNames() const override;
 
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override {
-    return std::nullopt;
+    return {};
   }
 
   /// Accepted register names: (n, m is unsigned integer, n < m)
diff --git a/clang/lib/Basic/Targets/ARC.h b/clang/lib/Basic/Targets/ARC.h
index fcbfdd6eec58..7f3d0aa15ab8 100644
--- a/clang/lib/Basic/Targets/ARC.h
+++ b/clang/lib/Basic/Targets/ARC.h
@@ -40,9 +40,7 @@ public:
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override {
-    return std::nullopt;
-  }
+  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::VoidPtrBuiltinVaList;
@@ -60,7 +58,7 @@ public:
   }
 
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override {
-    return std::nullopt;
+    return {};
   }
 
   bool validateAsmConstraint(const char *&Name,
diff --git a/clang/lib/Basic/Targets/AVR.h b/clang/lib/Basic/Targets/AVR.h
index 0a2f51747f8a..df1f8d171efb 100644
--- a/clang/lib/Basic/Targets/AVR.h
+++ b/clang/lib/Basic/Targets/AVR.h
@@ -63,9 +63,7 @@ public:
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override {
-    return std::nullopt;
-  }
+  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
 
   bool allowsLargerPreferedTypeAlignment() const override { return false; }
 
@@ -84,7 +82,7 @@ public:
   }
 
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override {
-    return std::nullopt;
+    return {};
   }
 
   ArrayRef<TargetInfo::AddlRegName> getGCCAddlRegNames() const override {
diff --git a/clang/lib/Basic/Targets/BPF.h b/clang/lib/Basic/Targets/BPF.h
index d19b37dd4df7..27a4b5f31497 100644
--- a/clang/lib/Basic/Targets/BPF.h
+++ b/clang/lib/Basic/Targets/BPF.h
@@ -67,9 +67,7 @@ public:
   }
 
   bool isValidGCCRegisterName(StringRef Name) const override { return true; }
-  ArrayRef<const char *> getGCCRegNames() const override {
-    return std::nullopt;
-  }
+  ArrayRef<const char *> getGCCRegNames() const override { return {}; }
 
   bool validateAsmConstraint(const char *&Name,
                              TargetInfo::ConstraintInfo &Info) const override {
@@ -86,7 +84,7 @@ public:
   }
 
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override {
-    return std::nullopt;
+    return {};
   }
 
   bool allowDebugInfoForExternalRef() const override { return true; }
diff --git a/clang/lib/Basic/Targets/DirectX.h b/clang/lib/Basic/Targets/DirectX.h
index 19b61252409b..ab22d1281a4d 100644
--- a/clang/lib/Basic/Targets/DirectX.h
+++ b/clang/lib/Basic/Targets/DirectX.h
@@ -72,15 +72,11 @@ public:
     return Feature == "directx";
   }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override {
-    return std::nullopt;
-  }
+  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
 
   std::string_view getClobbers() const override { return ""; }
 
-  ArrayRef<const char *> getGCCRegNames() const override {
-    return std::nullopt;
-  }
+  ArrayRef<const char *> getGCCRegNames() const override { return {}; }
 
   bool validateAsmConstraint(const char *&Name,
                              TargetInfo::ConstraintInfo &info) const override {
@@ -88,7 +84,7 @@ public:
   }
 
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override {
-    return std::nullopt;
+    return {};
   }
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
diff --git a/clang/lib/Basic/Targets/Lanai.h b/clang/lib/Basic/Targets/Lanai.h
index 144cbc7de989..f7e439c7c9e1 100644
--- a/clang/lib/Basic/Targets/Lanai.h
+++ b/clang/lib/Basic/Targets/Lanai.h
@@ -78,9 +78,7 @@ public:
     return TargetInfo::VoidPtrBuiltinVaList;
   }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override {
-    return std::nullopt;
-  }
+  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
 
   bool validateAsmConstraint(const char *&Name,
                              TargetInfo::ConstraintInfo &info) const override {
diff --git a/clang/lib/Basic/Targets/M68k.cpp b/clang/lib/Basic/Targets/M68k.cpp
index 8b8bf97d6f99..b5b29fd86756 100644
--- a/clang/lib/Basic/Targets/M68k.cpp
+++ b/clang/lib/Basic/Targets/M68k.cpp
@@ -117,7 +117,7 @@ void M68kTargetInfo::getTargetDefines(const LangOptions &Opts,
 
 ArrayRef<Builtin::Info> M68kTargetInfo::getTargetBuiltins() const {
   // FIXME: Implement.
-  return std::nullopt;
+  return {};
 }
 
 bool M68kTargetInfo::hasFeature(StringRef Feature) const {
diff --git a/clang/lib/Basic/Targets/MSP430.h b/clang/lib/Basic/Targets/MSP430.h
index 25639b8c1e0a..2266ada25c1d 100644
--- a/clang/lib/Basic/Targets/MSP430.h
+++ b/clang/lib/Basic/Targets/MSP430.h
@@ -52,7 +52,7 @@ public:
 
   ArrayRef<Builtin::Info> getTargetBuiltins() const override {
     // FIXME: Implement.
-    return std::nullopt;
+    return {};
   }
 
   bool allowsLargerPreferedTypeAlignment() const override { return false; }
diff --git a/clang/lib/Basic/Targets/NVPTX.h b/clang/lib/Basic/Targets/NVPTX.h
index 1ef20ce73391..165b28a60fb2 100644
--- a/clang/lib/Basic/Targets/NVPTX.h
+++ b/clang/lib/Basic/Targets/NVPTX.h
@@ -93,7 +93,7 @@ public:
 
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override {
     // No aliases.
-    return std::nullopt;
+    return {};
   }
 
   bool validateAsmConstraint(const char *&Name,
diff --git a/clang/lib/Basic/Targets/PNaCl.cpp b/clang/lib/Basic/Targets/PNaCl.cpp
index 51b6452b0c20..c4adfbefb9c7 100644
--- a/clang/lib/Basic/Targets/PNaCl.cpp
+++ b/clang/lib/Basic/Targets/PNaCl.cpp
@@ -16,12 +16,10 @@
 using namespace clang;
 using namespace clang::targets;
 
-ArrayRef<const char *> PNaClTargetInfo::getGCCRegNames() const {
-  return std::nullopt;
-}
+ArrayRef<const char *> PNaClTargetInfo::getGCCRegNames() const { return {}; }
 
 ArrayRef<TargetInfo::GCCRegAlias> PNaClTargetInfo::getGCCRegAliases() const {
-  return std::nullopt;
+  return {};
 }
 
 void PNaClTargetInfo::getArchDefines(const LangOptions &Opts,
diff --git a/clang/lib/Basic/Targets/PNaCl.h b/clang/lib/Basic/Targets/PNaCl.h
index 595c4d83b1d1..7e0e10aa362d 100644
--- a/clang/lib/Basic/Targets/PNaCl.h
+++ b/clang/lib/Basic/Targets/PNaCl.h
@@ -52,9 +52,7 @@ public:
     return Feature == "pnacl";
   }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override {
-    return std::nullopt;
-  }
+  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::PNaClABIBuiltinVaList;
diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h
index cc79562de287..30c7ac8d9c03 100644
--- a/clang/lib/Basic/Targets/SPIR.h
+++ b/clang/lib/Basic/Targets/SPIR.h
@@ -159,15 +159,11 @@ public:
   // memcpy as per section 3 of the SPIR spec.
   bool useFP16ConversionIntrinsics() const override { return false; }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override {
-    return std::nullopt;
-  }
+  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
 
   std::string_view getClobbers() const override { return ""; }
 
-  ArrayRef<const char *> getGCCRegNames() const override {
-    return std::nullopt;
-  }
+  ArrayRef<const char *> getGCCRegNames() const override { return {}; }
 
   bool validateAsmConstraint(const char *&Name,
                              TargetInfo::ConstraintInfo &info) const override {
@@ -175,7 +171,7 @@ public:
   }
 
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override {
-    return std::nullopt;
+    return {};
   }
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
diff --git a/clang/lib/Basic/Targets/Sparc.h b/clang/lib/Basic/Targets/Sparc.h
index ee0d3e2b4329..9c529a5bc5e7 100644
--- a/clang/lib/Basic/Targets/Sparc.h
+++ b/clang/lib/Basic/Targets/Sparc.h
@@ -50,7 +50,7 @@ public:
 
   ArrayRef<Builtin::Info> getTargetBuiltins() const override {
     // FIXME: Implement!
-    return std::nullopt;
+    return {};
   }
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::VoidPtrBuiltinVaList;
diff --git a/clang/lib/Basic/Targets/SystemZ.h b/clang/lib/Basic/Targets/SystemZ.h
index f05ea473017b..ef9a07033a6e 100644
--- a/clang/lib/Basic/Targets/SystemZ.h
+++ b/clang/lib/Basic/Targets/SystemZ.h
@@ -105,7 +105,7 @@ public:
 
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override {
     // No aliases.
-    return std::nullopt;
+    return {};
   }
 
   ArrayRef<TargetInfo::AddlRegName> getGCCAddlRegNames() const override;
diff --git a/clang/lib/Basic/Targets/TCE.h b/clang/lib/Basic/Targets/TCE.h
index dcf684fe6dbc..d6280b02f07b 100644
--- a/clang/lib/Basic/Targets/TCE.h
+++ b/clang/lib/Basic/Targets/TCE.h
@@ -95,9 +95,7 @@ public:
 
   bool hasFeature(StringRef Feature) const override { return Feature == "tce"; }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override {
-    return std::nullopt;
-  }
+  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
 
   std::string_view getClobbers() const override { return ""; }
 
@@ -105,9 +103,7 @@ public:
     return TargetInfo::VoidPtrBuiltinVaList;
   }
 
-  ArrayRef<const char *> getGCCRegNames() const override {
-    return std::nullopt;
-  }
+  ArrayRef<const char *> getGCCRegNames() const override { return {}; }
 
   bool validateAsmConstraint(const char *&Name,
                              TargetInfo::ConstraintInfo &info) const override {
@@ -115,7 +111,7 @@ public:
   }
 
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override {
-    return std::nullopt;
+    return {};
   }
 };
 
diff --git a/clang/lib/Basic/Targets/WebAssembly.h b/clang/lib/Basic/Targets/WebAssembly.h
index c8e93c79bb65..6c2fe8049ff4 100644
--- a/clang/lib/Basic/Targets/WebAssembly.h
+++ b/clang/lib/Basic/Targets/WebAssembly.h
@@ -124,10 +124,10 @@ private:
     return VoidPtrBuiltinVaList;
   }
 
-  ArrayRef<const char *> getGCCRegNames() const final { return std::nullopt; }
+  ArrayRef<const char *> getGCCRegNames() const final { return {}; }
 
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const final {
-    return std::nullopt;
+    return {};
   }
 
   bool validateAsmConstraint(const char *&Name,
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index a99ae62984c7..2affb659a160 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -210,7 +210,7 @@ public:
   ArrayRef<const char *> getGCCRegNames() const override;
 
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override {
-    return std::nullopt;
+    return {};
   }
 
   ArrayRef<TargetInfo::AddlRegName> getGCCAddlRegNames() const override;
diff --git a/clang/lib/Basic/Targets/XCore.h b/clang/lib/Basic/Targets/XCore.h
index a58d3e8acf47..84fd59d1a71e 100644
--- a/clang/lib/Basic/Targets/XCore.h
+++ b/clang/lib/Basic/Targets/XCore.h
@@ -60,7 +60,7 @@ public:
   }
 
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override {
-    return std::nullopt;
+    return {};
   }
 
   bool validateAsmConstraint(const char *&Name,
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 3f28b7f26c36..e2d03eff8ab4 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2024,7 +2024,7 @@ Value *CodeGenFunction::EmitCheckedArgForBuiltin(const Expr *E,
             SanitizerHandler::InvalidBuiltin,
             {EmitCheckSourceLocation(E->getExprLoc()),
              llvm::ConstantInt::get(Builder.getInt8Ty(), Kind)},
-            std::nullopt);
+            {});
   return ArgValue;
 }
 
@@ -18530,9 +18530,9 @@ Value *EmitAMDGPUWorkGroupSize(CodeGenFunction &CGF, unsigned Index) {
       APInt(16, CGF.getTarget().getMaxOpenCLWorkGroupSize() + 1));
   LD->setMetadata(llvm::LLVMContext::MD_range, RNode);
   LD->setMetadata(llvm::LLVMContext::MD_noundef,
-                  llvm::MDNode::get(CGF.getLLVMContext(), std::nullopt));
+                  llvm::MDNode::get(CGF.getLLVMContext(), {}));
   LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
-                  llvm::MDNode::get(CGF.getLLVMContext(), std::nullopt));
+                  llvm::MDNode::get(CGF.getLLVMContext(), {}));
   return LD;
 }
 
@@ -18546,7 +18546,7 @@ Value *EmitAMDGPUGridSize(CodeGenFunction &CGF, unsigned Index) {
   auto *LD = CGF.Builder.CreateLoad(
       Address(GEP, CGF.Int32Ty, CharUnits::fromQuantity(4)));
   LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
-                  llvm::MDNode::get(CGF.getLLVMContext(), std::nullopt));
+                  llvm::MDNode::get(CGF.getLLVMContext(), {}));
   return LD;
 }
 } // namespace
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 4ae981e4013e..5f50ab6f34ff 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -118,8 +118,8 @@ CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
   // When translating an unprototyped function type, always use a
   // variadic type.
   return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
-                                 FnInfoOpts::None, std::nullopt,
-                                 FTNP->getExtInfo(), {}, RequiredArgs(0));
+                                 FnInfoOpts::None, {}, FTNP->getExtInfo(), {},
+                                 RequiredArgs(0));
 }
 
 static void addExtParameterInfosForCall(
@@ -473,7 +473,7 @@ CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) {
   // non-variadic type.
   if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
     return arrangeLLVMFunctionInfo(noProto->getReturnType(), FnInfoOpts::None,
-                                   std::nullopt, noProto->getExtInfo(), {},
+                                   {}, noProto->getExtInfo(), {},
                                    RequiredArgs::All);
   }
 
@@ -719,8 +719,8 @@ CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
 }
 
 const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
-  return arrangeLLVMFunctionInfo(getContext().VoidTy, FnInfoOpts::None,
-                                 std::nullopt, FunctionType::ExtInfo(), {},
+  return arrangeLLVMFunctionInfo(getContext().VoidTy, FnInfoOpts::None, {},
+                                 FunctionType::ExtInfo(), {},
                                  RequiredArgs::All);
 }
 
@@ -4423,7 +4423,7 @@ void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
       EmitCheckSourceLocation(ArgLoc), EmitCheckSourceLocation(AttrLoc),
       llvm::ConstantInt::get(Int32Ty, ArgNo + 1),
   };
-  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, std::nullopt);
+  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, {});
 }
 
 void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType,
@@ -4814,7 +4814,7 @@ CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
 /// runtime function.
 llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
                                                  const llvm::Twine &name) {
-  return EmitRuntimeCall(callee, std::nullopt, name);
+  return EmitRuntimeCall(callee, {}, name);
 }
 
 // Calls which may throw must have operand bundles indicating which funclet
@@ -4881,7 +4881,7 @@ void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
 llvm::CallBase *
 CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
                                          const Twine &name) {
-  return EmitRuntimeCallOrInvoke(callee, std::nullopt, name);
+  return EmitRuntimeCallOrInvoke(callee, {}, name);
 }
 
 /// Emits a call or invoke instruction to the given runtime function.
diff --git a/clang/lib/CodeGen/CGCleanup.cpp b/clang/lib/CodeGen/CGCleanup.cpp
index 5d253c92a38a..d9c0dbe45d6c 100644
--- a/clang/lib/CodeGen/CGCleanup.cpp
+++ b/clang/lib/CodeGen/CGCleanup.cpp
@@ -1329,8 +1329,7 @@ static void EmitSehScope(CodeGenFunction &CGF,
       CGF.getBundlesForFunclet(SehCppScope.getCallee());
   if (CGF.CurrentFuncletPad)
     BundleList.emplace_back("funclet", CGF.CurrentFuncletPad);
-  CGF.Builder.CreateInvoke(SehCppScope, Cont, InvokeDest, std::nullopt,
-                           BundleList);
+  CGF.Builder.CreateInvoke(SehCppScope, Cont, InvokeDest, {}, BundleList);
   CGF.EmitBlock(Cont);
 }
 
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 06015a9e541e..27bbbfc6f531 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -2642,7 +2642,7 @@ void CGDebugInfo::addHeapAllocSiteMetadata(llvm::CallBase *CI,
     return;
   llvm::MDNode *node;
   if (AllocatedTy->isVoidType())
-    node = llvm::MDNode::get(CGM.getLLVMContext(), std::nullopt);
+    node = llvm::MDNode::get(CGM.getLLVMContext(), {});
   else
     node = getOrCreateType(AllocatedTy, getOrCreateFile(Loc));
 
@@ -4324,8 +4324,7 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateFunctionType(const Decl *D,
              !CGM.getCodeGenOpts().EmitCodeView))
     // Create fake but valid subroutine type. Otherwise -verify would fail, and
     // subprogram DIE will miss DW_AT_decl_file and DW_AT_decl_line fields.
-    return DBuilder.createSubroutineType(
-        DBuilder.getOrCreateTypeArray(std::nullopt));
+    return DBuilder.createSubroutineType(DBuilder.getOrCreateTypeArray({}));
 
   if (const auto *Method = dyn_cast<CXXMethodDecl>(D))
     return getOrCreateMethodType(Method, F);
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 1e8ffb53b53a..e0ea65bcaf36 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -2042,7 +2042,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
     if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) {
       Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo);
       Load->setMetadata(llvm::LLVMContext::MD_noundef,
-                        llvm::MDNode::get(getLLVMContext(), std::nullopt));
+                        llvm::MDNode::get(getLLVMContext(), {}));
     }
 
   return EmitFromMemory(Load, Ty);
@@ -3375,9 +3375,9 @@ llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
   // Format the type name as if for a diagnostic, including quotes and
   // optionally an 'aka'.
   SmallString<32> Buffer;
-  CGM.getDiags().ConvertArgToString(
-      DiagnosticsEngine::ak_qualtype, (intptr_t)T.getAsOpaquePtr(), StringRef(),
-      StringRef(), std::nullopt, Buffer, std::nullopt);
+  CGM.getDiags().ConvertArgToString(DiagnosticsEngine::ak_qualtype,
+                                    (intptr_t)T.getAsOpaquePtr(), StringRef(),
+                                    StringRef(), {}, Buffer, {});
 
   if (IsBitInt) {
     // The Structure is: 0 to end the string, 32 bit unsigned integer in target
@@ -3884,7 +3884,7 @@ void CodeGenFunction::EmitUnreachable(SourceLocation Loc) {
     EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()),
                              SanitizerKind::Unreachable),
               SanitizerHandler::BuiltinUnreachable,
-              EmitCheckSourceLocation(Loc), std::nullopt);
+              EmitCheckSourceLocation(Loc), {});
   }
   Builder.CreateUnreachable();
 }
diff --git a/clang/lib/CodeGen/CGLoopInfo.cpp b/clang/lib/CodeGen/CGLoopInfo.cpp
index 6b886bd6b6d2..cdff7e50c4ee 100644
--- a/clang/lib/CodeGen/CGLoopInfo.cpp
+++ b/clang/lib/CodeGen/CGLoopInfo.cpp
@@ -506,7 +506,7 @@ LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
       Attrs.CodeAlign == 0 && !StartLoc && !EndLoc && !Attrs.MustProgress)
     return;
 
-  TempLoopID = MDNode::getTemporary(Header->getContext(), std::nullopt);
+  TempLoopID = MDNode::getTemporary(Header->getContext(), {});
 }
 
 void LoopInfo::finish() {
diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp
index a73a71f2d268..28da353ff2d7 100644
--- a/clang/lib/CodeGen/CGObjC.cpp
+++ b/clang/lib/CodeGen/CGObjC.cpp
@@ -140,7 +140,7 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E,
     llvm::Value *Ptr = EmitLoadOfScalar(LV, E->getBeginLoc());
     cast<llvm::LoadInst>(Ptr)->setMetadata(
         llvm::LLVMContext::MD_invariant_load,
-        llvm::MDNode::get(getLLVMContext(), std::nullopt));
+        llvm::MDNode::get(getLLVMContext(), {}));
     return Builder.CreateBitCast(Ptr, ConvertType(E->getType()));
   }
 
@@ -2319,7 +2319,7 @@ llvm::Value *CodeGenFunction::EmitARCRetainBlock(llvm::Value *value,
            CGM.getObjCEntrypoints().objc_retainBlock);
 
     call->setMetadata("clang.arc.copy_on_escape",
-                      llvm::MDNode::get(Builder.getContext(), std::nullopt));
+                      llvm::MDNode::get(Builder.getContext(), {}));
   }
 
   return result;
@@ -2361,8 +2361,7 @@ static void emitAutoreleasedReturnValueMarker(CodeGenFunction &CGF) {
 
   // Call the marker asm if we made one, which we do only at -O0.
   if (marker)
-    CGF.Builder.CreateCall(marker, std::nullopt,
-                           CGF.getBundlesForFunclet(marker));
+    CGF.Builder.CreateCall(marker, {}, CGF.getBundlesForFunclet(marker));
 }
 
 static llvm::Value *emitOptimizedARCReturnCall(llvm::Value *value,
@@ -2449,7 +2448,7 @@ void CodeGenFunction::EmitARCRelease(llvm::Value *value,
 
   if (precise == ARCImpreciseLifetime) {
     call->setMetadata("clang.imprecise_release",
-                      llvm::MDNode::get(Builder.getContext(), std::nullopt));
+                      llvm::MDNode::get(Builder.getContext(), {}));
   }
 }
 
@@ -2843,7 +2842,7 @@ void CodeGenFunction::EmitObjCRelease(llvm::Value *value,
 
   if (precise == ARCImpreciseLifetime) {
     call->setMetadata("clang.imprecise_release",
-                      llvm::MDNode::get(Builder.getContext(), std::nullopt));
+                      llvm::MDNode::get(Builder.getContext(), {}));
   }
 }
 
diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp
index 6280e9465ecb..7a07284f8a8a 100644
--- a/clang/lib/CodeGen/CGObjCGNU.cpp
+++ b/clang/lib/CodeGen/CGObjCGNU.cpp
@@ -69,7 +69,7 @@ public:
       FTy = llvm::FunctionType::get(RetTy, ArgTys, false);
     }
     else {
-      FTy = llvm::FunctionType::get(RetTy, std::nullopt, false);
+      FTy = llvm::FunctionType::get(RetTy, {}, false);
     }
   }
 
diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp
index 30f3911a8b03..1c16d273a553 100644
--- a/clang/lib/CodeGen/CGObjCMac.cpp
+++ b/clang/lib/CodeGen/CGObjCMac.cpp
@@ -7191,7 +7191,7 @@ CGObjCNonFragileABIMac::EmitIvarOffset(CodeGen::CodeGenFunction &CGF,
     if (IsIvarOffsetKnownIdempotent(CGF, Ivar))
       cast<llvm::LoadInst>(IvarOffsetValue)
           ->setMetadata(llvm::LLVMContext::MD_invariant_load,
-                        llvm::MDNode::get(VMContext, std::nullopt));
+                        llvm::MDNode::get(VMContext, {}));
   }
 
   // This could be 32bit int or 64bit integer depending on the architecture.
@@ -7589,7 +7589,7 @@ llvm::Value *CGObjCNonFragileABIMac::EmitSelector(CodeGenFunction &CGF,
 
   llvm::LoadInst* LI = CGF.Builder.CreateLoad(Addr);
   LI->setMetadata(llvm::LLVMContext::MD_invariant_load,
-                  llvm::MDNode::get(VMContext, std::nullopt));
+                  llvm::MDNode::get(VMContext, {}));
   return LI;
 }
 
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 5e9f89b18918..c0931e82d987 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -5075,7 +5075,7 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
   };
   RegionCodeGenTy RCG(CodeGen);
   CommonActionTy Action(
-      nullptr, std::nullopt,
+      nullptr, {},
       OMPBuilder.getOrCreateRuntimeFunction(
           CGM.getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait
                                       : OMPRTL___kmpc_end_reduce),
@@ -5197,7 +5197,7 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
         ThreadId,  // i32 <gtid>
         Lock       // kmp_critical_name *&<lock>
     };
-    CommonActionTy Action(nullptr, std::nullopt,
+    CommonActionTy Action(nullptr, {},
                           OMPBuilder.getOrCreateRuntimeFunction(
                               CGM.getModule(), OMPRTL___kmpc_end_reduce),
                           EndArgs);
@@ -6819,7 +6819,7 @@ private:
       const ValueDecl *Mapper = nullptr, bool ForDeviceAddr = false,
       const ValueDecl *BaseDecl = nullptr, const Expr *MapExpr = nullptr,
       ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
-          OverlappedElements = std::nullopt,
+          OverlappedElements = {},
       bool AreBothBasePtrAndPteeMapped = false) const {
     // The following summarizes what has to be generated for each map and the
     // types below. The generated information is expressed in this order:
@@ -7693,7 +7693,7 @@ private:
       // for map(to: lambda): using user specified map type.
       return getMapTypeBits(
           I->getSecond()->getMapType(), I->getSecond()->getMapTypeModifiers(),
-          /*MotionModifiers=*/std::nullopt, I->getSecond()->isImplicit(),
+          /*MotionModifiers=*/{}, I->getSecond()->isImplicit(),
           /*AddPtrFlag=*/false,
           /*AddIsTargetParamFlag=*/false,
           /*isNonContiguous=*/false);
@@ -7818,7 +7818,7 @@ private:
       for (const auto L : C->component_lists()) {
         const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr;
         InfoGen(std::get<0>(L), Kind, std::get<1>(L), C->getMapType(),
-                C->getMapTypeModifiers(), std::nullopt,
+                C->getMapTypeModifiers(), {},
                 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(L),
                 E);
         ++EI;
@@ -7834,7 +7834,7 @@ private:
         Kind = Present;
       const auto *EI = C->getVarRefs().begin();
       for (const auto L : C->component_lists()) {
-        InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_to, std::nullopt,
+        InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_to, {},
                 C->getMotionModifiers(), /*ReturnDevicePointer=*/false,
                 C->isImplicit(), std::get<2>(L), *EI);
         ++EI;
@@ -7850,8 +7850,8 @@ private:
         Kind = Present;
       const auto *EI = C->getVarRefs().begin();
       for (const auto L : C->component_lists()) {
-        InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_from,
-                std::nullopt, C->getMotionModifiers(),
+        InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_from, {},
+                C->getMotionModifiers(),
                 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(L),
                 *EI);
         ++EI;
@@ -7903,9 +7903,9 @@ private:
             // processed. Nonetheless, generateInfoForComponentList must be
             // called to take the pointer into account for the calculation of
             // the range of the partial struct.
-            InfoGen(nullptr, Other, Components, OMPC_MAP_unknown, std::nullopt,
-                    std::nullopt, /*ReturnDevicePointer=*/false, IsImplicit,
-                    nullptr, nullptr, IsDevAddr);
+            InfoGen(nullptr, Other, Components, OMPC_MAP_unknown, {}, {},
+                    /*ReturnDevicePointer=*/false, IsImplicit, nullptr, nullptr,
+                    IsDevAddr);
             DeferredInfo[nullptr].emplace_back(IE, VD, IsDevAddr);
           } else {
             llvm::Value *Ptr;
@@ -8056,7 +8056,7 @@ private:
               CurInfo, StructBaseCurInfo, PartialStruct,
               /*IsFirstComponentList=*/false, L.IsImplicit,
               /*GenerateAllInfoForClauses*/ true, L.Mapper, L.ForDeviceAddr, VD,
-              L.VarRef, /*OverlappedElements*/ std::nullopt,
+              L.VarRef, /*OverlappedElements*/ {},
               HasMapBasePtr && HasMapArraySec);
 
           // If this entry relates to a device pointer, set the relevant
@@ -8692,7 +8692,7 @@ public:
       ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
           OverlappedComponents = Pair.getSecond();
       generateInfoForComponentList(
-          MapType, MapModifiers, std::nullopt, Components, CombinedInfo,
+          MapType, MapModifiers, {}, Components, CombinedInfo,
           StructBaseCombinedInfo, PartialStruct, IsFirstComponentList,
           IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper,
           /*ForDeviceAddr=*/false, VD, VarRef, OverlappedComponents);
@@ -8711,12 +8711,11 @@ public:
       auto It = OverlappedData.find(&L);
       if (It == OverlappedData.end())
         generateInfoForComponentList(
-            MapType, MapModifiers, std::nullopt, Components, CombinedInfo,
+            MapType, MapModifiers, {}, Components, CombinedInfo,
             StructBaseCombinedInfo, PartialStruct, IsFirstComponentList,
             IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper,
             /*ForDeviceAddr=*/false, VD, VarRef,
-            /*OverlappedElements*/ std::nullopt,
-            HasMapBasePtr && HasMapArraySec);
+            /*OverlappedElements*/ {}, HasMapBasePtr && HasMapArraySec);
       IsFirstComponentList = false;
     }
   }
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index f65314d014c0..5e7715743afb 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -352,7 +352,7 @@ protected:
   /// Emits \p Callee function call with arguments \p Args with location \p Loc.
   void emitCall(CodeGenFunction &CGF, SourceLocation Loc,
                 llvm::FunctionCallee Callee,
-                ArrayRef<llvm::Value *> Args = std::nullopt) const;
+                ArrayRef<llvm::Value *> Args = {}) const;
 
   /// Emits address of the word in a memory where current thread id is
   /// stored.
@@ -1543,7 +1543,7 @@ public:
   virtual void
   emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc,
                            llvm::FunctionCallee OutlinedFn,
-                           ArrayRef<llvm::Value *> Args = std::nullopt) const;
+                           ArrayRef<llvm::Value *> Args = {}) const;
 
   /// Emits OpenMP-specific function prolog.
   /// Required for device constructs.
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 2e5ab6e7b9ac..c66d5d11b0bb 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -985,8 +985,8 @@ llvm::Function *CGOpenMPRuntimeGPU::emitTeamsOutlinedFunction(
     getDistributeLastprivateVars(CGM.getContext(), D, LastPrivatesReductions);
     if (!LastPrivatesReductions.empty()) {
       GlobalizedRD = ::buildRecordForGlobalizedVars(
-          CGM.getContext(), std::nullopt, LastPrivatesReductions,
-          MappedDeclsFields, WarpSize);
+          CGM.getContext(), {}, LastPrivatesReductions, MappedDeclsFields,
+          WarpSize);
     }
   } else if (!LastPrivatesReductions.empty()) {
     assert(!TeamAndReductions.first &&
@@ -1681,7 +1681,7 @@ void CGOpenMPRuntimeGPU::emitReduction(
     ++Cnt;
   }
   const RecordDecl *ReductionRec = ::buildRecordForGlobalizedVars(
-      CGM.getContext(), PrivatesReductions, std::nullopt, VarFieldMap, 1);
+      CGM.getContext(), PrivatesReductions, {}, VarFieldMap, 1);
 
   if (TeamsReduction)
     TeamsReductions.push_back(ReductionRec);
@@ -2345,11 +2345,11 @@ llvm::Value *CGOpenMPRuntimeGPU::getGPUNumThreads(CodeGenFunction &CGF) {
   const char *LocSize = "__kmpc_get_hardware_num_threads_in_block";
   llvm::Function *F = M->getFunction(LocSize);
   if (!F) {
-    F = llvm::Function::Create(
-        llvm::FunctionType::get(CGF.Int32Ty, std::nullopt, false),
-        llvm::GlobalVariable::ExternalLinkage, LocSize, &CGF.CGM.getModule());
+    F = llvm::Function::Create(llvm::FunctionType::get(CGF.Int32Ty, {}, false),
+                               llvm::GlobalVariable::ExternalLinkage, LocSize,
+                               &CGF.CGM.getModule());
   }
-  return Bld.CreateCall(F, std::nullopt, "nvptx_num_threads");
+  return Bld.CreateCall(F, {}, "nvptx_num_threads");
 }
 
 llvm::Value *CGOpenMPRuntimeGPU::getGPUThreadID(CodeGenFunction &CGF) {
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
index 4d586ec972f8..b59f43a6915d 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
@@ -294,9 +294,10 @@ public:
 
   /// Emits call of the outlined function with the provided arguments,
   /// translating these arguments to correct target-specific arguments.
-  void emitOutlinedFunctionCall(
-      CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
-      ArrayRef<llvm::Value *> Args = std::nullopt) const override;
+  void
+  emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc,
+                           llvm::FunctionCallee OutlinedFn,
+                           ArrayRef<llvm::Value *> Args = {}) const override;
 
   /// Emits OpenMP-specific function prolog.
   /// Required for device constructs.
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 71a27d0c6bc1..0da7855ab05c 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -501,7 +501,7 @@ static llvm::Function *emitOutlinedFunctionPrologue(
   FunctionDecl *DebugFunctionDecl = nullptr;
   if (!FO.UIntPtrCastRequired) {
     FunctionProtoType::ExtProtoInfo EPI;
-    QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, std::nullopt, EPI);
+    QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, {}, EPI);
     DebugFunctionDecl = FunctionDecl::Create(
         Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
         SourceLocation(), DeclarationName(), FunctionTy,
@@ -5509,7 +5509,7 @@ void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
         if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
           return llvm::ArrayRef(FlushClause->varlist_begin(),
                                 FlushClause->varlist_end());
-        return std::nullopt;
+        return {};
       }(),
       S.getBeginLoc(), AO);
 }
@@ -6199,7 +6199,7 @@ static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
   case llvm::AtomicOrdering::Acquire:
   case llvm::AtomicOrdering::AcquireRelease:
   case llvm::AtomicOrdering::SequentiallyConsistent:
-    CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc,
+    CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
                                          llvm::AtomicOrdering::Acquire);
     break;
   case llvm::AtomicOrdering::Monotonic:
@@ -6228,7 +6228,7 @@ static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF,
   case llvm::AtomicOrdering::Release:
   case llvm::AtomicOrdering::AcquireRelease:
   case llvm::AtomicOrdering::SequentiallyConsistent:
-    CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc,
+    CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
                                          llvm::AtomicOrdering::Release);
     break;
   case llvm::AtomicOrdering::Acquire:
@@ -6418,7 +6418,7 @@ static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF,
   case llvm::AtomicOrdering::Release:
   case llvm::AtomicOrdering::AcquireRelease:
   case llvm::AtomicOrdering::SequentiallyConsistent:
-    CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc,
+    CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
                                          llvm::AtomicOrdering::Release);
     break;
   case llvm::AtomicOrdering::Acquire:
@@ -6533,17 +6533,17 @@ static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF,
     // operation is also an acquire flush.
     switch (AO) {
     case llvm::AtomicOrdering::Release:
-      CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc,
+      CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
                                            llvm::AtomicOrdering::Release);
       break;
     case llvm::AtomicOrdering::Acquire:
-      CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc,
+      CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
                                            llvm::AtomicOrdering::Acquire);
       break;
     case llvm::AtomicOrdering::AcquireRelease:
     case llvm::AtomicOrdering::SequentiallyConsistent:
       CGF.CGM.getOpenMPRuntime().emitFlush(
-          CGF, std::nullopt, Loc, llvm::AtomicOrdering::AcquireRelease);
+          CGF, {}, Loc, llvm::AtomicOrdering::AcquireRelease);
       break;
     case llvm::AtomicOrdering::Monotonic:
       break;
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 465dc8c661af..573ced0857d5 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -1595,7 +1595,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
       llvm::Value *IsFalse = Builder.getFalse();
       EmitCheck(std::make_pair(IsFalse, SanitizerKind::Return),
                 SanitizerHandler::MissingReturn,
-                EmitCheckSourceLocation(FD->getLocation()), std::nullopt);
+                EmitCheckSourceLocation(FD->getLocation()), {});
     } else if (ShouldEmitUnreachable) {
       if (CGM.getCodeGenOpts().OptimizationLevel == 0)
         EmitTrapCall(llvm::Intrinsic::trap);
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 5f203fe0b128..750a6cc24bad 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3524,7 +3524,7 @@ public:
   /// This function may clear the current insertion point; callers should use
   /// EnsureInsertPoint if they wish to subsequently generate code without first
   /// calling EmitBlock, EmitBranch, or EmitStmt.
-  void EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs = std::nullopt);
+  void EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs = {});
 
   /// EmitSimpleStmt - Try to emit a "simple" statement which does not
   /// necessarily require an insertion point or debug information; typically
@@ -3551,11 +3551,9 @@ public:
   void EmitIndirectGotoStmt(const IndirectGotoStmt &S);
   void EmitIfStmt(const IfStmt &S);
 
-  void EmitWhileStmt(const WhileStmt &S,
-                     ArrayRef<const Attr *> Attrs = std::nullopt);
-  void EmitDoStmt(const DoStmt &S, ArrayRef<const Attr *> Attrs = std::nullopt);
-  void EmitForStmt(const ForStmt &S,
-                   ArrayRef<const Attr *> Attrs = std::nullopt);
+  void EmitWhileStmt(const WhileStmt &S, ArrayRef<const Attr *> Attrs = {});
+  void EmitDoStmt(const DoStmt &S, ArrayRef<const Attr *> Attrs = {});
+  void EmitForStmt(const ForStmt &S, ArrayRef<const Attr *> Attrs = {});
   void EmitReturnStmt(const ReturnStmt &S);
   void EmitDeclStmt(const DeclStmt &S);
   void EmitBreakStmt(const BreakStmt &S);
@@ -3632,7 +3630,7 @@ public:
                                     llvm::Value *ParentFP);
 
   void EmitCXXForRangeStmt(const CXXForRangeStmt &S,
-                           ArrayRef<const Attr *> Attrs = std::nullopt);
+                           ArrayRef<const Attr *> Attrs = {});
 
   /// Controls insertion of cancellation exit blocks in worksharing constructs.
   class OMPCancelStackRAII {
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index 1b77490e261c..8d5787769382 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -753,8 +753,7 @@ public:
 
   llvm::MDNode *getNoObjCARCExceptionsMetadata() {
     if (!NoObjCARCExceptionsMetadata)
-      NoObjCARCExceptionsMetadata =
-          llvm::MDNode::get(getLLVMContext(), std::nullopt);
+      NoObjCARCExceptionsMetadata = llvm::MDNode::get(getLLVMContext(), {});
     return NoObjCARCExceptionsMetadata;
   }
 
@@ -1181,8 +1180,7 @@ public:
   llvm::Constant *getBuiltinLibFunction(const FunctionDecl *FD,
                                         unsigned BuiltinID);
 
-  llvm::Function *getIntrinsic(unsigned IID,
-                               ArrayRef<llvm::Type *> Tys = std::nullopt);
+  llvm::Function *getIntrinsic(unsigned IID, ArrayRef<llvm::Type *> Tys = {});
 
   /// Emit code for a single top level declaration.
   void EmitTopLevelDecl(Decl *D);
diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp
index 0a63c50d44f4..45518736a2ac 100644
--- a/clang/lib/CodeGen/CoverageMappingGen.cpp
+++ b/clang/lib/CodeGen/CoverageMappingGen.cpp
@@ -649,7 +649,7 @@ struct EmptyCoverageMappingBuilder : public CoverageMappingBuilder {
     if (MappingRegions.empty())
       return;
 
-    CoverageMappingWriter Writer(FileIDMapping, std::nullopt, MappingRegions);
+    CoverageMappingWriter Writer(FileIDMapping, {}, MappingRegions);
     Writer.write(OS);
   }
 };
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index 89f945752382..9b3c2f1b2af6 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -1416,7 +1416,7 @@ void ItaniumCXXABI::emitRethrow(CodeGenFunction &CGF, bool isNoReturn) {
   llvm::FunctionCallee Fn = CGM.CreateRuntimeFunction(FTy, "__cxa_rethrow");
 
   if (isNoReturn)
-    CGF.EmitNoreturnRuntimeCallOrInvoke(Fn, std::nullopt);
+    CGF.EmitNoreturnRuntimeCallOrInvoke(Fn, {});
   else
     CGF.EmitRuntimeCallOrInvoke(Fn);
 }
diff --git a/clang/lib/Frontend/DiagnosticRenderer.cpp b/clang/lib/Frontend/DiagnosticRenderer.cpp
index 017ce1c17d5c..8b32af133725 100644
--- a/clang/lib/Frontend/DiagnosticRenderer.cpp
+++ b/clang/lib/Frontend/DiagnosticRenderer.cpp
@@ -146,7 +146,7 @@ void DiagnosticRenderer::emitStoredDiagnostic(StoredDiagnostic &Diag) {
 
 void DiagnosticRenderer::emitBasicNote(StringRef Message) {
   emitDiagnosticMessage(FullSourceLoc(), PresumedLoc(), DiagnosticsEngine::Note,
-                        Message, std::nullopt, DiagOrStoredDiag());
+                        Message, {}, DiagOrStoredDiag());
 }
 
 /// Prints an include stack when appropriate for a particular
@@ -451,7 +451,7 @@ void DiagnosticRenderer::emitSingleMacroExpansion(
     Message << "expanded from macro '" << MacroName << "'";
 
   emitDiagnostic(SpellingLoc, DiagnosticsEngine::Note, Message.str(),
-                 SpellingRanges, std::nullopt);
+                 SpellingRanges, {});
 }
 
 /// Check that the macro argument location of Loc starts with ArgumentLoc.
diff --git a/clang/lib/Index/IndexingContext.h b/clang/lib/Index/IndexingContext.h
index 89363b529fe9..3020b33bea38 100644
--- a/clang/lib/Index/IndexingContext.h
+++ b/clang/lib/Index/IndexingContext.h
@@ -68,17 +68,17 @@ public:
   static bool isTemplateImplicitInstantiation(const Decl *D);
 
   bool handleDecl(const Decl *D, SymbolRoleSet Roles = SymbolRoleSet(),
-                  ArrayRef<SymbolRelation> Relations = std::nullopt);
+                  ArrayRef<SymbolRelation> Relations = {});
 
   bool handleDecl(const Decl *D, SourceLocation Loc,
                   SymbolRoleSet Roles = SymbolRoleSet(),
-                  ArrayRef<SymbolRelation> Relations = std::nullopt,
+                  ArrayRef<SymbolRelation> Relations = {},
                   const DeclContext *DC = nullptr);
 
   bool handleReference(const NamedDecl *D, SourceLocation Loc,
                        const NamedDecl *Parent, const DeclContext *DC,
                        SymbolRoleSet Roles = SymbolRoleSet(),
-                       ArrayRef<SymbolRelation> Relations = std::nullopt,
+                       ArrayRef<SymbolRelation> Relations = {},
                        const Expr *RefE = nullptr);
 
   void handleMacroDefined(const IdentifierInfo &Name, SourceLocation Loc,
@@ -94,8 +94,7 @@ public:
 
   bool indexDecl(const Decl *D);
 
-  void indexTagDecl(const TagDecl *D,
-                    ArrayRef<SymbolRelation> Relations = std::nullopt);
+  void indexTagDecl(const TagDecl *D, ArrayRef<SymbolRelation> Relations = {});
 
   void indexTypeSourceInfo(TypeSourceInfo *TInfo, const NamedDecl *Parent,
                            const DeclContext *DC = nullptr,
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index fd6bc17ae9cd..0a02a63deba3 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -707,7 +707,7 @@ ModuleMap::findAllModulesForHeader(FileEntryRef File) {
   if (findOrCreateModuleForHeaderInUmbrellaDir(File))
     return Headers.find(File)->second;
 
-  return std::nullopt;
+  return {};
 }
 
 ArrayRef<ModuleMap::KnownHeader>
@@ -716,7 +716,7 @@ ModuleMap::findResolvedModulesForHeader(FileEntryRef File) const {
   resolveHeaderDirectives(File);
   auto It = Headers.find(File);
   if (It == Headers.end())
-    return std::nullopt;
+    return {};
   return It->second;
 }
 
diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp
index d48bb8a9a9cd..3eef3dc9b0f3 100644
--- a/clang/lib/Lex/PPMacroExpansion.cpp
+++ b/clang/lib/Lex/PPMacroExpansion.cpp
@@ -286,8 +286,8 @@ void Preprocessor::dumpMacroInfo(const IdentifierInfo *II) {
 
   // Dump module macros.
   llvm::DenseSet<ModuleMacro*> Active;
-  for (auto *MM :
-       State ? State->getActiveModuleMacros(*this, II) : std::nullopt)
+  for (auto *MM : State ? State->getActiveModuleMacros(*this, II)
+                        : ArrayRef<ModuleMacro *>())
     Active.insert(MM);
   llvm::DenseSet<ModuleMacro*> Visited;
   llvm::SmallVector<ModuleMacro *, 16> Worklist(Leaf);
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 122a05be1c03..31984453487a 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -2445,8 +2445,8 @@ Parser::DeclGroupPtrTy Parser::ParseDeclGroup(ParsingDeclSpec &DS,
               // Recover as if it were an explicit specialization.
               TemplateParameterLists FakedParamLists;
               FakedParamLists.push_back(Actions.ActOnTemplateParameterList(
-                  0, SourceLocation(), TemplateInfo.TemplateLoc, LAngleLoc,
-                  std::nullopt, LAngleLoc, nullptr));
+                  0, SourceLocation(), TemplateInfo.TemplateLoc, LAngleLoc, {},
+                  LAngleLoc, nullptr));
 
               TheDecl = ParseFunctionDefinition(
                   D,
@@ -2787,8 +2787,8 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
         // Recover as if it were an explicit specialization.
         TemplateParameterLists FakedParamLists;
         FakedParamLists.push_back(Actions.ActOnTemplateParameterList(
-            0, SourceLocation(), TemplateInfo.TemplateLoc, LAngleLoc,
-            std::nullopt, LAngleLoc, nullptr));
+            0, SourceLocation(), TemplateInfo.TemplateLoc, LAngleLoc, {},
+            LAngleLoc, nullptr));
 
         ThisDecl =
             Actions.ActOnTemplateDeclarator(getCurScope(), FakedParamLists, D);
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index b28c2a9db91b..60aab1411a96 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -2220,8 +2220,8 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
           // "template<>", so that we treat this construct as a class
           // template specialization.
           FakedParamLists.push_back(Actions.ActOnTemplateParameterList(
-              0, SourceLocation(), TemplateInfo.TemplateLoc, LAngleLoc,
-              std::nullopt, LAngleLoc, nullptr));
+              0, SourceLocation(), TemplateInfo.TemplateLoc, LAngleLoc, {},
+              LAngleLoc, nullptr));
           TemplateParams = &FakedParamLists;
         }
       }
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 2fb4be0035b6..4570a18bc0d5 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -3855,8 +3855,8 @@ ExprResult Parser::ParseBlockLiteralExpression() {
                                      /*NumExceptions=*/0,
                                      /*NoexceptExpr=*/nullptr,
                                      /*ExceptionSpecTokens=*/nullptr,
-                                     /*DeclsInPrototype=*/std::nullopt,
-                                     CaretLoc, CaretLoc, ParamInfo),
+                                     /*DeclsInPrototype=*/{}, CaretLoc,
+                                     CaretLoc, ParamInfo),
         CaretLoc);
 
     MaybeParseGNUAttributes(ParamInfo);
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index e96cddf88a13..ce3624f366a2 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -1579,9 +1579,8 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
                       DynamicExceptionRanges.data(), DynamicExceptions.size(),
                       NoexceptExpr.isUsable() ? NoexceptExpr.get() : nullptr,
                       /*ExceptionSpecTokens*/ nullptr,
-                      /*DeclsInPrototype=*/std::nullopt, LParenLoc,
-                      FunLocalRangeEnd, D, TrailingReturnType,
-                      TrailingReturnTypeLoc, &DS),
+                      /*DeclsInPrototype=*/{}, LParenLoc, FunLocalRangeEnd, D,
+                      TrailingReturnType, TrailingReturnTypeLoc, &DS),
                   std::move(Attributes), DeclEndLoc);
 
     // We have called ActOnLambdaClosureQualifiers for parentheses-less cases
diff --git a/clang/lib/Parse/ParseInit.cpp b/clang/lib/Parse/ParseInit.cpp
index dd59cb23236d..07f133408c38 100644
--- a/clang/lib/Parse/ParseInit.cpp
+++ b/clang/lib/Parse/ParseInit.cpp
@@ -487,7 +487,7 @@ ExprResult Parser::ParseBraceInitializer() {
                           : diag::ext_c_empty_initializer);
     }
     // Match the '}'.
-    return Actions.ActOnInitList(LBraceLoc, std::nullopt, ConsumeBrace());
+    return Actions.ActOnInitList(LBraceLoc, {}, ConsumeBrace());
   }
 
   // Enter an appropriate expression evaluation context for an initializer list.
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index dad39d8dc736..28ccd3061f84 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -3227,13 +3227,13 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
     cutOffParsing();
     if (SuperLoc.isValid())
       Actions.CodeCompletion().CodeCompleteObjCSuperMessage(
-          getCurScope(), SuperLoc, std::nullopt, false);
+          getCurScope(), SuperLoc, {}, false);
     else if (ReceiverType)
       Actions.CodeCompletion().CodeCompleteObjCClassMessage(
-          getCurScope(), ReceiverType, std::nullopt, false);
+          getCurScope(), ReceiverType, {}, false);
     else
       Actions.CodeCompletion().CodeCompleteObjCInstanceMessage(
-          getCurScope(), ReceiverExpr, std::nullopt, false);
+          getCurScope(), ReceiverExpr, {}, false);
     return ExprError();
   }
 
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 108b532be416..16f731174fd0 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -2561,7 +2561,7 @@ StmtResult Parser::ParseOpenMPExecutableDirective(
              DKind == OMPD_target_exit_data) {
     Actions.OpenMP().ActOnOpenMPRegionStart(DKind, getCurScope());
     AssociatedStmt = (Sema::CompoundScopeRAII(Actions),
-                      Actions.ActOnCompoundStmt(Loc, Loc, std::nullopt,
+                      Actions.ActOnCompoundStmt(Loc, Loc, {},
                                                 /*isStmtExpr=*/false));
     AssociatedStmt =
         Actions.OpenMP().ActOnOpenMPRegionEnd(AssociatedStmt, Clauses);
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index 60d647da48f0..7d727efb2287 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -2558,8 +2558,7 @@ Decl *Parser::ParseFunctionStatementBody(Decl *Decl, ParseScope &BodyScope) {
   // If the function body could not be parsed, make a bogus compoundstmt.
   if (FnBody.isInvalid()) {
     Sema::CompoundScopeRAII CompoundScope(Actions);
-    FnBody =
-        Actions.ActOnCompoundStmt(LBraceLoc, LBraceLoc, std::nullopt, false);
+    FnBody = Actions.ActOnCompoundStmt(LBraceLoc, LBraceLoc, {}, false);
   }
 
   BodyScope.Exit();
@@ -2596,8 +2595,7 @@ Decl *Parser::ParseFunctionTryBlock(Decl *Decl, ParseScope &BodyScope) {
   // compound statement as the body.
   if (FnBody.isInvalid()) {
     Sema::CompoundScopeRAII CompoundScope(Actions);
-    FnBody =
-        Actions.ActOnCompoundStmt(LBraceLoc, LBraceLoc, std::nullopt, false);
+    FnBody = Actions.ActOnCompoundStmt(LBraceLoc, LBraceLoc, {}, false);
   }
 
   BodyScope.Exit();
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 5e9886a10946..2b51765e8086 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -2536,8 +2536,8 @@ bool Sema::tryExprAsCall(Expr &E, QualType &ZeroArgCallReturnTy,
   // with default arguments, etc.
   if (IsMemExpr && !E.isTypeDependent()) {
     Sema::TentativeAnalysisScope Trap(*this);
-    ExprResult R = BuildCallToMemberFunction(nullptr, &E, SourceLocation(),
-                                             std::nullopt, SourceLocation());
+    ExprResult R = BuildCallToMemberFunction(nullptr, &E, SourceLocation(), {},
+                                             SourceLocation());
     if (R.isUsable()) {
       ZeroArgCallReturnTy = R.get()->getType();
       return true;
@@ -2689,7 +2689,7 @@ bool Sema::tryToRecoverWithCall(ExprResult &E, const PartialDiagnostic &PD,
 
       // FIXME: Try this before emitting the fixit, and suppress diagnostics
       // while doing so.
-      E = BuildCallExpr(nullptr, E.get(), Range.getEnd(), std::nullopt,
+      E = BuildCallExpr(nullptr, E.get(), Range.getEnd(), {},
                         Range.getEnd().getLocWithOffset(1));
       return true;
     }
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 2bcb930acdcb..27b274d74ce7 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -6300,7 +6300,7 @@ public:
   EmitFormatDiagnostic(Sema &S, bool inFunctionCall, const Expr *ArgumentExpr,
                        const PartialDiagnostic &PDiag, SourceLocation StringLoc,
                        bool IsStringLocation, Range StringRange,
-                       ArrayRef<FixItHint> Fixit = std::nullopt);
+                       ArrayRef<FixItHint> Fixit = {});
 
 protected:
   bool HandleInvalidConversionSpecifier(unsigned argIndex, SourceLocation Loc,
@@ -6327,7 +6327,7 @@ protected:
   template <typename Range>
   void EmitFormatDiagnostic(PartialDiagnostic PDiag, SourceLocation StringLoc,
                             bool IsStringLocation, Range StringRange,
-                            ArrayRef<FixItHint> Fixit = std::nullopt);
+                            ArrayRef<FixItHint> Fixit = {});
 };
 
 } // namespace
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 3e31f3d82657..16a76ff9b5c2 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -4567,8 +4567,7 @@ void SemaCodeCompletion::CodeCompleteDeclSpec(Scope *S, DeclSpec &DS,
           0) {
     ParsedType T = DS.getRepAsType();
     if (!T.get().isNull() && T.get()->isObjCObjectOrInterfaceType())
-      AddClassMessageCompletions(SemaRef, S, T, std::nullopt, false, false,
-                                 Results);
+      AddClassMessageCompletions(SemaRef, S, T, {}, false, false, Results);
   }
 
   // Note that we intentionally suppress macro results here, since we do not
@@ -4931,7 +4930,7 @@ void SemaCodeCompletion::CodeCompletePostfixExpression(Scope *S, ExprResult E,
   if (E.isInvalid())
     CodeCompleteExpression(S, PreferredType);
   else if (getLangOpts().ObjC)
-    CodeCompleteObjCInstanceMessage(S, E.get(), std::nullopt, false);
+    CodeCompleteObjCInstanceMessage(S, E.get(), {}, false);
 }
 
 /// The set of properties that have already been added, referenced by
@@ -7747,8 +7746,8 @@ void SemaCodeCompletion::CodeCompleteObjCPropertyGetter(Scope *S) {
   Results.EnterNewScope();
 
   VisitedSelectorSet Selectors;
-  AddObjCMethods(Class, true, MK_ZeroArgSelector, std::nullopt,
-                 SemaRef.CurContext, Selectors,
+  AddObjCMethods(Class, true, MK_ZeroArgSelector, {}, SemaRef.CurContext,
+                 Selectors,
                  /*AllowSameLength=*/true, Results);
   Results.ExitScope();
   HandleCodeCompleteResults(&SemaRef, CodeCompleter,
@@ -7776,8 +7775,8 @@ void SemaCodeCompletion::CodeCompleteObjCPropertySetter(Scope *S) {
   Results.EnterNewScope();
 
   VisitedSelectorSet Selectors;
-  AddObjCMethods(Class, true, MK_OneArgSelector, std::nullopt,
-                 SemaRef.CurContext, Selectors,
+  AddObjCMethods(Class, true, MK_OneArgSelector, {}, SemaRef.CurContext,
+                 Selectors,
                  /*AllowSameLength=*/true, Results);
 
   Results.ExitScope();
@@ -8075,8 +8074,7 @@ void SemaCodeCompletion::CodeCompleteObjCMessageReceiver(Scope *S) {
       if (Iface->getSuperClass()) {
         Results.AddResult(Result("super"));
 
-        AddSuperSendCompletion(SemaRef, /*NeedSuperKeyword=*/true, std::nullopt,
-                               Results);
+        AddSuperSendCompletion(SemaRef, /*NeedSuperKeyword=*/true, {}, Results);
       }
 
   if (getLangOpts().CPlusPlus11)
diff --git a/clang/lib/Sema/SemaCoroutine.cpp b/clang/lib/Sema/SemaCoroutine.cpp
index 89a0beadc61f..3724aaf804c9 100644
--- a/clang/lib/Sema/SemaCoroutine.cpp
+++ b/clang/lib/Sema/SemaCoroutine.cpp
@@ -341,7 +341,7 @@ static Expr *maybeTailCall(Sema &S, QualType RetType, Expr *E,
   // EvaluateBinaryTypeTrait(BTT_IsConvertible, ...) which is at the moment
   // a private function in SemaExprCXX.cpp
 
-  ExprResult AddressExpr = buildMemberCall(S, E, Loc, "address", std::nullopt);
+  ExprResult AddressExpr = buildMemberCall(S, E, Loc, "address", {});
   if (AddressExpr.isInvalid())
     return nullptr;
 
@@ -392,8 +392,8 @@ static ReadySuspendResumeResult buildCoawaitCalls(Sema &S, VarDecl *CoroPromise,
     return Result.get();
   };
 
-  CallExpr *AwaitReady = cast_or_null<CallExpr>(
-      BuildSubExpr(ACT::ACT_Ready, "await_ready", std::nullopt));
+  CallExpr *AwaitReady =
+      cast_or_null<CallExpr>(BuildSubExpr(ACT::ACT_Ready, "await_ready", {}));
   if (!AwaitReady)
     return Calls;
   if (!AwaitReady->getType()->isDependentType()) {
@@ -454,7 +454,7 @@ static ReadySuspendResumeResult buildCoawaitCalls(Sema &S, VarDecl *CoroPromise,
     }
   }
 
-  BuildSubExpr(ACT::ACT_Resume, "await_resume", std::nullopt);
+  BuildSubExpr(ACT::ACT_Resume, "await_resume", {});
 
   // Make sure the awaiter object gets a chance to be cleaned up.
   S.Cleanup.setExprNeedsCleanups(true);
@@ -722,8 +722,8 @@ bool Sema::ActOnCoroutineBodyStart(Scope *SC, SourceLocation KWLoc,
   SourceLocation Loc = Fn->getLocation();
   // Build the initial suspend point
   auto buildSuspends = [&](StringRef Name) mutable -> StmtResult {
-    ExprResult Operand = buildPromiseCall(*this, ScopeInfo->CoroutinePromise,
-                                          Loc, Name, std::nullopt);
+    ExprResult Operand =
+        buildPromiseCall(*this, ScopeInfo->CoroutinePromise, Loc, Name, {});
     if (Operand.isInvalid())
       return StmtError();
     ExprResult Suspend =
@@ -1049,7 +1049,7 @@ StmtResult Sema::BuildCoreturnStmt(SourceLocation Loc, Expr *E,
     PC = buildPromiseCall(*this, Promise, Loc, "return_value", E);
   } else {
     E = MakeFullDiscardedValueExpr(E).get();
-    PC = buildPromiseCall(*this, Promise, Loc, "return_void", std::nullopt);
+    PC = buildPromiseCall(*this, Promise, Loc, "return_void", {});
   }
   if (PC.isInvalid())
     return StmtError();
@@ -1735,8 +1735,8 @@ bool CoroutineStmtBuilder::makeOnException() {
   if (!S.getLangOpts().CXXExceptions)
     return true;
 
-  ExprResult UnhandledException = buildPromiseCall(
-      S, Fn.CoroutinePromise, Loc, "unhandled_exception", std::nullopt);
+  ExprResult UnhandledException =
+      buildPromiseCall(S, Fn.CoroutinePromise, Loc, "unhandled_exception", {});
   UnhandledException = S.ActOnFinishFullExpr(UnhandledException.get(), Loc,
                                              /*DiscardedValue*/ false);
   if (UnhandledException.isInvalid())
@@ -1759,8 +1759,8 @@ bool CoroutineStmtBuilder::makeReturnObject() {
   // [dcl.fct.def.coroutine]p7
   // The expression promise.get_return_object() is used to initialize the
   // returned reference or prvalue result object of a call to a coroutine.
-  ExprResult ReturnObject = buildPromiseCall(S, Fn.CoroutinePromise, Loc,
-                                             "get_return_object", std::nullopt);
+  ExprResult ReturnObject =
+      buildPromiseCall(S, Fn.CoroutinePromise, Loc, "get_return_object", {});
   if (ReturnObject.isInvalid())
     return false;
 
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 229c9080d558..25061f02c13f 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14135,8 +14135,8 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl) {
     InitializationKind Kind
       = InitializationKind::CreateDefault(Var->getLocation());
 
-    InitializationSequence InitSeq(*this, Entity, Kind, std::nullopt);
-    ExprResult Init = InitSeq.Perform(*this, Entity, Kind, std::nullopt);
+    InitializationSequence InitSeq(*this, Entity, Kind, {});
+    ExprResult Init = InitSeq.Perform(*this, Entity, Kind, {});
 
     if (Init.get()) {
       Var->setInit(MaybeCreateExprWithCleanups(Init.get()));
@@ -16428,8 +16428,8 @@ NamedDecl *Sema::ImplicitlyDefineFunction(SourceLocation Loc,
                                              /*NumExceptions=*/0,
                                              /*NoexceptExpr=*/nullptr,
                                              /*ExceptionSpecTokens=*/nullptr,
-                                             /*DeclsInPrototype=*/std::nullopt,
-                                             Loc, Loc, D),
+                                             /*DeclsInPrototype=*/{}, Loc, Loc,
+                                             D),
                 std::move(DS.getAttributes()), SourceLocation());
   D.SetIdentifier(&II, Loc);
 
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 43ec25b23d97..1a691c0e1689 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -1281,7 +1281,7 @@ static bool checkTupleLikeDecomposition(Sema &S,
       if (E.isInvalid())
         return true;
 
-      E = S.BuildCallExpr(nullptr, E.get(), Loc, std::nullopt, Loc);
+      E = S.BuildCallExpr(nullptr, E.get(), Loc, {}, Loc);
     } else {
       //   Otherwise, the initializer is get<i-1>(e), where get is looked up
       //   in the associated namespaces.
@@ -4797,8 +4797,8 @@ BuildImplicitBaseInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
   case IIK_Default: {
     InitializationKind InitKind
       = InitializationKind::CreateDefault(Constructor->getLocation());
-    InitializationSequence InitSeq(SemaRef, InitEntity, InitKind, std::nullopt);
-    BaseInit = InitSeq.Perform(SemaRef, InitEntity, InitKind, std::nullopt);
+    InitializationSequence InitSeq(SemaRef, InitEntity, InitKind, {});
+    BaseInit = InitSeq.Perform(SemaRef, InitEntity, InitKind, {});
     break;
   }
 
@@ -4962,9 +4962,8 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     InitializationKind InitKind =
       InitializationKind::CreateDefault(Loc);
 
-    InitializationSequence InitSeq(SemaRef, InitEntity, InitKind, std::nullopt);
-    ExprResult MemberInit =
-        InitSeq.Perform(SemaRef, InitEntity, InitKind, std::nullopt);
+    InitializationSequence InitSeq(SemaRef, InitEntity, InitKind, {});
+    ExprResult MemberInit = InitSeq.Perform(SemaRef, InitEntity, InitKind, {});
 
     MemberInit = SemaRef.MaybeCreateExprWithCleanups(MemberInit);
     if (MemberInit.isInvalid())
@@ -10992,7 +10991,7 @@ QualType Sema::CheckDestructorDeclarator(Declarator &D, QualType R,
   EPI.Variadic = false;
   EPI.TypeQuals = Qualifiers();
   EPI.RefQualifier = RQ_None;
-  return Context.getFunctionType(Context.VoidTy, std::nullopt, EPI);
+  return Context.getFunctionType(Context.VoidTy, {}, EPI);
 }
 
 static void extendLeft(SourceRange &R, SourceRange Before) {
@@ -11172,8 +11171,7 @@ void Sema::CheckConversionDeclarator(Declarator &D, QualType &R,
   // of the errors above fired) and with the conversion type as the
   // return type.
   if (D.isInvalidType())
-    R = Context.getFunctionType(ConvType, std::nullopt,
-                                Proto->getExtProtoInfo());
+    R = Context.getFunctionType(ConvType, {}, Proto->getExtProtoInfo());
 
   // C++0x explicit conversion operators.
   if (DS.hasExplicitSpecifier() && !getLangOpts().CPlusPlus20)
@@ -13867,7 +13865,7 @@ CXXConstructorDecl *Sema::DeclareImplicitDefaultConstructor(
   DefaultCon->setAccess(AS_public);
   DefaultCon->setDefaulted();
 
-  setupImplicitSpecialMemberType(DefaultCon, Context.VoidTy, std::nullopt);
+  setupImplicitSpecialMemberType(DefaultCon, Context.VoidTy, {});
 
   if (getLangOpts().CUDA)
     CUDA().inferTargetForImplicitSpecialMember(
@@ -14153,7 +14151,7 @@ CXXDestructorDecl *Sema::DeclareImplicitDestructor(CXXRecordDecl *ClassDecl) {
   Destructor->setAccess(AS_public);
   Destructor->setDefaulted();
 
-  setupImplicitSpecialMemberType(Destructor, Context.VoidTy, std::nullopt);
+  setupImplicitSpecialMemberType(Destructor, Context.VoidTy, {});
 
   if (getLangOpts().CUDA)
     CUDA().inferTargetForImplicitSpecialMember(
@@ -14312,8 +14310,7 @@ void Sema::AdjustDestructorExceptionSpec(CXXDestructorDecl *Destructor) {
   FunctionProtoType::ExtProtoInfo EPI = DtorType->getExtProtoInfo();
   EPI.ExceptionSpec.Type = EST_Unevaluated;
   EPI.ExceptionSpec.SourceDecl = Destructor;
-  Destructor->setType(
-      Context.getFunctionType(Context.VoidTy, std::nullopt, EPI));
+  Destructor->setType(Context.getFunctionType(Context.VoidTy, {}, EPI));
 
   // FIXME: If the destructor has a body that could throw, and the newly created
   // spec doesn't allow exceptions, we should emit a warning, because this
@@ -15639,8 +15636,7 @@ void Sema::DefineImplicitCopyConstructor(SourceLocation CurrentLocation,
                              : CopyConstructor->getLocation();
     Sema::CompoundScopeRAII CompoundScope(*this);
     CopyConstructor->setBody(
-        ActOnCompoundStmt(Loc, Loc, std::nullopt, /*isStmtExpr=*/false)
-            .getAs<Stmt>());
+        ActOnCompoundStmt(Loc, Loc, {}, /*isStmtExpr=*/false).getAs<Stmt>());
     CopyConstructor->markUsed(Context);
   }
 
@@ -15771,8 +15767,7 @@ void Sema::DefineImplicitMoveConstructor(SourceLocation CurrentLocation,
                              : MoveConstructor->getLocation();
     Sema::CompoundScopeRAII CompoundScope(*this);
     MoveConstructor->setBody(
-        ActOnCompoundStmt(Loc, Loc, std::nullopt, /*isStmtExpr=*/false)
-            .getAs<Stmt>());
+        ActOnCompoundStmt(Loc, Loc, {}, /*isStmtExpr=*/false).getAs<Stmt>());
     MoveConstructor->markUsed(Context);
   }
 
@@ -17228,8 +17223,7 @@ bool Sema::EvaluateStaticAssertMessageAsString(Expr *Message,
         CXXScopeSpec(), SourceLocation(), nullptr, LR, nullptr, nullptr);
     if (Res.isInvalid())
       return ExprError();
-    Res = BuildCallExpr(nullptr, Res.get(), Loc, std::nullopt, Loc, nullptr,
-                        false, true);
+    Res = BuildCallExpr(nullptr, Res.get(), Loc, {}, Loc, nullptr, false, true);
     if (Res.isInvalid())
       return ExprError();
     if (Res.get()->isTypeDependent() || Res.get()->isValueDependent())
diff --git a/clang/lib/Sema/SemaDeclObjC.cpp b/clang/lib/Sema/SemaDeclObjC.cpp
index d8cd09b82729..78acfeddb786 100644
--- a/clang/lib/Sema/SemaDeclObjC.cpp
+++ b/clang/lib/Sema/SemaDeclObjC.cpp
@@ -5511,10 +5511,9 @@ void SemaObjC::SetIvarInitializers(ObjCImplementationDecl *ObjCImplementation) {
       InitializationKind InitKind =
           InitializationKind::CreateDefault(ObjCImplementation->getLocation());
 
-      InitializationSequence InitSeq(SemaRef, InitEntity, InitKind,
-                                     std::nullopt);
+      InitializationSequence InitSeq(SemaRef, InitEntity, InitKind, {});
       ExprResult MemberInit =
-          InitSeq.Perform(SemaRef, InitEntity, InitKind, std::nullopt);
+          InitSeq.Perform(SemaRef, InitEntity, InitKind, {});
       MemberInit = SemaRef.MaybeCreateExprWithCleanups(MemberInit);
       // Note, MemberInit could actually come back empty if no initialization
       // is required (e.g., because it would call a trivial default constructor)
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 6807f44562f6..ff6616901016 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -1079,8 +1079,8 @@ ExprResult Sema::DefaultVariadicArgumentPromotion(Expr *E, VariadicCallType CT,
     if (TrapFn.isInvalid())
       return ExprError();
 
-    ExprResult Call = BuildCallExpr(TUScope, TrapFn.get(), E->getBeginLoc(),
-                                    std::nullopt, E->getEndLoc());
+    ExprResult Call = BuildCallExpr(TUScope, TrapFn.get(), E->getBeginLoc(), {},
+                                    E->getEndLoc());
     if (Call.isInvalid())
       return ExprError();
 
@@ -2164,8 +2164,8 @@ Sema::ActOnStringLiteral(ArrayRef<Token> StringToks, Scope *UDLScope) {
     TemplateArgument Arg(Lit);
     TemplateArgumentLocInfo ArgInfo(Lit);
     ExplicitArgs.addArgument(TemplateArgumentLoc(Arg, ArgInfo));
-    return BuildLiteralOperatorCall(R, OpNameInfo, std::nullopt,
-                                    StringTokLocs.back(), &ExplicitArgs);
+    return BuildLiteralOperatorCall(R, OpNameInfo, {}, StringTokLocs.back(),
+                                    &ExplicitArgs);
   }
 
   case LOLR_StringTemplatePack: {
@@ -2185,8 +2185,8 @@ Sema::ActOnStringLiteral(ArrayRef<Token> StringToks, Scope *UDLScope) {
       TemplateArgumentLocInfo ArgInfo;
       ExplicitArgs.addArgument(TemplateArgumentLoc(Arg, ArgInfo));
     }
-    return BuildLiteralOperatorCall(R, OpNameInfo, std::nullopt,
-                                    StringTokLocs.back(), &ExplicitArgs);
+    return BuildLiteralOperatorCall(R, OpNameInfo, {}, StringTokLocs.back(),
+                                    &ExplicitArgs);
   }
   case LOLR_Raw:
   case LOLR_ErrorNoDiagnostic:
@@ -2801,7 +2801,7 @@ Sema::ActOnIdExpression(Scope *S, CXXScopeSpec &SS,
     // a template name, but we happen to have always already looked up the name
     // before we get here if it must be a template name.
     if (DiagnoseEmptyLookup(S, SS, R, CCC ? *CCC : DefaultValidator, nullptr,
-                            std::nullopt, nullptr, &TE)) {
+                            {}, nullptr, &TE)) {
       if (TE && KeywordReplacement) {
         auto &State = getTypoExprState(TE);
         auto BestTC = State.Consumer->getNextCorrection();
@@ -3787,8 +3787,7 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok, Scope *UDLScope) {
         TemplateArgumentLocInfo ArgInfo;
         ExplicitArgs.addArgument(TemplateArgumentLoc(Arg, ArgInfo));
       }
-      return BuildLiteralOperatorCall(R, OpNameInfo, std::nullopt, TokLoc,
-                                      &ExplicitArgs);
+      return BuildLiteralOperatorCall(R, OpNameInfo, {}, TokLoc, &ExplicitArgs);
     }
     case LOLR_StringTemplatePack:
       llvm_unreachable("unexpected literal operator lookup result");
@@ -16245,7 +16244,7 @@ ExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
     if (isa<FunctionNoProtoType>(FTy)) {
       FunctionProtoType::ExtProtoInfo EPI;
       EPI.ExtInfo = Ext;
-      BlockTy = Context.getFunctionType(RetTy, std::nullopt, EPI);
+      BlockTy = Context.getFunctionType(RetTy, {}, EPI);
 
       // Otherwise, if we don't need to change anything about the function type,
       // preserve its sugar structure.
@@ -16266,7 +16265,7 @@ ExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
   } else {
     FunctionProtoType::ExtProtoInfo EPI;
     EPI.ExtInfo = FunctionType::ExtInfo().withNoReturn(NoReturn);
-    BlockTy = Context.getFunctionType(RetTy, std::nullopt, EPI);
+    BlockTy = Context.getFunctionType(RetTy, {}, EPI);
   }
 
   DiagnoseUnusedParameters(BD->parameters());
@@ -20144,7 +20143,8 @@ bool Sema::DiagRuntimeBehavior(SourceLocation Loc, ArrayRef<const Stmt*> Stmts,
 bool Sema::DiagRuntimeBehavior(SourceLocation Loc, const Stmt *Statement,
                                const PartialDiagnostic &PD) {
   return DiagRuntimeBehavior(
-      Loc, Statement ? llvm::ArrayRef(Statement) : std::nullopt, PD);
+      Loc, Statement ? llvm::ArrayRef(Statement) : llvm::ArrayRef<Stmt *>(),
+      PD);
 }
 
 bool Sema::CheckCallReturnType(QualType ReturnType, SourceLocation Loc,
diff --git a/clang/lib/Sema/SemaExprObjC.cpp b/clang/lib/Sema/SemaExprObjC.cpp
index 2f914ddc22a3..35fbc4e7c30e 100644
--- a/clang/lib/Sema/SemaExprObjC.cpp
+++ b/clang/lib/Sema/SemaExprObjC.cpp
@@ -305,7 +305,7 @@ static ObjCMethodDecl *getNSNumberFactoryMethod(SemaObjC &S, SourceLocation Loc,
         ParmVarDecl::Create(S.SemaRef.Context, Method, SourceLocation(),
                             SourceLocation(), &CX.Idents.get("value"),
                             NumberType, /*TInfo=*/nullptr, SC_None, nullptr);
-    Method->setMethodParams(S.SemaRef.Context, value, std::nullopt);
+    Method->setMethodParams(S.SemaRef.Context, value, {});
   }
 
   if (!validateBoxingMethod(S.SemaRef, Loc, S.NSNumberDecl, Sel, Method))
@@ -588,7 +588,7 @@ ExprResult SemaObjC::BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr) {
                                 Context.getPointerType(ConstCharType),
                                 /*TInfo=*/nullptr,
                                 SC_None, nullptr);
-          M->setMethodParams(Context, value, std::nullopt);
+          M->setMethodParams(Context, value, {});
           BoxingMethod = M;
         }
 
@@ -713,7 +713,7 @@ ExprResult SemaObjC::BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr) {
                             SC_None, nullptr);
         Params.push_back(type);
 
-        M->setMethodParams(Context, Params, std::nullopt);
+        M->setMethodParams(Context, Params, {});
         BoxingMethod = M;
       }
 
@@ -843,7 +843,7 @@ ExprResult SemaObjC::BuildObjCArrayLiteral(SourceRange SR,
                                              /*TInfo=*/nullptr, SC_None,
                                              nullptr);
       Params.push_back(cnt);
-      Method->setMethodParams(Context, Params, std::nullopt);
+      Method->setMethodParams(Context, Params, {});
     }
 
     if (!validateBoxingMethod(SemaRef, Loc, NSArrayDecl, Sel, Method))
@@ -1012,7 +1012,7 @@ ExprResult SemaObjC::BuildObjCDictionaryLiteral(
                                              /*TInfo=*/nullptr, SC_None,
                                              nullptr);
       Params.push_back(cnt);
-      Method->setMethodParams(Context, Params, std::nullopt);
+      Method->setMethodParams(Context, Params, {});
     }
 
     if (!validateBoxingMethod(SemaRef, SR.getBegin(), NSDictionaryDecl, Sel,
@@ -4388,7 +4388,7 @@ bool SemaObjC::CheckObjCBridgeRelatedConversions(SourceLocation Loc,
 
         ExprResult msg = BuildInstanceMessageImplicit(
             SrcExpr, SrcType, InstanceMethod->getLocation(),
-            InstanceMethod->getSelector(), InstanceMethod, std::nullopt);
+            InstanceMethod->getSelector(), InstanceMethod, {});
         SrcExpr = msg.get();
       }
       return true;
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index f560865681fa..573e90aced3e 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -569,7 +569,7 @@ ExprResult InitListChecker::PerformEmptyInit(SourceLocation Loc,
                                                             true);
   MultiExprArg SubInit;
   Expr *InitExpr;
-  InitListExpr DummyInitList(SemaRef.Context, Loc, std::nullopt, Loc);
+  InitListExpr DummyInitList(SemaRef.Context, Loc, {}, Loc);
 
   // C++ [dcl.init.aggr]p7:
   //   If there are fewer initializer-clauses in the list than there are
@@ -588,10 +588,9 @@ ExprResult InitListChecker::PerformEmptyInit(SourceLocation Loc,
     //
     // Only do this if we're initializing a class type, to avoid filling in
     // the initializer list where possible.
-    InitExpr = VerifyOnly
-                   ? &DummyInitList
-                   : new (SemaRef.Context)
-                         InitListExpr(SemaRef.Context, Loc, std::nullopt, Loc);
+    InitExpr = VerifyOnly ? &DummyInitList
+                          : new (SemaRef.Context)
+                                InitListExpr(SemaRef.Context, Loc, {}, Loc);
     InitExpr->setType(SemaRef.Context.VoidTy);
     SubInit = InitExpr;
     Kind = InitializationKind::CreateCopy(Loc, Loc);
@@ -3403,7 +3402,7 @@ InitListChecker::createInitListExpr(QualType CurrentObjectType,
                                     SourceRange InitRange,
                                     unsigned ExpectedNumInits) {
   InitListExpr *Result = new (SemaRef.Context) InitListExpr(
-      SemaRef.Context, InitRange.getBegin(), std::nullopt, InitRange.getEnd());
+      SemaRef.Context, InitRange.getBegin(), {}, InitRange.getEnd());
 
   QualType ResultType = CurrentObjectType;
   if (!ResultType->isArrayType())
@@ -5650,7 +5649,7 @@ static void TryDefaultInitialization(Sema &S,
   //       constructor for T is called (and the initialization is ill-formed if
   //       T has no accessible default constructor);
   if (DestType->isRecordType() && S.getLangOpts().CPlusPlus) {
-    TryConstructorInitialization(S, Entity, Kind, std::nullopt, DestType,
+    TryConstructorInitialization(S, Entity, Kind, {}, DestType,
                                  Entity.getType(), Sequence);
     return;
   }
@@ -5687,11 +5686,13 @@ static void TryOrBuildParenListInitialization(
                                      const InitializationKind &SubKind,
                                      Expr *Arg, Expr **InitExpr = nullptr) {
     InitializationSequence IS = InitializationSequence(
-        S, SubEntity, SubKind, Arg ? MultiExprArg(Arg) : std::nullopt);
+        S, SubEntity, SubKind,
+        Arg ? MultiExprArg(Arg) : MutableArrayRef<Expr *>());
 
     if (IS.Failed()) {
       if (!VerifyOnly) {
-        IS.Diagnose(S, SubEntity, SubKind, Arg ? ArrayRef(Arg) : std::nullopt);
+        IS.Diagnose(S, SubEntity, SubKind,
+                    Arg ? ArrayRef(Arg) : ArrayRef<Expr *>());
       } else {
         Sequence.SetFailed(
             InitializationSequence::FK_ParenthesizedListInitFailed);
@@ -5702,7 +5703,7 @@ static void TryOrBuildParenListInitialization(
     if (!VerifyOnly) {
       ExprResult ER;
       ER = IS.Perform(S, SubEntity, SubKind,
-                      Arg ? MultiExprArg(Arg) : std::nullopt);
+                      Arg ? MultiExprArg(Arg) : MutableArrayRef<Expr *>());
 
       if (ER.isInvalid())
         return false;
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index 7b9d5f4ff7eb..e7afa0f4c81f 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -909,8 +909,8 @@ getDummyLambdaType(Sema &S, SourceLocation Loc = SourceLocation()) {
   QualType DefaultTypeForNoTrailingReturn = S.getLangOpts().CPlusPlus14
                                                 ? S.Context.getAutoDeductType()
                                                 : S.Context.DependentTy;
-  QualType MethodTy = S.Context.getFunctionType(DefaultTypeForNoTrailingReturn,
-                                                std::nullopt, EPI);
+  QualType MethodTy =
+      S.Context.getFunctionType(DefaultTypeForNoTrailingReturn, {}, EPI);
   return S.Context.getTrivialTypeSourceInfo(MethodTy, Loc);
 }
 
@@ -1681,8 +1681,7 @@ static void addFunctionPointerConversion(Sema &S, SourceRange IntroducerRange,
   ConvExtInfo.TypeQuals = Qualifiers();
   ConvExtInfo.TypeQuals.addConst();
   ConvExtInfo.ExceptionSpec.Type = EST_BasicNoexcept;
-  QualType ConvTy =
-      S.Context.getFunctionType(PtrToFunctionTy, std::nullopt, ConvExtInfo);
+  QualType ConvTy = S.Context.getFunctionType(PtrToFunctionTy, {}, ConvExtInfo);
 
   SourceLocation Loc = IntroducerRange.getBegin();
   DeclarationName ConversionName
@@ -1864,8 +1863,7 @@ static void addBlockPointerConversion(Sema &S,
           /*IsVariadic=*/false, /*IsCXXMethod=*/true));
   ConversionEPI.TypeQuals = Qualifiers();
   ConversionEPI.TypeQuals.addConst();
-  QualType ConvTy =
-      S.Context.getFunctionType(BlockPtrTy, std::nullopt, ConversionEPI);
+  QualType ConvTy = S.Context.getFunctionType(BlockPtrTy, {}, ConversionEPI);
 
   SourceLocation Loc = IntroducerRange.getBegin();
   DeclarationName Name
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index e5db11369221..8468e9a730c2 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -1200,7 +1200,7 @@ static bool LookupDirect(Sema &S, LookupResult &R, const DeclContext *DC) {
     EPI.ExtInfo = EPI.ExtInfo.withCallingConv(CC_C);
     EPI.ExceptionSpec = EST_None;
     QualType ExpectedType = R.getSema().Context.getFunctionType(
-        R.getLookupName().getCXXNameType(), std::nullopt, EPI);
+        R.getLookupName().getCXXNameType(), {}, EPI);
 
     // Perform template argument deduction against the type that we would
     // expect the function to have.
diff --git a/clang/lib/Sema/SemaObjCProperty.cpp b/clang/lib/Sema/SemaObjCProperty.cpp
index 031f2a6af877..5c4dc93ff16c 100644
--- a/clang/lib/Sema/SemaObjCProperty.cpp
+++ b/clang/lib/Sema/SemaObjCProperty.cpp
@@ -2552,7 +2552,7 @@ void SemaObjC::ProcessPropertyDecl(ObjCPropertyDecl *property) {
                                                   /*TInfo=*/nullptr,
                                                   SC_None,
                                                   nullptr);
-      SetterMethod->setMethodParams(Context, Argument, std::nullopt);
+      SetterMethod->setMethodParams(Context, Argument, {});
 
       AddPropertyAttrs(SemaRef, SetterMethod, property);
 
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index d33b0d0c1c30..aadcd7183e71 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -1986,8 +1986,7 @@ ExprResult SemaOpenACC::ActOnArraySectionExpr(Expr *Base, SourceLocation LBLoc,
     // Fill in a dummy 'length' so that when we instantiate this we don't
     // double-diagnose here.
     ExprResult Recovery = SemaRef.CreateRecoveryExpr(
-        ColonLoc, SourceLocation(), ArrayRef<Expr *>{std::nullopt},
-        Context.IntTy);
+        ColonLoc, SourceLocation(), ArrayRef<Expr *>(), Context.IntTy);
     Length = Recovery.isUsable() ? Recovery.get() : nullptr;
   }
 
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index fa81fc42c0ee..79e1536288e6 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -15901,9 +15901,9 @@ OMPClause *SemaOpenMP::ActOnOpenMPSimpleClause(
   return Res;
 }
 
-static std::string
-getListOfPossibleValues(OpenMPClauseKind K, unsigned First, unsigned Last,
-                        ArrayRef<unsigned> Exclude = std::nullopt) {
+static std::string getListOfPossibleValues(OpenMPClauseKind K, unsigned First,
+                                           unsigned Last,
+                                           ArrayRef<unsigned> Exclude = {}) {
   SmallString<256> Buffer;
   llvm::raw_svector_ostream Out(Buffer);
   unsigned Skipped = Exclude.size();
@@ -21296,7 +21296,7 @@ static void checkMappableExpressionList(
     CXXScopeSpec &MapperIdScopeSpec, DeclarationNameInfo MapperId,
     ArrayRef<Expr *> UnresolvedMappers,
     OpenMPMapClauseKind MapType = OMPC_MAP_unknown,
-    ArrayRef<OpenMPMapModifierKind> Modifiers = std::nullopt,
+    ArrayRef<OpenMPMapModifierKind> Modifiers = {},
     bool IsMapTypeImplicit = false, bool NoDiagnose = false) {
   // We only expect mappable expressions in 'to', 'from', and 'map' clauses.
   assert((CKind == OMPC_map || CKind == OMPC_to || CKind == OMPC_from) &&
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 7b86299561a3..4aeceba128b2 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -5502,7 +5502,7 @@ TryListConversion(Sema &S, InitListExpr *From, QualType ToType,
         }
         if (CT->getSize().ugt(e)) {
           // Need an init from empty {}, is there one?
-          InitListExpr EmptyList(S.Context, From->getEndLoc(), std::nullopt,
+          InitListExpr EmptyList(S.Context, From->getEndLoc(), {},
                                  From->getEndLoc());
           EmptyList.setType(S.Context.VoidTy);
           DfltElt = TryListConversion(
@@ -7479,7 +7479,7 @@ void Sema::AddMethodCandidate(DeclAccessPair FoundDecl, QualType ObjectType,
   } else {
     AddMethodCandidate(cast<CXXMethodDecl>(Decl), FoundDecl, ActingContext,
                        ObjectType, ObjectClassification, Args, CandidateSet,
-                       SuppressUserConversions, false, std::nullopt, PO);
+                       SuppressUserConversions, false, {}, PO);
   }
 }
 
@@ -8125,7 +8125,7 @@ void Sema::AddConversionCandidate(
   }
 
   if (EnableIfAttr *FailedAttr =
-          CheckEnableIf(Conversion, CandidateSet.getLocation(), std::nullopt)) {
+          CheckEnableIf(Conversion, CandidateSet.getLocation(), {})) {
     Candidate.Viable = false;
     Candidate.FailureKind = ovl_fail_enable_if;
     Candidate.DeductionFailure.Data = FailedAttr;
@@ -8306,7 +8306,7 @@ void Sema::AddSurrogateCandidate(CXXConversionDecl *Conversion,
   }
 
   if (EnableIfAttr *FailedAttr =
-          CheckEnableIf(Conversion, CandidateSet.getLocation(), std::nullopt)) {
+          CheckEnableIf(Conversion, CandidateSet.getLocation(), {})) {
     Candidate.Viable = false;
     Candidate.FailureKind = ovl_fail_enable_if;
     Candidate.DeductionFailure.Data = FailedAttr;
@@ -8346,10 +8346,10 @@ void Sema::AddNonMemberOperatorCandidates(
         continue;
       AddOverloadCandidate(FD, F.getPair(), FunctionArgs, CandidateSet);
       if (CandidateSet.getRewriteInfo().shouldAddReversed(*this, Args, FD))
-        AddOverloadCandidate(
-            FD, F.getPair(), {FunctionArgs[1], FunctionArgs[0]}, CandidateSet,
-            false, false, true, false, ADLCallKind::NotADL, std::nullopt,
-            OverloadCandidateParamOrder::Reversed);
+        AddOverloadCandidate(FD, F.getPair(),
+                             {FunctionArgs[1], FunctionArgs[0]}, CandidateSet,
+                             false, false, true, false, ADLCallKind::NotADL, {},
+                             OverloadCandidateParamOrder::Reversed);
     }
   }
 }
@@ -10102,8 +10102,7 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
             FD, FoundDecl, {Args[1], Args[0]}, CandidateSet,
             /*SuppressUserConversions=*/false, PartialOverloading,
             /*AllowExplicit=*/true, /*AllowExplicitConversion=*/false,
-            ADLCallKind::UsesADL, std::nullopt,
-            OverloadCandidateParamOrder::Reversed);
+            ADLCallKind::UsesADL, {}, OverloadCandidateParamOrder::Reversed);
       }
     } else {
       auto *FTD = cast<FunctionTemplateDecl>(*I);
@@ -15954,7 +15953,7 @@ Sema::BuildOverloadedArrowExpr(Scope *S, Expr *Base, SourceLocation OpLoc,
   for (LookupResult::iterator Oper = R.begin(), OperEnd = R.end();
        Oper != OperEnd; ++Oper) {
     AddMethodCandidate(Oper.getPair(), Base->getType(), Base->Classify(Context),
-                       std::nullopt, CandidateSet,
+                       {}, CandidateSet,
                        /*SuppressUserConversion=*/false);
   }
 
@@ -16149,8 +16148,7 @@ Sema::BuildForRangeBeginEndCall(SourceLocation Loc,
       *CallExpr = ExprError();
       return FRS_DiagnosticIssued;
     }
-    *CallExpr =
-        BuildCallExpr(S, MemberRef.get(), Loc, std::nullopt, Loc, nullptr);
+    *CallExpr = BuildCallExpr(S, MemberRef.get(), Loc, {}, Loc, nullptr);
     if (CallExpr->isInvalid()) {
       *CallExpr = ExprError();
       return FRS_DiagnosticIssued;
diff --git a/clang/lib/Sema/SemaPseudoObject.cpp b/clang/lib/Sema/SemaPseudoObject.cpp
index 30ed47e6e56e..c9379d25f780 100644
--- a/clang/lib/Sema/SemaPseudoObject.cpp
+++ b/clang/lib/Sema/SemaPseudoObject.cpp
@@ -746,11 +746,11 @@ ExprResult ObjCPropertyOpBuilder::buildGet() {
     assert(InstanceReceiver || RefExpr->isSuperReceiver());
     msg = S.ObjC().BuildInstanceMessageImplicit(
         InstanceReceiver, receiverType, GenericLoc, Getter->getSelector(),
-        Getter, std::nullopt);
+        Getter, {});
   } else {
     msg = S.ObjC().BuildClassMessageImplicit(
         receiverType, RefExpr->isSuperReceiver(), GenericLoc,
-        Getter->getSelector(), Getter, std::nullopt);
+        Getter->getSelector(), Getter, {});
   }
   return msg;
 }
@@ -1131,7 +1131,7 @@ bool ObjCSubscriptOpBuilder::findAtIndexGetter() {
                                                 /*TInfo=*/nullptr,
                                                 SC_None,
                                                 nullptr);
-    AtIndexGetter->setMethodParams(S.Context, Argument, std::nullopt);
+    AtIndexGetter->setMethodParams(S.Context, Argument, {});
   }
 
   if (!AtIndexGetter) {
@@ -1243,7 +1243,7 @@ bool ObjCSubscriptOpBuilder::findAtIndexSetter() {
                                                 SC_None,
                                                 nullptr);
     Params.push_back(key);
-    AtIndexSetter->setMethodParams(S.Context, Params, std::nullopt);
+    AtIndexSetter->setMethodParams(S.Context, Params, {});
   }
 
   if (!AtIndexSetter) {
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index 9e235a46707c..38ae6d8116c3 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -390,7 +390,7 @@ void Sema::DiagnoseUnusedExprResult(const Stmt *S, unsigned DiagID) {
   // type of the left operand could be used for SFINAE, so technically it is
   // *used*.
   if (DiagID != diag::warn_unused_comma_left_operand || !isSFINAEContext())
-    DiagIfReachable(Loc, S ? llvm::ArrayRef(S) : std::nullopt,
+    DiagIfReachable(Loc, S ? llvm::ArrayRef(S) : llvm::ArrayRef<Stmt *>(),
                     PDiag(DiagID) << R1 << R2);
 }
 
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 62f13610b528..fcf05798d9c7 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -2991,7 +2991,7 @@ TemplateParameterList *Sema::MatchTemplateParametersToScopeSpecifier(
 
       // Fabricate an empty template parameter list for the invented header.
       return TemplateParameterList::Create(Context, SourceLocation(),
-                                           SourceLocation(), std::nullopt,
+                                           SourceLocation(), {},
                                            SourceLocation(), nullptr);
     }
 
diff --git a/clang/lib/Sema/SemaTemplateDeductionGuide.cpp b/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
index 3a87128a7db0..698e0760d808 100644
--- a/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
+++ b/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
@@ -1388,7 +1388,7 @@ void Sema::DeclareImplicitDeductionGuides(TemplateDecl *Template,
   //    additional function template derived as above from a hypothetical
   //    constructor C().
   if (!AddedAny)
-    Transform.buildSimpleDeductionGuide(std::nullopt);
+    Transform.buildSimpleDeductionGuide({});
 
   //    -- An additional function template derived as above from a hypothetical
   //    constructor C(C), called the copy deduction candidate.
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 198442dd9821..6a55861fe5af 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -732,8 +732,7 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(
     : InstantiatingTemplate(
           SemaRef, CodeSynthesisContext::RequirementInstantiation,
           PointOfInstantiation, InstantiationRange, /*Entity=*/nullptr,
-          /*Template=*/nullptr, /*TemplateArgs=*/std::nullopt, &DeductionInfo) {
-}
+          /*Template=*/nullptr, /*TemplateArgs=*/{}, &DeductionInfo) {}
 
 Sema::InstantiatingTemplate::InstantiatingTemplate(
     Sema &SemaRef, SourceLocation PointOfInstantiation,
@@ -742,7 +741,7 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(
     : InstantiatingTemplate(
           SemaRef, CodeSynthesisContext::NestedRequirementConstraintsCheck,
           PointOfInstantiation, InstantiationRange, /*Entity=*/nullptr,
-          /*Template=*/nullptr, /*TemplateArgs=*/std::nullopt) {}
+          /*Template=*/nullptr, /*TemplateArgs=*/{}) {}
 
 Sema::InstantiatingTemplate::InstantiatingTemplate(
     Sema &SemaRef, SourceLocation PointOfInstantiation, const RequiresExpr *RE,
@@ -750,8 +749,7 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(
     : InstantiatingTemplate(
           SemaRef, CodeSynthesisContext::RequirementParameterInstantiation,
           PointOfInstantiation, InstantiationRange, /*Entity=*/nullptr,
-          /*Template=*/nullptr, /*TemplateArgs=*/std::nullopt, &DeductionInfo) {
-}
+          /*Template=*/nullptr, /*TemplateArgs=*/{}, &DeductionInfo) {}
 
 Sema::InstantiatingTemplate::InstantiatingTemplate(
     Sema &SemaRef, SourceLocation PointOfInstantiation,
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 6387fe9f1129..5d043e668457 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -803,7 +803,7 @@ static void maybeSynthesizeBlockSignature(TypeProcessingState &state,
       /*NumExceptions=*/0,
       /*NoexceptExpr=*/nullptr,
       /*ExceptionSpecTokens=*/nullptr,
-      /*DeclsInPrototype=*/std::nullopt, loc, loc, declarator));
+      /*DeclsInPrototype=*/{}, loc, loc, declarator));
 
   // For consistency, make sure the state still has us as processing
   // the decl spec.
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index bd474cc637a4..d24d8d5335e2 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -3423,8 +3423,7 @@ public:
   ExprResult RebuildCXXScalarValueInitExpr(TypeSourceInfo *TSInfo,
                                            SourceLocation LParenLoc,
                                            SourceLocation RParenLoc) {
-    return getSema().BuildCXXTypeConstructExpr(TSInfo, LParenLoc, std::nullopt,
-                                               RParenLoc,
+    return getSema().BuildCXXTypeConstructExpr(TSInfo, LParenLoc, {}, RParenLoc,
                                                /*ListInitialization=*/false);
   }
 
@@ -4243,13 +4242,13 @@ ExprResult TreeTransform<Derived>::TransformInitializer(Expr *Init,
   // Revert value-initialization back to empty parens.
   if (CXXScalarValueInitExpr *VIE = dyn_cast<CXXScalarValueInitExpr>(Init)) {
     SourceRange Parens = VIE->getSourceRange();
-    return getDerived().RebuildParenListExpr(Parens.getBegin(), std::nullopt,
+    return getDerived().RebuildParenListExpr(Parens.getBegin(), {},
                                              Parens.getEnd());
   }
 
   // FIXME: We shouldn't build ImplicitValueInitExprs for direct-initialization.
   if (isa<ImplicitValueInitExpr>(Init))
-    return getDerived().RebuildParenListExpr(SourceLocation(), std::nullopt,
+    return getDerived().RebuildParenListExpr(SourceLocation(), {},
                                              SourceLocation());
 
   // Revert initialization by constructor back to a parenthesized or braced list
@@ -15444,7 +15443,7 @@ TreeTransform<Derived>::TransformSizeOfPackExpr(SizeOfPackExpr *E) {
       return ExprError();
     return getDerived().RebuildSizeOfPackExpr(
         E->getOperatorLoc(), Pack, E->getPackLoc(), E->getRParenLoc(),
-        std::nullopt, std::nullopt);
+        std::nullopt, {});
   }
 
   // Try to compute the result without performing a partial substitution.
@@ -15488,9 +15487,9 @@ TreeTransform<Derived>::TransformSizeOfPackExpr(SizeOfPackExpr *E) {
   // Common case: we could determine the number of expansions without
   // substituting.
   if (Result)
-    return getDerived().RebuildSizeOfPackExpr(
-        E->getOperatorLoc(), E->getPack(), E->getPackLoc(), E->getRParenLoc(),
-        *Result, std::nullopt);
+    return getDerived().RebuildSizeOfPackExpr(E->getOperatorLoc(), E->getPack(),
+                                              E->getPackLoc(),
+                                              E->getRParenLoc(), *Result, {});
 
   TemplateArgumentListInfo TransformedPackArgs(E->getPackLoc(),
                                                E->getPackLoc());
@@ -15521,7 +15520,7 @@ TreeTransform<Derived>::TransformSizeOfPackExpr(SizeOfPackExpr *E) {
 
   return getDerived().RebuildSizeOfPackExpr(E->getOperatorLoc(), E->getPack(),
                                             E->getPackLoc(), E->getRParenLoc(),
-                                            Args.size(), std::nullopt);
+                                            Args.size(), {});
 }
 
 template <typename Derived>
@@ -15565,7 +15564,7 @@ TreeTransform<Derived>::TransformPackIndexingExpr(PackIndexingExpr *E) {
         return ExprError();
       return getDerived().RebuildPackIndexingExpr(
           E->getEllipsisLoc(), E->getRSquareLoc(), Pack.get(), IndexExpr.get(),
-          std::nullopt);
+          {});
     }
     for (unsigned I = 0; I != *NumExpansions; ++I) {
       Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(getSema(), I);
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index f21cbd11b6ab..b5fe16bf6e78 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -207,7 +207,7 @@ namespace clang {
       return Common->PartialSpecializations;
     }
     ArrayRef<Decl> getPartialSpecializations(FunctionTemplateDecl::Common *) {
-      return std::nullopt;
+      return {};
     }
 
     template<typename DeclTy>
diff --git a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
index 1efac6ac1c57..0ead95083e0e 100644
--- a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
@@ -554,7 +554,7 @@ std::optional<SVal> CallEvent::getReturnValueUnderConstruction() const {
 ArrayRef<ParmVarDecl*> AnyFunctionCall::parameters() const {
   const FunctionDecl *D = getDecl();
   if (!D)
-    return std::nullopt;
+    return {};
   return D->parameters();
 }
 
@@ -884,7 +884,7 @@ const BlockDataRegion *BlockCall::getBlockRegion() const {
 ArrayRef<ParmVarDecl*> BlockCall::parameters() const {
   const BlockDecl *D = getDecl();
   if (!D)
-    return std::nullopt;
+    return {};
   return D->parameters();
 }
 
@@ -981,7 +981,7 @@ RuntimeDefinition CXXDestructorCall::getRuntimeDefinition() const {
 ArrayRef<ParmVarDecl*> ObjCMethodCall::parameters() const {
   const ObjCMethodDecl *D = getDecl();
   if (!D)
-    return std::nullopt;
+    return {};
   return D->parameters();
 }
 
diff --git a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
index 693791c3aee8..0a29a050bbc2 100644
--- a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
+++ b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
@@ -1079,7 +1079,7 @@ const VarRegion *MemRegionManager::getVarRegion(const VarDecl *D,
             T = getContext().VoidTy;
           if (!T->getAs<FunctionType>()) {
             FunctionProtoType::ExtProtoInfo Ext;
-            T = getContext().getFunctionType(T, std::nullopt, Ext);
+            T = getContext().getFunctionType(T, {}, Ext);
           }
           T = getContext().getBlockPointerType(T);
 
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 55dd3d7b6911..5475f7143db9 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -4238,8 +4238,7 @@ clang_parseTranslationUnit_Impl(CXIndex CIdx, const char *source_filename,
 
   LibclangInvocationReporter InvocationReporter(
       *CXXIdx, LibclangInvocationReporter::OperationKind::ParseOperation,
-      options, llvm::ArrayRef(*Args), /*InvocationArgs=*/std::nullopt,
-      unsaved_files);
+      options, llvm::ArrayRef(*Args), /*InvocationArgs=*/{}, unsaved_files);
   std::unique_ptr<ASTUnit> Unit = ASTUnit::LoadFromCommandLine(
       Args->data(), Args->data() + Args->size(),
       CXXIdx->getPCHContainerOperations(), Diags,
diff --git a/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp b/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
index 0209703395bc..22fe347c4255 100644
--- a/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
@@ -1371,9 +1371,9 @@ protected:
                   new DiagnosticOptions());
               TextDiagnostic TD(llvm::errs(), AO.ASTCtx.getLangOpts(),
                                 DiagOpts.get());
-              TD.emitDiagnostic(
-                  FullSourceLoc(Loc, SrcMgr), DiagnosticsEngine::Error,
-                  "unexpected diagnostic", std::nullopt, std::nullopt);
+              TD.emitDiagnostic(FullSourceLoc(Loc, SrcMgr),
+                                DiagnosticsEngine::Error,
+                                "unexpected diagnostic", {}, {});
             }
           }
 
diff --git a/clang/unittests/Frontend/ASTUnitTest.cpp b/clang/unittests/Frontend/ASTUnitTest.cpp
index 19b5d9bb4146..bd5d5d009818 100644
--- a/clang/unittests/Frontend/ASTUnitTest.cpp
+++ b/clang/unittests/Frontend/ASTUnitTest.cpp
@@ -168,9 +168,9 @@ TEST_F(ASTUnitTest, LoadFromCommandLineEarlyError) {
 
   std::unique_ptr<ASTUnit> AST = ASTUnit::LoadFromCommandLine(
       &Args[0], &Args[4], PCHContainerOps, Diags, "", false, "", false,
-      CaptureDiagsKind::All, std::nullopt, true, 0, TU_Complete, false, false,
-      false, SkipFunctionBodiesScope::None, false, true, false, false,
-      std::nullopt, &ErrUnit, nullptr);
+      CaptureDiagsKind::All, {}, true, 0, TU_Complete, false, false, false,
+      SkipFunctionBodiesScope::None, false, true, false, false, std::nullopt,
+      &ErrUnit, nullptr);
 
   ASSERT_EQ(AST, nullptr);
   ASSERT_NE(ErrUnit, nullptr);
@@ -195,9 +195,9 @@ TEST_F(ASTUnitTest, LoadFromCommandLineWorkingDirectory) {
 
   std::unique_ptr<ASTUnit> AST = ASTUnit::LoadFromCommandLine(
       &Args[0], &Args[4], PCHContainerOps, Diags, "", false, "", false,
-      CaptureDiagsKind::All, std::nullopt, true, 0, TU_Complete, false, false,
-      false, SkipFunctionBodiesScope::None, false, true, false, false,
-      std::nullopt, &ErrUnit, nullptr);
+      CaptureDiagsKind::All, {}, true, 0, TU_Complete, false, false, false,
+      SkipFunctionBodiesScope::None, false, true, false, false, std::nullopt,
+      &ErrUnit, nullptr);
 
   ASSERT_NE(AST, nullptr);
   ASSERT_FALSE(Diags->hasErrorOccurred());
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index 59c023ca3360..35c8fcf69910 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -1685,7 +1685,7 @@ Intrinsic::DagEmitter::emitDagShuffle(const DagInit *DI) {
                  std::make_unique<Rev>(Arg1.first.getElementSizeInBits()));
   ST.addExpander("MaskExpand",
                  std::make_unique<MaskExpander>(Arg1.first.getNumElements()));
-  ST.evaluate(DI->getArg(2), Elts, std::nullopt);
+  ST.evaluate(DI->getArg(2), Elts, {});
 
   std::string S = "__builtin_shufflevector(" + Arg1.second + ", " + Arg2.second;
   for (auto &E : Elts) {
-- 
GitLab


From f52b89561f2d929c0c6f37fd818229fbcad3b26c Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Thu, 24 Oct 2024 10:27:56 +0100
Subject: [PATCH 039/693] [lldb][AArch64] Read fpmr register from core files
 (#110104)

https://developer.arm.com/documentation/ddi0601/2024-06/AArch64-Registers/FPMR--Floating-point-Mode-Register
for details of the register.
---
 .../RegisterContextPOSIXCore_arm64.cpp        |  13 +++++++
 .../elf-core/RegisterContextPOSIXCore_arm64.h |   1 +
 .../Process/elf-core/RegisterUtilities.h      |   4 +++
 .../aarch64/fpmr/TestAArch64LinuxFPMR.py      |  32 ++++++++++++++----
 lldb/test/API/linux/aarch64/fpmr/corefile     | Bin 0 -> 20480 bytes
 lldb/test/API/linux/aarch64/fpmr/main.c       |   7 +++-
 6 files changed, 50 insertions(+), 7 deletions(-)
 create mode 100644 lldb/test/API/linux/aarch64/fpmr/corefile

diff --git a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp
index 413bf1bbdb2a..2ddf8440aeb0 100644
--- a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp
+++ b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp
@@ -64,6 +64,11 @@ RegisterContextCorePOSIX_arm64::Create(Thread &thread, const ArchSpec &arch,
   if (zt_data.GetByteSize() >= 64)
     opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskZT);
 
+  DataExtractor fpmr_data =
+      getRegset(notes, arch.GetTriple(), AARCH64_FPMR_Desc);
+  if (fpmr_data.GetByteSize() >= sizeof(uint64_t))
+    opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskFPMR);
+
   auto register_info_up =
       std::make_unique<RegisterInfoPOSIX_arm64>(arch, opt_regsets);
   return std::unique_ptr<RegisterContextCorePOSIX_arm64>(
@@ -128,6 +133,9 @@ RegisterContextCorePOSIX_arm64::RegisterContextCorePOSIX_arm64(
   if (m_register_info_up->IsZTPresent())
     m_zt_data = getRegset(notes, target_triple, AARCH64_ZT_Desc);
 
+  if (m_register_info_up->IsFPMRPresent())
+    m_fpmr_data = getRegset(notes, target_triple, AARCH64_FPMR_Desc);
+
   ConfigureRegisterContext();
 }
 
@@ -370,6 +378,11 @@ bool RegisterContextCorePOSIX_arm64::ReadRegister(const RegisterInfo *reg_info,
           *reg_info, reinterpret_cast<uint8_t *>(&m_sme_pseudo_regs) + offset,
           reg_info->byte_size, lldb_private::endian::InlHostByteOrder(), error);
     }
+  } else if (IsFPMR(reg)) {
+    offset = reg_info->byte_offset - m_register_info_up->GetFPMROffset();
+    assert(offset < m_fpmr_data.GetByteSize());
+    value.SetFromMemoryData(*reg_info, m_fpmr_data.GetDataStart() + offset,
+                            reg_info->byte_size, lldb::eByteOrderLittle, error);
   } else
     return false;
 
diff --git a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h
index ff94845e58d6..35588c40c2eb 100644
--- a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h
+++ b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h
@@ -62,6 +62,7 @@ private:
   lldb_private::DataExtractor m_za_data;
   lldb_private::DataExtractor m_mte_data;
   lldb_private::DataExtractor m_zt_data;
+  lldb_private::DataExtractor m_fpmr_data;
 
   SVEState m_sve_state = SVEState::Unknown;
   uint16_t m_sve_vector_length = 0;
diff --git a/lldb/source/Plugins/Process/elf-core/RegisterUtilities.h b/lldb/source/Plugins/Process/elf-core/RegisterUtilities.h
index 12aa5f72371c..b97279b0d735 100644
--- a/lldb/source/Plugins/Process/elf-core/RegisterUtilities.h
+++ b/lldb/source/Plugins/Process/elf-core/RegisterUtilities.h
@@ -144,6 +144,10 @@ constexpr RegsetDesc AARCH64_MTE_Desc[] = {
      llvm::ELF::NT_ARM_TAGGED_ADDR_CTRL},
 };
 
+constexpr RegsetDesc AARCH64_FPMR_Desc[] = {
+    {llvm::Triple::Linux, llvm::Triple::aarch64, llvm::ELF::NT_ARM_FPMR},
+};
+
 constexpr RegsetDesc PPC_VMX_Desc[] = {
     {llvm::Triple::FreeBSD, llvm::Triple::UnknownArch, llvm::ELF::NT_PPC_VMX},
     {llvm::Triple::Linux, llvm::Triple::UnknownArch, llvm::ELF::NT_PPC_VMX},
diff --git a/lldb/test/API/linux/aarch64/fpmr/TestAArch64LinuxFPMR.py b/lldb/test/API/linux/aarch64/fpmr/TestAArch64LinuxFPMR.py
index d022c8eb3d6c..7f8dc811c5df 100644
--- a/lldb/test/API/linux/aarch64/fpmr/TestAArch64LinuxFPMR.py
+++ b/lldb/test/API/linux/aarch64/fpmr/TestAArch64LinuxFPMR.py
@@ -11,9 +11,13 @@ from lldbsuite.test import lldbutil
 class AArch64LinuxFPMR(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
+    # The value set by the inferior.
+    EXPECTED_FPMR = (0b101010 << 32) | 0b101
+    EXPECTED_FPMR_FIELDS = ["LSCALE2 = 42", "F8S1 = FP8_E4M3 | 0x4"]
+
     @skipUnlessArch("aarch64")
     @skipUnlessPlatform(["linux"])
-    def test_fpmr_register(self):
+    def test_fpmr_register_live(self):
         if not self.isAArch64FPMR():
             self.skipTest("FPMR must be present.")
 
@@ -39,16 +43,16 @@ class AArch64LinuxFPMR(TestBase):
         )
 
         # This has been set by the program.
-        expected_fpmr = (0b101010 << 32) | 0b101
         self.expect(
             "register read --all",
-            substrs=["Floating Point Mode Register", f"fpmr = {expected_fpmr:#018x}"],
+            substrs=[
+                "Floating Point Mode Register",
+                f"fpmr = {self.EXPECTED_FPMR:#018x}",
+            ],
         )
 
         if self.hasXMLSupport():
-            self.expect(
-                "register read fpmr", substrs=["LSCALE2 = 42", "F8S1 = FP8_E4M3 | 0x4"]
-            )
+            self.expect("register read fpmr", substrs=self.EXPECTED_FPMR_FIELDS)
 
         # Write a value for the program to find. Same fields but with bit values
         # inverted.
@@ -61,3 +65,19 @@ class AArch64LinuxFPMR(TestBase):
 
         # 0 means the program found the new value in the sysreg as expected.
         self.expect("continue", substrs=["exited with status = 0"])
+
+    @skipIfLLVMTargetMissing("AArch64")
+    def test_fpmr_register_core(self):
+        if not self.isAArch64FPMR():
+            self.skipTest("FPMR must be present.")
+
+        self.runCmd("target create --core corefile")
+
+        self.expect(
+            "register read --all",
+            substrs=[
+                "Floating Point Mode Register",
+                f"fpmr = {self.EXPECTED_FPMR:#018x}",
+            ],
+        )
+        self.expect("register read fpmr", substrs=self.EXPECTED_FPMR_FIELDS)
diff --git a/lldb/test/API/linux/aarch64/fpmr/corefile b/lldb/test/API/linux/aarch64/fpmr/corefile
new file mode 100644
index 0000000000000000000000000000000000000000..8496b6a774df95ab24c4792ab41f78049430f606
GIT binary patch
literal 20480
zcmeHPZ){Y@5g*?_+hCIiO_Y$N^@9-CP&gZcXlUB<e8zxPlOki<C~9)H&xdXNkKCOp
zNT?VKgNZ>v>4y-$xM?79)en3iX`M>Ng+K|W5{1%IDN@}00Y_8~p=ugP;@6pdGro7{
zbJH}9gv9P~Z+2#9c4mJ&yL*Scx23#tnI|g?r1;>MkmLX?X>d^s9tMMXAJajvOTkqg
z5ydop^5G|$Ne?QPdX9UDJyWt%K26aql}akjU>6__lhus>-YzxHzx>*^xeqtldym7p
z;#2~YmG-`$+3p&*yX;cN*Lgj&oi*MrWqgAZ*qNVVH#~ve;CQ<fTn7J<InE;<$Ivyt
zlsJulWwtZN`%A&4B+X9rg}VHyI4$mPRhk{^jU}|pPWfBKb~R~sEKcT=BgjIc#wT{2
z|D#>a?i(5N%t*zP#@2&Q$#2<rtI8<~HD0P<an~IQz)D=JA0Z=D<mq(XSJEY!d5Y*R
ze3P#^_|AE9kBU7nU?G<*H(y@*?f~pQyZ3b`$11*8T1^4}nN?&eYF13J#-yvg4yH9h
zhxV<$qH-vQ*EAypA3ov4LD*2amS3u<#d9(i-;vaum-t0r8R28q51js1@VJC;Opene
zUGYbF^x&JeMwuI^LWT<4j9??ZWiEu)2HQIlj;4ABi<QK5i9T1AV(Y|r#qx^P5*x{=
zNc^sGISpU%ll-qo=d^w#rWxXs*X={hub<{rBxiQme|V21YVA>l8>UhnOT9zXTu{}*
z7Zsm+0-k725e#yf&t?j|zvT$eRP+XYmXb_nIn61RJcp<`rfPl~KP8R7n|UGAxw5gw
z$-R4--)|p0_QLc%y9=1V*6y_5!F&(v-^cv@Y1CvsozVE#;sPqKtXL*<FiDgIY{m`@
zOInzgIyC)k72b5(af-R{I!`|vh3B(xrx?gt$Y&evd#W%G!_GS1H74gXY5fPOXN>IS
zybsWxrHbs;ea^lzm&L`6?XA(``q~Zg^-U%7orMUWLD&(G6}L3i71!3rB8`g{&TDCE
z>)1H2p{;|Ykzl+%xCn}G9i@5>ki(_;=Ksg|{yzrJ`9a4U{u^8W#c2hbcq@d}E6OV?
z^9J*HP0LCHaDP1WmZZMU=HbFbRiVxTMS(I;_Q6xH$t{Ke?eDY=W#PtMS@Gza$FOb|
zrhdjEogCZ)JT4tWHfC(Qor_@O)!2+n1BDGrr?{}8^N`L5YC|#jTx~WMlAE|;`CW42
zkKhLd3tv>ywy<Ip={-y#b9+7{XC`OJ8B^!X?U@vxGHS{hI`{HiIYYdGyyOh&SoqF&
z=^*>LiuetT>vIeDyG8Ad^u1`3OGmzkkUh2Ky4u)K+0p-+-I1Y9bTrinXaqC@8Uc-f
zMnEH=5zq)|1T+E~0gZr0KqH_L&<JP*Gy)m{jetf#BcKt`2xtT}0vZ90fJQ(gpb^jr
zXaqC@8Uc-fMnEH=5zq)|1d<4BNxT2=mHYpEf6-#qYjV*MdAk4a#pe{f%}7`6<jy3k
zrjQkECsgiFdQc1h>=$xakWNj$!zWbi$cEm1)IkPX?TA4r=~AKL(BC3cp}IR}b_EiX
z=#ZS=WcLbKS@?&{eZ-R<^>HBj$fc`$Htu`1=5b?bU!nO)Y5yB2&RBvdorWD)e4(_3
zwU#0^P=1c(GR}1<p9=50ud*o%`g>{W=NTTriyS-=lB>Y5{tV*-;c#;_))s9EM_Sq=
z&EbY<BGJ?uO^R92zP7%0Q&Kvn9E--2(pVxKZ)%7|6CJTOaBeYfsK@Jp>V}}YjTK%H
zToin0F@@?Gdl8q#ub&?Kd;Xos(|y#(sR4_FX%_4p3TIOr6&gPknm>{e-pe0d19QH0
zv-0(eLtA?adq=;0|MZv6d6o~+{K!04(%nkrVRAstn}P=)V{&S9m4a`=QzL$;o(Y{*
z5o2pr$Q-=4{>rFP2HtgM0IGY=nZ5aNWZDj}2D1I|^<#^@7rT7cK*$FR3rz3D6T^>O
ztOM_vHzPB9!(iR$E-IV7tH<gO%nS9C?oPC`A-6ee)UJ*|-q1T<|C~DTJUzVRz<DeF
z!u8Nx|H06#GtY(ru2y70_1jAh{Md`JJiqH#*Qx&y_<AEgm`d$0+IICR>w#Xt_QD+K
zAM}R*R5YdYJPz-VRHwMYkl#G$h3X|}d%?Ev4*TwlZ3U=4KKj)~vqnaZvnYej*8Xf9
zlRpRcbsPTPE{w4&%Ng5Y977?%jl({qKDa?+*pBUC2sHl|$3VW5zOtJw#E5*ohy4~9
z@Kny`&HE!y*v9dC+lUxl&o)6$ZxrqQxlaFe7-QW{TVZz`v0Db7GnXTUy>Fo32E?xQ
zh}B=n*tMbEF_@yT+lKvwAb;qJH+xP9@}7PR@$0-Z)E}CgeXxvhL<~dV?>z;8^WeQo
zalVV=COipC!V!}Mar7!2PgthH@d3sWm(IS!-?t6;+vW?Wv5%AJ`;ga(IXl-<bNL#|
zmSp*xXW%?+N1b;thHZ^CAC{y4LB#6{`Us&s4F0RRJeR{>|Ifw0dmi87c@+PI;me32
z(Z3>HdSTSa!#To@>L}&jjzBW^jt4U5-u78#vwJc3x{I9L>p?pkaya)U4_)>KRPG@r
z=a&*@MFGsc*D&{dS7&3qmpS(iE4%`ncRq#JA&fWj%Fe2?yys$}%|AOB`grXH%%6V3
zxfI|f%8BQHKWca}=DL-((T($b47|<TFbBGZt$~>U16^+;^}#?gfAX3KagJ!rzr!&X
z-IdJcvhUai<;_W7GH*Qg1bKtwZuy1Pzm~_{h;{`yPqgMRS2$-TtDNaJL;WgWin7(%
zLzq{mFmEU?{5TewH~0EG<jR|q)`Pvj;d#1*RP7%gs6KIW)Cl0%X&;G}+eQP9U+pE+
zwpQ7=b|OYM;k<4J;!JDOLh3QMo_{X8?>b|6bP4BT-H0{trq?s)CicI=wZ01wC)#g_
z(%Q+yH%@ghtpCWi&*A;VFX0LMlaFm1*S)Hr{=~LVbN%S3ZNJKO`5gk;lgF<5^Ij?Y
z>j?Az9JSl2dJk?sROCDzW}fzBDyx{E<goBVrg9yZ&NC_)qUx`a&n*m7@p6pGPmu_J
zjoTj+fz!<w_LR>j6FFF3R`w0EXiZ&5TcX2U99$SInK!>far1w)pd?tbFt}j0Bgf01
z@y2)}mZ+_RApQtIu(3AY2*LVIZShU54o$?ImJQKZys5n{g$dU~u&q52g<vAOF#*Ba
zmPAt<cGnR}plV!oP}|xRfncn?zBW+{!AN^+YqTvfQRJUZPWQTKt_60<2cdJhJ36na
z<TwPkUZ>E!e}Ho*_L4tBA7;IDdlswTnV~+!Uh+%mURET3lzQS{^6bkfqrSyn@>Qth
zC#tBz=_@Qeh&5a?WARJ=36)<Ekod)q&;@8uxh=fpyHFWF#ZNjCKUP&K#V37K6#G7|
z2*m?Br3mgKT#gkQldMnqyeRZinnAk%m25BmFY)JAp(WyoX*zySvTwmh)|F6`4I~Wl
zFZ)pg`)9XM#NRK3n(SAI*bAjig<Pb^Dc?`NCWcHUem9la-KuepW20M$b+JnmYgQw0
HOCa!Xfmm<-

literal 0
HcmV?d00001

diff --git a/lldb/test/API/linux/aarch64/fpmr/main.c b/lldb/test/API/linux/aarch64/fpmr/main.c
index bdb7d8f40b64..0a9a9c53eb78 100644
--- a/lldb/test/API/linux/aarch64/fpmr/main.c
+++ b/lldb/test/API/linux/aarch64/fpmr/main.c
@@ -37,5 +37,10 @@ int main(int argc, char *argv[]) {
   uint64_t new_fpmr = get_fpmr(); // Set break point at this line.
   uint64_t expected_fpmr = ((uint64_t)0b010101 << 32) | (uint64_t)0b010;
 
-  return new_fpmr == expected_fpmr ? 0 : 1;
+  // If the debugger failed to update the value, exit uncleanly.
+  // This also allows you to run this program standalone to create a core file.
+  if (new_fpmr != expected_fpmr)
+    __builtin_trap();
+
+  return 0;
 }
-- 
GitLab


From 76f4f950f67b9f3f2d2789abbdffc24ea4506a09 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Fri, 27 Sep 2024 16:44:05 +0100
Subject: [PATCH 040/693] [lldb][AArch64] Add release note for AArch64 Linux
 FPMR register

---
 llvm/docs/ReleaseNotes.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index c8f5d22c1547..504fce4160a5 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -268,6 +268,10 @@ Changes to the LLVM tools
 Changes to LLDB
 ---------------------------------
 
+* LLDB can now read the `fpmr` register from AArch64 Linux processes and core
+  files.
+
+
 Changes to BOLT
 ---------------------------------
 
-- 
GitLab


From cd0373e029fdca7d6d99677b805e728016574a1f Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Thu, 24 Oct 2024 11:21:17 +0100
Subject: [PATCH 041/693] [AArch64] Allow single-element vector FP converts
 with +sme2p2 (#112905)

Follow up to #112213 now that the +sme2p2 feature flag has landed. The
single-element vector variants of FCVTZS, FCVTZU, UCVTF, and SCVTF are
allowed in streaming SVE mode with +sme2p2.

Reference:
-
https://developer.arm.com/documentation/ddi0602/2024-09/SIMD-FP-Instructions/FCVTZS--vector--integer---Floating-point-convert-to-signed-integer--rounding-toward-zero--vector--
-
https://developer.arm.com/documentation/ddi0602/2024-09/SIMD-FP-Instructions/UCVTF--vector--integer---Unsigned-integer-convert-to-floating-point--vector--
-
https://developer.arm.com/documentation/ddi0602/2024-09/SIMD-FP-Instructions/SCVTF--vector--integer---Signed-integer-convert-to-floating-point--vector--
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   | 13 ++--
 .../sve-streaming-mode-cvt-fp-int-fp.ll       | 71 ++++++++++---------
 2 files changed, 43 insertions(+), 41 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 99e3ed31643b..fe3c8578b52a 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -287,6 +287,10 @@ def HasSMEF16F16orSMEF8F16
 def HasNEONandIsStreamingSafe
     : Predicate<"Subtarget->hasNEON()">,
       AssemblerPredicateWithAll<(any_of FeatureNEON), "neon">;
+// A subset of NEON instructions are legal in Streaming SVE mode only with +sme2p2.
+def HasNEONandIsSME2p2StreamingSafe
+    : Predicate<"Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasSME2p2())">,
+    AssemblerPredicateWithAll<(any_of FeatureNEON), "neon">;
 def HasRCPC          : Predicate<"Subtarget->hasRCPC()">,
                                  AssemblerPredicateWithAll<(all_of FeatureRCPC), "rcpc">;
 def HasAltNZCV       : Predicate<"Subtarget->hasAlternativeNZCV()">,
@@ -6315,8 +6319,7 @@ def : Pat<(v2f64 (AArch64frsqrts (v2f64 FPR128:$Rn), (v2f64 FPR128:$Rm))),
 // Some float -> int -> float conversion patterns for which we want to keep the
 // int values in FP registers using the corresponding NEON instructions to
 // avoid more costly int <-> fp register transfers.
-// TODO: Allow these in streaming[-compatible] functions with +sme2p2.
-let Predicates = [HasNEON] in {
+let Predicates = [HasNEONandIsSME2p2StreamingSafe] in {
 def : Pat<(f64 (any_sint_to_fp (i64 (any_fp_to_sint f64:$Rn)))),
           (SCVTFv1i64 (i64 (FCVTZSv1i64 f64:$Rn)))>;
 def : Pat<(f32 (any_sint_to_fp (i32 (any_fp_to_sint f32:$Rn)))),
@@ -6326,8 +6329,7 @@ def : Pat<(f64 (any_uint_to_fp (i64 (any_fp_to_uint f64:$Rn)))),
 def : Pat<(f32 (any_uint_to_fp (i32 (any_fp_to_uint f32:$Rn)))),
           (UCVTFv1i32 (i32 (FCVTZUv1i32 f32:$Rn)))>;
 
-// TODO: Allow these in streaming[-compatible] functions with +sme2p2.
-let Predicates = [HasNEON, HasFullFP16] in {
+let Predicates = [HasNEONandIsSME2p2StreamingSafe, HasFullFP16] in {
 def : Pat<(f16 (any_sint_to_fp (i32 (any_fp_to_sint f16:$Rn)))),
           (SCVTFv1i16 (f16 (FCVTZSv1f16 f16:$Rn)))>;
 def : Pat<(f16 (any_uint_to_fp (i32 (any_fp_to_uint f16:$Rn)))),
@@ -6350,8 +6352,7 @@ def : Pat<(f64 (uint_to_fp (i64 (vector_extract (v2i64 FPR128:$Rn), (i64 0))))),
 
 // fp16: integer extraction from vector must be at least 32-bits to be legal.
 // Actual extraction result is then an in-reg sign-extension of lower 16-bits.
-// TODO: Allow these in streaming[-compatible] functions with +sme2p2.
-let Predicates = [HasNEON, HasFullFP16] in {
+let Predicates = [HasNEONandIsSME2p2StreamingSafe, HasFullFP16] in {
 def : Pat<(f16 (sint_to_fp (i32 (sext_inreg (i32 (vector_extract
                 (v8i16 FPR128:$Rn), (i64 0))), i16)))),
           (SCVTFv1i16 (f16 (EXTRACT_SUBREG (v8i16 FPR128:$Rn), hsub)))>;
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-cvt-fp-int-fp.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-cvt-fp-int-fp.ll
index 9aadf3133ba1..f402463de7be 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-cvt-fp-int-fp.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-cvt-fp-int-fp.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -force-streaming-compatible  < %s | FileCheck %s
-; RUN: llc < %s | FileCheck %s --check-prefix=NON-STREAMING
+; RUN: llc -force-streaming-compatible -mattr=+sme2p2  < %s | FileCheck %s --check-prefix=USE-NEON-NO-GPRS
+; RUN: llc < %s | FileCheck %s --check-prefix=USE-NEON-NO-GPRS
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -11,11 +12,11 @@ define double @t1(double %x) {
 ; CHECK-NEXT:    scvtf d0, x8
 ; CHECK-NEXT:    ret
 ;
-; NON-STREAMING-LABEL: t1:
-; NON-STREAMING:       // %bb.0: // %entry
-; NON-STREAMING-NEXT:    fcvtzs d0, d0
-; NON-STREAMING-NEXT:    scvtf d0, d0
-; NON-STREAMING-NEXT:    ret
+; USE-NEON-NO-GPRS-LABEL: t1:
+; USE-NEON-NO-GPRS:       // %bb.0: // %entry
+; USE-NEON-NO-GPRS-NEXT:    fcvtzs d0, d0
+; USE-NEON-NO-GPRS-NEXT:    scvtf d0, d0
+; USE-NEON-NO-GPRS-NEXT:    ret
 entry:
   %conv = fptosi double %x to i64
   %conv1 = sitofp i64 %conv to double
@@ -29,11 +30,11 @@ define float @t2(float %x) {
 ; CHECK-NEXT:    scvtf s0, w8
 ; CHECK-NEXT:    ret
 ;
-; NON-STREAMING-LABEL: t2:
-; NON-STREAMING:       // %bb.0: // %entry
-; NON-STREAMING-NEXT:    fcvtzs s0, s0
-; NON-STREAMING-NEXT:    scvtf s0, s0
-; NON-STREAMING-NEXT:    ret
+; USE-NEON-NO-GPRS-LABEL: t2:
+; USE-NEON-NO-GPRS:       // %bb.0: // %entry
+; USE-NEON-NO-GPRS-NEXT:    fcvtzs s0, s0
+; USE-NEON-NO-GPRS-NEXT:    scvtf s0, s0
+; USE-NEON-NO-GPRS-NEXT:    ret
 entry:
   %conv = fptosi float %x to i32
   %conv1 = sitofp i32 %conv to float
@@ -49,13 +50,13 @@ define half @t3(half %x)  {
 ; CHECK-NEXT:    fcvt h0, s0
 ; CHECK-NEXT:    ret
 ;
-; NON-STREAMING-LABEL: t3:
-; NON-STREAMING:       // %bb.0: // %entry
-; NON-STREAMING-NEXT:    fcvt s0, h0
-; NON-STREAMING-NEXT:    fcvtzs s0, s0
-; NON-STREAMING-NEXT:    scvtf s0, s0
-; NON-STREAMING-NEXT:    fcvt h0, s0
-; NON-STREAMING-NEXT:    ret
+; USE-NEON-NO-GPRS-LABEL: t3:
+; USE-NEON-NO-GPRS:       // %bb.0: // %entry
+; USE-NEON-NO-GPRS-NEXT:    fcvt s0, h0
+; USE-NEON-NO-GPRS-NEXT:    fcvtzs s0, s0
+; USE-NEON-NO-GPRS-NEXT:    scvtf s0, s0
+; USE-NEON-NO-GPRS-NEXT:    fcvt h0, s0
+; USE-NEON-NO-GPRS-NEXT:    ret
 entry:
   %conv = fptosi half %x to i32
   %conv1 = sitofp i32 %conv to half
@@ -69,11 +70,11 @@ define double @t4(double %x) {
 ; CHECK-NEXT:    ucvtf d0, x8
 ; CHECK-NEXT:    ret
 ;
-; NON-STREAMING-LABEL: t4:
-; NON-STREAMING:       // %bb.0: // %entry
-; NON-STREAMING-NEXT:    fcvtzu d0, d0
-; NON-STREAMING-NEXT:    ucvtf d0, d0
-; NON-STREAMING-NEXT:    ret
+; USE-NEON-NO-GPRS-LABEL: t4:
+; USE-NEON-NO-GPRS:       // %bb.0: // %entry
+; USE-NEON-NO-GPRS-NEXT:    fcvtzu d0, d0
+; USE-NEON-NO-GPRS-NEXT:    ucvtf d0, d0
+; USE-NEON-NO-GPRS-NEXT:    ret
 entry:
   %conv = fptoui double %x to i64
   %conv1 = uitofp i64 %conv to double
@@ -87,11 +88,11 @@ define float @t5(float %x) {
 ; CHECK-NEXT:    ucvtf s0, w8
 ; CHECK-NEXT:    ret
 ;
-; NON-STREAMING-LABEL: t5:
-; NON-STREAMING:       // %bb.0: // %entry
-; NON-STREAMING-NEXT:    fcvtzu s0, s0
-; NON-STREAMING-NEXT:    ucvtf s0, s0
-; NON-STREAMING-NEXT:    ret
+; USE-NEON-NO-GPRS-LABEL: t5:
+; USE-NEON-NO-GPRS:       // %bb.0: // %entry
+; USE-NEON-NO-GPRS-NEXT:    fcvtzu s0, s0
+; USE-NEON-NO-GPRS-NEXT:    ucvtf s0, s0
+; USE-NEON-NO-GPRS-NEXT:    ret
 entry:
   %conv = fptoui float %x to i32
   %conv1 = uitofp i32 %conv to float
@@ -107,13 +108,13 @@ define half @t6(half %x)  {
 ; CHECK-NEXT:    fcvt h0, s0
 ; CHECK-NEXT:    ret
 ;
-; NON-STREAMING-LABEL: t6:
-; NON-STREAMING:       // %bb.0: // %entry
-; NON-STREAMING-NEXT:    fcvt s0, h0
-; NON-STREAMING-NEXT:    fcvtzu s0, s0
-; NON-STREAMING-NEXT:    ucvtf s0, s0
-; NON-STREAMING-NEXT:    fcvt h0, s0
-; NON-STREAMING-NEXT:    ret
+; USE-NEON-NO-GPRS-LABEL: t6:
+; USE-NEON-NO-GPRS:       // %bb.0: // %entry
+; USE-NEON-NO-GPRS-NEXT:    fcvt s0, h0
+; USE-NEON-NO-GPRS-NEXT:    fcvtzu s0, s0
+; USE-NEON-NO-GPRS-NEXT:    ucvtf s0, s0
+; USE-NEON-NO-GPRS-NEXT:    fcvt h0, s0
+; USE-NEON-NO-GPRS-NEXT:    ret
 entry:
   %conv = fptoui half %x to i32
   %conv1 = uitofp i32 %conv to half
-- 
GitLab


From c07abf727220e3bd2c78b129a683ad79f735e43c Mon Sep 17 00:00:00 2001
From: Abid Qadeer <haqadeer@amd.com>
Date: Thu, 24 Oct 2024 11:38:17 +0100
Subject: [PATCH 042/693] [flang][debug] Support fir::ReferenceType. (#113480)

---
 flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp |  5 +++++
 flang/test/Transforms/debug-ref-type.fir              | 10 ++++++++++
 2 files changed, 15 insertions(+)
 create mode 100644 flang/test/Transforms/debug-ref-type.fir

diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index 71e534b4f2e2..84f7873a9fd0 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -511,6 +511,11 @@ DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
                                 /*hasDescriptor=*/false);
   } else if (auto recTy = mlir::dyn_cast_or_null<fir::RecordType>(Ty)) {
     return convertRecordType(recTy, fileAttr, scope, declOp);
+  } else if (auto refTy = mlir::dyn_cast_if_present<fir::ReferenceType>(Ty)) {
+    auto elTy = refTy.getEleTy();
+    return convertPointerLikeType(elTy, fileAttr, scope, declOp,
+                                  /*genAllocated=*/false,
+                                  /*genAssociated=*/false);
   } else if (auto boxTy = mlir::dyn_cast_or_null<fir::BoxType>(Ty)) {
     auto elTy = boxTy.getElementType();
     if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(elTy))
diff --git a/flang/test/Transforms/debug-ref-type.fir b/flang/test/Transforms/debug-ref-type.fir
new file mode 100644
index 000000000000..2b3af485385d
--- /dev/null
+++ b/flang/test/Transforms/debug-ref-type.fir
@@ -0,0 +1,10 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func private @_FortranAioBeginExternalListOutput(i8) -> !fir.ref<i8> loc(#loc1)
+}
+#loc1 = loc("test.f90":5:1)
+
+// CHECK: #[[INT8_TY:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 8, encoding = DW_ATE_signed>
+// CHECK: #[[REF_TY:.*]] = #llvm.di_derived_type<tag = DW_TAG_pointer_type, name = "", baseType = #[[INT8_TY]]{{.*}}>
+// CHECK: #llvm.di_subroutine_type<{{.*}}types = #[[REF_TY]], #[[INT8_TY]]>
-- 
GitLab


From 9575ab28c1d48efba273d189f82be66a7e484da1 Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac@arm.com>
Date: Thu, 24 Oct 2024 11:52:24 +0100
Subject: [PATCH 043/693] [AArch64] Update feature dep. for Armv9.6 extensions
 (#113466)

Co-authored-by: Jonathan Thackray <jonathan.thackray@arm.com>
Co-authored-by: SpencerAbson <Spencer.Abson@arm.com>
---
 llvm/lib/Target/AArch64/AArch64Features.td    | 13 +++---
 .../TargetParser/TargetParserTest.cpp         | 44 +++++++++++++++++--
 2 files changed, 47 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 476d5a4e093e..afa96821984b 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -526,23 +526,23 @@ def FeatureCMPBR : ExtensionWithMArch<"cmpbr", "CMPBR", "FEAT_CMPBR",
   "Enable Armv9.6-A base compare and branch instructions">;
 
 def FeatureF8F32MM: ExtensionWithMArch<"f8f32mm", "F8F32MM", "FEAT_F8F32MM",
-  "Enable Armv9.6-A FP8 to Single-Precision Matrix Multiplication">;
+  "Enable Armv9.6-A FP8 to Single-Precision Matrix Multiplication", [FeatureNEON, FeatureFP8]>;
 
 def FeatureF8F16MM: ExtensionWithMArch<"f8f16mm", "F8F16MM", "FEAT_F8F16MM",
-  "Enable Armv9.6-A FP8 to Half-Precision Matrix Multiplication">;
+  "Enable Armv9.6-A FP8 to Half-Precision Matrix Multiplication", [FeatureNEON, FeatureFP8]>;
 
 def FeatureFPRCVT: ExtensionWithMArch<"fprcvt", "FPRCVT", "FEAT_FPRCVT",
   "Enable Armv9.6-A base convert instructions for SIMD&FP scalar register operands of"
-  " different input and output sizes">;
+  " different input and output sizes", [FeatureFPARMv8]>;
 
 def FeatureLSFE : ExtensionWithMArch<"lsfe", "LSFE", "FEAT_LSFE",
-  "Enable Armv9.6-A base Atomic floating-point in-memory instructions">;
+  "Enable Armv9.6-A base Atomic floating-point in-memory instructions", [FeatureFPARMv8]>;
 
 def FeatureSME2p2: ExtensionWithMArch<"sme2p2", "SME2p2", "FEAT_SME2p2",
   "Enable Armv9.6-A Scalable Matrix Extension 2.2 instructions", [FeatureSME2p1]>;
 
 def FeatureSSVE_AES : ExtensionWithMArch<"ssve-aes", "SSVE_AES", "FEAT_SSVE_AES",
-  "Enable Armv9.6-A SVE2 AES support in streaming SVE mode">;
+  "Enable Armv9.6-A SVE2 AES support in streaming SVE mode", [FeatureSME2, FeatureSVE2AES]>;
 
 def FeatureSVE2p2 : ExtensionWithMArch<"sve2p2", "SVE2p2", "FEAT_SVE2p2",
   "Enable Armv9.6-A Scalable Vector Extension 2.2 instructions", [FeatureSVE2p1]>;
@@ -554,7 +554,8 @@ def FeatureSVEBFSCALE: ExtensionWithMArch<"sve-bfscale", "SVE_BFSCALE", "FEAT_SV
   "Enable Armv9.6-A SVE BFloat16 scaling instructions">;
 
 def FeatureSVE_F16F32MM: ExtensionWithMArch<"sve-f16f32mm", "SVE_F16F32MM", "FEAT_SVE_F16F32MM",
-  "Enable Armv9.6-A FP16 to FP32 Matrix Multiply instructions">;
+  "Enable Armv9.6-A FP16 to FP32 Matrix Multiply instructions", [FeatureSVE]>;
+
 def FeatureLSUI: ExtensionWithMArch<"lsui", "LSUI", "FEAT_LSUI",
   "Enable Armv9.6-A unprivileged load/store instructions">;
 
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index 572006166d0d..51ea9cb6d644 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -1782,7 +1782,15 @@ AArch64ExtensionDependenciesBaseArchTestParams
         {AArch64::ARMV8A, {"nofp", "jscvt"}, {"fp-armv8", "jsconv"}, {}},
         {AArch64::ARMV8A, {"jscvt", "nofp"}, {}, {"fp-armv8", "jsconv"}},
 
-        // simd -> {aes, sha2, sha3, sm4}
+        // fp -> lsfe
+        {AArch64::ARMV9_6A, {"nofp", "lsfe"}, {"fp-armv8", "lsfe"}, {}},
+        {AArch64::ARMV9_6A, {"lsfe", "nofp"}, {}, {"fp-armv8", "lsfe"}},
+
+        // fp -> fprcvt
+        {AArch64::ARMV9_6A, {"nofp", "fprcvt"}, {"fp-armv8", "fprcvt"}, {}},
+        {AArch64::ARMV9_6A, {"fprcvt", "nofp"}, {}, {"fp-armv8", "fprcvt"}},
+
+        // simd -> {aes, sha2, sha3, sm4, f8f16mm, f8f32mm}
         {AArch64::ARMV8A, {"nosimd", "aes"}, {"neon", "aes"}, {}},
         {AArch64::ARMV8A, {"aes", "nosimd"}, {}, {"neon", "aes"}},
         {AArch64::ARMV8A, {"nosimd", "sha2"}, {"neon", "sha2"}, {}},
@@ -1791,6 +1799,10 @@ AArch64ExtensionDependenciesBaseArchTestParams
         {AArch64::ARMV8A, {"sha3", "nosimd"}, {}, {"neon", "sha3"}},
         {AArch64::ARMV8A, {"nosimd", "sm4"}, {"neon", "sm4"}, {}},
         {AArch64::ARMV8A, {"sm4", "nosimd"}, {}, {"neon", "sm4"}},
+        {AArch64::ARMV9_6A, {"nosimd", "f8f16mm"}, {"neon", "f8f16mm"}, {}},
+        {AArch64::ARMV9_6A, {"f8f16mm", "nosimd"}, {}, {"neon", "f8f16mm"}},
+        {AArch64::ARMV9_6A, {"nosimd", "f8f32mm"}, {"neon", "f8f32mm"}, {}},
+        {AArch64::ARMV9_6A, {"f8f32mm", "nosimd"}, {}, {"neon", "f8f32mm"}},
 
         // simd -> {rdm, dotprod, fcma}
         {AArch64::ARMV8A, {"nosimd", "rdm"}, {"neon", "rdm"}, {}},
@@ -1810,13 +1822,21 @@ AArch64ExtensionDependenciesBaseArchTestParams
         {AArch64::ARMV8A, {"nobf16", "sme"}, {"bf16", "sme"}, {}},
         {AArch64::ARMV8A, {"sme", "nobf16"}, {}, {"bf16", "sme"}},
 
-        // sve -> {sve2, f32mm, f64mm}
+        // sve -> {sve2, f32mm, f64mm, sve-f16f32mm}
         {AArch64::ARMV8A, {"nosve", "sve2"}, {"sve", "sve2"}, {}},
         {AArch64::ARMV8A, {"sve2", "nosve"}, {}, {"sve", "sve2"}},
         {AArch64::ARMV8A, {"nosve", "f32mm"}, {"sve", "f32mm"}, {}},
         {AArch64::ARMV8A, {"f32mm", "nosve"}, {}, {"sve", "f32mm"}},
         {AArch64::ARMV8A, {"nosve", "f64mm"}, {"sve", "f64mm"}, {}},
         {AArch64::ARMV8A, {"f64mm", "nosve"}, {}, {"sve", "f64mm"}},
+        {AArch64::ARMV9_6A,
+         {"nosve", "sve-f16f32mm"},
+         {"sve", "sve-f16f32mm"},
+         {}},
+        {AArch64::ARMV9_6A,
+         {"sve-f16f32mm", "nosve"},
+         {},
+         {"sve", "sve-f16f32mm"}},
 
         // sve2 -> {sve2p1, sve2-bitperm, sve2-sha3, sve2-sm4}
         {AArch64::ARMV8A, {"nosve2", "sve2p1"}, {"sve2", "sve2p1"}, {}},
@@ -1861,7 +1881,7 @@ AArch64ExtensionDependenciesBaseArchTestParams
         {AArch64::ARMV8A, {"sme-fa64", "nosme"}, {}, {"sme", "sme-fa64"}},
 
         // sme2 -> {sme2p1, ssve-fp8fma, ssve-fp8dot2, ssve-fp8dot4, sme-f8f16,
-        // sme-f8f32, sme-b16b16}
+        // sme-f8f32, sme-b16b16, ssve-aes}
         {AArch64::ARMV8A, {"nosme2", "sme2p1"}, {"sme2", "sme2p1"}, {}},
         {AArch64::ARMV8A, {"sme2p1", "nosme2"}, {}, {"sme2", "sme2p1"}},
         {AArch64::ARMV8A,
@@ -1894,16 +1914,22 @@ AArch64ExtensionDependenciesBaseArchTestParams
         {AArch64::ARMV8A, {"sme-f8f32", "nosme2"}, {}, {"sme2", "sme-f8f32"}},
         {AArch64::ARMV8A, {"nosme2", "sme-b16b16"}, {"sme2", "sme-b16b16"}, {}},
         {AArch64::ARMV8A, {"sme-b16b16", "nosme2"}, {}, {"sme2", "sme-b16b16"}},
+        {AArch64::ARMV9_6A, {"nosme2", "ssve-aes"}, {"sme2", "ssve-aes"}, {}},
+        {AArch64::ARMV9_6A, {"ssve-aes", "nosme2"}, {}, {"ssve-aes", "sme2"}},
 
         // sme2p1 -> {sme2p2}
         {AArch64::ARMV9_6A, {"nosme2p1", "sme2p2"}, {"sme2p2", "sme2p1"}, {}},
         {AArch64::ARMV9_6A, {"sme2p2", "nosme2p1"}, {}, {"sme2p1", "sme2p2"}},
 
-        // fp8 -> {sme-f8f16, sme-f8f32}
+        // fp8 -> {sme-f8f16, sme-f8f32, f8f16mm, f8f32mm}
         {AArch64::ARMV8A, {"nofp8", "sme-f8f16"}, {"fp8", "sme-f8f16"}, {}},
         {AArch64::ARMV8A, {"sme-f8f16", "nofp8"}, {}, {"fp8", "sme-f8f16"}},
         {AArch64::ARMV8A, {"nofp8", "sme-f8f32"}, {"fp8", "sme-f8f32"}, {}},
         {AArch64::ARMV8A, {"sme-f8f32", "nofp8"}, {}, {"fp8", "sme-f8f32"}},
+        {AArch64::ARMV9_6A, {"nofp8", "f8f16mm"}, {"fp8", "f8f16mm"}, {}},
+        {AArch64::ARMV9_6A, {"f8f16mm", "nofp8"}, {}, {"fp8", "f8f16mm"}},
+        {AArch64::ARMV9_6A, {"nofp8", "f8f32mm"}, {"fp8", "f8f32mm"}, {}},
+        {AArch64::ARMV9_6A, {"f8f32mm", "nofp8"}, {}, {"fp8", "f8f32mm"}},
 
         // lse -> lse128
         {AArch64::ARMV8A, {"nolse", "lse128"}, {"lse", "lse128"}, {}},
@@ -1926,6 +1952,16 @@ AArch64ExtensionDependenciesBaseArchTestParams
         // rcpc -> rcpc3
         {AArch64::ARMV8A, {"norcpc", "rcpc3"}, {"rcpc", "rcpc3"}, {}},
         {AArch64::ARMV8A, {"rcpc3", "norcpc"}, {}, {"rcpc", "rcpc3"}},
+
+        // sve2-aes -> ssve-aes
+        {AArch64::ARMV9_6A,
+         {"nosve2-aes", "ssve-aes"},
+         {"sve2-aes", "ssve-aes"},
+         {}},
+        {AArch64::ARMV9_6A,
+         {"ssve-aes", "nosve2-aes"},
+         {},
+         {"ssve-aes", "sve2-aes"}},
 };
 
 INSTANTIATE_TEST_SUITE_P(
-- 
GitLab


From ea3534b385a713639953fb5dfd287af87b52bead Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Thu, 24 Oct 2024 05:54:35 -0500
Subject: [PATCH 044/693] [flang][OpenMP] Parse AFFINITY clause, lowering not
 supported yet (#113485)

Implement parsing of the AFFINITY clause on TASK construct, conversion
from the parser class to omp::Clause.
Lowering to HLFIR is unsupported, a TODO message is displayed.
---
 flang/include/flang/Parser/dump-parse-tree.h  |  1 +
 flang/include/flang/Parser/parse-tree.h       |  7 ++
 flang/lib/Lower/OpenMP/Clauses.cpp            | 11 ++-
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  3 +-
 flang/lib/Parser/openmp-parsers.cpp           |  7 ++
 flang/lib/Parser/unparse.cpp                  |  4 +
 .../Lower/OpenMP/Todo/affinity-clause.f90     | 10 +++
 flang/test/Parser/OpenMP/affinity-clause.f90  | 79 +++++++++++++++++++
 .../test/Semantics/OpenMP/affinity-clause.f90 |  9 +++
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |  1 +
 10 files changed, 129 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/Todo/affinity-clause.f90
 create mode 100644 flang/test/Parser/OpenMP/affinity-clause.f90
 create mode 100644 flang/test/Semantics/OpenMP/affinity-clause.f90

diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 040065c0cbc0..76d2f164fc4b 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -476,6 +476,7 @@ public:
   NODE(parser, OldParameterStmt)
   NODE(parser, OmpIteratorSpecifier)
   NODE(parser, OmpIteratorModifier)
+  NODE(parser, OmpAffinityClause)
   NODE(parser, OmpAlignedClause)
   NODE(parser, OmpAtomic)
   NODE(parser, OmpAtomicCapture)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index f727525a759e..c1884f6e88d1 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3458,6 +3458,13 @@ struct OmpReductionOperator {
 
 // --- Clauses
 
+// OMP 5.0 2.10.1 affinity([aff-modifier:] locator-list)
+//                aff-modifier: interator-modifier
+struct OmpAffinityClause {
+  TUPLE_CLASS_BOILERPLATE(OmpAffinityClause);
+  std::tuple<std::optional<OmpIteratorModifier>, OmpObjectList> t;
+};
+
 // 2.8.1 aligned-clause -> ALIGNED (variable-name-list[ : scalar-constant])
 struct OmpAlignedClause {
   TUPLE_CLASS_BOILERPLATE(OmpAlignedClause);
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 8974b4211b96..ee3d74a7c631 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -352,8 +352,15 @@ Absent make(const parser::OmpClause::Absent &inp,
 
 Affinity make(const parser::OmpClause::Affinity &inp,
               semantics::SemanticsContext &semaCtx) {
-  // inp -> empty
-  llvm_unreachable("Empty: affinity");
+  // inp.v -> parser::OmpAffinityClause
+  auto &t0 = std::get<std::optional<parser::OmpIteratorModifier>>(inp.v.t);
+  auto &t1 = std::get<parser::OmpObjectList>(inp.v.t);
+
+  auto &&maybeIter =
+      maybeApply([&](auto &&s) { return makeIterator(s, semaCtx); }, t0);
+
+  return Affinity{{/*Iterator=*/std::move(maybeIter),
+                   /*LocatorList=*/makeObjects(t1, semaCtx)}};
 }
 
 Align make(const parser::OmpClause::Align &inp,
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 52a077cd5a79..fc54da8babe6 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2769,7 +2769,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
 
   for (const Clause &clause : clauses) {
     mlir::Location clauseLocation = converter.genLocation(clause.source);
-    if (!std::holds_alternative<clause::Allocate>(clause.u) &&
+    if (!std::holds_alternative<clause::Affinity>(clause.u) &&
+        !std::holds_alternative<clause::Allocate>(clause.u) &&
         !std::holds_alternative<clause::Copyin>(clause.u) &&
         !std::holds_alternative<clause::Copyprivate>(clause.u) &&
         !std::holds_alternative<clause::Default>(clause.u) &&
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 4a1daed04f3e..59a8757e58e8 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -177,6 +177,11 @@ TYPE_PARSER(construct<OmpIteratorSpecifier>(
 TYPE_PARSER(construct<OmpIteratorModifier>("ITERATOR" >>
     parenthesized(nonemptyList(sourced(Parser<OmpIteratorSpecifier>{})))))
 
+// [5.0] 2.10.1 affinity([aff-modifier:] locator-list)
+//              aff-modifier: interator-modifier
+TYPE_PARSER(construct<OmpAffinityClause>(
+    maybe(Parser<OmpIteratorModifier>{} / ":"), Parser<OmpObjectList>{}))
+
 // 2.15.3.1 DEFAULT (PRIVATE | FIRSTPRIVATE | SHARED | NONE)
 TYPE_PARSER(construct<OmpDefaultClause>(
     "PRIVATE" >> pure(OmpDefaultClause::Type::Private) ||
@@ -415,6 +420,8 @@ TYPE_PARSER(construct<OmpLastprivateClause>(
 TYPE_PARSER(
     "ACQUIRE" >> construct<OmpClause>(construct<OmpClause::Acquire>()) ||
     "ACQ_REL" >> construct<OmpClause>(construct<OmpClause::AcqRel>()) ||
+    "AFFINITY" >> construct<OmpClause>(construct<OmpClause::Affinity>(
+                      parenthesized(Parser<OmpAffinityClause>{}))) ||
     "ALIGNED" >> construct<OmpClause>(construct<OmpClause::Aligned>(
                      parenthesized(Parser<OmpAlignedClause>{}))) ||
     "ALLOCATE" >> construct<OmpClause>(construct<OmpClause::Allocate>(
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 5870aba0132c..04df988223e8 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2129,6 +2129,10 @@ public:
     Walk(std::get<std::optional<OmpDeviceClause::DeviceModifier>>(x.t), ":");
     Walk(std::get<ScalarIntExpr>(x.t));
   }
+  void Unparse(const OmpAffinityClause &x) {
+    Walk(std::get<std::optional<OmpIteratorModifier>>(x.t), ":");
+    Walk(std::get<OmpObjectList>(x.t));
+  }
   void Unparse(const OmpAlignedClause &x) {
     Walk(std::get<OmpObjectList>(x.t));
     Put(",");
diff --git a/flang/test/Lower/OpenMP/Todo/affinity-clause.f90 b/flang/test/Lower/OpenMP/Todo/affinity-clause.f90
new file mode 100644
index 000000000000..3459dd219e42
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/affinity-clause.f90
@@ -0,0 +1,10 @@
+!RUN: %not_todo_cmd bbc -emit-hlfir -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
+!RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
+
+!CHECK: not yet implemented: Unhandled clause AFFINITY in TASK construct
+subroutine f00(x)
+  integer :: x(10)
+!$omp task affinity(x)
+  x = x + 1
+!$omp end task
+end
diff --git a/flang/test/Parser/OpenMP/affinity-clause.f90 b/flang/test/Parser/OpenMP/affinity-clause.f90
new file mode 100644
index 000000000000..804723cad7b2
--- /dev/null
+++ b/flang/test/Parser/OpenMP/affinity-clause.f90
@@ -0,0 +1,79 @@
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=50 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=50 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f00(x)
+  integer :: x(10)
+!$omp task affinity(x)
+  x = x + 1
+!$omp end task
+end
+
+!UNPARSE: SUBROUTINE f00 (x)
+!UNPARSE:  INTEGER x(10_4)
+!UNPARSE: !$OMP TASK  AFFINITY(x)
+!UNPARSE:   x=x+1_4
+!UNPARSE: !$OMP END TASK
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = task
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Affinity -> OmpAffinityClause
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+
+subroutine f01(x)
+  integer :: x(10)
+!$omp task affinity(x(1), x(3))
+  x = x + 1
+!$omp end task
+end
+
+!UNPARSE: SUBROUTINE f01 (x)
+!UNPARSE:  INTEGER x(10_4)
+!UNPARSE: !$OMP TASK  AFFINITY(x(1_4),x(3_4))
+!UNPARSE:   x=x+1_4
+!UNPARSE: !$OMP END TASK
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = task
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Affinity -> OmpAffinityClause
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> ArrayElement
+!PARSE-TREE: | | | DataRef -> Name = 'x'
+!PARSE-TREE: | | | SectionSubscript -> Integer -> Expr = '1_4'
+!PARSE-TREE: | | | | LiteralConstant -> IntLiteralConstant = '1'
+!PARSE-TREE: | | OmpObject -> Designator -> DataRef -> ArrayElement
+!PARSE-TREE: | | | DataRef -> Name = 'x'
+!PARSE-TREE: | | | SectionSubscript -> Integer -> Expr = '3_4'
+!PARSE-TREE: | | | | LiteralConstant -> IntLiteralConstant = '3'
+
+subroutine f02(x)
+  integer :: x(10)
+!$omp task affinity(iterator(i = 1:3): x(i))
+  x = x + 1
+!$omp end task
+end
+
+!UNPARSE: SUBROUTINE f02 (x)
+!UNPARSE:  INTEGER x(10_4)
+!UNPARSE: !$OMP TASK  AFFINITY(ITERATOR(INTEGER i = 1_4:3_4):x(i))
+!UNPARSE:   x=x+1_4
+!UNPARSE: !$OMP END TASK
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = task
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Affinity -> OmpAffinityClause
+!PARSE-TREE: | | OmpIteratorModifier -> OmpIteratorSpecifier
+!PARSE-TREE: | | | TypeDeclarationStmt
+!PARSE-TREE: | | | | DeclarationTypeSpec -> IntrinsicTypeSpec -> IntegerTypeSpec ->
+!PARSE-TREE: | | | | EntityDecl
+!PARSE-TREE: | | | | | Name = 'i'
+!PARSE-TREE: | | | SubscriptTriplet
+!PARSE-TREE: | | | | Scalar -> Integer -> Expr = '1_4'
+!PARSE-TREE: | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!PARSE-TREE: | | | | Scalar -> Integer -> Expr = '3_4'
+!PARSE-TREE: | | | | | LiteralConstant -> IntLiteralConstant = '3'
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> ArrayElement
+!PARSE-TREE: | | | DataRef -> Name = 'x'
+!PARSE-TREE: | | | SectionSubscript -> Integer -> Expr = 'i'
+!PARSE-TREE: | | | | Designator -> DataRef -> Name = 'i'
diff --git a/flang/test/Semantics/OpenMP/affinity-clause.f90 b/flang/test/Semantics/OpenMP/affinity-clause.f90
new file mode 100644
index 000000000000..53b2c4fc5667
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/affinity-clause.f90
@@ -0,0 +1,9 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=45
+
+subroutine f00(x)
+  integer :: x(10)
+!ERROR: AFFINITY clause is not allowed on directive TASK in OpenMP v4.5, try -fopenmp-version=50
+!$omp task affinity(x)
+  x = x + 1
+!$omp end task
+end
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index f784c37cbe95..1834ad4d037f 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -45,6 +45,7 @@ def OMPC_AdjustArgs : Clause<"adjust_args"> {
 }
 def OMPC_Affinity : Clause<"affinity"> {
   let clangClass = "OMPAffinityClause";
+  let flangClass = "OmpAffinityClause";
 }
 def OMPC_Align : Clause<"align"> {
   let clangClass = "OMPAlignClause";
-- 
GitLab


From 4affb2d59a541d8163404d7e21252b4686f1c3b8 Mon Sep 17 00:00:00 2001
From: Balazs Benics <benicsbalazs@gmail.com>
Date: Thu, 24 Oct 2024 13:22:19 +0200
Subject: [PATCH 045/693] [analyzer] Use dynamic type when invalidating by a
 member function call (#111138)

When instantiating "callable<T>", the "class CallableType" nested type
will only have a declaration in the copy for the instantiation - because
it's not refereed to directly by any other code that would need a
complete definition.

However, in the past, when conservative eval calling member function, we
took the static type of the "this" expr, and looked up the CXXRecordDecl
it refereed to to see if it has any mutable members (to decide if it
needs to refine invalidation or not). Unfortunately, that query needs a
definition, and it asserts otherwise, thus we crashed.

To fix this, we should consult the dynamic type of the object, because
that will have the definition.
I anyways added a check for "hasDefinition" just to be on the safe side.

Fixes #77378
---
 .../Core/PathSensitive/CallEvent.h            |  5 ++
 clang/lib/StaticAnalyzer/Core/CallEvent.cpp   | 57 ++++++++++---------
 clang/test/Analysis/const-method-call.cpp     | 46 +++++++++++++++
 3 files changed, 80 insertions(+), 28 deletions(-)

diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h
index 549c864dc91e..4552df2a2a31 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h
@@ -690,6 +690,11 @@ protected:
       ValueList &Values,
       RegionAndSymbolInvalidationTraits *ETraits) const override;
 
+  /// Returns the decl refered to by the "dynamic type" of the current object
+  /// and if the class can be a sub-class or not.
+  /// If the Pointer is null, the flag has no meaning.
+  std::pair<const CXXRecordDecl *, bool> getDeclForDynamicType() const;
+
 public:
   /// Returns the expression representing the implicit 'this' object.
   virtual const Expr *getCXXThisExpr() const { return nullptr; }
diff --git a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
index 0ead95083e0e..0fdef7487b98 100644
--- a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
@@ -711,18 +711,17 @@ void CXXInstanceCall::getExtraInvalidatedValues(
   if (const auto *D = cast_or_null<CXXMethodDecl>(getDecl())) {
     if (!D->isConst())
       return;
-    // Get the record decl for the class of 'This'. D->getParent() may return a
-    // base class decl, rather than the class of the instance which needs to be
-    // checked for mutable fields.
-    // TODO: We might as well look at the dynamic type of the object.
-    const Expr *Ex = getCXXThisExpr()->IgnoreParenBaseCasts();
-    QualType T = Ex->getType();
-    if (T->isPointerType()) // Arrow or implicit-this syntax?
-      T = T->getPointeeType();
-    const CXXRecordDecl *ParentRecord = T->getAsCXXRecordDecl();
-    assert(ParentRecord);
+
+    // Get the record decl for the class of 'This'. D->getParent() may return
+    // a base class decl, rather than the class of the instance which needs to
+    // be checked for mutable fields.
+    const CXXRecordDecl *ParentRecord = getDeclForDynamicType().first;
+    if (!ParentRecord || !ParentRecord->hasDefinition())
+      return;
+
     if (ParentRecord->hasMutableFields())
       return;
+
     // Preserve CXXThis.
     const MemRegion *ThisRegion = ThisVal.getAsRegion();
     if (!ThisRegion)
@@ -748,6 +747,21 @@ SVal CXXInstanceCall::getCXXThisVal() const {
   return ThisVal;
 }
 
+std::pair<const CXXRecordDecl *, bool>
+CXXInstanceCall::getDeclForDynamicType() const {
+  const MemRegion *R = getCXXThisVal().getAsRegion();
+  if (!R)
+    return {};
+
+  DynamicTypeInfo DynType = getDynamicTypeInfo(getState(), R);
+  if (!DynType.isValid())
+    return {};
+
+  assert(!DynType.getType()->getPointeeType().isNull());
+  return {DynType.getType()->getPointeeCXXRecordDecl(),
+          DynType.canBeASubClass()};
+}
+
 RuntimeDefinition CXXInstanceCall::getRuntimeDefinition() const {
   // Do we have a decl at all?
   const Decl *D = getDecl();
@@ -759,21 +773,7 @@ RuntimeDefinition CXXInstanceCall::getRuntimeDefinition() const {
   if (!MD->isVirtual())
     return AnyFunctionCall::getRuntimeDefinition();
 
-  // Do we know the implicit 'this' object being called?
-  const MemRegion *R = getCXXThisVal().getAsRegion();
-  if (!R)
-    return {};
-
-  // Do we know anything about the type of 'this'?
-  DynamicTypeInfo DynType = getDynamicTypeInfo(getState(), R);
-  if (!DynType.isValid())
-    return {};
-
-  // Is the type a C++ class? (This is mostly a defensive check.)
-  QualType RegionType = DynType.getType()->getPointeeType();
-  assert(!RegionType.isNull() && "DynamicTypeInfo should always be a pointer.");
-
-  const CXXRecordDecl *RD = RegionType->getAsCXXRecordDecl();
+  auto [RD, CanBeSubClass] = getDeclForDynamicType();
   if (!RD || !RD->hasDefinition())
     return {};
 
@@ -800,7 +800,7 @@ RuntimeDefinition CXXInstanceCall::getRuntimeDefinition() const {
   // Does the decl that we found have an implementation?
   const FunctionDecl *Definition;
   if (!Result->hasBody(Definition)) {
-    if (!DynType.canBeASubClass())
+    if (!CanBeSubClass)
       return AnyFunctionCall::getRuntimeDefinition();
     return {};
   }
@@ -808,8 +808,9 @@ RuntimeDefinition CXXInstanceCall::getRuntimeDefinition() const {
   // We found a definition. If we're not sure that this devirtualization is
   // actually what will happen at runtime, make sure to provide the region so
   // that ExprEngine can decide what to do with it.
-  if (DynType.canBeASubClass())
-    return RuntimeDefinition(Definition, R->StripCasts());
+  if (CanBeSubClass)
+    return RuntimeDefinition(Definition,
+                             getCXXThisVal().getAsRegion()->StripCasts());
   return RuntimeDefinition(Definition, /*DispatchRegion=*/nullptr);
 }
 
diff --git a/clang/test/Analysis/const-method-call.cpp b/clang/test/Analysis/const-method-call.cpp
index 8e1fd3b125f0..7da7ca5554a2 100644
--- a/clang/test/Analysis/const-method-call.cpp
+++ b/clang/test/Analysis/const-method-call.cpp
@@ -271,3 +271,49 @@ void checkThatConstMethodCallDoesInvalidateObjectForCircularReferences() {
   // FIXME: Should be UNKNOWN.
   clang_analyzer_eval(t.x); // expected-warning{{TRUE}}
 }
+
+namespace gh77378 {
+template <typename Signature> class callable;
+
+template <typename R> class callable<R()> {
+  struct CallableType {
+    bool operator()();
+  };
+  using MethodType = R (CallableType::*)();
+  CallableType *object_{nullptr};
+  MethodType method_;
+
+public:
+  callable() = default;
+
+  template <typename T>
+  constexpr callable(const T &obj)
+      : object_{reinterpret_cast<CallableType *>(&const_cast<T &>(obj))},
+        method_{reinterpret_cast<MethodType>(
+            static_cast<bool (T::*)() const>(&T::operator()))} {}
+
+  constexpr bool const_method() const {
+    return (object_->*(method_))();
+  }
+
+  callable call() const & {
+    static const auto L = [this]() {
+      while (true) {
+        // This should not crash when conservative eval calling the member function
+        // when it unwinds the call stack due to exhausting the budget or reaching
+        // the inlining limit.
+        if (this->const_method()) {
+          break;
+        }
+      }
+      return true;
+    };
+    return L;
+  }
+};
+
+void entry() {
+  callable<bool()>{}.call().const_method();
+  // expected-warning@-1 {{Address of stack memory associated with temporary object of type 'callable<bool ()>' is still referred to by the static variable 'L' upon returning to the caller.  This will be a dangling reference}}
+}
+} // namespace gh77378
-- 
GitLab


From 76edf72501cd6f66788c631fada95972a797a4a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= <stefan.graenitz@gmail.com>
Date: Wed, 23 Oct 2024 13:21:22 +0200
Subject: [PATCH 046/693] Reland: [lldb] Fix crash missing MSInheritanceAttr
 with DWARF on Windows (#112928)

Member pointers refer to data or function members of a `CXXRecordDecl`,
which require a `MSInheritanceAttr` in order to be complete. Without that
we cannot calculate the size of a member pointer in memory. The attempt
has been causing a crash further down in the clang AST context. In order
to implement the feature, DWARF will need a new attribtue to convey the
information. For the moment, this patch teaches LLDB to handle to
situation and avoid the crash.
---
 .../TypeSystem/Clang/TypeSystemClang.cpp      |  3 +++
 .../SymbolFile/DWARF/x86/member-pointers.cpp  | 24 +++++++++++++++++++
 2 files changed, 27 insertions(+)
 create mode 100644 lldb/test/Shell/SymbolFile/DWARF/x86/member-pointers.cpp

diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index a57cd8efce8a..f5063175d6e0 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -2771,6 +2771,9 @@ static bool GetCompleteQualType(clang::ASTContext *ast,
         ast, llvm::cast<clang::AttributedType>(qual_type)->getModifiedType(),
         allow_completion);
 
+  case clang::Type::MemberPointer:
+    return !qual_type.getTypePtr()->isIncompleteType();
+
   default:
     break;
   }
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/member-pointers.cpp b/lldb/test/Shell/SymbolFile/DWARF/x86/member-pointers.cpp
new file mode 100644
index 000000000000..c7bd95774d33
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/DWARF/x86/member-pointers.cpp
@@ -0,0 +1,24 @@
+// REQUIRES: lld
+
+// Itanium ABI:
+// RUN: %clang --target=x86_64-pc-linux -gdwarf -c -o %t_linux.o %s
+// RUN: %lldb -f %t_linux.o -b -o "target variable mp" | FileCheck %s
+//
+// CHECK: (char SI::*) mp = 0x0000000000000000
+
+// Microsoft ABI:
+// RUN: %clang_cl --target=x86_64-windows-msvc -c -gdwarf -o %t_win.obj -- %s
+// RUN: lld-link /out:%t_win.exe %t_win.obj /nodefaultlib /entry:main /debug
+// RUN: %lldb -f %t_win.exe -b -o "target variable mp" | FileCheck --check-prefix=CHECK-MSVC %s
+//
+// DWARF has no representation of MSInheritanceAttr, so we cannot determine the size
+// of member-pointers yet. For the moment, make sure we don't crash on such variables.
+// CHECK-MSVC: error: Unable to determine byte size.
+
+struct SI {
+  char si;
+};
+
+char SI::*mp = &SI::si;
+
+int main() { return 0; }
-- 
GitLab


From 47c1abf4afd9120921001ef4bc04492cf949dce7 Mon Sep 17 00:00:00 2001
From: Abid Qadeer <haqadeer@amd.com>
Date: Thu, 24 Oct 2024 13:22:28 +0100
Subject: [PATCH 047/693] [flang][debug] Fix array lower bounds in derived type
 members. (#113183)

The lower bound information for the array members of a derived type
can't be obtained from the `DeclareOp`. It has to be extracted from the
`TypeInfoOp`. That was left as FIXME in the code. This PR adds the
missing functionality to fix the issue.

I tried the following approaches before settling on the current one that
is to generate `DITypeAttr` for array members right where the components
are being processed.

1. Generate a temp XDeclareOp with the shift information obtained from
the `TypeInfoOp`. This caused a few issues mostly related to
`unrealized_conversion_cast`.

2. Change the shift operands in the `declOp` that was passed in the
function before calling `convertType`. The code can be seen in the
abcf031a8e5a02f0081e7f293858302e7bf47bec. It essentially looked like the
following. It works correctly but I was not sure if temporarily changing
the `declOp` is the safe thing to do.

```
mlir::OperandRange originalShift = declOp.getShift();
mlir::MutableOperandRange mutableOpRange = declOp.getShiftMutable();
mutableOpRange.assign(shiftOpers);
elemTy = convertType(fieldTy, fileAttr, scope, declOp);
mutableOpRange.assign(originalShift);
```

Fixes #113178.
---
 .../Transforms/DebugTypeGenerator.cpp         | 36 +++++++++++++++++--
 .../test/Transforms/debug-derived-type-2.fir  | 16 +++++++++
 2 files changed, 49 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/Transforms/debug-derived-type-2.fir

diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index 84f7873a9fd0..c84a530eb382 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -15,6 +15,7 @@
 #include "DebugTypeGenerator.h"
 #include "flang/Optimizer/CodeGen/DescriptorModel.h"
 #include "flang/Optimizer/Support/InternalNames.h"
+#include "flang/Optimizer/Support/Utils.h"
 #include "mlir/Pass/Pass.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/BinaryFormat/Dwarf.h"
@@ -298,6 +299,8 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
   fir::TypeInfoOp tiOp = symbolTable->lookup<fir::TypeInfoOp>(Ty.getName());
   unsigned line = (tiOp) ? getLineFromLoc(tiOp.getLoc()) : 1;
 
+  mlir::OpBuilder builder(context);
+  mlir::IntegerType intTy = mlir::IntegerType::get(context, 64);
   std::uint64_t offset = 0;
   for (auto [fieldName, fieldTy] : Ty.getTypeList()) {
     mlir::Type llvmTy;
@@ -307,11 +310,38 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
     else
       llvmTy = llvmTypeConverter.convertType(fieldTy);
 
-    // FIXME: Handle non defaults array bound in derived types
     uint64_t byteSize = dataLayout->getTypeSize(llvmTy);
     unsigned short byteAlign = dataLayout->getTypeABIAlignment(llvmTy);
-    mlir::LLVM::DITypeAttr elemTy =
-        convertType(fieldTy, fileAttr, scope, /*declOp=*/nullptr);
+    std::optional<llvm::ArrayRef<int64_t>> lowerBounds =
+        fir::getComponentLowerBoundsIfNonDefault(Ty, fieldName, module,
+                                                 symbolTable);
+    auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(fieldTy);
+
+    // For members of the derived types, the information about the shift in
+    // lower bounds is not part of the declOp but has to be extracted from the
+    // TypeInfoOp (using getComponentLowerBoundsIfNonDefault).
+    mlir::LLVM::DITypeAttr elemTy;
+    if (lowerBounds && seqTy &&
+        lowerBounds->size() == seqTy.getShape().size()) {
+      llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
+      for (auto [bound, dim] :
+           llvm::zip_equal(*lowerBounds, seqTy.getShape())) {
+        auto countAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, dim));
+        auto lowerAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, bound));
+        auto subrangeTy = mlir::LLVM::DISubrangeAttr::get(
+            context, countAttr, lowerAttr, /*upperBound=*/nullptr,
+            /*stride=*/nullptr);
+        elements.push_back(subrangeTy);
+      }
+      elemTy = mlir::LLVM::DICompositeTypeAttr::get(
+          context, llvm::dwarf::DW_TAG_array_type, /*name=*/nullptr,
+          /*file=*/nullptr, /*line=*/0, /*scope=*/nullptr,
+          convertType(seqTy.getEleTy(), fileAttr, scope, declOp),
+          mlir::LLVM::DIFlags::Zero, /*sizeInBits=*/0, /*alignInBits=*/0,
+          elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
+          /*allocated=*/nullptr, /*associated=*/nullptr);
+    } else
+      elemTy = convertType(fieldTy, fileAttr, scope, /*declOp=*/nullptr);
     offset = llvm::alignTo(offset, byteAlign);
     mlir::LLVM::DIDerivedTypeAttr tyAttr = mlir::LLVM::DIDerivedTypeAttr::get(
         context, llvm::dwarf::DW_TAG_member,
diff --git a/flang/test/Transforms/debug-derived-type-2.fir b/flang/test/Transforms/debug-derived-type-2.fir
new file mode 100644
index 000000000000..63e842619edb
--- /dev/null
+++ b/flang/test/Transforms/debug-derived-type-2.fir
@@ -0,0 +1,16 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  fir.global @_QMmEvar : !fir.type<_QMmTt1{elm:!fir.array<5xi32>,elm2:!fir.array<5x8xi32>}> {} loc(#loc1)
+  fir.type_info @_QMmTt1 noinit nodestroy nofinal : !fir.type<_QMmTt1{elm:!fir.array<5xi32>,elm2:!fir.array<5x8xi32>}> component_info {
+    fir.dt_component "elm" lbs [2]
+    fir.dt_component "elm2" lbs [1, 3]
+  } loc(#loc1)
+}
+#loc1 = loc("derived.f90":24:1)
+
+
+// CHECK-DAG: #[[TY1:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type, {{.*}} = #{{.*}}, elements = #llvm.di_subrange<count = 5 : i64, lowerBound = 2 : i64>>
+// CHECK-DAG: #[[TY2:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type, baseType = #{{.*}}, elements = #llvm.di_subrange<count = 5 : i64, lowerBound = 1 : i64>, #llvm.di_subrange<count = 8 : i64, lowerBound = 3 : i64>>
+// CHECK-DAG: #llvm.di_derived_type<tag = DW_TAG_member, name = "elm", baseType = #[[TY1]], sizeInBits = {{.*}}, alignInBits = {{.*}}>
+// CHECK-DAG: #llvm.di_derived_type<tag = DW_TAG_member, name = "elm2", baseType = #[[TY2]], sizeInBits = {{.*}}, alignInBits = {{.*}}>
-- 
GitLab


From 927559d27d5bfe97ad668a562489d732b8bc246f Mon Sep 17 00:00:00 2001
From: Longsheng Mou <longshengmou@gmail.com>
Date: Thu, 24 Oct 2024 20:28:42 +0800
Subject: [PATCH 048/693] [mlir][vector] Fix a crash in `VectorToGPU` (#113454)

This PR fixes a crash in `VectorToGPU` when the operand of `extOp` is a
function argument, which cannot be retrieved using `getDefiningOp`.
Fixes #107967.
---
 .../Conversion/VectorToGPU/VectorToGPU.cpp    |  4 +++-
 .../VectorToGPU/vector-to-mma-ops.mlir        | 19 +++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
index e059d31ca584..034f3e2d16e9 100644
--- a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
+++ b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
@@ -200,7 +200,9 @@ static bool broadcastSupportsMMAMatrixType(vector::BroadcastOp broadcastOp) {
 /// Return true if this integer extend op can be folded into a contract op.
 template <typename ExtOpTy>
 static bool integerExtendSupportsMMAMatrixType(ExtOpTy extOp) {
-  if (!isa<vector::TransferReadOp>(extOp.getOperand().getDefiningOp()))
+  auto transferReadOp =
+      extOp.getOperand().template getDefiningOp<vector::TransferReadOp>();
+  if (!transferReadOp)
     return false;
   return llvm::all_of(extOp->getUsers(), llvm::IsaPred<vector::ContractionOp>);
 }
diff --git a/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir b/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir
index 8526ff139259..b8ac63f89af3 100644
--- a/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir
+++ b/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir
@@ -517,3 +517,22 @@ func.func @cast_f16_to_f32_read(%arg0: memref<16x16xf16>, %arg1: memref<16x16xf1
   vector.transfer_write %D, %arg3[%c0, %c0] {in_bounds = [true, true]} : vector<16x16xf32>, memref<16x16xf32>
   return
 }
+
+// -----
+
+#map = affine_map<(d0, d1, d2) -> (d0, d2)>
+#map1 = affine_map<(d0, d1, d2) -> (d2, d1)>
+#map2 = affine_map<(d0, d1, d2) -> (d0, d1)>
+
+// Ensure that no crash occurs when the predecessor operation
+// of `ext` is not `transfer_read`.
+
+// CHECK-LABEL: func @test_unsupported
+//       CHECK:    vector.contract
+func.func @test_unsupported(%arg0: vector<4x4xi32>, %arg1: vector<4x4xi32>, %arg2: vector<4x4xi64>) -> vector<4x4xi64 > {
+  %0 = arith.extui %arg0 : vector<4x4xi32> to vector<4x4xi64>
+  %1 = arith.extui %arg1 : vector<4x4xi32> to vector<4x4xi64>
+  %2 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>}
+                        %0, %1, %arg2 : vector<4x4xi64>, vector<4x4xi64> into vector<4x4xi64>
+  return %2 : vector<4x4xi64>
+}
-- 
GitLab


From 7e61d893739ff309a1c2e01dee31b1a5e6ed91b8 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Thu, 24 Oct 2024 12:32:49 +0000
Subject: [PATCH 049/693] [mlir] Apply ClangTidy performance finding

loop variable is copied but only used as const reference
---
 mlir/tools/mlir-rewrite/mlir-rewrite.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mlir/tools/mlir-rewrite/mlir-rewrite.cpp b/mlir/tools/mlir-rewrite/mlir-rewrite.cpp
index e70aa5d41aa0..bbb6bd6617a1 100644
--- a/mlir/tools/mlir-rewrite/mlir-rewrite.cpp
+++ b/mlir/tools/mlir-rewrite/mlir-rewrite.cpp
@@ -38,7 +38,7 @@ SMRange getOpRange(const OperationDefinition &op) {
   const char *startOp = op.scopeLoc.Start.getPointer();
   const char *endOp = op.scopeLoc.End.getPointer();
 
-  for (auto res : op.resultGroups) {
+  for (const auto &res : op.resultGroups) {
     SMRange range = res.definition.loc;
     startOp = std::min(startOp, range.Start.getPointer());
   }
@@ -317,7 +317,7 @@ static mlir::RewriterRegistration rewriteSimpleRename("simple-rename",
 
 // Rewriter that insert range markers.
 LogicalResult markRanges(RewritePad &rewriteState, raw_ostream &os) {
-  for (auto it : rewriteState.getOpDefs()) {
+  for (const auto &it : rewriteState.getOpDefs()) {
     auto [startOp, endOp] = getOpRange(it);
 
     rewriteState.insertText(startOp, "<");
-- 
GitLab


From 37832d5de2abb425c460375a4054478ce9f948fe Mon Sep 17 00:00:00 2001
From: Abid Qadeer <haqadeer@amd.com>
Date: Thu, 24 Oct 2024 13:37:32 +0100
Subject: [PATCH 050/693] [flang][debug] Support fir.vector type. (#112951)

This PR converts the `fir.vector<>` to
`DICompositeTypeAttr(DW_TAG_array_type)` with `vector` flag set.
---
 .../Transforms/DebugTypeGenerator.cpp         | 35 +++++++++++++++++++
 .../Optimizer/Transforms/DebugTypeGenerator.h |  4 +++
 flang/test/Transforms/debug-vector-type.fir   | 23 ++++++++++++
 3 files changed, 62 insertions(+)
 create mode 100644 flang/test/Transforms/debug-vector-type.fir

diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index c84a530eb382..1ab6c76dae8e 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -431,6 +431,39 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertSequenceType(
       /*associated=*/nullptr);
 }
 
+mlir::LLVM::DITypeAttr DebugTypeGenerator::convertVectorType(
+    fir::VectorType vecTy, mlir::LLVM::DIFileAttr fileAttr,
+    mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp) {
+  mlir::MLIRContext *context = module.getContext();
+
+  llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
+  mlir::LLVM::DITypeAttr elemTy =
+      convertType(vecTy.getEleTy(), fileAttr, scope, declOp);
+  auto intTy = mlir::IntegerType::get(context, 64);
+  auto countAttr =
+      mlir::IntegerAttr::get(intTy, llvm::APInt(64, vecTy.getLen()));
+  auto subrangeTy = mlir::LLVM::DISubrangeAttr::get(
+      context, countAttr, /*lowerBound=*/nullptr, /*upperBound=*/nullptr,
+      /*stride=*/nullptr);
+  elements.push_back(subrangeTy);
+  mlir::Type llvmTy = llvmTypeConverter.convertType(vecTy.getEleTy());
+  uint64_t sizeInBits = dataLayout->getTypeSize(llvmTy) * vecTy.getLen() * 8;
+  std::string name("vector");
+  // The element type of the vector must be integer or real so it will be a
+  // DIBasicTypeAttr.
+  if (auto ty = mlir::dyn_cast_if_present<mlir::LLVM::DIBasicTypeAttr>(elemTy))
+    name += " " + ty.getName().str();
+
+  name += " (" + std::to_string(vecTy.getLen()) + ")";
+  return mlir::LLVM::DICompositeTypeAttr::get(
+      context, llvm::dwarf::DW_TAG_array_type,
+      mlir::StringAttr::get(context, name),
+      /*file=*/nullptr, /*line=*/0, /*scope=*/nullptr, elemTy,
+      mlir::LLVM::DIFlags::Vector, sizeInBits, /*alignInBits=*/0, elements,
+      /*dataLocation=*/nullptr, /*rank=*/nullptr, /*allocated=*/nullptr,
+      /*associated=*/nullptr);
+}
+
 mlir::LLVM::DITypeAttr DebugTypeGenerator::convertCharacterType(
     fir::CharacterType charTy, mlir::LLVM::DIFileAttr fileAttr,
     mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp,
@@ -546,6 +579,8 @@ DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
     return convertPointerLikeType(elTy, fileAttr, scope, declOp,
                                   /*genAllocated=*/false,
                                   /*genAssociated=*/false);
+  } else if (auto vecTy = mlir::dyn_cast_or_null<fir::VectorType>(Ty)) {
+    return convertVectorType(vecTy, fileAttr, scope, declOp);
   } else if (auto boxTy = mlir::dyn_cast_or_null<fir::BoxType>(Ty)) {
     auto elTy = boxTy.getElementType();
     if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(elTy))
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
index ff600e751eb0..eeefb6c463d9 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
@@ -43,6 +43,10 @@ private:
                                              mlir::LLVM::DIFileAttr fileAttr,
                                              mlir::LLVM::DIScopeAttr scope,
                                              fir::cg::XDeclareOp declOp);
+  mlir::LLVM::DITypeAttr convertVectorType(fir::VectorType vecTy,
+                                           mlir::LLVM::DIFileAttr fileAttr,
+                                           mlir::LLVM::DIScopeAttr scope,
+                                           fir::cg::XDeclareOp declOp);
 
   /// The 'genAllocated' is true when we want to generate 'allocated' field
   /// in the DICompositeType. It is needed for the allocatable arrays.
diff --git a/flang/test/Transforms/debug-vector-type.fir b/flang/test/Transforms/debug-vector-type.fir
new file mode 100644
index 000000000000..63846ce006c6
--- /dev/null
+++ b/flang/test/Transforms/debug-vector-type.fir
@@ -0,0 +1,23 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+func.func private @foo1(%arg0: !fir.vector<20:bf16>)
+// CHECK-DAG: #[[F16:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real", sizeInBits = 16, encoding = DW_ATE_float>
+// CHECK-DAG: #llvm.di_composite_type<tag = DW_TAG_array_type, name = "vector real (20)", baseType = #[[F16]], flags = Vector, sizeInBits = 320, elements = #llvm.di_subrange<count = 20 : i64>>
+
+func.func private @foo2(%arg0: !fir.vector<30:f32>)
+// CHECK-DAG: #[[F32:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real", sizeInBits = 32, encoding = DW_ATE_float>
+// CHECK-DAG: #llvm.di_composite_type<tag = DW_TAG_array_type, name = "vector real (30)", baseType = #[[F32]], flags = Vector, sizeInBits = 960, elements = #llvm.di_subrange<count = 30 : i64>>
+
+func.func private @foo3(%arg0: !fir.vector<10:f64>)
+// CHECK-DAG: #[[F64:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real", sizeInBits = 64, encoding = DW_ATE_float>
+// CHECK-DAG: #llvm.di_composite_type<tag = DW_TAG_array_type, name = "vector real (10)", baseType = #[[F64]], flags = Vector, sizeInBits = 640, elements = #llvm.di_subrange<count = 10 : i64>>
+
+func.func private @foo4(%arg0: !fir.vector<5:i32>)
+// CHECK-DAG: #[[I32:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 32, encoding = DW_ATE_signed>
+// CHECK-DAG: #llvm.di_composite_type<tag = DW_TAG_array_type, name = "vector integer (5)", baseType = #[[I32]], flags = Vector, sizeInBits = 160, elements = #llvm.di_subrange<count = 5 : i64>>
+
+func.func private @foo5(%arg0: !fir.vector<2:i64>)
+// CHECK-DAG: #[[I64:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 64, encoding = DW_ATE_signed>
+// CHECK-DAG: #llvm.di_composite_type<tag = DW_TAG_array_type, name = "vector integer (2)", baseType = #[[I64]], flags = Vector, sizeInBits = 128, elements = #llvm.di_subrange<count = 2 : i64>>
+}
-- 
GitLab


From 046b87fb3ca8e84c42c915f2bd66a95004e136fc Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 23 Oct 2024 07:50:25 -0700
Subject: [PATCH 051/693] Fix MSVC not all control paths return a value
 warning. NFC.

---
 .../llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h      | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
index 8f698f44c7c0..bcfafd75d4ca 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
@@ -41,6 +41,7 @@ struct ToStr {
     case LegalityResultID::Widen:
       return "Widen";
     }
+    llvm_unreachable("Unknown LegalityResultID enum");
   }
 
   static const char *getVecReason(ResultReason Reason) {
@@ -50,6 +51,7 @@ struct ToStr {
     case ResultReason::DiffTypes:
       return "DiffTypes";
     }
+    llvm_unreachable("Unknown ResultReason enum");
   }
 };
 #endif // NDEBUG
-- 
GitLab


From b34d64921b2f878b6e1ac7205fc4b13d54a7d8db Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 23 Oct 2024 07:53:07 -0700
Subject: [PATCH 052/693] [X86] ReplaceNodeResults - adjust assert to allow XOP
 or GFNI subtargets to split i64 BITREVERSE nodes on 32-bit targets

Fixes #113353
Fixes #113034
---
 llvm/lib/Target/X86/X86ISelLowering.cpp |   6 +-
 llvm/test/CodeGen/X86/bitreverse.ll     | 433 +++++++++++++++++-------
 2 files changed, 314 insertions(+), 125 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7f4dc12a2083..a6d77873ec29 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -34158,9 +34158,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
   }
   case ISD::BITREVERSE: {
     assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!");
-    assert(Subtarget.hasXOP() && "Expected XOP");
-    // We can use VPPERM by copying to a vector register and back. We'll need
-    // to move the scalar in two i32 pieces.
+    assert((Subtarget.hasXOP() || Subtarget.hasGFNI()) && "Expected XOP/GFNI");
+    // We can use VPPERM/GF2P8AFFINEQB by copying to a vector register and back.
+    // We'll need to move the scalar in two i32 pieces.
     Results.push_back(LowerBITREVERSE(SDValue(N, 0), Subtarget, DAG));
     return;
   }
diff --git a/llvm/test/CodeGen/X86/bitreverse.ll b/llvm/test/CodeGen/X86/bitreverse.ll
index e256b811ee83..d92e1a1e7b9d 100644
--- a/llvm/test/CodeGen/X86/bitreverse.ll
+++ b/llvm/test/CodeGen/X86/bitreverse.ll
@@ -2,7 +2,8 @@
 ; RUN: llc < %s -mtriple=i686-unknown | FileCheck %s --check-prefixes=CHECK,X86
 ; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s --check-prefixes=CHECK,X64
 ; RUN: llc < %s -mtriple=i686-unknown -mattr=+xop | FileCheck %s --check-prefixes=CHECK,X86XOP
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512bw,+avx512vl,+gfni | FileCheck %s --check-prefixes=CHECK,GFNI
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx512bw,+avx512vl,+gfni | FileCheck %s --check-prefixes=CHECK,GFNI,X86GFNI
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512bw,+avx512vl,+gfni | FileCheck %s --check-prefixes=CHECK,GFNI,X64GFNI
 
 ; These tests just check that the plumbing is in place for @llvm.bitreverse. The
 ; actual output is massive at the moment as llvm.bitreverse is not yet legal.
@@ -86,11 +87,17 @@ define <2 x i16> @test_bitreverse_v2i16(<2 x i16> %a) nounwind {
 ; X86XOP-NEXT:    vpperm {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0, %xmm0
 ; X86XOP-NEXT:    retl
 ;
-; GFNI-LABEL: test_bitreverse_v2i16:
-; GFNI:       # %bb.0:
-; GFNI-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
-; GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
-; GFNI-NEXT:    retq
+; X86GFNI-LABEL: test_bitreverse_v2i16:
+; X86GFNI:       # %bb.0:
+; X86GFNI-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
+; X86GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}{1to2}, %xmm0, %xmm0
+; X86GFNI-NEXT:    retl
+;
+; X64GFNI-LABEL: test_bitreverse_v2i16:
+; X64GFNI:       # %bb.0:
+; X64GFNI-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
+; X64GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; X64GFNI-NEXT:    retq
   %b = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> %a)
   ret <2 x i16> %b
 }
@@ -170,13 +177,23 @@ define i64 @test_bitreverse_i64(i64 %a) nounwind {
 ; X86XOP-NEXT:    vpextrd $1, %xmm0, %edx
 ; X86XOP-NEXT:    retl
 ;
-; GFNI-LABEL: test_bitreverse_i64:
-; GFNI:       # %bb.0:
-; GFNI-NEXT:    vmovq %rdi, %xmm0
-; GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
-; GFNI-NEXT:    vmovq %xmm0, %rax
-; GFNI-NEXT:    bswapq %rax
-; GFNI-NEXT:    retq
+; X86GFNI-LABEL: test_bitreverse_i64:
+; X86GFNI:       # %bb.0:
+; X86GFNI-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; X86GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}{1to2}, %xmm0, %xmm0
+; X86GFNI-NEXT:    vpextrd $1, %xmm0, %eax
+; X86GFNI-NEXT:    bswapl %eax
+; X86GFNI-NEXT:    vmovd %xmm0, %edx
+; X86GFNI-NEXT:    bswapl %edx
+; X86GFNI-NEXT:    retl
+;
+; X64GFNI-LABEL: test_bitreverse_i64:
+; X64GFNI:       # %bb.0:
+; X64GFNI-NEXT:    vmovq %rdi, %xmm0
+; X64GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; X64GFNI-NEXT:    vmovq %xmm0, %rax
+; X64GFNI-NEXT:    bswapq %rax
+; X64GFNI-NEXT:    retq
   %b = call i64 @llvm.bitreverse.i64(i64 %a)
   ret i64 %b
 }
@@ -235,13 +252,21 @@ define i32 @test_bitreverse_i32(i32 %a) nounwind {
 ; X86XOP-NEXT:    vmovd %xmm0, %eax
 ; X86XOP-NEXT:    retl
 ;
-; GFNI-LABEL: test_bitreverse_i32:
-; GFNI:       # %bb.0:
-; GFNI-NEXT:    vmovd %edi, %xmm0
-; GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
-; GFNI-NEXT:    vmovd %xmm0, %eax
-; GFNI-NEXT:    bswapl %eax
-; GFNI-NEXT:    retq
+; X86GFNI-LABEL: test_bitreverse_i32:
+; X86GFNI:       # %bb.0:
+; X86GFNI-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}{1to2}, %xmm0, %xmm0
+; X86GFNI-NEXT:    vmovd %xmm0, %eax
+; X86GFNI-NEXT:    bswapl %eax
+; X86GFNI-NEXT:    retl
+;
+; X64GFNI-LABEL: test_bitreverse_i32:
+; X64GFNI:       # %bb.0:
+; X64GFNI-NEXT:    vmovd %edi, %xmm0
+; X64GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; X64GFNI-NEXT:    vmovd %xmm0, %eax
+; X64GFNI-NEXT:    bswapl %eax
+; X64GFNI-NEXT:    retq
   %b = call i32 @llvm.bitreverse.i32(i32 %a)
   ret i32 %b
 }
@@ -303,14 +328,23 @@ define i24 @test_bitreverse_i24(i24 %a) nounwind {
 ; X86XOP-NEXT:    shrl $8, %eax
 ; X86XOP-NEXT:    retl
 ;
-; GFNI-LABEL: test_bitreverse_i24:
-; GFNI:       # %bb.0:
-; GFNI-NEXT:    vmovd %edi, %xmm0
-; GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
-; GFNI-NEXT:    vmovd %xmm0, %eax
-; GFNI-NEXT:    bswapl %eax
-; GFNI-NEXT:    shrl $8, %eax
-; GFNI-NEXT:    retq
+; X86GFNI-LABEL: test_bitreverse_i24:
+; X86GFNI:       # %bb.0:
+; X86GFNI-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}{1to2}, %xmm0, %xmm0
+; X86GFNI-NEXT:    vmovd %xmm0, %eax
+; X86GFNI-NEXT:    bswapl %eax
+; X86GFNI-NEXT:    shrl $8, %eax
+; X86GFNI-NEXT:    retl
+;
+; X64GFNI-LABEL: test_bitreverse_i24:
+; X64GFNI:       # %bb.0:
+; X64GFNI-NEXT:    vmovd %edi, %xmm0
+; X64GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; X64GFNI-NEXT:    vmovd %xmm0, %eax
+; X64GFNI-NEXT:    bswapl %eax
+; X64GFNI-NEXT:    shrl $8, %eax
+; X64GFNI-NEXT:    retq
   %b = call i24 @llvm.bitreverse.i24(i24 %a)
   ret i24 %b
 }
@@ -372,14 +406,23 @@ define i16 @test_bitreverse_i16(i16 %a) nounwind {
 ; X86XOP-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86XOP-NEXT:    retl
 ;
-; GFNI-LABEL: test_bitreverse_i16:
-; GFNI:       # %bb.0:
-; GFNI-NEXT:    vmovd %edi, %xmm0
-; GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
-; GFNI-NEXT:    vmovd %xmm0, %eax
-; GFNI-NEXT:    rolw $8, %ax
-; GFNI-NEXT:    # kill: def $ax killed $ax killed $eax
-; GFNI-NEXT:    retq
+; X86GFNI-LABEL: test_bitreverse_i16:
+; X86GFNI:       # %bb.0:
+; X86GFNI-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}{1to2}, %xmm0, %xmm0
+; X86GFNI-NEXT:    vmovd %xmm0, %eax
+; X86GFNI-NEXT:    rolw $8, %ax
+; X86GFNI-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86GFNI-NEXT:    retl
+;
+; X64GFNI-LABEL: test_bitreverse_i16:
+; X64GFNI:       # %bb.0:
+; X64GFNI-NEXT:    vmovd %edi, %xmm0
+; X64GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; X64GFNI-NEXT:    vmovd %xmm0, %eax
+; X64GFNI-NEXT:    rolw $8, %ax
+; X64GFNI-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64GFNI-NEXT:    retq
   %b = call i16 @llvm.bitreverse.i16(i16 %a)
   ret i16 %b
 }
@@ -430,13 +473,21 @@ define i8 @test_bitreverse_i8(i8 %a) {
 ; X86XOP-NEXT:    # kill: def $al killed $al killed $eax
 ; X86XOP-NEXT:    retl
 ;
-; GFNI-LABEL: test_bitreverse_i8:
-; GFNI:       # %bb.0:
-; GFNI-NEXT:    vmovd %edi, %xmm0
-; GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
-; GFNI-NEXT:    vmovd %xmm0, %eax
-; GFNI-NEXT:    # kill: def $al killed $al killed $eax
-; GFNI-NEXT:    retq
+; X86GFNI-LABEL: test_bitreverse_i8:
+; X86GFNI:       # %bb.0:
+; X86GFNI-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}{1to2}, %xmm0, %xmm0
+; X86GFNI-NEXT:    vmovd %xmm0, %eax
+; X86GFNI-NEXT:    # kill: def $al killed $al killed $eax
+; X86GFNI-NEXT:    retl
+;
+; X64GFNI-LABEL: test_bitreverse_i8:
+; X64GFNI:       # %bb.0:
+; X64GFNI-NEXT:    vmovd %edi, %xmm0
+; X64GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; X64GFNI-NEXT:    vmovd %xmm0, %eax
+; X64GFNI-NEXT:    # kill: def $al killed $al killed $eax
+; X64GFNI-NEXT:    retq
   %b = call i8 @llvm.bitreverse.i8(i8 %a)
   ret i8 %b
 }
@@ -489,14 +540,23 @@ define i4 @test_bitreverse_i4(i4 %a) {
 ; X86XOP-NEXT:    # kill: def $al killed $al killed $eax
 ; X86XOP-NEXT:    retl
 ;
-; GFNI-LABEL: test_bitreverse_i4:
-; GFNI:       # %bb.0:
-; GFNI-NEXT:    vmovd %edi, %xmm0
-; GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
-; GFNI-NEXT:    vmovd %xmm0, %eax
-; GFNI-NEXT:    shrb $4, %al
-; GFNI-NEXT:    # kill: def $al killed $al killed $eax
-; GFNI-NEXT:    retq
+; X86GFNI-LABEL: test_bitreverse_i4:
+; X86GFNI:       # %bb.0:
+; X86GFNI-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}{1to2}, %xmm0, %xmm0
+; X86GFNI-NEXT:    vmovd %xmm0, %eax
+; X86GFNI-NEXT:    shrb $4, %al
+; X86GFNI-NEXT:    # kill: def $al killed $al killed $eax
+; X86GFNI-NEXT:    retl
+;
+; X64GFNI-LABEL: test_bitreverse_i4:
+; X64GFNI:       # %bb.0:
+; X64GFNI-NEXT:    vmovd %edi, %xmm0
+; X64GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; X64GFNI-NEXT:    vmovd %xmm0, %eax
+; X64GFNI-NEXT:    shrb $4, %al
+; X64GFNI-NEXT:    # kill: def $al killed $al killed $eax
+; X64GFNI-NEXT:    retq
   %b = call i4 @llvm.bitreverse.i4(i4 %a)
   ret i4 %b
 }
@@ -523,7 +583,7 @@ define <2 x i16> @fold_v2i16() {
 ; GFNI-LABEL: fold_v2i16:
 ; GFNI:       # %bb.0:
 ; GFNI-NEXT:    vmovss {{.*#+}} xmm0 = [61440,240,0,0,0,0,0,0]
-; GFNI-NEXT:    retq
+; GFNI-NEXT:    ret{{[l|q]}}
   %b = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> <i16 15, i16 3840>)
   ret <2 x i16> %b
 }
@@ -574,11 +634,16 @@ define i8 @identity_i8(i8 %a) {
 ; X86XOP-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86XOP-NEXT:    retl
 ;
-; GFNI-LABEL: identity_i8:
-; GFNI:       # %bb.0:
-; GFNI-NEXT:    movl %edi, %eax
-; GFNI-NEXT:    # kill: def $al killed $al killed $eax
-; GFNI-NEXT:    retq
+; X86GFNI-LABEL: identity_i8:
+; X86GFNI:       # %bb.0:
+; X86GFNI-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86GFNI-NEXT:    retl
+;
+; X64GFNI-LABEL: identity_i8:
+; X64GFNI:       # %bb.0:
+; X64GFNI-NEXT:    movl %edi, %eax
+; X64GFNI-NEXT:    # kill: def $al killed $al killed $eax
+; X64GFNI-NEXT:    retq
   %b = call i8 @llvm.bitreverse.i8(i8 %a)
   %c = call i8 @llvm.bitreverse.i8(i8 %b)
   ret i8 %c
@@ -601,7 +666,7 @@ define <2 x i16> @identity_v2i16(<2 x i16> %a) {
 ;
 ; GFNI-LABEL: identity_v2i16:
 ; GFNI:       # %bb.0:
-; GFNI-NEXT:    retq
+; GFNI-NEXT:    ret{{[l|q]}}
   %b = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> %a)
   %c = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> %b)
   ret <2 x i16> %c
@@ -1335,70 +1400,194 @@ define i528 @large_promotion(i528 %A) nounwind {
 ; X86XOP-NEXT:    popl %ebp
 ; X86XOP-NEXT:    retl $4
 ;
-; GFNI-LABEL: large_promotion:
-; GFNI:       # %bb.0:
-; GFNI-NEXT:    pushq %r14
-; GFNI-NEXT:    pushq %rbx
-; GFNI-NEXT:    movq %rdi, %rax
-; GFNI-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
-; GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; GFNI-NEXT:    vmovq %xmm1, %r10
-; GFNI-NEXT:    bswapq %r10
-; GFNI-NEXT:    vmovq %r9, %xmm1
-; GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; GFNI-NEXT:    vmovq %xmm1, %rdi
-; GFNI-NEXT:    bswapq %rdi
-; GFNI-NEXT:    vmovq %r8, %xmm1
-; GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; GFNI-NEXT:    vmovq %xmm1, %r8
-; GFNI-NEXT:    bswapq %r8
-; GFNI-NEXT:    movq %r8, %r9
-; GFNI-NEXT:    shldq $16, %rdi, %r9
-; GFNI-NEXT:    shldq $16, %r10, %rdi
-; GFNI-NEXT:    vmovq %rcx, %xmm1
-; GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; GFNI-NEXT:    vmovq %xmm1, %rcx
-; GFNI-NEXT:    bswapq %rcx
-; GFNI-NEXT:    shrdq $48, %rcx, %r8
-; GFNI-NEXT:    vmovq %rdx, %xmm1
-; GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; GFNI-NEXT:    vmovq %xmm1, %rdx
-; GFNI-NEXT:    bswapq %rdx
-; GFNI-NEXT:    shrdq $48, %rdx, %rcx
-; GFNI-NEXT:    vmovq %rsi, %xmm1
-; GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; GFNI-NEXT:    vmovq %xmm1, %rsi
-; GFNI-NEXT:    bswapq %rsi
-; GFNI-NEXT:    shrdq $48, %rsi, %rdx
-; GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; GFNI-NEXT:    vmovq %xmm1, %r11
-; GFNI-NEXT:    bswapq %r11
-; GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; GFNI-NEXT:    vmovq %xmm1, %rbx
-; GFNI-NEXT:    bswapq %rbx
-; GFNI-NEXT:    shrdq $48, %rbx, %r11
-; GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm0
-; GFNI-NEXT:    vmovq %xmm0, %r14
-; GFNI-NEXT:    bswapq %r14
-; GFNI-NEXT:    shrdq $48, %r14, %rbx
-; GFNI-NEXT:    shrdq $48, %r10, %r14
-; GFNI-NEXT:    shrq $48, %rsi
-; GFNI-NEXT:    movq %r14, 16(%rax)
-; GFNI-NEXT:    movq %rbx, 8(%rax)
-; GFNI-NEXT:    movq %r11, (%rax)
-; GFNI-NEXT:    movq %rdx, 56(%rax)
-; GFNI-NEXT:    movq %rcx, 48(%rax)
-; GFNI-NEXT:    movq %r8, 40(%rax)
-; GFNI-NEXT:    movq %r9, 32(%rax)
-; GFNI-NEXT:    movq %rdi, 24(%rax)
-; GFNI-NEXT:    movw %si, 64(%rax)
-; GFNI-NEXT:    popq %rbx
-; GFNI-NEXT:    popq %r14
-; GFNI-NEXT:    retq
+; X86GFNI-LABEL: large_promotion:
+; X86GFNI:       # %bb.0:
+; X86GFNI-NEXT:    pushl %ebp
+; X86GFNI-NEXT:    pushl %ebx
+; X86GFNI-NEXT:    pushl %edi
+; X86GFNI-NEXT:    pushl %esi
+; X86GFNI-NEXT:    subl $44, %esp
+; X86GFNI-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; X86GFNI-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X86GFNI-NEXT:    vmovd %xmm1, %eax
+; X86GFNI-NEXT:    bswapl %eax
+; X86GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X86GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X86GFNI-NEXT:    vpextrd $1, %xmm1, %ecx
+; X86GFNI-NEXT:    bswapl %ecx
+; X86GFNI-NEXT:    shrdl $16, %ecx, %eax
+; X86GFNI-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT:    vmovd %xmm1, %eax
+; X86GFNI-NEXT:    bswapl %eax
+; X86GFNI-NEXT:    shrdl $16, %eax, %ecx
+; X86GFNI-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X86GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X86GFNI-NEXT:    vpextrd $1, %xmm1, %ecx
+; X86GFNI-NEXT:    bswapl %ecx
+; X86GFNI-NEXT:    shrdl $16, %ecx, %eax
+; X86GFNI-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT:    vmovd %xmm1, %eax
+; X86GFNI-NEXT:    bswapl %eax
+; X86GFNI-NEXT:    shrdl $16, %eax, %ecx
+; X86GFNI-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X86GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X86GFNI-NEXT:    vpextrd $1, %xmm1, %ecx
+; X86GFNI-NEXT:    bswapl %ecx
+; X86GFNI-NEXT:    shrdl $16, %ecx, %eax
+; X86GFNI-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT:    vmovd %xmm1, %eax
+; X86GFNI-NEXT:    bswapl %eax
+; X86GFNI-NEXT:    shrdl $16, %eax, %ecx
+; X86GFNI-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X86GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X86GFNI-NEXT:    vpextrd $1, %xmm1, %ecx
+; X86GFNI-NEXT:    bswapl %ecx
+; X86GFNI-NEXT:    shrdl $16, %ecx, %eax
+; X86GFNI-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT:    vmovd %xmm1, %eax
+; X86GFNI-NEXT:    bswapl %eax
+; X86GFNI-NEXT:    shrdl $16, %eax, %ecx
+; X86GFNI-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X86GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X86GFNI-NEXT:    vpextrd $1, %xmm1, %ecx
+; X86GFNI-NEXT:    bswapl %ecx
+; X86GFNI-NEXT:    shrdl $16, %ecx, %eax
+; X86GFNI-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT:    vmovd %xmm1, %eax
+; X86GFNI-NEXT:    bswapl %eax
+; X86GFNI-NEXT:    shrdl $16, %eax, %ecx
+; X86GFNI-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X86GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X86GFNI-NEXT:    vpextrd $1, %xmm1, %ebp
+; X86GFNI-NEXT:    bswapl %ebp
+; X86GFNI-NEXT:    shrdl $16, %ebp, %eax
+; X86GFNI-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86GFNI-NEXT:    vmovd %xmm1, %ebx
+; X86GFNI-NEXT:    bswapl %ebx
+; X86GFNI-NEXT:    shrdl $16, %ebx, %ebp
+; X86GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X86GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X86GFNI-NEXT:    vpextrd $1, %xmm1, %edi
+; X86GFNI-NEXT:    bswapl %edi
+; X86GFNI-NEXT:    shrdl $16, %edi, %ebx
+; X86GFNI-NEXT:    vmovd %xmm1, %edx
+; X86GFNI-NEXT:    bswapl %edx
+; X86GFNI-NEXT:    shrdl $16, %edx, %edi
+; X86GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X86GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm0
+; X86GFNI-NEXT:    vpextrd $1, %xmm0, %ecx
+; X86GFNI-NEXT:    bswapl %ecx
+; X86GFNI-NEXT:    shrdl $16, %ecx, %edx
+; X86GFNI-NEXT:    vmovd %xmm0, %esi
+; X86GFNI-NEXT:    bswapl %esi
+; X86GFNI-NEXT:    shrdl $16, %esi, %ecx
+; X86GFNI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86GFNI-NEXT:    movl %ecx, 60(%eax)
+; X86GFNI-NEXT:    movl %edx, 56(%eax)
+; X86GFNI-NEXT:    movl %edi, 52(%eax)
+; X86GFNI-NEXT:    movl %ebx, 48(%eax)
+; X86GFNI-NEXT:    movl %ebp, 44(%eax)
+; X86GFNI-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; X86GFNI-NEXT:    movl %ecx, 40(%eax)
+; X86GFNI-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86GFNI-NEXT:    movl %ecx, 36(%eax)
+; X86GFNI-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86GFNI-NEXT:    movl %ecx, 32(%eax)
+; X86GFNI-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86GFNI-NEXT:    movl %ecx, 28(%eax)
+; X86GFNI-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86GFNI-NEXT:    movl %ecx, 24(%eax)
+; X86GFNI-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86GFNI-NEXT:    movl %ecx, 20(%eax)
+; X86GFNI-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86GFNI-NEXT:    movl %ecx, 16(%eax)
+; X86GFNI-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86GFNI-NEXT:    movl %ecx, 12(%eax)
+; X86GFNI-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86GFNI-NEXT:    movl %ecx, 8(%eax)
+; X86GFNI-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86GFNI-NEXT:    movl %ecx, 4(%eax)
+; X86GFNI-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86GFNI-NEXT:    movl %ecx, (%eax)
+; X86GFNI-NEXT:    shrl $16, %esi
+; X86GFNI-NEXT:    movw %si, 64(%eax)
+; X86GFNI-NEXT:    addl $44, %esp
+; X86GFNI-NEXT:    popl %esi
+; X86GFNI-NEXT:    popl %edi
+; X86GFNI-NEXT:    popl %ebx
+; X86GFNI-NEXT:    popl %ebp
+; X86GFNI-NEXT:    retl $4
+;
+; X64GFNI-LABEL: large_promotion:
+; X64GFNI:       # %bb.0:
+; X64GFNI-NEXT:    pushq %r14
+; X64GFNI-NEXT:    pushq %rbx
+; X64GFNI-NEXT:    movq %rdi, %rax
+; X64GFNI-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; X64GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X64GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X64GFNI-NEXT:    vmovq %xmm1, %r10
+; X64GFNI-NEXT:    bswapq %r10
+; X64GFNI-NEXT:    vmovq %r9, %xmm1
+; X64GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X64GFNI-NEXT:    vmovq %xmm1, %rdi
+; X64GFNI-NEXT:    bswapq %rdi
+; X64GFNI-NEXT:    vmovq %r8, %xmm1
+; X64GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X64GFNI-NEXT:    vmovq %xmm1, %r8
+; X64GFNI-NEXT:    bswapq %r8
+; X64GFNI-NEXT:    movq %r8, %r9
+; X64GFNI-NEXT:    shldq $16, %rdi, %r9
+; X64GFNI-NEXT:    shldq $16, %r10, %rdi
+; X64GFNI-NEXT:    vmovq %rcx, %xmm1
+; X64GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X64GFNI-NEXT:    vmovq %xmm1, %rcx
+; X64GFNI-NEXT:    bswapq %rcx
+; X64GFNI-NEXT:    shrdq $48, %rcx, %r8
+; X64GFNI-NEXT:    vmovq %rdx, %xmm1
+; X64GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X64GFNI-NEXT:    vmovq %xmm1, %rdx
+; X64GFNI-NEXT:    bswapq %rdx
+; X64GFNI-NEXT:    shrdq $48, %rdx, %rcx
+; X64GFNI-NEXT:    vmovq %rsi, %xmm1
+; X64GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X64GFNI-NEXT:    vmovq %xmm1, %rsi
+; X64GFNI-NEXT:    bswapq %rsi
+; X64GFNI-NEXT:    shrdq $48, %rsi, %rdx
+; X64GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X64GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X64GFNI-NEXT:    vmovq %xmm1, %r11
+; X64GFNI-NEXT:    bswapq %r11
+; X64GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X64GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
+; X64GFNI-NEXT:    vmovq %xmm1, %rbx
+; X64GFNI-NEXT:    bswapq %rbx
+; X64GFNI-NEXT:    shrdq $48, %rbx, %r11
+; X64GFNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X64GFNI-NEXT:    vgf2p8affineqb $0, %xmm0, %xmm1, %xmm0
+; X64GFNI-NEXT:    vmovq %xmm0, %r14
+; X64GFNI-NEXT:    bswapq %r14
+; X64GFNI-NEXT:    shrdq $48, %r14, %rbx
+; X64GFNI-NEXT:    shrdq $48, %r10, %r14
+; X64GFNI-NEXT:    shrq $48, %rsi
+; X64GFNI-NEXT:    movq %r14, 16(%rax)
+; X64GFNI-NEXT:    movq %rbx, 8(%rax)
+; X64GFNI-NEXT:    movq %r11, (%rax)
+; X64GFNI-NEXT:    movq %rdx, 56(%rax)
+; X64GFNI-NEXT:    movq %rcx, 48(%rax)
+; X64GFNI-NEXT:    movq %r8, 40(%rax)
+; X64GFNI-NEXT:    movq %r9, 32(%rax)
+; X64GFNI-NEXT:    movq %rdi, 24(%rax)
+; X64GFNI-NEXT:    movw %si, 64(%rax)
+; X64GFNI-NEXT:    popq %rbx
+; X64GFNI-NEXT:    popq %r14
+; X64GFNI-NEXT:    retq
   %Z = call i528 @llvm.bitreverse.i528(i528 %A)
   ret i528 %Z
 }
-- 
GitLab


From c7aff2ab4cfcc3cc4d43e8b9a3663096ab1e8f10 Mon Sep 17 00:00:00 2001
From: Simon Tatham <simon.tatham@arm.com>
Date: Thu, 24 Oct 2024 14:51:38 +0100
Subject: [PATCH 053/693] [libc] Pass through LIBC_CONF_MATH_OPTIMIZATIONS
 correctly (#113540)

It's checked in a cmake function that builds up a list called
`config_options`. But the check was appending to a list called
`compile_options`, so the intended `-DLIBC_MATH=whatever` didn't end up
on the actual compile command lines.
---
 libc/cmake/modules/LLVMLibCCompileOptionRules.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
index 0c658c6866c4..e30f7d322eb0 100644
--- a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
+++ b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
@@ -84,7 +84,7 @@ function(_get_compile_options_from_config output_var)
   endif()
 
   if(LIBC_CONF_MATH_OPTIMIZATIONS)
-    list(APPEND compile_options "-DLIBC_MATH=${LIBC_CONF_MATH_OPTIMIZATIONS}")
+    list(APPEND config_options "-DLIBC_MATH=${LIBC_CONF_MATH_OPTIMIZATIONS}")
   endif()
 
   set(${output_var} ${config_options} PARENT_SCOPE)
-- 
GitLab


From 522f34cfff693fd3f198808b7b3d17fa02f44120 Mon Sep 17 00:00:00 2001
From: Lei Huang <lei@ca.ibm.com>
Date: Thu, 24 Oct 2024 10:05:18 -0400
Subject: [PATCH 054/693] [PowerPC] Expand global named register support
 (#113482)

Enable all valid registers for intrinsics that read from and write
to global named registers.
---
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   |  36 +++--
 .../CodeGen/PowerPC/named-reg-alloc-r0.ll     |   4 +-
 .../CodeGen/PowerPC/named-reg-alloc-r2-64.ll  |   4 +-
 .../CodeGen/PowerPC/named-reg-alloc-r2.ll     |   4 +-
 llvm/test/CodeGen/PowerPC/named-reg-alloc.ll  | 144 ++++++++++++++++++
 5 files changed, 169 insertions(+), 23 deletions(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/named-reg-alloc.ll

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 7199fac9b110..ab31898e262e 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -17367,25 +17367,33 @@ SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op,
   return FrameAddr;
 }
 
-// FIXME? Maybe this could be a TableGen attribute on some registers and
-// this table could be generated automatically from RegInfo.
-Register PPCTargetLowering::getRegisterByName(const char* RegName, LLT VT,
+#define GET_REGISTER_MATCHER
+#include "PPCGenAsmMatcher.inc"
+
+Register PPCTargetLowering::getRegisterByName(const char *RegName, LLT VT,
                                               const MachineFunction &MF) const {
-  bool isPPC64 = Subtarget.isPPC64();
+  bool IsPPC64 = Subtarget.isPPC64();
 
-  bool is64Bit = isPPC64 && VT == LLT::scalar(64);
-  if (!is64Bit && VT != LLT::scalar(32))
+  bool Is64Bit = IsPPC64 && VT == LLT::scalar(64);
+  if (!Is64Bit && VT != LLT::scalar(32))
     report_fatal_error("Invalid register global variable type");
 
-  Register Reg = StringSwitch<Register>(RegName)
-                     .Case("r1", is64Bit ? PPC::X1 : PPC::R1)
-                     .Case("r2", isPPC64 ? Register() : PPC::R2)
-                     .Case("r13", (is64Bit ? PPC::X13 : PPC::R13))
-                     .Default(Register());
+  Register Reg = MatchRegisterName(RegName);
+  if (!Reg)
+    report_fatal_error(
+        Twine("Invalid global name register \"" + StringRef(RegName) + "\"."));
+
+  // FIXME: Unable to generate code for `-O2` but okay for `-O0`.
+  // Need followup investigation as to why.
+  if ((IsPPC64 && Reg == PPC::R2) || Reg == PPC::R0)
+    report_fatal_error(Twine("Trying to reserve an invalid register \"" +
+                             StringRef(RegName) + "\"."));
+
+  // Convert GPR to GP8R register for 64bit.
+  if (Is64Bit && StringRef(RegName).starts_with_insensitive("r"))
+    Reg = Reg.id() - PPC::R0 + PPC::X0;
 
-  if (Reg)
-    return Reg;
-  report_fatal_error("Invalid register name global variable");
+  return Reg;
 }
 
 bool PPCTargetLowering::isAccessedAsGotIndirect(SDValue GA) const {
diff --git a/llvm/test/CodeGen/PowerPC/named-reg-alloc-r0.ll b/llvm/test/CodeGen/PowerPC/named-reg-alloc-r0.ll
index 11cb72296e2c..8f4351f795ec 100644
--- a/llvm/test/CodeGen/PowerPC/named-reg-alloc-r0.ll
+++ b/llvm/test/CodeGen/PowerPC/named-reg-alloc-r0.ll
@@ -1,11 +1,9 @@
 ; RUN: not --crash llc < %s -mtriple=powerpc-unknown-linux-gnu 2>&1 | FileCheck %s
-; RUN: not --crash llc < %s -mtriple=powerpc-unknown-linux-gnu 2>&1 | FileCheck %s
 ; RUN: not --crash llc < %s -mtriple=powerpc64-unknown-linux-gnu 2>&1 | FileCheck %s
 
 define i32 @get_reg() nounwind {
 entry:
-; FIXME: Include an allocatable-specific error message
-; CHECK: Invalid register name global variable
+; CHECK: Trying to reserve an invalid register "r0".
         %reg = call i32 @llvm.read_register.i32(metadata !0)
   ret i32 %reg
 }
diff --git a/llvm/test/CodeGen/PowerPC/named-reg-alloc-r2-64.ll b/llvm/test/CodeGen/PowerPC/named-reg-alloc-r2-64.ll
index 3df778f445c7..fdf7ea2711d8 100644
--- a/llvm/test/CodeGen/PowerPC/named-reg-alloc-r2-64.ll
+++ b/llvm/test/CodeGen/PowerPC/named-reg-alloc-r2-64.ll
@@ -1,10 +1,8 @@
 ; RUN: not --crash llc < %s -mtriple=powerpc64-unknown-linux-gnu 2>&1 | FileCheck %s
-; RUN: not --crash llc < %s -mtriple=powerpc64-unknown-linux-gnu 2>&1 | FileCheck %s
 
 define i64 @get_reg() nounwind {
 entry:
-; FIXME: Include an allocatable-specific error message
-; CHECK: Invalid register name global variable
+; CHECK: Trying to reserve an invalid register "r2".
         %reg = call i64 @llvm.read_register.i64(metadata !0)
   ret i64 %reg
 }
diff --git a/llvm/test/CodeGen/PowerPC/named-reg-alloc-r2.ll b/llvm/test/CodeGen/PowerPC/named-reg-alloc-r2.ll
index ca79f857548e..c09b8bb3f49c 100644
--- a/llvm/test/CodeGen/PowerPC/named-reg-alloc-r2.ll
+++ b/llvm/test/CodeGen/PowerPC/named-reg-alloc-r2.ll
@@ -3,11 +3,9 @@
 
 define i32 @get_reg() nounwind {
 entry:
-; FIXME: Include an allocatable-specific error message
-; CHECK-NOTPPC32: Invalid register name global variable
+; CHECK-NOTPPC32: Trying to reserve an invalid register "r2".
         %reg = call i32 @llvm.read_register.i32(metadata !0)
   ret i32 %reg
-
 ; CHECK-LABEL: @get_reg
 ; CHECK: mr 3, 2
 }
diff --git a/llvm/test/CodeGen/PowerPC/named-reg-alloc.ll b/llvm/test/CodeGen/PowerPC/named-reg-alloc.ll
new file mode 100644
index 000000000000..38d22475ead9
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/named-reg-alloc.ll
@@ -0,0 +1,144 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -O0 -verify-machineinstrs < %s -mtriple=powerpc-unknown-linux-gnu 2>&1 | FileCheck %s
+; RUN: llc -O0 -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu 2>&1 | FileCheck %s --check-prefix=CHECK64
+
+@mVal = dso_local global i32 15, align 4
+@myGVal = dso_local global i32 0, align 4
+
+define dso_local void @testSetIntReg(i32 noundef signext %xx) {
+; CHECK-LABEL: testSetIntReg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mr 5, 3
+; CHECK-NEXT:    blr
+;
+; CHECK64-LABEL: testSetIntReg:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    mr 5, 3
+; CHECK64-NEXT:    blr
+entry:
+  tail call void @llvm.write_register.i32(metadata !0, i32 %xx)
+  ret void
+}
+
+declare void @llvm.write_register.i32(metadata, i32)
+
+define dso_local signext range(i32 0, 2) i32 @testCmpReg() {
+; CHECK-LABEL: testCmpReg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lis 3, mVal@ha
+; CHECK-NEXT:    lwz 3, mVal@l(3)
+; CHECK-NEXT:    xori 3, 3, 15
+; CHECK-NEXT:    cntlzw 3, 3
+; CHECK-NEXT:    srwi 3, 3, 5
+; CHECK-NEXT:    blr
+;
+; CHECK64-LABEL: testCmpReg:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    addis 3, 2, mVal@toc@ha
+; CHECK64-NEXT:    addi 3, 3, mVal@toc@l
+; CHECK64-NEXT:    lwz 3, 0(3)
+; CHECK64-NEXT:    xori 3, 3, 15
+; CHECK64-NEXT:    cntlzw 3, 3
+; CHECK64-NEXT:    srwi 3, 3, 5
+; CHECK64-NEXT:    extsw 3, 3
+; CHECK64-NEXT:    blr
+entry:
+  tail call void @llvm.write_register.i32(metadata !0, i32 15)
+  %0 = load i32, ptr @mVal, align 4
+  %1 = tail call i32 @llvm.read_register.i32(metadata !0)
+  %cmp = icmp eq i32 %0, %1
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+declare i32 @llvm.read_register.i32(metadata)
+
+define dso_local void @testSetIntReg2(i32 noundef signext %xx) {
+; CHECK-LABEL: testSetIntReg2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -48(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset r23, -36
+; CHECK-NEXT:    stw 23, 12(1) # 4-byte Folded Spill
+; CHECK-NEXT:    mr 23, 3
+; CHECK-NEXT:    lwz 23, 12(1) # 4-byte Folded Reload
+; CHECK-NEXT:    addi 1, 1, 48
+; CHECK-NEXT:    blr
+;
+; CHECK64-LABEL: testSetIntReg2:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    std 23, -72(1) # 8-byte Folded Spill
+; CHECK64-NEXT:    mr 23, 3
+; CHECK64-NEXT:    ld 23, -72(1) # 8-byte Folded Reload
+; CHECK64-NEXT:    blr
+entry:
+  tail call void @llvm.write_register.i32(metadata !1, i32 %xx)
+  ret void
+}
+
+define dso_local signext i32 @testReturnReg() {
+; CHECK-LABEL: testReturnReg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -48(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset r23, -36
+; CHECK-NEXT:    stw 23, 12(1) # 4-byte Folded Spill
+; CHECK-NEXT:    li 23, 125
+; CHECK-NEXT:    mr 3, 23
+; CHECK-NEXT:    lwz 23, 12(1) # 4-byte Folded Reload
+; CHECK-NEXT:    addi 1, 1, 48
+; CHECK-NEXT:    blr
+;
+; CHECK64-LABEL: testReturnReg:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    std 23, -72(1) # 8-byte Folded Spill
+; CHECK64-NEXT:    li 23, 125
+; CHECK64-NEXT:    extsw 3, 23
+; CHECK64-NEXT:    ld 23, -72(1) # 8-byte Folded Reload
+; CHECK64-NEXT:    blr
+entry:
+  tail call void @llvm.write_register.i32(metadata !1, i32 125)
+  %0 = tail call i32 @llvm.read_register.i32(metadata !1)
+  ret i32 %0
+}
+
+define dso_local void @testViaASM(i32 noundef signext %xx) {
+; CHECK-LABEL: testViaASM:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -64(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset r20, -48
+; CHECK-NEXT:    stw 20, 16(1) # 4-byte Folded Spill
+; CHECK-NEXT:    mr 20, 3
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi 3, 1, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    lis 4, myGVal@ha
+; CHECK-NEXT:    stw 3, myGVal@l(4)
+; CHECK-NEXT:    lwz 20, 16(1) # 4-byte Folded Reload
+; CHECK-NEXT:    addi 1, 1, 64
+; CHECK-NEXT:    blr
+;
+; CHECK64-LABEL: testViaASM:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    std 20, -96(1) # 8-byte Folded Spill
+; CHECK64-NEXT:    mr 20, 3
+; CHECK64-NEXT:    #APP
+; CHECK64-NEXT:    addi 3, 1, 1
+; CHECK64-NEXT:    #NO_APP
+; CHECK64-NEXT:    addis 4, 2, myGVal@toc@ha
+; CHECK64-NEXT:    addi 4, 4, myGVal@toc@l
+; CHECK64-NEXT:    stw 3, 0(4)
+; CHECK64-NEXT:    ld 20, -96(1) # 8-byte Folded Reload
+; CHECK64-NEXT:    blr
+entry:
+  tail call void @llvm.write_register.i32(metadata !2, i32 %xx)
+  %0 = tail call i32 @llvm.read_register.i32(metadata !2)
+  %1 = tail call i32 asm "addi $0, $2, $2", "=r,{r20},K"(i32 %0, i32 1)
+  store i32 %1, ptr @myGVal, align 4
+  ret void
+}
+
+!0 = !{!"r5"}
+!1 = !{!"r23"}
+!2 = !{!"r20"}
-- 
GitLab


From 2e686d6d17c4cc7608510a856055e6ca79fcb917 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Thu, 24 Oct 2024 16:11:34 +0200
Subject: [PATCH 055/693] [libc++] Remove workaround which allows setting
 _LIBCPP_OVERRIDABLE_FUNC_VIS externally (#113139)

`-fvisibility-global-new-delete` has been added in Clang 18, so there is
no more need to specify the visibility of new/delete via libc++-internal
macros.

Some people used a custom new/delete, which requires them to have
default visibility, but didn't want to leak any symbols otherwise. Since
the new/delete visibility can now be controlled by a compiler flag,
there is no reason to set it with out macro.

https://reviews.llvm.org/D128007 originally tried to remove the option
to override the macro, but was partially reverted because people
actually set this specific macro.
---
 libcxx/include/__config | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/libcxx/include/__config b/libcxx/include/__config
index fcba56f7e3d5..4155fd33dabc 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -408,11 +408,7 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_VISIBILITY("default")
 #    define _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS _LIBCPP_VISIBILITY("default")
 #    define _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS
-
-// TODO: Make this a proper customization point or remove the option to override it.
-#    ifndef _LIBCPP_OVERRIDABLE_FUNC_VIS
-#      define _LIBCPP_OVERRIDABLE_FUNC_VIS _LIBCPP_VISIBILITY("default")
-#    endif
+#    define _LIBCPP_OVERRIDABLE_FUNC_VIS _LIBCPP_VISIBILITY("default")
 
 #    if !defined(_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS)
 // The inline should be removed once PR32114 is resolved
-- 
GitLab


From f3131c99bf4816ac2fe48dcb5fa7b1f252b07f20 Mon Sep 17 00:00:00 2001
From: Zaara Syeda <syzaara@ca.ibm.com>
Date: Thu, 24 Oct 2024 10:16:39 -0400
Subject: [PATCH 056/693] [GlobalMerge] Aggressively merge constants to reduce
 TOC entries (#111756)

Symbols that get mapped into the read-only section are loaded as part of
the text segment and will always need a TOC entry to be addressable. Add
an option to aggressively merge these read only globals to reduce TOC
usage.
---
 llvm/lib/CodeGen/GlobalMerge.cpp              |  7 +-
 .../PowerPC/aix-aggressive-merge-const.ll     | 93 +++++++++++++++++++
 2 files changed, 99 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-aggressive-merge-const.ll

diff --git a/llvm/lib/CodeGen/GlobalMerge.cpp b/llvm/lib/CodeGen/GlobalMerge.cpp
index 007bea9a6585..4c6c8c600ee2 100644
--- a/llvm/lib/CodeGen/GlobalMerge.cpp
+++ b/llvm/lib/CodeGen/GlobalMerge.cpp
@@ -119,6 +119,11 @@ static cl::opt<bool> GlobalMergeGroupByUse(
     "global-merge-group-by-use", cl::Hidden,
     cl::desc("Improve global merge pass to look at uses"), cl::init(true));
 
+static cl::opt<bool> GlobalMergeAllConst(
+    "global-merge-all-const", cl::Hidden,
+    cl::desc("Merge all const globals without looking at uses"),
+    cl::init(false));
+
 static cl::opt<bool> GlobalMergeIgnoreSingleUse(
     "global-merge-ignore-single-use", cl::Hidden,
     cl::desc("Improve global merge pass to ignore globals only used alone"),
@@ -263,7 +268,7 @@ bool GlobalMergeImpl::doMerge(SmallVectorImpl<GlobalVariable *> &Globals,
       });
 
   // If we want to just blindly group all globals together, do so.
-  if (!GlobalMergeGroupByUse) {
+  if (!GlobalMergeGroupByUse || (GlobalMergeAllConst && isConst)) {
     BitVector AllGlobals(Globals.size());
     AllGlobals.set();
     return doMerge(Globals, AllGlobals, M, isConst, AddrSpace);
diff --git a/llvm/test/CodeGen/PowerPC/aix-aggressive-merge-const.ll b/llvm/test/CodeGen/PowerPC/aix-aggressive-merge-const.ll
new file mode 100644
index 000000000000..ec2d409bfb97
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-aggressive-merge-const.ll
@@ -0,0 +1,93 @@
+; RUN: llc --verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -global-merge-all-const=true < %s | FileCheck %s
+
+; RUN: llc --verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -global-merge-all-const=false < %s | FileCheck --check-prefix=NOMERGE %s
+
+%struct.pc_t = type { i8 }
+%struct.S = type { i32, i32, i32, i32, [9 x i32] }
+
+@constinit = private unnamed_addr constant <{ i32, i32, i32, i32, [9 x i32] }> <{ i32 0, i32 0, i32 0, i32 2, [9 x i32] zeroinitializer }>, align 4
+@.str = private unnamed_addr constant [6 x i8] c"hello\00", align 1
+@.str.1 = private unnamed_addr constant [6 x i8] c"world\00", align 1
+@.str.2 = private unnamed_addr constant [6 x i8] c"abcde\00", align 1
+@.str.3 = private unnamed_addr constant [6 x i8] c"fghij\00", align 1
+@pc = internal constant %struct.pc_t zeroinitializer, align 1
+@s = internal constant %struct.S { i32 1, i32 2, i32 3, i32 4, [9 x i32] [i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13] }, align 4
+
+; Function Attrs: mustprogress
+define noundef i32 @f5() {
+entry:
+  %call = tail call noundef i32 @f4(ptr noundef nonnull @pc)
+  ret i32 %call
+}
+
+declare noundef i32 @f4(ptr noundef)
+declare noundef i32 @printf(ptr nocapture noundef readonly, ...)
+
+define noundef i32 @f1() {
+entry:
+  %call = tail call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(1) @.str)
+  ret i32 %call
+}
+
+
+; Function Attrs: mustprogress nofree nounwind
+define noundef i32 @f2() {
+entry:
+  %call = tail call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(1) @.str.1)
+  ret i32 %call
+}
+
+define noundef i32 @f3() {
+entry:
+  %call = tail call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(1) @.str.2)
+  ret i32 %call
+}
+
+define noundef i32 @f7() {
+entry:
+  %call = tail call noundef i32 @f6(ptr noundef nonnull @s)
+  ret i32 %call
+}
+
+declare noundef i32 @f6(ptr noundef)
+
+; CHECK:             .csect L.._MergedGlobals[RO],2
+; CHECK-NEXT:        .lglobl pc                          # @_MergedGlobals
+; CHECK-NEXT:        .lglobl s
+; CHECK-NEXT:        .align  2
+; CHECK-NEXT:pc:
+; CHECK-NEXT:        .space  1
+; CHECK-NEXT:L...str:
+; CHECK-NEXT:        .string "hello"
+; CHECK-NEXT:L...str.1:
+; CHECK-NEXT:        .string "world"
+; CHECK-NEXT:L...str.2:
+; CHECK-NEXT:        .string "abcde"
+; CHECK-NEXT:L...str.3:
+; CHECK-NEXT:        .string "fghij"
+; CHECK-NEXT:        .space  3
+; CHECK-NEXT:L..constinit:
+; CHECK-NEXT:        .vbyte  4, 0                            # 0x0
+; CHECK-NEXT:        .vbyte  4, 0                            # 0x0
+; CHECK-NEXT:        .vbyte  4, 0                            # 0x0
+; CHECK-NEXT:        .vbyte  4, 2                            # 0x2
+; CHECK-NEXT:        .space  36
+; CHECK-NEXT:s:
+; CHECK-NEXT:        .vbyte  4, 1                            # 0x1
+; CHECK-NEXT:        .vbyte  4, 2                            # 0x2
+; CHECK-NEXT:        .vbyte  4, 3                            # 0x3
+; CHECK-NEXT:        .vbyte  4, 4                            # 0x4
+; CHECK-NEXT:        .vbyte  4, 5                            # 0x5
+; CHECK-NEXT:        .vbyte  4, 6                            # 0x6
+; CHECK-NEXT:        .vbyte  4, 7                            # 0x7
+; CHECK-NEXT:        .vbyte  4, 8                            # 0x8
+; CHECK-NEXT:        .vbyte  4, 9                            # 0x9
+; CHECK-NEXT:        .vbyte  4, 10                           # 0xa
+; CHECK-NEXT:        .vbyte  4, 11                           # 0xb
+; CHECK-NEXT:        .vbyte  4, 12                           # 0xc
+; CHECK-NEXT:        .vbyte  4, 13                           # 0xd
+
+
+; NOMERGE-NOT: L.._MergedGGlobals[RO]
-- 
GitLab


From 8a7318eb00fcd833aefa8b0028690e9fae221ce6 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Thu, 24 Oct 2024 10:19:21 -0400
Subject: [PATCH 057/693] [libc++] Refactor vector::push_back to use
 vector::emplace (#113481)

This removes some duplicate code. I suspect this was originally written
that way because vector::emplace didn't exist in C++03 mode, which
stopped being relevant when Clang implemented rvalue references in
C++03.
---
 libcxx/include/vector | 45 ++-----------------------------------------
 1 file changed, 2 insertions(+), 43 deletions(-)

diff --git a/libcxx/include/vector b/libcxx/include/vector
index bfbf1ea1cfc9..ce412a829ea8 100644
--- a/libcxx/include/vector
+++ b/libcxx/include/vector
@@ -689,9 +689,9 @@ public:
     return std::__to_address(this->__begin_);
   }
 
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_back(const_reference __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_back(const_reference __x) { emplace_back(__x); }
 
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __x) { emplace_back(std::move(__x)); }
 
   template <class... _Args>
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
@@ -898,9 +898,6 @@ private:
     __annotate_shrink(__old_size);
   }
 
-  template <class _Up>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI inline pointer __push_back_slow_path(_Up&& __x);
-
   template <class... _Args>
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI inline pointer __emplace_back_slow_path(_Args&&... __args);
 
@@ -1491,44 +1488,6 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::shrink_to_fit() _NOE
   }
 }
 
-template <class _Tp, class _Allocator>
-template <class _Up>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::pointer
-vector<_Tp, _Allocator>::__push_back_slow_path(_Up&& __x) {
-  allocator_type& __a = this->__alloc();
-  __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), size(), __a);
-  // __v.push_back(std::forward<_Up>(__x));
-  __alloc_traits::construct(__a, std::__to_address(__v.__end_), std::forward<_Up>(__x));
-  __v.__end_++;
-  __swap_out_circular_buffer(__v);
-  return this->__end_;
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void
-vector<_Tp, _Allocator>::push_back(const_reference __x) {
-  pointer __end = this->__end_;
-  if (__end < this->__end_cap()) {
-    __construct_one_at_end(__x);
-    ++__end;
-  } else {
-    __end = __push_back_slow_path(__x);
-  }
-  this->__end_ = __end;
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void vector<_Tp, _Allocator>::push_back(value_type&& __x) {
-  pointer __end = this->__end_;
-  if (__end < this->__end_cap()) {
-    __construct_one_at_end(std::move(__x));
-    ++__end;
-  } else {
-    __end = __push_back_slow_path(std::move(__x));
-  }
-  this->__end_ = __end;
-}
-
 template <class _Tp, class _Allocator>
 template <class... _Args>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::pointer
-- 
GitLab


From cb445e8d1d66daf294e64a07bf69b6266fe1cded Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Thu, 24 Oct 2024 10:27:51 -0400
Subject: [PATCH 058/693] [cmake] Promote message when failing to find
 compiler-rt to warning (#111834)

This makes the message stand out a bit more, which can be helpful to
debug situations where compiler-rt is missing but shouldn't.
---
 cmake/Modules/HandleCompilerRT.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/Modules/HandleCompilerRT.cmake b/cmake/Modules/HandleCompilerRT.cmake
index 0a7c56bc6309..9398d99afdd0 100644
--- a/cmake/Modules/HandleCompilerRT.cmake
+++ b/cmake/Modules/HandleCompilerRT.cmake
@@ -5,7 +5,7 @@
 #    COMPILER_RT_LIBRARY-<name>-<target> to NOTFOUND
 function(cache_compiler_rt_library err_flag name target library_file)
   if(err_flag OR NOT EXISTS "${library_file}")
-    message(STATUS "Failed to find compiler-rt ${name} library for ${target}")
+    message(WARNING "Failed to find compiler-rt ${name} library for ${target}")
     set(COMPILER_RT_LIBRARY_${name}_${target} "NOTFOUND" CACHE INTERNAL
         "compiler-rt ${name} library for ${target}")
   else()
-- 
GitLab


From 0af6c304e48e0484672b53be49a15f411d173e59 Mon Sep 17 00:00:00 2001
From: Eisuke Kawashima <e.kawaschima+github@gmail.com>
Date: Thu, 24 Oct 2024 23:32:10 +0900
Subject: [PATCH 059/693] [NFC][compiler-rt] fix(compiler-rt/**.py): fix
 comparison to None (#94015)

from PEP8
(https://peps.python.org/pep-0008/#programming-recommendations):

> Comparisons to singletons like None should always be done with is or
is not, never the equality operators.
---
 compiler-rt/test/asan/lit.cfg.py                                | 2 +-
 compiler-rt/test/builtins/Unit/lit.cfg.py                       | 2 +-
 compiler-rt/test/ctx_profile/lit.cfg.py                         | 2 +-
 compiler-rt/test/lsan/lit.common.cfg.py                         | 2 +-
 compiler-rt/test/memprof/lit.cfg.py                             | 2 +-
 compiler-rt/test/profile/lit.cfg.py                             | 2 +-
 .../test/sanitizer_common/android_commands/android_compile.py   | 2 +-
 .../test/sanitizer_common/ios_commands/iossim_compile.py        | 2 +-
 compiler-rt/test/ubsan/lit.common.cfg.py                        | 2 +-
 compiler-rt/test/ubsan_minimal/lit.common.cfg.py                | 2 +-
 10 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/compiler-rt/test/asan/lit.cfg.py b/compiler-rt/test/asan/lit.cfg.py
index dac3ef00a99a..c57f5ca0fa65 100644
--- a/compiler-rt/test/asan/lit.cfg.py
+++ b/compiler-rt/test/asan/lit.cfg.py
@@ -10,7 +10,7 @@ import lit.formats
 
 def get_required_attr(config, attr_name):
     attr_value = getattr(config, attr_name, None)
-    if attr_value == None:
+    if attr_value is None:
         lit_config.fatal(
             "No attribute %r in test configuration! You may need to run "
             "tests from your build directory or add this attribute "
diff --git a/compiler-rt/test/builtins/Unit/lit.cfg.py b/compiler-rt/test/builtins/Unit/lit.cfg.py
index f63d15919888..c18c973d54c1 100644
--- a/compiler-rt/test/builtins/Unit/lit.cfg.py
+++ b/compiler-rt/test/builtins/Unit/lit.cfg.py
@@ -21,7 +21,7 @@ else:
 
 def get_required_attr(config, attr_name):
     attr_value = getattr(config, attr_name, None)
-    if attr_value == None:
+    if attr_value is None:
         lit_config.fatal(
             "No attribute %r in test configuration! You may need to run "
             "tests from your build directory or add this attribute "
diff --git a/compiler-rt/test/ctx_profile/lit.cfg.py b/compiler-rt/test/ctx_profile/lit.cfg.py
index 3034fadbb7a6..74d9bfd11ae2 100644
--- a/compiler-rt/test/ctx_profile/lit.cfg.py
+++ b/compiler-rt/test/ctx_profile/lit.cfg.py
@@ -13,7 +13,7 @@ if config.host_os not in ["Linux"]:
 
 def get_required_attr(config, attr_name):
     attr_value = getattr(config, attr_name, None)
-    if attr_value == None:
+    if attr_value is None:
         lit_config.fatal(
             "No attribute %r in test configuration! You may need to run "
             "tests from your build directory or add this attribute "
diff --git a/compiler-rt/test/lsan/lit.common.cfg.py b/compiler-rt/test/lsan/lit.common.cfg.py
index e9b974955730..9426b7d108bb 100644
--- a/compiler-rt/test/lsan/lit.common.cfg.py
+++ b/compiler-rt/test/lsan/lit.common.cfg.py
@@ -10,7 +10,7 @@ import lit.util
 
 def get_required_attr(config, attr_name):
     attr_value = getattr(config, attr_name, None)
-    if attr_value == None:
+    if attr_value is None:
         lit_config.fatal(
             "No attribute %r in test configuration! You may need to run "
             "tests from your build directory or add this attribute "
diff --git a/compiler-rt/test/memprof/lit.cfg.py b/compiler-rt/test/memprof/lit.cfg.py
index 4e5d7ba405a2..4057da0c65b5 100644
--- a/compiler-rt/test/memprof/lit.cfg.py
+++ b/compiler-rt/test/memprof/lit.cfg.py
@@ -9,7 +9,7 @@ import lit.formats
 
 def get_required_attr(config, attr_name):
     attr_value = getattr(config, attr_name, None)
-    if attr_value == None:
+    if attr_value is None:
         lit_config.fatal(
             "No attribute %r in test configuration! You may need to run "
             "tests from your build directory or add this attribute "
diff --git a/compiler-rt/test/profile/lit.cfg.py b/compiler-rt/test/profile/lit.cfg.py
index c8c78a746b4c..ca6df08ad043 100644
--- a/compiler-rt/test/profile/lit.cfg.py
+++ b/compiler-rt/test/profile/lit.cfg.py
@@ -6,7 +6,7 @@ import re
 
 def get_required_attr(config, attr_name):
     attr_value = getattr(config, attr_name, None)
-    if attr_value == None:
+    if attr_value is None:
         lit_config.fatal(
             "No attribute %r in test configuration! You may need to run "
             "tests from your build directory or add this attribute "
diff --git a/compiler-rt/test/sanitizer_common/android_commands/android_compile.py b/compiler-rt/test/sanitizer_common/android_commands/android_compile.py
index f86831ffef89..e9c6738a09a3 100755
--- a/compiler-rt/test/sanitizer_common/android_commands/android_compile.py
+++ b/compiler-rt/test/sanitizer_common/android_commands/android_compile.py
@@ -20,7 +20,7 @@ while args:
     elif arg == "-o":
         output = args.pop(0)
 
-if output == None:
+if output is None:
     print("No output file name!")
     sys.exit(1)
 
diff --git a/compiler-rt/test/sanitizer_common/ios_commands/iossim_compile.py b/compiler-rt/test/sanitizer_common/ios_commands/iossim_compile.py
index 39b211b093c9..2eac1a9bab45 100755
--- a/compiler-rt/test/sanitizer_common/ios_commands/iossim_compile.py
+++ b/compiler-rt/test/sanitizer_common/ios_commands/iossim_compile.py
@@ -19,7 +19,7 @@ while args:
     elif arg == "-o":
         output = args.pop(0)
 
-if output == None:
+if output is None:
     print("No output file name!")
     sys.exit(1)
 
diff --git a/compiler-rt/test/ubsan/lit.common.cfg.py b/compiler-rt/test/ubsan/lit.common.cfg.py
index 61f6818cce06..04d6f24de5a9 100644
--- a/compiler-rt/test/ubsan/lit.common.cfg.py
+++ b/compiler-rt/test/ubsan/lit.common.cfg.py
@@ -5,7 +5,7 @@ import os
 
 def get_required_attr(config, attr_name):
     attr_value = getattr(config, attr_name, None)
-    if attr_value == None:
+    if attr_value is None:
         lit_config.fatal(
             "No attribute %r in test configuration! You may need to run "
             "tests from your build directory or add this attribute "
diff --git a/compiler-rt/test/ubsan_minimal/lit.common.cfg.py b/compiler-rt/test/ubsan_minimal/lit.common.cfg.py
index d398d3a0a816..e60aed6512fb 100644
--- a/compiler-rt/test/ubsan_minimal/lit.common.cfg.py
+++ b/compiler-rt/test/ubsan_minimal/lit.common.cfg.py
@@ -5,7 +5,7 @@ import os
 
 def get_required_attr(config, attr_name):
     attr_value = getattr(config, attr_name, None)
-    if attr_value == None:
+    if attr_value is None:
         lit_config.fatal(
             "No attribute %r in test configuration! You may need to run "
             "tests from your build directory or add this attribute "
-- 
GitLab


From cb5046da26399c0ab545c3ed764e0e68e997ff89 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Thu, 24 Oct 2024 04:33:41 -0700
Subject: [PATCH 060/693] [SLP]Do not ignore undefs when trying to replace with
 "poisonous" shuffles

Need to consider undefs correctly, when trying to replace them with
potentially poisonous values in shuffles. Such elements should not be
silently replaced by poison values, instead complex analysis should be
implemented to see if it is safe to do it.

Fixes #113425
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 104 +++++++++++-------
 .../RISCV/revec-getGatherCost.ll              |   6 +-
 .../Transforms/SLPVectorizer/RISCV/revec.ll   |   3 +-
 .../X86/matched-shuffled-entries.ll           |   4 +-
 .../X86/remark_extract_broadcast.ll           |   2 +-
 .../insertelement-across-zero.ll              |   6 +-
 llvm/test/Transforms/SLPVectorizer/revec.ll   |  19 +---
 .../SLPVectorizer/splat-buildvector.ll        |   2 +-
 8 files changed, 82 insertions(+), 64 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 889c4d94ba5e..f76e913a57d5 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9076,14 +9076,14 @@ protected:
           continue;
         ExtMask[Idx] = SV->getMaskValue(I);
       }
-      bool IsOp1Undef =
-          isUndefVector(SV->getOperand(0),
-                        buildUseMask(LocalVF, ExtMask, UseMask::FirstArg))
-              .all();
-      bool IsOp2Undef =
-          isUndefVector(SV->getOperand(1),
-                        buildUseMask(LocalVF, ExtMask, UseMask::SecondArg))
-              .all();
+      bool IsOp1Undef = isUndefVector</*isPoisonOnly=*/true>(
+                            SV->getOperand(0),
+                            buildUseMask(LocalVF, ExtMask, UseMask::FirstArg))
+                            .all();
+      bool IsOp2Undef = isUndefVector</*isPoisonOnly=*/true>(
+                            SV->getOperand(1),
+                            buildUseMask(LocalVF, ExtMask, UseMask::SecondArg))
+                            .all();
       if (!IsOp1Undef && !IsOp2Undef) {
         // Update mask and mark undef elems.
         for (int &I : Mask) {
@@ -13305,8 +13305,17 @@ Value *BoUpSLP::gather(ArrayRef<Value *> VL, Value *Root, Type *ScalarTy) {
     return Vec;
   };
   auto *VecTy = getWidenedType(ScalarTy, VL.size());
-  Value *Vec = Root ? Root : PoisonValue::get(VecTy);
+  Value *Vec = PoisonValue::get(VecTy);
   SmallVector<int> NonConsts;
+  SmallVector<int> Mask(VL.size());
+  std::iota(Mask.begin(), Mask.end(), 0);
+  Value *OriginalRoot = Root;
+  if (auto *SV = dyn_cast_or_null<ShuffleVectorInst>(Root);
+      SV && isa<PoisonValue>(SV->getOperand(1)) &&
+      SV->getOperand(0)->getType() == VecTy) {
+    Root = SV->getOperand(0);
+    Mask.assign(SV->getShuffleMask().begin(), SV->getShuffleMask().end());
+  }
   // Insert constant values at first.
   for (int I = 0, E = VL.size(); I < E; ++I) {
     if (PostponedIndices.contains(I))
@@ -13315,19 +13324,20 @@ Value *BoUpSLP::gather(ArrayRef<Value *> VL, Value *Root, Type *ScalarTy) {
       NonConsts.push_back(I);
       continue;
     }
-    if (Root) {
-      if (!isa<UndefValue>(VL[I])) {
-        NonConsts.push_back(I);
-        continue;
-      }
-      if (isa<PoisonValue>(VL[I]))
-        continue;
-      if (auto *SV = dyn_cast<ShuffleVectorInst>(Root)) {
-        if (SV->getMaskValue(I) == PoisonMaskElem)
-          continue;
-      }
-    }
+    if (isa<PoisonValue>(VL[I]))
+      continue;
     Vec = CreateInsertElement(Vec, VL[I], I, ScalarTy);
+    Mask[I] = I + E;
+  }
+  if (Root) {
+    if (isa<PoisonValue>(Vec)) {
+      Vec = OriginalRoot;
+    } else {
+      Vec = Builder.CreateShuffleVector(Root, Vec, Mask);
+      if (auto *OI = dyn_cast<Instruction>(OriginalRoot);
+          OI && OI->hasNUses(0))
+        eraseInstruction(OI);
+    }
   }
   // Insert non-constant values.
   for (int I : NonConsts)
@@ -14041,7 +14051,8 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
   if (!ReorderMask.empty())
     reorderScalars(GatheredScalars, ReorderMask);
   auto FindReusedSplat = [&](MutableArrayRef<int> Mask, unsigned InputVF,
-                             unsigned I, unsigned SliceSize) {
+                             unsigned I, unsigned SliceSize,
+                             bool IsNotPoisonous) {
     if (!isSplat(E->Scalars) || none_of(E->Scalars, [](Value *V) {
           return isa<UndefValue>(V) && !isa<PoisonValue>(V);
         }))
@@ -14050,14 +14061,29 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
     unsigned EdgeIdx = E->UserTreeIndices.back().EdgeIdx;
     if (UserTE->getNumOperands() != 2)
       return false;
-    auto *It =
-        find_if(VectorizableTree, [=](const std::unique_ptr<TreeEntry> &TE) {
-          return find_if(TE->UserTreeIndices, [=](const EdgeInfo &EI) {
-                   return EI.UserTE == UserTE && EI.EdgeIdx != EdgeIdx;
-                 }) != TE->UserTreeIndices.end();
-        });
-    if (It == VectorizableTree.end())
-      return false;
+    if (!IsNotPoisonous) {
+      auto *It =
+          find_if(VectorizableTree, [=](const std::unique_ptr<TreeEntry> &TE) {
+            return find_if(TE->UserTreeIndices, [=](const EdgeInfo &EI) {
+                     return EI.UserTE == UserTE && EI.EdgeIdx != EdgeIdx;
+                   }) != TE->UserTreeIndices.end();
+          });
+      if (It == VectorizableTree.end())
+        return false;
+      SmallVector<Value *> GS((*It)->Scalars.begin(), (*It)->Scalars.end());
+      if (!(*It)->ReorderIndices.empty()) {
+        inversePermutation((*It)->ReorderIndices, ReorderMask);
+        reorderScalars(GS, ReorderMask);
+      }
+      if (!all_of(zip(GatheredScalars, GS), [&](const auto &P) {
+            Value *V0 = std::get<0>(P);
+            Value *V1 = std::get<1>(P);
+            return !isa<UndefValue>(V0) || isa<PoisonValue>(V0) ||
+                   (isa<UndefValue>(V0) && !isa<PoisonValue>(V0) &&
+                    is_contained(E->Scalars, V1));
+          }))
+        return false;
+    }
     int Idx;
     if ((Mask.size() < InputVF &&
          ShuffleVectorInst::isExtractSubvectorMask(Mask, InputVF, Idx) &&
@@ -14330,12 +14356,13 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
             isGuaranteedNotToBePoison(Vec1) && isGuaranteedNotToBePoison(Vec2);
         ShuffleBuilder.add(Vec1, Vec2, ExtractMask);
       } else if (Vec1) {
+        bool IsNotPoisonedVec = isGuaranteedNotToBePoison(Vec1);
         IsUsedInExpr &= FindReusedSplat(
             ExtractMask,
             cast<FixedVectorType>(Vec1->getType())->getNumElements(), 0,
-            ExtractMask.size());
+            ExtractMask.size(), IsNotPoisonedVec);
         ShuffleBuilder.add(Vec1, ExtractMask, /*ForExtracts=*/true);
-        IsNonPoisoned &= isGuaranteedNotToBePoison(Vec1);
+        IsNonPoisoned &= IsNotPoisonedVec;
       } else {
         IsUsedInExpr = false;
         ShuffleBuilder.add(PoisonValue::get(VecTy), ExtractMask,
@@ -14358,12 +14385,15 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
         VecMask.assign(VecMask.size(), PoisonMaskElem);
         copy(SubMask, std::next(VecMask.begin(), I * SliceSize));
         if (TEs.size() == 1) {
-          IsUsedInExpr &= FindReusedSplat(
-              VecMask, TEs.front()->getVectorFactor(), I, SliceSize);
+          bool IsNotPoisonedVec =
+              TEs.front()->VectorizedValue
+                  ? isGuaranteedNotToBePoison(TEs.front()->VectorizedValue)
+                  : true;
+          IsUsedInExpr &=
+              FindReusedSplat(VecMask, TEs.front()->getVectorFactor(), I,
+                              SliceSize, IsNotPoisonedVec);
           ShuffleBuilder.add(*TEs.front(), VecMask);
-          if (TEs.front()->VectorizedValue)
-            IsNonPoisoned &=
-                isGuaranteedNotToBePoison(TEs.front()->VectorizedValue);
+          IsNonPoisoned &= IsNotPoisonedVec;
         } else {
           IsUsedInExpr = false;
           ShuffleBuilder.add(*TEs.front(), *TEs.back(), VecMask);
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll
index a0cb52a853b7..2565d5be32e2 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll
@@ -19,8 +19,7 @@ define void @test1(<4 x float> %load6, <4 x float> %load7, <4 x float> %load8, <
 ; CHECK-NEXT:    [[VEXT309_I:%.*]] = shufflevector <4 x float> [[LOAD7]], <4 x float> [[LOAD8:%.*]], <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[TMP0:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> poison, <4 x float> [[VEXT165_I]], i64 0)
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP0]], <4 x float> [[VEXT309_I]], i64 4)
-; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> poison, <4 x float> poison, i64 4)
-; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP2]], <4 x float> [[LOAD17:%.*]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> poison, <4 x float> [[LOAD17:%.*]], i64 0)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> poison, <4 x float> [[FMULADD7:%.*]], i64 0)
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP5]], <4 x float> [[FMULADD16:%.*]], i64 4)
@@ -58,8 +57,7 @@ define void @test2(<8 x float> %load6, <8 x float> %load7, <8 x float> %load8, <
 ; CHECK-NEXT:    [[VEXT309_I:%.*]] = shufflevector <8 x float> [[LOAD7]], <8 x float> [[LOAD8:%.*]], <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    [[TMP0:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> poison, <8 x float> [[VEXT165_I]], i64 0)
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> [[TMP0]], <8 x float> [[VEXT309_I]], i64 8)
-; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> poison, <8 x float> poison, i64 8)
-; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> [[TMP2]], <8 x float> [[LOAD17:%.*]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> poison, <8 x float> [[LOAD17:%.*]], i64 0)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x float> [[TMP3]], <16 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> poison, <8 x float> [[FMULADD7:%.*]], i64 0)
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> [[TMP5]], <8 x float> [[FMULADD16:%.*]], i64 8)
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll
index d822a24220df..65d0078080d2 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll
@@ -7,8 +7,7 @@ define i32 @test() {
 ; CHECK-NEXT:    br label [[IF_END_I87:%.*]]
 ; CHECK:       if.end.i87:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> getelementptr (i32, <4 x ptr> <ptr inttoptr (i64 64036 to ptr), ptr inttoptr (i64 64036 to ptr), ptr inttoptr (i64 64064 to ptr), ptr inttoptr (i64 64064 to ptr)>, <4 x i64> <i64 0, i64 1, i64 0, i64 1>), i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> poison)
-; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.vector.insert.v4i32.v2i32(<4 x i32> poison, <2 x i32> poison, i64 0)
-; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.vector.insert.v4i32.v2i32(<4 x i32> [[TMP1]], <2 x i32> zeroinitializer, i64 2)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.vector.insert.v4i32.v2i32(<4 x i32> poison, <2 x i32> zeroinitializer, i64 2)
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    switch i32 0, label [[SW_BB509_I:%.*]] [
 ; CHECK-NEXT:      i32 1, label [[SW_BB509_I]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/matched-shuffled-entries.ll b/llvm/test/Transforms/SLPVectorizer/X86/matched-shuffled-entries.ll
index 3d12da79ebf9..81845fed1134 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/matched-shuffled-entries.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/matched-shuffled-entries.ll
@@ -16,8 +16,8 @@ define i32 @bar() local_unnamed_addr {
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <16 x i32> [[TMP2]], i32 [[SUB86_1]], i32 7
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <16 x i32> [[TMP3]], i32 [[ADD78_2]], i32 9
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i32> [[TMP4]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 9, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i32> [[TMP4]], <16 x i32> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 7, i32 6, i32 5, i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x i32> [[TMP6]], i32 [[SUB102_3]], i32 12
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <16 x i32> [[TMP4]], <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 poison, i32 poison, i32 poison, i32 poison, i32 undef, i32 undef, i32 undef, i32 undef, i32 poison, i32 undef, i32 undef, i32 poison>, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 7, i32 6, i32 5, i32 4, i32 24, i32 25, i32 26, i32 27, i32 poison, i32 29, i32 30, i32 poison>
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x i32> [[TMP18]], i32 [[SUB102_3]], i32 12
 ; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i32> [[TMP7]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 12>
 ; CHECK-NEXT:    [[TMP9:%.*]] = add nsw <16 x i32> [[TMP5]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = sub nsw <16 x i32> [[TMP5]], [[TMP8]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/remark_extract_broadcast.ll b/llvm/test/Transforms/SLPVectorizer/X86/remark_extract_broadcast.ll
index a439db003f7b..049bb2e5073f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/remark_extract_broadcast.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/remark_extract_broadcast.ll
@@ -8,7 +8,7 @@ define void @fextr(ptr %ptr) {
 ; CHECK-NEXT:    [[LD:%.*]] = load <8 x i16>, ptr undef, align 16
 ; CHECK-NEXT:    br label [[T:%.*]]
 ; CHECK:       t:
-; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <8 x i16> [[LD]], <8 x i16> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <8 x i16> [[LD]], <8 x i16> <i16 poison, i16 undef, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison>, <8 x i32> <i32 0, i32 9, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <8 x i16> [[LD]], [[TMP0]]
 ; CHECK-NEXT:    store <8 x i16> [[TMP1]], ptr [[PTR:%.*]], align 2
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/insertelement-across-zero.ll b/llvm/test/Transforms/SLPVectorizer/insertelement-across-zero.ll
index 28afa40640bf..dbf24bc7721c 100644
--- a/llvm/test/Transforms/SLPVectorizer/insertelement-across-zero.ll
+++ b/llvm/test/Transforms/SLPVectorizer/insertelement-across-zero.ll
@@ -8,11 +8,11 @@ define void @test(i8 %0, i8 %1) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr null, i32 8), align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i8> [[TMP2]], <8 x i8> poison, <16 x i32> <i32 7, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[LUPTO132421:%.*]] = shufflevector <16 x i8> zeroinitializer, <16 x i8> [[TMP3]], <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <16 x i8> [[LUPTO132421]], i8 [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i8> [[LUPTO132421]], <16 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <16 x i8> [[TMP6]], i8 [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <16 x i8> [[TMP4]], i8 [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <16 x i8> [[TMP5]], i8 0, i32 7
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i8> [[TMP2]], <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP6]], <16 x i8> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP5]], <16 x i8> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <16 x i8> [[TMP8]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 7, i32 0, i32 9, i32 0, i32 0, i32 0, i32 0, i32 0, i32 15>
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i8> zeroinitializer, [[TMP9]]
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index 1d7a48981a3b..f32e31514276 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -93,14 +93,12 @@ define void @test4(ptr %in, ptr %out) {
 ; CHECK-LABEL: @test4(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x float>, ptr [[IN:%.*]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> poison, <8 x float> poison, i64 8)
-; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> [[TMP1]], <8 x float> [[TMP0]], i64 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> poison, <8 x float> [[TMP0]], i64 0)
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x float> [[TMP2]], <16 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> poison, <8 x float> zeroinitializer, i64 0)
 ; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> [[TMP4]], <8 x float> zeroinitializer, i64 8)
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul <16 x float> [[TMP3]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> poison, <8 x float> poison, i64 0)
-; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> [[TMP7]], <8 x float> zeroinitializer, i64 8)
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> poison, <8 x float> zeroinitializer, i64 8)
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <16 x float> [[TMP2]], <16 x float> [[TMP8]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[TMP10:%.*]] = fadd <16 x float> [[TMP9]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fcmp ogt <16 x float> [[TMP10]], [[TMP5]]
@@ -153,20 +151,14 @@ define <4 x i1> @test6(ptr %in1, ptr %in2) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[IN1:%.*]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[IN2:%.*]], align 2
-; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> poison, <4 x i32> poison, i64 4)
-; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> [[TMP2]], <4 x i32> poison, i64 8)
-; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> [[TMP3]], <4 x i32> poison, i64 12)
-; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> [[TMP4]], <4 x i32> [[TMP0]], i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> poison, <4 x i32> [[TMP0]], i64 0)
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> poison, <4 x i32> zeroinitializer, i64 0)
 ; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> [[TMP7]], <4 x i32> zeroinitializer, i64 4)
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> [[TMP8]], <4 x i32> zeroinitializer, i64 8)
 ; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> [[TMP9]], <4 x i32> zeroinitializer, i64 12)
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp ugt <16 x i32> [[TMP6]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v4i16(<16 x i16> poison, <4 x i16> poison, i64 4)
-; CHECK-NEXT:    [[TMP13:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v4i16(<16 x i16> [[TMP12]], <4 x i16> poison, i64 8)
-; CHECK-NEXT:    [[TMP14:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v4i16(<16 x i16> [[TMP13]], <4 x i16> poison, i64 12)
-; CHECK-NEXT:    [[TMP15:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v4i16(<16 x i16> [[TMP14]], <4 x i16> [[TMP1]], i64 0)
+; CHECK-NEXT:    [[TMP15:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v4i16(<16 x i16> poison, <4 x i16> [[TMP1]], i64 0)
 ; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <16 x i16> [[TMP15]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP17:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v4i16(<16 x i16> poison, <4 x i16> zeroinitializer, i64 0)
 ; CHECK-NEXT:    [[TMP18:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v4i16(<16 x i16> [[TMP17]], <4 x i16> zeroinitializer, i64 4)
@@ -301,8 +293,7 @@ define void @test10() {
 ; CHECK-LABEL: @test10(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr null, align 1
-; CHECK-NEXT:    [[TMP1:%.*]] = call <32 x i8> @llvm.vector.insert.v32i8.v16i8(<32 x i8> poison, <16 x i8> poison, i64 16)
-; CHECK-NEXT:    [[TMP2:%.*]] = call <32 x i8> @llvm.vector.insert.v32i8.v16i8(<32 x i8> [[TMP1]], <16 x i8> [[TMP0]], i64 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <32 x i8> @llvm.vector.insert.v32i8.v16i8(<32 x i8> poison, <16 x i8> [[TMP0]], i64 0)
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i8> [[TMP2]], <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i8> [[TMP2]], <32 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP5:%.*]] = sext <16 x i8> [[TMP4]] to <16 x i16>
diff --git a/llvm/test/Transforms/SLPVectorizer/splat-buildvector.ll b/llvm/test/Transforms/SLPVectorizer/splat-buildvector.ll
index 7532cd728225..c60ef6a18cba 100644
--- a/llvm/test/Transforms/SLPVectorizer/splat-buildvector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/splat-buildvector.ll
@@ -11,7 +11,7 @@ define i8 @foo(i64 %val_i64_57) {
 ; CHECK-NEXT:    [[VAL_I64_58:%.*]] = call i64 @llvm.smax.i64(i64 0, i64 1)
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i64> <i64 0, i64 poison, i64 poison, i64 0>, i64 [[VAL_I64_57]], i32 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i64> [[TMP0]], i64 [[VAL_I64_58]], i32 2
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> poison, <4 x i32> <i32 2, i32 2, i32 0, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> <i64 poison, i64 poison, i64 poison, i64 undef>, <4 x i32> <i32 2, i32 2, i32 2, i32 7>
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule <4 x i64> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp sle <4 x i64> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i1> [[TMP3]], <4 x i1> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
-- 
GitLab


From 0aec4d2b78a08f5f3e4b0ad0340a99e486d00110 Mon Sep 17 00:00:00 2001
From: tltao <tony.le.tao@gmail.com>
Date: Thu, 24 Oct 2024 11:02:36 -0400
Subject: [PATCH 061/693] [SystemZ] Update large ada tests after HLASM syntax
 change (#113578)

Fix buildbot failures seen on:
https://lab.llvm.org/buildbot/#/builders/42/builds/1597

caused by:  https://github.com/llvm/llvm-project/pull/113369

Co-authored-by: Tony Tao <tonytao@ca.ibm.com>
---
 llvm/test/CodeGen/SystemZ/Large/large-ada-01.py | 4 ++--
 llvm/test/CodeGen/SystemZ/Large/large-ada-02.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/test/CodeGen/SystemZ/Large/large-ada-01.py b/llvm/test/CodeGen/SystemZ/Large/large-ada-01.py
index 17efde5b519c..86d3af13232a 100644
--- a/llvm/test/CodeGen/SystemZ/Large/large-ada-01.py
+++ b/llvm/test/CodeGen/SystemZ/Large/large-ada-01.py
@@ -3,8 +3,8 @@
 # generation of a different instruction sequence
 # RUN: %python %s | llc -mtriple=s390x-ibm-zos -O2 | FileCheck %s
 
-# CHECK: llilf	1, {{[0-9]+}}
-# CHECK-NEXT: la	1, 0(1,8)
+# CHECK: llilf	1,{{[0-9]+}}
+# CHECK-NEXT: la	1,0(1,8)
 
 from __future__ import print_function
 
diff --git a/llvm/test/CodeGen/SystemZ/Large/large-ada-02.py b/llvm/test/CodeGen/SystemZ/Large/large-ada-02.py
index 45bcb276f9b9..99b260f09b31 100644
--- a/llvm/test/CodeGen/SystemZ/Large/large-ada-02.py
+++ b/llvm/test/CodeGen/SystemZ/Large/large-ada-02.py
@@ -3,8 +3,8 @@
 # generation of a different instruction sequence
 # RUN: %python %s | llc -mtriple=s390x-ibm-zos -O2 | FileCheck %s
 
-# CHECK: algfi	8, {{[0-9]+}}
-# CHECK: la	8, 0(8)
+# CHECK: algfi	8,{{[0-9]+}}
+# CHECK: la	8,0(8)
 
 from __future__ import print_function
 
-- 
GitLab


From d2e7ee77d33e8b3be3b1d4e9bc5bc4c60b62b554 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Thu, 24 Oct 2024 08:15:21 -0700
Subject: [PATCH 062/693] [SLP]Do not check for clustered loads only

Since SLP support "clusterization" of the non-load instructions, the
restriction for reduced values for loads only should be removed to avoid
compiler crash.

Fixes #113516
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  2 +-
 .../X86/non-load-reduced-as-part-of-bv.ll     | 38 +++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/non-load-reduced-as-part-of-bv.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index f76e913a57d5..012d85353f08 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -16118,7 +16118,7 @@ BoUpSLP::vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
     for (Instruction *I : RemovedInsts) {
       const TreeEntry *IE = getTreeEntry(I);
       if (IE->Idx != 0 &&
-          !(VectorizableTree.front()->isGather() && isa<LoadInst>(I) &&
+          !(VectorizableTree.front()->isGather() &&
             !IE->UserTreeIndices.empty() &&
             any_of(IE->UserTreeIndices,
                    [&](const EdgeInfo &EI) {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-load-reduced-as-part-of-bv.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-load-reduced-as-part-of-bv.ll
new file mode 100644
index 000000000000..2f49a2e6a212
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-load-reduced-as-part-of-bv.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i1 @foo() {
+; CHECK-LABEL: define i1 @foo() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TOBOOL_NOT_NOT509_I_2329_I_I:%.*]] = icmp ne i32 0, 0
+; CHECK-NEXT:    [[STOREMERGE_2333_I_I:%.*]] = select i1 [[TOBOOL_NOT_NOT509_I_2329_I_I]], i32 0, i32 0
+; CHECK-NEXT:    [[TOBOOL_NOT_NOT509_I_1_2_I_I:%.*]] = icmp ne i32 [[STOREMERGE_2333_I_I]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i1> poison, i1 [[TOBOOL_NOT_NOT509_I_1_2_I_I]], i32 4
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i1> [[TMP0]], i1 [[TOBOOL_NOT_NOT509_I_2329_I_I]], i32 5
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.vector.insert.v8i1.v4i1(<8 x i1> [[TMP1]], <4 x i1> zeroinitializer, i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i1> @llvm.vector.insert.v8i1.v2i1(<8 x i1> [[TMP2]], <2 x i1> zeroinitializer, i64 6)
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze <8 x i1> [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP4]])
+; CHECK-NEXT:    [[OP_RDX:%.*]] = select i1 false, i1 [[TMP5]], i1 false
+; CHECK-NEXT:    ret i1 [[OP_RDX]]
+;
+entry:
+  %tobool.not.not509.i.1311.i.i = icmp ne i32 0, 0
+  %cmp80.1319.i.i189 = icmp sgt i32 0, 0
+  %tobool.not.not509.i.1.1.i.i = icmp ne i32 0, 0
+  %cmp80.1.1.i.i190 = icmp sgt i32 0, 0
+  %tobool.not.not509.i.2329.i.i = icmp ne i32 0, 0
+  %storemerge.2333.i.i = select i1 %tobool.not.not509.i.2329.i.i, i32 0, i32 0
+  %cmp80.2337.i.i192 = icmp sgt i32 0, 0
+  %tobool.not.not509.i.1.2.i.i = icmp ne i32 %storemerge.2333.i.i, 0
+  %cmp80.1.2.i.i193 = icmp sgt i32 0, 0
+  %cmp80.1.2.i.i = select i1 %tobool.not.not509.i.1.2.i.i, i1 %cmp80.1.2.i.i193, i1 false
+  %0 = select i1 %cmp80.1.2.i.i, i1 %tobool.not.not509.i.2329.i.i, i1 false
+  %1 = select i1 %0, i1 %cmp80.2337.i.i192, i1 false
+  %2 = select i1 %1, i1 false, i1 false
+  %3 = select i1 %2, i1 %tobool.not.not509.i.1.1.i.i, i1 false
+  %4 = select i1 %3, i1 %cmp80.1.1.i.i190, i1 false
+  %5 = select i1 %4, i1 %tobool.not.not509.i.1311.i.i, i1 false
+  %6 = select i1 %5, i1 %cmp80.1319.i.i189, i1 false
+  ret i1 %6
+}
-- 
GitLab


From 61a76f58ebf161c739fb196d56c1899735c7cea8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Endre=20F=C3=BCl=C3=B6p?= <endre.fulop@sigmatechnology.com>
Date: Thu, 24 Oct 2024 17:37:04 +0200
Subject: [PATCH 063/693] [clang][analyzer][doc] Migrate ClangSA www FAQ
 section (#112831)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The ClangSA documentation lives in RST format, and the FAQ section of
the old webpage is also migrated from HTML with this change.

---------

Co-authored-by: Donát Nagy <donat.nagy@ericsson.com>
---
 clang/docs/analyzer/user-docs.rst     |   1 +
 clang/docs/analyzer/user-docs/FAQ.rst | 208 ++++++++++++++++++++++
 clang/www/analyzer/faq.html           | 246 +-------------------------
 3 files changed, 216 insertions(+), 239 deletions(-)
 create mode 100644 clang/docs/analyzer/user-docs/FAQ.rst

diff --git a/clang/docs/analyzer/user-docs.rst b/clang/docs/analyzer/user-docs.rst
index 08cb5119e810..dd53ae143148 100644
--- a/clang/docs/analyzer/user-docs.rst
+++ b/clang/docs/analyzer/user-docs.rst
@@ -12,3 +12,4 @@ Contents:
    user-docs/FilingBugs
    user-docs/CrossTranslationUnit
    user-docs/TaintAnalysisConfiguration
+   user-docs/FAQ
diff --git a/clang/docs/analyzer/user-docs/FAQ.rst b/clang/docs/analyzer/user-docs/FAQ.rst
new file mode 100644
index 000000000000..af52e99c91d6
--- /dev/null
+++ b/clang/docs/analyzer/user-docs/FAQ.rst
@@ -0,0 +1,208 @@
+FAQ and How to Deal with Common False Positives
+===============================================
+
+.. contents::
+   :local:
+
+Custom Assertions
+-----------------
+
+Q: How do I tell the analyzer that I do not want the bug being reported here since my custom error handler will safely end the execution before the bug is reached?
+
+You can tell the analyzer that this path is unreachable by teaching it about your `custom assertion handlers <annotations.html#custom_assertions>`_. For example, you can modify the code segment as following:
+
+.. code-block:: c
+
+   void customAssert() __attribute__((analyzer_noreturn));
+   int foo(int *b) {
+     if (!b)
+       customAssert();
+     return *b;
+   }
+
+Null Pointer Dereference
+------------------------
+
+Q: The analyzer reports a null dereference, but I know that the pointer is never null. How can I tell the analyzer that a pointer can never be null?
+
+The reason the analyzer often thinks that a pointer can be null is because the preceding code checked compared it against null. If you are absolutely sure that it cannot be null, remove the preceding check and, preferably, add an assertion as well. For example:
+
+.. code-block:: c
+
+   void usePointer(int *b);
+   int foo(int *b) {
+     usePointer(b);
+     return *b;
+   }
+
+Dead Store
+----------
+
+Q: How do I tell the static analyzer that I don't care about a specific dead store?
+
+When the analyzer sees that a value stored into a variable is never used, it's going to produce a message similar to this one:
+
+.. code-block:: none
+
+   Value stored to 'x' is never read
+
+You can use the ``(void)x;`` idiom to acknowledge that there is a dead store in your code but you do not want it to be reported in the future.
+
+Unused Instance Variable
+------------------------
+
+Q: How do I tell the static analyzer that I don't care about a specific unused instance variable in Objective-C?
+
+When the analyzer sees that a value stored into a variable is never used, it is going to produce a message similar to this one:
+
+.. code-block:: none
+
+   Instance variable 'commonName' in class 'HappyBird' is never used by the methods in its @implementation
+
+You can add ``__attribute__((unused))`` to the instance variable declaration to suppress the warning.
+
+Unlocalized String
+------------------
+
+Q: How do I tell the static analyzer that I don't care about a specific unlocalized string?
+
+When the analyzer sees that an unlocalized string is passed to a method that will present that string to the user, it is going to produce a message similar to this one:
+
+.. code-block:: none
+
+   User-facing text should use localized string macro
+
+If your project deliberately uses unlocalized user-facing strings (for example, in a debugging UI that is never shown to users), you can suppress the analyzer warnings (and document your intent) with a function that just returns its input but is annotated to return a localized string:
+
+.. code-block:: objc
+
+   __attribute__((annotate("returns_localized_nsstring")))
+   static inline NSString *LocalizationNotNeeded(NSString *s) {
+     return s;
+   }
+
+You can then call this function when creating your debugging UI:
+
+.. code-block:: objc
+
+   [field setStringValue:LocalizationNotNeeded(@"Debug")];
+
+Some projects may also find it useful to use NSLocalizedString but add "DNL" or "Do Not Localize" to the string contents as a convention:
+
+.. code-block:: objc
+
+   UILabel *testLabel = [[UILabel alloc] init];
+   NSString *s = NSLocalizedString(@"Hello <Do Not Localize>", @"For debug purposes");
+   [testLabel setText:s];
+
+Dealloc in Manual Retain/Release
+--------------------------------
+
+Q: How do I tell the analyzer that my instance variable does not need to be released in -dealloc under Manual Retain/Release?
+
+If your class only uses an instance variable for part of its lifetime, it may maintain an invariant guaranteeing that the instance variable is always released before -dealloc. In this case, you can silence a warning about a missing release by either adding ``assert(_ivar == nil)`` or an explicit release ``[_ivar release]`` (which will be a no-op when the variable is nil) in -dealloc.
+
+Deciding Nullability
+--------------------
+
+Q: How do I decide whether a method's return type should be _Nullable or _Nonnull?
+
+Depending on the implementation of the method, this puts you in one of five situations:
+
+1. You actually never return nil.
+2. You do return nil sometimes, and callers are supposed to handle that. This includes cases where your method is documented to return nil given certain inputs.
+3. You return nil based on some external condition (such as an out-of-memory error), but the client can't do anything about it either.
+4. You return nil only when the caller passes input documented to be invalid. That means it's the client's fault.
+5. You return nil in some totally undocumented case.
+
+In (1) you should annotate the method as returning a ``_Nonnull`` object.
+
+In (2) the method should be marked ``_Nullable``.
+
+In (3) you should probably annotate the method ``_Nonnull``. Why? Because no callers will actually check for nil, given that they can't do anything about the situation and don't know what went wrong. At this point things have gone so poorly that there's basically no way to recover.
+
+The least happy case is (4) because the resulting program will almost certainly either crash or just silently do the wrong thing. If this is a new method or you control the callers, you can use ``NSParameterAssert()`` (or the equivalent) to check the precondition and remove the nil return. But if you don't control the callers and they rely on this behavior, you should return mark the method ``_Nonnull`` and return nil cast to _Nonnull anyway.
+
+If you're in (5), document it, then figure out if you're now in (2), (3), or (4).
+
+Intentional Nullability Violation
+---------------------------------
+
+Q: How do I tell the analyzer that I am intentionally violating nullability?
+
+In some cases, it may make sense for methods to intentionally violate nullability. For example, your method may — for reasons of backward compatibility — chose to return nil and log an error message in a method with a non-null return type when the client violated a documented precondition rather than check the precondition with ``NSAssert()``. In these cases, you can suppress the analyzer warning with a cast:
+
+.. code-block:: objc
+
+   return (id _Nonnull)nil;
+
+Note that this cast does not affect code generation.
+
+Ensuring Loop Body Execution
+----------------------------
+
+Q: The analyzer assumes that a loop body is never entered. How can I tell it that the loop body will be entered at least once?
+
+In cases where you know that a loop will always be entered at least once, you can use assertions to inform the analyzer. For example:
+
+.. code-block:: c
+
+   int foo(int length) {
+     int x = 0;
+     assert(length > 0);
+     for (int i = 0; i < length; i++)
+       x += 1;
+     return length/x;
+   }
+
+By adding ``assert(length > 0)`` in the beginning of the function, you tell the analyzer that your code is never expecting a zero or a negative value, so it won't need to test the correctness of those paths.
+
+Suppressing Specific Warnings
+-----------------------------
+
+Q: How can I suppress a specific analyzer warning?
+
+When you encounter an analyzer bug/false positive, check if it's one of the issues discussed above or if the analyzer `annotations <annotations.html#custom_assertions>`_ can resolve the issue by helping the static analyzer understand the code better. Second, please `report it <filing_bugs.html>`_ to help us improve user experience.
+
+Sometimes there's really no "good" way to eliminate the issue. In such cases you can "silence" it directly by annotating the problematic line of code with the help of Clang attribute 'suppress':
+
+.. code-block:: c
+
+   int foo() {
+     int *x = nullptr;
+     ...
+     [[clang::suppress]] {
+       // all warnings in this scope are suppressed
+       int y = *x;
+     }
+
+     // null pointer dereference warning suppressed on the next line
+     [[clang::suppress]]
+     return *x
+   }
+
+   int bar(bool coin_flip) {
+     // suppress all memory leak warnings about this allocation
+     [[clang::suppress]]
+     int *result = (int *)malloc(sizeof(int));
+
+     if (coin_flip)
+       return 0;      // including this leak path
+
+     return *result;  // as well as this leak path
+   }
+
+Excluding Code from Analysis
+----------------------------
+
+Q: How can I selectively exclude code the analyzer examines?
+
+When the static analyzer is using clang to parse source files, it implicitly defines the preprocessor macro ``__clang_analyzer__``. One can use this macro to selectively exclude code the analyzer examines. Here is an example:
+
+.. code-block:: c
+
+   #ifndef __clang_analyzer__
+   // Code not to be analyzed
+   #endif
+
+This usage is discouraged because it makes the code dead to the analyzer from now on. Instead, we prefer that users file bugs against the analyzer when it flags false positives.
diff --git a/clang/www/analyzer/faq.html b/clang/www/analyzer/faq.html
index 156d383db2f2..b447968087fe 100644
--- a/clang/www/analyzer/faq.html
+++ b/clang/www/analyzer/faq.html
@@ -3,12 +3,11 @@
 <html>
 <head>
   <title>FAQ and How to Deal with Common False Positives</title>
+  <link rel="canonical" href="https://clang.llvm.org/docs/analyzer/user-docs/FAQ.html"/>
+  <meta http-equiv="refresh" content="0;url=https://clang.llvm.org/docs/analyzer/user-docs/FAQ.html" />
   <link type="text/css" rel="stylesheet" href="menu.css">
   <link type="text/css" rel="stylesheet" href="content.css">
   <script type="text/javascript" src="scripts/menu.js"></script>
-  <style type="text/css">
-    tr:first-child { width:20%; }
-  </style>
 </head>
 <body>
 
@@ -18,242 +17,11 @@
 <div id="content">
 
 <h1>FAQ and How to Deal with Common False Positives</h1>
+<p style="color:red; font-size:200%">This page is deprecated and will be removed in release 21.0</p>
+<p>Its content was migrated to <a href="https://clang.llvm.org/docs/analyzer/user-docs/FAQ.html">the regular LLVM documentation</a>.</p>
+<script>window.location='https://clang.llvm.org/docs/analyzer/user-docs/FAQ.html'</script>
 
-<ol>
-  <li><a href="#custom_assert">How do I tell the analyzer that I do not want the bug being
-reported here since my custom error handler will safely end the execution before
-the bug is reached?</a></li>
-  <li><a href="#null_pointer">The analyzer reports a null dereference, but I know that the
-pointer is never null. How can I tell the analyzer that a pointer can never be
-null?</a></li>
-  <li><a href="#dead_store">How do I tell the static analyzer that I don't care about a specific dead store?</a></li>
-  <li><a href="#unused_ivar">How do I tell the static analyzer that I don't care about a specific unused instance variable in Objective C?</a></li>
-  <li><a href="#unlocalized_string">How do I tell the static analyzer that I don't care about a specific unlocalized string?</a></li>
-  <li><a href="#dealloc_mrr">How do I tell the analyzer that my instance variable does not need to be released in -dealloc under Manual Retain/Release?</a></li>
-  <li><a href="#decide_nullability">How do I decide whether a method's return type should be _Nullable or _Nonnull?</a></li>
-  <li><a href="#nullability_intentional_violation">How do I tell the analyzer that I am intentionally violating nullability?</a></li>
-  <li><a href="#use_assert">The analyzer assumes that a loop body is never entered.  How can I tell it that the loop body will be entered at least once?</a></li>
-  <li><a href="#suppress_issue">How can I suppress a specific analyzer warning?</a></li>
-  <li><a href="#exclude_code">How can I selectively exclude code the analyzer examines?</a></li>
-</ol>
-
-
-<h4 id="custom_assert" class="faq">Q: How do I tell the analyzer that I do not want the bug being
-reported here since my custom error handler will safely end the execution before
-the bug is reached?</h4>
-
-<img src="images/example_custom_assert.png" alt="example custom assert">
-
-<p>You can tell the analyzer that this path is unreachable by teaching it about your <a href = "annotations.html#custom_assertions" >custom assertion handlers</a>. For example, you can modify the code segment as following.</p>
-
-<pre class="code_example">
-void customAssert() <span class="code_highlight">__attribute__((analyzer_noreturn))</span>;
-int foo(int *b) {
-  if (!b)
-    customAssert();
-  return *b;
-}</pre>
-
-
-<h4 id="null_pointer" class="faq">Q: The analyzer reports a null dereference, but I know that the
-pointer is never null. How can I tell the analyzer that a pointer can never be
-null?</h4>
-
-<img src="images/example_null_pointer.png" alt="example null pointer">
-
-<p>The reason the analyzer often thinks that a pointer can be null is because the preceding code checked compared it against null. So if you are absolutely sure that it cannot be null, remove the preceding check and, preferably, add an assertion as well. For example, in the code segment above, it will be sufficient to remove the <tt>if (!b)</tt> check. </p>
-
-<pre class="code_example">
-void usePointer(int *b);
-int foo(int *b) {
-  usePointer(b);
-  return *b;
-}</pre>
-
-<h4 id="dead_store" class="faq">Q: How do I tell the static analyzer that I don't care about a specific dead store?</h4>
-
-<p>When the analyzer sees that a value stored into a variable is never used, it's going to produce a message similar to this one:
-<pre class="code_example">Value stored to 'x' is never read</pre>
-You can use the <tt>(void)x;</tt> idiom to acknowledge that there is a dead store in your code but you do not want it to be reported in the future.</p>
-
-<h4 id="unused_ivar" class="faq">Q: How do I tell the static analyzer that I don't care about a specific unused instance variable in Objective C?</h4>
-
-<p>When the analyzer sees that a value stored into a variable is never used, it is going to produce a message similar to this one:
-<pre class="code_example">Instance variable 'commonName' in class 'HappyBird' is never used by the methods in its @implementation</pre>
-You can add <tt>__attribute__((unused))</tt> to the instance variable declaration to suppress the warning.</p>
-
-<h4 id="unlocalized_string" class="faq">Q: How do I tell the static analyzer that I don't care about a specific unlocalized string?</h4>
-
-<p>When the analyzer sees that an unlocalized string is passed to a method that will present that string to the user, it is going to produce a message similar to this one:
-<pre class="code_example">User-facing text should use localized string macro</pre>
-
-If your project deliberately uses unlocalized user-facing strings (for example, in a debugging UI that is never shown to users), you can suppress the analyzer warnings (and document your intent) with a function that just returns its input but is annotated to return a localized string:
-<pre class="code_example">
-__attribute__((annotate("returns_localized_nsstring")))
-static inline NSString *LocalizationNotNeeded(NSString *s) {
-  return s;
-}
-</pre>
-
-You can then call this function when creating your debugging UI:
-<pre class="code_example">
-[field setStringValue:LocalizationNotNeeded(@"Debug")];
-</pre>
-
-Some projects may also find it useful to use NSLocalizedString but add "DNL" or "Do Not Localize" to the string contents as a convention:
-<pre class="code_example">
-UILabel *testLabel = [[UILabel alloc] init];
-NSString *s = NSLocalizedString(@"Hello &lt;Do Not Localize&gt;", @"For debug purposes");
-[testLabel setText:s];
-</pre>
-</p>
-
-<h4 id="dealloc_mrr" class="faq">Q: How do I tell the analyzer that my instance variable does not need to be released in -dealloc under Manual Retain/Release?</h4>
-
-<p>If your class only uses an instance variable for part of its lifetime, it may
-maintain an invariant guaranteeing that the instance variable is always released
-before -dealloc. In this case, you can silence a warning about a missing release
-by either adding <tt>assert(_ivar == nil)</tt> or an explicit release
-<tt>[_ivar release]</tt> (which will be a no-op when the variable is nil) in
--dealloc. </p>
-
-<h4 id="decide_nullability" class="faq">Q: How do I decide whether a method's return type should be _Nullable or _Nonnull?</h4>
-
-<p> Depending on the implementation of the method, this puts you in one of five situations:
-<ol>
-<li>You actually never return nil.</li>
-<li>You do return nil sometimes, and callers are supposed to handle that. This
-includes cases where your method is documented to return nil given certain
-inputs.</li>
-<li>You return nil based on some external condition (such as an out-of-memory
-error), but the client can't do anything about it either.</li>
-<li>You return nil only when the caller passes input documented to be invalid.
-That means it's the client's fault.</li>
-<li>You return nil in some totally undocumented case.</li>
-</ol>
-</p>
-
-<p>In (1) you should annotate the method as returning a <tt>_Nonnull</tt>
-object.</p>
-<p>In (2) the method should be marked <tt>_Nullable.</tt></p>
-<p>In (3) you should probably annotate the method <tt>_Nonnull</tt>. Why?
-Because no callers will actually check for nil, given that they can't do
-anything about the situation and don't know what went wrong. At this point
-things have gone so poorly that there's basically no way to recover.</p>
-<p>The least happy case is (4) because the resulting program will almost
-certainly either crash or just silently do the wrong thing.
-If this is a new method or you control the callers, you can use
-<tt>NSParameterAssert()</tt> (or the equivalent) to check the precondition and
-remove the nil return. But if you don't control the callers and they rely on
-this behavior, you should return mark the method <tt>_Nonnull</tt> and return
-nil <a href="#nullability_intentional_violation">cast to _Nonnull</a> anyway.
-(Note that (4) doesn't apply in cases where the caller can't know they passed
-bad parameters. For example,
-<tt>+[NSData dataWithContentsOfFile:options:error:]</tt> will fail if the file
-doesn't exist, but there's no way to check for that in advance. This means
-you're really in (2).)</p>
-<p>If you're in (5), document it, then figure out if you're now in (2), (3), or
-(4). :-)</p>
-
-<h4 id="nullability_intentional_violation" class="faq">Q: How do I tell the analyzer that I am intentionally violating nullability?</h4>
-
-<p>In some cases, it may make sense for methods to intentionally violate
-nullability. For example, your method may &mdash; for reasons of backward
-compatibility &mdash; chose to return nil and log an error message in a method
-with a non-null return type when the client violated a documented precondition
-rather than check the precondition with <tt>NSAssert()</tt>. In these cases, you
-can suppress the analyzer warning with a cast:
-<pre class="code_example">
-    return (id _Nonnull)nil;
-</pre>
-Note that this cast does not affect code generation.
-</p>
-
-<h4 id="use_assert" class="faq">Q: The analyzer assumes that a loop body is never entered.  How can I tell it that the loop body will be entered at least once?</h4>
-
-<img src="images/example_use_assert.png" alt="example use assert">
-
-<p> In the contrived example above, the analyzer has detected that the body of
-the loop is never entered for the case where <tt>length <= 0</tt>. In this
-particular example, you may know that the loop will always be entered because
-the input parameter <tt>length</tt> will be greater than zero in all calls to this
-function. You can teach the analyzer facts about your code as well as document
-it by using assertions. By adding <tt>assert(length > 0)</tt> in the beginning
-of the function, you tell the analyzer that your code is never expecting a zero
-or a negative value, so it won't need to test the correctness of those paths.
-</p>
-
-<pre class="code_example">
-int foo(int length) {
-  int x = 0;
-  <span class="code_highlight">assert(length > 0);</span>
-  for (int i = 0; i < length; i++)
-    x += 1;
-  return length/x;
-}
-</pre>
-
-<h4 id="suppress_issue" class="faq">Q: How can I suppress a specific analyzer warning?</h4>
-
-<p>When you encounter an analyzer bug/false positive, check if it's one of the
-issues discussed above or if the analyzer
-<a href = "annotations.html#custom_assertions" >annotations</a> can
-resolve the issue by helping the static analyzer understand the code better.
-Second, please <a href = "filing_bugs.html">report it</a> to help us improve
-user experience.</p>
-
-<p>Sometimes there's really no "good" way to eliminate the issue. In such cases
-you can "silence" it directly by annotating the problematic line of code with
-the help of Clang attribute '<a href="https://clang.llvm.org/docs/AttributeReference.html#suppress">suppress</a>':
-
-<pre class="code_example">
-int foo() {
-  int *x = nullptr;
-  ...
-  <span class="code_highlight">[[clang::suppress]]</span> {
-    // all warnings in this scope are suppressed
-    int y = *x;
-  }
-
-  // null pointer dereference warning suppressed on the next line
-  <span class="code_highlight">[[clang::suppress]]</span>
-  return *x
-}
-
-int bar(bool coin_flip) {
-  // suppress all memory leak warnings about this allocation
-  <span class="code_highlight">[[clang::suppress]]</span>
-  int *result = (int *)malloc(sizeof(int));
-
-  if (coin_flip)
-    return 0;      // including this leak path
-
-  return *result;  // as well as this leak path
-}
-</pre>
-
-
-<p>You can also consider using <tt>__clang_analyzer__</tt> macro
-<a href = "faq.html#exclude_code" >described below</a>.</p>
-
-<h4 id="exclude_code" class="faq">Q: How can I selectively exclude code the analyzer examines?</h4>
-
-<p>When the static analyzer is using clang to parse source files, it implicitly
-defines the preprocessor macro <tt>__clang_analyzer__</tt>. One can use this
-macro to selectively exclude code the analyzer examines. Here is an example:
-
-<pre class="code_example">
-#ifndef __clang_analyzer__
-// Code not to be analyzed
-#endif
-</pre>
-
-This usage is discouraged because it makes the code dead to the analyzer from
-now on. Instead, we prefer that users file bugs against the analyzer when it flags
-false positives.
-</p>
-
-</div>
-</div>
+</div> <!-- content -->
+</div> <!-- page -->
 </body>
 </html>
-- 
GitLab


From 3a2c957c4b11babb6c2111b8827a324ffac5bab8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Endre=20F=C3=BCl=C3=B6p?= <endre.fulop@sigmatechnology.com>
Date: Thu, 24 Oct 2024 17:42:54 +0200
Subject: [PATCH 064/693] [clang][analyzer][doc] Update Clang SA www docs
 index.html (#112833)

Downloads and releases sections are removed, users are advised to use
the conventional ways of acquiring the analyzer.
---
 clang/www/analyzer/index.html | 67 +++++++++++++----------------------
 1 file changed, 24 insertions(+), 43 deletions(-)

diff --git a/clang/www/analyzer/index.html b/clang/www/analyzer/index.html
index ea0ed4967ec7..736ac4aa0f65 100644
--- a/clang/www/analyzer/index.html
+++ b/clang/www/analyzer/index.html
@@ -72,45 +72,9 @@
 <p>The Clang Static Analyzer is a source code analysis tool that finds bugs in
 C, C++, and Objective-C programs.</p>
 
-<p>Currently it can be run either from the <a href="command-line.html">command
-  line</a> or if you use macOS then <a href="xcode.html">within Xcode</a>. When
-invoked from the command line, it is intended to be run in tandem with a build
-of a codebase.</p>
-
-<p>The analyzer is 100% open source and is part of the <a
-href="https://clang.llvm.org">Clang</a> project. Like the rest of Clang, the
-analyzer is implemented as a C++ library that can be used by other tools and
-applications.</p>
-
-<h2>Download</h2>
-
-<div style="padding:0px; font-size: 90%">
- <b class="spiffy">
- <b class="spiffy1"><b></b></b>
- <b class="spiffy2"><b></b></b>
- <b class="spiffy3"></b>
- <b class="spiffy4"></b>
- <b class="spiffy5"></b></b>
- <div class="spiffyfg">
-  <div style="padding:15px">
-   <h3 style="margin:0px;padding:0px">Mac OS X</h3>
-   <ul>
-    <li>Latest build (10.8+):<br>
-     <b><a href="downloads/checker-279.tar.bz2">checker-279.tar.bz2</a></b> (built November 14, 2016)
-    </li>
-    <li><a href="/release_notes.html">Release notes</a></li>
-    <li>This build can be used both from the command line and from within Xcode</li>
-    <li><a href="/installation.html">Installation</a> and <a href="/scan-build.html">usage</a></li>
-   </ul>
-  </div>
- </div>
- <b class="spiffy">
- <b class="spiffy5"></b>
- <b class="spiffy4"></b>
- <b class="spiffy3"></b>
- <b class="spiffy2"><b></b></b>
- <b class="spiffy1"><b></b></b></b>
-</div>
+<p>The analyzer is 100% open source and is part of the <a href="https://clang.llvm.org">Clang</a>
+project. Like the rest of Clang, the analyzer is implemented as a C++ library
+that can be used by other tools and applications.</p>
 
 <div style="padding:0; margin-top:10px; font-size: 90%">
  <b class="spiffy">
@@ -121,10 +85,27 @@ applications.</p>
  <b class="spiffy5"></b></b>
  <div class="spiffyfg">
   <div style="padding:15px">
-   <h3 style="margin:0px;padding:0px">Other Platforms</h3>
-   <p>For other platforms, please follow the instructions for <a
-   href="/installation#OtherPlatforms">building the analyzer</a> from
-   source code.<p>
+
+    <p>To get started with the Clang Static Analyzer, visit the
+    <a href="https://releases.llvm.org/download.html">LLVM
+    releases page</a> for download and installation instructions. The official
+    releases include both the analyzer and <a href="scan-build.html">scan-build</a>,
+    a command-line tool for running the analyzer on your codebase.</p>
+
+    <p>If you're installing Clang from a different source, such as
+    a Linux package repository, then scan-build may be packaged
+    separately as an individual package, or as part of
+    a "clang tools" package.</p>
+
+    <p>If your IDE is using Clang, it may natively integrate the static analyzer.
+    On macOS, the easiest way to use the static analyzer is to invoke it
+    <a href="https://clang.llvm.org/docs/analyzer/user-docs/UsingWithXCode.html">directly
+    from Xcode</a>.</p>
+
+    <p>Additionally, if you're using <a href="https://clang.llvm.org/extra/clang-tidy/">clang-tidy</a>,
+    you can naturally make the static analyzer run alongside clang-tidy
+    by enabling the <a href="https://clang.llvm.org/extra/clang-tidy/checks/list.html">clang-analyzer</a>
+    checks.</p>
   </div>
  </div>
  <b class="spiffy">
-- 
GitLab


From 493529fbcee07b1ea17c3abfdea85cfa226a04a4 Mon Sep 17 00:00:00 2001
From: Oliver Stannard <oliver.stannard@arm.com>
Date: Thu, 17 Oct 2024 10:32:44 +0200
Subject: [PATCH 065/693] Re-land: [ARM] Fix frame chains with M-profile PACBTI
 (#110285)

When using AAPCS-compliant frame chains with PACBTI return address
signing, there ware a number of bugs in the generation of the frame
pointer and function prologues. The most obvious was that we sometimes
would modify r11 before pushing it to the stack, so it wasn't preserved
as required by the PCS. We also sometimes did not push R11 and LR
adjacent to one another on the stack, or used R11 as a frame pointer
without pointing it at the saved value of R11, both of which are
required to have an AAPCS compliant frame chain.

The original work of this patch was done by James Westwood, reviewed as
 #82801 and #81249, with some tidy-ups done by Mark Murray and myself.
---
 llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp   |   5 +-
 llvm/lib/Target/ARM/ARMCallingConv.td         |  19 ++-
 llvm/lib/Target/ARM/ARMFrameLowering.cpp      | 145 +++++++++++------
 llvm/lib/Target/ARM/ARMSubtarget.cpp          |   7 +
 llvm/lib/Target/ARM/ARMSubtarget.h            |  12 ++
 .../CodeGen/Thumb2/pacbti-m-frame-chain.ll    | 150 ++++++++++++++++++
 6 files changed, 281 insertions(+), 57 deletions(-)
 create mode 100644 llvm/test/CodeGen/Thumb2/pacbti-m-frame-chain.ll

diff --git a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
index 3f28ce8ca4b5..aad305cce039 100644
--- a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
@@ -116,9 +116,12 @@ ARMBaseRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
     return CSR_iOS_SaveList;
 
   if (PushPopSplit == ARMSubtarget::SplitR7)
-    return STI.createAAPCSFrameChain() ? CSR_AAPCS_SplitPush_SaveList
+    return STI.createAAPCSFrameChain() ? CSR_AAPCS_SplitPush_R7_SaveList
                                        : CSR_ATPCS_SplitPush_SaveList;
 
+  if (PushPopSplit == ARMSubtarget::SplitR11AAPCSSignRA)
+    return CSR_AAPCS_SplitPush_R11_SaveList;
+
   return CSR_AAPCS_SaveList;
 }
 
diff --git a/llvm/lib/Target/ARM/ARMCallingConv.td b/llvm/lib/Target/ARM/ARMCallingConv.td
index d14424c2deca..27f175a70033 100644
--- a/llvm/lib/Target/ARM/ARMCallingConv.td
+++ b/llvm/lib/Target/ARM/ARMCallingConv.td
@@ -301,14 +301,17 @@ def CSR_ATPCS_SplitPush_SwiftError : CalleeSavedRegs<(sub CSR_ATPCS_SplitPush,
 def CSR_ATPCS_SplitPush_SwiftTail : CalleeSavedRegs<(sub CSR_ATPCS_SplitPush,
                                                      R10)>;
 
-// When enforcing an AAPCS compliant frame chain, R11 is used as the frame
-// pointer even for Thumb targets, where split pushes are necessary.
-// This AAPCS alternative makes sure the frame index slots match the push
-// order in that case.
-def CSR_AAPCS_SplitPush : CalleeSavedRegs<(add LR, R11,
-                                               R7, R6, R5, R4,
-                                               R10, R9, R8,
-                                               (sequence "D%u", 15, 8))>;
+// Sometimes we need to split the push of the callee-saved GPRs into two
+// regions, to ensure that the frame chain record is set up correctly. These
+// list the callee-saved registers in the order they end up on the stack, which
+// depends on whether the frame pointer is r7 or r11.
+def CSR_AAPCS_SplitPush_R11 : CalleeSavedRegs<(add R10, R9, R8, R7, R6, R5, R4,
+                                                   LR, R11,
+                                                   (sequence "D%u", 15, 8))>;
+def CSR_AAPCS_SplitPush_R7 : CalleeSavedRegs<(add LR, R11,
+                                                  R7, R6, R5, R4,
+                                                  R10, R9, R8,
+                                                  (sequence "D%u", 15, 8))>;
 
 // Constructors and destructors return 'this' in the ARM C++ ABI; since 'this'
 // and the pointer return value are both passed in R0 in these cases, this can
diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
index e0703457aa81..4f366dcffcd2 100644
--- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
@@ -199,6 +199,11 @@ SpillArea getSpillArea(Register Reg,
   // push {r0-r10, r12}  GPRCS1
   // vpush {r8-d15}      DPRCS1
   // push {r11, lr}      GPRCS2
+  //
+  // SplitR11AAPCSSignRA:
+  // push {r0-r10, r12}  GPRSC1
+  // push {r11, lr}      GPRCS2
+  // vpush {r8-d15}      DPRCS1
 
   // If FPCXTNS is spilled (for CMSE secure entryfunctions), it is always at
   // the top of the stack frame.
@@ -246,7 +251,8 @@ SpillArea getSpillArea(Register Reg,
       return SpillArea::GPRCS1;
 
   case ARM::LR:
-    if (Variation == ARMSubtarget::SplitR11WindowsSEH)
+    if (Variation == ARMSubtarget::SplitR11WindowsSEH ||
+        Variation == ARMSubtarget::SplitR11AAPCSSignRA)
       return SpillArea::GPRCS2;
     else
       return SpillArea::GPRCS1;
@@ -863,6 +869,9 @@ static int getMaxFPOffset(const ARMSubtarget &STI, const ARMFunctionInfo &AFI,
   // This is a conservative estimation: Assume the frame pointer being r7 and
   // pc("r15") up to r8 getting spilled before (= 8 registers).
   int MaxRegBytes = 8 * 4;
+  if (PushPopSplit == ARMSubtarget::SplitR11AAPCSSignRA)
+    // Here, r11 can be stored below all of r4-r15.
+    MaxRegBytes = 11 * 4;
   if (PushPopSplit == ARMSubtarget::SplitR11WindowsSEH) {
     // Here, r11 can be stored below all of r4-r15 plus d8-d15.
     MaxRegBytes = 11 * 4 + 8 * 8;
@@ -935,17 +944,23 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
   }
 
   // Determine spill area sizes, and some important frame indices.
+  SpillArea FramePtrSpillArea = SpillArea::GPRCS1;
+  bool BeforeFPPush = true;
   for (const CalleeSavedInfo &I : CSI) {
     Register Reg = I.getReg();
     int FI = I.getFrameIdx();
 
-    if (Reg == FramePtr)
+    SpillArea Area = getSpillArea(Reg, PushPopSplit,
+                                  AFI->getNumAlignedDPRCS2Regs(), RegInfo);
+
+    if (Reg == FramePtr) {
       FramePtrSpillFI = FI;
+      FramePtrSpillArea = Area;
+    }
     if (Reg == ARM::D8)
       D8SpillFI = FI;
 
-    switch (getSpillArea(Reg, PushPopSplit, AFI->getNumAlignedDPRCS2Regs(),
-                         RegInfo)) {
+    switch (Area) {
     case SpillArea::FPCXT:
       FPCXTSaveSize += 4;
       break;
@@ -972,7 +987,7 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
   // Move past FPCXT area.
   if (FPCXTSaveSize > 0) {
     LastPush = MBBI++;
-    DefCFAOffsetCandidates.addInst(LastPush, FPCXTSaveSize, true);
+    DefCFAOffsetCandidates.addInst(LastPush, FPCXTSaveSize, BeforeFPPush);
   }
 
   // Allocate the vararg register save area.
@@ -980,13 +995,15 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
     emitSPUpdate(isARM, MBB, MBBI, dl, TII, -ArgRegsSaveSize,
                  MachineInstr::FrameSetup);
     LastPush = std::prev(MBBI);
-    DefCFAOffsetCandidates.addInst(LastPush, ArgRegsSaveSize, true);
+    DefCFAOffsetCandidates.addInst(LastPush, ArgRegsSaveSize, BeforeFPPush);
   }
 
   // Move past area 1.
   if (GPRCS1Size > 0) {
     GPRCS1Push = LastPush = MBBI++;
-    DefCFAOffsetCandidates.addInst(LastPush, GPRCS1Size, true);
+    DefCFAOffsetCandidates.addInst(LastPush, GPRCS1Size, BeforeFPPush);
+    if (FramePtrSpillArea == SpillArea::GPRCS1)
+      BeforeFPPush = false;
   }
 
   // Determine starting offsets of spill areas. These offsets are all positive
@@ -1010,7 +1027,6 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
   } else {
     DPRCSOffset = GPRCS2Offset - DPRGapSize - DPRCSSize;
   }
-  int FramePtrOffsetInPush = 0;
   if (HasFP) {
     // Offset from the CFA to the saved frame pointer, will be negative.
     [[maybe_unused]] int FPOffset = MFI.getObjectOffset(FramePtrSpillFI);
@@ -1018,13 +1034,6 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
                       << ", FPOffset: " << FPOffset << "\n");
     assert(getMaxFPOffset(STI, *AFI, MF) <= FPOffset &&
            "Max FP estimation is wrong");
-    // Offset from the top of the GPRCS1 area to the saved frame pointer, will
-    // be negative.
-    FramePtrOffsetInPush = FPOffset + ArgRegsSaveSize + FPCXTSaveSize;
-    LLVM_DEBUG(dbgs() << "FramePtrOffsetInPush=" << FramePtrOffsetInPush
-                      << ", FramePtrSpillOffset="
-                      << (MFI.getObjectOffset(FramePtrSpillFI) + NumBytes)
-                      << "\n");
     AFI->setFramePtrSpillOffset(MFI.getObjectOffset(FramePtrSpillFI) +
                                 NumBytes);
   }
@@ -1036,7 +1045,9 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
   // after DPRCS1.
   if (GPRCS2Size > 0 && PushPopSplit != ARMSubtarget::SplitR11WindowsSEH) {
     GPRCS2Push = LastPush = MBBI++;
-    DefCFAOffsetCandidates.addInst(LastPush, GPRCS2Size);
+    DefCFAOffsetCandidates.addInst(LastPush, GPRCS2Size, BeforeFPPush);
+    if (FramePtrSpillArea == SpillArea::GPRCS2)
+      BeforeFPPush = false;
   }
 
   // Prolog/epilog inserter assumes we correctly align DPRs on the stack, so our
@@ -1049,7 +1060,7 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
     else {
       emitSPUpdate(isARM, MBB, MBBI, dl, TII, -DPRGapSize,
                    MachineInstr::FrameSetup);
-      DefCFAOffsetCandidates.addInst(std::prev(MBBI), DPRGapSize);
+      DefCFAOffsetCandidates.addInst(std::prev(MBBI), DPRGapSize, BeforeFPPush);
     }
   }
 
@@ -1058,7 +1069,8 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
     // Since vpush register list cannot have gaps, there may be multiple vpush
     // instructions in the prologue.
     while (MBBI != MBB.end() && MBBI->getOpcode() == ARM::VSTMDDB_UPD) {
-      DefCFAOffsetCandidates.addInst(MBBI, sizeOfSPAdjustment(*MBBI));
+      DefCFAOffsetCandidates.addInst(MBBI, sizeOfSPAdjustment(*MBBI),
+                                     BeforeFPPush);
       LastPush = MBBI++;
     }
   }
@@ -1077,7 +1089,9 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
   // Move GPRCS2, if using using SplitR11WindowsSEH.
   if (GPRCS2Size > 0 && PushPopSplit == ARMSubtarget::SplitR11WindowsSEH) {
     GPRCS2Push = LastPush = MBBI++;
-    DefCFAOffsetCandidates.addInst(LastPush, GPRCS2Size);
+    DefCFAOffsetCandidates.addInst(LastPush, GPRCS2Size, BeforeFPPush);
+    if (FramePtrSpillArea == SpillArea::GPRCS2)
+      BeforeFPPush = false;
   }
 
   bool NeedsWinCFIStackAlloc = NeedsWinCFI;
@@ -1178,28 +1192,51 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
   // into spill area 1, including the FP in R11.  In either case, it
   // is in area one and the adjustment needs to take place just after
   // that push.
-  // FIXME: The above is not necessary true when PACBTI is enabled.
-  // AAPCS requires use of R11, and PACBTI gets in the way of regular pushes,
-  // so FP ends up on area two.
   MachineBasicBlock::iterator AfterPush;
   if (HasFP) {
-    AfterPush = std::next(GPRCS1Push);
-    unsigned PushSize = sizeOfSPAdjustment(*GPRCS1Push);
-    int FPOffset = PushSize + FramePtrOffsetInPush;
-    if (PushPopSplit == ARMSubtarget::SplitR11WindowsSEH) {
-      AfterPush = std::next(GPRCS2Push);
-      emitRegPlusImmediate(!AFI->isThumbFunction(), MBB, AfterPush, dl, TII,
-                           FramePtr, ARM::SP, 0, MachineInstr::FrameSetup);
-    } else {
-      emitRegPlusImmediate(!AFI->isThumbFunction(), MBB, AfterPush, dl, TII,
-                           FramePtr, ARM::SP, FPOffset,
-                           MachineInstr::FrameSetup);
+    MachineBasicBlock::iterator FPPushInst;
+    // Offset from SP immediately after the push which saved the FP to the FP
+    // save slot.
+    int64_t FPOffsetAfterPush;
+    switch (FramePtrSpillArea) {
+    case SpillArea::GPRCS1:
+      FPPushInst = GPRCS1Push;
+      FPOffsetAfterPush = MFI.getObjectOffset(FramePtrSpillFI) +
+                          ArgRegsSaveSize + FPCXTSaveSize +
+                          sizeOfSPAdjustment(*FPPushInst);
+      LLVM_DEBUG(dbgs() << "Frame pointer in GPRCS1, offset "
+                        << FPOffsetAfterPush << "  after that push\n");
+      break;
+    case SpillArea::GPRCS2:
+      FPPushInst = GPRCS2Push;
+      FPOffsetAfterPush = MFI.getObjectOffset(FramePtrSpillFI) +
+                          ArgRegsSaveSize + FPCXTSaveSize + GPRCS1Size +
+                          sizeOfSPAdjustment(*FPPushInst);
+      if (PushPopSplit == ARMSubtarget::SplitR11WindowsSEH)
+        FPOffsetAfterPush += DPRCSSize + DPRGapSize;
+      LLVM_DEBUG(dbgs() << "Frame pointer in GPRCS2, offset "
+                        << FPOffsetAfterPush << "  after that push\n");
+      break;
+    default:
+      llvm_unreachable("frame pointer in unknown spill area");
+      break;
     }
+    AfterPush = std::next(FPPushInst);
+    if (PushPopSplit == ARMSubtarget::SplitR11WindowsSEH)
+      assert(FPOffsetAfterPush == 0);
+
+    // Emit the MOV or ADD to set up the frame pointer register.
+    emitRegPlusImmediate(!AFI->isThumbFunction(), MBB, AfterPush, dl, TII,
+                         FramePtr, ARM::SP, FPOffsetAfterPush,
+                         MachineInstr::FrameSetup);
+
     if (!NeedsWinCFI) {
-      if (FramePtrOffsetInPush + PushSize != 0) {
+      // Emit DWARF info to find the CFA using the frame pointer from this
+      // point onward.
+      if (FPOffsetAfterPush != 0) {
         unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
             nullptr, MRI->getDwarfRegNum(FramePtr, true),
-            FPCXTSaveSize + ArgRegsSaveSize - FramePtrOffsetInPush));
+            -MFI.getObjectOffset(FramePtrSpillFI)));
         BuildMI(MBB, AfterPush, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
             .addCFIIndex(CFIIndex)
             .setMIFlags(MachineInstr::FrameSetup);
@@ -1712,7 +1749,8 @@ void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB,
       if (Reg == ARM::LR && !isTailCall && !isVarArg && !isInterrupt &&
           !isCmseEntry && !isTrap && AFI->getArgumentStackToRestore() == 0 &&
           STI.hasV5TOps() && MBB.succ_empty() && !hasPAC &&
-          PushPopSplit != ARMSubtarget::SplitR11WindowsSEH) {
+          (PushPopSplit != ARMSubtarget::SplitR11WindowsSEH &&
+           PushPopSplit != ARMSubtarget::SplitR11AAPCSSignRA)) {
         Reg = ARM::PC;
         // Fold the return instruction into the LDM.
         DeleteRet = true;
@@ -2945,18 +2983,29 @@ bool ARMFrameLowering::assignCalleeSavedSpillSlots(
   const auto &AFI = *MF.getInfo<ARMFunctionInfo>();
   if (AFI.shouldSignReturnAddress()) {
     // The order of register must match the order we push them, because the
-    // PEI assigns frame indices in that order. When compiling for return
-    // address sign and authenication, we use split push, therefore the orders
-    // we want are:
-    // LR, R7, R6, R5, R4, <R12>, R11, R10,  R9,  R8, D15-D8
-    CSI.insert(find_if(CSI,
-                       [=](const auto &CS) {
-                         Register Reg = CS.getReg();
-                         return Reg == ARM::R10 || Reg == ARM::R11 ||
-                                Reg == ARM::R8 || Reg == ARM::R9 ||
-                                ARM::DPRRegClass.contains(Reg);
-                       }),
-               CalleeSavedInfo(ARM::R12));
+    // PEI assigns frame indices in that order. That order depends on the
+    // PushPopSplitVariation, there are only two cases which we use with return
+    // address signing:
+    switch (STI.getPushPopSplitVariation(MF)) {
+    case ARMSubtarget::SplitR7:
+      // LR, R7, R6, R5, R4, <R12>, R11, R10,  R9,  R8, D15-D8
+      CSI.insert(find_if(CSI,
+                         [=](const auto &CS) {
+                           Register Reg = CS.getReg();
+                           return Reg == ARM::R10 || Reg == ARM::R11 ||
+                                  Reg == ARM::R8 || Reg == ARM::R9 ||
+                                  ARM::DPRRegClass.contains(Reg);
+                         }),
+                 CalleeSavedInfo(ARM::R12));
+      break;
+    case ARMSubtarget::SplitR11AAPCSSignRA:
+      // With SplitR11AAPCSSignRA, R12 will always be the highest-addressed CSR
+      // on the stack.
+      CSI.insert(CSI.begin(), CalleeSavedInfo(ARM::R12));
+      break;
+    default:
+      llvm_unreachable("Unexpected CSR split with return address signing");
+    }
   }
 
   return false;
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.cpp b/llvm/lib/Target/ARM/ARMSubtarget.cpp
index c4a782bc4091..9adfb1fab5f0 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.cpp
+++ b/llvm/lib/Target/ARM/ARMSubtarget.cpp
@@ -514,5 +514,12 @@ ARMSubtarget::getPushPopSplitVariation(const MachineFunction &MF) const {
       F.needsUnwindTableEntry() &&
       (MFI.hasVarSizedObjects() || getRegisterInfo()->hasStackRealignment(MF)))
     return SplitR11WindowsSEH;
+
+  // Returns R11SplitAAPCSBranchSigning if R11 and lr are not adjacent to each
+  // other in the list of callee saved registers in a frame, and branch
+  // signing is enabled.
+  if (MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress() &&
+      getFramePointerReg() == ARM::R11)
+    return SplitR11AAPCSSignRA;
   return NoSplit;
 }
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h
index 7917ddc17bdb..214c5f1b45e5 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -105,6 +105,18 @@ public:
     /// vpush {d8-d15}
     /// push {r11, lr}
     SplitR11WindowsSEH,
+
+    /// When generating AAPCS-compilant frame chains, R11 is the frame pointer,
+    /// and must be pushed adjacent to the return address (LR). Normally this
+    /// isn't a problem, because the only register between them is r12, which is
+    /// the intra-procedure-call scratch register, so doesn't need to be saved.
+    /// However, when PACBTI is in use, r12 contains the authentication code, so
+    /// does need to be saved. This means that we need a separate push for R11
+    /// and LR.
+    /// push {r0-r10, r12}
+    /// push {r11, lr}
+    /// vpush {d8-d15}
+    SplitR11AAPCSSignRA,
   };
 
 protected:
diff --git a/llvm/test/CodeGen/Thumb2/pacbti-m-frame-chain.ll b/llvm/test/CodeGen/Thumb2/pacbti-m-frame-chain.ll
new file mode 100644
index 000000000000..8bcf87130c54
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/pacbti-m-frame-chain.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=thumbv8.1m.main-none-eabi < %s --force-dwarf-frame-section -frame-pointer=all -mattr=+aapcs-frame-chain | FileCheck %s
+
+; int test1() {
+;     return 0;
+; }
+define i32 @test1() "sign-return-address"="non-leaf" {
+; CHECK-LABEL: test1:
+; CHECK:         .cfi_sections .debug_frame
+; CHECK-NEXT:    .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    .save {ra_auth_code}
+; CHECK-NEXT:    str r12, [sp, #-4]!
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -4
+; CHECK-NEXT:    .save {r11, lr}
+; CHECK-NEXT:    push.w {r11, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    .cfi_offset lr, -8
+; CHECK-NEXT:    .cfi_offset r11, -12
+; CHECK-NEXT:    .setfp r11, sp
+; CHECK-NEXT:    mov r11, sp
+; CHECK-NEXT:    .cfi_def_cfa_register r11
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    pop.w {r11, lr}
+; CHECK-NEXT:    ldr r12, [sp], #4
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    bx lr
+entry:
+    ret i32 0
+}
+
+; void foo(int n) {
+;   int a[n];
+;   bar(a);
+; }
+define dso_local void @test2(i32 noundef %n) "sign-return-address"="non-leaf" {
+; CHECK-LABEL: test2:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    .save {r4, r7, ra_auth_code}
+; CHECK-NEXT:    push.w {r4, r7, r12}
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -4
+; CHECK-NEXT:    .cfi_offset r7, -8
+; CHECK-NEXT:    .cfi_offset r4, -12
+; CHECK-NEXT:    .save {r11, lr}
+; CHECK-NEXT:    push.w {r11, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 20
+; CHECK-NEXT:    .cfi_offset lr, -16
+; CHECK-NEXT:    .cfi_offset r11, -20
+; CHECK-NEXT:    .setfp r11, sp
+; CHECK-NEXT:    mov r11, sp
+; CHECK-NEXT:    .cfi_def_cfa_register r11
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    movs r1, #7
+; CHECK-NEXT:    add.w r0, r1, r0, lsl #2
+; CHECK-NEXT:    bic r0, r0, #7
+; CHECK-NEXT:    sub.w r0, sp, r0
+; CHECK-NEXT:    mov sp, r0
+; CHECK-NEXT:    bl take_ptr
+; CHECK-NEXT:    mov sp, r11
+; CHECK-NEXT:    pop.w {r11, lr}
+; CHECK-NEXT:    pop.w {r4, r7, r12}
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    bx lr
+entry:
+  %vla = alloca i32, i32 %n, align 4
+  call void @take_ptr(ptr noundef nonnull %vla)
+  ret void
+}
+
+; void test3(int c, float e, int z) {
+;   if (c)
+;     knr();
+;   take_ptr(alloca(z));
+;   if (e)
+;     knr();
+; }
+define void @test3(i32 noundef %c, float noundef %e, i32 noundef %z) "sign-return-address"="non-leaf" {
+; CHECK-LABEL: test3:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    .save {r4, r5, r6, r7, ra_auth_code}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r12}
+; CHECK-NEXT:    .cfi_def_cfa_offset 20
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -4
+; CHECK-NEXT:    .cfi_offset r7, -8
+; CHECK-NEXT:    .cfi_offset r6, -12
+; CHECK-NEXT:    .cfi_offset r5, -16
+; CHECK-NEXT:    .cfi_offset r4, -20
+; CHECK-NEXT:    .save {r11, lr}
+; CHECK-NEXT:    push.w {r11, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 28
+; CHECK-NEXT:    .cfi_offset lr, -24
+; CHECK-NEXT:    .cfi_offset r11, -28
+; CHECK-NEXT:    .setfp r11, sp
+; CHECK-NEXT:    mov r11, sp
+; CHECK-NEXT:    .cfi_def_cfa_register r11
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r5, r2
+; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    blne knr
+; CHECK-NEXT:    adds r0, r5, #7
+; CHECK-NEXT:    bic r0, r0, #7
+; CHECK-NEXT:    sub.w r0, sp, r0
+; CHECK-NEXT:    mov sp, r0
+; CHECK-NEXT:    bl take_ptr
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    bl __aeabi_fcmpeq
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    bleq knr
+; CHECK-NEXT:    mov sp, r11
+; CHECK-NEXT:    pop.w {r11, lr}
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r12}
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    bx lr
+entry:
+  %tobool.not = icmp eq i32 %c, 0
+  br i1 %tobool.not, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void @knr()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %0 = alloca i8, i32 %z, align 8
+  call void @take_ptr(ptr noundef nonnull %0)
+  %tobool1 = fcmp une float %e, 0.000000e+00
+  br i1 %tobool1, label %if.then2, label %if.end3
+
+if.then2:                                         ; preds = %if.end
+  call void @knr()
+  br label %if.end3
+
+if.end3:                                          ; preds = %if.then2, %if.end
+  ret void
+}
+
+declare void @knr(...)
+declare void @take_ptr(ptr noundef)
-- 
GitLab


From 3cec720449d0b419570772a31aa137296bba6423 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Thu, 24 Oct 2024 11:51:31 -0400
Subject: [PATCH 066/693] Revert "[DSE] Apply initializes attribute to DSE"
 (#113589)

Reverts llvm/llvm-project#107282

Seems to be causing invalid analysis caching as mentioned in
https://github.com/llvm/llvm-project/pull/107282#issuecomment-2435083978.
---
 .../Scalar/DeadStoreElimination.cpp           | 250 +++------------
 .../DeadStoreElimination/inter-procedural.ll  | 301 ------------------
 2 files changed, 42 insertions(+), 509 deletions(-)
 delete mode 100644 llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll

diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 411f7cc34823..6fce46a624c9 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -52,7 +52,6 @@
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
-#include "llvm/IR/ConstantRangeList.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfo.h"
@@ -165,11 +164,6 @@ static cl::opt<bool>
     OptimizeMemorySSA("dse-optimize-memoryssa", cl::init(true), cl::Hidden,
                       cl::desc("Allow DSE to optimize memory accesses."));
 
-// TODO: turn on and remove this flag.
-static cl::opt<bool> EnableInitializesImprovement(
-    "enable-dse-initializes-attr-improvement", cl::init(false), cl::Hidden,
-    cl::desc("Enable the initializes attr improvement in DSE"));
-
 //===----------------------------------------------------------------------===//
 // Helper functions
 //===----------------------------------------------------------------------===//
@@ -815,10 +809,8 @@ bool canSkipDef(MemoryDef *D, bool DefVisibleToCaller) {
 // A memory location wrapper that represents a MemoryLocation, `MemLoc`,
 // defined by `MemDef`.
 struct MemoryLocationWrapper {
-  MemoryLocationWrapper(MemoryLocation MemLoc, MemoryDef *MemDef,
-                        bool DefByInitializesAttr)
-      : MemLoc(MemLoc), MemDef(MemDef),
-        DefByInitializesAttr(DefByInitializesAttr) {
+  MemoryLocationWrapper(MemoryLocation MemLoc, MemoryDef *MemDef)
+      : MemLoc(MemLoc), MemDef(MemDef) {
     assert(MemLoc.Ptr && "MemLoc should be not null");
     UnderlyingObject = getUnderlyingObject(MemLoc.Ptr);
     DefInst = MemDef->getMemoryInst();
@@ -828,59 +820,20 @@ struct MemoryLocationWrapper {
   const Value *UnderlyingObject;
   MemoryDef *MemDef;
   Instruction *DefInst;
-  bool DefByInitializesAttr = false;
 };
 
 // A memory def wrapper that represents a MemoryDef and the MemoryLocation(s)
 // defined by this MemoryDef.
 struct MemoryDefWrapper {
-  MemoryDefWrapper(MemoryDef *MemDef,
-                   ArrayRef<std::pair<MemoryLocation, bool>> MemLocations) {
+  MemoryDefWrapper(MemoryDef *MemDef, std::optional<MemoryLocation> MemLoc) {
     DefInst = MemDef->getMemoryInst();
-    for (auto &[MemLoc, DefByInitializesAttr] : MemLocations)
-      DefinedLocations.push_back(
-          MemoryLocationWrapper(MemLoc, MemDef, DefByInitializesAttr));
+    if (MemLoc.has_value())
+      DefinedLocation = MemoryLocationWrapper(*MemLoc, MemDef);
   }
   Instruction *DefInst;
-  SmallVector<MemoryLocationWrapper, 1> DefinedLocations;
-};
-
-bool hasInitializesAttr(Instruction *I) {
-  CallBase *CB = dyn_cast<CallBase>(I);
-  return CB && CB->getArgOperandWithAttribute(Attribute::Initializes);
-}
-
-struct ArgumentInitInfo {
-  unsigned Idx;
-  bool IsDeadOrInvisibleOnUnwind;
-  ConstantRangeList Inits;
+  std::optional<MemoryLocationWrapper> DefinedLocation = std::nullopt;
 };
 
-// Return the intersected range list of the initializes attributes of "Args".
-// "Args" are call arguments that alias to each other.
-// If any argument in "Args" doesn't have dead_on_unwind attr and
-// "CallHasNoUnwindAttr" is false, return empty.
-ConstantRangeList getIntersectedInitRangeList(ArrayRef<ArgumentInitInfo> Args,
-                                              bool CallHasNoUnwindAttr) {
-  if (Args.empty())
-    return {};
-
-  // To address unwind, the function should have nounwind attribute or the
-  // arguments have dead or invisible on unwind. Otherwise, return empty.
-  for (const auto &Arg : Args) {
-    if (!CallHasNoUnwindAttr && !Arg.IsDeadOrInvisibleOnUnwind)
-      return {};
-    if (Arg.Inits.empty())
-      return {};
-  }
-
-  ConstantRangeList IntersectedIntervals = Args.front().Inits;
-  for (auto &Arg : Args.drop_front())
-    IntersectedIntervals = IntersectedIntervals.intersectWith(Arg.Inits);
-
-  return IntersectedIntervals;
-}
-
 struct DSEState {
   Function &F;
   AliasAnalysis &AA;
@@ -958,8 +911,7 @@ struct DSEState {
 
         auto *MD = dyn_cast_or_null<MemoryDef>(MA);
         if (MD && MemDefs.size() < MemorySSADefsPerBlockLimit &&
-            (getLocForWrite(&I) || isMemTerminatorInst(&I) ||
-             (EnableInitializesImprovement && hasInitializesAttr(&I))))
+            (getLocForWrite(&I) || isMemTerminatorInst(&I)))
           MemDefs.push_back(MD);
       }
     }
@@ -1195,26 +1147,13 @@ struct DSEState {
     return MemoryLocation::getOrNone(I);
   }
 
-  // Returns a list of <MemoryLocation, bool> pairs written by I.
-  // The bool means whether the write is from Initializes attr.
-  SmallVector<std::pair<MemoryLocation, bool>, 1>
-  getLocForInst(Instruction *I, bool ConsiderInitializesAttr) {
-    SmallVector<std::pair<MemoryLocation, bool>, 1> Locations;
+  std::optional<MemoryLocation> getLocForInst(Instruction *I) {
     if (isMemTerminatorInst(I)) {
-      if (auto Loc = getLocForTerminator(I))
-        Locations.push_back(std::make_pair(Loc->first, false));
-      return Locations;
-    }
-
-    if (auto Loc = getLocForWrite(I))
-      Locations.push_back(std::make_pair(*Loc, false));
-
-    if (ConsiderInitializesAttr) {
-      for (auto &MemLoc : getInitializesArgMemLoc(I)) {
-        Locations.push_back(std::make_pair(MemLoc, true));
+      if (auto Loc = getLocForTerminator(I)) {
+        return Loc->first;
       }
     }
-    return Locations;
+    return getLocForWrite(I);
   }
 
   /// Assuming this instruction has a dead analyzable write, can we delete
@@ -1426,8 +1365,7 @@ struct DSEState {
   getDomMemoryDef(MemoryDef *KillingDef, MemoryAccess *StartAccess,
                   const MemoryLocation &KillingLoc, const Value *KillingUndObj,
                   unsigned &ScanLimit, unsigned &WalkerStepLimit,
-                  bool IsMemTerm, unsigned &PartialLimit,
-                  bool IsInitializesAttrMemLoc) {
+                  bool IsMemTerm, unsigned &PartialLimit) {
     if (ScanLimit == 0 || WalkerStepLimit == 0) {
       LLVM_DEBUG(dbgs() << "\n    ...  hit scan limit\n");
       return std::nullopt;
@@ -1664,16 +1602,7 @@ struct DSEState {
 
       // Uses which may read the original MemoryDef mean we cannot eliminate the
       // original MD. Stop walk.
-      // If KillingDef is a CallInst with "initializes" attribute, the reads in
-      // the callee would be dominated by initializations, so it should be safe.
-      bool IsKillingDefFromInitAttr = false;
-      if (IsInitializesAttrMemLoc) {
-        if (KillingI == UseInst &&
-            KillingUndObj == getUnderlyingObject(MaybeDeadLoc.Ptr))
-          IsKillingDefFromInitAttr = true;
-      }
-
-      if (isReadClobber(MaybeDeadLoc, UseInst) && !IsKillingDefFromInitAttr) {
+      if (isReadClobber(MaybeDeadLoc, UseInst)) {
         LLVM_DEBUG(dbgs() << "    ... found read clobber\n");
         return std::nullopt;
       }
@@ -2242,16 +2171,6 @@ struct DSEState {
     return MadeChange;
   }
 
-  // Return the locations written by the initializes attribute.
-  // Note that this function considers:
-  // 1. Unwind edge: use "initializes" attribute only if the callee has
-  //    "nounwind" attribute, or the argument has "dead_on_unwind" attribute,
-  //    or the argument is invisible to caller on unwind. That is, we don't
-  //    perform incorrect DSE on unwind edges in the current function.
-  // 2. Argument alias: for aliasing arguments, the "initializes" attribute is
-  //    the intersected range list of their "initializes" attributes.
-  SmallVector<MemoryLocation, 1> getInitializesArgMemLoc(const Instruction *I);
-
   // Try to eliminate dead defs that access `KillingLocWrapper.MemLoc` and are
   // killed by `KillingLocWrapper.MemDef`. Return whether
   // any changes were made, and whether `KillingLocWrapper.DefInst` was deleted.
@@ -2263,75 +2182,6 @@ struct DSEState {
   bool eliminateDeadDefs(const MemoryDefWrapper &KillingDefWrapper);
 };
 
-SmallVector<MemoryLocation, 1>
-DSEState::getInitializesArgMemLoc(const Instruction *I) {
-  const CallBase *CB = dyn_cast<CallBase>(I);
-  if (!CB)
-    return {};
-
-  // Collect aliasing arguments and their initializes ranges.
-  SmallMapVector<Value *, SmallVector<ArgumentInitInfo, 2>, 2> Arguments;
-  for (unsigned Idx = 0, Count = CB->arg_size(); Idx < Count; ++Idx) {
-    ConstantRangeList Inits;
-    Attribute InitializesAttr = CB->getParamAttr(Idx, Attribute::Initializes);
-    if (InitializesAttr.isValid())
-      Inits = InitializesAttr.getValueAsConstantRangeList();
-
-    Value *CurArg = CB->getArgOperand(Idx);
-    // We don't perform incorrect DSE on unwind edges in the current function,
-    // and use the "initializes" attribute to kill dead stores if:
-    // - The call does not throw exceptions, "CB->doesNotThrow()".
-    // - Or the callee parameter has "dead_on_unwind" attribute.
-    // - Or the argument is invisible to caller on unwind, and there are no
-    //   unwind edges from this call in the current function (e.g. `CallInst`).
-    bool IsDeadOrInvisibleOnUnwind =
-        CB->paramHasAttr(Idx, Attribute::DeadOnUnwind) ||
-        (isa<CallInst>(CB) && isInvisibleToCallerOnUnwind(CurArg));
-    ArgumentInitInfo InitInfo{Idx, IsDeadOrInvisibleOnUnwind, Inits};
-    bool FoundAliasing = false;
-    for (auto &[Arg, AliasList] : Arguments) {
-      auto AAR = BatchAA.alias(MemoryLocation::getBeforeOrAfter(Arg),
-                               MemoryLocation::getBeforeOrAfter(CurArg));
-      if (AAR == AliasResult::NoAlias) {
-        continue;
-      } else if (AAR == AliasResult::MustAlias) {
-        FoundAliasing = true;
-        AliasList.push_back(InitInfo);
-      } else {
-        // For PartialAlias and MayAlias, there is an offset or may be an
-        // unknown offset between the arguments and we insert an empty init
-        // range to discard the entire initializes info while intersecting.
-        FoundAliasing = true;
-        AliasList.push_back(ArgumentInitInfo{Idx, IsDeadOrInvisibleOnUnwind,
-                                             ConstantRangeList()});
-      }
-    }
-    if (!FoundAliasing)
-      Arguments[CurArg] = {InitInfo};
-  }
-
-  SmallVector<MemoryLocation, 1> Locations;
-  for (const auto &[_, Args] : Arguments) {
-    auto IntersectedRanges =
-        getIntersectedInitRangeList(Args, CB->doesNotThrow());
-    if (IntersectedRanges.empty())
-      continue;
-
-    for (const auto &Arg : Args) {
-      for (const auto &Range : IntersectedRanges) {
-        int64_t Start = Range.getLower().getSExtValue();
-        int64_t End = Range.getUpper().getSExtValue();
-        // For now, we only handle locations starting at offset 0.
-        if (Start == 0)
-          Locations.push_back(MemoryLocation(CB->getArgOperand(Arg.Idx),
-                                             LocationSize::precise(End - Start),
-                                             CB->getAAMetadata()));
-      }
-    }
-  }
-  return Locations;
-}
-
 std::pair<bool, bool>
 DSEState::eliminateDeadDefs(const MemoryLocationWrapper &KillingLocWrapper) {
   bool Changed = false;
@@ -2358,8 +2208,7 @@ DSEState::eliminateDeadDefs(const MemoryLocationWrapper &KillingLocWrapper) {
     std::optional<MemoryAccess *> MaybeDeadAccess = getDomMemoryDef(
         KillingLocWrapper.MemDef, Current, KillingLocWrapper.MemLoc,
         KillingLocWrapper.UnderlyingObject, ScanLimit, WalkerStepLimit,
-        isMemTerminatorInst(KillingLocWrapper.DefInst), PartialLimit,
-        KillingLocWrapper.DefByInitializesAttr);
+        isMemTerminatorInst(KillingLocWrapper.DefInst), PartialLimit);
 
     if (!MaybeDeadAccess) {
       LLVM_DEBUG(dbgs() << "  finished walk\n");
@@ -2382,20 +2231,10 @@ DSEState::eliminateDeadDefs(const MemoryLocationWrapper &KillingLocWrapper) {
       }
       continue;
     }
-    // We cannot apply the initializes attribute to DeadAccess/DeadDef.
-    // It would incorrectly consider a call instruction as redundant store
-    // and remove this call instruction.
-    // TODO: this conflates the existence of a MemoryLocation with being able
-    // to delete the instruction. Fix isRemovable() to consider calls with
-    // side effects that cannot be removed, e.g. calls with the initializes
-    // attribute, and remove getLocForInst(ConsiderInitializesAttr = false).
     MemoryDefWrapper DeadDefWrapper(
         cast<MemoryDef>(DeadAccess),
-        getLocForInst(cast<MemoryDef>(DeadAccess)->getMemoryInst(),
-                      /*ConsiderInitializesAttr=*/false));
-    assert(DeadDefWrapper.DefinedLocations.size() == 1);
-    MemoryLocationWrapper &DeadLocWrapper =
-        DeadDefWrapper.DefinedLocations.front();
+        getLocForInst(cast<MemoryDef>(DeadAccess)->getMemoryInst()));
+    MemoryLocationWrapper &DeadLocWrapper = *DeadDefWrapper.DefinedLocation;
     LLVM_DEBUG(dbgs() << " (" << *DeadLocWrapper.DefInst << ")\n");
     ToCheck.insert(DeadLocWrapper.MemDef->getDefiningAccess());
     NumGetDomMemoryDefPassed++;
@@ -2470,41 +2309,37 @@ DSEState::eliminateDeadDefs(const MemoryLocationWrapper &KillingLocWrapper) {
 }
 
 bool DSEState::eliminateDeadDefs(const MemoryDefWrapper &KillingDefWrapper) {
-  if (KillingDefWrapper.DefinedLocations.empty()) {
+  if (!KillingDefWrapper.DefinedLocation.has_value()) {
     LLVM_DEBUG(dbgs() << "Failed to find analyzable write location for "
                       << *KillingDefWrapper.DefInst << "\n");
     return false;
   }
 
-  bool MadeChange = false;
-  for (auto &KillingLocWrapper : KillingDefWrapper.DefinedLocations) {
-    LLVM_DEBUG(dbgs() << "Trying to eliminate MemoryDefs killed by "
-                      << *KillingLocWrapper.MemDef << " ("
-                      << *KillingLocWrapper.DefInst << ")\n");
-    auto [Changed, DeletedKillingLoc] = eliminateDeadDefs(KillingLocWrapper);
-
-    // Check if the store is a no-op.
-    if (!DeletedKillingLoc && storeIsNoop(KillingLocWrapper.MemDef,
-                                          KillingLocWrapper.UnderlyingObject)) {
-      LLVM_DEBUG(dbgs() << "DSE: Remove No-Op Store:\n  DEAD: "
-                        << *KillingLocWrapper.DefInst << '\n');
-      deleteDeadInstruction(KillingLocWrapper.DefInst);
-      NumRedundantStores++;
-      MadeChange = true;
-      continue;
-    }
-    // Can we form a calloc from a memset/malloc pair?
-    if (!DeletedKillingLoc &&
-        tryFoldIntoCalloc(KillingLocWrapper.MemDef,
-                          KillingLocWrapper.UnderlyingObject)) {
-      LLVM_DEBUG(dbgs() << "DSE: Remove memset after forming calloc:\n"
-                        << "  DEAD: " << *KillingLocWrapper.DefInst << '\n');
-      deleteDeadInstruction(KillingLocWrapper.DefInst);
-      MadeChange = true;
-      continue;
-    }
+  auto &KillingLocWrapper = *KillingDefWrapper.DefinedLocation;
+  LLVM_DEBUG(dbgs() << "Trying to eliminate MemoryDefs killed by "
+                    << *KillingLocWrapper.MemDef << " ("
+                    << *KillingLocWrapper.DefInst << ")\n");
+  auto [Changed, DeletedKillingLoc] = eliminateDeadDefs(KillingLocWrapper);
+
+  // Check if the store is a no-op.
+  if (!DeletedKillingLoc && storeIsNoop(KillingLocWrapper.MemDef,
+                                        KillingLocWrapper.UnderlyingObject)) {
+    LLVM_DEBUG(dbgs() << "DSE: Remove No-Op Store:\n  DEAD: "
+                      << *KillingLocWrapper.DefInst << '\n');
+    deleteDeadInstruction(KillingLocWrapper.DefInst);
+    NumRedundantStores++;
+    return true;
   }
-  return MadeChange;
+  // Can we form a calloc from a memset/malloc pair?
+  if (!DeletedKillingLoc &&
+      tryFoldIntoCalloc(KillingLocWrapper.MemDef,
+                        KillingLocWrapper.UnderlyingObject)) {
+    LLVM_DEBUG(dbgs() << "DSE: Remove memset after forming calloc:\n"
+                      << "  DEAD: " << *KillingLocWrapper.DefInst << '\n');
+    deleteDeadInstruction(KillingLocWrapper.DefInst);
+    return true;
+  }
+  return Changed;
 }
 
 static bool eliminateDeadStores(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
@@ -2520,8 +2355,7 @@ static bool eliminateDeadStores(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
       continue;
 
     MemoryDefWrapper KillingDefWrapper(
-        KillingDef, State.getLocForInst(KillingDef->getMemoryInst(),
-                                        EnableInitializesImprovement));
+        KillingDef, State.getLocForInst(KillingDef->getMemoryInst()));
     MadeChange |= State.eliminateDeadDefs(KillingDefWrapper);
   }
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll b/llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll
deleted file mode 100644
index d93da9b6612b..000000000000
--- a/llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll
+++ /dev/null
@@ -1,301 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=dse -enable-dse-initializes-attr-improvement -S | FileCheck %s
-
-declare void @p1_write_only(ptr nocapture noundef writeonly initializes((0, 2)) dead_on_unwind)
-declare void @p1_write_then_read(ptr nocapture noundef initializes((0, 2)) dead_on_unwind)
-declare void @p1_clobber(ptr nocapture noundef)
-declare void @p2_same_range(ptr nocapture noundef initializes((0, 2)) dead_on_unwind, ptr nocapture noundef initializes((0, 2)) dead_on_unwind)
-declare void @p2_no_init(ptr nocapture noundef initializes((0, 2)) dead_on_unwind, ptr nocapture noundef dead_on_unwind)
-declare void @p2_no_dead_on_unwind(ptr nocapture noundef initializes((0, 2)) dead_on_unwind, ptr nocapture noundef initializes((0, 2)))
-declare void @p2_no_dead_on_unwind_but_nounwind(ptr nocapture noundef initializes((0, 2)) dead_on_unwind, ptr nocapture noundef initializes((0, 2))) nounwind
-
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @p1_write_only_caller() {
-; CHECK-LABEL: @p1_write_only_caller(
-; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
-; CHECK-NEXT:    call void @p1_write_only(ptr [[PTR]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  %ptr = alloca i16
-  store i16 0, ptr %ptr
-  call void @p1_write_only(ptr %ptr)
-  %l = load i16, ptr %ptr
-  ret i16 %l
-}
-
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @p1_write_then_read_caller() {
-; CHECK-LABEL: @p1_write_then_read_caller(
-; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
-; CHECK-NEXT:    call void @p1_write_then_read(ptr [[PTR]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  %ptr = alloca i16
-  store i16 0, ptr %ptr
-  call void @p1_write_then_read(ptr %ptr)
-  %l = load i16, ptr %ptr
-  ret i16 %l
-}
-
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @p1_write_then_read_caller_with_clobber() {
-; CHECK-LABEL: @p1_write_then_read_caller_with_clobber(
-; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
-; CHECK-NEXT:    store i16 0, ptr [[PTR]], align 2
-; CHECK-NEXT:    call void @p1_clobber(ptr [[PTR]])
-; CHECK-NEXT:    call void @p1_write_then_read(ptr [[PTR]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  %ptr = alloca i16
-  store i16 0, ptr %ptr
-  call void @p1_clobber(ptr %ptr)
-  call void @p1_write_then_read(ptr %ptr)
-  %l = load i16, ptr %ptr
-  ret i16 %l
-}
-
-declare void @p1_write_then_read_raw(ptr nocapture noundef initializes((0, 2)))
-define i16 @p1_initializes_invoke() personality ptr undef {
-; CHECK-LABEL: @p1_initializes_invoke(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
-; CHECK-NEXT:    store i16 0, ptr [[PTR]], align 2
-; CHECK-NEXT:    invoke void @p1_write_then_read_raw(ptr [[PTR]])
-; CHECK-NEXT:            to label [[BB1:%.*]] unwind label [[BB2:%.*]]
-; CHECK:       bb1:
-; CHECK-NEXT:    ret i16 0
-; CHECK:       bb2:
-; CHECK-NEXT:    [[TMP:%.*]] = landingpad { ptr, i32 }
-; CHECK-NEXT:            cleanup
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-entry:
-  %ptr = alloca i16
-  store i16 0, ptr %ptr
-  invoke void @p1_write_then_read_raw(ptr %ptr) to label %bb1 unwind label %bb2
-bb1:
-  ret i16 0
-bb2:
-  %tmp = landingpad { ptr, i32 }
-  cleanup
-  %l = load i16, ptr %ptr
-  ret i16 %l
-}
-
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @p2_same_range_noalias_caller() {
-; CHECK-LABEL: @p2_same_range_noalias_caller(
-; CHECK-NEXT:    [[PTR1:%.*]] = alloca i16, align 2
-; CHECK-NEXT:    [[PTR2:%.*]] = alloca i16, align 2
-; CHECK-NEXT:    call void @p2_same_range(ptr [[PTR1]], ptr [[PTR2]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR1]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  %ptr1 = alloca i16
-  %ptr2 = alloca i16
-  store i16 0, ptr %ptr1
-  store i16 0, ptr %ptr2
-  call void @p2_same_range(ptr %ptr1, ptr %ptr2)
-  %l = load i16, ptr %ptr1
-  ret i16 %l
-}
-
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @p2_same_range_must_alias_caller() {
-; CHECK-LABEL: @p2_same_range_must_alias_caller(
-; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
-; CHECK-NEXT:    call void @p2_same_range(ptr [[PTR]], ptr [[PTR]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  %ptr = alloca i16
-  store i16 0, ptr %ptr
-  call void @p2_same_range(ptr %ptr, ptr %ptr)
-  %l = load i16, ptr %ptr
-  ret i16 %l
-}
-
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @p2_same_range_may_or_partial_alias_caller1(ptr %base, i1 %x) {
-; CHECK-LABEL: @p2_same_range_may_or_partial_alias_caller1(
-; CHECK-NEXT:    [[BASEPLUS:%.*]] = getelementptr i8, ptr [[BASE:%.*]], i64 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[X:%.*]], ptr [[BASEPLUS]], ptr [[BASE]]
-; CHECK-NEXT:    store i32 0, ptr [[BASE]], align 4
-; CHECK-NEXT:    call void @p2_same_range(ptr [[BASE]], ptr [[SEL]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[BASE]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  %baseplus = getelementptr i8, ptr %base, i64 1
-  %sel = select i1 %x, ptr %baseplus, ptr %base
-  store i32 0, ptr %base
-  call void @p2_same_range(ptr %base, ptr %sel)
-  %l = load i16, ptr %base
-  ret i16 %l
-}
-
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @p2_same_range_may_or_partial_alias_caller2(ptr %base1, ptr %base2) {
-; CHECK-LABEL: @p2_same_range_may_or_partial_alias_caller2(
-; CHECK-NEXT:    store i32 0, ptr [[BASE1:%.*]], align 4
-; CHECK-NEXT:    call void @p2_same_range(ptr [[BASE1]], ptr [[BASE2:%.*]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[BASE1]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  store i32 0, ptr %base1
-  call void @p2_same_range(ptr %base1, ptr %base2)
-  %l = load i16, ptr %base1
-  ret i16 %l
-}
-
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @p2_no_init_alias_caller() {
-; CHECK-LABEL: @p2_no_init_alias_caller(
-; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
-; CHECK-NEXT:    store i16 0, ptr [[PTR]], align 2
-; CHECK-NEXT:    call void @p2_no_init(ptr [[PTR]], ptr [[PTR]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  %ptr = alloca i16
-  store i16 0, ptr %ptr
-  call void @p2_no_init(ptr %ptr, ptr %ptr)
-  %l = load i16, ptr %ptr
-  ret i16 %l
-}
-
-; Althrough the 2nd parameter of `p2_no_dead_on_unwind` doesn't have
-; the 'dead_on_unwind' attribute, it's invisble to caller on unwind.
-; DSE still uses the 'initializes' attribute and kill the dead store.
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @p2_no_dead_on_unwind_but_invisible_to_caller_alias_caller() {
-; CHECK-LABEL: @p2_no_dead_on_unwind_but_invisible_to_caller_alias_caller(
-; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
-; CHECK-NEXT:    call void @p2_no_dead_on_unwind(ptr [[PTR]], ptr [[PTR]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  %ptr = alloca i16
-  store i16 0, ptr %ptr
-  call void @p2_no_dead_on_unwind(ptr %ptr, ptr %ptr)
-  %l = load i16, ptr %ptr
-  ret i16 %l
-}
-
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @p2_no_dead_on_unwind_alias_caller(ptr %ptr) {
-; CHECK-LABEL: @p2_no_dead_on_unwind_alias_caller(
-; CHECK-NEXT:    store i16 0, ptr [[PTR:%.*]], align 2
-; CHECK-NEXT:    call void @p2_no_dead_on_unwind(ptr [[PTR]], ptr [[PTR]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  store i16 0, ptr %ptr
-  call void @p2_no_dead_on_unwind(ptr %ptr, ptr %ptr)
-  %l = load i16, ptr %ptr
-  ret i16 %l
-}
-
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @p2_no_dead_on_unwind_but_nounwind_alias_caller() {
-; CHECK-LABEL: @p2_no_dead_on_unwind_but_nounwind_alias_caller(
-; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
-; CHECK-NEXT:    call void @p2_no_dead_on_unwind_but_nounwind(ptr [[PTR]], ptr [[PTR]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  %ptr = alloca i16
-  store i16 0, ptr %ptr
-  call void @p2_no_dead_on_unwind_but_nounwind(ptr %ptr, ptr %ptr)
-  %l = load i16, ptr %ptr
-  ret i16 %l
-}
-
-declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i1) nounwind
-declare void @large_p1(ptr nocapture noundef initializes((0, 200))) nounwind
-declare void @large_p2(ptr nocapture noundef initializes((0, 200)), ptr nocapture noundef initializes((0, 100))) nounwind
-
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @large_p1_caller() {
-; CHECK-LABEL: @large_p1_caller(
-; CHECK-NEXT:    [[PTR:%.*]] = alloca [200 x i8], align 1
-; CHECK-NEXT:    call void @large_p1(ptr [[PTR]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  %ptr = alloca [200 x i8]
-  call void @llvm.memset.p0.i64(ptr %ptr, i8 42, i64 100, i1 false)
-  call void @large_p1(ptr %ptr)
-  %l = load i16, ptr %ptr
-  ret i16 %l
-}
-
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @large_p2_nonalias_caller() {
-; CHECK-LABEL: @large_p2_nonalias_caller(
-; CHECK-NEXT:    [[PTR1:%.*]] = alloca [200 x i8], align 1
-; CHECK-NEXT:    [[PTR2:%.*]] = alloca [100 x i8], align 1
-; CHECK-NEXT:    call void @large_p2(ptr [[PTR1]], ptr [[PTR2]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR1]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  %ptr1 = alloca [200 x i8]
-  %ptr2 = alloca [100 x i8]
-  call void @llvm.memset.p0.i64(ptr %ptr1, i8 42, i64 200, i1 false)
-  call void @llvm.memset.p0.i64(ptr %ptr2, i8 42, i64 100, i1 false)
-  call void @large_p2(ptr %ptr1, ptr %ptr2)
-  %l = load i16, ptr %ptr1
-  ret i16 %l
-}
-
-
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @large_p2_must_alias_caller() {
-; CHECK-LABEL: @large_p2_must_alias_caller(
-; CHECK-NEXT:    [[PTR:%.*]] = alloca [300 x i8], align 1
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 100
-; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP1]], i8 42, i64 200, i1 false)
-; CHECK-NEXT:    call void @large_p2(ptr [[PTR]], ptr [[PTR]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  %ptr = alloca [300 x i8]
-  call void @llvm.memset.p0.i64(ptr %ptr, i8 42, i64 300, i1 false)
-  call void @large_p2(ptr %ptr, ptr %ptr)
-  %l = load i16, ptr %ptr
-  ret i16 %l
-}
-
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @large_p2_may_or_partial_alias_caller1(ptr %base) {
-; CHECK-LABEL: @large_p2_may_or_partial_alias_caller1(
-; CHECK-NEXT:    [[BASEPLUS:%.*]] = getelementptr i8, ptr [[BASE:%.*]], i64 100
-; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[BASE]], i8 42, i64 300, i1 false)
-; CHECK-NEXT:    call void @large_p2(ptr [[BASE]], ptr [[BASEPLUS]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[BASE]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  %baseplus = getelementptr i8, ptr %base, i64 100
-  call void @llvm.memset.p0.i64(ptr %base, i8 42, i64 300, i1 false)
-  call void @large_p2(ptr %base, ptr %baseplus)
-  %l = load i16, ptr %base
-  ret i16 %l
-}
-
-; Function Attrs: mustprogress nounwind uwtable
-define i16 @large_p2_may_or_partial_alias_caller2(ptr %base1, ptr %base2) {
-; CHECK-LABEL: @large_p2_may_or_partial_alias_caller2(
-; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[BASE1:%.*]], i8 42, i64 300, i1 false)
-; CHECK-NEXT:    call void @large_p2(ptr [[BASE1]], ptr [[BASE2:%.*]])
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[BASE1]], align 2
-; CHECK-NEXT:    ret i16 [[L]]
-;
-  call void @llvm.memset.p0.i64(ptr %base1, i8 42, i64 300, i1 false)
-  call void @large_p2(ptr %base1, ptr %base2)
-  %l = load i16, ptr %base1
-  ret i16 %l
-}
-
-- 
GitLab


From a21573bb99735e6364bb150c4bbae3423bfc2973 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Thu, 24 Oct 2024 08:51:51 -0700
Subject: [PATCH 067/693] [gn] port 6512a8d more

24293e690 added the files added there, but didn't do the tblgen changes.
---
 .../llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn  | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn
index 3897967d138c..edc3edc8d50f 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn
@@ -1,11 +1,20 @@
 import("//llvm/utils/TableGen/tablegen.gni")
 
-tablegen("SystemZGenAsmWriter") {
+tablegen("SystemZGenGNUAsmWriter") {
   visibility = [ ":MCTargetDesc" ]
   args = [ "-gen-asm-writer" ]
   td_file = "../SystemZ.td"
 }
 
+tablegen("SystemZGenHLAsmWriter") {
+  visibility = [ ":MCTargetDesc" ]
+  args = [
+    "-gen-asm-writer",
+    "-asmwriternum=1",
+  ]
+  td_file = "../SystemZ.td"
+}
+
 tablegen("SystemZGenInstrInfo") {
   visibility = [ ":MCTargetDesc" ]
   args = [ "-gen-instr-info" ]
@@ -42,7 +51,8 @@ static_library("MCTargetDesc") {
     ":SystemZGenSubtargetInfo",
   ]
   deps = [
-    ":SystemZGenAsmWriter",
+    ":SystemZGenGNUAsmWriter",
+    ":SystemZGenHLAsmWriter",
     ":SystemZGenMCCodeEmitter",
     "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
-- 
GitLab


From 460406affeecf811d20241f073c800a7b31c5592 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Thu, 24 Oct 2024 11:57:42 -0400
Subject: [PATCH 068/693] [libc++] Exclude C++03 frozen headers from the
 modulemap (#113572)

This is a temporary patch, eventually these headers should be included
in the modulemap as well.
---
 libcxx/utils/libcxx/header_information.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/libcxx/utils/libcxx/header_information.py b/libcxx/utils/libcxx/header_information.py
index c5849e80f085..528eb9995e19 100644
--- a/libcxx/utils/libcxx/header_information.py
+++ b/libcxx/utils/libcxx/header_information.py
@@ -95,6 +95,10 @@ class Header:
         removed_in_20 = ["ccomplex", "ciso646", "cstdbool", "ctgmath"]
         return self.is_public() and not self.is_experimental() and not self.is_C_compatibility() and not self._name in removed_in_20
 
+    def is_cxx03_frozen_header(self) -> bool:
+        """Returns whether the header is a frozen C++03 support header."""
+        return self._name.startswith("__cxx03/")
+
     def is_in_modulemap(self) -> bool:
         """Returns whether a header should be listed in the modulemap."""
         # TODO: Should `__config_site` be in the modulemap?
@@ -118,6 +122,11 @@ class Header:
         # burden ourself with maintaining them in any way.
         if self._name.startswith("ext/"):
             return False
+
+        # TODO: Frozen C++03 headers should probably be in the modulemap as well
+        if self.is_cxx03_frozen_header():
+            return False
+
         return True
 
     def __str__(self) -> str:
-- 
GitLab


From 907c13638819b36ecc5154775217068b1a1f4cfc Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Thu, 24 Oct 2024 08:57:39 -0700
Subject: [PATCH 069/693] [gn] port 6512a8d yet more

Follow-up to a21573bb9973 which had a typo.
---
 .../secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn
index edc3edc8d50f..f0db7906bbab 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn
@@ -6,7 +6,7 @@ tablegen("SystemZGenGNUAsmWriter") {
   td_file = "../SystemZ.td"
 }
 
-tablegen("SystemZGenHLAsmWriter") {
+tablegen("SystemZGenHLASMAsmWriter") {
   visibility = [ ":MCTargetDesc" ]
   args = [
     "-gen-asm-writer",
@@ -52,7 +52,7 @@ static_library("MCTargetDesc") {
   ]
   deps = [
     ":SystemZGenGNUAsmWriter",
-    ":SystemZGenHLAsmWriter",
+    ":SystemZGenHLASMAsmWriter",
     ":SystemZGenMCCodeEmitter",
     "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
-- 
GitLab


From e914421d7fbf22059ea2180233add0c914ec80f0 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Thu, 24 Oct 2024 08:57:07 -0700
Subject: [PATCH 070/693] [SLP]Do correct signedness analysis for externally
 used scalars

If the scalars is used externally is in the root node, it may have
incorrect signedness info because of the conflict with the demanded bits
analysis. Need to perform exact signedness analysis and compute it
rather than rely on the precomputed value, which might be incorrect for
alternate zext/sext nodes.

Fixes #113520
---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp        | 10 ++++++----
 .../SLPVectorizer/SystemZ/ext-alt-node-must-ext.ll     |  2 +-
 .../SLPVectorizer/resized-alt-shuffle-after-minbw.ll   |  2 +-
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 012d85353f08..2afd02dae3a8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -12068,8 +12068,9 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
     auto It = MinBWs.find(Entry);
     if (It != MinBWs.end()) {
       auto *MinTy = IntegerType::get(F->getContext(), It->second.first);
-      unsigned Extend =
-          It->second.second ? Instruction::SExt : Instruction::ZExt;
+      unsigned Extend = isKnownNonNegative(EU.Scalar, SimplifyQuery(*DL))
+                            ? Instruction::ZExt
+                            : Instruction::SExt;
       VecTy = getWidenedType(MinTy, BundleWidth);
       ExtraCost = TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
                                                 VecTy, EU.Lane);
@@ -15784,8 +15785,9 @@ BoUpSLP::vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
           // to the larger type.
           ExV = Ex;
           if (Scalar->getType() != Ex->getType())
-            ExV = Builder.CreateIntCast(Ex, Scalar->getType(),
-                                        MinBWs.find(E)->second.second);
+            ExV = Builder.CreateIntCast(
+                Ex, Scalar->getType(),
+                !isKnownNonNegative(Scalar, SimplifyQuery(*DL)));
           auto *I = dyn_cast<Instruction>(Ex);
           ScalarToEEs[Scalar].try_emplace(I ? I->getParent()
                                             : &F->getEntryBlock(),
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/ext-alt-node-must-ext.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/ext-alt-node-must-ext.ll
index 979d0ea66bac..d0c50b4dc3e4 100644
--- a/llvm/test/Transforms/SLPVectorizer/SystemZ/ext-alt-node-must-ext.ll
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/ext-alt-node-must-ext.ll
@@ -17,7 +17,7 @@ define i32 @test(ptr %0, ptr %1) {
 ; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i8> [[TMP12]], i32 0
 ; CHECK-NEXT:    [[DOTNEG:%.*]] = sext i8 [[TMP13]] to i32
 ; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i8> [[TMP12]], i32 1
-; CHECK-NEXT:    [[TMP8:%.*]] = sext i8 [[TMP15]] to i32
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i8 [[TMP15]] to i32
 ; CHECK-NEXT:    [[TMP10:%.*]] = add nsw i32 [[DOTNEG]], [[TMP8]]
 ; CHECK-NEXT:    ret i32 [[TMP10]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll b/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll
index 56281424c711..61a84a67c9ff 100644
--- a/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll
+++ b/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll
@@ -80,7 +80,7 @@ define void @func(i32 %0) {
 ; CHECK-NEXT:    [[TMP75:%.*]] = extractelement <32 x i1> [[TMP20]], i32 4
 ; CHECK-NEXT:    [[TMP76:%.*]] = and i1 false, [[TMP75]]
 ; CHECK-NEXT:    [[TMP77:%.*]] = extractelement <32 x i32> [[TMP18]], i32 0
-; CHECK-NEXT:    [[TMP78:%.*]] = sext i32 [[TMP77]] to i64
+; CHECK-NEXT:    [[TMP78:%.*]] = zext i32 [[TMP77]] to i64
 ; CHECK-NEXT:    [[TMP79:%.*]] = getelementptr float, ptr addrspace(1) null, i64 [[TMP78]]
 ; CHECK-NEXT:    ret void
 ;
-- 
GitLab


From 2646ce2a224c67435638a7d2ae72ba4e1a2c9f96 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Thu, 24 Oct 2024 09:08:43 -0700
Subject: [PATCH 071/693] [gn] port 522f34cfff693fd3f (PPC AsmMatcher dep)

Similar to 4168a2e9de35f. See also comment on #113482.
---
 .../gn/secondary/llvm/lib/Target/PowerPC/AsmParser/BUILD.gn  | 5 ++++-
 llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn     | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/AsmParser/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/AsmParser/BUILD.gn
index 250ff3ae11cf..34c88bed0738 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/AsmParser/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/AsmParser/BUILD.gn
@@ -1,7 +1,10 @@
 import("//llvm/utils/TableGen/tablegen.gni")
 
 tablegen("PPCGenAsmMatcher") {
-  visibility = [ ":AsmParser" ]
+  visibility = [
+    ":AsmParser",
+    "//llvm/lib/Target/PowerPC:LLVMPowerPCCodeGen",
+  ]
   args = [ "-gen-asm-matcher" ]
   td_file = "../PPC.td"
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
index 4ebeaff28c14..9e5ad929a313 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
@@ -37,6 +37,7 @@ static_library("LLVMPowerPCCodeGen") {
     ":PPCGenFastISel",
     ":PPCGenGlobalISel",
     ":PPCGenRegisterBank",
+    "AsmParser:PPCGenAsmMatcher",
     "MCTargetDesc",
     "TargetInfo",
     "//llvm/include/llvm/Config:llvm-config",
-- 
GitLab


From 762e65cf845c29bb5047f85244d8afc65a0665d0 Mon Sep 17 00:00:00 2001
From: SpencerAbson <Spencer.Abson@arm.com>
Date: Thu, 24 Oct 2024 17:14:42 +0100
Subject: [PATCH 072/693] [AArch64] Add assembly/disassembly for
 FRINT{32,64}{X,Z} (merging) (#113562)

This patch adds assembly/disassembly support for the following SVE2.2
instructions

    - FRINT32X (merging)
    - FRINT32Z (merging)
    - FRINT64X (merging)
    - FRINT64Z (merging)
- In accordance with:
https://developer.arm.com/documentation/ddi0602/latest/
---
 llvm/lib/Target/AArch64/AArch64.td            |  9 +++-
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |  7 +++
 llvm/lib/Target/AArch64/SVEInstrFormats.td    |  5 ++
 .../MC/AArch64/SVE2p2/frint32x-diagnostics.s  | 32 ++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frint32x.s        | 49 +++++++++++++++++++
 .../MC/AArch64/SVE2p2/frint32z-diagnostics.s  | 32 ++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frint32z.s        | 49 +++++++++++++++++++
 .../MC/AArch64/SVE2p2/frint64x-diagnostics.s  | 32 ++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frint64x.s        | 49 +++++++++++++++++++
 .../MC/AArch64/SVE2p2/frint64z-diagnostics.s  | 32 ++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frint64z.s        | 49 +++++++++++++++++++
 11 files changed, 343 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint32x-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint32x.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint32z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint32z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint64x-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint64x.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint64z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint64z.s

diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index 2262ad1dfd0c..9bb508b783c3 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -73,8 +73,13 @@ def SVEUnsupported : AArch64Unsupported {
                       SVE2Unsupported.F);
 }
 
-let F = [HasSME2p1, HasSVE2p1_or_HasSME2p1, HasSVE2p1orSSVE_AES] in
-def SME2p1Unsupported : AArch64Unsupported;
+let F = [HasSME2p2, HasSVE2p2orSME2p2] in
+def SME2p2Unsupported : AArch64Unsupported;
+
+def SME2p1Unsupported : AArch64Unsupported {
+  let F = !listconcat([HasSME2p1, HasSVE2p1_or_HasSME2p1, HasSVE2p1orSSVE_AES],
+                      SME2p2Unsupported.F);
+}
 
 def SME2Unsupported : AArch64Unsupported {
   let F = !listconcat([HasSME2, HasSVE2orSME2, HasSVE2p1_or_HasSME2, HasSSVE_FP8FMA,
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index a097c138c5d6..7b56f607a9de 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4226,6 +4226,13 @@ let Predicates = [HasSVE2p2orSME2p2] in {
 
   // SVE2p2 floating-point convert single-to-bf (placing odd), zeroing predicate
   def BFCVTNT_ZPzZ      : sve_fp_fcvt2z<0b1010, "bfcvtnt", ZPR16, ZPR32>;
+
+  // Floating point round to integral fp value in integer size range
+  // Merging
+  defm FRINT32Z_ZPmZ : sve_fp_2op_p_zd_frint<0b00, "frint32z">;
+  defm FRINT32X_ZPmZ : sve_fp_2op_p_zd_frint<0b01, "frint32x">;
+  defm FRINT64X_ZPmZ : sve_fp_2op_p_zd_frint<0b10, "frint64z">;
+  defm FRINT64Z_ZPmZ : sve_fp_2op_p_zd_frint<0b11, "frint64x">;
 } // End HasSME2p2orSVE2p2
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 42e00b534008..b45119512f8c 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -3104,6 +3104,11 @@ multiclass sve2_fp_convert_down_odd_rounding<string asm, string op, SDPatternOpe
   def : SVE_1_Op_Passthru_Pat<nxv2f32, ir_op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;
 }
 
+multiclass sve_fp_2op_p_zd_frint<bits<2> opc, string asm> {
+  def _S : sve_fp_2op_p_zd<{ 0b0010, opc{1}, 0, opc{0} }, asm, ZPR32, ZPR32, ElementSizeS>;
+  def _D : sve_fp_2op_p_zd<{ 0b0010, opc{1}, 1, opc{0} }, asm, ZPR64, ZPR64, ElementSizeD>;
+}
+
 //===----------------------------------------------------------------------===//
 // SVE Floating Point Unary Operations - Unpredicated Group
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint32x-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frint32x-diagnostics.s
new file mode 100644
index 000000000000..8daeb7c93623
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint32x-diagnostics.s
@@ -0,0 +1,32 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frint32x     z31.b, p7/m, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32x     z31.b, p7/m, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint32x     z31.h, p7/m, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32x     z31.h, p7/m, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint32x     z31.s, p7/m, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32x     z31.s, p7/m, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint32x     z31.d, p7/m, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32x     z31.d, p7/m, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frint32x     z31.s, p8/m, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frint32x     z31.s, p8/m, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint32x.s b/llvm/test/MC/AArch64/SVE2p2/frint32x.s
new file mode 100644
index 000000000000..abf402a1903a
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint32x.s
@@ -0,0 +1,49 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+movprfx  z23.d, p3/m, z31.d
+frint32x z23.d, p3/m, z13.d  // 01100101-00010011-10101101-10110111
+// CHECK-INST:  movprfx  z23.d, p3/m, z31.d
+// CHECK-INST: frint32x z23.d, p3/m, z13.d
+// CHECK-ENCODING: [0xb7,0xad,0x13,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6513adb7 <unknown>
+
+movprfx z23, z31
+frint32x z23.s, p3/m, z13.s  // 01100101-00010001-10101101-10110111
+// CHECK-INST:  movprfx z23, z31
+// CHECK-INST: frint32x z23.s, p3/m, z13.s
+// CHECK-ENCODING: [0xb7,0xad,0x11,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6511adb7 <unknown>
+
+frint32x z0.s, p0/m, z0.s  // 01100101-00010001-10100000-00000000
+// CHECK-INST: frint32x z0.s, p0/m, z0.s
+// CHECK-ENCODING: [0x00,0xa0,0x11,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6511a000 <unknown>
+
+frint32x z21.d, p5/m, z10.d  // 01100101-00010011-10110101-01010101
+// CHECK-INST: frint32x z21.d, p5/m, z10.d
+// CHECK-ENCODING: [0x55,0xb5,0x13,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6513b555 <unknown>
+
+frint32x z31.d, p7/m, z31.d  // 01100101-00010011-10111111-11111111
+// CHECK-INST: frint32x z31.d, p7/m, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0x13,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6513bfff <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint32z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frint32z-diagnostics.s
new file mode 100644
index 000000000000..9fb6f5f22530
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint32z-diagnostics.s
@@ -0,0 +1,32 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frint32z     z31.b, p7/m, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32z     z31.b, p7/m, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint32z     z31.h, p7/m, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32z     z31.h, p7/m, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint32z     z31.s, p7/m, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32z     z31.s, p7/m, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint32z     z31.d, p7/m, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32z     z31.d, p7/m, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frint32z     z31.s, p8/m, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frint32z     z31.s, p8/m, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint32z.s b/llvm/test/MC/AArch64/SVE2p2/frint32z.s
new file mode 100644
index 000000000000..117d8715a12c
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint32z.s
@@ -0,0 +1,49 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+movprfx  z23.d, p3/m, z31.d
+frint32z z23.d, p3/m, z13.d  // 01100101-00010010-10101101-10110111
+// CHECK-INST:  movprfx  z23.d, p3/m, z31.d
+// CHECK-INST: frint32z z23.d, p3/m, z13.d
+// CHECK-ENCODING: [0xb7,0xad,0x12,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6512adb7 <unknown>
+
+movprfx z23, z31
+frint32z z23.s, p3/m, z13.s  // 01100101-00010000-10101101-10110111
+// CHECK-INST:  movprfx z23, z31
+// CHECK-INST: frint32z z23.s, p3/m, z13.s
+// CHECK-ENCODING: [0xb7,0xad,0x10,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6510adb7 <unknown>
+
+frint32z z0.s, p0/m, z0.s  // 01100101-00010000-10100000-00000000
+// CHECK-INST: frint32z z0.s, p0/m, z0.s
+// CHECK-ENCODING: [0x00,0xa0,0x10,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6510a000 <unknown>
+
+frint32z z21.d, p5/m, z10.d  // 01100101-00010010-10110101-01010101
+// CHECK-INST: frint32z z21.d, p5/m, z10.d
+// CHECK-ENCODING: [0x55,0xb5,0x12,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6512b555 <unknown>
+
+frint32z z31.d, p7/m, z31.d  // 01100101-00010010-10111111-11111111
+// CHECK-INST: frint32z z31.d, p7/m, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0x12,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6512bfff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint64x-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frint64x-diagnostics.s
new file mode 100644
index 000000000000..d37e0488564a
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint64x-diagnostics.s
@@ -0,0 +1,32 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frint64x     z31.b, p7/m, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64x     z31.b, p7/m, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint64x     z31.h, p7/m, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64x     z31.h, p7/m, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint64x     z31.s, p7/m, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64x     z31.s, p7/m, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint64x     z31.d, p7/m, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64x     z31.d, p7/m, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frint64x     z31.s, p8/m, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frint64x     z31.s, p8/m, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint64x.s b/llvm/test/MC/AArch64/SVE2p2/frint64x.s
new file mode 100644
index 000000000000..25e3f3b17395
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint64x.s
@@ -0,0 +1,49 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+movprfx  z23.d, p3/m, z31.d
+frint64x z23.d, p3/m, z13.d  // 01100101-00010111-10101101-10110111
+// CHECK-INST:  movprfx  z23.d, p3/m, z31.d
+// CHECK-INST: frint64x z23.d, p3/m, z13.d
+// CHECK-ENCODING: [0xb7,0xad,0x17,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6517adb7 <unknown>
+
+movprfx z23, z31
+frint64x z23.s, p3/m, z13.s  // 01100101-00010101-10101101-10110111
+// CHECK-INST:  movprfx z23, z31
+// CHECK-INST: frint64x z23.s, p3/m, z13.s
+// CHECK-ENCODING: [0xb7,0xad,0x15,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6515adb7 <unknown>
+
+frint64x z21.d, p5/m, z10.d  // 01100101-00010111-10110101-01010101
+// CHECK-INST: frint64x z21.d, p5/m, z10.d
+// CHECK-ENCODING: [0x55,0xb5,0x17,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6517b555 <unknown>
+
+frint64x z31.d, p7/m, z31.d  // 01100101-00010111-10111111-11111111
+// CHECK-INST: frint64x z31.d, p7/m, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0x17,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6517bfff <unknown>
+
+frint64x z31.s, p7/m, z31.s  // 01100101-00010101-10111111-11111111
+// CHECK-INST: frint64x z31.s, p7/m, z31.s
+// CHECK-ENCODING: [0xff,0xbf,0x15,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6515bfff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint64z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frint64z-diagnostics.s
new file mode 100644
index 000000000000..b6dff833f9f5
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint64z-diagnostics.s
@@ -0,0 +1,32 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frint64z     z31.b, p7/m, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64z     z31.b, p7/m, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint64z     z31.h, p7/m, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64z     z31.h, p7/m, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint64z     z31.s, p7/m, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64z     z31.s, p7/m, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint64z     z31.d, p7/m, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64z     z31.d, p7/m, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frint64z     z31.s, p8/m, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frint64z     z31.s, p8/m, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint64z.s b/llvm/test/MC/AArch64/SVE2p2/frint64z.s
new file mode 100644
index 000000000000..d871d70112f0
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint64z.s
@@ -0,0 +1,49 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+movprfx  z23.d, p3/m, z31.d
+frint64z z23.d, p3/m, z13.d  // 01100101-00010110-10101101-10110111
+// CHECK-INST:  movprfx  z23.d, p3/m, z31.d
+// CHECK-INST: frint64z z23.d, p3/m, z13.d
+// CHECK-ENCODING: [0xb7,0xad,0x16,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6516adb7 <unknown>
+
+movprfx z23, z31
+frint64z z23.s, p3/m, z13.s  // 01100101-00010100-10101101-10110111
+// CHECK-INST:  movprfx z23, z31
+// CHECK-INST: frint64z z23.s, p3/m, z13.s
+// CHECK-ENCODING: [0xb7,0xad,0x14,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6514adb7 <unknown>
+
+frint64z z0.s, p0/m, z0.s  // 01100101-00010100-10100000-00000000
+// CHECK-INST: frint64z z0.s, p0/m, z0.s
+// CHECK-ENCODING: [0x00,0xa0,0x14,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6514a000 <unknown>
+
+frint64z z21.d, p5/m, z10.d  // 01100101-00010110-10110101-01010101
+// CHECK-INST: frint64z z21.d, p5/m, z10.d
+// CHECK-ENCODING: [0x55,0xb5,0x16,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6516b555 <unknown>
+
+frint64z z31.d, p7/m, z31.d  // 01100101-00010110-10111111-11111111
+// CHECK-INST: frint64z z31.d, p7/m, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0x16,0x65]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6516bfff <unknown>
\ No newline at end of file
-- 
GitLab


From 9a8292f91454573036b5de46992a93d4de7f9fa3 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Thu, 24 Oct 2024 09:19:53 -0700
Subject: [PATCH 073/693] [gn build] Port 8a12e0131f3d

---
 llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
index 1479e1c355d9..c46b969f67a3 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
@@ -40,7 +40,6 @@ static_library("Utils") {
     "GlobalStatus.cpp",
     "GuardUtils.cpp",
     "HelloWorld.cpp",
-    "IRNormalizer.cpp",
     "InjectTLIMappings.cpp",
     "InlineFunction.cpp",
     "InstructionNamer.cpp",
-- 
GitLab


From db0e3760442324d48adb68228d1939667807b2b8 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Thu, 24 Oct 2024 17:41:07 +0100
Subject: [PATCH 074/693] [AArch64] Fix failure with inline asm and svcount
 (#112537)

This fixes an issue where the compiler runs into an assertion failure
for the following example:

  register svcount_t pred asm("pn8") = svptrue_c8();
  asm("ld1w { z0.s, z4.s, z8.s, z12.s }, %[pred]/z, [x0]\n"
    :
    : [pred] "Uph" (pred)
    : "memory", "cc");

Here the register constraint that ends up in the LLVM IR is "{pn8}", but
the code in `TargetRegisterInfo::getRegForInlineAsmConstraint` that
parses that string, follows a path where it queries a suitable register
class for this register (<=> PPRorPNR regclass), for which it then
chooses `nxv16i1` as a suitable type. These choices individually are
correct, but the combined result isn't, because the type should be
`aarch64svcount`.
This then results in issues later on in SelectionDAGBuilder.cpp in
CopyToReg because the type of the actual value and the computed type
from the constraint don't match.

This PR pre-empts this issue by parsing the predicate explicitly and
returning the correct register class.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    | 32 ++++++++++
 llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll  | 64 +++++++++++++++++++
 2 files changed, 96 insertions(+)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 9adb11292376..5a848ada9dd8 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -11867,6 +11867,36 @@ const char *AArch64TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
 
 enum class PredicateConstraint { Uph, Upl, Upa };
 
+// Returns a {Reg, RegisterClass} tuple if the constraint is
+// a specific predicate register.
+//
+// For some constraint like "{pn3}" the default path in
+// TargetLowering::getRegForInlineAsmConstraint() leads it to determine that a
+// suitable register class for this register is "PPRorPNR", after which it
+// determines that nxv16i1 is an appropriate type for the constraint, which is
+// not what we want. The code here pre-empts this by matching the register
+// explicitly.
+static std::optional<std::pair<unsigned, const TargetRegisterClass *>>
+parsePredicateRegAsConstraint(StringRef Constraint) {
+  if (!Constraint.starts_with('{') || !Constraint.ends_with('}') ||
+      Constraint[1] != 'p')
+    return std::nullopt;
+
+  Constraint = Constraint.substr(2, Constraint.size() - 3);
+  bool IsPredicateAsCount = Constraint.starts_with("n");
+  if (IsPredicateAsCount)
+    Constraint = Constraint.drop_front(1);
+
+  unsigned V;
+  if (Constraint.getAsInteger(10, V) || V > 31)
+    return std::nullopt;
+
+  if (IsPredicateAsCount)
+    return std::make_pair(AArch64::PN0 + V, &AArch64::PNRRegClass);
+  else
+    return std::make_pair(AArch64::P0 + V, &AArch64::PPRRegClass);
+}
+
 static std::optional<PredicateConstraint>
 parsePredicateConstraint(StringRef Constraint) {
   return StringSwitch<std::optional<PredicateConstraint>>(Constraint)
@@ -12114,6 +12144,8 @@ AArch64TargetLowering::getRegForInlineAsmConstraint(
       break;
     }
   } else {
+    if (const auto P = parsePredicateRegAsConstraint(Constraint))
+      return *P;
     if (const auto PC = parsePredicateConstraint(Constraint))
       if (const auto *RegClass = getPredicateRegisterClass(*PC, VT))
         return std::make_pair(0U, RegClass);
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll
index 9f8897575b3d..ff66206228a4 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll
@@ -119,3 +119,67 @@ define <vscale x 8 x half> @test_svfadd_f16_Uph_constraint(<vscale x 16 x i1> %P
   %1 = tail call <vscale x 8 x half> asm "fadd $0.h, $1/m, $2.h, $3.h", "=w,@3Uph,w,w"(<vscale x 16 x i1> %Pg, <vscale x 8 x half> %Zn, <vscale x 8 x half> %Zm)
   ret <vscale x 8 x half> %1
 }
+
+define void @explicit_p0(ptr %p) {
+  ; CHECK-LABEL: name: explicit_p0
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $x0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+  ; CHECK-NEXT:   [[PTRUE_B:%[0-9]+]]:ppr = PTRUE_B 31, implicit $vg
+  ; CHECK-NEXT:   $p0 = COPY [[PTRUE_B]]
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr64common = COPY [[COPY]]
+  ; CHECK-NEXT:   INLINEASM &"ld4w { z0.s, z1.s, z2.s, z3.s }, $1/z, [$0]", 1 /* sideeffect attdialect */, 3538954 /* regdef:GPR64common */, def %1, 9 /* reguse */, $p0, 2147483657 /* reguse tiedto:$0 */, [[COPY1]](tied-def 3)
+  ; CHECK-NEXT:   RET_ReallyLR
+  %1 = tail call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.b8(i32 31)
+  %2 = tail call i64 asm sideeffect "ld4w { z0.s, z1.s, z2.s, z3.s }, $1/z, [$0]", "=r,{p0},0"(<vscale x 16 x i1> %1, ptr %p)
+  ret void
+}
+
+define void @explicit_p8_invalid(ptr %p) {
+  ; CHECK-LABEL: name: explicit_p8_invalid
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $x0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+  ; CHECK-NEXT:   [[PTRUE_B:%[0-9]+]]:ppr = PTRUE_B 31, implicit $vg
+  ; CHECK-NEXT:   $p8 = COPY [[PTRUE_B]]
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr64common = COPY [[COPY]]
+  ; CHECK-NEXT:   INLINEASM &"ld4w { z0.s, z1.s, z2.s, z3.s }, $1/z, [$0]", 1 /* sideeffect attdialect */, 3538954 /* regdef:GPR64common */, def %1, 9 /* reguse */, $p8, 2147483657 /* reguse tiedto:$0 */, [[COPY1]](tied-def 3)
+  ; CHECK-NEXT:   RET_ReallyLR
+  %1 = tail call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.b8(i32 31)
+  %2 = tail call i64 asm sideeffect "ld4w { z0.s, z1.s, z2.s, z3.s }, $1/z, [$0]", "=r,{p8},0"(<vscale x 16 x i1> %1, ptr %p)
+  ret void
+}
+
+define void @explicit_pn8(ptr %p) {
+  ; CHECK-LABEL: name: explicit_pn8
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $x0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+  ; CHECK-NEXT:   [[PTRUE_C_B:%[0-9]+]]:pnr_p8to15 = PTRUE_C_B implicit $vg
+  ; CHECK-NEXT:   $pn8 = COPY [[PTRUE_C_B]]
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr64common = COPY [[COPY]]
+  ; CHECK-NEXT:   INLINEASM &"ld1w { z0.s, z4.s, z8.s, z12.s }, $1/z, [$0]", 1 /* sideeffect attdialect */, 3538954 /* regdef:GPR64common */, def %1, 9 /* reguse */, $pn8, 2147483657 /* reguse tiedto:$0 */, [[COPY1]](tied-def 3)
+  ; CHECK-NEXT:   RET_ReallyLR
+  %1 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %2 = tail call i64 asm sideeffect "ld1w { z0.s, z4.s, z8.s, z12.s }, $1/z, [$0]", "=r,{pn8},0"(target("aarch64.svcount") %1, ptr %p)
+  ret void
+}
+
+define void @explicit_pn0_invalid(ptr %p) {
+  ; CHECK-LABEL: name: explicit_pn0_invalid
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $x0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+  ; CHECK-NEXT:   [[PTRUE_C_B:%[0-9]+]]:pnr_p8to15 = PTRUE_C_B implicit $vg
+  ; CHECK-NEXT:   $pn0 = COPY [[PTRUE_C_B]]
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr64common = COPY [[COPY]]
+  ; CHECK-NEXT:   INLINEASM &"ld1w { z0.s, z4.s, z8.s, z12.s }, $1/z, [$0]", 1 /* sideeffect attdialect */, 3538954 /* regdef:GPR64common */, def %1, 9 /* reguse */, $pn0, 2147483657 /* reguse tiedto:$0 */, [[COPY1]](tied-def 3)
+  ; CHECK-NEXT:   RET_ReallyLR
+  %1 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+  %2 = tail call i64 asm sideeffect "ld1w { z0.s, z4.s, z8.s, z12.s }, $1/z, [$0]", "=r,{pn0},0"(target("aarch64.svcount") %1, ptr %p)
+  ret void
+}
-- 
GitLab


From 0ef90bbdefc385bafb15e27cc2bfbeb34629ac40 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 24 Oct 2024 09:43:57 -0700
Subject: [PATCH 075/693] [MCParser] De-capitalize COFFAsmParser functions. NFC

---
 llvm/lib/MC/MCParser/COFFAsmParser.cpp | 217 +++++++++++++------------
 1 file changed, 109 insertions(+), 108 deletions(-)

diff --git a/llvm/lib/MC/MCParser/COFFAsmParser.cpp b/llvm/lib/MC/MCParser/COFFAsmParser.cpp
index a69276c36c56..53c0270255e8 100644
--- a/llvm/lib/MC/MCParser/COFFAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/COFFAsmParser.cpp
@@ -36,110 +36,111 @@ class COFFAsmParser : public MCAsmParserExtension {
     getParser().addDirectiveHandler(Directive, Handler);
   }
 
-  bool ParseSectionSwitch(StringRef Section, unsigned Characteristics);
+  bool parseSectionSwitch(StringRef Section, unsigned Characteristics);
 
-  bool ParseSectionSwitch(StringRef Section, unsigned Characteristics,
+  bool parseSectionSwitch(StringRef Section, unsigned Characteristics,
                           StringRef COMDATSymName, COFF::COMDATType Type);
 
-  bool ParseSectionName(StringRef &SectionName);
-  bool ParseSectionFlags(StringRef SectionName, StringRef FlagsString,
+  bool parseSectionName(StringRef &SectionName);
+  bool parseSectionFlags(StringRef SectionName, StringRef FlagsString,
                          unsigned *Flags);
 
   void Initialize(MCAsmParser &Parser) override {
     // Call the base implementation.
     MCAsmParserExtension::Initialize(Parser);
 
-    addDirectiveHandler<&COFFAsmParser::ParseSectionDirectiveText>(".text");
-    addDirectiveHandler<&COFFAsmParser::ParseSectionDirectiveData>(".data");
-    addDirectiveHandler<&COFFAsmParser::ParseSectionDirectiveBSS>(".bss");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectiveSection>(".section");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectivePushSection>(
+    addDirectiveHandler<&COFFAsmParser::parseSectionDirectiveText>(".text");
+    addDirectiveHandler<&COFFAsmParser::parseSectionDirectiveData>(".data");
+    addDirectiveHandler<&COFFAsmParser::parseSectionDirectiveBSS>(".bss");
+    addDirectiveHandler<&COFFAsmParser::parseDirectiveSection>(".section");
+    addDirectiveHandler<&COFFAsmParser::parseDirectivePushSection>(
         ".pushsection");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectivePopSection>(
+    addDirectiveHandler<&COFFAsmParser::parseDirectivePopSection>(
         ".popsection");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectiveDef>(".def");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectiveScl>(".scl");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectiveType>(".type");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectiveEndef>(".endef");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectiveSecRel32>(".secrel32");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectiveSymIdx>(".symidx");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectiveSafeSEH>(".safeseh");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectiveSecIdx>(".secidx");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectiveLinkOnce>(".linkonce");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectiveRVA>(".rva");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectiveSymbolAttribute>(".weak");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectiveSymbolAttribute>(".weak_anti_dep");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectiveCGProfile>(".cg_profile");
+    addDirectiveHandler<&COFFAsmParser::parseDirectiveDef>(".def");
+    addDirectiveHandler<&COFFAsmParser::parseDirectiveScl>(".scl");
+    addDirectiveHandler<&COFFAsmParser::parseDirectiveType>(".type");
+    addDirectiveHandler<&COFFAsmParser::parseDirectiveEndef>(".endef");
+    addDirectiveHandler<&COFFAsmParser::parseDirectiveSecRel32>(".secrel32");
+    addDirectiveHandler<&COFFAsmParser::parseDirectiveSymIdx>(".symidx");
+    addDirectiveHandler<&COFFAsmParser::parseDirectiveSafeSEH>(".safeseh");
+    addDirectiveHandler<&COFFAsmParser::parseDirectiveSecIdx>(".secidx");
+    addDirectiveHandler<&COFFAsmParser::parseDirectiveLinkOnce>(".linkonce");
+    addDirectiveHandler<&COFFAsmParser::parseDirectiveRVA>(".rva");
+    addDirectiveHandler<&COFFAsmParser::parseDirectiveSymbolAttribute>(".weak");
+    addDirectiveHandler<&COFFAsmParser::parseDirectiveSymbolAttribute>(
+        ".weak_anti_dep");
+    addDirectiveHandler<&COFFAsmParser::parseDirectiveCGProfile>(".cg_profile");
 
     // Win64 EH directives.
-    addDirectiveHandler<&COFFAsmParser::ParseSEHDirectiveStartProc>(
-                                                                   ".seh_proc");
-    addDirectiveHandler<&COFFAsmParser::ParseSEHDirectiveEndProc>(
-                                                                ".seh_endproc");
-    addDirectiveHandler<&COFFAsmParser::ParseSEHDirectiveEndFuncletOrFunc>(
-                                                                ".seh_endfunclet");
-    addDirectiveHandler<&COFFAsmParser::ParseSEHDirectiveStartChained>(
-                                                           ".seh_startchained");
-    addDirectiveHandler<&COFFAsmParser::ParseSEHDirectiveEndChained>(
-                                                             ".seh_endchained");
-    addDirectiveHandler<&COFFAsmParser::ParseSEHDirectiveHandler>(
-                                                                ".seh_handler");
-    addDirectiveHandler<&COFFAsmParser::ParseSEHDirectiveHandlerData>(
-                                                            ".seh_handlerdata");
-    addDirectiveHandler<&COFFAsmParser::ParseSEHDirectiveAllocStack>(
-                                                             ".seh_stackalloc");
-    addDirectiveHandler<&COFFAsmParser::ParseSEHDirectiveEndProlog>(
-                                                            ".seh_endprologue");
+    addDirectiveHandler<&COFFAsmParser::parseSEHDirectiveStartProc>(
+        ".seh_proc");
+    addDirectiveHandler<&COFFAsmParser::parseSEHDirectiveEndProc>(
+        ".seh_endproc");
+    addDirectiveHandler<&COFFAsmParser::parseSEHDirectiveEndFuncletOrFunc>(
+        ".seh_endfunclet");
+    addDirectiveHandler<&COFFAsmParser::parseSEHDirectiveStartChained>(
+        ".seh_startchained");
+    addDirectiveHandler<&COFFAsmParser::parseSEHDirectiveEndChained>(
+        ".seh_endchained");
+    addDirectiveHandler<&COFFAsmParser::parseSEHDirectiveHandler>(
+        ".seh_handler");
+    addDirectiveHandler<&COFFAsmParser::parseSEHDirectiveHandlerData>(
+        ".seh_handlerdata");
+    addDirectiveHandler<&COFFAsmParser::parseSEHDirectiveAllocStack>(
+        ".seh_stackalloc");
+    addDirectiveHandler<&COFFAsmParser::parseSEHDirectiveEndProlog>(
+        ".seh_endprologue");
   }
 
-  bool ParseSectionDirectiveText(StringRef, SMLoc) {
-    return ParseSectionSwitch(".text", COFF::IMAGE_SCN_CNT_CODE |
+  bool parseSectionDirectiveText(StringRef, SMLoc) {
+    return parseSectionSwitch(".text", COFF::IMAGE_SCN_CNT_CODE |
                                            COFF::IMAGE_SCN_MEM_EXECUTE |
                                            COFF::IMAGE_SCN_MEM_READ);
   }
 
-  bool ParseSectionDirectiveData(StringRef, SMLoc) {
-    return ParseSectionSwitch(".data", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+  bool parseSectionDirectiveData(StringRef, SMLoc) {
+    return parseSectionSwitch(".data", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
                                            COFF::IMAGE_SCN_MEM_READ |
                                            COFF::IMAGE_SCN_MEM_WRITE);
   }
 
-  bool ParseSectionDirectiveBSS(StringRef, SMLoc) {
-    return ParseSectionSwitch(".bss", COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA |
+  bool parseSectionDirectiveBSS(StringRef, SMLoc) {
+    return parseSectionSwitch(".bss", COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA |
                                           COFF::IMAGE_SCN_MEM_READ |
                                           COFF::IMAGE_SCN_MEM_WRITE);
   }
 
-  bool ParseDirectiveSection(StringRef, SMLoc);
+  bool parseDirectiveSection(StringRef, SMLoc);
   bool parseSectionArguments(StringRef, SMLoc);
-  bool ParseDirectivePushSection(StringRef, SMLoc);
-  bool ParseDirectivePopSection(StringRef, SMLoc);
-  bool ParseDirectiveDef(StringRef, SMLoc);
-  bool ParseDirectiveScl(StringRef, SMLoc);
-  bool ParseDirectiveType(StringRef, SMLoc);
-  bool ParseDirectiveEndef(StringRef, SMLoc);
-  bool ParseDirectiveSecRel32(StringRef, SMLoc);
-  bool ParseDirectiveSecIdx(StringRef, SMLoc);
-  bool ParseDirectiveSafeSEH(StringRef, SMLoc);
-  bool ParseDirectiveSymIdx(StringRef, SMLoc);
+  bool parseDirectivePushSection(StringRef, SMLoc);
+  bool parseDirectivePopSection(StringRef, SMLoc);
+  bool parseDirectiveDef(StringRef, SMLoc);
+  bool parseDirectiveScl(StringRef, SMLoc);
+  bool parseDirectiveType(StringRef, SMLoc);
+  bool parseDirectiveEndef(StringRef, SMLoc);
+  bool parseDirectiveSecRel32(StringRef, SMLoc);
+  bool parseDirectiveSecIdx(StringRef, SMLoc);
+  bool parseDirectiveSafeSEH(StringRef, SMLoc);
+  bool parseDirectiveSymIdx(StringRef, SMLoc);
   bool parseCOMDATType(COFF::COMDATType &Type);
-  bool ParseDirectiveLinkOnce(StringRef, SMLoc);
-  bool ParseDirectiveRVA(StringRef, SMLoc);
-  bool ParseDirectiveCGProfile(StringRef, SMLoc);
+  bool parseDirectiveLinkOnce(StringRef, SMLoc);
+  bool parseDirectiveRVA(StringRef, SMLoc);
+  bool parseDirectiveCGProfile(StringRef, SMLoc);
 
   // Win64 EH directives.
-  bool ParseSEHDirectiveStartProc(StringRef, SMLoc);
-  bool ParseSEHDirectiveEndProc(StringRef, SMLoc);
-  bool ParseSEHDirectiveEndFuncletOrFunc(StringRef, SMLoc);
-  bool ParseSEHDirectiveStartChained(StringRef, SMLoc);
-  bool ParseSEHDirectiveEndChained(StringRef, SMLoc);
-  bool ParseSEHDirectiveHandler(StringRef, SMLoc);
-  bool ParseSEHDirectiveHandlerData(StringRef, SMLoc);
-  bool ParseSEHDirectiveAllocStack(StringRef, SMLoc);
-  bool ParseSEHDirectiveEndProlog(StringRef, SMLoc);
-
-  bool ParseAtUnwindOrAtExcept(bool &unwind, bool &except);
-  bool ParseDirectiveSymbolAttribute(StringRef Directive, SMLoc);
+  bool parseSEHDirectiveStartProc(StringRef, SMLoc);
+  bool parseSEHDirectiveEndProc(StringRef, SMLoc);
+  bool parseSEHDirectiveEndFuncletOrFunc(StringRef, SMLoc);
+  bool parseSEHDirectiveStartChained(StringRef, SMLoc);
+  bool parseSEHDirectiveEndChained(StringRef, SMLoc);
+  bool parseSEHDirectiveHandler(StringRef, SMLoc);
+  bool parseSEHDirectiveHandlerData(StringRef, SMLoc);
+  bool parseSEHDirectiveAllocStack(StringRef, SMLoc);
+  bool parseSEHDirectiveEndProlog(StringRef, SMLoc);
+
+  bool parseAtUnwindOrAtExcept(bool &unwind, bool &except);
+  bool parseDirectiveSymbolAttribute(StringRef Directive, SMLoc);
 
 public:
   COFFAsmParser() = default;
@@ -147,7 +148,7 @@ public:
 
 } // end anonymous namespace.
 
-bool COFFAsmParser::ParseSectionFlags(StringRef SectionName,
+bool COFFAsmParser::parseSectionFlags(StringRef SectionName,
                                       StringRef FlagsString, unsigned *Flags) {
   enum {
     None = 0,
@@ -269,7 +270,7 @@ bool COFFAsmParser::ParseSectionFlags(StringRef SectionName,
 
 /// ParseDirectiveSymbolAttribute
 ///  ::= { ".weak", ... } [ identifier ( , identifier )* ]
-bool COFFAsmParser::ParseDirectiveSymbolAttribute(StringRef Directive, SMLoc) {
+bool COFFAsmParser::parseDirectiveSymbolAttribute(StringRef Directive, SMLoc) {
   MCSymbolAttr Attr = StringSwitch<MCSymbolAttr>(Directive)
     .Case(".weak", MCSA_Weak)
     .Case(".weak_anti_dep", MCSA_WeakAntiDep)
@@ -299,16 +300,16 @@ bool COFFAsmParser::ParseDirectiveSymbolAttribute(StringRef Directive, SMLoc) {
   return false;
 }
 
-bool COFFAsmParser::ParseDirectiveCGProfile(StringRef S, SMLoc Loc) {
+bool COFFAsmParser::parseDirectiveCGProfile(StringRef S, SMLoc Loc) {
   return MCAsmParserExtension::ParseDirectiveCGProfile(S, Loc);
 }
 
-bool COFFAsmParser::ParseSectionSwitch(StringRef Section,
+bool COFFAsmParser::parseSectionSwitch(StringRef Section,
                                        unsigned Characteristics) {
-  return ParseSectionSwitch(Section, Characteristics, "", (COFF::COMDATType)0);
+  return parseSectionSwitch(Section, Characteristics, "", (COFF::COMDATType)0);
 }
 
-bool COFFAsmParser::ParseSectionSwitch(StringRef Section,
+bool COFFAsmParser::parseSectionSwitch(StringRef Section,
                                        unsigned Characteristics,
                                        StringRef COMDATSymName,
                                        COFF::COMDATType Type) {
@@ -322,7 +323,7 @@ bool COFFAsmParser::ParseSectionSwitch(StringRef Section,
   return false;
 }
 
-bool COFFAsmParser::ParseSectionName(StringRef &SectionName) {
+bool COFFAsmParser::parseSectionName(StringRef &SectionName) {
   if (!getLexer().is(AsmToken::Identifier) && !getLexer().is(AsmToken::String))
     return true;
 
@@ -331,7 +332,7 @@ bool COFFAsmParser::ParseSectionName(StringRef &SectionName) {
   return false;
 }
 
-bool COFFAsmParser::ParseDirectiveSection(StringRef directive, SMLoc loc) {
+bool COFFAsmParser::parseDirectiveSection(StringRef directive, SMLoc loc) {
   return parseSectionArguments(directive, loc);
 }
 
@@ -354,7 +355,7 @@ bool COFFAsmParser::ParseDirectiveSection(StringRef directive, SMLoc loc) {
 bool COFFAsmParser::parseSectionArguments(StringRef, SMLoc) {
   StringRef SectionName;
 
-  if (ParseSectionName(SectionName))
+  if (parseSectionName(SectionName))
     return TokError("expected identifier in directive");
 
   unsigned Flags = COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
@@ -370,7 +371,7 @@ bool COFFAsmParser::parseSectionArguments(StringRef, SMLoc) {
     StringRef FlagsStr = getTok().getStringContents();
     Lex();
 
-    if (ParseSectionFlags(SectionName, FlagsStr, &Flags))
+    if (parseSectionFlags(SectionName, FlagsStr, &Flags))
       return true;
   }
 
@@ -405,11 +406,11 @@ bool COFFAsmParser::parseSectionArguments(StringRef, SMLoc) {
     if (T.getArch() == Triple::arm || T.getArch() == Triple::thumb)
       Flags |= COFF::IMAGE_SCN_MEM_16BIT;
   }
-  ParseSectionSwitch(SectionName, Flags, COMDATSymName, Type);
+  parseSectionSwitch(SectionName, Flags, COMDATSymName, Type);
   return false;
 }
 
-bool COFFAsmParser::ParseDirectivePushSection(StringRef directive, SMLoc loc) {
+bool COFFAsmParser::parseDirectivePushSection(StringRef directive, SMLoc loc) {
   getStreamer().pushSection();
 
   if (parseSectionArguments(directive, loc)) {
@@ -420,13 +421,13 @@ bool COFFAsmParser::ParseDirectivePushSection(StringRef directive, SMLoc loc) {
   return false;
 }
 
-bool COFFAsmParser::ParseDirectivePopSection(StringRef, SMLoc) {
+bool COFFAsmParser::parseDirectivePopSection(StringRef, SMLoc) {
   if (!getStreamer().popSection())
     return TokError(".popsection without corresponding .pushsection");
   return false;
 }
 
-bool COFFAsmParser::ParseDirectiveDef(StringRef, SMLoc) {
+bool COFFAsmParser::parseDirectiveDef(StringRef, SMLoc) {
   StringRef SymbolName;
 
   if (getParser().parseIdentifier(SymbolName))
@@ -440,7 +441,7 @@ bool COFFAsmParser::ParseDirectiveDef(StringRef, SMLoc) {
   return false;
 }
 
-bool COFFAsmParser::ParseDirectiveScl(StringRef, SMLoc) {
+bool COFFAsmParser::parseDirectiveScl(StringRef, SMLoc) {
   int64_t SymbolStorageClass;
   if (getParser().parseAbsoluteExpression(SymbolStorageClass))
     return true;
@@ -453,7 +454,7 @@ bool COFFAsmParser::ParseDirectiveScl(StringRef, SMLoc) {
   return false;
 }
 
-bool COFFAsmParser::ParseDirectiveType(StringRef, SMLoc) {
+bool COFFAsmParser::parseDirectiveType(StringRef, SMLoc) {
   int64_t Type;
   if (getParser().parseAbsoluteExpression(Type))
     return true;
@@ -466,13 +467,13 @@ bool COFFAsmParser::ParseDirectiveType(StringRef, SMLoc) {
   return false;
 }
 
-bool COFFAsmParser::ParseDirectiveEndef(StringRef, SMLoc) {
+bool COFFAsmParser::parseDirectiveEndef(StringRef, SMLoc) {
   Lex();
   getStreamer().endCOFFSymbolDef();
   return false;
 }
 
-bool COFFAsmParser::ParseDirectiveSecRel32(StringRef, SMLoc) {
+bool COFFAsmParser::parseDirectiveSecRel32(StringRef, SMLoc) {
   StringRef SymbolID;
   if (getParser().parseIdentifier(SymbolID))
     return TokError("expected identifier in directive");
@@ -501,7 +502,7 @@ bool COFFAsmParser::ParseDirectiveSecRel32(StringRef, SMLoc) {
   return false;
 }
 
-bool COFFAsmParser::ParseDirectiveRVA(StringRef, SMLoc) {
+bool COFFAsmParser::parseDirectiveRVA(StringRef, SMLoc) {
   auto parseOp = [&]() -> bool {
     StringRef SymbolID;
     if (getParser().parseIdentifier(SymbolID))
@@ -532,7 +533,7 @@ bool COFFAsmParser::ParseDirectiveRVA(StringRef, SMLoc) {
   return false;
 }
 
-bool COFFAsmParser::ParseDirectiveSafeSEH(StringRef, SMLoc) {
+bool COFFAsmParser::parseDirectiveSafeSEH(StringRef, SMLoc) {
   StringRef SymbolID;
   if (getParser().parseIdentifier(SymbolID))
     return TokError("expected identifier in directive");
@@ -547,7 +548,7 @@ bool COFFAsmParser::ParseDirectiveSafeSEH(StringRef, SMLoc) {
   return false;
 }
 
-bool COFFAsmParser::ParseDirectiveSecIdx(StringRef, SMLoc) {
+bool COFFAsmParser::parseDirectiveSecIdx(StringRef, SMLoc) {
   StringRef SymbolID;
   if (getParser().parseIdentifier(SymbolID))
     return TokError("expected identifier in directive");
@@ -562,7 +563,7 @@ bool COFFAsmParser::ParseDirectiveSecIdx(StringRef, SMLoc) {
   return false;
 }
 
-bool COFFAsmParser::ParseDirectiveSymIdx(StringRef, SMLoc) {
+bool COFFAsmParser::parseDirectiveSymIdx(StringRef, SMLoc) {
   StringRef SymbolID;
   if (getParser().parseIdentifier(SymbolID))
     return TokError("expected identifier in directive");
@@ -601,7 +602,7 @@ bool COFFAsmParser::parseCOMDATType(COFF::COMDATType &Type) {
 
 /// ParseDirectiveLinkOnce
 ///  ::= .linkonce [ identifier ]
-bool COFFAsmParser::ParseDirectiveLinkOnce(StringRef, SMLoc Loc) {
+bool COFFAsmParser::parseDirectiveLinkOnce(StringRef, SMLoc Loc) {
   COFF::COMDATType Type = COFF::IMAGE_COMDAT_SELECT_ANY;
   if (getLexer().is(AsmToken::Identifier))
     if (parseCOMDATType(Type))
@@ -625,7 +626,7 @@ bool COFFAsmParser::ParseDirectiveLinkOnce(StringRef, SMLoc Loc) {
   return false;
 }
 
-bool COFFAsmParser::ParseSEHDirectiveStartProc(StringRef, SMLoc Loc) {
+bool COFFAsmParser::parseSEHDirectiveStartProc(StringRef, SMLoc Loc) {
   StringRef SymbolID;
   if (getParser().parseIdentifier(SymbolID))
     return true;
@@ -640,31 +641,31 @@ bool COFFAsmParser::ParseSEHDirectiveStartProc(StringRef, SMLoc Loc) {
   return false;
 }
 
-bool COFFAsmParser::ParseSEHDirectiveEndProc(StringRef, SMLoc Loc) {
+bool COFFAsmParser::parseSEHDirectiveEndProc(StringRef, SMLoc Loc) {
   Lex();
   getStreamer().emitWinCFIEndProc(Loc);
   return false;
 }
 
-bool COFFAsmParser::ParseSEHDirectiveEndFuncletOrFunc(StringRef, SMLoc Loc) {
+bool COFFAsmParser::parseSEHDirectiveEndFuncletOrFunc(StringRef, SMLoc Loc) {
   Lex();
   getStreamer().emitWinCFIFuncletOrFuncEnd(Loc);
   return false;
 }
 
-bool COFFAsmParser::ParseSEHDirectiveStartChained(StringRef, SMLoc Loc) {
+bool COFFAsmParser::parseSEHDirectiveStartChained(StringRef, SMLoc Loc) {
   Lex();
   getStreamer().emitWinCFIStartChained(Loc);
   return false;
 }
 
-bool COFFAsmParser::ParseSEHDirectiveEndChained(StringRef, SMLoc Loc) {
+bool COFFAsmParser::parseSEHDirectiveEndChained(StringRef, SMLoc Loc) {
   Lex();
   getStreamer().emitWinCFIEndChained(Loc);
   return false;
 }
 
-bool COFFAsmParser::ParseSEHDirectiveHandler(StringRef, SMLoc Loc) {
+bool COFFAsmParser::parseSEHDirectiveHandler(StringRef, SMLoc Loc) {
   StringRef SymbolID;
   if (getParser().parseIdentifier(SymbolID))
     return true;
@@ -673,11 +674,11 @@ bool COFFAsmParser::ParseSEHDirectiveHandler(StringRef, SMLoc Loc) {
     return TokError("you must specify one or both of @unwind or @except");
   Lex();
   bool unwind = false, except = false;
-  if (ParseAtUnwindOrAtExcept(unwind, except))
+  if (parseAtUnwindOrAtExcept(unwind, except))
     return true;
   if (getLexer().is(AsmToken::Comma)) {
     Lex();
-    if (ParseAtUnwindOrAtExcept(unwind, except))
+    if (parseAtUnwindOrAtExcept(unwind, except))
       return true;
   }
   if (getLexer().isNot(AsmToken::EndOfStatement))
@@ -690,13 +691,13 @@ bool COFFAsmParser::ParseSEHDirectiveHandler(StringRef, SMLoc Loc) {
   return false;
 }
 
-bool COFFAsmParser::ParseSEHDirectiveHandlerData(StringRef, SMLoc Loc) {
+bool COFFAsmParser::parseSEHDirectiveHandlerData(StringRef, SMLoc Loc) {
   Lex();
   getStreamer().emitWinEHHandlerData();
   return false;
 }
 
-bool COFFAsmParser::ParseSEHDirectiveAllocStack(StringRef, SMLoc Loc) {
+bool COFFAsmParser::parseSEHDirectiveAllocStack(StringRef, SMLoc Loc) {
   int64_t Size;
   if (getParser().parseAbsoluteExpression(Size))
     return true;
@@ -709,13 +710,13 @@ bool COFFAsmParser::ParseSEHDirectiveAllocStack(StringRef, SMLoc Loc) {
   return false;
 }
 
-bool COFFAsmParser::ParseSEHDirectiveEndProlog(StringRef, SMLoc Loc) {
+bool COFFAsmParser::parseSEHDirectiveEndProlog(StringRef, SMLoc Loc) {
   Lex();
   getStreamer().emitWinCFIEndProlog(Loc);
   return false;
 }
 
-bool COFFAsmParser::ParseAtUnwindOrAtExcept(bool &unwind, bool &except) {
+bool COFFAsmParser::parseAtUnwindOrAtExcept(bool &unwind, bool &except) {
   StringRef identifier;
   if (getLexer().isNot(AsmToken::At) && getLexer().isNot(AsmToken::Percent))
     return TokError("a handler attribute must begin with '@' or '%'");
-- 
GitLab


From 5d37415a589177c4449a8262553bee5366c1bad6 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Thu, 24 Oct 2024 11:44:05 -0500
Subject: [PATCH 076/693] Unsupport flang/test/Driver/embed.f90 on Windows

The test fails due to Windows' line-endings, and it's blocking
pre-checkin tests.
---
 flang/test/Driver/embed.f90 | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flang/test/Driver/embed.f90 b/flang/test/Driver/embed.f90
index 60d532a8a94f..ce82beec1c6a 100644
--- a/flang/test/Driver/embed.f90
+++ b/flang/test/Driver/embed.f90
@@ -1,3 +1,5 @@
+!Windows uses 0d+0a as line-endings, and the check fails.
+!UNSUPPORTED: system-windows
 
 !----------
 ! RUN lines
-- 
GitLab


From 5d07162bba0648f5a5733039a7795eb7e9913863 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Thu, 24 Oct 2024 12:52:17 -0400
Subject: [PATCH 077/693] [OpenMP] Fix the test issue when `libomp` is built as
 a static library (#113522)

---
 openmp/runtime/test/tasking/hidden_helper_task/gtid.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/openmp/runtime/test/tasking/hidden_helper_task/gtid.cpp b/openmp/runtime/test/tasking/hidden_helper_task/gtid.cpp
index bc02caccb69e..9a07564406f7 100644
--- a/openmp/runtime/test/tasking/hidden_helper_task/gtid.cpp
+++ b/openmp/runtime/test/tasking/hidden_helper_task/gtid.cpp
@@ -43,7 +43,7 @@ struct anon {
 };
 }
 
-kmp_int32 __kmp_hidden_helper_threads_num;
+static kmp_int32 __kmp_hidden_helper_threads_num;
 
 kmp_int32 omp_task_entry(kmp_int32 gtid, kmp_task_t_with_privates *task) {
   auto shareds = reinterpret_cast<anon *>(task->task.shareds);
-- 
GitLab


From e2e7d565bf7228b5c4994257950c64d419cc8f03 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Thu, 24 Oct 2024 12:10:18 -0500
Subject: [PATCH 078/693] [flang][OpenMP] Make Symbol::OmpFlagToClauseName
 static (#113586)

It doesn't need the Symbol object for anything.
---
 flang/include/flang/Semantics/symbol.h     | 2 +-
 flang/lib/Semantics/resolve-directives.cpp | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h
index cf0350735b5b..0767d8ea84bc 100644
--- a/flang/include/flang/Semantics/symbol.h
+++ b/flang/include/flang/Semantics/symbol.h
@@ -778,7 +778,7 @@ public:
   void set_offset(std::size_t offset) { offset_ = offset; }
   // Give the symbol a name with a different source location but same chars.
   void ReplaceName(const SourceName &);
-  std::string OmpFlagToClauseName(Flag ompFlag);
+  static std::string OmpFlagToClauseName(Flag ompFlag);
 
   // Does symbol have this type of details?
   template <typename D> bool has() const {
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 490d802cddf4..33936ba4c2b3 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -2320,7 +2320,7 @@ void OmpAttributeVisitor::ResolveOmpObject(
               if (auto *symbol{ResolveOmp(*name, ompFlag, currScope())}) {
                 auto checkExclusivelists =
                     [&](const Symbol *symbol1, Symbol::Flag firstOmpFlag,
-                        Symbol *symbol2, Symbol::Flag secondOmpFlag) {
+                        const Symbol *symbol2, Symbol::Flag secondOmpFlag) {
                       if ((symbol1->test(firstOmpFlag) &&
                               symbol2->test(secondOmpFlag)) ||
                           (symbol1->test(secondOmpFlag) &&
@@ -2330,9 +2330,8 @@ void OmpAttributeVisitor::ResolveOmpObject(
                             "appear on both %s and %s "
                             "clauses on a %s construct"_err_en_US,
                             symbol2->name(),
-                            const_cast<Symbol *>(symbol1)->OmpFlagToClauseName(
-                                firstOmpFlag),
-                            symbol2->OmpFlagToClauseName(secondOmpFlag),
+                            Symbol::OmpFlagToClauseName(firstOmpFlag),
+                            Symbol::OmpFlagToClauseName(secondOmpFlag),
                             parser::ToUpperCaseLetters(
                                 llvm::omp::getOpenMPDirectiveName(
                                     GetContext().directive)
-- 
GitLab


From 100582ed4f5189c661c365e1ba8b07eb3af0f844 Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Thu, 24 Oct 2024 19:27:38 +0200
Subject: [PATCH 079/693] Pass std::initializer_list by value to ArrayRef
 constructor. (#113590)

The std::initializer_list is a lightweight object, it is passed by value
in general.

This would also avoid a false positive when adding the lifetimebound
annotation (#113547)

```
ArrayRef<int> foo(std::initializer_list<int> list) {
  return ArrayRef<int>(list);
}
```
---
 llvm/include/llvm/ADT/ArrayRef.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/ADT/ArrayRef.h b/llvm/include/llvm/ADT/ArrayRef.h
index d9897320ce09..bf6b55923b84 100644
--- a/llvm/include/llvm/ADT/ArrayRef.h
+++ b/llvm/include/llvm/ADT/ArrayRef.h
@@ -113,7 +113,7 @@ namespace llvm {
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Winit-list-lifetime"
 #endif
-    constexpr /*implicit*/ ArrayRef(const std::initializer_list<T> &Vec)
+    constexpr /*implicit*/ ArrayRef(std::initializer_list<T> Vec)
         : Data(Vec.begin() == Vec.end() ? (T *)nullptr : Vec.begin()),
           Length(Vec.size()) {}
 #if LLVM_GNUC_PREREQ(9, 0, 0)
-- 
GitLab


From 87b3c07ae696883098f4780765b7cf1fcf75716d Mon Sep 17 00:00:00 2001
From: SpencerAbson <Spencer.Abson@arm.com>
Date: Thu, 24 Oct 2024 18:29:05 +0100
Subject: [PATCH 080/693] [AArch64] Add assembly/disassembly for zeroing FRINT
 and FRECPX/FSQRT (#113543)

This patch adds assembly/disassembly support for the following
predicated SVE2.2 instructions

    - FRINT32X, FRINT32Z (zeroing)
    - FRINT64X, FRINT64Z (zeroing)
    - FRINT{N,P,M,Z,A,X,I} (zeroing)
    - FRECPX, FSQRT (zeroing)

- Updates the diagnostics tests for existing merging variants of these
instructions.
- In accordance with:
https://developer.arm.com/documentation/ddi0602/latest/
---
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 19 ++++++++
 llvm/lib/Target/AArch64/SVEInstrFormats.td    | 38 +++++++++++++++
 llvm/test/MC/AArch64/SVE/frecpx-diagnostics.s |  2 +-
 llvm/test/MC/AArch64/SVE/frinta-diagnostics.s |  2 +-
 llvm/test/MC/AArch64/SVE/frinti-diagnostics.s |  2 +-
 llvm/test/MC/AArch64/SVE/frintm-diagnostics.s |  2 +-
 llvm/test/MC/AArch64/SVE/frintn-diagnostics.s |  2 +-
 llvm/test/MC/AArch64/SVE/frintp-diagnostics.s |  2 +-
 llvm/test/MC/AArch64/SVE/frintx-diagnostics.s |  2 +-
 llvm/test/MC/AArch64/SVE/frintz-diagnostics.s |  2 +-
 llvm/test/MC/AArch64/SVE/fsqrt-diagnostics.s  |  2 +-
 .../MC/AArch64/SVE2p2/frecpx_z-diagnostics.s  | 47 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frecpx_z.s        | 33 +++++++++++++
 .../AArch64/SVE2p2/frint32x_z-diagnostics.s   | 47 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frint32x_z.s      | 33 +++++++++++++
 .../AArch64/SVE2p2/frint32z_z-diagnostics.s   | 47 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frint32z_z.s      | 33 +++++++++++++
 .../AArch64/SVE2p2/frint64x_z-diagnostics.s   | 47 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frint64x_z.s      | 33 +++++++++++++
 .../AArch64/SVE2p2/frint64z_z-diagnostics.s   | 47 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frint64z_z.s      | 33 +++++++++++++
 .../MC/AArch64/SVE2p2/frinta_z-diagnostics.s  | 47 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frinta_z.s        | 34 ++++++++++++++
 .../MC/AArch64/SVE2p2/frinti_z-diagnostics.s  | 47 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frinti_z.s        | 33 +++++++++++++
 .../MC/AArch64/SVE2p2/frintm_z-diagnostics.s  | 47 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frintm_z.s        | 33 +++++++++++++
 .../MC/AArch64/SVE2p2/frintn_z-diagnostics.s  | 47 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frintn_z.s        | 33 +++++++++++++
 .../MC/AArch64/SVE2p2/frintp_z-diagnostics.s  | 47 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frintp_z.s        | 33 +++++++++++++
 .../MC/AArch64/SVE2p2/frintx_z-diagnostics.s  | 47 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frintx_z.s        | 33 +++++++++++++
 .../MC/AArch64/SVE2p2/frintz_z-diagnostics.s  | 47 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/frintz_z.s        | 33 +++++++++++++
 .../MC/AArch64/SVE2p2/fsqrt_z-diagnostics.s   | 47 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/fsqrt_z.s         | 33 +++++++++++++
 37 files changed, 1107 insertions(+), 9 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frecpx_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frecpx_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint32x_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint32x_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint32z_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint32z_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint64x_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint64x_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint64z_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frint64z_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frinta_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frinta_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frinti_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frinti_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frintm_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frintm_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frintn_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frintn_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frintp_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frintp_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frintx_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frintx_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frintz_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/frintz_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/fsqrt_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/fsqrt_z.s

diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 7b56f607a9de..59859cb7442d 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4233,6 +4233,25 @@ let Predicates = [HasSVE2p2orSME2p2] in {
   defm FRINT32X_ZPmZ : sve_fp_2op_p_zd_frint<0b01, "frint32x">;
   defm FRINT64X_ZPmZ : sve_fp_2op_p_zd_frint<0b10, "frint64z">;
   defm FRINT64Z_ZPmZ : sve_fp_2op_p_zd_frint<0b11, "frint64x">;
+  // Zeroing
+  defm FRINT32Z_ZPzZ : sve_fp_z2op_p_zd_frint<0b00, "frint32z">;
+  defm FRINT32X_ZPzZ : sve_fp_z2op_p_zd_frint<0b01, "frint32x">;
+  defm FRINT64Z_ZPzZ : sve_fp_z2op_p_zd_frint<0b10, "frint64z">;
+  defm FRINT64X_ZPzZ : sve_fp_z2op_p_zd_frint<0b11, "frint64x">;
+
+  // Floating-point round to integral fp value, zeroing predicate
+  defm FRINTN_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00000, "frintn">;
+  defm FRINTP_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00001, "frintp">;
+  defm FRINTM_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00010, "frintm">;
+  defm FRINTZ_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00011, "frintz">;
+  defm FRINTA_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00100, "frinta">;
+  defm FRINTX_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00110, "frintx">;
+  defm FRINTI_ZPzZ : sve_fp_z2op_p_zd_hsd<0b00111, "frinti">;
+  // Floating-point invert exponent, zeroing predicate
+  defm FRECPX_ZPzZ : sve_fp_z2op_p_zd_hsd<0b01100, "frecpx">;
+  // Floating-point square root, zeroing predicate
+  defm FSQRT_ZPZz  : sve_fp_z2op_p_zd_hsd<0b01101, "fsqrt">;
+
 } // End HasSME2p2orSVE2p2
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index b45119512f8c..fc2e889d3a1a 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -3143,6 +3143,44 @@ multiclass sve_fp_2op_u_zd<bits<3> opc, string asm, SDPatternOperator op> {
   def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>;
 }
 
+//===----------------------------------------------------------------------===//
+// SVE Floating Point Unary Operations - Zeroing Predicate Group
+//===----------------------------------------------------------------------===//
+
+class sve_fp_z2op_p_zd<bits<7> opc,string asm, RegisterOperand i_zprtype,
+                       RegisterOperand o_zprtype>
+: I<(outs o_zprtype:$Zd), (ins PPR3bAny:$Pg, i_zprtype:$Zn),
+  asm, "\t$Zd, $Pg/z, $Zn",
+  "",
+  []>, Sched<[]> {
+  bits<3> Pg;
+  bits<5> Zd;
+  bits<5> Zn;
+  let Inst{31-24} = 0b01100100;
+  let Inst{23-22} = opc{6-5};
+  let Inst{21-19} = 0b011;
+  let Inst{18-16} = opc{4-2};
+  let Inst{15}    = 0b1;
+  let Inst{14-13} = opc{1-0};
+  let Inst{12-10} = Pg;
+  let Inst{9-5}   = Zn;
+  let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
+}
+
+multiclass sve_fp_z2op_p_zd_hsd<bits<5> opc, string asm> {
+  def _H : sve_fp_z2op_p_zd<{ 0b01, opc }, asm, ZPR16, ZPR16>;
+  def _S : sve_fp_z2op_p_zd<{ 0b10, opc }, asm, ZPR32, ZPR32>;
+  def _D : sve_fp_z2op_p_zd<{ 0b11, opc }, asm, ZPR64, ZPR64>;
+}
+
+multiclass sve_fp_z2op_p_zd_frint<bits<2> opc, string asm> {
+  def _S : sve_fp_z2op_p_zd<{ 0b0010, opc{1}, 0, opc{0} }, asm, ZPR32, ZPR32>;
+  def _D : sve_fp_z2op_p_zd<{ 0b0010, opc{1}, 1, opc{0} }, asm, ZPR64, ZPR64>;
+}
+
 //===----------------------------------------------------------------------===//
 // SVE Integer Arithmetic - Binary Predicated Group
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AArch64/SVE/frecpx-diagnostics.s b/llvm/test/MC/AArch64/SVE/frecpx-diagnostics.s
index a24f9c4a74c8..9b8d6889e5ff 100644
--- a/llvm/test/MC/AArch64/SVE/frecpx-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE/frecpx-diagnostics.s
@@ -6,7 +6,7 @@ frecpx  z0.b, p0/m, z0.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 frecpx  z0.s, p0/z, z0.s
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: frecpx  z0.s, p0/z, z0.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE/frinta-diagnostics.s b/llvm/test/MC/AArch64/SVE/frinta-diagnostics.s
index 136e123cb7eb..9ed088e9e9d1 100644
--- a/llvm/test/MC/AArch64/SVE/frinta-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE/frinta-diagnostics.s
@@ -6,7 +6,7 @@ frinta  z0.b, p0/m, z0.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 frinta  z0.s, p0/z, z0.s
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: frinta  z0.s, p0/z, z0.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE/frinti-diagnostics.s b/llvm/test/MC/AArch64/SVE/frinti-diagnostics.s
index 8c619214c64d..c8dda15a2cbd 100644
--- a/llvm/test/MC/AArch64/SVE/frinti-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE/frinti-diagnostics.s
@@ -6,7 +6,7 @@ frinti  z0.b, p0/m, z0.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 frinti  z0.s, p0/z, z0.s
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: frinti  z0.s, p0/z, z0.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE/frintm-diagnostics.s b/llvm/test/MC/AArch64/SVE/frintm-diagnostics.s
index a8ba48d353d0..cc6e72523464 100644
--- a/llvm/test/MC/AArch64/SVE/frintm-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE/frintm-diagnostics.s
@@ -6,7 +6,7 @@ frintm  z0.b, p0/m, z0.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 frintm  z0.s, p0/z, z0.s
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: frintm  z0.s, p0/z, z0.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE/frintn-diagnostics.s b/llvm/test/MC/AArch64/SVE/frintn-diagnostics.s
index f5fb07052436..09f64fc1ad6c 100644
--- a/llvm/test/MC/AArch64/SVE/frintn-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE/frintn-diagnostics.s
@@ -6,7 +6,7 @@ frintn  z0.b, p0/m, z0.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 frintn  z0.s, p0/z, z0.s
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: frintn  z0.s, p0/z, z0.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE/frintp-diagnostics.s b/llvm/test/MC/AArch64/SVE/frintp-diagnostics.s
index 2766299d4c9b..ce7b7eafd839 100644
--- a/llvm/test/MC/AArch64/SVE/frintp-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE/frintp-diagnostics.s
@@ -6,7 +6,7 @@ frintp  z0.b, p0/m, z0.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 frintp  z0.s, p0/z, z0.s
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: frintp  z0.s, p0/z, z0.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE/frintx-diagnostics.s b/llvm/test/MC/AArch64/SVE/frintx-diagnostics.s
index 98620898cde9..edfa713ca00a 100644
--- a/llvm/test/MC/AArch64/SVE/frintx-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE/frintx-diagnostics.s
@@ -6,7 +6,7 @@ frintx  z0.b, p0/m, z0.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 frintx  z0.s, p0/z, z0.s
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: frintx  z0.s, p0/z, z0.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE/frintz-diagnostics.s b/llvm/test/MC/AArch64/SVE/frintz-diagnostics.s
index 8bb6717ce3ac..ec974446bed3 100644
--- a/llvm/test/MC/AArch64/SVE/frintz-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE/frintz-diagnostics.s
@@ -6,7 +6,7 @@ frintz  z0.b, p0/m, z0.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 frintz  z0.s, p0/z, z0.s
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: frintz  z0.s, p0/z, z0.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE/fsqrt-diagnostics.s b/llvm/test/MC/AArch64/SVE/fsqrt-diagnostics.s
index 8b1abedb752c..4eb14ec34304 100644
--- a/llvm/test/MC/AArch64/SVE/fsqrt-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE/fsqrt-diagnostics.s
@@ -6,7 +6,7 @@ fsqrt   z0.b, p0/m, z0.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 fsqrt   z0.s, p0/z, z0.s
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: fsqrt   z0.s, p0/z, z0.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE2p2/frecpx_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frecpx_z-diagnostics.s
new file mode 100644
index 000000000000..b2aed8c7d592
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frecpx_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frecpx     z31.b, p7/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frecpx     z31.b, p7/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frecpx     z31.h, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frecpx     z31.h, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frecpx     z31.s, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frecpx     z31.s, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frecpx     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frecpx     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frecpx     z31.h, p8/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frecpx     z31.h, p8/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.h, p0/z, z7.h
+frecpx z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frecpx z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+frecpx z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frecpx z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frecpx_z.s b/llvm/test/MC/AArch64/SVE2p2/frecpx_z.s
new file mode 100644
index 000000000000..c089c8d623ba
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frecpx_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+frecpx  z0.h, p0/z, z0.h  // 01100100-01011011-10000000-00000000
+// CHECK-INST: frecpx  z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0x80,0x5b,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 645b8000 <unknown>
+
+frecpx  z23.s, p3/z, z13.s  // 01100100-10011011-10001101-10110111
+// CHECK-INST: frecpx  z23.s, p3/z, z13.s
+// CHECK-ENCODING: [0xb7,0x8d,0x9b,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 649b8db7 <unknown>
+
+frecpx  z31.d, p7/z, z31.d  // 01100100-11011011-10011111-11111111
+// CHECK-INST: frecpx  z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0x9f,0xdb,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64db9fff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint32x_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frint32x_z-diagnostics.s
new file mode 100644
index 000000000000..a01eb73d5082
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint32x_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frint32x     z31.b, p7/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32x     z31.b, p7/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint32x     z31.h, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32x     z31.h, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint32x     z31.s, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32x     z31.s, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint32x     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32x     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frint32x     z31.s, p8/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frint32x     z31.s, p8/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.s, p0/z, z7.s
+frint32x z0.s, p0/z, z3.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frint32x z0.s, p0/z, z3.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+frint32x z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frint32x z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint32x_z.s b/llvm/test/MC/AArch64/SVE2p2/frint32x_z.s
new file mode 100644
index 000000000000..abb0f78fb1e6
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint32x_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+frint32x z0.d, p0/z, z0.d  // 01100100-00011100-11100000-00000000
+// CHECK-INST: frint32x z0.d, p0/z, z0.d
+// CHECK-ENCODING: [0x00,0xe0,0x1c,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641ce000 <unknown>
+
+frint32x z23.d, p3/z, z13.d  // 01100100-00011100-11101101-10110111
+// CHECK-INST: frint32x z23.d, p3/z, z13.d
+// CHECK-ENCODING: [0xb7,0xed,0x1c,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641cedb7 <unknown>
+
+frint32x z31.s, p7/z, z31.s  // 01100100-00011100-10111111-11111111
+// CHECK-INST: frint32x z31.s, p7/z, z31.s
+// CHECK-ENCODING: [0xff,0xbf,0x1c,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641cbfff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint32z_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frint32z_z-diagnostics.s
new file mode 100644
index 000000000000..29bcfeca145a
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint32z_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frint32z     z31.b, p7/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32z     z31.b, p7/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint32z     z31.h, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32z     z31.h, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint32z     z31.s, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32z     z31.s, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint32z     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint32z     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frint32z     z31.s, p8/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frint32z     z31.s, p8/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.s, p0/z, z7.s
+frint32z z0.s, p0/z, z3.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frint32z z0.s, p0/z, z3.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+frint32z z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frint32z z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint32z_z.s b/llvm/test/MC/AArch64/SVE2p2/frint32z_z.s
new file mode 100644
index 000000000000..13140e4bdda9
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint32z_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+frint32z z0.d, p0/z, z0.d  // 01100100-00011100-11000000-00000000
+// CHECK-INST: frint32z z0.d, p0/z, z0.d
+// CHECK-ENCODING: [0x00,0xc0,0x1c,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641cc000 <unknown>
+
+frint32z z23.s, p3/z, z13.s  // 01100100-00011100-10001101-10110111
+// CHECK-INST: frint32z z23.s, p3/z, z13.s
+// CHECK-ENCODING: [0xb7,0x8d,0x1c,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641c8db7 <unknown>
+
+frint32z z31.s, p7/z, z31.s  // 01100100-00011100-10011111-11111111
+// CHECK-INST: frint32z z31.s, p7/z, z31.s
+// CHECK-ENCODING: [0xff,0x9f,0x1c,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641c9fff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint64x_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frint64x_z-diagnostics.s
new file mode 100644
index 000000000000..82ffa84645be
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint64x_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frint64x     z31.b, p7/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64x     z31.b, p7/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint64x     z31.h, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64x     z31.h, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint64x     z31.s, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64x     z31.s, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint64x     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64x     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frint64x     z31.s, p8/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frint64x     z31.s, p8/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.s, p0/z, z7.s
+frint64x z0.s, p0/z, z3.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frint64x z0.s, p0/z, z3.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+frint64x z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frint64x z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint64x_z.s b/llvm/test/MC/AArch64/SVE2p2/frint64x_z.s
new file mode 100644
index 000000000000..1e834e972c2d
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint64x_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+frint64x z0.d, p0/z, z0.d  // 01100100-00011101-11100000-00000000
+// CHECK-INST: frint64x z0.d, p0/z, z0.d
+// CHECK-ENCODING: [0x00,0xe0,0x1d,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641de000 <unknown>
+
+frint64x z21.s, p5/z, z10.s  // 01100100-00011101-10110101-01010101
+// CHECK-INST: frint64x z21.s, p5/z, z10.s
+// CHECK-ENCODING: [0x55,0xb5,0x1d,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641db555 <unknown>
+
+frint64x z31.s, p7/z, z31.s  // 01100100-00011101-10111111-11111111
+// CHECK-INST: frint64x z31.s, p7/z, z31.s
+// CHECK-ENCODING: [0xff,0xbf,0x1d,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641dbfff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint64z_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frint64z_z-diagnostics.s
new file mode 100644
index 000000000000..b2176ab345ab
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint64z_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frint64z     z31.b, p7/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64z     z31.b, p7/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint64z     z31.h, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64z     z31.h, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint64z     z31.s, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64z     z31.s, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frint64z     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frint64z     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frint64z     z31.s, p8/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frint64z     z31.s, p8/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.s, p0/z, z7.s
+frint64z z0.s, p0/z, z3.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frint64z z0.s, p0/z, z3.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+frint64z z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frint64z z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frint64z_z.s b/llvm/test/MC/AArch64/SVE2p2/frint64z_z.s
new file mode 100644
index 000000000000..2cf724a7d6f8
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frint64z_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+frint64z z0.d, p0/z, z0.d  // 01100100-00011101-11000000-00000000
+// CHECK-INST: frint64z z0.d, p0/z, z0.d
+// CHECK-ENCODING: [0x00,0xc0,0x1d,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641dc000 <unknown>
+
+frint64z z21.s, p5/z, z10.s  // 01100100-00011101-10010101-01010101
+// CHECK-INST: frint64z z21.s, p5/z, z10.s
+// CHECK-ENCODING: [0x55,0x95,0x1d,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641d9555 <unknown>
+
+frint64z z31.s, p7/z, z31.s  // 01100100-00011101-10011111-11111111
+// CHECK-INST: frint64z z31.s, p7/z, z31.s
+// CHECK-ENCODING: [0xff,0x9f,0x1d,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641d9fff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frinta_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frinta_z-diagnostics.s
new file mode 100644
index 000000000000..c3aa1f73b5ad
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frinta_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frinta     z31.b, p7/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frinta     z31.b, p7/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frinta     z31.h, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frinta     z31.h, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frinta     z31.s, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frinta     z31.s, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frinta     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frinta     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frinta     z31.h, p8/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frinta     z31.h, p8/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.h, p0/z, z7.h
+frinta z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frinta z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+frinta z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frinta z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frinta_z.s b/llvm/test/MC/AArch64/SVE2p2/frinta_z.s
new file mode 100644
index 000000000000..ef0a49e4f83b
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frinta_z.s
@@ -0,0 +1,34 @@
+
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+frinta  z0.h, p0/z, z0.h  // 01100100-01011001-10000000-00000000
+// CHECK-INST: frinta  z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0x80,0x59,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64598000 <unknown>
+
+frinta  z21.s, p5/z, z10.s  // 01100100-10011001-10010101-01010101
+// CHECK-INST: frinta  z21.s, p5/z, z10.s
+// CHECK-ENCODING: [0x55,0x95,0x99,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64999555 <unknown>
+
+frinta  z31.d, p7/z, z31.d  // 01100100-11011001-10011111-11111111
+// CHECK-INST: frinta  z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0x9f,0xd9,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64d99fff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frinti_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frinti_z-diagnostics.s
new file mode 100644
index 000000000000..8334d2d2f8ad
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frinti_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frinti     z31.b, p7/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frinti     z31.b, p7/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frinti     z31.h, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frinti     z31.h, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frinti     z31.s, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frinti     z31.s, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frinti     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frinti     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frinti     z31.h, p8/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frinti     z31.h, p8/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.h, p0/z, z7.h
+frinti z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frinti z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+frinti z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frinti z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frinti_z.s b/llvm/test/MC/AArch64/SVE2p2/frinti_z.s
new file mode 100644
index 000000000000..2cb15d4582de
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frinti_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+frinti  z0.h, p0/z, z0.h  // 01100100-01011001-11100000-00000000
+// CHECK-INST: frinti  z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xe0,0x59,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6459e000 <unknown>
+
+frinti  z21.s, p5/z, z10.s  // 01100100-10011001-11110101-01010101
+// CHECK-INST: frinti  z21.s, p5/z, z10.s
+// CHECK-ENCODING: [0x55,0xf5,0x99,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6499f555 <unknown>
+
+frinti  z31.d, p7/z, z31.d  // 01100100-11011001-11111111-11111111
+// CHECK-INST: frinti  z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xff,0xd9,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64d9ffff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frintm_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frintm_z-diagnostics.s
new file mode 100644
index 000000000000..fe72e6a7e264
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frintm_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frintm     z31.b, p7/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintm     z31.b, p7/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frintm     z31.h, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintm     z31.h, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frintm     z31.s, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintm     z31.s, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frintm     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintm     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frintm     z31.h, p8/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frintm     z31.h, p8/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.h, p0/z, z7.h
+frintm z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frintm z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+frintm z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frintm z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frintm_z.s b/llvm/test/MC/AArch64/SVE2p2/frintm_z.s
new file mode 100644
index 000000000000..60505bc114d0
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frintm_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+frintm  z0.h, p0/z, z0.h  // 01100100-01011000-11000000-00000000
+// CHECK-INST: frintm  z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xc0,0x58,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6458c000 <unknown>
+
+frintm  z23.s, p3/z, z13.s  // 01100100-10011000-11001101-10110111
+// CHECK-INST: frintm  z23.s, p3/z, z13.s
+// CHECK-ENCODING: [0xb7,0xcd,0x98,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6498cdb7 <unknown>
+
+frintm  z31.d, p7/z, z31.d  // 01100100-11011000-11011111-11111111
+// CHECK-INST: frintm  z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xdf,0xd8,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64d8dfff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frintn_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frintn_z-diagnostics.s
new file mode 100644
index 000000000000..07f54df87f2b
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frintn_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frintn     z31.b, p7/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintn     z31.b, p7/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frintn     z31.h, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintn     z31.h, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frintn     z31.s, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintn     z31.s, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frintn     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintn     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frintn     z31.h, p8/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frintn     z31.h, p8/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.h, p0/z, z7.h
+frintn z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frintn z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+frintn z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frintn z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frintn_z.s b/llvm/test/MC/AArch64/SVE2p2/frintn_z.s
new file mode 100644
index 000000000000..89461fc38c52
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frintn_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+frintn  z0.h, p0/z, z0.h  // 01100100-01011000-10000000-00000000
+// CHECK-INST: frintn  z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0x80,0x58,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64588000 <unknown>
+
+frintn  z23.s, p3/z, z13.s  // 01100100-10011000-10001101-10110111
+// CHECK-INST: frintn  z23.s, p3/z, z13.s
+// CHECK-ENCODING: [0xb7,0x8d,0x98,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64988db7 <unknown>
+
+frintn  z31.d, p7/z, z31.d  // 01100100-11011000-10011111-11111111
+// CHECK-INST: frintn  z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0x9f,0xd8,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64d89fff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frintp_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frintp_z-diagnostics.s
new file mode 100644
index 000000000000..5a9535b3affd
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frintp_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frintp     z31.b, p7/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintp     z31.b, p7/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frintp     z31.h, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintp     z31.h, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frintp     z31.s, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintp     z31.s, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frintp     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintp     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frintp     z31.h, p8/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frintp     z31.h, p8/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.h, p0/z, z7.h
+frintp z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frintp z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+frintp z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frintp z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frintp_z.s b/llvm/test/MC/AArch64/SVE2p2/frintp_z.s
new file mode 100644
index 000000000000..b728e4322c2c
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frintp_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+frintp  z0.h, p0/z, z0.h  // 01100100-01011000-10100000-00000000
+// CHECK-INST: frintp  z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xa0,0x58,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6458a000 <unknown>
+
+frintp  z23.s, p3/z, z13.s  // 01100100-10011000-10101101-10110111
+// CHECK-INST: frintp  z23.s, p3/z, z13.s
+// CHECK-ENCODING: [0xb7,0xad,0x98,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6498adb7 <unknown>
+
+frintp  z31.d, p7/z, z31.d  // 01100100-11011000-10111111-11111111
+// CHECK-INST: frintp  z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xd8,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64d8bfff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frintx_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frintx_z-diagnostics.s
new file mode 100644
index 000000000000..026ad7be5473
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frintx_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frintx     z31.b, p7/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintx     z31.b, p7/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frintx     z31.h, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintx     z31.h, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frintx     z31.s, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintx     z31.s, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frintx     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintx     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frintx     z31.h, p8/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frintx     z31.h, p8/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.h, p0/z, z7.h
+frintx z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frintx z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+frintx z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frintx z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frintx_z.s b/llvm/test/MC/AArch64/SVE2p2/frintx_z.s
new file mode 100644
index 000000000000..1cf77342bc25
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frintx_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+frintx  z0.h, p0/z, z0.h  // 01100100-01011001-11000000-00000000
+// CHECK-INST: frintx  z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xc0,0x59,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6459c000 <unknown>
+
+frintx  z23.s, p3/z, z13.s  // 01100100-10011001-11001101-10110111
+// CHECK-INST: frintx  z23.s, p3/z, z13.s
+// CHECK-ENCODING: [0xb7,0xcd,0x99,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6499cdb7 <unknown>
+
+frintx  z31.d, p7/z, z31.d  // 01100100-11011001-11011111-11111111
+// CHECK-INST: frintx  z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xdf,0xd9,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64d9dfff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frintz_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/frintz_z-diagnostics.s
new file mode 100644
index 000000000000..cb8632184e6b
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frintz_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+frintz     z31.b, p7/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintz     z31.b, p7/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frintz     z31.h, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintz     z31.h, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frintz     z31.s, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintz     z31.s, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+frintz     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: frintz     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+frintz     z31.h, p8/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: frintz     z31.h, p8/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.h, p0/z, z7.h
+frintz z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frintz z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+frintz z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: frintz z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/frintz_z.s b/llvm/test/MC/AArch64/SVE2p2/frintz_z.s
new file mode 100644
index 000000000000..ef28d565db05
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/frintz_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+frintz  z0.h, p0/z, z0.h  // 01100100-01011000-11100000-00000000
+// CHECK-INST: frintz  z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xe0,0x58,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6458e000 <unknown>
+
+frintz  z21.s, p5/z, z10.s  // 01100100-10011000-11110101-01010101
+// CHECK-INST: frintz  z21.s, p5/z, z10.s
+// CHECK-ENCODING: [0x55,0xf5,0x98,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 6498f555 <unknown>
+
+frintz  z31.d, p7/z, z31.d  // 01100100-11011000-11111111-11111111
+// CHECK-INST: frintz  z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xff,0xd8,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64d8ffff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/fsqrt_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/fsqrt_z-diagnostics.s
new file mode 100644
index 000000000000..8974b6b8adf0
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/fsqrt_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+fsqrt     z31.b, p7/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fsqrt     z31.b, p7/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fsqrt     z31.h, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fsqrt     z31.h, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fsqrt     z31.s, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fsqrt     z31.s, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fsqrt     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fsqrt     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+fsqrt     z31.h, p8/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: fsqrt     z31.h, p8/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.h, p0/z, z7.h
+fsqrt z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: fsqrt z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+fsqrt z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: fsqrt z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/fsqrt_z.s b/llvm/test/MC/AArch64/SVE2p2/fsqrt_z.s
new file mode 100644
index 000000000000..fe02dc8e16ce
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/fsqrt_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+fsqrt   z0.h, p0/z, z0.h  // 01100100-01011011-10100000-00000000
+// CHECK-INST: fsqrt   z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xa0,0x5b,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 645ba000 <unknown>
+
+fsqrt   z21.s, p5/z, z10.s  // 01100100-10011011-10110101-01010101
+// CHECK-INST: fsqrt   z21.s, p5/z, z10.s
+// CHECK-ENCODING: [0x55,0xb5,0x9b,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 649bb555 <unknown>
+
+fsqrt   z31.d, p7/z, z31.d  // 01100100-11011011-10111111-11111111
+// CHECK-INST: fsqrt   z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xdb,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64dbbfff <unknown>
\ No newline at end of file
-- 
GitLab


From 4bce21480fc52bf583b6bfe5072f9eb70c21f6e4 Mon Sep 17 00:00:00 2001
From: Dimitry Andric <dimitry@andric.com>
Date: Thu, 24 Oct 2024 19:30:38 +0200
Subject: [PATCH 081/693] Ensure !NDEBUG with LLVM_ENABLE_ABI_BREAKING_CHECKS
 does not segfault (#113588)

In SelectionDAG, `TargetTransformInfo::hasBranchDivergence()` can be
called when both `NDEBUG` and `LLVM_ENABLE_ABI_BREAKING_CHECKS` are
enabled. In that case, the class member `TTI` is still initialized to
`nullptr`, causing a segfault.

Fix this by ensuring that all the calls to `hasBranchDivergence` and
`VerifyDAGDivergence` only occur when `NDEBUG` is disabled, and
`LLVM_ENABLE_ABI_BREAKING_CHECKS` is enabled.
---
 llvm/include/llvm/CodeGen/SelectionDAG.h       |  2 +-
 llvm/include/llvm/CodeGen/SelectionDAGISel.h   |  2 +-
 llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp |  2 +-
 .../CodeGen/SelectionDAG/SelectionDAGISel.cpp  | 18 +++++++++---------
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 12ff36c89e33..e82bdb690616 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -584,7 +584,7 @@ public:
     return Root;
   }
 
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
   void VerifyDAGDivergence();
 #endif
 
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index f6191c6fdb7f..32b2ea48179f 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -56,7 +56,7 @@ public:
   AssumptionCache *AC = nullptr;
   GCFunctionInfo *GFI = nullptr;
   SSPLayoutInfo *SP = nullptr;
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
   TargetTransformInfo *TTI = nullptr;
 #endif
   CodeGenOptLevel OptLevel;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 55cebc28e492..0770355ec18c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -11744,7 +11744,7 @@ void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
   }
 }
 
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
 void SelectionDAG::VerifyDAGDivergence() {
   std::vector<SDNode *> TopoOrder;
   CreateTopologicalOrder(TopoOrder);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 9bd894dd952b..981ab18b59c1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -940,7 +940,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
                    << "'\n";
             CurDAG->dump());
 
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
   if (TTI->hasBranchDivergence())
     CurDAG->VerifyDAGDivergence();
 #endif
@@ -960,7 +960,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
                    << "'\n";
             CurDAG->dump());
 
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
   if (TTI->hasBranchDivergence())
     CurDAG->VerifyDAGDivergence();
 #endif
@@ -982,7 +982,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
                    << "'\n";
             CurDAG->dump());
 
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
   if (TTI->hasBranchDivergence())
     CurDAG->VerifyDAGDivergence();
 #endif
@@ -1006,7 +1006,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
                      << "'\n";
               CurDAG->dump());
 
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
     if (TTI->hasBranchDivergence())
       CurDAG->VerifyDAGDivergence();
 #endif
@@ -1024,7 +1024,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
                      << "'\n";
               CurDAG->dump());
 
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
     if (TTI->hasBranchDivergence())
       CurDAG->VerifyDAGDivergence();
 #endif
@@ -1040,7 +1040,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
                      << "'\n";
               CurDAG->dump());
 
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
     if (TTI->hasBranchDivergence())
       CurDAG->VerifyDAGDivergence();
 #endif
@@ -1060,7 +1060,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
                      << "'\n";
               CurDAG->dump());
 
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
     if (TTI->hasBranchDivergence())
       CurDAG->VerifyDAGDivergence();
 #endif
@@ -1080,7 +1080,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
                    << "'\n";
             CurDAG->dump());
 
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
   if (TTI->hasBranchDivergence())
     CurDAG->VerifyDAGDivergence();
 #endif
@@ -1100,7 +1100,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
                    << "'\n";
             CurDAG->dump());
 
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
   if (TTI->hasBranchDivergence())
     CurDAG->VerifyDAGDivergence();
 #endif
-- 
GitLab


From af872d5473e7e1132e0ec8e146d28af239196c53 Mon Sep 17 00:00:00 2001
From: Helena Kotas <hekotas@microsoft.com>
Date: Thu, 24 Oct 2024 10:34:04 -0700
Subject: [PATCH 082/693] [HLSL] Add RWStructuredBuffer definition to
 HLSLExternalSemaSource (#113477)

Add tests for RWStructuredBuffer class definition. Use shared test files
for all structured buffers' constructor and subscript tests. Keep AST
and element-type tests separate for each buffer type because they longer
and failures would be harder to match.

Fixes #112775
---
 clang/lib/Sema/HLSLExternalSemaSource.cpp     | 11 +++
 .../test/AST/HLSL/RWStructuredBuffer-AST.hlsl | 64 +++++++++++++++++
 clang/test/AST/HLSL/StructuredBuffer-AST.hlsl |  2 +-
 .../RWStructuredBuffer-elementtype.hlsl       | 70 +++++++++++++++++++
 .../StructuredBuffer-elementtype.hlsl         | 12 ++--
 ...sl => StructuredBuffers-constructors.hlsl} | 17 +++--
 ...hlsl => StructuredBuffers-subscripts.hlsl} |  6 +-
 7 files changed, 166 insertions(+), 16 deletions(-)
 create mode 100644 clang/test/AST/HLSL/RWStructuredBuffer-AST.hlsl
 create mode 100644 clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
 rename clang/test/CodeGenHLSL/builtins/{StructuredBuffer-constructor.hlsl => StructuredBuffers-constructors.hlsl} (57%)
 rename clang/test/CodeGenHLSL/builtins/{StructuredBuffer-subscript.hlsl => StructuredBuffers-subscripts.hlsl} (64%)

diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp
index 50bb307982d3..ce8564429b38 100644
--- a/clang/lib/Sema/HLSLExternalSemaSource.cpp
+++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp
@@ -519,6 +519,17 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
         .addArraySubscriptOperators()
         .completeDefinition();
   });
+
+  Decl = BuiltinTypeDeclBuilder(*SemaPtr, HLSLNamespace, "RWStructuredBuffer")
+             .addSimpleTemplateParams(*SemaPtr, {"element_type"})
+             .Record;
+  onCompletion(Decl, [this](CXXRecordDecl *Decl) {
+    setupBufferType(Decl, *SemaPtr, ResourceClass::UAV,
+                    ResourceKind::TypedBuffer, /*IsROV=*/false,
+                    /*RawBuffer=*/true)
+        .addArraySubscriptOperators()
+        .completeDefinition();
+  });
 }
 
 void HLSLExternalSemaSource::onCompletion(CXXRecordDecl *Record,
diff --git a/clang/test/AST/HLSL/RWStructuredBuffer-AST.hlsl b/clang/test/AST/HLSL/RWStructuredBuffer-AST.hlsl
new file mode 100644
index 000000000000..f95d74b30acd
--- /dev/null
+++ b/clang/test/AST/HLSL/RWStructuredBuffer-AST.hlsl
@@ -0,0 +1,64 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY %s | FileCheck -check-prefix=EMPTY %s
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump %s | FileCheck %s
+
+
+// This test tests two different AST generations. The "EMPTY" test mode verifies
+// the AST generated by forward declaration of the HLSL types which happens on
+// initializing the HLSL external AST with an AST Context.
+
+// The non-empty mode has a use that requires the RWStructuredBuffer type be complete,
+// which results in the AST being populated by the external AST source. That
+// case covers the full implementation of the template declaration and the
+// instantiated specialization.
+
+// EMPTY: ClassTemplateDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit RWStructuredBuffer
+// EMPTY-NEXT: TemplateTypeParmDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> class depth 0 index 0 element_type
+// EMPTY-NEXT: CXXRecordDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit <undeserialized declarations> class RWStructuredBuffer
+// EMPTY-NEXT: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
+
+// There should be no more occurrances of RWStructuredBuffer
+// EMPTY-NOT: {{[^[:alnum:]]}}RWStructuredBuffer
+
+#ifndef EMPTY
+
+RWStructuredBuffer<int> Buffer;
+
+#endif
+
+// CHECK: ClassTemplateDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit RWStructuredBuffer
+// CHECK-NEXT: TemplateTypeParmDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> class depth 0 index 0 element_type
+// CHECK-NEXT: CXXRecordDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit class RWStructuredBuffer definition
+
+// CHECK: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
+// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit h '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
+// CHECK-NEXT: HLSLResourceAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit TypedBuffer
+
+// CHECK: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'element_type &const (unsigned int) const'
+// CHECK-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Idx 'unsigned int'
+// CHECK-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' lvalue .e 0x{{[0-9A-Fa-f]+}}
+// CHECK-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'const RWStructuredBuffer<element_type>' lvalue implicit this
+// CHECK-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
+
+// CHECK-NEXT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'element_type &(unsigned int)'
+// CHECK-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Idx 'unsigned int'
+// CHECK-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' lvalue .e 0x{{[0-9A-Fa-f]+}}
+// CHECK-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'RWStructuredBuffer<element_type>' lvalue implicit this
+// CHECK-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
+
+// CHECK: ClassTemplateSpecializationDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> class RWStructuredBuffer definition
+
+// CHECK: TemplateArgument type 'int'
+// CHECK-NEXT: BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
+// CHECK-NEXT: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
+// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit h '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-SAME{LITERAL}: [[hlsl::contained_type(int)]]
+// CHECK-NEXT: HLSLResourceAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit TypedBuffer
diff --git a/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl b/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl
index 2c12e993270c..6c39be8ba519 100644
--- a/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl
+++ b/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl
@@ -17,7 +17,7 @@
 // EMPTY-NEXT: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
 
 // There should be no more occurrances of StructuredBuffer
-// EMPTY-NOT: StructuredBuffer
+// EMPTY-NOT: {{[^[:alnum:]]}}StructuredBuffer
 
 #ifndef EMPTY
 
diff --git a/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
new file mode 100644
index 000000000000..727f416cde57
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
@@ -0,0 +1,70 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s
+
+// NOTE: The number in type name and whether the struct is packed or not will mostly
+// likely change once subscript operators are properly implemented (llvm/llvm-project#95956) 
+// and theinterim field of the contained type is removed.
+
+// CHECK: %"class.hlsl::RWStructuredBuffer" = type <{ target("dx.RawBuffer", i16, 1, 0)
+// CHECK: %"class.hlsl::RWStructuredBuffer.0" = type <{ target("dx.RawBuffer", i16, 1, 0)
+// CHECK: %"class.hlsl::RWStructuredBuffer.2" = type { target("dx.RawBuffer", i32, 1, 0)
+// CHECK: %"class.hlsl::RWStructuredBuffer.3" = type { target("dx.RawBuffer", i32, 1, 0)
+// CHECK: %"class.hlsl::RWStructuredBuffer.4" = type { target("dx.RawBuffer", i64, 1, 0)
+// CHECK: %"class.hlsl::RWStructuredBuffer.5" = type { target("dx.RawBuffer", i64, 1, 0)
+// CHECK: %"class.hlsl::RWStructuredBuffer.6" = type <{ target("dx.RawBuffer", half, 1, 0) 
+// CHECK: %"class.hlsl::RWStructuredBuffer.8" = type { target("dx.RawBuffer", float, 1, 0)
+// CHECK: %"class.hlsl::RWStructuredBuffer.9" = type { target("dx.RawBuffer", double, 1, 0)
+// CHECK: %"class.hlsl::RWStructuredBuffer.10" = type { target("dx.RawBuffer", <4 x i16>, 1, 0)
+// CHECK: %"class.hlsl::RWStructuredBuffer.11" = type { target("dx.RawBuffer", <3 x i32>, 1, 0)
+// CHECK: %"class.hlsl::RWStructuredBuffer.12" = type { target("dx.RawBuffer", <2 x half>, 1, 0)
+// CHECK: %"class.hlsl::RWStructuredBuffer.13" = type { target("dx.RawBuffer", <3 x float>, 1, 0)
+
+RWStructuredBuffer<int16_t> BufI16;
+RWStructuredBuffer<uint16_t> BufU16;
+RWStructuredBuffer<int> BufI32;
+RWStructuredBuffer<uint> BufU32;
+RWStructuredBuffer<int64_t> BufI64;
+RWStructuredBuffer<uint64_t> BufU64;
+RWStructuredBuffer<half> BufF16;
+RWStructuredBuffer<float> BufF32;
+RWStructuredBuffer<double> BufF64;
+RWStructuredBuffer< vector<int16_t, 4> > BufI16x4;
+RWStructuredBuffer< vector<uint, 3> > BufU32x3;
+RWStructuredBuffer<half2> BufF16x2;
+RWStructuredBuffer<float3> BufF32x3;
+// TODO: RWStructuredBuffer<snorm half> BufSNormF16;
+// TODO: RWStructuredBuffer<unorm half> BufUNormF16;
+// TODO: RWStructuredBuffer<snorm float> BufSNormF32;
+// TODO: RWStructuredBuffer<unorm float> BufUNormF32;
+// TODO: RWStructuredBuffer<snorm double> BufSNormF64;
+// TODO: RWStructuredBuffer<unorm double> BufUNormF64;
+
+[numthreads(1,1,1)]
+void main(int GI : SV_GroupIndex) {
+  BufI16[GI] = 0;
+  BufU16[GI] = 0;
+  BufI32[GI] = 0;
+  BufU32[GI] = 0;
+  BufI64[GI] = 0;
+  BufU64[GI] = 0;
+  BufF16[GI] = 0;
+  BufF32[GI] = 0;
+  BufF64[GI] = 0;
+  BufI16x4[GI] = 0;
+  BufU32x3[GI] = 0;
+  BufF16x2[GI] = 0;
+  BufF32x3[GI] = 0;
+}
+
+// CHECK: !{{[0-9]+}} = !{ptr @BufI16, i32 10, i32 2,
+// CHECK: !{{[0-9]+}} = !{ptr @BufU16, i32 10, i32 3,
+// CHECK: !{{[0-9]+}} = !{ptr @BufI32, i32 10, i32 4,
+// CHECK: !{{[0-9]+}} = !{ptr @BufU32, i32 10, i32 5,
+// CHECK: !{{[0-9]+}} = !{ptr @BufI64, i32 10, i32 6,
+// CHECK: !{{[0-9]+}} = !{ptr @BufU64, i32 10, i32 7,
+// CHECK: !{{[0-9]+}} = !{ptr @BufF16, i32 10, i32 8,
+// CHECK: !{{[0-9]+}} = !{ptr @BufF32, i32 10, i32 9,
+// CHECK: !{{[0-9]+}} = !{ptr @BufF64, i32 10, i32 10,
+// CHECK: !{{[0-9]+}} = !{ptr @BufI16x4, i32 10, i32 2,
+// CHECK: !{{[0-9]+}} = !{ptr @BufU32x3, i32 10, i32 5,
+// CHECK: !{{[0-9]+}} = !{ptr @BufF16x2, i32 10, i32 8,
+// CHECK: !{{[0-9]+}} = !{ptr @BufF32x3, i32 10, i32 9,
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
index d22958f902b5..4c30119498ff 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
@@ -31,12 +31,12 @@ StructuredBuffer< vector<int16_t, 4> > BufI16x4;
 StructuredBuffer< vector<uint, 3> > BufU32x3;
 StructuredBuffer<half2> BufF16x2;
 StructuredBuffer<float3> BufF32x3;
-// TODO: StructuredBuffer<snorm half> BufSNormF16; -> 11
-// TODO: StructuredBuffer<unorm half> BufUNormF16; -> 12
-// TODO: StructuredBuffer<snorm float> BufSNormF32; -> 13
-// TODO: StructuredBuffer<unorm float> BufUNormF32; -> 14
-// TODO: StructuredBuffer<snorm double> BufSNormF64; -> 15
-// TODO: StructuredBuffer<unorm double> BufUNormF64; -> 16
+// TODO: StructuredBuffer<snorm half> BufSNormF16;
+// TODO: StructuredBuffer<unorm half> BufUNormF16;
+// TODO: StructuredBuffer<snorm float> BufSNormF32;
+// TODO: StructuredBuffer<unorm float> BufUNormF32;
+// TODO: StructuredBuffer<snorm double> BufSNormF64;
+// TODO: StructuredBuffer<unorm double> BufUNormF64;
 
 [numthreads(1,1,1)]
 void main(int GI : SV_GroupIndex) {
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-constructor.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-constructors.hlsl
similarity index 57%
rename from clang/test/CodeGenHLSL/builtins/StructuredBuffer-constructor.hlsl
rename to clang/test/CodeGenHLSL/builtins/StructuredBuffers-constructors.hlsl
index 8189aed89d74..4f2f8d079074 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-constructor.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-constructors.hlsl
@@ -4,21 +4,30 @@
 // NOTE: SPIRV codegen for resource types is not yet implemented
 
 StructuredBuffer<float> Buf : register(t10);
+RWStructuredBuffer<float> Buf2 : register(u5, space1);
 
 // CHECK: %"class.hlsl::StructuredBuffer" = type { target("dx.RawBuffer", float, 0, 0), float }
+// CHECK: %"class.hlsl::RWStructuredBuffer" = type { target("dx.RawBuffer", float, 1, 0), float }
+
 // CHECK: @Buf = global %"class.hlsl::StructuredBuffer" zeroinitializer, align 4
+// CHECK: @Buf2 = global %"class.hlsl::RWStructuredBuffer" zeroinitializer, align 4
 
 // CHECK: define linkonce_odr void @_ZN4hlsl16StructuredBufferIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(8) %this)
 // CHECK-NEXT: entry:
-
-// CHECK: define internal void @_GLOBAL__sub_I_StructuredBuffer_constructor.hlsl()
+// CHECK: define linkonce_odr void @_ZN4hlsl18RWStructuredBufferIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(8) %this)
 // CHECK-NEXT: entry:
-// CHECK-NEXT: call void @__cxx_global_var_init()
-// CHECK-NEXT: call void @_init_resource_bindings()
+
+// CHECK: define internal void @_GLOBAL__sub_I_StructuredBuffers_constructors.hlsl()
+// CHECK: entry:
+// CHECK: call void @_init_resource_bindings()
 
 // CHECK: define internal void @_init_resource_bindings() {
 // CHECK-NEXT: entry:
 // CHECK-DXIL-NEXT: %Buf_h = call target("dx.RawBuffer", float, 0, 0) @llvm.dx.handle.fromBinding.tdx.RawBuffer_f32_0_0t(i32 0, i32 10, i32 1, i32 0, i1 false)
 // CHECK-DXIL-NEXT: store target("dx.RawBuffer", float, 0, 0) %Buf_h, ptr @Buf, align 4
+// CHECK-DXIL-NEXT: %Buf2_h = call target("dx.RawBuffer", float, 1, 0) @llvm.dx.handle.fromBinding.tdx.RawBuffer_f32_1_0t(i32 1, i32 5, i32 1, i32 0, i1 false)
+// CHECK-DXIL-NEXT: store target("dx.RawBuffer", float, 1, 0) %Buf2_h, ptr @Buf2, align 4
 // CHECK-SPIRV-NEXT: %Buf_h = call target("dx.RawBuffer", float, 0, 0) @llvm.spv.handle.fromBinding.tdx.RawBuffer_f32_0_0t(i32 0, i32 10, i32 1, i32 0, i1 false)
 // CHECK-SPIRV-NEXT: store target("dx.RawBuffer", float, 0, 0) %Buf_h, ptr @Buf", align 4
+// CHECK-SPIRV-NEXT: %Buf2_h = call target("dx.RawBuffer", float, 1, 0) @llvm.spv.handle.fromBinding.tdx.RawBuffer_f32_1_0t(i32 1, i32 5, i32 1, i32 0, i1 false)
+// CHECK-SPIRV-NEXT: store target("dx.RawBuffer", float, 1, 0) %Buf2_h, ptr @Buf2", align 4
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-subscript.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-subscripts.hlsl
similarity index 64%
rename from clang/test/CodeGenHLSL/builtins/StructuredBuffer-subscript.hlsl
rename to clang/test/CodeGenHLSL/builtins/StructuredBuffers-subscripts.hlsl
index db75d534f854..b73b743c9b3f 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-subscript.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-subscripts.hlsl
@@ -1,11 +1,7 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -emit-llvm -o - -O0 %s | FileCheck %s
 
-// FIXME: This tests will fail once StructuredBuffer subscript operator is properly
-// implemented (llvm-project/llvm#112977). StructuredBuffer is a read-only resource
-// and the result of a subscript operator cannot be assigned to.
-
 StructuredBuffer<int> In;
-StructuredBuffer<int> Out;
+RWStructuredBuffer<int> Out;
 
 [numthreads(1,1,1)]
 void main(unsigned GI : SV_GroupIndex) {
-- 
GitLab


From 1fe64fe04687e5c95c33be9abac921cdd2a8e2ff Mon Sep 17 00:00:00 2001
From: Thomas Fransham <tfransham@gmail.com>
Date: Thu, 24 Oct 2024 18:41:34 +0100
Subject: [PATCH 083/693] [ORC] Add visibility macros to functions needed by
 lli and jitlink (#113271)

Annotating these symbols will fix missing symbols errors for lli and
llvm-jitlink when when the LLVM is built as shared library on windows
with explicit symbol visibility macros enabled.

This is part of the work to enable LLVM_BUILD_LLVM_DYLIB and plugins on
window.
---
 .../llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h  | 5 +++--
 .../llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h | 7 ++++---
 .../ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h     | 7 ++++---
 .../ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h   | 5 +++--
 4 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h
index 9f91a64e95ce..1563d1346898 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h
@@ -14,6 +14,7 @@
 #define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERGDB_H
 
 #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
 // Keep in sync with gdb/gdb/jit.h
@@ -42,10 +43,10 @@ struct jit_descriptor {
 };
 }
 
-extern "C" llvm::orc::shared::CWrapperFunctionResult
+extern "C" LLVM_ABI llvm::orc::shared::CWrapperFunctionResult
 llvm_orc_registerJITLoaderGDBWrapper(const char *Data, uint64_t Size);
 
-extern "C" llvm::orc::shared::CWrapperFunctionResult
+extern "C" LLVM_ABI llvm::orc::shared::CWrapperFunctionResult
 llvm_orc_registerJITLoaderGDBAllocAction(const char *Data, size_t Size);
 
 #endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERGDB_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h
index 1d8e33f8013b..aa9951a23b99 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h
@@ -14,15 +14,16 @@
 #define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERPERF_H
 
 #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
-extern "C" llvm::orc::shared::CWrapperFunctionResult
+extern "C" LLVM_ABI llvm::orc::shared::CWrapperFunctionResult
 llvm_orc_registerJITLoaderPerfImpl(const char *Data, uint64_t Size);
 
-extern "C" llvm::orc::shared::CWrapperFunctionResult
+extern "C" LLVM_ABI llvm::orc::shared::CWrapperFunctionResult
 llvm_orc_registerJITLoaderPerfStart(const char *Data, uint64_t Size);
 
-extern "C" llvm::orc::shared::CWrapperFunctionResult
+extern "C" LLVM_ABI llvm::orc::shared::CWrapperFunctionResult
 llvm_orc_registerJITLoaderPerfEnd(const char *Data, uint64_t Size);
 
 #endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERPERF_H
\ No newline at end of file
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h
index afb7df592faf..63a0d7325d5a 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h
@@ -15,15 +15,16 @@
 #define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERVTUNE_H
 
 #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
-extern "C" llvm::orc::shared::CWrapperFunctionResult
+extern "C" LLVM_ABI llvm::orc::shared::CWrapperFunctionResult
 llvm_orc_registerVTuneImpl(const char *Data, uint64_t Size);
 
-extern "C" llvm::orc::shared::CWrapperFunctionResult
+extern "C" LLVM_ABI llvm::orc::shared::CWrapperFunctionResult
 llvm_orc_unregisterVTuneImpl(const char *Data, uint64_t Size);
 
-extern "C" llvm::orc::shared::CWrapperFunctionResult
+extern "C" LLVM_ABI llvm::orc::shared::CWrapperFunctionResult
 llvm_orc_test_registerVTuneImpl(const char *Data, uint64_t Size);
 
 #endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERVTUNE_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h
index 04790afa6e31..d375f6034616 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h
@@ -17,6 +17,7 @@
 #define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_REGISTEREHFRAMES_H
 
 #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -33,10 +34,10 @@ Error deregisterEHFrameSection(const void *EHFrameSectionAddr,
 } // end namespace orc
 } // end namespace llvm
 
-extern "C" llvm::orc::shared::CWrapperFunctionResult
+extern "C" LLVM_ABI llvm::orc::shared::CWrapperFunctionResult
 llvm_orc_registerEHFrameSectionWrapper(const char *Data, uint64_t Size);
 
-extern "C" llvm::orc::shared::CWrapperFunctionResult
+extern "C" LLVM_ABI llvm::orc::shared::CWrapperFunctionResult
 llvm_orc_deregisterEHFrameSectionWrapper(const char *Data, uint64_t Size);
 
 #endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_REGISTEREHFRAMES_H
-- 
GitLab


From 6d8d7a3acdebf5979b93629a2641ba1bf06c574b Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Thu, 24 Oct 2024 10:57:02 -0700
Subject: [PATCH 084/693] [gn] disable building libc++ shared lib

It's currently broken by #91651. Stop buildling libcxx src files
until that is sorted out.
---
 llvm/utils/gn/secondary/BUILD.gn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/BUILD.gn b/llvm/utils/gn/secondary/BUILD.gn
index 7f6b4cb43239..5590a5ba195e 100644
--- a/llvm/utils/gn/secondary/BUILD.gn
+++ b/llvm/utils/gn/secondary/BUILD.gn
@@ -15,7 +15,8 @@ group("default") {
     "//lldb/test",
     "//llvm/test",
   ]
-  if (current_os == "linux") {
+  # FIXME: Reenable once libc hand-in-hand is figured out.
+  if (false && current_os == "linux") {
     deps += [
       "//libcxx",
       "//libcxxabi",
-- 
GitLab


From 564fd62aedfde6358baa1776a2de975b45bc7778 Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer@google.com>
Date: Thu, 24 Oct 2024 11:05:39 -0700
Subject: [PATCH 085/693] [FlowSensitive] Allow to dump nested
 RecordStorageLocation (#112457)

We have an internal analysis that uses them, and the HTML dump would
fail on the assertion.
---
 clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp b/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp
index a36cb41a63df..557df2188379 100644
--- a/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp
+++ b/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp
@@ -133,8 +133,7 @@ public:
       for (const auto &Child : RLoc->children())
         JOS.attributeObject("f:" + Child.first->getNameAsString(), [&] {
           if (Child.second)
-            if (Value *Val = Env.getValue(*Child.second))
-              dump(*Val);
+            dump(*Child.second);
         });
 
       for (const auto &SyntheticField : RLoc->synthetic_fields())
-- 
GitLab


From 99a52f95018ddebc93caf26498c00cd9cc36c1d6 Mon Sep 17 00:00:00 2001
From: James Westwood <james.westwood@arm.com>
Date: Thu, 24 Oct 2024 19:53:03 +0100
Subject: [PATCH 086/693] [AArch64] Add support for Armv9.6-A FEAT_SPE_EXC and
 FEAT_TRBE_EXC (#113463)

Add support for the following Armv9.6-A architecture extensions:
  * FEAT_SPE_EXC  - Statistical Profiling Extension profiling exceptions
  * FEAT_TRBE_EXC - Trace Buffer Management Events profiling exceptions

as documented here:

https://developer.arm.com/documentation/109697/2024_09/Feature-descriptions/The-Armv9-6-architecture-extension

Co-authored-by: Jonathan Thackray <jonathan.thackray@arm.com>
---
 .../Target/AArch64/AArch64SystemOperands.td   | 13 +++++++++++++
 .../AArch64/armv9.6a-statistical-profiling.s  | 19 +++++++++++++++++++
 .../test/MC/AArch64/armv9.6a-trbe-exception.s | 15 +++++++++++++++
 .../armv9.6a-statistical-profiling.txt        | 15 +++++++++++++++
 .../AArch64/armv9.6a-trbe-exception.txt       | 15 +++++++++++++++
 5 files changed, 77 insertions(+)
 create mode 100644 llvm/test/MC/AArch64/armv9.6a-statistical-profiling.s
 create mode 100644 llvm/test/MC/AArch64/armv9.6a-trbe-exception.s
 create mode 100644 llvm/test/MC/Disassembler/AArch64/armv9.6a-statistical-profiling.txt
 create mode 100644 llvm/test/MC/Disassembler/AArch64/armv9.6a-trbe-exception.txt

diff --git a/llvm/lib/Target/AArch64/AArch64SystemOperands.td b/llvm/lib/Target/AArch64/AArch64SystemOperands.td
index 1d2bb1d73234..052cdf9e9392 100644
--- a/llvm/lib/Target/AArch64/AArch64SystemOperands.td
+++ b/llvm/lib/Target/AArch64/AArch64SystemOperands.td
@@ -2048,3 +2048,16 @@ let Requires = [{ {AArch64::FeaturePCDPHINT} }] in {
 
 // v9.6a Realm management extension enhancements
 def : RWSysReg<"GPCBW_EL3", 0b11, 0b110, 0b0010, 0b0001, 0b101>;
+
+// v9.6a Statistical Profiling Extension exception registers (FEAT_SPE_EXC)
+//                                Op0   Op1    CRn     CRm     Op2
+def : RWSysReg<"PMBMAR_EL1",      0b11, 0b000, 0b1001, 0b1010, 0b101>;
+def : RWSysReg<"PMBSR_EL12",      0b11, 0b101, 0b1001, 0b1010, 0b011>;
+def : RWSysReg<"PMBSR_EL2",       0b11, 0b100, 0b1001, 0b1010, 0b011>;
+def : RWSysReg<"PMBSR_EL3",       0b11, 0b110, 0b1001, 0b1010, 0b011>;
+
+// v9.6a Trace Buffer Management Events exception registers (FEAT_TRBE_EXC)
+//                                Op0   Op1    CRn     CRm     Op2
+def : RWSysReg<"TRBSR_EL12",      0b11, 0b101, 0b1001, 0b1011, 0b011>;
+def : RWSysReg<"TRBSR_EL2",       0b11, 0b100, 0b1001, 0b1011, 0b011>;
+def : RWSysReg<"TRBSR_EL3",       0b11, 0b110, 0b1001, 0b1011, 0b011>;
diff --git a/llvm/test/MC/AArch64/armv9.6a-statistical-profiling.s b/llvm/test/MC/AArch64/armv9.6a-statistical-profiling.s
new file mode 100644
index 000000000000..2314c414d4d9
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv9.6a-statistical-profiling.s
@@ -0,0 +1,19 @@
+// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding     < %s 2> %t | FileCheck %s
+
+  msr pmbmar_el1, x0
+  msr pmbsr_el12, x0
+  msr pmbsr_el2, x0
+  msr pmbsr_el3, x0
+// CHECK:     msr PMBMAR_EL1, x0          // encoding: [0xa0,0x9a,0x18,0xd5]
+// CHECK:     msr PMBSR_EL12, x0          // encoding: [0x60,0x9a,0x1d,0xd5]
+// CHECK:     msr PMBSR_EL2, x0           // encoding: [0x60,0x9a,0x1c,0xd5]
+// CHECK:     msr PMBSR_EL3, x0           // encoding: [0x60,0x9a,0x1e,0xd5]
+
+  mrs x0, pmbmar_el1
+  mrs x0, pmbsr_el12
+  mrs x0, pmbsr_el2
+  mrs x0, pmbsr_el3
+// CHECK:    mrs x0, PMBMAR_EL1          // encoding: [0xa0,0x9a,0x38,0xd5]
+// CHECK:    mrs x0, PMBSR_EL12          // encoding: [0x60,0x9a,0x3d,0xd5]
+// CHECK:    mrs x0, PMBSR_EL2           // encoding: [0x60,0x9a,0x3c,0xd5]
+// CHECK:    mrs x0, PMBSR_EL3           // encoding: [0x60,0x9a,0x3e,0xd5]
diff --git a/llvm/test/MC/AArch64/armv9.6a-trbe-exception.s b/llvm/test/MC/AArch64/armv9.6a-trbe-exception.s
new file mode 100644
index 000000000000..a8ba7c442e4c
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv9.6a-trbe-exception.s
@@ -0,0 +1,15 @@
+// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding      < %s 2> %t | FileCheck %s
+
+  msr trbsr_el12, x0
+  msr trbsr_el2, x0
+  msr trbsr_el3, x0
+// CHECK:     msr TRBSR_EL12, x0          // encoding: [0x60,0x9b,0x1d,0xd5]
+// CHECK:     msr TRBSR_EL2, x0           // encoding: [0x60,0x9b,0x1c,0xd5]
+// CHECK:     msr TRBSR_EL3, x0           // encoding: [0x60,0x9b,0x1e,0xd5]
+
+  mrs x0, trbsr_el12
+  mrs x0, trbsr_el2
+  mrs x0, trbsr_el3
+// CHECK:    mrs x0, TRBSR_EL12          // encoding: [0x60,0x9b,0x3d,0xd5]
+// CHECK:    mrs x0, TRBSR_EL2           // encoding: [0x60,0x9b,0x3c,0xd5]
+// CHECK:    mrs x0, TRBSR_EL3           // encoding: [0x60,0x9b,0x3e,0xd5]
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.6a-statistical-profiling.txt b/llvm/test/MC/Disassembler/AArch64/armv9.6a-statistical-profiling.txt
new file mode 100644
index 000000000000..446e2f0eb05c
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AArch64/armv9.6a-statistical-profiling.txt
@@ -0,0 +1,15 @@
+# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck %s
+
+[0x60,0x9a,0x1d,0xd5]
+# CHECK: msr PMBSR_EL12, x0
+[0x60,0x9a,0x1c,0xd5]
+# CHECK: msr PMBSR_EL2, x0
+[0x60,0x9a,0x1e,0xd5]
+# CHECK: msr PMBSR_EL3, x0
+
+[0x60,0x9a,0x3d,0xd5]
+# CHECK: mrs x0, PMBSR_EL12
+[0x60,0x9a,0x3c,0xd5]
+# CHECK: mrs x0, PMBSR_EL2
+[0x60,0x9a,0x3e,0xd5]
+# CHECK: mrs x0, PMBSR_EL3
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.6a-trbe-exception.txt b/llvm/test/MC/Disassembler/AArch64/armv9.6a-trbe-exception.txt
new file mode 100644
index 000000000000..4b39e107538d
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AArch64/armv9.6a-trbe-exception.txt
@@ -0,0 +1,15 @@
+# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck %s
+
+[0x60,0x9b,0x1d,0xd5]
+# CHECK: msr TRBSR_EL12, x0
+[0x60,0x9b,0x1c,0xd5]
+# CHECK: msr TRBSR_EL2, x0
+[0x60,0x9b,0x1e,0xd5]
+# CHECK: msr TRBSR_EL3, x0
+
+[0x60,0x9b,0x3d,0xd5]
+# CHECK: mrs x0, TRBSR_EL12
+[0x60,0x9b,0x3c,0xd5]
+# CHECK: mrs x0, TRBSR_EL2
+[0x60,0x9b,0x3e,0xd5]
+# CHECK: mrs x0, TRBSR_EL3
-- 
GitLab


From 789fdd536d0446b9effa034ea37f3b56461d8500 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke@amd.com>
Date: Fri, 25 Oct 2024 00:24:44 +0530
Subject: [PATCH 087/693] [CodeGen][NewPM] Make MFProperties methods const NFC
 (#113304)

Makes them congruent with the legacy PM methods.
---
 llvm/include/llvm/CodeGen/MachineCSE.h                | 2 +-
 llvm/include/llvm/CodeGen/MachinePassManager.h        | 4 ++--
 llvm/include/llvm/CodeGen/RegAllocFast.h              | 6 +++---
 llvm/include/llvm/CodeGen/TwoAddressInstructionPass.h | 2 +-
 llvm/lib/Target/AMDGPU/GCNDPPCombine.h                | 2 +-
 llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.h         | 2 +-
 llvm/lib/Target/AMDGPU/SILowerSGPRSpills.h            | 2 +-
 7 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/MachineCSE.h b/llvm/include/llvm/CodeGen/MachineCSE.h
index f83c25bf3912..16a313508547 100644
--- a/llvm/include/llvm/CodeGen/MachineCSE.h
+++ b/llvm/include/llvm/CodeGen/MachineCSE.h
@@ -18,7 +18,7 @@ public:
   PreservedAnalyses run(MachineFunction &MF,
                         MachineFunctionAnalysisManager &MFAM);
 
-  MachineFunctionProperties getRequiredProperties() {
+  MachineFunctionProperties getRequiredProperties() const {
     return MachineFunctionProperties().set(
         MachineFunctionProperties::Property::IsSSA);
   }
diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h
index 253fabdac001..69b5f6e92940 100644
--- a/llvm/include/llvm/CodeGen/MachinePassManager.h
+++ b/llvm/include/llvm/CodeGen/MachinePassManager.h
@@ -41,7 +41,7 @@ using MachineFunctionAnalysisManager = AnalysisManager<MachineFunction>;
 /// MachineFunctionProperties properly.
 template <typename PassT> class MFPropsModifier {
 public:
-  MFPropsModifier(PassT &P_, MachineFunction &MF_) : P(P_), MF(MF_) {
+  MFPropsModifier(const PassT &P_, MachineFunction &MF_) : P(P_), MF(MF_) {
     auto &MFProps = MF.getProperties();
 #ifndef NDEBUG
     if constexpr (has_get_required_properties_v<PassT>) {
@@ -71,7 +71,7 @@ public:
   }
 
 private:
-  PassT &P;
+  const PassT &P;
   MachineFunction &MF;
 
   template <typename T>
diff --git a/llvm/include/llvm/CodeGen/RegAllocFast.h b/llvm/include/llvm/CodeGen/RegAllocFast.h
index c99c715daacf..440264a06ae8 100644
--- a/llvm/include/llvm/CodeGen/RegAllocFast.h
+++ b/llvm/include/llvm/CodeGen/RegAllocFast.h
@@ -27,12 +27,12 @@ public:
   RegAllocFastPass(RegAllocFastPassOptions Opts = RegAllocFastPassOptions())
       : Opts(Opts) {}
 
-  MachineFunctionProperties getRequiredProperties() {
+  MachineFunctionProperties getRequiredProperties() const {
     return MachineFunctionProperties().set(
         MachineFunctionProperties::Property::NoPHIs);
   }
 
-  MachineFunctionProperties getSetProperties() {
+  MachineFunctionProperties getSetProperties() const {
     if (Opts.ClearVRegs) {
       return MachineFunctionProperties().set(
           MachineFunctionProperties::Property::NoVRegs);
@@ -41,7 +41,7 @@ public:
     return MachineFunctionProperties();
   }
 
-  MachineFunctionProperties getClearedProperties() {
+  MachineFunctionProperties getClearedProperties() const {
     return MachineFunctionProperties().set(
         MachineFunctionProperties::Property::IsSSA);
   }
diff --git a/llvm/include/llvm/CodeGen/TwoAddressInstructionPass.h b/llvm/include/llvm/CodeGen/TwoAddressInstructionPass.h
index 7f2a070c5843..d4d47f29cc84 100644
--- a/llvm/include/llvm/CodeGen/TwoAddressInstructionPass.h
+++ b/llvm/include/llvm/CodeGen/TwoAddressInstructionPass.h
@@ -18,7 +18,7 @@ class TwoAddressInstructionPass
 public:
   PreservedAnalyses run(MachineFunction &MF,
                         MachineFunctionAnalysisManager &MFAM);
-  MachineFunctionProperties getSetProperties() {
+  MachineFunctionProperties getSetProperties() const {
     return MachineFunctionProperties().set(
         MachineFunctionProperties::Property::TiedOpsRewritten);
   }
diff --git a/llvm/lib/Target/AMDGPU/GCNDPPCombine.h b/llvm/lib/Target/AMDGPU/GCNDPPCombine.h
index 8f119054e6c0..ac45e578157e 100644
--- a/llvm/lib/Target/AMDGPU/GCNDPPCombine.h
+++ b/llvm/lib/Target/AMDGPU/GCNDPPCombine.h
@@ -17,7 +17,7 @@ public:
   PreservedAnalyses run(MachineFunction &MF,
                         MachineFunctionAnalysisManager &MAM);
 
-  MachineFunctionProperties getRequiredProperties() {
+  MachineFunctionProperties getRequiredProperties() const {
     return MachineFunctionProperties().set(
         MachineFunctionProperties::Property::IsSSA);
   }
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.h b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.h
index 6c20401d6bf5..33188c6ebb67 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.h
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.h
@@ -19,7 +19,7 @@ public:
   PreservedAnalyses run(MachineFunction &MF,
                         MachineFunctionAnalysisManager &MFAM);
 
-  MachineFunctionProperties getRequiredProperties() {
+  MachineFunctionProperties getRequiredProperties() const {
     return MachineFunctionProperties().set(
         MachineFunctionProperties::Property::IsSSA);
   }
diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.h b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.h
index 730b3f8c617b..a9ffb5705d09 100644
--- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.h
+++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.h
@@ -17,7 +17,7 @@ public:
   PreservedAnalyses run(MachineFunction &MF,
                         MachineFunctionAnalysisManager &MFAM);
 
-  MachineFunctionProperties getClearedProperties() {
+  MachineFunctionProperties getClearedProperties() const {
     // SILowerSGPRSpills introduces new Virtual VGPRs for spilling SGPRs.
     return MachineFunctionProperties()
         .set(MachineFunctionProperties::Property::IsSSA)
-- 
GitLab


From 6c64c8a6f3f77c30745c751d4163ff6bf2fc323b Mon Sep 17 00:00:00 2001
From: Artem Belevich <tra@google.com>
Date: Thu, 24 Oct 2024 12:00:56 -0700
Subject: [PATCH 088/693] [NVPTX] add an optional early copy of byval arguments
 (#113384)

byval arguments in NVPTX are special. We're only allowed to read from
them using a special instruction, and if we ever need to write to them
or take an address, we must make a local copy and use it, instead.

The problem is that local copies are very expensive, and we create them
very late in the compilation pipeline, so LLVM does not have much of a
chance to eliminate them, if they turn out to be unnecessary.

One way around that is to create such copies early on, and let them
percolate through the optimizations. The copying itself will never
trigger creation of another copy later on, as the reads are allowed. If
LLVM can eliminate it, it's a win. It the full optimization pipeline
can't remove the copy, that's as good as it gets in terms of the effort
we could've done, and it's certainly a much better effort than what we
do now.

This early injection of the copies has potential to create undesireable
side-effects, so it's disabled by default, for now, until it sees more
testing.
---
 llvm/lib/Target/NVPTX/NVPTX.h                |   4 +
 llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp     |  74 ++-
 llvm/lib/Target/NVPTX/NVPTXPassRegistry.def  |   1 +
 llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp |  25 +
 llvm/test/CodeGen/NVPTX/lower-byval-args.ll  | 638 ++++++++++++++-----
 5 files changed, 569 insertions(+), 173 deletions(-)

diff --git a/llvm/lib/Target/NVPTX/NVPTX.h b/llvm/lib/Target/NVPTX/NVPTX.h
index f6ab81d3ca0b..ca915cd3f373 100644
--- a/llvm/lib/Target/NVPTX/NVPTX.h
+++ b/llvm/lib/Target/NVPTX/NVPTX.h
@@ -70,6 +70,10 @@ struct GenericToNVVMPass : PassInfoMixin<GenericToNVVMPass> {
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
+struct NVPTXCopyByValArgsPass : PassInfoMixin<NVPTXCopyByValArgsPass> {
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+
 namespace NVPTX {
 enum DrvInterface {
   NVCL,
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
index bb76cfd6fdb7..17d605d95eff 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
@@ -543,6 +543,33 @@ struct ArgUseChecker : PtrUseVisitor<ArgUseChecker> {
       PI.setAborted(&II);
   }
 }; // struct ArgUseChecker
+
+void copyByValParam(Function &F, Argument &Arg) {
+  LLVM_DEBUG(dbgs() << "Creating a local copy of " << Arg << "\n");
+  // Otherwise we have to create a temporary copy.
+  BasicBlock::iterator FirstInst = F.getEntryBlock().begin();
+  Type *StructType = Arg.getParamByValType();
+  const DataLayout &DL = F.getDataLayout();
+  AllocaInst *AllocA = new AllocaInst(StructType, DL.getAllocaAddrSpace(),
+                                      Arg.getName(), FirstInst);
+  // Set the alignment to alignment of the byval parameter. This is because,
+  // later load/stores assume that alignment, and we are going to replace
+  // the use of the byval parameter with this alloca instruction.
+  AllocA->setAlignment(F.getParamAlign(Arg.getArgNo())
+                           .value_or(DL.getPrefTypeAlign(StructType)));
+  Arg.replaceAllUsesWith(AllocA);
+
+  Value *ArgInParam = new AddrSpaceCastInst(
+      &Arg, PointerType::get(Arg.getContext(), ADDRESS_SPACE_PARAM),
+      Arg.getName(), FirstInst);
+  // Be sure to propagate alignment to this load; LLVM doesn't know that NVPTX
+  // addrspacecast preserves alignment.  Since params are constant, this load
+  // is definitely not volatile.
+  const auto ArgSize = *AllocA->getAllocationSize(DL);
+  IRBuilder<> IRB(&*FirstInst);
+  IRB.CreateMemCpy(AllocA, AllocA->getAlign(), ArgInParam, AllocA->getAlign(),
+                   ArgSize);
+}
 } // namespace
 
 void NVPTXLowerArgs::handleByValParam(const NVPTXTargetMachine &TM,
@@ -558,7 +585,7 @@ void NVPTXLowerArgs::handleByValParam(const NVPTXTargetMachine &TM,
 
   ArgUseChecker AUC(DL, IsGridConstant);
   ArgUseChecker::PtrInfo PI = AUC.visitArgPtr(*Arg);
-  bool ArgUseIsReadOnly  = !(PI.isEscaped() || PI.isAborted());
+  bool ArgUseIsReadOnly = !(PI.isEscaped() || PI.isAborted());
   // Easy case, accessing parameter directly is fine.
   if (ArgUseIsReadOnly && AUC.Conditionals.empty()) {
     // Convert all loads and intermediate operations to use parameter AS and
@@ -587,7 +614,6 @@ void NVPTXLowerArgs::handleByValParam(const NVPTXTargetMachine &TM,
   // However, we're still not allowed to write to it. If the user specified
   // `__grid_constant__` for the argument, we'll consider escaped pointer as
   // read-only.
-  unsigned AS = DL.getAllocaAddrSpace();
   if (HasCvtaParam && (ArgUseIsReadOnly || IsGridConstant)) {
     LLVM_DEBUG(dbgs() << "Using non-copy pointer to " << *Arg << "\n");
     // Replace all argument pointer uses (which might include a device function
@@ -612,29 +638,8 @@ void NVPTXLowerArgs::handleByValParam(const NVPTXTargetMachine &TM,
 
     // Do not replace Arg in the cast to param space
     CastToParam->setOperand(0, Arg);
-  } else {
-    LLVM_DEBUG(dbgs() << "Creating a local copy of " << *Arg << "\n");
-    // Otherwise we have to create a temporary copy.
-    AllocaInst *AllocA =
-        new AllocaInst(StructType, AS, Arg->getName(), FirstInst);
-    // Set the alignment to alignment of the byval parameter. This is because,
-    // later load/stores assume that alignment, and we are going to replace
-    // the use of the byval parameter with this alloca instruction.
-    AllocA->setAlignment(Func->getParamAlign(Arg->getArgNo())
-                             .value_or(DL.getPrefTypeAlign(StructType)));
-    Arg->replaceAllUsesWith(AllocA);
-
-    Value *ArgInParam = new AddrSpaceCastInst(
-        Arg, PointerType::get(Arg->getContext(), ADDRESS_SPACE_PARAM),
-        Arg->getName(), FirstInst);
-    // Be sure to propagate alignment to this load; LLVM doesn't know that NVPTX
-    // addrspacecast preserves alignment.  Since params are constant, this load
-    // is definitely not volatile.
-    const auto ArgSize = *AllocA->getAllocationSize(DL);
-    IRBuilder<> IRB(&*FirstInst);
-    IRB.CreateMemCpy(AllocA, AllocA->getAlign(), ArgInParam, AllocA->getAlign(),
-                     ArgSize);
-  }
+  } else
+    copyByValParam(*Func, *Arg);
 }
 
 void NVPTXLowerArgs::markPointerAsGlobal(Value *Ptr) {
@@ -734,3 +739,22 @@ bool NVPTXLowerArgs::runOnFunction(Function &F) {
 }
 
 FunctionPass *llvm::createNVPTXLowerArgsPass() { return new NVPTXLowerArgs(); }
+
+static bool copyFunctionByValArgs(Function &F) {
+  LLVM_DEBUG(dbgs() << "Creating a copy of byval args of " << F.getName()
+                    << "\n");
+  bool Changed = false;
+  for (Argument &Arg : F.args())
+    if (Arg.getType()->isPointerTy() && Arg.hasByValAttr() &&
+        !(isParamGridConstant(Arg) && isKernelFunction(F))) {
+      copyByValParam(F, Arg);
+      Changed = true;
+    }
+  return Changed;
+}
+
+PreservedAnalyses NVPTXCopyByValArgsPass::run(Function &F,
+                                              FunctionAnalysisManager &AM) {
+  return copyFunctionByValArgs(F) ? PreservedAnalyses::none()
+                                  : PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def b/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def
index 6ff15ab6f13c..28ea9dd9c022 100644
--- a/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def
+++ b/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def
@@ -37,4 +37,5 @@ FUNCTION_ALIAS_ANALYSIS("nvptx-aa", NVPTXAA())
 #endif
 FUNCTION_PASS("nvvm-intr-range", NVVMIntrRangePass())
 FUNCTION_PASS("nvvm-reflect", NVVMReflectPass())
+FUNCTION_PASS("nvptx-copy-byval-args", NVPTXCopyByValArgsPass())
 #undef FUNCTION_PASS
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
index d0ea6023ddf2..1d6f39b29053 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
@@ -64,6 +64,29 @@ static cl::opt<bool> UseShortPointersOpt(
         "Use 32-bit pointers for accessing const/local/shared address spaces."),
     cl::init(false), cl::Hidden);
 
+// byval arguments in NVPTX are special. We're only allowed to read from them
+// using a special instruction, and if we ever need to write to them or take an
+// address, we must make a local copy and use it, instead.
+//
+// The problem is that local copies are very expensive, and we create them very
+// late in the compilation pipeline, so LLVM does not have much of a chance to
+// eliminate them, if they turn out to be unnecessary.
+//
+// One way around that is to create such copies early on, and let them percolate
+// through the optimizations. The copying itself will never trigger creation of
+// another copy later on, as the reads are allowed. If LLVM can eliminate it,
+// it's a win. It the full optimization pipeline can't remove the copy, that's
+// as good as it gets in terms of the effort we could've done, and it's
+// certainly a much better effort than what we do now.
+//
+// This early injection of the copies has potential to create undesireable
+// side-effects, so it's disabled by default, for now, until it sees more
+// testing.
+static cl::opt<bool> EarlyByValArgsCopy(
+    "nvptx-early-byval-copy",
+    cl::desc("Create a copy of byval function arguments early."),
+    cl::init(false), cl::Hidden);
+
 namespace llvm {
 
 void initializeGenericToNVVMLegacyPassPass(PassRegistry &);
@@ -236,6 +259,8 @@ void NVPTXTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
         // Note: NVVMIntrRangePass was causing numerical discrepancies at one
         // point, if issues crop up, consider disabling.
         FPM.addPass(NVVMIntrRangePass());
+        if (EarlyByValArgsCopy)
+          FPM.addPass(NVPTXCopyByValArgsPass());
         PM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
       });
 }
diff --git a/llvm/test/CodeGen/NVPTX/lower-byval-args.ll b/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
index 013694277039..5c52626a711f 100644
--- a/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --scrub-attributes --version 5
 ; RUN: opt < %s -mtriple nvptx64 -mcpu=sm_60 -mattr=ptx77 -nvptx-lower-args -S | FileCheck %s --check-prefixes=COMMON,SM_60
 ; RUN: opt < %s -mtriple nvptx64 -mcpu=sm_70 -mattr=ptx77 -nvptx-lower-args -S | FileCheck %s --check-prefixes=COMMON,SM_70
+; RUN: opt < %s -mtriple nvptx64 -mcpu=sm_70 -mattr=ptx77 -passes=nvptx-copy-byval-args -S | FileCheck %s --check-prefixes=COMMON,COPY
 source_filename = "<stdin>"
 target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
 target triple = "nvptx64-nvidia-cuda"
@@ -24,15 +25,35 @@ declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #2
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local void @read_only(ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
-; COMMON-LABEL: define dso_local void @read_only(
-; COMMON-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-; COMMON-NEXT:  [[ENTRY:.*:]]
-; COMMON-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
-; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
-; COMMON-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[S3]], align 4
-; COMMON-NEXT:    store i32 [[I]], ptr [[OUT2]], align 4
-; COMMON-NEXT:    ret void
+; SM_60-LABEL: define dso_local void @read_only(
+; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; SM_60-NEXT:  [[ENTRY:.*:]]
+; SM_60-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_60-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_60-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[S3]], align 4
+; SM_60-NEXT:    store i32 [[I]], ptr [[OUT2]], align 4
+; SM_60-NEXT:    ret void
+;
+; SM_70-LABEL: define dso_local void @read_only(
+; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; SM_70-NEXT:  [[ENTRY:.*:]]
+; SM_70-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_70-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_70-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[S3]], align 4
+; SM_70-NEXT:    store i32 [[I]], ptr [[OUT2]], align 4
+; SM_70-NEXT:    ret void
+;
+; COPY-LABEL: define dso_local void @read_only(
+; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; COPY-NEXT:  [[ENTRY:.*:]]
+; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
+; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
+; COPY-NEXT:    [[I:%.*]] = load i32, ptr [[S1]], align 4
+; COPY-NEXT:    store i32 [[I]], ptr [[OUT]], align 4
+; COPY-NEXT:    ret void
 ;
 entry:
   %i = load i32, ptr %s, align 4
@@ -42,16 +63,38 @@ entry:
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local void @read_only_gep(ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
-; COMMON-LABEL: define dso_local void @read_only_gep(
-; COMMON-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; COMMON-NEXT:  [[ENTRY:.*:]]
-; COMMON-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
-; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
-; COMMON-NEXT:    [[B4:%.*]] = getelementptr inbounds i8, ptr addrspace(101) [[S3]], i64 4
-; COMMON-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[B4]], align 4
-; COMMON-NEXT:    store i32 [[I]], ptr [[OUT2]], align 4
-; COMMON-NEXT:    ret void
+; SM_60-LABEL: define dso_local void @read_only_gep(
+; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-NEXT:  [[ENTRY:.*:]]
+; SM_60-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_60-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_60-NEXT:    [[B4:%.*]] = getelementptr inbounds i8, ptr addrspace(101) [[S3]], i64 4
+; SM_60-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[B4]], align 4
+; SM_60-NEXT:    store i32 [[I]], ptr [[OUT2]], align 4
+; SM_60-NEXT:    ret void
+;
+; SM_70-LABEL: define dso_local void @read_only_gep(
+; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-NEXT:  [[ENTRY:.*:]]
+; SM_70-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_70-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_70-NEXT:    [[B4:%.*]] = getelementptr inbounds i8, ptr addrspace(101) [[S3]], i64 4
+; SM_70-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[B4]], align 4
+; SM_70-NEXT:    store i32 [[I]], ptr [[OUT2]], align 4
+; SM_70-NEXT:    ret void
+;
+; COPY-LABEL: define dso_local void @read_only_gep(
+; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-NEXT:  [[ENTRY:.*:]]
+; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
+; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
+; COPY-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S1]], i64 4
+; COPY-NEXT:    [[I:%.*]] = load i32, ptr [[B]], align 4
+; COPY-NEXT:    store i32 [[I]], ptr [[OUT]], align 4
+; COPY-NEXT:    ret void
 ;
 entry:
   %b = getelementptr inbounds nuw i8, ptr %s, i64 4
@@ -62,16 +105,39 @@ entry:
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local void @read_only_gep_asc(ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
-; COMMON-LABEL: define dso_local void @read_only_gep_asc(
-; COMMON-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; COMMON-NEXT:  [[ENTRY:.*:]]
-; COMMON-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
-; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
-; COMMON-NEXT:    [[B4:%.*]] = getelementptr inbounds i8, ptr addrspace(101) [[S3]], i64 4
-; COMMON-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[B4]], align 4
-; COMMON-NEXT:    store i32 [[I]], ptr [[OUT2]], align 4
-; COMMON-NEXT:    ret void
+; SM_60-LABEL: define dso_local void @read_only_gep_asc(
+; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-NEXT:  [[ENTRY:.*:]]
+; SM_60-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_60-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_60-NEXT:    [[B4:%.*]] = getelementptr inbounds i8, ptr addrspace(101) [[S3]], i64 4
+; SM_60-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[B4]], align 4
+; SM_60-NEXT:    store i32 [[I]], ptr [[OUT2]], align 4
+; SM_60-NEXT:    ret void
+;
+; SM_70-LABEL: define dso_local void @read_only_gep_asc(
+; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-NEXT:  [[ENTRY:.*:]]
+; SM_70-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_70-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_70-NEXT:    [[B4:%.*]] = getelementptr inbounds i8, ptr addrspace(101) [[S3]], i64 4
+; SM_70-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[B4]], align 4
+; SM_70-NEXT:    store i32 [[I]], ptr [[OUT2]], align 4
+; SM_70-NEXT:    ret void
+;
+; COPY-LABEL: define dso_local void @read_only_gep_asc(
+; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-NEXT:  [[ENTRY:.*:]]
+; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
+; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
+; COPY-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S1]], i64 4
+; COPY-NEXT:    [[ASC:%.*]] = addrspacecast ptr [[B]] to ptr addrspace(101)
+; COPY-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[ASC]], align 4
+; COPY-NEXT:    store i32 [[I]], ptr [[OUT]], align 4
+; COPY-NEXT:    ret void
 ;
 entry:
   %b = getelementptr inbounds nuw i8, ptr %s, i64 4
@@ -83,20 +149,48 @@ entry:
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local void @read_only_gep_asc0(ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
-; COMMON-LABEL: define dso_local void @read_only_gep_asc0(
-; COMMON-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; COMMON-NEXT:  [[ENTRY:.*:]]
-; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
-; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
-; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
-; COMMON-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S3]], i64 4
-; COMMON-NEXT:    [[ASC:%.*]] = addrspacecast ptr [[B]] to ptr addrspace(101)
-; COMMON-NEXT:    [[ASC0:%.*]] = addrspacecast ptr addrspace(101) [[ASC]] to ptr
-; COMMON-NEXT:    [[I:%.*]] = load i32, ptr [[ASC0]], align 4
-; COMMON-NEXT:    store i32 [[I]], ptr [[OUT2]], align 4
-; COMMON-NEXT:    ret void
+; SM_60-LABEL: define dso_local void @read_only_gep_asc0(
+; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-NEXT:  [[ENTRY:.*:]]
+; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_60-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_60-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S3]], i64 4
+; SM_60-NEXT:    [[ASC:%.*]] = addrspacecast ptr [[B]] to ptr addrspace(101)
+; SM_60-NEXT:    [[ASC0:%.*]] = addrspacecast ptr addrspace(101) [[ASC]] to ptr
+; SM_60-NEXT:    [[I:%.*]] = load i32, ptr [[ASC0]], align 4
+; SM_60-NEXT:    store i32 [[I]], ptr [[OUT2]], align 4
+; SM_60-NEXT:    ret void
+;
+; SM_70-LABEL: define dso_local void @read_only_gep_asc0(
+; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-NEXT:  [[ENTRY:.*:]]
+; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_70-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_70-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_70-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S3]], i64 4
+; SM_70-NEXT:    [[ASC:%.*]] = addrspacecast ptr [[B]] to ptr addrspace(101)
+; SM_70-NEXT:    [[ASC0:%.*]] = addrspacecast ptr addrspace(101) [[ASC]] to ptr
+; SM_70-NEXT:    [[I:%.*]] = load i32, ptr [[ASC0]], align 4
+; SM_70-NEXT:    store i32 [[I]], ptr [[OUT2]], align 4
+; SM_70-NEXT:    ret void
+;
+; COPY-LABEL: define dso_local void @read_only_gep_asc0(
+; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-NEXT:  [[ENTRY:.*:]]
+; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
+; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
+; COPY-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S1]], i64 4
+; COPY-NEXT:    [[ASC:%.*]] = addrspacecast ptr [[B]] to ptr addrspace(101)
+; COPY-NEXT:    [[ASC0:%.*]] = addrspacecast ptr addrspace(101) [[ASC]] to ptr
+; COPY-NEXT:    [[I:%.*]] = load i32, ptr [[ASC0]], align 4
+; COPY-NEXT:    store i32 [[I]], ptr [[OUT]], align 4
+; COPY-NEXT:    ret void
 ;
 entry:
   %b = getelementptr inbounds nuw i8, ptr %s, i64 4
@@ -109,16 +203,36 @@ entry:
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local void @escape_ptr(ptr nocapture noundef readnone %out, ptr noundef byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
-; COMMON-LABEL: define dso_local void @escape_ptr(
-; COMMON-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; COMMON-NEXT:  [[ENTRY:.*:]]
-; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
-; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
-; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
-; COMMON-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[S3]])
-; COMMON-NEXT:    ret void
+; SM_60-LABEL: define dso_local void @escape_ptr(
+; SM_60-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-NEXT:  [[ENTRY:.*:]]
+; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_60-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_60-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[S3]])
+; SM_60-NEXT:    ret void
+;
+; SM_70-LABEL: define dso_local void @escape_ptr(
+; SM_70-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-NEXT:  [[ENTRY:.*:]]
+; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_70-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_70-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_70-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[S3]])
+; SM_70-NEXT:    ret void
+;
+; COPY-LABEL: define dso_local void @escape_ptr(
+; COPY-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-NEXT:  [[ENTRY:.*:]]
+; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
+; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
+; COPY-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[S1]])
+; COPY-NEXT:    ret void
 ;
 entry:
   call void @_Z6escapePv(ptr noundef nonnull %s) #0
@@ -127,17 +241,39 @@ entry:
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local void @escape_ptr_gep(ptr nocapture noundef readnone %out, ptr noundef byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
-; COMMON-LABEL: define dso_local void @escape_ptr_gep(
-; COMMON-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; COMMON-NEXT:  [[ENTRY:.*:]]
-; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
-; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
-; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
-; COMMON-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S3]], i64 4
-; COMMON-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[B]])
-; COMMON-NEXT:    ret void
+; SM_60-LABEL: define dso_local void @escape_ptr_gep(
+; SM_60-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-NEXT:  [[ENTRY:.*:]]
+; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_60-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_60-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S3]], i64 4
+; SM_60-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[B]])
+; SM_60-NEXT:    ret void
+;
+; SM_70-LABEL: define dso_local void @escape_ptr_gep(
+; SM_70-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-NEXT:  [[ENTRY:.*:]]
+; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_70-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_70-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_70-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S3]], i64 4
+; SM_70-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[B]])
+; SM_70-NEXT:    ret void
+;
+; COPY-LABEL: define dso_local void @escape_ptr_gep(
+; COPY-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-NEXT:  [[ENTRY:.*:]]
+; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
+; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
+; COPY-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S1]], i64 4
+; COPY-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[B]])
+; COPY-NEXT:    ret void
 ;
 entry:
   %b = getelementptr inbounds nuw i8, ptr %s, i64 4
@@ -147,16 +283,36 @@ entry:
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local void @escape_ptr_store(ptr nocapture noundef writeonly %out, ptr noundef byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
-; COMMON-LABEL: define dso_local void @escape_ptr_store(
-; COMMON-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; COMMON-NEXT:  [[ENTRY:.*:]]
-; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
-; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
-; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
-; COMMON-NEXT:    store ptr [[S3]], ptr [[OUT2]], align 8
-; COMMON-NEXT:    ret void
+; SM_60-LABEL: define dso_local void @escape_ptr_store(
+; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-NEXT:  [[ENTRY:.*:]]
+; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_60-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_60-NEXT:    store ptr [[S3]], ptr [[OUT2]], align 8
+; SM_60-NEXT:    ret void
+;
+; SM_70-LABEL: define dso_local void @escape_ptr_store(
+; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-NEXT:  [[ENTRY:.*:]]
+; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_70-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_70-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_70-NEXT:    store ptr [[S3]], ptr [[OUT2]], align 8
+; SM_70-NEXT:    ret void
+;
+; COPY-LABEL: define dso_local void @escape_ptr_store(
+; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-NEXT:  [[ENTRY:.*:]]
+; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
+; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
+; COPY-NEXT:    store ptr [[S1]], ptr [[OUT]], align 8
+; COPY-NEXT:    ret void
 ;
 entry:
   store ptr %s, ptr %out, align 8
@@ -165,17 +321,39 @@ entry:
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local void @escape_ptr_gep_store(ptr nocapture noundef writeonly %out, ptr noundef byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
-; COMMON-LABEL: define dso_local void @escape_ptr_gep_store(
-; COMMON-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; COMMON-NEXT:  [[ENTRY:.*:]]
-; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
-; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
-; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
-; COMMON-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S3]], i64 4
-; COMMON-NEXT:    store ptr [[B]], ptr [[OUT2]], align 8
-; COMMON-NEXT:    ret void
+; SM_60-LABEL: define dso_local void @escape_ptr_gep_store(
+; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-NEXT:  [[ENTRY:.*:]]
+; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_60-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_60-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S3]], i64 4
+; SM_60-NEXT:    store ptr [[B]], ptr [[OUT2]], align 8
+; SM_60-NEXT:    ret void
+;
+; SM_70-LABEL: define dso_local void @escape_ptr_gep_store(
+; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-NEXT:  [[ENTRY:.*:]]
+; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_70-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_70-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_70-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S3]], i64 4
+; SM_70-NEXT:    store ptr [[B]], ptr [[OUT2]], align 8
+; SM_70-NEXT:    ret void
+;
+; COPY-LABEL: define dso_local void @escape_ptr_gep_store(
+; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-NEXT:  [[ENTRY:.*:]]
+; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
+; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
+; COPY-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S1]], i64 4
+; COPY-NEXT:    store ptr [[B]], ptr [[OUT]], align 8
+; COPY-NEXT:    ret void
 ;
 entry:
   %b = getelementptr inbounds nuw i8, ptr %s, i64 4
@@ -185,17 +363,39 @@ entry:
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local void @escape_ptrtoint(ptr nocapture noundef writeonly %out, ptr noundef byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
-; COMMON-LABEL: define dso_local void @escape_ptrtoint(
-; COMMON-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; COMMON-NEXT:  [[ENTRY:.*:]]
-; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
-; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
-; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
-; COMMON-NEXT:    [[I:%.*]] = ptrtoint ptr [[S3]] to i64
-; COMMON-NEXT:    store i64 [[I]], ptr [[OUT2]], align 8
-; COMMON-NEXT:    ret void
+; SM_60-LABEL: define dso_local void @escape_ptrtoint(
+; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-NEXT:  [[ENTRY:.*:]]
+; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_60-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_60-NEXT:    [[I:%.*]] = ptrtoint ptr [[S3]] to i64
+; SM_60-NEXT:    store i64 [[I]], ptr [[OUT2]], align 8
+; SM_60-NEXT:    ret void
+;
+; SM_70-LABEL: define dso_local void @escape_ptrtoint(
+; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-NEXT:  [[ENTRY:.*:]]
+; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_70-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_70-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_70-NEXT:    [[I:%.*]] = ptrtoint ptr [[S3]] to i64
+; SM_70-NEXT:    store i64 [[I]], ptr [[OUT2]], align 8
+; SM_70-NEXT:    ret void
+;
+; COPY-LABEL: define dso_local void @escape_ptrtoint(
+; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-NEXT:  [[ENTRY:.*:]]
+; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
+; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
+; COPY-NEXT:    [[I:%.*]] = ptrtoint ptr [[S1]] to i64
+; COPY-NEXT:    store i64 [[I]], ptr [[OUT]], align 8
+; COPY-NEXT:    ret void
 ;
 entry:
   %i = ptrtoint ptr %s to i64
@@ -205,14 +405,32 @@ entry:
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local void @memcpy_from_param(ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
-; COMMON-LABEL: define dso_local void @memcpy_from_param(
-; COMMON-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; COMMON-NEXT:  [[ENTRY:.*:]]
-; COMMON-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
-; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
-; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr [[OUT2]], ptr addrspace(101) [[S3]], i64 16, i1 true)
-; COMMON-NEXT:    ret void
+; SM_60-LABEL: define dso_local void @memcpy_from_param(
+; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-NEXT:  [[ENTRY:.*:]]
+; SM_60-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_60-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr [[OUT2]], ptr addrspace(101) [[S3]], i64 16, i1 true)
+; SM_60-NEXT:    ret void
+;
+; SM_70-LABEL: define dso_local void @memcpy_from_param(
+; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-NEXT:  [[ENTRY:.*:]]
+; SM_70-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_70-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_70-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr [[OUT2]], ptr addrspace(101) [[S3]], i64 16, i1 true)
+; SM_70-NEXT:    ret void
+;
+; COPY-LABEL: define dso_local void @memcpy_from_param(
+; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-NEXT:  [[ENTRY:.*:]]
+; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
+; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
+; COPY-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr [[OUT]], ptr [[S1]], i64 16, i1 true)
+; COPY-NEXT:    ret void
 ;
 entry:
   tail call void @llvm.memcpy.p0.p0.i64(ptr %out, ptr %s, i64 16, i1 true)
@@ -221,14 +439,32 @@ entry:
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local void @memcpy_from_param_noalign (ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) %s) local_unnamed_addr #0 {
-; COMMON-LABEL: define dso_local void @memcpy_from_param_noalign(
-; COMMON-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; COMMON-NEXT:  [[ENTRY:.*:]]
-; COMMON-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
-; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
-; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr [[OUT2]], ptr addrspace(101) [[S3]], i64 16, i1 true)
-; COMMON-NEXT:    ret void
+; SM_60-LABEL: define dso_local void @memcpy_from_param_noalign(
+; SM_60-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-NEXT:  [[ENTRY:.*:]]
+; SM_60-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_60-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_60-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr [[OUT2]], ptr addrspace(101) [[S3]], i64 16, i1 true)
+; SM_60-NEXT:    ret void
+;
+; SM_70-LABEL: define dso_local void @memcpy_from_param_noalign(
+; SM_70-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-NEXT:  [[ENTRY:.*:]]
+; SM_70-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_70-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_70-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
+; SM_70-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr [[OUT2]], ptr addrspace(101) [[S3]], i64 16, i1 true)
+; SM_70-NEXT:    ret void
+;
+; COPY-LABEL: define dso_local void @memcpy_from_param_noalign(
+; COPY-SAME: ptr nocapture noundef writeonly [[OUT:%.*]], ptr nocapture noundef readonly byval([[STRUCT_S:%.*]]) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-NEXT:  [[ENTRY:.*:]]
+; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 8
+; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 8 [[S1]], ptr addrspace(101) align 8 [[S2]], i64 8, i1 false)
+; COPY-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr [[OUT]], ptr [[S1]], i64 16, i1 true)
+; COPY-NEXT:    ret void
 ;
 entry:
   tail call void @llvm.memcpy.p0.p0.i64(ptr %out, ptr %s, i64 16, i1 true)
@@ -237,16 +473,36 @@ entry:
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local void @memcpy_to_param(ptr nocapture noundef readonly %in, ptr nocapture noundef readnone byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
-; COMMON-LABEL: define dso_local void @memcpy_to_param(
-; COMMON-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture noundef readnone byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; COMMON-NEXT:  [[ENTRY:.*:]]
-; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
-; COMMON-NEXT:    [[IN1:%.*]] = addrspacecast ptr [[IN]] to ptr addrspace(1)
-; COMMON-NEXT:    [[IN2:%.*]] = addrspacecast ptr addrspace(1) [[IN1]] to ptr
-; COMMON-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr [[S3]], ptr [[IN2]], i64 16, i1 true)
-; COMMON-NEXT:    ret void
+; SM_60-LABEL: define dso_local void @memcpy_to_param(
+; SM_60-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture noundef readnone byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-NEXT:  [[ENTRY:.*:]]
+; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_60-NEXT:    [[IN1:%.*]] = addrspacecast ptr [[IN]] to ptr addrspace(1)
+; SM_60-NEXT:    [[IN2:%.*]] = addrspacecast ptr addrspace(1) [[IN1]] to ptr
+; SM_60-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr [[S3]], ptr [[IN2]], i64 16, i1 true)
+; SM_60-NEXT:    ret void
+;
+; SM_70-LABEL: define dso_local void @memcpy_to_param(
+; SM_70-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture noundef readnone byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-NEXT:  [[ENTRY:.*:]]
+; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_70-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_70-NEXT:    [[IN1:%.*]] = addrspacecast ptr [[IN]] to ptr addrspace(1)
+; SM_70-NEXT:    [[IN2:%.*]] = addrspacecast ptr addrspace(1) [[IN1]] to ptr
+; SM_70-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr [[S3]], ptr [[IN2]], i64 16, i1 true)
+; SM_70-NEXT:    ret void
+;
+; COPY-LABEL: define dso_local void @memcpy_to_param(
+; COPY-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture noundef readnone byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-NEXT:  [[ENTRY:.*:]]
+; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
+; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
+; COPY-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr [[S1]], ptr [[IN]], i64 16, i1 true)
+; COPY-NEXT:    ret void
 ;
 entry:
   tail call void @llvm.memcpy.p0.p0.i64(ptr %s, ptr %in, i64 16, i1 true)
@@ -255,17 +511,39 @@ entry:
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local void @copy_on_store(ptr nocapture noundef readonly %in, ptr nocapture noundef byval(%struct.S) align 4 %s, i1 noundef zeroext %b) local_unnamed_addr #0 {
-; COMMON-LABEL: define dso_local void @copy_on_store(
-; COMMON-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]], i1 noundef zeroext [[B:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; COMMON-NEXT:  [[BB:.*:]]
-; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
-; COMMON-NEXT:    [[IN1:%.*]] = addrspacecast ptr [[IN]] to ptr addrspace(1)
-; COMMON-NEXT:    [[IN2:%.*]] = addrspacecast ptr addrspace(1) [[IN1]] to ptr
-; COMMON-NEXT:    [[I:%.*]] = load i32, ptr [[IN2]], align 4
-; COMMON-NEXT:    store i32 [[I]], ptr [[S3]], align 4
-; COMMON-NEXT:    ret void
+; SM_60-LABEL: define dso_local void @copy_on_store(
+; SM_60-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]], i1 noundef zeroext [[B:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_60-NEXT:  [[BB:.*:]]
+; SM_60-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_60-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_60-NEXT:    [[IN1:%.*]] = addrspacecast ptr [[IN]] to ptr addrspace(1)
+; SM_60-NEXT:    [[IN2:%.*]] = addrspacecast ptr addrspace(1) [[IN1]] to ptr
+; SM_60-NEXT:    [[I:%.*]] = load i32, ptr [[IN2]], align 4
+; SM_60-NEXT:    store i32 [[I]], ptr [[S3]], align 4
+; SM_60-NEXT:    ret void
+;
+; SM_70-LABEL: define dso_local void @copy_on_store(
+; SM_70-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]], i1 noundef zeroext [[B:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; SM_70-NEXT:  [[BB:.*:]]
+; SM_70-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
+; SM_70-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; SM_70-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
+; SM_70-NEXT:    [[IN1:%.*]] = addrspacecast ptr [[IN]] to ptr addrspace(1)
+; SM_70-NEXT:    [[IN2:%.*]] = addrspacecast ptr addrspace(1) [[IN1]] to ptr
+; SM_70-NEXT:    [[I:%.*]] = load i32, ptr [[IN2]], align 4
+; SM_70-NEXT:    store i32 [[I]], ptr [[S3]], align 4
+; SM_70-NEXT:    ret void
+;
+; COPY-LABEL: define dso_local void @copy_on_store(
+; COPY-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]], i1 noundef zeroext [[B:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COPY-NEXT:  [[BB:.*:]]
+; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
+; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
+; COPY-NEXT:    [[I:%.*]] = load i32, ptr [[IN]], align 4
+; COPY-NEXT:    store i32 [[I]], ptr [[S1]], align 4
+; COPY-NEXT:    ret void
 ;
 bb:
   %i = load i32, ptr %in, align 4
@@ -304,6 +582,20 @@ define void @test_select(ptr byval(i32) align 4 %input1, ptr byval(i32) %input2,
 ; SM_70-NEXT:    store i32 [[VALLOADED]], ptr [[OUT2]], align 4
 ; SM_70-NEXT:    ret void
 ;
+; COPY-LABEL: define void @test_select(
+; COPY-SAME: ptr byval(i32) align 4 [[INPUT1:%.*]], ptr byval(i32) [[INPUT2:%.*]], ptr [[OUT:%.*]], i1 [[COND:%.*]]) #[[ATTR3:[0-9]+]] {
+; COPY-NEXT:  [[BB:.*:]]
+; COPY-NEXT:    [[INPUT23:%.*]] = alloca i32, align 4
+; COPY-NEXT:    [[INPUT24:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT23]], ptr addrspace(101) align 4 [[INPUT24]], i64 4, i1 false)
+; COPY-NEXT:    [[INPUT11:%.*]] = alloca i32, align 4
+; COPY-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 4, i1 false)
+; COPY-NEXT:    [[PTRNEW:%.*]] = select i1 [[COND]], ptr [[INPUT11]], ptr [[INPUT23]]
+; COPY-NEXT:    [[VALLOADED:%.*]] = load i32, ptr [[PTRNEW]], align 4
+; COPY-NEXT:    store i32 [[VALLOADED]], ptr [[OUT]], align 4
+; COPY-NEXT:    ret void
+;
 bb:
   %ptrnew = select i1 %cond, ptr %input1, ptr %input2
   %valloaded = load i32, ptr %ptrnew, align 4
@@ -312,20 +604,48 @@ bb:
 }
 
 define void @test_select_write(ptr byval(i32) align 4 %input1, ptr byval(i32) %input2, ptr %out, i1 %cond) {
-; COMMON-LABEL: define void @test_select_write(
-; COMMON-SAME: ptr byval(i32) align 4 [[INPUT1:%.*]], ptr byval(i32) [[INPUT2:%.*]], ptr [[OUT:%.*]], i1 [[COND:%.*]]) #[[ATTR3:[0-9]+]] {
-; COMMON-NEXT:  [[BB:.*:]]
-; COMMON-NEXT:    [[OUT7:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
-; COMMON-NEXT:    [[OUT8:%.*]] = addrspacecast ptr addrspace(1) [[OUT7]] to ptr
-; COMMON-NEXT:    [[INPUT24:%.*]] = alloca i32, align 4
-; COMMON-NEXT:    [[INPUT25:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
-; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT24]], ptr addrspace(101) align 4 [[INPUT25]], i64 4, i1 false)
-; COMMON-NEXT:    [[INPUT11:%.*]] = alloca i32, align 4
-; COMMON-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
-; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 4, i1 false)
-; COMMON-NEXT:    [[PTRNEW:%.*]] = select i1 [[COND]], ptr [[INPUT11]], ptr [[INPUT24]]
-; COMMON-NEXT:    store i32 1, ptr [[PTRNEW]], align 4
-; COMMON-NEXT:    ret void
+; SM_60-LABEL: define void @test_select_write(
+; SM_60-SAME: ptr byval(i32) align 4 [[INPUT1:%.*]], ptr byval(i32) [[INPUT2:%.*]], ptr [[OUT:%.*]], i1 [[COND:%.*]]) #[[ATTR3]] {
+; SM_60-NEXT:  [[BB:.*:]]
+; SM_60-NEXT:    [[OUT5:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_60-NEXT:    [[OUT6:%.*]] = addrspacecast ptr addrspace(1) [[OUT5]] to ptr
+; SM_60-NEXT:    [[INPUT23:%.*]] = alloca i32, align 4
+; SM_60-NEXT:    [[INPUT24:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT23]], ptr addrspace(101) align 4 [[INPUT24]], i64 4, i1 false)
+; SM_60-NEXT:    [[INPUT11:%.*]] = alloca i32, align 4
+; SM_60-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 4, i1 false)
+; SM_60-NEXT:    [[PTRNEW:%.*]] = select i1 [[COND]], ptr [[INPUT11]], ptr [[INPUT23]]
+; SM_60-NEXT:    store i32 1, ptr [[PTRNEW]], align 4
+; SM_60-NEXT:    ret void
+;
+; SM_70-LABEL: define void @test_select_write(
+; SM_70-SAME: ptr byval(i32) align 4 [[INPUT1:%.*]], ptr byval(i32) [[INPUT2:%.*]], ptr [[OUT:%.*]], i1 [[COND:%.*]]) #[[ATTR3]] {
+; SM_70-NEXT:  [[BB:.*:]]
+; SM_70-NEXT:    [[OUT5:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
+; SM_70-NEXT:    [[OUT6:%.*]] = addrspacecast ptr addrspace(1) [[OUT5]] to ptr
+; SM_70-NEXT:    [[INPUT23:%.*]] = alloca i32, align 4
+; SM_70-NEXT:    [[INPUT24:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
+; SM_70-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT23]], ptr addrspace(101) align 4 [[INPUT24]], i64 4, i1 false)
+; SM_70-NEXT:    [[INPUT11:%.*]] = alloca i32, align 4
+; SM_70-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
+; SM_70-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 4, i1 false)
+; SM_70-NEXT:    [[PTRNEW:%.*]] = select i1 [[COND]], ptr [[INPUT11]], ptr [[INPUT23]]
+; SM_70-NEXT:    store i32 1, ptr [[PTRNEW]], align 4
+; SM_70-NEXT:    ret void
+;
+; COPY-LABEL: define void @test_select_write(
+; COPY-SAME: ptr byval(i32) align 4 [[INPUT1:%.*]], ptr byval(i32) [[INPUT2:%.*]], ptr [[OUT:%.*]], i1 [[COND:%.*]]) #[[ATTR3]] {
+; COPY-NEXT:  [[BB:.*:]]
+; COPY-NEXT:    [[INPUT23:%.*]] = alloca i32, align 4
+; COPY-NEXT:    [[INPUT24:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT23]], ptr addrspace(101) align 4 [[INPUT24]], i64 4, i1 false)
+; COPY-NEXT:    [[INPUT11:%.*]] = alloca i32, align 4
+; COPY-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 4, i1 false)
+; COPY-NEXT:    [[PTRNEW:%.*]] = select i1 [[COND]], ptr [[INPUT11]], ptr [[INPUT23]]
+; COPY-NEXT:    store i32 1, ptr [[PTRNEW]], align 4
+; COPY-NEXT:    ret void
 ;
 bb:
   %ptrnew = select i1 %cond, ptr %input1, ptr %input2
@@ -380,6 +700,28 @@ define void @test_phi(ptr byval(%struct.S) align 4 %input1, ptr byval(%struct.S)
 ; SM_70-NEXT:    store i32 [[VALLOADED]], ptr [[INOUT2]], align 4
 ; SM_70-NEXT:    ret void
 ;
+; COPY-LABEL: define void @test_phi(
+; COPY-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT1:%.*]], ptr byval([[STRUCT_S]]) [[INPUT2:%.*]], ptr [[INOUT:%.*]], i1 [[COND:%.*]]) #[[ATTR3]] {
+; COPY-NEXT:  [[BB:.*:]]
+; COPY-NEXT:    [[INPUT23:%.*]] = alloca [[STRUCT_S]], align 8
+; COPY-NEXT:    [[INPUT24:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 8 [[INPUT23]], ptr addrspace(101) align 8 [[INPUT24]], i64 8, i1 false)
+; COPY-NEXT:    [[INPUT11:%.*]] = alloca [[STRUCT_S]], align 4
+; COPY-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
+; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 8, i1 false)
+; COPY-NEXT:    br i1 [[COND]], label %[[FIRST:.*]], label %[[SECOND:.*]]
+; COPY:       [[FIRST]]:
+; COPY-NEXT:    [[PTR1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[INPUT11]], i32 0, i32 0
+; COPY-NEXT:    br label %[[MERGE:.*]]
+; COPY:       [[SECOND]]:
+; COPY-NEXT:    [[PTR2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[INPUT23]], i32 0, i32 1
+; COPY-NEXT:    br label %[[MERGE]]
+; COPY:       [[MERGE]]:
+; COPY-NEXT:    [[PTRNEW:%.*]] = phi ptr [ [[PTR1]], %[[FIRST]] ], [ [[PTR2]], %[[SECOND]] ]
+; COPY-NEXT:    [[VALLOADED:%.*]] = load i32, ptr [[PTRNEW]], align 4
+; COPY-NEXT:    store i32 [[VALLOADED]], ptr [[INOUT]], align 4
+; COPY-NEXT:    ret void
+;
 bb:
   br i1 %cond, label %first, label %second
 
@@ -400,7 +742,7 @@ merge:                                            ; preds = %second, %first
 
 define void @test_phi_write(ptr byval(%struct.S) align 4 %input1, ptr byval(%struct.S) %input2, i1 %cond) {
 ; COMMON-LABEL: define void @test_phi_write(
-; COMMON-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT1:%.*]], ptr byval([[STRUCT_S]]) [[INPUT2:%.*]], i1 [[COND:%.*]]) #[[ATTR3]] {
+; COMMON-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT1:%.*]], ptr byval([[STRUCT_S]]) [[INPUT2:%.*]], i1 [[COND:%.*]]) #[[ATTR3:[0-9]+]] {
 ; COMMON-NEXT:  [[BB:.*:]]
 ; COMMON-NEXT:    [[INPUT24:%.*]] = alloca [[STRUCT_S]], align 8
 ; COMMON-NEXT:    [[INPUT25:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
-- 
GitLab


From 8aa69a0d02e99e50db0242f75a192b1c2d826528 Mon Sep 17 00:00:00 2001
From: Jan Voung <jvoung@google.com>
Date: Thu, 24 Oct 2024 15:23:40 -0400
Subject: [PATCH 089/693] [clang][dataflow] Disambiguate a ref to "internal" in
 CachedConstAccessorsLattice (#113601)

Disambiguate to fix a build error (e.g., on windows with clang-cl)
---
 .../Analysis/FlowSensitive/CachedConstAccessorsLattice.h   | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/clang/include/clang/Analysis/FlowSensitive/CachedConstAccessorsLattice.h b/clang/include/clang/Analysis/FlowSensitive/CachedConstAccessorsLattice.h
index 3402d105746e..48c528736773 100644
--- a/clang/include/clang/Analysis/FlowSensitive/CachedConstAccessorsLattice.h
+++ b/clang/include/clang/Analysis/FlowSensitive/CachedConstAccessorsLattice.h
@@ -154,11 +154,12 @@ LatticeEffect CachedConstAccessorsLattice<Base>::join(
   // are non-identical but equivalent. This is likely to be sufficient in
   // practice, and it reduces implementation complexity considerably.
 
-  ConstMethodReturnValues = internal::joinConstMethodMap<Value>(
-      ConstMethodReturnValues, Other.ConstMethodReturnValues, Effect);
+  ConstMethodReturnValues =
+      clang::dataflow::internal::joinConstMethodMap<dataflow::Value>(
+          ConstMethodReturnValues, Other.ConstMethodReturnValues, Effect);
 
   ConstMethodReturnStorageLocations =
-      internal::joinConstMethodMap<StorageLocation>(
+      clang::dataflow::internal::joinConstMethodMap<dataflow::StorageLocation>(
           ConstMethodReturnStorageLocations,
           Other.ConstMethodReturnStorageLocations, Effect);
 
-- 
GitLab


From 76316ec91410bfe4775f828e3ebae5958a9d2a1d Mon Sep 17 00:00:00 2001
From: Hubert Tong <hubert.reinterpretcast@gmail.com>
Date: Thu, 24 Oct 2024 16:14:22 -0400
Subject: [PATCH 090/693] [Legacy ThinLTO] NFC: Use explicit `static`; shrink
 anonymous namespace

Improve compliance with
https://llvm.org/docs/CodingStandards.html#anonymous-namespaces:
> [Only] use [anonymous namespaces] for class declarations.
---
 llvm/lib/LTO/ThinLTOCodeGenerator.cpp | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
index 311510ed9fab..4522f4adcebe 100644
--- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -85,8 +85,6 @@ extern cl::opt<std::optional<uint64_t>, false, remarks::HotnessThresholdParser>
 extern cl::opt<std::string> RemarksFormat;
 }
 
-namespace {
-
 // Default to using all available threads in the system, but using only one
 // thred per core, as indicated by the usage of
 // heavyweight_hardware_concurrency() below.
@@ -326,8 +324,8 @@ computeGUIDPreservedSymbols(const lto::InputFile &File,
   return GUIDPreservedSymbols;
 }
 
-std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
-                                            TargetMachine &TM) {
+static std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
+                                                   TargetMachine &TM) {
   SmallVector<char, 128> OutputBuffer;
 
   // CodeGen
@@ -347,6 +345,7 @@ std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
       std::move(OutputBuffer), /*RequiresNullTerminator=*/false);
 }
 
+namespace {
 /// Manage caching for a single Module.
 class ModuleCacheEntry {
   SmallString<128> EntryPath;
@@ -424,6 +423,7 @@ public:
                                        toString(std::move(Err)).c_str()));
   }
 };
+} // end anonymous namespace
 
 static std::unique_ptr<MemoryBuffer>
 ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
@@ -541,8 +541,6 @@ static void initTMBuilder(TargetMachineBuilder &TMBuilder,
   TMBuilder.TheTriple = std::move(TheTriple);
 }
 
-} // end anonymous namespace
-
 void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
   MemoryBufferRef Buffer(Data, Identifier);
 
-- 
GitLab


From 59b6c1b2d5290c40f59d2820b5144e2dec731635 Mon Sep 17 00:00:00 2001
From: Jessica Clarke <jrtc27@jrtc27.com>
Date: Thu, 24 Oct 2024 21:38:42 +0100
Subject: [PATCH 091/693] [DXIL] Add to LINK_COMPONENTS to fix
 BUILD_SHARED_LIBS build

Fixes: 324bdd662dedfd03b884e082f577a8ad6dc1f8a6
---
 llvm/lib/Target/DirectX/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/lib/Target/DirectX/CMakeLists.txt b/llvm/lib/Target/DirectX/CMakeLists.txt
index c8ef0ef6f7e7..5d1dc50fdb0d 100644
--- a/llvm/lib/Target/DirectX/CMakeLists.txt
+++ b/llvm/lib/Target/DirectX/CMakeLists.txt
@@ -44,6 +44,7 @@ add_llvm_target(DirectXCodeGen
   DirectXPointerTypeAnalysis
   FrontendHLSL
   MC
+  ScalarOpts
   SelectionDAG
   Support
   Target
-- 
GitLab


From a14a83d9a102253eca7c02ff4c35a2ce3f7de6e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Thu, 24 Oct 2024 23:45:14 +0300
Subject: [PATCH 092/693] [compiler-rt] [test] Fix using toolchains that rely
 on Clang default configs (#113491)

The use of CLANG_NO_DEFAULT_CONFIG in the tests was added because some
Linux distributions had a global default config file, that added flags
relating to hardening, which interfere with the sanitizer tests. By
setting CLANG_NO_DEFAULT_CONFIG, the global default config files that
are found are ignored, and the sanitizers get the expected default
compiler behaviour.

(This was https://github.com/llvm/llvm-project/issues/60394, which was
fixed in 8ab762557fb057af1a3015211ee116a975027e78.)

However, some toolchains may rely on default config files for mandatory
parts required for functioning at all - setting things like sysroots,
-rtlib, -unwindlib, -stdlib, -fuse-ld etc. In such a case we can't
forcibly disable any default config, because it will break the otherwise
working toolchain.

Add a test for whether the compiler works while passing
--no-default-config to it. If the option is accepted and the toolchain
still works while that is set, set CLANG_NO_DEFAULT_CONFIG while running
tests.

(This adds a little bit of inconsistency, as we're testing for the
command line option, while using the environment variable. However doing
compile testing with an environment variable isn't quite as easily
doable, and passing an extra command line flag to all compile commands
while testing, is a bit clumsy - therefore this inconsistency.)
---
 compiler-rt/CMakeLists.txt                | 16 ++++++++++++++++
 compiler-rt/test/CMakeLists.txt           |  2 ++
 compiler-rt/test/lit.common.cfg.py        |  6 +++++-
 compiler-rt/test/lit.common.configured.in |  1 +
 4 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt
index f6072aa40fad..20054c6e85a4 100644
--- a/compiler-rt/CMakeLists.txt
+++ b/compiler-rt/CMakeLists.txt
@@ -44,6 +44,22 @@ include(CompilerRTUtils)
 include(CMakeDependentOption)
 include(GetDarwinLinkerVersion)
 
+include(CheckCXXCompilerFlag)
+
+# Check if we can compile with --no-default-config, or if that omits a config
+# file that is essential for the toolchain to work properly.
+#
+# Using CMAKE_REQUIRED_FLAGS to make sure the flag is used both for compilation
+# and for linking.
+#
+# Doing this test early on, to see if the flag works on the toolchain
+# out of the box. Later on, we end up adding -nostdlib and similar flags
+# to all test compiles, which easily can give false positives on this test.
+set(OLD_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}")
+set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} --no-default-config")
+check_cxx_compiler_flag("" COMPILER_RT_HAS_NO_DEFAULT_CONFIG_FLAG)
+set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS}")
+
 option(COMPILER_RT_BUILD_BUILTINS "Build builtins" ON)
 mark_as_advanced(COMPILER_RT_BUILD_BUILTINS)
 option(COMPILER_RT_DISABLE_AARCH64_FMV "Disable AArch64 Function Multi Versioning support" OFF)
diff --git a/compiler-rt/test/CMakeLists.txt b/compiler-rt/test/CMakeLists.txt
index 84a98f367474..f9e23710d3e4 100644
--- a/compiler-rt/test/CMakeLists.txt
+++ b/compiler-rt/test/CMakeLists.txt
@@ -12,6 +12,8 @@ pythonize_bool(COMPILER_RT_ENABLE_INTERNAL_SYMBOLIZER)
 
 pythonize_bool(COMPILER_RT_HAS_AARCH64_SME)
 
+pythonize_bool(COMPILER_RT_HAS_NO_DEFAULT_CONFIG_FLAG)
+
 configure_compiler_rt_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.common.configured.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.common.configured)
diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py
index c3435dd2c52a..4bbce4c07366 100644
--- a/compiler-rt/test/lit.common.cfg.py
+++ b/compiler-rt/test/lit.common.cfg.py
@@ -993,7 +993,11 @@ if config.host_os == "Darwin":
 # default configs for the test runs. In particular, anything hardening
 # related is likely to cause issues with sanitizer tests, because it may
 # preempt something we're looking to trap (e.g. _FORTIFY_SOURCE vs our ASAN).
-config.environment["CLANG_NO_DEFAULT_CONFIG"] = "1"
+#
+# Only set this if we know we can still build for the target while disabling
+# default configs.
+if config.has_no_default_config_flag:
+    config.environment["CLANG_NO_DEFAULT_CONFIG"] = "1"
 
 if config.has_compiler_rt_libatomic:
   base_lib = os.path.join(config.compiler_rt_libdir, "libclang_rt.atomic%s.so"
diff --git a/compiler-rt/test/lit.common.configured.in b/compiler-rt/test/lit.common.configured.in
index 049133dc965c..66935c358afe 100644
--- a/compiler-rt/test/lit.common.configured.in
+++ b/compiler-rt/test/lit.common.configured.in
@@ -53,6 +53,7 @@ set_default("test_standalone_build_libs", @COMPILER_RT_TEST_STANDALONE_BUILD_LIB
 set_default("has_compiler_rt_libatomic", @COMPILER_RT_BUILD_STANDALONE_LIBATOMIC_PYBOOL@)
 set_default("aarch64_sme", @COMPILER_RT_HAS_AARCH64_SME_PYBOOL@)
 set_default("darwin_linker_version", "@COMPILER_RT_DARWIN_LINKER_VERSION@")
+set_default("has_no_default_config_flag", @COMPILER_RT_HAS_NO_DEFAULT_CONFIG_FLAG_PYBOOL@)
 # True iff the test suite supports ignoring the test compiler's runtime library path
 # and using `config.compiler_rt_libdir` instead. This only matters when the runtime
 # library paths differ.
-- 
GitLab


From 75d0281bc81f0040c24d15bdf9c5cc46e9237224 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Thu, 24 Oct 2024 23:46:04 +0300
Subject: [PATCH 093/693] [runtimes] Probe for -nostdlib++ and -nostdinc++ with
 the C compiler (#108357)

While these flags semantically are relevant only for C++, we do add them
to CMAKE_REQUIRED_FLAGS if they are detected. All flags in that variable
are used both when testing compilation of C and C++ (and for detecting
libraries, which uses the C compiler driver).

Therefore, to be sure we safely can add the flags to
CMAKE_REQUIRED_FLAGS, test for the option with the C language.

This should fix compilation with GCC; newer versions of GCC do support
the -nostdlib++ option, but it's only supported by the C++ compiler
driver, not the C driver. (However, many builds of GCC also do accept
the option with the C driver, if GCC was compiled with Ada support
enabled, see [1]. That's why this issue isn't noticed in all
configurations with GCC.)

Clang does support these options in both C and C++ driver modes.

This should fix #90332.

[1]
https://github.com/llvm/llvm-project/issues/90332#issuecomment-2325099254
---
 libcxx/cmake/config-ix.cmake    | 12 ++++++++----
 libcxxabi/cmake/config-ix.cmake | 12 ++++++++----
 libcxxabi/src/CMakeLists.txt    |  4 ++--
 libunwind/cmake/config-ix.cmake | 12 ++++++++----
 libunwind/src/CMakeLists.txt    |  2 +-
 runtimes/CMakeLists.txt         | 12 ++++++++----
 6 files changed, 35 insertions(+), 19 deletions(-)

diff --git a/libcxx/cmake/config-ix.cmake b/libcxx/cmake/config-ix.cmake
index 270d80575adc..192bad7a7a7f 100644
--- a/libcxx/cmake/config-ix.cmake
+++ b/libcxx/cmake/config-ix.cmake
@@ -38,9 +38,13 @@ check_cxx_compiler_flag(-nolibc CXX_SUPPORTS_NOLIBC_FLAG)
 # required during compilation (which has the -nostdlib++ or -nodefaultlibs). libc is
 # required for the link to go through. We remove sanitizers from the
 # configuration checks to avoid spurious link errors.
+#
+# Adding flags to CMAKE_REQUIRED_FLAGS will include the flags both when testing
+# compilation of C and C++. Therefore test to make sure that the flags are
+# supported by the C compiler driver, before deciding to include them.
 
-check_cxx_compiler_flag(-nostdlib++ CXX_SUPPORTS_NOSTDLIBXX_FLAG)
-if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
+check_c_compiler_flag(-nostdlib++ C_SUPPORTS_NOSTDLIBXX_FLAG)
+if (C_SUPPORTS_NOSTDLIBXX_FLAG)
   set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdlib++")
 else()
   check_c_compiler_flag(-nodefaultlibs C_SUPPORTS_NODEFAULTLIBS_FLAG)
@@ -51,7 +55,7 @@ endif()
 
 # Only link against compiler-rt manually if we use -nodefaultlibs, since
 # otherwise the compiler will do the right thing on its own.
-if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
+if (NOT C_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (LIBCXX_USE_COMPILER_RT)
     include(HandleCompilerRT)
     find_compiler_rt_library(builtins LIBCXX_BUILTINS_LIBRARY
@@ -81,7 +85,7 @@ if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
   endif()
 endif()
 
-if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
+if (C_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (CMAKE_C_FLAGS MATCHES -fsanitize OR CMAKE_CXX_FLAGS MATCHES -fsanitize)
     set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -fno-sanitize=all")
   endif ()
diff --git a/libcxxabi/cmake/config-ix.cmake b/libcxxabi/cmake/config-ix.cmake
index 10f2087c68c5..ab74ad79a654 100644
--- a/libcxxabi/cmake/config-ix.cmake
+++ b/libcxxabi/cmake/config-ix.cmake
@@ -22,9 +22,13 @@ endif ()
 # required during compilation (which has the -nodefaultlibs). libc is
 # required for the link to go through. We remove sanitizers from the
 # configuration checks to avoid spurious link errors.
+#
+# Adding flags to CMAKE_REQUIRED_FLAGS will include the flags both when testing
+# compilation of C and C++. Therefore test to make sure that the flags are
+# supported by the C compiler driver, before deciding to include them.
 
-check_cxx_compiler_flag(-nostdlib++ CXX_SUPPORTS_NOSTDLIBXX_FLAG)
-if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
+check_c_compiler_flag(-nostdlib++ C_SUPPORTS_NOSTDLIBXX_FLAG)
+if (C_SUPPORTS_NOSTDLIBXX_FLAG)
   set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdlib++")
 else()
   check_c_compiler_flag(-nodefaultlibs C_SUPPORTS_NODEFAULTLIBS_FLAG)
@@ -35,7 +39,7 @@ endif()
 
 # Only link against compiler-rt manually if we use -nodefaultlibs, since
 # otherwise the compiler will do the right thing on its own.
-if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
+if (NOT C_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (LIBCXXABI_HAS_C_LIB)
     list(APPEND CMAKE_REQUIRED_LIBRARIES c)
   endif ()
@@ -71,7 +75,7 @@ if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
   endif()
 endif()
 
-if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
+if (C_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (CMAKE_C_FLAGS MATCHES -fsanitize OR CMAKE_CXX_FLAGS MATCHES -fsanitize)
     set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -fno-sanitize=all")
   endif ()
diff --git a/libcxxabi/src/CMakeLists.txt b/libcxxabi/src/CMakeLists.txt
index 84fe2784bec5..af8071cbdc85 100644
--- a/libcxxabi/src/CMakeLists.txt
+++ b/libcxxabi/src/CMakeLists.txt
@@ -91,7 +91,7 @@ if (ANDROID AND ANDROID_PLATFORM_LEVEL LESS 21)
 endif()
 
 # Setup flags.
-if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
+if (C_SUPPORTS_NOSTDLIBXX_FLAG)
   add_link_flags_if_supported(-nostdlib++)
 else()
   add_link_flags_if_supported(-nodefaultlibs)
@@ -167,7 +167,7 @@ if (LIBCXXABI_USE_LLVM_UNWINDER)
   endif()
 endif()
 target_link_libraries(cxxabi_shared_objects PRIVATE cxx-headers ${LIBCXXABI_LIBRARIES})
-if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG)
+if (NOT C_SUPPORTS_NOSTDLIBXX_FLAG)
   target_link_libraries(cxxabi_shared_objects PRIVATE ${LIBCXXABI_BUILTINS_LIBRARY})
 endif()
 target_link_libraries(cxxabi_shared_objects PUBLIC cxxabi-headers)
diff --git a/libunwind/cmake/config-ix.cmake b/libunwind/cmake/config-ix.cmake
index 126c872f0d48..88a13b4ec0e4 100644
--- a/libunwind/cmake/config-ix.cmake
+++ b/libunwind/cmake/config-ix.cmake
@@ -34,9 +34,13 @@ endif()
 # required during compilation (which has the -nostdlib++ or -nodefaultlibs). libc is
 # required for the link to go through. We remove sanitizers from the
 # configuration checks to avoid spurious link errors.
+#
+# Adding flags to CMAKE_REQUIRED_FLAGS will include the flags both when testing
+# compilation of C and C++. Therefore test to make sure that the flags are
+# supported by the C compiler driver, before deciding to include them.
 
-llvm_check_compiler_linker_flag(CXX "-nostdlib++" CXX_SUPPORTS_NOSTDLIBXX_FLAG)
-if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
+llvm_check_compiler_linker_flag(C "-nostdlib++" C_SUPPORTS_NOSTDLIBXX_FLAG)
+if (C_SUPPORTS_NOSTDLIBXX_FLAG)
   set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdlib++")
 else()
   llvm_check_compiler_linker_flag(C "-nodefaultlibs" C_SUPPORTS_NODEFAULTLIBS_FLAG)
@@ -47,7 +51,7 @@ endif()
 
 # Only link against compiler-rt manually if we use -nodefaultlibs, since
 # otherwise the compiler will do the right thing on its own.
-if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
+if (NOT C_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (LIBUNWIND_HAS_C_LIB)
     list(APPEND CMAKE_REQUIRED_LIBRARIES c)
   endif ()
@@ -82,7 +86,7 @@ if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
   endif()
 endif()
 
-if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
+if (C_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (CMAKE_C_FLAGS MATCHES -fsanitize OR CMAKE_CXX_FLAGS MATCHES -fsanitize)
     set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -fno-sanitize=all")
   endif ()
diff --git a/libunwind/src/CMakeLists.txt b/libunwind/src/CMakeLists.txt
index 2e18b1096563..c58b848026c8 100644
--- a/libunwind/src/CMakeLists.txt
+++ b/libunwind/src/CMakeLists.txt
@@ -66,7 +66,7 @@ set(LIBUNWIND_SOURCES
     ${LIBUNWIND_ASM_SOURCES})
 
 # Generate library list.
-if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
+if (C_SUPPORTS_NOSTDLIBXX_FLAG)
   add_link_flags_if_supported(-nostdlib++)
 else()
   if (LIBUNWIND_USE_COMPILER_RT)
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 830165c799c2..8db2b3fb9309 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -143,12 +143,16 @@ endif()
 # Check for -nostdlib++ first; if there's no C++ standard library yet,
 # all check_cxx_compiler_flag commands will fail until we add -nostdlib++
 # (or -nodefaultlibs).
-llvm_check_compiler_linker_flag(CXX "-nostdlib++" CXX_SUPPORTS_NOSTDLIBXX_FLAG)
-if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
+#
+# Adding flags to CMAKE_REQUIRED_FLAGS will include the flags both when testing
+# compilation of C and C++. Therefore test to make sure that the flags are
+# supported by the C compiler driver, before deciding to include them.
+llvm_check_compiler_linker_flag(C "-nostdlib++" C_SUPPORTS_NOSTDLIBXX_FLAG)
+if (C_SUPPORTS_NOSTDLIBXX_FLAG)
   set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdlib++")
 endif()
-check_cxx_compiler_flag(-nostdinc++ CXX_SUPPORTS_NOSTDINCXX_FLAG)
-if (CXX_SUPPORTS_NOSTDINCXX_FLAG)
+check_c_compiler_flag(-nostdinc++ C_SUPPORTS_NOSTDINCXX_FLAG)
+if (C_SUPPORTS_NOSTDINCXX_FLAG)
   set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdinc++")
 endif()
 
-- 
GitLab


From 1b8cff9a52eab8718ba55996847ece0a96271bb7 Mon Sep 17 00:00:00 2001
From: Daniel Hoekwater <hoekwater@google.com>
Date: Thu, 24 Oct 2024 17:07:27 -0400
Subject: [PATCH 094/693] Reland "CFIFixup] Factor CFI remember/restore
 insertion into a helper (NFC)" (#113387)

The original patch (ac1a01f) dereferenced an end iterator, breaking some
tests (e.g. https://lab.llvm.org/buildbot/#/builders/17/builds/3116).
This updated patch only accesses the iterator when it's valid.
---
 llvm/lib/CodeGen/CFIFixup.cpp | 60 ++++++++++++++++++++++++-----------
 1 file changed, 42 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/CodeGen/CFIFixup.cpp b/llvm/lib/CodeGen/CFIFixup.cpp
index 61888a426665..ccf0738c197e 100644
--- a/llvm/lib/CodeGen/CFIFixup.cpp
+++ b/llvm/lib/CodeGen/CFIFixup.cpp
@@ -116,6 +116,41 @@ findPrologueEnd(MachineFunction &MF, MachineBasicBlock::iterator &PrologueEnd) {
   return nullptr;
 }
 
+// Represents the point within a basic block where we can insert an instruction.
+// Note that we need the MachineBasicBlock* as well as the iterator since the
+// iterator can point to the end of the block. Instructions are inserted
+// *before* the iterator.
+struct InsertionPoint {
+  MachineBasicBlock *MBB;
+  MachineBasicBlock::iterator Iterator;
+};
+
+// Inserts a `.cfi_remember_state` instruction before PrologueEnd and a
+// `.cfi_restore_state` instruction before DstInsertPt. Returns an iterator
+// to the first instruction after the inserted `.cfi_restore_state` instruction.
+static InsertionPoint
+insertRememberRestorePair(const InsertionPoint &RememberInsertPt,
+                          const InsertionPoint &RestoreInsertPt) {
+  MachineFunction &MF = *RememberInsertPt.MBB->getParent();
+  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
+
+  // Insert the `.cfi_remember_state` instruction.
+  unsigned CFIIndex =
+      MF.addFrameInst(MCCFIInstruction::createRememberState(nullptr));
+  BuildMI(*RememberInsertPt.MBB, RememberInsertPt.Iterator, DebugLoc(),
+          TII.get(TargetOpcode::CFI_INSTRUCTION))
+      .addCFIIndex(CFIIndex);
+
+  // Insert the `.cfi_restore_state` instruction.
+  CFIIndex = MF.addFrameInst(MCCFIInstruction::createRestoreState(nullptr));
+
+  return {RestoreInsertPt.MBB,
+          std::next(BuildMI(*RestoreInsertPt.MBB, RestoreInsertPt.Iterator,
+                            DebugLoc(), TII.get(TargetOpcode::CFI_INSTRUCTION))
+                        .addCFIIndex(CFIIndex)
+                        ->getIterator())};
+}
+
 bool CFIFixup::runOnMachineFunction(MachineFunction &MF) {
   const TargetFrameLowering &TFL = *MF.getSubtarget().getFrameLowering();
   if (!TFL.enableCFIFixup(MF))
@@ -174,15 +209,13 @@ bool CFIFixup::runOnMachineFunction(MachineFunction &MF) {
   // Every block inherits the frame state (as recorded in the unwind tables)
   // of the previous block. If the intended frame state is different, insert
   // compensating CFI instructions.
-  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
   bool Change = false;
   // `InsertPt` always points to the point in a preceding block where we have to
   // insert a `.cfi_remember_state`, in the case that the current block needs a
   // `.cfi_restore_state`.
-  MachineBasicBlock *InsertMBB = PrologueBlock;
-  MachineBasicBlock::iterator InsertPt = PrologueEnd;
+  InsertionPoint InsertPt = {PrologueBlock, PrologueEnd};
 
-  assert(InsertPt != PrologueBlock->begin() &&
+  assert(PrologueEnd != PrologueBlock->begin() &&
          "Inconsistent notion of \"prologue block\"");
 
   // No point starting before the prologue block.
@@ -210,20 +243,11 @@ bool CFIFixup::runOnMachineFunction(MachineFunction &MF) {
     if (!Info.StrongNoFrameOnEntry && Info.HasFrameOnEntry && !HasFrame) {
       // Reset to the "after prologue" state.
 
-      // Insert a `.cfi_remember_state` into the last block known to have a
-      // stack frame.
-      unsigned CFIIndex =
-          MF.addFrameInst(MCCFIInstruction::createRememberState(nullptr));
-      BuildMI(*InsertMBB, InsertPt, DebugLoc(),
-              TII.get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex);
-      // Insert a `.cfi_restore_state` at the beginning of the current block.
-      CFIIndex = MF.addFrameInst(MCCFIInstruction::createRestoreState(nullptr));
-      InsertPt = BuildMI(*CurrBB, CurrBB->begin(), DebugLoc(),
-                         TII.get(TargetOpcode::CFI_INSTRUCTION))
-                     .addCFIIndex(CFIIndex);
-      ++InsertPt;
-      InsertMBB = &*CurrBB;
+      // There's an earlier block known to have a stack frame. Insert a
+      // `.cfi_remember_state` instruction into that block and a
+      // `.cfi_restore_state` instruction at the beginning of the current block.
+      InsertPt = insertRememberRestorePair(
+          InsertPt, InsertionPoint{&*CurrBB, CurrBB->begin()});
       Change = true;
     } else if ((Info.StrongNoFrameOnEntry || !Info.HasFrameOnEntry) &&
                HasFrame) {
-- 
GitLab


From 4c8779388fd2f25730a1b044f9f10d6717833fce Mon Sep 17 00:00:00 2001
From: Alexander Richardson <alexrichardson@google.com>
Date: Thu, 24 Oct 2024 14:09:05 -0700
Subject: [PATCH 095/693] sanitizer_allocator.cpp: Ensure at least
 sizeof(void*) alignment

Some platforms (e.g. 64-bit CHERI) have stronger alignment requirements
on values returned from allocators. For all other platforms this does
not result in any functional change.

Reviewed By: cjappl, vitalybuka

Pull Request: https://github.com/llvm/llvm-project/pull/84440
---
 compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
index 1d5058c81acb..9d899371c2dd 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
@@ -59,7 +59,7 @@ static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache,
 
 static void *RawInternalRealloc(void *ptr, uptr size,
                                 InternalAllocatorCache *cache) {
-  uptr alignment = 8;
+  constexpr usize alignment = Max<usize>(8, sizeof(void *));
   if (cache == 0) {
     SpinMutexLock l(&internal_allocator_cache_mu);
     return internal_allocator()->Reallocate(&internal_allocator_cache, ptr,
@@ -137,7 +137,8 @@ void InternalAllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
 }
 
 // LowLevelAllocator
-constexpr uptr kLowLevelAllocatorDefaultAlignment = 8;
+constexpr usize kLowLevelAllocatorDefaultAlignment =
+    Max<usize>(8, sizeof(void *));
 constexpr uptr kMinNumPagesRounded = 16;
 constexpr uptr kMinRoundedSize = 65536;
 static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment;
-- 
GitLab


From 118445841d4d50b57ea2892a18f0b656526f804d Mon Sep 17 00:00:00 2001
From: Dan Gohman <dev@sunfishcode.online>
Date: Thu, 24 Oct 2024 14:13:58 -0700
Subject: [PATCH 096/693] [WebAssembly] Protect memory.fill and memory.copy
 from zero-length ranges. (#112617)

WebAssembly's `memory.fill` and `memory.copy` instructions trap if the
pointers are out of bounds, even if the length is zero. This is
different from LLVM, which expects that it can call `memcpy` on
arbitrary invalid pointers if the length is zero. To avoid spurious
traps, branch around `memory.fill` and `memory.copy` when the length is
zero.

---------

Co-authored-by: Heejin Ahn <aheejin@gmail.com>
---
 .../lib/Target/WebAssembly/WebAssemblyISD.def |   9 +-
 .../WebAssembly/WebAssemblyISelLowering.cpp   | 140 +++++++++++++++++
 .../WebAssembly/WebAssemblyInstrBulkMemory.td |  73 ++++++---
 .../WebAssemblySelectionDAGInfo.cpp           |  19 ++-
 llvm/test/CodeGen/WebAssembly/bulk-memory.ll  | 102 ++++++++++---
 .../test/CodeGen/WebAssembly/bulk-memory64.ll | 141 +++++++++++++-----
 6 files changed, 404 insertions(+), 80 deletions(-)

diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
index f1ef58ef1f34..3502c47016c6 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
@@ -42,8 +42,6 @@ HANDLE_NODETYPE(PROMOTE_LOW)
 HANDLE_NODETYPE(TRUNC_SAT_ZERO_S)
 HANDLE_NODETYPE(TRUNC_SAT_ZERO_U)
 HANDLE_NODETYPE(DEMOTE_ZERO)
-HANDLE_NODETYPE(MEMORY_COPY)
-HANDLE_NODETYPE(MEMORY_FILL)
 HANDLE_NODETYPE(I64_ADD128)
 HANDLE_NODETYPE(I64_SUB128)
 HANDLE_NODETYPE(I64_MUL_WIDE_S)
@@ -54,3 +52,10 @@ HANDLE_MEM_NODETYPE(GLOBAL_GET)
 HANDLE_MEM_NODETYPE(GLOBAL_SET)
 HANDLE_MEM_NODETYPE(TABLE_GET)
 HANDLE_MEM_NODETYPE(TABLE_SET)
+
+// Bulk memory instructions. These follow LLVM's expected semantics of
+// supporting out-of-bounds pointers if the length is zero, by inserting
+// a branch around Wasm's `memory.copy` and `memory.fill`, which would
+// otherwise trap.
+HANDLE_NODETYPE(MEMCPY)
+HANDLE_NODETYPE(MEMSET)
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index fb9f980cfff2..629e033336f9 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -568,6 +568,138 @@ static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL,
   return DoneMBB;
 }
 
+// Lower a `MEMCPY` instruction into a CFG triangle around a `MEMORY_COPY`
+// instuction to handle the zero-length case.
+static MachineBasicBlock *LowerMemcpy(MachineInstr &MI, DebugLoc DL,
+                                      MachineBasicBlock *BB,
+                                      const TargetInstrInfo &TII, bool Int64) {
+  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
+
+  MachineOperand DstMem = MI.getOperand(0);
+  MachineOperand SrcMem = MI.getOperand(1);
+  MachineOperand Dst = MI.getOperand(2);
+  MachineOperand Src = MI.getOperand(3);
+  MachineOperand Len = MI.getOperand(4);
+
+  // We're going to add an extra use to `Len` to test if it's zero; that
+  // use shouldn't be a kill, even if the original use is.
+  MachineOperand NoKillLen = Len;
+  NoKillLen.setIsKill(false);
+
+  // Decide on which `MachineInstr` opcode we're going to use.
+  unsigned Eqz = Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32;
+  unsigned MemoryCopy =
+      Int64 ? WebAssembly::MEMORY_COPY_A64 : WebAssembly::MEMORY_COPY_A32;
+
+  // Create two new basic blocks; one for the new `memory.fill` that we can
+  // branch over, and one for the rest of the instructions after the original
+  // `memory.fill`.
+  const BasicBlock *LLVMBB = BB->getBasicBlock();
+  MachineFunction *F = BB->getParent();
+  MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
+  MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);
+
+  MachineFunction::iterator It = ++BB->getIterator();
+  F->insert(It, TrueMBB);
+  F->insert(It, DoneMBB);
+
+  // Transfer the remainder of BB and its successor edges to DoneMBB.
+  DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
+  DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
+
+  // Connect the CFG edges.
+  BB->addSuccessor(TrueMBB);
+  BB->addSuccessor(DoneMBB);
+  TrueMBB->addSuccessor(DoneMBB);
+
+  // Create a virtual register for the `Eqz` result.
+  unsigned EqzReg;
+  EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
+
+  // Erase the original `memory.copy`.
+  MI.eraseFromParent();
+
+  // Test if `Len` is zero.
+  BuildMI(BB, DL, TII.get(Eqz), EqzReg).add(NoKillLen);
+
+  // Insert a new `memory.copy`.
+  BuildMI(TrueMBB, DL, TII.get(MemoryCopy))
+      .add(DstMem)
+      .add(SrcMem)
+      .add(Dst)
+      .add(Src)
+      .add(Len);
+
+  // Create the CFG triangle.
+  BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(DoneMBB).addReg(EqzReg);
+  BuildMI(TrueMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
+
+  return DoneMBB;
+}
+
+// Lower a `MEMSET` instruction into a CFG triangle around a `MEMORY_FILL`
+// instuction to handle the zero-length case.
+static MachineBasicBlock *LowerMemset(MachineInstr &MI, DebugLoc DL,
+                                      MachineBasicBlock *BB,
+                                      const TargetInstrInfo &TII, bool Int64) {
+  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
+
+  MachineOperand Mem = MI.getOperand(0);
+  MachineOperand Dst = MI.getOperand(1);
+  MachineOperand Val = MI.getOperand(2);
+  MachineOperand Len = MI.getOperand(3);
+
+  // We're going to add an extra use to `Len` to test if it's zero; that
+  // use shouldn't be a kill, even if the original use is.
+  MachineOperand NoKillLen = Len;
+  NoKillLen.setIsKill(false);
+
+  // Decide on which `MachineInstr` opcode we're going to use.
+  unsigned Eqz = Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32;
+  unsigned MemoryFill =
+      Int64 ? WebAssembly::MEMORY_FILL_A64 : WebAssembly::MEMORY_FILL_A32;
+
+  // Create two new basic blocks; one for the new `memory.fill` that we can
+  // branch over, and one for the rest of the instructions after the original
+  // `memory.fill`.
+  const BasicBlock *LLVMBB = BB->getBasicBlock();
+  MachineFunction *F = BB->getParent();
+  MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
+  MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);
+
+  MachineFunction::iterator It = ++BB->getIterator();
+  F->insert(It, TrueMBB);
+  F->insert(It, DoneMBB);
+
+  // Transfer the remainder of BB and its successor edges to DoneMBB.
+  DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
+  DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
+
+  // Connect the CFG edges.
+  BB->addSuccessor(TrueMBB);
+  BB->addSuccessor(DoneMBB);
+  TrueMBB->addSuccessor(DoneMBB);
+
+  // Create a virtual register for the `Eqz` result.
+  unsigned EqzReg;
+  EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
+
+  // Erase the original `memory.fill`.
+  MI.eraseFromParent();
+
+  // Test if `Len` is zero.
+  BuildMI(BB, DL, TII.get(Eqz), EqzReg).add(NoKillLen);
+
+  // Insert a new `memory.copy`.
+  BuildMI(TrueMBB, DL, TII.get(MemoryFill)).add(Mem).add(Dst).add(Val).add(Len);
+
+  // Create the CFG triangle.
+  BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(DoneMBB).addReg(EqzReg);
+  BuildMI(TrueMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
+
+  return DoneMBB;
+}
+
 static MachineBasicBlock *
 LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB,
                  const WebAssemblySubtarget *Subtarget,
@@ -725,6 +857,14 @@ MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
   case WebAssembly::FP_TO_UINT_I64_F64:
     return LowerFPToInt(MI, DL, BB, TII, true, true, true,
                         WebAssembly::I64_TRUNC_U_F64);
+  case WebAssembly::MEMCPY_A32:
+    return LowerMemcpy(MI, DL, BB, TII, false);
+  case WebAssembly::MEMCPY_A64:
+    return LowerMemcpy(MI, DL, BB, TII, true);
+  case WebAssembly::MEMSET_A32:
+    return LowerMemset(MI, DL, BB, TII, false);
+  case WebAssembly::MEMSET_A64:
+    return LowerMemset(MI, DL, BB, TII, true);
   case WebAssembly::CALL_RESULTS:
   case WebAssembly::RET_CALL_RESULTS:
     return LowerCallResults(MI, DL, BB, Subtarget, TII);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td
index 7aeae54d95a8..0772afb039f8 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td
@@ -21,22 +21,31 @@ multiclass BULK_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
 }
 
 // Bespoke types and nodes for bulk memory ops
+
 def wasm_memcpy_t : SDTypeProfile<0, 5,
   [SDTCisInt<0>, SDTCisInt<1>, SDTCisPtrTy<2>, SDTCisPtrTy<3>, SDTCisInt<4>]
 >;
-def wasm_memcpy : SDNode<"WebAssemblyISD::MEMORY_COPY", wasm_memcpy_t,
-                         [SDNPHasChain, SDNPMayLoad, SDNPMayStore]>;
-
 def wasm_memset_t : SDTypeProfile<0, 4,
   [SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisInt<2>, SDTCisInt<3>]
 >;
-def wasm_memset : SDNode<"WebAssemblyISD::MEMORY_FILL", wasm_memset_t,
+
+// memory.copy with a branch to avoid trapping in the case of out-of-bounds
+// pointers with empty ranges.
+def wasm_memcpy : SDNode<"WebAssemblyISD::MEMCPY", wasm_memcpy_t,
+                         [SDNPHasChain, SDNPMayLoad, SDNPMayStore]>;
+
+// memory.fill with a branch to avoid trapping in the case of out-of-bounds
+// pointers with empty ranges.
+def wasm_memset : SDNode<"WebAssemblyISD::MEMSET", wasm_memset_t,
                          [SDNPHasChain, SDNPMayStore]>;
 
+// A multiclass for defining Wasm's raw bulk-memory `memory.*` instructions.
+// `memory.copy` and `memory.fill` have Wasm's behavior rather than
+// `memcpy`/`memset` behavior.
 multiclass BulkMemoryOps<WebAssemblyRegClass rc, string B> {
 
 let mayStore = 1, hasSideEffects = 1 in
-defm MEMORY_INIT_A#B :
+defm INIT_A#B :
   BULK_I<(outs),
          (ins i32imm_op:$seg, i32imm_op:$idx, rc:$dest,
               I32:$offset, I32:$size),
@@ -45,31 +54,57 @@ defm MEMORY_INIT_A#B :
          "memory.init\t$seg, $idx, $dest, $offset, $size",
          "memory.init\t$seg, $idx", 0x08>;
 
-let hasSideEffects = 1 in
-defm DATA_DROP :
-  BULK_I<(outs), (ins i32imm_op:$seg), (outs), (ins i32imm_op:$seg),
-         [],
-         "data.drop\t$seg", "data.drop\t$seg", 0x09>;
-
 let mayLoad = 1, mayStore = 1 in
-defm MEMORY_COPY_A#B :
+defm COPY_A#B :
   BULK_I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx,
                       rc:$dst, rc:$src, rc:$len),
          (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx),
-         [(wasm_memcpy (i32 imm:$src_idx), (i32 imm:$dst_idx),
-           rc:$dst, rc:$src, rc:$len
-         )],
+         [],
          "memory.copy\t$src_idx, $dst_idx, $dst, $src, $len",
          "memory.copy\t$src_idx, $dst_idx", 0x0a>;
 
 let mayStore = 1 in
-defm MEMORY_FILL_A#B :
+defm FILL_A#B :
   BULK_I<(outs), (ins i32imm_op:$idx, rc:$dst, I32:$value, rc:$size),
          (outs), (ins i32imm_op:$idx),
-         [(wasm_memset (i32 imm:$idx), rc:$dst, I32:$value, rc:$size)],
+         [],
          "memory.fill\t$idx, $dst, $value, $size",
          "memory.fill\t$idx", 0x0b>;
 }
 
-defm : BulkMemoryOps<I32, "32">;
-defm : BulkMemoryOps<I64, "64">;
+defm MEMORY_ : BulkMemoryOps<I32, "32">;
+defm MEMORY_ : BulkMemoryOps<I64, "64">;
+
+// A multiclass for defining `memcpy`/`memset` pseudo instructions. These have
+// the behavior the rest of LLVM CodeGen expects, and we lower them into code
+// sequences that include the Wasm `memory.fill` and `memory.copy` instructions
+// using custom inserters, because they introduce new control flow.
+multiclass BulkMemOps<WebAssemblyRegClass rc, string B> {
+
+let usesCustomInserter = 1, isCodeGenOnly = 1, mayLoad = 1, mayStore = 1 in
+defm CPY_A#B : I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx,
+                              rc:$dst, rc:$src, rc:$len),
+                 (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx),
+                 [(wasm_memcpy (i32 imm:$src_idx), (i32 imm:$dst_idx),
+                   rc:$dst, rc:$src, rc:$len
+                 )],
+                 "", "", 0>,
+                  Requires<[HasBulkMemory]>;
+
+let usesCustomInserter = 1, isCodeGenOnly = 1, mayStore = 1 in
+defm SET_A#B : I<(outs), (ins i32imm_op:$idx, rc:$dst, I32:$value, rc:$size),
+                 (outs), (ins i32imm_op:$idx),
+                 [(wasm_memset (i32 imm:$idx), rc:$dst, I32:$value, rc:$size)],
+                 "", "", 0>,
+                 Requires<[HasBulkMemory]>;
+
+}
+
+defm MEM : BulkMemOps<I32, "32">;
+defm MEM : BulkMemOps<I64, "64">;
+
+let hasSideEffects = 1 in
+defm DATA_DROP :
+  BULK_I<(outs), (ins i32imm_op:$seg), (outs), (ins i32imm_op:$seg),
+         [],
+         "data.drop\t$seg", "data.drop\t$seg", 0x09>;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp
index 74af4c8873f7..d51bfeb6d859 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp
@@ -28,9 +28,13 @@ SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemcpy(
 
   SDValue MemIdx = DAG.getConstant(0, DL, MVT::i32);
   auto LenMVT = ST.hasAddr64() ? MVT::i64 : MVT::i32;
-  return DAG.getNode(WebAssemblyISD::MEMORY_COPY, DL, MVT::Other,
-                     {Chain, MemIdx, MemIdx, Dst, Src,
-                      DAG.getZExtOrTrunc(Size, DL, LenMVT)});
+
+  // Use `MEMCPY` here instead of `MEMORY_COPY` because `memory.copy` traps
+  // if the pointers are invalid even if the length is zero. `MEMCPY` gets
+  // extra code to handle this in the way that LLVM IR expects.
+  return DAG.getNode(
+      WebAssemblyISD::MEMCPY, DL, MVT::Other,
+      {Chain, MemIdx, MemIdx, Dst, Src, DAG.getZExtOrTrunc(Size, DL, LenMVT)});
 }
 
 SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemmove(
@@ -52,8 +56,13 @@ SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemset(
 
   SDValue MemIdx = DAG.getConstant(0, DL, MVT::i32);
   auto LenMVT = ST.hasAddr64() ? MVT::i64 : MVT::i32;
+
+  // Use `MEMSET` here instead of `MEMORY_FILL` because `memory.fill` traps
+  // if the pointers are invalid even if the length is zero. `MEMSET` gets
+  // extra code to handle this in the way that LLVM IR expects.
+  //
   // Only low byte matters for val argument, so anyext the i8
-  return DAG.getNode(WebAssemblyISD::MEMORY_FILL, DL, MVT::Other, Chain, MemIdx,
-                     Dst, DAG.getAnyExtOrTrunc(Val, DL, MVT::i32),
+  return DAG.getNode(WebAssemblyISD::MEMSET, DL, MVT::Other, Chain, MemIdx, Dst,
+                     DAG.getAnyExtOrTrunc(Val, DL, MVT::i32),
                      DAG.getZExtOrTrunc(Size, DL, LenMVT));
 }
diff --git a/llvm/test/CodeGen/WebAssembly/bulk-memory.ll b/llvm/test/CodeGen/WebAssembly/bulk-memory.ll
index dc29dc81c13e..ae170d757a30 100644
--- a/llvm/test/CodeGen/WebAssembly/bulk-memory.ll
+++ b/llvm/test/CodeGen/WebAssembly/bulk-memory.ll
@@ -17,7 +17,12 @@ declare void @llvm.memset.p0.i32(ptr, i8, i32, i1)
 ; CHECK-LABEL: memcpy_i8:
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memcpy_i8 (i32, i32, i32) -> ()
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i32.eqz $push0=, $2
+; BULK-MEM-NEXT: br_if 0, $pop0
 ; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memcpy_i8(ptr %dest, ptr %src, i8 zeroext %len) {
   call void @llvm.memcpy.p0.p0.i8(ptr %dest, ptr %src, i8 %len, i1 0)
@@ -27,7 +32,12 @@ define void @memcpy_i8(ptr %dest, ptr %src, i8 zeroext %len) {
 ; CHECK-LABEL: memmove_i8:
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memmove_i8 (i32, i32, i32) -> ()
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i32.eqz $push0=, $2
+; BULK-MEM-NEXT: br_if 0, $pop0
 ; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memmove_i8(ptr %dest, ptr %src, i8 zeroext %len) {
   call void @llvm.memmove.p0.p0.i8(ptr %dest, ptr %src, i8 %len, i1 0)
@@ -37,7 +47,12 @@ define void @memmove_i8(ptr %dest, ptr %src, i8 zeroext %len) {
 ; CHECK-LABEL: memset_i8:
 ; NO-BULK-MEM-NOT: memory.fill
 ; BULK-MEM-NEXT: .functype memset_i8 (i32, i32, i32) -> ()
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i32.eqz $push0=, $2
+; BULK-MEM-NEXT: br_if 0, $pop0
 ; BULK-MEM-NEXT: memory.fill 0, $0, $1, $2
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memset_i8(ptr %dest, i8 %val, i8 zeroext %len) {
   call void @llvm.memset.p0.i8(ptr %dest, i8 %val, i8 %len, i1 0)
@@ -47,7 +62,12 @@ define void @memset_i8(ptr %dest, i8 %val, i8 zeroext %len) {
 ; CHECK-LABEL: memcpy_i32:
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memcpy_i32 (i32, i32, i32) -> ()
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i32.eqz $push0=, $2
+; BULK-MEM-NEXT: br_if 0, $pop0
 ; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memcpy_i32(ptr %dest, ptr %src, i32 %len) {
   call void @llvm.memcpy.p0.p0.i32(ptr %dest, ptr %src, i32 %len, i1 0)
@@ -57,7 +77,12 @@ define void @memcpy_i32(ptr %dest, ptr %src, i32 %len) {
 ; CHECK-LABEL: memmove_i32:
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memmove_i32 (i32, i32, i32) -> ()
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i32.eqz $push0=, $2
+; BULK-MEM-NEXT: br_if 0, $pop0
 ; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memmove_i32(ptr %dest, ptr %src, i32 %len) {
   call void @llvm.memmove.p0.p0.i32(ptr %dest, ptr %src, i32 %len, i1 0)
@@ -67,7 +92,12 @@ define void @memmove_i32(ptr %dest, ptr %src, i32 %len) {
 ; CHECK-LABEL: memset_i32:
 ; NO-BULK-MEM-NOT: memory.fill
 ; BULK-MEM-NEXT: .functype memset_i32 (i32, i32, i32) -> ()
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i32.eqz $push0=, $2
+; BULK-MEM-NEXT: br_if 0, $pop0
 ; BULK-MEM-NEXT: memory.fill 0, $0, $1, $2
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memset_i32(ptr %dest, i8 %val, i32 %len) {
   call void @llvm.memset.p0.i32(ptr %dest, i8 %val, i32 %len, i1 0)
@@ -107,8 +137,14 @@ define void @memset_1(ptr %dest, i8 %val) {
 ; CHECK-LABEL: memcpy_1024:
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memcpy_1024 (i32, i32) -> ()
+; BULK-MEM-NEXT: block
 ; BULK-MEM-NEXT: i32.const $push[[L0:[0-9]+]]=, 1024
-; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: i32.eqz $push[[L1:[0-9]+]]=, $pop[[L0]]
+; BULK-MEM-NEXT: br_if 0, $pop[[L1]]
+; BULK-MEM-NEXT: i32.const $push[[L2:[0-9]+]]=, 1024
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L2]]
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memcpy_1024(ptr %dest, ptr %src) {
   call void @llvm.memcpy.p0.p0.i32(ptr %dest, ptr %src, i32 1024, i1 0)
@@ -118,8 +154,14 @@ define void @memcpy_1024(ptr %dest, ptr %src) {
 ; CHECK-LABEL: memmove_1024:
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memmove_1024 (i32, i32) -> ()
+; BULK-MEM-NEXT: block
 ; BULK-MEM-NEXT: i32.const $push[[L0:[0-9]+]]=, 1024
-; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: i32.eqz $push[[L1:[0-9]+]]=, $pop[[L0]]
+; BULK-MEM-NEXT: br_if 0, $pop[[L1]]
+; BULK-MEM-NEXT: i32.const $push[[L2:[0-9]+]]=, 1024
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L2]]
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memmove_1024(ptr %dest, ptr %src) {
   call void @llvm.memmove.p0.p0.i32(ptr %dest, ptr %src, i32 1024, i1 0)
@@ -129,8 +171,14 @@ define void @memmove_1024(ptr %dest, ptr %src) {
 ; CHECK-LABEL: memset_1024:
 ; NO-BULK-MEM-NOT: memory.fill
 ; BULK-MEM-NEXT: .functype memset_1024 (i32, i32) -> ()
+; BULK-MEM-NEXT: block
 ; BULK-MEM-NEXT: i32.const $push[[L0:[0-9]+]]=, 1024
-; BULK-MEM-NEXT: memory.fill 0, $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: i32.eqz $push[[L1:[0-9]+]]=, $pop[[L0]]
+; BULK-MEM-NEXT: br_if 0, $pop[[L1]]
+; BULK-MEM-NEXT: i32.const $push[[L2:[0-9]+]]=, 1024
+; BULK-MEM-NEXT: memory.fill 0, $0, $1, $pop[[L2]]
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memset_1024(ptr %dest, i8 %val) {
   call void @llvm.memset.p0.i32(ptr %dest, i8 %val, i32 1024, i1 0)
@@ -153,11 +201,17 @@ define void @memset_1024(ptr %dest, i8 %val) {
 ; BULK-MEM-NEXT: .functype memcpy_alloca_src (i32) -> ()
 ; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer
 ; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 112
-; BULK-MEM-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
-; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 12
-; BULK-MEM-NEXT: i32.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]
-; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 100
-; BULK-MEM-NEXT: memory.copy 0, 0, $0, $pop[[L4]], $pop[[L5]]
+; BULK-MEM-NEXT: i32.sub $[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 100
+; BULK-MEM-NEXT: i32.eqz $push[[L4:[0-9]+]]=, $pop[[L3]]
+; BULK-MEM-NEXT: br_if 0, $pop[[L4]]
+; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 12
+; BULK-MEM-NEXT: i32.add $push[[L6:[0-9]+]]=, $[[L2]], $pop[[L5]]
+; BULK-MEM-NEXT: i32.const $push[[L7:[0-9]+]]=, 100
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $pop[[L6]], $pop[[L7]]
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memcpy_alloca_src(ptr %dst) {
   %a = alloca [100 x i8]
@@ -170,11 +224,17 @@ define void @memcpy_alloca_src(ptr %dst) {
 ; BULK-MEM-NEXT: .functype memcpy_alloca_dst (i32) -> ()
 ; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer
 ; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 112
-; BULK-MEM-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
-; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 12
-; BULK-MEM-NEXT: i32.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]
-; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 100
-; BULK-MEM-NEXT: memory.copy 0, 0, $pop[[L4]], $0, $pop[[L5]]
+; BULK-MEM-NEXT: i32.sub $[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 100
+; BULK-MEM-NEXT: i32.eqz $push[[L4:[0-9]+]]=, $pop[[L3]]
+; BULK-MEM-NEXT: br_if 0, $pop[[L4]]
+; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 12
+; BULK-MEM-NEXT: i32.add $push[[L6:[0-9]+]]=, $[[L2]], $pop[[L5]]
+; BULK-MEM-NEXT: i32.const $push[[L7:[0-9]+]]=, 100
+; BULK-MEM-NEXT: memory.copy 0, 0, $pop[[L6]], $0, $pop[[L7]]
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memcpy_alloca_dst(ptr %src) {
   %a = alloca [100 x i8]
@@ -187,11 +247,17 @@ define void @memcpy_alloca_dst(ptr %src) {
 ; BULK-MEM-NEXT: .functype memset_alloca (i32) -> ()
 ; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer
 ; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 112
-; BULK-MEM-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
-; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 12
-; BULK-MEM-NEXT: i32.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]
-; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 100
-; BULK-MEM-NEXT: memory.fill 0, $pop[[L4]], $0, $pop[[L5]]
+; BULK-MEM-NEXT: i32.sub $1=, $pop[[L0]], $pop[[L1]]
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i32.const $push[[L2:[0-9]+]]=, 100
+; BULK-MEM-NEXT: i32.eqz $push[[L3:[0-9]+]]=, $pop[[L2]]
+; BULK-MEM-NEXT: br_if 0, $pop[[L3]]
+; BULK-MEM-NEXT: i32.const $push[[L4:[0-9]+]]=, 12
+; BULK-MEM-NEXT: i32.add $push[[L5:[0-9]+]]=, $1, $pop[[L4]]
+; BULK-MEM-NEXT: i32.const $push[[L6:[0-9]+]]=, 100
+; BULK-MEM-NEXT: memory.fill 0, $pop[[L5]], $0, $pop[[L6]]
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memset_alloca(i8 %val) {
   %a = alloca [100 x i8]
diff --git a/llvm/test/CodeGen/WebAssembly/bulk-memory64.ll b/llvm/test/CodeGen/WebAssembly/bulk-memory64.ll
index 8ee5f6314381..0cf8493a995f 100644
--- a/llvm/test/CodeGen/WebAssembly/bulk-memory64.ll
+++ b/llvm/test/CodeGen/WebAssembly/bulk-memory64.ll
@@ -17,8 +17,14 @@ declare void @llvm.memset.p0.i64(ptr, i8, i64, i1)
 ; CHECK-LABEL: memcpy_i8:
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memcpy_i8 (i64, i64, i32) -> ()
-; BULK-MEM-NEXT: i64.extend_i32_u $push0=, $2
-; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop0
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i64.extend_i32_u $push[[L0:[0-9]+]]=, $2
+; BULK-MEM-NEXT: local.tee $push[[L1:[0-9]+]]=, $3=, $pop[[L0]]
+; BULK-MEM-NEXT: i64.eqz $push0=, $pop[[L1]]
+; BULK-MEM-NEXT: br_if 0, $pop0
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $3
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memcpy_i8(ptr %dest, ptr %src, i8 zeroext %len) {
   call void @llvm.memcpy.p0.p0.i8(ptr %dest, ptr %src, i8 %len, i1 0)
@@ -28,8 +34,14 @@ define void @memcpy_i8(ptr %dest, ptr %src, i8 zeroext %len) {
 ; CHECK-LABEL: memmove_i8:
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memmove_i8 (i64, i64, i32) -> ()
-; BULK-MEM-NEXT: i64.extend_i32_u $push0=, $2
-; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop0
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i64.extend_i32_u $push[[L0:[0-9]+]]=, $2
+; BULK-MEM-NEXT: local.tee $push[[L1:[0-9]+]]=, $3=, $pop[[L0]]
+; BULK-MEM-NEXT: i64.eqz $push0=, $pop[[L1]]
+; BULK-MEM-NEXT: br_if 0, $pop0
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $3
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memmove_i8(ptr %dest, ptr %src, i8 zeroext %len) {
   call void @llvm.memmove.p0.p0.i8(ptr %dest, ptr %src, i8 %len, i1 0)
@@ -39,8 +51,14 @@ define void @memmove_i8(ptr %dest, ptr %src, i8 zeroext %len) {
 ; CHECK-LABEL: memset_i8:
 ; NO-BULK-MEM-NOT: memory.fill
 ; BULK-MEM-NEXT: .functype memset_i8 (i64, i32, i32) -> ()
-; BULK-MEM-NEXT: i64.extend_i32_u $push0=, $2
-; BULK-MEM-NEXT: memory.fill 0, $0, $1, $pop0
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i64.extend_i32_u $push[[L0:[0-9]+]]=, $2
+; BULK-MEM-NEXT: local.tee $push[[L1:[0-9]+]]=, $3=, $pop[[L0]]
+; BULK-MEM-NEXT: i64.eqz $push0=, $pop[[L1]]
+; BULK-MEM-NEXT: br_if 0, $pop0
+; BULK-MEM-NEXT: memory.fill 0, $0, $1, $3
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memset_i8(ptr %dest, i8 %val, i8 zeroext %len) {
   call void @llvm.memset.p0.i8(ptr %dest, i8 %val, i8 %len, i1 0)
@@ -50,7 +68,12 @@ define void @memset_i8(ptr %dest, i8 %val, i8 zeroext %len) {
 ; CHECK-LABEL: memcpy_i32:
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memcpy_i32 (i64, i64, i64) -> ()
-; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i64.eqz 	$push0=, $2
+; BULK-MEM-NEXT: br_if   	0, $pop0
+; BULK-MEM-NEXT: memory.copy	0, 0, $0, $1, $2
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memcpy_i32(ptr %dest, ptr %src, i64 %len) {
   call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %src, i64 %len, i1 0)
@@ -60,7 +83,12 @@ define void @memcpy_i32(ptr %dest, ptr %src, i64 %len) {
 ; CHECK-LABEL: memmove_i32:
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memmove_i32 (i64, i64, i64) -> ()
-; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i64.eqz 	$push0=, $2
+; BULK-MEM-NEXT: br_if   	0, $pop0
+; BULK-MEM-NEXT: memory.copy	0, 0, $0, $1, $2
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memmove_i32(ptr %dest, ptr %src, i64 %len) {
   call void @llvm.memmove.p0.p0.i64(ptr %dest, ptr %src, i64 %len, i1 0)
@@ -70,7 +98,12 @@ define void @memmove_i32(ptr %dest, ptr %src, i64 %len) {
 ; CHECK-LABEL: memset_i32:
 ; NO-BULK-MEM-NOT: memory.fill
 ; BULK-MEM-NEXT: .functype memset_i32 (i64, i32, i64) -> ()
-; BULK-MEM-NEXT: memory.fill 0, $0, $1, $2
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i64.eqz 	$push0=, $2
+; BULK-MEM-NEXT: br_if   	0, $pop0
+; BULK-MEM-NEXT: memory.fill	0, $0, $1, $2
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memset_i32(ptr %dest, i8 %val, i64 %len) {
   call void @llvm.memset.p0.i64(ptr %dest, i8 %val, i64 %len, i1 0)
@@ -110,8 +143,14 @@ define void @memset_1(ptr %dest, i8 %val) {
 ; CHECK-LABEL: memcpy_1024:
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memcpy_1024 (i64, i64) -> ()
-; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 1024
-; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i64.const	$push[[L1:[0-9]+]]=, 1024
+; BULK-MEM-NEXT: i64.eqz 	$push0=, $pop[[L1]]
+; BULK-MEM-NEXT: br_if   	0, $pop0
+; BULK-MEM-NEXT: i64.const	$push[[L0:[0-9]+]]=, 1024
+; BULK-MEM-NEXT: memory.copy	0, 0, $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memcpy_1024(ptr %dest, ptr %src) {
   call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %src, i64 1024, i1 0)
@@ -121,8 +160,14 @@ define void @memcpy_1024(ptr %dest, ptr %src) {
 ; CHECK-LABEL: memmove_1024:
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memmove_1024 (i64, i64) -> ()
-; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 1024
-; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i64.const	$push[[L1:[0-9]+]]=, 1024
+; BULK-MEM-NEXT: i64.eqz 	$push0=, $pop[[L1]]
+; BULK-MEM-NEXT: br_if   	0, $pop0
+; BULK-MEM-NEXT: i64.const	$push[[L0:[0-9]+]]=, 1024
+; BULK-MEM-NEXT: memory.copy	0, 0, $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memmove_1024(ptr %dest, ptr %src) {
   call void @llvm.memmove.p0.p0.i64(ptr %dest, ptr %src, i64 1024, i1 0)
@@ -132,8 +177,14 @@ define void @memmove_1024(ptr %dest, ptr %src) {
 ; CHECK-LABEL: memset_1024:
 ; NO-BULK-MEM-NOT: memory.fill
 ; BULK-MEM-NEXT: .functype memset_1024 (i64, i32) -> ()
-; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 1024
-; BULK-MEM-NEXT: memory.fill 0, $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i64.const	$push[[L1:[0-9]+]]=, 1024
+; BULK-MEM-NEXT: i64.eqz 	$push0=, $pop[[L1]]
+; BULK-MEM-NEXT: br_if   	0, $pop0
+; BULK-MEM-NEXT: i64.const	$push[[L0:[0-9]+]]=, 1024
+; BULK-MEM-NEXT: memory.fill	0, $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memset_1024(ptr %dest, i8 %val) {
   call void @llvm.memset.p0.i64(ptr %dest, i8 %val, i64 1024, i1 0)
@@ -154,13 +205,19 @@ define void @memset_1024(ptr %dest, i8 %val) {
 ; CHECK-LABEL: memcpy_alloca_src:
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memcpy_alloca_src (i64) -> ()
-; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer
-; BULK-MEM-NEXT: i64.const $push[[L1:[0-9]+]]=, 112
-; BULK-MEM-NEXT: i64.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
-; BULK-MEM-NEXT: i64.const $push[[L3:[0-9]+]]=, 12
-; BULK-MEM-NEXT: i64.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]
-; BULK-MEM-NEXT: i64.const $push[[L5:[0-9]+]]=, 100
-; BULK-MEM-NEXT: memory.copy 0, 0, $0, $pop[[L4]], $pop[[L5]]
+; BULK-MEM-NEXT: global.get	$push[[L1:[0-9]+]]=, __stack_pointer
+; BULK-MEM-NEXT: i64.const	$push[[L0:[0-9]+]]=, 112
+; BULK-MEM-NEXT: i64.sub 	$[[L2:[0-9]+]]=, $pop[[L1]], $pop[[L0]]
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i64.const	$push[[L3:[0-9]+]]=, 100
+; BULK-MEM-NEXT: i64.eqz 	$push[[L4:[0-9]+]]=, $pop[[L3]]
+; BULK-MEM-NEXT: br_if   	0, $pop[[L4]]
+; BULK-MEM-NEXT: i64.const	$push[[L5:[0-9]+]]=, 12
+; BULK-MEM-NEXT: i64.add 	$push[[L6:[0-9]+]]=, $[[L2]], $pop[[L5]]
+; BULK-MEM-NEXT: i64.const	$push[[L7:[0-9]+]]=, 100
+; BULK-MEM-NEXT: memory.copy	0, 0, $0, $pop[[L6]], $pop[[L7]]
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memcpy_alloca_src(ptr %dst) {
   %a = alloca [100 x i8]
@@ -171,13 +228,19 @@ define void @memcpy_alloca_src(ptr %dst) {
 ; CHECK-LABEL: memcpy_alloca_dst:
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memcpy_alloca_dst (i64) -> ()
-; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer
-; BULK-MEM-NEXT: i64.const $push[[L1:[0-9]+]]=, 112
-; BULK-MEM-NEXT: i64.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
-; BULK-MEM-NEXT: i64.const $push[[L3:[0-9]+]]=, 12
-; BULK-MEM-NEXT: i64.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]
-; BULK-MEM-NEXT: i64.const $push[[L5:[0-9]+]]=, 100
-; BULK-MEM-NEXT: memory.copy 0, 0, $pop[[L4]], $0, $pop[[L5]]
+; BULK-MEM-NEXT: global.get	$push[[L1:[0-9]+]]=, __stack_pointer
+; BULK-MEM-NEXT: i64.const	$push[[L0:[0-9]+]]=, 112
+; BULK-MEM-NEXT: i64.sub 	$[[L2:[0-9]+]]=, $pop[[L1]], $pop[[L0]]
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i64.const	$push[[L3:[0-9]+]]=, 100
+; BULK-MEM-NEXT: i64.eqz 	$push[[L4:[0-9]+]]=, $pop[[L3]]
+; BULK-MEM-NEXT: br_if   	0, $pop[[L4]]
+; BULK-MEM-NEXT: i64.const	$push[[L5:[0-9]+]]=, 12
+; BULK-MEM-NEXT: i64.add 	$push[[L6:[0-9]+]]=, $[[L2]], $pop[[L5]]
+; BULK-MEM-NEXT: i64.const	$push[[L7:[0-9]+]]=, 100
+; BULK-MEM-NEXT: memory.copy	0, 0, $pop[[L6]], $0, $pop[[L7]]
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memcpy_alloca_dst(ptr %src) {
   %a = alloca [100 x i8]
@@ -188,13 +251,19 @@ define void @memcpy_alloca_dst(ptr %src) {
 ; CHECK-LABEL: memset_alloca:
 ; NO-BULK-MEM-NOT: memory.fill
 ; BULK-MEM-NEXT: .functype memset_alloca (i32) -> ()
-; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer
-; BULK-MEM-NEXT: i64.const $push[[L1:[0-9]+]]=, 112
-; BULK-MEM-NEXT: i64.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
-; BULK-MEM-NEXT: i64.const $push[[L3:[0-9]+]]=, 12
-; BULK-MEM-NEXT: i64.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]
-; BULK-MEM-NEXT: i64.const $push[[L5:[0-9]+]]=, 100
-; BULK-MEM-NEXT: memory.fill 0, $pop[[L4]], $0, $pop[[L5]]
+; BULK-MEM-NEXT: global.get	$push[[L1:[0-9]+]]=, __stack_pointer
+; BULK-MEM-NEXT: i64.const	$push[[L0:[0-9]+]]=, 112
+; BULK-MEM-NEXT: i64.sub 	$1=, $pop[[L1]], $pop[[L0]]
+; BULK-MEM-NEXT: block
+; BULK-MEM-NEXT: i64.const	$push[[L2:[0-9]+]]=, 100
+; BULK-MEM-NEXT: i64.eqz 	$push[[L3:[0-9]+]]=, $pop[[L2]]
+; BULK-MEM-NEXT: br_if   	0, $pop[[L3]]
+; BULK-MEM-NEXT: i64.const	$push[[L4:[0-9]+]]=, 12
+; BULK-MEM-NEXT: i64.add 	$push[[L5:[0-9]+]]=, $1, $pop[[L4]]
+; BULK-MEM-NEXT: i64.const	$push[[L6:[0-9]+]]=, 100
+; BULK-MEM-NEXT: memory.fill	0, $pop[[L5]], $0, $pop[[L6]]
+; BULK-MEM-NEXT: .LBB{{.*}}:
+; BULK-MEM-NEXT: end_block
 ; BULK-MEM-NEXT: return
 define void @memset_alloca(i8 %val) {
   %a = alloca [100 x i8]
-- 
GitLab


From a291f00edaeeea622a9e3e2c706e3bdb25f93f09 Mon Sep 17 00:00:00 2001
From: Rashmi Mudduluru <r_mudduluru@apple.com>
Date: Thu, 24 Oct 2024 14:23:29 -0700
Subject: [PATCH 097/693] [WebKit Checkers] Make TrivialFunctionAnalysis
 recognize std::array::operator[] as trivial (#113377)

TFA wasn't recognizing `__libcpp_verbose_abort` as trivial causing `std::array::operator[]` also not being recognized as trivial.
---
 .../Checkers/WebKit/PtrTypesSemantics.cpp     |  3 +-
 .../WebKit/uncounted-obj-arg-std-array.cpp    | 39 +++++++++++++++++++
 .../Checkers/WebKit/uncounted-obj-arg.cpp     |  6 +++
 3 files changed, 47 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg-std-array.cpp

diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
index e043806eadd6..71440e6d08a1 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
@@ -414,7 +414,8 @@ public:
         Name == "isMainThreadOrGCThread" || Name == "isMainRunLoop" ||
         Name == "isWebThread" || Name == "isUIThread" ||
         Name == "mayBeGCThread" || Name == "compilerFenceForCrash" ||
-        Name == "bitwise_cast" || Name.find("__builtin") == 0)
+        Name == "bitwise_cast" || Name.find("__builtin") == 0 ||
+        Name == "__libcpp_verbose_abort")
       return true;
 
     return IsFunctionTrivial(Callee);
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg-std-array.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg-std-array.cpp
new file mode 100644
index 000000000000..ed28a64bba05
--- /dev/null
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg-std-array.cpp
@@ -0,0 +1,39 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.UncountedCallArgsChecker -verify %s
+// expected-no-diagnostics
+
+#include "mock-types.h"
+
+void __libcpp_verbose_abort(const char *__format, ...);
+
+using size_t = __typeof(sizeof(int));
+namespace std{
+template <class T, size_t N>
+class array {
+  T elements[N];
+  
+  public:
+  T& operator[](unsigned i) {
+    if (i >= N) {
+      __libcpp_verbose_abort("%s", "aborting");
+    }
+    return elements[i];
+  }
+};
+}
+
+class ArrayClass {
+public:
+  void ref() const;
+  void deref() const;
+  typedef std::array<std::array<double, 4>, 4> Matrix;
+  double e() { return matrix[3][0]; }
+  Matrix matrix;
+};
+
+class AnotherClass {
+  RefPtr<ArrayClass> matrix;
+  void test() {
+    double val = { matrix->e()};
+  }
+};
+
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
index 10da776f8157..e1dacdd9e25b 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
@@ -238,6 +238,8 @@ public:
   using BaseType::BaseType;
 };
 
+void __libcpp_verbose_abort(const char *__format, ...);
+
 class RefCounted {
 public:
   void ref() const;
@@ -361,6 +363,9 @@ public:
   void trivial62() { WTFReportBacktrace(); }
   SomeType trivial63() { return SomeType(0); }
   SomeType trivial64() { return SomeType(); }
+  void trivial65() {
+    __libcpp_verbose_abort("%s", "aborting");
+  }
 
   static RefCounted& singleton() {
     static RefCounted s_RefCounted;
@@ -544,6 +549,7 @@ public:
     getFieldTrivial().trivial62(); // no-warning
     getFieldTrivial().trivial63(); // no-warning
     getFieldTrivial().trivial64(); // no-warning
+    getFieldTrivial().trivial65(); // no-warning
 
     RefCounted::singleton().trivial18(); // no-warning
     RefCounted::singleton().someFunction(); // no-warning
-- 
GitLab


From 0558fe3bbf575f562e8bde4cb73cf47e0b6a8153 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 24 Oct 2024 14:49:08 -0700
Subject: [PATCH 098/693] [VPlan] Update unit tests to use getVectorLoopRegion
 (NFC).

Use getVectorLoopRegion to retrieve the vector loop header instead of
relying on getting the successor of the entry block. This makes sure the
code still works correctly as more parts of the skeleton are modeled in
VPlan.
---
 .../Transforms/Vectorize/VPlanHCFGTest.cpp    |  4 ++--
 .../Transforms/Vectorize/VPlanSlpTest.cpp     | 24 +++++++++----------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index 00a3c737c0e4..01d630124a4b 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -50,7 +50,7 @@ TEST_F(VPlanHCFGTest, testBuildHCFGInnerLoop) {
 
   // Check that the region following the preheader is a single basic-block
   // region (loop).
-  VPBasicBlock *VecBB = Entry->getSingleSuccessor()->getEntryBasicBlock();
+  VPBasicBlock *VecBB = Plan->getVectorLoopRegion()->getEntryBasicBlock();
   EXPECT_EQ(8u, VecBB->size());
   EXPECT_EQ(0u, VecBB->getNumPredecessors());
   EXPECT_EQ(0u, VecBB->getNumSuccessors());
@@ -193,7 +193,7 @@ TEST_F(VPlanHCFGTest, testVPInstructionToVPRecipesInner) {
 
   // Check that the region following the preheader is a single basic-block
   // region (loop).
-  VPBasicBlock *VecBB = Entry->getSingleSuccessor()->getEntryBasicBlock();
+  VPBasicBlock *VecBB = Plan->getVectorLoopRegion()->getEntryBasicBlock();
   EXPECT_EQ(8u, VecBB->size());
   EXPECT_EQ(0u, VecBB->getNumPredecessors());
   EXPECT_EQ(0u, VecBB->getNumSuccessors());
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
index 910fc24455a6..1b993b63898c 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
@@ -99,7 +99,7 @@ TEST_F(VPlanSlpTest, testSlpSimple_2) {
 
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
-  VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
+  VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
@@ -171,7 +171,7 @@ TEST_F(VPlanSlpTest, testSlpSimple_3) {
 
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
-  VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
+  VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
@@ -243,7 +243,7 @@ TEST_F(VPlanSlpTest, testSlpReuse_1) {
 
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
-  VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
+  VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 8));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10));
@@ -307,7 +307,7 @@ TEST_F(VPlanSlpTest, testSlpReuse_2) {
 
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
-  VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
+  VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 5));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10));
@@ -444,7 +444,7 @@ TEST_F(VPlanSlpTest, testSlpReorder_1) {
 
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
-  VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
+  VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
@@ -516,7 +516,7 @@ TEST_F(VPlanSlpTest, testSlpReorder_2) {
 
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
-  VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
+  VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
@@ -588,7 +588,7 @@ TEST_F(VPlanSlpTest, testSlpReorder_3) {
 
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
-  VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
+  VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
@@ -664,7 +664,7 @@ TEST_F(VPlanSlpTest, testSlpReorder_4) {
 
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
-  VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
+  VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
@@ -725,7 +725,7 @@ TEST_F(VPlanSlpTest, testInstrsInDifferentBBs) {
 
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
-  VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
+  VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
   VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 3));
@@ -788,7 +788,7 @@ TEST_F(VPlanSlpTest, testInstrsInDifferentBBs2) {
 
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
-  VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
+  VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
   VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 1));
@@ -848,7 +848,7 @@ TEST_F(VPlanSlpTest, testSlpAtomicLoad) {
 
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
-  VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
+  VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
@@ -907,7 +907,7 @@ TEST_F(VPlanSlpTest, testSlpAtomicStore) {
 
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
-  VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
+  VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
-- 
GitLab


From 7b9f988a535c3549b71025e951e3a36a2bf0fa03 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 24 Oct 2024 15:08:35 -0700
Subject: [PATCH 099/693] [VPlan] Limit stride replacement to vector region and
 middle VPBB (NFC).

At the moment this in NFC, but ensures we only replace uses that are
dominated by runtime checks as we model more of the skeleton in VPlan.
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 60a94ca1f86e..e1173ddd71af 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9189,6 +9189,14 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
 
   // Replace VPValues for known constant strides guaranteed by predicate scalar
   // evolution.
+  auto CanUseVersionedStride = [&Plan](VPUser &U, unsigned) {
+    auto *R = dyn_cast<VPRecipeBase>(&U);
+    if (!R)
+      return false;
+    return R->getParent()->getParent() ||
+           R->getParent() ==
+               Plan->getVectorLoopRegion()->getSinglePredecessor();
+  };
   for (auto [_, Stride] : Legal->getLAI()->getSymbolicStrides()) {
     auto *StrideV = cast<SCEVUnknown>(Stride)->getValue();
     auto *ScevStride = dyn_cast<SCEVConstant>(PSE.getSCEV(StrideV));
@@ -9199,7 +9207,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
     auto *CI = Plan->getOrAddLiveIn(
         ConstantInt::get(Stride->getType(), ScevStride->getAPInt()));
     if (VPValue *StrideVPV = Plan->getLiveIn(StrideV))
-      StrideVPV->replaceAllUsesWith(CI);
+      StrideVPV->replaceUsesWithIf(CI, CanUseVersionedStride);
 
     // The versioned value may not be used in the loop directly but through a
     // sext/zext. Add new live-ins in those cases.
@@ -9213,7 +9221,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
       APInt C = isa<SExtInst>(U) ? ScevStride->getAPInt().sext(BW)
                                  : ScevStride->getAPInt().zext(BW);
       VPValue *CI = Plan->getOrAddLiveIn(ConstantInt::get(U->getType(), C));
-      StrideVPV->replaceAllUsesWith(CI);
+      StrideVPV->replaceUsesWithIf(CI, CanUseVersionedStride);
     }
   }
 
-- 
GitLab


From 633a6c91a3d8b8c97de661d7e611ff9720b857ab Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Thu, 24 Oct 2024 15:53:42 -0700
Subject: [PATCH 100/693] [Support] Extend ExtensibleRTTI utility to support
 basic multiple inheritance. (#112643)

Clients can now pass multiple parent classes to RTTIExtends. Each parent
class will be inherited from publicly (and non-virtually). The isa,
cast, and dyn_cast methods will now work as expected for types with
multiple inheritance.
---
 llvm/include/llvm/Support/ExtensibleRTTI.h    | 44 +++++++----
 llvm/unittests/Support/ExtensibleRTTITest.cpp | 78 ++++++++++++++++---
 2 files changed, 99 insertions(+), 23 deletions(-)

diff --git a/llvm/include/llvm/Support/ExtensibleRTTI.h b/llvm/include/llvm/Support/ExtensibleRTTI.h
index d3193be6f529..e11e3bd06e93 100644
--- a/llvm/include/llvm/Support/ExtensibleRTTI.h
+++ b/llvm/include/llvm/Support/ExtensibleRTTI.h
@@ -81,10 +81,6 @@ public:
     return ClassID == classID();
   }
 
-  /// Check whether this instance is a subclass of QueryT.
-  template <typename QueryT>
-  bool isA() const { return isA(QueryT::classID()); }
-
 private:
   virtual void anchor();
 
@@ -93,13 +89,15 @@ private:
 
 /// Inheritance utility for extensible RTTI.
 ///
-/// Supports single inheritance only: A class can only have one
-/// ExtensibleRTTI-parent (i.e. a parent for which the isa<> test will work),
-/// though it can have many non-ExtensibleRTTI parents.
+/// Multiple inheritance is supported, but RTTIExtends only inherits
+/// constructors from the first base class. All subsequent bases will be
+/// default constructed. Virtual and non-public inheritance are not supported.
 ///
 /// RTTIExtents uses CRTP so the first template argument to RTTIExtends is the
-/// newly introduced type, and the *second* argument is the parent class.
+/// newly introduced type, and the *second and later* arguments are the parent
+/// classes.
 ///
+/// @code{.cpp}
 /// class MyType : public RTTIExtends<MyType, RTTIRoot> {
 /// public:
 ///   static char ID;
@@ -110,21 +108,41 @@ private:
 ///   static char ID;
 /// };
 ///
-template <typename ThisT, typename ParentT>
-class RTTIExtends : public ParentT {
+/// class MyOtherType : public RTTIExtends<MyOtherType, MyType> {
+/// public:
+///   static char ID;
+/// };
+///
+/// class MyMultipleInheritanceType
+///   : public RTTIExtends<MyMultipleInheritanceType,
+///                        MyDerivedType, MyOtherType> {
+/// public:
+///   static char ID;
+/// };
+///
+/// @endcode
+///
+template <typename ThisT, typename ParentT, typename... ParentTs>
+class RTTIExtends : public ParentT, public ParentTs... {
 public:
-  // Inherit constructors from ParentT.
+  // Inherit constructors from the first Parent.
   using ParentT::ParentT;
 
   static const void *classID() { return &ThisT::ID; }
 
   const void *dynamicClassID() const override { return &ThisT::ID; }
 
+  /// Check whether this instance is a subclass of QueryT.
+  template <typename QueryT> bool isA() const { return isA(QueryT::classID()); }
+
   bool isA(const void *const ClassID) const override {
-    return ClassID == classID() || ParentT::isA(ClassID);
+    return ClassID == classID() || ParentT::isA(ClassID) ||
+           (ParentTs::isA(ClassID) || ...);
   }
 
-  static bool classof(const RTTIRoot *R) { return R->isA<ThisT>(); }
+  template <typename T> static bool classof(const T *R) {
+    return R->template isA<ThisT>();
+  }
 };
 
 } // end namespace llvm
diff --git a/llvm/unittests/Support/ExtensibleRTTITest.cpp b/llvm/unittests/Support/ExtensibleRTTITest.cpp
index 9715d26a9de2..8b0ff89745b2 100644
--- a/llvm/unittests/Support/ExtensibleRTTITest.cpp
+++ b/llvm/unittests/Support/ExtensibleRTTITest.cpp
@@ -36,15 +36,42 @@ public:
   static char ID;
 };
 
+class MyMultipleInheritanceType
+    : public RTTIExtends<MyMultipleInheritanceType, MyDerivedType,
+                         MyOtherDerivedType> {
+public:
+  static char ID;
+};
+
+class MyTypeWithConstructor
+    : public RTTIExtends<MyTypeWithConstructor, MyBaseType> {
+public:
+  static char ID;
+
+  MyTypeWithConstructor(int) {}
+};
+
+class MyDerivedTypeWithConstructor
+    : public RTTIExtends<MyDerivedTypeWithConstructor, MyTypeWithConstructor> {
+public:
+  static char ID;
+
+  MyDerivedTypeWithConstructor(int x) : RTTIExtends(x) {}
+};
+
 char MyBaseType::ID = 0;
 char MyDerivedType::ID = 0;
 char MyOtherDerivedType::ID = 0;
 char MyDeeperDerivedType::ID = 0;
+char MyMultipleInheritanceType::ID = 0;
+char MyTypeWithConstructor::ID = 0;
+char MyDerivedTypeWithConstructor::ID = 0;
 
 TEST(ExtensibleRTTI, isa) {
   MyBaseType B;
   MyDerivedType D;
   MyDeeperDerivedType DD;
+  MyMultipleInheritanceType MI;
 
   EXPECT_TRUE(isa<MyBaseType>(B));
   EXPECT_FALSE(isa<MyDerivedType>(B));
@@ -60,26 +87,57 @@ TEST(ExtensibleRTTI, isa) {
   EXPECT_TRUE(isa<MyDerivedType>(DD));
   EXPECT_FALSE(isa<MyOtherDerivedType>(DD));
   EXPECT_TRUE(isa<MyDeeperDerivedType>(DD));
+
+  EXPECT_TRUE(isa<MyBaseType>(MI));
+  EXPECT_TRUE(isa<MyDerivedType>(MI));
+  EXPECT_TRUE(isa<MyOtherDerivedType>(MI));
+  EXPECT_FALSE(isa<MyDeeperDerivedType>(MI));
+  EXPECT_TRUE(isa<MyMultipleInheritanceType>(MI));
 }
 
 TEST(ExtensibleRTTI, cast) {
-  MyDerivedType D;
-  MyBaseType &BD = D;
-
-  (void)cast<MyBaseType>(D);
-  (void)cast<MyBaseType>(BD);
-  (void)cast<MyDerivedType>(BD);
+  MyMultipleInheritanceType MI;
+  MyDerivedType &D = MI;
+  MyOtherDerivedType &OD = MI;
+  MyBaseType &B = D;
+
+  EXPECT_EQ(&cast<MyBaseType>(D), &B);
+  EXPECT_EQ(&cast<MyDerivedType>(MI), &D);
+  EXPECT_EQ(&cast<MyOtherDerivedType>(MI), &OD);
+  EXPECT_EQ(&cast<MyMultipleInheritanceType>(MI), &MI);
 }
 
 TEST(ExtensibleRTTI, dyn_cast) {
-  MyBaseType B;
-  MyDerivedType D;
+  MyMultipleInheritanceType MI;
+  MyDerivedType &D = MI;
+  MyOtherDerivedType &OD = MI;
   MyBaseType &BD = D;
+  MyBaseType &BOD = OD;
 
-  EXPECT_EQ(dyn_cast<MyDerivedType>(&B), nullptr);
-  EXPECT_EQ(dyn_cast<MyDerivedType>(&D), &D);
   EXPECT_EQ(dyn_cast<MyBaseType>(&BD), &BD);
   EXPECT_EQ(dyn_cast<MyDerivedType>(&BD), &D);
+
+  EXPECT_EQ(dyn_cast<MyBaseType>(&BOD), &BOD);
+  EXPECT_EQ(dyn_cast<MyOtherDerivedType>(&BOD), &OD);
+
+  EXPECT_EQ(dyn_cast<MyBaseType>(&D), &BD);
+  EXPECT_EQ(dyn_cast<MyDerivedType>(&D), &D);
+  EXPECT_EQ(dyn_cast<MyMultipleInheritanceType>(&D), &MI);
+
+  EXPECT_EQ(dyn_cast<MyBaseType>(&OD), &BOD);
+  EXPECT_EQ(dyn_cast<MyOtherDerivedType>(&OD), &OD);
+  EXPECT_EQ(dyn_cast<MyMultipleInheritanceType>(&OD), &MI);
+
+  EXPECT_EQ(dyn_cast<MyDerivedType>(&MI), &D);
+  EXPECT_EQ(dyn_cast<MyMultipleInheritanceType>(&MI), &MI);
+
+  EXPECT_EQ(dyn_cast<MyDerivedType>(&MI), &D);
+  EXPECT_EQ(dyn_cast<MyOtherDerivedType>(&MI), &OD);
+  EXPECT_EQ(dyn_cast<MyMultipleInheritanceType>(&MI), &MI);
+}
+
+TEST(ExtensibleRTTI, multiple_inheritance_constructor) {
+  MyDerivedTypeWithConstructor V(42);
 }
 
 } // namespace
-- 
GitLab


From 19b04533612e96a442bd5c63d75fd4e67d796193 Mon Sep 17 00:00:00 2001
From: Jun Wang <jwang86@yahoo.com>
Date: Thu, 24 Oct 2024 16:19:18 -0700
Subject: [PATCH 101/693] [AMDGPU][MC] Fix disassembler problem for
 image_atomic with TFE (#112622)

For image_atomic instructions with TFE, in some cases (e.g., when
dmask=3) the disassembler produces dst register with wrong size (e.g.,
image_atomic_smin v5, v1, s[8:15] dmask:0x3 tfe, instead of v[5:7]).
This patch fixes the VDataDwords values for image atomic instructions.
---
 llvm/lib/Target/AMDGPU/MIMGInstructions.td    |  22 ++--
 .../llvm.amdgcn.image.atomic.dim.mir          |  60 +++++------
 llvm/test/CodeGen/AMDGPU/release-vgprs.mir    |   4 +-
 .../AMDGPU/gfx10_mimg_features.txt            |  84 +++++++++++++++
 .../AMDGPU/gfx11_dasm_mimg_features.txt       |  83 ++++++++++++++
 .../AMDGPU/gfx12_dasm_vimage_features.txt     | 101 ++++++++++++++++++
 .../AMDGPU/gfx8_mimg_features.txt             |  85 ++++++++++++++-
 .../AMDGPU/gfx9_mimg_features.txt             |  83 ++++++++++++++
 8 files changed, 480 insertions(+), 42 deletions(-)
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx10_mimg_features.txt
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage_features.txt
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx9_mimg_features.txt

diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
index 5c49a8116ae7..e0d1cde28452 100644
--- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
@@ -1119,14 +1119,22 @@ multiclass MIMG_Atomic <mimgopc op, string asm, bit isCmpSwap = 0, bit isFP = 0,
       // _V* variants have different dst size, but the size is encoded implicitly,
       // using dmask and tfe. Only 32-bit variant is registered with disassembler.
       // Other variants are reconstructed by disassembler using dmask and tfe.
-      let VDataDwords = !if(isCmpSwap, 2, 1) in
-      defm _V1 : MIMG_Atomic_Addr_Helper_m <op, asm, !if(isCmpSwap, VReg_64, VGPR_32), 1, isFP, renamed>;
-      let VDataDwords = !if(isCmpSwap, 4, 2) in
-      defm _V2 : MIMG_Atomic_Addr_Helper_m <op, asm, !if(isCmpSwap, VReg_128, VReg_64), 0, isFP, renamed>;
-      let VDataDwords = !if(isCmpSwap, 2, 2) in
+      if !not(isCmpSwap) then {
+        let VDataDwords = 1 in
+        defm _V1 : MIMG_Atomic_Addr_Helper_m <op, asm, VGPR_32, 1, isFP, renamed>;
+      }
+
+      let VDataDwords = 2 in
+      defm _V2 : MIMG_Atomic_Addr_Helper_m <op, asm, VReg_64, isCmpSwap, isFP, renamed>;
+      let VDataDwords = 3 in
       defm _V3 : MIMG_Atomic_Addr_Helper_m <op, asm, VReg_96, 0, isFP, renamed>;
-      let VDataDwords = !if(isCmpSwap, 4, 4) in
-      defm _V4 : MIMG_Atomic_Addr_Helper_m <op, asm, VReg_160, 0, isFP, renamed>;
+
+      if isCmpSwap then {
+        let VDataDwords = 4 in
+        defm _V4 : MIMG_Atomic_Addr_Helper_m <op, asm, VReg_128, 0, isFP, renamed>;
+        let VDataDwords = 5 in
+        defm _V5 : MIMG_Atomic_Addr_Helper_m <op, asm, VReg_160, 0, isFP, renamed>;
+      }
     }
   } // End IsAtomicRet = 1
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.mir
index fb8c76bb6f51..292fa4be1ca1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.mir
@@ -21,8 +21,8 @@ body: |
     ; GFX6-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_si:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_si [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
-    ; GFX6-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V1_V1_si]].sub0
+    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_si:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V2_V1_si [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
+    ; GFX6-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V2_V1_si]].sub0
     ; GFX6-NEXT: $vgpr0 = COPY [[COPY3]]
     ; GFX6-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
     ; GFX8-LABEL: name: atomic_cmpswap_i32_1d
@@ -31,8 +31,8 @@ body: |
     ; GFX8-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_vi:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
-    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V1_V1_vi]].sub0
+    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_vi:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V2_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V2_V1_vi]].sub0
     ; GFX8-NEXT: $vgpr0 = COPY [[COPY3]]
     ; GFX8-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
     ; GFX10-LABEL: name: atomic_cmpswap_i32_1d
@@ -41,8 +41,8 @@ body: |
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx10_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
-    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx10_]].sub0
+    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10_]].sub0
     ; GFX10-NEXT: $vgpr0 = COPY [[COPY3]]
     ; GFX10-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
     ; GFX11-LABEL: name: atomic_cmpswap_i32_1d
@@ -51,8 +51,8 @@ body: |
     ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx11_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
-    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx11_]].sub0
+    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11_]].sub0
     ; GFX11-NEXT: $vgpr0 = COPY [[COPY3]]
     ; GFX11-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
     ; GFX12-LABEL: name: atomic_cmpswap_i32_1d
@@ -61,8 +61,8 @@ body: |
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX12-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx12_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx12 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
-    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx12_]].sub0
+    ; GFX12-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx12_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx12 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx12_]].sub0
     ; GFX12-NEXT: $vgpr0 = COPY [[COPY3]]
     ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
     %0:sgpr(<8 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
@@ -89,7 +89,7 @@ body: |
     ; GFX6-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_si:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_si [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
+    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_si:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V2_V1_si [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
     ; GFX6-NEXT: S_ENDPGM 0
     ; GFX8-LABEL: name: atomic_cmpswap_i32_1d_no_return
     ; GFX8: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $vgpr0_vgpr1, $vgpr2
@@ -97,7 +97,7 @@ body: |
     ; GFX8-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_vi:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
+    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_vi:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V2_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
     ; GFX8-NEXT: S_ENDPGM 0
     ; GFX10-LABEL: name: atomic_cmpswap_i32_1d_no_return
     ; GFX10: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $vgpr0_vgpr1, $vgpr2
@@ -105,7 +105,7 @@ body: |
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx10_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
+    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
     ; GFX10-NEXT: S_ENDPGM 0
     ; GFX11-LABEL: name: atomic_cmpswap_i32_1d_no_return
     ; GFX11: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $vgpr0_vgpr1, $vgpr2
@@ -113,7 +113,7 @@ body: |
     ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx11_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
+    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
     ; GFX11-NEXT: S_ENDPGM 0
     ; GFX12-LABEL: name: atomic_cmpswap_i32_1d_no_return
     ; GFX12: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $vgpr0_vgpr1, $vgpr2
@@ -121,7 +121,7 @@ body: |
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX12-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx12_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx12 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
+    ; GFX12-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx12_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx12 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
     ; GFX12-NEXT: S_ENDPGM 0
     %0:sgpr(<8 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     %1:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
@@ -146,8 +146,8 @@ body: |
     ; GFX6-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_si:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_si [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
-    ; GFX6-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V2_V1_si]].sub0_sub1
+    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V4_V1_si:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V4_V1_si [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
+    ; GFX6-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V4_V1_si]].sub0_sub1
     ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[COPY3]]
     ; GFX6-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_vgpr1
     ; GFX8-LABEL: name: atomic_cmpswap_i64_1d
@@ -156,8 +156,8 @@ body: |
     ; GFX8-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_vi:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
-    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V2_V1_vi]].sub0_sub1
+    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V4_V1_vi:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V4_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V4_V1_vi]].sub0_sub1
     ; GFX8-NEXT: $vgpr0_vgpr1 = COPY [[COPY3]]
     ; GFX8-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_vgpr1
     ; GFX10-LABEL: name: atomic_cmpswap_i64_1d
@@ -166,8 +166,8 @@ body: |
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
-    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10_]].sub0_sub1
+    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx10_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx10_]].sub0_sub1
     ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[COPY3]]
     ; GFX10-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_vgpr1
     ; GFX11-LABEL: name: atomic_cmpswap_i64_1d
@@ -176,8 +176,8 @@ body: |
     ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
-    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11_]].sub0_sub1
+    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx11_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx11_]].sub0_sub1
     ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[COPY3]]
     ; GFX11-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_vgpr1
     ; GFX12-LABEL: name: atomic_cmpswap_i64_1d
@@ -186,8 +186,8 @@ body: |
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX12-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx12_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx12 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
-    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx12_]].sub0_sub1
+    ; GFX12-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx12_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx12 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx12_]].sub0_sub1
     ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[COPY3]]
     ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_vgpr1
     %0:sgpr(<8 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
@@ -214,7 +214,7 @@ body: |
     ; GFX6-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_si:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_si [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
+    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V4_V1_si:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V4_V1_si [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
     ; GFX6-NEXT: S_ENDPGM 0
     ; GFX8-LABEL: name: atomic_cmpswap_i64_1d_no_return
     ; GFX8: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
@@ -222,7 +222,7 @@ body: |
     ; GFX8-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_vi:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
+    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V4_V1_vi:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V4_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
     ; GFX8-NEXT: S_ENDPGM 0
     ; GFX10-LABEL: name: atomic_cmpswap_i64_1d_no_return
     ; GFX10: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
@@ -230,7 +230,7 @@ body: |
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
+    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx10_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
     ; GFX10-NEXT: S_ENDPGM 0
     ; GFX11-LABEL: name: atomic_cmpswap_i64_1d_no_return
     ; GFX11: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
@@ -238,7 +238,7 @@ body: |
     ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
+    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx11_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
     ; GFX11-NEXT: S_ENDPGM 0
     ; GFX12-LABEL: name: atomic_cmpswap_i64_1d_no_return
     ; GFX12: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
@@ -246,7 +246,7 @@ body: |
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX12-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx12_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx12 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
+    ; GFX12-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx12_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx12 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
     ; GFX12-NEXT: S_ENDPGM 0
     %0:sgpr(<8 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     %1:vgpr(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/release-vgprs.mir b/llvm/test/CodeGen/AMDGPU/release-vgprs.mir
index a9cb169d1a09..c845a4c82b9c 100644
--- a/llvm/test/CodeGen/AMDGPU/release-vgprs.mir
+++ b/llvm/test/CodeGen/AMDGPU/release-vgprs.mir
@@ -548,9 +548,9 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; CHECK-LABEL: name: image_atomic
-    ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11 killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $vgpr4, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 7)
+    ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx11 killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $vgpr4, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 7)
     ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr97
-    renamable $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11 killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $vgpr4, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 7)
+    renamable $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_ATOMIC_CMPSWAP_V4_V1_gfx11 killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $vgpr4, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 7)
     S_ENDPGM 0, implicit $vgpr97
 ...
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx10_mimg_features.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx10_mimg_features.txt
new file mode 100644
index 000000000000..133af6b0c1da
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx10_mimg_features.txt
@@ -0,0 +1,84 @@
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1010 -disassemble -show-encoding < %s | FileCheck %s -check-prefix=GFX1010
+
+#===------------------------------------------------------------------------===#
+# Image atomics
+#===------------------------------------------------------------------------===#
+
+# GFX1010: image_atomic_add v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x45,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_add v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x45,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_and v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x61,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_and v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x61,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_cmpswap v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x41,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_cmpswap v[5:9], v1, s[8:15] dmask:0xf dim:SQ_RSRC_IMG_1D tfe
+0x00,0x0f,0x41,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_dec v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x71,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_dec v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x71,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_inc v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x6d,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_inc v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x6d,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_or v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x65,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_or v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x65,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_smax v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x59,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_smax v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x59,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_smin v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x51,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_smin v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x51,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_sub v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x49,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_sub v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x49,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_swap v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x3d,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_swap v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x3d,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_umax v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x5d,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_umax v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x5d,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_umin v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x55,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_umin v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x55,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_xor v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x69,0xf0,0x01,0x05,0x02,0x00
+
+# GFX1010: image_atomic_xor v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x69,0xf0,0x01,0x05,0x02,0x00
+
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_mimg_features.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_mimg_features.txt
index 46488a9aa4ec..fdf376ed6d69 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_mimg_features.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_mimg_features.txt
@@ -135,6 +135,89 @@
 # GFX11: image_atomic_dec v4, v32, s[96:103] dmask:0x1 dim:SQ_RSRC_IMG_1D glc ; encoding: [0x00,0x41,0x58,0xf0,0x20,0x04,0x18,0x00]
 0x00,0x41,0x58,0xf0,0x20,0x04,0x18,0x00
 
+#===------------------------------------------------------------------------===#
+# TFE in image_atomic
+#===------------------------------------------------------------------------===#
+
+# GFX11: image_atomic_add v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x30,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_add v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x30,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_and v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x48,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_and v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x48,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_cmpswap v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x2c,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_cmpswap v[5:9], v1, s[8:15] dmask:0xf dim:SQ_RSRC_IMG_1D tfe
+0x00,0x0f,0x2c,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_dec v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x58,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_dec v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x58,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_inc v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x54,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_inc v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x54,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_or v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x4c,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_or v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x4c,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_smax v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x40,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_smax v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x40,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_smin v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x38,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_smin v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x38,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_sub v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x34,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_sub v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x34,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_swap v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x28,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_swap v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x28,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_umax v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x44,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_umax v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x44,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_umin v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x3c,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_umin v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x3c,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_xor v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x01,0x50,0xf0,0x01,0x05,0x22,0x00
+
+# GFX11: image_atomic_xor v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x03,0x50,0xf0,0x01,0x05,0x22,0x00
+
+
 # GFX11: image_sample v[64:66], v32, s[4:11], s[100:103] dmask:0x7 dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x07,0x6c,0xf0,0x20,0x40,0x01,0x64]
 0x00,0x07,0x6c,0xf0,0x20,0x40,0x01,0x64
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage_features.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage_features.txt
new file mode 100644
index 000000000000..2e747adbd072
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage_features.txt
@@ -0,0 +1,101 @@
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s
+
+#===------------------------------------------------------------------------===#
+# TFE in image_atomic
+#===------------------------------------------------------------------------===#
+
+# GFX12: image_atomic_add_flt v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0xc0,0x60,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_add_flt v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0xc0,0xe0,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_add_uint v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x00,0x43,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_add_uint v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x00,0xc3,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_and v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x80,0x44,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_and v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x80,0xc4,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_cmpswap v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0xc0,0xc2,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_cmpswap v[5:9], v1, s[8:15] dmask:0xf dim:SQ_RSRC_IMG_1D tfe
+0x00,0xc0,0xc2,0xd3,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_dec_uint v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x80,0x45,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_dec_uint v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x80,0xc5,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_inc_uint v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x40,0x45,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_inc_uint v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x40,0xc5,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_max_int v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x00,0x44,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_max_int v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x00,0xc4,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_max_uint v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x40,0x44,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_max_uint v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x40,0xc4,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_min_int v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x80,0x43,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_min_int v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x80,0xc3,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_min_uint v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0xc0,0x43,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_min_uint v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0xc0,0xc3,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_or v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0xc0,0x44,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_or v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0xc0,0xc4,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_sub_uint v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x40,0x43,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_sub_uint v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x40,0xc3,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_swap v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x80,0x42,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_swap v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x80,0xc2,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_max_uint v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x40,0x44,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_max_uint v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x40,0xc4,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_min_uint v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0xc0,0x43,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_min_uint v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0xc0,0xc3,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_xor v[5:6], v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x00,0x45,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
+
+# GFX12: image_atomic_xor v[5:7], v1, s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe
+0x00,0x00,0xc5,0xd0,0x05,0x10,0x80,0x00,0x01,0x00,0x00,0x00
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx8_mimg_features.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx8_mimg_features.txt
index 0a5bafc55f4d..1bb1014a431d 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx8_mimg_features.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx8_mimg_features.txt
@@ -184,6 +184,85 @@
 # VI: image_atomic_cmpswap v[5:8], v1, s[8:15] dmask:0xf unorm ; encoding: [0x00,0x1f,0x44,0xf0,0x01,0x05,0x02,0x00]
 0x00,0x1f,0x44,0xf0,0x01,0x05,0x02,0x00
 
+# VI: image_atomic_add v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x49,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_add v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x49,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_and v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x61,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_and v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x61,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_cmpswap v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x45,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_cmpswap v[5:9], v1, s[8:15] dmask:0xf tfe
+0x00,0x0f,0x45,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_dec v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x71,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_dec v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x71,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_inc v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x6d,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_inc v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x6d,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_or v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x65,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_or v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x65,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_smax v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x59,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_smax v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x59,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_smin v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x51,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_smin v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x51,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_sub v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x4d,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_sub v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x4d,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_swap v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x41,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_swap v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x41,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_umax v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x5d,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_umax v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x5d,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_umin v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x55,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_umin v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x55,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_xor v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x69,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_xor v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x69,0xf0,0x01,0x05,0x02,0x00
+
+
 #===------------------------------------------------------------------------===#
 # Invalid image atomics (incorrect dmask value).
 # Disassembler may produce a partially incorrect instruction but should not fail.
@@ -192,10 +271,10 @@
 # VI: image_atomic_add v5, v1, s[8:15] dmask:0x2 unorm ; encoding: [0x00,0x12,0x48,0xf0,0x01,0x05,0x02,0x00]
 0x00,0x12,0x48,0xf0,0x01,0x05,0x02,0x00
 
-# VI: image_atomic_add v5, v1, s[8:15] dmask:0x7 unorm ; encoding: [0x00,0x17,0x48,0xf0,0x01,0x05,0x02,0x00]
+# VI: image_atomic_add v[5:7], v1, s[8:15] dmask:0x7 unorm ; encoding: [0x00,0x17,0x48,0xf0,0x01,0x05,0x02,0x00]
 0x00,0x17,0x48,0xf0,0x01,0x05,0x02,0x00
 
-# VI: image_atomic_add v[5:9], v1, s[8:15] dmask:0xf unorm ; encoding: [0x00,0x1f,0x48,0xf0,0x01,0x05,0x02,0x00]
+# VI: image_atomic_add v5, v1, s[8:15] dmask:0xf unorm ; encoding: [0x00,0x1f,0x48,0xf0,0x01,0x05,0x02,0x00]
 0x00,0x1f,0x48,0xf0,0x01,0x05,0x02,0x00
 
 # VI: image_atomic_cmpswap v[5:6], v1, s[8:15] unorm ; encoding: [0x00,0x10,0x44,0xf0,0x01,0x05,0x02,0x00]
@@ -204,7 +283,7 @@
 # VI: image_atomic_cmpswap v[5:6], v1, s[8:15] dmask:0x1 unorm ; encoding: [0x00,0x11,0x44,0xf0,0x01,0x05,0x02,0x00]
 0x00,0x11,0x44,0xf0,0x01,0x05,0x02,0x00
 
-# VI: image_atomic_cmpswap v[5:6], v1, s[8:15] dmask:0xe unorm ; encoding: [0x00,0x1e,0x44,0xf0,0x01,0x05,0x02,0x00]
+# VI: image_atomic_cmpswap v[5:7], v1, s[8:15] dmask:0xe unorm ; encoding: [0x00,0x1e,0x44,0xf0,0x01,0x05,0x02,0x00]
 0x00,0x1e,0x44,0xf0,0x01,0x05,0x02,0x00
 
 #===------------------------------------------------------------------------===#
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx9_mimg_features.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx9_mimg_features.txt
new file mode 100644
index 000000000000..8300fe4e9db5
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx9_mimg_features.txt
@@ -0,0 +1,83 @@
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx900 -disassemble -show-encoding < %s | FileCheck %s -check-prefix=GFX900
+
+#===------------------------------------------------------------------------===#
+# Image atomics
+#===------------------------------------------------------------------------===#
+# GFX900: image_atomic_add v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x49,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_add v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x49,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_and v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x61,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_and v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x61,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_cmpswap v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x45,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_cmpswap v[5:9], v1, s[8:15] dmask:0xf tfe
+0x00,0x0f,0x45,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_dec v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x71,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_dec v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x71,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_inc v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x6d,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_inc v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x6d,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_or v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x65,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_or v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x65,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_smax v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x59,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_smax v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x59,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_smin v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x51,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_smin v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x51,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_sub v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x4d,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_sub v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x4d,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_swap v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x41,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_swap v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x41,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_umax v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x5d,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_umax v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x5d,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_umin v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x55,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_umin v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x55,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_xor v[5:6], v1, s[8:15] dmask:0x1 tfe
+0x00,0x01,0x69,0xf0,0x01,0x05,0x02,0x00
+
+# GFX900: image_atomic_xor v[5:7], v1, s[8:15] dmask:0x3 tfe
+0x00,0x03,0x69,0xf0,0x01,0x05,0x02,0x00
+
-- 
GitLab


From 8c2e8b5124a0365664bb79b0caa9cec74d8148ff Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Thu, 24 Oct 2024 16:32:09 -0700
Subject: [PATCH 102/693] [gn] port a14a83d9a102

---
 llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
index 281698cab31d..5fbda794ff17 100644
--- a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
@@ -53,6 +53,7 @@ write_cmake_config("lit_common_configured") {
     "COMPILER_RT_BUILD_STANDALONE_LIBATOMIC_PYBOOL=False",
     "COMPILER_RT_DEBUG_PYBOOL=False",
     "COMPILER_RT_ENABLE_INTERNAL_SYMBOLIZER_PYBOOL=False",
+    "COMPILER_RT_HAS_NO_DEFAULT_CONFIG_FLAG_PYBOOL=True",
     "COMPILER_RT_INTERCEPT_LIBDISPATCH_PYBOOL=False",
     "COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR=" +
         rebase_path(crt_current_out_dir),
-- 
GitLab


From 455f71d28541c3dcb628c4c6f7b53d6eae0f1829 Mon Sep 17 00:00:00 2001
From: Ian Wood <75152913+IanWood1@users.noreply.github.com>
Date: Thu, 24 Oct 2024 17:04:02 -0700
Subject: [PATCH 103/693] [mlir] Convert `expand_shape` to more static form
 (#112265)

Add pattern that converts a `tensor.expand_shape` op to a more static
form.

This matches the pattern: `tensor.cast` -> `tensor.expand_shape` if it
has a foldable `tensor.cast` and some constant foldable `output_shape`
operands for the `tensor.expand_shape`. This makes the
`tensor.expand_shape` more static, as well as allowing the static
information to be propagated further down in the program.
---
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp   | 83 +++++++++++++++++++++-
 mlir/test/Dialect/Tensor/canonicalize.mlir | 54 ++++++++++++++
 2 files changed, 136 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 603e86ca3d76..c2d6bc610cd9 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -24,6 +24,7 @@
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/DestinationStyleOpInterface.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
+#include "mlir/Support/LLVM.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallBitVector.h"
@@ -1982,6 +1983,86 @@ struct FoldDimOfCollapseShape : public OpRewritePattern<DimOp> {
     return success();
   }
 };
+
+/// Fold/sink a producer `tensor.cast` with a consumer `tensor.expand_shape` by
+/// matching constant output_shape operands of the expand. This makes the
+/// `tensor.expand_shape` more static and creates a consumer cast that can be
+/// propagated further.
+struct ConvertToStaticExpandShape : public OpRewritePattern<ExpandShapeOp> {
+  using OpRewritePattern<ExpandShapeOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(ExpandShapeOp expandOp,
+                                PatternRewriter &rewriter) const override {
+    auto castOp = expandOp.getSrc().getDefiningOp<CastOp>();
+    if (!canFoldIntoConsumerOp(castOp))
+      return failure();
+
+    ArrayRef<int64_t> castSrcShape = castOp.getSource().getType().getShape();
+    SmallVector<ReassociationIndices, 4> reassoc =
+        expandOp.getReassociationIndices();
+
+    SmallVector<int64_t> newOutputShape(expandOp.getResultType().getShape());
+    SmallVector<Value> dynamicOutputShape;
+    auto outputIt = expandOp.getOutputShape().begin();
+
+    for (const auto &[inputDim, innerReassoc] : llvm::enumerate(reassoc)) {
+      for (uint64_t outDim : innerReassoc) {
+        if (!ShapedType::isDynamic(newOutputShape[outDim]))
+          continue;
+
+        // If the cast's src type is dynamic, don't infer any of the
+        // corresponding expanded dimensions. `tensor.expand_shape` requires at
+        // least one of the expanded dimensions to be dynamic if the input is
+        // dynamic.
+        Value val = *outputIt;
+        ++outputIt;
+        if (ShapedType::isDynamic(castSrcShape[inputDim])) {
+          dynamicOutputShape.push_back(val);
+          continue;
+        }
+
+        APInt cst;
+        if (matchPattern(val, m_ConstantInt(&cst))) {
+          newOutputShape[outDim] = cst.getSExtValue();
+        } else {
+          dynamicOutputShape.push_back(val);
+        }
+      }
+    }
+
+    // Couldn't match any values, nothing to change
+    if (expandOp.getOutputShape().size() == dynamicOutputShape.size())
+      return failure();
+
+    // Calculate the input shape from the output
+    SmallVector<int64_t> newInputShape(expandOp.getSrcType().getRank(), 1l);
+    for (auto inDim : llvm::seq<int>(0, newInputShape.size())) {
+      for (auto outDim : reassoc[inDim]) {
+        auto ofr = newOutputShape[outDim];
+        if (ShapedType::isDynamic(ofr)) {
+          newInputShape[inDim] = ShapedType::kDynamic;
+          break;
+        }
+        newInputShape[inDim] *= ofr;
+      }
+    }
+
+    SmallVector<OpFoldResult> outputOfr =
+        getMixedValues(newOutputShape, dynamicOutputShape, rewriter);
+    auto inputType = RankedTensorType::get(
+        newInputShape, expandOp.getSrcType().getElementType());
+    auto outputType = RankedTensorType::get(
+        newOutputShape, expandOp.getSrcType().getElementType());
+    auto inputCast = rewriter.create<CastOp>(expandOp.getLoc(), inputType,
+                                             expandOp.getSrc());
+    auto newExpand = rewriter.create<ExpandShapeOp>(
+        expandOp.getLoc(), outputType, inputCast.getResult(),
+        expandOp.getReassociationIndices(), outputOfr);
+    rewriter.replaceOpWithNewOp<CastOp>(expandOp, expandOp.getType(),
+                                        newExpand.getResult());
+    return success();
+  }
+};
 } // namespace
 
 void ExpandShapeOp::getCanonicalizationPatterns(RewritePatternSet &results,
@@ -1989,7 +2070,7 @@ void ExpandShapeOp::getCanonicalizationPatterns(RewritePatternSet &results,
   results.add<
       ComposeReassociativeReshapeOps<ExpandShapeOp, ReshapeOpKind::kExpand>,
       ComposeExpandOfCollapseOp<ExpandShapeOp, CollapseShapeOp>,
-      FoldReshapeWithConstant<ExpandShapeOp>,
+      ConvertToStaticExpandShape, FoldReshapeWithConstant<ExpandShapeOp>,
       FoldReshapeWithSplat<ExpandShapeOp>,
       FoldReshapeWithFromElements<ExpandShapeOp>, FoldDimOfExpandShape,
       FoldDimOfCollapseShape>(context);
diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir
index dbf0f0b81f61..693079c3aa2f 100644
--- a/mlir/test/Dialect/Tensor/canonicalize.mlir
+++ b/mlir/test/Dialect/Tensor/canonicalize.mlir
@@ -2741,3 +2741,57 @@ func.func @pack_dont_drop_attributes(%arg0: tensor<?x?x?xf16>, %arg1: tensor<128
   %pack = tensor.pack %arg0 padding_value(%cst : f16) outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %arg1 {test_attr} : tensor<?x?x?xf16> -> tensor<128x?x100x16x1xf16>
   return %pack : tensor<128x?x100x16x1xf16>
 }
+
+// -----
+
+func.func @fold_expand_of_cast(%arg0 : tensor<10x10xf32>)
+    -> tensor<10x1x10xf32> {
+  %c1 = arith.constant 1 : index 
+  %c10 = arith.constant 10 : index 
+  %0 = tensor.cast %arg0 : tensor<10x10xf32> to tensor<?x?xf32>
+  %1 = tensor.expand_shape %0 [[0, 1], [2]] output_shape [%c10, %c1, %c10]
+      : tensor<?x?xf32> into tensor<?x?x?xf32>
+  %2 = tensor.cast %1 : tensor<?x?x?xf32> to tensor<10x1x10xf32>
+  return %2 : tensor<10x1x10xf32>
+}
+// CHECK-LABEL:  func.func @fold_expand_of_cast
+//       CHECK:   %[[RES:.+]] = tensor.expand_shape %{{.*}} {{\[}}[0, 1], [2]] output_shape [10, 1, 10]
+//       CHECK:   return %[[RES]]
+
+// -----
+
+func.func @sink_expand_of_cast(%arg0 : tensor<?x10xf32>)
+    -> tensor<?x?x?xf32> {
+  %c1 = arith.constant 1 : index
+  %c10 = arith.constant 10 : index
+  %0 = tensor.cast %arg0 : tensor<?x10xf32> to tensor<?x?xf32>
+  %1 = tensor.expand_shape %0 [[0, 1], [2]] output_shape [%c10, %c1, %c10]
+      : tensor<?x?xf32> into tensor<?x?x?xf32>
+  return %1 : tensor<?x?x?xf32>
+}
+// CHECK-LABEL:  func.func @sink_expand_of_cast
+//   CHECK-DAG:   %[[C10:.*]] = arith.constant 10
+//   CHECK-DAG:   %[[C1:.*]] = arith.constant 1
+//       CHECK:   %[[EXPAND:.+]] = tensor.expand_shape %{{.*}} {{\[}}[0, 1], [2]] 
+//  CHECK-SAME:     output_shape [%[[C10]], %[[C1]], 10]
+//       CHECK:   %[[RES:.+]] = tensor.cast %[[EXPAND]]
+//       CHECK:   return %[[RES]]
+
+// -----
+
+func.func @partial_sink_expand_of_cast(%arg0 : tensor<10x10xf32>, %arg1 : index, %arg2 : index)
+    -> tensor<?x?x?xf32> {
+  %c10 = arith.constant 10 : index
+  %0 = tensor.cast %arg0 : tensor<10x10xf32> to tensor<?x?xf32>
+  %1 = tensor.expand_shape %0 [[0, 1], [2]] output_shape [%arg1, %arg2, %c10]
+      : tensor<?x?xf32> into tensor<?x?x?xf32>
+  return %1 : tensor<?x?x?xf32>
+}
+// CHECK-LABEL:  func.func @partial_sink_expand_of_cast
+//       CHECK:   %[[CAST:.+]] = tensor.cast
+//  CHECK-SAME:     tensor<10x10xf32> to tensor<?x10xf32>
+//       CHECK:   %[[EXPAND:.+]] = tensor.expand_shape %{{.*}} {{\[}}[0, 1], [2]] 
+//  CHECK-SAME:     output_shape [%{{.*}}, %{{.*}}, 10]
+//       CHECK:   %[[RES:.+]] = tensor.cast %[[EXPAND]]
+//  CHECK-SAME:     tensor<?x?x10xf32> to tensor<?x?x?xf32>
+//       CHECK:   return %[[RES]]
-- 
GitLab


From 25909b811a7ddc983d042b15cb54ec271a673d63 Mon Sep 17 00:00:00 2001
From: jeffreytan81 <jeffreytan@meta.com>
Date: Thu, 24 Oct 2024 17:13:32 -0700
Subject: [PATCH 104/693] Fix pointer to reference type (#113596)

We have got customer reporting "v &obj" and "p &obj" reporting different
results.
Turns out it only happens for obj that is itself a reference type which
"v &obj" reports the address of the reference itself instead of the
target object the reference points to. This diverged from C++ semantics.

This PR fixes this issue by returning the address of the dereferenced
object if it is reference type.

A new test is added which fails before.

Co-authored-by: jeffreytan81 <jeffreytan@fb.com>
---
 lldb/source/Core/ValueObject.cpp              |  9 ++++++++
 .../TestCPPDereferencingReferences.py         | 21 +++++++++++++++++++
 .../cpp/dereferencing_references/main.cpp     |  2 ++
 3 files changed, 32 insertions(+)

diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp
index 5b1c171c01f2..5e3839b89ce0 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/Core/ValueObject.cpp
@@ -2911,6 +2911,15 @@ ValueObjectSP ValueObject::AddressOf(Status &error) {
 
   AddressType address_type = eAddressTypeInvalid;
   const bool scalar_is_load_address = false;
+
+  // For reference type we need to get the address of the object that
+  // it refers to.
+  if (GetCompilerType().IsReferenceType()) {
+    ValueObjectSP deref_obj = Dereference(error);
+    if (error.Fail() || !deref_obj)
+      return ValueObjectSP();
+    return deref_obj->AddressOf(error);
+  }
   addr_t addr = GetAddressOf(scalar_is_load_address, &address_type);
   error.Clear();
   if (addr != LLDB_INVALID_ADDRESS && address_type != eAddressTypeHost) {
diff --git a/lldb/test/API/lang/cpp/dereferencing_references/TestCPPDereferencingReferences.py b/lldb/test/API/lang/cpp/dereferencing_references/TestCPPDereferencingReferences.py
index 938fb1a6edf3..1374d4e1ec67 100644
--- a/lldb/test/API/lang/cpp/dereferencing_references/TestCPPDereferencingReferences.py
+++ b/lldb/test/API/lang/cpp/dereferencing_references/TestCPPDereferencingReferences.py
@@ -25,3 +25,24 @@ class TestCase(TestBase):
         # Typedef to a reference should dereference to the underlying type.
         td_val = self.expect_var_path("td_to_ref_type", type="td_int_ref")
         self.assertEqual(td_val.Dereference().GetType().GetName(), "int")
+
+    def test_take_address_of_reference(self):
+        """Tests taking address of lvalue/rvalue references in lldb works correctly."""
+        self.build()
+        lldbutil.run_to_source_breakpoint(
+            self, "// break here", lldb.SBFileSpec("main.cpp")
+        )
+
+        plref_val_from_code = self.expect_var_path("pl_ref", type="TTT *")
+        plref_val_from_expr_path = self.expect_var_path("&l_ref", type="TTT *")
+        self.assertEqual(
+            plref_val_from_code.GetValueAsAddress(),
+            plref_val_from_expr_path.GetValueAsAddress(),
+        )
+
+        prref_val_from_code = self.expect_var_path("pr_ref", type="TTT *")
+        prref_val_from_expr_path = self.expect_var_path("&r_ref", type="TTT *")
+        self.assertEqual(
+            prref_val_from_code.GetValueAsAddress(),
+            prref_val_from_expr_path.GetValueAsAddress(),
+        )
diff --git a/lldb/test/API/lang/cpp/dereferencing_references/main.cpp b/lldb/test/API/lang/cpp/dereferencing_references/main.cpp
index b64978a9029f..4ddffd167dde 100644
--- a/lldb/test/API/lang/cpp/dereferencing_references/main.cpp
+++ b/lldb/test/API/lang/cpp/dereferencing_references/main.cpp
@@ -9,5 +9,7 @@ int main() {
   // typedef of a reference
   td_int_ref td_to_ref_type = i;
 
+  TTT *pl_ref = &l_ref;
+  TTT *pr_ref = &r_ref;
   return l_ref; // break here
 }
-- 
GitLab


From b1be21394e9c8fee1d2092c9e6a789121ab214be Mon Sep 17 00:00:00 2001
From: Alexander Richardson <alexrichardson@google.com>
Date: Thu, 24 Oct 2024 17:35:07 -0700
Subject: [PATCH 105/693] [runtimes] Allow building against an installed LLVM
 tree

I am currently trying to test the LLVM runtimes (including compiler-rt)
against an installed LLVM tree rather than a build tree (since that is
no longer available). Currently, the runtimes build of compiler-rt assumes
that LLVM_BINARY_DIR is writable since it uses configure_file() to write
there during the CMake configure stage. Instead, generate this file inside
CMAKE_CURRENT_BINARY_DIR, which will match LLVM_BINARY_DIR when invoked
from llvm/runtimes/CMakeLists.txt.

I also needed to make a minor change to the hwasan tests: hwasan_symbolize
was previously found in the LLVM_BINARY_DIR, but since it is generated as
part of the compiler-rt build it is now inside the CMake build directory
instead. I fixed this by passing the output directory to lit as
config.compiler_rt_bindir and using llvm_config.add_tool_substitutions().

For testing that we no longer write to the LLVM install directory as
part of testing or configuration, I created a read-only bind mount and
configured the runtimes builds as follows:
```
$ sudo mount --bind --read-only ~/llvm-install /tmp/upstream-llvm-readonly
$ cmake -DCMAKE_BUILD_TYPE=Debug \
  -DCMAKE_C_COMPILER=/tmp/upstream-llvm-readonly/bin/clang \
  -DCMAKE_CXX_COMPILER=/tmp/upstream-llvm-readonly/bin/clang++ \
  -DLLVM_INCLUDE_TESTS=TRUE -DLLVM_ENABLE_ASSERTIONS=TRUE \
  -DCOMPILER_RT_INCLUDE_TESTS=TRUE -DCOMPILER_RT_DEBUG=OFF \
  -DLLVM_ENABLE_RUNTIMES=compiler-rt \
  -DLLVM_BINARY_DIR=/tmp/upstream-llvm-readonly \
  -G Ninja -S ~/upstream-llvm-project/runtimes \
  -B ~/upstream-llvm-project/runtimes/cmake-build-debug-llvm-git
```

Reviewed By: ldionne

Pull Request: https://github.com/llvm/llvm-project/pull/86209
---
 compiler-rt/cmake/Modules/AddCompilerRT.cmake |  1 +
 compiler-rt/test/hwasan/lit.cfg.py            |  9 ++++++
 compiler-rt/test/lit.common.configured.in     |  1 +
 runtimes/CMakeLists.txt                       | 29 ++++++++++++-------
 4 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
index e3d81d241b10..b2f33d1a961c 100644
--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake
+++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
@@ -773,6 +773,7 @@ function(configure_compiler_rt_lit_site_cfg input output)
 
   string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} COMPILER_RT_RESOLVED_TEST_COMPILER ${COMPILER_RT_TEST_COMPILER})
   string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} COMPILER_RT_RESOLVED_OUTPUT_DIR ${COMPILER_RT_OUTPUT_DIR})
+  string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} COMPILER_RT_RESOLVED_EXEC_OUTPUT_DIR ${COMPILER_RT_EXEC_OUTPUT_DIR})
   string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR ${output_dir})
 
   configure_lit_site_cfg(${input} ${output})
diff --git a/compiler-rt/test/hwasan/lit.cfg.py b/compiler-rt/test/hwasan/lit.cfg.py
index 594f3294a84a..bbf23e683240 100644
--- a/compiler-rt/test/hwasan/lit.cfg.py
+++ b/compiler-rt/test/hwasan/lit.cfg.py
@@ -2,6 +2,9 @@
 
 import os
 
+from lit.llvm import llvm_config
+from lit.llvm.subst import ToolSubst, FindTool
+
 # Setup config name.
 config.name = "HWAddressSanitizer" + getattr(config, "name_suffix", "default")
 
@@ -74,6 +77,12 @@ config.substitutions.append(
     ("%env_hwasan_opts=", "env HWASAN_OPTIONS=" + default_hwasan_opts_str)
 )
 
+# Ensure that we can use hwasan_symbolize from the expected location
+llvm_config.add_tool_substitutions(
+    [ToolSubst("hwasan_symbolize", unresolved="fatal")],
+    search_dirs=[config.compiler_rt_bindir],
+)
+
 # Default test suffixes.
 config.suffixes = [".c", ".cpp"]
 
diff --git a/compiler-rt/test/lit.common.configured.in b/compiler-rt/test/lit.common.configured.in
index 66935c358afe..050792b6b262 100644
--- a/compiler-rt/test/lit.common.configured.in
+++ b/compiler-rt/test/lit.common.configured.in
@@ -28,6 +28,7 @@ set_default("python_executable", "@Python3_EXECUTABLE@")
 set_default("compiler_rt_debug", @COMPILER_RT_DEBUG_PYBOOL@)
 set_default("compiler_rt_intercept_libdispatch", @COMPILER_RT_INTERCEPT_LIBDISPATCH_PYBOOL@)
 set_default("compiler_rt_output_dir", "@COMPILER_RT_RESOLVED_OUTPUT_DIR@")
+set_default("compiler_rt_bindir", "@COMPILER_RT_RESOLVED_EXEC_OUTPUT_DIR@")
 set_default("compiler_rt_libdir", "@COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR@")
 set_default("emulator", "@COMPILER_RT_EMULATOR@")
 set_default("asan_shadow_scale", "@COMPILER_RT_ASAN_SHADOW_SCALE@")
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 8db2b3fb9309..250ae77fa57b 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -243,6 +243,23 @@ foreach(entry ${runtimes})
 endforeach()
 
 if(LLVM_INCLUDE_TESTS)
+  # If built with the runtimes build (rooted at runtimes/CMakeLists.txt), we
+  # won't have llvm-lit. If built with the bootstrapping build (rooted at
+  # llvm/CMakeLists.txt), the top-level llvm CMake invocation already generated
+  # the llvm-lit script.
+  if (NOT HAVE_LLVM_LIT)
+    # Add lit before adding any runtimes since their CMake tests configuration
+    # might depend on lit being present.
+    set(LLVM_LIT_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/bin)
+    add_subdirectory(${LLVM_MAIN_SRC_DIR}/utils/llvm-lit
+                     ${CMAKE_CURRENT_BINARY_DIR}/llvm-lit)
+    # Ensure that the testsuites use the local lit rather than
+    # ${LLVM_INSTALL_DIR}/bin/llvm-lit (which may not exist if LLVM_BINARY_DIR
+    # points at an installed LLVM tree rather than a build tree).
+    get_llvm_lit_path(_base_dir _file_name)
+    set(LLVM_EXTERNAL_LIT "${_base_dir}/${_file_name}" CACHE STRING "Command used to spawn lit" FORCE)
+  endif()
+
   set(LIT_ARGS_DEFAULT "-sv --show-xfail --show-unsupported")
   if (MSVC OR XCODE)
     set(LIT_ARGS_DEFAULT "${LIT_ARGS_DEFAULT} --no-progress-bar")
@@ -276,14 +293,6 @@ if(LLVM_INCLUDE_TESTS)
   # and we know the total set of lit testsuites.
   umbrella_lit_testsuite_end(check-runtimes)
 
-  if (NOT HAVE_LLVM_LIT)
-    # If built by manually invoking cmake on this directory, we don't have
-    # llvm-lit. If invoked via llvm/runtimes, the toplevel llvm cmake
-    # invocation already generated the llvm-lit script.
-    add_subdirectory(${LLVM_MAIN_SRC_DIR}/utils/llvm-lit
-                     ${CMAKE_CURRENT_BINARY_DIR}/llvm-lit)
-  endif()
-
   get_property(LLVM_RUNTIMES_LIT_TESTSUITES GLOBAL PROPERTY LLVM_RUNTIMES_LIT_TESTSUITES)
   string(REPLACE ";" "\n" LLVM_RUNTIMES_LIT_TESTSUITES "${LLVM_RUNTIMES_LIT_TESTSUITES}")
   file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/lit.tests ${LLVM_RUNTIMES_LIT_TESTSUITES})
@@ -313,10 +322,10 @@ if(SUB_COMPONENTS)
   if(LLVM_RUNTIMES_TARGET)
     configure_file(
       ${CMAKE_CURRENT_SOURCE_DIR}/Components.cmake.in
-      ${LLVM_BINARY_DIR}/runtimes/${LLVM_RUNTIMES_TARGET}/Components.cmake)
+      ${CMAKE_CURRENT_BINARY_DIR}/runtimes/${LLVM_RUNTIMES_TARGET}/Components.cmake)
   else()
     configure_file(
       ${CMAKE_CURRENT_SOURCE_DIR}/Components.cmake.in
-      ${LLVM_BINARY_DIR}/runtimes/Components.cmake)
+      ${CMAKE_CURRENT_BINARY_DIR}/runtimes/Components.cmake)
   endif()
 endif()
-- 
GitLab


From c03d09ce3eed336fea4d9283232383f6d4d4057d Mon Sep 17 00:00:00 2001
From: Tex Riddell <texr@microsoft.com>
Date: Thu, 24 Oct 2024 17:53:12 -0700
Subject: [PATCH 106/693] [aarch64] atan2 intrinsic lowering (p5) (#112611)

This change is part of this proposal:
https://discourse.llvm.org/t/rfc-all-the-math-intrinsics/78294

- `VecFuncs.def`: define intrinsic to sleef/armpl mapping
- `LegalizerHelper.cpp`: add missing fewerElementsVector handling for
the new atan2 intrinsic
- `AArch64ISelLowering.cpp`: Add arch64 specializations for lowering
like neon instructions
- `AArch64LegalizerInfo.cpp`: Legalize atan2.

Part 5 for Implement the atan2 HLSL Function #70096.
---
 llvm/include/llvm/Analysis/VecFuncs.def       |  11 +
 .../Target/GlobalISel/SelectionDAGCompat.td   |   1 +
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |   5 +
 llvm/lib/CodeGen/GlobalISel/Utils.cpp         |   2 +
 .../Target/AArch64/AArch64ISelLowering.cpp    |  29 +-
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |   7 +-
 .../AArch64/GlobalISel/arm64-irtranslator.ll  |   8 +
 .../AArch64/GlobalISel/legalize-atan2.mir     | 333 ++++++++++++++++++
 .../GlobalISel/legalizer-info-validation.mir  |   5 +-
 llvm/test/CodeGen/AArch64/f16-instructions.ll |   6 +
 .../CodeGen/AArch64/fp-intrinsics-fp16.ll     |  16 +
 llvm/test/CodeGen/AArch64/fp-intrinsics.ll    |  31 ++
 .../test/CodeGen/AArch64/illegal-float-ops.ll |  21 ++
 .../AArch64/replace-with-veclib-armpl.ll      |  47 ++-
 .../replace-with-veclib-sleef-scalable.ll     |  20 +-
 .../AArch64/replace-with-veclib-sleef.ll      |  20 +-
 llvm/test/CodeGen/AArch64/vec-libcalls.ll     |  35 ++
 .../AArch64/veclib-calls-libsystem-darwin.ll  |  48 +++
 .../AArch64/veclib-intrinsic-calls.ll         |  75 +++-
 19 files changed, 698 insertions(+), 22 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-atan2.mir

diff --git a/llvm/include/llvm/Analysis/VecFuncs.def b/llvm/include/llvm/Analysis/VecFuncs.def
index 532a3ca334b1..c4586894e3e4 100644
--- a/llvm/include/llvm/Analysis/VecFuncs.def
+++ b/llvm/include/llvm/Analysis/VecFuncs.def
@@ -92,7 +92,9 @@ TLI_DEFINE_VECFUNC("llvm.atan.f64", "_simd_atan_d2", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("atanf", "_simd_atan_f4", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("llvm.atan.f32", "_simd_atan_f4", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("atan2", "_simd_atan2_d2", FIXED(2), "_ZGV_LLVM_N2vv")
+TLI_DEFINE_VECFUNC("llvm.atan2.f64", "_simd_atan2_d2", FIXED(2), "_ZGV_LLVM_N2vv")
 TLI_DEFINE_VECFUNC("atan2f", "_simd_atan2_f4", FIXED(4), "_ZGV_LLVM_N4vv")
+TLI_DEFINE_VECFUNC("llvm.atan2.f32", "_simd_atan2_f4", FIXED(4), "_ZGV_LLVM_N4vv")
 
 TLI_DEFINE_VECFUNC("cos", "_simd_cos_d2", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("llvm.cos.f64", "_simd_cos_d2", FIXED(2), "_ZGV_LLVM_N2v")
@@ -531,6 +533,7 @@ TLI_DEFINE_VECFUNC("atan", "_ZGVnN2v_atan", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("llvm.atan.f64", "_ZGVnN2v_atan", FIXED(2), "_ZGV_LLVM_N2v")
 
 TLI_DEFINE_VECFUNC("atan2", "_ZGVnN2vv_atan2", FIXED(2), "_ZGV_LLVM_N2vv")
+TLI_DEFINE_VECFUNC("llvm.atan2.f64", "_ZGVnN2vv_atan2", FIXED(2), "_ZGV_LLVM_N2vv")
 
 TLI_DEFINE_VECFUNC("atanh", "_ZGVnN2v_atanh", FIXED(2), "_ZGV_LLVM_N2v")
 
@@ -635,6 +638,7 @@ TLI_DEFINE_VECFUNC("atanf", "_ZGVnN4v_atanf", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("llvm.atan.f32", "_ZGVnN4v_atanf", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC("atan2f", "_ZGVnN4vv_atan2f", FIXED(4), "_ZGV_LLVM_N4vv")
+TLI_DEFINE_VECFUNC("llvm.atan2.f32", "_ZGVnN4vv_atan2f", FIXED(4), "_ZGV_LLVM_N4vv")
 
 TLI_DEFINE_VECFUNC("atanhf", "_ZGVnN4v_atanhf", FIXED(4), "_ZGV_LLVM_N4v")
 
@@ -748,6 +752,8 @@ TLI_DEFINE_VECFUNC("llvm.atan.f32", "_ZGVsMxv_atanf", SCALABLE(4), MASKED, "_ZGV
 
 TLI_DEFINE_VECFUNC("atan2", "_ZGVsMxvv_atan2",  SCALABLE(2), MASKED, "_ZGVsMxvv")
 TLI_DEFINE_VECFUNC("atan2f", "_ZGVsMxvv_atan2f", SCALABLE(4), MASKED, "_ZGVsMxvv")
+TLI_DEFINE_VECFUNC("llvm.atan2.f64", "_ZGVsMxvv_atan2",  SCALABLE(2), MASKED, "_ZGVsMxvv")
+TLI_DEFINE_VECFUNC("llvm.atan2.f32", "_ZGVsMxvv_atan2f", SCALABLE(4), MASKED, "_ZGVsMxvv")
 
 TLI_DEFINE_VECFUNC("atanh", "_ZGVsMxv_atanh",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("atanhf", "_ZGVsMxv_atanhf", SCALABLE(4), MASKED, "_ZGVsMxv")
@@ -933,6 +939,11 @@ TLI_DEFINE_VECFUNC("atan2f", "armpl_vatan2q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N
 TLI_DEFINE_VECFUNC("atan2", "armpl_svatan2_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv")
 TLI_DEFINE_VECFUNC("atan2f", "armpl_svatan2_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv")
 
+TLI_DEFINE_VECFUNC("llvm.atan2.f64", "armpl_vatan2q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv")
+TLI_DEFINE_VECFUNC("llvm.atan2.f32", "armpl_vatan2q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv")
+TLI_DEFINE_VECFUNC("llvm.atan2.f64", "armpl_svatan2_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvv")
+TLI_DEFINE_VECFUNC("llvm.atan2.f32", "armpl_svatan2_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv")
+
 TLI_DEFINE_VECFUNC("atanh", "armpl_vatanhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("atanhf", "armpl_vatanhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("atanh", "armpl_svatanh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index 2d19e36cc842..d1d0c5ff8734 100644
--- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -155,6 +155,7 @@ def : GINodeEquiv<G_FTAN, ftan>;
 def : GINodeEquiv<G_FACOS, facos>;
 def : GINodeEquiv<G_FASIN, fasin>;
 def : GINodeEquiv<G_FATAN, fatan>;
+def : GINodeEquiv<G_FATAN2, fatan2>;
 def : GINodeEquiv<G_FCOSH, fcosh>;
 def : GINodeEquiv<G_FSINH, fsinh>;
 def : GINodeEquiv<G_FTANH, ftanh>;
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 98aece0d68d6..062dbbe904de 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -457,6 +457,8 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
     RTLIBCASE(ACOS_F);
   case TargetOpcode::G_FATAN:
     RTLIBCASE(ATAN_F);
+  case TargetOpcode::G_FATAN2:
+    RTLIBCASE(ATAN2_F);
   case TargetOpcode::G_FSINH:
     RTLIBCASE(SINH_F);
   case TargetOpcode::G_FCOSH:
@@ -1202,6 +1204,7 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
   case TargetOpcode::G_FACOS:
   case TargetOpcode::G_FASIN:
   case TargetOpcode::G_FATAN:
+  case TargetOpcode::G_FATAN2:
   case TargetOpcode::G_FCOSH:
   case TargetOpcode::G_FSINH:
   case TargetOpcode::G_FTANH:
@@ -3122,6 +3125,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
   case TargetOpcode::G_FACOS:
   case TargetOpcode::G_FASIN:
   case TargetOpcode::G_FATAN:
+  case TargetOpcode::G_FATAN2:
   case TargetOpcode::G_FCOSH:
   case TargetOpcode::G_FSINH:
   case TargetOpcode::G_FTANH:
@@ -5141,6 +5145,7 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
   case G_FACOS:
   case G_FASIN:
   case G_FATAN:
+  case G_FATAN2:
   case G_FCOSH:
   case G_FSINH:
   case G_FTANH:
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 9574464207d9..722ceea29c95 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -828,6 +828,7 @@ bool llvm::isKnownNeverNaN(Register Val, const MachineRegisterInfo &MRI,
   case TargetOpcode::G_FACOS:
   case TargetOpcode::G_FASIN:
   case TargetOpcode::G_FATAN:
+  case TargetOpcode::G_FATAN2:
   case TargetOpcode::G_FCOSH:
   case TargetOpcode::G_FSINH:
   case TargetOpcode::G_FTANH:
@@ -1715,6 +1716,7 @@ bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) {
   case TargetOpcode::G_FACOS:
   case TargetOpcode::G_FASIN:
   case TargetOpcode::G_FATAN:
+  case TargetOpcode::G_FATAN2:
   case TargetOpcode::G_FCOSH:
   case TargetOpcode::G_FSINH:
   case TargetOpcode::G_FTANH:
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 5a848ada9dd8..4c0cd1ac3d45 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -734,18 +734,19 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Promote);
   }
 
-  for (auto Op : {ISD::FREM,         ISD::FPOW,          ISD::FPOWI,
-                  ISD::FCOS,         ISD::FSIN,          ISD::FSINCOS,
-                  ISD::FACOS,        ISD::FASIN,         ISD::FATAN,
-                  ISD::FCOSH,        ISD::FSINH,         ISD::FTANH,
-                  ISD::FTAN,         ISD::FEXP,          ISD::FEXP2,
-                  ISD::FEXP10,       ISD::FLOG,          ISD::FLOG2,
-                  ISD::FLOG10,       ISD::STRICT_FREM,   ISD::STRICT_FPOW,
-                  ISD::STRICT_FPOWI, ISD::STRICT_FCOS,   ISD::STRICT_FSIN,
-                  ISD::STRICT_FACOS, ISD::STRICT_FASIN,  ISD::STRICT_FATAN,
-                  ISD::STRICT_FCOSH, ISD::STRICT_FSINH,  ISD::STRICT_FTANH,
-                  ISD::STRICT_FEXP,  ISD::STRICT_FEXP2,  ISD::STRICT_FLOG,
-                  ISD::STRICT_FLOG2, ISD::STRICT_FLOG10, ISD::STRICT_FTAN}) {
+  for (auto Op : {ISD::FREM,          ISD::FPOW,          ISD::FPOWI,
+                  ISD::FCOS,          ISD::FSIN,          ISD::FSINCOS,
+                  ISD::FACOS,         ISD::FASIN,         ISD::FATAN,
+                  ISD::FATAN2,        ISD::FCOSH,         ISD::FSINH,
+                  ISD::FTANH,         ISD::FTAN,          ISD::FEXP,
+                  ISD::FEXP2,         ISD::FEXP10,        ISD::FLOG,
+                  ISD::FLOG2,         ISD::FLOG10,        ISD::STRICT_FREM,
+                  ISD::STRICT_FPOW,   ISD::STRICT_FPOWI,  ISD::STRICT_FCOS,
+                  ISD::STRICT_FSIN,   ISD::STRICT_FACOS,  ISD::STRICT_FASIN,
+                  ISD::STRICT_FATAN,  ISD::STRICT_FATAN2, ISD::STRICT_FCOSH,
+                  ISD::STRICT_FSINH,  ISD::STRICT_FTANH,  ISD::STRICT_FEXP,
+                  ISD::STRICT_FEXP2,  ISD::STRICT_FLOG,   ISD::STRICT_FLOG2,
+                  ISD::STRICT_FLOG10, ISD::STRICT_FTAN}) {
     setOperationAction(Op, MVT::f16, Promote);
     setOperationAction(Op, MVT::v4f16, Expand);
     setOperationAction(Op, MVT::v8f16, Expand);
@@ -1190,7 +1191,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     // silliness like this:
     // clang-format off
     for (auto Op :
-         {ISD::SELECT,            ISD::SELECT_CC,
+         {ISD::SELECT,            ISD::SELECT_CC,      ISD::FATAN2,
           ISD::BR_CC,             ISD::FADD,           ISD::FSUB,
           ISD::FMUL,              ISD::FDIV,           ISD::FMA,
           ISD::FNEG,              ISD::FABS,           ISD::FCEIL,
@@ -1649,6 +1650,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::FACOS, VT, Expand);
       setOperationAction(ISD::FASIN, VT, Expand);
       setOperationAction(ISD::FATAN, VT, Expand);
+      setOperationAction(ISD::FATAN2, VT, Expand);
       setOperationAction(ISD::FCOSH, VT, Expand);
       setOperationAction(ISD::FSINH, VT, Expand);
       setOperationAction(ISD::FTANH, VT, Expand);
@@ -1904,6 +1906,7 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT) {
     setOperationAction(ISD::FASIN, VT, Expand);
     setOperationAction(ISD::FACOS, VT, Expand);
     setOperationAction(ISD::FATAN, VT, Expand);
+    setOperationAction(ISD::FATAN2, VT, Expand);
     setOperationAction(ISD::FSINH, VT, Expand);
     setOperationAction(ISD::FCOSH, VT, Expand);
     setOperationAction(ISD::FTANH, VT, Expand);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index fb6c23a96456..5cd1fea75025 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -269,9 +269,10 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .libcallFor({{s64, s128}})
       .minScalarOrElt(1, MinFPScalar);
 
-  getActionDefinitionsBuilder(
-      {G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2, G_FLOG10, G_FTAN, G_FEXP,
-       G_FEXP2, G_FEXP10, G_FACOS, G_FASIN, G_FATAN, G_FCOSH, G_FSINH, G_FTANH})
+  getActionDefinitionsBuilder({G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2,
+                               G_FLOG10, G_FTAN, G_FEXP, G_FEXP2, G_FEXP10,
+                               G_FACOS, G_FASIN, G_FATAN, G_FATAN2, G_FCOSH,
+                               G_FSINH, G_FTANH})
       // We need a call for these, so we always need to scalarize.
       .scalarize(0)
       // Regardless of FP16 support, widen 16-bit elements to 32-bits.
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
index 314c5458e309..ed7bcff5160f 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
@@ -2345,6 +2345,14 @@ define float @test_atan_f32(float %x) {
   ret float %y
 }
 
+declare float @llvm.atan2.f32(float, float)
+define float @test_atan2_f32(float %x, float %y) {
+  ; CHECK-LABEL: name:            test_atan2_f32
+  ; CHECK: %{{[0-9]+}}:_(s32) = G_FATAN2 %{{[0-9]+}}
+  %z = call float @llvm.atan2.f32(float %x, float %y)
+  ret float %z
+}
+
 declare float @llvm.cosh.f32(float)
 define float @test_cosh_f32(float %x) {
   ; CHECK-LABEL: name:            test_cosh_f32
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-atan2.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-atan2.mir
new file mode 100644
index 000000000000..7e794c4556f1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-atan2.mir
@@ -0,0 +1,333 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=legalizer -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            test_v4f16.atan2
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0, $d1
+
+    ; CHECK-LABEL: name: test_v4f16.atan2
+    ; CHECK: liveins: $d0, $d1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $d0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $d1
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; CHECK-NEXT: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; CHECK-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; CHECK-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV4]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[FPEXT]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[FPEXT1]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY2]](s32)
+    ; CHECK-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; CHECK-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV5]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[FPEXT2]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[FPEXT3]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY3]](s32)
+    ; CHECK-NEXT: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; CHECK-NEXT: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[UV6]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[FPEXT4]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[FPEXT5]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY4]](s32)
+    ; CHECK-NEXT: [[FPEXT6:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; CHECK-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[UV7]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[FPEXT6]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[FPEXT7]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY5]](s32)
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16), [[FPTRUNC3]](s16)
+    ; CHECK-NEXT: $d0 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $d0
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = COPY $d1
+    %2:_(<4 x s16>) = G_FATAN2 %0, %1
+    $d0 = COPY %2(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v8f16.atan2
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: test_v8f16.atan2
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<8 x s16>)
+    ; CHECK-NEXT: [[UV8:%[0-9]+]]:_(s16), [[UV9:%[0-9]+]]:_(s16), [[UV10:%[0-9]+]]:_(s16), [[UV11:%[0-9]+]]:_(s16), [[UV12:%[0-9]+]]:_(s16), [[UV13:%[0-9]+]]:_(s16), [[UV14:%[0-9]+]]:_(s16), [[UV15:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<8 x s16>)
+    ; CHECK-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; CHECK-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV8]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[FPEXT]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[FPEXT1]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY2]](s32)
+    ; CHECK-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; CHECK-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV9]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[FPEXT2]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[FPEXT3]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY3]](s32)
+    ; CHECK-NEXT: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; CHECK-NEXT: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[UV10]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[FPEXT4]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[FPEXT5]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY4]](s32)
+    ; CHECK-NEXT: [[FPEXT6:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; CHECK-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[UV11]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[FPEXT6]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[FPEXT7]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY5]](s32)
+    ; CHECK-NEXT: [[FPEXT8:%[0-9]+]]:_(s32) = G_FPEXT [[UV4]](s16)
+    ; CHECK-NEXT: [[FPEXT9:%[0-9]+]]:_(s32) = G_FPEXT [[UV12]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[FPEXT8]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[FPEXT9]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[FPTRUNC4:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY6]](s32)
+    ; CHECK-NEXT: [[FPEXT10:%[0-9]+]]:_(s32) = G_FPEXT [[UV5]](s16)
+    ; CHECK-NEXT: [[FPEXT11:%[0-9]+]]:_(s32) = G_FPEXT [[UV13]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[FPEXT10]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[FPEXT11]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[FPTRUNC5:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY7]](s32)
+    ; CHECK-NEXT: [[FPEXT12:%[0-9]+]]:_(s32) = G_FPEXT [[UV6]](s16)
+    ; CHECK-NEXT: [[FPEXT13:%[0-9]+]]:_(s32) = G_FPEXT [[UV14]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[FPEXT12]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[FPEXT13]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY8:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[FPTRUNC6:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY8]](s32)
+    ; CHECK-NEXT: [[FPEXT14:%[0-9]+]]:_(s32) = G_FPEXT [[UV7]](s16)
+    ; CHECK-NEXT: [[FPEXT15:%[0-9]+]]:_(s32) = G_FPEXT [[UV15]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[FPEXT14]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[FPEXT15]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY9:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[FPTRUNC7:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY9]](s32)
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16), [[FPTRUNC3]](s16), [[FPTRUNC4]](s16), [[FPTRUNC5]](s16), [[FPTRUNC6]](s16), [[FPTRUNC7]](s16)
+    ; CHECK-NEXT: $q0 = COPY [[BUILD_VECTOR]](<8 x s16>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = COPY $q1
+    %2:_(<8 x s16>) = G_FATAN2 %0, %1
+    $q0 = COPY %2(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f32.atan2
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0, $d1
+
+    ; CHECK-LABEL: name: test_v2f32.atan2
+    ; CHECK: liveins: $d0, $d1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[UV]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[UV2]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[UV1]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[UV3]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: $d0 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $d0
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = COPY $d1
+    %2:_(<2 x s32>) = G_FATAN2 %0, %1
+    $d0 = COPY %2(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v4f32.atan2
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: test_v4f32.atan2
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
+    ; CHECK-NEXT: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[UV]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[UV4]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[UV1]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[UV5]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[UV2]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[UV6]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[UV3]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[UV7]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32)
+    ; CHECK-NEXT: $q0 = COPY [[BUILD_VECTOR]](<4 x s32>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = COPY $q1
+    %2:_(<4 x s32>) = G_FATAN2 %0, %1
+    $q0 = COPY %2(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f64.atan2
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: test_v2f64.atan2
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; CHECK-NEXT: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $d0 = COPY [[UV]](s64)
+    ; CHECK-NEXT: $d1 = COPY [[UV2]](s64)
+    ; CHECK-NEXT: BL &atan2, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $d0, implicit $d1, implicit-def $d0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $d0 = COPY [[UV1]](s64)
+    ; CHECK-NEXT: $d1 = COPY [[UV3]](s64)
+    ; CHECK-NEXT: BL &atan2, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $d0, implicit $d1, implicit-def $d0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[COPY2]](s64), [[COPY3]](s64)
+    ; CHECK-NEXT: $q0 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = COPY $q1
+    %2:_(<2 x s64>) = G_FATAN2 %0, %1
+    $q0 = COPY %2(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_atan2_half
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $h0, $h1
+
+    ; CHECK-LABEL: name: test_atan2_half
+    ; CHECK: liveins: $h0, $h1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY $h0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s16) = COPY $h1
+    ; CHECK-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[COPY]](s16)
+    ; CHECK-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[COPY1]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $s0 = COPY [[FPEXT]](s32)
+    ; CHECK-NEXT: $s1 = COPY [[FPEXT1]](s32)
+    ; CHECK-NEXT: BL &atan2f, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s0, implicit $s1, implicit-def $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY2]](s32)
+    ; CHECK-NEXT: $h0 = COPY [[FPTRUNC]](s16)
+    ; CHECK-NEXT: RET_ReallyLR implicit $h0
+    %0:_(s16) = COPY $h0
+    %1:_(s16) = COPY $h1
+    %2:_(s16) = G_FATAN2 %0, %1
+    $h0 = COPY %2(s16)
+    RET_ReallyLR implicit $h0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 4d096b7231c7..80b6e4f6d528 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -718,8 +718,9 @@
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FATAN2 (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined 
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FCOSH (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
diff --git a/llvm/test/CodeGen/AArch64/f16-instructions.ll b/llvm/test/CodeGen/AArch64/f16-instructions.ll
index e058c83f274f..5460a376931a 100644
--- a/llvm/test/CodeGen/AArch64/f16-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/f16-instructions.ll
@@ -1114,6 +1114,7 @@ declare half @llvm.tan.f16(half %a) #0
 declare half @llvm.asin.f16(half %a) #0
 declare half @llvm.acos.f16(half %a) #0
 declare half @llvm.atan.f16(half %a) #0
+declare half @llvm.atan2.f16(half %a, half %b) #0
 declare half @llvm.sinh.f16(half %a) #0
 declare half @llvm.cosh.f16(half %a) #0
 declare half @llvm.tanh.f16(half %a) #0
@@ -1246,6 +1247,11 @@ define half @test_atan(half %a) #0 {
   ret half %r
 }
 
+define half @test_atan2(half %a, half %b) #0 {
+  %r = call half @llvm.atan2.f16(half %a, half %b)
+  ret half %r
+}
+
 define half @test_cosh(half %a) #0 {
 ; CHECK-LABEL: test_cosh:
 ; CHECK:       // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/fp-intrinsics-fp16.ll b/llvm/test/CodeGen/AArch64/fp-intrinsics-fp16.ll
index cbdfb4c93277..3aeefab52c6f 100644
--- a/llvm/test/CodeGen/AArch64/fp-intrinsics-fp16.ll
+++ b/llvm/test/CodeGen/AArch64/fp-intrinsics-fp16.ll
@@ -398,6 +398,22 @@ define half @atan_f16(half %x) #0 {
   ret half %val
 }
 
+define half @atan2_f16(half %x, half %y) #0 {
+; CHECK-LABEL: atan2_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl atan2f
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %val = call half @llvm.experimental.constrained.atan2.f16(half %x, half %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret half %val
+}
+
 define half @sinh_f16(half %x) #0 {
 ; CHECK-LABEL: sinh_f16:
 ; CHECK:       // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/fp-intrinsics.ll b/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
index 62b4a79b26d8..1664fa3ce56a 100644
--- a/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
@@ -174,6 +174,13 @@ define float @atan_f32(float %x) #0 {
   ret float %val
 }
 
+; CHECK-LABEL: atan2_f32:
+; CHECK: bl atan2f
+define float @atan2_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.atan2.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
 ; CHECK-LABEL: sinh_f32:
 ; CHECK: bl sinhf
 define float @sinh_f32(float %x) #0 {
@@ -707,6 +714,13 @@ define double @atan_f64(double %x) #0 {
   ret double %val
 }
 
+; CHECK-LABEL: atan2_f64:
+; CHECK: bl atan2
+define double @atan2_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.atan2.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
 ; CHECK-LABEL: sinh_f64:
 ; CHECK: bl sinh
 define double @sinh_f64(double %x) #0 {
@@ -1240,6 +1254,13 @@ define fp128 @atan_f128(fp128 %x) #0 {
   ret fp128 %val
 }
 
+; CHECK-LABEL: atan2_f128:
+; CHECK: bl atan2l
+define fp128 @atan2_f128(fp128 %x, fp128 %y) #0 {
+  %val = call fp128 @llvm.experimental.constrained.atan2.f128(fp128 %x, fp128 %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
 ; CHECK-LABEL: sinh_f128:
 ; CHECK: bl sinhl
 define fp128 @sinh_f128(fp128 %x) #0 {
@@ -1666,6 +1687,13 @@ define <1 x double> @atan_v1f64(<1 x double> %x, <1 x double> %y) #0 {
   ret <1 x double> %val
 }
 
+; CHECK-LABEL: atan2_v1f64:
+; CHECK: bl atan2
+define <1 x double> @atan2_v1f64(<1 x double> %x, <1 x double> %y) #0 {
+  %val = call <1 x double> @llvm.experimental.constrained.atan2.v1f64(<1 x double> %x, <1 x double> %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret <1 x double> %val
+}
+
 ; CHECK-LABEL: sinh_v1f64:
 ; CHECK: bl sinh
 define <1 x double> @sinh_v1f64(<1 x double> %x, <1 x double> %y) #0 {
@@ -1755,6 +1783,7 @@ declare float @llvm.experimental.constrained.tan.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.asin.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.acos.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.atan.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.atan2.f32(float, float, metadata, metadata)
 declare float @llvm.experimental.constrained.sinh.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.cosh.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.tanh.f32(float, metadata, metadata)
@@ -1806,6 +1835,7 @@ declare double @llvm.experimental.constrained.tan.f64(double, metadata, metadata
 declare double @llvm.experimental.constrained.asin.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.acos.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.atan.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.atan2.f64(double, double, metadata, metadata)
 declare double @llvm.experimental.constrained.sinh.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.cosh.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.tanh.f64(double, metadata, metadata)
@@ -1857,6 +1887,7 @@ declare fp128 @llvm.experimental.constrained.tan.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.asin.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.acos.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.atan.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.atan2.f128(fp128, fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.sinh.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.cosh.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.tanh.f128(fp128, metadata, metadata)
diff --git a/llvm/test/CodeGen/AArch64/illegal-float-ops.ll b/llvm/test/CodeGen/AArch64/illegal-float-ops.ll
index 08f6bb6f2853..5ec66b784c03 100644
--- a/llvm/test/CodeGen/AArch64/illegal-float-ops.ll
+++ b/llvm/test/CodeGen/AArch64/illegal-float-ops.ll
@@ -243,6 +243,27 @@ define void @test_atan(float %float, double %double, fp128 %fp128) {
   ret void
 }
 
+declare float @llvm.atan2.f32(float, float)
+declare double @llvm.atan2.f64(double, double)
+declare fp128 @llvm.atan2.f128(fp128, fp128)
+
+define void @test_atan2(float %float1, double %double1, fp128 %fp1281, float %float2, double %double2, fp128 %fp1282) {
+; CHECK-LABEL: test_atan2:
+
+   %atan2float = call float @llvm.atan2.f32(float %float1, float %float2)
+   store float %atan2float, ptr @varfloat
+; CHECK: bl atan2f
+
+   %atan2double = call double @llvm.atan2.f64(double %double1, double %double2)
+   store double %atan2double, ptr @vardouble
+; CHECK: bl atan2
+
+   %atan2fp128 = call fp128 @llvm.atan2.f128(fp128 %fp1281, fp128 %fp1282)
+   store fp128 %atan2fp128, ptr @varfp128
+; CHECK: bl atan2l
+  ret void
+}
+
 declare float @llvm.cosh.f32(float)
 declare double @llvm.cosh.f64(double)
 declare fp128 @llvm.cosh.f128(fp128)
diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll
index 7b173bda5615..26fb4ca602da 100644
--- a/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll
+++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll
@@ -15,7 +15,7 @@ declare <vscale x 2 x double> @llvm.cos.nxv2f64(<vscale x 2 x double>)
 declare <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float>)
 
 ;.
-; CHECK: @llvm.compiler.used = appending global [64 x ptr] [ptr @armpl_vcosq_f64, ptr @armpl_vcosq_f32, ptr @armpl_svcos_f64_x, ptr @armpl_svcos_f32_x, ptr @armpl_vexpq_f64, ptr @armpl_vexpq_f32, ptr @armpl_svexp_f64_x, ptr @armpl_svexp_f32_x, ptr @armpl_vexp10q_f64, ptr @armpl_vexp10q_f32, ptr @armpl_svexp10_f64_x, ptr @armpl_svexp10_f32_x, ptr @armpl_vexp2q_f64, ptr @armpl_vexp2q_f32, ptr @armpl_svexp2_f64_x, ptr @armpl_svexp2_f32_x, ptr @armpl_vlogq_f64, ptr @armpl_vlogq_f32, ptr @armpl_svlog_f64_x, ptr @armpl_svlog_f32_x, ptr @armpl_vlog10q_f64, ptr @armpl_vlog10q_f32, ptr @armpl_svlog10_f64_x, ptr @armpl_svlog10_f32_x, ptr @armpl_vlog2q_f64, ptr @armpl_vlog2q_f32, ptr @armpl_svlog2_f64_x, ptr @armpl_svlog2_f32_x, ptr @armpl_vpowq_f64, ptr @armpl_vpowq_f32, ptr @armpl_svpow_f64_x, ptr @armpl_svpow_f32_x, ptr @armpl_vsinq_f64, ptr @armpl_vsinq_f32, ptr @armpl_svsin_f64_x, ptr @armpl_svsin_f32_x, ptr @armpl_vtanq_f64, ptr @armpl_vtanq_f32, ptr @armpl_svtan_f64_x, ptr @armpl_svtan_f32_x, ptr @armpl_vacosq_f64, ptr @armpl_vacosq_f32, ptr @armpl_svacos_f64_x, ptr @armpl_svacos_f32_x, ptr @armpl_vasinq_f64, ptr @armpl_vasinq_f32, ptr @armpl_svasin_f64_x, ptr @armpl_svasin_f32_x, ptr @armpl_vatanq_f64, ptr @armpl_vatanq_f32, ptr @armpl_svatan_f64_x, ptr @armpl_svatan_f32_x, ptr @armpl_vcoshq_f64, ptr @armpl_vcoshq_f32, ptr @armpl_svcosh_f64_x, ptr @armpl_svcosh_f32_x, ptr @armpl_vsinhq_f64, ptr @armpl_vsinhq_f32, ptr @armpl_svsinh_f64_x, ptr @armpl_svsinh_f32_x, ptr @armpl_vtanhq_f64, ptr @armpl_vtanhq_f32, ptr @armpl_svtanh_f64_x, ptr @armpl_svtanh_f32_x], section "llvm.metadata"
+; CHECK: @llvm.compiler.used = appending global [68 x ptr] [ptr @armpl_vcosq_f64, ptr @armpl_vcosq_f32, ptr @armpl_svcos_f64_x, ptr @armpl_svcos_f32_x, ptr @armpl_vexpq_f64, ptr @armpl_vexpq_f32, ptr @armpl_svexp_f64_x, ptr @armpl_svexp_f32_x, ptr @armpl_vexp10q_f64, ptr @armpl_vexp10q_f32, ptr @armpl_svexp10_f64_x, ptr @armpl_svexp10_f32_x, ptr @armpl_vexp2q_f64, ptr @armpl_vexp2q_f32, ptr @armpl_svexp2_f64_x, ptr @armpl_svexp2_f32_x, ptr @armpl_vlogq_f64, ptr @armpl_vlogq_f32, ptr @armpl_svlog_f64_x, ptr @armpl_svlog_f32_x, ptr @armpl_vlog10q_f64, ptr @armpl_vlog10q_f32, ptr @armpl_svlog10_f64_x, ptr @armpl_svlog10_f32_x, ptr @armpl_vlog2q_f64, ptr @armpl_vlog2q_f32, ptr @armpl_svlog2_f64_x, ptr @armpl_svlog2_f32_x, ptr @armpl_vpowq_f64, ptr @armpl_vpowq_f32, ptr @armpl_svpow_f64_x, ptr @armpl_svpow_f32_x, ptr @armpl_vsinq_f64, ptr @armpl_vsinq_f32, ptr @armpl_svsin_f64_x, ptr @armpl_svsin_f32_x, ptr @armpl_vtanq_f64, ptr @armpl_vtanq_f32, ptr @armpl_svtan_f64_x, ptr @armpl_svtan_f32_x, ptr @armpl_vacosq_f64, ptr @armpl_vacosq_f32, ptr @armpl_svacos_f64_x, ptr @armpl_svacos_f32_x, ptr @armpl_vasinq_f64, ptr @armpl_vasinq_f32, ptr @armpl_svasin_f64_x, ptr @armpl_svasin_f32_x, ptr @armpl_vatanq_f64, ptr @armpl_vatanq_f32, ptr @armpl_svatan_f64_x, ptr @armpl_svatan_f32_x, ptr @armpl_vatan2q_f64, ptr @armpl_vatan2q_f32, ptr @armpl_svatan2_f64_x, ptr @armpl_svatan2_f32_x, ptr @armpl_vcoshq_f64, ptr @armpl_vcoshq_f32, ptr @armpl_svcosh_f64_x, ptr @armpl_svcosh_f32_x, ptr @armpl_vsinhq_f64, ptr @armpl_vsinhq_f32, ptr @armpl_svsinh_f64_x, ptr @armpl_svsinh_f32_x, ptr @armpl_vtanhq_f64, ptr @armpl_vtanhq_f32, ptr @armpl_svtanh_f64_x, ptr @armpl_svtanh_f32_x], section "llvm.metadata"
 
 ;.
 define <2 x double> @llvm_cos_f64(<2 x double> %in) {
@@ -598,6 +598,51 @@ define <vscale x 4 x float> @llvm_atan_vscale_f32(<vscale x 4 x float> %in) #0 {
   ret <vscale x 4 x float> %1
 }
 
+declare <2 x double> @llvm.atan2.v2f64(<2 x double>, <2 x double>)
+declare <4 x float> @llvm.atan2.v4f32(<4 x float>, <4 x float>)
+declare <vscale x 2 x double> @llvm.atan2.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+declare <vscale x 4 x float> @llvm.atan2.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+
+define <2 x double> @llvm_atan2_f64(<2 x double> %in1, <2 x double> %in2) {
+; CHECK-LABEL: define <2 x double> @llvm_atan2_f64
+; CHECK-SAME: (<2 x double> [[IN1:%.*]], <2 x double> [[IN2:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @armpl_vatan2q_f64(<2 x double> [[IN1]], <2 x double> [[IN2]])
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call fast <2 x double> @llvm.atan2.v2f64(<2 x double> %in1, <2 x double> %in2)
+  ret <2 x double> %1
+}
+
+define <4 x float> @llvm_atan2_f32(<4 x float> %in1, <4 x float> %in2) {
+; CHECK-LABEL: define <4 x float> @llvm_atan2_f32
+; CHECK-SAME: (<4 x float> [[IN1:%.*]], <4 x float> [[IN2:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @armpl_vatan2q_f32(<4 x float> [[IN1]], <4 x float> [[IN2]])
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call fast <4 x float> @llvm.atan2.v4f32(<4 x float> %in1, <4 x float> %in2)
+  ret <4 x float> %1
+}
+
+define <vscale x 2 x double> @llvm_atan2_vscale_f64(<vscale x 2 x double> %in1, <vscale x 2 x double> %in2) #0 {
+; CHECK-LABEL: define <vscale x 2 x double> @llvm_atan2_vscale_f64
+; CHECK-SAME: (<vscale x 2 x double> [[IN1:%.*]], <vscale x 2 x double> [[IN2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svatan2_f64_x(<vscale x 2 x double> [[IN1]], <vscale x 2 x double> [[IN2]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+;
+  %1 = call fast <vscale x 2 x double> @llvm.atan2.nxv2f64(<vscale x 2 x double> %in1, <vscale x 2 x double> %in2)
+  ret <vscale x 2 x double> %1
+}
+
+define <vscale x 4 x float> @llvm_atan2_vscale_f32(<vscale x 4 x float> %in1, <vscale x 4 x float> %in2) #0 {
+; CHECK-LABEL: define <vscale x 4 x float> @llvm_atan2_vscale_f32
+; CHECK-SAME: (<vscale x 4 x float> [[IN1:%.*]], <vscale x 4 x float> [[IN2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svatan2_f32_x(<vscale x 4 x float> [[IN1]], <vscale x 4 x float> [[IN2]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %1 = call fast <vscale x 4 x float> @llvm.atan2.nxv4f32(<vscale x 4 x float> %in1, <vscale x 4 x float> %in2)
+  ret <vscale x 4 x float> %1
+}
+
 declare <2 x double> @llvm.cosh.v2f64(<2 x double>)
 declare <4 x float> @llvm.cosh.v4f32(<4 x float>)
 declare <vscale x 2 x double> @llvm.cosh.nxv2f64(<vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll
index 155587026b46..07edb4649569 100644
--- a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll
@@ -4,7 +4,7 @@
 target triple = "aarch64-unknown-linux-gnu"
 
 ;.
-; CHECK: @llvm.compiler.used = appending global [32 x ptr] [ptr @_ZGVsMxv_cos, ptr @_ZGVsMxv_cosf, ptr @_ZGVsMxv_exp, ptr @_ZGVsMxv_expf, ptr @_ZGVsMxv_exp10, ptr @_ZGVsMxv_exp10f, ptr @_ZGVsMxv_exp2, ptr @_ZGVsMxv_exp2f, ptr @_ZGVsMxv_log, ptr @_ZGVsMxv_logf, ptr @_ZGVsMxv_log10, ptr @_ZGVsMxv_log10f, ptr @_ZGVsMxv_log2, ptr @_ZGVsMxv_log2f, ptr @_ZGVsMxvv_pow, ptr @_ZGVsMxvv_powf, ptr @_ZGVsMxv_sin, ptr @_ZGVsMxv_sinf, ptr @_ZGVsMxv_tan, ptr @_ZGVsMxv_tanf, ptr @_ZGVsMxv_acos, ptr @_ZGVsMxv_acosf, ptr @_ZGVsMxv_asin, ptr @_ZGVsMxv_asinf, ptr @_ZGVsMxv_atan, ptr @_ZGVsMxv_atanf, ptr @_ZGVsMxv_cosh, ptr @_ZGVsMxv_coshf, ptr @_ZGVsMxv_sinh, ptr @_ZGVsMxv_sinhf, ptr @_ZGVsMxv_tanh, ptr @_ZGVsMxv_tanhf], section "llvm.metadata"
+; CHECK: @llvm.compiler.used = appending global [34 x ptr] [ptr @_ZGVsMxv_cos, ptr @_ZGVsMxv_cosf, ptr @_ZGVsMxv_exp, ptr @_ZGVsMxv_expf, ptr @_ZGVsMxv_exp10, ptr @_ZGVsMxv_exp10f, ptr @_ZGVsMxv_exp2, ptr @_ZGVsMxv_exp2f, ptr @_ZGVsMxv_log, ptr @_ZGVsMxv_logf, ptr @_ZGVsMxv_log10, ptr @_ZGVsMxv_log10f, ptr @_ZGVsMxv_log2, ptr @_ZGVsMxv_log2f, ptr @_ZGVsMxvv_pow, ptr @_ZGVsMxvv_powf, ptr @_ZGVsMxv_sin, ptr @_ZGVsMxv_sinf, ptr @_ZGVsMxv_tan, ptr @_ZGVsMxv_tanf, ptr @_ZGVsMxv_acos, ptr @_ZGVsMxv_acosf, ptr @_ZGVsMxv_asin, ptr @_ZGVsMxv_asinf, ptr @_ZGVsMxv_atan, ptr @_ZGVsMxv_atanf, ptr @_ZGVsMxvv_atan2, ptr @_ZGVsMxvv_atan2f, ptr @_ZGVsMxv_cosh, ptr @_ZGVsMxv_coshf, ptr @_ZGVsMxv_sinh, ptr @_ZGVsMxv_sinhf, ptr @_ZGVsMxv_tanh, ptr @_ZGVsMxv_tanhf], section "llvm.metadata"
 ;.
 define <vscale x 2 x double> @llvm_ceil_vscale_f64(<vscale x 2 x double> %in) {
 ; CHECK-LABEL: @llvm_ceil_vscale_f64(
@@ -438,6 +438,24 @@ define <vscale x 4 x float> @llvm_atan_vscale_f32(<vscale x 4 x float> %in) {
   ret <vscale x 4 x float> %1
 }
 
+define <vscale x 2 x double> @llvm_atan2_vscale_f64(<vscale x 2 x double> %x, <vscale x 2 x double> %y) {
+; CHECK-LABEL: @llvm_atan2_vscale_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxvv_atan2(<vscale x 2 x double> [[INX:%.*]], <vscale x 2 x double> [[INY:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+;
+  %1 = call fast <vscale x 2 x double> @llvm.atan2.nxv2f64(<vscale x 2 x double> %x, <vscale x 2 x double> %y)
+  ret <vscale x 2 x double> %1
+}
+
+define <vscale x 4 x float> @llvm_atan2_vscale_f32(<vscale x 4 x float> %x, <vscale x 4 x float> %y) {
+; CHECK-LABEL: @llvm_atan2_vscale_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxvv_atan2f(<vscale x 4 x float> [[INX:%.*]], <vscale x 4 x float> [[INY:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %1 = call fast <vscale x 4 x float> @llvm.atan2.nxv4f32(<vscale x 4 x float> %x, <vscale x 4 x float> %y)
+  ret <vscale x 4 x float> %1
+}
+
 define <vscale x 2 x double> @llvm_cosh_vscale_f64(<vscale x 2 x double> %in) {
 ; CHECK-LABEL: @llvm_cosh_vscale_f64(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll
index b89bf3d6f2ca..ebe5e7866768 100644
--- a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll
+++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll
@@ -4,7 +4,7 @@
 target triple = "aarch64-unknown-linux-gnu"
 
 ;.
-; CHECK: @llvm.compiler.used = appending global [32 x ptr] [ptr @_ZGVnN2v_cos, ptr @_ZGVnN4v_cosf, ptr @_ZGVnN2v_exp, ptr @_ZGVnN4v_expf, ptr @_ZGVnN2v_exp10, ptr @_ZGVnN4v_exp10f, ptr @_ZGVnN2v_exp2, ptr @_ZGVnN4v_exp2f, ptr @_ZGVnN2v_log, ptr @_ZGVnN4v_logf, ptr @_ZGVnN2v_log10, ptr @_ZGVnN4v_log10f, ptr @_ZGVnN2v_log2, ptr @_ZGVnN4v_log2f, ptr @_ZGVnN2vv_pow, ptr @_ZGVnN4vv_powf, ptr @_ZGVnN2v_sin, ptr @_ZGVnN4v_sinf, ptr @_ZGVnN2v_tan, ptr @_ZGVnN4v_tanf, ptr @_ZGVnN2v_acos, ptr @_ZGVnN4v_acosf, ptr @_ZGVnN2v_asin, ptr @_ZGVnN4v_asinf, ptr @_ZGVnN2v_atan, ptr @_ZGVnN4v_atanf, ptr @_ZGVnN2v_cosh, ptr @_ZGVnN4v_coshf, ptr @_ZGVnN2v_sinh, ptr @_ZGVnN4v_sinhf, ptr @_ZGVnN2v_tanh, ptr @_ZGVnN4v_tanhf], section "llvm.metadata"
+; CHECK: @llvm.compiler.used = appending global [34 x ptr] [ptr @_ZGVnN2v_cos, ptr @_ZGVnN4v_cosf, ptr @_ZGVnN2v_exp, ptr @_ZGVnN4v_expf, ptr @_ZGVnN2v_exp10, ptr @_ZGVnN4v_exp10f, ptr @_ZGVnN2v_exp2, ptr @_ZGVnN4v_exp2f, ptr @_ZGVnN2v_log, ptr @_ZGVnN4v_logf, ptr @_ZGVnN2v_log10, ptr @_ZGVnN4v_log10f, ptr @_ZGVnN2v_log2, ptr @_ZGVnN4v_log2f, ptr @_ZGVnN2vv_pow, ptr @_ZGVnN4vv_powf, ptr @_ZGVnN2v_sin, ptr @_ZGVnN4v_sinf, ptr @_ZGVnN2v_tan, ptr @_ZGVnN4v_tanf, ptr @_ZGVnN2v_acos, ptr @_ZGVnN4v_acosf, ptr @_ZGVnN2v_asin, ptr @_ZGVnN4v_asinf, ptr @_ZGVnN2v_atan, ptr @_ZGVnN4v_atanf, ptr @_ZGVnN2vv_atan2, ptr @_ZGVnN4vv_atan2f, ptr @_ZGVnN2v_cosh, ptr @_ZGVnN4v_coshf, ptr @_ZGVnN2v_sinh, ptr @_ZGVnN4v_sinhf, ptr @_ZGVnN2v_tanh, ptr @_ZGVnN4v_tanhf], section "llvm.metadata"
 ;.
 define <2 x double> @llvm_ceil_f64(<2 x double> %in) {
 ; CHECK-LABEL: @llvm_ceil_f64(
@@ -438,6 +438,24 @@ define <4 x float> @llvm_atan_f32(<4 x float> %in) {
   ret <4 x float> %1
 }
 
+define <2 x double> @llvm_atan2_f64(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: @llvm_atan2_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @_ZGVnN2vv_atan2(<2 x double> [[INX:%.*]], <2 x double> [[INY:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call fast <2 x double> @llvm.atan2.v2f64(<2 x double> %x, <2 x double> %y)
+  ret <2 x double> %1
+}
+
+define <4 x float> @llvm_atan2_f32(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @llvm_atan2_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @_ZGVnN4vv_atan2f(<4 x float> [[INX:%.*]], <4 x float> [[INY:%.*]])
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call fast <4 x float> @llvm.atan2.v4f32(<4 x float> %x, <4 x float> %y)
+  ret <4 x float> %1
+}
+
 define <2 x double> @llvm_cosh_f64(<2 x double> %in) {
 ; CHECK-LABEL: @llvm_cosh_f64(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @_ZGVnN2v_cosh(<2 x double> [[IN:%.*]])
diff --git a/llvm/test/CodeGen/AArch64/vec-libcalls.ll b/llvm/test/CodeGen/AArch64/vec-libcalls.ll
index 43c183981817..854a2d75d4b8 100644
--- a/llvm/test/CodeGen/AArch64/vec-libcalls.ll
+++ b/llvm/test/CodeGen/AArch64/vec-libcalls.ll
@@ -24,6 +24,7 @@ declare <3 x float> @llvm.tan.v3f32(<3 x float>)
 declare <3 x float> @llvm.asin.v3f32(<3 x float>)
 declare <3 x float> @llvm.acos.v3f32(<3 x float>)
 declare <3 x float> @llvm.atan.v3f32(<3 x float>)
+declare <3 x float> @llvm.atan2.v3f32(<3 x float>, <3 x float>)
 declare <3 x float> @llvm.sinh.v3f32(<3 x float>)
 declare <3 x float> @llvm.cosh.v3f32(<3 x float>)
 declare <3 x float> @llvm.tanh.v3f32(<3 x float>)
@@ -428,6 +429,40 @@ define <3 x float> @atan_v3f32(<3 x float> %x) nounwind {
   ret <3 x float> %r
 }
 
+define <3 x float> @atan2_v3f32(<3 x float> %x, <3 x float> %y) nounwind {
+; CHECK-LABEL: atan2_v3f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    stp q0, q1, [sp, #16] // 32-byte Folded Spill
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    mov s1, v1.s[1]
+; CHECK-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-NEXT:    bl atan2f
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldp q0, q1, [sp, #16] // 32-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    // kill: def $s1 killed $s1 killed $q1
+; CHECK-NEXT:    bl atan2f
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldp q0, q1, [sp, #16] // 32-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    mov s1, v1.s[2]
+; CHECK-NEXT:    bl atan2f
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v1.s[2], v0.s[0]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+  %r = call <3 x float> @llvm.atan2.v3f32(<3 x float> %x, <3 x float> %y)
+  ret <3 x float> %r
+}
+
 define <3 x float> @sinh_v3f32(<3 x float> %x) nounwind {
 ; CHECK-LABEL: sinh_v3f32:
 ; CHECK:       // %bb.0:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll
index ac6907609f5e..c06755d29d17 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll
@@ -579,6 +579,54 @@ for.end:
   ret void
 }
 
+declare float @llvm.atan2.f32(float, float) nounwind readnone
+define void @atan2_v4f32_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @atan2_v4f32_intrinsic(
+; CHECK: call <4 x float> @_simd_atan2_f4(<4 x float>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds float, ptr %y, i64 %iv
+  %lv = load float, ptr %gep.y, align 4
+  %call = tail call float @llvm.atan2.f32(float %lv, float %lv)
+  %gep.x = getelementptr inbounds float, ptr %x, i64 %iv
+  store float %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare double @llvm.atan2.f64(double, double) nounwind readnone
+define void @atan2_v2f64_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @atan2_v2f64_intrinsic(
+; CHECK: call <2 x double> @_simd_atan2_d2(<2 x double>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds double, ptr %y, i64 %iv
+  %lv = load double, ptr %gep.y, align 4
+  %call = tail call double @llvm.atan2.f64(double %lv, double %lv)
+  %gep.x = getelementptr inbounds double, ptr %x, i64 %iv
+  store double %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
 declare float @llvm.cosh.f32(float) nounwind readnone
 define void @cosh_v4f32_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
 ; CHECK-LABEL: @cosh_v4f32_intrinsic(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
index f3b47d04d216..30c53bf0177e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --scrub-attributes --filter "call.*(acos|asin|atan|cos|cosh|exp|log|sin|sinh|pow|ceil|copysign|fabs|floor|fma|m..num|nearbyint|rint|round|sqrt|tan|tanh|trunc)" --version 2
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --scrub-attributes --filter "call.*(acos|asin|atan|atan2|cos|cosh|exp|log|sin|sinh|pow|ceil|copysign|fabs|floor|fma|m..num|nearbyint|rint|round|sqrt|tan|tanh|trunc)" --version 2
 
 ; RUN: opt -mattr=+neon -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -S < %s | FileCheck %s --check-prefix=SLEEF-NEON
 ; RUN: opt -mattr=+sve -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S < %s | FileCheck %s --check-prefix=SLEEF-SVE
@@ -231,6 +231,79 @@ define void @atan_f32(ptr noalias %in.ptr, ptr %out.ptr) {
   ret void
 }
 
+declare double @llvm.atan2.f64(double, double)
+declare float @llvm.atan2.f32(float, float)
+
+define void @atan2_f64(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @atan2_f64
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2vv_atan2(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+;
+; SLEEF-SVE-LABEL: define void @atan2_f64
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_atan2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @atan2_f64
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vatan2q_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+;
+; ARMPL-SVE-LABEL: define void @atan2_f64
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svatan2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds double, ptr %in.ptr, i64 %iv
+  %in = load double, ptr %in.gep, align 8
+  %call = tail call double @llvm.atan2.f64(double %in, double %in)
+  %out.gep = getelementptr inbounds double, ptr %out.ptr, i64 %iv
+  store double %call, ptr %out.gep, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @atan2_f32(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @atan2_f32
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4vv_atan2f(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+;
+; SLEEF-SVE-LABEL: define void @atan2_f32
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_atan2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @atan2_f32
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vatan2q_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+;
+; ARMPL-SVE-LABEL: define void @atan2_f32
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svatan2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds float, ptr %in.ptr, i64 %iv
+  %in = load float, ptr %in.gep, align 8
+  %call = tail call float @llvm.atan2.f32(float %in, float %in)
+  %out.gep = getelementptr inbounds float, ptr %out.ptr, i64 %iv
+  store float %call, ptr %out.gep, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
 declare double @llvm.ceil.f64(double)
 declare float @llvm.ceil.f32(float)
 
-- 
GitLab


From c4248fa3edd3e2e18219b4fb46561ca1bfd65728 Mon Sep 17 00:00:00 2001
From: Freddy Ye <freddy.ye@intel.com>
Date: Fri, 25 Oct 2024 09:00:19 +0800
Subject: [PATCH 107/693] [X86] Support MOVRS and AVX10.2 instructions.
 (#113274)

Ref.: https://cdrdv2.intel.com/v1/dl/getContent/671368
---
 clang/docs/ReleaseNotes.rst                   |   3 +
 clang/include/clang/Basic/BuiltinsX86_64.def  |  14 +
 clang/include/clang/Driver/Options.td         |   2 +
 clang/lib/Basic/Targets/X86.cpp               |   6 +
 clang/lib/Basic/Targets/X86.h                 |   1 +
 clang/lib/Headers/CMakeLists.txt              |   2 +
 clang/lib/Headers/immintrin.h                 |  10 +
 clang/lib/Headers/movrs_avx10_2_512intrin.h   |  98 ++++++
 clang/lib/Headers/movrs_avx10_2intrin.h       | 174 +++++++++
 .../X86/movrs-avx10.2-512-builtins-error-32.c |  50 +++
 .../CodeGen/X86/movrs-avx10.2-512-builtins.c  |  87 +++++
 .../X86/movrs-avx10.2-builtins-error-32.c     |  98 ++++++
 .../test/CodeGen/X86/movrs-avx10.2-builtins.c | 171 +++++++++
 clang/test/CodeGen/target-builtin-noerror.c   |   1 +
 clang/test/Driver/x86-target-features.c       |   5 +
 clang/test/Preprocessor/x86_target_features.c |   6 +
 compiler-rt/lib/builtins/cpu_model/x86.c      |   3 +
 llvm/docs/ReleaseNotes.md                     |   2 +
 llvm/include/llvm/IR/IntrinsicsX86.td         |  39 +++
 .../llvm/TargetParser/X86TargetParser.def     |   2 +
 llvm/lib/Target/X86/X86.td                    |   2 +
 llvm/lib/Target/X86/X86InstrAVX10.td          |  28 ++
 llvm/lib/Target/X86/X86InstrPredicates.td     |   1 +
 llvm/lib/TargetParser/Host.cpp                |   1 +
 llvm/lib/TargetParser/X86TargetParser.cpp     |   2 +
 .../X86/movrs-avx10.2-512-intrinsics.ll       | 163 +++++++++
 .../CodeGen/X86/movrs-avx10.2-intrinsics.ll   | 329 ++++++++++++++++++
 .../MC/Disassembler/X86/movrs-avx10-64.txt    |  98 ++++++
 llvm/test/MC/X86/movrs-avx10-att-64.s         |  98 ++++++
 llvm/test/MC/X86/movrs-avx10-intel-64.s       |  97 ++++++
 30 files changed, 1593 insertions(+)
 create mode 100644 clang/lib/Headers/movrs_avx10_2_512intrin.h
 create mode 100644 clang/lib/Headers/movrs_avx10_2intrin.h
 create mode 100644 clang/test/CodeGen/X86/movrs-avx10.2-512-builtins-error-32.c
 create mode 100644 clang/test/CodeGen/X86/movrs-avx10.2-512-builtins.c
 create mode 100644 clang/test/CodeGen/X86/movrs-avx10.2-builtins-error-32.c
 create mode 100644 clang/test/CodeGen/X86/movrs-avx10.2-builtins.c
 create mode 100644 llvm/test/CodeGen/X86/movrs-avx10.2-512-intrinsics.ll
 create mode 100644 llvm/test/CodeGen/X86/movrs-avx10.2-intrinsics.ll
 create mode 100644 llvm/test/MC/Disassembler/X86/movrs-avx10-64.txt
 create mode 100644 llvm/test/MC/X86/movrs-avx10-att-64.s
 create mode 100644 llvm/test/MC/X86/movrs-avx10-intel-64.s

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 46021c9c17fe..d7b47cd2e2ee 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -624,6 +624,9 @@ X86 Support
 
 - All intrinsics in tbmintrin.h can now be used in constant expressions.
 
+- Supported intrinsics for ``MOVRS AND AVX10.2``.
+  * Supported intrinsics of ``_mm(256|512)_(mask(z))_loadrs_epi(8|16|32|64)``.
+
 Arm and AArch64 Support
 ^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/include/clang/Basic/BuiltinsX86_64.def b/clang/include/clang/Basic/BuiltinsX86_64.def
index 2c591edb2835..e1e613560167 100644
--- a/clang/include/clang/Basic/BuiltinsX86_64.def
+++ b/clang/include/clang/Basic/BuiltinsX86_64.def
@@ -161,6 +161,20 @@ TARGET_BUILTIN(__builtin_ia32_aand64, "vv*SOi", "n", "raoint")
 TARGET_BUILTIN(__builtin_ia32_aor64, "vv*SOi", "n", "raoint")
 TARGET_BUILTIN(__builtin_ia32_axor64, "vv*SOi", "n", "raoint")
 
+// MOVRS and AVX10.2
+TARGET_BUILTIN(__builtin_ia32_vmovrsb128, "V16cV16cC*", "nV:128:", "movrs,avx10.2-256")
+TARGET_BUILTIN(__builtin_ia32_vmovrsb256, "V32cV32cC*", "nV:256:", "movrs,avx10.2-256")
+TARGET_BUILTIN(__builtin_ia32_vmovrsb512, "V64cV64cC*", "nV:512:", "movrs,avx10.2-512")
+TARGET_BUILTIN(__builtin_ia32_vmovrsd128, "V4iV4iC*", "nV:128:", "movrs,avx10.2-256")
+TARGET_BUILTIN(__builtin_ia32_vmovrsd256, "V8iV8iC*", "nV:256:", "movrs,avx10.2-256")
+TARGET_BUILTIN(__builtin_ia32_vmovrsd512, "V16iV16iC*", "nV:512:", "movrs,avx10.2-512")
+TARGET_BUILTIN(__builtin_ia32_vmovrsq128, "V2OiV2OiC*", "nV:128:", "movrs,avx10.2-256")
+TARGET_BUILTIN(__builtin_ia32_vmovrsq256, "V4OiV4OiC*", "nV:256:", "movrs,avx10.2-256")
+TARGET_BUILTIN(__builtin_ia32_vmovrsq512, "V8OiV8OiC*", "nV:512:", "movrs,avx10.2-512")
+TARGET_BUILTIN(__builtin_ia32_vmovrsw128, "V8sV8sC*", "nV:128:", "movrs,avx10.2-256")
+TARGET_BUILTIN(__builtin_ia32_vmovrsw256, "V16sV16sC*", "nV:256:", "movrs,avx10.2-256")
+TARGET_BUILTIN(__builtin_ia32_vmovrsw512, "V32sV32sC*", "nV:512:", "movrs,avx10.2-512")
+
 #undef BUILTIN
 #undef TARGET_BUILTIN
 #undef TARGET_HEADER_BUILTIN
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 2ddb2f531214..62603969eaf8 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -6424,6 +6424,8 @@ def mmovdiri : Flag<["-"], "mmovdiri">, Group<m_x86_Features_Group>;
 def mno_movdiri : Flag<["-"], "mno-movdiri">, Group<m_x86_Features_Group>;
 def mmovdir64b : Flag<["-"], "mmovdir64b">, Group<m_x86_Features_Group>;
 def mno_movdir64b : Flag<["-"], "mno-movdir64b">, Group<m_x86_Features_Group>;
+def mmovrs : Flag<["-"], "mmovrs">, Group<m_x86_Features_Group>;
+def mno_movrs : Flag<["-"], "mno-movrs">, Group<m_x86_Features_Group>;
 def mmwaitx : Flag<["-"], "mmwaitx">, Group<m_x86_Features_Group>;
 def mno_mwaitx : Flag<["-"], "mno-mwaitx">, Group<m_x86_Features_Group>;
 def mpku : Flag<["-"], "mpku">, Group<m_x86_Features_Group>;
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 5448bd841959..d067ec218b52 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -348,6 +348,8 @@ bool X86TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasSM4 = true;
     } else if (Feature == "+movbe") {
       HasMOVBE = true;
+    } else if (Feature == "+movrs") {
+      HasMOVRS = true;
     } else if (Feature == "+sgx") {
       HasSGX = true;
     } else if (Feature == "+cx8") {
@@ -915,6 +917,8 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__MOVDIRI__");
   if (HasMOVDIR64B)
     Builder.defineMacro("__MOVDIR64B__");
+  if (HasMOVRS)
+    Builder.defineMacro("__MOVRS__");
   if (HasPCONFIG)
     Builder.defineMacro("__PCONFIG__");
   if (HasPTWRITE)
@@ -1116,6 +1120,7 @@ bool X86TargetInfo::isValidFeatureName(StringRef Name) const {
       .Case("lzcnt", true)
       .Case("mmx", true)
       .Case("movbe", true)
+      .Case("movrs", true)
       .Case("movdiri", true)
       .Case("movdir64b", true)
       .Case("mwaitx", true)
@@ -1233,6 +1238,7 @@ bool X86TargetInfo::hasFeature(StringRef Feature) const {
       .Case("lzcnt", HasLZCNT)
       .Case("mmx", HasMMX)
       .Case("movbe", HasMOVBE)
+      .Case("movrs", HasMOVRS)
       .Case("movdiri", HasMOVDIRI)
       .Case("movdir64b", HasMOVDIR64B)
       .Case("mwaitx", HasMWAITX)
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index 2affb659a160..e8aad3ec5a74 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -130,6 +130,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
   bool HasCLFLUSHOPT = false;
   bool HasCLWB = false;
   bool HasMOVBE = false;
+  bool HasMOVRS = false;
   bool HasPREFETCHI = false;
   bool HasRDPID = false;
   bool HasRDPRU = false;
diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index ff392e7122a4..e97953d87a2f 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -221,6 +221,8 @@ set(x86_files
   mm3dnow.h
   mmintrin.h
   movdirintrin.h
+  movrs_avx10_2_512intrin.h
+  movrs_avx10_2intrin.h
   mwaitxintrin.h
   nmmintrin.h
   pconfigintrin.h
diff --git a/clang/lib/Headers/immintrin.h b/clang/lib/Headers/immintrin.h
index 3fbabffa98df..5f296d0a3324 100644
--- a/clang/lib/Headers/immintrin.h
+++ b/clang/lib/Headers/immintrin.h
@@ -605,6 +605,16 @@ _storebe_i64(void * __P, long long __D) {
 #include <movdirintrin.h>
 #endif
 
+#if !defined(__SCE__) || __has_feature(modules) ||                             \
+    (defined(__AVX10_2__) && defined(__MOVRS__))
+#include <movrs_avx10_2intrin.h>
+#endif
+
+#if !defined(__SCE__) || __has_feature(modules) ||                             \
+    (defined(__AVX10_2_512__) && defined(__MOVRS__))
+#include <movrs_avx10_2_512intrin.h>
+#endif
+
 #if !defined(__SCE__) || __has_feature(modules) || defined(__PCONFIG__)
 #include <pconfigintrin.h>
 #endif
diff --git a/clang/lib/Headers/movrs_avx10_2_512intrin.h b/clang/lib/Headers/movrs_avx10_2_512intrin.h
new file mode 100644
index 000000000000..5cd907a59734
--- /dev/null
+++ b/clang/lib/Headers/movrs_avx10_2_512intrin.h
@@ -0,0 +1,98 @@
+/*===----- movrs_avx10_2_512intrin.h - AVX10.2-512-MOVRS intrinsics --------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+#ifndef __IMMINTRIN_H
+#error                                                                         \
+    "Never use <movrs_avx10_2_512intrin.h> directly; include <immintrin.h> instead."
+#endif
+
+#ifndef __MOVRS_AVX10_2_512INTRIN_H
+#define __MOVRS_AVX10_2_512INTRIN_H
+#ifdef __x86_64__
+
+/* Define the default attributes for the functions in this file. */
+#define __DEFAULT_FN_ATTRS512                                                  \
+  __attribute__((__always_inline__, __nodebug__,                               \
+                 __target__("movrs, avx10.2-512"), __min_vector_width__(512)))
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS512
+_mm512_loadrs_epi8(void const *__A) {
+  return (__m512i)__builtin_ia32_vmovrsb512((const __v64qi *)(__A));
+}
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS512
+_mm512_mask_loadrs_epi8(__m512i __W, __mmask64 __U, void const *__A) {
+  return (__m512i)__builtin_ia32_selectb_512(
+      (__mmask64)__U, (__v64qi)_mm512_loadrs_epi8(__A), (__v64qi)__W);
+}
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS512
+_mm512_maskz_loadrs_epi8(__mmask64 __U, void const *__A) {
+  return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
+                                             (__v64qi)_mm512_loadrs_epi8(__A),
+                                             (__v64qi)_mm512_setzero_si512());
+}
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS512
+_mm512_loadrs_epi32(void const *__A) {
+  return (__m512i)__builtin_ia32_vmovrsd512((const __v16si *)(__A));
+}
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS512
+_mm512_mask_loadrs_epi32(__m512i __W, __mmask16 __U, void const *__A) {
+  return (__m512i)__builtin_ia32_selectd_512(
+      (__mmask16)__U, (__v16si)_mm512_loadrs_epi32(__A), (__v16si)__W);
+}
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS512
+_mm512_maskz_loadrs_epi32(__mmask16 __U, void const *__A) {
+  return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
+                                             (__v16si)_mm512_loadrs_epi32(__A),
+                                             (__v16si)_mm512_setzero_si512());
+}
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS512
+_mm512_loadrs_epi64(void const *__A) {
+  return (__m512i)__builtin_ia32_vmovrsq512((const __v8di *)(__A));
+}
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS512
+_mm512_mask_loadrs_epi64(__m512i __W, __mmask8 __U, void const *__A) {
+  return (__m512i)__builtin_ia32_selectq_512(
+      (__mmask8)__U, (__v8di)_mm512_loadrs_epi64(__A), (__v8di)__W);
+}
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS512
+_mm512_maskz_loadrs_epi64(__mmask8 __U, void const *__A) {
+  return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
+                                             (__v8di)_mm512_loadrs_epi64(__A),
+                                             (__v8di)_mm512_setzero_si512());
+}
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS512
+_mm512_loadrs_epi16(void const *__A) {
+  return (__m512i)__builtin_ia32_vmovrsw512((const __v32hi *)(__A));
+}
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS512
+_mm512_mask_loadrs_epi16(__m512i __W, __mmask32 __U, void const *__A) {
+  return (__m512i)__builtin_ia32_selectw_512(
+      (__mmask32)__U, (__v32hi)_mm512_loadrs_epi16(__A), (__v32hi)__W);
+}
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS512
+_mm512_maskz_loadrs_epi16(__mmask32 __U, void const *__A) {
+  return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
+                                             (__v32hi)_mm512_loadrs_epi16(__A),
+                                             (__v32hi)_mm512_setzero_si512());
+}
+
+#undef __DEFAULT_FN_ATTRS512
+
+#endif /* __x86_64__ */
+#endif /* __MOVRS_AVX10_2_512INTRIN_H */
diff --git a/clang/lib/Headers/movrs_avx10_2intrin.h b/clang/lib/Headers/movrs_avx10_2intrin.h
new file mode 100644
index 000000000000..27b625b6b431
--- /dev/null
+++ b/clang/lib/Headers/movrs_avx10_2intrin.h
@@ -0,0 +1,174 @@
+/*===--------- movrs_avx10_2intrin.h - AVX10.2-MOVRS intrinsics ------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+#ifndef __IMMINTRIN_H
+#error                                                                         \
+    "Never use <movrs_avx10_2intrin.h> directly; include <immintrin.h> instead."
+#endif
+
+#ifndef __MOVRS_AVX10_2INTRIN_H
+#define __MOVRS_AVX10_2INTRIN_H
+#ifdef __x86_64__
+
+/* Define the default attributes for the functions in this file. */
+#define __DEFAULT_FN_ATTRS128                                                  \
+  __attribute__((__always_inline__, __nodebug__,                               \
+                 __target__("movrs,avx10.2-256"), __min_vector_width__(128)))
+#define __DEFAULT_FN_ATTRS256                                                  \
+  __attribute__((__always_inline__, __nodebug__,                               \
+                 __target__("movrs,avx10.2-256"), __min_vector_width__(256)))
+
+static __inline__ __m128i __DEFAULT_FN_ATTRS128
+_mm_loadrs_epi8(void const *__A) {
+  return (__m128i)__builtin_ia32_vmovrsb128((const __v16qi *)(__A));
+}
+
+static __inline__ __m128i __DEFAULT_FN_ATTRS128
+_mm_mask_loadrs_epi8(__m128i __W, __mmask16 __U, void const *__A) {
+  return (__m128i)__builtin_ia32_selectb_128(
+      (__mmask16)__U, (__v16qi)_mm_loadrs_epi8(__A), (__v16qi)__W);
+}
+
+static __inline__ __m128i __DEFAULT_FN_ATTRS128
+_mm_maskz_loadrs_epi8(__mmask16 __U, void const *__A) {
+  return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
+                                             (__v16qi)_mm_loadrs_epi8(__A),
+                                             (__v16qi)_mm_setzero_si128());
+}
+
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_loadrs_epi8(void const *__A) {
+  return (__m256i)__builtin_ia32_vmovrsb256((const __v32qi *)(__A));
+}
+
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_mask_loadrs_epi8(__m256i __W, __mmask32 __U, void const *__A) {
+  return (__m256i)__builtin_ia32_selectb_256(
+      (__mmask32)__U, (__v32qi)_mm256_loadrs_epi8(__A), (__v32qi)__W);
+}
+
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_maskz_loadrs_epi8(__mmask32 __U, void const *__A) {
+  return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
+                                             (__v32qi)_mm256_loadrs_epi8(__A),
+                                             (__v32qi)_mm256_setzero_si256());
+}
+
+static __inline__ __m128i __DEFAULT_FN_ATTRS128
+_mm_loadrs_epi32(void const *__A) {
+  return (__m128i)__builtin_ia32_vmovrsd128((const __v4si *)(__A));
+}
+
+static __inline__ __m128i __DEFAULT_FN_ATTRS128
+_mm_mask_loadrs_epi32(__m128i __W, __mmask8 __U, void const *__A) {
+  return (__m128i)__builtin_ia32_selectd_128(
+      (__mmask8)__U, (__v4si)_mm_loadrs_epi32(__A), (__v4si)__W);
+}
+
+static __inline__ __m128i __DEFAULT_FN_ATTRS128
+_mm_maskz_loadrs_epi32(__mmask8 __U, void const *__A) {
+  return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
+                                             (__v4si)_mm_loadrs_epi32(__A),
+                                             (__v4si)_mm_setzero_si128());
+}
+
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_loadrs_epi32(void const *__A) {
+  return (__m256i)__builtin_ia32_vmovrsd256((const __v8si *)(__A));
+}
+
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_mask_loadrs_epi32(__m256i __W, __mmask8 __U, void const *__A) {
+  return (__m256i)__builtin_ia32_selectd_256(
+      (__mmask8)__U, (__v8si)_mm256_loadrs_epi32(__A), (__v8si)__W);
+}
+
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_maskz_loadrs_epi32(__mmask8 __U, void const *__A) {
+  return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
+                                             (__v8si)_mm256_loadrs_epi32(__A),
+                                             (__v8si)_mm256_setzero_si256());
+}
+
+static __inline__ __m128i __DEFAULT_FN_ATTRS128
+_mm_loadrs_epi64(void const *__A) {
+  return (__m128i)__builtin_ia32_vmovrsq128((const __v2di *)(__A));
+}
+
+static __inline__ __m128i __DEFAULT_FN_ATTRS128
+_mm_mask_loadrs_epi64(__m128i __W, __mmask8 __U, void const *__A) {
+  return (__m128i)__builtin_ia32_selectq_128(
+      (__mmask8)__U, (__v2di)_mm_loadrs_epi64(__A), (__v2di)__W);
+}
+
+static __inline__ __m128i __DEFAULT_FN_ATTRS128
+_mm_maskz_loadrs_epi64(__mmask8 __U, void const *__A) {
+  return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
+                                             (__v2di)_mm_loadrs_epi64(__A),
+                                             (__v2di)_mm_setzero_si128());
+}
+
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_loadrs_epi64(void const *__A) {
+  return (__m256i)__builtin_ia32_vmovrsq256((const __v4di *)(__A));
+}
+
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_mask_loadrs_epi64(__m256i __W, __mmask8 __U, void const *__A) {
+  return (__m256i)__builtin_ia32_selectq_256(
+      (__mmask8)__U, (__v4di)_mm256_loadrs_epi64(__A), (__v4di)__W);
+}
+
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_maskz_loadrs_epi64(__mmask8 __U, void const *__A) {
+  return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
+                                             (__v4di)_mm256_loadrs_epi64(__A),
+                                             (__v4di)_mm256_setzero_si256());
+}
+
+static __inline__ __m128i __DEFAULT_FN_ATTRS128
+_mm_loadrs_epi16(void const *__A) {
+  return (__m128i)__builtin_ia32_vmovrsw128((const __v8hi *)(__A));
+}
+
+static __inline__ __m128i __DEFAULT_FN_ATTRS128
+_mm_mask_loadrs_epi16(__m128i __W, __mmask8 __U, void const *__A) {
+  return (__m128i)__builtin_ia32_selectw_128(
+      (__mmask8)__U, (__v8hi)_mm_loadrs_epi16(__A), (__v8hi)__W);
+}
+
+static __inline__ __m128i __DEFAULT_FN_ATTRS128
+_mm_maskz_loadrs_epi16(__mmask8 __U, void const *__A) {
+  return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
+                                             (__v8hi)_mm_loadrs_epi16(__A),
+                                             (__v8hi)_mm_setzero_si128());
+}
+
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_loadrs_epi16(void const *__A) {
+  return (__m256i)__builtin_ia32_vmovrsw256((const __v16hi *)(__A));
+}
+
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_mask_loadrs_epi16(__m256i __W, __mmask16 __U, void const *__A) {
+  return (__m256i)__builtin_ia32_selectw_256(
+      (__mmask16)__U, (__v16hi)_mm256_loadrs_epi16(__A), (__v16hi)__W);
+}
+
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_maskz_loadrs_epi16(__mmask16 __U, void const *__A) {
+  return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
+                                             (__v16hi)_mm256_loadrs_epi16(__A),
+                                             (__v16hi)_mm256_setzero_si256());
+}
+
+#undef __DEFAULT_FN_ATTRS128
+#undef __DEFAULT_FN_ATTRS256
+
+#endif /* __x86_64__ */
+#endif /* __MOVRS_AVX10_2INTRIN_H */
diff --git a/clang/test/CodeGen/X86/movrs-avx10.2-512-builtins-error-32.c b/clang/test/CodeGen/X86/movrs-avx10.2-512-builtins-error-32.c
new file mode 100644
index 000000000000..944033724a6a
--- /dev/null
+++ b/clang/test/CodeGen/X86/movrs-avx10.2-512-builtins-error-32.c
@@ -0,0 +1,50 @@
+// RUN: %clang_cc1 -ffreestanding %s -Wno-implicit-function-declaration -triple=i386-- -target-feature +movrs -target-feature +avx10.2-512 -emit-llvm -verify
+
+#include <immintrin.h>
+__m512i test_mm512_loadrs_epi8(const __m512i * __A) {
+  return _mm512_loadrs_epi8(__A); // expected-error {{returning 'int' from a function with incompatible result type '__m512i' (vector of 8 'long long' values)}}
+}
+
+__m512i test_mm512_mask_loadrs_epi8(__m512i __A, __mmask64 __B, const __m512i * __C) {
+  return _mm512_mask_loadrs_epi8(__A, __B, __C); // expected-error {{returning 'int' from a function with incompatible result type '__m512i' (vector of 8 'long long' values)}}
+}
+
+__m512i test_mm512_maskz_loadrs_epi8(__mmask64 __A, const __m512i * __B) {
+  return _mm512_maskz_loadrs_epi8(__A, __B); // expected-error {{returning 'int' from a function with incompatible result type '__m512i' (vector of 8 'long long' values)}}
+}
+
+__m512i test_mm512_loadrs_epi32(const __m512i * __A) {
+  return _mm512_loadrs_epi32(__A); // expected-error {{returning 'int' from a function with incompatible result type '__m512i' (vector of 8 'long long' values)}}
+}
+
+__m512i test_mm512_mask_loadrs_epi32(__m512i __A, __mmask16 __B, const __m512i * __C) {
+  return _mm512_mask_loadrs_epi32(__A, __B, __C); // expected-error {{returning 'int' from a function with incompatible result type '__m512i' (vector of 8 'long long' values)}}
+}
+
+__m512i test_mm512_maskz_loadrs_epi32(__mmask16 __A, const __m512i * __B) {
+  return _mm512_maskz_loadrs_epi32(__A, __B); // expected-error {{returning 'int' from a function with incompatible result type '__m512i' (vector of 8 'long long' values)}}
+}
+
+__m512i test_mm512_loadrs_epi64(const __m512i * __A) {
+  return _mm512_loadrs_epi64(__A); // expected-error {{returning 'int' from a function with incompatible result type '__m512i' (vector of 8 'long long' values)}}
+}
+
+__m512i test_mm512_mask_loadrs_epi64(__m512i __A, __mmask8 __B, const __m512i * __C) {
+  return _mm512_mask_loadrs_epi64(__A, __B, __C); // expected-error {{returning 'int' from a function with incompatible result type '__m512i' (vector of 8 'long long' values)}}
+}
+
+__m512i test_mm512_maskz_loadrs_epi64(__mmask8 __A, const __m512i * __B) {
+  return _mm512_maskz_loadrs_epi64(__A, __B); // expected-error {{returning 'int' from a function with incompatible result type '__m512i' (vector of 8 'long long' values)}}
+}
+
+__m512i test_mm512_loadrs_epi16(const __m512i * __A) {
+  return _mm512_loadrs_epi16(__A); // expected-error {{returning 'int' from a function with incompatible result type '__m512i' (vector of 8 'long long' values)}}
+}
+
+__m512i test_mm512_mask_loadrs_epi16(__m512i __A, __mmask32 __B, const __m512i * __C) {
+  return _mm512_mask_loadrs_epi16(__A, __B, __C); // expected-error {{returning 'int' from a function with incompatible result type '__m512i' (vector of 8 'long long' values)}}
+}
+
+__m512i test_mm512_maskz_loadrs_epi16(__mmask32 __A, const __m512i * __B) {
+  return _mm512_maskz_loadrs_epi16(__A, __B); // expected-error {{returning 'int' from a function with incompatible result type '__m512i' (vector of 8 'long long' values)}}
+}
diff --git a/clang/test/CodeGen/X86/movrs-avx10.2-512-builtins.c b/clang/test/CodeGen/X86/movrs-avx10.2-512-builtins.c
new file mode 100644
index 000000000000..997d6dbc53a8
--- /dev/null
+++ b/clang/test/CodeGen/X86/movrs-avx10.2-512-builtins.c
@@ -0,0 +1,87 @@
+// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-- -target-feature +movrs -target-feature +avx10.2-512 -emit-llvm -o - -Wall -Werror | FileCheck %s
+
+#include <immintrin.h>
+
+__m512i test_mm512_loadrs_epi8(const __m512i * __A) {
+  // CHECK-LABEL: @test_mm512_loadrs_epi8(
+  // CHECK: call <64 x i8> @llvm.x86.avx10.vmovrsb512(
+  return _mm512_loadrs_epi8(__A);
+}
+
+__m512i test_mm512_mask_loadrs_epi8(__m512i __A, __mmask64 __B, const __m512i * __C) {
+  // CHECK-LABEL: @test_mm512_mask_loadrs_epi8(
+  // CHECK: call <64 x i8> @llvm.x86.avx10.vmovrsb512(
+  // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
+  return _mm512_mask_loadrs_epi8(__A, __B, __C);
+}
+
+__m512i test_mm512_maskz_loadrs_epi8(__mmask64 __A, const __m512i * __B) {
+  // CHECK-LABEL: @test_mm512_maskz_loadrs_epi8(
+  // CHECK: call <64 x i8> @llvm.x86.avx10.vmovrsb512(
+  // CHECK: store <8 x i64> zeroinitializer
+  // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
+  return _mm512_maskz_loadrs_epi8(__A, __B);
+}
+
+__m512i test_mm512_loadrs_epi32(const __m512i * __A) {
+  // CHECK-LABEL: @test_mm512_loadrs_epi32(
+  // CHECK: call <16 x i32> @llvm.x86.avx10.vmovrsd512(
+  return _mm512_loadrs_epi32(__A);
+}
+
+__m512i test_mm512_mask_loadrs_epi32(__m512i __A, __mmask16 __B, const __m512i * __C) {
+  // CHECK-LABEL: @test_mm512_mask_loadrs_epi32(
+  // CHECK: call <16 x i32> @llvm.x86.avx10.vmovrsd512(
+  // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
+  return _mm512_mask_loadrs_epi32(__A, __B, __C);
+}
+
+__m512i test_mm512_maskz_loadrs_epi32(__mmask16 __A, const __m512i * __B) {
+  // CHECK-LABEL: @test_mm512_maskz_loadrs_epi32(
+  // CHECK: call <16 x i32> @llvm.x86.avx10.vmovrsd512(
+  // CHECK: store <8 x i64> zeroinitializer
+  // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
+  return _mm512_maskz_loadrs_epi32(__A, __B);
+}
+
+__m512i test_mm512_loadrs_epi64(const __m512i * __A) {
+  // CHECK-LABEL: @test_mm512_loadrs_epi64(
+  // CHECK: call <8 x i64> @llvm.x86.avx10.vmovrsq512(
+  return _mm512_loadrs_epi64(__A);
+}
+
+__m512i test_mm512_mask_loadrs_epi64(__m512i __A, __mmask8 __B, const __m512i * __C) {
+  // CHECK-LABEL: @test_mm512_mask_loadrs_epi64(
+  // CHECK: call <8 x i64> @llvm.x86.avx10.vmovrsq512(
+  // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
+  return _mm512_mask_loadrs_epi64(__A, __B, __C);
+}
+
+__m512i test_mm512_maskz_loadrs_epi64(__mmask8 __A, const __m512i * __B) {
+  // CHECK-LABEL: @test_mm512_maskz_loadrs_epi64(
+  // CHECK: call <8 x i64> @llvm.x86.avx10.vmovrsq512(
+  // CHECK: store <8 x i64> zeroinitializer
+  // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
+  return _mm512_maskz_loadrs_epi64(__A, __B);
+}
+
+__m512i test_mm512_loadrs_epi16(const __m512i * __A) {
+  // CHECK-LABEL: @test_mm512_loadrs_epi16(
+  // CHECK: call <32 x i16> @llvm.x86.avx10.vmovrsw512(
+  return _mm512_loadrs_epi16(__A);
+}
+
+__m512i test_mm512_mask_loadrs_epi16(__m512i __A, __mmask32 __B, const __m512i * __C) {
+  // CHECK-LABEL: @test_mm512_mask_loadrs_epi16(
+  // CHECK: call <32 x i16> @llvm.x86.avx10.vmovrsw512(
+  // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
+  return _mm512_mask_loadrs_epi16(__A, __B, __C);
+}
+
+__m512i test_mm512_maskz_loadrs_epi16(__mmask32 __A, const __m512i * __B) {
+  // CHECK-LABEL: @test_mm512_maskz_loadrs_epi16(
+  // CHECK: call <32 x i16> @llvm.x86.avx10.vmovrsw512(
+  // CHECK: store <8 x i64> zeroinitializer
+  // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
+  return _mm512_maskz_loadrs_epi16(__A, __B);
+}
diff --git a/clang/test/CodeGen/X86/movrs-avx10.2-builtins-error-32.c b/clang/test/CodeGen/X86/movrs-avx10.2-builtins-error-32.c
new file mode 100644
index 000000000000..68608b0cbff0
--- /dev/null
+++ b/clang/test/CodeGen/X86/movrs-avx10.2-builtins-error-32.c
@@ -0,0 +1,98 @@
+// RUN: %clang_cc1 -ffreestanding %s -Wno-implicit-function-declaration -triple=i386-unknown-unknown -target-feature +movrs -target-feature +avx10.2-256 -emit-llvm -verify
+
+#include <immintrin.h>
+__m128i test_mm_loadrs_epi8(const __m128i * __A) {
+  return _mm_loadrs_epi8(__A); // expected-error {{returning 'int' from a function with incompatible result type '__m128i' (vector of 2 'long long' values)}}
+}
+
+__m128i test_mm_mask_loadrs_epi8(__m128i __A, __mmask16 __B, const __m128i * __C) {
+  return _mm_mask_loadrs_epi8(__A, __B, __C); // expected-error {{returning 'int' from a function with incompatible result type '__m128i' (vector of 2 'long long' values)}}
+}
+
+__m128i test_mm_maskz_loadrs_epi8(__mmask16 __A, const __m128i * __B) {
+  return _mm_maskz_loadrs_epi8(__A, __B); // expected-error {{returning 'int' from a function with incompatible result type '__m128i' (vector of 2 'long long' values)}}
+}
+
+__m256i test_mm256_loadrs_epi8(const __m256i * __A) {
+  return _mm256_loadrs_epi8(__A); // expected-error {{returning 'int' from a function with incompatible result type '__m256i' (vector of 4 'long long' values)}}
+}
+
+__m256i test_mm256_mask_loadrs_epi8(__m256i __A, __mmask32 __B, const __m256i * __C) {
+  return _mm256_mask_loadrs_epi8(__A, __B, __C); // expected-error {{returning 'int' from a function with incompatible result type '__m256i' (vector of 4 'long long' values)}}
+}
+
+__m256i test_mm256_maskz_loadrs_epi8(__mmask32 __A, const __m256i * __B) {
+  return _mm256_maskz_loadrs_epi8(__A, __B); // expected-error {{returning 'int' from a function with incompatible result type '__m256i' (vector of 4 'long long' values)}}
+}
+
+__m128i test_mm_loadrs_epi32(const __m128i * __A) {
+  return _mm_loadrs_epi32(__A); // expected-error {{returning 'int' from a function with incompatible result type '__m128i' (vector of 2 'long long' values)}}
+}
+
+__m128i test_mm_mask_loadrs_epi32(__m128i __A, __mmask8 __B, const __m128i * __C) {
+  return _mm_mask_loadrs_epi32(__A, __B, __C); // expected-error {{returning 'int' from a function with incompatible result type '__m128i' (vector of 2 'long long' values)}}
+}
+
+__m128i test_mm_maskz_loadrs_epi32(__mmask8 __A, const __m128i * __B) {
+  return _mm_maskz_loadrs_epi32(__A, __B); // expected-error {{returning 'int' from a function with incompatible result type '__m128i' (vector of 2 'long long' values)}}
+}
+
+__m256i test_mm256_loadrs_epi32(const __m256i * __A) {
+  return _mm256_loadrs_epi32(__A); // expected-error {{returning 'int' from a function with incompatible result type '__m256i' (vector of 4 'long long' values)}}
+}
+
+__m256i test_mm256_mask_loadrs_epi32(__m256i __A, __mmask8 __B, const __m256i * __C) {
+  return _mm256_mask_loadrs_epi32(__A, __B, __C); // expected-error {{returning 'int' from a function with incompatible result type '__m256i' (vector of 4 'long long' values)}}
+}
+
+__m256i test_mm256_maskz_loadrs_epi32(__mmask8 __A, const __m256i * __B) {
+  return _mm256_maskz_loadrs_epi32(__A, __B); // expected-error {{returning 'int' from a function with incompatible result type '__m256i' (vector of 4 'long long' values)}}
+}
+
+__m128i test_mm_loadrs_epi64(const __m128i * __A) {
+  return _mm_loadrs_epi64(__A); // expected-error {{returning 'int' from a function with incompatible result type '__m128i' (vector of 2 'long long' values)}}
+}
+
+__m128i test_mm_mask_loadrs_epi64(__m128i __A, __mmask8 __B, const __m128i * __C) {
+  return _mm_mask_loadrs_epi64(__A, __B, __C); // expected-error {{returning 'int' from a function with incompatible result type '__m128i' (vector of 2 'long long' values)}}
+}
+
+__m128i test_mm_maskz_loadrs_epi64(__mmask8 __A, const __m128i * __B) {
+  return _mm_maskz_loadrs_epi64(__A, __B); // expected-error {{returning 'int' from a function with incompatible result type '__m128i' (vector of 2 'long long' values)}}
+}
+
+__m256i test_mm256_loadrs_epi64(const __m256i * __A) {
+  return _mm256_loadrs_epi64(__A); // expected-error {{returning 'int' from a function with incompatible result type '__m256i' (vector of 4 'long long' values)}}
+}
+
+__m256i test_mm256_mask_loadrs_epi64(__m256i __A, __mmask8 __B, const __m256i * __C) {
+  return _mm256_mask_loadrs_epi64(__A, __B, __C); // expected-error {{returning 'int' from a function with incompatible result type '__m256i' (vector of 4 'long long' values)}}
+}
+
+__m256i test_mm256_maskz_loadrs_epi64(__mmask8 __A, const __m256i * __B) {
+  return _mm256_maskz_loadrs_epi64(__A, __B); // expected-error {{returning 'int' from a function with incompatible result type '__m256i' (vector of 4 'long long' values)}}
+}
+
+__m128i test_mm_loadrs_epi16(const __m128i * __A) {
+  return _mm_loadrs_epi16(__A); // expected-error {{returning 'int' from a function with incompatible result type '__m128i' (vector of 2 'long long' values)}}
+}
+
+__m128i test_mm_mask_loadrs_epi16(__m128i __A, __mmask8 __B, const __m128i * __C) {
+  return _mm_mask_loadrs_epi16(__A, __B, __C); // expected-error {{returning 'int' from a function with incompatible result type '__m128i' (vector of 2 'long long' values)}}
+}
+
+__m128i test_mm_maskz_loadrs_epi16(__mmask8 __A, const __m128i * __B) {
+  return _mm_maskz_loadrs_epi16(__A, __B); // expected-error {{returning 'int' from a function with incompatible result type '__m128i' (vector of 2 'long long' values)}}
+}
+
+__m256i test_mm256_loadrs_epi16(const __m256i * __A) {
+  return _mm256_loadrs_epi16(__A); // expected-error {{returning 'int' from a function with incompatible result type '__m256i' (vector of 4 'long long' values)}}
+}
+
+__m256i test_mm256_mask_loadrs_epi16(__m256i __A, __mmask16 __B, const __m256i * __C) {
+  return _mm256_mask_loadrs_epi16(__A, __B, __C); // expected-error {{returning 'int' from a function with incompatible result type '__m256i' (vector of 4 'long long' values)}}
+}
+
+__m256i test_mm256_maskz_loadrs_epi16(__mmask16 __A, const __m256i * __B) {
+  return _mm256_maskz_loadrs_epi16(__A, __B); // expected-error {{returning 'int' from a function with incompatible result type '__m256i' (vector of 4 'long long' values)}}
+}
diff --git a/clang/test/CodeGen/X86/movrs-avx10.2-builtins.c b/clang/test/CodeGen/X86/movrs-avx10.2-builtins.c
new file mode 100644
index 000000000000..2011b2a86247
--- /dev/null
+++ b/clang/test/CodeGen/X86/movrs-avx10.2-builtins.c
@@ -0,0 +1,171 @@
+// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-- -target-feature +movrs -target-feature +avx10.2-256 -emit-llvm -o - -Wall -Werror | FileCheck %s
+
+#include <immintrin.h>
+
+__m128i test_mm_loadrs_epi8(const __m128i * __A) {
+  // CHECK-LABEL: @test_mm_loadrs_epi8(
+  // CHECK: call <16 x i8> @llvm.x86.avx10.vmovrsb128(
+  return _mm_loadrs_epi8(__A);
+}
+
+__m128i test_mm_mask_loadrs_epi8(__m128i __A, __mmask16 __B, const __m128i * __C) {
+  // CHECK-LABEL: @test_mm_mask_loadrs_epi8(
+  // CHECK: call <16 x i8> @llvm.x86.avx10.vmovrsb128(
+  // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
+  return _mm_mask_loadrs_epi8(__A, __B, __C);
+}
+
+__m128i test_mm_maskz_loadrs_epi8(__mmask16 __A, const __m128i * __B) {
+  // CHECK-LABEL: @test_mm_maskz_loadrs_epi8(
+  // CHECK: call <16 x i8> @llvm.x86.avx10.vmovrsb128(
+  // CHECK: store <2 x i64> zeroinitializer
+  // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
+  return _mm_maskz_loadrs_epi8(__A, __B);
+}
+
+__m256i test_mm256_loadrs_epi8(const __m256i * __A) {
+  // CHECK-LABEL: @test_mm256_loadrs_epi8(
+  // CHECK: call <32 x i8> @llvm.x86.avx10.vmovrsb256(
+  return _mm256_loadrs_epi8(__A);
+}
+
+__m256i test_mm256_mask_loadrs_epi8(__m256i __A, __mmask32 __B, const __m256i * __C) {
+  // CHECK-LABEL: @test_mm256_mask_loadrs_epi8(
+  // CHECK: call <32 x i8> @llvm.x86.avx10.vmovrsb256(
+  // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
+  return _mm256_mask_loadrs_epi8(__A, __B, __C);
+}
+
+__m256i test_mm256_maskz_loadrs_epi8(__mmask32 __A, const __m256i * __B) {
+  // CHECK-LABEL: @test_mm256_maskz_loadrs_epi8(
+  // CHECK: call <32 x i8> @llvm.x86.avx10.vmovrsb256(
+  // CHECK: store <4 x i64> zeroinitializer
+  // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
+  return _mm256_maskz_loadrs_epi8(__A, __B);
+}
+
+__m128i test_mm_loadrs_epi32(const __m128i * __A) {
+  // CHECK-LABEL: @test_mm_loadrs_epi32(
+  // CHECK: call <4 x i32> @llvm.x86.avx10.vmovrsd128(
+  return _mm_loadrs_epi32(__A);
+}
+
+__m128i test_mm_mask_loadrs_epi32(__m128i __A, __mmask8 __B, const __m128i * __C) {
+  // CHECK-LABEL: @test_mm_mask_loadrs_epi32(
+  // CHECK: call <4 x i32> @llvm.x86.avx10.vmovrsd128(
+  // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
+  return _mm_mask_loadrs_epi32(__A, __B, __C);
+}
+
+__m128i test_mm_maskz_loadrs_epi32(__mmask8 __A, const __m128i * __B) {
+  // CHECK-LABEL: @test_mm_maskz_loadrs_epi32(
+  // CHECK: call <4 x i32> @llvm.x86.avx10.vmovrsd128(
+  // CHECK: store <2 x i64> zeroinitializer
+  // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
+  return _mm_maskz_loadrs_epi32(__A, __B);
+}
+
+__m256i test_mm256_loadrs_epi32(const __m256i * __A) {
+  // CHECK-LABEL: @test_mm256_loadrs_epi32(
+  // CHECK: call <8 x i32> @llvm.x86.avx10.vmovrsd256(
+  return _mm256_loadrs_epi32(__A);
+}
+
+__m256i test_mm256_mask_loadrs_epi32(__m256i __A, __mmask8 __B, const __m256i * __C) {
+  // CHECK-LABEL: @test_mm256_mask_loadrs_epi32(
+  // CHECK: call <8 x i32> @llvm.x86.avx10.vmovrsd256(
+  // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
+  return _mm256_mask_loadrs_epi32(__A, __B, __C);
+}
+
+__m256i test_mm256_maskz_loadrs_epi32(__mmask8 __A, const __m256i * __B) {
+  // CHECK-LABEL: @test_mm256_maskz_loadrs_epi32(
+  // CHECK: call <8 x i32> @llvm.x86.avx10.vmovrsd256(
+  // CHECK: store <4 x i64> zeroinitializer
+  // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
+  return _mm256_maskz_loadrs_epi32(__A, __B);
+}
+
+__m128i test_mm_loadrs_epi64(const __m128i * __A) {
+  // CHECK-LABEL: @test_mm_loadrs_epi64(
+  // CHECK: call <2 x i64> @llvm.x86.avx10.vmovrsq128(
+  return _mm_loadrs_epi64(__A);
+}
+
+__m128i test_mm_mask_loadrs_epi64(__m128i __A, __mmask8 __B, const __m128i * __C) {
+  // CHECK-LABEL: @test_mm_mask_loadrs_epi64(
+  // CHECK: call <2 x i64> @llvm.x86.avx10.vmovrsq128(
+  // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
+  return _mm_mask_loadrs_epi64(__A, __B, __C);
+}
+
+__m128i test_mm_maskz_loadrs_epi64(__mmask8 __A, const __m128i * __B) {
+  // CHECK-LABEL: @test_mm_maskz_loadrs_epi64(
+  // CHECK: call <2 x i64> @llvm.x86.avx10.vmovrsq128(
+  // CHECK: store <2 x i64> zeroinitializer
+  // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
+  return _mm_maskz_loadrs_epi64(__A, __B);
+}
+
+__m256i test_mm256_loadrs_epi64(const __m256i * __A) {
+  // CHECK-LABEL: @test_mm256_loadrs_epi64(
+  // CHECK: call <4 x i64> @llvm.x86.avx10.vmovrsq256(
+  return _mm256_loadrs_epi64(__A);
+}
+
+__m256i test_mm256_mask_loadrs_epi64(__m256i __A, __mmask8 __B, const __m256i * __C) {
+  // CHECK-LABEL: @test_mm256_mask_loadrs_epi64(
+  // CHECK: call <4 x i64> @llvm.x86.avx10.vmovrsq256(
+  // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
+  return _mm256_mask_loadrs_epi64(__A, __B, __C);
+}
+
+__m256i test_mm256_maskz_loadrs_epi64(__mmask8 __A, const __m256i * __B) {
+  // CHECK-LABEL: @test_mm256_maskz_loadrs_epi64(
+  // CHECK: call <4 x i64> @llvm.x86.avx10.vmovrsq256(
+  // CHECK: store <4 x i64> zeroinitializer
+  // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
+  return _mm256_maskz_loadrs_epi64(__A, __B);
+}
+
+__m128i test_mm_loadrs_epi16(const __m128i * __A) {
+  // CHECK-LABEL: @test_mm_loadrs_epi16(
+  // CHECK: call <8 x i16> @llvm.x86.avx10.vmovrsw128(
+  return _mm_loadrs_epi16(__A);
+}
+
+__m128i test_mm_mask_loadrs_epi16(__m128i __A, __mmask8 __B, const __m128i * __C) {
+  // CHECK-LABEL: @test_mm_mask_loadrs_epi16(
+  // CHECK: call <8 x i16> @llvm.x86.avx10.vmovrsw128(
+  // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
+  return _mm_mask_loadrs_epi16(__A, __B, __C);
+}
+
+__m128i test_mm_maskz_loadrs_epi16(__mmask8 __A, const __m128i * __B) {
+  // CHECK-LABEL: @test_mm_maskz_loadrs_epi16(
+  // CHECK: call <8 x i16> @llvm.x86.avx10.vmovrsw128(
+  // CHECK: store <2 x i64> zeroinitializer
+  // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
+  return _mm_maskz_loadrs_epi16(__A, __B);
+}
+
+__m256i test_mm256_loadrs_epi16(const __m256i * __A) {
+  // CHECK-LABEL: @test_mm256_loadrs_epi16(
+  // CHECK: call <16 x i16> @llvm.x86.avx10.vmovrsw256(
+  return _mm256_loadrs_epi16(__A);
+}
+
+__m256i test_mm256_mask_loadrs_epi16(__m256i __A, __mmask16 __B, const __m256i * __C) {
+  // CHECK-LABEL: @test_mm256_mask_loadrs_epi16(
+  // CHECK: call <16 x i16> @llvm.x86.avx10.vmovrsw256(
+  // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
+  return _mm256_mask_loadrs_epi16(__A, __B, __C);
+}
+
+__m256i test_mm256_maskz_loadrs_epi16(__mmask16 __A, const __m256i * __B) {
+  // CHECK-LABEL: @test_mm256_maskz_loadrs_epi16(
+  // CHECK: call <16 x i16> @llvm.x86.avx10.vmovrsw256(
+  // CHECK: store <4 x i64> zeroinitializer
+  // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
+  return _mm256_maskz_loadrs_epi16(__A, __B);
+}
diff --git a/clang/test/CodeGen/target-builtin-noerror.c b/clang/test/CodeGen/target-builtin-noerror.c
index 2a05074d7c2b..1e53621bc6b5 100644
--- a/clang/test/CodeGen/target-builtin-noerror.c
+++ b/clang/test/CodeGen/target-builtin-noerror.c
@@ -145,6 +145,7 @@ void verifyfeaturestrings(void) {
   (void)__builtin_cpu_supports("avx10.1-512");
   (void)__builtin_cpu_supports("avx10.2-256");
   (void)__builtin_cpu_supports("avx10.2-512");
+  (void)__builtin_cpu_supports("movrs");
 }
 
 void verifycpustrings(void) {
diff --git a/clang/test/Driver/x86-target-features.c b/clang/test/Driver/x86-target-features.c
index ddfbb29a48f8..02370ef60b7f 100644
--- a/clang/test/Driver/x86-target-features.c
+++ b/clang/test/Driver/x86-target-features.c
@@ -404,6 +404,11 @@
 // USERMSR: "-target-feature" "+usermsr"
 // NO-USERMSR: "-target-feature" "-usermsr"
 
+// RUN: %clang --target=i386 -mmovrs %s -### -o %t.o 2>&1 | FileCheck -check-prefix=MOVRS %s
+// RUN: %clang --target=i386 -mno-movrs %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-MOVRS %s
+// MOVRS: "-target-feature" "+movrs"
+// NO-MOVRS: "-target-feature" "-movrs"
+
 // RUN: %clang --target=i386 -march=i386 -mcrc32 %s -### 2>&1 | FileCheck -check-prefix=CRC32 %s
 // RUN: %clang --target=i386 -march=i386 -mno-crc32 %s -### 2>&1 | FileCheck -check-prefix=NO-CRC32 %s
 // CRC32: "-target-feature" "+crc32"
diff --git a/clang/test/Preprocessor/x86_target_features.c b/clang/test/Preprocessor/x86_target_features.c
index 8b4e6bdc0922..2d1d2e57bdc7 100644
--- a/clang/test/Preprocessor/x86_target_features.c
+++ b/clang/test/Preprocessor/x86_target_features.c
@@ -740,6 +740,12 @@
 // RUN: %clang -target i686-unknown-linux-gnu -march=atom -mno-usermsr -x c -E -dM -o - %s | FileCheck  -check-prefix=NO-USERMSR %s
 // NO-USERMSR-NOT: #define __USERMSR__ 1
 
+// RUN: %clang -target i686-unknown-linux-gnu -march=atom -mmovrs -x c -E -dM -o - %s | FileCheck  -check-prefix=MOVRS %s
+// MOVRS: #define __MOVRS__ 1
+
+// RUN: %clang -target i686-unknown-linux-gnu -march=atom -mno-movrs -x c -E -dM -o - %s | FileCheck  -check-prefix=NO-MOVRS %s
+// NO-MOVRS-NOT: #define __MOVRS__ 1
+
 // RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mcrc32 -x c -E -dM -o - %s | FileCheck -check-prefix=CRC32 %s
 
 // CRC32: #define __CRC32__ 1
diff --git a/compiler-rt/lib/builtins/cpu_model/x86.c b/compiler-rt/lib/builtins/cpu_model/x86.c
index 7fa4b9e2b660..bfa478c4427a 100644
--- a/compiler-rt/lib/builtins/cpu_model/x86.c
+++ b/compiler-rt/lib/builtins/cpu_model/x86.c
@@ -229,6 +229,7 @@ enum ProcessorFeatures {
   FEATURE_AVX10_1_512,
   FEATURE_AVX10_2_256,
   FEATURE_AVX10_2_512,
+  FEATURE_MOVRS,
   CPU_FEATURE_MAX
 };
 
@@ -972,6 +973,8 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
     setFeature(FEATURE_HRESET);
   if (HasLeaf7Subleaf1 && ((EAX >> 23) & 1) && HasAVXSave)
     setFeature(FEATURE_AVXIFMA);
+  if (HasLeaf7Subleaf1 && ((EAX >> 31) & 1))
+    setFeature(FEATURE_MOVRS);
 
   if (HasLeaf7Subleaf1 && ((EDX >> 4) & 1) && HasAVXSave)
     setFeature(FEATURE_AVXVNNIINT8);
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 504fce4160a5..706546980cf6 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -196,6 +196,8 @@ Changes to the X86 Backend
 
 * Support ISA of `AVX10.2-256` and `AVX10.2-512`.
 
+* Supported instructions of `MOVRS AND AVX10.2`
+
 Changes to the OCaml bindings
 -----------------------------
 
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index 5262e3154ff7..d0083017fb93 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -7572,3 +7572,42 @@ def int_x86_avx10_vfnmsub231nepbf16128 : ClangBuiltin<"__builtin_ia32_vfnmsub231
         DefaultAttrsIntrinsic<[llvm_v8bf16_ty], [llvm_v8bf16_ty, llvm_v8bf16_ty, llvm_v8bf16_ty ],
                               [IntrNoMem]>;
 }
+
+let TargetPrefix = "x86" in {
+def int_x86_avx10_vmovrsb128 : ClangBuiltin<"__builtin_ia32_vmovrsb128">,
+        DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+def int_x86_avx10_vmovrsb256 : ClangBuiltin<"__builtin_ia32_vmovrsb256">,
+        DefaultAttrsIntrinsic<[llvm_v32i8_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+def int_x86_avx10_vmovrsb512 : ClangBuiltin<"__builtin_ia32_vmovrsb512">,
+        DefaultAttrsIntrinsic<[llvm_v64i8_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+def int_x86_avx10_vmovrsd128 : ClangBuiltin<"__builtin_ia32_vmovrsd128">,
+        DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+def int_x86_avx10_vmovrsd256 : ClangBuiltin<"__builtin_ia32_vmovrsd256">,
+        DefaultAttrsIntrinsic<[llvm_v8i32_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+def int_x86_avx10_vmovrsd512 : ClangBuiltin<"__builtin_ia32_vmovrsd512">,
+        DefaultAttrsIntrinsic<[llvm_v16i32_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+def int_x86_avx10_vmovrsq128 : ClangBuiltin<"__builtin_ia32_vmovrsq128">,
+        DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+def int_x86_avx10_vmovrsq256 : ClangBuiltin<"__builtin_ia32_vmovrsq256">,
+        DefaultAttrsIntrinsic<[llvm_v4i64_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+def int_x86_avx10_vmovrsq512 : ClangBuiltin<"__builtin_ia32_vmovrsq512">,
+        DefaultAttrsIntrinsic<[llvm_v8i64_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+def int_x86_avx10_vmovrsw128 : ClangBuiltin<"__builtin_ia32_vmovrsw128">,
+        DefaultAttrsIntrinsic<[llvm_v8i16_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+def int_x86_avx10_vmovrsw256 : ClangBuiltin<"__builtin_ia32_vmovrsw256">,
+        DefaultAttrsIntrinsic<[llvm_v16i16_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+def int_x86_avx10_vmovrsw512 : ClangBuiltin<"__builtin_ia32_vmovrsw512">,
+        DefaultAttrsIntrinsic<[llvm_v32i16_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+}
diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.def b/llvm/include/llvm/TargetParser/X86TargetParser.def
index e5bf196559ba..073e19f8187c 100644
--- a/llvm/include/llvm/TargetParser/X86TargetParser.def
+++ b/llvm/include/llvm/TargetParser/X86TargetParser.def
@@ -261,6 +261,8 @@ X86_FEATURE_COMPAT(AVX10_1,         "avx10.1-256",           36)
 X86_FEATURE_COMPAT(AVX10_1_512,     "avx10.1-512",           37)
 X86_FEATURE_COMPAT(AVX10_2,         "avx10.2-256",            0)
 X86_FEATURE_COMPAT(AVX10_2_512,     "avx10.2-512",            0)
+//FIXME: make MOVRS _COMPAT defined when gcc landed relate patch.
+X86_FEATURE       (MOVRS,           "movrs")
 X86_FEATURE       (ZU,              "zu")
 // These features aren't really CPU features, but the frontend can set them.
 X86_FEATURE       (RETPOLINE_EXTERNAL_THUNK,    "retpoline-external-thunk")
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index d57450d91ea2..6bedf9e1d13a 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -351,6 +351,8 @@ def FeatureZU : SubtargetFeature<"zu", "HasZU", "true",
 def FeatureUseGPR32InInlineAsm
     : SubtargetFeature<"inline-asm-use-gpr32", "UseInlineAsmGPR32", "true",
                        "Enable use of GPR32 in inline assembly for APX">;
+def FeatureMOVRS   : SubtargetFeature<"movrs", "HasMOVRS", "true",
+                           "Enable MOVRS", []>;
 
 // Ivy Bridge and newer processors have enhanced REP MOVSB and STOSB (aka
 // "string operations"). See "REP String Enhancement" in the Intel Software
diff --git a/llvm/lib/Target/X86/X86InstrAVX10.td b/llvm/lib/Target/X86/X86InstrAVX10.td
index 625f2e01d472..9ef2debb57fa 100644
--- a/llvm/lib/Target/X86/X86InstrAVX10.td
+++ b/llvm/lib/Target/X86/X86InstrAVX10.td
@@ -1647,3 +1647,31 @@ let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in
   def : InstAlias<"vmovw.s\t{$src, $dst|$dst, $src}",
                   (VMOVZPWILo2PWIZrr2 VR128X:$dst, VR128X:$src), 0>;
 }
+
+// MOVRS
+multiclass vmovrs_p<bits<8> opc, string OpStr, X86VectorVTInfo _> {
+  let ExeDomain = _.ExeDomain in {
+    defm m: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                            (ins _.MemOp:$src), OpStr, "$src", "$src",
+                            (_.VT (!cast<Intrinsic>("int_x86_avx10_"#OpStr#_.Size)
+                                   addr:$src))>, EVEX;
+  }
+}
+
+multiclass vmovrs_p_vl<bits<8> opc, string OpStr, AVX512VLVectorVTInfo _Vec> {
+  let Predicates = [HasMOVRS, HasAVX10_2_512] in
+    defm Z : vmovrs_p<opc, OpStr, _Vec.info512>, EVEX_V512;
+  let Predicates = [HasMOVRS, HasAVX10_2] in {
+    defm Z128 : vmovrs_p<opc, OpStr, _Vec.info128>, EVEX_V128;
+    defm Z256 : vmovrs_p<opc, OpStr, _Vec.info256>, EVEX_V256;
+  }
+}
+
+defm VMOVRSB : vmovrs_p_vl<0x6f, "vmovrsb", avx512vl_i8_info>,
+                          T_MAP5, XD, EVEX_CD8<8, CD8VF>, Sched<[WriteVecLoad]>;
+defm VMOVRSW : vmovrs_p_vl<0x6f, "vmovrsw", avx512vl_i16_info>,
+                          T_MAP5, XD, REX_W, EVEX_CD8<16, CD8VF>, Sched<[WriteVecLoad]>;
+defm VMOVRSD : vmovrs_p_vl<0x6f, "vmovrsd", avx512vl_i32_info>,
+                          T_MAP5, XS, EVEX_CD8<32, CD8VF>, Sched<[WriteVecLoad]>;
+defm VMOVRSQ : vmovrs_p_vl<0x6f, "vmovrsq", avx512vl_i64_info>,
+                          T_MAP5, XS, REX_W, EVEX_CD8<64, CD8VF>, Sched<[WriteVecLoad]>;
diff --git a/llvm/lib/Target/X86/X86InstrPredicates.td b/llvm/lib/Target/X86/X86InstrPredicates.td
index a815ddc9714f..7fb566fba518 100644
--- a/llvm/lib/Target/X86/X86InstrPredicates.td
+++ b/llvm/lib/Target/X86/X86InstrPredicates.td
@@ -152,6 +152,7 @@ def HasCLZERO    : Predicate<"Subtarget->hasCLZERO()">;
 def HasCLDEMOTE  : Predicate<"Subtarget->hasCLDEMOTE()">;
 def HasMOVDIRI   : Predicate<"Subtarget->hasMOVDIRI()">;
 def HasMOVDIR64B : Predicate<"Subtarget->hasMOVDIR64B()">;
+def HasMOVRS     : Predicate<"Subtarget->hasMOVRS()">;
 def HasPTWRITE   : Predicate<"Subtarget->hasPTWRITE()">;
 def FPStackf32   : Predicate<"!Subtarget->hasSSE1()">;
 def FPStackf64   : Predicate<"!Subtarget->hasSSE2()">;
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index 1f608f47ef79..5c4e3a9dc52b 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -1841,6 +1841,7 @@ const StringMap<bool> sys::getHostCPUFeatures() {
   Features["cmpccxadd"]  = HasLeaf7Subleaf1 && ((EAX >> 7) & 1);
   Features["hreset"]     = HasLeaf7Subleaf1 && ((EAX >> 22) & 1);
   Features["avxifma"]    = HasLeaf7Subleaf1 && ((EAX >> 23) & 1) && HasAVXSave;
+  Features["movrs"] = HasLeaf7Subleaf1 && ((EAX >> 31) & 1);
   Features["avxvnniint8"] = HasLeaf7Subleaf1 && ((EDX >> 4) & 1) && HasAVXSave;
   Features["avxneconvert"] = HasLeaf7Subleaf1 && ((EDX >> 5) & 1) && HasAVXSave;
   Features["amx-complex"] = HasLeaf7Subleaf1 && ((EDX >> 8) & 1) && HasAMXSave;
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index 09d4312918ac..586df5748aa8 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -639,6 +639,8 @@ constexpr FeatureBitset ImpliedFeaturesNF = {};
 constexpr FeatureBitset ImpliedFeaturesCF = {};
 constexpr FeatureBitset ImpliedFeaturesZU = {};
 
+constexpr FeatureBitset ImpliedFeaturesMOVRS = {};
+
 constexpr FeatureInfo FeatureInfos[X86::CPU_FEATURE_MAX] = {
 #define X86_FEATURE(ENUM, STR) {{"+" STR}, ImpliedFeatures##ENUM},
 #include "llvm/TargetParser/X86TargetParser.def"
diff --git a/llvm/test/CodeGen/X86/movrs-avx10.2-512-intrinsics.ll b/llvm/test/CodeGen/X86/movrs-avx10.2-512-intrinsics.ll
new file mode 100644
index 000000000000..a730ef519c01
--- /dev/null
+++ b/llvm/test/CodeGen/X86/movrs-avx10.2-512-intrinsics.ll
@@ -0,0 +1,163 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+movrs,+avx10.2-512 -verify-machineinstrs --show-mc-encoding | FileCheck %s --check-prefixes=CHECK
+
+declare <64 x i8> @llvm.x86.avx10.vmovrsb512(ptr)
+declare <16 x i32> @llvm.x86.avx10.vmovrsd512(ptr)
+declare <8 x i64> @llvm.x86.avx10.vmovrsq512(ptr)
+declare <32 x i16> @llvm.x86.avx10.vmovrsw512(ptr)
+
+define <8 x i64> @test_mm512_movrsb_epi8(ptr %__A) {
+; CHECK-LABEL: test_mm512_movrsb_epi8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovrsb (%rdi), %zmm0 # encoding: [0x62,0xf5,0x7f,0x48,0x6f,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <64 x i8> @llvm.x86.avx10.vmovrsb512(ptr %__A)
+  %1 = bitcast <64 x i8> %0 to <8 x i64>
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @test_mm512_mask_movrsb_epi8(<8 x i64> %__A, i64 %__B, ptr %__C) {
+; CHECK-LABEL: test_mm512_mask_movrsb_epi8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsb (%rsi), %zmm0 {%k1} # encoding: [0x62,0xf5,0x7f,0x49,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <64 x i8> @llvm.x86.avx10.vmovrsb512(ptr %__C)
+  %1 = bitcast <8 x i64> %__A to <64 x i8>
+  %2 = bitcast i64 %__B to <64 x i1>
+  %3 = select <64 x i1> %2, <64 x i8> %0, <64 x i8> %1
+  %4 = bitcast <64 x i8> %3 to <8 x i64>
+  ret <8 x i64> %4
+}
+
+define dso_local <8 x i64> @test_mm512_maskz_movrsb_epi8(i64 %__A, ptr %__B) {
+; CHECK-LABEL: test_mm512_maskz_movrsb_epi8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsb (%rsi), %zmm0 {%k1} {z} # encoding: [0x62,0xf5,0x7f,0xc9,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <64 x i8> @llvm.x86.avx10.vmovrsb512(ptr %__B)
+  %1 = bitcast i64 %__A to <64 x i1>
+  %2 = select <64 x i1> %1, <64 x i8> %0, <64 x i8> zeroinitializer
+  %3 = bitcast <64 x i8> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @test_mm512_movrsd_epi32(ptr %__A) {
+; CHECK-LABEL: test_mm512_movrsd_epi32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovrsd (%rdi), %zmm0 # encoding: [0x62,0xf5,0x7e,0x48,0x6f,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <16 x i32> @llvm.x86.avx10.vmovrsd512(ptr %__A)
+  %1 = bitcast <16 x i32> %0 to <8 x i64>
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @test_mm512_mask_movrsd_epi32(<8 x i64> %__A, i16 zeroext %__B, ptr %__C) {
+; CHECK-LABEL: test_mm512_mask_movrsd_epi32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsd (%rsi), %zmm0 {%k1} # encoding: [0x62,0xf5,0x7e,0x49,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <16 x i32> @llvm.x86.avx10.vmovrsd512(ptr %__C)
+  %1 = bitcast <8 x i64> %__A to <16 x i32>
+  %2 = bitcast i16 %__B to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %0, <16 x i32> %1
+  %4 = bitcast <16 x i32> %3 to <8 x i64>
+  ret <8 x i64> %4
+}
+
+define <8 x i64> @test_mm512_maskz_movrsd_epi32(i16 zeroext %__A, ptr %__B) {
+; CHECK-LABEL: test_mm512_maskz_movrsd_epi32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsd (%rsi), %zmm0 {%k1} {z} # encoding: [0x62,0xf5,0x7e,0xc9,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <16 x i32> @llvm.x86.avx10.vmovrsd512(ptr %__B)
+  %1 = bitcast i16 %__A to <16 x i1>
+  %2 = select <16 x i1> %1, <16 x i32> %0, <16 x i32> zeroinitializer
+  %3 = bitcast <16 x i32> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @test_mm512_movrsq_epi64(ptr %__A) {
+; CHECK-LABEL: test_mm512_movrsq_epi64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovrsq (%rdi), %zmm0 # encoding: [0x62,0xf5,0xfe,0x48,0x6f,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <8 x i64> @llvm.x86.avx10.vmovrsq512(ptr %__A)
+  ret <8 x i64> %0
+}
+
+define <8 x i64> @test_mm512_mask_movrsq_epi64(<8 x i64> %__A, i8 zeroext %__B, ptr %__C) {
+; CHECK-LABEL: test_mm512_mask_movrsq_epi64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsq (%rsi), %zmm0 {%k1} # encoding: [0x62,0xf5,0xfe,0x49,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <8 x i64> @llvm.x86.avx10.vmovrsq512(ptr %__C)
+  %1 = bitcast i8 %__B to <8 x i1>
+  %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> %__A
+  ret <8 x i64> %2
+}
+
+define <8 x i64> @test_mm512_maskz_movrsq_epi64(i8 zeroext %__A, ptr %__B) {
+; CHECK-LABEL: test_mm512_maskz_movrsq_epi64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsq (%rsi), %zmm0 {%k1} {z} # encoding: [0x62,0xf5,0xfe,0xc9,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <8 x i64> @llvm.x86.avx10.vmovrsq512(ptr %__B)
+  %1 = bitcast i8 %__A to <8 x i1>
+  %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> zeroinitializer
+  ret <8 x i64> %2
+}
+
+define <8 x i64> @test_mm512_movrsw_epi16(ptr %__A) {
+; CHECK-LABEL: test_mm512_movrsw_epi16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovrsw (%rdi), %zmm0 # encoding: [0x62,0xf5,0xff,0x48,0x6f,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <32 x i16> @llvm.x86.avx10.vmovrsw512(ptr %__A)
+  %1 = bitcast <32 x i16> %0 to <8 x i64>
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @test_mm512_mask_movrsw_epi16(<8 x i64> %__A, i32 %__B, ptr %__C) {
+; CHECK-LABEL: test_mm512_mask_movrsw_epi16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsw (%rsi), %zmm0 {%k1} # encoding: [0x62,0xf5,0xff,0x49,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <32 x i16> @llvm.x86.avx10.vmovrsw512(ptr %__C)
+  %1 = bitcast <8 x i64> %__A to <32 x i16>
+  %2 = bitcast i32 %__B to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %0, <32 x i16> %1
+  %4 = bitcast <32 x i16> %3 to <8 x i64>
+  ret <8 x i64> %4
+}
+
+define <8 x i64> @test_mm512_maskz_movrsw_epi16(i32 %__A, ptr %__B) {
+; CHECK-LABEL: test_mm512_maskz_movrsw_epi16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsw (%rsi), %zmm0 {%k1} {z} # encoding: [0x62,0xf5,0xff,0xc9,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <32 x i16> @llvm.x86.avx10.vmovrsw512(ptr %__B)
+  %1 = bitcast i32 %__A to <32 x i1>
+  %2 = select <32 x i1> %1, <32 x i16> %0, <32 x i16> zeroinitializer
+  %3 = bitcast <32 x i16> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
diff --git a/llvm/test/CodeGen/X86/movrs-avx10.2-intrinsics.ll b/llvm/test/CodeGen/X86/movrs-avx10.2-intrinsics.ll
new file mode 100644
index 000000000000..583e16351652
--- /dev/null
+++ b/llvm/test/CodeGen/X86/movrs-avx10.2-intrinsics.ll
@@ -0,0 +1,329 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+movrs,+avx10.2-256 -verify-machineinstrs --show-mc-encoding | FileCheck %s --check-prefixes=CHECK
+
+define <2 x i64> @test_mm_movrsb_epu8(ptr %__A) {
+; CHECK-LABEL: test_mm_movrsb_epu8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovrsb (%rdi), %xmm0 # encoding: [0x62,0xf5,0x7f,0x08,0x6f,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <16 x i8> @llvm.x86.avx10.vmovrsb128(ptr %__A)
+  %1 = bitcast <16 x i8> %0 to <2 x i64>
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_mm_mask_movrsb_epu8(<2 x i64> %__A, i16 zeroext %__B, ptr %__C) {
+; CHECK-LABEL: test_mm_mask_movrsb_epu8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsb (%rsi), %xmm0 {%k1} # encoding: [0x62,0xf5,0x7f,0x09,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <16 x i8> @llvm.x86.avx10.vmovrsb128(ptr %__C)
+  %1 = bitcast <2 x i64> %__A to <16 x i8>
+  %2 = bitcast i16 %__B to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %0, <16 x i8> %1
+  %4 = bitcast <16 x i8> %3 to <2 x i64>
+  ret <2 x i64> %4
+}
+
+define <2 x i64> @test_mm_maskz_movrsb_epu8(i16 zeroext %__A, ptr %__B) {
+; CHECK-LABEL: test_mm_maskz_movrsb_epu8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsb (%rsi), %xmm0 {%k1} {z} # encoding: [0x62,0xf5,0x7f,0x89,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <16 x i8> @llvm.x86.avx10.vmovrsb128(ptr %__B )
+  %1 = bitcast i16 %__A to <16 x i1>
+  %2 = select <16 x i1> %1, <16 x i8> %0, <16 x i8> zeroinitializer
+  %3 = bitcast <16 x i8> %2 to <2 x i64>
+  ret <2 x i64> %3
+}
+
+define <4 x i64> @test_mm256_movrsb_epu8(ptr %__A) {
+; CHECK-LABEL: test_mm256_movrsb_epu8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovrsb (%rdi), %ymm0 # encoding: [0x62,0xf5,0x7f,0x28,0x6f,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <32 x i8> @llvm.x86.avx10.vmovrsb256(ptr %__A)
+  %1 = bitcast <32 x i8> %0 to <4 x i64>
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @test_mm256_mask_movrsb_epu8(<4 x i64> %__A, i32 %__B, ptr %__C) {
+; CHECK-LABEL: test_mm256_mask_movrsb_epu8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsb (%rsi), %ymm0 {%k1} # encoding: [0x62,0xf5,0x7f,0x29,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <32 x i8> @llvm.x86.avx10.vmovrsb256(ptr %__C)
+  %1 = bitcast <4 x i64> %__A to <32 x i8>
+  %2 = bitcast i32 %__B to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %0, <32 x i8> %1
+  %4 = bitcast <32 x i8> %3 to <4 x i64>
+  ret <4 x i64> %4
+}
+
+define <4 x i64> @test_mm256_maskz_movrsb_epu8(i32 %__A, ptr %__B) {
+; CHECK-LABEL: test_mm256_maskz_movrsb_epu8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsb (%rsi), %ymm0 {%k1} {z} # encoding: [0x62,0xf5,0x7f,0xa9,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <32 x i8> @llvm.x86.avx10.vmovrsb256(ptr %__B)
+  %1 = bitcast i32 %__A to <32 x i1>
+  %2 = select <32 x i1> %1, <32 x i8> %0, <32 x i8> zeroinitializer
+  %3 = bitcast <32 x i8> %2 to <4 x i64>
+  ret <4 x i64> %3
+}
+
+define <2 x i64> @test_mm_movrsd_epu32(ptr %__A) {
+; CHECK-LABEL: test_mm_movrsd_epu32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovrsd (%rdi), %xmm0 # encoding: [0x62,0xf5,0x7e,0x08,0x6f,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <4 x i32> @llvm.x86.avx10.vmovrsd128(ptr %__A)
+  %1 = bitcast <4 x i32> %0 to <2 x i64>
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_mm_mask_movrsd_epu32(<2 x i64> %__A, i8 zeroext %__B, ptr %__C) {
+; CHECK-LABEL: test_mm_mask_movrsd_epu32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsd (%rsi), %xmm0 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <4 x i32> @llvm.x86.avx10.vmovrsd128(ptr %__C)
+  %1 = bitcast <2 x i64> %__A to <4 x i32>
+  %2 = bitcast i8 %__B to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %2, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract.i, <4 x i32> %0, <4 x i32> %1
+  %4 = bitcast <4 x i32> %3 to <2 x i64>
+  ret <2 x i64> %4
+}
+
+define <2 x i64> @test_mm_maskz_movrsd_epu32(i8 zeroext %__A, ptr %__B) {
+; CHECK-LABEL: test_mm_maskz_movrsd_epu32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsd (%rsi), %xmm0 {%k1} {z} # encoding: [0x62,0xf5,0x7e,0x89,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <4 x i32> @llvm.x86.avx10.vmovrsd128(ptr %__B)
+  %1 = bitcast i8 %__A to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = select <4 x i1> %extract.i, <4 x i32> %0, <4 x i32> zeroinitializer
+  %3 = bitcast <4 x i32> %2 to <2 x i64>
+  ret <2 x i64> %3
+}
+
+define <4 x i64> @test_mm256_movrsd_epu32(ptr %__A) {
+; CHECK-LABEL: test_mm256_movrsd_epu32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovrsd (%rdi), %ymm0 # encoding: [0x62,0xf5,0x7e,0x28,0x6f,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <8 x i32> @llvm.x86.avx10.vmovrsd256(ptr %__A)
+  %1 = bitcast <8 x i32> %0 to <4 x i64>
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @test_mm256_mask_movrsd_epu32(<4 x i64> %__A, i8 zeroext %__B, ptr %__C) {
+; CHECK-LABEL: test_mm256_mask_movrsd_epu32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsd (%rsi), %ymm0 {%k1} # encoding: [0x62,0xf5,0x7e,0x29,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <8 x i32> @llvm.x86.avx10.vmovrsd256(ptr %__C)
+  %1 = bitcast <4 x i64> %__A to <8 x i32>
+  %2 = bitcast i8 %__B to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %0, <8 x i32> %1
+  %4 = bitcast <8 x i32> %3 to <4 x i64>
+  ret <4 x i64> %4
+}
+
+define <4 x i64> @test_mm256_maskz_movrsd_epu32(i8 zeroext %__A, ptr %__B) {
+; CHECK-LABEL: test_mm256_maskz_movrsd_epu32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsd (%rsi), %ymm0 {%k1} {z} # encoding: [0x62,0xf5,0x7e,0xa9,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <8 x i32> @llvm.x86.avx10.vmovrsd256(ptr %__B)
+  %1 = bitcast i8 %__A to <8 x i1>
+  %2 = select <8 x i1> %1, <8 x i32> %0, <8 x i32> zeroinitializer
+  %3 = bitcast <8 x i32> %2 to <4 x i64>
+  ret <4 x i64> %3
+}
+
+define <2 x i64> @test_mm_movrsq_epu64(ptr %__A) {
+; CHECK-LABEL: test_mm_movrsq_epu64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovrsq (%rdi), %xmm0 # encoding: [0x62,0xf5,0xfe,0x08,0x6f,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <2 x i64> @llvm.x86.avx10.vmovrsq128(ptr %__A)
+  ret <2 x i64> %0
+}
+
+define <2 x i64> @test_mm_mask_movrsq_epu64(<2 x i64> %__A, i8 zeroext %__B, ptr %__C) {
+; CHECK-LABEL: test_mm_mask_movrsq_epu64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsq (%rsi), %xmm0 {%k1} # encoding: [0x62,0xf5,0xfe,0x09,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <2 x i64> @llvm.x86.avx10.vmovrsq128(ptr %__C)
+  %1 = bitcast i8 %__B to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
+  %2 = select <2 x i1> %extract.i, <2 x i64> %0, <2 x i64> %__A
+  ret <2 x i64> %2
+}
+
+define <2 x i64> @test_mm_maskz_movrsq_epu64(i8 zeroext %__A, ptr %__B) {
+; CHECK-LABEL: test_mm_maskz_movrsq_epu64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsq (%rsi), %xmm0 {%k1} {z} # encoding: [0x62,0xf5,0xfe,0x89,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <2 x i64> @llvm.x86.avx10.vmovrsq128(ptr %__B)
+  %1 = bitcast i8 %__A to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
+  %2 = select <2 x i1> %extract.i, <2 x i64> %0, <2 x i64> zeroinitializer
+  ret <2 x i64> %2
+}
+
+define <4 x i64> @test_mm256_movrsq_epu64(ptr %__A) {
+; CHECK-LABEL: test_mm256_movrsq_epu64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovrsq (%rdi), %ymm0 # encoding: [0x62,0xf5,0xfe,0x28,0x6f,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <4 x i64> @llvm.x86.avx10.vmovrsq256(ptr %__A)
+  ret <4 x i64> %0
+}
+
+define <4 x i64> @test_mm256_mask_movrsq_epu64(<4 x i64> %__A, i8 zeroext %__B, ptr %__C) {
+; CHECK-LABEL: test_mm256_mask_movrsq_epu64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsq (%rsi), %ymm0 {%k1} # encoding: [0x62,0xf5,0xfe,0x29,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <4 x i64> @llvm.x86.avx10.vmovrsq256(ptr %__C)
+  %1 = bitcast i8 %__B to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = select <4 x i1> %extract.i, <4 x i64> %0, <4 x i64> %__A
+  ret <4 x i64> %2
+}
+
+define <4 x i64> @test_mm256_maskz_movrsq_epu64(i8 zeroext %__A, ptr %__B) {
+; CHECK-LABEL: test_mm256_maskz_movrsq_epu64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsq (%rsi), %ymm0 {%k1} {z} # encoding: [0x62,0xf5,0xfe,0xa9,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <4 x i64> @llvm.x86.avx10.vmovrsq256(ptr %__B)
+  %1 = bitcast i8 %__A to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = select <4 x i1> %extract.i, <4 x i64> %0, <4 x i64> zeroinitializer
+  ret <4 x i64> %2
+}
+
+define <2 x i64> @test_mm_movrsw_epu16(ptr %__A) {
+; CHECK-LABEL: test_mm_movrsw_epu16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovrsw (%rdi), %xmm0 # encoding: [0x62,0xf5,0xff,0x08,0x6f,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <8 x i16> @llvm.x86.avx10.vmovrsw128(ptr %__A)
+  %1 = bitcast <8 x i16> %0 to <2 x i64>
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_mm_mask_movrsw_epu16(<2 x i64> %__A, i8 zeroext %__B, ptr %__C) {
+; CHECK-LABEL: test_mm_mask_movrsw_epu16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsw (%rsi), %xmm0 {%k1} # encoding: [0x62,0xf5,0xff,0x09,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <8 x i16> @llvm.x86.avx10.vmovrsw128(ptr %__C)
+  %1 = bitcast <2 x i64> %__A to <8 x i16>
+  %2 = bitcast i8 %__B to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %0, <8 x i16> %1
+  %4 = bitcast <8 x i16> %3 to <2 x i64>
+  ret <2 x i64> %4
+}
+
+define <2 x i64> @test_mm_maskz_movrsw_epu16(i8 zeroext %__A, ptr %__B) {
+; CHECK-LABEL: test_mm_maskz_movrsw_epu16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsw (%rsi), %xmm0 {%k1} {z} # encoding: [0x62,0xf5,0xff,0x89,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <8 x i16> @llvm.x86.avx10.vmovrsw128(ptr %__B)
+  %1 = bitcast i8 %__A to <8 x i1>
+  %2 = select <8 x i1> %1, <8 x i16> %0, <8 x i16> zeroinitializer
+  %3 = bitcast <8 x i16> %2 to <2 x i64>
+  ret <2 x i64> %3
+}
+
+define <4 x i64> @test_mm256_movrsw_epu16(ptr %__A) {
+; CHECK-LABEL: test_mm256_movrsw_epu16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovrsw (%rdi), %ymm0 # encoding: [0x62,0xf5,0xff,0x28,0x6f,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <16 x i16> @llvm.x86.avx10.vmovrsw256(ptr %__A)
+  %1 = bitcast <16 x i16> %0 to <4 x i64>
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @test_mm256_mask_movrsw_epu16(<4 x i64> %__A, i16 zeroext %__B, ptr %__C) {
+; CHECK-LABEL: test_mm256_mask_movrsw_epu16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsw (%rsi), %ymm0 {%k1} # encoding: [0x62,0xf5,0xff,0x29,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <16 x i16> @llvm.x86.avx10.vmovrsw256(ptr %__C)
+  %1 = bitcast <4 x i64> %__A to <16 x i16>
+  %2 = bitcast i16 %__B to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %0, <16 x i16> %1
+  %4 = bitcast <16 x i16> %3 to <4 x i64>
+  ret <4 x i64> %4
+}
+
+define <4 x i64> @test_mm256_maskz_movrsw_epu16(i16 zeroext %__A, ptr %__B) {
+; CHECK-LABEL: test_mm256_maskz_movrsw_epu16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vmovrsw (%rsi), %ymm0 {%k1} {z} # encoding: [0x62,0xf5,0xff,0xa9,0x6f,0x06]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = tail call <16 x i16> @llvm.x86.avx10.vmovrsw256(ptr %__B)
+  %1 = bitcast i16 %__A to <16 x i1>
+  %2 = select <16 x i1> %1, <16 x i16> %0, <16 x i16> zeroinitializer
+  %3 = bitcast <16 x i16> %2 to <4 x i64>
+  ret <4 x i64> %3
+}
+
+declare <16 x i8> @llvm.x86.avx10.vmovrsb128(ptr)
+declare <32 x i8> @llvm.x86.avx10.vmovrsb256(ptr)
+declare <4 x i32> @llvm.x86.avx10.vmovrsd128(ptr)
+declare <8 x i32> @llvm.x86.avx10.vmovrsd256(ptr)
+declare <2 x i64> @llvm.x86.avx10.vmovrsq128(ptr)
+declare <4 x i64> @llvm.x86.avx10.vmovrsq256(ptr)
+declare <8 x i16> @llvm.x86.avx10.vmovrsw128(ptr)
+declare <16 x i16> @llvm.x86.avx10.vmovrsw256(ptr)
diff --git a/llvm/test/MC/Disassembler/X86/movrs-avx10-64.txt b/llvm/test/MC/Disassembler/X86/movrs-avx10-64.txt
new file mode 100644
index 000000000000..e25e66ae5774
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/movrs-avx10-64.txt
@@ -0,0 +1,98 @@
+# RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding --disassemble < %s  | FileCheck %s --check-prefixes=ATT
+# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL
+
+# ATT:   vmovrsb  268435456(%rbp,%r14,8), %zmm22
+# INTEL: vmovrsb zmm22, zmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xa5,0x7f,0x48,0x6f,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   vmovrsb  291(%r8,%rax,4), %zmm22 {%k7}
+# INTEL: vmovrsb zmm22 {k7}, zmmword ptr [r8 + 4*rax + 291]
+0x62,0xc5,0x7f,0x4f,0x6f,0xb4,0x80,0x23,0x01,0x00,0x00
+
+# ATT:   vmovrsb  (%rip), %zmm22
+# INTEL: vmovrsb zmm22, zmmword ptr [rip]
+0x62,0xe5,0x7f,0x48,0x6f,0x35,0x00,0x00,0x00,0x00
+
+# ATT:   vmovrsb  -2048(,%rbp,2), %zmm22
+# INTEL: vmovrsb zmm22, zmmword ptr [2*rbp - 2048]
+0x62,0xe5,0x7f,0x48,0x6f,0x34,0x6d,0x00,0xf8,0xff,0xff
+
+# ATT:   vmovrsb  8128(%rcx), %zmm22 {%k7} {z}
+# INTEL: vmovrsb zmm22 {k7} {z}, zmmword ptr [rcx + 8128]
+0x62,0xe5,0x7f,0xcf,0x6f,0x71,0x7f
+
+# ATT:   vmovrsb  -8192(%rdx), %zmm22 {%k7} {z}
+# INTEL: vmovrsb zmm22 {k7} {z}, zmmword ptr [rdx - 8192]
+0x62,0xe5,0x7f,0xcf,0x6f,0x72,0x80
+
+# ATT:   vmovrsd  268435456(%rbp,%r14,8), %zmm22
+# INTEL: vmovrsd zmm22, zmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xa5,0x7e,0x48,0x6f,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   vmovrsd  291(%r8,%rax,4), %zmm22 {%k7}
+# INTEL: vmovrsd zmm22 {k7}, zmmword ptr [r8 + 4*rax + 291]
+0x62,0xc5,0x7e,0x4f,0x6f,0xb4,0x80,0x23,0x01,0x00,0x00
+
+# ATT:   vmovrsd  (%rip), %zmm22
+# INTEL: vmovrsd zmm22, zmmword ptr [rip]
+0x62,0xe5,0x7e,0x48,0x6f,0x35,0x00,0x00,0x00,0x00
+
+# ATT:   vmovrsd  -2048(,%rbp,2), %zmm22
+# INTEL: vmovrsd zmm22, zmmword ptr [2*rbp - 2048]
+0x62,0xe5,0x7e,0x48,0x6f,0x34,0x6d,0x00,0xf8,0xff,0xff
+
+# ATT:   vmovrsd  8128(%rcx), %zmm22 {%k7} {z}
+# INTEL: vmovrsd zmm22 {k7} {z}, zmmword ptr [rcx + 8128]
+0x62,0xe5,0x7e,0xcf,0x6f,0x71,0x7f
+
+# ATT:   vmovrsd  -8192(%rdx), %zmm22 {%k7} {z}
+# INTEL: vmovrsd zmm22 {k7} {z}, zmmword ptr [rdx - 8192]
+0x62,0xe5,0x7e,0xcf,0x6f,0x72,0x80
+
+# ATT:   vmovrsq  268435456(%rbp,%r14,8), %zmm22
+# INTEL: vmovrsq zmm22, zmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xa5,0xfe,0x48,0x6f,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   vmovrsq  291(%r8,%rax,4), %zmm22 {%k7}
+# INTEL: vmovrsq zmm22 {k7}, zmmword ptr [r8 + 4*rax + 291]
+0x62,0xc5,0xfe,0x4f,0x6f,0xb4,0x80,0x23,0x01,0x00,0x00
+
+# ATT:   vmovrsq  (%rip), %zmm22
+# INTEL: vmovrsq zmm22, zmmword ptr [rip]
+0x62,0xe5,0xfe,0x48,0x6f,0x35,0x00,0x00,0x00,0x00
+
+# ATT:   vmovrsq  -2048(,%rbp,2), %zmm22
+# INTEL: vmovrsq zmm22, zmmword ptr [2*rbp - 2048]
+0x62,0xe5,0xfe,0x48,0x6f,0x34,0x6d,0x00,0xf8,0xff,0xff
+
+# ATT:   vmovrsq  8128(%rcx), %zmm22 {%k7} {z}
+# INTEL: vmovrsq zmm22 {k7} {z}, zmmword ptr [rcx + 8128]
+0x62,0xe5,0xfe,0xcf,0x6f,0x71,0x7f
+
+# ATT:   vmovrsq  -8192(%rdx), %zmm22 {%k7} {z}
+# INTEL: vmovrsq zmm22 {k7} {z}, zmmword ptr [rdx - 8192]
+0x62,0xe5,0xfe,0xcf,0x6f,0x72,0x80
+
+# ATT:   vmovrsw  268435456(%rbp,%r14,8), %zmm22
+# INTEL: vmovrsw zmm22, zmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xa5,0xff,0x48,0x6f,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   vmovrsw  291(%r8,%rax,4), %zmm22 {%k7}
+# INTEL: vmovrsw zmm22 {k7}, zmmword ptr [r8 + 4*rax + 291]
+0x62,0xc5,0xff,0x4f,0x6f,0xb4,0x80,0x23,0x01,0x00,0x00
+
+# ATT:   vmovrsw  (%rip), %zmm22
+# INTEL: vmovrsw zmm22, zmmword ptr [rip]
+0x62,0xe5,0xff,0x48,0x6f,0x35,0x00,0x00,0x00,0x00
+
+# ATT:   vmovrsw  -2048(,%rbp,2), %zmm22
+# INTEL: vmovrsw zmm22, zmmword ptr [2*rbp - 2048]
+0x62,0xe5,0xff,0x48,0x6f,0x34,0x6d,0x00,0xf8,0xff,0xff
+
+# ATT:   vmovrsw  8128(%rcx), %zmm22 {%k7} {z}
+# INTEL: vmovrsw zmm22 {k7} {z}, zmmword ptr [rcx + 8128]
+0x62,0xe5,0xff,0xcf,0x6f,0x71,0x7f
+
+# ATT:   vmovrsw  -8192(%rdx), %zmm22 {%k7} {z}
+# INTEL: vmovrsw zmm22 {k7} {z}, zmmword ptr [rdx - 8192]
+0x62,0xe5,0xff,0xcf,0x6f,0x72,0x80
\ No newline at end of file
diff --git a/llvm/test/MC/X86/movrs-avx10-att-64.s b/llvm/test/MC/X86/movrs-avx10-att-64.s
new file mode 100644
index 000000000000..982b7a1d41c0
--- /dev/null
+++ b/llvm/test/MC/X86/movrs-avx10-att-64.s
@@ -0,0 +1,98 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding < %s  | FileCheck %s
+
+// CHECK: vmovrsb  268435456(%rbp,%r14,8), %zmm22
+// CHECK: encoding: [0x62,0xa5,0x7f,0x48,0x6f,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          vmovrsb  268435456(%rbp,%r14,8), %zmm22
+
+// CHECK: vmovrsb  291(%r8,%rax,4), %zmm22 {%k7}
+// CHECK: encoding: [0x62,0xc5,0x7f,0x4f,0x6f,0xb4,0x80,0x23,0x01,0x00,0x00]
+          vmovrsb  291(%r8,%rax,4), %zmm22 {%k7}
+
+// CHECK: vmovrsb  (%rip), %zmm22
+// CHECK: encoding: [0x62,0xe5,0x7f,0x48,0x6f,0x35,0x00,0x00,0x00,0x00]
+          vmovrsb  (%rip), %zmm22
+
+// CHECK: vmovrsb  -2048(,%rbp,2), %zmm22
+// CHECK: encoding: [0x62,0xe5,0x7f,0x48,0x6f,0x34,0x6d,0x00,0xf8,0xff,0xff]
+          vmovrsb  -2048(,%rbp,2), %zmm22
+
+// CHECK: vmovrsb  8128(%rcx), %zmm22 {%k7} {z}
+// CHECK: encoding: [0x62,0xe5,0x7f,0xcf,0x6f,0x71,0x7f]
+          vmovrsb  8128(%rcx), %zmm22 {%k7} {z}
+
+// CHECK: vmovrsb  -8192(%rdx), %zmm22 {%k7} {z}
+// CHECK: encoding: [0x62,0xe5,0x7f,0xcf,0x6f,0x72,0x80]
+          vmovrsb  -8192(%rdx), %zmm22 {%k7} {z}
+
+// CHECK: vmovrsd  268435456(%rbp,%r14,8), %zmm22
+// CHECK: encoding: [0x62,0xa5,0x7e,0x48,0x6f,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          vmovrsd  268435456(%rbp,%r14,8), %zmm22
+
+// CHECK: vmovrsd  291(%r8,%rax,4), %zmm22 {%k7}
+// CHECK: encoding: [0x62,0xc5,0x7e,0x4f,0x6f,0xb4,0x80,0x23,0x01,0x00,0x00]
+          vmovrsd  291(%r8,%rax,4), %zmm22 {%k7}
+
+// CHECK: vmovrsd  (%rip), %zmm22
+// CHECK: encoding: [0x62,0xe5,0x7e,0x48,0x6f,0x35,0x00,0x00,0x00,0x00]
+          vmovrsd  (%rip), %zmm22
+
+// CHECK: vmovrsd  -2048(,%rbp,2), %zmm22
+// CHECK: encoding: [0x62,0xe5,0x7e,0x48,0x6f,0x34,0x6d,0x00,0xf8,0xff,0xff]
+          vmovrsd  -2048(,%rbp,2), %zmm22
+
+// CHECK: vmovrsd  8128(%rcx), %zmm22 {%k7} {z}
+// CHECK: encoding: [0x62,0xe5,0x7e,0xcf,0x6f,0x71,0x7f]
+          vmovrsd  8128(%rcx), %zmm22 {%k7} {z}
+
+// CHECK: vmovrsd  -8192(%rdx), %zmm22 {%k7} {z}
+// CHECK: encoding: [0x62,0xe5,0x7e,0xcf,0x6f,0x72,0x80]
+          vmovrsd  -8192(%rdx), %zmm22 {%k7} {z}
+
+// CHECK: vmovrsq  268435456(%rbp,%r14,8), %zmm22
+// CHECK: encoding: [0x62,0xa5,0xfe,0x48,0x6f,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          vmovrsq  268435456(%rbp,%r14,8), %zmm22
+
+// CHECK: vmovrsq  291(%r8,%rax,4), %zmm22 {%k7}
+// CHECK: encoding: [0x62,0xc5,0xfe,0x4f,0x6f,0xb4,0x80,0x23,0x01,0x00,0x00]
+          vmovrsq  291(%r8,%rax,4), %zmm22 {%k7}
+
+// CHECK: vmovrsq  (%rip), %zmm22
+// CHECK: encoding: [0x62,0xe5,0xfe,0x48,0x6f,0x35,0x00,0x00,0x00,0x00]
+          vmovrsq  (%rip), %zmm22
+
+// CHECK: vmovrsq  -2048(,%rbp,2), %zmm22
+// CHECK: encoding: [0x62,0xe5,0xfe,0x48,0x6f,0x34,0x6d,0x00,0xf8,0xff,0xff]
+          vmovrsq  -2048(,%rbp,2), %zmm22
+
+// CHECK: vmovrsq  8128(%rcx), %zmm22 {%k7} {z}
+// CHECK: encoding: [0x62,0xe5,0xfe,0xcf,0x6f,0x71,0x7f]
+          vmovrsq  8128(%rcx), %zmm22 {%k7} {z}
+
+// CHECK: vmovrsq  -8192(%rdx), %zmm22 {%k7} {z}
+// CHECK: encoding: [0x62,0xe5,0xfe,0xcf,0x6f,0x72,0x80]
+          vmovrsq  -8192(%rdx), %zmm22 {%k7} {z}
+
+// CHECK: vmovrsw  268435456(%rbp,%r14,8), %zmm22
+// CHECK: encoding: [0x62,0xa5,0xff,0x48,0x6f,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          vmovrsw  268435456(%rbp,%r14,8), %zmm22
+
+// CHECK: vmovrsw  291(%r8,%rax,4), %zmm22 {%k7}
+// CHECK: encoding: [0x62,0xc5,0xff,0x4f,0x6f,0xb4,0x80,0x23,0x01,0x00,0x00]
+          vmovrsw  291(%r8,%rax,4), %zmm22 {%k7}
+
+// CHECK: vmovrsw  (%rip), %zmm22
+// CHECK: encoding: [0x62,0xe5,0xff,0x48,0x6f,0x35,0x00,0x00,0x00,0x00]
+          vmovrsw  (%rip), %zmm22
+
+// CHECK: vmovrsw  -2048(,%rbp,2), %zmm22
+// CHECK: encoding: [0x62,0xe5,0xff,0x48,0x6f,0x34,0x6d,0x00,0xf8,0xff,0xff]
+          vmovrsw  -2048(,%rbp,2), %zmm22
+
+// CHECK: vmovrsw  8128(%rcx), %zmm22 {%k7} {z}
+// CHECK: encoding: [0x62,0xe5,0xff,0xcf,0x6f,0x71,0x7f]
+          vmovrsw  8128(%rcx), %zmm22 {%k7} {z}
+
+// CHECK: vmovrsw  -8192(%rdx), %zmm22 {%k7} {z}
+// CHECK: encoding: [0x62,0xe5,0xff,0xcf,0x6f,0x72,0x80]
+          vmovrsw  -8192(%rdx), %zmm22 {%k7} {z}
+
diff --git a/llvm/test/MC/X86/movrs-avx10-intel-64.s b/llvm/test/MC/X86/movrs-avx10-intel-64.s
new file mode 100644
index 000000000000..d61e41abe7d6
--- /dev/null
+++ b/llvm/test/MC/X86/movrs-avx10-intel-64.s
@@ -0,0 +1,97 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
+
+// CHECK: vmovrsb zmm22, zmmword ptr [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0x62,0xa5,0x7f,0x48,0x6f,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          vmovrsb zmm22, zmmword ptr [rbp + 8*r14 + 268435456]
+
+// CHECK: vmovrsb zmm22 {k7}, zmmword ptr [r8 + 4*rax + 291]
+// CHECK: encoding: [0x62,0xc5,0x7f,0x4f,0x6f,0xb4,0x80,0x23,0x01,0x00,0x00]
+          vmovrsb zmm22 {k7}, zmmword ptr [r8 + 4*rax + 291]
+
+// CHECK: vmovrsb zmm22, zmmword ptr [rip]
+// CHECK: encoding: [0x62,0xe5,0x7f,0x48,0x6f,0x35,0x00,0x00,0x00,0x00]
+          vmovrsb zmm22, zmmword ptr [rip]
+
+// CHECK: vmovrsb zmm22, zmmword ptr [2*rbp - 2048]
+// CHECK: encoding: [0x62,0xe5,0x7f,0x48,0x6f,0x34,0x6d,0x00,0xf8,0xff,0xff]
+          vmovrsb zmm22, zmmword ptr [2*rbp - 2048]
+
+// CHECK: vmovrsb zmm22 {k7} {z}, zmmword ptr [rcx + 8128]
+// CHECK: encoding: [0x62,0xe5,0x7f,0xcf,0x6f,0x71,0x7f]
+          vmovrsb zmm22 {k7} {z}, zmmword ptr [rcx + 8128]
+
+// CHECK: vmovrsb zmm22 {k7} {z}, zmmword ptr [rdx - 8192]
+// CHECK: encoding: [0x62,0xe5,0x7f,0xcf,0x6f,0x72,0x80]
+          vmovrsb zmm22 {k7} {z}, zmmword ptr [rdx - 8192]
+
+// CHECK: vmovrsd zmm22, zmmword ptr [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0x62,0xa5,0x7e,0x48,0x6f,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          vmovrsd zmm22, zmmword ptr [rbp + 8*r14 + 268435456]
+
+// CHECK: vmovrsd zmm22 {k7}, zmmword ptr [r8 + 4*rax + 291]
+// CHECK: encoding: [0x62,0xc5,0x7e,0x4f,0x6f,0xb4,0x80,0x23,0x01,0x00,0x00]
+          vmovrsd zmm22 {k7}, zmmword ptr [r8 + 4*rax + 291]
+
+// CHECK: vmovrsd zmm22, zmmword ptr [rip]
+// CHECK: encoding: [0x62,0xe5,0x7e,0x48,0x6f,0x35,0x00,0x00,0x00,0x00]
+          vmovrsd zmm22, zmmword ptr [rip]
+
+// CHECK: vmovrsd zmm22, zmmword ptr [2*rbp - 2048]
+// CHECK: encoding: [0x62,0xe5,0x7e,0x48,0x6f,0x34,0x6d,0x00,0xf8,0xff,0xff]
+          vmovrsd zmm22, zmmword ptr [2*rbp - 2048]
+
+// CHECK: vmovrsd zmm22 {k7} {z}, zmmword ptr [rcx + 8128]
+// CHECK: encoding: [0x62,0xe5,0x7e,0xcf,0x6f,0x71,0x7f]
+          vmovrsd zmm22 {k7} {z}, zmmword ptr [rcx + 8128]
+
+// CHECK: vmovrsd zmm22 {k7} {z}, zmmword ptr [rdx - 8192]
+// CHECK: encoding: [0x62,0xe5,0x7e,0xcf,0x6f,0x72,0x80]
+          vmovrsd zmm22 {k7} {z}, zmmword ptr [rdx - 8192]
+
+// CHECK: vmovrsq zmm22, zmmword ptr [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0x62,0xa5,0xfe,0x48,0x6f,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          vmovrsq zmm22, zmmword ptr [rbp + 8*r14 + 268435456]
+
+// CHECK: vmovrsq zmm22 {k7}, zmmword ptr [r8 + 4*rax + 291]
+// CHECK: encoding: [0x62,0xc5,0xfe,0x4f,0x6f,0xb4,0x80,0x23,0x01,0x00,0x00]
+          vmovrsq zmm22 {k7}, zmmword ptr [r8 + 4*rax + 291]
+
+// CHECK: vmovrsq zmm22, zmmword ptr [rip]
+// CHECK: encoding: [0x62,0xe5,0xfe,0x48,0x6f,0x35,0x00,0x00,0x00,0x00]
+          vmovrsq zmm22, zmmword ptr [rip]
+
+// CHECK: vmovrsq zmm22, zmmword ptr [2*rbp - 2048]
+// CHECK: encoding: [0x62,0xe5,0xfe,0x48,0x6f,0x34,0x6d,0x00,0xf8,0xff,0xff]
+          vmovrsq zmm22, zmmword ptr [2*rbp - 2048]
+
+// CHECK: vmovrsq zmm22 {k7} {z}, zmmword ptr [rcx + 8128]
+// CHECK: encoding: [0x62,0xe5,0xfe,0xcf,0x6f,0x71,0x7f]
+          vmovrsq zmm22 {k7} {z}, zmmword ptr [rcx + 8128]
+
+// CHECK: vmovrsq zmm22 {k7} {z}, zmmword ptr [rdx - 8192]
+// CHECK: encoding: [0x62,0xe5,0xfe,0xcf,0x6f,0x72,0x80]
+          vmovrsq zmm22 {k7} {z}, zmmword ptr [rdx - 8192]
+
+// CHECK: vmovrsw zmm22, zmmword ptr [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0x62,0xa5,0xff,0x48,0x6f,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          vmovrsw zmm22, zmmword ptr [rbp + 8*r14 + 268435456]
+
+// CHECK: vmovrsw zmm22 {k7}, zmmword ptr [r8 + 4*rax + 291]
+// CHECK: encoding: [0x62,0xc5,0xff,0x4f,0x6f,0xb4,0x80,0x23,0x01,0x00,0x00]
+          vmovrsw zmm22 {k7}, zmmword ptr [r8 + 4*rax + 291]
+
+// CHECK: vmovrsw zmm22, zmmword ptr [rip]
+// CHECK: encoding: [0x62,0xe5,0xff,0x48,0x6f,0x35,0x00,0x00,0x00,0x00]
+          vmovrsw zmm22, zmmword ptr [rip]
+
+// CHECK: vmovrsw zmm22, zmmword ptr [2*rbp - 2048]
+// CHECK: encoding: [0x62,0xe5,0xff,0x48,0x6f,0x34,0x6d,0x00,0xf8,0xff,0xff]
+          vmovrsw zmm22, zmmword ptr [2*rbp - 2048]
+
+// CHECK: vmovrsw zmm22 {k7} {z}, zmmword ptr [rcx + 8128]
+// CHECK: encoding: [0x62,0xe5,0xff,0xcf,0x6f,0x71,0x7f]
+          vmovrsw zmm22 {k7} {z}, zmmword ptr [rcx + 8128]
+
+// CHECK: vmovrsw zmm22 {k7} {z}, zmmword ptr [rdx - 8192]
+// CHECK: encoding: [0x62,0xe5,0xff,0xcf,0x6f,0x72,0x80]
+          vmovrsw zmm22 {k7} {z}, zmmword ptr [rdx - 8192]
-- 
GitLab


From 08159e6e117f5717d30230cb6568d7f3686b2c86 Mon Sep 17 00:00:00 2001
From: "A. Jiang" <de34@live.cn>
Date: Fri, 25 Oct 2024 09:04:42 +0800
Subject: [PATCH 108/693] [libc++] __uglify `[[clang::noescape]]` (#113280)

Identifiers `clang` and `noescape` are not reserved by the C++ standard,
so perhaps we need to use the equivalent reserved forms.

Also changes the occurrences of that attribute to a macro, following the
convention for `[[_Clang::__lifetimebound__]]`.

Addresses
https://github.com/llvm/llvm-project/pull/91651#discussion_r1807852646.
---
 libcxx/include/__charconv/from_chars_floating_point.h | 6 +++---
 libcxx/include/__config                               | 6 ++++++
 libcxx/src/charconv.cpp                               | 6 +++---
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/libcxx/include/__charconv/from_chars_floating_point.h b/libcxx/include/__charconv/from_chars_floating_point.h
index 2860b0e8da83..5cd3fc4a41ea 100644
--- a/libcxx/include/__charconv/from_chars_floating_point.h
+++ b/libcxx/include/__charconv/from_chars_floating_point.h
@@ -37,13 +37,13 @@ struct __from_chars_result {
 
 template <class _Fp>
 _LIBCPP_EXPORTED_FROM_ABI __from_chars_result<_Fp> __from_chars_floating_point(
-    [[clang::noescape]] const char* __first, [[clang::noescape]] const char* __last, chars_format __fmt);
+    _LIBCPP_NOESCAPE const char* __first, _LIBCPP_NOESCAPE const char* __last, chars_format __fmt);
 
 extern template __from_chars_result<float> __from_chars_floating_point(
-    [[clang::noescape]] const char* __first, [[clang::noescape]] const char* __last, chars_format __fmt);
+    _LIBCPP_NOESCAPE const char* __first, _LIBCPP_NOESCAPE const char* __last, chars_format __fmt);
 
 extern template __from_chars_result<double> __from_chars_floating_point(
-    [[clang::noescape]] const char* __first, [[clang::noescape]] const char* __last, chars_format __fmt);
+    _LIBCPP_NOESCAPE const char* __first, _LIBCPP_NOESCAPE const char* __last, chars_format __fmt);
 
 template <class _Fp>
 _LIBCPP_HIDE_FROM_ABI from_chars_result
diff --git a/libcxx/include/__config b/libcxx/include/__config
index 4155fd33dabc..fc09a97274d7 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -1163,6 +1163,12 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_LIFETIMEBOUND
 #  endif
 
+#  if __has_cpp_attribute(_Clang::__noescape__)
+#    define _LIBCPP_NOESCAPE [[_Clang::__noescape__]]
+#  else
+#    define _LIBCPP_NOESCAPE
+#  endif
+
 #  if __has_attribute(__nodebug__)
 #    define _LIBCPP_NODEBUG __attribute__((__nodebug__))
 #  else
diff --git a/libcxx/src/charconv.cpp b/libcxx/src/charconv.cpp
index 3fe0afec0e28..5e8cb7d97703 100644
--- a/libcxx/src/charconv.cpp
+++ b/libcxx/src/charconv.cpp
@@ -77,13 +77,13 @@ to_chars_result to_chars(char* __first, char* __last, long double __value, chars
 
 template <class _Fp>
 __from_chars_result<_Fp> __from_chars_floating_point(
-    [[clang::noescape]] const char* __first, [[clang::noescape]] const char* __last, chars_format __fmt) {
+    _LIBCPP_NOESCAPE const char* __first, _LIBCPP_NOESCAPE const char* __last, chars_format __fmt) {
   return std::__from_chars_floating_point_impl<_Fp>(__first, __last, __fmt);
 }
 
 template __from_chars_result<float> __from_chars_floating_point(
-    [[clang::noescape]] const char* __first, [[clang::noescape]] const char* __last, chars_format __fmt);
+    _LIBCPP_NOESCAPE const char* __first, _LIBCPP_NOESCAPE const char* __last, chars_format __fmt);
 
 template __from_chars_result<double> __from_chars_floating_point(
-    [[clang::noescape]] const char* __first, [[clang::noescape]] const char* __last, chars_format __fmt);
+    _LIBCPP_NOESCAPE const char* __first, _LIBCPP_NOESCAPE const char* __last, chars_format __fmt);
 _LIBCPP_END_NAMESPACE_STD
-- 
GitLab


From c9b10a94b56411ba3b7cc3a148edb257b7e88fcd Mon Sep 17 00:00:00 2001
From: William Junda Huang <williamjhuang@google.com>
Date: Thu, 24 Oct 2024 21:05:45 -0400
Subject: [PATCH 109/693] [ThinLTO] Do not duplicate import a function that is
 actually defined in the current module #110064 (#111933)

Trying to land #110064  again after fixing test case
---
 llvm/lib/Linker/IRMover.cpp                   |  6 +-
 .../Inputs/ditemplatevalueparameter-remap.ll  | 29 ++++++
 .../X86/ditemplatevalueparameter-remap.ll     | 93 +++++++++++++++++++
 3 files changed, 127 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/ThinLTO/X86/Inputs/ditemplatevalueparameter-remap.ll
 create mode 100644 llvm/test/ThinLTO/X86/ditemplatevalueparameter-remap.ll

diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp
index 3a6c2678cd15..5067fbff2e27 100644
--- a/llvm/lib/Linker/IRMover.cpp
+++ b/llvm/lib/Linker/IRMover.cpp
@@ -595,11 +595,15 @@ Value *IRLinker::materialize(Value *V, bool ForIndirectSymbol) {
   if (!SGV)
     return nullptr;
 
+  // If SGV is from dest, it was already materialized when dest was loaded.
+  if (SGV->getParent() == &DstM)
+    return nullptr;
+
   // When linking a global from other modules than source & dest, skip
   // materializing it because it would be mapped later when its containing
   // module is linked. Linking it now would potentially pull in many types that
   // may not be mapped properly.
-  if (SGV->getParent() != &DstM && SGV->getParent() != SrcM.get())
+  if (SGV->getParent() != SrcM.get())
     return nullptr;
 
   Expected<Constant *> NewProto = linkGlobalValueProto(SGV, ForIndirectSymbol);
diff --git a/llvm/test/ThinLTO/X86/Inputs/ditemplatevalueparameter-remap.ll b/llvm/test/ThinLTO/X86/Inputs/ditemplatevalueparameter-remap.ll
new file mode 100644
index 000000000000..be93160b9433
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/ditemplatevalueparameter-remap.ll
@@ -0,0 +1,29 @@
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @_Z8thinlto1v() unnamed_addr {
+  %3 = alloca i64, align 4
+    #dbg_declare(ptr %3, !14, !DIExpression(), !15)
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "B.cpp", directory: ".")
+!2 = !{i32 7, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 8, !"PIC Level", i32 2}
+!10 = distinct !DISubprogram(name: "thinlto1", linkageName: "_Z8thinlto1v", scope: !11, file: !11, line: 8, type: !12, scopeLine: 8, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!11 = !DIFile(filename: "b.cpp", directory: ".")
+!12 = !DISubroutineType(types: !13)
+!13 = !{null}
+!14 = !DILocalVariable(name: "a", arg: 1, scope: !10, file: !11, line: 18, type: !16)
+!15 = !DILocation(line: 18, column: 19, scope: !10)
+!16 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "S<&func1>", file: !11, line: 2, size: 8, flags: DIFlagTypePassByValue, elements: !17, templateParams: !18, identifier: "_ZTS1SIXadL_Z5func1vEEE")
+!17 = !{}
+!18 = !{!19}
+!19 = !DITemplateValueParameter(name: "Func", type: !20, value: ptr undef)
+!20 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
diff --git a/llvm/test/ThinLTO/X86/ditemplatevalueparameter-remap.ll b/llvm/test/ThinLTO/X86/ditemplatevalueparameter-remap.ll
new file mode 100644
index 000000000000..a008485dfbfd
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/ditemplatevalueparameter-remap.ll
@@ -0,0 +1,93 @@
+; https://github.com/llvm/llvm-project/pull/110064
+; This test case checks if thinLTO correctly links metadata values in a specific
+; situation. Assume we are linking module B into module A, where an extern
+; function used in A is defined in B, but the function body has a
+; DITemplateValueParameter referring to another function back in A. The
+; compiler must check this other function is actually coming from A, thus
+; already materialized and does not require remapping. The IR here is modified
+; from the following source code.
+;
+; // A.h
+; template <void (*Func)()>
+; struct S {
+;   void Impl() {
+;     Func();
+;   }
+; };
+;
+; void func1();
+;
+; // A.cpp
+; #include "A.h"
+; __attribute__((weak)) void func1() {}
+; extern void thinlto1();
+; void bar() {
+;   S<func1> s; // Force instantiation of S<func1> in this compilation unit.
+;   s.Impl();
+;   thinlto1();
+; }
+;
+; // B.cpp
+; #include "A.h"
+; void thinlto1() {
+;   S<func1> s;
+; }
+;
+; RUN: opt -module-summary -o %t1.bc %s
+; RUN: opt -module-summary -o %t2.bc %S/Inputs/ditemplatevalueparameter-remap.ll
+; RUN: llvm-lto2 run %t1.bc %t2.bc -o %t3 -save-temps \
+; RUN:   -r=%t1.bc,_Z5func1v,p    \
+; RUN:   -r=%t1.bc,_Z3bazv,px     \
+; RUN:   -r=%t1.bc,_Z8thinlto1v,x \
+; RUN:   -r=%t1.bc,_Z3barv,px     \
+; RUN:   -r=%t2.bc,_Z8thinlto1v,px
+; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+$_Z5func1v = comdat any
+
+define linkonce_odr void @_Z5func1v() unnamed_addr !dbg !10 {
+  ret void
+}
+
+; Dummy function to use _Z5func1v so that it is not treated as dead symbol.
+define void @_Z3bazv() {
+  tail call void @_Z5func1v()
+  ret void
+}
+
+declare void @_Z8thinlto1v() unnamed_addr
+
+; Check _Z8thinlto1v is inlined after thinLTO.
+; CHECK: void @_Z3barv()
+; CHECK-NOT: @_Z8thinlto1v()
+; CHECK-NEXT: ret void
+define void @_Z3barv() unnamed_addr !dbg !14 {
+  tail call void @_Z8thinlto1v(), !dbg !25
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "A.cpp", directory: ".")
+!2 = !{i32 7, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 8, !"PIC Level", i32 2}
+!10 = distinct !DISubprogram(name: "func1", linkageName: "_Z5func1v", scope: !11, file: !11, line: 6, type: !12, scopeLine: 6, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!11 = !DIFile(filename: "a.h", directory: ".")
+!12 = !DISubroutineType(types: !13)
+!13 = !{null}
+!14 = distinct !DISubprogram(name: "bar", linkageName: "_Z3barv", scope: !11, file: !11, line: 15, type: !12, scopeLine: 15, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !16)
+!16 = !{!17}
+!17 = !DILocalVariable(name: "s", scope: !14, file: !11, line: 10, type: !18)
+!18 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "S<&func1>", file: !11, line: 2, size: 8, flags: DIFlagTypePassByValue, elements: !19, templateParams: !20, identifier: "_ZTS1SIXadL_Z5func1vEEE")
+!19 = !{}
+!20 = !{!21}
+!21 = !DITemplateValueParameter(name: "Func", type: !22, value: ptr @_Z5func1v)
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
+!25 = !DILocation(line: 16, column: 5, scope: !14)
-- 
GitLab


From a35ebcf255046bc78ea86b3c524e0836425a1156 Mon Sep 17 00:00:00 2001
From: Alexander Richardson <alexrichardson@google.com>
Date: Thu, 24 Oct 2024 18:11:35 -0700
Subject: [PATCH 110/693] [lld][WebAssembly] Update datalayout strings to
 latest version

These mismatched data layouts are exposed by the refactoring in
https://github.com/llvm/llvm-project/pull/105734 that now also
compares the non-integral pointers list of the data layout.

I dropped the explicit definition from all tests that use opt/llc since
those tools will insert the correct value, but for the llvm-as based
tests and explicit layout is required.

Reviewed By: MaskRay

Pull Request: https://github.com/llvm/llvm-project/pull/107276
---
 lld/test/wasm/lto/Inputs/archive.ll                 | 2 +-
 lld/test/wasm/lto/Inputs/cache.ll                   | 2 +-
 lld/test/wasm/lto/Inputs/comdat_ordering1.ll        | 2 +-
 lld/test/wasm/lto/Inputs/comdat_ordering2.ll        | 2 +-
 lld/test/wasm/lto/Inputs/foo.ll                     | 2 +-
 lld/test/wasm/lto/Inputs/libcall-archive.ll         | 2 +-
 lld/test/wasm/lto/Inputs/libcall-truncsfhf2.ll      | 2 +-
 lld/test/wasm/lto/Inputs/libcall.ll                 | 2 +-
 lld/test/wasm/lto/Inputs/save-temps.ll              | 2 +-
 lld/test/wasm/lto/Inputs/thin1.ll                   | 2 +-
 lld/test/wasm/lto/Inputs/thin2.ll                   | 2 +-
 lld/test/wasm/lto/Inputs/thinlto.ll                 | 2 +-
 lld/test/wasm/lto/Inputs/used.ll                    | 2 +-
 lld/test/wasm/lto/archive.ll                        | 2 +-
 lld/test/wasm/lto/atomics.ll                        | 2 +-
 lld/test/wasm/lto/cache-warnings.ll                 | 1 -
 lld/test/wasm/lto/cache.ll                          | 1 -
 lld/test/wasm/lto/cgo.ll                            | 2 +-
 lld/test/wasm/lto/comdat.ll                         | 2 +-
 lld/test/wasm/lto/diagnostics.ll                    | 2 +-
 lld/test/wasm/lto/export.ll                         | 2 +-
 lld/test/wasm/lto/import-attributes.ll              | 2 +-
 lld/test/wasm/lto/internalize-basic.ll              | 2 +-
 lld/test/wasm/lto/libcall-archive.ll                | 2 +-
 lld/test/wasm/lto/libcall-truncsfhf2.ll             | 2 +-
 lld/test/wasm/lto/lto-start.ll                      | 2 +-
 lld/test/wasm/lto/new-pass-manager.ll               | 2 +-
 lld/test/wasm/lto/opt-level.ll                      | 2 +-
 lld/test/wasm/lto/parallel.ll                       | 2 +-
 lld/test/wasm/lto/relocatable-undefined.ll          | 2 +-
 lld/test/wasm/lto/relocatable.ll                    | 2 +-
 lld/test/wasm/lto/save-temps.ll                     | 2 +-
 lld/test/wasm/lto/thin-archivecollision.ll          | 1 -
 lld/test/wasm/lto/thinlto-thin-archive-collision.ll | 1 -
 lld/test/wasm/lto/thinlto.ll                        | 1 -
 lld/test/wasm/lto/undef.ll                          | 2 +-
 lld/test/wasm/lto/used.ll                           | 1 -
 lld/test/wasm/lto/verify-invalid.ll                 | 2 +-
 lld/test/wasm/lto/weak-undefined.ll                 | 2 +-
 lld/test/wasm/lto/weak.ll                           | 2 +-
 40 files changed, 34 insertions(+), 40 deletions(-)

diff --git a/lld/test/wasm/lto/Inputs/archive.ll b/lld/test/wasm/lto/Inputs/archive.ll
index 5f4e3c00e0f0..9e174c36136a 100644
--- a/lld/test/wasm/lto/Inputs/archive.ll
+++ b/lld/test/wasm/lto/Inputs/archive.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define void @f() {
diff --git a/lld/test/wasm/lto/Inputs/cache.ll b/lld/test/wasm/lto/Inputs/cache.ll
index 914654ef5f82..ac76649dc85b 100644
--- a/lld/test/wasm/lto/Inputs/cache.ll
+++ b/lld/test/wasm/lto/Inputs/cache.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define i32 @_start() {
diff --git a/lld/test/wasm/lto/Inputs/comdat_ordering1.ll b/lld/test/wasm/lto/Inputs/comdat_ordering1.ll
index b866c6efeba1..d8da715d619c 100644
--- a/lld/test/wasm/lto/Inputs/comdat_ordering1.ll
+++ b/lld/test/wasm/lto/Inputs/comdat_ordering1.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 ; Generated from this C++ code and simplified manually:
diff --git a/lld/test/wasm/lto/Inputs/comdat_ordering2.ll b/lld/test/wasm/lto/Inputs/comdat_ordering2.ll
index 58ab5122bad8..1baa2ebc16de 100644
--- a/lld/test/wasm/lto/Inputs/comdat_ordering2.ll
+++ b/lld/test/wasm/lto/Inputs/comdat_ordering2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 ; Generated from this C++ code and simplified manually:
diff --git a/lld/test/wasm/lto/Inputs/foo.ll b/lld/test/wasm/lto/Inputs/foo.ll
index 5fd7651b49cb..f3b54cc7d009 100644
--- a/lld/test/wasm/lto/Inputs/foo.ll
+++ b/lld/test/wasm/lto/Inputs/foo.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define void @foo() local_unnamed_addr {
diff --git a/lld/test/wasm/lto/Inputs/libcall-archive.ll b/lld/test/wasm/lto/Inputs/libcall-archive.ll
index def1452bdf35..9d05efdeae08 100644
--- a/lld/test/wasm/lto/Inputs/libcall-archive.ll
+++ b/lld/test/wasm/lto/Inputs/libcall-archive.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define void @memcpy() {
diff --git a/lld/test/wasm/lto/Inputs/libcall-truncsfhf2.ll b/lld/test/wasm/lto/Inputs/libcall-truncsfhf2.ll
index b8609af28795..1439d7f8b4cb 100644
--- a/lld/test/wasm/lto/Inputs/libcall-truncsfhf2.ll
+++ b/lld/test/wasm/lto/Inputs/libcall-truncsfhf2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define half @__truncsfhf2(float) {
diff --git a/lld/test/wasm/lto/Inputs/libcall.ll b/lld/test/wasm/lto/Inputs/libcall.ll
index 23eb1f756284..182d847bf1f0 100644
--- a/lld/test/wasm/lto/Inputs/libcall.ll
+++ b/lld/test/wasm/lto/Inputs/libcall.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 ; This function, when compiled will generate a new undefined reference to
diff --git a/lld/test/wasm/lto/Inputs/save-temps.ll b/lld/test/wasm/lto/Inputs/save-temps.ll
index b25b8d92441f..e709ef8cc026 100644
--- a/lld/test/wasm/lto/Inputs/save-temps.ll
+++ b/lld/test/wasm/lto/Inputs/save-temps.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define void @bar() {
diff --git a/lld/test/wasm/lto/Inputs/thin1.ll b/lld/test/wasm/lto/Inputs/thin1.ll
index 04fa091ad288..653b1f3067af 100644
--- a/lld/test/wasm/lto/Inputs/thin1.ll
+++ b/lld/test/wasm/lto/Inputs/thin1.ll
@@ -1,6 +1,6 @@
 ; Copied from lld/test/ELF/lto/Inputs/thin1.ll
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define i32 @foo(i32 %goo) {
diff --git a/lld/test/wasm/lto/Inputs/thin2.ll b/lld/test/wasm/lto/Inputs/thin2.ll
index 236781aed3de..9b331c16a9cb 100644
--- a/lld/test/wasm/lto/Inputs/thin2.ll
+++ b/lld/test/wasm/lto/Inputs/thin2.ll
@@ -1,6 +1,6 @@
 ; Copied from lld/test/ELF/lto/Inputs/thin2.ll
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define i32 @blah(i32 %meh) #0 {
diff --git a/lld/test/wasm/lto/Inputs/thinlto.ll b/lld/test/wasm/lto/Inputs/thinlto.ll
index 2c4fe7177f56..a13f9c5ca56f 100644
--- a/lld/test/wasm/lto/Inputs/thinlto.ll
+++ b/lld/test/wasm/lto/Inputs/thinlto.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define void @g() {
diff --git a/lld/test/wasm/lto/Inputs/used.ll b/lld/test/wasm/lto/Inputs/used.ll
index 93dd4acc5039..840776d6ec2f 100644
--- a/lld/test/wasm/lto/Inputs/used.ll
+++ b/lld/test/wasm/lto/Inputs/used.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 @foo = hidden global i32 1
diff --git a/lld/test/wasm/lto/archive.ll b/lld/test/wasm/lto/archive.ll
index b5c9e459eb55..713c53a58f39 100644
--- a/lld/test/wasm/lto/archive.ll
+++ b/lld/test/wasm/lto/archive.ll
@@ -5,7 +5,7 @@
 ; RUN: wasm-ld --export-dynamic %t2.o %t.a -o %t3
 ; RUN: obj2yaml %t3 | FileCheck %s
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define void @_start() {
diff --git a/lld/test/wasm/lto/atomics.ll b/lld/test/wasm/lto/atomics.ll
index 9378519e0999..c7712f7b82d6 100644
--- a/lld/test/wasm/lto/atomics.ll
+++ b/lld/test/wasm/lto/atomics.ll
@@ -4,7 +4,7 @@
 ; Atomic operations will not fail to compile if atomics are not
 ; enabled because LLVM atomics will be lowered to regular ops.
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown-wasm"
 
 @foo = hidden global i32 1
diff --git a/lld/test/wasm/lto/cache-warnings.ll b/lld/test/wasm/lto/cache-warnings.ll
index 575e3a8b2ba4..9175fc97a075 100644
--- a/lld/test/wasm/lto/cache-warnings.ll
+++ b/lld/test/wasm/lto/cache-warnings.ll
@@ -35,7 +35,6 @@
 ; SIZE: warning: ThinLTO cache pruning happens since the total size of{{.*}}--thinlto-cache-policy
 ; WARN-NOT: warning: ThinLTO cache pruning happens{{.*}}--thinlto-cache-policy
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown-wasm"
 
 define void @globalfunc() #0 {
diff --git a/lld/test/wasm/lto/cache.ll b/lld/test/wasm/lto/cache.ll
index 570b38948f66..9ddf27630f1d 100644
--- a/lld/test/wasm/lto/cache.ll
+++ b/lld/test/wasm/lto/cache.ll
@@ -36,7 +36,6 @@
 ; RUN: wasm-ld --thinlto-cache-dir=%t.cache --thinlto-cache-policy prune_after=0s:cache_size=0%:cache_size_files=1:prune_interval=0s -o %t.wasm %t2.o %t.o
 ; RUN: ls %t.cache | count 3
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown-wasm"
 
 define void @globalfunc() #0 {
diff --git a/lld/test/wasm/lto/cgo.ll b/lld/test/wasm/lto/cgo.ll
index c37ae6e8bcf3..007e379e0632 100644
--- a/lld/test/wasm/lto/cgo.ll
+++ b/lld/test/wasm/lto/cgo.ll
@@ -19,7 +19,7 @@
 ; ERROR-O4: wasm-ld: error: invalid optimization level for LTO: 4
 ; ERROR-CGO4: wasm-ld: error: invalid codegen optimization level for LTO: 4
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown-wasm"
 
 define void @_start() {
diff --git a/lld/test/wasm/lto/comdat.ll b/lld/test/wasm/lto/comdat.ll
index d9f414a4dac0..634bcf84dde1 100644
--- a/lld/test/wasm/lto/comdat.ll
+++ b/lld/test/wasm/lto/comdat.ll
@@ -6,7 +6,7 @@
 ; RUN: wasm-ld %t.bc %t.o -o %t.wasm
 ; RUN: wasm-ld %t.o %t.bc -o %t.wasm
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 $foo = comdat any
diff --git a/lld/test/wasm/lto/diagnostics.ll b/lld/test/wasm/lto/diagnostics.ll
index 2f1daa99d848..e075a3219926 100644
--- a/lld/test/wasm/lto/diagnostics.ll
+++ b/lld/test/wasm/lto/diagnostics.ll
@@ -4,7 +4,7 @@
 ; RUN: llvm-as %s -o %t.o
 ; RUN: not wasm-ld --lto-O0 %t.o -o %t2 2>&1 | FileCheck %s
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define void @_start() {
diff --git a/lld/test/wasm/lto/export.ll b/lld/test/wasm/lto/export.ll
index ff5af8ccbbd3..e2e84f4056af 100644
--- a/lld/test/wasm/lto/export.ll
+++ b/lld/test/wasm/lto/export.ll
@@ -3,7 +3,7 @@
 ; RUN: wasm-ld --export=hidden_function -o %t.wasm %t.bc
 ; RUN: obj2yaml %t.wasm | FileCheck %s
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define hidden i32 @hidden_function() local_unnamed_addr {
diff --git a/lld/test/wasm/lto/import-attributes.ll b/lld/test/wasm/lto/import-attributes.ll
index 72198fa8adb4..c1398f975cb5 100644
--- a/lld/test/wasm/lto/import-attributes.ll
+++ b/lld/test/wasm/lto/import-attributes.ll
@@ -3,7 +3,7 @@
 ; RUN: obj2yaml %t.wasm | FileCheck %s
 
 target triple = "wasm32-unknown-unknown-wasm"
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 
 define void @_start() {
   call void @foo();
diff --git a/lld/test/wasm/lto/internalize-basic.ll b/lld/test/wasm/lto/internalize-basic.ll
index 95a1fa3e055c..e44c8a5f66de 100644
--- a/lld/test/wasm/lto/internalize-basic.ll
+++ b/lld/test/wasm/lto/internalize-basic.ll
@@ -2,7 +2,7 @@
 ; RUN: wasm-ld %t.o -o %t2 -save-temps
 ; RUN: llvm-dis < %t2.0.2.internalize.bc | FileCheck %s
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown-wasm"
 
 define void @_start() {
diff --git a/lld/test/wasm/lto/libcall-archive.ll b/lld/test/wasm/lto/libcall-archive.ll
index 365ce180f144..2f785b98976e 100644
--- a/lld/test/wasm/lto/libcall-archive.ll
+++ b/lld/test/wasm/lto/libcall-archive.ll
@@ -5,7 +5,7 @@
 ; RUN: wasm-ld -o %t %t.o %t.a
 ; RUN: obj2yaml %t | FileCheck %s
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define void @_start(ptr %a, ptr %b) {
diff --git a/lld/test/wasm/lto/libcall-truncsfhf2.ll b/lld/test/wasm/lto/libcall-truncsfhf2.ll
index 08b334ace9ba..fd07bb53890f 100644
--- a/lld/test/wasm/lto/libcall-truncsfhf2.ll
+++ b/lld/test/wasm/lto/libcall-truncsfhf2.ll
@@ -4,7 +4,7 @@
 ; RUN: llvm-ar rcs %t.a %t.truncsfhf2.o
 ; RUN: not wasm-ld --export-all %t.o %t.a -o %t.wasm 2>&1 | FileCheck %s
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 @g_float = global float 0.0
diff --git a/lld/test/wasm/lto/lto-start.ll b/lld/test/wasm/lto/lto-start.ll
index ebbfd8d53e4e..4af43103e53d 100644
--- a/lld/test/wasm/lto/lto-start.ll
+++ b/lld/test/wasm/lto/lto-start.ll
@@ -8,7 +8,7 @@
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Name:            _start
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown-wasm"
 
 define void @_start() {
diff --git a/lld/test/wasm/lto/new-pass-manager.ll b/lld/test/wasm/lto/new-pass-manager.ll
index 377b31674ccc..7fc70541ecec 100644
--- a/lld/test/wasm/lto/new-pass-manager.ll
+++ b/lld/test/wasm/lto/new-pass-manager.ll
@@ -3,7 +3,7 @@
 
 ; CHECK: Running pass: GlobalOptPass
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define void @_start() local_unnamed_addr {
diff --git a/lld/test/wasm/lto/opt-level.ll b/lld/test/wasm/lto/opt-level.ll
index 6556f2ef02b5..b2698fffdabd 100644
--- a/lld/test/wasm/lto/opt-level.ll
+++ b/lld/test/wasm/lto/opt-level.ll
@@ -15,7 +15,7 @@
 ; RUN:   FileCheck --check-prefix=INVALIDNEGATIVE %s
 ; INVALIDNEGATIVE: invalid optimization level for LTO: 4294967295
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown-wasm"
 
 ; CHECK-O0: Name: foo
diff --git a/lld/test/wasm/lto/parallel.ll b/lld/test/wasm/lto/parallel.ll
index e4ea35f60d89..a9e8b653d11a 100644
--- a/lld/test/wasm/lto/parallel.ll
+++ b/lld/test/wasm/lto/parallel.ll
@@ -4,7 +4,7 @@
 ; RUN: llvm-nm %t.lto.o | FileCheck --check-prefix=CHECK0 %s
 ; RUN: llvm-nm %t1.lto.o | FileCheck --check-prefix=CHECK1 %s
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown-wasm"
 
 ; CHECK0-NOT: bar
diff --git a/lld/test/wasm/lto/relocatable-undefined.ll b/lld/test/wasm/lto/relocatable-undefined.ll
index e8145cf39810..37ec50e13196 100644
--- a/lld/test/wasm/lto/relocatable-undefined.ll
+++ b/lld/test/wasm/lto/relocatable-undefined.ll
@@ -2,7 +2,7 @@
 ; RUN: wasm-ld -r -o %t.wasm %t.o
 ; RUN: obj2yaml %t.wasm | FileCheck %s
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 @missing_data = external global i32
diff --git a/lld/test/wasm/lto/relocatable.ll b/lld/test/wasm/lto/relocatable.ll
index a127db3663b3..5f3010724dcd 100644
--- a/lld/test/wasm/lto/relocatable.ll
+++ b/lld/test/wasm/lto/relocatable.ll
@@ -12,7 +12,7 @@
 ; CHECK-NEXT:   }
 ; CHECK-NEXT: ]
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define void @foo() {
diff --git a/lld/test/wasm/lto/save-temps.ll b/lld/test/wasm/lto/save-temps.ll
index bcc420178803..773978ef01f8 100644
--- a/lld/test/wasm/lto/save-temps.ll
+++ b/lld/test/wasm/lto/save-temps.ll
@@ -8,7 +8,7 @@
 ; RUN: llvm-nm a.out.lto.o | FileCheck %s
 ; RUN: llvm-dis a.out.0.0.preopt.bc
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define void @foo() {
diff --git a/lld/test/wasm/lto/thin-archivecollision.ll b/lld/test/wasm/lto/thin-archivecollision.ll
index 5b3a423de402..c3d133e2a713 100644
--- a/lld/test/wasm/lto/thin-archivecollision.ll
+++ b/lld/test/wasm/lto/thin-archivecollision.ll
@@ -18,7 +18,6 @@
 ; CHECK: Name:            foo
 ; CHECK: Name:            blah
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
 define i32 @_start() #0 {
diff --git a/lld/test/wasm/lto/thinlto-thin-archive-collision.ll b/lld/test/wasm/lto/thinlto-thin-archive-collision.ll
index d1000b371136..fb90b75eee2c 100644
--- a/lld/test/wasm/lto/thinlto-thin-archive-collision.ll
+++ b/lld/test/wasm/lto/thinlto-thin-archive-collision.ll
@@ -15,7 +15,6 @@
 ; CHECK: thinlto-archives/thin.a(thin.o at {{[1-9][0-9]+}})
 ; CHECK-NEXT: -r=thinlto-archives/thin.a(thin.o at {{[1-9][0-9]+}}),blah,p
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
 declare i32 @blah(i32 %meh)
diff --git a/lld/test/wasm/lto/thinlto.ll b/lld/test/wasm/lto/thinlto.ll
index 169b827fe308..c7dd73adcd4b 100644
--- a/lld/test/wasm/lto/thinlto.ll
+++ b/lld/test/wasm/lto/thinlto.ll
@@ -52,7 +52,6 @@
 ; NM1: T f
 ; NM2: T g
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
 declare void @g(...)
diff --git a/lld/test/wasm/lto/undef.ll b/lld/test/wasm/lto/undef.ll
index cb9e7fc39e93..0cadd75b78e4 100644
--- a/lld/test/wasm/lto/undef.ll
+++ b/lld/test/wasm/lto/undef.ll
@@ -2,7 +2,7 @@
 ; RUN: wasm-ld %t.o -o %t.wasm --allow-undefined
 ; RUN: obj2yaml %t.wasm | FileCheck %s
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 declare i32 @bar()
diff --git a/lld/test/wasm/lto/used.ll b/lld/test/wasm/lto/used.ll
index 414e72d6b9a4..a1851035fa24 100644
--- a/lld/test/wasm/lto/used.ll
+++ b/lld/test/wasm/lto/used.ll
@@ -5,7 +5,6 @@
 
 ; Verify that symbols references from regular objects are preserved by LTO
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
 declare void @bar()
diff --git a/lld/test/wasm/lto/verify-invalid.ll b/lld/test/wasm/lto/verify-invalid.ll
index 67ed5b5498f4..ee3a8142bc8a 100644
--- a/lld/test/wasm/lto/verify-invalid.ll
+++ b/lld/test/wasm/lto/verify-invalid.ll
@@ -4,7 +4,7 @@
 ; RUN: wasm-ld %t.o -o %t2 --lto-debug-pass-manager \
 ; RUN:   -disable-verify 2>&1 | FileCheck -check-prefix=DISABLE-NPM %s
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 define void @_start() {
diff --git a/lld/test/wasm/lto/weak-undefined.ll b/lld/test/wasm/lto/weak-undefined.ll
index 9a42c2bb5dbf..39730425270d 100644
--- a/lld/test/wasm/lto/weak-undefined.ll
+++ b/lld/test/wasm/lto/weak-undefined.ll
@@ -6,7 +6,7 @@
 ; We had a bug where stub function generation was failing because functions
 ; that are in bitcode (pre-LTO) don't have signatures assigned.
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
 declare extern_weak i32 @foo()
diff --git a/lld/test/wasm/lto/weak.ll b/lld/test/wasm/lto/weak.ll
index 6b83400b8e8a..84bcba91882f 100644
--- a/lld/test/wasm/lto/weak.ll
+++ b/lld/test/wasm/lto/weak.ll
@@ -2,7 +2,7 @@
 ; RUN: wasm-ld %t.o %t.o -o %t.wasm -r
 ; RUN: llvm-readobj --symbols %t.wasm | FileCheck %s
 
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown-wasm"
 
 define weak void @f() {
-- 
GitLab


From efd8938d575d1f8058bfe220e4c672d969c82be0 Mon Sep 17 00:00:00 2001
From: kakkoko <parenthesized.ko@gmail.com>
Date: Fri, 25 Oct 2024 10:13:08 +0900
Subject: [PATCH 111/693] [clang-format] Fix working -assume-filename with
 .clang-format-ignore (#113100)

Fixes #113099.

---------

Co-authored-by: Owen Pan <owenpiano@gmail.com>
---
 clang/test/Format/clang-format-ignore.cpp | 10 ++++++++++
 clang/tools/clang-format/ClangFormat.cpp  |  5 ++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/clang/test/Format/clang-format-ignore.cpp b/clang/test/Format/clang-format-ignore.cpp
index fb49fa9dd52c..67d68aebde5d 100644
--- a/clang/test/Format/clang-format-ignore.cpp
+++ b/clang/test/Format/clang-format-ignore.cpp
@@ -46,5 +46,15 @@
 // CHECK5-NEXT: {{Formatting \[4/5] .*foo\.c}}
 // CHECK5-NOT: foo.js
 
+// RUN: echo "foo.*" > .clang-format-ignore
+// RUN: echo "int i ;" > foo.c
+// RUN: clang-format -assume-filename=foo.c < foo.c \
+// RUN:   | FileCheck %s -check-prefix=CHECK6 -allow-empty
+// CHECK6-NOT: int
+
+// RUN: clang-format -assume-filename=bar.c < foo.c \
+// RUN:   | FileCheck %s -check-prefix=CHECK7 -match-full-lines
+// CHECK7: int i;
+
 // RUN: cd ..
 // RUN: rm -r %t.dir
diff --git a/clang/tools/clang-format/ClangFormat.cpp b/clang/tools/clang-format/ClangFormat.cpp
index 108db7204aa6..96fb85e99bf5 100644
--- a/clang/tools/clang-format/ClangFormat.cpp
+++ b/clang/tools/clang-format/ClangFormat.cpp
@@ -707,8 +707,11 @@ int main(int argc, const char **argv) {
     errs() << "Clang-formatting " << LineNo << " files\n";
   }
 
-  if (FileNames.empty())
+  if (FileNames.empty()) {
+    if (isIgnored(AssumeFileName))
+      return 0;
     return clang::format::format("-", FailOnIncompleteFormat);
+  }
 
   if (FileNames.size() > 1 &&
       (!Offsets.empty() || !Lengths.empty() || !LineRanges.empty())) {
-- 
GitLab


From 08916cef7e7ae334a04769126c2fefc144f8a5a0 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Thu, 24 Oct 2024 18:28:44 -0700
Subject: [PATCH 112/693] [BOLT] Set RawBranchCount in DataAggregator

Align DataAggregator (Linux perf and pre-aggregated profile reader) to
DataReader (fdata profile reader) behavior: set BF->RawBranchCount which
is used in profile density computation (#101094).

Reviewers: ayermolo, maksfb, dcci, rafaelauler, WenleiHe

Reviewed By: WenleiHe

Pull Request: https://github.com/llvm/llvm-project/pull/101093
---
 bolt/lib/Profile/DataAggregator.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index 0a63148379d9..2e83ca03dbde 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -638,8 +638,12 @@ void DataAggregator::processProfile(BinaryContext &BC) {
                                             : BinaryFunction::PF_LBR;
   for (auto &BFI : BC.getBinaryFunctions()) {
     BinaryFunction &BF = BFI.second;
-    if (getBranchData(BF) || getFuncSampleData(BF.getNames()))
+    FuncBranchData *FBD = getBranchData(BF);
+    if (FBD || getFuncSampleData(BF.getNames())) {
       BF.markProfiled(Flags);
+      if (FBD)
+        BF.RawBranchCount = FBD->getNumExecutedBranches();
+    }
   }
 
   for (auto &FuncBranches : NamesToBranches)
-- 
GitLab


From 6ee5ff95abe6ba6dfc4ca3a07a27b796fbf3664e Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Thu, 24 Oct 2024 18:30:59 -0700
Subject: [PATCH 113/693] [BOLT] Add profile density computation

Reuse the definition of profile density from llvm-profgen (#92144):
- the density is computed in perf2bolt using raw samples (perf.data or
  pre-aggregated data),
- function density is the ratio of dynamically executed function bytes
  to the static function size in bytes,
- profile density:
  - functions are sorted by density in decreasing order, accumulating
    their respective sample counts,
  - profile density is the smallest density covering 99% of total sample
    count.

In other words, BOLT binary profile density is the minimum amount of
profile information per function (excluding functions in tail 1% sample
count) which is sufficient to optimize the binary well.

The density threshold of 60 was determined through experiments with
large binaries by reducing the sample count and checking resulting
profile density and performance. The threshold is conservative.

perf2bolt would print the warning if the density is below the threshold
and suggest to increase the sampling duration and/or frequency to reach
a given density, e.g.:
```
BOLT-WARNING: BOLT is estimated to optimize better with 2.8x more samples.
```

Test Plan: updated pre-aggregated-perf.test

Reviewers: maksfb, wlei-llvm, rafaelauler, ayermolo, dcci, WenleiHe

Reviewed By: WenleiHe, wlei-llvm

Pull Request: https://github.com/llvm/llvm-project/pull/101094
---
 bolt/include/bolt/Core/BinaryFunction.h   |  6 ++
 bolt/include/bolt/Utils/CommandLineOpts.h |  1 +
 bolt/lib/Passes/BinaryPasses.cpp          | 74 +++++++++++++++++++++++
 bolt/lib/Profile/DataAggregator.cpp       |  9 ++-
 bolt/lib/Utils/CommandLineOpts.cpp        |  4 ++
 bolt/test/X86/pre-aggregated-perf.test    | 14 ++++-
 bolt/tools/driver/llvm-bolt.cpp           |  1 +
 7 files changed, 106 insertions(+), 3 deletions(-)

diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index fc0375b75533..0b875ed68810 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -386,6 +386,9 @@ private:
   /// Raw branch count for this function in the profile.
   uint64_t RawBranchCount{0};
 
+  /// Dynamically executed function bytes, used for density computation.
+  uint64_t SampleCountInBytes{0};
+
   /// Indicates the type of profile the function is using.
   uint16_t ProfileFlags{PF_NONE};
 
@@ -1844,6 +1847,9 @@ public:
   /// to this function.
   void setRawBranchCount(uint64_t Count) { RawBranchCount = Count; }
 
+  /// Return the number of dynamically executed bytes, from raw perf data.
+  uint64_t getSampleCountInBytes() const { return SampleCountInBytes; }
+
   /// Return the execution count for functions with known profile.
   /// Return 0 if the function has no profile.
   uint64_t getKnownExecutionCount() const {
diff --git a/bolt/include/bolt/Utils/CommandLineOpts.h b/bolt/include/bolt/Utils/CommandLineOpts.h
index baabeab577fb..04bf7db5de95 100644
--- a/bolt/include/bolt/Utils/CommandLineOpts.h
+++ b/bolt/include/bolt/Utils/CommandLineOpts.h
@@ -55,6 +55,7 @@ extern llvm::cl::opt<bool> PrintSections;
 enum ProfileFormatKind { PF_Fdata, PF_YAML };
 
 extern llvm::cl::opt<ProfileFormatKind> ProfileFormat;
+extern llvm::cl::opt<bool> ShowDensity;
 extern llvm::cl::opt<bool> SplitEH;
 extern llvm::cl::opt<bool> StrictMode;
 extern llvm::cl::opt<bool> TimeOpts;
diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp
index fa95ad7324ac..5a676185227e 100644
--- a/bolt/lib/Passes/BinaryPasses.cpp
+++ b/bolt/lib/Passes/BinaryPasses.cpp
@@ -15,6 +15,7 @@
 #include "bolt/Core/ParallelUtilities.h"
 #include "bolt/Passes/ReorderAlgorithm.h"
 #include "bolt/Passes/ReorderFunctions.h"
+#include "bolt/Utils/CommandLineOpts.h"
 #include "llvm/Support/CommandLine.h"
 #include <atomic>
 #include <mutex>
@@ -223,6 +224,18 @@ static cl::opt<unsigned> TopCalledLimit(
              "functions section"),
     cl::init(100), cl::Hidden, cl::cat(BoltCategory));
 
+// Profile density options, synced with llvm-profgen/ProfileGenerator.cpp
+static cl::opt<int> ProfileDensityCutOffHot(
+    "profile-density-cutoff-hot", cl::init(990000),
+    cl::desc("Total samples cutoff for functions used to calculate "
+             "profile density."));
+
+static cl::opt<double> ProfileDensityThreshold(
+    "profile-density-threshold", cl::init(60),
+    cl::desc("If the profile density is below the given threshold, it "
+             "will be suggested to increase the sampling rate."),
+    cl::Optional);
+
 } // namespace opts
 
 namespace llvm {
@@ -1383,6 +1396,7 @@ Error PrintProgramStats::runOnFunctions(BinaryContext &BC) {
   uint64_t StaleSampleCount = 0;
   uint64_t InferredSampleCount = 0;
   std::vector<const BinaryFunction *> ProfiledFunctions;
+  std::vector<std::pair<double, uint64_t>> FuncDensityList;
   const char *StaleFuncsHeader = "BOLT-INFO: Functions with stale profile:\n";
   for (auto &BFI : BC.getBinaryFunctions()) {
     const BinaryFunction &Function = BFI.second;
@@ -1441,6 +1455,22 @@ Error PrintProgramStats::runOnFunctions(BinaryContext &BC) {
       StaleSampleCount += SampleCount;
       ++NumAllStaleFunctions;
     }
+
+    if (opts::ShowDensity) {
+      uint64_t Size = Function.getSize();
+      // In case of BOLT split functions registered in BAT, executed traces are
+      // automatically attributed to the main fragment. Add up function sizes
+      // for all fragments.
+      if (IsHotParentOfBOLTSplitFunction)
+        for (const BinaryFunction *Fragment : Function.getFragments())
+          Size += Fragment->getSize();
+      double Density = (double)1.0 * Function.getSampleCountInBytes() / Size;
+      FuncDensityList.emplace_back(Density, SampleCount);
+      LLVM_DEBUG(BC.outs() << Function << ": executed bytes "
+                           << Function.getSampleCountInBytes() << ", size (b) "
+                           << Size << ", density " << Density
+                           << ", sample count " << SampleCount << '\n');
+    }
   }
   BC.NumProfiledFuncs = ProfiledFunctions.size();
   BC.NumStaleProfileFuncs = NumStaleProfileFunctions;
@@ -1684,6 +1714,50 @@ Error PrintProgramStats::runOnFunctions(BinaryContext &BC) {
       BC.outs() << ". Use -print-unknown to see the list.";
     BC.outs() << '\n';
   }
+
+  if (opts::ShowDensity) {
+    double Density = 0.0;
+    // Sorted by the density in descending order.
+    llvm::stable_sort(FuncDensityList,
+                      [&](const std::pair<double, uint64_t> &A,
+                          const std::pair<double, uint64_t> &B) {
+                        if (A.first != B.first)
+                          return A.first > B.first;
+                        return A.second < B.second;
+                      });
+
+    uint64_t AccumulatedSamples = 0;
+    uint32_t I = 0;
+    assert(opts::ProfileDensityCutOffHot <= 1000000 &&
+           "The cutoff value is greater than 1000000(100%)");
+    while (AccumulatedSamples <
+               TotalSampleCount *
+                   static_cast<float>(opts::ProfileDensityCutOffHot) /
+                   1000000 &&
+           I < FuncDensityList.size()) {
+      AccumulatedSamples += FuncDensityList[I].second;
+      Density = FuncDensityList[I].first;
+      I++;
+    }
+    if (Density == 0.0) {
+      BC.errs() << "BOLT-WARNING: the output profile is empty or the "
+                   "--profile-density-cutoff-hot option is "
+                   "set too low. Please check your command.\n";
+    } else if (Density < opts::ProfileDensityThreshold) {
+      BC.errs()
+          << "BOLT-WARNING: BOLT is estimated to optimize better with "
+          << format("%.1f", opts::ProfileDensityThreshold / Density)
+          << "x more samples. Please consider increasing sampling rate or "
+             "profiling for longer duration to get more samples.\n";
+    }
+
+    BC.outs() << "BOLT-INFO: Functions with density >= "
+              << format("%.1f", Density) << " account for "
+              << format("%.2f",
+                        static_cast<double>(opts::ProfileDensityCutOffHot) /
+                            10000)
+              << "% total sample counts.\n";
+  }
   return Error::success();
 }
 
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index 2e83ca03dbde..ffd693f9bbae 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -849,6 +849,12 @@ bool DataAggregator::doTrace(const LBREntry &First, const LBREntry &Second,
     return false;
   }
 
+  // Set ParentFunc to BAT parent function or FromFunc itself.
+  BinaryFunction *ParentFunc = getBATParentFunction(*FromFunc);
+  if (!ParentFunc)
+    ParentFunc = FromFunc;
+  ParentFunc->SampleCountInBytes += Count * (Second.From - First.To);
+
   std::optional<BoltAddressTranslation::FallthroughListTy> FTs =
       BAT ? BAT->getFallthroughsInTrace(FromFunc->getAddress(), First.To,
                                         Second.From)
@@ -868,13 +874,12 @@ bool DataAggregator::doTrace(const LBREntry &First, const LBREntry &Second,
                     << FromFunc->getPrintName() << ":"
                     << Twine::utohexstr(First.To) << " to "
                     << Twine::utohexstr(Second.From) << ".\n");
-  BinaryFunction *ParentFunc = getBATParentFunction(*FromFunc);
   for (auto [From, To] : *FTs) {
     if (BAT) {
       From = BAT->translate(FromFunc->getAddress(), From, /*IsBranchSrc=*/true);
       To = BAT->translate(FromFunc->getAddress(), To, /*IsBranchSrc=*/false);
     }
-    doIntraBranch(ParentFunc ? *ParentFunc : *FromFunc, From, To, Count, false);
+    doIntraBranch(*ParentFunc, From, To, Count, false);
   }
 
   return true;
diff --git a/bolt/lib/Utils/CommandLineOpts.cpp b/bolt/lib/Utils/CommandLineOpts.cpp
index 435a8fa9cafc..de82420a1671 100644
--- a/bolt/lib/Utils/CommandLineOpts.cpp
+++ b/bolt/lib/Utils/CommandLineOpts.cpp
@@ -175,6 +175,10 @@ cl::opt<std::string> SaveProfile("w",
                                  cl::desc("save recorded profile to a file"),
                                  cl::cat(BoltOutputCategory));
 
+cl::opt<bool> ShowDensity("show-density",
+                          cl::desc("show profile density details"),
+                          cl::Optional, cl::cat(AggregatorCategory));
+
 cl::opt<bool> SplitEH("split-eh", cl::desc("split C++ exception handling code"),
                       cl::Hidden, cl::cat(BoltOptCategory));
 
diff --git a/bolt/test/X86/pre-aggregated-perf.test b/bolt/test/X86/pre-aggregated-perf.test
index 90252f9ff68d..3242ba22f591 100644
--- a/bolt/test/X86/pre-aggregated-perf.test
+++ b/bolt/test/X86/pre-aggregated-perf.test
@@ -11,7 +11,19 @@ REQUIRES: system-linux
 
 RUN: yaml2obj %p/Inputs/blarge.yaml &> %t.exe
 RUN: perf2bolt %t.exe -o %t --pa -p %p/Inputs/pre-aggregated.txt -w %t.new \
-RUN:   --profile-use-dfs | FileCheck %s
+RUN:   --profile-density-threshold=9 --profile-density-cutoff-hot=970000 \
+RUN:   --profile-use-dfs | FileCheck %s --check-prefix=CHECK-P2B
+
+CHECK-P2B: BOLT-INFO: 4 out of 7 functions in the binary (57.1%) have non-empty execution profile
+CHECK-P2B: BOLT-INFO: Functions with density >= 21.7 account for 97.00% total sample counts.
+
+RUN: perf2bolt %t.exe -o %t --pa -p %p/Inputs/pre-aggregated.txt -w %t.new \
+RUN:   --profile-density-cutoff-hot=970000 \
+RUN:   --profile-use-dfs 2>&1 | FileCheck %s --check-prefix=CHECK-WARNING
+
+CHECK-WARNING: BOLT-INFO: 4 out of 7 functions in the binary (57.1%) have non-empty execution profile
+CHECK-WARNING: BOLT-WARNING: BOLT is estimated to optimize better with 2.8x more samples.
+CHECK-WARNING: BOLT-INFO: Functions with density >= 21.7 account for 97.00% total sample counts.
 
 RUN: llvm-bolt %t.exe -data %t -o %t.null | FileCheck %s
 RUN: llvm-bolt %t.exe -data %t.new -o %t.null | FileCheck %s
diff --git a/bolt/tools/driver/llvm-bolt.cpp b/bolt/tools/driver/llvm-bolt.cpp
index a8d1ac648089..efa06cd68cb9 100644
--- a/bolt/tools/driver/llvm-bolt.cpp
+++ b/bolt/tools/driver/llvm-bolt.cpp
@@ -129,6 +129,7 @@ void perf2boltMode(int argc, char **argv) {
     exit(1);
   }
   opts::AggregateOnly = true;
+  opts::ShowDensity = true;
 }
 
 void boltDiffMode(int argc, char **argv) {
-- 
GitLab


From f1595ecfdce5387e41826fd72ff930a1a39ae398 Mon Sep 17 00:00:00 2001
From: Max191 <44243577+Max191@users.noreply.github.com>
Date: Thu, 24 Oct 2024 18:32:45 -0700
Subject: [PATCH 114/693] [mlir] Fix bug in UnPackOp tiling implementation
 causing infinite loop (#113571)

This fixes a bug in the tiling implementation of tensor.unpack that was
causing an infinite loop when certain unpack ops get tiled and fused as
a producer. The tiled implementation of tensor.unpack sometimes needs to
create an additional tensor.extract_slice on the result of the tiled
unpack op, but this slice was getting added to the `generatedSlices` of
the tiling result. The `generatedSlices` are used to find the next
producers to fuse, so it caused an infinite loop of fusing the same
unpack op after it was already in the loop. This fixes the bug by adding
the slice of the source instead of the result.

Signed-off-by: Max Dawkins <max.dawkins@gmail.com>
---
 .../Tensor/IR/TensorTilingInterfaceImpl.cpp   |  8 ++--
 .../tile-and-fuse-using-interface.mlir        | 47 +++++++++++++++++++
 2 files changed, 51 insertions(+), 4 deletions(-)

diff --git a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
index 104d6ae1f9f6..ba41904b3709 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
@@ -554,13 +554,14 @@ struct UnPackOpTiling
     sliceSrcIndices.append(numInnerTiles, zeroAttr);
     sliceSrcSizes.append(unpackOp.getMixedTiles());
     sliceSrcStrides.append(numInnerTiles, oneAttr);
-    Value sliceSource =
+    SmallVector<Operation *> generatedSlices;
+    ExtractSliceOp sliceSource =
         b.create<ExtractSliceOp>(loc, unpackOp.getSource(), sliceSrcIndices,
                                  sliceSrcSizes, sliceSrcStrides);
+    generatedSlices.push_back(sliceSource);
 
     SmallVector<OpFoldResult> destStrides(destRank, oneAttr);
     Value sliceDest;
-    SmallVector<Operation *> generatedSlices;
     if (isPerfectTilingCase) {
       auto destSliceOp = b.create<ExtractSliceOp>(loc, unpackOp.getDest(),
                                                   offsets, sizes, destStrides);
@@ -571,7 +572,7 @@ struct UnPackOpTiling
                                     unpackOp.getDestType().getElementType());
     }
 
-    SmallVector<Value> tiledOperands = {sliceSource, sliceDest};
+    SmallVector<Value> tiledOperands = {sliceSource.getResult(), sliceDest};
     for (auto tile : unpackOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
@@ -586,7 +587,6 @@ struct UnPackOpTiling
     auto extractSlice =
         b.create<ExtractSliceOp>(loc, tiledUnpackOp->getResult(0),
                                  resultOffsetsFromDest, sizes, destStrides);
-    generatedSlices.push_back(extractSlice);
     return TilingResult{
         {tiledUnpackOp}, {extractSlice.getResult()}, generatedSlices};
   }
diff --git a/mlir/test/Interfaces/TilingInterface/tile-and-fuse-using-interface.mlir b/mlir/test/Interfaces/TilingInterface/tile-and-fuse-using-interface.mlir
index 3ea1929e4ed7..5f7663af773a 100644
--- a/mlir/test/Interfaces/TilingInterface/tile-and-fuse-using-interface.mlir
+++ b/mlir/test/Interfaces/TilingInterface/tile-and-fuse-using-interface.mlir
@@ -587,3 +587,50 @@ module attributes {transform.with_named_sequence} {
 //       CHECK:     %[[INSERT_SLICE:.+]] = tensor.insert_slice %[[IF_RESULT]]
 //       CHECK:     scf.yield %[[INSERT_SLICE]]
 //       CHECK:   return %[[FOR_RESULT]]
+
+// -----
+
+func.func @imperfect_unpack_producer_fusion(%source: tensor<1x1x288x8x4xf32>, %dest: tensor<1x2x1152xf32>) -> tensor<1x2x1152xf32> {
+  %0 = tensor.unpack %source
+      outer_dims_perm = [0, 1, 2]
+      inner_dims_pos = [1, 2]
+      inner_tiles = [8, 4] into %dest
+      : tensor<1x1x288x8x4xf32> -> tensor<1x2x1152xf32>
+  %1 = tensor.empty() : tensor<1x2x1152xf32>
+  %cst = arith.constant 1.0 : f32
+  %2 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>,
+                                        affine_map<(d0, d1, d2) -> (d0, d1, d2)>],
+                       iterator_types = ["parallel", "parallel", "parallel"]}
+                       ins(%0 : tensor<1x2x1152xf32>)
+                       outs(%1 : tensor<1x2x1152xf32>) {
+  ^bb0(%in: f32, %out: f32):
+    %7 = arith.addf %in, %cst : f32
+    linalg.yield %7 : f32
+  } -> tensor<1x2x1152xf32>
+  return %2 : tensor<1x2x1152xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
+    %matmul = transform.structured.match ops{["linalg.generic"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    %a, %b = transform.structured.fuse %matmul [0, 1, 0]
+      : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK-LABEL: func @imperfect_unpack_producer_fusion
+//  CHECK-SAME:     %[[ARG0:.+]]: tensor<1x1x288x8x4xf32>
+//  CHECK-SAME:     %[[ARG1:.+]]: tensor<1x2x1152xf32>
+//       CHECK:   %[[FOR_RESULT:.+]] = scf.for{{.*}}iter_args(%[[ITER_ARG:.+]] = {{.*}})
+//       CHECK:     %[[SLICE:.+]] = tensor.extract_slice %[[ARG0]]
+//       CHECK:     %[[UNPACK:.+]] = tensor.unpack %[[SLICE]]
+//   CHECK-DAG:     %[[UNPACK_SLICE:.+]] = tensor.extract_slice %[[UNPACK]]
+//   CHECK-DAG:     %[[INIT_SLICE:.+]] = tensor.extract_slice %[[ITER_ARG]]
+//       CHECK:     %[[GENERIC:.+]] = linalg.generic
+//  CHECK-SAME:         ins(%[[UNPACK_SLICE]]
+//  CHECK-SAME:         outs(%[[INIT_SLICE]]
+//       CHECK:     %[[INSERT_SLICE:.+]] = tensor.insert_slice %[[GENERIC]] into %[[ITER_ARG]]
+//       CHECK:     scf.yield %[[INSERT_SLICE]]
+//       CHECK:   return %[[FOR_RESULT]]
-- 
GitLab


From 78e7d9592d7c3b500ae2bae097d8cd7cd9369671 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Thu, 24 Oct 2024 18:34:29 -0700
Subject: [PATCH 115/693] [clang-format][NFC] Use CLANG_SOURCE_DIR in
 CMakeLists

---
 clang/lib/Format/CMakeLists.txt | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/clang/lib/Format/CMakeLists.txt b/clang/lib/Format/CMakeLists.txt
index b787b11ac7b7..ff987cc608fb 100644
--- a/clang/lib/Format/CMakeLists.txt
+++ b/clang/lib/Format/CMakeLists.txt
@@ -33,10 +33,10 @@ add_clang_library(clangFormat
 file(GLOB_RECURSE files
   *.cpp
   *.h
-  ../../include/clang/Format/*.h
-  ../../tools/clang-format/*.cpp
-  ../../unittests/Format/*.cpp
-  ../../unittests/Format/*.h
+  ${CLANG_SOURCE_DIR}/include/clang/Format/*.h
+  ${CLANG_SOURCE_DIR}/tools/clang-format/*.cpp
+  ${CLANG_SOURCE_DIR}/unittests/Format/*.cpp
+  ${CLANG_SOURCE_DIR}/unittests/Format/*.h
   )
 
 set(check_format_depends)
@@ -46,8 +46,8 @@ foreach (file IN LISTS files)
     COMMAND clang-format ${file} | diff -u ${file} -
     VERBATIM
     COMMENT "Checking format of ${file}..."
-  )
-  list(APPEND check_format_depends "clang-format-check-format${i}")
+    )
+  list(APPEND check_format_depends clang-format-check-format${i})
 
   math(EXPR i ${i}+1)
 endforeach ()
-- 
GitLab


From a31ce36f568723ffa5b4155d0ba009403ccd0d9e Mon Sep 17 00:00:00 2001
From: Haopeng Liu <153236845+haopliu@users.noreply.github.com>
Date: Thu, 24 Oct 2024 18:43:20 -0700
Subject: [PATCH 116/693] Apply initializes attribute to DSE (#113630)

retry #107282

Fixed with `MadeChange |= Changed;` and confirmed it works.

```
cmake -DLLVM_CCACHE_BUILD=ON -DLLVM_ENABLE_EXPENSIVE_CHECKS=ON -DLLVM_ENABLE_WERROR=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS=-U_GLIBCXX_DEBUG '-DLLVM_LIT_ARGS=-v -vv -j96' '-DLLVM_ENABLE_PROJECTS=llvm;lld' -DLLVM_ENABLE_ASSERTIONS=ON -GNinja ../llvm

ninja check-llvm
```
---
 .../Scalar/DeadStoreElimination.cpp           | 251 ++++++++++++---
 .../DeadStoreElimination/inter-procedural.ll  | 301 ++++++++++++++++++
 2 files changed, 510 insertions(+), 42 deletions(-)
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll

diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 6fce46a624c9..8c3cf84c58cd 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -52,6 +52,7 @@
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
+#include "llvm/IR/ConstantRangeList.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfo.h"
@@ -164,6 +165,11 @@ static cl::opt<bool>
     OptimizeMemorySSA("dse-optimize-memoryssa", cl::init(true), cl::Hidden,
                       cl::desc("Allow DSE to optimize memory accesses."));
 
+// TODO: turn on and remove this flag.
+static cl::opt<bool> EnableInitializesImprovement(
+    "enable-dse-initializes-attr-improvement", cl::init(false), cl::Hidden,
+    cl::desc("Enable the initializes attr improvement in DSE"));
+
 //===----------------------------------------------------------------------===//
 // Helper functions
 //===----------------------------------------------------------------------===//
@@ -809,8 +815,10 @@ bool canSkipDef(MemoryDef *D, bool DefVisibleToCaller) {
 // A memory location wrapper that represents a MemoryLocation, `MemLoc`,
 // defined by `MemDef`.
 struct MemoryLocationWrapper {
-  MemoryLocationWrapper(MemoryLocation MemLoc, MemoryDef *MemDef)
-      : MemLoc(MemLoc), MemDef(MemDef) {
+  MemoryLocationWrapper(MemoryLocation MemLoc, MemoryDef *MemDef,
+                        bool DefByInitializesAttr)
+      : MemLoc(MemLoc), MemDef(MemDef),
+        DefByInitializesAttr(DefByInitializesAttr) {
     assert(MemLoc.Ptr && "MemLoc should be not null");
     UnderlyingObject = getUnderlyingObject(MemLoc.Ptr);
     DefInst = MemDef->getMemoryInst();
@@ -820,20 +828,59 @@ struct MemoryLocationWrapper {
   const Value *UnderlyingObject;
   MemoryDef *MemDef;
   Instruction *DefInst;
+  bool DefByInitializesAttr = false;
 };
 
 // A memory def wrapper that represents a MemoryDef and the MemoryLocation(s)
 // defined by this MemoryDef.
 struct MemoryDefWrapper {
-  MemoryDefWrapper(MemoryDef *MemDef, std::optional<MemoryLocation> MemLoc) {
+  MemoryDefWrapper(MemoryDef *MemDef,
+                   ArrayRef<std::pair<MemoryLocation, bool>> MemLocations) {
     DefInst = MemDef->getMemoryInst();
-    if (MemLoc.has_value())
-      DefinedLocation = MemoryLocationWrapper(*MemLoc, MemDef);
+    for (auto &[MemLoc, DefByInitializesAttr] : MemLocations)
+      DefinedLocations.push_back(
+          MemoryLocationWrapper(MemLoc, MemDef, DefByInitializesAttr));
   }
   Instruction *DefInst;
-  std::optional<MemoryLocationWrapper> DefinedLocation = std::nullopt;
+  SmallVector<MemoryLocationWrapper, 1> DefinedLocations;
+};
+
+bool hasInitializesAttr(Instruction *I) {
+  CallBase *CB = dyn_cast<CallBase>(I);
+  return CB && CB->getArgOperandWithAttribute(Attribute::Initializes);
+}
+
+struct ArgumentInitInfo {
+  unsigned Idx;
+  bool IsDeadOrInvisibleOnUnwind;
+  ConstantRangeList Inits;
 };
 
+// Return the intersected range list of the initializes attributes of "Args".
+// "Args" are call arguments that alias to each other.
+// If any argument in "Args" doesn't have dead_on_unwind attr and
+// "CallHasNoUnwindAttr" is false, return empty.
+ConstantRangeList getIntersectedInitRangeList(ArrayRef<ArgumentInitInfo> Args,
+                                              bool CallHasNoUnwindAttr) {
+  if (Args.empty())
+    return {};
+
+  // To address unwind, the function should have nounwind attribute or the
+  // arguments have dead or invisible on unwind. Otherwise, return empty.
+  for (const auto &Arg : Args) {
+    if (!CallHasNoUnwindAttr && !Arg.IsDeadOrInvisibleOnUnwind)
+      return {};
+    if (Arg.Inits.empty())
+      return {};
+  }
+
+  ConstantRangeList IntersectedIntervals = Args.front().Inits;
+  for (auto &Arg : Args.drop_front())
+    IntersectedIntervals = IntersectedIntervals.intersectWith(Arg.Inits);
+
+  return IntersectedIntervals;
+}
+
 struct DSEState {
   Function &F;
   AliasAnalysis &AA;
@@ -911,7 +958,8 @@ struct DSEState {
 
         auto *MD = dyn_cast_or_null<MemoryDef>(MA);
         if (MD && MemDefs.size() < MemorySSADefsPerBlockLimit &&
-            (getLocForWrite(&I) || isMemTerminatorInst(&I)))
+            (getLocForWrite(&I) || isMemTerminatorInst(&I) ||
+             (EnableInitializesImprovement && hasInitializesAttr(&I))))
           MemDefs.push_back(MD);
       }
     }
@@ -1147,13 +1195,26 @@ struct DSEState {
     return MemoryLocation::getOrNone(I);
   }
 
-  std::optional<MemoryLocation> getLocForInst(Instruction *I) {
+  // Returns a list of <MemoryLocation, bool> pairs written by I.
+  // The bool means whether the write is from Initializes attr.
+  SmallVector<std::pair<MemoryLocation, bool>, 1>
+  getLocForInst(Instruction *I, bool ConsiderInitializesAttr) {
+    SmallVector<std::pair<MemoryLocation, bool>, 1> Locations;
     if (isMemTerminatorInst(I)) {
-      if (auto Loc = getLocForTerminator(I)) {
-        return Loc->first;
+      if (auto Loc = getLocForTerminator(I))
+        Locations.push_back(std::make_pair(Loc->first, false));
+      return Locations;
+    }
+
+    if (auto Loc = getLocForWrite(I))
+      Locations.push_back(std::make_pair(*Loc, false));
+
+    if (ConsiderInitializesAttr) {
+      for (auto &MemLoc : getInitializesArgMemLoc(I)) {
+        Locations.push_back(std::make_pair(MemLoc, true));
       }
     }
-    return getLocForWrite(I);
+    return Locations;
   }
 
   /// Assuming this instruction has a dead analyzable write, can we delete
@@ -1365,7 +1426,8 @@ struct DSEState {
   getDomMemoryDef(MemoryDef *KillingDef, MemoryAccess *StartAccess,
                   const MemoryLocation &KillingLoc, const Value *KillingUndObj,
                   unsigned &ScanLimit, unsigned &WalkerStepLimit,
-                  bool IsMemTerm, unsigned &PartialLimit) {
+                  bool IsMemTerm, unsigned &PartialLimit,
+                  bool IsInitializesAttrMemLoc) {
     if (ScanLimit == 0 || WalkerStepLimit == 0) {
       LLVM_DEBUG(dbgs() << "\n    ...  hit scan limit\n");
       return std::nullopt;
@@ -1602,7 +1664,16 @@ struct DSEState {
 
       // Uses which may read the original MemoryDef mean we cannot eliminate the
       // original MD. Stop walk.
-      if (isReadClobber(MaybeDeadLoc, UseInst)) {
+      // If KillingDef is a CallInst with "initializes" attribute, the reads in
+      // the callee would be dominated by initializations, so it should be safe.
+      bool IsKillingDefFromInitAttr = false;
+      if (IsInitializesAttrMemLoc) {
+        if (KillingI == UseInst &&
+            KillingUndObj == getUnderlyingObject(MaybeDeadLoc.Ptr))
+          IsKillingDefFromInitAttr = true;
+      }
+
+      if (isReadClobber(MaybeDeadLoc, UseInst) && !IsKillingDefFromInitAttr) {
         LLVM_DEBUG(dbgs() << "    ... found read clobber\n");
         return std::nullopt;
       }
@@ -2171,6 +2242,16 @@ struct DSEState {
     return MadeChange;
   }
 
+  // Return the locations written by the initializes attribute.
+  // Note that this function considers:
+  // 1. Unwind edge: use "initializes" attribute only if the callee has
+  //    "nounwind" attribute, or the argument has "dead_on_unwind" attribute,
+  //    or the argument is invisible to caller on unwind. That is, we don't
+  //    perform incorrect DSE on unwind edges in the current function.
+  // 2. Argument alias: for aliasing arguments, the "initializes" attribute is
+  //    the intersected range list of their "initializes" attributes.
+  SmallVector<MemoryLocation, 1> getInitializesArgMemLoc(const Instruction *I);
+
   // Try to eliminate dead defs that access `KillingLocWrapper.MemLoc` and are
   // killed by `KillingLocWrapper.MemDef`. Return whether
   // any changes were made, and whether `KillingLocWrapper.DefInst` was deleted.
@@ -2182,6 +2263,75 @@ struct DSEState {
   bool eliminateDeadDefs(const MemoryDefWrapper &KillingDefWrapper);
 };
 
+SmallVector<MemoryLocation, 1>
+DSEState::getInitializesArgMemLoc(const Instruction *I) {
+  const CallBase *CB = dyn_cast<CallBase>(I);
+  if (!CB)
+    return {};
+
+  // Collect aliasing arguments and their initializes ranges.
+  SmallMapVector<Value *, SmallVector<ArgumentInitInfo, 2>, 2> Arguments;
+  for (unsigned Idx = 0, Count = CB->arg_size(); Idx < Count; ++Idx) {
+    ConstantRangeList Inits;
+    Attribute InitializesAttr = CB->getParamAttr(Idx, Attribute::Initializes);
+    if (InitializesAttr.isValid())
+      Inits = InitializesAttr.getValueAsConstantRangeList();
+
+    Value *CurArg = CB->getArgOperand(Idx);
+    // We don't perform incorrect DSE on unwind edges in the current function,
+    // and use the "initializes" attribute to kill dead stores if:
+    // - The call does not throw exceptions, "CB->doesNotThrow()".
+    // - Or the callee parameter has "dead_on_unwind" attribute.
+    // - Or the argument is invisible to caller on unwind, and there are no
+    //   unwind edges from this call in the current function (e.g. `CallInst`).
+    bool IsDeadOrInvisibleOnUnwind =
+        CB->paramHasAttr(Idx, Attribute::DeadOnUnwind) ||
+        (isa<CallInst>(CB) && isInvisibleToCallerOnUnwind(CurArg));
+    ArgumentInitInfo InitInfo{Idx, IsDeadOrInvisibleOnUnwind, Inits};
+    bool FoundAliasing = false;
+    for (auto &[Arg, AliasList] : Arguments) {
+      auto AAR = BatchAA.alias(MemoryLocation::getBeforeOrAfter(Arg),
+                               MemoryLocation::getBeforeOrAfter(CurArg));
+      if (AAR == AliasResult::NoAlias) {
+        continue;
+      } else if (AAR == AliasResult::MustAlias) {
+        FoundAliasing = true;
+        AliasList.push_back(InitInfo);
+      } else {
+        // For PartialAlias and MayAlias, there is an offset or may be an
+        // unknown offset between the arguments and we insert an empty init
+        // range to discard the entire initializes info while intersecting.
+        FoundAliasing = true;
+        AliasList.push_back(ArgumentInitInfo{Idx, IsDeadOrInvisibleOnUnwind,
+                                             ConstantRangeList()});
+      }
+    }
+    if (!FoundAliasing)
+      Arguments[CurArg] = {InitInfo};
+  }
+
+  SmallVector<MemoryLocation, 1> Locations;
+  for (const auto &[_, Args] : Arguments) {
+    auto IntersectedRanges =
+        getIntersectedInitRangeList(Args, CB->doesNotThrow());
+    if (IntersectedRanges.empty())
+      continue;
+
+    for (const auto &Arg : Args) {
+      for (const auto &Range : IntersectedRanges) {
+        int64_t Start = Range.getLower().getSExtValue();
+        int64_t End = Range.getUpper().getSExtValue();
+        // For now, we only handle locations starting at offset 0.
+        if (Start == 0)
+          Locations.push_back(MemoryLocation(CB->getArgOperand(Arg.Idx),
+                                             LocationSize::precise(End - Start),
+                                             CB->getAAMetadata()));
+      }
+    }
+  }
+  return Locations;
+}
+
 std::pair<bool, bool>
 DSEState::eliminateDeadDefs(const MemoryLocationWrapper &KillingLocWrapper) {
   bool Changed = false;
@@ -2208,7 +2358,8 @@ DSEState::eliminateDeadDefs(const MemoryLocationWrapper &KillingLocWrapper) {
     std::optional<MemoryAccess *> MaybeDeadAccess = getDomMemoryDef(
         KillingLocWrapper.MemDef, Current, KillingLocWrapper.MemLoc,
         KillingLocWrapper.UnderlyingObject, ScanLimit, WalkerStepLimit,
-        isMemTerminatorInst(KillingLocWrapper.DefInst), PartialLimit);
+        isMemTerminatorInst(KillingLocWrapper.DefInst), PartialLimit,
+        KillingLocWrapper.DefByInitializesAttr);
 
     if (!MaybeDeadAccess) {
       LLVM_DEBUG(dbgs() << "  finished walk\n");
@@ -2231,10 +2382,20 @@ DSEState::eliminateDeadDefs(const MemoryLocationWrapper &KillingLocWrapper) {
       }
       continue;
     }
+    // We cannot apply the initializes attribute to DeadAccess/DeadDef.
+    // It would incorrectly consider a call instruction as redundant store
+    // and remove this call instruction.
+    // TODO: this conflates the existence of a MemoryLocation with being able
+    // to delete the instruction. Fix isRemovable() to consider calls with
+    // side effects that cannot be removed, e.g. calls with the initializes
+    // attribute, and remove getLocForInst(ConsiderInitializesAttr = false).
     MemoryDefWrapper DeadDefWrapper(
         cast<MemoryDef>(DeadAccess),
-        getLocForInst(cast<MemoryDef>(DeadAccess)->getMemoryInst()));
-    MemoryLocationWrapper &DeadLocWrapper = *DeadDefWrapper.DefinedLocation;
+        getLocForInst(cast<MemoryDef>(DeadAccess)->getMemoryInst(),
+                      /*ConsiderInitializesAttr=*/false));
+    assert(DeadDefWrapper.DefinedLocations.size() == 1);
+    MemoryLocationWrapper &DeadLocWrapper =
+        DeadDefWrapper.DefinedLocations.front();
     LLVM_DEBUG(dbgs() << " (" << *DeadLocWrapper.DefInst << ")\n");
     ToCheck.insert(DeadLocWrapper.MemDef->getDefiningAccess());
     NumGetDomMemoryDefPassed++;
@@ -2309,37 +2470,42 @@ DSEState::eliminateDeadDefs(const MemoryLocationWrapper &KillingLocWrapper) {
 }
 
 bool DSEState::eliminateDeadDefs(const MemoryDefWrapper &KillingDefWrapper) {
-  if (!KillingDefWrapper.DefinedLocation.has_value()) {
+  if (KillingDefWrapper.DefinedLocations.empty()) {
     LLVM_DEBUG(dbgs() << "Failed to find analyzable write location for "
                       << *KillingDefWrapper.DefInst << "\n");
     return false;
   }
 
-  auto &KillingLocWrapper = *KillingDefWrapper.DefinedLocation;
-  LLVM_DEBUG(dbgs() << "Trying to eliminate MemoryDefs killed by "
-                    << *KillingLocWrapper.MemDef << " ("
-                    << *KillingLocWrapper.DefInst << ")\n");
-  auto [Changed, DeletedKillingLoc] = eliminateDeadDefs(KillingLocWrapper);
-
-  // Check if the store is a no-op.
-  if (!DeletedKillingLoc && storeIsNoop(KillingLocWrapper.MemDef,
-                                        KillingLocWrapper.UnderlyingObject)) {
-    LLVM_DEBUG(dbgs() << "DSE: Remove No-Op Store:\n  DEAD: "
-                      << *KillingLocWrapper.DefInst << '\n');
-    deleteDeadInstruction(KillingLocWrapper.DefInst);
-    NumRedundantStores++;
-    return true;
-  }
-  // Can we form a calloc from a memset/malloc pair?
-  if (!DeletedKillingLoc &&
-      tryFoldIntoCalloc(KillingLocWrapper.MemDef,
-                        KillingLocWrapper.UnderlyingObject)) {
-    LLVM_DEBUG(dbgs() << "DSE: Remove memset after forming calloc:\n"
-                      << "  DEAD: " << *KillingLocWrapper.DefInst << '\n');
-    deleteDeadInstruction(KillingLocWrapper.DefInst);
-    return true;
+  bool MadeChange = false;
+  for (auto &KillingLocWrapper : KillingDefWrapper.DefinedLocations) {
+    LLVM_DEBUG(dbgs() << "Trying to eliminate MemoryDefs killed by "
+                      << *KillingLocWrapper.MemDef << " ("
+                      << *KillingLocWrapper.DefInst << ")\n");
+    auto [Changed, DeletedKillingLoc] = eliminateDeadDefs(KillingLocWrapper);
+    MadeChange |= Changed;
+
+    // Check if the store is a no-op.
+    if (!DeletedKillingLoc && storeIsNoop(KillingLocWrapper.MemDef,
+                                          KillingLocWrapper.UnderlyingObject)) {
+      LLVM_DEBUG(dbgs() << "DSE: Remove No-Op Store:\n  DEAD: "
+                        << *KillingLocWrapper.DefInst << '\n');
+      deleteDeadInstruction(KillingLocWrapper.DefInst);
+      NumRedundantStores++;
+      MadeChange = true;
+      continue;
+    }
+    // Can we form a calloc from a memset/malloc pair?
+    if (!DeletedKillingLoc &&
+        tryFoldIntoCalloc(KillingLocWrapper.MemDef,
+                          KillingLocWrapper.UnderlyingObject)) {
+      LLVM_DEBUG(dbgs() << "DSE: Remove memset after forming calloc:\n"
+                        << "  DEAD: " << *KillingLocWrapper.DefInst << '\n');
+      deleteDeadInstruction(KillingLocWrapper.DefInst);
+      MadeChange = true;
+      continue;
+    }
   }
-  return Changed;
+  return MadeChange;
 }
 
 static bool eliminateDeadStores(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
@@ -2355,7 +2521,8 @@ static bool eliminateDeadStores(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
       continue;
 
     MemoryDefWrapper KillingDefWrapper(
-        KillingDef, State.getLocForInst(KillingDef->getMemoryInst()));
+        KillingDef, State.getLocForInst(KillingDef->getMemoryInst(),
+                                        EnableInitializesImprovement));
     MadeChange |= State.eliminateDeadDefs(KillingDefWrapper);
   }
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll b/llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll
new file mode 100644
index 000000000000..d93da9b6612b
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/inter-procedural.ll
@@ -0,0 +1,301 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=dse -enable-dse-initializes-attr-improvement -S | FileCheck %s
+
+declare void @p1_write_only(ptr nocapture noundef writeonly initializes((0, 2)) dead_on_unwind)
+declare void @p1_write_then_read(ptr nocapture noundef initializes((0, 2)) dead_on_unwind)
+declare void @p1_clobber(ptr nocapture noundef)
+declare void @p2_same_range(ptr nocapture noundef initializes((0, 2)) dead_on_unwind, ptr nocapture noundef initializes((0, 2)) dead_on_unwind)
+declare void @p2_no_init(ptr nocapture noundef initializes((0, 2)) dead_on_unwind, ptr nocapture noundef dead_on_unwind)
+declare void @p2_no_dead_on_unwind(ptr nocapture noundef initializes((0, 2)) dead_on_unwind, ptr nocapture noundef initializes((0, 2)))
+declare void @p2_no_dead_on_unwind_but_nounwind(ptr nocapture noundef initializes((0, 2)) dead_on_unwind, ptr nocapture noundef initializes((0, 2))) nounwind
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p1_write_only_caller() {
+; CHECK-LABEL: @p1_write_only_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    call void @p1_write_only(ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  call void @p1_write_only(ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p1_write_then_read_caller() {
+; CHECK-LABEL: @p1_write_then_read_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    call void @p1_write_then_read(ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  call void @p1_write_then_read(ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p1_write_then_read_caller_with_clobber() {
+; CHECK-LABEL: @p1_write_then_read_caller_with_clobber(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    store i16 0, ptr [[PTR]], align 2
+; CHECK-NEXT:    call void @p1_clobber(ptr [[PTR]])
+; CHECK-NEXT:    call void @p1_write_then_read(ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  call void @p1_clobber(ptr %ptr)
+  call void @p1_write_then_read(ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+declare void @p1_write_then_read_raw(ptr nocapture noundef initializes((0, 2)))
+define i16 @p1_initializes_invoke() personality ptr undef {
+; CHECK-LABEL: @p1_initializes_invoke(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    store i16 0, ptr [[PTR]], align 2
+; CHECK-NEXT:    invoke void @p1_write_then_read_raw(ptr [[PTR]])
+; CHECK-NEXT:            to label [[BB1:%.*]] unwind label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    ret i16 0
+; CHECK:       bb2:
+; CHECK-NEXT:    [[TMP:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT:            cleanup
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+entry:
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  invoke void @p1_write_then_read_raw(ptr %ptr) to label %bb1 unwind label %bb2
+bb1:
+  ret i16 0
+bb2:
+  %tmp = landingpad { ptr, i32 }
+  cleanup
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_same_range_noalias_caller() {
+; CHECK-LABEL: @p2_same_range_noalias_caller(
+; CHECK-NEXT:    [[PTR1:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    [[PTR2:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    call void @p2_same_range(ptr [[PTR1]], ptr [[PTR2]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR1]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr1 = alloca i16
+  %ptr2 = alloca i16
+  store i16 0, ptr %ptr1
+  store i16 0, ptr %ptr2
+  call void @p2_same_range(ptr %ptr1, ptr %ptr2)
+  %l = load i16, ptr %ptr1
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_same_range_must_alias_caller() {
+; CHECK-LABEL: @p2_same_range_must_alias_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    call void @p2_same_range(ptr [[PTR]], ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  call void @p2_same_range(ptr %ptr, ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_same_range_may_or_partial_alias_caller1(ptr %base, i1 %x) {
+; CHECK-LABEL: @p2_same_range_may_or_partial_alias_caller1(
+; CHECK-NEXT:    [[BASEPLUS:%.*]] = getelementptr i8, ptr [[BASE:%.*]], i64 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[X:%.*]], ptr [[BASEPLUS]], ptr [[BASE]]
+; CHECK-NEXT:    store i32 0, ptr [[BASE]], align 4
+; CHECK-NEXT:    call void @p2_same_range(ptr [[BASE]], ptr [[SEL]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[BASE]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %baseplus = getelementptr i8, ptr %base, i64 1
+  %sel = select i1 %x, ptr %baseplus, ptr %base
+  store i32 0, ptr %base
+  call void @p2_same_range(ptr %base, ptr %sel)
+  %l = load i16, ptr %base
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_same_range_may_or_partial_alias_caller2(ptr %base1, ptr %base2) {
+; CHECK-LABEL: @p2_same_range_may_or_partial_alias_caller2(
+; CHECK-NEXT:    store i32 0, ptr [[BASE1:%.*]], align 4
+; CHECK-NEXT:    call void @p2_same_range(ptr [[BASE1]], ptr [[BASE2:%.*]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[BASE1]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  store i32 0, ptr %base1
+  call void @p2_same_range(ptr %base1, ptr %base2)
+  %l = load i16, ptr %base1
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_no_init_alias_caller() {
+; CHECK-LABEL: @p2_no_init_alias_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    store i16 0, ptr [[PTR]], align 2
+; CHECK-NEXT:    call void @p2_no_init(ptr [[PTR]], ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  call void @p2_no_init(ptr %ptr, ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Althrough the 2nd parameter of `p2_no_dead_on_unwind` doesn't have
+; the 'dead_on_unwind' attribute, it's invisble to caller on unwind.
+; DSE still uses the 'initializes' attribute and kill the dead store.
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_no_dead_on_unwind_but_invisible_to_caller_alias_caller() {
+; CHECK-LABEL: @p2_no_dead_on_unwind_but_invisible_to_caller_alias_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    call void @p2_no_dead_on_unwind(ptr [[PTR]], ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  call void @p2_no_dead_on_unwind(ptr %ptr, ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_no_dead_on_unwind_alias_caller(ptr %ptr) {
+; CHECK-LABEL: @p2_no_dead_on_unwind_alias_caller(
+; CHECK-NEXT:    store i16 0, ptr [[PTR:%.*]], align 2
+; CHECK-NEXT:    call void @p2_no_dead_on_unwind(ptr [[PTR]], ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  store i16 0, ptr %ptr
+  call void @p2_no_dead_on_unwind(ptr %ptr, ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @p2_no_dead_on_unwind_but_nounwind_alias_caller() {
+; CHECK-LABEL: @p2_no_dead_on_unwind_but_nounwind_alias_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    call void @p2_no_dead_on_unwind_but_nounwind(ptr [[PTR]], ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca i16
+  store i16 0, ptr %ptr
+  call void @p2_no_dead_on_unwind_but_nounwind(ptr %ptr, ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i1) nounwind
+declare void @large_p1(ptr nocapture noundef initializes((0, 200))) nounwind
+declare void @large_p2(ptr nocapture noundef initializes((0, 200)), ptr nocapture noundef initializes((0, 100))) nounwind
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @large_p1_caller() {
+; CHECK-LABEL: @large_p1_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca [200 x i8], align 1
+; CHECK-NEXT:    call void @large_p1(ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca [200 x i8]
+  call void @llvm.memset.p0.i64(ptr %ptr, i8 42, i64 100, i1 false)
+  call void @large_p1(ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @large_p2_nonalias_caller() {
+; CHECK-LABEL: @large_p2_nonalias_caller(
+; CHECK-NEXT:    [[PTR1:%.*]] = alloca [200 x i8], align 1
+; CHECK-NEXT:    [[PTR2:%.*]] = alloca [100 x i8], align 1
+; CHECK-NEXT:    call void @large_p2(ptr [[PTR1]], ptr [[PTR2]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR1]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr1 = alloca [200 x i8]
+  %ptr2 = alloca [100 x i8]
+  call void @llvm.memset.p0.i64(ptr %ptr1, i8 42, i64 200, i1 false)
+  call void @llvm.memset.p0.i64(ptr %ptr2, i8 42, i64 100, i1 false)
+  call void @large_p2(ptr %ptr1, ptr %ptr2)
+  %l = load i16, ptr %ptr1
+  ret i16 %l
+}
+
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @large_p2_must_alias_caller() {
+; CHECK-LABEL: @large_p2_must_alias_caller(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca [300 x i8], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 100
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP1]], i8 42, i64 200, i1 false)
+; CHECK-NEXT:    call void @large_p2(ptr [[PTR]], ptr [[PTR]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[PTR]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %ptr = alloca [300 x i8]
+  call void @llvm.memset.p0.i64(ptr %ptr, i8 42, i64 300, i1 false)
+  call void @large_p2(ptr %ptr, ptr %ptr)
+  %l = load i16, ptr %ptr
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @large_p2_may_or_partial_alias_caller1(ptr %base) {
+; CHECK-LABEL: @large_p2_may_or_partial_alias_caller1(
+; CHECK-NEXT:    [[BASEPLUS:%.*]] = getelementptr i8, ptr [[BASE:%.*]], i64 100
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[BASE]], i8 42, i64 300, i1 false)
+; CHECK-NEXT:    call void @large_p2(ptr [[BASE]], ptr [[BASEPLUS]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[BASE]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  %baseplus = getelementptr i8, ptr %base, i64 100
+  call void @llvm.memset.p0.i64(ptr %base, i8 42, i64 300, i1 false)
+  call void @large_p2(ptr %base, ptr %baseplus)
+  %l = load i16, ptr %base
+  ret i16 %l
+}
+
+; Function Attrs: mustprogress nounwind uwtable
+define i16 @large_p2_may_or_partial_alias_caller2(ptr %base1, ptr %base2) {
+; CHECK-LABEL: @large_p2_may_or_partial_alias_caller2(
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[BASE1:%.*]], i8 42, i64 300, i1 false)
+; CHECK-NEXT:    call void @large_p2(ptr [[BASE1]], ptr [[BASE2:%.*]])
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[BASE1]], align 2
+; CHECK-NEXT:    ret i16 [[L]]
+;
+  call void @llvm.memset.p0.i64(ptr %base1, i8 42, i64 300, i1 false)
+  call void @large_p2(ptr %base1, ptr %base2)
+  %l = load i16, ptr %base1
+  ret i16 %l
+}
+
-- 
GitLab


From b852fb1ec5fa15f0b913cc4988cbd09239b19904 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Thu, 24 Oct 2024 20:20:48 -0700
Subject: [PATCH 117/693] [lldb] Move ValueObject into its own library (NFC)
 (#113393)

ValueObject is part of lldbCore for historical reasons, but conceptually
it deserves to be its own library. This does introduce a (link-time) circular
dependency between lldbCore and lldbValueObject, which is unfortunate
but probably unavoidable because so many things in LLDB rely on
ValueObject. We already have cycles and these libraries are never built
as dylibs so while this doesn't improve the situation, it also doesn't
make things worse.

The header includes were updated with the following command:

```
find . -type f -exec sed -i.bak "s%include \"lldb/Core/ValueObject%include \"lldb/ValueObject/ValueObject%" '{}' \;
```
---
 lldb/include/lldb/Core/IOHandler.h            |  1 -
 .../lldb/DataFormatters/FormattersContainer.h |  2 +-
 lldb/include/lldb/DataFormatters/TypeFormat.h |  2 +-
 .../lldb/DataFormatters/TypeSynthetic.h       |  2 +-
 .../lldb/Expression/ExpressionVariable.h      |  2 +-
 .../OptionGroupValueObjectDisplay.h           |  2 +-
 lldb/include/lldb/Target/LanguageRuntime.h    |  2 +-
 lldb/include/lldb/Target/StackFrame.h         |  2 +-
 .../lldb/Target/StackFrameRecognizer.h        |  4 +-
 .../lldb/{Core => ValueObject}/ValueObject.h  | 12 ++---
 .../{Core => ValueObject}/ValueObjectCast.h   | 11 ++---
 .../{Core => ValueObject}/ValueObjectChild.h  | 15 +++---
 .../ValueObjectConstResult.h                  | 19 ++++----
 .../ValueObjectConstResultCast.h              | 10 ++--
 .../ValueObjectConstResultChild.h             | 23 ++++-----
 .../ValueObjectConstResultImpl.h              |  8 ++--
 .../ValueObjectDynamicValue.h                 |  8 ++--
 .../{Core => ValueObject}/ValueObjectList.h   |  7 +--
 .../{Core => ValueObject}/ValueObjectMemory.h |  8 ++--
 .../ValueObjectRegister.h                     |  8 ++--
 .../ValueObjectSyntheticFilter.h              |  8 ++--
 .../ValueObjectUpdater.h                      |  8 ++--
 .../{Core => ValueObject}/ValueObjectVTable.h |  8 ++--
 .../ValueObjectVariable.h                     | 10 ++--
 lldb/source/API/CMakeLists.txt                |  1 +
 lldb/source/API/SBBlock.cpp                   |  2 +-
 lldb/source/API/SBFrame.cpp                   |  6 +--
 lldb/source/API/SBModule.cpp                  |  4 +-
 lldb/source/API/SBTarget.cpp                  |  6 +--
 lldb/source/API/SBThread.cpp                  |  2 +-
 lldb/source/API/SBType.cpp                    |  2 +-
 lldb/source/API/SBValue.cpp                   |  4 +-
 lldb/source/API/SBValueList.cpp               |  2 +-
 lldb/source/Breakpoint/BreakpointLocation.cpp |  2 +-
 lldb/source/Breakpoint/CMakeLists.txt         |  1 +
 lldb/source/Breakpoint/Watchpoint.cpp         |  4 +-
 lldb/source/CMakeLists.txt                    |  1 +
 lldb/source/Commands/CMakeLists.txt           |  1 +
 .../Commands/CommandObjectDWIMPrint.cpp       |  2 +-
 lldb/source/Commands/CommandObjectFrame.cpp   |  2 +-
 lldb/source/Commands/CommandObjectMemory.cpp  |  2 +-
 lldb/source/Commands/CommandObjectTarget.cpp  |  2 +-
 lldb/source/Commands/CommandObjectThread.cpp  |  2 +-
 .../Commands/CommandObjectWatchpoint.cpp      |  2 +-
 lldb/source/Core/CMakeLists.txt               | 16 +------
 lldb/source/Core/DumpRegisterValue.cpp        |  4 +-
 lldb/source/Core/FormatEntity.cpp             |  4 +-
 lldb/source/Core/IOHandlerCursesGUI.cpp       |  6 +--
 lldb/source/DataFormatters/CMakeLists.txt     |  3 +-
 .../DataFormatters/CXXFunctionPointer.cpp     |  2 +-
 .../DataFormatters/DumpValueObjectOptions.cpp |  2 +-
 lldb/source/DataFormatters/FormatManager.cpp  |  2 +-
 lldb/source/DataFormatters/StringPrinter.cpp  |  2 +-
 lldb/source/DataFormatters/TypeSummary.cpp    |  2 +-
 .../DataFormatters/ValueObjectPrinter.cpp     |  2 +-
 lldb/source/DataFormatters/VectorType.cpp     |  4 +-
 lldb/source/Expression/CMakeLists.txt         |  1 +
 lldb/source/Expression/FunctionCaller.cpp     |  4 +-
 lldb/source/Expression/IRInterpreter.cpp      |  2 +-
 lldb/source/Expression/LLVMUserExpression.cpp |  2 +-
 lldb/source/Expression/Materializer.cpp       |  4 +-
 lldb/source/Expression/UserExpression.cpp     |  2 +-
 .../Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp   |  2 +-
 .../Plugins/ABI/AArch64/ABISysV_arm64.cpp     |  2 +-
 .../source/Plugins/ABI/AArch64/CMakeLists.txt |  1 +
 lldb/source/Plugins/ABI/ARC/ABISysV_arc.cpp   |  6 +--
 lldb/source/Plugins/ABI/ARC/CMakeLists.txt    |  1 +
 lldb/source/Plugins/ABI/ARM/ABIMacOSX_arm.cpp |  2 +-
 lldb/source/Plugins/ABI/ARM/ABISysV_arm.cpp   |  2 +-
 lldb/source/Plugins/ABI/ARM/CMakeLists.txt    |  1 +
 .../Plugins/ABI/Hexagon/ABISysV_hexagon.cpp   |  6 +--
 .../source/Plugins/ABI/Hexagon/CMakeLists.txt |  1 +
 .../Plugins/ABI/MSP430/ABISysV_msp430.cpp     |  6 +--
 lldb/source/Plugins/ABI/MSP430/CMakeLists.txt |  1 +
 lldb/source/Plugins/ABI/Mips/ABISysV_mips.cpp |  6 +--
 .../Plugins/ABI/Mips/ABISysV_mips64.cpp       |  6 +--
 lldb/source/Plugins/ABI/Mips/CMakeLists.txt   |  1 +
 .../Plugins/ABI/PowerPC/ABISysV_ppc.cpp       |  6 +--
 .../Plugins/ABI/PowerPC/ABISysV_ppc64.cpp     |  6 +--
 .../source/Plugins/ABI/PowerPC/CMakeLists.txt |  1 +
 .../Plugins/ABI/RISCV/ABISysV_riscv.cpp       |  2 +-
 lldb/source/Plugins/ABI/RISCV/CMakeLists.txt  |  1 +
 .../Plugins/ABI/SystemZ/ABISysV_s390x.cpp     |  6 +--
 .../source/Plugins/ABI/SystemZ/CMakeLists.txt |  1 +
 .../source/Plugins/ABI/X86/ABIMacOSX_i386.cpp |  2 +-
 lldb/source/Plugins/ABI/X86/ABISysV_i386.cpp  |  6 +--
 .../source/Plugins/ABI/X86/ABISysV_x86_64.cpp |  6 +--
 .../Plugins/ABI/X86/ABIWindows_x86_64.cpp     |  6 +--
 lldb/source/Plugins/ABI/X86/CMakeLists.txt    |  1 +
 .../ExpressionParser/Clang/CMakeLists.txt     |  1 +
 .../Clang/ClangExpressionDeclMap.cpp          |  4 +-
 .../Clang/ClangExpressionUtil.cpp             |  2 +-
 .../Clang/ClangExpressionVariable.cpp         |  2 +-
 .../Clang/ClangFunctionCaller.cpp             |  4 +-
 .../Clang/ClangFunctionCaller.h               |  2 +-
 .../Clang/ClangUserExpression.cpp             |  2 +-
 .../TSan/CMakeLists.txt                       |  1 +
 .../TSan/InstrumentationRuntimeTSan.cpp       |  2 +-
 .../UBSan/CMakeLists.txt                      |  1 +
 .../UBSan/InstrumentationRuntimeUBSan.cpp     |  2 +-
 .../Utility/CMakeLists.txt                    |  1 +
 .../Utility/ReportRetriever.cpp               |  2 +-
 .../Language/CPlusPlus/BlockPointer.cpp       |  4 +-
 .../Plugins/Language/CPlusPlus/CMakeLists.txt |  1 +
 .../Language/CPlusPlus/CPlusPlusLanguage.cpp  |  2 +-
 .../Plugins/Language/CPlusPlus/Coroutines.h   |  2 +-
 .../Language/CPlusPlus/CxxStringTypes.cpp     |  4 +-
 .../Language/CPlusPlus/CxxStringTypes.h       |  2 +-
 .../Plugins/Language/CPlusPlus/Generic.h      |  2 +-
 .../Plugins/Language/CPlusPlus/LibCxx.cpp     |  4 +-
 .../Plugins/Language/CPlusPlus/LibCxx.h       |  2 +-
 .../Plugins/Language/CPlusPlus/LibCxxAtomic.h |  2 +-
 .../CPlusPlus/LibCxxInitializerList.cpp       |  2 +-
 .../Plugins/Language/CPlusPlus/LibCxxList.cpp |  4 +-
 .../Plugins/Language/CPlusPlus/LibCxxMap.cpp  |  4 +-
 .../Language/CPlusPlus/LibCxxProxyArray.cpp   |  2 +-
 .../CPlusPlus/LibCxxRangesRefView.cpp         |  2 +-
 .../Language/CPlusPlus/LibCxxSliceArray.cpp   |  2 +-
 .../Plugins/Language/CPlusPlus/LibCxxSpan.cpp |  2 +-
 .../Language/CPlusPlus/LibCxxUnorderedMap.cpp |  4 +-
 .../Language/CPlusPlus/LibCxxValarray.cpp     |  2 +-
 .../Language/CPlusPlus/LibCxxVariant.h        |  2 +-
 .../Language/CPlusPlus/LibCxxVector.cpp       |  2 +-
 .../Plugins/Language/CPlusPlus/LibStdcpp.cpp  |  4 +-
 .../Plugins/Language/CPlusPlus/LibStdcpp.h    |  2 +-
 .../Language/CPlusPlus/LibStdcppTuple.cpp     |  2 +-
 .../CPlusPlus/LibStdcppUniquePointer.cpp      |  2 +-
 lldb/source/Plugins/Language/ObjC/CF.cpp      |  4 +-
 lldb/source/Plugins/Language/ObjC/CF.h        |  2 +-
 .../Plugins/Language/ObjC/CMakeLists.txt      |  1 +
 lldb/source/Plugins/Language/ObjC/Cocoa.cpp   |  4 +-
 lldb/source/Plugins/Language/ObjC/Cocoa.h     |  2 +-
 lldb/source/Plugins/Language/ObjC/CoreMedia.h |  2 +-
 lldb/source/Plugins/Language/ObjC/NSArray.cpp |  4 +-
 .../Plugins/Language/ObjC/NSDictionary.cpp    |  4 +-
 .../Plugins/Language/ObjC/NSDictionary.h      |  2 +-
 lldb/source/Plugins/Language/ObjC/NSError.cpp |  4 +-
 .../Plugins/Language/ObjC/NSException.cpp     |  4 +-
 .../Plugins/Language/ObjC/NSIndexPath.cpp     |  4 +-
 lldb/source/Plugins/Language/ObjC/NSSet.cpp   |  4 +-
 lldb/source/Plugins/Language/ObjC/NSSet.h     |  2 +-
 .../source/Plugins/Language/ObjC/NSString.cpp |  4 +-
 lldb/source/Plugins/Language/ObjC/NSString.h  |  2 +-
 .../Plugins/Language/ObjC/ObjCLanguage.cpp    |  2 +-
 .../CPlusPlus/ItaniumABI/CMakeLists.txt       |  1 +
 .../ItaniumABI/ItaniumABILanguageRuntime.cpp  |  4 +-
 .../AppleObjCRuntime/AppleObjCRuntime.cpp     |  4 +-
 .../AppleObjCRuntime/AppleObjCRuntimeV2.cpp   |  4 +-
 .../ObjC/AppleObjCRuntime/CMakeLists.txt      |  1 +
 .../ObjC/GNUstepObjCRuntime/CMakeLists.txt    |  1 +
 .../GNUstepObjCRuntime/GNUstepObjCRuntime.cpp |  2 +-
 .../ObjC/ObjCLanguageRuntime.cpp              |  2 +-
 .../Plugins/MemoryHistory/asan/CMakeLists.txt |  1 +
 .../MemoryHistory/asan/MemoryHistoryASan.cpp  |  2 +-
 .../OperatingSystem/Python/CMakeLists.txt     |  1 +
 .../Python/OperatingSystemPython.cpp          |  2 +-
 .../Plugins/Platform/Android/CMakeLists.txt   |  1 +
 .../Platform/Android/PlatformAndroid.cpp      |  2 +-
 .../Plugins/Platform/POSIX/CMakeLists.txt     |  1 +
 .../Plugins/Platform/POSIX/PlatformPOSIX.cpp  |  2 +-
 .../Plugins/Process/Utility/CMakeLists.txt    |  1 +
 .../Process/Utility/InferiorCallPOSIX.cpp     |  2 +-
 .../ScriptInterpreter/Python/CMakeLists.txt   |  1 +
 .../Python/ScriptInterpreterPython.cpp        |  2 +-
 .../Plugins/SymbolFile/DWARF/CMakeLists.txt   |  1 +
 .../SymbolFile/DWARF/DWARFASTParser.cpp       |  2 +-
 .../AbortWithPayloadFrameRecognizer.cpp       |  2 +-
 .../SystemRuntime/MacOSX/CMakeLists.txt       |  1 +
 lldb/source/Symbol/CMakeLists.txt             |  1 +
 lldb/source/Symbol/Variable.cpp               |  4 +-
 lldb/source/Target/ABI.cpp                    |  2 +-
 lldb/source/Target/CMakeLists.txt             |  1 +
 lldb/source/Target/StackFrame.cpp             |  6 +--
 lldb/source/Target/StopInfo.cpp               |  2 +-
 lldb/source/Target/Target.cpp                 |  4 +-
 lldb/source/Target/Thread.cpp                 |  4 +-
 lldb/source/Target/ThreadPlanStepOut.cpp      |  2 +-
 lldb/source/ValueObject/CMakeLists.txt        | 35 ++++++++++++++
 .../{Core => ValueObject}/ValueObject.cpp     | 47 ++++++++-----------
 .../{Core => ValueObject}/ValueObjectCast.cpp |  7 ++-
 .../ValueObjectChild.cpp                      | 12 ++---
 .../ValueObjectConstResult.cpp                | 14 +++---
 .../ValueObjectConstResultCast.cpp            |  2 +-
 .../ValueObjectConstResultChild.cpp           | 16 +++----
 .../ValueObjectConstResultImpl.cpp            | 13 +++--
 .../ValueObjectDynamicValue.cpp               |  6 +--
 .../{Core => ValueObject}/ValueObjectList.cpp |  4 +-
 .../ValueObjectMemory.cpp                     |  4 +-
 .../ValueObjectRegister.cpp                   |  2 +-
 .../ValueObjectSyntheticFilter.cpp            | 10 ++--
 .../ValueObjectUpdater.cpp                    |  2 +-
 .../ValueObjectVTable.cpp                     | 11 ++---
 .../ValueObjectVariable.cpp                   | 10 ++--
 lldb/unittests/ValueObject/CMakeLists.txt     |  1 +
 .../DumpValueObjectOptionsTests.cpp           |  4 +-
 195 files changed, 413 insertions(+), 380 deletions(-)
 rename lldb/include/lldb/{Core => ValueObject}/ValueObject.h (99%)
 rename lldb/include/lldb/{Core => ValueObject}/ValueObjectCast.h (86%)
 rename lldb/include/lldb/{Core => ValueObject}/ValueObjectChild.h (86%)
 rename lldb/include/lldb/{Core => ValueObject}/ValueObjectConstResult.h (90%)
 rename lldb/include/lldb/{Core => ValueObject}/ValueObjectConstResultCast.h (89%)
 rename lldb/include/lldb/{Core => ValueObject}/ValueObjectConstResultChild.h (74%)
 rename lldb/include/lldb/{Core => ValueObject}/ValueObjectConstResultImpl.h (92%)
 rename lldb/include/lldb/{Core => ValueObject}/ValueObjectDynamicValue.h (94%)
 rename lldb/include/lldb/{Core => ValueObject}/ValueObjectList.h (92%)
 rename lldb/include/lldb/{Core => ValueObject}/ValueObjectMemory.h (93%)
 rename lldb/include/lldb/{Core => ValueObject}/ValueObjectRegister.h (95%)
 rename lldb/include/lldb/{Core => ValueObject}/ValueObjectSyntheticFilter.h (96%)
 rename lldb/include/lldb/{Core => ValueObject}/ValueObjectUpdater.h (89%)
 rename lldb/include/lldb/{Core => ValueObject}/ValueObjectVTable.h (95%)
 rename lldb/include/lldb/{Core => ValueObject}/ValueObjectVariable.h (93%)
 create mode 100644 lldb/source/ValueObject/CMakeLists.txt
 rename lldb/source/{Core => ValueObject}/ValueObject.cpp (99%)
 rename lldb/source/{Core => ValueObject}/ValueObjectCast.cpp (95%)
 rename lldb/source/{Core => ValueObject}/ValueObjectChild.cpp (95%)
 rename lldb/source/{Core => ValueObject}/ValueObjectConstResult.cpp (96%)
 rename lldb/source/{Core => ValueObject}/ValueObjectConstResultCast.cpp (96%)
 rename lldb/source/{Core => ValueObject}/ValueObjectConstResultChild.cpp (80%)
 rename lldb/source/{Core => ValueObject}/ValueObjectConstResultImpl.cpp (96%)
 rename lldb/source/{Core => ValueObject}/ValueObjectDynamicValue.cpp (99%)
 rename lldb/source/{Core => ValueObject}/ValueObjectList.cpp (97%)
 rename lldb/source/{Core => ValueObject}/ValueObjectMemory.cpp (98%)
 rename lldb/source/{Core => ValueObject}/ValueObjectRegister.cpp (99%)
 rename lldb/source/{Core => ValueObject}/ValueObjectSyntheticFilter.cpp (98%)
 rename lldb/source/{Core => ValueObject}/ValueObjectUpdater.cpp (97%)
 rename lldb/source/{Core => ValueObject}/ValueObjectVTable.cpp (96%)
 rename lldb/source/{Core => ValueObject}/ValueObjectVariable.cpp (98%)

diff --git a/lldb/include/lldb/Core/IOHandler.h b/lldb/include/lldb/Core/IOHandler.h
index f10d63198aef..cb14d7241320 100644
--- a/lldb/include/lldb/Core/IOHandler.h
+++ b/lldb/include/lldb/Core/IOHandler.h
@@ -9,7 +9,6 @@
 #ifndef LLDB_CORE_IOHANDLER_H
 #define LLDB_CORE_IOHANDLER_H
 
-#include "lldb/Core/ValueObjectList.h"
 #include "lldb/Host/Config.h"
 #include "lldb/Utility/CompletionRequest.h"
 #include "lldb/Utility/Flags.h"
diff --git a/lldb/include/lldb/DataFormatters/FormattersContainer.h b/lldb/include/lldb/DataFormatters/FormattersContainer.h
index fd046e773b69..7898621fd18a 100644
--- a/lldb/include/lldb/DataFormatters/FormattersContainer.h
+++ b/lldb/include/lldb/DataFormatters/FormattersContainer.h
@@ -17,7 +17,6 @@
 
 #include "lldb/lldb-public.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/FormatClasses.h"
 #include "lldb/DataFormatters/TypeFormat.h"
 #include "lldb/DataFormatters/TypeSummary.h"
@@ -25,6 +24,7 @@
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/StringLexer.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 
diff --git a/lldb/include/lldb/DataFormatters/TypeFormat.h b/lldb/include/lldb/DataFormatters/TypeFormat.h
index 4e19d4cb14a1..63d4765bdf27 100644
--- a/lldb/include/lldb/DataFormatters/TypeFormat.h
+++ b/lldb/include/lldb/DataFormatters/TypeFormat.h
@@ -18,7 +18,7 @@
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-public.h"
 
-#include "lldb/Core/ValueObject.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 class TypeFormatImpl {
diff --git a/lldb/include/lldb/DataFormatters/TypeSynthetic.h b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
index ede7442a02bf..bf6dc6a0c3c6 100644
--- a/lldb/include/lldb/DataFormatters/TypeSynthetic.h
+++ b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
@@ -20,8 +20,8 @@
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-public.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Utility/StructuredData.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 class SyntheticChildrenFrontEnd {
diff --git a/lldb/include/lldb/Expression/ExpressionVariable.h b/lldb/include/lldb/Expression/ExpressionVariable.h
index 277b00a91668..fc36793b3a47 100644
--- a/lldb/include/lldb/Expression/ExpressionVariable.h
+++ b/lldb/include/lldb/Expression/ExpressionVariable.h
@@ -15,8 +15,8 @@
 
 #include "llvm/ADT/DenseMap.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-public.h"
 #include "llvm/Support/ExtensibleRTTI.h"
 
diff --git a/lldb/include/lldb/Interpreter/OptionGroupValueObjectDisplay.h b/lldb/include/lldb/Interpreter/OptionGroupValueObjectDisplay.h
index 9b545c46fba4..ebf26cea95cf 100644
--- a/lldb/include/lldb/Interpreter/OptionGroupValueObjectDisplay.h
+++ b/lldb/include/lldb/Interpreter/OptionGroupValueObjectDisplay.h
@@ -9,8 +9,8 @@
 #ifndef LLDB_INTERPRETER_OPTIONGROUPVALUEOBJECTDISPLAY_H
 #define LLDB_INTERPRETER_OPTIONGROUPVALUEOBJECTDISPLAY_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Interpreter/Options.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 
diff --git a/lldb/include/lldb/Target/LanguageRuntime.h b/lldb/include/lldb/Target/LanguageRuntime.h
index 954d454c7858..21bdc61b8cbc 100644
--- a/lldb/include/lldb/Target/LanguageRuntime.h
+++ b/lldb/include/lldb/Target/LanguageRuntime.h
@@ -14,11 +14,11 @@
 #include "lldb/Breakpoint/BreakpointResolverName.h"
 #include "lldb/Core/PluginInterface.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Expression/LLVMUserExpression.h"
 #include "lldb/Symbol/DeclVendor.h"
 #include "lldb/Target/ExecutionContextScope.h"
 #include "lldb/Target/Runtime.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-private.h"
 #include "lldb/lldb-public.h"
 #include <optional>
diff --git a/lldb/include/lldb/Target/StackFrame.h b/lldb/include/lldb/Target/StackFrame.h
index fdbe1f567eab..3881137583b9 100644
--- a/lldb/include/lldb/Target/StackFrame.h
+++ b/lldb/include/lldb/Target/StackFrame.h
@@ -16,7 +16,6 @@
 #include "lldb/Utility/Flags.h"
 
 #include "lldb/Core/FormatEntity.h"
-#include "lldb/Core/ValueObjectList.h"
 #include "lldb/Symbol/SymbolContext.h"
 #include "lldb/Target/ExecutionContextScope.h"
 #include "lldb/Target/StackID.h"
@@ -24,6 +23,7 @@
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/UserID.h"
+#include "lldb/ValueObject/ValueObjectList.h"
 
 namespace lldb_private {
 
diff --git a/lldb/include/lldb/Target/StackFrameRecognizer.h b/lldb/include/lldb/Target/StackFrameRecognizer.h
index 6c67a7fb4f68..8cf69a82eb8c 100644
--- a/lldb/include/lldb/Target/StackFrameRecognizer.h
+++ b/lldb/include/lldb/Target/StackFrameRecognizer.h
@@ -9,11 +9,11 @@
 #ifndef LLDB_TARGET_STACKFRAMERECOGNIZER_H
 #define LLDB_TARGET_STACKFRAMERECOGNIZER_H
 
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectList.h"
 #include "lldb/Symbol/VariableList.h"
 #include "lldb/Target/StopInfo.h"
 #include "lldb/Utility/StructuredData.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectList.h"
 #include "lldb/lldb-private-forward.h"
 #include "lldb/lldb-public.h"
 
diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/ValueObject/ValueObject.h
similarity index 99%
rename from lldb/include/lldb/Core/ValueObject.h
rename to lldb/include/lldb/ValueObject/ValueObject.h
index 93eb3e8f590f..4f77384bb8f1 100644
--- a/lldb/include/lldb/Core/ValueObject.h
+++ b/lldb/include/lldb/ValueObject/ValueObject.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_CORE_VALUEOBJECT_H
-#define LLDB_CORE_VALUEOBJECT_H
+#ifndef LLDB_VALUEOBJECT_VALUEOBJECT_H
+#define LLDB_VALUEOBJECT_VALUEOBJECT_H
 
 #include "lldb/Core/Value.h"
 #include "lldb/Symbol/CompilerType.h"
@@ -1004,8 +1004,8 @@ protected:
   } m_flags;
 
   friend class ValueObjectChild;
-  friend class ExpressionVariable;     // For SetName
-  friend class Target;                 // For SetName
+  friend class ExpressionVariable; // For SetName
+  friend class Target;             // For SetName
   friend class ValueObjectConstResultImpl;
   friend class ValueObjectSynthetic; // For ClearUserVisibleData
 
@@ -1079,7 +1079,7 @@ protected:
   void SetPreferredDisplayLanguageIfNeeded(lldb::LanguageType);
 
 protected:
-  virtual void DoUpdateChildrenAddressType(ValueObject &valobj){};
+  virtual void DoUpdateChildrenAddressType(ValueObject &valobj) {};
 
 private:
   virtual CompilerType MaybeCalculateCompleteType();
@@ -1100,4 +1100,4 @@ private:
 
 } // namespace lldb_private
 
-#endif // LLDB_CORE_VALUEOBJECT_H
+#endif // LLDB_VALUEOBJECT_VALUEOBJECT_H
diff --git a/lldb/include/lldb/Core/ValueObjectCast.h b/lldb/include/lldb/ValueObject/ValueObjectCast.h
similarity index 86%
rename from lldb/include/lldb/Core/ValueObjectCast.h
rename to lldb/include/lldb/ValueObject/ValueObjectCast.h
index ba25e166f326..9d174ae5ca60 100644
--- a/lldb/include/lldb/Core/ValueObjectCast.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectCast.h
@@ -6,11 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_CORE_VALUEOBJECTCAST_H
-#define LLDB_CORE_VALUEOBJECTCAST_H
+#ifndef LLDB_VALUEOBJECT_VALUEOBJECTCAST_H
+#define LLDB_VALUEOBJECT_VALUEOBJECTCAST_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Symbol/CompilerType.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
@@ -27,8 +27,7 @@ class ValueObjectCast : public ValueObject {
 public:
   ~ValueObjectCast() override;
 
-  static lldb::ValueObjectSP Create(ValueObject &parent,
-                                    ConstString name,
+  static lldb::ValueObjectSP Create(ValueObject &parent, ConstString name,
                                     const CompilerType &cast_type);
 
   std::optional<uint64_t> GetByteSize() override;
@@ -64,4 +63,4 @@ private:
 
 } // namespace lldb_private
 
-#endif // LLDB_CORE_VALUEOBJECTCAST_H
+#endif // LLDB_VALUEOBJECT_VALUEOBJECTCAST_H
diff --git a/lldb/include/lldb/Core/ValueObjectChild.h b/lldb/include/lldb/ValueObject/ValueObjectChild.h
similarity index 86%
rename from lldb/include/lldb/Core/ValueObjectChild.h
rename to lldb/include/lldb/ValueObject/ValueObjectChild.h
index 1f88e607cb57..e8c974a3a10a 100644
--- a/lldb/include/lldb/Core/ValueObjectChild.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectChild.h
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_CORE_VALUEOBJECTCHILD_H
-#define LLDB_CORE_VALUEOBJECTCHILD_H
+#ifndef LLDB_VALUEOBJECT_VALUEOBJECTCHILD_H
+#define LLDB_VALUEOBJECT_VALUEOBJECTCHILD_H
 
-#include "lldb/Core/ValueObject.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Utility/ConstString.h"
@@ -76,10 +76,9 @@ protected:
   friend class ValueObjectVTable;
 
   ValueObjectChild(ValueObject &parent, const CompilerType &compiler_type,
-                   ConstString name, uint64_t byte_size,
-                   int32_t byte_offset, uint32_t bitfield_bit_size,
-                   uint32_t bitfield_bit_offset, bool is_base_class,
-                   bool is_deref_of_parent,
+                   ConstString name, uint64_t byte_size, int32_t byte_offset,
+                   uint32_t bitfield_bit_size, uint32_t bitfield_bit_offset,
+                   bool is_base_class, bool is_deref_of_parent,
                    AddressType child_ptr_or_ref_addr_type,
                    uint64_t language_flags);
 
@@ -89,4 +88,4 @@ protected:
 
 } // namespace lldb_private
 
-#endif // LLDB_CORE_VALUEOBJECTCHILD_H
+#endif // LLDB_VALUEOBJECT_VALUEOBJECTCHILD_H
diff --git a/lldb/include/lldb/Core/ValueObjectConstResult.h b/lldb/include/lldb/ValueObject/ValueObjectConstResult.h
similarity index 90%
rename from lldb/include/lldb/Core/ValueObjectConstResult.h
rename to lldb/include/lldb/ValueObject/ValueObjectConstResult.h
index 9c34617af71d..e4ed1f399bf6 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResult.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectConstResult.h
@@ -6,15 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_CORE_VALUEOBJECTCONSTRESULT_H
-#define LLDB_CORE_VALUEOBJECTCONSTRESULT_H
+#ifndef LLDB_VALUEOBJECT_VALUEOBJECTCONSTRESULT_H
+#define LLDB_VALUEOBJECT_VALUEOBJECTCONSTRESULT_H
 
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResultImpl.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResultImpl.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
@@ -50,10 +50,11 @@ public:
          lldb::ByteOrder byte_order, uint32_t addr_size,
          lldb::addr_t address = LLDB_INVALID_ADDRESS);
 
-  static lldb::ValueObjectSP
-  Create(ExecutionContextScope *exe_scope, const CompilerType &compiler_type,
-         ConstString name, lldb::addr_t address,
-         AddressType address_type, uint32_t addr_byte_size);
+  static lldb::ValueObjectSP Create(ExecutionContextScope *exe_scope,
+                                    const CompilerType &compiler_type,
+                                    ConstString name, lldb::addr_t address,
+                                    AddressType address_type,
+                                    uint32_t addr_byte_size);
 
   static lldb::ValueObjectSP Create(ExecutionContextScope *exe_scope,
                                     Value &value, ConstString name,
@@ -162,4 +163,4 @@ private:
 
 } // namespace lldb_private
 
-#endif // LLDB_CORE_VALUEOBJECTCONSTRESULT_H
+#endif // LLDB_VALUEOBJECT_VALUEOBJECTCONSTRESULT_H
diff --git a/lldb/include/lldb/Core/ValueObjectConstResultCast.h b/lldb/include/lldb/ValueObject/ValueObjectConstResultCast.h
similarity index 89%
rename from lldb/include/lldb/Core/ValueObjectConstResultCast.h
rename to lldb/include/lldb/ValueObject/ValueObjectConstResultCast.h
index 911a08363b39..e786467eae6e 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResultCast.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectConstResultCast.h
@@ -6,13 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_CORE_VALUEOBJECTCONSTRESULTCAST_H
-#define LLDB_CORE_VALUEOBJECTCONSTRESULTCAST_H
+#ifndef LLDB_VALUEOBJECT_VALUEOBJECTCONSTRESULTCAST_H
+#define LLDB_VALUEOBJECT_VALUEOBJECTCONSTRESULTCAST_H
 
-#include "lldb/Core/ValueObjectCast.h"
-#include "lldb/Core/ValueObjectConstResultImpl.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObjectCast.h"
+#include "lldb/ValueObject/ValueObjectConstResultImpl.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-types.h"
@@ -72,4 +72,4 @@ private:
 
 } // namespace lldb_private
 
-#endif // LLDB_CORE_VALUEOBJECTCONSTRESULTCAST_H
+#endif // LLDB_VALUEOBJECT_VALUEOBJECTCONSTRESULTCAST_H
diff --git a/lldb/include/lldb/Core/ValueObjectConstResultChild.h b/lldb/include/lldb/ValueObject/ValueObjectConstResultChild.h
similarity index 74%
rename from lldb/include/lldb/Core/ValueObjectConstResultChild.h
rename to lldb/include/lldb/ValueObject/ValueObjectConstResultChild.h
index 71a3c53befe7..ad97b885684e 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResultChild.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectConstResultChild.h
@@ -6,13 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_CORE_VALUEOBJECTCONSTRESULTCHILD_H
-#define LLDB_CORE_VALUEOBJECTCONSTRESULTCHILD_H
+#ifndef LLDB_VALUEOBJECT_VALUEOBJECTCONSTRESULTCHILD_H
+#define LLDB_VALUEOBJECT_VALUEOBJECTCONSTRESULTCHILD_H
 
-#include "lldb/Core/ValueObjectChild.h"
-#include "lldb/Core/ValueObjectConstResultImpl.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObjectChild.h"
+#include "lldb/ValueObject/ValueObjectConstResultImpl.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-types.h"
@@ -28,14 +28,11 @@ class ValueObject;
 // A child of a ValueObjectConstResult.
 class ValueObjectConstResultChild : public ValueObjectChild {
 public:
-  ValueObjectConstResultChild(ValueObject &parent,
-                              const CompilerType &compiler_type,
-                              ConstString name, uint32_t byte_size,
-                              int32_t byte_offset, uint32_t bitfield_bit_size,
-                              uint32_t bitfield_bit_offset, bool is_base_class,
-                              bool is_deref_of_parent,
-                              lldb::addr_t live_address,
-                              uint64_t language_flags);
+  ValueObjectConstResultChild(
+      ValueObject &parent, const CompilerType &compiler_type, ConstString name,
+      uint32_t byte_size, int32_t byte_offset, uint32_t bitfield_bit_size,
+      uint32_t bitfield_bit_offset, bool is_base_class, bool is_deref_of_parent,
+      lldb::addr_t live_address, uint64_t language_flags);
 
   ~ValueObjectConstResultChild() override;
 
@@ -81,4 +78,4 @@ private:
 
 } // namespace lldb_private
 
-#endif // LLDB_CORE_VALUEOBJECTCONSTRESULTCHILD_H
+#endif // LLDB_VALUEOBJECT_VALUEOBJECTCONSTRESULTCHILD_H
diff --git a/lldb/include/lldb/Core/ValueObjectConstResultImpl.h b/lldb/include/lldb/ValueObject/ValueObjectConstResultImpl.h
similarity index 92%
rename from lldb/include/lldb/Core/ValueObjectConstResultImpl.h
rename to lldb/include/lldb/ValueObject/ValueObjectConstResultImpl.h
index 68ba8ae7fba2..dbd68160acb4 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResultImpl.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectConstResultImpl.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_CORE_VALUEOBJECTCONSTRESULTIMPL_H
-#define LLDB_CORE_VALUEOBJECTCONSTRESULTIMPL_H
+#ifndef LLDB_VALUEOBJECT_VALUEOBJECTCONSTRESULTIMPL_H
+#define LLDB_VALUEOBJECT_VALUEOBJECTCONSTRESULTIMPL_H
 
 #include "lldb/Utility/ConstString.h"
 #include "lldb/lldb-defines.h"
@@ -22,7 +22,7 @@ class CompilerType;
 class DataExtractor;
 class Status;
 class ValueObject;
-}
+} // namespace lldb_private
 
 namespace lldb_private {
 
@@ -77,4 +77,4 @@ private:
 
 } // namespace lldb_private
 
-#endif // LLDB_CORE_VALUEOBJECTCONSTRESULTIMPL_H
+#endif // LLDB_VALUEOBJECT_VALUEOBJECTCONSTRESULTIMPL_H
diff --git a/lldb/include/lldb/Core/ValueObjectDynamicValue.h b/lldb/include/lldb/ValueObject/ValueObjectDynamicValue.h
similarity index 94%
rename from lldb/include/lldb/Core/ValueObjectDynamicValue.h
rename to lldb/include/lldb/ValueObject/ValueObjectDynamicValue.h
index 82c20eee0cd4..145a46e29556 100644
--- a/lldb/include/lldb/Core/ValueObjectDynamicValue.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectDynamicValue.h
@@ -6,14 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_CORE_VALUEOBJECTDYNAMICVALUE_H
-#define LLDB_CORE_VALUEOBJECTDYNAMICVALUE_H
+#ifndef LLDB_VALUEOBJECT_VALUEOBJECTDYNAMICVALUE_H
+#define LLDB_VALUEOBJECT_VALUEOBJECTDYNAMICVALUE_H
 
 #include "lldb/Core/Address.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Symbol/Type.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
@@ -126,4 +126,4 @@ private:
 
 } // namespace lldb_private
 
-#endif // LLDB_CORE_VALUEOBJECTDYNAMICVALUE_H
+#endif // LLDB_VALUEOBJECT_VALUEOBJECTDYNAMICVALUE_H
diff --git a/lldb/include/lldb/Core/ValueObjectList.h b/lldb/include/lldb/ValueObject/ValueObjectList.h
similarity index 92%
rename from lldb/include/lldb/Core/ValueObjectList.h
rename to lldb/include/lldb/ValueObject/ValueObjectList.h
index fcb358e21a26..5d63c65a96e5 100644
--- a/lldb/include/lldb/Core/ValueObjectList.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectList.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_CORE_VALUEOBJECTLIST_H
-#define LLDB_CORE_VALUEOBJECTLIST_H
+#ifndef LLDB_VALUEOBJECT_VALUEOBJECTLIST_H
+#define LLDB_VALUEOBJECT_VALUEOBJECTLIST_H
 
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-types.h"
@@ -51,6 +51,7 @@ public:
   const std::vector<lldb::ValueObjectSP> &GetObjects() const {
     return m_value_objects;
   }
+
 protected:
   typedef std::vector<lldb::ValueObjectSP> collection;
   // Classes that inherit from ValueObjectList can see and modify these
@@ -59,4 +60,4 @@ protected:
 
 } // namespace lldb_private
 
-#endif // LLDB_CORE_VALUEOBJECTLIST_H
+#endif // LLDB_VALUEOBJECT_VALUEOBJECTLIST_H
diff --git a/lldb/include/lldb/Core/ValueObjectMemory.h b/lldb/include/lldb/ValueObject/ValueObjectMemory.h
similarity index 93%
rename from lldb/include/lldb/Core/ValueObjectMemory.h
rename to lldb/include/lldb/ValueObject/ValueObjectMemory.h
index a8fb0353d601..cfc36d0d610d 100644
--- a/lldb/include/lldb/Core/ValueObjectMemory.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectMemory.h
@@ -6,13 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_CORE_VALUEOBJECTMEMORY_H
-#define LLDB_CORE_VALUEOBJECTMEMORY_H
+#ifndef LLDB_VALUEOBJECT_VALUEOBJECTMEMORY_H
+#define LLDB_VALUEOBJECT_VALUEOBJECTMEMORY_H
 
 #include "lldb/Core/Address.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
@@ -79,4 +79,4 @@ private:
 
 } // namespace lldb_private
 
-#endif // LLDB_CORE_VALUEOBJECTMEMORY_H
+#endif // LLDB_VALUEOBJECT_VALUEOBJECTMEMORY_H
diff --git a/lldb/include/lldb/Core/ValueObjectRegister.h b/lldb/include/lldb/ValueObject/ValueObjectRegister.h
similarity index 95%
rename from lldb/include/lldb/Core/ValueObjectRegister.h
rename to lldb/include/lldb/ValueObject/ValueObjectRegister.h
index d948c663a4f8..fafbfd034111 100644
--- a/lldb/include/lldb/Core/ValueObjectRegister.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectRegister.h
@@ -6,13 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_CORE_VALUEOBJECTREGISTER_H
-#define LLDB_CORE_VALUEOBJECTREGISTER_H
+#ifndef LLDB_VALUEOBJECT_VALUEOBJECTREGISTER_H
+#define LLDB_VALUEOBJECT_VALUEOBJECTREGISTER_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/RegisterValue.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
@@ -140,4 +140,4 @@ private:
 
 } // namespace lldb_private
 
-#endif // LLDB_CORE_VALUEOBJECTREGISTER_H
+#endif // LLDB_VALUEOBJECT_VALUEOBJECTREGISTER_H
diff --git a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h b/lldb/include/lldb/ValueObject/ValueObjectSyntheticFilter.h
similarity index 96%
rename from lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
rename to lldb/include/lldb/ValueObject/ValueObjectSyntheticFilter.h
index ca6d6c728005..2811658fd8f1 100644
--- a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectSyntheticFilter.h
@@ -6,12 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_CORE_VALUEOBJECTSYNTHETICFILTER_H
-#define LLDB_CORE_VALUEOBJECTSYNTHETICFILTER_H
+#ifndef LLDB_VALUEOBJECT_VALUEOBJECTSYNTHETICFILTER_H
+#define LLDB_VALUEOBJECT_VALUEOBJECTSYNTHETICFILTER_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
@@ -176,4 +176,4 @@ private:
 
 } // namespace lldb_private
 
-#endif // LLDB_CORE_VALUEOBJECTSYNTHETICFILTER_H
+#endif // LLDB_VALUEOBJECT_VALUEOBJECTSYNTHETICFILTER_H
diff --git a/lldb/include/lldb/Core/ValueObjectUpdater.h b/lldb/include/lldb/ValueObject/ValueObjectUpdater.h
similarity index 89%
rename from lldb/include/lldb/Core/ValueObjectUpdater.h
rename to lldb/include/lldb/ValueObject/ValueObjectUpdater.h
index 54fcb31076ad..3f1d887b47a7 100644
--- a/lldb/include/lldb/Core/ValueObjectUpdater.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectUpdater.h
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_CORE_VALUEOBJECTUPDATER_H
-#define LLDB_CORE_VALUEOBJECTUPDATER_H
+#ifndef LLDB_VALUEOBJECT_VALUEOBJECTUPDATER_H
+#define LLDB_VALUEOBJECT_VALUEOBJECTUPDATER_H
 
-#include "lldb/Core/ValueObject.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 
@@ -40,4 +40,4 @@ public:
 
 } // namespace lldb_private
 
-#endif // LLDB_CORE_VALUEOBJECTUPDATER_H
+#endif // LLDB_VALUEOBJECT_VALUEOBJECTUPDATER_H
diff --git a/lldb/include/lldb/Core/ValueObjectVTable.h b/lldb/include/lldb/ValueObject/ValueObjectVTable.h
similarity index 95%
rename from lldb/include/lldb/Core/ValueObjectVTable.h
rename to lldb/include/lldb/ValueObject/ValueObjectVTable.h
index 7087dcc1d1be..9cf13be093a8 100644
--- a/lldb/include/lldb/Core/ValueObjectVTable.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectVTable.h
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_CORE_VALUEOBJECTVTABLE_H
-#define LLDB_CORE_VALUEOBJECTVTABLE_H
+#ifndef LLDB_VALUEOBJECT_VALUEOBJECTVTABLE_H
+#define LLDB_VALUEOBJECT_VALUEOBJECTVTABLE_H
 
-#include "lldb/Core/ValueObject.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 
@@ -104,4 +104,4 @@ private:
 
 } // namespace lldb_private
 
-#endif // LLDB_CORE_VALUEOBJECTVTABLE_H
+#endif // LLDB_VALUEOBJECT_VALUEOBJECTVTABLE_H
diff --git a/lldb/include/lldb/Core/ValueObjectVariable.h b/lldb/include/lldb/ValueObject/ValueObjectVariable.h
similarity index 93%
rename from lldb/include/lldb/Core/ValueObjectVariable.h
rename to lldb/include/lldb/ValueObject/ValueObjectVariable.h
index db3847f14a0b..9f66af808425 100644
--- a/lldb/include/lldb/Core/ValueObjectVariable.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectVariable.h
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_CORE_VALUEOBJECTVARIABLE_H
-#define LLDB_CORE_VALUEOBJECTVARIABLE_H
+#ifndef LLDB_VALUEOBJECT_VALUEOBJECTVARIABLE_H
+#define LLDB_VALUEOBJECT_VALUEOBJECTVARIABLE_H
 
-#include "lldb/Core/ValueObject.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include "lldb/Core/Value.h"
 #include "lldb/Symbol/CompilerType.h"
@@ -68,7 +68,7 @@ public:
 
 protected:
   bool UpdateValue() override;
-  
+
   void DoUpdateChildrenAddressType(ValueObject &valobj) override;
 
   CompilerType GetCompilerTypeImpl() override;
@@ -90,4 +90,4 @@ private:
 
 } // namespace lldb_private
 
-#endif // LLDB_CORE_VALUEOBJECTVARIABLE_H
+#endif // LLDB_VALUEOBJECT_VALUEOBJECTVARIABLE_H
diff --git a/lldb/source/API/CMakeLists.txt b/lldb/source/API/CMakeLists.txt
index a32bc58507d8..d8308841c05d 100644
--- a/lldb/source/API/CMakeLists.txt
+++ b/lldb/source/API/CMakeLists.txt
@@ -137,6 +137,7 @@ add_lldb_library(liblldb SHARED ${option_framework}
     lldbSymbol
     lldbTarget
     lldbUtility
+    lldbValueObject
     lldbVersion
     ${LLDB_ALL_PLUGINS}
   LINK_COMPONENTS
diff --git a/lldb/source/API/SBBlock.cpp b/lldb/source/API/SBBlock.cpp
index 2577b14920f0..b921ccd98024 100644
--- a/lldb/source/API/SBBlock.cpp
+++ b/lldb/source/API/SBBlock.cpp
@@ -14,7 +14,6 @@
 #include "lldb/API/SBValue.h"
 #include "lldb/Core/AddressRange.h"
 #include "lldb/Core/AddressRangeListImpl.h"
-#include "lldb/Core/ValueObjectVariable.h"
 #include "lldb/Symbol/Block.h"
 #include "lldb/Symbol/Function.h"
 #include "lldb/Symbol/SymbolContext.h"
@@ -22,6 +21,7 @@
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/Instrumentation.h"
+#include "lldb/ValueObject/ValueObjectVariable.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/API/SBFrame.cpp b/lldb/source/API/SBFrame.cpp
index 30c44b974988..dc41e80b457d 100644
--- a/lldb/source/API/SBFrame.cpp
+++ b/lldb/source/API/SBFrame.cpp
@@ -17,9 +17,6 @@
 #include "Utils.h"
 #include "lldb/Core/Address.h"
 #include "lldb/Core/Debugger.h"
-#include "lldb/Core/ValueObjectRegister.h"
-#include "lldb/Core/ValueObjectVariable.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Expression/ExpressionVariable.h"
 #include "lldb/Expression/UserExpression.h"
 #include "lldb/Host/Host.h"
@@ -41,6 +38,9 @@
 #include "lldb/Utility/Instrumentation.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectRegister.h"
+#include "lldb/ValueObject/ValueObjectVariable.h"
 
 #include "lldb/API/SBAddress.h"
 #include "lldb/API/SBDebugger.h"
diff --git a/lldb/source/API/SBModule.cpp b/lldb/source/API/SBModule.cpp
index 262e26c6bf43..985107ec68ef 100644
--- a/lldb/source/API/SBModule.cpp
+++ b/lldb/source/API/SBModule.cpp
@@ -15,8 +15,6 @@
 #include "lldb/API/SBSymbolContextList.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/Section.h"
-#include "lldb/Core/ValueObjectList.h"
-#include "lldb/Core/ValueObjectVariable.h"
 #include "lldb/Symbol/ObjectFile.h"
 #include "lldb/Symbol/SymbolFile.h"
 #include "lldb/Symbol/Symtab.h"
@@ -25,6 +23,8 @@
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/Instrumentation.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObjectList.h"
+#include "lldb/ValueObject/ValueObjectVariable.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/API/SBTarget.cpp b/lldb/source/API/SBTarget.cpp
index d5017ad6bff1..28bdf47a3413 100644
--- a/lldb/source/API/SBTarget.cpp
+++ b/lldb/source/API/SBTarget.cpp
@@ -41,9 +41,6 @@
 #include "lldb/Core/SearchFilter.h"
 #include "lldb/Core/Section.h"
 #include "lldb/Core/StructuredDataImpl.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectList.h"
-#include "lldb/Core/ValueObjectVariable.h"
 #include "lldb/Host/Host.h"
 #include "lldb/Symbol/DeclVendor.h"
 #include "lldb/Symbol/ObjectFile.h"
@@ -63,6 +60,9 @@
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/ProcessInfo.h"
 #include "lldb/Utility/RegularExpression.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectList.h"
+#include "lldb/ValueObject/ValueObjectVariable.h"
 
 #include "Commands/CommandObjectBreakpoint.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
diff --git a/lldb/source/API/SBThread.cpp b/lldb/source/API/SBThread.cpp
index a99456e06d03..4e61c83889b0 100644
--- a/lldb/source/API/SBThread.cpp
+++ b/lldb/source/API/SBThread.cpp
@@ -24,7 +24,6 @@
 #include "lldb/Breakpoint/BreakpointLocation.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/StructuredDataImpl.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Symbol/CompileUnit.h"
 #include "lldb/Symbol/SymbolContext.h"
@@ -43,6 +42,7 @@
 #include "lldb/Utility/State.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StructuredData.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-enumerations.h"
 
 #include <memory>
diff --git a/lldb/source/API/SBType.cpp b/lldb/source/API/SBType.cpp
index 8a063e5ad61d..4cc16c64e475 100644
--- a/lldb/source/API/SBType.cpp
+++ b/lldb/source/API/SBType.cpp
@@ -13,7 +13,6 @@
 #include "lldb/API/SBStream.h"
 #include "lldb/API/SBTypeEnumMember.h"
 #include "lldb/Core/Mangled.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Symbol/CompilerDecl.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Symbol/Type.h"
@@ -23,6 +22,7 @@
 #include "lldb/Utility/Instrumentation.h"
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "llvm/ADT/APSInt.h"
 #include "llvm/Support/MathExtras.h"
diff --git a/lldb/source/API/SBValue.cpp b/lldb/source/API/SBValue.cpp
index e1a31708d46f..b35c82250d6b 100644
--- a/lldb/source/API/SBValue.cpp
+++ b/lldb/source/API/SBValue.cpp
@@ -21,8 +21,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/Section.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/DataVisualization.h"
 #include "lldb/DataFormatters/DumpValueObjectOptions.h"
 #include "lldb/Symbol/Block.h"
@@ -38,6 +36,8 @@
 #include "lldb/Utility/DataExtractor.h"
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "lldb/API/SBDebugger.h"
 #include "lldb/API/SBExpressionOptions.h"
diff --git a/lldb/source/API/SBValueList.cpp b/lldb/source/API/SBValueList.cpp
index 63b915094baf..0a90ea66f974 100644
--- a/lldb/source/API/SBValueList.cpp
+++ b/lldb/source/API/SBValueList.cpp
@@ -10,9 +10,9 @@
 #include "lldb/API/SBError.h"
 #include "lldb/API/SBStream.h"
 #include "lldb/API/SBValue.h"
-#include "lldb/Core/ValueObjectList.h"
 #include "lldb/Utility/Instrumentation.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectList.h"
 #include <vector>
 
 using namespace lldb;
diff --git a/lldb/source/Breakpoint/BreakpointLocation.cpp b/lldb/source/Breakpoint/BreakpointLocation.cpp
index 35058a713aef..ad9057c8141e 100644
--- a/lldb/source/Breakpoint/BreakpointLocation.cpp
+++ b/lldb/source/Breakpoint/BreakpointLocation.cpp
@@ -11,7 +11,6 @@
 #include "lldb/Breakpoint/StoppointCallbackContext.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/Module.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Expression/DiagnosticManager.h"
 #include "lldb/Expression/ExpressionVariable.h"
 #include "lldb/Expression/UserExpression.h"
@@ -25,6 +24,7 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/Breakpoint/CMakeLists.txt b/lldb/source/Breakpoint/CMakeLists.txt
index 2fa659f803c2..6cd3c396a2c5 100644
--- a/lldb/source/Breakpoint/CMakeLists.txt
+++ b/lldb/source/Breakpoint/CMakeLists.txt
@@ -33,6 +33,7 @@ add_lldb_library(lldbBreakpoint NO_PLUGIN_DEPENDENCIES
     lldbSymbol
     lldbTarget
     lldbUtility
+    lldbValueObject
 
   LINK_COMPONENTS
     Support
diff --git a/lldb/source/Breakpoint/Watchpoint.cpp b/lldb/source/Breakpoint/Watchpoint.cpp
index 715e83c76697..2df848aaa057 100644
--- a/lldb/source/Breakpoint/Watchpoint.cpp
+++ b/lldb/source/Breakpoint/Watchpoint.cpp
@@ -11,8 +11,6 @@
 #include "lldb/Breakpoint/StoppointCallbackContext.h"
 #include "lldb/Breakpoint/WatchpointResource.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectMemory.h"
 #include "lldb/DataFormatters/DumpValueObjectOptions.h"
 #include "lldb/Expression/UserExpression.h"
 #include "lldb/Symbol/TypeSystem.h"
@@ -22,6 +20,8 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/CMakeLists.txt b/lldb/source/CMakeLists.txt
index 54e790358066..51c9f9c90826 100644
--- a/lldb/source/CMakeLists.txt
+++ b/lldb/source/CMakeLists.txt
@@ -13,6 +13,7 @@ add_subdirectory(Plugins)
 add_subdirectory(Symbol)
 add_subdirectory(Target)
 add_subdirectory(Utility)
+add_subdirectory(ValueObject)
 add_subdirectory(Version)
 
 # Build API last.  Since liblldb needs to link against every other target, it needs
diff --git a/lldb/source/Commands/CMakeLists.txt b/lldb/source/Commands/CMakeLists.txt
index 76397227d535..71339ccd3a6e 100644
--- a/lldb/source/Commands/CMakeLists.txt
+++ b/lldb/source/Commands/CMakeLists.txt
@@ -53,6 +53,7 @@ add_lldb_library(lldbCommands NO_PLUGIN_DEPENDENCIES
     lldbSymbol
     lldbTarget
     lldbUtility
+    lldbValueObject
     lldbVersion
 
   LINK_COMPONENTS
diff --git a/lldb/source/Commands/CommandObjectDWIMPrint.cpp b/lldb/source/Commands/CommandObjectDWIMPrint.cpp
index ddf0aca25fb3..f5aa6a287b6f 100644
--- a/lldb/source/Commands/CommandObjectDWIMPrint.cpp
+++ b/lldb/source/Commands/CommandObjectDWIMPrint.cpp
@@ -8,7 +8,6 @@
 
 #include "CommandObjectDWIMPrint.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/DumpValueObjectOptions.h"
 #include "lldb/Expression/ExpressionVariable.h"
 #include "lldb/Expression/UserExpression.h"
@@ -19,6 +18,7 @@
 #include "lldb/Interpreter/OptionGroupValueObjectDisplay.h"
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
diff --git a/lldb/source/Commands/CommandObjectFrame.cpp b/lldb/source/Commands/CommandObjectFrame.cpp
index e2203292e71e..a5709b36f52e 100644
--- a/lldb/source/Commands/CommandObjectFrame.cpp
+++ b/lldb/source/Commands/CommandObjectFrame.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 #include "CommandObjectFrame.h"
 #include "lldb/Core/Debugger.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/DataVisualization.h"
 #include "lldb/DataFormatters/ValueObjectPrinter.h"
 #include "lldb/Host/Config.h"
@@ -30,6 +29,7 @@
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
 #include "lldb/Utility/Args.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include <memory>
 #include <optional>
diff --git a/lldb/source/Commands/CommandObjectMemory.cpp b/lldb/source/Commands/CommandObjectMemory.cpp
index f93081d3f45a..b5612f21f115 100644
--- a/lldb/source/Commands/CommandObjectMemory.cpp
+++ b/lldb/source/Commands/CommandObjectMemory.cpp
@@ -10,7 +10,6 @@
 #include "CommandObjectMemoryTag.h"
 #include "lldb/Core/DumpDataExtractor.h"
 #include "lldb/Core/Section.h"
-#include "lldb/Core/ValueObjectMemory.h"
 #include "lldb/Expression/ExpressionVariable.h"
 #include "lldb/Host/OptionParser.h"
 #include "lldb/Interpreter/CommandOptionArgumentTable.h"
@@ -36,6 +35,7 @@
 #include "lldb/Utility/Args.h"
 #include "lldb/Utility/DataBufferHeap.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
 #include "llvm/Support/MathExtras.h"
 #include <cinttypes>
 #include <memory>
diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index e950fb346c25..6c870bbe3312 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -15,7 +15,6 @@
 #include "lldb/Core/ModuleSpec.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Section.h"
-#include "lldb/Core/ValueObjectVariable.h"
 #include "lldb/DataFormatters/ValueObjectPrinter.h"
 #include "lldb/Host/OptionParser.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
@@ -56,6 +55,7 @@
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StructuredData.h"
 #include "lldb/Utility/Timer.h"
+#include "lldb/ValueObject/ValueObjectVariable.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-private-enumerations.h"
 
diff --git a/lldb/source/Commands/CommandObjectThread.cpp b/lldb/source/Commands/CommandObjectThread.cpp
index 2e2b0edeec0f..4e2c4c1126bc 100644
--- a/lldb/source/Commands/CommandObjectThread.cpp
+++ b/lldb/source/Commands/CommandObjectThread.cpp
@@ -15,7 +15,6 @@
 #include "CommandObjectThreadUtil.h"
 #include "CommandObjectTrace.h"
 #include "lldb/Core/PluginManager.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Host/OptionParser.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/CommandOptionArgumentTable.h"
@@ -37,6 +36,7 @@
 #include "lldb/Target/Trace.h"
 #include "lldb/Target/TraceDumper.h"
 #include "lldb/Utility/State.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/Commands/CommandObjectWatchpoint.cpp b/lldb/source/Commands/CommandObjectWatchpoint.cpp
index f8f1fac62002..766d650a2ca0 100644
--- a/lldb/source/Commands/CommandObjectWatchpoint.cpp
+++ b/lldb/source/Commands/CommandObjectWatchpoint.cpp
@@ -16,7 +16,6 @@
 
 #include "lldb/Breakpoint/Watchpoint.h"
 #include "lldb/Breakpoint/WatchpointList.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Host/OptionParser.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/CommandOptionArgumentTable.h"
@@ -27,6 +26,7 @@
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/Core/CMakeLists.txt b/lldb/source/Core/CMakeLists.txt
index dbc620b91b1e..6d14f7a87764 100644
--- a/lldb/source/Core/CMakeLists.txt
+++ b/lldb/source/Core/CMakeLists.txt
@@ -55,21 +55,6 @@ add_lldb_library(lldbCore
   ThreadedCommunication.cpp
   UserSettingsController.cpp
   Value.cpp
-  ValueObject.cpp
-  ValueObjectCast.cpp
-  ValueObjectChild.cpp
-  ValueObjectConstResult.cpp
-  ValueObjectConstResultCast.cpp
-  ValueObjectConstResultChild.cpp
-  ValueObjectConstResultImpl.cpp
-  ValueObjectDynamicValue.cpp
-  ValueObjectList.cpp
-  ValueObjectMemory.cpp
-  ValueObjectRegister.cpp
-  ValueObjectSyntheticFilter.cpp
-  ValueObjectUpdater.cpp
-  ValueObjectVariable.cpp
-  ValueObjectVTable.cpp
 
   DEPENDS
     clang-tablegen-targets
@@ -83,6 +68,7 @@ add_lldb_library(lldbCore
     lldbSymbol
     lldbTarget
     lldbUtility
+    lldbValueObject
     lldbPluginCPlusPlusLanguage
     lldbPluginObjCLanguage
     ${LLDB_CURSES_LIBS}
diff --git a/lldb/source/Core/DumpRegisterValue.cpp b/lldb/source/Core/DumpRegisterValue.cpp
index 90b31fd0e865..aff4d2c621d7 100644
--- a/lldb/source/Core/DumpRegisterValue.cpp
+++ b/lldb/source/Core/DumpRegisterValue.cpp
@@ -8,14 +8,14 @@
 
 #include "lldb/Core/DumpRegisterValue.h"
 #include "lldb/Core/DumpDataExtractor.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/DumpValueObjectOptions.h"
 #include "lldb/Target/RegisterFlags.h"
 #include "lldb/Utility/DataExtractor.h"
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 #include "lldb/lldb-private-types.h"
 #include "llvm/ADT/bit.h"
 
diff --git a/lldb/source/Core/FormatEntity.cpp b/lldb/source/Core/FormatEntity.cpp
index 8f28cd4b5fb2..36214c173af6 100644
--- a/lldb/source/Core/FormatEntity.cpp
+++ b/lldb/source/Core/FormatEntity.cpp
@@ -13,8 +13,6 @@
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/DumpRegisterValue.h"
 #include "lldb/Core/Module.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectVariable.h"
 #include "lldb/DataFormatters/DataVisualization.h"
 #include "lldb/DataFormatters/FormatClasses.h"
 #include "lldb/DataFormatters/FormatManager.h"
@@ -52,6 +50,8 @@
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/StringList.h"
 #include "lldb/Utility/StructuredData.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectVariable.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-forward.h"
 #include "llvm/ADT/STLExtras.h"
diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index 3d69aedb6b13..456ce7d16e10 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -25,13 +25,13 @@
 #include <string>
 
 #include "lldb/Core/Debugger.h"
-#include "lldb/Core/ValueObjectUpdater.h"
 #include "lldb/Host/File.h"
 #include "lldb/Utility/AnsiTerminal.h"
 #include "lldb/Utility/Predicate.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/StringList.h"
+#include "lldb/ValueObject/ValueObjectUpdater.h"
 #include "lldb/lldb-forward.h"
 
 #include "lldb/Interpreter/CommandCompletions.h"
@@ -42,8 +42,6 @@
 #include "lldb/Breakpoint/BreakpointLocation.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectRegister.h"
 #include "lldb/Symbol/Block.h"
 #include "lldb/Symbol/CompileUnit.h"
 #include "lldb/Symbol/Function.h"
@@ -56,6 +54,8 @@
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
 #include "lldb/Utility/State.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectRegister.h"
 #endif
 
 #include "llvm/ADT/StringRef.h"
diff --git a/lldb/source/DataFormatters/CMakeLists.txt b/lldb/source/DataFormatters/CMakeLists.txt
index 7f48a2785c73..13faf65227d2 100644
--- a/lldb/source/DataFormatters/CMakeLists.txt
+++ b/lldb/source/DataFormatters/CMakeLists.txt
@@ -22,7 +22,8 @@ add_lldb_library(lldbDataFormatters NO_PLUGIN_DEPENDENCIES
     lldbSymbol
     lldbTarget
     lldbUtility
-  
+    lldbValueObject
+
   LINK_COMPONENTS
     Support
   )
diff --git a/lldb/source/DataFormatters/CXXFunctionPointer.cpp b/lldb/source/DataFormatters/CXXFunctionPointer.cpp
index 6543433d17ff..6d56e39fa973 100644
--- a/lldb/source/DataFormatters/CXXFunctionPointer.cpp
+++ b/lldb/source/DataFormatters/CXXFunctionPointer.cpp
@@ -8,11 +8,11 @@
 
 #include "lldb/DataFormatters/CXXFunctionPointer.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Target/ABI.h"
 #include "lldb/Target/SectionLoadList.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-enumerations.h"
 
 #include <string>
diff --git a/lldb/source/DataFormatters/DumpValueObjectOptions.cpp b/lldb/source/DataFormatters/DumpValueObjectOptions.cpp
index c5e84810cdc8..18d590d47d9a 100644
--- a/lldb/source/DataFormatters/DumpValueObjectOptions.cpp
+++ b/lldb/source/DataFormatters/DumpValueObjectOptions.cpp
@@ -8,7 +8,7 @@
 
 #include "lldb/DataFormatters/DumpValueObjectOptions.h"
 
-#include "lldb/Core/ValueObject.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/DataFormatters/FormatManager.cpp b/lldb/source/DataFormatters/FormatManager.cpp
index 7e19989a8264..3b891cecb1c8 100644
--- a/lldb/source/DataFormatters/FormatManager.cpp
+++ b/lldb/source/DataFormatters/FormatManager.cpp
@@ -9,7 +9,6 @@
 #include "lldb/DataFormatters/FormatManager.h"
 
 #include "lldb/Core/Debugger.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/DataFormatters/LanguageCategory.h"
 #include "lldb/Interpreter/ScriptInterpreter.h"
@@ -17,6 +16,7 @@
 #include "lldb/Target/Language.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "llvm/ADT/STLExtras.h"
 
 using namespace lldb;
diff --git a/lldb/source/DataFormatters/StringPrinter.cpp b/lldb/source/DataFormatters/StringPrinter.cpp
index ab07c74fd185..257178425017 100644
--- a/lldb/source/DataFormatters/StringPrinter.cpp
+++ b/lldb/source/DataFormatters/StringPrinter.cpp
@@ -9,11 +9,11 @@
 #include "lldb/DataFormatters/StringPrinter.h"
 
 #include "lldb/Core/Debugger.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Target/Language.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/ConvertUTF.h"
diff --git a/lldb/source/DataFormatters/TypeSummary.cpp b/lldb/source/DataFormatters/TypeSummary.cpp
index 339068e8cc6a..3538c1b60c8e 100644
--- a/lldb/source/DataFormatters/TypeSummary.cpp
+++ b/lldb/source/DataFormatters/TypeSummary.cpp
@@ -15,13 +15,13 @@
 #include "lldb/lldb-public.h"
 
 #include "lldb/Core/Debugger.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/ValueObjectPrinter.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/DataFormatters/ValueObjectPrinter.cpp b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
index ce24a7866e53..face38253efa 100644
--- a/lldb/source/DataFormatters/ValueObjectPrinter.cpp
+++ b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
@@ -8,12 +8,12 @@
 
 #include "lldb/DataFormatters/ValueObjectPrinter.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/DataVisualization.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Target/Language.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "llvm/Support/MathExtras.h"
 #include <cstdint>
 
diff --git a/lldb/source/DataFormatters/VectorType.cpp b/lldb/source/DataFormatters/VectorType.cpp
index f6c38f76fea3..cba107b7da89 100644
--- a/lldb/source/DataFormatters/VectorType.cpp
+++ b/lldb/source/DataFormatters/VectorType.cpp
@@ -8,12 +8,12 @@
 
 #include "lldb/DataFormatters/VectorType.h"
 
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Symbol/TypeSystem.h"
 #include "lldb/Target/Target.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "lldb/Utility/LLDBAssert.h"
 #include "lldb/Utility/Log.h"
diff --git a/lldb/source/Expression/CMakeLists.txt b/lldb/source/Expression/CMakeLists.txt
index be1e132f7aaa..9e1c341947e9 100644
--- a/lldb/source/Expression/CMakeLists.txt
+++ b/lldb/source/Expression/CMakeLists.txt
@@ -27,6 +27,7 @@ add_lldb_library(lldbExpression NO_PLUGIN_DEPENDENCIES
     lldbSymbol
     lldbTarget
     lldbUtility
+    lldbValueObject
 
   LINK_COMPONENTS
     Core
diff --git a/lldb/source/Expression/FunctionCaller.cpp b/lldb/source/Expression/FunctionCaller.cpp
index d1dd350b09f2..67f9cd5758be 100644
--- a/lldb/source/Expression/FunctionCaller.cpp
+++ b/lldb/source/Expression/FunctionCaller.cpp
@@ -8,8 +8,6 @@
 
 #include "lldb/Expression/FunctionCaller.h"
 #include "lldb/Core/Module.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectList.h"
 #include "lldb/Expression/DiagnosticManager.h"
 #include "lldb/Expression/IRExecutionUnit.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
@@ -27,6 +25,8 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/State.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectList.h"
 
 using namespace lldb_private;
 
diff --git a/lldb/source/Expression/IRInterpreter.cpp b/lldb/source/Expression/IRInterpreter.cpp
index 4909310db7a6..fa74e8828a57 100644
--- a/lldb/source/Expression/IRInterpreter.cpp
+++ b/lldb/source/Expression/IRInterpreter.cpp
@@ -10,7 +10,6 @@
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/ModuleSpec.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Expression/DiagnosticManager.h"
 #include "lldb/Expression/IRExecutionUnit.h"
 #include "lldb/Expression/IRMemoryMap.h"
@@ -22,6 +21,7 @@
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include "lldb/Target/ABI.h"
 #include "lldb/Target/ExecutionContext.h"
diff --git a/lldb/source/Expression/LLVMUserExpression.cpp b/lldb/source/Expression/LLVMUserExpression.cpp
index d76f04d9b86e..529ac462dfd8 100644
--- a/lldb/source/Expression/LLVMUserExpression.cpp
+++ b/lldb/source/Expression/LLVMUserExpression.cpp
@@ -8,7 +8,6 @@
 
 #include "lldb/Expression/LLVMUserExpression.h"
 #include "lldb/Core/Module.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Expression/DiagnosticManager.h"
 #include "lldb/Expression/ExpressionVariable.h"
 #include "lldb/Expression/IRExecutionUnit.h"
@@ -33,6 +32,7 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/Expression/Materializer.cpp b/lldb/source/Expression/Materializer.cpp
index 8097e38919b0..8cd050f9fdb7 100644
--- a/lldb/source/Expression/Materializer.cpp
+++ b/lldb/source/Expression/Materializer.cpp
@@ -8,8 +8,6 @@
 
 #include "lldb/Expression/Materializer.h"
 #include "lldb/Core/DumpDataExtractor.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectVariable.h"
 #include "lldb/Expression/ExpressionVariable.h"
 #include "lldb/Symbol/Symbol.h"
 #include "lldb/Symbol/Type.h"
@@ -22,6 +20,8 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegisterValue.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectVariable.h"
 #include "lldb/lldb-forward.h"
 
 #include <memory>
diff --git a/lldb/source/Expression/UserExpression.cpp b/lldb/source/Expression/UserExpression.cpp
index b3c81af24893..ed3734cbb943 100644
--- a/lldb/source/Expression/UserExpression.cpp
+++ b/lldb/source/Expression/UserExpression.cpp
@@ -14,7 +14,6 @@
 #include <string>
 
 #include "lldb/Core/Module.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Expression/DiagnosticManager.h"
 #include "lldb/Expression/ExpressionVariable.h"
 #include "lldb/Expression/IRExecutionUnit.h"
@@ -39,6 +38,7 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/State.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 
 using namespace lldb_private;
diff --git a/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp b/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp
index 57b8f52f9145..cb121c14048f 100644
--- a/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp
+++ b/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp
@@ -17,7 +17,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -29,6 +28,7 @@
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "Utility/ARM64_DWARF_Registers.h"
 
diff --git a/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp b/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp
index 2ca69378d7ec..93b8141e97ef 100644
--- a/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp
+++ b/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp
@@ -17,7 +17,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -29,6 +28,7 @@
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "Utility/ARM64_DWARF_Registers.h"
 
diff --git a/lldb/source/Plugins/ABI/AArch64/CMakeLists.txt b/lldb/source/Plugins/ABI/AArch64/CMakeLists.txt
index dcf15ef5b6cd..e1e555249450 100644
--- a/lldb/source/Plugins/ABI/AArch64/CMakeLists.txt
+++ b/lldb/source/Plugins/ABI/AArch64/CMakeLists.txt
@@ -7,6 +7,7 @@ add_lldb_library(lldbPluginABIAArch64 PLUGIN
     lldbCore
     lldbSymbol
     lldbTarget
+    lldbValueObject
   LINK_COMPONENTS
     Support
     TargetParser
diff --git a/lldb/source/Plugins/ABI/ARC/ABISysV_arc.cpp b/lldb/source/Plugins/ABI/ARC/ABISysV_arc.cpp
index 945fc56775b2..6f60663ca51e 100644
--- a/lldb/source/Plugins/ABI/ARC/ABISysV_arc.cpp
+++ b/lldb/source/Plugins/ABI/ARC/ABISysV_arc.cpp
@@ -22,9 +22,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectMemory.h"
-#include "lldb/Core/ValueObjectRegister.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -34,6 +31,9 @@
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
+#include "lldb/ValueObject/ValueObjectRegister.h"
 
 #define DEFINE_REG_NAME(reg_num)      ConstString(#reg_num).GetCString()
 #define DEFINE_REG_NAME_STR(reg_name) ConstString(reg_name).GetCString()
diff --git a/lldb/source/Plugins/ABI/ARC/CMakeLists.txt b/lldb/source/Plugins/ABI/ARC/CMakeLists.txt
index 1ad4cf16716b..8654461a6e7b 100644
--- a/lldb/source/Plugins/ABI/ARC/CMakeLists.txt
+++ b/lldb/source/Plugins/ABI/ARC/CMakeLists.txt
@@ -6,6 +6,7 @@ add_lldb_library(lldbPluginABIARC PLUGIN
     lldbSymbol
     lldbTarget
     lldbPluginProcessUtility
+    lldbValueObject
   LINK_COMPONENTS
     Support
     TargetParser
diff --git a/lldb/source/Plugins/ABI/ARM/ABIMacOSX_arm.cpp b/lldb/source/Plugins/ABI/ARM/ABIMacOSX_arm.cpp
index 005c5cb166e6..08c613c7b0d0 100644
--- a/lldb/source/Plugins/ABI/ARM/ABIMacOSX_arm.cpp
+++ b/lldb/source/Plugins/ABI/ARM/ABIMacOSX_arm.cpp
@@ -17,7 +17,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -27,6 +26,7 @@
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "Plugins/Process/Utility/ARMDefines.h"
 #include "Utility/ARM_DWARF_Registers.h"
diff --git a/lldb/source/Plugins/ABI/ARM/ABISysV_arm.cpp b/lldb/source/Plugins/ABI/ARM/ABISysV_arm.cpp
index b70ece159b5c..1a0e44f1936b 100644
--- a/lldb/source/Plugins/ABI/ARM/ABISysV_arm.cpp
+++ b/lldb/source/Plugins/ABI/ARM/ABISysV_arm.cpp
@@ -17,7 +17,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -27,6 +26,7 @@
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "Plugins/Process/Utility/ARMDefines.h"
 #include "Utility/ARM_DWARF_Registers.h"
diff --git a/lldb/source/Plugins/ABI/ARM/CMakeLists.txt b/lldb/source/Plugins/ABI/ARM/CMakeLists.txt
index b0914d54f9b6..f73848947e5d 100644
--- a/lldb/source/Plugins/ABI/ARM/CMakeLists.txt
+++ b/lldb/source/Plugins/ABI/ARM/CMakeLists.txt
@@ -8,6 +8,7 @@ add_lldb_library(lldbPluginABIARM PLUGIN
     lldbSymbol
     lldbTarget
     lldbPluginProcessUtility
+    lldbValueObject
   LINK_COMPONENTS
     Support
     TargetParser
diff --git a/lldb/source/Plugins/ABI/Hexagon/ABISysV_hexagon.cpp b/lldb/source/Plugins/ABI/Hexagon/ABISysV_hexagon.cpp
index 1fb293bb3d53..5419f054cbe2 100644
--- a/lldb/source/Plugins/ABI/Hexagon/ABISysV_hexagon.cpp
+++ b/lldb/source/Plugins/ABI/Hexagon/ABISysV_hexagon.cpp
@@ -14,9 +14,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectMemory.h"
-#include "lldb/Core/ValueObjectRegister.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -28,6 +25,9 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
+#include "lldb/ValueObject/ValueObjectRegister.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/Plugins/ABI/Hexagon/CMakeLists.txt b/lldb/source/Plugins/ABI/Hexagon/CMakeLists.txt
index c2d80ee15385..2bfc08d62a88 100644
--- a/lldb/source/Plugins/ABI/Hexagon/CMakeLists.txt
+++ b/lldb/source/Plugins/ABI/Hexagon/CMakeLists.txt
@@ -5,6 +5,7 @@ add_lldb_library(lldbPluginABIHexagon PLUGIN
     lldbCore
     lldbSymbol
     lldbTarget
+    lldbValueObject
   LINK_COMPONENTS
     Support
     TargetParser
diff --git a/lldb/source/Plugins/ABI/MSP430/ABISysV_msp430.cpp b/lldb/source/Plugins/ABI/MSP430/ABISysV_msp430.cpp
index 7fb651aa6c18..6f0fd3a6e8f6 100644
--- a/lldb/source/Plugins/ABI/MSP430/ABISysV_msp430.cpp
+++ b/lldb/source/Plugins/ABI/MSP430/ABISysV_msp430.cpp
@@ -12,9 +12,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectMemory.h"
-#include "lldb/Core/ValueObjectRegister.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -25,6 +22,9 @@
 #include "lldb/Utility/DataExtractor.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegisterValue.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
+#include "lldb/ValueObject/ValueObjectRegister.h"
 
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/TargetParser/Triple.h"
diff --git a/lldb/source/Plugins/ABI/MSP430/CMakeLists.txt b/lldb/source/Plugins/ABI/MSP430/CMakeLists.txt
index 81abb2d9b16e..6aff7a2f7bf2 100644
--- a/lldb/source/Plugins/ABI/MSP430/CMakeLists.txt
+++ b/lldb/source/Plugins/ABI/MSP430/CMakeLists.txt
@@ -5,6 +5,7 @@ add_lldb_library(lldbPluginABIMSP430 PLUGIN
     lldbCore
     lldbSymbol
     lldbTarget
+    lldbValueObject
   LINK_COMPONENTS
     Support
     TargetParser
diff --git a/lldb/source/Plugins/ABI/Mips/ABISysV_mips.cpp b/lldb/source/Plugins/ABI/Mips/ABISysV_mips.cpp
index 58293c3e7f64..d21ee8ac04a2 100644
--- a/lldb/source/Plugins/ABI/Mips/ABISysV_mips.cpp
+++ b/lldb/source/Plugins/ABI/Mips/ABISysV_mips.cpp
@@ -14,9 +14,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectMemory.h"
-#include "lldb/Core/ValueObjectRegister.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -29,6 +26,9 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
+#include "lldb/ValueObject/ValueObjectRegister.h"
 #include <optional>
 
 using namespace lldb;
diff --git a/lldb/source/Plugins/ABI/Mips/ABISysV_mips64.cpp b/lldb/source/Plugins/ABI/Mips/ABISysV_mips64.cpp
index cd9c3442cebc..100d52bfd1c8 100644
--- a/lldb/source/Plugins/ABI/Mips/ABISysV_mips64.cpp
+++ b/lldb/source/Plugins/ABI/Mips/ABISysV_mips64.cpp
@@ -14,9 +14,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectMemory.h"
-#include "lldb/Core/ValueObjectRegister.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -29,6 +26,9 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
+#include "lldb/ValueObject/ValueObjectRegister.h"
 #include <optional>
 
 using namespace lldb;
diff --git a/lldb/source/Plugins/ABI/Mips/CMakeLists.txt b/lldb/source/Plugins/ABI/Mips/CMakeLists.txt
index 7e2cfc80b55f..c7d0cc69d727 100644
--- a/lldb/source/Plugins/ABI/Mips/CMakeLists.txt
+++ b/lldb/source/Plugins/ABI/Mips/CMakeLists.txt
@@ -7,6 +7,7 @@ add_lldb_library(lldbPluginABIMips PLUGIN
     lldbCore
     lldbSymbol
     lldbTarget
+    lldbValueObject
   LINK_COMPONENTS
     Support
     TargetParser
diff --git a/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc.cpp b/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc.cpp
index 655c52b71fe0..e482b8d4c4c4 100644
--- a/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc.cpp
+++ b/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc.cpp
@@ -14,9 +14,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectMemory.h"
-#include "lldb/Core/ValueObjectRegister.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -29,6 +26,9 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
+#include "lldb/ValueObject/ValueObjectRegister.h"
 #include <optional>
 
 using namespace lldb;
diff --git a/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp b/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp
index 34c34ff7dbad..0aa51ef65427 100644
--- a/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp
+++ b/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp
@@ -17,9 +17,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectMemory.h"
-#include "lldb/Core/ValueObjectRegister.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -32,6 +29,9 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
+#include "lldb/ValueObject/ValueObjectRegister.h"
 
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Attr.h"
diff --git a/lldb/source/Plugins/ABI/PowerPC/CMakeLists.txt b/lldb/source/Plugins/ABI/PowerPC/CMakeLists.txt
index 3ab251e4c64d..eff8cd664e4c 100644
--- a/lldb/source/Plugins/ABI/PowerPC/CMakeLists.txt
+++ b/lldb/source/Plugins/ABI/PowerPC/CMakeLists.txt
@@ -8,6 +8,7 @@ add_lldb_library(lldbPluginABIPowerPC PLUGIN
     lldbSymbol
     lldbTarget
     lldbPluginTypeSystemClang
+    lldbValueObject
   LINK_COMPONENTS
     Support
     TargetParser
diff --git a/lldb/source/Plugins/ABI/RISCV/ABISysV_riscv.cpp b/lldb/source/Plugins/ABI/RISCV/ABISysV_riscv.cpp
index 38d452b6ab56..8412991933d2 100644
--- a/lldb/source/Plugins/ABI/RISCV/ABISysV_riscv.cpp
+++ b/lldb/source/Plugins/ABI/RISCV/ABISysV_riscv.cpp
@@ -18,12 +18,12 @@
 #include "Utility/RISCV_DWARF_Registers.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Target/RegisterContext.h"
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Target/Thread.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/RegisterValue.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #define DEFINE_REG_NAME(reg_num) ConstString(#reg_num).GetCString()
 #define DEFINE_REG_NAME_STR(reg_name) ConstString(reg_name).GetCString()
diff --git a/lldb/source/Plugins/ABI/RISCV/CMakeLists.txt b/lldb/source/Plugins/ABI/RISCV/CMakeLists.txt
index a556480ecdcf..5b25bf7c6af8 100755
--- a/lldb/source/Plugins/ABI/RISCV/CMakeLists.txt
+++ b/lldb/source/Plugins/ABI/RISCV/CMakeLists.txt
@@ -6,6 +6,7 @@ add_lldb_library(lldbPluginABIRISCV PLUGIN
     lldbSymbol
     lldbTarget
     lldbPluginProcessUtility
+    lldbValueObject
   LINK_COMPONENTS
     Support
     TargetParser
diff --git a/lldb/source/Plugins/ABI/SystemZ/ABISysV_s390x.cpp b/lldb/source/Plugins/ABI/SystemZ/ABISysV_s390x.cpp
index ac2d1988a176..82853c96a0f7 100644
--- a/lldb/source/Plugins/ABI/SystemZ/ABISysV_s390x.cpp
+++ b/lldb/source/Plugins/ABI/SystemZ/ABISysV_s390x.cpp
@@ -14,9 +14,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectMemory.h"
-#include "lldb/Core/ValueObjectRegister.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -29,6 +26,9 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
+#include "lldb/ValueObject/ValueObjectRegister.h"
 #include <optional>
 
 using namespace lldb;
diff --git a/lldb/source/Plugins/ABI/SystemZ/CMakeLists.txt b/lldb/source/Plugins/ABI/SystemZ/CMakeLists.txt
index 11d659b03b55..b6164ed87d9d 100644
--- a/lldb/source/Plugins/ABI/SystemZ/CMakeLists.txt
+++ b/lldb/source/Plugins/ABI/SystemZ/CMakeLists.txt
@@ -5,6 +5,7 @@ add_lldb_library(lldbPluginABISystemZ PLUGIN
     lldbCore
     lldbSymbol
     lldbTarget
+    lldbValueObject
   LINK_COMPONENTS
     Support
     TargetParser
diff --git a/lldb/source/Plugins/ABI/X86/ABIMacOSX_i386.cpp b/lldb/source/Plugins/ABI/X86/ABIMacOSX_i386.cpp
index 1abf7365e83c..f03acbcedb35 100644
--- a/lldb/source/Plugins/ABI/X86/ABIMacOSX_i386.cpp
+++ b/lldb/source/Plugins/ABI/X86/ABIMacOSX_i386.cpp
@@ -16,7 +16,6 @@
 
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -26,6 +25,7 @@
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/Plugins/ABI/X86/ABISysV_i386.cpp b/lldb/source/Plugins/ABI/X86/ABISysV_i386.cpp
index ecf68e6131d3..19ec5fa49bbe 100644
--- a/lldb/source/Plugins/ABI/X86/ABISysV_i386.cpp
+++ b/lldb/source/Plugins/ABI/X86/ABISysV_i386.cpp
@@ -13,9 +13,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectMemory.h"
-#include "lldb/Core/ValueObjectRegister.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -27,6 +24,9 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
+#include "lldb/ValueObject/ValueObjectRegister.h"
 #include <optional>
 
 using namespace lldb;
diff --git a/lldb/source/Plugins/ABI/X86/ABISysV_x86_64.cpp b/lldb/source/Plugins/ABI/X86/ABISysV_x86_64.cpp
index d036630aeba3..ad13a4406cfd 100644
--- a/lldb/source/Plugins/ABI/X86/ABISysV_x86_64.cpp
+++ b/lldb/source/Plugins/ABI/X86/ABISysV_x86_64.cpp
@@ -15,9 +15,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectMemory.h"
-#include "lldb/Core/ValueObjectRegister.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -30,6 +27,9 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
+#include "lldb/ValueObject/ValueObjectRegister.h"
 
 #include <optional>
 #include <vector>
diff --git a/lldb/source/Plugins/ABI/X86/ABIWindows_x86_64.cpp b/lldb/source/Plugins/ABI/X86/ABIWindows_x86_64.cpp
index 2f88a329eb41..098293c1d604 100644
--- a/lldb/source/Plugins/ABI/X86/ABIWindows_x86_64.cpp
+++ b/lldb/source/Plugins/ABI/X86/ABIWindows_x86_64.cpp
@@ -15,9 +15,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectMemory.h"
-#include "lldb/Core/ValueObjectRegister.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -30,6 +27,9 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
+#include "lldb/ValueObject/ValueObjectRegister.h"
 #include <optional>
 
 using namespace lldb;
diff --git a/lldb/source/Plugins/ABI/X86/CMakeLists.txt b/lldb/source/Plugins/ABI/X86/CMakeLists.txt
index 32a9df14b4b9..eeab6297e6f2 100644
--- a/lldb/source/Plugins/ABI/X86/CMakeLists.txt
+++ b/lldb/source/Plugins/ABI/X86/CMakeLists.txt
@@ -10,6 +10,7 @@ add_lldb_library(lldbPluginABIX86 PLUGIN
     lldbCore
     lldbSymbol
     lldbTarget
+    lldbValueObject
   LINK_COMPONENTS
     Support
     TargetParser
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/CMakeLists.txt b/lldb/source/Plugins/ExpressionParser/Clang/CMakeLists.txt
index e0ae3922788e..675cd066507b 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/CMakeLists.txt
+++ b/lldb/source/Plugins/ExpressionParser/Clang/CMakeLists.txt
@@ -37,6 +37,7 @@ add_lldb_library(lldbPluginExpressionParserClang
     lldbSymbol
     lldbTarget
     lldbUtility
+    lldbValueObject
     lldbPluginCPlusPlusLanguage
     lldbPluginCPPRuntime
     lldbPluginObjCRuntime
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
index 5edaa9e4e053..bbe401b827f1 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
@@ -20,8 +20,6 @@
 #include "lldb/Core/Address.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/ModuleSpec.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectVariable.h"
 #include "lldb/Expression/DiagnosticManager.h"
 #include "lldb/Expression/Materializer.h"
 #include "lldb/Symbol/CompileUnit.h"
@@ -47,6 +45,8 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectVariable.h"
 #include "lldb/lldb-private-types.h"
 #include "lldb/lldb-private.h"
 #include "clang/AST/ASTConsumer.h"
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionUtil.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionUtil.cpp
index 723a3d7e5984..d4b2775b8c03 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionUtil.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionUtil.cpp
@@ -8,9 +8,9 @@
 
 #include "ClangExpressionUtil.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 namespace ClangExpressionUtil {
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionVariable.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionVariable.cpp
index ac3cb0b1bdf3..e2fb4a054daf 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionVariable.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionVariable.cpp
@@ -9,12 +9,12 @@
 #include "ClangExpressionVariable.h"
 
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Target/ExecutionContext.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/DataExtractor.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 #include "clang/AST/ASTContext.h"
 
 using namespace lldb_private;
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangFunctionCaller.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangFunctionCaller.cpp
index 59321e375bdc..e4a094f3aa51 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangFunctionCaller.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangFunctionCaller.cpp
@@ -23,8 +23,6 @@
 
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "lldb/Core/Module.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectList.h"
 #include "lldb/Expression/IRExecutionUnit.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
 #include "lldb/Symbol/Function.h"
@@ -40,6 +38,8 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/State.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectList.h"
 
 using namespace lldb_private;
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangFunctionCaller.h b/lldb/source/Plugins/ExpressionParser/Clang/ClangFunctionCaller.h
index 2a5c863b51a2..9329ce76ec3b 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangFunctionCaller.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangFunctionCaller.h
@@ -13,10 +13,10 @@
 
 #include "lldb/Core/Address.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectList.h"
 #include "lldb/Expression/FunctionCaller.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Target/Process.h"
+#include "lldb/ValueObject/ValueObjectList.h"
 
 namespace lldb_private {
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
index 52d52826c946..ae34a983612f 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
@@ -27,7 +27,6 @@
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/Module.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Expression/ExpressionSourceCode.h"
 #include "lldb/Expression/IRExecutionUnit.h"
 #include "lldb/Expression/IRInterpreter.h"
@@ -51,6 +50,7 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
diff --git a/lldb/source/Plugins/InstrumentationRuntime/TSan/CMakeLists.txt b/lldb/source/Plugins/InstrumentationRuntime/TSan/CMakeLists.txt
index a388cbb2ddfe..673e75fa2faf 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/TSan/CMakeLists.txt
+++ b/lldb/source/Plugins/InstrumentationRuntime/TSan/CMakeLists.txt
@@ -8,5 +8,6 @@ add_lldb_library(lldbPluginInstrumentationRuntimeTSan PLUGIN
     lldbInterpreter
     lldbSymbol
     lldbTarget
+    lldbValueObject
     lldbPluginProcessUtility
   )
diff --git a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
index 5516de19d863..b0b17263ed6f 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
@@ -14,7 +14,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginInterface.h"
 #include "lldb/Core/PluginManager.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Expression/UserExpression.h"
 #include "lldb/Host/StreamFile.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
@@ -31,6 +30,7 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include <memory>
 
diff --git a/lldb/source/Plugins/InstrumentationRuntime/UBSan/CMakeLists.txt b/lldb/source/Plugins/InstrumentationRuntime/UBSan/CMakeLists.txt
index 12e1956bbe88..156bd3ac454e 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/UBSan/CMakeLists.txt
+++ b/lldb/source/Plugins/InstrumentationRuntime/UBSan/CMakeLists.txt
@@ -8,6 +8,7 @@ add_lldb_library(lldbPluginInstrumentationRuntimeUBSan PLUGIN
     lldbInterpreter
     lldbSymbol
     lldbTarget
+    lldbValueObject
   LINK_COMPONENTS
     Support
   )
diff --git a/lldb/source/Plugins/InstrumentationRuntime/UBSan/InstrumentationRuntimeUBSan.cpp b/lldb/source/Plugins/InstrumentationRuntime/UBSan/InstrumentationRuntimeUBSan.cpp
index 80935915e6c6..06d455e0676b 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/UBSan/InstrumentationRuntimeUBSan.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/UBSan/InstrumentationRuntimeUBSan.cpp
@@ -14,7 +14,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginInterface.h"
 #include "lldb/Core/PluginManager.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Expression/UserExpression.h"
 #include "lldb/Host/StreamFile.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
@@ -29,6 +28,7 @@
 #include "lldb/Target/Thread.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include <cctype>
 
 #include <memory>
diff --git a/lldb/source/Plugins/InstrumentationRuntime/Utility/CMakeLists.txt b/lldb/source/Plugins/InstrumentationRuntime/Utility/CMakeLists.txt
index 05f05c813922..dcca47fb0e25 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/Utility/CMakeLists.txt
+++ b/lldb/source/Plugins/InstrumentationRuntime/Utility/CMakeLists.txt
@@ -7,4 +7,5 @@ add_lldb_library(lldbPluginInstrumentationRuntimeUtility
     lldbExpression
     lldbSymbol
     lldbTarget
+    lldbValueObject
   )
diff --git a/lldb/source/Plugins/InstrumentationRuntime/Utility/ReportRetriever.cpp b/lldb/source/Plugins/InstrumentationRuntime/Utility/ReportRetriever.cpp
index 298b63bc716f..2f1c78d07fc0 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/Utility/ReportRetriever.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/Utility/ReportRetriever.cpp
@@ -11,9 +11,9 @@
 #include "lldb/Breakpoint/StoppointCallbackContext.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/Module.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Expression/UserExpression.h"
 #include "lldb/Target/InstrumentationRuntimeStopInfo.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
index f3c137d99703..d7d4654a6b5f 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
@@ -11,8 +11,6 @@
 #include "Plugins/ExpressionParser/Clang/ClangASTImporter.h"
 #include "Plugins/ExpressionParser/Clang/ClangPersistentVariables.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Symbol/TypeSystem.h"
@@ -20,6 +18,8 @@
 #include "lldb/Utility/LLDBAssert.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt b/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt
index f59032c42388..ccdc4d0ae99b 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt
+++ b/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt
@@ -33,6 +33,7 @@ add_lldb_library(lldbPluginCPlusPlusLanguage PLUGIN
     lldbSymbol
     lldbTarget
     lldbUtility
+    lldbValueObject
     lldbPluginClangCommon
     lldbPluginCPPRuntime
     lldbPluginTypeSystemClang
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
index 06c827c2543f..2bf574e97768 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
@@ -23,7 +23,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/UniqueCStringMap.h"
-#include "lldb/Core/ValueObjectVariable.h"
 #include "lldb/DataFormatters/CXXFunctionPointer.h"
 #include "lldb/DataFormatters/DataVisualization.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
@@ -34,6 +33,7 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
+#include "lldb/ValueObject/ValueObjectVariable.h"
 
 #include "BlockPointer.h"
 #include "CPlusPlusNameParser.h"
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
index 1d4bc65e2637..f9765f3255d2 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
@@ -9,10 +9,10 @@
 #ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_COROUTINES_H
 #define LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_COROUTINES_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
 #include "lldb/DataFormatters/TypeSynthetic.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CxxStringTypes.cpp b/lldb/source/Plugins/Language/CPlusPlus/CxxStringTypes.cpp
index 6781c96210a4..6d810cd30cd8 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CxxStringTypes.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CxxStringTypes.cpp
@@ -11,8 +11,6 @@
 #include "llvm/Support/ConvertUTF.h"
 
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/DataFormatters/StringPrinter.h"
 #include "lldb/DataFormatters/TypeSummary.h"
@@ -24,6 +22,8 @@
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include <algorithm>
 #include <optional>
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CxxStringTypes.h b/lldb/source/Plugins/Language/CPlusPlus/CxxStringTypes.h
index 2713ded45929..a2b606d28cac 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CxxStringTypes.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/CxxStringTypes.h
@@ -10,9 +10,9 @@
 #ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_CXXSTRINGTYPES_H
 #define LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_CXXSTRINGTYPES_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 namespace formatters {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Generic.h b/lldb/source/Plugins/Language/CPlusPlus/Generic.h
index bfb28bebf90b..34efef9e82e5 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/Generic.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/Generic.h
@@ -9,9 +9,9 @@
 #ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_GENERIC_H
 #define LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_GENERIC_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 namespace formatters {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
index 7d3b2410a729..6d0ccdbbe4a7 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
@@ -10,8 +10,6 @@
 
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/FormatEntity.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/DataFormatters/StringPrinter.h"
 #include "lldb/DataFormatters/TypeSummary.h"
@@ -23,6 +21,8 @@
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "Plugins/LanguageRuntime/CPlusPlus/CPPLanguageRuntime.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
index dad033611b38..cb9ceaf09330 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
@@ -10,10 +10,10 @@
 #ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_LIBCXX_H
 #define LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_LIBCXX_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
 #include "lldb/DataFormatters/TypeSynthetic.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 namespace formatters {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.h
index 6fcceb645c7b..93274460ce35 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.h
@@ -10,10 +10,10 @@
 #ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_LIBCXXATOMIC_H
 #define LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_LIBCXXATOMIC_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
 #include "lldb/DataFormatters/TypeSynthetic.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 namespace formatters {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
index bd9c72497664..67c6d1d3e550 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
@@ -8,9 +8,9 @@
 
 #include "LibCxx.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include <optional>
 
 using namespace lldb;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
index 4479f592fc2d..f33b148249ab 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
@@ -9,14 +9,14 @@
 #include "LibCxx.h"
 
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/DataBufferHeap.h"
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 #include "lldb/lldb-enumerations.h"
 
 using namespace lldb;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
index af3e41b0daf8..ebaf60a16b06 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
@@ -9,14 +9,14 @@
 #include "LibCxx.h"
 
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/DataBufferHeap.h"
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
 #include <cstdint>
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxProxyArray.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxProxyArray.cpp
index 726f06523b97..c659adbb9ab2 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxProxyArray.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxProxyArray.cpp
@@ -8,8 +8,8 @@
 
 #include "LibCxx.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include <optional>
 
 using namespace lldb;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
index 01a7b8f142ec..f3fe56525789 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
@@ -8,9 +8,9 @@
 
 #include "LibCxx.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "llvm/ADT/APSInt.h"
 
 using namespace lldb;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSliceArray.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSliceArray.cpp
index c33d8e91af70..5d607709d2c6 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSliceArray.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSliceArray.cpp
@@ -8,8 +8,8 @@
 
 #include "LibCxx.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include <optional>
 
 using namespace lldb;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
index 9895f336bfd0..15040295efe6 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
@@ -8,9 +8,9 @@
 
 #include "LibCxx.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "llvm/ADT/APSInt.h"
 #include <optional>
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
index 2f65c726aa51..bf91fc42482f 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
@@ -9,8 +9,6 @@
 #include "LibCxx.h"
 
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/ConstString.h"
@@ -18,6 +16,8 @@
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
index 99f94406e99a..3f519f8c585f 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
@@ -8,8 +8,8 @@
 
 #include "LibCxx.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include <optional>
 
 using namespace lldb;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.h
index 9488d94efda7..c6764f6178ae 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.h
@@ -9,10 +9,10 @@
 #ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_LIBCXXVARIANT_H
 #define LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_LIBCXXVARIANT_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
 #include "lldb/DataFormatters/TypeSynthetic.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 namespace formatters {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
index 3609219e7055..b762379a07d3 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
@@ -8,9 +8,9 @@
 
 #include "LibCxx.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
 #include <optional>
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
index 86bb575af5ca..0a1877471916 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
@@ -10,8 +10,6 @@
 #include "LibCxx.h"
 
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/StringPrinter.h"
 #include "lldb/DataFormatters/VectorIterator.h"
 #include "lldb/Target/Target.h"
@@ -19,6 +17,8 @@
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 #include <optional>
 
 using namespace lldb;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h
index 1c1c8fdb9ea4..c7b2130275ad 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h
@@ -9,10 +9,10 @@
 #ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_LIBSTDCPP_H
 #define LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_LIBSTDCPP_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
 #include "lldb/DataFormatters/TypeSynthetic.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 namespace formatters {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
index 05199ba35b9a..f59969d4cd7a 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
@@ -8,10 +8,10 @@
 
 #include "LibStdcpp.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/DataFormatters/TypeSynthetic.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include <memory>
 #include <vector>
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
index 92f540d9ca52..9447f7463f64 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
@@ -8,10 +8,10 @@
 
 #include "LibStdcpp.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/DataFormatters/TypeSynthetic.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include <memory>
 #include <vector>
diff --git a/lldb/source/Plugins/Language/ObjC/CF.cpp b/lldb/source/Plugins/Language/ObjC/CF.cpp
index 0926192a4f38..eba905b7b94f 100644
--- a/lldb/source/Plugins/Language/ObjC/CF.cpp
+++ b/lldb/source/Plugins/Language/ObjC/CF.cpp
@@ -9,8 +9,6 @@
 #include "CF.h"
 
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Target/Language.h"
 #include "lldb/Target/StackFrame.h"
@@ -19,6 +17,8 @@
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.h"
 
diff --git a/lldb/source/Plugins/Language/ObjC/CF.h b/lldb/source/Plugins/Language/ObjC/CF.h
index 6165e1c235bc..a4e6031b08f6 100644
--- a/lldb/source/Plugins/Language/ObjC/CF.h
+++ b/lldb/source/Plugins/Language/ObjC/CF.h
@@ -9,9 +9,9 @@
 #ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_OBJC_CF_H
 #define LLDB_SOURCE_PLUGINS_LANGUAGE_OBJC_CF_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 namespace formatters {
diff --git a/lldb/source/Plugins/Language/ObjC/CMakeLists.txt b/lldb/source/Plugins/Language/ObjC/CMakeLists.txt
index c998d5455de3..93c23fd32524 100644
--- a/lldb/source/Plugins/Language/ObjC/CMakeLists.txt
+++ b/lldb/source/Plugins/Language/ObjC/CMakeLists.txt
@@ -30,6 +30,7 @@ add_lldb_library(lldbPluginObjCLanguage PLUGIN
     lldbSymbol
     lldbTarget
     lldbUtility
+    lldbValueObject
     lldbPluginAppleObjCRuntime
     lldbPluginClangCommon
     lldbPluginTypeSystemClang
diff --git a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
index 341923108e32..bbe5d4c611f8 100644
--- a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
+++ b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
@@ -13,8 +13,6 @@
 #include "Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "lldb/Core/Mangled.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/DataFormatters/StringPrinter.h"
 #include "lldb/DataFormatters/TypeSummary.h"
@@ -27,6 +25,8 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/bit.h"
diff --git a/lldb/source/Plugins/Language/ObjC/Cocoa.h b/lldb/source/Plugins/Language/ObjC/Cocoa.h
index a195d622ce58..9617cb07cd8d 100644
--- a/lldb/source/Plugins/Language/ObjC/Cocoa.h
+++ b/lldb/source/Plugins/Language/ObjC/Cocoa.h
@@ -10,10 +10,10 @@
 #ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_OBJC_COCOA_H
 #define LLDB_SOURCE_PLUGINS_LANGUAGE_OBJC_COCOA_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
 #include "lldb/DataFormatters/TypeSynthetic.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include "Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.h"
 
diff --git a/lldb/source/Plugins/Language/ObjC/CoreMedia.h b/lldb/source/Plugins/Language/ObjC/CoreMedia.h
index 7fd8560d20e1..cc926f9c480a 100644
--- a/lldb/source/Plugins/Language/ObjC/CoreMedia.h
+++ b/lldb/source/Plugins/Language/ObjC/CoreMedia.h
@@ -10,9 +10,9 @@
 #ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_OBJC_COREMEDIA_H
 #define LLDB_SOURCE_PLUGINS_LANGUAGE_OBJC_COREMEDIA_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 namespace formatters {
diff --git a/lldb/source/Plugins/Language/ObjC/NSArray.cpp b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
index 67d0cd08f51a..072b8b5a6c86 100644
--- a/lldb/source/Plugins/Language/ObjC/NSArray.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
@@ -14,8 +14,6 @@
 #include "Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Expression/FunctionCaller.h"
 #include "lldb/Target/Language.h"
@@ -24,6 +22,8 @@
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
index ec6fd756394a..cf8750fd4976 100644
--- a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
@@ -16,8 +16,6 @@
 #include "Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Target/Language.h"
 #include "lldb/Target/StackFrame.h"
@@ -26,6 +24,8 @@
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/Plugins/Language/ObjC/NSDictionary.h b/lldb/source/Plugins/Language/ObjC/NSDictionary.h
index 57dacd6759d2..a65298de56b9 100644
--- a/lldb/source/Plugins/Language/ObjC/NSDictionary.h
+++ b/lldb/source/Plugins/Language/ObjC/NSDictionary.h
@@ -9,11 +9,11 @@
 #ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_OBJC_NSDICTIONARY_H
 #define LLDB_SOURCE_PLUGINS_LANGUAGE_OBJC_NSDICTIONARY_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
 #include "lldb/DataFormatters/TypeSynthetic.h"
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include <map>
 #include <memory>
diff --git a/lldb/source/Plugins/Language/ObjC/NSError.cpp b/lldb/source/Plugins/Language/ObjC/NSError.cpp
index 5ef7edc7e80c..2356bc4ef4ba 100644
--- a/lldb/source/Plugins/Language/ObjC/NSError.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSError.cpp
@@ -11,14 +11,14 @@
 #include "Cocoa.h"
 
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/DataBufferHeap.h"
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "Plugins/Language/ObjC/NSString.h"
 #include "Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.h"
diff --git a/lldb/source/Plugins/Language/ObjC/NSException.cpp b/lldb/source/Plugins/Language/ObjC/NSException.cpp
index e7ce26ea4c6f..b7d42bc5745e 100644
--- a/lldb/source/Plugins/Language/ObjC/NSException.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSException.cpp
@@ -10,14 +10,14 @@
 
 #include "Cocoa.h"
 
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/DataBufferHeap.h"
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "Plugins/Language/ObjC/NSString.h"
 #include "Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.h"
diff --git a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
index a434cee09d38..e29f625b405b 100644
--- a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
@@ -9,12 +9,12 @@
 #include "Cocoa.h"
 
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/DataFormatters/TypeSynthetic.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/Target.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.h"
 using namespace lldb;
diff --git a/lldb/source/Plugins/Language/ObjC/NSSet.cpp b/lldb/source/Plugins/Language/ObjC/NSSet.cpp
index 7d0a6a507211..a184ec624b63 100644
--- a/lldb/source/Plugins/Language/ObjC/NSSet.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSSet.cpp
@@ -11,8 +11,6 @@
 
 #include "Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Target/Language.h"
 #include "lldb/Target/Target.h"
@@ -20,6 +18,8 @@
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/Plugins/Language/ObjC/NSSet.h b/lldb/source/Plugins/Language/ObjC/NSSet.h
index 3ad1f694befe..2c490b72961f 100644
--- a/lldb/source/Plugins/Language/ObjC/NSSet.h
+++ b/lldb/source/Plugins/Language/ObjC/NSSet.h
@@ -10,11 +10,11 @@
 #ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_OBJC_NSSET_H
 #define LLDB_SOURCE_PLUGINS_LANGUAGE_OBJC_NSSET_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
 #include "lldb/DataFormatters/TypeSynthetic.h"
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 namespace lldb_private {
 namespace formatters {
diff --git a/lldb/source/Plugins/Language/ObjC/NSString.cpp b/lldb/source/Plugins/Language/ObjC/NSString.cpp
index 0a30737d9723..a99d042572bf 100644
--- a/lldb/source/Plugins/Language/ObjC/NSString.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSString.cpp
@@ -8,8 +8,6 @@
 
 #include "NSString.h"
 
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/DataFormatters/StringPrinter.h"
 #include "lldb/Target/Language.h"
@@ -19,6 +17,8 @@
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/Plugins/Language/ObjC/NSString.h b/lldb/source/Plugins/Language/ObjC/NSString.h
index a68cc6c056b0..8c9fcf955f1f 100644
--- a/lldb/source/Plugins/Language/ObjC/NSString.h
+++ b/lldb/source/Plugins/Language/ObjC/NSString.h
@@ -10,9 +10,9 @@
 #ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_OBJC_NSSTRING_H
 #define LLDB_SOURCE_PLUGINS_LANGUAGE_OBJC_NSSTRING_H
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include "Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.h"
 
diff --git a/lldb/source/Plugins/Language/ObjC/ObjCLanguage.cpp b/lldb/source/Plugins/Language/ObjC/ObjCLanguage.cpp
index 742ae7b14945..b44971e36c6d 100644
--- a/lldb/source/Plugins/Language/ObjC/ObjCLanguage.cpp
+++ b/lldb/source/Plugins/Language/ObjC/ObjCLanguage.cpp
@@ -14,13 +14,13 @@
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/PluginManager.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/DataVisualization.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include "llvm/Support/Threading.h"
 
diff --git a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/CMakeLists.txt b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/CMakeLists.txt
index 5ab494414ac5..a5406c73be93 100644
--- a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/CMakeLists.txt
+++ b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/CMakeLists.txt
@@ -7,6 +7,7 @@ add_lldb_library(lldbPluginCXXItaniumABI PLUGIN
     lldbInterpreter
     lldbSymbol
     lldbTarget
+    lldbValueObject
     lldbPluginCPPRuntime
     lldbPluginTypeSystemClang
   )
diff --git a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
index 4c547afe30fe..66cdab1307ce 100644
--- a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
@@ -13,8 +13,6 @@
 #include "lldb/Core/Mangled.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectMemory.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Expression/DiagnosticManager.h"
 #include "lldb/Expression/FunctionCaller.h"
@@ -35,6 +33,8 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
 
 #include <vector>
 
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
index 5ff267720629..ceee19c136d2 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
@@ -18,8 +18,6 @@
 #include "lldb/Core/ModuleList.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Section.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Expression/DiagnosticManager.h"
 #include "lldb/Expression/FunctionCaller.h"
@@ -37,6 +35,8 @@
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 #include "clang/AST/Type.h"
 
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
index 41492ca836fa..4c794b81809c 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
@@ -14,8 +14,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Section.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectVariable.h"
 #include "lldb/Expression/DiagnosticManager.h"
 #include "lldb/Expression/FunctionCaller.h"
 #include "lldb/Expression/UtilityFunction.h"
@@ -48,6 +46,8 @@
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/Timer.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectVariable.h"
 #include "lldb/lldb-enumerations.h"
 
 #include "AppleObjCClassDescriptorV2.h"
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/CMakeLists.txt b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/CMakeLists.txt
index 3789f5632598..53320d3da4cf 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/CMakeLists.txt
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/CMakeLists.txt
@@ -17,6 +17,7 @@ add_lldb_library(lldbPluginAppleObjCRuntime PLUGIN
     lldbSymbol
     lldbTarget
     lldbUtility
+    lldbValueObject
     lldbPluginExpressionParserClang
     lldbPluginCPPRuntime
     lldbPluginTypeSystemClang
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/CMakeLists.txt b/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/CMakeLists.txt
index 1da3ac58ec9f..0c7bff701dc0 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/CMakeLists.txt
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/CMakeLists.txt
@@ -10,6 +10,7 @@ add_lldb_library(lldbPluginGNUstepObjCRuntime PLUGIN
     lldbSymbol
     lldbTarget
     lldbUtility
+    lldbValueObject
     lldbPluginExpressionParserClang
     lldbPluginTypeSystemClang
   CLANG_LIBS
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.cpp
index 58b838752be5..d6ffb03ab55e 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.cpp
@@ -12,13 +12,13 @@
 
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Expression/UtilityFunction.h"
 #include "lldb/Target/ExecutionContext.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/ArchSpec.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
index ccde6c03db2e..cb745135b024 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
@@ -12,7 +12,6 @@
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Symbol/SymbolContext.h"
 #include "lldb/Symbol/SymbolFile.h"
 #include "lldb/Symbol/Type.h"
@@ -23,6 +22,7 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Timer.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/DJB.h"
diff --git a/lldb/source/Plugins/MemoryHistory/asan/CMakeLists.txt b/lldb/source/Plugins/MemoryHistory/asan/CMakeLists.txt
index 08deb75aabb1..a03723500f3a 100644
--- a/lldb/source/Plugins/MemoryHistory/asan/CMakeLists.txt
+++ b/lldb/source/Plugins/MemoryHistory/asan/CMakeLists.txt
@@ -5,5 +5,6 @@ add_lldb_library(lldbPluginMemoryHistoryASan PLUGIN
     lldbCore
     lldbExpression
     lldbTarget
+    lldbValueObject
     lldbPluginProcessUtility
   )
diff --git a/lldb/source/Plugins/MemoryHistory/asan/MemoryHistoryASan.cpp b/lldb/source/Plugins/MemoryHistory/asan/MemoryHistoryASan.cpp
index fe6c5a0544be..7363f606d1a7 100644
--- a/lldb/source/Plugins/MemoryHistory/asan/MemoryHistoryASan.cpp
+++ b/lldb/source/Plugins/MemoryHistory/asan/MemoryHistoryASan.cpp
@@ -15,12 +15,12 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginInterface.h"
 #include "lldb/Core/PluginManager.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Expression/UserExpression.h"
 #include "lldb/Target/ExecutionContext.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
 #include "lldb/Target/ThreadList.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-private.h"
 
 #include <sstream>
diff --git a/lldb/source/Plugins/OperatingSystem/Python/CMakeLists.txt b/lldb/source/Plugins/OperatingSystem/Python/CMakeLists.txt
index 91d7901011bc..b5be50d02bdb 100644
--- a/lldb/source/Plugins/OperatingSystem/Python/CMakeLists.txt
+++ b/lldb/source/Plugins/OperatingSystem/Python/CMakeLists.txt
@@ -6,5 +6,6 @@ add_lldb_library(lldbPluginOperatingSystemPython PLUGIN
     lldbInterpreter
     lldbSymbol
     lldbTarget
+    lldbValueObject
     lldbPluginProcessUtility
   )
diff --git a/lldb/source/Plugins/OperatingSystem/Python/OperatingSystemPython.cpp b/lldb/source/Plugins/OperatingSystem/Python/OperatingSystemPython.cpp
index 106e38b6e25a..3848a2b1deb9 100644
--- a/lldb/source/Plugins/OperatingSystem/Python/OperatingSystemPython.cpp
+++ b/lldb/source/Plugins/OperatingSystem/Python/OperatingSystemPython.cpp
@@ -18,7 +18,6 @@
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
-#include "lldb/Core/ValueObjectVariable.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/ScriptInterpreter.h"
 #include "lldb/Symbol/ObjectFile.h"
@@ -33,6 +32,7 @@
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/StructuredData.h"
+#include "lldb/ValueObject/ValueObjectVariable.h"
 
 #include <memory>
 
diff --git a/lldb/source/Plugins/Platform/Android/CMakeLists.txt b/lldb/source/Plugins/Platform/Android/CMakeLists.txt
index 518041b6cb31..7ffccc30a19f 100644
--- a/lldb/source/Plugins/Platform/Android/CMakeLists.txt
+++ b/lldb/source/Plugins/Platform/Android/CMakeLists.txt
@@ -14,6 +14,7 @@ add_lldb_library(lldbPluginPlatformAndroid PLUGIN
   LINK_LIBS
     lldbCore
     lldbHost
+    lldbValueObject
     lldbPluginPlatformLinux
     lldbPluginPlatformGDB
   LINK_COMPONENTS
diff --git a/lldb/source/Plugins/Platform/Android/PlatformAndroid.cpp b/lldb/source/Plugins/Platform/Android/PlatformAndroid.cpp
index 1fd098b4c42b..5bc9cc133fbd 100644
--- a/lldb/source/Plugins/Platform/Android/PlatformAndroid.cpp
+++ b/lldb/source/Plugins/Platform/Android/PlatformAndroid.cpp
@@ -9,12 +9,12 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Section.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Host/HostInfo.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/UriParser.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include "AdbClient.h"
 #include "PlatformAndroid.h"
diff --git a/lldb/source/Plugins/Platform/POSIX/CMakeLists.txt b/lldb/source/Plugins/Platform/POSIX/CMakeLists.txt
index 78eb7bc60e0e..ebca113a45d5 100644
--- a/lldb/source/Plugins/Platform/POSIX/CMakeLists.txt
+++ b/lldb/source/Plugins/Platform/POSIX/CMakeLists.txt
@@ -7,6 +7,7 @@ add_lldb_library(lldbPluginPlatformPOSIX
     lldbHost
     lldbInterpreter
     lldbTarget
+    lldbValueObject
     lldbPluginPlatformGDB
     lldbPluginTypeSystemClang
   )
diff --git a/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp b/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
index e9830c9f8722..4a8f669a84ec 100644
--- a/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
+++ b/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
@@ -12,7 +12,6 @@
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/Module.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Expression/DiagnosticManager.h"
 #include "lldb/Expression/FunctionCaller.h"
 #include "lldb/Expression/UserExpression.h"
@@ -32,6 +31,7 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "llvm/ADT/ScopeExit.h"
 #include <optional>
 
diff --git a/lldb/source/Plugins/Process/Utility/CMakeLists.txt b/lldb/source/Plugins/Process/Utility/CMakeLists.txt
index 5df4a9e5ac5c..308ea29e31ad 100644
--- a/lldb/source/Plugins/Process/Utility/CMakeLists.txt
+++ b/lldb/source/Plugins/Process/Utility/CMakeLists.txt
@@ -66,6 +66,7 @@ add_lldb_library(lldbPluginProcessUtility
     lldbSymbol
     lldbTarget
     lldbUtility
+    lldbValueObject
   LINK_COMPONENTS
     Support
     TargetParser
diff --git a/lldb/source/Plugins/Process/Utility/InferiorCallPOSIX.cpp b/lldb/source/Plugins/Process/Utility/InferiorCallPOSIX.cpp
index 32c71d87c7f5..99426c7d1f26 100644
--- a/lldb/source/Plugins/Process/Utility/InferiorCallPOSIX.cpp
+++ b/lldb/source/Plugins/Process/Utility/InferiorCallPOSIX.cpp
@@ -9,7 +9,6 @@
 #include "InferiorCallPOSIX.h"
 #include "lldb/Core/Address.h"
 #include "lldb/Core/Module.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Expression/DiagnosticManager.h"
 #include "lldb/Host/Config.h"
 #include "lldb/Symbol/SymbolContext.h"
@@ -19,6 +18,7 @@
 #include "lldb/Target/Process.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/ThreadPlanCallFunction.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #if LLDB_ENABLE_POSIX
 #include <sys/mman.h>
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/CMakeLists.txt b/lldb/source/Plugins/ScriptInterpreter/Python/CMakeLists.txt
index 7523d65abf0f..fd3118aa9c0a 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/CMakeLists.txt
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/CMakeLists.txt
@@ -33,6 +33,7 @@ add_lldb_library(lldbPluginScriptInterpreterPython PLUGIN
     lldbHost
     lldbInterpreter
     lldbTarget
+    lldbValueObject
     lldbPluginScriptInterpreterPythonInterfaces
     ${Python3_LIBRARIES}
     ${LLDB_LIBEDIT_LIBS}
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index db1a10e73a66..7cc38da6a6a9 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -29,7 +29,6 @@
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/ThreadedCommunication.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/HostInfo.h"
@@ -41,6 +40,7 @@
 #include "lldb/Utility/Instrumentation.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Timer.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/CMakeLists.txt b/lldb/source/Plugins/SymbolFile/DWARF/CMakeLists.txt
index 83c4f1a4cf89..ea644a61b1bd 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/CMakeLists.txt
+++ b/lldb/source/Plugins/SymbolFile/DWARF/CMakeLists.txt
@@ -45,6 +45,7 @@ add_lldb_library(lldbPluginSymbolFileDWARF PLUGIN
     lldbSymbol
     lldbTarget
     lldbUtility
+    lldbValueObject
     lldbPluginObjCLanguage
     lldbPluginCPlusPlusLanguage
     lldbPluginExpressionParserClang
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.cpp
index 4ed523bbb9e7..e53e930665a6 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.cpp
@@ -11,9 +11,9 @@
 #include "DWARFDIE.h"
 #include "SymbolFileDWARF.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Symbol/SymbolFile.h"
 #include "lldb/Target/StackFrame.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include <optional>
 
 using namespace lldb;
diff --git a/lldb/source/Plugins/SystemRuntime/MacOSX/AbortWithPayloadFrameRecognizer.cpp b/lldb/source/Plugins/SystemRuntime/MacOSX/AbortWithPayloadFrameRecognizer.cpp
index fd04db7807a4..7cf875075cd7 100644
--- a/lldb/source/Plugins/SystemRuntime/MacOSX/AbortWithPayloadFrameRecognizer.cpp
+++ b/lldb/source/Plugins/SystemRuntime/MacOSX/AbortWithPayloadFrameRecognizer.cpp
@@ -9,7 +9,6 @@
 #include "AbortWithPayloadFrameRecognizer.h"
 
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Target/ABI.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/StackFrame.h"
@@ -18,6 +17,7 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/StructuredData.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 
diff --git a/lldb/source/Plugins/SystemRuntime/MacOSX/CMakeLists.txt b/lldb/source/Plugins/SystemRuntime/MacOSX/CMakeLists.txt
index 17fccdf43c82..9dc196f45d60 100644
--- a/lldb/source/Plugins/SystemRuntime/MacOSX/CMakeLists.txt
+++ b/lldb/source/Plugins/SystemRuntime/MacOSX/CMakeLists.txt
@@ -14,6 +14,7 @@ add_lldb_library(lldbPluginSystemRuntimeMacOSX PLUGIN
     lldbSymbol
     lldbTarget
     lldbUtility
+    lldbValueObject
     lldbPluginProcessUtility
     lldbPluginTypeSystemClang
   )
diff --git a/lldb/source/Symbol/CMakeLists.txt b/lldb/source/Symbol/CMakeLists.txt
index e49477df06c9..86fa8583972b 100644
--- a/lldb/source/Symbol/CMakeLists.txt
+++ b/lldb/source/Symbol/CMakeLists.txt
@@ -39,6 +39,7 @@ add_lldb_library(lldbSymbol NO_PLUGIN_DEPENDENCIES
     lldbHost
     lldbTarget
     lldbUtility
+    lldbValueObject
 
   LINK_COMPONENTS
     Support
diff --git a/lldb/source/Symbol/Variable.cpp b/lldb/source/Symbol/Variable.cpp
index d4e1ce43ef1f..a63e4f973537 100644
--- a/lldb/source/Symbol/Variable.cpp
+++ b/lldb/source/Symbol/Variable.cpp
@@ -9,8 +9,6 @@
 #include "lldb/Symbol/Variable.h"
 
 #include "lldb/Core/Module.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectVariable.h"
 #include "lldb/Symbol/Block.h"
 #include "lldb/Symbol/CompileUnit.h"
 #include "lldb/Symbol/CompilerDecl.h"
@@ -31,6 +29,8 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectVariable.h"
 
 #include "llvm/ADT/Twine.h"
 
diff --git a/lldb/source/Target/ABI.cpp b/lldb/source/Target/ABI.cpp
index 1a301d4cae7a..b86fef6bf03e 100644
--- a/lldb/source/Target/ABI.cpp
+++ b/lldb/source/Target/ABI.cpp
@@ -9,7 +9,6 @@
 #include "lldb/Target/ABI.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Expression/ExpressionVariable.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Symbol/TypeSystem.h"
@@ -17,6 +16,7 @@
 #include "lldb/Target/Thread.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 #include "llvm/MC/TargetRegistry.h"
 #include <cctype>
 
diff --git a/lldb/source/Target/CMakeLists.txt b/lldb/source/Target/CMakeLists.txt
index a6d2eace9754..d3545913a690 100644
--- a/lldb/source/Target/CMakeLists.txt
+++ b/lldb/source/Target/CMakeLists.txt
@@ -90,6 +90,7 @@ add_lldb_library(lldbTarget
     lldbInterpreter
     lldbSymbol
     lldbUtility
+    lldbValueObject
     lldbPluginProcessUtility
 
   LINK_COMPONENTS
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index ed493e353161..3761b867452c 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -13,9 +13,6 @@
 #include "lldb/Core/Mangled.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectMemory.h"
-#include "lldb/Core/ValueObjectVariable.h"
 #include "lldb/Symbol/CompileUnit.h"
 #include "lldb/Symbol/Function.h"
 #include "lldb/Symbol/Symbol.h"
@@ -33,6 +30,9 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegisterValue.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
+#include "lldb/ValueObject/ValueObjectVariable.h"
 
 #include "lldb/lldb-enumerations.h"
 
diff --git a/lldb/source/Target/StopInfo.cpp b/lldb/source/Target/StopInfo.cpp
index bd7032b803df..60aa65ed38c7 100644
--- a/lldb/source/Target/StopInfo.cpp
+++ b/lldb/source/Target/StopInfo.cpp
@@ -14,7 +14,6 @@
 #include "lldb/Breakpoint/Watchpoint.h"
 #include "lldb/Breakpoint/WatchpointResource.h"
 #include "lldb/Core/Debugger.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Expression/UserExpression.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/StopInfo.h"
@@ -26,6 +25,7 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index 04395e37f042..199efae8a728 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -24,8 +24,6 @@
 #include "lldb/Core/Section.h"
 #include "lldb/Core/SourceManager.h"
 #include "lldb/Core/StructuredDataImpl.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Expression/DiagnosticManager.h"
 #include "lldb/Expression/ExpressionVariable.h"
 #include "lldb/Expression/REPL.h"
@@ -65,6 +63,8 @@
 #include "lldb/Utility/State.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/Timer.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SetVector.h"
diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp
index 902fbb2b519e..8373cdc36268 100644
--- a/lldb/source/Target/Thread.cpp
+++ b/lldb/source/Target/Thread.cpp
@@ -12,8 +12,6 @@
 #include "lldb/Core/FormatEntity.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/StructuredDataImpl.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Host/Host.h"
 #include "lldb/Interpreter/OptionValueFileSpecList.h"
 #include "lldb/Interpreter/OptionValueProperties.h"
@@ -50,6 +48,8 @@
 #include "lldb/Utility/State.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 #include "lldb/lldb-enumerations.h"
 
 #include <memory>
diff --git a/lldb/source/Target/ThreadPlanStepOut.cpp b/lldb/source/Target/ThreadPlanStepOut.cpp
index 8ca1dbc2fe4c..c0ea53e4a8cb 100644
--- a/lldb/source/Target/ThreadPlanStepOut.cpp
+++ b/lldb/source/Target/ThreadPlanStepOut.cpp
@@ -9,7 +9,6 @@
 #include "lldb/Target/ThreadPlanStepOut.h"
 #include "lldb/Breakpoint/Breakpoint.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/Symbol/Block.h"
 #include "lldb/Symbol/Function.h"
 #include "lldb/Symbol/Symbol.h"
@@ -23,6 +22,7 @@
 #include "lldb/Target/ThreadPlanStepThrough.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include <memory>
 
diff --git a/lldb/source/ValueObject/CMakeLists.txt b/lldb/source/ValueObject/CMakeLists.txt
new file mode 100644
index 000000000000..70cb3d6d53f0
--- /dev/null
+++ b/lldb/source/ValueObject/CMakeLists.txt
@@ -0,0 +1,35 @@
+add_lldb_library(lldbValueObject
+  ValueObject.cpp
+  ValueObjectCast.cpp
+  ValueObjectChild.cpp
+  ValueObjectConstResult.cpp
+  ValueObjectConstResultCast.cpp
+  ValueObjectConstResultChild.cpp
+  ValueObjectConstResultImpl.cpp
+  ValueObjectDynamicValue.cpp
+  ValueObjectList.cpp
+  ValueObjectMemory.cpp
+  ValueObjectRegister.cpp
+  ValueObjectSyntheticFilter.cpp
+  ValueObjectUpdater.cpp
+  ValueObjectVariable.cpp
+  ValueObjectVTable.cpp
+
+
+  LINK_LIBS
+    lldbBreakpoint
+    lldbDataFormatters
+    lldbExpression
+    lldbHost
+    lldbInterpreter
+    lldbSymbol
+    lldbTarget
+    lldbUtility
+    lldbPluginCPlusPlusLanguage
+    lldbPluginObjCLanguage
+
+  LINK_COMPONENTS
+    Support
+    Demangle
+    TargetParser
+  )
diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/ValueObject/ValueObject.cpp
similarity index 99%
rename from lldb/source/Core/ValueObject.cpp
rename to lldb/source/ValueObject/ValueObject.cpp
index 5e3839b89ce0..aca43175d12f 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/ValueObject/ValueObject.cpp
@@ -6,18 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/ValueObject.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include "lldb/Core/Address.h"
 #include "lldb/Core/Declaration.h"
 #include "lldb/Core/Module.h"
-#include "lldb/Core/ValueObjectCast.h"
-#include "lldb/Core/ValueObjectChild.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectDynamicValue.h"
-#include "lldb/Core/ValueObjectMemory.h"
-#include "lldb/Core/ValueObjectSyntheticFilter.h"
-#include "lldb/Core/ValueObjectVTable.h"
 #include "lldb/DataFormatters/DataVisualization.h"
 #include "lldb/DataFormatters/DumpValueObjectOptions.h"
 #include "lldb/DataFormatters/FormatManager.h"
@@ -48,6 +41,14 @@
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectCast.h"
+#include "lldb/ValueObject/ValueObjectChild.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectDynamicValue.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
+#include "lldb/ValueObject/ValueObjectSyntheticFilter.h"
+#include "lldb/ValueObject/ValueObjectVTable.h"
 #include "lldb/lldb-private-types.h"
 
 #include "llvm/Support/Compiler.h"
@@ -64,8 +65,6 @@
 #include <cstdio>
 #include <cstring>
 
-#include <lldb/Core/ValueObject.h>
-
 namespace lldb_private {
 class ExecutionContextScope;
 }
@@ -273,8 +272,6 @@ CompilerType ValueObject::MaybeCalculateCompleteType() {
   return compiler_type;
 }
 
-
-
 DataExtractor &ValueObject::GetDataExtractor() {
   UpdateValueIfNeeded(false);
   return m_data;
@@ -585,8 +582,8 @@ bool ValueObject::GetSummaryAsCString(TypeSummaryImpl *summary_ptr,
   // a valid summary string or function because the type is not complete and
   // no member variables or member functions will be available.
   if (GetCompilerType().IsForcefullyCompleted()) {
-      destination = "<incomplete type>";
-      return true;
+    destination = "<incomplete type>";
+    return true;
   }
 
   // ideally we would like to bail out if passing NULL, but if we do so we end
@@ -1732,8 +1729,7 @@ bool ValueObject::GetDeclaration(Declaration &decl) {
   return false;
 }
 
-void ValueObject::AddSyntheticChild(ConstString key,
-                                    ValueObject *valobj) {
+void ValueObject::AddSyntheticChild(ConstString key, ValueObject *valobj) {
   m_synthetic_children[key] = valobj;
 }
 
@@ -2958,13 +2954,13 @@ ValueObjectSP ValueObject::AddressOf(Status &error) {
 }
 
 ValueObjectSP ValueObject::DoCast(const CompilerType &compiler_type) {
-    return ValueObjectCast::Create(*this, GetName(), compiler_type);
+  return ValueObjectCast::Create(*this, GetName(), compiler_type);
 }
 
 ValueObjectSP ValueObject::Cast(const CompilerType &compiler_type) {
   // Only allow casts if the original type is equal or larger than the cast
   // type, unless we know this is a load address.  Getting the size wrong for
-  // a host side storage could leak lldb memory, so we absolutely want to 
+  // a host side storage could leak lldb memory, so we absolutely want to
   // prevent that.  We may not always get the right value, for instance if we
   // have an expression result value that's copied into a storage location in
   // the target may not have copied enough memory.  I'm not trying to fix that
@@ -2979,13 +2975,12 @@ ValueObjectSP ValueObject::Cast(const CompilerType &compiler_type) {
   Status error;
   CompilerType my_type = GetCompilerType();
 
-  ExecutionContextScope *exe_scope
-      = ExecutionContext(GetExecutionContextRef())
-          .GetBestExecutionContextScope();
-  if (compiler_type.GetByteSize(exe_scope)
-      <= GetCompilerType().GetByteSize(exe_scope) 
-      || m_value.GetValueType() == Value::ValueType::LoadAddress)
-        return DoCast(compiler_type);
+  ExecutionContextScope *exe_scope =
+      ExecutionContext(GetExecutionContextRef()).GetBestExecutionContextScope();
+  if (compiler_type.GetByteSize(exe_scope) <=
+          GetCompilerType().GetByteSize(exe_scope) ||
+      m_value.GetValueType() == Value::ValueType::LoadAddress)
+    return DoCast(compiler_type);
 
   error = Status::FromErrorString(
       "Can only cast to a type that is equal to or smaller "
@@ -3773,8 +3768,6 @@ bool ValueObject::CanProvideValue() {
   return (!type.IsValid()) || (0 != (type.GetTypeInfo() & eTypeHasValue));
 }
 
-
-
 ValueObjectSP ValueObject::Persist() {
   if (!UpdateValueIfNeeded())
     return nullptr;
diff --git a/lldb/source/Core/ValueObjectCast.cpp b/lldb/source/ValueObject/ValueObjectCast.cpp
similarity index 95%
rename from lldb/source/Core/ValueObjectCast.cpp
rename to lldb/source/ValueObject/ValueObjectCast.cpp
index 308fa161180d..6241f2397936 100644
--- a/lldb/source/Core/ValueObjectCast.cpp
+++ b/lldb/source/ValueObject/ValueObjectCast.cpp
@@ -6,14 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/ValueObjectCast.h"
+#include "lldb/ValueObject/ValueObjectCast.h"
 
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Target/ExecutionContext.h"
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include <optional>
 
 namespace lldb_private {
@@ -43,8 +43,7 @@ CompilerType ValueObjectCast::GetCompilerTypeImpl() { return m_cast_type; }
 
 llvm::Expected<uint32_t> ValueObjectCast::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
-  auto children_count = GetCompilerType().GetNumChildren(
-      true, &exe_ctx);
+  auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
   if (!children_count)
     return children_count;
   return *children_count <= max ? *children_count : max;
diff --git a/lldb/source/Core/ValueObjectChild.cpp b/lldb/source/ValueObject/ValueObjectChild.cpp
similarity index 95%
rename from lldb/source/Core/ValueObjectChild.cpp
rename to lldb/source/ValueObject/ValueObjectChild.cpp
index d4ced2f02299..ea211530591c 100644
--- a/lldb/source/Core/ValueObjectChild.cpp
+++ b/lldb/source/ValueObject/ValueObjectChild.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/ValueObjectChild.h"
+#include "lldb/ValueObject/ValueObjectChild.h"
 
 #include "lldb/Core/Value.h"
 #include "lldb/Symbol/CompilerType.h"
@@ -27,10 +27,9 @@
 using namespace lldb_private;
 
 ValueObjectChild::ValueObjectChild(
-    ValueObject &parent, const CompilerType &compiler_type,
-    ConstString name, uint64_t byte_size, int32_t byte_offset,
-    uint32_t bitfield_bit_size, uint32_t bitfield_bit_offset,
-    bool is_base_class, bool is_deref_of_parent,
+    ValueObject &parent, const CompilerType &compiler_type, ConstString name,
+    uint64_t byte_size, int32_t byte_offset, uint32_t bitfield_bit_size,
+    uint32_t bitfield_bit_offset, bool is_base_class, bool is_deref_of_parent,
     AddressType child_ptr_or_ref_addr_type, uint64_t language_flags)
     : ValueObject(parent), m_compiler_type(compiler_type),
       m_byte_size(byte_size), m_byte_offset(byte_offset),
@@ -200,8 +199,7 @@ bool ValueObjectChild::UpdateValue() {
             GetExecutionContextRef().Lock(thread_and_frame_only_if_stopped));
         if (GetCompilerType().GetTypeInfo() & lldb::eTypeHasValue) {
           Value &value = is_instance_ptr_base ? m_parent->GetValue() : m_value;
-          m_error =
-              value.GetValueAsData(&exe_ctx, m_data, GetModule().get());
+          m_error = value.GetValueAsData(&exe_ctx, m_data, GetModule().get());
         } else {
           m_error.Clear(); // No value so nothing to read...
         }
diff --git a/lldb/source/Core/ValueObjectConstResult.cpp b/lldb/source/ValueObject/ValueObjectConstResult.cpp
similarity index 96%
rename from lldb/source/Core/ValueObjectConstResult.cpp
rename to lldb/source/ValueObject/ValueObjectConstResult.cpp
index 60850c15e6a8..ba4f7aa24462 100644
--- a/lldb/source/Core/ValueObjectConstResult.cpp
+++ b/lldb/source/ValueObject/ValueObjectConstResult.cpp
@@ -6,9 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
-#include "lldb/Core/ValueObjectDynamicValue.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Target/ExecutionContext.h"
 #include "lldb/Target/ExecutionContextScope.h"
@@ -17,6 +16,7 @@
 #include "lldb/Utility/DataBufferHeap.h"
 #include "lldb/Utility/DataExtractor.h"
 #include "lldb/Utility/Scalar.h"
+#include "lldb/ValueObject/ValueObjectDynamicValue.h"
 #include <optional>
 
 namespace lldb_private {
@@ -97,8 +97,7 @@ ValueObjectSP ValueObjectConstResult::Create(ExecutionContextScope *exe_scope,
 }
 
 ValueObjectSP ValueObjectConstResult::Create(ExecutionContextScope *exe_scope,
-                                             Value &value,
-                                             ConstString name,
+                                             Value &value, ConstString name,
                                              Module *module) {
   auto manager_sp = ValueObjectManager::Create();
   return (new ValueObjectConstResult(exe_scope, *manager_sp, value, name,
@@ -141,8 +140,7 @@ ValueObjectConstResult::ValueObjectConstResult(
     ExecutionContextScope *exe_scope, ValueObjectManager &manager,
     const CompilerType &compiler_type, ConstString name, lldb::addr_t address,
     AddressType address_type, uint32_t addr_byte_size)
-    : ValueObject(exe_scope, manager), m_type_name(),
-      m_impl(this, address) {
+    : ValueObject(exe_scope, manager), m_type_name(), m_impl(this, address) {
   m_value.GetScalar() = address;
   m_data.SetAddressByteSize(addr_byte_size);
   m_value.GetScalar().GetData(m_data, addr_byte_size);
@@ -208,8 +206,8 @@ lldb::ValueType ValueObjectConstResult::GetValueType() const {
 std::optional<uint64_t> ValueObjectConstResult::GetByteSize() {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   if (!m_byte_size) {
-    if (auto size =
-        GetCompilerType().GetByteSize(exe_ctx.GetBestExecutionContextScope()))
+    if (auto size = GetCompilerType().GetByteSize(
+            exe_ctx.GetBestExecutionContextScope()))
       SetByteSize(*size);
   }
   return m_byte_size;
diff --git a/lldb/source/Core/ValueObjectConstResultCast.cpp b/lldb/source/ValueObject/ValueObjectConstResultCast.cpp
similarity index 96%
rename from lldb/source/Core/ValueObjectConstResultCast.cpp
rename to lldb/source/ValueObject/ValueObjectConstResultCast.cpp
index bf7a12dc6823..ac6eef5e415e 100644
--- a/lldb/source/Core/ValueObjectConstResultCast.cpp
+++ b/lldb/source/ValueObject/ValueObjectConstResultCast.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/ValueObjectConstResultCast.h"
+#include "lldb/ValueObject/ValueObjectConstResultCast.h"
 
 namespace lldb_private {
 class DataExtractor;
diff --git a/lldb/source/Core/ValueObjectConstResultChild.cpp b/lldb/source/ValueObject/ValueObjectConstResultChild.cpp
similarity index 80%
rename from lldb/source/Core/ValueObjectConstResultChild.cpp
rename to lldb/source/ValueObject/ValueObjectConstResultChild.cpp
index 39fc0c9fbb35..b1c800dfe8c3 100644
--- a/lldb/source/Core/ValueObjectConstResultChild.cpp
+++ b/lldb/source/ValueObject/ValueObjectConstResultChild.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/ValueObjectConstResultChild.h"
+#include "lldb/ValueObject/ValueObjectConstResultChild.h"
 
 #include "lldb/lldb-private-enumerations.h"
 namespace lldb_private {
@@ -22,11 +22,10 @@ class ValueObject;
 using namespace lldb_private;
 
 ValueObjectConstResultChild::ValueObjectConstResultChild(
-    ValueObject &parent, const CompilerType &compiler_type,
-    ConstString name, uint32_t byte_size, int32_t byte_offset,
-    uint32_t bitfield_bit_size, uint32_t bitfield_bit_offset,
-    bool is_base_class, bool is_deref_of_parent, lldb::addr_t live_address,
-    uint64_t language_flags)
+    ValueObject &parent, const CompilerType &compiler_type, ConstString name,
+    uint32_t byte_size, int32_t byte_offset, uint32_t bitfield_bit_size,
+    uint32_t bitfield_bit_offset, bool is_base_class, bool is_deref_of_parent,
+    lldb::addr_t live_address, uint64_t language_flags)
     : ValueObjectChild(parent, compiler_type, name, byte_size, byte_offset,
                        bitfield_bit_size, bitfield_bit_offset, is_base_class,
                        is_deref_of_parent, eAddressTypeLoad, language_flags),
@@ -51,8 +50,9 @@ lldb::ValueObjectSP ValueObjectConstResultChild::AddressOf(Status &error) {
   return m_impl.AddressOf(error);
 }
 
-lldb::addr_t ValueObjectConstResultChild::GetAddressOf(
-  bool scalar_is_load_address, AddressType* address_type) {
+lldb::addr_t
+ValueObjectConstResultChild::GetAddressOf(bool scalar_is_load_address,
+                                          AddressType *address_type) {
   return m_impl.GetAddressOf(scalar_is_load_address, address_type);
 }
 
diff --git a/lldb/source/Core/ValueObjectConstResultImpl.cpp b/lldb/source/ValueObject/ValueObjectConstResultImpl.cpp
similarity index 96%
rename from lldb/source/Core/ValueObjectConstResultImpl.cpp
rename to lldb/source/ValueObject/ValueObjectConstResultImpl.cpp
index 2a7c90770076..d79f655b015a 100644
--- a/lldb/source/Core/ValueObjectConstResultImpl.cpp
+++ b/lldb/source/ValueObject/ValueObjectConstResultImpl.cpp
@@ -6,13 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/ValueObjectConstResultImpl.h"
+#include "lldb/ValueObject/ValueObjectConstResultImpl.h"
 
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
-#include "lldb/Core/ValueObjectConstResultCast.h"
-#include "lldb/Core/ValueObjectConstResultChild.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Target/ExecutionContext.h"
 #include "lldb/Utility/DataBufferHeap.h"
@@ -20,6 +16,10 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Scalar.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
+#include "lldb/ValueObject/ValueObjectConstResultCast.h"
+#include "lldb/ValueObject/ValueObjectConstResultChild.h"
 
 #include <string>
 
@@ -36,8 +36,7 @@ using namespace lldb_private;
 ValueObjectConstResultImpl::ValueObjectConstResultImpl(
     ValueObject *valobj, lldb::addr_t live_address)
     : m_impl_backend(valobj), m_live_address(live_address),
-      m_live_address_type(eAddressTypeLoad),
-      m_address_of_backend() {}
+      m_live_address_type(eAddressTypeLoad), m_address_of_backend() {}
 
 lldb::ValueObjectSP ValueObjectConstResultImpl::Dereference(Status &error) {
   if (m_impl_backend == nullptr)
diff --git a/lldb/source/Core/ValueObjectDynamicValue.cpp b/lldb/source/ValueObject/ValueObjectDynamicValue.cpp
similarity index 99%
rename from lldb/source/Core/ValueObjectDynamicValue.cpp
rename to lldb/source/ValueObject/ValueObjectDynamicValue.cpp
index 67311ea6e3be..588c644bbfd0 100644
--- a/lldb/source/Core/ValueObjectDynamicValue.cpp
+++ b/lldb/source/ValueObject/ValueObjectDynamicValue.cpp
@@ -6,9 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/ValueObjectDynamicValue.h"
+#include "lldb/ValueObject/ValueObjectDynamicValue.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Symbol/Type.h"
 #include "lldb/Target/ExecutionContext.h"
@@ -20,6 +19,7 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-types.h"
 
 #include <cstring>
@@ -203,7 +203,7 @@ bool ValueObjectDynamicValue::UpdateValue() {
 
   // If we don't have a dynamic type, set ourselves to be invalid and return
   // false.  We used to try to produce a dynamic ValueObject that behaved "like"
-  // its parent, but that failed for ValueObjectConstResult, which is too 
+  // its parent, but that failed for ValueObjectConstResult, which is too
   // complex a beast to try to emulate.  If we return an invalid ValueObject,
   // clients will end up getting the static value instead, which behaves
   // correctly.
diff --git a/lldb/source/Core/ValueObjectList.cpp b/lldb/source/ValueObject/ValueObjectList.cpp
similarity index 97%
rename from lldb/source/Core/ValueObjectList.cpp
rename to lldb/source/ValueObject/ValueObjectList.cpp
index 28907261f0a6..a79a040bdf7f 100644
--- a/lldb/source/Core/ValueObjectList.cpp
+++ b/lldb/source/ValueObject/ValueObjectList.cpp
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/ValueObjectList.h"
+#include "lldb/ValueObject/ValueObjectList.h"
 
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include <utility>
 
diff --git a/lldb/source/Core/ValueObjectMemory.cpp b/lldb/source/ValueObject/ValueObjectMemory.cpp
similarity index 98%
rename from lldb/source/Core/ValueObjectMemory.cpp
rename to lldb/source/ValueObject/ValueObjectMemory.cpp
index 8013a0ea92ba..e2b5e8bfadf5 100644
--- a/lldb/source/Core/ValueObjectMemory.cpp
+++ b/lldb/source/ValueObject/ValueObjectMemory.cpp
@@ -6,15 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/ValueObjectMemory.h"
+#include "lldb/ValueObject/ValueObjectMemory.h"
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/Symbol/Type.h"
 #include "lldb/Target/ExecutionContext.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/DataExtractor.h"
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-types.h"
 #include "llvm/Support/ErrorHandling.h"
 
diff --git a/lldb/source/Core/ValueObjectRegister.cpp b/lldb/source/ValueObject/ValueObjectRegister.cpp
similarity index 99%
rename from lldb/source/Core/ValueObjectRegister.cpp
rename to lldb/source/ValueObject/ValueObjectRegister.cpp
index 1d1ae23f6ce2..805c921bfc9b 100644
--- a/lldb/source/Core/ValueObjectRegister.cpp
+++ b/lldb/source/ValueObject/ValueObjectRegister.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/ValueObjectRegister.h"
+#include "lldb/ValueObject/ValueObjectRegister.h"
 
 #include "lldb/Core/Module.h"
 #include "lldb/Core/Value.h"
diff --git a/lldb/source/Core/ValueObjectSyntheticFilter.cpp b/lldb/source/ValueObject/ValueObjectSyntheticFilter.cpp
similarity index 98%
rename from lldb/source/Core/ValueObjectSyntheticFilter.cpp
rename to lldb/source/ValueObject/ValueObjectSyntheticFilter.cpp
index 091b8f883b60..fbb329b0896d 100644
--- a/lldb/source/Core/ValueObjectSyntheticFilter.cpp
+++ b/lldb/source/ValueObject/ValueObjectSyntheticFilter.cpp
@@ -6,16 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/ValueObjectSyntheticFilter.h"
+#include "lldb/ValueObject/ValueObjectSyntheticFilter.h"
 
 #include "lldb/Core/Value.h"
-#include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSynthetic.h"
 #include "lldb/Target/ExecutionContext.h"
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/ValueObject/ValueObject.h"
 
 #include "llvm/ADT/STLExtras.h"
 #include <optional>
@@ -143,11 +143,9 @@ lldb::ValueType ValueObjectSynthetic::GetValueType() const {
 
 void ValueObjectSynthetic::CreateSynthFilter() {
   ValueObject *valobj_for_frontend = m_parent;
-  if (m_synth_sp->WantsDereference())
-  {
+  if (m_synth_sp->WantsDereference()) {
     CompilerType type = m_parent->GetCompilerType();
-    if (type.IsValid() && type.IsPointerOrReferenceType())
-    {
+    if (type.IsValid() && type.IsPointerOrReferenceType()) {
       Status error;
       lldb::ValueObjectSP deref_sp = m_parent->Dereference(error);
       if (error.Success())
diff --git a/lldb/source/Core/ValueObjectUpdater.cpp b/lldb/source/ValueObject/ValueObjectUpdater.cpp
similarity index 97%
rename from lldb/source/Core/ValueObjectUpdater.cpp
rename to lldb/source/ValueObject/ValueObjectUpdater.cpp
index af7f976a6d27..9260ceae8aa7 100644
--- a/lldb/source/Core/ValueObjectUpdater.cpp
+++ b/lldb/source/ValueObject/ValueObjectUpdater.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/ValueObjectUpdater.h"
+#include "lldb/ValueObject/ValueObjectUpdater.h"
 
 using namespace lldb_private;
 
diff --git a/lldb/source/Core/ValueObjectVTable.cpp b/lldb/source/ValueObject/ValueObjectVTable.cpp
similarity index 96%
rename from lldb/source/Core/ValueObjectVTable.cpp
rename to lldb/source/ValueObject/ValueObjectVTable.cpp
index e38f0a83df99..7171154339f9 100644
--- a/lldb/source/Core/ValueObjectVTable.cpp
+++ b/lldb/source/ValueObject/ValueObjectVTable.cpp
@@ -6,12 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/ValueObjectVTable.h"
+#include "lldb/ValueObject/ValueObjectVTable.h"
 #include "lldb/Core/Module.h"
-#include "lldb/Core/ValueObjectChild.h"
 #include "lldb/Symbol/Function.h"
 #include "lldb/Target/Language.h"
 #include "lldb/Target/LanguageRuntime.h"
+#include "lldb/ValueObject/ValueObjectChild.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
@@ -84,7 +84,6 @@ protected:
       return false;
     }
 
-
     // Set our value to be the load address of the function pointer in memory
     // and our type to be the function pointer type.
     m_value.SetValueType(Value::ValueType::LoadAddress);
@@ -109,7 +108,7 @@ protected:
       // the language from it correctly.
       ValueObject *val = parent->GetParent();
       auto type_system = target_sp->GetScratchTypeSystemForLanguage(
-            val ? val->GetObjectRuntimeLanguage() : eLanguageTypeC_plus_plus);
+          val ? val->GetObjectRuntimeLanguage() : eLanguageTypeC_plus_plus);
       if (type_system) {
         m_value.SetCompilerType(
             (*type_system)->CreateGenericFunctionPrototype().GetPointerType());
@@ -124,7 +123,7 @@ protected:
     if (m_error.Success()) {
       const bool thread_and_frame_only_if_stopped = true;
       ExecutionContext exe_ctx(
-        GetExecutionContextRef().Lock(thread_and_frame_only_if_stopped));
+          GetExecutionContextRef().Lock(thread_and_frame_only_if_stopped));
       m_error = m_value.GetValueAsData(&exe_ctx, m_data, GetModule().get());
     }
     SetValueDidChange(true);
@@ -255,7 +254,7 @@ bool ValueObjectVTable::UpdateValue() {
   m_value.SetValueType(Value::ValueType::LoadAddress);
   m_value.GetScalar() = parent->GetAddressOf();
   auto type_system_or_err =
-        target_sp->GetScratchTypeSystemForLanguage(eLanguageTypeC_plus_plus);
+      target_sp->GetScratchTypeSystemForLanguage(eLanguageTypeC_plus_plus);
   if (type_system_or_err) {
     m_value.SetCompilerType(
         (*type_system_or_err)->GetBasicTypeFromAST(eBasicTypeUnsignedLong));
diff --git a/lldb/source/Core/ValueObjectVariable.cpp b/lldb/source/ValueObject/ValueObjectVariable.cpp
similarity index 98%
rename from lldb/source/Core/ValueObjectVariable.cpp
rename to lldb/source/ValueObject/ValueObjectVariable.cpp
index 29aefb270c92..db664ce9a7a2 100644
--- a/lldb/source/Core/ValueObjectVariable.cpp
+++ b/lldb/source/ValueObject/ValueObjectVariable.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/ValueObjectVariable.h"
+#include "lldb/ValueObject/ValueObjectVariable.h"
 
 #include "lldb/Core/Address.h"
 #include "lldb/Core/AddressRange.h"
@@ -209,8 +209,7 @@ bool ValueObjectVariable::UpdateValue() {
       case Value::ValueType::Scalar:
         // The variable value is in the Scalar value inside the m_value. We can
         // point our m_data right to it.
-        m_error =
-            m_value.GetValueAsData(&exe_ctx, m_data, GetModule().get());
+        m_error = m_value.GetValueAsData(&exe_ctx, m_data, GetModule().get());
         break;
 
       case Value::ValueType::FileAddress:
@@ -238,8 +237,7 @@ bool ValueObjectVariable::UpdateValue() {
           // extract read its value into m_data appropriately
           Value value(m_value);
           value.SetContext(Value::ContextType::Variable, variable);
-          m_error =
-              value.GetValueAsData(&exe_ctx, m_data, GetModule().get());
+          m_error = value.GetValueAsData(&exe_ctx, m_data, GetModule().get());
 
           SetValueDidChange(value_type != old_value.GetValueType() ||
                             m_value.GetScalar() != old_value.GetScalar());
@@ -311,8 +309,6 @@ void ValueObjectVariable::DoUpdateChildrenAddressType(ValueObject &valobj) {
   }
 }
 
-
-
 bool ValueObjectVariable::IsInScope() {
   const ExecutionContextRef &exe_ctx_ref = GetExecutionContextRef();
   if (exe_ctx_ref.HasFrameRef()) {
diff --git a/lldb/unittests/ValueObject/CMakeLists.txt b/lldb/unittests/ValueObject/CMakeLists.txt
index fb31f7650628..8fcc8d62a799 100644
--- a/lldb/unittests/ValueObject/CMakeLists.txt
+++ b/lldb/unittests/ValueObject/CMakeLists.txt
@@ -2,6 +2,7 @@ add_lldb_unittest(LLDBValueObjectTests
   DumpValueObjectOptionsTests.cpp
 
   LINK_LIBS
+    lldbValueObject
     lldbPluginPlatformLinux
     lldbPluginScriptInterpreterNone
 
diff --git a/lldb/unittests/ValueObject/DumpValueObjectOptionsTests.cpp b/lldb/unittests/ValueObject/DumpValueObjectOptionsTests.cpp
index 950e981a3f5a..c3fb8cbeb60a 100644
--- a/lldb/unittests/ValueObject/DumpValueObjectOptionsTests.cpp
+++ b/lldb/unittests/ValueObject/DumpValueObjectOptionsTests.cpp
@@ -12,9 +12,9 @@
 #include "TestingSupport/SubsystemRAII.h"
 #include "TestingSupport/Symbol/ClangTestUtils.h"
 #include "lldb/Core/Debugger.h"
-#include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/DumpValueObjectOptions.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "gtest/gtest.h"
 
-- 
GitLab


From cf8d24531e0464d32031e5490cf2772f47645674 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Thu, 24 Oct 2024 20:46:49 -0700
Subject: [PATCH 118/693] [msan] Reduces overhead of #113200, by 10% (#113201)

CTMark #113200 size overhead was 5.3%, now it's 4.7%.

The patch affects only signed integers.

https://alive2.llvm.org/ce/z/Lv5hyi

* The patch replaces code which extracted sign bit,
maximized/minimized it, then packed it back, with
simple sign bit flip. The another way to think about
transformation is as a subtraction of MIN_SINT from
A/B. Then we map MIN_SINT to 0, 0 to -MIN_SINT, and
MAX_SINT to MAX_UINT.

* Then to maximize/minimize A/B we don't need
to extract sign bit, we can apply shadow the
same way as to other bits.

* After sign bit flip, we had to switch to unsigned
version of the predicates.

* After change above  getHighestPossibleValue/getLowestPossibleValue
became very similar, so we can combine into a single function.

* Because the function does sign bit flip and
requires unsigned predicates used for returned values,
there is no point in keeping it as a member of class,
to hide, we switch to function local lambda.
---
 .../Instrumentation/MemorySanitizer.cpp       |  64 +-
 .../MemorySanitizer/msan_basic.ll             | 606 ++++++------------
 .../MemorySanitizer/pr32842.ll                |  36 +-
 3 files changed, 217 insertions(+), 489 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 0208c800a4fc..391fb30d95e2 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -2694,40 +2694,6 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOriginForNaryOp(I);
   }
 
-  /// Build the lowest possible value of V, taking into account V's
-  ///        uninitialized bits.
-  Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
-                                bool isSigned) {
-    if (isSigned) {
-      // Split shadow into sign bit and other bits.
-      Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
-      Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
-      // Maximise the undefined shadow bit, minimize other undefined bits.
-      return IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)),
-                          SaSignBit);
-    } else {
-      // Minimize undefined bits.
-      return IRB.CreateAnd(A, IRB.CreateNot(Sa));
-    }
-  }
-
-  /// Build the highest possible value of V, taking into account V's
-  ///        uninitialized bits.
-  Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
-                                 bool isSigned) {
-    if (isSigned) {
-      // Split shadow into sign bit and other bits.
-      Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
-      Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
-      // Minimise the undefined shadow bit, maximise other undefined bits.
-      return IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)),
-                          SaOtherBits);
-    } else {
-      // Maximize undefined bits.
-      return IRB.CreateOr(A, Sa);
-    }
-  }
-
   /// Instrument relational comparisons.
   ///
   /// This function does exact shadow propagation for all relational
@@ -2750,12 +2716,30 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     // its undefined bits. Let [b0, b1] be the interval of possible values of B.
     // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
     bool IsSigned = I.isSigned();
-    Value *Amin = getLowestPossibleValue(IRB, A, Sa, IsSigned);
-    Value *Bmax = getHighestPossibleValue(IRB, B, Sb, IsSigned);
-    Value *S1 = IRB.CreateICmp(I.getPredicate(), Amin, Bmax);
-    Value *Amax = getHighestPossibleValue(IRB, A, Sa, IsSigned);
-    Value *Bmin = getLowestPossibleValue(IRB, B, Sb, IsSigned);
-    Value *S2 = IRB.CreateICmp(I.getPredicate(), Amax, Bmin);
+
+    auto GetMinMaxUnsigned = [&](Value *V, Value *S) {
+      if (IsSigned) {
+        // Sign-flip to map from signed range to unsigned range. Relation A vs B
+        // should be preserved, if checked with `getUnsignedPredicate()`.
+        // Relationship between Amin, Amax, Bmin, Bmax also will not be
+        // affected, as they are created by effectively adding/substructing from
+        // A (or B) a value, derived from shadow, with no overflow, either
+        // before or after sign flip.
+        APInt MinVal =
+            APInt::getSignedMinValue(V->getType()->getScalarSizeInBits());
+        V = IRB.CreateXor(V, ConstantInt::get(V->getType(), MinVal));
+      }
+      // Minimize undefined bits.
+      Value *Min = IRB.CreateAnd(V, IRB.CreateNot(S));
+      Value *Max = IRB.CreateOr(V, S);
+      return std::make_pair(Min, Max);
+    };
+
+    auto [Amin, Amax] = GetMinMaxUnsigned(A, Sa);
+    auto [Bmin, Bmax] = GetMinMaxUnsigned(B, Sb);
+    Value *S1 = IRB.CreateICmp(I.getUnsignedPredicate(), Amin, Bmax);
+    Value *S2 = IRB.CreateICmp(I.getUnsignedPredicate(), Amax, Bmin);
+
     Value *Si = IRB.CreateXor(S1, S2);
     setShadow(&I, Si);
     setOriginForNaryOp(I);
diff --git a/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll b/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll
index 7aeb763b3090..8746f7f19023 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll
@@ -1622,20 +1622,12 @@ define zeroext i1 @ICmpSLTZero(i32 %x) nounwind uwtable readnone sanitize_memory
 ; CHECK-SAME: i32 [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = lshr i32 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], -1
-; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[X]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp slt i32 [[TMP7]], 0
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = lshr i32 [[TMP9]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = xor i32 [[TMP1]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP11]], -1
-; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[X]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or i32 [[TMP13]], [[TMP10]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt i32 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[X]], -2147483648
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ult i32 [[TMP4]], -2147483648
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ult i32 [[TMP5]], -2147483648
 ; CHECK-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP8]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp slt i32 [[X]], 0
 ; CHECK-NEXT:    store i1 [[TMP16]], ptr @__msan_retval_tls, align 8
@@ -1646,20 +1638,12 @@ define zeroext i1 @ICmpSLTZero(i32 %x) nounwind uwtable readnone sanitize_memory
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; ORIGIN-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; ORIGIN-NEXT:    call void @llvm.donothing()
-; ORIGIN-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; ORIGIN-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; ORIGIN-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP4]], -1
-; ORIGIN-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; ORIGIN-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP5]]
-; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp slt i32 [[TMP8]], 0
-; ORIGIN-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; ORIGIN-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; ORIGIN-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP12]], -1
-; ORIGIN-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; ORIGIN-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP11]]
-; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp slt i32 [[TMP15]], 0
+; ORIGIN-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; ORIGIN-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; ORIGIN-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; ORIGIN-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp ult i32 [[TMP5]], -2147483648
+; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp ult i32 [[TMP6]], -2147483648
 ; ORIGIN-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; ORIGIN-NEXT:    [[TMP18:%.*]] = icmp slt i32 [[X]], 0
 ; ORIGIN-NEXT:    store i1 [[TMP17]], ptr @__msan_retval_tls, align 8
@@ -1671,20 +1655,12 @@ define zeroext i1 @ICmpSLTZero(i32 %x) nounwind uwtable readnone sanitize_memory
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; CALLS-NEXT:    call void @llvm.donothing()
-; CALLS-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; CALLS-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; CALLS-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP4]], -1
-; CALLS-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; CALLS-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP5]]
-; CALLS-NEXT:    [[TMP9:%.*]] = icmp slt i32 [[TMP8]], 0
-; CALLS-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; CALLS-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; CALLS-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP12]], -1
-; CALLS-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; CALLS-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP11]]
-; CALLS-NEXT:    [[TMP16:%.*]] = icmp slt i32 [[TMP15]], 0
+; CALLS-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; CALLS-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; CALLS-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; CALLS-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; CALLS-NEXT:    [[TMP9:%.*]] = icmp ult i32 [[TMP5]], -2147483648
+; CALLS-NEXT:    [[TMP16:%.*]] = icmp ult i32 [[TMP6]], -2147483648
 ; CALLS-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; CALLS-NEXT:    [[TMP18:%.*]] = icmp slt i32 [[X]], 0
 ; CALLS-NEXT:    store i1 [[TMP17]], ptr @__msan_retval_tls, align 8
@@ -1701,20 +1677,12 @@ define zeroext i1 @ICmpSGEZero(i32 %x) nounwind uwtable readnone sanitize_memory
 ; CHECK-SAME: i32 [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = lshr i32 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], -1
-; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[X]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp sge i32 [[TMP7]], 0
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = lshr i32 [[TMP9]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = xor i32 [[TMP1]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP11]], -1
-; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[X]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or i32 [[TMP13]], [[TMP10]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp sge i32 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[X]], -2147483648
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp uge i32 [[TMP4]], -2147483648
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp uge i32 [[TMP5]], -2147483648
 ; CHECK-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP8]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp sge i32 [[X]], 0
 ; CHECK-NEXT:    store i1 [[TMP16]], ptr @__msan_retval_tls, align 8
@@ -1725,20 +1693,12 @@ define zeroext i1 @ICmpSGEZero(i32 %x) nounwind uwtable readnone sanitize_memory
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; ORIGIN-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; ORIGIN-NEXT:    call void @llvm.donothing()
-; ORIGIN-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; ORIGIN-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; ORIGIN-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP4]], -1
-; ORIGIN-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; ORIGIN-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP5]]
-; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp sge i32 [[TMP8]], 0
-; ORIGIN-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; ORIGIN-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; ORIGIN-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP12]], -1
-; ORIGIN-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; ORIGIN-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP11]]
-; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp sge i32 [[TMP15]], 0
+; ORIGIN-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; ORIGIN-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; ORIGIN-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; ORIGIN-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp uge i32 [[TMP5]], -2147483648
+; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp uge i32 [[TMP6]], -2147483648
 ; ORIGIN-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; ORIGIN-NEXT:    [[TMP18:%.*]] = icmp sge i32 [[X]], 0
 ; ORIGIN-NEXT:    store i1 [[TMP17]], ptr @__msan_retval_tls, align 8
@@ -1750,20 +1710,12 @@ define zeroext i1 @ICmpSGEZero(i32 %x) nounwind uwtable readnone sanitize_memory
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; CALLS-NEXT:    call void @llvm.donothing()
-; CALLS-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; CALLS-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; CALLS-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP4]], -1
-; CALLS-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; CALLS-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP5]]
-; CALLS-NEXT:    [[TMP9:%.*]] = icmp sge i32 [[TMP8]], 0
-; CALLS-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; CALLS-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; CALLS-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP12]], -1
-; CALLS-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; CALLS-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP11]]
-; CALLS-NEXT:    [[TMP16:%.*]] = icmp sge i32 [[TMP15]], 0
+; CALLS-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; CALLS-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; CALLS-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; CALLS-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; CALLS-NEXT:    [[TMP9:%.*]] = icmp uge i32 [[TMP5]], -2147483648
+; CALLS-NEXT:    [[TMP16:%.*]] = icmp uge i32 [[TMP6]], -2147483648
 ; CALLS-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; CALLS-NEXT:    [[TMP18:%.*]] = icmp sge i32 [[X]], 0
 ; CALLS-NEXT:    store i1 [[TMP17]], ptr @__msan_retval_tls, align 8
@@ -1780,20 +1732,12 @@ define zeroext i1 @ICmpSGTZero(i32 %x) nounwind uwtable readnone sanitize_memory
 ; CHECK-SAME: i32 [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = lshr i32 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP4]], -1
-; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[X]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP3]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp sgt i32 0, [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = lshr i32 [[TMP9]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = xor i32 [[TMP1]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], -1
-; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[X]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or i32 [[TMP13]], [[TMP11]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt i32 0, [[TMP14]]
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[X]], -2147483648
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i32 -2147483648, [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ugt i32 -2147483648, [[TMP4]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP8]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp sgt i32 0, [[X]]
 ; CHECK-NEXT:    store i1 [[TMP16]], ptr @__msan_retval_tls, align 8
@@ -1804,20 +1748,12 @@ define zeroext i1 @ICmpSGTZero(i32 %x) nounwind uwtable readnone sanitize_memory
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; ORIGIN-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; ORIGIN-NEXT:    call void @llvm.donothing()
-; ORIGIN-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; ORIGIN-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; ORIGIN-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP5]], -1
-; ORIGIN-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; ORIGIN-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP4]]
-; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp sgt i32 0, [[TMP8]]
-; ORIGIN-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; ORIGIN-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; ORIGIN-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP11]], -1
-; ORIGIN-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; ORIGIN-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP12]]
-; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp sgt i32 0, [[TMP15]]
+; ORIGIN-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; ORIGIN-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; ORIGIN-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; ORIGIN-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp ugt i32 -2147483648, [[TMP6]]
+; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp ugt i32 -2147483648, [[TMP5]]
 ; ORIGIN-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; ORIGIN-NEXT:    [[TMP18:%.*]] = icmp ne i32 [[TMP1]], 0
 ; ORIGIN-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP2]], i32 0
@@ -1831,20 +1767,12 @@ define zeroext i1 @ICmpSGTZero(i32 %x) nounwind uwtable readnone sanitize_memory
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; CALLS-NEXT:    call void @llvm.donothing()
-; CALLS-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; CALLS-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; CALLS-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP5]], -1
-; CALLS-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; CALLS-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP4]]
-; CALLS-NEXT:    [[TMP9:%.*]] = icmp sgt i32 0, [[TMP8]]
-; CALLS-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; CALLS-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; CALLS-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP11]], -1
-; CALLS-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; CALLS-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP12]]
-; CALLS-NEXT:    [[TMP16:%.*]] = icmp sgt i32 0, [[TMP15]]
+; CALLS-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; CALLS-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; CALLS-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; CALLS-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; CALLS-NEXT:    [[TMP9:%.*]] = icmp ugt i32 -2147483648, [[TMP6]]
+; CALLS-NEXT:    [[TMP16:%.*]] = icmp ugt i32 -2147483648, [[TMP5]]
 ; CALLS-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; CALLS-NEXT:    [[TMP18:%.*]] = icmp ne i32 [[TMP1]], 0
 ; CALLS-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP2]], i32 0
@@ -1863,20 +1791,12 @@ define zeroext i1 @ICmpSLEZero(i32 %x) nounwind uwtable readnone sanitize_memory
 ; CHECK-SAME: i32 [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = lshr i32 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP4]], -1
-; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[X]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP3]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp sle i32 0, [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = lshr i32 [[TMP9]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = xor i32 [[TMP1]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], -1
-; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[X]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or i32 [[TMP13]], [[TMP11]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp sle i32 0, [[TMP14]]
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[X]], -2147483648
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ule i32 -2147483648, [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ule i32 -2147483648, [[TMP4]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP8]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp sle i32 0, [[X]]
 ; CHECK-NEXT:    store i1 [[TMP16]], ptr @__msan_retval_tls, align 8
@@ -1887,20 +1807,12 @@ define zeroext i1 @ICmpSLEZero(i32 %x) nounwind uwtable readnone sanitize_memory
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; ORIGIN-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; ORIGIN-NEXT:    call void @llvm.donothing()
-; ORIGIN-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; ORIGIN-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; ORIGIN-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP5]], -1
-; ORIGIN-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; ORIGIN-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP4]]
-; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp sle i32 0, [[TMP8]]
-; ORIGIN-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; ORIGIN-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; ORIGIN-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP11]], -1
-; ORIGIN-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; ORIGIN-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP12]]
-; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp sle i32 0, [[TMP15]]
+; ORIGIN-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; ORIGIN-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; ORIGIN-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; ORIGIN-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp ule i32 -2147483648, [[TMP6]]
+; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp ule i32 -2147483648, [[TMP5]]
 ; ORIGIN-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; ORIGIN-NEXT:    [[TMP18:%.*]] = icmp ne i32 [[TMP1]], 0
 ; ORIGIN-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP2]], i32 0
@@ -1914,20 +1826,12 @@ define zeroext i1 @ICmpSLEZero(i32 %x) nounwind uwtable readnone sanitize_memory
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; CALLS-NEXT:    call void @llvm.donothing()
-; CALLS-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; CALLS-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; CALLS-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP5]], -1
-; CALLS-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; CALLS-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP4]]
-; CALLS-NEXT:    [[TMP9:%.*]] = icmp sle i32 0, [[TMP8]]
-; CALLS-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; CALLS-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; CALLS-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP11]], -1
-; CALLS-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; CALLS-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP12]]
-; CALLS-NEXT:    [[TMP16:%.*]] = icmp sle i32 0, [[TMP15]]
+; CALLS-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; CALLS-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; CALLS-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; CALLS-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; CALLS-NEXT:    [[TMP9:%.*]] = icmp ule i32 -2147483648, [[TMP6]]
+; CALLS-NEXT:    [[TMP16:%.*]] = icmp ule i32 -2147483648, [[TMP5]]
 ; CALLS-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; CALLS-NEXT:    [[TMP18:%.*]] = icmp ne i32 [[TMP1]], 0
 ; CALLS-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP2]], i32 0
@@ -1949,20 +1853,12 @@ define zeroext i1 @ICmpSLTAllOnes(i32 %x) nounwind uwtable readnone sanitize_mem
 ; CHECK-SAME: i32 [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = lshr i32 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP4]], -1
-; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[X]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP3]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp slt i32 -1, [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = lshr i32 [[TMP9]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = xor i32 [[TMP1]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], -1
-; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[X]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or i32 [[TMP13]], [[TMP11]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt i32 -1, [[TMP14]]
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[X]], -2147483648
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ult i32 2147483647, [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ult i32 2147483647, [[TMP4]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP8]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp slt i32 -1, [[X]]
 ; CHECK-NEXT:    store i1 [[TMP16]], ptr @__msan_retval_tls, align 8
@@ -1973,20 +1869,12 @@ define zeroext i1 @ICmpSLTAllOnes(i32 %x) nounwind uwtable readnone sanitize_mem
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; ORIGIN-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; ORIGIN-NEXT:    call void @llvm.donothing()
-; ORIGIN-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; ORIGIN-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; ORIGIN-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP5]], -1
-; ORIGIN-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; ORIGIN-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP4]]
-; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp slt i32 -1, [[TMP8]]
-; ORIGIN-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; ORIGIN-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; ORIGIN-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP11]], -1
-; ORIGIN-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; ORIGIN-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP12]]
-; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp slt i32 -1, [[TMP15]]
+; ORIGIN-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; ORIGIN-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; ORIGIN-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; ORIGIN-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp ult i32 2147483647, [[TMP6]]
+; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp ult i32 2147483647, [[TMP5]]
 ; ORIGIN-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; ORIGIN-NEXT:    [[TMP18:%.*]] = icmp ne i32 [[TMP1]], 0
 ; ORIGIN-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP2]], i32 0
@@ -2000,20 +1888,12 @@ define zeroext i1 @ICmpSLTAllOnes(i32 %x) nounwind uwtable readnone sanitize_mem
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; CALLS-NEXT:    call void @llvm.donothing()
-; CALLS-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; CALLS-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; CALLS-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP5]], -1
-; CALLS-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; CALLS-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP4]]
-; CALLS-NEXT:    [[TMP9:%.*]] = icmp slt i32 -1, [[TMP8]]
-; CALLS-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; CALLS-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; CALLS-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP11]], -1
-; CALLS-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; CALLS-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP12]]
-; CALLS-NEXT:    [[TMP16:%.*]] = icmp slt i32 -1, [[TMP15]]
+; CALLS-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; CALLS-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; CALLS-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; CALLS-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; CALLS-NEXT:    [[TMP9:%.*]] = icmp ult i32 2147483647, [[TMP6]]
+; CALLS-NEXT:    [[TMP16:%.*]] = icmp ult i32 2147483647, [[TMP5]]
 ; CALLS-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; CALLS-NEXT:    [[TMP18:%.*]] = icmp ne i32 [[TMP1]], 0
 ; CALLS-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP2]], i32 0
@@ -2032,20 +1912,12 @@ define zeroext i1 @ICmpSGEAllOnes(i32 %x) nounwind uwtable readnone sanitize_mem
 ; CHECK-SAME: i32 [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = lshr i32 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP4]], -1
-; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[X]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP3]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp sge i32 -1, [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = lshr i32 [[TMP9]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = xor i32 [[TMP1]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], -1
-; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[X]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or i32 [[TMP13]], [[TMP11]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp sge i32 -1, [[TMP14]]
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[X]], -2147483648
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp uge i32 2147483647, [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp uge i32 2147483647, [[TMP4]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP8]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp sge i32 -1, [[X]]
 ; CHECK-NEXT:    store i1 [[TMP16]], ptr @__msan_retval_tls, align 8
@@ -2056,20 +1928,12 @@ define zeroext i1 @ICmpSGEAllOnes(i32 %x) nounwind uwtable readnone sanitize_mem
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; ORIGIN-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; ORIGIN-NEXT:    call void @llvm.donothing()
-; ORIGIN-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; ORIGIN-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; ORIGIN-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP5]], -1
-; ORIGIN-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; ORIGIN-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP4]]
-; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp sge i32 -1, [[TMP8]]
-; ORIGIN-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; ORIGIN-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; ORIGIN-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP11]], -1
-; ORIGIN-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; ORIGIN-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP12]]
-; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp sge i32 -1, [[TMP15]]
+; ORIGIN-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; ORIGIN-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; ORIGIN-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; ORIGIN-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp uge i32 2147483647, [[TMP6]]
+; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp uge i32 2147483647, [[TMP5]]
 ; ORIGIN-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; ORIGIN-NEXT:    [[TMP18:%.*]] = icmp ne i32 [[TMP1]], 0
 ; ORIGIN-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP2]], i32 0
@@ -2083,20 +1947,12 @@ define zeroext i1 @ICmpSGEAllOnes(i32 %x) nounwind uwtable readnone sanitize_mem
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; CALLS-NEXT:    call void @llvm.donothing()
-; CALLS-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; CALLS-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; CALLS-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP5]], -1
-; CALLS-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; CALLS-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP4]]
-; CALLS-NEXT:    [[TMP9:%.*]] = icmp sge i32 -1, [[TMP8]]
-; CALLS-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; CALLS-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; CALLS-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP11]], -1
-; CALLS-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; CALLS-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP12]]
-; CALLS-NEXT:    [[TMP16:%.*]] = icmp sge i32 -1, [[TMP15]]
+; CALLS-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; CALLS-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; CALLS-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; CALLS-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; CALLS-NEXT:    [[TMP9:%.*]] = icmp uge i32 2147483647, [[TMP6]]
+; CALLS-NEXT:    [[TMP16:%.*]] = icmp uge i32 2147483647, [[TMP5]]
 ; CALLS-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; CALLS-NEXT:    [[TMP18:%.*]] = icmp ne i32 [[TMP1]], 0
 ; CALLS-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP2]], i32 0
@@ -2115,20 +1971,12 @@ define zeroext i1 @ICmpSGTAllOnes(i32 %x) nounwind uwtable readnone sanitize_mem
 ; CHECK-SAME: i32 [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = lshr i32 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], -1
-; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[X]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp sgt i32 [[TMP7]], -1
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = lshr i32 [[TMP9]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = xor i32 [[TMP1]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP11]], -1
-; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[X]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or i32 [[TMP13]], [[TMP10]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt i32 [[TMP14]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[X]], -2147483648
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i32 [[TMP4]], 2147483647
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ugt i32 [[TMP5]], 2147483647
 ; CHECK-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP8]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp sgt i32 [[X]], -1
 ; CHECK-NEXT:    store i1 [[TMP16]], ptr @__msan_retval_tls, align 8
@@ -2139,20 +1987,12 @@ define zeroext i1 @ICmpSGTAllOnes(i32 %x) nounwind uwtable readnone sanitize_mem
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; ORIGIN-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; ORIGIN-NEXT:    call void @llvm.donothing()
-; ORIGIN-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; ORIGIN-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; ORIGIN-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP4]], -1
-; ORIGIN-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; ORIGIN-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP5]]
-; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], -1
-; ORIGIN-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; ORIGIN-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; ORIGIN-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP12]], -1
-; ORIGIN-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; ORIGIN-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP11]]
-; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP15]], -1
+; ORIGIN-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; ORIGIN-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; ORIGIN-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; ORIGIN-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp ugt i32 [[TMP5]], 2147483647
+; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp ugt i32 [[TMP6]], 2147483647
 ; ORIGIN-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; ORIGIN-NEXT:    [[TMP18:%.*]] = icmp sgt i32 [[X]], -1
 ; ORIGIN-NEXT:    store i1 [[TMP17]], ptr @__msan_retval_tls, align 8
@@ -2164,20 +2004,12 @@ define zeroext i1 @ICmpSGTAllOnes(i32 %x) nounwind uwtable readnone sanitize_mem
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; CALLS-NEXT:    call void @llvm.donothing()
-; CALLS-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; CALLS-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; CALLS-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP4]], -1
-; CALLS-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; CALLS-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP5]]
-; CALLS-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], -1
-; CALLS-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; CALLS-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; CALLS-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP12]], -1
-; CALLS-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; CALLS-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP11]]
-; CALLS-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP15]], -1
+; CALLS-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; CALLS-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; CALLS-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; CALLS-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; CALLS-NEXT:    [[TMP9:%.*]] = icmp ugt i32 [[TMP5]], 2147483647
+; CALLS-NEXT:    [[TMP16:%.*]] = icmp ugt i32 [[TMP6]], 2147483647
 ; CALLS-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; CALLS-NEXT:    [[TMP18:%.*]] = icmp sgt i32 [[X]], -1
 ; CALLS-NEXT:    store i1 [[TMP17]], ptr @__msan_retval_tls, align 8
@@ -2194,20 +2026,12 @@ define zeroext i1 @ICmpSLEAllOnes(i32 %x) nounwind uwtable readnone sanitize_mem
 ; CHECK-SAME: i32 [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = lshr i32 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], -1
-; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[X]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp sle i32 [[TMP7]], -1
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = lshr i32 [[TMP9]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = xor i32 [[TMP1]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP11]], -1
-; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[X]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or i32 [[TMP13]], [[TMP10]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp sle i32 [[TMP14]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[X]], -2147483648
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ule i32 [[TMP4]], 2147483647
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ule i32 [[TMP5]], 2147483647
 ; CHECK-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP8]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp sle i32 [[X]], -1
 ; CHECK-NEXT:    store i1 [[TMP16]], ptr @__msan_retval_tls, align 8
@@ -2218,20 +2042,12 @@ define zeroext i1 @ICmpSLEAllOnes(i32 %x) nounwind uwtable readnone sanitize_mem
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; ORIGIN-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; ORIGIN-NEXT:    call void @llvm.donothing()
-; ORIGIN-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; ORIGIN-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; ORIGIN-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP4]], -1
-; ORIGIN-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; ORIGIN-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP5]]
-; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp sle i32 [[TMP8]], -1
-; ORIGIN-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; ORIGIN-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; ORIGIN-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; ORIGIN-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP12]], -1
-; ORIGIN-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; ORIGIN-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP11]]
-; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp sle i32 [[TMP15]], -1
+; ORIGIN-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; ORIGIN-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; ORIGIN-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; ORIGIN-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp ule i32 [[TMP5]], 2147483647
+; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp ule i32 [[TMP6]], 2147483647
 ; ORIGIN-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; ORIGIN-NEXT:    [[TMP18:%.*]] = icmp sle i32 [[X]], -1
 ; ORIGIN-NEXT:    store i1 [[TMP17]], ptr @__msan_retval_tls, align 8
@@ -2243,20 +2059,12 @@ define zeroext i1 @ICmpSLEAllOnes(i32 %x) nounwind uwtable readnone sanitize_mem
 ; CALLS-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; CALLS-NEXT:    call void @llvm.donothing()
-; CALLS-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 1
-; CALLS-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP1]], [[TMP4]]
-; CALLS-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP4]], -1
-; CALLS-NEXT:    [[TMP7:%.*]] = and i32 [[X]], [[TMP6]]
-; CALLS-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP5]]
-; CALLS-NEXT:    [[TMP9:%.*]] = icmp sle i32 [[TMP8]], -1
-; CALLS-NEXT:    [[TMP10:%.*]] = shl i32 [[TMP1]], 1
-; CALLS-NEXT:    [[TMP11:%.*]] = lshr i32 [[TMP10]], 1
-; CALLS-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP1]], [[TMP11]]
-; CALLS-NEXT:    [[TMP13:%.*]] = xor i32 [[TMP12]], -1
-; CALLS-NEXT:    [[TMP14:%.*]] = and i32 [[X]], [[TMP13]]
-; CALLS-NEXT:    [[TMP15:%.*]] = or i32 [[TMP14]], [[TMP11]]
-; CALLS-NEXT:    [[TMP16:%.*]] = icmp sle i32 [[TMP15]], -1
+; CALLS-NEXT:    [[TMP3:%.*]] = xor i32 [[X]], -2147483648
+; CALLS-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP1]], -1
+; CALLS-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], [[TMP4]]
+; CALLS-NEXT:    [[TMP6:%.*]] = or i32 [[TMP3]], [[TMP1]]
+; CALLS-NEXT:    [[TMP9:%.*]] = icmp ule i32 [[TMP5]], 2147483647
+; CALLS-NEXT:    [[TMP16:%.*]] = icmp ule i32 [[TMP6]], 2147483647
 ; CALLS-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP9]], [[TMP16]]
 ; CALLS-NEXT:    [[TMP18:%.*]] = icmp sle i32 [[X]], -1
 ; CALLS-NEXT:    store i1 [[TMP17]], ptr @__msan_retval_tls, align 8
@@ -2278,20 +2086,12 @@ define <2 x i1> @ICmpSLT_vector_Zero(<2 x ptr> %x) nounwind uwtable readnone san
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint <2 x ptr> [[X]] to <2 x i64>
-; CHECK-NEXT:    [[TMP3:%.*]] = shl <2 x i64> [[TMP1]], <i64 1, i64 1>
-; CHECK-NEXT:    [[TMP4:%.*]] = lshr <2 x i64> [[TMP3]], <i64 1, i64 1>
-; CHECK-NEXT:    [[TMP5:%.*]] = xor <2 x i64> [[TMP1]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = xor <2 x i64> [[TMP4]], <i64 -1, i64 -1>
-; CHECK-NEXT:    [[TMP7:%.*]] = and <2 x i64> [[TMP2]], [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = or <2 x i64> [[TMP7]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp slt <2 x i64> [[TMP8]], zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = shl <2 x i64> [[TMP1]], <i64 1, i64 1>
-; CHECK-NEXT:    [[TMP11:%.*]] = lshr <2 x i64> [[TMP10]], <i64 1, i64 1>
-; CHECK-NEXT:    [[TMP12:%.*]] = xor <2 x i64> [[TMP1]], [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = xor <2 x i64> [[TMP12]], <i64 -1, i64 -1>
-; CHECK-NEXT:    [[TMP14:%.*]] = and <2 x i64> [[TMP2]], [[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = or <2 x i64> [[TMP14]], [[TMP11]]
-; CHECK-NEXT:    [[TMP16:%.*]] = icmp slt <2 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <2 x i64> [[TMP2]], <i64 -9223372036854775808, i64 -9223372036854775808>
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i64> [[TMP1]], <i64 -1, i64 -1>
+; CHECK-NEXT:    [[TMP5:%.*]] = and <2 x i64> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <2 x i64> [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ult <2 x i64> [[TMP5]], <i64 -9223372036854775808, i64 -9223372036854775808>
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ult <2 x i64> [[TMP6]], <i64 -9223372036854775808, i64 -9223372036854775808>
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor <2 x i1> [[TMP9]], [[TMP16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp slt <2 x ptr> [[X]], zeroinitializer
 ; CHECK-NEXT:    store <2 x i1> [[TMP17]], ptr @__msan_retval_tls, align 8
@@ -2303,20 +2103,12 @@ define <2 x i1> @ICmpSLT_vector_Zero(<2 x ptr> %x) nounwind uwtable readnone san
 ; ORIGIN-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; ORIGIN-NEXT:    call void @llvm.donothing()
 ; ORIGIN-NEXT:    [[TMP3:%.*]] = ptrtoint <2 x ptr> [[X]] to <2 x i64>
-; ORIGIN-NEXT:    [[TMP4:%.*]] = shl <2 x i64> [[TMP1]], <i64 1, i64 1>
-; ORIGIN-NEXT:    [[TMP5:%.*]] = lshr <2 x i64> [[TMP4]], <i64 1, i64 1>
-; ORIGIN-NEXT:    [[TMP6:%.*]] = xor <2 x i64> [[TMP1]], [[TMP5]]
-; ORIGIN-NEXT:    [[TMP7:%.*]] = xor <2 x i64> [[TMP5]], <i64 -1, i64 -1>
-; ORIGIN-NEXT:    [[TMP8:%.*]] = and <2 x i64> [[TMP3]], [[TMP7]]
-; ORIGIN-NEXT:    [[TMP9:%.*]] = or <2 x i64> [[TMP8]], [[TMP6]]
-; ORIGIN-NEXT:    [[TMP10:%.*]] = icmp slt <2 x i64> [[TMP9]], zeroinitializer
-; ORIGIN-NEXT:    [[TMP11:%.*]] = shl <2 x i64> [[TMP1]], <i64 1, i64 1>
-; ORIGIN-NEXT:    [[TMP12:%.*]] = lshr <2 x i64> [[TMP11]], <i64 1, i64 1>
-; ORIGIN-NEXT:    [[TMP13:%.*]] = xor <2 x i64> [[TMP1]], [[TMP12]]
-; ORIGIN-NEXT:    [[TMP14:%.*]] = xor <2 x i64> [[TMP13]], <i64 -1, i64 -1>
-; ORIGIN-NEXT:    [[TMP15:%.*]] = and <2 x i64> [[TMP3]], [[TMP14]]
-; ORIGIN-NEXT:    [[TMP16:%.*]] = or <2 x i64> [[TMP15]], [[TMP12]]
-; ORIGIN-NEXT:    [[TMP17:%.*]] = icmp slt <2 x i64> [[TMP16]], zeroinitializer
+; ORIGIN-NEXT:    [[TMP4:%.*]] = xor <2 x i64> [[TMP3]], <i64 -9223372036854775808, i64 -9223372036854775808>
+; ORIGIN-NEXT:    [[TMP5:%.*]] = xor <2 x i64> [[TMP1]], <i64 -1, i64 -1>
+; ORIGIN-NEXT:    [[TMP6:%.*]] = and <2 x i64> [[TMP4]], [[TMP5]]
+; ORIGIN-NEXT:    [[TMP7:%.*]] = or <2 x i64> [[TMP4]], [[TMP1]]
+; ORIGIN-NEXT:    [[TMP10:%.*]] = icmp ult <2 x i64> [[TMP6]], <i64 -9223372036854775808, i64 -9223372036854775808>
+; ORIGIN-NEXT:    [[TMP17:%.*]] = icmp ult <2 x i64> [[TMP7]], <i64 -9223372036854775808, i64 -9223372036854775808>
 ; ORIGIN-NEXT:    [[TMP18:%.*]] = xor <2 x i1> [[TMP10]], [[TMP17]]
 ; ORIGIN-NEXT:    [[TMP19:%.*]] = icmp slt <2 x ptr> [[X]], zeroinitializer
 ; ORIGIN-NEXT:    store <2 x i1> [[TMP18]], ptr @__msan_retval_tls, align 8
@@ -2329,20 +2121,12 @@ define <2 x i1> @ICmpSLT_vector_Zero(<2 x ptr> %x) nounwind uwtable readnone san
 ; CALLS-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; CALLS-NEXT:    call void @llvm.donothing()
 ; CALLS-NEXT:    [[TMP3:%.*]] = ptrtoint <2 x ptr> [[X]] to <2 x i64>
-; CALLS-NEXT:    [[TMP4:%.*]] = shl <2 x i64> [[TMP1]], <i64 1, i64 1>
-; CALLS-NEXT:    [[TMP5:%.*]] = lshr <2 x i64> [[TMP4]], <i64 1, i64 1>
-; CALLS-NEXT:    [[TMP6:%.*]] = xor <2 x i64> [[TMP1]], [[TMP5]]
-; CALLS-NEXT:    [[TMP7:%.*]] = xor <2 x i64> [[TMP5]], <i64 -1, i64 -1>
-; CALLS-NEXT:    [[TMP8:%.*]] = and <2 x i64> [[TMP3]], [[TMP7]]
-; CALLS-NEXT:    [[TMP9:%.*]] = or <2 x i64> [[TMP8]], [[TMP6]]
-; CALLS-NEXT:    [[TMP10:%.*]] = icmp slt <2 x i64> [[TMP9]], zeroinitializer
-; CALLS-NEXT:    [[TMP11:%.*]] = shl <2 x i64> [[TMP1]], <i64 1, i64 1>
-; CALLS-NEXT:    [[TMP12:%.*]] = lshr <2 x i64> [[TMP11]], <i64 1, i64 1>
-; CALLS-NEXT:    [[TMP13:%.*]] = xor <2 x i64> [[TMP1]], [[TMP12]]
-; CALLS-NEXT:    [[TMP14:%.*]] = xor <2 x i64> [[TMP13]], <i64 -1, i64 -1>
-; CALLS-NEXT:    [[TMP15:%.*]] = and <2 x i64> [[TMP3]], [[TMP14]]
-; CALLS-NEXT:    [[TMP16:%.*]] = or <2 x i64> [[TMP15]], [[TMP12]]
-; CALLS-NEXT:    [[TMP17:%.*]] = icmp slt <2 x i64> [[TMP16]], zeroinitializer
+; CALLS-NEXT:    [[TMP4:%.*]] = xor <2 x i64> [[TMP3]], <i64 -9223372036854775808, i64 -9223372036854775808>
+; CALLS-NEXT:    [[TMP5:%.*]] = xor <2 x i64> [[TMP1]], <i64 -1, i64 -1>
+; CALLS-NEXT:    [[TMP6:%.*]] = and <2 x i64> [[TMP4]], [[TMP5]]
+; CALLS-NEXT:    [[TMP7:%.*]] = or <2 x i64> [[TMP4]], [[TMP1]]
+; CALLS-NEXT:    [[TMP10:%.*]] = icmp ult <2 x i64> [[TMP6]], <i64 -9223372036854775808, i64 -9223372036854775808>
+; CALLS-NEXT:    [[TMP17:%.*]] = icmp ult <2 x i64> [[TMP7]], <i64 -9223372036854775808, i64 -9223372036854775808>
 ; CALLS-NEXT:    [[TMP18:%.*]] = xor <2 x i1> [[TMP10]], [[TMP17]]
 ; CALLS-NEXT:    [[TMP19:%.*]] = icmp slt <2 x ptr> [[X]], zeroinitializer
 ; CALLS-NEXT:    store <2 x i1> [[TMP18]], ptr @__msan_retval_tls, align 8
@@ -2362,20 +2146,12 @@ define <2 x i1> @ICmpSLT_vector_AllOnes(<2 x i32> %x) nounwind uwtable readnone
 ; CHECK-SAME: <2 x i32> [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl <2 x i32> [[TMP1]], <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = lshr <2 x i32> [[TMP2]], <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = xor <2 x i32> [[TMP4]], <i32 -1, i32 -1>
-; CHECK-NEXT:    [[TMP6:%.*]] = and <2 x i32> [[X]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = or <2 x i32> [[TMP6]], [[TMP3]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp slt <2 x i32> <i32 -1, i32 -1>, [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = shl <2 x i32> [[TMP1]], <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP10:%.*]] = lshr <2 x i32> [[TMP9]], <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP11:%.*]] = xor <2 x i32> [[TMP1]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = xor <2 x i32> [[TMP10]], <i32 -1, i32 -1>
-; CHECK-NEXT:    [[TMP13:%.*]] = and <2 x i32> [[X]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <2 x i32> [[TMP13]], [[TMP11]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt <2 x i32> <i32 -1, i32 -1>, [[TMP14]]
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <2 x i32> [[X]], <i32 -2147483648, i32 -2147483648>
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <2 x i32> [[TMP1]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[TMP4:%.*]] = and <2 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <2 x i32> [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ult <2 x i32> <i32 2147483647, i32 2147483647>, [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ult <2 x i32> <i32 2147483647, i32 2147483647>, [[TMP4]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = xor <2 x i1> [[TMP8]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp slt <2 x i32> <i32 -1, i32 -1>, [[X]]
 ; CHECK-NEXT:    store <2 x i1> [[TMP16]], ptr @__msan_retval_tls, align 8
@@ -2386,20 +2162,12 @@ define <2 x i1> @ICmpSLT_vector_AllOnes(<2 x i32> %x) nounwind uwtable readnone
 ; ORIGIN-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
 ; ORIGIN-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; ORIGIN-NEXT:    call void @llvm.donothing()
-; ORIGIN-NEXT:    [[TMP3:%.*]] = shl <2 x i32> [[TMP1]], <i32 1, i32 1>
-; ORIGIN-NEXT:    [[TMP4:%.*]] = lshr <2 x i32> [[TMP3]], <i32 1, i32 1>
-; ORIGIN-NEXT:    [[TMP5:%.*]] = xor <2 x i32> [[TMP1]], [[TMP4]]
-; ORIGIN-NEXT:    [[TMP6:%.*]] = xor <2 x i32> [[TMP5]], <i32 -1, i32 -1>
-; ORIGIN-NEXT:    [[TMP7:%.*]] = and <2 x i32> [[X]], [[TMP6]]
-; ORIGIN-NEXT:    [[TMP8:%.*]] = or <2 x i32> [[TMP7]], [[TMP4]]
-; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp slt <2 x i32> <i32 -1, i32 -1>, [[TMP8]]
-; ORIGIN-NEXT:    [[TMP10:%.*]] = shl <2 x i32> [[TMP1]], <i32 1, i32 1>
-; ORIGIN-NEXT:    [[TMP11:%.*]] = lshr <2 x i32> [[TMP10]], <i32 1, i32 1>
-; ORIGIN-NEXT:    [[TMP12:%.*]] = xor <2 x i32> [[TMP1]], [[TMP11]]
-; ORIGIN-NEXT:    [[TMP13:%.*]] = xor <2 x i32> [[TMP11]], <i32 -1, i32 -1>
-; ORIGIN-NEXT:    [[TMP14:%.*]] = and <2 x i32> [[X]], [[TMP13]]
-; ORIGIN-NEXT:    [[TMP15:%.*]] = or <2 x i32> [[TMP14]], [[TMP12]]
-; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp slt <2 x i32> <i32 -1, i32 -1>, [[TMP15]]
+; ORIGIN-NEXT:    [[TMP3:%.*]] = xor <2 x i32> [[X]], <i32 -2147483648, i32 -2147483648>
+; ORIGIN-NEXT:    [[TMP4:%.*]] = xor <2 x i32> [[TMP1]], <i32 -1, i32 -1>
+; ORIGIN-NEXT:    [[TMP5:%.*]] = and <2 x i32> [[TMP3]], [[TMP4]]
+; ORIGIN-NEXT:    [[TMP6:%.*]] = or <2 x i32> [[TMP3]], [[TMP1]]
+; ORIGIN-NEXT:    [[TMP9:%.*]] = icmp ult <2 x i32> <i32 2147483647, i32 2147483647>, [[TMP6]]
+; ORIGIN-NEXT:    [[TMP16:%.*]] = icmp ult <2 x i32> <i32 2147483647, i32 2147483647>, [[TMP5]]
 ; ORIGIN-NEXT:    [[TMP17:%.*]] = xor <2 x i1> [[TMP9]], [[TMP16]]
 ; ORIGIN-NEXT:    [[TMP18:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
 ; ORIGIN-NEXT:    [[TMP19:%.*]] = icmp ne i64 [[TMP18]], 0
@@ -2414,20 +2182,12 @@ define <2 x i1> @ICmpSLT_vector_AllOnes(<2 x i32> %x) nounwind uwtable readnone
 ; CALLS-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
 ; CALLS-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_origin_tls, align 4
 ; CALLS-NEXT:    call void @llvm.donothing()
-; CALLS-NEXT:    [[TMP3:%.*]] = shl <2 x i32> [[TMP1]], <i32 1, i32 1>
-; CALLS-NEXT:    [[TMP4:%.*]] = lshr <2 x i32> [[TMP3]], <i32 1, i32 1>
-; CALLS-NEXT:    [[TMP5:%.*]] = xor <2 x i32> [[TMP1]], [[TMP4]]
-; CALLS-NEXT:    [[TMP6:%.*]] = xor <2 x i32> [[TMP5]], <i32 -1, i32 -1>
-; CALLS-NEXT:    [[TMP7:%.*]] = and <2 x i32> [[X]], [[TMP6]]
-; CALLS-NEXT:    [[TMP8:%.*]] = or <2 x i32> [[TMP7]], [[TMP4]]
-; CALLS-NEXT:    [[TMP9:%.*]] = icmp slt <2 x i32> <i32 -1, i32 -1>, [[TMP8]]
-; CALLS-NEXT:    [[TMP10:%.*]] = shl <2 x i32> [[TMP1]], <i32 1, i32 1>
-; CALLS-NEXT:    [[TMP11:%.*]] = lshr <2 x i32> [[TMP10]], <i32 1, i32 1>
-; CALLS-NEXT:    [[TMP12:%.*]] = xor <2 x i32> [[TMP1]], [[TMP11]]
-; CALLS-NEXT:    [[TMP13:%.*]] = xor <2 x i32> [[TMP11]], <i32 -1, i32 -1>
-; CALLS-NEXT:    [[TMP14:%.*]] = and <2 x i32> [[X]], [[TMP13]]
-; CALLS-NEXT:    [[TMP15:%.*]] = or <2 x i32> [[TMP14]], [[TMP12]]
-; CALLS-NEXT:    [[TMP16:%.*]] = icmp slt <2 x i32> <i32 -1, i32 -1>, [[TMP15]]
+; CALLS-NEXT:    [[TMP3:%.*]] = xor <2 x i32> [[X]], <i32 -2147483648, i32 -2147483648>
+; CALLS-NEXT:    [[TMP4:%.*]] = xor <2 x i32> [[TMP1]], <i32 -1, i32 -1>
+; CALLS-NEXT:    [[TMP5:%.*]] = and <2 x i32> [[TMP3]], [[TMP4]]
+; CALLS-NEXT:    [[TMP6:%.*]] = or <2 x i32> [[TMP3]], [[TMP1]]
+; CALLS-NEXT:    [[TMP9:%.*]] = icmp ult <2 x i32> <i32 2147483647, i32 2147483647>, [[TMP6]]
+; CALLS-NEXT:    [[TMP16:%.*]] = icmp ult <2 x i32> <i32 2147483647, i32 2147483647>, [[TMP5]]
 ; CALLS-NEXT:    [[TMP17:%.*]] = xor <2 x i1> [[TMP9]], [[TMP16]]
 ; CALLS-NEXT:    [[TMP18:%.*]] = bitcast <2 x i32> [[TMP1]] to i64
 ; CALLS-NEXT:    [[TMP19:%.*]] = icmp ne i64 [[TMP18]], 0
@@ -2454,8 +2214,8 @@ define zeroext i1 @ICmpUGTConst(i32 %x) nounwind uwtable readnone sanitize_memor
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[TMP0]], -1
 ; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[X]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[TMP2]], 7
 ; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[X]], [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[TMP2]], 7
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp ugt i32 [[TMP4]], 7
 ; CHECK-NEXT:    [[TMP6:%.*]] = xor i1 [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X]], 7
@@ -2470,8 +2230,8 @@ define zeroext i1 @ICmpUGTConst(i32 %x) nounwind uwtable readnone sanitize_memor
 ; ORIGIN-NEXT:    call void @llvm.donothing()
 ; ORIGIN-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP0]], -1
 ; ORIGIN-NEXT:    [[TMP3:%.*]] = and i32 [[X]], [[TMP2]]
-; ORIGIN-NEXT:    [[TMP4:%.*]] = icmp ugt i32 [[TMP3]], 7
 ; ORIGIN-NEXT:    [[TMP5:%.*]] = or i32 [[X]], [[TMP0]]
+; ORIGIN-NEXT:    [[TMP4:%.*]] = icmp ugt i32 [[TMP3]], 7
 ; ORIGIN-NEXT:    [[TMP6:%.*]] = icmp ugt i32 [[TMP5]], 7
 ; ORIGIN-NEXT:    [[TMP7:%.*]] = xor i1 [[TMP4]], [[TMP6]]
 ; ORIGIN-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X]], 7
@@ -2487,8 +2247,8 @@ define zeroext i1 @ICmpUGTConst(i32 %x) nounwind uwtable readnone sanitize_memor
 ; CALLS-NEXT:    call void @llvm.donothing()
 ; CALLS-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP0]], -1
 ; CALLS-NEXT:    [[TMP3:%.*]] = and i32 [[X]], [[TMP2]]
-; CALLS-NEXT:    [[TMP4:%.*]] = icmp ugt i32 [[TMP3]], 7
 ; CALLS-NEXT:    [[TMP5:%.*]] = or i32 [[X]], [[TMP0]]
+; CALLS-NEXT:    [[TMP4:%.*]] = icmp ugt i32 [[TMP3]], 7
 ; CALLS-NEXT:    [[TMP6:%.*]] = icmp ugt i32 [[TMP5]], 7
 ; CALLS-NEXT:    [[TMP7:%.*]] = xor i1 [[TMP4]], [[TMP6]]
 ; CALLS-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X]], 7
diff --git a/llvm/test/Instrumentation/MemorySanitizer/pr32842.ll b/llvm/test/Instrumentation/MemorySanitizer/pr32842.ll
index 17fe78d292bd..6d275b3e2d38 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/pr32842.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/pr32842.ll
@@ -15,32 +15,16 @@ define zeroext i1 @_Z1fii(i32 %x, i32 %y) sanitize_memory {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP28:%.*]] = shl i32 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = lshr i32 [[TMP28]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP0]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], -1
-; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[X]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP9:%.*]] = lshr i32 [[TMP8]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = xor i32 [[TMP1]], [[TMP9]]
-; CHECK-NEXT:    [[TMP11:%.*]] = xor i32 [[TMP10]], -1
-; CHECK-NEXT:    [[TMP12:%.*]] = and i32 [[Y]], [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = or i32 [[TMP12]], [[TMP9]]
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp slt i32 [[TMP7]], [[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = shl i32 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP16:%.*]] = lshr i32 [[TMP15]], 1
-; CHECK-NEXT:    [[TMP17:%.*]] = xor i32 [[TMP0]], [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = xor i32 [[TMP17]], -1
-; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[X]], [[TMP18]]
-; CHECK-NEXT:    [[TMP20:%.*]] = or i32 [[TMP19]], [[TMP16]]
-; CHECK-NEXT:    [[TMP21:%.*]] = shl i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP22:%.*]] = lshr i32 [[TMP21]], 1
-; CHECK-NEXT:    [[TMP23:%.*]] = xor i32 [[TMP1]], [[TMP22]]
-; CHECK-NEXT:    [[TMP24:%.*]] = xor i32 [[TMP22]], -1
-; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[Y]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = or i32 [[TMP25]], [[TMP23]]
-; CHECK-NEXT:    [[TMP27:%.*]] = icmp slt i32 [[TMP20]], [[TMP26]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i32 [[X]], -2147483648
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP0]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP10]], [[TMP0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i32 [[Y]], -2147483648
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP8:%.*]] = and i32 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or i32 [[TMP6]], [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ult i32 [[TMP4]], [[TMP9]]
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ult i32 [[TMP5]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = xor i1 [[TMP14]], [[TMP27]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X]], [[Y]]
 ; CHECK-NEXT:    store i1 [[TMP2]], ptr @__msan_retval_tls, align 8
-- 
GitLab


From ce858e2127beb7e8f5780996d5a87994fbad39d4 Mon Sep 17 00:00:00 2001
From: tangaac <tangyan01@loongson.cn>
Date: Fri, 25 Oct 2024 11:59:27 +0800
Subject: [PATCH 119/693] [LoongArch] fix description of clang option
 -m[no-]lam-bh (#113632)

Fix the description of option `-mlam-bh` and `-mno-lam-bh`
Previous decription causes a `build docs-clang-html` error in #112727
---
 clang/include/clang/Driver/Options.td | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 62603969eaf8..5757c385d224 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5400,9 +5400,9 @@ def mfrecipe : Flag<["-"], "mfrecipe">, Group<m_loongarch_Features_Group>,
 def mno_frecipe : Flag<["-"], "mno-frecipe">, Group<m_loongarch_Features_Group>,
   HelpText<"Disable frecipe.{s/d} and frsqrte.{s/d}">;
 def mlam_bh : Flag<["-"], "mlam-bh">, Group<m_loongarch_Features_Group>,
-  HelpText<"Enable amswap_[db].{b/h} and amadd_[db].{b/h}">;
+  HelpText<"Enable amswap[_db].{b/h} and amadd[_db].{b/h}">;
 def mno_lam_bh : Flag<["-"], "mno-lam-bh">, Group<m_loongarch_Features_Group>,
-  HelpText<"Disable amswap_[db].{b/h} and amadd_[db].{b/h}">;
+  HelpText<"Disable amswap[_db].{b/h} and amadd[_db].{b/h}">;
 def mannotate_tablejump : Flag<["-"], "mannotate-tablejump">, Group<m_loongarch_Features_Group>,
   HelpText<"Enable annotate table jump instruction to correlate it with the jump table.">;
 def mno_annotate_tablejump : Flag<["-"], "mno-annotate-tablejump">, Group<m_loongarch_Features_Group>,
-- 
GitLab


From a9d51f0e18ec840bdb361daca48b4ef00e2ba8ac Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 24 Oct 2024 22:13:55 -0700
Subject: [PATCH 120/693] [MCParser] De-capitalize ParseDirectiveCGProfile. NFC

---
 llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h | 2 +-
 llvm/lib/MC/MCParser/COFFAsmParser.cpp               | 2 +-
 llvm/lib/MC/MCParser/DarwinAsmParser.cpp             | 2 +-
 llvm/lib/MC/MCParser/ELFAsmParser.cpp                | 2 +-
 llvm/lib/MC/MCParser/MCAsmParserExtension.cpp        | 4 ++--
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h b/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h
index e596a7195447..f1c1b2641ab1 100644
--- a/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h
+++ b/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h
@@ -98,7 +98,7 @@ public:
     return getParser().parseOptionalToken(T);
   }
 
-  bool ParseDirectiveCGProfile(StringRef, SMLoc);
+  bool parseDirectiveCGProfile(StringRef, SMLoc);
 
   bool check(bool P, const Twine &Msg) {
     return getParser().check(P, Msg);
diff --git a/llvm/lib/MC/MCParser/COFFAsmParser.cpp b/llvm/lib/MC/MCParser/COFFAsmParser.cpp
index 53c0270255e8..f97da2d479b4 100644
--- a/llvm/lib/MC/MCParser/COFFAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/COFFAsmParser.cpp
@@ -301,7 +301,7 @@ bool COFFAsmParser::parseDirectiveSymbolAttribute(StringRef Directive, SMLoc) {
 }
 
 bool COFFAsmParser::parseDirectiveCGProfile(StringRef S, SMLoc Loc) {
-  return MCAsmParserExtension::ParseDirectiveCGProfile(S, Loc);
+  return MCAsmParserExtension::parseDirectiveCGProfile(S, Loc);
 }
 
 bool COFFAsmParser::parseSectionSwitch(StringRef Section,
diff --git a/llvm/lib/MC/MCParser/DarwinAsmParser.cpp b/llvm/lib/MC/MCParser/DarwinAsmParser.cpp
index a97b72997ae3..5dc9cb64c46a 100644
--- a/llvm/lib/MC/MCParser/DarwinAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/DarwinAsmParser.cpp
@@ -1205,7 +1205,7 @@ bool DarwinAsmParser::parseBuildVersion(StringRef Directive, SMLoc Loc) {
 /// parseDirectiveCGProfile
 ///   ::= .cg_profile from, to, count
 bool DarwinAsmParser::parseDirectiveCGProfile(StringRef S, SMLoc Loc) {
-  return MCAsmParserExtension::ParseDirectiveCGProfile(S, Loc);
+  return MCAsmParserExtension::parseDirectiveCGProfile(S, Loc);
 }
 
 namespace llvm {
diff --git a/llvm/lib/MC/MCParser/ELFAsmParser.cpp b/llvm/lib/MC/MCParser/ELFAsmParser.cpp
index 28672a19aa70..9cdc2aafe3d2 100644
--- a/llvm/lib/MC/MCParser/ELFAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/ELFAsmParser.cpp
@@ -925,7 +925,7 @@ bool ELFAsmParser::parseDirectiveSubsection(StringRef, SMLoc) {
 }
 
 bool ELFAsmParser::parseDirectiveCGProfile(StringRef S, SMLoc Loc) {
-  return MCAsmParserExtension::ParseDirectiveCGProfile(S, Loc);
+  return MCAsmParserExtension::parseDirectiveCGProfile(S, Loc);
 }
 
 namespace llvm {
diff --git a/llvm/lib/MC/MCParser/MCAsmParserExtension.cpp b/llvm/lib/MC/MCParser/MCAsmParserExtension.cpp
index f5a10ce9805b..444ce395a4de 100644
--- a/llvm/lib/MC/MCParser/MCAsmParserExtension.cpp
+++ b/llvm/lib/MC/MCParser/MCAsmParserExtension.cpp
@@ -22,9 +22,9 @@ void MCAsmParserExtension::Initialize(MCAsmParser &Parser) {
   this->Parser = &Parser;
 }
 
-/// ParseDirectiveCGProfile
+/// parseDirectiveCGProfile
 ///  ::= .cg_profile identifier, identifier, <number>
-bool MCAsmParserExtension::ParseDirectiveCGProfile(StringRef, SMLoc) {
+bool MCAsmParserExtension::parseDirectiveCGProfile(StringRef, SMLoc) {
   StringRef From;
   SMLoc FromLoc = getLexer().getLoc();
   if (getParser().parseIdentifier(From))
-- 
GitLab


From 2e6d451d1565814415e2692ef8e5c3942d4c11a2 Mon Sep 17 00:00:00 2001
From: Roland McGrath <mcgrathr@google.com>
Date: Thu, 24 Oct 2024 22:17:57 -0700
Subject: [PATCH 121/693] [libc] Use idiomatic main() function in
 newhdrgen/yaml_to_classes.py (#113419)

This changes the entry-point Python script of newhdrgen to use
the idiomatic main() function of no arguments that does the whole
job of the script.  This replaces the unusual pattern of having
an idiosyncratic main(...) signature and having the script's
direct code do argument parsing outside the main function.  The
idiomatic pattern makes it possible to usefully wrap the script
in Python and still use its full command-line functionality.
---
 libc/newhdrgen/yaml_to_classes.py | 83 +++++++++++--------------------
 1 file changed, 29 insertions(+), 54 deletions(-)

diff --git a/libc/newhdrgen/yaml_to_classes.py b/libc/newhdrgen/yaml_to_classes.py
index 237dd21aa5df..a295058f7dc8 100644
--- a/libc/newhdrgen/yaml_to_classes.py
+++ b/libc/newhdrgen/yaml_to_classes.py
@@ -216,52 +216,7 @@ def add_function_to_yaml(yaml_file, function_details):
     print(f"Added function {new_function.name} to {yaml_file}")
 
 
-def main(
-    yaml_file,
-    output_dir=None,
-    h_def_file=None,
-    add_function=None,
-    entry_points=None,
-    export_decls=False,
-):
-    """
-    Main function to generate header files from YAML and .h.def templates.
-
-    Args:
-        yaml_file: Path to the YAML file containing header specification.
-        h_def_file: Path to the .h.def template file.
-        output_dir: Directory to output the generated header file.
-        add_function: Details of the function to be added to the YAML file (if any).
-        entry_points: A list of specific function names to include in the header.
-        export_decls: Flag to use GpuHeader for exporting declarations.
-    """
-    if add_function:
-        add_function_to_yaml(yaml_file, add_function)
-
-    header_class = GpuHeader if export_decls else HeaderFile
-    header = load_yaml_file(yaml_file, header_class, entry_points)
-
-    header_str = str(header)
-
-    if output_dir:
-        output_file_path = Path(output_dir)
-        if output_file_path.is_dir():
-            output_file_path /= f"{Path(yaml_file).stem}.h"
-    else:
-        output_file_path = Path(f"{Path(yaml_file).stem}.h")
-
-    if not export_decls and h_def_file:
-        with open(h_def_file, "r") as f:
-            h_def_content = f.read()
-        final_header_content = fill_public_api(header_str, h_def_content)
-        with open(output_file_path, "w") as f:
-            f.write(final_header_content)
-    else:
-        with open(output_file_path, "w") as f:
-            f.write(header_str)
-
-
-if __name__ == "__main__":
+def main():
     parser = argparse.ArgumentParser(description="Generate header files from YAML")
     parser.add_argument(
         "yaml_file", help="Path to the YAML file containing header specification"
@@ -297,11 +252,31 @@ if __name__ == "__main__":
     )
     args = parser.parse_args()
 
-    main(
-        args.yaml_file,
-        args.output_dir,
-        args.h_def_file,
-        args.add_function,
-        args.entry_points,
-        args.export_decls,
-    )
+    if args.add_function:
+        add_function_to_yaml(yaml_file, args.add_function)
+
+    header_class = GpuHeader if args.export_decls else HeaderFile
+    header = load_yaml_file(args.yaml_file, header_class, args.entry_points)
+
+    header_str = str(header)
+
+    if args.output_dir:
+        output_file_path = Path(args.output_dir)
+        if output_file_path.is_dir():
+            output_file_path /= f"{Path(args.yaml_file).stem}.h"
+    else:
+        output_file_path = Path(f"{Path(args.yaml_file).stem}.h")
+
+    if not args.export_decls and args.h_def_file:
+        with open(args.h_def_file, "r") as f:
+            h_def_content = f.read()
+        final_header_content = fill_public_api(header_str, h_def_content)
+        with open(output_file_path, "w") as f:
+            f.write(final_header_content)
+    else:
+        with open(output_file_path, "w") as f:
+            f.write(header_str)
+
+
+if __name__ == "__main__":
+    main()
-- 
GitLab


From 075581f34035c01659cc883d0d69336c279ef0d5 Mon Sep 17 00:00:00 2001
From: Anutosh Bhat <87052487+anutosh491@users.noreply.github.com>
Date: Fri, 25 Oct 2024 11:39:14 +0530
Subject: [PATCH 122/693] [clang-repl] Fix undefined lld::wasm::link symbol
 while building clangInterpreter for wasm (#113446)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

While building llvm (clang, lld) for wasm using emscripten (recipe
hosted on emscripten-forge
https://github.com/emscripten-forge/recipes/tree/main/recipes/recipes_emscripten/llvm)
I ended up with this error
```
 │ │ wasm-ld: error: ../../../../lib/libclangInterpreter.a(Wasm.cpp.o): undefined symbol: lld::wasm::link(llvm::ArrayRef<char const*>, llvm::raw_ostream&, llvm:
 │ │ :raw_ostream&, bool, bool)
 ```
 This is due to the link function here
 https://github.com/llvm/llvm-project/blob/a4819bd46d8baebc3aaa8b38f78065de33593199/clang/lib/Interpreter/Wasm.cpp#L25-L30

 This was added through this PR (https://github.com/llvm/llvm-project/pull/86402) as an attempt to support running clang-repl and executing C++ code interactively inside a Javascript engine using WebAssembly when built with Emscripten.

 The definition for link is present in lldwasm and when building for the emscripten platform we should be linking against it.
---
 clang/lib/Interpreter/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/clang/lib/Interpreter/CMakeLists.txt b/clang/lib/Interpreter/CMakeLists.txt
index 2cc7c59b61d3..d5ffe78251d2 100644
--- a/clang/lib/Interpreter/CMakeLists.txt
+++ b/clang/lib/Interpreter/CMakeLists.txt
@@ -14,6 +14,7 @@ set(LLVM_LINK_COMPONENTS
 
 if (EMSCRIPTEN AND "lld" IN_LIST LLVM_ENABLE_PROJECTS)
   set(WASM_SRC Wasm.cpp)
+  set(WASM_LINK lldWasm)
 endif()
 
 add_clang_library(clangInterpreter
@@ -44,6 +45,7 @@ add_clang_library(clangInterpreter
   clangParse
   clangSema
   clangSerialization
+  ${WASM_LINK}
   )
 
 if ((MINGW OR CYGWIN) AND BUILD_SHARED_LIBS)
-- 
GitLab


From d0c81509ffe7297903371111e7ee1f6f910c3dac Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 24 Oct 2024 23:09:26 -0700
Subject: [PATCH 123/693] [MCParser] De-capitalize COFFMasmParser functions.
 NFC

---
 llvm/lib/MC/MCParser/COFFMasmParser.cpp | 92 ++++++++++++-------------
 1 file changed, 46 insertions(+), 46 deletions(-)

diff --git a/llvm/lib/MC/MCParser/COFFMasmParser.cpp b/llvm/lib/MC/MCParser/COFFMasmParser.cpp
index 912e2b996792..c323e64a40ae 100644
--- a/llvm/lib/MC/MCParser/COFFMasmParser.cpp
+++ b/llvm/lib/MC/MCParser/COFFMasmParser.cpp
@@ -35,23 +35,23 @@ class COFFMasmParser : public MCAsmParserExtension {
     getParser().addDirectiveHandler(Directive, Handler);
   }
 
-  bool ParseSectionSwitch(StringRef SectionName, unsigned Characteristics);
+  bool parseSectionSwitch(StringRef SectionName, unsigned Characteristics);
 
-  bool ParseSectionSwitch(StringRef SectionName, unsigned Characteristics,
+  bool parseSectionSwitch(StringRef SectionName, unsigned Characteristics,
                           StringRef COMDATSymName, COFF::COMDATType Type,
                           Align Alignment);
 
-  bool ParseDirectiveProc(StringRef, SMLoc);
-  bool ParseDirectiveEndProc(StringRef, SMLoc);
-  bool ParseDirectiveSegment(StringRef, SMLoc);
-  bool ParseDirectiveSegmentEnd(StringRef, SMLoc);
-  bool ParseDirectiveIncludelib(StringRef, SMLoc);
-  bool ParseDirectiveOption(StringRef, SMLoc);
+  bool parseDirectiveProc(StringRef, SMLoc);
+  bool parseDirectiveEndProc(StringRef, SMLoc);
+  bool parseDirectiveSegment(StringRef, SMLoc);
+  bool parseDirectiveSegmentEnd(StringRef, SMLoc);
+  bool parseDirectiveIncludelib(StringRef, SMLoc);
+  bool parseDirectiveOption(StringRef, SMLoc);
 
-  bool ParseDirectiveAlias(StringRef, SMLoc);
+  bool parseDirectiveAlias(StringRef, SMLoc);
 
-  bool ParseSEHDirectiveAllocStack(StringRef, SMLoc);
-  bool ParseSEHDirectiveEndProlog(StringRef, SMLoc);
+  bool parseSEHDirectiveAllocStack(StringRef, SMLoc);
+  bool parseSEHDirectiveEndProlog(StringRef, SMLoc);
 
   bool IgnoreDirective(StringRef, SMLoc) {
     while (!getLexer().is(AsmToken::EndOfStatement)) {
@@ -65,9 +65,9 @@ class COFFMasmParser : public MCAsmParserExtension {
     MCAsmParserExtension::Initialize(Parser);
 
     // x64 directives
-    addDirectiveHandler<&COFFMasmParser::ParseSEHDirectiveAllocStack>(
+    addDirectiveHandler<&COFFMasmParser::parseSEHDirectiveAllocStack>(
         ".allocstack");
-    addDirectiveHandler<&COFFMasmParser::ParseSEHDirectiveEndProlog>(
+    addDirectiveHandler<&COFFMasmParser::parseSEHDirectiveEndProlog>(
         ".endprolog");
 
     // Code label directives
@@ -115,20 +115,20 @@ class COFFMasmParser : public MCAsmParserExtension {
     // goto
 
     // Miscellaneous directives
-    addDirectiveHandler<&COFFMasmParser::ParseDirectiveAlias>("alias");
+    addDirectiveHandler<&COFFMasmParser::parseDirectiveAlias>("alias");
     // assume
     // .fpo
-    addDirectiveHandler<&COFFMasmParser::ParseDirectiveIncludelib>(
+    addDirectiveHandler<&COFFMasmParser::parseDirectiveIncludelib>(
         "includelib");
-    addDirectiveHandler<&COFFMasmParser::ParseDirectiveOption>("option");
+    addDirectiveHandler<&COFFMasmParser::parseDirectiveOption>("option");
     // popcontext
     // pushcontext
     // .safeseh
 
     // Procedure directives
-    addDirectiveHandler<&COFFMasmParser::ParseDirectiveEndProc>("endp");
+    addDirectiveHandler<&COFFMasmParser::parseDirectiveEndProc>("endp");
     // invoke (32-bit only)
-    addDirectiveHandler<&COFFMasmParser::ParseDirectiveProc>("proc");
+    addDirectiveHandler<&COFFMasmParser::parseDirectiveProc>("proc");
     // proto
 
     // Processor directives; all ignored
@@ -153,17 +153,17 @@ class COFFMasmParser : public MCAsmParserExtension {
     // .alpha (32-bit only, order segments alphabetically)
     // .dosseg (32-bit only, order segments in DOS convention)
     // .seq (32-bit only, order segments sequentially)
-    addDirectiveHandler<&COFFMasmParser::ParseDirectiveSegmentEnd>("ends");
+    addDirectiveHandler<&COFFMasmParser::parseDirectiveSegmentEnd>("ends");
     // group (32-bit only)
-    addDirectiveHandler<&COFFMasmParser::ParseDirectiveSegment>("segment");
+    addDirectiveHandler<&COFFMasmParser::parseDirectiveSegment>("segment");
 
     // Simplified segment directives
-    addDirectiveHandler<&COFFMasmParser::ParseSectionDirectiveCode>(".code");
+    addDirectiveHandler<&COFFMasmParser::parseSectionDirectiveCode>(".code");
     // .const
+    addDirectiveHandler<&COFFMasmParser::parseSectionDirectiveInitializedData>(
+        ".data");
     addDirectiveHandler<
-        &COFFMasmParser::ParseSectionDirectiveInitializedData>(".data");
-    addDirectiveHandler<
-        &COFFMasmParser::ParseSectionDirectiveUninitializedData>(".data?");
+        &COFFMasmParser::parseSectionDirectiveUninitializedData>(".data?");
     // .exit
     // .fardata
     // .fardata?
@@ -182,20 +182,20 @@ class COFFMasmParser : public MCAsmParserExtension {
     // typedef
   }
 
-  bool ParseSectionDirectiveCode(StringRef, SMLoc) {
-    return ParseSectionSwitch(".text", COFF::IMAGE_SCN_CNT_CODE |
+  bool parseSectionDirectiveCode(StringRef, SMLoc) {
+    return parseSectionSwitch(".text", COFF::IMAGE_SCN_CNT_CODE |
                                            COFF::IMAGE_SCN_MEM_EXECUTE |
                                            COFF::IMAGE_SCN_MEM_READ);
   }
 
-  bool ParseSectionDirectiveInitializedData(StringRef, SMLoc) {
-    return ParseSectionSwitch(".data", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+  bool parseSectionDirectiveInitializedData(StringRef, SMLoc) {
+    return parseSectionSwitch(".data", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
                                            COFF::IMAGE_SCN_MEM_READ |
                                            COFF::IMAGE_SCN_MEM_WRITE);
   }
 
-  bool ParseSectionDirectiveUninitializedData(StringRef, SMLoc) {
-    return ParseSectionSwitch(".bss", COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA |
+  bool parseSectionDirectiveUninitializedData(StringRef, SMLoc) {
+    return parseSectionSwitch(".bss", COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA |
                                           COFF::IMAGE_SCN_MEM_READ |
                                           COFF::IMAGE_SCN_MEM_WRITE);
   }
@@ -210,13 +210,13 @@ public:
 
 } // end anonymous namespace.
 
-bool COFFMasmParser::ParseSectionSwitch(StringRef SectionName,
+bool COFFMasmParser::parseSectionSwitch(StringRef SectionName,
                                         unsigned Characteristics) {
-  return ParseSectionSwitch(SectionName, Characteristics, "",
+  return parseSectionSwitch(SectionName, Characteristics, "",
                             (COFF::COMDATType)0, Align(16));
 }
 
-bool COFFMasmParser::ParseSectionSwitch(StringRef SectionName,
+bool COFFMasmParser::parseSectionSwitch(StringRef SectionName,
                                         unsigned Characteristics,
                                         StringRef COMDATSymName,
                                         COFF::COMDATType Type,
@@ -233,7 +233,7 @@ bool COFFMasmParser::ParseSectionSwitch(StringRef SectionName,
   return false;
 }
 
-bool COFFMasmParser::ParseDirectiveSegment(StringRef Directive, SMLoc Loc) {
+bool COFFMasmParser::parseDirectiveSegment(StringRef Directive, SMLoc Loc) {
   StringRef SegmentName;
   if (!getLexer().is(AsmToken::Identifier))
     return TokError("expected identifier in directive");
@@ -367,9 +367,9 @@ bool COFFMasmParser::ParseDirectiveSegment(StringRef Directive, SMLoc Loc) {
   return false;
 }
 
-/// ParseDirectiveSegmentEnd
+/// parseDirectiveSegmentEnd
 ///  ::= identifier "ends"
-bool COFFMasmParser::ParseDirectiveSegmentEnd(StringRef Directive, SMLoc Loc) {
+bool COFFMasmParser::parseDirectiveSegmentEnd(StringRef Directive, SMLoc Loc) {
   StringRef SegmentName;
   if (!getLexer().is(AsmToken::Identifier))
     return TokError("expected identifier in directive");
@@ -380,9 +380,9 @@ bool COFFMasmParser::ParseDirectiveSegmentEnd(StringRef Directive, SMLoc Loc) {
   return false;
 }
 
-/// ParseDirectiveIncludelib
+/// parseDirectiveIncludelib
 ///  ::= "includelib" identifier
-bool COFFMasmParser::ParseDirectiveIncludelib(StringRef Directive, SMLoc Loc) {
+bool COFFMasmParser::parseDirectiveIncludelib(StringRef Directive, SMLoc Loc) {
   StringRef Lib;
   if (getParser().parseIdentifier(Lib))
     return TokError("expected identifier in includelib directive");
@@ -398,9 +398,9 @@ bool COFFMasmParser::ParseDirectiveIncludelib(StringRef Directive, SMLoc Loc) {
   return false;
 }
 
-/// ParseDirectiveOption
+/// parseDirectiveOption
 ///  ::= "option" option-list
-bool COFFMasmParser::ParseDirectiveOption(StringRef Directive, SMLoc Loc) {
+bool COFFMasmParser::parseDirectiveOption(StringRef Directive, SMLoc Loc) {
   auto parseOption = [&]() -> bool {
     StringRef Option;
     if (getParser().parseIdentifier(Option))
@@ -435,12 +435,12 @@ bool COFFMasmParser::ParseDirectiveOption(StringRef Directive, SMLoc Loc) {
   return false;
 }
 
-/// ParseDirectiveProc
+/// parseDirectiveProc
 /// TODO(epastor): Implement parameters and other attributes.
 ///  ::= label "proc" [[distance]]
 ///          statements
 ///      label "endproc"
-bool COFFMasmParser::ParseDirectiveProc(StringRef Directive, SMLoc Loc) {
+bool COFFMasmParser::parseDirectiveProc(StringRef Directive, SMLoc Loc) {
   StringRef Label;
   if (getParser().parseIdentifier(Label))
     return Error(Loc, "expected identifier for procedure");
@@ -476,7 +476,7 @@ bool COFFMasmParser::ParseDirectiveProc(StringRef Directive, SMLoc Loc) {
   CurrentProceduresFramed.push_back(Framed);
   return false;
 }
-bool COFFMasmParser::ParseDirectiveEndProc(StringRef Directive, SMLoc Loc) {
+bool COFFMasmParser::parseDirectiveEndProc(StringRef Directive, SMLoc Loc) {
   StringRef Label;
   SMLoc LabelLoc = getTok().getLoc();
   if (getParser().parseIdentifier(Label))
@@ -496,7 +496,7 @@ bool COFFMasmParser::ParseDirectiveEndProc(StringRef Directive, SMLoc Loc) {
   return false;
 }
 
-bool COFFMasmParser::ParseDirectiveAlias(StringRef Directive, SMLoc Loc) {
+bool COFFMasmParser::parseDirectiveAlias(StringRef Directive, SMLoc Loc) {
   std::string AliasName, ActualName;
   if (getTok().isNot(AsmToken::Less) ||
       getParser().parseAngleBracketString(AliasName))
@@ -515,7 +515,7 @@ bool COFFMasmParser::ParseDirectiveAlias(StringRef Directive, SMLoc Loc) {
   return false;
 }
 
-bool COFFMasmParser::ParseSEHDirectiveAllocStack(StringRef Directive,
+bool COFFMasmParser::parseSEHDirectiveAllocStack(StringRef Directive,
                                                  SMLoc Loc) {
   int64_t Size;
   SMLoc SizeLoc = getTok().getLoc();
@@ -527,7 +527,7 @@ bool COFFMasmParser::ParseSEHDirectiveAllocStack(StringRef Directive,
   return false;
 }
 
-bool COFFMasmParser::ParseSEHDirectiveEndProlog(StringRef Directive,
+bool COFFMasmParser::parseSEHDirectiveEndProlog(StringRef Directive,
                                                 SMLoc Loc) {
   getStreamer().emitWinCFIEndProlog(Loc);
   return false;
-- 
GitLab


From 240e4780680f0d6a760be1b9ea432f410424b3a5 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Thu, 24 Oct 2024 23:12:32 -0700
Subject: [PATCH 124/693] Revert "[runtimes] Probe for -nostdlib++ and
 -nostdinc++ with the C compiler" (#113653)

Reverts llvm/llvm-project#108357

Breaks https://lab.llvm.org/buildbot/#/builders/164/builds/3908 and
similar bots
---
 libcxx/cmake/config-ix.cmake    | 12 ++++--------
 libcxxabi/cmake/config-ix.cmake | 12 ++++--------
 libcxxabi/src/CMakeLists.txt    |  4 ++--
 libunwind/cmake/config-ix.cmake | 12 ++++--------
 libunwind/src/CMakeLists.txt    |  2 +-
 runtimes/CMakeLists.txt         | 12 ++++--------
 6 files changed, 19 insertions(+), 35 deletions(-)

diff --git a/libcxx/cmake/config-ix.cmake b/libcxx/cmake/config-ix.cmake
index 192bad7a7a7f..270d80575adc 100644
--- a/libcxx/cmake/config-ix.cmake
+++ b/libcxx/cmake/config-ix.cmake
@@ -38,13 +38,9 @@ check_cxx_compiler_flag(-nolibc CXX_SUPPORTS_NOLIBC_FLAG)
 # required during compilation (which has the -nostdlib++ or -nodefaultlibs). libc is
 # required for the link to go through. We remove sanitizers from the
 # configuration checks to avoid spurious link errors.
-#
-# Adding flags to CMAKE_REQUIRED_FLAGS will include the flags both when testing
-# compilation of C and C++. Therefore test to make sure that the flags are
-# supported by the C compiler driver, before deciding to include them.
 
-check_c_compiler_flag(-nostdlib++ C_SUPPORTS_NOSTDLIBXX_FLAG)
-if (C_SUPPORTS_NOSTDLIBXX_FLAG)
+check_cxx_compiler_flag(-nostdlib++ CXX_SUPPORTS_NOSTDLIBXX_FLAG)
+if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
   set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdlib++")
 else()
   check_c_compiler_flag(-nodefaultlibs C_SUPPORTS_NODEFAULTLIBS_FLAG)
@@ -55,7 +51,7 @@ endif()
 
 # Only link against compiler-rt manually if we use -nodefaultlibs, since
 # otherwise the compiler will do the right thing on its own.
-if (NOT C_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
+if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (LIBCXX_USE_COMPILER_RT)
     include(HandleCompilerRT)
     find_compiler_rt_library(builtins LIBCXX_BUILTINS_LIBRARY
@@ -85,7 +81,7 @@ if (NOT C_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
   endif()
 endif()
 
-if (C_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
+if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (CMAKE_C_FLAGS MATCHES -fsanitize OR CMAKE_CXX_FLAGS MATCHES -fsanitize)
     set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -fno-sanitize=all")
   endif ()
diff --git a/libcxxabi/cmake/config-ix.cmake b/libcxxabi/cmake/config-ix.cmake
index ab74ad79a654..10f2087c68c5 100644
--- a/libcxxabi/cmake/config-ix.cmake
+++ b/libcxxabi/cmake/config-ix.cmake
@@ -22,13 +22,9 @@ endif ()
 # required during compilation (which has the -nodefaultlibs). libc is
 # required for the link to go through. We remove sanitizers from the
 # configuration checks to avoid spurious link errors.
-#
-# Adding flags to CMAKE_REQUIRED_FLAGS will include the flags both when testing
-# compilation of C and C++. Therefore test to make sure that the flags are
-# supported by the C compiler driver, before deciding to include them.
 
-check_c_compiler_flag(-nostdlib++ C_SUPPORTS_NOSTDLIBXX_FLAG)
-if (C_SUPPORTS_NOSTDLIBXX_FLAG)
+check_cxx_compiler_flag(-nostdlib++ CXX_SUPPORTS_NOSTDLIBXX_FLAG)
+if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
   set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdlib++")
 else()
   check_c_compiler_flag(-nodefaultlibs C_SUPPORTS_NODEFAULTLIBS_FLAG)
@@ -39,7 +35,7 @@ endif()
 
 # Only link against compiler-rt manually if we use -nodefaultlibs, since
 # otherwise the compiler will do the right thing on its own.
-if (NOT C_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
+if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (LIBCXXABI_HAS_C_LIB)
     list(APPEND CMAKE_REQUIRED_LIBRARIES c)
   endif ()
@@ -75,7 +71,7 @@ if (NOT C_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
   endif()
 endif()
 
-if (C_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
+if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (CMAKE_C_FLAGS MATCHES -fsanitize OR CMAKE_CXX_FLAGS MATCHES -fsanitize)
     set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -fno-sanitize=all")
   endif ()
diff --git a/libcxxabi/src/CMakeLists.txt b/libcxxabi/src/CMakeLists.txt
index af8071cbdc85..84fe2784bec5 100644
--- a/libcxxabi/src/CMakeLists.txt
+++ b/libcxxabi/src/CMakeLists.txt
@@ -91,7 +91,7 @@ if (ANDROID AND ANDROID_PLATFORM_LEVEL LESS 21)
 endif()
 
 # Setup flags.
-if (C_SUPPORTS_NOSTDLIBXX_FLAG)
+if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
   add_link_flags_if_supported(-nostdlib++)
 else()
   add_link_flags_if_supported(-nodefaultlibs)
@@ -167,7 +167,7 @@ if (LIBCXXABI_USE_LLVM_UNWINDER)
   endif()
 endif()
 target_link_libraries(cxxabi_shared_objects PRIVATE cxx-headers ${LIBCXXABI_LIBRARIES})
-if (NOT C_SUPPORTS_NOSTDLIBXX_FLAG)
+if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG)
   target_link_libraries(cxxabi_shared_objects PRIVATE ${LIBCXXABI_BUILTINS_LIBRARY})
 endif()
 target_link_libraries(cxxabi_shared_objects PUBLIC cxxabi-headers)
diff --git a/libunwind/cmake/config-ix.cmake b/libunwind/cmake/config-ix.cmake
index 88a13b4ec0e4..126c872f0d48 100644
--- a/libunwind/cmake/config-ix.cmake
+++ b/libunwind/cmake/config-ix.cmake
@@ -34,13 +34,9 @@ endif()
 # required during compilation (which has the -nostdlib++ or -nodefaultlibs). libc is
 # required for the link to go through. We remove sanitizers from the
 # configuration checks to avoid spurious link errors.
-#
-# Adding flags to CMAKE_REQUIRED_FLAGS will include the flags both when testing
-# compilation of C and C++. Therefore test to make sure that the flags are
-# supported by the C compiler driver, before deciding to include them.
 
-llvm_check_compiler_linker_flag(C "-nostdlib++" C_SUPPORTS_NOSTDLIBXX_FLAG)
-if (C_SUPPORTS_NOSTDLIBXX_FLAG)
+llvm_check_compiler_linker_flag(CXX "-nostdlib++" CXX_SUPPORTS_NOSTDLIBXX_FLAG)
+if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
   set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdlib++")
 else()
   llvm_check_compiler_linker_flag(C "-nodefaultlibs" C_SUPPORTS_NODEFAULTLIBS_FLAG)
@@ -51,7 +47,7 @@ endif()
 
 # Only link against compiler-rt manually if we use -nodefaultlibs, since
 # otherwise the compiler will do the right thing on its own.
-if (NOT C_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
+if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (LIBUNWIND_HAS_C_LIB)
     list(APPEND CMAKE_REQUIRED_LIBRARIES c)
   endif ()
@@ -86,7 +82,7 @@ if (NOT C_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
   endif()
 endif()
 
-if (C_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
+if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (CMAKE_C_FLAGS MATCHES -fsanitize OR CMAKE_CXX_FLAGS MATCHES -fsanitize)
     set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -fno-sanitize=all")
   endif ()
diff --git a/libunwind/src/CMakeLists.txt b/libunwind/src/CMakeLists.txt
index c58b848026c8..2e18b1096563 100644
--- a/libunwind/src/CMakeLists.txt
+++ b/libunwind/src/CMakeLists.txt
@@ -66,7 +66,7 @@ set(LIBUNWIND_SOURCES
     ${LIBUNWIND_ASM_SOURCES})
 
 # Generate library list.
-if (C_SUPPORTS_NOSTDLIBXX_FLAG)
+if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
   add_link_flags_if_supported(-nostdlib++)
 else()
   if (LIBUNWIND_USE_COMPILER_RT)
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 250ae77fa57b..67368dcedb3b 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -143,16 +143,12 @@ endif()
 # Check for -nostdlib++ first; if there's no C++ standard library yet,
 # all check_cxx_compiler_flag commands will fail until we add -nostdlib++
 # (or -nodefaultlibs).
-#
-# Adding flags to CMAKE_REQUIRED_FLAGS will include the flags both when testing
-# compilation of C and C++. Therefore test to make sure that the flags are
-# supported by the C compiler driver, before deciding to include them.
-llvm_check_compiler_linker_flag(C "-nostdlib++" C_SUPPORTS_NOSTDLIBXX_FLAG)
-if (C_SUPPORTS_NOSTDLIBXX_FLAG)
+llvm_check_compiler_linker_flag(CXX "-nostdlib++" CXX_SUPPORTS_NOSTDLIBXX_FLAG)
+if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
   set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdlib++")
 endif()
-check_c_compiler_flag(-nostdinc++ C_SUPPORTS_NOSTDINCXX_FLAG)
-if (C_SUPPORTS_NOSTDINCXX_FLAG)
+check_cxx_compiler_flag(-nostdinc++ CXX_SUPPORTS_NOSTDINCXX_FLAG)
+if (CXX_SUPPORTS_NOSTDINCXX_FLAG)
   set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdinc++")
 endif()
 
-- 
GitLab


From 3a4ccebb5c5142c15fb6d72aedadb27205ef33d5 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 24 Oct 2024 23:15:19 -0700
Subject: [PATCH 125/693] [Driver] Use != instead of compare to compare strings
 (NFC) (#113651)

---
 clang/lib/Driver/ToolChains/Clang.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 06c44a660e98..04b3832327a9 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2838,7 +2838,7 @@ static std::string ComplexArithmeticStr(LangOptions::ComplexRangeKind Range) {
 
 static void EmitComplexRangeDiag(const Driver &D, std::string str1,
                                  std::string str2) {
-  if ((str1.compare(str2) != 0) && !str2.empty() && !str1.empty()) {
+  if (str1 != str2 && !str2.empty() && !str1.empty()) {
     D.Diag(clang::diag::warn_drv_overriding_option) << str1 << str2;
   }
 }
@@ -2996,8 +2996,8 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
         EmitComplexRangeDiag(D, RenderComplexRangeOption(Range),
                              "-fno-cx-limited-range");
       } else {
-        if (GccRangeComplexOption.compare("-fcx-limited-range") != 0 &&
-            GccRangeComplexOption.compare("-fno-cx-fortran-rules") != 0)
+        if (GccRangeComplexOption != "-fcx-limited-range" &&
+            GccRangeComplexOption != "-fno-cx-fortran-rules")
           EmitComplexRangeDiag(D, GccRangeComplexOption,
                                "-fno-cx-limited-range");
       }
@@ -3042,8 +3042,8 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
         break;
       }
       if (!GccRangeComplexOption.empty()) {
-        if (GccRangeComplexOption.compare("-fcx-limited-range") != 0) {
-          if (GccRangeComplexOption.compare("-fcx-fortran-rules") != 0) {
+        if (GccRangeComplexOption != "-fcx-limited-range") {
+          if (GccRangeComplexOption != "-fcx-fortran-rules") {
             if (RangeVal != LangOptions::ComplexRangeKind::CX_Improved)
               EmitComplexRangeDiag(D, GccRangeComplexOption,
                                    ComplexArithmeticStr(RangeVal));
-- 
GitLab


From 96bb375f5cedcfcc5dcd96296ba54ff933b39d4d Mon Sep 17 00:00:00 2001
From: Yusuke MINATO <minato.yusuke@fujitsu.com>
Date: Fri, 25 Oct 2024 15:20:23 +0900
Subject: [PATCH 126/693] [flang] Integrate the option
 -flang-experimental-integer-overflow into -fno-wrapv (#110063)

nsw is now added to do-variable increment when -fno-wrapv is enabled as
GFortran seems to do.
That means the option introduced by #91579 isn't necessary any more.

Note that the feature of -flang-experimental-integer-overflow is enabled
by default.
---
 clang/include/clang/Driver/Options.td         |   4 -
 clang/lib/Driver/ToolChains/Flang.cpp         |   1 -
 flang/include/flang/Lower/LoweringOptions.def |   5 -
 .../flang/Optimizer/Transforms/Passes.h       |   4 +-
 .../flang/Optimizer/Transforms/Passes.td      |   2 +-
 flang/include/flang/Tools/CrossToolHelpers.h  |   2 +-
 flang/lib/Frontend/CompilerInvocation.cpp     |   6 -
 flang/lib/Frontend/FrontendActions.cpp        |   4 +-
 flang/lib/Lower/Bridge.cpp                    |   2 +-
 flang/lib/Lower/IO.cpp                        |   2 +-
 flang/lib/Optimizer/Passes/Pipelines.cpp      |   6 +-
 .../Transforms/ControlFlowConverter.cpp       |   8 +-
 flang/test/Driver/frontend-forwarding.f90     |   2 -
 .../Fir/convert-to-llvm-openmp-and-fir.fir    |  10 +-
 flang/test/Fir/loop01.fir                     | 230 +-----------------
 flang/test/Fir/loop02.fir                     |   4 +-
 flang/test/Lower/HLFIR/goto-do-body.f90       |   4 +-
 .../OpenMP/parallel-private-clause-fixes.f90  |   4 +-
 ...oop-reduction-allocatable-array-minmax.f90 |   4 +-
 flang/test/Lower/OpenMP/wsloop-variable.f90   |   4 +-
 flang/test/Lower/array-character.f90          |   2 +-
 .../test/Lower/array-derived-assignments.f90  |   2 +-
 flang/test/Lower/array-derived.f90            |   2 +-
 .../array-elemental-calls-char-byval.f90      |   2 +-
 .../test/Lower/array-elemental-calls-char.f90 |   2 +-
 .../Lower/array-expression-assumed-size.f90   |   2 +-
 flang/test/Lower/array-expression-slice-1.f90 |   2 +-
 flang/test/Lower/array-substring.f90          |  46 +---
 flang/test/Lower/array-temp.f90               |   2 +-
 flang/test/Lower/components.f90               |   2 +-
 flang/test/Lower/do_loop.f90                  |  73 ++----
 flang/test/Lower/do_loop_unstructured.f90     | 203 +---------------
 flang/test/Lower/goto-do-body.f90             |   4 +-
 flang/test/Lower/host-associated.f90          |   2 +-
 flang/test/Lower/infinite_loop.f90            |  41 +---
 flang/test/Lower/io-implied-do-fixes.f90      |  49 +---
 flang/test/Lower/loops2.f90                   |   2 +-
 flang/test/Lower/mixed_loops.f90              |   6 +-
 flang/test/Lower/vector-subscript-io.f90      |  34 +--
 flang/tools/bbc/bbc.cpp                       |   9 +-
 40 files changed, 119 insertions(+), 676 deletions(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 5757c385d224..8263fe6f7d7f 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -6782,10 +6782,6 @@ def flang_deprecated_no_hlfir : Flag<["-"], "flang-deprecated-no-hlfir">,
   Flags<[HelpHidden]>, Visibility<[FlangOption, FC1Option]>,
   HelpText<"Do not use HLFIR lowering (deprecated)">;
 
-def flang_experimental_integer_overflow : Flag<["-"], "flang-experimental-integer-overflow">,
-  Flags<[HelpHidden]>, Visibility<[FlangOption, FC1Option]>,
-  HelpText<"Add nsw flag to internal operations such as do-variable increment (experimental)">;
-
 //===----------------------------------------------------------------------===//
 // FLangOption + CoreOption + NoXarchOption
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index a9d2b7a4dc48..366cadc2e547 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -148,7 +148,6 @@ void Flang::addCodegenOptions(const ArgList &Args,
 
   Args.addAllArgs(CmdArgs, {options::OPT_flang_experimental_hlfir,
                             options::OPT_flang_deprecated_no_hlfir,
-                            options::OPT_flang_experimental_integer_overflow,
                             options::OPT_fno_ppc_native_vec_elem_order,
                             options::OPT_fppc_native_vec_elem_order});
 }
diff --git a/flang/include/flang/Lower/LoweringOptions.def b/flang/include/flang/Lower/LoweringOptions.def
index 231de533fbd3..0b22e54b648e 100644
--- a/flang/include/flang/Lower/LoweringOptions.def
+++ b/flang/include/flang/Lower/LoweringOptions.def
@@ -38,10 +38,5 @@ ENUM_LOWERINGOPT(Underscoring, unsigned, 1, 1)
 /// (i.e. wraps around as two's complement). Off by default.
 ENUM_LOWERINGOPT(IntegerWrapAround, unsigned, 1, 0)
 
-/// If true, add nsw flags to loop variable increments.
-/// Off by default.
-/// TODO: integrate this option with the above
-ENUM_LOWERINGOPT(NSWOnLoopVarInc, unsigned, 1, 0)
-
 #undef LOWERINGOPT
 #undef ENUM_LOWERINGOPT
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
index e8f0a8444a31..e1453cfa374b 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.h
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -66,7 +66,7 @@ namespace fir {
 std::unique_ptr<mlir::Pass> createAffineDemotionPass();
 std::unique_ptr<mlir::Pass>
 createArrayValueCopyPass(fir::ArrayValueCopyOptions options = {});
-std::unique_ptr<mlir::Pass> createCFGConversionPassWithNSW();
+std::unique_ptr<mlir::Pass> createCFGConversionPassWithoutNSW();
 std::unique_ptr<mlir::Pass> createMemDataFlowOptPass();
 std::unique_ptr<mlir::Pass> createPromoteToAffinePass();
 std::unique_ptr<mlir::Pass>
@@ -83,7 +83,7 @@ createVScaleAttrPass(std::pair<unsigned, unsigned> vscaleAttr);
 
 void populateCfgConversionRewrites(mlir::RewritePatternSet &patterns,
                                    bool forceLoopToExecuteOnce = false,
-                                   bool setNSW = false);
+                                   bool setNSW = true);
 
 // declarative passes
 #define GEN_PASS_REGISTRATION
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
index a41f0f348f27..54b43adbfc6b 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -153,7 +153,7 @@ def CFGConversion : Pass<"cfg-conversion"> {
            /*default=*/"false",
            "force the body of a loop to execute at least once">,
     Option<"setNSW", "set-nsw", "bool",
-           /*default=*/"false",
+           /*default=*/"true",
            "set nsw on loop variable increment">
   ];
 }
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index df4b21ada058..1626970600ef 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -122,7 +122,7 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
   bool NoSignedZerosFPMath =
       false; ///< Set no-signed-zeros-fp-math attribute for functions.
   bool UnsafeFPMath = false; ///< Set unsafe-fp-math attribute for functions.
-  bool NSWOnLoopVarInc = false; ///< Add nsw flag to loop variable increments.
+  bool NSWOnLoopVarInc = true; ///< Add nsw flag to loop variable increments.
 };
 
 struct OffloadModuleOpts {
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 94d3d1154178..5da5236af2b0 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1350,12 +1350,6 @@ bool CompilerInvocation::createFromArgs(
     invoc.loweringOpts.setNoPPCNativeVecElemOrder(true);
   }
 
-  // -flang-experimental-integer-overflow
-  if (args.hasArg(
-          clang::driver::options::OPT_flang_experimental_integer_overflow)) {
-    invoc.loweringOpts.setNSWOnLoopVarInc(true);
-  }
-
   // Preserve all the remark options requested, i.e. -Rpass, -Rpass-missed or
   // -Rpass-analysis. This will be used later when processing and outputting the
   // remarks generated by LLVM in ExecuteCompilerInvocation.cpp.
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index f2e460fc53a6..3ea242315484 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -828,8 +828,8 @@ void CodeGenAction::generateLLVMIR() {
     config.VScaleMax = vsr->second;
   }
 
-  if (ci.getInvocation().getLoweringOpts().getNSWOnLoopVarInc())
-    config.NSWOnLoopVarInc = true;
+  if (ci.getInvocation().getLoweringOpts().getIntegerWrapAround())
+    config.NSWOnLoopVarInc = false;
 
   // Create the pass pipeline
   fir::createMLIRToLLVMPassPipeline(pm, config, getCurrentFile());
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 877fe122265d..a3bd1ace11da 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -2271,7 +2271,7 @@ private:
     assert(!incrementLoopNestInfo.empty() && "empty loop nest");
     mlir::Location loc = toLocation();
     mlir::arith::IntegerOverflowFlags flags{};
-    if (getLoweringOptions().getNSWOnLoopVarInc())
+    if (!getLoweringOptions().getIntegerWrapAround())
       flags = bitEnumSet(flags, mlir::arith::IntegerOverflowFlags::nsw);
     auto iofAttr = mlir::arith::IntegerOverflowFlagsAttr::get(
         builder->getContext(), flags);
diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index 1894b0cfd1be..b534c81a605a 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -929,7 +929,7 @@ static void genIoLoop(Fortran::lower::AbstractConverter &converter,
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   mlir::Location loc = converter.getCurrentLocation();
   mlir::arith::IntegerOverflowFlags flags{};
-  if (converter.getLoweringOptions().getNSWOnLoopVarInc())
+  if (!converter.getLoweringOptions().getIntegerWrapAround())
     flags = bitEnumSet(flags, mlir::arith::IntegerOverflowFlags::nsw);
   auto iofAttr =
       mlir::arith::IntegerOverflowFlagsAttr::get(builder.getContext(), flags);
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 3c139f7e9340..091b7c4c164e 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -35,11 +35,11 @@ void addCanonicalizerPassWithoutRegionSimplification(mlir::OpPassManager &pm) {
 void addCfgConversionPass(mlir::PassManager &pm,
                           const MLIRToLLVMPassPipelineConfig &config) {
   if (config.NSWOnLoopVarInc)
-    addNestedPassToAllTopLevelOperationsConditionally(
-        pm, disableCfgConversion, fir::createCFGConversionPassWithNSW);
-  else
     addNestedPassToAllTopLevelOperationsConditionally(pm, disableCfgConversion,
                                                       fir::createCFGConversion);
+  else
+    addNestedPassToAllTopLevelOperationsConditionally(
+        pm, disableCfgConversion, fir::createCFGConversionPassWithoutNSW);
 }
 
 void addAVC(mlir::PassManager &pm, const llvm::OptimizationLevel &optLevel) {
diff --git a/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp b/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
index 3b79d6d311b7..411bf7f364a6 100644
--- a/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
+++ b/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
@@ -332,8 +332,6 @@ class CfgConversion : public fir::impl::CFGConversionBase<CfgConversion> {
 public:
   using CFGConversionBase<CfgConversion>::CFGConversionBase;
 
-  CfgConversion(bool setNSW) { this->setNSW = setNSW; }
-
   void runOnOperation() override {
     auto *context = &this->getContext();
     mlir::RewritePatternSet patterns(context);
@@ -366,6 +364,8 @@ void fir::populateCfgConversionRewrites(mlir::RewritePatternSet &patterns,
       patterns.getContext(), forceLoopToExecuteOnce, setNSW);
 }
 
-std::unique_ptr<mlir::Pass> fir::createCFGConversionPassWithNSW() {
-  return std::make_unique<CfgConversion>(true);
+std::unique_ptr<mlir::Pass> fir::createCFGConversionPassWithoutNSW() {
+  fir::CFGConversionOptions options;
+  options.setNSW = false;
+  return fir::createCFGConversion(options);
 }
diff --git a/flang/test/Driver/frontend-forwarding.f90 b/flang/test/Driver/frontend-forwarding.f90
index ff2d66095214..55a74ccf4046 100644
--- a/flang/test/Driver/frontend-forwarding.f90
+++ b/flang/test/Driver/frontend-forwarding.f90
@@ -20,7 +20,6 @@
 ! RUN:     -fversion-loops-for-stride \
 ! RUN:     -flang-experimental-hlfir \
 ! RUN:     -flang-deprecated-no-hlfir \
-! RUN:     -flang-experimental-integer-overflow \
 ! RUN:     -fno-ppc-native-vector-element-order \
 ! RUN:     -fppc-native-vector-element-order \
 ! RUN:     -mllvm -print-before-all \
@@ -52,7 +51,6 @@
 ! CHECK: "-fversion-loops-for-stride"
 ! CHECK: "-flang-experimental-hlfir"
 ! CHECK: "-flang-deprecated-no-hlfir"
-! CHECK: "-flang-experimental-integer-overflow"
 ! CHECK: "-fno-ppc-native-vector-element-order"
 ! CHECK: "-fppc-native-vector-element-order"
 ! CHECK: "-Rpass"
diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index 335877e7c9a8..00f8e6e6cc9a 100644
--- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -358,10 +358,10 @@ func.func @_QPopenmp_target_data_region() {
       %9 = arith.subi %8, %c1_i64 : i64
       %10 = fir.coordinate_of %0, %9 : (!fir.ref<!fir.array<1024xi32>>, i64) -> !fir.ref<i32>
       fir.store %6 to %10 : !fir.ref<i32>
-      %11 = arith.addi %arg0, %c1 : index
+      %11 = arith.addi %arg0, %c1 overflow<nsw> : index
       %12 = fir.convert %c1 : (index) -> i32
       %13 = fir.load %1 : !fir.ref<i32>
-      %14 = arith.addi %13, %12 : i32
+      %14 = arith.addi %13, %12 overflow<nsw> : i32
       fir.result %11, %14 : index, i32
     }
     fir.store %5#1 to %1 : !fir.ref<i32>
@@ -404,11 +404,11 @@ func.func @_QPopenmp_target_data_region() {
 // CHECK:             %[[VAL_21:.*]] = llvm.sub %[[VAL_19]], %[[VAL_20]]  : i64
 // CHECK:             %[[VAL_22:.*]] = llvm.getelementptr %[[VAL_1]][0, %[[VAL_21]]] : (!llvm.ptr, i64) -> !llvm.ptr
 // CHECK:             llvm.store %[[VAL_17]], %[[VAL_22]] : i32, !llvm.ptr
-// CHECK:             %[[VAL_23:.*]] = llvm.add %[[VAL_12]], %[[VAL_8]]  : i64
+// CHECK:             %[[VAL_23:.*]] = llvm.add %[[VAL_12]], %[[VAL_8]] overflow<nsw> : i64
 // CHECK:             %[[VAL_24:.*]] = llvm.trunc %[[VAL_8]] : i64 to i32
 // CHECK:             %[[VAL_25:.*]] = llvm.load %[[VAL_3]] : !llvm.ptr -> i32
-// CHECK:             %[[VAL_26:.*]] = llvm.add %[[VAL_25]], %[[VAL_24]]  : i32
-// CHECK:             %[[VAL_27:.*]] = llvm.add %[[VAL_12]], %[[VAL_8]]  : i64
+// CHECK:             %[[VAL_26:.*]] = llvm.add %[[VAL_25]], %[[VAL_24]] overflow<nsw> : i32
+// CHECK:             %[[VAL_27:.*]] = llvm.add %[[VAL_12]], %[[VAL_8]] overflow<nsw> : i64
 // CHECK:             %[[VAL_28:.*]] = llvm.mlir.constant(1 : index) : i64
 // CHECK:             %[[VAL_29:.*]] = llvm.sub %[[VAL_14]], %[[VAL_28]]  : i64
 // CHECK:             llvm.br ^bb1(%[[VAL_27]], %[[VAL_26]], %[[VAL_29]] : i64, i32, i64)
diff --git a/flang/test/Fir/loop01.fir b/flang/test/Fir/loop01.fir
index c1cbb522c378..30d10b9bbdb9 100644
--- a/flang/test/Fir/loop01.fir
+++ b/flang/test/Fir/loop01.fir
@@ -1,5 +1,7 @@
 // RUN: fir-opt --split-input-file --cfg-conversion %s | FileCheck %s
-// RUN: fir-opt --split-input-file --cfg-conversion="set-nsw=true" %s | FileCheck %s --check-prefix=NSW
+// RUN: fir-opt --split-input-file --cfg-conversion="set-nsw=false" %s | FileCheck %s --check-prefix=NO-NSW
+
+// NO-NSW-NOT: overflow<nsw>
 
 func.func @x(%lb : index, %ub : index, %step : index, %b : i1, %addr : !fir.ref<index>) {
   fir.do_loop %iv = %lb to %ub step %step unordered {
@@ -35,7 +37,7 @@ func.func private @f2() -> i1
 // CHECK:       fir.store %[[VAL_12]] to %[[VAL_4]] : !fir.ref<index>
 // CHECK:       br ^bb5
 // CHECK:     ^bb5:
-// CHECK:       %[[VAL_13:.*]] = arith.addi %[[VAL_8]], %[[VAL_2]] : index
+// CHECK:       %[[VAL_13:.*]] = arith.addi %[[VAL_8]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:       %[[VAL_14:.*]] = arith.constant 1 : index
 // CHECK:       %[[VAL_15:.*]] = arith.subi %[[VAL_9]], %[[VAL_14]] : index
 // CHECK:       br ^bb1(%[[VAL_13]], %[[VAL_15]] : index, index)
@@ -44,34 +46,6 @@ func.func private @f2() -> i1
 // CHECK:     }
 // CHECK:     func private @f2() -> i1
 
-// NSW:     func @x(%[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: index, %[[VAL_3:.*]]: i1, %[[VAL_4:.*]]: !fir.ref<index>) {
-// NSW:       %[[VAL_5:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
-// NSW:       %[[VAL_6:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] : index
-// NSW:       %[[VAL_7:.*]] = arith.divsi %[[VAL_6]], %[[VAL_2]] : index
-// NSW:       br ^bb1(%[[VAL_0]], %[[VAL_7]] : index, index)
-// NSW:     ^bb1(%[[VAL_8:.*]]: index, %[[VAL_9:.*]]: index):
-// NSW:       %[[VAL_10:.*]] = arith.constant 0 : index
-// NSW:       %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_10]] : index
-// NSW:       cond_br %[[VAL_11]], ^bb2, ^bb6
-// NSW:     ^bb2:
-// NSW:       cond_br %[[VAL_3]], ^bb3, ^bb4
-// NSW:     ^bb3:
-// NSW:       fir.store %[[VAL_8]] to %[[VAL_4]] : !fir.ref<index>
-// NSW:       br ^bb5
-// NSW:     ^bb4:
-// NSW:       %[[VAL_12:.*]] = arith.constant 0 : index
-// NSW:       fir.store %[[VAL_12]] to %[[VAL_4]] : !fir.ref<index>
-// NSW:       br ^bb5
-// NSW:     ^bb5:
-// NSW:       %[[VAL_13:.*]] = arith.addi %[[VAL_8]], %[[VAL_2]] overflow<nsw> : index
-// NSW:       %[[VAL_14:.*]] = arith.constant 1 : index
-// NSW:       %[[VAL_15:.*]] = arith.subi %[[VAL_9]], %[[VAL_14]] : index
-// NSW:       br ^bb1(%[[VAL_13]], %[[VAL_15]] : index, index)
-// NSW:     ^bb6:
-// NSW:       return
-// NSW:     }
-// NSW:     func private @f2() -> i1
-
 // -----
 
 func.func @x2(%lo : index, %up : index, %ok : i1) {
@@ -101,36 +75,13 @@ func.func private @f3(i16)
 // CHECK:     cond_br %[[VAL_14]], ^bb2, ^bb3
 // CHECK:   ^bb2:
 // CHECK:     %[[VAL_15:.*]] = fir.call @f2() : () -> i1
-// CHECK:     %[[VAL_16:.*]] = arith.addi %[[VAL_4]], %[[VAL_3]] : index
+// CHECK:     %[[VAL_16:.*]] = arith.addi %[[VAL_4]], %[[VAL_3]] overflow<nsw> : index
 // CHECK:     br ^bb1(%[[VAL_16]], %[[VAL_15]] : index, i1)
 // CHECK:   ^bb3:
 // CHECK:     return
 // CHECK:   }
 // CHECK:   func private @f3(i16)
 
-// NSW:   func @x2(%[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: i1) {
-// NSW:     %[[VAL_3:.*]] = arith.constant 1 : index
-// NSW:     br ^bb1(%[[VAL_0]], %[[VAL_2]] : index, i1)
-// NSW:   ^bb1(%[[VAL_4:.*]]: index, %[[VAL_5:.*]]: i1):
-// NSW:     %[[VAL_6:.*]] = arith.constant 0 : index
-// NSW:     %[[VAL_7:.*]] = arith.cmpi slt, %[[VAL_6]], %[[VAL_3]] : index
-// NSW:     %[[VAL_8:.*]] = arith.cmpi sle, %[[VAL_4]], %[[VAL_1]] : index
-// NSW:     %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_3]], %[[VAL_6]] : index
-// NSW:     %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_4]] : index
-// NSW:     %[[VAL_11:.*]] = arith.andi %[[VAL_7]], %[[VAL_8]] : i1
-// NSW:     %[[VAL_12:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
-// NSW:     %[[VAL_13:.*]] = arith.ori %[[VAL_11]], %[[VAL_12]] : i1
-// NSW:     %[[VAL_14:.*]] = arith.andi %[[VAL_5]], %[[VAL_13]] : i1
-// NSW:     cond_br %[[VAL_14]], ^bb2, ^bb3
-// NSW:   ^bb2:
-// NSW:     %[[VAL_15:.*]] = fir.call @f2() : () -> i1
-// NSW:     %[[VAL_16:.*]] = arith.addi %[[VAL_4]], %[[VAL_3]] overflow<nsw> : index
-// NSW:     br ^bb1(%[[VAL_16]], %[[VAL_15]] : index, i1)
-// NSW:   ^bb3:
-// NSW:     return
-// NSW:   }
-// NSW:   func private @f3(i16)
-
 // -----
 
 // do_loop with an extra loop-carried value
@@ -159,7 +110,7 @@ func.func @x3(%lo : index, %up : index) -> i1 {
 // CHECK:           cond_br %[[VAL_11]], ^bb2, ^bb3
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_12:.*]] = fir.call @f2() : () -> i1
-// CHECK:           %[[VAL_13:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_13:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:           %[[VAL_14:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_15:.*]] = arith.subi %[[VAL_9]], %[[VAL_14]] : index
 // CHECK:           br ^bb1(%[[VAL_13]], %[[VAL_12]], %[[VAL_15]] : index, i1, index)
@@ -167,29 +118,6 @@ func.func @x3(%lo : index, %up : index) -> i1 {
 // CHECK:           return %[[VAL_8]] : i1
 // CHECK:         }
 
-// NSW-LABEL:   func @x3(
-// NSW-SAME:             %[[VAL_0:.*]]: index,
-// NSW-SAME:             %[[VAL_1:.*]]: index) -> i1 {
-// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_3:.*]] = arith.constant true
-// NSW:           %[[VAL_4:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
-// NSW:           %[[VAL_5:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] : index
-// NSW:           %[[VAL_6:.*]] = arith.divsi %[[VAL_5]], %[[VAL_2]] : index
-// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]], %[[VAL_6]] : index, i1, index)
-// NSW:         ^bb1(%[[VAL_7:.*]]: index, %[[VAL_8:.*]]: i1, %[[VAL_9:.*]]: index):
-// NSW:           %[[VAL_10:.*]] = arith.constant 0 : index
-// NSW:           %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_10]] : index
-// NSW:           cond_br %[[VAL_11]], ^bb2, ^bb3
-// NSW:         ^bb2:
-// NSW:           %[[VAL_12:.*]] = fir.call @f2() : () -> i1
-// NSW:           %[[VAL_13:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] overflow<nsw> : index
-// NSW:           %[[VAL_14:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_15:.*]] = arith.subi %[[VAL_9]], %[[VAL_14]] : index
-// NSW:           br ^bb1(%[[VAL_13]], %[[VAL_12]], %[[VAL_15]] : index, i1, index)
-// NSW:         ^bb3:
-// NSW:           return %[[VAL_8]] : i1
-// NSW:         }
-
 // -----
 
 // iterate_while with an extra loop-carried value
@@ -227,7 +155,7 @@ func.func private @f4(i32) -> i1
 // CHECK:           cond_br %[[VAL_16]], ^bb2, ^bb3
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_17:.*]] = fir.call @f2() : () -> i1
-// CHECK:           %[[VAL_18:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_18:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:           br ^bb1(%[[VAL_18]], %[[VAL_6]], %[[VAL_17]] : index, i1, i1)
 // CHECK:         ^bb3:
 // CHECK:           %[[VAL_19:.*]] = arith.andi %[[VAL_6]], %[[VAL_7]] : i1
@@ -235,34 +163,6 @@ func.func private @f4(i32) -> i1
 // CHECK:         }
 // CHECK:         func private @f4(i32) -> i1
 
-// NSW-LABEL:   func @y3(
-// NSW-SAME:             %[[VAL_0:.*]]: index,
-// NSW-SAME:             %[[VAL_1:.*]]: index) -> i1 {
-// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_3:.*]] = arith.constant true
-// NSW:           %[[VAL_4:.*]] = fir.call @f2() : () -> i1
-// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]], %[[VAL_4]] : index, i1, i1)
-// NSW:         ^bb1(%[[VAL_5:.*]]: index, %[[VAL_6:.*]]: i1, %[[VAL_7:.*]]: i1):
-// NSW:           %[[VAL_8:.*]] = arith.constant 0 : index
-// NSW:           %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_8]], %[[VAL_2]] : index
-// NSW:           %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_5]], %[[VAL_1]] : index
-// NSW:           %[[VAL_11:.*]] = arith.cmpi slt, %[[VAL_2]], %[[VAL_8]] : index
-// NSW:           %[[VAL_12:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_5]] : index
-// NSW:           %[[VAL_13:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
-// NSW:           %[[VAL_14:.*]] = arith.andi %[[VAL_11]], %[[VAL_12]] : i1
-// NSW:           %[[VAL_15:.*]] = arith.ori %[[VAL_13]], %[[VAL_14]] : i1
-// NSW:           %[[VAL_16:.*]] = arith.andi %[[VAL_6]], %[[VAL_15]] : i1
-// NSW:           cond_br %[[VAL_16]], ^bb2, ^bb3
-// NSW:         ^bb2:
-// NSW:           %[[VAL_17:.*]] = fir.call @f2() : () -> i1
-// NSW:           %[[VAL_18:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] overflow<nsw> : index
-// NSW:           br ^bb1(%[[VAL_18]], %[[VAL_6]], %[[VAL_17]] : index, i1, i1)
-// NSW:         ^bb3:
-// NSW:           %[[VAL_19:.*]] = arith.andi %[[VAL_6]], %[[VAL_7]] : i1
-// NSW:           return %[[VAL_19]] : i1
-// NSW:         }
-// NSW:         func private @f4(i32) -> i1
-
 // -----
 
 // do_loop that returns the final value of the induction
@@ -291,7 +191,7 @@ func.func @x4(%lo : index, %up : index) -> index {
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_6]] : (index) -> i32
 // CHECK:           %[[VAL_11:.*]] = fir.call @f4(%[[VAL_10]]) : (i32) -> i1
-// CHECK:           %[[VAL_12:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_12:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:           %[[VAL_13:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_14:.*]] = arith.subi %[[VAL_7]], %[[VAL_13]] : index
 // CHECK:           br ^bb1(%[[VAL_12]], %[[VAL_14]] : index, index)
@@ -299,29 +199,6 @@ func.func @x4(%lo : index, %up : index) -> index {
 // CHECK:           return %[[VAL_6]] : index
 // CHECK:         }
 
-// NSW-LABEL:   func @x4(
-// NSW-SAME:             %[[VAL_0:.*]]: index,
-// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
-// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_3:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
-// NSW:           %[[VAL_4:.*]] = arith.addi %[[VAL_3]], %[[VAL_2]] : index
-// NSW:           %[[VAL_5:.*]] = arith.divsi %[[VAL_4]], %[[VAL_2]] : index
-// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_5]] : index, index)
-// NSW:         ^bb1(%[[VAL_6:.*]]: index, %[[VAL_7:.*]]: index):
-// NSW:           %[[VAL_8:.*]] = arith.constant 0 : index
-// NSW:           %[[VAL_9:.*]] = arith.cmpi sgt, %[[VAL_7]], %[[VAL_8]] : index
-// NSW:           cond_br %[[VAL_9]], ^bb2, ^bb3
-// NSW:         ^bb2:
-// NSW:           %[[VAL_10:.*]] = fir.convert %[[VAL_6]] : (index) -> i32
-// NSW:           %[[VAL_11:.*]] = fir.call @f4(%[[VAL_10]]) : (i32) -> i1
-// NSW:           %[[VAL_12:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] overflow<nsw> : index
-// NSW:           %[[VAL_13:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_14:.*]] = arith.subi %[[VAL_7]], %[[VAL_13]] : index
-// NSW:           br ^bb1(%[[VAL_12]], %[[VAL_14]] : index, index)
-// NSW:         ^bb3:
-// NSW:           return %[[VAL_6]] : index
-// NSW:         }
-
 // -----
 
 // iterate_while that returns the final value of both inductions
@@ -356,38 +233,12 @@ func.func @y4(%lo : index, %up : index) -> index {
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
 // CHECK:           %[[VAL_16:.*]] = fir.call @f4(%[[VAL_15]]) : (i32) -> i1
-// CHECK:           %[[VAL_17:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_17:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:           br ^bb1(%[[VAL_17]], %[[VAL_16]] : index, i1)
 // CHECK:         ^bb3:
 // CHECK:           return %[[VAL_4]] : index
 // CHECK:         }
 
-// NSW-LABEL:   func @y4(
-// NSW-SAME:             %[[VAL_0:.*]]: index,
-// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
-// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_3:.*]] = arith.constant true
-// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]] : index, i1)
-// NSW:         ^bb1(%[[VAL_4:.*]]: index, %[[VAL_5:.*]]: i1):
-// NSW:           %[[VAL_6:.*]] = arith.constant 0 : index
-// NSW:           %[[VAL_7:.*]] = arith.cmpi slt, %[[VAL_6]], %[[VAL_2]] : index
-// NSW:           %[[VAL_8:.*]] = arith.cmpi sle, %[[VAL_4]], %[[VAL_1]] : index
-// NSW:           %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_2]], %[[VAL_6]] : index
-// NSW:           %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_4]] : index
-// NSW:           %[[VAL_11:.*]] = arith.andi %[[VAL_7]], %[[VAL_8]] : i1
-// NSW:           %[[VAL_12:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
-// NSW:           %[[VAL_13:.*]] = arith.ori %[[VAL_11]], %[[VAL_12]] : i1
-// NSW:           %[[VAL_14:.*]] = arith.andi %[[VAL_5]], %[[VAL_13]] : i1
-// NSW:           cond_br %[[VAL_14]], ^bb2, ^bb3
-// NSW:         ^bb2:
-// NSW:           %[[VAL_15:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
-// NSW:           %[[VAL_16:.*]] = fir.call @f4(%[[VAL_15]]) : (i32) -> i1
-// NSW:           %[[VAL_17:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] overflow<nsw> : index
-// NSW:           br ^bb1(%[[VAL_17]], %[[VAL_16]] : index, i1)
-// NSW:         ^bb3:
-// NSW:           return %[[VAL_4]] : index
-// NSW:         }
-
 // -----
 
 // do_loop that returns the final induction value
@@ -420,7 +271,7 @@ func.func @x5(%lo : index, %up : index) -> index {
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_12:.*]] = fir.call @f2() : () -> i1
 // CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i1) -> i16
-// CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:           %[[VAL_15:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_16:.*]] = arith.subi %[[VAL_9]], %[[VAL_15]] : index
 // CHECK:           br ^bb1(%[[VAL_14]], %[[VAL_13]], %[[VAL_16]] : index, i16, index)
@@ -429,31 +280,6 @@ func.func @x5(%lo : index, %up : index) -> index {
 // CHECK:           return %[[VAL_7]] : index
 // CHECK:         }
 
-// NSW-LABEL:   func @x5(
-// NSW-SAME:             %[[VAL_0:.*]]: index,
-// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
-// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_3:.*]] = arith.constant 42 : i16
-// NSW:           %[[VAL_4:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
-// NSW:           %[[VAL_5:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] : index
-// NSW:           %[[VAL_6:.*]] = arith.divsi %[[VAL_5]], %[[VAL_2]] : index
-// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]], %[[VAL_6]] : index, i16, index)
-// NSW:         ^bb1(%[[VAL_7:.*]]: index, %[[VAL_8:.*]]: i16, %[[VAL_9:.*]]: index):
-// NSW:           %[[VAL_10:.*]] = arith.constant 0 : index
-// NSW:           %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_10]] : index
-// NSW:           cond_br %[[VAL_11]], ^bb2, ^bb3
-// NSW:         ^bb2:
-// NSW:           %[[VAL_12:.*]] = fir.call @f2() : () -> i1
-// NSW:           %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i1) -> i16
-// NSW:           %[[VAL_14:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] overflow<nsw> : index
-// NSW:           %[[VAL_15:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_16:.*]] = arith.subi %[[VAL_9]], %[[VAL_15]] : index
-// NSW:           br ^bb1(%[[VAL_14]], %[[VAL_13]], %[[VAL_16]] : index, i16, index)
-// NSW:         ^bb3:
-// NSW:           fir.call @f3(%[[VAL_8]]) : (i16) -> ()
-// NSW:           return %[[VAL_7]] : index
-// NSW:         }
-
 // -----
 
 // iterate_while that returns the both induction values
@@ -496,7 +322,7 @@ func.func @y5(%lo : index, %up : index) -> index {
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_17:.*]] = fir.call @f2() : () -> i1
 // CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (i1) -> i16
-// CHECK:           %[[VAL_19:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_19:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:           br ^bb1(%[[VAL_19]], %[[VAL_17]], %[[VAL_18]] : index, i1, i16)
 // CHECK:         ^bb3:
 // CHECK:           cond_br %[[VAL_6]], ^bb4, ^bb5
@@ -508,37 +334,3 @@ func.func @y5(%lo : index, %up : index) -> index {
 // CHECK:           fir.call @f3(%[[VAL_7]]) : (i16) -> ()
 // CHECK:           return %[[VAL_5]] : index
 // CHECK:         }
-
-// NSW-LABEL:   func @y5(
-// NSW-SAME:             %[[VAL_0:.*]]: index,
-// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
-// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_3:.*]] = arith.constant 42 : i16
-// NSW:           %[[VAL_4:.*]] = arith.constant true
-// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_4]], %[[VAL_3]] : index, i1, i16)
-// NSW:         ^bb1(%[[VAL_5:.*]]: index, %[[VAL_6:.*]]: i1, %[[VAL_7:.*]]: i16):
-// NSW:           %[[VAL_8:.*]] = arith.constant 0 : index
-// NSW:           %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_8]], %[[VAL_2]] : index
-// NSW:           %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_5]], %[[VAL_1]] : index
-// NSW:           %[[VAL_11:.*]] = arith.cmpi slt, %[[VAL_2]], %[[VAL_8]] : index
-// NSW:           %[[VAL_12:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_5]] : index
-// NSW:           %[[VAL_13:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
-// NSW:           %[[VAL_14:.*]] = arith.andi %[[VAL_11]], %[[VAL_12]] : i1
-// NSW:           %[[VAL_15:.*]] = arith.ori %[[VAL_13]], %[[VAL_14]] : i1
-// NSW:           %[[VAL_16:.*]] = arith.andi %[[VAL_6]], %[[VAL_15]] : i1
-// NSW:           cond_br %[[VAL_16]], ^bb2, ^bb3
-// NSW:         ^bb2:
-// NSW:           %[[VAL_17:.*]] = fir.call @f2() : () -> i1
-// NSW:           %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (i1) -> i16
-// NSW:           %[[VAL_19:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] overflow<nsw> : index
-// NSW:           br ^bb1(%[[VAL_19]], %[[VAL_17]], %[[VAL_18]] : index, i1, i16)
-// NSW:         ^bb3:
-// NSW:           cond_br %[[VAL_6]], ^bb4, ^bb5
-// NSW:         ^bb4:
-// NSW:           %[[VAL_20:.*]] = arith.constant 0 : i32
-// NSW:           %[[VAL_21:.*]] = fir.call @f4(%[[VAL_20]]) : (i32) -> i1
-// NSW:           br ^bb5
-// NSW:         ^bb5:
-// NSW:           fir.call @f3(%[[VAL_7]]) : (i16) -> ()
-// NSW:           return %[[VAL_5]] : index
-// NSW:         }
diff --git a/flang/test/Fir/loop02.fir b/flang/test/Fir/loop02.fir
index 50948e0e7aa6..fb209a9dfeb4 100644
--- a/flang/test/Fir/loop02.fir
+++ b/flang/test/Fir/loop02.fir
@@ -31,7 +31,7 @@ func.func private @y(%addr : !fir.ref<index>)
 // CHECK:           cond_br %[[VAL_13]], ^bb2, ^bb3
 // CHECK:         ^bb2:
 // CHECK:           fir.call @y(%[[VAL_0]]) : (!fir.ref<index>) -> ()
-// CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_10]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_10]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:           %[[VAL_15:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_16:.*]] = arith.subi %[[VAL_11]], %[[VAL_15]] : index
 // CHECK:           br ^bb1(%[[VAL_14]], %[[VAL_16]] : index, index)
@@ -54,7 +54,7 @@ func.func private @y(%addr : !fir.ref<index>)
 // NOOPT:           cond_br %[[VAL_9]], ^bb2, ^bb3
 // NOOPT:         ^bb2:
 // NOOPT:           fir.call @y(%[[VAL_0]]) : (!fir.ref<index>) -> ()
-// NOOPT:           %[[VAL_10:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] : index
+// NOOPT:           %[[VAL_10:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] overflow<nsw> : index
 // NOOPT:           %[[VAL_11:.*]] = arith.constant 1 : index
 // NOOPT:           %[[VAL_12:.*]] = arith.subi %[[VAL_7]], %[[VAL_11]] : index
 // NOOPT:           br ^bb1(%[[VAL_10]], %[[VAL_12]] : index, index)
diff --git a/flang/test/Lower/HLFIR/goto-do-body.f90 b/flang/test/Lower/HLFIR/goto-do-body.f90
index 383b839e591e..5f5b09ccb8f7 100644
--- a/flang/test/Lower/HLFIR/goto-do-body.f90
+++ b/flang/test/Lower/HLFIR/goto-do-body.f90
@@ -40,7 +40,7 @@ subroutine sub1()
 ! CHECK:    %[[TMP5:.*]] = arith.subi %[[TMP4]], %[[C1]] : i32
 ! CHECK:    fir.store %[[TMP5]] to %[[TRIP]] : !fir.ref<i32>
 ! CHECK:    %[[TMP6:.*]] = fir.load %[[I]]#1 : !fir.ref<i32>
-! CHECK:    %[[TMP7:.*]] = arith.addi %[[TMP6]], %[[C1]] : i32
+! CHECK:    %[[TMP7:.*]] = arith.addi %[[TMP6]], %[[C1]] overflow<nsw> : i32
 ! CHECK:    fir.store %[[TMP7]] to %[[I]]#1 : !fir.ref<i32>
 ! CHECK:    cf.br ^[[HEADER]]
   end do
@@ -104,7 +104,7 @@ subroutine sub2()
 ! CHECK:    fir.store %[[TMP9]] to %[[TRIP]] : !fir.ref<i32>
 ! CHECK:    %[[TMP10:.*]] = fir.load %[[I]]#1 : !fir.ref<i32>
 ! CHECK:    %[[STEP_VAL:.*]] = fir.load %[[STEP_VAR]] : !fir.ref<i32>
-! CHECK:    %[[TMP11:.*]] = arith.addi %[[TMP10]], %[[STEP_VAL]] : i32
+! CHECK:    %[[TMP11:.*]] = arith.addi %[[TMP10]], %[[STEP_VAL]] overflow<nsw> : i32
 ! CHECK:    fir.store %[[TMP11]] to %[[I]]#1 : !fir.ref<i32>
 ! CHECK:    cf.br ^[[HEADER]]
   end do
diff --git a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90 b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
index 5e76e8ff1663..99323e69113b 100644
--- a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
+++ b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
@@ -82,10 +82,10 @@
 ! CHECK:                 %[[VAL_15:.*]] = fir.load %[[PRIV_J_DECL]]#0 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_16:.*]] = arith.addi %[[LOAD]], %[[VAL_15]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_16]] to %[[PRIV_X_DECL]]#0 : i32, !fir.ref<i32>
-! CHECK:                 %[[VAL_17:.*]] = arith.addi %[[VAL_13]], %[[VAL_11]] : index
+! CHECK:                 %[[VAL_17:.*]] = arith.addi %[[VAL_13]], %[[VAL_11]] overflow<nsw> : index
 ! CHECK:                 %[[STEPCAST:.*]] = fir.convert %[[VAL_11]] : (index) -> i32
 ! CHECK:                 %[[IVLOAD:.*]] = fir.load %[[PRIV_J_DECL]]#1 : !fir.ref<i32>
-! CHECK:                 %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST]]
+! CHECK:                 %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST]] overflow<nsw> :
 ! CHECK:                 fir.result %[[VAL_17]], %[[IVINC]] : index, i32
 ! CHECK:               }
 ! CHECK:               fir.store %[[VAL_12]]#1 to %[[PRIV_J_DECL]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
index a49eba69ff38..ce45d09d77a2 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
@@ -206,10 +206,10 @@ end program
 ! CHECK:             %[[VAL_48:.*]] = fir.convert %[[VAL_47]] : (i32) -> i64
 ! CHECK:             %[[VAL_49:.*]] = hlfir.designate %[[VAL_46]] (%[[VAL_48]])  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, i64) -> !fir.ref<i32>
 ! CHECK:             hlfir.assign %[[VAL_45]] to %[[VAL_49]] : i32, !fir.ref<i32>
-! CHECK:             %[[VAL_50:.*]] = arith.addi %[[VAL_43]], %[[VAL_40]] : index
+! CHECK:             %[[VAL_50:.*]] = arith.addi %[[VAL_43]], %[[VAL_40]] overflow<nsw> : index
 ! CHECK:             %[[VAL_51:.*]] = fir.convert %[[VAL_40]] : (index) -> i32
 ! CHECK:             %[[VAL_52:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref<i32>
-! CHECK:             %[[VAL_53:.*]] = arith.addi %[[VAL_52]], %[[VAL_51]] : i32
+! CHECK:             %[[VAL_53:.*]] = arith.addi %[[VAL_52]], %[[VAL_51]] overflow<nsw> : i32
 ! CHECK:             fir.result %[[VAL_50]], %[[VAL_53]] : index, i32
 ! CHECK:           }
 ! CHECK:           fir.store %[[VAL_54:.*]]#1 to %[[VAL_3]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-variable.f90 b/flang/test/Lower/OpenMP/wsloop-variable.f90
index 8d235c10fa1d..cc77ce754d97 100644
--- a/flang/test/Lower/OpenMP/wsloop-variable.f90
+++ b/flang/test/Lower/OpenMP/wsloop-variable.f90
@@ -150,10 +150,10 @@ subroutine wsloop_variable_sub
 !CHECK:                 %[[VAL_42:.*]] = arith.addi %[[VAL_40]], %[[VAL_41]] : i64
 !CHECK:                 %[[VAL_43:.*]] = fir.convert %[[VAL_42]] : (i64) -> f32
 !CHECK:                 hlfir.assign %[[VAL_43]] to %[[VAL_21]]#0 : f32, !fir.ref<f32>
-!CHECK:                 %[[VAL_44:.*]] = arith.addi %[[VAL_37]], %[[VAL_34]] : index
+!CHECK:                 %[[VAL_44:.*]] = arith.addi %[[VAL_37]], %[[VAL_34]] overflow<nsw> : index
 !CHECK:                 %[[VAL_45:.*]] = fir.convert %[[VAL_34]] : (index) -> i64
 !CHECK:                 %[[VAL_46:.*]] = fir.load %[[VAL_17]]#1 : !fir.ref<i64>
-!CHECK:                 %[[VAL_47:.*]] = arith.addi %[[VAL_46]], %[[VAL_45]] : i64
+!CHECK:                 %[[VAL_47:.*]] = arith.addi %[[VAL_46]], %[[VAL_45]] overflow<nsw> : i64
 !CHECK:                 fir.result %[[VAL_44]], %[[VAL_47]] : index, i64
 !CHECK:               }
 !CHECK:               fir.store %[[VAL_48:.*]]#1 to %[[VAL_17]]#1 : !fir.ref<i64>
diff --git a/flang/test/Lower/array-character.f90 b/flang/test/Lower/array-character.f90
index c93ef4be3082..53adc5c02958 100644
--- a/flang/test/Lower/array-character.f90
+++ b/flang/test/Lower/array-character.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -hlfir=false %s -o - | fir-opt --canonicalize --cse | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv %s -o - | fir-opt --canonicalize --cse | FileCheck %s
 
 ! CHECK-LABEL: func @_QPissue(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.boxchar<1>{{.*}}, %[[VAL_1:.*]]: !fir.boxchar<1>{{.*}}) {
diff --git a/flang/test/Lower/array-derived-assignments.f90 b/flang/test/Lower/array-derived-assignments.f90
index 71e61f651302..f4e51271d593 100644
--- a/flang/test/Lower/array-derived-assignments.f90
+++ b/flang/test/Lower/array-derived-assignments.f90
@@ -1,5 +1,5 @@
 ! Test derived type assignment lowering inside array expression
-! RUN: bbc -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s
 
 module array_derived_assign
   type simple_copy
diff --git a/flang/test/Lower/array-derived.f90 b/flang/test/Lower/array-derived.f90
index b5eb7621c90f..a0c55f5d8825 100644
--- a/flang/test/Lower/array-derived.f90
+++ b/flang/test/Lower/array-derived.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s
 
 module cs
   type r
diff --git a/flang/test/Lower/array-elemental-calls-char-byval.f90 b/flang/test/Lower/array-elemental-calls-char-byval.f90
index c321614e7fc5..682191fc7895 100644
--- a/flang/test/Lower/array-elemental-calls-char-byval.f90
+++ b/flang/test/Lower/array-elemental-calls-char-byval.f90
@@ -1,6 +1,6 @@
 ! Test lowering of elemental calls with character argument
 ! with the VALUE attribute.
-! RUN: bbc -hlfir=false -o - %s | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv -o - %s | FileCheck %s
 
 
 module char_elem_byval
diff --git a/flang/test/Lower/array-elemental-calls-char.f90 b/flang/test/Lower/array-elemental-calls-char.f90
index 603cc677805f..00e2f8e8f9c1 100644
--- a/flang/test/Lower/array-elemental-calls-char.f90
+++ b/flang/test/Lower/array-elemental-calls-char.f90
@@ -1,6 +1,6 @@
 ! Test lowering of elemental calls with character argument
 ! without the VALUE attribute.
-! RUN: bbc -hlfir=false -o - %s | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv -o - %s | FileCheck %s
 
 module char_elem
 
diff --git a/flang/test/Lower/array-expression-assumed-size.f90 b/flang/test/Lower/array-expression-assumed-size.f90
index ae35da951538..2fbf315aff11 100644
--- a/flang/test/Lower/array-expression-assumed-size.f90
+++ b/flang/test/Lower/array-expression-assumed-size.f90
@@ -1,5 +1,5 @@
 ! RUN: bbc --emit-fir -hlfir=false %s -o - | FileCheck %s
-! RUN: bbc -hlfir=false %s -o - | FileCheck --check-prefix=PostOpt %s
+! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck --check-prefix=PostOpt %s
 
 
 subroutine assumed_size_test(a)
diff --git a/flang/test/Lower/array-expression-slice-1.f90 b/flang/test/Lower/array-expression-slice-1.f90
index 152450902432..b597814bc0d9 100644
--- a/flang/test/Lower/array-expression-slice-1.f90
+++ b/flang/test/Lower/array-expression-slice-1.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -hlfir=false -o - --outline-intrinsics %s | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv -o - --outline-intrinsics %s | FileCheck %s
 
 ! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "p"} {
 ! CHECK-DAG:         %[[VAL_0:.*]] = arith.constant 10 : index
diff --git a/flang/test/Lower/array-substring.f90 b/flang/test/Lower/array-substring.f90
index 2e283997e3e0..02101039120e 100644
--- a/flang/test/Lower/array-substring.f90
+++ b/flang/test/Lower/array-substring.f90
@@ -1,5 +1,7 @@
 ! RUN: bbc -hlfir=false %s -o - | FileCheck %s
-! RUN: bbc -hlfir=false -integer-overflow %s -o - | FileCheck %s --check-prefix=NSW
+! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s --check-prefix=NO-NSW
+
+! NO-NSW-NOT: overflow<nsw>
 
 ! CHECK-LABEL: func @_QPtest(
 ! CHECK-SAME:     %[[VAL_0:.*]]: !fir.boxchar<1>{{.*}}) -> !fir.array<1x!fir.logical<4>> {
@@ -32,8 +34,9 @@
 ! CHECK:         %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:         %[[VAL_27:.*]] = fir.array_coor %[[VAL_8]](%[[VAL_9]]) %[[VAL_15]] : (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.shape<1>, index) -> !fir.ref<!fir.logical<4>>
 ! CHECK:         fir.store %[[VAL_26]] to %[[VAL_27]] : !fir.ref<!fir.logical<4>>
+! CHECK:         %[[VAL_15_NSW:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] overflow<nsw> : index
 ! CHECK:         %[[VAL_28:.*]] = arith.subi %[[VAL_13]], %[[VAL_1]] : index
-! CHECK:         br ^bb1(%[[VAL_15]], %[[VAL_28]] : index, index)
+! CHECK:         br ^bb1(%[[VAL_15_NSW]], %[[VAL_28]] : index, index)
 ! CHECK:       ^bb3:
 ! CHECK:         %[[VAL_29:.*]] = fir.load %[[VAL_8]] : !fir.ref<!fir.array<1x!fir.logical<4>>>
 ! CHECK:         return %[[VAL_29]] : !fir.array<1x!fir.logical<4>>
@@ -46,42 +49,3 @@ function test(C)
 
   test = C(1:1)(1:8) == (/'ABCDabcd'/) 
 end function test
-
-! NSW-LABEL: func @_QPtest(
-! NSW-SAME:     %[[VAL_0:.*]]: !fir.boxchar<1>{{.*}}) -> !fir.array<1x!fir.logical<4>> {
-! NSW-DAG:         %[[VAL_1:.*]] = arith.constant 1 : index
-! NSW-DAG:         %[[VAL_2:.*]] = arith.constant 0 : index
-! NSW-DAG:         %[[VAL_3:.*]] = arith.constant 0 : i32
-! NSW-DAG:         %[[VAL_4:.*]] = arith.constant 8 : index
-! NSW:         %[[VAL_6:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! NSW:         %[[VAL_7:.*]] = fir.convert %[[VAL_6]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<1x!fir.char<1,12>>>
-! NSW:         %[[VAL_8:.*]] = fir.alloca !fir.array<1x!fir.logical<4>> {bindc_name = "test", uniq_name = "_QFtestEtest"}
-! NSW:         %[[VAL_9:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
-! NSW:         %[[VAL_10:.*]] = fir.slice %[[VAL_1]], %[[VAL_1]], %[[VAL_1]] : (index, index, index) -> !fir.slice<1>
-! NSW:         %[[VAL_11:.*]] = fir.address_of(@_QQ{{.*}}) : !fir.ref<!fir.array<1x!fir.char<1,8>>>
-! NSW:         br ^bb1(%[[VAL_2]], %[[VAL_1]] : index, index)
-! NSW:       ^bb1(%[[VAL_12:.*]]: index, %[[VAL_13:.*]]: index):
-! NSW:         %[[VAL_14:.*]] = arith.cmpi sgt, %[[VAL_13]], %[[VAL_2]] : index
-! NSW:         cond_br %[[VAL_14]], ^bb2, ^bb3
-! NSW:       ^bb2:
-! NSW:         %[[VAL_15:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] : index
-! NSW:         %[[VAL_16:.*]] = fir.array_coor %[[VAL_7]](%[[VAL_9]]) {{\[}}%[[VAL_10]]] %[[VAL_15]] : (!fir.ref<!fir.array<1x!fir.char<1,12>>>, !fir.shape<1>, !fir.slice<1>, index) -> !fir.ref<!fir.char<1,12>>
-! NSW:         %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (!fir.ref<!fir.char<1,12>>) -> !fir.ref<!fir.array<12x!fir.char<1>>>
-! NSW:         %[[VAL_18:.*]] = fir.coordinate_of %[[VAL_17]], %[[VAL_2]] : (!fir.ref<!fir.array<12x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
-! NSW:         %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<!fir.char<1,?>>
-! NSW:         %[[VAL_20:.*]] = fir.array_coor %[[VAL_11]](%[[VAL_9]]) %[[VAL_15]] : (!fir.ref<!fir.array<1x!fir.char<1,8>>>, !fir.shape<1>, index) -> !fir.ref<!fir.char<1,8>>
-! NSW:         %[[VAL_21:.*]] = fir.convert %[[VAL_19]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
-! NSW:         %[[VAL_22:.*]] = fir.convert %[[VAL_20]] : (!fir.ref<!fir.char<1,8>>) -> !fir.ref<i8>
-! NSW:         %[[VAL_23:.*]] = fir.convert %[[VAL_4]] : (index) -> i64
-! NSW:         %[[VAL_24:.*]] = fir.call @_FortranACharacterCompareScalar1(%[[VAL_21]], %[[VAL_22]], %[[VAL_23]], %[[VAL_23]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64, i64) -> i32
-! NSW:         %[[VAL_25:.*]] = arith.cmpi eq, %[[VAL_24]], %[[VAL_3]] : i32
-! NSW:         %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
-! NSW:         %[[VAL_27:.*]] = fir.array_coor %[[VAL_8]](%[[VAL_9]]) %[[VAL_15]] : (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.shape<1>, index) -> !fir.ref<!fir.logical<4>>
-! NSW:         fir.store %[[VAL_26]] to %[[VAL_27]] : !fir.ref<!fir.logical<4>>
-! NSW:         %[[VAL_15_NSW:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] overflow<nsw> : index
-! NSW:         %[[VAL_28:.*]] = arith.subi %[[VAL_13]], %[[VAL_1]] : index
-! NSW:         br ^bb1(%[[VAL_15_NSW]], %[[VAL_28]] : index, index)
-! NSW:       ^bb3:
-! NSW:         %[[VAL_29:.*]] = fir.load %[[VAL_8]] : !fir.ref<!fir.array<1x!fir.logical<4>>>
-! NSW:         return %[[VAL_29]] : !fir.array<1x!fir.logical<4>>
-! NSW:       }
diff --git a/flang/test/Lower/array-temp.f90 b/flang/test/Lower/array-temp.f90
index 10c5ee91d44b..718aef84a4e8 100644
--- a/flang/test/Lower/array-temp.f90
+++ b/flang/test/Lower/array-temp.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPss1()
 subroutine ss1
diff --git a/flang/test/Lower/components.f90 b/flang/test/Lower/components.f90
index e1582a8a31e0..28e836c5d104 100644
--- a/flang/test/Lower/components.f90
+++ b/flang/test/Lower/components.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s
 
 module components_test
   type t1
diff --git a/flang/test/Lower/do_loop.f90 b/flang/test/Lower/do_loop.f90
index a46e6c947391..5d8343b8d68a 100644
--- a/flang/test/Lower/do_loop.f90
+++ b/flang/test/Lower/do_loop.f90
@@ -1,17 +1,17 @@
 ! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false -o - %s | FileCheck %s
 ! RUN: %flang_fc1 -mllvm --use-desc-for-alloc=false -emit-fir -flang-deprecated-no-hlfir -o - %s | FileCheck %s
-! RUN: %flang_fc1 -mllvm --use-desc-for-alloc=false -emit-fir -flang-deprecated-no-hlfir -flang-experimental-integer-overflow -o - %s | FileCheck %s --check-prefix=NSW
+! RUN: %flang_fc1 -mllvm --use-desc-for-alloc=false -emit-fir -flang-deprecated-no-hlfir -fwrapv -o - %s | FileCheck %s --check-prefix=NO-NSW
 
 ! Simple tests for structured ordered loops with loop-control.
 ! Tests the structure of the loop, storage to index variable and return and 
 ! storage of the final value of the index variable.
 
+! NO-NSW-NOT: overflow<nsw>
+
 ! Test a simple loop with the final value of the index variable read outside the loop
 ! CHECK-LABEL: simple_loop
-! NSW-LABEL:   simple_loop
 subroutine simple_loop
   ! CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_loopEi"}
-  ! NSW:   %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_loopEi"}
   integer :: i
 
   ! CHECK: %[[C1:.*]] = arith.constant 1 : i32
@@ -21,18 +21,14 @@ subroutine simple_loop
   ! CHECK: %[[C1:.*]] = arith.constant 1 : index
   ! CHECK: %[[LB:.*]] = fir.convert %[[C1_CVT]] : (index) -> i32
   ! CHECK: %[[LI_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
-  ! NSW:   %[[LI_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[C1_CVT]] to %[[C5_CVT]] step %[[C1]]
   ! CHECK-SAME: iter_args(%[[IV:.*]] = %[[LB]]) -> (index, i32) {
   do i=1,5
   ! CHECK:   fir.store %[[IV]] to %[[I_REF]] : !fir.ref<i32>
-  ! CHECK:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[C1]] : index
-  ! NSW:     %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[C1:.*]] overflow<nsw> : index
+  ! CHECK:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[C1]] overflow<nsw> : index
   ! CHECK:   %[[STEPCAST:.*]] = fir.convert %[[C1]] : (index) -> i32
   ! CHECK:   %[[IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-  ! NSW:     %[[IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-  ! CHECK:   %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST]] : i32
-  ! NSW:     %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST:.*]] overflow<nsw> : i32
+  ! CHECK:   %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST]] overflow<nsw> : i32
   ! CHECK:  fir.result %[[LI_NEXT]], %[[IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -44,14 +40,11 @@ end subroutine
 
 ! Test a 2-nested loop with a body composed of a reduction. Values are read from a 2d array.
 ! CHECK-LABEL: nested_loop
-! NSW-LABEL:   nested_loop
 subroutine nested_loop
   ! CHECK: %[[ARR_REF:.*]] = fir.alloca !fir.array<5x5xi32> {bindc_name = "arr", uniq_name = "_QFnested_loopEarr"}
   ! CHECK: %[[ASUM_REF:.*]] = fir.alloca i32 {bindc_name = "asum", uniq_name = "_QFnested_loopEasum"}
   ! CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_loopEi"}
-  ! NSW:   %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_loopEi"}
   ! CHECK: %[[J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_loopEj"}
-  ! NSW:   %[[J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_loopEj"}
   integer :: asum, arr(5,5)
   integer :: i, j
   asum = 0
@@ -62,7 +55,6 @@ subroutine nested_loop
   ! CHECK: %[[ST_I:.*]] = arith.constant 1 : index
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[S_I_CVT]] : (index) -> i32
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
-  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[S_I_CVT]] to %[[E_I_CVT]] step %[[ST_I]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do i=1,5
@@ -74,7 +66,6 @@ subroutine nested_loop
     ! CHECK: %[[ST_J:.*]] = arith.constant 1 : index
     ! CHECK: %[[J_LB:.*]] = fir.convert %[[S_J_CVT]] : (index) -> i32
     ! CHECK: %[[J_RES:.*]]:2 = fir.do_loop %[[LJ:[^ ]*]] =
-    ! NSW:   %[[J_RES:.*]]:2 = fir.do_loop %[[LJ:[^ ]*]] =
     ! CHECK-SAME: %[[S_J_CVT]] to %[[E_J_CVT]] step %[[ST_J]]
     ! CHECK-SAME: iter_args(%[[J_IV:.*]] = %[[J_LB]]) -> (index, i32) {
     do j=1,5
@@ -93,24 +84,18 @@ subroutine nested_loop
       ! CHECK: %[[ASUM_NEW:.*]] = arith.addi %[[ASUM]], %[[ARR_VAL]] : i32
       ! CHECK: fir.store %[[ASUM_NEW]] to %[[ASUM_REF]] : !fir.ref<i32>
       asum = asum + arr(i,j)
-      ! CHECK: %[[LJ_NEXT:.*]] = arith.addi %[[LJ]], %[[ST_J]] : index
-      ! NSW:   %[[LJ_NEXT:.*]] = arith.addi %[[LJ]], %[[ST_J:.*]] overflow<nsw> : index
+      ! CHECK: %[[LJ_NEXT:.*]] = arith.addi %[[LJ]], %[[ST_J]] overflow<nsw> : index
       ! CHECK: %[[J_STEPCAST:.*]] = fir.convert %[[ST_J]] : (index) -> i32
       ! CHECK: %[[J_IVLOAD:.*]] = fir.load %[[J_REF]] : !fir.ref<i32>
-      ! NSW:   %[[J_IVLOAD:.*]] = fir.load %[[J_REF]] : !fir.ref<i32>
-      ! CHECK: %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] : i32
-      ! NSW:   %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST:.*]] overflow<nsw> : i32
+      ! CHECK: %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] overflow<nsw> : i32
       ! CHECK: fir.result %[[LJ_NEXT]], %[[J_IVINC]] : index, i32
     ! CHECK: }
     end do
     ! CHECK: fir.store %[[J_RES]]#1 to %[[J_REF]] : !fir.ref<i32>
-    ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_I]] : index
-    ! NSW:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_I:.*]] overflow<nsw> : index
+    ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_I]] overflow<nsw> : index
     ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[ST_I]] : (index) -> i32
     ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-    ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-    ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
-    ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
+    ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i32
     ! CHECK: fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -119,11 +104,9 @@ end subroutine
 
 ! Test a downcounting loop
 ! CHECK-LABEL: down_counting_loop
-! NSW-LABEL:   down_counting_loop
 subroutine down_counting_loop()
   integer :: i
   ! CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFdown_counting_loopEi"}
-  ! NSW:   %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFdown_counting_loopEi"}
 
   ! CHECK: %[[C5:.*]] = arith.constant 5 : i32
   ! CHECK: %[[C5_CVT:.*]] = fir.convert %[[C5]] : (i32) -> index
@@ -133,18 +116,14 @@ subroutine down_counting_loop()
   ! CHECK: %[[CMINUS1_STEP_CVT:.*]] = fir.convert %[[CMINUS1]] : (i32) -> index
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[C5_CVT]] : (index) -> i32
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
-  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[C5_CVT]] to %[[C1_CVT]] step %[[CMINUS1_STEP_CVT]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do i=5,1,-1
   ! CHECK: fir.store %[[I_IV]] to %[[I_REF]] : !fir.ref<i32>
-  ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[CMINUS1_STEP_CVT]] : index
-  ! NSW:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[CMINUS1_STEP_CVT:.*]] overflow<nsw> : index
+  ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[CMINUS1_STEP_CVT]] overflow<nsw> : index
   ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[CMINUS1_STEP_CVT]] : (index) -> i32
   ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-  ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-  ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
-  ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
+  ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i32
   ! CHECK: fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -153,7 +132,6 @@ end subroutine
 
 ! Test a general loop with a variable step
 ! CHECK-LABEL: loop_with_variable_step
-! NSW-LABEL:   loop_with_variable_step
 ! CHECK-SAME: (%[[S_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "s"}, %[[E_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "e"}, %[[ST_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "st"}) {
 subroutine loop_with_variable_step(s,e,st)
   integer :: s, e, st
@@ -166,18 +144,14 @@ subroutine loop_with_variable_step(s,e,st)
   ! CHECK: %[[ST_CVT:.*]] = fir.convert %[[ST]] : (i32) -> index
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[S_CVT]] : (index) -> i32
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
-  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[S_CVT]] to %[[E_CVT]] step %[[ST_CVT]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do i=s,e,st
   ! CHECK:  fir.store %[[I_IV]] to %[[I_REF]] : !fir.ref<i32>
-  ! CHECK:  %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] : index
-  ! NSW:    %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT:.*]] overflow<nsw> : index
+  ! CHECK:  %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] overflow<nsw> : index
   ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[ST_CVT]] : (index) -> i32
   ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-  ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-  ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
-  ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
+  ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i32
   ! CHECK:  fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -186,13 +160,11 @@ end subroutine
 
 ! Test usage of pointer variables as index, start, end and step variables
 ! CHECK-LABEL: loop_with_pointer_variables
-! NSW-LABEL:   loop_with_pointer_variables
 ! CHECK-SAME: (%[[S_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "s", fir.target}, %[[E_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "e", fir.target}, %[[ST_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "st", fir.target}) {
 subroutine loop_with_pointer_variables(s,e,st)
 ! CHECK:  %[[E_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEeptr.addr"}
 ! CHECK:  %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", fir.target, uniq_name = "_QFloop_with_pointer_variablesEi"}
 ! CHECK:  %[[I_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEiptr.addr"}
-! NSW:    %[[I_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEiptr.addr"}
 ! CHECK:  %[[S_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEsptr.addr"}
 ! CHECK:  %[[ST_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEstptr.addr"}
   integer, target :: i
@@ -213,7 +185,6 @@ subroutine loop_with_pointer_variables(s,e,st)
   stptr => st
 
 ! CHECK:  %[[I_PTR:.*]] = fir.load %[[I_PTR_REF]] : !fir.ref<!fir.ptr<i32>>
-! NSW:    %[[I_PTR:.*]] = fir.load %[[I_PTR_REF]] : !fir.ref<!fir.ptr<i32>>
 ! CHECK:  %[[S_PTR:.*]] = fir.load %[[S_PTR_REF]] : !fir.ref<!fir.ptr<i32>>
 ! CHECK:  %[[S:.*]] = fir.load %[[S_PTR]] : !fir.ptr<i32>
 ! CHECK:  %[[S_CVT:.*]] = fir.convert %[[S]] : (i32) -> index
@@ -225,18 +196,14 @@ subroutine loop_with_pointer_variables(s,e,st)
 ! CHECK:  %[[ST_CVT:.*]] = fir.convert %[[ST]] : (i32) -> index
 ! CHECK:  %[[I_LB:.*]] = fir.convert %[[S_CVT]] : (index) -> i32
 ! CHECK:  %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
-! NSW:    %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
 ! CHECK-SAME: %[[S_CVT]] to %[[E_CVT]] step %[[ST_CVT]]
 ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do iptr=sptr,eptr,stptr
 ! CHECK:    fir.store %[[I_IV]] to %[[I_PTR]] : !fir.ptr<i32>
-! CHECK:    %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] : index
-! NSW:      %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT:.*]] overflow<nsw> : index
+! CHECK:    %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] overflow<nsw> : index
 ! CHECK:    %[[I_STEPCAST:.*]] = fir.convert %[[ST_CVT]] : (index) -> i32
 ! CHECK:    %[[I_IVLOAD:.*]] = fir.load %[[I_PTR]] : !fir.ptr<i32>
-! NSW:      %[[I_IVLOAD:.*]] = fir.load %[[I_PTR]] : !fir.ptr<i32>
-! CHECK:    %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
-! NSW:      %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
+! CHECK:    %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i32
 ! CHECK:    fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   end do
 ! CHECK:  }
@@ -245,11 +212,9 @@ end subroutine
 
 ! Test usage of non-default integer kind for loop control and loop index variable
 ! CHECK-LABEL: loop_with_non_default_integer
-! NSW-LABEL:   loop_with_non_default_integer
 ! CHECK-SAME: (%[[S_REF:.*]]: !fir.ref<i64> {fir.bindc_name = "s"}, %[[E_REF:.*]]: !fir.ref<i64> {fir.bindc_name = "e"}, %[[ST_REF:.*]]: !fir.ref<i64> {fir.bindc_name = "st"}) {
 subroutine loop_with_non_default_integer(s,e,st)
   ! CHECK: %[[I_REF:.*]] = fir.alloca i64 {bindc_name = "i", uniq_name = "_QFloop_with_non_default_integerEi"}
-  ! NSW:   %[[I_REF:.*]] = fir.alloca i64 {bindc_name = "i", uniq_name = "_QFloop_with_non_default_integerEi"}
   integer(kind=8):: i
   ! CHECK: %[[S:.*]] = fir.load %[[S_REF]] : !fir.ref<i64>
   ! CHECK: %[[S_CVT:.*]] = fir.convert %[[S]] : (i64) -> index
@@ -261,18 +226,14 @@ subroutine loop_with_non_default_integer(s,e,st)
 
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[S_CVT]] : (index) -> i64
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
-  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[S_CVT]] to %[[E_CVT]] step %[[ST_CVT]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i64) {
   do i=s,e,st
     ! CHECK: fir.store %[[I_IV]] to %[[I_REF]] : !fir.ref<i64>
-    ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] : index
-    ! NSW:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT:.*]] overflow<nsw> : index
+    ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] overflow<nsw> : index
     ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[ST_CVT]] : (index) -> i64
     ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i64>
-    ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i64>
-    ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i64
-    ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i64
+    ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i64
     ! CHECK: fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i64
   end do
   ! CHECK: }
diff --git a/flang/test/Lower/do_loop_unstructured.f90 b/flang/test/Lower/do_loop_unstructured.f90
index e1a669e09c9a..d8890b2d0926 100644
--- a/flang/test/Lower/do_loop_unstructured.f90
+++ b/flang/test/Lower/do_loop_unstructured.f90
@@ -1,9 +1,11 @@
 ! RUN: bbc -emit-fir -hlfir=false -o - %s | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -o - %s | FileCheck %s
-! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -flang-experimental-integer-overflow -o - %s | FileCheck %s --check-prefix=NSW
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fwrapv -o - %s | FileCheck %s --check-prefix=NO-NSW
 
 ! Tests for unstructured loops.
 
+! NO-NSW-NOT: overflow<nsw>
+
 ! Test a simple unstructured loop. Test for the existence of,
 ! -> The initialization of the trip-count and loop-variable
 ! -> The branch to the body or the exit inside the header
@@ -39,42 +41,12 @@ end subroutine
 ! CHECK:   fir.store %[[TRIP_VAR_NEXT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR:.*]] = fir.load %[[LOOP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   %[[STEP_ONE_2:.*]] = arith.constant 1 : i32
-! CHECK:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_ONE_2]] : i32
+! CHECK:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_ONE_2]] overflow<nsw> : i32
 ! CHECK:   fir.store %[[LOOP_VAR_NEXT]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER]]
 ! CHECK: ^[[EXIT]]:
 ! CHECK:   return
 
-! NSW-LABEL: simple_unstructured
-! NSW:   %[[TRIP_VAR_REF:.*]] = fir.alloca i32
-! NSW:   %[[LOOP_VAR_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_unstructuredEi"}
-! NSW:   %[[ONE:.*]] = arith.constant 1 : i32
-! NSW:   %[[HUNDRED:.*]] = arith.constant 100 : i32
-! NSW:   %[[STEP_ONE:.*]] = arith.constant 1 : i32
-! NSW:   %[[TMP1:.*]] = arith.subi %[[HUNDRED]], %[[ONE]] : i32
-! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[STEP_ONE]] : i32
-! NSW:   %[[TRIP_COUNT:.*]] = arith.divsi %[[TMP2]], %[[STEP_ONE]] : i32
-! NSW:   fir.store %[[TRIP_COUNT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   fir.store %[[ONE]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER:.*]]
-! NSW: ^[[HEADER]]:
-! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[ZERO:.*]] = arith.constant 0 : i32
-! NSW:   %[[COND:.*]] = arith.cmpi sgt, %[[TRIP_VAR]], %[[ZERO]] : i32
-! NSW:   cf.cond_br %[[COND]], ^[[BODY:.*]], ^[[EXIT:.*]]
-! NSW: ^[[BODY]]:
-! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
-! NSW:   %[[TRIP_VAR_NEXT:.*]] = arith.subi %[[TRIP_VAR]], %[[ONE_1]] : i32
-! NSW:   fir.store %[[TRIP_VAR_NEXT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[LOOP_VAR:.*]] = fir.load %[[LOOP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[STEP_ONE_2:.*]] = arith.constant 1 : i32
-! NSW:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_ONE_2]] overflow<nsw> : i32
-! NSW:   fir.store %[[LOOP_VAR_NEXT]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER]]
-! NSW: ^[[EXIT]]:
-! NSW:   return
-
 ! Test an unstructured loop with a step. Mostly similar to the previous one.
 ! Only difference is a non-unit step.
 subroutine simple_unstructured_with_step()
@@ -108,42 +80,12 @@ end subroutine
 ! CHECK:   fir.store %[[TRIP_VAR_NEXT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR:.*]] = fir.load %[[LOOP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   %[[STEP_2:.*]] = arith.constant 2 : i32
-! CHECK:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_2]] : i32
+! CHECK:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_2]] overflow<nsw> : i32
 ! CHECK:   fir.store %[[LOOP_VAR_NEXT]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER]]
 ! CHECK: ^[[EXIT]]:
 ! CHECK:   return
 
-! NSW-LABEL: simple_unstructured_with_step
-! NSW:   %[[TRIP_VAR_REF:.*]] = fir.alloca i32
-! NSW:   %[[LOOP_VAR_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_unstructured_with_stepEi"}
-! NSW:   %[[ONE:.*]] = arith.constant 1 : i32
-! NSW:   %[[HUNDRED:.*]] = arith.constant 100 : i32
-! NSW:   %[[STEP:.*]] = arith.constant 2 : i32
-! NSW:   %[[TMP1:.*]] = arith.subi %[[HUNDRED]], %[[ONE]] : i32
-! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[STEP]] : i32
-! NSW:   %[[TRIP_COUNT:.*]] = arith.divsi %[[TMP2]], %[[STEP]] : i32
-! NSW:   fir.store %[[TRIP_COUNT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   fir.store %[[ONE]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER:.*]]
-! NSW: ^[[HEADER]]:
-! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[ZERO:.*]] = arith.constant 0 : i32
-! NSW:   %[[COND:.*]] = arith.cmpi sgt, %[[TRIP_VAR]], %[[ZERO]] : i32
-! NSW:   cf.cond_br %[[COND]], ^[[BODY:.*]], ^[[EXIT:.*]]
-! NSW: ^[[BODY]]:
-! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
-! NSW:   %[[TRIP_VAR_NEXT:.*]] = arith.subi %[[TRIP_VAR]], %[[ONE_1]] : i32
-! NSW:   fir.store %[[TRIP_VAR_NEXT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[LOOP_VAR:.*]] = fir.load %[[LOOP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[STEP_2:.*]] = arith.constant 2 : i32
-! NSW:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_2]] overflow<nsw> : i32
-! NSW:   fir.store %[[LOOP_VAR_NEXT]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER]]
-! NSW: ^[[EXIT]]:
-! NSW:   return
-
 ! Test a three nested unstructured loop. Three nesting is the basic case where
 ! we have loops that are neither innermost or outermost.
 subroutine nested_unstructured()
@@ -215,7 +157,7 @@ end subroutine
 ! CHECK:   fir.store %[[TRIP_VAR_K_NEXT]] to %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR_K:.*]] = fir.load %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
 ! CHECK:   %[[K_STEP_2:.*]] = arith.constant 1 : i32
-! CHECK:   %[[LOOP_VAR_K_NEXT:.*]] = arith.addi %[[LOOP_VAR_K]], %[[K_STEP_2]] : i32
+! CHECK:   %[[LOOP_VAR_K_NEXT:.*]] = arith.addi %[[LOOP_VAR_K]], %[[K_STEP_2]] overflow<nsw> : i32
 ! CHECK:   fir.store %[[LOOP_VAR_K_NEXT]] to %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER_K]]
 ! CHECK: ^[[EXIT_K]]:
@@ -225,7 +167,7 @@ end subroutine
 ! CHECK:   fir.store %[[TRIP_VAR_J_NEXT]] to %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR_J:.*]] = fir.load %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
 ! CHECK:   %[[J_STEP_2:.*]] = arith.constant 1 : i32
-! CHECK:   %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %[[J_STEP_2]] : i32
+! CHECK:   %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %[[J_STEP_2]] overflow<nsw> : i32
 ! CHECK:   fir.store %[[LOOP_VAR_J_NEXT]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER_J]]
 ! CHECK: ^[[EXIT_J]]:
@@ -235,96 +177,12 @@ end subroutine
 ! CHECK:   fir.store %[[TRIP_VAR_I_NEXT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR_I:.*]] = fir.load %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[I_STEP_2:.*]] = arith.constant 1 : i32
-! CHECK:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] : i32
+! CHECK:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] overflow<nsw> : i32
 ! CHECK:   fir.store %[[LOOP_VAR_I_NEXT]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER_I]]
 ! CHECK: ^[[EXIT_I]]:
 ! CHECK:   return
 
-! NSW-LABEL: nested_unstructured
-! NSW:   %[[TRIP_VAR_K_REF:.*]] = fir.alloca i32
-! NSW:   %[[TRIP_VAR_J_REF:.*]] = fir.alloca i32
-! NSW:   %[[TRIP_VAR_I_REF:.*]] = fir.alloca i32
-! NSW:   %[[LOOP_VAR_I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_unstructuredEi"}
-! NSW:   %[[LOOP_VAR_J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_unstructuredEj"}
-! NSW:   %[[LOOP_VAR_K_REF:.*]] = fir.alloca i32 {bindc_name = "k", uniq_name = "_QFnested_unstructuredEk"}
-! NSW:   %[[I_START:.*]] = arith.constant 1 : i32
-! NSW:   %[[I_END:.*]] = arith.constant 100 : i32
-! NSW:   %[[I_STEP:.*]] = arith.constant 1 : i32
-! NSW:   %[[TMP1:.*]] = arith.subi %[[I_END]], %[[I_START]] : i32
-! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[I_STEP]] : i32
-! NSW:   %[[TRIP_COUNT_I:.*]] = arith.divsi %[[TMP2]], %[[I_STEP]] : i32
-! NSW:   fir.store %[[TRIP_COUNT_I]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   fir.store %[[I_START]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER_I:.*]]
-! NSW: ^[[HEADER_I]]:
-! NSW:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[ZERO_1:.*]] = arith.constant 0 : i32
-! NSW:   %[[COND_I:.*]] = arith.cmpi sgt, %[[TRIP_VAR_I]], %[[ZERO_1]] : i32
-! NSW:   cf.cond_br %[[COND_I]], ^[[BODY_I:.*]], ^[[EXIT_I:.*]]
-! NSW: ^[[BODY_I]]:
-! NSW:   %[[J_START:.*]] = arith.constant 1 : i32
-! NSW:   %[[J_END:.*]] = arith.constant 200 : i32
-! NSW:   %[[J_STEP:.*]] = arith.constant 1 : i32
-! NSW:   %[[TMP3:.*]] = arith.subi %[[J_END]], %[[J_START]] : i32
-! NSW:   %[[TMP4:.*]] = arith.addi %[[TMP3]], %[[J_STEP]] : i32
-! NSW:   %[[TRIP_COUNT_J:.*]] = arith.divsi %[[TMP4]], %[[J_STEP]] : i32
-! NSW:   fir.store %[[TRIP_COUNT_J]] to %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:   fir.store %[[J_START]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER_J:.*]]
-! NSW: ^[[HEADER_J]]:
-! NSW:   %[[TRIP_VAR_J:.*]] = fir.load %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:   %[[ZERO_2:.*]] = arith.constant 0 : i32
-! NSW:   %[[COND_J:.*]] = arith.cmpi sgt, %[[TRIP_VAR_J]], %[[ZERO_2]] : i32
-! NSW:   cf.cond_br %[[COND_J]], ^[[BODY_J:.*]], ^[[EXIT_J:.*]]
-! NSW: ^[[BODY_J]]:
-! NSW:   %[[K_START:.*]] = arith.constant 1 : i32
-! NSW:   %[[K_END:.*]] = arith.constant 300 : i32
-! NSW:   %[[K_STEP:.*]] = arith.constant 1 : i32
-! NSW:   %[[TMP3:.*]] = arith.subi %[[K_END]], %[[K_START]] : i32
-! NSW:   %[[TMP4:.*]] = arith.addi %[[TMP3]], %[[K_STEP]] : i32
-! NSW:   %[[TRIP_COUNT_K:.*]] = arith.divsi %[[TMP4]], %[[K_STEP]] : i32
-! NSW:   fir.store %[[TRIP_COUNT_K]] to %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
-! NSW:   fir.store %[[K_START]] to %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER_K:.*]]
-! NSW: ^[[HEADER_K]]:
-! NSW:   %[[TRIP_VAR_K:.*]] = fir.load %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
-! NSW:   %[[ZERO_2:.*]] = arith.constant 0 : i32
-! NSW:   %[[COND_K:.*]] = arith.cmpi sgt, %[[TRIP_VAR_K]], %[[ZERO_2]] : i32
-! NSW:   cf.cond_br %[[COND_K]], ^[[BODY_K:.*]], ^[[EXIT_K:.*]]
-! NSW: ^[[BODY_K]]:
-! NSW:   %[[TRIP_VAR_K:.*]] = fir.load %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
-! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
-! NSW:   %[[TRIP_VAR_K_NEXT:.*]] = arith.subi %[[TRIP_VAR_K]], %[[ONE_1]] : i32
-! NSW:   fir.store %[[TRIP_VAR_K_NEXT]] to %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
-! NSW:   %[[LOOP_VAR_K:.*]] = fir.load %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
-! NSW:   %[[K_STEP_2:.*]] = arith.constant 1 : i32
-! NSW:   %[[LOOP_VAR_K_NEXT:.*]] = arith.addi %[[LOOP_VAR_K]], %[[K_STEP_2]] overflow<nsw> : i32
-! NSW:   fir.store %[[LOOP_VAR_K_NEXT]] to %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER_K]]
-! NSW: ^[[EXIT_K]]:
-! NSW:   %[[TRIP_VAR_J:.*]] = fir.load %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
-! NSW:   %[[TRIP_VAR_J_NEXT:.*]] = arith.subi %[[TRIP_VAR_J]], %[[ONE_1]] : i32
-! NSW:   fir.store %[[TRIP_VAR_J_NEXT]] to %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:   %[[LOOP_VAR_J:.*]] = fir.load %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:   %[[J_STEP_2:.*]] = arith.constant 1 : i32
-! NSW:   %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %[[J_STEP_2]] overflow<nsw> : i32
-! NSW:   fir.store %[[LOOP_VAR_J_NEXT]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER_J]]
-! NSW: ^[[EXIT_J]]:
-! NSW:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
-! NSW:   %[[TRIP_VAR_I_NEXT:.*]] = arith.subi %[[TRIP_VAR_I]], %[[ONE_1]] : i32
-! NSW:   fir.store %[[TRIP_VAR_I_NEXT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[LOOP_VAR_I:.*]] = fir.load %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[I_STEP_2:.*]] = arith.constant 1 : i32
-! NSW:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] overflow<nsw> : i32
-! NSW:   fir.store %[[LOOP_VAR_I_NEXT]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER_I]]
-! NSW: ^[[EXIT_I]]:
-! NSW:   return
-
 ! Test the existence of a structured loop inside an unstructured loop.
 ! Only minimal checks are inserted for the structured loop.
 subroutine nested_structured_in_unstructured()
@@ -359,9 +217,9 @@ end subroutine
 ! CHECK-SAME: %{{.*}} to %{{.*}} step %[[ST:[^ ]*]]
 ! CHECK-SAME: iter_args(%[[J_IV:.*]] = %{{.*}}) -> (index, i32) {
 ! CHECK:     fir.store %[[J_IV]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! CHECK:     %[[J_INDEX_NEXT:.*]] = arith.addi %[[J_INDEX]], %[[ST]] : index
+! CHECK:     %[[J_INDEX_NEXT:.*]] = arith.addi %[[J_INDEX]], %[[ST]] overflow<nsw> : index
 ! CHECK:     %[[LOOP_VAR_J:.*]] = fir.load %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! CHECK:     %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %{{[^ ]*}} : i32
+! CHECK:     %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %{{[^ ]*}} overflow<nsw> : i32
 ! CHECK:   }
 ! CHECK:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[C1_3:.*]] = arith.constant 1 : i32
@@ -369,47 +227,8 @@ end subroutine
 ! CHECK:   fir.store %[[TRIP_VAR_I_NEXT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR_I:.*]] = fir.load %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[I_STEP_2:.*]] = arith.constant 1 : i32
-! CHECK:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] : i32
+! CHECK:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] overflow<nsw> : i32
 ! CHECK:   fir.store %[[LOOP_VAR_I_NEXT]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER]]
 ! CHECK: ^[[EXIT]]:
 ! CHECK:   return
-
-! NSW-LABEL: nested_structured_in_unstructured
-! NSW:   %[[TRIP_VAR_I_REF:.*]] = fir.alloca i32
-! NSW:   %[[LOOP_VAR_I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_structured_in_unstructuredEi"}
-! NSW:   %[[LOOP_VAR_J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_structured_in_unstructuredEj"}
-! NSW:   %[[I_START:.*]] = arith.constant 1 : i32
-! NSW:   %[[I_END:.*]] = arith.constant 100 : i32
-! NSW:   %[[I_STEP:.*]] = arith.constant 1 : i32
-! NSW:   %[[TMP1:.*]] = arith.subi %[[I_END]], %[[I_START]] : i32
-! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[I_STEP]] : i32
-! NSW:   %[[TRIP_COUNT:.*]] = arith.divsi %[[TMP2]], %[[I_STEP]] : i32
-! NSW:   fir.store %[[TRIP_COUNT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   fir.store %[[I_START]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER:.*]]
-! NSW: ^[[HEADER]]:
-! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[ZERO:.*]] = arith.constant 0 : i32
-! NSW:   %[[COND:.*]] = arith.cmpi sgt, %[[TRIP_VAR]], %[[ZERO]] : i32
-! NSW:   cf.cond_br %[[COND]], ^[[BODY:.*]], ^[[EXIT:.*]]
-! NSW: ^[[BODY]]:
-! NSW:   %{{.*}} = fir.do_loop %[[J_INDEX:[^ ]*]] =
-! NSW-SAME: %{{.*}} to %{{.*}} step %[[ST:[^ ]*]]
-! NSW-SAME: iter_args(%[[J_IV:.*]] = %{{.*}}) -> (index, i32) {
-! NSW:     fir.store %[[J_IV]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:     %[[J_INDEX_NEXT:.*]] = arith.addi %[[J_INDEX]], %[[ST]] overflow<nsw> : index
-! NSW:     %[[LOOP_VAR_J:.*]] = fir.load %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:     %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %{{[^ ]*}} overflow<nsw> : i32
-! NSW:   }
-! NSW:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[C1_3:.*]] = arith.constant 1 : i32
-! NSW:   %[[TRIP_VAR_I_NEXT:.*]] = arith.subi %[[TRIP_VAR_I]], %[[C1_3]] : i32
-! NSW:   fir.store %[[TRIP_VAR_I_NEXT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[LOOP_VAR_I:.*]] = fir.load %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[I_STEP_2:.*]] = arith.constant 1 : i32
-! NSW:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] overflow<nsw> : i32
-! NSW:   fir.store %[[LOOP_VAR_I_NEXT]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER]]
-! NSW: ^[[EXIT]]:
-! NSW:   return
diff --git a/flang/test/Lower/goto-do-body.f90 b/flang/test/Lower/goto-do-body.f90
index 910e55f1839f..89e4a7a64a87 100644
--- a/flang/test/Lower/goto-do-body.f90
+++ b/flang/test/Lower/goto-do-body.f90
@@ -48,7 +48,7 @@ subroutine sub1()
 ! CHECK:    fir.store %[[TMP8]] to %[[TRIP]] : !fir.ref<i32>
 ! CHECK:    %[[TMP9:.*]] = fir.load %[[I]] : !fir.ref<i32>
 ! CHECK:    %[[C1_4:.*]] = arith.constant 1 : i32
-! CHECK:    %[[TMP10:.*]] = arith.addi %[[TMP9]], %[[C1_4]] : i32
+! CHECK:    %[[TMP10:.*]] = arith.addi %[[TMP9]], %[[C1_4]] overflow<nsw> : i32
 ! CHECK:    fir.store %[[TMP10]] to %[[I]] : !fir.ref<i32>
 ! CHECK:    cf.br ^[[HEADER]]
   end do
@@ -115,7 +115,7 @@ subroutine sub2()
 ! CHECK:    fir.store %[[TMP10]] to %[[TRIP]] : !fir.ref<i32>
 ! CHECK:    %[[TMP11:.*]] = fir.load %[[I]] : !fir.ref<i32>
 ! CHECK:    %[[STEP_VAL:.*]] = fir.load %[[STEP_VAR]] : !fir.ref<i32>
-! CHECK:    %[[TMP12:.*]] = arith.addi %[[TMP11]], %[[STEP_VAL]] : i32
+! CHECK:    %[[TMP12:.*]] = arith.addi %[[TMP11]], %[[STEP_VAL]] overflow<nsw> : i32
 ! CHECK:    fir.store %[[TMP12]] to %[[I]] : !fir.ref<i32>
 ! CHECK:    cf.br ^[[HEADER]]
   end do
diff --git a/flang/test/Lower/host-associated.f90 b/flang/test/Lower/host-associated.f90
index 9b4269df7bfc..33acdff1bb74 100644
--- a/flang/test/Lower/host-associated.f90
+++ b/flang/test/Lower/host-associated.f90
@@ -1,5 +1,5 @@
 ! Test internal procedure host association lowering.
-! RUN: bbc -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s
 
 ! -----------------------------------------------------------------------------
 !     Test non character intrinsic scalars
diff --git a/flang/test/Lower/infinite_loop.f90 b/flang/test/Lower/infinite_loop.f90
index 6942dda8d7a2..de0bee779c5b 100644
--- a/flang/test/Lower/infinite_loop.f90
+++ b/flang/test/Lower/infinite_loop.f90
@@ -1,9 +1,11 @@
 ! RUN: bbc -emit-fir -hlfir=false -o - %s | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -o - %s | FileCheck %s
-! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -flang-experimental-integer-overflow -o - %s | FileCheck %s --check-prefix=NSW
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fwrapv -o - %s | FileCheck %s --check-prefix=NO-NSW
 
 ! Tests for infinite loop.
 
+! NO-NSW-NOT: overflow<nsw>
+
 subroutine empty_infinite()
   do
   end do
@@ -96,10 +98,10 @@ end subroutine
 ! CHECK-SAME: %[[C1_INDEX]] to %[[C10_INDEX]] step %[[C1_1]]
 ! CHECK-SAME: iter_args(%[[J_IV:.*]] = %[[J_LB]]) -> (index, i32) {
 ! CHECK:    fir.store %[[J_IV]] to %[[J_REF]] : !fir.ref<i32>
-! CHECK:    %[[J_NEXT:.*]] = arith.addi %[[J]], %[[C1_1]] : index
+! CHECK:    %[[J_NEXT:.*]] = arith.addi %[[J]], %[[C1_1]] overflow<nsw> : index
 ! CHECK:    %[[J_STEPCAST:.*]] = fir.convert %[[C1_1]] : (index) -> i32
 ! CHECK:    %[[J_IVLOAD:.*]] = fir.load %[[J_REF]] : !fir.ref<i32>
-! CHECK:    %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] : i32
+! CHECK:    %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] overflow<nsw> : i32
 ! CHECK:    fir.result %[[J_NEXT]], %[[J_IVINC]] : index, i32
 ! CHECK:  }
 ! CHECK:  fir.store %[[J_FINAL]]#1 to %[[J_REF]] : !fir.ref<i32>
@@ -107,39 +109,6 @@ end subroutine
 ! CHECK: ^[[RETURN]]:
 ! CHECK:   return
 
-! NSW-LABEL: structured_loop_in_infinite
-! NSW-SAME: %[[I_REF:.*]]: !fir.ref<i32>
-! NSW:  %[[J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFstructured_loop_in_infiniteEj"}
-! NSW:  cf.br ^[[BODY1:.*]]
-! NSW: ^[[BODY1]]:
-! NSW:  %[[I:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-! NSW:  %[[C100:.*]] = arith.constant 100 : i32
-! NSW:  %[[COND:.*]] = arith.cmpi sgt, %[[I]], %[[C100]] : i32
-! NSW:  cf.cond_br %[[COND]], ^[[EXIT:.*]], ^[[BODY2:.*]]
-! NSW: ^[[EXIT]]:
-! NSW:  cf.br ^[[RETURN:.*]]
-! NSW: ^[[BODY2:.*]]:
-! NSW:  %[[C1:.*]] = arith.constant 1 : i32
-! NSW:  %[[C1_INDEX:.*]] = fir.convert %[[C1]] : (i32) -> index
-! NSW:  %[[C10:.*]] = arith.constant 10 : i32
-! NSW:  %[[C10_INDEX:.*]] = fir.convert %[[C10]] : (i32) -> index
-! NSW:  %[[C1_1:.*]] = arith.constant 1 : index
-! NSW:  %[[J_LB:.*]] = fir.convert %[[C1_INDEX]] : (index) -> i32
-! NSW:  %[[J_FINAL:.*]]:2 = fir.do_loop %[[J:[^ ]*]] =
-! NSW-SAME: %[[C1_INDEX]] to %[[C10_INDEX]] step %[[C1_1]]
-! NSW-SAME: iter_args(%[[J_IV:.*]] = %[[J_LB]]) -> (index, i32) {
-! NSW:    fir.store %[[J_IV]] to %[[J_REF]] : !fir.ref<i32>
-! NSW:    %[[J_NEXT:.*]] = arith.addi %[[J]], %[[C1_1]] overflow<nsw> : index
-! NSW:    %[[J_STEPCAST:.*]] = fir.convert %[[C1_1]] : (index) -> i32
-! NSW:    %[[J_IVLOAD:.*]] = fir.load %[[J_REF]] : !fir.ref<i32>
-! NSW:    %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] overflow<nsw> : i32
-! NSW:    fir.result %[[J_NEXT]], %[[J_IVINC]] : index, i32
-! NSW:  }
-! NSW:  fir.store %[[J_FINAL]]#1 to %[[J_REF]] : !fir.ref<i32>
-! NSW:  cf.br ^[[BODY1]]
-! NSW: ^[[RETURN]]:
-! NSW:   return
-
 subroutine empty_infinite_in_while(i)
   integer :: i
   do while (i .gt. 50)
diff --git a/flang/test/Lower/io-implied-do-fixes.f90 b/flang/test/Lower/io-implied-do-fixes.f90
index a6c115fa80de..cd4fd43e0519 100644
--- a/flang/test/Lower/io-implied-do-fixes.f90
+++ b/flang/test/Lower/io-implied-do-fixes.f90
@@ -1,30 +1,20 @@
 ! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
-! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false -integer-overflow %s -o - | FileCheck %s --check-prefix=NSW
+! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false -fwrapv %s -o - | FileCheck %s --check-prefix=NO-NSW
 ! UNSUPPORTED: system-windows
 
+! NO-NSW-NOT: overflow<nsw>
+
 ! CHECK-LABEL: func @_QPido1
 ! CHECK: %[[J_REF_ADDR:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFido1Eiptr.addr"}
 ! CHECK: %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.ptr<i32>>
 ! CHECK: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
 ! CHECK:   %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
 ! CHECK:   fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.ptr<i32>
-! CHECK:   %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} : index
+! CHECK:   %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
 ! CHECK:   fir.result %[[J_VAL_NEXT]] : index
 ! CHECK: }
 ! CHECK: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
 ! CHECK: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.ptr<i32>
-
-! NSW-LABEL: func @_QPido1
-! NSW: %[[J_REF_ADDR:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFido1Eiptr.addr"}
-! NSW: %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.ptr<i32>>
-! NSW: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
-! NSW:   %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
-! NSW:   fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.ptr<i32>
-! NSW:   %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
-! NSW:   fir.result %[[J_VAL_NEXT]] : index
-! NSW: }
-! NSW: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
-! NSW: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.ptr<i32>
 subroutine ido1
   integer, pointer :: iptr
   integer, target :: itgt
@@ -38,23 +28,11 @@ end subroutine
 ! CHECK: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
 ! CHECK: %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
 ! CHECK: fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
-! CHECK: %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} : index
+! CHECK: %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
 ! CHECK: fir.result %[[J_VAL_NEXT]] : index
 ! CHECK: }
 ! CHECK: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
 ! CHECK: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32>
-
-! NSW-LABEL: func @_QPido2
-! NSW: %[[J_REF_ADDR:.*]] = fir.alloca !fir.heap<i32> {uniq_name = "_QFido2Eiptr.addr"}
-! NSW: %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.heap<i32>>
-! NSW: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
-! NSW: %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
-! NSW: fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
-! NSW: %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
-! NSW: fir.result %[[J_VAL_NEXT]] : index
-! NSW: }
-! NSW: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
-! NSW: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32>
 subroutine ido2
   integer, allocatable :: iptr
   allocate(iptr)
@@ -69,27 +47,12 @@ end subroutine
 ! CHECK:    fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
 ! CHECK:    %[[RES:.*]] = fir.if %[[OK]] -> (i1) {
 ! CHECK:    }
-! CHECK:   %[[J_VAL_INC:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} : index
+! CHECK:   %[[J_VAL_INC:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
 ! CHECK:   %[[J_VAL_NEXT:.*]] = arith.select %[[RES]], %[[J_VAL_INC]], %[[J_VAL]] : index
 ! CHECK:   fir.result %[[J_VAL_NEXT]], %[[RES]] : index, i1
 ! CHECK:  }
 ! CHECK:  %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]]#0 : (index) -> i32
 ! CHECK:  fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32
-
-! NSW-LABEL: func @_QPido3
-! NSW:  %[[J_REF_ADDR:.*]] = fir.alloca !fir.heap<i32> {uniq_name = "_QFido3Ej.addr"}
-! NSW:  %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.heap<i32>>
-! NSW:  %[[J_VAL_FINAL:.*]]:2 = fir.iterate_while (%[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}}) and (%[[OK:.*]] = {{.*}}) -> (index, i1) {
-! NSW:    %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
-! NSW:    fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
-! NSW:    %[[RES:.*]] = fir.if %[[OK]] -> (i1) {
-! NSW:    }
-! NSW:   %[[J_VAL_INC:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
-! NSW:   %[[J_VAL_NEXT:.*]] = arith.select %[[RES]], %[[J_VAL_INC]], %[[J_VAL]] : index
-! NSW:   fir.result %[[J_VAL_NEXT]], %[[RES]] : index, i1
-! NSW:  }
-! NSW:  %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]]#0 : (index) -> i32
-! NSW:  fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32
 subroutine ido3
   integer, allocatable :: j
   allocate(j)
diff --git a/flang/test/Lower/loops2.f90 b/flang/test/Lower/loops2.f90
index 0a587234a991..60a6bf6c5311 100644
--- a/flang/test/Lower/loops2.f90
+++ b/flang/test/Lower/loops2.f90
@@ -107,7 +107,7 @@ contains
 ! CHECK:       ^bb4:
 ! CHECK:         %[[VAL_20:.*]] = fir.load %[[VAL_3]] : !fir.ptr<i32>
 ! CHECK:         %[[VAL_21:.*]] = arith.constant 1 : i32
-! CHECK:         %[[VAL_22:.*]] = arith.addi %[[VAL_20]], %[[VAL_21]] : i32
+! CHECK:         %[[VAL_22:.*]] = arith.addi %[[VAL_20]], %[[VAL_21]] overflow<nsw> : i32
 ! CHECK:         fir.store %[[VAL_22]] to %[[VAL_3]] : !fir.ptr<i32>
 ! CHECK:         br ^bb1
 ! CHECK:       ^bb5:
diff --git a/flang/test/Lower/mixed_loops.f90 b/flang/test/Lower/mixed_loops.f90
index 1aa0225129be..991fd7aa82bb 100644
--- a/flang/test/Lower/mixed_loops.f90
+++ b/flang/test/Lower/mixed_loops.f90
@@ -53,7 +53,7 @@ subroutine while_inside_do_loop
   ! CHECK: fir.store %[[TDEC]] to %[[T_REF]]
   ! CHECK: %[[I3:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
   ! CHECK: %[[C1_2:.*]] = arith.constant 1 : i32
-  ! CHECK: %[[IINC:.*]] = arith.addi %[[I3]], %[[C1_2]] : i32
+  ! CHECK: %[[IINC:.*]] = arith.addi %[[I3]], %[[C1_2]] overflow<nsw> : i32
   ! CHECK: fir.store %[[IINC]] to %[[I_REF]] : !fir.ref<i32>
   ! CHECK: br ^[[HDR1]]
   end do
@@ -100,10 +100,10 @@ subroutine do_inside_while_loop
         ! CHECK-DAG: %[[C2:.*]] = arith.constant 2 : i32
         ! CHECK: %[[JINC:.*]] = arith.muli %[[C2]], %[[J2]] : i32
         ! CHECK: fir.store %[[JINC]] to %[[J_REF]] : !fir.ref<i32>
-        ! CHECK: %[[IINC:.*]] = arith.addi %[[IDX]], %[[C1]] : index
+        ! CHECK: %[[IINC:.*]] = arith.addi %[[IDX]], %[[C1]] overflow<nsw> : index
         ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[C1]] : (index) -> i32
         ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-        ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
+        ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i32
         ! CHECK: fir.result %[[IINC]], %[[I_IVINC]] : index, i32
       do i=8,13
         j=j*2
diff --git a/flang/test/Lower/vector-subscript-io.f90 b/flang/test/Lower/vector-subscript-io.f90
index 129e3ee1206c..372130fd0990 100644
--- a/flang/test/Lower/vector-subscript-io.f90
+++ b/flang/test/Lower/vector-subscript-io.f90
@@ -30,7 +30,7 @@ subroutine simple(x, y)
 ! CHECK:   %[[VAL_19:.*]] = fir.array_coor %[[VAL_20]](%[[VAL_10]]) {{\[}}%[[VAL_11]]] %[[VAL_18]] : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, !fir.slice<1>, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_21:.*]] = fir.convert %[[VAL_19]] : (!fir.ref<i32>) -> !fir.ref<i64>
 ! CHECK:   %[[VAL_22:.*]] = fir.call @_FortranAioInputInteger(%[[VAL_9]], %[[VAL_21]], %[[VAL_3]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
-! CHECK:   %[[VAL_23:.*]] = arith.addi %[[VAL_12]], %[[VAL_6]] : index
+! CHECK:   %[[VAL_23:.*]] = arith.addi %[[VAL_12]], %[[VAL_6]] overflow<nsw> : index
 ! CHECK:   %[[VAL_24:.*]] = arith.subi %[[VAL_13]], %[[VAL_6]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_23]], %[[VAL_24]] : index, index)
 ! CHECK: ^bb3:
@@ -79,7 +79,7 @@ subroutine only_once(x)
 ! CHECK:   %[[VAL_49:.*]] = fir.convert %[[VAL_48]] : (i32) -> index
 ! CHECK:   %[[VAL_50:.*]] = fir.array_coor %[[VAL_51]] {{\[}}%[[VAL_42]]] %[[VAL_46]], %[[VAL_49]] : (!fir.box<!fir.array<?x?xf32>>, !fir.slice<2>, index, index) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_52:.*]] = fir.call @_FortranAioInputReal32(%[[VAL_34]], %[[VAL_50]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
-! CHECK:   %[[VAL_53:.*]] = arith.addi %[[VAL_43]], %[[VAL_30]] : index
+! CHECK:   %[[VAL_53:.*]] = arith.addi %[[VAL_43]], %[[VAL_30]] overflow<nsw> : index
 ! CHECK:   %[[VAL_54:.*]] = arith.subi %[[VAL_44]], %[[VAL_30]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_53]], %[[VAL_54]] : index, index)
 ! CHECK: ^bb3:
@@ -122,7 +122,7 @@ subroutine with_assumed_shapes(x, y)
 ! CHECK:   %[[VAL_77:.*]] = fir.array_coor %[[VAL_78]] {{\[}}%[[VAL_70]]] %[[VAL_76]] : (!fir.box<!fir.array<?xi32>>, !fir.slice<1>, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_79:.*]] = fir.convert %[[VAL_77]] : (!fir.ref<i32>) -> !fir.ref<i64>
 ! CHECK:   %[[VAL_80:.*]] = fir.call @_FortranAioInputInteger(%[[VAL_67]], %[[VAL_79]], %[[VAL_62]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
-! CHECK:   %[[VAL_81:.*]] = arith.addi %[[VAL_71]], %[[VAL_64]] : index
+! CHECK:   %[[VAL_81:.*]] = arith.addi %[[VAL_71]], %[[VAL_64]] overflow<nsw> : index
 ! CHECK:   %[[VAL_82:.*]] = arith.subi %[[VAL_72]], %[[VAL_64]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_81]], %[[VAL_82]] : index, index)
 ! CHECK: ^bb3:
@@ -162,7 +162,7 @@ subroutine lower_bounds(x, y)
 ! CHECK:   %[[VAL_107:.*]] = fir.array_coor %[[VAL_108]](%[[VAL_97]]) {{\[}}%[[VAL_99]]] %[[VAL_91]], %[[VAL_106]] : (!fir.ref<!fir.array<4x6xi32>>, !fir.shapeshift<2>, !fir.slice<2>, index, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_109:.*]] = fir.convert %[[VAL_107]] : (!fir.ref<i32>) -> !fir.ref<i64>
 ! CHECK:   %[[VAL_110:.*]] = fir.call @_FortranAioInputInteger(%[[VAL_96]], %[[VAL_109]], %[[VAL_90]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
-! CHECK:   %[[VAL_111:.*]] = arith.addi %[[VAL_100]], %[[VAL_93]] : index
+! CHECK:   %[[VAL_111:.*]] = arith.addi %[[VAL_100]], %[[VAL_93]] overflow<nsw> : index
 ! CHECK:   %[[VAL_112:.*]] = arith.subi %[[VAL_101]], %[[VAL_93]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_111]], %[[VAL_112]] : index, index)
 ! CHECK: ^bb3:
@@ -202,11 +202,11 @@ subroutine two_vectors(x, y1, y2)
 ! CHECK:   %[[VAL_138:.*]] = fir.convert %[[VAL_137]] : (i32) -> index
 ! CHECK:   %[[VAL_139:.*]] = fir.array_coor %[[VAL_140]](%[[VAL_123]]) {{\[}}%[[VAL_124]]] %[[VAL_134]], %[[VAL_138]] : (!fir.ref<!fir.array<4x4xf32>>, !fir.shape<2>, !fir.slice<2>, index, index) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_141:.*]] = fir.call @_FortranAioInputReal32(%[[VAL_122]], %[[VAL_139]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
-! CHECK:   %[[VAL_142:.*]] = arith.addi %[[VAL_128]], %[[VAL_119]] : index
+! CHECK:   %[[VAL_142:.*]] = arith.addi %[[VAL_128]], %[[VAL_119]] overflow<nsw> : index
 ! CHECK:   %[[VAL_143:.*]] = arith.subi %[[VAL_129]], %[[VAL_119]] : index
 ! CHECK:   cf.br ^bb2(%[[VAL_142]], %[[VAL_143]] : index, index)
 ! CHECK: ^bb4:
-! CHECK:   %[[VAL_144:.*]] = arith.addi %[[VAL_125]], %[[VAL_119]] : index
+! CHECK:   %[[VAL_144:.*]] = arith.addi %[[VAL_125]], %[[VAL_119]] overflow<nsw> : index
 ! CHECK:   %[[VAL_145:.*]] = arith.subi %[[VAL_126]], %[[VAL_119]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_144]], %[[VAL_145]] : index, index)
 ! CHECK: ^bb5:
@@ -245,11 +245,11 @@ subroutine triplets_and_vector(x, y)
 ! CHECK:   %[[VAL_169:.*]] = fir.array_coor %[[VAL_170]](%[[VAL_157]]) {{\[}}%[[VAL_158]]] %[[VAL_162]], %[[VAL_168]] : (!fir.ref<!fir.array<4x4xcomplex<f32>>>, !fir.shape<2>, !fir.slice<2>, index, index) -> !fir.ref<complex<f32>>
 ! CHECK:   %[[VAL_171:.*]] = fir.convert %[[VAL_169]] : (!fir.ref<complex<f32>>) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_172:.*]] = fir.call @_FortranAioInputComplex32(%[[VAL_156]], %[[VAL_171]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
-! CHECK:   %[[VAL_173:.*]] = arith.addi %[[VAL_162]], %[[VAL_153]] : index
+! CHECK:   %[[VAL_173:.*]] = arith.addi %[[VAL_162]], %[[VAL_153]] overflow<nsw> : index
 ! CHECK:   %[[VAL_174:.*]] = arith.subi %[[VAL_163]], %[[VAL_153]] : index
 ! CHECK:   cf.br ^bb2(%[[VAL_173]], %[[VAL_174]] : index, index)
 ! CHECK: ^bb4:
-! CHECK:   %[[VAL_175:.*]] = arith.addi %[[VAL_159]], %[[VAL_153]] : index
+! CHECK:   %[[VAL_175:.*]] = arith.addi %[[VAL_159]], %[[VAL_153]] overflow<nsw> : index
 ! CHECK:   %[[VAL_176:.*]] = arith.subi %[[VAL_160]], %[[VAL_153]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_175]], %[[VAL_176]] : index, index)
 ! CHECK: ^bb5:
@@ -287,7 +287,7 @@ subroutine simple_char(x, y)
 ! CHECK:   %[[VAL_200:.*]] = fir.convert %[[VAL_199]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
 ! CHECK:   %[[VAL_201:.*]] = fir.convert %[[VAL_184]]#1 : (index) -> i64
 ! CHECK:   %[[VAL_202:.*]] = fir.call @_FortranAioInputAscii(%[[VAL_189]], %[[VAL_200]], %[[VAL_201]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64) -> i1
-! CHECK:   %[[VAL_203:.*]] = arith.addi %[[VAL_192]], %[[VAL_183]] : index
+! CHECK:   %[[VAL_203:.*]] = arith.addi %[[VAL_192]], %[[VAL_183]] overflow<nsw> : index
 ! CHECK:   %[[VAL_204:.*]] = arith.subi %[[VAL_193]], %[[VAL_183]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_203]], %[[VAL_204]] : index, index)
 ! CHECK: ^bb3:
@@ -333,7 +333,7 @@ subroutine substring(x, y, i, j)
 ! CHECK:   %[[VAL_238:.*]] = fir.convert %[[VAL_233]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
 ! CHECK:   %[[VAL_239:.*]] = fir.convert %[[VAL_237]] : (index) -> i64
 ! CHECK:   %[[VAL_240:.*]] = fir.call @_FortranAioInputAscii(%[[VAL_213]], %[[VAL_238]], %[[VAL_239]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64) -> i1
-! CHECK:   %[[VAL_241:.*]] = arith.addi %[[VAL_221]], %[[VAL_210]] : index
+! CHECK:   %[[VAL_241:.*]] = arith.addi %[[VAL_221]], %[[VAL_210]] overflow<nsw> : index
 ! CHECK:   %[[VAL_242:.*]] = arith.subi %[[VAL_222]], %[[VAL_210]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_241]], %[[VAL_242]] : index, index)
 ! CHECK: ^bb3:
@@ -366,7 +366,7 @@ subroutine complex_part(z, y)
 ! CHECK:   %[[VAL_260:.*]] = fir.convert %[[VAL_259]] : (i32) -> index
 ! CHECK:   %[[VAL_261:.*]] = fir.array_coor %[[VAL_262]] {{\[}}%[[VAL_254]]] %[[VAL_260]] : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.slice<1>, index) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_263:.*]] = fir.call @_FortranAioInputReal32(%[[VAL_251]], %[[VAL_261]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
-! CHECK:   %[[VAL_264:.*]] = arith.addi %[[VAL_255]], %[[VAL_248]] : index
+! CHECK:   %[[VAL_264:.*]] = arith.addi %[[VAL_255]], %[[VAL_248]] overflow<nsw> : index
 ! CHECK:   %[[VAL_265:.*]] = arith.subi %[[VAL_256]], %[[VAL_248]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_264]], %[[VAL_265]] : index, index)
 ! CHECK: ^bb3:
@@ -414,7 +414,7 @@ subroutine simple_derived(x, y)
 ! CHECK:   %[[VAL_288:.*]] = fir.embox %[[VAL_286]] : (!fir.ref<!fir.type<_QMderived_typesTt{i:i32,c:!fir.char<1,2>}>>) -> !fir.box<!fir.type<_QMderived_typesTt{i:i32,c:!fir.char<1,2>}>>
 ! CHECK:   %[[VAL_289:.*]] = fir.convert %[[VAL_288]] : (!fir.box<!fir.type<_QMderived_typesTt{i:i32,c:!fir.char<1,2>}>>) -> !fir.box<none>
 ! CHECK:   %[[VAL_290:.*]] = fir.call @_FortranAioInputDerivedType(%[[VAL_276]], %[[VAL_289]], {{.*}}) {{.*}}: (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
-! CHECK:   %[[VAL_291:.*]] = arith.addi %[[VAL_279]], %[[VAL_273]] : index
+! CHECK:   %[[VAL_291:.*]] = arith.addi %[[VAL_279]], %[[VAL_273]] overflow<nsw> : index
 ! CHECK:   %[[VAL_292:.*]] = arith.subi %[[VAL_280]], %[[VAL_273]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_291]], %[[VAL_292]] : index, index)
 ! CHECK: ^bb3:
@@ -463,11 +463,11 @@ subroutine with_path(b, i)
 ! CHECK:   %[[VAL_325:.*]] = fir.array_coor %[[VAL_326:.*]](%[[VAL_313]]) {{\[}}%[[VAL_315]]] %[[VAL_301]], %[[VAL_324]], %[[VAL_316]] : (!fir.box<!fir.array<?x?x?x!fir.type<_QMderived_typesTt2{a:!fir.array<5x5x!fir.type<_QMderived_typesTt{i:i32,c:!fir.char<1,2>}>>}>>>, !fir.shift<3>, !fir.slice<3>, index, index, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_327:.*]] = fir.convert %[[VAL_325]] : (!fir.ref<i32>) -> !fir.ref<i64>
 ! CHECK:   %[[VAL_328:.*]] = fir.call @_FortranAioInputInteger(%[[VAL_308]], %[[VAL_327]], %[[VAL_302]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
-! CHECK:   %[[VAL_329:.*]] = arith.addi %[[VAL_319]], %[[VAL_305]] : index
+! CHECK:   %[[VAL_329:.*]] = arith.addi %[[VAL_319]], %[[VAL_305]] overflow<nsw> : index
 ! CHECK:   %[[VAL_330:.*]] = arith.subi %[[VAL_320]], %[[VAL_305]] : index
 ! CHECK:   cf.br ^bb2(%[[VAL_329]], %[[VAL_330]] : index, index)
 ! CHECK: ^bb4:
-! CHECK:   %[[VAL_331:.*]] = arith.addi %[[VAL_316]], %[[VAL_305]] : index
+! CHECK:   %[[VAL_331:.*]] = arith.addi %[[VAL_316]], %[[VAL_305]] overflow<nsw> : index
 ! CHECK:   %[[VAL_332:.*]] = arith.subi %[[VAL_317]], %[[VAL_305]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_331]], %[[VAL_332]] : index, index)
 ! CHECK: ^bb5:
@@ -505,7 +505,7 @@ subroutine simple_iostat(x, y, j, stat)
 ! CHECK:   %[[VAL_355:.*]] = fir.convert %[[VAL_354]] : (i32) -> index
 ! CHECK:   %[[VAL_356:.*]] = fir.array_coor %[[VAL_357]] {{\[}}%[[VAL_347]]] %[[VAL_355]] : (!fir.box<!fir.array<?xf32>>, !fir.slice<1>, index) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_358:.*]] = fir.call @_FortranAioInputReal32(%[[VAL_343]], %[[VAL_356]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
-! CHECK:   %[[VAL_359:.*]] = arith.addi %[[VAL_349]], %[[VAL_338]] : index
+! CHECK:   %[[VAL_359:.*]] = arith.addi %[[VAL_349]], %[[VAL_338]] overflow<nsw> : index
 ! CHECK:   cf.br ^bb1(%[[VAL_359]], %[[VAL_358]] : index, i1)
 ! CHECK: ^bb3:
 ! CHECK:   cf.cond_br %[[VAL_350]], ^bb4, ^bb5
@@ -568,10 +568,10 @@ subroutine iostat_in_io_loop(k, j, stat)
 ! CHECK:   %[[VAL_399:.*]] = fir.array_coor %[[VAL_400]](%[[VAL_387]]) {{\[}}%[[VAL_389]]] %[[VAL_394]], %[[VAL_398]] : (!fir.ref<!fir.array<3x5xi32>>, !fir.shape<2>, !fir.slice<2>, index, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_401:.*]] = fir.convert %[[VAL_399]] : (!fir.ref<i32>) -> !fir.ref<i64>
 ! CHECK:   %[[VAL_402:.*]] = fir.call @_FortranAioInputInteger(%[[VAL_378]], %[[VAL_401]], %[[VAL_374]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
-! CHECK:   %[[VAL_403:.*]] = arith.addi %[[VAL_390]], %[[VAL_371]] : index
+! CHECK:   %[[VAL_403:.*]] = arith.addi %[[VAL_390]], %[[VAL_371]] overflow<nsw> : index
 ! CHECK:   cf.br ^bb4(%[[VAL_403]], %[[VAL_402]] : index, i1)
 ! CHECK: ^bb6(%[[VAL_404:.*]]: i1):
-! CHECK:   %[[VAL_405:.*]] = arith.addi %[[VAL_380]], %[[VAL_371]] : index
+! CHECK:   %[[VAL_405:.*]] = arith.addi %[[VAL_380]], %[[VAL_371]] overflow<nsw> : index
 ! CHECK:   cf.br ^bb1(%[[VAL_405]], %[[VAL_404]] : index, i1)
 ! CHECK: ^bb7:
 ! CHECK:   %[[VAL_406:.*]] = fir.convert %[[VAL_380]] : (index) -> i32
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index fe5e36f704c7..af38bf272b02 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -234,12 +234,6 @@ static llvm::cl::opt<bool> integerWrapAround(
     llvm::cl::desc("Treat signed integer overflow as two's complement"),
     llvm::cl::init(false));
 
-// TODO: integrate this option with the above
-static llvm::cl::opt<bool>
-    setNSW("integer-overflow",
-           llvm::cl::desc("add nsw flag to internal operations"),
-           llvm::cl::init(false));
-
 #define FLANG_EXCLUDE_CODEGEN
 #include "flang/Optimizer/Passes/CommandLineOpts.h"
 #include "flang/Optimizer/Passes/Pipelines.h"
@@ -381,7 +375,6 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
   loweringOptions.setNoPPCNativeVecElemOrder(enableNoPPCNativeVecElemOrder);
   loweringOptions.setLowerToHighLevelFIR(useHLFIR || emitHLFIR);
   loweringOptions.setIntegerWrapAround(integerWrapAround);
-  loweringOptions.setNSWOnLoopVarInc(setNSW);
   std::vector<Fortran::lower::EnvironmentDefault> envDefaults = {};
   Fortran::frontend::TargetOptions targetOpts;
   Fortran::frontend::CodeGenOptions cgOpts;
@@ -467,7 +460,7 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
 
     // Add O2 optimizer pass pipeline.
     MLIRToLLVMPassPipelineConfig config(llvm::OptimizationLevel::O2);
-    config.NSWOnLoopVarInc = setNSW;
+    config.NSWOnLoopVarInc = !integerWrapAround;
     fir::registerDefaultInlinerPass(config);
     fir::createDefaultFIROptimizerPassPipeline(pm, config);
   }
-- 
GitLab


From 21b3769d1dfed38ef8d8fa35ee98973926f7de62 Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com>
Date: Thu, 24 Oct 2024 23:26:02 -0700
Subject: [PATCH 127/693] [Scalarizer] Fix to only scalarize if intrinsic was
 marked as isTriviallyScalarizable (#113625)

fixes #113624
---
 llvm/lib/Transforms/Scalar/Scalarizer.cpp        | 11 +++++++++++
 llvm/test/Transforms/Scalarizer/uadd_overflow.ll | 16 ++++++++++++++++
 2 files changed, 27 insertions(+)
 create mode 100644 llvm/test/Transforms/Scalarizer/uadd_overflow.ll

diff --git a/llvm/lib/Transforms/Scalar/Scalarizer.cpp b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
index 772f4c6c35dd..03d069c9fcb3 100644
--- a/llvm/lib/Transforms/Scalar/Scalarizer.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
@@ -1084,6 +1084,17 @@ bool ScalarizerVisitor::visitExtractValueInst(ExtractValueInst &EVI) {
   ValueVector Res;
   if (!isStructOfMatchingFixedVectors(OpTy))
     return false;
+  if (CallInst *CI = dyn_cast<CallInst>(Op)) {
+    Function *F = CI->getCalledFunction();
+    if (!F)
+      return false;
+    Intrinsic::ID ID = F->getIntrinsicID();
+    if (ID == Intrinsic::not_intrinsic || !isTriviallyScalarizable(ID))
+      return false;
+    // Note: Fall through means Operand is a`CallInst` and it is defined in
+    // `isTriviallyScalarizable`.
+  } else
+    return false;
   Type *VecType = cast<FixedVectorType>(OpTy->getContainedType(0));
   std::optional<VectorSplit> VS = getVectorSplit(VecType);
   if (!VS)
diff --git a/llvm/test/Transforms/Scalarizer/uadd_overflow.ll b/llvm/test/Transforms/Scalarizer/uadd_overflow.ll
new file mode 100644
index 000000000000..39094451523a
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/uadd_overflow.ll
@@ -0,0 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt %s -passes='function(scalarizer)' -S | FileCheck %s
+
+; Test to make sure that struct return intrinsics that are not `isTriviallyScalarizable` do not get scalarized.
+
+define <3 x i32> @test_(<3 x i32> %a, <3 x i32> %b) {
+; CHECK-LABEL: define <3 x i32> @test_(
+; CHECK-SAME: <3 x i32> [[A:%.*]], <3 x i32> [[B:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call { <3 x i32>, <3 x i1> } @llvm.uadd.with.overflow.v3i32(<3 x i32> [[B]], <3 x i32> [[B]])
+; CHECK-NEXT:    [[EL:%.*]] = extractvalue { <3 x i32>, <3 x i1> } [[R]], 0
+; CHECK-NEXT:    ret <3 x i32> [[EL]]
+;
+  %r = call { <3 x i32>, <3 x i1> } @llvm.uadd.with.overflow.v3i32(<3 x i32> %b, <3 x i32> %b)
+  %el = extractvalue { <3 x i32>, <3 x i1> } %r, 0
+  ret <3 x i32> %el
+}
-- 
GitLab


From 5827334ed2c68a660ac73ae251f974333310d30b Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Fri, 25 Oct 2024 00:14:15 -0700
Subject: [PATCH 128/693] [gn] port 105d54726b1d7 (lldbValueObject)

---
 .../gn/secondary/lldb/source/API/BUILD.gn     |  1 +
 .../secondary/lldb/source/Breakpoint/BUILD.gn |  1 +
 .../secondary/lldb/source/Commands/BUILD.gn   |  1 +
 .../gn/secondary/lldb/source/Core/BUILD.gn    | 16 +------
 .../lldb/source/DataFormatters/BUILD.gn       |  1 +
 .../secondary/lldb/source/Expression/BUILD.gn |  1 +
 .../source/Plugins/SymbolFile/DWARF/BUILD.gn  |  1 +
 .../gn/secondary/lldb/source/Symbol/BUILD.gn  |  1 +
 .../gn/secondary/lldb/source/Target/BUILD.gn  |  1 +
 .../lldb/source/ValueObject/BUILD.gn          | 43 +++++++++++++++++++
 10 files changed, 52 insertions(+), 15 deletions(-)
 create mode 100644 llvm/utils/gn/secondary/lldb/source/ValueObject/BUILD.gn

diff --git a/llvm/utils/gn/secondary/lldb/source/API/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/API/BUILD.gn
index 148f2cbf7204..fae0c22710b0 100644
--- a/llvm/utils/gn/secondary/lldb/source/API/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/API/BUILD.gn
@@ -31,6 +31,7 @@ target(liblldb_type, "liblldb") {
     "//lldb/source/Symbol",
     "//lldb/source/Target",
     "//lldb/source/Utility",
+    "//lldb/source/ValueObject",
     "//lldb/source/Version",
     "//lldb/tools/argdumper:lldb-argdumper",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/lldb/source/Breakpoint/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Breakpoint/BUILD.gn
index 354aa6db95c3..24a1657759c5 100644
--- a/llvm/utils/gn/secondary/lldb/source/Breakpoint/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Breakpoint/BUILD.gn
@@ -9,6 +9,7 @@ static_library("Breakpoint") {
     "//lldb/source/Symbol",
     "//lldb/source/Target",
     "//lldb/source/Utility",
+    "//lldb/source/ValueObject",
     "//llvm/lib/Support",
   ]
   sources = [
diff --git a/llvm/utils/gn/secondary/lldb/source/Commands/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Commands/BUILD.gn
index 164ee901f285..1fc119c07244 100644
--- a/llvm/utils/gn/secondary/lldb/source/Commands/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Commands/BUILD.gn
@@ -26,6 +26,7 @@ static_library("Commands") {
     #"//lldb/source/Symbol", # FIXME: many-hop dependency cycle.
     #"//lldb/source/Target", # FIXME: 2-hop dependency cycle.
     "//lldb/source/Utility",
+    "//lldb/source/ValueObject",
     "//lldb/source/Version",
   ]
   sources = [
diff --git a/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn
index 0c9632a0a191..7b690b5c8782 100644
--- a/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn
@@ -24,6 +24,7 @@ static_library("Core") {
     #"//lldb/source/DataFormatters", # Dependency cycle.
     #"//lldb/source/Expression", # Dependency cycle.
     "//lldb/source/Host",
+    "//lldb/source/ValueObject",
 
     #"//lldb/source/Interpreter", # Dependency cycle.
     #"//lldb/source/Symbol", # Dependency cycle.
@@ -80,20 +81,5 @@ static_library("Core") {
     "ThreadedCommunication.cpp",
     "UserSettingsController.cpp",
     "Value.cpp",
-    "ValueObject.cpp",
-    "ValueObjectCast.cpp",
-    "ValueObjectChild.cpp",
-    "ValueObjectConstResult.cpp",
-    "ValueObjectConstResultCast.cpp",
-    "ValueObjectConstResultChild.cpp",
-    "ValueObjectConstResultImpl.cpp",
-    "ValueObjectDynamicValue.cpp",
-    "ValueObjectList.cpp",
-    "ValueObjectMemory.cpp",
-    "ValueObjectRegister.cpp",
-    "ValueObjectSyntheticFilter.cpp",
-    "ValueObjectUpdater.cpp",
-    "ValueObjectVTable.cpp",
-    "ValueObjectVariable.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/lldb/source/DataFormatters/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/DataFormatters/BUILD.gn
index 5dcbc01053fb..52e203c009c7 100644
--- a/llvm/utils/gn/secondary/lldb/source/DataFormatters/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/DataFormatters/BUILD.gn
@@ -8,6 +8,7 @@ static_library("DataFormatters") {
     #"//lldb/source/Symbol", # FIXME: 3-hop Dependency cycle (DataFormatters->Symbol->Expression->Interpreter->DataFormatters)
     #"//lldb/source/Target",  # FIXME: 3-hop Dependency cycle (DataFormatters->Target->Expression->Interpreter->DataFormatters)
     "//lldb/source/Utility",
+    "//lldb/source/ValueObject",
     "//llvm/lib/Support",
   ]
   sources = [
diff --git a/llvm/utils/gn/secondary/lldb/source/Expression/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Expression/BUILD.gn
index 973c10d14766..2163847cf81b 100644
--- a/llvm/utils/gn/secondary/lldb/source/Expression/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Expression/BUILD.gn
@@ -9,6 +9,7 @@ static_library("Expression") {
     #"//lldb/source/Symbol", # FIXME: Dependency cycle.
     #"//lldb/source/Target", # FIXME: Dependency cycle.
     "//lldb/source/Utility",
+    "//lldb/source/ValueObject",
     "//llvm/lib/ExecutionEngine",
     "//llvm/lib/IR",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/SymbolFile/DWARF/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/SymbolFile/DWARF/BUILD.gn
index b2f572ad3b66..b1405232ef2c 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/SymbolFile/DWARF/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/SymbolFile/DWARF/BUILD.gn
@@ -31,6 +31,7 @@ static_library("DWARF") {
     "//lldb/source/Symbol",
     "//lldb/source/Target",
     "//lldb/source/Utility",
+    "//lldb/source/ValueObject",
 
     #"//lldb/source/Plugins/TypeSystem/Clang", # Dependency cycle.
     "//llvm/lib/DebugInfo/DWARF",
diff --git a/llvm/utils/gn/secondary/lldb/source/Symbol/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Symbol/BUILD.gn
index 3b1df82a6b2d..75d2f6d1b21a 100644
--- a/llvm/utils/gn/secondary/lldb/source/Symbol/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Symbol/BUILD.gn
@@ -8,6 +8,7 @@ static_library("Symbol") {
 
     #"//lldb/source/Target", # FIXME: Dependency cycle.
     "//lldb/source/Utility",
+    "//lldb/source/ValueObject",
     "//llvm/lib/Support",
   ]
   include_dirs = [ ".." ]  # FIXME: Reaches into Utility and Host internals.
diff --git a/llvm/utils/gn/secondary/lldb/source/Target/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Target/BUILD.gn
index acf9e7aa701e..18eaaae14412 100644
--- a/llvm/utils/gn/secondary/lldb/source/Target/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Target/BUILD.gn
@@ -28,6 +28,7 @@ static_library("Target") {
     "//lldb/source/Plugins/Process/Utility",
     "//lldb/source/Symbol",
     "//lldb/source/Utility",
+    "//lldb/source/ValueObject",
     "//llvm/lib/MC",
     "//llvm/lib/Support",
   ]
diff --git a/llvm/utils/gn/secondary/lldb/source/ValueObject/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/ValueObject/BUILD.gn
new file mode 100644
index 000000000000..110f0a676146
--- /dev/null
+++ b/llvm/utils/gn/secondary/lldb/source/ValueObject/BUILD.gn
@@ -0,0 +1,43 @@
+static_library("ValueObject") {
+  output_name = "lldbValueObject"
+  configs += [ "//llvm/utils/gn/build:lldb_code" ]
+  deps = [
+    # FIXME: 3-hop dependency cycle
+    #"//lldb/source/Breakpoint",
+    # FIXME: 3-hop dependency cycle
+    #"//lldb/source/DataFormatters",
+    # FIXME: 3-hop dependency cycle
+    #"//lldb/source/Expression",
+    # FIXME: 3-hop dependency cycle
+    #"//lldb/source/Symbol",
+    # FIXME: 3-hop dependency cycle
+    #"//lldb/source/Target",
+    "//lldb/source/Utility",
+
+    # FIXME: 3-hop dependency cycle
+    #"//lldb/source/Plugins/Language/CPlusPlus",
+    # FIXME: 3-hop dependency cycle
+    #"//lldb/source/Plugins/Language/ObjC",
+    "//llvm/lib/Demangle",
+    "//llvm/lib/Support",
+    "//llvm/lib/TargetParser",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "ValueObject.cpp",
+    "ValueObjectCast.cpp",
+    "ValueObjectChild.cpp",
+    "ValueObjectConstResult.cpp",
+    "ValueObjectConstResultCast.cpp",
+    "ValueObjectConstResultChild.cpp",
+    "ValueObjectConstResultImpl.cpp",
+    "ValueObjectDynamicValue.cpp",
+    "ValueObjectList.cpp",
+    "ValueObjectMemory.cpp",
+    "ValueObjectRegister.cpp",
+    "ValueObjectSyntheticFilter.cpp",
+    "ValueObjectUpdater.cpp",
+    "ValueObjectVariable.cpp",
+    "ValueObjectVTable.cpp",
+  ]
+}
-- 
GitLab


From 93ee9d1c549873ef7912959330c89ef2d36784fa Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Fri, 25 Oct 2024 00:21:40 -0700
Subject: [PATCH 129/693] [gn] port 105d54726b1d7 more

Also, it's really b852fb1ec5f, not 105d54726b1d7.
---
 llvm/utils/gn/secondary/lldb/source/Plugins/ABI/X86/BUILD.gn     | 1 +
 .../lldb/source/Plugins/ExpressionParser/Clang/BUILD.gn          | 1 +
 .../gn/secondary/lldb/source/Plugins/Language/CPlusPlus/BUILD.gn | 1 +
 .../gn/secondary/lldb/source/Plugins/Language/ObjC/BUILD.gn      | 1 +
 .../source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/BUILD.gn | 1 +
 .../Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/BUILD.gn       | 1 +
 .../gn/secondary/lldb/source/Plugins/Platform/POSIX/BUILD.gn     | 1 +
 .../gn/secondary/lldb/source/Plugins/Process/Utility/BUILD.gn    | 1 +
 8 files changed, 8 insertions(+)

diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/ABI/X86/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/ABI/X86/BUILD.gn
index 0999f27ec07c..18c5543e0593 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/ABI/X86/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/ABI/X86/BUILD.gn
@@ -5,6 +5,7 @@ static_library("X86") {
     "//lldb/source/Core",
     "//lldb/source/Symbol",
     "//lldb/source/Target",
+    "//lldb/source/ValueObject",
     "//llvm/lib/Support",
     "//llvm/lib/TargetParser",
   ]
diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/ExpressionParser/Clang/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/ExpressionParser/Clang/BUILD.gn
index d73a904c5aba..5efc1531857b 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/ExpressionParser/Clang/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/ExpressionParser/Clang/BUILD.gn
@@ -27,6 +27,7 @@ static_library("Clang") {
     "//lldb/source/Symbol",
     "//lldb/source/Target",
     "//lldb/source/Utility",
+    "//lldb/source/ValueObject",
     "//llvm/lib/ExecutionEngine",
     "//llvm/lib/ExecutionEngine/MCJIT",
 
diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/Language/CPlusPlus/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/Language/CPlusPlus/BUILD.gn
index 91e5be0c8e27..e736cfe22bad 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/Language/CPlusPlus/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/Language/CPlusPlus/BUILD.gn
@@ -14,6 +14,7 @@ static_library("CPlusPlus") {
     "//lldb/source/Symbol",
     "//lldb/source/Target",
     "//lldb/source/Utility",
+    "//lldb/source/ValueObject",
     "//llvm/lib/Support",
   ]
 
diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/Language/ObjC/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/Language/ObjC/BUILD.gn
index 1704395730aa..1e2bf9dbd03b 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/Language/ObjC/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/Language/ObjC/BUILD.gn
@@ -15,6 +15,7 @@ static_library("ObjC") {
     "//lldb/source/Symbol",
     "//lldb/source/Target",
     "//lldb/source/Utility",
+    "//lldb/source/ValueObject",
 
     #"//lldb/source/Plugins/TypeSystem/Clang", # 3-hop dependency cycle.
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/BUILD.gn
index 7ca3dc376ba3..c62e98fbb853 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/BUILD.gn
@@ -12,6 +12,7 @@ static_library("ItaniumABI") {
     "//lldb/source/Plugins/TypeSystem/Clang",
     "//lldb/source/Symbol",
     "//lldb/source/Target",
+    "//lldb/source/ValueObject",
   ]
 
   # Reaches into Plugins/LanguageRuntime/CPlusPlus.
diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/BUILD.gn
index 632ef5ec71a5..2a805d46ad8a 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/BUILD.gn
@@ -16,6 +16,7 @@ static_library("AppleObjCRuntime") {
     "//lldb/source/Symbol",
     "//lldb/source/Target",
     "//lldb/source/Utility",
+    "//lldb/source/ValueObject",
 
     #"//lldb/source/Plugins/TypeSystem/Clang", # 4-hop dependency cycle.
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/Platform/POSIX/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/Platform/POSIX/BUILD.gn
index c5565bc20f5a..457407e98071 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/Platform/POSIX/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/Platform/POSIX/BUILD.gn
@@ -12,6 +12,7 @@ static_library("POSIX") {
     "//lldb/source/Plugins/Platform/gdb-server",
     "//lldb/source/Plugins/TypeSystem/Clang",
     "//lldb/source/Target",
+    "//lldb/source/ValueObject",
   ]
 
   # Reaches into Plugins/TypeSystem/Clang.
diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/Process/Utility/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/Process/Utility/BUILD.gn
index acf279bf1637..d4af7f3b0f07 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/Process/Utility/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/Process/Utility/BUILD.gn
@@ -11,6 +11,7 @@ static_library("Utility") {
 
     #"//lldb/source/Target", # Dependency cycle.
     "//lldb/source/Utility",
+    "//lldb/source/ValueObject",
     "//llvm/lib/Support",
     "//llvm/lib/TargetParser",
   ]
-- 
GitLab


From 18f4b7e4a862c11816e62cc72fb2a4ca8fac1987 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Fri, 25 Oct 2024 00:24:42 -0700
Subject: [PATCH 130/693] [gn] port b1be21394e9c

---
 llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
index 5fbda794ff17..7194c47b1bec 100644
--- a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
@@ -55,6 +55,7 @@ write_cmake_config("lit_common_configured") {
     "COMPILER_RT_ENABLE_INTERNAL_SYMBOLIZER_PYBOOL=False",
     "COMPILER_RT_HAS_NO_DEFAULT_CONFIG_FLAG_PYBOOL=True",
     "COMPILER_RT_INTERCEPT_LIBDISPATCH_PYBOOL=False",
+    "COMPILER_RT_RESOLVED_EXEC_OUTPUT_DIR=" + rebase_path(crt_current_out_dir),
     "COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR=" +
         rebase_path(crt_current_out_dir),
     "COMPILER_RT_RESOLVED_OUTPUT_DIR=" + rebase_path(crt_current_out_dir),
-- 
GitLab


From b89f09f5cc072b927917844a5781be3600acb72a Mon Sep 17 00:00:00 2001
From: "Oleksandr T." <oleksandr.tarasiuk@outlook.com>
Date: Fri, 25 Oct 2024 10:40:44 +0300
Subject: [PATCH 131/693] [Clang] Enhance handling of [[deprecated]] attribute
 diagnostics for local variables (#113575)

Fixes #90073
---
 clang/docs/ReleaseNotes.rst         |  2 ++
 clang/lib/Sema/SemaAvailability.cpp |  6 ++++++
 clang/test/SemaCXX/deprecated.cpp   | 20 ++++++++++++++++++++
 3 files changed, 28 insertions(+)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index d7b47cd2e2ee..ed0c0e369fca 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -424,6 +424,8 @@ Improvements to Clang's diagnostics
   name was a reserved name, which we improperly allowed to suppress the
   diagnostic.
 
+- Clang now diagnoses ``[[deprecated]]`` attribute usage on local variables (#GH90073).
+
 Improvements to Clang's time-trace
 ----------------------------------
 
diff --git a/clang/lib/Sema/SemaAvailability.cpp b/clang/lib/Sema/SemaAvailability.cpp
index 798cabaa31a4..076d3489fa94 100644
--- a/clang/lib/Sema/SemaAvailability.cpp
+++ b/clang/lib/Sema/SemaAvailability.cpp
@@ -182,6 +182,12 @@ static bool ShouldDiagnoseAvailabilityInContext(
       return false;
   }
 
+  if (K == AR_Deprecated) {
+    if (const auto *VD = dyn_cast<VarDecl>(OffendingDecl))
+      if (VD->isLocalVarDeclOrParm() && VD->isDeprecated())
+        return true;
+  }
+
   // Checks if we should emit the availability diagnostic in the context of C.
   auto CheckContext = [&](const Decl *C) {
     if (K == AR_NotYetIntroduced) {
diff --git a/clang/test/SemaCXX/deprecated.cpp b/clang/test/SemaCXX/deprecated.cpp
index a93331023f7e..667f4d7d3edb 100644
--- a/clang/test/SemaCXX/deprecated.cpp
+++ b/clang/test/SemaCXX/deprecated.cpp
@@ -260,5 +260,25 @@ namespace ArrayComp {
   bool b7 = arr1 == +f();
 }
 
+namespace GH90073 {
+[[deprecated]] int f1() { // expected-note {{'f1' has been explicitly marked deprecated here}}
+  [[deprecated]] int a;  // expected-note {{'a' has been explicitly marked deprecated here}} \
+                         // expected-note {{'a' has been explicitly marked deprecated here}}
+  a = 0;    // expected-warning {{'a' is deprecated}}
+  return a; // expected-warning {{'a' is deprecated}}
+}
+
+[[deprecated]] void f2([[deprecated]] int x) { // expected-note {{'f2' has been explicitly marked deprecated here}} \
+                                               // expected-note {{'x' has been explicitly marked deprecated here}}
+  x = 4; // expected-warning {{'x' is deprecated}}
+}
+
+int main() {
+  f1();  // expected-warning {{'f1' is deprecated}}
+  f2(1); // expected-warning {{'f2' is deprecated}}
+  return 0;
+}
+}
+
 # 1 "/usr/include/system-header.h" 1 3
 void system_header_function(void) throw();
-- 
GitLab


From ed6ddffb583beb450c4b0e1747ccd14f7e063105 Mon Sep 17 00:00:00 2001
From: dong-miao <601183878@qq.com>
Date: Fri, 25 Oct 2024 00:41:21 -0700
Subject: [PATCH 132/693] [RISCV] Add Smrnmi extension (#111668)

This commit has completed the Extension for Resumable Non Maskable
Interrupts, adding four CRSs and one Trap-Return instruction.
Specification link:["Smrnmi"
Extension](https://github.com/riscv/riscv-isa-manual/blob/main/src/rnmi.adoc)

---------

Co-authored-by: Sam Elliott <sam@lenary.co.uk>
---
 .../Driver/print-supported-extensions-riscv.c |  1 +
 .../test/Preprocessor/riscv-target-features.c |  9 +++
 llvm/docs/RISCVUsage.rst                      |  1 +
 llvm/lib/Target/RISCV/RISCVFeatures.td        |  7 +++
 llvm/lib/Target/RISCV/RISCVInstrInfo.td       |  8 +++
 llvm/test/CodeGen/RISCV/attributes.ll         |  4 ++
 llvm/test/MC/RISCV/attribute-arch.s           |  3 +
 llvm/test/MC/RISCV/machine-csr-names.s        | 60 +++++++++++++++++++
 llvm/test/MC/RISCV/smrnmi-invalid.s           |  4 ++
 llvm/test/MC/RISCV/smrnmi-valid.s             | 14 +++++
 .../TargetParser/RISCVISAInfoTest.cpp         |  1 +
 11 files changed, 112 insertions(+)
 create mode 100644 llvm/test/MC/RISCV/smrnmi-invalid.s
 create mode 100644 llvm/test/MC/RISCV/smrnmi-valid.s

diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index a39c1ab36b1d..65375b79cb68 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -120,6 +120,7 @@
 // CHECK-NEXT:     smcdeleg             1.0       'Smcdeleg' (Counter Delegation Machine Level)
 // CHECK-NEXT:     smcsrind             1.0       'Smcsrind' (Indirect CSR Access Machine Level)
 // CHECK-NEXT:     smepmp               1.0       'Smepmp' (Enhanced Physical Memory Protection)
+// CHECK-NEXT:     smrnmi               1.0       'Smrnmi' (Resumable Non-Maskable Interrupts)
 // CHECK-NEXT:     smstateen            1.0       'Smstateen' (Machine-mode view of the state-enable extension)
 // CHECK-NEXT:     ssaia                1.0       'Ssaia' (Advanced Interrupt Architecture Supervisor Level)
 // CHECK-NEXT:     ssccfg               1.0       'Ssccfg' (Counter Configuration Supervisor Level)
diff --git a/clang/test/Preprocessor/riscv-target-features.c b/clang/test/Preprocessor/riscv-target-features.c
index 05a8534ba13d..9e986f0143ae 100644
--- a/clang/test/Preprocessor/riscv-target-features.c
+++ b/clang/test/Preprocessor/riscv-target-features.c
@@ -30,6 +30,7 @@
 // CHECK-NOT: __riscv_smcdeleg {{.*$}}
 // CHECK-NOT: __riscv_smcsrind {{.*$}}
 // CHECK-NOT: __riscv_smepmp {{.*$}}
+// CHECK-NOT: __riscv_smrnmi {{.*$}}
 // CHECK-NOT: __riscv_smstateen {{.*$}}
 // CHECK-NOT: __riscv_ssaia {{.*$}}
 // CHECK-NOT: __riscv_ssccfg {{.*$}}
@@ -1449,6 +1450,14 @@
 // RUN:   -o - | FileCheck --check-prefix=CHECK-SMEPMP-EXT %s
 // CHECK-SMEPMP-EXT: __riscv_smepmp  1000000{{$}}
 
+// RUN: %clang --target=riscv32 \
+// RUN:   -march=rv32ismrnmi1p0 -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-SMRNMI-EXT %s
+// RUN: %clang --target=riscv64 \
+// RUN:   -march=rv64ismrnmi1p0 -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-SMRNMI-EXT %s
+// CHECK-SMRNMI-EXT: __riscv_smrnmi  1000000{{$}}
+
 // RUN: %clang --target=riscv32-unknown-linux-gnu \
 // RUN:   -march=rv32izfa -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-ZFA-EXT %s
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index 5736f3807f13..2135ed3b1114 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -129,6 +129,7 @@ on support follow.
      ``Smcdeleg``      Supported
      ``Smcsrind``      Supported
      ``Smepmp``        Supported
+     ``Smrnmi``        Assembly Support
      ``Smstateen``     Assembly Support
      ``Ssaia``         Supported
      ``Ssccfg``        Supported
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 3d0e1dae801d..7ccce2e1c70b 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -931,6 +931,13 @@ def FeatureStdExtSmepmp
     : RISCVExtension<"smepmp", 1, 0,
                      "'Smepmp' (Enhanced Physical Memory Protection)">;
 
+def FeatureStdExtSmrnmi
+    : RISCVExtension<"smrnmi", 1, 0,
+                     "'Smrnmi' (Resumable Non-Maskable Interrupts)">;
+def HasStdExtSmrnmi : Predicate<"Subtarget->hasStdExtSmrnmi()">,
+                       AssemblerPredicate<(all_of FeatureStdExtSmrnmi),
+                           "'Smrnmi' (Resumable Non-Maskable Interrupts)">; 
+
 def FeatureStdExtSmcdeleg
     : RISCVExtension<"smcdeleg", 1, 0,
                      "'Smcdeleg' (Counter Delegation Machine Level)">;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 5d329dceac65..86cc638fd04a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -815,6 +815,14 @@ def MRET : Priv<"mret", 0b0011000>, Sched<[]> {
 }
 } // isBarrier = 1, isReturn = 1, isTerminator = 1
 
+let Predicates = [HasStdExtSmrnmi] in {
+def MNRET : Priv<"mnret", 0b0111000>, Sched<[]> {
+  let rd = 0;
+  let rs1 = 0;
+  let rs2 = 0b00010;
+}
+}// Predicates = [HasStdExtSmrnmi]
+
 def WFI : Priv<"wfi", 0b0001000>, Sched<[]> {
   let rd = 0;
   let rs1 = 0;
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index 86ce368bc1db..aa27d63bfa62 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -118,6 +118,7 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+ssqosid %s -o - | FileCheck --check-prefix=RV32SSQOSID %s
 ; RUN: llc -mtriple=riscv32 -mattr=+smcdeleg %s -o - | FileCheck --check-prefixes=CHECK,RV32SMCDELEG %s
 ; RUN: llc -mtriple=riscv32 -mattr=+smepmp %s -o - | FileCheck --check-prefixes=CHECK,RV32SMEPMP %s
+; RUN: llc -mtriple=riscv32 -mattr=+smrnmi %s -o - | FileCheck --check-prefixes=CHECK,RV32SMRNMI %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zfbfmin %s -o - | FileCheck --check-prefixes=CHECK,RV32ZFBFMIN %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zvfbfmin %s -o - | FileCheck --check-prefixes=CHECK,RV32ZVFBFMIN %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zvfbfwma %s -o - | FileCheck --check-prefixes=CHECK,RV32ZVFBFWMA %s
@@ -261,6 +262,7 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+ssqosid %s -o - | FileCheck --check-prefix=RV64SSQOSID %s
 ; RUN: llc -mtriple=riscv64 -mattr=+smcdeleg %s -o - | FileCheck --check-prefixes=CHECK,RV64SMCDELEG %s
 ; RUN: llc -mtriple=riscv64 -mattr=+smepmp %s -o - | FileCheck --check-prefixes=CHECK,RV64SMEPMP %s
+; RUN: llc -mtriple=riscv64 -mattr=+smrnmi %s -o - | FileCheck --check-prefixes=CHECK,RV64SMRNMI %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zfbfmin %s -o - | FileCheck --check-prefixes=CHECK,RV64ZFBFMIN %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zvfbfmin %s -o - | FileCheck --check-prefixes=CHECK,RV64ZVFBFMIN %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zvfbfwma %s -o - | FileCheck --check-prefixes=CHECK,RV64ZVFBFWMA %s
@@ -410,6 +412,7 @@
 ; RV32SSQOSID: .attribute 5, "rv32i2p1_ssqosid1p0"
 ; RV32SMCDELEG: .attribute 5, "rv32i2p1_smcdeleg1p0"
 ; RV32SMEPMP: .attribute 5, "rv32i2p1_smepmp1p0"
+; RV32SMRNMI: .attribute 5, "rv32i2p1_smrnmi1p0"
 ; RV32ZFBFMIN: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zfbfmin1p0"
 ; RV32ZVFBFMIN: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zve32f1p0_zve32x1p0_zvfbfmin1p0_zvl32b1p0"
 ; RV32ZVFBFWMA: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zfbfmin1p0_zve32f1p0_zve32x1p0_zvfbfmin1p0_zvfbfwma1p0_zvl32b1p0"
@@ -551,6 +554,7 @@
 ; RV64SSQOSID: .attribute 5, "rv64i2p1_ssqosid1p0"
 ; RV64SMCDELEG: .attribute 5, "rv64i2p1_smcdeleg1p0"
 ; RV64SMEPMP: .attribute 5, "rv64i2p1_smepmp1p0"
+; RV64SMRNMI: .attribute 5, "rv64i2p1_smrnmi1p0"
 ; RV64ZFBFMIN: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zfbfmin1p0"
 ; RV64ZVFBFMIN: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zve32f1p0_zve32x1p0_zvfbfmin1p0_zvl32b1p0"
 ; RV64ZVFBFWMA: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zfbfmin1p0_zve32f1p0_zve32x1p0_zvfbfmin1p0_zvfbfwma1p0_zvl32b1p0"
diff --git a/llvm/test/MC/RISCV/attribute-arch.s b/llvm/test/MC/RISCV/attribute-arch.s
index 1c0b2a59d069..2332bcacc99d 100644
--- a/llvm/test/MC/RISCV/attribute-arch.s
+++ b/llvm/test/MC/RISCV/attribute-arch.s
@@ -327,6 +327,9 @@
 .attribute arch, "rv32i_smepmp1p0"
 # CHECK: attribute      5, "rv32i2p1_smepmp1p0"
 
+.attribute arch, "rv32i_smrnmi1p0"
+# CHECK: attribute      5, "rv32i2p1_smrnmi1p0"
+
 .attribute arch, "rv32i_ssccfg1p0"
 # CHECK: attribute      5, "rv32i2p1_ssccfg1p0"
 
diff --git a/llvm/test/MC/RISCV/machine-csr-names.s b/llvm/test/MC/RISCV/machine-csr-names.s
index 8cfdf7ee116c..d509f9eadeb5 100644
--- a/llvm/test/MC/RISCV/machine-csr-names.s
+++ b/llvm/test/MC/RISCV/machine-csr-names.s
@@ -1913,6 +1913,66 @@ csrrs t1, mhpmcounter31, zero
 csrrs t2, 0xB1F, zero
 
 
+######################################
+# Machine Counter Setup
+######################################
+# mnscratch
+# name
+# CHECK-INST: csrrs t1, mnscratch, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x00,0x74]
+# CHECK-INST-ALIAS: csrr t1, mnscratch
+# uimm12
+# CHECK-INST: csrrs t2, mnscratch, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x00,0x74]
+# CHECK-INST-ALIAS: csrr t2, mnscratch
+# name
+csrrs t1, mnscratch, zero
+# uimm12
+csrrs t2, 0x740, zero
+
+# mnepc
+# name
+# CHECK-INST: csrrs t1, mnepc, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x10,0x74]
+# CHECK-INST-ALIAS: csrr t1, mnepc
+# uimm12
+# CHECK-INST: csrrs t2, mnepc, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x10,0x74]
+# CHECK-INST-ALIAS: csrr t2, mnepc
+# name
+csrrs t1, mnepc, zero
+# uimm12
+csrrs t2, 0x741, zero
+
+# mncause
+# name
+# CHECK-INST: csrrs t1, mncause, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x20,0x74]
+# CHECK-INST-ALIAS: csrr t1, mncause
+# uimm12
+# CHECK-INST: csrrs t2, mncause, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x20,0x74]
+# CHECK-INST-ALIAS: csrr t2, mncause
+# name
+csrrs t1, mncause, zero
+# uimm12
+csrrs t2, 0x742, zero
+
+# mnstatus
+# name
+# CHECK-INST: csrrs t1, mnstatus, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x40,0x74]
+# CHECK-INST-ALIAS: csrr t1, mnstatus
+# uimm12
+# CHECK-INST: csrrs t2, mnstatus, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x40,0x74]
+# CHECK-INST-ALIAS: csrr t2, mnstatus
+# name
+csrrs t1, mnstatus, zero
+# uimm12
+csrrs t2, 0x744, zero
+
+
 ######################################
 # Machine Counter Setup
 ######################################
diff --git a/llvm/test/MC/RISCV/smrnmi-invalid.s b/llvm/test/MC/RISCV/smrnmi-invalid.s
new file mode 100644
index 000000000000..1a5e692148b3
--- /dev/null
+++ b/llvm/test/MC/RISCV/smrnmi-invalid.s
@@ -0,0 +1,4 @@
+# RUN: not llvm-mc -triple riscv32 -mattr=+smrnmi < %s 2>&1 | FileCheck %s
+# RUN: not llvm-mc -triple riscv64 -mattr=+smrnmi < %s 2>&1 | FileCheck %s
+
+mnret 0x10 # CHECK: :[[@LINE]]:7: error: invalid operand for instruction
diff --git a/llvm/test/MC/RISCV/smrnmi-valid.s b/llvm/test/MC/RISCV/smrnmi-valid.s
new file mode 100644
index 000000000000..d330ecefa605
--- /dev/null
+++ b/llvm/test/MC/RISCV/smrnmi-valid.s
@@ -0,0 +1,14 @@
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+smrnmi -riscv-no-aliases -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-INST %s
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+smrnmi -riscv-no-aliases -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-INST %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+smrnmi < %s \
+# RUN:     | llvm-objdump --mattr=+smrnmi -M no-aliases -d - \
+# RUN:     | FileCheck -check-prefix=CHECK-INST %s
+# RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+smrnmi < %s \
+# RUN:     | llvm-objdump --mattr=+smrnmi -M no-aliases -d - \
+# RUN:     | FileCheck -check-prefix=CHECK-INST %s
+
+# CHECK-INST: mnret
+# CHECK: encoding: [0x73,0x00,0x20,0x70]
+mnret
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 33944b64dc15..ded43a4ff787 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -1065,6 +1065,7 @@ R"(All available -march extensions for RISC-V
     smcdeleg             1.0
     smcsrind             1.0
     smepmp               1.0
+    smrnmi               1.0
     smstateen            1.0
     ssaia                1.0
     ssccfg               1.0
-- 
GitLab


From e738a5d8e33911381dbd0e1bc5a0109e9ebb62f2 Mon Sep 17 00:00:00 2001
From: Sebastian Kreutzer <SebastianKreutzer@gmx.net>
Date: Fri, 25 Oct 2024 09:15:25 +0100
Subject: [PATCH 133/693] Reapply " [XRay] Add support for instrumentation of
 DSOs on x86_64 (#90959)" (#113548)

This fixes remaining issues in my previous PR #90959.

Changes:
- Removed dependency on LLVM header in `xray_interface.cpp`
- Fixed XRay patching for some targets due to missing changes in
architecture-specific patching functions
- Addressed some remaining compiler warnings that I missed in the
previous patch
- Formatting

I have tested these changes on `x86_64` (natively), as well as
`ppc64le`, `aarch64` and `arm32` (cross-compiled and emulated using
qemu).

**Original description:**

This PR introduces shared library (DSO) support for XRay based on a
revised version of the implementation outlined in [this
RFC](https://discourse.llvm.org/t/rfc-upstreaming-dso-instrumentation-support-for-xray/73000).
The feature enables the patching and handling of events from DSOs,
supporting both libraries linked at startup or explicitly loaded, e.g.
via `dlopen`.
This patch adds the following:
- The `-fxray-shared` flag to enable the feature (turned off by default)
- A small runtime library that is linked into every instrumented DSO,
providing position-independent trampolines and code to register with the
main XRay runtime
- Changes to the XRay runtime to support management and patching of
multiple objects

These changes are fully backward compatible, i.e. running without
instrumented DSOs will produce identical traces (in terms of recorded
function IDs) to the previous implementation.

Due to my limited ability to test on other architectures, this feature
is only implemented and tested with x86_64. Extending support to other
architectures is fairly straightforward, requiring only a
position-independent implementation of the architecture-specific
trampoline implementation (see
`compiler-rt/lib/xray/xray_trampoline_x86_64.S` for reference).

This patch does not include any functionality to resolve function IDs
from DSOs for the provided logging/tracing modes. These modes still work
and will record calls from DSOs, but symbol resolution for these
functions in not available. Getting this to work properly requires
recording information about the loaded DSOs and should IMO be discussed
in a separate RFC, as there are mulitple feasible approaches.

---------

Co-authored-by: Sebastian Kreutzer <sebastian.kreutzer@tu-darmstadt.de>
---
 clang/include/clang/Basic/CodeGenOptions.def  |   2 +
 clang/include/clang/Driver/Options.td         |   5 +
 clang/include/clang/Driver/XRayArgs.h         |   2 +
 clang/lib/Driver/ToolChains/CommonArgs.cpp    |  12 +-
 clang/lib/Driver/XRayArgs.cpp                 |  21 ++
 clang/test/Driver/XRay/xray-shared.cpp        |  17 ++
 .../cmake/Modules/AllSupportedArchDefs.cmake  |   1 +
 compiler-rt/cmake/config-ix.cmake             |   4 +
 compiler-rt/include/xray/xray_interface.h     |  65 +++-
 compiler-rt/lib/xray/CMakeLists.txt           |  86 +++++-
 compiler-rt/lib/xray/xray_AArch64.cpp         |  19 +-
 compiler-rt/lib/xray/xray_arm.cpp             |  19 +-
 compiler-rt/lib/xray/xray_dso_init.cpp        |  62 ++++
 compiler-rt/lib/xray/xray_hexagon.cpp         |  19 +-
 compiler-rt/lib/xray/xray_init.cpp            | 183 ++++++++++--
 compiler-rt/lib/xray/xray_interface.cpp       | 282 ++++++++++++++----
 .../lib/xray/xray_interface_internal.h        |  76 ++++-
 compiler-rt/lib/xray/xray_loongarch64.cpp     |  23 +-
 compiler-rt/lib/xray/xray_mips.cpp            |  23 +-
 compiler-rt/lib/xray/xray_mips64.cpp          |  23 +-
 compiler-rt/lib/xray/xray_powerpc64.cpp       |  30 +-
 compiler-rt/lib/xray/xray_trampoline_x86_64.S |  24 +-
 compiler-rt/lib/xray/xray_x86_64.cpp          |  30 +-
 .../xray/TestCases/Posix/basic-mode-dso.cpp   |  47 +++
 .../TestCases/Posix/clang-xray-shared.cpp     |  14 +
 .../test/xray/TestCases/Posix/dlopen.cpp      | 107 +++++++
 .../xray/TestCases/Posix/dso-dep-chains.cpp   | 197 ++++++++++++
 .../TestCases/Posix/patch-premain-dso.cpp     |  45 +++
 .../Posix/patching-unpatching-dso.cpp         |  75 +++++
 29 files changed, 1319 insertions(+), 194 deletions(-)
 create mode 100644 clang/test/Driver/XRay/xray-shared.cpp
 create mode 100644 compiler-rt/lib/xray/xray_dso_init.cpp
 create mode 100644 compiler-rt/test/xray/TestCases/Posix/basic-mode-dso.cpp
 create mode 100644 compiler-rt/test/xray/TestCases/Posix/clang-xray-shared.cpp
 create mode 100644 compiler-rt/test/xray/TestCases/Posix/dlopen.cpp
 create mode 100644 compiler-rt/test/xray/TestCases/Posix/dso-dep-chains.cpp
 create mode 100644 compiler-rt/test/xray/TestCases/Posix/patch-premain-dso.cpp
 create mode 100644 compiler-rt/test/xray/TestCases/Posix/patching-unpatching-dso.cpp

diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index eac831278ee2..e45370bde74a 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -136,6 +136,8 @@ CODEGENOPT(XRayIgnoreLoops , 1, 0)
 ///< Emit the XRay function index section.
 CODEGENOPT(XRayFunctionIndex , 1, 1)
 
+///< Set when -fxray-shared is enabled
+CODEGENOPT(XRayShared , 1, 0)
 
 ///< Set the minimum number of instructions in a function to determine selective
 ///< XRay instrumentation.
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 8263fe6f7d7f..5df6ddd5e6a0 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -2948,6 +2948,11 @@ def fxray_selected_function_group :
   HelpText<"When using -fxray-function-groups, select which group of functions to instrument. Valid range is 0 to fxray-function-groups - 1">,
   MarshallingInfoInt<CodeGenOpts<"XRaySelectedFunctionGroup">, "0">;
 
+defm xray_shared : BoolFOption<"xray-shared",
+  CodeGenOpts<"XRayShared">, DefaultFalse,
+  PosFlag<SetTrue, [], [ClangOption, CC1Option],
+          "Enable shared library instrumentation with XRay">,
+  NegFlag<SetFalse>>;
 
 defm fine_grained_bitfield_accesses : BoolOption<"f", "fine-grained-bitfield-accesses",
   CodeGenOpts<"FineGrainedBitfieldAccesses">, DefaultFalse,
diff --git a/clang/include/clang/Driver/XRayArgs.h b/clang/include/clang/Driver/XRayArgs.h
index bdd3d979547e..1b5c4a4c42f1 100644
--- a/clang/include/clang/Driver/XRayArgs.h
+++ b/clang/include/clang/Driver/XRayArgs.h
@@ -27,6 +27,7 @@ class XRayArgs {
   XRayInstrSet InstrumentationBundle;
   llvm::opt::Arg *XRayInstrument = nullptr;
   bool XRayRT = true;
+  bool XRayShared = false;
 
 public:
   /// Parses the XRay arguments from an argument list.
@@ -35,6 +36,7 @@ public:
                llvm::opt::ArgStringList &CmdArgs, types::ID InputType) const;
 
   bool needsXRayRt() const { return XRayInstrument && XRayRT; }
+  bool needsXRayDSORt() const { return XRayInstrument && XRayRT && XRayShared; }
   llvm::ArrayRef<std::string> modeList() const { return Modes; }
   XRayInstrSet instrumentationBundle() const { return InstrumentationBundle; }
 };
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 91605a67a37f..1c3c8c816594 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1623,10 +1623,14 @@ bool tools::addSanitizerRuntimes(const ToolChain &TC, const ArgList &Args,
 }
 
 bool tools::addXRayRuntime(const ToolChain&TC, const ArgList &Args, ArgStringList &CmdArgs) {
-  if (Args.hasArg(options::OPT_shared))
-    return false;
-
-  if (TC.getXRayArgs().needsXRayRt()) {
+  if (Args.hasArg(options::OPT_shared)) {
+    if (TC.getXRayArgs().needsXRayDSORt()) {
+      CmdArgs.push_back("--whole-archive");
+      CmdArgs.push_back(TC.getCompilerRTArgString(Args, "xray-dso"));
+      CmdArgs.push_back("--no-whole-archive");
+      return true;
+    }
+  } else if (TC.getXRayArgs().needsXRayRt()) {
     CmdArgs.push_back("--whole-archive");
     CmdArgs.push_back(TC.getCompilerRTArgString(Args, "xray"));
     for (const auto &Mode : TC.getXRayArgs().modeList())
diff --git a/clang/lib/Driver/XRayArgs.cpp b/clang/lib/Driver/XRayArgs.cpp
index 8c5134e25013..d0bb5d4887c1 100644
--- a/clang/lib/Driver/XRayArgs.cpp
+++ b/clang/lib/Driver/XRayArgs.cpp
@@ -63,6 +63,23 @@ XRayArgs::XRayArgs(const ToolChain &TC, const ArgList &Args) {
         << XRayInstrument->getSpelling() << Triple.str();
   }
 
+  if (Args.hasFlag(options::OPT_fxray_shared, options::OPT_fno_xray_shared,
+                   false)) {
+    XRayShared = true;
+
+    // DSO instrumentation is currently limited to x86_64
+    if (Triple.getArch() != llvm::Triple::x86_64) {
+      D.Diag(diag::err_drv_unsupported_opt_for_target)
+          << "-fxray-shared" << Triple.str();
+    }
+
+    unsigned PICLvl = std::get<1>(tools::ParsePICArgs(TC, Args));
+    if (!PICLvl) {
+      D.Diag(diag::err_opt_not_valid_without_opt) << "-fxray-shared"
+                                                  << "-fPIC";
+    }
+  }
+
   // Both XRay and -fpatchable-function-entry use
   // TargetOpcode::PATCHABLE_FUNCTION_ENTER.
   if (Arg *A = Args.getLastArg(options::OPT_fpatchable_function_entry_EQ))
@@ -177,6 +194,10 @@ void XRayArgs::addArgs(const ToolChain &TC, const ArgList &Args,
   Args.addOptOutFlag(CmdArgs, options::OPT_fxray_function_index,
                      options::OPT_fno_xray_function_index);
 
+  if (XRayShared)
+    Args.addOptInFlag(CmdArgs, options::OPT_fxray_shared,
+                      options::OPT_fno_xray_shared);
+
   if (const Arg *A =
           Args.getLastArg(options::OPT_fxray_instruction_threshold_EQ)) {
     int Value;
diff --git a/clang/test/Driver/XRay/xray-shared.cpp b/clang/test/Driver/XRay/xray-shared.cpp
new file mode 100644
index 000000000000..215854e1fc7c
--- /dev/null
+++ b/clang/test/Driver/XRay/xray-shared.cpp
@@ -0,0 +1,17 @@
+// RUN: %clang -### --target=x86_64-unknown-linux-gnu -fPIC -fxray-instrument -fxray-shared -c %s -o /dev/null 2>&1 | FileCheck %s
+// RUN: %clang -### --target=x86_64-unknown-linux-gnu -fpic -fxray-instrument -fxray-shared -c %s -o /dev/null 2>&1 | FileCheck %s
+// RUN: %clang -### --target=x86_64-unknown-linux-gnu -fxray-instrument -fxray-shared -c %s -o /dev/null 2>&1 | FileCheck %s
+// RUN: not %clang -### --target=x86_64-unknown-linux-gnu -fno-PIC -fxray-instrument -fxray-shared -c %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR-PIC
+// RUN: not %clang -### --target=x86_64-unknown-linux-gnu -fno-pic -fxray-instrument -fxray-shared -c %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR-PIC
+
+// On 64 bit darwin, PIC is always enabled
+// RUN: %clang -### --target=x86_64-apple-darwin -fxray-instrument -fxray-shared -c %s -o /dev/null 2>&1 | FileCheck %s
+
+// Check unsupported targets
+// RUN: not %clang -### --target=aarch64-pc-freebsd -fPIC -fxray-instrument -fxray-shared -c %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR-TARGET
+// RUN: not %clang -### --target=arm64-apple-macos -fPIC -fxray-instrument -fxray-shared -c %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR-TARGET
+
+// CHECK: "-cc1" {{.*}}"-fxray-instrument" {{.*}}"-fxray-shared"
+// ERR-TARGET:   error: unsupported option '-fxray-shared' for target
+// ERR-PIC:   error: option '-fxray-shared' cannot be specified without '-fPIC'
+
diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
index d00d39518104..fb4dfa7bd09d 100644
--- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
+++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
@@ -104,6 +104,7 @@ else()
 set(ALL_XRAY_SUPPORTED_ARCH ${X86_64} ${ARM32} ${ARM64} ${MIPS32} ${MIPS64}
 		powerpc64le ${HEXAGON} ${LOONGARCH64})
 endif()
+set(ALL_XRAY_DSO_SUPPORTED_ARCH ${X86_64})
 set(ALL_SHADOWCALLSTACK_SUPPORTED_ARCH ${ARM64})
 
 if (UNIX)
diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake
index a494e0532a50..431f544e8ad6 100644
--- a/compiler-rt/cmake/config-ix.cmake
+++ b/compiler-rt/cmake/config-ix.cmake
@@ -668,6 +668,9 @@ if(APPLE)
   list_intersect(XRAY_SUPPORTED_ARCH
     ALL_XRAY_SUPPORTED_ARCH
     SANITIZER_COMMON_SUPPORTED_ARCH)
+  list_intersect(XRAY_DSO_SUPPORTED_ARCH
+    ALL_XRAY_DSO_SUPPORTED_ARCH
+    SANITIZER_COMMON_SUPPORTED_ARCH)
   list_intersect(SHADOWCALLSTACK_SUPPORTED_ARCH
     ALL_SHADOWCALLSTACK_SUPPORTED_ARCH
     SANITIZER_COMMON_SUPPORTED_ARCH)
@@ -702,6 +705,7 @@ else()
   filter_available_targets(CFI_SUPPORTED_ARCH ${ALL_CFI_SUPPORTED_ARCH})
   filter_available_targets(SCUDO_STANDALONE_SUPPORTED_ARCH ${ALL_SCUDO_STANDALONE_SUPPORTED_ARCH})
   filter_available_targets(XRAY_SUPPORTED_ARCH ${ALL_XRAY_SUPPORTED_ARCH})
+  filter_available_targets(XRAY_DSO_SUPPORTED_ARCH ${ALL_XRAY_DSO_SUPPORTED_ARCH})
   filter_available_targets(SHADOWCALLSTACK_SUPPORTED_ARCH
     ${ALL_SHADOWCALLSTACK_SUPPORTED_ARCH})
   filter_available_targets(GWP_ASAN_SUPPORTED_ARCH ${ALL_GWP_ASAN_SUPPORTED_ARCH})
diff --git a/compiler-rt/include/xray/xray_interface.h b/compiler-rt/include/xray/xray_interface.h
index 727431c04e4f..675ea0cbc48c 100644
--- a/compiler-rt/include/xray/xray_interface.h
+++ b/compiler-rt/include/xray/xray_interface.h
@@ -93,31 +93,78 @@ enum XRayPatchingStatus {
   FAILED = 3,
 };
 
-/// This tells XRay to patch the instrumentation points. See XRayPatchingStatus
-/// for possible result values.
+/// This tells XRay to patch the instrumentation points in all currently loaded
+/// objects. See XRayPatchingStatus for possible result values.
 extern XRayPatchingStatus __xray_patch();
 
+/// This tells XRay to patch the instrumentation points in the given object.
+/// See XRayPatchingStatus for possible result values.
+extern XRayPatchingStatus __xray_patch_object(int32_t ObjId);
+
 /// Reverses the effect of __xray_patch(). See XRayPatchingStatus for possible
 /// result values.
 extern XRayPatchingStatus __xray_unpatch();
 
-/// This patches a specific function id. See XRayPatchingStatus for possible
+/// Reverses the effect of __xray_patch_object. See XRayPatchingStatus for
+/// possible result values.
+extern XRayPatchingStatus __xray_unpatch_object(int32_t ObjId);
+
+/// This unpacks the given (packed) function id and patches
+/// the corresponding function.  See XRayPatchingStatus for possible
 /// result values.
 extern XRayPatchingStatus __xray_patch_function(int32_t FuncId);
 
-/// This unpatches a specific function id. See XRayPatchingStatus for possible
+/// This patches a specific function in the given object. See XRayPatchingStatus
+/// for possible result values.
+extern XRayPatchingStatus __xray_patch_function_in_object(int32_t FuncId,
+                                                          int32_t ObjId);
+
+/// This unpacks the given (packed) function id and unpatches
+/// the corresponding function. See XRayPatchingStatus for possible
 /// result values.
 extern XRayPatchingStatus __xray_unpatch_function(int32_t FuncId);
 
-/// This function returns the address of the function provided a valid function
-/// id. We return 0 if we encounter any error, even if 0 may be a valid function
-/// address.
+/// This unpatches a specific function in the given object.
+/// See XRayPatchingStatus for possible result values.
+extern XRayPatchingStatus __xray_unpatch_function_in_object(int32_t FuncId,
+                                                            int32_t ObjId);
+
+/// This function unpacks the given (packed) function id and returns the address
+/// of the corresponding function. We return 0 if we encounter any error, even
+/// if 0 may be a valid function address.
 extern uintptr_t __xray_function_address(int32_t FuncId);
 
-/// This function returns the maximum valid function id. Returns 0 if we
-/// encounter errors (when there are no instrumented functions, etc.).
+/// This function returns the address of the function in the given object
+/// provided valid function and object ids. We return 0 if we encounter any
+/// error, even if 0 may be a valid function address.
+extern uintptr_t __xray_function_address_in_object(int32_t FuncId,
+                                                   int32_t ObjId);
+
+/// This function returns the maximum valid function id for the main executable
+/// (object id = 0). Returns 0 if we encounter errors (when there are no
+/// instrumented functions, etc.).
 extern size_t __xray_max_function_id();
 
+/// This function returns the maximum valid function id for the given object.
+/// Returns 0 if we encounter errors (when there are no instrumented functions,
+/// etc.).
+extern size_t __xray_max_function_id_in_object(int32_t ObjId);
+
+/// This function returns the number of previously registered objects
+/// (executable + loaded DSOs). Returns 0 if XRay has not been initialized.
+extern size_t __xray_num_objects();
+
+/// Unpacks the function id from the given packed id.
+extern int32_t __xray_unpack_function_id(int32_t PackedId);
+
+/// Unpacks the object id from the given packed id.
+extern int32_t __xray_unpack_object_id(int32_t PackedId);
+
+/// Creates and returns a packed id from the given function and object ids.
+/// If the ids do not fit within the reserved number of bits for each part, the
+/// high bits are truncated.
+extern int32_t __xray_pack_id(int32_t FuncId, int32_t ObjId);
+
 /// Initialize the required XRay data structures. This is useful in cases where
 /// users want to control precisely when the XRay instrumentation data
 /// structures are initialized, for example when the XRay library is built with
diff --git a/compiler-rt/lib/xray/CMakeLists.txt b/compiler-rt/lib/xray/CMakeLists.txt
index cf7b5062aae3..f38c07420c9a 100644
--- a/compiler-rt/lib/xray/CMakeLists.txt
+++ b/compiler-rt/lib/xray/CMakeLists.txt
@@ -10,6 +10,10 @@ set(XRAY_SOURCES
   xray_utils.cpp
   )
 
+set(XRAY_DSO_SOURCES
+  xray_dso_init.cpp
+  )
+
 # Implementation files for all XRay modes.
 set(XRAY_FDR_MODE_SOURCES
   xray_fdr_flags.cpp
@@ -33,6 +37,11 @@ set(x86_64_SOURCES
   xray_trampoline_x86_64.S
   )
 
+set(x86_64_DSO_SOURCES
+   xray_trampoline_x86_64.S
+   )
+
+
 set(arm_SOURCES
   xray_arm.cpp
   xray_trampoline_arm.S
@@ -128,10 +137,12 @@ set(XRAY_IMPL_HEADERS
 # consumption by tests.
 set(XRAY_ALL_SOURCE_FILES
   ${XRAY_SOURCES}
+  ${XRAY_DSO_SOURCES}
   ${XRAY_FDR_MODE_SOURCES}
   ${XRAY_BASIC_MODE_SOURCES}
   ${XRAY_PROFILING_MODE_SOURCES}
   ${x86_64_SOURCES}
+  ${x86_64_DSO_SOURCES}
   ${arm_SOURCES}
   ${armhf_SOURCES}
   ${hexagon_SOURCES}
@@ -162,6 +173,9 @@ set(XRAY_CFLAGS
   ${COMPILER_RT_CXX_CFLAGS})
 set(XRAY_COMMON_DEFINITIONS SANITIZER_COMMON_NO_REDEFINE_BUILTINS XRAY_HAS_EXCEPTIONS=1)
 
+# DSO trampolines need to be compiled with GOT addressing
+set(XRAY_COMMON_DEFINITIONS_DSO ${XRAY_COMMON_DEFINITIONS} XRAY_PIC)
+
 # Too many existing bugs, needs cleanup.
 append_list_if(COMPILER_RT_HAS_WNO_FORMAT -Wno-format XRAY_CFLAGS)
 
@@ -201,7 +215,16 @@ if (APPLE)
     CFLAGS ${XRAY_CFLAGS}
     DEFS ${XRAY_COMMON_DEFINITIONS}
     DEPS ${XRAY_DEPS})
+  add_compiler_rt_object_libraries(RTXrayDSO
+    OS ${XRAY_SUPPORTED_OS}
+    ARCHS ${XRAY_DSO_SUPPORTED_ARCH}
+    SOURCES ${XRAY_DSO_SOURCES}
+    ADDITIONAL_HEADERS ${XRAY_IMPL_HEADERS}
+    CFLAGS ${XRAY_CFLAGS}
+    DEFS ${XRAY_COMMON_DEFINITIONS_DSO}
+    DEPS ${XRAY_DEPS})
   set(XRAY_RTXRAY_ARCH_LIBS "")
+  set(XRAY_DSO_RTXRAY_ARCH_LIBS "")
   foreach(arch ${XRAY_SUPPORTED_ARCH})
     if(NOT ${arch} IN_LIST XRAY_SOURCE_ARCHS)
       continue()
@@ -215,6 +238,17 @@ if (APPLE)
       DEFS ${XRAY_COMMON_DEFINITIONS}
       DEPS ${XRAY_DEPS})
     list(APPEND XRAY_RTXRAY_ARCH_LIBS RTXray_${arch})
+    if (${arch} IN_LIST XRAY_DSO_SUPPORTED_ARCH)
+      add_compiler_rt_object_libraries(RTXrayDSO_${arch}
+        OS ${XRAY_SUPPORTED_OS}
+        ARCHS ${XRAY_DSO_SUPPORTED_ARCH}
+        SOURCES ${${arch}_DSO_SOURCES}
+        ADDITIONAL_HEADERS ${XRAY_IMPL_HEADERS}
+        CFLAGS ${XRAY_CFLAGS}
+        DEFS ${XRAY_COMMON_DEFINITIONS_DSO}
+        DEPS ${XRAY_DEPS})
+      list(APPEND XRAY_DSO_RTXRAY_ARCH_LIBS RTXrayDSO_${arch})
+    endif()
   endforeach()
   add_compiler_rt_object_libraries(RTXrayFDR
     OS ${XRAY_SUPPORTED_OS}
@@ -252,6 +286,17 @@ if (APPLE)
     LINK_FLAGS ${XRAY_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
     LINK_LIBS ${XRAY_LINK_LIBS}
     PARENT_TARGET xray)
+  add_compiler_rt_runtime(clang_rt.xray-dso
+    STATIC
+    OS ${XRAY_SUPPORTED_OS}
+    ARCHS ${XRAY_DSO_SUPPORTED_ARCH}
+    OBJECT_LIBS RTXrayDSO ${XRAY_DSO_RTXRAY_ARCH_LIBS}
+    CFLAGS ${XRAY_CFLAGS}
+    DEFS ${XRAY_COMMON_DEFINITIONS}
+    LINK_FLAGS ${XRAY_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
+    LINK_LIBS ${XRAY_LINK_LIBS}
+    PARENT_TARGET xray)
+
   add_compiler_rt_runtime(clang_rt.xray-fdr
     STATIC
     OS ${XRAY_SUPPORTED_OS}
@@ -346,16 +391,37 @@ else() # not Apple
       DEFS ${XRAY_COMMON_DEFINITIONS}
       OBJECT_LIBS RTXrayBASIC
       PARENT_TARGET xray)
-   # Profiler Mode runtime
-   add_compiler_rt_runtime(clang_rt.xray-profiling
-     STATIC
-     ARCHS ${arch}
-     CFLAGS ${XRAY_CFLAGS}
-     LINK_FLAGS ${XRAY_LINK_FLAGS}
-     LINK_LIBS ${XRAY_LINK_LIBS}
-     DEFS ${XRAY_COMMON_DEFINITIONS}
-     OBJECT_LIBS RTXrayPROFILING
-     PARENT_TARGET xray)
+    # Profiler Mode runtime
+    add_compiler_rt_runtime(clang_rt.xray-profiling
+      STATIC
+      ARCHS ${arch}
+      CFLAGS ${XRAY_CFLAGS}
+      LINK_FLAGS ${XRAY_LINK_FLAGS}
+      LINK_LIBS ${XRAY_LINK_LIBS}
+      DEFS ${XRAY_COMMON_DEFINITIONS}
+      OBJECT_LIBS RTXrayPROFILING
+      PARENT_TARGET xray)
+
+    if (${arch} IN_LIST XRAY_DSO_SUPPORTED_ARCH)
+      # TODO: Only implemented for X86 at the moment
+      add_compiler_rt_object_libraries(RTXrayDSO
+        ARCHS ${arch}
+        SOURCES ${XRAY_DSO_SOURCES} ${${arch}_DSO_SOURCES} 
+        ADDITIONAL_HEADERS ${XRAY_IMPL_HEADERS}
+        CFLAGS ${XRAY_CFLAGS}
+        DEFS ${XRAY_COMMON_DEFINITIONS_DSO}
+        DEPS ${XRAY_DEPS})
+      # DSO runtime archive
+      add_compiler_rt_runtime(clang_rt.xray-dso
+        STATIC
+        ARCHS ${arch}
+        CFLAGS ${XRAY_CFLAGS}
+        LINK_FLAGS ${XRAY_LINK_FLAGS}
+        LINK_LIBS ${XRAY_LINK_LIBS}
+        DEFS ${XRAY_COMMON_DEFINITIONS}
+        OBJECT_LIBS RTXrayDSO
+        PARENT_TARGET xray)
+    endif()
   endforeach()
 endif() # not Apple
 
diff --git a/compiler-rt/lib/xray/xray_AArch64.cpp b/compiler-rt/lib/xray/xray_AArch64.cpp
index c1d77758946e..2b151162ce4b 100644
--- a/compiler-rt/lib/xray/xray_AArch64.cpp
+++ b/compiler-rt/lib/xray/xray_AArch64.cpp
@@ -89,18 +89,23 @@ inline static bool patchSled(const bool Enable, const uint32_t FuncId,
 
 bool patchFunctionEntry(const bool Enable, const uint32_t FuncId,
                         const XRaySledEntry &Sled,
-                        void (*Trampoline)()) XRAY_NEVER_INSTRUMENT {
+                        const XRayTrampolines &Trampolines,
+                        bool LogArgs) XRAY_NEVER_INSTRUMENT {
+  auto Trampoline =
+      LogArgs ? Trampolines.LogArgsTrampoline : Trampolines.EntryTrampoline;
   return patchSled(Enable, FuncId, Sled, Trampoline);
 }
 
-bool patchFunctionExit(const bool Enable, const uint32_t FuncId,
-                       const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
-  return patchSled(Enable, FuncId, Sled, __xray_FunctionExit);
+bool patchFunctionExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+  return patchSled(Enable, FuncId, Sled, Trampolines.ExitTrampoline);
 }
 
-bool patchFunctionTailExit(const bool Enable, const uint32_t FuncId,
-                           const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
-  return patchSled(Enable, FuncId, Sled, __xray_FunctionTailExit);
+bool patchFunctionTailExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+  return patchSled(Enable, FuncId, Sled, Trampolines.TailExitTrampoline);
 }
 
 // AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL generates this code sequence:
diff --git a/compiler-rt/lib/xray/xray_arm.cpp b/compiler-rt/lib/xray/xray_arm.cpp
index e1818555906c..e318bb7070e8 100644
--- a/compiler-rt/lib/xray/xray_arm.cpp
+++ b/compiler-rt/lib/xray/xray_arm.cpp
@@ -128,18 +128,23 @@ inline static bool patchSled(const bool Enable, const uint32_t FuncId,
 
 bool patchFunctionEntry(const bool Enable, const uint32_t FuncId,
                         const XRaySledEntry &Sled,
-                        void (*Trampoline)()) XRAY_NEVER_INSTRUMENT {
+                        const XRayTrampolines &Trampolines,
+                        bool LogArgs) XRAY_NEVER_INSTRUMENT {
+  auto Trampoline =
+      LogArgs ? Trampolines.LogArgsTrampoline : Trampolines.EntryTrampoline;
   return patchSled(Enable, FuncId, Sled, Trampoline);
 }
 
-bool patchFunctionExit(const bool Enable, const uint32_t FuncId,
-                       const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
-  return patchSled(Enable, FuncId, Sled, __xray_FunctionExit);
+bool patchFunctionExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+  return patchSled(Enable, FuncId, Sled, Trampolines.ExitTrampoline);
 }
 
-bool patchFunctionTailExit(const bool Enable, const uint32_t FuncId,
-                           const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
-  return patchSled(Enable, FuncId, Sled, __xray_FunctionTailExit);
+bool patchFunctionTailExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+  return patchSled(Enable, FuncId, Sled, Trampolines.TailExitTrampoline);
 }
 
 bool patchCustomEvent(const bool Enable, const uint32_t FuncId,
diff --git a/compiler-rt/lib/xray/xray_dso_init.cpp b/compiler-rt/lib/xray/xray_dso_init.cpp
new file mode 100644
index 000000000000..eb754db54c64
--- /dev/null
+++ b/compiler-rt/lib/xray/xray_dso_init.cpp
@@ -0,0 +1,62 @@
+//===-- xray_init.cpp -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// XRay initialisation logic for DSOs.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "xray_defs.h"
+#include "xray_flags.h"
+#include "xray_interface_internal.h"
+
+using namespace __sanitizer;
+
+extern "C" {
+extern const XRaySledEntry __start_xray_instr_map[] __attribute__((weak))
+__attribute__((visibility("hidden")));
+extern const XRaySledEntry __stop_xray_instr_map[] __attribute__((weak))
+__attribute__((visibility("hidden")));
+extern const XRayFunctionSledIndex __start_xray_fn_idx[] __attribute__((weak))
+__attribute__((visibility("hidden")));
+extern const XRayFunctionSledIndex __stop_xray_fn_idx[] __attribute__((weak))
+__attribute__((visibility("hidden")));
+
+#if SANITIZER_APPLE
+// HACK: This is a temporary workaround to make XRay build on
+// Darwin, but it will probably not work at runtime.
+extern const XRaySledEntry __start_xray_instr_map[] = {};
+extern const XRaySledEntry __stop_xray_instr_map[] = {};
+extern const XRayFunctionSledIndex __start_xray_fn_idx[] = {};
+extern const XRayFunctionSledIndex __stop_xray_fn_idx[] = {};
+#endif
+}
+
+// Handler functions to call in the patched entry/exit sled.
+extern atomic_uintptr_t XRayPatchedFunction;
+extern atomic_uintptr_t XRayArgLogger;
+extern atomic_uintptr_t XRayPatchedCustomEvent;
+extern atomic_uintptr_t XRayPatchedTypedEvent;
+
+static int __xray_object_id{-1};
+
+// Note: .preinit_array initialization does not work for DSOs
+__attribute__((constructor(0))) static void
+__xray_init_dso() XRAY_NEVER_INSTRUMENT {
+  // Register sleds in main XRay runtime.
+  __xray_object_id =
+      __xray_register_dso(__start_xray_instr_map, __stop_xray_instr_map,
+                          __start_xray_fn_idx, __stop_xray_fn_idx, {});
+}
+
+__attribute__((destructor(0))) static void
+__xray_finalize_dso() XRAY_NEVER_INSTRUMENT {
+  // Inform the main runtime that this DSO is no longer used.
+  __xray_deregister_dso(__xray_object_id);
+}
diff --git a/compiler-rt/lib/xray/xray_hexagon.cpp b/compiler-rt/lib/xray/xray_hexagon.cpp
index 7f127b2b499c..1c07bb046903 100644
--- a/compiler-rt/lib/xray/xray_hexagon.cpp
+++ b/compiler-rt/lib/xray/xray_hexagon.cpp
@@ -135,18 +135,23 @@ inline static bool patchSled(const bool Enable, const uint32_t FuncId,
 
 bool patchFunctionEntry(const bool Enable, const uint32_t FuncId,
                         const XRaySledEntry &Sled,
-                        void (*Trampoline)()) XRAY_NEVER_INSTRUMENT {
+                        const XRayTrampolines &Trampolines,
+                        bool LogArgs) XRAY_NEVER_INSTRUMENT {
+  auto Trampoline =
+      LogArgs ? Trampolines.LogArgsTrampoline : Trampolines.EntryTrampoline;
   return patchSled(Enable, FuncId, Sled, Trampoline);
 }
 
-bool patchFunctionExit(const bool Enable, const uint32_t FuncId,
-                       const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
-  return patchSled(Enable, FuncId, Sled, __xray_FunctionExit);
+bool patchFunctionExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+  return patchSled(Enable, FuncId, Sled, Trampolines.ExitTrampoline);
 }
 
-bool patchFunctionTailExit(const bool Enable, const uint32_t FuncId,
-                           const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
-  return patchSled(Enable, FuncId, Sled, __xray_FunctionExit);
+bool patchFunctionTailExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+  return patchSled(Enable, FuncId, Sled, Trampolines.TailExitTrampoline);
 }
 
 bool patchCustomEvent(const bool Enable, const uint32_t FuncId,
diff --git a/compiler-rt/lib/xray/xray_init.cpp b/compiler-rt/lib/xray/xray_init.cpp
index f22a31b95686..020bfe52b532 100644
--- a/compiler-rt/lib/xray/xray_init.cpp
+++ b/compiler-rt/lib/xray/xray_init.cpp
@@ -16,6 +16,8 @@
 #include <unistd.h>
 
 #include "sanitizer_common/sanitizer_common.h"
+#include "xray/xray_interface.h"
+#include "xray_allocator.h"
 #include "xray_defs.h"
 #include "xray_flags.h"
 #include "xray_interface_internal.h"
@@ -28,7 +30,7 @@ extern const XRayFunctionSledIndex __start_xray_fn_idx[] __attribute__((weak));
 extern const XRayFunctionSledIndex __stop_xray_fn_idx[] __attribute__((weak));
 
 #if SANITIZER_APPLE
-// HACK: This is a temporary workaround to make XRay build on 
+// HACK: This is a temporary workaround to make XRay build on
 // Darwin, but it will probably not work at runtime.
 const XRaySledEntry __start_xray_instr_map[] = {};
 extern const XRaySledEntry __stop_xray_instr_map[] = {};
@@ -43,14 +45,16 @@ using namespace __xray;
 // the weak symbols defined above (__start_xray_inst_map and
 // __stop_xray_instr_map) to initialise the instrumentation map that XRay uses
 // for runtime patching/unpatching of instrumentation points.
-//
-// FIXME: Support DSO instrumentation maps too. The current solution only works
-// for statically linked executables.
 atomic_uint8_t XRayInitialized{0};
 
 // This should always be updated before XRayInitialized is updated.
 SpinMutex XRayInstrMapMutex;
-XRaySledMap XRayInstrMap;
+
+//  Contains maps for the main executable as well as DSOs.
+XRaySledMap *XRayInstrMaps;
+
+// Number of binary objects registered.
+atomic_uint32_t XRayNumObjects{0};
 
 // Global flag to determine whether the flags have been initialized.
 atomic_uint8_t XRayFlagsInitialized{0};
@@ -58,6 +62,63 @@ atomic_uint8_t XRayFlagsInitialized{0};
 // A mutex to allow only one thread to initialize the XRay data structures.
 SpinMutex XRayInitMutex;
 
+// Registers XRay sleds and trampolines coming from the main executable or one
+// of the linked DSOs.
+// Returns the object ID if registration is successful, -1 otherwise.
+int32_t
+__xray_register_sleds(const XRaySledEntry *SledsBegin,
+                      const XRaySledEntry *SledsEnd,
+                      const XRayFunctionSledIndex *FnIndexBegin,
+                      const XRayFunctionSledIndex *FnIndexEnd, bool FromDSO,
+                      XRayTrampolines Trampolines) XRAY_NEVER_INSTRUMENT {
+  if (!SledsBegin || !SledsEnd) {
+    Report("Invalid XRay sleds.\n");
+    return -1;
+  }
+  XRaySledMap SledMap;
+  SledMap.FromDSO = FromDSO;
+  SledMap.Loaded = true;
+  SledMap.Trampolines = Trampolines;
+  SledMap.Sleds = SledsBegin;
+  SledMap.Entries = SledsEnd - SledsBegin;
+  if (FnIndexBegin != nullptr) {
+    SledMap.SledsIndex = FnIndexBegin;
+    SledMap.Functions = FnIndexEnd - FnIndexBegin;
+  } else {
+    size_t CountFunctions = 0;
+    uint64_t LastFnAddr = 0;
+
+    for (std::size_t I = 0; I < SledMap.Entries; I++) {
+      const auto &Sled = SledMap.Sleds[I];
+      const auto Function = Sled.function();
+      if (Function != LastFnAddr) {
+        CountFunctions++;
+        LastFnAddr = Function;
+      }
+    }
+    SledMap.SledsIndex = nullptr;
+    SledMap.Functions = CountFunctions;
+  }
+  if (SledMap.Functions >= XRayMaxFunctions) {
+    Report("Too many functions! Maximum is %ld\n", XRayMaxFunctions);
+    return -1;
+  }
+
+  if (Verbosity())
+    Report("Registering %d new functions!\n", SledMap.Functions);
+
+  {
+    SpinMutexLock Guard(&XRayInstrMapMutex);
+    auto Idx = atomic_fetch_add(&XRayNumObjects, 1, memory_order_acq_rel);
+    if (Idx >= XRayMaxObjects) {
+      Report("Too many objects registered! Maximum is %ld\n", XRayMaxObjects);
+      return -1;
+    }
+    XRayInstrMaps[Idx] = std::move(SledMap);
+    return Idx;
+  }
+}
+
 // __xray_init() will do the actual loading of the current process' memory map
 // and then proceed to look for the .xray_instr_map section/segment.
 void __xray_init() XRAY_NEVER_INSTRUMENT {
@@ -80,29 +141,21 @@ void __xray_init() XRAY_NEVER_INSTRUMENT {
     return;
   }
 
-  {
-    SpinMutexLock Guard(&XRayInstrMapMutex);
-    XRayInstrMap.Sleds = __start_xray_instr_map;
-    XRayInstrMap.Entries = __stop_xray_instr_map - __start_xray_instr_map;
-    if (__start_xray_fn_idx != nullptr) {
-      XRayInstrMap.SledsIndex = __start_xray_fn_idx;
-      XRayInstrMap.Functions = __stop_xray_fn_idx - __start_xray_fn_idx;
-    } else {
-      size_t CountFunctions = 0;
-      uint64_t LastFnAddr = 0;
-
-      for (std::size_t I = 0; I < XRayInstrMap.Entries; I++) {
-        const auto &Sled = XRayInstrMap.Sleds[I];
-        const auto Function = Sled.function();
-        if (Function != LastFnAddr) {
-          CountFunctions++;
-          LastFnAddr = Function;
-        }
-      }
+  atomic_store(&XRayNumObjects, 0, memory_order_release);
 
-      XRayInstrMap.Functions = CountFunctions;
-    }
+  // Pre-allocation takes up approx. 5kB for XRayMaxObjects=64.
+  XRayInstrMaps = allocateBuffer<XRaySledMap>(XRayMaxObjects);
+
+  int MainBinaryId =
+      __xray_register_sleds(__start_xray_instr_map, __stop_xray_instr_map,
+                            __start_xray_fn_idx, __stop_xray_fn_idx, false, {});
+
+  // The executable should always get ID 0.
+  if (MainBinaryId != 0) {
+    Report("Registering XRay sleds failed.\n");
+    return;
   }
+
   atomic_store(&XRayInitialized, true, memory_order_release);
 
 #ifndef XRAY_NO_PREINIT
@@ -111,6 +164,84 @@ void __xray_init() XRAY_NEVER_INSTRUMENT {
 #endif
 }
 
+// Registers XRay sleds and trampolines of an instrumented DSO.
+// Returns the object ID if registration is successful, -1 otherwise.
+//
+// Default visibility is hidden, so we have to explicitly make it visible to
+// DSO.
+SANITIZER_INTERFACE_ATTRIBUTE int32_t __xray_register_dso(
+    const XRaySledEntry *SledsBegin, const XRaySledEntry *SledsEnd,
+    const XRayFunctionSledIndex *FnIndexBegin,
+    const XRayFunctionSledIndex *FnIndexEnd,
+    XRayTrampolines Trampolines) XRAY_NEVER_INSTRUMENT {
+  // Make sure XRay has been initialized in the main executable.
+  __xray_init();
+
+  if (__xray_num_objects() == 0) {
+    if (Verbosity())
+      Report("No XRay instrumentation map in main executable. Not initializing "
+             "XRay for DSO.\n");
+    return -1;
+  }
+
+  // Register sleds in global map.
+  int ObjId = __xray_register_sleds(SledsBegin, SledsEnd, FnIndexBegin,
+                                    FnIndexEnd, true, Trampolines);
+
+#ifndef XRAY_NO_PREINIT
+  if (ObjId >= 0 && flags()->patch_premain)
+    __xray_patch_object(ObjId);
+#endif
+
+  return ObjId;
+}
+
+// Deregisters a DSO from the main XRay runtime.
+// Called from the DSO-local runtime when the library is unloaded (e.g. if
+// dlclose is called).
+// Returns true if the object ID is valid and the DSO was successfully
+// deregistered.
+SANITIZER_INTERFACE_ATTRIBUTE bool
+__xray_deregister_dso(int32_t ObjId) XRAY_NEVER_INSTRUMENT {
+
+  if (!atomic_load(&XRayInitialized, memory_order_acquire)) {
+    if (Verbosity())
+      Report("XRay has not been initialized. Cannot deregister DSO.\n");
+    return false;
+  }
+
+  if (ObjId <= 0 || static_cast<uint32_t>(ObjId) >= __xray_num_objects()) {
+    if (Verbosity())
+      Report("Can't deregister object with ID %d: ID is invalid.\n", ObjId);
+    return false;
+  }
+
+  {
+    SpinMutexLock Guard(&XRayInstrMapMutex);
+    auto &Entry = XRayInstrMaps[ObjId];
+    if (!Entry.FromDSO) {
+      if (Verbosity())
+        Report("Can't deregister object with ID %d: object does not correspond "
+               "to a shared library.\n",
+               ObjId);
+      return false;
+    }
+    if (!Entry.Loaded) {
+      if (Verbosity())
+        Report("Can't deregister object with ID %d: object is not loaded.\n",
+               ObjId);
+      return true;
+    }
+    // Mark DSO as unloaded. No need to unpatch.
+    Entry.Loaded = false;
+  }
+
+  if (Verbosity())
+    Report("Deregistered object with ID %d.\n", ObjId);
+
+  return true;
+}
+
 // FIXME: Make check-xray tests work on FreeBSD without
 // SANITIZER_CAN_USE_PREINIT_ARRAY.
 // See sanitizer_internal_defs.h where the macro is defined.
diff --git a/compiler-rt/lib/xray/xray_interface.cpp b/compiler-rt/lib/xray/xray_interface.cpp
index 5839043fcb93..b6f0e6762f16 100644
--- a/compiler-rt/lib/xray/xray_interface.cpp
+++ b/compiler-rt/lib/xray/xray_interface.cpp
@@ -36,7 +36,8 @@
 
 extern __sanitizer::SpinMutex XRayInstrMapMutex;
 extern __sanitizer::atomic_uint8_t XRayInitialized;
-extern __xray::XRaySledMap XRayInstrMap;
+extern __xray::XRaySledMap *XRayInstrMaps;
+extern __sanitizer::atomic_uint32_t XRayNumObjects;
 
 namespace __xray {
 
@@ -61,16 +62,16 @@ static const int16_t cSledLength = 20;
 #endif /* CPU architecture */
 
 // This is the function to call when we encounter the entry or exit sleds.
-atomic_uintptr_t XRayPatchedFunction{0};
+atomic_uintptr_t XRayPatchedFunction SANITIZER_INTERFACE_ATTRIBUTE{0};
 
 // This is the function to call from the arg1-enabled sleds/trampolines.
-atomic_uintptr_t XRayArgLogger{0};
+atomic_uintptr_t XRayArgLogger SANITIZER_INTERFACE_ATTRIBUTE{0};
 
 // This is the function to call when we encounter a custom event log call.
-atomic_uintptr_t XRayPatchedCustomEvent{0};
+atomic_uintptr_t XRayPatchedCustomEvent SANITIZER_INTERFACE_ATTRIBUTE{0};
 
 // This is the function to call when we encounter a typed event log call.
-atomic_uintptr_t XRayPatchedTypedEvent{0};
+atomic_uintptr_t XRayPatchedTypedEvent SANITIZER_INTERFACE_ATTRIBUTE{0};
 
 // This is the global status to determine whether we are currently
 // patching/unpatching.
@@ -150,21 +151,32 @@ public:
 
 namespace {
 
-bool patchSled(const XRaySledEntry &Sled, bool Enable,
-               int32_t FuncId) XRAY_NEVER_INSTRUMENT {
+bool isObjectLoaded(int32_t ObjId) {
+  SpinMutexLock Guard(&XRayInstrMapMutex);
+  if (ObjId < 0 || static_cast<uint32_t>(ObjId) >=
+                       atomic_load(&XRayNumObjects, memory_order_acquire)) {
+    return false;
+  }
+  return XRayInstrMaps[ObjId].Loaded;
+}
+
+bool patchSled(const XRaySledEntry &Sled, bool Enable, int32_t FuncId,
+               const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
   bool Success = false;
   switch (Sled.Kind) {
   case XRayEntryType::ENTRY:
-    Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_FunctionEntry);
+    Success = patchFunctionEntry(Enable, FuncId, Sled, Trampolines,
+                                 /*LogArgs=*/false);
     break;
   case XRayEntryType::EXIT:
-    Success = patchFunctionExit(Enable, FuncId, Sled);
+    Success = patchFunctionExit(Enable, FuncId, Sled, Trampolines);
     break;
   case XRayEntryType::TAIL:
-    Success = patchFunctionTailExit(Enable, FuncId, Sled);
+    Success = patchFunctionTailExit(Enable, FuncId, Sled, Trampolines);
     break;
   case XRayEntryType::LOG_ARGS_ENTRY:
-    Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_ArgLoggerEntry);
+    Success = patchFunctionEntry(Enable, FuncId, Sled, Trampolines,
+                                 /*LogArgs=*/true);
     break;
   case XRayEntryType::CUSTOM_EVENT:
     Success = patchCustomEvent(Enable, FuncId, Sled);
@@ -205,10 +217,9 @@ findFunctionSleds(int32_t FuncId,
   return Index;
 }
 
-XRayPatchingStatus patchFunction(int32_t FuncId,
+XRayPatchingStatus patchFunction(int32_t FuncId, int32_t ObjId,
                                  bool Enable) XRAY_NEVER_INSTRUMENT {
-  if (!atomic_load(&XRayInitialized,
-                                memory_order_acquire))
+  if (!atomic_load(&XRayInitialized, memory_order_acquire))
     return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
 
   uint8_t NotPatching = false;
@@ -220,13 +231,24 @@ XRayPatchingStatus patchFunction(int32_t FuncId,
   XRaySledMap InstrMap;
   {
     SpinMutexLock Guard(&XRayInstrMapMutex);
-    InstrMap = XRayInstrMap;
+    if (ObjId < 0 || static_cast<uint32_t>(ObjId) >=
+                         atomic_load(&XRayNumObjects, memory_order_acquire)) {
+      Report("Unable to patch function: invalid sled map index: %d", ObjId);
+      return XRayPatchingStatus::FAILED;
+    }
+    InstrMap = XRayInstrMaps[ObjId];
   }
 
   // If we don't have an index, we can't patch individual functions.
   if (InstrMap.Functions == 0)
     return XRayPatchingStatus::NOT_INITIALIZED;
 
+  // Check if the corresponding DSO has been unloaded.
+  if (!InstrMap.Loaded) {
+    Report("Invalid function id provided: %d\n", FuncId);
+    return XRayPatchingStatus::NOT_INITIALIZED;
+  }
+
   // FuncId must be a positive number, less than the number of functions
   // instrumented.
   if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
@@ -234,6 +256,8 @@ XRayPatchingStatus patchFunction(int32_t FuncId,
     return XRayPatchingStatus::FAILED;
   }
 
+  auto PackedId = __xray::MakePackedId(FuncId, ObjId);
+
   // Now we patch ths sleds for this specific function.
   XRayFunctionSledIndex SledRange;
   if (InstrMap.SledsIndex) {
@@ -242,13 +266,13 @@ XRayPatchingStatus patchFunction(int32_t FuncId,
   } else {
     SledRange = findFunctionSleds(FuncId, InstrMap);
   }
+
   auto *f = SledRange.Begin;
   bool SucceedOnce = false;
   for (size_t i = 0; i != SledRange.Size; ++i)
-    SucceedOnce |= patchSled(f[i], Enable, FuncId);
+    SucceedOnce |= patchSled(f[i], Enable, PackedId, InstrMap.Trampolines);
 
-  atomic_store(&XRayPatching, false,
-                            memory_order_release);
+  atomic_store(&XRayPatching, false, memory_order_release);
 
   if (!SucceedOnce) {
     Report("Failed patching any sled for function '%d'.", FuncId);
@@ -261,32 +285,31 @@ XRayPatchingStatus patchFunction(int32_t FuncId,
 // controlPatching implements the common internals of the patching/unpatching
 // implementation. |Enable| defines whether we're enabling or disabling the
 // runtime XRay instrumentation.
-XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
-  if (!atomic_load(&XRayInitialized,
-                                memory_order_acquire))
-    return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
-
-  uint8_t NotPatching = false;
-  if (!atomic_compare_exchange_strong(
-          &XRayPatching, &NotPatching, true, memory_order_acq_rel))
-    return XRayPatchingStatus::ONGOING; // Already patching.
-
-  uint8_t PatchingSuccess = false;
-  auto XRayPatchingStatusResetter =
-      at_scope_exit([&PatchingSuccess] {
-        if (!PatchingSuccess)
-          atomic_store(&XRayPatching, false,
-                                    memory_order_release);
-      });
-
+// This function should only be called after ensuring that XRay is initialized
+// and no other thread is currently patching.
+XRayPatchingStatus controlPatchingObjectUnchecked(bool Enable, int32_t ObjId) {
   XRaySledMap InstrMap;
   {
     SpinMutexLock Guard(&XRayInstrMapMutex);
-    InstrMap = XRayInstrMap;
+    if (ObjId < 0 || static_cast<uint32_t>(ObjId) >=
+                         atomic_load(&XRayNumObjects, memory_order_acquire)) {
+      Report("Unable to patch functions: invalid sled map index: %d\n", ObjId);
+      return XRayPatchingStatus::FAILED;
+    }
+    InstrMap = XRayInstrMaps[ObjId];
   }
   if (InstrMap.Entries == 0)
     return XRayPatchingStatus::NOT_INITIALIZED;
 
+  if (Verbosity())
+    Report("Patching object %d with %d functions.\n", ObjId, InstrMap.Entries);
+
+  // Check if the corresponding DSO has been unloaded.
+  if (!InstrMap.Loaded) {
+    Report("Object is not loaded at index: %d\n", ObjId);
+    return XRayPatchingStatus::FAILED;
+  }
+
   uint32_t FuncId = 1;
   uint64_t CurFun = 0;
 
@@ -336,20 +359,94 @@ XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
       ++FuncId;
       CurFun = F;
     }
-    patchSled(Sled, Enable, FuncId);
+    auto PackedId = __xray::MakePackedId(FuncId, ObjId);
+    patchSled(Sled, Enable, PackedId, InstrMap.Trampolines);
   }
-  atomic_store(&XRayPatching, false,
-                            memory_order_release);
-  PatchingSuccess = true;
+  atomic_store(&XRayPatching, false, memory_order_release);
   return XRayPatchingStatus::SUCCESS;
 }
 
-XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId,
+// Controls patching for all registered objects.
+// Returns: SUCCESS, if patching succeeds for all objects.
+//          NOT_INITIALIZED, if one or more objects returned NOT_INITIALIZED
+//             but none failed.
+//          FAILED, if patching of one or more objects failed.
+XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
+  if (!atomic_load(&XRayInitialized, memory_order_acquire))
+    return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
+
+  uint8_t NotPatching = false;
+  if (!atomic_compare_exchange_strong(&XRayPatching, &NotPatching, true,
+                                      memory_order_acq_rel))
+    return XRayPatchingStatus::ONGOING; // Already patching.
+
+  auto XRayPatchingStatusResetter = at_scope_exit(
+      [] { atomic_store(&XRayPatching, false, memory_order_release); });
+
+  unsigned NumObjects = __xray_num_objects();
+
+  XRayPatchingStatus CombinedStatus{NOT_INITIALIZED};
+  for (unsigned I = 0; I < NumObjects; ++I) {
+    if (!isObjectLoaded(I))
+      continue;
+    auto LastStatus = controlPatchingObjectUnchecked(Enable, I);
+    switch (LastStatus) {
+    case SUCCESS:
+      if (CombinedStatus == NOT_INITIALIZED)
+        CombinedStatus = SUCCESS;
+      break;
+    case FAILED:
+      // Report failure, but try to patch the remaining objects
+      CombinedStatus = FAILED;
+      break;
+    case NOT_INITIALIZED:
+      // XRay has been initialized but there are no sleds available for this
+      // object. Try to patch remaining objects.
+      if (CombinedStatus != FAILED)
+        CombinedStatus = NOT_INITIALIZED;
+      break;
+    case ONGOING:
+      UNREACHABLE("Status ONGOING should not appear at this point");
+    }
+  }
+  return CombinedStatus;
+}
+
+// Controls patching for one object.
+XRayPatchingStatus controlPatching(bool Enable,
+                                   int32_t ObjId) XRAY_NEVER_INSTRUMENT {
+
+  if (!atomic_load(&XRayInitialized, memory_order_acquire))
+    return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
+
+  uint8_t NotPatching = false;
+  if (!atomic_compare_exchange_strong(&XRayPatching, &NotPatching, true,
+                                      memory_order_acq_rel))
+    return XRayPatchingStatus::ONGOING; // Already patching.
+
+  auto XRayPatchingStatusResetter = at_scope_exit(
+      [] { atomic_store(&XRayPatching, false, memory_order_release); });
+
+  return controlPatchingObjectUnchecked(Enable, ObjId);
+}
+
+XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId, int32_t ObjId,
                                             bool Enable) XRAY_NEVER_INSTRUMENT {
   XRaySledMap InstrMap;
   {
     SpinMutexLock Guard(&XRayInstrMapMutex);
-    InstrMap = XRayInstrMap;
+    if (ObjId < 0 || static_cast<uint32_t>(ObjId) >=
+                         atomic_load(&XRayNumObjects, memory_order_acquire)) {
+      Report("Unable to patch function: invalid sled map index: %d\n", ObjId);
+      return XRayPatchingStatus::FAILED;
+    }
+    InstrMap = XRayInstrMaps[ObjId];
+  }
+
+  // Check if the corresponding DSO has been unloaded.
+  if (!InstrMap.Loaded) {
+    Report("Object is not loaded at index: %d\n", ObjId);
+    return XRayPatchingStatus::FAILED;
   }
 
   // FuncId must be a positive number, less than the number of functions
@@ -398,7 +495,7 @@ XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId,
     Report("Failed mprotect: %d\n", errno);
     return XRayPatchingStatus::FAILED;
   }
-  return patchFunction(FuncId, Enable);
+  return patchFunction(FuncId, ObjId, Enable);
 }
 
 } // namespace
@@ -412,12 +509,10 @@ using namespace __xray;
 
 int __xray_set_handler(void (*entry)(int32_t,
                                      XRayEntryType)) XRAY_NEVER_INSTRUMENT {
-  if (atomic_load(&XRayInitialized,
-                               memory_order_acquire)) {
+  if (atomic_load(&XRayInitialized, memory_order_acquire)) {
 
     atomic_store(&__xray::XRayPatchedFunction,
-                              reinterpret_cast<uintptr_t>(entry),
-                              memory_order_release);
+                 reinterpret_cast<uintptr_t>(entry), memory_order_release);
     return 1;
   }
   return 0;
@@ -425,11 +520,9 @@ int __xray_set_handler(void (*entry)(int32_t,
 
 int __xray_set_customevent_handler(void (*entry)(void *, size_t))
     XRAY_NEVER_INSTRUMENT {
-  if (atomic_load(&XRayInitialized,
-                               memory_order_acquire)) {
+  if (atomic_load(&XRayInitialized, memory_order_acquire)) {
     atomic_store(&__xray::XRayPatchedCustomEvent,
-                              reinterpret_cast<uintptr_t>(entry),
-                              memory_order_release);
+                 reinterpret_cast<uintptr_t>(entry), memory_order_release);
     return 1;
   }
   return 0;
@@ -437,11 +530,9 @@ int __xray_set_customevent_handler(void (*entry)(void *, size_t))
 
 int __xray_set_typedevent_handler(void (*entry)(size_t, const void *,
                                                 size_t)) XRAY_NEVER_INSTRUMENT {
-  if (atomic_load(&XRayInitialized,
-                               memory_order_acquire)) {
+  if (atomic_load(&XRayInitialized, memory_order_acquire)) {
     atomic_store(&__xray::XRayPatchedTypedEvent,
-                              reinterpret_cast<uintptr_t>(entry),
-                              memory_order_release);
+                 reinterpret_cast<uintptr_t>(entry), memory_order_release);
     return 1;
   }
   return 0;
@@ -474,39 +565,78 @@ XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT {
   return controlPatching(true);
 }
 
+XRayPatchingStatus __xray_patch_object(int32_t ObjId) XRAY_NEVER_INSTRUMENT {
+  return controlPatching(true, ObjId);
+}
+
 XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT {
   return controlPatching(false);
 }
 
+XRayPatchingStatus __xray_unpatch_object(int32_t ObjId) XRAY_NEVER_INSTRUMENT {
+  return controlPatching(false, ObjId);
+}
+
 XRayPatchingStatus __xray_patch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
-  return mprotectAndPatchFunction(FuncId, true);
+  auto Ids = __xray::UnpackId(FuncId);
+  auto ObjId = Ids.first;
+  auto FnId = Ids.second;
+  return mprotectAndPatchFunction(FnId, ObjId, true);
+}
+
+XRayPatchingStatus
+__xray_patch_function_in_object(int32_t FuncId,
+                                int32_t ObjId) XRAY_NEVER_INSTRUMENT {
+  return mprotectAndPatchFunction(FuncId, ObjId, true);
 }
 
 XRayPatchingStatus
 __xray_unpatch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
-  return mprotectAndPatchFunction(FuncId, false);
+  auto Ids = __xray::UnpackId(FuncId);
+  auto ObjId = Ids.first;
+  auto FnId = Ids.second;
+  return mprotectAndPatchFunction(FnId, ObjId, false);
+}
+
+XRayPatchingStatus
+__xray_unpatch_function_in_object(int32_t FuncId,
+                                  int32_t ObjId) XRAY_NEVER_INSTRUMENT {
+  return mprotectAndPatchFunction(FuncId, ObjId, false);
 }
 
 int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) {
-  if (!atomic_load(&XRayInitialized,
-                                memory_order_acquire))
+  if (!atomic_load(&XRayInitialized, memory_order_acquire))
     return 0;
 
   // A relaxed write might not be visible even if the current thread gets
   // scheduled on a different CPU/NUMA node.  We need to wait for everyone to
   // have this handler installed for consistency of collected data across CPUs.
   atomic_store(&XRayArgLogger, reinterpret_cast<uint64_t>(entry),
-                            memory_order_release);
+               memory_order_release);
   return 1;
 }
 
 int __xray_remove_handler_arg1() { return __xray_set_handler_arg1(nullptr); }
 
-uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
+uintptr_t
+__xray_function_address(int32_t CombinedFuncId) XRAY_NEVER_INSTRUMENT {
+  auto Ids = __xray::UnpackId(CombinedFuncId);
+  return __xray_function_address_in_object(Ids.second, Ids.first);
+}
+
+uintptr_t __xray_function_address_in_object(int32_t FuncId, int32_t ObjId)
+    XRAY_NEVER_INSTRUMENT {
   XRaySledMap InstrMap;
   {
     SpinMutexLock Guard(&XRayInstrMapMutex);
-    InstrMap = XRayInstrMap;
+    auto count = atomic_load(&XRayNumObjects, memory_order_acquire);
+    if (ObjId < 0 || static_cast<uint32_t>(ObjId) >= count) {
+      Report("Unable to determine function address: invalid sled map index %d "
+             "(size is %d)\n",
+             ObjId, (int)count);
+      return 0;
+    }
+    InstrMap = XRayInstrMaps[ObjId];
   }
 
   if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions)
@@ -525,6 +655,30 @@ uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
 }
 
 size_t __xray_max_function_id() XRAY_NEVER_INSTRUMENT {
+  return __xray_max_function_id_in_object(0);
+}
+
+size_t __xray_max_function_id_in_object(int32_t ObjId) XRAY_NEVER_INSTRUMENT {
+  SpinMutexLock Guard(&XRayInstrMapMutex);
+  if (ObjId < 0 || static_cast<uint32_t>(ObjId) >=
+                       atomic_load(&XRayNumObjects, memory_order_acquire))
+    return 0;
+  return XRayInstrMaps[ObjId].Functions;
+}
+
+size_t __xray_num_objects() XRAY_NEVER_INSTRUMENT {
   SpinMutexLock Guard(&XRayInstrMapMutex);
-  return XRayInstrMap.Functions;
+  return atomic_load(&XRayNumObjects, memory_order_acquire);
+}
+
+int32_t __xray_unpack_function_id(int32_t PackedId) {
+  return __xray::UnpackId(PackedId).second;
+}
+
+int32_t __xray_unpack_object_id(int32_t PackedId) {
+  return __xray::UnpackId(PackedId).first;
+}
+
+int32_t __xray_pack_id(int32_t FuncId, int32_t ObjId) {
+  return __xray::MakePackedId(FuncId, ObjId);
 }
diff --git a/compiler-rt/lib/xray/xray_interface_internal.h b/compiler-rt/lib/xray/xray_interface_internal.h
index 80c07c167f64..a8cfe0fde84d 100644
--- a/compiler-rt/lib/xray/xray_interface_internal.h
+++ b/compiler-rt/lib/xray/xray_interface_internal.h
@@ -18,6 +18,18 @@
 #include "xray/xray_interface.h"
 #include <cstddef>
 #include <cstdint>
+#include <utility>
+
+extern "C" {
+// The following functions have to be defined in assembler, on a per-platform
+// basis. See xray_trampoline_*.S files for implementations.
+extern void __xray_FunctionEntry();
+extern void __xray_FunctionExit();
+extern void __xray_FunctionTailExit();
+extern void __xray_ArgLoggerEntry();
+extern void __xray_CustomEvent();
+extern void __xray_TypedEvent();
+}
 
 extern "C" {
 
@@ -67,36 +79,72 @@ struct XRayFunctionSledIndex {
                                                    uintptr_t(Begin));
   }
 };
+
+struct XRayTrampolines {
+  void (*EntryTrampoline)();
+  void (*ExitTrampoline)();
+  void (*TailExitTrampoline)();
+  void (*LogArgsTrampoline)();
+
+  XRayTrampolines() {
+    // These resolve to the definitions in the respective executable or DSO.
+    EntryTrampoline = __xray_FunctionEntry;
+    ExitTrampoline = __xray_FunctionExit;
+    TailExitTrampoline = __xray_FunctionTailExit;
+    LogArgsTrampoline = __xray_ArgLoggerEntry;
+  }
+};
+
+extern int32_t __xray_register_dso(const XRaySledEntry *SledsBegin,
+                                   const XRaySledEntry *SledsEnd,
+                                   const XRayFunctionSledIndex *FnIndexBegin,
+                                   const XRayFunctionSledIndex *FnIndexEnd,
+                                   XRayTrampolines Trampolines);
+
+extern bool __xray_deregister_dso(int32_t ObjId);
 }
 
 namespace __xray {
 
+constexpr uint32_t XRayNFnBits = 24;
+constexpr uint32_t XRayNObjBits = 8;
+
+constexpr uint32_t XRayFnBitMask = 0x00FFFFFF;
+constexpr uint32_t XRayObjBitMask = 0xFF000000;
+
+constexpr size_t XRayMaxFunctions = 1 << XRayNFnBits;
+constexpr size_t XRayMaxObjects = 1 << XRayNObjBits;
+
+inline int32_t MakePackedId(int32_t FnId, int32_t ObjId) {
+  return ((ObjId << XRayNFnBits) & XRayObjBitMask) | (FnId & XRayFnBitMask);
+}
+
+inline std::pair<int32_t, int32_t> UnpackId(int32_t PackedId) {
+  uint32_t ObjId = (PackedId & XRayObjBitMask) >> XRayNFnBits;
+  uint32_t FnId = PackedId & XRayFnBitMask;
+  return {ObjId, FnId};
+}
+
 struct XRaySledMap {
   const XRaySledEntry *Sleds;
   size_t Entries;
   const XRayFunctionSledIndex *SledsIndex;
   size_t Functions;
+  XRayTrampolines Trampolines;
+  bool FromDSO;
+  bool Loaded;
 };
 
 bool patchFunctionEntry(bool Enable, uint32_t FuncId, const XRaySledEntry &Sled,
-                        void (*Trampoline)());
-bool patchFunctionExit(bool Enable, uint32_t FuncId, const XRaySledEntry &Sled);
+                        const XRayTrampolines &Trampolines, bool LogArgs);
+bool patchFunctionExit(bool Enable, uint32_t FuncId, const XRaySledEntry &Sled,
+                       const XRayTrampolines &Trampolines);
 bool patchFunctionTailExit(bool Enable, uint32_t FuncId,
-                           const XRaySledEntry &Sled);
+                           const XRaySledEntry &Sled,
+                           const XRayTrampolines &Trampolines);
 bool patchCustomEvent(bool Enable, uint32_t FuncId, const XRaySledEntry &Sled);
 bool patchTypedEvent(bool Enable, uint32_t FuncId, const XRaySledEntry &Sled);
 
 } // namespace __xray
 
-extern "C" {
-// The following functions have to be defined in assembler, on a per-platform
-// basis. See xray_trampoline_*.S files for implementations.
-extern void __xray_FunctionEntry();
-extern void __xray_FunctionExit();
-extern void __xray_FunctionTailExit();
-extern void __xray_ArgLoggerEntry();
-extern void __xray_CustomEvent();
-extern void __xray_TypedEvent();
-}
-
 #endif
diff --git a/compiler-rt/lib/xray/xray_loongarch64.cpp b/compiler-rt/lib/xray/xray_loongarch64.cpp
index b839adba00d2..b72417d4f2aa 100644
--- a/compiler-rt/lib/xray/xray_loongarch64.cpp
+++ b/compiler-rt/lib/xray/xray_loongarch64.cpp
@@ -126,20 +126,25 @@ static inline bool patchSled(const bool Enable, const uint32_t FuncId,
 
 bool patchFunctionEntry(const bool Enable, const uint32_t FuncId,
                         const XRaySledEntry &Sled,
-                        void (*Trampoline)()) XRAY_NEVER_INSTRUMENT {
+                        const XRayTrampolines &Trampolines,
+                        bool LogArgs) XRAY_NEVER_INSTRUMENT {
+  auto Trampoline =
+      LogArgs ? Trampolines.LogArgsTrampoline : Trampolines.EntryTrampoline;
   return patchSled(Enable, FuncId, Sled, Trampoline);
 }
 
-bool patchFunctionExit(const bool Enable, const uint32_t FuncId,
-                       const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
-  return patchSled(Enable, FuncId, Sled, __xray_FunctionExit);
+bool patchFunctionExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+  return patchSled(Enable, FuncId, Sled, Trampolines.ExitTrampoline);
 }
 
-bool patchFunctionTailExit(const bool Enable, const uint32_t FuncId,
-                           const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+bool patchFunctionTailExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
   // TODO: In the future we'd need to distinguish between non-tail exits and
   // tail exits for better information preservation.
-  return patchSled(Enable, FuncId, Sled, __xray_FunctionExit);
+  return patchSled(Enable, FuncId, Sled, Trampolines.ExitTrampoline);
 }
 
 bool patchCustomEvent(const bool Enable, const uint32_t FuncId,
@@ -158,3 +163,7 @@ bool patchTypedEvent(const bool Enable, const uint32_t FuncId,
 extern "C" void __xray_ArgLoggerEntry() XRAY_NEVER_INSTRUMENT {
   // TODO: This will have to be implemented in the trampoline assembly file.
 }
+
+extern "C" void __xray_FunctionTailExit() XRAY_NEVER_INSTRUMENT {
+  // FIXME: this will have to be implemented in the trampoline assembly file
+}
diff --git a/compiler-rt/lib/xray/xray_mips.cpp b/compiler-rt/lib/xray/xray_mips.cpp
index dc9e837a555d..4d2a7c82bddb 100644
--- a/compiler-rt/lib/xray/xray_mips.cpp
+++ b/compiler-rt/lib/xray/xray_mips.cpp
@@ -136,20 +136,25 @@ inline static bool patchSled(const bool Enable, const uint32_t FuncId,
 
 bool patchFunctionEntry(const bool Enable, const uint32_t FuncId,
                         const XRaySledEntry &Sled,
-                        void (*Trampoline)()) XRAY_NEVER_INSTRUMENT {
+                        const XRayTrampolines &Trampolines,
+                        bool LogArgs) XRAY_NEVER_INSTRUMENT {
+  auto Trampoline =
+      LogArgs ? Trampolines.LogArgsTrampoline : Trampolines.EntryTrampoline;
   return patchSled(Enable, FuncId, Sled, Trampoline);
 }
 
-bool patchFunctionExit(const bool Enable, const uint32_t FuncId,
-                       const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
-  return patchSled(Enable, FuncId, Sled, __xray_FunctionExit);
+bool patchFunctionExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+  return patchSled(Enable, FuncId, Sled, Trampolines.ExitTrampoline);
 }
 
-bool patchFunctionTailExit(const bool Enable, const uint32_t FuncId,
-                           const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+bool patchFunctionTailExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
   // FIXME: In the future we'd need to distinguish between non-tail exits and
   // tail exits for better information preservation.
-  return patchSled(Enable, FuncId, Sled, __xray_FunctionExit);
+  return patchSled(Enable, FuncId, Sled, Trampolines.ExitTrampoline);
 }
 
 bool patchCustomEvent(const bool Enable, const uint32_t FuncId,
@@ -169,3 +174,7 @@ bool patchTypedEvent(const bool Enable, const uint32_t FuncId,
 extern "C" void __xray_ArgLoggerEntry() XRAY_NEVER_INSTRUMENT {
   // FIXME: this will have to be implemented in the trampoline assembly file
 }
+
+extern "C" void __xray_FunctionTailExit() XRAY_NEVER_INSTRUMENT {
+  // FIXME: this will have to be implemented in the trampoline assembly file
+}
\ No newline at end of file
diff --git a/compiler-rt/lib/xray/xray_mips64.cpp b/compiler-rt/lib/xray/xray_mips64.cpp
index 5b221bb6ddc0..a088a322cc15 100644
--- a/compiler-rt/lib/xray/xray_mips64.cpp
+++ b/compiler-rt/lib/xray/xray_mips64.cpp
@@ -144,20 +144,25 @@ inline static bool patchSled(const bool Enable, const uint32_t FuncId,
 
 bool patchFunctionEntry(const bool Enable, const uint32_t FuncId,
                         const XRaySledEntry &Sled,
-                        void (*Trampoline)()) XRAY_NEVER_INSTRUMENT {
+                        const XRayTrampolines &Trampolines,
+                        bool LogArgs) XRAY_NEVER_INSTRUMENT {
+  auto Trampoline =
+      LogArgs ? Trampolines.LogArgsTrampoline : Trampolines.EntryTrampoline;
   return patchSled(Enable, FuncId, Sled, Trampoline);
 }
 
-bool patchFunctionExit(const bool Enable, const uint32_t FuncId,
-                       const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
-  return patchSled(Enable, FuncId, Sled, __xray_FunctionExit);
+bool patchFunctionExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+  return patchSled(Enable, FuncId, Sled, Trampolines.ExitTrampoline);
 }
 
-bool patchFunctionTailExit(const bool Enable, const uint32_t FuncId,
-                           const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+bool patchFunctionTailExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
   // FIXME: In the future we'd need to distinguish between non-tail exits and
   // tail exits for better information preservation.
-  return patchSled(Enable, FuncId, Sled, __xray_FunctionExit);
+  return patchSled(Enable, FuncId, Sled, Trampolines.ExitTrampoline);
 }
 
 bool patchCustomEvent(const bool Enable, const uint32_t FuncId,
@@ -176,3 +181,7 @@ bool patchTypedEvent(const bool Enable, const uint32_t FuncId,
 extern "C" void __xray_ArgLoggerEntry() XRAY_NEVER_INSTRUMENT {
   // FIXME: this will have to be implemented in the trampoline assembly file
 }
+
+extern "C" void __xray_FunctionTailExit() XRAY_NEVER_INSTRUMENT {
+  // FIXME: this will have to be implemented in the trampoline assembly file
+}
diff --git a/compiler-rt/lib/xray/xray_powerpc64.cpp b/compiler-rt/lib/xray/xray_powerpc64.cpp
index c3553d848313..a568c568c418 100644
--- a/compiler-rt/lib/xray/xray_powerpc64.cpp
+++ b/compiler-rt/lib/xray/xray_powerpc64.cpp
@@ -51,7 +51,12 @@ namespace __xray {
 
 bool patchFunctionEntry(const bool Enable, uint32_t FuncId,
                         const XRaySledEntry &Sled,
-                        void (*Trampoline)()) XRAY_NEVER_INSTRUMENT {
+                        const XRayTrampolines &Trampolines,
+                        bool LogArgs) XRAY_NEVER_INSTRUMENT {
+  // TODO: Trampoline addresses are currently inserted at compile-time, using
+  //       __xray_FunctionEntry and __xray_FunctionExit only.
+  //       To support DSO instrumentation, trampolines have to be written during
+  //       patching (see implementation on X86_64, e.g.).
   const uint64_t Address = Sled.address();
   if (Enable) {
     // lis 0, FuncId[16..32]
@@ -68,8 +73,13 @@ bool patchFunctionEntry(const bool Enable, uint32_t FuncId,
   return true;
 }
 
-bool patchFunctionExit(const bool Enable, uint32_t FuncId,
-                       const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+bool patchFunctionExit(
+    const bool Enable, uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+  // TODO: Trampoline addresses are currently inserted at compile-time, using
+  //       __xray_FunctionEntry and __xray_FunctionExit only.
+  //       To support DSO instrumentation, trampolines have to be written during
+  //       patching (see implementation on X86_64, e.g.).
   const uint64_t Address = Sled.address();
   if (Enable) {
     // lis 0, FuncId[16..32]
@@ -86,9 +96,10 @@ bool patchFunctionExit(const bool Enable, uint32_t FuncId,
   return true;
 }
 
-bool patchFunctionTailExit(const bool Enable, const uint32_t FuncId,
-                           const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
-  return patchFunctionExit(Enable, FuncId, Sled);
+bool patchFunctionTailExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+  return patchFunctionExit(Enable, FuncId, Sled, Trampolines);
 }
 
 // FIXME: Maybe implement this better?
@@ -111,3 +122,10 @@ bool patchTypedEvent(const bool Enable, const uint32_t FuncId,
 extern "C" void __xray_ArgLoggerEntry() XRAY_NEVER_INSTRUMENT {
   // FIXME: this will have to be implemented in the trampoline assembly file
 }
+
+extern "C" void __xray_FunctionTailExit() XRAY_NEVER_INSTRUMENT {
+  // For PowerPC, calls to __xray_FunctionEntry and __xray_FunctionExit
+  // are statically inserted into the sled. Tail exits are handled like normal
+  // function exits. This trampoline is therefore not implemented.
+  // This stub is placed here to avoid linking issues.
+}
diff --git a/compiler-rt/lib/xray/xray_trampoline_x86_64.S b/compiler-rt/lib/xray/xray_trampoline_x86_64.S
index 01098f60eeab..0f480547b52c 100644
--- a/compiler-rt/lib/xray/xray_trampoline_x86_64.S
+++ b/compiler-rt/lib/xray/xray_trampoline_x86_64.S
@@ -107,6 +107,16 @@
 	.section __TEXT,__text
 #endif
 
+.macro LOAD_HANDLER_ADDR handler
+#if !defined(XRAY_PIC)
+	movq	ASM_SYMBOL(\handler)(%rip), %rax
+#else
+	movq	ASM_SYMBOL(\handler)@GOTPCREL(%rip), %rax
+	movq	(%rax), %rax
+#endif
+.endm
+
+
 //===----------------------------------------------------------------------===//
 
 	.globl ASM_SYMBOL(__xray_FunctionEntry)
@@ -121,7 +131,7 @@ ASM_SYMBOL(__xray_FunctionEntry):
 
 	// This load has to be atomic, it's concurrent with __xray_patch().
 	// On x86/amd64, a simple (type-aligned) MOV instruction is enough.
-	movq	ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax
+	LOAD_HANDLER_ADDR _ZN6__xray19XRayPatchedFunctionE
 	testq	%rax, %rax
 	je	LOCAL_LABEL(tmp0)
 
@@ -159,7 +169,7 @@ ASM_SYMBOL(__xray_FunctionExit):
 	movupd	%xmm1, 16(%rsp)
 	movq	%rax, 8(%rsp)
 	movq	%rdx, 0(%rsp)
-	movq	ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax
+	LOAD_HANDLER_ADDR _ZN6__xray19XRayPatchedFunctionE
 	testq %rax,%rax
 	je	LOCAL_LABEL(tmp2)
 
@@ -195,7 +205,7 @@ ASM_SYMBOL(__xray_FunctionTailExit):
 	SAVE_REGISTERS
 	ALIGN_STACK_16B
 
-	movq	ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax
+	LOAD_HANDLER_ADDR _ZN6__xray19XRayPatchedFunctionE
 	testq %rax,%rax
 	je	LOCAL_LABEL(tmp4)
 
@@ -224,12 +234,12 @@ ASM_SYMBOL(__xray_ArgLoggerEntry):
 	ALIGN_STACK_16B
 
 	// Again, these function pointer loads must be atomic; MOV is fine.
-	movq	ASM_SYMBOL(_ZN6__xray13XRayArgLoggerE)(%rip), %rax
+	LOAD_HANDLER_ADDR _ZN6__xray13XRayArgLoggerE
 	testq	%rax, %rax
 	jne	LOCAL_LABEL(arg1entryLog)
 
 	// If [arg1 logging handler] not set, defer to no-arg logging.
-	movq	ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax
+	LOAD_HANDLER_ADDR _ZN6__xray19XRayPatchedFunctionE
 	testq	%rax, %rax
 	je	LOCAL_LABEL(arg1entryFail)
 
@@ -268,7 +278,7 @@ ASM_SYMBOL(__xray_CustomEvent):
 
 	// We take two arguments to this trampoline, which should be in rdi	and rsi
 	// already.
-	movq ASM_SYMBOL(_ZN6__xray22XRayPatchedCustomEventE)(%rip), %rax
+	LOAD_HANDLER_ADDR _ZN6__xray22XRayPatchedCustomEventE
 	testq %rax,%rax
 	je LOCAL_LABEL(customEventCleanup)
 
@@ -293,7 +303,7 @@ ASM_SYMBOL(__xray_TypedEvent):
 
 	// We pass three arguments to this trampoline, which should be in rdi, rsi
 	// and rdx without our intervention.
-	movq ASM_SYMBOL(_ZN6__xray21XRayPatchedTypedEventE)(%rip), %rax
+	LOAD_HANDLER_ADDR _ZN6__xray21XRayPatchedTypedEventE
 	testq %rax,%rax
 	je LOCAL_LABEL(typedEventCleanup)
 
diff --git a/compiler-rt/lib/xray/xray_x86_64.cpp b/compiler-rt/lib/xray/xray_x86_64.cpp
index b9666a40861d..8542351ca268 100644
--- a/compiler-rt/lib/xray/xray_x86_64.cpp
+++ b/compiler-rt/lib/xray/xray_x86_64.cpp
@@ -122,7 +122,8 @@ static constexpr int64_t MaxOffset{std::numeric_limits<int32_t>::max()};
 
 bool patchFunctionEntry(const bool Enable, const uint32_t FuncId,
                         const XRaySledEntry &Sled,
-                        void (*Trampoline)()) XRAY_NEVER_INSTRUMENT {
+                        const XRayTrampolines &Trampolines,
+                        bool LogArgs) XRAY_NEVER_INSTRUMENT {
   // Here we do the dance of replacing the following sled:
   //
   // xray_sled_n:
@@ -144,6 +145,8 @@ bool patchFunctionEntry(const bool Enable, const uint32_t FuncId,
   // opcode and first operand.
   //
   // Prerequisite is to compute the relative offset to the trampoline's address.
+  auto Trampoline =
+      LogArgs ? Trampolines.LogArgsTrampoline : Trampolines.EntryTrampoline;
   const uint64_t Address = Sled.address();
   int64_t TrampolineOffset = reinterpret_cast<int64_t>(Trampoline) -
                              (static_cast<int64_t>(Address) + 11);
@@ -169,8 +172,9 @@ bool patchFunctionEntry(const bool Enable, const uint32_t FuncId,
   return true;
 }
 
-bool patchFunctionExit(const bool Enable, const uint32_t FuncId,
-                       const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+bool patchFunctionExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
   // Here we do the dance of replacing the following sled:
   //
   // xray_sled_n:
@@ -191,12 +195,13 @@ bool patchFunctionExit(const bool Enable, const uint32_t FuncId,
   //
   // Prerequisite is to compute the relative offset fo the
   // __xray_FunctionExit function's address.
+  auto Trampoline = Trampolines.ExitTrampoline;
   const uint64_t Address = Sled.address();
-  int64_t TrampolineOffset = reinterpret_cast<int64_t>(__xray_FunctionExit) -
+  int64_t TrampolineOffset = reinterpret_cast<int64_t>(Trampoline) -
                              (static_cast<int64_t>(Address) + 11);
   if (TrampolineOffset < MinOffset || TrampolineOffset > MaxOffset) {
     Report("XRay Exit trampoline (%p) too far from sled (%p)\n",
-           reinterpret_cast<void *>(__xray_FunctionExit),
+           reinterpret_cast<void *>(Trampoline),
            reinterpret_cast<void *>(Address));
     return false;
   }
@@ -216,17 +221,18 @@ bool patchFunctionExit(const bool Enable, const uint32_t FuncId,
   return true;
 }
 
-bool patchFunctionTailExit(const bool Enable, const uint32_t FuncId,
-                           const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+bool patchFunctionTailExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
   // Here we do the dance of replacing the tail call sled with a similar
   // sequence as the entry sled, but calls the tail exit sled instead.
+  auto Trampoline = Trampolines.TailExitTrampoline;
   const uint64_t Address = Sled.address();
-  int64_t TrampolineOffset =
-      reinterpret_cast<int64_t>(__xray_FunctionTailExit) -
-      (static_cast<int64_t>(Address) + 11);
+  int64_t TrampolineOffset = reinterpret_cast<int64_t>(Trampoline) -
+                             (static_cast<int64_t>(Address) + 11);
   if (TrampolineOffset < MinOffset || TrampolineOffset > MaxOffset) {
     Report("XRay Tail Exit trampoline (%p) too far from sled (%p)\n",
-           reinterpret_cast<void *>(__xray_FunctionTailExit),
+           reinterpret_cast<void *>(Trampoline),
            reinterpret_cast<void *>(Address));
     return false;
   }
@@ -275,7 +281,7 @@ bool patchCustomEvent(const bool Enable, const uint32_t FuncId,
 }
 
 bool patchTypedEvent(const bool Enable, const uint32_t FuncId,
-                      const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+                     const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
   // Here we do the dance of replacing the following sled:
   //
   // xray_sled_n:
diff --git a/compiler-rt/test/xray/TestCases/Posix/basic-mode-dso.cpp b/compiler-rt/test/xray/TestCases/Posix/basic-mode-dso.cpp
new file mode 100644
index 000000000000..31c615bd1f81
--- /dev/null
+++ b/compiler-rt/test/xray/TestCases/Posix/basic-mode-dso.cpp
@@ -0,0 +1,47 @@
+// Testing shared library support in basic logging mode.
+
+// RUN: split-file %s %t
+// RUN: %clangxx_xray -g -fPIC -fxray-instrument -fxray-shared -shared -std=c++11 %t/testlib.cpp -o %t/testlib.so
+// RUN: %clangxx_xray -g -fPIC -fxray-instrument -fxray-shared -std=c++11 %t/main.cpp %t/testlib.so -Wl,-rpath,%t -o %t/main.o
+
+// RUN: XRAY_OPTIONS="patch_premain=false,xray_mode=xray-basic,xray_logfile_base=basic-mode-dso-,verbosity=1" XRAY_BASIC_OPTIONS="func_duration_threshold_us=0" %run %t/main.o 2>&1 | FileCheck %s
+// RUN: %llvm_xray account --format=csv --sort=funcid "`ls basic-mode-dso-* | head -1`" | FileCheck --check-prefix=ACCOUNT %s
+// RUN: rm basic-mode-dso-*
+
+// REQUIRES: target=x86_64{{.*}}
+
+//--- main.cpp
+
+#include "xray/xray_interface.h"
+
+#include <cstdio>
+#include <unistd.h>
+
+[[clang::xray_always_instrument]] void instrumented_in_executable() {
+  printf("instrumented_in_executable called\n");
+  sleep(1);
+}
+
+extern void instrumented_in_dso();
+
+int main() {
+  // Explicit patching to ensure the DSO has been loaded
+  __xray_patch();
+  instrumented_in_executable();
+  // CHECK: instrumented_in_executable called
+  instrumented_in_dso();
+  // CHECK-NEXT: instrumented_in_dso called
+}
+
+//--- testlib.cpp
+
+#include <cstdio>
+#include <unistd.h>
+
+[[clang::xray_always_instrument]] void instrumented_in_dso() {
+  printf("instrumented_in_dso called\n");
+}
+
+// ACCOUNT: funcid,count,min,median,90%ile,99%ile,max,sum,debug,function
+// ACCOUNT-NEXT: 1,1,{{.*}}
+// ACCOUNT-NEXT: 16777217,1,{{.*}}
diff --git a/compiler-rt/test/xray/TestCases/Posix/clang-xray-shared.cpp b/compiler-rt/test/xray/TestCases/Posix/clang-xray-shared.cpp
new file mode 100644
index 000000000000..92f3c29e970d
--- /dev/null
+++ b/compiler-rt/test/xray/TestCases/Posix/clang-xray-shared.cpp
@@ -0,0 +1,14 @@
+// Test that the DSO-local runtime library has been linked if -fxray-shared is passed.
+//
+// RUN: %clangxx -fxray-instrument -fxray-shared %s -shared -o %t.so
+// RUN: llvm-nm %t.so | FileCheck %s --check-prefix ENABLED
+
+// RUN: %clangxx -fxray-instrument %s -shared -o %t.so
+// RUN: llvm-nm %t.so | FileCheck %s --check-prefix DISABLED
+//
+// REQUIRES: target=x86_64{{.*}}
+
+[[clang::xray_always_instrument]] int always_instrumented() { return 42; }
+
+// ENABLED: __start_xray_instr_map
+// DISABLED-NOT: __start_xray_instr_map
diff --git a/compiler-rt/test/xray/TestCases/Posix/dlopen.cpp b/compiler-rt/test/xray/TestCases/Posix/dlopen.cpp
new file mode 100644
index 000000000000..9db411d5ff1c
--- /dev/null
+++ b/compiler-rt/test/xray/TestCases/Posix/dlopen.cpp
@@ -0,0 +1,107 @@
+// Check that we can patch and un-patch DSOs loaded with dlopen.
+//
+
+// RUN: split-file %s %t
+// RUN: %clangxx_xray -g -fPIC -fxray-instrument -fxray-shared -shared -std=c++11 %t/testlib.cpp -o %t/testlib.so
+// RUN: %clangxx_xray -g -fPIC -rdynamic -fxray-instrument -fxray-shared -std=c++11 %t/main.cpp -o %t/main.o
+//
+// RUN: XRAY_OPTIONS="patch_premain=true" %run %t/main.o %t/testlib.so 2>&1 | FileCheck %s
+
+// REQUIRES: target=x86_64{{.*}}
+
+//--- main.cpp
+
+#include "xray/xray_interface.h"
+
+#include <cstdio>
+#include <dlfcn.h>
+
+void test_handler(int32_t fid, XRayEntryType type) {
+  printf("called: %d, type=%d\n", fid, static_cast<int32_t>(type));
+}
+
+[[clang::xray_always_instrument]] void instrumented_in_executable() {
+  printf("instrumented_in_executable called\n");
+}
+
+typedef void (*dso_func_type)();
+
+int main(int argc, char **argv) {
+  if (argc < 2) {
+    printf("Shared library argument missing\n");
+    // CHECK-NOT: Shared library argument missing
+    return 1;
+  }
+
+  const char *dso_path = argv[1];
+
+  void *dso_handle = dlopen(dso_path, RTLD_LAZY);
+  if (!dso_handle) {
+    printf("Failed to load shared library\n");
+    char *error = dlerror();
+    if (error) {
+      fprintf(stderr, "%s\n", error);
+      return 1;
+    }
+    return 1;
+  }
+
+  dso_func_type instrumented_in_dso =
+      (dso_func_type)dlsym(dso_handle, "_Z19instrumented_in_dsov");
+  if (!instrumented_in_dso) {
+    printf("Failed to find symbol\n");
+    char *error = dlerror();
+    if (error) {
+      fprintf(stderr, "%s\n", error);
+      return 1;
+    }
+    return 1;
+  }
+
+  __xray_set_handler(test_handler);
+
+  instrumented_in_executable();
+  // CHECK: called: {{.*}}, type=0
+  // CHECK-NEXT: instrumented_in_executable called
+  // CHECK-NEXT: called: {{.*}}, type=1
+  instrumented_in_dso();
+  // CHECK-NEXT: called: {{.*}}, type=0
+  // CHECK-NEXT: instrumented_in_dso called
+  // CHECK-NEXT: called: {{.*}}, type=1
+
+  auto status = __xray_unpatch();
+  printf("unpatching status: %d\n", static_cast<int32_t>(status));
+  // CHECK-NEXT: unpatching status: 1
+
+  instrumented_in_executable();
+  // CHECK-NEXT: instrumented_in_executable called
+  instrumented_in_dso();
+  // CHECK-NEXT: instrumented_in_dso called
+
+  status = __xray_patch();
+  printf("patching status: %d\n", static_cast<int32_t>(status));
+  // CHECK-NEXT: patching status: 1
+
+  instrumented_in_executable();
+  // CHECK-NEXT: called: {{.*}}, type=0
+  // CHECK-NEXT: instrumented_in_executable called
+  // CHECK-NEXT: called: {{.*}}, type=1
+  instrumented_in_dso();
+  // CHECK-NEXT: called: {{.*}}, type=0
+  // CHECK-NEXT: instrumented_in_dso called
+  // CHECK-NEXT: called: {{.*}}, type=1
+
+  dlclose(dso_handle);
+
+  status = __xray_unpatch();
+  printf("unpatching status: %d\n", static_cast<int32_t>(status));
+  // CHECK-NEXT: unpatching status: 1
+}
+
+//--- testlib.cpp
+
+#include <cstdio>
+
+[[clang::xray_always_instrument]] void instrumented_in_dso() {
+  printf("instrumented_in_dso called\n");
+}
diff --git a/compiler-rt/test/xray/TestCases/Posix/dso-dep-chains.cpp b/compiler-rt/test/xray/TestCases/Posix/dso-dep-chains.cpp
new file mode 100644
index 000000000000..89da2764c35c
--- /dev/null
+++ b/compiler-rt/test/xray/TestCases/Posix/dso-dep-chains.cpp
@@ -0,0 +1,197 @@
+// Check that loading libraries with different modes (RTLD_LOCAL/RTLD_GLOBAL)
+// and dependencies on other DSOs work correctly.
+//
+
+// RUN: split-file %s %t
+//
+// Build shared libs with dependencies b->c and e->f
+// RUN: %clangxx_xray -g -fPIC -fxray-instrument -fxray-shared -shared -std=c++11 %t/testliba.cpp -o %t/testliba.so
+// RUN: %clangxx_xray -g -fPIC -fxray-instrument -fxray-shared -shared -std=c++11 %t/testlibc.cpp -o %t/testlibc.so
+// RUN: %clangxx_xray -g -fPIC -fxray-instrument -fxray-shared -shared -std=c++11 %t/testlibb.cpp %t/testlibc.so -o %t/testlibb.so
+// RUN: %clangxx_xray -g -fPIC -fxray-instrument -fxray-shared -shared -std=c++11 %t/testlibd.cpp -o %t/testlibd.so
+// RUN: %clangxx_xray -g -fPIC -fxray-instrument -fxray-shared -shared -std=c++11 %t/testlibf.cpp -o %t/testlibf.so
+// RUN: %clangxx_xray -g -fPIC -fxray-instrument -fxray-shared -shared -std=c++11 %t/testlibe.cpp %t/testlibf.so -o %t/testlibe.so
+//
+// Executable links with a and b explicitly and loads d and e at runtime.
+// RUN: %clangxx_xray -g -fPIC -rdynamic -fxray-instrument -fxray-shared -std=c++11 %t/main.cpp %t/testliba.so %t/testlibb.so -o %t/main.o
+//
+// RUN:  XRAY_OPTIONS="patch_premain=true" %run %t/main.o %t/testlibd.so %t/testlibe.so 2>&1 | FileCheck %s
+
+// REQUIRES: target=x86_64{{.*}}
+
+//--- main.cpp
+
+#include "xray/xray_interface.h"
+
+#include <cstdio>
+#include <dlfcn.h>
+
+[[clang::xray_never_instrument]] void test_handler(int32_t fid,
+                                                   XRayEntryType type) {
+  printf("called: %d, object=%d, fn=%d, type=%d\n", fid, (fid >> 24) & 0xFF,
+         fid & 0x00FFFFFF, static_cast<int32_t>(type));
+}
+
+[[clang::xray_always_instrument]] void instrumented_in_executable() {
+  printf("instrumented_in_executable called\n");
+}
+
+typedef void (*dso_func_type)();
+
+[[clang::xray_never_instrument]] void *load_dso(const char *path, int mode) {
+  void *dso_handle = dlopen(path, mode);
+  if (!dso_handle) {
+    printf("failed to load shared library\n");
+    char *error = dlerror();
+    if (error) {
+      fprintf(stderr, "%s\n", error);
+    }
+    return nullptr;
+  }
+  return dso_handle;
+}
+
+[[clang::xray_never_instrument]] void find_and_call(void *dso_handle,
+                                                    const char *fn) {
+  dso_func_type dso_fn = (dso_func_type)dlsym(dso_handle, fn);
+  if (!dso_fn) {
+    printf("failed to find symbol\n");
+    char *error = dlerror();
+    if (error) {
+      fprintf(stderr, "%s\n", error);
+    }
+    return;
+  }
+  dso_fn();
+}
+
+extern void a();
+extern void b();
+
+int main(int argc, char **argv) {
+
+  if (argc < 3) {
+    printf("Shared library arguments missing\n");
+    // CHECK-NOT: Shared library arguments missing
+    return 1;
+  }
+
+  const char *dso_path_d = argv[1];
+  const char *dso_path_e = argv[2];
+
+  __xray_set_handler(test_handler);
+
+  instrumented_in_executable();
+  // CHECK: called: {{[0-9]+}}, object=0, fn={{[0-9]+}}, type=0
+  // CHECK-NEXT: instrumented_in_executable called
+  // CHECK-NEXT: called: {{[0-9]+}}, object=0, fn={{[0-9]+}}, type=1
+
+  a();
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ1:[0-9]+]], fn=1, type=0
+  // CHECK-NEXT: a called
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ1]], fn=1, type=1
+
+  // Make sure this object ID does not appear again
+  // CHECK-NOT: called: {{[0-9]+}}, object=[[OBJ1]]
+
+  b(); // b calls c
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ2:[0-9]+]], fn=1, type=0
+  // CHECK-NEXT: b called
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ3:[0-9]+]], fn=1, type=0
+  // CHECK-NEXT: c called
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ3]], fn=1, type=1
+  // CHECK-NOT: called: {{[0-9]+}}, object=[[OBJ3]]
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ2]], fn=1, type=1
+  // CHECK-NOT: called: {{[0-9]+}}, object=[[OBJ2]]
+
+  // Now check explicit loading with RTLD_LOCAL
+
+  void *dso_handle_d = load_dso(dso_path_d, RTLD_LAZY | RTLD_LOCAL);
+  void *dso_handle_e = load_dso(dso_path_e, RTLD_LAZY | RTLD_LOCAL);
+  // CHECK-NOT: failed to load shared library
+
+  find_and_call(dso_handle_d, "_Z1dv");
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ4:[0-9]+]], fn=1, type=0
+  // CHECK-NEXT: d called
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ4]], fn=1, type=1
+  // CHECK-NOT: called: {{[0-9]+}}, object=[[OBJ4]]
+
+  find_and_call(dso_handle_e, "_Z1ev");
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ5:[0-9]+]], fn=1, type=0
+  // CHECK-NEXT: e called
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ6:[0-9]+]], fn=1, type=0
+  // CHECK-NEXT: f called
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ6]], fn=1, type=1
+  // CHECK-NOT: called: {{[0-9]+}}, object=[[OBJ6]]
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ5]], fn=1, type=1
+  // CHECK-NOT: called: {{[0-9]+}}, object=[[OBJ5]]
+
+  // Unload DSOs
+  dlclose(dso_handle_d);
+  dlclose(dso_handle_e);
+
+  // Repeat test with RTLD_GLOBAL
+  dso_handle_d = load_dso(dso_path_d, RTLD_LAZY | RTLD_GLOBAL);
+  dso_handle_e = load_dso(dso_path_e, RTLD_LAZY | RTLD_GLOBAL);
+  // CHECK-NOT: failed to load shared library
+
+  find_and_call(dso_handle_d, "_Z1dv");
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ7:[0-9]+]], fn=1, type=0
+  // CHECK-NEXT: d called
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ7]], fn=1, type=1
+  // CHECK-NOT: called: {{[0-9]+}}, object=[[OBJ7]]
+
+  find_and_call(dso_handle_e, "_Z1ev");
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ8:[0-9]+]], fn=1, type=0
+  // CHECK-NEXT: e called
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ9:[0-9]+]], fn=1, type=0
+  // CHECK-NEXT: f called
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ9]], fn=1, type=1
+  // CHECK-NOT: called: {{[0-9]+}}, object=[[OBJ9]]
+  // CHECK-NEXT: called: {{[0-9]+}}, object=[[OBJ8]], fn=1, type=1
+  // CHECK-NOT: called: {{[0-9]+}}, object=[[OBJ8]]
+
+  auto status = __xray_unpatch();
+  printf("unpatching status: %d\n", static_cast<int32_t>(status));
+  // CHECK-NEXT: unpatching status: 1
+
+  dlclose(dso_handle_d);
+  dlclose(dso_handle_e);
+}
+
+//--- libgenmacro.inc
+#include <cstdio>
+// Helper macros to quickly generate libraries containing a single function.
+#define GENERATE_LIB(NAME)                                                     \
+  [[clang::xray_always_instrument]] void NAME() { printf(#NAME " called\n"); }
+
+#define GENERATE_LIB_WITH_CALL(NAME, FN)                                       \
+  extern void FN();                                                            \
+  [[clang::xray_always_instrument]] void NAME() {                              \
+    printf(#NAME " called\n");                                                 \
+    FN();                                                                      \
+  }
+
+//--- testliba.cpp
+#include "libgenmacro.inc"
+GENERATE_LIB(a)
+
+//--- testlibb.cpp
+#include "libgenmacro.inc"
+GENERATE_LIB_WITH_CALL(b, c)
+
+//--- testlibc.cpp
+#include "libgenmacro.inc"
+GENERATE_LIB(c)
+
+//--- testlibd.cpp
+#include "libgenmacro.inc"
+GENERATE_LIB(d)
+
+//--- testlibe.cpp
+#include "libgenmacro.inc"
+GENERATE_LIB_WITH_CALL(e, f)
+
+//--- testlibf.cpp
+#include "libgenmacro.inc"
+GENERATE_LIB(f)
diff --git a/compiler-rt/test/xray/TestCases/Posix/patch-premain-dso.cpp b/compiler-rt/test/xray/TestCases/Posix/patch-premain-dso.cpp
new file mode 100644
index 000000000000..0708d0383439
--- /dev/null
+++ b/compiler-rt/test/xray/TestCases/Posix/patch-premain-dso.cpp
@@ -0,0 +1,45 @@
+// Checking that DSOs are automatically patched upon load, if patch_premain is passed.
+
+// RUN: split-file %s %t
+// RUN: %clangxx_xray -g -fPIC -fxray-instrument -fxray-shared -shared -std=c++11 %t/testlib.cpp -o %t/testlib.so
+// RUN: %clangxx_xray -g -fPIC -fxray-instrument -fxray-shared -std=c++11 %t/main.cpp %t/testlib.so -Wl,-rpath,%t -o %t/main.o
+
+// RUN: XRAY_OPTIONS="patch_premain=true,verbosity=1" %run %t/main.o 2>&1 | FileCheck %s
+
+// REQUIRES: target=x86_64{{.*}}
+
+//--- main.cpp
+
+#include "xray/xray_interface.h"
+
+#include <cstdio>
+
+void test_handler(int32_t fid, XRayEntryType type) {
+  printf("called: %d, type=%d\n", fid, static_cast<int32_t>(type));
+}
+
+[[clang::xray_always_instrument]] void instrumented_in_executable() {
+  printf("instrumented_in_executable called\n");
+}
+
+extern void instrumented_in_dso();
+
+int main() {
+  __xray_set_handler(test_handler);
+  instrumented_in_executable();
+  // CHECK: called: {{.*}}, type=0
+  // CHECK-NEXT: instrumented_in_executable called
+  // CHECK-NEXT: called: {{.*}}, type=1
+  instrumented_in_dso();
+  // CHECK-NEXT: called: {{.*}}, type=0
+  // CHECK-NEXT: instrumented_in_dso called
+  // CHECK-NEXT: called: {{.*}}, type=1
+}
+
+//--- testlib.cpp
+
+#include <cstdio>
+
+[[clang::xray_always_instrument]] void instrumented_in_dso() {
+  printf("instrumented_in_dso called\n");
+}
diff --git a/compiler-rt/test/xray/TestCases/Posix/patching-unpatching-dso.cpp b/compiler-rt/test/xray/TestCases/Posix/patching-unpatching-dso.cpp
new file mode 100644
index 000000000000..d3e992dd4977
--- /dev/null
+++ b/compiler-rt/test/xray/TestCases/Posix/patching-unpatching-dso.cpp
@@ -0,0 +1,75 @@
+// Check that we can patch and un-patch on demand, and that logging gets invoked
+// appropriately.
+//
+
+// RUN: split-file %s %t
+// RUN: %clangxx_xray -g -fPIC -fxray-instrument -fxray-shared -shared -std=c++11 %t/testlib.cpp -o %t/testlib.so
+// RUN: %clangxx_xray -g -fPIC -fxray-instrument -fxray-shared -std=c++11 %t/main.cpp %t/testlib.so -Wl,-rpath,%t -o %t/main.o
+
+// RUN: XRAY_OPTIONS="patch_premain=false" %run %t/main.o 2>&1 | FileCheck %s
+
+// REQUIRES: target=x86_64{{.*}}
+
+//--- main.cpp
+
+#include "xray/xray_interface.h"
+
+#include <cstdio>
+
+bool called = false;
+
+void test_handler(int32_t fid, XRayEntryType type) {
+  printf("called: %d, type=%d\n", fid, static_cast<int32_t>(type));
+  called = true;
+}
+
+[[clang::xray_always_instrument]] void instrumented_in_executable() {
+  printf("instrumented_in_executable called\n");
+}
+
+extern void instrumented_in_dso();
+
+int main() {
+  __xray_set_handler(test_handler);
+  instrumented_in_executable();
+  // CHECK: instrumented_in_executable called
+  instrumented_in_dso();
+  // CHECK: instrumented_in_dso called
+  auto status = __xray_patch();
+  printf("patching status: %d\n", static_cast<int32_t>(status));
+  // CHECK-NEXT: patching status: 1
+  instrumented_in_executable();
+  // CHECK-NEXT: called: {{.*}}, type=0
+  // CHECK-NEXT: instrumented_in_executable called
+  // CHECK-NEXT: called: {{.*}}, type=1
+  instrumented_in_dso();
+  // CHECK-NEXT: called: {{.*}}, type=0
+  // CHECK-NEXT: instrumented_in_dso called
+  // CHECK-NEXT: called: {{.*}}, type=1
+  status = __xray_unpatch();
+  printf("patching status: %d\n", static_cast<int32_t>(status));
+  // CHECK-NEXT: patching status: 1
+  instrumented_in_executable();
+  // CHECK-NEXT: instrumented_in_executable called
+  instrumented_in_dso();
+  // CHECK-NEXT: instrumented_in_dso called
+  status = __xray_patch();
+  printf("patching status: %d\n", static_cast<int32_t>(status));
+  // CHECK-NEXT: patching status: 1
+  __xray_remove_handler();
+  instrumented_in_executable();
+  // CHECK-NEXT: instrumented_in_executable called
+  instrumented_in_dso();
+  // CHECK-NEXT: instrumented_in_dso called
+  status = __xray_unpatch();
+  printf("patching status: %d\n", static_cast<int32_t>(status));
+  // CHECK-NEXT: patching status: 1
+}
+
+//--- testlib.cpp
+
+#include <cstdio>
+
+[[clang::xray_always_instrument]] void instrumented_in_dso() {
+  printf("instrumented_in_dso called\n");
+}
-- 
GitLab


From e3f218096ca31863cc245fdd411f0b6e5cbfb1cb Mon Sep 17 00:00:00 2001
From: Oliver Stannard <oliver.stannard@arm.com>
Date: Thu, 9 May 2024 12:58:41 +0100
Subject: [PATCH 134/693] [ARM] Re-generate a test

---
 llvm/test/CodeGen/ARM/fp-arg-shuffle.ll | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/llvm/test/CodeGen/ARM/fp-arg-shuffle.ll b/llvm/test/CodeGen/ARM/fp-arg-shuffle.ll
index 4996cc8ecbf0..36f5a4b30af4 100644
--- a/llvm/test/CodeGen/ARM/fp-arg-shuffle.ll
+++ b/llvm/test/CodeGen/ARM/fp-arg-shuffle.ll
@@ -1,8 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mtriple=arm-eabi -mattr=+neon -float-abi=soft %s -o - | FileCheck %s
 
 ; CHECK: function1
 ; CHECK-NOT: vmov
 define double @function1(double %a, double %b, double %c, double %d, double %e, double %f) nounwind noinline ssp {
+; CHECK-LABEL: function1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r11, lr}
+; CHECK-NEXT:    push {r4, r5, r11, lr}
+; CHECK-NEXT:    .pad #32
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    add lr, sp, #64
+; CHECK-NEXT:    vldr d16, [sp, #56]
+; CHECK-NEXT:    str r2, [sp, #16]
+; CHECK-NEXT:    ldm lr, {r4, r5, r12, lr}
+; CHECK-NEXT:    str r3, [sp, #20]
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    str r0, [sp, #24]
+; CHECK-NEXT:    mov r0, r12
+; CHECK-NEXT:    str r1, [sp, #28]
+; CHECK-NEXT:    mov r1, lr
+; CHECK-NEXT:    mov r2, r4
+; CHECK-NEXT:    vldr d17, [sp, #48]
+; CHECK-NEXT:    vstmia sp, {d16, d17}
+; CHECK-NEXT:    bl function2
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    pop {r4, r5, r11, lr}
+; CHECK-NEXT:    mov pc, lr
 entry:
   %call = tail call double @function2(double %f, double %e, double %d, double %c, double %b, double %a) nounwind
   ret double %call
-- 
GitLab


From 8e289e4fa60eb59fc12c39b88987d4956c1d38ea Mon Sep 17 00:00:00 2001
From: Oliver Stannard <oliver.stannard@arm.com>
Date: Tue, 24 Sep 2024 10:46:47 +0100
Subject: [PATCH 135/693] [ARM] Fix comment typo

---
 llvm/lib/Target/ARM/ARMISelLowering.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index a4f01e55f53c..76350f8022e6 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -2408,8 +2408,8 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     isTailCall = false;
 
   // For both the non-secure calls and the returns from a CMSE entry function,
-  // the function needs to do some extra work afte r the call, or before the
-  // return, respectively, thus it cannot end with atail call
+  // the function needs to do some extra work after the call, or before the
+  // return, respectively, thus it cannot end with a tail call
   if (isCmseNSCall || AFI->isCmseNSEntryFunction())
     isTailCall = false;
 
-- 
GitLab


From 246baeb5fe7d9176f3bc143ab8abe9199a8636f4 Mon Sep 17 00:00:00 2001
From: Oliver Stannard <oliver.stannard@arm.com>
Date: Thu, 9 May 2024 13:00:46 +0100
Subject: [PATCH 136/693] [ARM] Add debug trace for tail-call optimisation

There are lots of reasons a call might not be eligible for tail-call
optimisation, this adds debug trace to help understand the compiler's
decisions here.
---
 llvm/lib/Target/ARM/ARMISelLowering.cpp | 64 +++++++++++++++++++------
 1 file changed, 49 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 76350f8022e6..25c74f52bd87 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -3047,8 +3047,10 @@ bool ARMTargetLowering::IsEligibleForTailCallOptimization(
     for (const CCValAssign &AL : ArgLocs)
       if (AL.isRegLoc())
         AddressRegisters.erase(AL.getLocReg());
-    if (AddressRegisters.empty())
+    if (AddressRegisters.empty()) {
+      LLVM_DEBUG(dbgs() << "false (no reg to hold function pointer)\n");
       return false;
+    }
   }
 
   // Look for obvious safe cases to perform tail call optimization that do not
@@ -3057,18 +3059,26 @@ bool ARMTargetLowering::IsEligibleForTailCallOptimization(
   // Exception-handling functions need a special set of instructions to indicate
   // a return to the hardware. Tail-calling another function would probably
   // break this.
-  if (CallerF.hasFnAttribute("interrupt"))
+  if (CallerF.hasFnAttribute("interrupt")) {
+    LLVM_DEBUG(dbgs() << "false (interrupt attribute)\n");
     return false;
+  }
 
-  if (canGuaranteeTCO(CalleeCC, getTargetMachine().Options.GuaranteedTailCallOpt))
+  if (canGuaranteeTCO(CalleeCC,
+                      getTargetMachine().Options.GuaranteedTailCallOpt)) {
+    LLVM_DEBUG(dbgs() << (CalleeCC == CallerCC ? "true" : "false")
+                      << " (guaranteed tail-call CC)\n");
     return CalleeCC == CallerCC;
+  }
 
   // Also avoid sibcall optimization if either caller or callee uses struct
   // return semantics.
   bool isCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
   bool isCallerStructRet = MF.getFunction().hasStructRetAttr();
-  if (isCalleeStructRet || isCallerStructRet)
+  if (isCalleeStructRet || isCallerStructRet) {
+    LLVM_DEBUG(dbgs() << "false (struct-ret)\n");
     return false;
+  }
 
   // Externally-defined functions with weak linkage should not be
   // tail-called on ARM when the OS does not support dynamic
@@ -3081,8 +3091,11 @@ bool ARMTargetLowering::IsEligibleForTailCallOptimization(
     const GlobalValue *GV = G->getGlobal();
     const Triple &TT = getTargetMachine().getTargetTriple();
     if (GV->hasExternalWeakLinkage() &&
-        (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO()))
+        (!TT.isOSWindows() || TT.isOSBinFormatELF() ||
+         TT.isOSBinFormatMachO())) {
+      LLVM_DEBUG(dbgs() << "false (external weak linkage)\n");
       return false;
+    }
   }
 
   // Check that the call results are passed in the same way.
@@ -3091,23 +3104,29 @@ bool ARMTargetLowering::IsEligibleForTailCallOptimization(
           getEffectiveCallingConv(CalleeCC, isVarArg),
           getEffectiveCallingConv(CallerCC, CallerF.isVarArg()), MF, C, Ins,
           CCAssignFnForReturn(CalleeCC, isVarArg),
-          CCAssignFnForReturn(CallerCC, CallerF.isVarArg())))
+          CCAssignFnForReturn(CallerCC, CallerF.isVarArg()))) {
+    LLVM_DEBUG(dbgs() << "false (incompatible results)\n");
     return false;
+  }
   // The callee has to preserve all registers the caller needs to preserve.
   const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
   const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
   if (CalleeCC != CallerCC) {
     const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
-    if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
+    if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved)) {
+      LLVM_DEBUG(dbgs() << "false (not all registers preserved)\n");
       return false;
+    }
   }
 
   // If Caller's vararg or byval argument has been split between registers and
   // stack, do not perform tail call, since part of the argument is in caller's
   // local frame.
   const ARMFunctionInfo *AFI_Caller = MF.getInfo<ARMFunctionInfo>();
-  if (AFI_Caller->getArgRegsSaveSize())
+  if (AFI_Caller->getArgRegsSaveSize()) {
+    LLVM_DEBUG(dbgs() << "false (arg reg save area)\n");
     return false;
+  }
 
   // If the callee takes no arguments then go on to check the results of the
   // call.
@@ -3125,36 +3144,51 @@ bool ARMTargetLowering::IsEligibleForTailCallOptimization(
         EVT RegVT = VA.getLocVT();
         SDValue Arg = OutVals[realArgIdx];
         ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
-        if (VA.getLocInfo() == CCValAssign::Indirect)
+        if (VA.getLocInfo() == CCValAssign::Indirect) {
+          LLVM_DEBUG(dbgs() << "false (indirect arg)\n");
           return false;
+        }
         if (VA.needsCustom() && (RegVT == MVT::f64 || RegVT == MVT::v2f64)) {
           // f64 and vector types are split into multiple registers or
           // register/stack-slot combinations.  The types will not match
           // the registers; give up on memory f64 refs until we figure
           // out what to do about this.
-          if (!VA.isRegLoc())
+          if (!VA.isRegLoc()) {
+            LLVM_DEBUG(dbgs() << "false (f64 not in register)\n");
             return false;
-          if (!ArgLocs[++i].isRegLoc())
+          }
+          if (!ArgLocs[++i].isRegLoc()) {
+            LLVM_DEBUG(dbgs() << "false (f64 not in register, second half)\n");
             return false;
+          }
           if (RegVT == MVT::v2f64) {
-            if (!ArgLocs[++i].isRegLoc())
+            if (!ArgLocs[++i].isRegLoc()) {
+              LLVM_DEBUG(dbgs() << "false (v2f64 not in register)\n");
               return false;
-            if (!ArgLocs[++i].isRegLoc())
+            }
+            if (!ArgLocs[++i].isRegLoc()) {
+              LLVM_DEBUG(dbgs() << "false (v2f64 not in register, second half)\n");
               return false;
+            }
           }
         } else if (!VA.isRegLoc()) {
           if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags,
-                                   MFI, MRI, TII))
+                                   MFI, MRI, TII)) {
+            LLVM_DEBUG(dbgs() << "false (non-matching stack offset)\n");
             return false;
+          }
         }
       }
     }
 
     const MachineRegisterInfo &MRI = MF.getRegInfo();
-    if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals))
+    if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals)) {
+      LLVM_DEBUG(dbgs() << "false (parameters in CSRs do not match)\n");
       return false;
+    }
   }
 
+  LLVM_DEBUG(dbgs() << "true\n");
   return true;
 }
 
-- 
GitLab


From c1eb790cd2f2a3fd48781167b50f091c0d20be8d Mon Sep 17 00:00:00 2001
From: Oliver Stannard <oliver.stannard@arm.com>
Date: Thu, 9 May 2024 13:11:33 +0100
Subject: [PATCH 137/693] [ARM] Tail-calls do not require caller and callee
 arguments to match

The ARM backend was checking that the outgoing values for a tail-call
matched the incoming argument values of the caller. This isn't
necessary, because the caller can change the values in both registers
and the stack before doing the tail-call. The actual limitation is that
the callee can't need more stack space for it's arguments than the
caller does.

This is needed for code using the musttail attribute, as well as
enabling tail calls as an optimisation in more cases.
---
 llvm/lib/Target/ARM/ARMISelLowering.cpp       | 109 ++----------------
 llvm/test/CodeGen/ARM/fp-arg-shuffle.ll       |  30 +++--
 llvm/test/CodeGen/ARM/fp16-vector-argument.ll |  41 +++----
 llvm/test/CodeGen/ARM/musttail.ll             |  97 ++++++++++++++++
 4 files changed, 134 insertions(+), 143 deletions(-)
 create mode 100644 llvm/test/CodeGen/ARM/musttail.ll

diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 25c74f52bd87..222f08dab03f 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -2962,50 +2962,6 @@ void ARMTargetLowering::HandleByVal(CCState *State, unsigned &Size,
   Size = std::max<int>(Size - Excess, 0);
 }
 
-/// MatchingStackOffset - Return true if the given stack call argument is
-/// already available in the same position (relatively) of the caller's
-/// incoming argument stack.
-static
-bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
-                         MachineFrameInfo &MFI, const MachineRegisterInfo *MRI,
-                         const TargetInstrInfo *TII) {
-  unsigned Bytes = Arg.getValueSizeInBits() / 8;
-  int FI = std::numeric_limits<int>::max();
-  if (Arg.getOpcode() == ISD::CopyFromReg) {
-    Register VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg();
-    if (!VR.isVirtual())
-      return false;
-    MachineInstr *Def = MRI->getVRegDef(VR);
-    if (!Def)
-      return false;
-    if (!Flags.isByVal()) {
-      if (!TII->isLoadFromStackSlot(*Def, FI))
-        return false;
-    } else {
-      return false;
-    }
-  } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
-    if (Flags.isByVal())
-      // ByVal argument is passed in as a pointer but it's now being
-      // dereferenced. e.g.
-      // define @foo(%struct.X* %A) {
-      //   tail call @bar(%struct.X* byval %A)
-      // }
-      return false;
-    SDValue Ptr = Ld->getBasePtr();
-    FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr);
-    if (!FINode)
-      return false;
-    FI = FINode->getIndex();
-  } else
-    return false;
-
-  assert(FI != std::numeric_limits<int>::max());
-  if (!MFI.isFixedObjectIndex(FI))
-    return false;
-  return Offset == MFI.getObjectOffset(FI) && Bytes == MFI.getObjectSize(FI);
-}
-
 /// IsEligibleForTailCallOptimization - Check whether the call is eligible
 /// for tail call optimization. Targets which want to do tail call
 /// optimization should implement this function. Note that this function also
@@ -3130,64 +3086,17 @@ bool ARMTargetLowering::IsEligibleForTailCallOptimization(
 
   // If the callee takes no arguments then go on to check the results of the
   // call.
-  if (!Outs.empty()) {
-    if (CCInfo.getStackSize()) {
-      // Check if the arguments are already laid out in the right way as
-      // the caller's fixed stack objects.
-      MachineFrameInfo &MFI = MF.getFrameInfo();
-      const MachineRegisterInfo *MRI = &MF.getRegInfo();
-      const TargetInstrInfo *TII = Subtarget->getInstrInfo();
-      for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
-           i != e;
-           ++i, ++realArgIdx) {
-        CCValAssign &VA = ArgLocs[i];
-        EVT RegVT = VA.getLocVT();
-        SDValue Arg = OutVals[realArgIdx];
-        ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
-        if (VA.getLocInfo() == CCValAssign::Indirect) {
-          LLVM_DEBUG(dbgs() << "false (indirect arg)\n");
-          return false;
-        }
-        if (VA.needsCustom() && (RegVT == MVT::f64 || RegVT == MVT::v2f64)) {
-          // f64 and vector types are split into multiple registers or
-          // register/stack-slot combinations.  The types will not match
-          // the registers; give up on memory f64 refs until we figure
-          // out what to do about this.
-          if (!VA.isRegLoc()) {
-            LLVM_DEBUG(dbgs() << "false (f64 not in register)\n");
-            return false;
-          }
-          if (!ArgLocs[++i].isRegLoc()) {
-            LLVM_DEBUG(dbgs() << "false (f64 not in register, second half)\n");
-            return false;
-          }
-          if (RegVT == MVT::v2f64) {
-            if (!ArgLocs[++i].isRegLoc()) {
-              LLVM_DEBUG(dbgs() << "false (v2f64 not in register)\n");
-              return false;
-            }
-            if (!ArgLocs[++i].isRegLoc()) {
-              LLVM_DEBUG(dbgs() << "false (v2f64 not in register, second half)\n");
-              return false;
-            }
-          }
-        } else if (!VA.isRegLoc()) {
-          if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags,
-                                   MFI, MRI, TII)) {
-            LLVM_DEBUG(dbgs() << "false (non-matching stack offset)\n");
-            return false;
-          }
-        }
-      }
-    }
-
-    const MachineRegisterInfo &MRI = MF.getRegInfo();
-    if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals)) {
-      LLVM_DEBUG(dbgs() << "false (parameters in CSRs do not match)\n");
-      return false;
-    }
+  const MachineRegisterInfo &MRI = MF.getRegInfo();
+  if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals)) {
+    LLVM_DEBUG(dbgs() << "false (parameters in CSRs do not match)\n");
+    return false;
   }
 
+  // If the stack arguments for this call do not fit into our own save area then
+  // the call cannot be made tail.
+  if (CCInfo.getStackSize() > AFI_Caller->getArgumentStackSize())
+    return false;
+
   LLVM_DEBUG(dbgs() << "true\n");
   return true;
 }
diff --git a/llvm/test/CodeGen/ARM/fp-arg-shuffle.ll b/llvm/test/CodeGen/ARM/fp-arg-shuffle.ll
index 36f5a4b30af4..1e46b081acfd 100644
--- a/llvm/test/CodeGen/ARM/fp-arg-shuffle.ll
+++ b/llvm/test/CodeGen/ARM/fp-arg-shuffle.ll
@@ -2,31 +2,29 @@
 ; RUN: llc -mtriple=arm-eabi -mattr=+neon -float-abi=soft %s -o - | FileCheck %s
 
 ; CHECK: function1
-; CHECK-NOT: vmov
 define double @function1(double %a, double %b, double %c, double %d, double %e, double %f) nounwind noinline ssp {
 ; CHECK-LABEL: function1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r11, lr}
 ; CHECK-NEXT:    push {r4, r5, r11, lr}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, sp, #32
-; CHECK-NEXT:    add lr, sp, #64
-; CHECK-NEXT:    vldr d16, [sp, #56]
-; CHECK-NEXT:    str r2, [sp, #16]
-; CHECK-NEXT:    ldm lr, {r4, r5, r12, lr}
-; CHECK-NEXT:    str r3, [sp, #20]
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    str r0, [sp, #24]
+; CHECK-NEXT:    vldr d16, [sp, #40]
+; CHECK-NEXT:    vldr d17, [sp, #32]
+; CHECK-NEXT:    vmov r12, lr, d16
+; CHECK-NEXT:    vldr d16, [sp, #16]
+; CHECK-NEXT:    vmov r4, r5, d17
+; CHECK-NEXT:    vldr d17, [sp, #24]
+; CHECK-NEXT:    str r3, [sp, #36]
+; CHECK-NEXT:    str r2, [sp, #32]
+; CHECK-NEXT:    str r1, [sp, #44]
+; CHECK-NEXT:    str r0, [sp, #40]
+; CHECK-NEXT:    vstr d17, [sp, #16]
+; CHECK-NEXT:    vstr d16, [sp, #24]
 ; CHECK-NEXT:    mov r0, r12
-; CHECK-NEXT:    str r1, [sp, #28]
 ; CHECK-NEXT:    mov r1, lr
 ; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    vldr d17, [sp, #48]
-; CHECK-NEXT:    vstmia sp, {d16, d17}
-; CHECK-NEXT:    bl function2
-; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    mov r3, r5
 ; CHECK-NEXT:    pop {r4, r5, r11, lr}
-; CHECK-NEXT:    mov pc, lr
+; CHECK-NEXT:    b function2
 entry:
   %call = tail call double @function2(double %f, double %e, double %d, double %c, double %b, double %a) nounwind
   ret double %call
diff --git a/llvm/test/CodeGen/ARM/fp16-vector-argument.ll b/llvm/test/CodeGen/ARM/fp16-vector-argument.ll
index 6fc56967bc7a..65aff46658fd 100644
--- a/llvm/test/CodeGen/ARM/fp16-vector-argument.ll
+++ b/llvm/test/CodeGen/ARM/fp16-vector-argument.ll
@@ -145,26 +145,21 @@ entry:
 define void @many_args_test(double, float, i16, <4 x half>, <8 x half>, <8 x half>, <8 x half>) {
 ; SOFT-LABEL: many_args_test:
 ; SOFT:       @ %bb.0: @ %entry
-; SOFT-NEXT:    push {r11, lr}
-; SOFT-NEXT:    sub sp, sp, #32
-; SOFT-NEXT:    add r12, sp, #80
+; SOFT-NEXT:    add r12, sp, #40
 ; SOFT-NEXT:    vld1.64 {d16, d17}, [r12]
-; SOFT-NEXT:    add r12, sp, #48
+; SOFT-NEXT:    add r12, sp, #8
 ; SOFT-NEXT:    vabs.f16 q8, q8
 ; SOFT-NEXT:    vld1.64 {d18, d19}, [r12]
-; SOFT-NEXT:    add r12, sp, #64
+; SOFT-NEXT:    add r12, sp, #24
 ; SOFT-NEXT:    vadd.f16 q8, q8, q9
 ; SOFT-NEXT:    vld1.64 {d18, d19}, [r12]
 ; SOFT-NEXT:    add r12, sp, #16
 ; SOFT-NEXT:    vmul.f16 q8, q9, q8
 ; SOFT-NEXT:    vst1.64 {d16, d17}, [r12]
-; SOFT-NEXT:    mov r12, sp
-; SOFT-NEXT:    vldr d16, [sp, #40]
-; SOFT-NEXT:    vst1.16 {d16}, [r12:64]!
-; SOFT-NEXT:    str r3, [r12]
-; SOFT-NEXT:    bl use
-; SOFT-NEXT:    add sp, sp, #32
-; SOFT-NEXT:    pop {r11, pc}
+; SOFT-NEXT:    vldr d16, [sp]
+; SOFT-NEXT:    vstr d16, [sp]
+; SOFT-NEXT:    str r3, [sp, #8]
+; SOFT-NEXT:    b use
 ;
 ; HARD-LABEL: many_args_test:
 ; HARD:       @ %bb.0: @ %entry
@@ -177,33 +172,25 @@ define void @many_args_test(double, float, i16, <4 x half>, <8 x half>, <8 x hal
 ;
 ; SOFTEB-LABEL: many_args_test:
 ; SOFTEB:       @ %bb.0: @ %entry
-; SOFTEB-NEXT:    .save {r11, lr}
-; SOFTEB-NEXT:    push {r11, lr}
-; SOFTEB-NEXT:    .pad #32
-; SOFTEB-NEXT:    sub sp, sp, #32
-; SOFTEB-NEXT:    add r12, sp, #80
-; SOFTEB-NEXT:    mov lr, sp
+; SOFTEB-NEXT:    add r12, sp, #40
 ; SOFTEB-NEXT:    vld1.64 {d16, d17}, [r12]
-; SOFTEB-NEXT:    add r12, sp, #48
+; SOFTEB-NEXT:    add r12, sp, #8
 ; SOFTEB-NEXT:    vrev64.16 q8, q8
 ; SOFTEB-NEXT:    vabs.f16 q8, q8
 ; SOFTEB-NEXT:    vld1.64 {d18, d19}, [r12]
-; SOFTEB-NEXT:    add r12, sp, #64
+; SOFTEB-NEXT:    add r12, sp, #24
 ; SOFTEB-NEXT:    vrev64.16 q9, q9
 ; SOFTEB-NEXT:    vadd.f16 q8, q8, q9
 ; SOFTEB-NEXT:    vld1.64 {d18, d19}, [r12]
 ; SOFTEB-NEXT:    add r12, sp, #16
 ; SOFTEB-NEXT:    vrev64.16 q9, q9
 ; SOFTEB-NEXT:    vmul.f16 q8, q9, q8
-; SOFTEB-NEXT:    vldr d18, [sp, #40]
-; SOFTEB-NEXT:    vrev64.16 d18, d18
-; SOFTEB-NEXT:    vst1.16 {d18}, [lr:64]!
-; SOFTEB-NEXT:    str r3, [lr]
+; SOFTEB-NEXT:    vldr d18, [sp]
 ; SOFTEB-NEXT:    vrev64.16 q8, q8
 ; SOFTEB-NEXT:    vst1.64 {d16, d17}, [r12]
-; SOFTEB-NEXT:    bl use
-; SOFTEB-NEXT:    add sp, sp, #32
-; SOFTEB-NEXT:    pop {r11, pc}
+; SOFTEB-NEXT:    vstr d18, [sp]
+; SOFTEB-NEXT:    str r3, [sp, #8]
+; SOFTEB-NEXT:    b use
 ;
 ; HARDEB-LABEL: many_args_test:
 ; HARDEB:       @ %bb.0: @ %entry
diff --git a/llvm/test/CodeGen/ARM/musttail.ll b/llvm/test/CodeGen/ARM/musttail.ll
new file mode 100644
index 000000000000..622bea3f8763
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/musttail.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=armv7a-none-eabi %s -o - | FileCheck %s
+
+declare i32 @many_args_callee(i32 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5)
+
+define i32 @many_args_tail(i32 %0, i32 %1, i32 %2, i32 %3, i32  %4, i32  %5) {
+; CHECK-LABEL: many_args_tail:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    mov r0, #5
+; CHECK-NEXT:    mov r1, #2
+; CHECK-NEXT:    str r0, [sp]
+; CHECK-NEXT:    mov r0, #6
+; CHECK-NEXT:    str r0, [sp, #4]
+; CHECK-NEXT:    mov r0, #1
+; CHECK-NEXT:    mov r2, #3
+; CHECK-NEXT:    mov r3, #4
+; CHECK-NEXT:    b many_args_callee
+  %ret = tail call i32 @many_args_callee(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6)
+  ret i32 %ret
+}
+
+define i32 @many_args_musttail(i32 %0, i32 %1, i32 %2, i32 %3, i32  %4, i32  %5) {
+; CHECK-LABEL: many_args_musttail:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    mov r0, #5
+; CHECK-NEXT:    mov r1, #2
+; CHECK-NEXT:    str r0, [sp]
+; CHECK-NEXT:    mov r0, #6
+; CHECK-NEXT:    str r0, [sp, #4]
+; CHECK-NEXT:    mov r0, #1
+; CHECK-NEXT:    mov r2, #3
+; CHECK-NEXT:    mov r3, #4
+; CHECK-NEXT:    b many_args_callee
+  %ret = musttail call i32 @many_args_callee(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6)
+  ret i32 %ret
+}
+
+; This function has more arguments than it's tail-callee. This isn't valid for
+; the musttail attribute, but can still be tail-called as a non-guaranteed
+; optimisation, because the outgoing arguments to @many_args_callee fit in the
+; stack space allocated by the caller of @more_args_tail.
+define i32 @more_args_tail(i32 %0, i32 %1, i32 %2, i32 %3, i32  %4, i32 %5, i32 %6) {
+; CHECK-LABEL: more_args_tail:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    mov r0, #5
+; CHECK-NEXT:    mov r1, #2
+; CHECK-NEXT:    str r0, [sp]
+; CHECK-NEXT:    mov r0, #6
+; CHECK-NEXT:    str r0, [sp, #4]
+; CHECK-NEXT:    mov r0, #1
+; CHECK-NEXT:    mov r2, #3
+; CHECK-NEXT:    mov r3, #4
+; CHECK-NEXT:    b many_args_callee
+  %ret = tail call i32 @many_args_callee(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6)
+  ret i32 %ret
+}
+
+; Again, this isn't valid for musttail, but can be tail-called in practice
+; because the stack size if the same.
+define i32 @different_args_tail(i64 %0, i64 %1, i64 %2) {
+; CHECK-LABEL: different_args_tail:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    mov r0, #5
+; CHECK-NEXT:    mov r1, #2
+; CHECK-NEXT:    str r0, [sp]
+; CHECK-NEXT:    mov r0, #6
+; CHECK-NEXT:    str r0, [sp, #4]
+; CHECK-NEXT:    mov r0, #1
+; CHECK-NEXT:    mov r2, #3
+; CHECK-NEXT:    mov r3, #4
+; CHECK-NEXT:    b many_args_callee
+  %ret = tail call i32 @many_args_callee(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6)
+  ret i32 %ret
+}
+
+; Here, the caller requires less stack space for it's arguments than the
+; callee, so it would not ba valid to do a tail-call.
+define i32 @fewer_args_tail(i32 %0, i32 %1, i32 %2, i32 %3, i32  %4) {
+; CHECK-LABEL: fewer_args_tail:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r11, lr}
+; CHECK-NEXT:    push {r11, lr}
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, sp, #8
+; CHECK-NEXT:    mov r1, #6
+; CHECK-NEXT:    mov r0, #5
+; CHECK-NEXT:    strd r0, r1, [sp]
+; CHECK-NEXT:    mov r0, #1
+; CHECK-NEXT:    mov r1, #2
+; CHECK-NEXT:    mov r2, #3
+; CHECK-NEXT:    mov r3, #4
+; CHECK-NEXT:    bl many_args_callee
+; CHECK-NEXT:    add sp, sp, #8
+; CHECK-NEXT:    pop {r11, pc}
+  %ret = tail call i32 @many_args_callee(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6)
+  ret i32 %ret
+}
-- 
GitLab


From 82e64721974b1ed9b112ca98466fa29d21dffc33 Mon Sep 17 00:00:00 2001
From: Oliver Stannard <oliver.stannard@arm.com>
Date: Tue, 24 Sep 2024 17:47:01 +0100
Subject: [PATCH 138/693] [ARM] Allow functions with sret returns to be
 tail-called

It is valid to tail-call a function which returns through an sret
argument, as long as we have an incoming sret pointer to pass on.
---
 llvm/lib/Target/ARM/ARMISelLowering.cpp |  2 +-
 llvm/test/CodeGen/ARM/musttail.ll       | 22 ++++++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 222f08dab03f..4ccfe1b8cd1a 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -3031,7 +3031,7 @@ bool ARMTargetLowering::IsEligibleForTailCallOptimization(
   // return semantics.
   bool isCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
   bool isCallerStructRet = MF.getFunction().hasStructRetAttr();
-  if (isCalleeStructRet || isCallerStructRet) {
+  if (isCalleeStructRet != isCallerStructRet) {
     LLVM_DEBUG(dbgs() << "false (struct-ret)\n");
     return false;
   }
diff --git a/llvm/test/CodeGen/ARM/musttail.ll b/llvm/test/CodeGen/ARM/musttail.ll
index 622bea3f8763..6db45aa9e628 100644
--- a/llvm/test/CodeGen/ARM/musttail.ll
+++ b/llvm/test/CodeGen/ARM/musttail.ll
@@ -95,3 +95,25 @@ define i32 @fewer_args_tail(i32 %0, i32 %1, i32 %2, i32 %3, i32  %4) {
   %ret = tail call i32 @many_args_callee(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6)
   ret i32 %ret
 }
+
+declare void @sret_callee(ptr sret({ double, double }) align 8)
+
+; Functions which return by sret can be tail-called because the incoming sret
+; pointer gets passed through to the callee.
+define void @sret_caller_tail(ptr sret({ double, double }) align 8 %result) {
+; CHECK-LABEL: sret_caller_tail:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    b sret_callee
+entry:
+  tail call void @sret_callee(ptr sret({ double, double }) align 8 %result)
+  ret void
+}
+
+define void @sret_caller_musttail(ptr sret({ double, double }) align 8 %result) {
+; CHECK-LABEL: sret_caller_musttail:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    b sret_callee
+entry:
+  musttail call void @sret_callee(ptr sret({ double, double }) align 8 %result)
+  ret void
+}
-- 
GitLab


From 78ec2e2ed5e3dc61339b51f9d614029314a42005 Mon Sep 17 00:00:00 2001
From: Oliver Stannard <oliver.stannard@arm.com>
Date: Tue, 24 Sep 2024 11:25:17 +0100
Subject: [PATCH 139/693] [ARM] Allow tail calls with byval args

Byval arguments which are passed partially in registers get stored into
the local stack frame, but it is valid to tail-call them because the
part which gets spilled is always re-loaded into registers before doing
the tail-call, so it's OK for the spill area to be deallocated.
---
 llvm/lib/Target/ARM/ARMISelLowering.cpp       |   8 +-
 .../ARM/2013-05-13-AAPCS-byval-padding.ll     |  16 +--
 .../ARM/2013-05-13-AAPCS-byval-padding2.ll    |  13 +-
 llvm/test/CodeGen/ARM/musttail.ll             | 112 ++++++++++++++++++
 4 files changed, 126 insertions(+), 23 deletions(-)

diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 4ccfe1b8cd1a..78d2b98b0a84 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -3075,11 +3075,11 @@ bool ARMTargetLowering::IsEligibleForTailCallOptimization(
     }
   }
 
-  // If Caller's vararg or byval argument has been split between registers and
-  // stack, do not perform tail call, since part of the argument is in caller's
-  // local frame.
+  // If Caller's vararg argument has been split between registers and stack, do
+  // not perform tail call, since part of the argument is in caller's local
+  // frame.
   const ARMFunctionInfo *AFI_Caller = MF.getInfo<ARMFunctionInfo>();
-  if (AFI_Caller->getArgRegsSaveSize()) {
+  if (CLI.IsVarArg && AFI_Caller->getArgRegsSaveSize()) {
     LLVM_DEBUG(dbgs() << "false (arg reg save area)\n");
     return false;
   }
diff --git a/llvm/test/CodeGen/ARM/2013-05-13-AAPCS-byval-padding.ll b/llvm/test/CodeGen/ARM/2013-05-13-AAPCS-byval-padding.ll
index d8e22f4f5312..e186ae3a9615 100644
--- a/llvm/test/CodeGen/ARM/2013-05-13-AAPCS-byval-padding.ll
+++ b/llvm/test/CodeGen/ARM/2013-05-13-AAPCS-byval-padding.ll
@@ -12,17 +12,11 @@ define void @check227(
 ; arg1 --> SP+188
 
 entry:
-
-;CHECK:  sub   sp, sp, #12
-;CHECK:  push  {r11, lr}
-;CHECK:  sub   sp, sp, #4
-;CHECK:  add   r0, sp, #12
-;CHECK:  stm   r0, {r1, r2, r3}
-;CHECK:  ldr   r0, [sp, #212]
-;CHECK:  bl    useInt
-;CHECK:  add   sp, sp, #4
-;CHECK:  pop   {r11, lr}
-;CHECK:  add   sp, sp, #12
+; CHECK: sub     sp, sp, #12
+; CHECK: stm     sp, {r1, r2, r3}
+; CHECK: ldr     r0, [sp, #200]
+; CHECK: add     sp, sp, #12
+; CHECK: b       useInt
 
   %0 = ptrtoint ptr %arg1 to i32
   tail call void @useInt(i32 %0)
diff --git a/llvm/test/CodeGen/ARM/2013-05-13-AAPCS-byval-padding2.ll b/llvm/test/CodeGen/ARM/2013-05-13-AAPCS-byval-padding2.ll
index 0c5d22984b99..efdecce9ae72 100644
--- a/llvm/test/CodeGen/ARM/2013-05-13-AAPCS-byval-padding2.ll
+++ b/llvm/test/CodeGen/ARM/2013-05-13-AAPCS-byval-padding2.ll
@@ -7,14 +7,11 @@
 define void @foo(ptr byval(%struct4bytes) %p0, ; --> R0
                  ptr byval(%struct20bytes) %p1 ; --> R1,R2,R3, [SP+0 .. SP+8)
 ) {
-;CHECK:  sub  sp, sp, #16
-;CHECK:  push  {r11, lr}
-;CHECK:  add  r12, sp, #8
-;CHECK:  stm  r12, {r0, r1, r2, r3}
-;CHECK:  add  r0, sp, #12
-;CHECK:  bl  useInt
-;CHECK:  pop  {r11, lr}
-;CHECK:  add  sp, sp, #16
+;CHECK:  sub     sp, sp, #16
+;CHECK:  stm     sp, {r0, r1, r2, r3}
+;CHECK:  add     r0, sp, #4
+;CHECK:  add     sp, sp, #16
+;CHECK:  b       useInt
 
   %1 = ptrtoint ptr %p1 to i32
   tail call void @useInt(i32 %1)
diff --git a/llvm/test/CodeGen/ARM/musttail.ll b/llvm/test/CodeGen/ARM/musttail.ll
index 6db45aa9e628..aecf8e4579b5 100644
--- a/llvm/test/CodeGen/ARM/musttail.ll
+++ b/llvm/test/CodeGen/ARM/musttail.ll
@@ -117,3 +117,115 @@ entry:
   musttail call void @sret_callee(ptr sret({ double, double }) align 8 %result)
   ret void
 }
+
+; Clang only uses byval for arguments of 65 bytes or larger, but we test with a
+; 20 byte struct to keep the tests more readable. This size was chosen to still
+; make sure that it will be split between registers and the stack, to test all
+; of the interesting code paths in the backend.
+%twenty_bytes = type { [5 x i32] }
+declare void @large_callee(%twenty_bytes* byval(%twenty_bytes) align 4)
+
+; Functions with byval parameters can be tail-called, because the value is
+; actually passed in registers and the stack in the same way for the caller and
+; callee. Within @large_caller the first 16 bytes of the argument are spilled
+; to the local stack frame, but for the tail-call they are passed in r0-r3, so
+; it's safe to de-allocate that memory before the call. Most of the code
+; generated for this isn't needed, but that's a missed optimisation, not a
+; correctness issue.
+define void @large_caller(%twenty_bytes* byval(%twenty_bytes) align 4 %a) {
+; CHECK-LABEL: large_caller:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    add r12, sp, #8
+; CHECK-NEXT:    add lr, sp, #24
+; CHECK-NEXT:    stm r12, {r0, r1, r2, r3}
+; CHECK-NEXT:    add r12, sp, #8
+; CHECK-NEXT:    add r12, r12, #16
+; CHECK-NEXT:    ldr r4, [r12], #4
+; CHECK-NEXT:    str r4, [lr], #4
+; CHECK-NEXT:    pop {r4, lr}
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    b large_callee
+entry:
+  musttail call void @large_callee(%twenty_bytes* byval(%twenty_bytes) align 4 %a)
+  ret void
+}
+
+; As above, but with some inline asm to test that the arguments in r0-r3 are
+; re-loaded before the call.
+define void @large_caller_check_regs(%twenty_bytes* byval(%twenty_bytes) align 4 %a) {
+; CHECK-LABEL: large_caller_check_regs:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    add r12, sp, #8
+; CHECK-NEXT:    add lr, sp, #24
+; CHECK-NEXT:    stm r12, {r0, r1, r2, r3}
+; CHECK-NEXT:    @APP
+; CHECK-NEXT:    @NO_APP
+; CHECK-NEXT:    add r3, sp, #8
+; CHECK-NEXT:    add r0, sp, #8
+; CHECK-NEXT:    add r12, r0, #16
+; CHECK-NEXT:    ldm r3, {r0, r1, r2, r3}
+; CHECK-NEXT:    ldr r4, [r12], #4
+; CHECK-NEXT:    str r4, [lr], #4
+; CHECK-NEXT:    pop {r4, lr}
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    b large_callee
+entry:
+  tail call void asm sideeffect "", "~{r0},~{r1},~{r2},~{r3}"()
+  musttail call void @large_callee(%twenty_bytes* byval(%twenty_bytes) align 4 %a)
+  ret void
+}
+
+; The IR for this one looks dodgy, because it has an alloca passed to a
+; musttail function, but it is passed as a byval argument, so will be copied
+; into the stack space allocated by @large_caller_new_value's caller, so is
+; valid.
+define void @large_caller_new_value(%twenty_bytes* byval(%twenty_bytes) align 4 %a) {
+; CHECK-LABEL: large_caller_new_value:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #36
+; CHECK-NEXT:    sub sp, sp, #36
+; CHECK-NEXT:    add r12, sp, #20
+; CHECK-NEXT:    stm r12, {r0, r1, r2, r3}
+; CHECK-NEXT:    mov r0, #4
+; CHECK-NEXT:    add r1, sp, #36
+; CHECK-NEXT:    str r0, [sp, #16]
+; CHECK-NEXT:    mov r0, #3
+; CHECK-NEXT:    str r0, [sp, #12]
+; CHECK-NEXT:    mov r0, #2
+; CHECK-NEXT:    str r0, [sp, #8]
+; CHECK-NEXT:    mov r0, #1
+; CHECK-NEXT:    str r0, [sp, #4]
+; CHECK-NEXT:    mov r0, #0
+; CHECK-NEXT:    str r0, [sp]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    add r0, r0, #16
+; CHECK-NEXT:    mov r3, #3
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    mov r0, #0
+; CHECK-NEXT:    mov r1, #1
+; CHECK-NEXT:    mov r2, #2
+; CHECK-NEXT:    add sp, sp, #36
+; CHECK-NEXT:    b large_callee
+entry:
+  %y = alloca %twenty_bytes, align 4
+  store i32 0, ptr %y, align 4
+  %0 = getelementptr inbounds i8, ptr %y, i32 4
+  store i32 1, ptr %0, align 4
+  %1 = getelementptr inbounds i8, ptr %y, i32 8
+  store i32 2, ptr %1, align 4
+  %2 = getelementptr inbounds i8, ptr %y, i32 12
+  store i32 3, ptr %2, align 4
+  %3 = getelementptr inbounds i8, ptr %y, i32 16
+  store i32 4, ptr %3, align 4
+  musttail call void @large_callee(%twenty_bytes* byval(%twenty_bytes) align 4 %y)
+  ret void
+}
-- 
GitLab


From a96c14eeb8fcb1a141c0f55ed5d28dd7f1f3e38e Mon Sep 17 00:00:00 2001
From: Kiran <kiran.sturt@arm.com>
Date: Mon, 19 Aug 2024 14:44:50 +0100
Subject: [PATCH 140/693] [Clang] Always forward sret parameters to musttail
 calls

If a call using the musttail attribute returns it's value through an
sret argument pointer, we must forward an incoming sret pointer to it,
instead of creating a new alloca. This is always possible because the
musttail attribute requires the caller and callee to have the same
argument and return types.
---
 clang/lib/CodeGen/CGCall.cpp         |  6 +-
 clang/test/CodeGen/musttail-sret.cpp | 84 ++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/CodeGen/musttail-sret.cpp

diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 5f50ab6f34ff..1949b4ceb7f2 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -5112,7 +5112,11 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   RawAddress SRetAlloca = RawAddress::invalid();
   llvm::Value *UnusedReturnSizePtr = nullptr;
   if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
-    if (IsVirtualFunctionPointerThunk && RetAI.isIndirect()) {
+    // For virtual function pointer thunks and musttail calls, we must always
+    // forward an incoming SRet pointer to the callee, because a local alloca
+    // would be de-allocated before the call. These cases both guarantee that
+    // there will be an incoming SRet argument of the correct type.
+    if ((IsVirtualFunctionPointerThunk || IsMustTail) && RetAI.isIndirect()) {
       SRetPtr = makeNaturalAddressForPointer(CurFn->arg_begin() +
                                                  IRFunctionArgs.getSRetArgNo(),
                                              RetTy, CharUnits::fromQuantity(1));
diff --git a/clang/test/CodeGen/musttail-sret.cpp b/clang/test/CodeGen/musttail-sret.cpp
new file mode 100644
index 000000000000..ca67c218cd67
--- /dev/null
+++ b/clang/test/CodeGen/musttail-sret.cpp
@@ -0,0 +1,84 @@
+// RUN: %clang_cc1 -triple=arm %s -emit-llvm -O3 -o - | FileCheck %s --check-prefix=CHECK-ARM
+// RUN: %clang_cc1 -triple=arm64 %s -emit-llvm -O3 -o - | FileCheck %s --check-prefix=CHECK-ARM64
+// RUN: %clang_cc1 -triple=i686 %s -emit-llvm -O3 -o - | FileCheck %s --check-prefix=CHECK-X86
+// RUN: %clang_cc1 -triple=x86_64 %s -emit-llvm -O3 -o - | FileCheck %s --check-prefix=CHECK-X64
+
+// Sret tests
+struct Big {
+  int a, b, c, d, e, f, g, h;
+};
+
+struct Big F1(signed short P0);
+
+struct Big F2(signed short P0) {
+  signed short P1 = 20391;
+  [[clang::musttail]] return F1(P1);
+}
+
+// CHECK-NOT: alloca
+// CHECK-ARM: musttail call arm_aapcscc void @_Z2F1s(ptr dead_on_unwind writable sret(%struct.Big) align 4 %agg.result, i16 noundef signext 20391)
+// CHECK-ARM64: musttail call void @_Z2F1s(ptr dead_on_unwind writable sret(%struct.Big) align 4 %agg.result, i16 noundef 20391)
+// CHECK-X86: musttail call void @_Z2F1s(ptr dead_on_unwind writable sret(%struct.Big) align 4 %agg.result, i16 noundef signext 20391)
+// CHECK-X64: musttail call void @_Z2F1s(ptr dead_on_unwind writable sret(%struct.Big) align 4 %agg.result, i16 noundef signext 20391)
+
+struct ReallyBig {
+  int a[100];
+};
+
+// Indirect sret tests
+// Function pointer for testing indirect musttail call.
+struct FunctionPointers {
+  ReallyBig (*F3)(int, int, int, int, float, double);
+  ReallyBig (*F4)(int, int, int, char, float, double);
+};
+
+struct ReallyBig F3(int P0, int P1, int P2, int P3, float P4, double P5);
+struct ReallyBig F4(int P0, int P1, int P2, char P3, float P4, double P5);
+
+static struct FunctionPointers FP = {F3, F4};
+
+struct ReallyBig F5 (int P0, int P1, int P2, int P3, float P4, double P5) {
+  [[clang::musttail]] return FP.F3(P0, P1, P2, P3, P4, P5);
+}
+
+// CHECK-NOT: alloca
+// CHECK-ARM: musttail call arm_aapcscc void @_Z2F3iiiifd(ptr dead_on_unwind writable sret(%struct.ReallyBig) align 4 %agg.result, i32 noundef %P0, i32 noundef %P1, i32 noundef %P2, i32 noundef %P3, float noundef %P4, double noundef %P5)
+// CHECK-ARM64: musttail call void @_Z2F3iiiifd(ptr dead_on_unwind writable sret(%struct.ReallyBig) align 4 %agg.result, i32 noundef %P0, i32 noundef %P1, i32 noundef %P2, i32 noundef %P3, float noundef %P4, double noundef %P5)
+// CHECK-X86: musttail call void @_Z2F3iiiifd(ptr dead_on_unwind writable sret(%struct.ReallyBig) align 4 %agg.result, i32 noundef %P0, i32 noundef %P1, i32 noundef %P2, i32 noundef %P3, float noundef %P4, double noundef %P5)
+// CHECK-X64: musttail call void @_Z2F3iiiifd(ptr dead_on_unwind writable sret(%struct.ReallyBig) align 4 %agg.result, i32 noundef %P0, i32 noundef %P1, i32 noundef %P2, i32 noundef %P3, float noundef %P4, double noundef %P5)
+
+struct ReallyBig F6 (int P0, int P1, int P2, char P3, float P4, double P5) {
+  [[clang::musttail]] return FP.F4(P0, P1, P2, P3, P4, P5);
+}
+
+// Complex and BitInt. Special cases for sret.
+// CHECK-NOT: alloca
+// CHECK-ARM: musttail call arm_aapcscc void @_Z2F4iiicfd(ptr dead_on_unwind writable sret(%struct.ReallyBig) align 4 %agg.result, i32 noundef %P0, i32 noundef %P1, i32 noundef %P2, i8 noundef signext %P3, float noundef %P4, double noundef %P5)
+// CHECK-ARM64: musttail call void @_Z2F4iiicfd(ptr dead_on_unwind writable sret(%struct.ReallyBig) align 4 %agg.result, i32 noundef %P0, i32 noundef %P1, i32 noundef %P2, i8 noundef %P3, float noundef %P4, double noundef %P5)
+// CHECK-X86: musttail call void @_Z2F4iiicfd(ptr dead_on_unwind writable sret(%struct.ReallyBig) align 4 %agg.result, i32 noundef %P0, i32 noundef %P1, i32 noundef %P2, i8 noundef signext %P3, float noundef %P4, double noundef %P5)
+// CHECK-X64: musttail call void @_Z2F4iiicfd(ptr dead_on_unwind writable sret(%struct.ReallyBig) align 4 %agg.result, i32 noundef %P0, i32 noundef %P1, i32 noundef %P2, i8 noundef signext %P3, float noundef %P4, double noundef %P5)
+
+double _Complex F7(signed short P0);
+
+double _Complex F8(signed short P0) {
+  signed short P1 = 20391;
+  [[clang::musttail]] return F7(P1);
+}
+
+// CHECK-NOT: alloca
+// CHECK-ARM: musttail call arm_aapcscc void @_Z2F7s(ptr dead_on_unwind writable sret({ double, double }) align 8 %agg.result, i16 noundef signext 20391)
+// CHECK-ARM64: musttail call noundef { double, double } @_Z2F7s(i16 noundef 20391)
+// CHECK-X86: musttail call void @_Z2F7s(ptr dead_on_unwind writable sret({ double, double }) align 4 %agg.result, i16 noundef signext 20391) 
+// CHECK-X64: musttail call noundef { double, double } @_Z2F7s(i16 noundef signext 20391)
+
+signed _BitInt(100) F9(float P0, float P1, double P2, char P3);
+
+signed _BitInt(100) F10(float P0, float P1, double P2, char P3) {
+  [[clang::musttail]] return F9(P0, P1, P2, P3);
+}
+
+// CHECK-NOT: alloca
+// CHECK-ARM: musttail call arm_aapcscc void @_Z2F9ffdc(ptr dead_on_unwind writable sret(i128) align 8 %agg.result, float noundef %P0, float noundef %P1, double noundef %P2, i8 noundef signext %P3)
+// CHECK-ARM64: musttail call noundef i100 @_Z2F9ffdc(float noundef %P0, float noundef %P1, double noundef %P2, i8 noundef %P3)
+// CHECK-X86: musttail call void @_Z2F9ffdc(ptr dead_on_unwind writable sret(i128) align 4 %agg.result, float noundef %P0, float noundef %P1, double noundef %P2, i8 noundef signext %P3)
+// CHECK-X64: musttail call noundef { i64, i64 } @_Z2F9ffdc(float noundef %P0, float noundef %P1, double noundef %P2, i8 noundef signext %P3)
\ No newline at end of file
-- 
GitLab


From 914a3990d1a055f5f1848c6979f621ceff97fa7c Mon Sep 17 00:00:00 2001
From: Oliver Stannard <oliver.stannard@arm.com>
Date: Thu, 26 Sep 2024 11:21:09 +0100
Subject: [PATCH 141/693] [ARM] Avoid clobbering byval arguments when passing
 to tail-calls

When passing byval arguments to tail-calls, we need to store them into
the stack memory in which this the caller received it's arguments. If
any of the outgoing arguments are forwarded from incoming byval
arguments, then the source of the copy is from the same stack memory.
This can result in the copy corrupting a value which is still to be
read.

The fix is to first make a copy of the outgoing byval arguments in local
stack space, and then copy them to their final location. This fixes the
correctness issue, but results in extra copying, which could be
optimised.
---
 llvm/lib/Target/ARM/ARMISelLowering.cpp |  53 +++++++-
 llvm/test/CodeGen/ARM/musttail.ll       | 158 ++++++++++++++++++++----
 2 files changed, 183 insertions(+), 28 deletions(-)

diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 78d2b98b0a84..6cc0b681193f 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -2380,6 +2380,7 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
   MachineFunction &MF = DAG.getMachineFunction();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
+  MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
   MachineFunction::CallSiteInfo CSInfo;
   bool isStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet();
   bool isThisReturn = false;
@@ -2492,6 +2493,45 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   RegsToPassVector RegsToPass;
   SmallVector<SDValue, 8> MemOpChains;
 
+  // If we are doing a tail-call, any byval arguments will be written to stack
+  // space which was used for incoming arguments. If any the values being used
+  // are incoming byval arguments to this function, then they might be
+  // overwritten by the stores of the outgoing arguments. To avoid this, we
+  // need to make a temporary copy of them in local stack space, then copy back
+  // to the argument area.
+  // TODO This could be optimised to skip byvals which are already being copied
+  // from local stack space, or which are copied from the incoming stack at the
+  // exact same location.
+  DenseMap<unsigned, SDValue> ByValTemporaries;
+  SDValue ByValTempChain;
+  if (isTailCall) {
+    for (unsigned ArgIdx = 0, e = OutVals.size(); ArgIdx != e; ++ArgIdx) {
+      SDValue Arg = OutVals[ArgIdx];
+      ISD::ArgFlagsTy Flags = Outs[ArgIdx].Flags;
+
+      if (Flags.isByVal()) {
+        int FrameIdx = MFI.CreateStackObject(
+            Flags.getByValSize(), Flags.getNonZeroByValAlign(), false);
+        SDValue Dst =
+            DAG.getFrameIndex(FrameIdx, getPointerTy(DAG.getDataLayout()));
+
+        SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
+        SDValue AlignNode =
+            DAG.getConstant(Flags.getNonZeroByValAlign().value(), dl, MVT::i32);
+
+        SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
+        SDValue Ops[] = { Chain, Dst, Arg, SizeNode, AlignNode};
+        MemOpChains.push_back(DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs,
+                                          Ops));
+        ByValTemporaries[ArgIdx] = Dst;
+      }
+    }
+    if (!MemOpChains.empty()) {
+      ByValTempChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
+      MemOpChains.clear();
+    }
+  }
+
   // During a tail call, stores to the argument area must happen after all of
   // the function's incoming arguments have been loaded because they may alias.
   // This is done by folding in a TokenFactor from LowerFormalArguments, but
@@ -2529,6 +2569,9 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
     if (isTailCall && VA.isMemLoc() && !AfterFormalArgLoads) {
       Chain = DAG.getStackArgumentTokenFactor(Chain);
+      if (ByValTempChain)
+        Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chain,
+                            ByValTempChain);
       AfterFormalArgLoads = true;
     }
 
@@ -2600,6 +2643,12 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
       unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
 
+      SDValue ByValSrc;
+      if (ByValTemporaries.contains(realArgIdx))
+        ByValSrc = ByValTemporaries[realArgIdx];
+      else
+        ByValSrc = Arg;
+
       if (CurByValIdx < ByValArgsCount) {
 
         unsigned RegBegin, RegEnd;
@@ -2610,7 +2659,7 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         unsigned int i, j;
         for (i = 0, j = RegBegin; j < RegEnd; i++, j++) {
           SDValue Const = DAG.getConstant(4*i, dl, MVT::i32);
-          SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
+          SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, ByValSrc, Const);
           SDValue Load =
               DAG.getLoad(PtrVT, dl, Chain, AddArg, MachinePointerInfo(),
                           DAG.InferPtrAlign(AddArg));
@@ -2632,7 +2681,7 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         std::tie(Dst, DstInfo) =
             computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
         SDValue SrcOffset = DAG.getIntPtrConstant(4*offset, dl);
-        SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, SrcOffset);
+        SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, ByValSrc, SrcOffset);
         SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl,
                                            MVT::i32);
         SDValue AlignNode =
diff --git a/llvm/test/CodeGen/ARM/musttail.ll b/llvm/test/CodeGen/ARM/musttail.ll
index aecf8e4579b5..3c577134f696 100644
--- a/llvm/test/CodeGen/ARM/musttail.ll
+++ b/llvm/test/CodeGen/ARM/musttail.ll
@@ -139,13 +139,28 @@ define void @large_caller(%twenty_bytes* byval(%twenty_bytes) align 4 %a) {
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    add r12, sp, #8
-; CHECK-NEXT:    add lr, sp, #24
+; CHECK-NEXT:    .pad #20
+; CHECK-NEXT:    sub sp, sp, #20
+; CHECK-NEXT:    add r12, sp, #28
+; CHECK-NEXT:    add lr, sp, #44
 ; CHECK-NEXT:    stm r12, {r0, r1, r2, r3}
-; CHECK-NEXT:    add r12, sp, #8
-; CHECK-NEXT:    add r12, r12, #16
+; CHECK-NEXT:    add r0, sp, #28
+; CHECK-NEXT:    mov r1, sp
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    add r12, r1, #16
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldm sp, {r0, r1, r2, r3}
 ; CHECK-NEXT:    ldr r4, [r12], #4
 ; CHECK-NEXT:    str r4, [lr], #4
+; CHECK-NEXT:    add sp, sp, #20
 ; CHECK-NEXT:    pop {r4, lr}
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    b large_callee
@@ -163,17 +178,30 @@ define void @large_caller_check_regs(%twenty_bytes* byval(%twenty_bytes) align 4
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    add r12, sp, #8
-; CHECK-NEXT:    add lr, sp, #24
+; CHECK-NEXT:    .pad #20
+; CHECK-NEXT:    sub sp, sp, #20
+; CHECK-NEXT:    add r12, sp, #28
+; CHECK-NEXT:    add lr, sp, #44
 ; CHECK-NEXT:    stm r12, {r0, r1, r2, r3}
 ; CHECK-NEXT:    @APP
 ; CHECK-NEXT:    @NO_APP
-; CHECK-NEXT:    add r3, sp, #8
-; CHECK-NEXT:    add r0, sp, #8
-; CHECK-NEXT:    add r12, r0, #16
-; CHECK-NEXT:    ldm r3, {r0, r1, r2, r3}
+; CHECK-NEXT:    add r0, sp, #28
+; CHECK-NEXT:    mov r1, sp
+; CHECK-NEXT:    add r12, r1, #16
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldm sp, {r0, r1, r2, r3}
 ; CHECK-NEXT:    ldr r4, [r12], #4
 ; CHECK-NEXT:    str r4, [lr], #4
+; CHECK-NEXT:    add sp, sp, #20
 ; CHECK-NEXT:    pop {r4, lr}
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    b large_callee
@@ -190,30 +218,44 @@ entry:
 define void @large_caller_new_value(%twenty_bytes* byval(%twenty_bytes) align 4 %a) {
 ; CHECK-LABEL: large_caller_new_value:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #36
-; CHECK-NEXT:    sub sp, sp, #36
-; CHECK-NEXT:    add r12, sp, #20
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .pad #40
+; CHECK-NEXT:    sub sp, sp, #40
+; CHECK-NEXT:    add r12, sp, #48
+; CHECK-NEXT:    add lr, sp, #64
 ; CHECK-NEXT:    stm r12, {r0, r1, r2, r3}
 ; CHECK-NEXT:    mov r0, #4
-; CHECK-NEXT:    add r1, sp, #36
-; CHECK-NEXT:    str r0, [sp, #16]
+; CHECK-NEXT:    mov r1, sp
+; CHECK-NEXT:    str r0, [sp, #36]
 ; CHECK-NEXT:    mov r0, #3
-; CHECK-NEXT:    str r0, [sp, #12]
+; CHECK-NEXT:    str r0, [sp, #32]
 ; CHECK-NEXT:    mov r0, #2
-; CHECK-NEXT:    str r0, [sp, #8]
+; CHECK-NEXT:    str r0, [sp, #28]
 ; CHECK-NEXT:    mov r0, #1
-; CHECK-NEXT:    str r0, [sp, #4]
+; CHECK-NEXT:    str r0, [sp, #24]
 ; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    str r0, [sp]
-; CHECK-NEXT:    mov r0, sp
-; CHECK-NEXT:    add r0, r0, #16
-; CHECK-NEXT:    mov r3, #3
+; CHECK-NEXT:    str r0, [sp, #20]
+; CHECK-NEXT:    add r0, sp, #20
+; CHECK-NEXT:    add r12, r1, #16
 ; CHECK-NEXT:    ldr r2, [r0], #4
 ; CHECK-NEXT:    str r2, [r1], #4
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    mov r1, #1
-; CHECK-NEXT:    mov r2, #2
-; CHECK-NEXT:    add sp, sp, #36
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldm sp, {r0, r1, r2, r3}
+; CHECK-NEXT:    ldr r4, [r12], #4
+; CHECK-NEXT:    str r4, [lr], #4
+; CHECK-NEXT:    add sp, sp, #40
+; CHECK-NEXT:    pop {r4, lr}
+; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    b large_callee
 entry:
   %y = alloca %twenty_bytes, align 4
@@ -229,3 +271,67 @@ entry:
   musttail call void @large_callee(%twenty_bytes* byval(%twenty_bytes) align 4 %y)
   ret void
 }
+
+declare void @two_byvals_callee(%twenty_bytes* byval(%twenty_bytes) align 4, %twenty_bytes* byval(%twenty_bytes) align 4)
+define void @swap_byvals(%twenty_bytes* byval(%twenty_bytes) align 4 %a, %twenty_bytes* byval(%twenty_bytes) align 4 %b) {
+; CHECK-LABEL: swap_byvals:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .save {r4, r5, r11, lr}
+; CHECK-NEXT:    push {r4, r5, r11, lr}
+; CHECK-NEXT:    .pad #40
+; CHECK-NEXT:    sub sp, sp, #40
+; CHECK-NEXT:    add r12, sp, #56
+; CHECK-NEXT:    add lr, sp, #20
+; CHECK-NEXT:    stm r12, {r0, r1, r2, r3}
+; CHECK-NEXT:    add r0, sp, #56
+; CHECK-NEXT:    mov r12, sp
+; CHECK-NEXT:    ldr r1, [r0], #4
+; CHECK-NEXT:    mov r2, r12
+; CHECK-NEXT:    str r1, [r2], #4
+; CHECK-NEXT:    add r3, sp, #20
+; CHECK-NEXT:    ldr r1, [r0], #4
+; CHECK-NEXT:    add r4, sp, #76
+; CHECK-NEXT:    str r1, [r2], #4
+; CHECK-NEXT:    ldr r1, [r0], #4
+; CHECK-NEXT:    str r1, [r2], #4
+; CHECK-NEXT:    ldr r1, [r0], #4
+; CHECK-NEXT:    str r1, [r2], #4
+; CHECK-NEXT:    ldr r1, [r0], #4
+; CHECK-NEXT:    add r0, sp, #76
+; CHECK-NEXT:    str r1, [r2], #4
+; CHECK-NEXT:    mov r2, lr
+; CHECK-NEXT:    ldr r1, [r0], #4
+; CHECK-NEXT:    str r1, [r2], #4
+; CHECK-NEXT:    ldr r1, [r0], #4
+; CHECK-NEXT:    str r1, [r2], #4
+; CHECK-NEXT:    ldr r1, [r0], #4
+; CHECK-NEXT:    str r1, [r2], #4
+; CHECK-NEXT:    ldr r1, [r0], #4
+; CHECK-NEXT:    str r1, [r2], #4
+; CHECK-NEXT:    ldr r1, [r0], #4
+; CHECK-NEXT:    str r1, [r2], #4
+; CHECK-NEXT:    ldm r3, {r0, r1, r2, r3}
+; CHECK-NEXT:    ldr r5, [r12], #4
+; CHECK-NEXT:    str r5, [r4], #4
+; CHECK-NEXT:    ldr r5, [r12], #4
+; CHECK-NEXT:    str r5, [r4], #4
+; CHECK-NEXT:    ldr r5, [r12], #4
+; CHECK-NEXT:    str r5, [r4], #4
+; CHECK-NEXT:    ldr r5, [r12], #4
+; CHECK-NEXT:    str r5, [r4], #4
+; CHECK-NEXT:    ldr r5, [r12], #4
+; CHECK-NEXT:    str r5, [r4], #4
+; CHECK-NEXT:    add r5, lr, #16
+; CHECK-NEXT:    add r12, sp, #72
+; CHECK-NEXT:    ldr r4, [r5], #4
+; CHECK-NEXT:    str r4, [r12], #4
+; CHECK-NEXT:    add sp, sp, #40
+; CHECK-NEXT:    pop {r4, r5, r11, lr}
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    b two_byvals_callee
+entry:
+  musttail call void @two_byvals_callee(%twenty_bytes* byval(%twenty_bytes) align 4 %b, %twenty_bytes* byval(%twenty_bytes) align 4 %a)
+  ret void
+}
-- 
GitLab


From 376d7b27fa3de4f72c2a3cec4f941c1ca3f1d7f2 Mon Sep 17 00:00:00 2001
From: Oliver Stannard <oliver.stannard@arm.com>
Date: Thu, 26 Sep 2024 16:34:03 +0100
Subject: [PATCH 142/693] [ARM] Optimise byval arguments in tail-calls

We don't need to copy byval arguments to tail calls via a temporary, if
we can prove that we are not copying from the outgoing argument area.
This patch does this when the source if the argument is one of:
* Memory in the local stack frame, which can't be used for tail-call
  arguments.
* A global variable.

We can also avoid doing the copy completely if the source and
destination are the same memory location, which is the case when the
caller and callee have the same signature, and pass some arguments
through unmodified.
---
 llvm/lib/Target/ARM/ARMISelLowering.cpp | 122 ++++++++++++++---
 llvm/lib/Target/ARM/ARMISelLowering.h   |  16 +++
 llvm/test/CodeGen/ARM/musttail.ll       | 175 ++++++++++++------------
 3 files changed, 207 insertions(+), 106 deletions(-)

diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 6cc0b681193f..a98b7a842092 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -2325,6 +2325,59 @@ std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
   return std::make_pair(DstAddr, DstInfo);
 }
 
+// Returns the type of copying which is required to set up a byval argument to
+// a tail-called function. This isn't needed for non-tail calls, because they
+// always need the equivalent of CopyOnce, but tail-calls sometimes need two to
+// avoid clobbering another argument (CopyViaTemp), and sometimes can be
+// optimised to zero copies when forwarding an argument from the caller's
+// caller (NoCopy).
+ARMTargetLowering::ByValCopyKind ARMTargetLowering::ByValNeedsCopyForTailCall(
+    SelectionDAG &DAG, SDValue Src, SDValue Dst, ISD::ArgFlagsTy Flags) const {
+  MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
+  ARMFunctionInfo *AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
+
+  // Globals are always safe to copy from.
+  if (isa<GlobalAddressSDNode>(Src) || isa<ExternalSymbolSDNode>(Src))
+    return CopyOnce;
+
+  // Can only analyse frame index nodes, conservatively assume we need a
+  // temporary.
+  auto *SrcFrameIdxNode = dyn_cast<FrameIndexSDNode>(Src);
+  auto *DstFrameIdxNode = dyn_cast<FrameIndexSDNode>(Dst);
+  if (!SrcFrameIdxNode || !DstFrameIdxNode)
+    return CopyViaTemp;
+
+  int SrcFI = SrcFrameIdxNode->getIndex();
+  int DstFI = DstFrameIdxNode->getIndex();
+  assert(MFI.isFixedObjectIndex(DstFI) &&
+         "byval passed in non-fixed stack slot");
+
+  int64_t SrcOffset = MFI.getObjectOffset(SrcFI);
+  int64_t DstOffset = MFI.getObjectOffset(DstFI);
+
+  // If the source is in the local frame, then the copy to the argument memory
+  // is always valid.
+  bool FixedSrc = MFI.isFixedObjectIndex(SrcFI);
+  if (!FixedSrc ||
+      (FixedSrc && SrcOffset < -(int64_t)AFI->getArgRegsSaveSize()))
+    return CopyOnce;
+
+  // In the case of byval arguments split between registers and the stack,
+  // computeAddrForCallArg returns a FrameIndex which corresponds only to the
+  // stack portion, but the Src SDValue will refer to the full value, including
+  // the local stack memory that the register portion gets stored into. We only
+  // need to compare them for equality, so normalise on the full value version.
+  uint64_t RegSize = Flags.getByValSize() - MFI.getObjectSize(DstFI);
+  DstOffset -= RegSize;
+
+  // If the value is already in the correct location, then no copying is
+  // needed. If not, then we need to copy via a temporary.
+  if (SrcOffset == DstOffset)
+    return NoCopy;
+  else
+    return CopyViaTemp;
+}
+
 void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG,
                                          SDValue Chain, SDValue &Arg,
                                          RegsToPassVector &RegsToPass,
@@ -2499,37 +2552,58 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // overwritten by the stores of the outgoing arguments. To avoid this, we
   // need to make a temporary copy of them in local stack space, then copy back
   // to the argument area.
-  // TODO This could be optimised to skip byvals which are already being copied
-  // from local stack space, or which are copied from the incoming stack at the
-  // exact same location.
   DenseMap<unsigned, SDValue> ByValTemporaries;
   SDValue ByValTempChain;
   if (isTailCall) {
-    for (unsigned ArgIdx = 0, e = OutVals.size(); ArgIdx != e; ++ArgIdx) {
-      SDValue Arg = OutVals[ArgIdx];
+    SmallVector<SDValue, 8> ByValCopyChains;
+    for (const CCValAssign &VA : ArgLocs) {
+      unsigned ArgIdx = VA.getValNo();
+      SDValue Src = OutVals[ArgIdx];
       ISD::ArgFlagsTy Flags = Outs[ArgIdx].Flags;
 
-      if (Flags.isByVal()) {
-        int FrameIdx = MFI.CreateStackObject(
+      if (!Flags.isByVal())
+        continue;
+
+      SDValue Dst;
+      MachinePointerInfo DstInfo;
+      std::tie(Dst, DstInfo) =
+          computeAddrForCallArg(dl, DAG, VA, SDValue(), true, SPDiff);
+      ByValCopyKind Copy = ByValNeedsCopyForTailCall(DAG, Src, Dst, Flags);
+
+      if (Copy == NoCopy) {
+        // If the argument is already at the correct offset on the stack
+        // (because we are forwarding a byval argument from our caller), we
+        // don't need any copying.
+        continue;
+      } else if (Copy == CopyOnce) {
+        // If the argument is in our local stack frame, no other argument
+        // preparation can clobber it, so we can copy it to the final location
+        // later.
+        ByValTemporaries[ArgIdx] = Src;
+      } else {
+        assert(Copy == CopyViaTemp && "unexpected enum value");
+        // If we might be copying this argument from the outgoing argument
+        // stack area, we need to copy via a temporary in the local stack
+        // frame.
+        int TempFrameIdx = MFI.CreateStackObject(
             Flags.getByValSize(), Flags.getNonZeroByValAlign(), false);
-        SDValue Dst =
-            DAG.getFrameIndex(FrameIdx, getPointerTy(DAG.getDataLayout()));
+        SDValue Temp =
+            DAG.getFrameIndex(TempFrameIdx, getPointerTy(DAG.getDataLayout()));
 
         SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
         SDValue AlignNode =
             DAG.getConstant(Flags.getNonZeroByValAlign().value(), dl, MVT::i32);
 
         SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
-        SDValue Ops[] = { Chain, Dst, Arg, SizeNode, AlignNode};
-        MemOpChains.push_back(DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs,
-                                          Ops));
-        ByValTemporaries[ArgIdx] = Dst;
+        SDValue Ops[] = {Chain, Temp, Src, SizeNode, AlignNode};
+        ByValCopyChains.push_back(
+            DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs, Ops));
+        ByValTemporaries[ArgIdx] = Temp;
       }
     }
-    if (!MemOpChains.empty()) {
-      ByValTempChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
-      MemOpChains.clear();
-    }
+    if (!ByValCopyChains.empty())
+      ByValTempChain =
+          DAG.getNode(ISD::TokenFactor, dl, MVT::Other, ByValCopyChains);
   }
 
   // During a tail call, stores to the argument area must happen after all of
@@ -2644,13 +2718,17 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
 
       SDValue ByValSrc;
-      if (ByValTemporaries.contains(realArgIdx))
+      bool NeedsStackCopy;
+      if (ByValTemporaries.contains(realArgIdx)) {
         ByValSrc = ByValTemporaries[realArgIdx];
-      else
+        NeedsStackCopy = true;
+      } else {
         ByValSrc = Arg;
+        NeedsStackCopy = !isTailCall;
+      }
 
+      // If part of the argument is in registers, load them.
       if (CurByValIdx < ByValArgsCount) {
-
         unsigned RegBegin, RegEnd;
         CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
 
@@ -2674,7 +2752,9 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         CCInfo.nextInRegsParam();
       }
 
-      if (Flags.getByValSize() > 4*offset) {
+      // If the memory part of the argument isn't already in the correct place
+      // (which can happen with tail calls), copy it into the argument area.
+      if (NeedsStackCopy && Flags.getByValSize() > 4 * offset) {
         auto PtrVT = getPointerTy(DAG.getDataLayout());
         SDValue Dst;
         MachinePointerInfo DstInfo;
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index ef651bc3d84c..0e086f3340cc 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -395,6 +395,19 @@ class VectorType;
   //  ARMTargetLowering - ARM Implementation of the TargetLowering interface
 
   class ARMTargetLowering : public TargetLowering {
+    // Copying needed for an outgoing byval argument.
+    enum ByValCopyKind {
+      // Argument is already in the correct location, no copy needed.
+      NoCopy,
+      // Argument value is currently in the local stack frame, needs copying to
+      // outgoing arguemnt area.
+      CopyOnce,
+      // Argument value is currently in the outgoing argument area, but not at
+      // the correct offset, so needs copying via a temporary in local stack
+      // space.
+      CopyViaTemp,
+    };
+
   public:
     explicit ARMTargetLowering(const TargetMachine &TM,
                                const ARMSubtarget &STI);
@@ -809,6 +822,9 @@ class VectorType;
     computeAddrForCallArg(const SDLoc &dl, SelectionDAG &DAG,
                           const CCValAssign &VA, SDValue StackPtr,
                           bool IsTailCall, int SPDiff) const;
+    ByValCopyKind ByValNeedsCopyForTailCall(SelectionDAG &DAG, SDValue Src,
+                                            SDValue Dst,
+                                            ISD::ArgFlagsTy Flags) const;
     SDValue LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/ARM/musttail.ll b/llvm/test/CodeGen/ARM/musttail.ll
index 3c577134f696..aa7b81d07002 100644
--- a/llvm/test/CodeGen/ARM/musttail.ll
+++ b/llvm/test/CodeGen/ARM/musttail.ll
@@ -129,40 +129,15 @@ declare void @large_callee(%twenty_bytes* byval(%twenty_bytes) align 4)
 ; actually passed in registers and the stack in the same way for the caller and
 ; callee. Within @large_caller the first 16 bytes of the argument are spilled
 ; to the local stack frame, but for the tail-call they are passed in r0-r3, so
-; it's safe to de-allocate that memory before the call. Most of the code
-; generated for this isn't needed, but that's a missed optimisation, not a
-; correctness issue.
+; it's safe to de-allocate that memory before the call.
+; TODO: The SUB and STM instructions are unnecessary and could be optimised
+; out, but the behaviour of this is still correct.
 define void @large_caller(%twenty_bytes* byval(%twenty_bytes) align 4 %a) {
 ; CHECK-LABEL: large_caller:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .pad #16
 ; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    .pad #20
-; CHECK-NEXT:    sub sp, sp, #20
-; CHECK-NEXT:    add r12, sp, #28
-; CHECK-NEXT:    add lr, sp, #44
-; CHECK-NEXT:    stm r12, {r0, r1, r2, r3}
-; CHECK-NEXT:    add r0, sp, #28
-; CHECK-NEXT:    mov r1, sp
-; CHECK-NEXT:    ldr r2, [r0], #4
-; CHECK-NEXT:    add r12, r1, #16
-; CHECK-NEXT:    str r2, [r1], #4
-; CHECK-NEXT:    ldr r2, [r0], #4
-; CHECK-NEXT:    str r2, [r1], #4
-; CHECK-NEXT:    ldr r2, [r0], #4
-; CHECK-NEXT:    str r2, [r1], #4
-; CHECK-NEXT:    ldr r2, [r0], #4
-; CHECK-NEXT:    str r2, [r1], #4
-; CHECK-NEXT:    ldr r2, [r0], #4
-; CHECK-NEXT:    str r2, [r1], #4
-; CHECK-NEXT:    ldm sp, {r0, r1, r2, r3}
-; CHECK-NEXT:    ldr r4, [r12], #4
-; CHECK-NEXT:    str r4, [lr], #4
-; CHECK-NEXT:    add sp, sp, #20
-; CHECK-NEXT:    pop {r4, lr}
-; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    stm sp!, {r0, r1, r2, r3}
 ; CHECK-NEXT:    b large_callee
 entry:
   musttail call void @large_callee(%twenty_bytes* byval(%twenty_bytes) align 4 %a)
@@ -176,34 +151,10 @@ define void @large_caller_check_regs(%twenty_bytes* byval(%twenty_bytes) align 4
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .pad #16
 ; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    .pad #20
-; CHECK-NEXT:    sub sp, sp, #20
-; CHECK-NEXT:    add r12, sp, #28
-; CHECK-NEXT:    add lr, sp, #44
-; CHECK-NEXT:    stm r12, {r0, r1, r2, r3}
+; CHECK-NEXT:    stm sp, {r0, r1, r2, r3}
 ; CHECK-NEXT:    @APP
 ; CHECK-NEXT:    @NO_APP
-; CHECK-NEXT:    add r0, sp, #28
-; CHECK-NEXT:    mov r1, sp
-; CHECK-NEXT:    add r12, r1, #16
-; CHECK-NEXT:    ldr r2, [r0], #4
-; CHECK-NEXT:    str r2, [r1], #4
-; CHECK-NEXT:    ldr r2, [r0], #4
-; CHECK-NEXT:    str r2, [r1], #4
-; CHECK-NEXT:    ldr r2, [r0], #4
-; CHECK-NEXT:    str r2, [r1], #4
-; CHECK-NEXT:    ldr r2, [r0], #4
-; CHECK-NEXT:    str r2, [r1], #4
-; CHECK-NEXT:    ldr r2, [r0], #4
-; CHECK-NEXT:    str r2, [r1], #4
-; CHECK-NEXT:    ldm sp, {r0, r1, r2, r3}
-; CHECK-NEXT:    ldr r4, [r12], #4
-; CHECK-NEXT:    str r4, [lr], #4
-; CHECK-NEXT:    add sp, sp, #20
-; CHECK-NEXT:    pop {r4, lr}
-; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    pop {r0, r1, r2, r3}
 ; CHECK-NEXT:    b large_callee
 entry:
   tail call void asm sideeffect "", "~{r0},~{r1},~{r2},~{r3}"()
@@ -218,44 +169,30 @@ entry:
 define void @large_caller_new_value(%twenty_bytes* byval(%twenty_bytes) align 4 %a) {
 ; CHECK-LABEL: large_caller_new_value:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    .pad #40
-; CHECK-NEXT:    sub sp, sp, #40
-; CHECK-NEXT:    add r12, sp, #48
-; CHECK-NEXT:    add lr, sp, #64
+; CHECK-NEXT:    .pad #36
+; CHECK-NEXT:    sub sp, sp, #36
+; CHECK-NEXT:    add r12, sp, #20
 ; CHECK-NEXT:    stm r12, {r0, r1, r2, r3}
 ; CHECK-NEXT:    mov r0, #4
-; CHECK-NEXT:    mov r1, sp
-; CHECK-NEXT:    str r0, [sp, #36]
+; CHECK-NEXT:    add r1, sp, #36
+; CHECK-NEXT:    str r0, [sp, #16]
 ; CHECK-NEXT:    mov r0, #3
-; CHECK-NEXT:    str r0, [sp, #32]
+; CHECK-NEXT:    str r0, [sp, #12]
 ; CHECK-NEXT:    mov r0, #2
-; CHECK-NEXT:    str r0, [sp, #28]
+; CHECK-NEXT:    str r0, [sp, #8]
 ; CHECK-NEXT:    mov r0, #1
-; CHECK-NEXT:    str r0, [sp, #24]
+; CHECK-NEXT:    str r0, [sp, #4]
 ; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    str r0, [sp, #20]
-; CHECK-NEXT:    add r0, sp, #20
-; CHECK-NEXT:    add r12, r1, #16
-; CHECK-NEXT:    ldr r2, [r0], #4
-; CHECK-NEXT:    str r2, [r1], #4
-; CHECK-NEXT:    ldr r2, [r0], #4
-; CHECK-NEXT:    str r2, [r1], #4
-; CHECK-NEXT:    ldr r2, [r0], #4
-; CHECK-NEXT:    str r2, [r1], #4
-; CHECK-NEXT:    ldr r2, [r0], #4
-; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    str r0, [sp]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    add r0, r0, #16
+; CHECK-NEXT:    mov r3, #3
 ; CHECK-NEXT:    ldr r2, [r0], #4
 ; CHECK-NEXT:    str r2, [r1], #4
-; CHECK-NEXT:    ldm sp, {r0, r1, r2, r3}
-; CHECK-NEXT:    ldr r4, [r12], #4
-; CHECK-NEXT:    str r4, [lr], #4
-; CHECK-NEXT:    add sp, sp, #40
-; CHECK-NEXT:    pop {r4, lr}
-; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    mov r0, #0
+; CHECK-NEXT:    mov r1, #1
+; CHECK-NEXT:    mov r2, #2
+; CHECK-NEXT:    add sp, sp, #36
 ; CHECK-NEXT:    b large_callee
 entry:
   %y = alloca %twenty_bytes, align 4
@@ -335,3 +272,71 @@ entry:
   musttail call void @two_byvals_callee(%twenty_bytes* byval(%twenty_bytes) align 4 %b, %twenty_bytes* byval(%twenty_bytes) align 4 %a)
   ret void
 }
+
+; A forwarded byval arg, but at a different offset on the stack, so it needs to
+; be copied to the local stack frame first. This can't be musttail because of
+; the different signatures, but is still tail-called as an optimisation.
+declare void @shift_byval_callee(%twenty_bytes* byval(%twenty_bytes) align 4)
+define void @shift_byval(i32 %a, %twenty_bytes* byval(%twenty_bytes) align 4 %b) {
+; CHECK-LABEL: shift_byval:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #12
+; CHECK-NEXT:    sub sp, sp, #12
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .pad #20
+; CHECK-NEXT:    sub sp, sp, #20
+; CHECK-NEXT:    add r0, sp, #28
+; CHECK-NEXT:    add lr, sp, #40
+; CHECK-NEXT:    stm r0, {r1, r2, r3}
+; CHECK-NEXT:    add r0, sp, #28
+; CHECK-NEXT:    mov r1, sp
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    add r12, r1, #16
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldr r2, [r0], #4
+; CHECK-NEXT:    str r2, [r1], #4
+; CHECK-NEXT:    ldm sp, {r0, r1, r2, r3}
+; CHECK-NEXT:    ldr r4, [r12], #4
+; CHECK-NEXT:    str r4, [lr], #4
+; CHECK-NEXT:    add sp, sp, #20
+; CHECK-NEXT:    pop {r4, lr}
+; CHECK-NEXT:    add sp, sp, #12
+; CHECK-NEXT:    b shift_byval_callee
+entry:
+  tail call void @shift_byval_callee(%twenty_bytes* byval(%twenty_bytes) align 4 %b)
+  ret void
+}
+
+; A global object passed to a byval argument, so it must be copied, but doesn't
+; need a stack temporary.
+@large_global = external global %twenty_bytes
+define void @large_caller_from_global(%twenty_bytes* byval(%twenty_bytes) align 4 %a) {
+; CHECK-LABEL: large_caller_from_global:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    add r12, sp, #8
+; CHECK-NEXT:    add lr, sp, #24
+; CHECK-NEXT:    stm r12, {r0, r1, r2, r3}
+; CHECK-NEXT:    movw r3, :lower16:large_global
+; CHECK-NEXT:    movt r3, :upper16:large_global
+; CHECK-NEXT:    add r12, r3, #16
+; CHECK-NEXT:    ldm r3, {r0, r1, r2, r3}
+; CHECK-NEXT:    ldr r4, [r12], #4
+; CHECK-NEXT:    str r4, [lr], #4
+; CHECK-NEXT:    pop {r4, lr}
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    b large_callee
+entry:
+  musttail call void @large_callee(%twenty_bytes* byval(%twenty_bytes) align 4 @large_global)
+  ret void
+}
-- 
GitLab


From 8be860ddc4d5c21829085f5eed9188d59cbd3cc7 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray@arm.com>
Date: Fri, 25 Oct 2024 09:51:47 +0100
Subject: [PATCH 143/693] [AArch64] Add support for Armv9.6-A FEAT_PoPS
 architecture extension (#113496)

Add support for the following Armv9.6-A architecture extensions:
  * FEAT_PoPS      - Point of Physical Storage

as documented here:

https://developer.arm.com/documentation/109697/2024_09/Feature-descriptions/The-Armv9-6-architecture-extension

Co-authored-by: Alfie Richards <alfie.richards@arm.com>
---
 clang/test/Driver/aarch64-v96a.c                     |  4 ++++
 .../test/Driver/print-supported-extensions-aarch64.c |  1 +
 llvm/lib/Target/AArch64/AArch64Features.td           |  3 +++
 llvm/lib/Target/AArch64/AArch64SystemOperands.td     |  7 +++++++
 .../Target/AArch64/AsmParser/AArch64AsmParser.cpp    |  1 +
 llvm/test/MC/AArch64/armv9.6a-ras.s                  | 12 ++++++++++++
 llvm/test/MC/Disassembler/AArch64/armv9.6a-ras.txt   | 11 +++++++++++
 llvm/unittests/TargetParser/TargetParserTest.cpp     |  4 +++-
 8 files changed, 42 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/MC/AArch64/armv9.6a-ras.s
 create mode 100644 llvm/test/MC/Disassembler/AArch64/armv9.6a-ras.txt

diff --git a/clang/test/Driver/aarch64-v96a.c b/clang/test/Driver/aarch64-v96a.c
index 343e347c928c..de7890140ebd 100644
--- a/clang/test/Driver/aarch64-v96a.c
+++ b/clang/test/Driver/aarch64-v96a.c
@@ -65,4 +65,8 @@
 // RUN: %clang -target aarch64 -march=armv9.6a+pcdphint -### -c %s 2>&1 | FileCheck -check-prefix=V96A-PCDPHINT %s
 // RUN: %clang -target aarch64 -march=armv9.6-a+pcdphint -### -c %s 2>&1 | FileCheck -check-prefix=V96A-PCDPHINT %s
 // V96A-PCDPHINT: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "generic" "-target-feature" "+v9.6a"{{.*}} "-target-feature" "+pcdphint"
+
+// RUN: %clang -target aarch64 -march=armv9.6a+pops -### -c %s 2>&1 | FileCheck -check-prefix=V96A-POPS %s
+// RUN: %clang -target aarch64 -march=armv9.6-a+pops -### -c %s 2>&1 | FileCheck -check-prefix=V96A-POPS %s
+// V96A-POPS: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "generic" "-target-feature" "+v9.6a"{{.*}} "-target-feature" "+pops"
 //
diff --git a/clang/test/Driver/print-supported-extensions-aarch64.c b/clang/test/Driver/print-supported-extensions-aarch64.c
index 7ff4f17beff7..03eacf99736f 100644
--- a/clang/test/Driver/print-supported-extensions-aarch64.c
+++ b/clang/test/Driver/print-supported-extensions-aarch64.c
@@ -50,6 +50,7 @@
 // CHECK-NEXT:     pauth-lr            FEAT_PAuth_LR                                          Enable Armv9.5-A PAC enhancements
 // CHECK-NEXT:     pcdphint            FEAT_PCDPHINT                                          Enable Armv9.6-A Producer Consumer Data Placement hints
 // CHECK-NEXT:     pmuv3               FEAT_PMUv3                                             Enable Armv8.0-A PMUv3 Performance Monitors extension
+// CHECK-NEXT:     pops                FEAT_PoPS                                              Enable Armv9.6-A Point Of Physical Storage (PoPS) DC instructions
 // CHECK-NEXT:     predres             FEAT_SPECRES                                           Enable Armv8.5-A execution and data prediction invalidation instructions
 // CHECK-NEXT:     rng                 FEAT_RNG                                               Enable Random Number generation instructions
 // CHECK-NEXT:     ras                 FEAT_RAS, FEAT_RASv1p1                                 Enable Armv8.0-A Reliability, Availability and Serviceability Extensions
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index afa96821984b..e481da74ba2d 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -565,6 +565,9 @@ def FeatureOCCMO: ExtensionWithMArch<"occmo", "OCCMO", "FEAT_OCCMO",
 def FeaturePCDPHINT: ExtensionWithMArch<"pcdphint", "PCDPHINT", "FEAT_PCDPHINT",
   "Enable Armv9.6-A Producer Consumer Data Placement hints">;
 
+def FeaturePoPS: ExtensionWithMArch<"pops", "PoPS", "FEAT_PoPS",
+  "Enable Armv9.6-A Point Of Physical Storage (PoPS) DC instructions">;
+
 //===----------------------------------------------------------------------===//
 //  Other Features
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/AArch64SystemOperands.td b/llvm/lib/Target/AArch64/AArch64SystemOperands.td
index 052cdf9e9392..f22e0242321e 100644
--- a/llvm/lib/Target/AArch64/AArch64SystemOperands.td
+++ b/llvm/lib/Target/AArch64/AArch64SystemOperands.td
@@ -2061,3 +2061,10 @@ def : RWSysReg<"PMBSR_EL3",       0b11, 0b110, 0b1001, 0b1010, 0b011>;
 def : RWSysReg<"TRBSR_EL12",      0b11, 0b101, 0b1001, 0b1011, 0b011>;
 def : RWSysReg<"TRBSR_EL2",       0b11, 0b100, 0b1001, 0b1011, 0b011>;
 def : RWSysReg<"TRBSR_EL3",       0b11, 0b110, 0b1001, 0b1011, 0b011>;
+
+// v9.6 FEAT_PoPS
+//
+let Requires = [{ {AArch64::FeaturePoPS} }] in {
+def : DC<"CIGDVAPS",  0b000, 0b0111, 0b1111, 0b101>;
+def : DC<"CIVAPS",    0b000, 0b0111, 0b1111, 0b001>;
+}
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index 3ab4ebbf6ebf..dfc5e04110cf 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -3794,6 +3794,7 @@ static const struct Extension {
     {"sme-fa64", {AArch64::FeatureSMEFA64}},
     {"cpa", {AArch64::FeatureCPA}},
     {"tlbiw", {AArch64::FeatureTLBIW}},
+    {"pops", {AArch64::FeaturePoPS}},
     {"cmpbr", {AArch64::FeatureCMPBR}},
     {"f8f32mm", {AArch64::FeatureF8F32MM}},
     {"f8f16mm", {AArch64::FeatureF8F16MM}},
diff --git a/llvm/test/MC/AArch64/armv9.6a-ras.s b/llvm/test/MC/AArch64/armv9.6a-ras.s
new file mode 100644
index 000000000000..2a188d7613d5
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv9.6a-ras.s
@@ -0,0 +1,12 @@
+// RUN: not llvm-mc -triple aarch64 -show-encoding < %s 2>&1 | FileCheck %s --check-prefix NO-POPS
+// RUN: llvm-mc -triple aarch64 -mattr=+pops -show-encoding < %s 2>&1 | FileCheck %s --check-prefix HAS-POPS
+
+dc CIGDVAPS, x3
+dc CIVAPS, x3
+// NO-POPS: error: DC CIGDVAPS requires: pops
+// NO-POPS: error: DC CIVAPS requires: pops
+
+# HAS-POPS:      	.text
+
+# HAS-POPS:      	dc	cigdvaps, x3                    // encoding: [0xa3,0x7f,0x08,0xd5]
+# HAS-POPS-NEXT: 	dc	civaps, x3                      // encoding: [0x23,0x7f,0x08,0xd5]
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.6a-ras.txt b/llvm/test/MC/Disassembler/AArch64/armv9.6a-ras.txt
new file mode 100644
index 000000000000..d6bf65765d3b
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AArch64/armv9.6a-ras.txt
@@ -0,0 +1,11 @@
+# RUN: llvm-mc -triple aarch64 -mattr=+pops -disassemble < %s | FileCheck %s
+
+#------------------------------------------------------------------------------
+# ARMV9.6-A RAS Extensions
+#------------------------------------------------------------------------------
+
+[0xa3,0x7f,0x08,0xd5]
+# CHECK: dc cigdvaps, x3
+
+[0x23,0x7f,0x08,0xd5]
+# CHECK: dc civaps, x3
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index 51ea9cb6d644..d69b2d6b13b1 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -1333,7 +1333,7 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
       AArch64::AEK_F8F16MM,      AArch64::AEK_LSFE,
       AArch64::AEK_FPRCVT,       AArch64::AEK_CMPBR,
       AArch64::AEK_LSUI,         AArch64::AEK_OCCMO,
-      AArch64::AEK_PCDPHINT,
+      AArch64::AEK_PCDPHINT,     AArch64::AEK_POPS,
   };
 
   std::vector<StringRef> Features;
@@ -1437,6 +1437,7 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
   EXPECT_TRUE(llvm::is_contained(Features, "+lsui"));
   EXPECT_TRUE(llvm::is_contained(Features, "+occmo"));
   EXPECT_TRUE(llvm::is_contained(Features, "+pcdphint"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+pops"));
 
   // Assuming we listed every extension above, this should produce the same
   // result.
@@ -1592,6 +1593,7 @@ TEST(TargetParserTest, AArch64ArchExtFeature) {
       {"lsui", "nolsui", "+lsui", "-lsui"},
       {"occmo", "nooccmo", "+occmo", "-occmo"},
       {"pcdphint", "nopcdphint", "+pcdphint", "-pcdphint"},
+      {"pops", "nopops", "+pops", "-pops"},
   };
 
   for (unsigned i = 0; i < std::size(ArchExt); i++) {
-- 
GitLab


From 2e43a304f10fd801f068d0f9831f01f2c5b0b2e2 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Fri, 25 Oct 2024 11:28:46 +0200
Subject: [PATCH 144/693] [libc++] Granularize <vector> (#99705)

|                    | old time | new time |
| ------------------ | -------- | -------- |
| functional - c++23 | 416ms    | 225ms    |
| random - c++23     | 513ms    | 392ms    |
| vector - c++17     | 206ms    | 100ms    |
---
 libcxx/include/CMakeLists.txt                 |    8 +
 libcxx/include/__chrono/tzdb.h                |    2 +-
 .../__functional/boyer_moore_searcher.h       |    3 +-
 libcxx/include/__fwd/vector.h                 |    3 +
 .../include/__random/discrete_distribution.h  |    3 +-
 .../piecewise_constant_distribution.h         |    3 +-
 .../__random/piecewise_linear_distribution.h  |    4 +-
 libcxx/include/__random/seed_seq.h            |    2 +-
 libcxx/include/__vector/comparison.h          |   71 +
 libcxx/include/__vector/container_traits.h    |   39 +
 libcxx/include/__vector/erase.h               |   50 +
 libcxx/include/__vector/pmr.h                 |   33 +
 libcxx/include/__vector/swap.h                |   29 +
 libcxx/include/__vector/vector.h              | 1495 +++++++++
 libcxx/include/__vector/vector_bool.h         | 1123 +++++++
 .../include/__vector/vector_bool_formatter.h  |   49 +
 libcxx/include/chrono                         |    1 +
 libcxx/include/functional                     |    2 +-
 libcxx/include/module.modulemap               |   35 +-
 libcxx/include/vector                         | 2685 +----------------
 .../vector.bool/abi.compile.pass.cpp          |    2 +-
 .../libcxx/memory/swap_allocator.pass.cpp     |    3 +-
 .../test/libcxx/transitive_includes/cxx23.csv |   16 -
 .../test/libcxx/transitive_includes/cxx26.csv |   16 -
 .../std/atomics/atomics.ref/assign.pass.cpp   |    1 +
 .../thread.lock.shared.cons/mutex.pass.cpp    |    1 +
 .../time.zone.leap/cons.copy.pass.cpp         |    3 +-
 .../time.zone.leap/members/value.pass.cpp     |    1 +
 .../time.zone.link.members/name.pass.cpp      |    1 +
 .../time.zone.link.members/target.pass.cpp    |    1 +
 .../time.zone.members/name.pass.cpp           |    1 +
 .../const_time_zone_default_zone.pass.cpp     |    2 +-
 .../dereference.single.pass.cpp               |    3 +-
 33 files changed, 2988 insertions(+), 2703 deletions(-)
 create mode 100644 libcxx/include/__vector/comparison.h
 create mode 100644 libcxx/include/__vector/container_traits.h
 create mode 100644 libcxx/include/__vector/erase.h
 create mode 100644 libcxx/include/__vector/pmr.h
 create mode 100644 libcxx/include/__vector/swap.h
 create mode 100644 libcxx/include/__vector/vector.h
 create mode 100644 libcxx/include/__vector/vector_bool.h
 create mode 100644 libcxx/include/__vector/vector_bool_formatter.h

diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 86d2fc2c2c67..975adc03ec81 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -882,6 +882,14 @@ set(files
   __utility/to_underlying.h
   __utility/unreachable.h
   __variant/monostate.h
+  __vector/comparison.h
+  __vector/container_traits.h
+  __vector/erase.h
+  __vector/pmr.h
+  __vector/swap.h
+  __vector/vector.h
+  __vector/vector_bool.h
+  __vector/vector_bool_formatter.h
   __verbose_abort
   algorithm
   any
diff --git a/libcxx/include/__chrono/tzdb.h b/libcxx/include/__chrono/tzdb.h
index dce97c3adfa1..5bbb82f44241 100644
--- a/libcxx/include/__chrono/tzdb.h
+++ b/libcxx/include/__chrono/tzdb.h
@@ -22,10 +22,10 @@
 #  include <__chrono/time_zone_link.h>
 #  include <__config>
 #  include <__memory/addressof.h>
+#  include <__vector/vector.h>
 #  include <stdexcept>
 #  include <string>
 #  include <string_view>
-#  include <vector>
 
 #  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #    pragma GCC system_header
diff --git a/libcxx/include/__functional/boyer_moore_searcher.h b/libcxx/include/__functional/boyer_moore_searcher.h
index 648b60c50521..52a58d57a8d4 100644
--- a/libcxx/include/__functional/boyer_moore_searcher.h
+++ b/libcxx/include/__functional/boyer_moore_searcher.h
@@ -22,9 +22,10 @@
 #include <__memory/shared_ptr.h>
 #include <__type_traits/make_unsigned.h>
 #include <__utility/pair.h>
+#include <__vector/vector.h>
 #include <array>
+#include <limits>
 #include <unordered_map>
-#include <vector>
 
 #if _LIBCPP_STD_VER >= 17
 
diff --git a/libcxx/include/__fwd/vector.h b/libcxx/include/__fwd/vector.h
index c9cc96137449..6980e40ec918 100644
--- a/libcxx/include/__fwd/vector.h
+++ b/libcxx/include/__fwd/vector.h
@@ -21,6 +21,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 template <class _Tp, class _Alloc = allocator<_Tp> >
 class _LIBCPP_TEMPLATE_VIS vector;
 
+template <class _Allocator>
+class vector<bool, _Allocator>;
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___FWD_VECTOR_H
diff --git a/libcxx/include/__random/discrete_distribution.h b/libcxx/include/__random/discrete_distribution.h
index bb72dd6cb507..931f7704ff97 100644
--- a/libcxx/include/__random/discrete_distribution.h
+++ b/libcxx/include/__random/discrete_distribution.h
@@ -13,10 +13,11 @@
 #include <__config>
 #include <__random/is_valid.h>
 #include <__random/uniform_real_distribution.h>
+#include <__vector/vector.h>
 #include <cstddef>
+#include <initializer_list>
 #include <iosfwd>
 #include <numeric>
-#include <vector>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__random/piecewise_constant_distribution.h b/libcxx/include/__random/piecewise_constant_distribution.h
index e19380f97c35..a864f848143b 100644
--- a/libcxx/include/__random/piecewise_constant_distribution.h
+++ b/libcxx/include/__random/piecewise_constant_distribution.h
@@ -13,9 +13,10 @@
 #include <__config>
 #include <__random/is_valid.h>
 #include <__random/uniform_real_distribution.h>
+#include <__vector/vector.h>
+#include <initializer_list>
 #include <iosfwd>
 #include <numeric>
-#include <vector>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__random/piecewise_linear_distribution.h b/libcxx/include/__random/piecewise_linear_distribution.h
index 43769dc825e6..24aa6cce91cf 100644
--- a/libcxx/include/__random/piecewise_linear_distribution.h
+++ b/libcxx/include/__random/piecewise_linear_distribution.h
@@ -13,9 +13,11 @@
 #include <__config>
 #include <__random/is_valid.h>
 #include <__random/uniform_real_distribution.h>
+#include <__vector/comparison.h>
+#include <__vector/vector.h>
 #include <cmath>
+#include <initializer_list>
 #include <iosfwd>
-#include <vector>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__random/seed_seq.h b/libcxx/include/__random/seed_seq.h
index c1a320a75c88..3f60069a916d 100644
--- a/libcxx/include/__random/seed_seq.h
+++ b/libcxx/include/__random/seed_seq.h
@@ -17,9 +17,9 @@
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_integral.h>
 #include <__type_traits/is_unsigned.h>
+#include <__vector/vector.h>
 #include <cstdint>
 #include <initializer_list>
-#include <vector>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__vector/comparison.h b/libcxx/include/__vector/comparison.h
new file mode 100644
index 000000000000..27178e41ec51
--- /dev/null
+++ b/libcxx/include/__vector/comparison.h
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___VECTOR_COMPARISON_H
+#define _LIBCPP___VECTOR_COMPARISON_H
+
+#include <__algorithm/equal.h>
+#include <__algorithm/lexicographical_compare.h>
+#include <__algorithm/lexicographical_compare_three_way.h>
+#include <__compare/synth_three_way.h>
+#include <__config>
+#include <__fwd/vector.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
+  const typename vector<_Tp, _Allocator>::size_type __sz = __x.size();
+  return __sz == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
+}
+
+#if _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
+  return !(__x == __y);
+}
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
+  return std::lexicographical_compare(__x.begin(), __x.end(), __y.begin(), __y.end());
+}
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
+  return __y < __x;
+}
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator>=(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
+  return !(__x < __y);
+}
+
+template <class _Tp, class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
+  return !(__y < __x);
+}
+
+#else // _LIBCPP_STD_VER <= 17
+
+template <class _Tp, class _Allocator>
+_LIBCPP_HIDE_FROM_ABI constexpr __synth_three_way_result<_Tp>
+operator<=>(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
+  return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
+}
+
+#endif // _LIBCPP_STD_VER <= 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___VECTOR_COMPARISON_H
diff --git a/libcxx/include/__vector/container_traits.h b/libcxx/include/__vector/container_traits.h
new file mode 100644
index 000000000000..7b1497a404f2
--- /dev/null
+++ b/libcxx/include/__vector/container_traits.h
@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___VECTOR_CONTAINER_TRAITS_H
+#define _LIBCPP___VECTOR_CONTAINER_TRAITS_H
+
+#include <__config>
+#include <__fwd/vector.h>
+#include <__memory/allocator_traits.h>
+#include <__type_traits/container_traits.h>
+#include <__type_traits/disjunction.h>
+#include <__type_traits/is_nothrow_constructible.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Allocator>
+struct __container_traits<vector<_Tp, _Allocator> > {
+  // http://eel.is/c++draft/vector.modifiers#2
+  //  If an exception is thrown other than by the copy constructor, move constructor, assignment operator, or move
+  //  assignment operator of T or by any InputIterator operation, there are no effects. If an exception is thrown while
+  //  inserting a single element at the end and T is Cpp17CopyInsertable or is_nothrow_move_constructible_v<T> is true,
+  //  there are no effects. Otherwise, if an exception is thrown by the move constructor of a non-Cpp17CopyInsertable T,
+  //  the effects are unspecified.
+  static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee =
+      _Or<is_nothrow_move_constructible<_Tp>, __is_cpp17_copy_insertable<_Allocator> >::value;
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___VECTOR_CONTAINER_TRAITS_H
diff --git a/libcxx/include/__vector/erase.h b/libcxx/include/__vector/erase.h
new file mode 100644
index 000000000000..d7379fc1d04e
--- /dev/null
+++ b/libcxx/include/__vector/erase.h
@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___VECTOR_ERASE_H
+#define _LIBCPP___VECTOR_ERASE_H
+
+#include <__algorithm/remove.h>
+#include <__algorithm/remove_if.h>
+#include <__config>
+#include <__fwd/vector.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Allocator, class _Up>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::size_type
+erase(vector<_Tp, _Allocator>& __c, const _Up& __v) {
+  auto __old_size = __c.size();
+  __c.erase(std::remove(__c.begin(), __c.end(), __v), __c.end());
+  return __old_size - __c.size();
+}
+
+template <class _Tp, class _Allocator, class _Predicate>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::size_type
+erase_if(vector<_Tp, _Allocator>& __c, _Predicate __pred) {
+  auto __old_size = __c.size();
+  __c.erase(std::remove_if(__c.begin(), __c.end(), __pred), __c.end());
+  return __old_size - __c.size();
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___VECTOR_ERASE_H
diff --git a/libcxx/include/__vector/pmr.h b/libcxx/include/__vector/pmr.h
new file mode 100644
index 000000000000..8ef603c739a1
--- /dev/null
+++ b/libcxx/include/__vector/pmr.h
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___VECTOR_PMR_H
+#define _LIBCPP___VECTOR_PMR_H
+
+#include <__config>
+#include <__fwd/vector.h>
+#include <__memory_resource/polymorphic_allocator.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace pmr {
+template <class _ValueT>
+using vector _LIBCPP_AVAILABILITY_PMR = std::vector<_ValueT, polymorphic_allocator<_ValueT>>;
+} // namespace pmr
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif
+
+#endif // _LIBCPP___VECTOR_PMR_H
diff --git a/libcxx/include/__vector/swap.h b/libcxx/include/__vector/swap.h
new file mode 100644
index 000000000000..7fe27a91419b
--- /dev/null
+++ b/libcxx/include/__vector/swap.h
@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___VECTOR_SWAP_H
+#define _LIBCPP___VECTOR_SWAP_H
+
+#include <__config>
+#include <__fwd/vector.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void
+swap(vector<_Tp, _Allocator>& __x, vector<_Tp, _Allocator>& __y) _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+  __x.swap(__y);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___VECTOR_SWAP_H
diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h
new file mode 100644
index 000000000000..7889e8c2201a
--- /dev/null
+++ b/libcxx/include/__vector/vector.h
@@ -0,0 +1,1495 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___VECTOR_VECTOR_H
+#define _LIBCPP___VECTOR_VECTOR_H
+
+#include <__algorithm/copy.h>
+#include <__algorithm/fill_n.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/max.h>
+#include <__algorithm/min.h>
+#include <__algorithm/move.h>
+#include <__algorithm/move_backward.h>
+#include <__algorithm/rotate.h>
+#include <__assert>
+#include <__config>
+#include <__debug_utils/sanitizers.h>
+#include <__format/enable_insertable.h>
+#include <__fwd/vector.h>
+#include <__iterator/bounded_iter.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/move_iterator.h>
+#include <__iterator/next.h>
+#include <__iterator/reverse_iterator.h>
+#include <__iterator/wrap_iter.h>
+#include <__memory/addressof.h>
+#include <__memory/allocate_at_least.h>
+#include <__memory/allocator.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/compressed_pair.h>
+#include <__memory/noexcept_move_assign_container.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/swap_allocator.h>
+#include <__memory/temp_value.h>
+#include <__memory/uninitialized_algorithms.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/container_compatible_range.h>
+#include <__ranges/from_range.h>
+#include <__split_buffer>
+#include <__type_traits/conditional.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_allocator.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__type_traits/type_identity.h>
+#include <__utility/exception_guard.h>
+#include <__utility/forward.h>
+#include <__utility/is_pointer_in_range.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+#include <__utility/swap.h>
+#include <initializer_list>
+#include <limits>
+#include <stdexcept>
+
+// These headers define parts of vectors definition, since they define ADL functions or class specializations.
+#include <__vector/comparison.h>
+#include <__vector/container_traits.h>
+#include <__vector/swap.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _Allocator /* = allocator<_Tp> */>
+class _LIBCPP_TEMPLATE_VIS vector {
+private:
+  typedef allocator<_Tp> __default_allocator_type;
+
+public:
+  typedef vector __self;
+  typedef _Tp value_type;
+  typedef _Allocator allocator_type;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+  typedef value_type& reference;
+  typedef const value_type& const_reference;
+  typedef typename __alloc_traits::size_type size_type;
+  typedef typename __alloc_traits::difference_type difference_type;
+  typedef typename __alloc_traits::pointer pointer;
+  typedef typename __alloc_traits::const_pointer const_pointer;
+#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
+  // Users might provide custom allocators, and prior to C++20 we have no existing way to detect whether the allocator's
+  // pointer type is contiguous (though it has to be by the Standard). Using the wrapper type ensures the iterator is
+  // considered contiguous.
+  typedef __bounded_iter<__wrap_iter<pointer> > iterator;
+  typedef __bounded_iter<__wrap_iter<const_pointer> > const_iterator;
+#else
+  typedef __wrap_iter<pointer> iterator;
+  typedef __wrap_iter<const_pointer> const_iterator;
+#endif
+  typedef std::reverse_iterator<iterator> reverse_iterator;
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+
+  // A vector containers the following members which may be trivially relocatable:
+  // - pointer: may be trivially relocatable, so it's checked
+  // - allocator_type: may be trivially relocatable, so it's checked
+  // vector doesn't contain any self-references, so it's trivially relocatable if its members are.
+  using __trivially_relocatable = __conditional_t<
+      __libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<allocator_type>::value,
+      vector,
+      void>;
+
+  static_assert(__check_valid_allocator<allocator_type>::value, "");
+  static_assert(is_same<typename allocator_type::value_type, value_type>::value,
+                "Allocator::value_type must be same type as value_type");
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector()
+      _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit vector(const allocator_type& __a)
+#if _LIBCPP_STD_VER <= 14
+      _NOEXCEPT_(is_nothrow_copy_constructible<allocator_type>::value)
+#else
+      noexcept
+#endif
+      : __alloc_(__a) {
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit vector(size_type __n) {
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+    if (__n > 0) {
+      __vallocate(__n);
+      __construct_at_end(__n);
+    }
+    __guard.__complete();
+  }
+
+#if _LIBCPP_STD_VER >= 14
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit vector(size_type __n, const allocator_type& __a)
+      : __alloc_(__a) {
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+    if (__n > 0) {
+      __vallocate(__n);
+      __construct_at_end(__n);
+    }
+    __guard.__complete();
+  }
+#endif
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(size_type __n, const value_type& __x) {
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+    if (__n > 0) {
+      __vallocate(__n);
+      __construct_at_end(__n, __x);
+    }
+    __guard.__complete();
+  }
+
+  template <__enable_if_t<__is_allocator<_Allocator>::value, int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+  vector(size_type __n, const value_type& __x, const allocator_type& __a)
+      : __alloc_(__a) {
+    if (__n > 0) {
+      __vallocate(__n);
+      __construct_at_end(__n, __x);
+    }
+  }
+
+  template <class _InputIterator,
+            __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                              is_constructible<value_type, typename iterator_traits<_InputIterator>::reference>::value,
+                          int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(_InputIterator __first, _InputIterator __last);
+  template <class _InputIterator,
+            __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                              is_constructible<value_type, typename iterator_traits<_InputIterator>::reference>::value,
+                          int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+  vector(_InputIterator __first, _InputIterator __last, const allocator_type& __a);
+
+  template <
+      class _ForwardIterator,
+      __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                        is_constructible<value_type, typename iterator_traits<_ForwardIterator>::reference>::value,
+                    int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(_ForwardIterator __first, _ForwardIterator __last);
+
+  template <
+      class _ForwardIterator,
+      __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                        is_constructible<value_type, typename iterator_traits<_ForwardIterator>::reference>::value,
+                    int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+  vector(_ForwardIterator __first, _ForwardIterator __last, const allocator_type& __a);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr vector(
+      from_range_t, _Range&& __range, const allocator_type& __alloc = allocator_type())
+      : __alloc_(__alloc) {
+    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      __init_with_size(ranges::begin(__range), ranges::end(__range), __n);
+
+    } else {
+      __init_with_sentinel(ranges::begin(__range), ranges::end(__range));
+    }
+  }
+#endif
+
+private:
+  class __destroy_vector {
+  public:
+    _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI __destroy_vector(vector& __vec) : __vec_(__vec) {}
+
+    _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void operator()() {
+      if (__vec_.__begin_ != nullptr) {
+        __vec_.__clear();
+        __vec_.__annotate_delete();
+        __alloc_traits::deallocate(__vec_.__alloc(), __vec_.__begin_, __vec_.capacity());
+      }
+    }
+
+  private:
+    vector& __vec_;
+  };
+
+public:
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~vector() { __destroy_vector (*this)(); }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(const vector& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+  vector(const vector& __x, const __type_identity_t<allocator_type>& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector& operator=(const vector& __x);
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(initializer_list<value_type> __il);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+  vector(initializer_list<value_type> __il, const allocator_type& __a);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector& operator=(initializer_list<value_type> __il) {
+    assign(__il.begin(), __il.end());
+    return *this;
+  }
+#endif // !_LIBCPP_CXX03_LANG
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(vector&& __x)
+#if _LIBCPP_STD_VER >= 17
+      noexcept;
+#else
+      _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value);
+#endif
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+  vector(vector&& __x, const __type_identity_t<allocator_type>& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector& operator=(vector&& __x)
+      _NOEXCEPT_(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value);
+
+  template <class _InputIterator,
+            __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                              is_constructible<value_type, typename iterator_traits<_InputIterator>::reference>::value,
+                          int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void assign(_InputIterator __first, _InputIterator __last);
+  template <
+      class _ForwardIterator,
+      __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                        is_constructible<value_type, typename iterator_traits<_ForwardIterator>::reference>::value,
+                    int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void assign(_ForwardIterator __first, _ForwardIterator __last);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr void assign_range(_Range&& __range) {
+    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      __assign_with_size(ranges::begin(__range), ranges::end(__range), __n);
+
+    } else {
+      __assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
+    }
+  }
+#endif
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void assign(size_type __n, const_reference __u);
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void assign(initializer_list<value_type> __il) {
+    assign(__il.begin(), __il.end());
+  }
+#endif
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
+    return this->__alloc();
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT {
+    return reverse_iterator(end());
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT {
+    return const_reverse_iterator(end());
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT {
+    return reverse_iterator(begin());
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
+    return const_reverse_iterator(begin());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT {
+    return rbegin();
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT {
+    return static_cast<size_type>(this->__end_ - this->__begin_);
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const _NOEXCEPT {
+    return static_cast<size_type>(__end_cap() - this->__begin_);
+  }
+  [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT {
+    return this->__begin_ == this->__end_;
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void reserve(size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void shrink_to_fit() _NOEXCEPT;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference operator[](size_type __n) _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference operator[](size_type __n) const _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference at(size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference at(size_type __n) const;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference front() _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "front() called on an empty vector");
+    return *this->__begin_;
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference front() const _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "front() called on an empty vector");
+    return *this->__begin_;
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "back() called on an empty vector");
+    return *(this->__end_ - 1);
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "back() called on an empty vector");
+    return *(this->__end_ - 1);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI value_type* data() _NOEXCEPT {
+    return std::__to_address(this->__begin_);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const value_type* data() const _NOEXCEPT {
+    return std::__to_address(this->__begin_);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_back(const_reference __x) { emplace_back(__x); }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __x) { emplace_back(std::move(__x)); }
+
+  template <class... _Args>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+#if _LIBCPP_STD_VER >= 17
+  reference
+  emplace_back(_Args&&... __args);
+#else
+  void
+  emplace_back(_Args&&... __args);
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr void append_range(_Range&& __range) {
+    insert_range(end(), std::forward<_Range>(__range));
+  }
+#endif
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_back();
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, const_reference __x);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, value_type&& __x);
+  template <class... _Args>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __position, _Args&&... __args);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  insert(const_iterator __position, size_type __n, const_reference __x);
+
+  template <class _InputIterator,
+            __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                              is_constructible< value_type, typename iterator_traits<_InputIterator>::reference>::value,
+                          int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  insert(const_iterator __position, _InputIterator __first, _InputIterator __last);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<_Tp> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator insert_range(const_iterator __position, _Range&& __range) {
+    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      return __insert_with_size(__position, ranges::begin(__range), ranges::end(__range), __n);
+
+    } else {
+      return __insert_with_sentinel(__position, ranges::begin(__range), ranges::end(__range));
+    }
+  }
+#endif
+
+  template <
+      class _ForwardIterator,
+      __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                        is_constructible< value_type, typename iterator_traits<_ForwardIterator>::reference>::value,
+                    int> = 0>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  insert(const_iterator __position, _ForwardIterator __first, _ForwardIterator __last);
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  insert(const_iterator __position, initializer_list<value_type> __il) {
+    return insert(__position, __il.begin(), __il.end());
+  }
+#endif
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __position);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT {
+    size_type __old_size = size();
+    __clear();
+    __annotate_shrink(__old_size);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void resize(size_type __sz);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void resize(size_type __sz, const_reference __x);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void swap(vector&)
+#if _LIBCPP_STD_VER >= 14
+      _NOEXCEPT;
+#else
+      _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>);
+#endif
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __invariants() const;
+
+private:
+  pointer __begin_ = nullptr;
+  pointer __end_   = nullptr;
+  _LIBCPP_COMPRESSED_PAIR(pointer, __cap_ = nullptr, allocator_type, __alloc_);
+
+  //  Allocate space for __n objects
+  //  throws length_error if __n > max_size()
+  //  throws (probably bad_alloc) if memory run out
+  //  Precondition:  __begin_ == __end_ == __end_cap() == 0
+  //  Precondition:  __n > 0
+  //  Postcondition:  capacity() >= __n
+  //  Postcondition:  size() == 0
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __vallocate(size_type __n) {
+    if (__n > max_size())
+      __throw_length_error();
+    auto __allocation = std::__allocate_at_least(__alloc(), __n);
+    __begin_          = __allocation.ptr;
+    __end_            = __allocation.ptr;
+    __end_cap()       = __begin_ + __allocation.count;
+    __annotate_new(0);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __vdeallocate() _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __recommend(size_type __new_size) const;
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_at_end(size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_at_end(size_type __n, const_reference __x);
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __init_with_size(_InputIterator __first, _Sentinel __last, size_type __n) {
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+
+    if (__n > 0) {
+      __vallocate(__n);
+      __construct_at_end(__first, __last, __n);
+    }
+
+    __guard.__complete();
+  }
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __init_with_sentinel(_InputIterator __first, _Sentinel __last) {
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+
+    for (; __first != __last; ++__first)
+      emplace_back(*__first);
+
+    __guard.__complete();
+  }
+
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __first, _Sentinel __last);
+
+  template <class _ForwardIterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __assign_with_size(_ForwardIterator __first, _Sentinel __last, difference_type __n);
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  __insert_with_sentinel(const_iterator __position, _InputIterator __first, _Sentinel __last);
+
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  __insert_with_size(const_iterator __position, _Iterator __first, _Sentinel __last, difference_type __n);
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __append(size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __append(size_type __n, const_reference __x);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator __make_iter(pointer __p) _NOEXCEPT {
+#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
+    // Bound the iterator according to the capacity, rather than the size.
+    //
+    // Vector guarantees that iterators stay valid as long as no reallocation occurs even if new elements are inserted
+    // into the container; for these cases, we need to make sure that the newly-inserted elements can be accessed
+    // through the bounded iterator without failing checks. The downside is that the bounded iterator won't catch
+    // access that is logically out-of-bounds, i.e., goes beyond the size, but is still within the capacity. With the
+    // current implementation, there is no connection between a bounded iterator and its associated container, so we
+    // don't have a way to update existing valid iterators when the container is resized and thus have to go with
+    // a laxer approach.
+    return std::__make_bounded_iter(
+        std::__wrap_iter<pointer>(__p),
+        std::__wrap_iter<pointer>(this->__begin_),
+        std::__wrap_iter<pointer>(this->__end_cap()));
+#else
+    return iterator(__p);
+#endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator __make_iter(const_pointer __p) const _NOEXCEPT {
+#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
+    // Bound the iterator according to the capacity, rather than the size.
+    return std::__make_bounded_iter(
+        std::__wrap_iter<const_pointer>(__p),
+        std::__wrap_iter<const_pointer>(this->__begin_),
+        std::__wrap_iter<const_pointer>(this->__end_cap()));
+#else
+    return const_iterator(__p);
+#endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer
+  __swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v, pointer __p);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __move_range(pointer __from_s, pointer __from_e, pointer __to);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign(vector& __c, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign(vector& __c, false_type)
+      _NOEXCEPT_(__alloc_traits::is_always_equal::value);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __destruct_at_end(pointer __new_last) _NOEXCEPT {
+    size_type __old_size = size();
+    __base_destruct_at_end(__new_last);
+    __annotate_shrink(__old_size);
+  }
+
+  template <class... _Args>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI inline pointer __emplace_back_slow_path(_Args&&... __args);
+
+  // The following functions are no-ops outside of AddressSanitizer mode.
+  // We call annotations for every allocator, unless explicitly disabled.
+  //
+  // To disable annotations for a particular allocator, change value of
+  // __asan_annotate_container_with_allocator to false.
+  // For more details, see the "Using libc++" documentation page or
+  // the documentation for __sanitizer_annotate_contiguous_container.
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __annotate_contiguous_container(const void* __old_mid, const void* __new_mid) const {
+    std::__annotate_contiguous_container<_Allocator>(data(), data() + capacity(), __old_mid, __new_mid);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_new(size_type __current_size) const _NOEXCEPT {
+    (void)__current_size;
+#if _LIBCPP_HAS_ASAN
+    __annotate_contiguous_container(data() + capacity(), data() + __current_size);
+#endif
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_delete() const _NOEXCEPT {
+#if _LIBCPP_HAS_ASAN
+    __annotate_contiguous_container(data() + size(), data() + capacity());
+#endif
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_increase(size_type __n) const _NOEXCEPT {
+    (void)__n;
+#if _LIBCPP_HAS_ASAN
+    __annotate_contiguous_container(data() + size(), data() + size() + __n);
+#endif
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_shrink(size_type __old_size) const _NOEXCEPT {
+    (void)__old_size;
+#if _LIBCPP_HAS_ASAN
+    __annotate_contiguous_container(data() + __old_size, data() + size());
+#endif
+  }
+
+  struct _ConstructTransaction {
+    _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit _ConstructTransaction(vector& __v, size_type __n)
+        : __v_(__v), __pos_(__v.__end_), __new_end_(__v.__end_ + __n) {
+#if _LIBCPP_HAS_ASAN
+      __v_.__annotate_increase(__n);
+#endif
+    }
+
+    _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~_ConstructTransaction() {
+      __v_.__end_ = __pos_;
+#if _LIBCPP_HAS_ASAN
+      if (__pos_ != __new_end_) {
+        __v_.__annotate_shrink(__new_end_ - __v_.__begin_);
+      }
+#endif
+    }
+
+    vector& __v_;
+    pointer __pos_;
+    const_pointer const __new_end_;
+
+    _ConstructTransaction(_ConstructTransaction const&)            = delete;
+    _ConstructTransaction& operator=(_ConstructTransaction const&) = delete;
+  };
+
+  template <class... _Args>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_one_at_end(_Args&&... __args) {
+    _ConstructTransaction __tx(*this, 1);
+    __alloc_traits::construct(this->__alloc(), std::__to_address(__tx.__pos_), std::forward<_Args>(__args)...);
+    ++__tx.__pos_;
+  }
+
+  // TODO: Remove these now redundant accessors
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type& __alloc() _NOEXCEPT { return this->__alloc_; }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const allocator_type& __alloc() const _NOEXCEPT {
+    return this->__alloc_;
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer& __end_cap() _NOEXCEPT { return this->__cap_; }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const pointer& __end_cap() const _NOEXCEPT {
+    return this->__cap_;
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __clear() _NOEXCEPT {
+    __base_destruct_at_end(this->__begin_);
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __base_destruct_at_end(pointer __new_last) _NOEXCEPT {
+    pointer __soon_to_be_end = this->__end_;
+    while (__new_last != __soon_to_be_end)
+      __alloc_traits::destroy(__alloc(), std::__to_address(--__soon_to_be_end));
+    this->__end_ = __new_last;
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const vector& __c) {
+    __copy_assign_alloc(__c, integral_constant<bool, __alloc_traits::propagate_on_container_copy_assignment::value>());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(vector& __c)
+      _NOEXCEPT_(!__alloc_traits::propagate_on_container_move_assignment::value ||
+                 is_nothrow_move_assignable<allocator_type>::value) {
+    __move_assign_alloc(__c, integral_constant<bool, __alloc_traits::propagate_on_container_move_assignment::value>());
+  }
+
+  [[__noreturn__]] _LIBCPP_HIDE_FROM_ABI void __throw_length_error() const { std::__throw_length_error("vector"); }
+
+  [[__noreturn__]] _LIBCPP_HIDE_FROM_ABI void __throw_out_of_range() const { std::__throw_out_of_range("vector"); }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const vector& __c, true_type) {
+    if (__alloc() != __c.__alloc()) {
+      __clear();
+      __annotate_delete();
+      __alloc_traits::deallocate(__alloc(), this->__begin_, capacity());
+      this->__begin_ = this->__end_ = __end_cap() = nullptr;
+    }
+    __alloc() = __c.__alloc();
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const vector&, false_type) {}
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(vector& __c, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
+    __alloc() = std::move(__c.__alloc());
+  }
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(vector&, false_type) _NOEXCEPT {}
+};
+
+#if _LIBCPP_STD_VER >= 17
+template <class _InputIterator,
+          class _Alloc = allocator<__iter_value_type<_InputIterator>>,
+          class        = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class        = enable_if_t<__is_allocator<_Alloc>::value> >
+vector(_InputIterator, _InputIterator) -> vector<__iter_value_type<_InputIterator>, _Alloc>;
+
+template <class _InputIterator,
+          class _Alloc,
+          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
+          class = enable_if_t<__is_allocator<_Alloc>::value> >
+vector(_InputIterator, _InputIterator, _Alloc) -> vector<__iter_value_type<_InputIterator>, _Alloc>;
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+template <ranges::input_range _Range,
+          class _Alloc = allocator<ranges::range_value_t<_Range>>,
+          class        = enable_if_t<__is_allocator<_Alloc>::value> >
+vector(from_range_t, _Range&&, _Alloc = _Alloc()) -> vector<ranges::range_value_t<_Range>, _Alloc>;
+#endif
+
+// __swap_out_circular_buffer relocates the objects in [__begin_, __end_) into the front of __v and swaps the buffers of
+// *this and __v. It is assumed that __v provides space for exactly (__end_ - __begin_) objects in the front. This
+// function has a strong exception guarantee.
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v) {
+  __annotate_delete();
+  auto __new_begin = __v.__begin_ - (__end_ - __begin_);
+  std::__uninitialized_allocator_relocate(
+      __alloc(), std::__to_address(__begin_), std::__to_address(__end_), std::__to_address(__new_begin));
+  __v.__begin_ = __new_begin;
+  __end_       = __begin_; // All the objects have been destroyed by relocating them.
+  std::swap(this->__begin_, __v.__begin_);
+  std::swap(this->__end_, __v.__end_);
+  std::swap(this->__end_cap(), __v.__end_cap());
+  __v.__first_ = __v.__begin_;
+  __annotate_new(size());
+}
+
+// __swap_out_circular_buffer relocates the objects in [__begin_, __p) into the front of __v, the objects in
+// [__p, __end_) into the back of __v and swaps the buffers of *this and __v. It is assumed that __v provides space for
+// exactly (__p - __begin_) objects in the front and space for at least (__end_ - __p) objects in the back. This
+// function has a strong exception guarantee if __begin_ == __p || __end_ == __p.
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::pointer
+vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v, pointer __p) {
+  __annotate_delete();
+  pointer __ret = __v.__begin_;
+
+  // Relocate [__p, __end_) first to avoid having a hole in [__begin_, __end_)
+  // in case something in [__begin_, __p) throws.
+  std::__uninitialized_allocator_relocate(
+      __alloc(), std::__to_address(__p), std::__to_address(__end_), std::__to_address(__v.__end_));
+  __v.__end_ += (__end_ - __p);
+  __end_           = __p; // The objects in [__p, __end_) have been destroyed by relocating them.
+  auto __new_begin = __v.__begin_ - (__p - __begin_);
+
+  std::__uninitialized_allocator_relocate(
+      __alloc(), std::__to_address(__begin_), std::__to_address(__p), std::__to_address(__new_begin));
+  __v.__begin_ = __new_begin;
+  __end_       = __begin_; // All the objects have been destroyed by relocating them.
+
+  std::swap(this->__begin_, __v.__begin_);
+  std::swap(this->__end_, __v.__end_);
+  std::swap(this->__end_cap(), __v.__end_cap());
+  __v.__first_ = __v.__begin_;
+  __annotate_new(size());
+  return __ret;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__vdeallocate() _NOEXCEPT {
+  if (this->__begin_ != nullptr) {
+    clear();
+    __annotate_delete();
+    __alloc_traits::deallocate(this->__alloc(), this->__begin_, capacity());
+    this->__begin_ = this->__end_ = this->__end_cap() = nullptr;
+  }
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::size_type
+vector<_Tp, _Allocator>::max_size() const _NOEXCEPT {
+  return std::min<size_type>(__alloc_traits::max_size(this->__alloc()), numeric_limits<difference_type>::max());
+}
+
+//  Precondition:  __new_size > capacity()
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::size_type
+vector<_Tp, _Allocator>::__recommend(size_type __new_size) const {
+  const size_type __ms = max_size();
+  if (__new_size > __ms)
+    this->__throw_length_error();
+  const size_type __cap = capacity();
+  if (__cap >= __ms / 2)
+    return __ms;
+  return std::max<size_type>(2 * __cap, __new_size);
+}
+
+//  Default constructs __n objects starting at __end_
+//  throws if construction throws
+//  Precondition:  __n > 0
+//  Precondition:  size() + __n <= capacity()
+//  Postcondition:  size() == size() + __n
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__construct_at_end(size_type __n) {
+  _ConstructTransaction __tx(*this, __n);
+  const_pointer __new_end = __tx.__new_end_;
+  for (pointer __pos = __tx.__pos_; __pos != __new_end; __tx.__pos_ = ++__pos) {
+    __alloc_traits::construct(this->__alloc(), std::__to_address(__pos));
+  }
+}
+
+//  Copy constructs __n objects starting at __end_ from __x
+//  throws if construction throws
+//  Precondition:  __n > 0
+//  Precondition:  size() + __n <= capacity()
+//  Postcondition:  size() == old size() + __n
+//  Postcondition:  [i] == __x for all i in [size() - __n, __n)
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void
+vector<_Tp, _Allocator>::__construct_at_end(size_type __n, const_reference __x) {
+  _ConstructTransaction __tx(*this, __n);
+  const_pointer __new_end = __tx.__new_end_;
+  for (pointer __pos = __tx.__pos_; __pos != __new_end; __tx.__pos_ = ++__pos) {
+    __alloc_traits::construct(this->__alloc(), std::__to_address(__pos), __x);
+  }
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+vector<_Tp, _Allocator>::__construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n) {
+  _ConstructTransaction __tx(*this, __n);
+  __tx.__pos_ = std::__uninitialized_allocator_copy(__alloc(), __first, __last, __tx.__pos_);
+}
+
+//  Default constructs __n objects starting at __end_
+//  throws if construction throws
+//  Postcondition:  size() == size() + __n
+//  Exception safety: strong.
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__append(size_type __n) {
+  if (static_cast<size_type>(this->__end_cap() - this->__end_) >= __n)
+    this->__construct_at_end(__n);
+  else {
+    allocator_type& __a = this->__alloc();
+    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), size(), __a);
+    __v.__construct_at_end(__n);
+    __swap_out_circular_buffer(__v);
+  }
+}
+
+//  Default constructs __n objects starting at __end_
+//  throws if construction throws
+//  Postcondition:  size() == size() + __n
+//  Exception safety: strong.
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__append(size_type __n, const_reference __x) {
+  if (static_cast<size_type>(this->__end_cap() - this->__end_) >= __n)
+    this->__construct_at_end(__n, __x);
+  else {
+    allocator_type& __a = this->__alloc();
+    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), size(), __a);
+    __v.__construct_at_end(__n, __x);
+    __swap_out_circular_buffer(__v);
+  }
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIterator,
+          __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_InputIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<_Tp, _Allocator>::vector(_InputIterator __first, _InputIterator __last) {
+  __init_with_sentinel(__first, __last);
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIterator,
+          __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_InputIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<_Tp, _Allocator>::vector(_InputIterator __first, _InputIterator __last, const allocator_type& __a)
+    : __alloc_(__a) {
+  __init_with_sentinel(__first, __last);
+}
+
+template <class _Tp, class _Allocator>
+template <class _ForwardIterator,
+          __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_ForwardIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<_Tp, _Allocator>::vector(_ForwardIterator __first, _ForwardIterator __last) {
+  size_type __n = static_cast<size_type>(std::distance(__first, __last));
+  __init_with_size(__first, __last, __n);
+}
+
+template <class _Tp, class _Allocator>
+template <class _ForwardIterator,
+          __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_ForwardIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<_Tp, _Allocator>::vector(_ForwardIterator __first, _ForwardIterator __last, const allocator_type& __a)
+    : __alloc_(__a) {
+  size_type __n = static_cast<size_type>(std::distance(__first, __last));
+  __init_with_size(__first, __last, __n);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<_Tp, _Allocator>::vector(const vector& __x)
+    : __alloc_(__alloc_traits::select_on_container_copy_construction(__x.__alloc())) {
+  __init_with_size(__x.__begin_, __x.__end_, __x.size());
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<_Tp, _Allocator>::vector(const vector& __x, const __type_identity_t<allocator_type>& __a)
+    : __alloc_(__a) {
+  __init_with_size(__x.__begin_, __x.__end_, __x.size());
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI vector<_Tp, _Allocator>::vector(vector&& __x)
+#if _LIBCPP_STD_VER >= 17
+    noexcept
+#else
+    _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value)
+#endif
+    : __alloc_(std::move(__x.__alloc())) {
+  this->__begin_    = __x.__begin_;
+  this->__end_      = __x.__end_;
+  this->__end_cap() = __x.__end_cap();
+  __x.__begin_ = __x.__end_ = __x.__end_cap() = nullptr;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI
+vector<_Tp, _Allocator>::vector(vector&& __x, const __type_identity_t<allocator_type>& __a)
+    : __alloc_(__a) {
+  if (__a == __x.__alloc()) {
+    this->__begin_    = __x.__begin_;
+    this->__end_      = __x.__end_;
+    this->__end_cap() = __x.__end_cap();
+    __x.__begin_ = __x.__end_ = __x.__end_cap() = nullptr;
+  } else {
+    typedef move_iterator<iterator> _Ip;
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+    assign(_Ip(__x.begin()), _Ip(__x.end()));
+    __guard.__complete();
+  }
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI
+vector<_Tp, _Allocator>::vector(initializer_list<value_type> __il) {
+  __init_with_size(__il.begin(), __il.end(), __il.size());
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI
+vector<_Tp, _Allocator>::vector(initializer_list<value_type> __il, const allocator_type& __a)
+    : __alloc_(__a) {
+  __init_with_size(__il.begin(), __il.end(), __il.size());
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI vector<_Tp, _Allocator>&
+vector<_Tp, _Allocator>::operator=(vector&& __x)
+    _NOEXCEPT_(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value) {
+  __move_assign(__x, integral_constant<bool, __alloc_traits::propagate_on_container_move_assignment::value>());
+  return *this;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__move_assign(vector& __c, false_type)
+    _NOEXCEPT_(__alloc_traits::is_always_equal::value) {
+  if (__alloc() != __c.__alloc()) {
+    typedef move_iterator<iterator> _Ip;
+    assign(_Ip(__c.begin()), _Ip(__c.end()));
+  } else
+    __move_assign(__c, true_type());
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__move_assign(vector& __c, true_type)
+    _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
+  __vdeallocate();
+  __move_assign_alloc(__c); // this can throw
+  this->__begin_    = __c.__begin_;
+  this->__end_      = __c.__end_;
+  this->__end_cap() = __c.__end_cap();
+  __c.__begin_ = __c.__end_ = __c.__end_cap() = nullptr;
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI vector<_Tp, _Allocator>&
+vector<_Tp, _Allocator>::operator=(const vector& __x) {
+  if (this != std::addressof(__x)) {
+    __copy_assign_alloc(__x);
+    assign(__x.__begin_, __x.__end_);
+  }
+  return *this;
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIterator,
+          __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_InputIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::assign(_InputIterator __first, _InputIterator __last) {
+  __assign_with_sentinel(__first, __last);
+}
+
+template <class _Tp, class _Allocator>
+template <class _Iterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+vector<_Tp, _Allocator>::__assign_with_sentinel(_Iterator __first, _Sentinel __last) {
+  clear();
+  for (; __first != __last; ++__first)
+    emplace_back(*__first);
+}
+
+template <class _Tp, class _Allocator>
+template <class _ForwardIterator,
+          __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_ForwardIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::assign(_ForwardIterator __first, _ForwardIterator __last) {
+  __assign_with_size(__first, __last, std::distance(__first, __last));
+}
+
+template <class _Tp, class _Allocator>
+template <class _ForwardIterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+vector<_Tp, _Allocator>::__assign_with_size(_ForwardIterator __first, _Sentinel __last, difference_type __n) {
+  size_type __new_size = static_cast<size_type>(__n);
+  if (__new_size <= capacity()) {
+    if (__new_size > size()) {
+      _ForwardIterator __mid = std::next(__first, size());
+      std::copy(__first, __mid, this->__begin_);
+      __construct_at_end(__mid, __last, __new_size - size());
+    } else {
+      pointer __m = std::__copy<_ClassicAlgPolicy>(__first, __last, this->__begin_).second;
+      this->__destruct_at_end(__m);
+    }
+  } else {
+    __vdeallocate();
+    __vallocate(__recommend(__new_size));
+    __construct_at_end(__first, __last, __new_size);
+  }
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::assign(size_type __n, const_reference __u) {
+  if (__n <= capacity()) {
+    size_type __s = size();
+    std::fill_n(this->__begin_, std::min(__n, __s), __u);
+    if (__n > __s)
+      __construct_at_end(__n - __s, __u);
+    else
+      this->__destruct_at_end(this->__begin_ + __n);
+  } else {
+    __vdeallocate();
+    __vallocate(__recommend(static_cast<size_type>(__n)));
+    __construct_at_end(__n, __u);
+  }
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::begin() _NOEXCEPT {
+  return __make_iter(this->__begin_);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::const_iterator
+vector<_Tp, _Allocator>::begin() const _NOEXCEPT {
+  return __make_iter(this->__begin_);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::end() _NOEXCEPT {
+  return __make_iter(this->__end_);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::const_iterator
+vector<_Tp, _Allocator>::end() const _NOEXCEPT {
+  return __make_iter(this->__end_);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::reference
+vector<_Tp, _Allocator>::operator[](size_type __n) _NOEXCEPT {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < size(), "vector[] index out of bounds");
+  return this->__begin_[__n];
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::const_reference
+vector<_Tp, _Allocator>::operator[](size_type __n) const _NOEXCEPT {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < size(), "vector[] index out of bounds");
+  return this->__begin_[__n];
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::reference vector<_Tp, _Allocator>::at(size_type __n) {
+  if (__n >= size())
+    this->__throw_out_of_range();
+  return this->__begin_[__n];
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::const_reference
+vector<_Tp, _Allocator>::at(size_type __n) const {
+  if (__n >= size())
+    this->__throw_out_of_range();
+  return this->__begin_[__n];
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::reserve(size_type __n) {
+  if (__n > capacity()) {
+    if (__n > max_size())
+      this->__throw_length_error();
+    allocator_type& __a = this->__alloc();
+    __split_buffer<value_type, allocator_type&> __v(__n, size(), __a);
+    __swap_out_circular_buffer(__v);
+  }
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::shrink_to_fit() _NOEXCEPT {
+  if (capacity() > size()) {
+#if _LIBCPP_HAS_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_EXCEPTIONS
+      allocator_type& __a = this->__alloc();
+      __split_buffer<value_type, allocator_type&> __v(size(), size(), __a);
+      // The Standard mandates shrink_to_fit() does not increase the capacity.
+      // With equal capacity keep the existing buffer. This avoids extra work
+      // due to swapping the elements.
+      if (__v.capacity() < capacity())
+        __swap_out_circular_buffer(__v);
+#if _LIBCPP_HAS_EXCEPTIONS
+    } catch (...) {
+    }
+#endif // _LIBCPP_HAS_EXCEPTIONS
+  }
+}
+
+template <class _Tp, class _Allocator>
+template <class... _Args>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::pointer
+vector<_Tp, _Allocator>::__emplace_back_slow_path(_Args&&... __args) {
+  allocator_type& __a = this->__alloc();
+  __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), size(), __a);
+  //    __v.emplace_back(std::forward<_Args>(__args)...);
+  __alloc_traits::construct(__a, std::__to_address(__v.__end_), std::forward<_Args>(__args)...);
+  __v.__end_++;
+  __swap_out_circular_buffer(__v);
+  return this->__end_;
+}
+
+template <class _Tp, class _Allocator>
+template <class... _Args>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline
+#if _LIBCPP_STD_VER >= 17
+    typename vector<_Tp, _Allocator>::reference
+#else
+    void
+#endif
+    vector<_Tp, _Allocator>::emplace_back(_Args&&... __args) {
+  pointer __end = this->__end_;
+  if (__end < this->__end_cap()) {
+    __construct_one_at_end(std::forward<_Args>(__args)...);
+    ++__end;
+  } else {
+    __end = __emplace_back_slow_path(std::forward<_Args>(__args)...);
+  }
+  this->__end_ = __end;
+#if _LIBCPP_STD_VER >= 17
+  return *(__end - 1);
+#endif
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void vector<_Tp, _Allocator>::pop_back() {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "vector::pop_back called on an empty vector");
+  this->__destruct_at_end(this->__end_ - 1);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::erase(const_iterator __position) {
+  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+      __position != end(), "vector::erase(iterator) called with a non-dereferenceable iterator");
+  difference_type __ps = __position - cbegin();
+  pointer __p          = this->__begin_ + __ps;
+  this->__destruct_at_end(std::move(__p + 1, this->__end_, __p));
+  return __make_iter(__p);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::erase(const_iterator __first, const_iterator __last) {
+  _LIBCPP_ASSERT_VALID_INPUT_RANGE(__first <= __last, "vector::erase(first, last) called with invalid range");
+  pointer __p = this->__begin_ + (__first - begin());
+  if (__first != __last) {
+    this->__destruct_at_end(std::move(__p + (__last - __first), this->__end_, __p));
+  }
+  return __make_iter(__p);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+vector<_Tp, _Allocator>::__move_range(pointer __from_s, pointer __from_e, pointer __to) {
+  pointer __old_last  = this->__end_;
+  difference_type __n = __old_last - __to;
+  {
+    pointer __i = __from_s + __n;
+    _ConstructTransaction __tx(*this, __from_e - __i);
+    for (pointer __pos = __tx.__pos_; __i < __from_e; ++__i, (void)++__pos, __tx.__pos_ = __pos) {
+      __alloc_traits::construct(this->__alloc(), std::__to_address(__pos), std::move(*__i));
+    }
+  }
+  std::move_backward(__from_s, __from_s + __n, __old_last);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::insert(const_iterator __position, const_reference __x) {
+  pointer __p = this->__begin_ + (__position - begin());
+  if (this->__end_ < this->__end_cap()) {
+    if (__p == this->__end_) {
+      __construct_one_at_end(__x);
+    } else {
+      __move_range(__p, this->__end_, __p + 1);
+      const_pointer __xr = pointer_traits<const_pointer>::pointer_to(__x);
+      if (std::__is_pointer_in_range(std::__to_address(__p), std::__to_address(__end_), std::addressof(__x)))
+        ++__xr;
+      *__p = *__xr;
+    }
+  } else {
+    allocator_type& __a = this->__alloc();
+    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, __a);
+    __v.push_back(__x);
+    __p = __swap_out_circular_buffer(__v, __p);
+  }
+  return __make_iter(__p);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::insert(const_iterator __position, value_type&& __x) {
+  pointer __p = this->__begin_ + (__position - begin());
+  if (this->__end_ < this->__end_cap()) {
+    if (__p == this->__end_) {
+      __construct_one_at_end(std::move(__x));
+    } else {
+      __move_range(__p, this->__end_, __p + 1);
+      *__p = std::move(__x);
+    }
+  } else {
+    allocator_type& __a = this->__alloc();
+    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, __a);
+    __v.push_back(std::move(__x));
+    __p = __swap_out_circular_buffer(__v, __p);
+  }
+  return __make_iter(__p);
+}
+
+template <class _Tp, class _Allocator>
+template <class... _Args>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::emplace(const_iterator __position, _Args&&... __args) {
+  pointer __p = this->__begin_ + (__position - begin());
+  if (this->__end_ < this->__end_cap()) {
+    if (__p == this->__end_) {
+      __construct_one_at_end(std::forward<_Args>(__args)...);
+    } else {
+      __temp_value<value_type, _Allocator> __tmp(this->__alloc(), std::forward<_Args>(__args)...);
+      __move_range(__p, this->__end_, __p + 1);
+      *__p = std::move(__tmp.get());
+    }
+  } else {
+    allocator_type& __a = this->__alloc();
+    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, __a);
+    __v.emplace_back(std::forward<_Args>(__args)...);
+    __p = __swap_out_circular_buffer(__v, __p);
+  }
+  return __make_iter(__p);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::insert(const_iterator __position, size_type __n, const_reference __x) {
+  pointer __p = this->__begin_ + (__position - begin());
+  if (__n > 0) {
+    // We can't compare unrelated pointers inside constant expressions
+    if (!__libcpp_is_constant_evaluated() && __n <= static_cast<size_type>(this->__end_cap() - this->__end_)) {
+      size_type __old_n  = __n;
+      pointer __old_last = this->__end_;
+      if (__n > static_cast<size_type>(this->__end_ - __p)) {
+        size_type __cx = __n - (this->__end_ - __p);
+        __construct_at_end(__cx, __x);
+        __n -= __cx;
+      }
+      if (__n > 0) {
+        __move_range(__p, __old_last, __p + __old_n);
+        const_pointer __xr = pointer_traits<const_pointer>::pointer_to(__x);
+        if (__p <= __xr && __xr < this->__end_)
+          __xr += __old_n;
+        std::fill_n(__p, __n, *__xr);
+      }
+    } else {
+      allocator_type& __a = this->__alloc();
+      __split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), __p - this->__begin_, __a);
+      __v.__construct_at_end(__n, __x);
+      __p = __swap_out_circular_buffer(__v, __p);
+    }
+  }
+  return __make_iter(__p);
+}
+template <class _Tp, class _Allocator>
+template <class _InputIterator,
+          __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_InputIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::insert(const_iterator __position, _InputIterator __first, _InputIterator __last) {
+  return __insert_with_sentinel(__position, __first, __last);
+}
+
+template <class _Tp, class _Allocator>
+template <class _InputIterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::__insert_with_sentinel(const_iterator __position, _InputIterator __first, _Sentinel __last) {
+  difference_type __off = __position - begin();
+  pointer __p           = this->__begin_ + __off;
+  allocator_type& __a   = this->__alloc();
+  pointer __old_last    = this->__end_;
+  for (; this->__end_ != this->__end_cap() && __first != __last; ++__first) {
+    __construct_one_at_end(*__first);
+  }
+  __split_buffer<value_type, allocator_type&> __v(__a);
+  if (__first != __last) {
+#if _LIBCPP_HAS_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_EXCEPTIONS
+      __v.__construct_at_end_with_sentinel(std::move(__first), std::move(__last));
+      difference_type __old_size = __old_last - this->__begin_;
+      difference_type __old_p    = __p - this->__begin_;
+      reserve(__recommend(size() + __v.size()));
+      __p        = this->__begin_ + __old_p;
+      __old_last = this->__begin_ + __old_size;
+#if _LIBCPP_HAS_EXCEPTIONS
+    } catch (...) {
+      erase(__make_iter(__old_last), end());
+      throw;
+    }
+#endif // _LIBCPP_HAS_EXCEPTIONS
+  }
+  __p = std::rotate(__p, __old_last, this->__end_);
+  insert(__make_iter(__p), std::make_move_iterator(__v.begin()), std::make_move_iterator(__v.end()));
+  return begin() + __off;
+}
+
+template <class _Tp, class _Allocator>
+template <class _ForwardIterator,
+          __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
+                            is_constructible<_Tp, typename iterator_traits<_ForwardIterator>::reference>::value,
+                        int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::insert(const_iterator __position, _ForwardIterator __first, _ForwardIterator __last) {
+  return __insert_with_size(__position, __first, __last, std::distance(__first, __last));
+}
+
+template <class _Tp, class _Allocator>
+template <class _Iterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
+vector<_Tp, _Allocator>::__insert_with_size(
+    const_iterator __position, _Iterator __first, _Sentinel __last, difference_type __n) {
+  auto __insertion_size = __n;
+  pointer __p           = this->__begin_ + (__position - begin());
+  if (__n > 0) {
+    if (__n <= this->__end_cap() - this->__end_) {
+      size_type __old_n    = __n;
+      pointer __old_last   = this->__end_;
+      _Iterator __m        = std::next(__first, __n);
+      difference_type __dx = this->__end_ - __p;
+      if (__n > __dx) {
+        __m                    = __first;
+        difference_type __diff = this->__end_ - __p;
+        std::advance(__m, __diff);
+        __construct_at_end(__m, __last, __n - __diff);
+        __n = __dx;
+      }
+      if (__n > 0) {
+        __move_range(__p, __old_last, __p + __old_n);
+        std::copy(__first, __m, __p);
+      }
+    } else {
+      allocator_type& __a = this->__alloc();
+      __split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), __p - this->__begin_, __a);
+      __v.__construct_at_end_with_size(__first, __insertion_size);
+      __p = __swap_out_circular_buffer(__v, __p);
+    }
+  }
+  return __make_iter(__p);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::resize(size_type __sz) {
+  size_type __cs = size();
+  if (__cs < __sz)
+    this->__append(__sz - __cs);
+  else if (__cs > __sz)
+    this->__destruct_at_end(this->__begin_ + __sz);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::resize(size_type __sz, const_reference __x) {
+  size_type __cs = size();
+  if (__cs < __sz)
+    this->__append(__sz - __cs, __x);
+  else if (__cs > __sz)
+    this->__destruct_at_end(this->__begin_ + __sz);
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::swap(vector& __x)
+#if _LIBCPP_STD_VER >= 14
+    _NOEXCEPT
+#else
+    _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>)
+#endif
+{
+  _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
+      __alloc_traits::propagate_on_container_swap::value || this->__alloc() == __x.__alloc(),
+      "vector::swap: Either propagate_on_container_swap must be true"
+      " or the allocators must compare equal");
+  std::swap(this->__begin_, __x.__begin_);
+  std::swap(this->__end_, __x.__end_);
+  std::swap(this->__end_cap(), __x.__end_cap());
+  std::__swap_allocator(this->__alloc(), __x.__alloc());
+}
+
+template <class _Tp, class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 bool vector<_Tp, _Allocator>::__invariants() const {
+  if (this->__begin_ == nullptr) {
+    if (this->__end_ != nullptr || this->__end_cap() != nullptr)
+      return false;
+  } else {
+    if (this->__begin_ > this->__end_)
+      return false;
+    if (this->__begin_ == this->__end_cap())
+      return false;
+    if (this->__end_ > this->__end_cap())
+      return false;
+  }
+  return true;
+}
+
+#if _LIBCPP_STD_VER >= 20
+template <>
+inline constexpr bool __format::__enable_insertable<vector<char>> = true;
+#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+template <>
+inline constexpr bool __format::__enable_insertable<vector<wchar_t>> = true;
+#  endif
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___VECTOR_VECTOR_H
diff --git a/libcxx/include/__vector/vector_bool.h b/libcxx/include/__vector/vector_bool.h
new file mode 100644
index 000000000000..462095fd07ac
--- /dev/null
+++ b/libcxx/include/__vector/vector_bool.h
@@ -0,0 +1,1123 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___VECTOR_VECTOR_BOOL_H
+#define _LIBCPP___VECTOR_VECTOR_BOOL_H
+
+#include <__algorithm/copy.h>
+#include <__algorithm/fill_n.h>
+#include <__algorithm/iterator_operations.h>
+#include <__algorithm/max.h>
+#include <__assert>
+#include <__bit_reference>
+#include <__config>
+#include <__functional/unary_function.h>
+#include <__fwd/functional.h>
+#include <__fwd/vector.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/reverse_iterator.h>
+#include <__memory/addressof.h>
+#include <__memory/allocate_at_least.h>
+#include <__memory/allocator.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/compressed_pair.h>
+#include <__memory/construct_at.h>
+#include <__memory/noexcept_move_assign_container.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/swap_allocator.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/container_compatible_range.h>
+#include <__ranges/from_range.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_nothrow_assignable.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/type_identity.h>
+#include <__utility/exception_guard.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
+#include <climits>
+#include <initializer_list>
+#include <limits>
+#include <stdexcept>
+
+// These headers define parts of vectors definition, since they define ADL functions or class specializations.
+#include <__vector/comparison.h>
+#include <__vector/container_traits.h>
+#include <__vector/swap.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Allocator>
+struct hash<vector<bool, _Allocator> >;
+
+template <class _Allocator>
+struct __has_storage_type<vector<bool, _Allocator> > {
+  static const bool value = true;
+};
+
+template <class _Allocator>
+class _LIBCPP_TEMPLATE_VIS vector<bool, _Allocator> {
+public:
+  typedef vector __self;
+  typedef bool value_type;
+  typedef _Allocator allocator_type;
+  typedef allocator_traits<allocator_type> __alloc_traits;
+  typedef typename __alloc_traits::size_type size_type;
+  typedef typename __alloc_traits::difference_type difference_type;
+  typedef size_type __storage_type;
+  typedef __bit_iterator<vector, false> pointer;
+  typedef __bit_iterator<vector, true> const_pointer;
+  typedef pointer iterator;
+  typedef const_pointer const_iterator;
+  typedef std::reverse_iterator<iterator> reverse_iterator;
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+
+private:
+  typedef __rebind_alloc<__alloc_traits, __storage_type> __storage_allocator;
+  typedef allocator_traits<__storage_allocator> __storage_traits;
+  typedef typename __storage_traits::pointer __storage_pointer;
+  typedef typename __storage_traits::const_pointer __const_storage_pointer;
+
+  __storage_pointer __begin_;
+  size_type __size_;
+  _LIBCPP_COMPRESSED_PAIR(size_type, __cap_, __storage_allocator, __alloc_);
+
+public:
+  typedef __bit_reference<vector> reference;
+#ifdef _LIBCPP_ABI_BITSET_VECTOR_BOOL_CONST_SUBSCRIPT_RETURN_BOOL
+  using const_reference = bool;
+#else
+  typedef __bit_const_reference<vector> const_reference;
+#endif
+
+private:
+  // TODO: Remove these now redundant accessors
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type& __cap() _NOEXCEPT { return __cap_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const size_type& __cap() const _NOEXCEPT { return __cap_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __storage_allocator& __alloc() _NOEXCEPT { return __alloc_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const __storage_allocator& __alloc() const _NOEXCEPT {
+    return __alloc_;
+  }
+
+  static const unsigned __bits_per_word = static_cast<unsigned>(sizeof(__storage_type) * CHAR_BIT);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static size_type
+  __internal_cap_to_external(size_type __n) _NOEXCEPT {
+    return __n * __bits_per_word;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static size_type
+  __external_cap_to_internal(size_type __n) _NOEXCEPT {
+    return (__n - 1) / __bits_per_word + 1;
+  }
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector()
+      _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit vector(const allocator_type& __a)
+#if _LIBCPP_STD_VER <= 14
+      _NOEXCEPT_(is_nothrow_copy_constructible<allocator_type>::value);
+#else
+      _NOEXCEPT;
+#endif
+
+private:
+  class __destroy_vector {
+  public:
+    _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI __destroy_vector(vector& __vec) : __vec_(__vec) {}
+
+    _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void operator()() {
+      if (__vec_.__begin_ != nullptr)
+        __storage_traits::deallocate(__vec_.__alloc(), __vec_.__begin_, __vec_.__cap());
+    }
+
+  private:
+    vector& __vec_;
+  };
+
+public:
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~vector() { __destroy_vector (*this)(); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit vector(size_type __n);
+#if _LIBCPP_STD_VER >= 14
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit vector(size_type __n, const allocator_type& __a);
+#endif
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(size_type __n, const value_type& __v);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  vector(size_type __n, const value_type& __v, const allocator_type& __a);
+  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(_InputIterator __first, _InputIterator __last);
+  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  vector(_InputIterator __first, _InputIterator __last, const allocator_type& __a);
+  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(_ForwardIterator __first, _ForwardIterator __last);
+  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  vector(_ForwardIterator __first, _ForwardIterator __last, const allocator_type& __a);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<bool> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr vector(from_range_t, _Range&& __range, const allocator_type& __a = allocator_type())
+      : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
+    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      __init_with_size(ranges::begin(__range), ranges::end(__range), __n);
+
+    } else {
+      __init_with_sentinel(ranges::begin(__range), ranges::end(__range));
+    }
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(const vector& __v);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(const vector& __v, const allocator_type& __a);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector& operator=(const vector& __v);
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(initializer_list<value_type> __il);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  vector(initializer_list<value_type> __il, const allocator_type& __a);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector& operator=(initializer_list<value_type> __il) {
+    assign(__il.begin(), __il.end());
+    return *this;
+  }
+
+#endif // !_LIBCPP_CXX03_LANG
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(vector&& __v)
+#if _LIBCPP_STD_VER >= 17
+      noexcept;
+#else
+      _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value);
+#endif
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  vector(vector&& __v, const __type_identity_t<allocator_type>& __a);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector& operator=(vector&& __v)
+      _NOEXCEPT_(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value);
+
+  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 assign(_InputIterator __first, _InputIterator __last);
+  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+  void _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 assign(_ForwardIterator __first, _ForwardIterator __last);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<bool> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr void assign_range(_Range&& __range) {
+    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      __assign_with_size(ranges::begin(__range), ranges::end(__range), __n);
+
+    } else {
+      __assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
+    }
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void assign(size_type __n, const value_type& __x);
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void assign(initializer_list<value_type> __il) {
+    assign(__il.begin(), __il.end());
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator_type get_allocator() const _NOEXCEPT {
+    return allocator_type(this->__alloc());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type max_size() const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type capacity() const _NOEXCEPT {
+    return __internal_cap_to_external(__cap());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type size() const _NOEXCEPT { return __size_; }
+  [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool empty() const _NOEXCEPT {
+    return __size_ == 0;
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void reserve(size_type __n);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void shrink_to_fit() _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator begin() _NOEXCEPT { return __make_iter(0); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator begin() const _NOEXCEPT { return __make_iter(0); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator end() _NOEXCEPT { return __make_iter(__size_); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator end() const _NOEXCEPT {
+    return __make_iter(__size_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rbegin() _NOEXCEPT {
+    return reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator rbegin() const _NOEXCEPT {
+    return const_reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rend() _NOEXCEPT {
+    return reverse_iterator(begin());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator rend() const _NOEXCEPT {
+    return const_reverse_iterator(begin());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cbegin() const _NOEXCEPT { return __make_iter(0); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cend() const _NOEXCEPT {
+    return __make_iter(__size_);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator crbegin() const _NOEXCEPT {
+    return rbegin();
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference operator[](size_type __n) { return __make_ref(__n); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference operator[](size_type __n) const {
+    return __make_ref(__n);
+  }
+  _LIBCPP_HIDE_FROM_ABI reference at(size_type __n);
+  _LIBCPP_HIDE_FROM_ABI const_reference at(size_type __n) const;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference front() { return __make_ref(0); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference front() const { return __make_ref(0); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference back() { return __make_ref(__size_ - 1); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference back() const { return __make_ref(__size_ - 1); }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void push_back(const value_type& __x);
+#if _LIBCPP_STD_VER >= 14
+  template <class... _Args>
+#  if _LIBCPP_STD_VER >= 17
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference emplace_back(_Args&&... __args)
+#  else
+  _LIBCPP_HIDE_FROM_ABI void emplace_back(_Args&&... __args)
+#  endif
+  {
+    push_back(value_type(std::forward<_Args>(__args)...));
+#  if _LIBCPP_STD_VER >= 17
+    return this->back();
+#  endif
+  }
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<bool> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr void append_range(_Range&& __range) {
+    insert_range(end(), std::forward<_Range>(__range));
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void pop_back() { --__size_; }
+
+#if _LIBCPP_STD_VER >= 14
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator emplace(const_iterator __position, _Args&&... __args) {
+    return insert(__position, value_type(std::forward<_Args>(__args)...));
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator insert(const_iterator __position, const value_type& __x);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
+  insert(const_iterator __position, size_type __n, const value_type& __x);
+  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
+  iterator _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  insert(const_iterator __position, _InputIterator __first, _InputIterator __last);
+  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
+  iterator _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  insert(const_iterator __position, _ForwardIterator __first, _ForwardIterator __last);
+
+#if _LIBCPP_STD_VER >= 23
+  template <_ContainerCompatibleRange<bool> _Range>
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator insert_range(const_iterator __position, _Range&& __range) {
+    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
+      auto __n = static_cast<size_type>(ranges::distance(__range));
+      return __insert_with_size(__position, ranges::begin(__range), ranges::end(__range), __n);
+
+    } else {
+      return __insert_with_sentinel(__position, ranges::begin(__range), ranges::end(__range));
+    }
+  }
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
+  insert(const_iterator __position, initializer_list<value_type> __il) {
+    return insert(__position, __il.begin(), __il.end());
+  }
+#endif
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator erase(const_iterator __position);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator erase(const_iterator __first, const_iterator __last);
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void clear() _NOEXCEPT { __size_ = 0; }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(vector&)
+#if _LIBCPP_STD_VER >= 14
+      _NOEXCEPT;
+#else
+      _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>);
+#endif
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void swap(reference __x, reference __y) _NOEXCEPT {
+    std::swap(__x, __y);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void resize(size_type __sz, value_type __x = false);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void flip() _NOEXCEPT;
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __invariants() const;
+
+private:
+  [[__noreturn__]] _LIBCPP_HIDE_FROM_ABI void __throw_length_error() const { std::__throw_length_error("vector"); }
+
+  [[__noreturn__]] _LIBCPP_HIDE_FROM_ABI void __throw_out_of_range() const { std::__throw_out_of_range("vector"); }
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+  __init_with_size(_InputIterator __first, _Sentinel __last, size_type __n) {
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+
+    if (__n > 0) {
+      __vallocate(__n);
+      __construct_at_end(std::move(__first), std::move(__last), __n);
+    }
+
+    __guard.__complete();
+  }
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+  __init_with_sentinel(_InputIterator __first, _Sentinel __last) {
+#if _LIBCPP_HAS_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_EXCEPTIONS
+      for (; __first != __last; ++__first)
+        push_back(*__first);
+#if _LIBCPP_HAS_EXCEPTIONS
+    } catch (...) {
+      if (__begin_ != nullptr)
+        __storage_traits::deallocate(__alloc(), __begin_, __cap());
+      throw;
+    }
+#endif // _LIBCPP_HAS_EXCEPTIONS
+  }
+
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __first, _Sentinel __last);
+
+  template <class _ForwardIterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+  __assign_with_size(_ForwardIterator __first, _Sentinel __last, difference_type __ns);
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  __insert_with_sentinel(const_iterator __position, _InputIterator __first, _Sentinel __last);
+
+  template <class _Iterator, class _Sentinel>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
+  __insert_with_size(const_iterator __position, _Iterator __first, _Sentinel __last, difference_type __n);
+
+  //  Allocate space for __n objects
+  //  throws length_error if __n > max_size()
+  //  throws (probably bad_alloc) if memory run out
+  //  Precondition:  __begin_ == __end_ == __cap() == 0
+  //  Precondition:  __n > 0
+  //  Postcondition:  capacity() >= __n
+  //  Postcondition:  size() == 0
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __vallocate(size_type __n) {
+    if (__n > max_size())
+      __throw_length_error();
+    auto __allocation = std::__allocate_at_least(__alloc(), __external_cap_to_internal(__n));
+    __begin_          = __allocation.ptr;
+    __size_           = 0;
+    __cap()           = __allocation.count;
+    if (__libcpp_is_constant_evaluated()) {
+      for (size_type __i = 0; __i != __cap(); ++__i)
+        std::__construct_at(std::__to_address(__begin_) + __i);
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __vdeallocate() _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static size_type __align_it(size_type __new_size) _NOEXCEPT {
+    return (__new_size + (__bits_per_word - 1)) & ~((size_type)__bits_per_word - 1);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type __recommend(size_type __new_size) const;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __construct_at_end(size_type __n, bool __x);
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+  __construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __append(size_type __n, const_reference __x);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference __make_ref(size_type __pos) _NOEXCEPT {
+    return reference(__begin_ + __pos / __bits_per_word, __storage_type(1) << __pos % __bits_per_word);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference __make_ref(size_type __pos) const _NOEXCEPT {
+    return __bit_const_reference<vector>(
+        __begin_ + __pos / __bits_per_word, __storage_type(1) << __pos % __bits_per_word);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator __make_iter(size_type __pos) _NOEXCEPT {
+    return iterator(__begin_ + __pos / __bits_per_word, static_cast<unsigned>(__pos % __bits_per_word));
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator __make_iter(size_type __pos) const _NOEXCEPT {
+    return const_iterator(__begin_ + __pos / __bits_per_word, static_cast<unsigned>(__pos % __bits_per_word));
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator __const_iterator_cast(const_iterator __p) _NOEXCEPT {
+    return begin() + (__p - cbegin());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __copy_assign_alloc(const vector& __v) {
+    __copy_assign_alloc(
+        __v, integral_constant<bool, __storage_traits::propagate_on_container_copy_assignment::value>());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __copy_assign_alloc(const vector& __c, true_type) {
+    if (__alloc() != __c.__alloc())
+      __vdeallocate();
+    __alloc() = __c.__alloc();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __copy_assign_alloc(const vector&, false_type) {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign(vector& __c, false_type);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign(vector& __c, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign_alloc(vector& __c)
+      _NOEXCEPT_(!__storage_traits::propagate_on_container_move_assignment::value ||
+                 is_nothrow_move_assignable<allocator_type>::value) {
+    __move_assign_alloc(
+        __c, integral_constant<bool, __storage_traits::propagate_on_container_move_assignment::value>());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign_alloc(vector& __c, true_type)
+      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
+    __alloc() = std::move(__c.__alloc());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign_alloc(vector&, false_type) _NOEXCEPT {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_t __hash_code() const _NOEXCEPT;
+
+  friend class __bit_reference<vector>;
+  friend class __bit_const_reference<vector>;
+  friend class __bit_iterator<vector, false>;
+  friend class __bit_iterator<vector, true>;
+  friend struct __bit_array<vector>;
+  friend struct _LIBCPP_TEMPLATE_VIS hash<vector>;
+};
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::__vdeallocate() _NOEXCEPT {
+  if (this->__begin_ != nullptr) {
+    __storage_traits::deallocate(this->__alloc(), this->__begin_, __cap());
+    this->__begin_ = nullptr;
+    this->__size_ = this->__cap() = 0;
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::size_type
+vector<bool, _Allocator>::max_size() const _NOEXCEPT {
+  size_type __amax = __storage_traits::max_size(__alloc());
+  size_type __nmax = numeric_limits<size_type>::max() / 2; // end() >= begin(), always
+  if (__nmax / __bits_per_word <= __amax)
+    return __nmax;
+  return __internal_cap_to_external(__amax);
+}
+
+//  Precondition:  __new_size > capacity()
+template <class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::size_type
+vector<bool, _Allocator>::__recommend(size_type __new_size) const {
+  const size_type __ms = max_size();
+  if (__new_size > __ms)
+    this->__throw_length_error();
+  const size_type __cap = capacity();
+  if (__cap >= __ms / 2)
+    return __ms;
+  return std::max(2 * __cap, __align_it(__new_size));
+}
+
+//  Default constructs __n objects starting at __end_
+//  Precondition:  __n > 0
+//  Precondition:  size() + __n <= capacity()
+//  Postcondition:  size() == size() + __n
+template <class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+vector<bool, _Allocator>::__construct_at_end(size_type __n, bool __x) {
+  size_type __old_size = this->__size_;
+  this->__size_ += __n;
+  if (__old_size == 0 || ((__old_size - 1) / __bits_per_word) != ((this->__size_ - 1) / __bits_per_word)) {
+    if (this->__size_ <= __bits_per_word)
+      this->__begin_[0] = __storage_type(0);
+    else
+      this->__begin_[(this->__size_ - 1) / __bits_per_word] = __storage_type(0);
+  }
+  std::fill_n(__make_iter(__old_size), __n, __x);
+}
+
+template <class _Allocator>
+template <class _InputIterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void
+vector<bool, _Allocator>::__construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n) {
+  size_type __old_size = this->__size_;
+  this->__size_ += __n;
+  if (__old_size == 0 || ((__old_size - 1) / __bits_per_word) != ((this->__size_ - 1) / __bits_per_word)) {
+    if (this->__size_ <= __bits_per_word)
+      this->__begin_[0] = __storage_type(0);
+    else
+      this->__begin_[(this->__size_ - 1) / __bits_per_word] = __storage_type(0);
+  }
+  std::__copy<_ClassicAlgPolicy>(__first, __last, __make_iter(__old_size));
+}
+
+template <class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector()
+    _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
+    : __begin_(nullptr), __size_(0), __cap_(0) {}
+
+template <class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const allocator_type& __a)
+#if _LIBCPP_STD_VER <= 14
+    _NOEXCEPT_(is_nothrow_copy_constructible<allocator_type>::value)
+#else
+        _NOEXCEPT
+#endif
+    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(size_type __n)
+    : __begin_(nullptr), __size_(0), __cap_(0) {
+  if (__n > 0) {
+    __vallocate(__n);
+    __construct_at_end(__n, false);
+  }
+}
+
+#if _LIBCPP_STD_VER >= 14
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(size_type __n, const allocator_type& __a)
+    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
+  if (__n > 0) {
+    __vallocate(__n);
+    __construct_at_end(__n, false);
+  }
+}
+#endif
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(size_type __n, const value_type& __x)
+    : __begin_(nullptr), __size_(0), __cap_(0) {
+  if (__n > 0) {
+    __vallocate(__n);
+    __construct_at_end(__n, __x);
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<bool, _Allocator>::vector(size_type __n, const value_type& __x, const allocator_type& __a)
+    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
+  if (__n > 0) {
+    __vallocate(__n);
+    __construct_at_end(__n, __x);
+  }
+}
+
+template <class _Allocator>
+template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(_InputIterator __first, _InputIterator __last)
+    : __begin_(nullptr), __size_(0), __cap_(0) {
+  __init_with_sentinel(__first, __last);
+}
+
+template <class _Allocator>
+template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<bool, _Allocator>::vector(_InputIterator __first, _InputIterator __last, const allocator_type& __a)
+    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
+  __init_with_sentinel(__first, __last);
+}
+
+template <class _Allocator>
+template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(_ForwardIterator __first, _ForwardIterator __last)
+    : __begin_(nullptr), __size_(0), __cap_(0) {
+  auto __n = static_cast<size_type>(std::distance(__first, __last));
+  __init_with_size(__first, __last, __n);
+}
+
+template <class _Allocator>
+template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<bool, _Allocator>::vector(_ForwardIterator __first, _ForwardIterator __last, const allocator_type& __a)
+    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
+  auto __n = static_cast<size_type>(std::distance(__first, __last));
+  __init_with_size(__first, __last, __n);
+}
+
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(initializer_list<value_type> __il)
+    : __begin_(nullptr), __size_(0), __cap_(0) {
+  size_type __n = static_cast<size_type>(__il.size());
+  if (__n > 0) {
+    __vallocate(__n);
+    __construct_at_end(__il.begin(), __il.end(), __n);
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<bool, _Allocator>::vector(initializer_list<value_type> __il, const allocator_type& __a)
+    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
+  size_type __n = static_cast<size_type>(__il.size());
+  if (__n > 0) {
+    __vallocate(__n);
+    __construct_at_end(__il.begin(), __il.end(), __n);
+  }
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const vector& __v)
+    : __begin_(nullptr),
+      __size_(0),
+      __cap_(0),
+      __alloc_(__storage_traits::select_on_container_copy_construction(__v.__alloc())) {
+  if (__v.size() > 0) {
+    __vallocate(__v.size());
+    __construct_at_end(__v.begin(), __v.end(), __v.size());
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const vector& __v, const allocator_type& __a)
+    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(__a) {
+  if (__v.size() > 0) {
+    __vallocate(__v.size());
+    __construct_at_end(__v.begin(), __v.end(), __v.size());
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>& vector<bool, _Allocator>::operator=(const vector& __v) {
+  if (this != std::addressof(__v)) {
+    __copy_assign_alloc(__v);
+    if (__v.__size_) {
+      if (__v.__size_ > capacity()) {
+        __vdeallocate();
+        __vallocate(__v.__size_);
+      }
+      std::copy(__v.__begin_, __v.__begin_ + __external_cap_to_internal(__v.__size_), __begin_);
+    }
+    __size_ = __v.__size_;
+  }
+  return *this;
+}
+
+template <class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(vector&& __v)
+#if _LIBCPP_STD_VER >= 17
+    _NOEXCEPT
+#else
+    _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value)
+#endif
+    : __begin_(__v.__begin_),
+      __size_(__v.__size_),
+      __cap_(__v.__cap_),
+      __alloc_(std::move(__v.__alloc_)) {
+  __v.__begin_ = nullptr;
+  __v.__size_  = 0;
+  __v.__cap()  = 0;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20
+vector<bool, _Allocator>::vector(vector&& __v, const __type_identity_t<allocator_type>& __a)
+    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(__a) {
+  if (__a == allocator_type(__v.__alloc())) {
+    this->__begin_ = __v.__begin_;
+    this->__size_  = __v.__size_;
+    this->__cap()  = __v.__cap();
+    __v.__begin_   = nullptr;
+    __v.__cap() = __v.__size_ = 0;
+  } else if (__v.size() > 0) {
+    __vallocate(__v.size());
+    __construct_at_end(__v.begin(), __v.end(), __v.size());
+  }
+}
+
+template <class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>&
+vector<bool, _Allocator>::operator=(vector&& __v)
+    _NOEXCEPT_(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value) {
+  __move_assign(__v, integral_constant<bool, __storage_traits::propagate_on_container_move_assignment::value>());
+  return *this;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::__move_assign(vector& __c, false_type) {
+  if (__alloc() != __c.__alloc())
+    assign(__c.begin(), __c.end());
+  else
+    __move_assign(__c, true_type());
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::__move_assign(vector& __c, true_type)
+    _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
+  __vdeallocate();
+  __move_assign_alloc(__c);
+  this->__begin_ = __c.__begin_;
+  this->__size_  = __c.__size_;
+  this->__cap()  = __c.__cap();
+  __c.__begin_   = nullptr;
+  __c.__cap() = __c.__size_ = 0;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::assign(size_type __n, const value_type& __x) {
+  __size_ = 0;
+  if (__n > 0) {
+    size_type __c = capacity();
+    if (__n <= __c)
+      __size_ = __n;
+    else {
+      vector __v(get_allocator());
+      __v.reserve(__recommend(__n));
+      __v.__size_ = __n;
+      swap(__v);
+    }
+    std::fill_n(begin(), __n, __x);
+  }
+}
+
+template <class _Allocator>
+template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::assign(_InputIterator __first, _InputIterator __last) {
+  __assign_with_sentinel(__first, __last);
+}
+
+template <class _Allocator>
+template <class _Iterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+vector<bool, _Allocator>::__assign_with_sentinel(_Iterator __first, _Sentinel __last) {
+  clear();
+  for (; __first != __last; ++__first)
+    push_back(*__first);
+}
+
+template <class _Allocator>
+template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::assign(_ForwardIterator __first, _ForwardIterator __last) {
+  __assign_with_size(__first, __last, std::distance(__first, __last));
+}
+
+template <class _Allocator>
+template <class _ForwardIterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+vector<bool, _Allocator>::__assign_with_size(_ForwardIterator __first, _Sentinel __last, difference_type __ns) {
+  _LIBCPP_ASSERT_VALID_INPUT_RANGE(__ns >= 0, "invalid range specified");
+
+  clear();
+
+  const size_t __n = static_cast<size_type>(__ns);
+  if (__n) {
+    if (__n > capacity()) {
+      __vdeallocate();
+      __vallocate(__n);
+    }
+    __construct_at_end(__first, __last, __n);
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::reserve(size_type __n) {
+  if (__n > capacity()) {
+    if (__n > max_size())
+      this->__throw_length_error();
+    vector __v(this->get_allocator());
+    __v.__vallocate(__n);
+    __v.__construct_at_end(this->begin(), this->end(), this->size());
+    swap(__v);
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::shrink_to_fit() _NOEXCEPT {
+  if (__external_cap_to_internal(size()) > __cap()) {
+#if _LIBCPP_HAS_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_EXCEPTIONS
+      vector(*this, allocator_type(__alloc())).swap(*this);
+#if _LIBCPP_HAS_EXCEPTIONS
+    } catch (...) {
+    }
+#endif // _LIBCPP_HAS_EXCEPTIONS
+  }
+}
+
+template <class _Allocator>
+typename vector<bool, _Allocator>::reference vector<bool, _Allocator>::at(size_type __n) {
+  if (__n >= size())
+    this->__throw_out_of_range();
+  return (*this)[__n];
+}
+
+template <class _Allocator>
+typename vector<bool, _Allocator>::const_reference vector<bool, _Allocator>::at(size_type __n) const {
+  if (__n >= size())
+    this->__throw_out_of_range();
+  return (*this)[__n];
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::push_back(const value_type& __x) {
+  if (this->__size_ == this->capacity())
+    reserve(__recommend(this->__size_ + 1));
+  ++this->__size_;
+  back() = __x;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::insert(const_iterator __position, const value_type& __x) {
+  iterator __r;
+  if (size() < capacity()) {
+    const_iterator __old_end = end();
+    ++__size_;
+    std::copy_backward(__position, __old_end, end());
+    __r = __const_iterator_cast(__position);
+  } else {
+    vector __v(get_allocator());
+    __v.reserve(__recommend(__size_ + 1));
+    __v.__size_ = __size_ + 1;
+    __r         = std::copy(cbegin(), __position, __v.begin());
+    std::copy_backward(__position, cend(), __v.end());
+    swap(__v);
+  }
+  *__r = __x;
+  return __r;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::insert(const_iterator __position, size_type __n, const value_type& __x) {
+  iterator __r;
+  size_type __c = capacity();
+  if (__n <= __c && size() <= __c - __n) {
+    const_iterator __old_end = end();
+    __size_ += __n;
+    std::copy_backward(__position, __old_end, end());
+    __r = __const_iterator_cast(__position);
+  } else {
+    vector __v(get_allocator());
+    __v.reserve(__recommend(__size_ + __n));
+    __v.__size_ = __size_ + __n;
+    __r         = std::copy(cbegin(), __position, __v.begin());
+    std::copy_backward(__position, cend(), __v.end());
+    swap(__v);
+  }
+  std::fill_n(__r, __n, __x);
+  return __r;
+}
+
+template <class _Allocator>
+template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::insert(const_iterator __position, _InputIterator __first, _InputIterator __last) {
+  return __insert_with_sentinel(__position, __first, __last);
+}
+
+template <class _Allocator>
+template <class _InputIterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::__insert_with_sentinel(const_iterator __position, _InputIterator __first, _Sentinel __last) {
+  difference_type __off = __position - begin();
+  iterator __p          = __const_iterator_cast(__position);
+  iterator __old_end    = end();
+  for (; size() != capacity() && __first != __last; ++__first) {
+    ++this->__size_;
+    back() = *__first;
+  }
+  vector __v(get_allocator());
+  if (__first != __last) {
+#if _LIBCPP_HAS_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_EXCEPTIONS
+      __v.__assign_with_sentinel(std::move(__first), std::move(__last));
+      difference_type __old_size = static_cast<difference_type>(__old_end - begin());
+      difference_type __old_p    = __p - begin();
+      reserve(__recommend(size() + __v.size()));
+      __p       = begin() + __old_p;
+      __old_end = begin() + __old_size;
+#if _LIBCPP_HAS_EXCEPTIONS
+    } catch (...) {
+      erase(__old_end, end());
+      throw;
+    }
+#endif // _LIBCPP_HAS_EXCEPTIONS
+  }
+  __p = std::rotate(__p, __old_end, end());
+  insert(__p, __v.begin(), __v.end());
+  return begin() + __off;
+}
+
+template <class _Allocator>
+template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::insert(const_iterator __position, _ForwardIterator __first, _ForwardIterator __last) {
+  return __insert_with_size(__position, __first, __last, std::distance(__first, __last));
+}
+
+template <class _Allocator>
+template <class _ForwardIterator, class _Sentinel>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::__insert_with_size(
+    const_iterator __position, _ForwardIterator __first, _Sentinel __last, difference_type __n_signed) {
+  _LIBCPP_ASSERT_VALID_INPUT_RANGE(__n_signed >= 0, "invalid range specified");
+  const size_type __n = static_cast<size_type>(__n_signed);
+  iterator __r;
+  size_type __c = capacity();
+  if (__n <= __c && size() <= __c - __n) {
+    const_iterator __old_end = end();
+    __size_ += __n;
+    std::copy_backward(__position, __old_end, end());
+    __r = __const_iterator_cast(__position);
+  } else {
+    vector __v(get_allocator());
+    __v.reserve(__recommend(__size_ + __n));
+    __v.__size_ = __size_ + __n;
+    __r         = std::copy(cbegin(), __position, __v.begin());
+    std::copy_backward(__position, cend(), __v.end());
+    swap(__v);
+  }
+  std::__copy<_ClassicAlgPolicy>(__first, __last, __r);
+  return __r;
+}
+
+template <class _Allocator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::erase(const_iterator __position) {
+  iterator __r = __const_iterator_cast(__position);
+  std::copy(__position + 1, this->cend(), __r);
+  --__size_;
+  return __r;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
+vector<bool, _Allocator>::erase(const_iterator __first, const_iterator __last) {
+  iterator __r        = __const_iterator_cast(__first);
+  difference_type __d = __last - __first;
+  std::copy(__last, this->cend(), __r);
+  __size_ -= __d;
+  return __r;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::swap(vector& __x)
+#if _LIBCPP_STD_VER >= 14
+    _NOEXCEPT
+#else
+    _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>)
+#endif
+{
+  std::swap(this->__begin_, __x.__begin_);
+  std::swap(this->__size_, __x.__size_);
+  std::swap(this->__cap(), __x.__cap());
+  std::__swap_allocator(this->__alloc(), __x.__alloc());
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::resize(size_type __sz, value_type __x) {
+  size_type __cs = size();
+  if (__cs < __sz) {
+    iterator __r;
+    size_type __c = capacity();
+    size_type __n = __sz - __cs;
+    if (__n <= __c && __cs <= __c - __n) {
+      __r = end();
+      __size_ += __n;
+    } else {
+      vector __v(get_allocator());
+      __v.reserve(__recommend(__size_ + __n));
+      __v.__size_ = __size_ + __n;
+      __r         = std::copy(cbegin(), cend(), __v.begin());
+      swap(__v);
+    }
+    std::fill_n(__r, __n, __x);
+  } else
+    __size_ = __sz;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::flip() _NOEXCEPT {
+  // do middle whole words
+  size_type __n         = __size_;
+  __storage_pointer __p = __begin_;
+  for (; __n >= __bits_per_word; ++__p, __n -= __bits_per_word)
+    *__p = ~*__p;
+  // do last partial word
+  if (__n > 0) {
+    __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+    __storage_type __b = *__p & __m;
+    *__p &= ~__m;
+    *__p |= ~__b & __m;
+  }
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 bool vector<bool, _Allocator>::__invariants() const {
+  if (this->__begin_ == nullptr) {
+    if (this->__size_ != 0 || this->__cap() != 0)
+      return false;
+  } else {
+    if (this->__cap() == 0)
+      return false;
+    if (this->__size_ > this->capacity())
+      return false;
+  }
+  return true;
+}
+
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 size_t vector<bool, _Allocator>::__hash_code() const _NOEXCEPT {
+  size_t __h = 0;
+  // do middle whole words
+  size_type __n         = __size_;
+  __storage_pointer __p = __begin_;
+  for (; __n >= __bits_per_word; ++__p, __n -= __bits_per_word)
+    __h ^= *__p;
+  // do last partial word
+  if (__n > 0) {
+    const __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+    __h ^= *__p & __m;
+  }
+  return __h;
+}
+
+template <class _Allocator>
+struct _LIBCPP_TEMPLATE_VIS hash<vector<bool, _Allocator> >
+    : public __unary_function<vector<bool, _Allocator>, size_t> {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_t
+  operator()(const vector<bool, _Allocator>& __vec) const _NOEXCEPT {
+    return __vec.__hash_code();
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___VECTOR_VECTOR_BOOL_H
diff --git a/libcxx/include/__vector/vector_bool_formatter.h b/libcxx/include/__vector/vector_bool_formatter.h
new file mode 100644
index 000000000000..017f9405e329
--- /dev/null
+++ b/libcxx/include/__vector/vector_bool_formatter.h
@@ -0,0 +1,49 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___VECTOR_VECTOR_BOOL_FORMATTER_H
+#define _LIBCPP___VECTOR_VECTOR_BOOL_FORMATTER_H
+
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__format/formatter.h>
+#include <__format/formatter_bool.h>
+#include <__fwd/vector.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp, class _CharT>
+// Since is-vector-bool-reference is only used once it's inlined here.
+  requires same_as<typename _Tp::__container, vector<bool, typename _Tp::__container::allocator_type>>
+struct _LIBCPP_TEMPLATE_VIS formatter<_Tp, _CharT> {
+private:
+  formatter<bool, _CharT> __underlying_;
+
+public:
+  template <class _ParseContext>
+  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
+    return __underlying_.parse(__ctx);
+  }
+
+  template <class _FormatContext>
+  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(const _Tp& __ref, _FormatContext& __ctx) const {
+    return __underlying_.format(__ref, __ctx);
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+#endif // _LIBCPP___VECTOR_VECTOR_BOOL_FORMATTER_H
diff --git a/libcxx/include/chrono b/libcxx/include/chrono
index 7bec5e5a26ef..0f4223270c9d 100644
--- a/libcxx/include/chrono
+++ b/libcxx/include/chrono
@@ -1009,6 +1009,7 @@ constexpr chrono::year                                  operator ""y(unsigned lo
 #  include <forward_list>
 #  include <string>
 #  include <tuple>
+#  include <vector>
 #endif
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER == 20
diff --git a/libcxx/include/functional b/libcxx/include/functional
index 489d82b2d43a..55b50eea84b7 100644
--- a/libcxx/include/functional
+++ b/libcxx/include/functional
@@ -578,7 +578,6 @@ POLICY:  For non-variadic implementations, the number of arguments is limited
 #  include <array>
 #  include <initializer_list>
 #  include <unordered_map>
-#  include <vector>
 #endif
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
@@ -593,6 +592,7 @@ POLICY:  For non-variadic implementations, the number of arguments is limited
 #  include <type_traits>
 #  include <typeinfo>
 #  include <utility>
+#  include <vector>
 #endif
 
 #endif // _LIBCPP_FUNCTIONAL
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index d775da489e35..b429d7cff702 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1523,6 +1523,7 @@ module std [system] {
     }
     module uninitialized_algorithms {
       header "__memory/uninitialized_algorithms.h"
+      export std.utility.pair
     }
     module unique_ptr {
       header "__memory/unique_ptr.h"
@@ -2006,7 +2007,39 @@ module std [system] {
   }
 
   module vector {
-    module fwd { header "__fwd/vector.h" }
+    module fwd                   { header "__fwd/vector.h" }
+
+    module comparison            { header "__vector/comparison.h" }
+    module container_traits      { header "__vector/container_traits.h" }
+    module erase                 { header "__vector/erase.h" }
+    module vector_bool_formatter {
+      header "__vector/vector_bool_formatter.h"
+
+      export std.format.formatter
+    }
+    module pmr                   {
+      header "__vector/pmr.h"
+
+      export std.memory_resource.polymorphic_allocator
+    }
+    module swap                  { header "__vector/swap.h" }
+    module vector_bool           {
+      header "__vector/vector_bool.h"
+      export std.bit_reference
+      export std.memory.allocator
+      export std.vector.comparison
+      export std.vector.fwd
+      export std.vector.swap
+    }
+    module vector                {
+      header "__vector/vector.h"
+      export std.iterator.bounded_iter
+      export std.iterator.wrap_iter
+      export std.memory.allocator
+      export std.vector.comparison
+      export std.vector.fwd
+      export std.vector.swap
+    }
 
     header "vector"
     export *
diff --git a/libcxx/include/vector b/libcxx/include/vector
index ce412a829ea8..bd874811714a 100644
--- a/libcxx/include/vector
+++ b/libcxx/include/vector
@@ -304,78 +304,25 @@ template<class T, class charT> requires is-vector-bool-reference<T> // Since C++
 
 // clang-format on
 
-#include <__algorithm/copy.h>
-#include <__algorithm/equal.h>
-#include <__algorithm/fill_n.h>
-#include <__algorithm/iterator_operations.h>
-#include <__algorithm/lexicographical_compare.h>
-#include <__algorithm/lexicographical_compare_three_way.h>
-#include <__algorithm/max.h>
-#include <__algorithm/min.h>
-#include <__algorithm/move.h>
-#include <__algorithm/move_backward.h>
-#include <__algorithm/remove.h>
-#include <__algorithm/remove_if.h>
-#include <__algorithm/rotate.h>
-#include <__algorithm/unwrap_iter.h>
-#include <__assert>
-#include <__bit_reference>
-#include <__concepts/same_as.h>
 #include <__config>
-#include <__debug_utils/sanitizers.h>
-#include <__format/enable_insertable.h>
-#include <__format/formatter.h>
-#include <__format/formatter_bool.h>
-#include <__functional/hash.h>
-#include <__functional/unary_function.h>
-#include <__fwd/vector.h>
-#include <__iterator/advance.h>
-#include <__iterator/bounded_iter.h>
-#include <__iterator/distance.h>
-#include <__iterator/iterator_traits.h>
-#include <__iterator/move_iterator.h>
-#include <__iterator/next.h>
-#include <__iterator/reverse_iterator.h>
-#include <__iterator/wrap_iter.h>
-#include <__memory/addressof.h>
-#include <__memory/allocate_at_least.h>
-#include <__memory/allocator.h>
-#include <__memory/allocator_traits.h>
-#include <__memory/compressed_pair.h>
-#include <__memory/construct_at.h>
-#include <__memory/noexcept_move_assign_container.h>
-#include <__memory/pointer_traits.h>
-#include <__memory/swap_allocator.h>
-#include <__memory/temp_value.h>
-#include <__memory/uninitialized_algorithms.h>
-#include <__memory_resource/polymorphic_allocator.h>
-#include <__ranges/access.h>
-#include <__ranges/concepts.h>
-#include <__ranges/container_compatible_range.h>
-#include <__ranges/from_range.h>
-#include <__ranges/size.h>
-#include <__split_buffer>
-#include <__type_traits/conditional.h>
-#include <__type_traits/container_traits.h>
-#include <__type_traits/disjunction.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/is_allocator.h>
-#include <__type_traits/is_constructible.h>
-#include <__type_traits/is_nothrow_assignable.h>
-#include <__type_traits/is_nothrow_constructible.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/is_trivially_relocatable.h>
-#include <__type_traits/type_identity.h>
-#include <__utility/exception_guard.h>
-#include <__utility/forward.h>
-#include <__utility/is_pointer_in_range.h>
-#include <__utility/move.h>
-#include <__utility/pair.h>
-#include <__utility/swap.h>
-#include <climits>
-#include <cstring>
-#include <limits>
-#include <stdexcept>
+
+#include <__vector/comparison.h>
+#include <__vector/swap.h>
+#include <__vector/vector.h>
+#include <__vector/vector_bool.h>
+
+#if _LIBCPP_STD_VER >= 17
+#  include <__vector/pmr.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__vector/erase.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+#  include <__vector/vector_bool_formatter.h>
+#endif
+
 #include <version>
 
 // standard-mandated includes
@@ -391,2605 +338,29 @@ template<class T, class charT> requires is-vector-bool-reference<T> // Since C++
 #include <compare>
 #include <initializer_list>
 
+// [vector.syn], [unord.hash]
+#include <__functional/hash.h>
+
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
-_LIBCPP_PUSH_MACROS
-#include <__undef_macros>
-
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-template <class _Tp, class _Allocator /* = allocator<_Tp> */>
-class _LIBCPP_TEMPLATE_VIS vector {
-private:
-  typedef allocator<_Tp> __default_allocator_type;
-
-public:
-  typedef vector __self;
-  typedef _Tp value_type;
-  typedef _Allocator allocator_type;
-  typedef allocator_traits<allocator_type> __alloc_traits;
-  typedef value_type& reference;
-  typedef const value_type& const_reference;
-  typedef typename __alloc_traits::size_type size_type;
-  typedef typename __alloc_traits::difference_type difference_type;
-  typedef typename __alloc_traits::pointer pointer;
-  typedef typename __alloc_traits::const_pointer const_pointer;
-#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
-  // Users might provide custom allocators, and prior to C++20 we have no existing way to detect whether the allocator's
-  // pointer type is contiguous (though it has to be by the Standard). Using the wrapper type ensures the iterator is
-  // considered contiguous.
-  typedef __bounded_iter<__wrap_iter<pointer> > iterator;
-  typedef __bounded_iter<__wrap_iter<const_pointer> > const_iterator;
-#else
-  typedef __wrap_iter<pointer> iterator;
-  typedef __wrap_iter<const_pointer> const_iterator;
-#endif
-  typedef std::reverse_iterator<iterator> reverse_iterator;
-  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
-
-  // A vector containers the following members which may be trivially relocatable:
-  // - pointer: may be trivially relocatable, so it's checked
-  // - allocator_type: may be trivially relocatable, so it's checked
-  // vector doesn't contain any self-references, so it's trivially relocatable if its members are.
-  using __trivially_relocatable = __conditional_t<
-      __libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<allocator_type>::value,
-      vector,
-      void>;
-
-  static_assert(__check_valid_allocator<allocator_type>::value, "");
-  static_assert(is_same<typename allocator_type::value_type, value_type>::value,
-                "Allocator::value_type must be same type as value_type");
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector()
-      _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value) {}
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit vector(const allocator_type& __a)
-#if _LIBCPP_STD_VER <= 14
-      _NOEXCEPT_(is_nothrow_copy_constructible<allocator_type>::value)
-#else
-      noexcept
-#endif
-      : __alloc_(__a) {
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit vector(size_type __n) {
-    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
-    if (__n > 0) {
-      __vallocate(__n);
-      __construct_at_end(__n);
-    }
-    __guard.__complete();
-  }
-
-#if _LIBCPP_STD_VER >= 14
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit vector(size_type __n, const allocator_type& __a)
-      : __alloc_(__a) {
-    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
-    if (__n > 0) {
-      __vallocate(__n);
-      __construct_at_end(__n);
-    }
-    __guard.__complete();
-  }
-#endif
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(size_type __n, const value_type& __x) {
-    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
-    if (__n > 0) {
-      __vallocate(__n);
-      __construct_at_end(__n, __x);
-    }
-    __guard.__complete();
-  }
-
-  template <__enable_if_t<__is_allocator<_Allocator>::value, int> = 0>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
-  vector(size_type __n, const value_type& __x, const allocator_type& __a)
-      : __alloc_(__a) {
-    if (__n > 0) {
-      __vallocate(__n);
-      __construct_at_end(__n, __x);
-    }
-  }
-
-  template <class _InputIterator,
-            __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
-                              is_constructible<value_type, typename iterator_traits<_InputIterator>::reference>::value,
-                          int> = 0>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(_InputIterator __first, _InputIterator __last);
-  template <class _InputIterator,
-            __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
-                              is_constructible<value_type, typename iterator_traits<_InputIterator>::reference>::value,
-                          int> = 0>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
-  vector(_InputIterator __first, _InputIterator __last, const allocator_type& __a);
-
-  template <
-      class _ForwardIterator,
-      __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
-                        is_constructible<value_type, typename iterator_traits<_ForwardIterator>::reference>::value,
-                    int> = 0>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(_ForwardIterator __first, _ForwardIterator __last);
-
-  template <
-      class _ForwardIterator,
-      __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
-                        is_constructible<value_type, typename iterator_traits<_ForwardIterator>::reference>::value,
-                    int> = 0>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
-  vector(_ForwardIterator __first, _ForwardIterator __last, const allocator_type& __a);
-
-#if _LIBCPP_STD_VER >= 23
-  template <_ContainerCompatibleRange<_Tp> _Range>
-  _LIBCPP_HIDE_FROM_ABI constexpr vector(
-      from_range_t, _Range&& __range, const allocator_type& __alloc = allocator_type())
-      : __alloc_(__alloc) {
-    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
-      auto __n = static_cast<size_type>(ranges::distance(__range));
-      __init_with_size(ranges::begin(__range), ranges::end(__range), __n);
-
-    } else {
-      __init_with_sentinel(ranges::begin(__range), ranges::end(__range));
-    }
-  }
-#endif
-
-private:
-  class __destroy_vector {
-  public:
-    _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI __destroy_vector(vector& __vec) : __vec_(__vec) {}
-
-    _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void operator()() {
-      if (__vec_.__begin_ != nullptr) {
-        __vec_.__clear();
-        __vec_.__annotate_delete();
-        __alloc_traits::deallocate(__vec_.__alloc(), __vec_.__begin_, __vec_.capacity());
-      }
-    }
-
-  private:
-    vector& __vec_;
-  };
-
-public:
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~vector() { __destroy_vector (*this)(); }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(const vector& __x);
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
-  vector(const vector& __x, const __type_identity_t<allocator_type>& __a);
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector& operator=(const vector& __x);
-
-#ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(initializer_list<value_type> __il);
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
-  vector(initializer_list<value_type> __il, const allocator_type& __a);
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector& operator=(initializer_list<value_type> __il) {
-    assign(__il.begin(), __il.end());
-    return *this;
-  }
-#endif // !_LIBCPP_CXX03_LANG
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(vector&& __x)
-#if _LIBCPP_STD_VER >= 17
-      noexcept;
-#else
-      _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value);
-#endif
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
-  vector(vector&& __x, const __type_identity_t<allocator_type>& __a);
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector& operator=(vector&& __x)
-      _NOEXCEPT_(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value);
-
-  template <class _InputIterator,
-            __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
-                              is_constructible<value_type, typename iterator_traits<_InputIterator>::reference>::value,
-                          int> = 0>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void assign(_InputIterator __first, _InputIterator __last);
-  template <
-      class _ForwardIterator,
-      __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
-                        is_constructible<value_type, typename iterator_traits<_ForwardIterator>::reference>::value,
-                    int> = 0>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void assign(_ForwardIterator __first, _ForwardIterator __last);
-
-#if _LIBCPP_STD_VER >= 23
-  template <_ContainerCompatibleRange<_Tp> _Range>
-  _LIBCPP_HIDE_FROM_ABI constexpr void assign_range(_Range&& __range) {
-    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
-      auto __n = static_cast<size_type>(ranges::distance(__range));
-      __assign_with_size(ranges::begin(__range), ranges::end(__range), __n);
-
-    } else {
-      __assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
-    }
-  }
-#endif
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void assign(size_type __n, const_reference __u);
-
-#ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void assign(initializer_list<value_type> __il) {
-    assign(__il.begin(), __il.end());
-  }
-#endif
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
-    return this->__alloc();
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT;
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT;
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT;
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT;
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT {
-    return reverse_iterator(end());
-  }
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT {
-    return const_reverse_iterator(end());
-  }
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT {
-    return reverse_iterator(begin());
-  }
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
-    return const_reverse_iterator(begin());
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT {
-    return rbegin();
-  }
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT {
-    return static_cast<size_type>(this->__end_ - this->__begin_);
-  }
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const _NOEXCEPT {
-    return static_cast<size_type>(__end_cap() - this->__begin_);
-  }
-  [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT {
-    return this->__begin_ == this->__end_;
-  }
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT;
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void reserve(size_type __n);
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void shrink_to_fit() _NOEXCEPT;
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference operator[](size_type __n) _NOEXCEPT;
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference operator[](size_type __n) const _NOEXCEPT;
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference at(size_type __n);
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference at(size_type __n) const;
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference front() _NOEXCEPT {
-    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "front() called on an empty vector");
-    return *this->__begin_;
-  }
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference front() const _NOEXCEPT {
-    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "front() called on an empty vector");
-    return *this->__begin_;
-  }
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() _NOEXCEPT {
-    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "back() called on an empty vector");
-    return *(this->__end_ - 1);
-  }
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT {
-    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "back() called on an empty vector");
-    return *(this->__end_ - 1);
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI value_type* data() _NOEXCEPT {
-    return std::__to_address(this->__begin_);
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const value_type* data() const _NOEXCEPT {
-    return std::__to_address(this->__begin_);
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_back(const_reference __x) { emplace_back(__x); }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __x) { emplace_back(std::move(__x)); }
-
-  template <class... _Args>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
-#if _LIBCPP_STD_VER >= 17
-  reference
-  emplace_back(_Args&&... __args);
-#else
-  void
-  emplace_back(_Args&&... __args);
-#endif
-
-#if _LIBCPP_STD_VER >= 23
-  template <_ContainerCompatibleRange<_Tp> _Range>
-  _LIBCPP_HIDE_FROM_ABI constexpr void append_range(_Range&& __range) {
-    insert_range(end(), std::forward<_Range>(__range));
-  }
-#endif
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_back();
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, const_reference __x);
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, value_type&& __x);
-  template <class... _Args>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __position, _Args&&... __args);
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
-  insert(const_iterator __position, size_type __n, const_reference __x);
-
-  template <class _InputIterator,
-            __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
-                              is_constructible< value_type, typename iterator_traits<_InputIterator>::reference>::value,
-                          int> = 0>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
-  insert(const_iterator __position, _InputIterator __first, _InputIterator __last);
-
-#if _LIBCPP_STD_VER >= 23
-  template <_ContainerCompatibleRange<_Tp> _Range>
-  _LIBCPP_HIDE_FROM_ABI constexpr iterator insert_range(const_iterator __position, _Range&& __range) {
-    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
-      auto __n = static_cast<size_type>(ranges::distance(__range));
-      return __insert_with_size(__position, ranges::begin(__range), ranges::end(__range), __n);
-
-    } else {
-      return __insert_with_sentinel(__position, ranges::begin(__range), ranges::end(__range));
-    }
-  }
-#endif
-
-  template <
-      class _ForwardIterator,
-      __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
-                        is_constructible< value_type, typename iterator_traits<_ForwardIterator>::reference>::value,
-                    int> = 0>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
-  insert(const_iterator __position, _ForwardIterator __first, _ForwardIterator __last);
-
-#ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
-  insert(const_iterator __position, initializer_list<value_type> __il) {
-    return insert(__position, __il.begin(), __il.end());
-  }
-#endif
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __position);
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last);
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT {
-    size_type __old_size = size();
-    __clear();
-    __annotate_shrink(__old_size);
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void resize(size_type __sz);
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void resize(size_type __sz, const_reference __x);
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void swap(vector&)
-#if _LIBCPP_STD_VER >= 14
-      _NOEXCEPT;
-#else
-      _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>);
-#endif
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __invariants() const;
-
-private:
-  pointer __begin_ = nullptr;
-  pointer __end_   = nullptr;
-  _LIBCPP_COMPRESSED_PAIR(pointer, __cap_ = nullptr, allocator_type, __alloc_);
-
-  //  Allocate space for __n objects
-  //  throws length_error if __n > max_size()
-  //  throws (probably bad_alloc) if memory run out
-  //  Precondition:  __begin_ == __end_ == __end_cap() == 0
-  //  Precondition:  __n > 0
-  //  Postcondition:  capacity() >= __n
-  //  Postcondition:  size() == 0
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __vallocate(size_type __n) {
-    if (__n > max_size())
-      __throw_length_error();
-    auto __allocation = std::__allocate_at_least(__alloc(), __n);
-    __begin_          = __allocation.ptr;
-    __end_            = __allocation.ptr;
-    __end_cap()       = __begin_ + __allocation.count;
-    __annotate_new(0);
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __vdeallocate() _NOEXCEPT;
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __recommend(size_type __new_size) const;
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_at_end(size_type __n);
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_at_end(size_type __n, const_reference __x);
-
-  template <class _InputIterator, class _Sentinel>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
-  __init_with_size(_InputIterator __first, _Sentinel __last, size_type __n) {
-    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
-
-    if (__n > 0) {
-      __vallocate(__n);
-      __construct_at_end(__first, __last, __n);
-    }
-
-    __guard.__complete();
-  }
-
-  template <class _InputIterator, class _Sentinel>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
-  __init_with_sentinel(_InputIterator __first, _Sentinel __last) {
-    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
-
-    for (; __first != __last; ++__first)
-      emplace_back(*__first);
-
-    __guard.__complete();
-  }
-
-  template <class _Iterator, class _Sentinel>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __first, _Sentinel __last);
-
-  template <class _ForwardIterator, class _Sentinel>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
-  __assign_with_size(_ForwardIterator __first, _Sentinel __last, difference_type __n);
-
-  template <class _InputIterator, class _Sentinel>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
-  __insert_with_sentinel(const_iterator __position, _InputIterator __first, _Sentinel __last);
-
-  template <class _Iterator, class _Sentinel>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
-  __insert_with_size(const_iterator __position, _Iterator __first, _Sentinel __last, difference_type __n);
-
-  template <class _InputIterator, class _Sentinel>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
-  __construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n);
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __append(size_type __n);
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __append(size_type __n, const_reference __x);
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator __make_iter(pointer __p) _NOEXCEPT {
-#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
-    // Bound the iterator according to the capacity, rather than the size.
-    //
-    // Vector guarantees that iterators stay valid as long as no reallocation occurs even if new elements are inserted
-    // into the container; for these cases, we need to make sure that the newly-inserted elements can be accessed
-    // through the bounded iterator without failing checks. The downside is that the bounded iterator won't catch
-    // access that is logically out-of-bounds, i.e., goes beyond the size, but is still within the capacity. With the
-    // current implementation, there is no connection between a bounded iterator and its associated container, so we
-    // don't have a way to update existing valid iterators when the container is resized and thus have to go with
-    // a laxer approach.
-    return std::__make_bounded_iter(
-        std::__wrap_iter<pointer>(__p),
-        std::__wrap_iter<pointer>(this->__begin_),
-        std::__wrap_iter<pointer>(this->__end_cap()));
-#else
-    return iterator(__p);
-#endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator __make_iter(const_pointer __p) const _NOEXCEPT {
-#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
-    // Bound the iterator according to the capacity, rather than the size.
-    return std::__make_bounded_iter(
-        std::__wrap_iter<const_pointer>(__p),
-        std::__wrap_iter<const_pointer>(this->__begin_),
-        std::__wrap_iter<const_pointer>(this->__end_cap()));
-#else
-    return const_iterator(__p);
-#endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
-  __swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v);
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer
-  __swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v, pointer __p);
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
-  __move_range(pointer __from_s, pointer __from_e, pointer __to);
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign(vector& __c, true_type)
-      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value);
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign(vector& __c, false_type)
-      _NOEXCEPT_(__alloc_traits::is_always_equal::value);
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __destruct_at_end(pointer __new_last) _NOEXCEPT {
-    size_type __old_size = size();
-    __base_destruct_at_end(__new_last);
-    __annotate_shrink(__old_size);
-  }
-
-  template <class... _Args>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI inline pointer __emplace_back_slow_path(_Args&&... __args);
-
-  // The following functions are no-ops outside of AddressSanitizer mode.
-  // We call annotations for every allocator, unless explicitly disabled.
-  //
-  // To disable annotations for a particular allocator, change value of
-  // __asan_annotate_container_with_allocator to false.
-  // For more details, see the "Using libc++" documentation page or
-  // the documentation for __sanitizer_annotate_contiguous_container.
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
-  __annotate_contiguous_container(const void* __old_mid, const void* __new_mid) const {
-    std::__annotate_contiguous_container<_Allocator>(data(), data() + capacity(), __old_mid, __new_mid);
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_new(size_type __current_size) const _NOEXCEPT {
-    (void)__current_size;
-#if _LIBCPP_HAS_ASAN
-    __annotate_contiguous_container(data() + capacity(), data() + __current_size);
-#endif
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_delete() const _NOEXCEPT {
-#if _LIBCPP_HAS_ASAN
-    __annotate_contiguous_container(data() + size(), data() + capacity());
-#endif
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_increase(size_type __n) const _NOEXCEPT {
-    (void)__n;
-#if _LIBCPP_HAS_ASAN
-    __annotate_contiguous_container(data() + size(), data() + size() + __n);
-#endif
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_shrink(size_type __old_size) const _NOEXCEPT {
-    (void)__old_size;
-#if _LIBCPP_HAS_ASAN
-    __annotate_contiguous_container(data() + __old_size, data() + size());
-#endif
-  }
-
-  struct _ConstructTransaction {
-    _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit _ConstructTransaction(vector& __v, size_type __n)
-        : __v_(__v), __pos_(__v.__end_), __new_end_(__v.__end_ + __n) {
-#if _LIBCPP_HAS_ASAN
-      __v_.__annotate_increase(__n);
-#endif
-    }
-
-    _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~_ConstructTransaction() {
-      __v_.__end_ = __pos_;
-#if _LIBCPP_HAS_ASAN
-      if (__pos_ != __new_end_) {
-        __v_.__annotate_shrink(__new_end_ - __v_.__begin_);
-      }
-#endif
-    }
-
-    vector& __v_;
-    pointer __pos_;
-    const_pointer const __new_end_;
-
-    _ConstructTransaction(_ConstructTransaction const&)            = delete;
-    _ConstructTransaction& operator=(_ConstructTransaction const&) = delete;
-  };
-
-  template <class... _Args>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_one_at_end(_Args&&... __args) {
-    _ConstructTransaction __tx(*this, 1);
-    __alloc_traits::construct(this->__alloc(), std::__to_address(__tx.__pos_), std::forward<_Args>(__args)...);
-    ++__tx.__pos_;
-  }
-
-  // TODO: Remove these now redundant accessors
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type& __alloc() _NOEXCEPT { return this->__alloc_; }
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const allocator_type& __alloc() const _NOEXCEPT {
-    return this->__alloc_;
-  }
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer& __end_cap() _NOEXCEPT { return this->__cap_; }
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const pointer& __end_cap() const _NOEXCEPT {
-    return this->__cap_;
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __clear() _NOEXCEPT {
-    __base_destruct_at_end(this->__begin_);
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __base_destruct_at_end(pointer __new_last) _NOEXCEPT {
-    pointer __soon_to_be_end = this->__end_;
-    while (__new_last != __soon_to_be_end)
-      __alloc_traits::destroy(__alloc(), std::__to_address(--__soon_to_be_end));
-    this->__end_ = __new_last;
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const vector& __c) {
-    __copy_assign_alloc(__c, integral_constant<bool, __alloc_traits::propagate_on_container_copy_assignment::value>());
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(vector& __c)
-      _NOEXCEPT_(!__alloc_traits::propagate_on_container_move_assignment::value ||
-                 is_nothrow_move_assignable<allocator_type>::value) {
-    __move_assign_alloc(__c, integral_constant<bool, __alloc_traits::propagate_on_container_move_assignment::value>());
-  }
-
-  [[__noreturn__]] _LIBCPP_HIDE_FROM_ABI void __throw_length_error() const { std::__throw_length_error("vector"); }
-
-  [[__noreturn__]] _LIBCPP_HIDE_FROM_ABI void __throw_out_of_range() const { std::__throw_out_of_range("vector"); }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const vector& __c, true_type) {
-    if (__alloc() != __c.__alloc()) {
-      __clear();
-      __annotate_delete();
-      __alloc_traits::deallocate(__alloc(), this->__begin_, capacity());
-      this->__begin_ = this->__end_ = __end_cap() = nullptr;
-    }
-    __alloc() = __c.__alloc();
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const vector&, false_type) {}
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(vector& __c, true_type)
-      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
-    __alloc() = std::move(__c.__alloc());
-  }
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(vector&, false_type) _NOEXCEPT {}
-};
-
-#if _LIBCPP_STD_VER >= 17
-template <class _InputIterator,
-          class _Alloc = allocator<__iter_value_type<_InputIterator>>,
-          class        = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
-          class        = enable_if_t<__is_allocator<_Alloc>::value> >
-vector(_InputIterator, _InputIterator) -> vector<__iter_value_type<_InputIterator>, _Alloc>;
-
-template <class _InputIterator,
-          class _Alloc,
-          class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
-          class = enable_if_t<__is_allocator<_Alloc>::value> >
-vector(_InputIterator, _InputIterator, _Alloc) -> vector<__iter_value_type<_InputIterator>, _Alloc>;
-#endif
-
-#if _LIBCPP_STD_VER >= 23
-template <ranges::input_range _Range,
-          class _Alloc = allocator<ranges::range_value_t<_Range>>,
-          class        = enable_if_t<__is_allocator<_Alloc>::value> >
-vector(from_range_t, _Range&&, _Alloc = _Alloc()) -> vector<ranges::range_value_t<_Range>, _Alloc>;
-#endif
-
-// __swap_out_circular_buffer relocates the objects in [__begin_, __end_) into the front of __v and swaps the buffers of
-// *this and __v. It is assumed that __v provides space for exactly (__end_ - __begin_) objects in the front. This
-// function has a strong exception guarantee.
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void
-vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v) {
-  __annotate_delete();
-  auto __new_begin = __v.__begin_ - (__end_ - __begin_);
-  std::__uninitialized_allocator_relocate(
-      __alloc(), std::__to_address(__begin_), std::__to_address(__end_), std::__to_address(__new_begin));
-  __v.__begin_ = __new_begin;
-  __end_       = __begin_; // All the objects have been destroyed by relocating them.
-  std::swap(this->__begin_, __v.__begin_);
-  std::swap(this->__end_, __v.__end_);
-  std::swap(this->__end_cap(), __v.__end_cap());
-  __v.__first_ = __v.__begin_;
-  __annotate_new(size());
-}
-
-// __swap_out_circular_buffer relocates the objects in [__begin_, __p) into the front of __v, the objects in
-// [__p, __end_) into the back of __v and swaps the buffers of *this and __v. It is assumed that __v provides space for
-// exactly (__p - __begin_) objects in the front and space for at least (__end_ - __p) objects in the back. This
-// function has a strong exception guarantee if __begin_ == __p || __end_ == __p.
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::pointer
-vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v, pointer __p) {
-  __annotate_delete();
-  pointer __ret = __v.__begin_;
-
-  // Relocate [__p, __end_) first to avoid having a hole in [__begin_, __end_)
-  // in case something in [__begin_, __p) throws.
-  std::__uninitialized_allocator_relocate(
-      __alloc(), std::__to_address(__p), std::__to_address(__end_), std::__to_address(__v.__end_));
-  __v.__end_ += (__end_ - __p);
-  __end_           = __p; // The objects in [__p, __end_) have been destroyed by relocating them.
-  auto __new_begin = __v.__begin_ - (__p - __begin_);
-
-  std::__uninitialized_allocator_relocate(
-      __alloc(), std::__to_address(__begin_), std::__to_address(__p), std::__to_address(__new_begin));
-  __v.__begin_ = __new_begin;
-  __end_       = __begin_; // All the objects have been destroyed by relocating them.
-
-  std::swap(this->__begin_, __v.__begin_);
-  std::swap(this->__end_, __v.__end_);
-  std::swap(this->__end_cap(), __v.__end_cap());
-  __v.__first_ = __v.__begin_;
-  __annotate_new(size());
-  return __ret;
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__vdeallocate() _NOEXCEPT {
-  if (this->__begin_ != nullptr) {
-    clear();
-    __annotate_delete();
-    __alloc_traits::deallocate(this->__alloc(), this->__begin_, capacity());
-    this->__begin_ = this->__end_ = this->__end_cap() = nullptr;
-  }
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::size_type
-vector<_Tp, _Allocator>::max_size() const _NOEXCEPT {
-  return std::min<size_type>(__alloc_traits::max_size(this->__alloc()), numeric_limits<difference_type>::max());
-}
-
-//  Precondition:  __new_size > capacity()
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::size_type
-vector<_Tp, _Allocator>::__recommend(size_type __new_size) const {
-  const size_type __ms = max_size();
-  if (__new_size > __ms)
-    this->__throw_length_error();
-  const size_type __cap = capacity();
-  if (__cap >= __ms / 2)
-    return __ms;
-  return std::max<size_type>(2 * __cap, __new_size);
-}
-
-//  Default constructs __n objects starting at __end_
-//  throws if construction throws
-//  Precondition:  __n > 0
-//  Precondition:  size() + __n <= capacity()
-//  Postcondition:  size() == size() + __n
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__construct_at_end(size_type __n) {
-  _ConstructTransaction __tx(*this, __n);
-  const_pointer __new_end = __tx.__new_end_;
-  for (pointer __pos = __tx.__pos_; __pos != __new_end; __tx.__pos_ = ++__pos) {
-    __alloc_traits::construct(this->__alloc(), std::__to_address(__pos));
-  }
-}
-
-//  Copy constructs __n objects starting at __end_ from __x
-//  throws if construction throws
-//  Precondition:  __n > 0
-//  Precondition:  size() + __n <= capacity()
-//  Postcondition:  size() == old size() + __n
-//  Postcondition:  [i] == __x for all i in [size() - __n, __n)
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void
-vector<_Tp, _Allocator>::__construct_at_end(size_type __n, const_reference __x) {
-  _ConstructTransaction __tx(*this, __n);
-  const_pointer __new_end = __tx.__new_end_;
-  for (pointer __pos = __tx.__pos_; __pos != __new_end; __tx.__pos_ = ++__pos) {
-    __alloc_traits::construct(this->__alloc(), std::__to_address(__pos), __x);
-  }
-}
-
-template <class _Tp, class _Allocator>
-template <class _InputIterator, class _Sentinel>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void
-vector<_Tp, _Allocator>::__construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n) {
-  _ConstructTransaction __tx(*this, __n);
-  __tx.__pos_ = std::__uninitialized_allocator_copy(__alloc(), __first, __last, __tx.__pos_);
-}
-
-//  Default constructs __n objects starting at __end_
-//  throws if construction throws
-//  Postcondition:  size() == size() + __n
-//  Exception safety: strong.
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__append(size_type __n) {
-  if (static_cast<size_type>(this->__end_cap() - this->__end_) >= __n)
-    this->__construct_at_end(__n);
-  else {
-    allocator_type& __a = this->__alloc();
-    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), size(), __a);
-    __v.__construct_at_end(__n);
-    __swap_out_circular_buffer(__v);
-  }
-}
-
-//  Default constructs __n objects starting at __end_
-//  throws if construction throws
-//  Postcondition:  size() == size() + __n
-//  Exception safety: strong.
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__append(size_type __n, const_reference __x) {
-  if (static_cast<size_type>(this->__end_cap() - this->__end_) >= __n)
-    this->__construct_at_end(__n, __x);
-  else {
-    allocator_type& __a = this->__alloc();
-    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), size(), __a);
-    __v.__construct_at_end(__n, __x);
-    __swap_out_circular_buffer(__v);
-  }
-}
-
-template <class _Tp, class _Allocator>
-template <class _InputIterator,
-          __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
-                            is_constructible<_Tp, typename iterator_traits<_InputIterator>::reference>::value,
-                        int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<_Tp, _Allocator>::vector(_InputIterator __first, _InputIterator __last) {
-  __init_with_sentinel(__first, __last);
-}
-
-template <class _Tp, class _Allocator>
-template <class _InputIterator,
-          __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
-                            is_constructible<_Tp, typename iterator_traits<_InputIterator>::reference>::value,
-                        int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20
-vector<_Tp, _Allocator>::vector(_InputIterator __first, _InputIterator __last, const allocator_type& __a)
-    : __alloc_(__a) {
-  __init_with_sentinel(__first, __last);
-}
-
-template <class _Tp, class _Allocator>
-template <class _ForwardIterator,
-          __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
-                            is_constructible<_Tp, typename iterator_traits<_ForwardIterator>::reference>::value,
-                        int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<_Tp, _Allocator>::vector(_ForwardIterator __first, _ForwardIterator __last) {
-  size_type __n = static_cast<size_type>(std::distance(__first, __last));
-  __init_with_size(__first, __last, __n);
-}
-
-template <class _Tp, class _Allocator>
-template <class _ForwardIterator,
-          __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
-                            is_constructible<_Tp, typename iterator_traits<_ForwardIterator>::reference>::value,
-                        int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20
-vector<_Tp, _Allocator>::vector(_ForwardIterator __first, _ForwardIterator __last, const allocator_type& __a)
-    : __alloc_(__a) {
-  size_type __n = static_cast<size_type>(std::distance(__first, __last));
-  __init_with_size(__first, __last, __n);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<_Tp, _Allocator>::vector(const vector& __x)
-    : __alloc_(__alloc_traits::select_on_container_copy_construction(__x.__alloc())) {
-  __init_with_size(__x.__begin_, __x.__end_, __x.size());
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20
-vector<_Tp, _Allocator>::vector(const vector& __x, const __type_identity_t<allocator_type>& __a)
-    : __alloc_(__a) {
-  __init_with_size(__x.__begin_, __x.__end_, __x.size());
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI vector<_Tp, _Allocator>::vector(vector&& __x)
-#if _LIBCPP_STD_VER >= 17
-    noexcept
-#else
-    _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value)
-#endif
-    : __alloc_(std::move(__x.__alloc())) {
-  this->__begin_    = __x.__begin_;
-  this->__end_      = __x.__end_;
-  this->__end_cap() = __x.__end_cap();
-  __x.__begin_ = __x.__end_ = __x.__end_cap() = nullptr;
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI
-vector<_Tp, _Allocator>::vector(vector&& __x, const __type_identity_t<allocator_type>& __a)
-    : __alloc_(__a) {
-  if (__a == __x.__alloc()) {
-    this->__begin_    = __x.__begin_;
-    this->__end_      = __x.__end_;
-    this->__end_cap() = __x.__end_cap();
-    __x.__begin_ = __x.__end_ = __x.__end_cap() = nullptr;
-  } else {
-    typedef move_iterator<iterator> _Ip;
-    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
-    assign(_Ip(__x.begin()), _Ip(__x.end()));
-    __guard.__complete();
-  }
-}
-
-#ifndef _LIBCPP_CXX03_LANG
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI
-vector<_Tp, _Allocator>::vector(initializer_list<value_type> __il) {
-  __init_with_size(__il.begin(), __il.end(), __il.size());
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI
-vector<_Tp, _Allocator>::vector(initializer_list<value_type> __il, const allocator_type& __a)
-    : __alloc_(__a) {
-  __init_with_size(__il.begin(), __il.end(), __il.size());
-}
-
-#endif // _LIBCPP_CXX03_LANG
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI vector<_Tp, _Allocator>&
-vector<_Tp, _Allocator>::operator=(vector&& __x)
-    _NOEXCEPT_(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value) {
-  __move_assign(__x, integral_constant<bool, __alloc_traits::propagate_on_container_move_assignment::value>());
-  return *this;
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__move_assign(vector& __c, false_type)
-    _NOEXCEPT_(__alloc_traits::is_always_equal::value) {
-  if (__alloc() != __c.__alloc()) {
-    typedef move_iterator<iterator> _Ip;
-    assign(_Ip(__c.begin()), _Ip(__c.end()));
-  } else
-    __move_assign(__c, true_type());
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__move_assign(vector& __c, true_type)
-    _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
-  __vdeallocate();
-  __move_assign_alloc(__c); // this can throw
-  this->__begin_    = __c.__begin_;
-  this->__end_      = __c.__end_;
-  this->__end_cap() = __c.__end_cap();
-  __c.__begin_ = __c.__end_ = __c.__end_cap() = nullptr;
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI vector<_Tp, _Allocator>&
-vector<_Tp, _Allocator>::operator=(const vector& __x) {
-  if (this != std::addressof(__x)) {
-    __copy_assign_alloc(__x);
-    assign(__x.__begin_, __x.__end_);
-  }
-  return *this;
-}
-
-template <class _Tp, class _Allocator>
-template <class _InputIterator,
-          __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
-                            is_constructible<_Tp, typename iterator_traits<_InputIterator>::reference>::value,
-                        int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::assign(_InputIterator __first, _InputIterator __last) {
-  __assign_with_sentinel(__first, __last);
-}
-
-template <class _Tp, class _Allocator>
-template <class _Iterator, class _Sentinel>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
-vector<_Tp, _Allocator>::__assign_with_sentinel(_Iterator __first, _Sentinel __last) {
-  clear();
-  for (; __first != __last; ++__first)
-    emplace_back(*__first);
-}
-
-template <class _Tp, class _Allocator>
-template <class _ForwardIterator,
-          __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
-                            is_constructible<_Tp, typename iterator_traits<_ForwardIterator>::reference>::value,
-                        int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::assign(_ForwardIterator __first, _ForwardIterator __last) {
-  __assign_with_size(__first, __last, std::distance(__first, __last));
-}
-
-template <class _Tp, class _Allocator>
-template <class _ForwardIterator, class _Sentinel>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
-vector<_Tp, _Allocator>::__assign_with_size(_ForwardIterator __first, _Sentinel __last, difference_type __n) {
-  size_type __new_size = static_cast<size_type>(__n);
-  if (__new_size <= capacity()) {
-    if (__new_size > size()) {
-      _ForwardIterator __mid = std::next(__first, size());
-      std::copy(__first, __mid, this->__begin_);
-      __construct_at_end(__mid, __last, __new_size - size());
-    } else {
-      pointer __m = std::__copy<_ClassicAlgPolicy>(__first, __last, this->__begin_).second;
-      this->__destruct_at_end(__m);
-    }
-  } else {
-    __vdeallocate();
-    __vallocate(__recommend(__new_size));
-    __construct_at_end(__first, __last, __new_size);
-  }
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::assign(size_type __n, const_reference __u) {
-  if (__n <= capacity()) {
-    size_type __s = size();
-    std::fill_n(this->__begin_, std::min(__n, __s), __u);
-    if (__n > __s)
-      __construct_at_end(__n - __s, __u);
-    else
-      this->__destruct_at_end(this->__begin_ + __n);
-  } else {
-    __vdeallocate();
-    __vallocate(__recommend(static_cast<size_type>(__n)));
-    __construct_at_end(__n, __u);
-  }
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::begin() _NOEXCEPT {
-  return __make_iter(this->__begin_);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::const_iterator
-vector<_Tp, _Allocator>::begin() const _NOEXCEPT {
-  return __make_iter(this->__begin_);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::end() _NOEXCEPT {
-  return __make_iter(this->__end_);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::const_iterator
-vector<_Tp, _Allocator>::end() const _NOEXCEPT {
-  return __make_iter(this->__end_);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::reference
-vector<_Tp, _Allocator>::operator[](size_type __n) _NOEXCEPT {
-  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < size(), "vector[] index out of bounds");
-  return this->__begin_[__n];
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::const_reference
-vector<_Tp, _Allocator>::operator[](size_type __n) const _NOEXCEPT {
-  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < size(), "vector[] index out of bounds");
-  return this->__begin_[__n];
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::reference vector<_Tp, _Allocator>::at(size_type __n) {
-  if (__n >= size())
-    this->__throw_out_of_range();
-  return this->__begin_[__n];
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::const_reference
-vector<_Tp, _Allocator>::at(size_type __n) const {
-  if (__n >= size())
-    this->__throw_out_of_range();
-  return this->__begin_[__n];
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::reserve(size_type __n) {
-  if (__n > capacity()) {
-    if (__n > max_size())
-      this->__throw_length_error();
-    allocator_type& __a = this->__alloc();
-    __split_buffer<value_type, allocator_type&> __v(__n, size(), __a);
-    __swap_out_circular_buffer(__v);
-  }
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::shrink_to_fit() _NOEXCEPT {
-  if (capacity() > size()) {
-#if _LIBCPP_HAS_EXCEPTIONS
-    try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
-      allocator_type& __a = this->__alloc();
-      __split_buffer<value_type, allocator_type&> __v(size(), size(), __a);
-      // The Standard mandates shrink_to_fit() does not increase the capacity.
-      // With equal capacity keep the existing buffer. This avoids extra work
-      // due to swapping the elements.
-      if (__v.capacity() < capacity())
-        __swap_out_circular_buffer(__v);
-#if _LIBCPP_HAS_EXCEPTIONS
-    } catch (...) {
-    }
-#endif // _LIBCPP_HAS_EXCEPTIONS
-  }
-}
-
-template <class _Tp, class _Allocator>
-template <class... _Args>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::pointer
-vector<_Tp, _Allocator>::__emplace_back_slow_path(_Args&&... __args) {
-  allocator_type& __a = this->__alloc();
-  __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), size(), __a);
-  //    __v.emplace_back(std::forward<_Args>(__args)...);
-  __alloc_traits::construct(__a, std::__to_address(__v.__end_), std::forward<_Args>(__args)...);
-  __v.__end_++;
-  __swap_out_circular_buffer(__v);
-  return this->__end_;
-}
-
-template <class _Tp, class _Allocator>
-template <class... _Args>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline
-#if _LIBCPP_STD_VER >= 17
-    typename vector<_Tp, _Allocator>::reference
-#else
-    void
-#endif
-    vector<_Tp, _Allocator>::emplace_back(_Args&&... __args) {
-  pointer __end = this->__end_;
-  if (__end < this->__end_cap()) {
-    __construct_one_at_end(std::forward<_Args>(__args)...);
-    ++__end;
-  } else {
-    __end = __emplace_back_slow_path(std::forward<_Args>(__args)...);
-  }
-  this->__end_ = __end;
-#if _LIBCPP_STD_VER >= 17
-  return *(__end - 1);
-#endif
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void vector<_Tp, _Allocator>::pop_back() {
-  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "vector::pop_back called on an empty vector");
-  this->__destruct_at_end(this->__end_ - 1);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::erase(const_iterator __position) {
-  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
-      __position != end(), "vector::erase(iterator) called with a non-dereferenceable iterator");
-  difference_type __ps = __position - cbegin();
-  pointer __p          = this->__begin_ + __ps;
-  this->__destruct_at_end(std::move(__p + 1, this->__end_, __p));
-  return __make_iter(__p);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::erase(const_iterator __first, const_iterator __last) {
-  _LIBCPP_ASSERT_VALID_INPUT_RANGE(__first <= __last, "vector::erase(first, last) called with invalid range");
-  pointer __p = this->__begin_ + (__first - begin());
-  if (__first != __last) {
-    this->__destruct_at_end(std::move(__p + (__last - __first), this->__end_, __p));
-  }
-  return __make_iter(__p);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void
-vector<_Tp, _Allocator>::__move_range(pointer __from_s, pointer __from_e, pointer __to) {
-  pointer __old_last  = this->__end_;
-  difference_type __n = __old_last - __to;
-  {
-    pointer __i = __from_s + __n;
-    _ConstructTransaction __tx(*this, __from_e - __i);
-    for (pointer __pos = __tx.__pos_; __i < __from_e; ++__i, (void)++__pos, __tx.__pos_ = __pos) {
-      __alloc_traits::construct(this->__alloc(), std::__to_address(__pos), std::move(*__i));
-    }
-  }
-  std::move_backward(__from_s, __from_s + __n, __old_last);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::insert(const_iterator __position, const_reference __x) {
-  pointer __p = this->__begin_ + (__position - begin());
-  if (this->__end_ < this->__end_cap()) {
-    if (__p == this->__end_) {
-      __construct_one_at_end(__x);
-    } else {
-      __move_range(__p, this->__end_, __p + 1);
-      const_pointer __xr = pointer_traits<const_pointer>::pointer_to(__x);
-      if (std::__is_pointer_in_range(std::__to_address(__p), std::__to_address(__end_), std::addressof(__x)))
-        ++__xr;
-      *__p = *__xr;
-    }
-  } else {
-    allocator_type& __a = this->__alloc();
-    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, __a);
-    __v.push_back(__x);
-    __p = __swap_out_circular_buffer(__v, __p);
-  }
-  return __make_iter(__p);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::insert(const_iterator __position, value_type&& __x) {
-  pointer __p = this->__begin_ + (__position - begin());
-  if (this->__end_ < this->__end_cap()) {
-    if (__p == this->__end_) {
-      __construct_one_at_end(std::move(__x));
-    } else {
-      __move_range(__p, this->__end_, __p + 1);
-      *__p = std::move(__x);
-    }
-  } else {
-    allocator_type& __a = this->__alloc();
-    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, __a);
-    __v.push_back(std::move(__x));
-    __p = __swap_out_circular_buffer(__v, __p);
-  }
-  return __make_iter(__p);
-}
-
-template <class _Tp, class _Allocator>
-template <class... _Args>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::emplace(const_iterator __position, _Args&&... __args) {
-  pointer __p = this->__begin_ + (__position - begin());
-  if (this->__end_ < this->__end_cap()) {
-    if (__p == this->__end_) {
-      __construct_one_at_end(std::forward<_Args>(__args)...);
-    } else {
-      __temp_value<value_type, _Allocator> __tmp(this->__alloc(), std::forward<_Args>(__args)...);
-      __move_range(__p, this->__end_, __p + 1);
-      *__p = std::move(__tmp.get());
-    }
-  } else {
-    allocator_type& __a = this->__alloc();
-    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, __a);
-    __v.emplace_back(std::forward<_Args>(__args)...);
-    __p = __swap_out_circular_buffer(__v, __p);
-  }
-  return __make_iter(__p);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::insert(const_iterator __position, size_type __n, const_reference __x) {
-  pointer __p = this->__begin_ + (__position - begin());
-  if (__n > 0) {
-    // We can't compare unrelated pointers inside constant expressions
-    if (!__libcpp_is_constant_evaluated() && __n <= static_cast<size_type>(this->__end_cap() - this->__end_)) {
-      size_type __old_n  = __n;
-      pointer __old_last = this->__end_;
-      if (__n > static_cast<size_type>(this->__end_ - __p)) {
-        size_type __cx = __n - (this->__end_ - __p);
-        __construct_at_end(__cx, __x);
-        __n -= __cx;
-      }
-      if (__n > 0) {
-        __move_range(__p, __old_last, __p + __old_n);
-        const_pointer __xr = pointer_traits<const_pointer>::pointer_to(__x);
-        if (__p <= __xr && __xr < this->__end_)
-          __xr += __old_n;
-        std::fill_n(__p, __n, *__xr);
-      }
-    } else {
-      allocator_type& __a = this->__alloc();
-      __split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), __p - this->__begin_, __a);
-      __v.__construct_at_end(__n, __x);
-      __p = __swap_out_circular_buffer(__v, __p);
-    }
-  }
-  return __make_iter(__p);
-}
-template <class _Tp, class _Allocator>
-template <class _InputIterator,
-          __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value &&
-                            is_constructible<_Tp, typename iterator_traits<_InputIterator>::reference>::value,
-                        int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::insert(const_iterator __position, _InputIterator __first, _InputIterator __last) {
-  return __insert_with_sentinel(__position, __first, __last);
-}
-
-template <class _Tp, class _Allocator>
-template <class _InputIterator, class _Sentinel>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::__insert_with_sentinel(const_iterator __position, _InputIterator __first, _Sentinel __last) {
-  difference_type __off = __position - begin();
-  pointer __p           = this->__begin_ + __off;
-  allocator_type& __a   = this->__alloc();
-  pointer __old_last    = this->__end_;
-  for (; this->__end_ != this->__end_cap() && __first != __last; ++__first) {
-    __construct_one_at_end(*__first);
-  }
-  __split_buffer<value_type, allocator_type&> __v(__a);
-  if (__first != __last) {
-#if _LIBCPP_HAS_EXCEPTIONS
-    try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
-      __v.__construct_at_end_with_sentinel(std::move(__first), std::move(__last));
-      difference_type __old_size = __old_last - this->__begin_;
-      difference_type __old_p    = __p - this->__begin_;
-      reserve(__recommend(size() + __v.size()));
-      __p        = this->__begin_ + __old_p;
-      __old_last = this->__begin_ + __old_size;
-#if _LIBCPP_HAS_EXCEPTIONS
-    } catch (...) {
-      erase(__make_iter(__old_last), end());
-      throw;
-    }
-#endif // _LIBCPP_HAS_EXCEPTIONS
-  }
-  __p = std::rotate(__p, __old_last, this->__end_);
-  insert(__make_iter(__p), std::make_move_iterator(__v.begin()), std::make_move_iterator(__v.end()));
-  return begin() + __off;
-}
-
-template <class _Tp, class _Allocator>
-template <class _ForwardIterator,
-          __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value &&
-                            is_constructible<_Tp, typename iterator_traits<_ForwardIterator>::reference>::value,
-                        int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::insert(const_iterator __position, _ForwardIterator __first, _ForwardIterator __last) {
-  return __insert_with_size(__position, __first, __last, std::distance(__first, __last));
-}
-
-template <class _Tp, class _Allocator>
-template <class _Iterator, class _Sentinel>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::__insert_with_size(
-    const_iterator __position, _Iterator __first, _Sentinel __last, difference_type __n) {
-  auto __insertion_size = __n;
-  pointer __p           = this->__begin_ + (__position - begin());
-  if (__n > 0) {
-    if (__n <= this->__end_cap() - this->__end_) {
-      size_type __old_n    = __n;
-      pointer __old_last   = this->__end_;
-      _Iterator __m        = std::next(__first, __n);
-      difference_type __dx = this->__end_ - __p;
-      if (__n > __dx) {
-        __m                    = __first;
-        difference_type __diff = this->__end_ - __p;
-        std::advance(__m, __diff);
-        __construct_at_end(__m, __last, __n - __diff);
-        __n = __dx;
-      }
-      if (__n > 0) {
-        __move_range(__p, __old_last, __p + __old_n);
-        std::copy(__first, __m, __p);
-      }
-    } else {
-      allocator_type& __a = this->__alloc();
-      __split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), __p - this->__begin_, __a);
-      __v.__construct_at_end_with_size(__first, __insertion_size);
-      __p = __swap_out_circular_buffer(__v, __p);
-    }
-  }
-  return __make_iter(__p);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::resize(size_type __sz) {
-  size_type __cs = size();
-  if (__cs < __sz)
-    this->__append(__sz - __cs);
-  else if (__cs > __sz)
-    this->__destruct_at_end(this->__begin_ + __sz);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::resize(size_type __sz, const_reference __x) {
-  size_type __cs = size();
-  if (__cs < __sz)
-    this->__append(__sz - __cs, __x);
-  else if (__cs > __sz)
-    this->__destruct_at_end(this->__begin_ + __sz);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::swap(vector& __x)
-#if _LIBCPP_STD_VER >= 14
-    _NOEXCEPT
-#else
-    _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>)
-#endif
-{
-  _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
-      __alloc_traits::propagate_on_container_swap::value || this->__alloc() == __x.__alloc(),
-      "vector::swap: Either propagate_on_container_swap must be true"
-      " or the allocators must compare equal");
-  std::swap(this->__begin_, __x.__begin_);
-  std::swap(this->__end_, __x.__end_);
-  std::swap(this->__end_cap(), __x.__end_cap());
-  std::__swap_allocator(this->__alloc(), __x.__alloc());
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 bool vector<_Tp, _Allocator>::__invariants() const {
-  if (this->__begin_ == nullptr) {
-    if (this->__end_ != nullptr || this->__end_cap() != nullptr)
-      return false;
-  } else {
-    if (this->__begin_ > this->__end_)
-      return false;
-    if (this->__begin_ == this->__end_cap())
-      return false;
-    if (this->__end_ > this->__end_cap())
-      return false;
-  }
-  return true;
-}
-
-// vector<bool>
-
-template <class _Allocator>
-class vector<bool, _Allocator>;
-
-template <class _Allocator>
-struct hash<vector<bool, _Allocator> >;
-
-template <class _Allocator>
-struct __has_storage_type<vector<bool, _Allocator> > {
-  static const bool value = true;
-};
-
-template <class _Allocator>
-class _LIBCPP_TEMPLATE_VIS vector<bool, _Allocator> {
-public:
-  typedef vector __self;
-  typedef bool value_type;
-  typedef _Allocator allocator_type;
-  typedef allocator_traits<allocator_type> __alloc_traits;
-  typedef typename __alloc_traits::size_type size_type;
-  typedef typename __alloc_traits::difference_type difference_type;
-  typedef size_type __storage_type;
-  typedef __bit_iterator<vector, false> pointer;
-  typedef __bit_iterator<vector, true> const_pointer;
-  typedef pointer iterator;
-  typedef const_pointer const_iterator;
-  typedef std::reverse_iterator<iterator> reverse_iterator;
-  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
-
-private:
-  typedef __rebind_alloc<__alloc_traits, __storage_type> __storage_allocator;
-  typedef allocator_traits<__storage_allocator> __storage_traits;
-  typedef typename __storage_traits::pointer __storage_pointer;
-  typedef typename __storage_traits::const_pointer __const_storage_pointer;
-
-  __storage_pointer __begin_;
-  size_type __size_;
-  _LIBCPP_COMPRESSED_PAIR(size_type, __cap_, __storage_allocator, __alloc_);
-
-public:
-  typedef __bit_reference<vector> reference;
-#ifdef _LIBCPP_ABI_BITSET_VECTOR_BOOL_CONST_SUBSCRIPT_RETURN_BOOL
-  using const_reference = bool;
-#else
-  typedef __bit_const_reference<vector> const_reference;
-#endif
-
-private:
-  // TODO: Remove these now redundant accessors
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type& __cap() _NOEXCEPT { return __cap_; }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const size_type& __cap() const _NOEXCEPT { return __cap_; }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __storage_allocator& __alloc() _NOEXCEPT { return __alloc_; }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const __storage_allocator& __alloc() const _NOEXCEPT {
-    return __alloc_;
-  }
-
-  static const unsigned __bits_per_word = static_cast<unsigned>(sizeof(__storage_type) * CHAR_BIT);
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static size_type
-  __internal_cap_to_external(size_type __n) _NOEXCEPT {
-    return __n * __bits_per_word;
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static size_type
-  __external_cap_to_internal(size_type __n) _NOEXCEPT {
-    return (__n - 1) / __bits_per_word + 1;
-  }
-
-public:
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector()
-      _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value);
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit vector(const allocator_type& __a)
-#if _LIBCPP_STD_VER <= 14
-      _NOEXCEPT_(is_nothrow_copy_constructible<allocator_type>::value);
-#else
-      _NOEXCEPT;
-#endif
-
-private:
-  class __destroy_vector {
-  public:
-    _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI __destroy_vector(vector& __vec) : __vec_(__vec) {}
-
-    _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void operator()() {
-      if (__vec_.__begin_ != nullptr)
-        __storage_traits::deallocate(__vec_.__alloc(), __vec_.__begin_, __vec_.__cap());
-    }
-
-  private:
-    vector& __vec_;
-  };
-
-public:
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~vector() { __destroy_vector (*this)(); }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit vector(size_type __n);
-#if _LIBCPP_STD_VER >= 14
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit vector(size_type __n, const allocator_type& __a);
-#endif
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(size_type __n, const value_type& __v);
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
-  vector(size_type __n, const value_type& __v, const allocator_type& __a);
-  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(_InputIterator __first, _InputIterator __last);
-  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
-  vector(_InputIterator __first, _InputIterator __last, const allocator_type& __a);
-  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(_ForwardIterator __first, _ForwardIterator __last);
-  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
-  vector(_ForwardIterator __first, _ForwardIterator __last, const allocator_type& __a);
-
-#if _LIBCPP_STD_VER >= 23
-  template <_ContainerCompatibleRange<bool> _Range>
-  _LIBCPP_HIDE_FROM_ABI constexpr vector(from_range_t, _Range&& __range, const allocator_type& __a = allocator_type())
-      : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
-    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
-      auto __n = static_cast<size_type>(ranges::distance(__range));
-      __init_with_size(ranges::begin(__range), ranges::end(__range), __n);
-
-    } else {
-      __init_with_sentinel(ranges::begin(__range), ranges::end(__range));
-    }
-  }
-#endif
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(const vector& __v);
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(const vector& __v, const allocator_type& __a);
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector& operator=(const vector& __v);
-
-#ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(initializer_list<value_type> __il);
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
-  vector(initializer_list<value_type> __il, const allocator_type& __a);
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector& operator=(initializer_list<value_type> __il) {
-    assign(__il.begin(), __il.end());
-    return *this;
-  }
-
-#endif // !_LIBCPP_CXX03_LANG
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector(vector&& __v)
-#if _LIBCPP_STD_VER >= 17
-      noexcept;
-#else
-      _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value);
-#endif
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
-  vector(vector&& __v, const __type_identity_t<allocator_type>& __a);
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector& operator=(vector&& __v)
-      _NOEXCEPT_(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value);
-
-  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
-  void _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 assign(_InputIterator __first, _InputIterator __last);
-  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
-  void _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 assign(_ForwardIterator __first, _ForwardIterator __last);
-
-#if _LIBCPP_STD_VER >= 23
-  template <_ContainerCompatibleRange<bool> _Range>
-  _LIBCPP_HIDE_FROM_ABI constexpr void assign_range(_Range&& __range) {
-    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
-      auto __n = static_cast<size_type>(ranges::distance(__range));
-      __assign_with_size(ranges::begin(__range), ranges::end(__range), __n);
-
-    } else {
-      __assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
-    }
-  }
-#endif
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void assign(size_type __n, const value_type& __x);
-
-#ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void assign(initializer_list<value_type> __il) {
-    assign(__il.begin(), __il.end());
-  }
-#endif
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator_type get_allocator() const _NOEXCEPT {
-    return allocator_type(this->__alloc());
-  }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type max_size() const _NOEXCEPT;
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type capacity() const _NOEXCEPT {
-    return __internal_cap_to_external(__cap());
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type size() const _NOEXCEPT { return __size_; }
-  [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool empty() const _NOEXCEPT {
-    return __size_ == 0;
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void reserve(size_type __n);
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void shrink_to_fit() _NOEXCEPT;
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator begin() _NOEXCEPT { return __make_iter(0); }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator begin() const _NOEXCEPT { return __make_iter(0); }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator end() _NOEXCEPT { return __make_iter(__size_); }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator end() const _NOEXCEPT {
-    return __make_iter(__size_);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rbegin() _NOEXCEPT {
-    return reverse_iterator(end());
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator rbegin() const _NOEXCEPT {
-    return const_reverse_iterator(end());
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rend() _NOEXCEPT {
-    return reverse_iterator(begin());
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator rend() const _NOEXCEPT {
-    return const_reverse_iterator(begin());
-  }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cbegin() const _NOEXCEPT { return __make_iter(0); }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cend() const _NOEXCEPT {
-    return __make_iter(__size_);
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator crbegin() const _NOEXCEPT {
-    return rbegin();
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference operator[](size_type __n) { return __make_ref(__n); }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference operator[](size_type __n) const {
-    return __make_ref(__n);
-  }
-  _LIBCPP_HIDE_FROM_ABI reference at(size_type __n);
-  _LIBCPP_HIDE_FROM_ABI const_reference at(size_type __n) const;
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference front() { return __make_ref(0); }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference front() const { return __make_ref(0); }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference back() { return __make_ref(__size_ - 1); }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference back() const { return __make_ref(__size_ - 1); }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void push_back(const value_type& __x);
-#if _LIBCPP_STD_VER >= 14
-  template <class... _Args>
-#  if _LIBCPP_STD_VER >= 17
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference emplace_back(_Args&&... __args)
-#  else
-  _LIBCPP_HIDE_FROM_ABI void emplace_back(_Args&&... __args)
-#  endif
-  {
-    push_back(value_type(std::forward<_Args>(__args)...));
-#  if _LIBCPP_STD_VER >= 17
-    return this->back();
-#  endif
-  }
-#endif
-
-#if _LIBCPP_STD_VER >= 23
-  template <_ContainerCompatibleRange<bool> _Range>
-  _LIBCPP_HIDE_FROM_ABI constexpr void append_range(_Range&& __range) {
-    insert_range(end(), std::forward<_Range>(__range));
-  }
-#endif
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void pop_back() { --__size_; }
-
-#if _LIBCPP_STD_VER >= 14
-  template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator emplace(const_iterator __position, _Args&&... __args) {
-    return insert(__position, value_type(std::forward<_Args>(__args)...));
-  }
-#endif
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator insert(const_iterator __position, const value_type& __x);
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
-  insert(const_iterator __position, size_type __n, const value_type& __x);
-  template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
-  iterator _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
-  insert(const_iterator __position, _InputIterator __first, _InputIterator __last);
-  template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
-  iterator _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
-  insert(const_iterator __position, _ForwardIterator __first, _ForwardIterator __last);
-
-#if _LIBCPP_STD_VER >= 23
-  template <_ContainerCompatibleRange<bool> _Range>
-  _LIBCPP_HIDE_FROM_ABI constexpr iterator insert_range(const_iterator __position, _Range&& __range) {
-    if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
-      auto __n = static_cast<size_type>(ranges::distance(__range));
-      return __insert_with_size(__position, ranges::begin(__range), ranges::end(__range), __n);
-
-    } else {
-      return __insert_with_sentinel(__position, ranges::begin(__range), ranges::end(__range));
-    }
-  }
-#endif
-
-#ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
-  insert(const_iterator __position, initializer_list<value_type> __il) {
-    return insert(__position, __il.begin(), __il.end());
-  }
-#endif
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator erase(const_iterator __position);
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator erase(const_iterator __first, const_iterator __last);
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void clear() _NOEXCEPT { __size_ = 0; }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(vector&)
-#if _LIBCPP_STD_VER >= 14
-      _NOEXCEPT;
-#else
-      _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>);
-#endif
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void swap(reference __x, reference __y) _NOEXCEPT {
-    std::swap(__x, __y);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void resize(size_type __sz, value_type __x = false);
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void flip() _NOEXCEPT;
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __invariants() const;
-
-private:
-  [[__noreturn__]] _LIBCPP_HIDE_FROM_ABI void __throw_length_error() const { std::__throw_length_error("vector"); }
-
-  [[__noreturn__]] _LIBCPP_HIDE_FROM_ABI void __throw_out_of_range() const { std::__throw_out_of_range("vector"); }
-
-  template <class _InputIterator, class _Sentinel>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
-  __init_with_size(_InputIterator __first, _Sentinel __last, size_type __n) {
-    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
-
-    if (__n > 0) {
-      __vallocate(__n);
-      __construct_at_end(std::move(__first), std::move(__last), __n);
-    }
-
-    __guard.__complete();
-  }
-
-  template <class _InputIterator, class _Sentinel>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
-  __init_with_sentinel(_InputIterator __first, _Sentinel __last) {
-#if _LIBCPP_HAS_EXCEPTIONS
-    try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
-      for (; __first != __last; ++__first)
-        push_back(*__first);
-#if _LIBCPP_HAS_EXCEPTIONS
-    } catch (...) {
-      if (__begin_ != nullptr)
-        __storage_traits::deallocate(__alloc(), __begin_, __cap());
-      throw;
-    }
-#endif // _LIBCPP_HAS_EXCEPTIONS
-  }
-
-  template <class _Iterator, class _Sentinel>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __first, _Sentinel __last);
-
-  template <class _ForwardIterator, class _Sentinel>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
-  __assign_with_size(_ForwardIterator __first, _Sentinel __last, difference_type __ns);
-
-  template <class _InputIterator, class _Sentinel>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
-  __insert_with_sentinel(const_iterator __position, _InputIterator __first, _Sentinel __last);
-
-  template <class _Iterator, class _Sentinel>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
-  __insert_with_size(const_iterator __position, _Iterator __first, _Sentinel __last, difference_type __n);
-
-  //  Allocate space for __n objects
-  //  throws length_error if __n > max_size()
-  //  throws (probably bad_alloc) if memory run out
-  //  Precondition:  __begin_ == __end_ == __cap() == 0
-  //  Precondition:  __n > 0
-  //  Postcondition:  capacity() >= __n
-  //  Postcondition:  size() == 0
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __vallocate(size_type __n) {
-    if (__n > max_size())
-      __throw_length_error();
-    auto __allocation = std::__allocate_at_least(__alloc(), __external_cap_to_internal(__n));
-    __begin_          = __allocation.ptr;
-    __size_           = 0;
-    __cap()           = __allocation.count;
-    if (__libcpp_is_constant_evaluated()) {
-      for (size_type __i = 0; __i != __cap(); ++__i)
-        std::__construct_at(std::__to_address(__begin_) + __i);
-    }
-  }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __vdeallocate() _NOEXCEPT;
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static size_type __align_it(size_type __new_size) _NOEXCEPT {
-    return (__new_size + (__bits_per_word - 1)) & ~((size_type)__bits_per_word - 1);
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type __recommend(size_type __new_size) const;
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __construct_at_end(size_type __n, bool __x);
-  template <class _InputIterator, class _Sentinel>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
-  __construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n);
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __append(size_type __n, const_reference __x);
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference __make_ref(size_type __pos) _NOEXCEPT {
-    return reference(__begin_ + __pos / __bits_per_word, __storage_type(1) << __pos % __bits_per_word);
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference __make_ref(size_type __pos) const _NOEXCEPT {
-    return __bit_const_reference<vector>(
-        __begin_ + __pos / __bits_per_word, __storage_type(1) << __pos % __bits_per_word);
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator __make_iter(size_type __pos) _NOEXCEPT {
-    return iterator(__begin_ + __pos / __bits_per_word, static_cast<unsigned>(__pos % __bits_per_word));
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator __make_iter(size_type __pos) const _NOEXCEPT {
-    return const_iterator(__begin_ + __pos / __bits_per_word, static_cast<unsigned>(__pos % __bits_per_word));
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator __const_iterator_cast(const_iterator __p) _NOEXCEPT {
-    return begin() + (__p - cbegin());
-  }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __copy_assign_alloc(const vector& __v) {
-    __copy_assign_alloc(
-        __v, integral_constant<bool, __storage_traits::propagate_on_container_copy_assignment::value>());
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __copy_assign_alloc(const vector& __c, true_type) {
-    if (__alloc() != __c.__alloc())
-      __vdeallocate();
-    __alloc() = __c.__alloc();
-  }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __copy_assign_alloc(const vector&, false_type) {}
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign(vector& __c, false_type);
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign(vector& __c, true_type)
-      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value);
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign_alloc(vector& __c)
-      _NOEXCEPT_(!__storage_traits::propagate_on_container_move_assignment::value ||
-                 is_nothrow_move_assignable<allocator_type>::value) {
-    __move_assign_alloc(
-        __c, integral_constant<bool, __storage_traits::propagate_on_container_move_assignment::value>());
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign_alloc(vector& __c, true_type)
-      _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
-    __alloc() = std::move(__c.__alloc());
-  }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign_alloc(vector&, false_type) _NOEXCEPT {}
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_t __hash_code() const _NOEXCEPT;
-
-  friend class __bit_reference<vector>;
-  friend class __bit_const_reference<vector>;
-  friend class __bit_iterator<vector, false>;
-  friend class __bit_iterator<vector, true>;
-  friend struct __bit_array<vector>;
-  friend struct _LIBCPP_TEMPLATE_VIS hash<vector>;
-};
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::__vdeallocate() _NOEXCEPT {
-  if (this->__begin_ != nullptr) {
-    __storage_traits::deallocate(this->__alloc(), this->__begin_, __cap());
-    this->__begin_ = nullptr;
-    this->__size_ = this->__cap() = 0;
-  }
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::size_type
-vector<bool, _Allocator>::max_size() const _NOEXCEPT {
-  size_type __amax = __storage_traits::max_size(__alloc());
-  size_type __nmax = numeric_limits<size_type>::max() / 2; // end() >= begin(), always
-  if (__nmax / __bits_per_word <= __amax)
-    return __nmax;
-  return __internal_cap_to_external(__amax);
-}
-
-//  Precondition:  __new_size > capacity()
-template <class _Allocator>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::size_type
-vector<bool, _Allocator>::__recommend(size_type __new_size) const {
-  const size_type __ms = max_size();
-  if (__new_size > __ms)
-    this->__throw_length_error();
-  const size_type __cap = capacity();
-  if (__cap >= __ms / 2)
-    return __ms;
-  return std::max(2 * __cap, __align_it(__new_size));
-}
-
-//  Default constructs __n objects starting at __end_
-//  Precondition:  __n > 0
-//  Precondition:  size() + __n <= capacity()
-//  Postcondition:  size() == size() + __n
-template <class _Allocator>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
-vector<bool, _Allocator>::__construct_at_end(size_type __n, bool __x) {
-  size_type __old_size = this->__size_;
-  this->__size_ += __n;
-  if (__old_size == 0 || ((__old_size - 1) / __bits_per_word) != ((this->__size_ - 1) / __bits_per_word)) {
-    if (this->__size_ <= __bits_per_word)
-      this->__begin_[0] = __storage_type(0);
-    else
-      this->__begin_[(this->__size_ - 1) / __bits_per_word] = __storage_type(0);
-  }
-  std::fill_n(__make_iter(__old_size), __n, __x);
-}
-
-template <class _Allocator>
-template <class _InputIterator, class _Sentinel>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void
-vector<bool, _Allocator>::__construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n) {
-  size_type __old_size = this->__size_;
-  this->__size_ += __n;
-  if (__old_size == 0 || ((__old_size - 1) / __bits_per_word) != ((this->__size_ - 1) / __bits_per_word)) {
-    if (this->__size_ <= __bits_per_word)
-      this->__begin_[0] = __storage_type(0);
-    else
-      this->__begin_[(this->__size_ - 1) / __bits_per_word] = __storage_type(0);
-  }
-  std::__copy<_ClassicAlgPolicy>(__first, __last, __make_iter(__old_size));
-}
-
-template <class _Allocator>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector()
-    _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
-    : __begin_(nullptr), __size_(0), __cap_(0) {}
-
-template <class _Allocator>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const allocator_type& __a)
-#if _LIBCPP_STD_VER <= 14
-    _NOEXCEPT_(is_nothrow_copy_constructible<allocator_type>::value)
-#else
-        _NOEXCEPT
-#endif
-    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(size_type __n)
-    : __begin_(nullptr), __size_(0), __cap_(0) {
-  if (__n > 0) {
-    __vallocate(__n);
-    __construct_at_end(__n, false);
-  }
-}
-
-#if _LIBCPP_STD_VER >= 14
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(size_type __n, const allocator_type& __a)
-    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
-  if (__n > 0) {
-    __vallocate(__n);
-    __construct_at_end(__n, false);
-  }
-}
-#endif
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(size_type __n, const value_type& __x)
-    : __begin_(nullptr), __size_(0), __cap_(0) {
-  if (__n > 0) {
-    __vallocate(__n);
-    __construct_at_end(__n, __x);
-  }
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20
-vector<bool, _Allocator>::vector(size_type __n, const value_type& __x, const allocator_type& __a)
-    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
-  if (__n > 0) {
-    __vallocate(__n);
-    __construct_at_end(__n, __x);
-  }
-}
-
-template <class _Allocator>
-template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(_InputIterator __first, _InputIterator __last)
-    : __begin_(nullptr), __size_(0), __cap_(0) {
-  __init_with_sentinel(__first, __last);
-}
-
-template <class _Allocator>
-template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20
-vector<bool, _Allocator>::vector(_InputIterator __first, _InputIterator __last, const allocator_type& __a)
-    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
-  __init_with_sentinel(__first, __last);
-}
-
-template <class _Allocator>
-template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(_ForwardIterator __first, _ForwardIterator __last)
-    : __begin_(nullptr), __size_(0), __cap_(0) {
-  auto __n = static_cast<size_type>(std::distance(__first, __last));
-  __init_with_size(__first, __last, __n);
-}
-
-template <class _Allocator>
-template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20
-vector<bool, _Allocator>::vector(_ForwardIterator __first, _ForwardIterator __last, const allocator_type& __a)
-    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
-  auto __n = static_cast<size_type>(std::distance(__first, __last));
-  __init_with_size(__first, __last, __n);
-}
-
-#ifndef _LIBCPP_CXX03_LANG
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(initializer_list<value_type> __il)
-    : __begin_(nullptr), __size_(0), __cap_(0) {
-  size_type __n = static_cast<size_type>(__il.size());
-  if (__n > 0) {
-    __vallocate(__n);
-    __construct_at_end(__il.begin(), __il.end(), __n);
-  }
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20
-vector<bool, _Allocator>::vector(initializer_list<value_type> __il, const allocator_type& __a)
-    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
-  size_type __n = static_cast<size_type>(__il.size());
-  if (__n > 0) {
-    __vallocate(__n);
-    __construct_at_end(__il.begin(), __il.end(), __n);
-  }
-}
-
-#endif // _LIBCPP_CXX03_LANG
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const vector& __v)
-    : __begin_(nullptr),
-      __size_(0),
-      __cap_(0),
-      __alloc_(__storage_traits::select_on_container_copy_construction(__v.__alloc())) {
-  if (__v.size() > 0) {
-    __vallocate(__v.size());
-    __construct_at_end(__v.begin(), __v.end(), __v.size());
-  }
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const vector& __v, const allocator_type& __a)
-    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(__a) {
-  if (__v.size() > 0) {
-    __vallocate(__v.size());
-    __construct_at_end(__v.begin(), __v.end(), __v.size());
-  }
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>& vector<bool, _Allocator>::operator=(const vector& __v) {
-  if (this != std::addressof(__v)) {
-    __copy_assign_alloc(__v);
-    if (__v.__size_) {
-      if (__v.__size_ > capacity()) {
-        __vdeallocate();
-        __vallocate(__v.__size_);
-      }
-      std::copy(__v.__begin_, __v.__begin_ + __external_cap_to_internal(__v.__size_), __begin_);
-    }
-    __size_ = __v.__size_;
-  }
-  return *this;
-}
-
-template <class _Allocator>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(vector&& __v)
-#if _LIBCPP_STD_VER >= 17
-    _NOEXCEPT
-#else
-    _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value)
-#endif
-    : __begin_(__v.__begin_),
-      __size_(__v.__size_),
-      __cap_(__v.__cap_),
-      __alloc_(std::move(__v.__alloc_)) {
-  __v.__begin_ = nullptr;
-  __v.__size_  = 0;
-  __v.__cap()  = 0;
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20
-vector<bool, _Allocator>::vector(vector&& __v, const __type_identity_t<allocator_type>& __a)
-    : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(__a) {
-  if (__a == allocator_type(__v.__alloc())) {
-    this->__begin_ = __v.__begin_;
-    this->__size_  = __v.__size_;
-    this->__cap()  = __v.__cap();
-    __v.__begin_   = nullptr;
-    __v.__cap() = __v.__size_ = 0;
-  } else if (__v.size() > 0) {
-    __vallocate(__v.size());
-    __construct_at_end(__v.begin(), __v.end(), __v.size());
-  }
-}
-
-template <class _Allocator>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>&
-vector<bool, _Allocator>::operator=(vector&& __v)
-    _NOEXCEPT_(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value) {
-  __move_assign(__v, integral_constant<bool, __storage_traits::propagate_on_container_move_assignment::value>());
-  return *this;
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::__move_assign(vector& __c, false_type) {
-  if (__alloc() != __c.__alloc())
-    assign(__c.begin(), __c.end());
-  else
-    __move_assign(__c, true_type());
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::__move_assign(vector& __c, true_type)
-    _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
-  __vdeallocate();
-  __move_assign_alloc(__c);
-  this->__begin_ = __c.__begin_;
-  this->__size_  = __c.__size_;
-  this->__cap()  = __c.__cap();
-  __c.__begin_   = nullptr;
-  __c.__cap() = __c.__size_ = 0;
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::assign(size_type __n, const value_type& __x) {
-  __size_ = 0;
-  if (__n > 0) {
-    size_type __c = capacity();
-    if (__n <= __c)
-      __size_ = __n;
-    else {
-      vector __v(get_allocator());
-      __v.reserve(__recommend(__n));
-      __v.__size_ = __n;
-      swap(__v);
-    }
-    std::fill_n(begin(), __n, __x);
-  }
-}
-
-template <class _Allocator>
-template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::assign(_InputIterator __first, _InputIterator __last) {
-  __assign_with_sentinel(__first, __last);
-}
-
-template <class _Allocator>
-template <class _Iterator, class _Sentinel>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
-vector<bool, _Allocator>::__assign_with_sentinel(_Iterator __first, _Sentinel __last) {
-  clear();
-  for (; __first != __last; ++__first)
-    push_back(*__first);
-}
-
-template <class _Allocator>
-template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::assign(_ForwardIterator __first, _ForwardIterator __last) {
-  __assign_with_size(__first, __last, std::distance(__first, __last));
-}
-
-template <class _Allocator>
-template <class _ForwardIterator, class _Sentinel>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
-vector<bool, _Allocator>::__assign_with_size(_ForwardIterator __first, _Sentinel __last, difference_type __ns) {
-  _LIBCPP_ASSERT_VALID_INPUT_RANGE(__ns >= 0, "invalid range specified");
-
-  clear();
-
-  const size_t __n = static_cast<size_type>(__ns);
-  if (__n) {
-    if (__n > capacity()) {
-      __vdeallocate();
-      __vallocate(__n);
-    }
-    __construct_at_end(__first, __last, __n);
-  }
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::reserve(size_type __n) {
-  if (__n > capacity()) {
-    if (__n > max_size())
-      this->__throw_length_error();
-    vector __v(this->get_allocator());
-    __v.__vallocate(__n);
-    __v.__construct_at_end(this->begin(), this->end(), this->size());
-    swap(__v);
-  }
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::shrink_to_fit() _NOEXCEPT {
-  if (__external_cap_to_internal(size()) > __cap()) {
-#if _LIBCPP_HAS_EXCEPTIONS
-    try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
-      vector(*this, allocator_type(__alloc())).swap(*this);
-#if _LIBCPP_HAS_EXCEPTIONS
-    } catch (...) {
-    }
-#endif // _LIBCPP_HAS_EXCEPTIONS
-  }
-}
-
-template <class _Allocator>
-typename vector<bool, _Allocator>::reference vector<bool, _Allocator>::at(size_type __n) {
-  if (__n >= size())
-    this->__throw_out_of_range();
-  return (*this)[__n];
-}
-
-template <class _Allocator>
-typename vector<bool, _Allocator>::const_reference vector<bool, _Allocator>::at(size_type __n) const {
-  if (__n >= size())
-    this->__throw_out_of_range();
-  return (*this)[__n];
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::push_back(const value_type& __x) {
-  if (this->__size_ == this->capacity())
-    reserve(__recommend(this->__size_ + 1));
-  ++this->__size_;
-  back() = __x;
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
-vector<bool, _Allocator>::insert(const_iterator __position, const value_type& __x) {
-  iterator __r;
-  if (size() < capacity()) {
-    const_iterator __old_end = end();
-    ++__size_;
-    std::copy_backward(__position, __old_end, end());
-    __r = __const_iterator_cast(__position);
-  } else {
-    vector __v(get_allocator());
-    __v.reserve(__recommend(__size_ + 1));
-    __v.__size_ = __size_ + 1;
-    __r         = std::copy(cbegin(), __position, __v.begin());
-    std::copy_backward(__position, cend(), __v.end());
-    swap(__v);
-  }
-  *__r = __x;
-  return __r;
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
-vector<bool, _Allocator>::insert(const_iterator __position, size_type __n, const value_type& __x) {
-  iterator __r;
-  size_type __c = capacity();
-  if (__n <= __c && size() <= __c - __n) {
-    const_iterator __old_end = end();
-    __size_ += __n;
-    std::copy_backward(__position, __old_end, end());
-    __r = __const_iterator_cast(__position);
-  } else {
-    vector __v(get_allocator());
-    __v.reserve(__recommend(__size_ + __n));
-    __v.__size_ = __size_ + __n;
-    __r         = std::copy(cbegin(), __position, __v.begin());
-    std::copy_backward(__position, cend(), __v.end());
-    swap(__v);
-  }
-  std::fill_n(__r, __n, __x);
-  return __r;
-}
-
-template <class _Allocator>
-template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
-vector<bool, _Allocator>::insert(const_iterator __position, _InputIterator __first, _InputIterator __last) {
-  return __insert_with_sentinel(__position, __first, __last);
-}
-
-template <class _Allocator>
-template <class _InputIterator, class _Sentinel>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<bool, _Allocator>::iterator
-vector<bool, _Allocator>::__insert_with_sentinel(const_iterator __position, _InputIterator __first, _Sentinel __last) {
-  difference_type __off = __position - begin();
-  iterator __p          = __const_iterator_cast(__position);
-  iterator __old_end    = end();
-  for (; size() != capacity() && __first != __last; ++__first) {
-    ++this->__size_;
-    back() = *__first;
-  }
-  vector __v(get_allocator());
-  if (__first != __last) {
-#if _LIBCPP_HAS_EXCEPTIONS
-    try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
-      __v.__assign_with_sentinel(std::move(__first), std::move(__last));
-      difference_type __old_size = static_cast<difference_type>(__old_end - begin());
-      difference_type __old_p    = __p - begin();
-      reserve(__recommend(size() + __v.size()));
-      __p       = begin() + __old_p;
-      __old_end = begin() + __old_size;
-#if _LIBCPP_HAS_EXCEPTIONS
-    } catch (...) {
-      erase(__old_end, end());
-      throw;
-    }
-#endif // _LIBCPP_HAS_EXCEPTIONS
-  }
-  __p = std::rotate(__p, __old_end, end());
-  insert(__p, __v.begin(), __v.end());
-  return begin() + __off;
-}
-
-template <class _Allocator>
-template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
-vector<bool, _Allocator>::insert(const_iterator __position, _ForwardIterator __first, _ForwardIterator __last) {
-  return __insert_with_size(__position, __first, __last, std::distance(__first, __last));
-}
-
-template <class _Allocator>
-template <class _ForwardIterator, class _Sentinel>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<bool, _Allocator>::iterator
-vector<bool, _Allocator>::__insert_with_size(
-    const_iterator __position, _ForwardIterator __first, _Sentinel __last, difference_type __n_signed) {
-  _LIBCPP_ASSERT_VALID_INPUT_RANGE(__n_signed >= 0, "invalid range specified");
-  const size_type __n = static_cast<size_type>(__n_signed);
-  iterator __r;
-  size_type __c = capacity();
-  if (__n <= __c && size() <= __c - __n) {
-    const_iterator __old_end = end();
-    __size_ += __n;
-    std::copy_backward(__position, __old_end, end());
-    __r = __const_iterator_cast(__position);
-  } else {
-    vector __v(get_allocator());
-    __v.reserve(__recommend(__size_ + __n));
-    __v.__size_ = __size_ + __n;
-    __r         = std::copy(cbegin(), __position, __v.begin());
-    std::copy_backward(__position, cend(), __v.end());
-    swap(__v);
-  }
-  std::__copy<_ClassicAlgPolicy>(__first, __last, __r);
-  return __r;
-}
-
-template <class _Allocator>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
-vector<bool, _Allocator>::erase(const_iterator __position) {
-  iterator __r = __const_iterator_cast(__position);
-  std::copy(__position + 1, this->cend(), __r);
-  --__size_;
-  return __r;
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<bool, _Allocator>::iterator
-vector<bool, _Allocator>::erase(const_iterator __first, const_iterator __last) {
-  iterator __r        = __const_iterator_cast(__first);
-  difference_type __d = __last - __first;
-  std::copy(__last, this->cend(), __r);
-  __size_ -= __d;
-  return __r;
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::swap(vector& __x)
-#if _LIBCPP_STD_VER >= 14
-    _NOEXCEPT
-#else
-    _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>)
-#endif
-{
-  std::swap(this->__begin_, __x.__begin_);
-  std::swap(this->__size_, __x.__size_);
-  std::swap(this->__cap(), __x.__cap());
-  std::__swap_allocator(this->__alloc(), __x.__alloc());
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::resize(size_type __sz, value_type __x) {
-  size_type __cs = size();
-  if (__cs < __sz) {
-    iterator __r;
-    size_type __c = capacity();
-    size_type __n = __sz - __cs;
-    if (__n <= __c && __cs <= __c - __n) {
-      __r = end();
-      __size_ += __n;
-    } else {
-      vector __v(get_allocator());
-      __v.reserve(__recommend(__size_ + __n));
-      __v.__size_ = __size_ + __n;
-      __r         = std::copy(cbegin(), cend(), __v.begin());
-      swap(__v);
-    }
-    std::fill_n(__r, __n, __x);
-  } else
-    __size_ = __sz;
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::flip() _NOEXCEPT {
-  // do middle whole words
-  size_type __n         = __size_;
-  __storage_pointer __p = __begin_;
-  for (; __n >= __bits_per_word; ++__p, __n -= __bits_per_word)
-    *__p = ~*__p;
-  // do last partial word
-  if (__n > 0) {
-    __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
-    __storage_type __b = *__p & __m;
-    *__p &= ~__m;
-    *__p |= ~__b & __m;
-  }
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 bool vector<bool, _Allocator>::__invariants() const {
-  if (this->__begin_ == nullptr) {
-    if (this->__size_ != 0 || this->__cap() != 0)
-      return false;
-  } else {
-    if (this->__cap() == 0)
-      return false;
-    if (this->__size_ > this->capacity())
-      return false;
-  }
-  return true;
-}
-
-template <class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 size_t vector<bool, _Allocator>::__hash_code() const _NOEXCEPT {
-  size_t __h = 0;
-  // do middle whole words
-  size_type __n         = __size_;
-  __storage_pointer __p = __begin_;
-  for (; __n >= __bits_per_word; ++__p, __n -= __bits_per_word)
-    __h ^= *__p;
-  // do last partial word
-  if (__n > 0) {
-    const __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
-    __h ^= *__p & __m;
-  }
-  return __h;
-}
-
-template <class _Allocator>
-struct _LIBCPP_TEMPLATE_VIS hash<vector<bool, _Allocator> >
-    : public __unary_function<vector<bool, _Allocator>, size_t> {
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_t
-  operator()(const vector<bool, _Allocator>& __vec) const _NOEXCEPT {
-    return __vec.__hash_code();
-  }
-};
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI bool
-operator==(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
-  const typename vector<_Tp, _Allocator>::size_type __sz = __x.size();
-  return __sz == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
-}
-
-#if _LIBCPP_STD_VER <= 17
-
-template <class _Tp, class _Allocator>
-inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
-  return !(__x == __y);
-}
-
-template <class _Tp, class _Allocator>
-inline _LIBCPP_HIDE_FROM_ABI bool operator<(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
-  return std::lexicographical_compare(__x.begin(), __x.end(), __y.begin(), __y.end());
-}
-
-template <class _Tp, class _Allocator>
-inline _LIBCPP_HIDE_FROM_ABI bool operator>(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
-  return __y < __x;
-}
-
-template <class _Tp, class _Allocator>
-inline _LIBCPP_HIDE_FROM_ABI bool operator>=(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
-  return !(__x < __y);
-}
-
-template <class _Tp, class _Allocator>
-inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
-  return !(__y < __x);
-}
-
-#else // _LIBCPP_STD_VER <= 17
-
-template <class _Tp, class _Allocator>
-_LIBCPP_HIDE_FROM_ABI constexpr __synth_three_way_result<_Tp>
-operator<=>(const vector<_Tp, _Allocator>& __x, const vector<_Tp, _Allocator>& __y) {
-  return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
-}
-
-#endif // _LIBCPP_STD_VER <= 17
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void
-swap(vector<_Tp, _Allocator>& __x, vector<_Tp, _Allocator>& __y) _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
-  __x.swap(__y);
-}
-
-#if _LIBCPP_STD_VER >= 20
-template <class _Tp, class _Allocator, class _Up>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::size_type
-erase(vector<_Tp, _Allocator>& __c, const _Up& __v) {
-  auto __old_size = __c.size();
-  __c.erase(std::remove(__c.begin(), __c.end(), __v), __c.end());
-  return __old_size - __c.size();
-}
-
-template <class _Tp, class _Allocator, class _Predicate>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::size_type
-erase_if(vector<_Tp, _Allocator>& __c, _Predicate __pred) {
-  auto __old_size = __c.size();
-  __c.erase(std::remove_if(__c.begin(), __c.end(), __pred), __c.end());
-  return __old_size - __c.size();
-}
-
-template <>
-inline constexpr bool __format::__enable_insertable<vector<char>> = true;
-#  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
-template <>
-inline constexpr bool __format::__enable_insertable<vector<wchar_t>> = true;
-#  endif
-
-#endif // _LIBCPP_STD_VER >= 20
-
-#if _LIBCPP_STD_VER >= 23
-template <class _Tp, class _CharT>
-// Since is-vector-bool-reference is only used once it's inlined here.
-  requires same_as<typename _Tp::__container, vector<bool, typename _Tp::__container::allocator_type>>
-struct _LIBCPP_TEMPLATE_VIS formatter<_Tp, _CharT> {
-private:
-  formatter<bool, _CharT> __underlying_;
-
-public:
-  template <class _ParseContext>
-  _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) {
-    return __underlying_.parse(__ctx);
-  }
-
-  template <class _FormatContext>
-  _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(const _Tp& __ref, _FormatContext& __ctx) const {
-    return __underlying_.format(__ref, __ctx);
-  }
-};
-#endif // _LIBCPP_STD_VER >= 23
-
-template <class _Tp, class _Allocator>
-struct __container_traits<vector<_Tp, _Allocator> > {
-  // http://eel.is/c++draft/vector.modifiers#2
-  //  If an exception is thrown other than by the copy constructor, move constructor, assignment operator, or move
-  //  assignment operator of T or by any InputIterator operation, there are no effects. If an exception is thrown while
-  //  inserting a single element at the end and T is Cpp17CopyInsertable or is_nothrow_move_constructible_v<T> is true,
-  //  there are no effects. Otherwise, if an exception is thrown by the move constructor of a non-Cpp17CopyInsertable T,
-  //  the effects are unspecified.
-  static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee =
-      _Or<is_nothrow_move_constructible<_Tp>, __is_cpp17_copy_insertable<_Allocator> >::value;
-};
-
-_LIBCPP_END_NAMESPACE_STD
-
-#if _LIBCPP_STD_VER >= 17
-_LIBCPP_BEGIN_NAMESPACE_STD
-namespace pmr {
-template <class _ValueT>
-using vector _LIBCPP_AVAILABILITY_PMR = std::vector<_ValueT, polymorphic_allocator<_ValueT>>;
-} // namespace pmr
-_LIBCPP_END_NAMESPACE_STD
-#endif
-
-_LIBCPP_POP_MACROS
-
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #  include <algorithm>
+#  include <array>
 #  include <atomic>
+#  include <cctype>
+#  include <cerrno>
+#  include <clocale>
 #  include <concepts>
+#  include <cstdint>
 #  include <cstdlib>
 #  include <iosfwd>
 #  if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
 #    include <locale>
 #  endif
+#  include <string>
+#  include <string_view>
 #  include <tuple>
 #  include <type_traits>
 #  include <typeinfo>
diff --git a/libcxx/test/libcxx/containers/sequences/vector.bool/abi.compile.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector.bool/abi.compile.pass.cpp
index 9920d131da3e..cc6b0d94e7da 100644
--- a/libcxx/test/libcxx/containers/sequences/vector.bool/abi.compile.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector.bool/abi.compile.pass.cpp
@@ -8,8 +8,8 @@
 
 // UNSUPPORTED: libcpp-abi-no-compressed-pair-padding
 
-#include <vector>
 #include <cstdint>
+#include <vector>
 
 #include "min_allocator.h"
 #include "test_allocator.h"
diff --git a/libcxx/test/libcxx/memory/swap_allocator.pass.cpp b/libcxx/test/libcxx/memory/swap_allocator.pass.cpp
index 5eeb1bede71f..311660290eae 100644
--- a/libcxx/test/libcxx/memory/swap_allocator.pass.cpp
+++ b/libcxx/test/libcxx/memory/swap_allocator.pass.cpp
@@ -12,10 +12,9 @@
 // template <typename _Alloc>
 // void __swap_allocator(_Alloc& __a1, _Alloc& __a2);
 
+#include <__memory/swap_allocator.h>
 #include <cassert>
 #include <memory>
-// Transitively includes `swap_allocator.h` (directly including it breaks the modules build).
-#include <vector>
 #include <utility>
 
 #include "test_macros.h"
diff --git a/libcxx/test/libcxx/transitive_includes/cxx23.csv b/libcxx/test/libcxx/transitive_includes/cxx23.csv
index 861fd2bb9ba9..c3db06f15477 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx23.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx23.csv
@@ -145,7 +145,6 @@ chrono string
 chrono string_view
 chrono tuple
 chrono typeinfo
-chrono vector
 chrono version
 cinttypes cstdint
 cmath limits
@@ -457,29 +456,20 @@ fstream typeinfo
 fstream version
 functional array
 functional cctype
-functional cerrno
-functional climits
-functional clocale
 functional compare
 functional cstddef
 functional cstdint
-functional cstdio
-functional cstdlib
 functional cstring
 functional cwchar
 functional cwctype
 functional initializer_list
-functional iosfwd
 functional limits
 functional new
 functional optional
 functional stdexcept
-functional string
-functional string_view
 functional tuple
 functional typeinfo
 functional unordered_map
-functional vector
 functional version
 future array
 future atomic
@@ -884,17 +874,13 @@ queue tuple
 queue typeinfo
 queue vector
 queue version
-random array
 random cctype
-random cerrno
 random climits
-random clocale
 random cmath
 random compare
 random cstddef
 random cstdint
 random cstdio
-random cstdlib
 random cstring
 random ctime
 random cwchar
@@ -910,8 +896,6 @@ random stdexcept
 random string
 random string_view
 random tuple
-random typeinfo
-random vector
 random version
 ranges cctype
 ranges compare
diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv
index a31139f1ee85..8d7560344ee5 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx26.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv
@@ -145,7 +145,6 @@ chrono string
 chrono string_view
 chrono tuple
 chrono typeinfo
-chrono vector
 chrono version
 cinttypes cstdint
 cmath limits
@@ -456,29 +455,20 @@ fstream typeinfo
 fstream version
 functional array
 functional cctype
-functional cerrno
-functional climits
-functional clocale
 functional compare
 functional cstddef
 functional cstdint
-functional cstdio
-functional cstdlib
 functional cstring
 functional cwchar
 functional cwctype
 functional initializer_list
-functional iosfwd
 functional limits
 functional new
 functional optional
 functional stdexcept
-functional string
-functional string_view
 functional tuple
 functional typeinfo
 functional unordered_map
-functional vector
 functional version
 future array
 future atomic
@@ -883,17 +873,13 @@ queue tuple
 queue typeinfo
 queue vector
 queue version
-random array
 random cctype
-random cerrno
 random climits
-random clocale
 random cmath
 random compare
 random cstddef
 random cstdint
 random cstdio
-random cstdlib
 random cstring
 random ctime
 random cwchar
@@ -909,8 +895,6 @@ random stdexcept
 random string
 random string_view
 random tuple
-random typeinfo
-random vector
 random version
 ranges cctype
 ranges compare
diff --git a/libcxx/test/std/atomics/atomics.ref/assign.pass.cpp b/libcxx/test/std/atomics/atomics.ref/assign.pass.cpp
index 3887211752c6..9b2f9042e983 100644
--- a/libcxx/test/std/atomics/atomics.ref/assign.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/assign.pass.cpp
@@ -13,6 +13,7 @@
 
 #include <atomic>
 #include <cassert>
+#include <concepts>
 #include <type_traits>
 
 #include "atomic_helpers.h"
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex.pass.cpp
index ece330134f2c..45d694ed5c25 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex.pass.cpp
@@ -22,6 +22,7 @@
 #include <cassert>
 #include <shared_mutex>
 #include <thread>
+#include <type_traits>
 #include <vector>
 
 #include "make_test_thread.h"
diff --git a/libcxx/test/std/time/time.zone/time.zone.leap/cons.copy.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.leap/cons.copy.pass.cpp
index 79ed08608115..8c20e45213c3 100644
--- a/libcxx/test/std/time/time.zone/time.zone.leap/cons.copy.pass.cpp
+++ b/libcxx/test/std/time/time.zone/time.zone.leap/cons.copy.pass.cpp
@@ -21,9 +21,10 @@
 //   ...
 // };
 
+#include <cassert>
 #include <chrono>
 #include <concepts>
-#include <cassert>
+#include <type_traits>
 
 #include "test_chrono_leap_second.h"
 
diff --git a/libcxx/test/std/time/time.zone/time.zone.leap/members/value.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.leap/members/value.pass.cpp
index c6181ee7df9f..4da807cd4f4b 100644
--- a/libcxx/test/std/time/time.zone/time.zone.leap/members/value.pass.cpp
+++ b/libcxx/test/std/time/time.zone/time.zone.leap/members/value.pass.cpp
@@ -20,6 +20,7 @@
 
 #include <cassert>
 #include <chrono>
+#include <concepts>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/time/time.zone/time.zone.link/time.zone.link.members/name.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.link/time.zone.link.members/name.pass.cpp
index 593675cfba45..1f9b15ec8f4a 100644
--- a/libcxx/test/std/time/time.zone/time.zone.link/time.zone.link.members/name.pass.cpp
+++ b/libcxx/test/std/time/time.zone/time.zone.link/time.zone.link.members/name.pass.cpp
@@ -20,6 +20,7 @@
 
 #include <cassert>
 #include <chrono>
+#include <concepts>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/time/time.zone/time.zone.link/time.zone.link.members/target.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.link/time.zone.link.members/target.pass.cpp
index aa97fdadcebc..069132b0a39f 100644
--- a/libcxx/test/std/time/time.zone/time.zone.link/time.zone.link.members/target.pass.cpp
+++ b/libcxx/test/std/time/time.zone/time.zone.link/time.zone.link.members/target.pass.cpp
@@ -20,6 +20,7 @@
 
 #include <cassert>
 #include <chrono>
+#include <concepts>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/name.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/name.pass.cpp
index b9f473829279..fd17d9a6978b 100644
--- a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/name.pass.cpp
+++ b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/name.pass.cpp
@@ -20,6 +20,7 @@
 
 #include <cassert>
 #include <chrono>
+#include <concepts>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/time/time.zone/time.zone.zonedtraits/const_time_zone_default_zone.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.zonedtraits/const_time_zone_default_zone.pass.cpp
index 930de9d0fec9..e22d62390a77 100644
--- a/libcxx/test/std/time/time.zone/time.zone.zonedtraits/const_time_zone_default_zone.pass.cpp
+++ b/libcxx/test/std/time/time.zone/time.zone.zonedtraits/const_time_zone_default_zone.pass.cpp
@@ -18,8 +18,8 @@
 
 // static const time_zone* default_zone();
 
-#include <chrono>
 #include <cassert>
+#include <chrono>
 #include <concepts>
 
 int main(int, char**) {
diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/dereference.single.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/dereference.single.pass.cpp
index ba7f8bbb7622..4063190838a2 100644
--- a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/dereference.single.pass.cpp
+++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/dereference.single.pass.cpp
@@ -12,8 +12,9 @@
 
 // test op*()
 
-#include <memory>
 #include <cassert>
+#include <memory>
+#include <utility>
 #include <vector>
 
 #include "test_macros.h"
-- 
GitLab


From b91ce7bc0eda8ec91c7ba724fbe78607843e0e06 Mon Sep 17 00:00:00 2001
From: Sayan Saha <sayans@mathworks.com>
Date: Fri, 25 Oct 2024 05:51:09 -0400
Subject: [PATCH 145/693] [Tosa] : Fix integer overflow for computing intmax+1
 in tosa.cast to linalg. (#112455)

This PR fixes an issue related to integer overflow when computing
`(intmax+1)` for `i64` during `tosa-to-linalg` pass for `tosa.cast`.

Found this issue while debugging a numerical mismatch for `deeplabv3`
model from `torchvision` represented in `tosa` dialect using the
`TorchToTosa` pipeline in `torch-mlir` repository. `torch.aten.to.dtype`
is converted to `tosa.cast` that casts `f32` to `i64` type. Technically
by the specification, `tosa.cast` doesn't handle casting `f32` to `i64`.
So it's possible to add a verifier to error out for such tosa ops
instead of producing incorrect code. However, I chose to fix the
overflow issue to still be able to represent the `deeplabv3` model with
`tosa` ops in the above-mentioned pipeline. Open to suggestions if
adding the verifier is more appropriate instead.
---
 .../Conversion/TosaToLinalg/TosaToLinalg.cpp  |  7 ++---
 .../TosaToLinalg/tosa-to-linalg.mlir          | 27 +++++++++++++++++++
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index 495f1b4f10b0..251c48859d2d 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -555,9 +555,10 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       auto intMaxPlusOneFP = rewriter.create<arith::ConstantOp>(
           loc, rewriter.getFloatAttr(
                    getElementTypeOrSelf(srcTy),
-                   APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())
-                           .getSExtValue() +
-                       1));
+                   static_cast<double>(
+                       APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())
+                           .getSExtValue()) +
+                       1.0f));
 
       auto intMax = rewriter.create<arith::ConstantOp>(
           loc, rewriter.getIntegerAttr(
diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
index f9d37f9427d4..1a29d3f9f350 100644
--- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
+++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
@@ -1929,3 +1929,30 @@ func.func @test_dynamic_fft2d(%arg0: tensor<?x?x?xf32>, %arg1: tensor<?x?x?xf32>
   %output_real, %output_imag = "tosa.fft2d"(%arg0, %arg1) {inverse = true} : (tensor<?x?x?xf32>, tensor<?x?x?xf32>) -> (tensor<?x?x?xf32>, tensor<?x?x?xf32>)
   return %output_real, %output_imag : tensor<?x?x?xf32>, tensor<?x?x?xf32>
 }
+
+
+// -----
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (0)>
+// CHECK: #[[$MAP1:.+]] = affine_map<(d0) -> (d0)>
+
+// CHECK-LABEL:   func.func @test_cast_fp32_i64(
+// CHECK-SAME:                                  %[[ARG0:.*]]: tensor<1xf32>) -> tensor<1xi64> {
+// CHECK:           %[[EMPTY_TENSOR:.*]] = tensor.empty() : tensor<1xi64>
+// CHECK:           %[[RESULT:.*]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]]], iterator_types = ["parallel"]} ins(%[[ARG0]] : tensor<1xf32>) outs(%[[EMPTY_TENSOR]] : tensor<1xi64>) {
+// CHECK:           ^bb0(%[[IN:.*]]: f32, %[[OUT:.*]]: i64):
+// CHECK:             %[[ROUND_EVEN:.*]] = math.roundeven %[[IN]] : f32
+// CHECK:             %[[FP_INT_MIN:.*]] = arith.constant -9.22337203E+18 : f32
+// CHECK:             %[[FP_INT_MAX_PLUS_ONE:.*]] = arith.constant 9.22337203E+18 : f32
+// CHECK:             %[[INT_MAX:.*]] = arith.constant 9223372036854775807 : i64
+// CHECK:             %[[MAX:.*]] = arith.maximumf %[[ROUND_EVEN]], %[[FP_INT_MIN]] : f32
+// CHECK:             %[[FPTOSI:.*]] = arith.fptosi %[[MAX]] : f32 to i64
+// CHECK:             %[[CMPF:.*]] = arith.cmpf uge, %[[ROUND_EVEN]], %[[FP_INT_MAX_PLUS_ONE]] : f32
+// CHECK:             %[[SELECT:.*]] = arith.select %[[CMPF]], %[[INT_MAX]], %[[FPTOSI]] : i64
+// CHECK:             linalg.yield %[[SELECT]] : i64
+// CHECK:           } -> tensor<1xi64>
+// CHECK:           return %[[RESULT]] : tensor<1xi64>
+func.func @test_cast_fp32_i64(%arg0: tensor<1xf32>) -> (tensor<1xi64>) {
+  %0 = tosa.cast %arg0 : (tensor<1xf32>) -> tensor<1xi64>
+  return %0: tensor<1xi64>
+}
-- 
GitLab


From 6bb6922de92d8413321f57f74769380c6b6f45ee Mon Sep 17 00:00:00 2001
From: Jacek Caban <jacek@codeweavers.com>
Date: Fri, 25 Oct 2024 11:53:47 +0200
Subject: [PATCH 146/693] [LLD][COFF] Allow overriding EC alias symbols with
 alternate names (#113456)

---
 lld/COFF/Driver.cpp              |  16 ++++-
 lld/test/COFF/arm64ec-altnames.s | 109 +++++++++++++++++++++++++++++++
 llvm/include/llvm/IR/Mangler.h   |   6 ++
 3 files changed, 128 insertions(+), 3 deletions(-)
 create mode 100644 lld/test/COFF/arm64ec-altnames.s

diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index e7f768789271..08c1476a595f 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -2518,9 +2518,19 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
         Symbol *sym = ctx.symtab.find(from);
         if (!sym)
           continue;
-        if (auto *u = dyn_cast<Undefined>(sym))
-          if (!u->weakAlias)
-            u->setWeakAlias(ctx.symtab.addUndefined(to));
+        if (auto *u = dyn_cast<Undefined>(sym)) {
+          if (u->weakAlias) {
+            // On ARM64EC, anti-dependency aliases are treated as undefined
+            // symbols unless a demangled symbol aliases a defined one, which is
+            // part of the implementation.
+            if (!isArm64EC(ctx.config.machine) || !u->isAntiDep)
+              continue;
+            if (!isa<Undefined>(u->weakAlias) &&
+                !isArm64ECMangledFunctionName(u->getName()))
+              continue;
+          }
+          u->setWeakAlias(ctx.symtab.addUndefined(to));
+        }
       }
 
       // If any inputs are bitcode files, the LTO code generator may create
diff --git a/lld/test/COFF/arm64ec-altnames.s b/lld/test/COFF/arm64ec-altnames.s
new file mode 100644
index 000000000000..fb28ae15895f
--- /dev/null
+++ b/lld/test/COFF/arm64ec-altnames.s
@@ -0,0 +1,109 @@
+REQUIRES: aarch64
+RUN: split-file %s %t.dir && cd %t.dir
+
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows ext.s -o ext.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows impl.s -o impl.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows impl-cpp.s -o impl-cpp.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows %S/Inputs/loadconfig-arm64ec.s -o loadconfig.obj
+
+# Ensure -alternatename can change a mangled function symbol aliasing a defined symbol (typically a guest exit thunk).
+
+RUN: lld-link -machine:arm64ec -dll -noentry -out:out1.dll ext.obj loadconfig.obj "-alternatename:#func=altsym"
+
+RUN: llvm-objdump -d out1.dll | FileCheck --check-prefix=DISASM %s
+DISASM:      0000000180001000 <.text>:
+DISASM-NEXT: 180001000: 52800020     mov     w0, #0x1                // =1
+DISASM-NEXT: 180001004: d65f03c0     ret
+DISASM-NOT: .thnk
+
+RUN: llvm-readobj --hex-dump=.test out1.dll | FileCheck --check-prefix=TESTSEC %s
+TESTSEC: 0x180004000 00100000 00100000
+
+# Ensure -alternatename can change a demangled function symbol aliasing an anti-dependency symbol.
+
+RUN: lld-link -machine:arm64ec -dll -noentry -out:out2.dll ext.obj loadconfig.obj -alternatename:func=altsym
+
+RUN: llvm-objdump -d out2.dll | FileCheck --check-prefix=DISASM2 %s
+DISASM2:      Disassembly of section .text:
+DISASM2-EMPTY:
+DISASM2-NEXT: 0000000180001000 <.text>:
+DISASM2-NEXT: 180001000: 52800020     mov     w0, #0x1                // =1
+DISASM2-NEXT: 180001004: d65f03c0     ret
+DISASM2-EMPTY:
+DISASM2-NEXT: Disassembly of section .thnk:
+DISASM2-EMPTY:
+DISASM2-NEXT: 0000000180005000 <.thnk>:
+DISASM2-NEXT: 180005000: 52800040     mov     w0, #0x2                // =2
+DISASM2-NEXT: 180005004: d65f03c0     ret
+
+RUN: llvm-readobj --hex-dump=.test out2.dll | FileCheck --check-prefix=TESTSEC2 %s
+TESTSEC2: 0x180004000 00100000 00500000
+
+# Ensure -alternatename cannot modify a demangled function symbol aliasing a defined symbol.
+
+RUN: lld-link -machine:arm64ec -dll -noentry -out:out3.dll impl.obj loadconfig.obj -alternatename:func=altsym
+RUN: llvm-objdump -d out3.dll | FileCheck --check-prefix=DISASM %s
+RUN: llvm-readobj --hex-dump=.test out3.dll | FileCheck --check-prefix=TESTSEC %s
+
+RUN: lld-link -machine:arm64ec -dll -noentry -out:out4.dll impl-cpp.obj loadconfig.obj -alternatename:func=altsym
+RUN: llvm-objdump -d out4.dll | FileCheck --check-prefix=DISASM %s
+RUN: llvm-readobj --hex-dump=.test out4.dll | FileCheck --check-prefix=TESTSEC %s
+
+#--- ext.s
+        .weak_anti_dep func
+.set func, "#func"
+        .weak_anti_dep "#func"
+.set "#func", thunksym
+
+        .section .test, "r"
+        .rva func
+        .rva "#func"
+
+        .section .thnk,"xr",discard,thunksym
+thunksym:
+        mov w0, #2
+        ret
+
+        .section .text,"xr",discard,altsym
+        .globl altsym
+altsym:
+        mov w0, #1
+        ret
+
+#--- impl.s
+        .weak_anti_dep func
+.set func, "#func"
+
+        .section .test, "r"
+        .rva func
+        .rva "#func"
+
+        .section .text,"xr",discard,"#func"
+"#func":
+        mov w0, #1
+        ret
+
+        .section .text,"xr",discard,altsym
+        .globl altsym
+altsym:
+        mov w0, #2
+        ret
+
+#--- impl-cpp.s
+        .weak_anti_dep func
+.set func, "?func@@$$hYAXXZ"
+
+        .section .test, "r"
+        .rva func
+        .rva "?func@@$$hYAXXZ"
+
+        .section .text,"xr",discard,"?func@@$$hYAXXZ"
+"?func@@$$hYAXXZ":
+        mov w0, #1
+        ret
+
+        .section .text,"xr",discard,altsym
+        .globl altsym
+altsym:
+        mov w0, #2
+        ret
diff --git a/llvm/include/llvm/IR/Mangler.h b/llvm/include/llvm/IR/Mangler.h
index 349f9e6e7523..3c3f0c6dce80 100644
--- a/llvm/include/llvm/IR/Mangler.h
+++ b/llvm/include/llvm/IR/Mangler.h
@@ -61,6 +61,12 @@ std::optional<std::string> getArm64ECMangledFunctionName(StringRef Name);
 /// mangled.
 std::optional<std::string> getArm64ECDemangledFunctionName(StringRef Name);
 
+/// Check if an ARM64EC function name is mangled.
+bool inline isArm64ECMangledFunctionName(StringRef Name) {
+  return Name[0] == '#' ||
+         (Name[0] == '?' && Name.find("$$h") != StringRef::npos);
+}
+
 } // End llvm namespace
 
 #endif
-- 
GitLab


From 86d65ae7949e0322f10e1856c5c33caa34ebfe2f Mon Sep 17 00:00:00 2001
From: T-Gruber <100079402+T-Gruber@users.noreply.github.com>
Date: Fri, 25 Oct 2024 11:59:16 +0200
Subject: [PATCH 147/693] [analyzer] Improve FieldRegion descriptive name
 (#112313)

The current implementation of MemRegion::getDescriptiveName fails for
FieldRegions whose SuperRegion is an ElementRegion. As outlined below:
```Cpp
struct val_struct { int val; };
extern struct val_struct val_struct_array[3];

void func(){
  // FieldRegion with ElementRegion as SuperRegion.
  val_struct_array[0].val;
}
```

For this special case, the expression cannot be pretty printed and must
therefore be obtained separately.
---
 clang/lib/StaticAnalyzer/Core/MemRegion.cpp   | 27 +++++++++++++----
 .../MemRegionDescriptiveNameTest.cpp          | 29 ++++++++++++++++++-
 2 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
index 0a29a050bbc2..02d1358a2001 100644
--- a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
+++ b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
@@ -722,6 +722,13 @@ std::string MemRegion::getDescriptiveName(bool UseQuotes) const {
   SmallString<50> buf;
   llvm::raw_svector_ostream os(buf);
 
+  // Enclose subject with single quotes if needed.
+  auto QuoteIfNeeded = [UseQuotes](const Twine &Subject) -> std::string {
+    if (UseQuotes)
+      return ("'" + Subject + "'").str();
+    return Subject.str();
+  };
+
   // Obtain array indices to add them to the variable name.
   const ElementRegion *ER = nullptr;
   while ((ER = R->getAs<ElementRegion>())) {
@@ -751,12 +758,20 @@ std::string MemRegion::getDescriptiveName(bool UseQuotes) const {
   }
 
   // Get variable name.
-  if (R && R->canPrintPrettyAsExpr()) {
-    R->printPrettyAsExpr(os);
-    if (UseQuotes)
-      return (llvm::Twine("'") + os.str() + ArrayIndices + "'").str();
-    else
-      return (llvm::Twine(os.str()) + ArrayIndices).str();
+  if (R) {
+    // MemRegion can be pretty printed.
+    if (R->canPrintPrettyAsExpr()) {
+      R->printPrettyAsExpr(os);
+      return QuoteIfNeeded(llvm::Twine(os.str()) + ArrayIndices);
+    }
+
+    // FieldRegion may have ElementRegion as SuperRegion.
+    if (const auto *FR = R->getAs<FieldRegion>()) {
+      std::string Super = FR->getSuperRegion()->getDescriptiveName(false);
+      if (Super.empty())
+        return "";
+      return QuoteIfNeeded(Super + "." + FR->getDecl()->getName());
+    }
   }
 
   return VariableName;
diff --git a/clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp b/clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp
index b13e7123ee52..0f6e49bf42f4 100644
--- a/clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp
+++ b/clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp
@@ -12,7 +12,6 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h"
 #include "gtest/gtest.h"
-#include <fstream>
 
 using namespace clang;
 using namespace ento;
@@ -143,4 +142,32 @@ void top() {
   EXPECT_EQ(Output, "DescriptiveNameChecker: array[x]\n");
 }
 
+TEST(MemRegionDescriptiveNameTest, FieldRegWithSuperElementReg) {
+  StringRef Code = R"cpp(
+void reportDescriptiveName(int *p);
+struct val_struct { int val; };
+extern struct val_struct val_struct_array[3];
+void top() {
+  reportDescriptiveName(&val_struct_array[0].val);
+})cpp";
+
+  std::string Output;
+  ASSERT_TRUE(runChecker(Code, Output));
+  EXPECT_EQ(Output, "DescriptiveNameChecker: val_struct_array[0].val\n");
+}
+
+TEST(MemRegionDescriptiveNameTest, FieldRegWithSuperMultidimElementReg) {
+  StringRef Code = R"cpp(
+void reportDescriptiveName(int *p);
+struct val_struct { int val; };
+extern struct val_struct val_struct_array[3][4];
+void top() {
+  reportDescriptiveName(&val_struct_array[1][2].val);
+})cpp";
+
+  std::string Output;
+  ASSERT_TRUE(runChecker(Code, Output));
+  EXPECT_EQ(Output, "DescriptiveNameChecker: val_struct_array[1][2].val\n");
+}
+
 } // namespace
-- 
GitLab


From d87964de78ce692fd132ea453c32e4435309a306 Mon Sep 17 00:00:00 2001
From: Sergio Afonso <safonsof@amd.com>
Date: Fri, 25 Oct 2024 11:30:16 +0100
Subject: [PATCH 148/693] [OpenMP][OMPIRBuilder] Error propagation across
 callbacks (#112533)

This patch implements an approach to communicate errors between the
OMPIRBuilder and its users. It introduces `llvm::Error` and
`llvm::Expected` objects to replace the values returned by callbacks
passed to `OMPIRBuilder` codegen functions. These functions then check
the result for errors when callbacks are called and forward them back to
the caller, which has the flexibility to recover, exit cleanly or dump a
stack trace.

This prevents a failed callback to leave the IR in an invalid state and
still continue the codegen process, triggering unrelated assertions or
segmentation faults. In the case of MLIR to LLVM IR translation of the
'omp' dialect, this change results in the compiler emitting errors and
exiting early instead of triggering a crash for not-yet-implemented
errors. The behavior in Clang and openmp-opt stays unchanged, since
callbacks will continue always returning 'success'.
---
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         |  32 +-
 clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp      |  13 +-
 clang/lib/CodeGen/CGStmtOpenMP.cpp            |  90 ++-
 .../llvm/Frontend/OpenMP/OMPIRBuilder.h       | 259 +++----
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 615 +++++++++++------
 llvm/lib/Transforms/IPO/OpenMPOpt.cpp         |  35 +-
 .../Frontend/OpenMPIRBuilderTest.cpp          | 649 ++++++++++++------
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 632 ++++++++++-------
 8 files changed, 1467 insertions(+), 858 deletions(-)

diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index c0931e82d987..d714af035d21 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -1192,6 +1192,7 @@ struct PushAndPopStackRAII {
       CodeGenFunction::JumpDest Dest =
           CGF.getOMPCancelDestination(OMPD_parallel);
       CGF.EmitBranchThroughCleanup(Dest);
+      return llvm::Error::success();
     };
 
     // TODO: Remove this once we emit parallel regions through the
@@ -2331,8 +2332,11 @@ void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
   auto *OMPRegionInfo =
       dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo);
   if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
-    CGF.Builder.restoreIP(OMPBuilder.createBarrier(
-        CGF.Builder, Kind, ForceSimpleCall, EmitChecks));
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+        OMPBuilder.createBarrier(CGF.Builder, Kind, ForceSimpleCall,
+                                 EmitChecks);
+    assert(AfterIP && "unexpected error creating barrier");
+    CGF.Builder.restoreIP(*AfterIP);
     return;
   }
 
@@ -5928,8 +5932,10 @@ void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
         return CGF.GenerateOpenMPCapturedStmtFunction(CS, D.getBeginLoc());
       };
 
-  OMPBuilder.emitTargetRegionFunction(EntryInfo, GenerateOutlinedFunction,
-                                      IsOffloadEntry, OutlinedFn, OutlinedFnID);
+  llvm::Error Err = OMPBuilder.emitTargetRegionFunction(
+      EntryInfo, GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
+      OutlinedFnID);
+  assert(!Err && "unexpected error creating target region");
 
   if (!OutlinedFn)
     return;
@@ -9670,9 +9676,12 @@ static void emitTargetCallKernelLaunch(
         NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,
         DynCGGroupMem, HasNoWait);
 
-    CGF.Builder.restoreIP(OMPRuntime->getOMPBuilder().emitKernelLaunch(
-        CGF.Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID,
-        RTLoc, AllocaIP));
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+        OMPRuntime->getOMPBuilder().emitKernelLaunch(
+            CGF.Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID,
+            RTLoc, AllocaIP);
+    assert(AfterIP && "unexpected error creating kernel launch");
+    CGF.Builder.restoreIP(*AfterIP);
   };
 
   if (RequiresOuterTask)
@@ -10349,9 +10358,12 @@ void CGOpenMPRuntime::emitTargetDataCalls(
   InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(),
                           CGF.Builder.GetInsertPoint());
   llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CodeGenIP);
-  CGF.Builder.restoreIP(OMPBuilder.createTargetData(
-      OmpLoc, AllocaIP, CodeGenIP, DeviceID, IfCondVal, Info, GenMapInfoCB,
-      /*MapperFunc=*/nullptr, BodyCB, DeviceAddrCB, CustomMapperCB, RTLoc));
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      OMPBuilder.createTargetData(
+          OmpLoc, AllocaIP, CodeGenIP, DeviceID, IfCondVal, Info, GenMapInfoCB,
+          /*MapperFunc=*/nullptr, BodyCB, DeviceAddrCB, CustomMapperCB, RTLoc);
+  assert(AfterIP && "unexpected error creating target data");
+  CGF.Builder.restoreIP(*AfterIP);
 }
 
 void CGOpenMPRuntime::emitTargetDataStandAloneCall(
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index c66d5d11b0bb..598b946ad88d 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -1753,11 +1753,14 @@ void CGOpenMPRuntimeGPU::emitReduction(
     Idx++;
   }
 
-  CGF.Builder.restoreIP(OMPBuilder.createReductionsGPU(
-      OmpLoc, AllocaIP, CodeGenIP, ReductionInfos, false, TeamsReduction,
-      DistributeReduction, llvm::OpenMPIRBuilder::ReductionGenCBKind::Clang,
-      CGF.getTarget().getGridValue(), C.getLangOpts().OpenMPCUDAReductionBufNum,
-      RTLoc));
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      OMPBuilder.createReductionsGPU(
+          OmpLoc, AllocaIP, CodeGenIP, ReductionInfos, false, TeamsReduction,
+          DistributeReduction, llvm::OpenMPIRBuilder::ReductionGenCBKind::Clang,
+          CGF.getTarget().getGridValue(),
+          C.getLangOpts().OpenMPCUDAReductionBufNum, RTLoc);
+  assert(AfterIP && "unexpected error creating GPU reductions");
+  CGF.Builder.restoreIP(*AfterIP);
   return;
 }
 
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 0da7855ab05c..1c32a675380c 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -1809,6 +1809,7 @@ void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
     // thus calls destructors etc.
     auto FiniCB = [this](InsertPointTy IP) {
       OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);
+      return llvm::Error::success();
     };
 
     // Privatization callback that performs appropriate action for
@@ -1831,15 +1832,18 @@ void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
                                InsertPointTy CodeGenIP) {
       OMPBuilderCBHelpers::EmitOMPOutlinedRegionBody(
           *this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP, "parallel");
+      return llvm::Error::success();
     };
 
     CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
     CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
     llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
         AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
-    Builder.restoreIP(
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
         OMPBuilder.createParallel(Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
-                                  IfCond, NumThreads, ProcBind, S.hasCancel()));
+                                  IfCond, NumThreads, ProcBind, S.hasCancel());
+    assert(AfterIP && "unexpected error creating parallel");
+    Builder.restoreIP(*AfterIP);
     return;
   }
 
@@ -2128,9 +2132,13 @@ void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) {
 
     RunCleanupsScope BodyScope(*this);
     EmitStmt(BodyStmt);
+    return llvm::Error::success();
   };
-  llvm::CanonicalLoopInfo *CL =
+
+  llvm::Expected<llvm::CanonicalLoopInfo *> Result =
       OMPBuilder.createCanonicalLoop(Builder, BodyGen, DistVal);
+  assert(Result && "unexpected error creating canonical loop");
+  llvm::CanonicalLoopInfo *CL = *Result;
 
   // Finish up the loop.
   Builder.restoreIP(CL->getAfterIP());
@@ -4016,11 +4024,13 @@ static void emitOMPForDirective(const OMPLoopDirective &S, CodeGenFunction &CGF,
           CGM.getOpenMPRuntime().getOMPBuilder();
       llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
           CGF.AllocaInsertPt->getParent(), CGF.AllocaInsertPt->getIterator());
-      OMPBuilder.applyWorkshareLoop(
-          CGF.Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
-          SchedKind, ChunkSize, /*HasSimdModifier=*/false,
-          /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false,
-          /*HasOrderedClause=*/false);
+      llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+          OMPBuilder.applyWorkshareLoop(
+              CGF.Builder.getCurrentDebugLocation(), CLI, AllocaIP,
+              NeedsBarrier, SchedKind, ChunkSize, /*HasSimdModifier=*/false,
+              /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false,
+              /*HasOrderedClause=*/false);
+      assert(AfterIP && "unexpected error creating workshare loop");
       return;
     }
 
@@ -4257,6 +4267,7 @@ void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
 
     auto FiniCB = [this](InsertPointTy IP) {
       OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);
+      return llvm::Error::success();
     };
 
     const CapturedStmt *ICS = S.getInnermostCapturedStmt();
@@ -4269,6 +4280,7 @@ void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
                                          InsertPointTy CodeGenIP) {
           OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
               *this, SubStmt, AllocaIP, CodeGenIP, "section");
+          return llvm::Error::success();
         };
         SectionCBVector.push_back(SectionCB);
       }
@@ -4277,6 +4289,7 @@ void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
                                             InsertPointTy CodeGenIP) {
         OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
             *this, CapturedStmt, AllocaIP, CodeGenIP, "section");
+        return llvm::Error::success();
       };
       SectionCBVector.push_back(SectionCB);
     }
@@ -4298,9 +4311,12 @@ void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
     CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
     llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
         AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
-    Builder.restoreIP(OMPBuilder.createSections(
-        Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
-        S.getSingleClause<OMPNowaitClause>()));
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+        OMPBuilder.createSections(Builder, AllocaIP, SectionCBVector, PrivCB,
+                                  FiniCB, S.hasCancel(),
+                                  S.getSingleClause<OMPNowaitClause>());
+    assert(AfterIP && "unexpected error creating sections");
+    Builder.restoreIP(*AfterIP);
     return;
   }
   {
@@ -4326,17 +4342,22 @@ void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
     const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
     auto FiniCB = [this](InsertPointTy IP) {
       OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);
+      return llvm::Error::success();
     };
 
     auto BodyGenCB = [SectionRegionBodyStmt, this](InsertPointTy AllocaIP,
                                                    InsertPointTy CodeGenIP) {
       OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
           *this, SectionRegionBodyStmt, AllocaIP, CodeGenIP, "section");
+      return llvm::Error::success();
     };
 
     LexicalScope Scope(*this, S.getSourceRange());
     EmitStopPoint(&S);
-    Builder.restoreIP(OMPBuilder.createSection(Builder, BodyGenCB, FiniCB));
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+        OMPBuilder.createSection(Builder, BodyGenCB, FiniCB);
+    assert(AfterIP && "unexpected error creating section");
+    Builder.restoreIP(*AfterIP);
 
     return;
   }
@@ -4407,17 +4428,22 @@ void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
 
     auto FiniCB = [this](InsertPointTy IP) {
       OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);
+      return llvm::Error::success();
     };
 
     auto BodyGenCB = [MasterRegionBodyStmt, this](InsertPointTy AllocaIP,
                                                   InsertPointTy CodeGenIP) {
       OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
           *this, MasterRegionBodyStmt, AllocaIP, CodeGenIP, "master");
+      return llvm::Error::success();
     };
 
     LexicalScope Scope(*this, S.getSourceRange());
     EmitStopPoint(&S);
-    Builder.restoreIP(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+        OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB);
+    assert(AfterIP && "unexpected error creating master");
+    Builder.restoreIP(*AfterIP);
 
     return;
   }
@@ -4453,18 +4479,22 @@ void CodeGenFunction::EmitOMPMaskedDirective(const OMPMaskedDirective &S) {
 
     auto FiniCB = [this](InsertPointTy IP) {
       OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);
+      return llvm::Error::success();
     };
 
     auto BodyGenCB = [MaskedRegionBodyStmt, this](InsertPointTy AllocaIP,
                                                   InsertPointTy CodeGenIP) {
       OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
           *this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP, "masked");
+      return llvm::Error::success();
     };
 
     LexicalScope Scope(*this, S.getSourceRange());
     EmitStopPoint(&S);
-    Builder.restoreIP(
-        OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, FilterVal));
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+        OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, FilterVal);
+    assert(AfterIP && "unexpected error creating masked");
+    Builder.restoreIP(*AfterIP);
 
     return;
   }
@@ -4493,19 +4523,23 @@ void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
 
     auto FiniCB = [this](InsertPointTy IP) {
       OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);
+      return llvm::Error::success();
     };
 
     auto BodyGenCB = [CriticalRegionBodyStmt, this](InsertPointTy AllocaIP,
                                                     InsertPointTy CodeGenIP) {
       OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
           *this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP, "critical");
+      return llvm::Error::success();
     };
 
     LexicalScope Scope(*this, S.getSourceRange());
     EmitStopPoint(&S);
-    Builder.restoreIP(OMPBuilder.createCritical(
-        Builder, BodyGenCB, FiniCB, S.getDirectiveName().getAsString(),
-        HintInst));
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+        OMPBuilder.createCritical(Builder, BodyGenCB, FiniCB,
+                                  S.getDirectiveName().getAsString(), HintInst);
+    assert(AfterIP && "unexpected error creating critical");
+    Builder.restoreIP(*AfterIP);
 
     return;
   }
@@ -5464,11 +5498,15 @@ void CodeGenFunction::EmitOMPTaskgroupDirective(
                                InsertPointTy CodeGenIP) {
       Builder.restoreIP(CodeGenIP);
       EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
+      return llvm::Error::success();
     };
     CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
     if (!CapturedStmtInfo)
       CapturedStmtInfo = &CapStmtInfo;
-    Builder.restoreIP(OMPBuilder.createTaskgroup(Builder, AllocaIP, BodyGenCB));
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+        OMPBuilder.createTaskgroup(Builder, AllocaIP, BodyGenCB);
+    assert(AfterIP && "unexpected error creating taskgroup");
+    Builder.restoreIP(*AfterIP);
     return;
   }
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
@@ -6041,6 +6079,7 @@ void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
 
       auto FiniCB = [this](InsertPointTy IP) {
         OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);
+        return llvm::Error::success();
       };
 
       auto BodyGenCB = [&S, C, this](InsertPointTy AllocaIP,
@@ -6064,11 +6103,14 @@ void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
           OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
               *this, CS->getCapturedStmt(), AllocaIP, CodeGenIP, "ordered");
         }
+        return llvm::Error::success();
       };
 
       OMPLexicalScope Scope(*this, S, OMPD_unknown);
-      Builder.restoreIP(
-          OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, !C));
+      llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+          OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, !C);
+      assert(AfterIP && "unexpected error creating ordered");
+      Builder.restoreIP(*AfterIP);
     }
     return;
   }
@@ -7344,8 +7386,10 @@ void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
       if (IfCond)
         IfCondition = EmitScalarExpr(IfCond,
                                      /*IgnoreResultAssign=*/true);
-      return Builder.restoreIP(
-          OMPBuilder.createCancel(Builder, IfCondition, S.getCancelRegion()));
+      llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+          OMPBuilder.createCancel(Builder, IfCondition, S.getCancelRegion());
+      assert(AfterIP && "unexpected error creating cancel");
+      return Builder.restoreIP(*AfterIP);
     }
   }
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index c4735ec41e71..3afb9d84278e 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -520,6 +520,9 @@ public:
   /// Type used throughout for insertion points.
   using InsertPointTy = IRBuilder<>::InsertPoint;
 
+  /// Type used to represent an insertion point or an error value.
+  using InsertPointOrErrorTy = Expected<InsertPointTy>;
+
   /// Get the create a name using the platform specific separators.
   /// \param Parts parts of the final name that needs separation
   /// The created name has a first separator between the first and second part
@@ -538,7 +541,7 @@ public:
   /// A finalize callback knows about all objects that need finalization, e.g.
   /// destruction, when the scope of the currently generated construct is left
   /// at the time, and location, the callback is invoked.
-  using FinalizeCallbackTy = std::function<void(InsertPointTy CodeGenIP)>;
+  using FinalizeCallbackTy = std::function<Error(InsertPointTy CodeGenIP)>;
 
   struct FinalizationInfo {
     /// The finalization callback provided by the last in-flight invocation of
@@ -589,15 +592,19 @@ public:
   ///                 not be split.
   /// \param CodeGenIP is the insertion point at which the body code should be
   ///                  placed.
+  ///
+  /// \return an error, if any were triggered during execution.
   using BodyGenCallbackTy =
-      function_ref<void(InsertPointTy AllocaIP, InsertPointTy CodeGenIP)>;
+      function_ref<Error(InsertPointTy AllocaIP, InsertPointTy CodeGenIP)>;
 
   // This is created primarily for sections construct as llvm::function_ref
   // (BodyGenCallbackTy) is not storable (as described in the comments of
   // function_ref class - function_ref contains non-ownable reference
   // to the callable.
+  ///
+  /// \return an error, if any were triggered during execution.
   using StorableBodyGenCallbackTy =
-      std::function<void(InsertPointTy AllocaIP, InsertPointTy CodeGenIP)>;
+      std::function<Error(InsertPointTy AllocaIP, InsertPointTy CodeGenIP)>;
 
   /// Callback type for loop body code generation.
   ///
@@ -607,8 +614,10 @@ public:
   ///                  terminated with an unconditional branch to the loop
   ///                  latch.
   /// \param IndVar    is the induction variable usable at the insertion point.
+  ///
+  /// \return an error, if any were triggered during execution.
   using LoopBodyGenCallbackTy =
-      function_ref<void(InsertPointTy CodeGenIP, Value *IndVar)>;
+      function_ref<Error(InsertPointTy CodeGenIP, Value *IndVar)>;
 
   /// Callback type for variable privatization (think copy & default
   /// constructor).
@@ -628,7 +637,7 @@ public:
   ///
   /// \returns The new insertion point where code generation continues and
   ///          \p ReplVal the replacement value.
-  using PrivatizeCallbackTy = function_ref<InsertPointTy(
+  using PrivatizeCallbackTy = function_ref<InsertPointOrErrorTy(
       InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &Original,
       Value &Inner, Value *&ReplVal)>;
 
@@ -658,9 +667,10 @@ public:
   /// \param ThreadID Optional parameter to pass in any existing ThreadID value.
   ///
   /// \returns The insertion point after the barrier.
-  InsertPointTy createBarrier(const LocationDescription &Loc,
-                              omp::Directive Kind, bool ForceSimpleCall = false,
-                              bool CheckCancelFlag = true);
+  InsertPointOrErrorTy createBarrier(const LocationDescription &Loc,
+                                     omp::Directive Kind,
+                                     bool ForceSimpleCall = false,
+                                     bool CheckCancelFlag = true);
 
   /// Generator for '#omp cancel'
   ///
@@ -669,8 +679,9 @@ public:
   /// \param CanceledDirective The kind of directive that is cancled.
   ///
   /// \returns The insertion point after the barrier.
-  InsertPointTy createCancel(const LocationDescription &Loc, Value *IfCondition,
-                             omp::Directive CanceledDirective);
+  InsertPointOrErrorTy createCancel(const LocationDescription &Loc,
+                                    Value *IfCondition,
+                                    omp::Directive CanceledDirective);
 
   /// Generator for '#omp parallel'
   ///
@@ -685,7 +696,7 @@ public:
   /// \param IsCancellable Flag to indicate a cancellable parallel region.
   ///
   /// \returns The insertion position *after* the parallel.
-  IRBuilder<>::InsertPoint
+  InsertPointOrErrorTy
   createParallel(const LocationDescription &Loc, InsertPointTy AllocaIP,
                  BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
                  FinalizeCallbackTy FiniCB, Value *IfCondition,
@@ -711,10 +722,10 @@ public:
   ///
   /// \returns An object representing the created control flow structure which
   ///          can be used for loop-associated directives.
-  CanonicalLoopInfo *createCanonicalLoop(const LocationDescription &Loc,
-                                         LoopBodyGenCallbackTy BodyGenCB,
-                                         Value *TripCount,
-                                         const Twine &Name = "loop");
+  Expected<CanonicalLoopInfo *>
+  createCanonicalLoop(const LocationDescription &Loc,
+                      LoopBodyGenCallbackTy BodyGenCB, Value *TripCount,
+                      const Twine &Name = "loop");
 
   /// Generator for the control flow structure of an OpenMP canonical loop.
   ///
@@ -764,12 +775,10 @@ public:
   ///
   /// \returns An object representing the created control flow structure which
   ///          can be used for loop-associated directives.
-  CanonicalLoopInfo *createCanonicalLoop(const LocationDescription &Loc,
-                                         LoopBodyGenCallbackTy BodyGenCB,
-                                         Value *Start, Value *Stop, Value *Step,
-                                         bool IsSigned, bool InclusiveStop,
-                                         InsertPointTy ComputeIP = {},
-                                         const Twine &Name = "loop");
+  Expected<CanonicalLoopInfo *> createCanonicalLoop(
+      const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
+      Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
+      InsertPointTy ComputeIP = {}, const Twine &Name = "loop");
 
   /// Collapse a loop nest into a single loop.
   ///
@@ -996,9 +1005,10 @@ private:
   ///                     the loop.
   ///
   /// \returns Point where to insert code after the workshare construct.
-  InsertPointTy applyStaticWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
-                                         InsertPointTy AllocaIP,
-                                         bool NeedsBarrier);
+  InsertPointOrErrorTy applyStaticWorkshareLoop(DebugLoc DL,
+                                                CanonicalLoopInfo *CLI,
+                                                InsertPointTy AllocaIP,
+                                                bool NeedsBarrier);
 
   /// Modifies the canonical loop a statically-scheduled workshare loop with a
   /// user-specified chunk size.
@@ -1013,11 +1023,11 @@ private:
   /// \param ChunkSize    The user-specified chunk size.
   ///
   /// \returns Point where to insert code after the workshare construct.
-  InsertPointTy applyStaticChunkedWorkshareLoop(DebugLoc DL,
-                                                CanonicalLoopInfo *CLI,
-                                                InsertPointTy AllocaIP,
-                                                bool NeedsBarrier,
-                                                Value *ChunkSize);
+  InsertPointOrErrorTy applyStaticChunkedWorkshareLoop(DebugLoc DL,
+                                                       CanonicalLoopInfo *CLI,
+                                                       InsertPointTy AllocaIP,
+                                                       bool NeedsBarrier,
+                                                       Value *ChunkSize);
 
   /// Modifies the canonical loop to be a dynamically-scheduled workshare loop.
   ///
@@ -1039,11 +1049,12 @@ private:
   ///                 scheduling. If \p nullptr, defaults to 1.
   ///
   /// \returns Point where to insert code after the workshare construct.
-  InsertPointTy applyDynamicWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
-                                          InsertPointTy AllocaIP,
-                                          omp::OMPScheduleType SchedType,
-                                          bool NeedsBarrier,
-                                          Value *Chunk = nullptr);
+  InsertPointOrErrorTy applyDynamicWorkshareLoop(DebugLoc DL,
+                                                 CanonicalLoopInfo *CLI,
+                                                 InsertPointTy AllocaIP,
+                                                 omp::OMPScheduleType SchedType,
+                                                 bool NeedsBarrier,
+                                                 Value *Chunk = nullptr);
 
   /// Create alternative version of the loop to support if clause
   ///
@@ -1094,7 +1105,7 @@ public:
   ///                 It corresponds to type of loop workshare OpenMP pragma.
   ///
   /// \returns Point where to insert code after the workshare construct.
-  InsertPointTy applyWorkshareLoop(
+  InsertPointOrErrorTy applyWorkshareLoop(
       DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
       bool NeedsBarrier,
       llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default,
@@ -1251,20 +1262,21 @@ public:
   ///                    cannot be resumed until execution of the structured
   ///                    block that is associated with the generated task is
   ///                    completed.
-  InsertPointTy createTask(const LocationDescription &Loc,
-                           InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB,
-                           bool Tied = true, Value *Final = nullptr,
-                           Value *IfCondition = nullptr,
-                           SmallVector<DependData> Dependencies = {});
+  InsertPointOrErrorTy createTask(const LocationDescription &Loc,
+                                  InsertPointTy AllocaIP,
+                                  BodyGenCallbackTy BodyGenCB, bool Tied = true,
+                                  Value *Final = nullptr,
+                                  Value *IfCondition = nullptr,
+                                  SmallVector<DependData> Dependencies = {});
 
   /// Generator for the taskgroup construct
   ///
   /// \param Loc The location where the taskgroup construct was encountered.
   /// \param AllocaIP The insertion point to be used for alloca instructions.
   /// \param BodyGenCB Callback that will generate the region code.
-  InsertPointTy createTaskgroup(const LocationDescription &Loc,
-                                InsertPointTy AllocaIP,
-                                BodyGenCallbackTy BodyGenCB);
+  InsertPointOrErrorTy createTaskgroup(const LocationDescription &Loc,
+                                       InsertPointTy AllocaIP,
+                                       BodyGenCallbackTy BodyGenCB);
 
   using FileIdentifierInfoCallbackTy =
       std::function<std::tuple<std::string, uint64_t>()>;
@@ -1302,15 +1314,15 @@ public:
   /// \param CodeGenIP InsertPoint for CodeGen.
   /// \param LHS Pass in the LHS Value to be used for CodeGen.
   /// \param RHS Pass in the RHS Value to be used for CodeGen.
-  using ReductionGenCBTy = std::function<InsertPointTy(
+  using ReductionGenCBTy = std::function<InsertPointOrErrorTy(
       InsertPointTy CodeGenIP, Value *LHS, Value *RHS, Value *&Res)>;
 
   /// Functions used to generate atomic reductions. Such functions take two
   /// Values representing pointers to LHS and RHS of the reduction, as well as
   /// the element type of these pointers. They are expected to atomically
   /// update the LHS to the reduced value.
-  using ReductionGenAtomicCBTy =
-      std::function<InsertPointTy(InsertPointTy, Type *, Value *, Value *)>;
+  using ReductionGenAtomicCBTy = std::function<InsertPointOrErrorTy(
+      InsertPointTy, Type *, Value *, Value *)>;
 
   /// Enum class for reduction evaluation types scalar, complex and aggregate.
   enum class EvalKind { Scalar, Complex, Aggregate };
@@ -1510,9 +1522,10 @@ private:
   ///                  need to be copied to the new function.
   ///
   /// \return The InterWarpCopy function.
-  Function *emitInterWarpCopyFunction(const LocationDescription &Loc,
-                                      ArrayRef<ReductionInfo> ReductionInfos,
-                                      AttributeList FuncAttrs);
+  Expected<Function *>
+  emitInterWarpCopyFunction(const LocationDescription &Loc,
+                            ArrayRef<ReductionInfo> ReductionInfos,
+                            AttributeList FuncAttrs);
 
   /// This function emits a helper that copies all the reduction variables from
   /// the team into the provided global buffer for the reduction variables.
@@ -1604,7 +1617,7 @@ private:
   ///                  need to be copied to the new function.
   ///
   /// \return The reduction function.
-  Function *createReductionFunction(
+  Expected<Function *> createReductionFunction(
       StringRef ReducerName, ArrayRef<ReductionInfo> ReductionInfos,
       ReductionGenCBKind ReductionGenCBKind = ReductionGenCBKind::MLIR,
       AttributeList FuncAttrs = {});
@@ -1871,7 +1884,7 @@ public:
   /// \param ReductionBufNum    Optional OpenMPCUDAReductionBufNumValue to be
   /// used for teams reduction.
   /// \param SrcLocInfo         Source location information global.
-  InsertPointTy createReductionsGPU(
+  InsertPointOrErrorTy createReductionsGPU(
       const LocationDescription &Loc, InsertPointTy AllocaIP,
       InsertPointTy CodeGenIP, ArrayRef<ReductionInfo> ReductionInfos,
       bool IsNoWait = false, bool IsTeamsReduction = false,
@@ -1943,10 +1956,11 @@ public:
   /// \param IsNoWait           A flag set if the reduction is marked as nowait.
   /// \param IsByRef            A flag set if the reduction is using reference
   /// or direct value.
-  InsertPointTy createReductions(const LocationDescription &Loc,
-                                 InsertPointTy AllocaIP,
-                                 ArrayRef<ReductionInfo> ReductionInfos,
-                                 ArrayRef<bool> IsByRef, bool IsNoWait = false);
+  InsertPointOrErrorTy createReductions(const LocationDescription &Loc,
+                                        InsertPointTy AllocaIP,
+                                        ArrayRef<ReductionInfo> ReductionInfos,
+                                        ArrayRef<bool> IsByRef,
+                                        bool IsNoWait = false);
 
   ///}
 
@@ -2002,9 +2016,11 @@ public:
   /// \param CancelFlag Flag indicating if the cancellation is performed.
   /// \param CanceledDirective The kind of directive that is cancled.
   /// \param ExitCB Extra code to be generated in the exit block.
-  void emitCancelationCheckImpl(Value *CancelFlag,
-                                omp::Directive CanceledDirective,
-                                FinalizeCallbackTy ExitCB = {});
+  ///
+  /// \return an error, if any were triggered during execution.
+  Error emitCancelationCheckImpl(Value *CancelFlag,
+                                 omp::Directive CanceledDirective,
+                                 FinalizeCallbackTy ExitCB = {});
 
   /// Generate a target region entry call.
   ///
@@ -2135,8 +2151,10 @@ public:
   /// } else {
   ///   ElseGen();
   /// }
-  void emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
-                    BodyGenCallbackTy ElseGen, InsertPointTy AllocaIP = {});
+  ///
+  /// \return an error, if any were triggered during execution.
+  Error emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
+                     BodyGenCallbackTy ElseGen, InsertPointTy AllocaIP = {});
 
   /// Create the global variable holding the offload mappings information.
   GlobalVariable *createOffloadMaptypes(SmallVectorImpl<uint64_t> &Mappings,
@@ -2340,7 +2358,8 @@ public:
   /// is executed when the kernel launch fails. It takes an insertion point as
   /// parameter where the code should be emitted. It returns an insertion point
   /// that points right after after the emitted code.
-  using EmitFallbackCallbackTy = function_ref<InsertPointTy(InsertPointTy)>;
+  using EmitFallbackCallbackTy =
+      function_ref<InsertPointOrErrorTy(InsertPointTy)>;
 
   /// Generate a target region entry call and host fallback call.
   ///
@@ -2352,7 +2371,7 @@ public:
   /// \param DeviceID Identifier for the device via the 'device' clause.
   /// \param RTLoc Source location identifier
   /// \param AllocaIP The insertion point to be used for alloca instructions.
-  InsertPointTy
+  InsertPointOrErrorTy
   emitKernelLaunch(const LocationDescription &Loc, Value *OutlinedFnID,
                    EmitFallbackCallbackTy EmitTargetCallFallbackCB,
                    TargetKernelArgs &Args, Value *DeviceID, Value *RTLoc,
@@ -2366,9 +2385,11 @@ public:
   /// \param RTLoc Source location identifier
   /// \Param TargetTaskAllocaIP Insertion point for the alloca block of the
   ///        generated task.
+  ///
+  /// \return an error, if any were triggered during execution.
   using TargetTaskBodyCallbackTy =
-      function_ref<void(Value *DeviceID, Value *RTLoc,
-                        IRBuilderBase::InsertPoint TargetTaskAllocaIP)>;
+      function_ref<Error(Value *DeviceID, Value *RTLoc,
+                         IRBuilderBase::InsertPoint TargetTaskAllocaIP)>;
 
   /// Generate a target-task for the target construct
   ///
@@ -2380,7 +2401,7 @@ public:
   ///        dependencies as specified by the 'depend' clause.
   /// \param HasNoWait True if the target construct had 'nowait' on it, false
   ///        otherwise
-  InsertPointTy emitTargetTask(
+  InsertPointOrErrorTy emitTargetTask(
       TargetTaskBodyCallbackTy TaskBodyCB, Value *DeviceID, Value *RTLoc,
       OpenMPIRBuilder::InsertPointTy AllocaIP,
       const SmallVector<llvm::OpenMPIRBuilder::DependData> &Dependencies,
@@ -2478,11 +2499,11 @@ public:
   /// \param CPFuncs copy functions to use for each copyprivate variable.
   ///
   /// \returns The insertion position *after* the single call.
-  InsertPointTy createSingle(const LocationDescription &Loc,
-                             BodyGenCallbackTy BodyGenCB,
-                             FinalizeCallbackTy FiniCB, bool IsNowait,
-                             ArrayRef<llvm::Value *> CPVars = {},
-                             ArrayRef<llvm::Function *> CPFuncs = {});
+  InsertPointOrErrorTy createSingle(const LocationDescription &Loc,
+                                    BodyGenCallbackTy BodyGenCB,
+                                    FinalizeCallbackTy FiniCB, bool IsNowait,
+                                    ArrayRef<llvm::Value *> CPVars = {},
+                                    ArrayRef<llvm::Function *> CPFuncs = {});
 
   /// Generator for '#omp master'
   ///
@@ -2491,9 +2512,9 @@ public:
   /// \param FiniCB Callback to finalize variable copies.
   ///
   /// \returns The insertion position *after* the master.
-  InsertPointTy createMaster(const LocationDescription &Loc,
-                             BodyGenCallbackTy BodyGenCB,
-                             FinalizeCallbackTy FiniCB);
+  InsertPointOrErrorTy createMaster(const LocationDescription &Loc,
+                                    BodyGenCallbackTy BodyGenCB,
+                                    FinalizeCallbackTy FiniCB);
 
   /// Generator for '#omp masked'
   ///
@@ -2502,9 +2523,9 @@ public:
   /// \param FiniCB Callback to finialize variable copies.
   ///
   /// \returns The insertion position *after* the masked.
-  InsertPointTy createMasked(const LocationDescription &Loc,
-                             BodyGenCallbackTy BodyGenCB,
-                             FinalizeCallbackTy FiniCB, Value *Filter);
+  InsertPointOrErrorTy createMasked(const LocationDescription &Loc,
+                                    BodyGenCallbackTy BodyGenCB,
+                                    FinalizeCallbackTy FiniCB, Value *Filter);
 
   /// Generator for '#omp critical'
   ///
@@ -2515,10 +2536,10 @@ public:
   /// \param HintInst Hint Instruction for hint clause associated with critical
   ///
   /// \returns The insertion position *after* the critical.
-  InsertPointTy createCritical(const LocationDescription &Loc,
-                               BodyGenCallbackTy BodyGenCB,
-                               FinalizeCallbackTy FiniCB,
-                               StringRef CriticalName, Value *HintInst);
+  InsertPointOrErrorTy createCritical(const LocationDescription &Loc,
+                                      BodyGenCallbackTy BodyGenCB,
+                                      FinalizeCallbackTy FiniCB,
+                                      StringRef CriticalName, Value *HintInst);
 
   /// Generator for '#omp ordered depend (source | sink)'
   ///
@@ -2544,10 +2565,10 @@ public:
   /// otherwise, with simd clause;
   ///
   /// \returns The insertion position *after* the ordered.
-  InsertPointTy createOrderedThreadsSimd(const LocationDescription &Loc,
-                                         BodyGenCallbackTy BodyGenCB,
-                                         FinalizeCallbackTy FiniCB,
-                                         bool IsThreads);
+  InsertPointOrErrorTy createOrderedThreadsSimd(const LocationDescription &Loc,
+                                                BodyGenCallbackTy BodyGenCB,
+                                                FinalizeCallbackTy FiniCB,
+                                                bool IsThreads);
 
   /// Generator for '#omp sections'
   ///
@@ -2560,12 +2581,11 @@ public:
   /// \param IsNowait If true, barrier - to ensure all sections are executed
   /// before moving forward will not be generated.
   /// \returns The insertion position *after* the sections.
-  InsertPointTy createSections(const LocationDescription &Loc,
-                               InsertPointTy AllocaIP,
-                               ArrayRef<StorableBodyGenCallbackTy> SectionCBs,
-                               PrivatizeCallbackTy PrivCB,
-                               FinalizeCallbackTy FiniCB, bool IsCancellable,
-                               bool IsNowait);
+  InsertPointOrErrorTy
+  createSections(const LocationDescription &Loc, InsertPointTy AllocaIP,
+                 ArrayRef<StorableBodyGenCallbackTy> SectionCBs,
+                 PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB,
+                 bool IsCancellable, bool IsNowait);
 
   /// Generator for '#omp section'
   ///
@@ -2573,9 +2593,9 @@ public:
   /// \param BodyGenCB Callback that will generate the region body code.
   /// \param FiniCB Callback to finalize variable copies.
   /// \returns The insertion position *after* the section.
-  InsertPointTy createSection(const LocationDescription &Loc,
-                              BodyGenCallbackTy BodyGenCB,
-                              FinalizeCallbackTy FiniCB);
+  InsertPointOrErrorTy createSection(const LocationDescription &Loc,
+                                     BodyGenCallbackTy BodyGenCB,
+                                     FinalizeCallbackTy FiniCB);
 
   /// Generator for `#omp teams`
   ///
@@ -2589,7 +2609,7 @@ public:
   ///        contention group created by each team.
   /// \param IfExpr is the integer argument value of the if condition on the
   ///        teams clause.
-  InsertPointTy
+  InsertPointOrErrorTy
   createTeams(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
               Value *NumTeamsLower = nullptr, Value *NumTeamsUpper = nullptr,
               Value *ThreadLimit = nullptr, Value *IfExpr = nullptr);
@@ -2764,7 +2784,8 @@ private:
 
 public:
   /// Functions used to generate a function with the given name.
-  using FunctionGenCallback = std::function<Function *(StringRef FunctionName)>;
+  using FunctionGenCallback =
+      std::function<Expected<Function *>(StringRef FunctionName)>;
 
   /// Create a unique name for the entry function using the source location
   /// information of the current target region. The name will be something like:
@@ -2797,10 +2818,10 @@ public:
   /// \param GenerateFunctionCallback The callback function to generate the code
   /// \param OutlinedFunction Pointer to the outlined function
   /// \param EntryFnIDName Name of the ID o be created
-  void emitTargetRegionFunction(TargetRegionEntryInfo &EntryInfo,
-                                FunctionGenCallback &GenerateFunctionCallback,
-                                bool IsOffloadEntry, Function *&OutlinedFn,
-                                Constant *&OutlinedFnID);
+  Error emitTargetRegionFunction(TargetRegionEntryInfo &EntryInfo,
+                                 FunctionGenCallback &GenerateFunctionCallback,
+                                 bool IsOffloadEntry, Function *&OutlinedFn,
+                                 Constant *&OutlinedFnID);
 
   /// Registers the given function and sets up the attribtues of the function
   /// Returns the FunctionID.
@@ -2851,22 +2872,22 @@ public:
   /// use_device_ptr and use_device_addr.
   /// \param CustomMapperCB Optional callback to generate code related to
   /// custom mappers.
-  OpenMPIRBuilder::InsertPointTy createTargetData(
+  InsertPointOrErrorTy createTargetData(
       const LocationDescription &Loc, InsertPointTy AllocaIP,
       InsertPointTy CodeGenIP, Value *DeviceID, Value *IfCond,
       TargetDataInfo &Info, GenMapInfoCallbackTy GenMapInfoCB,
       omp::RuntimeFunction *MapperFunc = nullptr,
-      function_ref<InsertPointTy(InsertPointTy CodeGenIP,
-                                 BodyGenTy BodyGenType)>
+      function_ref<InsertPointOrErrorTy(InsertPointTy CodeGenIP,
+                                        BodyGenTy BodyGenType)>
           BodyGenCB = nullptr,
       function_ref<void(unsigned int, Value *)> DeviceAddrCB = nullptr,
       function_ref<Value *(unsigned int)> CustomMapperCB = nullptr,
       Value *SrcLocInfo = nullptr);
 
-  using TargetBodyGenCallbackTy = function_ref<InsertPointTy(
+  using TargetBodyGenCallbackTy = function_ref<InsertPointOrErrorTy(
       InsertPointTy AllocaIP, InsertPointTy CodeGenIP)>;
 
-  using TargetGenArgAccessorsCallbackTy = function_ref<InsertPointTy(
+  using TargetGenArgAccessorsCallbackTy = function_ref<InsertPointOrErrorTy(
       Argument &Arg, Value *Input, Value *&RetVal, InsertPointTy AllocaIP,
       InsertPointTy CodeGenIP)>;
 
@@ -2887,7 +2908,7 @@ public:
   /// \param Dependencies A vector of DependData objects that carry
   // dependency information as passed in the depend clause
   // \param HasNowait Whether the target construct has a `nowait` clause or not.
-  InsertPointTy createTarget(
+  InsertPointOrErrorTy createTarget(
       const LocationDescription &Loc, bool IsOffloadEntry,
       OpenMPIRBuilder::InsertPointTy AllocaIP,
       OpenMPIRBuilder::InsertPointTy CodeGenIP,
@@ -2969,10 +2990,10 @@ private:
   ///        should be called.
   ///
   /// \return The insertion position in exit block
-  InsertPointTy emitCommonDirectiveExit(omp::Directive OMPD,
-                                        InsertPointTy FinIP,
-                                        Instruction *ExitCall,
-                                        bool HasFinalize = true);
+  InsertPointOrErrorTy emitCommonDirectiveExit(omp::Directive OMPD,
+                                               InsertPointTy FinIP,
+                                               Instruction *ExitCall,
+                                               bool HasFinalize = true);
 
   /// Common Interface to generate OMP inlined regions
   ///
@@ -2990,8 +3011,7 @@ private:
   /// \param IsCancellable if HasFinalize is set to true, indicate if the
   ///        the directive should be cancellable.
   /// \return The insertion point after the region
-
-  InsertPointTy
+  InsertPointOrErrorTy
   EmitOMPInlinedRegion(omp::Directive OMPD, Instruction *EntryCall,
                        Instruction *ExitCall, BodyGenCallbackTy BodyGenCB,
                        FinalizeCallbackTy FiniCB, bool Conditional = false,
@@ -3027,7 +3047,7 @@ private:
   ///
   /// \returns Value to update X to.
   using AtomicUpdateCallbackTy =
-      const function_ref<Value *(Value *XOld, IRBuilder<> &IRB)>;
+      const function_ref<Expected<Value *>(Value *XOld, IRBuilder<> &IRB)>;
 
 private:
   enum AtomicKind { Read, Write, Update, Capture, Compare };
@@ -3066,7 +3086,7 @@ private:
   ///
   /// \returns A pair of the old value of X before the update, and the value
   ///          used for the update.
-  std::pair<Value *, Value *>
+  Expected<std::pair<Value *, Value *>>
   emitAtomicUpdate(InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
                    AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
                    AtomicUpdateCallbackTy &UpdateOp, bool VolatileX,
@@ -3143,12 +3163,11 @@ public:
   ///	                    (e.g. true for X = X BinOp Expr)
   ///
   /// \return Insertion point after generated atomic update IR.
-  InsertPointTy createAtomicUpdate(const LocationDescription &Loc,
-                                   InsertPointTy AllocaIP, AtomicOpValue &X,
-                                   Value *Expr, AtomicOrdering AO,
-                                   AtomicRMWInst::BinOp RMWOp,
-                                   AtomicUpdateCallbackTy &UpdateOp,
-                                   bool IsXBinopExpr);
+  InsertPointOrErrorTy
+  createAtomicUpdate(const LocationDescription &Loc, InsertPointTy AllocaIP,
+                     AtomicOpValue &X, Value *Expr, AtomicOrdering AO,
+                     AtomicRMWInst::BinOp RMWOp,
+                     AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr);
 
   /// Emit atomic update for constructs: --- Only Scalar data types
   /// V = X; X = X BinOp Expr ,
@@ -3179,7 +3198,7 @@ public:
   ///                        'v', not an updated one.
   ///
   /// \return Insertion point after generated atomic capture IR.
-  InsertPointTy
+  InsertPointOrErrorTy
   createAtomicCapture(const LocationDescription &Loc, InsertPointTy AllocaIP,
                       AtomicOpValue &X, AtomicOpValue &V, Value *Expr,
                       AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index be93f9f2e1fd..d2e4dc1c85df 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -945,7 +945,7 @@ Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) {
       "omp_global_thread_num");
 }
 
-OpenMPIRBuilder::InsertPointTy
+OpenMPIRBuilder::InsertPointOrErrorTy
 OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive Kind,
                                bool ForceSimpleCall, bool CheckCancelFlag) {
   if (!updateToLocation(Loc))
@@ -992,12 +992,13 @@ OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive Kind,
                          Args);
 
   if (UseCancelBarrier && CheckCancelFlag)
-    emitCancelationCheckImpl(Result, OMPD_parallel);
+    if (Error Err = emitCancelationCheckImpl(Result, OMPD_parallel))
+      return Err;
 
   return Builder.saveIP();
 }
 
-OpenMPIRBuilder::InsertPointTy
+OpenMPIRBuilder::InsertPointOrErrorTy
 OpenMPIRBuilder::createCancel(const LocationDescription &Loc,
                               Value *IfCondition,
                               omp::Directive CanceledDirective) {
@@ -1029,18 +1030,22 @@ OpenMPIRBuilder::createCancel(const LocationDescription &Loc,
   Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
   Value *Result = Builder.CreateCall(
       getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_cancel), Args);
-  auto ExitCB = [this, CanceledDirective, Loc](InsertPointTy IP) {
+  auto ExitCB = [this, CanceledDirective, Loc](InsertPointTy IP) -> Error {
     if (CanceledDirective == OMPD_parallel) {
       IRBuilder<>::InsertPointGuard IPG(Builder);
       Builder.restoreIP(IP);
-      createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
-                    omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
-                    /* CheckCancelFlag */ false);
+      return createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
+                           omp::Directive::OMPD_unknown,
+                           /* ForceSimpleCall */ false,
+                           /* CheckCancelFlag */ false)
+          .takeError();
     }
+    return Error::success();
   };
 
   // The actual cancel logic is shared with others, e.g., cancel_barriers.
-  emitCancelationCheckImpl(Result, CanceledDirective, ExitCB);
+  if (Error Err = emitCancelationCheckImpl(Result, CanceledDirective, ExitCB))
+    return Err;
 
   // Update the insertion point and remove the terminator we introduced.
   Builder.SetInsertPoint(UI->getParent());
@@ -1079,7 +1084,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel(
   return Builder.saveIP();
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitKernelLaunch(
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitKernelLaunch(
     const LocationDescription &Loc, Value *OutlinedFnID,
     EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args,
     Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP) {
@@ -1134,15 +1139,18 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitKernelLaunch(
 
   auto CurFn = Builder.GetInsertBlock()->getParent();
   emitBlock(OffloadFailedBlock, CurFn);
-  Builder.restoreIP(EmitTargetCallFallbackCB(Builder.saveIP()));
+  InsertPointOrErrorTy AfterIP = EmitTargetCallFallbackCB(Builder.saveIP());
+  if (!AfterIP)
+    return AfterIP.takeError();
+  Builder.restoreIP(*AfterIP);
   emitBranch(OffloadContBlock);
   emitBlock(OffloadContBlock, CurFn, /*IsFinished=*/true);
   return Builder.saveIP();
 }
 
-void OpenMPIRBuilder::emitCancelationCheckImpl(Value *CancelFlag,
-                                               omp::Directive CanceledDirective,
-                                               FinalizeCallbackTy ExitCB) {
+Error OpenMPIRBuilder::emitCancelationCheckImpl(
+    Value *CancelFlag, omp::Directive CanceledDirective,
+    FinalizeCallbackTy ExitCB) {
   assert(isLastFinalizationInfoCancellable(CanceledDirective) &&
          "Unexpected cancellation!");
 
@@ -1171,12 +1179,15 @@ void OpenMPIRBuilder::emitCancelationCheckImpl(Value *CancelFlag,
   // post finalization block that is known to the FiniCB callback.
   Builder.SetInsertPoint(CancellationBlock);
   if (ExitCB)
-    ExitCB(Builder.saveIP());
+    if (Error Err = ExitCB(Builder.saveIP()))
+      return Err;
   auto &FI = FinalizationStack.back();
-  FI.FiniCB(Builder.saveIP());
+  if (Error Err = FI.FiniCB(Builder.saveIP()))
+    return Err;
 
   // The continuation block is where code generation continues.
   Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin());
+  return Error::success();
 }
 
 // Callback used to create OpenMP runtime calls to support
@@ -1355,7 +1366,7 @@ hostParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn,
   }
 }
 
-IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel(
     const LocationDescription &Loc, InsertPointTy OuterAllocaIP,
     BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
     FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads,
@@ -1496,7 +1507,8 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
   // Let the caller create the body.
   assert(BodyGenCB && "Expected body generation callback!");
   InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin());
-  BodyGenCB(InnerAllocaIP, CodeGenIP);
+  if (Error Err = BodyGenCB(InnerAllocaIP, CodeGenIP))
+    return Err;
 
   LLVM_DEBUG(dbgs() << "After  body codegen: " << *OuterFn << "\n");
 
@@ -1565,10 +1577,10 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
   FunctionCallee TIDRTLFn =
       getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num);
 
-  auto PrivHelper = [&](Value &V) {
+  auto PrivHelper = [&](Value &V) -> Error {
     if (&V == TIDAddr || &V == ZeroAddr) {
       OI.ExcludeArgsFromAggregate.push_back(&V);
-      return;
+      return Error::success();
     }
 
     SetVector<Use *> Uses;
@@ -1608,8 +1620,11 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
     if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) {
       ReplacementValue = PrivTID;
     } else {
-      Builder.restoreIP(
-          PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue));
+      InsertPointOrErrorTy AfterIP =
+          PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue);
+      if (!AfterIP)
+        return AfterIP.takeError();
+      Builder.restoreIP(*AfterIP);
       InnerAllocaIP = {
           InnerAllocaIP.getBlock(),
           InnerAllocaIP.getBlock()->getTerminator()->getIterator()};
@@ -1617,11 +1632,13 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
       assert(ReplacementValue &&
              "Expected copy/create callback to set replacement value!");
       if (ReplacementValue == &V)
-        return;
+        return Error::success();
     }
 
     for (Use *UPtr : Uses)
       UPtr->set(ReplacementValue);
+
+    return Error::success();
   };
 
   // Reset the inner alloca insertion as it will be used for loading the values
@@ -1640,7 +1657,8 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
 
   for (Value *Input : Inputs) {
     LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n");
-    PrivHelper(*Input);
+    if (Error Err = PrivHelper(*Input))
+      return Err;
   }
   LLVM_DEBUG({
     for (Value *Output : Outputs)
@@ -1666,7 +1684,8 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
   Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();
 
   InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator());
-  FiniCB(PreFiniIP);
+  if (Error Err = FiniCB(PreFiniIP))
+    return Err;
 
   // Register the outlined info.
   addOutlineInfo(std::move(OI));
@@ -1797,7 +1816,7 @@ static Value *emitTaskDependencies(
   return DepArray;
 }
 
-OpenMPIRBuilder::InsertPointTy
+OpenMPIRBuilder::InsertPointOrErrorTy
 OpenMPIRBuilder::createTask(const LocationDescription &Loc,
                             InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB,
                             bool Tied, Value *Final, Value *IfCondition,
@@ -1833,7 +1852,8 @@ OpenMPIRBuilder::createTask(const LocationDescription &Loc,
   InsertPointTy TaskAllocaIP =
       InsertPointTy(TaskAllocaBB, TaskAllocaBB->begin());
   InsertPointTy TaskBodyIP = InsertPointTy(TaskBodyBB, TaskBodyBB->begin());
-  BodyGenCB(TaskAllocaIP, TaskBodyIP);
+  if (Error Err = BodyGenCB(TaskAllocaIP, TaskBodyIP))
+    return Err;
 
   OutlineInfo OI;
   OI.EntryBB = TaskAllocaBB;
@@ -2048,7 +2068,7 @@ OpenMPIRBuilder::createTask(const LocationDescription &Loc,
   return Builder.saveIP();
 }
 
-OpenMPIRBuilder::InsertPointTy
+OpenMPIRBuilder::InsertPointOrErrorTy
 OpenMPIRBuilder::createTaskgroup(const LocationDescription &Loc,
                                  InsertPointTy AllocaIP,
                                  BodyGenCallbackTy BodyGenCB) {
@@ -2066,7 +2086,8 @@ OpenMPIRBuilder::createTaskgroup(const LocationDescription &Loc,
   Builder.CreateCall(TaskgroupFn, {Ident, ThreadID});
 
   BasicBlock *TaskgroupExitBB = splitBB(Builder, true, "taskgroup.exit");
-  BodyGenCB(AllocaIP, Builder.saveIP());
+  if (Error Err = BodyGenCB(AllocaIP, Builder.saveIP()))
+    return Err;
 
   Builder.SetInsertPoint(TaskgroupExitBB);
   // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup
@@ -2077,7 +2098,7 @@ OpenMPIRBuilder::createTaskgroup(const LocationDescription &Loc,
   return Builder.saveIP();
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections(
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSections(
     const LocationDescription &Loc, InsertPointTy AllocaIP,
     ArrayRef<StorableBodyGenCallbackTy> SectionCBs, PrivatizeCallbackTy PrivCB,
     FinalizeCallbackTy FiniCB, bool IsCancellable, bool IsNowait) {
@@ -2124,7 +2145,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections(
   // ...
   // section_loop.after:
   // <FiniCB>;
-  auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, Value *IndVar) {
+  auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, Value *IndVar) -> Error {
     Builder.restoreIP(CodeGenIP);
     BasicBlock *Continue =
         splitBBWithSuffix(Builder, /*CreateBranch=*/false, ".sections.after");
@@ -2138,12 +2159,14 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections(
       SwitchStmt->addCase(Builder.getInt32(CaseNumber), CaseBB);
       Builder.SetInsertPoint(CaseBB);
       BranchInst *CaseEndBr = Builder.CreateBr(Continue);
-      SectionCB(InsertPointTy(),
-                {CaseEndBr->getParent(), CaseEndBr->getIterator()});
+      if (Error Err = SectionCB(InsertPointTy(), {CaseEndBr->getParent(),
+                                                  CaseEndBr->getIterator()}))
+        return Err;
       CaseNumber++;
     }
     // remove the existing terminator from body BB since there can be no
     // terminators after switch/case
+    return Error::success();
   };
   // Loop body ends here
   // LowerBound, UpperBound, and STride for createCanonicalLoop
@@ -2151,10 +2174,16 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections(
   Value *LB = ConstantInt::get(I32Ty, 0);
   Value *UB = ConstantInt::get(I32Ty, SectionCBs.size());
   Value *ST = ConstantInt::get(I32Ty, 1);
-  llvm::CanonicalLoopInfo *LoopInfo = createCanonicalLoop(
+  Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
       Loc, LoopBodyGenCB, LB, UB, ST, true, false, AllocaIP, "section_loop");
-  InsertPointTy AfterIP =
-      applyStaticWorkshareLoop(Loc.DL, LoopInfo, AllocaIP, !IsNowait);
+  if (!LoopInfo)
+    return LoopInfo.takeError();
+
+  InsertPointOrErrorTy WsloopIP =
+      applyStaticWorkshareLoop(Loc.DL, *LoopInfo, AllocaIP, !IsNowait);
+  if (!WsloopIP)
+    return WsloopIP.takeError();
+  InsertPointTy AfterIP = *WsloopIP;
 
   // Apply the finalization callback in LoopAfterBB
   auto FiniInfo = FinalizationStack.pop_back_val();
@@ -2164,14 +2193,15 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections(
     Builder.restoreIP(AfterIP);
     BasicBlock *FiniBB =
         splitBBWithSuffix(Builder, /*CreateBranch=*/true, "sections.fini");
-    CB(Builder.saveIP());
+    if (Error Err = CB(Builder.saveIP()))
+      return Err;
     AfterIP = {FiniBB, FiniBB->begin()};
   }
 
   return AfterIP;
 }
 
-OpenMPIRBuilder::InsertPointTy
+OpenMPIRBuilder::InsertPointOrErrorTy
 OpenMPIRBuilder::createSection(const LocationDescription &Loc,
                                BodyGenCallbackTy BodyGenCB,
                                FinalizeCallbackTy FiniCB) {
@@ -2502,7 +2532,7 @@ void OpenMPIRBuilder::emitReductionListCopy(
   }
 }
 
-Function *OpenMPIRBuilder::emitInterWarpCopyFunction(
+Expected<Function *> OpenMPIRBuilder::emitInterWarpCopyFunction(
     const LocationDescription &Loc, ArrayRef<ReductionInfo> ReductionInfos,
     AttributeList FuncAttrs) {
   InsertPointTy SavedIP = Builder.saveIP();
@@ -2621,10 +2651,13 @@ Function *OpenMPIRBuilder::emitInterWarpCopyFunction(
       }
 
       // kmpc_barrier.
-      createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
-                    omp::Directive::OMPD_unknown,
-                    /* ForceSimpleCall */ false,
-                    /* CheckCancelFlag */ true);
+      InsertPointOrErrorTy BarrierIP1 =
+          createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
+                        omp::Directive::OMPD_unknown,
+                        /* ForceSimpleCall */ false,
+                        /* CheckCancelFlag */ true);
+      if (!BarrierIP1)
+        return BarrierIP1.takeError();
       BasicBlock *ThenBB = BasicBlock::Create(Ctx, "then");
       BasicBlock *ElseBB = BasicBlock::Create(Ctx, "else");
       BasicBlock *MergeBB = BasicBlock::Create(Ctx, "ifcont");
@@ -2666,10 +2699,13 @@ Function *OpenMPIRBuilder::emitInterWarpCopyFunction(
 
       // endif
       emitBlock(MergeBB, Builder.GetInsertBlock()->getParent());
-      createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
-                    omp::Directive::OMPD_unknown,
-                    /* ForceSimpleCall */ false,
-                    /* CheckCancelFlag */ true);
+      InsertPointOrErrorTy BarrierIP2 =
+          createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
+                        omp::Directive::OMPD_unknown,
+                        /* ForceSimpleCall */ false,
+                        /* CheckCancelFlag */ true);
+      if (!BarrierIP2)
+        return BarrierIP2.takeError();
 
       // Warp 0 copies reduce element from transfer medium
       BasicBlock *W0ThenBB = BasicBlock::Create(Ctx, "then");
@@ -3286,7 +3322,7 @@ std::string OpenMPIRBuilder::getReductionFuncName(StringRef Name) const {
   return (Name + Suffix).str();
 }
 
-Function *OpenMPIRBuilder::createReductionFunction(
+Expected<Function *> OpenMPIRBuilder::createReductionFunction(
     StringRef ReducerName, ArrayRef<ReductionInfo> ReductionInfos,
     ReductionGenCBKind ReductionGenCBKind, AttributeList FuncAttrs) {
   auto *FuncTy = FunctionType::get(Builder.getVoidTy(),
@@ -3352,7 +3388,10 @@ Function *OpenMPIRBuilder::createReductionFunction(
       Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
       Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
       Value *Reduced;
-      RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
+      InsertPointOrErrorTy AfterIP =
+          RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
+      if (!AfterIP)
+        return AfterIP.takeError();
       if (!Builder.GetInsertBlock())
         return ReductionFunc;
       Builder.CreateStore(Reduced, LHSPtr);
@@ -3405,7 +3444,7 @@ checkReductionInfos(ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
   }
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductionsGPU(
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductionsGPU(
     const LocationDescription &Loc, InsertPointTy AllocaIP,
     InsertPointTy CodeGenIP, ArrayRef<ReductionInfo> ReductionInfos,
     bool IsNoWait, bool IsTeamsReduction, bool HasDistribute,
@@ -3435,11 +3474,13 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductionsGPU(
   AttrBldr.removeAttribute(Attribute::OptimizeNone);
   FuncAttrs = FuncAttrs.addFnAttributes(Ctx, AttrBldr);
 
-  Function *ReductionFunc = nullptr;
   CodeGenIP = Builder.saveIP();
-  ReductionFunc =
+  Expected<Function *> ReductionResult =
       createReductionFunction(Builder.GetInsertBlock()->getParent()->getName(),
                               ReductionInfos, ReductionGenCBKind, FuncAttrs);
+  if (!ReductionResult)
+    return ReductionResult.takeError();
+  Function *ReductionFunc = *ReductionResult;
   Builder.restoreIP(CodeGenIP);
 
   // Set the grid value in the config needed for lowering later on
@@ -3480,7 +3521,11 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductionsGPU(
   CodeGenIP = Builder.saveIP();
   Function *SarFunc =
       emitShuffleAndReduceFunction(ReductionInfos, ReductionFunc, FuncAttrs);
-  Function *WcFunc = emitInterWarpCopyFunction(Loc, ReductionInfos, FuncAttrs);
+  Expected<Function *> CopyResult =
+      emitInterWarpCopyFunction(Loc, ReductionInfos, FuncAttrs);
+  if (!CopyResult)
+    return CopyResult.takeError();
+  Function *WcFunc = *CopyResult;
   Builder.restoreIP(CodeGenIP);
 
   Value *RL = Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList, PtrTy);
@@ -3595,7 +3640,7 @@ static Function *getFreshReductionFunc(Module &M) {
                           ".omp.reduction.func", &M);
 }
 
-OpenMPIRBuilder::InsertPointTy
+OpenMPIRBuilder::InsertPointOrErrorTy
 OpenMPIRBuilder::createReductions(const LocationDescription &Loc,
                                   InsertPointTy AllocaIP,
                                   ArrayRef<ReductionInfo> ReductionInfos,
@@ -3688,7 +3733,7 @@ OpenMPIRBuilder::createReductions(const LocationDescription &Loc,
     Type *ValueType = RI.ElementType;
     // We have one less load for by-ref case because that load is now inside of
     // the reduction region
-    Value *RedValue = nullptr;
+    Value *RedValue = RI.Variable;
     if (!IsByRef[En.index()]) {
       RedValue = Builder.CreateLoad(ValueType, RI.Variable,
                                     "red.value." + Twine(En.index()));
@@ -3697,13 +3742,12 @@ OpenMPIRBuilder::createReductions(const LocationDescription &Loc,
         Builder.CreateLoad(ValueType, RI.PrivateVariable,
                            "red.private.value." + Twine(En.index()));
     Value *Reduced;
-    if (IsByRef[En.index()]) {
-      Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), RI.Variable,
-                                        PrivateRedValue, Reduced));
-    } else {
-      Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), RedValue,
-                                        PrivateRedValue, Reduced));
-    }
+    InsertPointOrErrorTy AfterIP =
+        RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced);
+    if (!AfterIP)
+      return AfterIP.takeError();
+    Builder.restoreIP(*AfterIP);
+
     if (!Builder.GetInsertBlock())
       return InsertPointTy();
     // for by-ref case, the load is inside of the reduction region
@@ -3722,8 +3766,11 @@ OpenMPIRBuilder::createReductions(const LocationDescription &Loc,
   Builder.SetInsertPoint(AtomicRedBlock);
   if (CanGenerateAtomic && llvm::none_of(IsByRef, [](bool P) { return P; })) {
     for (const ReductionInfo &RI : ReductionInfos) {
-      Builder.restoreIP(RI.AtomicReductionGen(Builder.saveIP(), RI.ElementType,
-                                              RI.Variable, RI.PrivateVariable));
+      InsertPointOrErrorTy AfterIP = RI.AtomicReductionGen(
+          Builder.saveIP(), RI.ElementType, RI.Variable, RI.PrivateVariable);
+      if (!AfterIP)
+        return AfterIP.takeError();
+      Builder.restoreIP(*AfterIP);
       if (!Builder.GetInsertBlock())
         return InsertPointTy();
     }
@@ -3755,7 +3802,11 @@ OpenMPIRBuilder::createReductions(const LocationDescription &Loc,
         Builder.CreateBitCast(RHSI8Ptr, RI.PrivateVariable->getType());
     Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
     Value *Reduced;
-    Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced));
+    InsertPointOrErrorTy AfterIP =
+        RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
+    if (!AfterIP)
+      return AfterIP.takeError();
+    Builder.restoreIP(*AfterIP);
     if (!Builder.GetInsertBlock())
       return InsertPointTy();
     // store is inside of the reduction region when using by-ref
@@ -3768,11 +3819,10 @@ OpenMPIRBuilder::createReductions(const LocationDescription &Loc,
   return Builder.saveIP();
 }
 
-OpenMPIRBuilder::InsertPointTy
+OpenMPIRBuilder::InsertPointOrErrorTy
 OpenMPIRBuilder::createMaster(const LocationDescription &Loc,
                               BodyGenCallbackTy BodyGenCB,
                               FinalizeCallbackTy FiniCB) {
-
   if (!updateToLocation(Loc))
     return Loc.IP;
 
@@ -3793,7 +3843,7 @@ OpenMPIRBuilder::createMaster(const LocationDescription &Loc,
                               /*Conditional*/ true, /*hasFinalize*/ true);
 }
 
-OpenMPIRBuilder::InsertPointTy
+OpenMPIRBuilder::InsertPointOrErrorTy
 OpenMPIRBuilder::createMasked(const LocationDescription &Loc,
                               BodyGenCallbackTy BodyGenCB,
                               FinalizeCallbackTy FiniCB, Value *Filter) {
@@ -3884,7 +3934,7 @@ CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
   return CL;
 }
 
-CanonicalLoopInfo *
+Expected<CanonicalLoopInfo *>
 OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
                                      LoopBodyGenCallbackTy BodyGenCB,
                                      Value *TripCount, const Twine &Name) {
@@ -3906,7 +3956,8 @@ OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
 
   // Emit the body content. We do it after connecting the loop to the CFG to
   // avoid that the callback encounters degenerate BBs.
-  BodyGenCB(CL->getBodyIP(), CL->getIndVar());
+  if (Error Err = BodyGenCB(CL->getBodyIP(), CL->getIndVar()))
+    return Err;
 
 #ifndef NDEBUG
   CL->assertOK();
@@ -3914,7 +3965,7 @@ OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
   return CL;
 }
 
-CanonicalLoopInfo *OpenMPIRBuilder::createCanonicalLoop(
+Expected<CanonicalLoopInfo *> OpenMPIRBuilder::createCanonicalLoop(
     const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
     Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
     InsertPointTy ComputeIP, const Twine &Name) {
@@ -3979,7 +4030,7 @@ CanonicalLoopInfo *OpenMPIRBuilder::createCanonicalLoop(
     Builder.restoreIP(CodeGenIP);
     Value *Span = Builder.CreateMul(IV, Step);
     Value *IndVar = Builder.CreateAdd(Span, Start);
-    BodyGenCB(Builder.saveIP(), IndVar);
+    return BodyGenCB(Builder.saveIP(), IndVar);
   };
   LocationDescription LoopLoc = ComputeIP.isSet() ? Loc.IP : Builder.saveIP();
   return createCanonicalLoop(LoopLoc, BodyGen, TripCount, Name);
@@ -4001,7 +4052,7 @@ static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M,
   llvm_unreachable("unknown OpenMP loop iterator bitwidth");
 }
 
-OpenMPIRBuilder::InsertPointTy
+OpenMPIRBuilder::InsertPointOrErrorTy
 OpenMPIRBuilder::applyStaticWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
                                           InsertPointTy AllocaIP,
                                           bool NeedsBarrier) {
@@ -4078,10 +4129,14 @@ OpenMPIRBuilder::applyStaticWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
   Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});
 
   // Add the barrier if requested.
-  if (NeedsBarrier)
-    createBarrier(LocationDescription(Builder.saveIP(), DL),
-                  omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
-                  /* CheckCancelFlag */ false);
+  if (NeedsBarrier) {
+    InsertPointOrErrorTy BarrierIP =
+        createBarrier(LocationDescription(Builder.saveIP(), DL),
+                      omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
+                      /* CheckCancelFlag */ false);
+    if (!BarrierIP)
+      return BarrierIP.takeError();
+  }
 
   InsertPointTy AfterIP = CLI->getAfterIP();
   CLI->invalidate();
@@ -4089,9 +4144,12 @@ OpenMPIRBuilder::applyStaticWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
   return AfterIP;
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(
-    DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
-    bool NeedsBarrier, Value *ChunkSize) {
+OpenMPIRBuilder::InsertPointOrErrorTy
+OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(DebugLoc DL,
+                                                 CanonicalLoopInfo *CLI,
+                                                 InsertPointTy AllocaIP,
+                                                 bool NeedsBarrier,
+                                                 Value *ChunkSize) {
   assert(CLI->isValid() && "Requires a valid canonical loop");
   assert(ChunkSize && "Chunk size is required");
 
@@ -4167,12 +4225,23 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(
   // Create outer "dispatch" loop for enumerating the chunks.
   BasicBlock *DispatchEnter = splitBB(Builder, true);
   Value *DispatchCounter;
-  CanonicalLoopInfo *DispatchCLI = createCanonicalLoop(
+  Expected<CanonicalLoopInfo *> LoopResult = createCanonicalLoop(
       {Builder.saveIP(), DL},
-      [&](InsertPointTy BodyIP, Value *Counter) { DispatchCounter = Counter; },
+      [&](InsertPointTy BodyIP, Value *Counter) {
+        DispatchCounter = Counter;
+        return Error::success();
+      },
       FirstChunkStart, CastedTripCount, NextChunkStride,
       /*IsSigned=*/false, /*InclusiveStop=*/false, /*ComputeIP=*/{},
       "dispatch");
+  if (!LoopResult) {
+    // It is safe to assume this didn't return an error because the callback
+    // passed into createCanonicalLoop is the only possible error source, and it
+    // always returns success. Need to still cast the result into bool to avoid
+    // runtime errors.
+    llvm_unreachable("unexpected error creating canonical loop");
+  }
+  CanonicalLoopInfo *DispatchCLI = *LoopResult;
 
   // Remember the BasicBlocks of the dispatch loop we need, then invalidate to
   // not have to preserve the canonical invariant.
@@ -4219,9 +4288,13 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(
   Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});
 
   // Add the barrier if requested.
-  if (NeedsBarrier)
-    createBarrier(LocationDescription(Builder.saveIP(), DL), OMPD_for,
-                  /*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false);
+  if (NeedsBarrier) {
+    InsertPointOrErrorTy AfterIP =
+        createBarrier(LocationDescription(Builder.saveIP(), DL), OMPD_for,
+                      /*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false);
+    if (!AfterIP)
+      return AfterIP.takeError();
+  }
 
 #ifndef NDEBUG
   // Even though we currently do not support applying additional methods to it,
@@ -4229,7 +4302,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(
   CLI->assertOK();
 #endif
 
-  return {DispatchAfter, DispatchAfter->getFirstInsertionPt()};
+  return InsertPointTy(DispatchAfter, DispatchAfter->getFirstInsertionPt());
 }
 
 // Returns an LLVM function to call for executing an OpenMP static worksharing
@@ -4462,7 +4535,7 @@ OpenMPIRBuilder::applyWorkshareLoopTarget(DebugLoc DL, CanonicalLoopInfo *CLI,
   return CLI->getAfterIP();
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyWorkshareLoop(
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyWorkshareLoop(
     DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
     bool NeedsBarrier, omp::ScheduleKind SchedKind, Value *ChunkSize,
     bool HasSimdModifier, bool HasMonotonicModifier,
@@ -4563,9 +4636,11 @@ getKmpcForDynamicFiniForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
   llvm_unreachable("unknown OpenMP loop iterator bitwidth");
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyDynamicWorkshareLoop(
-    DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
-    OMPScheduleType SchedType, bool NeedsBarrier, Value *Chunk) {
+OpenMPIRBuilder::InsertPointOrErrorTy
+OpenMPIRBuilder::applyDynamicWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
+                                           InsertPointTy AllocaIP,
+                                           OMPScheduleType SchedType,
+                                           bool NeedsBarrier, Value *Chunk) {
   assert(CLI->isValid() && "Requires a valid canonical loop");
   assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&
          "Require dedicated allocate IP");
@@ -4681,9 +4756,12 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyDynamicWorkshareLoop(
   // Add the barrier if requested.
   if (NeedsBarrier) {
     Builder.SetInsertPoint(&Exit->back());
-    createBarrier(LocationDescription(Builder.saveIP(), DL),
-                  omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
-                  /* CheckCancelFlag */ false);
+    InsertPointOrErrorTy BarrierIP =
+        createBarrier(LocationDescription(Builder.saveIP(), DL),
+                      omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
+                      /* CheckCancelFlag */ false);
+    if (!BarrierIP)
+      return BarrierIP.takeError();
   }
 
   CLI->invalidate();
@@ -5542,7 +5620,7 @@ OpenMPIRBuilder::createCopyPrivate(const LocationDescription &Loc,
   return Builder.saveIP();
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSingle(
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSingle(
     const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
     FinalizeCallbackTy FiniCB, bool IsNowait, ArrayRef<llvm::Value *> CPVars,
     ArrayRef<llvm::Function *> CPFuncs) {
@@ -5571,14 +5649,17 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSingle(
   Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_single);
   Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
 
-  auto FiniCBWrapper = [&](InsertPointTy IP) {
-    FiniCB(IP);
+  auto FiniCBWrapper = [&](InsertPointTy IP) -> Error {
+    if (Error Err = FiniCB(IP))
+      return Err;
 
     // The thread that executes the single region must set `DidIt` to 1.
     // This is used by __kmpc_copyprivate, to know if the caller is the
     // single thread or not.
     if (DidIt)
       Builder.CreateStore(Builder.getInt32(1), DidIt);
+
+    return Error::success();
   };
 
   // generates the following:
@@ -5589,9 +5670,12 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSingle(
   // __kmpc_copyprivate
   // __kmpc_barrier
 
-  EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCBWrapper,
-                       /*Conditional*/ true,
-                       /*hasFinalize*/ true);
+  InsertPointOrErrorTy AfterIP =
+      EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCBWrapper,
+                           /*Conditional*/ true,
+                           /*hasFinalize*/ true);
+  if (!AfterIP)
+    return AfterIP.takeError();
 
   if (DidIt) {
     for (size_t I = 0, E = CPVars.size(); I < E; ++I)
@@ -5600,14 +5684,18 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSingle(
                         /*BufSize=*/ConstantInt::get(Int64, 0), CPVars[I],
                         CPFuncs[I], DidIt);
     // NOTE __kmpc_copyprivate already inserts a barrier
-  } else if (!IsNowait)
-    createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
-                  omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
-                  /* CheckCancelFlag */ false);
+  } else if (!IsNowait) {
+    InsertPointOrErrorTy AfterIP =
+        createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
+                      omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
+                      /* CheckCancelFlag */ false);
+    if (!AfterIP)
+      return AfterIP.takeError();
+  }
   return Builder.saveIP();
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCritical(
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createCritical(
     const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
     FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) {
 
@@ -5688,7 +5776,7 @@ OpenMPIRBuilder::createOrderedDepend(const LocationDescription &Loc,
   return Builder.saveIP();
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createOrderedThreadsSimd(
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createOrderedThreadsSimd(
     const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
     FinalizeCallbackTy FiniCB, bool IsThreads) {
   if (!updateToLocation(Loc))
@@ -5717,7 +5805,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createOrderedThreadsSimd(
                               /*Conditional*/ false, /*hasFinalize*/ true);
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::EmitOMPInlinedRegion(
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::EmitOMPInlinedRegion(
     Directive OMPD, Instruction *EntryCall, Instruction *ExitCall,
     BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional,
     bool HasFinalize, bool IsCancellable) {
@@ -5739,15 +5827,19 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::EmitOMPInlinedRegion(
   emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
 
   // generate body
-  BodyGenCB(/* AllocaIP */ InsertPointTy(),
-            /* CodeGenIP */ Builder.saveIP());
+  if (Error Err = BodyGenCB(/* AllocaIP */ InsertPointTy(),
+                            /* CodeGenIP */ Builder.saveIP()))
+    return Err;
 
   // emit exit call and do any needed finalization.
   auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt());
   assert(FiniBB->getTerminator()->getNumSuccessors() == 1 &&
          FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&
          "Unexpected control flow graph state!!");
-  emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
+  InsertPointOrErrorTy AfterIP =
+      emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
+  if (!AfterIP)
+    return AfterIP.takeError();
   assert(FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB &&
          "Unexpected Control Flow State!");
   MergeBlockIntoPredecessor(FiniBB);
@@ -5796,7 +5888,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry(
   return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt());
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveExit(
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitCommonDirectiveExit(
     omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall,
     bool HasFinalize) {
 
@@ -5810,7 +5902,8 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveExit(
     FinalizationInfo Fi = FinalizationStack.pop_back_val();
     assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!");
 
-    Fi.FiniCB(FinIP);
+    if (Error Err = Fi.FiniCB(FinIP))
+      return Err;
 
     BasicBlock *FiniBB = FinIP.getBlock();
     Instruction *FiniBBTI = FiniBB->getTerminator();
@@ -6319,7 +6412,7 @@ Constant *OpenMPIRBuilder::createTargetRegionEntryAddr(Function *OutlinedFn,
       Constant::getNullValue(Builder.getInt8Ty()), EntryFnName);
 }
 
-void OpenMPIRBuilder::emitTargetRegionFunction(
+Error OpenMPIRBuilder::emitTargetRegionFunction(
     TargetRegionEntryInfo &EntryInfo,
     FunctionGenCallback &GenerateFunctionCallback, bool IsOffloadEntry,
     Function *&OutlinedFn, Constant *&OutlinedFnID) {
@@ -6327,15 +6420,20 @@ void OpenMPIRBuilder::emitTargetRegionFunction(
   SmallString<64> EntryFnName;
   OffloadInfoManager.getTargetRegionEntryFnName(EntryFnName, EntryInfo);
 
-  OutlinedFn = Config.isTargetDevice() || !Config.openMPOffloadMandatory()
-                   ? GenerateFunctionCallback(EntryFnName)
-                   : nullptr;
+  if (Config.isTargetDevice() || !Config.openMPOffloadMandatory()) {
+    Expected<Function *> CBResult = GenerateFunctionCallback(EntryFnName);
+    if (!CBResult)
+      return CBResult.takeError();
+    OutlinedFn = *CBResult;
+  } else {
+    OutlinedFn = nullptr;
+  }
 
   // If this target outline function is not an offload entry, we don't need to
   // register it. This may be in the case of a false if clause, or if there are
   // no OpenMP targets.
   if (!IsOffloadEntry)
-    return;
+    return Error::success();
 
   std::string EntryFnIDName =
       Config.isTargetDevice()
@@ -6344,6 +6442,7 @@ void OpenMPIRBuilder::emitTargetRegionFunction(
 
   OutlinedFnID = registerTargetRegionFunction(EntryInfo, OutlinedFn,
                                               EntryFnName, EntryFnIDName);
+  return Error::success();
 }
 
 Constant *OpenMPIRBuilder::registerTargetRegionFunction(
@@ -6359,12 +6458,13 @@ Constant *OpenMPIRBuilder::registerTargetRegionFunction(
   return OutlinedFnID;
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData(
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData(
     const LocationDescription &Loc, InsertPointTy AllocaIP,
     InsertPointTy CodeGenIP, Value *DeviceID, Value *IfCond,
     TargetDataInfo &Info, GenMapInfoCallbackTy GenMapInfoCB,
     omp::RuntimeFunction *MapperFunc,
-    function_ref<InsertPointTy(InsertPointTy CodeGenIP, BodyGenTy BodyGenType)>
+    function_ref<InsertPointOrErrorTy(InsertPointTy CodeGenIP,
+                                      BodyGenTy BodyGenType)>
         BodyGenCB,
     function_ref<void(unsigned int, Value *)> DeviceAddrCB,
     function_ref<Value *(unsigned int)> CustomMapperCB, Value *SrcLocInfo) {
@@ -6374,8 +6474,13 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData(
   Builder.restoreIP(CodeGenIP);
   // Disable TargetData CodeGen on Device pass.
   if (Config.IsTargetDevice.value_or(false)) {
-    if (BodyGenCB)
-      Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv));
+    if (BodyGenCB) {
+      InsertPointOrErrorTy AfterIP =
+          BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv);
+      if (!AfterIP)
+        return AfterIP.takeError();
+      Builder.restoreIP(*AfterIP);
+    }
     return Builder.saveIP();
   }
 
@@ -6384,7 +6489,8 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData(
   // Generate the code for the opening of the data environment. Capture all the
   // arguments of the runtime call by reference because they are used in the
   // closing of the region.
-  auto BeginThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+  auto BeginThenGen = [&](InsertPointTy AllocaIP,
+                          InsertPointTy CodeGenIP) -> Error {
     MapInfo = &GenMapInfoCB(Builder.saveIP());
     emitOffloadingArrays(AllocaIP, Builder.saveIP(), *MapInfo, Info,
                          /*IsNonContiguous=*/true, DeviceAddrCB,
@@ -6413,7 +6519,8 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData(
     if (IsStandAlone) {
       assert(MapperFunc && "MapperFunc missing for standalone target data");
 
-      auto TaskBodyCB = [&](Value *, Value *, IRBuilderBase::InsertPoint) {
+      auto TaskBodyCB = [&](Value *, Value *,
+                            IRBuilderBase::InsertPoint) -> Error {
         if (Info.HasNoWait) {
           OffloadingArgs.append({llvm::Constant::getNullValue(Int32),
                                  llvm::Constant::getNullValue(VoidPtr),
@@ -6431,16 +6538,20 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData(
           emitBlock(OffloadContBlock, CurFn, /*IsFinished=*/true);
           Builder.restoreIP(Builder.saveIP());
         }
+        return Error::success();
       };
 
       bool RequiresOuterTargetTask = Info.HasNoWait;
-
-      if (!RequiresOuterTargetTask)
-        TaskBodyCB(/*DeviceID=*/nullptr, /*RTLoc=*/nullptr,
-                   /*TargetTaskAllocaIP=*/{});
-      else
-        emitTargetTask(TaskBodyCB, DeviceID, SrcLocInfo, AllocaIP,
-                       /*Dependencies=*/{}, Info.HasNoWait);
+      if (!RequiresOuterTargetTask) {
+        Error Err = TaskBodyCB(/*DeviceID=*/nullptr, /*RTLoc=*/nullptr,
+                               /*TargetTaskAllocaIP=*/{});
+        assert(!Err && "TaskBodyCB expected to succeed");
+      } else {
+        InsertPointOrErrorTy AfterIP =
+            emitTargetTask(TaskBodyCB, DeviceID, SrcLocInfo, AllocaIP,
+                           /*Dependencies=*/{}, Info.HasNoWait);
+        assert(AfterIP && "TaskBodyCB expected to succeed");
+      }
     } else {
       Function *BeginMapperFunc = getOrCreateRuntimeFunctionPtr(
           omp::OMPRTL___tgt_target_data_begin_mapper);
@@ -6458,15 +6569,26 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData(
       // If device pointer privatization is required, emit the body of the
       // region here. It will have to be duplicated: with and without
       // privatization.
-      Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::Priv));
+      InsertPointOrErrorTy AfterIP =
+          BodyGenCB(Builder.saveIP(), BodyGenTy::Priv);
+      if (!AfterIP)
+        return AfterIP.takeError();
+      Builder.restoreIP(*AfterIP);
     }
+    return Error::success();
   };
 
   // If we need device pointer privatization, we need to emit the body of the
   // region with no privatization in the 'else' branch of the conditional.
   // Otherwise, we don't have to do anything.
-  auto BeginElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
-    Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv));
+  auto BeginElseGen = [&](InsertPointTy AllocaIP,
+                          InsertPointTy CodeGenIP) -> Error {
+    InsertPointOrErrorTy AfterIP =
+        BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv);
+    if (!AfterIP)
+      return AfterIP.takeError();
+    Builder.restoreIP(*AfterIP);
+    return Error::success();
   };
 
   // Generate code for the closing of the data region.
@@ -6494,35 +6616,45 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData(
         getOrCreateRuntimeFunctionPtr(omp::OMPRTL___tgt_target_data_end_mapper);
 
     Builder.CreateCall(EndMapperFunc, OffloadingArgs);
+    return Error::success();
   };
 
   // We don't have to do anything to close the region if the if clause evaluates
   // to false.
-  auto EndElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
+  auto EndElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    return Error::success();
+  };
 
-  if (BodyGenCB) {
-    if (IfCond) {
-      emitIfClause(IfCond, BeginThenGen, BeginElseGen, AllocaIP);
-    } else {
-      BeginThenGen(AllocaIP, Builder.saveIP());
+  Error Err = [&]() -> Error {
+    if (BodyGenCB) {
+      Error Err = [&]() {
+        if (IfCond)
+          return emitIfClause(IfCond, BeginThenGen, BeginElseGen, AllocaIP);
+        return BeginThenGen(AllocaIP, Builder.saveIP());
+      }();
+
+      if (Err)
+        return Err;
+
+      // If we don't require privatization of device pointers, we emit the body
+      // in between the runtime calls. This avoids duplicating the body code.
+      InsertPointOrErrorTy AfterIP =
+          BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv);
+      if (!AfterIP)
+        return AfterIP.takeError();
+      Builder.restoreIP(*AfterIP);
+
+      if (IfCond)
+        return emitIfClause(IfCond, EndThenGen, EndElseGen, AllocaIP);
+      return EndThenGen(AllocaIP, Builder.saveIP());
     }
+    if (IfCond)
+      return emitIfClause(IfCond, BeginThenGen, EndElseGen, AllocaIP);
+    return BeginThenGen(AllocaIP, Builder.saveIP());
+  }();
 
-    // If we don't require privatization of device pointers, we emit the body in
-    // between the runtime calls. This avoids duplicating the body code.
-    Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv));
-
-    if (IfCond) {
-      emitIfClause(IfCond, EndThenGen, EndElseGen, AllocaIP);
-    } else {
-      EndThenGen(AllocaIP, Builder.saveIP());
-    }
-  } else {
-    if (IfCond) {
-      emitIfClause(IfCond, BeginThenGen, EndElseGen, AllocaIP);
-    } else {
-      BeginThenGen(AllocaIP, Builder.saveIP());
-    }
-  }
+  if (Err)
+    return Err;
 
   return Builder.saveIP();
 }
@@ -6591,7 +6723,7 @@ FunctionCallee OpenMPIRBuilder::createDispatchDeinitFunction() {
   return getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_dispatch_deinit);
 }
 
-static Function *createOutlinedFunction(
+static Expected<Function *> createOutlinedFunction(
     OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, StringRef FuncName,
     SmallVectorImpl<Value *> &Inputs,
     OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,
@@ -6671,7 +6803,11 @@ static Function *createOutlinedFunction(
     OMPBuilder.ConstantAllocaRaiseCandidates.emplace_back(Func);
 
   // Insert target deinit call in the device compilation pass.
-  Builder.restoreIP(CBFunc(Builder.saveIP(), Builder.saveIP()));
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      CBFunc(Builder.saveIP(), Builder.saveIP());
+  if (!AfterIP)
+    return AfterIP.takeError();
+  Builder.restoreIP(*AfterIP);
   if (OMPBuilder.Config.isTargetDevice())
     OMPBuilder.createTargetDeinit(Builder);
 
@@ -6726,8 +6862,11 @@ static Function *createOutlinedFunction(
     Argument &Arg = std::get<1>(InArg);
     Value *InputCopy = nullptr;
 
-    Builder.restoreIP(
-        ArgAccessorFuncCB(Arg, Input, InputCopy, AllocaIP, Builder.saveIP()));
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+        ArgAccessorFuncCB(Arg, Input, InputCopy, AllocaIP, Builder.saveIP());
+    if (!AfterIP)
+      return AfterIP.takeError();
+    Builder.restoreIP(*AfterIP);
 
     // In certain cases a Global may be set up for replacement, however, this
     // Global may be used in multiple arguments to the kernel, just segmented
@@ -6847,7 +6986,8 @@ static Function *emitTargetTaskProxyFunction(OpenMPIRBuilder &OMPBuilder,
   Builder.CreateRetVoid();
   return ProxyFn;
 }
-static void emitTargetOutlinedFunction(
+
+static Error emitTargetOutlinedFunction(
     OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, bool IsOffloadEntry,
     TargetRegionEntryInfo &EntryInfo, Function *&OutlinedFn,
     Constant *&OutlinedFnID, SmallVectorImpl<Value *> &Inputs,
@@ -6861,11 +7001,12 @@ static void emitTargetOutlinedFunction(
                                       CBFunc, ArgAccessorFuncCB);
       };
 
-  OMPBuilder.emitTargetRegionFunction(EntryInfo, GenerateOutlinedFunction,
-                                      IsOffloadEntry, OutlinedFn, OutlinedFnID);
+  return OMPBuilder.emitTargetRegionFunction(
+      EntryInfo, GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
+      OutlinedFnID);
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetTask(
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitTargetTask(
     TargetTaskBodyCallbackTy TaskBodyCB, Value *DeviceID, Value *RTLoc,
     OpenMPIRBuilder::InsertPointTy AllocaIP,
     const SmallVector<llvm::OpenMPIRBuilder::DependData> &Dependencies,
@@ -6983,7 +7124,8 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetTask(
 
   Builder.restoreIP(TargetTaskBodyIP);
 
-  TaskBodyCB(DeviceID, RTLoc, TargetTaskAllocaIP);
+  if (Error Err = TaskBodyCB(DeviceID, RTLoc, TargetTaskAllocaIP))
+    return Err;
 
   OI.ExitBB = Builder.saveIP().getBlock();
   OI.PostOutlineCB = [this, ToBeDeleted, Dependencies, HasNoWait,
@@ -7161,8 +7303,8 @@ emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
   // Generate a function call to the host fallback implementation of the target
   // region. This is called by the host when no offload entry was generated for
   // the target region and when the offloading call fails at runtime.
-  auto &&EmitTargetCallFallbackCB =
-      [&](OpenMPIRBuilder::InsertPointTy IP) -> OpenMPIRBuilder::InsertPointTy {
+  auto &&EmitTargetCallFallbackCB = [&](OpenMPIRBuilder::InsertPointTy IP)
+      -> OpenMPIRBuilder::InsertPointOrErrorTy {
     Builder.restoreIP(IP);
     Builder.CreateCall(OutlinedFn, Args);
     return Builder.saveIP();
@@ -7173,9 +7315,10 @@ emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
 
   OpenMPIRBuilder::TargetKernelArgs KArgs;
 
-  auto TaskBodyCB = [&](Value *DeviceID, Value *RTLoc,
-                        IRBuilderBase::InsertPoint TargetTaskAllocaIP) {
-    if (OutlinedFnID) {
+  auto TaskBodyCB =
+      [&](Value *DeviceID, Value *RTLoc,
+          IRBuilderBase::InsertPoint TargetTaskAllocaIP) -> Error {
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = [&]() {
       // emitKernelLaunch makes the necessary runtime call to offload the
       // kernel. We then outline all that code into a separate function
       // ('kernel_launch_function' in the pseudo code above). This function is
@@ -7183,31 +7326,41 @@ emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
       // '@.omp_target_task_proxy_func' in the pseudo code above)
       // "@.omp_target_task_proxy_func' is generated by
       // emitTargetTaskProxyFunction.
-      Builder.restoreIP(OMPBuilder.emitKernelLaunch(
-          Builder, OutlinedFnID, EmitTargetCallFallbackCB, KArgs, DeviceID,
-          RTLoc, TargetTaskAllocaIP));
-    } else {
-      // When OutlinedFnID is set to nullptr, then it's not an offloading
-      // call. In this case, we execute the host implementation directly.
-      OMPBuilder.Builder.restoreIP(
-          EmitTargetCallFallbackCB(OMPBuilder.Builder.saveIP()));
-    }
+      if (OutlinedFnID)
+        return OMPBuilder.emitKernelLaunch(Builder, OutlinedFnID,
+                                           EmitTargetCallFallbackCB, KArgs,
+                                           DeviceID, RTLoc, TargetTaskAllocaIP);
+      // When OutlinedFnID is set to nullptr, then it's not an offloading call.
+      // In this case, we execute the host implementation directly.
+      return EmitTargetCallFallbackCB(OMPBuilder.Builder.saveIP());
+    }();
+
+    if (!AfterIP)
+      return AfterIP.takeError();
+
+    OMPBuilder.Builder.restoreIP(*AfterIP);
+    return Error::success();
   };
 
   // If we don't have an ID for the target region, it means an offload entry
   // wasn't created. In this case we just run the host fallback directly.
   if (!OutlinedFnID) {
-    if (RequiresOuterTargetTask) {
-      // Arguments that are intended to be directly forwarded to an
-      // emitKernelLaunch call are pased as nullptr, since OutlinedFnID=nullptr
-      // results in that call not being done.
-      Builder.restoreIP(OMPBuilder.emitTargetTask(TaskBodyCB,
-                                                  /*DeviceID=*/nullptr,
-                                                  /*RTLoc=*/nullptr, AllocaIP,
-                                                  Dependencies, HasNoWait));
-    } else {
-      Builder.restoreIP(EmitTargetCallFallbackCB(Builder.saveIP()));
-    }
+    OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = [&]() {
+      if (RequiresOuterTargetTask) {
+        // Arguments that are intended to be directly forwarded to an
+        // emitKernelLaunch call are pased as nullptr, since
+        // OutlinedFnID=nullptr results in that call not being done.
+        return OMPBuilder.emitTargetTask(TaskBodyCB, /*DeviceID=*/nullptr,
+                                         /*RTLoc=*/nullptr, AllocaIP,
+                                         Dependencies, HasNoWait);
+      }
+      return EmitTargetCallFallbackCB(Builder.saveIP());
+    }();
+
+    // Assume no error was returned because EmitTargetCallFallbackCB doesn't
+    // produce any. The 'if' check enables accessing the returned value.
+    if (AfterIP)
+      Builder.restoreIP(*AfterIP);
     return;
   }
 
@@ -7247,17 +7400,24 @@ emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
 
   // The presence of certain clauses on the target directive require the
   // explicit generation of the target task.
-  if (RequiresOuterTargetTask) {
-    Builder.restoreIP(OMPBuilder.emitTargetTask(
-        TaskBodyCB, DeviceID, RTLoc, AllocaIP, Dependencies, HasNoWait));
-  } else {
-    Builder.restoreIP(OMPBuilder.emitKernelLaunch(
-        Builder, OutlinedFnID, EmitTargetCallFallbackCB, KArgs, DeviceID, RTLoc,
-        AllocaIP));
-  }
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = [&]() {
+    if (RequiresOuterTargetTask)
+      return OMPBuilder.emitTargetTask(TaskBodyCB, DeviceID, RTLoc, AllocaIP,
+                                       Dependencies, HasNoWait);
+
+    return OMPBuilder.emitKernelLaunch(Builder, OutlinedFnID,
+                                       EmitTargetCallFallbackCB, KArgs,
+                                       DeviceID, RTLoc, AllocaIP);
+  }();
+
+  // Assume no error was returned because TaskBodyCB and
+  // EmitTargetCallFallbackCB don't produce any. The 'if' check enables
+  // accessing the returned value.
+  if (AfterIP)
+    Builder.restoreIP(*AfterIP);
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTarget(
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTarget(
     const LocationDescription &Loc, bool IsOffloadEntry, InsertPointTy AllocaIP,
     InsertPointTy CodeGenIP, TargetRegionEntryInfo &EntryInfo,
     ArrayRef<int32_t> NumTeams, ArrayRef<int32_t> NumThreads,
@@ -7276,9 +7436,10 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTarget(
   // The target region is outlined into its own function. The LLVM IR for
   // the target region itself is generated using the callbacks CBFunc
   // and ArgAccessorFuncCB
-  emitTargetOutlinedFunction(*this, Builder, IsOffloadEntry, EntryInfo,
-                             OutlinedFn, OutlinedFnID, Args, CBFunc,
-                             ArgAccessorFuncCB);
+  if (Error Err = emitTargetOutlinedFunction(
+          *this, Builder, IsOffloadEntry, EntryInfo, OutlinedFn, OutlinedFnID,
+          Args, CBFunc, ArgAccessorFuncCB))
+    return Err;
 
   // If we are not on the target device, then we need to generate code
   // to make a remote call (offload) to the previously outlined function
@@ -7767,18 +7928,17 @@ void OpenMPIRBuilder::emitBlock(BasicBlock *BB, Function *CurFn,
   Builder.SetInsertPoint(BB);
 }
 
-void OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
-                                   BodyGenCallbackTy ElseGen,
-                                   InsertPointTy AllocaIP) {
+Error OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
+                                    BodyGenCallbackTy ElseGen,
+                                    InsertPointTy AllocaIP) {
   // If the condition constant folds and can be elided, try to avoid emitting
   // the condition and the dead arm of the if/else.
   if (auto *CI = dyn_cast<ConstantInt>(Cond)) {
     auto CondConstant = CI->getSExtValue();
     if (CondConstant)
-      ThenGen(AllocaIP, Builder.saveIP());
-    else
-      ElseGen(AllocaIP, Builder.saveIP());
-    return;
+      return ThenGen(AllocaIP, Builder.saveIP());
+
+    return ElseGen(AllocaIP, Builder.saveIP());
   }
 
   Function *CurFn = Builder.GetInsertBlock()->getParent();
@@ -7791,16 +7951,19 @@ void OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
   Builder.CreateCondBr(Cond, ThenBlock, ElseBlock);
   // Emit the 'then' code.
   emitBlock(ThenBlock, CurFn);
-  ThenGen(AllocaIP, Builder.saveIP());
+  if (Error Err = ThenGen(AllocaIP, Builder.saveIP()))
+    return Err;
   emitBranch(ContBlock);
   // Emit the 'else' code if present.
   // There is no need to emit line number for unconditional branch.
   emitBlock(ElseBlock, CurFn);
-  ElseGen(AllocaIP, Builder.saveIP());
+  if (Error Err = ElseGen(AllocaIP, Builder.saveIP()))
+    return Err;
   // There is no need to emit line number for unconditional branch.
   emitBranch(ContBlock);
   // Emit the continuation block for code after the if.
   emitBlock(ContBlock, CurFn, /*IsFinished=*/true);
+  return Error::success();
 }
 
 bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(
@@ -7948,7 +8111,7 @@ OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,
   return Builder.saveIP();
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicUpdate(
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicUpdate(
     const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
     Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
     AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr) {
@@ -7969,8 +8132,11 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicUpdate(
            "OpenMP atomic does not support LT or GT operations");
   });
 
-  emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, RMWOp, UpdateOp,
-                   X.IsVolatile, IsXBinopExpr);
+  Expected<std::pair<Value *, Value *>> AtomicResult =
+      emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, RMWOp, UpdateOp,
+                       X.IsVolatile, IsXBinopExpr);
+  if (!AtomicResult)
+    return AtomicResult.takeError();
   checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Update);
   return Builder.saveIP();
 }
@@ -8010,7 +8176,7 @@ Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
   llvm_unreachable("Unsupported atomic update operation");
 }
 
-std::pair<Value *, Value *> OpenMPIRBuilder::emitAtomicUpdate(
+Expected<std::pair<Value *, Value *>> OpenMPIRBuilder::emitAtomicUpdate(
     InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
     AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
     AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr) {
@@ -8072,7 +8238,10 @@ std::pair<Value *, Value *> OpenMPIRBuilder::emitAtomicUpdate(
     llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(), 2);
     PHI->addIncoming(AtomicLoadRes.first, CurBB);
     Value *OldExprVal = PHI;
-    Value *Upd = UpdateOp(OldExprVal, Builder);
+    Expected<Value *> CBResult = UpdateOp(OldExprVal, Builder);
+    if (!CBResult)
+      return CBResult.takeError();
+    Value *Upd = *CBResult;
     Builder.CreateStore(Upd, NewAtomicAddr);
     AtomicOrdering Failure =
         llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
@@ -8129,7 +8298,10 @@ std::pair<Value *, Value *> OpenMPIRBuilder::emitAtomicUpdate(
       }
     }
 
-    Value *Upd = UpdateOp(OldExprVal, Builder);
+    Expected<Value *> CBResult = UpdateOp(OldExprVal, Builder);
+    if (!CBResult)
+      return CBResult.takeError();
+    Value *Upd = *CBResult;
     Builder.CreateStore(Upd, NewAtomicAddr);
     LoadInst *DesiredVal = Builder.CreateLoad(IntCastTy, NewAtomicAddr);
     AtomicOrdering Failure =
@@ -8158,7 +8330,7 @@ std::pair<Value *, Value *> OpenMPIRBuilder::emitAtomicUpdate(
   return Res;
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCapture(
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicCapture(
     const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
     AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
     AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
@@ -8181,11 +8353,13 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCapture(
   // If UpdateExpr is 'x' updated with some `expr` not based on 'x',
   // 'x' is simply atomically rewritten with 'expr'.
   AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);
-  std::pair<Value *, Value *> Result =
+  Expected<std::pair<Value *, Value *>> AtomicResult =
       emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, AtomicOp, UpdateOp,
                        X.IsVolatile, IsXBinopExpr);
-
-  Value *CapturedVal = (IsPostfixUpdate ? Result.first : Result.second);
+  if (!AtomicResult)
+    return AtomicResult.takeError();
+  Value *CapturedVal =
+      (IsPostfixUpdate ? AtomicResult->first : AtomicResult->second);
   Builder.CreateStore(CapturedVal, V.Var, V.IsVolatile);
 
   checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Capture);
@@ -8380,7 +8554,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
   return Builder.saveIP();
 }
 
-OpenMPIRBuilder::InsertPointTy
+OpenMPIRBuilder::InsertPointOrErrorTy
 OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
                              BodyGenCallbackTy BodyGenCB, Value *NumTeamsLower,
                              Value *NumTeamsUpper, Value *ThreadLimit,
@@ -8463,7 +8637,8 @@ OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
   // Generate the body of teams.
   InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin());
   InsertPointTy CodeGenIP(BodyBB, BodyBB->begin());
-  BodyGenCB(AllocaIP, CodeGenIP);
+  if (Error Err = BodyGenCB(AllocaIP, CodeGenIP))
+    return Err;
 
   OutlineInfo OI;
   OI.EntryBB = AllocaBB;
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index 35664a5c7a2a..9e25620710fc 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -1093,6 +1093,7 @@ private:
       CGStartBB->getTerminator()->setSuccessor(0, StartBB);
       assert(EndBB != nullptr && "EndBB should not be null");
       EndBB->getTerminator()->setSuccessor(0, CGEndBB);
+      return Error::success();
     };
 
     auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &,
@@ -1101,7 +1102,7 @@ private:
       return CodeGenIP;
     };
 
-    auto FiniCB = [&](InsertPointTy CodeGenIP) {};
+    auto FiniCB = [&](InsertPointTy CodeGenIP) { return Error::success(); };
 
     /// Create a sequential execution region within a merged parallel region,
     /// encapsulated in a master construct with a barrier for synchronization.
@@ -1132,8 +1133,9 @@ private:
         CGStartBB->getTerminator()->setSuccessor(0, SeqStartBB);
         assert(SeqEndBB != nullptr && "SeqEndBB should not be null");
         SeqEndBB->getTerminator()->setSuccessor(0, CGEndBB);
+        return Error::success();
       };
-      auto FiniCB = [&](InsertPointTy CodeGenIP) {};
+      auto FiniCB = [&](InsertPointTy CodeGenIP) { return Error::success(); };
 
       // Find outputs from the sequential region to outside users and
       // broadcast their values to them.
@@ -1176,12 +1178,15 @@ private:
 
       OpenMPIRBuilder::LocationDescription Loc(
           InsertPointTy(ParentBB, ParentBB->end()), DL);
-      InsertPointTy SeqAfterIP =
+      OpenMPIRBuilder::InsertPointOrErrorTy SeqAfterIP =
           OMPInfoCache.OMPBuilder.createMaster(Loc, BodyGenCB, FiniCB);
+      assert(SeqAfterIP && "Unexpected error creating master");
 
-      OMPInfoCache.OMPBuilder.createBarrier(SeqAfterIP, OMPD_parallel);
+      OpenMPIRBuilder::InsertPointOrErrorTy BarrierAfterIP =
+          OMPInfoCache.OMPBuilder.createBarrier(*SeqAfterIP, OMPD_parallel);
+      assert(BarrierAfterIP && "Unexpected error creating barrier");
 
-      BranchInst::Create(SeqAfterBB, SeqAfterIP.getBlock());
+      BranchInst::Create(SeqAfterBB, SeqAfterIP->getBlock());
 
       LLVM_DEBUG(dbgs() << TAG << "After sequential inlining " << *OuterFn
                         << "\n");
@@ -1251,10 +1256,12 @@ private:
           OriginalFn->getEntryBlock().getFirstInsertionPt());
       // Create the merged parallel region with default proc binding, to
       // avoid overriding binding settings, and without explicit cancellation.
-      InsertPointTy AfterIP = OMPInfoCache.OMPBuilder.createParallel(
-          Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr, nullptr,
-          OMP_PROC_BIND_default, /* IsCancellable */ false);
-      BranchInst::Create(AfterBB, AfterIP.getBlock());
+      OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+          OMPInfoCache.OMPBuilder.createParallel(
+              Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr, nullptr,
+              OMP_PROC_BIND_default, /* IsCancellable */ false);
+      assert(AfterIP && "Unexpected error creating parallel");
+      BranchInst::Create(AfterBB, AfterIP->getBlock());
 
       // Perform the actual outlining.
       OMPInfoCache.OMPBuilder.finalize(OriginalFn);
@@ -1290,10 +1297,12 @@ private:
         if (CI != MergableCIs.back()) {
           // TODO: Remove barrier if the merged parallel region includes the
           // 'nowait' clause.
-          OMPInfoCache.OMPBuilder.createBarrier(
-              InsertPointTy(NewCI->getParent(),
-                            NewCI->getNextNode()->getIterator()),
-              OMPD_parallel);
+          OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+              OMPInfoCache.OMPBuilder.createBarrier(
+                  InsertPointTy(NewCI->getParent(),
+                                NewCI->getNextNode()->getIterator()),
+                  OMPD_parallel);
+          assert(AfterIP && "Unexpected error creating barrier");
         }
 
         CI->eraseFromParent();
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index fe04cbbce12d..630cd03c6880 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -27,6 +27,20 @@
 using namespace llvm;
 using namespace omp;
 
+// Wrapper lambdas to allow using EXPECT*() macros inside of error-returning
+// callbacks.
+#define FINICB_WRAPPER(cb)                                                     \
+  [&cb](InsertPointTy IP) -> Error {                                           \
+    cb(IP);                                                                    \
+    return Error::success();                                                   \
+  }
+
+#define BODYGENCB_WRAPPER(cb)                                                  \
+  [&cb](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) -> Error {            \
+    cb(AllocaIP, CodeGenIP);                                                   \
+    return Error::success();                                                   \
+  }
+
 namespace {
 
 /// Create an instruction that uses the values in \p Values. We use "printf"
@@ -218,9 +232,13 @@ protected:
       CallInst *CallInst = createPrintfCall(Builder, "%d\\n", {LC});
       if (Call)
         *Call = CallInst;
+
+      return Error::success();
     };
-    CanonicalLoopInfo *Loop =
+    Expected<CanonicalLoopInfo *> LoopResult =
         OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, CastedTripCount);
+    assert(LoopResult && "unexpected error");
+    CanonicalLoopInfo *Loop = *LoopResult;
 
     // Finalize the function.
     Builder.restoreIP(Loop->getAfterIP());
@@ -327,14 +345,18 @@ TEST_F(OpenMPIRBuilderTest, CreateBarrier) {
 
   IRBuilder<> Builder(BB);
 
-  OMPBuilder.createBarrier({IRBuilder<>::InsertPoint()}, OMPD_for);
+  OpenMPIRBuilder::InsertPointOrErrorTy BarrierIP1 =
+      OMPBuilder.createBarrier({IRBuilder<>::InsertPoint()}, OMPD_for);
+  assert(BarrierIP1 && "unexpected error");
   EXPECT_TRUE(M->global_empty());
   EXPECT_EQ(M->size(), 1U);
   EXPECT_EQ(F->size(), 1U);
   EXPECT_EQ(BB->size(), 0U);
 
   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
-  OMPBuilder.createBarrier(Loc, OMPD_for);
+  OpenMPIRBuilder::InsertPointOrErrorTy BarrierIP2 =
+      OMPBuilder.createBarrier(Loc, OMPD_for);
+  assert(BarrierIP2 && "unexpected error");
   EXPECT_FALSE(M->global_empty());
   EXPECT_EQ(M->size(), 3U);
   EXPECT_EQ(F->size(), 1U);
@@ -372,13 +394,15 @@ TEST_F(OpenMPIRBuilderTest, CreateCancel) {
     ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
     BranchInst::Create(CBB, IP.getBlock());
   };
-  OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
+  OMPBuilder.pushFinalizationCB({FINICB_WRAPPER(FiniCB), OMPD_parallel, true});
 
   IRBuilder<> Builder(BB);
 
   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
-  auto NewIP = OMPBuilder.createCancel(Loc, nullptr, OMPD_parallel);
-  Builder.restoreIP(NewIP);
+  OpenMPIRBuilder::InsertPointOrErrorTy NewIP =
+      OMPBuilder.createCancel(Loc, nullptr, OMPD_parallel);
+  assert(NewIP && "unexpected error");
+  Builder.restoreIP(*NewIP);
   EXPECT_FALSE(M->global_empty());
   EXPECT_EQ(M->size(), 4U);
   EXPECT_EQ(F->size(), 4U);
@@ -400,7 +424,7 @@ TEST_F(OpenMPIRBuilderTest, CreateCancel) {
   EXPECT_EQ(Cancel->getNumUses(), 1U);
   Instruction *CancelBBTI = Cancel->getParent()->getTerminator();
   EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);
-  EXPECT_EQ(CancelBBTI->getSuccessor(0), NewIP.getBlock());
+  EXPECT_EQ(CancelBBTI->getSuccessor(0), NewIP->getBlock());
   EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U);
   CallInst *GTID1 = dyn_cast<CallInst>(&CancelBBTI->getSuccessor(1)->front());
   EXPECT_NE(GTID1, nullptr);
@@ -439,13 +463,15 @@ TEST_F(OpenMPIRBuilderTest, CreateCancelIfCond) {
     ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
     BranchInst::Create(CBB, IP.getBlock());
   };
-  OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
+  OMPBuilder.pushFinalizationCB({FINICB_WRAPPER(FiniCB), OMPD_parallel, true});
 
   IRBuilder<> Builder(BB);
 
   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
-  auto NewIP = OMPBuilder.createCancel(Loc, Builder.getTrue(), OMPD_parallel);
-  Builder.restoreIP(NewIP);
+  OpenMPIRBuilder::InsertPointOrErrorTy NewIP =
+      OMPBuilder.createCancel(Loc, Builder.getTrue(), OMPD_parallel);
+  assert(NewIP && "unexpected error");
+  Builder.restoreIP(*NewIP);
   EXPECT_FALSE(M->global_empty());
   EXPECT_EQ(M->size(), 4U);
   EXPECT_EQ(F->size(), 7U);
@@ -473,7 +499,7 @@ TEST_F(OpenMPIRBuilderTest, CreateCancelIfCond) {
   EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);
   EXPECT_EQ(CancelBBTI->getSuccessor(0)->size(), 1U);
   EXPECT_EQ(CancelBBTI->getSuccessor(0)->getUniqueSuccessor(),
-            NewIP.getBlock());
+            NewIP->getBlock());
   EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U);
   CallInst *GTID1 = dyn_cast<CallInst>(&CancelBBTI->getSuccessor(1)->front());
   EXPECT_NE(GTID1, nullptr);
@@ -512,13 +538,15 @@ TEST_F(OpenMPIRBuilderTest, CreateCancelBarrier) {
     ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
     BranchInst::Create(CBB, IP.getBlock());
   };
-  OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
+  OMPBuilder.pushFinalizationCB({FINICB_WRAPPER(FiniCB), OMPD_parallel, true});
 
   IRBuilder<> Builder(BB);
 
   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
-  auto NewIP = OMPBuilder.createBarrier(Loc, OMPD_for);
-  Builder.restoreIP(NewIP);
+  OpenMPIRBuilder::InsertPointOrErrorTy NewIP =
+      OMPBuilder.createBarrier(Loc, OMPD_for);
+  assert(NewIP && "unexpected error");
+  Builder.restoreIP(*NewIP);
   EXPECT_FALSE(M->global_empty());
   EXPECT_EQ(M->size(), 3U);
   EXPECT_EQ(F->size(), 4U);
@@ -540,7 +568,7 @@ TEST_F(OpenMPIRBuilderTest, CreateCancelBarrier) {
   EXPECT_EQ(Barrier->getNumUses(), 1U);
   Instruction *BarrierBBTI = Barrier->getParent()->getTerminator();
   EXPECT_EQ(BarrierBBTI->getNumSuccessors(), 2U);
-  EXPECT_EQ(BarrierBBTI->getSuccessor(0), NewIP.getBlock());
+  EXPECT_EQ(BarrierBBTI->getSuccessor(0), NewIP->getBlock());
   EXPECT_EQ(BarrierBBTI->getSuccessor(1)->size(), 1U);
   EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
             1U);
@@ -563,7 +591,9 @@ TEST_F(OpenMPIRBuilderTest, DbgLoc) {
   IRBuilder<> Builder(BB);
 
   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
-  OMPBuilder.createBarrier(Loc, OMPD_for);
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      OMPBuilder.createBarrier(Loc, OMPD_for);
+  assert(AfterIP && "unexpected error");
   CallInst *GTID = dyn_cast<CallInst>(&BB->front());
   CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());
   EXPECT_EQ(GTID->getDebugLoc(), DL);
@@ -627,6 +657,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimpleGPU) {
     Instruction *ThenTerm, *ElseTerm;
     SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
                                   &ThenTerm, &ElseTerm);
+    return Error::success();
   };
 
   auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
@@ -654,19 +685,23 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimpleGPU) {
     return CodeGenIP;
   };
 
-  auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
+  auto FiniCB = [&](InsertPointTy CodeGenIP) {
+    ++NumFinalizationPoints;
+    return Error::success();
+  };
 
   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
                                     F->getEntryBlock().getFirstInsertionPt());
-  IRBuilder<>::InsertPoint AfterIP =
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
       OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
                                 nullptr, nullptr, OMP_PROC_BIND_default, false);
+  assert(AfterIP && "unexpected error");
 
   EXPECT_EQ(NumBodiesGenerated, 1U);
   EXPECT_EQ(NumPrivatizedVars, 1U);
   EXPECT_EQ(NumFinalizationPoints, 1U);
 
-  Builder.restoreIP(AfterIP);
+  Builder.restoreIP(*AfterIP);
   Builder.CreateRetVoid();
 
   OMPBuilder.finalize();
@@ -735,6 +770,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimple) {
     Instruction *ThenTerm, *ElseTerm;
     SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
                                   &ThenTerm, &ElseTerm);
+    return Error::success();
   };
 
   auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
@@ -762,18 +798,22 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimple) {
     return CodeGenIP;
   };
 
-  auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
+  auto FiniCB = [&](InsertPointTy CodeGenIP) {
+    ++NumFinalizationPoints;
+    return Error::success();
+  };
 
   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
                                     F->getEntryBlock().getFirstInsertionPt());
-  IRBuilder<>::InsertPoint AfterIP =
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
       OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
                                 nullptr, nullptr, OMP_PROC_BIND_default, false);
+  assert(AfterIP && "unexpected error");
   EXPECT_EQ(NumBodiesGenerated, 1U);
   EXPECT_EQ(NumPrivatizedVars, 1U);
   EXPECT_EQ(NumFinalizationPoints, 1U);
 
-  Builder.restoreIP(AfterIP);
+  Builder.restoreIP(*AfterIP);
   Builder.CreateRetVoid();
 
   OMPBuilder.finalize();
@@ -826,6 +866,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested) {
 
   auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
     ++NumInnerBodiesGenerated;
+    return Error::success();
   };
 
   auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
@@ -841,7 +882,10 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested) {
     return CodeGenIP;
   };
 
-  auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
+  auto FiniCB = [&](InsertPointTy CodeGenIP) {
+    ++NumFinalizationPoints;
+    return Error::success();
+  };
 
   auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
     ++NumOuterBodiesGenerated;
@@ -849,27 +893,29 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested) {
     BasicBlock *CGBB = CodeGenIP.getBlock();
     BasicBlock *NewBB = SplitBlock(CGBB, &*CodeGenIP.getPoint());
     CGBB->getTerminator()->eraseFromParent();
-    ;
 
-    IRBuilder<>::InsertPoint AfterIP = OMPBuilder.createParallel(
+    OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createParallel(
         InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB,
         FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
+    assert(AfterIP && "unexpected error");
 
-    Builder.restoreIP(AfterIP);
+    Builder.restoreIP(*AfterIP);
     Builder.CreateBr(NewBB);
+    return Error::success();
   };
 
   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
                                     F->getEntryBlock().getFirstInsertionPt());
-  IRBuilder<>::InsertPoint AfterIP =
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
       OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB,
                                 nullptr, nullptr, OMP_PROC_BIND_default, false);
+  assert(AfterIP && "unexpected error");
 
   EXPECT_EQ(NumInnerBodiesGenerated, 1U);
   EXPECT_EQ(NumOuterBodiesGenerated, 1U);
   EXPECT_EQ(NumFinalizationPoints, 2U);
 
-  Builder.restoreIP(AfterIP);
+  Builder.restoreIP(*AfterIP);
   Builder.CreateRetVoid();
 
   OMPBuilder.finalize();
@@ -920,6 +966,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
 
   auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
     ++NumInnerBodiesGenerated;
+    return Error::success();
   };
 
   auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
@@ -935,7 +982,10 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
     return CodeGenIP;
   };
 
-  auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
+  auto FiniCB = [&](InsertPointTy CodeGenIP) {
+    ++NumFinalizationPoints;
+    return Error::success();
+  };
 
   auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
     ++NumOuterBodiesGenerated;
@@ -948,32 +998,36 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
     NewBB1->getTerminator()->eraseFromParent();
     ;
 
-    IRBuilder<>::InsertPoint AfterIP1 = OMPBuilder.createParallel(
+    OpenMPIRBuilder::InsertPointOrErrorTy AfterIP1 = OMPBuilder.createParallel(
         InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB,
         FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
+    assert(AfterIP1 && "unexpected error");
 
-    Builder.restoreIP(AfterIP1);
+    Builder.restoreIP(*AfterIP1);
     Builder.CreateBr(NewBB1);
 
-    IRBuilder<>::InsertPoint AfterIP2 = OMPBuilder.createParallel(
+    OpenMPIRBuilder::InsertPointOrErrorTy AfterIP2 = OMPBuilder.createParallel(
         InsertPointTy(NewBB1, NewBB1->end()), AllocaIP, InnerBodyGenCB, PrivCB,
         FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
+    assert(AfterIP2 && "unexpected error");
 
-    Builder.restoreIP(AfterIP2);
+    Builder.restoreIP(*AfterIP2);
     Builder.CreateBr(NewBB2);
+    return Error::success();
   };
 
   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
                                     F->getEntryBlock().getFirstInsertionPt());
-  IRBuilder<>::InsertPoint AfterIP =
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
       OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB,
                                 nullptr, nullptr, OMP_PROC_BIND_default, false);
+  assert(AfterIP && "unexpected error");
 
   EXPECT_EQ(NumInnerBodiesGenerated, 2U);
   EXPECT_EQ(NumOuterBodiesGenerated, 1U);
   EXPECT_EQ(NumFinalizationPoints, 3U);
 
-  Builder.restoreIP(AfterIP);
+  Builder.restoreIP(*AfterIP);
   Builder.CreateRetVoid();
 
   OMPBuilder.finalize();
@@ -1043,6 +1097,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelIfCond) {
     Instruction *ThenTerm, *ElseTerm;
     SplitBlockAndInsertIfThenElse(Cmp, &*Builder.GetInsertPoint(), &ThenTerm,
                                   &ElseTerm);
+    return Error::success();
   };
 
   auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
@@ -1073,20 +1128,22 @@ TEST_F(OpenMPIRBuilderTest, ParallelIfCond) {
   auto FiniCB = [&](InsertPointTy CodeGenIP) {
     ++NumFinalizationPoints;
     // No destructors.
+    return Error::success();
   };
 
   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
                                     F->getEntryBlock().getFirstInsertionPt());
-  IRBuilder<>::InsertPoint AfterIP =
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
       OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
                                 Builder.CreateIsNotNull(F->arg_begin()),
                                 nullptr, OMP_PROC_BIND_default, false);
+  assert(AfterIP && "unexpected error");
 
   EXPECT_EQ(NumBodiesGenerated, 1U);
   EXPECT_EQ(NumPrivatizedVars, 1U);
   EXPECT_EQ(NumFinalizationPoints, 1U);
 
-  Builder.restoreIP(AfterIP);
+  Builder.restoreIP(*AfterIP);
   Builder.CreateRetVoid();
   OMPBuilder.finalize();
 
@@ -1141,8 +1198,10 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
     // Create three barriers, two cancel barriers but only one checked.
     Function *CBFn, *BFn;
 
-    Builder.restoreIP(
-        OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel));
+    OpenMPIRBuilder::InsertPointOrErrorTy BarrierIP1 =
+        OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel);
+    assert(BarrierIP1 && "unexpected error");
+    Builder.restoreIP(*BarrierIP1);
 
     CBFn = M->getFunction("__kmpc_cancel_barrier");
     BFn = M->getFunction("__kmpc_barrier");
@@ -1153,8 +1212,10 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
     ASSERT_EQ(CBFn->user_back()->getNumUses(), 1U);
     CheckedBarrier = cast<CallInst>(CBFn->user_back());
 
-    Builder.restoreIP(
-        OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel, true));
+    OpenMPIRBuilder::InsertPointOrErrorTy BarrierIP2 =
+        OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel, true);
+    assert(BarrierIP2 && "unexpected error");
+    Builder.restoreIP(*BarrierIP2);
     CBFn = M->getFunction("__kmpc_cancel_barrier");
     BFn = M->getFunction("__kmpc_barrier");
     ASSERT_NE(CBFn, nullptr);
@@ -1164,8 +1225,10 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
     ASSERT_TRUE(isa<CallInst>(BFn->user_back()));
     ASSERT_EQ(BFn->user_back()->getNumUses(), 0U);
 
-    Builder.restoreIP(OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel,
-                                               false, false));
+    OpenMPIRBuilder::InsertPointOrErrorTy BarrierIP3 =
+        OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel, false, false);
+    assert(BarrierIP3 && "unexpected error");
+    Builder.restoreIP(*BarrierIP3);
     ASSERT_EQ(CBFn->getNumUses(), 2U);
     ASSERT_EQ(BFn->getNumUses(), 1U);
     ASSERT_TRUE(CBFn->user_back() != CheckedBarrier);
@@ -1190,21 +1253,23 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
     Builder.restoreIP(IP);
     Builder.CreateCall(FakeDestructor,
                        {Builder.getInt32(NumFinalizationPoints)});
+    return Error::success();
   };
 
   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
                                     F->getEntryBlock().getFirstInsertionPt());
-  IRBuilder<>::InsertPoint AfterIP =
-      OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
-                                Builder.CreateIsNotNull(F->arg_begin()),
-                                nullptr, OMP_PROC_BIND_default, true);
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createParallel(
+      Loc, AllocaIP, BODYGENCB_WRAPPER(BodyGenCB), PrivCB, FiniCB,
+      Builder.CreateIsNotNull(F->arg_begin()), nullptr, OMP_PROC_BIND_default,
+      true);
+  assert(AfterIP && "unexpected error");
 
   EXPECT_EQ(NumBodiesGenerated, 1U);
   EXPECT_EQ(NumPrivatizedVars, 0U);
   EXPECT_EQ(NumFinalizationPoints, 2U);
   EXPECT_EQ(FakeDestructor->getNumUses(), 2U);
 
-  Builder.restoreIP(AfterIP);
+  Builder.restoreIP(*AfterIP);
   Builder.CreateRetVoid();
   OMPBuilder.finalize();
 
@@ -1269,20 +1334,22 @@ TEST_F(OpenMPIRBuilderTest, ParallelForwardAsPointers) {
     Builder.CreateCall(TakeI32PtrFunc, I32PtrVal);
     Builder.CreateCall(TakeStructFunc, StructVal);
     Builder.CreateCall(TakeStructPtrFunc, StructPtrVal);
+    return Error::success();
   };
   auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &,
                     Value &Inner, Value *&ReplacementValue) {
     ReplacementValue = &Inner;
     return CodeGenIP;
   };
-  auto FiniCB = [](InsertPointTy) {};
+  auto FiniCB = [](InsertPointTy) { return Error::success(); };
 
   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
                                     F->getEntryBlock().getFirstInsertionPt());
-  IRBuilder<>::InsertPoint AfterIP =
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
       OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
                                 nullptr, nullptr, OMP_PROC_BIND_default, false);
-  Builder.restoreIP(AfterIP);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   Builder.CreateRetVoid();
 
   OMPBuilder.finalize();
@@ -1312,10 +1379,13 @@ TEST_F(OpenMPIRBuilderTest, CanonicalLoopSimple) {
     Instruction *ThenTerm, *ElseTerm;
     SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
                                   &ThenTerm, &ElseTerm);
+    return Error::success();
   };
 
-  CanonicalLoopInfo *Loop =
+  Expected<CanonicalLoopInfo *> LoopResult =
       OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, TripCount);
+  assert(LoopResult && "unexpected error");
+  CanonicalLoopInfo *Loop = *LoopResult;
 
   Builder.restoreIP(Loop->getAfterIP());
   ReturnInst *RetInst = Builder.CreateRetVoid();
@@ -1367,10 +1437,14 @@ TEST_F(OpenMPIRBuilderTest, CanonicalLoopBounds) {
     Value *StartVal = ConstantInt::get(LCTy, Start);
     Value *StopVal = ConstantInt::get(LCTy, Stop);
     Value *StepVal = ConstantInt::get(LCTy, Step);
-    auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {};
-    CanonicalLoopInfo *Loop =
+    auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {
+      return Error::success();
+    };
+    Expected<CanonicalLoopInfo *> LoopResult =
         OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, StartVal, StopVal,
                                        StepVal, IsSigned, InclusiveStop);
+    assert(LoopResult && "unexpected error");
+    CanonicalLoopInfo *Loop = *LoopResult;
     Loop->assertOK();
     Builder.restoreIP(Loop->getAfterIP());
     Value *TripCount = Loop->getTripCount();
@@ -1463,16 +1537,22 @@ TEST_F(OpenMPIRBuilderTest, CollapseNestedLoops) {
                                   Value *InnerLC) {
       Builder.restoreIP(InnerCodeGenIP);
       Call = createPrintfCall(Builder, "body i=%d j=%d\\n", {OuterLC, InnerLC});
+      return Error::success();
     };
-    InnerLoop = OMPBuilder.createCanonicalLoop(
+    Expected<CanonicalLoopInfo *> LoopResult = OMPBuilder.createCanonicalLoop(
         Builder.saveIP(), InnerLoopBodyGenCB, InnerTripCount, "inner");
+    assert(LoopResult && "unexpected error");
+    InnerLoop = *LoopResult;
 
     Builder.restoreIP(InnerLoop->getAfterIP());
     InbetweenTrail =
         createPrintfCall(Builder, "In-between trail i=%d\\n", {OuterLC});
+    return Error::success();
   };
-  CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
+  Expected<CanonicalLoopInfo *> LoopResult = OMPBuilder.createCanonicalLoop(
       OuterLoc, OuterLoopBodyGenCB, OuterTripCount, "outer");
+  assert(LoopResult && "unexpected error");
+  CanonicalLoopInfo *OuterLoop = *LoopResult;
 
   // Finish the function.
   Builder.restoreIP(OuterLoop->getAfterIP());
@@ -1582,12 +1662,18 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoops) {
 
       // Add something that consumes the induction variables to the body.
       createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC});
+      return Error::success();
     };
-    InnerLoop = OMPBuilder.createCanonicalLoop(
+    Expected<CanonicalLoopInfo *> LoopResult = OMPBuilder.createCanonicalLoop(
         OuterCodeGenIP, InnerLoopBodyGenCB, TripCount, "inner");
+    assert(LoopResult && "unexpected error");
+    InnerLoop = *LoopResult;
+    return Error::success();
   };
-  CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
+  Expected<CanonicalLoopInfo *> LoopResult = OMPBuilder.createCanonicalLoop(
       Loc, OuterLoopBodyGenCB, TripCount, "outer");
+  assert(LoopResult && "unexpected error");
+  CanonicalLoopInfo *OuterLoop = *LoopResult;
 
   // Finalize the function.
   Builder.restoreIP(OuterLoop->getAfterIP());
@@ -1682,14 +1768,20 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) {
 
       // Add something that consumes the induction variable to the body.
       Call = createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC});
+      return Error::success();
     };
-    InnerLoop = OMPBuilder.createCanonicalLoop(
+    Expected<CanonicalLoopInfo *> LoopResult = OMPBuilder.createCanonicalLoop(
         OuterCodeGenIP, InnerLoopBodyGenCB, InnerStartVal, InnerStopVal,
         InnerStep, false, false, ComputeIP, "inner");
+    assert(LoopResult && "unexpected error");
+    InnerLoop = *LoopResult;
+    return Error::success();
   };
-  CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
+  Expected<CanonicalLoopInfo *> LoopResult = OMPBuilder.createCanonicalLoop(
       Loc, OuterLoopBodyGenCB, OuterStartVal, OuterStopVal, OuterStep, false,
       false, ComputeIP, "outer");
+  assert(LoopResult && "unexpected error");
+  CanonicalLoopInfo *OuterLoop = *LoopResult;
 
   // Finalize the function
   Builder.restoreIP(OuterLoop->getAfterIP());
@@ -1793,10 +1885,14 @@ TEST_F(OpenMPIRBuilderTest, TileSingleLoopCounts) {
     Value *StepVal = ConstantInt::get(LCTy, Step);
 
     // Generate a loop.
-    auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {};
-    CanonicalLoopInfo *Loop =
+    auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {
+      return Error::success();
+    };
+    Expected<CanonicalLoopInfo *> LoopResult =
         OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, StartVal, StopVal,
                                        StepVal, IsSigned, InclusiveStop);
+    assert(LoopResult && "unexpected error");
+    CanonicalLoopInfo *Loop = *LoopResult;
     InsertPointTy AfterIP = Loop->getAfterIP();
 
     // Tile the loop.
@@ -2245,19 +2341,22 @@ TEST_F(OpenMPIRBuilderTest, StaticWorkshareLoopTarget) {
   Value *StartVal = ConstantInt::get(LCTy, 10);
   Value *StopVal = ConstantInt::get(LCTy, 52);
   Value *StepVal = ConstantInt::get(LCTy, 2);
-  auto LoopBodyGen = [&](InsertPointTy, Value *) {};
+  auto LoopBodyGen = [&](InsertPointTy, Value *) { return Error::success(); };
 
-  CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
+  Expected<CanonicalLoopInfo *> LoopResult = OMPBuilder.createCanonicalLoop(
       Loc, LoopBodyGen, StartVal, StopVal, StepVal, false, false);
+  assert(LoopResult && "unexpected error");
+  CanonicalLoopInfo *CLI = *LoopResult;
   BasicBlock *Preheader = CLI->getPreheader();
   Value *TripCount = CLI->getTripCount();
 
   Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
 
-  IRBuilder<>::InsertPoint AfterIP = OMPBuilder.applyWorkshareLoop(
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.applyWorkshareLoop(
       DL, CLI, AllocaIP, true, OMP_SCHEDULE_Static, nullptr, false, false,
       false, false, WorksharingLoopType::ForStaticLoop);
-  Builder.restoreIP(AfterIP);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   Builder.CreateRetVoid();
 
   OMPBuilder.finalize();
@@ -2306,11 +2405,15 @@ TEST_F(OpenMPIRBuilderTest, StaticWorkShareLoop) {
   Value *StartVal = ConstantInt::get(LCTy, 10);
   Value *StopVal = ConstantInt::get(LCTy, 52);
   Value *StepVal = ConstantInt::get(LCTy, 2);
-  auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
+  auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {
+    return Error::success();
+  };
 
-  CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
+  Expected<CanonicalLoopInfo *> LoopResult = OMPBuilder.createCanonicalLoop(
       Loc, LoopBodyGen, StartVal, StopVal, StepVal,
       /*IsSigned=*/false, /*InclusiveStop=*/false);
+  assert(LoopResult && "unexpected error");
+  CanonicalLoopInfo *CLI = *LoopResult;
   BasicBlock *Preheader = CLI->getPreheader();
   BasicBlock *Body = CLI->getBody();
   Value *IV = CLI->getIndVar();
@@ -2319,8 +2422,9 @@ TEST_F(OpenMPIRBuilderTest, StaticWorkShareLoop) {
   Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
   InsertPointTy AllocaIP = Builder.saveIP();
 
-  OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true,
-                                OMP_SCHEDULE_Static);
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.applyWorkshareLoop(
+      DL, CLI, AllocaIP, /*NeedsBarrier=*/true, OMP_SCHEDULE_Static);
+  assert(AfterIP && "unexpected error");
 
   BasicBlock *Cond = Body->getSinglePredecessor();
   Instruction *Cmp = &*Cond->begin();
@@ -2412,8 +2516,9 @@ TEST_P(OpenMPIRBuilderTestWithIVBits, StaticChunkedWorkshareLoop) {
   Value *ChunkSize = ConstantInt::get(LCTy, 5);
   InsertPointTy AllocaIP{&F->getEntryBlock(),
                          F->getEntryBlock().getFirstInsertionPt()};
-  OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true,
-                                OMP_SCHEDULE_Static, ChunkSize);
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.applyWorkshareLoop(
+      DL, CLI, AllocaIP, /*NeedsBarrier=*/true, OMP_SCHEDULE_Static, ChunkSize);
+  assert(AfterIP && "unexpected error");
 
   OMPBuilder.finalize();
   EXPECT_FALSE(verifyModule(*M, &errs()));
@@ -2500,11 +2605,15 @@ TEST_P(OpenMPIRBuilderTestWithParams, DynamicWorkShareLoop) {
   Value *StepVal = ConstantInt::get(LCTy, 2);
   Value *ChunkVal =
       (ChunkSize == 1) ? nullptr : ConstantInt::get(LCTy, ChunkSize);
-  auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
+  auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {
+    return Error::success();
+  };
 
-  CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
+  Expected<CanonicalLoopInfo *> LoopResult = OMPBuilder.createCanonicalLoop(
       Loc, LoopBodyGen, StartVal, StopVal, StepVal,
       /*IsSigned=*/false, /*InclusiveStop=*/false);
+  assert(LoopResult && "unexpected error");
+  CanonicalLoopInfo *CLI = *LoopResult;
 
   Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
   InsertPointTy AllocaIP = Builder.saveIP();
@@ -2517,7 +2626,7 @@ TEST_P(OpenMPIRBuilderTestWithParams, DynamicWorkShareLoop) {
   BasicBlock *LatchBlock = CLI->getLatch();
   Value *IV = CLI->getIndVar();
 
-  InsertPointTy EndIP = OMPBuilder.applyWorkshareLoop(
+  OpenMPIRBuilder::InsertPointOrErrorTy EndIP = OMPBuilder.applyWorkshareLoop(
       DL, CLI, AllocaIP, /*NeedsBarrier=*/true, getSchedKind(SchedType),
       ChunkVal, /*Simd=*/false,
       (SchedType & omp::OMPScheduleType::ModifierMonotonic) ==
@@ -2525,10 +2634,11 @@ TEST_P(OpenMPIRBuilderTestWithParams, DynamicWorkShareLoop) {
       (SchedType & omp::OMPScheduleType::ModifierNonmonotonic) ==
           omp::OMPScheduleType::ModifierNonmonotonic,
       /*Ordered=*/false);
+  assert(EndIP && "unexpected error");
 
   // The returned value should be the "after" point.
-  ASSERT_EQ(EndIP.getBlock(), AfterIP.getBlock());
-  ASSERT_EQ(EndIP.getPoint(), AfterIP.getPoint());
+  ASSERT_EQ(EndIP->getBlock(), AfterIP.getBlock());
+  ASSERT_EQ(EndIP->getPoint(), AfterIP.getPoint());
 
   auto AllocaIter = BB->begin();
   ASSERT_GE(std::distance(BB->begin(), BB->end()), 4);
@@ -2603,7 +2713,7 @@ TEST_P(OpenMPIRBuilderTestWithParams, DynamicWorkShareLoop) {
   EXPECT_EQ(NumCallsInExitBlock, 2u);
 
   // Add a termination to our block and check that it is internally consistent.
-  Builder.restoreIP(EndIP);
+  Builder.restoreIP(*EndIP);
   Builder.CreateRetVoid();
   OMPBuilder.finalize();
   EXPECT_FALSE(verifyModule(*M, &errs()));
@@ -2642,11 +2752,15 @@ TEST_F(OpenMPIRBuilderTest, DynamicWorkShareLoopOrdered) {
   Value *StopVal = ConstantInt::get(LCTy, 52);
   Value *StepVal = ConstantInt::get(LCTy, 2);
   Value *ChunkVal = ConstantInt::get(LCTy, ChunkSize);
-  auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
+  auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {
+    return llvm::Error::success();
+  };
 
-  CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
+  Expected<CanonicalLoopInfo *> LoopResult = OMPBuilder.createCanonicalLoop(
       Loc, LoopBodyGen, StartVal, StopVal, StepVal,
       /*IsSigned=*/false, /*InclusiveStop=*/false);
+  assert(LoopResult && "unexpected error");
+  CanonicalLoopInfo *CLI = *LoopResult;
 
   Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
   InsertPointTy AllocaIP = Builder.saveIP();
@@ -2658,14 +2772,15 @@ TEST_F(OpenMPIRBuilderTest, DynamicWorkShareLoopOrdered) {
   BasicBlock *LatchBlock = CLI->getLatch();
   Value *IV = CLI->getIndVar();
 
-  InsertPointTy EndIP = OMPBuilder.applyWorkshareLoop(
+  OpenMPIRBuilder::InsertPointOrErrorTy EndIP = OMPBuilder.applyWorkshareLoop(
       DL, CLI, AllocaIP, /*NeedsBarrier=*/true, OMP_SCHEDULE_Static, ChunkVal,
       /*HasSimdModifier=*/false, /*HasMonotonicModifier=*/false,
       /*HasNonmonotonicModifier=*/false,
       /*HasOrderedClause=*/true);
+  assert(EndIP && "unexpected error");
 
   // Add a termination to our block and check that it is internally consistent.
-  Builder.restoreIP(EndIP);
+  Builder.restoreIP(*EndIP);
   Builder.CreateRetVoid();
   OMPBuilder.finalize();
   EXPECT_FALSE(verifyModule(*M, &errs()));
@@ -2749,7 +2864,10 @@ TEST_F(OpenMPIRBuilderTest, MasterDirective) {
     EXPECT_NE(IPBB->end(), IP.getPoint());
   };
 
-  Builder.restoreIP(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createMaster(
+      Builder, BODYGENCB_WRAPPER(BodyGenCB), FINICB_WRAPPER(FiniCB));
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   Value *EntryBBTI = EntryBB->getTerminator();
   EXPECT_NE(EntryBBTI, nullptr);
   EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
@@ -2827,8 +2945,10 @@ TEST_F(OpenMPIRBuilderTest, MaskedDirective) {
   };
 
   Constant *Filter = ConstantInt::get(Type::getInt32Ty(M->getContext()), 0);
-  Builder.restoreIP(
-      OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, Filter));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createMasked(
+      Builder, BODYGENCB_WRAPPER(BodyGenCB), FINICB_WRAPPER(FiniCB), Filter);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   Value *EntryBBTI = EntryBB->getTerminator();
   EXPECT_NE(EntryBBTI, nullptr);
   EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
@@ -2893,8 +3013,11 @@ TEST_F(OpenMPIRBuilderTest, CriticalDirective) {
   };
   BasicBlock *EntryBB = Builder.GetInsertBlock();
 
-  Builder.restoreIP(OMPBuilder.createCritical(Builder, BodyGenCB, FiniCB,
-                                              "testCRT", nullptr));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      OMPBuilder.createCritical(Builder, BODYGENCB_WRAPPER(BodyGenCB),
+                                FINICB_WRAPPER(FiniCB), "testCRT", nullptr);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
 
   CallInst *CriticalEntryCI = nullptr;
   for (auto &EI : *EntryBB) {
@@ -3141,8 +3264,11 @@ TEST_F(OpenMPIRBuilderTest, OrderedDirectiveThreads) {
 
   // Test for "#omp ordered [threads]"
   BasicBlock *EntryBB = Builder.GetInsertBlock();
-  Builder.restoreIP(
-      OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, true));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      OMPBuilder.createOrderedThreadsSimd(Builder, BODYGENCB_WRAPPER(BodyGenCB),
+                                          FINICB_WRAPPER(FiniCB), true);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
 
   Builder.CreateRetVoid();
   OMPBuilder.finalize();
@@ -3212,8 +3338,11 @@ TEST_F(OpenMPIRBuilderTest, OrderedDirectiveSimd) {
 
   // Test for "#omp ordered simd"
   BasicBlock *EntryBB = Builder.GetInsertBlock();
-  Builder.restoreIP(
-      OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, false));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      OMPBuilder.createOrderedThreadsSimd(Builder, BODYGENCB_WRAPPER(BodyGenCB),
+                                          FINICB_WRAPPER(FiniCB), false);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
 
   Builder.CreateRetVoid();
   OMPBuilder.finalize();
@@ -3326,8 +3455,11 @@ TEST_F(OpenMPIRBuilderTest, SingleDirective) {
     EXPECT_NE(IPBB->end(), IP.getPoint());
   };
 
-  Builder.restoreIP(
-      OMPBuilder.createSingle(Builder, BodyGenCB, FiniCB, /*IsNowait*/ false));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      OMPBuilder.createSingle(Builder, BODYGENCB_WRAPPER(BodyGenCB),
+                              FINICB_WRAPPER(FiniCB), /*IsNowait*/ false);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   Value *EntryBBTI = EntryBB->getTerminator();
   EXPECT_NE(EntryBBTI, nullptr);
   EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
@@ -3416,8 +3548,11 @@ TEST_F(OpenMPIRBuilderTest, SingleDirectiveNowait) {
     EXPECT_NE(IPBB->end(), IP.getPoint());
   };
 
-  Builder.restoreIP(
-      OMPBuilder.createSingle(Builder, BodyGenCB, FiniCB, /*IsNowait*/ true));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      OMPBuilder.createSingle(Builder, BODYGENCB_WRAPPER(BodyGenCB),
+                              FINICB_WRAPPER(FiniCB), /*IsNowait*/ true);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   Value *EntryBBTI = EntryBB->getTerminator();
   EXPECT_NE(EntryBBTI, nullptr);
   EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
@@ -3535,9 +3670,11 @@ TEST_F(OpenMPIRBuilderTest, SingleDirectiveCopyPrivate) {
     EXPECT_NE(IPBB->end(), IP.getPoint());
   };
 
-  Builder.restoreIP(OMPBuilder.createSingle(Builder, BodyGenCB, FiniCB,
-                                            /*IsNowait*/ false, {CPVar},
-                                            {CopyFunc}));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createSingle(
+      Builder, BODYGENCB_WRAPPER(BodyGenCB), FINICB_WRAPPER(FiniCB),
+      /*IsNowait*/ false, {CPVar}, {CopyFunc});
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   Value *EntryBBTI = EntryBB->getTerminator();
   EXPECT_NE(EntryBBTI, nullptr);
   EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
@@ -3798,8 +3935,10 @@ TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdate) {
     Sub = IRB.CreateSub(ConstVal, Atomic);
     return Sub;
   };
-  Builder.restoreIP(OMPBuilder.createAtomicUpdate(
-      Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createAtomicUpdate(
+      Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   BasicBlock *ContBB = EntryBB->getSingleSuccessor();
   BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());
   EXPECT_NE(ContTI, nullptr);
@@ -3865,8 +4004,10 @@ TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateFloat) {
     Sub = IRB.CreateFSub(ConstVal, Atomic);
     return Sub;
   };
-  Builder.restoreIP(OMPBuilder.createAtomicUpdate(
-      Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createAtomicUpdate(
+      Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   BasicBlock *ContBB = EntryBB->getSingleSuccessor();
   BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());
   EXPECT_NE(ContTI, nullptr);
@@ -3931,8 +4072,10 @@ TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateIntr) {
     Sub = IRB.CreateSub(ConstVal, Atomic);
     return Sub;
   };
-  Builder.restoreIP(OMPBuilder.createAtomicUpdate(
-      Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createAtomicUpdate(
+      Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   BasicBlock *ContBB = EntryBB->getSingleSuccessor();
   BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());
   EXPECT_NE(ContTI, nullptr);
@@ -4003,9 +4146,12 @@ TEST_F(OpenMPIRBuilderTest, OMPAtomicCapture) {
   // integer update - not used
   auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) { return nullptr; };
 
-  Builder.restoreIP(OMPBuilder.createAtomicCapture(
-      Builder, AllocaIP, X, V, Expr, AO, RMWOp, UpdateOp, UpdateExpr,
-      IsPostfixUpdate, IsXLHSInRHSPart));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      OMPBuilder.createAtomicCapture(Builder, AllocaIP, X, V, Expr, AO, RMWOp,
+                                     UpdateOp, UpdateExpr, IsPostfixUpdate,
+                                     IsXLHSInRHSPart);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   EXPECT_EQ(EntryBB->getParent()->size(), 1U);
   AtomicRMWInst *ARWM = dyn_cast<AtomicRMWInst>(Init->getNextNode());
   EXPECT_NE(ARWM, nullptr);
@@ -4361,12 +4507,15 @@ TEST_F(OpenMPIRBuilderTest, CreateTeams) {
     Instruction *ThenTerm, *ElseTerm;
     SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
                                   &ThenTerm, &ElseTerm);
+    return Error::success();
   };
 
   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
-  Builder.restoreIP(OMPBuilder.createTeams(
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTeams(
       Builder, BodyGenCB, /*NumTeamsLower=*/nullptr, /*NumTeamsUpper=*/nullptr,
-      /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr));
+      /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
 
   OMPBuilder.finalize();
   Builder.CreateRetVoid();
@@ -4423,14 +4572,16 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithThreadLimit) {
   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
     Builder.restoreIP(CodeGenIP);
     Builder.CreateCall(FakeFunction, {});
+    return Error::success();
   };
 
   // `F` has an argument - an integer, so we use that as the thread limit.
-  Builder.restoreIP(OMPBuilder.createTeams(/*=*/Builder, BodyGenCB,
-                                           /*NumTeamsLower=*/nullptr,
-                                           /*NumTeamsUpper=*/nullptr,
-                                           /*ThreadLimit=*/F->arg_begin(),
-                                           /*IfExpr=*/nullptr));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTeams(
+      /*=*/Builder, BodyGenCB, /*NumTeamsLower=*/nullptr,
+      /*NumTeamsUpper=*/nullptr, /*ThreadLimit=*/F->arg_begin(),
+      /*IfExpr=*/nullptr);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
 
   Builder.CreateRetVoid();
   OMPBuilder.finalize();
@@ -4474,15 +4625,19 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsUpper) {
   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
     Builder.restoreIP(CodeGenIP);
     Builder.CreateCall(FakeFunction, {});
+    return Error::success();
   };
 
   // `F` already has an integer argument, so we use that as upper bound to
   // `num_teams`
-  Builder.restoreIP(OMPBuilder.createTeams(Builder, BodyGenCB,
-                                           /*NumTeamsLower=*/nullptr,
-                                           /*NumTeamsUpper=*/F->arg_begin(),
-                                           /*ThreadLimit=*/nullptr,
-                                           /*IfExpr=*/nullptr));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      OMPBuilder.createTeams(Builder, BodyGenCB,
+                             /*NumTeamsLower=*/nullptr,
+                             /*NumTeamsUpper=*/F->arg_begin(),
+                             /*ThreadLimit=*/nullptr,
+                             /*IfExpr=*/nullptr);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
 
   Builder.CreateRetVoid();
   OMPBuilder.finalize();
@@ -4531,13 +4686,16 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsBoth) {
   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
     Builder.restoreIP(CodeGenIP);
     Builder.CreateCall(FakeFunction, {});
+    return Error::success();
   };
 
   // `F` already has an integer argument, so we use that as upper bound to
   // `num_teams`
-  Builder.restoreIP(
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
       OMPBuilder.createTeams(Builder, BodyGenCB, NumTeamsLower, NumTeamsUpper,
-                             /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr));
+                             /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
 
   Builder.CreateRetVoid();
   OMPBuilder.finalize();
@@ -4593,11 +4751,14 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsAndThreadLimit) {
   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
     Builder.restoreIP(CodeGenIP);
     Builder.CreateCall(FakeFunction, {});
+    return Error::success();
   };
 
   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
-  Builder.restoreIP(OMPBuilder.createTeams(
-      Builder, BodyGenCB, NumTeamsLower, NumTeamsUpper, ThreadLimit, nullptr));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTeams(
+      Builder, BodyGenCB, NumTeamsLower, NumTeamsUpper, ThreadLimit, nullptr);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
 
   Builder.CreateRetVoid();
   OMPBuilder.finalize();
@@ -4644,13 +4805,16 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfCondition) {
   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
     Builder.restoreIP(CodeGenIP);
     Builder.CreateCall(FakeFunction, {});
+    return Error::success();
   };
 
   // `F` already has an integer argument, so we use that as upper bound to
   // `num_teams`
-  Builder.restoreIP(OMPBuilder.createTeams(
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTeams(
       Builder, BodyGenCB, /*NumTeamsLower=*/nullptr, /*NumTeamsUpper=*/nullptr,
-      /*ThreadLimit=*/nullptr, IfExpr));
+      /*ThreadLimit=*/nullptr, IfExpr);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
 
   Builder.CreateRetVoid();
   OMPBuilder.finalize();
@@ -4707,12 +4871,15 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfConditionAndNumTeams) {
   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
     Builder.restoreIP(CodeGenIP);
     Builder.CreateCall(FakeFunction, {});
+    return Error::success();
   };
 
   // `F` already has an integer argument, so we use that as upper bound to
   // `num_teams`
-  Builder.restoreIP(OMPBuilder.createTeams(Builder, BodyGenCB, NumTeamsLower,
-                                           NumTeamsUpper, ThreadLimit, IfExpr));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTeams(
+      Builder, BodyGenCB, NumTeamsLower, NumTeamsUpper, ThreadLimit, IfExpr);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
 
   Builder.CreateRetVoid();
   OMPBuilder.finalize();
@@ -4937,6 +5104,7 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) {
 
     BodyIP = Builder.saveIP();
     BodyAllocaIP = InnerAllocaIP;
+    return Error::success();
   };
 
   // Privatization for reduction creates local copies of reduction variables and
@@ -4969,14 +5137,15 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) {
   };
 
   // Do nothing in finalization.
-  auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; };
+  auto FiniCB = [&](InsertPointTy CodeGenIP) { return Error::success(); };
 
-  InsertPointTy AfterIP =
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
       OMPBuilder.createParallel(Loc, OuterAllocaIP, BodyGenCB, PrivCB, FiniCB,
                                 /* IfCondition */ nullptr,
                                 /* NumThreads */ nullptr, OMP_PROC_BIND_default,
                                 /* IsCancellable */ false);
-  Builder.restoreIP(AfterIP);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
 
   OpenMPIRBuilder::ReductionInfo ReductionInfos[] = {
       {SumType, SumReduced, SumPrivatized,
@@ -4989,10 +5158,12 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) {
 
   bool ReduceVariableByRef[] = {false, false};
 
-  OMPBuilder.createReductions(BodyIP, BodyAllocaIP, ReductionInfos,
-                              ReduceVariableByRef);
+  OpenMPIRBuilder::InsertPointOrErrorTy ReductionsIP =
+      OMPBuilder.createReductions(BodyIP, BodyAllocaIP, ReductionInfos,
+                                  ReduceVariableByRef);
+  assert(ReductionsIP && "unexpected error");
 
-  Builder.restoreIP(AfterIP);
+  Builder.restoreIP(*AfterIP);
   Builder.CreateRetVoid();
 
   OMPBuilder.finalize(F);
@@ -5172,6 +5343,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
 
     FirstBodyIP = Builder.saveIP();
     FirstBodyAllocaIP = InnerAllocaIP;
+    return Error::success();
   };
 
   InsertPointTy SecondBodyIP, SecondBodyAllocaIP;
@@ -5190,6 +5362,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
 
     SecondBodyIP = Builder.saveIP();
     SecondBodyAllocaIP = InnerAllocaIP;
+    return Error::success();
   };
 
   // Privatization for reduction creates local copies of reduction variables and
@@ -5224,36 +5397,44 @@ TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
   };
 
   // Do nothing in finalization.
-  auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; };
+  auto FiniCB = [&](InsertPointTy CodeGenIP) { return Error::success(); };
 
-  Builder.restoreIP(
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP1 =
       OMPBuilder.createParallel(Loc, OuterAllocaIP, FirstBodyGenCB, PrivCB,
                                 FiniCB, /* IfCondition */ nullptr,
                                 /* NumThreads */ nullptr, OMP_PROC_BIND_default,
-                                /* IsCancellable */ false));
-  InsertPointTy AfterIP = OMPBuilder.createParallel(
+                                /* IsCancellable */ false);
+  assert(AfterIP1 && "unexpected error");
+  Builder.restoreIP(*AfterIP1);
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP2 = OMPBuilder.createParallel(
       {Builder.saveIP(), DL}, OuterAllocaIP, SecondBodyGenCB, PrivCB, FiniCB,
       /* IfCondition */ nullptr,
       /* NumThreads */ nullptr, OMP_PROC_BIND_default,
       /* IsCancellable */ false);
+  assert(AfterIP2 && "unexpected error");
+  Builder.restoreIP(*AfterIP2);
 
   OMPBuilder.Config.setIsGPU(false);
   bool ReduceVariableByRef[] = {false};
 
-  OMPBuilder.createReductions(
-      FirstBodyIP, FirstBodyAllocaIP,
-      {{SumType, SumReduced, SumPrivatized,
-        /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar, sumReduction,
-        /*ReductionGenClang=*/nullptr, sumAtomicReduction}},
-      ReduceVariableByRef);
-  OMPBuilder.createReductions(
-      SecondBodyIP, SecondBodyAllocaIP,
-      {{XorType, XorReduced, XorPrivatized,
-        /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar, xorReduction,
-        /*ReductionGenClang=*/nullptr, xorAtomicReduction}},
-      ReduceVariableByRef);
-
-  Builder.restoreIP(AfterIP);
+  OpenMPIRBuilder::InsertPointOrErrorTy ReductionsIP1 =
+      OMPBuilder.createReductions(
+          FirstBodyIP, FirstBodyAllocaIP,
+          {{SumType, SumReduced, SumPrivatized,
+            /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar, sumReduction,
+            /*ReductionGenClang=*/nullptr, sumAtomicReduction}},
+          ReduceVariableByRef);
+  assert(ReductionsIP1 && "unexpected error");
+  OpenMPIRBuilder::InsertPointOrErrorTy ReductionsIP2 =
+      OMPBuilder.createReductions(
+          SecondBodyIP, SecondBodyAllocaIP,
+          {{XorType, XorReduced, XorPrivatized,
+            /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar, xorReduction,
+            /*ReductionGenClang=*/nullptr, xorAtomicReduction}},
+          ReduceVariableByRef);
+  assert(ReductionsIP2 && "unexpected error");
+
+  Builder.restoreIP(*AfterIP2);
   Builder.CreateRetVoid();
 
   OMPBuilder.finalize(F);
@@ -5320,8 +5501,10 @@ TEST_F(OpenMPIRBuilderTest, CreateSectionsSimple) {
   llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
   llvm::SmallVector<BasicBlock *, 4> CaseBBs;
 
-  auto FiniCB = [&](InsertPointTy IP) {};
-  auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
+  auto FiniCB = [&](InsertPointTy IP) { return Error::success(); };
+  auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    return Error::success();
+  };
   SectionCBVector.push_back(SectionCB);
 
   auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
@@ -5329,8 +5512,10 @@ TEST_F(OpenMPIRBuilderTest, CreateSectionsSimple) {
                    llvm::Value *&ReplVal) { return CodeGenIP; };
   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
                                     F->getEntryBlock().getFirstInsertionPt());
-  Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,
-                                              PrivCB, FiniCB, false, false));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createSections(
+      Loc, AllocaIP, SectionCBVector, PrivCB, FiniCB, false, false);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   Builder.CreateRetVoid(); // Required at the end of the function
   EXPECT_NE(F->getEntryBlock().getTerminator(), nullptr);
   EXPECT_FALSE(verifyModule(*M, &errs()));
@@ -5371,6 +5556,7 @@ TEST_F(OpenMPIRBuilderTest, CreateSections) {
     Value *PrivLoad =
         Builder.CreateLoad(F->arg_begin()->getType(), PrivAI, "local.alloca");
     Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
+    return Error::success();
   };
   auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
                    llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
@@ -5383,8 +5569,11 @@ TEST_F(OpenMPIRBuilderTest, CreateSections) {
 
   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
                                     F->getEntryBlock().getFirstInsertionPt());
-  Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,
-                                              PrivCB, FiniCB, false, false));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector, PrivCB,
+                                FINICB_WRAPPER(FiniCB), false, false);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   Builder.CreateRetVoid(); // Required at the end of the function
 
   // Switch BB's predecessor is loop condition BB, whose successor at index 1 is
@@ -5468,10 +5657,12 @@ TEST_F(OpenMPIRBuilderTest, CreateSectionsNoWait) {
   auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
                    llvm::Value &, llvm::Value &Val,
                    llvm::Value *&ReplVal) { return CodeGenIP; };
-  auto FiniCB = [&](InsertPointTy IP) {};
+  auto FiniCB = [&](InsertPointTy IP) { return Error::success(); };
 
-  Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,
-                                              PrivCB, FiniCB, false, true));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createSections(
+      Loc, AllocaIP, SectionCBVector, PrivCB, FiniCB, false, true);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   Builder.CreateRetVoid(); // Required at the end of the function
   for (auto &Inst : instructions(*F)) {
     EXPECT_FALSE(isa<CallInst>(Inst) &&
@@ -5692,9 +5883,11 @@ TEST_F(OpenMPIRBuilderTest, TargetEnterData) {
   OMPBuilder.Config.setIsGPU(true);
 
   llvm::omp::RuntimeFunction RTLFunc = OMPRTL___tgt_target_data_begin_mapper;
-  Builder.restoreIP(OMPBuilder.createTargetData(
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTargetData(
       Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),
-      /* IfCond= */ nullptr, Info, GenMapInfoCB, &RTLFunc));
+      /* IfCond= */ nullptr, Info, GenMapInfoCB, &RTLFunc);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
 
   CallInst *TargetDataCall = dyn_cast<CallInst>(&BB->back());
   EXPECT_NE(TargetDataCall, nullptr);
@@ -5751,9 +5944,11 @@ TEST_F(OpenMPIRBuilderTest, TargetExitData) {
   OMPBuilder.Config.setIsGPU(true);
 
   llvm::omp::RuntimeFunction RTLFunc = OMPRTL___tgt_target_data_end_mapper;
-  Builder.restoreIP(OMPBuilder.createTargetData(
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTargetData(
       Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),
-      /* IfCond= */ nullptr, Info, GenMapInfoCB, &RTLFunc));
+      /* IfCond= */ nullptr, Info, GenMapInfoCB, &RTLFunc);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
 
   CallInst *TargetDataCall = dyn_cast<CallInst>(&BB->back());
   EXPECT_NE(TargetDataCall, nullptr);
@@ -5859,9 +6054,12 @@ TEST_F(OpenMPIRBuilderTest, TargetDataRegion) {
     return Builder.saveIP();
   };
 
-  Builder.restoreIP(OMPBuilder.createTargetData(
-      Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),
-      /* IfCond= */ nullptr, Info, GenMapInfoCB, nullptr, BodyCB));
+  OpenMPIRBuilder::InsertPointOrErrorTy TargetDataIP1 =
+      OMPBuilder.createTargetData(
+          Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),
+          /* IfCond= */ nullptr, Info, GenMapInfoCB, nullptr, BodyCB);
+  assert(TargetDataIP1 && "unexpected error");
+  Builder.restoreIP(*TargetDataIP1);
 
   CallInst *TargetDataCall = dyn_cast<CallInst>(&BB->back());
   EXPECT_NE(TargetDataCall, nullptr);
@@ -5884,9 +6082,12 @@ TEST_F(OpenMPIRBuilderTest, TargetDataRegion) {
     EXPECT_EQ(TargetDataCall, nullptr);
     return Builder.saveIP();
   };
-  Builder.restoreIP(OMPBuilder.createTargetData(
-      Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),
-      /* IfCond= */ nullptr, Info, GenMapInfoCB, nullptr, BodyTargetCB));
+  OpenMPIRBuilder::InsertPointOrErrorTy TargetDataIP2 =
+      OMPBuilder.createTargetData(
+          Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),
+          /* IfCond= */ nullptr, Info, GenMapInfoCB, nullptr, BodyTargetCB);
+  assert(TargetDataIP2 && "unexpected error");
+  Builder.restoreIP(*TargetDataIP2);
   EXPECT_TRUE(CheckDevicePassBodyGen);
 
   Builder.CreateRetVoid();
@@ -5981,9 +6182,11 @@ TEST_F(OpenMPIRBuilderTest, TargetRegion) {
 
   TargetRegionEntryInfo EntryInfo("func", 42, 4711, 17);
   OpenMPIRBuilder::LocationDescription OmpLoc({Builder.saveIP(), DL});
-  Builder.restoreIP(OMPBuilder.createTarget(
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTarget(
       OmpLoc, /*IsOffloadEntry=*/true, Builder.saveIP(), Builder.saveIP(),
-      EntryInfo, -1, 0, Inputs, GenMapInfoCB, BodyGenCB, SimpleArgAccessorCB));
+      EntryInfo, -1, 0, Inputs, GenMapInfoCB, BodyGenCB, SimpleArgAccessorCB);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   OMPBuilder.finalize();
   Builder.CreateRetVoid();
 
@@ -6089,11 +6292,13 @@ TEST_F(OpenMPIRBuilderTest, TargetRegionDevice) {
   TargetRegionEntryInfo EntryInfo("parent", /*DeviceID=*/1, /*FileID=*/2,
                                   /*Line=*/3, /*Count=*/0);
 
-  Builder.restoreIP(
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
       OMPBuilder.createTarget(Loc, /*IsOffloadEntry=*/true, EntryIP, EntryIP,
                               EntryInfo, /*NumTeams=*/-1,
                               /*NumThreads=*/0, CapturedArgs, GenMapInfoCB,
-                              BodyGenCB, SimpleArgAccessorCB));
+                              BodyGenCB, SimpleArgAccessorCB);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
 
   Builder.CreateRetVoid();
   OMPBuilder.finalize();
@@ -6238,11 +6443,13 @@ TEST_F(OpenMPIRBuilderTest, ConstantAllocaRaise) {
   TargetRegionEntryInfo EntryInfo("parent", /*DeviceID=*/1, /*FileID=*/2,
                                   /*Line=*/3, /*Count=*/0);
 
-  Builder.restoreIP(
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
       OMPBuilder.createTarget(Loc, /*IsOffloadEntry=*/true, EntryIP, EntryIP,
                               EntryInfo, /*NumTeams=*/-1,
                               /*NumThreads=*/0, CapturedArgs, GenMapInfoCB,
-                              BodyGenCB, SimpleArgAccessorCB));
+                              BodyGenCB, SimpleArgAccessorCB);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
 
   Builder.CreateRetVoid();
   OMPBuilder.finalize();
@@ -6354,15 +6561,17 @@ TEST_F(OpenMPIRBuilderTest, CreateTask) {
     Instruction *ThenTerm, *ElseTerm;
     SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
                                   &ThenTerm, &ElseTerm);
+    return Error::success();
   };
 
   BasicBlock *AllocaBB = Builder.GetInsertBlock();
   BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
   OpenMPIRBuilder::LocationDescription Loc(
       InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
-  Builder.restoreIP(OMPBuilder.createTask(
-      Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
-      BodyGenCB));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTask(
+      Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   OMPBuilder.finalize();
   Builder.CreateRetVoid();
 
@@ -6460,15 +6669,18 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskNoArgs) {
   F->setName("func");
   IRBuilder<> Builder(BB);
 
-  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
+  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    return Error::success();
+  };
 
   BasicBlock *AllocaBB = Builder.GetInsertBlock();
   BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
   OpenMPIRBuilder::LocationDescription Loc(
       InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
-  Builder.restoreIP(OMPBuilder.createTask(
-      Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
-      BodyGenCB));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTask(
+      Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   OMPBuilder.finalize();
   Builder.CreateRetVoid();
 
@@ -6490,14 +6702,18 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskUntied) {
   OMPBuilder.initialize();
   F->setName("func");
   IRBuilder<> Builder(BB);
-  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
+  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    return Error::success();
+  };
   BasicBlock *AllocaBB = Builder.GetInsertBlock();
   BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
   OpenMPIRBuilder::LocationDescription Loc(
       InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
-  Builder.restoreIP(OMPBuilder.createTask(
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTask(
       Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB,
-      /*Tied=*/false));
+      /*Tied=*/false);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   OMPBuilder.finalize();
   Builder.CreateRetVoid();
 
@@ -6520,7 +6736,9 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskDepend) {
   OMPBuilder.initialize();
   F->setName("func");
   IRBuilder<> Builder(BB);
-  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
+  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    return Error::success();
+  };
   BasicBlock *AllocaBB = Builder.GetInsertBlock();
   BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
   OpenMPIRBuilder::LocationDescription Loc(
@@ -6532,9 +6750,11 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskDepend) {
                                      Type::getInt32Ty(M->getContext()), InDep);
     DDS.push_back(DDIn);
   }
-  Builder.restoreIP(OMPBuilder.createTask(
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTask(
       Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB,
-      /*Tied=*/false, /*Final*/ nullptr, /*IfCondition*/ nullptr, DDS));
+      /*Tied=*/false, /*Final*/ nullptr, /*IfCondition*/ nullptr, DDS);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   OMPBuilder.finalize();
   Builder.CreateRetVoid();
 
@@ -6594,7 +6814,9 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskFinal) {
   OMPBuilder.initialize();
   F->setName("func");
   IRBuilder<> Builder(BB);
-  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
+  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    return Error::success();
+  };
   BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
   IRBuilderBase::InsertPoint AllocaIP = Builder.saveIP();
   Builder.SetInsertPoint(BodyBB);
@@ -6602,8 +6824,11 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskFinal) {
       CmpInst::Predicate::ICMP_EQ, F->getArg(0),
       ConstantInt::get(Type::getInt32Ty(M->getContext()), 0U));
   OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
-  Builder.restoreIP(OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,
-                                          /*Tied=*/false, Final));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,
+                            /*Tied=*/false, Final);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   OMPBuilder.finalize();
   Builder.CreateRetVoid();
 
@@ -6648,7 +6873,9 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskIfCondition) {
   OMPBuilder.initialize();
   F->setName("func");
   IRBuilder<> Builder(BB);
-  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
+  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    return Error::success();
+  };
   BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
   IRBuilderBase::InsertPoint AllocaIP = Builder.saveIP();
   Builder.SetInsertPoint(BodyBB);
@@ -6656,9 +6883,11 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskIfCondition) {
       CmpInst::Predicate::ICMP_EQ, F->getArg(0),
       ConstantInt::get(Type::getInt32Ty(M->getContext()), 0U));
   OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
-  Builder.restoreIP(OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,
-                                          /*Tied=*/false, /*Final=*/nullptr,
-                                          IfCondition));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,
+                            /*Tied=*/false, /*Final=*/nullptr, IfCondition);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   OMPBuilder.finalize();
   Builder.CreateRetVoid();
 
@@ -6742,15 +6971,17 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskgroup) {
     SplitBlockAndInsertIfThenElse(InternalIfCmp,
                                   CodeGenIP.getBlock()->getTerminator(),
                                   &ThenTerm, &ElseTerm);
+    return Error::success();
   };
 
   BasicBlock *AllocaBB = Builder.GetInsertBlock();
   BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
   OpenMPIRBuilder::LocationDescription Loc(
       InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
-  Builder.restoreIP(OMPBuilder.createTaskgroup(
-      Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
-      BodyGenCB));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTaskgroup(
+      Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   OMPBuilder.finalize();
   Builder.CreateRetVoid();
 
@@ -6823,9 +7054,13 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskgroupWithTasks) {
           Builder.CreateLoad(Alloca64->getAllocatedType(), Alloca64);
       Value *AddInst = Builder.CreateAdd(LoadValue, Builder.getInt64(64));
       Builder.CreateStore(AddInst, Alloca64);
+      return Error::success();
     };
     OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
-    Builder.restoreIP(OMPBuilder.createTask(Loc, AllocaIP, TaskBodyGenCB1));
+    OpenMPIRBuilder::InsertPointOrErrorTy TaskIP1 =
+        OMPBuilder.createTask(Loc, AllocaIP, TaskBodyGenCB1);
+    assert(TaskIP1 && "unexpected error");
+    Builder.restoreIP(*TaskIP1);
 
     auto TaskBodyGenCB2 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
       Builder.restoreIP(CodeGenIP);
@@ -6833,18 +7068,24 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskgroupWithTasks) {
           Builder.CreateLoad(Alloca32->getAllocatedType(), Alloca32);
       Value *AddInst = Builder.CreateAdd(LoadValue, Builder.getInt32(32));
       Builder.CreateStore(AddInst, Alloca32);
+      return Error::success();
     };
     OpenMPIRBuilder::LocationDescription Loc2(Builder.saveIP(), DL);
-    Builder.restoreIP(OMPBuilder.createTask(Loc2, AllocaIP, TaskBodyGenCB2));
+    OpenMPIRBuilder::InsertPointOrErrorTy TaskIP2 =
+        OMPBuilder.createTask(Loc2, AllocaIP, TaskBodyGenCB2);
+    assert(TaskIP2 && "unexpected error");
+    Builder.restoreIP(*TaskIP2);
+    return Error::success();
   };
 
   BasicBlock *AllocaBB = Builder.GetInsertBlock();
   BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
   OpenMPIRBuilder::LocationDescription Loc(
       InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
-  Builder.restoreIP(OMPBuilder.createTaskgroup(
-      Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
-      BodyGenCB));
+  OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTaskgroup(
+      Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB);
+  assert(AfterIP && "unexpected error");
+  Builder.restoreIP(*AfterIP);
   OMPBuilder.finalize();
   Builder.CreateRetVoid();
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 27cd38dc3c62..fc2f88b766f1 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -137,9 +137,9 @@ findAllocaInsertPoint(llvm::IRBuilderBase &builder,
 /// region, and a branch from any block with an successor-less OpenMP terminator
 /// to `continuationBlock`. Populates `continuationBlockPHIs` with the PHI nodes
 /// of the continuation block if provided.
-static llvm::BasicBlock *convertOmpOpRegions(
+static llvm::Expected<llvm::BasicBlock *> convertOmpOpRegions(
     Region &region, StringRef blockName, llvm::IRBuilderBase &builder,
-    LLVM::ModuleTranslation &moduleTranslation, LogicalResult &bodyGenStatus,
+    LLVM::ModuleTranslation &moduleTranslation,
     SmallVectorImpl<llvm::PHINode *> *continuationBlockPHIs = nullptr) {
   llvm::BasicBlock *continuationBlock =
       splitBB(builder, true, "omp.region.cont");
@@ -215,10 +215,8 @@ static llvm::BasicBlock *convertOmpOpRegions(
 
     llvm::IRBuilderBase::InsertPointGuard guard(builder);
     if (failed(
-            moduleTranslation.convertBlock(*bb, bb->isEntryBlock(), builder))) {
-      bodyGenStatus = failure();
-      return continuationBlock;
-    }
+            moduleTranslation.convertBlock(*bb, bb->isEntryBlock(), builder)))
+      return llvm::createStringError("failed region translation");
 
     // Special handling for `omp.yield` and `omp.terminator` (we may have more
     // than one): they return the control to the parent OpenMP dialect operation
@@ -270,21 +268,19 @@ convertOmpMasked(Operation &opInst, llvm::IRBuilderBase &builder,
                  LLVM::ModuleTranslation &moduleTranslation) {
   auto maskedOp = cast<omp::MaskedOp>(opInst);
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
-  // TODO: support error propagation in OpenMPIRBuilder and use it instead of
-  // relying on captured variables.
-  LogicalResult bodyGenStatus = success();
 
   auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
     // MaskedOp has only one region associated with it.
     auto &region = maskedOp.getRegion();
     builder.restoreIP(codeGenIP);
-    convertOmpOpRegions(region, "omp.masked.region", builder, moduleTranslation,
-                        bodyGenStatus);
+    return convertOmpOpRegions(region, "omp.masked.region", builder,
+                               moduleTranslation)
+        .takeError();
   };
 
   // TODO: Perform finalization actions for variables. This has to be
   // called for variables which have destructors/finalizers.
-  auto finiCB = [&](InsertPointTy codeGenIP) {};
+  auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };
 
   llvm::Value *filterVal = nullptr;
   if (auto filterVar = maskedOp.getFilteredThreadId()) {
@@ -296,8 +292,14 @@ convertOmpMasked(Operation &opInst, llvm::IRBuilderBase &builder,
   }
   assert(filterVal != nullptr);
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
-  builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createMasked(
-      ompLoc, bodyGenCB, finiCB, filterVal));
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
+      moduleTranslation.getOpenMPBuilder()->createMasked(ompLoc, bodyGenCB,
+                                                         finiCB, filterVal);
+
+  if (!afterIP)
+    return opInst.emitError(llvm::toString(afterIP.takeError()));
+
+  builder.restoreIP(*afterIP);
   return success();
 }
 
@@ -306,25 +308,28 @@ static LogicalResult
 convertOmpMaster(Operation &opInst, llvm::IRBuilderBase &builder,
                  LLVM::ModuleTranslation &moduleTranslation) {
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
-  // TODO: support error propagation in OpenMPIRBuilder and use it instead of
-  // relying on captured variables.
-  LogicalResult bodyGenStatus = success();
-
   auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
     // MasterOp has only one region associated with it.
     auto &region = cast<omp::MasterOp>(opInst).getRegion();
     builder.restoreIP(codeGenIP);
-    convertOmpOpRegions(region, "omp.master.region", builder, moduleTranslation,
-                        bodyGenStatus);
+    return convertOmpOpRegions(region, "omp.master.region", builder,
+                               moduleTranslation)
+        .takeError();
   };
 
   // TODO: Perform finalization actions for variables. This has to be
   // called for variables which have destructors/finalizers.
-  auto finiCB = [&](InsertPointTy codeGenIP) {};
+  auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };
 
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
-  builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createMaster(
-      ompLoc, bodyGenCB, finiCB));
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
+      moduleTranslation.getOpenMPBuilder()->createMaster(ompLoc, bodyGenCB,
+                                                         finiCB);
+
+  if (!afterIP)
+    return opInst.emitError(llvm::toString(afterIP.takeError()));
+
+  builder.restoreIP(*afterIP);
   return success();
 }
 
@@ -334,21 +339,19 @@ convertOmpCritical(Operation &opInst, llvm::IRBuilderBase &builder,
                    LLVM::ModuleTranslation &moduleTranslation) {
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
   auto criticalOp = cast<omp::CriticalOp>(opInst);
-  // TODO: support error propagation in OpenMPIRBuilder and use it instead of
-  // relying on captured variables.
-  LogicalResult bodyGenStatus = success();
 
   auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
     // CriticalOp has only one region associated with it.
     auto &region = cast<omp::CriticalOp>(opInst).getRegion();
     builder.restoreIP(codeGenIP);
-    convertOmpOpRegions(region, "omp.critical.region", builder,
-                        moduleTranslation, bodyGenStatus);
+    return convertOmpOpRegions(region, "omp.critical.region", builder,
+                               moduleTranslation)
+        .takeError();
   };
 
   // TODO: Perform finalization actions for variables. This has to be
   // called for variables which have destructors/finalizers.
-  auto finiCB = [&](InsertPointTy codeGenIP) {};
+  auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };
 
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
   llvm::LLVMContext &llvmContext = moduleTranslation.getLLVMContext();
@@ -366,8 +369,14 @@ convertOmpCritical(Operation &opInst, llvm::IRBuilderBase &builder,
         llvm::ConstantInt::get(llvm::Type::getInt32Ty(llvmContext),
                                static_cast<int>(criticalDeclareOp.getHint()));
   }
-  builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createCritical(
-      ompLoc, bodyGenCB, finiCB, criticalOp.getName().value_or(""), hint));
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
+      moduleTranslation.getOpenMPBuilder()->createCritical(
+          ompLoc, bodyGenCB, finiCB, criticalOp.getName().value_or(""), hint);
+
+  if (!afterIP)
+    return opInst.emitError(llvm::toString(afterIP.takeError()));
+
+  builder.restoreIP(*afterIP);
   return success();
 }
 
@@ -468,27 +477,30 @@ static LogicalResult inlineConvertOmpRegions(
     return success();
   }
 
-  LogicalResult bodyGenStatus = success();
   SmallVector<llvm::PHINode *> phis;
-  llvm::BasicBlock *continuationBlock = convertOmpOpRegions(
-      region, blockName, builder, moduleTranslation, bodyGenStatus, &phis);
-  if (failed(bodyGenStatus))
-    return failure();
+  llvm::Expected<llvm::BasicBlock *> continuationBlock =
+      convertOmpOpRegions(region, blockName, builder, moduleTranslation, &phis);
+
+  if (!continuationBlock)
+    return region.getParentOp()->emitError(
+        llvm::toString(continuationBlock.takeError()));
+
   if (continuationBlockArgs)
     llvm::append_range(*continuationBlockArgs, phis);
-  builder.SetInsertPoint(continuationBlock,
-                         continuationBlock->getFirstInsertionPt());
+  builder.SetInsertPoint(*continuationBlock,
+                         (*continuationBlock)->getFirstInsertionPt());
   return success();
 }
 
 namespace {
 /// Owning equivalents of OpenMPIRBuilder::(Atomic)ReductionGen that are used to
 /// store lambdas with capture.
-using OwningReductionGen = std::function<llvm::OpenMPIRBuilder::InsertPointTy(
-    llvm::OpenMPIRBuilder::InsertPointTy, llvm::Value *, llvm::Value *,
-    llvm::Value *&)>;
+using OwningReductionGen =
+    std::function<llvm::OpenMPIRBuilder::InsertPointOrErrorTy(
+        llvm::OpenMPIRBuilder::InsertPointTy, llvm::Value *, llvm::Value *,
+        llvm::Value *&)>;
 using OwningAtomicReductionGen =
-    std::function<llvm::OpenMPIRBuilder::InsertPointTy(
+    std::function<llvm::OpenMPIRBuilder::InsertPointOrErrorTy(
         llvm::OpenMPIRBuilder::InsertPointTy, llvm::Type *, llvm::Value *,
         llvm::Value *)>;
 } // namespace
@@ -505,19 +517,20 @@ makeReductionGen(omp::DeclareReductionOp decl, llvm::IRBuilderBase &builder,
   OwningReductionGen gen =
       [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint,
                 llvm::Value *lhs, llvm::Value *rhs,
-                llvm::Value *&result) mutable {
-        moduleTranslation.mapValue(decl.getReductionLhsArg(), lhs);
-        moduleTranslation.mapValue(decl.getReductionRhsArg(), rhs);
-        builder.restoreIP(insertPoint);
-        SmallVector<llvm::Value *> phis;
-        if (failed(inlineConvertOmpRegions(decl.getReductionRegion(),
-                                           "omp.reduction.nonatomic.body",
-                                           builder, moduleTranslation, &phis)))
-          return llvm::OpenMPIRBuilder::InsertPointTy();
-        assert(phis.size() == 1);
-        result = phis[0];
-        return builder.saveIP();
-      };
+                llvm::Value *&result) mutable
+      -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
+    moduleTranslation.mapValue(decl.getReductionLhsArg(), lhs);
+    moduleTranslation.mapValue(decl.getReductionRhsArg(), rhs);
+    builder.restoreIP(insertPoint);
+    SmallVector<llvm::Value *> phis;
+    if (failed(inlineConvertOmpRegions(decl.getReductionRegion(),
+                                       "omp.reduction.nonatomic.body", builder,
+                                       moduleTranslation, &phis)))
+      return llvm::createStringError("failed reduction region translation");
+    assert(phis.size() == 1);
+    result = phis[0];
+    return builder.saveIP();
+  };
   return gen;
 }
 
@@ -537,18 +550,19 @@ makeAtomicReductionGen(omp::DeclareReductionOp decl,
   // avoid the dangling reference after the parent function returns.
   OwningAtomicReductionGen atomicGen =
       [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, llvm::Type *,
-                llvm::Value *lhs, llvm::Value *rhs) mutable {
-        moduleTranslation.mapValue(decl.getAtomicReductionLhsArg(), lhs);
-        moduleTranslation.mapValue(decl.getAtomicReductionRhsArg(), rhs);
-        builder.restoreIP(insertPoint);
-        SmallVector<llvm::Value *> phis;
-        if (failed(inlineConvertOmpRegions(decl.getAtomicReductionRegion(),
-                                           "omp.reduction.atomic.body", builder,
-                                           moduleTranslation, &phis)))
-          return llvm::OpenMPIRBuilder::InsertPointTy();
-        assert(phis.empty());
-        return builder.saveIP();
-      };
+                llvm::Value *lhs, llvm::Value *rhs) mutable
+      -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
+    moduleTranslation.mapValue(decl.getAtomicReductionLhsArg(), lhs);
+    moduleTranslation.mapValue(decl.getAtomicReductionRhsArg(), rhs);
+    builder.restoreIP(insertPoint);
+    SmallVector<llvm::Value *> phis;
+    if (failed(inlineConvertOmpRegions(decl.getAtomicReductionRegion(),
+                                       "omp.reduction.atomic.body", builder,
+                                       moduleTranslation, &phis)))
+      return llvm::createStringError("failed reduction region translation");
+    assert(phis.empty());
+    return builder.saveIP();
+  };
   return atomicGen;
 }
 
@@ -593,27 +607,29 @@ convertOmpOrderedRegion(Operation &opInst, llvm::IRBuilderBase &builder,
   if (orderedRegionOp.getParLevelSimd())
     return failure();
 
-  // TODO: support error propagation in OpenMPIRBuilder and use it instead of
-  // relying on captured variables.
-  LogicalResult bodyGenStatus = success();
-
   auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
     // OrderedOp has only one region associated with it.
     auto &region = cast<omp::OrderedRegionOp>(opInst).getRegion();
     builder.restoreIP(codeGenIP);
-    convertOmpOpRegions(region, "omp.ordered.region", builder,
-                        moduleTranslation, bodyGenStatus);
+    return convertOmpOpRegions(region, "omp.ordered.region", builder,
+                               moduleTranslation)
+        .takeError();
   };
 
   // TODO: Perform finalization actions for variables. This has to be
   // called for variables which have destructors/finalizers.
-  auto finiCB = [&](InsertPointTy codeGenIP) {};
+  auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };
 
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
-  builder.restoreIP(
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
       moduleTranslation.getOpenMPBuilder()->createOrderedThreadsSimd(
-          ompLoc, bodyGenCB, finiCB, !orderedRegionOp.getParLevelSimd()));
-  return bodyGenStatus;
+          ompLoc, bodyGenCB, finiCB, !orderedRegionOp.getParLevelSimd());
+
+  if (!afterIP)
+    return opInst.emitError(llvm::toString(afterIP.takeError()));
+
+  builder.restoreIP(*afterIP);
+  return success();
 }
 
 namespace {
@@ -811,15 +827,24 @@ static LogicalResult createReductionsAndCleanup(
   // and remove it later.
   llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable();
   builder.SetInsertPoint(tempTerminator);
-  llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint =
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy contInsertPoint =
       ompBuilder->createReductions(builder.saveIP(), allocaIP, reductionInfos,
                                    isByRef, op.getNowait());
-  if (!contInsertPoint.getBlock())
+
+  if (!contInsertPoint)
+    return op.emitError(llvm::toString(contInsertPoint.takeError()));
+
+  if (!contInsertPoint->getBlock())
     return op->emitOpError() << "failed to convert reductions";
-  auto nextInsertionPoint =
-      ompBuilder->createBarrier(contInsertPoint, llvm::omp::OMPD_for);
+
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
+      ompBuilder->createBarrier(*contInsertPoint, llvm::omp::OMPD_for);
+
+  if (!afterIP)
+    return op.emitError(llvm::toString(afterIP.takeError()));
+
   tempTerminator->eraseFromParent();
-  builder.restoreIP(nextInsertionPoint);
+  builder.restoreIP(*afterIP);
 
   // after the construct, deallocate private reduction variables
   SmallVector<Region *> reductionRegions;
@@ -958,7 +983,6 @@ convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder,
   LLVM::ModuleTranslation::SaveStack<OpenMPVarMappingStackFrame> mappingGuard(
       moduleTranslation, reductionVariableMap);
 
-  LogicalResult bodyGenStatus = success();
   SmallVector<StorableBodyGenCallbackTy> sectionCBs;
 
   for (Operation &op : *sectionsOp.getRegion().begin()) {
@@ -967,9 +991,8 @@ convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder,
       continue;
 
     Region &region = sectionOp.getRegion();
-    auto sectionCB = [&sectionsOp, &region, &builder, &moduleTranslation,
-                      &bodyGenStatus](InsertPointTy allocaIP,
-                                      InsertPointTy codeGenIP) {
+    auto sectionCB = [&sectionsOp, &region, &builder, &moduleTranslation](
+                         InsertPointTy allocaIP, InsertPointTy codeGenIP) {
       builder.restoreIP(codeGenIP);
 
       // map the omp.section reduction block argument to the omp.sections block
@@ -985,8 +1008,9 @@ convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder,
         moduleTranslation.mapValue(sectionArg, llvmVal);
       }
 
-      convertOmpOpRegions(region, "omp.section.region", builder,
-                          moduleTranslation, bodyGenStatus);
+      return convertOmpOpRegions(region, "omp.section.region", builder,
+                                 moduleTranslation)
+          .takeError();
     };
     sectionCBs.push_back(sectionCB);
   }
@@ -1003,24 +1027,27 @@ convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder,
   // attribute (shared, private, firstprivate, ...) of variables.
   // Currently defaults to shared.
   auto privCB = [&](InsertPointTy, InsertPointTy codeGenIP, llvm::Value &,
-                    llvm::Value &vPtr,
-                    llvm::Value *&replacementValue) -> InsertPointTy {
+                    llvm::Value &vPtr, llvm::Value *&replacementValue)
+      -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
     replacementValue = &vPtr;
     return codeGenIP;
   };
 
   // TODO: Perform finalization actions for variables. This has to be
   // called for variables which have destructors/finalizers.
-  auto finiCB = [&](InsertPointTy codeGenIP) {};
+  auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };
 
   allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
-  builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createSections(
-      ompLoc, allocaIP, sectionCBs, privCB, finiCB, false,
-      sectionsOp.getNowait()));
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
+      moduleTranslation.getOpenMPBuilder()->createSections(
+          ompLoc, allocaIP, sectionCBs, privCB, finiCB, false,
+          sectionsOp.getNowait());
+
+  if (!afterIP)
+    return opInst.emitError(llvm::toString(afterIP.takeError()));
 
-  if (failed(bodyGenStatus))
-    return bodyGenStatus;
+  builder.restoreIP(*afterIP);
 
   // Process the reductions if required.
   return createReductionsAndCleanup(sectionsOp, builder, moduleTranslation,
@@ -1034,16 +1061,17 @@ convertOmpSingle(omp::SingleOp &singleOp, llvm::IRBuilderBase &builder,
                  LLVM::ModuleTranslation &moduleTranslation) {
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
-  LogicalResult bodyGenStatus = success();
+
   if (!singleOp.getPrivateVars().empty() || singleOp.getPrivateSyms())
     return singleOp.emitError("unhandled clauses for translation to LLVM IR");
 
   auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
     builder.restoreIP(codegenIP);
-    convertOmpOpRegions(singleOp.getRegion(), "omp.single.region", builder,
-                        moduleTranslation, bodyGenStatus);
+    return convertOmpOpRegions(singleOp.getRegion(), "omp.single.region",
+                               builder, moduleTranslation)
+        .takeError();
   };
-  auto finiCB = [&](InsertPointTy codeGenIP) {};
+  auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };
 
   // Handle copyprivate
   Operation::operand_range cpVars = singleOp.getCopyprivateVars();
@@ -1058,9 +1086,16 @@ convertOmpSingle(omp::SingleOp &singleOp, llvm::IRBuilderBase &builder,
         moduleTranslation.lookupFunction(llvmFuncOp.getName()));
   }
 
-  builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createSingle(
-      ompLoc, bodyCB, finiCB, singleOp.getNowait(), llvmCPVars, llvmCPFuncs));
-  return bodyGenStatus;
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
+      moduleTranslation.getOpenMPBuilder()->createSingle(
+          ompLoc, bodyCB, finiCB, singleOp.getNowait(), llvmCPVars,
+          llvmCPFuncs);
+
+  if (!afterIP)
+    return singleOp.emitError(llvm::toString(afterIP.takeError()));
+
+  builder.restoreIP(*afterIP);
+  return success();
 }
 
 // Convert an OpenMP Teams construct to LLVM IR using OpenMPIRBuilder
@@ -1068,7 +1103,6 @@ static LogicalResult
 convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder,
                 LLVM::ModuleTranslation &moduleTranslation) {
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
-  LogicalResult bodyGenStatus = success();
   if (!op.getAllocatorVars().empty() || op.getReductionSyms() ||
       !op.getPrivateVars().empty() || op.getPrivateSyms())
     return op.emitError("unhandled clauses for translation to LLVM IR");
@@ -1077,8 +1111,9 @@ convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder,
     LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(
         moduleTranslation, allocaIP);
     builder.restoreIP(codegenIP);
-    convertOmpOpRegions(op.getRegion(), "omp.teams.region", builder,
-                        moduleTranslation, bodyGenStatus);
+    return convertOmpOpRegions(op.getRegion(), "omp.teams.region", builder,
+                               moduleTranslation)
+        .takeError();
   };
 
   llvm::Value *numTeamsLower = nullptr;
@@ -1098,9 +1133,15 @@ convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder,
     ifExpr = moduleTranslation.lookupValue(ifVar);
 
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
-  builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTeams(
-      ompLoc, bodyCB, numTeamsLower, numTeamsUpper, threadLimit, ifExpr));
-  return bodyGenStatus;
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
+      moduleTranslation.getOpenMPBuilder()->createTeams(
+          ompLoc, bodyCB, numTeamsLower, numTeamsUpper, threadLimit, ifExpr);
+
+  if (!afterIP)
+    return op.emitError(llvm::toString(afterIP.takeError()));
+
+  builder.restoreIP(*afterIP);
+  return success();
 }
 
 static void
@@ -1134,7 +1175,6 @@ static LogicalResult
 convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,
                  LLVM::ModuleTranslation &moduleTranslation) {
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
-  LogicalResult bodyGenStatus = success();
   if (taskOp.getUntiedAttr() || taskOp.getMergeableAttr() ||
       taskOp.getInReductionSyms() || taskOp.getPriority() ||
       !taskOp.getAllocateVars().empty() || !taskOp.getPrivateVars().empty() ||
@@ -1148,8 +1188,9 @@ convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,
         moduleTranslation, allocaIP);
 
     builder.restoreIP(codegenIP);
-    convertOmpOpRegions(taskOp.getRegion(), "omp.task.region", builder,
-                        moduleTranslation, bodyGenStatus);
+    return convertOmpOpRegions(taskOp.getRegion(), "omp.task.region", builder,
+                               moduleTranslation)
+        .takeError();
   };
 
   SmallVector<llvm::OpenMPIRBuilder::DependData> dds;
@@ -1159,11 +1200,17 @@ convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,
   llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
       findAllocaInsertPoint(builder, moduleTranslation);
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
-  builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTask(
-      ompLoc, allocaIP, bodyCB, !taskOp.getUntied(),
-      moduleTranslation.lookupValue(taskOp.getFinal()),
-      moduleTranslation.lookupValue(taskOp.getIfExpr()), dds));
-  return bodyGenStatus;
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
+      moduleTranslation.getOpenMPBuilder()->createTask(
+          ompLoc, allocaIP, bodyCB, !taskOp.getUntied(),
+          moduleTranslation.lookupValue(taskOp.getFinal()),
+          moduleTranslation.lookupValue(taskOp.getIfExpr()), dds);
+
+  if (!afterIP)
+    return taskOp.emitError(llvm::toString(afterIP.takeError()));
+
+  builder.restoreIP(*afterIP);
+  return success();
 }
 
 /// Converts an OpenMP taskgroup construct into LLVM IR using OpenMPIRBuilder.
@@ -1171,20 +1218,27 @@ static LogicalResult
 convertOmpTaskgroupOp(omp::TaskgroupOp tgOp, llvm::IRBuilderBase &builder,
                       LLVM::ModuleTranslation &moduleTranslation) {
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
-  LogicalResult bodyGenStatus = success();
-  if (!tgOp.getTaskReductionVars().empty() || !tgOp.getAllocateVars().empty()) {
+  if (!tgOp.getTaskReductionVars().empty() || !tgOp.getAllocateVars().empty())
     return tgOp.emitError("unhandled clauses for translation to LLVM IR");
-  }
+
   auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
     builder.restoreIP(codegenIP);
-    convertOmpOpRegions(tgOp.getRegion(), "omp.taskgroup.region", builder,
-                        moduleTranslation, bodyGenStatus);
+    return convertOmpOpRegions(tgOp.getRegion(), "omp.taskgroup.region",
+                               builder, moduleTranslation)
+        .takeError();
   };
+
   InsertPointTy allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
-  builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTaskgroup(
-      ompLoc, allocaIP, bodyCB));
-  return bodyGenStatus;
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
+      moduleTranslation.getOpenMPBuilder()->createTaskgroup(ompLoc, allocaIP,
+                                                            bodyCB);
+
+  if (!afterIP)
+    return tgOp.emitError(llvm::toString(afterIP.takeError()));
+
+  builder.restoreIP(*afterIP);
+  return success();
 }
 
 static LogicalResult
@@ -1258,12 +1312,10 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
 
   // Generator of the canonical loop body.
-  // TODO: support error propagation in OpenMPIRBuilder and use it instead of
-  // relying on captured variables.
   SmallVector<llvm::CanonicalLoopInfo *> loopInfos;
   SmallVector<llvm::OpenMPIRBuilder::InsertPointTy> bodyInsertPoints;
-  LogicalResult bodyGenStatus = success();
-  auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) {
+  auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip,
+                     llvm::Value *iv) -> llvm::Error {
     // Make sure further conversions know about the induction variable.
     moduleTranslation.mapValue(
         loopOp.getRegion().front().getArgument(loopInfos.size()), iv);
@@ -1274,12 +1326,13 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
     bodyInsertPoints.push_back(ip);
 
     if (loopInfos.size() != loopOp.getNumLoops() - 1)
-      return;
+      return llvm::Error::success();
 
     // Convert the body of the loop.
     builder.restoreIP(ip);
-    convertOmpOpRegions(loopOp.getRegion(), "omp.wsloop.region", builder,
-                        moduleTranslation, bodyGenStatus);
+    return convertOmpOpRegions(loopOp.getRegion(), "omp.wsloop.region", builder,
+                               moduleTranslation)
+        .takeError();
   };
 
   // Delegate actual loop construction to the OpenMP IRBuilder.
@@ -1304,12 +1357,16 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
       loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back());
       computeIP = loopInfos.front()->getPreheaderIP();
     }
-    loopInfos.push_back(ompBuilder->createCanonicalLoop(
-        loc, bodyGen, lowerBound, upperBound, step,
-        /*IsSigned=*/true, loopOp.getLoopInclusive(), computeIP));
 
-    if (failed(bodyGenStatus))
-      return failure();
+    llvm::Expected<llvm::CanonicalLoopInfo *> result =
+        ompBuilder->createCanonicalLoop(
+            loc, bodyGen, lowerBound, upperBound, step,
+            /*IsSigned=*/true, loopOp.getLoopInclusive(), computeIP);
+
+    if (!result)
+      return loopOp.emitError(llvm::toString(result.takeError()));
+
+    loopInfos.push_back(*result);
   }
 
   // Collapse loops. Store the insertion point because LoopInfos may get
@@ -1325,11 +1382,15 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
   std::optional<omp::ScheduleModifier> scheduleMod = wsloopOp.getScheduleMod();
   bool isSimd = wsloopOp.getScheduleSimd();
 
-  ompBuilder->applyWorkshareLoop(
-      ompLoc.DL, loopInfo, allocaIP, !wsloopOp.getNowait(),
-      convertToScheduleKind(schedule), chunk, isSimd,
-      scheduleMod == omp::ScheduleModifier::monotonic,
-      scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered);
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
+      ompBuilder->applyWorkshareLoop(
+          ompLoc.DL, loopInfo, allocaIP, !wsloopOp.getNowait(),
+          convertToScheduleKind(schedule), chunk, isSimd,
+          scheduleMod == omp::ScheduleModifier::monotonic,
+          scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered);
+
+  if (!wsloopIP)
+    return opInst.emitError(llvm::toString(wsloopIP.takeError()));
 
   // Continue building IR after the loop. Note that the LoopInfo returned by
   // `collapseLoops` points inside the outermost loop and is intended for
@@ -1350,10 +1411,6 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
   ArrayRef<bool> isByRef = getIsByRef(opInst.getReductionByref());
   assert(isByRef.size() == opInst.getNumReductionVars());
-
-  // TODO: support error propagation in OpenMPIRBuilder and use it instead of
-  // relying on captured variables.
-  LogicalResult bodyGenStatus = success();
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
 
   // Collect delayed privatization declarations
@@ -1372,7 +1429,8 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       opInst.getNumReductionVars());
   SmallVector<DeferredStore> deferredStores;
 
-  auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
+  auto bodyGenCB = [&](InsertPointTy allocaIP,
+                       InsertPointTy codeGenIP) -> llvm::Error {
     // Allocate private vars
     llvm::BranchInst *allocaTerminator =
         llvm::cast<llvm::BranchInst>(allocaIP.getBlock()->getTerminator());
@@ -1418,10 +1476,11 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
         builder.SetInsertPoint(privAllocBlock->getTerminator());
       }
       if (failed(inlineConvertOmpRegions(allocRegion, "omp.private.alloc",
-                                         builder, moduleTranslation, &phis))) {
-        bodyGenStatus = failure();
-        return;
-      }
+                                         builder, moduleTranslation, &phis)))
+        return llvm::createStringError(
+            "failed to inline `alloc` region of an `omp.private` op in the "
+            "parallel region");
+
       assert(phis.size() == 1 && "expected one allocation to be yielded");
 
       moduleTranslation.mapValue(privateBlockArgs[i], phis[0]);
@@ -1447,7 +1506,7 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
             opInst, reductionArgs, builder, moduleTranslation, allocaIP,
             reductionDecls, privateReductionVariables, reductionVariableMap,
             deferredStores, isByRef)))
-      bodyGenStatus = failure();
+      return llvm::createStringError("failed reduction vars allocation");
 
     // Apply copy region for firstprivate.
     bool needsFirstprivate =
@@ -1486,10 +1545,10 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       // in-place convert copy region
       builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator());
       if (failed(inlineConvertOmpRegions(copyRegion, "omp.private.copy",
-                                         builder, moduleTranslation))) {
-        bodyGenStatus = failure();
-        return;
-      }
+                                         builder, moduleTranslation)))
+        return llvm::createStringError(
+            "failed to inline `copy` region of an `omp.private` op in the "
+            "parallel region");
 
       // ignore unused value yielded from copy region
 
@@ -1538,7 +1597,9 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       if (failed(inlineConvertOmpRegions(
               reductionDecls[i].getInitializerRegion(), "omp.reduction.neutral",
               builder, moduleTranslation, &phis)))
-        bodyGenStatus = failure();
+        return llvm::createStringError(
+            "failed to inline `init` region of an `omp.declare_reduction` op "
+            "in the parallel region");
       assert(phis.size() == 1 &&
              "expected one value to be yielded from the "
              "reduction neutral element declaration region");
@@ -1582,9 +1643,10 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
 
     // ParallelOp has only one region associated with it.
     builder.restoreIP(codeGenIP);
-    auto regionBlock =
-        convertOmpOpRegions(opInst.getRegion(), "omp.par.region", builder,
-                            moduleTranslation, bodyGenStatus);
+    llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions(
+        opInst.getRegion(), "omp.par.region", builder, moduleTranslation);
+    if (!regionBlock)
+      return regionBlock.takeError();
 
     // Process the reductions if required.
     if (opInst.getNumReductionVars() > 0) {
@@ -1597,23 +1659,25 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
                            privateReductionVariables, reductionInfos);
 
       // Move to region cont block
-      builder.SetInsertPoint(regionBlock->getTerminator());
+      builder.SetInsertPoint((*regionBlock)->getTerminator());
 
       // Generate reductions from info
       llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable();
       builder.SetInsertPoint(tempTerminator);
 
-      llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint =
+      llvm::OpenMPIRBuilder::InsertPointOrErrorTy contInsertPoint =
           ompBuilder->createReductions(builder.saveIP(), allocaIP,
                                        reductionInfos, isByRef, false);
-      if (!contInsertPoint.getBlock()) {
-        bodyGenStatus = opInst->emitOpError() << "failed to convert reductions";
-        return;
-      }
+      if (!contInsertPoint)
+        return contInsertPoint.takeError();
+
+      if (!contInsertPoint->getBlock())
+        return llvm::createStringError("failed to convert reductions");
 
       tempTerminator->eraseFromParent();
-      builder.restoreIP(contInsertPoint);
+      builder.restoreIP(*contInsertPoint);
     }
+    return llvm::Error::success();
   };
 
   auto privCB = [](InsertPointTy allocaIP, InsertPointTy codeGenIP,
@@ -1626,7 +1690,7 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
 
   // TODO: Perform finalization actions for variables. This has to be
   // called for variables which have destructors/finalizers.
-  auto finiCB = [&](InsertPointTy codeGenIP) {
+  auto finiCB = [&](InsertPointTy codeGenIP) -> llvm::Error {
     InsertPointTy oldIP = builder.saveIP();
     builder.restoreIP(codeGenIP);
 
@@ -1640,7 +1704,9 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
     if (failed(inlineOmpRegionCleanup(
             reductionCleanupRegions, privateReductionVariables,
             moduleTranslation, builder, "omp.reduction.cleanup")))
-      bodyGenStatus = failure();
+      return llvm::createStringError(
+          "failed to inline `cleanup` region of an `omp.declare_reduction` op "
+          "in the parallel region");
 
     SmallVector<Region *> privateCleanupRegions;
     llvm::transform(privateDecls, std::back_inserter(privateCleanupRegions),
@@ -1651,9 +1717,11 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
     if (failed(inlineOmpRegionCleanup(
             privateCleanupRegions, llvmPrivateVars, moduleTranslation, builder,
             "omp.private.dealloc", /*shouldLoadCleanupRegionArg=*/false)))
-      bodyGenStatus = failure();
+      return llvm::createStringError("failed to inline `dealloc` region of an "
+                                     "`omp.private` op in the parallel region");
 
     builder.restoreIP(oldIP);
+    return llvm::Error::success();
   };
 
   llvm::Value *ifCond = nullptr;
@@ -1672,11 +1740,14 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       findAllocaInsertPoint(builder, moduleTranslation);
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
 
-  builder.restoreIP(
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
       ompBuilder->createParallel(ompLoc, allocaIP, bodyGenCB, privCB, finiCB,
-                                 ifCond, numThreads, pbKind, isCancellable));
+                                 ifCond, numThreads, pbKind, isCancellable);
+  if (!afterIP)
+    return opInst.emitError(llvm::toString(afterIP.takeError()));
 
-  return bodyGenStatus;
+  builder.restoreIP(*afterIP);
+  return success();
 }
 
 /// Convert Order attribute to llvm::omp::OrderKind.
@@ -1718,12 +1789,10 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
 
   // Generator of the canonical loop body.
-  // TODO: support error propagation in OpenMPIRBuilder and use it instead of
-  // relying on captured variables.
   SmallVector<llvm::CanonicalLoopInfo *> loopInfos;
   SmallVector<llvm::OpenMPIRBuilder::InsertPointTy> bodyInsertPoints;
-  LogicalResult bodyGenStatus = success();
-  auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) {
+  auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip,
+                     llvm::Value *iv) -> llvm::Error {
     // Make sure further conversions know about the induction variable.
     moduleTranslation.mapValue(
         loopOp.getRegion().front().getArgument(loopInfos.size()), iv);
@@ -1734,12 +1803,13 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
     bodyInsertPoints.push_back(ip);
 
     if (loopInfos.size() != loopOp.getNumLoops() - 1)
-      return;
+      return llvm::Error::success();
 
     // Convert the body of the loop.
     builder.restoreIP(ip);
-    convertOmpOpRegions(loopOp.getRegion(), "omp.simd.region", builder,
-                        moduleTranslation, bodyGenStatus);
+    return convertOmpOpRegions(loopOp.getRegion(), "omp.simd.region", builder,
+                               moduleTranslation)
+        .takeError();
   };
 
   // Delegate actual loop construction to the OpenMP IRBuilder.
@@ -1765,12 +1835,16 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
                                                        ompLoc.DL);
       computeIP = loopInfos.front()->getPreheaderIP();
     }
-    loopInfos.push_back(ompBuilder->createCanonicalLoop(
-        loc, bodyGen, lowerBound, upperBound, step,
-        /*IsSigned=*/true, /*Inclusive=*/true, computeIP));
 
-    if (failed(bodyGenStatus))
-      return failure();
+    llvm::Expected<llvm::CanonicalLoopInfo *> result =
+        ompBuilder->createCanonicalLoop(
+            loc, bodyGen, lowerBound, upperBound, step,
+            /*IsSigned=*/true, /*InclusiveStop=*/true, computeIP);
+
+    if (!result)
+      return loopOp->emitError(llvm::toString(result.takeError()));
+
+    loopInfos.push_back(*result);
   }
 
   // Collapse loops.
@@ -1921,18 +1995,17 @@ convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst,
       convertAtomicOrdering(opInst.getMemoryOrder());
 
   // Generate update code.
-  LogicalResult updateGenStatus = success();
-  auto updateFn = [&opInst, &moduleTranslation, &updateGenStatus](
-                      llvm::Value *atomicx,
-                      llvm::IRBuilder<> &builder) -> llvm::Value * {
+  auto updateFn =
+      [&opInst, &moduleTranslation](
+          llvm::Value *atomicx,
+          llvm::IRBuilder<> &builder) -> llvm::Expected<llvm::Value *> {
     Block &bb = *opInst.getRegion().begin();
     moduleTranslation.mapValue(*opInst.getRegion().args_begin(), atomicx);
     moduleTranslation.mapBlock(&bb, builder.GetInsertBlock());
-    if (failed(moduleTranslation.convertBlock(bb, true, builder))) {
-      updateGenStatus = (opInst.emitError()
-                         << "unable to convert update operation to llvm IR");
-      return nullptr;
-    }
+    if (failed(moduleTranslation.convertBlock(bb, true, builder)))
+      return llvm::createStringError(
+          "unable to convert update operation to llvm IR");
+
     omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.getTerminator());
     assert(yieldop && yieldop.getResults().size() == 1 &&
            "terminator must be omp.yield op and it must have exactly one "
@@ -1943,10 +2016,16 @@ convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst,
   // Handle ambiguous alloca, if any.
   auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
-  builder.restoreIP(ompBuilder->createAtomicUpdate(
-      ompLoc, allocaIP, llvmAtomicX, llvmExpr, atomicOrdering, binop, updateFn,
-      isXBinopExpr));
-  return updateGenStatus;
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
+      ompBuilder->createAtomicUpdate(ompLoc, allocaIP, llvmAtomicX, llvmExpr,
+                                     atomicOrdering, binop, updateFn,
+                                     isXBinopExpr);
+
+  if (!afterIP)
+    return opInst.emitError(llvm::toString(afterIP.takeError()));
+
+  builder.restoreIP(*afterIP);
+  return success();
 }
 
 static LogicalResult
@@ -2007,20 +2086,19 @@ convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,
   llvm::AtomicOrdering atomicOrdering =
       convertAtomicOrdering(atomicCaptureOp.getMemoryOrder());
 
-  LogicalResult updateGenStatus = success();
-  auto updateFn = [&](llvm::Value *atomicx,
-                      llvm::IRBuilder<> &builder) -> llvm::Value * {
+  auto updateFn =
+      [&](llvm::Value *atomicx,
+          llvm::IRBuilder<> &builder) -> llvm::Expected<llvm::Value *> {
     if (atomicWriteOp)
       return moduleTranslation.lookupValue(atomicWriteOp.getExpr());
     Block &bb = *atomicUpdateOp.getRegion().begin();
     moduleTranslation.mapValue(*atomicUpdateOp.getRegion().args_begin(),
                                atomicx);
     moduleTranslation.mapBlock(&bb, builder.GetInsertBlock());
-    if (failed(moduleTranslation.convertBlock(bb, true, builder))) {
-      updateGenStatus = (atomicUpdateOp.emitError()
-                         << "unable to convert update operation to llvm IR");
-      return nullptr;
-    }
+    if (failed(moduleTranslation.convertBlock(bb, true, builder)))
+      return llvm::createStringError(
+          "unable to convert update operation to llvm IR");
+
     omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.getTerminator());
     assert(yieldop && yieldop.getResults().size() == 1 &&
            "terminator must be omp.yield op and it must have exactly one "
@@ -2031,10 +2109,16 @@ convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,
   // Handle ambiguous alloca, if any.
   auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
-  builder.restoreIP(ompBuilder->createAtomicCapture(
-      ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering,
-      binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr));
-  return updateGenStatus;
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
+      ompBuilder->createAtomicCapture(
+          ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering,
+          binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr);
+
+  if (!afterIP)
+    return atomicCaptureOp.emitError(llvm::toString(afterIP.takeError()));
+
+  builder.restoreIP(*afterIP);
+  return success();
 }
 
 /// Converts an OpenMP Threadprivate operation into LLVM IR using
@@ -3019,8 +3103,8 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
       };
 
   using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
-  LogicalResult bodyGenStatus = success();
-  auto bodyGenCB = [&](InsertPointTy codeGenIP, BodyGenTy bodyGenType) {
+  auto bodyGenCB = [&](InsertPointTy codeGenIP, BodyGenTy bodyGenType)
+      -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
     assert(isa<omp::TargetDataOp>(op) &&
            "BodyGen requested for non TargetDataOp");
     auto blockArgIface = cast<omp::BlockArgOpenMPOpInterface>(op);
@@ -3046,8 +3130,10 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
                        return info.DevicePtrInfoMap[basePointer].second;
                      });
 
-        bodyGenStatus = inlineConvertOmpRegions(region, "omp.data.region",
-                                                builder, moduleTranslation);
+        if (failed(inlineConvertOmpRegions(region, "omp.data.region", builder,
+                                           moduleTranslation)))
+          return llvm::createStringError(
+              "failed to inline region of an `omp.target_data` op");
       }
       break;
     case BodyGenTy::DupNoPriv:
@@ -3067,8 +3153,10 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
                        mapData.BasePointers, mapData.DevicePointers);
         }
 
-        bodyGenStatus = inlineConvertOmpRegions(region, "omp.data.region",
-                                                builder, moduleTranslation);
+        if (failed(inlineConvertOmpRegions(region, "omp.data.region", builder,
+                                           moduleTranslation)))
+          return llvm::createStringError(
+              "failed to inline region of an `omp.target_data` op");
       }
       break;
     }
@@ -3078,17 +3166,21 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
   llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
       findAllocaInsertPoint(builder, moduleTranslation);
-  if (isa<omp::TargetDataOp>(op)) {
-    builder.restoreIP(ompBuilder->createTargetData(
-        ompLoc, allocaIP, builder.saveIP(), builder.getInt64(deviceID), ifCond,
-        info, genMapInfoCB, nullptr, bodyGenCB));
-  } else {
-    builder.restoreIP(ompBuilder->createTargetData(
-        ompLoc, allocaIP, builder.saveIP(), builder.getInt64(deviceID), ifCond,
-        info, genMapInfoCB, &RTLFn));
-  }
-
-  return bodyGenStatus;
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = [&]() {
+    if (isa<omp::TargetDataOp>(op))
+      return ompBuilder->createTargetData(
+          ompLoc, allocaIP, builder.saveIP(), builder.getInt64(deviceID),
+          ifCond, info, genMapInfoCB, nullptr, bodyGenCB);
+    return ompBuilder->createTargetData(ompLoc, allocaIP, builder.saveIP(),
+                                        builder.getInt64(deviceID), ifCond,
+                                        info, genMapInfoCB, &RTLFn);
+  }();
+
+  if (!afterIP)
+    return op->emitError(llvm::toString(afterIP.takeError()));
+
+  builder.restoreIP(*afterIP);
+  return success();
 }
 
 /// Lowers the FlagsAttr which is applied to the module on the device
@@ -3320,6 +3412,8 @@ createDeviceArgumentAccessor(MapInfoData &mapData, llvm::Argument &arg,
   }
   case omp::VariableCaptureKind::This:
   case omp::VariableCaptureKind::VLAType:
+    // TODO: Consider returning error to use standard reporting for
+    // unimplemented features.
     assert(false && "Currently unsupported capture kind");
     break;
   }
@@ -3350,10 +3444,9 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
   bool isOffloadEntry =
       isTargetDevice || !ompBuilder->Config.TargetTriples.empty();
 
-  LogicalResult bodyGenStatus = success();
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
-  auto bodyCB = [&](InsertPointTy allocaIP,
-                    InsertPointTy codeGenIP) -> InsertPointTy {
+  auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP)
+      -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
     // Forward target-cpu and target-features function attributes from the
     // original function to the new outlined function.
     llvm::Function *llvmParentFn =
@@ -3396,34 +3489,35 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
         if (privatizer.getDataSharingType() ==
                 omp::DataSharingClauseType::FirstPrivate ||
             !privatizer.getDeallocRegion().empty()) {
-          opInst.emitError("Translation of omp.target from MLIR to LLVMIR "
-                           "failed because translation of firstprivate and "
-                           " private allocatables is not supported yet");
-          bodyGenStatus = failure();
-        } else {
-          moduleTranslation.mapValue(privatizer.getAllocMoldArg(),
-                                     moduleTranslation.lookupValue(privVar));
-          Region &allocRegion = privatizer.getAllocRegion();
-          SmallVector<llvm::Value *, 1> yieldedValues;
-          if (failed(inlineConvertOmpRegions(
-                  allocRegion, "omp.targetop.privatizer", builder,
-                  moduleTranslation, &yieldedValues))) {
-            opInst.emitError(
-                "failed to inline `alloc` region of an `omp.private` "
-                "op in the target region");
-            bodyGenStatus = failure();
-          } else {
-            assert(yieldedValues.size() == 1);
-            moduleTranslation.mapValue(privBlockArg, yieldedValues.front());
-          }
-          moduleTranslation.forgetMapping(allocRegion);
-          builder.restoreIP(builder.saveIP());
+          return llvm::createStringError(
+              "Translation of omp.target from MLIR to LLVMIR "
+              "failed because translation of firstprivate and "
+              " private allocatables is not supported yet");
         }
+        moduleTranslation.mapValue(privatizer.getAllocMoldArg(),
+                                   moduleTranslation.lookupValue(privVar));
+        Region &allocRegion = privatizer.getAllocRegion();
+        SmallVector<llvm::Value *, 1> yieldedValues;
+        if (failed(inlineConvertOmpRegions(
+                allocRegion, "omp.targetop.privatizer", builder,
+                moduleTranslation, &yieldedValues))) {
+          return llvm::createStringError(
+              "failed to inline `alloc` region of an `omp.private` "
+              "op in the target region");
+        }
+        assert(yieldedValues.size() == 1);
+        moduleTranslation.mapValue(privBlockArg, yieldedValues.front());
+        moduleTranslation.forgetMapping(allocRegion);
+        builder.restoreIP(builder.saveIP());
       }
     }
-    llvm::BasicBlock *exitBlock = convertOmpOpRegions(
-        targetRegion, "omp.target", builder, moduleTranslation, bodyGenStatus);
-    builder.SetInsertPoint(exitBlock);
+
+    llvm::Expected<llvm::BasicBlock *> exitBlock = convertOmpOpRegions(
+        targetRegion, "omp.target", builder, moduleTranslation);
+    if (!exitBlock)
+      return exitBlock.takeError();
+
+    builder.SetInsertPoint(*exitBlock);
     return builder.saveIP();
   };
 
@@ -3455,7 +3549,8 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
 
   auto argAccessorCB = [&](llvm::Argument &arg, llvm::Value *input,
                            llvm::Value *&retVal, InsertPointTy allocaIP,
-                           InsertPointTy codeGenIP) {
+                           InsertPointTy codeGenIP)
+      -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
     // We just return the unaltered argument for the host function
     // for now, some alterations may be required in the future to
     // keep host fallback functions working identically to the device
@@ -3486,10 +3581,16 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
   buildDependData(targetOp.getDependKinds(), targetOp.getDependVars(),
                   moduleTranslation, dds);
 
-  builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTarget(
-      ompLoc, isOffloadEntry, allocaIP, builder.saveIP(), entryInfo,
-      defaultValTeams, defaultValThreads, kernelInput, genMapInfoCB, bodyCB,
-      argAccessorCB, dds, targetOp.getNowait()));
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy result =
+      moduleTranslation.getOpenMPBuilder()->createTarget(
+          ompLoc, isOffloadEntry, allocaIP, builder.saveIP(), entryInfo,
+          defaultValTeams, defaultValThreads, kernelInput, genMapInfoCB, bodyCB,
+          argAccessorCB, dds, targetOp.getNowait());
+
+  if (!result)
+    return opInst.emitError(llvm::toString(result.takeError()));
+
+  builder.restoreIP(*result);
 
   // Remap access operations to declare target reference pointers for the
   // device, essentially generating extra loadop's as necessary
@@ -3497,7 +3598,7 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
     handleDeclareTargetMapVar(mapData, moduleTranslation, builder,
                               llvmOutlinedFn);
 
-  return bodyGenStatus;
+  return success();
 }
 
 static LogicalResult
@@ -3618,8 +3719,13 @@ convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder,
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
 
   return llvm::TypeSwitch<Operation *, LogicalResult>(op)
-      .Case([&](omp::BarrierOp) {
-        ompBuilder->createBarrier(builder.saveIP(), llvm::omp::OMPD_barrier);
+      .Case([&](omp::BarrierOp) -> LogicalResult {
+        llvm::OpenMPIRBuilder::InsertPointOrErrorTy result =
+            ompBuilder->createBarrier(builder.saveIP(),
+                                      llvm::omp::OMPD_barrier);
+        if (!result)
+          return op->emitError(llvm::toString(result.takeError()));
+
         return success();
       })
       .Case([&](omp::TaskyieldOp) {
-- 
GitLab


From 85af1926f74e88e14c9f58946537e025a74ccf7e Mon Sep 17 00:00:00 2001
From: Abid Qadeer <haqadeer@amd.com>
Date: Fri, 25 Oct 2024 11:43:25 +0100
Subject: [PATCH 149/693] [flang][debug] Support mlir::NoneType. (#113550)

---
 .../Optimizer/Transforms/DebugTypeGenerator.cpp    |  2 ++
 flang/test/Transforms/debug-none-type.fir          | 14 ++++++++++++++
 2 files changed, 16 insertions(+)
 create mode 100644 flang/test/Transforms/debug-none-type.fir

diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index 1ab6c76dae8e..e387e27533a0 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -581,6 +581,8 @@ DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
                                   /*genAssociated=*/false);
   } else if (auto vecTy = mlir::dyn_cast_or_null<fir::VectorType>(Ty)) {
     return convertVectorType(vecTy, fileAttr, scope, declOp);
+  } else if (mlir::isa<mlir::NoneType>(Ty)) {
+    return mlir::LLVM::DINullTypeAttr::get(context);
   } else if (auto boxTy = mlir::dyn_cast_or_null<fir::BoxType>(Ty)) {
     auto elTy = boxTy.getElementType();
     if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(elTy))
diff --git a/flang/test/Transforms/debug-none-type.fir b/flang/test/Transforms/debug-none-type.fir
new file mode 100644
index 000000000000..5eee6a633219
--- /dev/null
+++ b/flang/test/Transforms/debug-none-type.fir
@@ -0,0 +1,14 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func private @_FortranAAssign(i8) -> none loc(#loc1)
+  func.func private @foo() -> !fir.ref<none>  loc(#loc2)
+}
+#loc1 = loc("test.f90":5:1)
+#loc2 = loc("test.f90":15:1)
+
+// CHECK: #[[INT8_TY:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 8, encoding = DW_ATE_signed>
+// CHECK: #[[NONE_TY:.*]] = #llvm.di_null_type
+// CHECK: #[[REFNONE_TY:.*]] = #llvm.di_derived_type<tag = DW_TAG_pointer_type, name = "", baseType = #di_null_type{{.*}}>
+// CHECK: #llvm.di_subroutine_type<{{.*}}types = #[[NONE_TY]], #[[INT8_TY]]>
+// CHECK: #llvm.di_subroutine_type<{{.*}}types = #[[REFNONE_TY]]>
-- 
GitLab


From c2d2b3b80866633b4db65c3841c40c16815267f3 Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Fri, 25 Oct 2024 18:43:40 +0800
Subject: [PATCH 150/693] [test] Avoid writing to a potentially write-protected
 dir (#113674)

---
 clang/test/CodeGen/X86/movrs-avx10.2-512-builtins-error-32.c | 2 +-
 clang/test/CodeGen/X86/movrs-avx10.2-builtins-error-32.c     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/test/CodeGen/X86/movrs-avx10.2-512-builtins-error-32.c b/clang/test/CodeGen/X86/movrs-avx10.2-512-builtins-error-32.c
index 944033724a6a..a4d887f0be41 100644
--- a/clang/test/CodeGen/X86/movrs-avx10.2-512-builtins-error-32.c
+++ b/clang/test/CodeGen/X86/movrs-avx10.2-512-builtins-error-32.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -ffreestanding %s -Wno-implicit-function-declaration -triple=i386-- -target-feature +movrs -target-feature +avx10.2-512 -emit-llvm -verify
+// RUN: %clang_cc1 -ffreestanding %s -Wno-implicit-function-declaration -triple=i386-- -target-feature +movrs -target-feature +avx10.2-512 -verify
 
 #include <immintrin.h>
 __m512i test_mm512_loadrs_epi8(const __m512i * __A) {
diff --git a/clang/test/CodeGen/X86/movrs-avx10.2-builtins-error-32.c b/clang/test/CodeGen/X86/movrs-avx10.2-builtins-error-32.c
index 68608b0cbff0..2a7204e39b83 100644
--- a/clang/test/CodeGen/X86/movrs-avx10.2-builtins-error-32.c
+++ b/clang/test/CodeGen/X86/movrs-avx10.2-builtins-error-32.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -ffreestanding %s -Wno-implicit-function-declaration -triple=i386-unknown-unknown -target-feature +movrs -target-feature +avx10.2-256 -emit-llvm -verify
+// RUN: %clang_cc1 -ffreestanding %s -Wno-implicit-function-declaration -triple=i386-unknown-unknown -target-feature +movrs -target-feature +avx10.2-256 -verify
 
 #include <immintrin.h>
 __m128i test_mm_loadrs_epi8(const __m128i * __A) {
-- 
GitLab


From 2d26ef09fc87472cd42ea219c8f9267599872958 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Fri, 25 Oct 2024 12:06:57 +0100
Subject: [PATCH 151/693] [lldb-dap][test] Set disableASLR to False for tests
 (#113593)

When running in constrained environments like docker, disabling ASLR
might fail with errors like:
```
AssertionError: False is not true : launch failed (Cannot launch
'/__w/.../lldb-dap/stackTrace/subtleFrames/TestDAP_subtleFrames.test_subtleFrames/a.out':
personality set failed: Operation not permitted)
```
E.g., https://github.com/llvm/llvm-project/pull/110303

Hence we already run `settings set target.disable-aslr false` as part of
the init-commands for the non-DAP tests (see
https://github.com/llvm/llvm-project/pull/88312 and
https://discourse.llvm.org/t/running-lldb-in-a-container/76801).

But we never adjusted it for the DAP tests. As a result we get
conflicting test logs like:
```
 {
    "arguments": {
      "commandEscapePrefix": null,
      "disableASLR": true,
     ....
      "initCommands": [
        ...
        "settings set target.disable-aslr false",
```

Disabling ASLR by default in tests isn't useulf (it's only really a
debugging aid for users). So this patch sets `disableASLR=False` by
default.
---
 .../Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
index 7e80912be446..a25466f07fa5 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
@@ -367,7 +367,7 @@ class DAPTestCaseBase(TestBase):
         cwd=None,
         env=None,
         stopOnEntry=False,
-        disableASLR=True,
+        disableASLR=False,
         disableSTDIO=False,
         shellExpandArguments=False,
         trace=False,
@@ -451,7 +451,7 @@ class DAPTestCaseBase(TestBase):
         cwd=None,
         env=None,
         stopOnEntry=False,
-        disableASLR=True,
+        disableASLR=False,
         disableSTDIO=False,
         shellExpandArguments=False,
         trace=False,
-- 
GitLab


From 2c0b34852af4dc9964f8bf6db303bd54a32856e7 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Fri, 25 Oct 2024 04:24:50 -0700
Subject: [PATCH 152/693] [RISCV] Mark pointer masking extensions as
 non-experimental (#113618)

These extensions were ratified very recently.

<https://lf-riscv.atlassian.net/wiki/spaces/HOME/pages/16154732/Ratified+Extensions>

I've ensured we have definitions for all extensions in the document
<https://drive.google.com/file/d/159QffOTbi3EEbdkKndYRZ2c46D25ZLmO/view?usp=drive_link>.
There are no additional CSRs.
---
 .../Driver/print-supported-extensions-riscv.c | 10 +++++-----
 clang/test/Driver/riscv-profiles.c            |  2 +-
 .../test/Preprocessor/riscv-target-features.c | 16 +++++++--------
 llvm/docs/RISCVUsage.rst                      |  8 +++++---
 llvm/docs/ReleaseNotes.md                     |  2 ++
 llvm/lib/Target/RISCV/RISCVFeatures.td        | 20 +++++++++----------
 llvm/test/CodeGen/RISCV/attributes.ll         | 20 +++++++++----------
 llvm/test/CodeGen/RISCV/rvv/pr107950.ll       |  2 +-
 .../TargetParser/RISCVISAInfoTest.cpp         | 10 +++++-----
 9 files changed, 47 insertions(+), 43 deletions(-)

diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index 65375b79cb68..342d6e921a5a 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -120,6 +120,8 @@
 // CHECK-NEXT:     smcdeleg             1.0       'Smcdeleg' (Counter Delegation Machine Level)
 // CHECK-NEXT:     smcsrind             1.0       'Smcsrind' (Indirect CSR Access Machine Level)
 // CHECK-NEXT:     smepmp               1.0       'Smepmp' (Enhanced Physical Memory Protection)
+// CHECK-NEXT:     smmpm                1.0       'Smmpm' (Machine-level Pointer Masking for M-mode)
+// CHECK-NEXT:     smnpm                1.0       'Smnpm' (Machine-level Pointer Masking for next lower privilege mode)
 // CHECK-NEXT:     smrnmi               1.0       'Smrnmi' (Resumable Non-Maskable Interrupts)
 // CHECK-NEXT:     smstateen            1.0       'Smstateen' (Machine-mode view of the state-enable extension)
 // CHECK-NEXT:     ssaia                1.0       'Ssaia' (Advanced Interrupt Architecture Supervisor Level)
@@ -128,6 +130,8 @@
 // CHECK-NEXT:     sscofpmf             1.0       'Sscofpmf' (Count Overflow and Mode-Based Filtering)
 // CHECK-NEXT:     sscounterenw         1.0       'Sscounterenw' (Support writeable scounteren enable bit for any hpmcounter that is not read-only zero)
 // CHECK-NEXT:     sscsrind             1.0       'Sscsrind' (Indirect CSR Access Supervisor Level)
+// CHECK-NEXT:     ssnpm                1.0       'Ssnpm' (Supervisor-level Pointer Masking for next lower privilege mode)
+// CHECK-NEXT:     sspm                 1.0       'Sspm' (Indicates Supervisor-mode Pointer Masking)
 // CHECK-NEXT:     ssqosid              1.0       'Ssqosid' (Quality-of-Service (QoS) Identifiers)
 // CHECK-NEXT:     ssstateen            1.0       'Ssstateen' (Supervisor-mode view of the state-enable extension)
 // CHECK-NEXT:     ssstrict             1.0       'Ssstrict' (No non-conforming extensions are present)
@@ -135,6 +139,7 @@
 // CHECK-NEXT:     sstvala              1.0       'Sstvala' (stval provides all needed values)
 // CHECK-NEXT:     sstvecd              1.0       'Sstvecd' (stvec supports Direct mode)
 // CHECK-NEXT:     ssu64xl              1.0       'Ssu64xl' (UXLEN=64 supported)
+// CHECK-NEXT:     supm                 1.0       'Supm' (Indicates User-mode Pointer Masking)
 // CHECK-NEXT:     svade                1.0       'Svade' (Raise exceptions on improper A/D bits)
 // CHECK-NEXT:     svadu                1.0       'Svadu' (Hardware A/D updates)
 // CHECK-NEXT:     svbare               1.0       'Svbare' $(satp mode Bare supported)
@@ -177,12 +182,7 @@
 // CHECK-NEXT:     zvbc32e              0.7       'Zvbc32e' (Vector Carryless Multiplication with 32-bits elements)
 // CHECK-NEXT:     zvkgs                0.7       'Zvkgs' (Vector-Scalar GCM instructions for Cryptography)
 // CHECK-NEXT:     smctr                1.0       'Smctr' (Control Transfer Records Machine Level)
-// CHECK-NEXT:     smmpm                1.0       'Smmpm' (Machine-level Pointer Masking for M-mode)
-// CHECK-NEXT:     smnpm                1.0       'Smnpm' (Machine-level Pointer Masking for next lower privilege mode)
 // CHECK-NEXT:     ssctr                1.0       'Ssctr' (Control Transfer Records Supervisor Level)
-// CHECK-NEXT:     ssnpm                1.0       'Ssnpm' (Supervisor-level Pointer Masking for next lower privilege mode)
-// CHECK-NEXT:     sspm                 1.0       'Sspm' (Indicates Supervisor-mode Pointer Masking)
-// CHECK-NEXT:     supm                 1.0       'Supm' (Indicates User-mode Pointer Masking)
 // CHECK-EMPTY:
 // CHECK-NEXT: Supported Profiles
 // CHECK-NEXT:     rva20s64
diff --git a/clang/test/Driver/riscv-profiles.c b/clang/test/Driver/riscv-profiles.c
index 55aa5b398cee..42e23cf57c88 100644
--- a/clang/test/Driver/riscv-profiles.c
+++ b/clang/test/Driver/riscv-profiles.c
@@ -195,7 +195,7 @@
 // RVA23S64: "-target-feature" "+ssccptr"
 // RVA23S64: "-target-feature" "+sscofpmf"
 // RVA23S64: "-target-feature" "+sscounterenw"
-// RVA23S64: "-target-feature" "+experimental-ssnpm"
+// RVA23S64: "-target-feature" "+ssnpm"
 // RVA23S64: "-target-feature" "+ssstateen"
 // RVA23S64: "-target-feature" "+sstc"
 // RVA23S64: "-target-feature" "+sstvala"
diff --git a/clang/test/Preprocessor/riscv-target-features.c b/clang/test/Preprocessor/riscv-target-features.c
index 9e986f0143ae..98ad564d2b84 100644
--- a/clang/test/Preprocessor/riscv-target-features.c
+++ b/clang/test/Preprocessor/riscv-target-features.c
@@ -1719,10 +1719,10 @@
 // RUN:   -o - | FileCheck --check-prefix=CHECK-ZICFISS-EXT %s
 // CHECK-ZICFISS-EXT: __riscv_zicfiss 1000000{{$}}
 
-// RUN: %clang --target=riscv32 -menable-experimental-extensions \
+// RUN: %clang --target=riscv32 \
 // RUN:   -march=rv32i_ssnpm1p0 -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-SSNPM-EXT %s
-// RUN: %clang --target=riscv64 -menable-experimental-extensions \
+// RUN: %clang --target=riscv64 \
 // RUN:   -march=rv64i_ssnpm1p0 -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-SSNPM-EXT %s
 // CHECK-SSNPM-EXT: __riscv_ssnpm 1000000{{$}}
@@ -1735,26 +1735,26 @@
 // RUN:   -o - | FileCheck --check-prefix=CHECK-SMNPM-EXT %s
 // CHECK-SMNPM-EXT: __riscv_smnpm 1000000{{$}}
 
-// RUN: %clang --target=riscv32 -menable-experimental-extensions \
+// RUN: %clang --target=riscv32 \
 // RUN:   -march=rv32i_smmpm1p0 -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-SMMPM-EXT %s
-// RUN: %clang --target=riscv64 -menable-experimental-extensions \
+// RUN: %clang --target=riscv64 \
 // RUN:   -march=rv64i_smmpm1p0 -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-SMMPM-EXT %s
 // CHECK-SMMPM-EXT: __riscv_smmpm 1000000{{$}}
 
-// RUN: %clang --target=riscv32 -menable-experimental-extensions \
+// RUN: %clang --target=riscv32 \
 // RUN:   -march=rv32i_sspm1p0 -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-SSPM-EXT %s
-// RUN: %clang --target=riscv64 -menable-experimental-extensions \
+// RUN: %clang --target=riscv64 \
 // RUN:   -march=rv64i_sspm1p0 -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-SSPM-EXT %s
 // CHECK-SSPM-EXT: __riscv_sspm 1000000{{$}}
 
-// RUN: %clang --target=riscv32 -menable-experimental-extensions \
+// RUN: %clang --target=riscv32 \
 // RUN:   -march=rv32i_supm1p0 -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-SUPM-EXT %s
-// RUN: %clang --target=riscv64 -menable-experimental-extensions \
+// RUN: %clang --target=riscv64 \
 // RUN:   -march=rv64i_supm1p0 -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-SUPM-EXT %s
 // CHECK-SUPM-EXT: __riscv_supm 1000000{{$}}
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index 2135ed3b1114..ab58cdaa1b2f 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -129,6 +129,8 @@ on support follow.
      ``Smcdeleg``      Supported
      ``Smcsrind``      Supported
      ``Smepmp``        Supported
+     ``Smmpm``         Supported
+     ``Smnpm``         Supported
      ``Smrnmi``        Assembly Support
      ``Smstateen``     Assembly Support
      ``Ssaia``         Supported
@@ -137,6 +139,8 @@ on support follow.
      ``Sscofpmf``      Assembly Support
      ``Sscounterenw``  Assembly Support (`See note <#riscv-profiles-extensions-note>`__)
      ``Sscsrind``      Supported
+     ``Ssnpm``         Supported
+     ``Sspm``          Supported
      ``Ssqosid``       Assembly Support
      ``Ssstateen``     Assembly Support (`See note <#riscv-profiles-extensions-note>`__)
      ``Ssstrict``      Assembly Support (`See note <#riscv-profiles-extensions-note>`__)
@@ -144,6 +148,7 @@ on support follow.
      ``Sstvala``       Assembly Support (`See note <#riscv-profiles-extensions-note>`__)
      ``Sstvecd``       Assembly Support (`See note <#riscv-profiles-extensions-note>`__)
      ``Ssu64xl``       Assembly Support (`See note <#riscv-profiles-extensions-note>`__)
+     ``Supm``          Supported
      ``Svade``         Assembly Support (`See note <#riscv-profiles-extensions-note>`__)
      ``Svadu``         Assembly Support
      ``Svbare``        Assembly Support (`See note <#riscv-profiles-extensions-note>`__)
@@ -308,9 +313,6 @@ LLVM supports (to various degrees) a number of experimental extensions.  All exp
 
 The primary goal of experimental support is to assist in the process of ratification by providing an existence proof of an implementation, and simplifying efforts to validate the value of a proposed extension against large code bases.  Experimental extensions are expected to either transition to ratified status, or be eventually removed.  The decision on whether to accept an experimental extension is currently done on an entirely case by case basis; if you want to propose one, attending the bi-weekly RISC-V sync-up call is strongly advised.
 
-``experimental-ssnpm``, ``experimental-smnpm``, ``experimental-smmpm``, ``experimental-sspm``, ``experimental-supm``
-  LLVM implements the `v1.0.0-rc2 specification <https://github.com/riscv/riscv-j-extension/releases/tag/pointer-masking-v1.0.0-rc2>`__.
-
 ``experimental-zalasr``
   LLVM implements the `0.0.5 draft specification <https://github.com/mehnadnerd/riscv-zalasr>`__.
 
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 706546980cf6..7cca9116a513 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -171,6 +171,8 @@ Changes to the RISC-V Backend
 * Added `Smctr` and `Ssctr` extensions.
 * `-mcpu=syntacore-scr7` was added.
 * The `Zacas` extension is no longer marked as experimental.
+* The `Smmpm`, `Smnpm`, `Ssnpm`, `Supm`, and `Sspm` pointer masking extensions
+  are no longer marked as experimental.
 
 Changes to the WebAssembly Backend
 ----------------------------------
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 7ccce2e1c70b..778df542022f 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -1035,33 +1035,33 @@ def FeatureStdExtSvpbmt
 // privilege mode (U-mode), and for VS- and VU-modes if the H extension is
 // present.
 def FeatureStdExtSsnpm
-    : RISCVExperimentalExtension<"ssnpm", 1, 0,
-                                 "'Ssnpm' (Supervisor-level Pointer Masking for next lower privilege mode)">;
+    : RISCVExtension<"ssnpm", 1, 0,
+                     "'Ssnpm' (Supervisor-level Pointer Masking for next lower privilege mode)">;
 
 // A machine-level extension that provides pointer masking for the next lower
 // privilege mode (S/HS if S-mode is implemented, or U-mode otherwise).
 def FeatureStdExtSmnpm
-    : RISCVExperimentalExtension<"smnpm", 1, 0,
-                                 "'Smnpm' (Machine-level Pointer Masking for next lower privilege mode)">;
+    : RISCVExtension<"smnpm", 1, 0,
+                     "'Smnpm' (Machine-level Pointer Masking for next lower privilege mode)">;
 
 // A machine-level extension that provides pointer masking for M-mode.
 def FeatureStdExtSmmpm
-    : RISCVExperimentalExtension<"smmpm", 1, 0,
-                                 "'Smmpm' (Machine-level Pointer Masking for M-mode)">;
+    : RISCVExtension<"smmpm", 1, 0,
+                     "'Smmpm' (Machine-level Pointer Masking for M-mode)">;
 
 // An extension that indicates that there is pointer-masking support available
 // in supervisor mode, with some facility provided in the supervisor execution
 // environment to control pointer masking.
 def FeatureStdExtSspm
-    : RISCVExperimentalExtension<"sspm", 1, 0,
-                                 "'Sspm' (Indicates Supervisor-mode Pointer Masking)">;
+    : RISCVExtension<"sspm", 1, 0,
+                     "'Sspm' (Indicates Supervisor-mode Pointer Masking)">;
 
 // An extension that indicates that there is pointer-masking support available
 // in user mode, with some facility provided in the application execution
 // environment to control pointer masking.
 def FeatureStdExtSupm
-    : RISCVExperimentalExtension<"supm", 1, 0,
-                                 "'Supm' (Indicates User-mode Pointer Masking)">;
+    : RISCVExtension<"supm", 1, 0,
+                     "'Supm' (Indicates User-mode Pointer Masking)">;
 
 def FeatureStdExtSmctr
     : RISCVExperimentalExtension<"smctr", 1, 0,
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index aa27d63bfa62..e9743d484f77 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -129,11 +129,11 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+a,+zabha %s -o - | FileCheck --check-prefix=RV32ZABHA %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zve32x -mattr=+experimental-zvbc32e  %s -o - | FileCheck --check-prefix=RV32ZVBC32E %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zve32x -mattr=+experimental-zvkgs  %s -o - | FileCheck --check-prefix=RV32ZVKGS %s
-; RUN: llc -mtriple=riscv32 -mattr=+experimental-ssnpm  %s -o - | FileCheck --check-prefix=RV32SSNPM %s
-; RUN: llc -mtriple=riscv32 -mattr=+experimental-smnpm  %s -o - | FileCheck --check-prefix=RV32SMNPM %s
-; RUN: llc -mtriple=riscv32 -mattr=+experimental-smmpm %s -o - | FileCheck --check-prefix=RV32SMMPM %s
-; RUN: llc -mtriple=riscv32 -mattr=+experimental-sspm %s -o - | FileCheck --check-prefix=RV32SSPM %s
-; RUN: llc -mtriple=riscv32 -mattr=+experimental-supm %s -o - | FileCheck --check-prefix=RV32SUPM %s
+; RUN: llc -mtriple=riscv32 -mattr=+ssnpm  %s -o - | FileCheck --check-prefix=RV32SSNPM %s
+; RUN: llc -mtriple=riscv32 -mattr=+smnpm  %s -o - | FileCheck --check-prefix=RV32SMNPM %s
+; RUN: llc -mtriple=riscv32 -mattr=+smmpm %s -o - | FileCheck --check-prefix=RV32SMMPM %s
+; RUN: llc -mtriple=riscv32 -mattr=+sspm %s -o - | FileCheck --check-prefix=RV32SSPM %s
+; RUN: llc -mtriple=riscv32 -mattr=+supm %s -o - | FileCheck --check-prefix=RV32SUPM %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-smctr  %s -o - | FileCheck --check-prefix=RV32SMCTR %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-ssctr  %s -o - | FileCheck --check-prefix=RV32SSCTR %s
 
@@ -272,11 +272,11 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+a,+zabha %s -o - | FileCheck --check-prefix=RV64ZABHA %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zve32x -mattr=+experimental-zvbc32e  %s -o - | FileCheck --check-prefix=RV64ZVBC32E %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zve32x -mattr=+experimental-zvkgs  %s -o - | FileCheck --check-prefix=RV64ZVKGS %s
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-ssnpm  %s -o - | FileCheck --check-prefix=RV64SSNPM %s
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-smnpm  %s -o - | FileCheck --check-prefix=RV64SMNPM %s
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-smmpm %s -o - | FileCheck --check-prefix=RV64SMMPM %s
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-sspm %s -o - | FileCheck --check-prefix=RV64SSPM %s
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-supm %s -o - | FileCheck --check-prefix=RV64SUPM %s
+; RUN: llc -mtriple=riscv64 -mattr=+ssnpm  %s -o - | FileCheck --check-prefix=RV64SSNPM %s
+; RUN: llc -mtriple=riscv64 -mattr=+smnpm  %s -o - | FileCheck --check-prefix=RV64SMNPM %s
+; RUN: llc -mtriple=riscv64 -mattr=+smmpm %s -o - | FileCheck --check-prefix=RV64SMMPM %s
+; RUN: llc -mtriple=riscv64 -mattr=+sspm %s -o - | FileCheck --check-prefix=RV64SSPM %s
+; RUN: llc -mtriple=riscv64 -mattr=+supm %s -o - | FileCheck --check-prefix=RV64SUPM %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-smctr  %s -o - | FileCheck --check-prefix=RV64SMCTR %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-ssctr  %s -o - | FileCheck --check-prefix=RV64SSCTR %s
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/pr107950.ll b/llvm/test/CodeGen/RISCV/rvv/pr107950.ll
index 8384008c245f..9d93ed317213 100644
--- a/llvm/test/CodeGen/RISCV/rvv/pr107950.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/pr107950.ll
@@ -26,5 +26,5 @@ entry:
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(read)
 declare <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr>, i32 immarg, <vscale x 4 x i1>, <vscale x 4 x i32>) #1
 
-attributes #0 = { "target-features"="+64bit,+d,+f,+relax,+v,+xsifivecdiscarddlone,+zicsr,+zve32f,+zve32x,+zve64d,+zve64f,+zve64x,+zvl128b,+zvl32b,+zvl64b,-a,-b,-c,-e,-experimental-smctr,-experimental-smmpm,-experimental-smnpm,-experimental-ssctr,-experimental-ssnpm,-experimental-sspm,-experimental-supm,-experimental-zacas,-experimental-zalasr,-experimental-zicfilp,-experimental-zicfiss,-experimental-zvbc32e,-experimental-zvkgs,-h,-m,-shcounterenw,-shgatpa,-shtvala,-shvsatpa,-shvstvala,-shvstvecd,-smaia,-smcdeleg,-smcsrind,-smepmp,-smstateen,-ssaia,-ssccfg,-ssccptr,-sscofpmf,-sscounterenw,-sscsrind,-ssqosid,-ssstateen,-ssstrict,-sstc,-sstvala,-sstvecd,-ssu64xl,-svade,-svadu,-svbare,-svinval,-svnapot,-svpbmt,-xcvalu,-xcvbi,-xcvbitmanip,-xcvelw,-xcvmac,-xcvmem,-xcvsimd,-xsfcease,-xsfvcp,-xsfvfnrclipxfqf,-xsfvfwmaccqqq,-xsfvqmaccdod,-xsfvqmaccqoq,-xsifivecflushdlone,-xtheadba,-xtheadbb,-xtheadbs,-xtheadcmo,-xtheadcondmov,-xtheadfmemidx,-xtheadmac,-xtheadmemidx,-xtheadmempair,-xtheadsync,-xtheadvdot,-xventanacondops,-xwchc,-za128rs,-za64rs,-zaamo,-zabha,-zalrsc,-zama16b,-zawrs,-zba,-zbb,-zbc,-zbkb,-zbkc,-zbkx,-zbs,-zca,-zcb,-zcd,-zce,-zcf,-zcmop,-zcmp,-zcmt,-zdinx,-zfa,-zfbfmin,-zfh,-zfhmin,-zfinx,-zhinx,-zhinxmin,-zic64b,-zicbom,-zicbop,-zicboz,-ziccamoa,-ziccif,-zicclsm,-ziccrse,-zicntr,-zicond,-zifencei,-zihintntl,-zihintpause,-zihpm,-zimop,-zk,-zkn,-zknd,-zkne,-zknh,-zkr,-zks,-zksed,-zksh,-zkt,-zmmul,-ztso,-zvbb,-zvbc,-zvfbfmin,-zvfbfwma,-zvfh,-zvfhmin,-zvkb,-zvkg,-zvkn,-zvknc,-zvkned,-zvkng,-zvknha,-zvknhb,-zvks,-zvksc,-zvksed,-zvksg,-zvksh,-zvkt,-zvl1024b,-zvl16384b,-zvl2048b,-zvl256b,-zvl32768b,-zvl4096b,-zvl512b,-zvl65536b,-zvl8192b" }
+attributes #0 = { "target-features"="+64bit,+d,+f,+relax,+v,+xsifivecdiscarddlone,+zicsr,+zve32f,+zve32x,+zve64d,+zve64f,+zve64x,+zvl128b,+zvl32b,+zvl64b,-a,-b,-c,-e,-experimental-smctr,-smmpm,-smnpm,-experimental-ssctr,-ssnpm,-sspm,-supm,-experimental-zacas,-experimental-zalasr,-experimental-zicfilp,-experimental-zicfiss,-experimental-zvbc32e,-experimental-zvkgs,-h,-m,-shcounterenw,-shgatpa,-shtvala,-shvsatpa,-shvstvala,-shvstvecd,-smaia,-smcdeleg,-smcsrind,-smepmp,-smstateen,-ssaia,-ssccfg,-ssccptr,-sscofpmf,-sscounterenw,-sscsrind,-ssqosid,-ssstateen,-ssstrict,-sstc,-sstvala,-sstvecd,-ssu64xl,-svade,-svadu,-svbare,-svinval,-svnapot,-svpbmt,-xcvalu,-xcvbi,-xcvbitmanip,-xcvelw,-xcvmac,-xcvmem,-xcvsimd,-xsfcease,-xsfvcp,-xsfvfnrclipxfqf,-xsfvfwmaccqqq,-xsfvqmaccdod,-xsfvqmaccqoq,-xsifivecflushdlone,-xtheadba,-xtheadbb,-xtheadbs,-xtheadcmo,-xtheadcondmov,-xtheadfmemidx,-xtheadmac,-xtheadmemidx,-xtheadmempair,-xtheadsync,-xtheadvdot,-xventanacondops,-xwchc,-za128rs,-za64rs,-zaamo,-zabha,-zalrsc,-zama16b,-zawrs,-zba,-zbb,-zbc,-zbkb,-zbkc,-zbkx,-zbs,-zca,-zcb,-zcd,-zce,-zcf,-zcmop,-zcmp,-zcmt,-zdinx,-zfa,-zfbfmin,-zfh,-zfhmin,-zfinx,-zhinx,-zhinxmin,-zic64b,-zicbom,-zicbop,-zicboz,-ziccamoa,-ziccif,-zicclsm,-ziccrse,-zicntr,-zicond,-zifencei,-zihintntl,-zihintpause,-zihpm,-zimop,-zk,-zkn,-zknd,-zkne,-zknh,-zkr,-zks,-zksed,-zksh,-zkt,-zmmul,-ztso,-zvbb,-zvbc,-zvfbfmin,-zvfbfwma,-zvfh,-zvfhmin,-zvkb,-zvkg,-zvkn,-zvknc,-zvkned,-zvkng,-zvknha,-zvknhb,-zvks,-zvksc,-zvksed,-zvksg,-zvksh,-zvkt,-zvl1024b,-zvl16384b,-zvl2048b,-zvl256b,-zvl32768b,-zvl4096b,-zvl512b,-zvl65536b,-zvl8192b" }
 attributes #1 = { nocallback nofree nosync nounwind willreturn memory(read) }
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index ded43a4ff787..2e9c548ae872 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -1065,6 +1065,8 @@ R"(All available -march extensions for RISC-V
     smcdeleg             1.0
     smcsrind             1.0
     smepmp               1.0
+    smmpm                1.0
+    smnpm                1.0
     smrnmi               1.0
     smstateen            1.0
     ssaia                1.0
@@ -1073,6 +1075,8 @@ R"(All available -march extensions for RISC-V
     sscofpmf             1.0
     sscounterenw         1.0
     sscsrind             1.0
+    ssnpm                1.0
+    sspm                 1.0
     ssqosid              1.0
     ssstateen            1.0
     ssstrict             1.0
@@ -1080,6 +1084,7 @@ R"(All available -march extensions for RISC-V
     sstvala              1.0
     sstvecd              1.0
     ssu64xl              1.0
+    supm                 1.0
     svade                1.0
     svadu                1.0
     svbare               1.0
@@ -1122,12 +1127,7 @@ Experimental extensions
     zvbc32e              0.7
     zvkgs                0.7
     smctr                1.0
-    smmpm                1.0
-    smnpm                1.0
     ssctr                1.0
-    ssnpm                1.0
-    sspm                 1.0
-    supm                 1.0
 
 Supported Profiles
     rva20s64
-- 
GitLab


From 90cdc03e7f5bda2e31573d48450a8ac8fa856efa Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Fri, 25 Oct 2024 12:56:10 +0100
Subject: [PATCH 153/693] [IR] Fix undiagnosed cases of structs containing
 scalable vectors (#113455)

Type::isScalableTy and StructType::containsScalableVectorType failed to
detect some cases of structs containing scalable vectors because
containsScalableVectorType did not call back into isScalableTy to check
the element types. Fix this, which requires sharing the same Visited set
in both functions. Also change the external API so that callers are
never required to pass in a Visited set, and normalize the naming to
isScalableTy.
---
 llvm/include/llvm/IR/DerivedTypes.h           |  4 +--
 llvm/include/llvm/IR/Type.h                   |  1 +
 llvm/lib/AsmParser/LLParser.cpp               |  2 +-
 llvm/lib/IR/Type.cpp                          | 29 ++++++++-----------
 llvm/lib/IR/Verifier.cpp                      |  3 +-
 .../InstCombine/InstructionCombining.cpp      |  2 +-
 llvm/test/Verifier/scalable-global-vars.ll    | 14 +++++++++
 7 files changed, 32 insertions(+), 23 deletions(-)

diff --git a/llvm/include/llvm/IR/DerivedTypes.h b/llvm/include/llvm/IR/DerivedTypes.h
index a24801d8bdf8..820b5c0707df 100644
--- a/llvm/include/llvm/IR/DerivedTypes.h
+++ b/llvm/include/llvm/IR/DerivedTypes.h
@@ -290,8 +290,8 @@ public:
   bool isSized(SmallPtrSetImpl<Type *> *Visited = nullptr) const;
 
   /// Returns true if this struct contains a scalable vector.
-  bool
-  containsScalableVectorType(SmallPtrSetImpl<Type *> *Visited = nullptr) const;
+  bool isScalableTy(SmallPtrSetImpl<const Type *> &Visited) const;
+  using Type::isScalableTy;
 
   /// Returns true if this struct contains homogeneous scalable vector types.
   /// Note that the definition of homogeneous scalable vector type is not
diff --git a/llvm/include/llvm/IR/Type.h b/llvm/include/llvm/IR/Type.h
index 2f53197df199..d563b25d600a 100644
--- a/llvm/include/llvm/IR/Type.h
+++ b/llvm/include/llvm/IR/Type.h
@@ -206,6 +206,7 @@ public:
   bool isScalableTargetExtTy() const;
 
   /// Return true if this is a type whose size is a known multiple of vscale.
+  bool isScalableTy(SmallPtrSetImpl<const Type *> &Visited) const;
   bool isScalableTy() const;
 
   /// Return true if this is a FP type or a vector of FP.
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 6a2372c97514..8ddb2efb0e26 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -8525,7 +8525,7 @@ int LLParser::parseGetElementPtr(Instruction *&Inst, PerFunctionState &PFS) {
     return error(Loc, "base element of getelementptr must be sized");
 
   auto *STy = dyn_cast<StructType>(Ty);
-  if (STy && STy->containsScalableVectorType())
+  if (STy && STy->isScalableTy())
     return error(Loc, "getelementptr cannot target structure that contains "
                       "scalable vector type");
 
diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp
index f618263f79c3..912b1a3960ef 100644
--- a/llvm/lib/IR/Type.cpp
+++ b/llvm/lib/IR/Type.cpp
@@ -58,16 +58,19 @@ bool Type::isIntegerTy(unsigned Bitwidth) const {
   return isIntegerTy() && cast<IntegerType>(this)->getBitWidth() == Bitwidth;
 }
 
-bool Type::isScalableTy() const {
+bool Type::isScalableTy(SmallPtrSetImpl<const Type *> &Visited) const {
   if (const auto *ATy = dyn_cast<ArrayType>(this))
-    return ATy->getElementType()->isScalableTy();
-  if (const auto *STy = dyn_cast<StructType>(this)) {
-    SmallPtrSet<Type *, 4> Visited;
-    return STy->containsScalableVectorType(&Visited);
-  }
+    return ATy->getElementType()->isScalableTy(Visited);
+  if (const auto *STy = dyn_cast<StructType>(this))
+    return STy->isScalableTy(Visited);
   return getTypeID() == ScalableVectorTyID || isScalableTargetExtTy();
 }
 
+bool Type::isScalableTy() const {
+  SmallPtrSet<const Type *, 4> Visited;
+  return isScalableTy(Visited);
+}
+
 const fltSemantics &Type::getFltSemantics() const {
   switch (getTypeID()) {
   case HalfTyID: return APFloat::IEEEhalf();
@@ -394,30 +397,22 @@ StructType *StructType::get(LLVMContext &Context, ArrayRef<Type*> ETypes,
   return ST;
 }
 
-bool StructType::containsScalableVectorType(
-    SmallPtrSetImpl<Type *> *Visited) const {
+bool StructType::isScalableTy(SmallPtrSetImpl<const Type *> &Visited) const {
   if ((getSubclassData() & SCDB_ContainsScalableVector) != 0)
     return true;
 
   if ((getSubclassData() & SCDB_NotContainsScalableVector) != 0)
     return false;
 
-  if (Visited && !Visited->insert(const_cast<StructType *>(this)).second)
+  if (!Visited.insert(this).second)
     return false;
 
   for (Type *Ty : elements()) {
-    if (isa<ScalableVectorType>(Ty)) {
+    if (Ty->isScalableTy(Visited)) {
       const_cast<StructType *>(this)->setSubclassData(
           getSubclassData() | SCDB_ContainsScalableVector);
       return true;
     }
-    if (auto *STy = dyn_cast<StructType>(Ty)) {
-      if (STy->containsScalableVectorType(Visited)) {
-        const_cast<StructType *>(this)->setSubclassData(
-            getSubclassData() | SCDB_ContainsScalableVector);
-        return true;
-      }
-    }
   }
 
   // For structures that are opaque, return false but do not set the
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index f34fe7594c86..60e65392218d 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4107,8 +4107,7 @@ void Verifier::visitGetElementPtrInst(GetElementPtrInst &GEP) {
   Check(GEP.getSourceElementType()->isSized(), "GEP into unsized type!", &GEP);
 
   if (auto *STy = dyn_cast<StructType>(GEP.getSourceElementType())) {
-    SmallPtrSet<Type *, 4> Visited;
-    Check(!STy->containsScalableVectorType(&Visited),
+    Check(!STy->isScalableTy(),
           "getelementptr cannot target structure that contains scalable vector"
           "type",
           &GEP);
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index c8b9f166b160..971ace2a4f47 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -4087,7 +4087,7 @@ Instruction *InstCombinerImpl::visitExtractValueInst(ExtractValueInst &EV) {
   if (LoadInst *L = dyn_cast<LoadInst>(Agg)) {
     // Bail out if the aggregate contains scalable vector type
     if (auto *STy = dyn_cast<StructType>(Agg->getType());
-        STy && STy->containsScalableVectorType())
+        STy && STy->isScalableTy())
       return nullptr;
 
     // If the (non-volatile) load only has one use, we can rewrite this to a
diff --git a/llvm/test/Verifier/scalable-global-vars.ll b/llvm/test/Verifier/scalable-global-vars.ll
index 81882261e664..fb9a3067acba 100644
--- a/llvm/test/Verifier/scalable-global-vars.ll
+++ b/llvm/test/Verifier/scalable-global-vars.ll
@@ -15,3 +15,17 @@
 ; CHECK-NEXT: ptr @ScalableVecStructGlobal
 @ScalableVecStructGlobal = global { i32,  <vscale x 4 x i32> } zeroinitializer
 
+; CHECK-NEXT: Globals cannot contain scalable types
+; CHECK-NEXT: ptr @StructTestGlobal
+%struct.test = type { <vscale x 1 x double>, <vscale x 1 x double> }
+@StructTestGlobal = global %struct.test zeroinitializer
+
+; CHECK-NEXT: Globals cannot contain scalable types
+; CHECK-NEXT: ptr @StructArrayTestGlobal
+%struct.array.test = type { [2 x <vscale x 1 x double>] }
+@StructArrayTestGlobal = global %struct.array.test zeroinitializer
+
+; CHECK-NEXT: Globals cannot contain scalable types
+; CHECK-NEXT: ptr @StructTargetTestGlobal
+%struct.target.test = type { target("aarch64.svcount"), target("aarch64.svcount") }
+@StructTargetTestGlobal = global %struct.target.test zeroinitializer
-- 
GitLab


From 403e4a2074910fcec139a8fbb77f3c2d15643916 Mon Sep 17 00:00:00 2001
From: SpencerAbson <Spencer.Abson@arm.com>
Date: Fri, 25 Oct 2024 13:03:07 +0100
Subject: [PATCH 154/693] [AArch64] Add assembly/disassembly for SVE zeroing
 int-float conversions (#113605)

This patch adds assembly/disassembly for the following predicated SVE2.2
instructions

    - SCVTF (zeroing)
    - UCVTF (zeroing)
    - FCVTZS (zeroing)
    - FCVTZU (zeroing)
    - FLOGB (zeroing)
- In accordance with:
https://developer.arm.com/documentation/ddi0602/latest/
---
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |  9 +++
 llvm/lib/Target/AArch64/SVEInstrFormats.td    | 26 ++++++++
 llvm/test/MC/AArch64/SVE2/flogb-diagnostics.s |  2 +-
 .../MC/AArch64/SVE2p2/fcvtzs_z-diagnostics.s  | 34 ++++++++++
 llvm/test/MC/AArch64/SVE2p2/fcvtzs_z.s        | 63 +++++++++++++++++++
 .../MC/AArch64/SVE2p2/fcvtzu_z-diagnostics.s  | 34 ++++++++++
 llvm/test/MC/AArch64/SVE2p2/fcvtzu_z.s        | 63 +++++++++++++++++++
 .../MC/AArch64/SVE2p2/flogb_z-diagnostics.s   | 47 ++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/flogb_z.s         | 33 ++++++++++
 .../MC/AArch64/SVE2p2/scvtf_z-diagnostics.s   | 34 ++++++++++
 llvm/test/MC/AArch64/SVE2p2/scvtf_z.s         | 63 +++++++++++++++++++
 .../MC/AArch64/SVE2p2/ucvtf_z-diagnostics.s   | 34 ++++++++++
 llvm/test/MC/AArch64/SVE2p2/ucvtf_z.s         | 63 +++++++++++++++++++
 13 files changed, 504 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/fcvtzs_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/fcvtzs_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/fcvtzu_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/fcvtzu_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/flogb_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/flogb_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/scvtf_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/scvtf_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/ucvtf_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/ucvtf_z.s

diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 59859cb7442d..bf6e25438633 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4227,6 +4227,15 @@ let Predicates = [HasSVE2p2orSME2p2] in {
   // SVE2p2 floating-point convert single-to-bf (placing odd), zeroing predicate
   def BFCVTNT_ZPzZ      : sve_fp_fcvt2z<0b1010, "bfcvtnt", ZPR16, ZPR32>;
 
+  // Floating-point convert to integer, zeroing predicate
+  defm FCVTZS_ZPzZ : sve_fp_z2op_p_zd_d<0b0, "fcvtzs">;
+  defm FCVTZU_ZPzZ : sve_fp_z2op_p_zd_d<0b1, "fcvtzu">;
+  // Integer convert to floating-point, zeroing predicate
+  defm SCVTF_ZPzZ  : sve_fp_z2op_p_zd_c<0b0, "scvtf">;
+  defm UCVTF_ZPzZ  : sve_fp_z2op_p_zd_c<0b1, "ucvtf">;
+  // Signed integer base 2 logarithm of fp value, zeroing predicate
+  defm FLOGB_ZPzZ : sve_fp_z2op_p_zd_d_flogb<"flogb">;
+
   // Floating point round to integral fp value in integer size range
   // Merging
   defm FRINT32Z_ZPmZ : sve_fp_2op_p_zd_frint<0b00, "frint32z">;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index fc2e889d3a1a..af5c96eb5c8c 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -3181,6 +3181,32 @@ multiclass sve_fp_z2op_p_zd_frint<bits<2> opc, string asm> {
   def _D : sve_fp_z2op_p_zd<{ 0b0010, opc{1}, 1, opc{0} }, asm, ZPR64, ZPR64>;
 }
 
+multiclass sve_fp_z2op_p_zd_d<bit U, string asm> {
+  def _HtoH : sve_fp_z2op_p_zd<{ 0b011101, U }, asm, ZPR16, ZPR16>;
+  def _HtoS : sve_fp_z2op_p_zd<{ 0b011110, U }, asm, ZPR16, ZPR32>;
+  def _HtoD : sve_fp_z2op_p_zd<{ 0b011111, U }, asm, ZPR16, ZPR64>;
+  def _StoS : sve_fp_z2op_p_zd<{ 0b101110, U }, asm, ZPR32, ZPR32>;
+  def _StoD : sve_fp_z2op_p_zd<{ 0b111110, U }, asm, ZPR32, ZPR64>;
+  def _DtoS : sve_fp_z2op_p_zd<{ 0b111100, U }, asm, ZPR64, ZPR32>;
+  def _DtoD : sve_fp_z2op_p_zd<{ 0b111111, U }, asm, ZPR64, ZPR64>;
+}
+
+multiclass sve_fp_z2op_p_zd_c<bit U, string asm> {
+  def _HtoH : sve_fp_z2op_p_zd<{ 0b011001, U }, asm, ZPR16, ZPR16>;
+  def _StoH : sve_fp_z2op_p_zd<{ 0b011010, U }, asm, ZPR32, ZPR16>;
+  def _StoS : sve_fp_z2op_p_zd<{ 0b101010, U }, asm, ZPR32, ZPR32>;
+  def _StoD : sve_fp_z2op_p_zd<{ 0b111000, U }, asm, ZPR32, ZPR64>;
+  def _DtoS : sve_fp_z2op_p_zd<{ 0b111010, U }, asm, ZPR64, ZPR32>;
+  def _DtoH : sve_fp_z2op_p_zd<{ 0b011011, U }, asm, ZPR64, ZPR16>;
+  def _DtoD : sve_fp_z2op_p_zd<{ 0b111011, U }, asm, ZPR64, ZPR64>;
+}
+
+multiclass sve_fp_z2op_p_zd_d_flogb<string asm> {
+  def _H : sve_fp_z2op_p_zd<0b0011001, asm, ZPR16, ZPR16>;
+  def _S : sve_fp_z2op_p_zd<0b0011010, asm, ZPR32, ZPR32>;
+  def _D : sve_fp_z2op_p_zd<0b0011011, asm, ZPR64, ZPR64>;
+}
+
 //===----------------------------------------------------------------------===//
 // SVE Integer Arithmetic - Binary Predicated Group
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AArch64/SVE2/flogb-diagnostics.s b/llvm/test/MC/AArch64/SVE2/flogb-diagnostics.s
index ddb8c4ff35b6..60d9f9e5e242 100644
--- a/llvm/test/MC/AArch64/SVE2/flogb-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE2/flogb-diagnostics.s
@@ -14,7 +14,7 @@ flogb   z0.b, p0/m, z0.b
 // Invalid predicate operation
 
 flogb   z0.s, p0/z, z0.s
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: flogb   z0.s, p0/z, z0.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE2p2/fcvtzs_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/fcvtzs_z-diagnostics.s
new file mode 100644
index 000000000000..1408cba4070b
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/fcvtzs_z-diagnostics.s
@@ -0,0 +1,34 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+fcvtzs    z0.h, p0/z, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fcvtzs    z0.h, p0/z, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fcvtzs    z0.h, p0/z, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fcvtzs    z0.h, p0/z, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+fcvtzs    z0.h, p8/z, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: fcvtzs    z0.h, p8/z, z0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.d, p0/z, z7.d
+fcvtzs z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: fcvtzs z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+fcvtzs z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: fcvtzs z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/fcvtzs_z.s b/llvm/test/MC/AArch64/SVE2p2/fcvtzs_z.s
new file mode 100644
index 000000000000..a37f83c0f97b
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/fcvtzs_z.s
@@ -0,0 +1,63 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sve - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// convert from half
+
+fcvtzs  z0.h, p0/z, z0.h  // 01100100-01011110-11000000-00000000
+// CHECK-INST: fcvtzs  z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xc0,0x5e,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 645ec000 <unknown>
+
+fcvtzs  z23.s, p3/z, z13.h  // 01100100-01011111-10001101-10110111
+// CHECK-INST: fcvtzs  z23.s, p3/z, z13.h
+// CHECK-ENCODING: [0xb7,0x8d,0x5f,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 645f8db7 <unknown>
+
+fcvtzs  z31.d, p7/z, z31.h  // 01100100-01011111-11011111-11111111
+// CHECK-INST: fcvtzs  z31.d, p7/z, z31.h
+// CHECK-ENCODING: [0xff,0xdf,0x5f,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 645fdfff <unknown>
+
+// convert from single
+
+fcvtzs  z0.s, p0/z, z0.s  // 01100100-10011111-10000000-00000000
+// CHECK-INST: fcvtzs  z0.s, p0/z, z0.s
+// CHECK-ENCODING: [0x00,0x80,0x9f,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 649f8000 <unknown>
+
+fcvtzs  z21.d, p5/z, z10.s  // 01100100-11011111-10010101-01010101
+// CHECK-INST: fcvtzs  z21.d, p5/z, z10.s
+// CHECK-ENCODING: [0x55,0x95,0xdf,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64df9555 <unknown>
+
+// convert from double
+
+fcvtzs  z23.s, p3/z, z13.d  // 01100100-11011110-10001101-10110111
+// CHECK-INST: fcvtzs  z23.s, p3/z, z13.d
+// CHECK-ENCODING: [0xb7,0x8d,0xde,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64de8db7 <unknown>
+
+fcvtzs  z31.d, p7/z, z31.d  // 01100100-11011111-11011111-11111111
+// CHECK-INST: fcvtzs  z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xdf,0xdf,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64dfdfff <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2p2/fcvtzu_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/fcvtzu_z-diagnostics.s
new file mode 100644
index 000000000000..fc4ecda82bd2
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/fcvtzu_z-diagnostics.s
@@ -0,0 +1,34 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+fcvtzu    z0.h, p0/z, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fcvtzu    z0.h, p0/z, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fcvtzu    z0.h, p0/z, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fcvtzu    z0.h, p0/z, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+fcvtzu    z0.h, p8/z, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: fcvtzu    z0.h, p8/z, z0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.d, p0/z, z7.d
+fcvtzu z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: fcvtzu z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+fcvtzu z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: fcvtzu z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/fcvtzu_z.s b/llvm/test/MC/AArch64/SVE2p2/fcvtzu_z.s
new file mode 100644
index 000000000000..df1ac4016689
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/fcvtzu_z.s
@@ -0,0 +1,63 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sve - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// convert from half
+
+fcvtzu  z0.h, p0/z, z0.h  // 01100100-01011110-11100000-00000000
+// CHECK-INST: fcvtzu  z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xe0,0x5e,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 645ee000 <unknown>
+
+fcvtzu  z21.s, p5/z, z10.h  // 01100100-01011111-10110101-01010101
+// CHECK-INST: fcvtzu  z21.s, p5/z, z10.h
+// CHECK-ENCODING: [0x55,0xb5,0x5f,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 645fb555 <unknown>
+
+fcvtzu  z23.d, p3/z, z13.h  // 01100100-01011111-11101101-10110111
+// CHECK-INST: fcvtzu  z23.d, p3/z, z13.h
+// CHECK-ENCODING: [0xb7,0xed,0x5f,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 645fedb7 <unknown>
+
+// convert from single
+
+fcvtzu  z21.s, p5/z, z10.s  // 01100100-10011111-10110101-01010101
+// CHECK-INST: fcvtzu  z21.s, p5/z, z10.s
+// CHECK-ENCODING: [0x55,0xb5,0x9f,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 649fb555 <unknown>
+
+fcvtzu  z31.d, p7/z, z31.s  // 01100100-11011111-10111111-11111111
+// CHECK-INST: fcvtzu  z31.d, p7/z, z31.s
+// CHECK-ENCODING: [0xff,0xbf,0xdf,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64dfbfff <unknown>
+
+// convert from double
+
+fcvtzu  z0.s, p0/z, z0.d  // 01100100-11011110-10100000-00000000
+// CHECK-INST: fcvtzu  z0.s, p0/z, z0.d
+// CHECK-ENCODING: [0x00,0xa0,0xde,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64dea000 <unknown>
+
+fcvtzu  z31.d, p7/z, z31.d  // 01100100-11011111-11111111-11111111
+// CHECK-INST: fcvtzu  z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xff,0xdf,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64dfffff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/flogb_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/flogb_z-diagnostics.s
new file mode 100644
index 000000000000..8fd528e1fc05
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/flogb_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2  2>&1 < %s| FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid element width
+
+flogb   z0.b, p0/z, z0.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: flogb   z0.b, p0/z, z0.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+flogb   z0.h, p0/z, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: flogb   z0.h, p0/z, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+flogb   z0.s, p0/z, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: flogb   z0.s, p0/z, z0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+flogb   z0.d, p0/z, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: flogb   z0.d, p0/z, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+flogb    z0.h, p8/z, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: flogb    z0.h, p8/z, z0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.d, p0/z, z7.d
+flogb z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: flogb z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+flogb z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: flogb z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/flogb_z.s b/llvm/test/MC/AArch64/SVE2p2/flogb_z.s
new file mode 100644
index 000000000000..1b056aa928ce
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/flogb_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sve - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+flogb   z0.h, p0/z, z0.h  // 01100100-00011110-10100000-00000000
+// CHECK-INST: flogb   z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xa0,0x1e,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641ea000 <unknown>
+
+flogb   z23.s, p3/z, z13.s  // 01100100-00011110-11001101-10110111
+// CHECK-INST: flogb   z23.s, p3/z, z13.s
+// CHECK-ENCODING: [0xb7,0xcd,0x1e,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641ecdb7 <unknown>
+
+flogb   z31.d, p7/z, z31.d  // 01100100-00011110-11111111-11111111
+// CHECK-INST: flogb   z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xff,0x1e,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641effff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/scvtf_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/scvtf_z-diagnostics.s
new file mode 100644
index 000000000000..9dd089271462
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/scvtf_z-diagnostics.s
@@ -0,0 +1,34 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+scvtf    z0.s, p0/z, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: scvtf    z0.s, p0/z, z0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+scvtf    z0.d, p0/z, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: scvtf    z0.d, p0/z, z0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+scvtf    z0.h, p8/z, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: scvtf    z0.h, p8/z, z0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.d, p0/z, z7.d
+scvtf z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: scvtf z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+scvtf z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: scvtf z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE2p2/scvtf_z.s b/llvm/test/MC/AArch64/SVE2p2/scvtf_z.s
new file mode 100644
index 000000000000..b8898c6485f6
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/scvtf_z.s
@@ -0,0 +1,63 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sve - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// convert to half
+
+scvtf   z0.h, p0/z, z0.h  // 01100100-01011100-11000000-00000000
+// CHECK-INST: scvtf   z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xc0,0x5c,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 645cc000 <unknown>
+
+scvtf   z21.h, p5/z, z10.s  // 01100100-01011101-10010101-01010101
+// CHECK-INST: scvtf   z21.h, p5/z, z10.s
+// CHECK-ENCODING: [0x55,0x95,0x5d,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 645d9555 <unknown>
+
+scvtf   z31.h, p7/z, z31.d  // 01100100-01011101-11011111-11111111
+// CHECK-INST: scvtf   z31.h, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xdf,0x5d,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 645ddfff <unknown>
+
+// convert to single
+
+scvtf   z0.s, p0/z, z0.s  // 01100100-10011101-10000000-00000000
+// CHECK-INST: scvtf   z0.s, p0/z, z0.s
+// CHECK-ENCODING: [0x00,0x80,0x9d,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 649d8000 <unknown>
+
+scvtf   z23.s, p3/z, z13.d  // 01100100-11011101-10001101-10110111
+// CHECK-INST: scvtf   z23.s, p3/z, z13.d
+// CHECK-ENCODING: [0xb7,0x8d,0xdd,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64dd8db7 <unknown>
+
+// convert to double
+
+scvtf   z21.d, p5/z, z10.s  // 01100100-11011100-10010101-01010101
+// CHECK-INST: scvtf   z21.d, p5/z, z10.s
+// CHECK-ENCODING: [0x55,0x95,0xdc,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64dc9555 <unknown>
+
+scvtf   z31.d, p7/z, z31.d  // 01100100-11011101-11011111-11111111
+// CHECK-INST: scvtf   z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xdf,0xdd,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64dddfff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/ucvtf_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/ucvtf_z-diagnostics.s
new file mode 100644
index 000000000000..1317428dafde
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/ucvtf_z-diagnostics.s
@@ -0,0 +1,34 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+ucvtf    z0.s, p0/z, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: ucvtf    z0.s, p0/z, z0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ucvtf    z0.d, p0/z, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: ucvtf    z0.d, p0/z, z0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+ucvtf    z0.h, p8/z, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: ucvtf    z0.h, p8/z, z0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.d, p0/z, z7.d
+ucvtf z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: ucvtf z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+ucvtf z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: ucvtf z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE2p2/ucvtf_z.s b/llvm/test/MC/AArch64/SVE2p2/ucvtf_z.s
new file mode 100644
index 000000000000..9e87afd60b40
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/ucvtf_z.s
@@ -0,0 +1,63 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sve - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// convert to half
+
+ucvtf   z0.h, p0/z, z0.h  // 01100100-01011100-11100000-00000000
+// CHECK-INST: ucvtf   z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xe0,0x5c,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 645ce000 <unknown>
+
+ucvtf   z21.h, p5/z, z10.s  // 01100100-01011101-10110101-01010101
+// CHECK-INST: ucvtf   z21.h, p5/z, z10.s
+// CHECK-ENCODING: [0x55,0xb5,0x5d,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 645db555 <unknown>
+
+ucvtf   z31.h, p7/z, z31.d  // 01100100-01011101-11111111-11111111
+// CHECK-INST: ucvtf   z31.h, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xff,0x5d,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 645dffff <unknown>
+
+// convert to single
+
+ucvtf   z23.s, p3/z, z13.s  // 01100100-10011101-10101101-10110111
+// CHECK-INST: ucvtf   z23.s, p3/z, z13.s
+// CHECK-ENCODING: [0xb7,0xad,0x9d,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 649dadb7 <unknown>
+
+ucvtf   z21.s, p5/z, z10.d  // 01100100-11011101-10110101-01010101
+// CHECK-INST: ucvtf   z21.s, p5/z, z10.d
+// CHECK-ENCODING: [0x55,0xb5,0xdd,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64ddb555 <unknown>
+
+// convert to double
+
+ucvtf   z0.d, p0/z, z0.s  // 01100100-11011100-10100000-00000000
+// CHECK-INST: ucvtf   z0.d, p0/z, z0.s
+// CHECK-ENCODING: [0x00,0xa0,0xdc,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64dca000 <unknown>
+
+ucvtf   z31.d, p7/z, z31.d  // 01100100-11011101-11111111-11111111
+// CHECK-INST: ucvtf   z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xff,0xdd,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64ddffff <unknown>
\ No newline at end of file
-- 
GitLab


From 183ddefff60f560466ec3bf271c6008067d746db Mon Sep 17 00:00:00 2001
From: Ivan Kosarev <ivan.kosarev@amd.com>
Date: Fri, 25 Oct 2024 15:06:32 +0300
Subject: [PATCH 155/693] [MC][ELF] Have an assert catching creating group
 sections without signatures. (#113553)

---
 llvm/include/llvm/MC/MCSectionELF.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/include/llvm/MC/MCSectionELF.h b/llvm/include/llvm/MC/MCSectionELF.h
index d43ffbd885c9..f09d30591a3c 100644
--- a/llvm/include/llvm/MC/MCSectionELF.h
+++ b/llvm/include/llvm/MC/MCSectionELF.h
@@ -62,6 +62,8 @@ private:
                   type == ELF::SHT_NOBITS, Begin),
         Type(type), Flags(flags), UniqueID(UniqueID), EntrySize(entrySize),
         Group(group, IsComdat), LinkedToSym(LinkedToSym) {
+    assert((!(Flags & ELF::SHF_GROUP) || Group.getPointer()) &&
+           "Group section without signature!");
     if (Group.getPointer())
       Group.getPointer()->setIsSignature();
   }
-- 
GitLab


From 43dda4bbeb59c9b327b9b5b1fe917a7d8a5bf3bc Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Fri, 25 Oct 2024 13:08:39 +0100
Subject: [PATCH 156/693] [clang][doc] Add release note for changes to
 `-fveclib={ArmPL,SLEEF}` (#113673)

Changed in #112580.
---
 clang/docs/ReleaseNotes.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index ed0c0e369fca..9e1558d8acc9 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -302,6 +302,11 @@ Modified Compiler Flags
   the ``promoted`` algorithm for complex division when possible rather than the
   less basic (limited range) algorithm.
 
+- The ``-fveclib`` option has been updated to enable ``-fno-math-errno`` for
+  ``-fveclib=ArmPL`` and ``-fveclib=SLEEF``. This gives Clang more opportunities
+  to utilize these vector libraries. The behavior for all other vector function
+  libraries remains unchanged.
+
 Removed Compiler Flags
 -------------------------
 
-- 
GitLab


From 6854ad90e39e9d119c990043f573db7157a2b097 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <benny.kra@googlemail.com>
Date: Fri, 25 Oct 2024 14:22:11 +0200
Subject: [PATCH 157/693] [bazel][lldb] "Fix" the build after
 b852fb1ec5fa15f0b913cc4988cbd09239b19904

b852fb1ec5fa15f0b913cc4988cbd09239b19904 split out ValueObject, but it's
still all a big pile of dependency spaghetti so just build it with Core.
---
 utils/bazel/llvm-project-overlay/lldb/BUILD.bazel | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
index 9dba2efc34f6..91eb04db3ee9 100644
--- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
@@ -607,7 +607,10 @@ gentbl_cc_library(
 
 cc_library(
     name = "CoreHeaders",
-    hdrs = glob(["include/lldb/Core/**/*.h"]),
+    hdrs = glob([
+        "include/lldb/Core/**/*.h",
+        "include/lldb/ValueObject/**/*.h",  # This should be its own library.
+    ]),
     strip_include_prefix = "include",
     deps = [
         ":BreakpointHeaders",
@@ -627,8 +630,14 @@ cc_library(
 
 cc_library(
     name = "Core",
-    srcs = glob(["source/Core/**/*.cpp"]),
-    hdrs = glob(["include/lldb/Core/**/*.h"]),
+    srcs = glob([
+        "source/Core/**/*.cpp",
+        "source/ValueObject/**/*.cpp",  # This should be its own library.
+    ]),
+    hdrs = glob([
+        "include/lldb/Core/**/*.h",
+        "include/lldb/ValueObject/**/*.h",  # This should be its own library.
+    ]),
     strip_include_prefix = "include",
     deps = [
         ":BreakpointHeaders",
-- 
GitLab


From 32baf2917357b4e5a566ff43c6e0156e5ad10cb8 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <benny.kra@googlemail.com>
Date: Fri, 25 Oct 2024 14:48:38 +0200
Subject: [PATCH 158/693] Make NoopLattice.h self-contained

---
 clang/include/clang/Analysis/FlowSensitive/NoopLattice.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang/include/clang/Analysis/FlowSensitive/NoopLattice.h b/clang/include/clang/Analysis/FlowSensitive/NoopLattice.h
index 0938091cd689..96c695473b67 100644
--- a/clang/include/clang/Analysis/FlowSensitive/NoopLattice.h
+++ b/clang/include/clang/Analysis/FlowSensitive/NoopLattice.h
@@ -14,6 +14,7 @@
 #define LLVM_CLANG_ANALYSIS_FLOWSENSITIVE_NOOP_LATTICE_H
 
 #include "clang/Analysis/FlowSensitive/DataflowLattice.h"
+#include "clang/Support/Compiler.h"
 #include "llvm/ADT/Any.h"
 #include <ostream>
 
-- 
GitLab


From 49940514e2c26ad82d86abee59ba4c2f6d8ec07d Mon Sep 17 00:00:00 2001
From: CarolineConcatto <caroline.concatto@arm.com>
Date: Fri, 25 Oct 2024 13:59:46 +0100
Subject: [PATCH 159/693] [CLANG][AArch64] Add the  modal 8 bit floating-point
 scalar type (#97277)

ARM ACLE PR#323[1] adds new modal types for 8-bit floating point
intrinsic.

From the PR#323:
```
ACLE defines the `__mfp8` type, which can be used for the E5M2 and E4M3
8-bit floating-point formats. It is a storage and interchange only type
with no arithmetic operations other than intrinsic calls.
````

The type should be an opaque type and its format in undefined in Clang.
Only defined in the backend by a status/format register, for AArch64 the
FPMR.

This patch is an attempt to the add the mfloat8_t scalar type. It has a
parser and codegen for the new scalar type.

The patch it is lowering to and 8bit unsigned as it has no format. But
maybe we should add another opaque type.

[1]  https://github.com/ARM-software/acle/pull/323
---
 .../clang/Basic/AArch64SVEACLETypes.def       |  2 +-
 .../include/clang/Serialization/ASTBitCodes.h |  2 +-
 clang/lib/CodeGen/CGDebugInfo.cpp             |  7 ++
 clang/test/AST/arm-mfp8.cpp                   | 91 +++++++++++++++++++
 clang/test/CodeGen/aarch64-debug-types.c      |  9 ++
 clang/test/CodeGen/arm-mfp8.c                 | 34 +++++++
 clang/test/Modules/no-external-type-id.cppm   |  2 +-
 clang/test/Sema/arm-mfp8.c                    | 11 +++
 clang/test/Sema/arm-mfp8.cpp                  | 34 ++++++-
 clang/utils/TableGen/NeonEmitter.cpp          |  1 +
 10 files changed, 189 insertions(+), 4 deletions(-)
 create mode 100644 clang/test/AST/arm-mfp8.cpp
 create mode 100644 clang/test/CodeGen/aarch64-debug-types.c
 create mode 100644 clang/test/Sema/arm-mfp8.c

diff --git a/clang/include/clang/Basic/AArch64SVEACLETypes.def b/clang/include/clang/Basic/AArch64SVEACLETypes.def
index 25abf5f3f86b..62f6087e9624 100644
--- a/clang/include/clang/Basic/AArch64SVEACLETypes.def
+++ b/clang/include/clang/Basic/AArch64SVEACLETypes.def
@@ -107,7 +107,6 @@
   AARCH64_VECTOR_TYPE(Name, MangledName, Id, SingletonId)
 #endif
 
-
 //===- Vector point types -----------------------------------------------===//
 
 SVE_VECTOR_TYPE_INT("__SVInt8_t",  "__SVInt8_t",  SveInt8,  SveInt8Ty, 16,  8, 1, true)
@@ -201,6 +200,7 @@ SVE_PREDICATE_TYPE_ALL("__clang_svboolx4_t", "svboolx4_t", SveBoolx4, SveBoolx4T
 
 SVE_OPAQUE_TYPE("__SVCount_t", "__SVCount_t", SveCount, SveCountTy)
 
+AARCH64_VECTOR_TYPE_MFLOAT("__MFloat8_t", "__MFloat8_t", MFloat8, MFloat8Ty, 1, 8, 1)
 AARCH64_VECTOR_TYPE_MFLOAT("__MFloat8x8_t", "__MFloat8x8_t", MFloat8x8, MFloat8x8Ty, 8, 8, 1)
 AARCH64_VECTOR_TYPE_MFLOAT("__MFloat8x16_t", "__MFloat8x16_t", MFloat8x16, MFloat8x16Ty, 16, 8, 1)
 
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 13173dc96e71..99232fd21357 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1149,7 +1149,7 @@ enum PredefinedTypeIDs {
 ///
 /// Type IDs for non-predefined types will start at
 /// NUM_PREDEF_TYPE_IDs.
-const unsigned NUM_PREDEF_TYPE_IDS = 511;
+const unsigned NUM_PREDEF_TYPE_IDS = 512;
 
 // Ensure we do not overrun the predefined types we reserved
 // in the enum PredefinedTypeIDs above.
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 27bbbfc6f531..59a761c2303c 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -783,6 +783,13 @@ llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) {
 #define SVE_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
 #include "clang/Basic/AArch64SVEACLETypes.def"
     {
+      if (BT->getKind() == BuiltinType::MFloat8) {
+        Encoding = llvm::dwarf::DW_ATE_unsigned_char;
+        BTName = BT->getName(CGM.getLangOpts());
+        // Bit size and offset of the type.
+        uint64_t Size = CGM.getContext().getTypeSize(BT);
+        return DBuilder.createBasicType(BTName, Size, Encoding);
+      }
       ASTContext::BuiltinVectorTypeInfo Info =
           // For svcount_t, only the lower 2 bytes are relevant.
           BT->getKind() == BuiltinType::SveCount
diff --git a/clang/test/AST/arm-mfp8.cpp b/clang/test/AST/arm-mfp8.cpp
new file mode 100644
index 000000000000..a00d055f7d96
--- /dev/null
+++ b/clang/test/AST/arm-mfp8.cpp
@@ -0,0 +1,91 @@
+// RUN: %clang_cc1 -std=c++11 -triple aarch64-arm-none-eabi -target-feature -fp8 -ast-dump %s | \
+// RUN:  FileCheck %s --strict-whitespace
+
+// REQUIRES: aarch64-registered-target || arm-registered-target
+
+/*  Various contexts where type __mfp8 can appear. */
+
+#include<arm_neon.h>
+/*  Namespace */
+namespace {
+  __mfp8 f2n;
+  __mfp8 arr1n[10];
+}
+
+//CHECK:       |-NamespaceDecl {{.*}}
+//CHECK-NEXT:  | |-VarDecl {{.*}} f2n '__mfp8':'__MFloat8_t'
+//CHECK-NEXT:  | `-VarDecl {{.*}} arr1n '__mfp8[10]'
+
+
+  const __mfp8 func1n(const __mfp8 mfp8) {
+    // this should fail
+    __mfp8 f1n;
+    f1n  = mfp8;
+    return f1n;
+  }
+//CHECK:    |-FunctionDecl {{.*}} func1n 'const __mfp8 (const __mfp8)'
+//CHECK:            | `-VarDecl {{.*}} f1n '__mfp8':'__MFloat8_t'
+//CHECK-NEXT:       |-BinaryOperator {{.*}} '__mfp8':'__MFloat8_t' lvalue '='
+//CHECK-NEXT:       | |-DeclRefExpr {{.*}} '__mfp8':'__MFloat8_t' lvalue Var {{.*}} 'f1n' '__mfp8':'__MFloat8_t'
+//CHECK-NEXT:       | `-ImplicitCastExpr {{.*}} '__mfp8':'__MFloat8_t' <LValueToRValue>
+//CHECK-NEXT:       |   `-DeclRefExpr {{.*}} 'const __mfp8':'const __MFloat8_t' lvalue ParmVar {{.*}} 'mfp8' 'const __mfp8':'const __MFloat8_t'
+//CHECK-NEXT:        `-ReturnStmt {{.*}}
+//CHECK-NEXT:         `-ImplicitCastExpr {{.*}} '__mfp8':'__MFloat8_t' <LValueToRValue>
+//CHECK-NEXT:           `-DeclRefExpr {{.*}} '__mfp8':'__MFloat8_t' lvalue Var {{.*}} 'f1n' '__mfp8':'__MFloat8_t'
+
+
+/* Class */
+
+class C1 {
+  __mfp8 f1c;
+  static const __mfp8 f2c;
+  volatile __MFloat8_t f3c;
+public:
+  C1(__mfp8 arg) : f1c(arg), f3c(arg) { }
+  __mfp8 func1c(__mfp8 arg ) {
+    return  arg;
+  }
+  static __mfp8 func2c(__mfp8 arg) {
+    return arg;
+  }
+};
+
+//CHECK:       | |-CXXRecordDecl {{.*}} referenced class C1
+//CHECK-NEXT:  | |-FieldDecl {{.*}} f1c '__mfp8':'__MFloat8_t'
+//CHECK-NEXT:  | |-VarDecl {{.*}} f2c 'const __mfp8':'const __MFloat8_t' static
+//CHECK-NEXT:  | |-FieldDecl {{.*}} f3c 'volatile __MFloat8_t'
+//CHECK-NEXT:  | |-AccessSpecDecl {{.*}}
+//CHECK-NEXT:  | |-CXXConstructorDecl {{.*}} C1 'void (__mfp8)' implicit-inline
+//CHECK-NEXT:  | | |-ParmVarDecl {{.*}} arg '__mfp8':'__MFloat8_t'
+//CHECK-NEXT:  | | |-CXXCtorInitializer {{.*}} 'f1c' '__mfp8':'__MFloat8_t'
+//CHECK-NEXT:  | | | `-ImplicitCastExpr {{.*}} '__mfp8':'__MFloat8_t' <LValueToRValue>
+//CHECK-NEXT:  | | |   `-DeclRefExpr {{.*}} '__mfp8':'__MFloat8_t' lvalue ParmVar {{.*}} 'arg' '__mfp8':'__MFloat8_t'
+//CHECK-NEXT:  | | |-CXXCtorInitializer {{.*}} 'f3c' 'volatile __MFloat8_t'
+//CHECK-NEXT:  | | | `-ImplicitCastExpr {{.*}} '__mfp8':'__MFloat8_t' <LValueToRValue>
+//CHECK-NEXT:  | | |   `-DeclRefExpr {{.*}} '__mfp8':'__MFloat8_t' lvalue ParmVar {{.*}} 'arg' '__mfp8':'__MFloat8_t'
+//CHECK-NEXT:  | | `-CompoundStmt {{.*}}
+//CHECK-NEXT:  | |-CXXMethodDecl {{.*}} func1c '__mfp8 (__mfp8)' implicit-inline
+//CHECK-NEXT:  | | |-ParmVarDecl {{.*}} arg '__mfp8':'__MFloat8_t'
+//CHECK-NEXT:  | | `-CompoundStmt {{.*}}
+//CHECK-NEXT:  | |   `-ReturnStmt {{.*}}
+//CHECK-NEXT:  | |     `-ImplicitCastExpr {{.*}} '__mfp8':'__MFloat8_t' <LValueToRValue>
+//CHECK-NEXT:  | |       `-DeclRefExpr {{.*}} '__mfp8':'__MFloat8_t' lvalue ParmVar {{.*}}8 'arg' '__mfp8':'__MFloat8_t'
+//CHECK-NEXT:  | `-CXXMethodDecl {{.*}} func2c '__mfp8 (__mfp8)' static implicit-inline
+//CHECK-NEXT:  |   |-ParmVarDecl {{.*}} arg '__mfp8':'__MFloat8_t'
+//CHECK-NEXT:  |   `-CompoundStmt {{.*}}
+//CHECK-NEXT:  |     `-ReturnStmt {{.*}}
+//CHECK-NEXT:  |       `-ImplicitCastExpr {{.*}} '__mfp8':'__MFloat8_t' <LValueToRValue>
+//CHECK-NEXT:  |         `-DeclRefExpr {{.*}} '__mfp8':'__MFloat8_t' lvalue ParmVar {{.*}} 'arg' '__mfp8':'__MFloat8_t'
+
+template <class C> struct S1 {
+  C mem1;
+};
+
+template <> struct S1<__mfp8> {
+  __mfp8 mem2;
+};
+
+//CHECK:       |-TemplateArgument type '__MFloat8_t'
+//CHECK-NEXT:  | `-BuiltinType {{.*}} '__MFloat8_t'
+//CHECK-NEXT:  |-CXXRecordDecl {{.*}} implicit struct S1
+//CHECK-NEXT:  `-FieldDecl {{.*}} mem2 '__mfp8':'__MFloat8_t'
diff --git a/clang/test/CodeGen/aarch64-debug-types.c b/clang/test/CodeGen/aarch64-debug-types.c
new file mode 100644
index 000000000000..c109610023ed
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-debug-types.c
@@ -0,0 +1,9 @@
+// RUN:  %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +neon  -target-feature +fp8 \
+// RUN:  -emit-llvm -o - %s -debug-info-kind=limited 2>&1 | FileCheck %s
+#include<arm_neon.h>
+
+void test_locals(void) {
+  // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "__MFloat8_t", {{.*}}, baseType: ![[ELTTYU8:[0-9]+]]
+  // CHECK-DAG: ![[ELTTYU8]] = !DIBasicType(name: "__MFloat8_t", size: 8, encoding: DW_ATE_unsigned_char)
+  __MFloat8_t mfp8;
+}
diff --git a/clang/test/CodeGen/arm-mfp8.c b/clang/test/CodeGen/arm-mfp8.c
index 35ec24c8a788..8c817fd5be1c 100644
--- a/clang/test/CodeGen/arm-mfp8.c
+++ b/clang/test/CodeGen/arm-mfp8.c
@@ -47,5 +47,39 @@ mfloat8x8_t test_ret_mfloat8x8_t(mfloat8x8_t v) {
   return v;
 }
 
+// CHECK-C-LABEL: define dso_local <1 x i8> @func1n(
+// CHECK-C-SAME: <1 x i8> [[MFP8:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[MFP8_ADDR:%.*]] = alloca <1 x i8>, align 1
+// CHECK-C-NEXT:    [[F1N:%.*]] = alloca [10 x <1 x i8>], align 1
+// CHECK-C-NEXT:    store <1 x i8> [[MFP8]], ptr [[MFP8_ADDR]], align 1
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <1 x i8>, ptr [[MFP8_ADDR]], align 1
+// CHECK-C-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x <1 x i8>], ptr [[F1N]], i64 0, i64 2
+// CHECK-C-NEXT:    store <1 x i8> [[TMP0]], ptr [[ARRAYIDX]], align 1
+// CHECK-C-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [10 x <1 x i8>], ptr [[F1N]], i64 0, i64 2
+// CHECK-C-NEXT:    [[TMP1:%.*]] = load <1 x i8>, ptr [[ARRAYIDX1]], align 1
+// CHECK-C-NEXT:    ret <1 x i8> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <1 x i8> @_Z6func1nu11__MFloat8_t(
+// CHECK-CXX-SAME: <1 x i8> [[MFP8:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    [[MFP8_ADDR:%.*]] = alloca <1 x i8>, align 1
+// CHECK-CXX-NEXT:    [[F1N:%.*]] = alloca [10 x <1 x i8>], align 1
+// CHECK-CXX-NEXT:    store <1 x i8> [[MFP8]], ptr [[MFP8_ADDR]], align 1
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = load <1 x i8>, ptr [[MFP8_ADDR]], align 1
+// CHECK-CXX-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x <1 x i8>], ptr [[F1N]], i64 0, i64 2
+// CHECK-CXX-NEXT:    store <1 x i8> [[TMP0]], ptr [[ARRAYIDX]], align 1
+// CHECK-CXX-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [10 x <1 x i8>], ptr [[F1N]], i64 0, i64 2
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = load <1 x i8>, ptr [[ARRAYIDX1]], align 1
+// CHECK-CXX-NEXT:    ret <1 x i8> [[TMP1]]
+//
+__mfp8 func1n(__mfp8 mfp8) {
+  __mfp8 f1n[10];
+  f1n[2] = mfp8;
+  return f1n[2];
+}
+
+
+
 //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 // CHECK: {{.*}}
diff --git a/clang/test/Modules/no-external-type-id.cppm b/clang/test/Modules/no-external-type-id.cppm
index b8b987403812..6385f3a8aa00 100644
--- a/clang/test/Modules/no-external-type-id.cppm
+++ b/clang/test/Modules/no-external-type-id.cppm
@@ -23,7 +23,7 @@ export module b;
 import a;
 export int b();
 
-// CHECK: <DECL_FUNCTION {{.*}} op8=4104
+// CHECK: <DECL_FUNCTION {{.*}} op8=4112
 // CHECK: <TYPE_FUNCTION_PROTO
 
 //--- a.v1.cppm
diff --git a/clang/test/Sema/arm-mfp8.c b/clang/test/Sema/arm-mfp8.c
new file mode 100644
index 000000000000..e917078f50ed
--- /dev/null
+++ b/clang/test/Sema/arm-mfp8.c
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -triple aarch64-arm-none-eabi -target-feature -fp8 %s
+
+// REQUIRES: aarch64-registered-target
+#include<arm_neon.h>
+__mfp8 test_cast_from_float(unsigned in) {
+  return (__mfp8)in; // expected-error {{used type '__mfp8' (aka '__MFloat8_t') where arithmetic or pointer type is required}}
+}
+
+unsigned test_cast_to_int(__mfp8 in) {
+  return (unsigned)in; // expected-error {{operand of type '__mfp8' (aka '__MFloat8_t') where arithmetic or pointer type is required}}
+}
diff --git a/clang/test/Sema/arm-mfp8.cpp b/clang/test/Sema/arm-mfp8.cpp
index f270168faceb..e882c382522c 100644
--- a/clang/test/Sema/arm-mfp8.cpp
+++ b/clang/test/Sema/arm-mfp8.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -verify=sve,neon -triple aarch64-arm-none-eabi \
+// RUN: %clang_cc1 -fsyntax-only -verify=sve,neon,scalar -triple aarch64-arm-none-eabi \
 // RUN: -target-feature -fp8 -target-feature +sve  -target-feature +neon %s
 
 // REQUIRES: aarch64-registered-target
@@ -29,3 +29,35 @@ void test_vector(mfloat8x8_t a, mfloat8x16_t b, uint8x8_t c) {
   c * b;  // neon-error {{cannot convert between vector and non-scalar values ('uint8x8_t' (vector of 8 'uint8_t' values) and 'mfloat8x16_t' (aka '__MFloat8x16_t'))}}
   c / b;  // neon-error {{cannot convert between vector and non-scalar values ('uint8x8_t' (vector of 8 'uint8_t' values) and 'mfloat8x16_t' (aka '__MFloat8x16_t'))}}
 }
+__mfp8 test_static_cast_from_char(char in) {
+  return static_cast<__mfp8>(in); // scalar-error {{static_cast from 'char' to '__mfp8' (aka '__MFloat8_t') is not allowed}}
+}
+
+char test_static_cast_to_char(__mfp8 in) {
+  return static_cast<char>(in); // scalar-error {{static_cast from '__mfp8' (aka '__MFloat8_t') to 'char' is not allowed}}
+}
+void test(bool b) {
+  __mfp8 mfp8;
+
+  mfp8 + mfp8;  // scalar-error {{invalid operands to binary expression ('__mfp8' (aka '__MFloat8_t') and '__mfp8')}}
+  mfp8 - mfp8;  // scalar-error {{invalid operands to binary expression ('__mfp8' (aka '__MFloat8_t') and '__mfp8')}}
+  mfp8 * mfp8;  // scalar-error {{invalid operands to binary expression ('__mfp8' (aka '__MFloat8_t') and '__mfp8')}}
+  mfp8 / mfp8;  // scalar-error {{invalid operands to binary expression ('__mfp8' (aka '__MFloat8_t') and '__mfp8')}}
+  ++mfp8;       // scalar-error {{cannot increment value of type '__mfp8' (aka '__MFloat8_t')}}
+  --mfp8;       // scalar-error {{cannot decrement value of type '__mfp8' (aka '__MFloat8_t')}}
+
+  char u8;
+
+  mfp8 + u8;   // scalar-error {{invalid operands to binary expression ('__mfp8' (aka '__MFloat8_t') and 'char')}}
+  u8 + mfp8;   // scalar-error {{invalid operands to binary expression ('char' and '__mfp8' (aka '__MFloat8_t'))}}
+  mfp8 - u8;   // scalar-error {{invalid operands to binary expression ('__mfp8' (aka '__MFloat8_t') and 'char')}}
+  u8 - mfp8;   // scalar-error {{invalid operands to binary expression ('char' and '__mfp8' (aka '__MFloat8_t'))}}
+  mfp8 * u8;   // scalar-error {{invalid operands to binary expression ('__mfp8' (aka '__MFloat8_t') and 'char')}}
+  u8 * mfp8;   // scalar-error {{invalid operands to binary expression ('char' and '__mfp8' (aka '__MFloat8_t'))}}
+  mfp8 / u8;   // scalar-error {{invalid operands to binary expression ('__mfp8' (aka '__MFloat8_t') and 'char')}}
+  u8 / mfp8;   // scalar-error {{invalid operands to binary expression ('char' and '__mfp8' (aka '__MFloat8_t'))}}
+  mfp8 = u8;   // scalar-error {{assigning to '__mfp8' (aka '__MFloat8_t') from incompatible type 'char'}}
+  u8 = mfp8;   // scalar-error {{assigning to 'char' from incompatible type '__mfp8' (aka '__MFloat8_t')}}
+  mfp8 + (b ? u8 : mfp8);  // scalar-error {{incompatible operand types ('char' and '__mfp8' (aka '__MFloat8_t'))}}
+}
+
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index 35c8fcf69910..8caba774649a 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -2588,6 +2588,7 @@ void NeonEmitter::runVectorTypes(raw_ostream &OS) {
   OS << "typedef __fp16 float16_t;\n";
 
   OS << "#if defined(__aarch64__) || defined(__arm64ec__)\n";
+  OS << "typedef __MFloat8_t __mfp8;\n";
   OS << "typedef __MFloat8x8_t mfloat8x8_t;\n";
   OS << "typedef __MFloat8x16_t mfloat8x16_t;\n";
   OS << "typedef double float64_t;\n";
-- 
GitLab


From a69d2a18d207947a25838dd01d2116bee384b75b Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Fri, 25 Oct 2024 09:05:27 -0400
Subject: [PATCH 160/693] [gn] try to port b1be21394e9c more

Bots are currently failing check-hwasan with

    llvm-lit: .../llvm/utils/lit/lit/llvm/subst.py:133: fatal: Did not find
        hwasan_symbolize in /.../out/gn/lib/clang/20/lib/linux

Maybe this fixes that.

See also llvm/utils/gn/secondary/compiler-rt/lib/hwasan/scripts/BUILD.gn.
---
 llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
index 7194c47b1bec..520f3b6c0166 100644
--- a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
@@ -55,7 +55,7 @@ write_cmake_config("lit_common_configured") {
     "COMPILER_RT_ENABLE_INTERNAL_SYMBOLIZER_PYBOOL=False",
     "COMPILER_RT_HAS_NO_DEFAULT_CONFIG_FLAG_PYBOOL=True",
     "COMPILER_RT_INTERCEPT_LIBDISPATCH_PYBOOL=False",
-    "COMPILER_RT_RESOLVED_EXEC_OUTPUT_DIR=" + rebase_path(crt_current_out_dir),
+    "COMPILER_RT_RESOLVED_EXEC_OUTPUT_DIR=" + rebase_path("$root_out_dir/bin"),
     "COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR=" +
         rebase_path(crt_current_out_dir),
     "COMPILER_RT_RESOLVED_OUTPUT_DIR=" + rebase_path(crt_current_out_dir),
-- 
GitLab


From 4a6b56960f445d111adc9aef799acad8c6ca41f0 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Fri, 25 Oct 2024 09:19:21 -0400
Subject: [PATCH 161/693] [gn] try to port b1be21394e9c even more

Bots are now failing check-hwasan with

    llvm-lit: .../llvm/utils/lit/lit/llvm/subst.py:133: fatal: Did not find
        hwasan_symbolize in .../out/gn/stage2_unix/bin

This time it's the right fix for sure!
---
 llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
index 520f3b6c0166..020f3e7d9acd 100644
--- a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
@@ -55,7 +55,8 @@ write_cmake_config("lit_common_configured") {
     "COMPILER_RT_ENABLE_INTERNAL_SYMBOLIZER_PYBOOL=False",
     "COMPILER_RT_HAS_NO_DEFAULT_CONFIG_FLAG_PYBOOL=True",
     "COMPILER_RT_INTERCEPT_LIBDISPATCH_PYBOOL=False",
-    "COMPILER_RT_RESOLVED_EXEC_OUTPUT_DIR=" + rebase_path("$root_out_dir/bin"),
+    "COMPILER_RT_RESOLVED_EXEC_OUTPUT_DIR=" +
+        rebase_path("$root_build_dir/bin"),
     "COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR=" +
         rebase_path(crt_current_out_dir),
     "COMPILER_RT_RESOLVED_OUTPUT_DIR=" + rebase_path(crt_current_out_dir),
-- 
GitLab


From 83e7e6a0ff9363da89d8917937407f9d37caec83 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 25 Oct 2024 13:20:14 +0000
Subject: [PATCH 162/693] [gn build] Port 2e43a304f10f

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 0a2552f4b061..1630c8004d31 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -954,6 +954,14 @@ if (current_toolchain == default_toolchain) {
       "__utility/to_underlying.h",
       "__utility/unreachable.h",
       "__variant/monostate.h",
+      "__vector/comparison.h",
+      "__vector/container_traits.h",
+      "__vector/erase.h",
+      "__vector/pmr.h",
+      "__vector/swap.h",
+      "__vector/vector.h",
+      "__vector/vector_bool.h",
+      "__vector/vector_bool_formatter.h",
       "__verbose_abort",
       "algorithm",
       "any",
-- 
GitLab


From a3dd6000a732d0b9a7019abd0dba5613786b9ff4 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 25 Oct 2024 13:20:15 +0000
Subject: [PATCH 163/693] [gn build] Port c4248fa3edd3

---
 llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
index 1bbec962ff68..0a97bcf59112 100644
--- a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
@@ -245,6 +245,8 @@ copy("Headers") {
     "mmintrin.h",
     "module.modulemap",
     "movdirintrin.h",
+    "movrs_avx10_2_512intrin.h",
+    "movrs_avx10_2intrin.h",
     "msa.h",
     "mwaitxintrin.h",
     "nmmintrin.h",
-- 
GitLab


From b3703fa50485cf90b04105e6a223ccdd1e29c9af Mon Sep 17 00:00:00 2001
From: Caroline Concatto <caroline.concatto@arm.com>
Date: Fri, 25 Oct 2024 13:26:58 +0000
Subject: [PATCH 164/693] [AArch64]Update test aarch64-debug-types.c

This patch fix the failing tests by adding
REQUIRES: aarch64-registered-target

This tests was failing in non aarch64 cpu.
The test was introduced by:
 [CLANG][AArch64] Add the  modal 8 bit floating-point scalar type (#97277)
---
 clang/test/CodeGen/aarch64-debug-types.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/clang/test/CodeGen/aarch64-debug-types.c b/clang/test/CodeGen/aarch64-debug-types.c
index c109610023ed..f1ab74c5c31b 100644
--- a/clang/test/CodeGen/aarch64-debug-types.c
+++ b/clang/test/CodeGen/aarch64-debug-types.c
@@ -1,5 +1,8 @@
 // RUN:  %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +neon  -target-feature +fp8 \
 // RUN:  -emit-llvm -o - %s -debug-info-kind=limited 2>&1 | FileCheck %s
+
+// REQUIRES: aarch64-registered-target
+
 #include<arm_neon.h>
 
 void test_locals(void) {
-- 
GitLab


From 9043bdbce4ab1c6f07e72ddfedf7165bdf2b3e40 Mon Sep 17 00:00:00 2001
From: Boaz Brickner <brickner@google.com>
Date: Fri, 25 Oct 2024 15:31:57 +0200
Subject: [PATCH 165/693] [clang] Output an error when [[lifetimebound]]
 attribute is applied on a function parameter while the function returns void
 (#113460)

Fixes: https://github.com/llvm/llvm-project/issues/107556
---
 clang/docs/ReleaseNotes.rst                      |  9 +++++++++
 clang/include/clang/Basic/DiagnosticSemaKinds.td |  3 +++
 clang/lib/Sema/SemaDecl.cpp                      | 14 +++++++++++++-
 clang/test/SemaCXX/attr-lifetimebound.cpp        | 12 ++++++++++--
 4 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 9e1558d8acc9..170c4cc28053 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -133,6 +133,15 @@ C++ Specific Potentially Breaking Changes
     // Fixed version:
     unsigned operator""_udl_name(unsigned long long);
 
+- Clang will now produce an error diagnostic when [[clang::lifetimebound]] is
+  applied on a parameter of a function that returns void. This was previously 
+  ignored and had no effect. (#GH107556)
+
+  .. code-block:: c++
+
+    // Now diagnoses with an error.
+    void f(int& i [[clang::lifetimebound]]);
+
 ABI Changes in This Version
 ---------------------------
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 8e4718008ece..9b9bdd7c800e 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -10097,6 +10097,9 @@ def err_lifetimebound_no_object_param : Error<
 def err_lifetimebound_ctor_dtor : Error<
   "'lifetimebound' attribute cannot be applied to a "
   "%select{constructor|destructor}0">;
+def err_lifetimebound_void_return_type : Error<
+  "'lifetimebound' attribute cannot be applied to a parameter of a function "
+  "that returns void">;
 
 // CHECK: returning address/reference of stack memory
 def warn_ret_stack_addr_ref : Warning<
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 25061f02c13f..f8e5f3c6d309 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -6940,7 +6940,7 @@ static void checkAttributesAfterMerging(Sema &S, NamedDecl &ND) {
     }
   }
 
-  // Check the attributes on the function type, if any.
+  // Check the attributes on the function type and function params, if any.
   if (const auto *FD = dyn_cast<FunctionDecl>(&ND)) {
     // Don't declare this variable in the second operand of the for-statement;
     // GCC miscompiles that by ending its lifetime before evaluating the
@@ -6970,6 +6970,18 @@ static void checkAttributesAfterMerging(Sema &S, NamedDecl &ND) {
         }
       }
     }
+
+    for (unsigned int I = 0; I < FD->getNumParams(); ++I) {
+      const ParmVarDecl *P = FD->getParamDecl(I);
+
+      // The [[lifetimebound]] attribute can be applied to a function parameter
+      // only if the function returns a value.
+      if (auto *A = P->getAttr<LifetimeBoundAttr>()) {
+        if (!isa<CXXConstructorDecl>(FD) && FD->getReturnType()->isVoidType()) {
+          S.Diag(A->getLocation(), diag::err_lifetimebound_void_return_type);
+        }
+      }
+    }
   }
 }
 
diff --git a/clang/test/SemaCXX/attr-lifetimebound.cpp b/clang/test/SemaCXX/attr-lifetimebound.cpp
index 1c5c79777c71..804d61fb62ca 100644
--- a/clang/test/SemaCXX/attr-lifetimebound.cpp
+++ b/clang/test/SemaCXX/attr-lifetimebound.cpp
@@ -1,8 +1,7 @@
 // RUN: %clang_cc1 -std=c++23 -verify %s
 
 namespace usage_invalid {
-  // FIXME: Should we diagnose a void return type?
-  void voidreturn(int &param [[clang::lifetimebound]]);
+  void void_return(int &param [[clang::lifetimebound]]); // expected-error {{'lifetimebound' attribute cannot be applied to a parameter of a function that returns void}}
 
   int *not_class_member() [[clang::lifetimebound]]; // expected-error {{non-member function has no implicit object parameter}}
   struct A {
@@ -12,6 +11,8 @@ namespace usage_invalid {
     int *explicit_object(this A&) [[clang::lifetimebound]]; // expected-error {{explicit object member function has no implicit object parameter}}
     int not_function [[clang::lifetimebound]]; // expected-error {{only applies to parameters and implicit object parameters}}
     int [[clang::lifetimebound]] also_not_function; // expected-error {{cannot be applied to types}}
+    // FIXME: Should diagnose a void return type.
+    void void_return_member() [[clang::lifetimebound]];
   };
   int *attr_with_param(int &param [[clang::lifetimebound(42)]]); // expected-error {{takes no arguments}}
 }
@@ -31,6 +32,13 @@ namespace usage_ok {
     return *(int*)param;
   }
 
+  template <class T, class R = void> R dependent_void(const T& t [[clang::lifetimebound]]);
+  void dependent_void_instantiation() {
+    dependent_void<int>(1); // OK: Returns void.
+    int x = dependent_void<int, int>(1); // expected-warning {{temporary whose address is used as value of local variable 'x' will be destroyed at the end of the full-expression}}
+    dependent_void<int, int>(1); // OK: Returns an unused value.
+  }
+
   struct A {
     A();
     A(int);
-- 
GitLab


From 21ecd4a9dff312b26f7a276dc78d1ddb3707e7c1 Mon Sep 17 00:00:00 2001
From: Anatoly Trosinenko <atrosinenko@accesssoftek.com>
Date: Fri, 25 Oct 2024 17:03:15 +0300
Subject: [PATCH 166/693] [AArch64][PAC] Factor out the emission of pointer
 check sequence (NFC) (#110702)

When pointer is authenticated or resigned, it may be required to
explicitly check the authenticated value to prevent introducing signing
or authentication oracles. While the check sequence is expensive in
general, a more efficient sequence can be emitted under specific
assumptions.

This commit factors out the emission of the code sequence to check the
authenticated pointer value in preparation for adding other variants
of checking code, as it is currently done when emitting tail calls.
---
 llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 168 ++++++++++--------
 1 file changed, 91 insertions(+), 77 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index a982ea67a0f2..6d2dd0ecbccf 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -150,6 +150,12 @@ public:
   // Emit the sequence for BRA/BLRA (authenticate + branch/call).
   void emitPtrauthBranch(const MachineInstr *MI);
 
+  void emitPtrauthCheckAuthenticatedValue(Register TestedReg,
+                                          Register ScratchReg,
+                                          AArch64PACKey::ID Key,
+                                          bool ShouldTrap,
+                                          const MCSymbol *OnFailure);
+
   // Emit the sequence for AUT or AUTPAC.
   void emitPtrauthAuthResign(const MachineInstr *MI);
 
@@ -1719,45 +1725,37 @@ unsigned AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
   return AArch64::X17;
 }
 
-void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
-  const bool IsAUTPAC = MI->getOpcode() == AArch64::AUTPAC;
-
-  // We can expand AUT/AUTPAC into 3 possible sequences:
-  // - unchecked:
-  //      autia x16, x0
-  //      pacib x16, x1 ; if AUTPAC
+/// Emits a code sequence to check an authenticated pointer value.
+///
+/// If OnFailure argument is passed, jump there on check failure instead
+/// of proceeding to the next instruction (only if ShouldTrap is false).
+void AArch64AsmPrinter::emitPtrauthCheckAuthenticatedValue(
+    Register TestedReg, Register ScratchReg, AArch64PACKey::ID Key,
+    bool ShouldTrap, const MCSymbol *OnFailure) {
+  // Insert a sequence to check if authentication of TestedReg succeeded,
+  // such as:
   //
   // - checked and clearing:
-  //      mov x17, x0
-  //      movk x17, #disc, lsl #48
-  //      autia x16, x17
+  //      ; x16 is TestedReg, x17 is ScratchReg
   //      mov x17, x16
   //      xpaci x17
   //      cmp x16, x17
   //      b.eq Lsuccess
   //      mov x16, x17
   //      b Lend
-  //     Lsuccess:
-  //      mov x17, x1
-  //      movk x17, #disc, lsl #48
-  //      pacib x16, x17
-  //     Lend:
-  //   Where we only emit the AUT if we started with an AUT.
+  //    Lsuccess:
+  //      ; skipped if authentication failed
+  //    Lend:
+  //      ...
   //
   // - checked and trapping:
-  //      mov x17, x0
-  //      movk x17, #disc, lsl #48
-  //      autia x16, x0
   //      mov x17, x16
   //      xpaci x17
   //      cmp x16, x17
   //      b.eq Lsuccess
   //      brk #<0xc470 + aut key>
-  //     Lsuccess:
-  //      mov x17, x1
-  //      movk x17, #disc, lsl #48
-  //      pacib x16, x17 ; if AUTPAC
-  //   Where the b.eq skips over the trap if the PAC is valid.
+  //    Lsuccess:
+  //      ...
   //
   // This sequence is expensive, but we need more information to be able to
   // do better.
@@ -1770,6 +1768,71 @@ void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
   // Either way, we also don't always know whether TBI is enabled or not for
   // the specific target environment.
 
+  unsigned XPACOpc = getXPACOpcodeForKey(Key);
+
+  MCSymbol *SuccessSym = createTempSymbol("auth_success_");
+
+  //  mov Xscratch, Xtested
+  emitMovXReg(ScratchReg, TestedReg);
+
+  //  xpac(i|d) Xscratch
+  EmitToStreamer(MCInstBuilder(XPACOpc).addReg(ScratchReg).addReg(ScratchReg));
+
+  //  cmp Xtested, Xscratch
+  EmitToStreamer(MCInstBuilder(AArch64::SUBSXrs)
+                     .addReg(AArch64::XZR)
+                     .addReg(TestedReg)
+                     .addReg(ScratchReg)
+                     .addImm(0));
+
+  //  b.eq Lsuccess
+  EmitToStreamer(MCInstBuilder(AArch64::Bcc)
+                     .addImm(AArch64CC::EQ)
+                     .addExpr(MCSymbolRefExpr::create(SuccessSym, OutContext)));
+
+  if (ShouldTrap) {
+    assert(!OnFailure && "Cannot specify OnFailure with ShouldTrap");
+    // Trapping sequences do a 'brk'.
+    //  brk #<0xc470 + aut key>
+    EmitToStreamer(MCInstBuilder(AArch64::BRK).addImm(0xc470 | Key));
+  } else {
+    // Non-trapping checked sequences return the stripped result in TestedReg,
+    // skipping over success-only code (such as re-signing the pointer) if
+    // there is one.
+    // Note that this can introduce an authentication oracle (such as based on
+    // the high bits of the re-signed value).
+
+    // FIXME: Can we simply return the AUT result, already in TestedReg?
+    //  mov Xtested, Xscratch
+    emitMovXReg(TestedReg, ScratchReg);
+
+    if (OnFailure) {
+      //  b Lend
+      EmitToStreamer(
+          MCInstBuilder(AArch64::B)
+              .addExpr(MCSymbolRefExpr::create(OnFailure, OutContext)));
+    }
+  }
+
+  // If the auth check succeeds, we can continue.
+  // Lsuccess:
+  OutStreamer->emitLabel(SuccessSym);
+}
+
+void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
+  const bool IsAUTPAC = MI->getOpcode() == AArch64::AUTPAC;
+
+  // We expand AUT/AUTPAC into a sequence of the form
+  //
+  //      ; authenticate x16
+  //      ; check pointer in x16
+  //    Lsuccess:
+  //      ; sign x16 (if AUTPAC)
+  //    Lend:   ; if not trapping on failure
+  //
+  // with the checking sequence chosen depending on whether we should check
+  // the pointer and whether we should trap on failure.
+
   // By default, auth/resign sequences check for auth failures.
   bool ShouldCheck = true;
   // In the checked sequence, we only trap if explicitly requested.
@@ -1800,8 +1863,6 @@ void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
   uint64_t AUTDisc = MI->getOperand(1).getImm();
   unsigned AUTAddrDisc = MI->getOperand(2).getReg();
 
-  unsigned XPACOpc = getXPACOpcodeForKey(AUTKey);
-
   // Compute aut discriminator into x17
   assert(isUInt<16>(AUTDisc));
   unsigned AUTDiscReg = emitPtrauthDiscriminator(AUTDisc, AUTAddrDisc);
@@ -1824,59 +1885,12 @@ void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
 
   MCSymbol *EndSym = nullptr;
 
-  // Checked sequences do an additional strip-and-compare.
   if (ShouldCheck) {
-    MCSymbol *SuccessSym = createTempSymbol("auth_success_");
-
-    // XPAC has tied src/dst: use x17 as a temporary copy.
-    //  mov x17, x16
-    emitMovXReg(AArch64::X17, AArch64::X16);
-
-    //  xpaci x17
-    EmitToStreamer(
-        *OutStreamer,
-        MCInstBuilder(XPACOpc).addReg(AArch64::X17).addReg(AArch64::X17));
-
-    //  cmp x16, x17
-    EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSXrs)
-                                     .addReg(AArch64::XZR)
-                                     .addReg(AArch64::X16)
-                                     .addReg(AArch64::X17)
-                                     .addImm(0));
-
-    //  b.eq Lsuccess
-    EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::Bcc)
-                                     .addImm(AArch64CC::EQ)
-                                     .addExpr(MCSymbolRefExpr::create(
-                                         SuccessSym, OutContext)));
-
-    if (ShouldTrap) {
-      // Trapping sequences do a 'brk'.
-      //  brk #<0xc470 + aut key>
-      EmitToStreamer(*OutStreamer,
-                     MCInstBuilder(AArch64::BRK).addImm(0xc470 | AUTKey));
-    } else {
-      // Non-trapping checked sequences return the stripped result in x16,
-      // skipping over the PAC if there is one.
-
-      // FIXME: can we simply return the AUT result, already in x16? without..
-      //        ..traps this is usable as an oracle anyway, based on high bits
-      //  mov x17, x16
-      emitMovXReg(AArch64::X16, AArch64::X17);
-
-      if (IsAUTPAC) {
-        EndSym = createTempSymbol("resign_end_");
-
-        //  b Lend
-        EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B)
-                                         .addExpr(MCSymbolRefExpr::create(
-                                             EndSym, OutContext)));
-      }
-    }
+    if (IsAUTPAC && !ShouldTrap)
+      EndSym = createTempSymbol("resign_end_");
 
-    // If the auth check succeeds, we can continue.
-    // Lsuccess:
-    OutStreamer->emitLabel(SuccessSym);
+    emitPtrauthCheckAuthenticatedValue(AArch64::X16, AArch64::X17, AUTKey,
+                                       ShouldTrap, EndSym);
   }
 
   // We already emitted unchecked and checked-but-non-trapping AUTs.
-- 
GitLab


From 800a47d6cd33ea1c2a888ceb67d566366c61e7ed Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Fri, 25 Oct 2024 10:17:16 -0400
Subject: [PATCH 167/693] [libc++][NFC] Fix include guards inside
 locale_base_api

---
 libcxx/include/__locale_dir/locale_base_api/android.h       | 6 +++---
 .../__locale_dir/locale_base_api/bsd_locale_defaults.h      | 6 +++---
 .../__locale_dir/locale_base_api/bsd_locale_fallbacks.h     | 6 +++---
 libcxx/include/__locale_dir/locale_base_api/fuchsia.h       | 6 +++---
 libcxx/include/__locale_dir/locale_base_api/ibm.h           | 6 +++---
 libcxx/include/__locale_dir/locale_base_api/locale_guard.h  | 6 +++---
 libcxx/include/__locale_dir/locale_base_api/musl.h          | 6 +++---
 libcxx/include/__locale_dir/locale_base_api/newlib.h        | 6 +++---
 libcxx/include/__locale_dir/locale_base_api/openbsd.h       | 6 +++---
 libcxx/include/__locale_dir/locale_base_api/win32.h         | 6 +++---
 10 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/libcxx/include/__locale_dir/locale_base_api/android.h b/libcxx/include/__locale_dir/locale_base_api/android.h
index 9965d8bbf6a2..08ef5407dedf 100644
--- a/libcxx/include/__locale_dir/locale_base_api/android.h
+++ b/libcxx/include/__locale_dir/locale_base_api/android.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_ANDROID_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_ANDROID_H
+#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_ANDROID_H
+#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_ANDROID_H
 
 #include <stdlib.h>
 
@@ -47,4 +47,4 @@ inline _LIBCPP_HIDE_FROM_ABI double strtod_l(const char* __nptr, char** __endptr
 #  endif // __NDK_MAJOR__ <= 16
 #endif   // __has_include(<android/ndk-version.h>)
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_ANDROID_H
+#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_ANDROID_H
diff --git a/libcxx/include/__locale_dir/locale_base_api/bsd_locale_defaults.h b/libcxx/include/__locale_dir/locale_base_api/bsd_locale_defaults.h
index 1f9607209842..e88eb4fa41d7 100644
--- a/libcxx/include/__locale_dir/locale_base_api/bsd_locale_defaults.h
+++ b/libcxx/include/__locale_dir/locale_base_api/bsd_locale_defaults.h
@@ -11,8 +11,8 @@
 // we will define the mapping from an internal macro to the real BSD symbol.
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_DEFAULTS_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_DEFAULTS_H
+#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_BSD_LOCALE_DEFAULTS_H
+#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_BSD_LOCALE_DEFAULTS_H
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -33,4 +33,4 @@
 #define __libcpp_asprintf_l(...) asprintf_l(__VA_ARGS__)
 #define __libcpp_sscanf_l(...) sscanf_l(__VA_ARGS__)
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_DEFAULTS_H
+#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_BSD_LOCALE_DEFAULTS_H
diff --git a/libcxx/include/__locale_dir/locale_base_api/bsd_locale_fallbacks.h b/libcxx/include/__locale_dir/locale_base_api/bsd_locale_fallbacks.h
index 76b94287cd6c..5f99c7aea02a 100644
--- a/libcxx/include/__locale_dir/locale_base_api/bsd_locale_fallbacks.h
+++ b/libcxx/include/__locale_dir/locale_base_api/bsd_locale_fallbacks.h
@@ -10,8 +10,8 @@
 // of those functions for non-BSD platforms.
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
+#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
+#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
 
 #include <__locale_dir/locale_base_api/locale_guard.h>
 #include <cstdio>
@@ -123,4 +123,4 @@ inline _LIBCPP_ATTRIBUTE_FORMAT(__scanf__, 3, 4) int __libcpp_sscanf_l(
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
+#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
diff --git a/libcxx/include/__locale_dir/locale_base_api/fuchsia.h b/libcxx/include/__locale_dir/locale_base_api/fuchsia.h
index 4c3440f981c6..f6ef454ba7ad 100644
--- a/libcxx/include/__locale_dir/locale_base_api/fuchsia.h
+++ b/libcxx/include/__locale_dir/locale_base_api/fuchsia.h
@@ -7,12 +7,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_FUCHSIA_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_FUCHSIA_H
+#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_FUCHSIA_H
+#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_FUCHSIA_H
 
 #include <__support/xlocale/__posix_l_fallback.h>
 #include <__support/xlocale/__strtonum_fallback.h>
 #include <cstdlib>
 #include <cwchar>
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_FUCHSIA_H
+#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_FUCHSIA_H
diff --git a/libcxx/include/__locale_dir/locale_base_api/ibm.h b/libcxx/include/__locale_dir/locale_base_api/ibm.h
index fa3bc1c3633f..1d1d15df9f79 100644
--- a/libcxx/include/__locale_dir/locale_base_api/ibm.h
+++ b/libcxx/include/__locale_dir/locale_base_api/ibm.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_IBM_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_IBM_H
+#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_IBM_H
+#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_IBM_H
 
 #if defined(__MVS__)
 #  include <__support/ibm/locale_mgmt_zos.h>
@@ -105,4 +105,4 @@ _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 2, 0) int vasprintf(char** strp, const char
   return str_size;
 }
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_IBM_H
+#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_IBM_H
diff --git a/libcxx/include/__locale_dir/locale_base_api/locale_guard.h b/libcxx/include/__locale_dir/locale_base_api/locale_guard.h
index 2baacb51cd06..7d15f2d253ad 100644
--- a/libcxx/include/__locale_dir/locale_base_api/locale_guard.h
+++ b/libcxx/include/__locale_dir/locale_base_api/locale_guard.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_LOCALE_GUARD_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_LOCALE_GUARD_H
+#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_LOCALE_GUARD_H
+#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_LOCALE_GUARD_H
 
 #include <__config>
 #include <__locale> // for locale_t
@@ -75,4 +75,4 @@ struct __libcpp_locale_guard {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_LOCALE_GUARD_H
+#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_LOCALE_GUARD_H
diff --git a/libcxx/include/__locale_dir/locale_base_api/musl.h b/libcxx/include/__locale_dir/locale_base_api/musl.h
index bf7b849d5863..1653214cdba1 100644
--- a/libcxx/include/__locale_dir/locale_base_api/musl.h
+++ b/libcxx/include/__locale_dir/locale_base_api/musl.h
@@ -14,8 +14,8 @@
 // in Musl.
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_MUSL_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_MUSL_H
+#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_MUSL_H
+#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_MUSL_H
 
 #include <cstdlib>
 #include <cwchar>
@@ -28,4 +28,4 @@ inline _LIBCPP_HIDE_FROM_ABI unsigned long long strtoull_l(const char* __nptr, c
   return ::strtoull(__nptr, __endptr, __base);
 }
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_MUSL_H
+#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_MUSL_H
diff --git a/libcxx/include/__locale_dir/locale_base_api/newlib.h b/libcxx/include/__locale_dir/locale_base_api/newlib.h
index a8c1cff16e6d..7da10e588984 100644
--- a/libcxx/include/__locale_dir/locale_base_api/newlib.h
+++ b/libcxx/include/__locale_dir/locale_base_api/newlib.h
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_NEWLIB_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_NEWLIB_H
+#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_NEWLIB_H
+#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_NEWLIB_H
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_NEWLIB_H
+#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_NEWLIB_H
diff --git a/libcxx/include/__locale_dir/locale_base_api/openbsd.h b/libcxx/include/__locale_dir/locale_base_api/openbsd.h
index 0c05d6a0f788..d4fb224e0c80 100644
--- a/libcxx/include/__locale_dir/locale_base_api/openbsd.h
+++ b/libcxx/include/__locale_dir/locale_base_api/openbsd.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_OPENBSD_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_OPENBSD_H
+#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_OPENBSD_H
+#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_OPENBSD_H
 
 #include <__support/xlocale/__strtonum_fallback.h>
 #include <clocale>
@@ -16,4 +16,4 @@
 #include <ctype.h>
 #include <cwctype>
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_OPENBSD_H
+#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_OPENBSD_H
diff --git a/libcxx/include/__locale_dir/locale_base_api/win32.h b/libcxx/include/__locale_dir/locale_base_api/win32.h
index f66baffb6920..f488a0dc0d69 100644
--- a/libcxx/include/__locale_dir/locale_base_api/win32.h
+++ b/libcxx/include/__locale_dir/locale_base_api/win32.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_WIN32_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_WIN32_H
+#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_WIN32_H
+#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_WIN32_H
 
 #include <__config>
 #include <cstddef>
@@ -232,4 +232,4 @@ _LIBCPP_EXPORTED_FROM_ABI int vasprintf_l(char** __ret, locale_t __loc, const ch
 // not-so-pressing FIXME: use locale to determine blank characters
 inline int iswblank_l(wint_t __c, locale_t /*loc*/) { return (__c == L' ' || __c == L'\t'); }
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_WIN32_H
+#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_WIN32_H
-- 
GitLab


From 577c7dd7cc4c5a9f62f9654cfa30ee9d55709426 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Fri, 25 Oct 2024 15:20:24 +0100
Subject: [PATCH 168/693] [AArch64] Add a phase-ordering test for vectorizing
 predicated selects. NFC

---
 .../AArch64/predicated-reduction.ll           | 294 ++++++++++++++++++
 1 file changed, 294 insertions(+)
 create mode 100644 llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll

diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll
new file mode 100644
index 000000000000..7274e9525676
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll
@@ -0,0 +1,294 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes="default<O3>" -S < %s  | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64"
+
+define nofpclass(nan inf) double @monte_simple(i32 noundef %nblocks, i32 noundef %RAND_BLOCK_LENGTH, ptr noundef %samples, double noundef nofpclass(nan inf) %Y, double noundef nofpclass(nan inf) %Z) {
+; CHECK-LABEL: define nofpclass(nan inf) double @monte_simple(
+; CHECK-SAME: i32 noundef [[NBLOCKS:%.*]], i32 noundef [[RAND_BLOCK_LENGTH:%.*]], ptr nocapture noundef readonly [[SAMPLES:%.*]], double noundef nofpclass(nan inf) [[Y:%.*]], double noundef nofpclass(nan inf) [[Z:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[RAND_BLOCK_LENGTH]], 0
+; CHECK-NEXT:    br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[RAND_BLOCK_LENGTH]] to i64
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[V1_011:%.*]] = phi double [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ], [ [[V1_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[V0_010:%.*]] = phi double [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ], [ [[V0_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SAMPLES]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP0]] to double
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[Y]], [[CONV]]
+; CHECK-NEXT:    [[SUB:%.*]] = fsub fast double [[MUL]], [[Z]]
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp fast ogt double [[SUB]], 0.000000e+00
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast double [[SUB]], [[V0_010]]
+; CHECK-NEXT:    [[MUL3:%.*]] = fmul fast double [[SUB]], [[SUB]]
+; CHECK-NEXT:    [[ADD4:%.*]] = fadd fast double [[MUL3]], [[V1_011]]
+; CHECK-NEXT:    [[V0_1]] = select i1 [[CMP1]], double [[ADD]], double [[V0_010]]
+; CHECK-NEXT:    [[V1_1]] = select i1 [[CMP1]], double [[ADD4]], double [[V1_011]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END_LOOPEXIT:.*]], label %[[FOR_BODY]]
+; CHECK:       [[FOR_END_LOOPEXIT]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd fast double [[V1_1]], [[V0_1]]
+; CHECK-NEXT:    br label %[[FOR_END]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    [[ADD5:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP1]], %[[FOR_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret double [[ADD5]]
+;
+entry:
+  %nblocks.addr = alloca i32, align 4
+  %RAND_BLOCK_LENGTH.addr = alloca i32, align 4
+  %samples.addr = alloca ptr, align 8
+  %Y.addr = alloca double, align 8
+  %Z.addr = alloca double, align 8
+  %i = alloca i32, align 4
+  %block = alloca i32, align 4
+  %rngVal = alloca double, align 8
+  %callValue = alloca double, align 8
+  %v0 = alloca double, align 8
+  %v1 = alloca double, align 8
+  store i32 %nblocks, ptr %nblocks.addr, align 4
+  store i32 %RAND_BLOCK_LENGTH, ptr %RAND_BLOCK_LENGTH.addr, align 4
+  store ptr %samples, ptr %samples.addr, align 8
+  store double %Y, ptr %Y.addr, align 8
+  store double %Z, ptr %Z.addr, align 8
+  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2
+  call void @llvm.lifetime.start.p0(i64 4, ptr %block) #2
+  call void @llvm.lifetime.start.p0(i64 8, ptr %rngVal) #2
+  call void @llvm.lifetime.start.p0(i64 8, ptr %callValue) #2
+  call void @llvm.lifetime.start.p0(i64 8, ptr %v0) #2
+  store double 0.000000e+00, ptr %v0, align 8
+  call void @llvm.lifetime.start.p0(i64 8, ptr %v1) #2
+  store double 0.000000e+00, ptr %v1, align 8
+  store i32 0, ptr %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, ptr %i, align 4
+  %1 = load i32, ptr %RAND_BLOCK_LENGTH.addr, align 4
+  %cmp = icmp slt i32 %0, %1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %2 = load ptr, ptr %samples.addr, align 8
+  %3 = load i32, ptr %i, align 4
+  %idxprom = sext i32 %3 to i64
+  %arrayidx = getelementptr inbounds float, ptr %2, i64 %idxprom
+  %4 = load float, ptr %arrayidx, align 4
+  %conv = fpext float %4 to double
+  store double %conv, ptr %rngVal, align 8
+  %5 = load double, ptr %Y.addr, align 8
+  %6 = load double, ptr %rngVal, align 8
+  %mul = fmul fast double %5, %6
+  %7 = load double, ptr %Z.addr, align 8
+  %sub = fsub fast double %mul, %7
+  store double %sub, ptr %callValue, align 8
+  %8 = load double, ptr %callValue, align 8
+  %cmp1 = fcmp fast ogt double %8, 0.000000e+00
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %9 = load double, ptr %callValue, align 8
+  %10 = load double, ptr %v0, align 8
+  %add = fadd fast double %10, %9
+  store double %add, ptr %v0, align 8
+  %11 = load double, ptr %callValue, align 8
+  %12 = load double, ptr %callValue, align 8
+  %mul3 = fmul fast double %11, %12
+  %13 = load double, ptr %v1, align 8
+  %add4 = fadd fast double %13, %mul3
+  store double %add4, ptr %v1, align 8
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %14 = load i32, ptr %i, align 4
+  %inc = add nsw i32 %14, 1
+  store i32 %inc, ptr %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %15 = load double, ptr %v0, align 8
+  %16 = load double, ptr %v1, align 8
+  %add5 = fadd fast double %15, %16
+  call void @llvm.lifetime.end.p0(i64 8, ptr %v1) #2
+  call void @llvm.lifetime.end.p0(i64 8, ptr %v0) #2
+  call void @llvm.lifetime.end.p0(i64 8, ptr %callValue) #2
+  call void @llvm.lifetime.end.p0(i64 8, ptr %rngVal) #2
+  call void @llvm.lifetime.end.p0(i64 4, ptr %block) #2
+  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2
+  ret double %add5
+}
+
+define nofpclass(nan inf) double @monte_exp(i32 noundef %nblocks, i32 noundef %RAND_BLOCK_LENGTH, ptr noundef %samples, double noundef nofpclass(nan inf) %Y, double noundef nofpclass(nan inf) %Z) {
+; CHECK-LABEL: define nofpclass(nan inf) double @monte_exp(
+; CHECK-SAME: i32 noundef [[NBLOCKS:%.*]], i32 noundef [[RAND_BLOCK_LENGTH:%.*]], ptr noundef [[SAMPLES:%.*]], double noundef nofpclass(nan inf) [[Y:%.*]], double noundef nofpclass(nan inf) [[Z:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[CMP16:%.*]] = icmp sgt i32 [[NBLOCKS]], 0
+; CHECK-NEXT:    br i1 [[CMP16]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END10:.*]]
+; CHECK:       [[FOR_BODY_LR_PH]]:
+; CHECK-NEXT:    [[CMP211:%.*]] = icmp sgt i32 [[RAND_BLOCK_LENGTH]], 0
+; CHECK-NEXT:    br i1 [[CMP211]], label %[[FOR_BODY_US_PREHEADER:.*]], label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY_US_PREHEADER]]:
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[RAND_BLOCK_LENGTH]] to i64
+; CHECK-NEXT:    br label %[[FOR_BODY_US:.*]]
+; CHECK:       [[FOR_BODY_US]]:
+; CHECK-NEXT:    [[V1_019_US:%.*]] = phi double [ [[V1_2_US:%.*]], %[[FOR_COND1_FOR_INC8_CRIT_EDGE_US:.*]] ], [ 0.000000e+00, %[[FOR_BODY_US_PREHEADER]] ]
+; CHECK-NEXT:    [[V0_018_US:%.*]] = phi double [ [[V0_2_US:%.*]], %[[FOR_COND1_FOR_INC8_CRIT_EDGE_US]] ], [ 0.000000e+00, %[[FOR_BODY_US_PREHEADER]] ]
+; CHECK-NEXT:    [[BLOCK_017_US:%.*]] = phi i32 [ [[INC9_US:%.*]], %[[FOR_COND1_FOR_INC8_CRIT_EDGE_US]] ], [ 0, %[[FOR_BODY_US_PREHEADER]] ]
+; CHECK-NEXT:    tail call void @resample(i32 noundef [[RAND_BLOCK_LENGTH]], ptr noundef [[SAMPLES]])
+; CHECK-NEXT:    br label %[[FOR_BODY3_US:.*]]
+; CHECK:       [[FOR_BODY3_US]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_BODY_US]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY3_US]] ]
+; CHECK-NEXT:    [[V1_114_US:%.*]] = phi double [ [[V1_019_US]], %[[FOR_BODY_US]] ], [ [[V1_2_US]], %[[FOR_BODY3_US]] ]
+; CHECK-NEXT:    [[V0_113_US:%.*]] = phi double [ [[V0_018_US]], %[[FOR_BODY_US]] ], [ [[V0_2_US]], %[[FOR_BODY3_US]] ]
+; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds float, ptr [[SAMPLES]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX_US]], align 4
+; CHECK-NEXT:    [[CONV_US:%.*]] = fpext float [[TMP0]] to double
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast double @llvm.exp2.f64(double [[CONV_US]])
+; CHECK-NEXT:    [[MUL_US:%.*]] = fmul fast double [[TMP1]], [[Y]]
+; CHECK-NEXT:    [[SUB_US:%.*]] = fsub fast double [[MUL_US]], [[Z]]
+; CHECK-NEXT:    [[CMP4_US:%.*]] = fcmp fast ogt double [[SUB_US]], 0.000000e+00
+; CHECK-NEXT:    [[ADD_US:%.*]] = fadd fast double [[SUB_US]], [[V0_113_US]]
+; CHECK-NEXT:    [[MUL6_US:%.*]] = fmul fast double [[SUB_US]], [[SUB_US]]
+; CHECK-NEXT:    [[ADD7_US:%.*]] = fadd fast double [[MUL6_US]], [[V1_114_US]]
+; CHECK-NEXT:    [[V0_2_US]] = select i1 [[CMP4_US]], double [[ADD_US]], double [[V0_113_US]]
+; CHECK-NEXT:    [[V1_2_US]] = select i1 [[CMP4_US]], double [[ADD7_US]], double [[V1_114_US]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND25_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND25_NOT]], label %[[FOR_COND1_FOR_INC8_CRIT_EDGE_US]], label %[[FOR_BODY3_US]]
+; CHECK:       [[FOR_COND1_FOR_INC8_CRIT_EDGE_US]]:
+; CHECK-NEXT:    [[INC9_US]] = add nuw nsw i32 [[BLOCK_017_US]], 1
+; CHECK-NEXT:    [[EXITCOND26_NOT:%.*]] = icmp eq i32 [[INC9_US]], [[NBLOCKS]]
+; CHECK-NEXT:    br i1 [[EXITCOND26_NOT]], label %[[FOR_END10]], label %[[FOR_BODY_US]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[BLOCK_017:%.*]] = phi i32 [ [[INC9:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_LR_PH]] ]
+; CHECK-NEXT:    tail call void @resample(i32 noundef [[RAND_BLOCK_LENGTH]], ptr noundef [[SAMPLES]])
+; CHECK-NEXT:    [[INC9]] = add nuw nsw i32 [[BLOCK_017]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC9]], [[NBLOCKS]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END10]], label %[[FOR_BODY]]
+; CHECK:       [[FOR_END10]]:
+; CHECK-NEXT:    [[V0_0_LCSSA:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[V0_2_US]], %[[FOR_COND1_FOR_INC8_CRIT_EDGE_US]] ], [ 0.000000e+00, %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[V1_0_LCSSA:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[V1_2_US]], %[[FOR_COND1_FOR_INC8_CRIT_EDGE_US]] ], [ 0.000000e+00, %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[ADD11:%.*]] = fadd fast double [[V1_0_LCSSA]], [[V0_0_LCSSA]]
+; CHECK-NEXT:    ret double [[ADD11]]
+;
+entry:
+  %nblocks.addr = alloca i32, align 4
+  %RAND_BLOCK_LENGTH.addr = alloca i32, align 4
+  %samples.addr = alloca ptr, align 8
+  %Y.addr = alloca double, align 8
+  %Z.addr = alloca double, align 8
+  %i = alloca i32, align 4
+  %block = alloca i32, align 4
+  %rngVal = alloca double, align 8
+  %callValue = alloca double, align 8
+  %v0 = alloca double, align 8
+  %v1 = alloca double, align 8
+  store i32 %nblocks, ptr %nblocks.addr, align 4
+  store i32 %RAND_BLOCK_LENGTH, ptr %RAND_BLOCK_LENGTH.addr, align 4
+  store ptr %samples, ptr %samples.addr, align 8
+  store double %Y, ptr %Y.addr, align 8
+  store double %Z, ptr %Z.addr, align 8
+  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #4
+  call void @llvm.lifetime.start.p0(i64 4, ptr %block) #4
+  call void @llvm.lifetime.start.p0(i64 8, ptr %rngVal) #4
+  call void @llvm.lifetime.start.p0(i64 8, ptr %callValue) #4
+  call void @llvm.lifetime.start.p0(i64 8, ptr %v0) #4
+  store double 0.000000e+00, ptr %v0, align 8
+  call void @llvm.lifetime.start.p0(i64 8, ptr %v1) #4
+  store double 0.000000e+00, ptr %v1, align 8
+  store i32 0, ptr %block, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc8, %entry
+  %0 = load i32, ptr %block, align 4
+  %1 = load i32, ptr %nblocks.addr, align 4
+  %cmp = icmp slt i32 %0, %1
+  br i1 %cmp, label %for.body, label %for.end10
+
+for.body:                                         ; preds = %for.cond
+  %2 = load i32, ptr %RAND_BLOCK_LENGTH.addr, align 4
+  %3 = load ptr, ptr %samples.addr, align 8
+  call void @resample(i32 noundef %2, ptr noundef %3)
+  store i32 0, ptr %i, align 4
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %for.inc, %for.body
+  %4 = load i32, ptr %i, align 4
+  %5 = load i32, ptr %RAND_BLOCK_LENGTH.addr, align 4
+  %cmp2 = icmp slt i32 %4, %5
+  br i1 %cmp2, label %for.body3, label %for.end
+
+for.body3:                                        ; preds = %for.cond1
+  %6 = load ptr, ptr %samples.addr, align 8
+  %7 = load i32, ptr %i, align 4
+  %idxprom = sext i32 %7 to i64
+  %arrayidx = getelementptr inbounds float, ptr %6, i64 %idxprom
+  %8 = load float, ptr %arrayidx, align 4
+  %conv = fpext float %8 to double
+  store double %conv, ptr %rngVal, align 8
+  %9 = load double, ptr %Y.addr, align 8
+  %10 = load double, ptr %rngVal, align 8
+  %11 = call fast double @llvm.exp2.f64(double %10)
+  %mul = fmul fast double %9, %11
+  %12 = load double, ptr %Z.addr, align 8
+  %sub = fsub fast double %mul, %12
+  store double %sub, ptr %callValue, align 8
+  %13 = load double, ptr %callValue, align 8
+  %cmp4 = fcmp fast ogt double %13, 0.000000e+00
+  br i1 %cmp4, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body3
+  %14 = load double, ptr %callValue, align 8
+  %15 = load double, ptr %v0, align 8
+  %add = fadd fast double %15, %14
+  store double %add, ptr %v0, align 8
+  %16 = load double, ptr %callValue, align 8
+  %17 = load double, ptr %callValue, align 8
+  %mul6 = fmul fast double %16, %17
+  %18 = load double, ptr %v1, align 8
+  %add7 = fadd fast double %18, %mul6
+  store double %add7, ptr %v1, align 8
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body3
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %19 = load i32, ptr %i, align 4
+  %inc = add nsw i32 %19, 1
+  store i32 %inc, ptr %i, align 4
+  br label %for.cond1
+
+for.end:                                          ; preds = %for.cond1
+  br label %for.inc8
+
+for.inc8:                                         ; preds = %for.end
+  %20 = load i32, ptr %block, align 4
+  %inc9 = add nsw i32 %20, 1
+  store i32 %inc9, ptr %block, align 4
+  br label %for.cond
+
+for.end10:                                        ; preds = %for.cond
+  %21 = load double, ptr %v0, align 8
+  %22 = load double, ptr %v1, align 8
+  %add11 = fadd fast double %21, %22
+  call void @llvm.lifetime.end.p0(i64 8, ptr %v1) #4
+  call void @llvm.lifetime.end.p0(i64 8, ptr %v0) #4
+  call void @llvm.lifetime.end.p0(i64 8, ptr %callValue) #4
+  call void @llvm.lifetime.end.p0(i64 8, ptr %rngVal) #4
+  call void @llvm.lifetime.end.p0(i64 4, ptr %block) #4
+  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #4
+  ret double %add11
+}
+
+declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @resample(i32 noundef, ptr noundef)
+declare double @llvm.exp2.f64(double)
+declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
-- 
GitLab


From 9f6c632ecda08bfff76b798c46d5d7cfde57b5e9 Mon Sep 17 00:00:00 2001
From: Andrea Faulds <andrea.faulds@amd.com>
Date: Fri, 25 Oct 2024 16:21:59 +0200
Subject: [PATCH 169/693] [mlir][mlir-spirv-cpu-runner] Move MLIR pass pipeline
 to mlir-opt (#113594)

Adds a new mlir-opt test-only pass, -test-spirv-cpu-runner-pipeline,
which runs the set of MLIR passes needed for the mlir-spirv-cpu-runner,
and removes them from the runner. The tests are changed to invoke
mlir-opt with this flag before running the runner. The eventual goal is
to move all host/device code generation steps out of the runner, like
with some of the other runners.

Recommit of 17e9752267ed9c81c8da87f3a6d0e01f130b0d04. It was reverted
due to a build failure, but the build failure had in fact already been
fixed in e7302319b52e3d231216d54d10622b0698928a96.
---
 mlir/test/lib/Pass/CMakeLists.txt             |  1 +
 .../lib/Pass/TestSPIRVCPURunnerPipeline.cpp   | 47 +++++++++++++++++++
 mlir/test/mlir-spirv-cpu-runner/double.mlir   |  3 +-
 .../mlir-spirv-cpu-runner/simple_add.mlir     |  3 +-
 mlir/tools/mlir-opt/mlir-opt.cpp              |  2 +
 .../mlir-spirv-cpu-runner.cpp                 | 24 ----------
 6 files changed, 54 insertions(+), 26 deletions(-)
 create mode 100644 mlir/test/lib/Pass/TestSPIRVCPURunnerPipeline.cpp

diff --git a/mlir/test/lib/Pass/CMakeLists.txt b/mlir/test/lib/Pass/CMakeLists.txt
index b190f054e50b..f489b7e51e50 100644
--- a/mlir/test/lib/Pass/CMakeLists.txt
+++ b/mlir/test/lib/Pass/CMakeLists.txt
@@ -3,6 +3,7 @@ get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
 add_mlir_library(MLIRTestPass
   TestDynamicPipeline.cpp
   TestPassManager.cpp
+  TestSPIRVCPURunnerPipeline.cpp
 
   EXCLUDE_FROM_LIBMLIR
 
diff --git a/mlir/test/lib/Pass/TestSPIRVCPURunnerPipeline.cpp b/mlir/test/lib/Pass/TestSPIRVCPURunnerPipeline.cpp
new file mode 100644
index 000000000000..ded0d22c3130
--- /dev/null
+++ b/mlir/test/lib/Pass/TestSPIRVCPURunnerPipeline.cpp
@@ -0,0 +1,47 @@
+//===------------------ TestSPIRVCPURunnerPipeline.cpp --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements a pipeline for use by mlir-spirv-cpu-runner tests.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/GPUToSPIRV/GPUToSPIRVPass.h"
+#include "mlir/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.h"
+#include "mlir/Dialect/GPU/Transforms/Passes.h"
+#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
+#include "mlir/Dialect/SPIRV/Transforms/Passes.h"
+#include "mlir/Pass/PassManager.h"
+
+using namespace mlir;
+
+namespace {
+
+void buildTestSPIRVCPURunnerPipeline(OpPassManager &passManager) {
+  passManager.addPass(createGpuKernelOutliningPass());
+  passManager.addPass(createConvertGPUToSPIRVPass(/*mapMemorySpace=*/true));
+
+  OpPassManager &nestedPM = passManager.nest<spirv::ModuleOp>();
+  nestedPM.addPass(spirv::createSPIRVLowerABIAttributesPass());
+  nestedPM.addPass(spirv::createSPIRVUpdateVCEPass());
+  passManager.addPass(createLowerHostCodeToLLVMPass());
+  passManager.addPass(createConvertSPIRVToLLVMPass());
+}
+
+} // namespace
+
+namespace mlir {
+namespace test {
+void registerTestSPIRVCPURunnerPipeline() {
+  PassPipelineRegistration<>(
+      "test-spirv-cpu-runner-pipeline",
+      "Runs a series of passes for lowering SPIR-V-dialect MLIR to "
+      "LLVM-dialect MLIR intended for mlir-spirv-cpu-runner.",
+      buildTestSPIRVCPURunnerPipeline);
+}
+} // namespace test
+} // namespace mlir
diff --git a/mlir/test/mlir-spirv-cpu-runner/double.mlir b/mlir/test/mlir-spirv-cpu-runner/double.mlir
index cd551ffb1bd0..35557ba1e94c 100644
--- a/mlir/test/mlir-spirv-cpu-runner/double.mlir
+++ b/mlir/test/mlir-spirv-cpu-runner/double.mlir
@@ -1,4 +1,5 @@
-// RUN: mlir-spirv-cpu-runner %s -e main --entry-point-result=void --shared-libs=%mlir_runner_utils,%mlir_test_spirv_cpu_runner_c_wrappers \
+// RUN: mlir-opt %s -test-spirv-cpu-runner-pipeline \
+// RUN: | mlir-spirv-cpu-runner - -e main --entry-point-result=void --shared-libs=%mlir_runner_utils,%mlir_test_spirv_cpu_runner_c_wrappers \
 // RUN: | FileCheck %s
 
 // CHECK: [8,  8,  8,  8,  8,  8]
diff --git a/mlir/test/mlir-spirv-cpu-runner/simple_add.mlir b/mlir/test/mlir-spirv-cpu-runner/simple_add.mlir
index 119e973e45e4..75675a69a675 100644
--- a/mlir/test/mlir-spirv-cpu-runner/simple_add.mlir
+++ b/mlir/test/mlir-spirv-cpu-runner/simple_add.mlir
@@ -1,4 +1,5 @@
-// RUN: mlir-spirv-cpu-runner %s -e main --entry-point-result=void --shared-libs=%mlir_runner_utils,%mlir_test_spirv_cpu_runner_c_wrappers \
+// RUN: mlir-opt %s -test-spirv-cpu-runner-pipeline \
+// RUN: | mlir-spirv-cpu-runner - -e main --entry-point-result=void --shared-libs=%mlir_runner_utils,%mlir_test_spirv_cpu_runner_c_wrappers \
 // RUN: | FileCheck %s
 
 // CHECK: data =
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index 36b142484bb0..002c3900056d 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -142,6 +142,7 @@ void registerTestSCFWhileOpBuilderPass();
 void registerTestSCFWrapInZeroTripCheckPasses();
 void registerTestShapeMappingPass();
 void registerTestSliceAnalysisPass();
+void registerTestSPIRVCPURunnerPipeline();
 void registerTestSPIRVFuncSignatureConversion();
 void registerTestSPIRVVectorUnrolling();
 void registerTestTensorCopyInsertionPass();
@@ -278,6 +279,7 @@ void registerTestPasses() {
   mlir::test::registerTestSCFWrapInZeroTripCheckPasses();
   mlir::test::registerTestShapeMappingPass();
   mlir::test::registerTestSliceAnalysisPass();
+  mlir::test::registerTestSPIRVCPURunnerPipeline();
   mlir::test::registerTestSPIRVFuncSignatureConversion();
   mlir::test::registerTestSPIRVVectorUnrolling();
   mlir::test::registerTestTensorCopyInsertionPass();
diff --git a/mlir/tools/mlir-spirv-cpu-runner/mlir-spirv-cpu-runner.cpp b/mlir/tools/mlir-spirv-cpu-runner/mlir-spirv-cpu-runner.cpp
index 7e0b51cac806..22ad1024db4a 100644
--- a/mlir/tools/mlir-spirv-cpu-runner/mlir-spirv-cpu-runner.cpp
+++ b/mlir/tools/mlir-spirv-cpu-runner/mlir-spirv-cpu-runner.cpp
@@ -12,18 +12,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h"
-#include "mlir/Conversion/GPUToSPIRV/GPUToSPIRVPass.h"
-#include "mlir/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
-#include "mlir/Dialect/GPU/Transforms/Passes.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
-#include "mlir/Dialect/SPIRV/Transforms/Passes.h"
 #include "mlir/ExecutionEngine/JitRunner.h"
 #include "mlir/ExecutionEngine/OptUtils.h"
 #include "mlir/Pass/Pass.h"
@@ -75,23 +69,6 @@ convertMLIRModule(Operation *op, llvm::LLVMContext &context) {
   return mainModule;
 }
 
-static LogicalResult runMLIRPasses(Operation *module,
-                                   JitRunnerOptions &options) {
-  PassManager passManager(module->getContext(),
-                          module->getName().getStringRef());
-  if (failed(applyPassManagerCLOptions(passManager)))
-    return failure();
-  passManager.addPass(createGpuKernelOutliningPass());
-  passManager.addPass(createConvertGPUToSPIRVPass(/*mapMemorySpace=*/true));
-
-  OpPassManager &nestedPM = passManager.nest<spirv::ModuleOp>();
-  nestedPM.addPass(spirv::createSPIRVLowerABIAttributesPass());
-  nestedPM.addPass(spirv::createSPIRVUpdateVCEPass());
-  passManager.addPass(createLowerHostCodeToLLVMPass());
-  passManager.addPass(createConvertSPIRVToLLVMPass());
-  return passManager.run(module);
-}
-
 int main(int argc, char **argv) {
   llvm::InitLLVM y(argc, argv);
 
@@ -99,7 +76,6 @@ int main(int argc, char **argv) {
   llvm::InitializeNativeTargetAsmPrinter();
 
   mlir::JitRunnerConfig jitRunnerConfig;
-  jitRunnerConfig.mlirTransformer = runMLIRPasses;
   jitRunnerConfig.llvmModuleBuilder = convertMLIRModule;
 
   mlir::DialectRegistry registry;
-- 
GitLab


From cbdfb18794026b0d662d7de1fa39c02ad6227abb Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Fri, 25 Oct 2024 15:39:07 +0100
Subject: [PATCH 170/693] [RISCV] Add Supm extension to RVA23 profiles
 (#113619)

This is mandatory for both RVA23U64 and RVA23S64 in the ratified version
of the specification

<https://github.com/riscv/riscv-profiles/blob/main/src/rva23-profile.adoc>.
---
 llvm/lib/Target/RISCV/RISCVProfiles.td | 3 ++-
 llvm/test/CodeGen/RISCV/attributes.ll  | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVProfiles.td b/llvm/lib/Target/RISCV/RISCVProfiles.td
index 157e087a64da..ce7d1973989f 100644
--- a/llvm/lib/Target/RISCV/RISCVProfiles.td
+++ b/llvm/lib/Target/RISCV/RISCVProfiles.td
@@ -73,7 +73,8 @@ defvar RVA23U64Features = !listconcat(RVA22U64Features,
                                        FeatureStdExtZcmop,
                                        FeatureStdExtZcb,
                                        FeatureStdExtZfa,
-                                       FeatureStdExtZawrs]);
+                                       FeatureStdExtZawrs,
+                                       FeatureStdExtSupm]);
 
 defvar RVA23S64BaseFeatures = !listconcat(RVA22S64BaseFeatures,
                                           [FeatureStdExtSvnapot,
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index e9743d484f77..9be9ddd05ee2 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -578,8 +578,8 @@
 ; RVA20S64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicsr2p0_zifencei2p0_zmmul1p0_za128rs1p0_ssccptr1p0_sstvala1p0_sstvecd1p0_svade1p0_svbare1p0"
 ; RVA22U64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicsr2p0_zihintpause2p0_zihpm2p0_zmmul1p0_za64rs1p0_zfhmin1p0_zba1p0_zbb1p0_zbs1p0_zkt1p0"
 ; RVA22S64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicsr2p0_zifencei2p0_zihintpause2p0_zihpm2p0_zmmul1p0_za64rs1p0_zfhmin1p0_zba1p0_zbb1p0_zbs1p0_zkt1p0_ssccptr1p0_sscounterenw1p0_sstvala1p0_sstvecd1p0_svade1p0_svbare1p0_svinval1p0_svpbmt1p0"
-; RVA23U64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_v1p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicond1p0_zicsr2p0_zihintntl1p0_zihintpause2p0_zihpm2p0_zimop1p0_zmmul1p0_za64rs1p0_zawrs1p0_zfa1p0_zfhmin1p0_zca1p0_zcb1p0_zcmop1p0_zba1p0_zbb1p0_zbs1p0_zkt1p0_zvbb1p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvfhmin1p0_zvkb1p0_zvkt1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0"
-; RVA23S64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_v1p0_h1p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicond1p0_zicsr2p0_zifencei2p0_zihintntl1p0_zihintpause2p0_zihpm2p0_zimop1p0_zmmul1p0_za64rs1p0_zawrs1p0_zfa1p0_zfhmin1p0_zca1p0_zcb1p0_zcmop1p0_zba1p0_zbb1p0_zbs1p0_zkt1p0_zvbb1p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvfhmin1p0_zvkb1p0_zvkt1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0_shcounterenw1p0_shgatpa1p0_shtvala1p0_shvsatpa1p0_shvstvala1p0_shvstvecd1p0_ssccptr1p0_sscofpmf1p0_sscounterenw1p0_ssnpm1p0_ssstateen1p0_sstc1p0_sstvala1p0_sstvecd1p0_ssu64xl1p0_svade1p0_svbare1p0_svinval1p0_svnapot1p0_svpbmt1p0"
+; RVA23U64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_v1p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicond1p0_zicsr2p0_zihintntl1p0_zihintpause2p0_zihpm2p0_zimop1p0_zmmul1p0_za64rs1p0_zawrs1p0_zfa1p0_zfhmin1p0_zca1p0_zcb1p0_zcmop1p0_zba1p0_zbb1p0_zbs1p0_zkt1p0_zvbb1p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvfhmin1p0_zvkb1p0_zvkt1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0_supm1p0"
+; RVA23S64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_v1p0_h1p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicond1p0_zicsr2p0_zifencei2p0_zihintntl1p0_zihintpause2p0_zihpm2p0_zimop1p0_zmmul1p0_za64rs1p0_zawrs1p0_zfa1p0_zfhmin1p0_zca1p0_zcb1p0_zcmop1p0_zba1p0_zbb1p0_zbs1p0_zkt1p0_zvbb1p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvfhmin1p0_zvkb1p0_zvkt1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0_shcounterenw1p0_shgatpa1p0_shtvala1p0_shvsatpa1p0_shvstvala1p0_shvstvecd1p0_ssccptr1p0_sscofpmf1p0_sscounterenw1p0_ssnpm1p0_ssstateen1p0_sstc1p0_sstvala1p0_sstvecd1p0_ssu64xl1p0_supm1p0_svade1p0_svbare1p0_svinval1p0_svnapot1p0_svpbmt1p0"
 ; RVB23U64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicond1p0_zicsr2p0_zihintntl1p0_zihintpause2p0_zihpm2p0_zimop1p0_zmmul1p0_za64rs1p0_zawrs1p0_zfa1p0_zca1p0_zcb1p0_zcmop1p0_zba1p0_zbb1p0_zbs1p0_zkt1p0"
 ; RVB23S64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicond1p0_zicsr2p0_zifencei2p0_zihintntl1p0_zihintpause2p0_zihpm2p0_zimop1p0_zmmul1p0_za64rs1p0_zawrs1p0_zfa1p0_zca1p0_zcb1p0_zcmop1p0_zba1p0_zbb1p0_zbs1p0_zkt1p0_ssccptr1p0_sscofpmf1p0_sscounterenw1p0_sstc1p0_sstvala1p0_sstvecd1p0_ssu64xl1p0_svade1p0_svbare1p0_svinval1p0_svnapot1p0_svpbmt1p0"
 ; RVM23U32: .attribute 5, "rv32i2p1_m2p0_zicbop1p0_zicond1p0_zicsr2p0_zihintntl1p0_zihintpause2p0_zimop1p0_zmmul1p0_zca1p0_zcb1p0_zce1p0_zcmop1p0_zcmp1p0_zcmt1p0_zba1p0_zbb1p0_zbs1p0"
-- 
GitLab


From bbc0e631d2d3facd5952aeafc7400761813acc3a Mon Sep 17 00:00:00 2001
From: Thomas Preud'homme <thomas.preudhomme@arm.com>
Date: Fri, 25 Oct 2024 15:41:39 +0100
Subject: [PATCH 171/693] [MLIR] Remove unneeded LLVMDialect.h include in
 ControlFlowToSCF.cpp (#113560)

This fixes the following failure when doing a clean build (in particular
no .ninja* lying around) of lib/libMLIRControlFlowToSCF.a only:
```
In file included from llvm/include/llvm/IR/Module.h:22,
                 from mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h:37,
                 from mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp:19
llvm/include/llvm/IR/Attributes.h:90:14: fatal error: llvm/IR/Attributes.inc: No such file or directory
```
---
 mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
index d3ee89743da9..1c592d665f3e 100644
--- a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
+++ b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
@@ -16,7 +16,6 @@
 #include "mlir/Dialect/ControlFlow/IR/ControlFlow.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/Pass/Pass.h"
-- 
GitLab


From e47bf3d08d51306f2e534951a1b77043dc540ceb Mon Sep 17 00:00:00 2001
From: Kai Nacke <kai.peter.nacke@ibm.com>
Date: Wed, 23 Oct 2024 16:13:39 -0400
Subject: [PATCH 172/693] [JIT] Fix crash in unit tests

The unit tests `ReOptimizeLayerTest.BasicReOptimization` and `JITLinkRedirectionManagerTest.BasicRedirectionOperation` are failing for me with the error:

```
Program aborted due to an unhandled Error:
Error value was Success. (Note: Success values must still be checked prior to being destroyed).
```

The error is raised when a value is assigned to `Err`, due to the the missing `ErrorAsOutParameter`.

The fix is to move the error handling out of the constructor.
---
 .../Orc/JITLinkRedirectableSymbolManager.h    | 33 +++++++++----------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
index 52f284c89bda..ef42cc5f798f 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
@@ -26,12 +26,16 @@ public:
   /// Create redirection manager that uses JITLink based implementaion.
   static Expected<std::unique_ptr<RedirectableSymbolManager>>
   Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &JD) {
-    Error Err = Error::success();
-    auto RM = std::unique_ptr<RedirectableSymbolManager>(
-        new JITLinkRedirectableSymbolManager(ObjLinkingLayer, JD, Err));
-    if (Err)
-      return Err;
-    return std::move(RM);
+    auto AnonymousPtrCreator(jitlink::getAnonymousPointerCreator(
+        ObjLinkingLayer.getExecutionSession().getTargetTriple()));
+    auto PtrJumpStubCreator(jitlink::getPointerJumpStubCreator(
+        ObjLinkingLayer.getExecutionSession().getTargetTriple()));
+    if (!AnonymousPtrCreator || !PtrJumpStubCreator)
+      return make_error<StringError>("Architecture not supported",
+                                     inconvertibleErrorCode());
+    return std::unique_ptr<RedirectableSymbolManager>(
+        new JITLinkRedirectableSymbolManager(
+            ObjLinkingLayer, JD, AnonymousPtrCreator, PtrJumpStubCreator));
   }
 
   void emitRedirectableSymbols(std::unique_ptr<MaterializationResponsibility> R,
@@ -52,18 +56,13 @@ private:
   constexpr static StringRef JumpStubTableName = "$IND_JUMP_";
   constexpr static StringRef StubPtrTableName = "$__IND_JUMP_PTRS";
 
-  JITLinkRedirectableSymbolManager(ObjectLinkingLayer &ObjLinkingLayer,
-                                   JITDylib &JD, Error &Err)
+  JITLinkRedirectableSymbolManager(
+      ObjectLinkingLayer &ObjLinkingLayer, JITDylib &JD,
+      jitlink::AnonymousPointerCreator &AnonymousPtrCreator,
+      jitlink::PointerJumpStubCreator &PtrJumpStubCreator)
       : ObjLinkingLayer(ObjLinkingLayer), JD(JD),
-        AnonymousPtrCreator(jitlink::getAnonymousPointerCreator(
-            ObjLinkingLayer.getExecutionSession().getTargetTriple())),
-        PtrJumpStubCreator(jitlink::getPointerJumpStubCreator(
-            ObjLinkingLayer.getExecutionSession().getTargetTriple())) {
-    if (!AnonymousPtrCreator || !PtrJumpStubCreator)
-      Err = make_error<StringError>("Architecture not supported",
-                                    inconvertibleErrorCode());
-    if (Err)
-      return;
+        AnonymousPtrCreator(std::move(AnonymousPtrCreator)),
+        PtrJumpStubCreator(std::move(PtrJumpStubCreator)) {
     ObjLinkingLayer.getExecutionSession().registerResourceManager(*this);
   }
 
-- 
GitLab


From aba39c3974c7e43a83a9d647dca9b67caca8572e Mon Sep 17 00:00:00 2001
From: Jonas Paulsson <paulson1@linux.ibm.com>
Date: Fri, 25 Oct 2024 17:40:00 +0200
Subject: [PATCH 173/693] [System] Precommit of test for #112491 (#113704)

---
 .../SystemZ/vec-elt-insertion.ll              | 128 ++++++++++++++++++
 1 file changed, 128 insertions(+)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/SystemZ/vec-elt-insertion.ll

diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/vec-elt-insertion.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/vec-elt-insertion.ll
new file mode 100644
index 000000000000..eb8dd72e0304
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/vec-elt-insertion.ll
@@ -0,0 +1,128 @@
+; RUN: opt < %s -mtriple=s390x-unknown-linux -mcpu=z16 -S -passes=slp-vectorizer \
+; RUN:   -pass-remarks-output=%t | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=REMARK %s
+;
+; NB! This is a pre-commit version (for #112491) with current codegen and remarks.
+;
+; Test functions that (at least currently) only gets vectorized if the
+; insertion cost for an element load is counted as free.
+
+; This function needs the free element load to be recognized in SLP
+; getGatherCost().
+define void @fun0(ptr nocapture %0, double %1) {
+; CHECK-LABEL: define void @fun0(
+; CHECK:         fmul double
+; CHECK:         call double @llvm.fmuladd.f64(
+; CHECK-NEXT:    call double @llvm.fmuladd.f64(
+; CHECK-NEXT:    call double @llvm.sqrt.f64(
+; CHECK:         fmul double
+; CHECK:         call double @llvm.fmuladd.f64(
+; CHECK-NEXT:    call double @llvm.fmuladd.f64(
+; CHECK-NEXT:    call double @llvm.sqrt.f64(
+;
+; REMARK-LABEL: Function: fun0
+; REMARK: Args:
+; REMARK-NEXT: - String:          'List vectorization was possible but not beneficial with cost '
+; REMARK-NEXT: - Cost:            '0'
+
+  %3 = fmul double %1, 2.000000e+00
+  %4 = tail call double @llvm.fmuladd.f64(double %3, double %3, double 0.000000e+00)
+  %5 = tail call double @llvm.fmuladd.f64(double %3, double %3, double %4)
+  %sqrt1 = tail call double @llvm.sqrt.f64(double %5)
+  %6 = load double, ptr %0, align 8
+  %7 = fmul double %6, 2.000000e+00
+  %8 = tail call double @llvm.fmuladd.f64(double %7, double %7, double 0.000000e+00)
+  %9 = tail call double @llvm.fmuladd.f64(double %7, double %7, double %8)
+  %sqrt = tail call double @llvm.sqrt.f64(double %9)
+  %10 = fadd double %sqrt1, %sqrt
+  store double %10, ptr %0, align 8
+  ret void
+}
+
+; This function needs the element-load to be recognized in SystemZ
+; getVectorInstrCost().
+define void @fun1(double %0) {
+; CHECK-LABEL: define void @fun1(
+; CHECK:         phi double
+; CHECK-NEXT:    phi double
+; CHECK-NEXT:    phi double
+; CHECK-NEXT:    phi double
+; CHECK-NEXT:    phi double
+; CHECK-NEXT:    phi double
+; CHECK-NEXT:    fsub double
+; CHECK-NEXT:    fsub double
+; CHECK-NEXT:    fmul double
+; CHECK-NEXT:    fmul double
+; CHECK-NEXT:    fsub double
+; CHECK-NEXT:    fsub double
+; CHECK-NEXT:    call double @llvm.fmuladd.f64(
+; CHECK-NEXT:    call double @llvm.fmuladd.f64(
+; CHECK-NEXT:    fsub double
+; CHECK-NEXT:    fsub double
+; CHECK-NEXT:    call double @llvm.fmuladd.f64(
+; CHECK-NEXT:    call double @llvm.fmuladd.f64(
+; CHECK:         fcmp olt double
+; CHECK-NEXT:    fcmp olt double
+; CHECK-NEXT:    or i1
+;
+; REMARK-LABEL: Function: fun1
+; REMARK: Args:
+; REMARK:      - String:          'List vectorization was possible but not beneficial with cost '
+; REMARK-NEXT: - Cost:            '0'
+
+  br label %2
+
+2:
+  %3 = phi double [ poison, %1 ], [ poison, %2 ]
+  %4 = phi double [ undef, %1 ], [ poison, %2 ]
+  %5 = phi double [ 0.000000e+00, %1 ], [ poison, %2 ]
+  %6 = phi double [ 0.000000e+00, %1 ], [ poison, %2 ]
+  %7 = phi double [ 0.000000e+00, %1 ], [ poison, %2 ]
+  %8 = phi double [ 0.000000e+00, %1 ], [ %21, %2 ]
+  %9 = fsub double 0.000000e+00, %8
+  %10 = fsub double 0.000000e+00, %7
+  %11 = fmul double %9, 0.000000e+00
+  %12 = fmul double %10, 0.000000e+00
+  %13 = fsub double 0.000000e+00, %6
+  %14 = fsub double 0.000000e+00, %5
+  %15 = tail call double @llvm.fmuladd.f64(double %13, double %13, double %11)
+  %16 = tail call double @llvm.fmuladd.f64(double %14, double %14, double %12)
+  %17 = fsub double 0.000000e+00, %4
+  %18 = fsub double 0.000000e+00, %3
+  %19 = tail call double @llvm.fmuladd.f64(double %17, double %17, double %15)
+  %20 = tail call double @llvm.fmuladd.f64(double %18, double %18, double %16)
+  %21 = load double, ptr null, align 8
+  %22 = fcmp olt double %19, %0
+  %23 = fcmp olt double %20, 0.000000e+00
+  %24 = or i1 %23, %22
+  br label %2
+}
+
+declare double @llvm.fmuladd.f64(double, double, double)
+
+; This should *not* be vectorized as the insertion into the vector isn't free,
+; which is recognized in SystemZTTImpl::getScalarizationOverhead().
+define void @fun2(ptr %0, ptr %Dst) {
+; CHECK-LABEL: define void @fun2(
+; CHECK: insertelement
+; CHECK: store <2 x i64>
+;
+; REMARK-LABEL: Function: fun2
+; REMARK: Args:
+; REMARK-NEXT: - String:          'Stores SLP vectorized with cost '
+; REMARK-NEXT: - Cost:            '-1'
+
+  %3 = load i64, ptr %0, align 8
+  %4 = icmp eq i64 %3, 0
+  br i1 %4, label %5, label %6
+
+5:
+  ret void
+
+6:
+  %7 = getelementptr i8, ptr %Dst, i64 24
+  store i64 %3, ptr %7, align 8
+  %8 = getelementptr i8, ptr %Dst, i64 16
+  store i64 0, ptr %8, align 8
+  br label %5
+}
-- 
GitLab


From 81e536ec87a108d012cf9156a2c3fc672fb92155 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Fri, 25 Oct 2024 15:43:47 +0000
Subject: [PATCH 174/693] [clang][test] Fix typo in arm-mfp8.cpp

New test added by https://github.com/llvm/llvm-project/pull/97277.
---
 clang/test/AST/arm-mfp8.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/test/AST/arm-mfp8.cpp b/clang/test/AST/arm-mfp8.cpp
index a00d055f7d96..51bebba067eb 100644
--- a/clang/test/AST/arm-mfp8.cpp
+++ b/clang/test/AST/arm-mfp8.cpp
@@ -69,7 +69,7 @@ public:
 //CHECK-NEXT:  | | `-CompoundStmt {{.*}}
 //CHECK-NEXT:  | |   `-ReturnStmt {{.*}}
 //CHECK-NEXT:  | |     `-ImplicitCastExpr {{.*}} '__mfp8':'__MFloat8_t' <LValueToRValue>
-//CHECK-NEXT:  | |       `-DeclRefExpr {{.*}} '__mfp8':'__MFloat8_t' lvalue ParmVar {{.*}}8 'arg' '__mfp8':'__MFloat8_t'
+//CHECK-NEXT:  | |       `-DeclRefExpr {{.*}} '__mfp8':'__MFloat8_t' lvalue ParmVar {{.*}} 'arg' '__mfp8':'__MFloat8_t'
 //CHECK-NEXT:  | `-CXXMethodDecl {{.*}} func2c '__mfp8 (__mfp8)' static implicit-inline
 //CHECK-NEXT:  |   |-ParmVarDecl {{.*}} arg '__mfp8':'__MFloat8_t'
 //CHECK-NEXT:  |   `-CompoundStmt {{.*}}
-- 
GitLab


From 5c20891b2bb60f82dd82a8e90b111f8c13a13ad4 Mon Sep 17 00:00:00 2001
From: Ryosuke Niwa <rniwa@webkit.org>
Date: Fri, 25 Oct 2024 08:52:56 -0700
Subject: [PATCH 175/693] [WebKit Checkers] Allow a guardian
 CheckedPtr/CheckedRef (#110222)

This PR makes WebKit checkers allow a guardian variable which is
CheckedPtr or CheckedRef as in addition to RefPtr or Ref.
---
 .../Checkers/WebKit/ASTUtils.cpp              | 16 +++---
 .../Checkers/WebKit/PtrTypesSemantics.cpp     | 43 +++++++++++++---
 .../Checkers/WebKit/PtrTypesSemantics.h       | 22 ++++++--
 .../WebKit/UncountedCallArgsChecker.cpp       |  2 +
 .../WebKit/UncountedLocalVarsChecker.cpp      |  1 +
 .../Checkers/WebKit/call-args-checked.cpp     | 46 +++++++++++++++++
 .../Analysis/Checkers/WebKit/mock-types.h     | 16 ++++--
 .../Checkers/WebKit/uncounted-local-vars.cpp  | 51 +++++++++++++++++++
 8 files changed, 177 insertions(+), 20 deletions(-)
 create mode 100644 clang/test/Analysis/Checkers/WebKit/call-args-checked.cpp

diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
index b7b2f8a16f07..9d34dfd3cea6 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
@@ -17,6 +17,10 @@
 
 namespace clang {
 
+bool isSafePtr(clang::CXXRecordDecl *Decl) {
+  return isRefCounted(Decl) || isCheckedPtr(Decl);
+}
+
 bool tryToFindPtrOrigin(
     const Expr *E, bool StopAtFirstRefCountedObj,
     std::function<bool(const clang::Expr *, bool)> callback) {
@@ -31,7 +35,7 @@ bool tryToFindPtrOrigin(
     }
     if (auto *tempExpr = dyn_cast<CXXTemporaryObjectExpr>(E)) {
       if (auto *C = tempExpr->getConstructor()) {
-        if (auto *Class = C->getParent(); Class && isRefCounted(Class))
+        if (auto *Class = C->getParent(); Class && isSafePtr(Class))
           return callback(E, true);
         break;
       }
@@ -56,7 +60,7 @@ bool tryToFindPtrOrigin(
       if (StopAtFirstRefCountedObj) {
         if (auto *ConversionFunc =
                 dyn_cast_or_null<FunctionDecl>(cast->getConversionFunction())) {
-          if (isCtorOfRefCounted(ConversionFunc))
+          if (isCtorOfSafePtr(ConversionFunc))
             return callback(E, true);
         }
       }
@@ -68,7 +72,7 @@ bool tryToFindPtrOrigin(
     if (auto *call = dyn_cast<CallExpr>(E)) {
       if (auto *memberCall = dyn_cast<CXXMemberCallExpr>(call)) {
         if (auto *decl = memberCall->getMethodDecl()) {
-          std::optional<bool> IsGetterOfRefCt = isGetterOfRefCounted(decl);
+          std::optional<bool> IsGetterOfRefCt = isGetterOfSafePtr(decl);
           if (IsGetterOfRefCt && *IsGetterOfRefCt) {
             E = memberCall->getImplicitObjectArgument();
             if (StopAtFirstRefCountedObj) {
@@ -87,7 +91,7 @@ bool tryToFindPtrOrigin(
       }
 
       if (auto *callee = call->getDirectCallee()) {
-        if (isCtorOfRefCounted(callee)) {
+        if (isCtorOfRefCounted(callee) || isCtorOfCheckedPtr(callee)) {
           if (StopAtFirstRefCountedObj)
             return callback(E, true);
 
@@ -95,7 +99,7 @@ bool tryToFindPtrOrigin(
           continue;
         }
 
-        if (isRefType(callee->getReturnType()))
+        if (isSafePtrType(callee->getReturnType()))
           return callback(E, true);
 
         if (isSingleton(callee))
@@ -114,7 +118,7 @@ bool tryToFindPtrOrigin(
     }
     if (auto *ObjCMsgExpr = dyn_cast<ObjCMessageExpr>(E)) {
       if (auto *Method = ObjCMsgExpr->getMethodDecl()) {
-        if (isRefType(Method->getReturnType()))
+        if (isSafePtrType(Method->getReturnType()))
           return callback(E, true);
       }
     }
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
index 71440e6d08a1..2293dcf1d4bd 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
@@ -135,7 +135,16 @@ bool isCtorOfRefCounted(const clang::FunctionDecl *F) {
          || FunctionName == "Identifier";
 }
 
-bool isRefType(const clang::QualType T) {
+bool isCtorOfCheckedPtr(const clang::FunctionDecl *F) {
+  assert(F);
+  return isCheckedPtr(safeGetName(F));
+}
+
+bool isCtorOfSafePtr(const clang::FunctionDecl *F) {
+  return isCtorOfRefCounted(F) || isCtorOfCheckedPtr(F);
+}
+
+bool isSafePtrType(const clang::QualType T) {
   QualType type = T;
   while (!type.isNull()) {
     if (auto *elaboratedT = type->getAs<ElaboratedType>()) {
@@ -145,7 +154,7 @@ bool isRefType(const clang::QualType T) {
     if (auto *specialT = type->getAs<TemplateSpecializationType>()) {
       if (auto *decl = specialT->getTemplateName().getAsTemplateDecl()) {
         auto name = decl->getNameAsString();
-        return isRefType(name);
+        return isRefType(name) || isCheckedPtr(name);
       }
       return false;
     }
@@ -177,6 +186,12 @@ std::optional<bool> isUncounted(const CXXRecordDecl* Class)
   return (*IsRefCountable);
 }
 
+std::optional<bool> isUnchecked(const CXXRecordDecl *Class) {
+  if (isCheckedPtr(Class))
+    return false; // Cheaper than below
+  return isCheckedPtrCapable(Class);
+}
+
 std::optional<bool> isUncountedPtr(const QualType T) {
   if (T->isPointerType() || T->isReferenceType()) {
     if (auto *CXXRD = T->getPointeeCXXRecordDecl())
@@ -185,8 +200,16 @@ std::optional<bool> isUncountedPtr(const QualType T) {
   return false;
 }
 
-std::optional<bool> isGetterOfRefCounted(const CXXMethodDecl* M)
-{
+std::optional<bool> isUnsafePtr(const QualType T) {
+  if (T->isPointerType() || T->isReferenceType()) {
+    if (auto *CXXRD = T->getPointeeCXXRecordDecl()) {
+      return isUncounted(CXXRD) || isUnchecked(CXXRD);
+    }
+  }
+  return false;
+}
+
+std::optional<bool> isGetterOfSafePtr(const CXXMethodDecl *M) {
   assert(M);
 
   if (isa<CXXMethodDecl>(M)) {
@@ -194,6 +217,9 @@ std::optional<bool> isGetterOfRefCounted(const CXXMethodDecl* M)
     auto className = safeGetName(calleeMethodsClass);
     auto method = safeGetName(M);
 
+    if (isCheckedPtr(className) && (method == "get" || method == "ptr"))
+      return true;
+
     if ((isRefType(className) && (method == "get" || method == "ptr")) ||
         ((className == "String" || className == "AtomString" ||
           className == "AtomStringImpl" || className == "UniqueString" ||
@@ -205,7 +231,12 @@ std::optional<bool> isGetterOfRefCounted(const CXXMethodDecl* M)
     // FIXME: Currently allowing any Ref<T> -> whatever cast.
     if (isRefType(className)) {
       if (auto *maybeRefToRawOperator = dyn_cast<CXXConversionDecl>(M))
-        return isUncountedPtr(maybeRefToRawOperator->getConversionType());
+        return isUnsafePtr(maybeRefToRawOperator->getConversionType());
+    }
+
+    if (isCheckedPtr(className)) {
+      if (auto *maybeRefToRawOperator = dyn_cast<CXXConversionDecl>(M))
+        return isUnsafePtr(maybeRefToRawOperator->getConversionType());
     }
   }
   return false;
@@ -448,7 +479,7 @@ public:
     if (!Callee)
       return false;
 
-    std::optional<bool> IsGetterOfRefCounted = isGetterOfRefCounted(Callee);
+    std::optional<bool> IsGetterOfRefCounted = isGetterOfSafePtr(Callee);
     if (IsGetterOfRefCounted && *IsGetterOfRefCounted)
       return true;
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
index 8e6aadf63b6d..4b41ca96e1df 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
@@ -63,18 +63,30 @@ std::optional<bool> isUncounted(const clang::CXXRecordDecl* Class);
 /// class, false if not, std::nullopt if inconclusive.
 std::optional<bool> isUncountedPtr(const clang::QualType T);
 
-/// \returns true if Name is a RefPtr, Ref, or its variant, false if not.
-bool isRefType(const std::string &Name);
+/// \returns true if \p T is a RefPtr, Ref, CheckedPtr, CheckedRef, or its
+/// variant, false if not.
+bool isSafePtrType(const clang::QualType T);
 
 /// \returns true if \p F creates ref-countable object from uncounted parameter,
 /// false if not.
 bool isCtorOfRefCounted(const clang::FunctionDecl *F);
 
-/// \returns true if \p T is RefPtr, Ref, or its variant, false if not.
-bool isRefType(const clang::QualType T);
+/// \returns true if \p F creates checked ptr object from uncounted parameter,
+/// false if not.
+bool isCtorOfCheckedPtr(const clang::FunctionDecl *F);
+
+/// \returns true if \p F creates ref-countable or checked ptr object from
+/// uncounted parameter, false if not.
+bool isCtorOfSafePtr(const clang::FunctionDecl *F);
+
+/// \returns true if \p Name is RefPtr, Ref, or its variant, false if not.
+bool isRefType(const std::string &Name);
+
+/// \returns true if \p Name is CheckedRef or CheckedPtr, false if not.
+bool isCheckedPtr(const std::string &Name);
 
 /// \returns true if \p M is getter of a ref-counted class, false if not.
-std::optional<bool> isGetterOfRefCounted(const clang::CXXMethodDecl* Method);
+std::optional<bool> isGetterOfSafePtr(const clang::CXXMethodDecl *Method);
 
 /// \returns true if \p F is a conversion between ref-countable or ref-counted
 /// pointer types.
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp
index cea3503fa2c3..1a5a7309a54f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp
@@ -96,6 +96,8 @@ public:
           auto name = safeGetName(MD);
           if (name == "ref" || name == "deref")
             return;
+          if (name == "incrementPtrCount" || name == "decrementPtrCount")
+            return;
         }
         auto *E = MemberCallExpr->getImplicitObjectArgument();
         QualType ArgType = MemberCallExpr->getObjectType().getCanonicalType();
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
index 81d21100de87..5cdf047738ab 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
@@ -227,6 +227,7 @@ public:
                       if (MaybeGuardianArgCXXRecord) {
                         if (MaybeGuardian->isLocalVarDecl() &&
                             (isRefCounted(MaybeGuardianArgCXXRecord) ||
+                             isCheckedPtr(MaybeGuardianArgCXXRecord) ||
                              isRefcountedStringsHack(MaybeGuardian)) &&
                             isGuardedScopeEmbeddedInGuardianScope(
                                 V, MaybeGuardian))
diff --git a/clang/test/Analysis/Checkers/WebKit/call-args-checked.cpp b/clang/test/Analysis/Checkers/WebKit/call-args-checked.cpp
new file mode 100644
index 000000000000..49b6bfcd7cad
--- /dev/null
+++ b/clang/test/Analysis/Checkers/WebKit/call-args-checked.cpp
@@ -0,0 +1,46 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.UncountedCallArgsChecker -verify %s
+
+#include "mock-types.h"
+
+RefCountableAndCheckable* makeObj();
+CheckedRef<RefCountableAndCheckable> makeObjChecked();
+void someFunction(RefCountableAndCheckable*);
+
+namespace call_args_unchecked_uncounted {
+
+static void foo() {
+  someFunction(makeObj());
+  // expected-warning@-1{{Call argument is uncounted and unsafe [alpha.webkit.UncountedCallArgsChecker]}}
+}
+
+} // namespace call_args_checked
+
+namespace call_args_checked {
+
+static void foo() {
+  CheckedPtr<RefCountableAndCheckable> ptr = makeObj();
+  someFunction(ptr.get());
+}
+
+static void bar() {
+  someFunction(CheckedPtr { makeObj() }.get());
+}
+
+static void baz() {
+  someFunction(makeObjChecked().ptr());
+}
+
+} // namespace call_args_checked
+
+namespace call_args_default {
+
+void someFunction(RefCountableAndCheckable* = makeObj());
+// expected-warning@-1{{Call argument is uncounted and unsafe [alpha.webkit.UncountedCallArgsChecker]}}
+void otherFunction(RefCountableAndCheckable* = makeObjChecked().ptr());
+
+void foo() {
+  someFunction();
+  otherFunction();
+}
+
+}
diff --git a/clang/test/Analysis/Checkers/WebKit/mock-types.h b/clang/test/Analysis/Checkers/WebKit/mock-types.h
index 933b4c5e62a7..8d8a90f0afae 100644
--- a/clang/test/Analysis/Checkers/WebKit/mock-types.h
+++ b/clang/test/Analysis/Checkers/WebKit/mock-types.h
@@ -114,8 +114,8 @@ private:
 
 public:
   CheckedRef() : t{} {};
-  CheckedRef(T &t) : t(t) { t->incrementPtrCount(); }
-  CheckedRef(const CheckedRef& o) : t(o.t) { if (t) t->incrementPtrCount(); }
+  CheckedRef(T &t) : t(&t) { t.incrementPtrCount(); }
+  CheckedRef(const CheckedRef &o) : t(o.t) { if (t) t->incrementPtrCount(); }
   ~CheckedRef() { if (t) t->decrementPtrCount(); }
   T &get() { return *t; }
   T *ptr() { return t; }
@@ -135,7 +135,7 @@ public:
     if (t)
       t->incrementPtrCount();
   }
-  CheckedPtr(Ref<T>&& o)
+  CheckedPtr(Ref<T> &&o)
     : t(o.leakRef())
   { }
   ~CheckedPtr() {
@@ -156,4 +156,14 @@ public:
   void decrementPtrCount();
 };
 
+class RefCountableAndCheckable {
+public:
+  void incrementPtrCount() const;
+  void decrementPtrCount() const;
+  void ref() const;
+  void deref() const;
+  void method();
+  int trivial() { return 0; }
+};
+
 #endif
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
index b5f6b8535bf4..1c0df42cdda6 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
@@ -290,6 +290,57 @@ void foo() {
 
 } // namespace local_assignment_to_global
 
+namespace local_refcountable_checkable_object {
+
+RefCountableAndCheckable* provide_obj();
+
+void local_raw_ptr() {
+  RefCountableAndCheckable* a = nullptr;
+  // expected-warning@-1{{Local variable 'a' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+  a = provide_obj();
+  a->method();
+}
+
+void local_checked_ptr() {
+  CheckedPtr<RefCountableAndCheckable> a = nullptr;
+  a = provide_obj();
+  a->method();
+}
+
+void local_var_with_guardian_checked_ptr() {
+  CheckedPtr<RefCountableAndCheckable> a = provide_obj();
+  {
+    auto* b = a.get();
+    b->method();
+  }
+}
+
+void local_var_with_guardian_checked_ptr_with_assignment() {
+  CheckedPtr<RefCountableAndCheckable> a = provide_obj();
+  {
+    RefCountableAndCheckable* b = a.get();
+    // expected-warning@-1{{Local variable 'b' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+    b = provide_obj();
+    b->method();
+  }
+}
+
+void local_var_with_guardian_checked_ref() {
+  CheckedRef<RefCountableAndCheckable> a = *provide_obj();
+  {
+    RefCountableAndCheckable& b = a;
+    b.method();
+  }
+}
+
+void static_var() {
+  static RefCountableAndCheckable* a = nullptr;
+  // expected-warning@-1{{Static local variable 'a' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+  a = provide_obj();
+}
+
+} // namespace local_refcountable_checkable_object
+
 namespace local_var_in_recursive_function {
 
 struct TreeNode {
-- 
GitLab


From 831ba662713334032088ee94cacb1b26903aa916 Mon Sep 17 00:00:00 2001
From: Lorenzo Albano <lorenzo.albano@bsc.es>
Date: Fri, 25 Oct 2024 16:11:18 +0000
Subject: [PATCH 176/693] Fix replacePHIsWithBlendRecipes transform.

---
 .../Transforms/Vectorize/VPlanTransforms.cpp  | 187 +++++++++++++-----
 1 file changed, 133 insertions(+), 54 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 33b9cf1b740c..54fb08b3a867 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2738,6 +2738,8 @@ static void replacePHIsInRegion(VPRegionBlock *Region, VPDominatorTree &DomTree,
   ReversePostOrderTraversal<GraphT> RPOT(Region->getEntry());
 
   // First, collect all PHI candidates to be replaced.
+  VERBOSE_DEBUG(
+      dbgs() << "Collect PHIs candidates to be replaced with blends.\n");
   SmallVector<VPWidenPHIRecipe *> CandidatesPHIs;
   for (VPBlockBase *Block : RPOT) {
     // Header PHIs should not be blended.
@@ -2749,22 +2751,25 @@ static void replacePHIsInRegion(VPRegionBlock *Region, VPDominatorTree &DomTree,
       continue;
     }
 
-    for (auto &Phi : cast<VPBasicBlock>(Block)->phis()) {
+    for (VPRecipeBase &Phi : cast<VPBasicBlock>(Block)->phis()) {
       auto *PhiR = dyn_cast<VPWidenPHIRecipe>(&Phi);
       if (!PhiR)
         continue;
       CandidatesPHIs.push_back(PhiR);
+      VERBOSE_DEBUG(dbgs() << "\t" << "Add "; Phi.dump(););
     }
   }
 
   MapVector<VPWidenPHIRecipe *, VPRecipeBase *> ReplacePHIWithBlend;
   for (VPWidenPHIRecipe *PhiR : CandidatesPHIs) {
+    VERBOSE_DEBUG(dbgs() << "Working on "; cast<VPRecipeBase>(PhiR)->dump(););
     // If there is only a single incoming value, or if all incoming values are
     // equal, no need for a recipe.
     unsigned NumIncoming = PhiR->getNumOperands();
     if (NumIncoming == 1 || llvm::all_equal(PhiR->operands())) {
       PhiR->getVPSingleValue()->replaceAllUsesWith(PhiR->getIncomingValue(0));
       PhiR->eraseFromParent();
+      VERBOSE_DEBUG(dbgs() << "\t" << "Recipe not needed: deleting it.\n");
       continue;
     }
 
@@ -2783,8 +2788,10 @@ static void replacePHIsInRegion(VPRegionBlock *Region, VPDominatorTree &DomTree,
           BlendOperands.push_back(IncomingM);
         }
       }
-      VPBlendRecipe *Blend = new VPBlendRecipe(BlendOperands);
+      auto *Blend = new VPBlendRecipe(BlendOperands);
       ReplacePHIWithBlend[PhiR] = Blend;
+      VERBOSE_DEBUG(dbgs() << "\t" << "Replacing with ";
+                    cast<VPRecipeBase>(Blend)->dump(););
       continue;
     }
 
@@ -2799,93 +2806,165 @@ static void replacePHIsInRegion(VPRegionBlock *Region, VPDominatorTree &DomTree,
         break;
       }
     }
-    if (IsValidPHI)
+    if (IsValidPHI) {
+      VERBOSE_DEBUG(dbgs() << "\t" << "PHI stil valid: keep it.\n");
       continue;
+    }
 
-    // Otherwise, we need to add PHIs in the CFG in order to blend current PHI.
+    VERBOSE_DEBUG(dbgs() << "CFG needs to be fixed before blending the PHI!\n");
+    // Otherwise, we need to add phis and/or blends in the CFG in order to make
+    // it possible to blend current PHI.
     Type *PHIType = PhiR->getUnderlyingInstr()->getType();
     // Map each incoming block of the original PHI to its incoming value.
-    DenseMap<VPBlockBase *, VPValue *> BlockHasDef;
+    DenseMap<VPBlockBase *, VPValue *> ValueForBlock;
     // Map each incoming value of the original PHI with the edge mask calculated
     // before linearization.
-    DenseMap<VPValue *, VPValue *> ValueToBlendMask;
+    DenseMap<VPValue *, VPValue *> MaskForValue;
     for (unsigned I = 0; I < NumIncoming; I++) {
       VPValue *IncomingV = PhiR->getIncomingValue(I);
       VPBasicBlock *IncomingB = PhiR->getIncomingBlock(I);
       VPValue *IncomingM = DTO.MaskForEdge[{IncomingB, PHIBlock}];
       assert(IncomingM && "Missing edge mask?");
 
-      BlockHasDef[IncomingB] = IncomingV;
-      ValueToBlendMask[IncomingV] = IncomingM;
+      ValueForBlock[IncomingB] = IncomingV;
+      MaskForValue[IncomingV] = IncomingM;
     }
+
     SmallVector<VPValue *> BlendOperands;
     // WARNING: what if a value comes from a VPRegionBlock?
     for (auto *Block : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
-      if (BlockHasDef.contains(Block))
-        continue;
-
-      MapVector<VPValue *, VPBasicBlock *> Defs;
-      for (VPBlockBase *P : Block->predecessors()) {
-        if (BlockHasDef.contains(P))
-          Defs[BlockHasDef[P]] = cast<VPBasicBlock>(P);
+      // Store the value(s) coming from the predecessors.
+      VPValue *NewValueForBlock = nullptr;
+
+      SmallVector<std::pair<VPValue *, VPBasicBlock *>> NewPhiOps;
+      SmallVector<VPBlockBase *> Preds(Block->predecessors());
+      for (VPBlockBase *P : Preds) {
+        if (ValueForBlock.contains(P))
+          NewPhiOps.push_back({ValueForBlock[P], cast<VPBasicBlock>(P)});
       }
-      if (Defs.empty())
-        continue;
-      if (Defs.size() == 1)
-        BlockHasDef[Block] = Defs.takeVector()[0].first;
-      else {
-        // Check if all predecessors have an assigned value.
-        SmallVector<VPBlockBase *> Preds(Block->predecessors());
-        // Create a PHI of all incoming defs.
+      if (NewPhiOps.size() == 1)
+        NewValueForBlock = NewPhiOps[0].first;
+      else if (NewPhiOps.size() > 1) {
+        // Create the mask to be used as blend operand for the new phi.
+        VPValue *BlendMask = nullptr;
+        // Create the new phi.
         auto *NewPHI = new VPWidenPHIRecipe();
         NewPHI->insertBefore(&Block->front());
-        // Create the mask to be used as blend operand for this new PHI.
-        VPValue *BlendMask = nullptr;
-        VPBuilder VPlanBuilder(&*Block->getFirstNonPhi());
-        for (auto [V, B] : Defs) {
+        for (auto [V, B] : NewPhiOps) {
           assert(is_contained(Preds, B));
           Preds.erase(std::find(Preds.begin(), Preds.end(), B));
-
           NewPHI->addIncoming(V, B);
-
-          VPValue *IncomingM = ValueToBlendMask[V];
-          BlendMask =
-              BlendMask
-                  ? (IncomingM != BlendMask
-                         ? VPlanBuilder.createLogicalOr(BlendMask, IncomingM)
-                         : BlendMask)
-                  : IncomingM;
+          VPValue *IncomingM = MaskForValue[V];
+          if (!BlendMask)
+            BlendMask = IncomingM;
+          else if (IncomingM != BlendMask) {
+            BlendMask = new VPInstruction(VPInstruction::LogicalOr,
+                                          {BlendMask, IncomingM}, {},
+                                          "blend.mask.for.phi");
+          }
         }
-        // Assign the calculated blend mask to the new PHI.
-        ValueToBlendMask[NewPHI->getVPSingleValue()] = BlendMask;
-        // If not all predecessors define a value, add them with an undef one.
+        // If all predecessor define a value, the blend mask for the new phi is
+        // the mask of current block.
+        VPRecipeBase *DefR = BlendMask->getDefiningRecipe();
+        if (Preds.empty()) {
+          if (DefR && !DefR->getParent())
+            delete BlendMask;
+          BlendMask = DTO.MaskForBlock[Block];
+        } else {
+          if (DefR && !DefR->getParent())
+            DefR->insertBefore(&*Block->getFirstNonPhi());
+        }
+        // Use undef as value for predecessors that do not define one.
         for (VPBlockBase *MissingP : Preds) {
           VPValue *Undef =
               Region->getPlan()->getOrAddLiveIn(PoisonValue::get(PHIType));
           NewPHI->addIncoming(Undef, cast<VPBasicBlock>(MissingP));
         }
-        // If the new PHI dominates the to-be-replaced PHI, add it to blend's
-        // operands, together with the calculated blend mask.
-        if (DomTree.dominates(Block, PHIBlock)) {
-          BlendOperands.push_back(NewPHI->getVPSingleValue());
-          BlendOperands.push_back(BlendMask);
-          // NOTE: current Block will not have an assigned definition: since the
-          // new PHI dominates the to-be-replaced PHI, no further sinking is
-          // needed.
+        NewValueForBlock = NewPHI->getVPSingleValue();
+        // Assign the calculated blend mask to the new PHI.
+        MaskForValue[NewValueForBlock] = BlendMask;
+        VERBOSE_DEBUG(
+            dbgs() << "\t" << "Prepending "; cast<VPRecipeBase>(NewPHI)->dump();
+            dbgs() << "\t" << "\t" << "to block " << Block->getName() << ".\n");
+      }
+
+      if (NewValueForBlock && ValueForBlock.contains(Block)) {
+        // More than one value reaches the to-be-replaced phi's block through
+        // this one: create a blend to be used as the new value for current
+        // block.
+        SmallVector<VPValue *> BlendOps;
+        // Add current block value as first operand.
+        BlendOps.push_back(ValueForBlock[Block]);
+        if (auto *B = dyn_cast_if_present<VPBlendRecipe>(
+                NewValueForBlock->getDefiningRecipe())) {
+          // If the new value is a blend, it means:
+          // - current block only has a predecessor
+          // - all operands of the blend dominate it
+          assert(Block->getNumPredecessors() == 1);
+          for (unsigned int I = 0; I < B->getNumIncomingValues(); I++) {
+            VPValue *Op = B->getIncomingValue(I);
+            BlendOps.push_back(Op);               // Value
+            BlendOps.push_back(MaskForValue[Op]); // Mask
+          }
+          if (B->getNumUsers() == 0) {
+            VERBOSE_DEBUG(dbgs() << "\t" << "Erasing ";
+                          cast<VPRecipeBase>(B)->dump();
+                          dbgs() << "\t" << "\t" << "from block "
+                                 << B->getParent()->getName() << ".\n");
+            B->eraseFromParent();
+          }
         } else {
-          // Set the new PHI as value defined by this block.
-          BlockHasDef[Block] = NewPHI->getVPSingleValue();
+          BlendOps.push_back(NewValueForBlock);               // Value
+          BlendOps.push_back(MaskForValue[NewValueForBlock]); // Mask
+        }
+        auto *BlendR = new VPBlendRecipe(BlendOps);
+        Block->appendRecipe(BlendR);
+        VERBOSE_DEBUG(
+            dbgs() << "\t" << "Appending "; cast<VPRecipeBase>(BlendR)->dump();
+            dbgs() << "\t" << "\t" << "to block " << Block->getName() << ".\n");
+        NewValueForBlock = BlendR->getVPSingleValue();
+        // Assign to this new blend the same mask used by the value assigned to
+        // current block.
+        MaskForValue[NewValueForBlock] = MaskForValue[ValueForBlock[Block]];
+      }
+      if (NewValueForBlock)
+        ValueForBlock[Block] = NewValueForBlock;
+
+      if (!ValueForBlock.contains(Block))
+        continue;
+
+      if (DomTree.dominates(Block, PHIBlock)) {
+        // The value can be used directly in the (to-be-created) blend.
+        VPValue *BlockValue = ValueForBlock[Block];
+        if (BlendOperands.empty())
+          BlendOperands.push_back(BlockValue);
+        else {
+          BlendOperands.push_back(BlockValue);               // Value
+          BlendOperands.push_back(MaskForValue[BlockValue]); // Mask
         }
+        // Since no further sinking is needed, current block should not have a
+        // value assigned.
+        ValueForBlock.erase(Block);
       }
     }
 
-    auto *Blend = new VPBlendRecipe(BlendOperands);
-    ReplacePHIWithBlend[PhiR] = Blend;
+    assert(!BlendOperands.empty());
+    if (BlendOperands.size() == 1) {
+      PhiR->getVPSingleValue()->replaceAllUsesWith(BlendOperands[0]);
+      PhiR->eraseFromParent();
+      VERBOSE_DEBUG(dbgs() << "\t" << "Replacing with ";
+                    BlendOperands[0]->dump(););
+    } else {
+      auto *Blend = new VPBlendRecipe(BlendOperands);
+      ReplacePHIWithBlend[PhiR] = Blend;
+      VERBOSE_DEBUG(dbgs() << "\t" << "Replacing with ";
+                    cast<VPRecipeBase>(Blend)->dump(););
+    }
   }
 
   for (auto [PHI, Recipe] : ReplacePHIWithBlend) {
-    // Insert the blend just before the first user of the PHI: sometimes one of
-    // the mask operands can be defined in the same block of the blend.
+    // Insert the blend just before the first user of the phi: it may happen
+    // that one of the mask operands can be defined in the same block.
     VPRecipeBase *InsPt = nullptr;
     for (VPRecipeBase &R : *PHI->getParent()) {
       if (is_contained(PHI->users(), &R)) {
@@ -3112,7 +3191,7 @@ void VPlanTransforms::cfgLinearization(VPlanPtr &Plan,
   // After a partial if-conversion, BlockMask recipes may need to be updated.
   if (UniformBranchesPreserved > 0) {
     VERBOSE_DEBUG(
-        dbgs() << "\n---> Updating block masks after partial control-flow "
+        dbgs() << "---> Updating block masks after partial control-flow "
                   "linearization <---\n");
     VPDominatorTree VPDomTree;
     VPDomTree.recalculate(*Plan);
-- 
GitLab


From 1f2b7ae6d78906df4f0c06961e3c9ed227986acf Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Fri, 25 Oct 2024 12:28:55 -0400
Subject: [PATCH 177/693] [libc++] Refactor locale_guard (#113694)

Rename __libcpp_locale_guard to just __locale_guard, since there's no
reason for it to have __libcpp_ in its name -- it's just an internal
utility.

Also, define __locale_guard unconditionally of
_LIBCPP_LOCALE__L_EXTENSIONS, since that header is only used on Windows
(where it has a custom definition) or from bsd_locale_fallbacks.h, which
is only included when the L extensions are not provided.
---
 libcxx/include/CMakeLists.txt                 |  2 +-
 .../locale_base_api/bsd_locale_fallbacks.h    | 30 ++++++-------
 .../{locale_base_api => }/locale_guard.h      | 42 +++++++++----------
 libcxx/include/module.modulemap               |  2 +-
 libcxx/src/iostream.cpp                       |  4 +-
 libcxx/src/support/win32/locale_win32.cpp     | 34 +++++++--------
 6 files changed, 56 insertions(+), 58 deletions(-)
 rename libcxx/include/__locale_dir/{locale_base_api => }/locale_guard.h (73%)

diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 975adc03ec81..63aa74e09bb1 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -496,11 +496,11 @@ set(files
   __locale_dir/locale_base_api/bsd_locale_fallbacks.h
   __locale_dir/locale_base_api/fuchsia.h
   __locale_dir/locale_base_api/ibm.h
-  __locale_dir/locale_base_api/locale_guard.h
   __locale_dir/locale_base_api/musl.h
   __locale_dir/locale_base_api/newlib.h
   __locale_dir/locale_base_api/openbsd.h
   __locale_dir/locale_base_api/win32.h
+  __locale_dir/locale_guard.h
   __math/abs.h
   __math/copysign.h
   __math/error_functions.h
diff --git a/libcxx/include/__locale_dir/locale_base_api/bsd_locale_fallbacks.h b/libcxx/include/__locale_dir/locale_base_api/bsd_locale_fallbacks.h
index 5f99c7aea02a..ae2db6ae70be 100644
--- a/libcxx/include/__locale_dir/locale_base_api/bsd_locale_fallbacks.h
+++ b/libcxx/include/__locale_dir/locale_base_api/bsd_locale_fallbacks.h
@@ -13,7 +13,7 @@
 #ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
 #define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
 
-#include <__locale_dir/locale_base_api/locale_guard.h>
+#include <__locale_dir/locale_guard.h>
 #include <cstdio>
 #include <stdarg.h>
 #include <stdlib.h>
@@ -29,64 +29,64 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 inline _LIBCPP_HIDE_FROM_ABI decltype(MB_CUR_MAX) __libcpp_mb_cur_max_l(locale_t __l) {
-  __libcpp_locale_guard __current(__l);
+  __locale_guard __current(__l);
   return MB_CUR_MAX;
 }
 
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
 inline _LIBCPP_HIDE_FROM_ABI wint_t __libcpp_btowc_l(int __c, locale_t __l) {
-  __libcpp_locale_guard __current(__l);
+  __locale_guard __current(__l);
   return btowc(__c);
 }
 
 inline _LIBCPP_HIDE_FROM_ABI int __libcpp_wctob_l(wint_t __c, locale_t __l) {
-  __libcpp_locale_guard __current(__l);
+  __locale_guard __current(__l);
   return wctob(__c);
 }
 
 inline _LIBCPP_HIDE_FROM_ABI size_t
 __libcpp_wcsnrtombs_l(char* __dest, const wchar_t** __src, size_t __nwc, size_t __len, mbstate_t* __ps, locale_t __l) {
-  __libcpp_locale_guard __current(__l);
+  __locale_guard __current(__l);
   return wcsnrtombs(__dest, __src, __nwc, __len, __ps);
 }
 
 inline _LIBCPP_HIDE_FROM_ABI size_t __libcpp_wcrtomb_l(char* __s, wchar_t __wc, mbstate_t* __ps, locale_t __l) {
-  __libcpp_locale_guard __current(__l);
+  __locale_guard __current(__l);
   return wcrtomb(__s, __wc, __ps);
 }
 
 inline _LIBCPP_HIDE_FROM_ABI size_t
 __libcpp_mbsnrtowcs_l(wchar_t* __dest, const char** __src, size_t __nms, size_t __len, mbstate_t* __ps, locale_t __l) {
-  __libcpp_locale_guard __current(__l);
+  __locale_guard __current(__l);
   return mbsnrtowcs(__dest, __src, __nms, __len, __ps);
 }
 
 inline _LIBCPP_HIDE_FROM_ABI size_t
 __libcpp_mbrtowc_l(wchar_t* __pwc, const char* __s, size_t __n, mbstate_t* __ps, locale_t __l) {
-  __libcpp_locale_guard __current(__l);
+  __locale_guard __current(__l);
   return mbrtowc(__pwc, __s, __n, __ps);
 }
 
 inline _LIBCPP_HIDE_FROM_ABI int __libcpp_mbtowc_l(wchar_t* __pwc, const char* __pmb, size_t __max, locale_t __l) {
-  __libcpp_locale_guard __current(__l);
+  __locale_guard __current(__l);
   return mbtowc(__pwc, __pmb, __max);
 }
 
 inline _LIBCPP_HIDE_FROM_ABI size_t __libcpp_mbrlen_l(const char* __s, size_t __n, mbstate_t* __ps, locale_t __l) {
-  __libcpp_locale_guard __current(__l);
+  __locale_guard __current(__l);
   return mbrlen(__s, __n, __ps);
 }
 #endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
 
 inline _LIBCPP_HIDE_FROM_ABI lconv* __libcpp_localeconv_l(locale_t __l) {
-  __libcpp_locale_guard __current(__l);
+  __locale_guard __current(__l);
   return localeconv();
 }
 
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
 inline _LIBCPP_HIDE_FROM_ABI size_t
 __libcpp_mbsrtowcs_l(wchar_t* __dest, const char** __src, size_t __len, mbstate_t* __ps, locale_t __l) {
-  __libcpp_locale_guard __current(__l);
+  __locale_guard __current(__l);
   return mbsrtowcs(__dest, __src, __len, __ps);
 }
 #endif
@@ -95,7 +95,7 @@ inline _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 4, 5) int __libcpp_snprintf_l(
     char* __s, size_t __n, locale_t __l, const char* __format, ...) {
   va_list __va;
   va_start(__va, __format);
-  __libcpp_locale_guard __current(__l);
+  __locale_guard __current(__l);
   int __res = vsnprintf(__s, __n, __format, __va);
   va_end(__va);
   return __res;
@@ -105,7 +105,7 @@ inline _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 3, 4) int __libcpp_asprintf_l(
     char** __s, locale_t __l, const char* __format, ...) {
   va_list __va;
   va_start(__va, __format);
-  __libcpp_locale_guard __current(__l);
+  __locale_guard __current(__l);
   int __res = vasprintf(__s, __format, __va);
   va_end(__va);
   return __res;
@@ -115,7 +115,7 @@ inline _LIBCPP_ATTRIBUTE_FORMAT(__scanf__, 3, 4) int __libcpp_sscanf_l(
     const char* __s, locale_t __l, const char* __format, ...) {
   va_list __va;
   va_start(__va, __format);
-  __libcpp_locale_guard __current(__l);
+  __locale_guard __current(__l);
   int __res = vsscanf(__s, __format, __va);
   va_end(__va);
   return __res;
diff --git a/libcxx/include/__locale_dir/locale_base_api/locale_guard.h b/libcxx/include/__locale_dir/locale_guard.h
similarity index 73%
rename from libcxx/include/__locale_dir/locale_base_api/locale_guard.h
rename to libcxx/include/__locale_dir/locale_guard.h
index 7d15f2d253ad..e0c414c001c4 100644
--- a/libcxx/include/__locale_dir/locale_base_api/locale_guard.h
+++ b/libcxx/include/__locale_dir/locale_guard.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_LOCALE_GUARD_H
-#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_LOCALE_GUARD_H
+#ifndef _LIBCPP___LOCALE_DIR_LOCALE_GUARD_H
+#define _LIBCPP___LOCALE_DIR_LOCALE_GUARD_H
 
 #include <__config>
 #include <__locale> // for locale_t
@@ -19,23 +19,9 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if !defined(_LIBCPP_LOCALE__L_EXTENSIONS)
-struct __libcpp_locale_guard {
-  _LIBCPP_HIDE_FROM_ABI __libcpp_locale_guard(locale_t& __loc) : __old_loc_(uselocale(__loc)) {}
-
-  _LIBCPP_HIDE_FROM_ABI ~__libcpp_locale_guard() {
-    if (__old_loc_)
-      uselocale(__old_loc_);
-  }
-
-  locale_t __old_loc_;
-
-  __libcpp_locale_guard(__libcpp_locale_guard const&)            = delete;
-  __libcpp_locale_guard& operator=(__libcpp_locale_guard const&) = delete;
-};
-#elif defined(_LIBCPP_MSVCRT_LIKE)
-struct __libcpp_locale_guard {
-  __libcpp_locale_guard(locale_t __l) : __status(_configthreadlocale(_ENABLE_PER_THREAD_LOCALE)) {
+#if defined(_LIBCPP_MSVCRT_LIKE)
+struct __locale_guard {
+  __locale_guard(locale_t __l) : __status(_configthreadlocale(_ENABLE_PER_THREAD_LOCALE)) {
     // Setting the locale can be expensive even when the locale given is
     // already the current locale, so do an explicit check to see if the
     // current locale is already the one we want.
@@ -51,7 +37,7 @@ struct __libcpp_locale_guard {
       __setlocale(__l.__get_locale());
     }
   }
-  ~__libcpp_locale_guard() {
+  ~__locale_guard() {
     // The CRT documentation doesn't explicitly say, but setlocale() does the
     // right thing when given a semicolon-separated list of locale settings
     // for the different categories in the same format as returned by
@@ -71,8 +57,22 @@ struct __libcpp_locale_guard {
   int __status;
   char* __locale_all = nullptr;
 };
+#else
+struct __locale_guard {
+  _LIBCPP_HIDE_FROM_ABI __locale_guard(locale_t& __loc) : __old_loc_(uselocale(__loc)) {}
+
+  _LIBCPP_HIDE_FROM_ABI ~__locale_guard() {
+    if (__old_loc_)
+      uselocale(__old_loc_);
+  }
+
+  locale_t __old_loc_;
+
+  __locale_guard(__locale_guard const&)            = delete;
+  __locale_guard& operator=(__locale_guard const&) = delete;
+};
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_LOCALE_GUARD_H
+#endif // _LIBCPP___LOCALE_DIR_LOCALE_GUARD_H
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index b429d7cff702..c79070c31875 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1443,7 +1443,7 @@ module std [system] {
   module locale {
     header "locale"
     header "__locale_dir/locale_base_api.h"
-    header "__locale_dir/locale_base_api/locale_guard.h"
+    header "__locale_dir/locale_guard.h"
     module locale_base_api {
       textual header "__locale_dir/locale_base_api/android.h"
       textual header "__locale_dir/locale_base_api/bsd_locale_defaults.h"
diff --git a/libcxx/src/iostream.cpp b/libcxx/src/iostream.cpp
index c5ad77a01916..48d2fdb866a3 100644
--- a/libcxx/src/iostream.cpp
+++ b/libcxx/src/iostream.cpp
@@ -12,7 +12,7 @@
 #include <string>
 
 #ifdef _LIBCPP_MSVCRT_LIKE
-#  include <__locale_dir/locale_base_api/locale_guard.h>
+#  include <__locale_dir/locale_guard.h>
 #endif
 
 #define _str(s) #s
@@ -109,7 +109,7 @@ static void force_locale_initialization() {
   static bool once = []() {
     auto loc = newlocale(LC_ALL_MASK, "C", 0);
     {
-      __libcpp_locale_guard g(loc); // forces initialization of locale TLS
+      __locale_guard g(loc); // forces initialization of locale TLS
       ((void)g);
     }
     freelocale(loc);
diff --git a/libcxx/src/support/win32/locale_win32.cpp b/libcxx/src/support/win32/locale_win32.cpp
index 57ef94932ba0..2a08e97b8645 100644
--- a/libcxx/src/support/win32/locale_win32.cpp
+++ b/libcxx/src/support/win32/locale_win32.cpp
@@ -11,12 +11,10 @@
 #include <memory>
 #include <type_traits>
 
-#include <__locale_dir/locale_base_api/locale_guard.h>
+#include <__locale_dir/locale_guard.h>
 
 int __libcpp_vasprintf(char** sptr, const char* __restrict fmt, va_list ap);
 
-using std::__libcpp_locale_guard;
-
 // FIXME: base and mask currently unused. Needs manual work to construct the new locale
 locale_t newlocale(int /*mask*/, const char* locale, locale_t /*base*/) {
   return {_create_locale(LC_ALL, locale), locale};
@@ -26,33 +24,33 @@ decltype(MB_CUR_MAX) MB_CUR_MAX_L(locale_t __l) {
 #if defined(_LIBCPP_MSVCRT)
   return ___mb_cur_max_l_func(__l);
 #else
-  __libcpp_locale_guard __current(__l);
+  std::__locale_guard __current(__l);
   return MB_CUR_MAX;
 #endif
 }
 
 lconv* localeconv_l(locale_t& loc) {
-  __libcpp_locale_guard __current(loc);
+  std::__locale_guard __current(loc);
   lconv* lc = localeconv();
   if (!lc)
     return lc;
   return loc.__store_lconv(lc);
 }
 size_t mbrlen_l(const char* __restrict s, size_t n, mbstate_t* __restrict ps, locale_t loc) {
-  __libcpp_locale_guard __current(loc);
+  std::__locale_guard __current(loc);
   return mbrlen(s, n, ps);
 }
 size_t
 mbsrtowcs_l(wchar_t* __restrict dst, const char** __restrict src, size_t len, mbstate_t* __restrict ps, locale_t loc) {
-  __libcpp_locale_guard __current(loc);
+  std::__locale_guard __current(loc);
   return mbsrtowcs(dst, src, len, ps);
 }
 size_t wcrtomb_l(char* __restrict s, wchar_t wc, mbstate_t* __restrict ps, locale_t loc) {
-  __libcpp_locale_guard __current(loc);
+  std::__locale_guard __current(loc);
   return wcrtomb(s, wc, ps);
 }
 size_t mbrtowc_l(wchar_t* __restrict pwc, const char* __restrict s, size_t n, mbstate_t* __restrict ps, locale_t loc) {
-  __libcpp_locale_guard __current(loc);
+  std::__locale_guard __current(loc);
   return mbrtowc(pwc, s, n, ps);
 }
 size_t mbsnrtowcs_l(wchar_t* __restrict dst,
@@ -61,7 +59,7 @@ size_t mbsnrtowcs_l(wchar_t* __restrict dst,
                     size_t len,
                     mbstate_t* __restrict ps,
                     locale_t loc) {
-  __libcpp_locale_guard __current(loc);
+  std::__locale_guard __current(loc);
   return mbsnrtowcs(dst, src, nms, len, ps);
 }
 size_t wcsnrtombs_l(char* __restrict dst,
@@ -70,15 +68,15 @@ size_t wcsnrtombs_l(char* __restrict dst,
                     size_t len,
                     mbstate_t* __restrict ps,
                     locale_t loc) {
-  __libcpp_locale_guard __current(loc);
+  std::__locale_guard __current(loc);
   return wcsnrtombs(dst, src, nwc, len, ps);
 }
 wint_t btowc_l(int c, locale_t loc) {
-  __libcpp_locale_guard __current(loc);
+  std::__locale_guard __current(loc);
   return btowc(c);
 }
 int wctob_l(wint_t c, locale_t loc) {
-  __libcpp_locale_guard __current(loc);
+  std::__locale_guard __current(loc);
   return wctob(c);
 }
 
@@ -90,7 +88,7 @@ int snprintf_l(char* ret, size_t n, locale_t loc, const char* format, ...) {
   int result = __stdio_common_vsprintf(
       _CRT_INTERNAL_LOCAL_PRINTF_OPTIONS | _CRT_INTERNAL_PRINTF_STANDARD_SNPRINTF_BEHAVIOR, ret, n, format, loc, ap);
 #else
-  __libcpp_locale_guard __current(loc);
+  std::__locale_guard __current(loc);
   _LIBCPP_DIAGNOSTIC_PUSH
   _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wformat-nonliteral")
   int result = vsnprintf(ret, n, format, ap);
@@ -108,25 +106,25 @@ int asprintf_l(char** ret, locale_t loc, const char* format, ...) {
   return result;
 }
 int vasprintf_l(char** ret, locale_t loc, const char* format, va_list ap) {
-  __libcpp_locale_guard __current(loc);
+  std::__locale_guard __current(loc);
   return __libcpp_vasprintf(ret, format, ap);
 }
 
 #if !defined(_LIBCPP_MSVCRT)
 float strtof_l(const char* nptr, char** endptr, locale_t loc) {
-  __libcpp_locale_guard __current(loc);
+  std::__locale_guard __current(loc);
   return strtof(nptr, endptr);
 }
 
 long double strtold_l(const char* nptr, char** endptr, locale_t loc) {
-  __libcpp_locale_guard __current(loc);
+  std::__locale_guard __current(loc);
   return strtold(nptr, endptr);
 }
 #endif
 
 #if defined(__MINGW32__) && __MSVCRT_VERSION__ < 0x0800
 size_t strftime_l(char* ret, size_t n, const char* format, const struct tm* tm, locale_t loc) {
-  __libcpp_locale_guard __current(loc);
+  std::__locale_guard __current(loc);
   return strftime(ret, n, format, tm);
 }
 #endif
-- 
GitLab


From ba81e1949a4f25216e2b3ea3a1507a52db88562a Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Fri, 25 Oct 2024 17:32:25 +0100
Subject: [PATCH 178/693] [AArch64] Add assembly/disassembly for BFMOP4{A,S}
 (widening) instructions (#113203)

The new instructions are described in
https://developer.arm.com/documentation/ddi0602/2024-09/SME-Instructions
---
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |   3 +
 llvm/lib/Target/AArch64/SMEInstrFormats.td    |  37 +++
 .../SME2p2/bfmop4as-widening-diagnostics.s    | 220 ++++++++++++++++++
 .../MC/AArch64/SME2p2/bfmop4as-widening.s     | 178 ++++++++++++++
 4 files changed, 438 insertions(+)
 create mode 100644 llvm/test/MC/AArch64/SME2p2/bfmop4as-widening-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SME2p2/bfmop4as-widening.s

diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index 6044b5bb7d81..b763aa15a7c3 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -1004,6 +1004,9 @@ let Predicates = [HasSME2p2] in {
   def FTMOPA_M2ZZZI_HtoS  : sme_tmopa_32b<0b11000, ZZ_h_mul_r, ZPR16, "ftmopa">;
   def FTMOPA_M2ZZZI_StoS  : sme_tmopa_32b<0b00000, ZZ_s_mul_r, ZPR32, "ftmopa">;
   def BFTMOPA_M2ZZZI_HtoS : sme_tmopa_32b<0b10000, ZZ_h_mul_r, ZPR16, "bftmopa">;
+
+  defm BFMOP4A : sme2_bfmop4as_widening<0, "bfmop4a">;
+  defm BFMOP4S : sme2_bfmop4as_widening<1, "bfmop4s">;
 } // [HasSME2p2]
 
 let Predicates = [HasSME2p2, HasSMEB16B16] in {
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index 08929ed5616b..4cfe18eddf48 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -5188,3 +5188,40 @@ class sme2_luti4_vector_vg4_strided<bits<2> sz, bits<2> op, string mnemonic>
   let Inst{3-2}   = 0b00;
   let Inst{1-0}   = Zd{1-0};
 }
+
+class sme2_bf16_fp32_quarter_tile_outer_product<bit M, bit N, bit S, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>
+    : I<(outs TileOp32:$ZAda),
+        (ins TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),
+        mnemonic, "\t$ZAda, $Zn, $Zm",
+        "", []>, Sched<[]> {
+  bits<2> ZAda;
+  bits<3> Zn;
+  bits<3> Zm;
+
+  let Inst{31-21} = 0b10000001000;
+  let Inst{20} = M;
+  let Inst{19-17} = Zm;
+  let Inst{16-10} = 0b0000000;
+  let Inst{9} = N;
+  let Inst{8-6} = Zn;
+  let Inst{5} = 0;
+  let Inst{4} = S;
+  let Inst{3-2} = 0b00;
+  let Inst{1-0} = ZAda;
+
+  let Constraints = "$ZAda = $_ZAda";
+}
+
+multiclass sme2_bfmop4as_widening<bit S, string mnemonic> {
+  // Single vectors
+  def _MZZ_S : sme2_bf16_fp32_quarter_tile_outer_product<0, 0, S, mnemonic, ZPR16Mul2_Lo, ZPR16Mul2_Hi>;
+
+  // Multiple and single vectors
+  def _M2ZZ_S : sme2_bf16_fp32_quarter_tile_outer_product<0, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZPR16Mul2_Hi>;
+
+  // Single and multiple vectors
+  def _MZ2Z_S : sme2_bf16_fp32_quarter_tile_outer_product<1, 0, S, mnemonic, ZPR16Mul2_Lo, ZZ_h_mul_r_Hi>;
+
+  // Multiple vectors
+  def _M2Z2Z_S : sme2_bf16_fp32_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi>;
+}
diff --git a/llvm/test/MC/AArch64/SME2p2/bfmop4as-widening-diagnostics.s b/llvm/test/MC/AArch64/SME2p2/bfmop4as-widening-diagnostics.s
new file mode 100644
index 000000000000..5906bcb07f15
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/bfmop4as-widening-diagnostics.s
@@ -0,0 +1,220 @@
+// RUN: not llvm-mc -triple=aarch64 -mattr=+sme2p2 < %s 2>&1 | FileCheck %s
+
+// BFMOP4A
+
+// Single vectors
+
+bfmop4a za0.d, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4a za4.s, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4a za0.s, z0.s, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4a za0.s, z15.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4a za0.s, z16.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4a za0.s, z0.h, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4a za0.s, z12.h, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4a za0.s, z12.h, z14.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4a za0.s, z12.h, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Single and multiple vectors
+
+bfmop4a za0.d, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4a za4.s, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4a za0.s, z0.s, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4a za0.s, z1.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4a za0.s, z16.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4a za0.s, z0.h, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4a za0.s, z0.h, {z17.h-z18.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4a za0.s, z0.h, {z12.h-z13.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+// Multiple and single vectors
+
+bfmop4a za0.d, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4a za4.s, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4a za0.s, {z0.s-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: mismatched register size suffix
+
+bfmop4a za0.s, {z1.h-z2.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4a za0.s, {z16.h-z17.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4a za0.s, {z0.h-z1.h}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4a za0.s, {z0.h-z1.h}, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4a za0.s, {z0.h-z1.h}, z12.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Multiple vectors
+
+bfmop4a za0.d, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4a za4.s, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4a za0.s, {z0.s-z1.s}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4a za0.s, {z1.h-z2.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4a za0.s, {z18.h-z19.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4a za0.s, {z0.h-z1.h}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4a za0.s, {z0.h-z1.h}, {z19.h-z20.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4a za0.s, {z0.h-z1.h}, {z10.h-z11.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+
+// BFMOP4S
+
+// Single vectors
+
+bfmop4s za0.d, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4s za4.s, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4s za0.s, z0.s, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4s za0.s, z15.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4s za0.s, z16.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4s za0.s, z0.h, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4s za0.s, z12.h, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4s za0.s, z12.h, z14.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4s za0.s, z12.h, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Single and multiple vectors
+
+bfmop4s za0.d, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4s za4.s, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4s za0.s, z0.s, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4s za0.s, z1.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4s za0.s, z16.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4s za0.s, z0.h, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4s za0.s, z0.h, {z17.h-z18.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4s za0.s, z0.h, {z12.h-z13.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+// Multiple and single vectors
+
+bfmop4s za0.d, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4s za4.s, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4s za0.s, {z0.s-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: mismatched register size suffix
+
+bfmop4s za0.s, {z1.h-z2.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4s za0.s, {z16.h-z17.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4s za0.s, {z0.h-z1.h}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4s za0.s, {z0.h-z1.h}, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4s za0.s, {z0.h-z1.h}, z12.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Multiple vectors
+
+bfmop4s za0.d, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4s za4.s, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4s za0.s, {z0.s-z1.s}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4s za0.s, {z1.h-z2.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4s za0.s, {z18.h-z19.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4s za0.s, {z0.h-z1.h}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4s za0.s, {z0.h-z1.h}, {z19.h-z20.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4s za0.s, {z0.h-z1.h}, {z10.h-z11.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
diff --git a/llvm/test/MC/AArch64/SME2p2/bfmop4as-widening.s b/llvm/test/MC/AArch64/SME2p2/bfmop4as-widening.s
new file mode 100644
index 000000000000..40d08e503c8b
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/bfmop4as-widening.s
@@ -0,0 +1,178 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sme2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2p2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// BFMOP4A
+
+// Single vectors
+
+bfmop4a za0.s, z0.h, z16.h  // 10000001-00000000-00000000-00000000
+// CHECK-INST: bfmop4a za0.s, z0.h, z16.h
+// CHECK-ENCODING: [0x00,0x00,0x00,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81000000 <unknown>
+
+bfmop4a za3.s, z14.h, z30.h  // 10000001-00001110-00000001-11000011
+// CHECK-INST: bfmop4a za3.s, z14.h, z30.h
+// CHECK-ENCODING: [0xc3,0x01,0x0e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 810e01c3 <unknown>
+
+bfmop4a za1.s, z10.h, z20.h  // 10000001-00000100-00000001-01000001
+// CHECK-INST: bfmop4a za1.s, z10.h, z20.h
+// CHECK-ENCODING: [0x41,0x01,0x04,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81040141 <unknown>
+
+// Single and multiple vectors
+
+bfmop4a za0.s, z0.h, {z16.h-z17.h}  // 10000001-00010000-00000000-00000000
+// CHECK-INST: bfmop4a za0.s, z0.h, { z16.h, z17.h }
+// CHECK-ENCODING: [0x00,0x00,0x10,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81100000 <unknown>
+
+bfmop4a za3.s, z14.h, {z30.h-z31.h}  // 10000001-00011110-00000001-11000011
+// CHECK-INST: bfmop4a za3.s, z14.h, { z30.h, z31.h }
+// CHECK-ENCODING: [0xc3,0x01,0x1e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 811e01c3 <unknown>
+
+bfmop4a za2.s, z12.h, {z24.h-z25.h}  // 10000001-00011000-00000001-10000010
+// CHECK-INST: bfmop4a za2.s, z12.h, { z24.h, z25.h }
+// CHECK-ENCODING: [0x82,0x01,0x18,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81180182 <unknown>
+
+// Multiple and single vectors
+
+bfmop4a za0.s, {z0.h-z1.h}, z16.h  // 10000001-00000000-00000010-00000000
+// CHECK-INST: bfmop4a za0.s, { z0.h, z1.h }, z16.h
+// CHECK-ENCODING: [0x00,0x02,0x00,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81000200 <unknown>
+
+bfmop4a za3.s, {z14.h-z15.h}, z30.h  // 10000001-00001110-00000011-11000011
+// CHECK-INST: bfmop4a za3.s, { z14.h, z15.h }, z30.h
+// CHECK-ENCODING: [0xc3,0x03,0x0e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 810e03c3 <unknown>
+
+bfmop4a za2.s, {z12.h-z13.h}, z28.h  // 10000001-00001100-00000011-10000010
+// CHECK-INST: bfmop4a za2.s, { z12.h, z13.h }, z28.h
+// CHECK-ENCODING: [0x82,0x03,0x0c,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 810c0382 <unknown>
+
+// Multiple vectors
+
+bfmop4a za0.s, {z0.h-z1.h}, {z16.h-z17.h}  // 10000001-00010000-00000010-00000000
+// CHECK-INST: bfmop4a za0.s, { z0.h, z1.h }, { z16.h, z17.h }
+// CHECK-ENCODING: [0x00,0x02,0x10,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81100200 <unknown>
+
+bfmop4a za3.s, {z14.h-z15.h}, {z30.h-z31.h}  // 10000001-00011110-00000011-11000011
+// CHECK-INST: bfmop4a za3.s, { z14.h, z15.h }, { z30.h, z31.h }
+// CHECK-ENCODING: [0xc3,0x03,0x1e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 811e03c3 <unknown>
+
+bfmop4a za2.s, {z12.h-z13.h}, {z26.h-z27.h}  // 10000001-00011010-00000011-10000010
+// CHECK-INST: bfmop4a za2.s, { z12.h, z13.h }, { z26.h, z27.h }
+// CHECK-ENCODING: [0x82,0x03,0x1a,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 811a0382 <unknown>
+
+
+// BFMOP4S
+
+// Single vectors
+
+bfmop4s za0.s, z0.h, z16.h  // 10000001-00000000-00000000-00010000
+// CHECK-INST: bfmop4s za0.s, z0.h, z16.h
+// CHECK-ENCODING: [0x10,0x00,0x00,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81000010 <unknown>
+
+bfmop4s za3.s, z14.h, z30.h  // 10000001-00001110-00000001-11010011
+// CHECK-INST: bfmop4s za3.s, z14.h, z30.h
+// CHECK-ENCODING: [0xd3,0x01,0x0e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 810e01d3 <unknown>
+
+bfmop4s za1.s, z10.h, z20.h  // 10000001-00000100-00000001-01010001
+// CHECK-INST: bfmop4s za1.s, z10.h, z20.h
+// CHECK-ENCODING: [0x51,0x01,0x04,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81040151 <unknown>
+
+// Single and multiple vectors
+
+bfmop4s za0.s, z0.h, {z16.h-z17.h}  // 10000001-00010000-00000000-00010000
+// CHECK-INST: bfmop4s za0.s, z0.h, { z16.h, z17.h }
+// CHECK-ENCODING: [0x10,0x00,0x10,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81100010 <unknown>
+
+bfmop4s za3.s, z14.h, {z30.h-z31.h}  // 10000001-00011110-00000001-11010011
+// CHECK-INST: bfmop4s za3.s, z14.h, { z30.h, z31.h }
+// CHECK-ENCODING: [0xd3,0x01,0x1e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 811e01d3 <unknown>
+
+bfmop4s za2.s, z12.h, {z24.h-z25.h}  // 10000001-00011000-00000001-10010010
+// CHECK-INST: bfmop4s za2.s, z12.h, { z24.h, z25.h }
+// CHECK-ENCODING: [0x92,0x01,0x18,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81180192 <unknown>
+
+// Multiple and single vectors
+
+bfmop4s za0.s, {z0.h-z1.h}, z16.h  // 10000001-00000000-00000010-00010000
+// CHECK-INST: bfmop4s za0.s, { z0.h, z1.h }, z16.h
+// CHECK-ENCODING: [0x10,0x02,0x00,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81000210 <unknown>
+
+bfmop4s za3.s, {z14.h-z15.h}, z30.h  // 10000001-00001110-00000011-11010011
+// CHECK-INST: bfmop4s za3.s, { z14.h, z15.h }, z30.h
+// CHECK-ENCODING: [0xd3,0x03,0x0e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 810e03d3 <unknown>
+
+bfmop4s za2.s, {z12.h-z13.h}, z28.h  // 10000001-00001100-00000011-10010010
+// CHECK-INST: bfmop4s za2.s, { z12.h, z13.h }, z28.h
+// CHECK-ENCODING: [0x92,0x03,0x0c,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 810c0392 <unknown>
+
+// Multiple vectors
+
+bfmop4s za0.s, {z0.h-z1.h}, {z16.h-z17.h}  // 10000001-00010000-00000010-00010000
+// CHECK-INST: bfmop4s za0.s, { z0.h, z1.h }, { z16.h, z17.h }
+// CHECK-ENCODING: [0x10,0x02,0x10,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81100210 <unknown>
+
+bfmop4s za3.s, {z14.h-z15.h}, {z30.h-z31.h}  // 10000001-00011110-00000011-11010011
+// CHECK-INST: bfmop4s za3.s, { z14.h, z15.h }, { z30.h, z31.h }
+// CHECK-ENCODING: [0xd3,0x03,0x1e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 811e03d3 <unknown>
+
+bfmop4s za2.s, {z12.h-z13.h}, {z26.h-z27.h}  // 10000001-00011010-00000011-10010010
+// CHECK-INST: bfmop4s za2.s, { z12.h, z13.h }, { z26.h, z27.h }
+// CHECK-ENCODING: [0x92,0x03,0x1a,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 811a0392 <unknown>
-- 
GitLab


From 4161ca2092d3b92034515190f577aa200ec615bf Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray@arm.com>
Date: Fri, 25 Oct 2024 14:54:56 +0100
Subject: [PATCH 179/693] [NFC][AArch64][LLVM] Update ReleaseNotes.md with
 Armv9.6-A (2024) arch extensions

---
 llvm/docs/ReleaseNotes.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 7cca9116a513..be51b0af56dd 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -110,6 +110,9 @@ Changes to the AArch64 Backend
   the required alignment space with a sequence of `0x0` bytes (the requested
   fill value) rather than NOPs.
 
+* Assembler/disassembler support has been added for Armv9.6-A (2024)
+  architecture extensions.
+
 Changes to the AMDGPU Backend
 -----------------------------
 
-- 
GitLab


From 2ec5c69b6872b8b474f3d37b9125d3d57d144d1b Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer@google.com>
Date: Fri, 25 Oct 2024 09:42:01 -0700
Subject: [PATCH 180/693] Revert "[Sanitizers] Intercept timer_create"
 (#113710)

Reverts llvm/llvm-project#112285
---
 .../lib/hwasan/hwasan_platform_interceptors.h |  3 ---
 compiler-rt/lib/msan/tests/msan_test.cpp      | 23 -------------------
 .../sanitizer_common_interceptors.inc         | 19 ---------------
 .../sanitizer_platform_interceptors.h         |  3 ---
 .../sanitizer_platform_limits_posix.h         |  4 ----
 5 files changed, 52 deletions(-)

diff --git a/compiler-rt/lib/hwasan/hwasan_platform_interceptors.h b/compiler-rt/lib/hwasan/hwasan_platform_interceptors.h
index e8011014c233..d92b51052194 100644
--- a/compiler-rt/lib/hwasan/hwasan_platform_interceptors.h
+++ b/compiler-rt/lib/hwasan/hwasan_platform_interceptors.h
@@ -200,9 +200,6 @@
 #undef SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID
 #define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID 0
 
-#undef SANITIZER_INTERCEPT_TIMER_CREATE
-#define SANITIZER_INTERCEPT_TIMER_CREATE 0
-
 #undef SANITIZER_INTERCEPT_GETITIMER
 #define SANITIZER_INTERCEPT_GETITIMER 0
 
diff --git a/compiler-rt/lib/msan/tests/msan_test.cpp b/compiler-rt/lib/msan/tests/msan_test.cpp
index ad265acf4c1e..41b99fabe84f 100644
--- a/compiler-rt/lib/msan/tests/msan_test.cpp
+++ b/compiler-rt/lib/msan/tests/msan_test.cpp
@@ -4881,27 +4881,4 @@ TEST(MemorySanitizer, throw_catch) {
     // pass
   }
 }
-
-#if defined(__linux__)
-TEST(MemorySanitizer, timer_create) {
-  timer_t timer;
-  EXPECT_POISONED(timer);
-  int res = timer_create(CLOCK_REALTIME, nullptr, &timer);
-  ASSERT_EQ(0, res);
-  EXPECT_NOT_POISONED(timer);
-
-  // Make sure the timer is usable.
-  struct itimerspec cur_value {};
-  cur_value.it_value.tv_sec = 1;
-  EXPECT_EQ(0, timer_settime(timer, 0, &cur_value, nullptr));
-
-  timer_t timer2;
-  EXPECT_POISONED(timer2);
-  // Use an invalid clock_id to make timer_create fail.
-  res = timer_create(INT_MAX, nullptr, &timer2);
-  ASSERT_EQ(-1, res);
-  EXPECT_POISONED(timer2);
-  timer_delete(timer);
-}
-#endif
 } // namespace
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index 211f9f70d7e4..b8627f8557af 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -2289,24 +2289,6 @@ INTERCEPTOR(int, pthread_getcpuclockid, uptr thread,
 #define INIT_CLOCK_GETCPUCLOCKID
 #endif
 
-#if SANITIZER_INTERCEPT_TIMER_CREATE
-INTERCEPTOR(int, timer_create, __sanitizer_clockid_t clockid, void *sevp,
-            __sanitizer_timer_t *timer) {
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, timer_create, clockid, sevp, timer);
-  int res = REAL(timer_create)(clockid, sevp, timer);
-  if (!res && timer) {
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, timer, sizeof *timer);
-  }
-  return res;
-}
-
-#  define INIT_TIMER_CREATE \
-    COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(timer_create, "GLIBC_2.3.3");
-#else
-#  define INIT_TIMER_CREATE
-#endif
-
 #if SANITIZER_INTERCEPT_GETITIMER
 INTERCEPTOR(int, getitimer, int which, void *curr_value) {
   void *ctx;
@@ -10284,7 +10266,6 @@ static void InitializeCommonInterceptors() {
   INIT_SETPWENT;
   INIT_CLOCK_GETTIME;
   INIT_CLOCK_GETCPUCLOCKID;
-  INIT_TIMER_CREATE;
   INIT_GETITIMER;
   INIT_TIME;
   INIT_GLOB;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
index 36fafdc64264..6959a6d52d60 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
@@ -237,9 +237,6 @@
   (SI_FREEBSD || SI_NETBSD || SI_LINUX || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID \
   (SI_LINUX || SI_FREEBSD || SI_NETBSD)
-// TODO: This should be SI_POSIX, adding Linux first until I have time
-// to verify all timer_t typedefs on other platforms.
-#define SANITIZER_INTERCEPT_TIMER_CREATE SI_LINUX
 #define SANITIZER_INTERCEPT_GETITIMER SI_POSIX
 #define SANITIZER_INTERCEPT_TIME SI_POSIX
 #define SANITIZER_INTERCEPT_GLOB (SI_GLIBC || SI_SOLARIS)
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
index b4ccf7b3d7be..e8c81aa8e281 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
@@ -1517,10 +1517,6 @@ extern const int si_SEGV_ACCERR;
 
 #define SIGACTION_SYMNAME sigaction
 
-#  if SANITIZER_LINUX
-typedef void *__sanitizer_timer_t;
-#  endif
-
 #endif  // SANITIZER_LINUX || SANITIZER_APPLE
 
 #endif
-- 
GitLab


From 9ea6fcd02b172ec12c9d4b9157d4a37765d83421 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 25 Oct 2024 16:47:08 +0000
Subject: [PATCH 181/693] [gn build] Port 1f2b7ae6d789

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 1630c8004d31..0586704850a5 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -568,11 +568,11 @@ if (current_toolchain == default_toolchain) {
       "__locale_dir/locale_base_api/bsd_locale_fallbacks.h",
       "__locale_dir/locale_base_api/fuchsia.h",
       "__locale_dir/locale_base_api/ibm.h",
-      "__locale_dir/locale_base_api/locale_guard.h",
       "__locale_dir/locale_base_api/musl.h",
       "__locale_dir/locale_base_api/newlib.h",
       "__locale_dir/locale_base_api/openbsd.h",
       "__locale_dir/locale_base_api/win32.h",
+      "__locale_dir/locale_guard.h",
       "__math/abs.h",
       "__math/copysign.h",
       "__math/error_functions.h",
-- 
GitLab


From 305a1ceae371b482375545650ba9fd9e4c165157 Mon Sep 17 00:00:00 2001
From: Alexander Richardson <alexrichardson@google.com>
Date: Fri, 25 Oct 2024 10:02:40 -0700
Subject: [PATCH 182/693] [DataLayout] Refactor storage of non-integral address
 spaces

Instead of storing this as a separate array of non-integral pointers,
add it to the PointerSpec class instead. This will allow for future
simplifications such as splitting the non-integral property into
multiple distinct ones: relocatable (i.e. non-stable representation) and
non-integral representation (i.e. pointers with metadata).

Reviewed By: arsenm

Pull Request: https://github.com/llvm/llvm-project/pull/105734
---
 llvm/include/llvm/IR/DataLayout.h | 28 ++++++++++++++++-----------
 llvm/lib/IR/DataLayout.cpp        | 32 ++++++++++++++++++++++---------
 2 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h
index 8f7ab2f9df38..93bd519f5727 100644
--- a/llvm/include/llvm/IR/DataLayout.h
+++ b/llvm/include/llvm/IR/DataLayout.h
@@ -78,7 +78,11 @@ public:
     Align ABIAlign;
     Align PrefAlign;
     uint32_t IndexBitWidth;
-
+    /// Pointers in this address space don't have a well-defined bitwise
+    /// representation (e.g. may be relocated by a copying garbage collector).
+    /// Additionally, they may also be non-integral (i.e. containing additional
+    /// metadata such as bounds information/permissions).
+    bool IsNonIntegral;
     bool operator==(const PointerSpec &Other) const;
   };
 
@@ -133,10 +137,6 @@ private:
   // The StructType -> StructLayout map.
   mutable void *LayoutMap = nullptr;
 
-  /// Pointers in these address spaces are non-integral, and don't have a
-  /// well-defined bitwise representation.
-  SmallVector<unsigned, 8> NonIntegralAddressSpaces;
-
   /// Sets or updates the specification for the given primitive type.
   void setPrimitiveSpec(char Specifier, uint32_t BitWidth, Align ABIAlign,
                         Align PrefAlign);
@@ -147,7 +147,8 @@ private:
 
   /// Sets or updates the specification for pointer in the given address space.
   void setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth, Align ABIAlign,
-                      Align PrefAlign, uint32_t IndexBitWidth);
+                      Align PrefAlign, uint32_t IndexBitWidth,
+                      bool IsNonIntegral);
 
   /// Internal helper to get alignment for integer of given bitwidth.
   Align getIntegerAlignment(uint32_t BitWidth, bool abi_or_pref) const;
@@ -165,7 +166,8 @@ private:
   Error parsePointerSpec(StringRef Spec);
 
   /// Attempts to parse a single specification.
-  Error parseSpecification(StringRef Spec);
+  Error parseSpecification(StringRef Spec,
+                           SmallVectorImpl<unsigned> &NonIntegralAddressSpaces);
 
   /// Attempts to parse a data layout string.
   Error parseLayoutString(StringRef LayoutString);
@@ -337,13 +339,17 @@ public:
 
   /// Return the address spaces containing non-integral pointers.  Pointers in
   /// this address space don't have a well-defined bitwise representation.
-  ArrayRef<unsigned> getNonIntegralAddressSpaces() const {
-    return NonIntegralAddressSpaces;
+  SmallVector<unsigned, 8> getNonIntegralAddressSpaces() const {
+    SmallVector<unsigned, 8> AddrSpaces;
+    for (const PointerSpec &PS : PointerSpecs) {
+      if (PS.IsNonIntegral)
+        AddrSpaces.push_back(PS.AddrSpace);
+    }
+    return AddrSpaces;
   }
 
   bool isNonIntegralAddressSpace(unsigned AddrSpace) const {
-    ArrayRef<unsigned> NonIntegralSpaces = getNonIntegralAddressSpaces();
-    return is_contained(NonIntegralSpaces, AddrSpace);
+    return getPointerSpec(AddrSpace).IsNonIntegral;
   }
 
   bool isNonIntegralPointerType(PointerType *PT) const {
diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index d295d1f5785e..a4af0ead07cf 100644
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -151,7 +151,8 @@ bool DataLayout::PrimitiveSpec::operator==(const PrimitiveSpec &Other) const {
 bool DataLayout::PointerSpec::operator==(const PointerSpec &Other) const {
   return AddrSpace == Other.AddrSpace && BitWidth == Other.BitWidth &&
          ABIAlign == Other.ABIAlign && PrefAlign == Other.PrefAlign &&
-         IndexBitWidth == Other.IndexBitWidth;
+         IndexBitWidth == Other.IndexBitWidth &&
+         IsNonIntegral == Other.IsNonIntegral;
 }
 
 namespace {
@@ -206,7 +207,8 @@ constexpr DataLayout::PrimitiveSpec DefaultVectorSpecs[] = {
 
 // Default pointer type specifications.
 constexpr DataLayout::PointerSpec DefaultPointerSpecs[] = {
-    {0, 64, Align::Constant<8>(), Align::Constant<8>(), 64} // p0:64:64:64:64
+    // p0:64:64:64:64
+    {0, 64, Align::Constant<8>(), Align::Constant<8>(), 64, false},
 };
 
 DataLayout::DataLayout()
@@ -239,13 +241,11 @@ DataLayout &DataLayout::operator=(const DataLayout &Other) {
   PointerSpecs = Other.PointerSpecs;
   StructABIAlignment = Other.StructABIAlignment;
   StructPrefAlignment = Other.StructPrefAlignment;
-  NonIntegralAddressSpaces = Other.NonIntegralAddressSpaces;
   return *this;
 }
 
 bool DataLayout::operator==(const DataLayout &Other) const {
   // NOTE: StringRepresentation might differ, it is not canonicalized.
-  // FIXME: NonIntegralAddressSpaces isn't compared.
   return BigEndian == Other.BigEndian &&
          AllocaAddrSpace == Other.AllocaAddrSpace &&
          ProgramAddrSpace == Other.ProgramAddrSpace &&
@@ -454,11 +454,13 @@ Error DataLayout::parsePointerSpec(StringRef Spec) {
     return createStringError(
         "index size cannot be larger than the pointer size");
 
-  setPointerSpec(AddrSpace, BitWidth, ABIAlign, PrefAlign, IndexBitWidth);
+  setPointerSpec(AddrSpace, BitWidth, ABIAlign, PrefAlign, IndexBitWidth,
+                 false);
   return Error::success();
 }
 
-Error DataLayout::parseSpecification(StringRef Spec) {
+Error DataLayout::parseSpecification(
+    StringRef Spec, SmallVectorImpl<unsigned> &NonIntegralAddressSpaces) {
   // The "ni" specifier is the only two-character specifier. Handle it first.
   if (Spec.starts_with("ni")) {
     // ni:<address space>[:<address space>]...
@@ -614,12 +616,23 @@ Error DataLayout::parseLayoutString(StringRef LayoutString) {
 
   // Split the data layout string into specifications separated by '-' and
   // parse each specification individually, updating internal data structures.
+  SmallVector<unsigned, 8> NonIntegralAddressSpaces;
   for (StringRef Spec : split(LayoutString, '-')) {
     if (Spec.empty())
       return createStringError("empty specification is not allowed");
-    if (Error Err = parseSpecification(Spec))
+    if (Error Err = parseSpecification(Spec, NonIntegralAddressSpaces))
       return Err;
   }
+  // Mark all address spaces that were qualified as non-integral now. This has
+  // to be done later since the non-integral property is not part of the data
+  // layout pointer specification.
+  for (unsigned AS : NonIntegralAddressSpaces) {
+    // If there is no special spec for a given AS, getPointerSpec(AS) returns
+    // the spec for AS0, and we then update that to mark it non-integral.
+    const PointerSpec &PS = getPointerSpec(AS);
+    setPointerSpec(AS, PS.BitWidth, PS.ABIAlign, PS.PrefAlign, PS.IndexBitWidth,
+                   true);
+  }
 
   return Error::success();
 }
@@ -666,16 +679,17 @@ DataLayout::getPointerSpec(uint32_t AddrSpace) const {
 
 void DataLayout::setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth,
                                 Align ABIAlign, Align PrefAlign,
-                                uint32_t IndexBitWidth) {
+                                uint32_t IndexBitWidth, bool IsNonIntegral) {
   auto I = lower_bound(PointerSpecs, AddrSpace, LessPointerAddrSpace());
   if (I == PointerSpecs.end() || I->AddrSpace != AddrSpace) {
     PointerSpecs.insert(I, PointerSpec{AddrSpace, BitWidth, ABIAlign, PrefAlign,
-                                       IndexBitWidth});
+                                       IndexBitWidth, IsNonIntegral});
   } else {
     I->BitWidth = BitWidth;
     I->ABIAlign = ABIAlign;
     I->PrefAlign = PrefAlign;
     I->IndexBitWidth = IndexBitWidth;
+    I->IsNonIntegral = IsNonIntegral;
   }
 }
 
-- 
GitLab


From 9d88543301f262e584a36ea969237a2cf054328b Mon Sep 17 00:00:00 2001
From: Abhina Sree <Abhina.Sreeskantharajan@ibm.com>
Date: Fri, 25 Oct 2024 13:06:02 -0400
Subject: [PATCH 183/693] [AIX] Use internal lit shell for TableGen instead of
 a global setting (#113627)

This is to address the latest lit regressions
https://lab.llvm.org/buildbot/#/builders/64/builds/1285 caused by using
the internal lit shell. This change will limit using the internal lit
shell to TableGen on AIX so we do not hit these regressions.
---
 llvm/test/TableGen/lit.local.cfg  | 8 ++++++++
 llvm/utils/lit/lit/llvm/config.py | 7 -------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/llvm/test/TableGen/lit.local.cfg b/llvm/test/TableGen/lit.local.cfg
index 0e827479cd41..9d6dfdc14bbf 100644
--- a/llvm/test/TableGen/lit.local.cfg
+++ b/llvm/test/TableGen/lit.local.cfg
@@ -1,2 +1,10 @@
+import platform
+import lit.formats
+
 config.suffixes = [".td"]
 config.excludes = ["Common", "Inputs"]
+
+# AIX 'diff' command doesn't support --strip-trailing-cr, but the internal
+# python implementation does, so use that for cross platform compatibility
+if platform.system() == "AIX":
+    config.test_format = lit.formats.ShTest()
diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py
index 1ef5796cd32e..5f762ec7f351 100644
--- a/llvm/utils/lit/lit/llvm/config.py
+++ b/llvm/utils/lit/lit/llvm/config.py
@@ -57,13 +57,6 @@ class LLVMConfig(object):
                 self.lit_config.note("using lit tools: {}".format(path))
                 lit_path_displayed = True
 
-        if platform.system() == "AIX":
-            # Diff on AIX doesn't have all the required features (see
-            # https://github.com/llvm/llvm-project/pull/108871 and
-            # https://github.com/llvm/llvm-project/pull/112997#issuecomment-2429656192)
-            # so always use the internal shell.
-            self.use_lit_shell = True
-
         if platform.system() == "OS/390":
             self.with_environment("_BPXK_AUTOCVT", "ON")
             self.with_environment("_TAG_REDIR_IN", "TXT")
-- 
GitLab


From f24c1dd08ea71fa7334a85fd2772c2f728de0c56 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 25 Oct 2024 18:11:20 +0100
Subject: [PATCH 184/693] Fix MSVC "signed/unsigned mismatch" warning. NFC.

---
 clang/tools/clang-format/ClangFormat.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/tools/clang-format/ClangFormat.cpp b/clang/tools/clang-format/ClangFormat.cpp
index 96fb85e99bf5..5522d05744a2 100644
--- a/clang/tools/clang-format/ClangFormat.cpp
+++ b/clang/tools/clang-format/ClangFormat.cpp
@@ -510,7 +510,7 @@ static bool format(StringRef FileName, bool ErrorOnIncompleteFormat = false) {
       reformat(*FormatStyle, *ChangedCode, Ranges, AssumedFileName, &Status);
   Replaces = Replaces.merge(FormatChanges);
   if (DryRun) {
-    return Replaces.size() > (IsJson ? 1 : 0) &&
+    return Replaces.size() > (IsJson ? 1u : 0u) &&
            emitReplacementWarnings(Replaces, AssumedFileName, Code);
   }
   if (OutputXML) {
-- 
GitLab


From e6917e95548f81e7f00b8bca70ce571780e2afc9 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 25 Oct 2024 21:15:21 +0400
Subject: [PATCH 185/693] =?UTF-8?q?[clang][NFC]=20Add=20test=20for=20CWG18?=
 =?UTF-8?q?98=20"Use=20of=20=E2=80=9Cequivalent=E2=80=9D=20in=20overload?=
 =?UTF-8?q?=20resolution"=20(#113439)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

[CWG1898](https://cplusplus.github.io/CWG/issues/1898.html) Use of
“equivalent” in overload resolution
====================

[P1787R6](https://wg21.link/p1787r6):
> CWG1898 is resolved by explicitly using the defined term
parameter-type-list.

Except that now it's called non-object-parameter-type-list, which is
defined in [dcl.fct] [p8](https://eel.is/c++draft/dcl.fct#8) and
[p4](https://eel.is/c++draft/dcl.fct#8).

As for the wording, the first sentence
[\_N4140\_.[over.dcl]/1](https://timsong-cpp.github.io/cppwp/n4140/over.dcl#1)
where the word "equivalent" was used:
> Two function declarations of the same name refer to the same function
if they are in the same scope and have equivalent parameter declarations
([over.load]).

was replaced with what is now known as "corresponding overloads",
defined in
[[basic.scope.scope]/4](https://eel.is/c++draft/basic.scope#scope-4).
The definition is present in P1787R6, but it's hard to reference,
because the "corresponding overloads" term was coined later.
---
 clang/test/CXX/drs/cwg18xx.cpp | 83 ++++++++++++++++++++++++++++++++++
 clang/www/cxx_dr_status.html   |  2 +-
 2 files changed, 84 insertions(+), 1 deletion(-)

diff --git a/clang/test/CXX/drs/cwg18xx.cpp b/clang/test/CXX/drs/cwg18xx.cpp
index 7f0fb8cf589d..b059492637bd 100644
--- a/clang/test/CXX/drs/cwg18xx.cpp
+++ b/clang/test/CXX/drs/cwg18xx.cpp
@@ -640,3 +640,86 @@ namespace H {
   struct S s;
 }
 }
+
+namespace cwg1898 { // cwg1898: 2.7
+void e(int) {} // #cwg1898-e
+void e(int) {}
+// expected-error@-1 {{redefinition of 'e'}}
+//   expected-note@#cwg1898-e {{previous definition is here}}
+
+void e2(int) {}
+void e2(long) {} // OK, different type
+
+void f(int) {} // #cwg1898-f
+void f(const int) {}
+// expected-error@-1 {{redefinition of 'f'}}
+//   expected-note@#cwg1898-f {{previous definition is here}}
+
+void g(int) {} // #cwg1898-g
+void g(volatile int) {}
+// since-cxx20-warning@-1 {{volatile-qualified parameter type 'volatile int' is deprecated}}
+// expected-error@-2 {{redefinition of 'g'}}
+//   expected-note@#cwg1898-g {{previous definition is here}}
+
+void h(int *) {} // #cwg1898-h
+void h(int[]) {}
+// expected-error@-1 {{redefinition of 'h'}}
+//   expected-note@#cwg1898-h {{previous definition is here}}
+
+void h2(int *) {} // #cwg1898-h2
+void h2(int[2]) {}
+// expected-error@-1 {{redefinition of 'h2'}}
+//   expected-note@#cwg1898-h2 {{previous definition is here}}
+
+void h3(int (*)[2]) {} // #cwg1898-h3
+void h3(int [3][2]) {}
+// expected-error@-1 {{redefinition of 'h3'}}
+//   expected-note@#cwg1898-h3 {{previous definition is here}}
+
+void h4(int (*)[2]) {}
+void h4(int [3][3]) {} // OK, differ in non-top-level extent of array
+
+void i(int *) {}
+void i(const int *) {} // OK, pointee cv-qualification is not discarded
+
+void i2(int *) {} // #cwg1898-i2
+void i2(int * const) {}
+// expected-error@-1 {{redefinition of 'i2'}}
+//   expected-note@#cwg1898-i2 {{previous definition is here}}
+
+void j(void(*)()) {} // #cwg1898-j
+void j(void()) {}
+// expected-error@-1 {{redefinition of 'j'}}
+//   expected-note@#cwg1898-j {{previous definition is here}}
+
+void j2(void(int)) {} // #cwg1898-j2
+void j2(void(const int)) {}
+// expected-error@-1 {{redefinition of 'j2'}}
+//   expected-note@#cwg1898-j2 {{previous definition is here}}
+
+struct A {
+  void k(int) {} // #cwg1898-k
+  void k(int) {}
+  // expected-error@-1 {{class member cannot be redeclared}}
+  //   expected-note@#cwg1898-k {{previous definition is here}}
+};
+
+struct B : A {
+  void k(int) {} // OK, shadows A::k
+};
+
+void l() {}
+void l(...) {}
+
+#if __cplusplus >= 201103L
+template <typename T>
+void m(T) {}
+template <typename... Ts>
+void m(Ts...) {}
+
+template <typename T, typename U>
+void m2(T, U) {}
+template <typename... Ts, typename U>
+void m2(Ts..., U) {}
+#endif
+} // namespace cwg1898
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 82ba9b370ba5..6640ed477a24 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -11219,7 +11219,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/1898.html">1898</a></td>
     <td>CD6</td>
     <td>Use of &#8220;equivalent&#8221; in overload resolution</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 2.7</td>
   </tr>
   <tr id="1899">
     <td><a href="https://cplusplus.github.io/CWG/issues/1899.html">1899</a></td>
-- 
GitLab


From d3c29e8d2f11742e83e2b80df47391598bf2e857 Mon Sep 17 00:00:00 2001
From: Yijia Gu <yijiagu@google.com>
Date: Fri, 25 Oct 2024 10:24:31 -0700
Subject: [PATCH 186/693] [mlir][test][bazel] add missing deps for TestPass

---
 utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index 34beb758a12d..c69f793943be 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -547,8 +547,13 @@ cc_library(
         ":TestDialect",
         "//llvm:Support",
         "//mlir:FuncDialect",
+        "//mlir:GPUToSPIRV",
+        "//mlir:GPUTransforms",
         "//mlir:IR",
         "//mlir:Pass",
+        "//mlir:SPIRVDialect",
+        "//mlir:SPIRVToLLVM",
+        "//mlir:SPIRVTransforms",
         "//mlir:Support",
     ],
 )
-- 
GitLab


From 6e7375031a1a3172d5e369cf2c108da2bcf65c8a Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <arvind.sudarsanam@intel.com>
Date: Fri, 25 Oct 2024 10:27:42 -0700
Subject: [PATCH 187/693] [clang-linker-wrapper] Add error handling for missing
 linker path (#113613)

In clang-linker-wrapper, we do not explicitly check if --linker-path is
provided.
This PR adds a check to capture this.

Thanks

---------

Signed-off-by: Arvind Sudarsanam <arvind.sudarsanam@intel.com>
---
 clang/test/Driver/linker-wrapper.c                      | 4 ++++
 clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/clang/test/Driver/linker-wrapper.c b/clang/test/Driver/linker-wrapper.c
index 068ea2d7d3c6..470af4d5d70c 100644
--- a/clang/test/Driver/linker-wrapper.c
+++ b/clang/test/Driver/linker-wrapper.c
@@ -250,3 +250,7 @@ __attribute__((visibility("protected"), used)) int x;
 //       MLLVM-SAME: -Xlinker -mllvm=-pass-remarks=foo,bar
 //  OFFLOAD-OPT-NOT: -Xlinker -mllvm=-pass-remarks=foo,bar
 // OFFLOAD-OPT-SAME: {{$}}
+
+// Error handling when --linker-path is not provided for clang-linker-wrapper
+// RUN: not clang-linker-wrapper 2>&1 | FileCheck --check-prefix=LINKER-PATH-NOT-PROVIDED %s
+// LINKER-PATH-NOT-PROVIDED: linker path missing, must pass 'linker-path'
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 9fea1fdcd5fb..9fcecaee318a 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -370,6 +370,8 @@ Error runLinker(ArrayRef<StringRef> Files, const ArgList &Args) {
   // Render the linker arguments and add the newly created image. We add it
   // after the output file to ensure it is linked with the correct libraries.
   StringRef LinkerPath = Args.getLastArgValue(OPT_linker_path_EQ);
+  if (LinkerPath.empty())
+    return createStringError("linker path missing, must pass 'linker-path'");
   ArgStringList NewLinkerArgs;
   for (const opt::Arg *Arg : Args) {
     // Do not forward arguments only intended for the linker wrapper.
-- 
GitLab


From ac4bd74190fedfbe025ef757ff308dd184a507f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Fri, 25 Oct 2024 10:39:26 -0700
Subject: [PATCH 188/693] [mlir] Add apply_patterns.linalg.pad_vectorization TD
 Op (#112504)

This PR simply wraps `populatePadOpVectorizationPatterns` into a new
Transform Dialect Op: `apply_patterns.linalg.pad_vectorization`.

This change makes it possible to run (and test) the corresponding
patterns _without_:

  `transform.structured.vectorize_children_and_apply_patterns`.

Note that the Op above only supports non-masked vectorisation (i.e. when
the inputs are static), so, effectively, only fixed-width vectorisation
(as opposed to scalable vectorisation). As such, this change is required
to construct vectorization pipelines for tensor.pad targeting scalable
vectors.

To test the new Op and the corresponding patterns, I added
"vectorization-pad-patterns.mlir" - most tests have been extracted from
"vectorization-with-patterns.mlir".
---
 .../Linalg/TransformOps/LinalgTransformOps.td |  20 ++
 .../TransformOps/LinalgTransformOps.cpp       |   5 +
 .../Linalg/Transforms/Vectorization.cpp       |   3 +
 .../Linalg/vectorization-pad-patterns.mlir    | 274 ++++++++++++++++++
 .../Linalg/vectorization-with-patterns.mlir   | 143 ---------
 5 files changed, 302 insertions(+), 143 deletions(-)
 create mode 100644 mlir/test/Dialect/Linalg/vectorization-pad-patterns.mlir

diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index 040c04b0410e..abf446887c54 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -84,6 +84,26 @@ def ApplyFoldAddIntoDestPatternsOp : Op<Transform_Dialect,
   let assemblyFormat = "attr-dict";
 }
 
+def ApplyPadVectorizationPatternsOp : Op<Transform_Dialect,
+    "apply_patterns.linalg.pad_vectorization",
+    [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
+  let description = [{
+    Apply patterns that vectorize tensor.pad.
+
+    These patterns rewrite tensor.pad Ops using vector.transfer_read and
+    vector.transfer_write operations. This is done either by:
+      1. Folding tensor.pad with an existing vector.transfer_read /
+      vector.transfer_write Op (generated prior to running these patterns). 
+      2. Rewriting it (when matched together with q tensor.insert_slice
+      consumer Op) as a vector.transfer_read + vector.transfer_write pair.
+
+    In both cases, these patterns look at producers and consumers for the
+    matched tensor.pad Op to find opportunities for vectorization.
+  }];
+
+  let assemblyFormat = "attr-dict";
+}
+
 //===----------------------------------------------------------------------===//
 // BufferizeToAllocationOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 1f1d8ad89ae2..3d3f0a93a382 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -253,6 +253,11 @@ void transform::ApplyFoldAddIntoDestPatternsOp::populatePatterns(
   linalg::populateFoldAddIntoDestPatterns(patterns);
 }
 
+void transform::ApplyPadVectorizationPatternsOp::populatePatterns(
+    RewritePatternSet &patterns) {
+  linalg::populatePadOpVectorizationPatterns(patterns);
+}
+
 //===----------------------------------------------------------------------===//
 // BufferizeToAllocationOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index e1b97fbf985d..0a2457176a1d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -2712,6 +2712,9 @@ struct PadOpVectorizationWithInsertSlicePattern
 
 void mlir::linalg::populatePadOpVectorizationPatterns(
     RewritePatternSet &patterns, PatternBenefit baseBenefit) {
+  // TODO: The following pattern implements "decomposition" and
+  // optional "vectorization". Seperate "decomposition" into a sepereate
+  // pre-processing pattern group.
   patterns.add<GenericPadOpVectorizationPattern>(patterns.getContext(),
                                                  baseBenefit);
   // Try these specialized patterns first before resorting to the generic one.
diff --git a/mlir/test/Dialect/Linalg/vectorization-pad-patterns.mlir b/mlir/test/Dialect/Linalg/vectorization-pad-patterns.mlir
new file mode 100644
index 000000000000..2aa4638af3f0
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/vectorization-pad-patterns.mlir
@@ -0,0 +1,274 @@
+// RUN: mlir-opt %s -transform-interpreter -split-input-file | FileCheck %s
+
+///----------------------------------------------------------------------------------------
+/// [Pattern: PadOpVectorizationWithTransferReadPattern]
+///----------------------------------------------------------------------------------------
+// CHECK-LABEL: func @pad_and_transfer_read
+//  CHECK-SAME:     %[[ARG0:.*]]: tensor<5x6xf32>
+//   CHECK-NOT:   tensor.pad
+//   CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+//   CHECK-DAG:   %[[C5:.*]] = arith.constant 5.0
+//       CHECK:   %[[RESULT:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C0]]], %[[C5]] : tensor<5x6xf32>, vector<7x9xf32>
+//       CHECK:   return %[[RESULT]]
+func.func @pad_and_transfer_read(%arg0: tensor<5x6xf32>) -> vector<7x9xf32> {
+  %c0 = arith.constant 0 : index
+  %c5 = arith.constant 5.0 : f32
+  %c6 = arith.constant 6.0 : f32
+  %0 = tensor.pad %arg0 low[0, 0] high[5, 7] {
+    ^bb0(%arg1: index, %arg2: index):
+      tensor.yield %c5 : f32
+  } : tensor<5x6xf32> to tensor<10x13xf32>
+  %1 = vector.transfer_read %0[%c0, %c0], %c6
+      : tensor<10x13xf32>, vector<7x9xf32>
+  return %1 : vector<7x9xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func_op = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.op<"func.func">
+
+    transform.apply_patterns to %func_op {
+      transform.apply_patterns.linalg.pad_vectorization
+    } : !transform.op<"func.func">
+    transform.yield
+  }
+}
+
+// -----
+
+///----------------------------------------------------------------------------------------
+/// [Pattern: PadOpVectorizationWithTransferWritePattern]
+///----------------------------------------------------------------------------------------
+func.func private @make_vector() -> vector<7x9xf32>
+
+// CHECK-LABEL: func @pad_and_transfer_write_static_low_and_high
+//  CHECK-SAME:     %[[ARG0:.*]]: tensor<5x6xf32>
+//   CHECK-NOT:   tensor.pad
+//       CHECK:   %[[C0:.*]] = arith.constant 0 : index
+//       CHECK:   %[[VEC0:.*]] = call @make_vector() : () -> vector<7x9xf32>
+//       CHECK:   %[[RESULT:.*]] = vector.transfer_write %[[VEC0]], %[[ARG0]][%[[C0]], %[[C0]]] : vector<7x9xf32>, tensor<5x6xf32>
+//       CHECK:   return %[[RESULT]]
+func.func @pad_and_transfer_write_static_low_and_high(
+    %arg0: tensor<5x6xf32>) -> tensor<5x6xf32> {
+  %c0 = arith.constant 0 : index
+  %c5 = arith.constant 5.0 : f32
+  %0 = tensor.pad %arg0 low[0, 0] high[5, 7] {
+    ^bb0(%arg2: index, %arg3: index):
+      tensor.yield %c5 : f32
+  } : tensor<5x6xf32> to tensor<10x13xf32>
+  %1 = call @make_vector() : () -> vector<7x9xf32>
+  %2 = vector.transfer_write %1, %0[%c0, %c0]
+      : vector<7x9xf32>, tensor<10x13xf32>
+  %3 = tensor.extract_slice %2[0, 0] [5, 6] [1, 1] : tensor<10x13xf32> to tensor<5x6xf32>
+  return %3 : tensor<5x6xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func_op = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.op<"func.func">
+
+    transform.apply_patterns to %func_op {
+      transform.apply_patterns.linalg.pad_vectorization
+    } : !transform.op<"func.func">
+    transform.yield
+  }
+}
+
+// -----
+
+func.func private @make_vector() -> vector<7x9xf32>
+
+// CHECK-LABEL: func @pad_and_transfer_write_static_low_dynamic_high
+//  CHECK-SAME:     %[[ARG0:.*]]: tensor<?x?xf32>, %[[SIZE:.*]]: index, %[[PADDING:.*]]: index
+//   CHECK-NOT:   tensor.pad
+//       CHECK:   %[[C0:.*]] = arith.constant 0 : index
+//       CHECK:   %[[SUB:.*]] = tensor.extract_slice %[[ARG0]][0, 0] [%[[SIZE]], 6] [1, 1] : tensor<?x?xf32> to tensor<?x6xf32>
+//       CHECK:   %[[VEC0:.*]] = call @make_vector() : () -> vector<7x9xf32>
+//       CHECK:   %[[RESULT:.*]] = vector.transfer_write %[[VEC0]], %[[SUB]][%[[C0]], %[[C0]]] : vector<7x9xf32>, tensor<?x6xf32>
+//       CHECK:   return %[[RESULT]]
+func.func @pad_and_transfer_write_static_low_dynamic_high(
+    %arg0: tensor<?x?xf32>, %size: index, %padding: index) -> tensor<?x6xf32> {
+  %c0 = arith.constant 0 : index
+  %c5 = arith.constant 5.0 : f32
+  %s = tensor.extract_slice %arg0[0, 0] [%size, 6] [1, 1]
+      : tensor<?x?xf32> to tensor<?x6xf32>
+  %0 = tensor.pad %s low[0, 0] high[%padding, 7] {
+    ^bb0(%arg2: index, %arg3: index):
+      tensor.yield %c5 : f32
+  } : tensor<?x6xf32> to tensor<?x13xf32>
+  %1 = call @make_vector() : () -> vector<7x9xf32>
+  %2 = vector.transfer_write %1, %0[%c0, %c0]
+      : vector<7x9xf32>, tensor<?x13xf32>
+  %3 = tensor.extract_slice %2[0, 0] [%size, 6] [1, 1] : tensor<?x13xf32> to tensor<?x6xf32>
+  return %3 : tensor<?x6xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func_op = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.op<"func.func">
+
+    transform.apply_patterns to %func_op {
+      transform.apply_patterns.linalg.pad_vectorization
+    } : !transform.op<"func.func">
+    transform.yield
+  }
+}
+
+
+// -----
+
+///----------------------------------------------------------------------------------------
+/// [Pattern: PadOpVectorizationWithInsertSlicePattern]
+///----------------------------------------------------------------------------------------
+
+func.func private @make_vector() -> tensor<12x13xf32>
+
+// CHECK-LABEL: func @pad_and_insert_slice_source
+//  CHECK-SAME:     %[[ARG0:.*]]: tensor<5x6xf32>
+//   CHECK-NOT:   tensor.pad
+//   CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+//   CHECK-DAG:   %[[C5:.*]] = arith.constant 5.0
+//       CHECK:   %[[VEC0:.*]] = call @make_vector() : () -> tensor<12x13xf32>
+//       CHECK:   %[[READ:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C0]]], %[[C5]] : tensor<5x6xf32>, vector<7x9xf32>
+//       CHECK:   %[[WRITE:.*]] = vector.transfer_write %[[READ]], %[[VEC0]][%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<7x9xf32>, tensor<12x13xf32>
+//       CHECK:   return %[[WRITE]]
+func.func @pad_and_insert_slice_source(
+    %arg0: tensor<5x6xf32>) -> tensor<12x13xf32> {
+  %c0 = arith.constant 0 : index
+  %c5 = arith.constant 5.0 : f32
+  %0 = tensor.pad %arg0 low[0, 0] high[2, 3] {
+    ^bb0(%arg2: index, %arg3: index):
+      tensor.yield %c5 : f32
+  } : tensor<5x6xf32> to tensor<7x9xf32>
+  %1 = call @make_vector() : () -> tensor<12x13xf32>
+  %r = tensor.insert_slice %0 into %1[0, 0][7, 9][1, 1] : tensor<7x9xf32> into tensor<12x13xf32>
+  return %r : tensor<12x13xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func_op = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.op<"func.func">
+
+    transform.apply_patterns to %func_op {
+      transform.apply_patterns.linalg.pad_vectorization
+    } : !transform.op<"func.func">
+    transform.yield
+  }
+}
+
+
+// -----
+
+///----------------------------------------------------------------------------------------
+/// tensor::PadOp -> tensor::EmptyOp + linalg::FillOp/tensor::GenerateOp + tensor::InsertSliceOp
+/// [Pattern: GenericPadOpVectorizationPattern]
+///----------------------------------------------------------------------------------------
+
+func.func private @make_vector() -> tensor<12x13xf32>
+
+// Same as @pad_and_insert_slice_dest in vectorization-with-patterns.mlir, but
+// over here linalg::fill is not vectorized (patterns for linalg.fill are not
+// included here)
+// CHECK-LABEL:   func.func @pad_and_insert_slice_dest(
+// CHECK-SAME:      %[[ARG_0:.*]]: tensor<1x5x6xf32>) -> tensor<1x12x13xf32> {
+//  CHECK-NOT:     tensor.pad
+//  CHECK-DAG:     %[[C0:.*]] = arith.constant 0 : index
+//  CHECK-DAG:     %[[PAD:.*]] = arith.constant 5.000000e+00 : f32
+//      CHECK:     %[[EMPTY:.*]] = tensor.empty() : tensor<1x12x13xf32>
+//      CHECK:     %[[FILL:.*]] = linalg.fill ins(%[[PAD]] : f32) outs(%[[EMPTY]] : tensor<1x12x13xf32>) -> tensor<1x12x13xf32>
+//      CHECK:     %[[READ:.*]] = vector.transfer_read %[[ARG_0]]{{\[}}%[[C0]], %[[C0]], %[[C0]]], %[[PAD]] {in_bounds = [true, true, true]} : tensor<1x5x6xf32>, vector<1x5x6xf32>
+//      CHECK:     %[[WRITE:.*]] = vector.transfer_write %[[READ]], %[[FILL]]{{\[}}%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x5x6xf32>, tensor<1x12x13xf32>
+//      CHECK:     %[[VEC:.*]] = call @make_vector() : () -> tensor<12x13xf32>
+//      CHECK:     %[[RES:.*]] = tensor.insert_slice %[[VEC]] into %[[WRITE]][0, 0, 0] [1, 12, 13] [1, 1, 1] : tensor<12x13xf32> into tensor<1x12x13xf32>
+//      CHECK:     return %[[RES]] : tensor<1x12x13xf32>
+
+func.func @pad_and_insert_slice_dest(
+    %arg0: tensor<1x5x6xf32>) -> tensor<1x12x13xf32> {
+  %c5 = arith.constant 5.0 : f32
+  %0 = tensor.pad %arg0 low[0, 0, 0] high[0, 7, 7] {
+    ^bb0(%arg2: index, %arg3: index, %arg4: index):
+      tensor.yield %c5 : f32
+  } : tensor<1x5x6xf32> to tensor<1x12x13xf32>
+  %1 = call @make_vector() : () -> tensor<12x13xf32>
+  %r = tensor.insert_slice %1 into %0[0, 0, 0][1, 12, 13][1, 1, 1] : tensor<12x13xf32> into tensor<1x12x13xf32>
+  return %r : tensor<1x12x13xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func_op = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.op<"func.func">
+
+    transform.apply_patterns to %func_op {
+      transform.apply_patterns.linalg.pad_vectorization
+    } : !transform.op<"func.func">
+    transform.yield
+  }
+}
+
+// -----
+func.func private @make_vector() -> vector<7x9xf32>
+
+// Variant of @pad_and_transfer_write_static
+
+// CHECK-LABEL: func @pad_and_transfer_write_static_non_zero_low_pad
+//   CHECK-NOT:   tensor.pad
+//       CHECK:   linalg.fill
+func.func @pad_and_transfer_write_static_non_zero_low_pad(
+    %arg0: tensor<5x6xf32>) -> tensor<5x6xf32> {
+  %c0 = arith.constant 0 : index
+  %c5 = arith.constant 5.0 : f32
+  %0 = tensor.pad %arg0 low[0, 1] high[5, 6] {
+    ^bb0(%arg2: index, %arg3: index):
+      tensor.yield %c5 : f32
+  } : tensor<5x6xf32> to tensor<10x13xf32>
+  %1 = call @make_vector() : () -> vector<7x9xf32>
+  %2 = vector.transfer_write %1, %0[%c0, %c0]
+      : vector<7x9xf32>, tensor<10x13xf32>
+  %3 = tensor.extract_slice %2[0, 0] [5, 6] [1, 1] : tensor<10x13xf32> to tensor<5x6xf32>
+  return %3 : tensor<5x6xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func_op = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.op<"func.func">
+
+    transform.apply_patterns to %func_op {
+      transform.apply_patterns.linalg.pad_vectorization
+    } : !transform.op<"func.func">
+    transform.yield
+  }
+}
+
+// -----
+func.func private @make_vector() -> vector<7x9xf32>
+
+// Variant of @pad_and_transfer_write_static
+
+// CHECK-LABEL: func @pad_and_transfer_write_static_non_zero_offset
+//   CHECK-NOT:   tensor.pad
+//       CHECK:   linalg.fill
+func.func @pad_and_transfer_write_static_non_zero_offset(
+    %arg0: tensor<5x6xf32>) -> tensor<5x6xf32> {
+  %c0 = arith.constant 0 : index
+  %c5 = arith.constant 5.0 : f32
+  %0 = tensor.pad %arg0 low[0, 1] high[5, 6] {
+    ^bb0(%arg2: index, %arg3: index):
+      tensor.yield %c5 : f32
+  } : tensor<5x6xf32> to tensor<10x13xf32>
+  %1 = call @make_vector() : () -> vector<7x9xf32>
+  %2 = vector.transfer_write %1, %0[%c0, %c0]
+      : vector<7x9xf32>, tensor<10x13xf32>
+  %3 = tensor.extract_slice %2[0, 1] [5, 6] [1, 1] : tensor<10x13xf32> to tensor<5x6xf32>
+  return %3 : tensor<5x6xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func_op = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.op<"func.func">
+
+    transform.apply_patterns to %func_op {
+      transform.apply_patterns.linalg.pad_vectorization
+    } : !transform.op<"func.func">
+    transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir b/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
index 1c6a786bfa43..189507d97d6d 100644
--- a/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
@@ -935,149 +935,6 @@ module attributes {transform.with_named_sequence} {
   }
 }
 
-// -----
-
-// CHECK-LABEL: func @pad_and_transfer_read
-//  CHECK-SAME:     %[[ARG0:.*]]: tensor<5x6xf32>
-//   CHECK-NOT:   tensor.pad
-//   CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
-//   CHECK-DAG:   %[[C5:.*]] = arith.constant 5.0
-//       CHECK:   %[[RESULT:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C0]]], %[[C5]] : tensor<5x6xf32>, vector<7x9xf32>
-//       CHECK:   return %[[RESULT]]
-func.func @pad_and_transfer_read(%arg0: tensor<5x6xf32>) -> vector<7x9xf32> {
-  %c0 = arith.constant 0 : index
-  %c5 = arith.constant 5.0 : f32
-  %c6 = arith.constant 6.0 : f32
-  %0 = tensor.pad %arg0 low[0, 0] high[5, 7] {
-    ^bb0(%arg1: index, %arg2: index):
-      tensor.yield %c5 : f32
-  } : tensor<5x6xf32> to tensor<10x13xf32>
-  %1 = vector.transfer_read %0[%c0, %c0], %c6
-      : tensor<10x13xf32>, vector<7x9xf32>
-  return %1 : vector<7x9xf32>
-}
-
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_padding } : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
-
-func.func private @make_vector() -> vector<7x9xf32>
-
-// CHECK-LABEL: func @pad_and_transfer_write_static
-//  CHECK-SAME:     %[[ARG0:.*]]: tensor<5x6xf32>
-//   CHECK-NOT:   tensor.pad
-//       CHECK:   %[[C0:.*]] = arith.constant 0 : index
-//       CHECK:   %[[VEC0:.*]] = call @make_vector() : () -> vector<7x9xf32>
-//       CHECK:   %[[RESULT:.*]] = vector.transfer_write %[[VEC0]], %[[ARG0]][%[[C0]], %[[C0]]] : vector<7x9xf32>, tensor<5x6xf32>
-//       CHECK:   return %[[RESULT]]
-func.func @pad_and_transfer_write_static(
-    %arg0: tensor<5x6xf32>) -> tensor<5x6xf32> {
-  %c0 = arith.constant 0 : index
-  %c5 = arith.constant 5.0 : f32
-  %0 = tensor.pad %arg0 low[0, 0] high[5, 7] {
-    ^bb0(%arg2: index, %arg3: index):
-      tensor.yield %c5 : f32
-  } : tensor<5x6xf32> to tensor<10x13xf32>
-  %1 = call @make_vector() : () -> vector<7x9xf32>
-  %2 = vector.transfer_write %1, %0[%c0, %c0]
-      : vector<7x9xf32>, tensor<10x13xf32>
-  %3 = tensor.extract_slice %2[0, 0] [5, 6] [1, 1] : tensor<10x13xf32> to tensor<5x6xf32>
-  return %3 : tensor<5x6xf32>
-}
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %3 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %4 = transform.get_parent_op %3 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %5 = transform.structured.vectorize_children_and_apply_patterns %4  { vectorize_padding } : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
-
-
-// -----
-
-func.func private @make_vector() -> vector<7x9xf32>
-
-// CHECK-LABEL: func @pad_and_transfer_write_dynamic_static
-//  CHECK-SAME:     %[[ARG0:.*]]: tensor<?x?xf32>, %[[SIZE:.*]]: index, %[[PADDING:.*]]: index
-//   CHECK-NOT:   tensor.pad
-//       CHECK:   %[[C0:.*]] = arith.constant 0 : index
-//       CHECK:   %[[SUB:.*]] = tensor.extract_slice %[[ARG0]][0, 0] [%[[SIZE]], 6] [1, 1] : tensor<?x?xf32> to tensor<?x6xf32>
-//       CHECK:   %[[VEC0:.*]] = call @make_vector() : () -> vector<7x9xf32>
-//       CHECK:   %[[RESULT:.*]] = vector.transfer_write %[[VEC0]], %[[SUB]][%[[C0]], %[[C0]]] : vector<7x9xf32>, tensor<?x6xf32>
-//       CHECK:   return %[[RESULT]]
-func.func @pad_and_transfer_write_dynamic_static(
-    %arg0: tensor<?x?xf32>, %size: index, %padding: index) -> tensor<?x6xf32> {
-  %c0 = arith.constant 0 : index
-  %c5 = arith.constant 5.0 : f32
-  %s = tensor.extract_slice %arg0[0, 0] [%size, 6] [1, 1]
-      : tensor<?x?xf32> to tensor<?x6xf32>
-  %0 = tensor.pad %s low[0, 0] high[%padding, 7] {
-    ^bb0(%arg2: index, %arg3: index):
-      tensor.yield %c5 : f32
-  } : tensor<?x6xf32> to tensor<?x13xf32>
-  %1 = call @make_vector() : () -> vector<7x9xf32>
-  %2 = vector.transfer_write %1, %0[%c0, %c0]
-      : vector<7x9xf32>, tensor<?x13xf32>
-  %3 = tensor.extract_slice %2[0, 0] [%size, 6] [1, 1] : tensor<?x13xf32> to tensor<?x6xf32>
-  return %3 : tensor<?x6xf32>
-}
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %3 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %4 = transform.get_parent_op %3 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %5 = transform.structured.vectorize_children_and_apply_patterns %4 { vectorize_padding } : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
-
-
-// -----
-
-func.func private @make_vector() -> tensor<12x13xf32>
-
-// CHECK-LABEL: func @pad_and_insert_slice_source
-//  CHECK-SAME:     %[[ARG0:.*]]: tensor<5x6xf32>
-//   CHECK-NOT:   tensor.pad
-//   CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
-//   CHECK-DAG:   %[[C5:.*]] = arith.constant 5.0
-//       CHECK:   %[[VEC0:.*]] = call @make_vector() : () -> tensor<12x13xf32>
-//       CHECK:   %[[READ:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C0]]], %[[C5]] : tensor<5x6xf32>, vector<7x9xf32>
-//       CHECK:   %[[WRITE:.*]] = vector.transfer_write %[[READ]], %[[VEC0]][%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<7x9xf32>, tensor<12x13xf32>
-//       CHECK:   return %[[WRITE]]
-func.func @pad_and_insert_slice_source(
-    %arg0: tensor<5x6xf32>) -> tensor<12x13xf32> {
-  %c0 = arith.constant 0 : index
-  %c5 = arith.constant 5.0 : f32
-  %0 = tensor.pad %arg0 low[0, 0] high[2, 3] {
-    ^bb0(%arg2: index, %arg3: index):
-      tensor.yield %c5 : f32
-  } : tensor<5x6xf32> to tensor<7x9xf32>
-  %1 = call @make_vector() : () -> tensor<12x13xf32>
-  %r = tensor.insert_slice %0 into %1[0, 0][7, 9][1, 1] : tensor<7x9xf32> into tensor<12x13xf32>
-  return %r : tensor<12x13xf32>
-}
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %3 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %4 = transform.get_parent_op %3 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %5 = transform.structured.vectorize_children_and_apply_patterns %4  { vectorize_padding } : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
-
-
 // -----
 
 func.func private @make_vector() -> tensor<12x13xf32>
-- 
GitLab


From 14db06946839729befd6bd3ced8142547f5fd139 Mon Sep 17 00:00:00 2001
From: ssijaric-nv <ssijaric@nvidia.com>
Date: Fri, 25 Oct 2024 10:47:39 -0700
Subject: [PATCH 189/693] [InstCombine] Fix a cycle when folding fneg(select)
 with scalable vector types (#112465)

The two folding operations are causing a cycle for the following case
with
scalable vector types:

define <vscale x 2 x double> @test_fneg_select_abs(<vscale x 2 x i1>
%cond, <vscale x 2 x double> %b) {
%1 = select <vscale x 2 x i1> %cond, <vscale x 2 x double>
zeroinitializer, <vscale x 2 x double> %b
  %2 = fneg fast <vscale x 2 x double> %1
  ret <vscale x 2 x double> %2
}

1) fold fneg:  -(Cond ? C : Y) -> Cond ? -C : -Y

2) fold select: (Cond ? -X : -Y) -> -(Cond ? X : Y)

1) results in the following since '<vscale x 2 x double>
zeroinitializer' passes
the check for the immediate constant:

%.neg = fneg fast <vscale x 2 x double> zeroinitializer
%b.neg = fneg fast <vscale x 2 x double> %b
%1 = select fast <vscale x 2 x i1> %cond, <vscale x 2 x double> %.neg,
<vscale x 2 x double> %b.neg

and so we end up going back and forth between 1) and 2).

Attempt to fold scalable vector constants, so that we end up with a
splat instead:

define <vscale x 2 x double> @test_fneg_select_abs(<vscale x 2 x i1>
%cond, <vscale x 2 x double> %b) {
  %b.neg = fneg fast <vscale x 2 x double> %b
%1 = select fast <vscale x 2 x i1> %cond, <vscale x 2 x double>
shufflevector (<vscale x 2 x double> insertelement (<vscale x 2 x
double> poison, double -0.000000e+00, i64 0), <vscale x 2 x double>
poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x double>
%b.neg
  ret <vscale x 2 x double> %1
}
---
 llvm/lib/IR/ConstantFold.cpp                | 29 ++++++++++---------
 llvm/test/Transforms/InstCombine/fneg.ll    | 32 +++++++++++++++++++++
 llvm/test/Transforms/InstSimplify/fp-nan.ll |  6 ++--
 3 files changed, 49 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index 57d9a03c9c22..07dfbc41e79b 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -581,26 +581,27 @@ Constant *llvm::ConstantFoldUnaryInstruction(unsigned Opcode, Constant *C) {
     case Instruction::FNeg:
       return ConstantFP::get(C->getContext(), neg(CV));
     }
-  } else if (auto *VTy = dyn_cast<FixedVectorType>(C->getType())) {
-
-    Type *Ty = IntegerType::get(VTy->getContext(), 32);
+  } else if (auto *VTy = dyn_cast<VectorType>(C->getType())) {
     // Fast path for splatted constants.
     if (Constant *Splat = C->getSplatValue())
       if (Constant *Elt = ConstantFoldUnaryInstruction(Opcode, Splat))
         return ConstantVector::getSplat(VTy->getElementCount(), Elt);
 
-    // Fold each element and create a vector constant from those constants.
-    SmallVector<Constant *, 16> Result;
-    for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) {
-      Constant *ExtractIdx = ConstantInt::get(Ty, i);
-      Constant *Elt = ConstantExpr::getExtractElement(C, ExtractIdx);
-      Constant *Res = ConstantFoldUnaryInstruction(Opcode, Elt);
-      if (!Res)
-        return nullptr;
-      Result.push_back(Res);
-    }
+    if (auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
+      // Fold each element and create a vector constant from those constants.
+      Type *Ty = IntegerType::get(FVTy->getContext(), 32);
+      SmallVector<Constant *, 16> Result;
+      for (unsigned i = 0, e = FVTy->getNumElements(); i != e; ++i) {
+        Constant *ExtractIdx = ConstantInt::get(Ty, i);
+        Constant *Elt = ConstantExpr::getExtractElement(C, ExtractIdx);
+        Constant *Res = ConstantFoldUnaryInstruction(Opcode, Elt);
+        if (!Res)
+          return nullptr;
+        Result.push_back(Res);
+      }
 
-    return ConstantVector::get(Result);
+      return ConstantVector::get(Result);
+    }
   }
 
   // We don't know how to fold this.
diff --git a/llvm/test/Transforms/InstCombine/fneg.ll b/llvm/test/Transforms/InstCombine/fneg.ll
index 3c4088832fea..6a9b3309bb34 100644
--- a/llvm/test/Transforms/InstCombine/fneg.ll
+++ b/llvm/test/Transforms/InstCombine/fneg.ll
@@ -1109,4 +1109,36 @@ define float @test_fneg_select_maxnum(float %x) {
   ret float %neg
 }
 
+; Check that there's no infinite loop.
+define <vscale x 2 x double> @test_fneg_select_svec(<vscale x 2 x i1> %cond, <vscale x 2 x double> %b) {
+; CHECK-LABEL: @test_fneg_select_svec(
+; CHECK-NEXT:    [[TMP2:%.*]] = fneg fast <vscale x 2 x double> [[TMP1:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = select fast <vscale x 2 x i1> [[COND:%.*]], <vscale x 2 x double> shufflevector (<vscale x 2 x double> insertelement (<vscale x 2 x double> poison, double -0.000000e+00, i64 0), <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x double> [[TMP2]]
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP3]]
+;
+  %1 = select <vscale x 2 x i1> %cond, <vscale x 2 x double> zeroinitializer, <vscale x 2 x double> %b
+  %2 = fneg fast <vscale x 2 x double> %1
+  ret <vscale x 2 x double> %2
+}
+
+define <vscale x 2 x double> @test_fneg_select_svec_2(<vscale x 2 x i1> %cond, <vscale x 2 x double> %a) {
+; CHECK-LABEL: @test_fneg_select_svec_2(
+; CHECK-NEXT:    [[A_NEG:%.*]] = fneg fast <vscale x 2 x double> [[A:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select fast <vscale x 2 x i1> [[COND:%.*]], <vscale x 2 x double> [[A_NEG]], <vscale x 2 x double> shufflevector (<vscale x 2 x double> insertelement (<vscale x 2 x double> poison, double -0.000000e+00, i64 0), <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+;
+  %1 = select <vscale x 2 x i1> %cond, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
+  %2 = fneg fast <vscale x 2 x double> %1
+  ret <vscale x 2 x double> %2
+}
+
+define <vscale x 2 x double> @test_fneg_select_svec_3(<vscale x 2 x i1> %cond, <vscale x 2 x double> %b) {
+; CHECK-LABEL: @test_fneg_select_svec_3(
+; CHECK-NEXT:    ret <vscale x 2 x double> shufflevector (<vscale x 2 x double> insertelement (<vscale x 2 x double> poison, double -0.000000e+00, i64 0), <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer)
+;
+  %1 = select <vscale x 2 x i1> %cond, <vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer
+  %2 = fneg fast <vscale x 2 x double> %1
+  ret <vscale x 2 x double> %2
+}
+
 !0 = !{}
diff --git a/llvm/test/Transforms/InstSimplify/fp-nan.ll b/llvm/test/Transforms/InstSimplify/fp-nan.ll
index bb557500822c..06b23200baff 100644
--- a/llvm/test/Transforms/InstSimplify/fp-nan.ll
+++ b/llvm/test/Transforms/InstSimplify/fp-nan.ll
@@ -237,8 +237,7 @@ define <2 x double> @unary_fneg_nan_2(<2 x double> %x) {
 ; FIXME: This doesn't behave the same way as the fixed-length vectors above
 define <vscale x 1 x double> @unary_fneg_nan_2_scalable_vec_0() {
 ; CHECK-LABEL: @unary_fneg_nan_2_scalable_vec_0(
-; CHECK-NEXT:    [[R:%.*]] = fneg <vscale x 1 x double> shufflevector (<vscale x 1 x double> insertelement (<vscale x 1 x double> poison, double 0xFFF1234567890ABC, i64 0), <vscale x 1 x double> poison, <vscale x 1 x i32> zeroinitializer)
-; CHECK-NEXT:    ret <vscale x 1 x double> [[R]]
+; CHECK-NEXT:    ret <vscale x 1 x double> shufflevector (<vscale x 1 x double> insertelement (<vscale x 1 x double> poison, double 0x7FF1234567890ABC, i64 0), <vscale x 1 x double> poison, <vscale x 1 x i32> zeroinitializer)
 ;
   %r = fneg <vscale x 1 x double> splat (double 0xFFF1234567890ABC)
   ret <vscale x 1 x double> %r
@@ -247,8 +246,7 @@ define <vscale x 1 x double> @unary_fneg_nan_2_scalable_vec_0() {
 ; FIXME: This doesn't behave the same way as the fixed-length vectors above
 define <vscale x 1 x double> @unary_fneg_nan_2_scalable_vec_1() {
 ; CHECK-LABEL: @unary_fneg_nan_2_scalable_vec_1(
-; CHECK-NEXT:    [[R:%.*]] = fneg <vscale x 1 x double> shufflevector (<vscale x 1 x double> insertelement (<vscale x 1 x double> poison, double 0x7FF0000000000001, i64 0), <vscale x 1 x double> poison, <vscale x 1 x i32> zeroinitializer)
-; CHECK-NEXT:    ret <vscale x 1 x double> [[R]]
+; CHECK-NEXT:    ret <vscale x 1 x double> shufflevector (<vscale x 1 x double> insertelement (<vscale x 1 x double> poison, double 0xFFF0000000000001, i64 0), <vscale x 1 x double> poison, <vscale x 1 x i32> zeroinitializer)
 ;
   %r = fneg <vscale x 1 x double> splat (double 0x7FF0000000000001)
   ret <vscale x 1 x double> %r
-- 
GitLab


From 843c2fbe7f983c2a2059f753e4494f06fb645a9e Mon Sep 17 00:00:00 2001
From: Kiran Chandramohan <kiran.chandramohan@arm.com>
Date: Fri, 25 Oct 2024 18:57:01 +0100
Subject: [PATCH 190/693] Add parser+semantics support for scope construct
 (#113700)

Test parsing, semantics and a couple of basic semantic checks for
block/worksharing constructs.
Add TODO message in lowering.
---
 .../flang/Semantics/openmp-directive-sets.h   |  2 ++
 flang/lib/Lower/OpenMP/OpenMP.cpp             | 12 ++++++++++
 flang/lib/Parser/openmp-parsers.cpp           |  1 +
 flang/lib/Parser/unparse.cpp                  |  3 +++
 flang/lib/Semantics/check-omp-structure.cpp   |  7 +++++-
 flang/lib/Semantics/resolve-directives.cpp    |  2 ++
 flang/test/Lower/OpenMP/Todo/scope.f90        | 13 ++++++++++
 flang/test/Parser/OpenMP/scope.f90            | 24 +++++++++++++++++++
 .../test/Semantics/OpenMP/invalid-branch.f90  |  8 +++++++
 flang/test/Semantics/OpenMP/nested01.f90      |  7 ++++++
 llvm/include/llvm/Frontend/OpenMP/OMP.td      | 10 +++++++-
 11 files changed, 87 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/Todo/scope.f90
 create mode 100644 flang/test/Parser/OpenMP/scope.f90

diff --git a/flang/include/flang/Semantics/openmp-directive-sets.h b/flang/include/flang/Semantics/openmp-directive-sets.h
index 8eb736bb098f..50d6d5b59ef7 100644
--- a/flang/include/flang/Semantics/openmp-directive-sets.h
+++ b/flang/include/flang/Semantics/openmp-directive-sets.h
@@ -211,6 +211,7 @@ static const OmpDirectiveSet blockConstructSet{
     Directive::OMPD_parallel,
     Directive::OMPD_parallel_masked,
     Directive::OMPD_parallel_workshare,
+    Directive::OMPD_scope,
     Directive::OMPD_single,
     Directive::OMPD_target,
     Directive::OMPD_target_data,
@@ -281,6 +282,7 @@ static const OmpDirectiveSet workShareSet{
         Directive::OMPD_workshare,
         Directive::OMPD_parallel_workshare,
         Directive::OMPD_parallel_sections,
+        Directive::OMPD_scope,
         Directive::OMPD_sections,
         Directive::OMPD_single,
     } | allDoSet,
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index fc54da8babe6..01a40d6e2204 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -1650,6 +1650,15 @@ genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
   return sectionsOp;
 }
 
+static void genScopeOp(lower::AbstractConverter &converter,
+                       lower::SymMap &symTable,
+                       semantics::SemanticsContext &semaCtx,
+                       lower::pft::Evaluation &eval, mlir::Location loc,
+                       const ConstructQueue &queue,
+                       ConstructQueue::const_iterator item) {
+  TODO(loc, "Scope construct");
+}
+
 static mlir::omp::SingleOp
 genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
             semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
@@ -2478,6 +2487,9 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
   case llvm::omp::Directive::OMPD_simd:
     genStandaloneSimd(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
+  case llvm::omp::Directive::OMPD_scope:
+    genScopeOp(converter, symTable, semaCtx, eval, loc, queue, item);
+    break;
   case llvm::omp::Directive::OMPD_single:
     genSingleOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 59a8757e58e8..e740c421ca80 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -697,6 +697,7 @@ TYPE_PARSER(construct<OmpBlockDirective>(first(
     "PARALLEL MASKED" >> pure(llvm::omp::Directive::OMPD_parallel_masked),
     "PARALLEL WORKSHARE" >> pure(llvm::omp::Directive::OMPD_parallel_workshare),
     "PARALLEL" >> pure(llvm::omp::Directive::OMPD_parallel),
+    "SCOPE" >> pure(llvm::omp::Directive::OMPD_scope),
     "SINGLE" >> pure(llvm::omp::Directive::OMPD_single),
     "TARGET DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
     "TARGET PARALLEL" >> pure(llvm::omp::Directive::OMPD_target_parallel),
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 04df988223e8..19ceb2a3ebc3 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2386,6 +2386,9 @@ public:
     case llvm::omp::Directive::OMPD_parallel:
       Word("PARALLEL ");
       break;
+    case llvm::omp::Directive::OMPD_scope:
+      Word("SCOPE ");
+      break;
     case llvm::omp::Directive::OMPD_single:
       Word("SINGLE ");
       break;
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 46486907ceb9..1c2cf304d0ee 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -972,6 +972,7 @@ void OmpStructureChecker::Enter(const parser::OpenMPBlockConstruct &x) {
     HasInvalidWorksharingNesting(
         beginDir.source, llvm::omp::nestedWorkshareErrSet);
     break;
+  case llvm::omp::Directive::OMPD_scope:
   case llvm::omp::Directive::OMPD_single:
     // TODO: This check needs to be extended while implementing nesting of
     // regions checks.
@@ -1864,6 +1865,9 @@ void OmpStructureChecker::Enter(const parser::OmpEndBlockDirective &x) {
   const auto &dir{std::get<parser::OmpBlockDirective>(x.t)};
   ResetPartialContext(dir.source);
   switch (dir.v) {
+  case llvm::omp::Directive::OMPD_scope:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_scope);
+    break;
   // 2.7.3 end-single-clause -> copyprivate-clause |
   //                            nowait-clause
   case llvm::omp::Directive::OMPD_single:
@@ -1886,7 +1890,8 @@ void OmpStructureChecker::Enter(const parser::OmpEndBlockDirective &x) {
 // end_workshareare popped as they are pushed while entering the
 // EndBlockDirective.
 void OmpStructureChecker::Leave(const parser::OmpEndBlockDirective &x) {
-  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_single) ||
+  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_scope) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_single) ||
       (GetContext().directive == llvm::omp::Directive::OMPD_end_workshare)) {
     dirContext_.pop_back();
   }
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 33936ba4c2b3..513e42bee976 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -1526,6 +1526,7 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_master:
   case llvm::omp::Directive::OMPD_ordered:
   case llvm::omp::Directive::OMPD_parallel:
+  case llvm::omp::Directive::OMPD_scope:
   case llvm::omp::Directive::OMPD_single:
   case llvm::omp::Directive::OMPD_target:
   case llvm::omp::Directive::OMPD_target_data:
@@ -1557,6 +1558,7 @@ void OmpAttributeVisitor::Post(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_masked:
   case llvm::omp::Directive::OMPD_parallel_masked:
   case llvm::omp::Directive::OMPD_parallel:
+  case llvm::omp::Directive::OMPD_scope:
   case llvm::omp::Directive::OMPD_single:
   case llvm::omp::Directive::OMPD_target:
   case llvm::omp::Directive::OMPD_task:
diff --git a/flang/test/Lower/OpenMP/Todo/scope.f90 b/flang/test/Lower/OpenMP/Todo/scope.f90
new file mode 100644
index 000000000000..16a067dc8f25
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/scope.f90
@@ -0,0 +1,13 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s -fopenmp-version=51 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s -fopenmp-version=51 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: Scope construct
+program omp_scope
+  integer i
+  i = 10
+
+  !$omp scope private(i)
+  print *, "omp scope", i
+  !$omp end scope
+
+end program omp_scope
diff --git a/flang/test/Parser/OpenMP/scope.f90 b/flang/test/Parser/OpenMP/scope.f90
new file mode 100644
index 000000000000..6574136311e7
--- /dev/null
+++ b/flang/test/Parser/OpenMP/scope.f90
@@ -0,0 +1,24 @@
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck --ignore-case %s
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+program omp_scope
+  integer i
+  i = 10
+
+!CHECK: !$OMP SCOPE  PRIVATE(i)
+!CHECK: !$OMP END SCOPE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: OmpBlockDirective -> llvm::omp::Directive = scope
+!PARSE-TREE: OmpClauseList -> OmpClause -> Private -> OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'i'
+!PARSE-TREE: Block
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> PrintStmt
+!PARSE-TREE: OmpEndBlockDirective
+!PARSE-TREE: OmpBlockDirective -> llvm::omp::Directive = scope
+!PARSE-TREE: OmpClauseList -> OmpClause -> Nowait
+
+  !$omp scope private(i)
+  print *, "omp scope", i
+  !$omp end scope nowait
+end program omp_scope
diff --git a/flang/test/Semantics/OpenMP/invalid-branch.f90 b/flang/test/Semantics/OpenMP/invalid-branch.f90
index ed9e4d268f65..28aab8b122f3 100644
--- a/flang/test/Semantics/OpenMP/invalid-branch.f90
+++ b/flang/test/Semantics/OpenMP/invalid-branch.f90
@@ -105,4 +105,12 @@ program omp_invalid_branch
   !$omp end parallel
   9 print *, "2nd alternate return"
 
+  !CHECK: invalid branch into an OpenMP structured block
+  goto 100
+  !$omp scope
+    100 continue
+    !CHECK: invalid branch leaving an OpenMP structured block
+    goto 200
+  !$omp end scope
+  200 continue
 end program
diff --git a/flang/test/Semantics/OpenMP/nested01.f90 b/flang/test/Semantics/OpenMP/nested01.f90
index 49c964ab86aa..0936e4c1b45a 100644
--- a/flang/test/Semantics/OpenMP/nested01.f90
+++ b/flang/test/Semantics/OpenMP/nested01.f90
@@ -25,6 +25,13 @@
    !$omp end target
   enddo
   
+  !$omp do
+  do i = 1, N
+     !ERROR: A worksharing region may not be closely nested inside a worksharing, explicit task, taskloop, critical, ordered, atomic, or master region
+     !$omp scope
+     !$omp end scope
+  end do
+  !$omp end do
 
   !$omp do
   do i = 1, N
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 1834ad4d037f..d592f369a17f 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -892,7 +892,7 @@ def OMP_Scan : Directive<"scan"> {
   let association = AS_Separating;
   let category = CA_Subsidiary;
 }
-def OMP_scope : Directive<"scope"> {
+def OMP_Scope : Directive<"scope"> {
   let allowedClauses = [
     VersionedClause<OMPC_Private, 51>,
     VersionedClause<OMPC_Reduction, 51>,
@@ -905,6 +905,14 @@ def OMP_scope : Directive<"scope"> {
   let association = AS_Block;
   let category = CA_Executable;
 }
+def OMP_EndScope : Directive<"end scope"> {
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_NoWait>,
+  ];
+  let leafConstructs = OMP_Scope.leafConstructs;
+  let association = OMP_Scope.association;
+  let category = OMP_Scope.category;
+}
 def OMP_Section : Directive<"section"> {
   let association = AS_Separating;
   let category = CA_Subsidiary;
-- 
GitLab


From 144ddca9ed6a439ad8a421c3ff2ea763532341ba Mon Sep 17 00:00:00 2001
From: Teresa Johnson <tejohnson@google.com>
Date: Fri, 25 Oct 2024 11:09:57 -0700
Subject: [PATCH 191/693] [MemProf] Avoid duplicate edges between nodes
 (#113337)

The recent change to add support for cloning indirect calls
inadvertantly caused duplicate edges to be created between the same
caller/callee pair. This is due to the new moveCalleeEdgeToNewCaller
not properly guarding the addition of a new edge (ironically I was
testing for that in an assertion, but failed to handle that case
specially otherwise). Now simply move the context ids over to any
existing edge.

This issue in turn led to some assumptions in cloning being violated,
resulting in a later crash.

Add a test for this case to checkNode.
---
 .../IPO/MemProfContextDisambiguation.cpp      | 21 +++++++++++-
 llvm/test/ThinLTO/X86/memprof-icp.ll          | 34 +++++++++++++++----
 2 files changed, 48 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
index 4efd683dfca3..905186edcbec 100644
--- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
+++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
@@ -1352,6 +1352,17 @@ static void checkNode(const ContextNode<DerivedCCG, FuncTy, CallTy> *Node,
     }
     assert(NodeContextIds == CalleeEdgeContextIds);
   }
+  // FIXME: Since this checking is only invoked under an option, we should
+  // change the error checking from using assert to something that will trigger
+  // an error on a release build.
+#ifndef NDEBUG
+  // Make sure we don't end up with duplicate edges between the same caller and
+  // callee.
+  DenseSet<ContextNode<DerivedCCG, FuncTy, CallTy> *> NodeSet;
+  for (const auto &E : Node->CalleeEdges)
+    NodeSet.insert(E->Callee);
+  assert(NodeSet.size() == Node->CalleeEdges.size());
+#endif
 }
 
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
@@ -3125,7 +3136,15 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
     // from the same callers as the old node. That should be true in the current
     // use case, where we will remove None-type edges after copying over all
     // caller edges from the callee.
-    assert(IsNewNode || NewCaller->findEdgeFromCaller(OldCallerEdge->Caller));
+    auto *ExistingCallerEdge =
+        NewCaller->findEdgeFromCaller(OldCallerEdge->Caller);
+    assert(IsNewNode || ExistingCallerEdge);
+    if (ExistingCallerEdge) {
+      ExistingCallerEdge->getContextIds().insert(EdgeContextIdsToMove.begin(),
+                                                 EdgeContextIdsToMove.end());
+      ExistingCallerEdge->AllocTypes |= computeAllocType(EdgeContextIdsToMove);
+      continue;
+    }
     auto NewEdge = std::make_shared<ContextEdge>(
         NewCaller, OldCallerEdge->Caller,
         computeAllocType(EdgeContextIdsToMove), EdgeContextIdsToMove);
diff --git a/llvm/test/ThinLTO/X86/memprof-icp.ll b/llvm/test/ThinLTO/X86/memprof-icp.ll
index f17e19e1f77e..99e071898765 100644
--- a/llvm/test/ThinLTO/X86/memprof-icp.ll
+++ b/llvm/test/ThinLTO/X86/memprof-icp.ll
@@ -186,9 +186,13 @@
 ; REMARKS-MAIN: created clone _ZN2B03barEj.memprof.1
 ; REMARKS-MAIN: call in clone _ZN2B03barEj marked with memprof allocation attribute notcold
 ; REMARKS-MAIN: call in clone _ZN2B03barEj.memprof.1 marked with memprof allocation attribute cold
+; REMARKS-MAIN: call in clone _ZN2B03barEj marked with memprof allocation attribute notcold
+; REMARKS-MAIN: call in clone _ZN2B03barEj.memprof.1 marked with memprof allocation attribute cold
 ; REMARKS-MAIN: created clone _ZN1B3barEj.memprof.1
 ; REMARKS-MAIN: call in clone _ZN1B3barEj marked with memprof allocation attribute notcold
 ; REMARKS-MAIN: call in clone _ZN1B3barEj.memprof.1 marked with memprof allocation attribute cold
+; REMARKS-MAIN: call in clone _ZN1B3barEj marked with memprof allocation attribute notcold
+; REMARKS-MAIN: call in clone _ZN1B3barEj.memprof.1 marked with memprof allocation attribute cold
 ; REMARKS-FOO: created clone _Z3fooR2B0j.memprof.1
 ;; In each version of foo we should have promoted the indirect call to two conditional
 ;; direct calls, one to B::bar and one to B0::bar. The cloned version of foo should call
@@ -208,10 +212,10 @@
 ; REMARKS-FOO: call in clone _ZN1B3barEj marked with memprof allocation attribute notcold
 ; REMARKS-FOO: call in clone _ZN1B3barEj.memprof.1 marked with memprof allocation attribute cold
 
-; STATS: 2 memprof-context-disambiguation - Number of cold static allocations (possibly cloned) during whole program analysis
-; STATS-BE: 4 memprof-context-disambiguation - Number of cold static allocations (possibly cloned) during ThinLTO backend
-; STATS: 2 memprof-context-disambiguation - Number of not cold static allocations (possibly cloned) during whole program analysis
-; STATS-BE: 4 memprof-context-disambiguation - Number of not cold static allocations (possibly cloned) during ThinLTO backend
+; STATS: 4 memprof-context-disambiguation - Number of cold static allocations (possibly cloned) during whole program analysis
+; STATS-BE: 8 memprof-context-disambiguation - Number of cold static allocations (possibly cloned) during ThinLTO backend
+; STATS: 4 memprof-context-disambiguation - Number of not cold static allocations (possibly cloned) during whole program analysis
+; STATS-BE: 8 memprof-context-disambiguation - Number of not cold static allocations (possibly cloned) during ThinLTO backend
 ; STATS: 3 memprof-context-disambiguation - Number of function clones created during whole program analysis
 ; STATS-BE: 5 memprof-context-disambiguation - Number of function clones created during ThinLTO backend
 
@@ -247,8 +251,8 @@
 ; IR: attributes #[[NOTCOLD]] = {{.*}} "memprof"="notcold"
 ; IR: attributes #[[COLD]] = {{.*}} "memprof"="cold"
 
-; STATS-BE-DISTRIB: 2 memprof-context-disambiguation - Number of cold static allocations (possibly cloned) during ThinLTO backend
-; STATS-BE-DISTRIB: 2 memprof-context-disambiguation - Number of not cold static allocations (possibly cloned) during ThinLTO backend
+; STATS-BE-DISTRIB: 4 memprof-context-disambiguation - Number of cold static allocations (possibly cloned) during ThinLTO backend
+; STATS-BE-DISTRIB: 4 memprof-context-disambiguation - Number of not cold static allocations (possibly cloned) during ThinLTO backend
 ; STATS-BE-DISTRIB: 3 memprof-context-disambiguation - Number of function clones created during ThinLTO backend
 
 ;--- foo.ll
@@ -298,6 +302,9 @@ declare i32 @_Z3fooR2B0j(ptr, i32)
 define i32 @_ZN2B03barEj(ptr %this, i32 %s) {
 entry:
   %call = tail call ptr @_Znwm(i64 noundef 4) #0, !memprof !33, !callsite !38
+  ;; Second allocation in this function, to ensure that indirect edges to the
+  ;; same callee are partitioned correctly.
+  %call2 = tail call ptr @_Znwm(i64 noundef 4) #0, !memprof !45, !callsite !50
   store volatile i32 0, ptr %call, align 4
   ret i32 0
 }
@@ -311,6 +318,9 @@ declare void @_ZdlPvm()
 define i32 @_ZN1B3barEj(ptr %this, i32 %s) {
 entry:
   %call = tail call ptr @_Znwm(i64 noundef 4) #0, !memprof !39, !callsite !44
+  ;; Second allocation in this function, to ensure that indirect edges to the
+  ;; same callee are partitioned correctly.
+  %call2 = tail call ptr @_Znwm(i64 noundef 4) #0, !memprof !51, !callsite !56
   store volatile i32 0, ptr %call, align 4
   ret i32 0
 }
@@ -367,3 +377,15 @@ attributes #0 = { builtin allocsize(0) }
 !42 = !{!43, !"cold"}
 !43 = !{i64 4457553070050523782, i64 -2101080423462424381, i64 -6490791336773930154}
 !44 = !{i64 4457553070050523782}
+!45 = !{!46, !48}
+!46 = !{!47, !"notcold"}
+!47 = !{i64 456, i64 -2101080423462424381, i64 5188446645037944434}
+!48 = !{!49, !"cold"}
+!49 = !{i64 456, i64 -2101080423462424381, i64 5583420417449503557}
+!50 = !{i64 456}
+!51 = !{!52, !54}
+!52 = !{!53, !"notcold"}
+!53 = !{i64 789, i64 -2101080423462424381, i64 132626519179914298}
+!54 = !{!55, !"cold"}
+!55 = !{i64 789, i64 -2101080423462424381, i64 -6490791336773930154}
+!56 = !{i64 789}
-- 
GitLab


From f4db221258cb44a8f9804ce852c0403328de39b2 Mon Sep 17 00:00:00 2001
From: "Stephan T. Lavavej" <stl@nuwen.net>
Date: Fri, 25 Oct 2024 11:12:41 -0700
Subject: [PATCH 192/693] [libc++][test] Use
 `ADDITIONAL_COMPILE_FLAGS(gcc-style-warnings)` for `-Wno-psabi` (#113608)

MSVC doesn't understand `-Wno-psabi`, which was introduced here by
@ldionne in #106077.

Using `ADDITIONAL_COMPILE_FLAGS(gcc-style-warnings)` (implemented by
#75317) avoids passing this to MSVC.
---
 .../std/atomics/atomics.lockfree/is_always_lock_free.pass.cpp   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcxx/test/std/atomics/atomics.lockfree/is_always_lock_free.pass.cpp b/libcxx/test/std/atomics/atomics.lockfree/is_always_lock_free.pass.cpp
index 5130758d5efd..abb12d6a3c24 100644
--- a/libcxx/test/std/atomics/atomics.lockfree/is_always_lock_free.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.lockfree/is_always_lock_free.pass.cpp
@@ -18,7 +18,7 @@
 
 // Ignore diagnostic about vector types changing the ABI on some targets, since
 // that is irrelevant for this test.
-// ADDITIONAL_COMPILE_FLAGS: -Wno-psabi
+// ADDITIONAL_COMPILE_FLAGS(gcc-style-warnings): -Wno-psabi
 
 #include <atomic>
 #include <cassert>
-- 
GitLab


From a0c318938a528cfbef509a2516b36dd2411a52b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?=
 <41161573+gbossu@users.noreply.github.com>
Date: Fri, 25 Oct 2024 20:19:22 +0200
Subject: [PATCH 193/693] [CodeGen][NFC] Properly split MachineLICM and
 EarlyMachineLICM (#113573)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Both are based on MachineLICMBase, and the functionality there is
"switched" based on a PreRegAlloc flag. This commit is simply about
trusting the original value of that flag, defined by the `MachineLICM`
and `EarlyMachineLICM` classes.

The `PreRegAlloc` flag used to be overwritten it based on MRI.isSSA(),
which is un-reliable due to how it is inferred by the MIRParser. I see
that we can now define isSSA in MIR (thanks @gargaroff ), meaning the
fix isn’t really needed anymore, but redefining that flag still feels
wrong.

Note that I'm looking into upstreaming more changes to MachineLICM, see
[the discourse
thread](https://discourse.llvm.org/t/extending-post-regalloc-machinelicm/82725).
---
 llvm/lib/CodeGen/MachineLICM.cpp                         | 6 ------
 llvm/test/CodeGen/AArch64/mlicm-stack-write-check.mir    | 9 +--------
 llvm/test/CodeGen/AMDGPU/licm-regpressure.mir            | 4 ++--
 llvm/test/CodeGen/AMDGPU/licm-valu.mir                   | 4 ++--
 llvm/test/CodeGen/X86/unfoldMemoryOperand.mir            | 2 +-
 llvm/test/DebugInfo/MIR/X86/mlicm-hoist-pre-regalloc.mir | 4 ++--
 6 files changed, 8 insertions(+), 21 deletions(-)

diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index 793ad75759cc..7ea07862b839 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -391,12 +391,6 @@ bool MachineLICMImpl::run(MachineFunction &MF) {
   MRI = &MF.getRegInfo();
   SchedModel.init(&ST);
 
-  // FIXME: Remove this assignment or convert to an assert? (dead variable PreRegAlloc)
-  // MachineLICM and PostRAMachineLICM were distinguished by introducing
-  // EarlyMachineLICM and MachineLICM respectively to avoid "using an unreliable
-  // MRI::isSSA() check to determine whether register allocation has happened"
-  // (See 4a7c8e7).
-  PreRegAlloc = MRI->isSSA();
   HasProfileData = MF.getFunction().hasProfileData();
 
   if (PreRegAlloc)
diff --git a/llvm/test/CodeGen/AArch64/mlicm-stack-write-check.mir b/llvm/test/CodeGen/AArch64/mlicm-stack-write-check.mir
index 406025c4fde3..90ff68d30a3a 100644
--- a/llvm/test/CodeGen/AArch64/mlicm-stack-write-check.mir
+++ b/llvm/test/CodeGen/AArch64/mlicm-stack-write-check.mir
@@ -3,9 +3,6 @@
 ---
 name: test
 tracksRegLiveness: true
-isSSA: false
-registers:
-  - { id: 0, class: gpr64 }
 stack:
   - { id: 0, size: 8, type: spill-slot }
 body: |
@@ -30,14 +27,11 @@ body: |
 
   bb.2:
     liveins: $x0
-    %0 = COPY $x0
 ...
+
 ---
 name: test2
 tracksRegLiveness: true
-isSSA: false
-registers:
-  - { id: 0, class: gpr64 }
 stack:
   - { id: 0, size: 8, type: spill-slot }
 body: |
@@ -62,5 +56,4 @@ body: |
 
   bb.2:
     liveins: $x0
-    %0 = COPY $x0
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/licm-regpressure.mir b/llvm/test/CodeGen/AMDGPU/licm-regpressure.mir
index e63009fdcb43..dd478f94e103 100644
--- a/llvm/test/CodeGen/AMDGPU/licm-regpressure.mir
+++ b/llvm/test/CodeGen/AMDGPU/licm-regpressure.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass machinelicm -o - %s | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -passes machinelicm -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass early-machinelicm -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -passes early-machinelicm -o - %s | FileCheck -check-prefix=GCN %s
 
 # MachineLICM shall limit hoisting of V_CVT instructions out of the loop keeping
 # register pressure within the budget. VGPR budget at occupancy 10 is 24 vgprs.
diff --git a/llvm/test/CodeGen/AMDGPU/licm-valu.mir b/llvm/test/CodeGen/AMDGPU/licm-valu.mir
index b4f5e057f532..6a28eee19d50 100644
--- a/llvm/test/CodeGen/AMDGPU/licm-valu.mir
+++ b/llvm/test/CodeGen/AMDGPU/licm-valu.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=machinelicm -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=machinelicm -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=early-machinelicm -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=early-machinelicm -o - %s | FileCheck -check-prefix=GCN %s
 
 ---
 name: hoist_move
diff --git a/llvm/test/CodeGen/X86/unfoldMemoryOperand.mir b/llvm/test/CodeGen/X86/unfoldMemoryOperand.mir
index ff3d9ca378db..135b14d6836a 100644
--- a/llvm/test/CodeGen/X86/unfoldMemoryOperand.mir
+++ b/llvm/test/CodeGen/X86/unfoldMemoryOperand.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=x86_64-- -passes machinelicm -mcpu=skx -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=x86_64-- -passes early-machinelicm -mcpu=skx -verify-machineinstrs -o - %s | FileCheck %s
 --- |
   @x = dso_local global i32 0, align 4
   @z = dso_local local_unnamed_addr global [1024 x i32] zeroinitializer, align 16
diff --git a/llvm/test/DebugInfo/MIR/X86/mlicm-hoist-pre-regalloc.mir b/llvm/test/DebugInfo/MIR/X86/mlicm-hoist-pre-regalloc.mir
index d4d59e14724e..b65a0e71af1d 100644
--- a/llvm/test/DebugInfo/MIR/X86/mlicm-hoist-pre-regalloc.mir
+++ b/llvm/test/DebugInfo/MIR/X86/mlicm-hoist-pre-regalloc.mir
@@ -1,6 +1,6 @@
 --- | 
-  ; RUN: llc -run-pass=machinelicm -o - %s | FileCheck %s
-  ; RUN: llc -passes=machinelicm -o - %s | FileCheck %s
+  ; RUN: llc -run-pass=early-machinelicm -o - %s | FileCheck %s
+  ; RUN: llc -passes=early-machinelicm -o - %s | FileCheck %s
   ; Line numbers should not be retained when loop invariant instructions are hoisted.
   ; Doing so causes poor stepping bevavior.
   ;
-- 
GitLab


From eccdb2489483ca58d2cb35bc38967a8e33117575 Mon Sep 17 00:00:00 2001
From: Ye Luo <yeluo@anl.gov>
Date: Fri, 25 Oct 2024 13:19:58 -0500
Subject: [PATCH 194/693] [OpenMP] Create versioned libgomp softlinks (#112973)

Add libgomp.1.dylib for MacOS and libgomp.so.1 for Linux

Linkers on Mac and Linux pick up versioned libgomp dynamic library
files. The existing softlinks (libgomp.dylib for MacOS and libgomp.so
for Linux) are insufficient. This helps alleviate the issue of mixing
libgomp and libomp at runtime.
---
 openmp/runtime/src/CMakeLists.txt | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 439cc20963a1..61c0bacc9f20 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -253,6 +253,17 @@ if(NOT WIN32)
       libiomp5${LIBOMP_LIBRARY_SUFFIX}
     WORKING_DIRECTORY ${LIBOMP_LIBRARY_DIR}
   )
+  if(LIBOMP_ENABLE_SHARED)
+    if(APPLE)
+      set(VERSIONED_LIBGOMP_NAME libgomp.1${LIBOMP_LIBRARY_SUFFIX})
+    else()
+      set(VERSIONED_LIBGOMP_NAME libgomp${LIBOMP_LIBRARY_SUFFIX}.1)
+    endif()
+    add_custom_command(TARGET omp POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E create_symlink ${LIBOMP_LIB_FILE} ${VERSIONED_LIBGOMP_NAME}
+      WORKING_DIRECTORY ${LIBOMP_LIBRARY_DIR}
+    )
+  endif()
 endif()
 
 # Definitions for testing, for reuse when testing libomptarget-nvptx.
@@ -439,13 +450,18 @@ else()
 
   if(${LIBOMP_INSTALL_ALIASES})
     # Create aliases (symlinks) of the library for backwards compatibility
+    extend_path(outdir "${CMAKE_INSTALL_PREFIX}" "${OPENMP_INSTALL_LIBDIR}")
     set(LIBOMP_ALIASES "libgomp;libiomp5")
     foreach(alias IN LISTS LIBOMP_ALIASES)
-      extend_path(outdir "${CMAKE_INSTALL_PREFIX}" "${OPENMP_INSTALL_LIBDIR}")
       install(CODE "execute_process(COMMAND \"\${CMAKE_COMMAND}\" -E create_symlink \"${LIBOMP_LIB_FILE}\"
         \"${alias}${LIBOMP_LIBRARY_SUFFIX}\" WORKING_DIRECTORY
         \"\$ENV{DESTDIR}${outdir}\")")
     endforeach()
+    if(LIBOMP_ENABLE_SHARED)
+      install(CODE "execute_process(COMMAND \"\${CMAKE_COMMAND}\" -E create_symlink \"${LIBOMP_LIB_FILE}\"
+        \"${VERSIONED_LIBGOMP_NAME}\" WORKING_DIRECTORY
+        \"\$ENV{DESTDIR}${outdir}\")")
+    endif()
   endif()
 endif()
 install(
-- 
GitLab


From 88cc7ac0cc43a739c25f6988c1bfe3949ca4da62 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Fri, 25 Oct 2024 14:21:43 -0400
Subject: [PATCH 195/693] [libc++][NFC] Remove unused functions from
 posix_l_fallbacks (#113709)

---
 .../__support/xlocale/__posix_l_fallback.h     | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/libcxx/include/__support/xlocale/__posix_l_fallback.h b/libcxx/include/__support/xlocale/__posix_l_fallback.h
index 8a3a6f27f48d..c83589181747 100644
--- a/libcxx/include/__support/xlocale/__posix_l_fallback.h
+++ b/libcxx/include/__support/xlocale/__posix_l_fallback.h
@@ -25,24 +25,10 @@
 #  include <wctype.h>
 #endif
 
-inline _LIBCPP_HIDE_FROM_ABI int isalnum_l(int __c, locale_t) { return ::isalnum(__c); }
-
-inline _LIBCPP_HIDE_FROM_ABI int isalpha_l(int __c, locale_t) { return ::isalpha(__c); }
-
-inline _LIBCPP_HIDE_FROM_ABI int iscntrl_l(int __c, locale_t) { return ::iscntrl(__c); }
-
 inline _LIBCPP_HIDE_FROM_ABI int isdigit_l(int __c, locale_t) { return ::isdigit(__c); }
 
-inline _LIBCPP_HIDE_FROM_ABI int isgraph_l(int __c, locale_t) { return ::isgraph(__c); }
-
 inline _LIBCPP_HIDE_FROM_ABI int islower_l(int __c, locale_t) { return ::islower(__c); }
 
-inline _LIBCPP_HIDE_FROM_ABI int isprint_l(int __c, locale_t) { return ::isprint(__c); }
-
-inline _LIBCPP_HIDE_FROM_ABI int ispunct_l(int __c, locale_t) { return ::ispunct(__c); }
-
-inline _LIBCPP_HIDE_FROM_ABI int isspace_l(int __c, locale_t) { return ::isspace(__c); }
-
 inline _LIBCPP_HIDE_FROM_ABI int isupper_l(int __c, locale_t) { return ::isupper(__c); }
 
 inline _LIBCPP_HIDE_FROM_ABI int isxdigit_l(int __c, locale_t) { return ::isxdigit(__c); }
@@ -52,8 +38,6 @@ inline _LIBCPP_HIDE_FROM_ABI int toupper_l(int __c, locale_t) { return ::toupper
 inline _LIBCPP_HIDE_FROM_ABI int tolower_l(int __c, locale_t) { return ::tolower(__c); }
 
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
-inline _LIBCPP_HIDE_FROM_ABI int iswalnum_l(wint_t __c, locale_t) { return ::iswalnum(__c); }
-
 inline _LIBCPP_HIDE_FROM_ABI int iswalpha_l(wint_t __c, locale_t) { return ::iswalpha(__c); }
 
 inline _LIBCPP_HIDE_FROM_ABI int iswblank_l(wint_t __c, locale_t) { return ::iswblank(__c); }
@@ -62,8 +46,6 @@ inline _LIBCPP_HIDE_FROM_ABI int iswcntrl_l(wint_t __c, locale_t) { return ::isw
 
 inline _LIBCPP_HIDE_FROM_ABI int iswdigit_l(wint_t __c, locale_t) { return ::iswdigit(__c); }
 
-inline _LIBCPP_HIDE_FROM_ABI int iswgraph_l(wint_t __c, locale_t) { return ::iswgraph(__c); }
-
 inline _LIBCPP_HIDE_FROM_ABI int iswlower_l(wint_t __c, locale_t) { return ::iswlower(__c); }
 
 inline _LIBCPP_HIDE_FROM_ABI int iswprint_l(wint_t __c, locale_t) { return ::iswprint(__c); }
-- 
GitLab


From 4ac0e7e400fe2a66d1fd5d5d1fa1c899dfb16716 Mon Sep 17 00:00:00 2001
From: Gang Chen <gangc@amd.com>
Date: Fri, 25 Oct 2024 11:24:47 -0700
Subject: [PATCH 196/693] [AMDGPU] Add a type for the named barrier (#113614)

---
 clang/include/clang/Basic/AMDGPUTypes.def     |  8 ++++
 clang/lib/CodeGen/CGDebugInfo.cpp             |  7 ++++
 clang/lib/CodeGen/CodeGenTypes.cpp            |  4 ++
 clang/test/AST/ast-dump-amdgpu-types.c        | 13 ++++--
 .../CodeGen/amdgpu-barrier-type-debug-info.c  |  8 ++++
 .../CodeGenCXX/amdgpu-barrier-typeinfo.cpp    | 10 +++++
 clang/test/CodeGenHIP/amdgpu-barrier-type.hip | 42 +++++++++++++++++++
 clang/test/SemaCXX/amdgpu-barrier.cpp         | 17 ++++++++
 clang/test/SemaHIP/amdgpu-barrier.hip         | 20 +++++++++
 clang/test/SemaOpenCL/amdgpu-barrier.cl       | 12 ++++++
 clang/test/SemaOpenMP/amdgpu-barrier.cpp      | 17 ++++++++
 llvm/lib/IR/Type.cpp                          | 14 +++++++
 12 files changed, 168 insertions(+), 4 deletions(-)
 create mode 100644 clang/test/CodeGen/amdgpu-barrier-type-debug-info.c
 create mode 100644 clang/test/CodeGenCXX/amdgpu-barrier-typeinfo.cpp
 create mode 100644 clang/test/CodeGenHIP/amdgpu-barrier-type.hip
 create mode 100644 clang/test/SemaCXX/amdgpu-barrier.cpp
 create mode 100644 clang/test/SemaHIP/amdgpu-barrier.hip
 create mode 100644 clang/test/SemaOpenCL/amdgpu-barrier.cl
 create mode 100644 clang/test/SemaOpenMP/amdgpu-barrier.cpp

diff --git a/clang/include/clang/Basic/AMDGPUTypes.def b/clang/include/clang/Basic/AMDGPUTypes.def
index e47e544fdc82..d3dff446f9ed 100644
--- a/clang/include/clang/Basic/AMDGPUTypes.def
+++ b/clang/include/clang/Basic/AMDGPUTypes.def
@@ -15,7 +15,15 @@
   AMDGPU_TYPE(Name, Id, SingletonId, Width, Align)
 #endif
 
+#ifndef AMDGPU_NAMED_BARRIER_TYPE
+#define AMDGPU_NAMED_BARRIER_TYPE(Name, Id, SingletonId, Width, Align, Scope) \
+  AMDGPU_TYPE(Name, Id, SingletonId, Width, Align)
+#endif
+
 AMDGPU_OPAQUE_PTR_TYPE("__amdgpu_buffer_rsrc_t", AMDGPUBufferRsrc, AMDGPUBufferRsrcTy, 128, 128, 8)
 
+AMDGPU_NAMED_BARRIER_TYPE("__amdgpu_named_workgroup_barrier_t", AMDGPUNamedWorkgroupBarrier, AMDGPUNamedWorkgroupBarrierTy, 128, 32, 0)
+
 #undef AMDGPU_TYPE
 #undef AMDGPU_OPAQUE_PTR_TYPE
+#undef AMDGPU_NAMED_BARRIER_TYPE
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 59a761c2303c..5fd6cfa63e6e 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -916,6 +916,13 @@ llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) {
                                      TheCU, TheCU->getFile(), 0);              \
     return SingletonId;                                                        \
   }
+#define AMDGPU_NAMED_BARRIER_TYPE(Name, Id, SingletonId, Width, Align, Scope)  \
+  case BuiltinType::Id: {                                                      \
+    if (!SingletonId)                                                          \
+      SingletonId =                                                            \
+          DBuilder.createBasicType(Name, Width, llvm::dwarf::DW_ATE_unsigned); \
+    return SingletonId;                                                        \
+  }
 #include "clang/Basic/AMDGPUTypes.def"
   case BuiltinType::UChar:
   case BuiltinType::Char_U:
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index f87184fc7783..09191a4901f4 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -564,6 +564,10 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
 #define AMDGPU_OPAQUE_PTR_TYPE(Name, Id, SingletonId, Width, Align, AS)        \
   case BuiltinType::Id:                                                        \
     return llvm::PointerType::get(getLLVMContext(), AS);
+#define AMDGPU_NAMED_BARRIER_TYPE(Name, Id, SingletonId, Width, Align, Scope)  \
+  case BuiltinType::Id:                                                        \
+    return llvm::TargetExtType::get(getLLVMContext(), "amdgcn.named.barrier",  \
+                                    {}, {Scope});
 #include "clang/Basic/AMDGPUTypes.def"
 #define HLSL_INTANGIBLE_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
 #include "clang/Basic/HLSLIntangibleTypes.def"
diff --git a/clang/test/AST/ast-dump-amdgpu-types.c b/clang/test/AST/ast-dump-amdgpu-types.c
index e032d678f1a0..f01461cdba23 100644
--- a/clang/test/AST/ast-dump-amdgpu-types.c
+++ b/clang/test/AST/ast-dump-amdgpu-types.c
@@ -1,10 +1,15 @@
 // REQUIRES: amdgpu-registered-target
 // Test without serialization:
-// RUN: %clang_cc1 -triple amdgcn -ast-dump -ast-dump-filter __amdgpu_buffer_rsrc_t %s | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn -ast-dump -ast-dump-filter __amdgpu_buffer_rsrc_t %s | FileCheck %s -check-prefix=BUFFER-RSRC
+// RUN: %clang_cc1 -triple amdgcn -ast-dump -ast-dump-filter __amdgpu_named_workgroup_barrier %s | FileCheck %s -check-prefix=WORKGROUP-BARRIER
 //
 // Test with serialization:
 // RUN: %clang_cc1 -triple amdgcn -emit-pch -o %t %s
-// RUN: %clang_cc1 -x c -triple amdgcn -include-pch %t -ast-dump-all -ast-dump-filter __amdgpu_buffer_rsrc_t /dev/null | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" | FileCheck %s
+// RUN: %clang_cc1 -x c -triple amdgcn -include-pch %t -ast-dump-all -ast-dump-filter __amdgpu_buffer_rsrc_t /dev/null | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" | FileCheck %s -check-prefix=BUFFER-RSRC
+// RUN: %clang_cc1 -x c -triple amdgcn -include-pch %t -ast-dump-all -ast-dump-filter __amdgpu_named_workgroup_barrier /dev/null | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" | FileCheck %s -check-prefix=WORKGROUP-BARRIER
 
-// CHECK: TypedefDecl {{.*}} implicit __amdgpu_buffer_rsrc_t
-// CHECK-NEXT: -BuiltinType {{.*}} '__amdgpu_buffer_rsrc_t'
+// BUFFER-RSRC: TypedefDecl {{.*}} implicit __amdgpu_buffer_rsrc_t
+// BUFFER-RSRC-NEXT: -BuiltinType {{.*}} '__amdgpu_buffer_rsrc_t'
+
+// WORKGROUP-BARRIER: TypedefDecl {{.*}} implicit __amdgpu_named_workgroup_barrier_t
+// WORKGROUP-BARRIER-NEXT: -BuiltinType {{.*}} '__amdgpu_named_workgroup_barrier_t'
diff --git a/clang/test/CodeGen/amdgpu-barrier-type-debug-info.c b/clang/test/CodeGen/amdgpu-barrier-type-debug-info.c
new file mode 100644
index 000000000000..f595f1b222c4
--- /dev/null
+++ b/clang/test/CodeGen/amdgpu-barrier-type-debug-info.c
@@ -0,0 +1,8 @@
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -triple amdgcn -emit-llvm -o - %s -debug-info-kind=limited 2>&1 | FileCheck %s
+
+// CHECK: name: "__amdgpu_named_workgroup_barrier_t",{{.*}}baseType: ![[BT:[0-9]+]]
+// CHECK: [[BT]] = !DIBasicType(name: "__amdgpu_named_workgroup_barrier_t", size: 128, encoding: DW_ATE_unsigned)
+void test_locals(void) {
+  __amdgpu_named_workgroup_barrier_t k0;
+}
diff --git a/clang/test/CodeGenCXX/amdgpu-barrier-typeinfo.cpp b/clang/test/CodeGenCXX/amdgpu-barrier-typeinfo.cpp
new file mode 100644
index 000000000000..a47f217dcd3d
--- /dev/null
+++ b/clang/test/CodeGenCXX/amdgpu-barrier-typeinfo.cpp
@@ -0,0 +1,10 @@
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -triple amdgcn %s -emit-llvm -o - | FileCheck %s
+
+namespace std { class type_info; };
+
+auto &b0 = typeid(__amdgpu_named_workgroup_barrier_t);
+
+// CHECK-DAG: @_ZTSu34__amdgpu_named_workgroup_barrier_t = {{.*}} c"u34__amdgpu_named_workgroup_barrier_t\00"
+// CHECK-DAG: @_ZTIu34__amdgpu_named_workgroup_barrier_t = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTSu34__amdgpu_named_workgroup_barrier_t
+
diff --git a/clang/test/CodeGenHIP/amdgpu-barrier-type.hip b/clang/test/CodeGenHIP/amdgpu-barrier-type.hip
new file mode 100644
index 000000000000..229e8b3c737c
--- /dev/null
+++ b/clang/test/CodeGenHIP/amdgpu-barrier-type.hip
@@ -0,0 +1,42 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature
+ // REQUIRES: amdgpu-registered-target
+ // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu verde -emit-llvm -o - %s | FileCheck %s
+
+#define __shared__ __attribute__((shared))
+
+__shared__ __amdgpu_named_workgroup_barrier_t bar;
+__shared__ __amdgpu_named_workgroup_barrier_t arr[2];
+__shared__ struct {
+  __amdgpu_named_workgroup_barrier_t x;
+  __amdgpu_named_workgroup_barrier_t y;
+} str;
+
+__amdgpu_named_workgroup_barrier_t *getBar();
+void useBar(__amdgpu_named_workgroup_barrier_t *);
+
+// CHECK-LABEL: define {{[^@]+}}@_Z7testSemPu34__amdgpu_named_workgroup_barrier_t
+// CHECK-SAME: (ptr noundef [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    [[P_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// CHECK-NEXT:    [[P_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[P_ADDR]] to ptr
+// CHECK-NEXT:    store ptr [[P]], ptr [[P_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    call void @_Z6useBarPu34__amdgpu_named_workgroup_barrier_t(ptr noundef [[TMP0]]) #[[ATTR2:[0-9]+]]
+// CHECK-NEXT:    call void @_Z6useBarPu34__amdgpu_named_workgroup_barrier_t(ptr noundef addrspacecast (ptr addrspace(1) @bar to ptr)) #[[ATTR2]]
+// CHECK-NEXT:    call void @_Z6useBarPu34__amdgpu_named_workgroup_barrier_t(ptr noundef getelementptr inbounds ([2 x target("amdgcn.named.barrier", 0)], ptr addrspacecast (ptr addrspace(1) @arr to ptr), i64 0, i64 1)) #[[ATTR2]]
+// CHECK-NEXT:    call void @_Z6useBarPu34__amdgpu_named_workgroup_barrier_t(ptr noundef getelementptr inbounds nuw ([[STRUCT_ANON:%.*]], ptr addrspacecast (ptr addrspace(1) @str to ptr), i32 0, i32 1)) #[[ATTR2]]
+// CHECK-NEXT:    [[CALL:%.*]] = call noundef ptr @_Z6getBarv() #[[ATTR2]]
+// CHECK-NEXT:    call void @_Z6useBarPu34__amdgpu_named_workgroup_barrier_t(ptr noundef [[CALL]]) #[[ATTR2]]
+// CHECK-NEXT:    [[CALL1:%.*]] = call noundef ptr @_Z6getBarv() #[[ATTR2]]
+// CHECK-NEXT:    ret ptr [[CALL1]]
+//
+__amdgpu_named_workgroup_barrier_t *testSem(__amdgpu_named_workgroup_barrier_t *p) {
+  useBar(p);
+  useBar(&bar);
+  useBar(&arr[1]);
+  useBar(&str.y);
+  useBar(getBar());
+  return getBar();
+}
diff --git a/clang/test/SemaCXX/amdgpu-barrier.cpp b/clang/test/SemaCXX/amdgpu-barrier.cpp
new file mode 100644
index 000000000000..a171433727dd
--- /dev/null
+++ b/clang/test/SemaCXX/amdgpu-barrier.cpp
@@ -0,0 +1,17 @@
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -fsyntax-only -verify -std=gnu++11 -triple amdgcn -Wno-unused-value %s
+
+void foo() {
+  int n = 100;
+  __amdgpu_named_workgroup_barrier_t v = 0; // expected-error {{cannot initialize a variable of type '__amdgpu_named_workgroup_barrier_t' with an rvalue of type 'int'}}
+  static_cast<__amdgpu_named_workgroup_barrier_t>(n); // expected-error {{static_cast from 'int' to '__amdgpu_named_workgroup_barrier_t' is not allowed}}
+  dynamic_cast<__amdgpu_named_workgroup_barrier_t>(n); // expected-error {{invalid target type '__amdgpu_named_workgroup_barrier_t' for dynamic_cast; target type must be a reference or pointer type to a defined class}}
+  reinterpret_cast<__amdgpu_named_workgroup_barrier_t>(n); // expected-error {{reinterpret_cast from 'int' to '__amdgpu_named_workgroup_barrier_t' is not allowed}}
+  int c(v); // expected-error {{cannot initialize a variable of type 'int' with an lvalue of type '__amdgpu_named_workgroup_barrier_t'}}
+  __amdgpu_named_workgroup_barrier_t k;
+  int *ip = (int *)k; // expected-error {{cannot cast from type '__amdgpu_named_workgroup_barrier_t' to pointer type 'int *'}}
+  void *vp = (void *)k; // expected-error {{cannot cast from type '__amdgpu_named_workgroup_barrier_t' to pointer type 'void *'}}
+}
+
+static_assert(sizeof(__amdgpu_named_workgroup_barrier_t) == 16, "wrong size");
+static_assert(alignof(__amdgpu_named_workgroup_barrier_t) == 4, "wrong alignment");
diff --git a/clang/test/SemaHIP/amdgpu-barrier.hip b/clang/test/SemaHIP/amdgpu-barrier.hip
new file mode 100644
index 000000000000..ccd99b1e2c1f
--- /dev/null
+++ b/clang/test/SemaHIP/amdgpu-barrier.hip
@@ -0,0 +1,20 @@
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -fsyntax-only -verify -triple amdgcn -Wno-unused-value %s
+// RUN: %clang_cc1 -fsyntax-only -verify -triple x86_64 -aux-triple amdgcn -Wno-unused-value %s
+
+#define __device__ __attribute__((device))
+
+__device__ void foo() {
+  int n = 100;
+  __amdgpu_named_workgroup_barrier_t v = 0; // expected-error {{cannot initialize a variable of type '__amdgpu_named_workgroup_barrier_t' with an rvalue of type 'int'}}
+  static_cast<__amdgpu_named_workgroup_barrier_t>(n); // expected-error {{static_cast from 'int' to '__amdgpu_named_workgroup_barrier_t' is not allowed}}
+  dynamic_cast<__amdgpu_named_workgroup_barrier_t>(n); // expected-error {{invalid target type '__amdgpu_named_workgroup_barrier_t' for dynamic_cast; target type must be a reference or pointer type to a defined class}}
+  reinterpret_cast<__amdgpu_named_workgroup_barrier_t>(n); // expected-error {{reinterpret_cast from 'int' to '__amdgpu_named_workgroup_barrier_t' is not allowed}}
+  int c(v); // expected-error {{cannot initialize a variable of type 'int' with an lvalue of type '__amdgpu_named_workgroup_barrier_t'}}
+  __amdgpu_named_workgroup_barrier_t k;
+  int *ip = (int *)k; // expected-error {{cannot cast from type '__amdgpu_named_workgroup_barrier_t' to pointer type 'int *'}}
+  void *vp = (void *)k; // expected-error {{cannot cast from type '__amdgpu_named_workgroup_barrier_t' to pointer type 'void *'}}
+}
+
+static_assert(sizeof(__amdgpu_named_workgroup_barrier_t) == 16, "wrong size");
+static_assert(alignof(__amdgpu_named_workgroup_barrier_t) == 4, "wrong alignment");
diff --git a/clang/test/SemaOpenCL/amdgpu-barrier.cl b/clang/test/SemaOpenCL/amdgpu-barrier.cl
new file mode 100644
index 000000000000..150c311c7c59
--- /dev/null
+++ b/clang/test/SemaOpenCL/amdgpu-barrier.cl
@@ -0,0 +1,12 @@
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -verify -cl-std=CL1.2 -triple amdgcn-amd-amdhsa -Wno-unused-value %s
+// RUN: %clang_cc1 -verify -cl-std=CL2.0 -triple amdgcn-amd-amdhsa -Wno-unused-value %s
+
+void foo() {
+    int n = 100;
+    __amdgpu_named_workgroup_barrier_t v = 0; // expected-error {{initializing '__private __amdgpu_named_workgroup_barrier_t' with an expression of incompatible type 'int'}}
+    int c = v; // expected-error {{initializing '__private int' with an expression of incompatible type '__private __amdgpu_named_workgroup_barrier_t'}}
+    __amdgpu_named_workgroup_barrier_t k;
+    int *ip = (int *)k; // expected-error {{operand of type '__amdgpu_named_workgroup_barrier_t' where arithmetic or pointer type is required}}
+    void *vp = (void *)k; // expected-error {{operand of type '__amdgpu_named_workgroup_barrier_t' where arithmetic or pointer type is required}}
+ }
diff --git a/clang/test/SemaOpenMP/amdgpu-barrier.cpp b/clang/test/SemaOpenMP/amdgpu-barrier.cpp
new file mode 100644
index 000000000000..70aaefd08088
--- /dev/null
+++ b/clang/test/SemaOpenMP/amdgpu-barrier.cpp
@@ -0,0 +1,17 @@
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa -triple amdgcn-amd-amdhsa -fopenmp-is-target-device -Wno-unused-value %s
+
+void foo() {
+#pragma omp target
+  {
+    int n = 100;
+    __amdgpu_named_workgroup_barrier_t v = 0; // expected-error {{cannot initialize a variable of type '__amdgpu_named_workgroup_barrier_t' with an rvalue of type 'int'}}
+    static_cast<__amdgpu_named_workgroup_barrier_t>(n); // expected-error {{static_cast from 'int' to '__amdgpu_named_workgroup_barrier_t' is not allowed}}
+    dynamic_cast<__amdgpu_named_workgroup_barrier_t>(n); // expected-error {{invalid target type '__amdgpu_named_workgroup_barrier_t' for dynamic_cast; target type must be a reference or pointer type to a defined class}}
+    reinterpret_cast<__amdgpu_named_workgroup_barrier_t>(n); // expected-error {{reinterpret_cast from 'int' to '__amdgpu_named_workgroup_barrier_t' is not allowed}}
+    int c(v); // expected-error {{cannot initialize a variable of type 'int' with an lvalue of type '__amdgpu_named_workgroup_barrier_t'}}
+    __amdgpu_named_workgroup_barrier_t k;
+    int *ip = (int *)k; // expected-error {{cannot cast from type '__amdgpu_named_workgroup_barrier_t' to pointer type 'int *'}}
+    void *vp = (void *)k; // expected-error {{cannot cast from type '__amdgpu_named_workgroup_barrier_t' to pointer type 'void *'}}
+  }
+ }
diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp
index 912b1a3960ef..e311cde41517 100644
--- a/llvm/lib/IR/Type.cpp
+++ b/llvm/lib/IR/Type.cpp
@@ -834,6 +834,14 @@ Expected<TargetExtType *> TargetExtType::checkParams(TargetExtType *TTy) {
         "target extension type riscv.vector.tuple should have one "
         "type parameter and one integer parameter");
 
+  // Opaque types in the AMDGPU name space.
+  if (TTy->Name == "amdgcn.named.barrier" &&
+      (TTy->getNumTypeParameters() != 0 || TTy->getNumIntParameters() != 1)) {
+    return createStringError("target extension type amdgcn.named.barrier "
+                             "should have no type parameters "
+                             "and one integer parameter");
+  }
+
   return TTy;
 }
 
@@ -879,6 +887,12 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) {
   if (Name.starts_with("dx."))
     return TargetTypeInfo(PointerType::get(C, 0));
 
+  // Opaque types in the AMDGPU name space.
+  if (Name == "amdgcn.named.barrier") {
+    return TargetTypeInfo(FixedVectorType::get(Type::getInt32Ty(C), 4),
+                          TargetExtType::CanBeGlobal);
+  }
+
   return TargetTypeInfo(Type::getVoidTy(C));
 }
 
-- 
GitLab


From 61946687bc68ccba763571cb420049b9a3749dfe Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Fri, 25 Oct 2024 11:33:44 -0700
Subject: [PATCH 197/693] [clang][modules] Shrink the size of `Module::Headers`
 (#113395)

This patch shrinks the size of the `Module` class from 2112B to 1624B. I
wasn't able to get a good data on the actual impact on memory usage, but
given my `clang-scan-deps` workload at hand (with tens of thousands of
instances), I think there should be some win here. This also speeds up
my benchmark by under 0.1%.
---
 .../modularize/CoverageChecker.cpp            |  7 ++---
 .../modularize/ModularizeUtilities.cpp        | 14 ++-------
 clang/include/clang/Basic/Module.h            | 31 ++++++++++++++-----
 clang/lib/Basic/Module.cpp                    |  2 +-
 clang/lib/Frontend/FrontendAction.cpp         |  2 +-
 clang/lib/Lex/ModuleMap.cpp                   | 21 +++++++------
 clang/lib/Serialization/ASTWriter.cpp         |  4 +--
 7 files changed, 45 insertions(+), 36 deletions(-)

diff --git a/clang-tools-extra/modularize/CoverageChecker.cpp b/clang-tools-extra/modularize/CoverageChecker.cpp
index 0e76c539aa3c..b536ee00497c 100644
--- a/clang-tools-extra/modularize/CoverageChecker.cpp
+++ b/clang-tools-extra/modularize/CoverageChecker.cpp
@@ -223,10 +223,9 @@ bool CoverageChecker::collectModuleHeaders(const Module &Mod) {
       return false;
   }
 
-  for (auto &HeaderKind : Mod.Headers)
-    for (auto &Header : HeaderKind)
-      ModuleMapHeadersSet.insert(
-          ModularizeUtilities::getCanonicalPath(Header.Entry.getName()));
+  for (const auto &Header : Mod.getAllHeaders())
+    ModuleMapHeadersSet.insert(
+        ModularizeUtilities::getCanonicalPath(Header.Entry.getName()));
 
   for (auto *Submodule : Mod.submodules())
     collectModuleHeaders(*Submodule);
diff --git a/clang-tools-extra/modularize/ModularizeUtilities.cpp b/clang-tools-extra/modularize/ModularizeUtilities.cpp
index b202b3aae8f8..476e13770a94 100644
--- a/clang-tools-extra/modularize/ModularizeUtilities.cpp
+++ b/clang-tools-extra/modularize/ModularizeUtilities.cpp
@@ -358,7 +358,7 @@ bool ModularizeUtilities::collectModuleHeaders(const clang::Module &Mod) {
   } else if (std::optional<clang::Module::DirectoryName> UmbrellaDir =
                  Mod.getUmbrellaDirAsWritten()) {
     // If there normal headers, assume these are umbrellas and skip collection.
-    if (Mod.Headers->size() == 0) {
+    if (Mod.getHeaders(Module::HK_Normal).empty()) {
       // Collect headers in umbrella directory.
       if (!collectUmbrellaHeaders(UmbrellaDir->Entry.getName(),
                                   UmbrellaDependents))
@@ -371,16 +371,8 @@ bool ModularizeUtilities::collectModuleHeaders(const clang::Module &Mod) {
   // modules or because they are meant to be included by another header,
   // and thus should be ignored by modularize.
 
-  int NormalHeaderCount = Mod.Headers[clang::Module::HK_Normal].size();
-
-  for (int Index = 0; Index < NormalHeaderCount; ++Index) {
-    DependentsVector NormalDependents;
-    // Collect normal header.
-    const clang::Module::Header &Header(
-      Mod.Headers[clang::Module::HK_Normal][Index]);
-    std::string HeaderPath = getCanonicalPath(Header.Entry.getName());
-    HeaderFileNames.push_back(HeaderPath);
-  }
+  for (const auto &Header : Mod.getHeaders(clang::Module::HK_Normal))
+    HeaderFileNames.push_back(getCanonicalPath(Header.Entry.getName()));
 
   int MissingCountThisModule = Mod.MissingHeaders.size();
 
diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h
index 9c5d33fbb562..1ab3b5e5f815 100644
--- a/clang/include/clang/Basic/Module.h
+++ b/clang/include/clang/Basic/Module.h
@@ -253,8 +253,6 @@ public:
     HK_PrivateTextual,
     HK_Excluded
   };
-  static const int NumHeaderKinds = HK_Excluded + 1;
-
   /// Information about a header directive as found in the module map
   /// file.
   struct Header {
@@ -263,17 +261,36 @@ public:
     FileEntryRef Entry;
   };
 
-  /// Information about a directory name as found in the module map
-  /// file.
+private:
+  static const int NumHeaderKinds = HK_Excluded + 1;
+  // The begin index for a HeaderKind also acts the end index of HeaderKind - 1.
+  // The extra element at the end acts as the end index of the last HeaderKind.
+  unsigned HeaderKindBeginIndex[NumHeaderKinds + 1] = {};
+  SmallVector<Header, 2> HeadersStorage;
+
+public:
+  ArrayRef<Header> getAllHeaders() const { return HeadersStorage; }
+  ArrayRef<Header> getHeaders(HeaderKind HK) const {
+    assert(HK < NumHeaderKinds && "Invalid Module::HeaderKind");
+    auto BeginIt = HeadersStorage.begin() + HeaderKindBeginIndex[HK];
+    auto EndIt = HeadersStorage.begin() + HeaderKindBeginIndex[HK + 1];
+    return {BeginIt, EndIt};
+  }
+  void addHeader(HeaderKind HK, Header H) {
+    assert(HK < NumHeaderKinds && "Invalid Module::HeaderKind");
+    auto EndIt = HeadersStorage.begin() + HeaderKindBeginIndex[HK + 1];
+    HeadersStorage.insert(EndIt, std::move(H));
+    for (unsigned HKI = HK + 1; HKI != NumHeaderKinds + 1; ++HKI)
+      ++HeaderKindBeginIndex[HKI];
+  }
+
+  /// Information about a directory name as found in the module map file.
   struct DirectoryName {
     std::string NameAsWritten;
     std::string PathRelativeToRootModuleDirectory;
     DirectoryEntryRef Entry;
   };
 
-  /// The headers that are part of this module.
-  SmallVector<Header, 2> Headers[5];
-
   /// Stored information about a header directive that was found in the
   /// module map file but has not been resolved to a file.
   struct UnresolvedHeaderDirective {
diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp
index ad52fccff5dc..a7a3f6b37efe 100644
--- a/clang/lib/Basic/Module.cpp
+++ b/clang/lib/Basic/Module.cpp
@@ -528,7 +528,7 @@ void Module::print(raw_ostream &OS, unsigned Indent, bool Dump) const {
 
   for (auto &K : Kinds) {
     assert(&K == &Kinds[K.Kind] && "kinds in wrong order");
-    for (auto &H : Headers[K.Kind]) {
+    for (auto &H : getHeaders(K.Kind)) {
       OS.indent(Indent + 2);
       OS << K.Prefix << "header \"";
       OS.write_escaped(H.NameAsWritten);
diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp
index 81eea9c4c4dc..8264bd702fe4 100644
--- a/clang/lib/Frontend/FrontendAction.cpp
+++ b/clang/lib/Frontend/FrontendAction.cpp
@@ -358,7 +358,7 @@ static std::error_code collectModuleHeaderIncludes(
 
   // Add includes for each of these headers.
   for (auto HK : {Module::HK_Normal, Module::HK_Private}) {
-    for (Module::Header &H : Module->Headers[HK]) {
+    for (const Module::Header &H : Module->getHeaders(HK)) {
       Module->addTopHeader(H.Entry);
       // Use the path as specified in the module map file. We'll look for this
       // file relative to the module build directory (the directory containing
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index 0a02a63deba3..bc76a54abd95 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -472,12 +472,12 @@ static bool violatesPrivateInclude(Module *RequestingModule,
     // as obtained from the lookup and as obtained from the module.
     // This check is not cheap, so enable it only for debugging.
     bool IsPrivate = false;
-    SmallVectorImpl<Module::Header> *HeaderList[] = {
-        &Header.getModule()->Headers[Module::HK_Private],
-        &Header.getModule()->Headers[Module::HK_PrivateTextual]};
-    for (auto *Hs : HeaderList)
+    ArrayRef<Module::Header> HeaderList[] = {
+        Header.getModule()->getHeaders(Module::HK_Private),
+        Header.getModule()->getHeaders(Module::HK_PrivateTextual)};
+    for (auto Hs : HeaderList)
       IsPrivate |= llvm::any_of(
-          *Hs, [&](const Module::Header &H) { return H.Entry == IncFileEnt; });
+          Hs, [&](const Module::Header &H) { return H.Entry == IncFileEnt; });
     assert(IsPrivate && "inconsistent headers and roles");
   }
 #endif
@@ -1296,27 +1296,28 @@ void ModuleMap::addHeader(Module *Mod, Module::Header Header,
                           ModuleHeaderRole Role, bool Imported) {
   KnownHeader KH(Mod, Role);
 
+  FileEntryRef HeaderEntry = Header.Entry;
+
   // Only add each header to the headers list once.
   // FIXME: Should we diagnose if a header is listed twice in the
   // same module definition?
-  auto &HeaderList = Headers[Header.Entry];
+  auto &HeaderList = Headers[HeaderEntry];
   if (llvm::is_contained(HeaderList, KH))
     return;
 
   HeaderList.push_back(KH);
-  Mod->Headers[headerRoleToKind(Role)].push_back(Header);
+  Mod->addHeader(headerRoleToKind(Role), std::move(Header));
 
   bool isCompilingModuleHeader = Mod->isForBuilding(LangOpts);
   if (!Imported || isCompilingModuleHeader) {
     // When we import HeaderFileInfo, the external source is expected to
     // set the isModuleHeader flag itself.
-    HeaderInfo.MarkFileModuleHeader(Header.Entry, Role,
-                                    isCompilingModuleHeader);
+    HeaderInfo.MarkFileModuleHeader(HeaderEntry, Role, isCompilingModuleHeader);
   }
 
   // Notify callbacks that we just added a new header.
   for (const auto &Cb : Callbacks)
-    Cb->moduleMapAddHeader(Header.Entry.getName());
+    Cb->moduleMapAddHeader(HeaderEntry.getName());
 }
 
 FileID ModuleMap::getContainingModuleMapFileID(const Module *Module) const {
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 494890284d2f..b576822fa704 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -3070,9 +3070,9 @@ void ASTWriter::WriteSubmodules(Module *WritingModule) {
         Module::HK_PrivateTextual},
       {SUBMODULE_EXCLUDED_HEADER, ExcludedHeaderAbbrev, Module::HK_Excluded}
     };
-    for (auto &HL : HeaderLists) {
+    for (const auto &HL : HeaderLists) {
       RecordData::value_type Record[] = {HL.RecordKind};
-      for (auto &H : Mod->Headers[HL.HeaderKind])
+      for (const auto &H : Mod->getHeaders(HL.HeaderKind))
         Stream.EmitRecordWithBlob(HL.Abbrev, Record, H.NameAsWritten);
     }
 
-- 
GitLab


From 9648271a3c5adf875680833ac74eb4bafb48678d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 25 Oct 2024 20:39:45 +0200
Subject: [PATCH 198/693] [LV] Pass flag indicating epilogue is vectorized to
 executePlan (NFC)

This clarifies the flag, which is now only passed if the epilogue loop
is being vectorized.
---
 .../Vectorize/LoopVectorizationPlanner.h           |  8 ++++----
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp    | 14 +++++++-------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 1c8d541ef2c5..b2745c81dec8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -435,9 +435,9 @@ public:
   /// Generate the IR code for the vectorized loop captured in VPlan \p BestPlan
   /// according to the best selected \p VF and  \p UF.
   ///
-  /// TODO: \p IsEpilogueVectorization is needed to avoid issues due to epilogue
-  /// vectorization re-using plans for both the main and epilogue vector loops.
-  /// It should be removed once the re-use issue has been fixed.
+  /// TODO: \p VectorizingEpilogue indicates if the executed VPlan is for the
+  /// epilogue vector loop. It should be removed once the re-use issue has been
+  /// fixed.
   /// \p ExpandedSCEVs is passed during execution of the plan for epilogue loop
   /// to re-use expansion results generated during main plan execution.
   ///
@@ -447,7 +447,7 @@ public:
   DenseMap<const SCEV *, Value *>
   executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan,
               InnerLoopVectorizer &LB, DominatorTree *DT,
-              bool IsEpilogueVectorization,
+              bool VectorizingEpilogue,
               const DenseMap<const SCEV *, Value *> *ExpandedSCEVs = nullptr);
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e1173ddd71af..865f5e3d2e58 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7626,16 +7626,16 @@ static void createAndCollectMergePhiForReduction(
 
 DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
     ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
-    InnerLoopVectorizer &ILV, DominatorTree *DT, bool IsEpilogueVectorization,
+    InnerLoopVectorizer &ILV, DominatorTree *DT, bool VectorizingEpilogue,
     const DenseMap<const SCEV *, Value *> *ExpandedSCEVs) {
   assert(BestVPlan.hasVF(BestVF) &&
          "Trying to execute plan with unsupported VF");
   assert(BestVPlan.hasUF(BestUF) &&
          "Trying to execute plan with unsupported UF");
   assert(
-      (IsEpilogueVectorization || !ExpandedSCEVs) &&
+      ((VectorizingEpilogue && ExpandedSCEVs) ||
+       (!VectorizingEpilogue && !ExpandedSCEVs)) &&
       "expanded SCEVs to reuse can only be used during epilogue vectorization");
-  (void)IsEpilogueVectorization;
 
   // TODO: Move to VPlan transform stage once the transition to the VPlan-based
   // cost model is complete for better cost estimates.
@@ -7661,8 +7661,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   if (!ILV.getTripCount())
     ILV.setTripCount(State.get(BestVPlan.getTripCount(), VPLane(0)));
   else
-    assert(IsEpilogueVectorization && "should only re-use the existing trip "
-                                      "count during epilogue vectorization");
+    assert(VectorizingEpilogue && "should only re-use the existing trip "
+                                  "count during epilogue vectorization");
 
   // 1. Set up the skeleton for vectorization, including vector pre-header and
   // middle block. The vector loop is created during VPlan execution.
@@ -7715,7 +7715,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   for (VPRecipeBase &R : *ExitVPBB) {
     createAndCollectMergePhiForReduction(
         dyn_cast<VPInstruction>(&R), State, OrigLoop,
-        State.CFG.VPBB2IRBB[ExitVPBB], ExpandedSCEVs);
+        State.CFG.VPBB2IRBB[ExitVPBB], VectorizingEpilogue);
   }
 
   // 2.6. Maintain Loop Hints
@@ -10233,7 +10233,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
 
         std::unique_ptr<VPlan> BestMainPlan(BestPlan.duplicate());
         auto ExpandedSCEVs = LVP.executePlan(EPI.MainLoopVF, EPI.MainLoopUF,
-                                             *BestMainPlan, MainILV, DT, true);
+                                             *BestMainPlan, MainILV, DT, false);
         ++LoopsVectorized;
 
         // Second pass vectorizes the epilogue and adjusts the control flow
-- 
GitLab


From 8c4bc1e75de27adfbaead34b895b0efbaf17bd02 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Fri, 25 Oct 2024 11:44:20 -0700
Subject: [PATCH 199/693] [mlir][Transforms] Merge 1:1 and 1:N type converters
 (#113032)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The 1:N type converter derived from the 1:1 type converter and extends
it with 1:N target materializations. This commit merges the two type
converters and stores 1:N target materializations in the 1:1 type
converter. This is in preparation of merging the 1:1 and 1:N dialect
conversion infrastructures.

1:1 target materializations (producing a single `Value`) will remain
valid. An additional API is added to the type converter to register 1:N
target materializations (producing a `SmallVector<Value>`). Internally,
all target materializations are stored as 1:N materializations.

The 1:N type converter is removed.

Note for LLVM integration: If you are using the `OneToNTypeConverter`,
simply switch all occurrences to `TypeConverter`.

---------

Co-authored-by: Markus Böck <markus.boeck02@gmail.com>
---
 .../Dialect/SparseTensor/Transforms/Passes.h  |  2 +-
 .../mlir/Transforms/DialectConversion.h       | 62 ++++++++++++++-----
 .../mlir/Transforms/OneToNTypeConversion.h    | 45 +-------------
 .../ArmSME/Transforms/VectorLegalization.cpp  |  2 +-
 .../Transforms/Utils/DialectConversion.cpp    | 26 ++++++--
 .../Transforms/Utils/OneToNTypeConversion.cpp | 44 +++++--------
 .../TestOneToNTypeConversionPass.cpp          | 18 ++++--
 7 files changed, 101 insertions(+), 98 deletions(-)

diff --git a/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h b/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h
index 6ccbc40bdd60..2e9c297f2018 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h
@@ -150,7 +150,7 @@ std::unique_ptr<Pass> createLowerForeachToSCFPass();
 //===----------------------------------------------------------------------===//
 
 /// Type converter for iter_space and iterator.
-struct SparseIterationTypeConverter : public OneToNTypeConverter {
+struct SparseIterationTypeConverter : public TypeConverter {
   SparseIterationTypeConverter();
 };
 
diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index 5ff36160dd61..5e5957170e64 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -173,7 +173,9 @@ public:
   /// conversion has finished.
   ///
   /// Note: Target materializations may optionally accept an additional Type
-  /// parameter, which is the original type of the SSA value.
+  /// parameter, which is the original type of the SSA value. Furthermore, `T`
+  /// can be a TypeRange; in that case, the function must return a
+  /// SmallVector<Value>.
 
   /// This method registers a materialization that will be called when
   /// converting (potentially multiple) block arguments that were the result of
@@ -210,6 +212,9 @@ public:
   /// will be invoked with: outputType = "t3", inputs = "v2",
   // originalType = "t1". Note  that the original type "t1" cannot be recovered
   /// from just "t3" and "v2"; that's why the originalType parameter exists.
+  ///
+  /// Note: During a 1:N conversion, the result types can be a TypeRange. In
+  /// that case the materialization produces a SmallVector<Value>.
   template <typename FnT, typename T = typename llvm::function_traits<
                               std::decay_t<FnT>>::template arg_t<1>>
   void addTargetMaterialization(FnT &&callback) {
@@ -316,6 +321,11 @@ public:
   Value materializeTargetConversion(OpBuilder &builder, Location loc,
                                     Type resultType, ValueRange inputs,
                                     Type originalType = {}) const;
+  SmallVector<Value> materializeTargetConversion(OpBuilder &builder,
+                                                 Location loc,
+                                                 TypeRange resultType,
+                                                 ValueRange inputs,
+                                                 Type originalType = {}) const;
 
   /// Convert an attribute present `attr` from within the type `type` using
   /// the registered conversion functions. If no applicable conversion has been
@@ -340,9 +350,9 @@ private:
 
   /// The signature of the callback used to materialize a target conversion.
   ///
-  /// Arguments: builder, result type, inputs, location, original type
-  using TargetMaterializationCallbackFn =
-      std::function<Value(OpBuilder &, Type, ValueRange, Location, Type)>;
+  /// Arguments: builder, result types, inputs, location, original type
+  using TargetMaterializationCallbackFn = std::function<SmallVector<Value>(
+      OpBuilder &, TypeRange, ValueRange, Location, Type)>;
 
   /// The signature of the callback used to convert a type attribute.
   using TypeAttributeConversionCallbackFn =
@@ -409,22 +419,46 @@ private:
   /// callback.
   ///
   /// With callback of form:
-  /// `Value(OpBuilder &, T, ValueRange, Location, Type)`
+  /// - Value(OpBuilder &, T, ValueRange, Location, Type)
+  /// - SmallVector<Value>(OpBuilder &, TypeRange, ValueRange, Location, Type)
   template <typename T, typename FnT>
   std::enable_if_t<
       std::is_invocable_v<FnT, OpBuilder &, T, ValueRange, Location, Type>,
       TargetMaterializationCallbackFn>
   wrapTargetMaterialization(FnT &&callback) const {
     return [callback = std::forward<FnT>(callback)](
-               OpBuilder &builder, Type resultType, ValueRange inputs,
-               Location loc, Type originalType) -> Value {
-      if (T derivedType = dyn_cast<T>(resultType))
-        return callback(builder, derivedType, inputs, loc, originalType);
-      return Value();
+               OpBuilder &builder, TypeRange resultTypes, ValueRange inputs,
+               Location loc, Type originalType) -> SmallVector<Value> {
+      SmallVector<Value> result;
+      if constexpr (std::is_same<T, TypeRange>::value) {
+        // This is a 1:N target materialization. Return the produces values
+        // directly.
+        result = callback(builder, resultTypes, inputs, loc, originalType);
+      } else if constexpr (std::is_assignable<Type, T>::value) {
+        // This is a 1:1 target materialization. Invoke the callback only if a
+        // single SSA value is requested.
+        if (resultTypes.size() == 1) {
+          // Invoke the callback only if the type class of the callback matches
+          // the requested result type.
+          if (T derivedType = dyn_cast<T>(resultTypes.front())) {
+            // 1:1 materializations produce single values, but we store 1:N
+            // target materialization functions in the type converter. Wrap the
+            // result value in a SmallVector<Value>.
+            Value val =
+                callback(builder, derivedType, inputs, loc, originalType);
+            if (val)
+              result.push_back(val);
+          }
+        }
+      } else {
+        static_assert(sizeof(T) == 0, "T must be a Type or a TypeRange");
+      }
+      return result;
     };
   }
   /// With callback of form:
-  /// `Value(OpBuilder &, T, ValueRange, Location)`
+  /// - Value(OpBuilder &, T, ValueRange, Location)
+  /// - SmallVector<Value>(OpBuilder &, TypeRange, ValueRange, Location)
   template <typename T, typename FnT>
   std::enable_if_t<
       std::is_invocable_v<FnT, OpBuilder &, T, ValueRange, Location>,
@@ -432,9 +466,9 @@ private:
   wrapTargetMaterialization(FnT &&callback) const {
     return wrapTargetMaterialization<T>(
         [callback = std::forward<FnT>(callback)](
-            OpBuilder &builder, T resultType, ValueRange inputs, Location loc,
-            Type originalType) -> Value {
-          return callback(builder, resultType, inputs, loc);
+            OpBuilder &builder, T resultTypes, ValueRange inputs, Location loc,
+            Type originalType) {
+          return callback(builder, resultTypes, inputs, loc);
         });
   }
 
diff --git a/mlir/include/mlir/Transforms/OneToNTypeConversion.h b/mlir/include/mlir/Transforms/OneToNTypeConversion.h
index c59a3a52f028..7b4dd65cbff7 100644
--- a/mlir/include/mlir/Transforms/OneToNTypeConversion.h
+++ b/mlir/include/mlir/Transforms/OneToNTypeConversion.h
@@ -33,49 +33,6 @@
 
 namespace mlir {
 
-/// Extends `TypeConverter` with 1:N target materializations. Such
-/// materializations have to provide the "reverse" of 1:N type conversions,
-/// i.e., they need to materialize N values with target types into one value
-/// with a source type (which isn't possible in the base class currently).
-class OneToNTypeConverter : public TypeConverter {
-public:
-  /// Callback that expresses user-provided materialization logic from the given
-  /// value to N values of the given types. This is useful for expressing target
-  /// materializations for 1:N type conversions, which materialize one value in
-  /// a source type as N values in target types.
-  using OneToNMaterializationCallbackFn =
-      std::function<std::optional<SmallVector<Value>>(OpBuilder &, TypeRange,
-                                                      Value, Location)>;
-
-  /// Creates the mapping of the given range of original types to target types
-  /// of the conversion and stores that mapping in the given (signature)
-  /// conversion. This function simply calls
-  /// `TypeConverter::convertSignatureArgs` and exists here with a different
-  /// name to reflect the broader semantic.
-  LogicalResult computeTypeMapping(TypeRange types,
-                                   SignatureConversion &result) const {
-    return convertSignatureArgs(types, result);
-  }
-
-  /// Applies one of the user-provided 1:N target materializations. If several
-  /// exists, they are tried out in the reverse order in which they have been
-  /// added until the first one succeeds. If none succeeds, the functions
-  /// returns `std::nullopt`.
-  std::optional<SmallVector<Value>>
-  materializeTargetConversion(OpBuilder &builder, Location loc,
-                              TypeRange resultTypes, Value input) const;
-
-  /// Adds a 1:N target materialization to the converter. Such materializations
-  /// build IR that converts N values with target types into 1 value of the
-  /// source type.
-  void addTargetMaterialization(OneToNMaterializationCallbackFn &&callback) {
-    oneToNTargetMaterializations.emplace_back(std::move(callback));
-  }
-
-private:
-  SmallVector<OneToNMaterializationCallbackFn> oneToNTargetMaterializations;
-};
-
 /// Stores a 1:N mapping of types and provides several useful accessors. This
 /// class extends `SignatureConversion`, which already supports 1:N type
 /// mappings but lacks some accessors into the mapping as well as access to the
@@ -295,7 +252,7 @@ public:
 /// not fail if some ops or types remain unconverted (i.e., the conversion is
 /// only "partial").
 LogicalResult
-applyPartialOneToNConversion(Operation *op, OneToNTypeConverter &typeConverter,
+applyPartialOneToNConversion(Operation *op, TypeConverter &typeConverter,
                              const FrozenRewritePatternSet &patterns);
 
 /// Add a pattern to the given pattern list to convert the signature of a
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
index 4968c4fc463d..e908a536e6fb 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
@@ -921,7 +921,7 @@ struct VectorLegalizationPass
     : public arm_sme::impl::VectorLegalizationBase<VectorLegalizationPass> {
   void runOnOperation() override {
     auto *context = &getContext();
-    OneToNTypeConverter converter;
+    TypeConverter converter;
     RewritePatternSet patterns(context);
     converter.addConversion([](Type type) { return type; });
     converter.addConversion(
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 3cfcaa965f35..3d0c81867e0c 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -2831,11 +2831,29 @@ Value TypeConverter::materializeTargetConversion(OpBuilder &builder,
                                                  Location loc, Type resultType,
                                                  ValueRange inputs,
                                                  Type originalType) const {
+  SmallVector<Value> result = materializeTargetConversion(
+      builder, loc, TypeRange(resultType), inputs, originalType);
+  if (result.empty())
+    return nullptr;
+  assert(result.size() == 1 && "expected single result");
+  return result.front();
+}
+
+SmallVector<Value> TypeConverter::materializeTargetConversion(
+    OpBuilder &builder, Location loc, TypeRange resultTypes, ValueRange inputs,
+    Type originalType) const {
   for (const TargetMaterializationCallbackFn &fn :
-       llvm::reverse(targetMaterializations))
-    if (Value result = fn(builder, resultType, inputs, loc, originalType))
-      return result;
-  return nullptr;
+       llvm::reverse(targetMaterializations)) {
+    SmallVector<Value> result =
+        fn(builder, resultTypes, inputs, loc, originalType);
+    if (result.empty())
+      continue;
+    assert(TypeRange(result) == resultTypes &&
+           "callback produced incorrect number of values or values with "
+           "incorrect types");
+    return result;
+  }
+  return {};
 }
 
 std::optional<TypeConverter::SignatureConversion>
diff --git a/mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp b/mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp
index 19e29d48623e..c208716891ef 100644
--- a/mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp
+++ b/mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp
@@ -17,20 +17,6 @@
 using namespace llvm;
 using namespace mlir;
 
-std::optional<SmallVector<Value>>
-OneToNTypeConverter::materializeTargetConversion(OpBuilder &builder,
-                                                 Location loc,
-                                                 TypeRange resultTypes,
-                                                 Value input) const {
-  for (const OneToNMaterializationCallbackFn &fn :
-       llvm::reverse(oneToNTargetMaterializations)) {
-    if (std::optional<SmallVector<Value>> result =
-            fn(builder, resultTypes, input, loc))
-      return *result;
-  }
-  return std::nullopt;
-}
-
 TypeRange OneToNTypeMapping::getConvertedTypes(unsigned originalTypeNo) const {
   TypeRange convertedTypes = getConvertedTypes();
   if (auto mapping = getInputMapping(originalTypeNo))
@@ -268,20 +254,20 @@ Block *OneToNPatternRewriter::applySignatureConversion(
 LogicalResult
 OneToNConversionPattern::matchAndRewrite(Operation *op,
                                          PatternRewriter &rewriter) const {
-  auto *typeConverter = getTypeConverter<OneToNTypeConverter>();
+  auto *typeConverter = getTypeConverter();
 
   // Construct conversion mapping for results.
   Operation::result_type_range originalResultTypes = op->getResultTypes();
   OneToNTypeMapping resultMapping(originalResultTypes);
-  if (failed(typeConverter->computeTypeMapping(originalResultTypes,
-                                               resultMapping)))
+  if (failed(typeConverter->convertSignatureArgs(originalResultTypes,
+                                                 resultMapping)))
     return failure();
 
   // Construct conversion mapping for operands.
   Operation::operand_type_range originalOperandTypes = op->getOperandTypes();
   OneToNTypeMapping operandMapping(originalOperandTypes);
-  if (failed(typeConverter->computeTypeMapping(originalOperandTypes,
-                                               operandMapping)))
+  if (failed(typeConverter->convertSignatureArgs(originalOperandTypes,
+                                                 operandMapping)))
     return failure();
 
   // Cast operands to target types.
@@ -318,7 +304,7 @@ namespace mlir {
 // inserted by this pass are annotated with a string attribute that also
 // documents which kind of the cast (source, argument, or target).
 LogicalResult
-applyPartialOneToNConversion(Operation *op, OneToNTypeConverter &typeConverter,
+applyPartialOneToNConversion(Operation *op, TypeConverter &typeConverter,
                              const FrozenRewritePatternSet &patterns) {
 #ifndef NDEBUG
   // Remember existing unrealized casts. This data structure is only used in
@@ -370,15 +356,13 @@ applyPartialOneToNConversion(Operation *op, OneToNTypeConverter &typeConverter,
       // Target materialization.
       assert(!areOperandTypesLegal && areResultsTypesLegal &&
              operands.size() == 1 && "found unexpected target cast");
-      std::optional<SmallVector<Value>> maybeResults =
-          typeConverter.materializeTargetConversion(
-              rewriter, castOp->getLoc(), resultTypes, operands.front());
-      if (!maybeResults) {
+      materializedResults = typeConverter.materializeTargetConversion(
+          rewriter, castOp->getLoc(), resultTypes, operands.front());
+      if (materializedResults.empty()) {
         emitError(castOp->getLoc())
             << "failed to create target materialization";
         return failure();
       }
-      materializedResults = maybeResults.value();
     } else {
       // Source and argument materializations.
       assert(areOperandTypesLegal && !areResultsTypesLegal &&
@@ -427,18 +411,18 @@ public:
                                 const OneToNTypeMapping &resultMapping,
                                 ValueRange convertedOperands) const override {
     auto funcOp = cast<FunctionOpInterface>(op);
-    auto *typeConverter = getTypeConverter<OneToNTypeConverter>();
+    auto *typeConverter = getTypeConverter();
 
     // Construct mapping for function arguments.
     OneToNTypeMapping argumentMapping(funcOp.getArgumentTypes());
-    if (failed(typeConverter->computeTypeMapping(funcOp.getArgumentTypes(),
-                                                 argumentMapping)))
+    if (failed(typeConverter->convertSignatureArgs(funcOp.getArgumentTypes(),
+                                                   argumentMapping)))
       return failure();
 
     // Construct mapping for function results.
     OneToNTypeMapping funcResultMapping(funcOp.getResultTypes());
-    if (failed(typeConverter->computeTypeMapping(funcOp.getResultTypes(),
-                                                 funcResultMapping)))
+    if (failed(typeConverter->convertSignatureArgs(funcOp.getResultTypes(),
+                                                   funcResultMapping)))
       return failure();
 
     // Nothing to do if the op doesn't have any non-identity conversions for its
diff --git a/mlir/test/lib/Conversion/OneToNTypeConversion/TestOneToNTypeConversionPass.cpp b/mlir/test/lib/Conversion/OneToNTypeConversion/TestOneToNTypeConversionPass.cpp
index 5c03ac12d1e5..b18dfd8bb22c 100644
--- a/mlir/test/lib/Conversion/OneToNTypeConversion/TestOneToNTypeConversionPass.cpp
+++ b/mlir/test/lib/Conversion/OneToNTypeConversion/TestOneToNTypeConversionPass.cpp
@@ -147,9 +147,14 @@ populateDecomposeTuplesTestPatterns(const TypeConverter &typeConverter,
 ///
 /// This function has been copied (with small adaptions) from
 /// TestDecomposeCallGraphTypes.cpp.
-static std::optional<SmallVector<Value>>
-buildGetTupleElementOps(OpBuilder &builder, TypeRange resultTypes, Value input,
-                        Location loc) {
+static SmallVector<Value> buildGetTupleElementOps(OpBuilder &builder,
+                                                  TypeRange resultTypes,
+                                                  ValueRange inputs,
+                                                  Location loc) {
+  if (inputs.size() != 1)
+    return {};
+  Value input = inputs.front();
+
   TupleType inputType = dyn_cast<TupleType>(input.getType());
   if (!inputType)
     return {};
@@ -222,7 +227,7 @@ void TestOneToNTypeConversionPass::runOnOperation() {
   auto *context = &getContext();
 
   // Assemble type converter.
-  OneToNTypeConverter typeConverter;
+  TypeConverter typeConverter;
 
   typeConverter.addConversion([](Type type) { return type; });
   typeConverter.addConversion(
@@ -234,6 +239,11 @@ void TestOneToNTypeConversionPass::runOnOperation() {
   typeConverter.addArgumentMaterialization(buildMakeTupleOp);
   typeConverter.addSourceMaterialization(buildMakeTupleOp);
   typeConverter.addTargetMaterialization(buildGetTupleElementOps);
+  // Test the other target materialization variant that takes the original type
+  // as additional argument. This materialization function always fails.
+  typeConverter.addTargetMaterialization(
+      [](OpBuilder &builder, TypeRange resultTypes, ValueRange inputs,
+         Location loc, Type originalType) -> SmallVector<Value> { return {}; });
 
   // Assemble patterns.
   RewritePatternSet patterns(context);
-- 
GitLab


From e724226da753f10fd36fbb0ea392f04ab0fdbdab Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 25 Oct 2024 12:35:33 +0100
Subject: [PATCH 200/693] [VPlan] Return cost of 0 for VPWidenCastRecipe
 without underlying value.

In some cases, VPWidenCastRecipes are created but not considered in the
legacy cost model, including truncates/extends when evaluating a reduction
in a smaller type. Return 0 for such casts for now, to avoid divergences
between VPlan and legacy cost models.

Fixes https://github.com/llvm/llvm-project/issues/113526.
---
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |  5 ++
 .../LoopVectorize/X86/cost-model.ll           | 65 +++++++++++++++++++
 2 files changed, 70 insertions(+)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 0eb4f7c7c88c..2080b77157b6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1524,6 +1524,11 @@ void VPWidenCastRecipe::execute(VPTransformState &State) {
 
 InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF,
                                                VPCostContext &Ctx) const {
+  // TODO: In some cases, VPWidenCastRecipes are created but not considered in
+  // the legacy cost model, including truncates/extends when evaluating a
+  // reduction in a smaller type.
+  if (!getUnderlyingValue())
+    return 0;
   // Computes the CastContextHint from a recipes that may access memory.
   auto ComputeCCH = [&](const VPRecipeBase *R) -> TTI::CastContextHint {
     if (VF.isScalar())
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 73647919aac3..29e54fabad0c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -1037,6 +1037,71 @@ exit:
   ret i64 %red.mul
 }
 
+; Test case for https://github.com/llvm/llvm-project/issues/113526.
+define i32 @narrowed_reduction(ptr %a, i1 %cmp) #0 {
+; CHECK-LABEL: @narrowed_reduction(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP:%.*]] to i32
+; CHECK-NEXT:    br i1 true, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[CONV]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = and <16 x i32> [[VEC_PHI]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <16 x i32> [[VEC_PHI1]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = or <16 x i32> [[TMP0]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i32> [[TMP1]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc <16 x i32> [[TMP2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc <16 x i32> [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6]] = zext <16 x i1> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7]] = zext <16 x i1> [[TMP5]] to <16 x i32>
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc <16 x i32> [[TMP6]] to <16 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = trunc <16 x i32> [[TMP7]] to <16 x i1>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = or <16 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[BIN_RDX]])
+; CHECK-NEXT:    [[TMP12:%.*]] = zext i1 [[TMP11]] to i32
+; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OR13:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[OR:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[OR13]], 1
+; CHECK-NEXT:    [[OR]] = or i32 [[AND]], [[CONV]]
+; CHECK-NEXT:    [[INC]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV]], 0
+; CHECK-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[OR_LCSSA:%.*]] = phi i32 [ [[OR]], [[LOOP]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[OR_LCSSA]]
+;
+entry:
+  %conv = zext i1 %cmp to i32
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 1, %entry ], [ %inc, %loop ]
+  %or13 = phi i32 [ 0, %entry ], [ %or, %loop ]
+  %and = and i32 %or13, 1
+  %or = or i32 %and, %conv
+  %inc = add i32 %iv, 1
+  %ec = icmp eq i32 %iv, 0
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret i32 %or
+}
+
 declare void @llvm.assume(i1 noundef) #0
 
 attributes #0 = { "target-cpu"="penryn" }
-- 
GitLab


From 75252e29ea6a0959f3c1670e641a03fc18fc65fa Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Fri, 25 Oct 2024 12:40:59 -0700
Subject: [PATCH 201/693] [clang][serialization] Bump `NUM_PREDEF_TYPE_IDS`

This fixes a build error caused by 4ac0e7e400fe2a66d1fd5d5d1fa1c899dfb16716.
---
 clang/include/clang/Serialization/ASTBitCodes.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 99232fd21357..3ddbc5fcd26c 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1149,7 +1149,7 @@ enum PredefinedTypeIDs {
 ///
 /// Type IDs for non-predefined types will start at
 /// NUM_PREDEF_TYPE_IDs.
-const unsigned NUM_PREDEF_TYPE_IDS = 512;
+const unsigned NUM_PREDEF_TYPE_IDS = 513;
 
 // Ensure we do not overrun the predefined types we reserved
 // in the enum PredefinedTypeIDs above.
-- 
GitLab


From 6c9bbbc818ae8a0d2849dbc1ebd84a220cc27d20 Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Fri, 25 Oct 2024 12:47:19 -0700
Subject: [PATCH 202/693] [SandboxVec][Legality] Reject non-instructions
 (#113190)

---
 .../Vectorize/SandboxVectorizer/Legality.h     | 10 +++++++++-
 .../Vectorize/SandboxVectorizer/Legality.cpp   | 18 +++++++++++++++++-
 .../SandboxVectorizer/Passes/BottomUpVec.cpp   |  2 +-
 .../SandboxVectorizer/LegalityTest.cpp         | 13 ++++++++++++-
 4 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
index bcfafd75d4ca..d4b0b54375b0 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
@@ -28,6 +28,7 @@ enum class LegalityResultID {
 
 /// The reason for vectorizing or not vectorizing.
 enum class ResultReason {
+  NotInstructions,
   DiffOpcodes,
   DiffTypes,
 };
@@ -46,6 +47,8 @@ struct ToStr {
 
   static const char *getVecReason(ResultReason Reason) {
     switch (Reason) {
+    case ResultReason::NotInstructions:
+      return "NotInstructions";
     case ResultReason::DiffOpcodes:
       return "DiffOpcodes";
     case ResultReason::DiffTypes:
@@ -67,6 +70,10 @@ protected:
   LegalityResult(LegalityResultID ID) : ID(ID) {}
   friend class LegalityAnalysis;
 
+  /// We shouldn't need copies.
+  LegalityResult(const LegalityResult &) = delete;
+  LegalityResult &operator=(const LegalityResult &) = delete;
+
 public:
   virtual ~LegalityResult() {}
   LegalityResultID getSubclassID() const { return ID; }
@@ -90,6 +97,7 @@ class LegalityResultWithReason : public LegalityResult {
   friend class Pack; // For constructor.
 
 public:
+  ResultReason getReason() const { return Reason; }
 #ifndef NDEBUG
   void print(raw_ostream &OS) const override {
     LegalityResult::print(OS);
@@ -138,7 +146,7 @@ public:
   }
   /// Checks if it's legal to vectorize the instructions in \p Bndl.
   /// \Returns a LegalityResult object owned by LegalityAnalysis.
-  LegalityResult &canVectorize(ArrayRef<Value *> Bndl);
+  const LegalityResult &canVectorize(ArrayRef<Value *> Bndl);
 };
 
 } // namespace llvm::sandboxir
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
index 0e2cd83c37b0..f1c4577cece7 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
@@ -7,11 +7,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
+#include "llvm/SandboxIR/Instruction.h"
+#include "llvm/SandboxIR/Utils.h"
 #include "llvm/SandboxIR/Value.h"
 #include "llvm/Support/Debug.h"
 
 namespace llvm::sandboxir {
 
+#define DEBUG_TYPE "SBVec:Legality"
+
 #ifndef NDEBUG
 void LegalityResult::dump() const {
   print(dbgs());
@@ -26,7 +30,19 @@ LegalityAnalysis::notVectorizableBasedOnOpcodesAndTypes(
   return std::nullopt;
 }
 
-LegalityResult &LegalityAnalysis::canVectorize(ArrayRef<Value *> Bndl) {
+static void dumpBndl(ArrayRef<Value *> Bndl) {
+  for (auto *V : Bndl)
+    dbgs() << *V << "\n";
+}
+
+const LegalityResult &LegalityAnalysis::canVectorize(ArrayRef<Value *> Bndl) {
+  // If Bndl contains values other than instructions, we need to Pack.
+  if (any_of(Bndl, [](auto *V) { return !isa<Instruction>(V); })) {
+    LLVM_DEBUG(dbgs() << "Not vectorizing: Not Instructions:\n";
+               dumpBndl(Bndl););
+    return createLegalityResult<Pack>(ResultReason::NotInstructions);
+  }
+
   if (auto ReasonOpt = notVectorizableBasedOnOpcodesAndTypes(Bndl))
     return createLegalityResult<Pack>(*ReasonOpt);
 
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index f11420e47f3e..ede41cd661b5 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -40,7 +40,7 @@ static SmallVector<Value *, 4> getOperand(ArrayRef<Value *> Bndl,
 }
 
 void BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl) {
-  auto LegalityRes = Legality.canVectorize(Bndl);
+  const auto &LegalityRes = Legality.canVectorize(Bndl);
   switch (LegalityRes.getSubclassID()) {
   case LegalityResultID::Widen: {
     auto *I = cast<Instruction>(Bndl[0]);
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
index 76e5a5ce5aed..56c6bf5f1ef1 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
@@ -52,8 +52,16 @@ define void @foo(ptr %ptr) {
   auto *St1 = cast<sandboxir::StoreInst>(&*It++);
 
   sandboxir::LegalityAnalysis Legality;
-  auto Result = Legality.canVectorize({St0, St1});
+  const auto &Result = Legality.canVectorize({St0, St1});
   EXPECT_TRUE(isa<sandboxir::Widen>(Result));
+
+  {
+    // Check NotInstructions
+    auto &Result = Legality.canVectorize({F, St0});
+    EXPECT_TRUE(isa<sandboxir::Pack>(Result));
+    EXPECT_EQ(cast<sandboxir::Pack>(Result).getReason(),
+              sandboxir::ResultReason::NotInstructions);
+  }
 }
 
 #ifndef NDEBUG
@@ -68,6 +76,9 @@ TEST_F(LegalityTest, LegalityResultDump) {
   sandboxir::LegalityAnalysis Legality;
   EXPECT_TRUE(
       Matches(Legality.createLegalityResult<sandboxir::Widen>(), "Widen"));
+  EXPECT_TRUE(Matches(Legality.createLegalityResult<sandboxir::Pack>(
+                          sandboxir::ResultReason::NotInstructions),
+                      "Pack Reason: NotInstructions"));
   EXPECT_TRUE(Matches(Legality.createLegalityResult<sandboxir::Pack>(
                           sandboxir::ResultReason::DiffOpcodes),
                       "Pack Reason: DiffOpcodes"));
-- 
GitLab


From eb9f4756bc3daaa4b19f4f46521dc05180814de4 Mon Sep 17 00:00:00 2001
From: Vasileios Porpodas <vporpodas@google.com>
Date: Fri, 25 Oct 2024 12:52:31 -0700
Subject: [PATCH 203/693] Revert "[SandboxVec][Legality] Reject
 non-instructions (#113190)"

This reverts commit 6c9bbbc818ae8a0d2849dbc1ebd84a220cc27d20.
---
 .../Vectorize/SandboxVectorizer/Legality.h     | 10 +---------
 .../Vectorize/SandboxVectorizer/Legality.cpp   | 18 +-----------------
 .../SandboxVectorizer/Passes/BottomUpVec.cpp   |  2 +-
 .../SandboxVectorizer/LegalityTest.cpp         | 13 +------------
 4 files changed, 4 insertions(+), 39 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
index d4b0b54375b0..bcfafd75d4ca 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
@@ -28,7 +28,6 @@ enum class LegalityResultID {
 
 /// The reason for vectorizing or not vectorizing.
 enum class ResultReason {
-  NotInstructions,
   DiffOpcodes,
   DiffTypes,
 };
@@ -47,8 +46,6 @@ struct ToStr {
 
   static const char *getVecReason(ResultReason Reason) {
     switch (Reason) {
-    case ResultReason::NotInstructions:
-      return "NotInstructions";
     case ResultReason::DiffOpcodes:
       return "DiffOpcodes";
     case ResultReason::DiffTypes:
@@ -70,10 +67,6 @@ protected:
   LegalityResult(LegalityResultID ID) : ID(ID) {}
   friend class LegalityAnalysis;
 
-  /// We shouldn't need copies.
-  LegalityResult(const LegalityResult &) = delete;
-  LegalityResult &operator=(const LegalityResult &) = delete;
-
 public:
   virtual ~LegalityResult() {}
   LegalityResultID getSubclassID() const { return ID; }
@@ -97,7 +90,6 @@ class LegalityResultWithReason : public LegalityResult {
   friend class Pack; // For constructor.
 
 public:
-  ResultReason getReason() const { return Reason; }
 #ifndef NDEBUG
   void print(raw_ostream &OS) const override {
     LegalityResult::print(OS);
@@ -146,7 +138,7 @@ public:
   }
   /// Checks if it's legal to vectorize the instructions in \p Bndl.
   /// \Returns a LegalityResult object owned by LegalityAnalysis.
-  const LegalityResult &canVectorize(ArrayRef<Value *> Bndl);
+  LegalityResult &canVectorize(ArrayRef<Value *> Bndl);
 };
 
 } // namespace llvm::sandboxir
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
index f1c4577cece7..0e2cd83c37b0 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
@@ -7,15 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
-#include "llvm/SandboxIR/Instruction.h"
-#include "llvm/SandboxIR/Utils.h"
 #include "llvm/SandboxIR/Value.h"
 #include "llvm/Support/Debug.h"
 
 namespace llvm::sandboxir {
 
-#define DEBUG_TYPE "SBVec:Legality"
-
 #ifndef NDEBUG
 void LegalityResult::dump() const {
   print(dbgs());
@@ -30,19 +26,7 @@ LegalityAnalysis::notVectorizableBasedOnOpcodesAndTypes(
   return std::nullopt;
 }
 
-static void dumpBndl(ArrayRef<Value *> Bndl) {
-  for (auto *V : Bndl)
-    dbgs() << *V << "\n";
-}
-
-const LegalityResult &LegalityAnalysis::canVectorize(ArrayRef<Value *> Bndl) {
-  // If Bndl contains values other than instructions, we need to Pack.
-  if (any_of(Bndl, [](auto *V) { return !isa<Instruction>(V); })) {
-    LLVM_DEBUG(dbgs() << "Not vectorizing: Not Instructions:\n";
-               dumpBndl(Bndl););
-    return createLegalityResult<Pack>(ResultReason::NotInstructions);
-  }
-
+LegalityResult &LegalityAnalysis::canVectorize(ArrayRef<Value *> Bndl) {
   if (auto ReasonOpt = notVectorizableBasedOnOpcodesAndTypes(Bndl))
     return createLegalityResult<Pack>(*ReasonOpt);
 
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index ede41cd661b5..f11420e47f3e 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -40,7 +40,7 @@ static SmallVector<Value *, 4> getOperand(ArrayRef<Value *> Bndl,
 }
 
 void BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl) {
-  const auto &LegalityRes = Legality.canVectorize(Bndl);
+  auto LegalityRes = Legality.canVectorize(Bndl);
   switch (LegalityRes.getSubclassID()) {
   case LegalityResultID::Widen: {
     auto *I = cast<Instruction>(Bndl[0]);
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
index 56c6bf5f1ef1..76e5a5ce5aed 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
@@ -52,16 +52,8 @@ define void @foo(ptr %ptr) {
   auto *St1 = cast<sandboxir::StoreInst>(&*It++);
 
   sandboxir::LegalityAnalysis Legality;
-  const auto &Result = Legality.canVectorize({St0, St1});
+  auto Result = Legality.canVectorize({St0, St1});
   EXPECT_TRUE(isa<sandboxir::Widen>(Result));
-
-  {
-    // Check NotInstructions
-    auto &Result = Legality.canVectorize({F, St0});
-    EXPECT_TRUE(isa<sandboxir::Pack>(Result));
-    EXPECT_EQ(cast<sandboxir::Pack>(Result).getReason(),
-              sandboxir::ResultReason::NotInstructions);
-  }
 }
 
 #ifndef NDEBUG
@@ -76,9 +68,6 @@ TEST_F(LegalityTest, LegalityResultDump) {
   sandboxir::LegalityAnalysis Legality;
   EXPECT_TRUE(
       Matches(Legality.createLegalityResult<sandboxir::Widen>(), "Widen"));
-  EXPECT_TRUE(Matches(Legality.createLegalityResult<sandboxir::Pack>(
-                          sandboxir::ResultReason::NotInstructions),
-                      "Pack Reason: NotInstructions"));
   EXPECT_TRUE(Matches(Legality.createLegalityResult<sandboxir::Pack>(
                           sandboxir::ResultReason::DiffOpcodes),
                       "Pack Reason: DiffOpcodes"));
-- 
GitLab


From 1540f772c793b3a29ae5d57e99456ec5d7ef4b39 Mon Sep 17 00:00:00 2001
From: Vasileios Porpodas <vporpodas@google.com>
Date: Fri, 25 Oct 2024 12:53:26 -0700
Subject: [PATCH 204/693] Reapply "[SandboxVec][Legality] Reject
 non-instructions (#113190)"

This reverts commit eb9f4756bc3daaa4b19f4f46521dc05180814de4.
---
 .../Vectorize/SandboxVectorizer/Legality.h    | 10 +++++++++-
 .../Vectorize/SandboxVectorizer/Legality.cpp  | 20 ++++++++++++++++++-
 .../SandboxVectorizer/Passes/BottomUpVec.cpp  |  2 +-
 .../SandboxVectorizer/LegalityTest.cpp        | 13 +++++++++++-
 4 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
index bcfafd75d4ca..d4b0b54375b0 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
@@ -28,6 +28,7 @@ enum class LegalityResultID {
 
 /// The reason for vectorizing or not vectorizing.
 enum class ResultReason {
+  NotInstructions,
   DiffOpcodes,
   DiffTypes,
 };
@@ -46,6 +47,8 @@ struct ToStr {
 
   static const char *getVecReason(ResultReason Reason) {
     switch (Reason) {
+    case ResultReason::NotInstructions:
+      return "NotInstructions";
     case ResultReason::DiffOpcodes:
       return "DiffOpcodes";
     case ResultReason::DiffTypes:
@@ -67,6 +70,10 @@ protected:
   LegalityResult(LegalityResultID ID) : ID(ID) {}
   friend class LegalityAnalysis;
 
+  /// We shouldn't need copies.
+  LegalityResult(const LegalityResult &) = delete;
+  LegalityResult &operator=(const LegalityResult &) = delete;
+
 public:
   virtual ~LegalityResult() {}
   LegalityResultID getSubclassID() const { return ID; }
@@ -90,6 +97,7 @@ class LegalityResultWithReason : public LegalityResult {
   friend class Pack; // For constructor.
 
 public:
+  ResultReason getReason() const { return Reason; }
 #ifndef NDEBUG
   void print(raw_ostream &OS) const override {
     LegalityResult::print(OS);
@@ -138,7 +146,7 @@ public:
   }
   /// Checks if it's legal to vectorize the instructions in \p Bndl.
   /// \Returns a LegalityResult object owned by LegalityAnalysis.
-  LegalityResult &canVectorize(ArrayRef<Value *> Bndl);
+  const LegalityResult &canVectorize(ArrayRef<Value *> Bndl);
 };
 
 } // namespace llvm::sandboxir
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
index 0e2cd83c37b0..e4546c2f9811 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
@@ -7,11 +7,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
+#include "llvm/SandboxIR/Instruction.h"
+#include "llvm/SandboxIR/Utils.h"
 #include "llvm/SandboxIR/Value.h"
 #include "llvm/Support/Debug.h"
 
 namespace llvm::sandboxir {
 
+#define DEBUG_TYPE "SBVec:Legality"
+
 #ifndef NDEBUG
 void LegalityResult::dump() const {
   print(dbgs());
@@ -26,7 +30,21 @@ LegalityAnalysis::notVectorizableBasedOnOpcodesAndTypes(
   return std::nullopt;
 }
 
-LegalityResult &LegalityAnalysis::canVectorize(ArrayRef<Value *> Bndl) {
+#ifndef NDEBUG
+static void dumpBndl(ArrayRef<Value *> Bndl) {
+  for (auto *V : Bndl)
+    dbgs() << *V << "\n";
+}
+#endif // NDEBUG
+
+const LegalityResult &LegalityAnalysis::canVectorize(ArrayRef<Value *> Bndl) {
+  // If Bndl contains values other than instructions, we need to Pack.
+  if (any_of(Bndl, [](auto *V) { return !isa<Instruction>(V); })) {
+    LLVM_DEBUG(dbgs() << "Not vectorizing: Not Instructions:\n";
+               dumpBndl(Bndl););
+    return createLegalityResult<Pack>(ResultReason::NotInstructions);
+  }
+
   if (auto ReasonOpt = notVectorizableBasedOnOpcodesAndTypes(Bndl))
     return createLegalityResult<Pack>(*ReasonOpt);
 
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index f11420e47f3e..ede41cd661b5 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -40,7 +40,7 @@ static SmallVector<Value *, 4> getOperand(ArrayRef<Value *> Bndl,
 }
 
 void BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl) {
-  auto LegalityRes = Legality.canVectorize(Bndl);
+  const auto &LegalityRes = Legality.canVectorize(Bndl);
   switch (LegalityRes.getSubclassID()) {
   case LegalityResultID::Widen: {
     auto *I = cast<Instruction>(Bndl[0]);
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
index 76e5a5ce5aed..56c6bf5f1ef1 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
@@ -52,8 +52,16 @@ define void @foo(ptr %ptr) {
   auto *St1 = cast<sandboxir::StoreInst>(&*It++);
 
   sandboxir::LegalityAnalysis Legality;
-  auto Result = Legality.canVectorize({St0, St1});
+  const auto &Result = Legality.canVectorize({St0, St1});
   EXPECT_TRUE(isa<sandboxir::Widen>(Result));
+
+  {
+    // Check NotInstructions
+    auto &Result = Legality.canVectorize({F, St0});
+    EXPECT_TRUE(isa<sandboxir::Pack>(Result));
+    EXPECT_EQ(cast<sandboxir::Pack>(Result).getReason(),
+              sandboxir::ResultReason::NotInstructions);
+  }
 }
 
 #ifndef NDEBUG
@@ -68,6 +76,9 @@ TEST_F(LegalityTest, LegalityResultDump) {
   sandboxir::LegalityAnalysis Legality;
   EXPECT_TRUE(
       Matches(Legality.createLegalityResult<sandboxir::Widen>(), "Widen"));
+  EXPECT_TRUE(Matches(Legality.createLegalityResult<sandboxir::Pack>(
+                          sandboxir::ResultReason::NotInstructions),
+                      "Pack Reason: NotInstructions"));
   EXPECT_TRUE(Matches(Legality.createLegalityResult<sandboxir::Pack>(
                           sandboxir::ResultReason::DiffOpcodes),
                       "Pack Reason: DiffOpcodes"));
-- 
GitLab


From cfde4fbccf5d8d949a8cade0a4f8ef9b0f47ca73 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Fri, 25 Oct 2024 16:46:38 -0400
Subject: [PATCH 205/693] [libc++] Remove obsolete Solaris and Newlib support
 for locales (#113721)

The solaris header file doesn't even exist, so that's definitely dead
code. The newlib header is empty, which means that localization can't
work on that platform. If someone is using libc++ with Newlib, they must
be providing LIBCXX_HAS_NO_LOCALIZATION today for anything to work, so
that header is basically dead code as well.
---
 libcxx/include/CMakeLists.txt                        |  1 -
 libcxx/include/__locale_dir/locale_base_api.h        |  4 ----
 libcxx/include/__locale_dir/locale_base_api/newlib.h | 12 ------------
 libcxx/include/module.modulemap                      |  1 -
 4 files changed, 18 deletions(-)
 delete mode 100644 libcxx/include/__locale_dir/locale_base_api/newlib.h

diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 63aa74e09bb1..506ed721d084 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -497,7 +497,6 @@ set(files
   __locale_dir/locale_base_api/fuchsia.h
   __locale_dir/locale_base_api/ibm.h
   __locale_dir/locale_base_api/musl.h
-  __locale_dir/locale_base_api/newlib.h
   __locale_dir/locale_base_api/openbsd.h
   __locale_dir/locale_base_api/win32.h
   __locale_dir/locale_guard.h
diff --git a/libcxx/include/__locale_dir/locale_base_api.h b/libcxx/include/__locale_dir/locale_base_api.h
index 8c000c558c52..eab7fa8bf62f 100644
--- a/libcxx/include/__locale_dir/locale_base_api.h
+++ b/libcxx/include/__locale_dir/locale_base_api.h
@@ -15,10 +15,6 @@
 #  include <__locale_dir/locale_base_api/ibm.h>
 #elif defined(__ANDROID__)
 #  include <__locale_dir/locale_base_api/android.h>
-#elif defined(__sun__)
-#  include <__locale_dir/locale_base_api/solaris.h>
-#elif defined(_NEWLIB_VERSION)
-#  include <__locale_dir/locale_base_api/newlib.h>
 #elif defined(__OpenBSD__)
 #  include <__locale_dir/locale_base_api/openbsd.h>
 #elif defined(__Fuchsia__)
diff --git a/libcxx/include/__locale_dir/locale_base_api/newlib.h b/libcxx/include/__locale_dir/locale_base_api/newlib.h
deleted file mode 100644
index 7da10e588984..000000000000
--- a/libcxx/include/__locale_dir/locale_base_api/newlib.h
+++ /dev/null
@@ -1,12 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_NEWLIB_H
-#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_NEWLIB_H
-
-#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_NEWLIB_H
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index c79070c31875..f92e8bf5fc9a 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1451,7 +1451,6 @@ module std [system] {
       textual header "__locale_dir/locale_base_api/fuchsia.h"
       textual header "__locale_dir/locale_base_api/ibm.h"
       textual header "__locale_dir/locale_base_api/musl.h"
-      textual header "__locale_dir/locale_base_api/newlib.h"
       textual header "__locale_dir/locale_base_api/openbsd.h"
       textual header "__locale_dir/locale_base_api/win32.h"
     }
-- 
GitLab


From 1bc2cd98c58a1059170dc38697c7a29a8e21160b Mon Sep 17 00:00:00 2001
From: Dan Gohman <dev@sunfishcode.online>
Date: Fri, 25 Oct 2024 13:52:51 -0700
Subject: [PATCH 206/693] [WebAssembly] Enable nontrapping-fptoint and
 bulk-memory by default. (#112049)

We were prepared to enable these features [back in February], but they
got pulled for what appear to be unrelated reasons. So let's have
another try at enabling them!

Another motivation here is that it'd be convenient for the [Lime1
proposal] if "lime1" is close to a subset of "generic" (missing only
for extended-const).

[back in February]:
https://github.com/WebAssembly/tool-conventions/issues/158#issuecomment-1931119512
[Lime1 proposal]: https://github.com/llvm/llvm-project/pull/112035
---
 clang/docs/ReleaseNotes.rst                   |  9 ++++++
 clang/lib/Basic/Targets/WebAssembly.cpp       |  4 +--
 .../test/Preprocessor/wasm-target-features.c  |  4 +--
 lld/test/wasm/custom-section-name.ll          |  2 +-
 lld/test/wasm/data-segments.ll                |  2 +-
 lld/test/wasm/lto/Inputs/libcall-archive.ll   |  4 ++-
 lld/test/wasm/lto/libcall-archive.ll          |  4 ++-
 lld/test/wasm/lto/stub-library-libcall.s      |  4 +--
 llvm/docs/ReleaseNotes.md                     |  9 ++++++
 llvm/lib/Target/WebAssembly/WebAssembly.td    |  3 +-
 .../WebAssemblyFixFunctionBitcasts.cpp        |  2 ++
 .../WebAssembly/WebAssemblyTargetMachine.cpp  | 29 +++++++++++++++----
 .../WebAssembly/cfg-stackify-eh-legacy.ll     | 10 +++----
 .../WebAssembly/target-features-cpus.ll       |  8 ++++-
 .../WebAssembly/extern-functype-intrinsic.ll  |  4 +--
 llvm/test/MC/WebAssembly/libcall.ll           |  2 +-
 16 files changed, 74 insertions(+), 26 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 170c4cc28053..6a9533781517 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -690,6 +690,15 @@ NetBSD Support
 WebAssembly Support
 ^^^^^^^^^^^^^^^^^^^
 
+The default target CPU, "generic", now enables the `-mnontrapping-fptoint`
+and `-mbulk-memory` flags, which correspond to the [Bulk Memory Operations]
+and [Non-trapping float-to-int Conversions] language features, which are
+[widely implemented in engines].
+
+[Bulk Memory Operations]: https://github.com/WebAssembly/bulk-memory-operations/blob/master/proposals/bulk-memory-operations/Overview.md
+[Non-trapping float-to-int Conversions]: https://github.com/WebAssembly/spec/blob/master/proposals/nontrapping-float-to-int-conversion/Overview.md
+[widely implemented in engines]: https://webassembly.org/features/
+
 AVR Support
 ^^^^^^^^^^^
 
diff --git a/clang/lib/Basic/Targets/WebAssembly.cpp b/clang/lib/Basic/Targets/WebAssembly.cpp
index 4c9df6007b78..0b380bdf835f 100644
--- a/clang/lib/Basic/Targets/WebAssembly.cpp
+++ b/clang/lib/Basic/Targets/WebAssembly.cpp
@@ -154,20 +154,20 @@ bool WebAssemblyTargetInfo::initFeatureMap(
     llvm::StringMap<bool> &Features, DiagnosticsEngine &Diags, StringRef CPU,
     const std::vector<std::string> &FeaturesVec) const {
   auto addGenericFeatures = [&]() {
+    Features["bulk-memory"] = true;
     Features["multivalue"] = true;
     Features["mutable-globals"] = true;
+    Features["nontrapping-fptoint"] = true;
     Features["reference-types"] = true;
     Features["sign-ext"] = true;
   };
   auto addBleedingEdgeFeatures = [&]() {
     addGenericFeatures();
     Features["atomics"] = true;
-    Features["bulk-memory"] = true;
     Features["exception-handling"] = true;
     Features["extended-const"] = true;
     Features["fp16"] = true;
     Features["multimemory"] = true;
-    Features["nontrapping-fptoint"] = true;
     Features["tail-call"] = true;
     Features["wide-arithmetic"] = true;
     setSIMDLevel(Features, RelaxedSIMD, true);
diff --git a/clang/test/Preprocessor/wasm-target-features.c b/clang/test/Preprocessor/wasm-target-features.c
index 14d2fbf4423d..71b7cf6a5d43 100644
--- a/clang/test/Preprocessor/wasm-target-features.c
+++ b/clang/test/Preprocessor/wasm-target-features.c
@@ -163,8 +163,10 @@
 // RUN:     -target wasm64-unknown-unknown -mcpu=generic \
 // RUN:   | FileCheck %s -check-prefix=GENERIC-INCLUDE
 //
+// GENERIC-INCLUDE-DAG: #define __wasm_bulk_memory__ 1{{$}}
 // GENERIC-INCLUDE-DAG: #define __wasm_multivalue__ 1{{$}}
 // GENERIC-INCLUDE-DAG: #define __wasm_mutable_globals__ 1{{$}}
+// GENERIC-INCLUDE-DAG: #define __wasm_nontrapping_fptoint__ 1{{$}}
 // GENERIC-INCLUDE-DAG: #define __wasm_reference_types__ 1{{$}}
 // GENERIC-INCLUDE-DAG: #define __wasm_sign_ext__ 1{{$}}
 //
@@ -176,12 +178,10 @@
 // RUN:   | FileCheck %s -check-prefix=GENERIC
 //
 // GENERIC-NOT: #define __wasm_atomics__ 1{{$}}
-// GENERIC-NOT: #define __wasm_bulk_memory__ 1{{$}}
 // GENERIC-NOT: #define __wasm_exception_handling__ 1{{$}}
 // GENERIC-NOT: #define __wasm_extended_const__ 1{{$}}
 // GENERIC-NOT: #define __wasm__fp16__ 1{{$}}
 // GENERIC-NOT: #define __wasm_multimemory__ 1{{$}}
-// GENERIC-NOT: #define __wasm_nontrapping_fptoint__ 1{{$}}
 // GENERIC-NOT: #define __wasm_relaxed_simd__ 1{{$}}
 // GENERIC-NOT: #define __wasm_simd128__ 1{{$}}
 // GENERIC-NOT: #define __wasm_tail_call__ 1{{$}}
diff --git a/lld/test/wasm/custom-section-name.ll b/lld/test/wasm/custom-section-name.ll
index b860ef5a83e8..8799fbf36056 100644
--- a/lld/test/wasm/custom-section-name.ll
+++ b/lld/test/wasm/custom-section-name.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=obj %s -o %t.o
+; RUN: llc -filetype=obj -mattr=-bulk-memory %s -o %t.o
 ; RUN: wasm-ld -no-gc-sections --no-entry -o %t.wasm %t.o
 ; RUN: obj2yaml %t.wasm | FileCheck %s --check-prefixes=CHECK,NO-BSS
 ; RUN: wasm-ld -no-gc-sections --no-entry --import-memory -o %t.bss.wasm %t.o
diff --git a/lld/test/wasm/data-segments.ll b/lld/test/wasm/data-segments.ll
index 670ac3c1f373..41868a0b2b50 100644
--- a/lld/test/wasm/data-segments.ll
+++ b/lld/test/wasm/data-segments.ll
@@ -1,4 +1,4 @@
-; RUN: llc --mtriple=wasm32-unknown-unknown -filetype=obj %s -o %t.atomics.o -mattr=+atomics
+; RUN: llc --mtriple=wasm32-unknown-unknown -filetype=obj %s -o %t.atomics.o -mattr=+atomics,-bulk-memory
 ; RUN: llc --mtriple=wasm32-unknown-unknown -filetype=obj %s -o %t.bulk-mem.o -mattr=+bulk-memory
 ; RUN: llc --mtriple=wasm64-unknown-unknown -filetype=obj %s -o %t.bulk-mem64.o -mattr=+bulk-memory
 ; RUN: llc --mtriple=wasm32-unknown-unknown -filetype=obj %s -o %t.atomics.bulk-mem.o -mattr=+atomics,+bulk-memory
diff --git a/lld/test/wasm/lto/Inputs/libcall-archive.ll b/lld/test/wasm/lto/Inputs/libcall-archive.ll
index 9d05efdeae08..7d8c34196dfe 100644
--- a/lld/test/wasm/lto/Inputs/libcall-archive.ll
+++ b/lld/test/wasm/lto/Inputs/libcall-archive.ll
@@ -1,6 +1,8 @@
 target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
-define void @memcpy() {
+define void @memcpy() #0 {
   ret void
 }
+
+attributes #0 = { "target-features"="-bulk-memory" }
diff --git a/lld/test/wasm/lto/libcall-archive.ll b/lld/test/wasm/lto/libcall-archive.ll
index 2f785b98976e..5c46d2f7ed78 100644
--- a/lld/test/wasm/lto/libcall-archive.ll
+++ b/lld/test/wasm/lto/libcall-archive.ll
@@ -8,7 +8,7 @@
 target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
-define void @_start(ptr %a, ptr %b) {
+define void @_start(ptr %a, ptr %b) #0 {
 entry:
   call void @llvm.memcpy.p0.p0.i64(ptr %a, ptr %b, i64 1024, i1 false)
   ret void
@@ -16,6 +16,8 @@ entry:
 
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1)
 
+attributes #0 = { "target-features"="-bulk-memory" }
+
 ; CHECK:       - Type:            CUSTOM
 ; CHECK-NEXT:    Name:            name
 ; CHECK-NEXT:    FunctionNames:
diff --git a/lld/test/wasm/lto/stub-library-libcall.s b/lld/test/wasm/lto/stub-library-libcall.s
index ce88a32dd99d..d65983c0cf5b 100644
--- a/lld/test/wasm/lto/stub-library-libcall.s
+++ b/lld/test/wasm/lto/stub-library-libcall.s
@@ -2,7 +2,7 @@
 # RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown -o %t_main.o %t/main.s
 # RUN: llvm-as %S/Inputs/foo.ll -o %t_foo.o
 # RUN: llvm-as %S/Inputs/libcall.ll -o %t_libcall.o
-# RUN: wasm-ld %t_main.o %t_libcall.o %t_foo.o %p/Inputs/stub.so -o %t.wasm
+# RUN: wasm-ld -mllvm -mattr=-bulk-memory %t_main.o %t_libcall.o %t_foo.o %p/Inputs/stub.so -o %t.wasm
 # RUN: obj2yaml %t.wasm | FileCheck %s
 
 # The function `func_with_libcall` will generate an undefined reference to
@@ -12,7 +12,7 @@
 # If %t_foo.o is not included in the link we get an undefined symbol reported
 # to the dependency of memcpy on the foo export:
 
-# RUN: not wasm-ld %t_main.o %t_libcall.o %p/Inputs/stub.so -o %t.wasm 2>&1 | FileCheck --check-prefix=MISSING %s
+# RUN: not wasm-ld -mllvm -mattr=-bulk-memory %t_main.o %t_libcall.o %p/Inputs/stub.so -o %t.wasm 2>&1 | FileCheck --check-prefix=MISSING %s
 # MISSING: stub.so: undefined symbol: foo. Required by memcpy
 
 #--- main.s
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index be51b0af56dd..e3d93f0dfd0e 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -180,6 +180,15 @@ Changes to the RISC-V Backend
 Changes to the WebAssembly Backend
 ----------------------------------
 
+The default target CPU, "generic", now enables the `-mnontrapping-fptoint`
+and `-mbulk-memory` flags, which correspond to the [Bulk Memory Operations]
+and [Non-trapping float-to-int Conversions] language features, which are
+[widely implemented in engines].
+
+[Bulk Memory Operations]: https://github.com/WebAssembly/bulk-memory-operations/blob/master/proposals/bulk-memory-operations/Overview.md
+[Non-trapping float-to-int Conversions]: https://github.com/WebAssembly/spec/blob/master/proposals/nontrapping-float-to-int-conversion/Overview.md
+[widely implemented in engines]: https://webassembly.org/features/
+
 Changes to the Windows Target
 -----------------------------
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssembly.td b/llvm/lib/Target/WebAssembly/WebAssembly.td
index 37d99690c25b..88628f2a7935 100644
--- a/llvm/lib/Target/WebAssembly/WebAssembly.td
+++ b/llvm/lib/Target/WebAssembly/WebAssembly.td
@@ -114,7 +114,8 @@ def : ProcessorModel<"mvp", NoSchedModel, []>;
 // consideration given to available support in relevant engines and tools, and
 // the importance of the features.
 def : ProcessorModel<"generic", NoSchedModel,
-                      [FeatureMultivalue, FeatureMutableGlobals,
+                      [FeatureBulkMemory, FeatureMultivalue,
+                       FeatureMutableGlobals, FeatureNontrappingFPToInt,
                        FeatureReferenceTypes, FeatureSignExt]>;
 
 // Latest and greatest experimental version of WebAssembly. Bugs included!
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp
index a3cc9bae4708..7c3e8d18ad27 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp
@@ -111,6 +111,7 @@ static Function *createWrapper(Function *F, FunctionType *Ty) {
 
   Function *Wrapper = Function::Create(Ty, Function::PrivateLinkage,
                                        F->getName() + "_bitcast", M);
+  Wrapper->setAttributes(F->getAttributes());
   BasicBlock *BB = BasicBlock::Create(M->getContext(), "body", Wrapper);
   const DataLayout &DL = BB->getDataLayout();
 
@@ -201,6 +202,7 @@ static Function *createWrapper(Function *F, FunctionType *Ty) {
     Wrapper->eraseFromParent();
     Wrapper = Function::Create(Ty, Function::PrivateLinkage,
                                F->getName() + "_bitcast_invalid", M);
+    Wrapper->setAttributes(F->getAttributes());
     BasicBlock *BB = BasicBlock::Create(M->getContext(), "body", Wrapper);
     new UnreachableInst(M->getContext(), BB);
     Wrapper->setName(F->getName() + "_bitcast_invalid");
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
index 3fe6ccf1c608..83cd57d0bbdd 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
@@ -233,13 +233,30 @@ public:
 
 private:
   FeatureBitset coalesceFeatures(const Module &M) {
-    FeatureBitset Features =
-        WasmTM
-            ->getSubtargetImpl(std::string(WasmTM->getTargetCPU()),
-                               std::string(WasmTM->getTargetFeatureString()))
-            ->getFeatureBits();
-    for (auto &F : M)
+    // Union the features of all defined functions. Start with an empty set, so
+    // that if a feature is disabled in every function, we'll compute it as
+    // disabled. If any function lacks a target-features attribute, it'll
+    // default to the target CPU from the `TargetMachine`.
+    FeatureBitset Features;
+    bool AnyDefinedFuncs = false;
+    for (auto &F : M) {
+      if (F.isDeclaration())
+        continue;
+
       Features |= WasmTM->getSubtargetImpl(F)->getFeatureBits();
+      AnyDefinedFuncs = true;
+    }
+
+    // If we have no defined functions, use the target CPU from the
+    // `TargetMachine`.
+    if (!AnyDefinedFuncs) {
+      Features =
+          WasmTM
+              ->getSubtargetImpl(std::string(WasmTM->getTargetCPU()),
+                                 std::string(WasmTM->getTargetFeatureString()))
+              ->getFeatureBits();
+    }
+
     return Features;
   }
 
diff --git a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.ll b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.ll
index cef92f459e4a..24a08267db6f 100644
--- a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.ll
+++ b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.ll
@@ -1,9 +1,9 @@
 ; REQUIRES: asserts
-; RUN: llc < %s -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling | FileCheck %s
-; RUN: llc < %s -disable-wasm-fallthrough-return-opt -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling
-; RUN: llc < %s -O0 -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -verify-machineinstrs -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling | FileCheck %s --check-prefix=NOOPT
-; RUN: llc < %s -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling -wasm-disable-ehpad-sort -stats 2>&1 | FileCheck %s --check-prefix=NOSORT
-; RUN: llc < %s -disable-wasm-fallthrough-return-opt -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling -wasm-disable-ehpad-sort | FileCheck %s --check-prefix=NOSORT-LOCALS
+; RUN: llc < %s -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling,bulk-memory | FileCheck %s
+; RUN: llc < %s -disable-wasm-fallthrough-return-opt -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling,bulk-memory
+; RUN: llc < %s -O0 -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -verify-machineinstrs -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling,-bulk-memory | FileCheck %s --check-prefix=NOOPT
+; RUN: llc < %s -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling,-bulk-memory -wasm-disable-ehpad-sort -stats 2>&1 | FileCheck %s --check-prefix=NOSORT
+; RUN: llc < %s -disable-wasm-fallthrough-return-opt -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling,-bulk-memory -wasm-disable-ehpad-sort | FileCheck %s --check-prefix=NOSORT-LOCALS
 
 target triple = "wasm32-unknown-unknown"
 
diff --git a/llvm/test/CodeGen/WebAssembly/target-features-cpus.ll b/llvm/test/CodeGen/WebAssembly/target-features-cpus.ll
index 77d1564409f7..ba10dd94a983 100644
--- a/llvm/test/CodeGen/WebAssembly/target-features-cpus.ll
+++ b/llvm/test/CodeGen/WebAssembly/target-features-cpus.ll
@@ -13,7 +13,10 @@ target triple = "wasm32-unknown-unknown"
 
 ; generic: +multivalue, +mutable-globals, +reference-types, +sign-ext
 ; GENERIC-LABEL: .custom_section.target_features,"",@
-; GENERIC-NEXT: .int8  4
+; GENERIC-NEXT: .int8  6
+; GENERIC-NEXT: .int8  43
+; GENERIC-NEXT: .int8  11
+; GENERIC-NEXT: .ascii  "bulk-memory"
 ; GENERIC-NEXT: .int8  43
 ; GENERIC-NEXT: .int8  10
 ; GENERIC-NEXT: .ascii  "multivalue"
@@ -21,6 +24,9 @@ target triple = "wasm32-unknown-unknown"
 ; GENERIC-NEXT: .int8  15
 ; GENERIC-NEXT: .ascii  "mutable-globals"
 ; GENERIC-NEXT: .int8  43
+; GENERIC-NEXT: .int8  19
+; GENERIC-NEXT: .ascii  "nontrapping-fptoint"
+; GENERIC-NEXT: .int8  43
 ; GENERIC-NEXT: .int8  15
 ; GENERIC-NEXT: .ascii  "reference-types"
 ; GENERIC-NEXT: .int8  43
diff --git a/llvm/test/MC/WebAssembly/extern-functype-intrinsic.ll b/llvm/test/MC/WebAssembly/extern-functype-intrinsic.ll
index 320b65356ba9..b321c0c82ad4 100644
--- a/llvm/test/MC/WebAssembly/extern-functype-intrinsic.ll
+++ b/llvm/test/MC/WebAssembly/extern-functype-intrinsic.ll
@@ -1,5 +1,5 @@
-; RUN: llc %s -o - | FileCheck %s
-; RUN: llc %s -o - | llvm-mc -triple=wasm32-unknown-unknown | FileCheck %s
+; RUN: llc %s -mattr=-bulk-memory -o - | FileCheck %s
+; RUN: llc %s -mattr=-bulk-memory -o - | llvm-mc -triple=wasm32-unknown-unknown | FileCheck %s
 
 ; ModuleID = 'test.c'
 source_filename = "test.c"
diff --git a/llvm/test/MC/WebAssembly/libcall.ll b/llvm/test/MC/WebAssembly/libcall.ll
index 8b81f150da89..ffd32abe2345 100644
--- a/llvm/test/MC/WebAssembly/libcall.ll
+++ b/llvm/test/MC/WebAssembly/libcall.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=obj %s -o - | obj2yaml | FileCheck %s
+; RUN: llc -filetype=obj -mattr=-bulk-memory %s -o - | obj2yaml | FileCheck %s
 
 target triple = "wasm32-unknown-unknown"
 
-- 
GitLab


From a93c952bab3569c84ae0f9619533f7d7c2477a41 Mon Sep 17 00:00:00 2001
From: Chris Apple <cja-private@pm.me>
Date: Fri, 25 Oct 2024 14:21:32 -0700
Subject: [PATCH 207/693] [rtsan][NFC] Documentation of suppression flag
 (#112727)

---
 clang/docs/RealtimeSanitizer.rst | 68 ++++++++++++++++++++++++++++++--
 1 file changed, 64 insertions(+), 4 deletions(-)

diff --git a/clang/docs/RealtimeSanitizer.rst b/clang/docs/RealtimeSanitizer.rst
index 103842e055db..b09162cd99f4 100644
--- a/clang/docs/RealtimeSanitizer.rst
+++ b/clang/docs/RealtimeSanitizer.rst
@@ -183,6 +183,10 @@ A **partial** list of flags RealtimeSanitizer respects:
      - ``true``
      - boolean
      - If set, use the symbolizer to turn virtual addresses to file/line locations. If false, can greatly speed up the error reporting.
+   * - ``suppressions``
+     - ""
+     - path
+     - If set to a valid suppressions file, will suppress issue reporting. See details in "Disabling", below.
 
 
 Some issues with flags can be debugged using the ``verbosity=$NUM`` flag:
@@ -194,12 +198,43 @@ Some issues with flags can be debugged using the ``verbosity=$NUM`` flag:
    misspelled_flag
    ...
 
-Disabling
----------
+Disabling and suppressing
+-------------------------
 
-In some circumstances, you may want to suppress error reporting in a specific scope.
+There are multiple ways to disable error reporting when using RealtimeSanitizer.
 
-In C++, this is achieved via  ``__rtsan::ScopedDisabler``. Within the scope where the ``ScopedDisabler`` object is instantiated, all sanitizer error reports are suppressed. This suppression applies to the current scope as well as all invoked functions, including any functions called transitively.
+In general, ``ScopedDisabler`` should be preferred, as it is the most performant.
+
+.. list-table:: Suppression methods
+   :widths: 30 15 15 10 70
+   :header-rows: 1
+
+   * - Method
+     - Specified at?
+     - Scope
+     - Run-time cost
+     - Description
+   * - ``ScopedDisabler``
+     - Compile-time
+     - Stack
+     - Very low
+     - Violations are ignored for the lifetime of the ``ScopedDisabler`` object.
+   * - ``function-name-matches`` suppression
+     - Run-time
+     - Single function
+     - Medium
+     - Suppresses intercepted and ``[[clang::blocking]]`` function calls by name.
+   * - ``call-stack-contains`` suppression
+     - Run-time
+     - Stack
+     - High
+     - Suppresses any stack trace contaning the specified pattern.
+    
+
+``ScopedDisabler``
+##################
+
+At compile time, RealtimeSanitizer may be disabled using ``__rtsan::ScopedDisabler``. RTSan ignores any errors originating within the ``ScopedDisabler`` instance variable scope.
 
 .. code-block:: c++
 
@@ -233,6 +268,31 @@ In C, you can use the ``__rtsan_disable()`` and ``rtsan_enable()`` functions to
 
 Each call to ``__rtsan_disable()`` must be paired with a subsequent call to ``__rtsan_enable()`` to restore normal sanitizer functionality. If a corresponding ``rtsan_enable()`` call is not made, the behavior is undefined.
 
+Suppression file
+################
+
+At run-time, suppressions may be specified using a suppressions file passed in ``RTSAN_OPTIONS``. Run-time suppression may be useful if the source cannot be changed.
+
+.. code-block:: console
+
+   > cat suppressions.supp
+   call-stack-contains:MallocViolation
+   call-stack-contains:std::*vector
+   function-name-matches:free
+   function-name-matches:CustomMarkedBlocking*
+   > RTSAN_OPTIONS="suppressions=suppressions.supp" ./a.out
+   ...
+
+Suppressions specified in this file are one of two flavors.
+
+``function-name-matches`` suppresses reporting of any intercepted library call, or function marked ``[[clang::blocking]]`` by name. If, for instance, you know that ``malloc`` is real-time safe on your system, you can disable the check for it via ``function-name-matches:malloc``.
+
+``call-stack-contains`` suppresses reporting of errors in any stack that contains a string matching the pattern specified. For example, suppressing error reporting of any non-real-time-safe behavior in ``std::vector`` may be specified ``call-stack-contains:std::*vector``. You must include symbols in your build for this method to be effective, unsymbolicated stack traces cannot be matched. ``call-stack-contains`` has the highest run-time cost of any method of suppression.
+
+Patterns may be exact matches or are "regex-light" patterns, containing special characters such as ``^$*``.
+
+The number of potential errors suppressed via this method may be seen on exit when using the ``print_stats_on_exit`` flag.
+
 Compile-time sanitizer detection
 --------------------------------
 
-- 
GitLab


From 590b1e31546572b62040066f90a35893a1b64f29 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Fri, 25 Oct 2024 15:00:07 -0700
Subject: [PATCH 208/693] [clang][modules] Only serialize info for
 locally-included headers (#113718)

I noticed that some PCM files contain `HeaderFileInfo` for headers only
included in a dependent PCM file, which is wasteful.

This patch changes the logic to only write headers that are included
locally. This makes the PCM files smaller and saves some superfluous
deserialization of `HeaderFileInfo` triggered by
`Preprocessor::alreadyIncluded()`.
---
 clang/include/clang/Lex/Preprocessor.h | 2 +-
 clang/lib/Lex/HeaderSearch.cpp         | 1 -
 clang/lib/Serialization/ASTWriter.cpp  | 6 +++---
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index f3f4de044fc4..38a527d2324f 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -1490,7 +1490,7 @@ public:
   /// Mark the file as included.
   /// Returns true if this is the first time the file was included.
   bool markIncluded(FileEntryRef File) {
-    HeaderInfo.getFileInfo(File);
+    HeaderInfo.getFileInfo(File).IsLocallyIncluded = true;
     return IncludedFiles.insert(File).second;
   }
 
diff --git a/clang/lib/Lex/HeaderSearch.cpp b/clang/lib/Lex/HeaderSearch.cpp
index 8826ab449df4..052be1395161 100644
--- a/clang/lib/Lex/HeaderSearch.cpp
+++ b/clang/lib/Lex/HeaderSearch.cpp
@@ -1582,7 +1582,6 @@ bool HeaderSearch::ShouldEnterIncludeFile(Preprocessor &PP,
     }
   }
 
-  FileInfo.IsLocallyIncluded = true;
   IsFirstIncludeOfFile = PP.markIncluded(File);
   return true;
 }
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index b576822fa704..c09a41f4d140 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -2163,8 +2163,8 @@ void ASTWriter::WriteHeaderSearch(const HeaderSearch &HS) {
       continue; // We have no information on this being a header file.
     if (!HFI->isCompilingModuleHeader && HFI->isModuleHeader)
       continue; // Header file info is tracked by the owning module file.
-    if (!HFI->isCompilingModuleHeader && !PP->alreadyIncluded(*File))
-      continue; // Non-modular header not included is not needed.
+    if (!HFI->isCompilingModuleHeader && !HFI->IsLocallyIncluded)
+      continue; // Header file info is tracked by the including module file.
 
     // Massage the file path into an appropriate form.
     StringRef Filename = File->getName();
@@ -2176,7 +2176,7 @@ void ASTWriter::WriteHeaderSearch(const HeaderSearch &HS) {
       SavedStrings.push_back(Filename.data());
     }
 
-    bool Included = PP->alreadyIncluded(*File);
+    bool Included = HFI->IsLocallyIncluded || PP->alreadyIncluded(*File);
 
     HeaderFileInfoTrait::key_type Key = {
       Filename, File->getSize(), getTimestampForOutput(*File)
-- 
GitLab


From 5f0a62817fc4e68e793e6d5e5c5fb9b614a1c8bf Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 25 Oct 2024 15:07:48 -0700
Subject: [PATCH 209/693] [RISCV] Remove duplicate Smrnmi CSR test cases. NFC

ed6ddffb583beb450c4b0e1747ccd14f7e063105 added tests that already
existed.
---
 llvm/test/MC/RISCV/machine-csr-names.s | 60 --------------------------
 1 file changed, 60 deletions(-)

diff --git a/llvm/test/MC/RISCV/machine-csr-names.s b/llvm/test/MC/RISCV/machine-csr-names.s
index d509f9eadeb5..8cfdf7ee116c 100644
--- a/llvm/test/MC/RISCV/machine-csr-names.s
+++ b/llvm/test/MC/RISCV/machine-csr-names.s
@@ -1913,66 +1913,6 @@ csrrs t1, mhpmcounter31, zero
 csrrs t2, 0xB1F, zero
 
 
-######################################
-# Machine Counter Setup
-######################################
-# mnscratch
-# name
-# CHECK-INST: csrrs t1, mnscratch, zero
-# CHECK-ENC: encoding: [0x73,0x23,0x00,0x74]
-# CHECK-INST-ALIAS: csrr t1, mnscratch
-# uimm12
-# CHECK-INST: csrrs t2, mnscratch, zero
-# CHECK-ENC: encoding: [0xf3,0x23,0x00,0x74]
-# CHECK-INST-ALIAS: csrr t2, mnscratch
-# name
-csrrs t1, mnscratch, zero
-# uimm12
-csrrs t2, 0x740, zero
-
-# mnepc
-# name
-# CHECK-INST: csrrs t1, mnepc, zero
-# CHECK-ENC: encoding: [0x73,0x23,0x10,0x74]
-# CHECK-INST-ALIAS: csrr t1, mnepc
-# uimm12
-# CHECK-INST: csrrs t2, mnepc, zero
-# CHECK-ENC: encoding: [0xf3,0x23,0x10,0x74]
-# CHECK-INST-ALIAS: csrr t2, mnepc
-# name
-csrrs t1, mnepc, zero
-# uimm12
-csrrs t2, 0x741, zero
-
-# mncause
-# name
-# CHECK-INST: csrrs t1, mncause, zero
-# CHECK-ENC: encoding: [0x73,0x23,0x20,0x74]
-# CHECK-INST-ALIAS: csrr t1, mncause
-# uimm12
-# CHECK-INST: csrrs t2, mncause, zero
-# CHECK-ENC: encoding: [0xf3,0x23,0x20,0x74]
-# CHECK-INST-ALIAS: csrr t2, mncause
-# name
-csrrs t1, mncause, zero
-# uimm12
-csrrs t2, 0x742, zero
-
-# mnstatus
-# name
-# CHECK-INST: csrrs t1, mnstatus, zero
-# CHECK-ENC: encoding: [0x73,0x23,0x40,0x74]
-# CHECK-INST-ALIAS: csrr t1, mnstatus
-# uimm12
-# CHECK-INST: csrrs t2, mnstatus, zero
-# CHECK-ENC: encoding: [0xf3,0x23,0x40,0x74]
-# CHECK-INST-ALIAS: csrr t2, mnstatus
-# name
-csrrs t1, mnstatus, zero
-# uimm12
-csrrs t2, 0x744, zero
-
-
 ######################################
 # Machine Counter Setup
 ######################################
-- 
GitLab


From ccc15cd6f52922e83164b44e927870059c168ddf Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 25 Oct 2024 15:35:01 -0700
Subject: [PATCH 210/693] [lldb] Avoid repeated hash lookups (NFC) (#113412)

---
 .../InstEmulation/UnwindAssemblyInstEmulation.cpp          | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp b/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp
index 49edd40544e3..1a680d80a9d3 100644
--- a/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp
+++ b/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp
@@ -461,8 +461,7 @@ size_t UnwindAssemblyInstEmulation::WriteMemory(
 
     if (reg_num != LLDB_INVALID_REGNUM &&
         generic_regnum != LLDB_REGNUM_GENERIC_SP) {
-      if (m_pushed_regs.find(reg_num) == m_pushed_regs.end()) {
-        m_pushed_regs[reg_num] = addr;
+      if (m_pushed_regs.try_emplace(reg_num, addr).second) {
         const int32_t offset = addr - m_initial_sp;
         m_curr_row->SetRegisterLocationToAtCFAPlusOffset(reg_num, offset,
                                                          /*can_replace=*/true);
@@ -608,8 +607,8 @@ bool UnwindAssemblyInstEmulation::WriteRegister(
         generic_regnum != LLDB_REGNUM_GENERIC_SP) {
       switch (context.GetInfoType()) {
       case EmulateInstruction::eInfoTypeAddress:
-        if (m_pushed_regs.find(reg_num) != m_pushed_regs.end() &&
-            context.info.address == m_pushed_regs[reg_num]) {
+        if (auto it = m_pushed_regs.find(reg_num);
+            it != m_pushed_regs.end() && context.info.address == it->second) {
           m_curr_row->SetRegisterLocationToSame(reg_num,
                                                 false /*must_replace*/);
           m_curr_row_modified = true;
-- 
GitLab


From 38caf282ab3dfd69cee213b0d4df2f1672e52b38 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman@google.com>
Date: Fri, 25 Oct 2024 15:39:52 -0700
Subject: [PATCH 211/693] [SHT_LLVM_BB_ADDR_MAP][AsmPrinter] Add none and all
 options to PGO Map (#111221)

This patch adds none and all options to the -pgo-analysis-map flag,
which do basically what they say on the tin. The none option is added to
enable forcing the pgo-analysis-map by overriding an earlier invocation
of the flag. The all option is just added for convenience.
---
 llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp    | 31 +++++++++++++------
 .../basic-block-address-map-pgo-features.ll   |  7 ++++-
 2 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index bf4c707cca06..2d444f2f970a 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -141,18 +141,22 @@ using namespace llvm;
 // `object::PGOAnalysisMap::Features::decode(PgoAnalysisMapFeatures.getBits())`
 // succeeds.
 enum class PGOMapFeaturesEnum {
+  None,
   FuncEntryCount,
   BBFreq,
   BrProb,
+  All,
 };
 static cl::bits<PGOMapFeaturesEnum> PgoAnalysisMapFeatures(
     "pgo-analysis-map", cl::Hidden, cl::CommaSeparated,
-    cl::values(clEnumValN(PGOMapFeaturesEnum::FuncEntryCount,
-                          "func-entry-count", "Function Entry Count"),
-               clEnumValN(PGOMapFeaturesEnum::BBFreq, "bb-freq",
-                          "Basic Block Frequency"),
-               clEnumValN(PGOMapFeaturesEnum::BrProb, "br-prob",
-                          "Branch Probability")),
+    cl::values(
+        clEnumValN(PGOMapFeaturesEnum::None, "none", "Disable all options"),
+        clEnumValN(PGOMapFeaturesEnum::FuncEntryCount, "func-entry-count",
+                   "Function Entry Count"),
+        clEnumValN(PGOMapFeaturesEnum::BBFreq, "bb-freq",
+                   "Basic Block Frequency"),
+        clEnumValN(PGOMapFeaturesEnum::BrProb, "br-prob", "Branch Probability"),
+        clEnumValN(PGOMapFeaturesEnum::All, "all", "Enable all options")),
     cl::desc(
         "Enable extended information within the SHT_LLVM_BB_ADDR_MAP that is "
         "extracted from PGO related analysis."));
@@ -1367,9 +1371,18 @@ static uint32_t getBBAddrMapMetadata(const MachineBasicBlock &MBB) {
 
 static llvm::object::BBAddrMap::Features
 getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges) {
-  return {PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::FuncEntryCount),
-          PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::BBFreq),
-          PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::BrProb),
+  bool NoFeatures = PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::None);
+  bool AllFeatures = PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::All);
+  bool FuncEntryCountEnabled =
+      AllFeatures || (!NoFeatures && PgoAnalysisMapFeatures.isSet(
+                                         PGOMapFeaturesEnum::FuncEntryCount));
+  bool BBFreqEnabled =
+      AllFeatures ||
+      (!NoFeatures && PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::BBFreq));
+  bool BrProbEnabled =
+      AllFeatures ||
+      (!NoFeatures && PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::BrProb));
+  return {FuncEntryCountEnabled, BBFreqEnabled, BrProbEnabled,
           MF.hasBBSections() && NumMBBSectionRanges > 1};
 }
 
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll b/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
index 73fe4f6ffedb..1c3db738a947 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
@@ -1,8 +1,10 @@
 ; Check the basic block sections labels option
-; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,BASIC
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,BASIC,PGO-NONE
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -pgo-analysis-map=none | FileCheck %s --check-prefixes=CHECK,BASIC,PGO-NONE
 
 ;; Also verify this holds for all PGO features enabled
 ; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -pgo-analysis-map=func-entry-count,bb-freq,br-prob | FileCheck %s --check-prefixes=CHECK,PGO-ALL,PGO-FEC,PGO-BBF,PGO-BRP
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -pgo-analysis-map=all | FileCheck %s --check-prefixes=CHECK,PGO-ALL,PGO-FEC,PGO-BBF,PGO-BRP
 
 ;; Also verify that pgo extension only includes the enabled feature
 ; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -pgo-analysis-map=func-entry-count | FileCheck %s --check-prefixes=CHECK,PGO-FEC,FEC-ONLY
@@ -93,6 +95,9 @@ declare i32 @__gxx_personality_v0(...)
 ; CHECK-NEXT:	.byte	4
 
 ;; PGO Analysis Map
+; PGO-NONE-NOT: .byte	100		# function entry count
+; PGO-NONE-NOT: .ascii	"\271\235\376\332\245\200\356\017"	# basic block frequency
+; PGO-NONE-NOT: .byte	2		# basic block successor count
 ; PGO-FEC-NEXT:	.byte	100		# function entry count
 ; PGO-BBF-NEXT:	.ascii	"\271\235\376\332\245\200\356\017"	# basic block frequency
 ; PGO-BRP-NEXT:	.byte	2		# basic block successor count
-- 
GitLab


From 242ccd2eb1069c817d44545010dfe185a4c3d0b1 Mon Sep 17 00:00:00 2001
From: Gang Chen <gangc@amd.com>
Date: Fri, 25 Oct 2024 15:52:31 -0700
Subject: [PATCH 212/693] [clang] update the number in no-external-type-id.cppm
 (#113738)

This should fix https://linaro.atlassian.net/browse/LLVM-1411
---
 clang/test/Modules/no-external-type-id.cppm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/test/Modules/no-external-type-id.cppm b/clang/test/Modules/no-external-type-id.cppm
index 6385f3a8aa00..d067e574e72e 100644
--- a/clang/test/Modules/no-external-type-id.cppm
+++ b/clang/test/Modules/no-external-type-id.cppm
@@ -23,7 +23,7 @@ export module b;
 import a;
 export int b();
 
-// CHECK: <DECL_FUNCTION {{.*}} op8=4112
+// CHECK: <DECL_FUNCTION {{.*}} op8=4120
 // CHECK: <TYPE_FUNCTION_PROTO
 
 //--- a.v1.cppm
-- 
GitLab


From 3acf856b501d529821c884fd5406aef7252c208c Mon Sep 17 00:00:00 2001
From: Renaud Kauffmann <rkauffmann@nvidia.com>
Date: Fri, 25 Oct 2024 16:08:45 -0700
Subject: [PATCH 213/693] Adding CUFCommon.{h,cpp} for CUF utilities (#113740)

---
 .../flang/Optimizer/Transforms/CUFCommon.h    | 25 +++++++++++++++
 flang/lib/Optimizer/Transforms/CMakeLists.txt |  1 +
 .../Transforms/CUFAddConstructor.cpp          |  7 ++---
 flang/lib/Optimizer/Transforms/CUFCommon.cpp  | 31 +++++++++++++++++++
 4 files changed, 60 insertions(+), 4 deletions(-)
 create mode 100644 flang/include/flang/Optimizer/Transforms/CUFCommon.h
 create mode 100644 flang/lib/Optimizer/Transforms/CUFCommon.cpp

diff --git a/flang/include/flang/Optimizer/Transforms/CUFCommon.h b/flang/include/flang/Optimizer/Transforms/CUFCommon.h
new file mode 100644
index 000000000000..b88133489df5
--- /dev/null
+++ b/flang/include/flang/Optimizer/Transforms/CUFCommon.h
@@ -0,0 +1,25 @@
+//===-- CUFCommon.h -------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_OPTIMIZER_TRANSFORMS_CUFCOMMON_H_
+#define FORTRAN_OPTIMIZER_TRANSFORMS_CUFCOMMON_H_
+
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/IR/BuiltinOps.h"
+
+static constexpr llvm::StringRef cudaDeviceModuleName = "cuda_device_mod";
+
+namespace cuf {
+
+/// Retrieve or create the CUDA Fortran GPU module in the given \p mod.
+mlir::gpu::GPUModuleOp getOrCreateGPUModule(mlir::ModuleOp mod,
+                                            mlir::SymbolTable &symTab);
+
+} // namespace cuf
+
+#endif // FORTRAN_OPTIMIZER_TRANSFORMS_CUFCOMMON_H_
diff --git a/flang/lib/Optimizer/Transforms/CMakeLists.txt b/flang/lib/Optimizer/Transforms/CMakeLists.txt
index d20d3bc4108c..9eafa4ec234b 100644
--- a/flang/lib/Optimizer/Transforms/CMakeLists.txt
+++ b/flang/lib/Optimizer/Transforms/CMakeLists.txt
@@ -9,6 +9,7 @@ add_flang_library(FIRTransforms
   CompilerGeneratedNames.cpp
   ConstantArgumentGlobalisation.cpp
   ControlFlowConverter.cpp
+  CUFCommon.cpp
   CUFAddConstructor.cpp
   CUFDeviceGlobal.cpp
   CUFOpConversion.cpp
diff --git a/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp b/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp
index f260437e7104..4da06be8ef7d 100644
--- a/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp
@@ -11,6 +11,7 @@
 #include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROpsSupport.h"
+#include "flang/Optimizer/Transforms/CUFCommon.h"
 #include "flang/Runtime/entry-names.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
@@ -24,8 +25,6 @@ namespace fir {
 
 namespace {
 
-static constexpr llvm::StringRef cudaModName{"cuda_device_mod"};
-
 static constexpr llvm::StringRef cudaFortranCtorName{
     "__cudaFortranConstructor"};
 
@@ -60,7 +59,7 @@ struct CUFAddConstructor
     builder.create<mlir::LLVM::CallOp>(loc, funcTy, cufRegisterAllocatorRef);
 
     // Register kernels
-    auto gpuMod = symTab.lookup<mlir::gpu::GPUModuleOp>(cudaModName);
+    auto gpuMod = symTab.lookup<mlir::gpu::GPUModuleOp>(cudaDeviceModuleName);
     if (gpuMod) {
       auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(ctx);
       auto registeredMod = builder.create<cuf::RegisterModuleOp>(
@@ -68,7 +67,7 @@ struct CUFAddConstructor
       for (auto func : gpuMod.getOps<mlir::gpu::GPUFuncOp>()) {
         if (func.isKernel()) {
           auto kernelName = mlir::SymbolRefAttr::get(
-              builder.getStringAttr(cudaModName),
+              builder.getStringAttr(cudaDeviceModuleName),
               {mlir::SymbolRefAttr::get(builder.getContext(), func.getName())});
           builder.create<cuf::RegisterKernelOp>(loc, kernelName, registeredMod);
         }
diff --git a/flang/lib/Optimizer/Transforms/CUFCommon.cpp b/flang/lib/Optimizer/Transforms/CUFCommon.cpp
new file mode 100644
index 000000000000..5eca86529f9e
--- /dev/null
+++ b/flang/lib/Optimizer/Transforms/CUFCommon.cpp
@@ -0,0 +1,31 @@
+//===-- CUFCommon.cpp - Shared functions between passes ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Transforms/CUFCommon.h"
+#include "mlir/Dialect/LLVMIR/NVVMDialect.h"
+
+/// Retrieve or create the CUDA Fortran GPU module in the give in \p mod.
+mlir::gpu::GPUModuleOp cuf::getOrCreateGPUModule(mlir::ModuleOp mod,
+                                                 mlir::SymbolTable &symTab) {
+  if (auto gpuMod = symTab.lookup<mlir::gpu::GPUModuleOp>(cudaDeviceModuleName))
+    return gpuMod;
+
+  auto *ctx = mod.getContext();
+  mod->setAttr(mlir::gpu::GPUDialect::getContainerModuleAttrName(),
+               mlir::UnitAttr::get(ctx));
+
+  mlir::OpBuilder builder(ctx);
+  auto gpuMod = builder.create<mlir::gpu::GPUModuleOp>(mod.getLoc(),
+                                                       cudaDeviceModuleName);
+  llvm::SmallVector<mlir::Attribute> targets;
+  targets.push_back(mlir::NVVM::NVVMTargetAttr::get(ctx));
+  gpuMod.setTargetsAttr(builder.getArrayAttr(targets));
+  mlir::Block::iterator insertPt(mod.getBodyRegion().front().end());
+  symTab.insert(gpuMod, insertPt);
+  return gpuMod;
+}
-- 
GitLab


From 75c1c2614aea23647871dcfa161cf0fad3b19312 Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs@apple.com>
Date: Wed, 25 Sep 2024 08:20:05 -0700
Subject: [PATCH 214/693] [llvm][TLI] Sort a switch's cases. NFC

---
 .../include/llvm/Analysis/TargetLibraryInfo.h | 38 +++++++++----------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
index 9e543b844ad7..aeb8de3973f7 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
@@ -408,35 +408,35 @@ public:
     switch (F) {
     default: break;
       // clang-format off
-    case LibFunc_copysign:     case LibFunc_copysignf:  case LibFunc_copysignl:
-    case LibFunc_fabs:         case LibFunc_fabsf:      case LibFunc_fabsl:
-    case LibFunc_sin:          case LibFunc_sinf:       case LibFunc_sinl:
-    case LibFunc_cos:          case LibFunc_cosf:       case LibFunc_cosl:
-    case LibFunc_tan:          case LibFunc_tanf:       case LibFunc_tanl:
-    case LibFunc_asin:         case LibFunc_asinf:      case LibFunc_asinl:
     case LibFunc_acos:         case LibFunc_acosf:      case LibFunc_acosl:
+    case LibFunc_asin:         case LibFunc_asinf:      case LibFunc_asinl:
     case LibFunc_atan:         case LibFunc_atanf:      case LibFunc_atanl:
-    case LibFunc_sinh:         case LibFunc_sinhf:      case LibFunc_sinhl:
+    case LibFunc_ceil:         case LibFunc_ceilf:      case LibFunc_ceill:
+    case LibFunc_copysign:     case LibFunc_copysignf:  case LibFunc_copysignl:
+    case LibFunc_cos:          case LibFunc_cosf:       case LibFunc_cosl:
     case LibFunc_cosh:         case LibFunc_coshf:      case LibFunc_coshl:
-    case LibFunc_tanh:         case LibFunc_tanhf:      case LibFunc_tanhl:
-    case LibFunc_sqrt:         case LibFunc_sqrtf:      case LibFunc_sqrtl:
-    case LibFunc_sqrt_finite:  case LibFunc_sqrtf_finite:
-                                                   case LibFunc_sqrtl_finite:
+    case LibFunc_exp2:         case LibFunc_exp2f:      case LibFunc_exp2l:
+    case LibFunc_fabs:         case LibFunc_fabsf:      case LibFunc_fabsl:
+    case LibFunc_floor:        case LibFunc_floorf:     case LibFunc_floorl:
     case LibFunc_fmax:         case LibFunc_fmaxf:      case LibFunc_fmaxl:
     case LibFunc_fmin:         case LibFunc_fminf:      case LibFunc_fminl:
-    case LibFunc_floor:        case LibFunc_floorf:     case LibFunc_floorl:
+    case LibFunc_ldexp:        case LibFunc_ldexpf:     case LibFunc_ldexpl:
+    case LibFunc_log2:         case LibFunc_log2f:      case LibFunc_log2l:
+    case LibFunc_memcmp:       case LibFunc_bcmp:       case LibFunc_strcmp:
+    case LibFunc_memcpy:       case LibFunc_memset:     case LibFunc_memmove:
     case LibFunc_nearbyint:    case LibFunc_nearbyintf: case LibFunc_nearbyintl:
-    case LibFunc_ceil:         case LibFunc_ceilf:      case LibFunc_ceill:
     case LibFunc_rint:         case LibFunc_rintf:      case LibFunc_rintl:
     case LibFunc_round:        case LibFunc_roundf:     case LibFunc_roundl:
-    case LibFunc_trunc:        case LibFunc_truncf:     case LibFunc_truncl:
-    case LibFunc_log2:         case LibFunc_log2f:      case LibFunc_log2l:
-    case LibFunc_exp2:         case LibFunc_exp2f:      case LibFunc_exp2l:
-    case LibFunc_ldexp:        case LibFunc_ldexpf:     case LibFunc_ldexpl:
-    case LibFunc_memcpy:       case LibFunc_memset:     case LibFunc_memmove:
-    case LibFunc_memcmp:       case LibFunc_bcmp:       case LibFunc_strcmp:
+    case LibFunc_sin:          case LibFunc_sinf:       case LibFunc_sinl:
+    case LibFunc_sinh:         case LibFunc_sinhf:      case LibFunc_sinhl:
+    case LibFunc_sqrt:         case LibFunc_sqrtf:      case LibFunc_sqrtl:
+    case LibFunc_sqrt_finite:  case LibFunc_sqrtf_finite:
+                                                   case LibFunc_sqrtl_finite:
     case LibFunc_strcpy:       case LibFunc_stpcpy:     case LibFunc_strlen:
     case LibFunc_strnlen:      case LibFunc_memchr:     case LibFunc_mempcpy:
+    case LibFunc_tan:          case LibFunc_tanf:       case LibFunc_tanl:
+    case LibFunc_tanh:         case LibFunc_tanhf:      case LibFunc_tanhl:
+    case LibFunc_trunc:        case LibFunc_truncf:     case LibFunc_truncl:
       // clang-format on
       return true;
     }
-- 
GitLab


From 054c23d78f2e1aba1dd353aec19e00993ca452fa Mon Sep 17 00:00:00 2001
From: Matthias Braun <matze@braunis.de>
Date: Fri, 25 Oct 2024 16:22:24 -0700
Subject: [PATCH 215/693] X86: Improve cost model of fp16 conversion (#113195)

Improve cost-modeling for x86 __fp16 conversions so the SLPVectorizer
transforms the patterns:

- Override `X86TTIImpl::getStoreMinimumVF` to report a minimum VF of 4 (SSE
  register can hold 4xfloat converted/stored to 4xf16) this is necessary as
  fp16 stores are neither modeled as trunc-stores nor can we mark direct Xxfp16
  stores as legal as we generally expand fp16 operations).
- Add missing cost entries to `X86TTIImpl::getCastInstrCost`
  conversion from/to fp16. Note that conversion from f64 to f16 is not
  supported by an X86 instruction.
---
 .../lib/Target/X86/X86TargetTransformInfo.cpp |  41 ++
 llvm/lib/Target/X86/X86TargetTransformInfo.h  |   3 +
 .../SLPVectorizer/X86/conversion-fp16.ll      | 606 ++++++++++++++++++
 3 files changed, 650 insertions(+)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/conversion-fp16.ll

diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 413ef0136d5c..bae223243b3d 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -2296,7 +2296,10 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
     { ISD::FP_EXTEND, MVT::v8f64,   MVT::v8f32,   { 1, 1, 1, 1 } },
     { ISD::FP_EXTEND, MVT::v8f64,   MVT::v16f32,  { 3, 1, 1, 1 } },
     { ISD::FP_EXTEND, MVT::v16f64,  MVT::v16f32,  { 4, 1, 1, 1 } }, // 2*vcvtps2pd+vextractf64x4
+    { ISD::FP_EXTEND, MVT::v16f32,  MVT::v16f16,  { 1, 1, 1, 1 } }, // vcvtph2ps
+    { ISD::FP_EXTEND, MVT::v8f64,   MVT::v8f16,   { 2, 1, 1, 1 } }, // vcvtph2ps+vcvtps2pd
     { ISD::FP_ROUND,  MVT::v8f32,   MVT::v8f64,   { 1, 1, 1, 1 } },
+    { ISD::FP_ROUND,  MVT::v16f16,  MVT::v16f32,  { 1, 1, 1, 1 } }, // vcvtps2ph
 
     { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i8,    { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
     { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i8,    { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
@@ -2973,6 +2976,17 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
     { ISD::TRUNCATE,    MVT::v4i32,  MVT::v2i64,  { 1, 1, 1, 1 } }, // PSHUFD
   };
 
+  static const TypeConversionCostKindTblEntry F16ConversionTbl[] = {
+    { ISD::FP_ROUND,  MVT::f16,     MVT::f32,     { 1, 1, 1, 1 } },
+    { ISD::FP_ROUND,  MVT::v8f16,   MVT::v8f32,   { 1, 1, 1, 1 } },
+    { ISD::FP_ROUND,  MVT::v4f16,   MVT::v4f32,   { 1, 1, 1, 1 } },
+    { ISD::FP_EXTEND, MVT::f32,     MVT::f16,     { 1, 1, 1, 1 } },
+    { ISD::FP_EXTEND, MVT::f64,     MVT::f16,     { 2, 1, 1, 1 } }, // vcvtph2ps+vcvtps2pd
+    { ISD::FP_EXTEND, MVT::v8f32,   MVT::v8f16,   { 1, 1, 1, 1 } },
+    { ISD::FP_EXTEND, MVT::v4f32,   MVT::v4f16,   { 1, 1, 1, 1 } },
+    { ISD::FP_EXTEND, MVT::v4f64,   MVT::v4f16,   { 2, 1, 1, 1 } }, // vcvtph2ps+vcvtps2pd
+  };
+
   // Attempt to map directly to (simple) MVT types to let us match custom entries.
   EVT SrcTy = TLI->getValueType(DL, Src);
   EVT DstTy = TLI->getValueType(DL, Dst);
@@ -3034,6 +3048,13 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
           return *KindCost;
     }
 
+    if (ST->hasF16C()) {
+      if (const auto *Entry = ConvertCostTableLookup(F16ConversionTbl, ISD,
+                                                     SimpleDstTy, SimpleSrcTy))
+        if (auto KindCost = Entry->Cost[CostKind])
+          return *KindCost;
+    }
+
     if (ST->hasSSE41()) {
       if (const auto *Entry = ConvertCostTableLookup(SSE41ConversionTbl, ISD,
                                                      SimpleDstTy, SimpleSrcTy))
@@ -3107,6 +3128,13 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
       if (auto KindCost = Entry->Cost[CostKind])
         return std::max(LTSrc.first, LTDest.first) * *KindCost;
 
+  if (ST->hasF16C()) {
+    if (const auto *Entry = ConvertCostTableLookup(F16ConversionTbl, ISD,
+                                                   LTDest.second, LTSrc.second))
+      if (auto KindCost = Entry->Cost[CostKind])
+        return std::max(LTSrc.first, LTDest.first) * *KindCost;
+  }
+
   if (ST->hasSSE41())
     if (const auto *Entry = ConvertCostTableLookup(SSE41ConversionTbl, ISD,
                                                    LTDest.second, LTSrc.second))
@@ -3146,6 +3174,11 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
                             TTI::CastContextHint::None, CostKind);
   }
 
+  if (ISD == ISD::FP_ROUND && LTDest.second.getScalarType() == MVT::f16) {
+    // Conversion requires a libcall.
+    return InstructionCost::getInvalid();
+  }
+
   // TODO: Allow non-throughput costs that aren't binary.
   auto AdjustCost = [&CostKind](InstructionCost Cost,
                                 InstructionCost N = 1) -> InstructionCost {
@@ -6923,6 +6956,14 @@ bool X86TTIImpl::isVectorShiftByScalarCheap(Type *Ty) const {
   return true;
 }
 
+unsigned X86TTIImpl::getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,
+                                       Type *ScalarValTy) const {
+  if (ST->hasF16C() && ScalarMemTy->isHalfTy()) {
+    return 4;
+  }
+  return BaseT::getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy);
+}
+
 bool X86TTIImpl::isProfitableToSinkOperands(Instruction *I,
                                             SmallVectorImpl<Use *> &Ops) const {
   using namespace llvm::PatternMatch;
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 0100f328ab4b..36d00cee0d18 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -302,6 +302,9 @@ public:
 
   bool isVectorShiftByScalarCheap(Type *Ty) const;
 
+  unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,
+                             Type *ScalarValTy) const;
+
 private:
   bool supportsGather() const;
   InstructionCost getGSVectorCost(unsigned Opcode, TTI::TargetCostKind CostKind,
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/conversion-fp16.ll b/llvm/test/Transforms/SLPVectorizer/X86/conversion-fp16.ll
new file mode 100644
index 000000000000..bcea147d724f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/conversion-fp16.ll
@@ -0,0 +1,606 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=x86_64-- -passes=slp-vectorizer -S -mattr=+avx2 | FileCheck %s --check-prefix=CHECK
+; RUN: opt < %s -mtriple=x86_64-- -passes=slp-vectorizer -S -mattr=+avx2 -mattr=+f16c | FileCheck %s --check-prefix=CHECK-F16C
+; RUN: opt < %s -mtriple=x86_64-- -passes=slp-vectorizer -S -mattr=+avx512f | FileCheck %s --check-prefix=CHECK-AVX512
+
+define void @fpext_v4xf16_v4xf32(ptr %s0, ptr %d0) {
+; CHECK-LABEL: define void @fpext_v4xf16_v4xf32(
+; CHECK-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[S1:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 1
+; CHECK-NEXT:    [[S2:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 2
+; CHECK-NEXT:    [[S3:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 3
+; CHECK-NEXT:    [[L0:%.*]] = load half, ptr [[S0]], align 2
+; CHECK-NEXT:    [[L1:%.*]] = load half, ptr [[S1]], align 2
+; CHECK-NEXT:    [[L2:%.*]] = load half, ptr [[S2]], align 2
+; CHECK-NEXT:    [[L3:%.*]] = load half, ptr [[S3]], align 2
+; CHECK-NEXT:    [[E0:%.*]] = fpext half [[L0]] to float
+; CHECK-NEXT:    [[E1:%.*]] = fpext half [[L1]] to float
+; CHECK-NEXT:    [[E2:%.*]] = fpext half [[L2]] to float
+; CHECK-NEXT:    [[E3:%.*]] = fpext half [[L3]] to float
+; CHECK-NEXT:    [[D1:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 1
+; CHECK-NEXT:    [[D2:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 2
+; CHECK-NEXT:    [[D3:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 3
+; CHECK-NEXT:    store float [[E0]], ptr [[D0]], align 8
+; CHECK-NEXT:    store float [[E1]], ptr [[D1]], align 8
+; CHECK-NEXT:    store float [[E2]], ptr [[D2]], align 8
+; CHECK-NEXT:    store float [[E3]], ptr [[D3]], align 8
+; CHECK-NEXT:    ret void
+;
+; CHECK-F16C-LABEL: define void @fpext_v4xf16_v4xf32(
+; CHECK-F16C-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-F16C-NEXT:    [[TMP1:%.*]] = load <4 x half>, ptr [[S0]], align 2
+; CHECK-F16C-NEXT:    [[TMP2:%.*]] = fpext <4 x half> [[TMP1]] to <4 x float>
+; CHECK-F16C-NEXT:    store <4 x float> [[TMP2]], ptr [[D0]], align 8
+; CHECK-F16C-NEXT:    ret void
+;
+; CHECK-AVX512-LABEL: define void @fpext_v4xf16_v4xf32(
+; CHECK-AVX512-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-AVX512-NEXT:    [[TMP1:%.*]] = load <4 x half>, ptr [[S0]], align 2
+; CHECK-AVX512-NEXT:    [[TMP2:%.*]] = fpext <4 x half> [[TMP1]] to <4 x float>
+; CHECK-AVX512-NEXT:    store <4 x float> [[TMP2]], ptr [[D0]], align 8
+; CHECK-AVX512-NEXT:    ret void
+;
+  %s1 = getelementptr inbounds half, ptr %s0, i64 1
+  %s2 = getelementptr inbounds half, ptr %s0, i64 2
+  %s3 = getelementptr inbounds half, ptr %s0, i64 3
+  %l0 = load half, ptr %s0, align 2
+  %l1 = load half, ptr %s1, align 2
+  %l2 = load half, ptr %s2, align 2
+  %l3 = load half, ptr %s3, align 2
+
+  %e0 = fpext half %l0 to float
+  %e1 = fpext half %l1 to float
+  %e2 = fpext half %l2 to float
+  %e3 = fpext half %l3 to float
+
+  %d1 = getelementptr inbounds float, ptr %d0, i64 1
+  %d2 = getelementptr inbounds float, ptr %d0, i64 2
+  %d3 = getelementptr inbounds float, ptr %d0, i64 3
+  store float %e0, ptr %d0, align 8
+  store float %e1, ptr %d1, align 8
+  store float %e2, ptr %d2, align 8
+  store float %e3, ptr %d3, align 8
+  ret void
+}
+
+define void @fpext_v4xf16_v4xf64(ptr %s0, ptr %d0) {
+; CHECK-LABEL: define void @fpext_v4xf16_v4xf64(
+; CHECK-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[S1:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 1
+; CHECK-NEXT:    [[S2:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 2
+; CHECK-NEXT:    [[S3:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 3
+; CHECK-NEXT:    [[L0:%.*]] = load half, ptr [[S0]], align 2
+; CHECK-NEXT:    [[L1:%.*]] = load half, ptr [[S1]], align 2
+; CHECK-NEXT:    [[L2:%.*]] = load half, ptr [[S2]], align 2
+; CHECK-NEXT:    [[L3:%.*]] = load half, ptr [[S3]], align 2
+; CHECK-NEXT:    [[E0:%.*]] = fpext half [[L0]] to double
+; CHECK-NEXT:    [[E1:%.*]] = fpext half [[L1]] to double
+; CHECK-NEXT:    [[E2:%.*]] = fpext half [[L2]] to double
+; CHECK-NEXT:    [[E3:%.*]] = fpext half [[L3]] to double
+; CHECK-NEXT:    [[D1:%.*]] = getelementptr inbounds double, ptr [[D0]], i64 1
+; CHECK-NEXT:    [[D2:%.*]] = getelementptr inbounds double, ptr [[D0]], i64 2
+; CHECK-NEXT:    [[D3:%.*]] = getelementptr inbounds double, ptr [[D0]], i64 3
+; CHECK-NEXT:    store double [[E0]], ptr [[D0]], align 8
+; CHECK-NEXT:    store double [[E1]], ptr [[D1]], align 8
+; CHECK-NEXT:    store double [[E2]], ptr [[D2]], align 8
+; CHECK-NEXT:    store double [[E3]], ptr [[D3]], align 8
+; CHECK-NEXT:    ret void
+;
+; CHECK-F16C-LABEL: define void @fpext_v4xf16_v4xf64(
+; CHECK-F16C-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
+; CHECK-F16C-NEXT:    [[TMP1:%.*]] = load <4 x half>, ptr [[S0]], align 2
+; CHECK-F16C-NEXT:    [[TMP2:%.*]] = fpext <4 x half> [[TMP1]] to <4 x double>
+; CHECK-F16C-NEXT:    store <4 x double> [[TMP2]], ptr [[D0]], align 8
+; CHECK-F16C-NEXT:    ret void
+;
+; CHECK-AVX512-LABEL: define void @fpext_v4xf16_v4xf64(
+; CHECK-AVX512-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
+; CHECK-AVX512-NEXT:    [[TMP1:%.*]] = load <4 x half>, ptr [[S0]], align 2
+; CHECK-AVX512-NEXT:    [[TMP2:%.*]] = fpext <4 x half> [[TMP1]] to <4 x double>
+; CHECK-AVX512-NEXT:    store <4 x double> [[TMP2]], ptr [[D0]], align 8
+; CHECK-AVX512-NEXT:    ret void
+;
+  %s1 = getelementptr inbounds half, ptr %s0, i64 1
+  %s2 = getelementptr inbounds half, ptr %s0, i64 2
+  %s3 = getelementptr inbounds half, ptr %s0, i64 3
+  %l0 = load half, ptr %s0, align 2
+  %l1 = load half, ptr %s1, align 2
+  %l2 = load half, ptr %s2, align 2
+  %l3 = load half, ptr %s3, align 2
+
+  %e0 = fpext half %l0 to double
+  %e1 = fpext half %l1 to double
+  %e2 = fpext half %l2 to double
+  %e3 = fpext half %l3 to double
+
+  %d1 = getelementptr inbounds double, ptr %d0, i64 1
+  %d2 = getelementptr inbounds double, ptr %d0, i64 2
+  %d3 = getelementptr inbounds double, ptr %d0, i64 3
+  store double %e0, ptr %d0, align 8
+  store double %e1, ptr %d1, align 8
+  store double %e2, ptr %d2, align 8
+  store double %e3, ptr %d3, align 8
+  ret void
+}
+
+define void @fpext_v16xf16_v16xf32(ptr %s0, ptr %d0) {
+; CHECK-LABEL: define void @fpext_v16xf16_v16xf32(
+; CHECK-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[S1:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 1
+; CHECK-NEXT:    [[S2:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 2
+; CHECK-NEXT:    [[S3:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 3
+; CHECK-NEXT:    [[S4:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 4
+; CHECK-NEXT:    [[S5:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 5
+; CHECK-NEXT:    [[S6:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 6
+; CHECK-NEXT:    [[S7:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 7
+; CHECK-NEXT:    [[S8:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 8
+; CHECK-NEXT:    [[S9:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 9
+; CHECK-NEXT:    [[S10:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 10
+; CHECK-NEXT:    [[S11:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 11
+; CHECK-NEXT:    [[S12:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 12
+; CHECK-NEXT:    [[S13:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 13
+; CHECK-NEXT:    [[S14:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 14
+; CHECK-NEXT:    [[S15:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 15
+; CHECK-NEXT:    [[L0:%.*]] = load half, ptr [[S0]], align 2
+; CHECK-NEXT:    [[L1:%.*]] = load half, ptr [[S1]], align 2
+; CHECK-NEXT:    [[L2:%.*]] = load half, ptr [[S2]], align 2
+; CHECK-NEXT:    [[L3:%.*]] = load half, ptr [[S3]], align 2
+; CHECK-NEXT:    [[L4:%.*]] = load half, ptr [[S4]], align 2
+; CHECK-NEXT:    [[L5:%.*]] = load half, ptr [[S5]], align 2
+; CHECK-NEXT:    [[L6:%.*]] = load half, ptr [[S6]], align 2
+; CHECK-NEXT:    [[L7:%.*]] = load half, ptr [[S7]], align 2
+; CHECK-NEXT:    [[L8:%.*]] = load half, ptr [[S8]], align 2
+; CHECK-NEXT:    [[L9:%.*]] = load half, ptr [[S9]], align 2
+; CHECK-NEXT:    [[L10:%.*]] = load half, ptr [[S10]], align 2
+; CHECK-NEXT:    [[L11:%.*]] = load half, ptr [[S11]], align 2
+; CHECK-NEXT:    [[L12:%.*]] = load half, ptr [[S12]], align 2
+; CHECK-NEXT:    [[L13:%.*]] = load half, ptr [[S13]], align 2
+; CHECK-NEXT:    [[L14:%.*]] = load half, ptr [[S14]], align 2
+; CHECK-NEXT:    [[L15:%.*]] = load half, ptr [[S15]], align 2
+; CHECK-NEXT:    [[E0:%.*]] = fpext half [[L0]] to float
+; CHECK-NEXT:    [[E1:%.*]] = fpext half [[L1]] to float
+; CHECK-NEXT:    [[E2:%.*]] = fpext half [[L2]] to float
+; CHECK-NEXT:    [[E3:%.*]] = fpext half [[L3]] to float
+; CHECK-NEXT:    [[E4:%.*]] = fpext half [[L4]] to float
+; CHECK-NEXT:    [[E5:%.*]] = fpext half [[L5]] to float
+; CHECK-NEXT:    [[E6:%.*]] = fpext half [[L6]] to float
+; CHECK-NEXT:    [[E7:%.*]] = fpext half [[L7]] to float
+; CHECK-NEXT:    [[E8:%.*]] = fpext half [[L8]] to float
+; CHECK-NEXT:    [[E9:%.*]] = fpext half [[L9]] to float
+; CHECK-NEXT:    [[E10:%.*]] = fpext half [[L10]] to float
+; CHECK-NEXT:    [[E11:%.*]] = fpext half [[L11]] to float
+; CHECK-NEXT:    [[E12:%.*]] = fpext half [[L12]] to float
+; CHECK-NEXT:    [[E13:%.*]] = fpext half [[L13]] to float
+; CHECK-NEXT:    [[E14:%.*]] = fpext half [[L14]] to float
+; CHECK-NEXT:    [[E15:%.*]] = fpext half [[L15]] to float
+; CHECK-NEXT:    [[D1:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 1
+; CHECK-NEXT:    [[D2:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 2
+; CHECK-NEXT:    [[D15:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 3
+; CHECK-NEXT:    [[D4:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 4
+; CHECK-NEXT:    [[D5:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 5
+; CHECK-NEXT:    [[D6:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 6
+; CHECK-NEXT:    [[D7:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 7
+; CHECK-NEXT:    [[D8:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 8
+; CHECK-NEXT:    [[D9:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 9
+; CHECK-NEXT:    [[D10:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 10
+; CHECK-NEXT:    [[D11:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 11
+; CHECK-NEXT:    [[D12:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 12
+; CHECK-NEXT:    [[D13:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 13
+; CHECK-NEXT:    [[D14:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 14
+; CHECK-NEXT:    [[D16:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 15
+; CHECK-NEXT:    store float [[E0]], ptr [[D0]], align 8
+; CHECK-NEXT:    store float [[E1]], ptr [[D1]], align 8
+; CHECK-NEXT:    store float [[E2]], ptr [[D2]], align 8
+; CHECK-NEXT:    store float [[E3]], ptr [[D15]], align 8
+; CHECK-NEXT:    store float [[E4]], ptr [[D4]], align 8
+; CHECK-NEXT:    store float [[E5]], ptr [[D5]], align 8
+; CHECK-NEXT:    store float [[E6]], ptr [[D6]], align 8
+; CHECK-NEXT:    store float [[E7]], ptr [[D7]], align 8
+; CHECK-NEXT:    store float [[E8]], ptr [[D8]], align 8
+; CHECK-NEXT:    store float [[E9]], ptr [[D9]], align 8
+; CHECK-NEXT:    store float [[E10]], ptr [[D10]], align 8
+; CHECK-NEXT:    store float [[E11]], ptr [[D11]], align 8
+; CHECK-NEXT:    store float [[E12]], ptr [[D12]], align 8
+; CHECK-NEXT:    store float [[E13]], ptr [[D13]], align 8
+; CHECK-NEXT:    store float [[E14]], ptr [[D14]], align 8
+; CHECK-NEXT:    store float [[E15]], ptr [[D16]], align 8
+; CHECK-NEXT:    ret void
+;
+; CHECK-F16C-LABEL: define void @fpext_v16xf16_v16xf32(
+; CHECK-F16C-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
+; CHECK-F16C-NEXT:    [[S8:%.*]] = getelementptr inbounds half, ptr [[S0]], i64 8
+; CHECK-F16C-NEXT:    [[D8:%.*]] = getelementptr inbounds float, ptr [[D0]], i64 8
+; CHECK-F16C-NEXT:    [[TMP1:%.*]] = load <8 x half>, ptr [[S0]], align 2
+; CHECK-F16C-NEXT:    [[TMP2:%.*]] = fpext <8 x half> [[TMP1]] to <8 x float>
+; CHECK-F16C-NEXT:    [[TMP3:%.*]] = load <8 x half>, ptr [[S8]], align 2
+; CHECK-F16C-NEXT:    [[TMP4:%.*]] = fpext <8 x half> [[TMP3]] to <8 x float>
+; CHECK-F16C-NEXT:    store <8 x float> [[TMP2]], ptr [[D0]], align 8
+; CHECK-F16C-NEXT:    store <8 x float> [[TMP4]], ptr [[D8]], align 8
+; CHECK-F16C-NEXT:    ret void
+;
+; CHECK-AVX512-LABEL: define void @fpext_v16xf16_v16xf32(
+; CHECK-AVX512-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
+; CHECK-AVX512-NEXT:    [[TMP1:%.*]] = load <16 x half>, ptr [[S0]], align 2
+; CHECK-AVX512-NEXT:    [[TMP2:%.*]] = fpext <16 x half> [[TMP1]] to <16 x float>
+; CHECK-AVX512-NEXT:    store <16 x float> [[TMP2]], ptr [[D0]], align 8
+; CHECK-AVX512-NEXT:    ret void
+;
+  %s1 = getelementptr inbounds half, ptr %s0, i64 1
+  %s2 = getelementptr inbounds half, ptr %s0, i64 2
+  %s3 = getelementptr inbounds half, ptr %s0, i64 3
+  %s4 = getelementptr inbounds half, ptr %s0, i64 4
+  %s5 = getelementptr inbounds half, ptr %s0, i64 5
+  %s6 = getelementptr inbounds half, ptr %s0, i64 6
+  %s7 = getelementptr inbounds half, ptr %s0, i64 7
+  %s8 = getelementptr inbounds half, ptr %s0, i64 8
+  %s9 = getelementptr inbounds half, ptr %s0, i64 9
+  %s10 = getelementptr inbounds half, ptr %s0, i64 10
+  %s11 = getelementptr inbounds half, ptr %s0, i64 11
+  %s12 = getelementptr inbounds half, ptr %s0, i64 12
+  %s13 = getelementptr inbounds half, ptr %s0, i64 13
+  %s14 = getelementptr inbounds half, ptr %s0, i64 14
+  %s15 = getelementptr inbounds half, ptr %s0, i64 15
+  %l0 = load half, ptr %s0, align 2
+  %l1 = load half, ptr %s1, align 2
+  %l2 = load half, ptr %s2, align 2
+  %l3 = load half, ptr %s3, align 2
+  %l4 = load half, ptr %s4, align 2
+  %l5 = load half, ptr %s5, align 2
+  %l6 = load half, ptr %s6, align 2
+  %l7 = load half, ptr %s7, align 2
+  %l8 = load half, ptr %s8, align 2
+  %l9 = load half, ptr %s9, align 2
+  %l10 = load half, ptr %s10, align 2
+  %l11 = load half, ptr %s11, align 2
+  %l12 = load half, ptr %s12, align 2
+  %l13 = load half, ptr %s13, align 2
+  %l14 = load half, ptr %s14, align 2
+  %l15 = load half, ptr %s15, align 2
+
+  %e0 = fpext half %l0 to float
+  %e1 = fpext half %l1 to float
+  %e2 = fpext half %l2 to float
+  %e3 = fpext half %l3 to float
+  %e4 = fpext half %l4 to float
+  %e5 = fpext half %l5 to float
+  %e6 = fpext half %l6 to float
+  %e7 = fpext half %l7 to float
+  %e8 = fpext half %l8 to float
+  %e9 = fpext half %l9 to float
+  %e10 = fpext half %l10 to float
+  %e11 = fpext half %l11 to float
+  %e12 = fpext half %l12 to float
+  %e13 = fpext half %l13 to float
+  %e14 = fpext half %l14 to float
+  %e15 = fpext half %l15 to float
+
+  %d1 = getelementptr inbounds float, ptr %d0, i64 1
+  %d2 = getelementptr inbounds float, ptr %d0, i64 2
+  %d3 = getelementptr inbounds float, ptr %d0, i64 3
+  %d4 = getelementptr inbounds float, ptr %d0, i64 4
+  %d5 = getelementptr inbounds float, ptr %d0, i64 5
+  %d6 = getelementptr inbounds float, ptr %d0, i64 6
+  %d7 = getelementptr inbounds float, ptr %d0, i64 7
+  %d8 = getelementptr inbounds float, ptr %d0, i64 8
+  %d9 = getelementptr inbounds float, ptr %d0, i64 9
+  %d10 = getelementptr inbounds float, ptr %d0, i64 10
+  %d11 = getelementptr inbounds float, ptr %d0, i64 11
+  %d12 = getelementptr inbounds float, ptr %d0, i64 12
+  %d13 = getelementptr inbounds float, ptr %d0, i64 13
+  %d14 = getelementptr inbounds float, ptr %d0, i64 14
+  %d15 = getelementptr inbounds float, ptr %d0, i64 15
+  store float %e0, ptr %d0, align 8
+  store float %e1, ptr %d1, align 8
+  store float %e2, ptr %d2, align 8
+  store float %e3, ptr %d3, align 8
+  store float %e4, ptr %d4, align 8
+  store float %e5, ptr %d5, align 8
+  store float %e6, ptr %d6, align 8
+  store float %e7, ptr %d7, align 8
+  store float %e8, ptr %d8, align 8
+  store float %e9, ptr %d9, align 8
+  store float %e10, ptr %d10, align 8
+  store float %e11, ptr %d11, align 8
+  store float %e12, ptr %d12, align 8
+  store float %e13, ptr %d13, align 8
+  store float %e14, ptr %d14, align 8
+  store float %e15, ptr %d15, align 8
+  ret void
+}
+
+define void @fpround_v4xf32_v4xf16(ptr %s0, ptr %d0) {
+; CHECK-LABEL: define void @fpround_v4xf32_v4xf16(
+; CHECK-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[S1:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 1
+; CHECK-NEXT:    [[S2:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 2
+; CHECK-NEXT:    [[S3:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 3
+; CHECK-NEXT:    [[L0:%.*]] = load float, ptr [[S0]], align 4
+; CHECK-NEXT:    [[L1:%.*]] = load float, ptr [[S1]], align 4
+; CHECK-NEXT:    [[L2:%.*]] = load float, ptr [[S2]], align 4
+; CHECK-NEXT:    [[L3:%.*]] = load float, ptr [[S3]], align 4
+; CHECK-NEXT:    [[T0:%.*]] = fptrunc float [[L0]] to half
+; CHECK-NEXT:    [[T1:%.*]] = fptrunc float [[L1]] to half
+; CHECK-NEXT:    [[T2:%.*]] = fptrunc float [[L2]] to half
+; CHECK-NEXT:    [[T3:%.*]] = fptrunc float [[L3]] to half
+; CHECK-NEXT:    [[D1:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 1
+; CHECK-NEXT:    [[D2:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 2
+; CHECK-NEXT:    [[D3:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 3
+; CHECK-NEXT:    store half [[T0]], ptr [[D0]], align 2
+; CHECK-NEXT:    store half [[T1]], ptr [[D1]], align 2
+; CHECK-NEXT:    store half [[T2]], ptr [[D2]], align 2
+; CHECK-NEXT:    store half [[T3]], ptr [[D3]], align 2
+; CHECK-NEXT:    ret void
+;
+; CHECK-F16C-LABEL: define void @fpround_v4xf32_v4xf16(
+; CHECK-F16C-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
+; CHECK-F16C-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[S0]], align 4
+; CHECK-F16C-NEXT:    [[TMP2:%.*]] = fptrunc <4 x float> [[TMP1]] to <4 x half>
+; CHECK-F16C-NEXT:    store <4 x half> [[TMP2]], ptr [[D0]], align 2
+; CHECK-F16C-NEXT:    ret void
+;
+; CHECK-AVX512-LABEL: define void @fpround_v4xf32_v4xf16(
+; CHECK-AVX512-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
+; CHECK-AVX512-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[S0]], align 4
+; CHECK-AVX512-NEXT:    [[TMP2:%.*]] = fptrunc <4 x float> [[TMP1]] to <4 x half>
+; CHECK-AVX512-NEXT:    store <4 x half> [[TMP2]], ptr [[D0]], align 2
+; CHECK-AVX512-NEXT:    ret void
+;
+  %s1 = getelementptr inbounds float, ptr %s0, i64 1
+  %s2 = getelementptr inbounds float, ptr %s0, i64 2
+  %s3 = getelementptr inbounds float, ptr %s0, i64 3
+  %l0 = load float, ptr %s0, align 4
+  %l1 = load float, ptr %s1, align 4
+  %l2 = load float, ptr %s2, align 4
+  %l3 = load float, ptr %s3, align 4
+
+  %t0 = fptrunc float %l0 to half
+  %t1 = fptrunc float %l1 to half
+  %t2 = fptrunc float %l2 to half
+  %t3 = fptrunc float %l3 to half
+
+  %d1 = getelementptr inbounds half, ptr %d0, i64 1
+  %d2 = getelementptr inbounds half, ptr %d0, i64 2
+  %d3 = getelementptr inbounds half, ptr %d0, i64 3
+  store half %t0, ptr %d0, align 2
+  store half %t1, ptr %d1, align 2
+  store half %t2, ptr %d2, align 2
+  store half %t3, ptr %d3, align 2
+  ret void
+}
+
+define void @fpround_v16xf32_v16xf16(ptr %s0, ptr %d0) {
+; CHECK-LABEL: define void @fpround_v16xf32_v16xf16(
+; CHECK-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[S1:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 1
+; CHECK-NEXT:    [[S2:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 2
+; CHECK-NEXT:    [[S3:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 3
+; CHECK-NEXT:    [[S4:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 4
+; CHECK-NEXT:    [[S5:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 5
+; CHECK-NEXT:    [[S6:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 6
+; CHECK-NEXT:    [[S7:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 7
+; CHECK-NEXT:    [[S8:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 8
+; CHECK-NEXT:    [[S9:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 9
+; CHECK-NEXT:    [[S10:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 10
+; CHECK-NEXT:    [[S11:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 11
+; CHECK-NEXT:    [[S12:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 12
+; CHECK-NEXT:    [[S13:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 13
+; CHECK-NEXT:    [[S14:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 14
+; CHECK-NEXT:    [[S15:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 15
+; CHECK-NEXT:    [[L0:%.*]] = load float, ptr [[S0]], align 4
+; CHECK-NEXT:    [[L1:%.*]] = load float, ptr [[S1]], align 4
+; CHECK-NEXT:    [[L2:%.*]] = load float, ptr [[S2]], align 4
+; CHECK-NEXT:    [[L3:%.*]] = load float, ptr [[S3]], align 4
+; CHECK-NEXT:    [[L4:%.*]] = load float, ptr [[S4]], align 4
+; CHECK-NEXT:    [[L5:%.*]] = load float, ptr [[S5]], align 4
+; CHECK-NEXT:    [[L6:%.*]] = load float, ptr [[S6]], align 4
+; CHECK-NEXT:    [[L7:%.*]] = load float, ptr [[S7]], align 4
+; CHECK-NEXT:    [[L8:%.*]] = load float, ptr [[S8]], align 4
+; CHECK-NEXT:    [[L9:%.*]] = load float, ptr [[S9]], align 4
+; CHECK-NEXT:    [[L10:%.*]] = load float, ptr [[S10]], align 4
+; CHECK-NEXT:    [[L11:%.*]] = load float, ptr [[S11]], align 4
+; CHECK-NEXT:    [[L12:%.*]] = load float, ptr [[S12]], align 4
+; CHECK-NEXT:    [[L13:%.*]] = load float, ptr [[S13]], align 4
+; CHECK-NEXT:    [[L14:%.*]] = load float, ptr [[S14]], align 4
+; CHECK-NEXT:    [[L15:%.*]] = load float, ptr [[S15]], align 4
+; CHECK-NEXT:    [[T0:%.*]] = fptrunc float [[L0]] to half
+; CHECK-NEXT:    [[T1:%.*]] = fptrunc float [[L1]] to half
+; CHECK-NEXT:    [[T2:%.*]] = fptrunc float [[L2]] to half
+; CHECK-NEXT:    [[T3:%.*]] = fptrunc float [[L3]] to half
+; CHECK-NEXT:    [[T4:%.*]] = fptrunc float [[L4]] to half
+; CHECK-NEXT:    [[T5:%.*]] = fptrunc float [[L5]] to half
+; CHECK-NEXT:    [[T6:%.*]] = fptrunc float [[L6]] to half
+; CHECK-NEXT:    [[T7:%.*]] = fptrunc float [[L7]] to half
+; CHECK-NEXT:    [[T8:%.*]] = fptrunc float [[L8]] to half
+; CHECK-NEXT:    [[T9:%.*]] = fptrunc float [[L9]] to half
+; CHECK-NEXT:    [[T10:%.*]] = fptrunc float [[L10]] to half
+; CHECK-NEXT:    [[T11:%.*]] = fptrunc float [[L11]] to half
+; CHECK-NEXT:    [[T12:%.*]] = fptrunc float [[L12]] to half
+; CHECK-NEXT:    [[T13:%.*]] = fptrunc float [[L13]] to half
+; CHECK-NEXT:    [[T14:%.*]] = fptrunc float [[L14]] to half
+; CHECK-NEXT:    [[T15:%.*]] = fptrunc float [[L15]] to half
+; CHECK-NEXT:    [[D1:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 1
+; CHECK-NEXT:    [[D2:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 2
+; CHECK-NEXT:    [[D3:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 3
+; CHECK-NEXT:    [[D4:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 4
+; CHECK-NEXT:    [[D5:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 5
+; CHECK-NEXT:    [[D6:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 6
+; CHECK-NEXT:    [[D7:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 7
+; CHECK-NEXT:    [[D8:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 8
+; CHECK-NEXT:    [[D9:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 9
+; CHECK-NEXT:    [[D10:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 10
+; CHECK-NEXT:    [[D11:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 11
+; CHECK-NEXT:    [[D12:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 12
+; CHECK-NEXT:    [[D13:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 13
+; CHECK-NEXT:    [[D14:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 14
+; CHECK-NEXT:    [[D15:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 15
+; CHECK-NEXT:    store half [[T0]], ptr [[D0]], align 2
+; CHECK-NEXT:    store half [[T1]], ptr [[D1]], align 2
+; CHECK-NEXT:    store half [[T2]], ptr [[D2]], align 2
+; CHECK-NEXT:    store half [[T3]], ptr [[D3]], align 2
+; CHECK-NEXT:    store half [[T4]], ptr [[D4]], align 2
+; CHECK-NEXT:    store half [[T5]], ptr [[D5]], align 2
+; CHECK-NEXT:    store half [[T6]], ptr [[D6]], align 2
+; CHECK-NEXT:    store half [[T7]], ptr [[D7]], align 2
+; CHECK-NEXT:    store half [[T8]], ptr [[D8]], align 2
+; CHECK-NEXT:    store half [[T9]], ptr [[D9]], align 2
+; CHECK-NEXT:    store half [[T10]], ptr [[D10]], align 2
+; CHECK-NEXT:    store half [[T11]], ptr [[D11]], align 2
+; CHECK-NEXT:    store half [[T12]], ptr [[D12]], align 2
+; CHECK-NEXT:    store half [[T13]], ptr [[D13]], align 2
+; CHECK-NEXT:    store half [[T14]], ptr [[D14]], align 2
+; CHECK-NEXT:    store half [[T15]], ptr [[D15]], align 2
+; CHECK-NEXT:    ret void
+;
+; CHECK-F16C-LABEL: define void @fpround_v16xf32_v16xf16(
+; CHECK-F16C-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
+; CHECK-F16C-NEXT:    [[S8:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 8
+; CHECK-F16C-NEXT:    [[D8:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 8
+; CHECK-F16C-NEXT:    [[TMP1:%.*]] = load <8 x float>, ptr [[S0]], align 4
+; CHECK-F16C-NEXT:    [[TMP2:%.*]] = fptrunc <8 x float> [[TMP1]] to <8 x half>
+; CHECK-F16C-NEXT:    [[TMP3:%.*]] = load <8 x float>, ptr [[S8]], align 4
+; CHECK-F16C-NEXT:    [[TMP4:%.*]] = fptrunc <8 x float> [[TMP3]] to <8 x half>
+; CHECK-F16C-NEXT:    store <8 x half> [[TMP2]], ptr [[D0]], align 2
+; CHECK-F16C-NEXT:    store <8 x half> [[TMP4]], ptr [[D8]], align 2
+; CHECK-F16C-NEXT:    ret void
+;
+; CHECK-AVX512-LABEL: define void @fpround_v16xf32_v16xf16(
+; CHECK-AVX512-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
+; CHECK-AVX512-NEXT:    [[TMP1:%.*]] = load <16 x float>, ptr [[S0]], align 4
+; CHECK-AVX512-NEXT:    [[TMP2:%.*]] = fptrunc <16 x float> [[TMP1]] to <16 x half>
+; CHECK-AVX512-NEXT:    store <16 x half> [[TMP2]], ptr [[D0]], align 2
+; CHECK-AVX512-NEXT:    ret void
+;
+  %s1 = getelementptr inbounds float, ptr %s0, i64 1
+  %s2 = getelementptr inbounds float, ptr %s0, i64 2
+  %s3 = getelementptr inbounds float, ptr %s0, i64 3
+  %s4 = getelementptr inbounds float, ptr %s0, i64 4
+  %s5 = getelementptr inbounds float, ptr %s0, i64 5
+  %s6 = getelementptr inbounds float, ptr %s0, i64 6
+  %s7 = getelementptr inbounds float, ptr %s0, i64 7
+  %s8 = getelementptr inbounds float, ptr %s0, i64 8
+  %s9 = getelementptr inbounds float, ptr %s0, i64 9
+  %s10 = getelementptr inbounds float, ptr %s0, i64 10
+  %s11 = getelementptr inbounds float, ptr %s0, i64 11
+  %s12 = getelementptr inbounds float, ptr %s0, i64 12
+  %s13 = getelementptr inbounds float, ptr %s0, i64 13
+  %s14 = getelementptr inbounds float, ptr %s0, i64 14
+  %s15 = getelementptr inbounds float, ptr %s0, i64 15
+  %l0 = load float, ptr %s0, align 4
+  %l1 = load float, ptr %s1, align 4
+  %l2 = load float, ptr %s2, align 4
+  %l3 = load float, ptr %s3, align 4
+  %l4 = load float, ptr %s4, align 4
+  %l5 = load float, ptr %s5, align 4
+  %l6 = load float, ptr %s6, align 4
+  %l7 = load float, ptr %s7, align 4
+  %l8 = load float, ptr %s8, align 4
+  %l9 = load float, ptr %s9, align 4
+  %l10 = load float, ptr %s10, align 4
+  %l11 = load float, ptr %s11, align 4
+  %l12 = load float, ptr %s12, align 4
+  %l13 = load float, ptr %s13, align 4
+  %l14 = load float, ptr %s14, align 4
+  %l15 = load float, ptr %s15, align 4
+
+  %t0 = fptrunc float %l0 to half
+  %t1 = fptrunc float %l1 to half
+  %t2 = fptrunc float %l2 to half
+  %t3 = fptrunc float %l3 to half
+  %t4 = fptrunc float %l4 to half
+  %t5 = fptrunc float %l5 to half
+  %t6 = fptrunc float %l6 to half
+  %t7 = fptrunc float %l7 to half
+  %t8 = fptrunc float %l8 to half
+  %t9 = fptrunc float %l9 to half
+  %t10 = fptrunc float %l10 to half
+  %t11 = fptrunc float %l11 to half
+  %t12 = fptrunc float %l12 to half
+  %t13 = fptrunc float %l13 to half
+  %t14 = fptrunc float %l14 to half
+  %t15 = fptrunc float %l15 to half
+
+  %d1 = getelementptr inbounds half, ptr %d0, i64 1
+  %d2 = getelementptr inbounds half, ptr %d0, i64 2
+  %d3 = getelementptr inbounds half, ptr %d0, i64 3
+  %d4 = getelementptr inbounds half, ptr %d0, i64 4
+  %d5 = getelementptr inbounds half, ptr %d0, i64 5
+  %d6 = getelementptr inbounds half, ptr %d0, i64 6
+  %d7 = getelementptr inbounds half, ptr %d0, i64 7
+  %d8 = getelementptr inbounds half, ptr %d0, i64 8
+  %d9 = getelementptr inbounds half, ptr %d0, i64 9
+  %d10 = getelementptr inbounds half, ptr %d0, i64 10
+  %d11 = getelementptr inbounds half, ptr %d0, i64 11
+  %d12 = getelementptr inbounds half, ptr %d0, i64 12
+  %d13 = getelementptr inbounds half, ptr %d0, i64 13
+  %d14 = getelementptr inbounds half, ptr %d0, i64 14
+  %d15 = getelementptr inbounds half, ptr %d0, i64 15
+  store half %t0, ptr %d0, align 2
+  store half %t1, ptr %d1, align 2
+  store half %t2, ptr %d2, align 2
+  store half %t3, ptr %d3, align 2
+  store half %t4, ptr %d4, align 2
+  store half %t5, ptr %d5, align 2
+  store half %t6, ptr %d6, align 2
+  store half %t7, ptr %d7, align 2
+  store half %t8, ptr %d8, align 2
+  store half %t9, ptr %d9, align 2
+  store half %t10, ptr %d10, align 2
+  store half %t11, ptr %d11, align 2
+  store half %t12, ptr %d12, align 2
+  store half %t13, ptr %d13, align 2
+  store half %t14, ptr %d14, align 2
+  store half %t15, ptr %d15, align 2
+  ret void
+
+}
+
+; There is no instruction to round f64 to f16; this should not get vectorized!
+define void @fpround_v2xf64_v2xf16(ptr %s0, ptr %d0) {
+; CHECK-LABEL: define void @fpround_v2xf64_v2xf16(
+; CHECK-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[S1:%.*]] = getelementptr inbounds double, ptr [[S0]], i64 1
+; CHECK-NEXT:    [[L0:%.*]] = load double, ptr [[S0]], align 4
+; CHECK-NEXT:    [[L1:%.*]] = load double, ptr [[S1]], align 4
+; CHECK-NEXT:    [[T0:%.*]] = fptrunc double [[L0]] to half
+; CHECK-NEXT:    [[T1:%.*]] = fptrunc double [[L1]] to half
+; CHECK-NEXT:    [[D1:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 1
+; CHECK-NEXT:    store half [[T0]], ptr [[D0]], align 2
+; CHECK-NEXT:    store half [[T1]], ptr [[D1]], align 2
+; CHECK-NEXT:    ret void
+;
+; CHECK-F16C-LABEL: define void @fpround_v2xf64_v2xf16(
+; CHECK-F16C-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
+; CHECK-F16C-NEXT:    [[S1:%.*]] = getelementptr inbounds double, ptr [[S0]], i64 1
+; CHECK-F16C-NEXT:    [[L0:%.*]] = load double, ptr [[S0]], align 4
+; CHECK-F16C-NEXT:    [[L1:%.*]] = load double, ptr [[S1]], align 4
+; CHECK-F16C-NEXT:    [[T0:%.*]] = fptrunc double [[L0]] to half
+; CHECK-F16C-NEXT:    [[T1:%.*]] = fptrunc double [[L1]] to half
+; CHECK-F16C-NEXT:    [[D1:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 1
+; CHECK-F16C-NEXT:    store half [[T0]], ptr [[D0]], align 2
+; CHECK-F16C-NEXT:    store half [[T1]], ptr [[D1]], align 2
+; CHECK-F16C-NEXT:    ret void
+;
+; CHECK-AVX512-LABEL: define void @fpround_v2xf64_v2xf16(
+; CHECK-AVX512-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
+; CHECK-AVX512-NEXT:    [[S1:%.*]] = getelementptr inbounds double, ptr [[S0]], i64 1
+; CHECK-AVX512-NEXT:    [[L0:%.*]] = load double, ptr [[S0]], align 4
+; CHECK-AVX512-NEXT:    [[L1:%.*]] = load double, ptr [[S1]], align 4
+; CHECK-AVX512-NEXT:    [[T0:%.*]] = fptrunc double [[L0]] to half
+; CHECK-AVX512-NEXT:    [[T1:%.*]] = fptrunc double [[L1]] to half
+; CHECK-AVX512-NEXT:    [[D1:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 1
+; CHECK-AVX512-NEXT:    store half [[T0]], ptr [[D0]], align 2
+; CHECK-AVX512-NEXT:    store half [[T1]], ptr [[D1]], align 2
+; CHECK-AVX512-NEXT:    ret void
+;
+  %s1 = getelementptr inbounds double, ptr %s0, i64 1
+  %l0 = load double, ptr %s0, align 4
+  %l1 = load double, ptr %s1, align 4
+
+  %t0 = fptrunc double %l0 to half
+  %t1 = fptrunc double %l1 to half
+
+  %d1 = getelementptr inbounds half, ptr %d0, i64 1
+  store half %t0, ptr %d0, align 2
+  store half %t1, ptr %d1, align 2
+  ret void
+}
-- 
GitLab


From 5ad500ca4a1cba1f39757ba2660d4e0c6e3559d3 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Fri, 25 Oct 2024 16:23:07 -0700
Subject: [PATCH 216/693] [RISCV] Coverage for a few missed vector idioms

---
 .../RISCV/rvv/fixed-vectors-int-buildvec.ll   | 97 +++++++++++++++----
 .../RISCV/rvv/fixed-vectors-mask-buildvec.ll  | 23 +++++
 .../CodeGen/RISCV/rvv/fold-binary-reduce.ll   | 75 ++++++++++++++
 3 files changed, 175 insertions(+), 20 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
index 7bf47d42de3b..ea4072f15712 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
@@ -428,6 +428,33 @@ define void @buildvec_dominant0_v8i16(ptr %x) {
   ret void
 }
 
+define void @buildvec_dominant0_v8i16_with_end_element(ptr %x) {
+; CHECK-LABEL: buildvec_dominant0_v8i16_with_end_element:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 8
+; CHECK-NEXT:    li a1, 3
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    vse16.v v8, (a0)
+; CHECK-NEXT:    ret
+  store <8 x i16> <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 3>, ptr %x
+  ret void
+}
+
+define void @buildvec_dominant0_v8i16_with_tail(ptr %x) {
+; CHECK-LABEL: buildvec_dominant0_v8i16_with_tail:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lui a1, %hi(.LCPI35_0)
+; CHECK-NEXT:    addi a1, a1, %lo(.LCPI35_0)
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vle16.v v8, (a1)
+; CHECK-NEXT:    vse16.v v8, (a0)
+; CHECK-NEXT:    ret
+  store <8 x i16> <i16 8, i16 8, i16 8, i16 8, i16 8, i16 undef, i16 2, i16 3>, ptr %x
+  ret void
+}
+
+
 define void @buildvec_dominant1_v8i16(ptr %x) {
 ; CHECK-LABEL: buildvec_dominant1_v8i16:
 ; CHECK:       # %bb.0:
@@ -494,8 +521,8 @@ define <2 x i8> @buildvec_dominant2_v2i8() {
 define void @buildvec_dominant0_v2i32(ptr %x) {
 ; RV32-LABEL: buildvec_dominant0_v2i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    lui a1, %hi(.LCPI38_0)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI38_0)
+; RV32-NEXT:    lui a1, %hi(.LCPI40_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI40_0)
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a1)
 ; RV32-NEXT:    vse32.v v8, (a0)
@@ -503,8 +530,8 @@ define void @buildvec_dominant0_v2i32(ptr %x) {
 ;
 ; RV64V-LABEL: buildvec_dominant0_v2i32:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    lui a1, %hi(.LCPI38_0)
-; RV64V-NEXT:    ld a1, %lo(.LCPI38_0)(a1)
+; RV64V-NEXT:    lui a1, %hi(.LCPI40_0)
+; RV64V-NEXT:    ld a1, %lo(.LCPI40_0)(a1)
 ; RV64V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV64V-NEXT:    vmv.v.i v8, -1
 ; RV64V-NEXT:    vsetvli zero, zero, e64, m1, tu, ma
@@ -514,8 +541,8 @@ define void @buildvec_dominant0_v2i32(ptr %x) {
 ;
 ; RV64ZVE32-LABEL: buildvec_dominant0_v2i32:
 ; RV64ZVE32:       # %bb.0:
-; RV64ZVE32-NEXT:    lui a1, %hi(.LCPI38_0)
-; RV64ZVE32-NEXT:    ld a1, %lo(.LCPI38_0)(a1)
+; RV64ZVE32-NEXT:    lui a1, %hi(.LCPI40_0)
+; RV64ZVE32-NEXT:    ld a1, %lo(.LCPI40_0)(a1)
 ; RV64ZVE32-NEXT:    li a2, -1
 ; RV64ZVE32-NEXT:    sd a1, 0(a0)
 ; RV64ZVE32-NEXT:    sd a2, 8(a0)
@@ -527,8 +554,8 @@ define void @buildvec_dominant0_v2i32(ptr %x) {
 define void @buildvec_dominant1_optsize_v2i32(ptr %x) optsize {
 ; RV32-LABEL: buildvec_dominant1_optsize_v2i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    lui a1, %hi(.LCPI39_0)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI39_0)
+; RV32-NEXT:    lui a1, %hi(.LCPI41_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI41_0)
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a1)
 ; RV32-NEXT:    vse32.v v8, (a0)
@@ -536,8 +563,8 @@ define void @buildvec_dominant1_optsize_v2i32(ptr %x) optsize {
 ;
 ; RV64V-LABEL: buildvec_dominant1_optsize_v2i32:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    lui a1, %hi(.LCPI39_0)
-; RV64V-NEXT:    addi a1, a1, %lo(.LCPI39_0)
+; RV64V-NEXT:    lui a1, %hi(.LCPI41_0)
+; RV64V-NEXT:    addi a1, a1, %lo(.LCPI41_0)
 ; RV64V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV64V-NEXT:    vle64.v v8, (a1)
 ; RV64V-NEXT:    vse64.v v8, (a0)
@@ -545,8 +572,8 @@ define void @buildvec_dominant1_optsize_v2i32(ptr %x) optsize {
 ;
 ; RV64ZVE32-LABEL: buildvec_dominant1_optsize_v2i32:
 ; RV64ZVE32:       # %bb.0:
-; RV64ZVE32-NEXT:    lui a1, %hi(.LCPI39_0)
-; RV64ZVE32-NEXT:    ld a1, %lo(.LCPI39_0)(a1)
+; RV64ZVE32-NEXT:    lui a1, %hi(.LCPI41_0)
+; RV64ZVE32-NEXT:    ld a1, %lo(.LCPI41_0)(a1)
 ; RV64ZVE32-NEXT:    li a2, -1
 ; RV64ZVE32-NEXT:    sd a1, 0(a0)
 ; RV64ZVE32-NEXT:    sd a2, 8(a0)
@@ -604,8 +631,8 @@ define void @buildvec_seq_v8i8_v2i32(ptr %x) {
 define void @buildvec_seq_v16i8_v2i64(ptr %x) {
 ; RV32-LABEL: buildvec_seq_v16i8_v2i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    lui a1, %hi(.LCPI42_0)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI42_0)
+; RV32-NEXT:    lui a1, %hi(.LCPI44_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI44_0)
 ; RV32-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; RV32-NEXT:    vle8.v v8, (a1)
 ; RV32-NEXT:    vse8.v v8, (a0)
@@ -613,8 +640,8 @@ define void @buildvec_seq_v16i8_v2i64(ptr %x) {
 ;
 ; RV64V-LABEL: buildvec_seq_v16i8_v2i64:
 ; RV64V:       # %bb.0:
-; RV64V-NEXT:    lui a1, %hi(.LCPI42_0)
-; RV64V-NEXT:    ld a1, %lo(.LCPI42_0)(a1)
+; RV64V-NEXT:    lui a1, %hi(.LCPI44_0)
+; RV64V-NEXT:    ld a1, %lo(.LCPI44_0)(a1)
 ; RV64V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV64V-NEXT:    vmv.v.x v8, a1
 ; RV64V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
@@ -623,8 +650,8 @@ define void @buildvec_seq_v16i8_v2i64(ptr %x) {
 ;
 ; RV64ZVE32-LABEL: buildvec_seq_v16i8_v2i64:
 ; RV64ZVE32:       # %bb.0:
-; RV64ZVE32-NEXT:    lui a1, %hi(.LCPI42_0)
-; RV64ZVE32-NEXT:    addi a1, a1, %lo(.LCPI42_0)
+; RV64ZVE32-NEXT:    lui a1, %hi(.LCPI44_0)
+; RV64ZVE32-NEXT:    addi a1, a1, %lo(.LCPI44_0)
 ; RV64ZVE32-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; RV64ZVE32-NEXT:    vle8.v v8, (a1)
 ; RV64ZVE32-NEXT:    vse8.v v8, (a0)
@@ -656,8 +683,8 @@ define void @buildvec_seq2_v16i8_v2i64(ptr %x) {
 ;
 ; RV64ZVE32-LABEL: buildvec_seq2_v16i8_v2i64:
 ; RV64ZVE32:       # %bb.0:
-; RV64ZVE32-NEXT:    lui a1, %hi(.LCPI43_0)
-; RV64ZVE32-NEXT:    addi a1, a1, %lo(.LCPI43_0)
+; RV64ZVE32-NEXT:    lui a1, %hi(.LCPI45_0)
+; RV64ZVE32-NEXT:    addi a1, a1, %lo(.LCPI45_0)
 ; RV64ZVE32-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; RV64ZVE32-NEXT:    vle8.v v8, (a1)
 ; RV64ZVE32-NEXT:    vse8.v v8, (a0)
@@ -3384,3 +3411,33 @@ define <1 x i32> @buildvec_v1i32_pack(i32 %e1) {
   ret <1 x i32> %v1
 }
 
+define <4 x i32> @buildvec_vslide1up(i32 %e1, i32 %e2) {
+; CHECK-LABEL: buildvec_vslide1up:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, tu, ma
+; CHECK-NEXT:    vmv.s.x v8, a1
+; CHECK-NEXT:    ret
+  %v1 = insertelement <4 x i32> poison, i32 %e2, i32 0
+  %v2 = insertelement <4 x i32> %v1, i32 %e1, i32 1
+  %v3 = insertelement <4 x i32> %v2, i32 %e1, i32 2
+  %v4 = insertelement <4 x i32> %v3, i32 %e1, i32 3
+  ret <4 x i32> %v4
+}
+
+define <4 x i1> @buildvec_i1_splat(i1 %e1) {
+; CHECK-LABEL: buildvec_i1_splat:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    andi a0, a0, 1
+; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmsne.vi v0, v8, 0
+; CHECK-NEXT:    ret
+  %v1 = insertelement <4 x i1> poison, i1 %e1, i32 0
+  %v2 = insertelement <4 x i1> %v1, i1 %e1, i32 1
+  %v3 = insertelement <4 x i1> %v2, i1 %e1, i32 2
+  %v4 = insertelement <4 x i1> %v3, i1 %e1, i32 3
+  ret <4 x i1> %v4
+}
+
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll
index 47cbb2509441..5b9af1a3cfe2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll
@@ -566,3 +566,26 @@ define <128 x i1> @buildvec_mask_optsize_v128i1() optsize {
 ; ZVE32F-NEXT:    ret
   ret <128 x i1> <i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 0, i1 1, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 1, i1 1, i1 1>
 }
+
+define <4 x i1> @buildvec_mask_splat(i1 %e1) {
+; CHECK-LABEL: buildvec_mask_splat:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    andi a0, a0, 1
+; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmsne.vi v0, v8, 0
+; CHECK-NEXT:    ret
+;
+; ZVE32F-LABEL: buildvec_mask_splat:
+; ZVE32F:       # %bb.0:
+; ZVE32F-NEXT:    andi a0, a0, 1
+; ZVE32F-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVE32F-NEXT:    vmv.v.x v8, a0
+; ZVE32F-NEXT:    vmsne.vi v0, v8, 0
+; ZVE32F-NEXT:    ret
+  %v1 = insertelement <4 x i1> poison, i1 %e1, i32 0
+  %v2 = insertelement <4 x i1> %v1, i1 %e1, i32 1
+  %v3 = insertelement <4 x i1> %v2, i1 %e1, i32 2
+  %v4 = insertelement <4 x i1> %v3, i1 %e1, i32 3
+  ret <4 x i1> %v4
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll b/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll
index 351c0bab9dca..adfae5ede7bb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll
@@ -366,3 +366,78 @@ entry:
   ret void
 }
 declare i16 @llvm.vector.reduce.add.v4i16(<4 x i16>)
+
+define i64 @op_then_reduce(<4 x i64> %v, <4 x i64> %v2) {
+; CHECK-LABEL: op_then_reduce:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v10
+; CHECK-NEXT:    vmv.s.x v10, zero
+; CHECK-NEXT:    vredsum.vs v8, v8, v10
+; CHECK-NEXT:    vmv.x.s a0, v8
+; CHECK-NEXT:    ret
+entry:
+  %rdx1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %v)
+  %rdx2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %v2)
+  %res = add i64 %rdx1, %rdx2
+  ret i64 %res
+}
+
+
+define i64 @two_reduce_scalar_bypass(<4 x i64> %v, <4 x i64> %v2) {
+; CHECK-LABEL: two_reduce_scalar_bypass:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmv.s.x v12, zero
+; CHECK-NEXT:    vredxor.vs v8, v8, v12
+; CHECK-NEXT:    vredsum.vs v8, v10, v8
+; CHECK-NEXT:    vmv.x.s a0, v8
+; CHECK-NEXT:    ret
+entry:
+  %rdx1 = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> %v)
+  %rdx2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %v2)
+  %res = add i64 %rdx1, %rdx2
+  ret i64 %res
+}
+
+define i64 @two_reduce_scalar_bypass_zext(<4 x i64> %v, <4 x i32> %v2) {
+; CHECK-LABEL: two_reduce_scalar_bypass_zext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.s.x v11, zero
+; CHECK-NEXT:    vredsum.vs v10, v10, v11
+; CHECK-NEXT:    vmv.x.s a0, v10
+; CHECK-NEXT:    slli a0, a0, 32
+; CHECK-NEXT:    srli a0, a0, 32
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vmv.s.x v10, a0
+; CHECK-NEXT:    vredsum.vs v8, v8, v10
+; CHECK-NEXT:    vmv.x.s a0, v8
+; CHECK-NEXT:    ret
+entry:
+  %rdx1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %v)
+  %rdx2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %v2)
+  %rdx2.zext = zext i32 %rdx2 to i64
+  %res = add i64 %rdx1, %rdx2.zext
+  ret i64 %res
+}
+
+define i64 @two_reduce_scalar_bypass_sext(<4 x i64> %v, <4 x i32> %v2) {
+; CHECK-LABEL: two_reduce_scalar_bypass_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.s.x v11, zero
+; CHECK-NEXT:    vredsum.vs v10, v10, v11
+; CHECK-NEXT:    vmv.x.s a0, v10
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vmv.s.x v10, a0
+; CHECK-NEXT:    vredsum.vs v8, v8, v10
+; CHECK-NEXT:    vmv.x.s a0, v8
+; CHECK-NEXT:    ret
+entry:
+  %rdx1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %v)
+  %rdx2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %v2)
+  %rdx2.zext = sext i32 %rdx2 to i64
+  %res = add i64 %rdx1, %rdx2.zext
+  ret i64 %res
+}
-- 
GitLab


From cae351f3453a0a26ec8eb2ddaf773c24a29d929e Mon Sep 17 00:00:00 2001
From: "A. Jiang" <de34@live.cn>
Date: Sat, 26 Oct 2024 07:38:52 +0800
Subject: [PATCH 217/693] [libc++] Bump `__cpp_lib_optional` and
 `__cpp_lib_variant` (#113650)

In C++20 mode, `__cpp_lib_optional` and `__cpp_lib_variant` should be
`202106L` due to DR P2231R1.

In C++26 mode, `__cpp_lib_variant` should be bumped to `202306L` due to
P2637R3.
- Clang 16/17 shouldn't get this bumping (as member `visit` requires
explicit object parameters), but it's very tricky to make the bumping
conditionally enabled. I _hope_ unconditionally bumping in C++26 will be
OK for LLVM 20 when the support for Clang 17 is dropped.

Related PRs:
- https://reviews.llvm.org/D102119
- #83335
- #76447
---
 libcxx/docs/FeatureTestMacroTable.rst         |  6 ++++++
 libcxx/docs/Status/Cxx20Papers.csv            |  2 +-
 libcxx/docs/Status/Cxx2cPapers.csv            |  2 +-
 libcxx/include/version                        | 11 ++++++++++-
 .../optional.version.compile.pass.cpp         |  5 +++--
 .../variant.version.compile.pass.cpp          | 14 ++++++++------
 .../version.version.compile.pass.cpp          | 19 +++++++++++--------
 .../generate_feature_test_macro_components.py | 10 +++++++---
 8 files changed, 47 insertions(+), 22 deletions(-)

diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index 05b08da52153..db24b65caca6 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -264,6 +264,8 @@ Status
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_move_iterator_concept``                        ``202207L``
     ---------------------------------------------------------- -----------------
+    ``__cpp_lib_optional``                                     ``202106L``
+    ---------------------------------------------------------- -----------------
     ``__cpp_lib_polymorphic_allocator``                        ``201902L``
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_ranges``                                       ``202110L``
@@ -300,6 +302,8 @@ Status
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_unwrap_ref``                                   ``201811L``
     ---------------------------------------------------------- -----------------
+    ``__cpp_lib_variant``                                      ``202106L``
+    ---------------------------------------------------------- -----------------
     **C++23**
     ----------------------------------------------------------------------------
     ``__cpp_lib_adaptor_iterator_pair_constructor``            ``202106L``
@@ -491,5 +495,7 @@ Status
     ``__cpp_lib_to_string``                                    *unimplemented*
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_tuple_like``                                   *unimplemented*
+    ---------------------------------------------------------- -----------------
+    ``__cpp_lib_variant``                                      ``202306L``
     ========================================================== =================
 
diff --git a/libcxx/docs/Status/Cxx20Papers.csv b/libcxx/docs/Status/Cxx20Papers.csv
index cc75d28f14aa..9a057be8ad05 100644
--- a/libcxx/docs/Status/Cxx20Papers.csv
+++ b/libcxx/docs/Status/Cxx20Papers.csv
@@ -192,7 +192,7 @@
 "`P2106R0 <https://wg21.link/P2106R0>`__","Alternative wording for GB315 and GB316","2020-02 (Prague)","|Complete|","15.0",""
 "`P2116R0 <https://wg21.link/P2116R0>`__","Remove tuple-like protocol support from fixed-extent span","2020-02 (Prague)","|Complete|","11.0",""
 "","","","","",""
-"`P2231R1 <https://wg21.link/P2231R1>`__","Missing constexpr in std::optional and std::variant","2021-06 (Virtual)","|Complete|","19.0",""
+"`P2231R1 <https://wg21.link/P2231R1>`__","Missing constexpr in std::optional and std::variant","2021-06 (Virtual)","|Complete|","19.0","Changes of feature-test macros are completed in LLVM 20."
 "`P2325R3 <https://wg21.link/P2325R3>`__","Views should not be required to be default constructible","2021-06 (Virtual)","|Complete|","16.0",""
 "`P2210R2 <https://wg21.link/P2210R2>`__","Superior String Splitting","2021-06 (Virtual)","|Complete|","16.0",""
 "`P2216R3 <https://wg21.link/P2216R3>`__","std::format improvements","2021-06 (Virtual)","|Complete|","15.0",""
diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv
index 8864b1ebe288..d5d5cdda065a 100644
--- a/libcxx/docs/Status/Cxx2cPapers.csv
+++ b/libcxx/docs/Status/Cxx2cPapers.csv
@@ -17,7 +17,7 @@
 "`P0792R14 <https://wg21.link/P0792R14>`__","``function_ref``: a type-erased callable reference","2023-06 (Varna)","","",""
 "`P2874R2 <https://wg21.link/P2874R2>`__","Mandating Annex D Require No More","2023-06 (Varna)","","",""
 "`P2757R3 <https://wg21.link/P2757R3>`__","Type-checking format args","2023-06 (Varna)","","",""
-"`P2637R3 <https://wg21.link/P2637R3>`__","Member ``visit``","2023-06 (Varna)","|Complete|","19.0",""
+"`P2637R3 <https://wg21.link/P2637R3>`__","Member ``visit``","2023-06 (Varna)","|Complete|","19.0","Change of ``__cpp_lib_variant`` is completed in LLVM 20. Change of ``__cpp_lib_format`` is blocked by `P2419R2 <https://wg21.link/P2419R2>`__."
 "`P2641R4 <https://wg21.link/P2641R4>`__","Checking if a ``union`` alternative is active","2023-06 (Varna)","","",""
 "`P1759R6 <https://wg21.link/P1759R6>`__","Native handles and file streams","2023-06 (Varna)","|Complete|","18.0",""
 "`P2697R1 <https://wg21.link/P2697R1>`__","Interfacing ``bitset`` with ``string_view``","2023-06 (Varna)","|Complete|","18.0",""
diff --git a/libcxx/include/version b/libcxx/include/version
index 5ab4f28a04d8..cb75f3b2db68 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -174,6 +174,7 @@ __cpp_lib_nonmember_container_access                    201411L <array> <deque>
 __cpp_lib_not_fn                                        201603L <functional>
 __cpp_lib_null_iterators                                201304L <iterator>
 __cpp_lib_optional                                      202110L <optional>
+                                                        202106L // C++20
                                                         201606L // C++17
 __cpp_lib_optional_range_support                        202406L <optional>
 __cpp_lib_out_ptr                                       202311L <memory>
@@ -261,7 +262,9 @@ __cpp_lib_uncaught_exceptions                           201411L <exception>
 __cpp_lib_unordered_map_try_emplace                     201411L <unordered_map>
 __cpp_lib_unreachable                                   202202L <utility>
 __cpp_lib_unwrap_ref                                    201811L <functional>
-__cpp_lib_variant                                       202102L <variant>
+__cpp_lib_variant                                       202306L <variant>
+                                                        202106L // C++20
+                                                        202102L // C++17
 __cpp_lib_void_t                                        201411L <type_traits>
 
 */
@@ -427,6 +430,8 @@ __cpp_lib_void_t                                        201411L <type_traits>
 # define __cpp_lib_list_remove_return_type              201806L
 # define __cpp_lib_math_constants                       201907L
 # define __cpp_lib_move_iterator_concept                202207L
+# undef  __cpp_lib_optional
+# define __cpp_lib_optional                             202106L
 # if _LIBCPP_AVAILABILITY_HAS_PMR
 #   define __cpp_lib_polymorphic_allocator              201902L
 # endif
@@ -453,6 +458,8 @@ __cpp_lib_void_t                                        201411L <type_traits>
 # define __cpp_lib_to_array                             201907L
 # define __cpp_lib_type_identity                        201806L
 # define __cpp_lib_unwrap_ref                           201811L
+# undef  __cpp_lib_variant
+# define __cpp_lib_variant                              202106L
 #endif
 
 #if _LIBCPP_STD_VER >= 23
@@ -570,6 +577,8 @@ __cpp_lib_void_t                                        201411L <type_traits>
 // # define __cpp_lib_to_string                            202306L
 # undef  __cpp_lib_tuple_like
 // # define __cpp_lib_tuple_like                           202311L
+# undef  __cpp_lib_variant
+# define __cpp_lib_variant                              202306L
 #endif
 
 // clang-format on
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/optional.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/optional.version.compile.pass.cpp
index f265be091f79..91abbbc77837 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/optional.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/optional.version.compile.pass.cpp
@@ -19,6 +19,7 @@
     __cpp_lib_constrained_equality      202403L [C++26]
     __cpp_lib_freestanding_optional     202311L [C++26]
     __cpp_lib_optional                  201606L [C++17]
+                                        202106L [C++20]
                                         202110L [C++23]
     __cpp_lib_optional_range_support    202406L [C++26]
 */
@@ -96,8 +97,8 @@
 # ifndef __cpp_lib_optional
 #   error "__cpp_lib_optional should be defined in c++20"
 # endif
-# if __cpp_lib_optional != 201606L
-#   error "__cpp_lib_optional should have the value 201606L in c++20"
+# if __cpp_lib_optional != 202106L
+#   error "__cpp_lib_optional should have the value 202106L in c++20"
 # endif
 
 # ifdef __cpp_lib_optional_range_support
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/variant.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/variant.version.compile.pass.cpp
index 4dcc477696bf..598e976bda3c 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/variant.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/variant.version.compile.pass.cpp
@@ -19,6 +19,8 @@
     __cpp_lib_constrained_equality    202403L [C++26]
     __cpp_lib_freestanding_variant    202311L [C++26]
     __cpp_lib_variant                 202102L [C++17]
+                                      202106L [C++20]
+                                      202306L [C++26]
 */
 
 #include <variant>
@@ -82,8 +84,8 @@
 # ifndef __cpp_lib_variant
 #   error "__cpp_lib_variant should be defined in c++20"
 # endif
-# if __cpp_lib_variant != 202102L
-#   error "__cpp_lib_variant should have the value 202102L in c++20"
+# if __cpp_lib_variant != 202106L
+#   error "__cpp_lib_variant should have the value 202106L in c++20"
 # endif
 
 #elif TEST_STD_VER == 23
@@ -99,8 +101,8 @@
 # ifndef __cpp_lib_variant
 #   error "__cpp_lib_variant should be defined in c++23"
 # endif
-# if __cpp_lib_variant != 202102L
-#   error "__cpp_lib_variant should have the value 202102L in c++23"
+# if __cpp_lib_variant != 202106L
+#   error "__cpp_lib_variant should have the value 202106L in c++23"
 # endif
 
 #elif TEST_STD_VER > 23
@@ -134,8 +136,8 @@
 # ifndef __cpp_lib_variant
 #   error "__cpp_lib_variant should be defined in c++26"
 # endif
-# if __cpp_lib_variant != 202102L
-#   error "__cpp_lib_variant should have the value 202102L in c++26"
+# if __cpp_lib_variant != 202306L
+#   error "__cpp_lib_variant should have the value 202306L in c++26"
 # endif
 
 #endif // TEST_STD_VER > 23
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index 0614f64a2ef0..5deaee16895f 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -158,6 +158,7 @@
     __cpp_lib_not_fn                                        201603L [C++17]
     __cpp_lib_null_iterators                                201304L [C++14]
     __cpp_lib_optional                                      201606L [C++17]
+                                                            202106L [C++20]
                                                             202110L [C++23]
     __cpp_lib_optional_range_support                        202406L [C++26]
     __cpp_lib_out_ptr                                       202106L [C++23]
@@ -244,6 +245,8 @@
     __cpp_lib_unreachable                                   202202L [C++23]
     __cpp_lib_unwrap_ref                                    201811L [C++20]
     __cpp_lib_variant                                       202102L [C++17]
+                                                            202106L [C++20]
+                                                            202306L [C++26]
     __cpp_lib_void_t                                        201411L [C++17]
 */
 
@@ -4087,8 +4090,8 @@
 # ifndef __cpp_lib_optional
 #   error "__cpp_lib_optional should be defined in c++20"
 # endif
-# if __cpp_lib_optional != 201606L
-#   error "__cpp_lib_optional should have the value 201606L in c++20"
+# if __cpp_lib_optional != 202106L
+#   error "__cpp_lib_optional should have the value 202106L in c++20"
 # endif
 
 # ifdef __cpp_lib_optional_range_support
@@ -4569,8 +4572,8 @@
 # ifndef __cpp_lib_variant
 #   error "__cpp_lib_variant should be defined in c++20"
 # endif
-# if __cpp_lib_variant != 202102L
-#   error "__cpp_lib_variant should have the value 202102L in c++20"
+# if __cpp_lib_variant != 202106L
+#   error "__cpp_lib_variant should have the value 202106L in c++20"
 # endif
 
 # ifndef __cpp_lib_void_t
@@ -6196,8 +6199,8 @@
 # ifndef __cpp_lib_variant
 #   error "__cpp_lib_variant should be defined in c++23"
 # endif
-# if __cpp_lib_variant != 202102L
-#   error "__cpp_lib_variant should have the value 202102L in c++23"
+# if __cpp_lib_variant != 202106L
+#   error "__cpp_lib_variant should have the value 202106L in c++23"
 # endif
 
 # ifndef __cpp_lib_void_t
@@ -8141,8 +8144,8 @@
 # ifndef __cpp_lib_variant
 #   error "__cpp_lib_variant should be defined in c++26"
 # endif
-# if __cpp_lib_variant != 202102L
-#   error "__cpp_lib_variant should have the value 202102L in c++26"
+# if __cpp_lib_variant != 202306L
+#   error "__cpp_lib_variant should have the value 202306L in c++26"
 # endif
 
 # ifndef __cpp_lib_void_t
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index 7ab1af93d177..197d6bbc6922 100755
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -942,7 +942,11 @@ feature_test_macros = [
         },
         {
             "name": "__cpp_lib_optional",
-            "values": {"c++17": 201606, "c++23": 202110},
+            "values": {
+                "c++17": 201606,
+                "c++20": 202106,  # P2231R1 Missing constexpr in std::optional and std::variant
+                "c++23": 202110,  # P0798R8 Monadic operations for std::optional + LWG3621 Remove feature-test macro __cpp_lib_monadic_optional
+            },
             "headers": ["optional"],
         },
         {
@@ -1406,8 +1410,8 @@ feature_test_macros = [
             "name": "__cpp_lib_variant",
             "values": {
                 "c++17": 202102,  # std::visit for classes derived from std::variant
-                # "c++20": 202106,  # Fully constexpr std::variant
-                # "c++26": 202306,  # Member visit (implemented)
+                "c++20": 202106,  # P2231R1 Missing constexpr in std::optional and std::variant
+                "c++26": 202306,  # P2637R3 Member visit
             },
             "headers": ["variant"],
         },
-- 
GitLab


From 5f7bad07b9d5b6c5cfa8c16a4e62cf1e128725be Mon Sep 17 00:00:00 2001
From: B I Mohammed Abbas <bimohammadabbas@gmail.com>
Date: Sat, 26 Oct 2024 05:35:23 +0530
Subject: [PATCH 218/693] Add extendhfxf2 into compiler rt (#111099)

Retry pr #109090 with updated extendhfxf2 test
---
 compiler-rt/lib/builtins/CMakeLists.txt       |  1 +
 compiler-rt/lib/builtins/extendhfxf2.c        | 18 +++++
 .../lib/builtins/macho_embedded/common.txt    |  1 +
 .../test/builtins/Unit/extendhfxf2_test.c     | 71 +++++++++++++++++++
 .../compiler-rt/lib/builtins/BUILD.gn         |  1 +
 5 files changed, 92 insertions(+)
 create mode 100644 compiler-rt/lib/builtins/extendhfxf2.c
 create mode 100644 compiler-rt/test/builtins/Unit/extendhfxf2_test.c

diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index 9a0a50ee7003..97a9e508d37a 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -104,6 +104,7 @@ set(GENERIC_SOURCES
   divti3.c
   extendsfdf2.c
   extendhfsf2.c
+  extendhfxf2.c
   ffsdi2.c
   ffssi2.c
   ffsti2.c
diff --git a/compiler-rt/lib/builtins/extendhfxf2.c b/compiler-rt/lib/builtins/extendhfxf2.c
new file mode 100644
index 000000000000..7425859f79f7
--- /dev/null
+++ b/compiler-rt/lib/builtins/extendhfxf2.c
@@ -0,0 +1,18 @@
+//===-- lib/extendhfxf2.c - half -> long double conversion --------*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#define SRC_HALF
+#define DST_DOUBLE
+#include "fp_extend_impl.inc"
+
+// Use a forwarding definition and noinline to implement a poor man's alias,
+// as there isn't a good cross-platform way of defining one.
+// Long double are expected to be as precise as double.
+COMPILER_RT_ABI NOINLINE long double __extendhfxf2(src_t a) {
+  return (long double)__extendXfYf2__(a);
+}
diff --git a/compiler-rt/lib/builtins/macho_embedded/common.txt b/compiler-rt/lib/builtins/macho_embedded/common.txt
index 819109768f52..fa99bc239e68 100644
--- a/compiler-rt/lib/builtins/macho_embedded/common.txt
+++ b/compiler-rt/lib/builtins/macho_embedded/common.txt
@@ -60,6 +60,7 @@ divsf3
 divsi3
 extendsfdf2
 extendhfsf2
+extendhfxf2
 ffssi2
 fixdfsi
 fixsfsi
diff --git a/compiler-rt/test/builtins/Unit/extendhfxf2_test.c b/compiler-rt/test/builtins/Unit/extendhfxf2_test.c
new file mode 100644
index 000000000000..9972b024ab41
--- /dev/null
+++ b/compiler-rt/test/builtins/Unit/extendhfxf2_test.c
@@ -0,0 +1,71 @@
+// RUN: %clang_builtins %s %librt -o %t && %run %t
+// REQUIRES: librt_has_extendhfxf2
+
+#include <limits.h>
+#include <math.h> // for isnan, isinf
+#include <stdio.h>
+
+#if __LDBL_MANT_DIG__ >= 64 && defined(COMPILER_RT_HAS_FLOAT16)
+
+long double __extendhfxf2(_Float16 f);
+
+int test_extendhfxf2(_Float16 a, long double expected) {
+  long double x = __extendhfxf2(a);
+  __uint16_t *b = (void *)&a;
+  int ret = !((isnan(x) && isnan(expected)) || x == expected);
+  if (ret) {
+    printf("error in test__extendhfxf2(%#.4x) = %.20Lf, "
+           "expected %.20Lf\n",
+           *b, x, expected);
+  }
+  return ret;
+}
+
+char assumption_1[sizeof(_Float16) * CHAR_BIT == 16] = {0};
+
+int main() {
+  // Small positive value
+  if (test_extendhfxf2(0.09997558593750000000f, 0.09997558593750000000L))
+    return 1;
+
+  // Small negative value
+  if (test_extendhfxf2(-0.09997558593750000000f, -0.09997558593750000000L))
+    return 1;
+
+  // Zero
+  if (test_extendhfxf2(0.0f, 0.0L))
+    return 1;
+
+  // Smallest positive non-zero value
+  if (test_extendhfxf2(0x1p-16f, 0x1p-16L))
+    return 1;
+
+  // Smallest negative non-zero value
+  if (test_extendhfxf2(-0x1p-16f, -0x1p-16L))
+    return 1;
+
+  // Positive infinity
+  if (test_extendhfxf2(__builtin_huge_valf16(), __builtin_huge_valf64x()))
+    return 1;
+
+  // Negative infinity
+  if (test_extendhfxf2(-__builtin_huge_valf16(),
+                       (long double)-__builtin_huge_valf64x()))
+    return 1;
+
+  // NaN
+  if (test_extendhfxf2(__builtin_nanf16(""),
+                       (long double)__builtin_nanf64x("")))
+    return 1;
+
+  return 0;
+}
+
+#else
+
+int main() {
+  printf("skipped\n");
+  return 0;
+}
+
+#endif
diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
index 8904aed28229..efbf01960bf9 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
@@ -126,6 +126,7 @@ static_library("builtins") {
     "divsi3.c",
     "divti3.c",
     "extendhfsf2.c",
+    "extendhfxf2.c"
     "extendsfdf2.c",
     "ffsdi2.c",
     "ffssi2.c",
-- 
GitLab


From 889b67c9d30e3024a1317431d66c22599f6c2011 Mon Sep 17 00:00:00 2001
From: donald chen <chenxunyu1993@gmail.com>
Date: Sat, 26 Oct 2024 08:07:51 +0800
Subject: [PATCH 219/693] [mlir] [memref] add more checks to the
 memref.reinterpret_cast (#112669)

Operation memref.reinterpret_cast was accept input like:

%out = memref.reinterpret_cast %in to offset: [%offset], sizes: [10],
strides: [1]
         : memref<?xf32> to memref<10xf32>

A problem arises: while lowering, the true offset of %out is %offset,
but its data type indicates an offset of 0. Permitting this
inconsistency can result in incorrect outcomes, as certain pass might
erroneously extract the offset from the data type of %out.

This patch fixes this by enforcing that the return value's data type
aligns
with the input parameter.
---
 .../GPU/Transforms/DecomposeMemRefs.cpp       | 13 ++++++++-
 mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp      | 27 ++++++++++---------
 .../Dialect/MemRef/Transforms/ExpandOps.cpp   | 22 ++++++++++++---
 .../Transforms/ExpandStridedMetadata.cpp      | 17 +++++-------
 .../expand-then-convert-to-llvm.mlir          | 10 ++-----
 mlir/test/Dialect/GPU/decompose-memrefs.mlir  | 12 ++++-----
 mlir/test/Dialect/MemRef/expand-ops.mlir      | 13 +++++----
 .../MemRef/expand-strided-metadata.mlir       | 21 +++++----------
 mlir/test/Dialect/MemRef/invalid.mlir         |  9 +++++++
 9 files changed, 81 insertions(+), 63 deletions(-)

diff --git a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
index 2b2d10a7733e..004d73a77e53 100644
--- a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
@@ -29,6 +29,17 @@ namespace mlir {
 
 using namespace mlir;
 
+static MemRefType inferCastResultType(Value source, OpFoldResult offset) {
+  auto sourceType = cast<BaseMemRefType>(source.getType());
+  SmallVector<int64_t> staticOffsets;
+  SmallVector<Value> dynamicOffsets;
+  dispatchIndexOpFoldResults(offset, dynamicOffsets, staticOffsets);
+  auto stridedLayout =
+      StridedLayoutAttr::get(source.getContext(), staticOffsets.front(), {});
+  return MemRefType::get({}, sourceType.getElementType(), stridedLayout,
+                         sourceType.getMemorySpace());
+}
+
 static void setInsertionPointToStart(OpBuilder &builder, Value val) {
   if (auto *parentOp = val.getDefiningOp()) {
     builder.setInsertionPointAfter(parentOp);
@@ -98,7 +109,7 @@ static Value getFlatMemref(OpBuilder &rewriter, Location loc, Value source,
   SmallVector<OpFoldResult> offsetsTemp = getAsOpFoldResult(offsets);
   auto &&[base, offset, ignore] =
       getFlatOffsetAndStrides(rewriter, loc, source, offsetsTemp);
-  auto retType = cast<MemRefType>(base.getType());
+  MemRefType retType = inferCastResultType(base, offset);
   return rewriter.create<memref::ReinterpretCastOp>(loc, retType, base, offset,
                                                     std::nullopt, std::nullopt);
 }
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index d579a27359df..2219505c9b80 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -1892,11 +1892,12 @@ LogicalResult ReinterpretCastOp::verify() {
   // Match sizes in result memref type and in static_sizes attribute.
   for (auto [idx, resultSize, expectedSize] :
        llvm::enumerate(resultType.getShape(), getStaticSizes())) {
-    if (!ShapedType::isDynamic(resultSize) &&
-        !ShapedType::isDynamic(expectedSize) && resultSize != expectedSize)
+    if (!ShapedType::isDynamic(resultSize) && resultSize != expectedSize)
       return emitError("expected result type with size = ")
-             << expectedSize << " instead of " << resultSize
-             << " in dim = " << idx;
+             << (ShapedType::isDynamic(expectedSize)
+                     ? std::string("dynamic")
+                     : std::to_string(expectedSize))
+             << " instead of " << resultSize << " in dim = " << idx;
   }
 
   // Match offset and strides in static_offset and static_strides attributes. If
@@ -1910,20 +1911,22 @@ LogicalResult ReinterpretCastOp::verify() {
 
   // Match offset in result memref type and in static_offsets attribute.
   int64_t expectedOffset = getStaticOffsets().front();
-  if (!ShapedType::isDynamic(resultOffset) &&
-      !ShapedType::isDynamic(expectedOffset) && resultOffset != expectedOffset)
+  if (!ShapedType::isDynamic(resultOffset) && resultOffset != expectedOffset)
     return emitError("expected result type with offset = ")
-           << expectedOffset << " instead of " << resultOffset;
+           << (ShapedType::isDynamic(expectedOffset)
+                   ? std::string("dynamic")
+                   : std::to_string(expectedOffset))
+           << " instead of " << resultOffset;
 
   // Match strides in result memref type and in static_strides attribute.
   for (auto [idx, resultStride, expectedStride] :
        llvm::enumerate(resultStrides, getStaticStrides())) {
-    if (!ShapedType::isDynamic(resultStride) &&
-        !ShapedType::isDynamic(expectedStride) &&
-        resultStride != expectedStride)
+    if (!ShapedType::isDynamic(resultStride) && resultStride != expectedStride)
       return emitError("expected result type with stride = ")
-             << expectedStride << " instead of " << resultStride
-             << " in dim = " << idx;
+             << (ShapedType::isDynamic(expectedStride)
+                     ? std::string("dynamic")
+                     : std::to_string(expectedStride))
+             << " instead of " << resultStride << " in dim = " << idx;
   }
 
   return success();
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
index faba12f5bf82..83683c7e617b 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
@@ -89,7 +89,8 @@ public:
     strides.resize(rank);
 
     Location loc = op.getLoc();
-    Value stride = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    Value stride = nullptr;
+    int64_t staticStride = 1;
     for (int i = rank - 1; i >= 0; --i) {
       Value size;
       // Load dynamic sizes from the shape input, use constants for static dims.
@@ -105,9 +106,22 @@ public:
         size = rewriter.create<arith::ConstantOp>(loc, sizeAttr);
         sizes[i] = sizeAttr;
       }
-      strides[i] = stride;
-      if (i > 0)
-        stride = rewriter.create<arith::MulIOp>(loc, stride, size);
+      if (stride)
+        strides[i] = stride;
+      else
+        strides[i] = rewriter.getIndexAttr(staticStride);
+
+      if (i > 0) {
+        if (stride) {
+          stride = rewriter.create<arith::MulIOp>(loc, stride, size);
+        } else if (op.getType().isDynamicDim(i)) {
+          stride = rewriter.create<arith::MulIOp>(
+              loc, rewriter.create<arith::ConstantIndexOp>(loc, staticStride),
+              size);
+        } else {
+          staticStride *= op.getType().getDimSize(i);
+        }
+      }
     }
     rewriter.replaceOpWithNewOp<memref::ReinterpretCastOp>(
         op, op.getType(), op.getSource(), /*offset=*/rewriter.getIndexAttr(0),
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
index a2049ba4a492..087d1fcc2b23 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
@@ -507,6 +507,8 @@ getCollapsedStride(memref::CollapseShapeOp collapseShape, OpBuilder &builder,
 
   SmallVector<OpFoldResult> groupStrides;
   ArrayRef<int64_t> srcShape = sourceType.getShape();
+
+  OpFoldResult lastValidStride = nullptr;
   for (int64_t currentDim : reassocGroup) {
     // Skip size-of-1 dimensions, since right now their strides may be
     // meaningless.
@@ -517,11 +519,11 @@ getCollapsedStride(memref::CollapseShapeOp collapseShape, OpBuilder &builder,
       continue;
 
     int64_t currentStride = strides[currentDim];
-    groupStrides.push_back(ShapedType::isDynamic(currentStride)
-                               ? origStrides[currentDim]
-                               : builder.getIndexAttr(currentStride));
+    lastValidStride = ShapedType::isDynamic(currentStride)
+                          ? origStrides[currentDim]
+                          : builder.getIndexAttr(currentStride);
   }
-  if (groupStrides.empty()) {
+  if (!lastValidStride) {
     // We're dealing with a 1x1x...x1 shape. The stride is meaningless,
     // but we still have to make the type system happy.
     MemRefType collapsedType = collapseShape.getResultType();
@@ -543,12 +545,7 @@ getCollapsedStride(memref::CollapseShapeOp collapseShape, OpBuilder &builder,
     return {builder.getIndexAttr(finalStride)};
   }
 
-  // For the general case, we just want the minimum stride
-  // since the collapsed dimensions are contiguous.
-  auto minMap = AffineMap::getMultiDimIdentityMap(groupStrides.size(),
-                                                  builder.getContext());
-  return {makeComposedFoldedAffineMin(builder, collapseShape.getLoc(), minMap,
-                                      groupStrides)};
+  return {lastValidStride};
 }
 
 /// From `reshape_like(memref, subSizes, subStrides))` compute
diff --git a/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir
index 55b1bc9c545a..ec5ceae57ccb 100644
--- a/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir
+++ b/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir
@@ -425,8 +425,6 @@ func.func @collapse_shape_dynamic_with_non_identity_layout(
 // CHECK:           %[[SIZE1:.*]] = llvm.extractvalue %[[MEM]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 // CHECK:           %[[SIZE2:.*]] = llvm.extractvalue %[[MEM]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 // CHECK:           %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
-// CHECK:           %[[STRIDE0_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[STRIDE0]] : i64 to index
-// CHECK:           %[[STRIDE0:.*]] = builtin.unrealized_conversion_cast %[[STRIDE0_TO_IDX]] : index to i64
 // CHECK:           %[[FINAL_SIZE1:.*]] = llvm.mul %[[SIZE1]], %[[SIZE2]]  : i64
 // CHECK:           %[[SIZE1_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[FINAL_SIZE1]] : i64 to index
 // CHECK:           %[[FINAL_SIZE1:.*]] = builtin.unrealized_conversion_cast %[[SIZE1_TO_IDX]] : index to i64
@@ -548,23 +546,19 @@ func.func @collapse_shape_dynamic(%arg0 : memref<1x2x?xf32>) -> memref<1x?xf32>
 // CHECK:           %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
 // CHECK:           %[[SIZE2:.*]] = llvm.extractvalue %[[MEM]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 // CHECK:           %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
-// CHECK:           %[[STRIDE1:.*]] = llvm.extractvalue %[[MEM]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 // CHECK:           %[[C2:.*]] = llvm.mlir.constant(2 : index) : i64
 // CHECK:           %[[FINAL_SIZE1:.*]] = llvm.mul %[[SIZE2]], %[[C2]]  : i64
 // CHECK:           %[[SIZE1_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[FINAL_SIZE1]] : i64 to index
 // CHECK:           %[[FINAL_SIZE1:.*]] = builtin.unrealized_conversion_cast %[[SIZE1_TO_IDX]] : index to i64
-// CHECK:           %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64
-// CHECK:           %[[MIN_STRIDE1:.*]] = llvm.intr.smin(%[[STRIDE1]], %[[C1]]) : (i64, i64) -> i64
-// CHECK:           %[[MIN_STRIDE1_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[MIN_STRIDE1]] : i64 to index
-// CHECK:           %[[MIN_STRIDE1:.*]] = builtin.unrealized_conversion_cast %[[MIN_STRIDE1_TO_IDX]] : index to i64
 // CHECK:           %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK:           %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64
 // CHECK:           %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[DESC5:.*]] = llvm.insertvalue %[[FINAL_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
-// CHECK:           %[[DESC6:.*]] = llvm.insertvalue %[[MIN_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK:           %[[DESC6:.*]] = llvm.insertvalue %[[C1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK:           %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC6]] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<1x?xf32>
 // CHECK:           return %[[RES]] : memref<1x?xf32>
 // CHECK:         }
diff --git a/mlir/test/Dialect/GPU/decompose-memrefs.mlir b/mlir/test/Dialect/GPU/decompose-memrefs.mlir
index 56fc9a66b7ac..1a1922194845 100644
--- a/mlir/test/Dialect/GPU/decompose-memrefs.mlir
+++ b/mlir/test/Dialect/GPU/decompose-memrefs.mlir
@@ -7,8 +7,8 @@
 //       CHECK:  gpu.launch
 //  CHECK-SAME:  threads(%[[TX:.*]], %[[TY:.*]], %[[TZ:.*]]) in
 //       CHECK:  %[[IDX:.*]] = affine.apply #[[MAP]]()[%[[TX]], %[[STRIDES]]#0, %[[TY]], %[[STRIDES]]#1, %[[TZ]]]
-//       CHECK:  %[[PTR:.*]] = memref.reinterpret_cast %[[BASE]] to offset: [%[[IDX]]], sizes: [], strides: [] : memref<f32> to memref<f32>
-//       CHECK:  memref.store %[[VAL]], %[[PTR]][] : memref<f32>
+//       CHECK:  %[[PTR:.*]] = memref.reinterpret_cast %[[BASE]] to offset: [%[[IDX]]], sizes: [], strides: [] : memref<f32> to memref<f32, strided<[], offset: ?>>
+//       CHECK:  memref.store %[[VAL]], %[[PTR]][] : memref<f32, strided<[], offset: ?>>
 func.func @decompose_store(%arg0 : f32, %arg1 : memref<?x?x?xf32>) {
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
@@ -33,8 +33,8 @@ func.func @decompose_store(%arg0 : f32, %arg1 : memref<?x?x?xf32>) {
 //       CHECK:  gpu.launch
 //  CHECK-SAME:  threads(%[[TX:.*]], %[[TY:.*]], %[[TZ:.*]]) in
 //       CHECK:  %[[IDX:.*]] = affine.apply #[[MAP]]()[%[[OFFSET]], %[[TX]], %[[STRIDES]]#0, %[[TY]], %[[STRIDES]]#1, %[[TZ]], %[[STRIDES]]#2]
-//       CHECK:  %[[PTR:.*]] = memref.reinterpret_cast %[[BASE]] to offset: [%[[IDX]]], sizes: [], strides: [] : memref<f32> to memref<f32>
-//       CHECK:  memref.store %[[VAL]], %[[PTR]][] : memref<f32>
+//       CHECK:  %[[PTR:.*]] = memref.reinterpret_cast %[[BASE]] to offset: [%[[IDX]]], sizes: [], strides: [] : memref<f32> to memref<f32, strided<[], offset: ?>>
+//       CHECK:  memref.store %[[VAL]], %[[PTR]][] : memref<f32, strided<[], offset: ?>>
 func.func @decompose_store_strided(%arg0 : f32, %arg1 : memref<?x?x?xf32, strided<[?, ?, ?], offset: ?>>) {
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
@@ -59,8 +59,8 @@ func.func @decompose_store_strided(%arg0 : f32, %arg1 : memref<?x?x?xf32, stride
 //       CHECK:  gpu.launch
 //  CHECK-SAME:  threads(%[[TX:.*]], %[[TY:.*]], %[[TZ:.*]]) in
 //       CHECK:  %[[IDX:.*]] = affine.apply #[[MAP]]()[%[[TX]], %[[STRIDES]]#0, %[[TY]], %[[STRIDES]]#1, %[[TZ]]]
-//       CHECK:  %[[PTR:.*]] = memref.reinterpret_cast %[[BASE]] to offset: [%[[IDX]]], sizes: [], strides: [] : memref<f32> to memref<f32>
-//       CHECK:  %[[RES:.*]] = memref.load %[[PTR]][] : memref<f32>
+//       CHECK:  %[[PTR:.*]] = memref.reinterpret_cast %[[BASE]] to offset: [%[[IDX]]], sizes: [], strides: [] : memref<f32> to memref<f32, strided<[], offset: ?>>
+//       CHECK:  %[[RES:.*]] = memref.load %[[PTR]][] : memref<f32, strided<[], offset: ?>>
 //       CHECK:  "test.test"(%[[RES]]) : (f32) -> ()
 func.func @decompose_load(%arg0 : memref<?x?x?xf32>) {
   %c0 = arith.constant 0 : index
diff --git a/mlir/test/Dialect/MemRef/expand-ops.mlir b/mlir/test/Dialect/MemRef/expand-ops.mlir
index f958a92b751a..65932b5814a6 100644
--- a/mlir/test/Dialect/MemRef/expand-ops.mlir
+++ b/mlir/test/Dialect/MemRef/expand-ops.mlir
@@ -52,14 +52,13 @@ func.func @memref_reshape(%input: memref<*xf32>,
 // CHECK-SAME: [[SRC:%.*]]: memref<*xf32>,
 // CHECK-SAME: [[SHAPE:%.*]]: memref<3xi32>) -> memref<?x?x8xf32> {
 
-// CHECK: [[C1:%.*]] = arith.constant 1 : index
 // CHECK: [[C8:%.*]] = arith.constant 8 : index
-// CHECK: [[STRIDE_1:%.*]] = arith.muli [[C1]], [[C8]] : index
-
-// CHECK: [[C1_:%.*]] = arith.constant 1 : index
-// CHECK: [[DIM_1:%.*]] = memref.load [[SHAPE]]{{\[}}[[C1_]]] : memref<3xi32>
+// CHECK: [[C1:%.*]] = arith.constant 1 : index
+// CHECK: [[DIM_1:%.*]] = memref.load [[SHAPE]]{{\[}}[[C1]]] : memref<3xi32>
 // CHECK: [[SIZE_1:%.*]] = arith.index_cast [[DIM_1]] : i32 to index
-// CHECK: [[STRIDE_0:%.*]] = arith.muli [[STRIDE_1]], [[SIZE_1]] : index
+
+// CHECK: [[C8_:%.*]] = arith.constant 8 : index
+// CHECK: [[STRIDE_0:%.*]] = arith.muli [[C8_]], [[SIZE_1]] : index
 
 // CHECK: [[C0:%.*]] = arith.constant 0 : index
 // CHECK: [[DIM_0:%.*]] = memref.load [[SHAPE]]{{\[}}[[C0]]] : memref<3xi32>
@@ -67,5 +66,5 @@ func.func @memref_reshape(%input: memref<*xf32>,
 
 // CHECK: [[RESULT:%.*]] = memref.reinterpret_cast [[SRC]]
 // CHECK-SAME: to offset: [0], sizes: {{\[}}[[SIZE_0]], [[SIZE_1]], 8],
-// CHECK-SAME: strides: {{\[}}[[STRIDE_0]], [[STRIDE_1]], [[C1]]]
+// CHECK-SAME: strides: {{\[}}[[STRIDE_0]], 8, 1]
 // CHECK-SAME: : memref<*xf32> to memref<?x?x8xf32>
diff --git a/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir b/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir
index 8aac802ba10a..647731db439c 100644
--- a/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir
+++ b/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir
@@ -931,19 +931,15 @@ func.func @extract_aligned_pointer_as_index_of_unranked_source(%arg0: memref<*xf
 //          = min(7, 1)
 //          = 1
 //
-//   CHECK-DAG: #[[$STRIDE0_MIN_MAP:.*]] = affine_map<()[s0] -> (s0)>
-//   CHECK-DAG: #[[$SIZE0_MAP:.*]] = affine_map<()[s0, s1] -> ((s0 * s1) * 4)>
-//   CHECK-DAG: #[[$STRIDE1_MIN_MAP:.*]] = affine_map<()[s0, s1] -> (s0, s1, 42)>
+//       CHECK: #[[$SIZE0_MAP:.*]] = affine_map<()[s0, s1] -> ((s0 * s1) * 4)>
 // CHECK-LABEL: func @simplify_collapse(
 //  CHECK-SAME: %[[ARG:.*]]: memref<?x?x4x?x6x7xi32>)
 //
 //       CHECK: %[[BASE:.*]], %[[OFFSET:.*]], %[[SIZES:.*]]:6, %[[STRIDES:.*]]:6 = memref.extract_strided_metadata %[[ARG]] : memref<?x?x4x?x6x7xi32>
 //
-//   CHECK-DAG: %[[DYN_STRIDE0:.*]] = affine.min #[[$STRIDE0_MIN_MAP]]()[%[[STRIDES]]#0]
-//   CHECK-DAG: %[[DYN_SIZE1:.*]] = affine.apply #[[$SIZE0_MAP]]()[%[[SIZES]]#1, %[[SIZES]]#3]
-//   CHECK-DAG: %[[DYN_STRIDE1:.*]] = affine.min #[[$STRIDE1_MIN_MAP]]()[%[[STRIDES]]#1, %[[STRIDES]]#2]
+//       CHECK: %[[DYN_SIZE1:.*]] = affine.apply #[[$SIZE0_MAP]]()[%[[SIZES]]#1, %[[SIZES]]#3]
 //
-//       CHECK: %[[COLLAPSE_VIEW:.*]] = memref.reinterpret_cast %[[BASE]] to offset: [0], sizes: [%[[SIZES]]#0, %[[DYN_SIZE1]], 42], strides: [%[[DYN_STRIDE0]], %[[DYN_STRIDE1]], 1]
+//       CHECK: %[[COLLAPSE_VIEW:.*]] = memref.reinterpret_cast %[[BASE]] to offset: [0], sizes: [%[[SIZES]]#0, %[[DYN_SIZE1]], 42], strides: [%[[STRIDES]]#0, 42, 1]
 func.func @simplify_collapse(%arg : memref<?x?x4x?x6x7xi32>)
   -> memref<?x?x42xi32> {
 
@@ -1046,15 +1042,12 @@ func.func @simplify_collapse_with_dim_of_size1_and_non_1_stride
 //           We just return the first dynamic one for this group.
 //
 //
-//   CHECK-DAG: #[[$STRIDE0_MIN_MAP:.*]] = affine_map<()[s0, s1] -> (s0, s1)>
 // CHECK-LABEL: func @simplify_collapse_with_dim_of_size1_and_resulting_dyn_stride(
 //  CHECK-SAME: %[[ARG:.*]]: memref<2x3x1x1x1xi32, strided<[?, ?, ?, ?, 2]
 //
 //       CHECK: %[[BASE:.*]], %[[OFFSET:.*]], %[[SIZES:.*]]:5, %[[STRIDES:.*]]:5 = memref.extract_strided_metadata %[[ARG]] : memref<2x3x1x1x1xi32, strided<[?, ?, ?, ?, 2], offset: ?>>
 //
-//   CHECK-DAG: %[[DYN_STRIDE0:.*]] = affine.min #[[$STRIDE0_MIN_MAP]]()[%[[STRIDES]]#0, %[[STRIDES]]#1]
-//
-//       CHECK: %[[COLLAPSE_VIEW:.*]] = memref.reinterpret_cast %[[BASE]] to offset: [%[[OFFSET]]], sizes: [6, 1], strides: [%[[DYN_STRIDE0]], %[[STRIDES]]#2]
+//       CHECK: %[[COLLAPSE_VIEW:.*]] = memref.reinterpret_cast %[[BASE]] to offset: [%[[OFFSET]]], sizes: [6, 1], strides: [%[[STRIDES]]#1, %[[STRIDES]]#2]
 func.func @simplify_collapse_with_dim_of_size1_and_resulting_dyn_stride
     (%arg0: memref<2x3x1x1x1xi32, strided<[?, ?, ?, ?, 2], offset: ?>>)
     -> memref<6x1xi32, strided<[?, ?], offset: ?>> {
@@ -1083,8 +1076,7 @@ func.func @simplify_collapse_with_dim_of_size1_and_resulting_dyn_stride
 // Stride 2 = origStride5
 //          = 1
 //
-//   CHECK-DAG: #[[$SIZE0_MAP:.*]] = affine_map<()[s0, s1] -> ((s0 * s1) * 4)>
-//   CHECK-DAG: #[[$STRIDE0_MAP:.*]] = affine_map<()[s0] -> (s0)>
+//       CHECK: #[[$SIZE0_MAP:.*]] = affine_map<()[s0, s1] -> ((s0 * s1) * 4)>
 // CHECK-LABEL: func @extract_strided_metadata_of_collapse(
 //  CHECK-SAME: %[[ARG:.*]]: memref<?x?x4x?x6x7xi32>)
 //
@@ -1094,10 +1086,9 @@ func.func @simplify_collapse_with_dim_of_size1_and_resulting_dyn_stride
 //
 //   CHECK-DAG: %[[BASE:.*]], %[[OFFSET:.*]], %[[SIZES:.*]]:6, %[[STRIDES:.*]]:6 = memref.extract_strided_metadata %[[ARG]] : memref<?x?x4x?x6x7xi32>
 //
-//   CHECK-DAG: %[[DYN_STRIDE0:.*]] = affine.min #[[$STRIDE0_MAP]]()[%[[STRIDES]]#0]
 //   CHECK-DAG: %[[DYN_SIZE1:.*]] = affine.apply #[[$SIZE0_MAP]]()[%[[SIZES]]#1, %[[SIZES]]#3]
 //
-//       CHECK: return %[[BASE]], %[[C0]], %[[SIZES]]#0, %[[DYN_SIZE1]], %[[C42]], %[[DYN_STRIDE0]], %[[C42]], %[[C1]]
+//       CHECK: return %[[BASE]], %[[C0]], %[[SIZES]]#0, %[[DYN_SIZE1]], %[[C42]], %[[STRIDES]]#0, %[[C42]], %[[C1]]
 func.func @extract_strided_metadata_of_collapse(%arg : memref<?x?x4x?x6x7xi32>)
   -> (memref<i32>, index,
       index, index, index,
diff --git a/mlir/test/Dialect/MemRef/invalid.mlir b/mlir/test/Dialect/MemRef/invalid.mlir
index 0f533cb95a0c..51c4781c9022 100644
--- a/mlir/test/Dialect/MemRef/invalid.mlir
+++ b/mlir/test/Dialect/MemRef/invalid.mlir
@@ -217,6 +217,15 @@ func.func @memref_reinterpret_cast_no_map_but_offset(%in: memref<?xf32>) {
 
 // -----
 
+func.func @memref_reinterpret_cast_offset_mismatch_dynamic(%in: memref<?xf32>, %offset : index) {
+  // expected-error @+1 {{expected result type with offset = dynamic instead of 0}}
+  %out = memref.reinterpret_cast %in to offset: [%offset], sizes: [10], strides: [1]
+         : memref<?xf32> to memref<10xf32>
+  return
+}
+
+// -----
+
 func.func @memref_reinterpret_cast_no_map_but_stride(%in: memref<?xf32>) {
   // expected-error @+1 {{expected result type with stride = 10 instead of 1 in dim = 0}}
   %out = memref.reinterpret_cast %in to offset: [0], sizes: [10], strides: [10]
-- 
GitLab


From a879073494e7989f670ab0432dd0047ffbf5d1cd Mon Sep 17 00:00:00 2001
From: lntue <lntue@google.com>
Date: Fri, 25 Oct 2024 22:11:07 -0400
Subject: [PATCH 220/693] [libc][math] Fix a bug in cbrt when the result is
 rounded to the next exponent. (#113749)

---
 libc/src/math/generic/cbrt.cpp         |  4 ++--
 libc/test/src/math/cbrt_test.cpp       | 13 +++++++------
 libc/test/src/math/smoke/cbrt_test.cpp |  2 ++
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/libc/src/math/generic/cbrt.cpp b/libc/src/math/generic/cbrt.cpp
index 036664c2aafa..4fa24c54fdee 100644
--- a/libc/src/math/generic/cbrt.cpp
+++ b/libc/src/math/generic/cbrt.cpp
@@ -235,10 +235,10 @@ LLVM_LIBC_FUNCTION(double, cbrt, (double x)) {
 
   // Lambda function to update the exponent of the result.
   auto update_exponent = [=](double r) -> double {
-    uint64_t r_m = FPBits(r).uintval() & 0x800F'FFFF'FFFF'FFFF;
+    uint64_t r_m = FPBits(r).uintval() - 0x3FF0'0000'0000'0000;
     // Adjust exponent and sign.
     uint64_t r_bits =
-        r_m | (static_cast<uint64_t>(out_e) << FPBits::FRACTION_LEN);
+        r_m + (static_cast<uint64_t>(out_e) << FPBits::FRACTION_LEN);
     return FPBits(r_bits).get_val();
   };
 
diff --git a/libc/test/src/math/cbrt_test.cpp b/libc/test/src/math/cbrt_test.cpp
index 2ef2140966f5..2e2de16fc859 100644
--- a/libc/test/src/math/cbrt_test.cpp
+++ b/libc/test/src/math/cbrt_test.cpp
@@ -87,12 +87,13 @@ TEST_F(LlvmLibcCbrtTest, InDoubleRange) {
 
 TEST_F(LlvmLibcCbrtTest, SpecialValues) {
   constexpr double INPUTS[] = {
-      0x1.4f61672324c8p-1028, 0x1.00152f57068b7p-1, 0x1.006509cda9886p-1,
-      0x1.018369b92e523p-1,   0x1.10af932ef2bf9p-1, 0x1.1a41117939fdbp-1,
-      0x1.2ae8076520d9ap-1,   0x1.a202bfc89ddffp-1, 0x1.a6bb8c803147bp-1,
-      0x1.000197b499b1bp+0,   0x1.00065ed266c6cp+0, 0x1.d4306c202c4c2p+0,
-      0x1.8fd409efe4851p+1,   0x1.95fd0eb31cc4p+1,  0x1.7cef1d276e335p+2,
-      0x1.94910c4fc98p+2,     0x1.a0cc1327bb4c4p+2, 0x1.e7d6ebed549c4p+2,
+      0x1.4f61672324c8p-1028, -0x1.fffffffffffffp-1021, 0x1.00152f57068b7p-1,
+      0x1.006509cda9886p-1,   0x1.018369b92e523p-1,     0x1.10af932ef2bf9p-1,
+      0x1.1a41117939fdbp-1,   0x1.2ae8076520d9ap-1,     0x1.a202bfc89ddffp-1,
+      0x1.a6bb8c803147bp-1,   0x1.000197b499b1bp+0,     0x1.00065ed266c6cp+0,
+      0x1.d4306c202c4c2p+0,   0x1.8fd409efe4851p+1,     0x1.95fd0eb31cc4p+1,
+      0x1.7cef1d276e335p+2,   0x1.94910c4fc98p+2,       0x1.a0cc1327bb4c4p+2,
+      0x1.e7d6ebed549c4p+2,
   };
   for (double v : INPUTS) {
     double x = FPBits(v).get_val();
diff --git a/libc/test/src/math/smoke/cbrt_test.cpp b/libc/test/src/math/smoke/cbrt_test.cpp
index 724e0e979dec..d57cdb20de27 100644
--- a/libc/test/src/math/smoke/cbrt_test.cpp
+++ b/libc/test/src/math/smoke/cbrt_test.cpp
@@ -32,4 +32,6 @@ TEST_F(LlvmLibcCbrtTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(-0x1.0p42, LIBC_NAMESPACE::cbrt(-0x1.0p126));
   EXPECT_FP_EQ_ALL_ROUNDING(0x1.0p341, LIBC_NAMESPACE::cbrt(0x1.0p1023));
   EXPECT_FP_EQ_ALL_ROUNDING(-0x1.0p341, LIBC_NAMESPACE::cbrt(-0x1.0p1023));
+  EXPECT_FP_EQ(-0x1.0p-340, LIBC_NAMESPACE::cbrt(-0x1.fffffffffffffp-1021));
+  EXPECT_FP_EQ(2.0, LIBC_NAMESPACE::cbrt(0x1.fffffffffffffp2));
 }
-- 
GitLab


From a8ef0b33a6e5a35f341aec702fa2ceeed444f1fe Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Fri, 25 Oct 2024 19:11:38 -0700
Subject: [PATCH 221/693] [mlir] Fix build (#113750)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

```
mlir/lib/Transforms/Utils/DialectConversion.cpp:2851:28: error: call of overloaded ‘TypeRange(llvm::SmallVector<mlir::Value>&)’ is ambiguous
     assert(TypeRange(result) == resultTypes &&
```
---
 mlir/lib/Transforms/Utils/DialectConversion.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 3d0c81867e0c..9f8a482d6e2d 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -2848,7 +2848,7 @@ SmallVector<Value> TypeConverter::materializeTargetConversion(
         fn(builder, resultTypes, inputs, loc, originalType);
     if (result.empty())
       continue;
-    assert(TypeRange(result) == resultTypes &&
+    assert(TypeRange(ValueRange(result)) == resultTypes &&
            "callback produced incorrect number of values or values with "
            "incorrect types");
     return result;
-- 
GitLab


From 801f3a5400ca2fbcfdeb73fd744e8cce0cebc722 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Fri, 25 Oct 2024 19:29:21 -0700
Subject: [PATCH 222/693] [clang-format] Print the names of unfound files in
 error messages (#113640)

Also fix the return status when `-i` is used with reading from stdin.

Fixes #113631.
---
 clang/test/Format/error-unfound-files.cpp | 5 +++++
 clang/tools/clang-format/ClangFormat.cpp  | 4 ++--
 2 files changed, 7 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/Format/error-unfound-files.cpp

diff --git a/clang/test/Format/error-unfound-files.cpp b/clang/test/Format/error-unfound-files.cpp
new file mode 100644
index 000000000000..1cc57ed064fb
--- /dev/null
+++ b/clang/test/Format/error-unfound-files.cpp
@@ -0,0 +1,5 @@
+// RUN: rm -f a.c b.c
+
+// RUN: not clang-format a.c b.c 2>&1 | FileCheck %s
+// CHECK: a.c:
+// CHECK-NEXT: b.c:
diff --git a/clang/tools/clang-format/ClangFormat.cpp b/clang/tools/clang-format/ClangFormat.cpp
index 5522d05744a2..cc735e487259 100644
--- a/clang/tools/clang-format/ClangFormat.cpp
+++ b/clang/tools/clang-format/ClangFormat.cpp
@@ -410,7 +410,7 @@ static bool format(StringRef FileName, bool ErrorOnIncompleteFormat = false) {
   const bool IsSTDIN = FileName == "-";
   if (!OutputXML && Inplace && IsSTDIN) {
     errs() << "error: cannot use -i when reading from stdin.\n";
-    return false;
+    return true;
   }
   // On Windows, overwriting a file with an open file mapping doesn't work,
   // so read the whole file into memory when formatting in-place.
@@ -419,7 +419,7 @@ static bool format(StringRef FileName, bool ErrorOnIncompleteFormat = false) {
           ? MemoryBuffer::getFileAsStream(FileName)
           : MemoryBuffer::getFileOrSTDIN(FileName, /*IsText=*/true);
   if (std::error_code EC = CodeOrErr.getError()) {
-    errs() << EC.message() << "\n";
+    errs() << FileName << ": " << EC.message() << "\n";
     return true;
   }
   std::unique_ptr<llvm::MemoryBuffer> Code = std::move(CodeOrErr.get());
-- 
GitLab


From ec2da0ca19c393053c7f11d5478ae21c27e54f5c Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 25 Oct 2024 19:37:55 -0700
Subject: [PATCH 223/693] [ADT] Use data() and size() within StringRef (NFC)
 (#113657)

This patch uses data() and size() within StringRef instead of Data and
Length.  This makes it easier to replace Data and Length with
std::string_view in the future, which in turn allows us to forward
most of StringRef functions to the counterparts in std::string_view.
---
 llvm/include/llvm/ADT/StringRef.h | 74 +++++++++++++++---------------
 llvm/lib/Support/StringRef.cpp    | 76 ++++++++++++++++---------------
 2 files changed, 77 insertions(+), 73 deletions(-)

diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h
index 17ab10b9181f..d5f30b88c4c6 100644
--- a/llvm/include/llvm/ADT/StringRef.h
+++ b/llvm/include/llvm/ADT/StringRef.h
@@ -102,7 +102,7 @@ namespace llvm {
 
     /// Construct a string ref from an std::string.
     /*implicit*/ StringRef(const std::string &Str)
-      : Data(Str.data()), Length(Str.length()) {}
+        : Data(Str.data()), Length(Str.length()) {}
 
     /// Construct a string ref from an std::string_view.
     /*implicit*/ constexpr StringRef(std::string_view Str)
@@ -112,9 +112,9 @@ namespace llvm {
     /// @name Iterators
     /// @{
 
-    iterator begin() const { return Data; }
+    iterator begin() const { return data(); }
 
-    iterator end() const { return Data + Length; }
+    iterator end() const { return data() + size(); }
 
     reverse_iterator rbegin() const {
       return std::make_reverse_iterator(end());
@@ -143,7 +143,7 @@ namespace llvm {
     [[nodiscard]] constexpr const char *data() const { return Data; }
 
     /// empty - Check if the string is empty.
-    [[nodiscard]] constexpr bool empty() const { return Length == 0; }
+    [[nodiscard]] constexpr bool empty() const { return size() == 0; }
 
     /// size - Get the string size.
     [[nodiscard]] constexpr size_t size() const { return Length; }
@@ -151,13 +151,13 @@ namespace llvm {
     /// front - Get the first character in the string.
     [[nodiscard]] char front() const {
       assert(!empty());
-      return Data[0];
+      return data()[0];
     }
 
     /// back - Get the last character in the string.
     [[nodiscard]] char back() const {
       assert(!empty());
-      return Data[Length-1];
+      return data()[size() - 1];
     }
 
     // copy - Allocate copy in Allocator and return StringRef to it.
@@ -166,14 +166,14 @@ namespace llvm {
       // Don't request a length 0 copy from the allocator.
       if (empty())
         return StringRef();
-      char *S = A.template Allocate<char>(Length);
+      char *S = A.template Allocate<char>(size());
       std::copy(begin(), end(), S);
-      return StringRef(S, Length);
+      return StringRef(S, size());
     }
 
     /// Check for string equality, ignoring case.
     [[nodiscard]] bool equals_insensitive(StringRef RHS) const {
-      return Length == RHS.Length && compare_insensitive(RHS) == 0;
+      return size() == RHS.size() && compare_insensitive(RHS) == 0;
     }
 
     /// compare - Compare two strings; the result is negative, zero, or positive
@@ -181,13 +181,14 @@ namespace llvm {
     /// the \p RHS.
     [[nodiscard]] int compare(StringRef RHS) const {
       // Check the prefix for a mismatch.
-      if (int Res = compareMemory(Data, RHS.Data, std::min(Length, RHS.Length)))
+      if (int Res =
+              compareMemory(data(), RHS.data(), std::min(size(), RHS.size())))
         return Res < 0 ? -1 : 1;
 
       // Otherwise the prefixes match, so we only need to check the lengths.
-      if (Length == RHS.Length)
+      if (size() == RHS.size())
         return 0;
-      return Length < RHS.Length ? -1 : 1;
+      return size() < RHS.size() ? -1 : 1;
     }
 
     /// Compare two strings, ignoring case.
@@ -225,8 +226,9 @@ namespace llvm {
 
     /// str - Get the contents as an std::string.
     [[nodiscard]] std::string str() const {
-      if (!Data) return std::string();
-      return std::string(Data, Length);
+      if (!data())
+        return std::string();
+      return std::string(data(), size());
     }
 
     /// @}
@@ -234,8 +236,8 @@ namespace llvm {
     /// @{
 
     [[nodiscard]] char operator[](size_t Index) const {
-      assert(Index < Length && "Invalid index!");
-      return Data[Index];
+      assert(Index < size() && "Invalid index!");
+      return data()[Index];
     }
 
     /// Disallow accidental assignment from a temporary std::string.
@@ -260,8 +262,8 @@ namespace llvm {
 
     /// Check if this string starts with the given \p Prefix.
     [[nodiscard]] bool starts_with(StringRef Prefix) const {
-      return Length >= Prefix.Length &&
-             compareMemory(Data, Prefix.Data, Prefix.Length) == 0;
+      return size() >= Prefix.size() &&
+             compareMemory(data(), Prefix.data(), Prefix.size()) == 0;
     }
     [[nodiscard]] bool starts_with(char Prefix) const {
       return !empty() && front() == Prefix;
@@ -272,9 +274,9 @@ namespace llvm {
 
     /// Check if this string ends with the given \p Suffix.
     [[nodiscard]] bool ends_with(StringRef Suffix) const {
-      return Length >= Suffix.Length &&
-             compareMemory(end() - Suffix.Length, Suffix.Data, Suffix.Length) ==
-                 0;
+      return size() >= Suffix.size() &&
+             compareMemory(end() - Suffix.size(), Suffix.data(),
+                           Suffix.size()) == 0;
     }
     [[nodiscard]] bool ends_with(char Suffix) const {
       return !empty() && back() == Suffix;
@@ -342,10 +344,10 @@ namespace llvm {
     /// \returns The index of the last occurrence of \p C, or npos if not
     /// found.
     [[nodiscard]] size_t rfind(char C, size_t From = npos) const {
-      size_t I = std::min(From, Length);
+      size_t I = std::min(From, size());
       while (I) {
         --I;
-        if (Data[I] == C)
+        if (data()[I] == C)
           return I;
       }
       return npos;
@@ -447,8 +449,8 @@ namespace llvm {
     /// Return the number of occurrences of \p C in the string.
     [[nodiscard]] size_t count(char C) const {
       size_t Count = 0;
-      for (size_t I = 0; I != Length; ++I)
-        if (Data[I] == C)
+      for (size_t I = 0; I != size(); ++I)
+        if (data()[I] == C)
           ++Count;
       return Count;
     }
@@ -567,8 +569,8 @@ namespace llvm {
     /// suffix (starting with \p Start) will be returned.
     [[nodiscard]] constexpr StringRef substr(size_t Start,
                                              size_t N = npos) const {
-      Start = std::min(Start, Length);
-      return StringRef(Data + Start, std::min(N, Length - Start));
+      Start = std::min(Start, size());
+      return StringRef(data() + Start, std::min(N, size() - Start));
     }
 
     /// Return a StringRef equal to 'this' but with only the first \p N
@@ -679,9 +681,9 @@ namespace llvm {
     /// will be returned. If this is less than \p Start, an empty string will
     /// be returned.
     [[nodiscard]] StringRef slice(size_t Start, size_t End) const {
-      Start = std::min(Start, Length);
-      End = std::clamp(End, Start, Length);
-      return StringRef(Data + Start, End - Start);
+      Start = std::min(Start, size());
+      End = std::clamp(End, Start, size());
+      return StringRef(data() + Start, End - Start);
     }
 
     /// Split into two substrings around the first occurrence of a separator
@@ -786,25 +788,25 @@ namespace llvm {
     /// Return string with consecutive \p Char characters starting from the
     /// the left removed.
     [[nodiscard]] StringRef ltrim(char Char) const {
-      return drop_front(std::min(Length, find_first_not_of(Char)));
+      return drop_front(std::min(size(), find_first_not_of(Char)));
     }
 
     /// Return string with consecutive characters in \p Chars starting from
     /// the left removed.
     [[nodiscard]] StringRef ltrim(StringRef Chars = " \t\n\v\f\r") const {
-      return drop_front(std::min(Length, find_first_not_of(Chars)));
+      return drop_front(std::min(size(), find_first_not_of(Chars)));
     }
 
     /// Return string with consecutive \p Char characters starting from the
     /// right removed.
     [[nodiscard]] StringRef rtrim(char Char) const {
-      return drop_back(Length - std::min(Length, find_last_not_of(Char) + 1));
+      return drop_back(size() - std::min(size(), find_last_not_of(Char) + 1));
     }
 
     /// Return string with consecutive characters in \p Chars starting from
     /// the right removed.
     [[nodiscard]] StringRef rtrim(StringRef Chars = " \t\n\v\f\r") const {
-      return drop_back(Length - std::min(Length, find_last_not_of(Chars) + 1));
+      return drop_back(size() - std::min(size(), find_last_not_of(Chars) + 1));
     }
 
     /// Return string with consecutive \p Char characters starting from the
@@ -831,9 +833,9 @@ namespace llvm {
         // If there is no carriage return, assume unix
         return "\n";
       }
-      if (Pos + 1 < Length && Data[Pos + 1] == '\n')
+      if (Pos + 1 < size() && data()[Pos + 1] == '\n')
         return "\r\n"; // Windows
-      if (Pos > 0 && Data[Pos - 1] == '\n')
+      if (Pos > 0 && data()[Pos - 1] == '\n')
         return "\n\r"; // You monster!
       return "\r";     // Classic Mac
     }
diff --git a/llvm/lib/Support/StringRef.cpp b/llvm/lib/Support/StringRef.cpp
index 4bbe41688209..4f5fcb4857e8 100644
--- a/llvm/lib/Support/StringRef.cpp
+++ b/llvm/lib/Support/StringRef.cpp
@@ -35,21 +35,23 @@ static int ascii_strncasecmp(const char *LHS, const char *RHS, size_t Length) {
 }
 
 int StringRef::compare_insensitive(StringRef RHS) const {
-  if (int Res = ascii_strncasecmp(Data, RHS.Data, std::min(Length, RHS.Length)))
+  if (int Res =
+          ascii_strncasecmp(data(), RHS.data(), std::min(size(), RHS.size())))
     return Res;
-  if (Length == RHS.Length)
+  if (size() == RHS.size())
     return 0;
-  return Length < RHS.Length ? -1 : 1;
+  return size() < RHS.size() ? -1 : 1;
 }
 
 bool StringRef::starts_with_insensitive(StringRef Prefix) const {
-  return Length >= Prefix.Length &&
-      ascii_strncasecmp(Data, Prefix.Data, Prefix.Length) == 0;
+  return size() >= Prefix.size() &&
+         ascii_strncasecmp(data(), Prefix.data(), Prefix.size()) == 0;
 }
 
 bool StringRef::ends_with_insensitive(StringRef Suffix) const {
-  return Length >= Suffix.Length &&
-      ascii_strncasecmp(end() - Suffix.Length, Suffix.Data, Suffix.Length) == 0;
+  return size() >= Suffix.size() &&
+         ascii_strncasecmp(end() - Suffix.size(), Suffix.data(),
+                           Suffix.size()) == 0;
 }
 
 size_t StringRef::find_insensitive(char C, size_t From) const {
@@ -59,33 +61,33 @@ size_t StringRef::find_insensitive(char C, size_t From) const {
 
 /// compare_numeric - Compare strings, handle embedded numbers.
 int StringRef::compare_numeric(StringRef RHS) const {
-  for (size_t I = 0, E = std::min(Length, RHS.Length); I != E; ++I) {
+  for (size_t I = 0, E = std::min(size(), RHS.size()); I != E; ++I) {
     // Check for sequences of digits.
-    if (isDigit(Data[I]) && isDigit(RHS.Data[I])) {
+    if (isDigit(data()[I]) && isDigit(RHS.data()[I])) {
       // The longer sequence of numbers is considered larger.
       // This doesn't really handle prefixed zeros well.
       size_t J;
       for (J = I + 1; J != E + 1; ++J) {
-        bool ld = J < Length && isDigit(Data[J]);
-        bool rd = J < RHS.Length && isDigit(RHS.Data[J]);
+        bool ld = J < size() && isDigit(data()[J]);
+        bool rd = J < RHS.size() && isDigit(RHS.data()[J]);
         if (ld != rd)
           return rd ? -1 : 1;
         if (!rd)
           break;
       }
       // The two number sequences have the same length (J-I), just memcmp them.
-      if (int Res = compareMemory(Data + I, RHS.Data + I, J - I))
+      if (int Res = compareMemory(data() + I, RHS.data() + I, J - I))
         return Res < 0 ? -1 : 1;
       // Identical number sequences, continue search after the numbers.
       I = J - 1;
       continue;
     }
-    if (Data[I] != RHS.Data[I])
-      return (unsigned char)Data[I] < (unsigned char)RHS.Data[I] ? -1 : 1;
+    if (data()[I] != RHS.data()[I])
+      return (unsigned char)data()[I] < (unsigned char)RHS.data()[I] ? -1 : 1;
   }
-  if (Length == RHS.Length)
+  if (size() == RHS.size())
     return 0;
-  return Length < RHS.Length ? -1 : 1;
+  return size() < RHS.size() ? -1 : 1;
 }
 
 // Compute the edit distance between the two given strings.
@@ -128,11 +130,11 @@ std::string StringRef::upper() const {
 /// \return - The index of the first occurrence of \arg Str, or npos if not
 /// found.
 size_t StringRef::find(StringRef Str, size_t From) const {
-  if (From > Length)
+  if (From > size())
     return npos;
 
-  const char *Start = Data + From;
-  size_t Size = Length - From;
+  const char *Start = data() + From;
+  size_t Size = size() - From;
 
   const char *Needle = Str.data();
   size_t N = Str.size();
@@ -142,7 +144,7 @@ size_t StringRef::find(StringRef Str, size_t From) const {
     return npos;
   if (N == 1) {
     const char *Ptr = (const char *)::memchr(Start, Needle[0], Size);
-    return Ptr == nullptr ? npos : Ptr - Data;
+    return Ptr == nullptr ? npos : Ptr - data();
   }
 
   const char *Stop = Start + (Size - N + 1);
@@ -153,7 +155,7 @@ size_t StringRef::find(StringRef Str, size_t From) const {
     // good enough.
     do {
       if (std::memcmp(Start, Needle, 2) == 0)
-        return Start - Data;
+        return Start - data();
       ++Start;
     } while (Start < Stop);
     return npos;
@@ -163,7 +165,7 @@ size_t StringRef::find(StringRef Str, size_t From) const {
   if (Size < 16 || N > 255) {
     do {
       if (std::memcmp(Start, Needle, N) == 0)
-        return Start - Data;
+        return Start - data();
       ++Start;
     } while (Start < Stop);
     return npos;
@@ -179,7 +181,7 @@ size_t StringRef::find(StringRef Str, size_t From) const {
     uint8_t Last = Start[N - 1];
     if (LLVM_UNLIKELY(Last == (uint8_t)Needle[N - 1]))
       if (std::memcmp(Start, Needle, N - 1) == 0)
-        return Start - Data;
+        return Start - data();
 
     // Otherwise skip the appropriate number of bytes.
     Start += BadCharSkip[Last];
@@ -200,11 +202,11 @@ size_t StringRef::find_insensitive(StringRef Str, size_t From) const {
 }
 
 size_t StringRef::rfind_insensitive(char C, size_t From) const {
-  From = std::min(From, Length);
+  From = std::min(From, size());
   size_t i = From;
   while (i != 0) {
     --i;
-    if (toLower(Data[i]) == toLower(C))
+    if (toLower(data()[i]) == toLower(C))
       return i;
   }
   return npos;
@@ -220,9 +222,9 @@ size_t StringRef::rfind(StringRef Str) const {
 
 size_t StringRef::rfind_insensitive(StringRef Str) const {
   size_t N = Str.size();
-  if (N > Length)
+  if (N > size())
     return npos;
-  for (size_t i = Length - N + 1, e = 0; i != e;) {
+  for (size_t i = size() - N + 1, e = 0; i != e;) {
     --i;
     if (substr(i, N).equals_insensitive(Str))
       return i;
@@ -240,8 +242,8 @@ StringRef::size_type StringRef::find_first_of(StringRef Chars,
   for (char C : Chars)
     CharBits.set((unsigned char)C);
 
-  for (size_type i = std::min(From, Length), e = Length; i != e; ++i)
-    if (CharBits.test((unsigned char)Data[i]))
+  for (size_type i = std::min(From, size()), e = size(); i != e; ++i)
+    if (CharBits.test((unsigned char)data()[i]))
       return i;
   return npos;
 }
@@ -262,8 +264,8 @@ StringRef::size_type StringRef::find_first_not_of(StringRef Chars,
   for (char C : Chars)
     CharBits.set((unsigned char)C);
 
-  for (size_type i = std::min(From, Length), e = Length; i != e; ++i)
-    if (!CharBits.test((unsigned char)Data[i]))
+  for (size_type i = std::min(From, size()), e = size(); i != e; ++i)
+    if (!CharBits.test((unsigned char)data()[i]))
       return i;
   return npos;
 }
@@ -278,8 +280,8 @@ StringRef::size_type StringRef::find_last_of(StringRef Chars,
   for (char C : Chars)
     CharBits.set((unsigned char)C);
 
-  for (size_type i = std::min(From, Length) - 1, e = -1; i != e; --i)
-    if (CharBits.test((unsigned char)Data[i]))
+  for (size_type i = std::min(From, size()) - 1, e = -1; i != e; --i)
+    if (CharBits.test((unsigned char)data()[i]))
       return i;
   return npos;
 }
@@ -287,8 +289,8 @@ StringRef::size_type StringRef::find_last_of(StringRef Chars,
 /// find_last_not_of - Find the last character in the string that is not
 /// \arg C, or npos if not found.
 StringRef::size_type StringRef::find_last_not_of(char C, size_t From) const {
-  for (size_type i = std::min(From, Length) - 1, e = -1; i != e; --i)
-    if (Data[i] != C)
+  for (size_type i = std::min(From, size()) - 1, e = -1; i != e; --i)
+    if (data()[i] != C)
       return i;
   return npos;
 }
@@ -303,8 +305,8 @@ StringRef::size_type StringRef::find_last_not_of(StringRef Chars,
   for (char C : Chars)
     CharBits.set((unsigned char)C);
 
-  for (size_type i = std::min(From, Length) - 1, e = -1; i != e; --i)
-    if (!CharBits.test((unsigned char)Data[i]))
+  for (size_type i = std::min(From, size()) - 1, e = -1; i != e; --i)
+    if (!CharBits.test((unsigned char)data()[i]))
       return i;
   return npos;
 }
-- 
GitLab


From 8f9fc6ce4713b209e6addeb313eb4432fe374c00 Mon Sep 17 00:00:00 2001
From: Longsheng Mou <longshengmou@gmail.com>
Date: Sat, 26 Oct 2024 11:22:08 +0800
Subject: [PATCH 224/693] [mlir][GPU] Add FunctionOpInterface check for
 `OpToFuncCallLowering` (#113449)

This PR adds a `FunctionOpInterface` check in `OpToFuncCallLowering` to
resolve a crash when ops not in function. Fixes #113334.
---
 .../Conversion/GPUCommon/OpToFuncCallLowering.h  |  5 +++++
 .../Conversion/MathToROCDL/math-to-rocdl.mlir    | 16 +++++++++++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
index 1cf8a1acb319..3b94abd88f9e 100644
--- a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
@@ -61,6 +61,11 @@ public:
                                   SourceOp>::value,
                   "expected op with same operand and result types");
 
+    if (!op->template getParentOfType<FunctionOpInterface>()) {
+      return rewriter.notifyMatchFailure(
+          op, "expected op to be within a function region");
+    }
+
     SmallVector<Value, 1> castedOperands;
     for (Value operand : adaptor.getOperands())
       castedOperands.push_back(maybeCast(operand, rewriter));
diff --git a/mlir/test/Conversion/MathToROCDL/math-to-rocdl.mlir b/mlir/test/Conversion/MathToROCDL/math-to-rocdl.mlir
index ddd96bf797e6..e0ea18d41f66 100644
--- a/mlir/test/Conversion/MathToROCDL/math-to-rocdl.mlir
+++ b/mlir/test/Conversion/MathToROCDL/math-to-rocdl.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-math-to-rocdl -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -convert-math-to-rocdl -allow-unregistered-dialect -split-input-file | FileCheck %s
 
 module @test_module {
   // CHECK: llvm.func @__ocml_fmod_f16(f16, f16) -> f16
@@ -481,3 +481,17 @@ module @test_module {
     func.return %resultf16, %resultf32, %resultf64, %resultbf16 : f16, f32, f64, bf16
   }
 }
+
+// -----
+
+// Math operation not inside function
+// Ensure it not crash
+
+module {
+  "test.some_op_with_region"() ({
+  ^bb0(%arg0: f64):
+    // CHECK: math.atan
+    %0 = math.atan %arg0 : f64
+    "test.possible_terminator"() : () -> ()
+  }) : () -> ()
+}
-- 
GitLab


From 5aa741d7ca21645572103d798d0b121f1a741a35 Mon Sep 17 00:00:00 2001
From: Longsheng Mou <longshengmou@gmail.com>
Date: Sat, 26 Oct 2024 11:22:57 +0800
Subject: [PATCH 225/693] [mlir][SPIRVToLLVM] Erase empty `spirv.mlir.loop` in
 `LoopPattern` (#113527)

This PR erases `spirv.mlir.loop` with an empty region in `LoopPattern`,
resolving a crash. Fixes #113404.
---
 mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp           | 6 ++++++
 .../Conversion/SPIRVToLLVM/control-flow-ops-to-llvm.mlir  | 8 ++++++++
 2 files changed, 14 insertions(+)

diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index f28473a108e1..87c0936cee22 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -1148,6 +1148,12 @@ public:
     if (loopOp.getLoopControl() != spirv::LoopControl::None)
       return failure();
 
+    // `spirv.mlir.loop` with empty region is redundant and should be erased.
+    if (loopOp.getBody().empty()) {
+      rewriter.eraseOp(loopOp);
+      return success();
+    }
+
     Location loc = loopOp.getLoc();
 
     // Split the current block after `spirv.mlir.loop`. The remaining ops will
diff --git a/mlir/test/Conversion/SPIRVToLLVM/control-flow-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/control-flow-ops-to-llvm.mlir
index 3557830e779e..756fc5415e20 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/control-flow-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/control-flow-ops-to-llvm.mlir
@@ -86,6 +86,14 @@ spirv.module Logical GLSL450 {
 //===----------------------------------------------------------------------===//
 
 spirv.module Logical GLSL450 {
+  // CHECK-LABEL: @empty_loop
+  spirv.func @empty_loop() "None" {
+    // CHECK: llvm.return
+    spirv.mlir.loop {
+    }
+    spirv.Return
+  }
+
   // CHECK-LABEL: @infinite_loop
   spirv.func @infinite_loop(%count : i32) -> () "None" {
     // CHECK:   llvm.br ^[[BB1:.*]]
-- 
GitLab


From bb00f5b1edd0ed77b7e7b0113dad223505564b18 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Fri, 25 Oct 2024 20:45:44 -0700
Subject: [PATCH 226/693] [mlir][vector] Remove unneeded mask restriction
 (#113742)

These were added when the only mapping was to LLVM.
---
 .../mlir/Dialect/Vector/IR/VectorOps.td       | 52 +++++++++++--------
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp      |  8 +--
 mlir/test/Dialect/Vector/invalid.mlir         |  4 +-
 3 files changed, 35 insertions(+), 29 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index c02b16ea9317..e859270cf9a5 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -1819,17 +1819,17 @@ def Vector_MaskedLoadOp :
   Vector_Op<"maskedload">,
     Arguments<(ins Arg<AnyMemRef, "", [MemRead]>:$base,
                Variadic<Index>:$indices,
-               VectorOfRankAndType<[1], [I1]>:$mask,
-               VectorOfRank<[1]>:$pass_thru)>,
-    Results<(outs VectorOfRank<[1]>:$result)> {
+               VectorOf<[I1]>:$mask,
+               AnyVector:$pass_thru)>,
+    Results<(outs AnyVector:$result)> {
 
   let summary = "loads elements from memory into a vector as defined by a mask vector";
 
   let description = [{
-    The masked load reads elements from memory into a 1-D vector as defined
-    by a base with indices and a 1-D mask vector. When the mask is set, the
+    The masked load reads elements from memory into a vector as defined
+    by a base with indices and a mask vector. When the mask is set, the
     element is read from memory. Otherwise, the corresponding element is taken
-    from a 1-D pass-through vector. Informally the semantics are:
+    from a pass-through vector. Informally the semantics are:
     ```
     result[0] := if mask[0] then base[i + 0] else pass_thru[0]
     result[1] := if mask[1] then base[i + 1] else pass_thru[1]
@@ -1882,14 +1882,14 @@ def Vector_MaskedStoreOp :
   Vector_Op<"maskedstore">,
     Arguments<(ins Arg<AnyMemRef, "", [MemWrite]>:$base,
                Variadic<Index>:$indices,
-               VectorOfRankAndType<[1], [I1]>:$mask,
-               VectorOfRank<[1]>:$valueToStore)> {
+               VectorOf<[I1]>:$mask,
+               AnyVector:$valueToStore)> {
 
   let summary = "stores elements from a vector into memory as defined by a mask vector";
 
   let description = [{
-    The masked store operation writes elements from a 1-D vector into memory
-    as defined by a base with indices and a 1-D mask vector. When the mask is
+    The masked store operation writes elements from a vector into memory
+    as defined by a base with indices and a mask vector. When the mask is
     set, the corresponding element from the vector is written to memory. Otherwise,
     no action is taken for the element. Informally the semantics are:
     ```
@@ -2076,23 +2076,26 @@ def Vector_ExpandLoadOp :
   Vector_Op<"expandload">,
     Arguments<(ins Arg<AnyMemRef, "", [MemRead]>:$base,
                Variadic<Index>:$indices,
-               VectorOfRankAndType<[1], [I1]>:$mask,
-               VectorOfRank<[1]>:$pass_thru)>,
-    Results<(outs VectorOfRank<[1]>:$result)> {
+               VectorOf<[I1]>:$mask,
+               AnyVector:$pass_thru)>,
+    Results<(outs AnyVector:$result)> {
 
   let summary = "reads elements from memory and spreads them into a vector as defined by a mask";
 
   let description = [{
-    The expand load reads elements from memory into a 1-D vector as defined
-    by a base with indices and a 1-D mask vector. When the mask is set, the
-    next element is read from memory. Otherwise, the corresponding element
-    is taken from a 1-D pass-through vector. Informally the semantics are:
+    The expand load reads elements from memory into a vector as defined by a
+    base with indices and a mask vector. Expansion only applies to the innermost
+    dimension. When the mask is set, the next element is read from memory.
+    Otherwise, the corresponding element is taken from a pass-through vector.
+    Informally the semantics are:
+
     ```
     index = i
     result[0] := if mask[0] then base[index++] else pass_thru[0]
     result[1] := if mask[1] then base[index++] else pass_thru[1]
     etc.
     ```
+
     Note that the index increment is done conditionally.
 
     If a mask bit is set and the corresponding index is out-of-bounds for the
@@ -2140,22 +2143,25 @@ def Vector_CompressStoreOp :
   Vector_Op<"compressstore">,
     Arguments<(ins Arg<AnyMemRef, "", [MemWrite]>:$base,
                Variadic<Index>:$indices,
-               VectorOfRankAndType<[1], [I1]>:$mask,
-               VectorOfRank<[1]>:$valueToStore)> {
+               VectorOf<[I1]>:$mask,
+               AnyVector:$valueToStore)> {
 
   let summary = "writes elements selectively from a vector as defined by a mask";
 
   let description = [{
-    The compress store operation writes elements from a 1-D vector into memory
-    as defined by a base with indices and a 1-D mask vector. When the mask is
-    set, the corresponding element from the vector is written next to memory.
-    Otherwise, no action is taken for the element. Informally the semantics are:
+    The compress store operation writes elements from a vector into memory as
+    defined by a base with indices and a mask vector. Compression only applies
+    to the innermost dimension. When the mask is set, the corresponding element
+    from the vector is written next to memory.  Otherwise, no action is taken
+    for the element. Informally the semantics are:
+
     ```
     index = i
     if (mask[0]) base[index++] = value[0]
     if (mask[1]) base[index++] = value[1]
     etc.
     ```
+
     Note that the index increment is done conditionally.
 
     If a mask bit is set and the corresponding index is out-of-bounds for the
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index a2abe1619454..d71a236f62f4 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -4977,8 +4977,8 @@ LogicalResult MaskedLoadOp::verify() {
     return emitOpError("base and result element type should match");
   if (llvm::size(getIndices()) != memType.getRank())
     return emitOpError("requires ") << memType.getRank() << " indices";
-  if (resVType.getDimSize(0) != maskVType.getDimSize(0))
-    return emitOpError("expected result dim to match mask dim");
+  if (resVType.getShape() != maskVType.getShape())
+    return emitOpError("expected result shape to match mask shape");
   if (resVType != passVType)
     return emitOpError("expected pass_thru of same type as result type");
   return success();
@@ -5030,8 +5030,8 @@ LogicalResult MaskedStoreOp::verify() {
     return emitOpError("base and valueToStore element type should match");
   if (llvm::size(getIndices()) != memType.getRank())
     return emitOpError("requires ") << memType.getRank() << " indices";
-  if (valueVType.getDimSize(0) != maskVType.getDimSize(0))
-    return emitOpError("expected valueToStore dim to match mask dim");
+  if (valueVType.getShape() != maskVType.getShape())
+    return emitOpError("expected valueToStore shape to match mask shape");
   return success();
 }
 
diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir
index 36d04bb77e3b..5b0fb537b356 100644
--- a/mlir/test/Dialect/Vector/invalid.mlir
+++ b/mlir/test/Dialect/Vector/invalid.mlir
@@ -1356,7 +1356,7 @@ func.func @maskedload_base_type_mismatch(%base: memref<?xf64>, %mask: vector<16x
 
 func.func @maskedload_dim_mask_mismatch(%base: memref<?xf32>, %mask: vector<15xi1>, %pass: vector<16xf32>) {
   %c0 = arith.constant 0 : index
-  // expected-error@+1 {{'vector.maskedload' op expected result dim to match mask dim}}
+  // expected-error@+1 {{'vector.maskedload' op expected result shape to match mask shape}}
   %0 = vector.maskedload %base[%c0], %mask, %pass : memref<?xf32>, vector<15xi1>, vector<16xf32> into vector<16xf32>
 }
 
@@ -1387,7 +1387,7 @@ func.func @maskedstore_base_type_mismatch(%base: memref<?xf64>, %mask: vector<16
 
 func.func @maskedstore_dim_mask_mismatch(%base: memref<?xf32>, %mask: vector<15xi1>, %value: vector<16xf32>) {
   %c0 = arith.constant 0 : index
-  // expected-error@+1 {{'vector.maskedstore' op expected valueToStore dim to match mask dim}}
+  // expected-error@+1 {{'vector.maskedstore' op expected valueToStore shape to match mask shape}}
   vector.maskedstore %base[%c0], %mask, %value : memref<?xf32>, vector<15xi1>, vector<16xf32>
 }
 
-- 
GitLab


From 13d6233e77982f2a596922a79365373e1466a968 Mon Sep 17 00:00:00 2001
From: Durgadoss R <durgadossr@nvidia.com>
Date: Sat, 26 Oct 2024 11:15:50 +0530
Subject: [PATCH 227/693] [MLIR][NVGPU] Fix nvgpu_arrive syntax in
 matmulBuilder.py (#113713)

This patch updates the syntax for nvgpu_arrive Op
in matmulBuilder.py. This fixes the compilation
error for this test.

For the warp-specialized matmul_kernel implementation,
removing the WaitGroupSyncOp (after the mma-main-loop)
fixes the hang observed.

With these two fixes, the test compiles and
executes successfully on an sm90a machine.

Signed-off-by: Durgadoss R <durgadossr@nvidia.com>
---
 .../GPU/CUDA/sm90/python/tools/matmulBuilder.py       | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/mlir/test/Integration/GPU/CUDA/sm90/python/tools/matmulBuilder.py b/mlir/test/Integration/GPU/CUDA/sm90/python/tools/matmulBuilder.py
index 75f0dc947e06..5394d4a32725 100644
--- a/mlir/test/Integration/GPU/CUDA/sm90/python/tools/matmulBuilder.py
+++ b/mlir/test/Integration/GPU/CUDA/sm90/python/tools/matmulBuilder.py
@@ -568,9 +568,7 @@ def generate_matmul_ws(
                                 barId,
                                 predicate=consumerPrimaryThread,
                             )
-                            nvgpu.mbarrier_arrive(
-                                ir.Type.parse("!nvgpu.mbarrier.token"), mbarDONE, barId
-                            )
+                            nvgpu.mbarrier_arrive(mbarDONE, barId)
                             debug_print(
                                 "[cons] iv={}  | mbarDONE[{}] arrive [done]",
                                 iv,
@@ -589,14 +587,9 @@ def generate_matmul_ws(
                         # Step 6.3.5. Yield
                         scf.yield_([new_acc, phaseParity])
 
-                    # Step 6.3. Wait All WGMMA
-                    nvvm.WgmmaWaitGroupSyncOp(0)
-
                     with ir.InsertionPoint(scf.IfOp(consumerPrimaryThread).then_block):
                         barId = c((K // BLOCK_K) % num_stages)
-                        nvgpu.mbarrier_arrive(
-                            ir.Type.parse("!nvgpu.mbarrier.token"), mbarDONE, barId
-                        )
+                        nvgpu.mbarrier_arrive(mbarDONE, barId)
                         scf.yield_([])
 
                     # Step 6.4. Epilogue (registers --> shared memory)
-- 
GitLab


From 3fc0d94ce57de2d0841e77c8fda7feef2923c4e0 Mon Sep 17 00:00:00 2001
From: "A. Jiang" <de34@live.cn>
Date: Sat, 26 Oct 2024 13:46:59 +0800
Subject: [PATCH 228/693] [libc++][test] Make macro detection more friendly to
 MSVC (#113633)

MSVC STL's test suite is a bit nervous about replacing non-macro-defined
identifiers with `0` (see also
https://learn.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-4-c4668?view=msvc-170).

On MSVC (and MS-compatible mode of other compilers), `long double` has
the same format (IEEE-754 binary64) as `double`, so it should be OK to
define `TEST_LONG_DOUBLE_IS_DOUBLE` when `_MSC_VER` is defined. Such
detection should be performed first.
---
 libcxx/test/support/test_macros.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcxx/test/support/test_macros.h b/libcxx/test/support/test_macros.h
index 5ef14e54dae2..1b6473b623c5 100644
--- a/libcxx/test/support/test_macros.h
+++ b/libcxx/test/support/test_macros.h
@@ -511,7 +511,7 @@ inline Tp const& DoNotOptimize(Tp const& value) {
 #  define TEST_CONSTEXPR_OPERATOR_NEW
 #endif
 
-#if __SIZEOF_LONG_DOUBLE__ == __SIZEOF_DOUBLE__
+#if defined(_MSC_VER) || __SIZEOF_LONG_DOUBLE__ == __SIZEOF_DOUBLE__
 #  define TEST_LONG_DOUBLE_IS_DOUBLE
 #endif
 
-- 
GitLab


From 7c9cf0c6f09115c2d948f16946aa3b36bf483f34 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman@google.com>
Date: Sat, 26 Oct 2024 08:15:34 +0000
Subject: [PATCH 229/693] [SHT_LLVM_BB_ADDR_MAP][AsmPrinter] Emit error on bad
 option combinatons

This patch makes it so that specifying all or none for -pgo-analysis-map
along with an explicit option causes an error as this set of options
does not really make sense.
---
 llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp             | 10 ++++++++++
 .../X86/basic-block-address-map-pgo-features.ll        |  4 ++++
 2 files changed, 14 insertions(+)

diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 2d444f2f970a..4ea71c9bd4ad 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -1371,6 +1371,16 @@ static uint32_t getBBAddrMapMetadata(const MachineBasicBlock &MBB) {
 
 static llvm::object::BBAddrMap::Features
 getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges) {
+  // Ensure that the user has not passed in additional options while also
+  // specifying all or none.
+  if ((PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::None) ||
+       PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::All)) &&
+      popcount(PgoAnalysisMapFeatures.getBits()) != 1) {
+    MF.getFunction().getContext().emitError(
+        "-pgo-anaylsis-map can accept only all or none with no additional "
+        "values.");
+  }
+
   bool NoFeatures = PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::None);
   bool AllFeatures = PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::All);
   bool FuncEntryCountEnabled =
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll b/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
index 1c3db738a947..fca5aa046b03 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
@@ -11,6 +11,10 @@
 ; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -pgo-analysis-map=bb-freq | FileCheck %s --check-prefixes=CHECK,PGO-BBF,BBF-ONLY
 ; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -pgo-analysis-map=br-prob | FileCheck %s --check-prefixes=CHECK,PGO-BRP,BRP-ONLY
 
+;; Verify that we emit an error if we try and specify values in addition to all/none
+; RUN: not llc < %s -mtriple=x86_64 -basic-block-address-map -pgo-analysis-map=all,bb-freq
+; RUN: not llc < %s -mtriple=x86_64 -basic-block-address-map -pgo-analysis-map=none,bb-freq
+
 
 define void @_Z3bazb(i1 zeroext, i1 zeroext) personality ptr @__gxx_personality_v0 !prof !0  {
   br i1 %0, label %3, label %8, !prof !1
-- 
GitLab


From ef9629c64ac05d426436a0b9fd1ae9a058805368 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Sat, 26 Oct 2024 12:34:45 +0300
Subject: [PATCH 230/693] Revert "Add extendhfxf2 into compiler rt (#111099)"

This reverts commit 5f7bad07b9d5b6c5cfa8c16a4e62cf1e128725be.

These tests fail to build in multiple configurations, see
https://github.com/llvm/llvm-project/pull/111099.
---
 compiler-rt/lib/builtins/CMakeLists.txt       |  1 -
 compiler-rt/lib/builtins/extendhfxf2.c        | 18 -----
 .../lib/builtins/macho_embedded/common.txt    |  1 -
 .../test/builtins/Unit/extendhfxf2_test.c     | 71 -------------------
 .../compiler-rt/lib/builtins/BUILD.gn         |  1 -
 5 files changed, 92 deletions(-)
 delete mode 100644 compiler-rt/lib/builtins/extendhfxf2.c
 delete mode 100644 compiler-rt/test/builtins/Unit/extendhfxf2_test.c

diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index 97a9e508d37a..9a0a50ee7003 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -104,7 +104,6 @@ set(GENERIC_SOURCES
   divti3.c
   extendsfdf2.c
   extendhfsf2.c
-  extendhfxf2.c
   ffsdi2.c
   ffssi2.c
   ffsti2.c
diff --git a/compiler-rt/lib/builtins/extendhfxf2.c b/compiler-rt/lib/builtins/extendhfxf2.c
deleted file mode 100644
index 7425859f79f7..000000000000
--- a/compiler-rt/lib/builtins/extendhfxf2.c
+++ /dev/null
@@ -1,18 +0,0 @@
-//===-- lib/extendhfxf2.c - half -> long double conversion --------*- C -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#define SRC_HALF
-#define DST_DOUBLE
-#include "fp_extend_impl.inc"
-
-// Use a forwarding definition and noinline to implement a poor man's alias,
-// as there isn't a good cross-platform way of defining one.
-// Long double are expected to be as precise as double.
-COMPILER_RT_ABI NOINLINE long double __extendhfxf2(src_t a) {
-  return (long double)__extendXfYf2__(a);
-}
diff --git a/compiler-rt/lib/builtins/macho_embedded/common.txt b/compiler-rt/lib/builtins/macho_embedded/common.txt
index fa99bc239e68..819109768f52 100644
--- a/compiler-rt/lib/builtins/macho_embedded/common.txt
+++ b/compiler-rt/lib/builtins/macho_embedded/common.txt
@@ -60,7 +60,6 @@ divsf3
 divsi3
 extendsfdf2
 extendhfsf2
-extendhfxf2
 ffssi2
 fixdfsi
 fixsfsi
diff --git a/compiler-rt/test/builtins/Unit/extendhfxf2_test.c b/compiler-rt/test/builtins/Unit/extendhfxf2_test.c
deleted file mode 100644
index 9972b024ab41..000000000000
--- a/compiler-rt/test/builtins/Unit/extendhfxf2_test.c
+++ /dev/null
@@ -1,71 +0,0 @@
-// RUN: %clang_builtins %s %librt -o %t && %run %t
-// REQUIRES: librt_has_extendhfxf2
-
-#include <limits.h>
-#include <math.h> // for isnan, isinf
-#include <stdio.h>
-
-#if __LDBL_MANT_DIG__ >= 64 && defined(COMPILER_RT_HAS_FLOAT16)
-
-long double __extendhfxf2(_Float16 f);
-
-int test_extendhfxf2(_Float16 a, long double expected) {
-  long double x = __extendhfxf2(a);
-  __uint16_t *b = (void *)&a;
-  int ret = !((isnan(x) && isnan(expected)) || x == expected);
-  if (ret) {
-    printf("error in test__extendhfxf2(%#.4x) = %.20Lf, "
-           "expected %.20Lf\n",
-           *b, x, expected);
-  }
-  return ret;
-}
-
-char assumption_1[sizeof(_Float16) * CHAR_BIT == 16] = {0};
-
-int main() {
-  // Small positive value
-  if (test_extendhfxf2(0.09997558593750000000f, 0.09997558593750000000L))
-    return 1;
-
-  // Small negative value
-  if (test_extendhfxf2(-0.09997558593750000000f, -0.09997558593750000000L))
-    return 1;
-
-  // Zero
-  if (test_extendhfxf2(0.0f, 0.0L))
-    return 1;
-
-  // Smallest positive non-zero value
-  if (test_extendhfxf2(0x1p-16f, 0x1p-16L))
-    return 1;
-
-  // Smallest negative non-zero value
-  if (test_extendhfxf2(-0x1p-16f, -0x1p-16L))
-    return 1;
-
-  // Positive infinity
-  if (test_extendhfxf2(__builtin_huge_valf16(), __builtin_huge_valf64x()))
-    return 1;
-
-  // Negative infinity
-  if (test_extendhfxf2(-__builtin_huge_valf16(),
-                       (long double)-__builtin_huge_valf64x()))
-    return 1;
-
-  // NaN
-  if (test_extendhfxf2(__builtin_nanf16(""),
-                       (long double)__builtin_nanf64x("")))
-    return 1;
-
-  return 0;
-}
-
-#else
-
-int main() {
-  printf("skipped\n");
-  return 0;
-}
-
-#endif
diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
index efbf01960bf9..8904aed28229 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
@@ -126,7 +126,6 @@ static_library("builtins") {
     "divsi3.c",
     "divti3.c",
     "extendhfsf2.c",
-    "extendhfxf2.c"
     "extendsfdf2.c",
     "ffsdi2.c",
     "ffssi2.c",
-- 
GitLab


From 69ead949d08ff0bb8cbbf4f7143aaa6687830f6b Mon Sep 17 00:00:00 2001
From: Thomas Fransham <tfransham@gmail.com>
Date: Sat, 26 Oct 2024 11:15:37 +0100
Subject: [PATCH 231/693] [llvm] Enable building Analysis plugins on windows
 (#112303)

Enable building InlineAdvisorPlugin and InlineOrderPlugin on windows for
shared library builds.

This is part of the work to enable LLVM_BUILD_LLVM_DYLIB and LLVM
plugins on window.
---
 llvm/unittests/Analysis/InlineAdvisorPlugin/CMakeLists.txt | 2 +-
 llvm/unittests/Analysis/InlineOrderPlugin/CMakeLists.txt   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/unittests/Analysis/InlineAdvisorPlugin/CMakeLists.txt b/llvm/unittests/Analysis/InlineAdvisorPlugin/CMakeLists.txt
index 22cd0cb1d9a0..deabf110f2e4 100644
--- a/llvm/unittests/Analysis/InlineAdvisorPlugin/CMakeLists.txt
+++ b/llvm/unittests/Analysis/InlineAdvisorPlugin/CMakeLists.txt
@@ -2,7 +2,7 @@
 # libraries, but expects them to exist in the process loading the plugin. This 
 # doesn't work with DLLs on Windows (where a shared library can't have undefined
 # references), so just skip this testcase on Windows.
-if (NOT WIN32 AND NOT CYGWIN)
+if ((NOT WIN32 OR LLVM_BUILD_LLVM_DYLIB) AND NOT CYGWIN)
   unset(LLVM_LINK_COMPONENTS)
   add_llvm_library(InlineAdvisorPlugin MODULE BUILDTREE_ONLY
     InlineAdvisorPlugin.cpp
diff --git a/llvm/unittests/Analysis/InlineOrderPlugin/CMakeLists.txt b/llvm/unittests/Analysis/InlineOrderPlugin/CMakeLists.txt
index cc470a934426..0b37cebe3da6 100644
--- a/llvm/unittests/Analysis/InlineOrderPlugin/CMakeLists.txt
+++ b/llvm/unittests/Analysis/InlineOrderPlugin/CMakeLists.txt
@@ -2,7 +2,7 @@
 # libraries, but expects them to exist in the process loading the plugin. This 
 # doesn't work with DLLs on Windows (where a shared library can't have undefined
 # references), so just skip this testcase on Windows.
-if (NOT WIN32 AND NOT CYGWIN)
+if ((NOT WIN32 OR LLVM_BUILD_LLVM_DYLIB) AND NOT CYGWIN)
   unset(LLVM_LINK_COMPONENTS)
   add_llvm_library(InlineOrderPlugin MODULE BUILDTREE_ONLY
     InlineOrderPlugin.cpp
-- 
GitLab


From 4102625380823e58d7b13f01b5bd979a29bce19e Mon Sep 17 00:00:00 2001
From: davidtrevelyan <davidtrevelyan@users.noreply.github.com>
Date: Sat, 26 Oct 2024 13:06:11 +0100
Subject: [PATCH 232/693] [rtsan][llvm][NFC] Rename sanitize_realtime_unsafe
 attr to sanitize_realtime_blocking (#113155)

# What

This PR renames the newly-introduced llvm attribute
`sanitize_realtime_unsafe` to `sanitize_realtime_blocking`. Likewise,
sibling variables such as `SanitizeRealtimeUnsafe` are renamed to
`SanitizeRealtimeBlocking` respectively. There are no other functional
changes.


# Why?

- There are a number of problems that can cause a function to be
real-time "unsafe",
- we wish to communicate what problems rtsan detects and *why* they're
unsafe, and
- a generic "unsafe" attribute is, in our opinion, too broad a net -
which may lead to future implementations that need extra contextual
information passed through them in order to communicate meaningful
reasons to users.
- We want to avoid this situation and make the runtime library boundary
API/ABI as simple as possible, and
- we believe that restricting the scope of attributes to names like
`sanitize_realtime_blocking` is an effective means of doing so.

We also feel that the symmetry between `[[clang::blocking]]` and
`sanitize_realtime_blocking` is easier to follow as a developer.

# Concerns

- I'm aware that the LLVM attribute `sanitize_realtime_unsafe` has been
part of the tree for a few weeks now (introduced here:
https://github.com/llvm/llvm-project/pull/106754). Given that it hasn't
been released in version 20 yet, am I correct in considering this to not
be a breaking change?
---
 clang/lib/CodeGen/CodeGenFunction.cpp                       | 2 +-
 clang/test/CodeGen/rtsan_attribute_inserted.c               | 2 +-
 clang/test/CodeGen/rtsan_no_attribute_sanitizer_disabled.c  | 2 +-
 compiler-rt/lib/rtsan/tests/rtsan_test_functional.cpp       | 4 ++--
 llvm/docs/LangRef.rst                                       | 2 +-
 llvm/include/llvm/Bitcode/LLVMBitCodes.h                    | 2 +-
 llvm/include/llvm/IR/Attributes.td                          | 4 ++--
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp                   | 4 ++--
 llvm/lib/Bitcode/Writer/BitcodeWriter.cpp                   | 4 ++--
 llvm/lib/IR/Verifier.cpp                                    | 4 ++--
 llvm/lib/Transforms/Instrumentation/RealtimeSanitizer.cpp   | 6 +++---
 llvm/lib/Transforms/Utils/CodeExtractor.cpp                 | 2 +-
 llvm/test/Bitcode/attributes.ll                             | 4 ++--
 llvm/test/Bitcode/compatibility.ll                          | 6 +++---
 .../{rtsan_unsafe.ll => rtsan_blocking.ll}                  | 4 ++--
 llvm/test/Verifier/rtsan-attrs.ll                           | 4 ++--
 16 files changed, 28 insertions(+), 28 deletions(-)
 rename llvm/test/Instrumentation/RealtimeSanitizer/{rtsan_unsafe.ll => rtsan_blocking.ll} (87%)

diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 573ced0857d5..6ead45793742 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -852,7 +852,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
         if (Fe.Effect.kind() == FunctionEffect::Kind::NonBlocking)
           Fn->addFnAttr(llvm::Attribute::SanitizeRealtime);
         else if (Fe.Effect.kind() == FunctionEffect::Kind::Blocking)
-          Fn->addFnAttr(llvm::Attribute::SanitizeRealtimeUnsafe);
+          Fn->addFnAttr(llvm::Attribute::SanitizeRealtimeBlocking);
       }
 
   // Apply fuzzing attribute to the function.
diff --git a/clang/test/CodeGen/rtsan_attribute_inserted.c b/clang/test/CodeGen/rtsan_attribute_inserted.c
index b21ecb6b6b06..cebfe43c8123 100644
--- a/clang/test/CodeGen/rtsan_attribute_inserted.c
+++ b/clang/test/CodeGen/rtsan_attribute_inserted.c
@@ -8,4 +8,4 @@ float process(float *a) [[clang::nonblocking]] { return *a; }
 int spinlock(int *a) [[clang::blocking]] { return *a; }
 // CHECK: @spinlock{{.*}} #1 {
 // CHECK: attributes #1 = {
-// CHECK-SAME: {{.*sanitize_realtime_unsafe .*}}
+// CHECK-SAME: {{.*sanitize_realtime_blocking .*}}
diff --git a/clang/test/CodeGen/rtsan_no_attribute_sanitizer_disabled.c b/clang/test/CodeGen/rtsan_no_attribute_sanitizer_disabled.c
index 0f43007c5e4c..86305080c94a 100644
--- a/clang/test/CodeGen/rtsan_no_attribute_sanitizer_disabled.c
+++ b/clang/test/CodeGen/rtsan_no_attribute_sanitizer_disabled.c
@@ -5,4 +5,4 @@ int spinlock(int *a) [[clang::blocking]] { return *a; }
 
 // Without the -fsanitize=realtime flag, we shouldn't attach the attributes.
 // CHECK-NOT: {{.*sanitize_realtime .*}}
-// CHECK-NOT: {{.*sanitize_realtime_unsafe .*}}
+// CHECK-NOT: {{.*sanitize_realtime_blocking .*}}
diff --git a/compiler-rt/lib/rtsan/tests/rtsan_test_functional.cpp b/compiler-rt/lib/rtsan/tests/rtsan_test_functional.cpp
index 9e455f0326a5..ed9ee4ded7b0 100644
--- a/compiler-rt/lib/rtsan/tests/rtsan_test_functional.cpp
+++ b/compiler-rt/lib/rtsan/tests/rtsan_test_functional.cpp
@@ -204,11 +204,11 @@ TEST(TestRtsan, ThrowingAnExceptionDiesWhenRealtime) {
 
 TEST(TestRtsan, DoesNotDieIfTurnedOff) {
   std::mutex mutex;
-  auto RealtimeUnsafeFunc = [&]() {
+  auto RealtimeBlockingFunc = [&]() {
     __rtsan_disable();
     mutex.lock();
     mutex.unlock();
     __rtsan_enable();
   };
-  RealtimeInvoke(RealtimeUnsafeFunc);
+  RealtimeInvoke(RealtimeBlockingFunc);
 }
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index b83675c6ed97..f9ec33da1b65 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -2334,7 +2334,7 @@ example:
     This attribute indicates that RealtimeSanitizer checks
     (realtime safety analysis - no allocations, syscalls or exceptions) are enabled
     for this function.
-``sanitize_realtime_unsafe``
+``sanitize_realtime_blocking``
     This attribute indicates that RealtimeSanitizer should error immediately
     if the attributed function is called during invocation of a function
     attributed with ``sanitize_realtime``.
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index 08574cc356e5..41a6447356c2 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -768,7 +768,7 @@ enum AttributeKindCodes {
   ATTR_KIND_INITIALIZES = 94,
   ATTR_KIND_HYBRID_PATCHABLE = 95,
   ATTR_KIND_SANITIZE_REALTIME = 96,
-  ATTR_KIND_SANITIZE_REALTIME_UNSAFE = 97,
+  ATTR_KIND_SANITIZE_REALTIME_BLOCKING = 97,
   ATTR_KIND_CORO_ELIDE_SAFE = 98,
   ATTR_KIND_NO_EXT = 99,
   ATTR_KIND_NO_DIVERGENCE_SOURCE = 100,
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index b6d36a5f7ae4..49f4527bde66 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -334,7 +334,7 @@ def SanitizeRealtime : EnumAttr<"sanitize_realtime", IntersectPreserve, [FnAttr]
 
 /// RealtimeSanitizer should error if a real-time unsafe function is invoked
 /// during a real-time sanitized function (see `sanitize_realtime`).
-def SanitizeRealtimeUnsafe : EnumAttr<"sanitize_realtime_unsafe", IntersectPreserve, [FnAttr]>;
+def SanitizeRealtimeBlocking : EnumAttr<"sanitize_realtime_blocking", IntersectPreserve, [FnAttr]>;
 
 /// Speculative Load Hardening is enabled.
 ///
@@ -430,7 +430,7 @@ def : CompatRule<"isEqual<SanitizeHWAddressAttr>">;
 def : CompatRule<"isEqual<SanitizeMemTagAttr>">;
 def : CompatRule<"isEqual<SanitizeNumericalStabilityAttr>">;
 def : CompatRule<"isEqual<SanitizeRealtimeAttr>">;
-def : CompatRule<"isEqual<SanitizeRealtimeUnsafeAttr>">;
+def : CompatRule<"isEqual<SanitizeRealtimeBlockingAttr>">;
 def : CompatRule<"isEqual<SafeStackAttr>">;
 def : CompatRule<"isEqual<ShadowCallStackAttr>">;
 def : CompatRule<"isEqual<UseSampleProfileAttr>">;
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 4aea059551de..446c98c8cecd 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -2165,8 +2165,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
     return Attribute::SanitizeNumericalStability;
   case bitc::ATTR_KIND_SANITIZE_REALTIME:
     return Attribute::SanitizeRealtime;
-  case bitc::ATTR_KIND_SANITIZE_REALTIME_UNSAFE:
-    return Attribute::SanitizeRealtimeUnsafe;
+  case bitc::ATTR_KIND_SANITIZE_REALTIME_BLOCKING:
+    return Attribute::SanitizeRealtimeBlocking;
   case bitc::ATTR_KIND_SPECULATIVE_LOAD_HARDENING:
     return Attribute::SpeculativeLoadHardening;
   case bitc::ATTR_KIND_SWIFT_ERROR:
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index d9002149fba5..ee9cc4b6e0c0 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -853,8 +853,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_SANITIZE_NUMERICAL_STABILITY;
   case Attribute::SanitizeRealtime:
     return bitc::ATTR_KIND_SANITIZE_REALTIME;
-  case Attribute::SanitizeRealtimeUnsafe:
-    return bitc::ATTR_KIND_SANITIZE_REALTIME_UNSAFE;
+  case Attribute::SanitizeRealtimeBlocking:
+    return bitc::ATTR_KIND_SANITIZE_REALTIME_BLOCKING;
   case Attribute::SpeculativeLoadHardening:
     return bitc::ATTR_KIND_SPECULATIVE_LOAD_HARDENING;
   case Attribute::SwiftError:
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 60e65392218d..ee807ca13787 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -2235,9 +2235,9 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
   }
 
   Check(!(Attrs.hasFnAttr(Attribute::SanitizeRealtime) &&
-          Attrs.hasFnAttr(Attribute::SanitizeRealtimeUnsafe)),
+          Attrs.hasFnAttr(Attribute::SanitizeRealtimeBlocking)),
         "Attributes "
-        "'sanitize_realtime and sanitize_realtime_unsafe' are incompatible!",
+        "'sanitize_realtime and sanitize_realtime_blocking' are incompatible!",
         V);
 
   if (Attrs.hasFnAttr(Attribute::OptimizeForDebugging)) {
diff --git a/llvm/lib/Transforms/Instrumentation/RealtimeSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/RealtimeSanitizer.cpp
index c4cb72ab2e4d..88cb04695217 100644
--- a/llvm/lib/Transforms/Instrumentation/RealtimeSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/RealtimeSanitizer.cpp
@@ -69,7 +69,7 @@ static PreservedAnalyses runSanitizeRealtime(Function &Fn) {
   return rtsanPreservedCFGAnalyses();
 }
 
-static PreservedAnalyses runSanitizeRealtimeUnsafe(Function &Fn) {
+static PreservedAnalyses runSanitizeRealtimeBlocking(Function &Fn) {
   IRBuilder<> Builder(&Fn.front().front());
   Value *Name = Builder.CreateGlobalString(demangle(Fn.getName()));
   insertCallAtFunctionEntryPoint(Fn, "__rtsan_notify_blocking_call", {Name});
@@ -84,8 +84,8 @@ PreservedAnalyses RealtimeSanitizerPass::run(Function &Fn,
   if (Fn.hasFnAttribute(Attribute::SanitizeRealtime))
     return runSanitizeRealtime(Fn);
 
-  if (Fn.hasFnAttribute(Attribute::SanitizeRealtimeUnsafe))
-    return runSanitizeRealtimeUnsafe(Fn);
+  if (Fn.hasFnAttribute(Attribute::SanitizeRealtimeBlocking))
+    return runSanitizeRealtimeBlocking(Fn);
 
   return PreservedAnalyses::all();
 }
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 15b26a38cc28..ed4ad15e5ab6 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -953,7 +953,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
       case Attribute::SanitizeHWAddress:
       case Attribute::SanitizeMemTag:
       case Attribute::SanitizeRealtime:
-      case Attribute::SanitizeRealtimeUnsafe:
+      case Attribute::SanitizeRealtimeBlocking:
       case Attribute::SpeculativeLoadHardening:
       case Attribute::StackProtect:
       case Attribute::StackProtectReq:
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index 737f49aa86a7..492de663884d 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -512,7 +512,7 @@ define void @f92() sanitize_realtime
 }
 
 ; CHECK: define void @f93() #54
-define void @f93() sanitize_realtime_unsafe {
+define void @f93() sanitize_realtime_blocking {
         ret void;
 }
 
@@ -616,7 +616,7 @@ define void @initializes(ptr initializes((-4, 0), (4, 8)) %a) {
 ; CHECK: attributes #51 = { uwtable(sync) }
 ; CHECK: attributes #52 = { nosanitize_bounds }
 ; CHECK: attributes #53 = { sanitize_realtime }
-; CHECK: attributes #54 = { sanitize_realtime_unsafe }
+; CHECK: attributes #54 = { sanitize_realtime_blocking }
 ; CHECK: attributes [[FNRETTHUNKEXTERN]] = { fn_ret_thunk_extern }
 ; CHECK: attributes [[SKIPPROFILE]] = { skipprofile }
 ; CHECK: attributes [[OPTDEBUG]] = { optdebug }
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index 280c3a99d753..a849789da536 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -2048,8 +2048,8 @@ declare void @f.sanitize_numerical_stability() sanitize_numerical_stability
 declare void @f.sanitize_realtime() sanitize_realtime
 ; CHECK: declare void @f.sanitize_realtime() #52
 
-declare void @f.sanitize_realtime_unsafe() sanitize_realtime_unsafe
-; CHECK: declare void @f.sanitize_realtime_unsafe() #53
+declare void @f.sanitize_realtime_blocking() sanitize_realtime_blocking
+; CHECK: declare void @f.sanitize_realtime_blocking() #53
 
 ; CHECK: declare nofpclass(snan) float @nofpclass_snan(float nofpclass(snan))
 declare nofpclass(snan) float @nofpclass_snan(float nofpclass(snan))
@@ -2183,7 +2183,7 @@ define float @nofpclass_callsites(float %arg, { float } %arg1) {
 ; CHECK: attributes #50 = { allockind("alloc,uninitialized") }
 ; CHECK: attributes #51 = { sanitize_numerical_stability }
 ; CHECK: attributes #52 = { sanitize_realtime }
-; CHECK: attributes #53 = { sanitize_realtime_unsafe }
+; CHECK: attributes #53 = { sanitize_realtime_blocking }
 ; CHECK: attributes #54 = { builtin }
 
 ;; Metadata
diff --git a/llvm/test/Instrumentation/RealtimeSanitizer/rtsan_unsafe.ll b/llvm/test/Instrumentation/RealtimeSanitizer/rtsan_blocking.ll
similarity index 87%
rename from llvm/test/Instrumentation/RealtimeSanitizer/rtsan_unsafe.ll
rename to llvm/test/Instrumentation/RealtimeSanitizer/rtsan_blocking.ll
index 5abf5de30448..80eb28c3923c 100644
--- a/llvm/test/Instrumentation/RealtimeSanitizer/rtsan_unsafe.ll
+++ b/llvm/test/Instrumentation/RealtimeSanitizer/rtsan_blocking.ll
@@ -25,7 +25,7 @@ define noundef i32 @main() #2 {
   ret i32 0
 }
 
-attributes #0 = { mustprogress noinline sanitize_realtime_unsafe optnone ssp uwtable(sync) }
+attributes #0 = { mustprogress noinline sanitize_realtime_blocking optnone ssp uwtable(sync) }
 ;.
-; CHECK: attributes #[[ATTR0]] = { mustprogress noinline optnone sanitize_realtime_unsafe ssp uwtable(sync) }
+; CHECK: attributes #[[ATTR0]] = { mustprogress noinline optnone sanitize_realtime_blocking ssp uwtable(sync) }
 ;.
diff --git a/llvm/test/Verifier/rtsan-attrs.ll b/llvm/test/Verifier/rtsan-attrs.ll
index fcc44d8d63c1..c813266b434f 100644
--- a/llvm/test/Verifier/rtsan-attrs.ll
+++ b/llvm/test/Verifier/rtsan-attrs.ll
@@ -1,9 +1,9 @@
 ; RUN: not llvm-as -disable-output %s 2>&1 | FileCheck %s
 
-; CHECK: Attributes 'sanitize_realtime and sanitize_realtime_unsafe' are incompatible!
+; CHECK: Attributes 'sanitize_realtime and sanitize_realtime_blocking' are incompatible!
 ; CHECK-NEXT: ptr @sanitize_unsafe
 define void @sanitize_unsafe() #0 {
   ret void
 }
 
-attributes #0 = { sanitize_realtime sanitize_realtime_unsafe }
+attributes #0 = { sanitize_realtime sanitize_realtime_blocking }
-- 
GitLab


From 0cf7aaf30067c4be2886a8c9127a27dcbfd63b92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Sat, 26 Oct 2024 05:54:04 -0700
Subject: [PATCH 233/693] [MLIR][Vector] Update
 Transfer{Read|Write}DropUnitDimsPattern patterns (#112394)

Updates `TransferWriteDropUnitDimsPattern` and
`TransferReadDropUnitDimsPattern` to inherit from
`MaskableOpRewritePattern` so that masked versions of
xfer_read/xfer_write Ops are also supported:

```mlir
    %v = vector.mask %mask {
      vector.transfer_read %arg[%c0, %c0, %c0, %c0], %cst :
        memref<1x1x3x2xi8, strided<[6, 6, 2, 1], offset: ?>>, vector<3x2xi8>
    } : vector<3x2xi1> -> vector<3x2xi8>
```
---
 .../Transforms/VectorTransferOpTransforms.cpp | 67 +++++++++++----
 mlir/test/Dialect/Vector/invalid.mlir         |  9 +++
 ...ctor-transfer-drop-unit-dims-patterns.mlir | 81 ++++++++++++++++++-
 3 files changed, 139 insertions(+), 18 deletions(-)

diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index e05c801121ff..3a30382114c8 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -354,11 +354,13 @@ namespace {
 /// inserting a memref.subview dropping those unit dims. The vector shapes are
 /// also reduced accordingly.
 class TransferReadDropUnitDimsPattern
-    : public OpRewritePattern<vector::TransferReadOp> {
-  using OpRewritePattern::OpRewritePattern;
+    : public vector::MaskableOpRewritePattern<vector::TransferReadOp> {
+  using MaskableOpRewritePattern::MaskableOpRewritePattern;
 
-  LogicalResult matchAndRewrite(vector::TransferReadOp transferReadOp,
-                                PatternRewriter &rewriter) const override {
+  FailureOr<Value>
+  matchAndRewriteMaskableOp(vector::TransferReadOp transferReadOp,
+                            vector::MaskingOpInterface maskingOp,
+                            PatternRewriter &rewriter) const override {
     auto loc = transferReadOp.getLoc();
     Value vector = transferReadOp.getVector();
     VectorType vectorType = cast<VectorType>(vector.getType());
@@ -376,6 +378,10 @@ class TransferReadDropUnitDimsPattern
     int reducedRank = getReducedRank(sourceType.getShape());
     if (reducedRank == sourceType.getRank())
       return failure();
+    // TODO: Extend vector.mask to support 0-d vectors. In the meantime, bail
+    // out.
+    if (reducedRank == 0 && maskingOp)
+      return failure();
     // Check if the reduced vector shape matches the reduced source shape.
     // Otherwise, this case is not supported yet.
     VectorType reducedVectorType = trimNonScalableUnitDims(vectorType);
@@ -406,15 +412,23 @@ class TransferReadDropUnitDimsPattern
     SmallVector<Value> zeros(reducedRank, c0);
     auto identityMap = rewriter.getMultiDimIdentityMap(reducedRank);
     SmallVector<bool> inBounds(reducedVectorType.getRank(), true);
-    auto newTransferReadOp = rewriter.create<vector::TransferReadOp>(
+    Operation *newTransferReadOp = rewriter.create<vector::TransferReadOp>(
         loc, reducedVectorType, reducedShapeSource, zeros, identityMap,
         transferReadOp.getPadding(), maskOp,
         rewriter.getBoolArrayAttr(inBounds));
+
+    if (maskingOp) {
+      auto shapeCastMask = rewriter.createOrFold<vector::ShapeCastOp>(
+          loc, reducedVectorType.cloneWith(std::nullopt, rewriter.getI1Type()),
+          maskingOp.getMask());
+      newTransferReadOp = mlir::vector::maskOperation(
+          rewriter, newTransferReadOp, shapeCastMask);
+    }
+
     auto shapeCast = rewriter.createOrFold<vector::ShapeCastOp>(
-        loc, vectorType, newTransferReadOp);
-    rewriter.replaceOp(transferReadOp, shapeCast);
+        loc, vectorType, newTransferReadOp->getResults()[0]);
 
-    return success();
+    return shapeCast;
   }
 };
 
@@ -422,11 +436,13 @@ class TransferReadDropUnitDimsPattern
 /// has unit dims, by inserting a `memref.subview` dropping those unit dims. The
 /// vector shapes are also reduced accordingly.
 class TransferWriteDropUnitDimsPattern
-    : public OpRewritePattern<vector::TransferWriteOp> {
-  using OpRewritePattern::OpRewritePattern;
+    : public vector::MaskableOpRewritePattern<vector::TransferWriteOp> {
+  using MaskableOpRewritePattern::MaskableOpRewritePattern;
 
-  LogicalResult matchAndRewrite(vector::TransferWriteOp transferWriteOp,
-                                PatternRewriter &rewriter) const override {
+  FailureOr<Value>
+  matchAndRewriteMaskableOp(vector::TransferWriteOp transferWriteOp,
+                            vector::MaskingOpInterface maskingOp,
+                            PatternRewriter &rewriter) const override {
     auto loc = transferWriteOp.getLoc();
     Value vector = transferWriteOp.getVector();
     VectorType vectorType = cast<VectorType>(vector.getType());
@@ -444,6 +460,10 @@ class TransferWriteDropUnitDimsPattern
     int reducedRank = getReducedRank(sourceType.getShape());
     if (reducedRank == sourceType.getRank())
       return failure();
+    // TODO: Extend vector.mask to support 0-d vectors. In the meantime, bail
+    // out.
+    if (reducedRank == 0 && maskingOp)
+      return failure();
     // Check if the reduced vector shape matches the reduced destination shape.
     // Otherwise, this case is not supported yet.
     VectorType reducedVectorType = trimNonScalableUnitDims(vectorType);
@@ -474,13 +494,26 @@ class TransferWriteDropUnitDimsPattern
     SmallVector<Value> zeros(reducedRank, c0);
     auto identityMap = rewriter.getMultiDimIdentityMap(reducedRank);
     SmallVector<bool> inBounds(reducedVectorType.getRank(), true);
-    auto shapeCast = rewriter.createOrFold<vector::ShapeCastOp>(
+    auto shapeCastSrc = rewriter.createOrFold<vector::ShapeCastOp>(
         loc, reducedVectorType, vector);
-    rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
-        transferWriteOp, Type(), shapeCast, reducedShapeSource, zeros,
-        identityMap, maskOp, rewriter.getBoolArrayAttr(inBounds));
+    Operation *newXferWrite = rewriter.create<vector::TransferWriteOp>(
+        loc, Type(), shapeCastSrc, reducedShapeSource, zeros, identityMap,
+        maskOp, rewriter.getBoolArrayAttr(inBounds));
+
+    if (maskingOp) {
+      auto shapeCastMask = rewriter.createOrFold<vector::ShapeCastOp>(
+          loc, reducedVectorType.cloneWith(std::nullopt, rewriter.getI1Type()),
+          maskingOp.getMask());
+      newXferWrite =
+          mlir::vector::maskOperation(rewriter, newXferWrite, shapeCastMask);
+    }
 
-    return success();
+    if (transferWriteOp.hasPureTensorSemantics())
+      return newXferWrite->getResults()[0];
+
+    // With Memref semantics, there's no return value. Use empty value to signal
+    // success.
+    return Value();
   }
 };
 
diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir
index 5b0fb537b356..56039d04549a 100644
--- a/mlir/test/Dialect/Vector/invalid.mlir
+++ b/mlir/test/Dialect/Vector/invalid.mlir
@@ -1717,6 +1717,15 @@ func.func @vector_mask_shape_mismatch(%a: vector<8xi32>, %m0: vector<16xi1>) ->
 
 // -----
 
+func.func @vector_mask_passthru_type_mismatch(%t0: tensor<f32>, %m0: vector<i1>) -> vector<f32> {
+  %ft0 = arith.constant 0.0 : f32
+  // expected-error@+1 {{'vector.mask' op operand #0 must be vector of 1-bit signless integer values, but got 'vector<i1>'}}
+  %0 = vector.mask %m0 { vector.transfer_read %t0[], %ft0 : tensor<f32>, vector<f32> } : vector<i1> -> vector<f32>
+  return %0 : vector<f32>
+}
+
+// -----
+
 // expected-note@+1 {{prior use here}}
 func.func @vector_mask_passthru_type_mismatch(%t0: tensor<?xf32>, %idx: index, %m0: vector<16xi1>, %pt0: vector<16xi32>) -> vector<16xf32> {
   %ft0 = arith.constant 0.0 : f32
diff --git a/mlir/test/Dialect/Vector/vector-transfer-drop-unit-dims-patterns.mlir b/mlir/test/Dialect/Vector/vector-transfer-drop-unit-dims-patterns.mlir
index e9d12b044e2c..8234351302f6 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-drop-unit-dims-patterns.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-drop-unit-dims-patterns.mlir
@@ -1,5 +1,9 @@
 // RUN: mlir-opt %s --transform-interpreter | FileCheck %s
 
+//-----------------------------------------------------------------------------
+// [Patterns: TransferWriteDropUnitDimsPattern, TransferReadeDropUnitDimsPattern]
+//-----------------------------------------------------------------------------
+
 func.func @transfer_read_rank_reducing(
       %arg : memref<1x1x3x2xi8, strided<[6, 6, 2, 1], offset: ?>>) -> vector<3x2xi8> {
     %c0 = arith.constant 0 : index
@@ -14,7 +18,29 @@ func.func @transfer_read_rank_reducing(
 //  CHECK-SAME:     memref<1x1x3x2xi8, {{.*}}> to memref<3x2xi8, {{.*}}>
 //       CHECK:   vector.transfer_read %[[SUBVIEW]]
 
-func.func @transfer_write_rank_reducing(%arg : memref<1x1x3x2xi8, strided<[6, 6, 2, 1], offset: ?>>, %vec : vector<3x2xi8>) {
+func.func @transfer_read_rank_reducing_masked(
+      %arg : memref<1x1x3x2xi8, strided<[6, 6, 2, 1], offset: ?>>,
+      %mask: vector<3x2xi1>) -> vector<3x2xi8> {
+    %c0 = arith.constant 0 : index
+    %cst = arith.constant 0 : i8
+    %v = vector.mask %mask {
+      vector.transfer_read %arg[%c0, %c0, %c0, %c0], %cst :
+        memref<1x1x3x2xi8, strided<[6, 6, 2, 1], offset: ?>>, vector<3x2xi8>
+    } : vector<3x2xi1> -> vector<3x2xi8>
+    return %v : vector<3x2xi8>
+}
+// CHECK-LABEL: func @transfer_read_rank_reducing_masked
+//  CHECK-SAME:     %[[ARG:.+]]: memref<1x1x3x2xi8
+//  CHECK-SAME:     %[[MASK:.+]]: vector<3x2xi1>
+//       CHECK:   %[[SUBVIEW:.+]] = memref.subview %[[ARG]][0, 0, 0, 0] [1, 1, 3, 2] [1, 1, 1, 1]
+//  CHECK-SAME:     memref<1x1x3x2xi8, {{.*}}> to memref<3x2xi8, {{.*}}>
+//       CHECK:   vector.mask %[[MASK]]
+//  CHECK-SAME:  vector.transfer_read %[[SUBVIEW]]
+
+func.func @transfer_write_rank_reducing(
+      %arg : memref<1x1x3x2xi8, strided<[6, 6, 2, 1], offset: ?>>,
+      %vec : vector<3x2xi8>) {
+
     %c0 = arith.constant 0 : index
     vector.transfer_write %vec, %arg [%c0, %c0, %c0, %c0] :
       vector<3x2xi8>, memref<1x1x3x2xi8, strided<[6, 6, 2, 1], offset: ?>>
@@ -26,6 +52,26 @@ func.func @transfer_write_rank_reducing(%arg : memref<1x1x3x2xi8, strided<[6, 6,
 //  CHECK-SAME:     memref<1x1x3x2xi8, {{.*}}> to memref<3x2xi8, {{.*}}>
 //       CHECK:   vector.transfer_write %{{.*}}, %[[SUBVIEW]]
 
+func.func @transfer_write_rank_reducing_masked(
+      %arg : memref<1x1x3x2xi8, strided<[6, 6, 2, 1], offset: ?>>,
+      %vec : vector<3x2xi8>,
+      %mask: vector<3x2xi1>) {
+    %c0 = arith.constant 0 : index
+    vector.mask %mask {
+      vector.transfer_write %vec, %arg [%c0, %c0, %c0, %c0] :
+        vector<3x2xi8>, memref<1x1x3x2xi8, strided<[6, 6, 2, 1], offset: ?>>
+    } : vector<3x2xi1>
+    return
+}
+// CHECK-LABEL: func @transfer_write_rank_reducing_masked
+//  CHECK-SAME:     %[[ARG:.+]]: memref<1x1x3x2xi8
+//  CHECK-SAME:     %[[VEC:.+]]: vector<3x2xi8>
+//  CHECK-SAME:     %[[MASK:.+]]: vector<3x2xi1>
+//       CHECK:   %[[SUBVIEW:.+]] = memref.subview %[[ARG]][0, 0, 0, 0] [1, 1, 3, 2] [1, 1, 1, 1]
+//  CHECK-SAME:     memref<1x1x3x2xi8, {{.*}}> to memref<3x2xi8, {{.*}}>
+//       CHECK:   vector.mask %[[MASK]]
+//  CHECK-SAME:   vector.transfer_write %{{.*}}, %[[SUBVIEW]]
+
 func.func @transfer_read_and_vector_rank_reducing(
       %arg : memref<1x1x3x2x1xf32>) -> vector<3x2x1xf32> {
     %c0 = arith.constant 0 : index
@@ -68,6 +114,22 @@ func.func @transfer_read_and_vector_rank_reducing_to_0d(
 //       CHECK:   %[[READ:.+]] = vector.transfer_read %[[SUBVIEW]]{{.*}} : memref<f32>, vector<f32>
 //       CHECK:   vector.shape_cast %[[READ]] : vector<f32> to vector<1x1x1xf32>
 
+func.func @transfer_read_and_vector_rank_reducing_to_0d_masked(
+      %arg : memref<1x1x1x1x1xf32>,
+      %mask: vector<1x1x1xi1>) -> vector<1x1x1xf32> {
+
+    %c0 = arith.constant 0 : index
+    %cst = arith.constant 0.0 : f32
+    %v = vector.mask %mask {
+      vector.transfer_read %arg[%c0, %c0, %c0, %c0, %c0], %cst
+        : memref<1x1x1x1x1xf32>, vector<1x1x1xf32>
+    } : vector<1x1x1xi1> -> vector<1x1x1xf32>
+    return %v : vector<1x1x1xf32>
+}
+// CHECK-LABEL: func @transfer_read_and_vector_rank_reducing_to_0d_masked
+//   CHECK-NOT:   vector.shape_cast
+//   CHECK-NOT:   memref.subview
+
 func.func @transfer_write_and_vector_rank_reducing_to_0d(
       %arg : memref<1x1x1x1x1xf32>,
       %vec : vector<1x1x1xf32>) {
@@ -82,6 +144,23 @@ func.func @transfer_write_and_vector_rank_reducing_to_0d(
 //       CHECK:   %[[SHCAST:.+]] = vector.shape_cast %[[VECTOR]] : vector<1x1x1xf32> to vector<f32>
 //       CHECK:   vector.transfer_write %[[SHCAST]], %[[SUBVIEW]]{{.*}} : vector<f32>, memref<f32>
 
+func.func @transfer_write_and_vector_rank_reducing_to_0d_masked(
+      %arg : memref<1x1x1x1x1xf32>,
+      %vec : vector<1x1x1xf32>,
+      %mask: vector<1x1x1xi1>) {
+
+    %c0 = arith.constant 0 : index
+    %cst = arith.constant 0.0 : f32
+    vector.mask %mask {
+      vector.transfer_write %vec, %arg[%c0, %c0, %c0, %c0, %c0] :
+        vector<1x1x1xf32>, memref<1x1x1x1x1xf32>
+    } : vector<1x1x1xi1>
+    return
+}
+// CHECK-LABEL: func @transfer_write_and_vector_rank_reducing_to_0d_masked
+//   CHECK-NOT:   vector.shape_cast
+//   CHECK-NOT:   memref.subview
+
 func.func @transfer_read_dynamic_rank_reducing(
       %arg : memref<?x1xi8, strided<[?, ?], offset: ?>>) -> vector<[16]x1xi8> {
     %c0 = arith.constant 0 : index
-- 
GitLab


From e146c1867e8decfd423034f63a3a863733e03f04 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Sat, 26 Oct 2024 09:03:04 -0400
Subject: [PATCH 234/693] [libc++] Split Apple and Freebsd locale support into
 their own headers (#113737)

For now these headers don't provide much benefit, however as we refactor
the locale base API they will provide a location to specify the
localization interface on these platforms.
---
 libcxx/include/CMakeLists.txt                     |  2 ++
 libcxx/include/__locale_dir/locale_base_api.h     |  6 ++++--
 .../include/__locale_dir/locale_base_api/apple.h  | 15 +++++++++++++++
 .../__locale_dir/locale_base_api/freebsd.h        | 15 +++++++++++++++
 libcxx/include/module.modulemap                   |  2 ++
 5 files changed, 38 insertions(+), 2 deletions(-)
 create mode 100644 libcxx/include/__locale_dir/locale_base_api/apple.h
 create mode 100644 libcxx/include/__locale_dir/locale_base_api/freebsd.h

diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 506ed721d084..bb152af82cad 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -492,8 +492,10 @@ set(files
   __locale
   __locale_dir/locale_base_api.h
   __locale_dir/locale_base_api/android.h
+  __locale_dir/locale_base_api/apple.h
   __locale_dir/locale_base_api/bsd_locale_defaults.h
   __locale_dir/locale_base_api/bsd_locale_fallbacks.h
+  __locale_dir/locale_base_api/freebsd.h
   __locale_dir/locale_base_api/fuchsia.h
   __locale_dir/locale_base_api/ibm.h
   __locale_dir/locale_base_api/musl.h
diff --git a/libcxx/include/__locale_dir/locale_base_api.h b/libcxx/include/__locale_dir/locale_base_api.h
index eab7fa8bf62f..b6c80255b4d1 100644
--- a/libcxx/include/__locale_dir/locale_base_api.h
+++ b/libcxx/include/__locale_dir/locale_base_api.h
@@ -21,8 +21,10 @@
 #  include <__locale_dir/locale_base_api/fuchsia.h>
 #elif defined(__wasi__) || defined(_LIBCPP_HAS_MUSL_LIBC)
 #  include <__locale_dir/locale_base_api/musl.h>
-#elif defined(__APPLE__) || defined(__FreeBSD__)
-#  include <xlocale.h>
+#elif defined(__APPLE__)
+#  include <__locale_dir/locale_base_api/apple.h>
+#elif defined(__FreeBSD__)
+#  include <__locale_dir/locale_base_api/freebsd.h>
 #endif
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__locale_dir/locale_base_api/apple.h b/libcxx/include/__locale_dir/locale_base_api/apple.h
new file mode 100644
index 000000000000..ec5986c3a19f
--- /dev/null
+++ b/libcxx/include/__locale_dir/locale_base_api/apple.h
@@ -0,0 +1,15 @@
+// -*- C++ -*-
+//===-----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_APPLE_H
+#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_APPLE_H
+
+#include <xlocale.h>
+
+#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_APPLE_H
diff --git a/libcxx/include/__locale_dir/locale_base_api/freebsd.h b/libcxx/include/__locale_dir/locale_base_api/freebsd.h
new file mode 100644
index 000000000000..45ecf1977471
--- /dev/null
+++ b/libcxx/include/__locale_dir/locale_base_api/freebsd.h
@@ -0,0 +1,15 @@
+// -*- C++ -*-
+//===-----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_FREEBSD_H
+#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_FREEBSD_H
+
+#include <xlocale.h>
+
+#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_FREEBSD_H
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index f92e8bf5fc9a..05d08cfbd7cd 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1446,8 +1446,10 @@ module std [system] {
     header "__locale_dir/locale_guard.h"
     module locale_base_api {
       textual header "__locale_dir/locale_base_api/android.h"
+      textual header "__locale_dir/locale_base_api/apple.h"
       textual header "__locale_dir/locale_base_api/bsd_locale_defaults.h"
       textual header "__locale_dir/locale_base_api/bsd_locale_fallbacks.h"
+      textual header "__locale_dir/locale_base_api/freebsd.h"
       textual header "__locale_dir/locale_base_api/fuchsia.h"
       textual header "__locale_dir/locale_base_api/ibm.h"
       textual header "__locale_dir/locale_base_api/musl.h"
-- 
GitLab


From 0df70c28d25aedbedbb4c89f2c03a0e10ec0e8f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alex=20R=C3=B8nne=20Petersen?= <alex@alexrp.com>
Date: Sat, 26 Oct 2024 16:53:01 +0200
Subject: [PATCH 235/693] [llvm][SystemZ] Remove some leftover code from
 #106014. NFC. (#113761)

Pointed out by @redstar here:
https://github.com/llvm/llvm-project/pull/106014/files#r1816845388
---
 llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
index 8fbd05eab5f6..f2fa7e7c9f9f 100644
--- a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
@@ -1450,11 +1450,6 @@ void SystemZXPLINKFrameLowering::inlineStackProbe(
 }
 
 bool SystemZXPLINKFrameLowering::hasFPImpl(const MachineFunction &MF) const {
-  // Naked functions have no stack frame pushed, so we don't have a frame
-  // pointer.
-  if (MF.getFunction().hasFnAttribute(Attribute::Naked))
-    return false;
-
   return (MF.getFrameInfo().hasVarSizedObjects());
 }
 
-- 
GitLab


From 0f4b3c409fbd756d826c89d5539d9ea22bcc56aa Mon Sep 17 00:00:00 2001
From: lntue <lntue@google.com>
Date: Sat, 26 Oct 2024 10:55:20 -0400
Subject: [PATCH 236/693] [libc][math] Add tests and fix some issues with
 FTZ/DAZ modes. (#113744)

---
 libc/src/math/generic/atan2.cpp           |  4 +--
 libc/src/math/generic/cbrt.cpp            |  5 +--
 libc/src/math/generic/cbrtf.cpp           |  5 +--
 libc/src/math/generic/log.cpp             |  2 +-
 libc/src/math/generic/log10.cpp           |  2 +-
 libc/src/math/generic/log10f.cpp          |  2 +-
 libc/src/math/generic/log1p.cpp           |  6 ++--
 libc/src/math/generic/log2.cpp            |  2 +-
 libc/src/math/generic/log2f.cpp           |  2 +-
 libc/src/math/generic/logf.cpp            |  2 +-
 libc/src/math/generic/pow.cpp             | 16 +++++-----
 libc/src/math/generic/powf.cpp            | 25 ++++++++-------
 libc/src/math/generic/sin.cpp             |  2 +-
 libc/src/math/generic/tan.cpp             |  2 +-
 libc/test/src/math/smoke/HypotTest.h      |  4 +--
 libc/test/src/math/smoke/acosf_test.cpp   | 24 +++++++++++++++
 libc/test/src/math/smoke/acoshf_test.cpp  | 24 +++++++++++++++
 libc/test/src/math/smoke/asinf_test.cpp   | 24 +++++++++++++++
 libc/test/src/math/smoke/asinhf_test.cpp  | 24 +++++++++++++++
 libc/test/src/math/smoke/atan2_test.cpp   | 37 +++++++++++++++++++++++
 libc/test/src/math/smoke/atanf_test.cpp   | 24 +++++++++++++++
 libc/test/src/math/smoke/atanhf_test.cpp  | 24 +++++++++++++++
 libc/test/src/math/smoke/cbrt_test.cpp    | 27 +++++++++++++++++
 libc/test/src/math/smoke/cbrtf_test.cpp   | 27 +++++++++++++++++
 libc/test/src/math/smoke/cos_test.cpp     | 27 +++++++++++++++++
 libc/test/src/math/smoke/cosf_test.cpp    | 27 +++++++++++++++++
 libc/test/src/math/smoke/coshf_test.cpp   | 27 +++++++++++++++++
 libc/test/src/math/smoke/cospif_test.cpp  | 27 +++++++++++++++++
 libc/test/src/math/smoke/erff_test.cpp    | 27 +++++++++++++++++
 libc/test/src/math/smoke/exp10_test.cpp   | 27 +++++++++++++++++
 libc/test/src/math/smoke/exp10f_test.cpp  | 27 +++++++++++++++++
 libc/test/src/math/smoke/exp2_test.cpp    | 27 +++++++++++++++++
 libc/test/src/math/smoke/exp2f_test.cpp   | 27 +++++++++++++++++
 libc/test/src/math/smoke/exp2m1f_test.cpp | 27 +++++++++++++++++
 libc/test/src/math/smoke/exp_test.cpp     | 27 +++++++++++++++++
 libc/test/src/math/smoke/expf_test.cpp    | 27 +++++++++++++++++
 libc/test/src/math/smoke/expm1_test.cpp   | 27 +++++++++++++++++
 libc/test/src/math/smoke/expm1f_test.cpp  | 27 +++++++++++++++++
 libc/test/src/math/smoke/hypotf_test.cpp  | 34 +++++++++++++++++++++
 libc/test/src/math/smoke/log10_test.cpp   | 26 ++++++++++++++++
 libc/test/src/math/smoke/log10f_test.cpp  | 26 ++++++++++++++++
 libc/test/src/math/smoke/log1p_test.cpp   | 24 +++++++++++++++
 libc/test/src/math/smoke/log1pf_test.cpp  | 24 +++++++++++++++
 libc/test/src/math/smoke/log2_test.cpp    | 26 ++++++++++++++++
 libc/test/src/math/smoke/log2f_test.cpp   | 25 +++++++++++++++
 libc/test/src/math/smoke/log_test.cpp     | 26 ++++++++++++++++
 libc/test/src/math/smoke/logf_test.cpp    | 25 +++++++++++++++
 libc/test/src/math/smoke/pow_test.cpp     | 27 +++++++++++++++++
 libc/test/src/math/smoke/powf_test.cpp    | 27 +++++++++++++++++
 libc/test/src/math/smoke/sin_test.cpp     | 27 +++++++++++++++++
 libc/test/src/math/smoke/sinf_test.cpp    | 27 +++++++++++++++++
 libc/test/src/math/smoke/sinhf_test.cpp   | 27 +++++++++++++++++
 libc/test/src/math/smoke/sinpif_test.cpp  | 27 +++++++++++++++++
 libc/test/src/math/smoke/tan_test.cpp     | 27 +++++++++++++++++
 libc/test/src/math/smoke/tanf_test.cpp    | 27 +++++++++++++++++
 libc/test/src/math/smoke/tanhf_test.cpp   | 27 +++++++++++++++++
 56 files changed, 1135 insertions(+), 38 deletions(-)

diff --git a/libc/src/math/generic/atan2.cpp b/libc/src/math/generic/atan2.cpp
index c39deebca4d4..1b16e15d29d0 100644
--- a/libc/src/math/generic/atan2.cpp
+++ b/libc/src/math/generic/atan2.cpp
@@ -230,8 +230,8 @@ LLVM_LIBC_FUNCTION(double, atan2, (double y, double x)) {
   if (LIBC_UNLIKELY(max_exp > 0x7ffU - 128U || min_exp < 128U)) {
     if (x_bits.is_nan() || y_bits.is_nan())
       return FPBits::quiet_nan().get_val();
-    unsigned x_except = x_abs == 0 ? 0 : (FPBits(x_abs).is_inf() ? 2 : 1);
-    unsigned y_except = y_abs == 0 ? 0 : (FPBits(y_abs).is_inf() ? 2 : 1);
+    unsigned x_except = x == 0.0 ? 0 : (FPBits(x_abs).is_inf() ? 2 : 1);
+    unsigned y_except = y == 0.0 ? 0 : (FPBits(y_abs).is_inf() ? 2 : 1);
 
     // Exceptional cases:
     //   EXCEPT[y_except][x_except][x_is_neg]
diff --git a/libc/src/math/generic/cbrt.cpp b/libc/src/math/generic/cbrt.cpp
index 4fa24c54fdee..ee7d69b2c211 100644
--- a/libc/src/math/generic/cbrt.cpp
+++ b/libc/src/math/generic/cbrt.cpp
@@ -151,9 +151,10 @@ LLVM_LIBC_FUNCTION(double, cbrt, (double x)) {
 
   if (LIBC_UNLIKELY(x_abs < FPBits::min_normal().uintval() ||
                     x_abs >= FPBits::inf().uintval())) {
-    if (x_abs == 0 || x_abs >= FPBits::inf().uintval())
+    if (x == 0.0 || x_abs >= FPBits::inf().uintval())
       // x is 0, Inf, or NaN.
-      return x;
+      // Make sure it works for FTZ/DAZ modes.
+      return static_cast<double>(x + x);
 
     // x is non-zero denormal number.
     // Normalize x.
diff --git a/libc/src/math/generic/cbrtf.cpp b/libc/src/math/generic/cbrtf.cpp
index 313961bf356b..0abbf6e87942 100644
--- a/libc/src/math/generic/cbrtf.cpp
+++ b/libc/src/math/generic/cbrtf.cpp
@@ -93,9 +93,10 @@ LLVM_LIBC_FUNCTION(float, cbrtf, (float x)) {
   uint32_t x_abs = x_bits.uintval() & 0x7fff'ffff;
   uint32_t sign_bit = (x_bits.uintval() >> 31) << DoubleBits::EXP_LEN;
 
-  if (LIBC_UNLIKELY(x_abs == 0 || x_abs >= 0x7f80'0000)) {
+  if (LIBC_UNLIKELY(x == 0.0f || x_abs >= 0x7f80'0000)) {
     // x is 0, Inf, or NaN.
-    return x;
+    // Make sure it works for FTZ/DAZ modes.
+    return x + x;
   }
 
   double xd = static_cast<double>(x);
diff --git a/libc/src/math/generic/log.cpp b/libc/src/math/generic/log.cpp
index 57c70e31730b..4302c64c8aba 100644
--- a/libc/src/math/generic/log.cpp
+++ b/libc/src/math/generic/log.cpp
@@ -749,7 +749,7 @@ LLVM_LIBC_FUNCTION(double, log, (double x)) {
 
   if (LIBC_UNLIKELY(xbits.uintval() < FPBits_t::min_normal().uintval() ||
                     xbits.uintval() > FPBits_t::max_normal().uintval())) {
-    if (xbits.is_zero()) {
+    if (x == 0.0) {
       // return -Inf and raise FE_DIVBYZERO.
       fputil::set_errno_if_required(ERANGE);
       fputil::raise_except_if_required(FE_DIVBYZERO);
diff --git a/libc/src/math/generic/log10.cpp b/libc/src/math/generic/log10.cpp
index b99b22b024fe..7df57ef85b81 100644
--- a/libc/src/math/generic/log10.cpp
+++ b/libc/src/math/generic/log10.cpp
@@ -751,7 +751,7 @@ LLVM_LIBC_FUNCTION(double, log10, (double x)) {
 
   if (LIBC_UNLIKELY(xbits.uintval() < FPBits_t::min_normal().uintval() ||
                     xbits.uintval() > FPBits_t::max_normal().uintval())) {
-    if (xbits.is_zero()) {
+    if (x == 0.0) {
       // return -Inf and raise FE_DIVBYZERO.
       fputil::set_errno_if_required(ERANGE);
       fputil::raise_except_if_required(FE_DIVBYZERO);
diff --git a/libc/src/math/generic/log10f.cpp b/libc/src/math/generic/log10f.cpp
index f7dd85cc08bf..c635fa4ef9b6 100644
--- a/libc/src/math/generic/log10f.cpp
+++ b/libc/src/math/generic/log10f.cpp
@@ -164,7 +164,7 @@ LLVM_LIBC_FUNCTION(float, log10f, (float x)) {
 
   if (LIBC_UNLIKELY(x_u < FPBits::min_normal().uintval() ||
                     x_u > FPBits::max_normal().uintval())) {
-    if (xbits.is_zero()) {
+    if (x == 0.0f) {
       // Return -inf and raise FE_DIVBYZERO
       fputil::set_errno_if_required(ERANGE);
       fputil::raise_except_if_required(FE_DIVBYZERO);
diff --git a/libc/src/math/generic/log1p.cpp b/libc/src/math/generic/log1p.cpp
index f301a5aba3a5..43eb8a924aef 100644
--- a/libc/src/math/generic/log1p.cpp
+++ b/libc/src/math/generic/log1p.cpp
@@ -927,8 +927,8 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) {
       //   log(1 + x) = nextafter(x, -inf) for FE_DOWNWARD, or
       //                                       FE_TOWARDZERO and x > 0,
       //              = x                  otherwise.
-      if (LIBC_UNLIKELY(xbits.is_zero()))
-        return x;
+      if (x == 0.0)
+        return x + x; // Handle FTZ/DAZ correctly.
 
       volatile float tp = 1.0f;
       volatile float tn = -1.0f;
@@ -943,7 +943,7 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) {
         return FPBits_t(x_u + 1).get_val();
       }
 
-      return x;
+      return (x + x == 0.0) ? x + x : x;
     }
     x_dd = fputil::exact_add(1.0, x);
   }
diff --git a/libc/src/math/generic/log2.cpp b/libc/src/math/generic/log2.cpp
index 7d868e2f6f61..37ea0c8f1343 100644
--- a/libc/src/math/generic/log2.cpp
+++ b/libc/src/math/generic/log2.cpp
@@ -871,7 +871,7 @@ LLVM_LIBC_FUNCTION(double, log2, (double x)) {
 
   if (LIBC_UNLIKELY(xbits.uintval() < FPBits_t::min_normal().uintval() ||
                     xbits.uintval() > FPBits_t::max_normal().uintval())) {
-    if (xbits.is_zero()) {
+    if (x == 0.0) {
       // return -Inf and raise FE_DIVBYZERO.
       fputil::set_errno_if_required(ERANGE);
       fputil::raise_except_if_required(FE_DIVBYZERO);
diff --git a/libc/src/math/generic/log2f.cpp b/libc/src/math/generic/log2f.cpp
index 9cad02d796b1..111f3f130bca 100644
--- a/libc/src/math/generic/log2f.cpp
+++ b/libc/src/math/generic/log2f.cpp
@@ -72,7 +72,7 @@ LLVM_LIBC_FUNCTION(float, log2f, (float x)) {
   // Exceptional inputs.
   if (LIBC_UNLIKELY(x_u < FPBits::min_normal().uintval() ||
                     x_u > FPBits::max_normal().uintval())) {
-    if (xbits.is_zero()) {
+    if (x == 0.0f) {
       fputil::set_errno_if_required(ERANGE);
       fputil::raise_except_if_required(FE_DIVBYZERO);
       return FPBits::inf(Sign::NEG).get_val();
diff --git a/libc/src/math/generic/logf.cpp b/libc/src/math/generic/logf.cpp
index f8ecf320568a..30c00edafe21 100644
--- a/libc/src/math/generic/logf.cpp
+++ b/libc/src/math/generic/logf.cpp
@@ -82,7 +82,7 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) {
     }
     // Subnormal inputs.
     if (LIBC_UNLIKELY(x_u < FPBits::min_normal().uintval())) {
-      if (x_u == 0) {
+      if (x == 0.0f) {
         // Return -inf and raise FE_DIVBYZERO
         fputil::set_errno_if_required(ERANGE);
         fputil::raise_except_if_required(FE_DIVBYZERO);
diff --git a/libc/src/math/generic/pow.cpp b/libc/src/math/generic/pow.cpp
index 181d3d40b3c9..213dbd959039 100644
--- a/libc/src/math/generic/pow.cpp
+++ b/libc/src/math/generic/pow.cpp
@@ -228,16 +228,18 @@ LLVM_LIBC_FUNCTION(double, pow, (double x, double y)) {
                     x_u >= FPBits::inf().uintval() ||
                     x_u < FPBits::min_normal().uintval())) {
     // Exceptional exponents.
-    switch (y_a) {
-    case 0: // y = +-0.0
+    if (y == 0.0)
       return 1.0;
+
+    switch (y_a) {
     case 0x3fe0'0000'0000'0000: { // y = +-0.5
       // TODO: speed up x^(-1/2) with rsqrt(x) when available.
-      if (LIBC_UNLIKELY(!y_sign && (x_u == FPBits::zero(Sign::NEG).uintval() ||
-                                    x_u == FPBits::inf(Sign::NEG).uintval()))) {
+      if (LIBC_UNLIKELY(
+              (x == 0.0 || x_u == FPBits::inf(Sign::NEG).uintval()))) {
         // pow(-0, 1/2) = +0
         // pow(-inf, 1/2) = +inf
-        return FPBits(x_abs).get_val();
+        // Make sure it works correctly for FTZ/DAZ.
+        return y_sign ? 1.0 / (x * x) : (x * x);
       }
       return y_sign ? (1.0 / fputil::sqrt<double>(x)) : fputil::sqrt<double>(x);
     }
@@ -269,7 +271,7 @@ LLVM_LIBC_FUNCTION(double, pow, (double x, double y)) {
           return 1.0;
         }
 
-        if (x_a == 0 && y_sign) {
+        if (x == 0.0 && y_sign) {
           // pow(+-0, -Inf) = +inf and raise FE_DIVBYZERO
           fputil::set_errno_if_required(EDOM);
           fputil::raise_except_if_required(FE_DIVBYZERO);
@@ -298,7 +300,7 @@ LLVM_LIBC_FUNCTION(double, pow, (double x, double y)) {
 
     // TODO: Speed things up with pow(2, y) = exp2(y) and pow(10, y) = exp10(y).
 
-    if (x_a == 0) {
+    if (x == 0.0) {
       bool out_is_neg = x_sign && is_odd_integer(y);
       if (y_sign) {
         // pow(0, negative number) = inf
diff --git a/libc/src/math/generic/powf.cpp b/libc/src/math/generic/powf.cpp
index 83477c6ef2ac..c84ce0da34b1 100644
--- a/libc/src/math/generic/powf.cpp
+++ b/libc/src/math/generic/powf.cpp
@@ -529,10 +529,10 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) {
   // Hence x^y will either overflow or underflow if x is not zero.
   if (LIBC_UNLIKELY((y_abs & 0x0007'ffff) == 0) || (y_abs > 0x4f170000)) {
     // Exceptional exponents.
-    switch (y_abs) {
-    case 0x0000'0000: { // y = +-0.0f
+    if (y == 0.0f)
       return 1.0f;
-    }
+
+    switch (y_abs) {
     case 0x7f80'0000: { // y = +-Inf
       if (x_abs > 0x7f80'0000) {
         // pow(NaN, +-Inf) = NaN
@@ -542,7 +542,7 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) {
         // pow(+-1, +-Inf) = 1.0f
         return 1.0f;
       }
-      if (x_abs == 0 && y_u == 0xff80'0000) {
+      if (x == 0.0f && y_u == 0xff80'0000) {
         // pow(+-0, -Inf) = +inf and raise FE_DIVBYZERO
         fputil::set_errno_if_required(EDOM);
         fputil::raise_except_if_required(FE_DIVBYZERO);
@@ -561,12 +561,15 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) {
       switch (y_u) {
       case 0x3f00'0000: // y = 0.5f
         // pow(x, 1/2) = sqrt(x)
-        if (LIBC_UNLIKELY(x_u == 0x8000'0000 || x_u == 0xff80'0000)) {
+        if (LIBC_UNLIKELY(x == 0.0f || x_u == 0xff80'0000)) {
           // pow(-0, 1/2) = +0
           // pow(-inf, 1/2) = +inf
-          return FloatBits(x_abs).get_val();
+          // Make sure it is correct for FTZ/DAZ.
+          return x * x;
         }
-        return fputil::sqrt<float>(x);
+        float r;
+        r = fputil::sqrt<float>(x);
+        return (FloatBits(r).uintval() != 0x8000'0000) ? r : 0.0f;
       case 0x3f80'0000: // y = 1.0f
         return x;
       case 0x4000'0000: // y = 2.0f
@@ -634,8 +637,7 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) {
 
     const bool x_is_neg = x_u >= FloatBits::SIGN_MASK;
 
-    switch (x_abs) {
-    case 0x0000'0000: { // x = +-0.0f
+    if (x == 0.0f) {
       const bool out_is_neg =
           x_is_neg && is_odd_integer(FloatBits(y_u).get_val());
       if (y_u > 0x8000'0000U) {
@@ -647,7 +649,9 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) {
       // pow(0, positive number) = 0
       return out_is_neg ? -0.0f : 0.0f;
     }
-    case 0x7f80'0000: { // x = +-Inf
+
+    if (x_abs == 0x7f80'0000) {
+      // x = +-Inf
       const bool out_is_neg =
           x_is_neg && is_odd_integer(FloatBits(y_u).get_val());
       if (y_u >= FloatBits::SIGN_MASK) {
@@ -655,7 +659,6 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) {
       }
       return FloatBits::inf(out_is_neg ? Sign::NEG : Sign::POS).get_val();
     }
-    }
 
     if (x_abs > 0x7f80'0000) {
       // x is NaN.
diff --git a/libc/src/math/generic/sin.cpp b/libc/src/math/generic/sin.cpp
index 2e1d3ffd5f37..b32486dff487 100644
--- a/libc/src/math/generic/sin.cpp
+++ b/libc/src/math/generic/sin.cpp
@@ -50,7 +50,7 @@ LLVM_LIBC_FUNCTION(double, sin, (double x)) {
       if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 26)) {
         // Signed zeros.
         if (LIBC_UNLIKELY(x == 0.0))
-          return x;
+          return x + x; // Make sure it works with FTZ/DAZ.
 
 #ifdef LIBC_TARGET_CPU_HAS_FMA
         return fputil::multiply_add(x, -0x1.0p-54, x);
diff --git a/libc/src/math/generic/tan.cpp b/libc/src/math/generic/tan.cpp
index f9be25ed866e..19d31a8441ef 100644
--- a/libc/src/math/generic/tan.cpp
+++ b/libc/src/math/generic/tan.cpp
@@ -138,7 +138,7 @@ LLVM_LIBC_FUNCTION(double, tan, (double x)) {
       if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 27)) {
         // Signed zeros.
         if (LIBC_UNLIKELY(x == 0.0))
-          return x;
+          return x + x; // Make sure it works with FTZ/DAZ.
 
 #ifdef LIBC_TARGET_CPU_HAS_FMA
         return fputil::multiply_add(x, 0x1.0p-54, x);
diff --git a/libc/test/src/math/smoke/HypotTest.h b/libc/test/src/math/smoke/HypotTest.h
index d7c62dcbeb0e..30d57a4fe2a2 100644
--- a/libc/test/src/math/smoke/HypotTest.h
+++ b/libc/test/src/math/smoke/HypotTest.h
@@ -14,13 +14,11 @@
 #include "test/UnitTest/Test.h"
 
 template <typename T>
-class HypotTestTemplate : public LIBC_NAMESPACE::testing::Test {
-private:
+struct HypotTestTemplate : public LIBC_NAMESPACE::testing::Test {
   using Func = T (*)(T, T);
 
   DECLARE_SPECIAL_CONSTANTS(T)
 
-public:
   void test_special_numbers(Func func) {
     constexpr int N = 4;
     // Pythagorean triples.
diff --git a/libc/test/src/math/smoke/acosf_test.cpp b/libc/test/src/math/smoke/acosf_test.cpp
index 039d8c201383..e5d56c70f272 100644
--- a/libc/test/src/math/smoke/acosf_test.cpp
+++ b/libc/test/src/math/smoke/acosf_test.cpp
@@ -38,3 +38,27 @@ TEST_F(LlvmLibcAcosfTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::acosf(-2.0f));
   EXPECT_MATH_ERRNO(EDOM);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcAcosfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0x1.921fb6p0f, LIBC_NAMESPACE::acosf(min_denormal));
+}
+
+TEST_F(LlvmLibcAcosfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0x1.921fb6p0f, LIBC_NAMESPACE::acosf(min_denormal));
+}
+
+TEST_F(LlvmLibcAcosfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0x1.921fb6p0f, LIBC_NAMESPACE::acosf(min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/acoshf_test.cpp b/libc/test/src/math/smoke/acoshf_test.cpp
index 91d433df8055..c4e88259919c 100644
--- a/libc/test/src/math/smoke/acoshf_test.cpp
+++ b/libc/test/src/math/smoke/acoshf_test.cpp
@@ -35,3 +35,27 @@ TEST_F(LlvmLibcAcoshfTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::acoshf(neg_inf));
   EXPECT_MATH_ERRNO(EDOM);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcAcoshfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_IS_NAN(LIBC_NAMESPACE::acoshf(min_denormal));
+}
+
+TEST_F(LlvmLibcAcoshfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_IS_NAN(LIBC_NAMESPACE::acoshf(min_denormal));
+}
+
+TEST_F(LlvmLibcAcoshfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_IS_NAN(LIBC_NAMESPACE::acoshf(min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/asinf_test.cpp b/libc/test/src/math/smoke/asinf_test.cpp
index 450255ccd302..ce1576e2b57d 100644
--- a/libc/test/src/math/smoke/asinf_test.cpp
+++ b/libc/test/src/math/smoke/asinf_test.cpp
@@ -41,3 +41,27 @@ TEST_F(LlvmLibcAsinfTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::asinf(-2.0f));
   EXPECT_MATH_ERRNO(EDOM);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcAsinfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::asinf(min_denormal));
+}
+
+TEST_F(LlvmLibcAsinfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::asinf(min_denormal));
+}
+
+TEST_F(LlvmLibcAsinfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::asinf(min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/asinhf_test.cpp b/libc/test/src/math/smoke/asinhf_test.cpp
index a8e54f379a1f..5b83ce646611 100644
--- a/libc/test/src/math/smoke/asinhf_test.cpp
+++ b/libc/test/src/math/smoke/asinhf_test.cpp
@@ -35,3 +35,27 @@ TEST_F(LlvmLibcAsinhfTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(neg_inf, LIBC_NAMESPACE::asinhf(neg_inf));
   EXPECT_MATH_ERRNO(0);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcAsinhfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::asinhf(min_denormal));
+}
+
+TEST_F(LlvmLibcAsinhfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::asinhf(min_denormal));
+}
+
+TEST_F(LlvmLibcAsinhfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::asinhf(min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/atan2_test.cpp b/libc/test/src/math/smoke/atan2_test.cpp
index 61dd6cab1049..1606c3f378cb 100644
--- a/libc/test/src/math/smoke/atan2_test.cpp
+++ b/libc/test/src/math/smoke/atan2_test.cpp
@@ -20,3 +20,40 @@ TEST_F(LlvmLibcAtan2Test, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(0.0, LIBC_NAMESPACE::atan2(1.0, inf));
   EXPECT_FP_EQ_ALL_ROUNDING(-0.0, LIBC_NAMESPACE::atan2(-1.0, inf));
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcAtan2Test, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0x1.921fb54442d18p-1,
+               LIBC_NAMESPACE::atan2(min_denormal, min_denormal));
+  EXPECT_FP_EQ(0x1.0000000000001p-52,
+               LIBC_NAMESPACE::atan2(min_denormal, max_denormal));
+  EXPECT_FP_EQ(0x1.921fb54442d17p0,
+               LIBC_NAMESPACE::atan2(max_denormal, min_denormal));
+  EXPECT_FP_EQ(0x1.921fb54442d18p-1,
+               LIBC_NAMESPACE::atan2(max_denormal, max_denormal));
+}
+
+TEST_F(LlvmLibcAtan2Test, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::atan2(min_denormal, min_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::atan2(min_denormal, max_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::atan2(max_denormal, min_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::atan2(max_denormal, max_denormal));
+}
+
+TEST_F(LlvmLibcAtan2Test, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::atan2(min_denormal, min_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::atan2(min_denormal, max_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::atan2(max_denormal, min_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::atan2(max_denormal, max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/atanf_test.cpp b/libc/test/src/math/smoke/atanf_test.cpp
index 0fe11d795338..346b8e8abd19 100644
--- a/libc/test/src/math/smoke/atanf_test.cpp
+++ b/libc/test/src/math/smoke/atanf_test.cpp
@@ -42,3 +42,27 @@ TEST_F(LlvmLibcAtanfTest, SpecialNumbers) {
   // EXPECT_FP_EXCEPTION(0);
   EXPECT_MATH_ERRNO(0);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcAtanfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::atanf(min_denormal));
+}
+
+TEST_F(LlvmLibcAtanfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::atanf(min_denormal));
+}
+
+TEST_F(LlvmLibcAtanfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::atanf(min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/atanhf_test.cpp b/libc/test/src/math/smoke/atanhf_test.cpp
index e22926bd2f03..8300b47ea9a3 100644
--- a/libc/test/src/math/smoke/atanhf_test.cpp
+++ b/libc/test/src/math/smoke/atanhf_test.cpp
@@ -76,3 +76,27 @@ TEST_F(LlvmLibcAtanhfTest, SpecialNumbers) {
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::atanhf(neg_inf), FE_INVALID);
   EXPECT_MATH_ERRNO(EDOM);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcAtanhfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::atanhf(min_denormal));
+}
+
+TEST_F(LlvmLibcAtanhfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::atanhf(min_denormal));
+}
+
+TEST_F(LlvmLibcAtanhfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::atanhf(min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/cbrt_test.cpp b/libc/test/src/math/smoke/cbrt_test.cpp
index d57cdb20de27..092e6dd1aeed 100644
--- a/libc/test/src/math/smoke/cbrt_test.cpp
+++ b/libc/test/src/math/smoke/cbrt_test.cpp
@@ -35,3 +35,30 @@ TEST_F(LlvmLibcCbrtTest, SpecialNumbers) {
   EXPECT_FP_EQ(-0x1.0p-340, LIBC_NAMESPACE::cbrt(-0x1.fffffffffffffp-1021));
   EXPECT_FP_EQ(2.0, LIBC_NAMESPACE::cbrt(0x1.fffffffffffffp2));
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcCbrtTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0x1.0p-358, LIBC_NAMESPACE::cbrt(min_denormal));
+  EXPECT_FP_EQ(0x1.428a2f98d728ap-341, LIBC_NAMESPACE::cbrt(max_denormal));
+}
+
+TEST_F(LlvmLibcCbrtTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::cbrt(min_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::cbrt(max_denormal));
+}
+
+TEST_F(LlvmLibcCbrtTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::cbrt(min_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::cbrt(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/cbrtf_test.cpp b/libc/test/src/math/smoke/cbrtf_test.cpp
index a68e57744bd0..202a5ce07335 100644
--- a/libc/test/src/math/smoke/cbrtf_test.cpp
+++ b/libc/test/src/math/smoke/cbrtf_test.cpp
@@ -31,3 +31,30 @@ TEST_F(LlvmLibcCbrtfTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(0x1.0p42f, LIBC_NAMESPACE::cbrtf(0x1.0p126f));
   EXPECT_FP_EQ_ALL_ROUNDING(-0x1.0p42f, LIBC_NAMESPACE::cbrtf(-0x1.0p126f));
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcCbrtfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0x1.428a3p-50f, LIBC_NAMESPACE::cbrtf(min_denormal));
+  EXPECT_FP_EQ(0x1.fffffep-43f, LIBC_NAMESPACE::cbrtf(max_denormal));
+}
+
+TEST_F(LlvmLibcCbrtfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::cbrtf(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::cbrtf(max_denormal));
+}
+
+TEST_F(LlvmLibcCbrtfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::cbrtf(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::cbrtf(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/cos_test.cpp b/libc/test/src/math/smoke/cos_test.cpp
index 81c8612dba26..88d8ead1af99 100644
--- a/libc/test/src/math/smoke/cos_test.cpp
+++ b/libc/test/src/math/smoke/cos_test.cpp
@@ -24,3 +24,30 @@ TEST_F(LlvmLibcCosTest, SpecialNumbers) {
   EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::cos(min_normal));
   EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::cos(min_denormal));
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcCosTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::cos(min_denormal));
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::cos(max_denormal));
+}
+
+TEST_F(LlvmLibcCosTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::cos(min_denormal));
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::cos(max_denormal));
+}
+
+TEST_F(LlvmLibcCosTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::cos(min_denormal));
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::cos(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/cosf_test.cpp b/libc/test/src/math/smoke/cosf_test.cpp
index 62f7ede9cf17..2e261f9fac3c 100644
--- a/libc/test/src/math/smoke/cosf_test.cpp
+++ b/libc/test/src/math/smoke/cosf_test.cpp
@@ -35,3 +35,30 @@ TEST_F(LlvmLibcCosfTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::cosf(neg_inf));
   EXPECT_MATH_ERRNO(EDOM);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcCosfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cosf(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cosf(max_denormal));
+}
+
+TEST_F(LlvmLibcCosfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cosf(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cosf(max_denormal));
+}
+
+TEST_F(LlvmLibcCosfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cosf(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cosf(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/coshf_test.cpp b/libc/test/src/math/smoke/coshf_test.cpp
index ddaa19f4c392..fd1556b10116 100644
--- a/libc/test/src/math/smoke/coshf_test.cpp
+++ b/libc/test/src/math/smoke/coshf_test.cpp
@@ -51,3 +51,30 @@ TEST_F(LlvmLibcCoshfTest, Overflow) {
       inf, LIBC_NAMESPACE::coshf(FPBits(0x42d00008U).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcCoshfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::coshf(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::coshf(max_denormal));
+}
+
+TEST_F(LlvmLibcCoshfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::coshf(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::coshf(max_denormal));
+}
+
+TEST_F(LlvmLibcCoshfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::coshf(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::coshf(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/cospif_test.cpp b/libc/test/src/math/smoke/cospif_test.cpp
index 007c4c45e3b1..bf6d86bcfe62 100644
--- a/libc/test/src/math/smoke/cospif_test.cpp
+++ b/libc/test/src/math/smoke/cospif_test.cpp
@@ -32,3 +32,30 @@ TEST_F(LlvmLibcCospifTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::cospif(neg_inf));
   EXPECT_MATH_ERRNO(EDOM);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcCospifTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cospif(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cospif(max_denormal));
+}
+
+TEST_F(LlvmLibcCospifTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cospif(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cospif(max_denormal));
+}
+
+TEST_F(LlvmLibcCospifTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cospif(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cospif(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/erff_test.cpp b/libc/test/src/math/smoke/erff_test.cpp
index 8a970f3a4b7e..7d2c1013752c 100644
--- a/libc/test/src/math/smoke/erff_test.cpp
+++ b/libc/test/src/math/smoke/erff_test.cpp
@@ -23,3 +23,30 @@ TEST_F(LlvmLibcErffTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::erff(zero));
   EXPECT_FP_EQ_ALL_ROUNDING(neg_zero, LIBC_NAMESPACE::erff(neg_zero));
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcErffTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::erff(min_denormal));
+  EXPECT_FP_EQ(0x1.20dd72p-126f, LIBC_NAMESPACE::erff(max_denormal));
+}
+
+TEST_F(LlvmLibcErffTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::erff(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::erff(max_denormal));
+}
+
+TEST_F(LlvmLibcErffTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::erff(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::erff(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/exp10_test.cpp b/libc/test/src/math/smoke/exp10_test.cpp
index 282ddc987b49..ca9fc359edeb 100644
--- a/libc/test/src/math/smoke/exp10_test.cpp
+++ b/libc/test/src/math/smoke/exp10_test.cpp
@@ -32,3 +32,30 @@ TEST_F(LlvmLibcExp10Test, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(100.0, LIBC_NAMESPACE::exp10(2.0));
   EXPECT_FP_EQ_ALL_ROUNDING(1000.0, LIBC_NAMESPACE::exp10(3.0));
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcExp10Test, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp10(min_denormal));
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp10(max_denormal));
+}
+
+TEST_F(LlvmLibcExp10Test, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp10(min_denormal));
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp10(max_denormal));
+}
+
+TEST_F(LlvmLibcExp10Test, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp10(min_denormal));
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp10(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/exp10f_test.cpp b/libc/test/src/math/smoke/exp10f_test.cpp
index 9fb15ae75348..bcbfc96efd72 100644
--- a/libc/test/src/math/smoke/exp10f_test.cpp
+++ b/libc/test/src/math/smoke/exp10f_test.cpp
@@ -54,3 +54,30 @@ TEST_F(LlvmLibcExp10fTest, Overflow) {
       inf, LIBC_NAMESPACE::exp10f(FPBits(0x43000001U).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcExp10fTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::exp10f(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::exp10f(max_denormal));
+}
+
+TEST_F(LlvmLibcExp10fTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::exp10f(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::exp10f(max_denormal));
+}
+
+TEST_F(LlvmLibcExp10fTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::exp10f(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::exp10f(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/exp2_test.cpp b/libc/test/src/math/smoke/exp2_test.cpp
index d148d27fad38..d97a384367a0 100644
--- a/libc/test/src/math/smoke/exp2_test.cpp
+++ b/libc/test/src/math/smoke/exp2_test.cpp
@@ -31,3 +31,30 @@ TEST_F(LlvmLibcExp2Test, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(4.0, LIBC_NAMESPACE::exp2(2.0));
   EXPECT_FP_EQ_ALL_ROUNDING(0.25, LIBC_NAMESPACE::exp2(-2.0));
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcExp2Test, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp2(min_denormal));
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp2(max_denormal));
+}
+
+TEST_F(LlvmLibcExp2Test, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp2(min_denormal));
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp2(max_denormal));
+}
+
+TEST_F(LlvmLibcExp2Test, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp2(min_denormal));
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp2(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/exp2f_test.cpp b/libc/test/src/math/smoke/exp2f_test.cpp
index 39228eb2f6d8..d9cdecbf0fe9 100644
--- a/libc/test/src/math/smoke/exp2f_test.cpp
+++ b/libc/test/src/math/smoke/exp2f_test.cpp
@@ -55,3 +55,30 @@ TEST_F(LlvmLibcExp2fTest, Overflow) {
       inf, LIBC_NAMESPACE::exp2f(FPBits(0x43000001U).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcExp2fTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::exp2f(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::exp2f(max_denormal));
+}
+
+TEST_F(LlvmLibcExp2fTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::exp2f(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::exp2f(max_denormal));
+}
+
+TEST_F(LlvmLibcExp2fTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::exp2f(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::exp2f(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/exp2m1f_test.cpp b/libc/test/src/math/smoke/exp2m1f_test.cpp
index 2df435385247..4657d088f07a 100644
--- a/libc/test/src/math/smoke/exp2m1f_test.cpp
+++ b/libc/test/src/math/smoke/exp2m1f_test.cpp
@@ -61,3 +61,30 @@ TEST_F(LlvmLibcExp2m1fTest, Underflow) {
                               FE_UNDERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcExp2m1fTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::exp2m1f(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::exp2m1f(max_denormal));
+}
+
+TEST_F(LlvmLibcExp2m1fTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::exp2m1f(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::exp2m1f(max_denormal));
+}
+
+TEST_F(LlvmLibcExp2m1fTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::exp2m1f(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::exp2m1f(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/exp_test.cpp b/libc/test/src/math/smoke/exp_test.cpp
index 5fe6f3e92f4a..d2467ff88389 100644
--- a/libc/test/src/math/smoke/exp_test.cpp
+++ b/libc/test/src/math/smoke/exp_test.cpp
@@ -27,3 +27,30 @@ TEST_F(LlvmLibcExpTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(1.0, LIBC_NAMESPACE::exp(0.0));
   EXPECT_FP_EQ_ALL_ROUNDING(1.0, LIBC_NAMESPACE::exp(-0.0));
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcExpTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp(min_denormal));
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp(max_denormal));
+}
+
+TEST_F(LlvmLibcExpTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp(min_denormal));
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp(max_denormal));
+}
+
+TEST_F(LlvmLibcExpTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp(min_denormal));
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::exp(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/expf_test.cpp b/libc/test/src/math/smoke/expf_test.cpp
index b954125afd7b..11181ed1402c 100644
--- a/libc/test/src/math/smoke/expf_test.cpp
+++ b/libc/test/src/math/smoke/expf_test.cpp
@@ -50,3 +50,30 @@ TEST_F(LlvmLibcExpfTest, Overflow) {
       inf, LIBC_NAMESPACE::expf(FPBits(0x42d00008U).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcExpfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::expf(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::expf(max_denormal));
+}
+
+TEST_F(LlvmLibcExpfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::expf(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::expf(max_denormal));
+}
+
+TEST_F(LlvmLibcExpfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::expf(min_denormal));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::expf(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/expm1_test.cpp b/libc/test/src/math/smoke/expm1_test.cpp
index bafdbda8af03..cebd2d757606 100644
--- a/libc/test/src/math/smoke/expm1_test.cpp
+++ b/libc/test/src/math/smoke/expm1_test.cpp
@@ -33,3 +33,30 @@ TEST_F(LlvmLibcExpm1Test, SpecialNumbers) {
   // log(2^-54)
   EXPECT_FP_EQ(-1.0, LIBC_NAMESPACE::expm1(-0x1.2b708872320e2p5));
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcExpm1Test, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::expm1(min_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::expm1(max_denormal));
+}
+
+TEST_F(LlvmLibcExpm1Test, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::expm1(min_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::expm1(max_denormal));
+}
+
+TEST_F(LlvmLibcExpm1Test, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::expm1(min_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::expm1(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/expm1f_test.cpp b/libc/test/src/math/smoke/expm1f_test.cpp
index 03b6e47b7c3b..f4138aa05ba7 100644
--- a/libc/test/src/math/smoke/expm1f_test.cpp
+++ b/libc/test/src/math/smoke/expm1f_test.cpp
@@ -50,3 +50,30 @@ TEST_F(LlvmLibcExpm1fTest, Overflow) {
       inf, LIBC_NAMESPACE::expm1f(FPBits(0x42d00008U).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcExpm1fTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::expm1f(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::expm1f(max_denormal));
+}
+
+TEST_F(LlvmLibcExpm1fTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::expm1f(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::expm1f(max_denormal));
+}
+
+TEST_F(LlvmLibcExpm1fTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::expm1f(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::expm1f(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/hypotf_test.cpp b/libc/test/src/math/smoke/hypotf_test.cpp
index 768e7f75e9d6..62399489987e 100644
--- a/libc/test/src/math/smoke/hypotf_test.cpp
+++ b/libc/test/src/math/smoke/hypotf_test.cpp
@@ -15,3 +15,37 @@ using LlvmLibcHypotfTest = HypotTestTemplate<float>;
 TEST_F(LlvmLibcHypotfTest, SpecialNumbers) {
   test_special_numbers(&LIBC_NAMESPACE::hypotf);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcHypotfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::hypotf(min_denormal, min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::hypotf(min_denormal, max_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::hypotf(max_denormal, min_denormal));
+  EXPECT_FP_EQ(0x1.6a09e4p-126f,
+               LIBC_NAMESPACE::hypotf(max_denormal, max_denormal));
+}
+
+TEST_F(LlvmLibcHypotfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::hypotf(min_denormal, min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::hypotf(min_denormal, max_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::hypotf(max_denormal, min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::hypotf(max_denormal, max_denormal));
+}
+
+TEST_F(LlvmLibcHypotfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::hypotf(min_denormal, min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::hypotf(min_denormal, max_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::hypotf(max_denormal, min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::hypotf(max_denormal, max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/log10_test.cpp b/libc/test/src/math/smoke/log10_test.cpp
index e03416ae20c8..9f159f282aad 100644
--- a/libc/test/src/math/smoke/log10_test.cpp
+++ b/libc/test/src/math/smoke/log10_test.cpp
@@ -33,3 +33,29 @@ TEST_F(LlvmLibcLog10Test, SpecialNumbers) {
     EXPECT_FP_EQ_ALL_ROUNDING(static_cast<double>(i), LIBC_NAMESPACE::log10(x));
   }
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcLog10Test, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(-0x1.434e6420f4374p8, LIBC_NAMESPACE::log10(min_denormal));
+}
+
+TEST_F(LlvmLibcLog10Test, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(FPBits::inf(Sign::NEG).get_val(),
+               LIBC_NAMESPACE::log10(min_denormal));
+}
+
+TEST_F(LlvmLibcLog10Test, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(FPBits::inf(Sign::NEG).get_val(),
+               LIBC_NAMESPACE::log10(min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/log10f_test.cpp b/libc/test/src/math/smoke/log10f_test.cpp
index 2524545e0181..4e3bf654ca91 100644
--- a/libc/test/src/math/smoke/log10f_test.cpp
+++ b/libc/test/src/math/smoke/log10f_test.cpp
@@ -32,3 +32,29 @@ TEST_F(LlvmLibcLog10fTest, SpecialNumbers) {
     EXPECT_FP_EQ_ALL_ROUNDING(static_cast<float>(i), LIBC_NAMESPACE::log10f(x));
   }
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcLog10fTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(-0x1.66d3e7bd9a403p5f, LIBC_NAMESPACE::log10f(min_denormal));
+}
+
+TEST_F(LlvmLibcLog10fTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(FPBits::inf(Sign::NEG).get_val(),
+               LIBC_NAMESPACE::log10f(min_denormal));
+}
+
+TEST_F(LlvmLibcLog10fTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(FPBits::inf(Sign::NEG).get_val(),
+               LIBC_NAMESPACE::log10f(min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/log1p_test.cpp b/libc/test/src/math/smoke/log1p_test.cpp
index 63237f3259b2..eba65f56df73 100644
--- a/libc/test/src/math/smoke/log1p_test.cpp
+++ b/libc/test/src/math/smoke/log1p_test.cpp
@@ -27,3 +27,27 @@ TEST_F(LlvmLibcLog1pTest, SpecialNumbers) {
   EXPECT_FP_EQ_WITH_EXCEPTION(neg_inf, LIBC_NAMESPACE::log1p(-1.0),
                               FE_DIVBYZERO);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcLog1pTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::log1p(min_denormal));
+}
+
+TEST_F(LlvmLibcLog1pTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::log1p(min_denormal));
+}
+
+TEST_F(LlvmLibcLog1pTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::log1p(min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/log1pf_test.cpp b/libc/test/src/math/smoke/log1pf_test.cpp
index c14d65529469..1b0a1d589e68 100644
--- a/libc/test/src/math/smoke/log1pf_test.cpp
+++ b/libc/test/src/math/smoke/log1pf_test.cpp
@@ -26,3 +26,27 @@ TEST_F(LlvmLibcLog1pfTest, SpecialNumbers) {
   EXPECT_FP_EQ_WITH_EXCEPTION(neg_inf, LIBC_NAMESPACE::log1pf(-1.0f),
                               FE_DIVBYZERO);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcLog1pfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::log1pf(min_denormal));
+}
+
+TEST_F(LlvmLibcLog1pfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::log1pf(min_denormal));
+}
+
+TEST_F(LlvmLibcLog1pfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::log1pf(min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/log2_test.cpp b/libc/test/src/math/smoke/log2_test.cpp
index 89d8e5651091..1570d60556df 100644
--- a/libc/test/src/math/smoke/log2_test.cpp
+++ b/libc/test/src/math/smoke/log2_test.cpp
@@ -27,3 +27,29 @@ TEST_F(LlvmLibcLog2Test, SpecialNumbers) {
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log2(-1.0), FE_INVALID);
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::log2(1.0));
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcLog2Test, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(-1074.0, LIBC_NAMESPACE::log2(min_denormal));
+}
+
+TEST_F(LlvmLibcLog2Test, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(FPBits::inf(Sign::NEG).get_val(),
+               LIBC_NAMESPACE::log2(min_denormal));
+}
+
+TEST_F(LlvmLibcLog2Test, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(FPBits::inf(Sign::NEG).get_val(),
+               LIBC_NAMESPACE::log2(min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/log2f_test.cpp b/libc/test/src/math/smoke/log2f_test.cpp
index 00bfb7c4abad..67b2c5b2db13 100644
--- a/libc/test/src/math/smoke/log2f_test.cpp
+++ b/libc/test/src/math/smoke/log2f_test.cpp
@@ -28,3 +28,28 @@ TEST_F(LlvmLibcLog2fTest, SpecialNumbers) {
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log2f(-1.0f), FE_INVALID);
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::log2f(1.0f));
 }
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcLog2fTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(-149.0f, LIBC_NAMESPACE::log2f(min_denormal));
+}
+
+TEST_F(LlvmLibcLog2fTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(FPBits::inf(Sign::NEG).get_val(),
+               LIBC_NAMESPACE::log2f(min_denormal));
+}
+
+TEST_F(LlvmLibcLog2fTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(FPBits::inf(Sign::NEG).get_val(),
+               LIBC_NAMESPACE::log2f(min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/log_test.cpp b/libc/test/src/math/smoke/log_test.cpp
index e7897add575f..20b974d7e167 100644
--- a/libc/test/src/math/smoke/log_test.cpp
+++ b/libc/test/src/math/smoke/log_test.cpp
@@ -26,3 +26,29 @@ TEST_F(LlvmLibcLogTest, SpecialNumbers) {
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log(-1.0), FE_INVALID);
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::log(1.0));
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcLogTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(-0x1.74385446d71c3p9, LIBC_NAMESPACE::log(min_denormal));
+}
+
+TEST_F(LlvmLibcLogTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(FPBits::inf(Sign::NEG).get_val(),
+               LIBC_NAMESPACE::log(min_denormal));
+}
+
+TEST_F(LlvmLibcLogTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(FPBits::inf(Sign::NEG).get_val(),
+               LIBC_NAMESPACE::log(min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/logf_test.cpp b/libc/test/src/math/smoke/logf_test.cpp
index a27206027614..1a3102ae2b14 100644
--- a/libc/test/src/math/smoke/logf_test.cpp
+++ b/libc/test/src/math/smoke/logf_test.cpp
@@ -27,3 +27,28 @@ TEST_F(LlvmLibcLogfTest, SpecialNumbers) {
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::logf(-1.0f), FE_INVALID);
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::logf(1.0f));
 }
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcLogfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(-0x1.9d1d9fccf477p6f, LIBC_NAMESPACE::logf(min_denormal));
+}
+
+TEST_F(LlvmLibcLogfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(FPBits::inf(Sign::NEG).get_val(),
+               LIBC_NAMESPACE::logf(min_denormal));
+}
+
+TEST_F(LlvmLibcLogfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(FPBits::inf(Sign::NEG).get_val(),
+               LIBC_NAMESPACE::logf(min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/pow_test.cpp b/libc/test/src/math/smoke/pow_test.cpp
index 7f0136d783c6..f9db7f102962 100644
--- a/libc/test/src/math/smoke/pow_test.cpp
+++ b/libc/test/src/math/smoke/pow_test.cpp
@@ -190,3 +190,30 @@ TEST_F(LlvmLibcPowTest, SpecialNumbers) {
     }
   }
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcPowTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_IS_NAN(LIBC_NAMESPACE::pow(-min_denormal, 0.5));
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::pow(2.0, min_denormal));
+}
+
+TEST_F(LlvmLibcPowTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::pow(-min_denormal, 0.5));
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::pow(2.0, min_denormal));
+}
+
+TEST_F(LlvmLibcPowTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::pow(-min_denormal, 0.5));
+  EXPECT_FP_EQ(1.0, LIBC_NAMESPACE::pow(2.0, min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/powf_test.cpp b/libc/test/src/math/smoke/powf_test.cpp
index a0f66f2733a1..9cc95ce0baef 100644
--- a/libc/test/src/math/smoke/powf_test.cpp
+++ b/libc/test/src/math/smoke/powf_test.cpp
@@ -194,3 +194,30 @@ TEST_F(LlvmLibcPowfTest, SpecialNumbers) {
   EXPECT_FP_EQ(-0.0f, LIBC_NAMESPACE::powf(-0.015625f, 25.0f));
   EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::powf(-0.015625f, 26.0f));
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcPowfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_IS_NAN(LIBC_NAMESPACE::powf(-min_denormal, 0.5f));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::powf(2.0f, min_denormal));
+}
+
+TEST_F(LlvmLibcPowfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::powf(-min_denormal, 0.5f));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::powf(2.0f, min_denormal));
+}
+
+TEST_F(LlvmLibcPowfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::powf(-min_denormal, 0.5f));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::powf(2.0f, min_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/sin_test.cpp b/libc/test/src/math/smoke/sin_test.cpp
index 16ced68709ca..7dd1b7fda625 100644
--- a/libc/test/src/math/smoke/sin_test.cpp
+++ b/libc/test/src/math/smoke/sin_test.cpp
@@ -24,3 +24,30 @@ TEST_F(LlvmLibcSinTest, SpecialNumbers) {
   EXPECT_FP_EQ(min_normal, LIBC_NAMESPACE::sin(min_normal));
   EXPECT_FP_EQ(min_denormal, LIBC_NAMESPACE::sin(min_denormal));
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcSinTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::sin(min_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::sin(max_denormal));
+}
+
+TEST_F(LlvmLibcSinTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::sin(min_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::sin(max_denormal));
+}
+
+TEST_F(LlvmLibcSinTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::sin(min_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::sin(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/sinf_test.cpp b/libc/test/src/math/smoke/sinf_test.cpp
index 1bf6eaa8b78d..776c66dcb37b 100644
--- a/libc/test/src/math/smoke/sinf_test.cpp
+++ b/libc/test/src/math/smoke/sinf_test.cpp
@@ -35,3 +35,30 @@ TEST_F(LlvmLibcSinfTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::sinf(neg_inf));
   EXPECT_MATH_ERRNO(EDOM);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcSinfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinf(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinf(max_denormal));
+}
+
+TEST_F(LlvmLibcSinfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinf(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinf(max_denormal));
+}
+
+TEST_F(LlvmLibcSinfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinf(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinf(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/sinhf_test.cpp b/libc/test/src/math/smoke/sinhf_test.cpp
index 635a10627a21..3cc065696758 100644
--- a/libc/test/src/math/smoke/sinhf_test.cpp
+++ b/libc/test/src/math/smoke/sinhf_test.cpp
@@ -62,3 +62,30 @@ TEST_F(LlvmLibcSinhfTest, Overflow) {
       inf, LIBC_NAMESPACE::sinhf(FPBits(0x42d00008U).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcSinhfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinhf(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinhf(max_denormal));
+}
+
+TEST_F(LlvmLibcSinhfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinhf(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinhf(max_denormal));
+}
+
+TEST_F(LlvmLibcSinhfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinhf(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinhf(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/sinpif_test.cpp b/libc/test/src/math/smoke/sinpif_test.cpp
index 0918294ab361..11bda0b6b28c 100644
--- a/libc/test/src/math/smoke/sinpif_test.cpp
+++ b/libc/test/src/math/smoke/sinpif_test.cpp
@@ -41,3 +41,30 @@ TEST_F(LlvmLibcSinpifTest, Integers) {
   EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::sinpif(0x1.cp+106));
   EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::sinpif(0x1.cp+21));
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcSinpifTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinpif(min_denormal));
+  EXPECT_FP_EQ(0x1.921fb2p-125f, LIBC_NAMESPACE::sinpif(max_denormal));
+}
+
+TEST_F(LlvmLibcSinpifTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinpif(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinpif(max_denormal));
+}
+
+TEST_F(LlvmLibcSinpifTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinpif(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::sinpif(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/tan_test.cpp b/libc/test/src/math/smoke/tan_test.cpp
index 498dba76b6e7..aa5c23d65886 100644
--- a/libc/test/src/math/smoke/tan_test.cpp
+++ b/libc/test/src/math/smoke/tan_test.cpp
@@ -24,3 +24,30 @@ TEST_F(LlvmLibcTanTest, SpecialNumbers) {
   EXPECT_FP_EQ(min_normal, LIBC_NAMESPACE::tan(min_normal));
   EXPECT_FP_EQ(min_denormal, LIBC_NAMESPACE::tan(min_denormal));
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcTanTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::tan(min_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::tan(max_denormal));
+}
+
+TEST_F(LlvmLibcTanTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::tan(min_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::tan(max_denormal));
+}
+
+TEST_F(LlvmLibcTanTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::tan(min_denormal));
+  EXPECT_FP_EQ(0.0, LIBC_NAMESPACE::tan(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/tanf_test.cpp b/libc/test/src/math/smoke/tanf_test.cpp
index b90c5da87418..93fbfded3f66 100644
--- a/libc/test/src/math/smoke/tanf_test.cpp
+++ b/libc/test/src/math/smoke/tanf_test.cpp
@@ -35,3 +35,30 @@ TEST_F(LlvmLibcTanfTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::tanf(neg_inf));
   EXPECT_MATH_ERRNO(EDOM);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcTanfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::tanf(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::tanf(max_denormal));
+}
+
+TEST_F(LlvmLibcTanfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::tanf(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::tanf(max_denormal));
+}
+
+TEST_F(LlvmLibcTanfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::tanf(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::tanf(max_denormal));
+}
+
+#endif
diff --git a/libc/test/src/math/smoke/tanhf_test.cpp b/libc/test/src/math/smoke/tanhf_test.cpp
index 748e6fe8c626..3b7faa81dac2 100644
--- a/libc/test/src/math/smoke/tanhf_test.cpp
+++ b/libc/test/src/math/smoke/tanhf_test.cpp
@@ -35,3 +35,30 @@ TEST_F(LlvmLibcTanhfTest, SpecialNumbers) {
   EXPECT_FP_EQ(-1.0f, LIBC_NAMESPACE::tanhf(neg_inf));
   EXPECT_MATH_ERRNO(0);
 }
+
+#ifdef LIBC_TEST_FTZ_DAZ
+
+using namespace LIBC_NAMESPACE::testing;
+
+TEST_F(LlvmLibcTanhfTest, FTZMode) {
+  ModifyMXCSR mxcsr(FTZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::tanhf(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::tanhf(max_denormal));
+}
+
+TEST_F(LlvmLibcTanhfTest, DAZMode) {
+  ModifyMXCSR mxcsr(DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::tanhf(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::tanhf(max_denormal));
+}
+
+TEST_F(LlvmLibcTanhfTest, FTZDAZMode) {
+  ModifyMXCSR mxcsr(FTZ | DAZ);
+
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::tanhf(min_denormal));
+  EXPECT_FP_EQ(0.0f, LIBC_NAMESPACE::tanhf(max_denormal));
+}
+
+#endif
-- 
GitLab


From 266ff98cba44b9e969e6894eaa30d4fe37647210 Mon Sep 17 00:00:00 2001
From: Shih-Po Hung <shihpo.hung@sifive.com>
Date: Sat, 26 Oct 2024 23:18:50 +0800
Subject: [PATCH 237/693] [LV][VPlan] Use VF VPValue in VPVectorPointerRecipe
 (#110974)

Refactors VPVectorPointerRecipe to use the VF VPValue to obtain the
runtime VF, similar to #95305.

Since only reverse vector pointers require the runtime VF, the patch
sets VPUnrollPart::PartOpIndex to 1 for vector pointers and 2 for
reverse vector pointers. As a result, the generation of reverse vector
pointers is moved into a separate recipe.
---
 .../Transforms/Vectorize/LoopVectorize.cpp    |  13 +-
 llvm/lib/Transforms/Vectorize/VPlan.h         |  64 +++++++--
 .../Transforms/Vectorize/VPlanAnalysis.cpp    |   7 +-
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |  80 +++++++----
 llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp |   4 +-
 llvm/lib/Transforms/Vectorize/VPlanValue.h    |   1 +
 .../AArch64/sve-vector-reverse.ll             | 130 ++++++++----------
 .../RISCV/riscv-vector-reverse.ll             |  20 +--
 ...-force-tail-with-evl-reverse-load-store.ll | 120 ++++++++--------
 ...orize-force-tail-with-evl-uniform-store.ll |   6 +-
 .../LoopVectorize/reverse_induction.ll        |  24 ++--
 .../vplan-sink-scalars-and-merge.ll           |   5 +-
 12 files changed, 267 insertions(+), 207 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 865f5e3d2e58..88086f24dfdc 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4492,6 +4492,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
       case VPDef::VPInstructionSC:
       case VPDef::VPCanonicalIVPHISC:
       case VPDef::VPVectorPointerSC:
+      case VPDef::VPReverseVectorPointerSC:
       case VPDef::VPExpandSCEVSC:
       case VPDef::VPEVLBasedIVPHISC:
       case VPDef::VPPredInstPHISC:
@@ -8278,9 +8279,15 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
   if (Consecutive) {
     auto *GEP = dyn_cast<GetElementPtrInst>(
         Ptr->getUnderlyingValue()->stripPointerCasts());
-    auto *VectorPtr = new VPVectorPointerRecipe(
-        Ptr, getLoadStoreType(I), Reverse, GEP ? GEP->isInBounds() : false,
-        I->getDebugLoc());
+    VPSingleDefRecipe *VectorPtr;
+    if (Reverse)
+      VectorPtr = new VPReverseVectorPointerRecipe(
+          Ptr, &Plan.getVF(), getLoadStoreType(I),
+          GEP ? GEP->isInBounds() : false, I->getDebugLoc());
+    else
+      VectorPtr = new VPVectorPointerRecipe(Ptr, getLoadStoreType(I),
+                                            GEP ? GEP->isInBounds() : false,
+                                            I->getDebugLoc());
     Builder.getInsertBlock()->appendRecipe(VectorPtr);
     Ptr = VectorPtr;
   }
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index f2e6729a2e26..a34e34a0d71f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -905,6 +905,7 @@ public:
     case VPRecipeBase::VPReplicateSC:
     case VPRecipeBase::VPScalarIVStepsSC:
     case VPRecipeBase::VPVectorPointerSC:
+    case VPRecipeBase::VPReverseVectorPointerSC:
     case VPRecipeBase::VPWidenCallSC:
     case VPRecipeBase::VPWidenCanonicalIVSC:
     case VPRecipeBase::VPWidenCastSC:
@@ -1110,6 +1111,7 @@ public:
            R->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
            R->getVPDefID() == VPRecipeBase::VPWidenCastSC ||
            R->getVPDefID() == VPRecipeBase::VPReplicateSC ||
+           R->getVPDefID() == VPRecipeBase::VPReverseVectorPointerSC ||
            R->getVPDefID() == VPRecipeBase::VPVectorPointerSC;
   }
 
@@ -1910,20 +1912,64 @@ public:
 #endif
 };
 
-/// A recipe to compute the pointers for widened memory accesses of IndexTy for
-/// all parts. If IsReverse is true, compute pointers for accessing the input in
-/// reverse order per part.
+/// A recipe to compute the pointers for widened memory accesses of IndexTy
+/// in reverse order.
+class VPReverseVectorPointerRecipe : public VPRecipeWithIRFlags,
+                                     public VPUnrollPartAccessor<2> {
+  Type *IndexedTy;
+
+public:
+  VPReverseVectorPointerRecipe(VPValue *Ptr, VPValue *VF, Type *IndexedTy,
+                               bool IsInBounds, DebugLoc DL)
+      : VPRecipeWithIRFlags(VPDef::VPReverseVectorPointerSC,
+                            ArrayRef<VPValue *>({Ptr, VF}),
+                            GEPFlagsTy(IsInBounds), DL),
+        IndexedTy(IndexedTy) {}
+
+  VP_CLASSOF_IMPL(VPDef::VPReverseVectorPointerSC)
+
+  VPValue *getVFValue() { return getOperand(1); }
+  const VPValue *getVFValue() const { return getOperand(1); }
+
+  void execute(VPTransformState &State) override;
+
+  bool onlyFirstLaneUsed(const VPValue *Op) const override {
+    assert(is_contained(operands(), Op) &&
+           "Op must be an operand of the recipe");
+    return true;
+  }
+
+  /// Returns true if the recipe only uses the first part of operand \p Op.
+  bool onlyFirstPartUsed(const VPValue *Op) const override {
+    assert(is_contained(operands(), Op) &&
+           "Op must be an operand of the recipe");
+    assert(getNumOperands() <= 2 && "must have at most two operands");
+    return true;
+  }
+
+  VPReverseVectorPointerRecipe *clone() override {
+    return new VPReverseVectorPointerRecipe(
+        getOperand(0), getVFValue(), IndexedTy, isInBounds(), getDebugLoc());
+  }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  /// Print the recipe.
+  void print(raw_ostream &O, const Twine &Indent,
+             VPSlotTracker &SlotTracker) const override;
+#endif
+};
+
+/// A recipe to compute the pointers for widened memory accesses of IndexTy.
 class VPVectorPointerRecipe : public VPRecipeWithIRFlags,
                               public VPUnrollPartAccessor<1> {
   Type *IndexedTy;
-  bool IsReverse;
 
 public:
-  VPVectorPointerRecipe(VPValue *Ptr, Type *IndexedTy, bool IsReverse,
-                        bool IsInBounds, DebugLoc DL)
+  VPVectorPointerRecipe(VPValue *Ptr, Type *IndexedTy, bool IsInBounds,
+                        DebugLoc DL)
       : VPRecipeWithIRFlags(VPDef::VPVectorPointerSC, ArrayRef<VPValue *>(Ptr),
                             GEPFlagsTy(IsInBounds), DL),
-        IndexedTy(IndexedTy), IsReverse(IsReverse) {}
+        IndexedTy(IndexedTy) {}
 
   VP_CLASSOF_IMPL(VPDef::VPVectorPointerSC)
 
@@ -1944,8 +1990,8 @@ public:
   }
 
   VPVectorPointerRecipe *clone() override {
-    return new VPVectorPointerRecipe(getOperand(0), IndexedTy, IsReverse,
-                                     isInBounds(), getDebugLoc());
+    return new VPVectorPointerRecipe(getOperand(0), IndexedTy, isInBounds(),
+                                     getDebugLoc());
   }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 3eb5f3f40f84..8b8ab6be99b0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -263,9 +263,10 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
               [](const auto *R) { return R->getScalarType(); })
           .Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,
                 VPScalarIVStepsRecipe, VPWidenGEPRecipe, VPVectorPointerRecipe,
-                VPWidenCanonicalIVRecipe>([this](const VPRecipeBase *R) {
-            return inferScalarType(R->getOperand(0));
-          })
+                VPReverseVectorPointerRecipe, VPWidenCanonicalIVRecipe>(
+              [this](const VPRecipeBase *R) {
+                return inferScalarType(R->getOperand(0));
+              })
           .Case<VPBlendRecipe, VPInstruction, VPWidenRecipe, VPWidenEVLRecipe,
                 VPReplicateRecipe, VPWidenCallRecipe, VPWidenMemoryRecipe,
                 VPWidenSelectRecipe>(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 2080b77157b6..b1e6086398c4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -162,6 +162,7 @@ bool VPRecipeBase::mayHaveSideEffects() const {
   case VPDerivedIVSC:
   case VPPredInstPHISC:
   case VPScalarCastSC:
+  case VPReverseVectorPointerSC:
     return false;
   case VPInstructionSC:
     return mayWriteToMemory();
@@ -1971,38 +1972,63 @@ void VPWidenGEPRecipe::print(raw_ostream &O, const Twine &Indent,
 }
 #endif
 
-void VPVectorPointerRecipe ::execute(VPTransformState &State) {
-  auto &Builder = State.Builder;
-  State.setDebugLocFrom(getDebugLoc());
-  unsigned CurrentPart = getUnrollPart(*this);
+static Type *getGEPIndexTy(bool IsScalable, bool IsReverse,
+                           unsigned CurrentPart, IRBuilderBase &Builder) {
   // Use i32 for the gep index type when the value is constant,
   // or query DataLayout for a more suitable index type otherwise.
   const DataLayout &DL = Builder.GetInsertBlock()->getDataLayout();
-  Type *IndexTy = State.VF.isScalable() && (IsReverse || CurrentPart > 0)
-                      ? DL.getIndexType(Builder.getPtrTy(0))
-                      : Builder.getInt32Ty();
+  return IsScalable && (IsReverse || CurrentPart > 0)
+             ? DL.getIndexType(Builder.getPtrTy(0))
+             : Builder.getInt32Ty();
+}
+
+void VPReverseVectorPointerRecipe::execute(VPTransformState &State) {
+  auto &Builder = State.Builder;
+  State.setDebugLocFrom(getDebugLoc());
+  unsigned CurrentPart = getUnrollPart(*this);
+  Type *IndexTy = getGEPIndexTy(State.VF.isScalable(), /*IsReverse*/ true,
+                                CurrentPart, Builder);
+
+  // The wide store needs to start at the last vector element.
+  Value *RunTimeVF = State.get(getVFValue(), VPLane(0));
+  if (IndexTy != RunTimeVF->getType())
+    RunTimeVF = Builder.CreateZExtOrTrunc(RunTimeVF, IndexTy);
+  // NumElt = -CurrentPart * RunTimeVF
+  Value *NumElt = Builder.CreateMul(
+      ConstantInt::get(IndexTy, -(int64_t)CurrentPart), RunTimeVF);
+  // LastLane = 1 - RunTimeVF
+  Value *LastLane = Builder.CreateSub(ConstantInt::get(IndexTy, 1), RunTimeVF);
   Value *Ptr = State.get(getOperand(0), VPLane(0));
   bool InBounds = isInBounds();
+  Value *ResultPtr = Builder.CreateGEP(IndexedTy, Ptr, NumElt, "", InBounds);
+  ResultPtr = Builder.CreateGEP(IndexedTy, ResultPtr, LastLane, "", InBounds);
 
-  Value *ResultPtr = nullptr;
-  if (IsReverse) {
-    // If the address is consecutive but reversed, then the
-    // wide store needs to start at the last vector element.
-    // RunTimeVF =  VScale * VF.getKnownMinValue()
-    // For fixed-width VScale is 1, then RunTimeVF = VF.getKnownMinValue()
-    Value *RunTimeVF = getRuntimeVF(Builder, IndexTy, State.VF);
-    // NumElt = -CurrentPart * RunTimeVF
-    Value *NumElt = Builder.CreateMul(
-        ConstantInt::get(IndexTy, -(int64_t)CurrentPart), RunTimeVF);
-    // LastLane = 1 - RunTimeVF
-    Value *LastLane =
-        Builder.CreateSub(ConstantInt::get(IndexTy, 1), RunTimeVF);
-    ResultPtr = Builder.CreateGEP(IndexedTy, Ptr, NumElt, "", InBounds);
-    ResultPtr = Builder.CreateGEP(IndexedTy, ResultPtr, LastLane, "", InBounds);
-  } else {
-    Value *Increment = createStepForVF(Builder, IndexTy, State.VF, CurrentPart);
-    ResultPtr = Builder.CreateGEP(IndexedTy, Ptr, Increment, "", InBounds);
-  }
+  State.set(this, ResultPtr, /*IsScalar*/ true);
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPReverseVectorPointerRecipe::print(raw_ostream &O, const Twine &Indent,
+                                         VPSlotTracker &SlotTracker) const {
+  O << Indent;
+  printAsOperand(O, SlotTracker);
+  O << " = reverse-vector-pointer ";
+  if (isInBounds())
+    O << "inbounds ";
+  printOperands(O, SlotTracker);
+}
+#endif
+
+void VPVectorPointerRecipe::execute(VPTransformState &State) {
+  auto &Builder = State.Builder;
+  State.setDebugLocFrom(getDebugLoc());
+  unsigned CurrentPart = getUnrollPart(*this);
+  Type *IndexTy = getGEPIndexTy(State.VF.isScalable(), /*IsReverse*/ false,
+                                CurrentPart, Builder);
+  Value *Ptr = State.get(getOperand(0), VPLane(0));
+  bool InBounds = isInBounds();
+
+  Value *Increment = createStepForVF(Builder, IndexTy, State.VF, CurrentPart);
+  Value *ResultPtr = Builder.CreateGEP(IndexedTy, Ptr, Increment, "", InBounds);
 
   State.set(this, ResultPtr, /*IsScalar*/ true);
 }
@@ -2013,8 +2039,6 @@ void VPVectorPointerRecipe::print(raw_ostream &O, const Twine &Indent,
   O << Indent;
   printAsOperand(O, SlotTracker);
   O << " = vector-pointer ";
-  if (IsReverse)
-    O << "(reverse) ";
 
   printOperands(O, SlotTracker);
 }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index ca78f32506ef..1e32865e8ee5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -316,12 +316,12 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
     // Add operand indicating the part to generate code for, to recipes still
     // requiring it.
     if (isa<VPScalarIVStepsRecipe, VPWidenCanonicalIVRecipe,
-            VPVectorPointerRecipe>(Copy) ||
+            VPVectorPointerRecipe, VPReverseVectorPointerRecipe>(Copy) ||
         match(Copy, m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>(
                         m_VPValue())))
       Copy->addOperand(getConstantVPV(Part));
 
-    if (isa<VPVectorPointerRecipe>(R))
+    if (isa<VPVectorPointerRecipe, VPReverseVectorPointerRecipe>(R))
       Copy->setOperand(0, R.getOperand(0));
   }
 }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 1900182f76e0..89b3ed72b8eb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -346,6 +346,7 @@ public:
     VPScalarCastSC,
     VPScalarIVStepsSC,
     VPVectorPointerSC,
+    VPReverseVectorPointerSC,
     VPWidenCallSC,
     VPWidenCanonicalIVSC,
     VPWidenCastSC,
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
index 81121019efe7..76562e80fbc4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
@@ -24,43 +24,36 @@ define void @vector_reverse_f64(i64 %N, ptr noalias %a, ptr noalias %b) #0{
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 3
+; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[INDEX]], -1
-; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[N]], [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds double, ptr [[B:%.*]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = shl i64 [[TMP9]], 3
-; CHECK-NEXT:    [[TMP11:%.*]] = sub i64 1, [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP8]], i64 [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP14:%.*]] = shl i64 [[TMP13]], 3
-; CHECK-NEXT:    [[TMP15:%.*]] = sub i64 0, [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = sub i64 1, [[TMP14]]
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds double, ptr [[TMP8]], i64 [[TMP15]]
-; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double, ptr [[TMP17]], i64 [[TMP16]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x double>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x double>, ptr [[TMP18]], align 8
-; CHECK-NEXT:    [[TMP19:%.*]] = fadd <vscale x 8 x double> [[WIDE_LOAD]], shufflevector (<vscale x 8 x double> insertelement (<vscale x 8 x double> poison, double 1.000000e+00, i64 0), <vscale x 8 x double> poison, <vscale x 8 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP20:%.*]] = fadd <vscale x 8 x double> [[WIDE_LOAD1]], shufflevector (<vscale x 8 x double> insertelement (<vscale x 8 x double> poison, double 1.000000e+00, i64 0), <vscale x 8 x double> poison, <vscale x 8 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP22]], 3
-; CHECK-NEXT:    [[TMP24:%.*]] = sub i64 1, [[TMP23]]
-; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds double, ptr [[TMP21]], i64 [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP27:%.*]] = shl i64 [[TMP26]], 3
-; CHECK-NEXT:    [[TMP28:%.*]] = sub i64 0, [[TMP27]]
-; CHECK-NEXT:    [[TMP29:%.*]] = sub i64 1, [[TMP27]]
-; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds double, ptr [[TMP21]], i64 [[TMP28]]
-; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds double, ptr [[TMP30]], i64 [[TMP29]]
-; CHECK-NEXT:    store <vscale x 8 x double> [[TMP19]], ptr [[TMP25]], align 8
-; CHECK-NEXT:    store <vscale x 8 x double> [[TMP20]], ptr [[TMP31]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[INDEX]], -1
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[N]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, ptr [[B:%.*]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub i64 1, [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = sub i64 1, [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[TMP13]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x double>, ptr [[TMP11]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x double>, ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = fadd <vscale x 8 x double> [[WIDE_LOAD]], shufflevector (<vscale x 8 x double> insertelement (<vscale x 8 x double> poison, double 1.000000e+00, i64 0), <vscale x 8 x double> poison, <vscale x 8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP17:%.*]] = fadd <vscale x 8 x double> [[WIDE_LOAD1]], shufflevector (<vscale x 8 x double> insertelement (<vscale x 8 x double> poison, double 1.000000e+00, i64 0), <vscale x 8 x double> poison, <vscale x 8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP19:%.*]] = sub i64 1, [[TMP5]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds double, ptr [[TMP18]], i64 [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = sub i64 1, [[TMP5]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds double, ptr [[TMP18]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds double, ptr [[TMP23]], i64 [[TMP22]]
+; CHECK-NEXT:    store <vscale x 8 x double> [[TMP16]], ptr [[TMP20]], align 8
+; CHECK-NEXT:    store <vscale x 8 x double> [[TMP17]], ptr [[TMP24]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
@@ -75,8 +68,8 @@ define void @vector_reverse_f64(i64 %N, ptr noalias %a, ptr noalias %b) #0{
 ; CHECK-NEXT:    [[I_08_IN:%.*]] = phi i64 [ [[I_08:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[I_08]] = add nsw i64 [[I_08_IN]], -1
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[I_08]]
-; CHECK-NEXT:    [[TMP33:%.*]] = load double, ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP33]], 1.000000e+00
+; CHECK-NEXT:    [[TMP26:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP26]], 1.000000e+00
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[I_08]]
 ; CHECK-NEXT:    store double [[ADD]], ptr [[ARRAYIDX1]], align 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[I_08_IN]], 1
@@ -126,43 +119,36 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP7]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP7]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[INDEX]], -1
-; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[N]], [[TMP9]]
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = shl i64 [[TMP12]], 3
-; CHECK-NEXT:    [[TMP14:%.*]] = sub i64 1, [[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i64 [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP17:%.*]] = shl i64 [[TMP16]], 3
-; CHECK-NEXT:    [[TMP18:%.*]] = sub i64 0, [[TMP17]]
-; CHECK-NEXT:    [[TMP19:%.*]] = sub i64 1, [[TMP17]]
-; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i64 [[TMP18]]
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[TMP20]], i64 [[TMP19]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP15]], align 8
-; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i64>, ptr [[TMP21]], align 8
-; CHECK-NEXT:    [[TMP22:%.*]] = add <vscale x 8 x i64> [[WIDE_LOAD]], shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 1, i64 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP23:%.*]] = add <vscale x 8 x i64> [[WIDE_LOAD3]], shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 1, i64 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP10]]
-; CHECK-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP26:%.*]] = shl i64 [[TMP25]], 3
-; CHECK-NEXT:    [[TMP27:%.*]] = sub i64 1, [[TMP26]]
-; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i64, ptr [[TMP24]], i64 [[TMP27]]
-; CHECK-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP30:%.*]] = shl i64 [[TMP29]], 3
-; CHECK-NEXT:    [[TMP31:%.*]] = sub i64 0, [[TMP30]]
-; CHECK-NEXT:    [[TMP32:%.*]] = sub i64 1, [[TMP30]]
-; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i64, ptr [[TMP24]], i64 [[TMP31]]
-; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i64, ptr [[TMP33]], i64 [[TMP32]]
-; CHECK-NEXT:    store <vscale x 8 x i64> [[TMP22]], ptr [[TMP28]], align 8
-; CHECK-NEXT:    store <vscale x 8 x i64> [[TMP23]], ptr [[TMP34]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
-; CHECK-NEXT:    [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP35]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[INDEX]], -1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[N]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = sub i64 1, [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = sub i64 0, [[TMP8]]
+; CHECK-NEXT:    [[TMP16:%.*]] = sub i64 1, [[TMP8]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP17]], i64 [[TMP16]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i64>, ptr [[TMP18]], align 8
+; CHECK-NEXT:    [[TMP19:%.*]] = add <vscale x 8 x i64> [[WIDE_LOAD]], shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 1, i64 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP20:%.*]] = add <vscale x 8 x i64> [[WIDE_LOAD3]], shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 1, i64 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP22:%.*]] = sub i64 1, [[TMP8]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i64, ptr [[TMP21]], i64 [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = sub i64 0, [[TMP8]]
+; CHECK-NEXT:    [[TMP25:%.*]] = sub i64 1, [[TMP8]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i64, ptr [[TMP21]], i64 [[TMP24]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i64, ptr [[TMP26]], i64 [[TMP25]]
+; CHECK-NEXT:    store <vscale x 8 x i64> [[TMP19]], ptr [[TMP23]], align 8
+; CHECK-NEXT:    store <vscale x 8 x i64> [[TMP20]], ptr [[TMP27]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
@@ -177,8 +163,8 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    [[I_09_IN:%.*]] = phi i64 [ [[I_09:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[I_09]] = add nsw i64 [[I_09_IN]], -1
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[I_09]]
-; CHECK-NEXT:    [[TMP36:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[TMP36]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[TMP29]], 1
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I_09]]
 ; CHECK-NEXT:    store i64 [[ADD]], ptr [[ARRAYIDX2]], align 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[I_09_IN]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index 9a716f775607..c7bb1ffab23e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -54,6 +54,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1
 ; CHECK-NEXT:  LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' {
+; CHECK-NEXT:  Live-in vp<[[VF:%.+]]> = VF
 ; CHECK-NEXT:  Live-in vp<[[VFxUF:%.+]]> = VF * UF
 ; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT:  vp<[[TC:%.+]]> = original trip-count
@@ -74,11 +75,11 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
 ; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
+; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx>, vp<[[VF]]>
 ; CHECK-NEXT:      WIDEN ir<%1> = load vp<[[VEC_PTR]]>
 ; CHECK-NEXT:      WIDEN ir<%add9> = add ir<%1>, ir<1>
 ; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
+; CHECK-NEXT:      vp<[[VEC_PTR2:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx3>, vp<[[VF]]>
 ; CHECK-NEXT:      WIDEN store vp<[[VEC_PTR2]]>, ir<%add9>
 ; CHECK-NEXT:      EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:      EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
@@ -138,6 +139,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LEV: Epilogue vectorization is not profitable for this loop
 ; CHECK-NEXT:  Executing best plan with VF=vscale x 4, UF=1
 ; CHECK-NEXT:  VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT:  Live-in vp<[[VF:%.+]]> = VF
 ; CHECK-NEXT:  Live-in vp<[[VFxUF:%.+]]> = VF * UF
 ; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT:  vp<[[TC:%.+]]> = original trip-count
@@ -158,11 +160,11 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
 ; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
+; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx>, vp<[[VF]]>
 ; CHECK-NEXT:      WIDEN ir<%13> = load vp<[[VEC_PTR]]>
 ; CHECK-NEXT:      WIDEN ir<%add9> = add ir<%13>, ir<1>
 ; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
+; CHECK-NEXT:      vp<[[VEC_PTR2:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx3>, vp<[[VF]]>
 ; CHECK-NEXT:      WIDEN store vp<[[VEC_PTR2]]>, ir<%add9>
 ; CHECK-NEXT:      EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:      EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
@@ -259,6 +261,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1
 ; CHECK-NEXT:  LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' {
+; CHECK-NEXT:  Live-in vp<[[VF:%.+]]> = VF
 ; CHECK-NEXT:  Live-in vp<[[VFxUF:%.+]]> = VF * UF
 ; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT:  vp<[[TC:%.+]]> = original trip-count
@@ -279,11 +282,11 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
 ; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
+; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx>, vp<[[VF]]>
 ; CHECK-NEXT:      WIDEN ir<%1> = load vp<[[VEC_PTR]]>
 ; CHECK-NEXT:      WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00>
 ; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
+; CHECK-NEXT:      vp<[[VEC_PTR2:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx3>, vp<[[VF]]>
 ; CHECK-NEXT:      WIDEN store vp<[[VEC_PTR2]]>, ir<%conv1>
 ; CHECK-NEXT:      EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:      EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
@@ -343,6 +346,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LEV: Epilogue vectorization is not profitable for this loop
 ; CHECK-NEXT:  Executing best plan with VF=vscale x 4, UF=1
 ; CHECK-NEXT:  VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT:  Live-in vp<[[VF:%.+]]> = VF
 ; CHECK-NEXT:  Live-in vp<[[VFxUF:%.+]]> = VF * UF
 ; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT:  vp<[[TC:%.+]]> = original trip-count
@@ -363,11 +367,11 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
 ; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
+; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx>, vp<[[VF]]>
 ; CHECK-NEXT:      WIDEN ir<%13> = load vp<[[VEC_PTR]]>
 ; CHECK-NEXT:      WIDEN ir<%conv1> = fadd ir<%13>, ir<1.000000e+00>
 ; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
+; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx3>, vp<[[VF]]>
 ; CHECK-NEXT:      WIDEN store vp<[[VEC_PTR]]>, ir<%conv1>
 ; CHECK-NEXT:      EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:      EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
index c1cf8b0fc541..9a001f36da7d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
@@ -28,34 +28,30 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[TMP5:%.*]] = sub i64 1024, [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP5]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 1024, [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 [[STARTVAL]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 0
-; IF-EVL-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], -1
-; IF-EVL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 [[TMP8]]
-; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
-; IF-EVL-NEXT:    [[TMP12:%.*]] = mul i64 0, [[TMP11]]
-; IF-EVL-NEXT:    [[TMP13:%.*]] = sub i64 1, [[TMP11]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 [[TMP12]]
-; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i64 [[TMP13]]
-; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP15]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP6]])
-; IF-EVL-NEXT:    [[VP_REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP6]])
-; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[PTR2:%.*]], i64 [[TMP8]]
-; IF-EVL-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 4
-; IF-EVL-NEXT:    [[TMP19:%.*]] = mul i64 0, [[TMP18]]
-; IF-EVL-NEXT:    [[TMP20:%.*]] = sub i64 1, [[TMP18]]
-; IF-EVL-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i64 [[TMP19]]
-; IF-EVL-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i64 [[TMP20]]
-; IF-EVL-NEXT:    [[VP_REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_REVERSE]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP6]])
-; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[VP_REVERSE3]], ptr align 4 [[TMP22]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP6]])
-; IF-EVL-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP6]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP23]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 0
+; IF-EVL-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], -1
+; IF-EVL-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 [[TMP7]]
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 0, [[TMP4]]
+; IF-EVL-NEXT:    [[TMP10:%.*]] = sub i64 1, [[TMP4]]
+; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 [[TMP9]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP5]])
+; IF-EVL-NEXT:    [[VP_REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP5]])
+; IF-EVL-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[PTR2:%.*]], i64 [[TMP7]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = mul i64 0, [[TMP4]]
+; IF-EVL-NEXT:    [[TMP15:%.*]] = sub i64 1, [[TMP4]]
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i64 [[TMP14]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i64 [[TMP15]]
+; IF-EVL-NEXT:    [[VP_REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_REVERSE]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP5]])
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[VP_REVERSE3]], ptr align 4 [[TMP17]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP5]])
+; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP5]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
-; IF-EVL-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br i1 true, label [[LOOPEND:%.*]], label [[SCALAR_PH]]
 ; IF-EVL:       scalar.ph:
@@ -131,49 +127,45 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[TMP5:%.*]] = sub i64 1024, [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP5]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 1024, [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 [[STARTVAL]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 0
+; IF-EVL-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; IF-EVL-NEXT:    [[OFFSET_IDX3:%.*]] = trunc i64 [[EVL_BASED_IV]] to i32
-; IF-EVL-NEXT:    [[TMP8:%.*]] = add i32 [[OFFSET_IDX3]], 0
+; IF-EVL-NEXT:    [[TMP7:%.*]] = add i32 [[OFFSET_IDX3]], 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[EVL_BASED_IV]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; IF-EVL-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP9]]
-; IF-EVL-NEXT:    [[VEC_IV:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP10]]
-; IF-EVL-NEXT:    [[TMP11:%.*]] = icmp ule <vscale x 4 x i64> [[VEC_IV]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1023, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
-; IF-EVL-NEXT:    [[TMP12:%.*]] = add i64 [[TMP7]], -1
-; IF-EVL-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i32 [[TMP8]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i32 0
-; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP6]])
-; IF-EVL-NEXT:    [[TMP15:%.*]] = icmp slt <vscale x 4 x i32> [[VP_OP_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 100, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
-; IF-EVL-NEXT:    [[TMP16:%.*]] = select <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i1> zeroinitializer
-; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr i32, ptr [[PTR1:%.*]], i64 [[TMP12]]
-; IF-EVL-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 4
-; IF-EVL-NEXT:    [[TMP20:%.*]] = mul i64 0, [[TMP19]]
-; IF-EVL-NEXT:    [[TMP21:%.*]] = sub i64 1, [[TMP19]]
-; IF-EVL-NEXT:    [[TMP22:%.*]] = getelementptr i32, ptr [[TMP17]], i64 [[TMP20]]
-; IF-EVL-NEXT:    [[TMP23:%.*]] = getelementptr i32, ptr [[TMP22]], i64 [[TMP21]]
-; IF-EVL-NEXT:    [[VP_REVERSE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.experimental.vp.reverse.nxv4i1(<vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP6]])
-; IF-EVL-NEXT:    [[VP_OP_LOAD4:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP23]], <vscale x 4 x i1> [[VP_REVERSE_MASK]], i32 [[TMP6]])
-; IF-EVL-NEXT:    [[VP_REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD4]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP6]])
-; IF-EVL-NEXT:    [[TMP24:%.*]] = getelementptr i32, ptr [[PTR2:%.*]], i64 [[TMP12]]
-; IF-EVL-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 4
-; IF-EVL-NEXT:    [[TMP27:%.*]] = mul i64 0, [[TMP26]]
-; IF-EVL-NEXT:    [[TMP28:%.*]] = sub i64 1, [[TMP26]]
-; IF-EVL-NEXT:    [[TMP29:%.*]] = getelementptr i32, ptr [[TMP24]], i64 [[TMP27]]
-; IF-EVL-NEXT:    [[TMP30:%.*]] = getelementptr i32, ptr [[TMP29]], i64 [[TMP28]]
-; IF-EVL-NEXT:    [[VP_REVERSE5:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_REVERSE]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP6]])
-; IF-EVL-NEXT:    [[VP_REVERSE_MASK6:%.*]] = call <vscale x 4 x i1> @llvm.experimental.vp.reverse.nxv4i1(<vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP6]])
-; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[VP_REVERSE5]], ptr align 4 [[TMP30]], <vscale x 4 x i1> [[VP_REVERSE_MASK6]], i32 [[TMP6]])
-; IF-EVL-NEXT:    [[TMP31:%.*]] = zext i32 [[TMP6]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP31]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; IF-EVL-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP8]]
+; IF-EVL-NEXT:    [[VEC_IV:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP9]]
+; IF-EVL-NEXT:    [[TMP10:%.*]] = icmp ule <vscale x 4 x i64> [[VEC_IV]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1023, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; IF-EVL-NEXT:    [[TMP11:%.*]] = add i64 [[TMP6]], -1
+; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i32 [[TMP7]]
+; IF-EVL-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP13]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP5]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = icmp slt <vscale x 4 x i32> [[VP_OP_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 100, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; IF-EVL-NEXT:    [[TMP15:%.*]] = select <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> zeroinitializer
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr i32, ptr [[PTR1:%.*]], i64 [[TMP11]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = mul i64 0, [[TMP4]]
+; IF-EVL-NEXT:    [[TMP18:%.*]] = sub i64 1, [[TMP4]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = getelementptr i32, ptr [[TMP16]], i64 [[TMP17]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = getelementptr i32, ptr [[TMP19]], i64 [[TMP18]]
+; IF-EVL-NEXT:    [[VP_REVERSE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.experimental.vp.reverse.nxv4i1(<vscale x 4 x i1> [[TMP15]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP5]])
+; IF-EVL-NEXT:    [[VP_OP_LOAD4:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP20]], <vscale x 4 x i1> [[VP_REVERSE_MASK]], i32 [[TMP5]])
+; IF-EVL-NEXT:    [[VP_REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD4]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP5]])
+; IF-EVL-NEXT:    [[TMP21:%.*]] = getelementptr i32, ptr [[PTR2:%.*]], i64 [[TMP11]]
+; IF-EVL-NEXT:    [[TMP22:%.*]] = mul i64 0, [[TMP4]]
+; IF-EVL-NEXT:    [[TMP23:%.*]] = sub i64 1, [[TMP4]]
+; IF-EVL-NEXT:    [[TMP24:%.*]] = getelementptr i32, ptr [[TMP21]], i64 [[TMP22]]
+; IF-EVL-NEXT:    [[TMP25:%.*]] = getelementptr i32, ptr [[TMP24]], i64 [[TMP23]]
+; IF-EVL-NEXT:    [[VP_REVERSE5:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_REVERSE]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP5]])
+; IF-EVL-NEXT:    [[VP_REVERSE_MASK6:%.*]] = call <vscale x 4 x i1> @llvm.experimental.vp.reverse.nxv4i1(<vscale x 4 x i1> [[TMP15]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP5]])
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[VP_REVERSE5]], ptr align 4 [[TMP25]], <vscale x 4 x i1> [[VP_REVERSE_MASK6]], i32 [[TMP5]])
+; IF-EVL-NEXT:    [[TMP26:%.*]] = zext i32 [[TMP5]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP26]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
-; IF-EVL-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IF-EVL-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br i1 true, label [[LOOPEND:%.*]], label [[SCALAR_PH]]
 ; IF-EVL:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
index 870925950ae4..c492b296903e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
@@ -38,10 +38,8 @@ define void @lshift_significand(i32 %n, ptr nocapture writeonly %dst) {
 ; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP13:%.*]] = sub nuw nsw i64 1, [[TMP12]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 2
-; CHECK-NEXT:    [[TMP17:%.*]] = mul i64 0, [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = sub i64 1, [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = mul i64 0, [[TMP9]]
+; CHECK-NEXT:    [[TMP18:%.*]] = sub i64 1, [[TMP9]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i64, ptr [[TMP14]], i64 [[TMP17]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i64, ptr [[TMP19]], i64 [[TMP18]]
 ; CHECK-NEXT:    [[VP_REVERSE:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.reverse.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP11]])
diff --git a/llvm/test/Transforms/LoopVectorize/reverse_induction.ll b/llvm/test/Transforms/LoopVectorize/reverse_induction.ll
index c1322792071e..d983c5138164 100644
--- a/llvm/test/Transforms/LoopVectorize/reverse_induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/reverse_induction.ll
@@ -20,11 +20,11 @@ define i32 @reverse_induction_i64(i64 %startval, ptr %ptr) {
 ; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 [[STARTVAL]], [[INDEX]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], -1
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP0]], -1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i32 -3
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 -4
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 -4
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 -3
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4
 ; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
@@ -93,11 +93,11 @@ define i32 @reverse_induction_i128(i128 %startval, ptr %ptr) {
 ; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i128 [[STARTVAL]], [[INDEX]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i128 [[OFFSET_IDX]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = add i128 [[TMP0]], -1
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i128 [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = add i128 [[TMP0]], -1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i128 [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i32 -3
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 -4
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 -4
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 -3
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4
 ; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
@@ -176,11 +176,11 @@ define i32 @reverse_induction_i16(i16 %startval, ptr %ptr) {
 ; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i16 [[STARTVAL]], [[DOTCAST]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i16 [[OFFSET_IDX]], 0
-; CHECK-NEXT:    [[TMP8:%.*]] = add i16 [[TMP4]], -1
-; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i16 [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = add i16 [[TMP4]], -1
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i16 [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i32 0
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i32 -3
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i32 -4
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i32 -4
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i32 -3
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
 ; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 0f3cd9d4ca4d..446b720ad1ba 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -1101,6 +1101,7 @@ exit:
 define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
 ; CHECK-LABEL: LV: Checking a loop in 'ptr_induction_remove_dead_recipe'
 ; CHECK:       VPlan 'Initial VPlan for VF={2},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF
 ; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
@@ -1115,11 +1116,11 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT:     vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<0> + vp<%3> * ir<-1>
+; CHECK-NEXT:     vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<-1>
 ; CHECK-NEXT:     vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
 ; CHECK-NEXT:     EMIT vp<[[PTR_IV:%.+]]> = ptradd ir<%start>, vp<[[STEPS]]>
 ; CHECK-NEXT:     CLONE ir<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
-; CHECK-NEXT:     vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%ptr.iv.next>
+; CHECK-NEXT:     vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%ptr.iv.next>, vp<[[VF]]>
 ; CHECK-NEXT:     WIDEN ir<%l> = load vp<[[VEC_PTR]]>
 ; CHECK-NEXT:     WIDEN ir<%c.1> = icmp eq ir<%l>, ir<0>
 ; CHECK-NEXT:     EMIT vp<[[NEG:%.+]]> = not ir<%c.1>
-- 
GitLab


From a917ae0b4fc0d792ee0e2c512c4ea539f98e1204 Mon Sep 17 00:00:00 2001
From: isuckatcs <65320245+isuckatcs@users.noreply.github.com>
Date: Sat, 26 Oct 2024 17:41:55 +0200
Subject: [PATCH 238/693] [analyzer] Fix a crash from element region
 construction during `ArrayInitLoopExpr` analysis (#113570)

This patch generalizes the way element regions are constructed when an
`ArrayInitLoopExpr` is being analyzed. Previously the base region of the
`ElementRegion` was determined with pattern matching, which led to
crashes, when an unhandled pattern was encountered.

Fixes #112813
---
 .../lib/StaticAnalyzer/Core/ExprEngineCXX.cpp | 69 ++++---------------
 clang/test/Analysis/array-init-loop.cpp       | 38 ++++++++++
 2 files changed, 50 insertions(+), 57 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
index c50db1e0e2f8..ccc3097e8d2f 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
@@ -513,70 +513,25 @@ ProgramStateRef ExprEngine::updateObjectsUnderConstruction(
 static ProgramStateRef
 bindRequiredArrayElementToEnvironment(ProgramStateRef State,
                                       const ArrayInitLoopExpr *AILE,
-                                      const LocationContext *LCtx, SVal Idx) {
-  // The ctor in this case is guaranteed to be a copy ctor, otherwise we hit a
-  // compile time error.
-  //
-  //  -ArrayInitLoopExpr                <-- we're here
-  //   |-OpaqueValueExpr
-  //   | `-DeclRefExpr                  <-- match this
-  //   `-CXXConstructExpr
-  //     `-ImplicitCastExpr
-  //       `-ArraySubscriptExpr
-  //         |-ImplicitCastExpr
-  //         | `-OpaqueValueExpr
-  //         |   `-DeclRefExpr
-  //         `-ArrayInitIndexExpr
-  //
-  // The resulting expression might look like the one below in an implicit
-  // copy/move ctor.
-  //
-  //   ArrayInitLoopExpr                <-- we're here
-  //   |-OpaqueValueExpr
-  //   | `-MemberExpr                   <-- match this
-  //   |  (`-CXXStaticCastExpr)         <-- move ctor only
-  //   |     `-DeclRefExpr
-  //   `-CXXConstructExpr
-  //     `-ArraySubscriptExpr
-  //       |-ImplicitCastExpr
-  //       | `-OpaqueValueExpr
-  //       |   `-MemberExpr
-  //       |     `-DeclRefExpr
-  //       `-ArrayInitIndexExpr
-  //
-  // The resulting expression for a multidimensional array.
-  // ArrayInitLoopExpr                  <-- we're here
-  // |-OpaqueValueExpr
-  // | `-DeclRefExpr                    <-- match this
-  // `-ArrayInitLoopExpr
-  //   |-OpaqueValueExpr
-  //   | `-ArraySubscriptExpr
-  //   |   |-ImplicitCastExpr
-  //   |   | `-OpaqueValueExpr
-  //   |   |   `-DeclRefExpr
-  //   |   `-ArrayInitIndexExpr
-  //   `-CXXConstructExpr             <-- extract this
-  //     ` ...
-
-  const auto *OVESrc = AILE->getCommonExpr()->getSourceExpr();
+                                      const LocationContext *LCtx, NonLoc Idx) {
+  SValBuilder &SVB = State->getStateManager().getSValBuilder();
+  MemRegionManager &MRMgr = SVB.getRegionManager();
+  ASTContext &Ctx = SVB.getContext();
 
   // HACK: There is no way we can put the index of the array element into the
   // CFG unless we unroll the loop, so we manually select and bind the required
   // parameter to the environment.
-  const auto *CE =
+  const Expr *SourceArray = AILE->getCommonExpr()->getSourceExpr();
+  const auto *Ctor =
       cast<CXXConstructExpr>(extractElementInitializerFromNestedAILE(AILE));
 
-  SVal Base = UnknownVal();
-  if (const auto *ME = dyn_cast<MemberExpr>(OVESrc))
-    Base = State->getSVal(ME, LCtx);
-  else if (const auto *DRE = dyn_cast<DeclRefExpr>(OVESrc))
-    Base = State->getLValue(cast<VarDecl>(DRE->getDecl()), LCtx);
-  else
-    llvm_unreachable("ArrayInitLoopExpr contains unexpected source expression");
-
-  SVal NthElem = State->getLValue(CE->getType(), Idx, Base);
+  const auto *SourceArrayRegion =
+      cast<SubRegion>(State->getSVal(SourceArray, LCtx).getAsRegion());
+  const ElementRegion *ElementRegion =
+      MRMgr.getElementRegion(Ctor->getType(), Idx, SourceArrayRegion, Ctx);
 
-  return State->BindExpr(CE->getArg(0), LCtx, NthElem);
+  return State->BindExpr(Ctor->getArg(0), LCtx,
+                         loc::MemRegionVal(ElementRegion));
 }
 
 void ExprEngine::handleConstructor(const Expr *E,
diff --git a/clang/test/Analysis/array-init-loop.cpp b/clang/test/Analysis/array-init-loop.cpp
index 4ab4489fc882..b28468b7f560 100644
--- a/clang/test/Analysis/array-init-loop.cpp
+++ b/clang/test/Analysis/array-init-loop.cpp
@@ -330,3 +330,41 @@ void no_crash() {
 }
 
 } // namespace crash
+
+namespace array_subscript_initializer {
+struct S {
+  int x;
+};
+
+void no_crash() {
+  S arr[][2] = {{1, 2}};
+
+  const auto [a, b] = arr[0]; // no-crash
+
+  clang_analyzer_eval(a.x == 1); // expected-warning{{TRUE}}
+  clang_analyzer_eval(b.x == 2); // expected-warning{{TRUE}}
+}
+} // namespace array_subscript_initializer
+
+namespace iterator_initializer {
+struct S {
+  int x;
+};
+
+void no_crash() {
+  S arr[][2] = {{1, 2}, {3, 4}};
+
+  int i = 0;
+  for (const auto [a, b] : arr) { // no-crash
+    if (i == 0) {
+      clang_analyzer_eval(a.x == 1); // expected-warning{{TRUE}}
+      clang_analyzer_eval(b.x == 2); // expected-warning{{TRUE}}
+    } else {
+      clang_analyzer_eval(a.x == 3); // expected-warning{{TRUE}}
+      clang_analyzer_eval(b.x == 4); // expected-warning{{TRUE}}
+    }
+
+    ++i;
+  }
+}
+} // namespace iterator_initializer
-- 
GitLab


From b667d161f0a9ff6b29cda0ccdb0081610c1e8b8c Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyulee@meta.com>
Date: Sat, 26 Oct 2024 09:20:26 -0700
Subject: [PATCH 239/693] [StructuralHash] Refactor (#112621)

This is largely NFC, and it prepares for #112638.
 - Use stable_hash instead of uint64_t
- Rename update* to hash* functions. They compute stable_hash locally and return it.

This is a patch for
https://discourse.llvm.org/t/rfc-global-function-merging/82608.
---
 llvm/include/llvm/IR/StructuralHash.h         |   7 +-
 llvm/lib/IR/StructuralHash.cpp                | 133 +++++++++++-------
 llvm/lib/Transforms/IPO/MergeFunctions.cpp    |   6 +-
 .../MergeFunc/call-and-invoke-with-ranges.ll  |  17 ++-
 4 files changed, 100 insertions(+), 63 deletions(-)

diff --git a/llvm/include/llvm/IR/StructuralHash.h b/llvm/include/llvm/IR/StructuralHash.h
index 57fb45db8491..e2e192cc9501 100644
--- a/llvm/include/llvm/IR/StructuralHash.h
+++ b/llvm/include/llvm/IR/StructuralHash.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_IR_STRUCTURALHASH_H
 #define LLVM_IR_STRUCTURALHASH_H
 
+#include "llvm/ADT/StableHashing.h"
 #include <cstdint>
 
 namespace llvm {
@@ -21,20 +22,18 @@ namespace llvm {
 class Function;
 class Module;
 
-using IRHash = uint64_t;
-
 /// Returns a hash of the function \p F.
 /// \param F The function to hash.
 /// \param DetailedHash Whether or not to encode additional information in the
 /// hash. The additional information added into the hash when this flag is set
 /// to true includes instruction and operand type information.
-IRHash StructuralHash(const Function &F, bool DetailedHash = false);
+stable_hash StructuralHash(const Function &F, bool DetailedHash = false);
 
 /// Returns a hash of the module \p M by hashing all functions and global
 /// variables contained within. \param M The module to hash. \param DetailedHash
 /// Whether or not to encode additional information in the function hashes that
 /// composed the module hash.
-IRHash StructuralHash(const Module &M, bool DetailedHash = false);
+stable_hash StructuralHash(const Module &M, bool DetailedHash = false);
 
 } // end namespace llvm
 
diff --git a/llvm/lib/IR/StructuralHash.cpp b/llvm/lib/IR/StructuralHash.cpp
index fb4f33a021a9..267a085c5af7 100644
--- a/llvm/lib/IR/StructuralHash.cpp
+++ b/llvm/lib/IR/StructuralHash.cpp
@@ -24,61 +24,93 @@ namespace {
 // by the MergeFunctions pass.
 
 class StructuralHashImpl {
-  uint64_t Hash = 4;
+  stable_hash Hash = 4;
 
-  void hash(uint64_t V) { Hash = hashing::detail::hash_16_bytes(Hash, V); }
+  bool DetailedHash;
+
+  // This random value acts as a block header, as otherwise the partition of
+  // opcodes into BBs wouldn't affect the hash, only the order of the opcodes.
+  static constexpr stable_hash BlockHeaderHash = 45798;
+  static constexpr stable_hash FunctionHeaderHash = 0x62642d6b6b2d6b72;
+  static constexpr stable_hash GlobalHeaderHash = 23456;
 
   // This will produce different values on 32-bit and 64-bit systens as
   // hash_combine returns a size_t. However, this is only used for
   // detailed hashing which, in-tree, only needs to distinguish between
   // differences in functions.
-  template <typename T> void hashArbitaryType(const T &V) {
-    hash(hash_combine(V));
+  // TODO: This is not stable.
+  template <typename T> stable_hash hashArbitaryType(const T &V) {
+    return hash_combine(V);
   }
 
-  void hashType(Type *ValueType) {
-    hash(ValueType->getTypeID());
+  stable_hash hashType(Type *ValueType) {
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(ValueType->getTypeID());
     if (ValueType->isIntegerTy())
-      hash(ValueType->getIntegerBitWidth());
+      Hashes.emplace_back(ValueType->getIntegerBitWidth());
+    return stable_hash_combine(Hashes);
   }
 
 public:
-  StructuralHashImpl() = default;
-
-  void updateOperand(Value *Operand) {
-    hashType(Operand->getType());
-
-    // The cases enumerated below are not exhaustive and are only aimed to
-    // get decent coverage over the function.
-    if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(Operand)) {
-      hashArbitaryType(ConstInt->getValue());
-    } else if (ConstantFP *ConstFP = dyn_cast<ConstantFP>(Operand)) {
-      hashArbitaryType(ConstFP->getValue());
-    } else if (Argument *Arg = dyn_cast<Argument>(Operand)) {
-      hash(Arg->getArgNo());
-    } else if (Function *Func = dyn_cast<Function>(Operand)) {
+  StructuralHashImpl() = delete;
+  explicit StructuralHashImpl(bool DetailedHash) : DetailedHash(DetailedHash) {}
+
+  stable_hash hashConstant(Constant *C) {
+    SmallVector<stable_hash> Hashes;
+    // TODO: hashArbitaryType() is not stable.
+    if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(C)) {
+      Hashes.emplace_back(hashArbitaryType(ConstInt->getValue()));
+    } else if (ConstantFP *ConstFP = dyn_cast<ConstantFP>(C)) {
+      Hashes.emplace_back(hashArbitaryType(ConstFP->getValue()));
+    } else if (Function *Func = dyn_cast<Function>(C)) {
       // Hashing the name will be deterministic as LLVM's hashing infrastructure
       // has explicit support for hashing strings and will not simply hash
       // the pointer.
-      hashArbitaryType(Func->getName());
+      Hashes.emplace_back(hashArbitaryType(Func->getName()));
     }
+
+    return stable_hash_combine(Hashes);
+  }
+
+  stable_hash hashValue(Value *V) {
+    // Check constant and return its hash.
+    Constant *C = dyn_cast<Constant>(V);
+    if (C)
+      return hashConstant(C);
+
+    // Hash argument number.
+    SmallVector<stable_hash> Hashes;
+    if (Argument *Arg = dyn_cast<Argument>(V))
+      Hashes.emplace_back(Arg->getArgNo());
+
+    return stable_hash_combine(Hashes);
   }
 
-  void updateInstruction(const Instruction &Inst, bool DetailedHash) {
-    hash(Inst.getOpcode());
+  stable_hash hashOperand(Value *Operand) {
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(hashType(Operand->getType()));
+    Hashes.emplace_back(hashValue(Operand));
+    return stable_hash_combine(Hashes);
+  }
+
+  stable_hash hashInstruction(const Instruction &Inst) {
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(Inst.getOpcode());
 
     if (!DetailedHash)
-      return;
+      return stable_hash_combine(Hashes);
 
-    hashType(Inst.getType());
+    Hashes.emplace_back(hashType(Inst.getType()));
 
     // Handle additional properties of specific instructions that cause
     // semantic differences in the IR.
     if (const auto *ComparisonInstruction = dyn_cast<CmpInst>(&Inst))
-      hash(ComparisonInstruction->getPredicate());
+      Hashes.emplace_back(ComparisonInstruction->getPredicate());
 
     for (const auto &Op : Inst.operands())
-      updateOperand(Op);
+      Hashes.emplace_back(hashOperand(Op));
+
+    return stable_hash_combine(Hashes);
   }
 
   // A function hash is calculated by considering only the number of arguments
@@ -97,15 +129,17 @@ public:
   // expensive checks for pass modification status). When modifying this
   // function, most changes should be gated behind an option and enabled
   // selectively.
-  void update(const Function &F, bool DetailedHash) {
+  void update(const Function &F) {
     // Declarations don't affect analyses.
     if (F.isDeclaration())
       return;
 
-    hash(0x62642d6b6b2d6b72); // Function header
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(Hash);
+    Hashes.emplace_back(FunctionHeaderHash);
 
-    hash(F.isVarArg());
-    hash(F.arg_size());
+    Hashes.emplace_back(F.isVarArg());
+    Hashes.emplace_back(F.arg_size());
 
     SmallVector<const BasicBlock *, 8> BBs;
     SmallPtrSet<const BasicBlock *, 16> VisitedBBs;
@@ -118,17 +152,17 @@ public:
     while (!BBs.empty()) {
       const BasicBlock *BB = BBs.pop_back_val();
 
-      // This random value acts as a block header, as otherwise the partition of
-      // opcodes into BBs wouldn't affect the hash, only the order of the
-      // opcodes
-      hash(45798);
+      Hashes.emplace_back(BlockHeaderHash);
       for (auto &Inst : *BB)
-        updateInstruction(Inst, DetailedHash);
+        Hashes.emplace_back(hashInstruction(Inst));
 
       for (const BasicBlock *Succ : successors(BB))
         if (VisitedBBs.insert(Succ).second)
           BBs.push_back(Succ);
     }
+
+    // Update the combined hash in place.
+    Hash = stable_hash_combine(Hashes);
   }
 
   void update(const GlobalVariable &GV) {
@@ -137,15 +171,20 @@ public:
     // we ignore anything with the `.llvm` prefix
     if (GV.isDeclaration() || GV.getName().starts_with("llvm."))
       return;
-    hash(23456); // Global header
-    hash(GV.getValueType()->getTypeID());
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(Hash);
+    Hashes.emplace_back(GlobalHeaderHash);
+    Hashes.emplace_back(GV.getValueType()->getTypeID());
+
+    // Update the combined hash in place.
+    Hash = stable_hash_combine(Hashes);
   }
 
-  void update(const Module &M, bool DetailedHash) {
+  void update(const Module &M) {
     for (const GlobalVariable &GV : M.globals())
       update(GV);
     for (const Function &F : M)
-      update(F, DetailedHash);
+      update(F);
   }
 
   uint64_t getHash() const { return Hash; }
@@ -153,14 +192,14 @@ public:
 
 } // namespace
 
-IRHash llvm::StructuralHash(const Function &F, bool DetailedHash) {
-  StructuralHashImpl H;
-  H.update(F, DetailedHash);
+stable_hash llvm::StructuralHash(const Function &F, bool DetailedHash) {
+  StructuralHashImpl H(DetailedHash);
+  H.update(F);
   return H.getHash();
 }
 
-IRHash llvm::StructuralHash(const Module &M, bool DetailedHash) {
-  StructuralHashImpl H;
-  H.update(M, DetailedHash);
+stable_hash llvm::StructuralHash(const Module &M, bool DetailedHash) {
+  StructuralHashImpl H(DetailedHash);
+  H.update(M);
   return H.getHash();
 }
diff --git a/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index b50a700e0903..ad16b0b35014 100644
--- a/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -172,14 +172,14 @@ namespace {
 
 class FunctionNode {
   mutable AssertingVH<Function> F;
-  IRHash Hash;
+  stable_hash Hash;
 
 public:
   // Note the hash is recalculated potentially multiple times, but it is cheap.
   FunctionNode(Function *F) : F(F), Hash(StructuralHash(*F)) {}
 
   Function *getFunc() const { return F; }
-  IRHash getHash() const { return Hash; }
+  stable_hash getHash() const { return Hash; }
 
   /// Replace the reference to the function F by the function G, assuming their
   /// implementations are equal.
@@ -420,7 +420,7 @@ bool MergeFunctions::runOnModule(Module &M) {
 
   // All functions in the module, ordered by hash. Functions with a unique
   // hash value are easily eliminated.
-  std::vector<std::pair<IRHash, Function *>> HashedFuncs;
+  std::vector<std::pair<stable_hash, Function *>> HashedFuncs;
   for (Function &Func : M) {
     if (isEligibleForMerging(Func)) {
       HashedFuncs.push_back({StructuralHash(Func), &Func});
diff --git a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
index e7718ca84d31..0ceb363a67b1 100644
--- a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
+++ b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
@@ -63,6 +63,14 @@ lpad:
   resume { ptr, i32 } zeroinitializer
 }
 
+define i8 @call_with_same_range() {
+; CHECK-LABEL: @call_with_same_range
+; CHECK: tail call i8 @call_with_range
+  bitcast i8 0 to i8
+  %out = call i8 @dummy(), !range !0
+  ret i8 %out
+}
+
 define i8 @invoke_with_same_range() personality ptr undef {
 ; CHECK-LABEL: @invoke_with_same_range()
 ; CHECK: tail call i8 @invoke_with_range()
@@ -76,15 +84,6 @@ lpad:
   resume { ptr, i32 } zeroinitializer
 }
 
-define i8 @call_with_same_range() {
-; CHECK-LABEL: @call_with_same_range
-; CHECK: tail call i8 @call_with_range
-  bitcast i8 0 to i8
-  %out = call i8 @dummy(), !range !0
-  ret i8 %out
-}
-
-
 declare i8 @dummy();
 declare i32 @__gxx_personality_v0(...)
 
-- 
GitLab


From 96723756231e316e68b0dacb104862eecbee01a2 Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyulee@meta.com>
Date: Sat, 26 Oct 2024 09:55:21 -0700
Subject: [PATCH 240/693] Revert "[StructuralHash] Refactor (#112621)"

This reverts commit b667d161f0a9ff6b29cda0ccdb0081610c1e8b8c.
---
 llvm/include/llvm/IR/StructuralHash.h         |   7 +-
 llvm/lib/IR/StructuralHash.cpp                | 133 +++++++-----------
 llvm/lib/Transforms/IPO/MergeFunctions.cpp    |   6 +-
 .../MergeFunc/call-and-invoke-with-ranges.ll  |  17 +--
 4 files changed, 63 insertions(+), 100 deletions(-)

diff --git a/llvm/include/llvm/IR/StructuralHash.h b/llvm/include/llvm/IR/StructuralHash.h
index e2e192cc9501..57fb45db8491 100644
--- a/llvm/include/llvm/IR/StructuralHash.h
+++ b/llvm/include/llvm/IR/StructuralHash.h
@@ -14,7 +14,6 @@
 #ifndef LLVM_IR_STRUCTURALHASH_H
 #define LLVM_IR_STRUCTURALHASH_H
 
-#include "llvm/ADT/StableHashing.h"
 #include <cstdint>
 
 namespace llvm {
@@ -22,18 +21,20 @@ namespace llvm {
 class Function;
 class Module;
 
+using IRHash = uint64_t;
+
 /// Returns a hash of the function \p F.
 /// \param F The function to hash.
 /// \param DetailedHash Whether or not to encode additional information in the
 /// hash. The additional information added into the hash when this flag is set
 /// to true includes instruction and operand type information.
-stable_hash StructuralHash(const Function &F, bool DetailedHash = false);
+IRHash StructuralHash(const Function &F, bool DetailedHash = false);
 
 /// Returns a hash of the module \p M by hashing all functions and global
 /// variables contained within. \param M The module to hash. \param DetailedHash
 /// Whether or not to encode additional information in the function hashes that
 /// composed the module hash.
-stable_hash StructuralHash(const Module &M, bool DetailedHash = false);
+IRHash StructuralHash(const Module &M, bool DetailedHash = false);
 
 } // end namespace llvm
 
diff --git a/llvm/lib/IR/StructuralHash.cpp b/llvm/lib/IR/StructuralHash.cpp
index 267a085c5af7..fb4f33a021a9 100644
--- a/llvm/lib/IR/StructuralHash.cpp
+++ b/llvm/lib/IR/StructuralHash.cpp
@@ -24,93 +24,61 @@ namespace {
 // by the MergeFunctions pass.
 
 class StructuralHashImpl {
-  stable_hash Hash = 4;
+  uint64_t Hash = 4;
 
-  bool DetailedHash;
-
-  // This random value acts as a block header, as otherwise the partition of
-  // opcodes into BBs wouldn't affect the hash, only the order of the opcodes.
-  static constexpr stable_hash BlockHeaderHash = 45798;
-  static constexpr stable_hash FunctionHeaderHash = 0x62642d6b6b2d6b72;
-  static constexpr stable_hash GlobalHeaderHash = 23456;
+  void hash(uint64_t V) { Hash = hashing::detail::hash_16_bytes(Hash, V); }
 
   // This will produce different values on 32-bit and 64-bit systens as
   // hash_combine returns a size_t. However, this is only used for
   // detailed hashing which, in-tree, only needs to distinguish between
   // differences in functions.
-  // TODO: This is not stable.
-  template <typename T> stable_hash hashArbitaryType(const T &V) {
-    return hash_combine(V);
+  template <typename T> void hashArbitaryType(const T &V) {
+    hash(hash_combine(V));
   }
 
-  stable_hash hashType(Type *ValueType) {
-    SmallVector<stable_hash> Hashes;
-    Hashes.emplace_back(ValueType->getTypeID());
+  void hashType(Type *ValueType) {
+    hash(ValueType->getTypeID());
     if (ValueType->isIntegerTy())
-      Hashes.emplace_back(ValueType->getIntegerBitWidth());
-    return stable_hash_combine(Hashes);
+      hash(ValueType->getIntegerBitWidth());
   }
 
 public:
-  StructuralHashImpl() = delete;
-  explicit StructuralHashImpl(bool DetailedHash) : DetailedHash(DetailedHash) {}
-
-  stable_hash hashConstant(Constant *C) {
-    SmallVector<stable_hash> Hashes;
-    // TODO: hashArbitaryType() is not stable.
-    if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(C)) {
-      Hashes.emplace_back(hashArbitaryType(ConstInt->getValue()));
-    } else if (ConstantFP *ConstFP = dyn_cast<ConstantFP>(C)) {
-      Hashes.emplace_back(hashArbitaryType(ConstFP->getValue()));
-    } else if (Function *Func = dyn_cast<Function>(C)) {
+  StructuralHashImpl() = default;
+
+  void updateOperand(Value *Operand) {
+    hashType(Operand->getType());
+
+    // The cases enumerated below are not exhaustive and are only aimed to
+    // get decent coverage over the function.
+    if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(Operand)) {
+      hashArbitaryType(ConstInt->getValue());
+    } else if (ConstantFP *ConstFP = dyn_cast<ConstantFP>(Operand)) {
+      hashArbitaryType(ConstFP->getValue());
+    } else if (Argument *Arg = dyn_cast<Argument>(Operand)) {
+      hash(Arg->getArgNo());
+    } else if (Function *Func = dyn_cast<Function>(Operand)) {
       // Hashing the name will be deterministic as LLVM's hashing infrastructure
       // has explicit support for hashing strings and will not simply hash
       // the pointer.
-      Hashes.emplace_back(hashArbitaryType(Func->getName()));
+      hashArbitaryType(Func->getName());
     }
-
-    return stable_hash_combine(Hashes);
-  }
-
-  stable_hash hashValue(Value *V) {
-    // Check constant and return its hash.
-    Constant *C = dyn_cast<Constant>(V);
-    if (C)
-      return hashConstant(C);
-
-    // Hash argument number.
-    SmallVector<stable_hash> Hashes;
-    if (Argument *Arg = dyn_cast<Argument>(V))
-      Hashes.emplace_back(Arg->getArgNo());
-
-    return stable_hash_combine(Hashes);
   }
 
-  stable_hash hashOperand(Value *Operand) {
-    SmallVector<stable_hash> Hashes;
-    Hashes.emplace_back(hashType(Operand->getType()));
-    Hashes.emplace_back(hashValue(Operand));
-    return stable_hash_combine(Hashes);
-  }
-
-  stable_hash hashInstruction(const Instruction &Inst) {
-    SmallVector<stable_hash> Hashes;
-    Hashes.emplace_back(Inst.getOpcode());
+  void updateInstruction(const Instruction &Inst, bool DetailedHash) {
+    hash(Inst.getOpcode());
 
     if (!DetailedHash)
-      return stable_hash_combine(Hashes);
+      return;
 
-    Hashes.emplace_back(hashType(Inst.getType()));
+    hashType(Inst.getType());
 
     // Handle additional properties of specific instructions that cause
     // semantic differences in the IR.
     if (const auto *ComparisonInstruction = dyn_cast<CmpInst>(&Inst))
-      Hashes.emplace_back(ComparisonInstruction->getPredicate());
+      hash(ComparisonInstruction->getPredicate());
 
     for (const auto &Op : Inst.operands())
-      Hashes.emplace_back(hashOperand(Op));
-
-    return stable_hash_combine(Hashes);
+      updateOperand(Op);
   }
 
   // A function hash is calculated by considering only the number of arguments
@@ -129,17 +97,15 @@ public:
   // expensive checks for pass modification status). When modifying this
   // function, most changes should be gated behind an option and enabled
   // selectively.
-  void update(const Function &F) {
+  void update(const Function &F, bool DetailedHash) {
     // Declarations don't affect analyses.
     if (F.isDeclaration())
       return;
 
-    SmallVector<stable_hash> Hashes;
-    Hashes.emplace_back(Hash);
-    Hashes.emplace_back(FunctionHeaderHash);
+    hash(0x62642d6b6b2d6b72); // Function header
 
-    Hashes.emplace_back(F.isVarArg());
-    Hashes.emplace_back(F.arg_size());
+    hash(F.isVarArg());
+    hash(F.arg_size());
 
     SmallVector<const BasicBlock *, 8> BBs;
     SmallPtrSet<const BasicBlock *, 16> VisitedBBs;
@@ -152,17 +118,17 @@ public:
     while (!BBs.empty()) {
       const BasicBlock *BB = BBs.pop_back_val();
 
-      Hashes.emplace_back(BlockHeaderHash);
+      // This random value acts as a block header, as otherwise the partition of
+      // opcodes into BBs wouldn't affect the hash, only the order of the
+      // opcodes
+      hash(45798);
       for (auto &Inst : *BB)
-        Hashes.emplace_back(hashInstruction(Inst));
+        updateInstruction(Inst, DetailedHash);
 
       for (const BasicBlock *Succ : successors(BB))
         if (VisitedBBs.insert(Succ).second)
           BBs.push_back(Succ);
     }
-
-    // Update the combined hash in place.
-    Hash = stable_hash_combine(Hashes);
   }
 
   void update(const GlobalVariable &GV) {
@@ -171,20 +137,15 @@ public:
     // we ignore anything with the `.llvm` prefix
     if (GV.isDeclaration() || GV.getName().starts_with("llvm."))
       return;
-    SmallVector<stable_hash> Hashes;
-    Hashes.emplace_back(Hash);
-    Hashes.emplace_back(GlobalHeaderHash);
-    Hashes.emplace_back(GV.getValueType()->getTypeID());
-
-    // Update the combined hash in place.
-    Hash = stable_hash_combine(Hashes);
+    hash(23456); // Global header
+    hash(GV.getValueType()->getTypeID());
   }
 
-  void update(const Module &M) {
+  void update(const Module &M, bool DetailedHash) {
     for (const GlobalVariable &GV : M.globals())
       update(GV);
     for (const Function &F : M)
-      update(F);
+      update(F, DetailedHash);
   }
 
   uint64_t getHash() const { return Hash; }
@@ -192,14 +153,14 @@ public:
 
 } // namespace
 
-stable_hash llvm::StructuralHash(const Function &F, bool DetailedHash) {
-  StructuralHashImpl H(DetailedHash);
-  H.update(F);
+IRHash llvm::StructuralHash(const Function &F, bool DetailedHash) {
+  StructuralHashImpl H;
+  H.update(F, DetailedHash);
   return H.getHash();
 }
 
-stable_hash llvm::StructuralHash(const Module &M, bool DetailedHash) {
-  StructuralHashImpl H(DetailedHash);
-  H.update(M);
+IRHash llvm::StructuralHash(const Module &M, bool DetailedHash) {
+  StructuralHashImpl H;
+  H.update(M, DetailedHash);
   return H.getHash();
 }
diff --git a/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index ad16b0b35014..b50a700e0903 100644
--- a/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -172,14 +172,14 @@ namespace {
 
 class FunctionNode {
   mutable AssertingVH<Function> F;
-  stable_hash Hash;
+  IRHash Hash;
 
 public:
   // Note the hash is recalculated potentially multiple times, but it is cheap.
   FunctionNode(Function *F) : F(F), Hash(StructuralHash(*F)) {}
 
   Function *getFunc() const { return F; }
-  stable_hash getHash() const { return Hash; }
+  IRHash getHash() const { return Hash; }
 
   /// Replace the reference to the function F by the function G, assuming their
   /// implementations are equal.
@@ -420,7 +420,7 @@ bool MergeFunctions::runOnModule(Module &M) {
 
   // All functions in the module, ordered by hash. Functions with a unique
   // hash value are easily eliminated.
-  std::vector<std::pair<stable_hash, Function *>> HashedFuncs;
+  std::vector<std::pair<IRHash, Function *>> HashedFuncs;
   for (Function &Func : M) {
     if (isEligibleForMerging(Func)) {
       HashedFuncs.push_back({StructuralHash(Func), &Func});
diff --git a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
index 0ceb363a67b1..e7718ca84d31 100644
--- a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
+++ b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
@@ -63,14 +63,6 @@ lpad:
   resume { ptr, i32 } zeroinitializer
 }
 
-define i8 @call_with_same_range() {
-; CHECK-LABEL: @call_with_same_range
-; CHECK: tail call i8 @call_with_range
-  bitcast i8 0 to i8
-  %out = call i8 @dummy(), !range !0
-  ret i8 %out
-}
-
 define i8 @invoke_with_same_range() personality ptr undef {
 ; CHECK-LABEL: @invoke_with_same_range()
 ; CHECK: tail call i8 @invoke_with_range()
@@ -84,6 +76,15 @@ lpad:
   resume { ptr, i32 } zeroinitializer
 }
 
+define i8 @call_with_same_range() {
+; CHECK-LABEL: @call_with_same_range
+; CHECK: tail call i8 @call_with_range
+  bitcast i8 0 to i8
+  %out = call i8 @dummy(), !range !0
+  ret i8 %out
+}
+
+
 declare i8 @dummy();
 declare i32 @__gxx_personality_v0(...)
 
-- 
GitLab


From a3181b11b5b758cfa852c0d27465f84ab3b079bb Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 26 Oct 2024 10:02:11 -0700
Subject: [PATCH 241/693] [ADT] Teach StringRef to derive from std::string_view
 (#113752)

This patch makes minimum changes to have StringRef derive from
std::string_view, with an eventual goal of removing StringRef
completely.  Subsequent patches will further clean up the remaining
bits.

I've chosen public inheritance over private one because our codebase
relies on implicit conversions to std::string_view, which is currently
provided by:

  constexpr operator std::string_view() const {
    return std::string_view(data(), size());
  }

This implicit conversion stops applying once we use std::string_view
as a base class, public or private.  If we chose a private base class,
we would lose the implicit conversion.
---
 llvm/include/llvm/ADT/StringRef.h | 41 ++++++++-----------------------
 1 file changed, 10 insertions(+), 31 deletions(-)

diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h
index d5f30b88c4c6..49b6b8ff52ab 100644
--- a/llvm/include/llvm/ADT/StringRef.h
+++ b/llvm/include/llvm/ADT/StringRef.h
@@ -48,7 +48,9 @@ namespace llvm {
   /// situations where the character data resides in some other buffer, whose
   /// lifetime extends past that of the StringRef. For this reason, it is not in
   /// general safe to store a StringRef.
-  class LLVM_GSL_POINTER StringRef {
+  class LLVM_GSL_POINTER StringRef : public std::string_view {
+    using Base = std::string_view;
+
   public:
     static constexpr size_t npos = ~size_t(0);
 
@@ -60,12 +62,6 @@ namespace llvm {
     using const_reverse_iterator = std::reverse_iterator<const_iterator>;
 
   private:
-    /// The start of the string, in an external buffer.
-    const char *Data = nullptr;
-
-    /// The length of the string.
-    size_t Length = 0;
-
     // Workaround memcmp issue with null pointers (undefined behavior)
     // by providing a specialized version
     static int compareMemory(const char *Lhs, const char *Rhs, size_t Length) {
@@ -86,27 +82,25 @@ namespace llvm {
 
     /// Construct a string ref from a cstring.
     /*implicit*/ constexpr StringRef(const char *Str)
-        : Data(Str), Length(Str ?
+        : Base(Str, Str ?
     // GCC 7 doesn't have constexpr char_traits. Fall back to __builtin_strlen.
 #if defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 8
-                                __builtin_strlen(Str)
+                        __builtin_strlen(Str)
 #else
-                                std::char_traits<char>::length(Str)
+                        std::char_traits<char>::length(Str)
 #endif
-                                : 0) {
+                        : 0) {
     }
 
     /// Construct a string ref from a pointer and length.
     /*implicit*/ constexpr StringRef(const char *data, size_t length)
-        : Data(data), Length(length) {}
+        : Base(data, length) {}
 
     /// Construct a string ref from an std::string.
-    /*implicit*/ StringRef(const std::string &Str)
-        : Data(Str.data()), Length(Str.length()) {}
+    /*implicit*/ StringRef(const std::string &Str) : Base(Str) {}
 
     /// Construct a string ref from an std::string_view.
-    /*implicit*/ constexpr StringRef(std::string_view Str)
-        : Data(Str.data()), Length(Str.size()) {}
+    /*implicit*/ constexpr StringRef(std::string_view Str) : Base(Str) {}
 
     /// @}
     /// @name Iterators
@@ -138,16 +132,9 @@ namespace llvm {
     /// @name String Operations
     /// @{
 
-    /// data - Get a pointer to the start of the string (which may not be null
-    /// terminated).
-    [[nodiscard]] constexpr const char *data() const { return Data; }
-
     /// empty - Check if the string is empty.
     [[nodiscard]] constexpr bool empty() const { return size() == 0; }
 
-    /// size - Get the string size.
-    [[nodiscard]] constexpr size_t size() const { return Length; }
-
     /// front - Get the first character in the string.
     [[nodiscard]] char front() const {
       assert(!empty());
@@ -248,14 +235,6 @@ namespace llvm {
     std::enable_if_t<std::is_same<T, std::string>::value, StringRef> &
     operator=(T &&Str) = delete;
 
-    /// @}
-    /// @name Type Conversions
-    /// @{
-
-    constexpr operator std::string_view() const {
-      return std::string_view(data(), size());
-    }
-
     /// @}
     /// @name String Predicates
     /// @{
-- 
GitLab


From ef886a25805b6aa0cd42886791af5ce26d575f10 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 26 Oct 2024 10:32:09 -0700
Subject: [PATCH 242/693] Revert "[ADT] Teach StringRef to derive from
 std::string_view" (#113767)

Reverts llvm/llvm-project#113752

Many build bot failures
---
 llvm/include/llvm/ADT/StringRef.h | 41 +++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h
index 49b6b8ff52ab..d5f30b88c4c6 100644
--- a/llvm/include/llvm/ADT/StringRef.h
+++ b/llvm/include/llvm/ADT/StringRef.h
@@ -48,9 +48,7 @@ namespace llvm {
   /// situations where the character data resides in some other buffer, whose
   /// lifetime extends past that of the StringRef. For this reason, it is not in
   /// general safe to store a StringRef.
-  class LLVM_GSL_POINTER StringRef : public std::string_view {
-    using Base = std::string_view;
-
+  class LLVM_GSL_POINTER StringRef {
   public:
     static constexpr size_t npos = ~size_t(0);
 
@@ -62,6 +60,12 @@ namespace llvm {
     using const_reverse_iterator = std::reverse_iterator<const_iterator>;
 
   private:
+    /// The start of the string, in an external buffer.
+    const char *Data = nullptr;
+
+    /// The length of the string.
+    size_t Length = 0;
+
     // Workaround memcmp issue with null pointers (undefined behavior)
     // by providing a specialized version
     static int compareMemory(const char *Lhs, const char *Rhs, size_t Length) {
@@ -82,25 +86,27 @@ namespace llvm {
 
     /// Construct a string ref from a cstring.
     /*implicit*/ constexpr StringRef(const char *Str)
-        : Base(Str, Str ?
+        : Data(Str), Length(Str ?
     // GCC 7 doesn't have constexpr char_traits. Fall back to __builtin_strlen.
 #if defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 8
-                        __builtin_strlen(Str)
+                                __builtin_strlen(Str)
 #else
-                        std::char_traits<char>::length(Str)
+                                std::char_traits<char>::length(Str)
 #endif
-                        : 0) {
+                                : 0) {
     }
 
     /// Construct a string ref from a pointer and length.
     /*implicit*/ constexpr StringRef(const char *data, size_t length)
-        : Base(data, length) {}
+        : Data(data), Length(length) {}
 
     /// Construct a string ref from an std::string.
-    /*implicit*/ StringRef(const std::string &Str) : Base(Str) {}
+    /*implicit*/ StringRef(const std::string &Str)
+        : Data(Str.data()), Length(Str.length()) {}
 
     /// Construct a string ref from an std::string_view.
-    /*implicit*/ constexpr StringRef(std::string_view Str) : Base(Str) {}
+    /*implicit*/ constexpr StringRef(std::string_view Str)
+        : Data(Str.data()), Length(Str.size()) {}
 
     /// @}
     /// @name Iterators
@@ -132,9 +138,16 @@ namespace llvm {
     /// @name String Operations
     /// @{
 
+    /// data - Get a pointer to the start of the string (which may not be null
+    /// terminated).
+    [[nodiscard]] constexpr const char *data() const { return Data; }
+
     /// empty - Check if the string is empty.
     [[nodiscard]] constexpr bool empty() const { return size() == 0; }
 
+    /// size - Get the string size.
+    [[nodiscard]] constexpr size_t size() const { return Length; }
+
     /// front - Get the first character in the string.
     [[nodiscard]] char front() const {
       assert(!empty());
@@ -235,6 +248,14 @@ namespace llvm {
     std::enable_if_t<std::is_same<T, std::string>::value, StringRef> &
     operator=(T &&Str) = delete;
 
+    /// @}
+    /// @name Type Conversions
+    /// @{
+
+    constexpr operator std::string_view() const {
+      return std::string_view(data(), size());
+    }
+
     /// @}
     /// @name String Predicates
     /// @{
-- 
GitLab


From c8140d0d7fa943f5ae339cb5e6f4fd8644d5ae89 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 26 Oct 2024 10:47:51 -0700
Subject: [PATCH 243/693] [MCParser] De-capitalize ELFAsmParser comments. NFC

---
 llvm/lib/MC/MCParser/ELFAsmParser.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/MC/MCParser/ELFAsmParser.cpp b/llvm/lib/MC/MCParser/ELFAsmParser.cpp
index 9cdc2aafe3d2..b58210b3c268 100644
--- a/llvm/lib/MC/MCParser/ELFAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/ELFAsmParser.cpp
@@ -163,7 +163,7 @@ private:
 
 } // end anonymous namespace
 
-/// ParseDirectiveSymbolAttribute
+/// parseDirectiveSymbolAttribute
 ///  ::= { ".local", ".weak", ... } [ identifier ( , identifier )* ]
 bool ELFAsmParser::parseDirectiveSymbolAttribute(StringRef Directive, SMLoc) {
   MCSymbolAttr Attr = StringSwitch<MCSymbolAttr>(Directive)
@@ -746,7 +746,7 @@ static MCSymbolAttr MCAttrForString(StringRef Type) {
           .Default(MCSA_Invalid);
 }
 
-/// ParseDirectiveELFType
+/// parseDirectiveELFType
 ///  ::= .type identifier , STT_<TYPE_IN_UPPER_CASE>
 ///  ::= .type identifier , #attribute
 ///  ::= .type identifier , @attribute
@@ -803,7 +803,7 @@ bool ELFAsmParser::parseDirectiveType(StringRef, SMLoc) {
   return false;
 }
 
-/// ParseDirectiveIdent
+/// parseDirectiveIdent
 ///  ::= .ident string
 bool ELFAsmParser::parseDirectiveIdent(StringRef, SMLoc) {
   if (getLexer().isNot(AsmToken::String))
@@ -821,7 +821,7 @@ bool ELFAsmParser::parseDirectiveIdent(StringRef, SMLoc) {
   return false;
 }
 
-/// ParseDirectiveSymver
+/// parseDirectiveSymver
 ///  ::= .symver foo, bar2@zed
 bool ELFAsmParser::parseDirectiveSymver(StringRef, SMLoc) {
   StringRef OriginalName, Name, Action;
@@ -858,7 +858,7 @@ bool ELFAsmParser::parseDirectiveSymver(StringRef, SMLoc) {
   return false;
 }
 
-/// ParseDirectiveVersion
+/// parseDirectiveVersion
 ///  ::= .version string
 bool ELFAsmParser::parseDirectiveVersion(StringRef, SMLoc) {
   if (getLexer().isNot(AsmToken::String))
@@ -882,7 +882,7 @@ bool ELFAsmParser::parseDirectiveVersion(StringRef, SMLoc) {
   return false;
 }
 
-/// ParseDirectiveWeakref
+/// parseDirectiveWeakref
 ///  ::= .weakref foo, bar
 bool ELFAsmParser::parseDirectiveWeakref(StringRef, SMLoc) {
   // FIXME: Share code with the other alias building directives.
-- 
GitLab


From 0be1883c36fc19e4020fea12902481c3dd3436d2 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990@gmail.com>
Date: Sat, 26 Oct 2024 18:58:53 +0100
Subject: [PATCH 244/693] [libc++] Implement P0429R9 `std::flat_map` (#98643)

Around half of the tests are based on the tests Arthur O'Dwyer's
original implementation of std::flat_map, with modifications and
removals.

partially implement #105190
---
 libcxx/docs/ReleaseNotes/20.rst               |    1 +
 libcxx/docs/Status/Cxx23Papers.csv            |    2 +-
 libcxx/include/CMakeLists.txt                 |    3 +
 libcxx/include/__flat_map/flat_map.h          | 1359 +++++++++++++++++
 libcxx/include/__flat_map/sorted_unique.h     |   31 +
 libcxx/include/__utility/exception_guard.h    |    6 +
 libcxx/include/flat_map                       |   54 +
 libcxx/include/module.modulemap               |    8 +
 libcxx/modules/std.compat.cppm.in             |    3 -
 libcxx/modules/std.cppm.in                    |    4 +-
 libcxx/modules/std/flat_map.inc               |   12 +-
 .../flat.map/assert.input_range.pass.cpp      |   66 +
 .../flat.map/assert.sorted_unique.pass.cpp    |  225 +++
 .../flat.map/container_stability.pass.cpp     |   68 +
 .../test/libcxx/transitive_includes/cxx03.csv |   44 +
 .../test/libcxx/transitive_includes/cxx11.csv |   44 +
 .../test/libcxx/transitive_includes/cxx14.csv |   45 +
 .../test/libcxx/transitive_includes/cxx17.csv |   44 +
 .../test/libcxx/transitive_includes/cxx20.csv |   44 +
 .../test/libcxx/transitive_includes/cxx23.csv |   25 +
 .../test/libcxx/transitive_includes/cxx26.csv |   25 +
 .../container.adaptors/NaiveStaticVector.h    |   94 ++
 .../flat.map.syn/sorted_unique.pass.cpp       |   44 +
 .../flat.map/flat.map.access/at.pass.cpp      |   92 ++
 .../flat.map.access/at_transparent.pass.cpp   |  111 ++
 .../flat.map.access/index_key.pass.cpp        |   77 +
 .../flat.map.access/index_rv_key.pass.cpp     |   68 +
 .../index_transparent.pass.cpp                |  107 ++
 .../flat.map/flat.map.capacity/empty.pass.cpp |   47 +
 .../flat.map.capacity/empty.verify.cpp        |   24 +
 .../flat.map.capacity/max_size.pass.cpp       |   76 +
 .../flat.map/flat.map.capacity/size.pass.cpp  |   65 +
 .../flat.map/flat.map.cons/alloc.pass.cpp     |   72 +
 .../assign_initializer_list.pass.cpp          |   59 +
 .../flat.map/flat.map.cons/compare.pass.cpp   |   93 ++
 .../flat.map.cons/containers.pass.cpp         |  184 +++
 .../flat.map/flat.map.cons/copy.pass.cpp      |   70 +
 .../flat.map.cons/copy_alloc.pass.cpp         |   67 +
 .../copy_assign.addressof.compile.pass.cpp    |   30 +
 .../flat.map.cons/copy_assign.pass.cpp        |   92 ++
 .../flat.map/flat.map.cons/deduct.pass.cpp    |  342 +++++
 .../flat.map/flat.map.cons/deduct.verify.cpp  |   97 ++
 .../flat.map.cons/deduct_pmr.pass.cpp         |  106 ++
 .../flat.map/flat.map.cons/default.pass.cpp   |   72 +
 .../flat.map.cons/default_noexcept.pass.cpp   |   57 +
 .../flat.map.cons/dtor_noexcept.pass.cpp      |   53 +
 .../flat.map.cons/initializer_list.pass.cpp   |  157 ++
 .../flat.map/flat.map.cons/iter_iter.pass.cpp |  154 ++
 .../iter_iter_stability.pass.cpp              |   65 +
 .../flat.map/flat.map.cons/move.pass.cpp      |   88 ++
 .../flat.map.cons/move_alloc.pass.cpp         |   82 +
 .../flat.map.cons/move_assign.pass.cpp        |   74 +
 .../flat.map.cons/move_assign_clears.pass.cpp |  104 ++
 .../move_assign_noexcept.pass.cpp             |  110 ++
 .../flat.map.cons/move_exceptions.pass.cpp    |   71 +
 .../flat.map.cons/move_noexcept.pass.cpp      |  102 ++
 .../flat.map/flat.map.cons/pmr.pass.cpp       |  361 +++++
 .../flat.map/flat.map.cons/range.pass.cpp     |  227 +++
 .../flat.map.cons/sorted_container.pass.cpp   |  165 ++
 .../sorted_initializer_list.pass.cpp          |  179 +++
 .../flat.map.cons/sorted_iter_iter.pass.cpp   |  171 +++
 .../flat.map.erasure/erase_if.pass.cpp        |   93 ++
 .../erase_if_exceptions.pass.cpp              |  155 ++
 .../flat.map.iterators/iterator.pass.cpp      |   96 ++
 .../iterator_comparison.pass.cpp              |  155 ++
 ...rator_concept_conformance.compile.pass.cpp |   82 +
 ...range_concept_conformance.compile.pass.cpp |   55 +
 .../reverse_iterator.pass.cpp                 |   90 ++
 .../flat.map.modifiers/clear.pass.cpp         |   64 +
 .../flat.map.modifiers/emplace.pass.cpp       |  103 ++
 .../flat.map.modifiers/emplace_hint.pass.cpp  |  102 ++
 .../flat.map.modifiers/erase_iter.pass.cpp    |  151 ++
 .../erase_iter_iter.pass.cpp                  |  109 ++
 .../flat.map.modifiers/erase_key.pass.cpp     |   91 ++
 .../erase_key_transparent.pass.cpp            |  144 ++
 .../flat.map.modifiers/extract.pass.cpp       |   91 ++
 .../flat.map.modifiers/insert_cv.pass.cpp     |   83 +
 .../insert_initializer_list.pass.cpp          |   68 +
 .../insert_iter_cv.pass.cpp                   |   79 +
 .../insert_iter_iter.pass.cpp                 |   89 ++
 .../insert_iter_rv.pass.cpp                   |   88 ++
 .../insert_or_assign.pass.cpp                 |  326 ++++
 .../insert_or_assign_transparent.pass.cpp     |  259 ++++
 .../flat.map.modifiers/insert_range.pass.cpp  |  109 ++
 .../insert_range_stability.pass.cpp           |   63 +
 .../flat.map.modifiers/insert_rv.pass.cpp     |  124 ++
 .../insert_sorted_initializer_list.pass.cpp   |   66 +
 .../insert_sorted_iter_iter.pass.cpp          |   86 ++
 .../insert_transparent.pass.cpp               |  167 ++
 .../flat.map.modifiers/replace.pass.cpp       |   80 +
 .../swap_exception.pass.cpp                   |   78 +
 .../flat.map.modifiers/swap_free.pass.cpp     |   97 ++
 .../flat.map.modifiers/swap_member.pass.cpp   |   95 ++
 .../flat.map.modifiers/try_emplace.pass.cpp   |  246 +++
 .../try_emplace_transparent.pass.cpp          |  182 +++
 .../flat.map/flat.map.observers/comp.pass.cpp |   96 ++
 .../flat.map.observers/keys_values.pass.cpp   |   57 +
 .../flat.map.operations/contains.pass.cpp     |   70 +
 .../contains_transparent.pass.cpp             |   71 +
 .../flat.map.operations/count.pass.cpp        |   69 +
 .../count_transparent.pass.cpp                |   72 +
 .../flat.map.operations/equal_range.pass.cpp  |   78 +
 .../equal_range_transparent.pass.cpp          |  100 ++
 .../flat.map.operations/find.pass.cpp         |   55 +
 .../find_transparent.pass.cpp                 |   88 ++
 .../flat.map.operations/lower_bound.pass.cpp  |   71 +
 .../lower_bound_transparent.pass.cpp          |   95 ++
 .../flat.map.operations/upper_bound.pass.cpp  |   72 +
 .../upper_bound_transparent.pass.cpp          |   94 ++
 .../container.adaptors/flat.map/helpers.h     |  394 +++++
 .../flat.map/incomplete_type.pass.cpp         |   32 +
 .../flat.map/op_compare.pass.cpp              |  118 ++
 .../flat.map/types.compile.pass.cpp           |  133 ++
 libcxx/test/support/MinSequenceContainer.h    |   83 +
 libcxx/utils/libcxx/header_information.py     |    2 +-
 115 files changed, 12225 insertions(+), 13 deletions(-)
 create mode 100644 libcxx/include/__flat_map/flat_map.h
 create mode 100644 libcxx/include/__flat_map/sorted_unique.h
 create mode 100644 libcxx/include/flat_map
 create mode 100644 libcxx/test/libcxx/containers/containers.adaptors/flat.map/assert.input_range.pass.cpp
 create mode 100644 libcxx/test/libcxx/containers/containers.adaptors/flat.map/assert.sorted_unique.pass.cpp
 create mode 100644 libcxx/test/libcxx/containers/containers.adaptors/flat.map/container_stability.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/NaiveStaticVector.h
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map.syn/sorted_unique.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at_transparent.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_key.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_rv_key.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_transparent.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.verify.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/max_size.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/size.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/alloc.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/assign_initializer_list.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/compare.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/containers.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy_alloc.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy_assign.addressof.compile.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy_assign.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.verify.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct_pmr.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/default.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/default_noexcept.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/dtor_noexcept.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/initializer_list.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/iter_iter.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/iter_iter_stability.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_alloc.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_assign.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_assign_clears.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_assign_noexcept.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_exceptions.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_noexcept.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/pmr.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/range.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/sorted_container.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/sorted_initializer_list.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/sorted_iter_iter.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.erasure/erase_if.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.erasure/erase_if_exceptions.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/iterator.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/iterator_comparison.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/iterator_concept_conformance.compile.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/range_concept_conformance.compile.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/reverse_iterator.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/clear.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/emplace.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/emplace_hint.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_iter.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_iter_iter.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_key.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_key_transparent.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/extract.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_cv.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_initializer_list.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_cv.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_rv.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign_transparent.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_range.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_range_stability.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_rv.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_sorted_initializer_list.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_sorted_iter_iter.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_transparent.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/replace.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_exception.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_free.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_member.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/try_emplace.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/try_emplace_transparent.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.observers/comp.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.observers/keys_values.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/contains.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/contains_transparent.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/count.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/count_transparent.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/equal_range.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/equal_range_transparent.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/find.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/find_transparent.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/lower_bound.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/lower_bound_transparent.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/upper_bound.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/upper_bound_transparent.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/helpers.h
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/incomplete_type.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/op_compare.pass.cpp
 create mode 100644 libcxx/test/std/containers/container.adaptors/flat.map/types.compile.pass.cpp
 create mode 100644 libcxx/test/support/MinSequenceContainer.h

diff --git a/libcxx/docs/ReleaseNotes/20.rst b/libcxx/docs/ReleaseNotes/20.rst
index 39546493ae8d..84080e7cbafe 100644
--- a/libcxx/docs/ReleaseNotes/20.rst
+++ b/libcxx/docs/ReleaseNotes/20.rst
@@ -43,6 +43,7 @@ Implemented Papers
 - P2985R0: A type trait for detecting virtual base classes (`Github <https://github.com/llvm/llvm-project/issues/105432>`__)
 - ``std::jthread`` and ``<stop_token>`` are not guarded behind ``-fexperimental-library`` anymore
 - P2674R1: A trait for implicit lifetime types (`Github <https://github.com/llvm/llvm-project/issues/105259>`__)
+- P0429R9: A Standard ``flat_map`` is partially implemented and ``flat_map`` is provided (`Github <https://github.com/llvm/llvm-project/issues/105190>`__)
 
 Improvements and New Features
 -----------------------------
diff --git a/libcxx/docs/Status/Cxx23Papers.csv b/libcxx/docs/Status/Cxx23Papers.csv
index c64f1c4171fc..6f1626da7350 100644
--- a/libcxx/docs/Status/Cxx23Papers.csv
+++ b/libcxx/docs/Status/Cxx23Papers.csv
@@ -52,7 +52,7 @@
 "`P2443R1 <https://wg21.link/P2443R1>`__","``views::chunk_by``","2022-02 (Virtual)","|Complete|","18.0",""
 "","","","","",""
 "`P0009R18 <https://wg21.link/P0009R18>`__","mdspan: A Non-Owning Multidimensional Array Reference","2022-07 (Virtual)","|Complete|","18.0",""
-"`P0429R9 <https://wg21.link/P0429R9>`__","A Standard ``flat_map``","2022-07 (Virtual)","","",""
+"`P0429R9 <https://wg21.link/P0429R9>`__","A Standard ``flat_map``","2022-07 (Virtual)","|In progress|","",""
 "`P1169R4 <https://wg21.link/P1169R4>`__","``static operator()``","2022-07 (Virtual)","|Complete|","16.0",""
 "`P1222R4 <https://wg21.link/P1222R4>`__","A Standard ``flat_set``","2022-07 (Virtual)","","",""
 "`P1223R5 <https://wg21.link/P1223R5>`__","``ranges::find_last()``, ``ranges::find_last_if()``, and ``ranges::find_last_if_not()``","2022-07 (Virtual)","|Complete|","19.0",""
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index bb152af82cad..e84a55e25f2f 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -358,6 +358,8 @@ set(files
   __filesystem/recursive_directory_iterator.h
   __filesystem/space_info.h
   __filesystem/u8path.h
+  __flat_map/flat_map.h
+  __flat_map/sorted_unique.h
   __format/buffer.h
   __format/concepts.h
   __format/container_adaptor.h
@@ -959,6 +961,7 @@ set(files
   ext/hash_set
   fenv.h
   filesystem
+  flat_map
   float.h
   format
   forward_list
diff --git a/libcxx/include/__flat_map/flat_map.h b/libcxx/include/__flat_map/flat_map.h
new file mode 100644
index 000000000000..9ca32d5295bd
--- /dev/null
+++ b/libcxx/include/__flat_map/flat_map.h
@@ -0,0 +1,1359 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FLAT_MAP_FLAT_MAP_H
+#define _LIBCPP___FLAT_MAP_FLAT_MAP_H
+
+#include <__algorithm/lexicographical_compare_three_way.h>
+#include <__algorithm/ranges_adjacent_find.h>
+#include <__algorithm/ranges_equal.h>
+#include <__algorithm/ranges_inplace_merge.h>
+#include <__algorithm/ranges_lower_bound.h>
+#include <__algorithm/ranges_partition_point.h>
+#include <__algorithm/ranges_stable_sort.h>
+#include <__algorithm/ranges_unique.h>
+#include <__algorithm/ranges_upper_bound.h>
+#include <__compare/synth_three_way.h>
+#include <__concepts/convertible_to.h>
+#include <__concepts/swappable.h>
+#include <__config>
+#include <__flat_map/sorted_unique.h>
+#include <__functional/invoke.h>
+#include <__functional/is_transparent.h>
+#include <__functional/operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/distance.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/next.h>
+#include <__iterator/ranges_iterator_traits.h>
+#include <__iterator/reverse_iterator.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/uses_allocator.h>
+#include <__memory/uses_allocator_construction.h>
+#include <__ranges/concepts.h>
+#include <__ranges/container_compatible_range.h>
+#include <__ranges/drop_view.h>
+#include <__ranges/ref_view.h>
+#include <__ranges/subrange.h>
+#include <__ranges/zip_view.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/container_traits.h>
+#include <__type_traits/invoke.h>
+#include <__type_traits/is_allocator.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/maybe_const.h>
+#include <__utility/exception_guard.h>
+#include <__utility/pair.h>
+#include <initializer_list>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Key,
+          class _Tp,
+          class _Compare         = less<_Key>,
+          class _KeyContainer    = vector<_Key>,
+          class _MappedContainer = vector<_Tp>>
+class flat_map {
+  template <bool _Const>
+  struct __iterator;
+
+  template <class, class, class, class, class>
+  friend class flat_map;
+
+  static_assert(is_same_v<_Key, typename _KeyContainer::value_type>);
+  static_assert(is_same_v<_Tp, typename _MappedContainer::value_type>);
+  static_assert(!is_same_v<_KeyContainer, std::vector<bool>>, "vector<bool> is not a sequence container");
+  static_assert(!is_same_v<_MappedContainer, std::vector<bool>>, "vector<bool> is not a sequence container");
+
+public:
+  // types
+  using key_type               = _Key;
+  using mapped_type            = _Tp;
+  using value_type             = pair<key_type, mapped_type>;
+  using key_compare            = __type_identity_t<_Compare>;
+  using reference              = pair<const key_type&, mapped_type&>;
+  using const_reference        = pair<const key_type&, const mapped_type&>;
+  using size_type              = size_t;
+  using difference_type        = ptrdiff_t;
+  using iterator               = __iterator<false>; // see [container.requirements]
+  using const_iterator         = __iterator<true>;  // see [container.requirements]
+  using reverse_iterator       = std::reverse_iterator<iterator>;
+  using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+  using key_container_type     = _KeyContainer;
+  using mapped_container_type  = _MappedContainer;
+
+  class value_compare {
+  private:
+    key_compare __comp_;
+    value_compare(key_compare __c) : __comp_(__c) {}
+    friend flat_map;
+
+  public:
+    _LIBCPP_HIDE_FROM_ABI bool operator()(const_reference __x, const_reference __y) const {
+      return __comp_(__x.first, __y.first);
+    }
+  };
+
+  struct containers {
+    key_container_type keys;
+    mapped_container_type values;
+  };
+
+private:
+  template <class _Allocator>
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __allocator_ctor_constraint =
+      _And<uses_allocator<key_container_type, _Allocator>, uses_allocator<mapped_container_type, _Allocator>>::value;
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __is_compare_transparent = __is_transparent_v<_Compare, _Compare>;
+
+  template <bool _Const>
+  struct __iterator {
+  private:
+    using __key_iterator    = ranges::iterator_t<const key_container_type>;
+    using __mapped_iterator = ranges::iterator_t<__maybe_const<_Const, mapped_container_type>>;
+    using __reference       = pair<iter_reference_t<__key_iterator>, iter_reference_t<__mapped_iterator>>;
+
+    struct __arrow_proxy {
+      __reference __ref_;
+      _LIBCPP_HIDE_FROM_ABI __reference* operator->() { return std::addressof(__ref_); }
+    };
+
+    __key_iterator __key_iter_;
+    __mapped_iterator __mapped_iter_;
+
+    friend flat_map;
+
+  public:
+    using iterator_concept = random_access_iterator_tag;
+    // `flat_map::iterator` only satisfy "Cpp17InputIterator" named requirements, because
+    // its `reference` is not a reference type.
+    // However, to avoid surprising runtime behaviour when it is used with the
+    // Cpp17 algorithms or operations, iterator_category is set to random_access_iterator_tag.
+    using iterator_category = random_access_iterator_tag;
+    using value_type        = flat_map::value_type;
+    using difference_type   = flat_map::difference_type;
+
+    _LIBCPP_HIDE_FROM_ABI __iterator() = default;
+
+    _LIBCPP_HIDE_FROM_ABI __iterator(__iterator<!_Const> __i)
+      requires _Const && convertible_to<ranges::iterator_t<key_container_type>, __key_iterator> &&
+                   convertible_to<ranges::iterator_t<mapped_container_type>, __mapped_iterator>
+        : __key_iter_(std::move(__i.__key_iter_)), __mapped_iter_(std::move(__i.__mapped_iter_)) {}
+
+    _LIBCPP_HIDE_FROM_ABI __iterator(__key_iterator __key_iter, __mapped_iterator __mapped_iter)
+        : __key_iter_(std::move(__key_iter)), __mapped_iter_(std::move(__mapped_iter)) {}
+
+    _LIBCPP_HIDE_FROM_ABI __reference operator*() const { return __reference(*__key_iter_, *__mapped_iter_); }
+    _LIBCPP_HIDE_FROM_ABI __arrow_proxy operator->() const { return __arrow_proxy{**this}; }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator& operator++() {
+      ++__key_iter_;
+      ++__mapped_iter_;
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator operator++(int) {
+      __iterator __tmp(*this);
+      ++*this;
+      return __tmp;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator& operator--() {
+      --__key_iter_;
+      --__mapped_iter_;
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator operator--(int) {
+      __iterator __tmp(*this);
+      --*this;
+      return __tmp;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator& operator+=(difference_type __x) {
+      __key_iter_ += __x;
+      __mapped_iter_ += __x;
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator& operator-=(difference_type __x) {
+      __key_iter_ -= __x;
+      __mapped_iter_ -= __x;
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __reference operator[](difference_type __n) const { return *(*this + __n); }
+
+    _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __iterator& __y) {
+      return __x.__key_iter_ == __y.__key_iter_;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend bool operator<(const __iterator& __x, const __iterator& __y) {
+      return __x.__key_iter_ < __y.__key_iter_;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend bool operator>(const __iterator& __x, const __iterator& __y) { return __y < __x; }
+
+    _LIBCPP_HIDE_FROM_ABI friend bool operator<=(const __iterator& __x, const __iterator& __y) { return !(__y < __x); }
+
+    _LIBCPP_HIDE_FROM_ABI friend bool operator>=(const __iterator& __x, const __iterator& __y) { return !(__x < __y); }
+
+    _LIBCPP_HIDE_FROM_ABI friend auto operator<=>(const __iterator& __x, const __iterator& __y)
+      requires three_way_comparable<__key_iterator>
+    {
+      return __x.__key_iter_ <=> __y.__key_iter_;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend __iterator operator+(const __iterator& __i, difference_type __n) {
+      auto __tmp = __i;
+      __tmp += __n;
+      return __tmp;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend __iterator operator+(difference_type __n, const __iterator& __i) { return __i + __n; }
+
+    _LIBCPP_HIDE_FROM_ABI friend __iterator operator-(const __iterator& __i, difference_type __n) {
+      auto __tmp = __i;
+      __tmp -= __n;
+      return __tmp;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend difference_type operator-(const __iterator& __x, const __iterator& __y) {
+      return difference_type(__x.__key_iter_ - __y.__key_iter_);
+    }
+  };
+
+public:
+  // [flat.map.cons], construct/copy/destroy
+  _LIBCPP_HIDE_FROM_ABI flat_map() noexcept(
+      is_nothrow_default_constructible_v<_KeyContainer> && is_nothrow_default_constructible_v<_MappedContainer> &&
+      is_nothrow_default_constructible_v<_Compare>)
+      : __containers_(), __compare_() {}
+
+  _LIBCPP_HIDE_FROM_ABI flat_map(const flat_map&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI flat_map(flat_map&& __other) noexcept(
+      is_nothrow_move_constructible_v<_KeyContainer> && is_nothrow_move_constructible_v<_MappedContainer> &&
+      is_nothrow_move_constructible_v<_Compare>)
+#  if _LIBCPP_HAS_EXCEPTIONS
+      try
+#  endif // _LIBCPP_HAS_EXCEPTIONS
+      : __containers_(std::move(__other.__containers_)), __compare_(std::move(__other.__compare_)) {
+    __other.clear();
+#  if _LIBCPP_HAS_EXCEPTIONS
+  } catch (...) {
+    __other.clear();
+    // gcc does not like the `throw` keyword in a conditional noexcept function
+    if constexpr (!(is_nothrow_move_constructible_v<_KeyContainer> &&
+                    is_nothrow_move_constructible_v<_MappedContainer> && is_nothrow_move_constructible_v<_Compare>)) {
+      throw;
+    }
+#  endif // _LIBCPP_HAS_EXCEPTIONS
+  }
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_map(const flat_map& __other, const _Allocator& __alloc)
+      : flat_map(__ctor_uses_allocator_tag{},
+                 __alloc,
+                 __other.__containers_.keys,
+                 __other.__containers_.values,
+                 __other.__compare_) {}
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_map(flat_map&& __other, const _Allocator& __alloc)
+#  if _LIBCPP_HAS_EXCEPTIONS
+      try
+#  endif // _LIBCPP_HAS_EXCEPTIONS
+      : flat_map(__ctor_uses_allocator_tag{},
+                 __alloc,
+                 std::move(__other.__containers_.keys),
+                 std::move(__other.__containers_.values),
+                 std::move(__other.__compare_)) {
+    __other.clear();
+#  if _LIBCPP_HAS_EXCEPTIONS
+  } catch (...) {
+    __other.clear();
+    throw;
+#  endif // _LIBCPP_HAS_EXCEPTIONS
+  }
+
+  _LIBCPP_HIDE_FROM_ABI flat_map(
+      key_container_type __key_cont, mapped_container_type __mapped_cont, const key_compare& __comp = key_compare())
+      : __containers_{.keys = std::move(__key_cont), .values = std::move(__mapped_cont)}, __compare_(__comp) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
+                                     "flat_map keys and mapped containers have different size");
+    __sort_and_unique();
+  }
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI
+  flat_map(const key_container_type& __key_cont, const mapped_container_type& __mapped_cont, const _Allocator& __alloc)
+      : flat_map(__ctor_uses_allocator_tag{}, __alloc, __key_cont, __mapped_cont) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
+                                     "flat_map keys and mapped containers have different size");
+    __sort_and_unique();
+  }
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI
+  flat_map(const key_container_type& __key_cont,
+           const mapped_container_type& __mapped_cont,
+           const key_compare& __comp,
+           const _Allocator& __alloc)
+      : flat_map(__ctor_uses_allocator_tag{}, __alloc, __key_cont, __mapped_cont, __comp) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
+                                     "flat_map keys and mapped containers have different size");
+    __sort_and_unique();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI
+  flat_map(sorted_unique_t,
+           key_container_type __key_cont,
+           mapped_container_type __mapped_cont,
+           const key_compare& __comp = key_compare())
+      : __containers_{.keys = std::move(__key_cont), .values = std::move(__mapped_cont)}, __compare_(__comp) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
+                                     "flat_map keys and mapped containers have different size");
+    _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
+        __is_sorted_and_unique(__containers_.keys), "Either the key container is not sorted or it contains duplicates");
+  }
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI
+  flat_map(sorted_unique_t,
+           const key_container_type& __key_cont,
+           const mapped_container_type& __mapped_cont,
+           const _Allocator& __alloc)
+      : flat_map(__ctor_uses_allocator_tag{}, __alloc, __key_cont, __mapped_cont) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
+                                     "flat_map keys and mapped containers have different size");
+    _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
+        __is_sorted_and_unique(__containers_.keys), "Either the key container is not sorted or it contains duplicates");
+  }
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI
+  flat_map(sorted_unique_t,
+           const key_container_type& __key_cont,
+           const mapped_container_type& __mapped_cont,
+           const key_compare& __comp,
+           const _Allocator& __alloc)
+      : flat_map(__ctor_uses_allocator_tag{}, __alloc, __key_cont, __mapped_cont, __comp) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
+                                     "flat_map keys and mapped containers have different size");
+    _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
+        __is_sorted_and_unique(__containers_.keys), "Either the key container is not sorted or it contains duplicates");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI explicit flat_map(const key_compare& __comp) : __containers_(), __compare_(__comp) {}
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_map(const key_compare& __comp, const _Allocator& __alloc)
+      : flat_map(__ctor_uses_allocator_empty_tag{}, __alloc, __comp) {}
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI explicit flat_map(const _Allocator& __alloc)
+      : flat_map(__ctor_uses_allocator_empty_tag{}, __alloc) {}
+
+  template <class _InputIterator>
+    requires __has_input_iterator_category<_InputIterator>::value
+  _LIBCPP_HIDE_FROM_ABI
+  flat_map(_InputIterator __first, _InputIterator __last, const key_compare& __comp = key_compare())
+      : __containers_(), __compare_(__comp) {
+    insert(__first, __last);
+  }
+
+  template <class _InputIterator, class _Allocator>
+    requires(__has_input_iterator_category<_InputIterator>::value && __allocator_ctor_constraint<_Allocator>)
+  _LIBCPP_HIDE_FROM_ABI
+  flat_map(_InputIterator __first, _InputIterator __last, const key_compare& __comp, const _Allocator& __alloc)
+      : flat_map(__ctor_uses_allocator_empty_tag{}, __alloc, __comp) {
+    insert(__first, __last);
+  }
+
+  template <class _InputIterator, class _Allocator>
+    requires(__has_input_iterator_category<_InputIterator>::value && __allocator_ctor_constraint<_Allocator>)
+  _LIBCPP_HIDE_FROM_ABI flat_map(_InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
+      : flat_map(__ctor_uses_allocator_empty_tag{}, __alloc) {
+    insert(__first, __last);
+  }
+
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI flat_map(from_range_t __fr, _Range&& __rg)
+      : flat_map(__fr, std::forward<_Range>(__rg), key_compare()) {}
+
+  template <_ContainerCompatibleRange<value_type> _Range, class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_map(from_range_t, _Range&& __rg, const _Allocator& __alloc)
+      : flat_map(__ctor_uses_allocator_empty_tag{}, __alloc) {
+    insert_range(std::forward<_Range>(__rg));
+  }
+
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI flat_map(from_range_t, _Range&& __rg, const key_compare& __comp) : flat_map(__comp) {
+    insert_range(std::forward<_Range>(__rg));
+  }
+
+  template <_ContainerCompatibleRange<value_type> _Range, class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_map(from_range_t, _Range&& __rg, const key_compare& __comp, const _Allocator& __alloc)
+      : flat_map(__ctor_uses_allocator_empty_tag{}, __alloc, __comp) {
+    insert_range(std::forward<_Range>(__rg));
+  }
+
+  template <class _InputIterator>
+    requires __has_input_iterator_category<_InputIterator>::value
+  _LIBCPP_HIDE_FROM_ABI
+  flat_map(sorted_unique_t, _InputIterator __first, _InputIterator __last, const key_compare& __comp = key_compare())
+      : __containers_(), __compare_(__comp) {
+    insert(sorted_unique, __first, __last);
+  }
+  template <class _InputIterator, class _Allocator>
+    requires(__has_input_iterator_category<_InputIterator>::value && __allocator_ctor_constraint<_Allocator>)
+  _LIBCPP_HIDE_FROM_ABI
+  flat_map(sorted_unique_t,
+           _InputIterator __first,
+           _InputIterator __last,
+           const key_compare& __comp,
+           const _Allocator& __alloc)
+      : flat_map(__ctor_uses_allocator_empty_tag{}, __alloc, __comp) {
+    insert(sorted_unique, __first, __last);
+  }
+
+  template <class _InputIterator, class _Allocator>
+    requires(__has_input_iterator_category<_InputIterator>::value && __allocator_ctor_constraint<_Allocator>)
+  _LIBCPP_HIDE_FROM_ABI
+  flat_map(sorted_unique_t, _InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
+      : flat_map(__ctor_uses_allocator_empty_tag{}, __alloc) {
+    insert(sorted_unique, __first, __last);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI flat_map(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
+      : flat_map(__il.begin(), __il.end(), __comp) {}
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI
+  flat_map(initializer_list<value_type> __il, const key_compare& __comp, const _Allocator& __alloc)
+      : flat_map(__il.begin(), __il.end(), __comp, __alloc) {}
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_map(initializer_list<value_type> __il, const _Allocator& __alloc)
+      : flat_map(__il.begin(), __il.end(), __alloc) {}
+
+  _LIBCPP_HIDE_FROM_ABI
+  flat_map(sorted_unique_t, initializer_list<value_type> __il, const key_compare& __comp = key_compare())
+      : flat_map(sorted_unique, __il.begin(), __il.end(), __comp) {}
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI
+  flat_map(sorted_unique_t, initializer_list<value_type> __il, const key_compare& __comp, const _Allocator& __alloc)
+      : flat_map(sorted_unique, __il.begin(), __il.end(), __comp, __alloc) {}
+
+  template <class _Allocator>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_map(sorted_unique_t, initializer_list<value_type> __il, const _Allocator& __alloc)
+      : flat_map(sorted_unique, __il.begin(), __il.end(), __alloc) {}
+
+  _LIBCPP_HIDE_FROM_ABI flat_map& operator=(initializer_list<value_type> __il) {
+    clear();
+    insert(__il);
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI flat_map& operator=(const flat_map&) = default;
+
+  _LIBCPP_HIDE_FROM_ABI flat_map& operator=(flat_map&& __other) noexcept(
+      is_nothrow_move_assignable_v<_KeyContainer> && is_nothrow_move_assignable_v<_MappedContainer> &&
+      is_nothrow_move_assignable_v<_Compare>) {
+    // No matter what happens, we always want to clear the other container before returning
+    // since we moved from it
+    auto __clear_other_guard = std::__make_scope_guard([&]() noexcept { __other.clear() /* noexcept */; });
+    {
+      // If an exception is thrown, we have no choice but to clear *this to preserve invariants
+      auto __on_exception = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
+      __containers_       = std::move(__other.__containers_);
+      __compare_          = std::move(__other.__compare_);
+      __on_exception.__complete();
+    }
+    return *this;
+  }
+
+  // iterators
+  _LIBCPP_HIDE_FROM_ABI iterator begin() noexcept {
+    return iterator(__containers_.keys.begin(), __containers_.values.begin());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const noexcept {
+    return const_iterator(__containers_.keys.begin(), __containers_.values.begin());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator end() noexcept {
+    return iterator(__containers_.keys.end(), __containers_.values.end());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator end() const noexcept {
+    return const_iterator(__containers_.keys.end(), __containers_.values.end());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() noexcept { return reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const noexcept { return const_reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() noexcept { return reverse_iterator(begin()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const noexcept { return const_reverse_iterator(begin()); }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const noexcept { return begin(); }
+  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const noexcept { return end(); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const noexcept { return const_reverse_iterator(end()); }
+  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const noexcept { return const_reverse_iterator(begin()); }
+
+  // [flat.map.capacity], capacity
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool empty() const noexcept { return __containers_.keys.empty(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type size() const noexcept { return __containers_.keys.size(); }
+
+  _LIBCPP_HIDE_FROM_ABI size_type max_size() const noexcept {
+    return std::min<size_type>(__containers_.keys.max_size(), __containers_.values.max_size());
+  }
+
+  // [flat.map.access], element access
+  _LIBCPP_HIDE_FROM_ABI mapped_type& operator[](const key_type& __x)
+    requires is_constructible_v<mapped_type>
+  {
+    return try_emplace(__x).first->second;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI mapped_type& operator[](key_type&& __x)
+    requires is_constructible_v<mapped_type>
+  {
+    return try_emplace(std::move(__x)).first->second;
+  }
+
+  template <class _Kp>
+    requires(__is_compare_transparent && is_constructible_v<key_type, _Kp> && is_constructible_v<mapped_type> &&
+             !is_convertible_v<_Kp &&, const_iterator> && !is_convertible_v<_Kp &&, iterator>)
+  _LIBCPP_HIDE_FROM_ABI mapped_type& operator[](_Kp&& __x) {
+    return try_emplace(std::forward<_Kp>(__x)).first->second;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI mapped_type& at(const key_type& __x) {
+    auto __it = find(__x);
+    if (__it == end()) {
+      std::__throw_out_of_range("flat_map::at(const key_type&): Key does not exist");
+    }
+    return __it->second;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const mapped_type& at(const key_type& __x) const {
+    auto __it = find(__x);
+    if (__it == end()) {
+      std::__throw_out_of_range("flat_map::at(const key_type&) const: Key does not exist");
+    }
+    return __it->second;
+  }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI mapped_type& at(const _Kp& __x) {
+    auto __it = find(__x);
+    if (__it == end()) {
+      std::__throw_out_of_range("flat_map::at(const K&): Key does not exist");
+    }
+    return __it->second;
+  }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI const mapped_type& at(const _Kp& __x) const {
+    auto __it = find(__x);
+    if (__it == end()) {
+      std::__throw_out_of_range("flat_map::at(const K&) const: Key does not exist");
+    }
+    return __it->second;
+  }
+
+  // [flat.map.modifiers], modifiers
+  template <class... _Args>
+    requires is_constructible_v<pair<key_type, mapped_type>, _Args...>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> emplace(_Args&&... __args) {
+    std::pair<key_type, mapped_type> __pair(std::forward<_Args>(__args)...);
+    return __try_emplace(std::move(__pair.first), std::move(__pair.second));
+  }
+
+  template <class... _Args>
+    requires is_constructible_v<pair<key_type, mapped_type>, _Args...>
+  _LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __hint, _Args&&... __args) {
+    std::pair<key_type, mapped_type> __pair(std::forward<_Args>(__args)...);
+    return __try_emplace_hint(__hint, std::move(__pair.first), std::move(__pair.second)).first;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(const value_type& __x) { return emplace(__x); }
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(value_type&& __x) { return emplace(std::move(__x)); }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, const value_type& __x) {
+    return emplace_hint(__hint, __x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, value_type&& __x) {
+    return emplace_hint(__hint, std::move(__x));
+  }
+
+  template <class _Pp>
+    requires is_constructible_v<pair<key_type, mapped_type>, _Pp>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(_Pp&& __x) {
+    return emplace(std::forward<_Pp>(__x));
+  }
+
+  template <class _Pp>
+    requires is_constructible_v<pair<key_type, mapped_type>, _Pp>
+  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, _Pp&& __x) {
+    return emplace_hint(__hint, std::forward<_Pp>(__x));
+  }
+
+  template <class _InputIterator>
+    requires __has_input_iterator_category<_InputIterator>::value
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last) {
+    if constexpr (sized_sentinel_for<_InputIterator, _InputIterator>) {
+      __reserve(__last - __first);
+    }
+    __append_sort_merge_unique</*WasSorted = */ false>(std::move(__first), std::move(__last));
+  }
+
+  template <class _InputIterator>
+    requires __has_input_iterator_category<_InputIterator>::value
+  void insert(sorted_unique_t, _InputIterator __first, _InputIterator __last) {
+    if constexpr (sized_sentinel_for<_InputIterator, _InputIterator>) {
+      __reserve(__last - __first);
+    }
+
+    __append_sort_merge_unique</*WasSorted = */ true>(std::move(__first), std::move(__last));
+  }
+
+  template <_ContainerCompatibleRange<value_type> _Range>
+  _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
+    if constexpr (ranges::sized_range<_Range>) {
+      __reserve(ranges::size(__range));
+    }
+
+    __append_sort_merge_unique</*WasSorted = */ false>(ranges::begin(__range), ranges::end(__range));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
+
+  _LIBCPP_HIDE_FROM_ABI void insert(sorted_unique_t, initializer_list<value_type> __il) {
+    insert(sorted_unique, __il.begin(), __il.end());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI containers extract() && {
+    auto __guard = std::__make_scope_guard([&]() noexcept { clear() /* noexcept */; });
+    auto __ret   = std::move(__containers_);
+    return __ret;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void replace(key_container_type&& __key_cont, mapped_container_type&& __mapped_cont) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(
+        __key_cont.size() == __mapped_cont.size(), "flat_map keys and mapped containers have different size");
+
+    _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
+        __is_sorted_and_unique(__key_cont), "Either the key container is not sorted or it contains duplicates");
+    auto __guard         = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
+    __containers_.keys   = std::move(__key_cont);
+    __containers_.values = std::move(__mapped_cont);
+    __guard.__complete();
+  }
+
+  template <class... _Args>
+    requires is_constructible_v<mapped_type, _Args...>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> try_emplace(const key_type& __key, _Args&&... __args) {
+    return __try_emplace(__key, std::forward<_Args>(__args)...);
+  }
+
+  template <class... _Args>
+    requires is_constructible_v<mapped_type, _Args...>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> try_emplace(key_type&& __key, _Args&&... __args) {
+    return __try_emplace(std::move(__key), std::forward<_Args>(__args)...);
+  }
+
+  template <class _Kp, class... _Args>
+    requires(__is_compare_transparent && is_constructible_v<key_type, _Kp> &&
+             is_constructible_v<mapped_type, _Args...> && !is_convertible_v<_Kp &&, const_iterator> &&
+             !is_convertible_v<_Kp &&, iterator>)
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> try_emplace(_Kp&& __key, _Args&&... __args) {
+    return __try_emplace(std::forward<_Kp>(__key), std::forward<_Args>(__args)...);
+  }
+
+  template <class... _Args>
+    requires is_constructible_v<mapped_type, _Args...>
+  _LIBCPP_HIDE_FROM_ABI iterator try_emplace(const_iterator __hint, const key_type& __key, _Args&&... __args) {
+    return __try_emplace_hint(__hint, __key, std::forward<_Args>(__args)...).first;
+  }
+
+  template <class... _Args>
+    requires is_constructible_v<mapped_type, _Args...>
+  _LIBCPP_HIDE_FROM_ABI iterator try_emplace(const_iterator __hint, key_type&& __key, _Args&&... __args) {
+    return __try_emplace_hint(__hint, std::move(__key), std::forward<_Args>(__args)...).first;
+  }
+
+  template <class _Kp, class... _Args>
+    requires __is_compare_transparent && is_constructible_v<key_type, _Kp> && is_constructible_v<mapped_type, _Args...>
+  _LIBCPP_HIDE_FROM_ABI iterator try_emplace(const_iterator __hint, _Kp&& __key, _Args&&... __args) {
+    return __try_emplace_hint(__hint, std::forward<_Kp>(__key), std::forward<_Args>(__args)...).first;
+  }
+
+  template <class _Mapped>
+    requires is_assignable_v<mapped_type&, _Mapped> && is_constructible_v<mapped_type, _Mapped>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert_or_assign(const key_type& __key, _Mapped&& __obj) {
+    return __insert_or_assign(__key, std::forward<_Mapped>(__obj));
+  }
+
+  template <class _Mapped>
+    requires is_assignable_v<mapped_type&, _Mapped> && is_constructible_v<mapped_type, _Mapped>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert_or_assign(key_type&& __key, _Mapped&& __obj) {
+    return __insert_or_assign(std::move(__key), std::forward<_Mapped>(__obj));
+  }
+
+  template <class _Kp, class _Mapped>
+    requires __is_compare_transparent && is_constructible_v<key_type, _Kp> && is_assignable_v<mapped_type&, _Mapped> &&
+             is_constructible_v<mapped_type, _Mapped>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert_or_assign(_Kp&& __key, _Mapped&& __obj) {
+    return __insert_or_assign(std::forward<_Kp>(__key), std::forward<_Mapped>(__obj));
+  }
+
+  template <class _Mapped>
+    requires is_assignable_v<mapped_type&, _Mapped> && is_constructible_v<mapped_type, _Mapped>
+  _LIBCPP_HIDE_FROM_ABI iterator insert_or_assign(const_iterator __hint, const key_type& __key, _Mapped&& __obj) {
+    return __insert_or_assign(__hint, __key, std::forward<_Mapped>(__obj));
+  }
+
+  template <class _Mapped>
+    requires is_assignable_v<mapped_type&, _Mapped> && is_constructible_v<mapped_type, _Mapped>
+  _LIBCPP_HIDE_FROM_ABI iterator insert_or_assign(const_iterator __hint, key_type&& __key, _Mapped&& __obj) {
+    return __insert_or_assign(__hint, std::move(__key), std::forward<_Mapped>(__obj));
+  }
+
+  template <class _Kp, class _Mapped>
+    requires __is_compare_transparent && is_constructible_v<key_type, _Kp> && is_assignable_v<mapped_type&, _Mapped> &&
+             is_constructible_v<mapped_type, _Mapped>
+  _LIBCPP_HIDE_FROM_ABI iterator insert_or_assign(const_iterator __hint, _Kp&& __key, _Mapped&& __obj) {
+    return __insert_or_assign(__hint, std::forward<_Kp>(__key), std::forward<_Mapped>(__obj));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(iterator __position) {
+    return __erase(__position.__key_iter_, __position.__mapped_iter_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __position) {
+    return __erase(__position.__key_iter_, __position.__mapped_iter_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __x) {
+    auto __iter = find(__x);
+    if (__iter != end()) {
+      erase(__iter);
+      return 1;
+    }
+    return 0;
+  }
+
+  template <class _Kp>
+    requires(__is_compare_transparent && !is_convertible_v<_Kp &&, iterator> &&
+             !is_convertible_v<_Kp &&, const_iterator>)
+  _LIBCPP_HIDE_FROM_ABI size_type erase(_Kp&& __x) {
+    auto [__first, __last] = equal_range(__x);
+    auto __res             = __last - __first;
+    erase(__first, __last);
+    return __res;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last) {
+    auto __on_failure = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
+    auto __key_it     = __containers_.keys.erase(__first.__key_iter_, __last.__key_iter_);
+    auto __mapped_it  = __containers_.values.erase(__first.__mapped_iter_, __last.__mapped_iter_);
+    __on_failure.__complete();
+    return iterator(std::move(__key_it), std::move(__mapped_it));
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void swap(flat_map& __y) noexcept {
+    // warning: The spec has unconditional noexcept, which means that
+    // if any of the following functions throw an exception,
+    // std::terminate will be called.
+    // This is discussed in P2767, which hasn't been voted on yet.
+    ranges::swap(__compare_, __y.__compare_);
+    ranges::swap(__containers_.keys, __y.__containers_.keys);
+    ranges::swap(__containers_.values, __y.__containers_.values);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void clear() noexcept {
+    __containers_.keys.clear();
+    __containers_.values.clear();
+  }
+
+  // observers
+  _LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __compare_; }
+  _LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return value_compare(__compare_); }
+
+  _LIBCPP_HIDE_FROM_ABI const key_container_type& keys() const noexcept { return __containers_.keys; }
+  _LIBCPP_HIDE_FROM_ABI const mapped_container_type& values() const noexcept { return __containers_.values; }
+
+  // map operations
+  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __x) { return __find_impl(*this, __x); }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __x) const { return __find_impl(*this, __x); }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI iterator find(const _Kp& __x) {
+    return __find_impl(*this, __x);
+  }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _Kp& __x) const {
+    return __find_impl(*this, __x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __x) const { return contains(__x) ? 1 : 0; }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI size_type count(const _Kp& __x) const {
+    return contains(__x) ? 1 : 0;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __x) const { return find(__x) != end(); }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI bool contains(const _Kp& __x) const {
+    return find(__x) != end();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __x) { return __lower_bound<iterator>(*this, __x); }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __x) const {
+    return __lower_bound<const_iterator>(*this, __x);
+  }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _Kp& __x) {
+    return __lower_bound<iterator>(*this, __x);
+  }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _Kp& __x) const {
+    return __lower_bound<const_iterator>(*this, __x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __x) { return __upper_bound<iterator>(*this, __x); }
+
+  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __x) const {
+    return __upper_bound<const_iterator>(*this, __x);
+  }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _Kp& __x) {
+    return __upper_bound<iterator>(*this, __x);
+  }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _Kp& __x) const {
+    return __upper_bound<const_iterator>(*this, __x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __x) {
+    return __equal_range_impl(*this, __x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __x) const {
+    return __equal_range_impl(*this, __x);
+  }
+
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _Kp& __x) {
+    return __equal_range_impl(*this, __x);
+  }
+  template <class _Kp>
+    requires __is_compare_transparent
+  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _Kp& __x) const {
+    return __equal_range_impl(*this, __x);
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const flat_map& __x, const flat_map& __y) {
+    return ranges::equal(__x, __y);
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI auto operator<=>(const flat_map& __x, const flat_map& __y) {
+    return std::lexicographical_compare_three_way(
+        __x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
+  }
+
+  friend _LIBCPP_HIDE_FROM_ABI void swap(flat_map& __x, flat_map& __y) noexcept { __x.swap(__y); }
+
+private:
+  struct __ctor_uses_allocator_tag {
+    explicit _LIBCPP_HIDE_FROM_ABI __ctor_uses_allocator_tag() = default;
+  };
+  struct __ctor_uses_allocator_empty_tag {
+    explicit _LIBCPP_HIDE_FROM_ABI __ctor_uses_allocator_empty_tag() = default;
+  };
+
+  template <class _Allocator, class _KeyCont, class _MappedCont, class... _CompArg>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI
+  flat_map(__ctor_uses_allocator_tag,
+           const _Allocator& __alloc,
+           _KeyCont&& __key_cont,
+           _MappedCont&& __mapped_cont,
+           _CompArg&&... __comp)
+      : __containers_{.keys = std::make_obj_using_allocator<key_container_type>(
+                          __alloc, std::forward<_KeyCont>(__key_cont)),
+                      .values = std::make_obj_using_allocator<mapped_container_type>(
+                          __alloc, std::forward<_MappedCont>(__mapped_cont))},
+        __compare_(std::forward<_CompArg>(__comp)...) {}
+
+  template <class _Allocator, class... _CompArg>
+    requires __allocator_ctor_constraint<_Allocator>
+  _LIBCPP_HIDE_FROM_ABI flat_map(__ctor_uses_allocator_empty_tag, const _Allocator& __alloc, _CompArg&&... __comp)
+      : __containers_{.keys   = std::make_obj_using_allocator<key_container_type>(__alloc),
+                      .values = std::make_obj_using_allocator<mapped_container_type>(__alloc)},
+        __compare_(std::forward<_CompArg>(__comp)...) {}
+
+  _LIBCPP_HIDE_FROM_ABI bool __is_sorted_and_unique(auto&& __key_container) const {
+    auto __greater_or_equal_to = [this](const auto& __x, const auto& __y) { return !__compare_(__x, __y); };
+    return ranges::adjacent_find(__key_container, __greater_or_equal_to) == ranges::end(__key_container);
+  }
+
+  // This function is only used in constructors. So there is not exception handling in this function.
+  // If the function exits via an exception, there will be no flat_map object constructed, thus, there
+  // is no invariant state to preserve
+  _LIBCPP_HIDE_FROM_ABI void __sort_and_unique() {
+    auto __zv = ranges::views::zip(__containers_.keys, __containers_.values);
+    // To be consistent with std::map's behaviour, we use stable_sort instead of sort.
+    // As a result, if there are duplicated keys, the first value in the original order will be taken.
+    ranges::stable_sort(__zv, __compare_, [](const auto& __p) -> decltype(auto) { return std::get<0>(__p); });
+    auto __dup_start = ranges::unique(__zv, __key_equiv(__compare_)).begin();
+    auto __dist      = ranges::distance(__zv.begin(), __dup_start);
+    __containers_.keys.erase(__containers_.keys.begin() + __dist, __containers_.keys.end());
+    __containers_.values.erase(__containers_.values.begin() + __dist, __containers_.values.end());
+  }
+
+  template <class _InputIterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI size_type __append(_InputIterator __first, _Sentinel __last) {
+    size_type __num_of_appended = 0;
+    for (; __first != __last; ++__first) {
+      value_type __kv = *__first;
+      __containers_.keys.insert(__containers_.keys.end(), std::move(__kv.first));
+      __containers_.values.insert(__containers_.values.end(), std::move(__kv.second));
+      ++__num_of_appended;
+    }
+    return __num_of_appended;
+  }
+
+  template <bool _WasSorted, class _InputIterator, class _Sentinel>
+  _LIBCPP_HIDE_FROM_ABI void __append_sort_merge_unique(_InputIterator __first, _Sentinel __last) {
+    auto __on_failure        = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
+    size_t __num_of_appended = __append(std::move(__first), std::move(__last));
+    if (__num_of_appended != 0) {
+      auto __zv                  = ranges::views::zip(__containers_.keys, __containers_.values);
+      auto __append_start_offset = __containers_.keys.size() - __num_of_appended;
+      auto __end                 = __zv.end();
+      auto __compare_key         = [this](const auto& __p1, const auto& __p2) {
+        return __compare_(std::get<0>(__p1), std::get<0>(__p2));
+      };
+      if constexpr (!_WasSorted) {
+        ranges::stable_sort(__zv.begin() + __append_start_offset, __end, __compare_key);
+      } else {
+        _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
+            __is_sorted_and_unique(__containers_.keys | ranges::views::drop(__append_start_offset)),
+            "Either the key container is not sorted or it contains duplicates");
+      }
+      ranges::inplace_merge(__zv.begin(), __zv.begin() + __append_start_offset, __end, __compare_key);
+
+      auto __dup_start = ranges::unique(__zv, __key_equiv(__compare_)).begin();
+      auto __dist      = ranges::distance(__zv.begin(), __dup_start);
+      __containers_.keys.erase(__containers_.keys.begin() + __dist, __containers_.keys.end());
+      __containers_.values.erase(__containers_.values.begin() + __dist, __containers_.values.end());
+    }
+    __on_failure.__complete();
+  }
+
+  template <class _Self, class _Kp>
+  _LIBCPP_HIDE_FROM_ABI static auto __find_impl(_Self&& __self, const _Kp& __key) {
+    auto __it   = __self.lower_bound(__key);
+    auto __last = __self.end();
+    if (__it == __last || __self.__compare_(__key, __it->first)) {
+      return __last;
+    }
+    return __it;
+  }
+
+  template <class _Self, class _Kp>
+  _LIBCPP_HIDE_FROM_ABI static auto __key_equal_range(_Self&& __self, const _Kp& __key) {
+    auto __it   = ranges::lower_bound(__self.__containers_.keys, __key, __self.__compare_);
+    auto __last = __self.__containers_.keys.end();
+    if (__it == __last || __self.__compare_(__key, *__it)) {
+      return std::make_pair(__it, __it);
+    }
+    return std::make_pair(__it, std::next(__it));
+  }
+
+  template <class _Self, class _Kp>
+  _LIBCPP_HIDE_FROM_ABI static auto __equal_range_impl(_Self&& __self, const _Kp& __key) {
+    auto [__key_first, __key_last] = __key_equal_range(__self, __key);
+
+    const auto __make_mapped_iter = [&](const auto& __key_iter) {
+      return __self.__containers_.values.begin() +
+             static_cast<ranges::range_difference_t<mapped_container_type>>(
+                 ranges::distance(__self.__containers_.keys.begin(), __key_iter));
+    };
+
+    using __iterator_type = ranges::iterator_t<decltype(__self)>;
+    return std::make_pair(__iterator_type(__key_first, __make_mapped_iter(__key_first)),
+                          __iterator_type(__key_last, __make_mapped_iter(__key_last)));
+  }
+
+  template <class _Res, class _Self, class _Kp>
+  _LIBCPP_HIDE_FROM_ABI static _Res __lower_bound(_Self&& __self, _Kp& __x) {
+    return __binary_search<_Res>(__self, ranges::lower_bound, __x);
+  }
+
+  template <class _Res, class _Self, class _Kp>
+  _LIBCPP_HIDE_FROM_ABI static _Res __upper_bound(_Self&& __self, _Kp& __x) {
+    return __binary_search<_Res>(__self, ranges::upper_bound, __x);
+  }
+
+  template <class _Res, class _Self, class _Fn, class _Kp>
+  _LIBCPP_HIDE_FROM_ABI static _Res __binary_search(_Self&& __self, _Fn __search_fn, _Kp& __x) {
+    auto __key_iter = __search_fn(__self.__containers_.keys, __x, __self.__compare_);
+    auto __mapped_iter =
+        __self.__containers_.values.begin() +
+        static_cast<ranges::range_difference_t<mapped_container_type>>(
+            ranges::distance(__self.__containers_.keys.begin(), __key_iter));
+
+    return _Res(std::move(__key_iter), std::move(__mapped_iter));
+  }
+
+  template <class _KeyArg, class... _MArgs>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __try_emplace(_KeyArg&& __key, _MArgs&&... __mapped_args) {
+    auto __key_it    = ranges::lower_bound(__containers_.keys, __key, __compare_);
+    auto __mapped_it = __containers_.values.begin() + ranges::distance(__containers_.keys.begin(), __key_it);
+
+    if (__key_it == __containers_.keys.end() || __compare_(__key, *__key_it)) {
+      return pair<iterator, bool>(
+          __try_emplace_exact_hint(
+              std::move(__key_it),
+              std::move(__mapped_it),
+              std::forward<_KeyArg>(__key),
+              std::forward<_MArgs>(__mapped_args)...),
+          true);
+    } else {
+      return pair<iterator, bool>(iterator(std::move(__key_it), std::move(__mapped_it)), false);
+    }
+  }
+
+  template <class _Kp>
+  _LIBCPP_HIDE_FROM_ABI bool __is_hint_correct(const_iterator __hint, _Kp&& __key) {
+    if (__hint != cbegin() && !__compare_((__hint - 1)->first, __key)) {
+      return false;
+    }
+    if (__hint != cend() && __compare_(__hint->first, __key)) {
+      return false;
+    }
+    return true;
+  }
+
+  template <class _Kp, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __try_emplace_hint(const_iterator __hint, _Kp&& __key, _Args&&... __args) {
+    if (__is_hint_correct(__hint, __key)) {
+      if (__hint == cend() || __compare_(__key, __hint->first)) {
+        return {
+            __try_emplace_exact_hint(
+                __hint.__key_iter_, __hint.__mapped_iter_, std::forward<_Kp>(__key), std::forward<_Args>(__args)...),
+            true};
+      } else {
+        // key equals
+        auto __dist = __hint - cbegin();
+        return {iterator(__containers_.keys.begin() + __dist, __containers_.values.begin() + __dist), false};
+      }
+    } else {
+      return __try_emplace(std::forward<_Kp>(__key), std::forward<_Args>(__args)...);
+    }
+  }
+
+  template <class _IterK, class _IterM, class _KeyArg, class... _MArgs>
+  _LIBCPP_HIDE_FROM_ABI iterator
+  __try_emplace_exact_hint(_IterK&& __it_key, _IterM&& __it_mapped, _KeyArg&& __key, _MArgs&&... __mapped_args) {
+    auto __on_key_failed = std::__make_exception_guard([&]() noexcept {
+      if constexpr (__container_traits<_KeyContainer>::__emplacement_has_strong_exception_safety_guarantee) {
+        // Nothing to roll back!
+      } else {
+        // we need to clear both because we don't know the state of our keys anymore
+        clear() /* noexcept */;
+      }
+    });
+    auto __key_it        = __containers_.keys.emplace(__it_key, std::forward<_KeyArg>(__key));
+    __on_key_failed.__complete();
+
+    auto __on_value_failed = std::__make_exception_guard([&]() noexcept {
+      if constexpr (!__container_traits<_MappedContainer>::__emplacement_has_strong_exception_safety_guarantee) {
+        // we need to clear both because we don't know the state of our values anymore
+        clear() /* noexcept */;
+      } else {
+        // In this case, we know the values are just like before we attempted emplacement,
+        // and we also know that the keys have been emplaced successfully. Just roll back the keys.
+#  if _LIBCPP_HAS_EXCEPTIONS
+        try {
+#  endif // _LIBCPP_HAS_EXCEPTIONS
+          __containers_.keys.erase(__key_it);
+#  if _LIBCPP_HAS_EXCEPTIONS
+        } catch (...) {
+          // Now things are funky for real. We're failing to rollback the keys.
+          // Just give up and clear the whole thing.
+          //
+          // Also, swallow the exception that happened during the rollback and let the
+          // original value-emplacement exception propagate normally.
+          clear() /* noexcept */;
+        }
+#  endif // _LIBCPP_HAS_EXCEPTIONS
+      }
+    });
+    auto __mapped_it = __containers_.values.emplace(__it_mapped, std::forward<_MArgs>(__mapped_args)...);
+    __on_value_failed.__complete();
+
+    return iterator(std::move(__key_it), std::move(__mapped_it));
+  }
+
+  template <class _Kp, class _Mapped>
+  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __insert_or_assign(_Kp&& __key, _Mapped&& __mapped) {
+    auto __r = try_emplace(std::forward<_Kp>(__key), std::forward<_Mapped>(__mapped));
+    if (!__r.second) {
+      __r.first->second = std::forward<_Mapped>(__mapped);
+    }
+    return __r;
+  }
+
+  template <class _Kp, class _Mapped>
+  _LIBCPP_HIDE_FROM_ABI iterator __insert_or_assign(const_iterator __hint, _Kp&& __key, _Mapped&& __mapped) {
+    auto __r = __try_emplace_hint(__hint, std::forward<_Kp>(__key), std::forward<_Mapped>(__mapped));
+    if (!__r.second) {
+      __r.first->second = std::forward<_Mapped>(__mapped);
+    }
+    return __r.first;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __reserve(size_t __size) {
+    if constexpr (requires { __containers_.keys.reserve(__size); }) {
+      __containers_.keys.reserve(__size);
+    }
+
+    if constexpr (requires { __containers_.values.reserve(__size); }) {
+      __containers_.values.reserve(__size);
+    }
+  }
+
+  template <class _KIter, class _MIter>
+  _LIBCPP_HIDE_FROM_ABI iterator __erase(_KIter __key_iter_to_remove, _MIter __mapped_iter_to_remove) {
+    auto __on_failure  = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
+    auto __key_iter    = __containers_.keys.erase(__key_iter_to_remove);
+    auto __mapped_iter = __containers_.values.erase(__mapped_iter_to_remove);
+    __on_failure.__complete();
+    return iterator(std::move(__key_iter), std::move(__mapped_iter));
+  }
+
+  template <class _Key2, class _Tp2, class _Compare2, class _KeyContainer2, class _MappedContainer2, class _Predicate>
+  friend typename flat_map<_Key2, _Tp2, _Compare2, _KeyContainer2, _MappedContainer2>::size_type
+  erase_if(flat_map<_Key2, _Tp2, _Compare2, _KeyContainer2, _MappedContainer2>&, _Predicate);
+
+  containers __containers_;
+  [[no_unique_address]] key_compare __compare_;
+
+  struct __key_equiv {
+    _LIBCPP_HIDE_FROM_ABI __key_equiv(key_compare __c) : __comp_(__c) {}
+    _LIBCPP_HIDE_FROM_ABI bool operator()(const_reference __x, const_reference __y) const {
+      return !__comp_(std::get<0>(__x), std::get<0>(__y)) && !__comp_(std::get<0>(__y), std::get<0>(__x));
+    }
+    key_compare __comp_;
+  };
+};
+
+template <class _KeyContainer, class _MappedContainer, class _Compare = less<typename _KeyContainer::value_type>>
+  requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
+           !__is_allocator<_MappedContainer>::value &&
+           is_invocable_v<const _Compare&,
+                          const typename _KeyContainer::value_type&,
+                          const typename _KeyContainer::value_type&>)
+flat_map(_KeyContainer, _MappedContainer, _Compare = _Compare())
+    -> flat_map<typename _KeyContainer::value_type,
+                typename _MappedContainer::value_type,
+                _Compare,
+                _KeyContainer,
+                _MappedContainer>;
+
+template <class _KeyContainer, class _MappedContainer, class _Allocator>
+  requires(uses_allocator_v<_KeyContainer, _Allocator> && uses_allocator_v<_MappedContainer, _Allocator> &&
+           !__is_allocator<_KeyContainer>::value && !__is_allocator<_MappedContainer>::value)
+flat_map(_KeyContainer, _MappedContainer, _Allocator)
+    -> flat_map<typename _KeyContainer::value_type,
+                typename _MappedContainer::value_type,
+                less<typename _KeyContainer::value_type>,
+                _KeyContainer,
+                _MappedContainer>;
+
+template <class _KeyContainer, class _MappedContainer, class _Compare, class _Allocator>
+  requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
+           !__is_allocator<_MappedContainer>::value && uses_allocator_v<_KeyContainer, _Allocator> &&
+           uses_allocator_v<_MappedContainer, _Allocator> &&
+           is_invocable_v<const _Compare&,
+                          const typename _KeyContainer::value_type&,
+                          const typename _KeyContainer::value_type&>)
+flat_map(_KeyContainer, _MappedContainer, _Compare, _Allocator)
+    -> flat_map<typename _KeyContainer::value_type,
+                typename _MappedContainer::value_type,
+                _Compare,
+                _KeyContainer,
+                _MappedContainer>;
+
+template <class _KeyContainer, class _MappedContainer, class _Compare = less<typename _KeyContainer::value_type>>
+  requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
+           !__is_allocator<_MappedContainer>::value &&
+           is_invocable_v<const _Compare&,
+                          const typename _KeyContainer::value_type&,
+                          const typename _KeyContainer::value_type&>)
+flat_map(sorted_unique_t, _KeyContainer, _MappedContainer, _Compare = _Compare())
+    -> flat_map<typename _KeyContainer::value_type,
+                typename _MappedContainer::value_type,
+                _Compare,
+                _KeyContainer,
+                _MappedContainer>;
+
+template <class _KeyContainer, class _MappedContainer, class _Allocator>
+  requires(uses_allocator_v<_KeyContainer, _Allocator> && uses_allocator_v<_MappedContainer, _Allocator> &&
+           !__is_allocator<_KeyContainer>::value && !__is_allocator<_MappedContainer>::value)
+flat_map(sorted_unique_t, _KeyContainer, _MappedContainer, _Allocator)
+    -> flat_map<typename _KeyContainer::value_type,
+                typename _MappedContainer::value_type,
+                less<typename _KeyContainer::value_type>,
+                _KeyContainer,
+                _MappedContainer>;
+
+template <class _KeyContainer, class _MappedContainer, class _Compare, class _Allocator>
+  requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
+           !__is_allocator<_MappedContainer>::value && uses_allocator_v<_KeyContainer, _Allocator> &&
+           uses_allocator_v<_MappedContainer, _Allocator> &&
+           is_invocable_v<const _Compare&,
+                          const typename _KeyContainer::value_type&,
+                          const typename _KeyContainer::value_type&>)
+flat_map(sorted_unique_t, _KeyContainer, _MappedContainer, _Compare, _Allocator)
+    -> flat_map<typename _KeyContainer::value_type,
+                typename _MappedContainer::value_type,
+                _Compare,
+                _KeyContainer,
+                _MappedContainer>;
+
+template <class _InputIterator, class _Compare = less<__iter_key_type<_InputIterator>>>
+  requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator<_Compare>::value)
+flat_map(_InputIterator, _InputIterator, _Compare = _Compare())
+    -> flat_map<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Compare>;
+
+template <class _InputIterator, class _Compare = less<__iter_key_type<_InputIterator>>>
+  requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator<_Compare>::value)
+flat_map(sorted_unique_t, _InputIterator, _InputIterator, _Compare = _Compare())
+    -> flat_map<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Compare>;
+
+template <ranges::input_range _Range,
+          class _Compare   = less<__range_key_type<_Range>>,
+          class _Allocator = allocator<byte>,
+          class            = __enable_if_t<!__is_allocator<_Compare>::value && __is_allocator<_Allocator>::value>>
+flat_map(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator())
+    -> flat_map<
+        __range_key_type<_Range>,
+        __range_mapped_type<_Range>,
+        _Compare,
+        vector<__range_key_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_key_type<_Range>>>,
+        vector<__range_mapped_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_mapped_type<_Range>>>>;
+
+template <ranges::input_range _Range, class _Allocator, class = __enable_if_t<__is_allocator<_Allocator>::value>>
+flat_map(from_range_t, _Range&&, _Allocator)
+    -> flat_map<
+        __range_key_type<_Range>,
+        __range_mapped_type<_Range>,
+        less<__range_key_type<_Range>>,
+        vector<__range_key_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_key_type<_Range>>>,
+        vector<__range_mapped_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_mapped_type<_Range>>>>;
+
+template <class _Key, class _Tp, class _Compare = less<_Key>>
+  requires(!__is_allocator<_Compare>::value)
+flat_map(initializer_list<pair<_Key, _Tp>>, _Compare = _Compare()) -> flat_map<_Key, _Tp, _Compare>;
+
+template <class _Key, class _Tp, class _Compare = less<_Key>>
+  requires(!__is_allocator<_Compare>::value)
+flat_map(sorted_unique_t, initializer_list<pair<_Key, _Tp>>, _Compare = _Compare()) -> flat_map<_Key, _Tp, _Compare>;
+
+template <class _Key, class _Tp, class _Compare, class _KeyContainer, class _MappedContainer, class _Allocator>
+struct uses_allocator<flat_map<_Key, _Tp, _Compare, _KeyContainer, _MappedContainer>, _Allocator>
+    : bool_constant<uses_allocator_v<_KeyContainer, _Allocator> && uses_allocator_v<_MappedContainer, _Allocator>> {};
+
+template <class _Key, class _Tp, class _Compare, class _KeyContainer, class _MappedContainer, class _Predicate>
+_LIBCPP_HIDE_FROM_ABI typename flat_map<_Key, _Tp, _Compare, _KeyContainer, _MappedContainer>::size_type
+erase_if(flat_map<_Key, _Tp, _Compare, _KeyContainer, _MappedContainer>& __flat_map, _Predicate __pred) {
+  auto __zv     = ranges::views::zip(__flat_map.__containers_.keys, __flat_map.__containers_.values);
+  auto __first  = __zv.begin();
+  auto __last   = __zv.end();
+  auto __guard  = std::__make_exception_guard([&] { __flat_map.clear(); });
+  auto __it     = std::remove_if(__first, __last, [&](auto&& __zipped) -> bool {
+    using _Ref = typename flat_map<_Key, _Tp, _Compare, _KeyContainer, _MappedContainer>::const_reference;
+    return __pred(_Ref(std::get<0>(__zipped), std::get<1>(__zipped)));
+  });
+  auto __res    = __last - __it;
+  auto __offset = __it - __first;
+
+  const auto __erase_container = [&](auto& __cont) { __cont.erase(__cont.begin() + __offset, __cont.end()); };
+
+  __erase_container(__flat_map.__containers_.keys);
+  __erase_container(__flat_map.__containers_.values);
+
+  __guard.__complete();
+  return __res;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FLAT_MAP_FLAT_MAP_H
diff --git a/libcxx/include/__flat_map/sorted_unique.h b/libcxx/include/__flat_map/sorted_unique.h
new file mode 100644
index 000000000000..0189a5ff1d56
--- /dev/null
+++ b/libcxx/include/__flat_map/sorted_unique.h
@@ -0,0 +1,31 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef _LIBCPP___FLAT_MAP_SORTED_UNIQUE_H
+#define _LIBCPP___FLAT_MAP_SORTED_UNIQUE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct sorted_unique_t {
+  explicit sorted_unique_t() = default;
+};
+inline constexpr sorted_unique_t sorted_unique{};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+#endif // _LIBCPP___FLAT_MAP_SORTED_UNIQUE_H
diff --git a/libcxx/include/__utility/exception_guard.h b/libcxx/include/__utility/exception_guard.h
index a03bd7e8f352..00b835d3e2a2 100644
--- a/libcxx/include/__utility/exception_guard.h
+++ b/libcxx/include/__utility/exception_guard.h
@@ -137,6 +137,12 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __exception_guard<_Rollback> __make_exce
   return __exception_guard<_Rollback>(std::move(__rollback));
 }
 
+template <class _Rollback>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __exception_guard_exceptions<_Rollback>
+__make_scope_guard(_Rollback __rollback) {
+  return __exception_guard_exceptions<_Rollback>(std::move(__rollback));
+}
+
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
diff --git a/libcxx/include/flat_map b/libcxx/include/flat_map
new file mode 100644
index 000000000000..15d79dd1ddca
--- /dev/null
+++ b/libcxx/include/flat_map
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_FLAT_MAP
+#define _LIBCPP_FLAT_MAP
+
+/*
+  Header <flat_map> synopsis
+
+#include <compare>              // see [compare.syn]
+#include <initializer_list>     // see [initializer.list.syn]
+
+namespace std {
+  // [flat.map], class template flat_map
+  template<class Key, class T, class Compare = less<Key>,
+           class KeyContainer = vector<Key>, class MappedContainer = vector<T>>
+    class flat_map;
+
+  struct sorted_unique_t { explicit sorted_unique_t() = default; };
+  inline constexpr sorted_unique_t sorted_unique{};
+
+  template<class Key, class T, class Compare, class KeyContainer, class MappedContainer,
+           class Allocator>
+    struct uses_allocator<flat_map<Key, T, Compare, KeyContainer, MappedContainer>,
+                          Allocator>;
+
+  // [flat.map.erasure], erasure for flat_map
+  template<class Key, class T, class Compare, class KeyContainer, class MappedContainer,
+           class Predicate>
+    typename flat_map<Key, T, Compare, KeyContainer, MappedContainer>::size_type
+      erase_if(flat_map<Key, T, Compare, KeyContainer, MappedContainer>& c, Predicate pred);
+*/
+
+#include <__assert> // all public C++ headers provide the assertion handler
+#include <__config>
+#include <__flat_map/flat_map.h>
+#include <__flat_map/sorted_unique.h>
+#include <version>
+
+// standard required includes
+#include <compare>
+#include <initializer_list>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#endif // _LIBCPP_FLAT_MAP
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 05d08cfbd7cd..c3561590e06d 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1222,6 +1222,14 @@ module std [system] {
     export *
   }
 
+  module flat_map {
+    module flat_map                       { header "__flat_map/flat_map.h" }
+    module sorted_unique                  { header "__flat_map/sorted_unique.h" }
+
+    header "flat_map"
+    export *
+  }
+  
   module format {
     module buffer                             { header "__format/buffer.h" }
     module concepts                           { header "__format/concepts.h" }
diff --git a/libcxx/modules/std.compat.cppm.in b/libcxx/modules/std.compat.cppm.in
index 797b413f68e2..fbc2c7d94cfa 100644
--- a/libcxx/modules/std.compat.cppm.in
+++ b/libcxx/modules/std.compat.cppm.in
@@ -53,9 +53,6 @@ module;
 #  if __has_include(<debugging>)
 #    error "please update the header information for <debugging> in headers_not_available in utils/libcxx/header_information.py"
 #  endif // __has_include(<debugging>)
-#  if __has_include(<flat_map>)
-#    error "please update the header information for <flat_map> in headers_not_available in utils/libcxx/header_information.py"
-#  endif // __has_include(<flat_map>)
 #  if __has_include(<flat_set>)
 #    error "please update the header information for <flat_set> in headers_not_available in utils/libcxx/header_information.py"
 #  endif // __has_include(<flat_set>)
diff --git a/libcxx/modules/std.cppm.in b/libcxx/modules/std.cppm.in
index 64ed8d4088cc..b4889e5a69e4 100644
--- a/libcxx/modules/std.cppm.in
+++ b/libcxx/modules/std.cppm.in
@@ -64,6 +64,7 @@ module;
 #include <execution>
 #include <expected>
 #include <filesystem>
+#include <flat_map>
 #include <format>
 #include <forward_list>
 #if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
@@ -161,9 +162,6 @@ module;
 #  if __has_include(<debugging>)
 #    error "please update the header information for <debugging> in headers_not_available in utils/libcxx/header_information.py"
 #  endif // __has_include(<debugging>)
-#  if __has_include(<flat_map>)
-#    error "please update the header information for <flat_map> in headers_not_available in utils/libcxx/header_information.py"
-#  endif // __has_include(<flat_map>)
 #  if __has_include(<flat_set>)
 #    error "please update the header information for <flat_set> in headers_not_available in utils/libcxx/header_information.py"
 #  endif // __has_include(<flat_set>)
diff --git a/libcxx/modules/std/flat_map.inc b/libcxx/modules/std/flat_map.inc
index 83cd20ad6189..6a86229bceab 100644
--- a/libcxx/modules/std/flat_map.inc
+++ b/libcxx/modules/std/flat_map.inc
@@ -8,8 +8,8 @@
 //===----------------------------------------------------------------------===//
 
 export namespace std {
-#if 0
-  // [flat.map], class template flat_­map
+#if _LIBCPP_STD_VER >= 23
+  // [flat.map], class template flat_map
   using std::flat_map;
 
   using std::sorted_unique;
@@ -17,15 +17,17 @@ export namespace std {
 
   using std::uses_allocator;
 
-  // [flat.map.erasure], erasure for flat_­map
+  // [flat.map.erasure], erasure for flat_map
   using std::erase_if;
 
-  // [flat.multimap], class template flat_­multimap
+#endif // _LIBCPP_STD_VER >= 23
+#if 0
+  // [flat.multimap], class template flat_multimap
   using std::flat_multimap;
 
   using std::sorted_equivalent;
   using std::sorted_equivalent_t;
 
-  // [flat.multimap.erasure], erasure for flat_­multimap
+  // [flat.multimap.erasure], erasure for flat_multimap
 #endif
 } // namespace std
diff --git a/libcxx/test/libcxx/containers/containers.adaptors/flat.map/assert.input_range.pass.cpp b/libcxx/test/libcxx/containers/containers.adaptors/flat.map/assert.input_range.pass.cpp
new file mode 100644
index 000000000000..2db803b53441
--- /dev/null
+++ b/libcxx/test/libcxx/containers/containers.adaptors/flat.map/assert.input_range.pass.cpp
@@ -0,0 +1,66 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: libcpp-hardening-mode=none
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+
+// <flat_map>
+
+// flat_map(key_container_type , mapped_container_type , const key_compare& __comp = key_compare())
+// flat_map(const key_container_type& , const mapped_container_type& , const _Allocator& )
+// flat_map(const key_container_type& , const mapped_container_type& , const key_compare&, const _Allocator& )
+// void replace(key_container_type&& , mapped_container_type&&)
+//
+
+#include <flat_map>
+#include <functional>
+#include <memory>
+#include <vector>
+
+#include "check_assertion.h"
+
+int main(int, char**) {
+  using M = std::flat_map<int, int>;
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] { M m({1, 2, 3}, {4}); }()), "flat_map keys and mapped containers have different size");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] { M m({1, 2, 3}, {4}, std::less<int>{}); }()), "flat_map keys and mapped containers have different size");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector keys{1, 2, 3};
+        const std::vector values{4};
+        const std::allocator<int> alloc{};
+        M m(keys, values, alloc);
+      }()),
+      "flat_map keys and mapped containers have different size");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector keys{1, 2, 3};
+        const std::vector values{4};
+        const std::less<int> key_compare{};
+        const std::allocator<int> alloc{};
+        M m(keys, values, key_compare, alloc);
+      }()),
+      "flat_map keys and mapped containers have different size");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::vector keys{1, 2, 3};
+        std::vector values{4};
+        M m;
+        m.replace(std::move(keys), std::move(values));
+      }()),
+      "flat_map keys and mapped containers have different size");
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/containers/containers.adaptors/flat.map/assert.sorted_unique.pass.cpp b/libcxx/test/libcxx/containers/containers.adaptors/flat.map/assert.sorted_unique.pass.cpp
new file mode 100644
index 000000000000..e6bd3f385af9
--- /dev/null
+++ b/libcxx/test/libcxx/containers/containers.adaptors/flat.map/assert.sorted_unique.pass.cpp
@@ -0,0 +1,225 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: libcpp-hardening-mode=none
+// REQUIRES: libcpp-hardening-mode=debug
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+
+// <flat_map>
+
+// flat_map(key_container_type , mapped_container_type , const key_compare& __comp = key_compare())
+// flat_map(const key_container_type& , const mapped_container_type& , const _Allocator& )
+// flat_map(const key_container_type& , const mapped_container_type& , const key_compare&, const _Allocator& )
+// void replace(key_container_type&& , mapped_container_type&&)
+//
+
+#include <flat_map>
+#include <functional>
+#include <initializer_list>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "check_assertion.h"
+
+int main(int, char**) {
+  using M = std::flat_map<int, int>;
+
+  TEST_LIBCPP_ASSERT_FAILURE(([] { M m(std::sorted_unique, {2, 2, 3}, {4, 5, 6}); }()),
+                             "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(([] { M m(std::sorted_unique, {4, 2, 3}, {4, 5, 6}); }()),
+                             "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(([] { M m(std::sorted_unique, {2, 2, 3}, {4, 5, 6}, std::less<int>{}); }()),
+                             "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(([] { M m(std::sorted_unique, {4, 2, 3}, {4, 5, 6}, std::less<int>{}); }()),
+                             "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector keys{2, 2, 3};
+        const std::vector values{4, 5, 6};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_unique, keys, values, alloc);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector keys{4, 2, 3};
+        const std::vector values{4, 5, 6};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_unique, keys, values, alloc);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector keys{2, 2, 3};
+        const std::vector values{4, 5, 6};
+        const std::allocator<int> alloc{};
+        const std::less<int> comp{};
+        M m(std::sorted_unique, keys, values, comp, alloc);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector keys{4, 2, 3};
+        const std::vector values{4, 5, 6};
+        const std::allocator<int> alloc{};
+        const std::less<int> comp{};
+        M m(std::sorted_unique, keys, values, comp, alloc);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{2, 4}, {2, 5}, {3, 6}};
+        const std::less<int> comp{};
+        M m(std::sorted_unique, v.begin(), v.end(), comp);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        const std::less<int> comp{};
+        M m(std::sorted_unique, v.begin(), v.end(), comp);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{2, 4}, {2, 5}, {3, 6}};
+        const std::less<int> comp{};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_unique, v.begin(), v.end(), comp, alloc);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        const std::less<int> comp{};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_unique, v.begin(), v.end(), comp, alloc);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{2, 4}, {2, 5}, {3, 6}};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_unique, v.begin(), v.end(), alloc);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_unique, v.begin(), v.end(), alloc);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{2, 4}, {2, 5}, {3, 6}};
+        const std::less<int> comp{};
+        M m(std::sorted_unique, v, comp);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        const std::less<int> comp{};
+        M m(std::sorted_unique, v, comp);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{2, 4}, {2, 5}, {3, 6}};
+        const std::less<int> comp{};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_unique, v, comp, alloc);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        const std::less<int> comp{};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_unique, v, comp, alloc);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{2, 4}, {2, 5}, {3, 6}};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_unique, v, alloc);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        const std::allocator<int> alloc{};
+        M m(std::sorted_unique, v, alloc);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{2, 4}, {2, 5}, {3, 6}};
+        M m;
+        m.insert(std::sorted_unique, v.begin(), v.end());
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        const std::vector<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        M m;
+        m.insert(std::sorted_unique, v.begin(), v.end());
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{2, 4}, {2, 5}, {3, 6}};
+        M m;
+        m.insert(std::sorted_unique, v);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::initializer_list<std::pair<int, int>> v{{4, 4}, {2, 5}, {3, 6}};
+        M m;
+        m.insert(std::sorted_unique, v);
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        std::vector keys{1, 1, 3};
+        std::vector values{4, 5, 6};
+        M m;
+        m.replace(std::move(keys), std::move(values));
+      }()),
+      "Either the key container is not sorted or it contains duplicates");
+  return 0;
+}
diff --git a/libcxx/test/libcxx/containers/containers.adaptors/flat.map/container_stability.pass.cpp b/libcxx/test/libcxx/containers/containers.adaptors/flat.map/container_stability.pass.cpp
new file mode 100644
index 000000000000..0d90c3250061
--- /dev/null
+++ b/libcxx/test/libcxx/containers/containers.adaptors/flat.map/container_stability.pass.cpp
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map(key_container_type key_cont, mapped_container_type mapped_cont);
+//
+// libc++ uses stable_sort to ensure that flat_map's behavior matches map's,
+// in terms of which duplicate items are kept.
+// This tests a conforming extension.
+
+#include <algorithm>
+#include <cassert>
+#include <cstdint>
+#include <flat_map>
+#include <random>
+#include <map>
+#include <vector>
+
+#include "test_macros.h"
+
+struct Mod256 {
+  bool operator()(int x, int y) const { return (x % 256) < (y % 256); }
+};
+
+int main(int, char**) {
+  std::mt19937 randomness;
+  std::vector<uint16_t> values;
+  std::vector<std::pair<uint16_t, uint16_t>> pairs;
+  for (int i = 0; i < 200; ++i) {
+    uint16_t r = randomness();
+    values.push_back(r);
+    pairs.emplace_back(r, r);
+  }
+
+  {
+    std::map<uint16_t, uint16_t, Mod256> m(pairs.begin(), pairs.end());
+    std::flat_map<uint16_t, uint16_t, Mod256> fm(values, values);
+    assert(fm.size() == m.size());
+    LIBCPP_ASSERT(std::ranges::equal(fm, m));
+  }
+  {
+    std::map<uint16_t, uint16_t, Mod256> m(pairs.begin(), pairs.end());
+    std::flat_map<uint16_t, uint16_t, Mod256> fm(values, values, Mod256());
+    assert(fm.size() == m.size());
+    LIBCPP_ASSERT(std::ranges::equal(fm, m));
+  }
+  {
+    std::map<uint16_t, uint16_t, Mod256> m(pairs.begin(), pairs.end());
+    std::flat_map<uint16_t, uint16_t, Mod256> fm(values, values, std::allocator<int>());
+    assert(fm.size() == m.size());
+    LIBCPP_ASSERT(std::ranges::equal(fm, m));
+  }
+  {
+    std::map<uint16_t, uint16_t, Mod256> m(pairs.begin(), pairs.end());
+    std::flat_map<uint16_t, uint16_t, Mod256> fm(values, values, Mod256(), std::allocator<int>());
+    assert(fm.size() == m.size());
+    LIBCPP_ASSERT(std::ranges::equal(fm, m));
+  }
+  return 0;
+}
diff --git a/libcxx/test/libcxx/transitive_includes/cxx03.csv b/libcxx/test/libcxx/transitive_includes/cxx03.csv
index 506b5cd02c44..2dc84963f089 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx03.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx03.csv
@@ -682,6 +682,50 @@ filesystem typeinfo
 filesystem utility
 filesystem variant
 filesystem version
+flat_map algorithm
+flat_map array
+flat_map atomic
+flat_map bit
+flat_map cctype
+flat_map cerrno
+flat_map climits
+flat_map clocale
+flat_map cmath
+flat_map compare
+flat_map concepts
+flat_map cstdarg
+flat_map cstddef
+flat_map cstdint
+flat_map cstdio
+flat_map cstdlib
+flat_map cstring
+flat_map ctime
+flat_map cwchar
+flat_map cwctype
+flat_map exception
+flat_map initializer_list
+flat_map ios
+flat_map iosfwd
+flat_map iterator
+flat_map limits
+flat_map locale
+flat_map memory
+flat_map mutex
+flat_map new
+flat_map optional
+flat_map ratio
+flat_map stdexcept
+flat_map streambuf
+flat_map string
+flat_map string_view
+flat_map system_error
+flat_map tuple
+flat_map type_traits
+flat_map typeinfo
+flat_map utility
+flat_map variant
+flat_map vector
+flat_map version
 format algorithm
 format array
 format atomic
diff --git a/libcxx/test/libcxx/transitive_includes/cxx11.csv b/libcxx/test/libcxx/transitive_includes/cxx11.csv
index 506b5cd02c44..2dc84963f089 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx11.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx11.csv
@@ -682,6 +682,50 @@ filesystem typeinfo
 filesystem utility
 filesystem variant
 filesystem version
+flat_map algorithm
+flat_map array
+flat_map atomic
+flat_map bit
+flat_map cctype
+flat_map cerrno
+flat_map climits
+flat_map clocale
+flat_map cmath
+flat_map compare
+flat_map concepts
+flat_map cstdarg
+flat_map cstddef
+flat_map cstdint
+flat_map cstdio
+flat_map cstdlib
+flat_map cstring
+flat_map ctime
+flat_map cwchar
+flat_map cwctype
+flat_map exception
+flat_map initializer_list
+flat_map ios
+flat_map iosfwd
+flat_map iterator
+flat_map limits
+flat_map locale
+flat_map memory
+flat_map mutex
+flat_map new
+flat_map optional
+flat_map ratio
+flat_map stdexcept
+flat_map streambuf
+flat_map string
+flat_map string_view
+flat_map system_error
+flat_map tuple
+flat_map type_traits
+flat_map typeinfo
+flat_map utility
+flat_map variant
+flat_map vector
+flat_map version
 format algorithm
 format array
 format atomic
diff --git a/libcxx/test/libcxx/transitive_includes/cxx14.csv b/libcxx/test/libcxx/transitive_includes/cxx14.csv
index 828e1d62c6ec..27e229755735 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx14.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx14.csv
@@ -700,6 +700,51 @@ filesystem typeinfo
 filesystem utility
 filesystem variant
 filesystem version
+flat_map algorithm
+flat_map array
+flat_map atomic
+flat_map bit
+flat_map cctype
+flat_map cerrno
+flat_map climits
+flat_map clocale
+flat_map cmath
+flat_map compare
+flat_map concepts
+flat_map cstdarg
+flat_map cstddef
+flat_map cstdint
+flat_map cstdio
+flat_map cstdlib
+flat_map cstring
+flat_map ctime
+flat_map cwchar
+flat_map cwctype
+flat_map exception
+flat_map execution
+flat_map initializer_list
+flat_map ios
+flat_map iosfwd
+flat_map iterator
+flat_map limits
+flat_map locale
+flat_map memory
+flat_map mutex
+flat_map new
+flat_map optional
+flat_map ratio
+flat_map stdexcept
+flat_map streambuf
+flat_map string
+flat_map string_view
+flat_map system_error
+flat_map tuple
+flat_map type_traits
+flat_map typeinfo
+flat_map utility
+flat_map variant
+flat_map vector
+flat_map version
 format algorithm
 format array
 format atomic
diff --git a/libcxx/test/libcxx/transitive_includes/cxx17.csv b/libcxx/test/libcxx/transitive_includes/cxx17.csv
index 0bee6e9beb7a..b17eb1f2347a 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx17.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx17.csv
@@ -706,6 +706,50 @@ filesystem utility
 filesystem variant
 filesystem vector
 filesystem version
+flat_map algorithm
+flat_map array
+flat_map atomic
+flat_map bit
+flat_map cctype
+flat_map cerrno
+flat_map climits
+flat_map clocale
+flat_map cmath
+flat_map compare
+flat_map concepts
+flat_map cstdarg
+flat_map cstddef
+flat_map cstdint
+flat_map cstdio
+flat_map cstdlib
+flat_map cstring
+flat_map ctime
+flat_map cwchar
+flat_map cwctype
+flat_map exception
+flat_map initializer_list
+flat_map ios
+flat_map iosfwd
+flat_map iterator
+flat_map limits
+flat_map locale
+flat_map memory
+flat_map mutex
+flat_map new
+flat_map optional
+flat_map ratio
+flat_map stdexcept
+flat_map streambuf
+flat_map string
+flat_map string_view
+flat_map system_error
+flat_map tuple
+flat_map type_traits
+flat_map typeinfo
+flat_map utility
+flat_map variant
+flat_map vector
+flat_map version
 format algorithm
 format array
 format atomic
diff --git a/libcxx/test/libcxx/transitive_includes/cxx20.csv b/libcxx/test/libcxx/transitive_includes/cxx20.csv
index 026c26f3bd98..9efec327889c 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx20.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx20.csv
@@ -694,6 +694,50 @@ filesystem utility
 filesystem variant
 filesystem vector
 filesystem version
+flat_map algorithm
+flat_map array
+flat_map atomic
+flat_map bit
+flat_map cctype
+flat_map cerrno
+flat_map climits
+flat_map clocale
+flat_map cmath
+flat_map compare
+flat_map concepts
+flat_map cstdarg
+flat_map cstddef
+flat_map cstdint
+flat_map cstdio
+flat_map cstdlib
+flat_map cstring
+flat_map ctime
+flat_map cwchar
+flat_map cwctype
+flat_map exception
+flat_map initializer_list
+flat_map ios
+flat_map iosfwd
+flat_map iterator
+flat_map limits
+flat_map locale
+flat_map memory
+flat_map mutex
+flat_map new
+flat_map optional
+flat_map ratio
+flat_map stdexcept
+flat_map streambuf
+flat_map string
+flat_map string_view
+flat_map system_error
+flat_map tuple
+flat_map type_traits
+flat_map typeinfo
+flat_map utility
+flat_map variant
+flat_map vector
+flat_map version
 format algorithm
 format array
 format atomic
diff --git a/libcxx/test/libcxx/transitive_includes/cxx23.csv b/libcxx/test/libcxx/transitive_includes/cxx23.csv
index c3db06f15477..e17f732663a9 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx23.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx23.csv
@@ -381,6 +381,31 @@ filesystem string_view
 filesystem tuple
 filesystem typeinfo
 filesystem version
+flat_map array
+flat_map cctype
+flat_map cerrno
+flat_map climits
+flat_map clocale
+flat_map compare
+flat_map cstddef
+flat_map cstdint
+flat_map cstdio
+flat_map cstdlib
+flat_map cstring
+flat_map cwchar
+flat_map cwctype
+flat_map initializer_list
+flat_map iosfwd
+flat_map limits
+flat_map new
+flat_map optional
+flat_map stdexcept
+flat_map string
+flat_map string_view
+flat_map tuple
+flat_map typeinfo
+flat_map vector
+flat_map version
 format array
 format cctype
 format cerrno
diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv
index 8d7560344ee5..c56f5cdfad00 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx26.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv
@@ -381,6 +381,31 @@ filesystem string_view
 filesystem tuple
 filesystem typeinfo
 filesystem version
+flat_map array
+flat_map cctype
+flat_map cerrno
+flat_map climits
+flat_map clocale
+flat_map compare
+flat_map cstddef
+flat_map cstdint
+flat_map cstdio
+flat_map cstdlib
+flat_map cstring
+flat_map cwchar
+flat_map cwctype
+flat_map initializer_list
+flat_map iosfwd
+flat_map limits
+flat_map new
+flat_map optional
+flat_map stdexcept
+flat_map string
+flat_map string_view
+flat_map tuple
+flat_map typeinfo
+flat_map vector
+flat_map version
 format array
 format cctype
 format cerrno
diff --git a/libcxx/test/std/containers/container.adaptors/NaiveStaticVector.h b/libcxx/test/std/containers/container.adaptors/NaiveStaticVector.h
new file mode 100644
index 000000000000..61fa3504e34e
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/NaiveStaticVector.h
@@ -0,0 +1,94 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SUPPORT_NAIVE_STATIC_VECTOR_H
+#define SUPPORT_NAIVE_STATIC_VECTOR_H
+
+#include <cstddef>
+#include <utility>
+#include "test_iterators.h"
+#include "test_macros.h"
+
+template <class T, std::size_t N>
+struct NaiveStaticVector {
+  struct CapacityError {};
+
+  using value_type      = T;
+  using difference_type = short;
+  using size_type       = unsigned short;
+  using iterator        = random_access_iterator<T*>;
+  using const_iterator  = random_access_iterator<const T*>;
+
+  explicit NaiveStaticVector() = default;
+  template <class It>
+  explicit NaiveStaticVector(It first, It last) {
+    while (first != last)
+      insert(*first++);
+  }
+
+  // Moving-from a NaiveStaticVector leaves the source vector holding moved-from objects.
+  // This is intentional (the "Naive" in the name).
+  // Specifically, moving-out-of a sorted+uniqued NaiveStaticVector<MoveOnly>
+  // will leave it in a non-sorted+uniqued state.
+
+  NaiveStaticVector(const NaiveStaticVector&)            = default;
+  NaiveStaticVector(NaiveStaticVector&&)                 = default; // deliberately don't reset size_
+  NaiveStaticVector& operator=(const NaiveStaticVector&) = default;
+  NaiveStaticVector& operator=(NaiveStaticVector&&)      = default;
+
+  iterator begin() { return iterator(data_); }
+  const_iterator begin() const { return const_iterator(data_); }
+  const_iterator cbegin() const { return const_iterator(data_); }
+  iterator end() { return begin() + size(); }
+  const_iterator end() const { return begin() + size(); }
+  size_type size() const { return size_; }
+  bool empty() const { return size_ == 0; }
+
+  void clear() { size_ = 0; }
+
+  template <class It>
+  iterator insert(const_iterator pos, It first, It last) {
+    iterator result = pos - cbegin() + begin();
+    while (first != last) {
+      insert(pos++, *first++);
+    }
+    return result;
+  }
+
+  iterator insert(const_iterator pos, T value) {
+    if (size_ == N) {
+      throw CapacityError();
+    }
+    int i = pos - cbegin();
+    size_ += 1;
+    std::move_backward(&data_[i], &data_[size_ - 1], &data_[size_]);
+    data_[i] = std::move(value);
+    return begin() + i;
+  }
+
+  template <class... Args>
+  iterator emplace(const_iterator pos, Args&&... args) {
+    return insert(pos, T(std::forward<Args>(args)...));
+  }
+
+  iterator erase(const_iterator first, const_iterator last) {
+    int i = first - cbegin();
+    int j = last - cbegin();
+    std::move(&data_[j], &data_[size_], &data_[i]);
+    size_ -= (last - first);
+    return begin() + i;
+  }
+
+  iterator erase(const_iterator pos) { return erase(pos, std::next(pos)); }
+
+private:
+  T data_[N];
+  std::size_t size_ = 0;
+};
+
+#endif // SUPPORT_NAIVE_STATIC_VECTOR_H
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map.syn/sorted_unique.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map.syn/sorted_unique.pass.cpp
new file mode 100644
index 000000000000..c602d2d3d38f
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map.syn/sorted_unique.pass.cpp
@@ -0,0 +1,44 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// struct sorted_unique_t { explicit sorted_unique_t() = default; };
+// inline constexpr sorted_unique_t sorted_unique{};
+
+#include <cassert>
+#include <concepts>
+#include <flat_map>
+#include <type_traits>
+
+template <class T>
+void implicit_test(T) {}
+
+template <class T>
+concept HasImplicitDefaultCtor = requires { implicit_test<T>({}); };
+
+static_assert(std::is_default_constructible_v<std::sorted_unique_t>);
+static_assert(std::is_trivially_default_constructible_v<std::sorted_unique_t>);
+static_assert(!HasImplicitDefaultCtor<std::sorted_unique_t>);
+
+constexpr bool test() {
+  { [[maybe_unused]] std::sorted_unique_t s; }
+  { [[maybe_unused]] std::same_as<const std::sorted_unique_t&> decltype(auto) s = (std::sorted_unique); }
+  { [[maybe_unused]] std::same_as<const std::sorted_unique_t> decltype(auto) copy = std::sorted_unique; }
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at.pass.cpp
new file mode 100644
index 000000000000..d30055bf1701
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at.pass.cpp
@@ -0,0 +1,92 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+//       mapped_type& at(const key_type& k);
+// const mapped_type& at(const key_type& k) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <stdexcept>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "min_allocator.h"
+#include "test_macros.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using P = std::pair<int, double>;
+  P ar[]  = {
+      P(1, 1.5),
+      P(2, 2.5),
+      P(3, 3.5),
+      P(4, 4.5),
+      P(5, 5.5),
+      P(7, 7.5),
+      P(8, 8.5),
+  };
+  const int one = 1;
+  {
+    std::flat_map<int, double, std::less<int>, KeyContainer, ValueContainer> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+    ASSERT_SAME_TYPE(decltype(m.at(one)), double&);
+    assert(m.size() == 7);
+    assert(m.at(one) == 1.5);
+    m.at(1) = -1.5;
+    assert(m.at(1) == -1.5);
+    assert(m.at(2) == 2.5);
+    assert(m.at(3) == 3.5);
+    assert(m.at(4) == 4.5);
+    assert(m.at(5) == 5.5);
+#ifndef TEST_HAS_NO_EXCEPTIONS
+    try {
+      TEST_IGNORE_NODISCARD m.at(6);
+      assert(false);
+    } catch (std::out_of_range&) {
+    }
+#endif
+    assert(m.at(7) == 7.5);
+    assert(m.at(8) == 8.5);
+    assert(m.size() == 7);
+  }
+  {
+    const std::flat_map<int, double, std::less<int>, KeyContainer, ValueContainer> m(
+        ar, ar + sizeof(ar) / sizeof(ar[0]));
+    ASSERT_SAME_TYPE(decltype(m.at(one)), const double&);
+    assert(m.size() == 7);
+    assert(m.at(one) == 1.5);
+    assert(m.at(2) == 2.5);
+    assert(m.at(3) == 3.5);
+    assert(m.at(4) == 4.5);
+    assert(m.at(5) == 5.5);
+#ifndef TEST_HAS_NO_EXCEPTIONS
+    try {
+      TEST_IGNORE_NODISCARD m.at(6);
+      assert(false);
+    } catch (std::out_of_range&) {
+    }
+#endif
+    assert(m.at(7) == 7.5);
+    assert(m.at(8) == 8.5);
+    assert(m.size() == 7);
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at_transparent.pass.cpp
new file mode 100644
index 000000000000..13edca915fd0
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at_transparent.pass.cpp
@@ -0,0 +1,111 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<class K> mapped_type&       at(const K& x);
+// template<class K> const mapped_type& at(const K& x) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <stdexcept>
+
+#include "../helpers.h"
+#include "min_allocator.h"
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+
+// Constraints: The qualified-id Compare::is_transparent is valid and denotes a type.
+template <class M>
+concept CanAt           = requires(M m, Transparent<int> k) { m.at(k); };
+using TransparentMap    = std::flat_map<int, double, TransparentComparator>;
+using NonTransparentMap = std::flat_map<int, double, NonTransparentComparator>;
+static_assert(CanAt<TransparentMap>);
+static_assert(CanAt<const TransparentMap>);
+static_assert(!CanAt<NonTransparentMap>);
+static_assert(!CanAt<const NonTransparentMap>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using P = std::pair<int, double>;
+  P ar[]  = {
+      P(1, 1.5),
+      P(2, 2.5),
+      P(3, 3.5),
+      P(4, 4.5),
+      P(5, 5.5),
+      P(7, 7.5),
+      P(8, 8.5),
+  };
+  const Transparent<int> one{1};
+  {
+    std::flat_map<int, double, TransparentComparator, KeyContainer, ValueContainer> m(
+        ar, ar + sizeof(ar) / sizeof(ar[0]));
+    ASSERT_SAME_TYPE(decltype(m.at(one)), double&);
+    assert(m.size() == 7);
+    assert(m.at(one) == 1.5);
+    m.at(one) = -1.5;
+    assert(m.at(Transparent<int>{1}) == -1.5);
+    assert(m.at(Transparent<int>{2}) == 2.5);
+    assert(m.at(Transparent<int>{3}) == 3.5);
+    assert(m.at(Transparent<int>{4}) == 4.5);
+    assert(m.at(Transparent<int>{5}) == 5.5);
+#ifndef TEST_HAS_NO_EXCEPTIONS
+    try {
+      TEST_IGNORE_NODISCARD m.at(Transparent<int>{6});
+      assert(false);
+    } catch (std::out_of_range&) {
+    }
+#endif
+    assert(m.at(Transparent<int>{7}) == 7.5);
+    assert(m.at(Transparent<int>{8}) == 8.5);
+    assert(m.size() == 7);
+  }
+  {
+    const std::flat_map<int, double, TransparentComparator, KeyContainer, ValueContainer> m(
+        ar, ar + sizeof(ar) / sizeof(ar[0]));
+    ASSERT_SAME_TYPE(decltype(m.at(one)), const double&);
+    assert(m.size() == 7);
+    assert(m.at(Transparent<int>{1}) == 1.5);
+    assert(m.at(Transparent<int>{2}) == 2.5);
+    assert(m.at(Transparent<int>{3}) == 3.5);
+    assert(m.at(Transparent<int>{4}) == 4.5);
+    assert(m.at(Transparent<int>{5}) == 5.5);
+#ifndef TEST_HAS_NO_EXCEPTIONS
+    try {
+      TEST_IGNORE_NODISCARD m.at(Transparent<int>{6});
+      assert(false);
+    } catch (std::out_of_range&) {
+    }
+#endif
+    assert(m.at(Transparent<int>{7}) == 7.5);
+    assert(m.at(Transparent<int>{8}) == 8.5);
+    assert(m.size() == 7);
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_map<int, int, TransparentComparator> m(std::sorted_unique, {{1, 1}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    m.at(Transparent<int>{3});
+    assert(transparent_used);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_key.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_key.pass.cpp
new file mode 100644
index 000000000000..ea2f5d800878
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_key.pass.cpp
@@ -0,0 +1,77 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// mapped_type& operator[](const key_type& k);
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "min_allocator.h"
+#include "test_macros.h"
+
+// Constraints: is_constructible_v<mapped_type> is true.
+template <class M, class Input>
+concept CanIndex = requires(M m, Input k) { m[k]; };
+
+static_assert(CanIndex<std::flat_map<int, double>, const int&>);
+static_assert(!CanIndex<std::flat_map<int, NoDefaultCtr>, const int&>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using P = std::pair<int, double>;
+  P ar[]  = {
+      P(1, 1.5),
+      P(2, 2.5),
+      P(3, 3.5),
+      P(4, 4.5),
+      P(5, 5.5),
+      P(7, 7.5),
+      P(8, 8.5),
+  };
+  const int one = 1;
+  std::flat_map<int, double, std::less<int>, KeyContainer, ValueContainer> m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+  ASSERT_SAME_TYPE(decltype(m[one]), double&);
+  assert(m.size() == 7);
+  assert(m[one] == 1.5);
+  assert(m.size() == 7);
+  m[1] = -1.5;
+  assert(m[1] == -1.5);
+  assert(m.size() == 7);
+  assert(m[6] == 0);
+  assert(m.size() == 8);
+  m[6] = 6.5;
+  assert(m[6] == 6.5);
+  assert(m.size() == 8);
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto index_func = [](auto& m, auto key_arg, auto value_arg) {
+      using FlatMap                             = std::decay_t<decltype(m)>;
+      const typename FlatMap::key_type key      = key_arg;
+      const typename FlatMap::mapped_type value = value_arg;
+      m[key]                                    = value;
+    };
+    test_emplace_exception_guarantee(index_func);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_rv_key.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_rv_key.pass.cpp
new file mode 100644
index 000000000000..faacc3cfe8f9
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_rv_key.pass.cpp
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// mapped_type& operator[](key_type&& k);
+
+#include <flat_map>
+#include <deque>
+#include <functional>
+#include <cassert>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "min_allocator.h"
+
+// Constraints: is_constructible_v<mapped_type> is true.
+template <class M, class Input>
+concept CanIndex = requires(M m, Input k) { m[k]; };
+
+static_assert(CanIndex<std::flat_map<int, double>, int&&>);
+static_assert(!CanIndex<std::flat_map<int, NoDefaultCtr>, int&&>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  {
+    std::flat_map<MoveOnly, double, std::less<MoveOnly>, KeyContainer, ValueContainer> m;
+    ASSERT_SAME_TYPE(decltype(m[MoveOnly{}]), double&);
+    assert(m.size() == 0);
+    assert(m[1] == 0.0);
+    assert(m.size() == 1);
+    m[1] = -1.5;
+    assert(m[1] == -1.5);
+    assert(m.size() == 1);
+    assert(m[6] == 0);
+    assert(m.size() == 2);
+    m[6] = 6.5;
+    assert(m[6] == 6.5);
+    assert(m.size() == 2);
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<MoveOnly>, std::vector<double>>();
+  test<std::deque<MoveOnly>, std::vector<double>>();
+  test<MinSequenceContainer<MoveOnly>, MinSequenceContainer<double>>();
+  test<std::vector<MoveOnly, min_allocator<MoveOnly>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto index_func = [](auto& m, auto key_arg, auto value_arg) {
+      using FlatMap                             = std::decay_t<decltype(m)>;
+      typename FlatMap::key_type key            = key_arg;
+      const typename FlatMap::mapped_type value = value_arg;
+      m[std::move(key)]                         = value;
+    };
+    test_emplace_exception_guarantee(index_func);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_transparent.pass.cpp
new file mode 100644
index 000000000000..24c08464f315
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_transparent.pass.cpp
@@ -0,0 +1,107 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<class K> mapped_type& operator[](K&& x);
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// Constraints:
+// The qualified-id Compare::is_transparent is valid and denotes a type.
+// is_constructible_v<key_type, K> is true.
+// is_constructible_v<mapped_type, Args...> is true.
+// is_convertible_v<K&&, const_iterator> and is_convertible_v<K&&, iterator> are both false
+template <class M, class Input>
+concept CanIndex                      = requires(M m, Input k) { m[k]; };
+using TransparentMap                  = std::flat_map<int, double, TransparentComparator>;
+using NonTransparentMap               = std::flat_map<int, double, NonTransparentComparator>;
+using TransparentNoDefaultCtrValueMap = std::flat_map<int, NoDefaultCtr, TransparentComparator>;
+
+static_assert(CanIndex<TransparentMap, ConvertibleTransparent<int>>);
+static_assert(!CanIndex<const TransparentMap, ConvertibleTransparent<int>>);
+
+static_assert(!CanIndex<NonTransparentMap, NonConvertibleTransparent<int>>);
+static_assert(!CanIndex<const NonTransparentMap, NonConvertibleTransparent<int>>);
+
+static_assert(!CanIndex<TransparentMap, NonConvertibleTransparent<int>>);
+static_assert(!CanIndex<const TransparentMap, NonConvertibleTransparent<int>>);
+
+static_assert(!CanIndex<TransparentNoDefaultCtrValueMap, ConvertibleTransparent<int>>);
+static_assert(!CanIndex<const TransparentNoDefaultCtrValueMap, ConvertibleTransparent<int>>);
+
+static_assert(!CanIndex<TransparentMap, TransparentMap::iterator>);
+static_assert(!CanIndex<TransparentMap, TransparentMap::const_iterator>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using P = std::pair<int, double>;
+  P ar[]  = {
+      P(1, 1.5),
+      P(2, 2.5),
+      P(3, 3.5),
+      P(4, 4.5),
+      P(5, 5.5),
+      P(7, 7.5),
+      P(8, 8.5),
+  };
+  const ConvertibleTransparent<int> one{1};
+  const ConvertibleTransparent<int> six{6};
+  {
+    std::flat_map<int, double, TransparentComparator, KeyContainer, ValueContainer> m(
+        ar, ar + sizeof(ar) / sizeof(ar[0]));
+    ASSERT_SAME_TYPE(decltype(m[one]), double&);
+    assert(m.size() == 7);
+    assert(m[one] == 1.5);
+    assert(m.size() == 7);
+    m[one] = -1.5;
+    assert(m[one] == -1.5);
+    assert(m.size() == 7);
+    assert(m[six] == 0);
+    assert(m.size() == 8);
+    m[six] = 6.5;
+    assert(m[six] == 6.5);
+    assert(m.size() == 8);
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_map<int, int, TransparentComparator> m(std::sorted_unique, {{1, 1}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    m[ConvertibleTransparent<int>{3}];
+    assert(transparent_used);
+  }
+  {
+    auto index_func = [](auto& m, auto key_arg, auto value_arg) {
+      using FlatMap                             = std::decay_t<decltype(m)>;
+      using Key                                 = typename FlatMap::key_type;
+      const typename FlatMap::mapped_type value = value_arg;
+      m[ConvertibleTransparent<Key>{key_arg}]   = value;
+    };
+    test_emplace_exception_guarantee(index_func);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.pass.cpp
new file mode 100644
index 000000000000..5ecc2cf7c917
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.pass.cpp
@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// [[nodiscard]] bool empty() const noexcept;
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using M = std::flat_map<int, double, std::less<int>, KeyContainer, ValueContainer>;
+  M m;
+  ASSERT_SAME_TYPE(decltype(m.empty()), bool);
+  ASSERT_NOEXCEPT(m.empty());
+  assert(m.empty());
+  assert(std::as_const(m).empty());
+  m = {{1, 1.0}};
+  assert(!m.empty());
+  m.clear();
+  assert(m.empty());
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.verify.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.verify.cpp
new file mode 100644
index 000000000000..cc8016182dcb
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/empty.verify.cpp
@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// [[nodiscard]] bool empty() const noexcept;
+
+#include <flat_map>
+
+#include "test_macros.h"
+
+int main(int, char**) {
+  std::flat_map<int, int> c;
+  c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/max_size.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/max_size.pass.cpp
new file mode 100644
index 000000000000..87acdfd2cf62
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/max_size.pass.cpp
@@ -0,0 +1,76 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// size_type max_size() const noexcept;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <limits>
+#include <type_traits>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "test_allocator.h"
+#include "test_macros.h"
+
+int main(int, char**) {
+  {
+    using A1 = limited_allocator<int, 10>;
+    using A2 = limited_allocator<int, 20>;
+    using C  = std::flat_map<int, int, std::less<int>, std::vector<int, A1>, std::vector<int, A2>>;
+    ASSERT_SAME_TYPE(C::difference_type, std::ptrdiff_t);
+    ASSERT_SAME_TYPE(C::size_type, std::size_t);
+    const C c;
+    ASSERT_NOEXCEPT(c.max_size());
+    ASSERT_SAME_TYPE(decltype(c.max_size()), C::size_type);
+    assert(c.max_size() <= 10);
+    LIBCPP_ASSERT(c.max_size() == 10);
+  }
+  {
+    using A1 = limited_allocator<int, 10>;
+    using A2 = limited_allocator<int, 20>;
+    using C  = std::flat_map<int, int, std::less<int>, std::vector<int, A2>, std::vector<int, A1>>;
+    ASSERT_SAME_TYPE(C::difference_type, std::ptrdiff_t);
+    ASSERT_SAME_TYPE(C::size_type, std::size_t);
+    const C c;
+    ASSERT_NOEXCEPT(c.max_size());
+    ASSERT_SAME_TYPE(decltype(c.max_size()), C::size_type);
+    assert(c.max_size() <= 10);
+    LIBCPP_ASSERT(c.max_size() == 10);
+  }
+  {
+    using A = limited_allocator<int, (size_t)-1>;
+    using C = std::flat_map<int, int, std::less<int>, std::vector<int, A>, std::vector<int, A>>;
+    ASSERT_SAME_TYPE(C::difference_type, std::ptrdiff_t);
+    ASSERT_SAME_TYPE(C::size_type, std::size_t);
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    const C c;
+    ASSERT_NOEXCEPT(c.max_size());
+    ASSERT_SAME_TYPE(decltype(c.max_size()), C::size_type);
+    assert(c.max_size() <= max_dist);
+    LIBCPP_ASSERT(c.max_size() == max_dist);
+  }
+  {
+    typedef std::flat_map<char, char> C;
+    ASSERT_SAME_TYPE(C::difference_type, std::ptrdiff_t);
+    ASSERT_SAME_TYPE(C::size_type, std::size_t);
+    const C::size_type max_dist = static_cast<C::size_type>(std::numeric_limits<C::difference_type>::max());
+    const C c;
+    ASSERT_NOEXCEPT(c.max_size());
+    ASSERT_SAME_TYPE(decltype(c.max_size()), C::size_type);
+    assert(c.max_size() <= max_dist);
+    assert(c.max_size() <= alloc_max_size(std::allocator<char>()));
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/size.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/size.pass.cpp
new file mode 100644
index 000000000000..957a86045009
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.capacity/size.pass.cpp
@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// size_type size() const noexcept;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using M = std::flat_map<int, char, std::less<int>, KeyContainer, ValueContainer>;
+  {
+    const M m = {{1, 'a'}, {1, 'b'}, {4, 'd'}, {5, 'e'}, {5, 'h'}};
+    ASSERT_SAME_TYPE(decltype(m.size()), std::size_t);
+    ASSERT_NOEXCEPT(m.size());
+    assert(m.size() == 3);
+  }
+  {
+    const M m = {{1, 'a'}};
+    ASSERT_SAME_TYPE(decltype(m.size()), std::size_t);
+    ASSERT_NOEXCEPT(m.size());
+    assert(m.size() == 1);
+  }
+  {
+    const M m;
+    ASSERT_SAME_TYPE(decltype(m.size()), std::size_t);
+    ASSERT_NOEXCEPT(m.size());
+    assert(m.size() == 0);
+  }
+  {
+    M m;
+    std::size_t s = 1000000;
+    for (auto i = 0u; i < s; ++i) {
+      m.emplace(i, 'a');
+    }
+    ASSERT_SAME_TYPE(decltype(m.size()), std::size_t);
+    ASSERT_NOEXCEPT(m.size());
+    assert(m.size() == s);
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/alloc.pass.cpp
new file mode 100644
index 000000000000..3f8d2ed332d6
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/alloc.pass.cpp
@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<class Allocator>
+//   explicit flat_map(const Allocator& a);
+
+#include <cassert>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_allocator.h"
+#include "../../../test_compare.h"
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_map<int, int, C, V1, V1>;
+    using M2 = std::flat_map<int, int, C, V1, V2>;
+    using M3 = std::flat_map<int, int, C, V2, V1>;
+    static_assert(std::is_constructible_v<M1, const A1&>);
+    static_assert(!std::is_constructible_v<M1, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const A2&>);
+    static_assert(!std::is_constructible_v<M3, const A2&>);
+  }
+  {
+    // explicit
+    using M =
+        std::flat_map<int,
+                      long,
+                      std::less<int>,
+                      std::vector<int, test_allocator<int>>,
+                      std::vector<long, test_allocator<long>>>;
+
+    static_assert(std::is_constructible_v<M, test_allocator<int>>);
+    static_assert(!std::is_convertible_v<test_allocator<int>, M>);
+  }
+  {
+    using A = test_allocator<short>;
+    using M =
+        std::flat_map<int,
+                      long,
+                      std::less<int>,
+                      std::vector<int, test_allocator<int>>,
+                      std::vector<long, test_allocator<long>>>;
+    M m(A(0, 5));
+    assert(m.empty());
+    assert(m.begin() == m.end());
+    assert(m.keys().get_allocator().get_id() == 5);
+    assert(m.values().get_allocator().get_id() == 5);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/assign_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/assign_initializer_list.pass.cpp
new file mode 100644
index 000000000000..06bde71e7994
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/assign_initializer_list.pass.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map& operator=(initializer_list<value_type> il);
+
+#include <algorithm>
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <ranges>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+#include "test_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  {
+    M m = {{8, 8}, {10, 10}};
+    assert(m.size() == 2);
+    m                              = {{3, 0}, {1, 0}, {2, 0}, {2, 1}, {3, 1}, {4, 0}, {3, 2}, {5, 0}, {6, 0}, {5, 1}};
+    std::pair<int, int> expected[] = {{1, 0}, {2, 0}, {3, 0}, {4, 0}, {5, 0}, {6, 0}};
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+  }
+  {
+    M m = {{10, 1}, {8, 1}};
+    assert(m.size() == 2);
+    m                                    = {{3, 2}};
+    std::pair<double, double> expected[] = {{3, 2}};
+    assert(std::ranges::equal(m, expected));
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<int>>();
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/compare.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/compare.pass.cpp
new file mode 100644
index 000000000000..40a1710f55e4
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/compare.pass.cpp
@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// explicit flat_map(const key_compare& comp);
+// template <class Alloc>
+//   flat_map(const key_compare& comp, const Alloc& a);
+
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <type_traits>
+#include <vector>
+
+#include "test_macros.h"
+#include "../../../test_compare.h"
+#include "test_allocator.h"
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using M1 = std::flat_map<int, int, C, std::vector<int, A1>, std::vector<int, A1>>;
+    using M2 = std::flat_map<int, int, C, std::vector<int, A1>, std::vector<int, A2>>;
+    using M3 = std::flat_map<int, int, C, std::vector<int, A2>, std::vector<int, A1>>;
+    static_assert(std::is_constructible_v<M1, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, const C&, const A2&>);
+  }
+  {
+    using C = test_less<int>;
+    auto m  = std::flat_map<int, char*, C>(C(3));
+    assert(m.empty());
+    assert(m.begin() == m.end());
+    assert(m.key_comp() == C(3));
+  }
+  {
+    // The one-argument ctor is explicit.
+    using C = test_less<int>;
+    static_assert(std::is_constructible_v<std::flat_map<int, char*, C>, C>);
+    static_assert(!std::is_convertible_v<C, std::flat_map<int, char*, C>>);
+
+    static_assert(std::is_constructible_v<std::flat_map<int, char*>, std::less<int>>);
+    static_assert(!std::is_convertible_v<std::less<int>, std::flat_map<int, char*>>);
+  }
+  {
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    auto m   = std::flat_map<int, short, C, std::vector<int, A1>, std::vector<short, A2>>(C(4), A1(5));
+    assert(m.empty());
+    assert(m.begin() == m.end());
+    assert(m.key_comp() == C(4));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // explicit(false)
+    using C                                                                    = test_less<int>;
+    using A1                                                                   = test_allocator<int>;
+    using A2                                                                   = test_allocator<short>;
+    std::flat_map<int, short, C, std::deque<int, A1>, std::deque<short, A2>> m = {C(4), A1(5)};
+    assert(m.empty());
+    assert(m.begin() == m.end());
+    assert(m.key_comp() == C(4));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // If an allocator is given, it must be usable by both containers.
+    using A = test_allocator<int>;
+    using M = std::flat_map<int, int, std::less<>, std::vector<int>, std::vector<int, A>>;
+    static_assert(std::is_constructible_v<M, std::less<>>);
+    static_assert(!std::is_constructible_v<M, std::less<>, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::less<>, A>);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/containers.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/containers.pass.cpp
new file mode 100644
index 000000000000..812e2c3e4f02
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/containers.pass.cpp
@@ -0,0 +1,184 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map(key_container_type key_cont, mapped_container_type mapped_cont,
+//           const key_compare& comp = key_compare());
+// template<class Allocator>
+//   flat_map(const key_container_type& key_cont, const mapped_container_type& mapped_cont,
+//            const Allocator& a);
+// template<class Alloc>
+//   flat_map(const key_container_type& key_cont, const mapped_container_type& mapped_cont,
+//            const key_compare& comp, const Alloc& a);
+
+#include <algorithm>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "min_allocator.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+
+struct P {
+  int first;
+  int second;
+  template <class T, class U>
+  bool operator==(const std::pair<T, U>& rhs) const {
+    return MoveOnly(first) == rhs.first && MoveOnly(second) == rhs.second;
+  }
+};
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_map<int, int, C, V1, V1>;
+    using M2 = std::flat_map<int, int, C, V1, V2>;
+    using M3 = std::flat_map<int, int, C, V2, V1>;
+    static_assert(std::is_constructible_v<M1, const V1&, const V1&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, const V1&, const V1&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const V1&, const V2&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, const V2&, const V1&, const A2&>);
+
+    static_assert(std::is_constructible_v<M1, const V1&, const V1&, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, const V1&, const V1&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const V1&, const V2&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, const V2&, const V1&, const C&, const A2&>);
+  }
+  {
+    // flat_map(key_container_type , mapped_container_type)
+    using M              = std::flat_map<int, char>;
+    std::vector<int> ks  = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    std::vector<char> vs = {1, 2, 3, 4, 5, 6, 7, 8, 9};
+    auto m               = M(ks, vs);
+    assert((m.keys() == std::vector<int>{1, 2, 3}));
+    LIBCPP_ASSERT((m.values() == std::vector<char>{1, 4, 6}));
+
+    // explicit(false)
+    M m2 = {ks, vs};
+    assert(m2 == m);
+
+    m = M(std::move(ks), std::move(vs));
+    assert(ks.empty()); // it was moved-from
+    assert(vs.empty()); // it was moved-from
+    assert((m.keys() == std::vector<int>{1, 2, 3}));
+    LIBCPP_ASSERT((m.values() == std::vector<char>{1, 4, 6}));
+  }
+  {
+    // flat_map(key_container_type , mapped_container_type)
+    // move-only
+    P expected[] = {{3, 2}, {2, 1}, {1, 3}};
+    using Ks     = std::deque<int, min_allocator<int>>;
+    using Vs     = std::vector<MoveOnly, min_allocator<MoveOnly>>;
+    using M      = std::flat_map<int, MoveOnly, std::greater<int>, Ks, Vs>;
+    Ks ks        = {1, 3, 2};
+    Vs vs;
+    vs.push_back(3);
+    vs.push_back(2);
+    vs.push_back(1);
+    auto m = M(std::move(ks), std::move(vs));
+    assert(ks.empty()); // it was moved-from
+    assert(vs.empty()); // it was moved-from
+    assert(std::ranges::equal(m, expected, std::equal_to<>()));
+  }
+  {
+    // flat_map(key_container_type , mapped_container_type)
+    // container's allocators are used
+    using A = test_allocator<int>;
+    using M = std::flat_map<int, int, std::less<int>, std::vector<int, A>, std::deque<int, A>>;
+    auto ks = std::vector<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(5));
+    auto vs = std::deque<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(6));
+    auto m  = M(std::move(ks), std::move(vs));
+    assert(ks.empty()); // it was moved-from
+    assert(vs.empty()); // it was moved-from
+    assert((m == M{{1, 1}, {2, 2}, {3, 3}}));
+    assert(m.keys().get_allocator() == A(5));
+    assert(m.values().get_allocator() == A(6));
+  }
+  {
+    // flat_map(key_container_type , mapped_container_type, key_compare)
+    using C              = test_less<int>;
+    using M              = std::flat_map<int, char, C>;
+    std::vector<int> ks  = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    std::vector<char> vs = {1, 2, 3, 4, 5, 6, 7, 8, 9};
+    auto m               = M(ks, vs, C(4));
+    assert((m.keys() == std::vector<int>{1, 2, 3}));
+    LIBCPP_ASSERT((m.values() == std::vector<char>{1, 4, 6}));
+    assert(m.key_comp() == C(4));
+
+    // explicit(false)
+    M m2 = {ks, vs, C(4)};
+    assert(m2 == m);
+    assert(m2.key_comp() == C(4));
+  }
+  {
+    // flat_map(key_container_type , mapped_container_type, const Allocator&)
+    using A = test_allocator<int>;
+    using M = std::flat_map<int, int, std::less<int>, std::vector<int, A>, std::deque<int, A>>;
+    auto ks = std::vector<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(5));
+    auto vs = std::deque<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(6));
+    auto m  = M(ks, vs, A(4)); // replaces the allocators
+    assert(!ks.empty());       // it was an lvalue above
+    assert(!vs.empty());       // it was an lvalue above
+    assert((m == M{{1, 1}, {2, 2}, {3, 3}}));
+    assert(m.keys().get_allocator() == A(4));
+    assert(m.values().get_allocator() == A(4));
+  }
+  {
+    // flat_map(key_container_type , mapped_container_type, const Allocator&)
+    // explicit(false)
+    using A = test_allocator<int>;
+    using M = std::flat_map<int, int, std::less<int>, std::vector<int, A>, std::deque<int, A>>;
+    auto ks = std::vector<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(5));
+    auto vs = std::deque<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(6));
+    M m     = {ks, vs, A(4)}; // implicit ctor
+    assert(!ks.empty());      // it was an lvalue above
+    assert(!vs.empty());      // it was an lvalue above
+    assert((m == M{{1, 1}, {2, 2}, {3, 3}}));
+    assert(m.keys().get_allocator() == A(4));
+    assert(m.values().get_allocator() == A(4));
+  }
+  {
+    // flat_map(key_container_type , mapped_container_type, key_compare, const Allocator&)
+    using C                = test_less<int>;
+    using A                = test_allocator<int>;
+    using M                = std::flat_map<int, int, C, std::vector<int, A>, std::vector<int, A>>;
+    std::vector<int, A> ks = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    std::vector<int, A> vs = {1, 2, 3, 4, 5, 6, 7, 8, 9};
+    auto m                 = M(ks, vs, C(4), A(5));
+    assert((m.keys() == std::vector<int, A>{1, 2, 3}));
+    LIBCPP_ASSERT((m.values() == std::vector<int, A>{1, 4, 6}));
+    assert(m.key_comp() == C(4));
+    assert(m.keys().get_allocator() == A(5));
+    assert(m.values().get_allocator() == A(5));
+
+    // explicit(false)
+    M m2 = {ks, vs, C(4), A(5)};
+    assert(m2 == m);
+    assert(m2.key_comp() == C(4));
+    assert(m2.keys().get_allocator() == A(5));
+    assert(m2.values().get_allocator() == A(5));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy.pass.cpp
new file mode 100644
index 000000000000..fcd0415088c1
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy.pass.cpp
@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map(const flat_map& m);
+
+#include <cassert>
+#include <flat_map>
+#include <vector>
+
+#include "test_macros.h"
+#include "../../../test_compare.h"
+#include "test_allocator.h"
+
+int main(int, char**) {
+  {
+    using C = test_less<int>;
+    std::vector<int, test_allocator<int>> ks({1, 3, 5}, test_allocator<int>(6));
+    std::vector<char, test_allocator<char>> vs({2, 2, 1}, test_allocator<char>(7));
+    using M = std::flat_map<int, char, C, decltype(ks), decltype(vs)>;
+    auto mo = M(ks, vs, C(5));
+    auto m  = mo;
+
+    assert(m.key_comp() == C(5));
+    assert(m.keys() == ks);
+    assert(m.values() == vs);
+    assert(m.keys().get_allocator() == test_allocator<int>(6));
+    assert(m.values().get_allocator() == test_allocator<char>(7));
+
+    // mo is unchanged
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys() == ks);
+    assert(mo.values() == vs);
+    assert(mo.keys().get_allocator() == test_allocator<int>(6));
+    assert(mo.values().get_allocator() == test_allocator<char>(7));
+  }
+  {
+    using C  = test_less<int>;
+    using Ks = std::vector<int, other_allocator<int>>;
+    using Vs = std::vector<char, other_allocator<char>>;
+    auto ks  = Ks({1, 3, 5}, other_allocator<int>(6));
+    auto vs  = Vs({2, 2, 1}, other_allocator<char>(7));
+    using M  = std::flat_map<int, char, C, Ks, Vs>;
+    auto mo  = M(Ks(ks, other_allocator<int>(6)), Vs(vs, other_allocator<int>(7)), C(5));
+    auto m   = mo;
+
+    assert(m.key_comp() == C(5));
+    assert(m.keys() == ks);
+    assert(m.values() == vs);
+    assert(m.keys().get_allocator() == other_allocator<int>(-2));
+    assert(m.values().get_allocator() == other_allocator<char>(-2));
+
+    // mo is unchanged
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys() == ks);
+    assert(mo.values() == vs);
+    assert(mo.keys().get_allocator() == other_allocator<int>(6));
+    assert(mo.values().get_allocator() == other_allocator<char>(7));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy_alloc.pass.cpp
new file mode 100644
index 000000000000..cbda6ea85326
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy_alloc.pass.cpp
@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map(const flat_map&, const allocator_type&);
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "test_macros.h"
+#include "../../../test_compare.h"
+#include "test_allocator.h"
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_map<int, int, C, V1, V1>;
+    using M2 = std::flat_map<int, int, C, V1, V2>;
+    using M3 = std::flat_map<int, int, C, V2, V1>;
+    static_assert(std::is_constructible_v<M1, const M1&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, const M1&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const M2&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, const M3&, const A2&>);
+  }
+  {
+    using C = test_less<int>;
+    std::vector<int, test_allocator<int>> ks({1, 3, 5}, test_allocator<int>(6));
+    std::vector<char, test_allocator<char>> vs({2, 2, 1}, test_allocator<char>(7));
+    using M = std::flat_map<int, char, C, decltype(ks), decltype(vs)>;
+    auto mo = M(ks, vs, C(5));
+    auto m  = M(mo, test_allocator<int>(3));
+
+    assert(m.key_comp() == C(5));
+    assert(m.keys() == ks);
+    assert(m.values() == vs);
+    assert(m.keys().get_allocator() == test_allocator<int>(3));
+    assert(m.values().get_allocator() == test_allocator<char>(3));
+
+    // mo is unchanged
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys() == ks);
+    assert(mo.values() == vs);
+    assert(mo.keys().get_allocator() == test_allocator<int>(6));
+    assert(mo.values().get_allocator() == test_allocator<char>(7));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy_assign.addressof.compile.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy_assign.addressof.compile.pass.cpp
new file mode 100644
index 000000000000..e9b752d5eb12
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy_assign.addressof.compile.pass.cpp
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map& operator=(const flat_map& s);
+
+// Validate whether the container can be copy-assigned (move-assigned, swapped)
+// with an ADL-hijacking operator&
+
+#include <flat_map>
+#include <utility>
+
+#include "test_macros.h"
+#include "operator_hijacker.h"
+
+void test() {
+  std::flat_map<operator_hijacker, operator_hijacker> so;
+  std::flat_map<operator_hijacker, operator_hijacker> s;
+  s = so;
+  s = std::move(so);
+  swap(s, so);
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy_assign.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy_assign.pass.cpp
new file mode 100644
index 000000000000..4f9797d5bf81
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/copy_assign.pass.cpp
@@ -0,0 +1,92 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map& operator=(const flat_map& m);
+
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "test_macros.h"
+#include "../../../test_compare.h"
+#include "test_allocator.h"
+
+int main(int, char**) {
+  {
+    // test_allocator is not propagated
+    using C = test_less<int>;
+    std::vector<int, test_allocator<int>> ks({1, 3, 5}, test_allocator<int>(6));
+    std::vector<char, test_allocator<char>> vs({2, 2, 1}, test_allocator<char>(7));
+    using M = std::flat_map<int, char, C, decltype(ks), decltype(vs)>;
+    auto mo = M(ks, vs, C(5));
+    auto m  = M({{3, 3}, {4, 4}, {5, 5}}, C(3), test_allocator<int>(2));
+    m       = mo;
+
+    assert(m.key_comp() == C(5));
+    assert(m.keys() == ks);
+    assert(m.values() == vs);
+    assert(m.keys().get_allocator() == test_allocator<int>(2));
+    assert(m.values().get_allocator() == test_allocator<char>(2));
+
+    // mo is unchanged
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys() == ks);
+    assert(mo.values() == vs);
+    assert(mo.keys().get_allocator() == test_allocator<int>(6));
+    assert(mo.values().get_allocator() == test_allocator<char>(7));
+  }
+  {
+    // other_allocator is propagated
+    using C  = test_less<int>;
+    using Ks = std::vector<int, other_allocator<int>>;
+    using Vs = std::vector<char, other_allocator<char>>;
+    auto ks  = Ks({1, 3, 5}, other_allocator<int>(6));
+    auto vs  = Vs({2, 2, 1}, other_allocator<char>(7));
+    using M  = std::flat_map<int, char, C, Ks, Vs>;
+    auto mo  = M(Ks(ks, other_allocator<int>(6)), Vs(vs, other_allocator<int>(7)), C(5));
+    auto m   = M({{3, 3}, {4, 4}, {5, 5}}, C(3), other_allocator<int>(2));
+    m        = mo;
+
+    assert(m.key_comp() == C(5));
+    assert(m.keys() == ks);
+    assert(m.values() == vs);
+    assert(m.keys().get_allocator() == other_allocator<int>(6));
+    assert(m.values().get_allocator() == other_allocator<char>(7));
+
+    // mo is unchanged
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys() == ks);
+    assert(mo.values() == vs);
+    assert(mo.keys().get_allocator() == other_allocator<int>(6));
+    assert(mo.values().get_allocator() == other_allocator<char>(7));
+  }
+  {
+    // comparator is copied and invariant is preserved
+    using M = std::flat_map<int, int, std::function<bool(int, int)>>;
+    M mo    = M({{1, 2}, {3, 4}}, std::less<int>());
+    M m     = M({{1, 2}, {3, 4}}, std::greater<int>());
+    assert(m.key_comp()(2, 1) == true);
+    assert(m != mo);
+    m = mo;
+    assert(m.key_comp()(2, 1) == false);
+    assert(m == mo);
+  }
+  {
+    // self-assignment
+    using M = std::flat_map<int, int>;
+    M m     = {{1, 2}, {3, 4}};
+    m       = static_cast<const M&>(m);
+    assert((m == M{{1, 2}, {3, 4}}));
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.pass.cpp
new file mode 100644
index 000000000000..d01bee9aae9c
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.pass.cpp
@@ -0,0 +1,342 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+#include <algorithm>
+#include <cassert>
+#include <climits>
+#include <deque>
+#include <initializer_list>
+#include <list>
+#include <flat_map>
+#include <functional>
+#include <ranges>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "deduction_guides_sfinae_checks.h"
+#include "test_allocator.h"
+
+using P  = std::pair<int, long>;
+using PC = std::pair<const int, long>;
+
+void test_copy() {
+  {
+    std::flat_map<long, short> source = {{1, 2}, {2, 3}};
+    std::flat_map s(source);
+    ASSERT_SAME_TYPE(decltype(s), decltype(source));
+    assert(s == source);
+  }
+  {
+    std::flat_map<long, short, std::greater<long>> source = {{1, 2}, {2, 3}};
+    std::flat_map s{source}; // braces instead of parens
+    ASSERT_SAME_TYPE(decltype(s), decltype(source));
+    assert(s == source);
+  }
+  {
+    std::flat_map<long, short, std::greater<long>> source = {{1, 2}, {2, 3}};
+    std::flat_map s(source, std::allocator<int>());
+    ASSERT_SAME_TYPE(decltype(s), decltype(source));
+    assert(s == source);
+  }
+}
+
+void test_containers() {
+  std::deque<int, test_allocator<int>> ks({1, 2, 1, INT_MAX, 3}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> vs({1, 2, 1, 4, 5}, test_allocator<int>(0, 43));
+  std::deque<int, test_allocator<int>> sorted_ks({1, 2, 3, INT_MAX}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> sorted_vs({1, 2, 5, 4}, test_allocator<int>(0, 43));
+  const std::pair<int, short> expected[] = {{1, 1}, {2, 2}, {3, 5}, {INT_MAX, 4}};
+  {
+    std::flat_map s(ks, vs);
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, short, std::less<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 42);
+    assert(s.values().get_allocator().get_id() == 43);
+  }
+  {
+    std::flat_map s(std::sorted_unique, sorted_ks, sorted_vs);
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, short, std::less<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 42);
+    assert(s.values().get_allocator().get_id() == 43);
+  }
+  {
+    std::flat_map s(ks, vs, test_allocator<long>(0, 44));
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, short, std::less<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 44);
+    assert(s.values().get_allocator().get_id() == 44);
+  }
+  {
+    std::flat_map s(std::sorted_unique, sorted_ks, sorted_vs, test_allocator<long>(0, 44));
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, short, std::less<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 44);
+    assert(s.values().get_allocator().get_id() == 44);
+  }
+}
+
+void test_containers_compare() {
+  std::deque<int, test_allocator<int>> ks({1, 2, 1, INT_MAX, 3}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> vs({1, 2, 1, 4, 5}, test_allocator<int>(0, 43));
+  std::deque<int, test_allocator<int>> sorted_ks({INT_MAX, 3, 2, 1}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> sorted_vs({4, 5, 2, 1}, test_allocator<int>(0, 43));
+  const std::pair<int, short> expected[] = {{INT_MAX, 4}, {3, 5}, {2, 2}, {1, 1}};
+  {
+    std::flat_map s(ks, vs, std::greater<int>());
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, short, std::greater<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 42);
+    assert(s.values().get_allocator().get_id() == 43);
+  }
+  {
+    std::flat_map s(std::sorted_unique, sorted_ks, sorted_vs, std::greater<int>());
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, short, std::greater<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 42);
+    assert(s.values().get_allocator().get_id() == 43);
+  }
+  {
+    std::flat_map s(ks, vs, std::greater<int>(), test_allocator<long>(0, 44));
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, short, std::greater<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 44);
+    assert(s.values().get_allocator().get_id() == 44);
+  }
+  {
+    std::flat_map s(std::sorted_unique, sorted_ks, sorted_vs, std::greater<int>(), test_allocator<long>(0, 44));
+
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, short, std::greater<int>, decltype(ks), decltype(vs)>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 44);
+    assert(s.values().get_allocator().get_id() == 44);
+  }
+}
+
+void test_iter_iter() {
+  const P arr[]          = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
+  const P sorted_arr[]   = {{1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
+  const PC arrc[]        = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
+  const PC sorted_arrc[] = {{1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
+  {
+    std::flat_map m(std::begin(arr), std::end(arr));
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_map<int, long>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_map m(std::begin(arrc), std::end(arrc));
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_map<int, long>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_map m(std::sorted_unique, std::begin(sorted_arr), std::end(sorted_arr));
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_map<int, long>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_map m(std::sorted_unique, std::begin(sorted_arrc), std::end(sorted_arrc));
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_map<int, long>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_map<int, short> mo;
+    std::flat_map m(mo.begin(), mo.end());
+    ASSERT_SAME_TYPE(decltype(m), decltype(mo));
+  }
+  {
+    std::flat_map<int, short> mo;
+    std::flat_map m(mo.cbegin(), mo.cend());
+    ASSERT_SAME_TYPE(decltype(m), decltype(mo));
+  }
+}
+
+void test_iter_iter_compare() {
+  const P arr[]          = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
+  const P sorted_arr[]   = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}};
+  const PC arrc[]        = {{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
+  const PC sorted_arrc[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}};
+  using C                = std::greater<long long>;
+  {
+    std::flat_map m(std::begin(arr), std::end(arr), C());
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_map<int, long, C>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_map m(std::begin(arrc), std::end(arrc), C());
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_map<int, long, C>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_map m(std::sorted_unique, std::begin(sorted_arr), std::end(sorted_arr), C());
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_map<int, long, C>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_map m(std::sorted_unique, std::begin(sorted_arrc), std::end(sorted_arrc), C());
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_map<int, long, C>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_map<int, short> mo;
+    std::flat_map m(mo.begin(), mo.end(), C());
+    ASSERT_SAME_TYPE(decltype(m), std::flat_map<int, short, C>);
+  }
+  {
+    std::flat_map<int, short> mo;
+    std::flat_map m(mo.cbegin(), mo.cend(), C());
+    ASSERT_SAME_TYPE(decltype(m), std::flat_map<int, short, C>);
+  }
+}
+
+void test_initializer_list() {
+  const P sorted_arr[] = {{1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}};
+  {
+    std::flat_map m{std::pair{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}};
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_map<int, long>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_map m(std::sorted_unique, {std::pair{1, 1L}, {2, 2L}, {3, 1L}, {INT_MAX, 1L}});
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_map<int, long>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+}
+
+void test_initializer_list_compare() {
+  const P sorted_arr[] = {{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}};
+  using C              = std::greater<long long>;
+  {
+    std::flat_map m({std::pair{1, 1L}, {2, 2L}, {1, 1L}, {INT_MAX, 1L}, {3, 1L}}, C());
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_map<int, long, C>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+  {
+    std::flat_map m(std::sorted_unique, {std::pair{INT_MAX, 1L}, {3, 1L}, {2, 2L}, {1, 1L}}, C());
+
+    ASSERT_SAME_TYPE(decltype(m), std::flat_map<int, long, C>);
+    assert(std::ranges::equal(m, sorted_arr));
+  }
+}
+
+void test_from_range() {
+  std::list<std::pair<int, short>> r     = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 4}, {3, 5}};
+  const std::pair<int, short> expected[] = {{1, 1}, {2, 2}, {3, 5}, {INT_MAX, 4}};
+  {
+    std::flat_map s(std::from_range, r);
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, short, std::less<int>>);
+    assert(std::ranges::equal(s, expected));
+  }
+  {
+    std::flat_map s(std::from_range, r, test_allocator<long>(0, 42));
+    ASSERT_SAME_TYPE(
+        decltype(s),
+        std::flat_map<int,
+                      short,
+                      std::less<int>,
+                      std::vector<int, test_allocator<int>>,
+                      std::vector<short, test_allocator<short>>>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 42);
+    assert(s.values().get_allocator().get_id() == 42);
+  }
+}
+
+void test_from_range_compare() {
+  std::list<std::pair<int, short>> r     = {{1, 1}, {2, 2}, {1, 1}, {INT_MAX, 4}, {3, 5}};
+  const std::pair<int, short> expected[] = {{INT_MAX, 4}, {3, 5}, {2, 2}, {1, 1}};
+  {
+    std::flat_map s(std::from_range, r, std::greater<int>());
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, short, std::greater<int>>);
+    assert(std::ranges::equal(s, expected));
+  }
+  {
+    std::flat_map s(std::from_range, r, std::greater<int>(), test_allocator<long>(0, 42));
+    ASSERT_SAME_TYPE(
+        decltype(s),
+        std::flat_map<int,
+                      short,
+                      std::greater<int>,
+                      std::vector<int, test_allocator<int>>,
+                      std::vector<short, test_allocator<short>>>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().get_id() == 42);
+    assert(s.values().get_allocator().get_id() == 42);
+  }
+}
+
+int main(int, char**) {
+  // Each test function also tests the sorted_unique-prefixed and allocator-suffixed overloads.
+  test_copy();
+  test_containers();
+  test_containers_compare();
+  test_iter_iter();
+  test_iter_iter_compare();
+  test_initializer_list();
+  test_initializer_list_compare();
+  test_from_range();
+  test_from_range_compare();
+
+  AssociativeContainerDeductionGuidesSfinaeAway<std::flat_map, std::flat_map<int, short>>();
+  {
+    std::flat_map s = {std::make_pair(1, 'a')}; // flat_map(initializer_list<pair<int, char>>)
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, char>);
+    assert(s.size() == 1);
+  }
+  {
+    using M = std::flat_map<int, short>;
+    M m;
+    std::flat_map s = {std::make_pair(m, m)}; // flat_map(initializer_list<pair<M, M>>)
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<M, M>);
+    assert(s.size() == 1);
+    assert(s[m] == m);
+  }
+
+  {
+    std::pair<int, int> source[3] = {{1, 1}, {2, 2}, {3, 3}};
+    std::flat_map s               = {source, source + 3}; // flat_map(InputIterator, InputIterator)
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, int>);
+    assert(s.size() == 3);
+  }
+  {
+    std::pair<int, int> source[3] = {{1, 1}, {2, 2}, {3, 3}};
+    std::flat_map s{source, source + 3}; // flat_map(InputIterator, InputIterator)
+    ASSERT_SAME_TYPE(decltype(s), std::flat_map<int, int>);
+    assert(s.size() == 3);
+  }
+  {
+    std::pair<int, int> source[3] = {{1, 1}, {2, 2}, {3, 3}};
+    std::flat_map s{std::sorted_unique, source, source + 3}; // flat_map(sorted_unique_t, InputIterator, InputIterator)
+    static_assert(std::is_same_v<decltype(s), std::flat_map<int, int>>);
+    assert(s.size() == 3);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.verify.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.verify.cpp
new file mode 100644
index 000000000000..08244f01cb24
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct.verify.cpp
@@ -0,0 +1,97 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// Test CTAD on cases where deduction should fail.
+
+#include <flat_map>
+#include <functional>
+#include <memory>
+#include <utility>
+#include <vector>
+
+struct NotAnAllocator {
+  friend bool operator<(NotAnAllocator, NotAnAllocator) { return false; }
+};
+
+using P  = std::pair<int, long>;
+using PC = std::pair<const int, long>;
+
+void test() {
+  {
+    // cannot deduce Key and T from just (KeyContainer), even if it's a container of pairs
+    std::vector<std::pair<int, int>> v;
+    std::flat_map s(v);
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
+  }
+  {
+    // cannot deduce Key and T from just (KeyContainer, Allocator)
+    std::vector<int> v;
+    std::flat_map s(v, std::allocator<std::pair<const int, int>>());
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
+  }
+  {
+    // cannot deduce Key and T from nothing
+    std::flat_map m;
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
+  }
+  {
+    // cannot deduce Key and T from just (Compare)
+    std::flat_map m(std::less<int>{});
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
+  }
+  {
+    // cannot deduce Key and T from just (Compare, Allocator)
+    std::flat_map m(std::less<int>{}, std::allocator<PC>{});
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
+  }
+  {
+    // cannot deduce Key and T from just (Allocator)
+    std::flat_map m(std::allocator<PC>{});
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
+  }
+  {
+    // cannot convert from some arbitrary unrelated type
+    NotAnAllocator a;
+    std::flat_map m(a);
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
+  }
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::flat_map m{{1, 1L}, {2, 2L}, {3, 3L}};
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
+  }
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::flat_map m({{1, 1L}, {2, 2L}, {3, 3L}}, std::less<int>());
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
+  }
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::flat_map m({{1, 1L}, {2, 2L}, {3, 3L}}, std::less<int>(), std::allocator<PC>());
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
+  }
+  {
+    // cannot deduce that the inner braced things should be std::pair and not something else
+    std::flat_map m({{1, 1L}, {2, 2L}, {3, 3L}}, std::allocator<PC>());
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
+  }
+  {
+    // since we have parens, not braces, this deliberately does not find the initializer_list constructor
+    std::flat_map m(P{1, 1L});
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
+  }
+  {
+    // since we have parens, not braces, this deliberately does not find the initializer_list constructor
+    std::flat_map m(PC{1, 1L});
+    // expected-error-re@-1{{{{no viable constructor or deduction guide for deduction of template arguments of '.*flat_map'}}}}
+  }
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct_pmr.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct_pmr.pass.cpp
new file mode 100644
index 000000000000..11c18ac13c76
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/deduct_pmr.pass.cpp
@@ -0,0 +1,106 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: availability-pmr-missing
+
+// <flat_map>
+
+#include <algorithm>
+#include <cassert>
+#include <climits>
+#include <deque>
+#include <initializer_list>
+#include <list>
+#include <flat_map>
+#include <functional>
+#include <memory_resource>
+#include <ranges>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "test_allocator.h"
+
+using P  = std::pair<int, long>;
+using PC = std::pair<const int, long>;
+
+void test_containers() {
+  std::deque<int, test_allocator<int>> ks({1, 2, 1, INT_MAX, 3}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> vs({1, 2, 1, 4, 5}, test_allocator<int>(0, 43));
+  std::deque<int, test_allocator<int>> sorted_ks({1, 2, 3, INT_MAX}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> sorted_vs({1, 2, 5, 4}, test_allocator<int>(0, 43));
+  const std::pair<int, short> expected[] = {{1, 1}, {2, 2}, {3, 5}, {INT_MAX, 4}};
+  {
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::monotonic_buffer_resource mr2;
+    std::pmr::deque<int> pks(ks.begin(), ks.end(), &mr);
+    std::pmr::deque<short> pvs(vs.begin(), vs.end(), &mr);
+    std::flat_map s(std::move(pks), std::move(pvs), &mr2);
+
+    ASSERT_SAME_TYPE(
+        decltype(s), std::flat_map<int, short, std::less<int>, std::pmr::deque<int>, std::pmr::deque<short>>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().resource() == &mr2);
+    assert(s.values().get_allocator().resource() == &mr2);
+  }
+  {
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::monotonic_buffer_resource mr2;
+    std::pmr::deque<int> pks(sorted_ks.begin(), sorted_ks.end(), &mr);
+    std::pmr::deque<short> pvs(sorted_vs.begin(), sorted_vs.end(), &mr);
+    std::flat_map s(std::sorted_unique, std::move(pks), std::move(pvs), &mr2);
+
+    ASSERT_SAME_TYPE(
+        decltype(s), std::flat_map<int, short, std::less<int>, std::pmr::deque<int>, std::pmr::deque<short>>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().resource() == &mr2);
+    assert(s.values().get_allocator().resource() == &mr2);
+  }
+}
+
+void test_containers_compare() {
+  std::deque<int, test_allocator<int>> ks({1, 2, 1, INT_MAX, 3}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> vs({1, 2, 1, 4, 5}, test_allocator<int>(0, 43));
+  std::deque<int, test_allocator<int>> sorted_ks({INT_MAX, 3, 2, 1}, test_allocator<int>(0, 42));
+  std::deque<short, test_allocator<short>> sorted_vs({4, 5, 2, 1}, test_allocator<int>(0, 43));
+  const std::pair<int, short> expected[] = {{INT_MAX, 4}, {3, 5}, {2, 2}, {1, 1}};
+  {
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::monotonic_buffer_resource mr2;
+    std::pmr::deque<int> pks(ks.begin(), ks.end(), &mr);
+    std::pmr::deque<short> pvs(vs.begin(), vs.end(), &mr);
+    std::flat_map s(std::move(pks), std::move(pvs), std::greater<int>(), &mr2);
+
+    ASSERT_SAME_TYPE(
+        decltype(s), std::flat_map<int, short, std::greater<int>, std::pmr::deque<int>, std::pmr::deque<short>>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().resource() == &mr2);
+    assert(s.values().get_allocator().resource() == &mr2);
+  }
+  {
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::monotonic_buffer_resource mr2;
+    std::pmr::deque<int> pks(sorted_ks.begin(), sorted_ks.end(), &mr);
+    std::pmr::deque<short> pvs(sorted_vs.begin(), sorted_vs.end(), &mr);
+    std::flat_map s(std::sorted_unique, std::move(pks), std::move(pvs), std::greater<int>(), &mr2);
+
+    ASSERT_SAME_TYPE(
+        decltype(s), std::flat_map<int, short, std::greater<int>, std::pmr::deque<int>, std::pmr::deque<short>>);
+    assert(std::ranges::equal(s, expected));
+    assert(s.keys().get_allocator().resource() == &mr2);
+    assert(s.values().get_allocator().resource() == &mr2);
+  }
+}
+
+int main(int, char**) {
+  test_containers();
+  test_containers_compare();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/default.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/default.pass.cpp
new file mode 100644
index 000000000000..c5b94896b929
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/default.pass.cpp
@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map();
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <type_traits>
+#include <vector>
+
+#include "test_macros.h"
+#include "min_allocator.h"
+#include "test_allocator.h"
+
+struct DefaultCtableComp {
+  explicit DefaultCtableComp() { default_constructed_ = true; }
+  bool operator()(int, int) const { return false; }
+  bool default_constructed_ = false;
+};
+
+int main(int, char**) {
+  {
+    std::flat_map<int, char*> m;
+    assert(m.empty());
+  }
+  {
+    // explicit(false)
+    std::flat_map<int, char*> m = {};
+    assert(m.empty());
+  }
+  {
+    std::flat_map<int, char*, DefaultCtableComp, std::deque<int, min_allocator<int>>> m;
+    assert(m.empty());
+    assert(m.begin() == m.end());
+    assert(m.key_comp().default_constructed_);
+  }
+  {
+    using A1 = explicit_allocator<int>;
+    using A2 = explicit_allocator<char*>;
+    {
+      std::flat_map<int, char*, DefaultCtableComp, std::vector<int, A1>, std::vector<char*, A2>> m;
+      assert(m.empty());
+      assert(m.key_comp().default_constructed_);
+    }
+    {
+      A1 a1;
+      std::flat_map<int, int, DefaultCtableComp, std::vector<int, A1>, std::vector<int, A1>> m(a1);
+      assert(m.empty());
+      assert(m.key_comp().default_constructed_);
+    }
+  }
+  {
+    // If an allocator is given, it must be usable by both containers.
+    using A = test_allocator<int>;
+    using M = std::flat_map<int, int, std::less<>, std::vector<int>, std::vector<int, A>>;
+    static_assert(std::is_constructible_v<M>);
+    static_assert(!std::is_constructible_v<M, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, A>);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/default_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/default_noexcept.pass.cpp
new file mode 100644
index 000000000000..ac24c8a8ac06
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/default_noexcept.pass.cpp
@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map()
+//    noexcept(
+//        is_nothrow_default_constructible_v<key_container_type> &&
+//        is_nothrow_default_constructible_v<mapped_container_type> &&
+//        is_nothrow_default_constructible_v<key_compare>);
+
+// This tests a conforming extension
+
+#include <cassert>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+struct ThrowingCtorComp {
+  ThrowingCtorComp() noexcept(false) {}
+  bool operator()(const auto&, const auto&) const { return false; }
+};
+
+int main(int, char**) {
+#if defined(_LIBCPP_VERSION)
+  {
+    using C = std::flat_map<MoveOnly, MoveOnly>;
+    static_assert(std::is_nothrow_default_constructible_v<C>);
+  }
+  {
+    using C = std::flat_map<MoveOnly, MoveOnly, std::less<MoveOnly>, std::vector<MoveOnly, test_allocator<MoveOnly>>>;
+    static_assert(std::is_nothrow_default_constructible_v<C>);
+  }
+#endif // _LIBCPP_VERSION
+  {
+    using C = std::flat_map<MoveOnly, MoveOnly, std::less<MoveOnly>, std::vector<MoveOnly, other_allocator<MoveOnly>>>;
+    static_assert(!std::is_nothrow_default_constructible_v<C>);
+    C c;
+  }
+  {
+    using C = std::flat_map<MoveOnly, MoveOnly, ThrowingCtorComp>;
+    static_assert(!std::is_nothrow_default_constructible_v<C>);
+    C c;
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/dtor_noexcept.pass.cpp
new file mode 100644
index 000000000000..e3ab33a55d95
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/dtor_noexcept.pass.cpp
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// ~flat_map();
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+struct ThrowingDtorComp {
+  bool operator()(const auto&, const auto&) const;
+  ~ThrowingDtorComp() noexcept(false);
+};
+
+int main(int, char**) {
+  {
+    using C = std::flat_map<MoveOnly, MoveOnly>;
+    static_assert(std::is_nothrow_destructible_v<C>);
+  }
+  {
+    using V = std::vector<MoveOnly, test_allocator<MoveOnly>>;
+    using C = std::flat_map<MoveOnly, MoveOnly, std::less<MoveOnly>, V, V>;
+    static_assert(std::is_nothrow_destructible_v<C>);
+  }
+  {
+    using V = std::deque<MoveOnly, other_allocator<MoveOnly>>;
+    using C = std::flat_map<MoveOnly, MoveOnly, std::greater<MoveOnly>, V, V>;
+    static_assert(std::is_nothrow_destructible_v<C>);
+  }
+#if defined(_LIBCPP_VERSION)
+  {
+    using C = std::flat_map<MoveOnly, MoveOnly, ThrowingDtorComp>;
+    static_assert(!std::is_nothrow_destructible_v<C>);
+  }
+#endif // _LIBCPP_VERSION
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/initializer_list.pass.cpp
new file mode 100644
index 000000000000..7a22746845d0
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/initializer_list.pass.cpp
@@ -0,0 +1,157 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map(initializer_list<value_type> il, const key_compare& comp = key_compare());
+// template<class Alloc>
+//    flat_map(initializer_list<value_type> il, const Alloc& a);
+// template<class Alloc>
+//    flat_map(initializer_list<value_type> il, const key_compare& comp, const Alloc& a);
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <type_traits>
+#include <vector>
+
+#include "test_macros.h"
+#include "min_allocator.h"
+#include "test_allocator.h"
+
+#include "../../../test_compare.h"
+
+struct DefaultCtableComp {
+  explicit DefaultCtableComp() { default_constructed_ = true; }
+  bool operator()(int, int) const { return false; }
+  bool default_constructed_ = false;
+};
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_map<int, int, C, V1, V1>;
+    using M2 = std::flat_map<int, int, C, V1, V2>;
+    using M3 = std::flat_map<int, int, C, V2, V1>;
+    using IL = std::initializer_list<std::pair<int, int>>;
+    static_assert(std::is_constructible_v<M1, IL, const A1&>);
+    static_assert(!std::is_constructible_v<M1, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M2, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M3, IL, const A2&>);
+
+    static_assert(std::is_constructible_v<M1, IL, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, IL, const C&, const A2&>);
+  }
+
+  {
+    // initializer_list<value_type> needs to match exactly
+    using M = std::flat_map<int, short>;
+    using C = typename M::key_compare;
+    static_assert(std::is_constructible_v<M, std::initializer_list<std::pair<int, short>>>);
+    static_assert(std::is_constructible_v<M, std::initializer_list<std::pair<int, short>>, C>);
+    static_assert(std::is_constructible_v<M, std::initializer_list<std::pair<int, short>>, C, std::allocator<int>>);
+    static_assert(std::is_constructible_v<M, std::initializer_list<std::pair<int, short>>, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<std::pair<const int, short>>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<std::pair<const int, short>>, C>);
+    static_assert(
+        !std::is_constructible_v<M, std::initializer_list<std::pair<const int, short>>, C, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<std::pair<const int, short>>, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<std::pair<const int, const short>>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<std::pair<const int, const short>>, C>);
+    static_assert(
+        !std::is_constructible_v<M, std::initializer_list<std::pair<const int, const short>>, C, std::allocator<int>>);
+    static_assert(
+        !std::is_constructible_v<M, std::initializer_list<std::pair<const int, const short>>, std::allocator<int>>);
+  }
+
+  std::pair<int, short> expected[] = {{1, 1}, {2, 2}, {3, 3}, {5, 2}};
+  {
+    // flat_map(initializer_list<value_type>);
+    using M                                         = std::flat_map<int, short>;
+    std::initializer_list<std::pair<int, short>> il = {{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}};
+    M m(il);
+    assert(std::equal(m.begin(), m.end(), expected, expected + 4));
+  }
+  {
+    // flat_map(initializer_list<value_type>);
+    // explicit(false)
+    using M = std::flat_map<int, short>;
+    M m     = {{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}};
+    assert(std::equal(m.begin(), m.end(), expected, expected + 4));
+  }
+  {
+    // flat_map(initializer_list<value_type>);
+    using M = std::flat_map<int, short, std::greater<int>, std::deque<int, min_allocator<int>>>;
+    M m     = {{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}};
+    assert(std::equal(m.rbegin(), m.rend(), expected, expected + 4));
+  }
+  {
+    using A = explicit_allocator<int>;
+    {
+      // flat_map(initializer_list<value_type>);
+      // different comparator
+      using M = std::flat_map<int, int, DefaultCtableComp, std::vector<int, A>, std::deque<int, A>>;
+      M m     = {{1, 1}, {2, 2}, {3, 3}};
+      assert(m.size() == 1);
+      assert(m.begin()->first == m.begin()->second);
+      LIBCPP_ASSERT(*m.begin() == std::make_pair(1, 1));
+      assert(m.key_comp().default_constructed_);
+    }
+    {
+      // flat_map(initializer_list<value_type>, const Allocator&);
+      using M = std::flat_map<int, int, std::greater<int>, std::deque<int, A>, std::vector<int, A>>;
+      A a;
+      M m({{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}}, a);
+      assert(std::equal(m.rbegin(), m.rend(), expected, expected + 4));
+    }
+  }
+  {
+    // flat_map(initializer_list<value_type>, const key_compare&);
+    using C = test_less<int>;
+    using M = std::flat_map<int, short, C>;
+    auto m  = M({{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}}, C(10));
+    assert(std::equal(m.begin(), m.end(), expected, expected + 4));
+    assert(m.key_comp() == C(10));
+
+    // explicit(false)
+    M m2 = {{{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}}, C(10)};
+    assert(m2 == m);
+    assert(m2.key_comp() == C(10));
+  }
+  {
+    // flat_map(initializer_list<value_type>, const key_compare&);
+    // Sorting uses the comparator that was passed in
+    using M = std::flat_map<int, short, std::function<bool(int, int)>, std::deque<int, min_allocator<int>>>;
+    auto m  = M({{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}}, std::greater<int>());
+    assert(std::equal(m.rbegin(), m.rend(), expected, expected + 4));
+    assert(m.key_comp()(2, 1) == true);
+  }
+  {
+    // flat_map(initializer_list<value_type> il, const key_compare& comp, const Alloc& a);
+    using A = explicit_allocator<int>;
+    using M = std::flat_map<int, int, std::greater<int>, std::deque<int, A>, std::vector<int, A>>;
+    A a;
+    M m({{5, 2}, {2, 2}, {2, 2}, {3, 3}, {1, 1}, {3, 3}}, {}, a);
+    assert(std::equal(m.rbegin(), m.rend(), expected, expected + 4));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/iter_iter.pass.cpp
new file mode 100644
index 000000000000..7c0c48796994
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/iter_iter.pass.cpp
@@ -0,0 +1,154 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template <class InputIterator>
+//   flat_map(InputIterator first, InputIterator last, const key_compare& comp = key_compare());
+// template<class InputIterator, class Allocator>
+//   flat_map(InputIterator first, InputIterator last, const Allocator& a);
+// template<class InputIterator, class Allocator>
+//   flat_map(InputIterator first, InputIterator last, const key_compare& comp, const Allocator& a);
+
+#include <algorithm>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C     = test_less<int>;
+    using A1    = test_allocator<int>;
+    using A2    = other_allocator<int>;
+    using V1    = std::vector<int, A1>;
+    using V2    = std::vector<int, A2>;
+    using M1    = std::flat_map<int, int, C, V1, V1>;
+    using M2    = std::flat_map<int, int, C, V1, V2>;
+    using M3    = std::flat_map<int, int, C, V2, V1>;
+    using Iter1 = typename M1::iterator;
+    using Iter2 = typename M2::iterator;
+    using Iter3 = typename M3::iterator;
+    static_assert(std::is_constructible_v<M1, Iter1, Iter1, const A1&>);
+    static_assert(!std::is_constructible_v<M1, Iter1, Iter1, const A2&>);
+    static_assert(!std::is_constructible_v<M2, Iter2, Iter2, const A2&>);
+    static_assert(!std::is_constructible_v<M3, Iter3, Iter3, const A2&>);
+
+    static_assert(std::is_constructible_v<M1, Iter1, Iter1, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, Iter1, Iter1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, Iter2, Iter2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, Iter3, Iter3, const C&, const A2&>);
+  }
+
+  using P      = std::pair<int, short>;
+  P ar[]       = {{1, 1}, {1, 2}, {1, 3}, {2, 4}, {2, 5}, {3, 6}, {2, 7}, {3, 8}, {3, 9}};
+  P expected[] = {{1, 1}, {2, 4}, {3, 6}};
+  {
+    // flat_map(InputIterator , InputIterator)
+    // cpp17_input_iterator
+    using M = std::flat_map<int, short>;
+    auto m  = M(cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 9));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+
+    // explicit(false)
+    M m2 = {cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 9)};
+    assert(m2 == m);
+  }
+  {
+    // flat_map(InputIterator , InputIterator)
+    // greater
+    using M = std::flat_map<int, short, std::greater<int>, std::deque<int, min_allocator<int>>, std::deque<short>>;
+    auto m  = M(cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 9));
+    assert((m.keys() == std::deque<int, min_allocator<int>>{3, 2, 1}));
+    LIBCPP_ASSERT((m.values() == std::deque<short>{6, 4, 1}));
+  }
+  {
+    // flat_map(InputIterator , InputIterator)
+    // Test when the operands are of array type (also contiguous iterator type)
+    using M = std::flat_map<int, short, std::greater<int>, std::vector<int, min_allocator<int>>>;
+    auto m  = M(ar, ar);
+    assert(m.empty());
+  }
+  {
+    // flat_map(InputIterator , InputIterator, const key_compare&)
+    using C = test_less<int>;
+    using M = std::flat_map<int, short, C, std::vector<int>, std::deque<short>>;
+    auto m  = M(ar, ar + 9, C(3));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.key_comp() == C(3));
+
+    // explicit(false)
+    M m2 = {ar, ar + 9, C(3)};
+    assert(m2 == m);
+    assert(m2.key_comp() == C(3));
+  }
+  {
+    // flat_map(InputIterator , InputIterator, const Allocator&)
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_map<int, short, std::less<int>, std::vector<int, A1>, std::deque<short, A2>>;
+    auto m   = M(ar, ar + 9, A1(5));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_map(InputIterator , InputIterator, const Allocator&)
+    // explicit(false)
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_map<int, short, std::less<int>, std::vector<int, A1>, std::deque<short, A2>>;
+    M m      = {ar, ar + 9, A1(5)}; // implicit ctor
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_map(InputIterator , InputIterator, const key_compare&, const Allocator&)
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_map<int, short, C, std::vector<int, A1>, std::deque<short, A2>>;
+    auto m   = M(ar, ar + 9, C(3), A1(5));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.key_comp() == C(3));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_map(InputIterator , InputIterator, const key_compare&, const Allocator&)
+    // explicit(false)
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_map<int, short, std::less<int>, std::deque<int, A1>, std::vector<short, A2>>;
+    M m      = {ar, ar + 9, {}, A2(5)}; // implicit ctor
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/iter_iter_stability.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/iter_iter_stability.pass.cpp
new file mode 100644
index 000000000000..1ce859f6c737
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/iter_iter_stability.pass.cpp
@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<class InputIterator>
+//   flat_map(InputIterator first, InputIterator last, const key_compare& comp = key_compare())
+//
+// libc++ uses stable_sort to ensure that flat_map's behavior matches map's,
+// in terms of which duplicate items are kept.
+// This tests a conforming extension.
+
+#include <algorithm>
+#include <cassert>
+#include <cstdint>
+#include <flat_map>
+#include <random>
+#include <map>
+
+#include "test_macros.h"
+
+struct Mod256 {
+  bool operator()(int x, int y) const { return (x % 256) < (y % 256); }
+};
+
+int main(int, char**) {
+  std::mt19937 randomness;
+  std::pair<uint16_t, uint16_t> pairs[200];
+  for (auto& pair : pairs) {
+    pair = {uint16_t(randomness()), uint16_t(randomness())};
+  }
+
+  {
+    std::map<uint16_t, uint16_t, Mod256> m(pairs, pairs + 200);
+    std::flat_map<uint16_t, uint16_t, Mod256> fm(pairs, pairs + 200);
+    assert(fm.size() == m.size());
+    LIBCPP_ASSERT(std::ranges::equal(fm, m));
+  }
+  {
+    std::map<uint16_t, uint16_t, Mod256> m(pairs, pairs + 200, std::allocator<int>());
+    std::flat_map<uint16_t, uint16_t, Mod256> fm(pairs, pairs + 200, std::allocator<int>());
+    assert(fm.size() == m.size());
+    LIBCPP_ASSERT(std::ranges::equal(fm, m));
+  }
+  {
+    std::map<uint16_t, uint16_t, Mod256> m(pairs, pairs + 200, Mod256());
+    std::flat_map<uint16_t, uint16_t, Mod256> fm(pairs, pairs + 200, Mod256());
+    assert(fm.size() == m.size());
+    LIBCPP_ASSERT(std::ranges::equal(fm, m));
+  }
+  {
+    std::map<uint16_t, uint16_t, Mod256> m(pairs, pairs + 200, Mod256(), std::allocator<int>());
+    std::flat_map<uint16_t, uint16_t, Mod256> fm(pairs, pairs + 200, Mod256(), std::allocator<int>());
+    assert(fm.size() == m.size());
+    LIBCPP_ASSERT(std::ranges::equal(fm, m));
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move.pass.cpp
new file mode 100644
index 000000000000..955d3156064a
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move.pass.cpp
@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map(flat_map&&);
+
+#include <algorithm>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "../helpers.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+#include "test_allocator.h"
+#include "min_allocator.h"
+
+int main(int, char**) {
+  {
+    using C = test_less<int>;
+    using A = test_allocator<int>;
+    using M = std::flat_map<int, int, C, std::vector<int, A>, std::deque<int, A>>;
+    M mo    = M({{1, 1}, {2, 2}, {3, 1}}, C(5), A(7));
+    M m     = std::move(mo);
+    assert((m == M{{1, 1}, {2, 2}, {3, 1}}));
+    assert(m.key_comp() == C(5));
+    assert(m.keys().get_allocator() == A(7));
+    assert(m.values().get_allocator() == A(7));
+
+    assert(mo.empty());
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys().get_allocator().get_id() == test_alloc_base::moved_value);
+    assert(mo.values().get_allocator().get_id() == test_alloc_base::moved_value);
+  }
+  {
+    using C = test_less<int>;
+    using A = min_allocator<int>;
+    using M = std::flat_map<int, int, C, std::vector<int, A>, std::deque<int, A>>;
+    M mo    = M({{1, 1}, {2, 2}, {3, 1}}, C(5), A());
+    M m     = std::move(mo);
+    assert((m == M{{1, 1}, {2, 2}, {3, 1}}));
+    assert(m.key_comp() == C(5));
+    assert(m.keys().get_allocator() == A());
+    assert(m.values().get_allocator() == A());
+
+    assert(mo.empty());
+    assert(mo.key_comp() == C(5));
+    assert(m.keys().get_allocator() == A());
+    assert(m.values().get_allocator() == A());
+  }
+  {
+    // A moved-from flat_map maintains its class invariant in the presence of moved-from comparators.
+    using M = std::flat_map<int, int, std::function<bool(int, int)>>;
+    M mo    = M({{1, 1}, {2, 2}, {3, 1}}, std::less<int>());
+    M m     = std::move(mo);
+    assert(m.size() == 3);
+    assert(std::is_sorted(m.begin(), m.end(), m.value_comp()));
+    assert(m.key_comp()(1, 2) == true);
+
+    assert(std::is_sorted(mo.begin(), mo.end(), mo.value_comp()));
+    LIBCPP_ASSERT(m.key_comp()(1, 2) == true);
+    LIBCPP_ASSERT(mo.empty());
+    mo.insert({{1, 1}, {2, 2}, {3, 1}}); // insert has no preconditions
+    assert(m == mo);
+  }
+  {
+    // moved-from object maintains invariant if one of underlying container does not clear after move
+    using M = std::flat_map<int, int, std::less<>, std::vector<int>, CopyOnlyVector<int>>;
+    M m1    = M({1, 2, 3}, {1, 2, 3});
+    M m2    = std::move(m1);
+    assert(m2.size() == 3);
+    check_invariant(m1);
+    LIBCPP_ASSERT(m1.empty());
+    LIBCPP_ASSERT(m1.keys().size() == 0);
+    LIBCPP_ASSERT(m1.values().size() == 0);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_alloc.pass.cpp
new file mode 100644
index 000000000000..93a397642252
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_alloc.pass.cpp
@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map(flat_map&&, const allocator_type&);
+
+#include <algorithm>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <ranges>
+#include <vector>
+
+#include "../helpers.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+#include "test_allocator.h"
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_map<int, int, C, V1, V1>;
+    using M2 = std::flat_map<int, int, C, V1, V2>;
+    using M3 = std::flat_map<int, int, C, V2, V1>;
+    static_assert(std::is_constructible_v<M1, M1&&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, M1&&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, M2&&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, M3&&, const A2&>);
+  }
+  {
+    std::pair<int, int> expected[] = {{1, 1}, {2, 2}, {3, 1}};
+    using C                        = test_less<int>;
+    using A                        = test_allocator<int>;
+    using M                        = std::flat_map<int, int, C, std::vector<int, A>, std::deque<int, A>>;
+    auto mo                        = M(expected, expected + 3, C(5), A(7));
+    auto m                         = M(std::move(mo), A(3));
+
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    auto [keys, values] = std::move(m).extract();
+    assert(keys.get_allocator() == A(3));
+    assert(values.get_allocator() == A(3));
+    assert(std::ranges::equal(keys, expected | std::views::elements<0>));
+    assert(std::ranges::equal(values, expected | std::views::elements<1>));
+
+    // The original flat_map is moved-from.
+    assert(std::is_sorted(mo.begin(), mo.end(), mo.value_comp()));
+    assert(mo.empty());
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys().get_allocator() == A(7));
+    assert(mo.values().get_allocator() == A(7));
+  }
+  {
+    // moved-from object maintains invariant if one of underlying container does not clear after move
+    using M = std::flat_map<int, int, std::less<>, std::vector<int>, CopyOnlyVector<int>>;
+    M m1    = M({1, 2, 3}, {1, 2, 3});
+    M m2(std::move(m1), std::allocator<int>{});
+    assert(m2.size() == 3);
+    check_invariant(m1);
+    LIBCPP_ASSERT(m1.empty());
+    LIBCPP_ASSERT(m1.keys().size() == 0);
+    LIBCPP_ASSERT(m1.values().size() == 0);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_assign.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_assign.pass.cpp
new file mode 100644
index 000000000000..a94c442c695d
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_assign.pass.cpp
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map& operator=(flat_map&&);
+
+#include <algorithm>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "../../../test_compare.h"
+#include "test_allocator.h"
+#include "min_allocator.h"
+
+int main(int, char**) {
+  {
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<char>;
+    using M  = std::flat_map<int, char, C, std::vector<int, A1>, std::vector<char, A2>>;
+    M mo     = M({{1, 1}, {2, 3}, {3, 2}}, C(5), A1(7));
+    M m      = M({}, C(3), A1(7));
+    m        = std::move(mo);
+    assert((m == M{{1, 1}, {2, 3}, {3, 2}}));
+    assert(m.key_comp() == C(5));
+    auto [ks, vs] = std::move(m).extract();
+    assert(ks.get_allocator() == A1(7));
+    assert(vs.get_allocator() == A2(7));
+    assert(mo.empty());
+  }
+  {
+    using C  = test_less<int>;
+    using A1 = other_allocator<int>;
+    using A2 = other_allocator<char>;
+    using M  = std::flat_map<int, char, C, std::deque<int, A1>, std::deque<char, A2>>;
+    M mo     = M({{4, 5}, {5, 4}}, C(5), A1(7));
+    M m      = M({{1, 1}, {2, 2}, {3, 3}, {4, 4}}, C(3), A1(7));
+    m        = std::move(mo);
+    assert((m == M{{4, 5}, {5, 4}}));
+    assert(m.key_comp() == C(5));
+    auto [ks, vs] = std::move(m).extract();
+    assert(ks.get_allocator() == A1(7));
+    assert(vs.get_allocator() == A2(7));
+    assert(mo.empty());
+  }
+  {
+    using A = min_allocator<int>;
+    using M = std::flat_map<int, int, std::greater<int>, std::vector<int, A>, std::vector<int, A>>;
+    M mo    = M({{5, 1}, {4, 2}, {3, 3}}, A());
+    M m     = M({{4, 4}, {3, 3}, {2, 2}, {1, 1}}, A());
+    m       = std::move(mo);
+    assert((m == M{{5, 1}, {4, 2}, {3, 3}}));
+    auto [ks, vs] = std::move(m).extract();
+    assert(ks.get_allocator() == A());
+    assert(vs.get_allocator() == A());
+    assert(mo.empty());
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_assign_clears.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_assign_clears.pass.cpp
new file mode 100644
index 000000000000..f28d52dd4e46
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_assign_clears.pass.cpp
@@ -0,0 +1,104 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map& operator=(flat_map&&);
+// Preserves the class invariant for the moved-from flat_map.
+
+#include <algorithm>
+#include <cassert>
+#include <compare>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "../helpers.h"
+#include "test_macros.h"
+
+struct MoveNegates {
+  int value_    = 0;
+  MoveNegates() = default;
+  MoveNegates(int v) : value_(v) {}
+  MoveNegates(MoveNegates&& rhs) : value_(rhs.value_) { rhs.value_ = -rhs.value_; }
+  MoveNegates& operator=(MoveNegates&& rhs) {
+    value_     = rhs.value_;
+    rhs.value_ = -rhs.value_;
+    return *this;
+  }
+  ~MoveNegates()                             = default;
+  auto operator<=>(const MoveNegates&) const = default;
+};
+
+struct MoveClears {
+  int value_   = 0;
+  MoveClears() = default;
+  MoveClears(int v) : value_(v) {}
+  MoveClears(MoveClears&& rhs) : value_(rhs.value_) { rhs.value_ = 0; }
+  MoveClears& operator=(MoveClears&& rhs) {
+    value_     = rhs.value_;
+    rhs.value_ = 0;
+    return *this;
+  }
+  ~MoveClears()                             = default;
+  auto operator<=>(const MoveClears&) const = default;
+};
+
+int main(int, char**) {
+  auto value_eq = [](auto&& p, auto&& q) { return p.first == q.first; };
+  {
+    const std::pair<int, int> expected[] = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}, {7, 7}, {8, 8}};
+    using M = std::flat_map<MoveNegates, int, std::less<MoveNegates>, std::vector<MoveNegates>>;
+    M m     = M(expected, expected + 8);
+    M m2    = M(expected, expected + 3);
+
+    m2 = std::move(m);
+
+    assert(std::equal(m2.begin(), m2.end(), expected, expected + 8));
+    LIBCPP_ASSERT(m.empty());
+    assert(std::is_sorted(m.begin(), m.end(), m.value_comp()));          // still sorted
+    assert(std::adjacent_find(m.begin(), m.end(), value_eq) == m.end()); // still contains no duplicates
+    m.insert({1, 1});
+    m.insert({2, 2});
+    assert(m.contains(1));
+    assert(m.find(2) != m.end());
+  }
+  {
+    const std::pair<int, int> expected[] = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}, {7, 7}, {8, 8}};
+    using M = std::flat_map<MoveClears, int, std::less<MoveClears>, std::vector<MoveClears>>;
+    M m     = M(expected, expected + 8);
+    M m2    = M(expected, expected + 3);
+
+    m2 = std::move(m);
+
+    assert(std::equal(m2.begin(), m2.end(), expected, expected + 8));
+    LIBCPP_ASSERT(m.empty());
+    assert(std::is_sorted(m.begin(), m.end(), m.value_comp()));          // still sorted
+    assert(std::adjacent_find(m.begin(), m.end(), value_eq) == m.end()); // still contains no duplicates
+    m.insert({1, 1});
+    m.insert({2, 2});
+    assert(m.contains(1));
+    assert(m.find(2) != m.end());
+  }
+  {
+    // moved-from object maintains invariant if one of underlying container does not clear after move
+    using M = std::flat_map<int, int, std::less<>, std::vector<int>, CopyOnlyVector<int>>;
+    M m1    = M({1, 2, 3}, {1, 2, 3});
+    M m2    = M({1, 2}, {1, 2});
+    m2      = std::move(m1);
+    assert(m2.size() == 3);
+    check_invariant(m1);
+    LIBCPP_ASSERT(m1.empty());
+    LIBCPP_ASSERT(m1.keys().size() == 0);
+    LIBCPP_ASSERT(m1.values().size() == 0);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_assign_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_assign_noexcept.pass.cpp
new file mode 100644
index 000000000000..665b763e6c4f
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_assign_noexcept.pass.cpp
@@ -0,0 +1,110 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map& operator=(flat_map&& c)
+//     noexcept(
+//          is_nothrow_move_assignable<key_container_type>::value &&
+//          is_nothrow_move_assignable<mapped_container_type>::value &&
+//          is_nothrow_copy_assignable<key_compare>::value);
+
+// This tests a conforming extension
+
+#include <flat_map>
+#include <functional>
+#include <memory_resource>
+#include <type_traits>
+#include <vector>
+
+#include "MoveOnly.h"
+#include "test_allocator.h"
+#include "test_macros.h"
+
+struct MoveSensitiveComp {
+  MoveSensitiveComp() noexcept(false)                         = default;
+  MoveSensitiveComp(const MoveSensitiveComp&) noexcept(false) = default;
+  MoveSensitiveComp(MoveSensitiveComp&& rhs) { rhs.is_moved_from_ = true; }
+  MoveSensitiveComp& operator=(const MoveSensitiveComp&) noexcept = default;
+  MoveSensitiveComp& operator=(MoveSensitiveComp&& rhs) {
+    rhs.is_moved_from_ = true;
+    return *this;
+  }
+  bool operator()(const auto&, const auto&) const { return false; }
+  bool is_moved_from_ = false;
+};
+
+struct MoveThrowsComp {
+  MoveThrowsComp(MoveThrowsComp&&) noexcept(false);
+  MoveThrowsComp(const MoveThrowsComp&) noexcept(true);
+  MoveThrowsComp& operator=(MoveThrowsComp&&) noexcept(false);
+  MoveThrowsComp& operator=(const MoveThrowsComp&) noexcept(true);
+  bool operator()(const auto&, const auto&) const;
+};
+
+int main(int, char**) {
+  {
+    using C = std::flat_map<int, int>;
+    LIBCPP_STATIC_ASSERT(std::is_nothrow_move_assignable_v<C>);
+  }
+  {
+    using C =
+        std::flat_map<MoveOnly,
+                      int,
+                      std::less<MoveOnly>,
+                      std::vector<MoveOnly, test_allocator<MoveOnly>>,
+                      std::vector<int, test_allocator<int>>>;
+    static_assert(!std::is_nothrow_move_assignable_v<C>);
+  }
+  {
+    using C =
+        std::flat_map<int,
+                      MoveOnly,
+                      std::less<int>,
+                      std::vector<int, test_allocator<int>>,
+                      std::vector<MoveOnly, test_allocator<MoveOnly>>>;
+    static_assert(!std::is_nothrow_move_assignable_v<C>);
+  }
+  {
+    using C =
+        std::flat_map<MoveOnly,
+                      int,
+                      std::less<MoveOnly>,
+                      std::vector<MoveOnly, other_allocator<MoveOnly>>,
+                      std::vector<int, other_allocator<int>>>;
+    LIBCPP_STATIC_ASSERT(std::is_nothrow_move_assignable_v<C>);
+  }
+  {
+    using C =
+        std::flat_map<int,
+                      MoveOnly,
+                      std::less<int>,
+                      std::vector<int, other_allocator<int>>,
+                      std::vector<MoveOnly, other_allocator<MoveOnly>>>;
+    LIBCPP_STATIC_ASSERT(std::is_nothrow_move_assignable_v<C>);
+  }
+  {
+    // Test with a comparator that throws on move-assignment.
+    using C = std::flat_map<int, int, MoveThrowsComp>;
+    LIBCPP_STATIC_ASSERT(!std::is_nothrow_move_assignable_v<C>);
+  }
+  {
+    // Test with a container that throws on move-assignment.
+    using C = std::flat_map<int, int, std::less<int>, std::pmr::vector<int>, std::vector<int>>;
+    static_assert(!std::is_nothrow_move_assignable_v<C>);
+  }
+  {
+    // Test with a container that throws on move-assignment.
+    using C = std::flat_map<int, int, std::less<int>, std::vector<int>, std::pmr::vector<int>>;
+    static_assert(!std::is_nothrow_move_assignable_v<C>);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_exceptions.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_exceptions.pass.cpp
new file mode 100644
index 000000000000..cb7e30c2b74f
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_exceptions.pass.cpp
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: no-exceptions
+
+// <flat_map>
+
+// flat_map(flat_map&& s);
+// If any member function in [flat.map.defn] exits via an exception, the invariant is restored.
+
+#include <algorithm>
+#include <cassert>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "../helpers.h"
+#include "test_macros.h"
+
+static int countdown = 0;
+
+struct EvilContainer : std::vector<int> {
+  EvilContainer() = default;
+  EvilContainer(EvilContainer&& rhs) {
+    // Throw on move-construction.
+    if (--countdown == 0) {
+      rhs.insert(rhs.end(), 0);
+      rhs.insert(rhs.end(), 0);
+      throw 42;
+    }
+  }
+};
+
+int main(int, char**) {
+  {
+    using M   = std::flat_map<int, int, std::less<int>, EvilContainer, std::vector<int>>;
+    M mo      = {{1, 1}, {2, 2}, {3, 3}};
+    countdown = 1;
+    try {
+      M m = std::move(mo);
+      assert(false); // not reached
+    } catch (int x) {
+      assert(x == 42);
+    }
+    // The source flat_map maintains its class invariant.
+    check_invariant(mo);
+    LIBCPP_ASSERT(mo.empty());
+  }
+  {
+    using M   = std::flat_map<int, int, std::less<int>, std::vector<int>, EvilContainer>;
+    M mo      = {{1, 1}, {2, 2}, {3, 3}};
+    countdown = 1;
+    try {
+      M m = std::move(mo);
+      assert(false); // not reached
+    } catch (int x) {
+      assert(x == 42);
+    }
+    // The source flat_map maintains its class invariant.
+    check_invariant(mo);
+    LIBCPP_ASSERT(mo.empty());
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_noexcept.pass.cpp
new file mode 100644
index 000000000000..d281dafbcf72
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/move_noexcept.pass.cpp
@@ -0,0 +1,102 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map(flat_map&&)
+//        noexcept(is_nothrow_move_constructible<key_container_type>::value &&
+//                 is_nothrow_move_constructible<mapped_container_type>::value &&
+//                 is_nothrow_copy_constructible<key_compare>::value);
+
+// This tests a conforming extension
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <memory>
+#include <type_traits>
+#include <vector>
+
+#include "test_macros.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+
+template <class T>
+struct ThrowingMoveAllocator {
+  using value_type                                    = T;
+  explicit ThrowingMoveAllocator()                    = default;
+  ThrowingMoveAllocator(const ThrowingMoveAllocator&) = default;
+  ThrowingMoveAllocator(ThrowingMoveAllocator&&) noexcept(false) {}
+  T* allocate(std::ptrdiff_t n) { return std::allocator<T>().allocate(n); }
+  void deallocate(T* p, std::ptrdiff_t n) { return std::allocator<T>().deallocate(p, n); }
+  friend bool operator==(ThrowingMoveAllocator, ThrowingMoveAllocator) = default;
+};
+
+struct ThrowingMoveComp {
+  ThrowingMoveComp() = default;
+  ThrowingMoveComp(const ThrowingMoveComp&) noexcept(true) {}
+  ThrowingMoveComp(ThrowingMoveComp&&) noexcept(false) {}
+  bool operator()(const auto&, const auto&) const { return false; }
+};
+
+struct MoveSensitiveComp {
+  MoveSensitiveComp() noexcept(false)                  = default;
+  MoveSensitiveComp(const MoveSensitiveComp&) noexcept = default;
+  MoveSensitiveComp(MoveSensitiveComp&& rhs) { rhs.is_moved_from_ = true; }
+  MoveSensitiveComp& operator=(const MoveSensitiveComp&) noexcept(false) = default;
+  MoveSensitiveComp& operator=(MoveSensitiveComp&& rhs) {
+    rhs.is_moved_from_ = true;
+    return *this;
+  }
+  bool operator()(const auto&, const auto&) const { return false; }
+  bool is_moved_from_ = false;
+};
+
+int main(int, char**) {
+  {
+    using C = std::flat_map<int, int>;
+    LIBCPP_STATIC_ASSERT(std::is_nothrow_move_constructible_v<C>);
+    C c;
+    C d = std::move(c);
+  }
+  {
+    using C = std::flat_map<int, int, std::less<int>, std::deque<int, test_allocator<int>>>;
+    LIBCPP_STATIC_ASSERT(std::is_nothrow_move_constructible_v<C>);
+    C c;
+    C d = std::move(c);
+  }
+#if _LIBCPP_VERSION
+  {
+    // Container fails to be nothrow-move-constructible; this relies on libc++'s support for non-nothrow-copyable allocators
+    using C = std::flat_map<int, int, std::less<int>, std::deque<int, ThrowingMoveAllocator<int>>, std::vector<int>>;
+    static_assert(!std::is_nothrow_move_constructible_v<std::deque<int, ThrowingMoveAllocator<int>>>);
+    static_assert(!std::is_nothrow_move_constructible_v<C>);
+    C c;
+    C d = std::move(c);
+  }
+  {
+    // Container fails to be nothrow-move-constructible; this relies on libc++'s support for non-nothrow-copyable allocators
+    using C = std::flat_map<int, int, std::less<int>, std::vector<int>, std::deque<int, ThrowingMoveAllocator<int>>>;
+    static_assert(!std::is_nothrow_move_constructible_v<std::deque<int, ThrowingMoveAllocator<int>>>);
+    static_assert(!std::is_nothrow_move_constructible_v<C>);
+    C c;
+    C d = std::move(c);
+  }
+#endif // _LIBCPP_VERSION
+  {
+    // Comparator fails to be nothrow-move-constructible
+    using C = std::flat_map<int, int, ThrowingMoveComp>;
+    static_assert(!std::is_nothrow_move_constructible_v<C>);
+    C c;
+    C d = std::move(c);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/pmr.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/pmr.pass.cpp
new file mode 100644
index 000000000000..154af11bb9b4
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/pmr.pass.cpp
@@ -0,0 +1,361 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: availability-pmr-missing
+
+// <flat_map>
+
+// Test various constructors with pmr
+
+#include <algorithm>
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <memory_resource>
+#include <ranges>
+#include <vector>
+#include <string>
+
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "test_allocator.h"
+#include "../../../test_compare.h"
+
+int main(int, char**) {
+  {
+    // flat_map(const Allocator& a);
+    using M = std::flat_map<int, short, std::less<int>, std::pmr::vector<int>, std::pmr::vector<short>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::polymorphic_allocator<int> pa = &mr;
+    auto m1                                 = M(pa);
+    assert(m1.empty());
+    assert(m1.keys().get_allocator() == pa);
+    assert(m1.values().get_allocator() == pa);
+    auto m2 = M(&mr);
+    assert(m2.empty());
+    assert(m2.keys().get_allocator() == pa);
+    assert(m2.values().get_allocator() == pa);
+  }
+  {
+    // flat_map(const key_compare& comp, const Alloc& a);
+    using M = std::flat_map<int, int, std::function<bool(int, int)>, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    vm.emplace_back(std::greater<int>());
+    assert(vm[0] == M{});
+    assert(vm[0].key_comp()(2, 1) == true);
+    assert(vm[0].value_comp()({2, 0}, {1, 0}) == true);
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    // flat_map(const key_container_type& key_cont, const mapped_container_type& mapped_cont,
+    //          const Allocator& a);
+    using M = std::flat_map<int, int, std::less<int>, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    std::pmr::vector<int> ks = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    std::pmr::vector<int> vs = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    assert(ks.get_allocator().resource() != &mr);
+    assert(vs.get_allocator().resource() != &mr);
+    vm.emplace_back(ks, vs);
+    assert(ks.size() == 9); // ks' value is unchanged, since it was an lvalue above
+    assert(vs.size() == 9); // vs' value is unchanged, since it was an lvalue above
+    assert((vm[0] == M{{1, 1}, {2, 2}, {3, 3}}));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    // flat_map(const flat_map&, const allocator_type&);
+    using C = test_less<int>;
+    using M = std::flat_map<int, int, C, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr1;
+    std::pmr::monotonic_buffer_resource mr2;
+    M mo = M({1, 2, 3}, {2, 2, 1}, C(5), &mr1);
+    M m  = {mo, &mr2}; // also test the implicitness of this constructor
+
+    assert(m.key_comp() == C(5));
+    assert((m.keys() == std::pmr::vector<int>{1, 2, 3}));
+    assert((m.values() == std::pmr::vector<int>{2, 2, 1}));
+    assert(m.keys().get_allocator().resource() == &mr2);
+    assert(m.values().get_allocator().resource() == &mr2);
+
+    // mo is unchanged
+    assert(mo.key_comp() == C(5));
+    assert((mo.keys() == std::pmr::vector<int>{1, 2, 3}));
+    assert((mo.values() == std::pmr::vector<int>{2, 2, 1}));
+    assert(mo.keys().get_allocator().resource() == &mr1);
+    assert(mo.values().get_allocator().resource() == &mr1);
+  }
+  {
+    // flat_map(const flat_map&, const allocator_type&);
+    using M = std::flat_map<int, int, std::less<>, std::pmr::vector<int>, std::pmr::deque<int>>;
+    std::pmr::vector<M> vs;
+    M m = {{1, 2}, {2, 2}, {3, 1}};
+    vs.push_back(m);
+    assert(vs[0] == m);
+  }
+  {
+    // flat_map& operator=(const flat_map& m);
+    // pmr allocator is not propagated
+    using M = std::flat_map<int, int, std::less<>, std::pmr::deque<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr1;
+    std::pmr::monotonic_buffer_resource mr2;
+    M mo = M({{1, 1}, {2, 2}, {3, 3}}, &mr1);
+    M m  = M({{4, 4}, {5, 5}}, &mr2);
+    m    = mo;
+    assert((m == M{{1, 1}, {2, 2}, {3, 3}}));
+    assert(m.keys().get_allocator().resource() == &mr2);
+    assert(m.values().get_allocator().resource() == &mr2);
+
+    // mo is unchanged
+    assert((mo == M{{1, 1}, {2, 2}, {3, 3}}));
+    assert(mo.keys().get_allocator().resource() == &mr1);
+  }
+  {
+    // flat_map(const flat_map& m);
+    using C = test_less<int>;
+    std::pmr::monotonic_buffer_resource mr;
+    using M = std::flat_map<int, int, C, std::pmr::vector<int>, std::pmr::vector<int>>;
+    auto mo = M({{1, 1}, {2, 2}, {3, 3}}, C(5), &mr);
+    auto m  = mo;
+
+    assert(m.key_comp() == C(5));
+    assert((m == M{{1, 1}, {2, 2}, {3, 3}}));
+    auto [ks, vs] = std::move(m).extract();
+    assert(ks.get_allocator().resource() == std::pmr::get_default_resource());
+    assert(vs.get_allocator().resource() == std::pmr::get_default_resource());
+
+    // mo is unchanged
+    assert(mo.key_comp() == C(5));
+    assert((mo == M{{1, 1}, {2, 2}, {3, 3}}));
+    auto [kso, vso] = std::move(mo).extract();
+    assert(kso.get_allocator().resource() == &mr);
+    assert(vso.get_allocator().resource() == &mr);
+  }
+  {
+    //  flat_map(initializer_list<value_type> il, const Alloc& a);
+    using M = std::flat_map<int, int, std::less<int>, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    std::initializer_list<M::value_type> il = {{3, 3}, {1, 1}, {4, 4}, {1, 1}, {5, 5}};
+    vm.emplace_back(il);
+    assert((vm[0] == M{{1, 1}, {3, 3}, {4, 4}, {5, 5}}));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    //  flat_map(initializer_list<value_type> il, const key_compare& comp, const Alloc& a);
+    using C = test_less<int>;
+    using M = std::flat_map<int, int, C, std::pmr::vector<int>, std::pmr::deque<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    std::initializer_list<M::value_type> il = {{3, 3}, {1, 1}, {4, 4}, {1, 1}, {5, 5}};
+    vm.emplace_back(il, C(5));
+    assert((vm[0] == M{{1, 1}, {3, 3}, {4, 4}, {5, 5}}));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+    assert(vm[0].key_comp() == C(5));
+  }
+  {
+    // flat_map(InputIterator first, InputIterator last, const Allocator& a);
+    using P      = std::pair<int, short>;
+    P ar[]       = {{1, 1}, {1, 2}, {1, 3}, {2, 4}, {2, 5}, {3, 6}, {2, 7}, {3, 8}, {3, 9}};
+    P expected[] = {{1, 1}, {2, 4}, {3, 6}};
+    {
+      //  cpp17 iterator
+      using M = std::flat_map<int, short, std::less<int>, std::pmr::vector<int>, std::pmr::vector<short>>;
+      std::pmr::monotonic_buffer_resource mr;
+      std::pmr::vector<M> vm(&mr);
+      vm.emplace_back(cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 9));
+      assert(std::ranges::equal(vm[0].keys(), expected | std::views::elements<0>));
+      LIBCPP_ASSERT(std::ranges::equal(vm[0], expected));
+      assert(vm[0].keys().get_allocator().resource() == &mr);
+      assert(vm[0].values().get_allocator().resource() == &mr);
+    }
+    {
+      using M = std::flat_map<int, short, std::less<int>, std::pmr::vector<int>, std::pmr::vector<short>>;
+      std::pmr::monotonic_buffer_resource mr;
+      std::pmr::vector<M> vm(&mr);
+      vm.emplace_back(ar, ar);
+      assert(vm[0].empty());
+      assert(vm[0].keys().get_allocator().resource() == &mr);
+      assert(vm[0].values().get_allocator().resource() == &mr);
+    }
+  }
+  {
+    // flat_map(flat_map&&, const allocator_type&);
+    std::pair<int, int> expected[] = {{1, 1}, {2, 2}, {3, 1}};
+    using C                        = test_less<int>;
+    using M                        = std::flat_map<int, int, C, std::pmr::vector<int>, std::pmr::deque<int>>;
+    std::pmr::monotonic_buffer_resource mr1;
+    std::pmr::monotonic_buffer_resource mr2;
+    M mo = M({{1, 1}, {3, 1}, {1, 1}, {2, 2}}, C(5), &mr1);
+    M m  = {std::move(mo), &mr2}; // also test the implicitness of this constructor
+
+    assert(m.key_comp() == C(5));
+    assert(m.size() == 3);
+    assert(m.keys().get_allocator().resource() == &mr2);
+    assert(m.values().get_allocator().resource() == &mr2);
+    assert(std::equal(m.begin(), m.end(), expected, expected + 3));
+
+    // The original flat_map is moved-from.
+    assert(std::is_sorted(mo.begin(), mo.end(), mo.value_comp()));
+    assert(mo.key_comp() == C(5));
+    assert(mo.keys().get_allocator().resource() == &mr1);
+    assert(mo.values().get_allocator().resource() == &mr1);
+  }
+  {
+    // flat_map(flat_map&&, const allocator_type&);
+    using M = std::flat_map<int, int, std::less<>, std::pmr::deque<int>, std::pmr::vector<int>>;
+    std::pmr::vector<M> vs;
+    M m = {{1, 1}, {3, 1}, {1, 1}, {2, 2}};
+    vs.push_back(std::move(m));
+    assert((vs[0].keys() == std::pmr::deque<int>{1, 2, 3}));
+    assert((vs[0].values() == std::pmr::vector<int>{1, 2, 1}));
+  }
+  {
+    // flat_map& operator=(flat_map&&);
+    using M =
+        std::flat_map<std::pmr::string, int, std::less<>, std::pmr::vector<std::pmr::string>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr1;
+    std::pmr::monotonic_buffer_resource mr2;
+    M mo = M({{"short", 1},
+              {"very long string that definitely won't fit in the SSO buffer and therefore becomes empty on move", 2}},
+             &mr1);
+    M m  = M({{"don't care", 3}}, &mr2);
+    m    = std::move(mo);
+    assert(m.size() == 2);
+    assert(std::is_sorted(m.begin(), m.end(), m.value_comp()));
+    assert(m.begin()->first.get_allocator().resource() == &mr2);
+
+    assert(std::is_sorted(mo.begin(), mo.end(), mo.value_comp()));
+    mo.insert({"foo", 1});
+    assert(mo.begin()->first.get_allocator().resource() == &mr1);
+  }
+  {
+    //  flat_map(from_range_t, R&&, const Alloc&);
+    using P      = std::pair<int, short>;
+    P ar[]       = {{1, 1}, {1, 2}, {1, 3}, {2, 4}, {2, 5}, {3, 6}, {2, 7}, {3, 8}, {3, 9}};
+    P expected[] = {{1, 1}, {2, 4}, {3, 6}};
+    {
+      // input_range
+      using M    = std::flat_map<int, short, std::less<int>, std::pmr::vector<int>, std::pmr::vector<short>>;
+      using Iter = cpp20_input_iterator<const P*>;
+      using Sent = sentinel_wrapper<Iter>;
+      using R    = std::ranges::subrange<Iter, Sent>;
+      std::pmr::monotonic_buffer_resource mr;
+      std::pmr::vector<M> vm(&mr);
+      vm.emplace_back(std::from_range, R(Iter(ar), Sent(Iter(ar + 9))));
+      assert(std::ranges::equal(vm[0].keys(), expected | std::views::elements<0>));
+      LIBCPP_ASSERT(std::ranges::equal(vm[0], expected));
+      assert(vm[0].keys().get_allocator().resource() == &mr);
+      assert(vm[0].values().get_allocator().resource() == &mr);
+    }
+    {
+      using M = std::flat_map<int, short, std::less<int>, std::pmr::vector<int>, std::pmr::vector<short>>;
+      using R = std::ranges::subrange<const P*>;
+      std::pmr::monotonic_buffer_resource mr;
+      std::pmr::vector<M> vm(&mr);
+      vm.emplace_back(std::from_range, R(ar, ar));
+      assert(vm[0].empty());
+      assert(vm[0].keys().get_allocator().resource() == &mr);
+      assert(vm[0].values().get_allocator().resource() == &mr);
+    }
+  }
+  {
+    // flat_map(sorted_unique_t, const key_container_type& key_cont,
+    //          const mapped_container_type& mapped_cont, const Alloc& a);
+    using M = std::flat_map<int, int, std::less<int>, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    std::pmr::vector<int> ks = {1, 2, 4, 10};
+    std::pmr::vector<int> vs = {4, 3, 2, 1};
+    vm.emplace_back(std::sorted_unique, ks, vs);
+    assert(!ks.empty()); // it was an lvalue above
+    assert(!vs.empty()); // it was an lvalue above
+    assert((vm[0] == M{{1, 4}, {2, 3}, {4, 2}, {10, 1}}));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    // flat_map(sorted_unique_t, const key_container_type& key_cont,
+    //          const mapped_container_type& mapped_cont, const Alloc& a);
+    using M = std::flat_map<int, int, std::less<int>, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    std::pmr::vector<int> ks({1, 2, 4, 10}, &mr);
+    std::pmr::vector<int> vs({4, 3, 2, 1}, &mr);
+    vm.emplace_back(std::sorted_unique, ks, vs);
+    assert((vm[0] == M{{1, 4}, {2, 3}, {4, 2}, {10, 1}}));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    // flat_map(sorted_unique_t, initializer_list<value_type> il, const Alloc& a);
+    // cpp_17
+    using C = test_less<int>;
+    using M = std::flat_map<int, int, C, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    using P = std::pair<int, int>;
+    P ar[]  = {{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    vm.emplace_back(
+        std::sorted_unique, cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 4), C(3));
+    assert((vm[0] == M{{1, 1}, {2, 2}, {4, 4}, {5, 5}}));
+    assert(vm[0].key_comp() == C(3));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    // flat_map(sorted_unique_t, initializer_list<value_type> il, const Alloc& a);
+    using C = test_less<int>;
+    using M = std::flat_map<int, int, C, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    std::pair<int, int> ar[1] = {{42, 42}};
+    vm.emplace_back(std::sorted_unique, ar, ar, C(4));
+    assert(vm[0] == M{});
+    assert(vm[0].key_comp() == C(4));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    // flat_map(InputIterator first, InputIterator last, const Alloc& a);
+    // cpp_17
+    using C = test_less<int>;
+    using M = std::flat_map<int, int, C, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    using P = std::pair<int, int>;
+    P ar[]  = {{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    vm.emplace_back(
+        std::sorted_unique, cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 4), C(3));
+    assert((vm[0] == M{{1, 1}, {2, 2}, {4, 4}, {5, 5}}));
+    assert(vm[0].key_comp() == C(3));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+  {
+    // flat_map(InputIterator first, InputIterator last, const Alloc& a);
+    using C = test_less<int>;
+    using M = std::flat_map<int, int, C, std::pmr::vector<int>, std::pmr::vector<int>>;
+    std::pmr::monotonic_buffer_resource mr;
+    std::pmr::vector<M> vm(&mr);
+    std::pair<int, int> ar[1] = {{42, 42}};
+    vm.emplace_back(std::sorted_unique, ar, ar, C(4));
+    assert(vm[0] == M{});
+    assert(vm[0].key_comp() == C(4));
+    assert(vm[0].keys().get_allocator().resource() == &mr);
+    assert(vm[0].values().get_allocator().resource() == &mr);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/range.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/range.pass.cpp
new file mode 100644
index 000000000000..282cc71f3199
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/range.pass.cpp
@@ -0,0 +1,227 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<container-compatible-range<value_type> R>
+//     flat_map(from_range_t, R&&)
+// template<container-compatible-range<value_type> R>
+//     flat_map(from_range_t, R&&, const key_compare&)
+// template<container-compatible-range<value_type> R, class Alloc>
+//      flat_map(from_range_t, R&&, const Alloc&);
+// template<container-compatible-range<value_type> R, class Alloc>
+//      flat_map(from_range_t, R&&, const key_compare&, const Alloc&);
+
+#include <algorithm>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <string>
+#include <vector>
+
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+
+// test constraint container-compatible-range
+
+template <class V>
+using RangeOf = std::ranges::subrange<V*>;
+using Map     = std::flat_map<int, double>;
+
+static_assert(std::is_constructible_v<Map, std::from_range_t, RangeOf<std::pair<int, double>>>);
+static_assert(std::is_constructible_v<Map, std::from_range_t, RangeOf<std::pair<short, double>>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<int>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<double>>);
+
+static_assert(std::is_constructible_v<Map, std::from_range_t, RangeOf<std::pair<int, double>>, std::less<int>>);
+static_assert(std::is_constructible_v<Map, std::from_range_t, RangeOf<std::pair<short, double>>, std::less<int>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<int>, std::less<int>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<double>, std::less<int>>);
+
+static_assert(std::is_constructible_v<Map, std::from_range_t, RangeOf<std::pair<int, double>>, std::allocator<int>>);
+static_assert(std::is_constructible_v<Map, std::from_range_t, RangeOf<std::pair<short, double>>, std::allocator<int>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<int>, std::allocator<int>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<double>, std::allocator<int>>);
+
+static_assert(std::is_constructible_v<Map,
+                                      std::from_range_t,
+                                      RangeOf<std::pair<int, double>>,
+                                      std::less<int>,
+                                      std::allocator<int>>);
+static_assert(std::is_constructible_v<Map,
+                                      std::from_range_t,
+                                      RangeOf<std::pair<short, double>>,
+                                      std::less<int>,
+                                      std::allocator<int>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<int>, std::less<int>, std::allocator<int>>);
+static_assert(!std::is_constructible_v<Map, std::from_range_t, RangeOf<double>, std::less<int>, std::allocator<int>>);
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_map<int, int, C, V1, V1>;
+    using M2 = std::flat_map<int, int, C, V1, V2>;
+    using M3 = std::flat_map<int, int, C, V2, V1>;
+    static_assert(std::is_constructible_v<M1, std::from_range_t, M1, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::from_range_t, M1, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::from_range_t, M2, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::from_range_t, M3, const A2&>);
+
+    static_assert(std::is_constructible_v<M1, std::from_range_t, M1, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::from_range_t, M1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::from_range_t, M2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::from_range_t, M3, const C&, const A2&>);
+  }
+  {
+    // container-compatible-range
+    using C           = test_less<int>;
+    using A1          = test_allocator<int>;
+    using A2          = test_allocator<std::string>;
+    using M           = std::flat_map<int, std::string, C, std::vector<int, A1>, std::vector<std::string, A2>>;
+    using Pair        = std::pair<int, std::string>;
+    using PairLike    = std::tuple<int, std::string>;
+    using NonPairLike = int;
+
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<Pair>&>);
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<PairLike>&>);
+    static_assert(!std::is_constructible_v<M, std::from_range_t, std::vector<NonPairLike>&>);
+
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<Pair>&, const C&>);
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<PairLike>&, const C&>);
+    static_assert(!std::is_constructible_v<M, std::from_range_t, std::vector<NonPairLike>&, const C&>);
+
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<Pair>&, const A1&>);
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<PairLike>&, const A1&>);
+    static_assert(!std::is_constructible_v<M, std::from_range_t, std::vector<NonPairLike>&, const A1&>);
+
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<Pair>&, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M, std::from_range_t, std::vector<PairLike>&, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M, std::from_range_t, std::vector<NonPairLike>&, const C&, const A1&>);
+  }
+
+  using P      = std::pair<int, short>;
+  P ar[]       = {{1, 1}, {1, 2}, {1, 3}, {2, 4}, {2, 5}, {3, 6}, {2, 7}, {3, 8}, {3, 9}};
+  P expected[] = {{1, 1}, {2, 4}, {3, 6}};
+  {
+    // flat_map(from_range_t, R&&)
+    // input_range && !common
+    using M    = std::flat_map<int, short>;
+    using Iter = cpp20_input_iterator<const P*>;
+    using Sent = sentinel_wrapper<Iter>;
+    using R    = std::ranges::subrange<Iter, Sent>;
+    auto m     = M(std::from_range, R(Iter(ar), Sent(Iter(ar + 9))));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+
+    // explicit(false)
+    M m2 = {std::from_range, R(Iter(ar), Sent(Iter(ar + 9)))};
+    assert(m2 == m);
+  }
+  {
+    // flat_map(from_range_t, R&&)
+    // greater
+    using M    = std::flat_map<int, short, std::greater<int>, std::deque<int, min_allocator<int>>, std::deque<short>>;
+    using Iter = cpp20_input_iterator<const P*>;
+    using Sent = sentinel_wrapper<Iter>;
+    using R    = std::ranges::subrange<Iter, Sent>;
+    auto m     = M(std::from_range, R(Iter(ar), Sent(Iter(ar + 9))));
+    assert((m.keys() == std::deque<int, min_allocator<int>>{3, 2, 1}));
+    LIBCPP_ASSERT((m.values() == std::deque<short>{6, 4, 1}));
+  }
+  {
+    // flat_map(from_range_t, R&&)
+    // contiguous range
+    using M = std::flat_map<int, short>;
+    using R = std::ranges::subrange<const P*>;
+    auto m  = M(std::from_range, R(ar, ar + 9));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+  }
+  {
+    // flat_map(from_range_t, R&&, const key_compare&)
+    using C = test_less<int>;
+    using M = std::flat_map<int, short, C, std::vector<int>, std::deque<short>>;
+    using R = std::ranges::subrange<const P*>;
+    auto m  = M(std::from_range, R(ar, ar + 9), C(3));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.key_comp() == C(3));
+
+    // explicit(false)
+    M m2 = {std::from_range, R(ar, ar + 9), C(3)};
+    assert(m2 == m);
+    assert(m2.key_comp() == C(3));
+  }
+  {
+    // flat_map(from_range_t, R&&, const Allocator&)
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_map<int, short, std::less<int>, std::vector<int, A1>, std::deque<short, A2>>;
+    using R  = std::ranges::subrange<const P*>;
+    auto m   = M(std::from_range, R(ar, ar + 9), A1(5));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_map(from_range_t, R&&, const Allocator&)
+    // explicit(false)
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_map<int, short, std::less<int>, std::vector<int, A1>, std::deque<short, A2>>;
+    using R  = std::ranges::subrange<const P*>;
+    M m      = {std::from_range, R(ar, ar + 9), A1(5)}; // implicit ctor
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_map(from_range_t, R&&, const key_compare&, const Allocator&)
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_map<int, short, C, std::vector<int, A1>, std::deque<short, A2>>;
+    using R  = std::ranges::subrange<const P*>;
+    auto m   = M(std::from_range, R(ar, ar + 9), C(3), A1(5));
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.key_comp() == C(3));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_map(from_range_t, R&&, const key_compare&, const Allocator&)
+    // explicit(false)
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_map<int, short, std::less<int>, std::deque<int, A1>, std::vector<short, A2>>;
+    using R  = std::ranges::subrange<const P*>;
+    M m      = {std::from_range, R(ar, ar + 9), {}, A2(5)}; // implicit ctor
+    assert(std::ranges::equal(m.keys(), expected | std::views::elements<0>));
+    LIBCPP_ASSERT(std::ranges::equal(m, expected));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/sorted_container.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/sorted_container.pass.cpp
new file mode 100644
index 000000000000..3c8868f2ff42
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/sorted_container.pass.cpp
@@ -0,0 +1,165 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map(sorted_unique_t, key_container_type key_cont, mapped_container_type mapped_cont,
+//          const key_compare& comp = key_compare());
+//
+// template<class Alloc>
+//   flat_map(sorted_unique_t, const key_container_type& key_cont,
+//            const mapped_container_type& mapped_cont, const Alloc& a);
+// template<class Alloc>
+//   flat_map(sorted_unique_t, const key_container_type& key_cont,
+//            const mapped_container_type& mapped_cont,
+//            const key_compare& comp, const Alloc& a);
+
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "min_allocator.h"
+#include "MoveOnly.h"
+#include "test_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_map<int, int, C, V1, V1>;
+    using M2 = std::flat_map<int, int, C, V1, V2>;
+    using M3 = std::flat_map<int, int, C, V2, V1>;
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const V1&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const V1&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, const V1&, const V2&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::sorted_unique_t, const V2&, const V1&, const A2&>);
+
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const V1&, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const V1&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, const V1&, const V2&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::sorted_unique_t, const V2&, const V1&, const C&, const A2&>);
+  }
+  {
+    // flat_map(sorted_unique_t, key_container_type , mapped_container_type)
+    using M              = std::flat_map<int, char>;
+    std::vector<int> ks  = {1, 2, 4, 10};
+    std::vector<char> vs = {4, 3, 2, 1};
+    auto ks2             = ks;
+    auto vs2             = vs;
+
+    auto m = M(std::sorted_unique, ks, vs);
+    assert((m == M{{1, 4}, {2, 3}, {4, 2}, {10, 1}}));
+    m = M(std::sorted_unique, std::move(ks), std::move(vs));
+    assert(ks.empty()); // it was moved-from
+    assert(vs.empty()); // it was moved-from
+    assert((m == M{{1, 4}, {2, 3}, {4, 2}, {10, 1}}));
+
+    // explicit(false)
+    M m2 = {std::sorted_unique, std::move(ks2), std::move(vs2)};
+    assert(m == m2);
+  }
+  {
+    // flat_map(sorted_unique_t, key_container_type , mapped_container_type)
+    // non-default container, comparator and allocator type
+    using Ks = std::deque<int, min_allocator<int>>;
+    using Vs = std::deque<char, min_allocator<char>>;
+    using M  = std::flat_map<int, char, std::greater<int>, Ks, Vs>;
+    Ks ks    = {10, 4, 2, 1};
+    Vs vs    = {1, 2, 3, 4};
+    auto m   = M(std::sorted_unique, ks, vs);
+    assert((m == M{{1, 4}, {2, 3}, {4, 2}, {10, 1}}));
+    m = M(std::sorted_unique, std::move(ks), std::move(vs));
+    assert(ks.empty()); // it was moved-from
+    assert(vs.empty()); // it was moved-from
+    assert((m == M{{1, 4}, {2, 3}, {4, 2}, {10, 1}}));
+  }
+  {
+    // flat_map(sorted_unique_t, key_container_type , mapped_container_type)
+    // allocator copied into the containers
+    using A = test_allocator<int>;
+    using M = std::flat_map<int, int, std::less<int>, std::vector<int, A>, std::deque<int, A>>;
+    auto ks = std::vector<int, A>({1, 2, 4, 10}, A(4));
+    auto vs = std::deque<int, A>({4, 3, 2, 1}, A(5));
+    auto m  = M(std::sorted_unique, std::move(ks), std::move(vs));
+    assert(ks.empty()); // it was moved-from
+    assert(vs.empty()); // it was moved-from
+    assert((m == M{{1, 4}, {2, 3}, {4, 2}, {10, 1}}));
+    assert(m.keys().get_allocator() == A(4));
+    assert(m.values().get_allocator() == A(5));
+  }
+  {
+    // flat_map(sorted_unique_t, key_container_type , mapped_container_type, key_compare)
+    using C              = test_less<int>;
+    using M              = std::flat_map<int, char, C>;
+    std::vector<int> ks  = {1, 2, 4, 10};
+    std::vector<char> vs = {4, 3, 2, 1};
+
+    auto m = M(std::sorted_unique, ks, vs, C(4));
+    assert((m == M{{1, 4}, {2, 3}, {4, 2}, {10, 1}}));
+    assert(m.key_comp() == C(4));
+
+    // explicit(false)
+    M m2 = {std::sorted_unique, ks, vs, C(4)};
+    assert(m2 == m);
+    assert(m2.key_comp() == C(4));
+  }
+  {
+    // flat_map(sorted_unique_t, key_container_type , mapped_container_type, key_compare, const Allocator&)
+    using C                = test_less<int>;
+    using A                = test_allocator<int>;
+    using M                = std::flat_map<int, int, C, std::vector<int, A>, std::vector<int, A>>;
+    std::vector<int, A> ks = {1, 2, 4, 10};
+    std::vector<int, A> vs = {4, 3, 2, 1};
+    auto m                 = M(std::sorted_unique, ks, vs, C(4), A(5));
+    assert((m == M{{1, 4}, {2, 3}, {4, 2}, {10, 1}}));
+    assert(m.key_comp() == C(4));
+    assert(m.keys().get_allocator() == A(5));
+    assert(m.values().get_allocator() == A(5));
+
+    // explicit(false)
+    M m2 = {ks, vs, C(4), A(5)};
+    assert(m2 == m);
+    assert(m2.key_comp() == C(4));
+    assert(m2.keys().get_allocator() == A(5));
+    assert(m2.values().get_allocator() == A(5));
+  }
+  {
+    // flat_map(sorted_unique_t, key_container_type , mapped_container_type, const Allocator&)
+    using A = test_allocator<int>;
+    using M = std::flat_map<int, int, std::less<int>, std::vector<int, A>, std::deque<int, A>>;
+    auto ks = std::vector<int, A>({1, 2, 4, 10}, A(4));
+    auto vs = std::deque<int, A>({4, 3, 2, 1}, A(5));
+    auto m  = M(std::sorted_unique, ks, vs, A(6)); // replaces the allocators
+    assert(!ks.empty());                           // it was an lvalue above
+    assert(!vs.empty());                           // it was an lvalue above
+    assert((m == M{{1, 4}, {2, 3}, {4, 2}, {10, 1}}));
+    assert(m.keys().get_allocator() == A(6));
+    assert(m.values().get_allocator() == A(6));
+
+    // explicit(false)
+    M m2 = {std::sorted_unique, ks, vs, A(6)};
+    assert(m2 == m);
+    assert(m2.keys().get_allocator() == A(6));
+    assert(m2.values().get_allocator() == A(6));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/sorted_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/sorted_initializer_list.pass.cpp
new file mode 100644
index 000000000000..26452472ba20
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/sorted_initializer_list.pass.cpp
@@ -0,0 +1,179 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template <class InputIterator>
+//   flat_map(sorted_unique_t s, initializer_list<value_type> il,
+//            const key_compare& comp = key_compare())
+// template<class Alloc>
+//   flat_map(sorted_unique_t, initializer_list<value_type> il, const Alloc& a);
+// template<class Alloc>
+//   flat_map(sorted_unique_t, initializer_list<value_type> il,
+//            const key_compare& comp, const Alloc& a);
+
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+
+template <class T, class U>
+std::initializer_list<std::pair<T, U>> il = {{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+
+const auto il1 = il<int, int>;
+const auto il2 = il<int, short>;
+const auto il3 = il<short, int>;
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_map<int, int, C, V1, V1>;
+    using M2 = std::flat_map<int, int, C, V1, V2>;
+    using M3 = std::flat_map<int, int, C, V2, V1>;
+    using IL = std::initializer_list<std::pair<int, int>>;
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, IL, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::sorted_unique_t, IL, const A2&>);
+
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, IL, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::sorted_unique_t, IL, const C&, const A2&>);
+  }
+  {
+    // initializer_list<value_type> needs to match exactly
+    using M = std::flat_map<int, short>;
+    using C = typename M::key_compare;
+    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<std::pair<int, short>>>);
+    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<std::pair<int, short>>, C>);
+    static_assert(std::is_constructible_v<M,
+                                          std::sorted_unique_t,
+                                          std::initializer_list<std::pair<int, short>>,
+                                          C,
+                                          std::allocator<int>>);
+    static_assert(std::is_constructible_v<M,
+                                          std::sorted_unique_t,
+                                          std::initializer_list<std::pair<int, short>>,
+                                          std::allocator<int>>);
+    static_assert(
+        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<std::pair<const int, short>>>);
+    static_assert(
+        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<std::pair<const int, short>>, C>);
+    static_assert(!std::is_constructible_v<M,
+                                           std::sorted_unique_t,
+                                           std::initializer_list<std::pair<const int, short>>,
+                                           C,
+                                           std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M,
+                                           std::sorted_unique_t,
+                                           std::initializer_list<std::pair<const int, short>>,
+                                           std::allocator<int>>);
+    static_assert(
+        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<std::pair<const int, const short>>>);
+    static_assert(
+        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<std::pair<const int, const short>>, C>);
+    static_assert(!std::is_constructible_v<M,
+                                           std::sorted_unique_t,
+                                           std::initializer_list<std::pair<const int, const short>>,
+                                           C,
+                                           std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M,
+                                           std::sorted_unique_t,
+                                           std::initializer_list<std::pair<const int, const short>>,
+                                           std::allocator<int>>);
+  }
+
+  {
+    // flat_map(sorted_unique_t, initializer_list<value_type>);
+    using M       = std::flat_map<int, int>;
+    auto m        = M(std::sorted_unique, il1);
+    auto expected = M{{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    assert(m == expected);
+
+    // explicit(false)
+    M m2 = {std::sorted_unique, il1};
+    assert(m2 == m);
+  }
+  {
+    // flat_map(sorted_unique_t, initializer_list<value_type>, const key_compare&);
+    using M = std::flat_map<int, int, std::function<bool(int, int)>>;
+    auto m  = M(std::sorted_unique, il1, std::less<int>());
+    assert(m == M({{1, 1}, {2, 2}, {4, 4}, {5, 5}}, std::less<>()));
+    assert(m.key_comp()(1, 2) == true);
+
+    // explicit(false)
+    M m2 = {std::sorted_unique, il1, std::less<int>()};
+    assert(m2 == m);
+  }
+  {
+    // flat_map(sorted_unique_t, initializer_list<value_type>, const key_compare&);
+    // greater
+    using M = std::flat_map<int, int, std::greater<int>, std::deque<int, min_allocator<int>>, std::vector<int>>;
+    std::initializer_list<std::pair<int, int>> il4{{5, 5}, {4, 4}, {2, 2}, {1, 1}};
+    auto m = M(std::sorted_unique, il4, std::greater<int>());
+    assert((m == M{{5, 5}, {4, 4}, {2, 2}, {1, 1}}));
+  }
+  {
+    // flat_map(sorted_unique_t, initializer_list<value_type>,  const Allocator&)
+    using A1      = test_allocator<int>;
+    using A2      = test_allocator<short>;
+    using M       = std::flat_map<int, short, std::less<int>, std::vector<int, A1>, std::deque<short, A2>>;
+    auto m        = M(std::sorted_unique, il2, A1(5));
+    auto expected = M{{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    assert(m == expected);
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+
+    // explicit(false)
+    M m2 = {std::sorted_unique, il2, A1(5)};
+    assert(m2 == m);
+    assert(m2.keys().get_allocator() == A1(5));
+    assert(m2.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_map(sorted_unique_t, initializer_list<value_type>, const key_compare&, const Allocator&);
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_map<int, short, C, std::vector<int, A1>, std::deque<short, A2>>;
+    auto m   = M(std::sorted_unique, il2, C(3), A1(5));
+    assert((m == M{{1, 1}, {2, 2}, {4, 4}, {5, 5}}));
+    assert(m.key_comp() == C(3));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_map(sorted_unique_t, initializer_list<value_type>, const key_compare&, const Allocator&);
+    // explicit(false)
+    using A1 = test_allocator<short>;
+    using A2 = test_allocator<int>;
+    using M  = std::flat_map<short, int, std::less<int>, std::deque<short, A1>, std::vector<int, A2>>;
+    M m      = {std::sorted_unique, il3, {}, A1(5)}; // implicit ctor
+    assert((m == M{{1, 1}, {2, 2}, {4, 4}, {5, 5}}));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/sorted_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/sorted_iter_iter.pass.cpp
new file mode 100644
index 000000000000..8eb7547e917c
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/sorted_iter_iter.pass.cpp
@@ -0,0 +1,171 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template <class InputIterator>
+//   flat_map(sorted_unique_t, InputIterator first, InputIterator last, const key_compare& comp = key_compare());
+// template<class InputIterator, class Alloc>
+//   flat_map(InputIterator first, InputIterator last, const Alloc& a);
+// template<class InputIterator, class Allocator>
+//   flat_map(sorted_unique_t, InputIterator first, InputIterator last, const key_compare& comp, const Allocator& a);
+
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "../../../test_compare.h"
+
+int main(int, char**) {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<key_container_type, Alloc> is true
+    // and uses_allocator_v<mapped_container_type, Alloc> is true.
+    using C     = test_less<int>;
+    using A1    = test_allocator<int>;
+    using A2    = other_allocator<int>;
+    using V1    = std::vector<int, A1>;
+    using V2    = std::vector<int, A2>;
+    using M1    = std::flat_map<int, int, C, V1, V1>;
+    using M2    = std::flat_map<int, int, C, V1, V2>;
+    using M3    = std::flat_map<int, int, C, V2, V1>;
+    using Iter1 = typename M1::iterator;
+    using Iter2 = typename M2::iterator;
+    using Iter3 = typename M3::iterator;
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::sorted_unique_t, Iter3, Iter3, const A2&>);
+
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const C&, const A1&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M3, std::sorted_unique_t, Iter3, Iter3, const C&, const A2&>);
+  }
+  {
+    // flat_map(sorted_unique_t, InputIterator, InputIterator);
+    // cpp17_input_iterator
+    using M       = std::flat_map<int, int>;
+    using P       = std::pair<int, int>;
+    P ar[]        = {{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    auto m        = M(std::sorted_unique, cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 4));
+    auto expected = M{{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    assert(m == expected);
+
+    // explicit(false)
+    M m2 = {std::sorted_unique, cpp17_input_iterator<const P*>(ar), cpp17_input_iterator<const P*>(ar + 4)};
+    assert(m2 == m);
+  }
+  {
+    // flat_map(sorted_unique_t, InputIterator, InputIterator);
+    // contiguous iterator
+    using C = test_less<int>;
+    using M = std::flat_map<int, int, C, std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>;
+    std::pair<int, int> ar[] = {{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    auto m                   = M(std::sorted_unique, ar, ar + 4);
+    auto expected            = M{{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    assert(m == expected);
+  }
+  {
+    // flat_map(sorted_unique_t, InputIterator, InputIterator, const key_compare&);
+    // cpp_17_input_iterator
+    using M = std::flat_map<int, int, std::function<bool(int, int)>>;
+    using P = std::pair<int, int>;
+    P ar[]  = {{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    auto m  = M(std::sorted_unique,
+               cpp17_input_iterator<const P*>(ar),
+               cpp17_input_iterator<const P*>(ar + 4),
+               std::less<int>());
+    assert(m == M({{1, 1}, {2, 2}, {4, 4}, {5, 5}}, std::less<>()));
+    assert(m.key_comp()(1, 2) == true);
+
+    // explicit(false)
+    M m2 = {std::sorted_unique,
+            cpp17_input_iterator<const P*>(ar),
+            cpp17_input_iterator<const P*>(ar + 4),
+            std::less<int>()};
+    assert(m2 == m);
+  }
+  {
+    // flat_map(sorted_unique_t, InputIterator, InputIterator, const key_compare&);
+    // greater
+    using M = std::flat_map<int, int, std::greater<int>, std::deque<int, min_allocator<int>>, std::vector<int>>;
+    using P = std::pair<int, int>;
+    P ar[]  = {{5, 5}, {4, 4}, {2, 2}, {1, 1}};
+    auto m  = M(std::sorted_unique,
+               cpp17_input_iterator<const P*>(ar),
+               cpp17_input_iterator<const P*>(ar + 4),
+               std::greater<int>());
+    assert((m == M{{5, 5}, {4, 4}, {2, 2}, {1, 1}}));
+  }
+  {
+    // flat_map(sorted_unique_t, InputIterator, InputIterator, const key_compare&);
+    // contiguous iterator
+    using C = test_less<int>;
+    using M = std::flat_map<int, int, C, std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>;
+    std::pair<int, int> ar[1] = {{42, 42}};
+    auto m                    = M(std::sorted_unique, ar, ar, C(5));
+    assert(m.empty());
+    assert(m.key_comp() == C(5));
+  }
+  {
+    // flat_map(sorted_unique_t, InputIterator , InputIterator, const Allocator&)
+    using A1      = test_allocator<int>;
+    using A2      = test_allocator<short>;
+    using M       = std::flat_map<int, short, std::less<int>, std::vector<int, A1>, std::deque<short, A2>>;
+    using P       = std::pair<int, int>;
+    P ar[]        = {{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    auto m        = M(std::sorted_unique, ar, ar + 4, A1(5));
+    auto expected = M{{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    assert(m == expected);
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+
+    // explicit(false)
+    M m2 = {std::sorted_unique, ar, ar + 4, A1(5)};
+    assert(m2 == m);
+    assert(m2.keys().get_allocator() == A1(5));
+    assert(m2.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_map(sorted_unique_t, InputIterator, InputIterator, const key_compare&, const Allocator&);
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = test_allocator<short>;
+    using M  = std::flat_map<int, short, C, std::vector<int, A1>, std::deque<short, A2>>;
+    using P  = std::pair<int, int>;
+    P ar[]   = {{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    auto m   = M(std::sorted_unique, ar, ar + 4, C(3), A1(5));
+    assert((m == M{{1, 1}, {2, 2}, {4, 4}, {5, 5}}));
+    assert(m.key_comp() == C(3));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+  {
+    // flat_map(sorted_unique_t, InputIterator, InputIterator, const key_compare&, const Allocator&);
+    // explicit(false)
+    using A1 = test_allocator<short>;
+    using A2 = test_allocator<int>;
+    using M  = std::flat_map<short, int, std::less<int>, std::deque<short, A1>, std::vector<int, A2>>;
+    using P  = std::pair<int, int>;
+    P ar[]   = {{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    M m      = {std::sorted_unique, ar, ar + 4, {}, A1(5)}; // implicit ctor
+    assert((m == M{{1, 1}, {2, 2}, {4, 4}, {5, 5}}));
+    assert(m.keys().get_allocator() == A1(5));
+    assert(m.values().get_allocator() == A2(5));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.erasure/erase_if.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.erasure/erase_if.pass.cpp
new file mode 100644
index 000000000000..fb0563eec537
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.erasure/erase_if.pass.cpp
@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<class Key, class T, class Compare, class KeyContainer, class MappedContainer, class Predicate>
+//   typename flat_map<Key, T, Compare, KeyContainer, MappedContainer>::size_type
+//   erase_if(flat_map<Key, T, Compare, KeyContainer, MappedContainer>& c, Predicate pred);
+
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <initializer_list>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_allocator.h"
+#include "min_allocator.h"
+
+// Verify that `flat_map` (like `map`) does NOT support std::erase.
+//
+template <class S>
+concept HasStdErase = requires(S& s, typename S::value_type x) { std::erase(s, x); };
+static_assert(HasStdErase<std::vector<int>>);
+static_assert(!HasStdErase<std::flat_map<int, int>>);
+
+template <class M>
+M make(std::initializer_list<int> vals) {
+  M ret;
+  for (int v : vals)
+    ret[static_cast<typename M::key_type>(v)] = static_cast<typename M::mapped_type>(v + 10);
+  return ret;
+}
+
+template <class M, class Pred>
+void test0(
+    std::initializer_list<int> vals, Pred p, std::initializer_list<int> expected, std::size_t expected_erased_count) {
+  M s = make<M>(vals);
+  ASSERT_SAME_TYPE(typename M::size_type, decltype(std::erase_if(s, p)));
+  assert(expected_erased_count == std::erase_if(s, p));
+  assert(s == make<M>(expected));
+}
+
+template <class S>
+void test() {
+  // Test all the plausible signatures for this predicate.
+  auto is1   = [](typename S::const_reference v) { return v.first == 1; };
+  auto is2   = [](typename S::value_type v) { return v.first == 2; };
+  auto is3   = [](const typename S::value_type& v) { return v.first == 3; };
+  auto is4   = [](auto v) { return v.first == 4; };
+  auto True  = [](const auto&) { return true; };
+  auto False = [](auto&&) { return false; };
+
+  test0<S>({}, is1, {}, 0);
+
+  test0<S>({1}, is1, {}, 1);
+  test0<S>({1}, is2, {1}, 0);
+
+  test0<S>({1, 2}, is1, {2}, 1);
+  test0<S>({1, 2}, is2, {1}, 1);
+  test0<S>({1, 2}, is3, {1, 2}, 0);
+
+  test0<S>({1, 2, 3}, is1, {2, 3}, 1);
+  test0<S>({1, 2, 3}, is2, {1, 3}, 1);
+  test0<S>({1, 2, 3}, is3, {1, 2}, 1);
+  test0<S>({1, 2, 3}, is4, {1, 2, 3}, 0);
+
+  test0<S>({1, 2, 3}, True, {}, 3);
+  test0<S>({1, 2, 3}, False, {1, 2, 3}, 0);
+}
+
+int main(int, char**) {
+  test<std::flat_map<int, char>>();
+  test<std::flat_map<int,
+                     char,
+                     std::less<int>,
+                     std::vector<int, min_allocator<int>>,
+                     std::vector<char, min_allocator<char>>>>();
+  test<std::flat_map<int, char, std::greater<int>, std::vector<int, test_allocator<int>>>>();
+  test<std::flat_map<int, char, std::less<int>, std::deque<int, min_allocator<int>>>>();
+  test<std::flat_map<int, char, std::greater<int>, std::deque<int, test_allocator<int>>>>();
+  test<std::flat_map<long, int>>();
+  test<std::flat_map<double, int>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.erasure/erase_if_exceptions.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.erasure/erase_if_exceptions.pass.cpp
new file mode 100644
index 000000000000..48fdec42db3f
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.erasure/erase_if_exceptions.pass.cpp
@@ -0,0 +1,155 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: no-exceptions
+
+// <flat_map>
+
+// template<class Key, class T, class Compare, class KeyContainer, class MappedContainer, class Predicate>
+//   typename flat_map<Key, T, Compare, KeyContainer, MappedContainer>::size_type
+//   erase_if(flat_map<Key, T, Compare, KeyContainer, MappedContainer>& c, Predicate pred);
+// If any member function in [flat.set.defn] exits via an exception, the invariant is restored.
+// (This is not a member function, but let's respect the invariant anyway.)
+
+#include <algorithm>
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "../helpers.h"
+#include "test_macros.h"
+
+struct Counter {
+  int c1, c2, throws;
+  void tick() {
+    c1 -= 1;
+    if (c1 == 0) {
+      c1 = c2;
+      throws += 1;
+      throw 42;
+    }
+  }
+};
+Counter g_counter = {0, 0, 0};
+
+struct ThrowingAssignment {
+  ThrowingAssignment(int i) : i_(i) {}
+  ThrowingAssignment(const ThrowingAssignment&) = default;
+  ThrowingAssignment& operator=(const ThrowingAssignment& rhs) {
+    g_counter.tick();
+    i_ = rhs.i_;
+    g_counter.tick();
+    return *this;
+  }
+  operator int() const { return i_; }
+  int i_;
+};
+
+struct ThrowingComparator {
+  bool operator()(const ThrowingAssignment& a, const ThrowingAssignment& b) const {
+    g_counter.tick();
+    return a.i_ < b.i_;
+  }
+};
+
+struct ErasurePredicate {
+  bool operator()(const auto& x) const { return (3 <= x.first && x.first <= 5); }
+};
+
+int main(int, char**) {
+  const std::pair<int, int> expected[] = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}, {7, 7}, {8, 8}};
+  {
+    using M = std::flat_map<ThrowingAssignment, int, ThrowingComparator>;
+    for (int first_throw = 1; first_throw < 99; ++first_throw) {
+      for (int second_throw = 1; second_throw < 99; ++second_throw) {
+        g_counter = {0, 0, 0};
+        M m       = M({1, 2, 3, 4, 5, 6, 7, 8}, {1, 2, 3, 4, 5, 6, 7, 8});
+        try {
+          g_counter = {first_throw, second_throw, 0};
+          auto n    = std::erase_if(m, ErasurePredicate());
+          assert(n == 3);
+          // If it didn't throw at all, we're done.
+          g_counter = {0, 0, 0};
+          assert((m == M{{1, 1}, {2, 2}, {6, 6}, {7, 7}, {8, 8}}));
+          first_throw = 99; // "done"
+          break;
+        } catch (int ex) {
+          assert(ex == 42);
+          check_invariant(m);
+          LIBCPP_ASSERT(m.empty() || std::equal(m.begin(), m.end(), expected, expected + 8));
+          if (g_counter.throws == 1) {
+            // We reached the first throw but not the second throw.
+            break;
+          }
+        }
+      }
+    }
+  }
+  {
+    using M = std::flat_map<int, ThrowingAssignment, ThrowingComparator>;
+    for (int first_throw = 1; first_throw < 99; ++first_throw) {
+      for (int second_throw = 1; second_throw < 99; ++second_throw) {
+        g_counter = {0, 0, 0};
+        M m       = M({1, 2, 3, 4, 5, 6, 7, 8}, {1, 2, 3, 4, 5, 6, 7, 8});
+        try {
+          g_counter = {first_throw, second_throw, 0};
+          auto n    = std::erase_if(m, ErasurePredicate());
+          assert(n == 3);
+          // If it didn't throw at all, we're done.
+          g_counter = {0, 0, 0};
+          assert((m == M{{1, 1}, {2, 2}, {6, 6}, {7, 7}, {8, 8}}));
+          first_throw = 99; // "done"
+          break;
+        } catch (int ex) {
+          assert(ex == 42);
+          check_invariant(m);
+          LIBCPP_ASSERT(m.empty() || std::equal(m.begin(), m.end(), expected, expected + 8));
+          if (g_counter.throws == 1) {
+            // We reached the first throw but not the second throw.
+            break;
+          }
+        }
+      }
+    }
+  }
+  {
+    using M =
+        std::flat_map<ThrowingAssignment, int, ThrowingComparator, std::deque<ThrowingAssignment>, std::deque<int>>;
+    for (int first_throw = 1; first_throw < 99; ++first_throw) {
+      for (int second_throw = 1; second_throw < 99; ++second_throw) {
+        g_counter                                = {0, 0, 0};
+        std::deque<ThrowingAssignment> container = {5, 6, 7, 8};
+        container.insert(container.begin(), {1, 2, 3, 4});
+        M m = M(std::move(container), {1, 2, 3, 4, 5, 6, 7, 8});
+        try {
+          g_counter = {first_throw, second_throw, 0};
+          auto n    = std::erase_if(m, ErasurePredicate());
+          assert(n == 3);
+          // If it didn't throw at all, we're done.
+          g_counter = {0, 0, 0};
+          assert((m == M{{1, 1}, {2, 2}, {6, 6}, {7, 7}, {8, 8}}));
+          first_throw = 99; // "done"
+          break;
+        } catch (int ex) {
+          assert(ex == 42);
+          check_invariant(m);
+          LIBCPP_ASSERT(m.empty() || std::equal(m.begin(), m.end(), expected, expected + 8));
+          if (g_counter.throws == 1) {
+            // We reached the first throw but not the second throw.
+            break;
+          }
+        }
+      }
+    }
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/iterator.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/iterator.pass.cpp
new file mode 100644
index 000000000000..b63ce6b19ee1
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/iterator.pass.cpp
@@ -0,0 +1,96 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+//       iterator begin()   noexcept;
+// const_iterator begin()   const noexcept
+//       iterator end()     noexcept;
+// const_iterator end()     const noexcept;
+//
+// const_iterator cbegin()  const noexcept;
+// const_iterator cend()    const noexcept;
+
+#include <cassert>
+#include <cstddef>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <string>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  M m         = {{1, 'a'}, {2, 'b'}, {3, 'c'}, {4, 'd'}};
+  const M& cm = m;
+  ASSERT_SAME_TYPE(decltype(m.begin()), typename M::iterator);
+  ASSERT_SAME_TYPE(decltype(m.cbegin()), typename M::const_iterator);
+  ASSERT_SAME_TYPE(decltype(cm.begin()), typename M::const_iterator);
+  ASSERT_SAME_TYPE(decltype(m.end()), typename M::iterator);
+  ASSERT_SAME_TYPE(decltype(m.cend()), typename M::const_iterator);
+  ASSERT_SAME_TYPE(decltype(cm.end()), typename M::const_iterator);
+  static_assert(noexcept(m.begin()));
+  static_assert(noexcept(cm.begin()));
+  static_assert(noexcept(m.cbegin()));
+  static_assert(noexcept(m.end()));
+  static_assert(noexcept(cm.end()));
+  static_assert(noexcept(m.cend()));
+  assert(m.size() == 4);
+  assert(std::distance(m.begin(), m.end()) == 4);
+  assert(std::distance(cm.begin(), cm.end()) == 4);
+  assert(std::distance(m.cbegin(), m.cend()) == 4);
+  typename M::iterator i;                   // default-construct
+  i                            = m.begin(); // move-assignment
+  typename M::const_iterator k = i;         // converting constructor
+  assert(i == k);                           // comparison
+  for (int j = 1; j <= 4; ++j, ++i) {       // pre-increment
+    assert(i->first == j);                  // operator->
+    assert(i->second == 'a' + j - 1);
+  }
+  assert(i == m.end());
+  for (int j = 4; j >= 1; --j) {
+    --i; // pre-decrement
+    assert((*i).first == j);
+    assert((*i).second == 'a' + j - 1);
+  }
+  assert(i == m.begin());
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  {
+    // N3644 testing
+    using C = std::flat_map<int, char>;
+    C::iterator ii1{}, ii2{};
+    C::iterator ii4 = ii1;
+    C::const_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
+    assert(!(ii1 != ii2));
+
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/iterator_comparison.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/iterator_comparison.pass.cpp
new file mode 100644
index 000000000000..1975d0ed86cc
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/iterator_comparison.pass.cpp
@@ -0,0 +1,155 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// flat_map iterators should be C++20 random access iterators
+
+#include <compare>
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using KI    = typename KeyContainer::iterator;
+  using I     = M::iterator;
+  using CI    = M::const_iterator;
+  using RI    = M::reverse_iterator;
+  using CRI   = M::const_reverse_iterator;
+
+  static_assert(std::equality_comparable<I>);
+  static_assert(std::equality_comparable<CI>);
+  static_assert(std::equality_comparable<RI>);
+  static_assert(std::equality_comparable<CRI>);
+
+  static_assert(std::totally_ordered<I>);
+  static_assert(std::totally_ordered<CI>);
+  static_assert(std::totally_ordered<RI>);
+  static_assert(std::totally_ordered<CRI>);
+
+  M m = {{1, 'a'}, {2, 'b'}, {3, 'c'}, {4, 'd'}};
+
+  I i1 = m.begin();
+  I i2 = m.begin() + 1;
+
+  assert(i1 == i1);
+  assert(!(i1 != i1));
+  assert(i1 != i2);
+  assert(!(i1 == i2));
+  assert(i1 < i2);
+  assert(!(i1 < i1));
+  assert(i1 <= i1);
+  assert(i1 <= i2);
+  assert(!(i2 <= i1));
+  assert(i2 > i1);
+  assert(!(i2 > i2));
+  assert(i2 >= i1);
+  assert(i2 >= i2);
+  assert(!(i1 >= i2));
+
+  CI ci1 = m.cbegin();
+  CI ci2 = m.cbegin() + 1;
+  assert(ci1 == ci1);
+  assert(!(ci1 != ci1));
+  assert(ci1 != ci2);
+  assert(!(ci1 == ci2));
+  assert(ci1 < ci2);
+  assert(!(ci1 < ci1));
+  assert(ci1 <= ci1);
+  assert(ci1 <= ci2);
+  assert(!(ci2 <= ci1));
+  assert(ci2 > ci1);
+  assert(!(ci2 > ci2));
+  assert(ci2 >= ci1);
+  assert(ci2 >= ci2);
+  assert(!(ci1 >= ci2));
+
+  RI ri1 = m.rbegin();
+  RI ri2 = m.rbegin() + 1;
+  assert(ri1 == ri1);
+  assert(!(ri1 != ri1));
+  assert(ri1 != ri2);
+  assert(!(ri1 == ri2));
+  assert(ri1 < ri2);
+  assert(!(ri1 < ri1));
+  assert(ri1 <= ri1);
+  assert(ri1 <= ri2);
+  assert(!(ri2 <= ri1));
+  assert(ri2 > ri1);
+  assert(!(ri2 > ri2));
+  assert(ri2 >= ri1);
+  assert(ri2 >= ri2);
+  assert(!(ri1 >= ri2));
+
+  CRI cri1 = m.crbegin();
+  CRI cri2 = m.crbegin() + 1;
+  assert(cri1 == cri1);
+  assert(!(cri1 != cri1));
+  assert(cri1 != cri2);
+  assert(!(cri1 == cri2));
+  assert(cri1 < cri2);
+  assert(!(cri1 < cri1));
+  assert(cri1 <= cri1);
+  assert(cri1 <= cri2);
+  assert(!(cri2 <= cri1));
+  assert(cri2 > cri1);
+  assert(!(cri2 > cri2));
+  assert(cri2 >= cri1);
+  assert(cri2 >= cri2);
+  assert(!(cri1 >= cri2));
+
+  if constexpr (std::three_way_comparable<KI>) {
+    static_assert(std::three_way_comparable<I>); // ...of course the wrapped iterators still support <=>.
+    static_assert(std::three_way_comparable<CI>);
+    static_assert(std::three_way_comparable<RI>);
+    static_assert(std::three_way_comparable<CRI>);
+    static_assert(std::same_as<decltype(I() <=> I()), std::strong_ordering>);
+    static_assert(std::same_as<decltype(I() <=> CI()), std::strong_ordering>);
+    static_assert(std::same_as<decltype(CI() <=> CI()), std::strong_ordering>);
+    static_assert(std::same_as<decltype(RI() <=> RI()), std::strong_ordering>);
+    static_assert(std::same_as<decltype(RI() <=> CRI()), std::strong_ordering>);
+    static_assert(std::same_as<decltype(CRI() <=> CRI()), std::strong_ordering>);
+
+    assert(i1 <=> i1 == std::strong_ordering::equivalent);
+    assert(i1 <=> i2 == std::strong_ordering::less);
+    assert(i2 <=> i1 == std::strong_ordering::greater);
+
+    assert(ci1 <=> ci1 == std::strong_ordering::equivalent);
+    assert(ci1 <=> ci2 == std::strong_ordering::less);
+    assert(ci2 <=> ci1 == std::strong_ordering::greater);
+
+    assert(ri1 <=> ri1 == std::strong_ordering::equivalent);
+    assert(ri1 <=> ri2 == std::strong_ordering::less);
+    assert(ri2 <=> ri1 == std::strong_ordering::greater);
+
+    assert(cri1 <=> cri1 == std::strong_ordering::equivalent);
+    assert(cri1 <=> cri2 == std::strong_ordering::less);
+    assert(cri2 <=> cri1 == std::strong_ordering::greater);
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/iterator_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/iterator_concept_conformance.compile.pass.cpp
new file mode 100644
index 000000000000..28814e2e37e3
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/iterator_concept_conformance.compile.pass.cpp
@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// iterator, const_iterator, reverse_iterator, const_reverse_iterator
+
+#include <flat_map>
+#include <deque>
+#include <functional>
+#include <iterator>
+#include <string>
+#include <vector>
+#include <type_traits>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using C     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using I     = C::iterator;
+  using CI    = C::const_iterator;
+  using RI    = C::reverse_iterator;
+  using CRI   = C::const_reverse_iterator;
+  static_assert(std::random_access_iterator<I>);
+  static_assert(std::random_access_iterator<CI>);
+  static_assert(std::random_access_iterator<RI>);
+  static_assert(std::random_access_iterator<CRI>);
+  static_assert(!std::contiguous_iterator<I>);
+  static_assert(!std::contiguous_iterator<CI>);
+  static_assert(!std::contiguous_iterator<RI>);
+  static_assert(!std::contiguous_iterator<CRI>);
+  static_assert(!std::indirectly_writable<I, std::pair<int, char>>);
+  static_assert(!std::indirectly_writable<CI, std::pair<int, char>>);
+  static_assert(!std::indirectly_writable<RI, std::pair<int, char>>);
+  static_assert(!std::indirectly_writable<CRI, std::pair<int, char>>);
+  static_assert(std::sentinel_for<I, I>);
+  static_assert(std::sentinel_for<I, CI>);
+  static_assert(!std::sentinel_for<I, RI>);
+  static_assert(!std::sentinel_for<I, CRI>);
+  static_assert(std::sentinel_for<CI, I>);
+  static_assert(std::sentinel_for<CI, CI>);
+  static_assert(!std::sentinel_for<CI, RI>);
+  static_assert(!std::sentinel_for<CI, CRI>);
+  static_assert(!std::sentinel_for<RI, I>);
+  static_assert(!std::sentinel_for<RI, CI>);
+  static_assert(std::sentinel_for<RI, RI>);
+  static_assert(std::sentinel_for<RI, CRI>);
+  static_assert(!std::sentinel_for<CRI, I>);
+  static_assert(!std::sentinel_for<CRI, CI>);
+  static_assert(std::sentinel_for<CRI, RI>);
+  static_assert(std::sentinel_for<CRI, CRI>);
+  static_assert(std::indirectly_movable_storable<I, std::pair<int, char>*>);
+  static_assert(std::indirectly_movable_storable<CI, std::pair<int, char>*>);
+  static_assert(std::indirectly_movable_storable<RI, std::pair<int, char>*>);
+  static_assert(std::indirectly_movable_storable<CRI, std::pair<int, char>*>);
+
+#ifdef _LIBCPP_VERSION
+  static_assert(std::is_same_v<typename std::iterator_traits<I>::iterator_category, std::random_access_iterator_tag>);
+  static_assert(std::is_same_v<typename std::iterator_traits<CI>::iterator_category, std::random_access_iterator_tag>);
+  static_assert(std::is_same_v<typename std::iterator_traits<RI>::iterator_category, std::random_access_iterator_tag>);
+  static_assert(std::is_same_v<typename std::iterator_traits<CRI>::iterator_category, std::random_access_iterator_tag>);
+#endif
+}
+
+void test() {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/range_concept_conformance.compile.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/range_concept_conformance.compile.pass.cpp
new file mode 100644
index 000000000000..abbad310f49c
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/range_concept_conformance.compile.pass.cpp
@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <ranges>
+#include <string>
+#include <vector>
+#include "MinSequenceContainer.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  {
+    using Key   = typename KeyContainer::value_type;
+    using Value = typename ValueContainer::value_type;
+    using C     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+    static_assert(std::same_as<std::ranges::iterator_t<C>, typename C::iterator>);
+    static_assert(std::ranges::random_access_range<C>);
+    static_assert(!std::ranges::contiguous_range<C>);
+    static_assert(std::ranges::common_range<C>);
+    static_assert(std::ranges::input_range<C>);
+    static_assert(!std::ranges::view<C>);
+    static_assert(std::ranges::sized_range<C>);
+    static_assert(!std::ranges::borrowed_range<C>);
+    static_assert(std::ranges::viewable_range<C>);
+
+    static_assert(std::same_as<std::ranges::iterator_t<const C>, typename C::const_iterator>);
+    static_assert(std::ranges::random_access_range<const C>);
+    static_assert(!std::ranges::contiguous_range<const C>);
+    static_assert(std::ranges::common_range<const C>);
+    static_assert(std::ranges::input_range<const C>);
+    static_assert(!std::ranges::view<const C>);
+    static_assert(std::ranges::sized_range<const C>);
+    static_assert(!std::ranges::borrowed_range<const C>);
+    static_assert(!std::ranges::viewable_range<const C>);
+  }
+}
+
+void test() {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/reverse_iterator.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/reverse_iterator.pass.cpp
new file mode 100644
index 000000000000..09e18986a7e8
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/reverse_iterator.pass.cpp
@@ -0,0 +1,90 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+//       reverse_iterator rbegin() noexcept;
+// const_reverse_iterator rbegin() const noexcept;
+//       reverse_iterator rend()   noexcept;
+// const_reverse_iterator rend()   const noexcept;
+//
+// const_reverse_iterator crbegin() const noexcept;
+// const_reverse_iterator crend()   const noexcept;
+
+#include <cassert>
+#include <cstddef>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <string>
+
+#include <iterator>
+
+#include "test_macros.h"
+#include <iostream>
+
+int main(int, char**) {
+  {
+    using M     = std::flat_map<int, char, std::less<int>, std::deque<int>, std::deque<char>>;
+    M m         = {{1, 'a'}, {2, 'b'}, {3, 'c'}, {4, 'd'}};
+    const M& cm = m;
+    ASSERT_SAME_TYPE(decltype(m.rbegin()), M::reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(m.crbegin()), M::const_reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(cm.rbegin()), M::const_reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(m.rend()), M::reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(m.crend()), M::const_reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(cm.rend()), M::const_reverse_iterator);
+    static_assert(noexcept(m.rbegin()));
+    static_assert(noexcept(cm.rbegin()));
+    static_assert(noexcept(m.crbegin()));
+    static_assert(noexcept(m.rend()));
+    static_assert(noexcept(cm.rend()));
+    static_assert(noexcept(m.crend()));
+    assert(m.size() == 4);
+    assert(std::distance(m.rbegin(), m.rend()) == 4);
+    assert(std::distance(cm.rbegin(), cm.rend()) == 4);
+    assert(std::distance(m.crbegin(), m.crend()) == 4);
+    assert(std::distance(cm.crbegin(), cm.crend()) == 4);
+    M::reverse_iterator i; // default-construct
+    ASSERT_SAME_TYPE(decltype(i->first), const int&);
+    ASSERT_SAME_TYPE(decltype(i->second), char&);
+    i                           = m.rbegin(); // move-assignment
+    M::const_reverse_iterator k = i;          // converting constructor
+    assert(i == k);                           // comparison
+    for (int j = 4; j >= 1; --j, ++i) {       // pre-increment
+      assert(i->first == j);                  // operator->
+      assert(i->second == 'a' + j - 1);
+    }
+    assert(i == m.rend());
+    for (int j = 1; j <= 4; ++j) {
+      --i; // pre-decrement
+      assert((*i).first == j);
+      assert((*i).second == 'a' + j - 1);
+    }
+    assert(i == m.rbegin());
+  }
+  {
+    // N3644 testing
+    using C = std::flat_map<int, char>;
+    C::reverse_iterator ii1{}, ii2{};
+    C::reverse_iterator ii4 = ii1;
+    C::const_reverse_iterator cii{};
+    assert(ii1 == ii2);
+    assert(ii1 == ii4);
+    assert(!(ii1 != ii2));
+
+    assert((ii1 == cii));
+    assert((cii == ii1));
+    assert(!(ii1 != cii));
+    assert(!(cii != ii1));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/clear.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/clear.pass.cpp
new file mode 100644
index 000000000000..30271eb55660
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/clear.pass.cpp
@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_map
+
+// void clear() noexcept;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// test noexcept
+
+template <class T>
+concept NoExceptClear = requires(T t) {
+  { t.clear() } noexcept;
+};
+
+static_assert(NoExceptClear<std::flat_map<int, int>>);
+#ifndef TEST_HAS_NO_EXCEPTIONS
+static_assert(
+    NoExceptClear<std::flat_map<int, int, std::less<int>, ThrowOnMoveContainer<int>, ThrowOnMoveContainer<int>>>);
+#endif
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  M m = {{1, 2}, {2, 1}, {3, 3}, {4, 1}, {5, 0}};
+  assert(m.size() == 5);
+  ASSERT_NOEXCEPT(m.clear());
+  ASSERT_SAME_TYPE(decltype(m.clear()), void);
+  m.clear();
+  assert(m.size() == 0);
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<int>>();
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/emplace.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/emplace.pass.cpp
new file mode 100644
index 000000000000..06631ac689f7
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/emplace.pass.cpp
@@ -0,0 +1,103 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template <class... Args>
+//   pair<iterator, bool> emplace(Args&&... args);
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+#include <tuple>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "../../../Emplaceable.h"
+#include "DefaultOnly.h"
+#include "min_allocator.h"
+
+// Constraints: is_constructible_v<pair<key_type, mapped_type>, Args...> is true.
+template <class M, class... Args>
+concept CanEmplace = requires(M m, Args&&... args) { m.emplace(std::forward<Args>(args)...); };
+
+using Map = std::flat_map<Emplaceable, Emplaceable>;
+static_assert(CanEmplace<Map>);
+static_assert(CanEmplace<Map, Emplaceable, Emplaceable>);
+static_assert(CanEmplace<Map, std::piecewise_construct_t, std::tuple<int, double>, std::tuple<int, double>>);
+static_assert(!CanEmplace<Map, Emplaceable>);
+static_assert(!CanEmplace<Map, int, double>);
+
+template <class KeyContainer, class ValueContainer>
+void test_simple() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using R     = std::pair<typename M::iterator, bool>;
+  M m;
+  ASSERT_SAME_TYPE(decltype(m.emplace()), R);
+  R r = m.emplace(typename M::value_type(2, 3.5));
+  assert(r.second);
+  assert(r.first == m.begin());
+  assert(m.size() == 1);
+  assert(m.begin()->first == 2);
+  assert(m.begin()->second == 3.5);
+}
+
+template <class KeyContainer, class ValueContainer>
+void test_emplaceable() {
+  using M = std::flat_map<int, Emplaceable, std::less<int>, KeyContainer, ValueContainer>;
+  using R = std::pair<typename M::iterator, bool>;
+
+  M m;
+  ASSERT_SAME_TYPE(decltype(m.emplace()), R);
+  R r = m.emplace(std::piecewise_construct, std::forward_as_tuple(2), std::forward_as_tuple());
+  assert(r.second);
+  assert(r.first == m.begin());
+  assert(m.size() == 1);
+  assert(m.begin()->first == 2);
+  assert(m.begin()->second == Emplaceable());
+  r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+  assert(r.second);
+  assert(r.first == m.begin());
+  assert(m.size() == 2);
+  assert(m.begin()->first == 1);
+  assert(m.begin()->second == Emplaceable(2, 3.5));
+  r = m.emplace(std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+  assert(!r.second);
+  assert(r.first == m.begin());
+  assert(m.size() == 2);
+  assert(m.begin()->first == 1);
+  assert(m.begin()->second == Emplaceable(2, 3.5));
+}
+
+int main(int, char**) {
+  test_simple<std::vector<int>, std::vector<double>>();
+  test_simple<std::deque<int>, std::vector<double>>();
+  test_simple<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test_simple<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  test_emplaceable<std::vector<int>, std::vector<Emplaceable>>();
+  test_emplaceable<std::deque<int>, std::vector<Emplaceable>>();
+  test_emplaceable<MinSequenceContainer<int>, MinSequenceContainer<Emplaceable>>();
+  test_emplaceable<std::vector<int, min_allocator<int>>, std::vector<Emplaceable, min_allocator<Emplaceable>>>();
+
+  {
+    auto emplace_func = [](auto& m, auto key_arg, auto value_arg) {
+      m.emplace(std::piecewise_construct, std::tuple(key_arg), std::tuple(value_arg));
+    };
+    test_emplace_exception_guarantee(emplace_func);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/emplace_hint.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/emplace_hint.pass.cpp
new file mode 100644
index 000000000000..cfee6cac5806
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/emplace_hint.pass.cpp
@@ -0,0 +1,102 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template <class... Args>
+//   iterator emplace_hint(const_iterator position, Args&&... args);
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "../../../Emplaceable.h"
+#include "DefaultOnly.h"
+#include "min_allocator.h"
+#include "../helpers.h"
+
+#if defined(_LIBCPP_VERSION)
+// spec only specifies `emplace(Args&&...)` is_constructible_v<pair<key_type, mapped_type>, Args...> is true.
+// nothing mentioned for emplace_hint
+template <class M, class... Args>
+concept CanEmplaceHint =
+    requires(M m, typename M::const_iterator i, Args&&... args) { m.emplace_hint(i, std::forward<Args>(args)...); };
+
+using Map = std::flat_map<Emplaceable, Emplaceable>;
+static_assert(CanEmplaceHint<Map>);
+static_assert(CanEmplaceHint<Map, Emplaceable, Emplaceable>);
+static_assert(CanEmplaceHint<Map, std::piecewise_construct_t, std::tuple<int, double>, std::tuple<int, double>>);
+static_assert(!CanEmplaceHint<Map, Emplaceable>);
+static_assert(!CanEmplaceHint<Map, int, double>);
+#endif
+
+template <class KeyContainer, class ValueContainer>
+void test_simple() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using R     = M::iterator;
+  M m;
+  ASSERT_SAME_TYPE(decltype(m.emplace_hint(m.cbegin())), R);
+  R r = m.emplace_hint(m.end(), typename M::value_type(2, 3.5));
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(m.begin()->first == 2);
+  assert(m.begin()->second == 3.5);
+}
+
+template <class KeyContainer, class ValueContainer>
+void test_emplaceable() {
+  using M = std::flat_map<int, Emplaceable, std::less<int>, KeyContainer, ValueContainer>;
+  using R = M::iterator;
+
+  M m;
+  ASSERT_SAME_TYPE(decltype(m.emplace_hint(m.cbegin())), R);
+  R r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(2), std::forward_as_tuple());
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(m.begin()->first == 2);
+  assert(m.begin()->second == Emplaceable());
+  r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(m.begin()->first == 1);
+  assert(m.begin()->second == Emplaceable(2, 3.5));
+  r = m.emplace_hint(m.end(), std::piecewise_construct, std::forward_as_tuple(1), std::forward_as_tuple(2, 3.5));
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(m.begin()->first == 1);
+  assert(m.begin()->second == Emplaceable(2, 3.5));
+}
+
+int main(int, char**) {
+  test_simple<std::vector<int>, std::vector<double>>();
+  test_simple<std::deque<int>, std::vector<double>>();
+  test_simple<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test_simple<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  test_emplaceable<std::vector<int>, std::vector<Emplaceable>>();
+  test_emplaceable<std::deque<int>, std::vector<Emplaceable>>();
+  test_emplaceable<MinSequenceContainer<int>, MinSequenceContainer<Emplaceable>>();
+  test_emplaceable<std::vector<int, min_allocator<int>>, std::vector<Emplaceable, min_allocator<Emplaceable>>>();
+
+  {
+    auto emplace_func = [](auto& m, auto key_arg, auto value_arg) {
+      m.emplace_hint(m.begin(), std::piecewise_construct, std::tuple(key_arg), std::tuple(value_arg));
+    };
+    test_emplace_exception_guarantee(emplace_func);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_iter.pass.cpp
new file mode 100644
index 000000000000..914e8b676a65
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_iter.pass.cpp
@@ -0,0 +1,151 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// iterator erase(iterator position);
+// iterator erase(const_iterator position);
+
+#include <compare>
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using P     = std::pair<Key, Value>;
+  using I     = M::iterator;
+
+  P ar[] = {
+      P(1, 1.5),
+      P(2, 2.5),
+      P(3, 3.5),
+      P(4, 4.5),
+      P(5, 5.5),
+      P(6, 6.5),
+      P(7, 7.5),
+      P(8, 8.5),
+  };
+  M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+  assert(m.size() == 8);
+  std::same_as<I> decltype(auto) i1 = m.erase(std::next(m.cbegin(), 3));
+  assert(m.size() == 7);
+  assert(i1 == std::next(m.begin(), 3));
+  assert(m.begin()->first == 1);
+  assert(m.begin()->second == 1.5);
+  assert(std::next(m.begin())->first == 2);
+  assert(std::next(m.begin())->second == 2.5);
+  assert(std::next(m.begin(), 2)->first == 3);
+  assert(std::next(m.begin(), 2)->second == 3.5);
+  assert(std::next(m.begin(), 3)->first == 5);
+  assert(std::next(m.begin(), 3)->second == 5.5);
+  assert(std::next(m.begin(), 4)->first == 6);
+  assert(std::next(m.begin(), 4)->second == 6.5);
+  assert(std::next(m.begin(), 5)->first == 7);
+  assert(std::next(m.begin(), 5)->second == 7.5);
+  assert(std::next(m.begin(), 6)->first == 8);
+  assert(std::next(m.begin(), 6)->second == 8.5);
+
+  std::same_as<I> decltype(auto) i2 = m.erase(std::next(m.begin(), 0));
+  assert(m.size() == 6);
+  assert(i2 == m.begin());
+  assert(m.begin()->first == 2);
+  assert(m.begin()->second == 2.5);
+  assert(std::next(m.begin())->first == 3);
+  assert(std::next(m.begin())->second == 3.5);
+  assert(std::next(m.begin(), 2)->first == 5);
+  assert(std::next(m.begin(), 2)->second == 5.5);
+  assert(std::next(m.begin(), 3)->first == 6);
+  assert(std::next(m.begin(), 3)->second == 6.5);
+  assert(std::next(m.begin(), 4)->first == 7);
+  assert(std::next(m.begin(), 4)->second == 7.5);
+  assert(std::next(m.begin(), 5)->first == 8);
+  assert(std::next(m.begin(), 5)->second == 8.5);
+
+  std::same_as<I> decltype(auto) i3 = m.erase(std::next(m.cbegin(), 5));
+  assert(m.size() == 5);
+  assert(i3 == m.end());
+  assert(m.begin()->first == 2);
+  assert(m.begin()->second == 2.5);
+  assert(std::next(m.begin())->first == 3);
+  assert(std::next(m.begin())->second == 3.5);
+  assert(std::next(m.begin(), 2)->first == 5);
+  assert(std::next(m.begin(), 2)->second == 5.5);
+  assert(std::next(m.begin(), 3)->first == 6);
+  assert(std::next(m.begin(), 3)->second == 6.5);
+  assert(std::next(m.begin(), 4)->first == 7);
+  assert(std::next(m.begin(), 4)->second == 7.5);
+
+  std::same_as<I> decltype(auto) i4 = m.erase(std::next(m.begin(), 1));
+  assert(m.size() == 4);
+  assert(i4 == std::next(m.begin()));
+  assert(m.begin()->first == 2);
+  assert(m.begin()->second == 2.5);
+  assert(std::next(m.begin())->first == 5);
+  assert(std::next(m.begin())->second == 5.5);
+  assert(std::next(m.begin(), 2)->first == 6);
+  assert(std::next(m.begin(), 2)->second == 6.5);
+  assert(std::next(m.begin(), 3)->first == 7);
+  assert(std::next(m.begin(), 3)->second == 7.5);
+
+  std::same_as<I> decltype(auto) i5 = m.erase(std::next(m.cbegin(), 2));
+  assert(m.size() == 3);
+  assert(i5 == std::next(m.begin(), 2));
+  assert(m.begin()->first == 2);
+  assert(m.begin()->second == 2.5);
+  assert(std::next(m.begin())->first == 5);
+  assert(std::next(m.begin())->second == 5.5);
+  assert(std::next(m.begin(), 2)->first == 7);
+  assert(std::next(m.begin(), 2)->second == 7.5);
+
+  std::same_as<I> decltype(auto) i6 = m.erase(std::next(m.begin(), 2));
+  assert(m.size() == 2);
+  assert(i6 == std::next(m.begin(), 2));
+  assert(m.begin()->first == 2);
+  assert(m.begin()->second == 2.5);
+  assert(std::next(m.begin())->first == 5);
+  assert(std::next(m.begin())->second == 5.5);
+
+  std::same_as<I> decltype(auto) i7 = m.erase(std::next(m.cbegin(), 0));
+  assert(m.size() == 1);
+  assert(i7 == std::next(m.begin(), 0));
+  assert(m.begin()->first == 5);
+  assert(m.begin()->second == 5.5);
+
+  std::same_as<I> decltype(auto) i8 = m.erase(m.begin());
+  assert(m.size() == 0);
+  assert(i8 == m.begin());
+  assert(i8 == m.end());
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto erase_function = [](auto& m, auto) { m.erase(m.begin() + 2); };
+    test_erase_exception_guarantee(erase_function);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_iter_iter.pass.cpp
new file mode 100644
index 000000000000..0bc920829402
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_iter_iter.pass.cpp
@@ -0,0 +1,109 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// iterator erase(const_iterator first, const_iterator last);
+
+#include <compare>
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using P     = std::pair<Key, Value>;
+  using I     = M::iterator;
+
+  P ar[] = {
+      P(1, 1.5),
+      P(2, 2.5),
+      P(3, 3.5),
+      P(4, 4.5),
+      P(5, 5.5),
+      P(6, 6.5),
+      P(7, 7.5),
+      P(8, 8.5),
+  };
+  M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
+  assert(m.size() == 8);
+  std::same_as<I> decltype(auto) i1 = m.erase(m.cbegin(), m.cbegin());
+  assert(m.size() == 8);
+  assert(i1 == m.begin());
+  assert(m.begin()->first == 1);
+  assert(m.begin()->second == 1.5);
+  assert(std::next(m.begin())->first == 2);
+  assert(std::next(m.begin())->second == 2.5);
+  assert(std::next(m.begin(), 2)->first == 3);
+  assert(std::next(m.begin(), 2)->second == 3.5);
+  assert(std::next(m.begin(), 3)->first == 4);
+  assert(std::next(m.begin(), 3)->second == 4.5);
+  assert(std::next(m.begin(), 4)->first == 5);
+  assert(std::next(m.begin(), 4)->second == 5.5);
+  assert(std::next(m.begin(), 5)->first == 6);
+  assert(std::next(m.begin(), 5)->second == 6.5);
+  assert(std::next(m.begin(), 6)->first == 7);
+  assert(std::next(m.begin(), 6)->second == 7.5);
+  assert(std::next(m.begin(), 7)->first == 8);
+  assert(std::next(m.begin(), 7)->second == 8.5);
+
+  std::same_as<I> decltype(auto) i2 = m.erase(m.cbegin(), std::next(m.cbegin(), 2));
+  assert(m.size() == 6);
+  assert(i2 == m.begin());
+  assert(std::next(m.begin(), 0)->first == 3);
+  assert(std::next(m.begin(), 0)->second == 3.5);
+  assert(std::next(m.begin(), 1)->first == 4);
+  assert(std::next(m.begin(), 1)->second == 4.5);
+  assert(std::next(m.begin(), 2)->first == 5);
+  assert(std::next(m.begin(), 2)->second == 5.5);
+  assert(std::next(m.begin(), 3)->first == 6);
+  assert(std::next(m.begin(), 3)->second == 6.5);
+  assert(std::next(m.begin(), 4)->first == 7);
+  assert(std::next(m.begin(), 4)->second == 7.5);
+  assert(std::next(m.begin(), 5)->first == 8);
+  assert(std::next(m.begin(), 5)->second == 8.5);
+
+  std::same_as<I> decltype(auto) i3 = m.erase(std::next(m.cbegin(), 2), std::next(m.cbegin(), 6));
+  assert(m.size() == 2);
+  assert(i3 == std::next(m.begin(), 2));
+  assert(std::next(m.begin(), 0)->first == 3);
+  assert(std::next(m.begin(), 0)->second == 3.5);
+  assert(std::next(m.begin(), 1)->first == 4);
+  assert(std::next(m.begin(), 1)->second == 4.5);
+
+  std::same_as<I> decltype(auto) i4 = m.erase(m.cbegin(), m.cend());
+  assert(m.size() == 0);
+  assert(i4 == m.begin());
+  assert(i4 == m.end());
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto erase_function = [](auto& m, auto) { m.erase(m.begin(), m.begin() + 2); };
+    test_erase_exception_guarantee(erase_function);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_key.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_key.pass.cpp
new file mode 100644
index 000000000000..ef57b1cb5512
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_key.pass.cpp
@@ -0,0 +1,91 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// size_type erase(const key_type& k);
+
+#include <compare>
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer, class Compare = std::less<>>
+void test() {
+  using M = std::flat_map<int, char, Compare, KeyContainer, ValueContainer>;
+
+  auto make = [](std::initializer_list<int> il) {
+    M m;
+    for (int i : il) {
+      m.emplace(i, i);
+    }
+    return m;
+  };
+  M m = make({1, 2, 3, 4, 5, 6, 7, 8});
+  ASSERT_SAME_TYPE(decltype(m.erase(9)), typename M::size_type);
+  auto n = m.erase(9);
+  assert(n == 0);
+  assert(m == make({1, 2, 3, 4, 5, 6, 7, 8}));
+  n = m.erase(4);
+  assert(n == 1);
+  assert(m == make({1, 2, 3, 5, 6, 7, 8}));
+  n = m.erase(1);
+  assert(n == 1);
+  assert(m == make({2, 3, 5, 6, 7, 8}));
+  n = m.erase(8);
+  assert(n == 1);
+  assert(m == make({2, 3, 5, 6, 7}));
+  n = m.erase(3);
+  assert(n == 1);
+  assert(m == make({2, 5, 6, 7}));
+  n = m.erase(4);
+  assert(n == 0);
+  assert(m == make({2, 5, 6, 7}));
+  n = m.erase(6);
+  assert(n == 1);
+  assert(m == make({2, 5, 7}));
+  n = m.erase(7);
+  assert(n == 1);
+  assert(m == make({2, 5}));
+  n = m.erase(2);
+  assert(n == 1);
+  assert(m == make({5}));
+  n = m.erase(5);
+  assert(n == 1);
+  assert(m.empty());
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::vector<int>, std::vector<char>, std::greater<>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  {
+    auto erase_function = [](auto& m, auto key_arg) {
+      using Map = std::decay_t<decltype(m)>;
+      using Key = typename Map::key_type;
+      const Key key{key_arg};
+      m.erase(key);
+    };
+    test_erase_exception_guarantee(erase_function);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_key_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_key_transparent.pass.cpp
new file mode 100644
index 000000000000..3ba30757bf2c
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_key_transparent.pass.cpp
@@ -0,0 +1,144 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// size_type erase(K&& k);
+
+#include <compare>
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// Constraints: The qualified-id Compare::is_transparent is valid and denotes a type.
+template <class M>
+concept CanErase        = requires(M m, Transparent<int> k) { m.erase(k); };
+using TransparentMap    = std::flat_map<int, double, TransparentComparator>;
+using NonTransparentMap = std::flat_map<int, double, NonTransparentComparator>;
+static_assert(CanErase<TransparentMap>);
+static_assert(!CanErase<const TransparentMap>);
+static_assert(!CanErase<NonTransparentMap>);
+static_assert(!CanErase<const NonTransparentMap>);
+
+template <class Key, class It>
+struct HeterogeneousKey {
+  explicit HeterogeneousKey(Key key, It it) : key_(key), it_(it) {}
+  operator It() && { return it_; }
+  auto operator<=>(Key key) const { return key_ <=> key; }
+  friend bool operator<(const HeterogeneousKey&, const HeterogeneousKey&) {
+    assert(false);
+    return false;
+  }
+  Key key_;
+  It it_;
+};
+
+template <class KeyContainer, class ValueContainer>
+void test_simple() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  M m = {{1, 1}, {2, 2}, {3, 3}, {4, 4}};
+  ASSERT_SAME_TYPE(decltype(m.erase(9)), typename M::size_type);
+  auto n = m.erase(3); // erase(K&&) [with K=int]
+  assert(n == 1);
+  assert((m == M{{1, 1}, {2, 2}, {4, 4}}));
+  typename M::key_type lvalue = 2;
+  n                           = m.erase(lvalue); // erase(K&&) [with K=int&]
+  assert(n == 1);
+  assert((m == M{{1, 1}, {4, 4}}));
+  const typename M::key_type const_lvalue = 1;
+  n                                       = m.erase(const_lvalue); // erase(const key_type&)
+  assert(n == 1);
+  assert((m == M{{4, 4}}));
+}
+
+template <class KeyContainer, class ValueContainer>
+void test_transparent_comparator() {
+  using M = std::flat_map<std::string, int, TransparentComparator, KeyContainer, ValueContainer>;
+  M m     = {{"alpha", 1}, {"beta", 2}, {"epsilon", 3}, {"eta", 4}, {"gamma", 5}};
+  ASSERT_SAME_TYPE(decltype(m.erase(Transparent<std::string>{"abc"})), typename M::size_type);
+
+  auto n = m.erase(Transparent<std::string>{"epsilon"});
+  assert(n == 1);
+
+  M expected = {{"alpha", 1}, {"beta", 2}, {"eta", 4}, {"gamma", 5}};
+  assert(m == expected);
+
+  auto n2 = m.erase(Transparent<std::string>{"aaa"});
+  assert(n2 == 0);
+  assert(m == expected);
+}
+
+int main(int, char**) {
+  test_simple<std::vector<int>, std::vector<double>>();
+  test_simple<std::deque<int>, std::vector<double>>();
+  test_simple<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test_simple<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  test_transparent_comparator<std::vector<std::string>, std::vector<int>>();
+  test_transparent_comparator<std::deque<std::string>, std::vector<int>>();
+  test_transparent_comparator<MinSequenceContainer<std::string>, MinSequenceContainer<int>>();
+  test_transparent_comparator<std::vector<std::string, min_allocator<std::string>>,
+                              std::vector<int, min_allocator<int>>>();
+
+  {
+    // P2077's HeterogeneousKey example
+    using M                           = std::flat_map<int, int, std::less<>>;
+    M m                               = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}, {7, 7}, {8, 8}};
+    auto h1                           = HeterogeneousKey<int, M::iterator>(8, m.begin());
+    std::same_as<M::size_type> auto n = m.erase(h1); // lvalue is not convertible to It; erase(K&&) is the best match
+    assert(n == 1);
+    assert((m == M{{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}, {7, 7}}));
+    std::same_as<M::iterator> auto it = m.erase(std::move(h1)); // rvalue is convertible to It; erase(K&&) drops out
+    assert(it == m.begin());
+    assert((m == M{{2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}, {7, 7}}));
+  }
+  {
+    using M                           = std::flat_map<int, int, std::less<>>;
+    M m                               = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}, {7, 7}, {8, 8}};
+    auto h1                           = HeterogeneousKey<int, M::const_iterator>(8, m.begin());
+    std::same_as<M::size_type> auto n = m.erase(h1); // lvalue is not convertible to It; erase(K&&) is the best match
+    assert(n == 1);
+    assert((m == M{{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}, {7, 7}}));
+    std::same_as<M::iterator> auto it = m.erase(std::move(h1)); // rvalue is convertible to It; erase(K&&) drops out
+    assert(it == m.begin());
+    assert((m == M{{2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}, {7, 7}}));
+  }
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_map<int, int, TransparentComparator> m(std::sorted_unique, {{1, 1}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto n = m.erase(Transparent<int>{3});
+    assert(n == 1);
+    assert(transparent_used);
+  }
+  {
+    auto erase_transparent = [](auto& m, auto key_arg) {
+      using Map = std::decay_t<decltype(m)>;
+      using Key = typename Map::key_type;
+      m.erase(Transparent<Key>{key_arg});
+    };
+    test_erase_exception_guarantee(erase_transparent);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/extract.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/extract.pass.cpp
new file mode 100644
index 000000000000..d8e4ce94efb9
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/extract.pass.cpp
@@ -0,0 +1,91 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// containers extract() &&;
+
+#include <algorithm>
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class T>
+concept CanExtract = requires(T&& t) { std::forward<T>(t).extract(); };
+
+static_assert(CanExtract<std::flat_map<int, int>&&>);
+static_assert(!CanExtract<std::flat_map<int, int>&>);
+static_assert(!CanExtract<std::flat_map<int, int> const&>);
+static_assert(!CanExtract<std::flat_map<int, int> const&&>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using M = std::flat_map<int, int, std::less<int>, KeyContainer, ValueContainer>;
+  M m     = M({1, 2, 3}, {4, 5, 6});
+
+  std::same_as<typename M::containers> auto containers = std::move(m).extract();
+
+  auto expected_keys   = {1, 2, 3};
+  auto expected_values = {4, 5, 6};
+  assert(std::ranges::equal(containers.keys, expected_keys));
+  assert(std::ranges::equal(containers.values, expected_values));
+  check_invariant(m);
+  LIBCPP_ASSERT(m.empty());
+  LIBCPP_ASSERT(m.keys().size() == 0);
+  LIBCPP_ASSERT(m.values().size() == 0);
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<int>>();
+  test<std::deque<int>, std::vector<int>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<int>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+  {
+    // extracted object maintains invariant if one of underlying container does not clear after move
+    using M = std::flat_map<int, int, std::less<>, std::vector<int>, CopyOnlyVector<int>>;
+    M m     = M({1, 2, 3}, {1, 2, 3});
+    std::same_as<M::containers> auto containers = std::move(m).extract();
+    assert(containers.keys.size() == 3);
+    assert(containers.values.size() == 3);
+    check_invariant(m);
+    LIBCPP_ASSERT(m.empty());
+    LIBCPP_ASSERT(m.keys().size() == 0);
+    LIBCPP_ASSERT(m.values().size() == 0);
+  }
+
+  {
+#ifndef TEST_HAS_NO_EXCEPTIONS
+    using KeyContainer   = std::vector<int>;
+    using ValueContainer = ThrowOnMoveContainer<int>;
+    using M              = std::flat_map<int, int, std::ranges::less, KeyContainer, ValueContainer>;
+
+    M m;
+    m.emplace(1, 1);
+    m.emplace(2, 2);
+    try {
+      auto c = std::move(m).extract();
+      assert(false);
+    } catch (int) {
+      check_invariant(m);
+      // In libc++, we try to erase the key after value emplacement failure.
+      // and after erasure failure, we clear the flat_map
+      LIBCPP_ASSERT(m.size() == 0);
+    }
+#endif
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_cv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_cv.pass.cpp
new file mode 100644
index 000000000000..7e667c4e4877
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_cv.pass.cpp
@@ -0,0 +1,83 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// pair<iterator, bool> insert(const value_type& v);
+
+#include <flat_map>
+#include <deque>
+#include <cassert>
+#include <functional>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "../helpers.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using R     = std::pair<typename M::iterator, bool>;
+  using VT    = typename M::value_type;
+  M m;
+
+  const VT v1(2, 2.5);
+  std::same_as<R> decltype(auto) r = m.insert(v1);
+  assert(r.second);
+  assert(r.first == m.begin());
+  assert(m.size() == 1);
+  assert(r.first->first == 2);
+  assert(r.first->second == 2.5);
+
+  const VT v2(1, 1.5);
+  r = m.insert(v2);
+  assert(r.second);
+  assert(r.first == m.begin());
+  assert(m.size() == 2);
+  assert(r.first->first == 1);
+  assert(r.first->second == 1.5);
+
+  const VT v3(3, 3.5);
+  r = m.insert(v3);
+  assert(r.second);
+  assert(r.first == std::ranges::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r.first->first == 3);
+  assert(r.first->second == 3.5);
+
+  const VT v4(3, 4.5);
+  r = m.insert(v4);
+  assert(!r.second);
+  assert(r.first == std::ranges::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r.first->first == 3);
+  assert(r.first->second == 3.5);
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto insert_func = [](auto& m, auto key_arg, auto value_arg) {
+      using FlatMap    = std::decay_t<decltype(m)>;
+      using value_type = typename FlatMap::value_type;
+      const value_type p(std::piecewise_construct, std::tuple(key_arg), std::tuple(value_arg));
+      m.insert(p);
+    };
+    test_emplace_exception_guarantee(insert_func);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_initializer_list.pass.cpp
new file mode 100644
index 000000000000..32be3ab8a95b
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_initializer_list.pass.cpp
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// void insert(initializer_list<value_type> il);
+
+#include <flat_map>
+#include <cassert>
+#include <functional>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using V     = std::pair<const int, double>;
+
+  M m = {{1, 1}, {1, 1.5}, {1, 2}, {3, 1}, {3, 1.5}, {3, 2}};
+  m.insert({
+      {4, 1},
+      {4, 1.5},
+      {4, 2},
+      {1, 1},
+      {1, 1.5},
+      {1, 2},
+      {2, 1},
+      {2, 1.5},
+      {2, 2},
+  });
+  assert(m.size() == 4);
+  assert(std::distance(m.begin(), m.end()) == 4);
+  assert(*m.begin() == V(1, 1));
+  assert(*std::next(m.begin()) == V(2, 1));
+  assert(*std::next(m.begin(), 2) == V(3, 1));
+  assert(*std::next(m.begin(), 3) == V(4, 1));
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto insert_func = [](auto& m, const auto& newValues) {
+      using FlatMap                        = std::decay_t<decltype(m)>;
+      using value_type                     = typename FlatMap::value_type;
+      std::initializer_list<value_type> il = {{newValues[0].first, newValues[0].second}};
+      m.insert(il);
+    };
+    test_insert_range_exception_guarantee(insert_func);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_cv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_cv.pass.cpp
new file mode 100644
index 000000000000..4bbe0628317d
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_cv.pass.cpp
@@ -0,0 +1,79 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// iterator insert(const_iterator position, const value_type& v);
+
+#include <flat_map>
+#include <cassert>
+#include <functional>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "../helpers.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using R     = typename M::iterator;
+  using VT    = typename M::value_type;
+
+  M m;
+  const VT v1(2, 2.5);
+  std::same_as<R> decltype(auto) r = m.insert(m.end(), v1);
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(r->first == 2);
+  assert(r->second == 2.5);
+
+  const VT v2(1, 1.5);
+  r = m.insert(m.end(), v2);
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(r->first == 1);
+  assert(r->second == 1.5);
+
+  const VT v3(3, 3.5);
+  r = m.insert(m.end(), v3);
+  assert(r == std::ranges::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3.5);
+
+  const VT v4(3, 4.5);
+  r = m.insert(m.end(), v4);
+  assert(r == std::ranges::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3.5);
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto insert_func = [](auto& m, auto key_arg, auto value_arg) {
+      using FlatMap    = std::decay_t<decltype(m)>;
+      using value_type = typename FlatMap::value_type;
+      const value_type p(std::piecewise_construct, std::tuple(key_arg), std::tuple(value_arg));
+      m.insert(m.begin(), p);
+    };
+    test_emplace_exception_guarantee(insert_func);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp
new file mode 100644
index 000000000000..8455b19475fe
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp
@@ -0,0 +1,89 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template <class InputIterator>
+//   void insert(InputIterator first, InputIterator last);
+
+#include <flat_map>
+#include <cassert>
+#include <functional>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "test_iterators.h"
+#include "min_allocator.h"
+
+// test constraint InputIterator
+template <class M, class... Args>
+concept CanInsert = requires(M m, Args&&... args) { m.insert(std::forward<Args>(args)...); };
+
+using Map  = std::flat_map<int, int>;
+using Pair = std::pair<int, int>;
+
+static_assert(CanInsert<Map, Pair*, Pair*>);
+static_assert(CanInsert<Map, cpp17_input_iterator<Pair*>, cpp17_input_iterator<Pair*>>);
+static_assert(!CanInsert<Map, int, int>);
+static_assert(!CanInsert<Map, cpp20_input_iterator<Pair*>, cpp20_input_iterator<Pair*>>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using P = std::pair<int, double>;
+  using M = std::flat_map<int, double, std::less<int>, KeyContainer, ValueContainer>;
+
+  P ar1[] = {
+      P(2, 1),
+      P(2, 1.5),
+      P(2, 2),
+      P(1, 1),
+      P(1, 1.5),
+      P(1, 2),
+      P(3, 1),
+      P(3, 1.5),
+      P(3, 2),
+  };
+  P ar2[] = {
+      P(4, 1),
+      P(4, 1.5),
+      P(4, 2),
+      P(1, 1),
+      P(1, 1.5),
+      P(1, 2),
+      P(0, 1),
+      P(0, 1.5),
+      P(0, 2),
+  };
+
+  M m;
+  m.insert(cpp17_input_iterator<P*>(ar1), cpp17_input_iterator<P*>(ar1 + sizeof(ar1) / sizeof(ar1[0])));
+  assert(m.size() == 3);
+  M expected{{1, 1}, {2, 1}, {3, 1}};
+  assert(m == expected);
+
+  m.insert(cpp17_input_iterator<P*>(ar2), cpp17_input_iterator<P*>(ar2 + sizeof(ar2) / sizeof(ar2[0])));
+  assert(m.size() == 5);
+  M expected2{{0, 1}, {1, 1}, {2, 1}, {3, 1}, {4, 1}};
+  assert(m == expected2);
+}
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto insert_func = [](auto& m, const auto& newValues) { m.insert(newValues.begin(), newValues.end()); };
+    test_insert_range_exception_guarantee(insert_func);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_rv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_rv.pass.cpp
new file mode 100644
index 000000000000..034941b55eb8
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_rv.pass.cpp
@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+//     iterator insert(const_iterator position, value_type&&);
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "MoveOnly.h"
+#include "min_allocator.h"
+#include "../helpers.h"
+#include "test_macros.h"
+
+template <class Container, class Pair>
+void do_insert_iter_rv_test() {
+  using M = Container;
+  using P = Pair;
+  using R = typename M::iterator;
+  M m;
+  std::same_as<R> decltype(auto) r = m.insert(m.end(), P(2, 2));
+  assert(r == m.begin());
+  assert(m.size() == 1);
+  assert(r->first == 2);
+  assert(r->second == 2);
+
+  r = m.insert(m.end(), P(1, 1));
+  assert(r == m.begin());
+  assert(m.size() == 2);
+  assert(r->first == 1);
+  assert(r->second == 1);
+
+  r = m.insert(m.end(), P(3, 3));
+  assert(r == std::ranges::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3);
+
+  r = m.insert(m.end(), P(3, 4));
+  assert(r == std::ranges::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r->first == 3);
+  assert(r->second == 3);
+}
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using P     = std::pair<Key, Value>;
+  using CP    = std::pair<const Key, Value>;
+
+  do_insert_iter_rv_test<M, P>();
+  do_insert_iter_rv_test<M, CP>();
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::vector<int>, std::vector<MoveOnly>>();
+  test<std::deque<int>, std::deque<double>>();
+  test<std::deque<int>, std::deque<MoveOnly>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<MoveOnly>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<MoveOnly, min_allocator<MoveOnly>>>();
+
+  {
+    auto insert_func = [](auto& m, auto key_arg, auto value_arg) {
+      using FlatMap    = std::decay_t<decltype(m)>;
+      using value_type = typename FlatMap::value_type;
+      value_type p(std::piecewise_construct, std::tuple(key_arg), std::tuple(value_arg));
+      m.insert(m.begin(), std::move(p));
+    };
+    test_emplace_exception_guarantee(insert_func);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign.pass.cpp
new file mode 100644
index 000000000000..398a7a1a4052
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign.pass.cpp
@@ -0,0 +1,326 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "MoveOnly.h"
+#include "min_allocator.h"
+#include "test_macros.h"
+#include "../helpers.h"
+
+// template<class M>
+//   pair<iterator, bool> insert_or_assign(const key_type& k, M&& obj);
+// template<class M>
+//   pair<iterator, bool> insert_or_assign(key_type&& k, M&& obj);
+// template<class M>
+//   iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj);
+// template<class M>
+//   iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj);
+
+// Constraints: is_assignable_v<mapped_type&, M> is true and is_constructible_v<mapped_type, M> is true.
+template <class Map, class K, class M>
+concept CanInsertOrAssign =
+    requires(Map map, K&& k, M&& m) { map.insert_or_assign(std::forward<K>(k), std::forward<M>(m)); };
+
+template <class Map, class K, class M>
+concept CanInsertOrAssignIter = requires(Map map, typename Map::const_iterator iter, K&& k, M&& m) {
+  map.insert_or_assign(iter, std::forward<K>(k), std::forward<M>(m));
+};
+
+template <class From>
+struct ConstructAndAssignFrom {
+  explicit ConstructAndAssignFrom(From);
+  ConstructAndAssignFrom& operator=(From);
+};
+
+template <class From>
+struct ConstructFrom {
+  explicit ConstructFrom(From);
+};
+
+template <class From>
+struct AssignFrom {
+  AssignFrom& operator=(From);
+};
+
+struct V {};
+
+static_assert(CanInsertOrAssign<std::flat_map<int, ConstructAndAssignFrom<V>>, const int&, V>);
+static_assert(!CanInsertOrAssign<std::flat_map<int, ConstructAndAssignFrom<V>>, const int&, int>);
+static_assert(!CanInsertOrAssign<std::flat_map<int, ConstructFrom<V>>, const int&, V>);
+static_assert(!CanInsertOrAssign<std::flat_map<int, AssignFrom<V>>, const int&, V>);
+
+static_assert(CanInsertOrAssign<std::flat_map<int, ConstructAndAssignFrom<V>>, int&&, V>);
+static_assert(!CanInsertOrAssign<std::flat_map<int, ConstructAndAssignFrom<V>>, int&&, int>);
+static_assert(!CanInsertOrAssign<std::flat_map<int, ConstructFrom<V>>, int&&, V>);
+static_assert(!CanInsertOrAssign<std::flat_map<int, AssignFrom<V>>, int&&, V>);
+
+static_assert(CanInsertOrAssignIter<std::flat_map<int, ConstructAndAssignFrom<V>>, const int&, V>);
+static_assert(!CanInsertOrAssignIter<std::flat_map<int, ConstructAndAssignFrom<V>>, const int&, int>);
+static_assert(!CanInsertOrAssignIter<std::flat_map<int, ConstructFrom<V>>, const int&, V>);
+static_assert(!CanInsertOrAssignIter<std::flat_map<int, AssignFrom<V>>, const int&, V>);
+
+static_assert(CanInsertOrAssignIter<std::flat_map<int, ConstructAndAssignFrom<V>>, int&&, V>);
+static_assert(!CanInsertOrAssignIter<std::flat_map<int, ConstructAndAssignFrom<V>>, int&&, int>);
+static_assert(!CanInsertOrAssignIter<std::flat_map<int, ConstructFrom<V>>, int&&, V>);
+static_assert(!CanInsertOrAssignIter<std::flat_map<int, AssignFrom<V>>, int&&, V>);
+
+template <class KeyContainer, class ValueContainer>
+void test_cv_key() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+  { // pair<iterator, bool> insert_or_assign(const key_type& k, M&& obj);
+    using R = std::pair<typename M::iterator, bool>;
+    M m;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(i, Moveable(i, (double)i));
+    assert(m.size() == 10);
+
+    for (int i = 0; i < 20; i += 2) {
+      Moveable mv(i + 1, i + 1);
+      std::same_as<R> decltype(auto) r1 = m.insert_or_assign(i, std::move(mv));
+      assert(m.size() == 10);
+      assert(!r1.second);                      // was not inserted
+      assert(mv.moved());                      // was moved from
+      assert(r1.first->first == i);            // key
+      assert(r1.first->second.get() == i + 1); // value
+    }
+
+    Moveable mv1(5, 5.0);
+    std::same_as<R> decltype(auto) r2 = m.insert_or_assign(-1, std::move(mv1));
+    assert(m.size() == 11);
+    assert(r2.second);                   // was inserted
+    assert(mv1.moved());                 // was moved from
+    assert(r2.first->first == -1);       // key
+    assert(r2.first->second.get() == 5); // value
+
+    Moveable mv2(9, 9.0);
+    std::same_as<R> decltype(auto) r3 = m.insert_or_assign(3, std::move(mv2));
+    assert(m.size() == 12);
+    assert(r3.second);                   // was inserted
+    assert(mv2.moved());                 // was moved from
+    assert(r3.first->first == 3);        // key
+    assert(r3.first->second.get() == 9); // value
+
+    Moveable mv3(-1, 5.0);
+    std::same_as<R> decltype(auto) r4 = m.insert_or_assign(117, std::move(mv3));
+    assert(m.size() == 13);
+    assert(r4.second);                    // was inserted
+    assert(mv3.moved());                  // was moved from
+    assert(r4.first->first == 117);       // key
+    assert(r4.first->second.get() == -1); // value
+  }
+
+  { // iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj);
+    M m;
+    using R = M::iterator;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(i, Moveable(i, (double)i));
+    assert(m.size() == 10);
+    typename M::const_iterator it = m.find(2);
+
+    Moveable mv1(3, 3.0);
+    std::same_as<R> decltype(auto) r1 = m.insert_or_assign(it, 2, std::move(mv1));
+    assert(m.size() == 10);
+    assert(mv1.moved());           // was moved from
+    assert(r1->first == 2);        // key
+    assert(r1->second.get() == 3); // value
+
+    Moveable mv2(5, 5.0);
+    std::same_as<R> decltype(auto) r2 = m.insert_or_assign(it, 3, std::move(mv2));
+    assert(m.size() == 11);
+    assert(mv2.moved());           // was moved from
+    assert(r2->first == 3);        // key
+    assert(r2->second.get() == 5); // value
+
+    // wrong hint: begin()
+    Moveable mv3(7, 7.0);
+    std::same_as<R> decltype(auto) r3 = m.insert_or_assign(m.begin(), 4, std::move(mv3));
+    assert(m.size() == 11);
+    assert(mv3.moved());           // was moved from
+    assert(r3->first == 4);        // key
+    assert(r3->second.get() == 7); // value
+
+    Moveable mv4(9, 9.0);
+    std::same_as<R> decltype(auto) r4 = m.insert_or_assign(m.begin(), 5, std::move(mv4));
+    assert(m.size() == 12);
+    assert(mv4.moved());           // was moved from
+    assert(r4->first == 5);        // key
+    assert(r4->second.get() == 9); // value
+
+    // wrong hint: end()
+    Moveable mv5(11, 11.0);
+    std::same_as<R> decltype(auto) r5 = m.insert_or_assign(m.end(), 6, std::move(mv5));
+    assert(m.size() == 12);
+    assert(mv5.moved());            // was moved from
+    assert(r5->first == 6);         // key
+    assert(r5->second.get() == 11); // value
+
+    Moveable mv6(13, 13.0);
+    std::same_as<R> decltype(auto) r6 = m.insert_or_assign(m.end(), 7, std::move(mv6));
+    assert(m.size() == 13);
+    assert(mv6.moved());            // was moved from
+    assert(r6->first == 7);         // key
+    assert(r6->second.get() == 13); // value
+
+    // wrong hint: third element
+    Moveable mv7(15, 15.0);
+    std::same_as<R> decltype(auto) r7 = m.insert_or_assign(std::next(m.begin(), 2), 8, std::move(mv7));
+    assert(m.size() == 13);
+    assert(mv7.moved());            // was moved from
+    assert(r7->first == 8);         // key
+    assert(r7->second.get() == 15); // value
+
+    Moveable mv8(17, 17.0);
+    std::same_as<R> decltype(auto) r8 = m.insert_or_assign(std::next(m.begin(), 2), 9, std::move(mv8));
+    assert(m.size() == 14);
+    assert(mv8.moved());            // was moved from
+    assert(r8->first == 9);         // key
+    assert(r8->second.get() == 17); // value
+  }
+}
+
+template <class KeyContainer, class ValueContainer>
+void test_rv_key() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  { // pair<iterator, bool> insert_or_assign(key_type&& k, M&& obj);
+    using R = std::pair<typename M::iterator, bool>;
+    M m;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(Moveable(i, (double)i), Moveable(i + 1, (double)i + 1));
+    assert(m.size() == 10);
+
+    Moveable mvkey1(2, 2.0);
+    Moveable mv1(4, 4.0);
+    std::same_as<R> decltype(auto) r1 = m.insert_or_assign(std::move(mvkey1), std::move(mv1));
+    assert(m.size() == 10);
+    assert(!r1.second);                  // was not inserted
+    assert(!mvkey1.moved());             // was not moved from
+    assert(mv1.moved());                 // was moved from
+    assert(r1.first->first == mvkey1);   // key
+    assert(r1.first->second.get() == 4); // value
+
+    Moveable mvkey2(3, 3.0);
+    Moveable mv2(5, 5.0);
+    std::same_as<R> decltype(auto) r2 = m.try_emplace(std::move(mvkey2), std::move(mv2));
+    assert(m.size() == 11);
+    assert(r2.second);                   // was inserted
+    assert(mv2.moved());                 // was moved from
+    assert(mvkey2.moved());              // was moved from
+    assert(r2.first->first.get() == 3);  // key
+    assert(r2.first->second.get() == 5); // value
+  }
+  { // iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj);
+    using R = M::iterator;
+    M m;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(Moveable(i, (double)i), Moveable(i + 1, (double)i + 1));
+    assert(m.size() == 10);
+    typename M::const_iterator it = std::next(m.cbegin());
+
+    Moveable mvkey1(2, 2.0);
+    Moveable mv1(4, 4.0);
+    std::same_as<R> decltype(auto) r1 = m.insert_or_assign(it, std::move(mvkey1), std::move(mv1));
+    assert(m.size() == 10);
+    assert(mv1.moved());           // was moved from
+    assert(!mvkey1.moved());       // was not moved from
+    assert(r1->first == mvkey1);   // key
+    assert(r1->second.get() == 4); // value
+
+    Moveable mvkey2(3, 3.0);
+    Moveable mv2(5, 5.0);
+    std::same_as<R> decltype(auto) r2 = m.insert_or_assign(it, std::move(mvkey2), std::move(mv2));
+    assert(m.size() == 11);
+    assert(mv2.moved());           // was moved from
+    assert(mvkey2.moved());        // was moved from
+    assert(r2->first.get() == 3);  // key
+    assert(r2->second.get() == 5); // value
+
+    // wrong hint: begin()
+    Moveable mvkey3(6, 6.0);
+    Moveable mv3(8, 8.0);
+    std::same_as<R> decltype(auto) r3 = m.insert_or_assign(m.begin(), std::move(mvkey3), std::move(mv3));
+    assert(m.size() == 11);
+    assert(mv3.moved());           // was moved from
+    assert(!mvkey3.moved());       // was not moved from
+    assert(r3->first == mvkey3);   // key
+    assert(r3->second.get() == 8); // value
+
+    Moveable mvkey4(7, 7.0);
+    Moveable mv4(9, 9.0);
+    std::same_as<R> decltype(auto) r4 = m.insert_or_assign(m.begin(), std::move(mvkey4), std::move(mv4));
+    assert(m.size() == 12);
+    assert(mv4.moved());           // was moved from
+    assert(mvkey4.moved());        // was moved from
+    assert(r4->first.get() == 7);  // key
+    assert(r4->second.get() == 9); // value
+
+    // wrong hint: end()
+    Moveable mvkey5(8, 8.0);
+    Moveable mv5(10, 10.0);
+    std::same_as<R> decltype(auto) r5 = m.insert_or_assign(m.end(), std::move(mvkey5), std::move(mv5));
+    assert(m.size() == 12);
+    assert(mv5.moved());            // was moved from
+    assert(!mvkey5.moved());        // was not moved from
+    assert(r5->first == mvkey5);    // key
+    assert(r5->second.get() == 10); // value
+
+    Moveable mvkey6(9, 9.0);
+    Moveable mv6(11, 11.0);
+    std::same_as<R> decltype(auto) r6 = m.insert_or_assign(m.end(), std::move(mvkey6), std::move(mv6));
+    assert(m.size() == 13);
+    assert(mv6.moved());            // was moved from
+    assert(mvkey6.moved());         // was moved from
+    assert(r6->first.get() == 9);   // key
+    assert(r6->second.get() == 11); // value
+
+    // wrong hint: third element
+    Moveable mvkey7(10, 10.0);
+    Moveable mv7(12, 12.0);
+    std::same_as<R> decltype(auto) r7 = m.insert_or_assign(std::next(m.begin(), 2), std::move(mvkey7), std::move(mv7));
+    assert(m.size() == 13);
+    assert(mv7.moved());            // was moved from
+    assert(!mvkey7.moved());        // was not moved from
+    assert(r7->first == mvkey7);    // key
+    assert(r7->second.get() == 12); // value
+
+    Moveable mvkey8(11, 11.0);
+    Moveable mv8(13, 13.0);
+    std::same_as<R> decltype(auto) r8 = m.insert_or_assign(std::next(m.begin(), 2), std::move(mvkey8), std::move(mv8));
+    assert(m.size() == 14);
+    assert(mv8.moved());            // was moved from
+    assert(mvkey8.moved());         // was moved from
+    assert(r8->first.get() == 11);  // key
+    assert(r8->second.get() == 13); // value
+  }
+}
+
+int main(int, char**) {
+  test_cv_key<std::vector<int>, std::vector<Moveable>>();
+  test_cv_key<std::deque<int>, std::vector<Moveable>>();
+  test_cv_key<MinSequenceContainer<int>, MinSequenceContainer<Moveable>>();
+  test_cv_key<std::vector<int, min_allocator<int>>, std::vector<Moveable, min_allocator<Moveable>>>();
+
+  test_rv_key<std::vector<Moveable>, std::vector<Moveable>>();
+  test_rv_key<std::deque<Moveable>, std::vector<Moveable>>();
+  test_rv_key<MinSequenceContainer<Moveable>, MinSequenceContainer<Moveable>>();
+  test_rv_key<std::vector<Moveable, min_allocator<Moveable>>, std::vector<Moveable, min_allocator<Moveable>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign_transparent.pass.cpp
new file mode 100644
index 000000000000..636c4edfe551
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign_transparent.pass.cpp
@@ -0,0 +1,259 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "MoveOnly.h"
+#include "min_allocator.h"
+#include "test_macros.h"
+#include "../helpers.h"
+
+// template<class K, class M>
+//   pair<iterator, bool> insert_or_assign(K&& k, M&& obj);
+// template<class K, class M>
+//   iterator insert_or_assign(const_iterator hint, K&& k, M&& obj);
+
+// Constraints:
+// The qualified-id Compare::is_transparent is valid and denotes a type.
+// is_constructible_v<key_type, K> is true.
+// is_assignable_v<mapped_type&, M> is true.
+// is_constructible_v<mapped_type, M> is true.
+
+template <class Map, class K, class M>
+concept CanInsertOrAssign =
+    requires(Map map, K&& k, M&& m) { map.insert_or_assign(std::forward<K>(k), std::forward<M>(m)); };
+
+template <class Map, class K, class M>
+concept CanInsertOrAssignIter = requires(Map map, typename Map::const_iterator iter, K&& k, M&& m) {
+  map.insert_or_assign(iter, std::forward<K>(k), std::forward<M>(m));
+};
+
+template <class From>
+struct ConstructAndAssignFrom {
+  explicit ConstructAndAssignFrom(From);
+  ConstructAndAssignFrom& operator=(From);
+};
+
+template <class From>
+struct ConstructFrom {
+  explicit ConstructFrom(From);
+};
+
+template <class From>
+struct AssignFrom {
+  AssignFrom& operator=(From);
+};
+
+struct V {};
+
+static_assert(CanInsertOrAssign<std::flat_map<int, ConstructAndAssignFrom<V>, TransparentComparator>,
+                                ConvertibleTransparent<int>,
+                                V>);
+static_assert(!CanInsertOrAssign<std::flat_map<int, ConstructAndAssignFrom<V>, TransparentComparator>,
+                                 NonConvertibleTransparent<int>,
+                                 V>);
+static_assert(!CanInsertOrAssign<std::flat_map<int, ConstructAndAssignFrom<V>, NonTransparentComparator>,
+                                 NonConvertibleTransparent<int>,
+                                 V>);
+static_assert(!CanInsertOrAssign<std::flat_map<int, ConstructAndAssignFrom<V>, TransparentComparator>,
+                                 ConvertibleTransparent<int>,
+                                 int>);
+static_assert(
+    !CanInsertOrAssign<std::flat_map<int, ConstructFrom<V>, TransparentComparator>, ConvertibleTransparent<int>, V>);
+static_assert(
+    !CanInsertOrAssign<std::flat_map<int, AssignFrom<V>, TransparentComparator>, ConvertibleTransparent<int>, V>);
+
+static_assert(CanInsertOrAssignIter<std::flat_map<int, ConstructAndAssignFrom<V>, TransparentComparator>,
+                                    ConvertibleTransparent<int>,
+                                    V>);
+static_assert(!CanInsertOrAssignIter<std::flat_map<int, ConstructAndAssignFrom<V>, TransparentComparator>,
+                                     NonConvertibleTransparent<int>,
+                                     V>);
+static_assert(!CanInsertOrAssignIter<std::flat_map<int, ConstructAndAssignFrom<V>, NonTransparentComparator>,
+                                     NonConvertibleTransparent<int>,
+                                     V>);
+static_assert(!CanInsertOrAssignIter<std::flat_map<int, ConstructAndAssignFrom<V>, TransparentComparator>,
+                                     ConvertibleTransparent<int>,
+                                     int>);
+static_assert(!CanInsertOrAssignIter<std::flat_map<int, ConstructFrom<V>, TransparentComparator>,
+                                     ConvertibleTransparent<int>,
+                                     V>);
+static_assert(
+    !CanInsertOrAssignIter<std::flat_map<int, AssignFrom<V>, TransparentComparator>, ConvertibleTransparent<int>, V>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+  {
+    // pair<iterator, bool> insert_or_assign(const key_type& k, M&& obj);
+    using R = std::pair<typename M::iterator, bool>;
+    M m;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(i, Moveable(i, (double)i));
+    assert(m.size() == 10);
+
+    for (int i = 0; i < 20; i += 2) {
+      Moveable mv(i + 1, i + 1);
+      std::same_as<R> decltype(auto) r1 = m.insert_or_assign(ConvertibleTransparent<int>{i}, std::move(mv));
+      assert(m.size() == 10);
+      assert(!r1.second);                      // was not inserted
+      assert(mv.moved());                      // was moved from
+      assert(r1.first->first == i);            // key
+      assert(r1.first->second.get() == i + 1); // value
+    }
+
+    Moveable mv1(5, 5.0);
+    std::same_as<R> decltype(auto) r2 = m.insert_or_assign(ConvertibleTransparent<int>{-1}, std::move(mv1));
+    assert(m.size() == 11);
+    assert(r2.second);                   // was inserted
+    assert(mv1.moved());                 // was moved from
+    assert(r2.first->first == -1);       // key
+    assert(r2.first->second.get() == 5); // value
+
+    Moveable mv2(9, 9.0);
+    std::same_as<R> decltype(auto) r3 = m.insert_or_assign(ConvertibleTransparent<int>{3}, std::move(mv2));
+    assert(m.size() == 12);
+    assert(r3.second);                   // was inserted
+    assert(mv2.moved());                 // was moved from
+    assert(r3.first->first == 3);        // key
+    assert(r3.first->second.get() == 9); // value
+
+    Moveable mv3(-1, 5.0);
+    std::same_as<R> decltype(auto) r4 = m.insert_or_assign(ConvertibleTransparent<int>{117}, std::move(mv3));
+    assert(m.size() == 13);
+    assert(r4.second);                    // was inserted
+    assert(mv3.moved());                  // was moved from
+    assert(r4.first->first == 117);       // key
+    assert(r4.first->second.get() == -1); // value
+  }
+  {
+    // iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj);
+    using R = M::iterator;
+    M m;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(i, Moveable(i, (double)i));
+    assert(m.size() == 10);
+    typename M::const_iterator it = m.find(2);
+
+    Moveable mv1(3, 3.0);
+    std::same_as<R> decltype(auto) r1 = m.insert_or_assign(it, ConvertibleTransparent<int>{2}, std::move(mv1));
+    assert(m.size() == 10);
+    assert(mv1.moved());           // was moved from
+    assert(r1->first == 2);        // key
+    assert(r1->second.get() == 3); // value
+
+    Moveable mv2(5, 5.0);
+    std::same_as<R> decltype(auto) r2 = m.insert_or_assign(it, ConvertibleTransparent<int>{3}, std::move(mv2));
+    assert(m.size() == 11);
+    assert(mv2.moved());           // was moved from
+    assert(r2->first == 3);        // key
+    assert(r2->second.get() == 5); // value
+
+    // wrong hint: begin()
+    Moveable mv3(7, 7.0);
+    std::same_as<R> decltype(auto) r3 = m.insert_or_assign(m.begin(), ConvertibleTransparent<int>{4}, std::move(mv3));
+    assert(m.size() == 11);
+    assert(mv3.moved());           // was moved from
+    assert(r3->first == 4);        // key
+    assert(r3->second.get() == 7); // value
+
+    Moveable mv4(9, 9.0);
+    std::same_as<R> decltype(auto) r4 = m.insert_or_assign(m.begin(), ConvertibleTransparent<int>{5}, std::move(mv4));
+    assert(m.size() == 12);
+    assert(mv4.moved());           // was moved from
+    assert(r4->first == 5);        // key
+    assert(r4->second.get() == 9); // value
+
+    // wrong hint: end()
+    Moveable mv5(11, 11.0);
+    std::same_as<R> decltype(auto) r5 = m.insert_or_assign(m.end(), ConvertibleTransparent<int>{6}, std::move(mv5));
+    assert(m.size() == 12);
+    assert(mv5.moved());            // was moved from
+    assert(r5->first == 6);         // key
+    assert(r5->second.get() == 11); // value
+
+    Moveable mv6(13, 13.0);
+    std::same_as<R> decltype(auto) r6 = m.insert_or_assign(m.end(), ConvertibleTransparent<int>{7}, std::move(mv6));
+    assert(m.size() == 13);
+    assert(mv6.moved());            // was moved from
+    assert(r6->first == 7);         // key
+    assert(r6->second.get() == 13); // value
+
+    // wrong hint: third element
+    Moveable mv7(15, 15.0);
+    std::same_as<R> decltype(auto) r7 =
+        m.insert_or_assign(std::next(m.begin(), 2), ConvertibleTransparent<int>{8}, std::move(mv7));
+    assert(m.size() == 13);
+    assert(mv7.moved());            // was moved from
+    assert(r7->first == 8);         // key
+    assert(r7->second.get() == 15); // value
+
+    Moveable mv8(17, 17.0);
+    std::same_as<R> decltype(auto) r8 =
+        m.insert_or_assign(std::next(m.begin(), 2), ConvertibleTransparent<int>{9}, std::move(mv8));
+    assert(m.size() == 14);
+    assert(mv8.moved());            // was moved from
+    assert(r8->first == 9);         // key
+    assert(r8->second.get() == 17); // value
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<Moveable>>();
+  test<std::deque<int>, std::vector<Moveable>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<Moveable>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<Moveable, min_allocator<Moveable>>>();
+
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_map<int, int, TransparentComparator> m(std::sorted_unique, {{1, 1}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto p = m.insert_or_assign(ConvertibleTransparent<int>{3}, 5);
+    assert(!p.second);
+    assert(transparent_used);
+  }
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_map<int, int, TransparentComparator> m(std::sorted_unique, {{1, 1}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto it = m.insert_or_assign(m.begin(), ConvertibleTransparent<int>{3}, 5);
+    assert(it->second == 5);
+    assert(transparent_used);
+  }
+
+  {
+    auto insert_or_assign = [](auto& m, auto key_arg, auto value_arg) {
+      using M   = std::decay_t<decltype(m)>;
+      using Key = typename M::key_type;
+      m.insert_or_assign(ConvertibleTransparent<Key>{key_arg}, value_arg);
+    };
+    test_emplace_exception_guarantee(insert_or_assign);
+  }
+
+  {
+    auto insert_or_assign_iter = [](auto& m, auto key_arg, auto value_arg) {
+      using M   = std::decay_t<decltype(m)>;
+      using Key = typename M::key_type;
+      m.insert_or_assign(m.begin(), ConvertibleTransparent<Key>{key_arg}, value_arg);
+    };
+    test_emplace_exception_guarantee(insert_or_assign_iter);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_range.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_range.pass.cpp
new file mode 100644
index 000000000000..a2e64431a3c2
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_range.pass.cpp
@@ -0,0 +1,109 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<container-compatible-range<value_type> R>
+//   void insert_range(R&& rg);
+
+#include <algorithm>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <ranges>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "MoveOnly.h"
+#include "test_macros.h"
+#include "test_iterators.h"
+#include "min_allocator.h"
+
+// test constraint container-compatible-range
+template <class M, class R>
+concept CanInsertRange = requires(M m, R&& r) { m.insert_range(std::forward<R>(r)); };
+
+using Map = std::flat_map<int, double>;
+
+static_assert(CanInsertRange<Map, std::ranges::subrange<std::pair<int, double>*>>);
+static_assert(CanInsertRange<Map, std::ranges::subrange<std::pair<short, double>*>>);
+static_assert(!CanInsertRange<Map, std::ranges::subrange<int*>>);
+static_assert(!CanInsertRange<Map, std::ranges::subrange<double*>>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+
+  {
+    using P                 = std::pair<int, int>;
+    using M                 = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+    using It                = forward_iterator<const P*>;
+    M m                     = {{10, 1}, {8, 2}, {5, 3}, {2, 4}, {1, 5}};
+    P ar[]                  = {{3, 1}, {1, 2}, {4, 3}, {1, 4}, {5, 5}, {9, 6}};
+    std::ranges::subrange r = {It(ar), It(ar + 6)};
+    static_assert(std::ranges::common_range<decltype(r)>);
+    m.insert_range(r);
+    assert((m == M{{1, 5}, {2, 4}, {3, 1}, {4, 3}, {5, 3}, {8, 2}, {9, 6}, {10, 1}}));
+  }
+  {
+    using P                 = std::pair<int, int>;
+    using M                 = std::flat_map<Key, Value, std::greater<>, KeyContainer, ValueContainer>;
+    using It                = cpp20_input_iterator<const P*>;
+    M m                     = {{8, 1}, {5, 2}, {3, 3}, {2, 4}};
+    P ar[]                  = {{3, 1}, {1, 2}, {4, 3}, {1, 4}, {5, 5}, {9, 6}};
+    std::ranges::subrange r = {It(ar), sentinel_wrapper<It>(It(ar + 6))};
+    static_assert(!std::ranges::common_range<decltype(r)>);
+    m.insert_range(r);
+    assert((m == M{{1, 2}, {2, 4}, {3, 3}, {4, 3}, {5, 2}, {8, 1}, {9, 6}}));
+  }
+  {
+    // The "uniquing" part uses the comparator, not operator==.
+    struct ModTen {
+      bool operator()(int a, int b) const { return (a % 10) < (b % 10); }
+    };
+    using P = std::pair<int, int>;
+    using M = std::flat_map<Key, Value, ModTen, KeyContainer, ValueContainer>;
+    M m     = {{21, 0}, {43, 0}, {15, 0}, {37, 0}};
+    P ar[]  = {{33, 1}, {18, 1}, {55, 1}, {18, 1}, {42, 1}};
+    m.insert_range(ar);
+    assert((m == M{{21, 0}, {42, 1}, {43, 0}, {15, 0}, {37, 0}, {18, 1}}));
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<int>>();
+  test<std::deque<int>, std::vector<int>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<int>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+  {
+    // Items are forwarded correctly from the input range (P2767).
+    std::pair<MoveOnly, MoveOnly> a[] = {{3, 3}, {1, 1}, {4, 4}, {1, 1}, {5, 5}};
+    std::flat_map<MoveOnly, MoveOnly> m;
+    m.insert_range(a | std::views::as_rvalue);
+    std::pair<MoveOnly, MoveOnly> expected[] = {{1, 1}, {3, 3}, {4, 4}, {5, 5}};
+    assert(std::ranges::equal(m, expected));
+  }
+  {
+    // The element type of the range doesn't need to be std::pair (P2767).
+    std::pair<int, int> pa[] = {{3, 3}, {1, 1}, {4, 4}, {1, 1}, {5, 5}};
+    std::deque<std::reference_wrapper<std::pair<int, int>>> a(pa, pa + 5);
+    std::flat_map<int, int> m;
+    m.insert_range(a);
+    std::pair<int, int> expected[] = {{1, 1}, {3, 3}, {4, 4}, {5, 5}};
+    assert(std::ranges::equal(m, expected));
+  }
+  {
+    auto insert_func = [](auto& m, const auto& newValues) { m.insert_range(newValues); };
+    test_insert_range_exception_guarantee(insert_func);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_range_stability.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_range_stability.pass.cpp
new file mode 100644
index 000000000000..fabcb1d216a7
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_range_stability.pass.cpp
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<container-compatible-range<value_type> R>
+//   void insert_range(R&& rg);
+//
+// libc++ uses stable_sort to ensure that flat_map's behavior matches map's,
+// in terms of which duplicate items are kept.
+// This tests a conforming extension.
+
+#include <algorithm>
+#include <cassert>
+#include <cstdint>
+#include <flat_map>
+#include <random>
+#include <ranges>
+#include <map>
+#include <vector>
+#include <utility>
+
+#include "test_macros.h"
+
+struct Mod256 {
+  bool operator()(int x, int y) const { return (x % 256) < (y % 256); }
+};
+
+int main(int, char**) {
+  {
+    std::mt19937 randomness;
+    std::pair<uint16_t, uint16_t> pairs[400];
+    for (int i = 0; i < 400; ++i) {
+      uint16_t r = randomness();
+      pairs[i]   = {r, r};
+    }
+
+    std::map<uint16_t, uint16_t, Mod256> m(pairs, pairs + 200);
+    std::flat_map<uint16_t, uint16_t, Mod256> fm(std::sorted_unique, m.begin(), m.end());
+    assert(std::ranges::equal(fm, m));
+
+    fm.insert_range(std::views::counted(pairs + 200, 200));
+    m.insert(pairs + 200, pairs + 400);
+    assert(fm.size() == m.size());
+    LIBCPP_ASSERT(std::ranges::equal(fm, m));
+  }
+
+  {
+    std::vector<std::pair<int, int>> v{{1, 2}, {1, 3}};
+    std::flat_map<int, int> m;
+    m.insert_range(v);
+    assert(m.size() == 1);
+    LIBCPP_ASSERT(m[1] == 2);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_rv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_rv.pass.cpp
new file mode 100644
index 000000000000..9ea7a6a63666
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_rv.pass.cpp
@@ -0,0 +1,124 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// class flat_map
+
+// pair<iterator, bool> insert( value_type&& v);
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "MoveOnly.h"
+#include "min_allocator.h"
+#include "test_macros.h"
+#include "../helpers.h"
+
+template <class Container, class Pair>
+void do_insert_rv_test() {
+  using M = Container;
+  using P = Pair;
+  using R = std::pair<typename M::iterator, bool>;
+  M m;
+  std::same_as<R> decltype(auto) r = m.insert(P(2, 2));
+  assert(r.second);
+  assert(r.first == m.begin());
+  assert(m.size() == 1);
+  assert(r.first->first == 2);
+  assert(r.first->second == 2);
+
+  r = m.insert(P(1, 1));
+  assert(r.second);
+  assert(r.first == m.begin());
+  assert(m.size() == 2);
+  assert(r.first->first == 1);
+  assert(r.first->second == 1);
+
+  r = m.insert(P(3, 3));
+  assert(r.second);
+  assert(r.first == std::ranges::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r.first->first == 3);
+  assert(r.first->second == 3);
+
+  r = m.insert(P(3, 3));
+  assert(!r.second);
+  assert(r.first == std::ranges::prev(m.end()));
+  assert(m.size() == 3);
+  assert(r.first->first == 3);
+  assert(r.first->second == 3);
+}
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  using P  = std::pair<Key, Value>;
+  using CP = std::pair<const Key, Value>;
+
+  do_insert_rv_test<M, P>();
+  do_insert_rv_test<M, CP>();
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<MoveOnly>>();
+  test<std::deque<int>, std::vector<MoveOnly>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<MoveOnly>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<MoveOnly, min_allocator<MoveOnly>>>();
+
+  {
+    using M = std::flat_map<int, MoveOnly>;
+    using R = std::pair<M::iterator, bool>;
+    M m;
+    R r = m.insert({2, MoveOnly(2)});
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 1);
+    assert(r.first->first == 2);
+    assert(r.first->second == 2);
+
+    r = m.insert({1, MoveOnly(1)});
+    assert(r.second);
+    assert(r.first == m.begin());
+    assert(m.size() == 2);
+    assert(r.first->first == 1);
+    assert(r.first->second == 1);
+
+    r = m.insert({3, MoveOnly(3)});
+    assert(r.second);
+    assert(r.first == std::ranges::prev(m.end()));
+    assert(m.size() == 3);
+    assert(r.first->first == 3);
+    assert(r.first->second == 3);
+
+    r = m.insert({3, MoveOnly(3)});
+    assert(!r.second);
+    assert(r.first == std::ranges::prev(m.end()));
+    assert(m.size() == 3);
+    assert(r.first->first == 3);
+    assert(r.first->second == 3);
+  }
+  {
+    auto insert_func = [](auto& m, auto key_arg, auto value_arg) {
+      using FlatMap    = std::decay_t<decltype(m)>;
+      using value_type = typename FlatMap::value_type;
+      value_type p(std::piecewise_construct, std::tuple(key_arg), std::tuple(value_arg));
+      m.insert(std::move(p));
+    };
+    test_emplace_exception_guarantee(insert_func);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_sorted_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_sorted_initializer_list.pass.cpp
new file mode 100644
index 000000000000..08d2caf34987
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_sorted_initializer_list.pass.cpp
@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// void insert(initializer_list<value_type> il);
+
+#include <flat_map>
+#include <cassert>
+#include <functional>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  using V = std::pair<const Key, Value>;
+  M m     = {{1, 1}, {1, 1.5}, {1, 2}, {3, 1}, {3, 1.5}, {3, 2}};
+  m.insert(std::sorted_unique,
+           {
+               {0, 1},
+               {1, 2},
+               {2, 1},
+               {4, 1},
+           });
+  assert(m.size() == 5);
+  assert(std::distance(m.begin(), m.end()) == 5);
+  assert(*m.begin() == V(0, 1));
+  assert(*std::next(m.begin()) == V(1, 1));
+  assert(*std::next(m.begin(), 2) == V(2, 1));
+  assert(*std::next(m.begin(), 3) == V(3, 1));
+  assert(*std::next(m.begin(), 4) == V(4, 1));
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto insert_func = [](auto& m, const auto& newValues) {
+      using FlatMap                        = std::decay_t<decltype(m)>;
+      using value_type                     = typename FlatMap::value_type;
+      std::initializer_list<value_type> il = {{newValues[0].first, newValues[0].second}};
+      m.insert(std::sorted_unique, il);
+    };
+    test_insert_range_exception_guarantee(insert_func);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_sorted_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_sorted_iter_iter.pass.cpp
new file mode 100644
index 000000000000..18a3b571a419
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_sorted_iter_iter.pass.cpp
@@ -0,0 +1,86 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template <class InputIterator>
+//   void insert(sorted_unique_t, InputIterator first, InputIterator last);
+
+#include <flat_map>
+#include <cassert>
+#include <functional>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "test_iterators.h"
+#include "min_allocator.h"
+
+// test constraint InputIterator
+template <class M, class... Args>
+concept CanInsert = requires(M m, Args&&... args) { m.insert(std::forward<Args>(args)...); };
+
+using Map  = std::flat_map<int, int>;
+using Pair = std::pair<int, int>;
+
+static_assert(CanInsert<Map, std::sorted_unique_t, Pair*, Pair*>);
+static_assert(CanInsert<Map, std::sorted_unique_t, cpp17_input_iterator<Pair*>, cpp17_input_iterator<Pair*>>);
+static_assert(!CanInsert<Map, std::sorted_unique_t, int, int>);
+static_assert(!CanInsert<Map, std::sorted_unique_t, cpp20_input_iterator<Pair*>, cpp20_input_iterator<Pair*>>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using P     = std::pair<Key, Value>;
+
+  P ar1[] = {
+      P(1, 1),
+      P(2, 1),
+      P(3, 1),
+  };
+
+  P ar2[] = {
+      P(0, 1),
+      P(2, 2),
+      P(4, 1),
+  };
+
+  M m;
+  m.insert(
+      std::sorted_unique, cpp17_input_iterator<P*>(ar1), cpp17_input_iterator<P*>(ar1 + sizeof(ar1) / sizeof(ar1[0])));
+  assert(m.size() == 3);
+  M expected{{1, 1}, {2, 1}, {3, 1}};
+  assert(m == expected);
+
+  m.insert(
+      std::sorted_unique, cpp17_input_iterator<P*>(ar2), cpp17_input_iterator<P*>(ar2 + sizeof(ar2) / sizeof(ar2[0])));
+  assert(m.size() == 5);
+  M expected2{{0, 1}, {1, 1}, {2, 1}, {3, 1}, {4, 1}};
+  assert(m == expected2);
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    auto insert_func = [](auto& m, const auto& newValues) {
+      m.insert(std::sorted_unique, newValues.begin(), newValues.end());
+    };
+    test_insert_range_exception_guarantee(insert_func);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_transparent.pass.cpp
new file mode 100644
index 000000000000..75cabb70630f
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_transparent.pass.cpp
@@ -0,0 +1,167 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<class K> pair<iterator, bool> insert(P&& x);
+// template<class K> iterator insert(const_iterator hint, P&& x);
+
+#include <algorithm>
+#include <compare>
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <tuple>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "test_iterators.h"
+#include "min_allocator.h"
+
+// Constraints: is_constructible_v<pair<key_type, mapped_type>, P> is true.
+template <class M, class... Args>
+concept CanInsert = requires(M m, Args&&... args) { m.insert(std::forward<Args>(args)...); };
+
+using Map  = std::flat_map<int, double>;
+using Iter = Map::const_iterator;
+
+static_assert(CanInsert<Map, std::pair<short, double>&&>);
+static_assert(CanInsert<Map, Iter, std::pair<short, double>&&>);
+static_assert(CanInsert<Map, std::tuple<short, double>&&>);
+static_assert(CanInsert<Map, Iter, std::tuple<short, double>&&>);
+static_assert(!CanInsert<Map, int>);
+static_assert(!CanInsert<Map, Iter, int>);
+
+static int expensive_comparisons = 0;
+static int cheap_comparisons     = 0;
+
+struct CompareCounter {
+  int i_ = 0;
+  CompareCounter(int i) : i_(i) {}
+  friend auto operator<=>(const CompareCounter& x, const CompareCounter& y) {
+    expensive_comparisons += 1;
+    return x.i_ <=> y.i_;
+  }
+  bool operator==(const CompareCounter&) const = default;
+  friend auto operator<=>(const CompareCounter& x, int y) {
+    cheap_comparisons += 1;
+    return x.i_ <=> y;
+  }
+};
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  const std::pair<int, int> expected[] = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}};
+  {
+    // insert(P&&)
+    //   Unlike flat_set, here we can't use key_compare to compare value_type versus P,
+    //   so we must eagerly convert to value_type.
+    M m                   = {{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    expensive_comparisons = 0;
+    cheap_comparisons     = 0;
+    std::same_as<std::pair<typename M::iterator, bool>> auto p =
+        m.insert(std::make_pair(3, 3)); // conversion happens first
+    assert(expensive_comparisons >= 2);
+    assert(cheap_comparisons == 0);
+    assert(p == std::make_pair(m.begin() + 2, true));
+    assert(std::ranges::equal(m, expected));
+  }
+  {
+    // insert(const_iterator, P&&)
+    M m                                        = {{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    expensive_comparisons                      = 0;
+    cheap_comparisons                          = 0;
+    std::same_as<typename M::iterator> auto it = m.insert(m.begin(), std::make_pair(3, 3));
+    assert(expensive_comparisons >= 2);
+    assert(cheap_comparisons == 0);
+    assert(it == m.begin() + 2);
+    assert(std::ranges::equal(m, expected));
+  }
+  {
+    // insert(value_type&&)
+    M m                   = {{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    expensive_comparisons = 0;
+    cheap_comparisons     = 0;
+    std::same_as<std::pair<typename M::iterator, bool>> auto p =
+        m.insert(std::make_pair(3, 3)); // conversion happens last
+    assert(expensive_comparisons >= 2);
+    assert(cheap_comparisons == 0);
+    assert(p == std::make_pair(m.begin() + 2, true));
+    assert(std::ranges::equal(m, expected));
+  }
+  {
+    // insert(const_iterator, value_type&&)
+    M m                                        = {{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    expensive_comparisons                      = 0;
+    cheap_comparisons                          = 0;
+    std::same_as<typename M::iterator> auto it = m.insert(m.begin(), std::make_pair(3, 3));
+    assert(expensive_comparisons >= 2);
+    assert(cheap_comparisons == 0);
+    assert(it == m.begin() + 2);
+    assert(std::ranges::equal(m, expected));
+  }
+  {
+    // emplace(Args&&...)
+    M m                   = {{1, 1}, {2, 2}, {4, 4}, {5, 5}};
+    expensive_comparisons = 0;
+    cheap_comparisons     = 0;
+    std::same_as<std::pair<typename M::iterator, bool>> auto p =
+        m.emplace(std::make_pair(3, 3)); // conversion happens first
+    assert(expensive_comparisons >= 2);
+    assert(cheap_comparisons == 0);
+    assert(p == std::make_pair(m.begin() + 2, true));
+    assert(std::ranges::equal(m, expected));
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<CompareCounter>, std::vector<double>>();
+  test<std::deque<CompareCounter>, std::vector<double>>();
+  test<MinSequenceContainer<CompareCounter>, MinSequenceContainer<double>>();
+  test<std::vector<CompareCounter, min_allocator<CompareCounter>>, std::vector<double, min_allocator<double>>>();
+
+  {
+    // no ambiguity between insert(pos, P&&) and insert(first, last)
+    using M = std::flat_map<int, int>;
+    struct Evil {
+      operator M::value_type() const;
+      operator M::const_iterator() const;
+    };
+    std::flat_map<int, int> m;
+    ASSERT_SAME_TYPE(decltype(m.insert(Evil())), std::pair<M::iterator, bool>);
+    ASSERT_SAME_TYPE(decltype(m.insert(m.begin(), Evil())), M::iterator);
+    ASSERT_SAME_TYPE(decltype(m.insert(m.begin(), m.end())), void);
+  }
+  {
+    auto insert_func = [](auto& m, auto key_arg, auto value_arg) {
+      using FlatMap    = std::decay_t<decltype(m)>;
+      using tuple_type = std::tuple<typename FlatMap::key_type, typename FlatMap::mapped_type>;
+      tuple_type t(key_arg, value_arg);
+      m.insert(t);
+    };
+    test_emplace_exception_guarantee(insert_func);
+  }
+  {
+    auto insert_func_iter = [](auto& m, auto key_arg, auto value_arg) {
+      using FlatMap    = std::decay_t<decltype(m)>;
+      using tuple_type = std::tuple<typename FlatMap::key_type, typename FlatMap::mapped_type>;
+      tuple_type t(key_arg, value_arg);
+      m.insert(m.begin(), t);
+    };
+    test_emplace_exception_guarantee(insert_func_iter);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/replace.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/replace.pass.cpp
new file mode 100644
index 000000000000..5ca811d76152
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/replace.pass.cpp
@@ -0,0 +1,80 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// void replace(key_container_type&& key_cont, mapped_container_type&& mapped_cont);
+
+#include <algorithm>
+#include <deque>
+#include <concepts>
+#include <flat_map>
+#include <functional>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class T, class... Args>
+concept CanReplace = requires(T t, Args&&... args) { t.replace(std::forward<Args>(args)...); };
+
+using Map = std::flat_map<int, int>;
+static_assert(CanReplace<Map, std::vector<int>, std::vector<int>>);
+static_assert(!CanReplace<Map, const std::vector<int>&, std::vector<int>>);
+static_assert(!CanReplace<Map, std::vector<int>, const std::vector<int>&>);
+static_assert(!CanReplace<Map, const std::vector<int>&, const std::vector<int>&>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  M m                       = M({1, 2, 3}, {4, 5, 6});
+  KeyContainer new_keys     = {7, 8};
+  ValueContainer new_values = {9, 10};
+  auto expected_keys        = new_keys;
+  auto expected_values      = new_values;
+  m.replace(std::move(new_keys), std::move(new_values));
+  assert(m.size() == 2);
+  assert(std::ranges::equal(m.keys(), expected_keys));
+  assert(std::ranges::equal(m.values(), expected_values));
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  {
+#ifndef TEST_HAS_NO_EXCEPTIONS
+    using KeyContainer   = std::vector<int>;
+    using ValueContainer = ThrowOnMoveContainer<int>;
+    using M              = std::flat_map<int, int, std::ranges::less, KeyContainer, ValueContainer>;
+
+    M m;
+    m.emplace(1, 1);
+    m.emplace(2, 2);
+    try {
+      KeyContainer new_keys{3, 4};
+      ValueContainer new_values{5, 6};
+      m.replace(std::move(new_keys), std::move(new_values));
+      assert(false);
+    } catch (int) {
+      check_invariant(m);
+      // In libc++, we clear the map
+      LIBCPP_ASSERT(m.size() == 0);
+    }
+#endif
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_exception.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_exception.pass.cpp
new file mode 100644
index 000000000000..f9708aac62c7
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_exception.pass.cpp
@@ -0,0 +1,78 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// `check_assertion.h` requires Unix headers and regex support.
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: no-localization
+// UNSUPPORTED: no-exceptions
+
+// <flat_map>
+
+// void swap(flat_map& y) noexcept;
+// friend void swap(flat_map& x, flat_map& y) noexcept
+
+// Test that std::terminate is called if any exception is thrown during swap
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+#include <functional>
+#include <vector>
+
+#include "test_macros.h"
+#include "../helpers.h"
+#include "check_assertion.h"
+
+template <class F>
+void test_swap_exception_guarantee([[maybe_unused]] F&& swap_function) {
+  {
+    // key swap throws
+    using KeyContainer   = ThrowOnMoveContainer<int>;
+    using ValueContainer = std::vector<int>;
+    using M              = std::flat_map<int, int, TransparentComparator, KeyContainer, ValueContainer>;
+
+    M m1, m2;
+    m1.emplace(1, 1);
+    m1.emplace(2, 2);
+    m2.emplace(3, 3);
+    m2.emplace(4, 4);
+    // swap is noexcept
+    EXPECT_STD_TERMINATE([&] { swap_function(m1, m2); });
+  }
+
+  {
+    // value swap throws
+    using KeyContainer   = std::vector<int>;
+    using ValueContainer = ThrowOnMoveContainer<int>;
+    using M              = std::flat_map<int, int, TransparentComparator, KeyContainer, ValueContainer>;
+
+    M m1, m2;
+    m1.emplace(1, 1);
+    m1.emplace(2, 2);
+    m2.emplace(3, 3);
+    m2.emplace(4, 4);
+
+    // swap is noexcept
+    EXPECT_STD_TERMINATE([&] { swap_function(m1, m2); });
+  }
+}
+
+int main(int, char**) {
+  {
+    auto swap_func = [](auto& m1, auto& m2) { swap(m1, m2); };
+    test_swap_exception_guarantee(swap_func);
+  }
+
+  {
+    auto swap_func = [](auto& m1, auto& m2) { m1.swap(m2); };
+    test_swap_exception_guarantee(swap_func);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_free.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_free.pass.cpp
new file mode 100644
index 000000000000..98c60c1488cf
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_free.pass.cpp
@@ -0,0 +1,97 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// friend void swap(flat_map& x, flat_map& y) noexcept
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "MoveOnly.h"
+#include "min_allocator.h"
+#include "test_macros.h"
+#include "../helpers.h"
+
+// test noexcept
+
+template <class T>
+concept NoExceptAdlSwap = requires(T t1, T t2) {
+  { swap(t1, t2) } noexcept;
+};
+
+static_assert(NoExceptAdlSwap<std::flat_map<int, int>>);
+
+#ifndef TEST_HAS_NO_EXCEPTIONS
+static_assert(
+    NoExceptAdlSwap<std::flat_map<int, int, std::less<int>, ThrowOnMoveContainer<int>, ThrowOnMoveContainer<int>>>);
+#endif
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using V     = std::pair<const Key, Value>;
+
+  {
+    M m1;
+    M m2;
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+  {
+    V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+    M m1;
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+  {
+    V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+    M m2;
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+  {
+    V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+    V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+    M m1_save = m1;
+    M m2_save = m2;
+    swap(m1, m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_member.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_member.pass.cpp
new file mode 100644
index 000000000000..d2d8f5673ede
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_member.pass.cpp
@@ -0,0 +1,95 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// void swap(flat_map& y) noexcept;
+
+#include <flat_map>
+#include <cassert>
+#include <deque>
+#include <functional>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "MoveOnly.h"
+#include "min_allocator.h"
+#include "test_macros.h"
+#include "../helpers.h"
+
+// test noexcept
+
+template <class T>
+concept NoExceptMemberSwap = requires(T t1, T t2) {
+  { t1.swap(t2) } noexcept;
+};
+
+static_assert(NoExceptMemberSwap<std::flat_map<int, int>>);
+#ifndef TEST_HAS_NO_EXCEPTIONS
+static_assert(
+    NoExceptMemberSwap<std::flat_map<int, int, std::less<int>, ThrowOnMoveContainer<int>, ThrowOnMoveContainer<int>>>);
+#endif
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+  using V     = std::pair<const Key, Value>;
+  {
+    M m1;
+    M m2;
+    M m1_save = m1;
+    M m2_save = m2;
+    m1.swap(m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+  {
+    V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+    M m1;
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+    M m1_save = m1;
+    M m2_save = m2;
+    m1.swap(m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+  {
+    V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+    M m2;
+    M m1_save = m1;
+    M m2_save = m2;
+    m1.swap(m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+  {
+    V ar1[] = {V(1, 1), V(2, 2), V(3, 3), V(4, 4)};
+    V ar2[] = {V(5, 5), V(6, 6), V(7, 7), V(8, 8), V(9, 9), V(10, 10), V(11, 11), V(12, 12)};
+    M m1(ar1, ar1 + sizeof(ar1) / sizeof(ar1[0]));
+    M m2(ar2, ar2 + sizeof(ar2) / sizeof(ar2[0]));
+    M m1_save = m1;
+    M m2_save = m2;
+    m1.swap(m2);
+    assert(m1 == m2_save);
+    assert(m2 == m1_save);
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<double>>();
+  test<std::deque<int>, std::vector<double>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/try_emplace.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/try_emplace.pass.cpp
new file mode 100644
index 000000000000..4be2fe1c4333
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/try_emplace.pass.cpp
@@ -0,0 +1,246 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<class... Args>
+//   pair<iterator, bool> try_emplace(const key_type& k, Args&&... args);
+// template<class... Args>
+//   pair<iterator, bool> try_emplace(key_type&& k, Args&&... args);
+// template<class... Args>
+//   iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args);
+// template<class... Args>
+//   iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args);
+
+#include <flat_map>
+#include <cassert>
+#include <functional>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "../helpers.h"
+#include "min_allocator.h"
+#include "../../../Emplaceable.h"
+
+// Constraints: is_constructible_v<mapped_type, Args...> is true.
+template <class M, class... Args>
+concept CanTryEmplace = requires(M m, Args&&... args) { m.try_emplace(std::forward<Args>(args)...); };
+
+using Map  = std::flat_map<Emplaceable, Emplaceable>;
+using Iter = typename Map::const_iterator;
+static_assert(!CanTryEmplace<Map>);
+
+static_assert(CanTryEmplace<Map, const Emplaceable&>);
+static_assert(CanTryEmplace<Map, const Emplaceable&, Emplaceable>);
+static_assert(CanTryEmplace<Map, const Emplaceable&, int, double>);
+static_assert(!CanTryEmplace<Map, const Emplaceable&, const Emplaceable&>);
+static_assert(!CanTryEmplace<Map, const Emplaceable&, int>);
+
+static_assert(CanTryEmplace<Map, Emplaceable>);
+static_assert(CanTryEmplace<Map, Emplaceable, Emplaceable>);
+static_assert(CanTryEmplace<Map, Emplaceable, int, double>);
+static_assert(!CanTryEmplace<Map, Emplaceable, const Emplaceable&>);
+static_assert(!CanTryEmplace<Map, Emplaceable, int>);
+
+static_assert(CanTryEmplace<Map, Iter, const Emplaceable&>);
+static_assert(CanTryEmplace<Map, Iter, const Emplaceable&, Emplaceable>);
+static_assert(CanTryEmplace<Map, Iter, const Emplaceable&, int, double>);
+static_assert(!CanTryEmplace<Map, Iter, const Emplaceable&, const Emplaceable&>);
+static_assert(!CanTryEmplace<Map, Iter, const Emplaceable&, int>);
+
+static_assert(CanTryEmplace<Map, Iter, Emplaceable>);
+static_assert(CanTryEmplace<Map, Iter, Emplaceable, Emplaceable>);
+static_assert(CanTryEmplace<Map, Iter, Emplaceable, int, double>);
+static_assert(!CanTryEmplace<Map, Iter, Emplaceable, const Emplaceable&>);
+static_assert(!CanTryEmplace<Map, Iter, Emplaceable, int>);
+
+template <class KeyContainer, class ValueContainer>
+void test_ck() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  { // pair<iterator, bool> try_emplace(const key_type& k, Args&&... args);
+    using R = std::pair<typename M::iterator, bool>;
+    M m;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(i, Moveable(i, (double)i));
+
+    assert(m.size() == 10);
+
+    Moveable mv1(3, 3.0);
+    for (int i = 0; i < 20; i += 2) {
+      std::same_as<R> decltype(auto) r = m.try_emplace(i, std::move(mv1));
+      assert(m.size() == 10);
+      assert(!r.second);           // was not inserted
+      assert(!mv1.moved());        // was not moved from
+      assert(r.first->first == i); // key
+    }
+
+    std::same_as<R> decltype(auto) r2 = m.try_emplace(-1, std::move(mv1));
+    assert(m.size() == 11);
+    assert(r2.second);                   // was inserted
+    assert(mv1.moved());                 // was moved from
+    assert(r2.first->first == -1);       // key
+    assert(r2.first->second.get() == 3); // value
+
+    Moveable mv2(5, 3.0);
+    std::same_as<R> decltype(auto) r3 = m.try_emplace(5, std::move(mv2));
+    assert(m.size() == 12);
+    assert(r3.second);                   // was inserted
+    assert(mv2.moved());                 // was moved from
+    assert(r3.first->first == 5);        // key
+    assert(r3.first->second.get() == 5); // value
+
+    Moveable mv3(-1, 3.0);
+    std::same_as<R> decltype(auto) r4 = m.try_emplace(117, std::move(mv2));
+    assert(m.size() == 13);
+    assert(r4.second);                    // was inserted
+    assert(mv2.moved());                  // was moved from
+    assert(r4.first->first == 117);       // key
+    assert(r4.first->second.get() == -1); // value
+  }
+
+  { // iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args);
+    using R = typename M::iterator;
+    M m;
+    for (int i = 0; i < 20; i += 2)
+      m.try_emplace(i, Moveable(i, (double)i));
+    assert(m.size() == 10);
+    typename M::const_iterator it = m.find(2);
+
+    Moveable mv1(3, 3.0);
+    for (int i = 0; i < 20; i += 2) {
+      std::same_as<R> decltype(auto) r1 = m.try_emplace(it, i, std::move(mv1));
+      assert(m.size() == 10);
+      assert(!mv1.moved());          // was not moved from
+      assert(r1->first == i);        // key
+      assert(r1->second.get() == i); // value
+    }
+
+    std::same_as<R> decltype(auto) r2 = m.try_emplace(it, 3, std::move(mv1));
+    assert(m.size() == 11);
+    assert(mv1.moved());           // was moved from
+    assert(r2->first == 3);        // key
+    assert(r2->second.get() == 3); // value
+  }
+}
+
+template <class KeyContainer, class ValueContainer>
+void test_rk() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  { // pair<iterator, bool> try_emplace(key_type&& k, Args&&... args);
+    using R = std::pair<typename M::iterator, bool>;
+    M m;
+    for (int i = 0; i < 20; i += 2) {
+      m.emplace(Moveable(i, (double)i), Moveable(i + 1, (double)i + 1));
+    }
+    assert(m.size() == 10);
+
+    Moveable mvkey1(2, 2.0);
+    Moveable mv1(4, 4.0);
+    std::same_as<R> decltype(auto) r1 = m.try_emplace(std::move(mvkey1), std::move(mv1));
+    assert(m.size() == 10);
+    assert(!r1.second);                // was not inserted
+    assert(!mv1.moved());              // was not moved from
+    assert(!mvkey1.moved());           // was not moved from
+    assert(r1.first->first == mvkey1); // key
+
+    Moveable mvkey2(3, 3.0);
+    std::same_as<R> decltype(auto) r2 = m.try_emplace(std::move(mvkey2), std::move(mv1));
+    assert(m.size() == 11);
+    assert(r2.second);                   // was inserted
+    assert(mv1.moved());                 // was moved from
+    assert(mvkey2.moved());              // was moved from
+    assert(r2.first->first.get() == 3);  // key
+    assert(r2.first->second.get() == 4); // value
+  }
+
+  { // iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args);
+    using R = typename M::iterator;
+    M m;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(Moveable(i, (double)i), Moveable(i + 1, (double)i + 1));
+    assert(m.size() == 10);
+    typename M::const_iterator it = std::next(m.cbegin());
+
+    Moveable mvkey1(2, 2.0);
+    Moveable mv1(4, 4.0);
+    std::same_as<R> decltype(auto) r1 = m.try_emplace(it, std::move(mvkey1), std::move(mv1));
+    assert(m.size() == 10);
+    assert(!mv1.moved());        // was not moved from
+    assert(!mvkey1.moved());     // was not moved from
+    assert(r1->first == mvkey1); // key
+
+    Moveable mvkey2(3, 3.0);
+    std::same_as<R> decltype(auto) r2 = m.try_emplace(it, std::move(mvkey2), std::move(mv1));
+    assert(m.size() == 11);
+    assert(mv1.moved());           // was moved from
+    assert(mvkey2.moved());        // was moved from
+    assert(r2->first.get() == 3);  // key
+    assert(r2->second.get() == 4); // value
+  }
+}
+
+int main(int, char**) {
+  test_ck<std::vector<int>, std::vector<Moveable>>();
+  test_ck<std::deque<int>, std::vector<Moveable>>();
+  test_ck<MinSequenceContainer<int>, MinSequenceContainer<Moveable>>();
+  test_ck<std::vector<int, min_allocator<int>>, std::vector<Moveable, min_allocator<Moveable>>>();
+
+  test_rk<std::vector<Moveable>, std::vector<Moveable>>();
+  test_rk<std::deque<Moveable>, std::vector<Moveable>>();
+  test_rk<MinSequenceContainer<Moveable>, MinSequenceContainer<Moveable>>();
+  test_rk<std::vector<Moveable, min_allocator<Moveable>>, std::vector<Moveable, min_allocator<Moveable>>>();
+
+  {
+    auto try_emplace_ck = [](auto& m, auto key_arg, auto value_arg) {
+      using M   = std::decay_t<decltype(m)>;
+      using Key = typename M::key_type;
+      const Key key{key_arg};
+      m.try_emplace(key, value_arg);
+    };
+    test_emplace_exception_guarantee(try_emplace_ck);
+  }
+
+  {
+    auto try_emplace_rk = [](auto& m, auto key_arg, auto value_arg) {
+      using M   = std::decay_t<decltype(m)>;
+      using Key = typename M::key_type;
+      m.try_emplace(Key{key_arg}, value_arg);
+    };
+    test_emplace_exception_guarantee(try_emplace_rk);
+  }
+
+  {
+    auto try_emplace_iter_ck = [](auto& m, auto key_arg, auto value_arg) {
+      using M   = std::decay_t<decltype(m)>;
+      using Key = typename M::key_type;
+      const Key key{key_arg};
+      m.try_emplace(m.begin(), key, value_arg);
+    };
+    test_emplace_exception_guarantee(try_emplace_iter_ck);
+  }
+
+  {
+    auto try_emplace_iter_rk = [](auto& m, auto key_arg, auto value_arg) {
+      using M   = std::decay_t<decltype(m)>;
+      using Key = typename M::key_type;
+      m.try_emplace(m.begin(), Key{key_arg}, value_arg);
+    };
+    test_emplace_exception_guarantee(try_emplace_iter_rk);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/try_emplace_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/try_emplace_transparent.pass.cpp
new file mode 100644
index 000000000000..21fda4378096
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/try_emplace_transparent.pass.cpp
@@ -0,0 +1,182 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<class K, class... Args>
+//   pair<iterator, bool> try_emplace(K&& k, Args&&... args);
+// template<class K, class... Args>
+//   iterator try_emplace(const_iterator hint, K&& k, Args&&... args);
+
+#include <flat_map>
+#include <cassert>
+#include <functional>
+#include <deque>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "../helpers.h"
+#include "min_allocator.h"
+#include "../../../Emplaceable.h"
+
+// Constraints:
+// The qualified-id Compare::is_transparent is valid and denotes a type.
+// is_constructible_v<key_type, K> is true.
+// is_constructible_v<mapped_type, Args...> is true.
+// For the first overload, is_convertible_v<K&&, const_iterator> and is_convertible_v<K&&, iterator> are both false
+template <class M, class... Args>
+concept CanTryEmplace = requires(M m, Args&&... args) { m.try_emplace(std::forward<Args>(args)...); };
+
+using TransparentMap    = std::flat_map<int, Emplaceable, TransparentComparator>;
+using NonTransparentMap = std::flat_map<int, Emplaceable, NonTransparentComparator>;
+
+using TransparentMapIter      = typename TransparentMap::iterator;
+using TransparentMapConstIter = typename TransparentMap::const_iterator;
+
+static_assert(!CanTryEmplace<TransparentMap>);
+static_assert(!CanTryEmplace<NonTransparentMap>);
+
+static_assert(CanTryEmplace<TransparentMap, ConvertibleTransparent<int>>);
+static_assert(CanTryEmplace<TransparentMap, ConvertibleTransparent<int>, Emplaceable>);
+static_assert(CanTryEmplace<TransparentMap, ConvertibleTransparent<int>, int, double>);
+static_assert(!CanTryEmplace<TransparentMap, ConvertibleTransparent<int>, const Emplaceable&>);
+static_assert(!CanTryEmplace<TransparentMap, ConvertibleTransparent<int>, int>);
+static_assert(!CanTryEmplace<TransparentMap, NonConvertibleTransparent<int>, Emplaceable>);
+static_assert(!CanTryEmplace<NonTransparentMap, NonConvertibleTransparent<int>, Emplaceable>);
+static_assert(!CanTryEmplace<TransparentMap, ConvertibleTransparent<int>, int>);
+static_assert(!CanTryEmplace<TransparentMap, TransparentMapIter, Emplaceable>);
+static_assert(!CanTryEmplace<TransparentMap, TransparentMapConstIter, Emplaceable>);
+
+static_assert(CanTryEmplace<TransparentMap, TransparentMapConstIter, ConvertibleTransparent<int>>);
+static_assert(CanTryEmplace<TransparentMap, TransparentMapConstIter, ConvertibleTransparent<int>, Emplaceable>);
+static_assert(CanTryEmplace<TransparentMap, TransparentMapConstIter, ConvertibleTransparent<int>, int, double>);
+static_assert(!CanTryEmplace<TransparentMap, TransparentMapConstIter, ConvertibleTransparent<int>, const Emplaceable&>);
+static_assert(!CanTryEmplace<TransparentMap, TransparentMapConstIter, ConvertibleTransparent<int>, int>);
+static_assert(!CanTryEmplace<TransparentMap, TransparentMapConstIter, NonConvertibleTransparent<int>, Emplaceable>);
+static_assert(!CanTryEmplace<NonTransparentMap, TransparentMapConstIter, NonConvertibleTransparent<int>, Emplaceable>);
+static_assert(!CanTryEmplace<TransparentMap, TransparentMapConstIter, ConvertibleTransparent<int>, int>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  { // pair<iterator, bool> try_emplace(K&& k, Args&&... args);
+    using R = std::pair<typename M::iterator, bool>;
+    M m;
+    for (int i = 0; i < 20; i += 2)
+      m.emplace(i, Moveable(i, (double)i));
+
+    assert(m.size() == 10);
+
+    Moveable mv1(3, 3.0);
+    for (int i = 0; i < 20; i += 2) {
+      std::same_as<R> decltype(auto) r = m.try_emplace(ConvertibleTransparent<int>{i}, std::move(mv1));
+      assert(m.size() == 10);
+      assert(!r.second);           // was not inserted
+      assert(!mv1.moved());        // was not moved from
+      assert(r.first->first == i); // key
+    }
+
+    std::same_as<R> decltype(auto) r2 = m.try_emplace(ConvertibleTransparent<int>{-1}, std::move(mv1));
+    assert(m.size() == 11);
+    assert(r2.second);                   // was inserted
+    assert(mv1.moved());                 // was moved from
+    assert(r2.first->first == -1);       // key
+    assert(r2.first->second.get() == 3); // value
+
+    Moveable mv2(5, 3.0);
+    std::same_as<R> decltype(auto) r3 = m.try_emplace(ConvertibleTransparent<int>{5}, std::move(mv2));
+    assert(m.size() == 12);
+    assert(r3.second);                   // was inserted
+    assert(mv2.moved());                 // was moved from
+    assert(r3.first->first == 5);        // key
+    assert(r3.first->second.get() == 5); // value
+
+    Moveable mv3(-1, 3.0);
+    std::same_as<R> decltype(auto) r4 = m.try_emplace(ConvertibleTransparent<int>{117}, std::move(mv2));
+    assert(m.size() == 13);
+    assert(r4.second);                    // was inserted
+    assert(mv2.moved());                  // was moved from
+    assert(r4.first->first == 117);       // key
+    assert(r4.first->second.get() == -1); // value
+  }
+
+  { // iterator try_emplace(const_iterator hint, K&& k, Args&&... args);
+    using R = typename M::iterator;
+    M m;
+    for (int i = 0; i < 20; i += 2)
+      m.try_emplace(i, Moveable(i, (double)i));
+    assert(m.size() == 10);
+    typename M::const_iterator it = m.find(2);
+
+    Moveable mv1(3, 3.0);
+    for (int i = 0; i < 20; i += 2) {
+      std::same_as<R> decltype(auto) r1 = m.try_emplace(it, ConvertibleTransparent<int>{i}, std::move(mv1));
+      assert(m.size() == 10);
+      assert(!mv1.moved());          // was not moved from
+      assert(r1->first == i);        // key
+      assert(r1->second.get() == i); // value
+    }
+
+    std::same_as<R> decltype(auto) r2 = m.try_emplace(it, ConvertibleTransparent<int>{3}, std::move(mv1));
+    assert(m.size() == 11);
+    assert(mv1.moved());           // was moved from
+    assert(r2->first == 3);        // key
+    assert(r2->second.get() == 3); // value
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<Moveable>>();
+  test<std::deque<int>, std::vector<Moveable>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<Moveable>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<Moveable, min_allocator<Moveable>>>();
+
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_map<int, int, TransparentComparator> m(std::sorted_unique, {{1, 1}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto p = m.try_emplace(ConvertibleTransparent<int>{3}, 3);
+    assert(!p.second);
+    assert(transparent_used);
+  }
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_map<int, int, TransparentComparator> m(std::sorted_unique, {{1, 1}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto p = m.try_emplace(m.begin(), ConvertibleTransparent<int>{3}, 3);
+    assert(p->second == 3);
+    assert(transparent_used);
+  }
+  {
+    auto try_emplace = [](auto& m, auto key_arg, auto value_arg) {
+      using M   = std::decay_t<decltype(m)>;
+      using Key = typename M::key_type;
+      m.try_emplace(ConvertibleTransparent<Key>{key_arg}, value_arg);
+    };
+    test_emplace_exception_guarantee(try_emplace);
+  }
+
+  {
+    auto try_emplace_iter = [](auto& m, auto key_arg, auto value_arg) {
+      using M   = std::decay_t<decltype(m)>;
+      using Key = typename M::key_type;
+      m.try_emplace(m.begin(), ConvertibleTransparent<Key>{key_arg}, value_arg);
+    };
+    test_emplace_exception_guarantee(try_emplace_iter);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.observers/comp.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.observers/comp.pass.cpp
new file mode 100644
index 000000000000..d86224952dee
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.observers/comp.pass.cpp
@@ -0,0 +1,96 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// key_compare key_comp() const;
+// value_compare value_comp() const;
+
+#include <cassert>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+
+#include "test_macros.h"
+
+int main(int, char**) {
+  {
+    using M    = std::flat_map<int, char>;
+    using Comp = std::less<int>; // the default
+    M m        = {};
+    ASSERT_SAME_TYPE(M::key_compare, Comp);
+    static_assert(!std::is_same_v<M::value_compare, Comp>);
+    ASSERT_SAME_TYPE(decltype(m.key_comp()), Comp);
+    ASSERT_SAME_TYPE(decltype(m.value_comp()), M::value_compare);
+    Comp kc = m.key_comp();
+    assert(kc(1, 2));
+    assert(!kc(2, 1));
+    auto vc = m.value_comp();
+    ASSERT_SAME_TYPE(decltype(vc(std::make_pair(1, 2), std::make_pair(1, 2))), bool);
+    assert(vc({1, '2'}, {2, '1'}));
+    assert(!vc({2, '1'}, {1, '2'}));
+  }
+  {
+    using Comp = std::function<bool(int, int)>;
+    using M    = std::flat_map<int, int, Comp>;
+    Comp comp  = std::greater<int>();
+    M m({}, comp);
+    ASSERT_SAME_TYPE(M::key_compare, Comp);
+    ASSERT_SAME_TYPE(decltype(m.key_comp()), Comp);
+    ASSERT_SAME_TYPE(decltype(m.value_comp()), M::value_compare);
+    Comp kc = m.key_comp();
+    assert(!kc(1, 2));
+    assert(kc(2, 1));
+    auto vc = m.value_comp();
+    auto a  = std::make_pair(1, 2);
+    ASSERT_SAME_TYPE(decltype(vc(a, a)), bool);
+    static_assert(!noexcept(vc(a, a)));
+    assert(!vc({1, 2}, {2, 1}));
+    assert(vc({2, 1}, {1, 2}));
+  }
+  {
+    using Comp = std::less<>;
+    using M    = std::flat_map<int, int, Comp>;
+    M m        = {};
+    ASSERT_SAME_TYPE(M::key_compare, Comp);
+    ASSERT_SAME_TYPE(decltype(m.key_comp()), Comp);
+    ASSERT_SAME_TYPE(decltype(m.value_comp()), M::value_compare);
+    Comp kc = m.key_comp();
+    assert(kc(1, 2));
+    assert(!kc(2, 1));
+    auto vc = m.value_comp();
+    auto a  = std::make_pair(1, 2);
+    ASSERT_SAME_TYPE(decltype(vc(a, a)), bool);
+    assert(vc({1, 2}, {2, 1}));
+    assert(!vc({2, 1}, {1, 2}));
+  }
+  {
+    using Comp = std::function<bool(const std::vector<int>&, const std::vector<int>&)>;
+    using M    = std::flat_map<std::vector<int>, int, Comp>;
+    Comp comp  = [i = 1](const auto& x, const auto& y) { return x[i] < y[i]; };
+    M m({}, comp);
+    auto vc = m.value_comp();
+    static_assert(sizeof(vc) >= sizeof(Comp));
+    comp = nullptr;
+    m    = M({}, nullptr);
+    assert(m.key_comp() == nullptr);
+    // At this point, m.key_comp() is disengaged.
+    // But the std::function captured by copy inside `vc` remains valid.
+    auto a = std::make_pair(std::vector<int>{2, 1, 4}, 42);
+    auto b = std::make_pair(std::vector<int>{1, 2, 3}, 42);
+    auto c = std::make_pair(std::vector<int>{0, 3, 2}, 42);
+    assert(vc(a, b));
+    assert(vc(b, c));
+    assert(!vc(b, a));
+    assert(!vc(c, b));
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.observers/keys_values.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.observers/keys_values.pass.cpp
new file mode 100644
index 000000000000..84d8f8344aaa
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.observers/keys_values.pass.cpp
@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// const key_container_type& keys() const noexcept
+// const mapped_container_type& values() const noexcept
+
+#include <algorithm>
+#include <cassert>
+#include <flat_map>
+#include <functional>
+#include <utility>
+#include <vector>
+#include <deque>
+#include <string>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "test_allocator.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  const M m                                                 = {{4, 'a'}, {2, 'b'}, {3, 'c'}};
+  std::same_as<const KeyContainer&> decltype(auto) keys     = m.keys();
+  std::same_as<const ValueContainer&> decltype(auto) values = m.values();
+
+  // noexcept
+  static_assert(noexcept(m.keys()));
+  static_assert(noexcept(m.values()));
+
+  auto expected_keys   = {2, 3, 4};
+  auto expected_values = {'b', 'c', 'a'};
+  assert(std::ranges::equal(keys, expected_keys));
+  assert(std::ranges::equal(values, expected_values));
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/contains.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/contains.pass.cpp
new file mode 100644
index 000000000000..208d6138fa68
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/contains.pass.cpp
@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// bool contains(const key_type& x) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  {
+    using M = std::flat_map<Key, Value, std::less<>, KeyContainer, ValueContainer>;
+    M m     = {{1, 1}, {2, 2}, {4, 4}, {5, 5}, {8, 8}};
+    assert(!m.contains(0));
+    assert(m.contains(1));
+    assert(m.contains(2));
+    assert(!m.contains(3));
+    assert(m.contains(4));
+    assert(m.contains(5));
+    assert(!m.contains(6));
+    assert(!m.contains(7));
+    assert(std::as_const(m).contains(8));
+    assert(!std::as_const(m).contains(9));
+    m.clear();
+    assert(!m.contains(1));
+  }
+  {
+    using M = std::flat_map<Key, Value, std::greater<int>, KeyContainer, ValueContainer>;
+    M m     = {{1, 0}, {2, 0}, {4, 0}, {5, 0}, {8, 0}};
+    assert(!m.contains(0));
+    assert(m.contains(1));
+    assert(m.contains(2));
+    assert(!m.contains(3));
+    assert(m.contains(4));
+    assert(m.contains(5));
+    assert(!m.contains(6));
+    assert(!m.contains(7));
+    assert(std::as_const(m).contains(8));
+    assert(!std::as_const(m).contains(9));
+    m.clear();
+    assert(!m.contains(1));
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<int>>();
+  test<std::deque<int>, std::vector<int>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<int>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/contains_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/contains_transparent.pass.cpp
new file mode 100644
index 000000000000..0493538ab6da
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/contains_transparent.pass.cpp
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<class K> bool contains(const K& x) const;
+
+#include <cassert>
+#include <flat_map>
+#include <string>
+#include <utility>
+#include <deque>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// Constraints: The qualified-id Compare::is_transparent is valid and denotes a type.
+template <class M>
+concept CanContains     = requires(M m, Transparent<int> k) { m.contains(k); };
+using TransparentMap    = std::flat_map<int, double, TransparentComparator>;
+using NonTransparentMap = std::flat_map<int, double, NonTransparentComparator>;
+static_assert(CanContains<TransparentMap>);
+static_assert(CanContains<const TransparentMap>);
+static_assert(!CanContains<NonTransparentMap>);
+static_assert(!CanContains<const NonTransparentMap>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  M m = {{"alpha", 1}, {"beta", 2}, {"epsilon", 3}, {"eta", 4}, {"gamma", 5}};
+  ASSERT_SAME_TYPE(decltype(m.contains(Transparent<std::string>{"abc"})), bool);
+  ASSERT_SAME_TYPE(decltype(std::as_const(m).contains(Transparent<std::string>{"b"})), bool);
+  assert(m.contains(Transparent<std::string>{"alpha"}) == true);
+  assert(m.contains(Transparent<std::string>{"beta"}) == true);
+  assert(m.contains(Transparent<std::string>{"epsilon"}) == true);
+  assert(m.contains(Transparent<std::string>{"eta"}) == true);
+  assert(m.contains(Transparent<std::string>{"gamma"}) == true);
+  assert(m.contains(Transparent<std::string>{"al"}) == false);
+  assert(m.contains(Transparent<std::string>{""}) == false);
+  assert(m.contains(Transparent<std::string>{"g"}) == false);
+}
+
+int main(int, char**) {
+  test<std::vector<std::string>, std::vector<int>>();
+  test<std::deque<std::string>, std::vector<int>>();
+  test<MinSequenceContainer<std::string>, MinSequenceContainer<int>>();
+  test<std::vector<std::string, min_allocator<std::string>>, std::vector<int, min_allocator<int>>>();
+
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_map<int, int, TransparentComparator> m(std::sorted_unique, {{1, 1}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto b = m.contains(Transparent<int>{3});
+    assert(b);
+    assert(transparent_used);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/count.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/count.pass.cpp
new file mode 100644
index 000000000000..db675854d5e9
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/count.pass.cpp
@@ -0,0 +1,69 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// size_type count(const key_type& x) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+
+  {
+    using M = std::flat_map<Key, Value, std::less<>, KeyContainer, ValueContainer>;
+    M m     = {{1, 1}, {2, 2}, {4, 4}, {5, 5}, {8, 8}};
+    ASSERT_SAME_TYPE(decltype(m.count(0)), size_t);
+    assert(m.count(0) == 0);
+    assert(m.count(1) == 1);
+    assert(m.count(2) == 1);
+    assert(m.count(3) == 0);
+    assert(m.count(4) == 1);
+    assert(m.count(5) == 1);
+    assert(m.count(6) == 0);
+    assert(m.count(7) == 0);
+    assert(std::as_const(m).count(8) == 1);
+    assert(std::as_const(m).count(9) == 0);
+  }
+  {
+    using M = std::flat_map<Key, Value, std::greater<int>, KeyContainer, ValueContainer>;
+    M m     = {{1, 0}, {2, 0}, {4, 0}, {5, 0}, {8, 0}};
+    ASSERT_SAME_TYPE(decltype(m.count(0)), size_t);
+    assert(m.count(0) == 0);
+    assert(m.count(1) == 1);
+    assert(m.count(2) == 1);
+    assert(m.count(3) == 0);
+    assert(m.count(4) == 1);
+    assert(m.count(5) == 1);
+    assert(m.count(6) == 0);
+    assert(m.count(7) == 0);
+    assert(std::as_const(m).count(8) == 1);
+    assert(std::as_const(m).count(9) == 0);
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<int>>();
+  test<std::deque<int>, std::vector<int>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<int>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/count_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/count_transparent.pass.cpp
new file mode 100644
index 000000000000..cd195ff1fa8b
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/count_transparent.pass.cpp
@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<class K> size_type count(const K& x) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <string>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// Constraints: The qualified-id Compare::is_transparent is valid and denotes a type.
+template <class M>
+concept CanCount        = requires(M m, Transparent<int> k) { m.count(k); };
+using TransparentMap    = std::flat_map<int, double, TransparentComparator>;
+using NonTransparentMap = std::flat_map<int, double, NonTransparentComparator>;
+static_assert(CanCount<TransparentMap>);
+static_assert(CanCount<const TransparentMap>);
+static_assert(!CanCount<NonTransparentMap>);
+static_assert(!CanCount<const NonTransparentMap>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  M m = {{"alpha", 1}, {"beta", 2}, {"epsilon", 3}, {"eta", 4}, {"gamma", 5}};
+  ASSERT_SAME_TYPE(decltype(m.count(Transparent<std::string>{"abc"})), typename M::size_type);
+  ASSERT_SAME_TYPE(decltype(std::as_const(m).count(Transparent<std::string>{"b"})), typename M::size_type);
+  assert(m.count(Transparent<std::string>{"alpha"}) == 1);
+  assert(m.count(Transparent<std::string>{"beta"}) == 1);
+  assert(m.count(Transparent<std::string>{"epsilon"}) == 1);
+  assert(m.count(Transparent<std::string>{"eta"}) == 1);
+  assert(m.count(Transparent<std::string>{"gamma"}) == 1);
+  assert(m.count(Transparent<std::string>{"al"}) == 0);
+  assert(m.count(Transparent<std::string>{""}) == 0);
+  assert(m.count(Transparent<std::string>{"g"}) == 0);
+}
+
+int main(int, char**) {
+  test<std::vector<std::string>, std::vector<int>>();
+  test<std::deque<std::string>, std::vector<int>>();
+  test<MinSequenceContainer<std::string>, MinSequenceContainer<int>>();
+  test<std::vector<std::string, min_allocator<std::string>>, std::vector<int, min_allocator<int>>>();
+
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_map<int, int, TransparentComparator> m(std::sorted_unique, {{1, 1}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto n = m.count(Transparent<int>{3});
+    assert(n == 1);
+    assert(transparent_used);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/equal_range.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/equal_range.pass.cpp
new file mode 100644
index 000000000000..8fa73d2a2eb5
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/equal_range.pass.cpp
@@ -0,0 +1,78 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// pair<iterator,iterator>             equal_range(const key_type& k);
+// pair<const_iterator,const_iterator> equal_range(const key_type& k) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  {
+    using M  = std::flat_map<Key, Value, std::less<>, KeyContainer, ValueContainer>;
+    using R  = std::pair<typename M::iterator, typename M::iterator>;
+    using CR = std::pair<typename M::const_iterator, typename M::const_iterator>;
+    M m      = {{1, 'a'}, {2, 'b'}, {4, 'd'}, {5, 'e'}, {8, 'h'}};
+    ASSERT_SAME_TYPE(decltype(m.equal_range(0)), R);
+    ASSERT_SAME_TYPE(decltype(std::as_const(m).equal_range(0)), CR);
+    auto begin = m.begin();
+    assert(m.equal_range(0) == std::pair(begin, begin));
+    assert(m.equal_range(1) == std::pair(begin, begin + 1));
+    assert(m.equal_range(2) == std::pair(begin + 1, begin + 2));
+    assert(m.equal_range(3) == std::pair(begin + 2, begin + 2));
+    assert(m.equal_range(4) == std::pair(begin + 2, begin + 3));
+    assert(m.equal_range(5) == std::pair(begin + 3, begin + 4));
+    assert(m.equal_range(6) == std::pair(begin + 4, begin + 4));
+    assert(m.equal_range(7) == std::pair(begin + 4, begin + 4));
+    assert(std::as_const(m).equal_range(8) == std::pair(m.cbegin() + 4, m.cbegin() + 5));
+    assert(std::as_const(m).equal_range(9) == std::pair(m.cbegin() + 5, m.cbegin() + 5));
+  }
+
+  {
+    using M  = std::flat_map<Key, Value, std::greater<int>, KeyContainer, ValueContainer>;
+    using R  = std::pair<typename M::iterator, typename M::iterator>;
+    using CR = std::pair<typename M::const_iterator, typename M::const_iterator>;
+    M m      = {{1, 'a'}, {2, 'b'}, {4, 'd'}, {5, 'e'}, {8, 'h'}};
+    ASSERT_SAME_TYPE(decltype(m.equal_range(0)), R);
+    ASSERT_SAME_TYPE(decltype(std::as_const(m).equal_range(0)), CR);
+    auto begin = m.begin();
+    assert(m.equal_range(0) == std::pair(begin + 5, begin + 5));
+    assert(m.equal_range(1) == std::pair(begin + 4, begin + 5));
+    assert(m.equal_range(2) == std::pair(begin + 3, begin + 4));
+    assert(m.equal_range(3) == std::pair(begin + 3, begin + 3));
+    assert(m.equal_range(4) == std::pair(begin + 2, begin + 3));
+    assert(m.equal_range(5) == std::pair(begin + 1, begin + 2));
+    assert(m.equal_range(6) == std::pair(begin + 1, begin + 1));
+    assert(m.equal_range(7) == std::pair(begin + 1, begin + 1));
+    assert(std::as_const(m).equal_range(8) == std::pair(m.cbegin(), m.cbegin() + 1));
+    assert(std::as_const(m).equal_range(9) == std::pair(m.cbegin(), m.cbegin()));
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/equal_range_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/equal_range_transparent.pass.cpp
new file mode 100644
index 000000000000..0198f433bdc4
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/equal_range_transparent.pass.cpp
@@ -0,0 +1,100 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<class K> pair<iterator,iterator>             equal_range(const K& x);
+// template<class K> pair<const_iterator,const_iterator> equal_range(const K& x) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <string>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// Constraints: The qualified-id Compare::is_transparent is valid and denotes a type.
+template <class M>
+concept CanEqualRange   = requires(M m, Transparent<int> k) { m.equal_range(k); };
+using TransparentMap    = std::flat_map<int, double, TransparentComparator>;
+using NonTransparentMap = std::flat_map<int, double, NonTransparentComparator>;
+static_assert(CanEqualRange<TransparentMap>);
+static_assert(CanEqualRange<const TransparentMap>);
+static_assert(!CanEqualRange<NonTransparentMap>);
+static_assert(!CanEqualRange<const NonTransparentMap>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  using R        = std::pair<typename M::iterator, typename M::iterator>;
+  using CR       = std::pair<typename M::const_iterator, typename M::const_iterator>;
+  M m            = {{"alpha", 1}, {"beta", 2}, {"epsilon", 3}, {"eta", 4}, {"gamma", 5}};
+  const auto& cm = m;
+  ASSERT_SAME_TYPE(decltype(m.equal_range(Transparent<std::string>{"abc"})), R);
+  ASSERT_SAME_TYPE(decltype(std::as_const(m).equal_range(Transparent<std::string>{"b"})), CR);
+
+  auto test_found = [&](auto&& map, const std::string& expected_key, int expected_value) {
+    auto [first, last] = map.equal_range(Transparent<std::string>{expected_key});
+    assert(last - first == 1);
+    auto [key, value] = *first;
+    assert(key == expected_key);
+    assert(value == expected_value);
+  };
+
+  auto test_not_found = [&](auto&& map, const std::string& expected_key, long expected_offset) {
+    auto [first, last] = map.equal_range(Transparent<std::string>{expected_key});
+    assert(first == last);
+    assert(first - m.begin() == expected_offset);
+  };
+
+  test_found(m, "alpha", 1);
+  test_found(m, "beta", 2);
+  test_found(m, "epsilon", 3);
+  test_found(m, "eta", 4);
+  test_found(m, "gamma", 5);
+  test_found(cm, "alpha", 1);
+  test_found(cm, "beta", 2);
+  test_found(cm, "epsilon", 3);
+  test_found(cm, "eta", 4);
+  test_found(cm, "gamma", 5);
+
+  test_not_found(m, "charlie", 2);
+  test_not_found(m, "aaa", 0);
+  test_not_found(m, "zzz", 5);
+  test_not_found(cm, "charlie", 2);
+  test_not_found(cm, "aaa", 0);
+  test_not_found(cm, "zzz", 5);
+}
+
+int main(int, char**) {
+  test<std::vector<std::string>, std::vector<int>>();
+  test<std::deque<std::string>, std::vector<int>>();
+  test<MinSequenceContainer<std::string>, MinSequenceContainer<int>>();
+  test<std::vector<std::string, min_allocator<std::string>>, std::vector<int, min_allocator<int>>>();
+
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_map<int, int, TransparentComparator> m(std::sorted_unique, {{1, 1}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto p = m.equal_range(Transparent<int>{3});
+    assert(p.first != p.second);
+    assert(transparent_used);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/find.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/find.pass.cpp
new file mode 100644
index 000000000000..9fae407c7d8f
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/find.pass.cpp
@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+//       iterator find(const key_type& k);
+// const_iterator find(const key_type& k) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <string>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+
+  M m = {{1, 'a'}, {2, 'b'}, {4, 'd'}, {5, 'e'}, {8, 'h'}};
+  ASSERT_SAME_TYPE(decltype(m.find(0)), typename M::iterator);
+  ASSERT_SAME_TYPE(decltype(std::as_const(m).find(0)), typename M::const_iterator);
+  assert(m.find(0) == m.end());
+  assert(m.find(1) == m.begin());
+  assert(m.find(2) == m.begin() + 1);
+  assert(m.find(3) == m.end());
+  assert(m.find(4) == m.begin() + 2);
+  assert(m.find(5) == m.begin() + 3);
+  assert(m.find(6) == m.end());
+  assert(m.find(7) == m.end());
+  assert(std::as_const(m).find(8) == m.begin() + 4);
+  assert(std::as_const(m).find(9) == m.end());
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/find_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/find_transparent.pass.cpp
new file mode 100644
index 000000000000..291577a89fc8
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/find_transparent.pass.cpp
@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<class K> iterator       find(const K& x);
+// template<class K> const_iterator find(const K& x) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <string>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// Constraints: The qualified-id Compare::is_transparent is valid and denotes a type.
+template <class M>
+concept CanFind         = requires(M m, Transparent<int> k) { m.find(k); };
+using TransparentMap    = std::flat_map<int, double, TransparentComparator>;
+using NonTransparentMap = std::flat_map<int, double, NonTransparentComparator>;
+static_assert(CanFind<TransparentMap>);
+static_assert(CanFind<const TransparentMap>);
+static_assert(!CanFind<NonTransparentMap>);
+static_assert(!CanFind<const NonTransparentMap>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  M m            = {{"alpha", 1}, {"beta", 2}, {"epsilon", 3}, {"eta", 4}, {"gamma", 5}};
+  const auto& cm = m;
+  ASSERT_SAME_TYPE(decltype(m.find(Transparent<std::string>{"abc"})), typename M::iterator);
+  ASSERT_SAME_TYPE(decltype(std::as_const(m).find(Transparent<std::string>{"b"})), typename M::const_iterator);
+
+  auto test_find = [&](auto&& map, const std::string& expected_key, long expected_offset) {
+    auto iter = map.find(Transparent<std::string>{expected_key});
+    assert(iter - map.begin() == expected_offset);
+  };
+
+  test_find(m, "alpha", 0);
+  test_find(m, "beta", 1);
+  test_find(m, "epsilon", 2);
+  test_find(m, "eta", 3);
+  test_find(m, "gamma", 4);
+  test_find(m, "charlie", 5);
+  test_find(m, "aaa", 5);
+  test_find(m, "zzz", 5);
+  test_find(cm, "alpha", 0);
+  test_find(cm, "beta", 1);
+  test_find(cm, "epsilon", 2);
+  test_find(cm, "eta", 3);
+  test_find(cm, "gamma", 4);
+  test_find(cm, "charlie", 5);
+  test_find(cm, "aaa", 5);
+  test_find(cm, "zzz", 5);
+}
+
+int main(int, char**) {
+  test<std::vector<std::string>, std::vector<int>>();
+  test<std::deque<std::string>, std::vector<int>>();
+  test<MinSequenceContainer<std::string>, MinSequenceContainer<int>>();
+  test<std::vector<std::string, min_allocator<std::string>>, std::vector<int, min_allocator<int>>>();
+
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_map<int, int, TransparentComparator> m(std::sorted_unique, {{1, 1}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto it = m.find(Transparent<int>{3});
+    assert(it != m.end());
+    assert(transparent_used);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/lower_bound.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/lower_bound.pass.cpp
new file mode 100644
index 000000000000..b5491f3b2267
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/lower_bound.pass.cpp
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+//       iterator lower_bound(const key_type& k);
+// const_iterator lower_bound(const key_type& k) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  {
+    using M = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+    M m     = {{1, 'a'}, {2, 'b'}, {4, 'd'}, {5, 'e'}, {8, 'h'}};
+    ASSERT_SAME_TYPE(decltype(m.lower_bound(0)), typename M::iterator);
+    ASSERT_SAME_TYPE(decltype(std::as_const(m).lower_bound(0)), typename M::const_iterator);
+    assert(m.lower_bound(0) == m.begin());
+    assert(m.lower_bound(1) == m.begin());
+    assert(m.lower_bound(2) == m.begin() + 1);
+    assert(m.lower_bound(3) == m.begin() + 2);
+    assert(m.lower_bound(4) == m.begin() + 2);
+    assert(m.lower_bound(5) == m.begin() + 3);
+    assert(m.lower_bound(6) == m.begin() + 4);
+    assert(m.lower_bound(7) == m.begin() + 4);
+    assert(std::as_const(m).lower_bound(8) == m.begin() + 4);
+    assert(std::as_const(m).lower_bound(9) == m.end());
+  }
+  {
+    using M = std::flat_map<Key, Value, std::greater<Key>, KeyContainer, ValueContainer>;
+    M m     = {{1, 'a'}, {2, 'b'}, {4, 'd'}, {5, 'e'}, {8, 'h'}};
+    ASSERT_SAME_TYPE(decltype(m.lower_bound(0)), typename M::iterator);
+    ASSERT_SAME_TYPE(decltype(std::as_const(m).lower_bound(0)), typename M::const_iterator);
+    assert(m.lower_bound(0) == m.end());
+    assert(m.lower_bound(1) == m.begin() + 4);
+    assert(m.lower_bound(2) == m.begin() + 3);
+    assert(m.lower_bound(3) == m.begin() + 3);
+    assert(m.lower_bound(4) == m.begin() + 2);
+    assert(m.lower_bound(5) == m.begin() + 1);
+    assert(m.lower_bound(6) == m.begin() + 1);
+    assert(m.lower_bound(7) == m.begin() + 1);
+    assert(std::as_const(m).lower_bound(8) == m.begin());
+    assert(std::as_const(m).lower_bound(9) == m.begin());
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/lower_bound_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/lower_bound_transparent.pass.cpp
new file mode 100644
index 000000000000..6a923c197e91
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/lower_bound_transparent.pass.cpp
@@ -0,0 +1,95 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<class K> iterator       lower_bound(const K& x);
+// template<class K> const_iterator lower_bound(const K& x) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <string>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// Constraints: The qualified-id Compare::is_transparent is valid and denotes a type.
+template <class M>
+concept CanLowerBound   = requires(M m, Transparent<int> k) { m.lower_bound(k); };
+using TransparentMap    = std::flat_map<int, double, TransparentComparator>;
+using NonTransparentMap = std::flat_map<int, double, NonTransparentComparator>;
+static_assert(CanLowerBound<TransparentMap>);
+static_assert(CanLowerBound<const TransparentMap>);
+static_assert(!CanLowerBound<NonTransparentMap>);
+static_assert(!CanLowerBound<const NonTransparentMap>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  M m            = {{"alpha", 1}, {"beta", 2}, {"epsilon", 3}, {"eta", 4}, {"gamma", 5}};
+  const auto& cm = m;
+  ASSERT_SAME_TYPE(decltype(m.lower_bound(Transparent<std::string>{"abc"})), typename M::iterator);
+  ASSERT_SAME_TYPE(decltype(std::as_const(m).lower_bound(Transparent<std::string>{"b"})), typename M::const_iterator);
+
+  auto test_lower_bound = [&](auto&& map, const std::string& expected_key, long expected_offset) {
+    auto iter = map.lower_bound(Transparent<std::string>{expected_key});
+    assert(iter - map.begin() == expected_offset);
+  };
+
+  test_lower_bound(m, "abc", 0);
+  test_lower_bound(m, "alpha", 0);
+  test_lower_bound(m, "beta", 1);
+  test_lower_bound(m, "bets", 2);
+  test_lower_bound(m, "charlie", 2);
+  test_lower_bound(m, "echo", 2);
+  test_lower_bound(m, "epsilon", 2);
+  test_lower_bound(m, "eta", 3);
+  test_lower_bound(m, "gamma", 4);
+  test_lower_bound(m, "golf", 5);
+  test_lower_bound(m, "zzz", 5);
+
+  test_lower_bound(cm, "abc", 0);
+  test_lower_bound(cm, "alpha", 0);
+  test_lower_bound(cm, "beta", 1);
+  test_lower_bound(cm, "bets", 2);
+  test_lower_bound(cm, "charlie", 2);
+  test_lower_bound(cm, "echo", 2);
+  test_lower_bound(cm, "epsilon", 2);
+  test_lower_bound(cm, "eta", 3);
+  test_lower_bound(cm, "gamma", 4);
+  test_lower_bound(cm, "golf", 5);
+  test_lower_bound(cm, "zzz", 5);
+}
+
+int main(int, char**) {
+  test<std::vector<std::string>, std::vector<int>>();
+  test<std::deque<std::string>, std::vector<int>>();
+  test<MinSequenceContainer<std::string>, MinSequenceContainer<int>>();
+  test<std::vector<std::string, min_allocator<std::string>>, std::vector<int, min_allocator<int>>>();
+
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_map<int, int, TransparentComparator> m(std::sorted_unique, {{1, 1}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto it = m.lower_bound(Transparent<int>{3});
+    assert(it != m.end());
+    assert(transparent_used);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/upper_bound.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/upper_bound.pass.cpp
new file mode 100644
index 000000000000..775e53286d62
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/upper_bound.pass.cpp
@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+//       iterator upper_bound(const key_type& k);
+// const_iterator upper_bound(const key_type& k) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  {
+    using M = std::flat_map<Key, Value, std::less<Key>, KeyContainer, ValueContainer>;
+    M m     = {{1, 'a'}, {2, 'b'}, {4, 'd'}, {5, 'e'}, {8, 'h'}};
+    ASSERT_SAME_TYPE(decltype(m.upper_bound(0)), typename M::iterator);
+    ASSERT_SAME_TYPE(decltype(std::as_const(m).upper_bound(0)), typename M::const_iterator);
+    assert(m.upper_bound(0) == m.begin());
+    assert(m.upper_bound(1) == m.begin() + 1);
+    assert(m.upper_bound(2) == m.begin() + 2);
+    assert(m.upper_bound(3) == m.begin() + 2);
+    assert(m.upper_bound(4) == m.begin() + 3);
+    assert(m.upper_bound(5) == m.begin() + 4);
+    assert(m.upper_bound(6) == m.begin() + 4);
+    assert(std::as_const(m).upper_bound(7) == m.begin() + 4);
+    assert(std::as_const(m).upper_bound(8) == m.end());
+    assert(std::as_const(m).upper_bound(9) == m.end());
+  }
+
+  {
+    using M = std::flat_map<Key, Value, std::greater<Key>, KeyContainer, ValueContainer>;
+    M m     = {{1, 'a'}, {2, 'b'}, {4, 'd'}, {5, 'e'}, {8, 'h'}};
+    ASSERT_SAME_TYPE(decltype(m.upper_bound(0)), typename M::iterator);
+    ASSERT_SAME_TYPE(decltype(std::as_const(m).upper_bound(0)), typename M::const_iterator);
+    assert(m.upper_bound(0) == m.end());
+    assert(m.upper_bound(1) == m.end());
+    assert(m.upper_bound(2) == m.begin() + 4);
+    assert(m.upper_bound(3) == m.begin() + 3);
+    assert(m.upper_bound(4) == m.begin() + 3);
+    assert(m.upper_bound(5) == m.begin() + 2);
+    assert(m.upper_bound(6) == m.begin() + 1);
+    assert(m.upper_bound(7) == m.begin() + 1);
+    assert(std::as_const(m).upper_bound(8) == m.begin() + 1);
+    assert(std::as_const(m).upper_bound(9) == m.begin());
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<char>>();
+  test<std::deque<int>, std::vector<char>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<char>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<char, min_allocator<char>>>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/upper_bound_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/upper_bound_transparent.pass.cpp
new file mode 100644
index 000000000000..4e83f920835d
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/upper_bound_transparent.pass.cpp
@@ -0,0 +1,94 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// template<class K> iterator       upper_bound(const K& x);
+// template<class K> const_iterator upper_bound(const K& x) const;
+
+#include <cassert>
+#include <deque>
+#include <flat_map>
+#include <string>
+#include <utility>
+
+#include "MinSequenceContainer.h"
+#include "../helpers.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+
+// Constraints: The qualified-id Compare::is_transparent is valid and denotes a type.
+template <class M>
+concept CanUpperBound   = requires(M m, Transparent<int> k) { m.upper_bound(k); };
+using TransparentMap    = std::flat_map<int, double, TransparentComparator>;
+using NonTransparentMap = std::flat_map<int, double, NonTransparentComparator>;
+static_assert(CanUpperBound<TransparentMap>);
+static_assert(CanUpperBound<const TransparentMap>);
+static_assert(!CanUpperBound<NonTransparentMap>);
+static_assert(!CanUpperBound<const NonTransparentMap>);
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+  using M     = std::flat_map<Key, Value, TransparentComparator, KeyContainer, ValueContainer>;
+
+  M m            = {{"alpha", 1}, {"beta", 2}, {"epsilon", 3}, {"eta", 4}, {"gamma", 5}};
+  const auto& cm = m;
+  ASSERT_SAME_TYPE(decltype(m.lower_bound(Transparent<std::string>{"abc"})), typename M::iterator);
+  ASSERT_SAME_TYPE(decltype(std::as_const(m).lower_bound(Transparent<std::string>{"b"})), typename M::const_iterator);
+
+  auto test_upper_bound = [&](auto&& map, const std::string& expected_key, long expected_offset) {
+    auto iter = map.upper_bound(Transparent<std::string>{expected_key});
+    assert(iter - map.begin() == expected_offset);
+  };
+
+  test_upper_bound(m, "abc", 0);
+  test_upper_bound(m, "alpha", 1);
+  test_upper_bound(m, "beta", 2);
+  test_upper_bound(m, "bets", 2);
+  test_upper_bound(m, "charlie", 2);
+  test_upper_bound(m, "echo", 2);
+  test_upper_bound(m, "epsilon", 3);
+  test_upper_bound(m, "eta", 4);
+  test_upper_bound(m, "gamma", 5);
+  test_upper_bound(m, "golf", 5);
+  test_upper_bound(m, "zzz", 5);
+
+  test_upper_bound(cm, "abc", 0);
+  test_upper_bound(cm, "alpha", 1);
+  test_upper_bound(cm, "beta", 2);
+  test_upper_bound(cm, "bets", 2);
+  test_upper_bound(cm, "charlie", 2);
+  test_upper_bound(cm, "echo", 2);
+  test_upper_bound(cm, "epsilon", 3);
+  test_upper_bound(cm, "eta", 4);
+  test_upper_bound(cm, "gamma", 5);
+  test_upper_bound(cm, "golf", 5);
+  test_upper_bound(cm, "zzz", 5);
+}
+
+int main(int, char**) {
+  test<std::vector<std::string>, std::vector<int>>();
+  test<std::deque<std::string>, std::vector<int>>();
+  test<MinSequenceContainer<std::string>, MinSequenceContainer<int>>();
+  test<std::vector<std::string, min_allocator<std::string>>, std::vector<int, min_allocator<int>>>();
+  {
+    bool transparent_used = false;
+    TransparentComparator c(transparent_used);
+    std::flat_map<int, int, TransparentComparator> m(std::sorted_unique, {{1, 1}, {2, 2}, {3, 3}}, c);
+    assert(!transparent_used);
+    auto it = m.upper_bound(Transparent<int>{2});
+    assert(it != m.end());
+    assert(transparent_used);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/helpers.h b/libcxx/test/std/containers/container.adaptors/flat.map/helpers.h
new file mode 100644
index 000000000000..8dbb85a6c0ac
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/helpers.h
@@ -0,0 +1,394 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SUPPORT_FLAT_MAP_HELPERS_H
+#define SUPPORT_FLAT_MAP_HELPERS_H
+
+#include <algorithm>
+#include <cassert>
+#include <string>
+#include <vector>
+#include <flat_map>
+
+#include "test_allocator.h"
+#include "test_macros.h"
+
+template <class... Args>
+void check_invariant(const std::flat_map<Args...>& m) {
+  assert(m.keys().size() == m.values().size());
+  const auto& keys = m.keys();
+  assert(std::is_sorted(keys.begin(), keys.end(), m.key_comp()));
+  auto key_equal = [&](const auto& x, const auto& y) {
+    const auto& c = m.key_comp();
+    return !c(x, y) && !c(y, x);
+  };
+  assert(std::adjacent_find(keys.begin(), keys.end(), key_equal) == keys.end());
+}
+
+struct StartsWith {
+  explicit StartsWith(char ch) : lower_(1, ch), upper_(1, ch + 1) {}
+  StartsWith(const StartsWith&)     = delete;
+  void operator=(const StartsWith&) = delete;
+  struct Less {
+    using is_transparent = void;
+    bool operator()(const std::string& a, const std::string& b) const { return a < b; }
+    bool operator()(const StartsWith& a, const std::string& b) const { return a.upper_ <= b; }
+    bool operator()(const std::string& a, const StartsWith& b) const { return a < b.lower_; }
+    bool operator()(const StartsWith&, const StartsWith&) const {
+      assert(false); // should not be called
+      return false;
+    }
+  };
+
+private:
+  std::string lower_;
+  std::string upper_;
+};
+
+template <class T>
+struct CopyOnlyVector : std::vector<T> {
+  using std::vector<T>::vector;
+
+  CopyOnlyVector(const CopyOnlyVector&) = default;
+  CopyOnlyVector(CopyOnlyVector&& other) : CopyOnlyVector(other) {}
+  CopyOnlyVector(CopyOnlyVector&& other, std::vector<T>::allocator_type alloc) : CopyOnlyVector(other, alloc) {}
+
+  CopyOnlyVector& operator=(const CopyOnlyVector&) = default;
+  CopyOnlyVector& operator=(CopyOnlyVector& other) { return this->operator=(other); }
+};
+
+template <class T, bool ConvertibleToT = false>
+struct Transparent {
+  T t;
+
+  operator T() const
+    requires ConvertibleToT
+  {
+    return t;
+  }
+};
+
+template <class T>
+using ConvertibleTransparent = Transparent<T, true>;
+
+template <class T>
+using NonConvertibleTransparent = Transparent<T, false>;
+
+struct TransparentComparator {
+  using is_transparent = void;
+
+  bool* transparent_used  = nullptr;
+  TransparentComparator() = default;
+  TransparentComparator(bool& used) : transparent_used(&used) {}
+
+  template <class T, bool Convertible>
+  bool operator()(const T& t, const Transparent<T, Convertible>& transparent) const {
+    if (transparent_used != nullptr) {
+      *transparent_used = true;
+    }
+    return t < transparent.t;
+  }
+
+  template <class T, bool Convertible>
+  bool operator()(const Transparent<T, Convertible>& transparent, const T& t) const {
+    if (transparent_used != nullptr) {
+      *transparent_used = true;
+    }
+    return transparent.t < t;
+  }
+
+  template <class T>
+  bool operator()(const T& t1, const T& t2) const {
+    return t1 < t2;
+  }
+};
+
+struct NonTransparentComparator {
+  template <class T, bool Convertible>
+  bool operator()(const T&, const Transparent<T, Convertible>&) const;
+
+  template <class T, bool Convertible>
+  bool operator()(const Transparent<T, Convertible>&, const T&) const;
+
+  template <class T>
+  bool operator()(const T&, const T&) const;
+};
+
+struct NoDefaultCtr {
+  NoDefaultCtr() = delete;
+};
+
+#ifndef TEST_HAS_NO_EXCEPTIONS
+template <class T>
+struct EmplaceUnsafeContainer : std::vector<T> {
+  using std::vector<T>::vector;
+
+  template <class... Args>
+  auto emplace(Args&&... args) -> decltype(std::declval<std::vector<T>>().emplace(std::forward<Args>(args)...)) {
+    if (this->size() > 1) {
+      auto it1 = this->begin();
+      auto it2 = it1 + 1;
+      // messing up the container
+      std::iter_swap(it1, it2);
+    }
+
+    throw 42;
+  }
+
+  template <class... Args>
+  auto insert(Args&&... args) -> decltype(std::declval<std::vector<T>>().insert(std::forward<Args>(args)...)) {
+    if (this->size() > 1) {
+      auto it1 = this->begin();
+      auto it2 = it1 + 1;
+      // messing up the container
+      std::iter_swap(it1, it2);
+    }
+
+    throw 42;
+  }
+};
+
+template <class T>
+struct ThrowOnEraseContainer : std::vector<T> {
+  using std::vector<T>::vector;
+
+  template <class... Args>
+  auto erase(Args&&... args) -> decltype(std::declval<std::vector<T>>().erase(std::forward<Args>(args)...)) {
+    throw 42;
+  }
+};
+
+template <class T>
+struct ThrowOnMoveContainer : std::vector<T> {
+  using std::vector<T>::vector;
+
+  ThrowOnMoveContainer(ThrowOnMoveContainer&&) { throw 42; }
+
+  ThrowOnMoveContainer& operator=(ThrowOnMoveContainer&&) { throw 42; }
+};
+
+#endif
+
+template <class F>
+void test_emplace_exception_guarantee([[maybe_unused]] F&& emplace_function) {
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  using C = TransparentComparator;
+  {
+    // Throw on emplace the key, and underlying has strong exception guarantee
+    using KeyContainer = std::vector<int, test_allocator<int>>;
+    using M            = std::flat_map<int, int, C, KeyContainer>;
+
+    LIBCPP_STATIC_ASSERT(std::__container_traits<KeyContainer>::__emplacement_has_strong_exception_safety_guarantee);
+
+    test_allocator_statistics stats;
+
+    KeyContainer a({1, 2, 3, 4}, test_allocator<int>{&stats});
+    std::vector<int> b                    = {5, 6, 7, 8};
+    [[maybe_unused]] auto expected_keys   = a;
+    [[maybe_unused]] auto expected_values = b;
+    M m(std::sorted_unique, std::move(a), std::move(b));
+
+    stats.throw_after = 1;
+    try {
+      emplace_function(m, 0, 0);
+      assert(false);
+    } catch (const std::bad_alloc&) {
+      check_invariant(m);
+      // In libc++, the flat_map is unchanged
+      LIBCPP_ASSERT(m.size() == 4);
+      LIBCPP_ASSERT(m.keys() == expected_keys);
+      LIBCPP_ASSERT(m.values() == expected_values);
+    }
+  }
+  {
+    // Throw on emplace the key, and underlying has no strong exception guarantee
+    using KeyContainer = EmplaceUnsafeContainer<int>;
+    using M            = std::flat_map<int, int, C, KeyContainer>;
+
+    LIBCPP_STATIC_ASSERT(!std::__container_traits<KeyContainer>::__emplacement_has_strong_exception_safety_guarantee);
+    KeyContainer a     = {1, 2, 3, 4};
+    std::vector<int> b = {5, 6, 7, 8};
+    M m(std::sorted_unique, std::move(a), std::move(b));
+    try {
+      emplace_function(m, 0, 0);
+      assert(false);
+    } catch (int) {
+      check_invariant(m);
+      // In libc++, the flat_map is cleared
+      LIBCPP_ASSERT(m.size() == 0);
+    }
+  }
+  {
+    // Throw on emplace the value, and underlying has strong exception guarantee
+    using ValueContainer = std::vector<int, test_allocator<int>>;
+    ;
+    using M = std::flat_map<int, int, C, std::vector<int>, ValueContainer>;
+
+    LIBCPP_STATIC_ASSERT(std::__container_traits<ValueContainer>::__emplacement_has_strong_exception_safety_guarantee);
+
+    std::vector<int> a = {1, 2, 3, 4};
+    test_allocator_statistics stats;
+    ValueContainer b({1, 2, 3, 4}, test_allocator<int>{&stats});
+
+    [[maybe_unused]] auto expected_keys   = a;
+    [[maybe_unused]] auto expected_values = b;
+    M m(std::sorted_unique, std::move(a), std::move(b));
+
+    stats.throw_after = 1;
+    try {
+      emplace_function(m, 0, 0);
+      assert(false);
+    } catch (const std::bad_alloc&) {
+      check_invariant(m);
+      // In libc++, the emplaced key is erased and the flat_map is unchanged
+      LIBCPP_ASSERT(m.size() == 4);
+      LIBCPP_ASSERT(m.keys() == expected_keys);
+      LIBCPP_ASSERT(m.values() == expected_values);
+    }
+  }
+  {
+    // Throw on emplace the value, and underlying has no strong exception guarantee
+    using ValueContainer = EmplaceUnsafeContainer<int>;
+    using M              = std::flat_map<int, int, C, std::vector<int>, ValueContainer>;
+
+    LIBCPP_STATIC_ASSERT(!std::__container_traits<ValueContainer>::__emplacement_has_strong_exception_safety_guarantee);
+    std::vector<int> a = {1, 2, 3, 4};
+    ValueContainer b   = {1, 2, 3, 4};
+
+    M m(std::sorted_unique, std::move(a), std::move(b));
+
+    try {
+      emplace_function(m, 0, 0);
+      assert(false);
+    } catch (int) {
+      check_invariant(m);
+      // In libc++, the flat_map is cleared
+      LIBCPP_ASSERT(m.size() == 0);
+    }
+  }
+  {
+    // Throw on emplace the value, then throw again on erasing the key
+    using KeyContainer   = ThrowOnEraseContainer<int>;
+    using ValueContainer = std::vector<int, test_allocator<int>>;
+    using M              = std::flat_map<int, int, C, KeyContainer, ValueContainer>;
+
+    LIBCPP_STATIC_ASSERT(std::__container_traits<ValueContainer>::__emplacement_has_strong_exception_safety_guarantee);
+
+    KeyContainer a = {1, 2, 3, 4};
+    test_allocator_statistics stats;
+    ValueContainer b({1, 2, 3, 4}, test_allocator<int>{&stats});
+
+    M m(std::sorted_unique, std::move(a), std::move(b));
+    stats.throw_after = 1;
+    try {
+      emplace_function(m, 0, 0);
+      assert(false);
+    } catch (const std::bad_alloc&) {
+      check_invariant(m);
+      // In libc++, we try to erase the key after value emplacement failure.
+      // and after erasure failure, we clear the flat_map
+      LIBCPP_ASSERT(m.size() == 0);
+    }
+  }
+#endif
+}
+
+template <class F>
+void test_insert_range_exception_guarantee([[maybe_unused]] F&& insert_function) {
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  using KeyContainer   = EmplaceUnsafeContainer<int>;
+  using ValueContainer = std::vector<int>;
+  using M              = std::flat_map<int, int, std::ranges::less, KeyContainer, ValueContainer>;
+  test_allocator_statistics stats;
+  KeyContainer a{1, 2, 3, 4};
+  ValueContainer b{1, 2, 3, 4};
+  M m(std::sorted_unique, std::move(a), std::move(b));
+
+  std::vector<std::pair<int, int>> newValues = {{0, 0}, {1, 1}, {5, 5}, {6, 6}, {7, 7}, {8, 8}};
+  stats.throw_after                          = 1;
+  try {
+    insert_function(m, newValues);
+    assert(false);
+  } catch (int) {
+    check_invariant(m);
+    // In libc++, we clear if anything goes wrong when inserting a range
+    LIBCPP_ASSERT(m.size() == 0);
+  }
+#endif
+}
+
+template <class F>
+void test_erase_exception_guarantee([[maybe_unused]] F&& erase_function) {
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  {
+    // key erase throws
+    using KeyContainer   = ThrowOnEraseContainer<int>;
+    using ValueContainer = std::vector<int>;
+    using M              = std::flat_map<int, int, TransparentComparator, KeyContainer, ValueContainer>;
+
+    KeyContainer a{1, 2, 3, 4};
+    ValueContainer b{1, 2, 3, 4};
+    M m(std::sorted_unique, std::move(a), std::move(b));
+    try {
+      erase_function(m, 3);
+      assert(false);
+    } catch (int) {
+      check_invariant(m);
+      // In libc++, we clear if anything goes wrong when erasing
+      LIBCPP_ASSERT(m.size() == 0);
+    }
+  }
+  {
+    // key erase throws
+    using KeyContainer   = std::vector<int>;
+    using ValueContainer = ThrowOnEraseContainer<int>;
+    using M              = std::flat_map<int, int, TransparentComparator, KeyContainer, ValueContainer>;
+
+    KeyContainer a{1, 2, 3, 4};
+    ValueContainer b{1, 2, 3, 4};
+    M m(std::sorted_unique, std::move(a), std::move(b));
+    try {
+      erase_function(m, 3);
+      assert(false);
+    } catch (int) {
+      check_invariant(m);
+      // In libc++, we clear if anything goes wrong when erasing
+      LIBCPP_ASSERT(m.size() == 0);
+    }
+  }
+#endif
+}
+class Moveable {
+  int int_;
+  double double_;
+
+public:
+  Moveable() : int_(0), double_(0) {}
+  Moveable(int i, double d) : int_(i), double_(d) {}
+  Moveable(Moveable&& x) : int_(x.int_), double_(x.double_) {
+    x.int_    = -1;
+    x.double_ = -1;
+  }
+  Moveable& operator=(Moveable&& x) {
+    int_      = x.int_;
+    x.int_    = -1;
+    double_   = x.double_;
+    x.double_ = -1;
+    return *this;
+  }
+
+  Moveable(const Moveable&)            = delete;
+  Moveable& operator=(const Moveable&) = delete;
+  bool operator==(const Moveable& x) const { return int_ == x.int_ && double_ == x.double_; }
+  bool operator<(const Moveable& x) const { return int_ < x.int_ || (int_ == x.int_ && double_ < x.double_); }
+
+  int get() const { return int_; }
+  bool moved() const { return int_ == -1; }
+};
+
+#endif // SUPPORT_FLAT_MAP_HELPERS_H
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/incomplete_type.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/incomplete_type.pass.cpp
new file mode 100644
index 000000000000..81c590ba73a1
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/incomplete_type.pass.cpp
@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// Check that std::flat_map and its iterators can be instantiated with an incomplete
+// type.
+
+#include <flat_map>
+
+struct A {
+  using Map = std::flat_map<A, A>;
+  int data;
+  Map m;
+  Map::iterator it;
+  Map::const_iterator cit;
+};
+
+// Implement the operator< required in order to instantiate flat_map<A, X>
+bool operator<(A const& L, A const& R) { return L.data < R.data; }
+
+int main(int, char**) {
+  A a;
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/op_compare.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/op_compare.pass.cpp
new file mode 100644
index 000000000000..fffe71158070
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/op_compare.pass.cpp
@@ -0,0 +1,118 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_map>
+
+// friend bool operator==(const flat_map& x, const flat_map& y);
+// friend synth-three-way-result<value_type>
+//   operator<=>(const flat_map& x, const flat_map& y);
+
+#include <algorithm>
+#include <cassert>
+#include <deque>
+#include <compare>
+#include <flat_map>
+#include <functional>
+#include <limits>
+#include <vector>
+
+#include "MinSequenceContainer.h"
+#include "test_macros.h"
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_comparisons.h"
+#include "test_container_comparisons.h"
+
+template <class KeyContainer, class ValueContainer>
+void test() {
+  using Key   = typename KeyContainer::value_type;
+  using Value = typename ValueContainer::value_type;
+
+  {
+    using C = std::flat_map<Key, Value>;
+    C s1    = {{1, 1}};
+    C s2    = {{2, 0}}; // {{1,1}} versus {{2,0}}
+    ASSERT_SAME_TYPE(decltype(s1 <=> s2), std::strong_ordering);
+    AssertComparisonsReturnBool<C>();
+    assert(testComparisons(s1, s2, false, true));
+    s2 = {{1, 1}}; // {{1,1}} versus {{1,1}}
+    assert(testComparisons(s1, s2, true, false));
+    s2 = {{1, 1}, {2, 0}}; // {{1,1}} versus {{1,1},{2,0}}
+    assert(testComparisons(s1, s2, false, true));
+    s1 = {{0, 0}, {1, 1}, {2, 2}}; // {{0,0},{1,1},{2,2}} versus {{1,1},{2,0}}
+    assert(testComparisons(s1, s2, false, true));
+    s2 = {{0, 0}, {1, 1}, {2, 3}}; // {{0,0},{1,1},{2,2}} versus {{0,0},{1,1},{2,3}}
+    assert(testComparisons(s1, s2, false, true));
+  }
+  {
+    // Comparisons use value_type's native operators, not the comparator
+    using C = std::flat_map<Key, Value, std::greater<Key>>;
+    C s1    = {{1, 1}};
+    C s2    = {{2, 0}}; // {{1,1}} versus {{2,0}}
+    ASSERT_SAME_TYPE(decltype(s1 <=> s2), std::strong_ordering);
+    AssertComparisonsReturnBool<C>();
+    assert(testComparisons(s1, s2, false, true));
+    s2 = {{1, 1}}; // {{1,1}} versus {{1,1}}
+    assert(testComparisons(s1, s2, true, false));
+    s2 = {{1, 1}, {2, 0}}; // {{1,1}} versus {{2,0},{1,1}}
+    assert(testComparisons(s1, s2, false, true));
+    s1 = {{0, 0}, {1, 1}, {2, 2}}; // {{2,2},{1,1},{0,0}} versus {2,0},{1,1}}
+    assert(testComparisons(s1, s2, false, false));
+    s2 = {{0, 0}, {1, 1}, {2, 3}}; // {{2,2},{1,1},{0,0}} versus {{2,3},{1,1},{0,0}}
+    assert(testComparisons(s1, s2, false, true));
+  }
+}
+
+int main(int, char**) {
+  test<std::vector<int>, std::vector<int>>();
+  test<std::deque<int>, std::deque<int>>();
+  test<MinSequenceContainer<int>, MinSequenceContainer<int>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+  test<std::vector<int, min_allocator<int>>, std::vector<int, min_allocator<int>>>();
+
+  {
+    using C = std::flat_map<double, int>;
+    C s1    = {{1, 1}};
+    C s2    = C(std::sorted_unique, {{std::numeric_limits<double>::quiet_NaN(), 2}});
+    ASSERT_SAME_TYPE(decltype(s1 <=> s2), std::partial_ordering);
+    AssertComparisonsReturnBool<C>();
+    assert(testComparisonsComplete(s1, s2, false, false, false));
+  }
+  {
+    using C = std::flat_map<int, double>;
+    C s1    = {{1, 1}};
+    C s2    = C(std::sorted_unique, {{2, std::numeric_limits<double>::quiet_NaN()}});
+    ASSERT_SAME_TYPE(decltype(s1 <=> s2), std::partial_ordering);
+    AssertComparisonsReturnBool<C>();
+    assert(testComparisonsComplete(s1, s2, false, true, false));
+    s2 = C(std::sorted_unique, {{1, std::numeric_limits<double>::quiet_NaN()}});
+    assert(testComparisonsComplete(s1, s2, false, false, false));
+  }
+  {
+    // Comparisons use value_type's native operators, not the comparator
+    struct StrongComp {
+      bool operator()(double a, double b) const { return std::strong_order(a, b) < 0; }
+    };
+    using C = std::flat_map<double, double, StrongComp>;
+    C s1    = {{1, 1}};
+    C s2    = {{std::numeric_limits<double>::quiet_NaN(), std::numeric_limits<double>::quiet_NaN()}};
+    ASSERT_SAME_TYPE(decltype(s1 <=> s2), std::partial_ordering);
+    AssertComparisonsReturnBool<C>();
+    assert(testComparisonsComplete(s1, s2, false, false, false));
+    s1 = {{{1, 1}, {std::numeric_limits<double>::quiet_NaN(), 1}}};
+    s2 = {{{std::numeric_limits<double>::quiet_NaN(), 1}, {1, 1}}};
+    assert(std::lexicographical_compare_three_way(
+               s1.keys().begin(), s1.keys().end(), s2.keys().begin(), s2.keys().end(), std::strong_order) ==
+           std::strong_ordering::equal);
+    assert(s1 != s2);
+    assert((s1 <=> s2) == std::partial_ordering::unordered);
+  }
+  return 0;
+}
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/types.compile.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/types.compile.pass.cpp
new file mode 100644
index 000000000000..ea9d4d7fca67
--- /dev/null
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/types.compile.pass.cpp
@@ -0,0 +1,133 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+//  using key_type               = Key;
+//  using mapped_type            = T;
+//  using value_type             = pair<key_type, mapped_type>;
+//  using key_compare            = Compare;
+//  using reference              = pair<const key_type&, mapped_type&>;
+//  using const_reference        = pair<const key_type&, const mapped_type&>;
+//  using size_type              = size_t;
+//  using difference_type        = ptrdiff_t;
+//  using iterator               = implementation-defined; // see [container.requirements]
+//  using const_iterator         = implementation-defined; // see [container.requirements]
+//  using reverse_iterator       = std::reverse_iterator<iterator>;
+//  using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+//  using key_container_type     = KeyContainer;
+//  using mapped_container_type  = MappedContainer;
+
+//  class value_compare;
+
+//  struct containers {
+//    key_container_type keys;
+//    mapped_container_type values;
+//  };
+
+#include <concepts>
+#include <deque>
+#include <flat_map>
+#include <functional>
+#include <ranges>
+#include <string>
+#include <vector>
+#include "min_allocator.h"
+
+void test() {
+  {
+    using M = std::flat_map<int, double>;
+    static_assert(std::is_same_v<typename M::key_type, int>);
+    static_assert(std::is_same_v<typename M::mapped_type, double>);
+    static_assert(std::is_same_v<typename M::value_type, std::pair<int, double>>);
+    static_assert(std::is_same_v<typename M::key_compare, std::less<int>>);
+    static_assert(std::is_same_v<typename M::reference, std::pair<const int&, double&>>);
+    static_assert(std::is_same_v<typename M::const_reference, std::pair<const int&, const double&>>);
+    static_assert(std::is_same_v<typename M::size_type, size_t>);
+    static_assert(std::is_same_v<typename M::difference_type, ptrdiff_t>);
+    static_assert(requires { typename M::iterator; });
+    static_assert(requires { typename M::const_iterator; });
+    static_assert(std::is_same_v<typename M::reverse_iterator, std::reverse_iterator<typename M::iterator>>);
+    static_assert(
+        std::is_same_v<typename M::const_reverse_iterator, std::reverse_iterator<typename M::const_iterator>>);
+    static_assert(std::is_same_v<typename M::key_container_type, std::vector<int>>);
+    static_assert(std::is_same_v<typename M::mapped_container_type, std::vector<double>>);
+    static_assert(requires { typename M::value_compare; });
+    static_assert(requires { typename M::containers; });
+    static_assert(std::is_same_v<decltype(M::containers::keys), std::vector<int>>);
+    static_assert(std::is_same_v<decltype(M::containers::values), std::vector<double>>);
+  }
+
+  {
+    struct A {};
+    struct Compare {
+      bool operator()(const std::string&, const std::string&) const;
+    };
+    using M = std::flat_map<std::string, A, Compare, std::deque<std::string>, std::deque<A>>;
+    static_assert(std::is_same_v<typename M::key_type, std::string>);
+    static_assert(std::is_same_v<typename M::mapped_type, A>);
+    static_assert(std::is_same_v<typename M::value_type, std::pair<std::string, A>>);
+    static_assert(std::is_same_v<typename M::key_compare, Compare>);
+    static_assert(std::is_same_v<typename M::reference, std::pair<const std::string&, A&>>);
+    static_assert(std::is_same_v<typename M::const_reference, std::pair<const std::string&, const A&>>);
+    static_assert(std::is_same_v<typename M::size_type, size_t>);
+    static_assert(std::is_same_v<typename M::difference_type, ptrdiff_t>);
+    static_assert(requires { typename M::iterator; });
+    static_assert(requires { typename M::const_iterator; });
+    static_assert(std::is_same_v<typename M::reverse_iterator, std::reverse_iterator<typename M::iterator>>);
+    static_assert(
+        std::is_same_v<typename M::const_reverse_iterator, std::reverse_iterator<typename M::const_iterator>>);
+    static_assert(std::is_same_v<typename M::key_container_type, std::deque<std::string>>);
+    static_assert(std::is_same_v<typename M::mapped_container_type, std::deque<A>>);
+    static_assert(requires { typename M::value_compare; });
+    static_assert(requires { typename M::containers; });
+    static_assert(std::is_same_v<decltype(M::containers::keys), std::deque<std::string>>);
+    static_assert(std::is_same_v<decltype(M::containers::values), std::deque<A>>);
+  }
+  {
+    using C = std::flat_map<int, short>;
+    static_assert(std::is_same_v<C::key_type, int>);
+    static_assert(std::is_same_v<C::mapped_type, short>);
+    static_assert(std::is_same_v<C::value_type, std::pair<int, short>>);
+    static_assert(std::is_same_v<C::key_compare, std::less<int>>);
+    static_assert(!std::is_same_v<C::value_compare, std::less<int>>);
+    static_assert(std::is_same_v<C::reference, std::pair<const int&, short&>>);
+    static_assert(std::is_same_v<C::const_reference, std::pair<const int&, const short&>>);
+    static_assert(std::random_access_iterator<C::iterator>);
+    static_assert(std::random_access_iterator<C::const_iterator>);
+    static_assert(std::random_access_iterator<C::reverse_iterator>);
+    static_assert(std::random_access_iterator<C::const_reverse_iterator>);
+    static_assert(std::is_same_v<C::reverse_iterator, std::reverse_iterator<C::iterator>>);
+    static_assert(std::is_same_v<C::const_reverse_iterator, std::reverse_iterator<C::const_iterator>>);
+    static_assert(std::is_same_v<C::size_type, std::size_t>);
+    static_assert(std::is_same_v<C::difference_type, std::ptrdiff_t>);
+    static_assert(std::is_same_v<C::key_container_type, std::vector<int>>);
+    static_assert(std::is_same_v<C::mapped_container_type, std::vector<short>>);
+  }
+  {
+    using C = std::flat_map<short, int, std::greater<long>, std::deque<short, min_allocator<short>>>;
+    static_assert(std::is_same_v<C::key_type, short>);
+    static_assert(std::is_same_v<C::mapped_type, int>);
+    static_assert(std::is_same_v<C::value_type, std::pair<short, int>>);
+    static_assert(std::is_same_v<C::key_compare, std::greater<long>>);
+    static_assert(!std::is_same_v<C::value_compare, std::greater<long>>);
+    static_assert(std::is_same_v<C::reference, std::pair<const short&, int&>>);
+    static_assert(std::is_same_v<C::const_reference, std::pair<const short&, const int&>>);
+    static_assert(std::random_access_iterator<C::iterator>);
+    static_assert(std::random_access_iterator<C::const_iterator>);
+    static_assert(std::random_access_iterator<C::reverse_iterator>);
+    static_assert(std::random_access_iterator<C::const_reverse_iterator>);
+    static_assert(std::is_same_v<C::reverse_iterator, std::reverse_iterator<C::iterator>>);
+    static_assert(std::is_same_v<C::const_reverse_iterator, std::reverse_iterator<C::const_iterator>>);
+    // size_type is invariably size_t
+    static_assert(std::is_same_v<C::size_type, std::size_t>);
+    static_assert(std::is_same_v<C::difference_type, std::ptrdiff_t>);
+    static_assert(std::is_same_v<C::key_container_type, std::deque<short, min_allocator<short>>>);
+    static_assert(std::is_same_v<C::mapped_container_type, std::vector<int>>);
+  }
+}
diff --git a/libcxx/test/support/MinSequenceContainer.h b/libcxx/test/support/MinSequenceContainer.h
new file mode 100644
index 000000000000..d0e29ae40c40
--- /dev/null
+++ b/libcxx/test/support/MinSequenceContainer.h
@@ -0,0 +1,83 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SUPPORT_MIN_SEQUENCE_CONTAINER_H
+#define SUPPORT_MIN_SEQUENCE_CONTAINER_H
+
+#include <initializer_list>
+#include <vector>
+
+#include "test_iterators.h"
+
+template <class T, class Iterator = random_access_iterator<T*>, class ConstIterator = random_access_iterator<const T*>>
+struct MinSequenceContainer {
+  using value_type      = T;
+  using difference_type = int;
+  using size_type       = unsigned int;
+  using iterator        = Iterator;
+  using const_iterator  = ConstIterator;
+
+  explicit MinSequenceContainer() = default;
+  template <class It>
+  explicit MinSequenceContainer(It first, It last) : data_(first, last) {}
+  MinSequenceContainer(std::initializer_list<T> il) : data_(il) {}
+  iterator begin() { return iterator(data_.data()); }
+  const_iterator begin() const { return const_iterator(data_.data()); }
+  const_iterator cbegin() const { return const_iterator(data_.data()); }
+  iterator end() { return begin() + size(); }
+  const_iterator end() const { return begin() + size(); }
+  size_type size() const { return data_.size(); }
+  bool empty() const { return data_.empty(); }
+
+  void clear() { data_.clear(); }
+
+  template <class It>
+  iterator insert(const_iterator p, It first, It last) {
+    return from_vector_iterator(data_.insert(to_vector_iterator(p), first, last));
+  }
+
+  iterator insert(const_iterator p, T value) {
+    return from_vector_iterator(data_.insert(to_vector_iterator(p), std::move(value)));
+  }
+
+  iterator erase(const_iterator first, const_iterator last) {
+    return from_vector_iterator(data_.erase(to_vector_iterator(first), to_vector_iterator(last)));
+  }
+
+  iterator erase(const_iterator iter) { return from_vector_iterator(data_.erase(to_vector_iterator(iter))); }
+
+  template <class... Args>
+  iterator emplace(const_iterator pos, Args&&... args) {
+    return from_vector_iterator(data_.emplace(to_vector_iterator(pos), std::forward<Args>(args)...));
+  }
+
+private:
+  std::vector<T>::const_iterator to_vector_iterator(const_iterator cit) const { return cit - cbegin() + data_.begin(); }
+
+  iterator from_vector_iterator(std::vector<T>::iterator it) { return it - data_.begin() + begin(); }
+
+  std::vector<T> data_;
+};
+
+namespace MinSequenceContainer_detail {
+
+// MinSequenceContainer is non-allocator-aware, because flat_set supports
+// such (non-STL) container types, and we want to make sure they are supported.
+template <class T>
+concept HasAllocatorType = requires { typename T::allocator_type; };
+static_assert(!HasAllocatorType<MinSequenceContainer<int>>);
+
+// MinSequenceContainer by itself doesn't support .emplace(), because we want
+// to at least somewhat support (non-STL) container types with nothing but .insert().
+template <class T>
+concept HasEmplace = requires(T& t) { t.emplace(42); };
+static_assert(!HasEmplace<MinSequenceContainer<int>>);
+
+} // namespace MinSequenceContainer_detail
+
+#endif // SUPPORT_MIN_SEQUENCE_CONTAINER_H
diff --git a/libcxx/utils/libcxx/header_information.py b/libcxx/utils/libcxx/header_information.py
index 528eb9995e19..3b12dcb9f56c 100644
--- a/libcxx/utils/libcxx/header_information.py
+++ b/libcxx/utils/libcxx/header_information.py
@@ -163,7 +163,6 @@ module_c_headers = [h for h in all_headers if h.has_cxx20_module() and h.is_cstd
 # modules will fail to build if a header is added but this list is not updated.
 headers_not_available = list(map(Header, [
     "debugging",
-    "flat_map",
     "flat_set",
     "generator",
     "hazard_pointer",
@@ -251,6 +250,7 @@ mandatory_inclusions = {
     "coroutine": ["compare"],
     "deque": ["compare", "initializer_list"],
     "filesystem": ["compare"],
+    "flat_map": ["compare", "initializer_list"],
     "forward_list": ["compare", "initializer_list"],
     "ios": ["iosfwd"],
     "iostream": ["ios", "istream", "ostream", "streambuf"],
-- 
GitLab


From 98ca9a635bd2fb98cee473a9558687a5b522e219 Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyulee@meta.com>
Date: Sat, 26 Oct 2024 09:20:26 -0700
Subject: [PATCH 245/693] Reland [StructuralHash] Refactor (#112621)

This is largely NFC, and it prepares for #112638.
 - Use stable_hash instead of uint64_t
 - Rename update* to hash* functions. They compute stable_hash locally and return it.

This is a patch for
https://discourse.llvm.org/t/rfc-global-function-merging/82608.
---
 llvm/include/llvm/IR/StructuralHash.h         |   7 +-
 llvm/lib/IR/StructuralHash.cpp                | 133 +++++++++++-------
 llvm/lib/Transforms/IPO/MergeFunctions.cpp    |   6 +-
 .../call-and-invoke-with-ranges-attr.ll       |  12 +-
 .../MergeFunc/call-and-invoke-with-ranges.ll  |   8 +-
 llvm/test/Transforms/MergeFunc/inline-asm.ll  |   8 +-
 6 files changed, 106 insertions(+), 68 deletions(-)

diff --git a/llvm/include/llvm/IR/StructuralHash.h b/llvm/include/llvm/IR/StructuralHash.h
index 57fb45db8491..e2e192cc9501 100644
--- a/llvm/include/llvm/IR/StructuralHash.h
+++ b/llvm/include/llvm/IR/StructuralHash.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_IR_STRUCTURALHASH_H
 #define LLVM_IR_STRUCTURALHASH_H
 
+#include "llvm/ADT/StableHashing.h"
 #include <cstdint>
 
 namespace llvm {
@@ -21,20 +22,18 @@ namespace llvm {
 class Function;
 class Module;
 
-using IRHash = uint64_t;
-
 /// Returns a hash of the function \p F.
 /// \param F The function to hash.
 /// \param DetailedHash Whether or not to encode additional information in the
 /// hash. The additional information added into the hash when this flag is set
 /// to true includes instruction and operand type information.
-IRHash StructuralHash(const Function &F, bool DetailedHash = false);
+stable_hash StructuralHash(const Function &F, bool DetailedHash = false);
 
 /// Returns a hash of the module \p M by hashing all functions and global
 /// variables contained within. \param M The module to hash. \param DetailedHash
 /// Whether or not to encode additional information in the function hashes that
 /// composed the module hash.
-IRHash StructuralHash(const Module &M, bool DetailedHash = false);
+stable_hash StructuralHash(const Module &M, bool DetailedHash = false);
 
 } // end namespace llvm
 
diff --git a/llvm/lib/IR/StructuralHash.cpp b/llvm/lib/IR/StructuralHash.cpp
index fb4f33a021a9..267a085c5af7 100644
--- a/llvm/lib/IR/StructuralHash.cpp
+++ b/llvm/lib/IR/StructuralHash.cpp
@@ -24,61 +24,93 @@ namespace {
 // by the MergeFunctions pass.
 
 class StructuralHashImpl {
-  uint64_t Hash = 4;
+  stable_hash Hash = 4;
 
-  void hash(uint64_t V) { Hash = hashing::detail::hash_16_bytes(Hash, V); }
+  bool DetailedHash;
+
+  // This random value acts as a block header, as otherwise the partition of
+  // opcodes into BBs wouldn't affect the hash, only the order of the opcodes.
+  static constexpr stable_hash BlockHeaderHash = 45798;
+  static constexpr stable_hash FunctionHeaderHash = 0x62642d6b6b2d6b72;
+  static constexpr stable_hash GlobalHeaderHash = 23456;
 
   // This will produce different values on 32-bit and 64-bit systens as
   // hash_combine returns a size_t. However, this is only used for
   // detailed hashing which, in-tree, only needs to distinguish between
   // differences in functions.
-  template <typename T> void hashArbitaryType(const T &V) {
-    hash(hash_combine(V));
+  // TODO: This is not stable.
+  template <typename T> stable_hash hashArbitaryType(const T &V) {
+    return hash_combine(V);
   }
 
-  void hashType(Type *ValueType) {
-    hash(ValueType->getTypeID());
+  stable_hash hashType(Type *ValueType) {
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(ValueType->getTypeID());
     if (ValueType->isIntegerTy())
-      hash(ValueType->getIntegerBitWidth());
+      Hashes.emplace_back(ValueType->getIntegerBitWidth());
+    return stable_hash_combine(Hashes);
   }
 
 public:
-  StructuralHashImpl() = default;
-
-  void updateOperand(Value *Operand) {
-    hashType(Operand->getType());
-
-    // The cases enumerated below are not exhaustive and are only aimed to
-    // get decent coverage over the function.
-    if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(Operand)) {
-      hashArbitaryType(ConstInt->getValue());
-    } else if (ConstantFP *ConstFP = dyn_cast<ConstantFP>(Operand)) {
-      hashArbitaryType(ConstFP->getValue());
-    } else if (Argument *Arg = dyn_cast<Argument>(Operand)) {
-      hash(Arg->getArgNo());
-    } else if (Function *Func = dyn_cast<Function>(Operand)) {
+  StructuralHashImpl() = delete;
+  explicit StructuralHashImpl(bool DetailedHash) : DetailedHash(DetailedHash) {}
+
+  stable_hash hashConstant(Constant *C) {
+    SmallVector<stable_hash> Hashes;
+    // TODO: hashArbitaryType() is not stable.
+    if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(C)) {
+      Hashes.emplace_back(hashArbitaryType(ConstInt->getValue()));
+    } else if (ConstantFP *ConstFP = dyn_cast<ConstantFP>(C)) {
+      Hashes.emplace_back(hashArbitaryType(ConstFP->getValue()));
+    } else if (Function *Func = dyn_cast<Function>(C)) {
       // Hashing the name will be deterministic as LLVM's hashing infrastructure
       // has explicit support for hashing strings and will not simply hash
       // the pointer.
-      hashArbitaryType(Func->getName());
+      Hashes.emplace_back(hashArbitaryType(Func->getName()));
     }
+
+    return stable_hash_combine(Hashes);
+  }
+
+  stable_hash hashValue(Value *V) {
+    // Check constant and return its hash.
+    Constant *C = dyn_cast<Constant>(V);
+    if (C)
+      return hashConstant(C);
+
+    // Hash argument number.
+    SmallVector<stable_hash> Hashes;
+    if (Argument *Arg = dyn_cast<Argument>(V))
+      Hashes.emplace_back(Arg->getArgNo());
+
+    return stable_hash_combine(Hashes);
   }
 
-  void updateInstruction(const Instruction &Inst, bool DetailedHash) {
-    hash(Inst.getOpcode());
+  stable_hash hashOperand(Value *Operand) {
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(hashType(Operand->getType()));
+    Hashes.emplace_back(hashValue(Operand));
+    return stable_hash_combine(Hashes);
+  }
+
+  stable_hash hashInstruction(const Instruction &Inst) {
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(Inst.getOpcode());
 
     if (!DetailedHash)
-      return;
+      return stable_hash_combine(Hashes);
 
-    hashType(Inst.getType());
+    Hashes.emplace_back(hashType(Inst.getType()));
 
     // Handle additional properties of specific instructions that cause
     // semantic differences in the IR.
     if (const auto *ComparisonInstruction = dyn_cast<CmpInst>(&Inst))
-      hash(ComparisonInstruction->getPredicate());
+      Hashes.emplace_back(ComparisonInstruction->getPredicate());
 
     for (const auto &Op : Inst.operands())
-      updateOperand(Op);
+      Hashes.emplace_back(hashOperand(Op));
+
+    return stable_hash_combine(Hashes);
   }
 
   // A function hash is calculated by considering only the number of arguments
@@ -97,15 +129,17 @@ public:
   // expensive checks for pass modification status). When modifying this
   // function, most changes should be gated behind an option and enabled
   // selectively.
-  void update(const Function &F, bool DetailedHash) {
+  void update(const Function &F) {
     // Declarations don't affect analyses.
     if (F.isDeclaration())
       return;
 
-    hash(0x62642d6b6b2d6b72); // Function header
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(Hash);
+    Hashes.emplace_back(FunctionHeaderHash);
 
-    hash(F.isVarArg());
-    hash(F.arg_size());
+    Hashes.emplace_back(F.isVarArg());
+    Hashes.emplace_back(F.arg_size());
 
     SmallVector<const BasicBlock *, 8> BBs;
     SmallPtrSet<const BasicBlock *, 16> VisitedBBs;
@@ -118,17 +152,17 @@ public:
     while (!BBs.empty()) {
       const BasicBlock *BB = BBs.pop_back_val();
 
-      // This random value acts as a block header, as otherwise the partition of
-      // opcodes into BBs wouldn't affect the hash, only the order of the
-      // opcodes
-      hash(45798);
+      Hashes.emplace_back(BlockHeaderHash);
       for (auto &Inst : *BB)
-        updateInstruction(Inst, DetailedHash);
+        Hashes.emplace_back(hashInstruction(Inst));
 
       for (const BasicBlock *Succ : successors(BB))
         if (VisitedBBs.insert(Succ).second)
           BBs.push_back(Succ);
     }
+
+    // Update the combined hash in place.
+    Hash = stable_hash_combine(Hashes);
   }
 
   void update(const GlobalVariable &GV) {
@@ -137,15 +171,20 @@ public:
     // we ignore anything with the `.llvm` prefix
     if (GV.isDeclaration() || GV.getName().starts_with("llvm."))
       return;
-    hash(23456); // Global header
-    hash(GV.getValueType()->getTypeID());
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(Hash);
+    Hashes.emplace_back(GlobalHeaderHash);
+    Hashes.emplace_back(GV.getValueType()->getTypeID());
+
+    // Update the combined hash in place.
+    Hash = stable_hash_combine(Hashes);
   }
 
-  void update(const Module &M, bool DetailedHash) {
+  void update(const Module &M) {
     for (const GlobalVariable &GV : M.globals())
       update(GV);
     for (const Function &F : M)
-      update(F, DetailedHash);
+      update(F);
   }
 
   uint64_t getHash() const { return Hash; }
@@ -153,14 +192,14 @@ public:
 
 } // namespace
 
-IRHash llvm::StructuralHash(const Function &F, bool DetailedHash) {
-  StructuralHashImpl H;
-  H.update(F, DetailedHash);
+stable_hash llvm::StructuralHash(const Function &F, bool DetailedHash) {
+  StructuralHashImpl H(DetailedHash);
+  H.update(F);
   return H.getHash();
 }
 
-IRHash llvm::StructuralHash(const Module &M, bool DetailedHash) {
-  StructuralHashImpl H;
-  H.update(M, DetailedHash);
+stable_hash llvm::StructuralHash(const Module &M, bool DetailedHash) {
+  StructuralHashImpl H(DetailedHash);
+  H.update(M);
   return H.getHash();
 }
diff --git a/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index b50a700e0903..ad16b0b35014 100644
--- a/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -172,14 +172,14 @@ namespace {
 
 class FunctionNode {
   mutable AssertingVH<Function> F;
-  IRHash Hash;
+  stable_hash Hash;
 
 public:
   // Note the hash is recalculated potentially multiple times, but it is cheap.
   FunctionNode(Function *F) : F(F), Hash(StructuralHash(*F)) {}
 
   Function *getFunc() const { return F; }
-  IRHash getHash() const { return Hash; }
+  stable_hash getHash() const { return Hash; }
 
   /// Replace the reference to the function F by the function G, assuming their
   /// implementations are equal.
@@ -420,7 +420,7 @@ bool MergeFunctions::runOnModule(Module &M) {
 
   // All functions in the module, ordered by hash. Functions with a unique
   // hash value are easily eliminated.
-  std::vector<std::pair<IRHash, Function *>> HashedFuncs;
+  std::vector<std::pair<stable_hash, Function *>> HashedFuncs;
   for (Function &Func : M) {
     if (isEligibleForMerging(Func)) {
       HashedFuncs.push_back({StructuralHash(Func), &Func});
diff --git a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll
index e5d62319bf9d..cbf14165548e 100644
--- a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll
+++ b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll
@@ -80,8 +80,8 @@ lpad:
 }
 
 define i8 @invoke_with_same_range() personality ptr undef {
-; CHECK-LABEL: @invoke_with_same_range()
-; CHECK: tail call i8 @invoke_with_range()
+; CHECK-DAG: @invoke_with_same_range()
+; CHECK-DAG: tail call i8 @invoke_with_range()
   %out = invoke range(i8 0, 2) i8 @dummy() to label %next unwind label %lpad
 
 next:
@@ -93,15 +93,15 @@ lpad:
 }
 
 define i8 @call_with_same_range() {
-; CHECK-LABEL: @call_with_same_range
-; CHECK: tail call i8 @call_with_range
+; CHECK-DAG: @call_with_same_range
+; CHECK-DAG: tail call i8 @call_with_range
   %out = call range(i8 0, 2) i8 @dummy()
   ret i8 %out
 }
 
 define i8 @call_with_same_range_attr(i8 range(i8 0, 2) %v) {
-; CHECK-LABEL: @call_with_same_range_attr
-; CHECK: tail call i8 @call_with_range_attr
+; CHECK-DAG: @call_with_same_range_attr
+; CHECK-DAG: tail call i8 @call_with_range_attr
   %out = call i8 @dummy2(i8 %v)
   ret i8 %out
 }
diff --git a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
index e7718ca84d31..39e5a11181a4 100644
--- a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
+++ b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
@@ -64,8 +64,8 @@ lpad:
 }
 
 define i8 @invoke_with_same_range() personality ptr undef {
-; CHECK-LABEL: @invoke_with_same_range()
-; CHECK: tail call i8 @invoke_with_range()
+; CHECK-DAG: @invoke_with_same_range()
+; CHECK-DAG: tail call i8 @invoke_with_range()
   %out = invoke i8 @dummy() to label %next unwind label %lpad, !range !0
 
 next:
@@ -77,8 +77,8 @@ lpad:
 }
 
 define i8 @call_with_same_range() {
-; CHECK-LABEL: @call_with_same_range
-; CHECK: tail call i8 @call_with_range
+; CHECK-DAG: @call_with_same_range
+; CHECK-DAG: tail call i8 @call_with_range
   bitcast i8 0 to i8
   %out = call i8 @dummy(), !range !0
   ret i8 %out
diff --git a/llvm/test/Transforms/MergeFunc/inline-asm.ll b/llvm/test/Transforms/MergeFunc/inline-asm.ll
index 7cc6afd2f8f7..970757e8d53a 100644
--- a/llvm/test/Transforms/MergeFunc/inline-asm.ll
+++ b/llvm/test/Transforms/MergeFunc/inline-asm.ll
@@ -3,11 +3,11 @@
 ; CHECK-LABEL: @int_ptr_arg_different
 ; CHECK-NEXT: call void asm
 
-; CHECK-LABEL: @int_ptr_null
-; CHECK-NEXT: tail call void @float_ptr_null()
+; CHECK-DAG: @int_ptr_null
+; CHECK-DAG: tail call void @float_ptr_null()
 
-; CHECK-LABEL: @int_ptr_arg_same
-; CHECK-NEXT: tail call void @float_ptr_arg_same(ptr %0)
+; CHECK-DAG: @int_ptr_arg_same
+; CHECK-DAG: tail call void @float_ptr_arg_same(ptr %0)
 
 ; Used to satisfy minimum size limit
 declare void @stuff()
-- 
GitLab


From d78d030b31bb3bde822115fd9d26ab887f890203 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 26 Oct 2024 12:26:11 -0700
Subject: [PATCH 246/693] [MC] Export llvm::XCOFFObjectwriter and access it
 from MCXCOFFStreamer

Similar to commit 9539a7796094ff5fb59d9c685140ea2e214b945c for
WinCOFFObjectWriter.

Close #100412
---
 llvm/include/llvm/MC/MCXCOFFObjectWriter.h |  17 ++-
 llvm/include/llvm/MC/MCXCOFFStreamer.h     |   3 +
 llvm/lib/MC/MCXCOFFStreamer.cpp            |  12 +-
 llvm/lib/MC/XCOFFObjectWriter.cpp          | 149 +++++++++------------
 4 files changed, 83 insertions(+), 98 deletions(-)

diff --git a/llvm/include/llvm/MC/MCXCOFFObjectWriter.h b/llvm/include/llvm/MC/MCXCOFFObjectWriter.h
index c0e32a70172d..968d938a6549 100644
--- a/llvm/include/llvm/MC/MCXCOFFObjectWriter.h
+++ b/llvm/include/llvm/MC/MCXCOFFObjectWriter.h
@@ -39,19 +39,18 @@ private:
   bool Is64Bit;
 };
 
+class XCOFFObjectWriter : public MCObjectWriter {
+public:
+  virtual void addExceptionEntry(const MCSymbol *Symbol, const MCSymbol *Trap,
+                                 unsigned LanguageCode, unsigned ReasonCode,
+                                 unsigned FunctionSize, bool hasDebug) = 0;
+  virtual void addCInfoSymEntry(StringRef Name, StringRef Metadata) = 0;
+};
+
 std::unique_ptr<MCObjectWriter>
 createXCOFFObjectWriter(std::unique_ptr<MCXCOFFObjectTargetWriter> MOTW,
                         raw_pwrite_stream &OS);
 
-namespace XCOFF {
-void addExceptionEntry(MCObjectWriter &Writer, const MCSymbol *Symbol,
-                       const MCSymbol *Trap, unsigned LanguageCode,
-                       unsigned ReasonCode, unsigned FunctionSize,
-                       bool hasDebug);
-void addCInfoSymEntry(MCObjectWriter &Writer, StringRef Name,
-                      StringRef Metadata);
-} // namespace XCOFF
-
 } // end namespace llvm
 
 #endif // LLVM_MC_MCXCOFFOBJECTWRITER_H
diff --git a/llvm/include/llvm/MC/MCXCOFFStreamer.h b/llvm/include/llvm/MC/MCXCOFFStreamer.h
index 8cae64fa33be..3c7abfea85ec 100644
--- a/llvm/include/llvm/MC/MCXCOFFStreamer.h
+++ b/llvm/include/llvm/MC/MCXCOFFStreamer.h
@@ -12,6 +12,7 @@
 #include "llvm/MC/MCObjectStreamer.h"
 
 namespace llvm {
+class XCOFFObjectWriter;
 
 class MCXCOFFStreamer : public MCObjectStreamer {
 public:
@@ -19,6 +20,8 @@ public:
                   std::unique_ptr<MCObjectWriter> OW,
                   std::unique_ptr<MCCodeEmitter> Emitter);
 
+  XCOFFObjectWriter &getWriter();
+
   bool emitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override;
   void emitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                         Align ByteAlignment) override;
diff --git a/llvm/lib/MC/MCXCOFFStreamer.cpp b/llvm/lib/MC/MCXCOFFStreamer.cpp
index 94aa1ebc8f9e..f5b83f29352c 100644
--- a/llvm/lib/MC/MCXCOFFStreamer.cpp
+++ b/llvm/lib/MC/MCXCOFFStreamer.cpp
@@ -32,6 +32,10 @@ MCXCOFFStreamer::MCXCOFFStreamer(MCContext &Context,
     : MCObjectStreamer(Context, std::move(MAB), std::move(OW),
                        std::move(Emitter)) {}
 
+XCOFFObjectWriter &MCXCOFFStreamer::getWriter() {
+  return static_cast<XCOFFObjectWriter &>(getAssembler().getWriter());
+}
+
 bool MCXCOFFStreamer::emitSymbolAttribute(MCSymbol *Sym,
                                           MCSymbolAttr Attribute) {
   auto *Symbol = cast<MCSymbolXCOFF>(Sym);
@@ -109,14 +113,12 @@ void MCXCOFFStreamer::emitXCOFFExceptDirective(const MCSymbol *Symbol,
                                                unsigned Lang, unsigned Reason,
                                                unsigned FunctionSize,
                                                bool hasDebug) {
-  // TODO: Export XCOFFObjectWriter to llvm/MC/MCXCOFFObjectWriter.h and access
-  // it from MCXCOFFStreamer.
-  XCOFF::addExceptionEntry(getAssembler().getWriter(), Symbol, Trap, Lang,
-                           Reason, FunctionSize, hasDebug);
+  getWriter().addExceptionEntry(Symbol, Trap, Lang, Reason, FunctionSize,
+                                hasDebug);
 }
 
 void MCXCOFFStreamer::emitXCOFFCInfoSym(StringRef Name, StringRef Metadata) {
-  XCOFF::addCInfoSymEntry(getAssembler().getWriter(), Name, Metadata);
+  getWriter().addCInfoSymEntry(Name, Metadata);
 }
 
 void MCXCOFFStreamer::emitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
diff --git a/llvm/lib/MC/XCOFFObjectWriter.cpp b/llvm/lib/MC/XCOFFObjectWriter.cpp
index c7f29c73eaac..5d8f3dbdaada 100644
--- a/llvm/lib/MC/XCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/XCOFFObjectWriter.cpp
@@ -290,8 +290,7 @@ struct CInfoSymSectionEntry : public SectionEntry {
   }
 };
 
-class XCOFFObjectWriter : public MCObjectWriter {
-
+class XCOFFWriter final : public XCOFFObjectWriter {
   uint32_t SymbolTableEntryCount = 0;
   uint64_t SymbolTableOffset = 0;
   uint16_t SectionCount = 0;
@@ -433,8 +432,8 @@ class XCOFFObjectWriter : public MCObjectWriter {
   }
 
 public:
-  XCOFFObjectWriter(std::unique_ptr<MCXCOFFObjectTargetWriter> MOTW,
-                    raw_pwrite_stream &OS);
+  XCOFFWriter(std::unique_ptr<MCXCOFFObjectTargetWriter> MOTW,
+              raw_pwrite_stream &OS);
 
   void writeWord(uint64_t Word) {
     is64Bit() ? W.write<uint64_t>(Word) : W.write<uint32_t>(Word);
@@ -442,12 +441,12 @@ public:
 
   void addExceptionEntry(const MCSymbol *Symbol, const MCSymbol *Trap,
                          unsigned LanguageCode, unsigned ReasonCode,
-                         unsigned FunctionSize, bool hasDebug);
-  void addCInfoSymEntry(StringRef Name, StringRef Metadata);
+                         unsigned FunctionSize, bool hasDebug) override;
+  void addCInfoSymEntry(StringRef Name, StringRef Metadata) override;
 };
 
-XCOFFObjectWriter::XCOFFObjectWriter(
-    std::unique_ptr<MCXCOFFObjectTargetWriter> MOTW, raw_pwrite_stream &OS)
+XCOFFWriter::XCOFFWriter(std::unique_ptr<MCXCOFFObjectTargetWriter> MOTW,
+                         raw_pwrite_stream &OS)
     : W(OS, llvm::endianness::big), TargetObjectWriter(std::move(MOTW)),
       Strings(StringTableBuilder::XCOFF),
       Text(".text", XCOFF::STYP_TEXT, /* IsVirtual */ false,
@@ -463,7 +462,7 @@ XCOFFObjectWriter::XCOFFObjectWriter(
       ExceptionSection(".except", XCOFF::STYP_EXCEPT),
       CInfoSymSection(".info", XCOFF::STYP_INFO) {}
 
-void XCOFFObjectWriter::reset() {
+void XCOFFWriter::reset() {
   // Clear the mappings we created.
   SymbolIndexMap.clear();
   SectionMap.clear();
@@ -479,7 +478,7 @@ void XCOFFObjectWriter::reset() {
   ExceptionSection.reset();
   CInfoSymSection.reset();
 
-  // Reset states in XCOFFObjectWriter.
+  // Reset states in XCOFFWriter.
   SymbolTableEntryCount = 0;
   SymbolTableOffset = 0;
   SectionCount = 0;
@@ -489,7 +488,7 @@ void XCOFFObjectWriter::reset() {
   MCObjectWriter::reset();
 }
 
-CsectGroup &XCOFFObjectWriter::getCsectGroup(const MCSectionXCOFF *MCSec) {
+CsectGroup &XCOFFWriter::getCsectGroup(const MCSectionXCOFF *MCSec) {
   switch (MCSec->getMappingClass()) {
   case XCOFF::XMC_PR:
     assert(XCOFF::XTY_SD == MCSec->getCSectType() &&
@@ -556,7 +555,7 @@ static MCSectionXCOFF *getContainingCsect(const MCSymbolXCOFF *XSym) {
   return XSym->getRepresentedCsect();
 }
 
-void XCOFFObjectWriter::executePostLayoutBinding(MCAssembler &Asm) {
+void XCOFFWriter::executePostLayoutBinding(MCAssembler &Asm) {
   for (const auto &S : Asm) {
     const auto *MCSec = cast<const MCSectionXCOFF>(&S);
     assert(!SectionMap.contains(MCSec) && "Cannot add a section twice.");
@@ -657,10 +656,9 @@ void XCOFFObjectWriter::executePostLayoutBinding(MCAssembler &Asm) {
   assignAddressesAndIndices(Asm);
 }
 
-void XCOFFObjectWriter::recordRelocation(MCAssembler &Asm,
-                                         const MCFragment *Fragment,
-                                         const MCFixup &Fixup, MCValue Target,
-                                         uint64_t &FixedValue) {
+void XCOFFWriter::recordRelocation(MCAssembler &Asm, const MCFragment *Fragment,
+                                   const MCFixup &Fixup, MCValue Target,
+                                   uint64_t &FixedValue) {
   auto getIndex = [this](const MCSymbol *Sym,
                          const MCSectionXCOFF *ContainingCsect) {
     // If we could not find the symbol directly in SymbolIndexMap, this symbol
@@ -812,7 +810,7 @@ void XCOFFObjectWriter::recordRelocation(MCAssembler &Asm,
   FixedValue -= getVirtualAddress(SymB, SymBSec);
 }
 
-void XCOFFObjectWriter::writeSections(const MCAssembler &Asm) {
+void XCOFFWriter::writeSections(const MCAssembler &Asm) {
   uint64_t CurrentAddressLocation = 0;
   for (const auto *Section : Sections)
     writeSectionForControlSectionEntry(Asm, *Section, CurrentAddressLocation);
@@ -824,7 +822,7 @@ void XCOFFObjectWriter::writeSections(const MCAssembler &Asm) {
                                       CurrentAddressLocation);
 }
 
-uint64_t XCOFFObjectWriter::writeObject(MCAssembler &Asm) {
+uint64_t XCOFFWriter::writeObject(MCAssembler &Asm) {
   // We always emit a timestamp of 0 for reproducibility, so ensure incremental
   // linking is not enabled, in case, like with Windows COFF, such a timestamp
   // is incompatible with incremental linking of XCOFF.
@@ -844,11 +842,11 @@ uint64_t XCOFFObjectWriter::writeObject(MCAssembler &Asm) {
   return W.OS.tell() - StartOffset;
 }
 
-bool XCOFFObjectWriter::nameShouldBeInStringTable(const StringRef &SymbolName) {
+bool XCOFFWriter::nameShouldBeInStringTable(const StringRef &SymbolName) {
   return SymbolName.size() > XCOFF::NameSize || is64Bit();
 }
 
-void XCOFFObjectWriter::writeSymbolName(const StringRef &SymbolName) {
+void XCOFFWriter::writeSymbolName(const StringRef &SymbolName) {
   // Magic, Offset or SymbolName.
   if (nameShouldBeInStringTable(SymbolName)) {
     W.write<int32_t>(0);
@@ -861,11 +859,10 @@ void XCOFFObjectWriter::writeSymbolName(const StringRef &SymbolName) {
   }
 }
 
-void XCOFFObjectWriter::writeSymbolEntry(StringRef SymbolName, uint64_t Value,
-                                         int16_t SectionNumber,
-                                         uint16_t SymbolType,
-                                         uint8_t StorageClass,
-                                         uint8_t NumberOfAuxEntries) {
+void XCOFFWriter::writeSymbolEntry(StringRef SymbolName, uint64_t Value,
+                                   int16_t SectionNumber, uint16_t SymbolType,
+                                   uint8_t StorageClass,
+                                   uint8_t NumberOfAuxEntries) {
   if (is64Bit()) {
     W.write<uint64_t>(Value);
     W.write<uint32_t>(Strings.getOffset(SymbolName));
@@ -879,9 +876,9 @@ void XCOFFObjectWriter::writeSymbolEntry(StringRef SymbolName, uint64_t Value,
   W.write<uint8_t>(NumberOfAuxEntries);
 }
 
-void XCOFFObjectWriter::writeSymbolAuxCsectEntry(uint64_t SectionOrLength,
-                                                 uint8_t SymbolAlignmentAndType,
-                                                 uint8_t StorageMappingClass) {
+void XCOFFWriter::writeSymbolAuxCsectEntry(uint64_t SectionOrLength,
+                                           uint8_t SymbolAlignmentAndType,
+                                           uint8_t StorageMappingClass) {
   W.write<uint32_t>(is64Bit() ? Lo_32(SectionOrLength) : SectionOrLength);
   W.write<uint32_t>(0); // ParameterHashIndex
   W.write<uint16_t>(0); // TypeChkSectNum
@@ -897,12 +894,12 @@ void XCOFFObjectWriter::writeSymbolAuxCsectEntry(uint64_t SectionOrLength,
   }
 }
 
-bool XCOFFObjectWriter::auxFileSymNameShouldBeInStringTable(
+bool XCOFFWriter::auxFileSymNameShouldBeInStringTable(
     const StringRef &SymbolName) {
   return SymbolName.size() > XCOFF::AuxFileEntNameSize;
 }
 
-void XCOFFObjectWriter::writeAuxFileSymName(const StringRef &SymbolName) {
+void XCOFFWriter::writeAuxFileSymName(const StringRef &SymbolName) {
   // Magic, Offset or SymbolName.
   if (auxFileSymNameShouldBeInStringTable(SymbolName)) {
     W.write<int32_t>(0);
@@ -916,8 +913,7 @@ void XCOFFObjectWriter::writeAuxFileSymName(const StringRef &SymbolName) {
   }
 }
 
-void XCOFFObjectWriter::writeSymbolAuxFileEntry(StringRef &Name,
-                                                uint8_t ftype) {
+void XCOFFWriter::writeSymbolAuxFileEntry(StringRef &Name, uint8_t ftype) {
   writeAuxFileSymName(Name);
   W.write<uint8_t>(ftype);
   W.OS.write_zeros(2);
@@ -927,8 +923,8 @@ void XCOFFObjectWriter::writeSymbolAuxFileEntry(StringRef &Name,
     W.OS.write_zeros(1);
 }
 
-void XCOFFObjectWriter::writeSymbolAuxDwarfEntry(
-    uint64_t LengthOfSectionPortion, uint64_t NumberOfRelocEnt) {
+void XCOFFWriter::writeSymbolAuxDwarfEntry(uint64_t LengthOfSectionPortion,
+                                           uint64_t NumberOfRelocEnt) {
   writeWord(LengthOfSectionPortion);
   if (!is64Bit())
     W.OS.write_zeros(4); // Reserved
@@ -941,7 +937,7 @@ void XCOFFObjectWriter::writeSymbolAuxDwarfEntry(
   }
 }
 
-void XCOFFObjectWriter::writeSymbolEntryForCsectMemberLabel(
+void XCOFFWriter::writeSymbolEntryForCsectMemberLabel(
     const Symbol &SymbolRef, const XCOFFSection &CSectionRef,
     int16_t SectionIndex, uint64_t SymbolOffset) {
   assert(SymbolOffset <= MaxRawDataSize - CSectionRef.Address &&
@@ -986,7 +982,7 @@ void XCOFFObjectWriter::writeSymbolEntryForCsectMemberLabel(
                            CSectionRef.MCSec->getMappingClass());
 }
 
-void XCOFFObjectWriter::writeSymbolEntryForDwarfSection(
+void XCOFFWriter::writeSymbolEntryForDwarfSection(
     const XCOFFSection &DwarfSectionRef, int16_t SectionIndex) {
   assert(DwarfSectionRef.MCSec->isDwarfSect() && "Not a DWARF section!");
 
@@ -996,7 +992,7 @@ void XCOFFObjectWriter::writeSymbolEntryForDwarfSection(
   writeSymbolAuxDwarfEntry(DwarfSectionRef.Size);
 }
 
-void XCOFFObjectWriter::writeSymbolEntryForControlSection(
+void XCOFFWriter::writeSymbolEntryForControlSection(
     const XCOFFSection &CSectionRef, int16_t SectionIndex,
     XCOFF::StorageClass StorageClass) {
   writeSymbolEntry(CSectionRef.getSymbolTableName(), CSectionRef.Address,
@@ -1006,10 +1002,10 @@ void XCOFFObjectWriter::writeSymbolEntryForControlSection(
                            CSectionRef.MCSec->getMappingClass());
 }
 
-void XCOFFObjectWriter::writeSymbolAuxFunctionEntry(uint32_t EntryOffset,
-                                                    uint32_t FunctionSize,
-                                                    uint64_t LineNumberPointer,
-                                                    uint32_t EndIndex) {
+void XCOFFWriter::writeSymbolAuxFunctionEntry(uint32_t EntryOffset,
+                                              uint32_t FunctionSize,
+                                              uint64_t LineNumberPointer,
+                                              uint32_t EndIndex) {
   if (is64Bit())
     writeWord(LineNumberPointer);
   else
@@ -1026,9 +1022,9 @@ void XCOFFObjectWriter::writeSymbolAuxFunctionEntry(uint32_t EntryOffset,
   }
 }
 
-void XCOFFObjectWriter::writeSymbolAuxExceptionEntry(uint64_t EntryOffset,
-                                                     uint32_t FunctionSize,
-                                                     uint32_t EndIndex) {
+void XCOFFWriter::writeSymbolAuxExceptionEntry(uint64_t EntryOffset,
+                                               uint32_t FunctionSize,
+                                               uint32_t EndIndex) {
   assert(is64Bit() && "Exception auxilliary entries are 64-bit only.");
   W.write<uint64_t>(EntryOffset);
   W.write<uint32_t>(FunctionSize);
@@ -1037,7 +1033,7 @@ void XCOFFObjectWriter::writeSymbolAuxExceptionEntry(uint64_t EntryOffset,
   W.write<uint8_t>(XCOFF::AUX_EXCEPT);
 }
 
-void XCOFFObjectWriter::writeFileHeader() {
+void XCOFFWriter::writeFileHeader() {
   W.write<uint16_t>(is64Bit() ? XCOFF::XCOFF64 : XCOFF::XCOFF32);
   W.write<uint16_t>(SectionCount);
   W.write<int32_t>(0); // TimeStamp
@@ -1053,7 +1049,7 @@ void XCOFFObjectWriter::writeFileHeader() {
   }
 }
 
-void XCOFFObjectWriter::writeAuxFileHeader() {
+void XCOFFWriter::writeAuxFileHeader() {
   if (!auxiliaryHeaderSize())
     return;
   W.write<uint16_t>(0); // Magic
@@ -1069,7 +1065,7 @@ void XCOFFObjectWriter::writeAuxFileHeader() {
   W.write<uint32_t>(Sections[1]->Address); // DataStartAddr
 }
 
-void XCOFFObjectWriter::writeSectionHeader(const SectionEntry *Sec) {
+void XCOFFWriter::writeSectionHeader(const SectionEntry *Sec) {
   bool IsDwarf = (Sec->Flags & XCOFF::STYP_DWARF) != 0;
   bool IsOvrflo = (Sec->Flags & XCOFF::STYP_OVRFLO) != 0;
   // Nothing to write for this Section.
@@ -1109,7 +1105,7 @@ void XCOFFObjectWriter::writeSectionHeader(const SectionEntry *Sec) {
   }
 }
 
-void XCOFFObjectWriter::writeSectionHeaderTable() {
+void XCOFFWriter::writeSectionHeaderTable() {
   for (const auto *CsectSec : Sections)
     writeSectionHeader(CsectSec);
   for (const auto &DwarfSec : DwarfSections)
@@ -1122,8 +1118,8 @@ void XCOFFObjectWriter::writeSectionHeaderTable() {
     writeSectionHeader(&CInfoSymSection);
 }
 
-void XCOFFObjectWriter::writeRelocation(XCOFFRelocation Reloc,
-                                        const XCOFFSection &Section) {
+void XCOFFWriter::writeRelocation(XCOFFRelocation Reloc,
+                                  const XCOFFSection &Section) {
   if (Section.MCSec->isCsect())
     writeWord(Section.Address + Reloc.FixupOffsetInCsect);
   else {
@@ -1136,7 +1132,7 @@ void XCOFFObjectWriter::writeRelocation(XCOFFRelocation Reloc,
   W.write<uint8_t>(Reloc.Type);
 }
 
-void XCOFFObjectWriter::writeRelocations() {
+void XCOFFWriter::writeRelocations() {
   for (const auto *Section : Sections) {
     if (Section->Index == SectionEntry::UninitializedIndex)
       // Nothing to write for this Section.
@@ -1158,7 +1154,7 @@ void XCOFFObjectWriter::writeRelocations() {
       writeRelocation(Reloc, *DwarfSection.DwarfSect);
 }
 
-void XCOFFObjectWriter::writeSymbolTable(MCAssembler &Asm) {
+void XCOFFWriter::writeSymbolTable(MCAssembler &Asm) {
   // Write C_FILE symbols.
   StringRef Vers = CompilerVersion;
 
@@ -1239,8 +1235,7 @@ void XCOFFObjectWriter::writeSymbolTable(MCAssembler &Asm) {
                                     DwarfSection.Index);
 }
 
-void XCOFFObjectWriter::finalizeRelocationInfo(SectionEntry *Sec,
-                                               uint64_t RelCount) {
+void XCOFFWriter::finalizeRelocationInfo(SectionEntry *Sec, uint64_t RelCount) {
   // Handles relocation field overflows in an XCOFF32 file. An XCOFF64 file
   // may not contain an overflow section header.
   if (!is64Bit() && (RelCount >= static_cast<uint32_t>(XCOFF::RelocOverflow))) {
@@ -1265,8 +1260,8 @@ void XCOFFObjectWriter::finalizeRelocationInfo(SectionEntry *Sec,
   }
 }
 
-void XCOFFObjectWriter::calcOffsetToRelocations(SectionEntry *Sec,
-                                                uint64_t &RawPointer) {
+void XCOFFWriter::calcOffsetToRelocations(SectionEntry *Sec,
+                                          uint64_t &RawPointer) {
   if (!Sec->RelocationCount)
     return;
 
@@ -1297,7 +1292,7 @@ void XCOFFObjectWriter::calcOffsetToRelocations(SectionEntry *Sec,
     report_fatal_error("Relocation data overflowed this object file.");
 }
 
-void XCOFFObjectWriter::finalizeSectionInfo() {
+void XCOFFWriter::finalizeSectionInfo() {
   for (auto *Section : Sections) {
     if (Section->Index == SectionEntry::UninitializedIndex)
       // Nothing to record for this Section.
@@ -1361,9 +1356,10 @@ void XCOFFObjectWriter::finalizeSectionInfo() {
     SymbolTableOffset = RawPointer;
 }
 
-void XCOFFObjectWriter::addExceptionEntry(
-    const MCSymbol *Symbol, const MCSymbol *Trap, unsigned LanguageCode,
-    unsigned ReasonCode, unsigned FunctionSize, bool hasDebug) {
+void XCOFFWriter::addExceptionEntry(const MCSymbol *Symbol,
+                                    const MCSymbol *Trap, unsigned LanguageCode,
+                                    unsigned ReasonCode, unsigned FunctionSize,
+                                    bool hasDebug) {
   // If a module had debug info, debugging is enabled and XCOFF emits the
   // exception auxilliary entry.
   if (hasDebug)
@@ -1383,7 +1379,7 @@ void XCOFFObjectWriter::addExceptionEntry(
       std::pair<const StringRef, ExceptionInfo>(Symbol->getName(), NewEntry));
 }
 
-unsigned XCOFFObjectWriter::getExceptionSectionSize() {
+unsigned XCOFFWriter::getExceptionSectionSize() {
   unsigned EntryNum = 0;
 
   for (const auto &TableEntry : ExceptionSection.ExceptionTable)
@@ -1395,7 +1391,7 @@ unsigned XCOFFObjectWriter::getExceptionSectionSize() {
                                : XCOFF::ExceptionSectionEntrySize32);
 }
 
-unsigned XCOFFObjectWriter::getExceptionOffset(const MCSymbol *Symbol) {
+unsigned XCOFFWriter::getExceptionOffset(const MCSymbol *Symbol) {
   unsigned EntryNum = 0;
   for (const auto &TableEntry : ExceptionSection.ExceptionTable) {
     if (Symbol == TableEntry.second.FunctionSymbol)
@@ -1406,13 +1402,13 @@ unsigned XCOFFObjectWriter::getExceptionOffset(const MCSymbol *Symbol) {
                                : XCOFF::ExceptionSectionEntrySize32);
 }
 
-void XCOFFObjectWriter::addCInfoSymEntry(StringRef Name, StringRef Metadata) {
+void XCOFFWriter::addCInfoSymEntry(StringRef Name, StringRef Metadata) {
   assert(!CInfoSymSection.Entry && "Multiple entries are not supported");
   CInfoSymSection.addEntry(
       std::make_unique<CInfoSymInfo>(Name.str(), Metadata.str()));
 }
 
-void XCOFFObjectWriter::assignAddressesAndIndices(MCAssembler &Asm) {
+void XCOFFWriter::assignAddressesAndIndices(MCAssembler &Asm) {
   // The symbol table starts with all the C_FILE symbols. Each C_FILE symbol
   // requires 1 or 2 auxiliary entries.
   uint32_t SymbolTableIndex =
@@ -1588,7 +1584,7 @@ void XCOFFObjectWriter::assignAddressesAndIndices(MCAssembler &Asm) {
   SymbolTableEntryCount = SymbolTableIndex;
 }
 
-void XCOFFObjectWriter::writeSectionForControlSectionEntry(
+void XCOFFWriter::writeSectionForControlSectionEntry(
     const MCAssembler &Asm, const CsectSectionEntry &CsectEntry,
     uint64_t &CurrentAddressLocation) {
   // Nothing to write for this Section.
@@ -1635,7 +1631,7 @@ void XCOFFObjectWriter::writeSectionForControlSectionEntry(
   }
 }
 
-void XCOFFObjectWriter::writeSectionForDwarfSectionEntry(
+void XCOFFWriter::writeSectionForDwarfSectionEntry(
     const MCAssembler &Asm, const DwarfSectionEntry &DwarfEntry,
     uint64_t &CurrentAddressLocation) {
   // There could be a gap (without corresponding zero padding) between
@@ -1663,7 +1659,7 @@ void XCOFFObjectWriter::writeSectionForDwarfSectionEntry(
   CurrentAddressLocation += TailPaddingSize;
 }
 
-void XCOFFObjectWriter::writeSectionForExceptionSectionEntry(
+void XCOFFWriter::writeSectionForExceptionSectionEntry(
     const MCAssembler &Asm, ExceptionSectionEntry &ExceptionEntry,
     uint64_t &CurrentAddressLocation) {
   for (const auto &TableEntry : ExceptionEntry.ExceptionTable) {
@@ -1685,7 +1681,7 @@ void XCOFFObjectWriter::writeSectionForExceptionSectionEntry(
   CurrentAddressLocation += getExceptionSectionSize();
 }
 
-void XCOFFObjectWriter::writeSectionForCInfoSymSectionEntry(
+void XCOFFWriter::writeSectionForCInfoSymSectionEntry(
     const MCAssembler &Asm, CInfoSymSectionEntry &CInfoSymEntry,
     uint64_t &CurrentAddressLocation) {
   if (!CInfoSymSection.Entry)
@@ -1737,20 +1733,5 @@ uint8_t getEncodedType(const MCSectionXCOFF *Sec) {
 std::unique_ptr<MCObjectWriter>
 llvm::createXCOFFObjectWriter(std::unique_ptr<MCXCOFFObjectTargetWriter> MOTW,
                               raw_pwrite_stream &OS) {
-  return std::make_unique<XCOFFObjectWriter>(std::move(MOTW), OS);
-}
-
-// TODO: Export XCOFFObjectWriter to llvm/MC/MCXCOFFObjectWriter.h and remove
-// the forwarders.
-void XCOFF::addExceptionEntry(MCObjectWriter &Writer, const MCSymbol *Symbol,
-                              const MCSymbol *Trap, unsigned LanguageCode,
-                              unsigned ReasonCode, unsigned FunctionSize,
-                              bool hasDebug) {
-  static_cast<XCOFFObjectWriter &>(Writer).addExceptionEntry(
-      Symbol, Trap, LanguageCode, ReasonCode, FunctionSize, hasDebug);
-}
-
-void XCOFF::addCInfoSymEntry(MCObjectWriter &Writer, StringRef Name,
-                             StringRef Metadata) {
-  static_cast<XCOFFObjectWriter &>(Writer).addCInfoSymEntry(Name, Metadata);
+  return std::make_unique<XCOFFWriter>(std::move(MOTW), OS);
 }
-- 
GitLab


From 64d7e45c40e75846d13f4f7a853f9a3ea3493faa Mon Sep 17 00:00:00 2001
From: jeanPerier <jperier@nvidia.com>
Date: Sat, 26 Oct 2024 21:38:54 +0200
Subject: [PATCH 247/693] Revert "[flang][debug] Support mlir::NoneType."
 (#113769)

Reverts llvm/llvm-project#113550

It turns out this causes compiler crashes with assumed-type arrays and -g.
See https://github.com/llvm/llvm-project/pull/113769 for a reproducer.
---
 .../Optimizer/Transforms/DebugTypeGenerator.cpp    |  2 --
 flang/test/Transforms/debug-none-type.fir          | 14 --------------
 2 files changed, 16 deletions(-)
 delete mode 100644 flang/test/Transforms/debug-none-type.fir

diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index e387e27533a0..1ab6c76dae8e 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -581,8 +581,6 @@ DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
                                   /*genAssociated=*/false);
   } else if (auto vecTy = mlir::dyn_cast_or_null<fir::VectorType>(Ty)) {
     return convertVectorType(vecTy, fileAttr, scope, declOp);
-  } else if (mlir::isa<mlir::NoneType>(Ty)) {
-    return mlir::LLVM::DINullTypeAttr::get(context);
   } else if (auto boxTy = mlir::dyn_cast_or_null<fir::BoxType>(Ty)) {
     auto elTy = boxTy.getElementType();
     if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(elTy))
diff --git a/flang/test/Transforms/debug-none-type.fir b/flang/test/Transforms/debug-none-type.fir
deleted file mode 100644
index 5eee6a633219..000000000000
--- a/flang/test/Transforms/debug-none-type.fir
+++ /dev/null
@@ -1,14 +0,0 @@
-// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
-
-module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
-  func.func private @_FortranAAssign(i8) -> none loc(#loc1)
-  func.func private @foo() -> !fir.ref<none>  loc(#loc2)
-}
-#loc1 = loc("test.f90":5:1)
-#loc2 = loc("test.f90":15:1)
-
-// CHECK: #[[INT8_TY:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 8, encoding = DW_ATE_signed>
-// CHECK: #[[NONE_TY:.*]] = #llvm.di_null_type
-// CHECK: #[[REFNONE_TY:.*]] = #llvm.di_derived_type<tag = DW_TAG_pointer_type, name = "", baseType = #di_null_type{{.*}}>
-// CHECK: #llvm.di_subroutine_type<{{.*}}types = #[[NONE_TY]], #[[INT8_TY]]>
-// CHECK: #llvm.di_subroutine_type<{{.*}}types = #[[REFNONE_TY]]>
-- 
GitLab


From d104b8e827ef5c3cb723aee92af4adfc8af18e9a Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyulee@meta.com>
Date: Sat, 26 Oct 2024 13:55:46 -0700
Subject: [PATCH 248/693] Revert "Reland [StructuralHash] Refactor (#112621)"

This reverts commit 98ca9a635bd2fb98cee473a9558687a5b522e219.
---
 llvm/include/llvm/IR/StructuralHash.h         |   7 +-
 llvm/lib/IR/StructuralHash.cpp                | 133 +++++++-----------
 llvm/lib/Transforms/IPO/MergeFunctions.cpp    |   6 +-
 .../call-and-invoke-with-ranges-attr.ll       |  12 +-
 .../MergeFunc/call-and-invoke-with-ranges.ll  |   8 +-
 llvm/test/Transforms/MergeFunc/inline-asm.ll  |   8 +-
 6 files changed, 68 insertions(+), 106 deletions(-)

diff --git a/llvm/include/llvm/IR/StructuralHash.h b/llvm/include/llvm/IR/StructuralHash.h
index e2e192cc9501..57fb45db8491 100644
--- a/llvm/include/llvm/IR/StructuralHash.h
+++ b/llvm/include/llvm/IR/StructuralHash.h
@@ -14,7 +14,6 @@
 #ifndef LLVM_IR_STRUCTURALHASH_H
 #define LLVM_IR_STRUCTURALHASH_H
 
-#include "llvm/ADT/StableHashing.h"
 #include <cstdint>
 
 namespace llvm {
@@ -22,18 +21,20 @@ namespace llvm {
 class Function;
 class Module;
 
+using IRHash = uint64_t;
+
 /// Returns a hash of the function \p F.
 /// \param F The function to hash.
 /// \param DetailedHash Whether or not to encode additional information in the
 /// hash. The additional information added into the hash when this flag is set
 /// to true includes instruction and operand type information.
-stable_hash StructuralHash(const Function &F, bool DetailedHash = false);
+IRHash StructuralHash(const Function &F, bool DetailedHash = false);
 
 /// Returns a hash of the module \p M by hashing all functions and global
 /// variables contained within. \param M The module to hash. \param DetailedHash
 /// Whether or not to encode additional information in the function hashes that
 /// composed the module hash.
-stable_hash StructuralHash(const Module &M, bool DetailedHash = false);
+IRHash StructuralHash(const Module &M, bool DetailedHash = false);
 
 } // end namespace llvm
 
diff --git a/llvm/lib/IR/StructuralHash.cpp b/llvm/lib/IR/StructuralHash.cpp
index 267a085c5af7..fb4f33a021a9 100644
--- a/llvm/lib/IR/StructuralHash.cpp
+++ b/llvm/lib/IR/StructuralHash.cpp
@@ -24,93 +24,61 @@ namespace {
 // by the MergeFunctions pass.
 
 class StructuralHashImpl {
-  stable_hash Hash = 4;
+  uint64_t Hash = 4;
 
-  bool DetailedHash;
-
-  // This random value acts as a block header, as otherwise the partition of
-  // opcodes into BBs wouldn't affect the hash, only the order of the opcodes.
-  static constexpr stable_hash BlockHeaderHash = 45798;
-  static constexpr stable_hash FunctionHeaderHash = 0x62642d6b6b2d6b72;
-  static constexpr stable_hash GlobalHeaderHash = 23456;
+  void hash(uint64_t V) { Hash = hashing::detail::hash_16_bytes(Hash, V); }
 
   // This will produce different values on 32-bit and 64-bit systens as
   // hash_combine returns a size_t. However, this is only used for
   // detailed hashing which, in-tree, only needs to distinguish between
   // differences in functions.
-  // TODO: This is not stable.
-  template <typename T> stable_hash hashArbitaryType(const T &V) {
-    return hash_combine(V);
+  template <typename T> void hashArbitaryType(const T &V) {
+    hash(hash_combine(V));
   }
 
-  stable_hash hashType(Type *ValueType) {
-    SmallVector<stable_hash> Hashes;
-    Hashes.emplace_back(ValueType->getTypeID());
+  void hashType(Type *ValueType) {
+    hash(ValueType->getTypeID());
     if (ValueType->isIntegerTy())
-      Hashes.emplace_back(ValueType->getIntegerBitWidth());
-    return stable_hash_combine(Hashes);
+      hash(ValueType->getIntegerBitWidth());
   }
 
 public:
-  StructuralHashImpl() = delete;
-  explicit StructuralHashImpl(bool DetailedHash) : DetailedHash(DetailedHash) {}
-
-  stable_hash hashConstant(Constant *C) {
-    SmallVector<stable_hash> Hashes;
-    // TODO: hashArbitaryType() is not stable.
-    if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(C)) {
-      Hashes.emplace_back(hashArbitaryType(ConstInt->getValue()));
-    } else if (ConstantFP *ConstFP = dyn_cast<ConstantFP>(C)) {
-      Hashes.emplace_back(hashArbitaryType(ConstFP->getValue()));
-    } else if (Function *Func = dyn_cast<Function>(C)) {
+  StructuralHashImpl() = default;
+
+  void updateOperand(Value *Operand) {
+    hashType(Operand->getType());
+
+    // The cases enumerated below are not exhaustive and are only aimed to
+    // get decent coverage over the function.
+    if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(Operand)) {
+      hashArbitaryType(ConstInt->getValue());
+    } else if (ConstantFP *ConstFP = dyn_cast<ConstantFP>(Operand)) {
+      hashArbitaryType(ConstFP->getValue());
+    } else if (Argument *Arg = dyn_cast<Argument>(Operand)) {
+      hash(Arg->getArgNo());
+    } else if (Function *Func = dyn_cast<Function>(Operand)) {
       // Hashing the name will be deterministic as LLVM's hashing infrastructure
       // has explicit support for hashing strings and will not simply hash
       // the pointer.
-      Hashes.emplace_back(hashArbitaryType(Func->getName()));
+      hashArbitaryType(Func->getName());
     }
-
-    return stable_hash_combine(Hashes);
-  }
-
-  stable_hash hashValue(Value *V) {
-    // Check constant and return its hash.
-    Constant *C = dyn_cast<Constant>(V);
-    if (C)
-      return hashConstant(C);
-
-    // Hash argument number.
-    SmallVector<stable_hash> Hashes;
-    if (Argument *Arg = dyn_cast<Argument>(V))
-      Hashes.emplace_back(Arg->getArgNo());
-
-    return stable_hash_combine(Hashes);
   }
 
-  stable_hash hashOperand(Value *Operand) {
-    SmallVector<stable_hash> Hashes;
-    Hashes.emplace_back(hashType(Operand->getType()));
-    Hashes.emplace_back(hashValue(Operand));
-    return stable_hash_combine(Hashes);
-  }
-
-  stable_hash hashInstruction(const Instruction &Inst) {
-    SmallVector<stable_hash> Hashes;
-    Hashes.emplace_back(Inst.getOpcode());
+  void updateInstruction(const Instruction &Inst, bool DetailedHash) {
+    hash(Inst.getOpcode());
 
     if (!DetailedHash)
-      return stable_hash_combine(Hashes);
+      return;
 
-    Hashes.emplace_back(hashType(Inst.getType()));
+    hashType(Inst.getType());
 
     // Handle additional properties of specific instructions that cause
     // semantic differences in the IR.
     if (const auto *ComparisonInstruction = dyn_cast<CmpInst>(&Inst))
-      Hashes.emplace_back(ComparisonInstruction->getPredicate());
+      hash(ComparisonInstruction->getPredicate());
 
     for (const auto &Op : Inst.operands())
-      Hashes.emplace_back(hashOperand(Op));
-
-    return stable_hash_combine(Hashes);
+      updateOperand(Op);
   }
 
   // A function hash is calculated by considering only the number of arguments
@@ -129,17 +97,15 @@ public:
   // expensive checks for pass modification status). When modifying this
   // function, most changes should be gated behind an option and enabled
   // selectively.
-  void update(const Function &F) {
+  void update(const Function &F, bool DetailedHash) {
     // Declarations don't affect analyses.
     if (F.isDeclaration())
       return;
 
-    SmallVector<stable_hash> Hashes;
-    Hashes.emplace_back(Hash);
-    Hashes.emplace_back(FunctionHeaderHash);
+    hash(0x62642d6b6b2d6b72); // Function header
 
-    Hashes.emplace_back(F.isVarArg());
-    Hashes.emplace_back(F.arg_size());
+    hash(F.isVarArg());
+    hash(F.arg_size());
 
     SmallVector<const BasicBlock *, 8> BBs;
     SmallPtrSet<const BasicBlock *, 16> VisitedBBs;
@@ -152,17 +118,17 @@ public:
     while (!BBs.empty()) {
       const BasicBlock *BB = BBs.pop_back_val();
 
-      Hashes.emplace_back(BlockHeaderHash);
+      // This random value acts as a block header, as otherwise the partition of
+      // opcodes into BBs wouldn't affect the hash, only the order of the
+      // opcodes
+      hash(45798);
       for (auto &Inst : *BB)
-        Hashes.emplace_back(hashInstruction(Inst));
+        updateInstruction(Inst, DetailedHash);
 
       for (const BasicBlock *Succ : successors(BB))
         if (VisitedBBs.insert(Succ).second)
           BBs.push_back(Succ);
     }
-
-    // Update the combined hash in place.
-    Hash = stable_hash_combine(Hashes);
   }
 
   void update(const GlobalVariable &GV) {
@@ -171,20 +137,15 @@ public:
     // we ignore anything with the `.llvm` prefix
     if (GV.isDeclaration() || GV.getName().starts_with("llvm."))
       return;
-    SmallVector<stable_hash> Hashes;
-    Hashes.emplace_back(Hash);
-    Hashes.emplace_back(GlobalHeaderHash);
-    Hashes.emplace_back(GV.getValueType()->getTypeID());
-
-    // Update the combined hash in place.
-    Hash = stable_hash_combine(Hashes);
+    hash(23456); // Global header
+    hash(GV.getValueType()->getTypeID());
   }
 
-  void update(const Module &M) {
+  void update(const Module &M, bool DetailedHash) {
     for (const GlobalVariable &GV : M.globals())
       update(GV);
     for (const Function &F : M)
-      update(F);
+      update(F, DetailedHash);
   }
 
   uint64_t getHash() const { return Hash; }
@@ -192,14 +153,14 @@ public:
 
 } // namespace
 
-stable_hash llvm::StructuralHash(const Function &F, bool DetailedHash) {
-  StructuralHashImpl H(DetailedHash);
-  H.update(F);
+IRHash llvm::StructuralHash(const Function &F, bool DetailedHash) {
+  StructuralHashImpl H;
+  H.update(F, DetailedHash);
   return H.getHash();
 }
 
-stable_hash llvm::StructuralHash(const Module &M, bool DetailedHash) {
-  StructuralHashImpl H(DetailedHash);
-  H.update(M);
+IRHash llvm::StructuralHash(const Module &M, bool DetailedHash) {
+  StructuralHashImpl H;
+  H.update(M, DetailedHash);
   return H.getHash();
 }
diff --git a/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index ad16b0b35014..b50a700e0903 100644
--- a/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -172,14 +172,14 @@ namespace {
 
 class FunctionNode {
   mutable AssertingVH<Function> F;
-  stable_hash Hash;
+  IRHash Hash;
 
 public:
   // Note the hash is recalculated potentially multiple times, but it is cheap.
   FunctionNode(Function *F) : F(F), Hash(StructuralHash(*F)) {}
 
   Function *getFunc() const { return F; }
-  stable_hash getHash() const { return Hash; }
+  IRHash getHash() const { return Hash; }
 
   /// Replace the reference to the function F by the function G, assuming their
   /// implementations are equal.
@@ -420,7 +420,7 @@ bool MergeFunctions::runOnModule(Module &M) {
 
   // All functions in the module, ordered by hash. Functions with a unique
   // hash value are easily eliminated.
-  std::vector<std::pair<stable_hash, Function *>> HashedFuncs;
+  std::vector<std::pair<IRHash, Function *>> HashedFuncs;
   for (Function &Func : M) {
     if (isEligibleForMerging(Func)) {
       HashedFuncs.push_back({StructuralHash(Func), &Func});
diff --git a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll
index cbf14165548e..e5d62319bf9d 100644
--- a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll
+++ b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll
@@ -80,8 +80,8 @@ lpad:
 }
 
 define i8 @invoke_with_same_range() personality ptr undef {
-; CHECK-DAG: @invoke_with_same_range()
-; CHECK-DAG: tail call i8 @invoke_with_range()
+; CHECK-LABEL: @invoke_with_same_range()
+; CHECK: tail call i8 @invoke_with_range()
   %out = invoke range(i8 0, 2) i8 @dummy() to label %next unwind label %lpad
 
 next:
@@ -93,15 +93,15 @@ lpad:
 }
 
 define i8 @call_with_same_range() {
-; CHECK-DAG: @call_with_same_range
-; CHECK-DAG: tail call i8 @call_with_range
+; CHECK-LABEL: @call_with_same_range
+; CHECK: tail call i8 @call_with_range
   %out = call range(i8 0, 2) i8 @dummy()
   ret i8 %out
 }
 
 define i8 @call_with_same_range_attr(i8 range(i8 0, 2) %v) {
-; CHECK-DAG: @call_with_same_range_attr
-; CHECK-DAG: tail call i8 @call_with_range_attr
+; CHECK-LABEL: @call_with_same_range_attr
+; CHECK: tail call i8 @call_with_range_attr
   %out = call i8 @dummy2(i8 %v)
   ret i8 %out
 }
diff --git a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
index 39e5a11181a4..e7718ca84d31 100644
--- a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
+++ b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
@@ -64,8 +64,8 @@ lpad:
 }
 
 define i8 @invoke_with_same_range() personality ptr undef {
-; CHECK-DAG: @invoke_with_same_range()
-; CHECK-DAG: tail call i8 @invoke_with_range()
+; CHECK-LABEL: @invoke_with_same_range()
+; CHECK: tail call i8 @invoke_with_range()
   %out = invoke i8 @dummy() to label %next unwind label %lpad, !range !0
 
 next:
@@ -77,8 +77,8 @@ lpad:
 }
 
 define i8 @call_with_same_range() {
-; CHECK-DAG: @call_with_same_range
-; CHECK-DAG: tail call i8 @call_with_range
+; CHECK-LABEL: @call_with_same_range
+; CHECK: tail call i8 @call_with_range
   bitcast i8 0 to i8
   %out = call i8 @dummy(), !range !0
   ret i8 %out
diff --git a/llvm/test/Transforms/MergeFunc/inline-asm.ll b/llvm/test/Transforms/MergeFunc/inline-asm.ll
index 970757e8d53a..7cc6afd2f8f7 100644
--- a/llvm/test/Transforms/MergeFunc/inline-asm.ll
+++ b/llvm/test/Transforms/MergeFunc/inline-asm.ll
@@ -3,11 +3,11 @@
 ; CHECK-LABEL: @int_ptr_arg_different
 ; CHECK-NEXT: call void asm
 
-; CHECK-DAG: @int_ptr_null
-; CHECK-DAG: tail call void @float_ptr_null()
+; CHECK-LABEL: @int_ptr_null
+; CHECK-NEXT: tail call void @float_ptr_null()
 
-; CHECK-DAG: @int_ptr_arg_same
-; CHECK-DAG: tail call void @float_ptr_arg_same(ptr %0)
+; CHECK-LABEL: @int_ptr_arg_same
+; CHECK-NEXT: tail call void @float_ptr_arg_same(ptr %0)
 
 ; Used to satisfy minimum size limit
 declare void @stuff()
-- 
GitLab


From d72e711e864dad7e3a434d66f3febad2b1596335 Mon Sep 17 00:00:00 2001
From: Jinsong Ji <jinsong.ji@intel.com>
Date: Sat, 26 Oct 2024 17:40:11 -0400
Subject: [PATCH 249/693] [NFC] [CMake] Add -Wno-dangling-else for GCC built
 unittests  (#112817)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is one of the many PRs to fix errors with LLVM_ENABLE_WERROR=on.
Built by GCC 11.

Fix warnings:

llvm/unittests/ProfileData/CoverageMappingTest.cpp: In member function
‘virtual void
{anonymous}::CoverageMappingTest_TVIdxBuilder_Test::TestBody()’:
llvm/unittests/ProfileData/CoverageMappingTest.cpp:1116:10: error:
suggest explicit braces to avoid ambiguous ‘else’
[-Werror=dangling-else]
 1116 |       if (Node.NextIDs[C] < 0)

The problem here is because these macros, eg: EXPECT_TRUE are expanded
to a single line multi-statement code with if/else, which is indeed
ambiguous after pre-processing. a simple example would be like:
https://godbolt.org/z/4zjn56qrP

    if(x)
     switch (0) case 0: default: if (...) ; else return;;

Given that omit braces in such cases is part of LLVM's style guide, and
it is hard to add braces in gtest just for GCC's warning, add
-Wno-dangling-else for GCC instead.
---
 llvm/unittests/CMakeLists.txt | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/llvm/unittests/CMakeLists.txt b/llvm/unittests/CMakeLists.txt
index 911ede701982..8892f3e75729 100644
--- a/llvm/unittests/CMakeLists.txt
+++ b/llvm/unittests/CMakeLists.txt
@@ -14,6 +14,20 @@ function(add_llvm_target_unittest test_dir_name)
   add_llvm_unittest(${test_dir_name} DISABLE_LLVM_LINK_LLVM_DYLIB ${ARGN})
 endfunction()
 
+# gtest macros like EXPECT_TRUE are expanded to a single line
+# multi-statement code with if/else. eg:
+#   if (...)
+#     EXPECT_TURE(...)
+# will be expanded into something like:
+#   if(...)
+#     switch (0) case 0: default: if (...) ; else return;;
+# GCC may emit false positive dangling-else warnings for such code.
+# However, such warnings are actually against LLVM's style guide.
+# disable the warning for GCC so that one can enbable Werror.
+if (CMAKE_COMPILER_IS_GNUCXX)
+  list(APPEND LLVM_COMPILE_FLAGS "-Wno-dangling-else")
+endif ()
+
 add_subdirectory(ADT)
 add_subdirectory(Analysis)
 add_subdirectory(AsmParser)
-- 
GitLab


From 1941c5180b91d792200d5e868d45c96e99bda35e Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyulee@meta.com>
Date: Sat, 26 Oct 2024 09:20:26 -0700
Subject: [PATCH 250/693] Reland (2nd attempt) [StructuralHash] Refactor
 (#112621)

This is largely NFC, and it prepares for #112638.
 - Use stable_hash instead of uint64_t
 - Rename update* to hash* functions. They compute stable_hash locally and return it.

This is a patch for
https://discourse.llvm.org/t/rfc-global-function-merging/82608.
---
 llvm/include/llvm/IR/StructuralHash.h         |   7 +-
 llvm/lib/IR/StructuralHash.cpp                | 133 +++++++++++-------
 llvm/lib/Transforms/IPO/MergeFunctions.cpp    |   6 +-
 .../call-and-invoke-with-ranges-attr.ll       |  12 +-
 .../MergeFunc/call-and-invoke-with-ranges.ll  |   8 +-
 llvm/test/Transforms/MergeFunc/inline-asm.ll  |   8 +-
 6 files changed, 106 insertions(+), 68 deletions(-)

diff --git a/llvm/include/llvm/IR/StructuralHash.h b/llvm/include/llvm/IR/StructuralHash.h
index 57fb45db8491..e2e192cc9501 100644
--- a/llvm/include/llvm/IR/StructuralHash.h
+++ b/llvm/include/llvm/IR/StructuralHash.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_IR_STRUCTURALHASH_H
 #define LLVM_IR_STRUCTURALHASH_H
 
+#include "llvm/ADT/StableHashing.h"
 #include <cstdint>
 
 namespace llvm {
@@ -21,20 +22,18 @@ namespace llvm {
 class Function;
 class Module;
 
-using IRHash = uint64_t;
-
 /// Returns a hash of the function \p F.
 /// \param F The function to hash.
 /// \param DetailedHash Whether or not to encode additional information in the
 /// hash. The additional information added into the hash when this flag is set
 /// to true includes instruction and operand type information.
-IRHash StructuralHash(const Function &F, bool DetailedHash = false);
+stable_hash StructuralHash(const Function &F, bool DetailedHash = false);
 
 /// Returns a hash of the module \p M by hashing all functions and global
 /// variables contained within. \param M The module to hash. \param DetailedHash
 /// Whether or not to encode additional information in the function hashes that
 /// composed the module hash.
-IRHash StructuralHash(const Module &M, bool DetailedHash = false);
+stable_hash StructuralHash(const Module &M, bool DetailedHash = false);
 
 } // end namespace llvm
 
diff --git a/llvm/lib/IR/StructuralHash.cpp b/llvm/lib/IR/StructuralHash.cpp
index fb4f33a021a9..267a085c5af7 100644
--- a/llvm/lib/IR/StructuralHash.cpp
+++ b/llvm/lib/IR/StructuralHash.cpp
@@ -24,61 +24,93 @@ namespace {
 // by the MergeFunctions pass.
 
 class StructuralHashImpl {
-  uint64_t Hash = 4;
+  stable_hash Hash = 4;
 
-  void hash(uint64_t V) { Hash = hashing::detail::hash_16_bytes(Hash, V); }
+  bool DetailedHash;
+
+  // This random value acts as a block header, as otherwise the partition of
+  // opcodes into BBs wouldn't affect the hash, only the order of the opcodes.
+  static constexpr stable_hash BlockHeaderHash = 45798;
+  static constexpr stable_hash FunctionHeaderHash = 0x62642d6b6b2d6b72;
+  static constexpr stable_hash GlobalHeaderHash = 23456;
 
   // This will produce different values on 32-bit and 64-bit systens as
   // hash_combine returns a size_t. However, this is only used for
   // detailed hashing which, in-tree, only needs to distinguish between
   // differences in functions.
-  template <typename T> void hashArbitaryType(const T &V) {
-    hash(hash_combine(V));
+  // TODO: This is not stable.
+  template <typename T> stable_hash hashArbitaryType(const T &V) {
+    return hash_combine(V);
   }
 
-  void hashType(Type *ValueType) {
-    hash(ValueType->getTypeID());
+  stable_hash hashType(Type *ValueType) {
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(ValueType->getTypeID());
     if (ValueType->isIntegerTy())
-      hash(ValueType->getIntegerBitWidth());
+      Hashes.emplace_back(ValueType->getIntegerBitWidth());
+    return stable_hash_combine(Hashes);
   }
 
 public:
-  StructuralHashImpl() = default;
-
-  void updateOperand(Value *Operand) {
-    hashType(Operand->getType());
-
-    // The cases enumerated below are not exhaustive and are only aimed to
-    // get decent coverage over the function.
-    if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(Operand)) {
-      hashArbitaryType(ConstInt->getValue());
-    } else if (ConstantFP *ConstFP = dyn_cast<ConstantFP>(Operand)) {
-      hashArbitaryType(ConstFP->getValue());
-    } else if (Argument *Arg = dyn_cast<Argument>(Operand)) {
-      hash(Arg->getArgNo());
-    } else if (Function *Func = dyn_cast<Function>(Operand)) {
+  StructuralHashImpl() = delete;
+  explicit StructuralHashImpl(bool DetailedHash) : DetailedHash(DetailedHash) {}
+
+  stable_hash hashConstant(Constant *C) {
+    SmallVector<stable_hash> Hashes;
+    // TODO: hashArbitaryType() is not stable.
+    if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(C)) {
+      Hashes.emplace_back(hashArbitaryType(ConstInt->getValue()));
+    } else if (ConstantFP *ConstFP = dyn_cast<ConstantFP>(C)) {
+      Hashes.emplace_back(hashArbitaryType(ConstFP->getValue()));
+    } else if (Function *Func = dyn_cast<Function>(C)) {
       // Hashing the name will be deterministic as LLVM's hashing infrastructure
       // has explicit support for hashing strings and will not simply hash
       // the pointer.
-      hashArbitaryType(Func->getName());
+      Hashes.emplace_back(hashArbitaryType(Func->getName()));
     }
+
+    return stable_hash_combine(Hashes);
+  }
+
+  stable_hash hashValue(Value *V) {
+    // Check constant and return its hash.
+    Constant *C = dyn_cast<Constant>(V);
+    if (C)
+      return hashConstant(C);
+
+    // Hash argument number.
+    SmallVector<stable_hash> Hashes;
+    if (Argument *Arg = dyn_cast<Argument>(V))
+      Hashes.emplace_back(Arg->getArgNo());
+
+    return stable_hash_combine(Hashes);
   }
 
-  void updateInstruction(const Instruction &Inst, bool DetailedHash) {
-    hash(Inst.getOpcode());
+  stable_hash hashOperand(Value *Operand) {
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(hashType(Operand->getType()));
+    Hashes.emplace_back(hashValue(Operand));
+    return stable_hash_combine(Hashes);
+  }
+
+  stable_hash hashInstruction(const Instruction &Inst) {
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(Inst.getOpcode());
 
     if (!DetailedHash)
-      return;
+      return stable_hash_combine(Hashes);
 
-    hashType(Inst.getType());
+    Hashes.emplace_back(hashType(Inst.getType()));
 
     // Handle additional properties of specific instructions that cause
     // semantic differences in the IR.
     if (const auto *ComparisonInstruction = dyn_cast<CmpInst>(&Inst))
-      hash(ComparisonInstruction->getPredicate());
+      Hashes.emplace_back(ComparisonInstruction->getPredicate());
 
     for (const auto &Op : Inst.operands())
-      updateOperand(Op);
+      Hashes.emplace_back(hashOperand(Op));
+
+    return stable_hash_combine(Hashes);
   }
 
   // A function hash is calculated by considering only the number of arguments
@@ -97,15 +129,17 @@ public:
   // expensive checks for pass modification status). When modifying this
   // function, most changes should be gated behind an option and enabled
   // selectively.
-  void update(const Function &F, bool DetailedHash) {
+  void update(const Function &F) {
     // Declarations don't affect analyses.
     if (F.isDeclaration())
       return;
 
-    hash(0x62642d6b6b2d6b72); // Function header
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(Hash);
+    Hashes.emplace_back(FunctionHeaderHash);
 
-    hash(F.isVarArg());
-    hash(F.arg_size());
+    Hashes.emplace_back(F.isVarArg());
+    Hashes.emplace_back(F.arg_size());
 
     SmallVector<const BasicBlock *, 8> BBs;
     SmallPtrSet<const BasicBlock *, 16> VisitedBBs;
@@ -118,17 +152,17 @@ public:
     while (!BBs.empty()) {
       const BasicBlock *BB = BBs.pop_back_val();
 
-      // This random value acts as a block header, as otherwise the partition of
-      // opcodes into BBs wouldn't affect the hash, only the order of the
-      // opcodes
-      hash(45798);
+      Hashes.emplace_back(BlockHeaderHash);
       for (auto &Inst : *BB)
-        updateInstruction(Inst, DetailedHash);
+        Hashes.emplace_back(hashInstruction(Inst));
 
       for (const BasicBlock *Succ : successors(BB))
         if (VisitedBBs.insert(Succ).second)
           BBs.push_back(Succ);
     }
+
+    // Update the combined hash in place.
+    Hash = stable_hash_combine(Hashes);
   }
 
   void update(const GlobalVariable &GV) {
@@ -137,15 +171,20 @@ public:
     // we ignore anything with the `.llvm` prefix
     if (GV.isDeclaration() || GV.getName().starts_with("llvm."))
       return;
-    hash(23456); // Global header
-    hash(GV.getValueType()->getTypeID());
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(Hash);
+    Hashes.emplace_back(GlobalHeaderHash);
+    Hashes.emplace_back(GV.getValueType()->getTypeID());
+
+    // Update the combined hash in place.
+    Hash = stable_hash_combine(Hashes);
   }
 
-  void update(const Module &M, bool DetailedHash) {
+  void update(const Module &M) {
     for (const GlobalVariable &GV : M.globals())
       update(GV);
     for (const Function &F : M)
-      update(F, DetailedHash);
+      update(F);
   }
 
   uint64_t getHash() const { return Hash; }
@@ -153,14 +192,14 @@ public:
 
 } // namespace
 
-IRHash llvm::StructuralHash(const Function &F, bool DetailedHash) {
-  StructuralHashImpl H;
-  H.update(F, DetailedHash);
+stable_hash llvm::StructuralHash(const Function &F, bool DetailedHash) {
+  StructuralHashImpl H(DetailedHash);
+  H.update(F);
   return H.getHash();
 }
 
-IRHash llvm::StructuralHash(const Module &M, bool DetailedHash) {
-  StructuralHashImpl H;
-  H.update(M, DetailedHash);
+stable_hash llvm::StructuralHash(const Module &M, bool DetailedHash) {
+  StructuralHashImpl H(DetailedHash);
+  H.update(M);
   return H.getHash();
 }
diff --git a/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index b50a700e0903..ad16b0b35014 100644
--- a/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -172,14 +172,14 @@ namespace {
 
 class FunctionNode {
   mutable AssertingVH<Function> F;
-  IRHash Hash;
+  stable_hash Hash;
 
 public:
   // Note the hash is recalculated potentially multiple times, but it is cheap.
   FunctionNode(Function *F) : F(F), Hash(StructuralHash(*F)) {}
 
   Function *getFunc() const { return F; }
-  IRHash getHash() const { return Hash; }
+  stable_hash getHash() const { return Hash; }
 
   /// Replace the reference to the function F by the function G, assuming their
   /// implementations are equal.
@@ -420,7 +420,7 @@ bool MergeFunctions::runOnModule(Module &M) {
 
   // All functions in the module, ordered by hash. Functions with a unique
   // hash value are easily eliminated.
-  std::vector<std::pair<IRHash, Function *>> HashedFuncs;
+  std::vector<std::pair<stable_hash, Function *>> HashedFuncs;
   for (Function &Func : M) {
     if (isEligibleForMerging(Func)) {
       HashedFuncs.push_back({StructuralHash(Func), &Func});
diff --git a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll
index e5d62319bf9d..b5f29906b051 100644
--- a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll
+++ b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll
@@ -80,8 +80,8 @@ lpad:
 }
 
 define i8 @invoke_with_same_range() personality ptr undef {
-; CHECK-LABEL: @invoke_with_same_range()
-; CHECK: tail call i8 @invoke_with_range()
+; CHECK-DAG: @invoke_with_same_range()
+; CHECK-DAG: tail call i8 @invoke_with_range()
   %out = invoke range(i8 0, 2) i8 @dummy() to label %next unwind label %lpad
 
 next:
@@ -93,15 +93,15 @@ lpad:
 }
 
 define i8 @call_with_same_range() {
-; CHECK-LABEL: @call_with_same_range
-; CHECK: tail call i8 @call_with_range
+; CHECK-DAG: @call_with_same_range()
+; CHECK-DAG: tail call i8 @call_with_range()
   %out = call range(i8 0, 2) i8 @dummy()
   ret i8 %out
 }
 
 define i8 @call_with_same_range_attr(i8 range(i8 0, 2) %v) {
-; CHECK-LABEL: @call_with_same_range_attr
-; CHECK: tail call i8 @call_with_range_attr
+; CHECK-DAG: @call_with_same_range_attr
+; CHECK-DAG: tail call i8 @call_with_range_attr
   %out = call i8 @dummy2(i8 %v)
   ret i8 %out
 }
diff --git a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
index e7718ca84d31..39e5a11181a4 100644
--- a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
+++ b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
@@ -64,8 +64,8 @@ lpad:
 }
 
 define i8 @invoke_with_same_range() personality ptr undef {
-; CHECK-LABEL: @invoke_with_same_range()
-; CHECK: tail call i8 @invoke_with_range()
+; CHECK-DAG: @invoke_with_same_range()
+; CHECK-DAG: tail call i8 @invoke_with_range()
   %out = invoke i8 @dummy() to label %next unwind label %lpad, !range !0
 
 next:
@@ -77,8 +77,8 @@ lpad:
 }
 
 define i8 @call_with_same_range() {
-; CHECK-LABEL: @call_with_same_range
-; CHECK: tail call i8 @call_with_range
+; CHECK-DAG: @call_with_same_range
+; CHECK-DAG: tail call i8 @call_with_range
   bitcast i8 0 to i8
   %out = call i8 @dummy(), !range !0
   ret i8 %out
diff --git a/llvm/test/Transforms/MergeFunc/inline-asm.ll b/llvm/test/Transforms/MergeFunc/inline-asm.ll
index 7cc6afd2f8f7..970757e8d53a 100644
--- a/llvm/test/Transforms/MergeFunc/inline-asm.ll
+++ b/llvm/test/Transforms/MergeFunc/inline-asm.ll
@@ -3,11 +3,11 @@
 ; CHECK-LABEL: @int_ptr_arg_different
 ; CHECK-NEXT: call void asm
 
-; CHECK-LABEL: @int_ptr_null
-; CHECK-NEXT: tail call void @float_ptr_null()
+; CHECK-DAG: @int_ptr_null
+; CHECK-DAG: tail call void @float_ptr_null()
 
-; CHECK-LABEL: @int_ptr_arg_same
-; CHECK-NEXT: tail call void @float_ptr_arg_same(ptr %0)
+; CHECK-DAG: @int_ptr_arg_same
+; CHECK-DAG: tail call void @float_ptr_arg_same(ptr %0)
 
 ; Used to satisfy minimum size limit
 declare void @stuff()
-- 
GitLab


From 597ccb800829af69ebc18cd7c75d878c8d21de6e Mon Sep 17 00:00:00 2001
From: Justin Fargnoli <justinfargnoli@gmail.com>
Date: Sat, 26 Oct 2024 17:15:19 -0700
Subject: [PATCH 251/693] [NFC] Fix typo in description of
 `AfterPassInvalidatedCallbacks` (#113779)

---
 llvm/include/llvm/IR/PassInstrumentation.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/IR/PassInstrumentation.h b/llvm/include/llvm/IR/PassInstrumentation.h
index 45ee372e7959..4e65804179ae 100644
--- a/llvm/include/llvm/IR/PassInstrumentation.h
+++ b/llvm/include/llvm/IR/PassInstrumentation.h
@@ -182,7 +182,7 @@ private:
       BeforeNonSkippedPassCallbacks;
   /// These are run on passes that have just run.
   SmallVector<llvm::unique_function<AfterPassFunc>, 4> AfterPassCallbacks;
-  /// These are run passes that have just run on invalidated IR.
+  /// These are run on passes that have just run on invalidated IR.
   SmallVector<llvm::unique_function<AfterPassInvalidatedFunc>, 4>
       AfterPassInvalidatedCallbacks;
   /// These are run on analyses that are about to be run.
-- 
GitLab


From 89b5d88fb81362b4fb2f833790aa40b7eaa186da Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 26 Oct 2024 19:13:56 -0700
Subject: [PATCH 252/693] [ADT] Use std::string_view inside StringRef (#113775)

This patch makes minimum changes to replace Data and Length with an
instance of std::string_view.

Previously, I opted for public inheritance (#113752), but I
encountered a lot of errors from gcc stemming from ambiguity between
std::string_view and StringRef.

The composition approach in this patch gives us greater control at the
expense of forwarder functions.
---
 llvm/include/llvm/ADT/StringRef.h | 26 ++++++++++----------------
 1 file changed, 10 insertions(+), 16 deletions(-)

diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h
index d5f30b88c4c6..f879bbf7164f 100644
--- a/llvm/include/llvm/ADT/StringRef.h
+++ b/llvm/include/llvm/ADT/StringRef.h
@@ -60,11 +60,7 @@ namespace llvm {
     using const_reverse_iterator = std::reverse_iterator<const_iterator>;
 
   private:
-    /// The start of the string, in an external buffer.
-    const char *Data = nullptr;
-
-    /// The length of the string.
-    size_t Length = 0;
+    std::string_view View;
 
     // Workaround memcmp issue with null pointers (undefined behavior)
     // by providing a specialized version
@@ -86,27 +82,25 @@ namespace llvm {
 
     /// Construct a string ref from a cstring.
     /*implicit*/ constexpr StringRef(const char *Str)
-        : Data(Str), Length(Str ?
+        : View(Str, Str ?
     // GCC 7 doesn't have constexpr char_traits. Fall back to __builtin_strlen.
 #if defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 8
-                                __builtin_strlen(Str)
+                        __builtin_strlen(Str)
 #else
-                                std::char_traits<char>::length(Str)
+                        std::char_traits<char>::length(Str)
 #endif
-                                : 0) {
+                        : 0) {
     }
 
     /// Construct a string ref from a pointer and length.
     /*implicit*/ constexpr StringRef(const char *data, size_t length)
-        : Data(data), Length(length) {}
+        : View(data, length) {}
 
     /// Construct a string ref from an std::string.
-    /*implicit*/ StringRef(const std::string &Str)
-        : Data(Str.data()), Length(Str.length()) {}
+    /*implicit*/ StringRef(const std::string &Str) : View(Str) {}
 
     /// Construct a string ref from an std::string_view.
-    /*implicit*/ constexpr StringRef(std::string_view Str)
-        : Data(Str.data()), Length(Str.size()) {}
+    /*implicit*/ constexpr StringRef(std::string_view Str) : View(Str) {}
 
     /// @}
     /// @name Iterators
@@ -140,13 +134,13 @@ namespace llvm {
 
     /// data - Get a pointer to the start of the string (which may not be null
     /// terminated).
-    [[nodiscard]] constexpr const char *data() const { return Data; }
+    [[nodiscard]] constexpr const char *data() const { return View.data(); }
 
     /// empty - Check if the string is empty.
     [[nodiscard]] constexpr bool empty() const { return size() == 0; }
 
     /// size - Get the string size.
-    [[nodiscard]] constexpr size_t size() const { return Length; }
+    [[nodiscard]] constexpr size_t size() const { return View.size(); }
 
     /// front - Get the first character in the string.
     [[nodiscard]] char front() const {
-- 
GitLab


From 242c77018f669c0b8f06b262050fcc4dde486738 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 26 Oct 2024 19:29:49 -0700
Subject: [PATCH 253/693] [ARM] clang-format (NFC)

I'm planning to post a patch in this area.
---
 .../lib/Target/ARM/AsmParser/ARMAsmParser.cpp | 34 +++++++++----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 906519fef45d..68f1199fd12e 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -5081,23 +5081,23 @@ ParseStatus ARMAsmParser::parseMemBarrierOptOperand(OperandVector &Operands) {
     StringRef OptStr = Tok.getString();
 
     Opt = StringSwitch<unsigned>(OptStr.slice(0, OptStr.size()).lower())
-      .Case("sy",    ARM_MB::SY)
-      .Case("st",    ARM_MB::ST)
-      .Case("ld",    ARM_MB::LD)
-      .Case("sh",    ARM_MB::ISH)
-      .Case("ish",   ARM_MB::ISH)
-      .Case("shst",  ARM_MB::ISHST)
-      .Case("ishst", ARM_MB::ISHST)
-      .Case("ishld", ARM_MB::ISHLD)
-      .Case("nsh",   ARM_MB::NSH)
-      .Case("un",    ARM_MB::NSH)
-      .Case("nshst", ARM_MB::NSHST)
-      .Case("nshld", ARM_MB::NSHLD)
-      .Case("unst",  ARM_MB::NSHST)
-      .Case("osh",   ARM_MB::OSH)
-      .Case("oshst", ARM_MB::OSHST)
-      .Case("oshld", ARM_MB::OSHLD)
-      .Default(~0U);
+              .Case("sy", ARM_MB::SY)
+              .Case("st", ARM_MB::ST)
+              .Case("ld", ARM_MB::LD)
+              .Case("sh", ARM_MB::ISH)
+              .Case("ish", ARM_MB::ISH)
+              .Case("shst", ARM_MB::ISHST)
+              .Case("ishst", ARM_MB::ISHST)
+              .Case("ishld", ARM_MB::ISHLD)
+              .Case("nsh", ARM_MB::NSH)
+              .Case("un", ARM_MB::NSH)
+              .Case("nshst", ARM_MB::NSHST)
+              .Case("nshld", ARM_MB::NSHLD)
+              .Case("unst", ARM_MB::NSHST)
+              .Case("osh", ARM_MB::OSH)
+              .Case("oshst", ARM_MB::OSHST)
+              .Case("oshld", ARM_MB::OSHLD)
+              .Default(~0U);
 
     // ishld, oshld, nshld and ld are only available from ARMv8.
     if (!hasV8Ops() && (Opt == ARM_MB::ISHLD || Opt == ARM_MB::OSHLD ||
-- 
GitLab


From 0dd9fdcf83cd00f51669b32c96937a97ef4b339e Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyulee@meta.com>
Date: Sat, 26 Oct 2024 20:02:05 -0700
Subject: [PATCH 254/693] [StructuralHash] Support Differences (#112638)

This computes a structural hash while allowing for selective ignoring of
certain operands based on a custom function that is provided. Instead of
a single hash value, it now returns FunctionHashInfo which includes a
hash value, an instruction mapping, and a map to track the operand
location and its corresponding hash value that is ignored.

Depends on https://github.com/llvm/llvm-project/pull/112621.
This is a patch for
https://discourse.llvm.org/t/rfc-global-function-merging/82608.
---
 llvm/include/llvm/Analysis/StructuralHash.h   |  13 +-
 llvm/include/llvm/IR/StructuralHash.h         |  45 ++++++
 llvm/lib/Analysis/StructuralHash.cpp          |  27 +++-
 llvm/lib/IR/StructuralHash.cpp                | 153 +++++++++++++++---
 llvm/lib/Passes/PassBuilder.cpp               |  14 +-
 llvm/lib/Passes/PassRegistry.def              |   7 +-
 .../StructuralHash/structural-hash-printer.ll |  24 ++-
 llvm/unittests/IR/StructuralHashTest.cpp      |  61 +++++++
 8 files changed, 304 insertions(+), 40 deletions(-)

diff --git a/llvm/include/llvm/Analysis/StructuralHash.h b/llvm/include/llvm/Analysis/StructuralHash.h
index 9f33c69aed34..4c9f063bc7d2 100644
--- a/llvm/include/llvm/Analysis/StructuralHash.h
+++ b/llvm/include/llvm/Analysis/StructuralHash.h
@@ -13,15 +13,22 @@
 
 namespace llvm {
 
+enum class StructuralHashOptions {
+  None,              /// Hash with opcode only.
+  Detailed,          /// Hash with opcode and operands.
+  CallTargetIgnored, /// Ignore call target operand when computing hash.
+};
+
 /// Printer pass for  StructuralHashes
 class StructuralHashPrinterPass
     : public PassInfoMixin<StructuralHashPrinterPass> {
   raw_ostream &OS;
-  bool EnableDetailedStructuralHash;
+  const StructuralHashOptions Options;
 
 public:
-  explicit StructuralHashPrinterPass(raw_ostream &OS, bool Detailed)
-      : OS(OS), EnableDetailedStructuralHash(Detailed) {}
+  explicit StructuralHashPrinterPass(raw_ostream &OS,
+                                     StructuralHashOptions Options)
+      : OS(OS), Options(Options) {}
 
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
 
diff --git a/llvm/include/llvm/IR/StructuralHash.h b/llvm/include/llvm/IR/StructuralHash.h
index e2e192cc9501..071575137ff5 100644
--- a/llvm/include/llvm/IR/StructuralHash.h
+++ b/llvm/include/llvm/IR/StructuralHash.h
@@ -14,7 +14,9 @@
 #ifndef LLVM_IR_STRUCTURALHASH_H
 #define LLVM_IR_STRUCTURALHASH_H
 
+#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/StableHashing.h"
+#include "llvm/IR/Instruction.h"
 #include <cstdint>
 
 namespace llvm {
@@ -35,6 +37,49 @@ stable_hash StructuralHash(const Function &F, bool DetailedHash = false);
 /// composed the module hash.
 stable_hash StructuralHash(const Module &M, bool DetailedHash = false);
 
+/// The pair of an instruction index and a operand index.
+using IndexPair = std::pair<unsigned, unsigned>;
+
+/// A map from an instruction index to an instruction pointer.
+using IndexInstrMap = MapVector<unsigned, Instruction *>;
+
+/// A map from an IndexPair to a stable hash.
+using IndexOperandHashMapType = DenseMap<IndexPair, stable_hash>;
+
+/// A function that takes an instruction and an operand index and returns true
+/// if the operand should be ignored in the function hash computation.
+using IgnoreOperandFunc = std::function<bool(const Instruction *, unsigned)>;
+
+struct FunctionHashInfo {
+  /// A hash value representing the structural content of the function
+  stable_hash FunctionHash;
+  /// A mapping from instruction indices to instruction pointers
+  std::unique_ptr<IndexInstrMap> IndexInstruction;
+  /// A mapping from pairs of instruction indices and operand indices
+  /// to the hashes of the operands. This can be used to analyze or
+  /// reconstruct the differences in ignored operands
+  std::unique_ptr<IndexOperandHashMapType> IndexOperandHashMap;
+
+  FunctionHashInfo(stable_hash FuntionHash,
+                   std::unique_ptr<IndexInstrMap> IndexInstruction,
+                   std::unique_ptr<IndexOperandHashMapType> IndexOperandHashMap)
+      : FunctionHash(FuntionHash),
+        IndexInstruction(std::move(IndexInstruction)),
+        IndexOperandHashMap(std::move(IndexOperandHashMap)) {}
+};
+
+/// Computes a structural hash of a given function, considering the structure
+/// and content of the function's instructions while allowing for selective
+/// ignoring of certain operands based on custom criteria. This hash can be used
+/// to identify functions that are structurally similar or identical, which is
+/// useful in optimizations, deduplication, or analysis tasks.
+/// \param F The function to hash.
+/// \param IgnoreOp A callable that takes an instruction and an operand index,
+/// and returns true if the operand should be ignored in the hash computation.
+/// \return A FunctionHashInfo structure
+FunctionHashInfo StructuralHashWithDifferences(const Function &F,
+                                               IgnoreOperandFunc IgnoreOp);
+
 } // end namespace llvm
 
 #endif
diff --git a/llvm/lib/Analysis/StructuralHash.cpp b/llvm/lib/Analysis/StructuralHash.cpp
index 3a2341fe59ad..4f2e003148b6 100644
--- a/llvm/lib/Analysis/StructuralHash.cpp
+++ b/llvm/lib/Analysis/StructuralHash.cpp
@@ -21,14 +21,33 @@ using namespace llvm;
 PreservedAnalyses StructuralHashPrinterPass::run(Module &M,
                                                  ModuleAnalysisManager &MAM) {
   OS << "Module Hash: "
-     << format("%016" PRIx64, StructuralHash(M, EnableDetailedStructuralHash))
+     << format("%016" PRIx64,
+               StructuralHash(M, Options != StructuralHashOptions::None))
      << "\n";
   for (Function &F : M) {
     if (F.isDeclaration())
       continue;
-    OS << "Function " << F.getName() << " Hash: "
-       << format("%016" PRIx64, StructuralHash(F, EnableDetailedStructuralHash))
-       << "\n";
+    if (Options == StructuralHashOptions::CallTargetIgnored) {
+      auto IgnoreOp = [&](const Instruction *I, unsigned OpndIdx) {
+        return I->getOpcode() == Instruction::Call &&
+               isa<Constant>(I->getOperand(OpndIdx));
+      };
+      auto FuncHashInfo = StructuralHashWithDifferences(F, IgnoreOp);
+      OS << "Function " << F.getName()
+         << " Hash: " << format("%016" PRIx64, FuncHashInfo.FunctionHash)
+         << "\n";
+      for (auto &[IndexPair, OpndHash] : *FuncHashInfo.IndexOperandHashMap) {
+        auto [InstIndex, OpndIndex] = IndexPair;
+        OS << "\tIgnored Operand Hash: " << format("%016" PRIx64, OpndHash)
+           << " at (" << InstIndex << "," << OpndIndex << ")\n";
+      }
+    } else {
+      OS << "Function " << F.getName() << " Hash: "
+         << format(
+                "%016" PRIx64,
+                StructuralHash(F, Options == StructuralHashOptions::Detailed))
+         << "\n";
+    }
   }
   return PreservedAnalyses::all();
 }
diff --git a/llvm/lib/IR/StructuralHash.cpp b/llvm/lib/IR/StructuralHash.cpp
index 267a085c5af7..a51f9124af04 100644
--- a/llvm/lib/IR/StructuralHash.cpp
+++ b/llvm/lib/IR/StructuralHash.cpp
@@ -34,14 +34,18 @@ class StructuralHashImpl {
   static constexpr stable_hash FunctionHeaderHash = 0x62642d6b6b2d6b72;
   static constexpr stable_hash GlobalHeaderHash = 23456;
 
-  // This will produce different values on 32-bit and 64-bit systens as
-  // hash_combine returns a size_t. However, this is only used for
-  // detailed hashing which, in-tree, only needs to distinguish between
-  // differences in functions.
-  // TODO: This is not stable.
-  template <typename T> stable_hash hashArbitaryType(const T &V) {
-    return hash_combine(V);
-  }
+  /// IgnoreOp is a function that returns true if the operand should be ignored.
+  IgnoreOperandFunc IgnoreOp = nullptr;
+  /// A mapping from instruction indices to instruction pointers.
+  /// The index represents the position of an instruction based on the order in
+  /// which it is first encountered.
+  std::unique_ptr<IndexInstrMap> IndexInstruction = nullptr;
+  /// A mapping from pairs of instruction indices and operand indices
+  /// to the hashes of the operands.
+  std::unique_ptr<IndexOperandHashMapType> IndexOperandHashMap = nullptr;
+
+  /// Assign a unique ID to each Value in the order they are first seen.
+  DenseMap<const Value *, int> ValueToId;
 
   stable_hash hashType(Type *ValueType) {
     SmallVector<stable_hash> Hashes;
@@ -53,23 +57,95 @@ class StructuralHashImpl {
 
 public:
   StructuralHashImpl() = delete;
-  explicit StructuralHashImpl(bool DetailedHash) : DetailedHash(DetailedHash) {}
+  explicit StructuralHashImpl(bool DetailedHash,
+                              IgnoreOperandFunc IgnoreOp = nullptr)
+      : DetailedHash(DetailedHash), IgnoreOp(IgnoreOp) {
+    if (IgnoreOp) {
+      IndexInstruction = std::make_unique<IndexInstrMap>();
+      IndexOperandHashMap = std::make_unique<IndexOperandHashMapType>();
+    }
+  }
+
+  stable_hash hashAPInt(const APInt &I) {
+    SmallVector<stable_hash> Hashes;
+    Hashes.emplace_back(I.getBitWidth());
+    auto RawVals = ArrayRef<uint64_t>(I.getRawData(), I.getNumWords());
+    Hashes.append(RawVals.begin(), RawVals.end());
+    return stable_hash_combine(Hashes);
+  }
+
+  stable_hash hashAPFloat(const APFloat &F) {
+    return hashAPInt(F.bitcastToAPInt());
+  }
+
+  stable_hash hashGlobalValue(const GlobalValue *GV) {
+    if (!GV->hasName())
+      return 0;
+    return stable_hash_name(GV->getName());
+  }
 
+  // Compute a hash for a Constant. This function is logically similar to
+  // FunctionComparator::cmpConstants() in FunctionComparator.cpp, but here
+  // we're interested in computing a hash rather than comparing two Constants.
+  // Some of the logic is simplified, e.g, we don't expand GEPOperator.
   stable_hash hashConstant(Constant *C) {
     SmallVector<stable_hash> Hashes;
-    // TODO: hashArbitaryType() is not stable.
-    if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(C)) {
-      Hashes.emplace_back(hashArbitaryType(ConstInt->getValue()));
-    } else if (ConstantFP *ConstFP = dyn_cast<ConstantFP>(C)) {
-      Hashes.emplace_back(hashArbitaryType(ConstFP->getValue()));
-    } else if (Function *Func = dyn_cast<Function>(C)) {
-      // Hashing the name will be deterministic as LLVM's hashing infrastructure
-      // has explicit support for hashing strings and will not simply hash
-      // the pointer.
-      Hashes.emplace_back(hashArbitaryType(Func->getName()));
+
+    Type *Ty = C->getType();
+    Hashes.emplace_back(hashType(Ty));
+
+    if (C->isNullValue()) {
+      Hashes.emplace_back(static_cast<stable_hash>('N'));
+      return stable_hash_combine(Hashes);
     }
 
-    return stable_hash_combine(Hashes);
+    if (auto *G = dyn_cast<GlobalValue>(C)) {
+      Hashes.emplace_back(hashGlobalValue(G));
+      return stable_hash_combine(Hashes);
+    }
+
+    if (const auto *Seq = dyn_cast<ConstantDataSequential>(C)) {
+      Hashes.emplace_back(xxh3_64bits(Seq->getRawDataValues()));
+      return stable_hash_combine(Hashes);
+    }
+
+    switch (C->getValueID()) {
+    case Value::ConstantIntVal: {
+      const APInt &Int = cast<ConstantInt>(C)->getValue();
+      Hashes.emplace_back(hashAPInt(Int));
+      return stable_hash_combine(Hashes);
+    }
+    case Value::ConstantFPVal: {
+      const APFloat &APF = cast<ConstantFP>(C)->getValueAPF();
+      Hashes.emplace_back(hashAPFloat(APF));
+      return stable_hash_combine(Hashes);
+    }
+    case Value::ConstantArrayVal:
+    case Value::ConstantStructVal:
+    case Value::ConstantVectorVal:
+    case Value::ConstantExprVal: {
+      for (const auto &Op : C->operands()) {
+        auto H = hashConstant(cast<Constant>(Op));
+        Hashes.emplace_back(H);
+      }
+      return stable_hash_combine(Hashes);
+    }
+    case Value::BlockAddressVal: {
+      const BlockAddress *BA = cast<BlockAddress>(C);
+      auto H = hashGlobalValue(BA->getFunction());
+      Hashes.emplace_back(H);
+      return stable_hash_combine(Hashes);
+    }
+    case Value::DSOLocalEquivalentVal: {
+      const auto *Equiv = cast<DSOLocalEquivalent>(C);
+      auto H = hashGlobalValue(Equiv->getGlobalValue());
+      Hashes.emplace_back(H);
+      return stable_hash_combine(Hashes);
+    }
+    default:
+      // Skip other types of constants for simplicity.
+      return stable_hash_combine(Hashes);
+    }
   }
 
   stable_hash hashValue(Value *V) {
@@ -83,6 +159,10 @@ public:
     if (Argument *Arg = dyn_cast<Argument>(V))
       Hashes.emplace_back(Arg->getArgNo());
 
+    // Get an index (an insertion order) for the non-constant value.
+    auto [It, WasInserted] = ValueToId.try_emplace(V, ValueToId.size());
+    Hashes.emplace_back(It->second);
+
     return stable_hash_combine(Hashes);
   }
 
@@ -107,8 +187,20 @@ public:
     if (const auto *ComparisonInstruction = dyn_cast<CmpInst>(&Inst))
       Hashes.emplace_back(ComparisonInstruction->getPredicate());
 
-    for (const auto &Op : Inst.operands())
-      Hashes.emplace_back(hashOperand(Op));
+    unsigned InstIdx = 0;
+    if (IndexInstruction) {
+      InstIdx = IndexInstruction->size();
+      IndexInstruction->try_emplace(InstIdx, const_cast<Instruction *>(&Inst));
+    }
+
+    for (const auto [OpndIdx, Op] : enumerate(Inst.operands())) {
+      auto OpndHash = hashOperand(Op);
+      if (IgnoreOp && IgnoreOp(&Inst, OpndIdx)) {
+        assert(IndexOperandHashMap);
+        IndexOperandHashMap->try_emplace({InstIdx, OpndIdx}, OpndHash);
+      } else
+        Hashes.emplace_back(OpndHash);
+    }
 
     return stable_hash_combine(Hashes);
   }
@@ -188,6 +280,14 @@ public:
   }
 
   uint64_t getHash() const { return Hash; }
+
+  std::unique_ptr<IndexInstrMap> getIndexInstrMap() {
+    return std::move(IndexInstruction);
+  }
+
+  std::unique_ptr<IndexOperandHashMapType> getIndexPairOpndHashMap() {
+    return std::move(IndexOperandHashMap);
+  }
 };
 
 } // namespace
@@ -203,3 +303,12 @@ stable_hash llvm::StructuralHash(const Module &M, bool DetailedHash) {
   H.update(M);
   return H.getHash();
 }
+
+FunctionHashInfo
+llvm::StructuralHashWithDifferences(const Function &F,
+                                    IgnoreOperandFunc IgnoreOp) {
+  StructuralHashImpl H(/*DetailedHash=*/true, IgnoreOp);
+  H.update(F);
+  return FunctionHashInfo(H.getHash(), H.getIndexInstrMap(),
+                          H.getIndexPairOpndHashMap());
+}
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index f5ce405ab8d9..d1f75dfb5350 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -1175,9 +1175,17 @@ Expected<std::string> parseMemProfUsePassOptions(StringRef Params) {
   return Result;
 }
 
-Expected<bool> parseStructuralHashPrinterPassOptions(StringRef Params) {
-  return PassBuilder::parseSinglePassOption(Params, "detailed",
-                                            "StructuralHashPrinterPass");
+Expected<StructuralHashOptions>
+parseStructuralHashPrinterPassOptions(StringRef Params) {
+  if (Params.empty())
+    return StructuralHashOptions::None;
+  if (Params == "detailed")
+    return StructuralHashOptions::Detailed;
+  if (Params == "call-target-ignored")
+    return StructuralHashOptions::CallTargetIgnored;
+  return make_error<StringError>(
+      formatv("invalid structural hash printer parameter '{0}' ", Params).str(),
+      inconvertibleErrorCode());
 }
 
 Expected<bool> parseWinEHPrepareOptions(StringRef Params) {
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 549c1359b585..017ae311c55e 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -220,10 +220,11 @@ MODULE_PASS_WITH_PARAMS(
     parseMSanPassOptions, "recover;kernel;eager-checks;track-origins=N")
 MODULE_PASS_WITH_PARAMS(
     "print<structural-hash>", "StructuralHashPrinterPass",
-    [](bool EnableDetailedStructuralHash) {
-      return StructuralHashPrinterPass(dbgs(), EnableDetailedStructuralHash);
+    [](StructuralHashOptions Options) {
+      return StructuralHashPrinterPass(dbgs(), Options);
     },
-    parseStructuralHashPrinterPassOptions, "detailed")
+    parseStructuralHashPrinterPassOptions, "detailed;call-target-ignored")
+
 #undef MODULE_PASS_WITH_PARAMS
 
 #ifndef CGSCC_ANALYSIS
diff --git a/llvm/test/Analysis/StructuralHash/structural-hash-printer.ll b/llvm/test/Analysis/StructuralHash/structural-hash-printer.ll
index 5936199bf32f..3c23b54d2973 100644
--- a/llvm/test/Analysis/StructuralHash/structural-hash-printer.ll
+++ b/llvm/test/Analysis/StructuralHash/structural-hash-printer.ll
@@ -1,17 +1,21 @@
 ; RUN: opt -passes='print<structural-hash>' -disable-output %s 2>&1 | FileCheck %s
 ; RUN: opt -passes='print<structural-hash><detailed>' -disable-output %s 2>&1 | FileCheck %s -check-prefix=DETAILED-HASH
+; RUN: opt -passes='print<structural-hash><call-target-ignored>' -disable-output %s 2>&1 | FileCheck %s -check-prefix=CALLTARGETIGNORED-HASH
 
 ; Add a declaration so that we can test we skip it.
-declare i64 @d1()
+declare i64 @d1(i64)
+declare i64 @e1(i64)
 
 define i64 @f1(i64 %a) {
 	%b = add i64 %a, 1
-	ret i64 %b
+	%c = call i64 @d1(i64 %b)
+	ret i64 %c
 }
 
-define i32 @f2(i32 %a) {
-	%b = add i32 %a, 2
-	ret i32 %b
+define i64 @f2(i64 %a) {
+	%b = add i64 %a, 1
+	%c = call i64 @e1(i64 %b)
+	ret i64 %c
 }
 
 ; CHECK: Module Hash: {{([a-f0-9]{16,})}}
@@ -22,3 +26,13 @@ define i32 @f2(i32 %a) {
 ; DETAILED-HASH-NEXT: Function f1 Hash: [[DF1H:([a-f0-9]{16,})]]
 ; DETAILED-HASH-NOT: [[DF1H]]
 ; DETAILED-HASH-NEXT: Function f2 Hash: {{([a-f0-9]{16,})}}
+
+; When ignoring the call target, check if `f1` and `f2` produce the same function hash.
+; The index for the call instruction is 1, and the index of the call target operand is 1.
+; The ignored operand hashes for different call targets should be different.
+; CALLTARGETIGNORED-HASH: Module Hash: {{([a-f0-9]{16,})}}
+; CALLTARGETIGNORED-HASH-NEXT: Function f1 Hash: [[IF1H:([a-f0-9]{16,})]]
+; CALLTARGETIGNORED-HASH-NEXT:   Ignored Operand Hash: [[IO1H:([a-f0-9]{16,})]] at (1,1)
+; CALLTARGETIGNORED-HASH-NEXT: Function f2 Hash: [[IF1H]]
+; CALLTARGETIGNORED-HASH-NOT: [[IO1H]]
+; CALLTARGETIGNORED-HASH-NEXT:   Ignored Operand Hash: {{([a-f0-9]{16,})}} at (1,1)
diff --git a/llvm/unittests/IR/StructuralHashTest.cpp b/llvm/unittests/IR/StructuralHashTest.cpp
index 64e66aa5f97a..81c17120a1f6 100644
--- a/llvm/unittests/IR/StructuralHashTest.cpp
+++ b/llvm/unittests/IR/StructuralHashTest.cpp
@@ -10,6 +10,7 @@
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/SourceMgr.h"
+#include "gmock/gmock-matchers.h"
 #include "gtest/gtest.h"
 
 #include <memory>
@@ -18,6 +19,11 @@ using namespace llvm;
 
 namespace {
 
+using testing::Contains;
+using testing::Key;
+using testing::Pair;
+using testing::SizeIs;
+
 std::unique_ptr<Module> parseIR(LLVMContext &Context, const char *IR) {
   SMDiagnostic Err;
   std::unique_ptr<Module> M = parseAssemblyString(IR, Err, Context);
@@ -239,4 +245,59 @@ TEST(StructuralHashTest, ArgumentNumber) {
   EXPECT_EQ(StructuralHash(*M1), StructuralHash(*M2));
   EXPECT_NE(StructuralHash(*M1, true), StructuralHash(*M2, true));
 }
+
+TEST(StructuralHashTest, Differences) {
+  LLVMContext Ctx;
+  std::unique_ptr<Module> M1 = parseIR(Ctx, "define i64 @f(i64 %a) {\n"
+                                            "  %c = add i64 %a, 1\n"
+                                            "  %b = call i64 @f1(i64 %c)\n"
+                                            "  ret i64 %b\n"
+                                            "}\n"
+                                            "declare i64 @f1(i64)");
+  auto *F1 = M1->getFunction("f");
+  std::unique_ptr<Module> M2 = parseIR(Ctx, "define i64 @g(i64 %a) {\n"
+                                            "  %c = add i64 %a, 1\n"
+                                            "  %b = call i64 @f2(i64 %c)\n"
+                                            "  ret i64 %b\n"
+                                            "}\n"
+                                            "declare i64 @f2(i64)");
+  auto *F2 = M2->getFunction("g");
+
+  // They are originally different when not ignoring any operand.
+  EXPECT_NE(StructuralHash(*F1, true), StructuralHash(*F2, true));
+  EXPECT_NE(StructuralHashWithDifferences(*F1, nullptr).FunctionHash,
+            StructuralHashWithDifferences(*F2, nullptr).FunctionHash);
+
+  // When we ignore the call target f1 vs f2, they have the same hash.
+  auto IgnoreOp = [&](const Instruction *I, unsigned OpndIdx) {
+    return I->getOpcode() == Instruction::Call && OpndIdx == 1;
+  };
+  auto FuncHashInfo1 = StructuralHashWithDifferences(*F1, IgnoreOp);
+  auto FuncHashInfo2 = StructuralHashWithDifferences(*F2, IgnoreOp);
+  EXPECT_EQ(FuncHashInfo1.FunctionHash, FuncHashInfo2.FunctionHash);
+
+  // There are total 3 instructions.
+  EXPECT_THAT(*FuncHashInfo1.IndexInstruction, SizeIs(3));
+  EXPECT_THAT(*FuncHashInfo2.IndexInstruction, SizeIs(3));
+
+  // The only 1 operand (the call target) has been ignored.
+  EXPECT_THAT(*FuncHashInfo1.IndexOperandHashMap, SizeIs(1u));
+  EXPECT_THAT(*FuncHashInfo2.IndexOperandHashMap, SizeIs(1u));
+
+  // The index pair of instruction and operand (1, 1) is a key in the map.
+  ASSERT_THAT(*FuncHashInfo1.IndexOperandHashMap, Contains(Key(Pair(1, 1))));
+  ASSERT_THAT(*FuncHashInfo2.IndexOperandHashMap, Contains(Key(Pair(1, 1))));
+
+  // The indexed instruciton must be the call instruction as shown in the
+  // IgnoreOp above.
+  EXPECT_EQ(FuncHashInfo1.IndexInstruction->lookup(1)->getOpcode(),
+            Instruction::Call);
+  EXPECT_EQ(FuncHashInfo2.IndexInstruction->lookup(1)->getOpcode(),
+            Instruction::Call);
+
+  // The ignored operand hashes (for f1 vs. f2) are different.
+  EXPECT_NE(FuncHashInfo1.IndexOperandHashMap->lookup({1, 1}),
+            FuncHashInfo2.IndexOperandHashMap->lookup({1, 1}));
+}
+
 } // end anonymous namespace
-- 
GitLab


From 93da6423af5f00a3bbee4d2ee571ccc7887f444d Mon Sep 17 00:00:00 2001
From: Sirui Mu <msrlancern@gmail.com>
Date: Sun, 27 Oct 2024 11:52:00 +0800
Subject: [PATCH 255/693] [mlir][LLVM] Add builders for llvm.intr.assume
 (#113317)

This patch adds several new builders for llvm.intr.assume that build the
operation with additional operand bundles.
---
 .../include/mlir/Dialect/LLVMIR/LLVMDialect.h |  8 ++++
 .../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td   |  9 ++++-
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp    | 39 ++++++++++++++++++-
 3 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
index d236cae0d808..63e007cdc335 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
@@ -33,6 +33,7 @@
 #include "mlir/Support/ThreadLocalCache.h"
 #include "llvm/ADT/PointerEmbeddedInt.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
@@ -87,6 +88,13 @@ public:
 } // namespace LLVM
 } // namespace mlir
 
+namespace mlir {
+namespace LLVM {
+struct AssumeAlignTag {};
+struct AssumeSeparateStorageTag {};
+} // namespace LLVM
+} // namespace mlir
+
 ///// Ops /////
 #define GET_OP_CLASSES
 #include "mlir/Dialect/LLVMIR/LLVMOps.h.inc"
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 845c88b1be77..d07ebbacc604 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -450,7 +450,14 @@ def LLVM_AssumeOp
   }];
 
   let builders = [
-    OpBuilder<(ins "Value":$cond)>
+    OpBuilder<(ins "Value":$cond)>,
+    OpBuilder<(ins "Value":$cond,
+                   "ArrayRef<llvm::OperandBundleDefT<Value>>":$opBundles)>,
+    OpBuilder<(ins "Value":$cond, "llvm::StringRef":$tag, "ValueRange":$args)>,
+    OpBuilder<(ins "Value":$cond, "AssumeAlignTag":$tag, "Value":$ptr,
+                   "Value":$align)>,
+    OpBuilder<(ins "Value":$cond, "AssumeSeparateStorageTag":$tag,
+                   "Value":$ptr1, "Value":$ptr2)>
   ];
 
   let hasVerifier = 1;
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index cc73878a64ff..c9bc9533ca2a 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -3438,7 +3438,44 @@ void InlineAsmOp::getEffects(
 void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
                            mlir::Value cond) {
   return build(builder, state, cond, /*op_bundle_operands=*/{},
-               /*op_bundle_tags=*/{});
+               /*op_bundle_tags=*/ArrayAttr{});
+}
+
+void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
+                           Value cond,
+                           ArrayRef<llvm::OperandBundleDefT<Value>> opBundles) {
+  SmallVector<ValueRange> opBundleOperands;
+  SmallVector<Attribute> opBundleTags;
+  opBundleOperands.reserve(opBundles.size());
+  opBundleTags.reserve(opBundles.size());
+
+  for (const llvm::OperandBundleDefT<Value> &bundle : opBundles) {
+    opBundleOperands.emplace_back(bundle.inputs());
+    opBundleTags.push_back(
+        StringAttr::get(builder.getContext(), bundle.getTag()));
+  }
+
+  auto opBundleTagsAttr = ArrayAttr::get(builder.getContext(), opBundleTags);
+  return build(builder, state, cond, opBundleOperands, opBundleTagsAttr);
+}
+
+void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
+                           Value cond, llvm::StringRef tag, ValueRange args) {
+  llvm::OperandBundleDefT<Value> opBundle(
+      tag.str(), SmallVector<Value>(args.begin(), args.end()));
+  return build(builder, state, cond, opBundle);
+}
+
+void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
+                           Value cond, AssumeAlignTag, Value ptr, Value align) {
+  return build(builder, state, cond, "align", ValueRange{ptr, align});
+}
+
+void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
+                           Value cond, AssumeSeparateStorageTag, Value ptr1,
+                           Value ptr2) {
+  return build(builder, state, cond, "separate_storage",
+               ValueRange{ptr1, ptr2});
 }
 
 LogicalResult LLVM::AssumeOp::verify() { return verifyOperandBundles(*this); }
-- 
GitLab


From 355e6948d44a97781cc184a22c9b51760cae6de0 Mon Sep 17 00:00:00 2001
From: Teresa Johnson <tejohnson@google.com>
Date: Sat, 26 Oct 2024 20:53:20 -0700
Subject: [PATCH 256/693] [MemProf] Fix clone edge comparison (#113753)

The issue fixed in PR113337 exposed a bug in the comparisons done in
allocTypesMatch, which compares a vector of alloc types to those in the
given vector of Edges. The form of std::equal used, which didn't provide
the end iterator for the Edges vector, will iterate through as many
entries in the Edges vector as in the InAllocTypes vector, which can
fail if there are fewer entries in the Edges vector, because we may
dereference a bogus Edge pointer. This function is called twice, once
for the Node, with its callee edges, in which case the number of edges
should always match the number of entries in allocTypesMatch, which is
computed from the Node's callee edges. It was also called for Node's
clones, and it turns out that after cloning and edge modifications done
for other allocations, the number of callee edges in Node and its clones
may no longer match. In some cases, more common with memprof ICP before
the PR113337, the number of clone edges can be smaller leading to a bad
dereference. I found for a large application even before adding memprof
ICP support we sometimes call this with fewer entries in the clone's
callee edges, but were getting lucky as they had allocation type None,
and we didn't end up attempting to dereference the bad edge pointer.

Fix this by passing Edges.end() to std::equal, which means std::equal
will fail if the number of entries in the 2 vectors are not equal.
However, this is too conservative, as clone edges may have been added or
removed since it was initially cloned, and in fact can be wrong as we
may not be comparing allocation types corresponding to the same callee.

Therefore, a couple of enhancements are made to avoid regressing and
improve the checking and cloning:
- Don't bother calling the alloc type comparison when the clone and the
  Node's alloc type for the current allocation are precise (have a
  single allocation type) and are the same (which is guaranteed by an
  earlier check, and an assert is added to confirm that). In that case
  we can trivially determine that the clone can be used.
- Split the alloc type matching handling into a separate function for
  the clone case. In that case, for each of the InAllocType entries,
  attempt to find and compare to the clone callee edge with the same
  callee as the corresponding original node callee.

To create a test case I needed to take a spec application (xalancbmk),
and repeatedly apply random hot/cold-ness to the memprof contexts
when building, until I hit the problematic case. I then reduced that
full LTO IR using llvm-reduce and then manually.
---
 .../IPO/MemProfContextDisambiguation.cpp      | 66 +++++++++++--
 .../fix_clone_checking.ll                     | 99 +++++++++++++++++++
 2 files changed, 159 insertions(+), 6 deletions(-)
 create mode 100644 llvm/test/Transforms/MemProfContextDisambiguation/fix_clone_checking.ll

diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
index 905186edcbec..da5ded23ecc0 100644
--- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
+++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
@@ -928,8 +928,11 @@ bool allocTypesMatch(
     const std::vector<uint8_t> &InAllocTypes,
     const std::vector<std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>>>
         &Edges) {
+  // This should be called only when the InAllocTypes vector was computed for
+  // this set of Edges. Make sure the sizes are the same.
+  assert(InAllocTypes.size() == Edges.size());
   return std::equal(
-      InAllocTypes.begin(), InAllocTypes.end(), Edges.begin(),
+      InAllocTypes.begin(), InAllocTypes.end(), Edges.begin(), Edges.end(),
       [](const uint8_t &l,
          const std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>> &r) {
         // Can share if one of the edges is None type - don't
@@ -942,6 +945,46 @@ bool allocTypesMatch(
       });
 }
 
+// Helper to check if the alloc types for all edges recorded in the
+// InAllocTypes vector match the alloc types for callee edges in the given
+// clone. Because the InAllocTypes were computed from the original node's callee
+// edges, and other cloning could have happened after this clone was created, we
+// need to find the matching clone callee edge, which may or may not exist.
+template <typename DerivedCCG, typename FuncTy, typename CallTy>
+bool allocTypesMatchClone(
+    const std::vector<uint8_t> &InAllocTypes,
+    const ContextNode<DerivedCCG, FuncTy, CallTy> *Clone) {
+  const ContextNode<DerivedCCG, FuncTy, CallTy> *Node = Clone->CloneOf;
+  assert(Node);
+  // InAllocTypes should have been computed for the original node's callee
+  // edges.
+  assert(InAllocTypes.size() == Node->CalleeEdges.size());
+  // First create a map of the clone callee edge callees to the edge alloc type.
+  DenseMap<const ContextNode<DerivedCCG, FuncTy, CallTy> *, uint8_t>
+      EdgeCalleeMap;
+  for (const auto &E : Clone->CalleeEdges) {
+    assert(!EdgeCalleeMap.contains(E->Callee));
+    EdgeCalleeMap[E->Callee] = E->AllocTypes;
+  }
+  // Next, walk the original node's callees, and look for the corresponding
+  // clone edge to that callee.
+  for (unsigned I = 0; I < Node->CalleeEdges.size(); I++) {
+    auto Iter = EdgeCalleeMap.find(Node->CalleeEdges[I]->Callee);
+    // Not found is ok, we will simply add an edge if we use this clone.
+    if (Iter == EdgeCalleeMap.end())
+      continue;
+    // Can share if one of the edges is None type - don't
+    // care about the type along that edge as it doesn't
+    // exist for those context ids.
+    if (InAllocTypes[I] == (uint8_t)AllocationType::None ||
+        Iter->second == (uint8_t)AllocationType::None)
+      continue;
+    if (allocTypeToUse(Iter->second) != allocTypeToUse(InAllocTypes[I]))
+      return false;
+  }
+  return true;
+}
+
 } // end anonymous namespace
 
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
@@ -3364,11 +3407,22 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones(
           allocTypeToUse(CallerAllocTypeForAlloc))
         continue;
 
-      if (!allocTypesMatch<DerivedCCG, FuncTy, CallTy>(
-              CalleeEdgeAllocTypesForCallerEdge, CurClone->CalleeEdges))
-        continue;
-      Clone = CurClone;
-      break;
+      bool BothSingleAlloc = hasSingleAllocType(CurClone->AllocTypes) &&
+                             hasSingleAllocType(CallerAllocTypeForAlloc);
+      // The above check should mean that if both have single alloc types that
+      // they should be equal.
+      assert(!BothSingleAlloc ||
+             CurClone->AllocTypes == CallerAllocTypeForAlloc);
+
+      // If either both have a single alloc type (which are the same), or if the
+      // clone's callee edges have the same alloc types as those for the current
+      // allocation on Node's callee edges (CalleeEdgeAllocTypesForCallerEdge),
+      // then we can reuse this clone.
+      if (BothSingleAlloc || allocTypesMatchClone<DerivedCCG, FuncTy, CallTy>(
+                                 CalleeEdgeAllocTypesForCallerEdge, CurClone)) {
+        Clone = CurClone;
+        break;
+      }
     }
 
     // The edge iterator is adjusted when we move the CallerEdge to the clone.
diff --git a/llvm/test/Transforms/MemProfContextDisambiguation/fix_clone_checking.ll b/llvm/test/Transforms/MemProfContextDisambiguation/fix_clone_checking.ll
new file mode 100644
index 000000000000..75cebae0b829
--- /dev/null
+++ b/llvm/test/Transforms/MemProfContextDisambiguation/fix_clone_checking.ll
@@ -0,0 +1,99 @@
+;; Test to make sure we don't fail when cloning in a case where we end up with
+;; a clone that has fewer edges than the node it was initially cloned from.
+;; This test was reduced and simplified from xalancbmk with some random hotness
+;; applied to the profile that reproduced the issue.
+
+; RUN: opt -passes=memprof-context-disambiguation -supports-hot-cold-new \
+; RUN:		-memprof-verify-ccg -memprof-verify-nodes \
+; RUN: 		-pass-remarks=memprof-context-disambiguation %s -S 2>&1 | FileCheck %s
+
+;; Make sure we created some clones
+; CHECK: created clone A.memprof.1
+; CHECK: created clone C.memprof.1
+; CHECK: created clone D.memprof.1
+; CHECK: created clone E.memprof.1
+; CHECK: created clone B.memprof.1
+; CHECK: created clone F.memprof.1
+; CHECK: created clone G.memprof.1
+
+; ModuleID = '<stdin>'
+source_filename = "reduced.ll"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+define void @A() {
+  call void @B(), !callsite !0
+  ret void
+}
+
+define void @C() {
+  call void @D(), !callsite !1
+  ret void
+}
+
+define void @D() {
+  call void @A(), !callsite !2
+  ret void
+}
+
+define void @E() {
+  %1 = call ptr @_Znwm(i64 0), !memprof !3, !callsite !20
+  ret void
+}
+
+define void @B() {
+  call void @F(), !callsite !21
+  ret void
+}
+
+define void @F() {
+  call void @E(), !callsite !22
+  call void @G(), !callsite !23
+  ret void
+}
+
+define void @G() {
+  %1 = call ptr @_Znwm(i64 0), !memprof !24, !callsite !37
+  ret void
+}
+
+declare ptr @_Znwm(i64)
+
+!0 = !{i64 1995602625719775354}
+!1 = !{i64 4312698517630782220}
+!2 = !{i64 5516454029445989383}
+!3 = !{!4, !6, !8, !10, !12, !14, !16, !18}
+!4 = !{!5, !"notcold"}
+!5 = !{i64 -2282665745786859978, i64 -6758300505622211768, i64 2938340307832638862, i64 7147584705143805656, i64 -6456074186195384663, i64 1995602625719775354}
+!6 = !{!7, !"cold"}
+!7 = !{i64 -2282665745786859978, i64 -6758300505622211768, i64 2938340307832638862, i64 7147584705143805656, i64 -6456074186195384663, i64 2077908580042347045, i64 4312698517630782220, i64 5379466077518675850}
+!8 = !{!9, !"cold"}
+!9 = !{i64 -2282665745786859978, i64 -6758300505622211768, i64 2938340307832638862, i64 7147584705143805656, i64 -6456074186195384663, i64 2077908580042347045, i64 4312698517630782220, i64 -7632894069000375689}
+!10 = !{!11, !"cold"}
+!11 = !{i64 -2282665745786859978, i64 -6758300505622211768, i64 2938340307832638862, i64 7147584705143805656, i64 -6456074186195384663, i64 2939944783060497247}
+!12 = !{!13, !"notcold"}
+!13 = !{i64 -2282665745786859978, i64 -6758300505622211768, i64 2938340307832638862, i64 7147584705143805656, i64 -6456074186195384663, i64 5642549674080861567, i64 5516454029445989383, i64 4312698517630782220, i64 -7632894069000375689}
+!14 = !{!15, !"cold"}
+!15 = !{i64 -2282665745786859978, i64 -6758300505622211768, i64 2938340307832638862, i64 7147584705143805656, i64 -6456074186195384663, i64 5642549674080861567, i64 5516454029445989383, i64 4312698517630782220, i64 -1805555115991223293}
+!16 = !{!17, !"notcold"}
+!17 = !{i64 -2282665745786859978, i64 -6758300505622211768, i64 2938340307832638862, i64 7147584705143805656, i64 -6456074186195384663, i64 -4746997736434041076, i64 5516454029445989383, i64 4312698517630782220, i64 -1805555115991223293}
+!18 = !{!19, !"notcold"}
+!19 = !{i64 -2282665745786859978, i64 -6758300505622211768, i64 2938340307832638862, i64 7147584705143805656, i64 -6456074186195384663, i64 -4637272929643682959}
+!20 = !{i64 -2282665745786859978, i64 -6758300505622211768, i64 2938340307832638862}
+!21 = !{i64 -6456074186195384663}
+!22 = !{i64 7147584705143805656}
+!23 = !{i64 3938822378769440754}
+!24 = !{!25, !27, !29, !31, !33, !35}
+!25 = !{!26, !"cold"}
+!26 = !{i64 -2282665745786859978, i64 -3548918226713766361, i64 4077289288013931196, i64 3938822378769440754, i64 -6456074186195384663, i64 1995602625719775354, i64 5516454029445989383, i64 4312698517630782220, i64 -1805555115991223293}
+!27 = !{!28, !"notcold"}
+!28 = !{i64 -2282665745786859978, i64 -3548918226713766361, i64 4077289288013931196, i64 3938822378769440754, i64 -6456074186195384663, i64 2077908580042347045, i64 4312698517630782220, i64 -7632894069000375689}
+!29 = !{!30, !"cold"}
+!30 = !{i64 -2282665745786859978, i64 -3548918226713766361, i64 4077289288013931196, i64 3938822378769440754, i64 -6456074186195384663, i64 -4746997736434041076, i64 5516454029445989383, i64 4312698517630782220, i64 -7632894069000375689}
+!31 = !{!32, !"notcold"}
+!32 = !{i64 -2282665745786859978, i64 -3548918226713766361, i64 4077289288013931196, i64 3938822378769440754, i64 -6456074186195384663, i64 -4746997736434041076, i64 5516454029445989383, i64 4312698517630782220, i64 -1805555115991223293}
+!33 = !{!34, !"cold"}
+!34 = !{i64 -2282665745786859978, i64 -3548918226713766361, i64 4077289288013931196, i64 3938822378769440754, i64 -6456074186195384663, i64 -4637272929643682959}
+!35 = !{!36, !"notcold"}
+!36 = !{i64 -2282665745786859978, i64 -3548918226713766361, i64 4077289288013931196, i64 3938822378769440754, i64 -6456074186195384663, i64 -4409412896859835674}
+!37 = !{i64 -2282665745786859978, i64 -3548918226713766361, i64 4077289288013931196}
-- 
GitLab


From d5b42db00f0b21855501b01e8cd80326e1ce763d Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Sun, 27 Oct 2024 04:56:53 +0100
Subject: [PATCH 257/693] [clang][bytecode][NFC] Only do CheckConstant checks
 for global pointers (#113786)

We can check isStatic() early here and save ourselves some work.
---
 clang/lib/AST/ByteCode/Interp.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index b7a6c224c80f..6d40fb03696d 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -400,7 +400,7 @@ bool CheckConstant(InterpState &S, CodePtr OpPC, const Descriptor *Desc) {
 }
 
 static bool CheckConstant(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
-  if (!Ptr.isBlockPointer())
+  if (!Ptr.isStatic() || !Ptr.isBlockPointer())
     return true;
   return CheckConstant(S, OpPC, Ptr.getDeclDesc());
 }
-- 
GitLab


From 7b88e7530d4329ff0c7c8638f69b39fa1e540218 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Sun, 27 Oct 2024 05:06:47 +0100
Subject: [PATCH 258/693] [clang][bytecode][NFC] Make CheckVolatile static
 (#113785)

---
 clang/lib/AST/ByteCode/Interp.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index 6d40fb03696d..6e45cfb7e8a2 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -513,8 +513,8 @@ bool CheckMutable(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
   return false;
 }
 
-bool CheckVolatile(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
-                   AccessKinds AK) {
+static bool CheckVolatile(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
+                          AccessKinds AK) {
   assert(Ptr.isLive());
 
   // FIXME: This check here might be kinda expensive. Maybe it would be better
-- 
GitLab


From 60d2feded5c0f55b21d042ee2f35227847d66ee0 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 26 Oct 2024 22:07:56 -0700
Subject: [PATCH 259/693] [ARM] Remove a redundant call to StringRef::slice
 (NFC) (#113783)

OptStr.slice(0, OptStr.size()) is exactly the same as OptStr.
---
 llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 68f1199fd12e..0df1c336a221 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -5080,7 +5080,7 @@ ParseStatus ARMAsmParser::parseMemBarrierOptOperand(OperandVector &Operands) {
   if (Tok.is(AsmToken::Identifier)) {
     StringRef OptStr = Tok.getString();
 
-    Opt = StringSwitch<unsigned>(OptStr.slice(0, OptStr.size()).lower())
+    Opt = StringSwitch<unsigned>(OptStr.lower())
               .Case("sy", ARM_MB::SY)
               .Case("st", ARM_MB::ST)
               .Case("ld", ARM_MB::LD)
-- 
GitLab


From d2e9532fe12dc2568f40c2648ff4bb3730141aed Mon Sep 17 00:00:00 2001
From: Eirik Byrkjeflot Anonsen <eirik@eirikba.org>
Date: Sun, 27 Oct 2024 10:09:39 +0100
Subject: [PATCH 260/693] [DemoteRegToStack] Use correct variable for branch
 instructions in DemoteRegToStack (#113798)

I happened to see this code, and it seems "obviously" wrong to me. So
here's what I think this code is supposed to look like.
---
 llvm/lib/Transforms/Utils/DemoteRegToStack.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp b/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp
index 3a33b591d355..6337913cdbbe 100644
--- a/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp
+++ b/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp
@@ -55,8 +55,8 @@ AllocaInst *llvm::DemoteRegToStack(Instruction &I, bool VolatileLoads,
     for (unsigned i = 0; i < CBI->getNumSuccessors(); i++) {
       auto *Succ = CBI->getSuccessor(i);
       if (!Succ->getSinglePredecessor()) {
-        assert(isCriticalEdge(II, i) && "Expected a critical edge!");
-        [[maybe_unused]] BasicBlock *BB = SplitCriticalEdge(II, i);
+        assert(isCriticalEdge(CBI, i) && "Expected a critical edge!");
+        [[maybe_unused]] BasicBlock *BB = SplitCriticalEdge(CBI, i);
         assert(BB && "Unable to split critical edge.");
       }
     }
-- 
GitLab


From 45c84d59c454ba2b57affcd09a74f8d91e93bff7 Mon Sep 17 00:00:00 2001
From: goldsteinn <35538541+goldsteinn@users.noreply.github.com>
Date: Sun, 27 Oct 2024 06:49:20 -0700
Subject: [PATCH 261/693] [libc] Add `__builtin_expect` tag on assert
 conditions; NFC (#99498)

---
 libc/src/__support/CMakeLists.txt |  1 +
 libc/src/__support/libc_assert.h  |  5 +++--
 libc/src/assert/assert.h          | 16 +++++++++++++---
 3 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index 4785895b562b..14a3acff8fae 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -277,6 +277,7 @@ add_header_library(
   DEPENDS
     .integer_to_string
     libc.src.__support.OSUtil.osutil
+    libc.src.__support.macros.optimization
 )
 
 add_header_library(
diff --git a/libc/src/__support/libc_assert.h b/libc/src/__support/libc_assert.h
index e21a58a0c8aa..3db179ff6721 100644
--- a/libc/src/__support/libc_assert.h
+++ b/libc/src/__support/libc_assert.h
@@ -24,7 +24,8 @@
 #include "src/__support/OSUtil/exit.h"
 #include "src/__support/OSUtil/io.h"
 #include "src/__support/integer_to_string.h"
-#include "src/__support/macros/attributes.h" // For LIBC_INLINE
+#include "src/__support/macros/attributes.h"   // For LIBC_INLINE
+#include "src/__support/macros/optimization.h" // For LIBC_UNLIKELY
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -71,7 +72,7 @@ LIBC_INLINE void report_assertion_failure(const char *assertion,
 
 #define LIBC_ASSERT(COND)                                                      \
   do {                                                                         \
-    if (!(COND)) {                                                             \
+    if (LIBC_UNLIKELY(!(COND))) {                                              \
       LIBC_NAMESPACE::write_to_stderr(__FILE__ ":" __LIBC_LINE_STR__           \
                                                ": Assertion failed: '" #COND   \
                                                "' in function: '");            \
diff --git a/libc/src/assert/assert.h b/libc/src/assert/assert.h
index 6f352af1988b..1ea19ea5554f 100644
--- a/libc/src/assert/assert.h
+++ b/libc/src/assert/assert.h
@@ -18,8 +18,18 @@
 #ifdef NDEBUG
 #define assert(e) (void)0
 #else
+
+#ifdef __has_builtin
+#if __has_builtin(__builtin_expect)
+#define __LIBC_ASSERT_LIKELY(e) __builtin_expect(e, 1)
+#endif
+#endif
+#ifndef __LIBC_ASSERT_LIKELY
+#define __LIBC_ASSERT_LIKELY(e) e
+#endif
+
 #define assert(e)                                                              \
-  ((e) ? (void)0                                                               \
-       : LIBC_NAMESPACE::__assert_fail(#e, __FILE__, __LINE__,                 \
-                                       __PRETTY_FUNCTION__))
+  (__LIBC_ASSERT_LIKELY(e) ? (void)0                                           \
+                           : LIBC_NAMESPACE::__assert_fail(                    \
+                                 #e, __FILE__, __LINE__, __PRETTY_FUNCTION__))
 #endif // NDEBUG
-- 
GitLab


From 5287a9b3456fe7aefa24c8da95ef265b8dba875b Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sun, 27 Oct 2024 07:28:27 -0700
Subject: [PATCH 262/693] [mlir] Prefer StringRef::substr to slice (NFC)
 (#113788)

I'm planning to migrate StringRef to std::string_view
eventually.  Since std::string_view does not have slice, this patch
migrates:

  slice(0, N)                to  substr(0, N)
  slice(N, StringRef::npos)  to  substr(N)
---
 mlir/include/mlir/Support/IndentedOstream.h | 3 +--
 mlir/lib/Query/QueryParser.cpp              | 4 ++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/mlir/include/mlir/Support/IndentedOstream.h b/mlir/include/mlir/Support/IndentedOstream.h
index 101aa8b631d2..eeab36806c4e 100644
--- a/mlir/include/mlir/Support/IndentedOstream.h
+++ b/mlir/include/mlir/Support/IndentedOstream.h
@@ -166,8 +166,7 @@ inline void mlir::raw_indented_ostream::write_impl(const char *ptr,
       break;
     }
 
-    auto split =
-        std::make_pair(str.slice(0, idx), str.slice(idx + 1, StringRef::npos));
+    auto split = std::make_pair(str.substr(0, idx), str.substr(idx + 1));
     // Print empty new line without spaces if line only has spaces and no extra
     // prefix is requested.
     if (!split.first.ltrim().empty() || !currentExtraPrefix.empty())
diff --git a/mlir/lib/Query/QueryParser.cpp b/mlir/lib/Query/QueryParser.cpp
index 13ee931cc522..31aead7d403d 100644
--- a/mlir/lib/Query/QueryParser.cpp
+++ b/mlir/lib/Query/QueryParser.cpp
@@ -181,8 +181,8 @@ QueryRef QueryParser::doParse() {
     if (!matcher) {
       return makeInvalidQueryFromDiagnostics(diag);
     }
-    auto actualSource = origMatcherSource.slice(0, origMatcherSource.size() -
-                                                       matcherSource.size());
+    auto actualSource = origMatcherSource.substr(0, origMatcherSource.size() -
+                                                        matcherSource.size());
     QueryRef query = new MatchQuery(actualSource, *matcher);
     query->remainingContent = matcherSource;
     return query;
-- 
GitLab


From 7fe149cdf09d04fb8390b97c91bd9214c968cd3e Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sun, 27 Oct 2024 16:22:18 +0100
Subject: [PATCH 263/693] [VPlan] Replace getIRBasicBlock with IRBB in
 VPIRBB::execute (NFC).

Suggested in https://github.com/llvm/llvm-project/pull/109975. This
makes the function consistent throughout.
---
 llvm/lib/Transforms/Vectorize/VPlan.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 44ffcb954a28..0484543d2d03 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -453,13 +453,13 @@ VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
 void VPIRBasicBlock::execute(VPTransformState *State) {
   assert(getHierarchicalSuccessors().size() <= 2 &&
          "VPIRBasicBlock can have at most two successors at the moment!");
-  State->Builder.SetInsertPoint(getIRBasicBlock()->getTerminator());
-  executeRecipes(State, getIRBasicBlock());
+  State->Builder.SetInsertPoint(IRBB->getTerminator());
+  executeRecipes(State, IRBB);
   if (getSingleSuccessor()) {
-    assert(isa<UnreachableInst>(getIRBasicBlock()->getTerminator()));
-    auto *Br = State->Builder.CreateBr(getIRBasicBlock());
+    assert(isa<UnreachableInst>(IRBB->getTerminator()));
+    auto *Br = State->Builder.CreateBr(IRBB);
     Br->setOperand(0, nullptr);
-    getIRBasicBlock()->getTerminator()->eraseFromParent();
+    IRBB->getTerminator()->eraseFromParent();
   }
 
   for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) {
-- 
GitLab


From e33aec89ef1378d80e8df2e965ac5e6e6aa2e3de Mon Sep 17 00:00:00 2001
From: Durgadoss R <durgadossr@nvidia.com>
Date: Sun, 27 Oct 2024 22:24:31 +0530
Subject: [PATCH 264/693] [MLIR][NVVM] Update the elect.sync Op to use
 intrinsics (#113757)

Recently, we added an intrinsic for the elect.sync PTX instruction (PR
104780). This patch updates the corresponding Op in NVVM Dialect
to lower to the intrinsic instead of inline-ptx.

The existing test under Conversion/ is migrated to check for the new
pattern. A separate test is added to verify the lowered intrinsic under
the Target/ directory.

Signed-off-by: Durgadoss R <durgadossr@nvidia.com>
---
 mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td   | 33 ++++++++++---------
 .../Conversion/NVVMToLLVM/nvvm-to-llvm.mlir   |  8 +----
 mlir/test/Target/LLVMIR/nvvmir.mlir           |  9 +++++
 3 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index 5806295cedb1..7cb4b5c346ad 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -783,24 +783,27 @@ def NVVM_SyncWarpOp :
   let assemblyFormat = "$mask attr-dict `:` type($mask)";
 }
 
-
-def NVVM_ElectSyncOp : NVVM_Op<"elect.sync", 
-                  [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]>
+def NVVM_ElectSyncOp : NVVM_Op<"elect.sync">
 {  
+  let summary = "Elect one leader thread";
+  let description = [{
+    The `elect.sync` instruction elects one predicated active leader
+    thread from among a set of threads specified in membermask.
+    The membermask is set to `0xFFFFFFFF` for the current version
+    of this Op. The predicate result is set to `True` for the
+    leader thread, and `False` for all other threads.
+
+    [For more information, see PTX ISA]
+    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-elect-sync)
+  }];
+
   let results = (outs I1:$pred);
   let assemblyFormat = "attr-dict `->` type(results)";  
-  let extraClassDefinition = [{        
-    std::string $cppClass::getPtx() { 
-      return std::string(
-        "{                                  \n"
-        ".reg .u32 rx;                      \n"
-        ".reg .pred px;                     \n"
-        " mov.pred %0, 0;                   \n"
-        "    elect.sync rx | px, 0xFFFFFFFF;\n"
-        "@px mov.pred %0, 1;                \n"
-        "}\n"
-      ); 
-    }
+  string llvmBuilder = [{
+    auto *resultTuple = createIntrinsicCall(builder,
+        llvm::Intrinsic::nvvm_elect_sync, {builder.getInt32(0xFFFFFFFF)});
+    // Extract the second value into $pred
+    $pred = builder.CreateExtractValue(resultTuple, 1);
   }];
 }
 
diff --git a/mlir/test/Conversion/NVVMToLLVM/nvvm-to-llvm.mlir b/mlir/test/Conversion/NVVMToLLVM/nvvm-to-llvm.mlir
index 375e2951a037..66b736c18718 100644
--- a/mlir/test/Conversion/NVVMToLLVM/nvvm-to-llvm.mlir
+++ b/mlir/test/Conversion/NVVMToLLVM/nvvm-to-llvm.mlir
@@ -579,13 +579,7 @@ func.func @wgmma_f32_e5m2_e4m3(%descA : i64, %descB : i64) -> !mat32f32 {
 // -----
 
 func.func @elect_one_leader_sync() {  
-  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "{
-  // CHECK-SAME: .reg .u32 rx;
-  // CHECK-SAME: .reg .pred px;
-  // CHECK-SAME: mov.pred $0, 0;
-  // CHECK-SAME: elect.sync rx | px, 0xFFFFFFFF;
-  // CHECK-SAME: @px mov.pred $0, 1;
-  // CHECK-SAME: "=b"  : () -> i1
+  // CHECK: %[[RES:.*]] = nvvm.elect.sync -> i1
   %cnd = nvvm.elect.sync -> i1 
   return 
 }
diff --git a/mlir/test/Target/LLVMIR/nvvmir.mlir b/mlir/test/Target/LLVMIR/nvvmir.mlir
index 0471e5faf845..75ce958b43fd 100644
--- a/mlir/test/Target/LLVMIR/nvvmir.mlir
+++ b/mlir/test/Target/LLVMIR/nvvmir.mlir
@@ -259,6 +259,15 @@ llvm.func @nvvm_vote(%0 : i32, %1 : i1) -> i32 {
   llvm.return %3 : i32
 }
 
+// CHECK-LABEL: @nvvm_elect_sync
+llvm.func @nvvm_elect_sync() -> i1 {
+  // CHECK: %[[RES:.*]] = call { i32, i1 } @llvm.nvvm.elect.sync(i32 -1)
+  // CHECK-NEXT: %[[PRED:.*]] = extractvalue { i32, i1 } %[[RES]], 1
+  // CHECK-NEXT: ret i1 %[[PRED]]
+  %0 = nvvm.elect.sync -> i1
+  llvm.return %0 : i1
+}
+
 // CHECK-LABEL: @nvvm_mma_mn8n8k4_row_col_f32_f32
 llvm.func @nvvm_mma_mn8n8k4_row_col_f32_f32(%a0 : vector<2xf16>, %a1 : vector<2xf16>,
                     %b0 : vector<2xf16>, %b1 : vector<2xf16>,
-- 
GitLab


From eef3766ae5a39fea6f7f81ac444f878969743d85 Mon Sep 17 00:00:00 2001
From: Kiran Chandramohan <kiran.chandramohan@arm.com>
Date: Sun, 27 Oct 2024 18:58:47 +0000
Subject: [PATCH 265/693] Assumed-size arrays are shared and cannot be
 privatized (#112963)

Do not error out if default(none) is specified and the region has an
assumed-size array.

Fixes #110442
---
 flang/lib/Semantics/resolve-directives.cpp   |  2 ++
 flang/test/Semantics/OpenMP/default-none.f90 | 11 +++++++++++
 2 files changed, 13 insertions(+)

diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 513e42bee976..979570a7d410 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -2052,6 +2052,8 @@ void OmpAttributeVisitor::Post(const parser::OpenMPAllocatorsConstruct &x) {
 static bool IsPrivatizable(const Symbol *sym) {
   auto *misc{sym->detailsIf<MiscDetails>()};
   return !IsProcedure(*sym) && !IsNamedConstant(*sym) &&
+      !semantics::IsAssumedSizeArray(
+          *sym) && /* OpenMP 5.2, 5.1.1: Assumed-size arrays are shared*/
       !sym->owner().IsDerivedType() &&
       sym->owner().kind() != Scope::Kind::ImpliedDos &&
       !sym->detailsIf<semantics::AssocEntityDetails>() &&
diff --git a/flang/test/Semantics/OpenMP/default-none.f90 b/flang/test/Semantics/OpenMP/default-none.f90
index 11ba878ea779..761c2385466a 100644
--- a/flang/test/Semantics/OpenMP/default-none.f90
+++ b/flang/test/Semantics/OpenMP/default-none.f90
@@ -47,3 +47,14 @@ subroutine sb4
     end do loop
   !$omp end parallel
 end subroutine
+
+! Test that default(none) does not error for assumed-size array
+subroutine sub( aaa)
+  real,dimension(*),intent(in)::aaa
+  integer::ip
+  real::ccc
+!$omp parallel do private(ip,ccc) default(none)
+  do ip = 1, 10
+     ccc= aaa(ip)
+  end do
+end subroutine sub
-- 
GitLab


From 5621929f7f4878c318deecb592cd03f5ecbb83ba Mon Sep 17 00:00:00 2001
From: Kiran Chandramohan <kiran.chandramohan@arm.com>
Date: Sun, 27 Oct 2024 20:16:24 +0000
Subject: [PATCH 266/693] [Flang][OpenMP] Add parser support for grainsize and
 num_tasks clause (#113136)

These clauses are applicable only for the taskloop directive. Since the
directive has a TODO error, skipping the addition of TODOs for these
clauses.
---
 flang/examples/FeatureList/FeatureList.cpp   |  4 ++
 flang/include/flang/Parser/dump-parse-tree.h |  4 ++
 flang/include/flang/Parser/parse-tree.h      | 14 +++++++
 flang/lib/Lower/OpenMP/Clauses.cpp           | 35 +++++++++++++----
 flang/lib/Parser/openmp-parsers.cpp          | 14 ++++++-
 flang/lib/Parser/unparse.cpp                 | 13 +++++++
 flang/lib/Semantics/check-omp-structure.cpp  |  4 +-
 flang/test/Parser/OpenMP/taskloop.f90        | 41 ++++++++++++++++++++
 llvm/include/llvm/Frontend/OpenMP/OMP.td     |  8 ++--
 9 files changed, 122 insertions(+), 15 deletions(-)
 create mode 100644 flang/test/Parser/OpenMP/taskloop.f90

diff --git a/flang/examples/FeatureList/FeatureList.cpp b/flang/examples/FeatureList/FeatureList.cpp
index 06ca12a492d2..9fce67e61ed3 100644
--- a/flang/examples/FeatureList/FeatureList.cpp
+++ b/flang/examples/FeatureList/FeatureList.cpp
@@ -483,6 +483,8 @@ public:
   READ_FEATURE(OmpEndCriticalDirective)
   READ_FEATURE(OmpEndLoopDirective)
   READ_FEATURE(OmpEndSectionsDirective)
+  READ_FEATURE(OmpGrainsizeClause)
+  READ_FEATURE(OmpGrainsizeClause::Prescriptiveness)
   READ_FEATURE(OmpIfClause)
   READ_FEATURE(OmpIfClause::DirectiveNameModifier)
   READ_FEATURE(OmpLinearClause)
@@ -494,6 +496,8 @@ public:
   READ_FEATURE(OmpMapClause)
   READ_FEATURE(OmpMapClause::TypeModifier)
   READ_FEATURE(OmpMapClause::Type)
+  READ_FEATURE(OmpNumTasksClause)
+  READ_FEATURE(OmpNumTasksClause::Prescriptiveness)
   READ_FEATURE(OmpObject)
   READ_FEATURE(OmpObjectList)
   READ_FEATURE(OmpOrderClause)
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 76d2f164fc4b..ccdfe980f6f3 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -547,6 +547,10 @@ public:
   NODE_ENUM(OmpOrderClause, Type)
   NODE(parser, OmpOrderModifier)
   NODE_ENUM(OmpOrderModifier, Kind)
+  NODE(parser, OmpGrainsizeClause)
+  NODE_ENUM(OmpGrainsizeClause, Prescriptiveness)
+  NODE(parser, OmpNumTasksClause)
+  NODE_ENUM(OmpNumTasksClause, Prescriptiveness)
   NODE(parser, OmpProcBindClause)
   NODE_ENUM(OmpProcBindClause, Type)
   NODE_ENUM(OmpReductionClause, ReductionModifier)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index c1884f6e88d1..2a312e29a3a4 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3559,6 +3559,13 @@ struct OmpDependClause {
   std::variant<Source, Sink, InOut> u;
 };
 
+// OMP 5.2 12.6.1 grainsize-clause -> grainsize ([prescriptiveness :] value)
+struct OmpGrainsizeClause {
+  TUPLE_CLASS_BOILERPLATE(OmpGrainsizeClause);
+  ENUM_CLASS(Prescriptiveness, Strict);
+  std::tuple<std::optional<Prescriptiveness>, ScalarIntExpr> t;
+};
+
 // 2.12 if-clause -> IF ([ directive-name-modifier :] scalar-logical-expr)
 struct OmpIfClause {
   TUPLE_CLASS_BOILERPLATE(OmpIfClause);
@@ -3688,6 +3695,13 @@ struct OmpScheduleClause {
       t;
 };
 
+// OMP 5.2 12.6.2 num_tasks-clause -> num_tasks ([prescriptiveness :] value)
+struct OmpNumTasksClause {
+  TUPLE_CLASS_BOILERPLATE(OmpNumTasksClause);
+  ENUM_CLASS(Prescriptiveness, Strict);
+  std::tuple<std::optional<Prescriptiveness>, ScalarIntExpr> t;
+};
+
 // OpenMP Clauses
 struct OmpClause {
   UNION_CLASS_BOILERPLATE(OmpClause);
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index ee3d74a7c631..3bd89b543288 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -721,10 +721,20 @@ From make(const parser::OmpClause::From &inp,
 // Full: empty
 
 Grainsize make(const parser::OmpClause::Grainsize &inp,
-               semantics::SemanticsContext &semaCtx) {
-  // inp.v -> parser::ScalarIntExpr
-  return Grainsize{{/*Prescriptiveness=*/std::nullopt,
-                    /*GrainSize=*/makeExpr(inp.v, semaCtx)}};
+            semantics::SemanticsContext &semaCtx) {
+  // inp.v -> parser::OmpGrainsizeClause
+  using wrapped = parser::OmpGrainsizeClause;
+
+  CLAUSET_ENUM_CONVERT( //
+      convert, parser::OmpGrainsizeClause::Prescriptiveness, Grainsize::Prescriptiveness,
+      // clang-format off
+      MS(Strict,   Strict)
+      // clang-format on
+  );
+  auto &t0 = std::get<std::optional<wrapped::Prescriptiveness>>(inp.v.t);
+  auto &t1 = std::get<parser::ScalarIntExpr>(inp.v.t);
+  return Grainsize{{/*Prescriptiveness=*/maybeApply(convert, t0),
+                    /*Grainsize=*/makeExpr(t1, semaCtx)}};
 }
 
 HasDeviceAddr make(const parser::OmpClause::HasDeviceAddr &inp,
@@ -971,9 +981,20 @@ Novariants make(const parser::OmpClause::Novariants &inp,
 
 NumTasks make(const parser::OmpClause::NumTasks &inp,
               semantics::SemanticsContext &semaCtx) {
-  // inp.v -> parser::ScalarIntExpr
-  return NumTasks{{/*Prescriptiveness=*/std::nullopt,
-                   /*NumTasks=*/makeExpr(inp.v, semaCtx)}};
+  // inp.v -> parser::OmpNumTasksClause
+  using wrapped = parser::OmpNumTasksClause;
+
+  CLAUSET_ENUM_CONVERT( //
+      convert, parser::OmpNumTasksClause::Prescriptiveness,
+      NumTasks::Prescriptiveness,
+      // clang-format off
+      MS(Strict,   Strict)
+      // clang-format on
+  );
+  auto &t0 = std::get<std::optional<wrapped::Prescriptiveness>>(inp.v.t);
+  auto &t1 = std::get<parser::ScalarIntExpr>(inp.v.t);
+  return NumTasks{{/*Prescriptiveness=*/maybeApply(convert, t0),
+                   /*NumTasks=*/makeExpr(t1, semaCtx)}};
 }
 
 NumTeams make(const parser::OmpClause::NumTeams &inp,
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index e740c421ca80..ae0c351fed56 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -408,6 +408,16 @@ TYPE_PARSER(construct<OmpOrderClause>(
     maybe(Parser<OmpOrderModifier>{} / ":"),
     "CONCURRENT" >> pure(OmpOrderClause::Type::Concurrent)))
 
+// OMP 5.2 12.6.1 grainsize([ prescriptiveness :] scalar-integer-expression)
+TYPE_PARSER(construct<OmpGrainsizeClause>(
+    maybe("STRICT" >> pure(OmpGrainsizeClause::Prescriptiveness::Strict) / ":"),
+    scalarIntExpr))
+
+// OMP 5.2 12.6.2 num_tasks([ prescriptiveness :] scalar-integer-expression)
+TYPE_PARSER(construct<OmpNumTasksClause>(
+    maybe("STRICT" >> pure(OmpNumTasksClause::Prescriptiveness::Strict) / ":"),
+    scalarIntExpr))
+
 TYPE_PARSER(
     construct<OmpObject>(designator) || construct<OmpObject>("/" >> name / "/"))
 
@@ -464,7 +474,7 @@ TYPE_PARSER(
     "FROM" >> construct<OmpClause>(construct<OmpClause::From>(
                   parenthesized(Parser<OmpObjectList>{}))) ||
     "GRAINSIZE" >> construct<OmpClause>(construct<OmpClause::Grainsize>(
-                       parenthesized(scalarIntExpr))) ||
+                       parenthesized(Parser<OmpGrainsizeClause>{}))) ||
     "HAS_DEVICE_ADDR" >>
         construct<OmpClause>(construct<OmpClause::HasDeviceAddr>(
             parenthesized(Parser<OmpObjectList>{}))) ||
@@ -491,7 +501,7 @@ TYPE_PARSER(
         construct<OmpClause>(construct<OmpClause::Notinbranch>()) ||
     "NOWAIT" >> construct<OmpClause>(construct<OmpClause::Nowait>()) ||
     "NUM_TASKS" >> construct<OmpClause>(construct<OmpClause::NumTasks>(
-                       parenthesized(scalarIntExpr))) ||
+                       parenthesized(Parser<OmpNumTasksClause>{}))) ||
     "NUM_TEAMS" >> construct<OmpClause>(construct<OmpClause::NumTeams>(
                        parenthesized(scalarIntExpr))) ||
     "NUM_THREADS" >> construct<OmpClause>(construct<OmpClause::NumThreads>(
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 19ceb2a3ebc3..ba4155469073 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2196,6 +2196,16 @@ public:
     Walk(std::get<std::optional<OmpOrderModifier>>(x.t), ":");
     Walk(std::get<OmpOrderClause::Type>(x.t));
   }
+  void Unparse(const OmpGrainsizeClause &x) {
+    Walk(std::get<std::optional<OmpGrainsizeClause::Prescriptiveness>>(x.t),
+        ":");
+    Walk(std::get<ScalarIntExpr>(x.t));
+  }
+  void Unparse(const OmpNumTasksClause &x) {
+    Walk(
+        std::get<std::optional<OmpNumTasksClause::Prescriptiveness>>(x.t), ":");
+    Walk(std::get<ScalarIntExpr>(x.t));
+  }
   void Unparse(const OmpDependSinkVecLength &x) {
     Walk(std::get<DefinedOperator>(x.t));
     Walk(std::get<ScalarIntConstantExpr>(x.t));
@@ -2829,6 +2839,9 @@ public:
   WALK_NESTED_ENUM(OmpCancelType, Type) // OMP cancel-type
   WALK_NESTED_ENUM(OmpOrderClause, Type) // OMP order-type
   WALK_NESTED_ENUM(OmpOrderModifier, Kind) // OMP order-modifier
+  WALK_NESTED_ENUM(
+      OmpGrainsizeClause, Prescriptiveness) // OMP grainsize-modifier
+  WALK_NESTED_ENUM(OmpNumTasksClause, Prescriptiveness) // OMP numtasks-modifier
   WALK_NESTED_ENUM(OmpMapClause, Type) // OMP map-type
   WALK_NESTED_ENUM(OmpMapClause, TypeModifier) // OMP map-type-modifier
 #undef WALK_NESTED_ENUM
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 1c2cf304d0ee..455322d610d6 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -2479,12 +2479,14 @@ CHECK_SIMPLE_CLAUSE(Final, OMPC_final)
 CHECK_SIMPLE_CLAUSE(Flush, OMPC_flush)
 CHECK_SIMPLE_CLAUSE(From, OMPC_from)
 CHECK_SIMPLE_CLAUSE(Full, OMPC_full)
+CHECK_SIMPLE_CLAUSE(Grainsize, OMPC_grainsize)
 CHECK_SIMPLE_CLAUSE(Hint, OMPC_hint)
 CHECK_SIMPLE_CLAUSE(Holds, OMPC_holds)
 CHECK_SIMPLE_CLAUSE(InReduction, OMPC_in_reduction)
 CHECK_SIMPLE_CLAUSE(Inclusive, OMPC_inclusive)
 CHECK_SIMPLE_CLAUSE(Match, OMPC_match)
 CHECK_SIMPLE_CLAUSE(Nontemporal, OMPC_nontemporal)
+CHECK_SIMPLE_CLAUSE(NumTasks, OMPC_num_tasks)
 CHECK_SIMPLE_CLAUSE(Order, OMPC_order)
 CHECK_SIMPLE_CLAUSE(Read, OMPC_read)
 CHECK_SIMPLE_CLAUSE(Threadprivate, OMPC_threadprivate)
@@ -2535,8 +2537,6 @@ CHECK_SIMPLE_CLAUSE(OmpxBare, OMPC_ompx_bare)
 CHECK_SIMPLE_CLAUSE(Fail, OMPC_fail)
 CHECK_SIMPLE_CLAUSE(Weak, OMPC_weak)
 
-CHECK_REQ_SCALAR_INT_CLAUSE(Grainsize, OMPC_grainsize)
-CHECK_REQ_SCALAR_INT_CLAUSE(NumTasks, OMPC_num_tasks)
 CHECK_REQ_SCALAR_INT_CLAUSE(NumTeams, OMPC_num_teams)
 CHECK_REQ_SCALAR_INT_CLAUSE(NumThreads, OMPC_num_threads)
 CHECK_REQ_SCALAR_INT_CLAUSE(OmpxDynCgroupMem, OMPC_ompx_dyn_cgroup_mem)
diff --git a/flang/test/Parser/OpenMP/taskloop.f90 b/flang/test/Parser/OpenMP/taskloop.f90
new file mode 100644
index 000000000000..a9c361046bd5
--- /dev/null
+++ b/flang/test/Parser/OpenMP/taskloop.f90
@@ -0,0 +1,41 @@
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp %s | FileCheck --ignore-case %s
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine parallel_work
+  integer :: i
+
+!CHECK: !$OMP TASKLOOP  GRAINSIZE(STRICT:500_4)
+!PARSE-TREE: OmpBeginLoopDirective
+!PARSE-TREE-NEXT: OmpLoopDirective -> llvm::omp::Directive = taskloop
+!PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Grainsize -> OmpGrainsizeClause
+!PARSE-TREE-NEXT: Prescriptiveness = Strict
+!PARSE-TREE-NEXT: Scalar -> Integer -> Expr = '500_4'
+  !$omp taskloop grainsize(strict: 500)
+  do i=1,10000
+    call loop_body(i)
+  end do
+  !$omp end taskloop
+
+!CHECK: !$OMP TASKLOOP  GRAINSIZE(500_4)
+!PARSE-TREE: OmpBeginLoopDirective
+!PARSE-TREE-NEXT: OmpLoopDirective -> llvm::omp::Directive = taskloop
+!PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Grainsize -> OmpGrainsizeClause
+!PARSE-TREE-NEXT: Scalar -> Integer -> Expr = '500_4'
+  !$omp taskloop grainsize(500)
+  do i=1,10000
+    call loop_body(i)
+  end do
+  !$omp end taskloop
+
+!CHECK: !$OMP TASKLOOP  NUM_TASKS(STRICT:500_4)
+!PARSE-TREE: OmpBeginLoopDirective
+!PARSE-TREE-NEXT: OmpLoopDirective -> llvm::omp::Directive = taskloop
+!PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> NumTasks -> OmpNumTasksClause
+!PARSE-TREE-NEXT: Prescriptiveness = Strict
+!PARSE-TREE-NEXT: Scalar -> Integer -> Expr = '500_4'
+  !$omp taskloop num_tasks(strict: 500)
+  do i=1,10000
+    call loop_body(i)
+  end do
+  !$omp end taskloop
+end subroutine parallel_work
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index d592f369a17f..70179bab4757 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -185,10 +185,10 @@ def OMPC_Full: Clause<"full"> {
   let clangClass = "OMPFullClause";
 }
 def OMP_GRAINSIZE_Strict : ClauseVal<"strict", 1, 1> {}
-def OMP_GRAINSIZE_Unknown : ClauseVal<"unkonwn", 2, 0> { let isDefault = 1; }
+def OMP_GRAINSIZE_Unknown : ClauseVal<"unknown", 2, 0> { let isDefault = 1; }
 def OMPC_GrainSize : Clause<"grainsize"> {
   let clangClass = "OMPGrainsizeClause";
-  let flangClass = "ScalarIntExpr";
+  let flangClass = "OmpGrainsizeClause";
   let enumClauseValue = "GrainsizeType";
   let allowedClauseValues = [
     OMP_GRAINSIZE_Strict,
@@ -301,10 +301,10 @@ def OMPC_NoWait : Clause<"nowait"> {
   let clangClass = "OMPNowaitClause";
 }
 def OMP_NUMTASKS_Strict : ClauseVal<"strict", 1, 1> {}
-def OMP_NUMTASKS_Unknown : ClauseVal<"unkonwn", 2, 0> { let isDefault = 1; }
+def OMP_NUMTASKS_Unknown : ClauseVal<"unknown", 2, 0> { let isDefault = 1; }
 def OMPC_NumTasks : Clause<"num_tasks"> {
   let clangClass = "OMPNumTasksClause";
-  let flangClass = "ScalarIntExpr";
+  let flangClass = "OmpNumTasksClause";
   let enumClauseValue = "NumTasksType";
   let allowedClauseValues = [
     OMP_NUMTASKS_Strict,
-- 
GitLab


From 7b3da7b3b2b0e2f322dddf1f343571cc7fd09b09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Sun, 27 Oct 2024 23:14:07 +0100
Subject: [PATCH 267/693] [GlobalISel][AArch64] Legalize G_ADD, G_SUB, G_AND,
 G_OR, and G_XOR for SVE (#110561)

Credits: https://github.com/llvm/llvm-project/pull/72976

LLVM ERROR: cannot select: %3:zpr(<vscale x 2 x s64>) = G_MUL %0:fpr,
%1:fpr (in function: xmulnxv2i64)

;; mul
define void @xmulnxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b,
ptr %p) {
entry:
  %c = mul <vscale x 2 x i64> %a, %b
  store <vscale x 2 x i64> %c, ptr %p, align 16
   ret void
}

define void @mulnxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b,
ptr %p) {
entry:
  %c = mul <vscale x 4 x i32> %a, %b
  store <vscale x 4 x i32> %c, ptr %p, align 16
   ret void
}

define void @mulnxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b,
ptr %p) {
entry:
  %c = mul <vscale x 8 x i16> %a, %b
  store <vscale x 8 x i16> %c, ptr %p, align 16
  ret void
}

define void @mulnxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b,
ptr %p) {
entry:
  %c = mul <vscale x 16 x i8> %a, %b
  store <vscale x 16 x i8> %c, ptr %p, align 16
  ret void
}
---
 .../llvm/CodeGen/GlobalISel/LegalizerInfo.h   |  12 +-
 .../AArch64/AArch64GenRegisterBankInfo.def    |   2 +
 .../AArch64/GISel/AArch64CallLowering.cpp     |   4 +-
 .../GISel/AArch64InstructionSelector.cpp      |  10 +-
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |  30 ++-
 .../GlobalISel/legalizer-info-validation.mir  |   1 -
 .../CodeGen/AArch64/GlobalISel/sve-integer.ll | 208 ++++++++++++++++++
 7 files changed, 257 insertions(+), 10 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/sve-integer.ll

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
index bcd44abb2088..6d71c150c8da 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
@@ -998,8 +998,7 @@ public:
         LegalizeAction::WidenScalar,
         [=](const LegalityQuery &Query) {
           const LLT VecTy = Query.Types[TypeIdx];
-          return VecTy.isVector() && !VecTy.isScalable() &&
-                 VecTy.getSizeInBits() < VectorSize;
+          return VecTy.isFixedVector() && VecTy.getSizeInBits() < VectorSize;
         },
         [=](const LegalityQuery &Query) {
           const LLT VecTy = Query.Types[TypeIdx];
@@ -1172,7 +1171,7 @@ public:
         LegalizeAction::MoreElements,
         [=](const LegalityQuery &Query) {
           LLT VecTy = Query.Types[TypeIdx];
-          return VecTy.isVector() && VecTy.getElementType() == EltTy &&
+          return VecTy.isFixedVector() && VecTy.getElementType() == EltTy &&
                  VecTy.getNumElements() < MinElements;
         },
         [=](const LegalityQuery &Query) {
@@ -1190,7 +1189,7 @@ public:
         LegalizeAction::MoreElements,
         [=](const LegalityQuery &Query) {
           LLT VecTy = Query.Types[TypeIdx];
-          return VecTy.isVector() && VecTy.getElementType() == EltTy &&
+          return VecTy.isFixedVector() && VecTy.getElementType() == EltTy &&
                  (VecTy.getNumElements() % NumElts != 0);
         },
         [=](const LegalityQuery &Query) {
@@ -1210,7 +1209,7 @@ public:
         LegalizeAction::FewerElements,
         [=](const LegalityQuery &Query) {
           LLT VecTy = Query.Types[TypeIdx];
-          return VecTy.isVector() && VecTy.getElementType() == EltTy &&
+          return VecTy.isFixedVector() && VecTy.getElementType() == EltTy &&
                  VecTy.getNumElements() > MaxElements;
         },
         [=](const LegalityQuery &Query) {
@@ -1231,6 +1230,9 @@ public:
     assert(MinTy.getElementType() == MaxTy.getElementType() &&
            "Expected element types to agree");
 
+    assert((!MinTy.isScalableVector() && !MaxTy.isScalableVector()) &&
+           "Unexpected scalable vectors");
+
     const LLT EltTy = MinTy.getElementType();
     return clampMinNumElements(TypeIdx, EltTy, MinTy.getNumElements())
         .clampMaxNumElements(TypeIdx, EltTy, MaxTy.getNumElements());
diff --git a/llvm/lib/Target/AArch64/AArch64GenRegisterBankInfo.def b/llvm/lib/Target/AArch64/AArch64GenRegisterBankInfo.def
index 82066b48c84b..8ff59f60968b 100644
--- a/llvm/lib/Target/AArch64/AArch64GenRegisterBankInfo.def
+++ b/llvm/lib/Target/AArch64/AArch64GenRegisterBankInfo.def
@@ -183,6 +183,8 @@ unsigned AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(unsigned RBIdx,
     const unsigned MinSize = Size.getKnownMinValue();
     assert((!Size.isScalable() || MinSize >= 128) &&
            "Scalable vector types should have size of at least 128 bits");
+    if (Size.isScalable())
+      return 3;
     if (MinSize <= 16)
       return 0;
     if (MinSize <= 32)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
index 5aee7804de3e..6cbfb018b318 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@@ -393,8 +393,8 @@ bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
       // i1 is a special case because SDAG i1 true is naturally zero extended
       // when widened using ANYEXT. We need to do it explicitly here.
       auto &Flags = CurArgInfo.Flags[0];
-      if (MRI.getType(CurVReg).getSizeInBits() == 1 && !Flags.isSExt() &&
-          !Flags.isZExt()) {
+      if (MRI.getType(CurVReg).getSizeInBits() == TypeSize::getFixed(1) &&
+          !Flags.isSExt() && !Flags.isZExt()) {
         CurVReg = MIRBuilder.buildZExt(LLT::scalar(8), CurVReg).getReg(0);
       } else if (TLI.getNumRegistersForCallingConv(Ctx, CC, SplitEVTs[i]) ==
                  1) {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index df0c09d32c07..afea08ab0925 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -615,6 +615,7 @@ getMinClassForRegBank(const RegisterBank &RB, TypeSize SizeInBits,
   unsigned RegBankID = RB.getID();
 
   if (RegBankID == AArch64::GPRRegBankID) {
+    assert(!SizeInBits.isScalable() && "Unexpected scalable register size");
     if (SizeInBits <= 32)
       return GetAllRegSet ? &AArch64::GPR32allRegClass
                           : &AArch64::GPR32RegClass;
@@ -626,6 +627,12 @@ getMinClassForRegBank(const RegisterBank &RB, TypeSize SizeInBits,
   }
 
   if (RegBankID == AArch64::FPRRegBankID) {
+    if (SizeInBits.isScalable()) {
+      assert(SizeInBits == TypeSize::getScalable(128) &&
+             "Unexpected scalable register size");
+      return &AArch64::ZPRRegClass;
+    }
+
     switch (SizeInBits) {
     default:
       return nullptr;
@@ -964,7 +971,8 @@ getRegClassesForCopy(MachineInstr &I, const TargetInstrInfo &TII,
   // then we can pull it into the helpers that get the appropriate class for a
   // register bank. Or make a new helper that carries along some constraint
   // information.
-  if (SrcRegBank != DstRegBank && (DstSize == 1 && SrcSize == 1))
+  if (SrcRegBank != DstRegBank &&
+      (DstSize == TypeSize::getFixed(1) && SrcSize == TypeSize::getFixed(1)))
     SrcSize = DstSize = TypeSize::getFixed(32);
 
   return {getMinClassForRegBank(SrcRegBank, SrcSize, true),
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 5cd1fea75025..dd65dbe594a6 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -91,6 +91,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
 
   const bool HasCSSC = ST.hasCSSC();
   const bool HasRCPC3 = ST.hasRCPC3();
+  const bool HasSVE = ST.hasSVE();
 
   getActionDefinitionsBuilder(
       {G_IMPLICIT_DEF, G_FREEZE, G_CONSTANT_FOLD_BARRIER})
@@ -127,7 +128,34 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .clampNumElements(0, v2s64, v2s64)
       .moreElementsToNextPow2(0);
 
-  getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR})
+  getActionDefinitionsBuilder({G_ADD, G_SUB, G_AND, G_OR, G_XOR})
+      .legalFor({s32, s64, v2s32, v2s64, v4s32, v4s16, v8s16, v16s8, v8s8})
+      .legalFor(HasSVE, {nxv16s8, nxv8s16, nxv4s32, nxv2s64})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, s32, s64)
+      .clampMaxNumElements(0, s8, 16)
+      .clampMaxNumElements(0, s16, 8)
+      .clampNumElements(0, v2s32, v4s32)
+      .clampNumElements(0, v2s64, v2s64)
+      .minScalarOrEltIf(
+          [=](const LegalityQuery &Query) {
+            return Query.Types[0].getNumElements() <= 2;
+          },
+          0, s32)
+      .minScalarOrEltIf(
+          [=](const LegalityQuery &Query) {
+            return Query.Types[0].getNumElements() <= 4;
+          },
+          0, s16)
+      .minScalarOrEltIf(
+          [=](const LegalityQuery &Query) {
+            return Query.Types[0].getNumElements() <= 16;
+          },
+          0, s8)
+      .scalarizeIf(scalarOrEltWiderThan(0, 64), 0)
+      .moreElementsToNextPow2(0);
+
+  getActionDefinitionsBuilder(G_MUL)
       .legalFor({s32, s64, v2s32, v2s64, v4s32, v4s16, v8s16, v16s8, v8s8})
       .widenScalarToNextPow2(0)
       .clampScalar(0, s32, s64)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 80b6e4f6d528..0af60a503c5f 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -24,7 +24,6 @@
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 #
 # DEBUG-NEXT: G_MUL (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 #
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/sve-integer.ll b/llvm/test/CodeGen/AArch64/GlobalISel/sve-integer.ll
new file mode 100644
index 000000000000..bc51cf7bac23
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/sve-integer.ll
@@ -0,0 +1,208 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc < %s -mtriple aarch64 -mattr=+sve -aarch64-enable-gisel-sve=1  | FileCheck %s
+; RUN: llc < %s -mtriple aarch64 -mattr=+sve -global-isel -aarch64-enable-gisel-sve=1 | FileCheck %s
+
+;; add
+define <vscale x 2 x i64> @addnxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: addnxv2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    add z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %c = add <vscale x 2 x i64> %a, %b
+  ret <vscale x 2 x i64> %c
+}
+
+define <vscale x 4 x i32> @addnxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: addnxv4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    add z0.s, z0.s, z1.s
+; CHECK-NEXT:    ret
+entry:
+  %c = add <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %c
+}
+
+define <vscale x 8 x i16> @addnxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, ptr %p) {
+; CHECK-LABEL: addnxv8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    add z0.h, z0.h, z1.h
+; CHECK-NEXT:    ret
+entry:
+  %c = add <vscale x 8 x i16> %a, %b
+  ret <vscale x 8 x i16> %c
+}
+
+define <vscale x 16 x i8> @addnxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: addnxv16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    add z0.b, z0.b, z1.b
+; CHECK-NEXT:    ret
+entry:
+  %c = add <vscale x 16 x i8> %a, %b
+  ret <vscale x 16 x i8> %c
+}
+
+;; sub
+define <vscale x 2 x i64> @subnxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: subnxv2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %c = sub <vscale x 2 x i64> %a, %b
+  ret <vscale x 2 x i64> %c
+}
+
+define <vscale x 4 x i32> @subnxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: subnxv4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub z0.s, z0.s, z1.s
+; CHECK-NEXT:    ret
+entry:
+  %c = sub <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %c
+}
+
+define <vscale x 8 x i16> @subnxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, ptr %p) {
+; CHECK-LABEL: subnxv8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub z0.h, z0.h, z1.h
+; CHECK-NEXT:    ret
+entry:
+  %c = sub <vscale x 8 x i16> %a, %b
+  ret <vscale x 8 x i16> %c
+}
+
+define <vscale x 16 x i8> @subnxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: subnxv16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub z0.b, z0.b, z1.b
+; CHECK-NEXT:    ret
+entry:
+  %c = sub <vscale x 16 x i8> %a, %b
+  ret <vscale x 16 x i8> %c
+}
+
+;; and
+define <vscale x 2 x i64> @andnxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: andnxv2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %c = and <vscale x 2 x i64> %a, %b
+  ret <vscale x 2 x i64> %c
+}
+
+define <vscale x 4 x i32> @andnxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: andnxv4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %c = and <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %c
+}
+
+define <vscale x 8 x i16> @andnxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, ptr %p) {
+; CHECK-LABEL: andnxv8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %c = and <vscale x 8 x i16> %a, %b
+  ret <vscale x 8 x i16> %c
+}
+
+define <vscale x 16 x i8> @andnxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: andnxv16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %c = and <vscale x 16 x i8> %a, %b
+  ret <vscale x 16 x i8> %c
+}
+
+;; or
+define <vscale x 2 x i64> @ornxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: ornxv2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %c = or <vscale x 2 x i64> %a, %b
+  ret <vscale x 2 x i64> %c
+}
+
+define <vscale x 4 x i32> @ornxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: ornxv4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %c = or <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %c
+}
+
+define <vscale x 8 x i16> @ornxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, ptr %p) {
+; CHECK-LABEL: ornxv8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %c = or <vscale x 8 x i16> %a, %b
+  ret <vscale x 8 x i16> %c
+}
+
+define <vscale x 16 x i8> @ornxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: ornxv16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %c = or <vscale x 16 x i8> %a, %b
+  ret <vscale x 16 x i8> %c
+}
+
+;; xor
+define <vscale x 2 x i64> @xornxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: xornxv2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    eor z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %c = xor <vscale x 2 x i64> %a, %b
+  ret <vscale x 2 x i64> %c
+}
+
+define <vscale x 4 x i32> @xornxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: xornxv4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    eor z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %c = xor <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %c
+}
+
+define <vscale x 8 x i16> @xornxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, ptr %p) {
+; CHECK-LABEL: xornxv8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    eor z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %c = xor <vscale x 8 x i16> %a, %b
+  ret <vscale x 8 x i16> %c
+}
+
+define <vscale x 16 x i8> @xornxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: xornxv16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    eor z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %c = xor <vscale x 16 x i8> %a, %b
+  ret <vscale x 16 x i8> %c
+}
-- 
GitLab


From fb33af08e4c105a05855f8beeb972d493410e72f Mon Sep 17 00:00:00 2001
From: Alex MacLean <amaclean@nvidia.com>
Date: Sun, 27 Oct 2024 16:14:13 -0700
Subject: [PATCH 268/693] [NVPTX] Remove nvvm.ldg.global.* intrinsics (#112834)

Remove these intrinsics which can be better represented by load
instructions with `!invariant.load` metadata:

- llvm.nvvm.ldg.global.i
- llvm.nvvm.ldg.global.f
- llvm.nvvm.ldg.global.p
---
 clang/lib/CodeGen/CGBuiltin.cpp               |  47 +++--
 .../builtins-nvptx-native-half-type-native.c  |   4 +-
 .../CodeGen/builtins-nvptx-native-half-type.c |   4 +-
 clang/test/CodeGen/builtins-nvptx.c           |  72 +++----
 llvm/docs/ReleaseNotes.md                     |   9 +-
 llvm/include/llvm/IR/IntrinsicsNVVM.td        |  18 +-
 llvm/lib/IR/AutoUpgrade.cpp                   |  14 ++
 llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp   | 189 +++++++-----------
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp   |  55 +----
 llvm/lib/Target/NVPTX/NVPTXISelLowering.h     |   2 -
 .../Assembler/auto_upgrade_nvvm_intrinsics.ll |  31 +++
 11 files changed, 197 insertions(+), 248 deletions(-)

diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index e2d03eff8ab4..911eec48bcb2 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -20492,8 +20492,8 @@ static NVPTXMmaInfo getNVPTXMmaInfo(unsigned BuiltinID) {
 #undef MMA_VARIANTS_B1_XOR
 }
 
-static Value *MakeLdgLdu(unsigned IntrinsicID, CodeGenFunction &CGF,
-                         const CallExpr *E) {
+static Value *MakeLdu(unsigned IntrinsicID, CodeGenFunction &CGF,
+                      const CallExpr *E) {
   Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
   QualType ArgType = E->getArg(0)->getType();
   clang::CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment(ArgType);
@@ -20503,6 +20503,21 @@ static Value *MakeLdgLdu(unsigned IntrinsicID, CodeGenFunction &CGF,
       {Ptr, ConstantInt::get(CGF.Builder.getInt32Ty(), Align.getQuantity())});
 }
 
+static Value *MakeLdg(CodeGenFunction &CGF, const CallExpr *E) {
+  Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
+  QualType ArgType = E->getArg(0)->getType();
+  clang::CharUnits AlignV = CGF.CGM.getNaturalPointeeTypeAlignment(ArgType);
+  llvm::Type *ElemTy = CGF.ConvertTypeForMem(ArgType->getPointeeType());
+
+  // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
+  auto *ASC = CGF.Builder.CreateAddrSpaceCast(Ptr, CGF.Builder.getPtrTy(1));
+  auto *LD = CGF.Builder.CreateAlignedLoad(ElemTy, ASC, AlignV.getAsAlign());
+  MDNode *MD = MDNode::get(CGF.Builder.getContext(), {});
+  LD->setMetadata(LLVMContext::MD_invariant_load, MD);
+
+  return LD;
+}
+
 static Value *MakeScopedAtomic(unsigned IntrinsicID, CodeGenFunction &CGF,
                                const CallExpr *E) {
   Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
@@ -20536,9 +20551,11 @@ static Value *MakeHalfType(unsigned IntrinsicID, unsigned BuiltinID,
     return nullptr;
   }
 
-  if (IntrinsicID == Intrinsic::nvvm_ldg_global_f ||
-      IntrinsicID == Intrinsic::nvvm_ldu_global_f)
-    return MakeLdgLdu(IntrinsicID, CGF, E);
+  if (BuiltinID == NVPTX::BI__nvvm_ldg_h || BuiltinID == NVPTX::BI__nvvm_ldg_h2)
+    return MakeLdg(CGF, E);
+
+  if (IntrinsicID == Intrinsic::nvvm_ldu_global_f)
+    return MakeLdu(IntrinsicID, CGF, E);
 
   SmallVector<Value *, 16> Args;
   auto *F = CGF.CGM.getIntrinsic(IntrinsicID);
@@ -20675,16 +20692,15 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
   case NVPTX::BI__nvvm_ldg_ul2:
   case NVPTX::BI__nvvm_ldg_ull:
   case NVPTX::BI__nvvm_ldg_ull2:
-    // PTX Interoperability section 2.2: "For a vector with an even number of
-    // elements, its alignment is set to number of elements times the alignment
-    // of its member: n*alignof(t)."
-    return MakeLdgLdu(Intrinsic::nvvm_ldg_global_i, *this, E);
   case NVPTX::BI__nvvm_ldg_f:
   case NVPTX::BI__nvvm_ldg_f2:
   case NVPTX::BI__nvvm_ldg_f4:
   case NVPTX::BI__nvvm_ldg_d:
   case NVPTX::BI__nvvm_ldg_d2:
-    return MakeLdgLdu(Intrinsic::nvvm_ldg_global_f, *this, E);
+    // PTX Interoperability section 2.2: "For a vector with an even number of
+    // elements, its alignment is set to number of elements times the alignment
+    // of its member: n*alignof(t)."
+    return MakeLdg(*this, E);
 
   case NVPTX::BI__nvvm_ldu_c:
   case NVPTX::BI__nvvm_ldu_sc:
@@ -20715,13 +20731,13 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
   case NVPTX::BI__nvvm_ldu_ul2:
   case NVPTX::BI__nvvm_ldu_ull:
   case NVPTX::BI__nvvm_ldu_ull2:
-    return MakeLdgLdu(Intrinsic::nvvm_ldu_global_i, *this, E);
+    return MakeLdu(Intrinsic::nvvm_ldu_global_i, *this, E);
   case NVPTX::BI__nvvm_ldu_f:
   case NVPTX::BI__nvvm_ldu_f2:
   case NVPTX::BI__nvvm_ldu_f4:
   case NVPTX::BI__nvvm_ldu_d:
   case NVPTX::BI__nvvm_ldu_d2:
-    return MakeLdgLdu(Intrinsic::nvvm_ldu_global_f, *this, E);
+    return MakeLdu(Intrinsic::nvvm_ldu_global_f, *this, E);
 
   case NVPTX::BI__nvvm_atom_cta_add_gen_i:
   case NVPTX::BI__nvvm_atom_cta_add_gen_l:
@@ -21195,14 +21211,11 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
     return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID, E,
                         *this);
   case NVPTX::BI__nvvm_ldg_h:
-    return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID, E, *this);
   case NVPTX::BI__nvvm_ldg_h2:
-    return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID, E, *this);
+    return MakeHalfType(Intrinsic::not_intrinsic, BuiltinID, E, *this);
   case NVPTX::BI__nvvm_ldu_h:
+  case NVPTX::BI__nvvm_ldu_h2:
     return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID, E, *this);
-  case NVPTX::BI__nvvm_ldu_h2: {
-    return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID, E, *this);
-  }
   case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
     return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_4,
                        Intrinsic::nvvm_cp_async_ca_shared_global_4_s, *this, E,
diff --git a/clang/test/CodeGen/builtins-nvptx-native-half-type-native.c b/clang/test/CodeGen/builtins-nvptx-native-half-type-native.c
index b594fc876d4b..035c4c6066be 100644
--- a/clang/test/CodeGen/builtins-nvptx-native-half-type-native.c
+++ b/clang/test/CodeGen/builtins-nvptx-native-half-type-native.c
@@ -52,8 +52,8 @@ typedef __fp16 __fp16v2 __attribute__((ext_vector_type(2)));
 // CHECK: call <2 x half> @llvm.nvvm.fmax.ftz.xorsign.abs.f16x2(<2 x half> {{.*}}, <2 x half> {{.*}})
 // CHECK: call <2 x half> @llvm.nvvm.fmax.nan.xorsign.abs.f16x2(<2 x half> {{.*}}, <2 x half> {{.*}})
 // CHECK: call <2 x half> @llvm.nvvm.fmax.ftz.nan.xorsign.abs.f16x2(<2 x half> {{.*}}, <2 x half> {{.*}})
-// CHECK: call half @llvm.nvvm.ldg.global.f.f16.p0(ptr {{.*}}, i32 2)
-// CHECK: call <2 x half> @llvm.nvvm.ldg.global.f.v2f16.p0(ptr {{.*}}, i32 4)
+// CHECK: load half, ptr addrspace(1) {{.*}}, align 2, !invariant.load
+// CHECK: load <2 x half>, ptr addrspace(1) {{.*}}, align 4, !invariant.load
 // CHECK: call half @llvm.nvvm.ldu.global.f.f16.p0(ptr {{.*}}, i32 2)
 // CHECK: call <2 x half> @llvm.nvvm.ldu.global.f.v2f16.p0(ptr {{.*}}, i32 4)
 __device__ void nvvm_native_half_types(void *a, void*b, void*c, __fp16* out) {
diff --git a/clang/test/CodeGen/builtins-nvptx-native-half-type.c b/clang/test/CodeGen/builtins-nvptx-native-half-type.c
index 4aeae953bc16..511497702ff7 100644
--- a/clang/test/CodeGen/builtins-nvptx-native-half-type.c
+++ b/clang/test/CodeGen/builtins-nvptx-native-half-type.c
@@ -177,9 +177,9 @@ typedef __fp16 __fp16v2 __attribute__((ext_vector_type(2)));
 
 // CHECK-LABEL: nvvm_ldg_native_half_types
 __device__ void nvvm_ldg_native_half_types(const void *p) {
-  // CHECK: call half @llvm.nvvm.ldg.global.f.f16.p0
+  // CHECK: load half, ptr addrspace(1) {{.*}}, align 2, !invariant.load
   __nvvm_ldg_h((const __fp16 *)p);
-  // CHECK: call <2 x half> @llvm.nvvm.ldg.global.f.v2f16.p0
+  // CHECK: load <2 x half>, ptr addrspace(1) {{.*}}, align 4, !invariant.load
   __nvvm_ldg_h2((const __fp16v2 *)p);
 }
 
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index 0d0e3ecdb90c..3406cbdde2bf 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -598,33 +598,33 @@ __device__ void nvvm_atom(float *fp, float f, double *dfp, double df,
 
 // CHECK-LABEL: nvvm_ldg
 __device__ void nvvm_ldg(const void *p) {
-  // CHECK: call i8 @llvm.nvvm.ldg.global.i.i8.p0(ptr {{%[0-9]+}}, i32 1)
-  // CHECK: call i8 @llvm.nvvm.ldg.global.i.i8.p0(ptr {{%[0-9]+}}, i32 1)
-  // CHECK: call i8 @llvm.nvvm.ldg.global.i.i8.p0(ptr {{%[0-9]+}}, i32 1)
+  // CHECK: load i8, ptr addrspace(1) {{%[0-9]+}}, align 1, !invariant.load
+  // CHECK: load i8, ptr addrspace(1) {{%[0-9]+}}, align 1, !invariant.load
+  // CHECK: load i8, ptr addrspace(1) {{%[0-9]+}}, align 1, !invariant.load
   __nvvm_ldg_c((const char *)p);
   __nvvm_ldg_uc((const unsigned char *)p);
   __nvvm_ldg_sc((const signed char *)p);
 
-  // CHECK: call i16 @llvm.nvvm.ldg.global.i.i16.p0(ptr {{%[0-9]+}}, i32 2)
-  // CHECK: call i16 @llvm.nvvm.ldg.global.i.i16.p0(ptr {{%[0-9]+}}, i32 2)
+  // CHECK: load i16, ptr addrspace(1) {{%[0-9]+}}, align 2, !invariant.load
+  // CHECK: load i16, ptr addrspace(1) {{%[0-9]+}}, align 2, !invariant.load
   __nvvm_ldg_s((const short *)p);
   __nvvm_ldg_us((const unsigned short *)p);
 
-  // CHECK: call i32 @llvm.nvvm.ldg.global.i.i32.p0(ptr {{%[0-9]+}}, i32 4)
-  // CHECK: call i32 @llvm.nvvm.ldg.global.i.i32.p0(ptr {{%[0-9]+}}, i32 4)
+  // CHECK: load i32, ptr addrspace(1) {{%[0-9]+}}, align 4, !invariant.load
+  // CHECK: load i32, ptr addrspace(1) {{%[0-9]+}}, align 4, !invariant.load
   __nvvm_ldg_i((const int *)p);
   __nvvm_ldg_ui((const unsigned int *)p);
 
-  // LP32: call i32 @llvm.nvvm.ldg.global.i.i32.p0(ptr {{%[0-9]+}}, i32 4)
-  // LP32: call i32 @llvm.nvvm.ldg.global.i.i32.p0(ptr {{%[0-9]+}}, i32 4)
-  // LP64: call i64 @llvm.nvvm.ldg.global.i.i64.p0(ptr {{%[0-9]+}}, i32 8)
-  // LP64: call i64 @llvm.nvvm.ldg.global.i.i64.p0(ptr {{%[0-9]+}}, i32 8)
+  // LP32: load i32, ptr addrspace(1) {{%[0-9]+}}, align 4, !invariant.load
+  // LP32: load i32, ptr addrspace(1) {{%[0-9]+}}, align 4, !invariant.load
+  // LP64: load i64, ptr addrspace(1) {{%[0-9]+}}, align 8, !invariant.load
+  // LP64: load i64, ptr addrspace(1) {{%[0-9]+}}, align 8, !invariant.load
   __nvvm_ldg_l((const long *)p);
   __nvvm_ldg_ul((const unsigned long *)p);
 
-  // CHECK: call float @llvm.nvvm.ldg.global.f.f32.p0(ptr {{%[0-9]+}}, i32 4)
+  // CHECK: load float, ptr addrspace(1) {{%[0-9]+}}, align 4, !invariant.load
   __nvvm_ldg_f((const float *)p);
-  // CHECK: call double @llvm.nvvm.ldg.global.f.f64.p0(ptr {{%[0-9]+}}, i32 8)
+  // CHECK: load double, ptr addrspace(1) {{%[0-9]+}}, align 8, !invariant.load
   __nvvm_ldg_d((const double *)p);
 
   // In practice, the pointers we pass to __ldg will be aligned as appropriate
@@ -636,9 +636,9 @@ __device__ void nvvm_ldg(const void *p) {
   // elements, its alignment is set to number of elements times the alignment of
   // its member: n*alignof(t)."
 
-  // CHECK: call <2 x i8> @llvm.nvvm.ldg.global.i.v2i8.p0(ptr {{%[0-9]+}}, i32 2)
-  // CHECK: call <2 x i8> @llvm.nvvm.ldg.global.i.v2i8.p0(ptr {{%[0-9]+}}, i32 2)
-  // CHECK: call <2 x i8> @llvm.nvvm.ldg.global.i.v2i8.p0(ptr {{%[0-9]+}}, i32 2)
+  // CHECK: load <2 x i8>, ptr addrspace(1) {{%[0-9]+}}, align 2, !invariant.load
+  // CHECK: load <2 x i8>, ptr addrspace(1) {{%[0-9]+}}, align 2, !invariant.load
+  // CHECK: load <2 x i8>, ptr addrspace(1) {{%[0-9]+}}, align 2, !invariant.load
   typedef char char2 __attribute__((ext_vector_type(2)));
   typedef unsigned char uchar2 __attribute__((ext_vector_type(2)));
   typedef signed char schar2 __attribute__((ext_vector_type(2)));
@@ -646,9 +646,9 @@ __device__ void nvvm_ldg(const void *p) {
   __nvvm_ldg_uc2((const uchar2 *)p);
   __nvvm_ldg_sc2((const schar2 *)p);
 
-  // CHECK: call <4 x i8> @llvm.nvvm.ldg.global.i.v4i8.p0(ptr {{%[0-9]+}}, i32 4)
-  // CHECK: call <4 x i8> @llvm.nvvm.ldg.global.i.v4i8.p0(ptr {{%[0-9]+}}, i32 4)
-  // CHECK: call <4 x i8> @llvm.nvvm.ldg.global.i.v4i8.p0(ptr {{%[0-9]+}}, i32 4)
+  // CHECK: load <4 x i8>, ptr addrspace(1) {{%[0-9]+}}, align 4, !invariant.load
+  // CHECK: load <4 x i8>, ptr addrspace(1) {{%[0-9]+}}, align 4, !invariant.load
+  // CHECK: load <4 x i8>, ptr addrspace(1) {{%[0-9]+}}, align 4, !invariant.load
   typedef char char4 __attribute__((ext_vector_type(4)));
   typedef unsigned char uchar4 __attribute__((ext_vector_type(4)));
   typedef signed char schar4 __attribute__((ext_vector_type(4)));
@@ -656,59 +656,59 @@ __device__ void nvvm_ldg(const void *p) {
   __nvvm_ldg_uc4((const uchar4 *)p);
   __nvvm_ldg_sc4((const schar4 *)p);
 
-  // CHECK: call <2 x i16> @llvm.nvvm.ldg.global.i.v2i16.p0(ptr {{%[0-9]+}}, i32 4)
-  // CHECK: call <2 x i16> @llvm.nvvm.ldg.global.i.v2i16.p0(ptr {{%[0-9]+}}, i32 4)
+  // CHECK: load <2 x i16>, ptr addrspace(1) {{%[0-9]+}}, align 4, !invariant.load
+  // CHECK: load <2 x i16>, ptr addrspace(1) {{%[0-9]+}}, align 4, !invariant.load
   typedef short short2 __attribute__((ext_vector_type(2)));
   typedef unsigned short ushort2 __attribute__((ext_vector_type(2)));
   __nvvm_ldg_s2((const short2 *)p);
   __nvvm_ldg_us2((const ushort2 *)p);
 
-  // CHECK: call <4 x i16> @llvm.nvvm.ldg.global.i.v4i16.p0(ptr {{%[0-9]+}}, i32 8)
-  // CHECK: call <4 x i16> @llvm.nvvm.ldg.global.i.v4i16.p0(ptr {{%[0-9]+}}, i32 8)
+  // CHECK: load <4 x i16>, ptr addrspace(1) {{%[0-9]+}}, align 8, !invariant.load
+  // CHECK: load <4 x i16>, ptr addrspace(1) {{%[0-9]+}}, align 8, !invariant.load
   typedef short short4 __attribute__((ext_vector_type(4)));
   typedef unsigned short ushort4 __attribute__((ext_vector_type(4)));
   __nvvm_ldg_s4((const short4 *)p);
   __nvvm_ldg_us4((const ushort4 *)p);
 
-  // CHECK: call <2 x i32> @llvm.nvvm.ldg.global.i.v2i32.p0(ptr {{%[0-9]+}}, i32 8)
-  // CHECK: call <2 x i32> @llvm.nvvm.ldg.global.i.v2i32.p0(ptr {{%[0-9]+}}, i32 8)
+  // CHECK: load <2 x i32>, ptr addrspace(1) {{%[0-9]+}}, align 8, !invariant.load
+  // CHECK: load <2 x i32>, ptr addrspace(1) {{%[0-9]+}}, align 8, !invariant.load
   typedef int int2 __attribute__((ext_vector_type(2)));
   typedef unsigned int uint2 __attribute__((ext_vector_type(2)));
   __nvvm_ldg_i2((const int2 *)p);
   __nvvm_ldg_ui2((const uint2 *)p);
 
-  // CHECK: call <4 x i32> @llvm.nvvm.ldg.global.i.v4i32.p0(ptr {{%[0-9]+}}, i32 16)
-  // CHECK: call <4 x i32> @llvm.nvvm.ldg.global.i.v4i32.p0(ptr {{%[0-9]+}}, i32 16)
+  // CHECK: load <4 x i32>, ptr addrspace(1) {{%[0-9]+}}, align 16, !invariant.load
+  // CHECK: load <4 x i32>, ptr addrspace(1) {{%[0-9]+}}, align 16, !invariant.load
   typedef int int4 __attribute__((ext_vector_type(4)));
   typedef unsigned int uint4 __attribute__((ext_vector_type(4)));
   __nvvm_ldg_i4((const int4 *)p);
   __nvvm_ldg_ui4((const uint4 *)p);
 
-  // LP32: call <2 x i32> @llvm.nvvm.ldg.global.i.v2i32.p0(ptr {{%[0-9]+}}, i32 8)
-  // LP32: call <2 x i32> @llvm.nvvm.ldg.global.i.v2i32.p0(ptr {{%[0-9]+}}, i32 8)
-  // LP64: call <2 x i64> @llvm.nvvm.ldg.global.i.v2i64.p0(ptr {{%[0-9]+}}, i32 16)
-  // LP64: call <2 x i64> @llvm.nvvm.ldg.global.i.v2i64.p0(ptr {{%[0-9]+}}, i32 16)
+  // LP32: load <2 x i32>, ptr addrspace(1) {{%[0-9]+}}, align 8, !invariant.load
+  // LP32: load <2 x i32>, ptr addrspace(1) {{%[0-9]+}}, align 8, !invariant.load
+  // LP64: load <2 x i64>, ptr addrspace(1) {{%[0-9]+}}, align 16, !invariant.load
+  // LP64: load <2 x i64>, ptr addrspace(1) {{%[0-9]+}}, align 16, !invariant.load
   typedef long long2 __attribute__((ext_vector_type(2)));
   typedef unsigned long ulong2 __attribute__((ext_vector_type(2)));
   __nvvm_ldg_l2((const long2 *)p);
   __nvvm_ldg_ul2((const ulong2 *)p);
 
-  // CHECK: call <2 x i64> @llvm.nvvm.ldg.global.i.v2i64.p0(ptr {{%[0-9]+}}, i32 16)
-  // CHECK: call <2 x i64> @llvm.nvvm.ldg.global.i.v2i64.p0(ptr {{%[0-9]+}}, i32 16)
+  // CHECK: load <2 x i64>, ptr addrspace(1) {{%[0-9]+}}, align 16, !invariant.load
+  // CHECK: load <2 x i64>, ptr addrspace(1) {{%[0-9]+}}, align 16, !invariant.load
   typedef long long longlong2 __attribute__((ext_vector_type(2)));
   typedef unsigned long long ulonglong2 __attribute__((ext_vector_type(2)));
   __nvvm_ldg_ll2((const longlong2 *)p);
   __nvvm_ldg_ull2((const ulonglong2 *)p);
 
-  // CHECK: call <2 x float> @llvm.nvvm.ldg.global.f.v2f32.p0(ptr {{%[0-9]+}}, i32 8)
+  // CHECK: load <2 x float>, ptr addrspace(1) {{%[0-9]+}}, align 8, !invariant.load
   typedef float float2 __attribute__((ext_vector_type(2)));
   __nvvm_ldg_f2((const float2 *)p);
 
-  // CHECK: call <4 x float> @llvm.nvvm.ldg.global.f.v4f32.p0(ptr {{%[0-9]+}}, i32 16)
+  // CHECK: load <4 x float>, ptr addrspace(1) {{%[0-9]+}}, align 16, !invariant.load
   typedef float float4 __attribute__((ext_vector_type(4)));
   __nvvm_ldg_f4((const float4 *)p);
 
-  // CHECK: call <2 x double> @llvm.nvvm.ldg.global.f.v2f64.p0(ptr {{%[0-9]+}}, i32 16)
+  // CHECK: load <2 x double>, ptr addrspace(1) {{%[0-9]+}}, align 16, !invariant.load
   typedef double double2 __attribute__((ext_vector_type(2)));
   __nvvm_ldg_d2((const double2 *)p);
 }
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index e3d93f0dfd0e..3cac3e57344d 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -88,7 +88,14 @@ Changes to the LLVM IR
   * `llvm.nvvm.ptr.shared.to.gen`
   * `llvm.nvvm.ptr.constant.to.gen`
   * `llvm.nvvm.ptr.local.to.gen`
-  
+
+* Remove the following intrinsics which can be relaced with a load from
+  addrspace(1) with an !invariant.load metadata
+
+  * `llvm.nvvm.ldg.global.i`
+  * `llvm.nvvm.ldg.global.f`
+  * `llvm.nvvm.ldg.global.p`
+
 * Operand bundle values can now be metadata strings.
 
 Changes to LLVM infrastructure
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index b4a06f583f2c..5164f873d00f 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -42,6 +42,9 @@
 //   * llvm.nvvm.ptr.shared.to.gen   --> ibid.
 //   * llvm.nvvm.ptr.constant.to.gen --> ibid.
 //   * llvm.nvvm.ptr.local.to.gen    --> ibid.
+//   * llvm.nvvm.ldg.global.i        --> load addrspace(1) !load.invariant
+//   * llvm.nvvm.ldg.global.f        --> ibid.
+//   * llvm.nvvm.ldg.global.p        --> ibid.
 
 def llvm_global_ptr_ty  : LLVMQualPointerType<1>;  // (global)ptr
 def llvm_shared_ptr_ty  : LLVMQualPointerType<3>;  // (shared)ptr
@@ -1605,21 +1608,6 @@ def int_nvvm_ldu_global_p : Intrinsic<[llvm_anyptr_ty],
   [IntrReadMem, IntrArgMemOnly, IntrNoCallback, IntrWillReturn, NoCapture<ArgIndex<0>>],
   "llvm.nvvm.ldu.global.p">;
 
-// Generated within nvvm. Use for ldg on sm_35 or later.  Second arg is the
-// pointer's alignment.
-def int_nvvm_ldg_global_i : Intrinsic<[llvm_anyint_ty],
-  [llvm_anyptr_ty, llvm_i32_ty],
-  [IntrReadMem, IntrArgMemOnly, IntrNoCallback, IntrWillReturn, NoCapture<ArgIndex<0>>],
-  "llvm.nvvm.ldg.global.i">;
-def int_nvvm_ldg_global_f : Intrinsic<[llvm_anyfloat_ty],
-  [llvm_anyptr_ty, llvm_i32_ty],
-  [IntrReadMem, IntrArgMemOnly, IntrNoCallback, IntrWillReturn, NoCapture<ArgIndex<0>>],
-  "llvm.nvvm.ldg.global.f">;
-def int_nvvm_ldg_global_p : Intrinsic<[llvm_anyptr_ty],
-  [llvm_anyptr_ty, llvm_i32_ty],
-  [IntrReadMem, IntrArgMemOnly, IntrNoCallback, IntrWillReturn, NoCapture<ArgIndex<0>>],
-  "llvm.nvvm.ldg.global.p">;
-
 // Used in nvvm internally to help address space opt and ptx code generation
 // This is for params that are passed to kernel functions by pointer by-val.
 def int_nvvm_ptr_gen_to_param: Intrinsic<[llvm_anyptr_ty],
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index bb03c9290e4c..73882fbc7a25 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -37,6 +37,7 @@
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/Value.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/Support/AMDGPUAddrSpace.h"
 #include "llvm/Support/CommandLine.h"
@@ -1301,6 +1302,10 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
             (Name.consume_front("local") || Name.consume_front("shared") ||
              Name.consume_front("global") || Name.consume_front("constant")) &&
             Name.starts_with(".to.gen");
+      else if (Name.consume_front("ldg.global."))
+        // nvvm.ldg.global.{i,p,f}
+        Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
+                  Name.starts_with("p."));
       else
         Expand = false;
 
@@ -2363,6 +2368,15 @@ static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
                Name.consume_front("constant")) &&
               Name.starts_with(".to.gen"))) {
     Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
+  } else if (Name.consume_front("ldg.global")) {
+    Value *Ptr = CI->getArgOperand(0);
+    Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
+    // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
+    Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
+    Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
+    MDNode *MD = MDNode::get(Builder.getContext(), {});
+    LD->setMetadata(LLVMContext::MD_invariant_load, MD);
+    return LD;
   } else {
     Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
     if (IID != Intrinsic::not_intrinsic &&
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 93c2d92ef7c1..965ed98630a2 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -126,8 +126,6 @@ void NVPTXDAGToDAGISel::Select(SDNode *N) {
     if (tryLoadVector(N))
       return;
     break;
-  case NVPTXISD::LDGV2:
-  case NVPTXISD::LDGV4:
   case NVPTXISD::LDUV2:
   case NVPTXISD::LDUV4:
     if (tryLDGLDU(N))
@@ -550,9 +548,6 @@ bool NVPTXDAGToDAGISel::tryIntrinsicChain(SDNode *N) {
   switch (IID) {
   default:
     return false;
-  case Intrinsic::nvvm_ldg_global_f:
-  case Intrinsic::nvvm_ldg_global_i:
-  case Intrinsic::nvvm_ldg_global_p:
   case Intrinsic::nvvm_ldu_global_f:
   case Intrinsic::nvvm_ldu_global_i:
   case Intrinsic::nvvm_ldu_global_p:
@@ -1559,34 +1554,11 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
 }
 
 bool NVPTXDAGToDAGISel::tryLDGLDU(SDNode *N) {
-  SDValue Op1;
-  MemSDNode *Mem;
-  bool IsLDG = true;
+  auto *Mem = cast<MemSDNode>(N);
 
   // If this is an LDG intrinsic, the address is the third operand. If its an
   // LDG/LDU SD node (from custom vector handling), then its the second operand
-  if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
-    Op1 = N->getOperand(2);
-    Mem = cast<MemIntrinsicSDNode>(N);
-    unsigned IID = N->getConstantOperandVal(1);
-    switch (IID) {
-    default:
-      return false;
-    case Intrinsic::nvvm_ldg_global_f:
-    case Intrinsic::nvvm_ldg_global_i:
-    case Intrinsic::nvvm_ldg_global_p:
-      IsLDG = true;
-      break;
-    case Intrinsic::nvvm_ldu_global_f:
-    case Intrinsic::nvvm_ldu_global_i:
-    case Intrinsic::nvvm_ldu_global_p:
-      IsLDG = false;
-      break;
-    }
-  } else {
-    Op1 = N->getOperand(1);
-    Mem = cast<MemSDNode>(N);
-  }
+  SDValue Op1 = N->getOperand(N->getOpcode() == ISD::INTRINSIC_W_CHAIN ? 2 : 1);
 
   EVT OrigType = N->getValueType(0);
   EVT EltVT = Mem->getMemoryVT();
@@ -1629,26 +1601,20 @@ bool NVPTXDAGToDAGISel::tryLDGLDU(SDNode *N) {
     default:
       return false;
     case ISD::LOAD:
+      Opcode = pickOpcodeForVT(
+          EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDG_GLOBAL_i8avar,
+          NVPTX::INT_PTX_LDG_GLOBAL_i16avar, NVPTX::INT_PTX_LDG_GLOBAL_i32avar,
+          NVPTX::INT_PTX_LDG_GLOBAL_i64avar, NVPTX::INT_PTX_LDG_GLOBAL_f32avar,
+          NVPTX::INT_PTX_LDG_GLOBAL_f64avar);
+      break;
     case ISD::INTRINSIC_W_CHAIN:
-      if (IsLDG)
-        Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
-                                 NVPTX::INT_PTX_LDG_GLOBAL_i8avar,
-                                 NVPTX::INT_PTX_LDG_GLOBAL_i16avar,
-                                 NVPTX::INT_PTX_LDG_GLOBAL_i32avar,
-                                 NVPTX::INT_PTX_LDG_GLOBAL_i64avar,
-                                 NVPTX::INT_PTX_LDG_GLOBAL_f32avar,
-                                 NVPTX::INT_PTX_LDG_GLOBAL_f64avar);
-      else
-        Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
-                                 NVPTX::INT_PTX_LDU_GLOBAL_i8avar,
-                                 NVPTX::INT_PTX_LDU_GLOBAL_i16avar,
-                                 NVPTX::INT_PTX_LDU_GLOBAL_i32avar,
-                                 NVPTX::INT_PTX_LDU_GLOBAL_i64avar,
-                                 NVPTX::INT_PTX_LDU_GLOBAL_f32avar,
-                                 NVPTX::INT_PTX_LDU_GLOBAL_f64avar);
+      Opcode = pickOpcodeForVT(
+          EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDU_GLOBAL_i8avar,
+          NVPTX::INT_PTX_LDU_GLOBAL_i16avar, NVPTX::INT_PTX_LDU_GLOBAL_i32avar,
+          NVPTX::INT_PTX_LDU_GLOBAL_i64avar, NVPTX::INT_PTX_LDU_GLOBAL_f32avar,
+          NVPTX::INT_PTX_LDU_GLOBAL_f64avar);
       break;
     case NVPTXISD::LoadV2:
-    case NVPTXISD::LDGV2:
       Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
                                NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar,
                                NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar,
@@ -1667,7 +1633,6 @@ bool NVPTXDAGToDAGISel::tryLDGLDU(SDNode *N) {
                                NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar);
       break;
     case NVPTXISD::LoadV4:
-    case NVPTXISD::LDGV4:
       Opcode = pickOpcodeForVT(
           EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar,
           NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar,
@@ -1693,26 +1658,24 @@ bool NVPTXDAGToDAGISel::tryLDGLDU(SDNode *N) {
       default:
         return false;
       case ISD::LOAD:
+        Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_i8ari64,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_i16ari64,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_i32ari64,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_i64ari64,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_f32ari64,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_f64ari64);
+        break;
       case ISD::INTRINSIC_W_CHAIN:
-        if (IsLDG)
-          Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
-                                       NVPTX::INT_PTX_LDG_GLOBAL_i8ari64,
-                                       NVPTX::INT_PTX_LDG_GLOBAL_i16ari64,
-                                       NVPTX::INT_PTX_LDG_GLOBAL_i32ari64,
-                                       NVPTX::INT_PTX_LDG_GLOBAL_i64ari64,
-                                       NVPTX::INT_PTX_LDG_GLOBAL_f32ari64,
-                                       NVPTX::INT_PTX_LDG_GLOBAL_f64ari64);
-        else
-          Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
-                                       NVPTX::INT_PTX_LDU_GLOBAL_i8ari64,
-                                       NVPTX::INT_PTX_LDU_GLOBAL_i16ari64,
-                                       NVPTX::INT_PTX_LDU_GLOBAL_i32ari64,
-                                       NVPTX::INT_PTX_LDU_GLOBAL_i64ari64,
-                                       NVPTX::INT_PTX_LDU_GLOBAL_f32ari64,
-                                       NVPTX::INT_PTX_LDU_GLOBAL_f64ari64);
+        Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_i8ari64,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_i16ari64,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_i32ari64,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_i64ari64,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_f32ari64,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_f64ari64);
         break;
       case NVPTXISD::LoadV2:
-      case NVPTXISD::LDGV2:
         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
                                      NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64,
                                      NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64,
@@ -1731,7 +1694,6 @@ bool NVPTXDAGToDAGISel::tryLDGLDU(SDNode *N) {
                                      NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64);
         break;
       case NVPTXISD::LoadV4:
-      case NVPTXISD::LDGV4:
         Opcode = pickOpcodeForVT(
             EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64,
             NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64,
@@ -1751,26 +1713,20 @@ bool NVPTXDAGToDAGISel::tryLDGLDU(SDNode *N) {
       default:
         return false;
       case ISD::LOAD:
+        Opcode = pickOpcodeForVT(
+            EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDG_GLOBAL_i8ari,
+            NVPTX::INT_PTX_LDG_GLOBAL_i16ari, NVPTX::INT_PTX_LDG_GLOBAL_i32ari,
+            NVPTX::INT_PTX_LDG_GLOBAL_i64ari, NVPTX::INT_PTX_LDG_GLOBAL_f32ari,
+            NVPTX::INT_PTX_LDG_GLOBAL_f64ari);
+        break;
       case ISD::INTRINSIC_W_CHAIN:
-        if (IsLDG)
-          Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
-                                   NVPTX::INT_PTX_LDG_GLOBAL_i8ari,
-                                   NVPTX::INT_PTX_LDG_GLOBAL_i16ari,
-                                   NVPTX::INT_PTX_LDG_GLOBAL_i32ari,
-                                   NVPTX::INT_PTX_LDG_GLOBAL_i64ari,
-                                   NVPTX::INT_PTX_LDG_GLOBAL_f32ari,
-                                   NVPTX::INT_PTX_LDG_GLOBAL_f64ari);
-        else
-          Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
-                                   NVPTX::INT_PTX_LDU_GLOBAL_i8ari,
-                                   NVPTX::INT_PTX_LDU_GLOBAL_i16ari,
-                                   NVPTX::INT_PTX_LDU_GLOBAL_i32ari,
-                                   NVPTX::INT_PTX_LDU_GLOBAL_i64ari,
-                                   NVPTX::INT_PTX_LDU_GLOBAL_f32ari,
-                                   NVPTX::INT_PTX_LDU_GLOBAL_f64ari);
+        Opcode = pickOpcodeForVT(
+            EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDU_GLOBAL_i8ari,
+            NVPTX::INT_PTX_LDU_GLOBAL_i16ari, NVPTX::INT_PTX_LDU_GLOBAL_i32ari,
+            NVPTX::INT_PTX_LDU_GLOBAL_i64ari, NVPTX::INT_PTX_LDU_GLOBAL_f32ari,
+            NVPTX::INT_PTX_LDU_GLOBAL_f64ari);
         break;
       case NVPTXISD::LoadV2:
-      case NVPTXISD::LDGV2:
         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
                                  NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32,
                                  NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32,
@@ -1789,7 +1745,6 @@ bool NVPTXDAGToDAGISel::tryLDGLDU(SDNode *N) {
                                  NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32);
         break;
       case NVPTXISD::LoadV4:
-      case NVPTXISD::LDGV4:
         Opcode = pickOpcodeForVT(
             EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32,
             NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32,
@@ -1815,26 +1770,24 @@ bool NVPTXDAGToDAGISel::tryLDGLDU(SDNode *N) {
       default:
         return false;
       case ISD::LOAD:
+        Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_i8areg64,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_i16areg64,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_i32areg64,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_i64areg64,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_f32areg64,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_f64areg64);
+        break;
       case ISD::INTRINSIC_W_CHAIN:
-        if (IsLDG)
-          Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
-                                       NVPTX::INT_PTX_LDG_GLOBAL_i8areg64,
-                                       NVPTX::INT_PTX_LDG_GLOBAL_i16areg64,
-                                       NVPTX::INT_PTX_LDG_GLOBAL_i32areg64,
-                                       NVPTX::INT_PTX_LDG_GLOBAL_i64areg64,
-                                       NVPTX::INT_PTX_LDG_GLOBAL_f32areg64,
-                                       NVPTX::INT_PTX_LDG_GLOBAL_f64areg64);
-        else
-          Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
-                                       NVPTX::INT_PTX_LDU_GLOBAL_i8areg64,
-                                       NVPTX::INT_PTX_LDU_GLOBAL_i16areg64,
-                                       NVPTX::INT_PTX_LDU_GLOBAL_i32areg64,
-                                       NVPTX::INT_PTX_LDU_GLOBAL_i64areg64,
-                                       NVPTX::INT_PTX_LDU_GLOBAL_f32areg64,
-                                       NVPTX::INT_PTX_LDU_GLOBAL_f64areg64);
+        Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_i8areg64,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_i16areg64,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_i32areg64,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_i64areg64,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_f32areg64,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_f64areg64);
         break;
       case NVPTXISD::LoadV2:
-      case NVPTXISD::LDGV2:
         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
                                      NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64,
                                      NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64,
@@ -1853,7 +1806,6 @@ bool NVPTXDAGToDAGISel::tryLDGLDU(SDNode *N) {
                                      NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64);
         break;
       case NVPTXISD::LoadV4:
-      case NVPTXISD::LDGV4:
         Opcode = pickOpcodeForVT(
             EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64,
             NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64,
@@ -1873,26 +1825,24 @@ bool NVPTXDAGToDAGISel::tryLDGLDU(SDNode *N) {
       default:
         return false;
       case ISD::LOAD:
+        Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_i8areg,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_i16areg,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_i32areg,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_i64areg,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_f32areg,
+                                 NVPTX::INT_PTX_LDG_GLOBAL_f64areg);
+        break;
       case ISD::INTRINSIC_W_CHAIN:
-        if (IsLDG)
-          Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
-                                   NVPTX::INT_PTX_LDG_GLOBAL_i8areg,
-                                   NVPTX::INT_PTX_LDG_GLOBAL_i16areg,
-                                   NVPTX::INT_PTX_LDG_GLOBAL_i32areg,
-                                   NVPTX::INT_PTX_LDG_GLOBAL_i64areg,
-                                   NVPTX::INT_PTX_LDG_GLOBAL_f32areg,
-                                   NVPTX::INT_PTX_LDG_GLOBAL_f64areg);
-        else
-          Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
-                                   NVPTX::INT_PTX_LDU_GLOBAL_i8areg,
-                                   NVPTX::INT_PTX_LDU_GLOBAL_i16areg,
-                                   NVPTX::INT_PTX_LDU_GLOBAL_i32areg,
-                                   NVPTX::INT_PTX_LDU_GLOBAL_i64areg,
-                                   NVPTX::INT_PTX_LDU_GLOBAL_f32areg,
-                                   NVPTX::INT_PTX_LDU_GLOBAL_f64areg);
+        Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_i8areg,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_i16areg,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_i32areg,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_i64areg,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_f32areg,
+                                 NVPTX::INT_PTX_LDU_GLOBAL_f64areg);
         break;
       case NVPTXISD::LoadV2:
-      case NVPTXISD::LDGV2:
         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
                                  NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32,
                                  NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32,
@@ -1911,7 +1861,6 @@ bool NVPTXDAGToDAGISel::tryLDGLDU(SDNode *N) {
                                  NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32);
         break;
       case NVPTXISD::LoadV4:
-      case NVPTXISD::LDGV4:
         Opcode = pickOpcodeForVT(
             EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32,
             NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32,
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 57bc5fe0ac36..a95cba586b8f 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -949,8 +949,6 @@ const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
     MAKE_CASE(NVPTXISD::ProxyReg)
     MAKE_CASE(NVPTXISD::LoadV2)
     MAKE_CASE(NVPTXISD::LoadV4)
-    MAKE_CASE(NVPTXISD::LDGV2)
-    MAKE_CASE(NVPTXISD::LDGV4)
     MAKE_CASE(NVPTXISD::LDUV2)
     MAKE_CASE(NVPTXISD::LDUV4)
     MAKE_CASE(NVPTXISD::StoreV2)
@@ -4774,26 +4772,6 @@ bool NVPTXTargetLowering::getTgtMemIntrinsic(
 
     return true;
   }
-  case Intrinsic::nvvm_ldg_global_i:
-  case Intrinsic::nvvm_ldg_global_f:
-  case Intrinsic::nvvm_ldg_global_p: {
-    auto &DL = I.getDataLayout();
-
-    Info.opc = ISD::INTRINSIC_W_CHAIN;
-    if (Intrinsic == Intrinsic::nvvm_ldg_global_i)
-      Info.memVT = getValueType(DL, I.getType());
-    else if(Intrinsic == Intrinsic::nvvm_ldg_global_p)
-      Info.memVT = getPointerTy(DL);
-    else
-      Info.memVT = getValueType(DL, I.getType());
-    Info.ptrVal = I.getArgOperand(0);
-    Info.offset = 0;
-    Info.flags = MachineMemOperand::MOLoad;
-    Info.align = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();
-
-    return true;
-  }
-
   case Intrinsic::nvvm_tex_1d_v4f32_s32:
   case Intrinsic::nvvm_tex_1d_v4f32_f32:
   case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
@@ -6308,9 +6286,6 @@ static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,
   switch (IntrinNo) {
   default:
     return;
-  case Intrinsic::nvvm_ldg_global_i:
-  case Intrinsic::nvvm_ldg_global_f:
-  case Intrinsic::nvvm_ldg_global_p:
   case Intrinsic::nvvm_ldu_global_i:
   case Intrinsic::nvvm_ldu_global_f:
   case Intrinsic::nvvm_ldu_global_p: {
@@ -6339,37 +6314,11 @@ static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,
       default:
         return;
       case 2:
-        switch (IntrinNo) {
-        default:
-          return;
-        case Intrinsic::nvvm_ldg_global_i:
-        case Intrinsic::nvvm_ldg_global_f:
-        case Intrinsic::nvvm_ldg_global_p:
-          Opcode = NVPTXISD::LDGV2;
-          break;
-        case Intrinsic::nvvm_ldu_global_i:
-        case Intrinsic::nvvm_ldu_global_f:
-        case Intrinsic::nvvm_ldu_global_p:
-          Opcode = NVPTXISD::LDUV2;
-          break;
-        }
+        Opcode = NVPTXISD::LDUV2;
         LdResVTs = DAG.getVTList(EltVT, EltVT, MVT::Other);
         break;
       case 4: {
-        switch (IntrinNo) {
-        default:
-          return;
-        case Intrinsic::nvvm_ldg_global_i:
-        case Intrinsic::nvvm_ldg_global_f:
-        case Intrinsic::nvvm_ldg_global_p:
-          Opcode = NVPTXISD::LDGV4;
-          break;
-        case Intrinsic::nvvm_ldu_global_i:
-        case Intrinsic::nvvm_ldu_global_f:
-        case Intrinsic::nvvm_ldu_global_p:
-          Opcode = NVPTXISD::LDUV4;
-          break;
-        }
+        Opcode = NVPTXISD::LDUV4;
         EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };
         LdResVTs = DAG.getVTList(ListVTs);
         break;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 8c3a597ce0b0..824a65967196 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -70,8 +70,6 @@ enum NodeType : unsigned {
 
   LoadV2 = ISD::FIRST_TARGET_MEMORY_OPCODE,
   LoadV4,
-  LDGV2, // LDG.v2
-  LDGV4, // LDG.v4
   LDUV2, // LDU.v2
   LDUV4, // LDU.v4
   StoreV2,
diff --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index 584c0ef7cfeb..5cc3a3027745 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -44,6 +44,13 @@ declare ptr @llvm.nvvm.ptr.shared.to.gen.p0.p3(ptr addrspace(3))
 declare ptr @llvm.nvvm.ptr.constant.to.gen.p0.p4(ptr addrspace(4))
 declare ptr @llvm.nvvm.ptr.local.to.gen.p0.p5(ptr addrspace(5))
 
+declare i32 @llvm.nvvm.ldg.global.i.i32.p1(ptr addrspace(1), i32)
+declare ptr @llvm.nvvm.ldg.global.p.p1(ptr addrspace(1), i32)
+declare float @llvm.nvvm.ldg.global.f.f32.p1(ptr addrspace(1), i32)
+declare i32 @llvm.nvvm.ldg.global.i.i32.p0(ptr, i32)
+declare ptr @llvm.nvvm.ldg.global.p.p0(ptr, i32)
+declare float @llvm.nvvm.ldg.global.f.f32.p0(ptr, i32)
+
 ; CHECK-LABEL: @simple_upgrade
 define void @simple_upgrade(i32 %a, i64 %b, i16 %c) {
 ; CHECK: call i32 @llvm.bitreverse.i32(i32 %a)
@@ -191,3 +198,27 @@ define void @addrspacecast(ptr %p0) {
 
   ret void
 }
+
+; CHECK-LABEL: @ldg
+define void @ldg(ptr %p0, ptr addrspace(1) %p1) {
+; CHECK: %1 = load i32, ptr addrspace(1) %p1, align 4, !invariant.load !0
+; CHECK: %2 = load ptr, ptr addrspace(1) %p1, align 8, !invariant.load !0
+; CHECK: %3 = load float, ptr addrspace(1) %p1, align 16, !invariant.load !0
+
+; CHECK: %4 = addrspacecast ptr %p0 to ptr addrspace(1)
+; CHECK: %5 = load i32, ptr addrspace(1) %4, align 4, !invariant.load !0
+; CHECK: %6 = addrspacecast ptr %p0 to ptr addrspace(1)
+; CHECK: %7 = load ptr, ptr addrspace(1) %6, align 8, !invariant.load !0
+; CHECK: %8 = addrspacecast ptr %p0 to ptr addrspace(1)
+; CHECK: %9 = load float, ptr addrspace(1) %8, align 16, !invariant.load !0
+;
+  %v1 = call i32 @llvm.nvvm.ldg.global.i.i32.p1(ptr addrspace(1) %p1, i32 4)
+  %v2 = call ptr @llvm.nvvm.ldg.global.p.p1(ptr addrspace(1) %p1, i32 8 )
+  %v3 = call float @llvm.nvvm.ldg.global.f.f32.p1(ptr addrspace(1) %p1, i32 16)
+
+  %v4 = call i32 @llvm.nvvm.ldg.global.i.i32.p0(ptr %p0, i32 4)
+  %v5 = call ptr @llvm.nvvm.ldg.global.p.p0(ptr %p0, i32 8)
+  %v6 = call float @llvm.nvvm.ldg.global.f.f32.p0(ptr %p0, i32 16)
+
+  ret void
+}
\ No newline at end of file
-- 
GitLab


From 1fe8e7838bb5118b9e48fa15fa21a4638bae8ae1 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 27 Oct 2024 17:29:38 -0700
Subject: [PATCH 269/693] [X86] Rename the x86-asm-syntax variable. NFC

Follow-up to #109360. x86-asm-syntax is for input instead of output.
---
 llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
index 47eb617c06ac..39b0f7c4c4c1 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
@@ -23,9 +23,9 @@ enum AsmWriterFlavorTy {
   ATT = 0, Intel = 1
 };
 
-static cl::opt<AsmWriterFlavorTy> AsmWriterFlavor(
+static cl::opt<AsmWriterFlavorTy> X86AsmSyntax(
     "x86-asm-syntax", cl::init(ATT), cl::Hidden,
-    cl::desc("Choose style of code to emit from X86 backend:"),
+    cl::desc("Select the assembly style for input"),
     cl::values(clEnumValN(ATT, "att", "Emit AT&T-style assembly"),
                clEnumValN(Intel, "intel", "Emit Intel-style assembly")));
 
@@ -41,7 +41,7 @@ X86MCAsmInfoDarwin::X86MCAsmInfoDarwin(const Triple &T) {
   if (is64Bit)
     CodePointerSize = CalleeSaveStackSlotSize = 8;
 
-  AssemblerDialect = AsmWriterFlavor;
+  AssemblerDialect = X86AsmSyntax;
 
   if (!is64Bit)
     Data64bitsDirective = nullptr;       // we can't emit a 64-bit unit
@@ -89,7 +89,7 @@ X86ELFMCAsmInfo::X86ELFMCAsmInfo(const Triple &T) {
   // OTOH, stack slot size is always 8 for x86-64, even with the x32 ABI.
   CalleeSaveStackSlotSize = is64Bit ? 8 : 4;
 
-  AssemblerDialect = AsmWriterFlavor;
+  AssemblerDialect = X86AsmSyntax;
 
   // Debug Information
   SupportsDebugInformation = true;
@@ -126,7 +126,7 @@ X86MCAsmInfoMicrosoft::X86MCAsmInfoMicrosoft(const Triple &Triple) {
 
   ExceptionsType = ExceptionHandling::WinEH;
 
-  AssemblerDialect = AsmWriterFlavor;
+  AssemblerDialect = X86AsmSyntax;
 
   AllowAtInName = true;
 }
@@ -159,7 +159,7 @@ X86MCAsmInfoGNUCOFF::X86MCAsmInfoGNUCOFF(const Triple &Triple) {
     ExceptionsType = ExceptionHandling::DwarfCFI;
   }
 
-  AssemblerDialect = AsmWriterFlavor;
+  AssemblerDialect = X86AsmSyntax;
 
   AllowAtInName = true;
 }
-- 
GitLab


From 40fffba9b29141d598bf36cdc4bb886e97143d06 Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Mon, 28 Oct 2024 09:54:29 +0800
Subject: [PATCH 270/693] [X86][AVX10.2] Fix wrong predicates for BF16 feature
 (#113800)

Since AVX10.2, we need to enable 128/256-bit vector by default and check
for 512 feature for 512-bit vector.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp    | 41 ++++++++++------------
 llvm/test/CodeGen/X86/avx10_2bf16-arith.ll | 22 ++++++++++++
 2 files changed, 41 insertions(+), 22 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index a6d77873ec29..9d447959faf5 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2406,7 +2406,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     addLegalFPImmediate(APFloat::getZero(APFloat::BFloat()));
   }
 
-  if (!Subtarget.useSoftFloat() && Subtarget.hasBF16()) {
+  if (!Subtarget.useSoftFloat() && Subtarget.hasBF16() &&
+      Subtarget.useAVX512Regs()) {
     addRegisterClass(MVT::v32bf16, &X86::VR512RegClass);
     setF16Action(MVT::v32bf16, Expand);
     for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV})
@@ -2419,27 +2420,23 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   }
 
   if (!Subtarget.useSoftFloat() && Subtarget.hasAVX10_2()) {
-    addRegisterClass(MVT::v8bf16, &X86::VR128XRegClass);
-    addRegisterClass(MVT::v16bf16, &X86::VR256XRegClass);
-    addRegisterClass(MVT::v32bf16, &X86::VR512RegClass);
-
-    setOperationAction(ISD::FADD, MVT::v32bf16, Legal);
-    setOperationAction(ISD::FSUB, MVT::v32bf16, Legal);
-    setOperationAction(ISD::FMUL, MVT::v32bf16, Legal);
-    setOperationAction(ISD::FDIV, MVT::v32bf16, Legal);
-    setOperationAction(ISD::FSQRT, MVT::v32bf16, Legal);
-    setOperationAction(ISD::FMA, MVT::v32bf16, Legal);
-    setOperationAction(ISD::SETCC, MVT::v32bf16, Custom);
-    if (Subtarget.hasVLX()) {
-      for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
-        setOperationAction(ISD::FADD, VT, Legal);
-        setOperationAction(ISD::FSUB, VT, Legal);
-        setOperationAction(ISD::FMUL, VT, Legal);
-        setOperationAction(ISD::FDIV, VT, Legal);
-        setOperationAction(ISD::FSQRT, VT, Legal);
-        setOperationAction(ISD::FMA, VT, Legal);
-        setOperationAction(ISD::SETCC, VT, Custom);
-      }
+    for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
+      setOperationAction(ISD::FADD, VT, Legal);
+      setOperationAction(ISD::FSUB, VT, Legal);
+      setOperationAction(ISD::FMUL, VT, Legal);
+      setOperationAction(ISD::FDIV, VT, Legal);
+      setOperationAction(ISD::FSQRT, VT, Legal);
+      setOperationAction(ISD::FMA, VT, Legal);
+      setOperationAction(ISD::SETCC, VT, Custom);
+    }
+    if (Subtarget.hasAVX10_2_512()) {
+      setOperationAction(ISD::FADD, MVT::v32bf16, Legal);
+      setOperationAction(ISD::FSUB, MVT::v32bf16, Legal);
+      setOperationAction(ISD::FMUL, MVT::v32bf16, Legal);
+      setOperationAction(ISD::FDIV, MVT::v32bf16, Legal);
+      setOperationAction(ISD::FSQRT, MVT::v32bf16, Legal);
+      setOperationAction(ISD::FMA, MVT::v32bf16, Legal);
+      setOperationAction(ISD::SETCC, MVT::v32bf16, Custom);
     }
   }
 
diff --git a/llvm/test/CodeGen/X86/avx10_2bf16-arith.ll b/llvm/test/CodeGen/X86/avx10_2bf16-arith.ll
index e0f5679e8ac9..c97d27ff324b 100644
--- a/llvm/test/CodeGen/X86/avx10_2bf16-arith.ll
+++ b/llvm/test/CodeGen/X86/avx10_2bf16-arith.ll
@@ -1166,3 +1166,25 @@ entry:
   %2 = select <8 x i1> %1, <8 x bfloat> %0, <8 x bfloat> zeroinitializer
   ret <8 x bfloat> %2
 }
+
+define <32 x bfloat> @addv(<32 x bfloat> %a, <32 x bfloat> %b) nounwind {
+; X64-LABEL: addv:
+; X64:       # %bb.0:
+; X64-NEXT:    vaddnepbf16 %ymm2, %ymm0, %ymm0 # encoding: [0x62,0xf5,0x7d,0x28,0x58,0xc2]
+; X64-NEXT:    vaddnepbf16 %ymm3, %ymm1, %ymm1 # encoding: [0x62,0xf5,0x75,0x28,0x58,0xcb]
+; X64-NEXT:    retq # encoding: [0xc3]
+;
+; X86-LABEL: addv:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp # encoding: [0x55]
+; X86-NEXT:    movl %esp, %ebp # encoding: [0x89,0xe5]
+; X86-NEXT:    andl $-32, %esp # encoding: [0x83,0xe4,0xe0]
+; X86-NEXT:    subl $32, %esp # encoding: [0x83,0xec,0x20]
+; X86-NEXT:    vaddnepbf16 %ymm2, %ymm0, %ymm0 # encoding: [0x62,0xf5,0x7d,0x28,0x58,0xc2]
+; X86-NEXT:    vaddnepbf16 8(%ebp), %ymm1, %ymm1 # encoding: [0x62,0xf5,0x75,0x28,0x58,0x8d,0x08,0x00,0x00,0x00]
+; X86-NEXT:    movl %ebp, %esp # encoding: [0x89,0xec]
+; X86-NEXT:    popl %ebp # encoding: [0x5d]
+; X86-NEXT:    retl # encoding: [0xc3]
+  %add = fadd <32 x bfloat> %a, %b
+  ret <32 x bfloat> %add
+}
-- 
GitLab


From 7ad63c0e44ef277591497a176991e7723165611e Mon Sep 17 00:00:00 2001
From: Longsheng Mou <longshengmou@gmail.com>
Date: Mon, 28 Oct 2024 09:54:51 +0800
Subject: [PATCH 271/693] [mlir][MathToFuncs] `MathToFuncs` only support
 integer type (#113693)

This PR fixes a bug in `MathToFuncs` where it incorrectly converts index
type for `math.ctlz` and `math.ipowi`, leading to a crash. Fixes
#108150.
---
 .../Conversion/MathToFuncs/MathToFuncs.cpp    | 21 +++++++++++++++----
 mlir/test/Conversion/MathToFuncs/ctlz.mlir    | 10 +++++++++
 mlir/test/Conversion/MathToFuncs/ipowi.mlir   | 11 ++++++++++
 3 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp b/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
index 3a567643ffdb..df5396ac628c 100644
--- a/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
+++ b/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
@@ -781,6 +781,9 @@ private:
   // or equal to minWidthOfFPowIExponent option value.
   bool isFPowIConvertible(math::FPowIOp op);
 
+  // Reture true, if operation is integer type.
+  bool isConvertible(Operation *op);
+
   // Generate outlined implementations for power operations
   // and store them in funcImpls map.
   void generateOpImplementations();
@@ -798,13 +801,17 @@ bool ConvertMathToFuncsPass::isFPowIConvertible(math::FPowIOp op) {
   return (expTy && expTy.getWidth() >= minWidthOfFPowIExponent);
 }
 
+bool ConvertMathToFuncsPass::isConvertible(Operation *op) {
+  return isa<IntegerType>(getElementTypeOrSelf(op->getResult(0).getType()));
+}
+
 void ConvertMathToFuncsPass::generateOpImplementations() {
   ModuleOp module = getOperation();
 
   module.walk([&](Operation *op) {
     TypeSwitch<Operation *>(op)
         .Case<math::CountLeadingZerosOp>([&](math::CountLeadingZerosOp op) {
-          if (!convertCtlz)
+          if (!convertCtlz || !isConvertible(op))
             return;
           Type resultType = getElementTypeOrSelf(op.getResult().getType());
 
@@ -816,6 +823,9 @@ void ConvertMathToFuncsPass::generateOpImplementations() {
             entry.first->second = createCtlzFunc(&module, resultType);
         })
         .Case<math::IPowIOp>([&](math::IPowIOp op) {
+          if (!isConvertible(op))
+            return;
+
           Type resultType = getElementTypeOrSelf(op.getResult().getType());
 
           // Generate the software implementation of this operation,
@@ -873,9 +883,12 @@ void ConvertMathToFuncsPass::runOnOperation() {
                          func::FuncDialect, scf::SCFDialect,
                          vector::VectorDialect>();
 
-  target.addIllegalOp<math::IPowIOp>();
-  if (convertCtlz)
-    target.addIllegalOp<math::CountLeadingZerosOp>();
+  target.addDynamicallyLegalOp<math::IPowIOp>(
+      [this](math::IPowIOp op) { return !isConvertible(op); });
+  if (convertCtlz) {
+    target.addDynamicallyLegalOp<math::CountLeadingZerosOp>(
+        [this](math::CountLeadingZerosOp op) { return !isConvertible(op); });
+  }
   target.addDynamicallyLegalOp<math::FPowIOp>(
       [this](math::FPowIOp op) { return !isFPowIConvertible(op); });
   if (failed(applyPartialConversion(module, target, std::move(patterns))))
diff --git a/mlir/test/Conversion/MathToFuncs/ctlz.mlir b/mlir/test/Conversion/MathToFuncs/ctlz.mlir
index 4e262417d6a9..b7ef0a892891 100644
--- a/mlir/test/Conversion/MathToFuncs/ctlz.mlir
+++ b/mlir/test/Conversion/MathToFuncs/ctlz.mlir
@@ -91,3 +91,13 @@ func.func @main(%arg0: i8) {
   func.return
 }
 
+// -----
+
+// Check that index is not converted
+
+// CHECK-LABEL: func.func @ctlz_index
+// CHECK:         math.ctlz
+func.func @ctlz_index(%arg0: index) {
+  %0 = math.ctlz %arg0 : index
+  func.return
+}
diff --git a/mlir/test/Conversion/MathToFuncs/ipowi.mlir b/mlir/test/Conversion/MathToFuncs/ipowi.mlir
index e464e9ca9564..2702a1e22e62 100644
--- a/mlir/test/Conversion/MathToFuncs/ipowi.mlir
+++ b/mlir/test/Conversion/MathToFuncs/ipowi.mlir
@@ -170,3 +170,14 @@ func.func @ipowi_vec(%arg0: vector<2x3xi64>, %arg1: vector<2x3xi64>) {
   %0 = math.ipowi %arg0, %arg1 : vector<2x3xi64>
   func.return
 }
+
+// -----
+
+// Check that index is not converted
+
+// CHECK-LABEL: func.func @ipowi_index
+// CHECK:         math.ipowi
+func.func @ipowi_index(%arg0: index, %arg1: index) {
+  %0 = math.ipowi %arg0, %arg1 : index
+  func.return
+}
-- 
GitLab


From 39ac64c1c0fc61a476aa22c53e6977608ead03cf Mon Sep 17 00:00:00 2001
From: donald chen <chenxunyu1993@gmail.com>
Date: Mon, 28 Oct 2024 10:14:44 +0800
Subject: [PATCH 272/693] [mlir][Arith] ValueBoundsInterface: speedup
 arith.select (#113531)

When calculating value bounds in the arith.select op , the compare
function is invoked to compare trueValue and falseValue. This function
rebuilds constraints, resulting in repeated computations of value
bounds.

In large-scale programs, this redundancy significantly impacts
compilation time.
---
 .../Arith/IR/ValueBoundsOpInterfaceImpl.cpp        | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/mlir/lib/Dialect/Arith/IR/ValueBoundsOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/IR/ValueBoundsOpInterfaceImpl.cpp
index 7cfcc4180539..6de151594e3e 100644
--- a/mlir/lib/Dialect/Arith/IR/ValueBoundsOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ValueBoundsOpInterfaceImpl.cpp
@@ -107,9 +107,10 @@ struct SelectOpInterface
     // If trueValue <= falseValue:
     // * result <= falseValue
     // * result >= trueValue
-    if (cstr.compare(/*lhs=*/{trueValue, dim},
-                     ValueBoundsConstraintSet::ComparisonOperator::LE,
-                     /*rhs=*/{falseValue, dim})) {
+    if (cstr.populateAndCompare(
+            /*lhs=*/{trueValue, dim},
+            ValueBoundsConstraintSet::ComparisonOperator::LE,
+            /*rhs=*/{falseValue, dim})) {
       if (dim) {
         cstr.bound(value)[*dim] >= cstr.getExpr(trueValue, dim);
         cstr.bound(value)[*dim] <= cstr.getExpr(falseValue, dim);
@@ -121,9 +122,10 @@ struct SelectOpInterface
     // If falseValue <= trueValue:
     // * result <= trueValue
     // * result >= falseValue
-    if (cstr.compare(/*lhs=*/{falseValue, dim},
-                     ValueBoundsConstraintSet::ComparisonOperator::LE,
-                     /*rhs=*/{trueValue, dim})) {
+    if (cstr.populateAndCompare(
+            /*lhs=*/{falseValue, dim},
+            ValueBoundsConstraintSet::ComparisonOperator::LE,
+            /*rhs=*/{trueValue, dim})) {
       if (dim) {
         cstr.bound(value)[*dim] >= cstr.getExpr(falseValue, dim);
         cstr.bound(value)[*dim] <= cstr.getExpr(trueValue, dim);
-- 
GitLab


From 5aa1275d03b679f45f47f29f206292f663afed83 Mon Sep 17 00:00:00 2001
From: Freddy Ye <freddy.ye@intel.com>
Date: Mon, 28 Oct 2024 10:46:16 +0800
Subject: [PATCH 273/693] [X86] Support SM4 EVEX version
 intrinsics/instructions. (#113402)

Ref.: https://cdrdv2.intel.com/v1/dl/getContent/671368
---
 clang/docs/ReleaseNotes.rst                   |   4 +
 clang/include/clang/Basic/BuiltinsX86.def     |   4 +
 clang/lib/Headers/CMakeLists.txt              |   1 +
 clang/lib/Headers/immintrin.h                 |   5 +
 clang/lib/Headers/sm4evexintrin.h             |  32 +++
 clang/test/CodeGen/X86/sm4-evex-builtins.c    |  19 ++
 llvm/docs/ReleaseNotes.md                     |   2 +
 llvm/include/llvm/IR/IntrinsicsX86.td         |  10 +
 llvm/lib/Target/X86/X86InstrAVX10.td          |  14 ++
 llvm/test/CodeGen/X86/sm4-evex-intrinsics.ll  |  64 +++++
 llvm/test/MC/Disassembler/X86/sm4-evex-32.txt | 170 +++++++++++++
 llvm/test/MC/Disassembler/X86/sm4-evex-64.txt | 170 +++++++++++++
 llvm/test/MC/X86/sm4-evex-32-att.s            | 169 +++++++++++++
 llvm/test/MC/X86/sm4-evex-32-intel.s          | 169 +++++++++++++
 llvm/test/MC/X86/sm4-evex-64-att.s            | 224 ++++++++++++++++++
 llvm/test/MC/X86/sm4-evex-64-intel.s          | 169 +++++++++++++
 llvm/test/TableGen/x86-fold-tables.inc        |   6 +
 17 files changed, 1232 insertions(+)
 create mode 100644 clang/lib/Headers/sm4evexintrin.h
 create mode 100644 clang/test/CodeGen/X86/sm4-evex-builtins.c
 create mode 100644 llvm/test/CodeGen/X86/sm4-evex-intrinsics.ll
 create mode 100644 llvm/test/MC/Disassembler/X86/sm4-evex-32.txt
 create mode 100644 llvm/test/MC/Disassembler/X86/sm4-evex-64.txt
 create mode 100644 llvm/test/MC/X86/sm4-evex-32-att.s
 create mode 100644 llvm/test/MC/X86/sm4-evex-32-intel.s
 create mode 100644 llvm/test/MC/X86/sm4-evex-64-att.s
 create mode 100644 llvm/test/MC/X86/sm4-evex-64-intel.s

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 6a9533781517..31ee4f7e516f 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -628,6 +628,10 @@ X86 Support
   * Supported MINMAX intrinsics of ``*_(mask(z)))_minmax(ne)_p[s|d|h|bh]`` and
   ``*_(mask(z)))_minmax_s[s|d|h]``.
 
+- Supported intrinsics for ``SM4 and AVX10.2``.
+  * Supported SM4 intrinsics of ``_mm512_sm4key4_epi32`` and
+  ``_mm512_sm4rnds4_epi32``.
+
 - All intrinsics in adcintrin.h can now be used in constant expressions.
 
 - All intrinsics in adxintrin.h can now be used in constant expressions.
diff --git a/clang/include/clang/Basic/BuiltinsX86.def b/clang/include/clang/Basic/BuiltinsX86.def
index 4c6b22cca421..4486eb73a11f 100644
--- a/clang/include/clang/Basic/BuiltinsX86.def
+++ b/clang/include/clang/Basic/BuiltinsX86.def
@@ -2179,6 +2179,10 @@ TARGET_BUILTIN(__builtin_ia32_vsm4key4256, "V8UiV8UiV8Ui", "nV:256:", "sm4")
 TARGET_BUILTIN(__builtin_ia32_vsm4rnds4128, "V4UiV4UiV4Ui", "nV:128:", "sm4")
 TARGET_BUILTIN(__builtin_ia32_vsm4rnds4256, "V8UiV8UiV8Ui", "nV:256:", "sm4")
 
+// SM4_EVEX
+TARGET_BUILTIN(__builtin_ia32_vsm4key4512, "V16UiV16UiV16Ui", "nV:512:", "avx10.2-512,sm4")
+TARGET_BUILTIN(__builtin_ia32_vsm4rnds4512, "V16UiV16UiV16Ui", "nV:512:", "avx10.2-512,sm4")
+
 // AVX10 MINMAX
 TARGET_BUILTIN(__builtin_ia32_vminmaxnepbf16128, "V8yV8yV8yIi", "nV:128:", "avx10.2-256")
 TARGET_BUILTIN(__builtin_ia32_vminmaxnepbf16256, "V16yV16yV16yIi", "nV:256:", "avx10.2-256")
diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index e97953d87a2f..0211d1870b30 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -243,6 +243,7 @@ set(x86_files
   shaintrin.h
   sm3intrin.h
   sm4intrin.h
+  sm4evexintrin.h
   smmintrin.h
   tbmintrin.h
   tmmintrin.h
diff --git a/clang/lib/Headers/immintrin.h b/clang/lib/Headers/immintrin.h
index 5f296d0a3324..65ad72bc479f 100644
--- a/clang/lib/Headers/immintrin.h
+++ b/clang/lib/Headers/immintrin.h
@@ -677,6 +677,11 @@ _storebe_i64(void * __P, long long __D) {
 #include <avx10_2_512satcvtintrin.h>
 #endif
 
+#if !defined(__SCE__) || __has_feature(modules) ||                             \
+    (defined(__AVX10_2_512__) && defined(__SM4__))
+#include <sm4evexintrin.h>
+#endif
+
 #if !defined(__SCE__) || __has_feature(modules) || defined(__ENQCMD__)
 #include <enqcmdintrin.h>
 #endif
diff --git a/clang/lib/Headers/sm4evexintrin.h b/clang/lib/Headers/sm4evexintrin.h
new file mode 100644
index 000000000000..f6ae0037baea
--- /dev/null
+++ b/clang/lib/Headers/sm4evexintrin.h
@@ -0,0 +1,32 @@
+/*===--------------- sm4evexintrin.h - SM4 EVEX intrinsics -----------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===----------------------------------------------------------------------===
+ */
+#ifndef __IMMINTRIN_H
+#error "Never use <sm4evexintrin.h> directly; include <immintrin.h> instead."
+#endif // __IMMINTRIN_H
+
+#ifndef __SM4EVEXINTRIN_H
+#define __SM4EVEXINTRIN_H
+
+#define __DEFAULT_FN_ATTRS512                                                  \
+  __attribute__((__always_inline__, __nodebug__,                               \
+                 __target__("sm4,avx10.2-512"), __min_vector_width__(512)))
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS512
+_mm512_sm4key4_epi32(__m512i __A, __m512i __B) {
+  return (__m512i)__builtin_ia32_vsm4key4512((__v16su)__A, (__v16su)__B);
+}
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS512
+_mm512_sm4rnds4_epi32(__m512i __A, __m512i __B) {
+  return (__m512i)__builtin_ia32_vsm4rnds4512((__v16su)__A, (__v16su)__B);
+}
+
+#undef __DEFAULT_FN_ATTRS512
+
+#endif // __SM4EVEXINTRIN_H
diff --git a/clang/test/CodeGen/X86/sm4-evex-builtins.c b/clang/test/CodeGen/X86/sm4-evex-builtins.c
new file mode 100644
index 000000000000..0e54bd008d4f
--- /dev/null
+++ b/clang/test/CodeGen/X86/sm4-evex-builtins.c
@@ -0,0 +1,19 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-- -target-feature +sm4 \
+// RUN: -target-feature +avx10.2-512 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 %s -ffreestanding -triple=i386-- -target-feature +sm4 \
+// RUN: -target-feature +avx10.2-512 -emit-llvm -o - -Wall -Werror | FileCheck %s
+
+#include <immintrin.h>
+#include <stddef.h>
+
+__m512i test_mm512_sm4key4_epi32(__m512i __A, __m512i __B) {
+  // CHECK-LABEL: @test_mm512_sm4key4_epi32(
+  // CHECK: call <16 x i32> @llvm.x86.vsm4key4512(<16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+  return _mm512_sm4key4_epi32(__A, __B);
+}
+
+__m512i test_mm512_sm4rnds4_epi32(__m512i __A, __m512i __B) {
+  // CHECK-LABEL: @test_mm512_sm4rnds4_epi32(
+  // CHECK: call <16 x i32> @llvm.x86.vsm4rnds4512(<16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+  return _mm512_sm4rnds4_epi32(__A, __B);
+}
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 3cac3e57344d..7c7e687e9474 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -219,6 +219,8 @@ Changes to the X86 Backend
 
 * Supported instructions of `MOVRS AND AVX10.2`
 
+* Supported ISA of `SM4(EVEX)`.
+
 Changes to the OCaml bindings
 -----------------------------
 
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index d0083017fb93..0ecca157077f 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -6099,6 +6099,11 @@ let TargetPrefix = "x86" in {
         DefaultAttrsIntrinsic<[llvm_v8i32_ty],
         [llvm_v8i32_ty, llvm_v8i32_ty],
         [IntrNoMem]>;
+  def int_x86_vsm4key4512
+      : ClangBuiltin<"__builtin_ia32_vsm4key4512">,
+        DefaultAttrsIntrinsic<[llvm_v16i32_ty],
+        [llvm_v16i32_ty, llvm_v16i32_ty],
+        [IntrNoMem]>;
   def int_x86_vsm4rnds4128
       : ClangBuiltin<"__builtin_ia32_vsm4rnds4128">,
         DefaultAttrsIntrinsic<[llvm_v4i32_ty],
@@ -6109,6 +6114,11 @@ let TargetPrefix = "x86" in {
         DefaultAttrsIntrinsic<[llvm_v8i32_ty],
         [llvm_v8i32_ty, llvm_v8i32_ty],
         [IntrNoMem]>;
+  def int_x86_vsm4rnds4512
+      : ClangBuiltin<"__builtin_ia32_vsm4rnds4512">,
+        DefaultAttrsIntrinsic<[llvm_v16i32_ty],
+        [llvm_v16i32_ty, llvm_v16i32_ty],
+        [IntrNoMem]>;
 }
 //===----------------------------------------------------------------------===//
 // RAO-INT intrinsics
diff --git a/llvm/lib/Target/X86/X86InstrAVX10.td b/llvm/lib/Target/X86/X86InstrAVX10.td
index 9ef2debb57fa..4d64eb776e09 100644
--- a/llvm/lib/Target/X86/X86InstrAVX10.td
+++ b/llvm/lib/Target/X86/X86InstrAVX10.td
@@ -1675,3 +1675,17 @@ defm VMOVRSD : vmovrs_p_vl<0x6f, "vmovrsd", avx512vl_i32_info>,
                           T_MAP5, XS, EVEX_CD8<32, CD8VF>, Sched<[WriteVecLoad]>;
 defm VMOVRSQ : vmovrs_p_vl<0x6f, "vmovrsq", avx512vl_i64_info>,
                           T_MAP5, XS, REX_W, EVEX_CD8<64, CD8VF>, Sched<[WriteVecLoad]>;
+
+// SM4(EVEX)
+multiclass avx10_sm4_base<string OpStr> {
+  // SM4_Base is in X86InstrSSE.td.
+  let Predicates = [HasSM4, HasAVX10_2], AddedComplexity = 1 in {
+    defm Z128 : SM4_Base<OpStr, VR128X, "128", loadv4i32, i128mem>, EVEX_V128;
+    defm Z256 : SM4_Base<OpStr, VR256X, "256", loadv8i32, i256mem>, EVEX_V256;
+  }
+  let Predicates = [HasSM4, HasAVX10_2_512] in
+    defm Z : SM4_Base<OpStr, VR512, "512", loadv16i32, i512mem>, EVEX_V512;
+}
+
+defm VSM4KEY4 : avx10_sm4_base<"vsm4key4">, T8, XS, EVEX, VVVV;
+defm VSM4RNDS4 : avx10_sm4_base<"vsm4rnds4">, T8, XD, EVEX, VVVV;
diff --git a/llvm/test/CodeGen/X86/sm4-evex-intrinsics.ll b/llvm/test/CodeGen/X86/sm4-evex-intrinsics.ll
new file mode 100644
index 000000000000..825a11d66cd4
--- /dev/null
+++ b/llvm/test/CodeGen/X86/sm4-evex-intrinsics.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-- --show-mc-encoding -mattr=+sm4,+avx10.2-512 | FileCheck %s
+; RUN: llc < %s -verify-machineinstrs -mtriple=i686-- --show-mc-encoding -mattr=+sm4,+avx10.2-512 | FileCheck %s
+
+define <4 x i32> @test_int_x86_vsm4key4128(<4 x i32> %A, <4 x i32> %B) {
+; CHECK-LABEL: test_int_x86_vsm4key4128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsm4key4 %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7a,0xda,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %ret = call <4 x i32> @llvm.x86.vsm4key4128(<4 x i32> %A, <4 x i32> %B)
+  ret <4 x i32> %ret
+}
+declare <4 x i32> @llvm.x86.vsm4key4128(<4 x i32> %A, <4 x i32> %B)
+
+define <8 x i32> @test_int_x86_vsm4key4256(<8 x i32> %A, <8 x i32> %B) {
+; CHECK-LABEL: test_int_x86_vsm4key4256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsm4key4 %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7e,0xda,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %ret = call <8 x i32> @llvm.x86.vsm4key4256(<8 x i32> %A, <8 x i32> %B)
+  ret <8 x i32> %ret
+}
+declare <8 x i32> @llvm.x86.vsm4key4256(<8 x i32> %A, <8 x i32> %B)
+
+define <16 x i32> @test_int_x86_vsm4key4512(<16 x i32> %A, <16 x i32> %B) {
+; CHECK-LABEL: test_int_x86_vsm4key4512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsm4key4 %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf2,0x7e,0x48,0xda,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %ret = call <16 x i32> @llvm.x86.vsm4key4512(<16 x i32> %A, <16 x i32> %B)
+  ret <16 x i32> %ret
+}
+declare <16 x i32> @llvm.x86.vsm4key4512(<16 x i32> %A, <16 x i32> %B)
+
+define <4 x i32> @test_int_x86_vsm4rnds4128(<4 x i32> %A, <4 x i32> %B) {
+; CHECK-LABEL: test_int_x86_vsm4rnds4128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsm4rnds4 %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7b,0xda,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %ret = call <4 x i32> @llvm.x86.vsm4rnds4128(<4 x i32> %A, <4 x i32> %B)
+  ret <4 x i32> %ret
+}
+declare <4 x i32> @llvm.x86.vsm4rnds4128(<4 x i32> %A, <4 x i32> %B)
+
+define <8 x i32> @test_int_x86_vsm4rnds4256(<8 x i32> %A, <8 x i32> %B) {
+; CHECK-LABEL: test_int_x86_vsm4rnds4256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsm4rnds4 %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7f,0xda,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %ret = call <8 x i32> @llvm.x86.vsm4rnds4256(<8 x i32> %A, <8 x i32> %B)
+  ret <8 x i32> %ret
+}
+declare <8 x i32> @llvm.x86.vsm4rnds4256(<8 x i32> %A, <8 x i32> %B)
+
+define <16 x i32> @test_int_x86_vsm4rnds4512(<16 x i32> %A, <16 x i32> %B) {
+; CHECK-LABEL: test_int_x86_vsm4rnds4512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsm4rnds4 %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf2,0x7f,0x48,0xda,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %ret = call <16 x i32> @llvm.x86.vsm4rnds4512(<16 x i32> %A, <16 x i32> %B)
+  ret <16 x i32> %ret
+}
+declare <16 x i32> @llvm.x86.vsm4rnds4512(<16 x i32> %A, <16 x i32> %B)
+
diff --git a/llvm/test/MC/Disassembler/X86/sm4-evex-32.txt b/llvm/test/MC/Disassembler/X86/sm4-evex-32.txt
new file mode 100644
index 000000000000..c1cb271a967b
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/sm4-evex-32.txt
@@ -0,0 +1,170 @@
+# RUN: llvm-mc --disassemble %s -triple=i386-unknown-unknown | FileCheck %s --check-prefixes=ATT
+# RUN: llvm-mc --disassemble %s -triple=i386-unknown-unknown -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL
+
+# ATT:        vsm4key4 %zmm4, %zmm3, %zmm2
+# INTEL:      vsm4key4 zmm2, zmm3, zmm4
+0x62,0xf2,0x66,0x48,0xda,0xd4
+
+# ATT:        vsm4key4  268435456(%esp,%esi,8), %zmm3, %zmm2
+# INTEL:      vsm4key4 zmm2, zmm3, zmmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x66,0x48,0xda,0x94,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:        vsm4key4  291(%edi,%eax,4), %zmm3, %zmm2
+# INTEL:      vsm4key4 zmm2, zmm3, zmmword ptr [edi + 4*eax + 291]
+0x62,0xf2,0x66,0x48,0xda,0x94,0x87,0x23,0x01,0x00,0x00
+
+# ATT:        vsm4key4  (%eax), %zmm3, %zmm2
+# INTEL:      vsm4key4 zmm2, zmm3, zmmword ptr [eax]
+0x62,0xf2,0x66,0x48,0xda,0x10
+
+# ATT:        vsm4key4  -2048(,%ebp,2), %zmm3, %zmm2
+# INTEL:      vsm4key4 zmm2, zmm3, zmmword ptr [2*ebp - 2048]
+0x62,0xf2,0x66,0x48,0xda,0x14,0x6d,0x00,0xf8,0xff,0xff
+
+# ATT:        vsm4key4  8128(%ecx), %zmm3, %zmm2
+# INTEL:      vsm4key4 zmm2, zmm3, zmmword ptr [ecx + 8128]
+0x62,0xf2,0x66,0x48,0xda,0x51,0x7f
+
+# ATT:        vsm4key4  -8192(%edx), %zmm3, %zmm2
+# INTEL:      vsm4key4 zmm2, zmm3, zmmword ptr [edx - 8192]
+0x62,0xf2,0x66,0x48,0xda,0x52,0x80
+
+# ATT:        vsm4rnds4 %zmm4, %zmm3, %zmm2
+# INTEL:      vsm4rnds4 zmm2, zmm3, zmm4
+0x62,0xf2,0x67,0x48,0xda,0xd4
+
+# ATT:        vsm4rnds4  268435456(%esp,%esi,8), %zmm3, %zmm2
+# INTEL:      vsm4rnds4 zmm2, zmm3, zmmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x67,0x48,0xda,0x94,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:        vsm4rnds4  291(%edi,%eax,4), %zmm3, %zmm2
+# INTEL:      vsm4rnds4 zmm2, zmm3, zmmword ptr [edi + 4*eax + 291]
+0x62,0xf2,0x67,0x48,0xda,0x94,0x87,0x23,0x01,0x00,0x00
+
+# ATT:        vsm4rnds4  (%eax), %zmm3, %zmm2
+# INTEL:      vsm4rnds4 zmm2, zmm3, zmmword ptr [eax]
+0x62,0xf2,0x67,0x48,0xda,0x10
+
+# ATT:        vsm4rnds4  -2048(,%ebp,2), %zmm3, %zmm2
+# INTEL:      vsm4rnds4 zmm2, zmm3, zmmword ptr [2*ebp - 2048]
+0x62,0xf2,0x67,0x48,0xda,0x14,0x6d,0x00,0xf8,0xff,0xff
+
+# ATT:        vsm4rnds4  8128(%ecx), %zmm3, %zmm2
+# INTEL:      vsm4rnds4 zmm2, zmm3, zmmword ptr [ecx + 8128]
+0x62,0xf2,0x67,0x48,0xda,0x51,0x7f
+
+# ATT:        vsm4rnds4  -8192(%edx), %zmm3, %zmm2
+# INTEL:      vsm4rnds4 zmm2, zmm3, zmmword ptr [edx - 8192]
+0x62,0xf2,0x67,0x48,0xda,0x52,0x80
+
+# ATT:        vsm4key4 %ymm4, %ymm3, %ymm2
+# INTEL:      vsm4key4 ymm2, ymm3, ymm4
+0x62,0xf2,0x66,0x28,0xda,0xd4
+
+# ATT:        vsm4key4 %xmm4, %xmm3, %xmm2
+# INTEL:      vsm4key4 xmm2, xmm3, xmm4
+0x62,0xf2,0x66,0x08,0xda,0xd4
+
+# ATT:        vsm4key4  268435456(%esp,%esi,8), %ymm3, %ymm2
+# INTEL:      vsm4key4 ymm2, ymm3, ymmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x66,0x28,0xda,0x94,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:        vsm4key4  291(%edi,%eax,4), %ymm3, %ymm2
+# INTEL:      vsm4key4 ymm2, ymm3, ymmword ptr [edi + 4*eax + 291]
+0x62,0xf2,0x66,0x28,0xda,0x94,0x87,0x23,0x01,0x00,0x00
+
+# ATT:        vsm4key4  (%eax), %ymm3, %ymm2
+# INTEL:      vsm4key4 ymm2, ymm3, ymmword ptr [eax]
+0x62,0xf2,0x66,0x28,0xda,0x10
+
+# ATT:        vsm4key4  -1024(,%ebp,2), %ymm3, %ymm2
+# INTEL:      vsm4key4 ymm2, ymm3, ymmword ptr [2*ebp - 1024]
+0x62,0xf2,0x66,0x28,0xda,0x14,0x6d,0x00,0xfc,0xff,0xff
+
+# ATT:        vsm4key4  4064(%ecx), %ymm3, %ymm2
+# INTEL:      vsm4key4 ymm2, ymm3, ymmword ptr [ecx + 4064]
+0x62,0xf2,0x66,0x28,0xda,0x51,0x7f
+
+# ATT:        vsm4key4  -4096(%edx), %ymm3, %ymm2
+# INTEL:      vsm4key4 ymm2, ymm3, ymmword ptr [edx - 4096]
+0x62,0xf2,0x66,0x28,0xda,0x52,0x80
+
+# ATT:        vsm4key4  268435456(%esp,%esi,8), %xmm3, %xmm2
+# INTEL:      vsm4key4 xmm2, xmm3, xmmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x66,0x08,0xda,0x94,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:        vsm4key4  291(%edi,%eax,4), %xmm3, %xmm2
+# INTEL:      vsm4key4 xmm2, xmm3, xmmword ptr [edi + 4*eax + 291]
+0x62,0xf2,0x66,0x08,0xda,0x94,0x87,0x23,0x01,0x00,0x00
+
+# ATT:        vsm4key4  (%eax), %xmm3, %xmm2
+# INTEL:      vsm4key4 xmm2, xmm3, xmmword ptr [eax]
+0x62,0xf2,0x66,0x08,0xda,0x10
+
+# ATT:        vsm4key4  -512(,%ebp,2), %xmm3, %xmm2
+# INTEL:      vsm4key4 xmm2, xmm3, xmmword ptr [2*ebp - 512]
+0x62,0xf2,0x66,0x08,0xda,0x14,0x6d,0x00,0xfe,0xff,0xff
+
+# ATT:        vsm4key4  2032(%ecx), %xmm3, %xmm2
+# INTEL:      vsm4key4 xmm2, xmm3, xmmword ptr [ecx + 2032]
+0x62,0xf2,0x66,0x08,0xda,0x51,0x7f
+
+# ATT:        vsm4key4  -2048(%edx), %xmm3, %xmm2
+# INTEL:      vsm4key4 xmm2, xmm3, xmmword ptr [edx - 2048]
+0x62,0xf2,0x66,0x08,0xda,0x52,0x80
+
+# ATT:        vsm4rnds4 %ymm4, %ymm3, %ymm2
+# INTEL:      vsm4rnds4 ymm2, ymm3, ymm4
+0x62,0xf2,0x67,0x28,0xda,0xd4
+
+# ATT:        vsm4rnds4 %xmm4, %xmm3, %xmm2
+# INTEL:      vsm4rnds4 xmm2, xmm3, xmm4
+0x62,0xf2,0x67,0x08,0xda,0xd4
+
+# ATT:        vsm4rnds4  268435456(%esp,%esi,8), %ymm3, %ymm2
+# INTEL:      vsm4rnds4 ymm2, ymm3, ymmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x67,0x28,0xda,0x94,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:        vsm4rnds4  291(%edi,%eax,4), %ymm3, %ymm2
+# INTEL:      vsm4rnds4 ymm2, ymm3, ymmword ptr [edi + 4*eax + 291]
+0x62,0xf2,0x67,0x28,0xda,0x94,0x87,0x23,0x01,0x00,0x00
+
+# ATT:        vsm4rnds4  (%eax), %ymm3, %ymm2
+# INTEL:      vsm4rnds4 ymm2, ymm3, ymmword ptr [eax]
+0x62,0xf2,0x67,0x28,0xda,0x10
+
+# ATT:        vsm4rnds4  -1024(,%ebp,2), %ymm3, %ymm2
+# INTEL:      vsm4rnds4 ymm2, ymm3, ymmword ptr [2*ebp - 1024]
+0x62,0xf2,0x67,0x28,0xda,0x14,0x6d,0x00,0xfc,0xff,0xff
+
+# ATT:        vsm4rnds4  4064(%ecx), %ymm3, %ymm2
+# INTEL:      vsm4rnds4 ymm2, ymm3, ymmword ptr [ecx + 4064]
+0x62,0xf2,0x67,0x28,0xda,0x51,0x7f
+
+# ATT:        vsm4rnds4  -4096(%edx), %ymm3, %ymm2
+# INTEL:      vsm4rnds4 ymm2, ymm3, ymmword ptr [edx - 4096]
+0x62,0xf2,0x67,0x28,0xda,0x52,0x80
+
+# ATT:        vsm4rnds4  268435456(%esp,%esi,8), %xmm3, %xmm2
+# INTEL:      vsm4rnds4 xmm2, xmm3, xmmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x67,0x08,0xda,0x94,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:        vsm4rnds4  291(%edi,%eax,4), %xmm3, %xmm2
+# INTEL:      vsm4rnds4 xmm2, xmm3, xmmword ptr [edi + 4*eax + 291]
+0x62,0xf2,0x67,0x08,0xda,0x94,0x87,0x23,0x01,0x00,0x00
+
+# ATT:        vsm4rnds4  (%eax), %xmm3, %xmm2
+# INTEL:      vsm4rnds4 xmm2, xmm3, xmmword ptr [eax]
+0x62,0xf2,0x67,0x08,0xda,0x10
+
+# ATT:        vsm4rnds4  -512(,%ebp,2), %xmm3, %xmm2
+# INTEL:      vsm4rnds4 xmm2, xmm3, xmmword ptr [2*ebp - 512]
+0x62,0xf2,0x67,0x08,0xda,0x14,0x6d,0x00,0xfe,0xff,0xff
+
+# ATT:        vsm4rnds4  2032(%ecx), %xmm3, %xmm2
+# INTEL:      vsm4rnds4 xmm2, xmm3, xmmword ptr [ecx + 2032]
+0x62,0xf2,0x67,0x08,0xda,0x51,0x7f
+
+# ATT:        vsm4rnds4  -2048(%edx), %xmm3, %xmm2
+# INTEL:      vsm4rnds4 xmm2, xmm3, xmmword ptr [edx - 2048]
+0x62,0xf2,0x67,0x08,0xda,0x52,0x80
diff --git a/llvm/test/MC/Disassembler/X86/sm4-evex-64.txt b/llvm/test/MC/Disassembler/X86/sm4-evex-64.txt
new file mode 100644
index 000000000000..f89f4b5a8c0f
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/sm4-evex-64.txt
@@ -0,0 +1,170 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64 | FileCheck %s --check-prefixes=ATT
+# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL
+
+# ATT:        vsm4key4 %zmm24, %zmm23, %zmm22
+# INTEL:      vsm4key4 zmm22, zmm23, zmm24
+0x62,0x82,0x46,0x40,0xda,0xf0
+
+# ATT:        vsm4key4  268435456(%rbp,%r14,8), %zmm23, %zmm22
+# INTEL:      vsm4key4 zmm22, zmm23, zmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xa2,0x46,0x40,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:        vsm4key4  291(%r8,%rax,4), %zmm23, %zmm22
+# INTEL:      vsm4key4 zmm22, zmm23, zmmword ptr [r8 + 4*rax + 291]
+0x62,0xc2,0x46,0x40,0xda,0xb4,0x80,0x23,0x01,0x00,0x00
+
+# ATT:        vsm4key4  (%rip), %zmm23, %zmm22
+# INTEL:      vsm4key4 zmm22, zmm23, zmmword ptr [rip]
+0x62,0xe2,0x46,0x40,0xda,0x35,0x00,0x00,0x00,0x00
+
+# ATT:        vsm4key4  -2048(,%rbp,2), %zmm23, %zmm22
+# INTEL:      vsm4key4 zmm22, zmm23, zmmword ptr [2*rbp - 2048]
+0x62,0xe2,0x46,0x40,0xda,0x34,0x6d,0x00,0xf8,0xff,0xff
+
+# ATT:        vsm4key4  8128(%rcx), %zmm23, %zmm22
+# INTEL:      vsm4key4 zmm22, zmm23, zmmword ptr [rcx + 8128]
+0x62,0xe2,0x46,0x40,0xda,0x71,0x7f
+
+# ATT:        vsm4key4  -8192(%rdx), %zmm23, %zmm22
+# INTEL:      vsm4key4 zmm22, zmm23, zmmword ptr [rdx - 8192]
+0x62,0xe2,0x46,0x40,0xda,0x72,0x80
+
+# ATT:        vsm4rnds4 %zmm24, %zmm23, %zmm22
+# INTEL:      vsm4rnds4 zmm22, zmm23, zmm24
+0x62,0x82,0x47,0x40,0xda,0xf0
+
+# ATT:        vsm4rnds4  268435456(%rbp,%r14,8), %zmm23, %zmm22
+# INTEL:      vsm4rnds4 zmm22, zmm23, zmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xa2,0x47,0x40,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:        vsm4rnds4  291(%r8,%rax,4), %zmm23, %zmm22
+# INTEL:      vsm4rnds4 zmm22, zmm23, zmmword ptr [r8 + 4*rax + 291]
+0x62,0xc2,0x47,0x40,0xda,0xb4,0x80,0x23,0x01,0x00,0x00
+
+# ATT:        vsm4rnds4  (%rip), %zmm23, %zmm22
+# INTEL:      vsm4rnds4 zmm22, zmm23, zmmword ptr [rip]
+0x62,0xe2,0x47,0x40,0xda,0x35,0x00,0x00,0x00,0x00
+
+# ATT:        vsm4rnds4  -2048(,%rbp,2), %zmm23, %zmm22
+# INTEL:      vsm4rnds4 zmm22, zmm23, zmmword ptr [2*rbp - 2048]
+0x62,0xe2,0x47,0x40,0xda,0x34,0x6d,0x00,0xf8,0xff,0xff
+
+# ATT:        vsm4rnds4  8128(%rcx), %zmm23, %zmm22
+# INTEL:      vsm4rnds4 zmm22, zmm23, zmmword ptr [rcx + 8128]
+0x62,0xe2,0x47,0x40,0xda,0x71,0x7f
+
+# ATT:        vsm4rnds4  -8192(%rdx), %zmm23, %zmm22
+# INTEL:      vsm4rnds4 zmm22, zmm23, zmmword ptr [rdx - 8192]
+0x62,0xe2,0x47,0x40,0xda,0x72,0x80
+
+# ATT:        vsm4key4 %ymm24, %ymm23, %ymm22
+# INTEL:      vsm4key4 ymm22, ymm23, ymm24
+0x62,0x82,0x46,0x20,0xda,0xf0
+
+# ATT:        vsm4key4 %xmm24, %xmm23, %xmm22
+# INTEL:      vsm4key4 xmm22, xmm23, xmm24
+0x62,0x82,0x46,0x00,0xda,0xf0
+
+# ATT:        vsm4key4  268435456(%rbp,%r14,8), %ymm23, %ymm22
+# INTEL:      vsm4key4 ymm22, ymm23, ymmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xa2,0x46,0x20,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:        vsm4key4  291(%r8,%rax,4), %ymm23, %ymm22
+# INTEL:      vsm4key4 ymm22, ymm23, ymmword ptr [r8 + 4*rax + 291]
+0x62,0xc2,0x46,0x20,0xda,0xb4,0x80,0x23,0x01,0x00,0x00
+
+# ATT:        vsm4key4  (%rip), %ymm23, %ymm22
+# INTEL:      vsm4key4 ymm22, ymm23, ymmword ptr [rip]
+0x62,0xe2,0x46,0x20,0xda,0x35,0x00,0x00,0x00,0x00
+
+# ATT:        vsm4key4  -1024(,%rbp,2), %ymm23, %ymm22
+# INTEL:      vsm4key4 ymm22, ymm23, ymmword ptr [2*rbp - 1024]
+0x62,0xe2,0x46,0x20,0xda,0x34,0x6d,0x00,0xfc,0xff,0xff
+
+# ATT:        vsm4key4  4064(%rcx), %ymm23, %ymm22
+# INTEL:      vsm4key4 ymm22, ymm23, ymmword ptr [rcx + 4064]
+0x62,0xe2,0x46,0x20,0xda,0x71,0x7f
+
+# ATT:        vsm4key4  -4096(%rdx), %ymm23, %ymm22
+# INTEL:      vsm4key4 ymm22, ymm23, ymmword ptr [rdx - 4096]
+0x62,0xe2,0x46,0x20,0xda,0x72,0x80
+
+# ATT:        vsm4key4  268435456(%rbp,%r14,8), %xmm23, %xmm22
+# INTEL:      vsm4key4 xmm22, xmm23, xmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xa2,0x46,0x00,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:        vsm4key4  291(%r8,%rax,4), %xmm23, %xmm22
+# INTEL:      vsm4key4 xmm22, xmm23, xmmword ptr [r8 + 4*rax + 291]
+0x62,0xc2,0x46,0x00,0xda,0xb4,0x80,0x23,0x01,0x00,0x00
+
+# ATT:        vsm4key4  (%rip), %xmm23, %xmm22
+# INTEL:      vsm4key4 xmm22, xmm23, xmmword ptr [rip]
+0x62,0xe2,0x46,0x00,0xda,0x35,0x00,0x00,0x00,0x00
+
+# ATT:        vsm4key4  -512(,%rbp,2), %xmm23, %xmm22
+# INTEL:      vsm4key4 xmm22, xmm23, xmmword ptr [2*rbp - 512]
+0x62,0xe2,0x46,0x00,0xda,0x34,0x6d,0x00,0xfe,0xff,0xff
+
+# ATT:        vsm4key4  2032(%rcx), %xmm23, %xmm22
+# INTEL:      vsm4key4 xmm22, xmm23, xmmword ptr [rcx + 2032]
+0x62,0xe2,0x46,0x00,0xda,0x71,0x7f
+
+# ATT:        vsm4key4  -2048(%rdx), %xmm23, %xmm22
+# INTEL:      vsm4key4 xmm22, xmm23, xmmword ptr [rdx - 2048]
+0x62,0xe2,0x46,0x00,0xda,0x72,0x80
+
+# ATT:        vsm4rnds4 %ymm24, %ymm23, %ymm22
+# INTEL:      vsm4rnds4 ymm22, ymm23, ymm24
+0x62,0x82,0x47,0x20,0xda,0xf0
+
+# ATT:        vsm4rnds4 %xmm24, %xmm23, %xmm22
+# INTEL:      vsm4rnds4 xmm22, xmm23, xmm24
+0x62,0x82,0x47,0x00,0xda,0xf0
+
+# ATT:        vsm4rnds4  268435456(%rbp,%r14,8), %ymm23, %ymm22
+# INTEL:      vsm4rnds4 ymm22, ymm23, ymmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xa2,0x47,0x20,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:        vsm4rnds4  291(%r8,%rax,4), %ymm23, %ymm22
+# INTEL:      vsm4rnds4 ymm22, ymm23, ymmword ptr [r8 + 4*rax + 291]
+0x62,0xc2,0x47,0x20,0xda,0xb4,0x80,0x23,0x01,0x00,0x00
+
+# ATT:        vsm4rnds4  (%rip), %ymm23, %ymm22
+# INTEL:      vsm4rnds4 ymm22, ymm23, ymmword ptr [rip]
+0x62,0xe2,0x47,0x20,0xda,0x35,0x00,0x00,0x00,0x00
+
+# ATT:        vsm4rnds4  -1024(,%rbp,2), %ymm23, %ymm22
+# INTEL:      vsm4rnds4 ymm22, ymm23, ymmword ptr [2*rbp - 1024]
+0x62,0xe2,0x47,0x20,0xda,0x34,0x6d,0x00,0xfc,0xff,0xff
+
+# ATT:        vsm4rnds4  4064(%rcx), %ymm23, %ymm22
+# INTEL:      vsm4rnds4 ymm22, ymm23, ymmword ptr [rcx + 4064]
+0x62,0xe2,0x47,0x20,0xda,0x71,0x7f
+
+# ATT:        vsm4rnds4  -4096(%rdx), %ymm23, %ymm22
+# INTEL:      vsm4rnds4 ymm22, ymm23, ymmword ptr [rdx - 4096]
+0x62,0xe2,0x47,0x20,0xda,0x72,0x80
+
+# ATT:        vsm4rnds4  268435456(%rbp,%r14,8), %xmm23, %xmm22
+# INTEL:      vsm4rnds4 xmm22, xmm23, xmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xa2,0x47,0x00,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:        vsm4rnds4  291(%r8,%rax,4), %xmm23, %xmm22
+# INTEL:      vsm4rnds4 xmm22, xmm23, xmmword ptr [r8 + 4*rax + 291]
+0x62,0xc2,0x47,0x00,0xda,0xb4,0x80,0x23,0x01,0x00,0x00
+
+# ATT:        vsm4rnds4  (%rip), %xmm23, %xmm22
+# INTEL:      vsm4rnds4 xmm22, xmm23, xmmword ptr [rip]
+0x62,0xe2,0x47,0x00,0xda,0x35,0x00,0x00,0x00,0x00
+
+# ATT:        vsm4rnds4  -512(,%rbp,2), %xmm23, %xmm22
+# INTEL:      vsm4rnds4 xmm22, xmm23, xmmword ptr [2*rbp - 512]
+0x62,0xe2,0x47,0x00,0xda,0x34,0x6d,0x00,0xfe,0xff,0xff
+
+# ATT:        vsm4rnds4  2032(%rcx), %xmm23, %xmm22
+# INTEL:      vsm4rnds4 xmm22, xmm23, xmmword ptr [rcx + 2032]
+0x62,0xe2,0x47,0x00,0xda,0x71,0x7f
+
+# ATT:        vsm4rnds4  -2048(%rdx), %xmm23, %xmm22
+# INTEL:      vsm4rnds4 xmm22, xmm23, xmmword ptr [rdx - 2048]
+0x62,0xe2,0x47,0x00,0xda,0x72,0x80
diff --git a/llvm/test/MC/X86/sm4-evex-32-att.s b/llvm/test/MC/X86/sm4-evex-32-att.s
new file mode 100644
index 000000000000..de10d95ac74d
--- /dev/null
+++ b/llvm/test/MC/X86/sm4-evex-32-att.s
@@ -0,0 +1,169 @@
+// RUN: llvm-mc -triple i686-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK:      vsm4key4 %zmm4, %zmm3, %zmm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0xda,0xd4]
+               vsm4key4 %zmm4, %zmm3, %zmm2
+
+// CHECK:      vsm4key4  268435456(%esp,%esi,8), %zmm3, %zmm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0xda,0x94,0xf4,0x00,0x00,0x00,0x10]
+               vsm4key4  268435456(%esp,%esi,8), %zmm3, %zmm2
+
+// CHECK:      vsm4key4  291(%edi,%eax,4), %zmm3, %zmm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0xda,0x94,0x87,0x23,0x01,0x00,0x00]
+               vsm4key4  291(%edi,%eax,4), %zmm3, %zmm2
+
+// CHECK:      vsm4key4  (%eax), %zmm3, %zmm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0xda,0x10]
+               vsm4key4  (%eax), %zmm3, %zmm2
+
+// CHECK:      vsm4key4  -2048(,%ebp,2), %zmm3, %zmm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0xda,0x14,0x6d,0x00,0xf8,0xff,0xff]
+               vsm4key4  -2048(,%ebp,2), %zmm3, %zmm2
+
+// CHECK:      vsm4key4  8128(%ecx), %zmm3, %zmm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0xda,0x51,0x7f]
+               vsm4key4  8128(%ecx), %zmm3, %zmm2
+
+// CHECK:      vsm4key4  -8192(%edx), %zmm3, %zmm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0xda,0x52,0x80]
+               vsm4key4  -8192(%edx), %zmm3, %zmm2
+
+// CHECK:      vsm4rnds4 %zmm4, %zmm3, %zmm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0xda,0xd4]
+               vsm4rnds4 %zmm4, %zmm3, %zmm2
+
+// CHECK:      vsm4rnds4  268435456(%esp,%esi,8), %zmm3, %zmm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0xda,0x94,0xf4,0x00,0x00,0x00,0x10]
+               vsm4rnds4  268435456(%esp,%esi,8), %zmm3, %zmm2
+
+// CHECK:      vsm4rnds4  291(%edi,%eax,4), %zmm3, %zmm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0xda,0x94,0x87,0x23,0x01,0x00,0x00]
+               vsm4rnds4  291(%edi,%eax,4), %zmm3, %zmm2
+
+// CHECK:      vsm4rnds4  (%eax), %zmm3, %zmm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0xda,0x10]
+               vsm4rnds4  (%eax), %zmm3, %zmm2
+
+// CHECK:      vsm4rnds4  -2048(,%ebp,2), %zmm3, %zmm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0xda,0x14,0x6d,0x00,0xf8,0xff,0xff]
+               vsm4rnds4  -2048(,%ebp,2), %zmm3, %zmm2
+
+// CHECK:      vsm4rnds4  8128(%ecx), %zmm3, %zmm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0xda,0x51,0x7f]
+               vsm4rnds4  8128(%ecx), %zmm3, %zmm2
+
+// CHECK:      vsm4rnds4  -8192(%edx), %zmm3, %zmm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0xda,0x52,0x80]
+               vsm4rnds4  -8192(%edx), %zmm3, %zmm2
+
+// CHECK:      {evex} vsm4key4 %ymm4, %ymm3, %ymm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x28,0xda,0xd4]
+               {evex} vsm4key4 %ymm4, %ymm3, %ymm2
+
+// CHECK:      {evex} vsm4key4 %xmm4, %xmm3, %xmm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x08,0xda,0xd4]
+               {evex} vsm4key4 %xmm4, %xmm3, %xmm2
+
+// CHECK:      {evex} vsm4key4  268435456(%esp,%esi,8), %ymm3, %ymm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x28,0xda,0x94,0xf4,0x00,0x00,0x00,0x10]
+               {evex} vsm4key4  268435456(%esp,%esi,8), %ymm3, %ymm2
+
+// CHECK:      {evex} vsm4key4  291(%edi,%eax,4), %ymm3, %ymm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x28,0xda,0x94,0x87,0x23,0x01,0x00,0x00]
+               {evex} vsm4key4  291(%edi,%eax,4), %ymm3, %ymm2
+
+// CHECK:      {evex} vsm4key4  (%eax), %ymm3, %ymm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x28,0xda,0x10]
+               {evex} vsm4key4  (%eax), %ymm3, %ymm2
+
+// CHECK:      {evex} vsm4key4  -1024(,%ebp,2), %ymm3, %ymm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x28,0xda,0x14,0x6d,0x00,0xfc,0xff,0xff]
+               {evex} vsm4key4  -1024(,%ebp,2), %ymm3, %ymm2
+
+// CHECK:      {evex} vsm4key4  4064(%ecx), %ymm3, %ymm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x28,0xda,0x51,0x7f]
+               {evex} vsm4key4  4064(%ecx), %ymm3, %ymm2
+
+// CHECK:      {evex} vsm4key4  -4096(%edx), %ymm3, %ymm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x28,0xda,0x52,0x80]
+               {evex} vsm4key4  -4096(%edx), %ymm3, %ymm2
+
+// CHECK:      {evex} vsm4key4  268435456(%esp,%esi,8), %xmm3, %xmm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x08,0xda,0x94,0xf4,0x00,0x00,0x00,0x10]
+               {evex} vsm4key4  268435456(%esp,%esi,8), %xmm3, %xmm2
+
+// CHECK:      {evex} vsm4key4  291(%edi,%eax,4), %xmm3, %xmm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x08,0xda,0x94,0x87,0x23,0x01,0x00,0x00]
+               {evex} vsm4key4  291(%edi,%eax,4), %xmm3, %xmm2
+
+// CHECK:      {evex} vsm4key4  (%eax), %xmm3, %xmm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x08,0xda,0x10]
+               {evex} vsm4key4  (%eax), %xmm3, %xmm2
+
+// CHECK:      {evex} vsm4key4  -512(,%ebp,2), %xmm3, %xmm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x08,0xda,0x14,0x6d,0x00,0xfe,0xff,0xff]
+               {evex} vsm4key4  -512(,%ebp,2), %xmm3, %xmm2
+
+// CHECK:      {evex} vsm4key4  2032(%ecx), %xmm3, %xmm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x08,0xda,0x51,0x7f]
+               {evex} vsm4key4  2032(%ecx), %xmm3, %xmm2
+
+// CHECK:      {evex} vsm4key4  -2048(%edx), %xmm3, %xmm2
+// CHECK: encoding: [0x62,0xf2,0x66,0x08,0xda,0x52,0x80]
+               {evex} vsm4key4  -2048(%edx), %xmm3, %xmm2
+
+// CHECK:      {evex} vsm4rnds4 %ymm4, %ymm3, %ymm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x28,0xda,0xd4]
+               {evex} vsm4rnds4 %ymm4, %ymm3, %ymm2
+
+// CHECK:      {evex} vsm4rnds4 %xmm4, %xmm3, %xmm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x08,0xda,0xd4]
+               {evex} vsm4rnds4 %xmm4, %xmm3, %xmm2
+
+// CHECK:      {evex} vsm4rnds4  268435456(%esp,%esi,8), %ymm3, %ymm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x28,0xda,0x94,0xf4,0x00,0x00,0x00,0x10]
+               {evex} vsm4rnds4  268435456(%esp,%esi,8), %ymm3, %ymm2
+
+// CHECK:      {evex} vsm4rnds4  291(%edi,%eax,4), %ymm3, %ymm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x28,0xda,0x94,0x87,0x23,0x01,0x00,0x00]
+               {evex} vsm4rnds4  291(%edi,%eax,4), %ymm3, %ymm2
+
+// CHECK:      {evex} vsm4rnds4  (%eax), %ymm3, %ymm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x28,0xda,0x10]
+               {evex} vsm4rnds4  (%eax), %ymm3, %ymm2
+
+// CHECK:      {evex} vsm4rnds4  -1024(,%ebp,2), %ymm3, %ymm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x28,0xda,0x14,0x6d,0x00,0xfc,0xff,0xff]
+               {evex} vsm4rnds4  -1024(,%ebp,2), %ymm3, %ymm2
+
+// CHECK:      {evex} vsm4rnds4  4064(%ecx), %ymm3, %ymm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x28,0xda,0x51,0x7f]
+               {evex} vsm4rnds4  4064(%ecx), %ymm3, %ymm2
+
+// CHECK:      {evex} vsm4rnds4  -4096(%edx), %ymm3, %ymm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x28,0xda,0x52,0x80]
+               {evex} vsm4rnds4  -4096(%edx), %ymm3, %ymm2
+
+// CHECK:      {evex} vsm4rnds4  268435456(%esp,%esi,8), %xmm3, %xmm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x08,0xda,0x94,0xf4,0x00,0x00,0x00,0x10]
+               {evex} vsm4rnds4  268435456(%esp,%esi,8), %xmm3, %xmm2
+
+// CHECK:      {evex} vsm4rnds4  291(%edi,%eax,4), %xmm3, %xmm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x08,0xda,0x94,0x87,0x23,0x01,0x00,0x00]
+               {evex} vsm4rnds4  291(%edi,%eax,4), %xmm3, %xmm2
+
+// CHECK:      {evex} vsm4rnds4  (%eax), %xmm3, %xmm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x08,0xda,0x10]
+               {evex} vsm4rnds4  (%eax), %xmm3, %xmm2
+
+// CHECK:      {evex} vsm4rnds4  -512(,%ebp,2), %xmm3, %xmm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x08,0xda,0x14,0x6d,0x00,0xfe,0xff,0xff]
+               {evex} vsm4rnds4  -512(,%ebp,2), %xmm3, %xmm2
+
+// CHECK:      {evex} vsm4rnds4  2032(%ecx), %xmm3, %xmm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x08,0xda,0x51,0x7f]
+               {evex} vsm4rnds4  2032(%ecx), %xmm3, %xmm2
+
+// CHECK:      {evex} vsm4rnds4  -2048(%edx), %xmm3, %xmm2
+// CHECK: encoding: [0x62,0xf2,0x67,0x08,0xda,0x52,0x80]
+               {evex} vsm4rnds4  -2048(%edx), %xmm3, %xmm2
\ No newline at end of file
diff --git a/llvm/test/MC/X86/sm4-evex-32-intel.s b/llvm/test/MC/X86/sm4-evex-32-intel.s
new file mode 100644
index 000000000000..812fdb13f809
--- /dev/null
+++ b/llvm/test/MC/X86/sm4-evex-32-intel.s
@@ -0,0 +1,169 @@
+// RUN: llvm-mc -triple i686-unknown-unknown -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
+
+// CHECK:      vsm4key4 zmm2, zmm3, zmm4
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0xda,0xd4]
+               vsm4key4 zmm2, zmm3, zmm4
+
+// CHECK:      vsm4key4 zmm2, zmm3, zmmword ptr [esp + 8*esi + 268435456]
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0xda,0x94,0xf4,0x00,0x00,0x00,0x10]
+               vsm4key4 zmm2, zmm3, zmmword ptr [esp + 8*esi + 268435456]
+
+// CHECK:      vsm4key4 zmm2, zmm3, zmmword ptr [edi + 4*eax + 291]
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0xda,0x94,0x87,0x23,0x01,0x00,0x00]
+               vsm4key4 zmm2, zmm3, zmmword ptr [edi + 4*eax + 291]
+
+// CHECK:      vsm4key4 zmm2, zmm3, zmmword ptr [eax]
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0xda,0x10]
+               vsm4key4 zmm2, zmm3, zmmword ptr [eax]
+
+// CHECK:      vsm4key4 zmm2, zmm3, zmmword ptr [2*ebp - 2048]
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0xda,0x14,0x6d,0x00,0xf8,0xff,0xff]
+               vsm4key4 zmm2, zmm3, zmmword ptr [2*ebp - 2048]
+
+// CHECK:      vsm4key4 zmm2, zmm3, zmmword ptr [ecx + 8128]
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0xda,0x51,0x7f]
+               vsm4key4 zmm2, zmm3, zmmword ptr [ecx + 8128]
+
+// CHECK:      vsm4key4 zmm2, zmm3, zmmword ptr [edx - 8192]
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0xda,0x52,0x80]
+               vsm4key4 zmm2, zmm3, zmmword ptr [edx - 8192]
+
+// CHECK:      vsm4rnds4 zmm2, zmm3, zmm4
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0xda,0xd4]
+               vsm4rnds4 zmm2, zmm3, zmm4
+
+// CHECK:      vsm4rnds4 zmm2, zmm3, zmmword ptr [esp + 8*esi + 268435456]
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0xda,0x94,0xf4,0x00,0x00,0x00,0x10]
+               vsm4rnds4 zmm2, zmm3, zmmword ptr [esp + 8*esi + 268435456]
+
+// CHECK:      vsm4rnds4 zmm2, zmm3, zmmword ptr [edi + 4*eax + 291]
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0xda,0x94,0x87,0x23,0x01,0x00,0x00]
+               vsm4rnds4 zmm2, zmm3, zmmword ptr [edi + 4*eax + 291]
+
+// CHECK:      vsm4rnds4 zmm2, zmm3, zmmword ptr [eax]
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0xda,0x10]
+               vsm4rnds4 zmm2, zmm3, zmmword ptr [eax]
+
+// CHECK:      vsm4rnds4 zmm2, zmm3, zmmword ptr [2*ebp - 2048]
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0xda,0x14,0x6d,0x00,0xf8,0xff,0xff]
+               vsm4rnds4 zmm2, zmm3, zmmword ptr [2*ebp - 2048]
+
+// CHECK:      vsm4rnds4 zmm2, zmm3, zmmword ptr [ecx + 8128]
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0xda,0x51,0x7f]
+               vsm4rnds4 zmm2, zmm3, zmmword ptr [ecx + 8128]
+
+// CHECK:      vsm4rnds4 zmm2, zmm3, zmmword ptr [edx - 8192]
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0xda,0x52,0x80]
+               vsm4rnds4 zmm2, zmm3, zmmword ptr [edx - 8192]
+
+// CHECK:      {evex} vsm4key4 ymm2, ymm3, ymm4
+// CHECK: encoding: [0x62,0xf2,0x66,0x28,0xda,0xd4]
+               {evex} vsm4key4 ymm2, ymm3, ymm4
+
+// CHECK:      {evex} vsm4key4 xmm2, xmm3, xmm4
+// CHECK: encoding: [0x62,0xf2,0x66,0x08,0xda,0xd4]
+               {evex} vsm4key4 xmm2, xmm3, xmm4
+
+// CHECK:      {evex} vsm4key4 ymm2, ymm3, ymmword ptr [esp + 8*esi + 268435456]
+// CHECK: encoding: [0x62,0xf2,0x66,0x28,0xda,0x94,0xf4,0x00,0x00,0x00,0x10]
+               {evex} vsm4key4 ymm2, ymm3, ymmword ptr [esp + 8*esi + 268435456]
+
+// CHECK:      {evex} vsm4key4 ymm2, ymm3, ymmword ptr [edi + 4*eax + 291]
+// CHECK: encoding: [0x62,0xf2,0x66,0x28,0xda,0x94,0x87,0x23,0x01,0x00,0x00]
+               {evex} vsm4key4 ymm2, ymm3, ymmword ptr [edi + 4*eax + 291]
+
+// CHECK:      {evex} vsm4key4 ymm2, ymm3, ymmword ptr [eax]
+// CHECK: encoding: [0x62,0xf2,0x66,0x28,0xda,0x10]
+               {evex} vsm4key4 ymm2, ymm3, ymmword ptr [eax]
+
+// CHECK:      {evex} vsm4key4 ymm2, ymm3, ymmword ptr [2*ebp - 1024]
+// CHECK: encoding: [0x62,0xf2,0x66,0x28,0xda,0x14,0x6d,0x00,0xfc,0xff,0xff]
+               {evex} vsm4key4 ymm2, ymm3, ymmword ptr [2*ebp - 1024]
+
+// CHECK:      {evex} vsm4key4 ymm2, ymm3, ymmword ptr [ecx + 4064]
+// CHECK: encoding: [0x62,0xf2,0x66,0x28,0xda,0x51,0x7f]
+               {evex} vsm4key4 ymm2, ymm3, ymmword ptr [ecx + 4064]
+
+// CHECK:      {evex} vsm4key4 ymm2, ymm3, ymmword ptr [edx - 4096]
+// CHECK: encoding: [0x62,0xf2,0x66,0x28,0xda,0x52,0x80]
+               {evex} vsm4key4 ymm2, ymm3, ymmword ptr [edx - 4096]
+
+// CHECK:      {evex} vsm4key4 xmm2, xmm3, xmmword ptr [esp + 8*esi + 268435456]
+// CHECK: encoding: [0x62,0xf2,0x66,0x08,0xda,0x94,0xf4,0x00,0x00,0x00,0x10]
+               {evex} vsm4key4 xmm2, xmm3, xmmword ptr [esp + 8*esi + 268435456]
+
+// CHECK:      {evex} vsm4key4 xmm2, xmm3, xmmword ptr [edi + 4*eax + 291]
+// CHECK: encoding: [0x62,0xf2,0x66,0x08,0xda,0x94,0x87,0x23,0x01,0x00,0x00]
+               {evex} vsm4key4 xmm2, xmm3, xmmword ptr [edi + 4*eax + 291]
+
+// CHECK:      {evex} vsm4key4 xmm2, xmm3, xmmword ptr [eax]
+// CHECK: encoding: [0x62,0xf2,0x66,0x08,0xda,0x10]
+               {evex} vsm4key4 xmm2, xmm3, xmmword ptr [eax]
+
+// CHECK:      {evex} vsm4key4 xmm2, xmm3, xmmword ptr [2*ebp - 512]
+// CHECK: encoding: [0x62,0xf2,0x66,0x08,0xda,0x14,0x6d,0x00,0xfe,0xff,0xff]
+               {evex} vsm4key4 xmm2, xmm3, xmmword ptr [2*ebp - 512]
+
+// CHECK:      {evex} vsm4key4 xmm2, xmm3, xmmword ptr [ecx + 2032]
+// CHECK: encoding: [0x62,0xf2,0x66,0x08,0xda,0x51,0x7f]
+               {evex} vsm4key4 xmm2, xmm3, xmmword ptr [ecx + 2032]
+
+// CHECK:      {evex} vsm4key4 xmm2, xmm3, xmmword ptr [edx - 2048]
+// CHECK: encoding: [0x62,0xf2,0x66,0x08,0xda,0x52,0x80]
+               {evex} vsm4key4 xmm2, xmm3, xmmword ptr [edx - 2048]
+
+// CHECK:      {evex} vsm4rnds4 ymm2, ymm3, ymm4
+// CHECK: encoding: [0x62,0xf2,0x67,0x28,0xda,0xd4]
+               {evex} vsm4rnds4 ymm2, ymm3, ymm4
+
+// CHECK:      {evex} vsm4rnds4 xmm2, xmm3, xmm4
+// CHECK: encoding: [0x62,0xf2,0x67,0x08,0xda,0xd4]
+               {evex} vsm4rnds4 xmm2, xmm3, xmm4
+
+// CHECK:      {evex} vsm4rnds4 ymm2, ymm3, ymmword ptr [esp + 8*esi + 268435456]
+// CHECK: encoding: [0x62,0xf2,0x67,0x28,0xda,0x94,0xf4,0x00,0x00,0x00,0x10]
+               {evex} vsm4rnds4 ymm2, ymm3, ymmword ptr [esp + 8*esi + 268435456]
+
+// CHECK:      {evex} vsm4rnds4 ymm2, ymm3, ymmword ptr [edi + 4*eax + 291]
+// CHECK: encoding: [0x62,0xf2,0x67,0x28,0xda,0x94,0x87,0x23,0x01,0x00,0x00]
+               {evex} vsm4rnds4 ymm2, ymm3, ymmword ptr [edi + 4*eax + 291]
+
+// CHECK:      {evex} vsm4rnds4 ymm2, ymm3, ymmword ptr [eax]
+// CHECK: encoding: [0x62,0xf2,0x67,0x28,0xda,0x10]
+               {evex} vsm4rnds4 ymm2, ymm3, ymmword ptr [eax]
+
+// CHECK:      {evex} vsm4rnds4 ymm2, ymm3, ymmword ptr [2*ebp - 1024]
+// CHECK: encoding: [0x62,0xf2,0x67,0x28,0xda,0x14,0x6d,0x00,0xfc,0xff,0xff]
+               {evex} vsm4rnds4 ymm2, ymm3, ymmword ptr [2*ebp - 1024]
+
+// CHECK:      {evex} vsm4rnds4 ymm2, ymm3, ymmword ptr [ecx + 4064]
+// CHECK: encoding: [0x62,0xf2,0x67,0x28,0xda,0x51,0x7f]
+               {evex} vsm4rnds4 ymm2, ymm3, ymmword ptr [ecx + 4064]
+
+// CHECK:      {evex} vsm4rnds4 ymm2, ymm3, ymmword ptr [edx - 4096]
+// CHECK: encoding: [0x62,0xf2,0x67,0x28,0xda,0x52,0x80]
+               {evex} vsm4rnds4 ymm2, ymm3, ymmword ptr [edx - 4096]
+
+// CHECK:      {evex} vsm4rnds4 xmm2, xmm3, xmmword ptr [esp + 8*esi + 268435456]
+// CHECK: encoding: [0x62,0xf2,0x67,0x08,0xda,0x94,0xf4,0x00,0x00,0x00,0x10]
+               {evex} vsm4rnds4 xmm2, xmm3, xmmword ptr [esp + 8*esi + 268435456]
+
+// CHECK:      {evex} vsm4rnds4 xmm2, xmm3, xmmword ptr [edi + 4*eax + 291]
+// CHECK: encoding: [0x62,0xf2,0x67,0x08,0xda,0x94,0x87,0x23,0x01,0x00,0x00]
+               {evex} vsm4rnds4 xmm2, xmm3, xmmword ptr [edi + 4*eax + 291]
+
+// CHECK:      {evex} vsm4rnds4 xmm2, xmm3, xmmword ptr [eax]
+// CHECK: encoding: [0x62,0xf2,0x67,0x08,0xda,0x10]
+               {evex} vsm4rnds4 xmm2, xmm3, xmmword ptr [eax]
+
+// CHECK:      {evex} vsm4rnds4 xmm2, xmm3, xmmword ptr [2*ebp - 512]
+// CHECK: encoding: [0x62,0xf2,0x67,0x08,0xda,0x14,0x6d,0x00,0xfe,0xff,0xff]
+               {evex} vsm4rnds4 xmm2, xmm3, xmmword ptr [2*ebp - 512]
+
+// CHECK:      {evex} vsm4rnds4 xmm2, xmm3, xmmword ptr [ecx + 2032]
+// CHECK: encoding: [0x62,0xf2,0x67,0x08,0xda,0x51,0x7f]
+               {evex} vsm4rnds4 xmm2, xmm3, xmmword ptr [ecx + 2032]
+
+// CHECK:      {evex} vsm4rnds4 xmm2, xmm3, xmmword ptr [edx - 2048]
+// CHECK: encoding: [0x62,0xf2,0x67,0x08,0xda,0x52,0x80]
+               {evex} vsm4rnds4 xmm2, xmm3, xmmword ptr [edx - 2048]
diff --git a/llvm/test/MC/X86/sm4-evex-64-att.s b/llvm/test/MC/X86/sm4-evex-64-att.s
new file mode 100644
index 000000000000..389a29b11897
--- /dev/null
+++ b/llvm/test/MC/X86/sm4-evex-64-att.s
@@ -0,0 +1,224 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK:      vsm4key4 %zmm24, %zmm23, %zmm22
+// CHECK: encoding: [0x62,0x82,0x46,0x40,0xda,0xf0]
+               vsm4key4 %zmm24, %zmm23, %zmm22
+
+// CHECK:      vsm4key4  268435456(%rbp,%r14,8), %zmm23, %zmm22
+// CHECK: encoding: [0x62,0xa2,0x46,0x40,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10]
+               vsm4key4  268435456(%rbp,%r14,8), %zmm23, %zmm22
+
+// CHECK:      vsm4key4  291(%r8,%rax,4), %zmm23, %zmm22
+// CHECK: encoding: [0x62,0xc2,0x46,0x40,0xda,0xb4,0x80,0x23,0x01,0x00,0x00]
+               vsm4key4  291(%r8,%rax,4), %zmm23, %zmm22
+
+// CHECK:      vsm4key4  (%rip), %zmm23, %zmm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x40,0xda,0x35,0x00,0x00,0x00,0x00]
+               vsm4key4  (%rip), %zmm23, %zmm22
+
+// CHECK:      vsm4key4  -2048(,%rbp,2), %zmm23, %zmm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x40,0xda,0x34,0x6d,0x00,0xf8,0xff,0xff]
+               vsm4key4  -2048(,%rbp,2), %zmm23, %zmm22
+
+// CHECK:      vsm4key4  8128(%rcx), %zmm23, %zmm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x40,0xda,0x71,0x7f]
+               vsm4key4  8128(%rcx), %zmm23, %zmm22
+
+// CHECK:      vsm4key4  -8192(%rdx), %zmm23, %zmm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x40,0xda,0x72,0x80]
+               vsm4key4  -8192(%rdx), %zmm23, %zmm22
+
+// CHECK:      vsm4rnds4 %zmm24, %zmm23, %zmm22
+// CHECK: encoding: [0x62,0x82,0x47,0x40,0xda,0xf0]
+               vsm4rnds4 %zmm24, %zmm23, %zmm22
+
+// CHECK:      vsm4rnds4  268435456(%rbp,%r14,8), %zmm23, %zmm22
+// CHECK: encoding: [0x62,0xa2,0x47,0x40,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10]
+               vsm4rnds4  268435456(%rbp,%r14,8), %zmm23, %zmm22
+
+// CHECK:      vsm4rnds4  291(%r8,%rax,4), %zmm23, %zmm22
+// CHECK: encoding: [0x62,0xc2,0x47,0x40,0xda,0xb4,0x80,0x23,0x01,0x00,0x00]
+               vsm4rnds4  291(%r8,%rax,4), %zmm23, %zmm22
+
+// CHECK:      vsm4rnds4  (%rip), %zmm23, %zmm22
+// CHECK: encoding: [0x62,0xe2,0x47,0x40,0xda,0x35,0x00,0x00,0x00,0x00]
+               vsm4rnds4  (%rip), %zmm23, %zmm22
+
+// CHECK:      vsm4rnds4  -2048(,%rbp,2), %zmm23, %zmm22
+// CHECK: encoding: [0x62,0xe2,0x47,0x40,0xda,0x34,0x6d,0x00,0xf8,0xff,0xff]
+               vsm4rnds4  -2048(,%rbp,2), %zmm23, %zmm22
+
+// CHECK:      vsm4rnds4  8128(%rcx), %zmm23, %zmm22
+// CHECK: encoding: [0x62,0xe2,0x47,0x40,0xda,0x71,0x7f]
+               vsm4rnds4  8128(%rcx), %zmm23, %zmm22
+
+// CHECK:      vsm4rnds4  -8192(%rdx), %zmm23, %zmm22
+// CHECK: encoding: [0x62,0xe2,0x47,0x40,0xda,0x72,0x80]
+               vsm4rnds4  -8192(%rdx), %zmm23, %zmm22
+
+// CHECK:      vsm4key4 %ymm24, %ymm23, %ymm22
+// CHECK: encoding: [0x62,0x82,0x46,0x20,0xda,0xf0]
+               vsm4key4 %ymm24, %ymm23, %ymm22
+
+// CHECK:      vsm4key4 %xmm24, %xmm23, %xmm22
+// CHECK: encoding: [0x62,0x82,0x46,0x00,0xda,0xf0]
+               vsm4key4 %xmm24, %xmm23, %xmm22
+
+// CHECK:      vsm4key4  268435456(%rbp,%r14,8), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xa2,0x46,0x20,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10]
+               vsm4key4  268435456(%rbp,%r14,8), %ymm23, %ymm22
+
+// CHECK:      vsm4key4  291(%r8,%rax,4), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xc2,0x46,0x20,0xda,0xb4,0x80,0x23,0x01,0x00,0x00]
+               vsm4key4  291(%r8,%rax,4), %ymm23, %ymm22
+
+// CHECK:      vsm4key4  (%rip), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x20,0xda,0x35,0x00,0x00,0x00,0x00]
+               vsm4key4  (%rip), %ymm23, %ymm22
+
+// CHECK:      vsm4key4  -1024(,%rbp,2), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x20,0xda,0x34,0x6d,0x00,0xfc,0xff,0xff]
+               vsm4key4  -1024(,%rbp,2), %ymm23, %ymm22
+
+// CHECK:      vsm4key4  4064(%rcx), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x20,0xda,0x71,0x7f]
+               vsm4key4  4064(%rcx), %ymm23, %ymm22
+
+// CHECK:      vsm4key4  -4096(%rdx), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x20,0xda,0x72,0x80]
+               vsm4key4  -4096(%rdx), %ymm23, %ymm22
+
+// CHECK:      vsm4key4  268435456(%rbp,%r14,8), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xa2,0x46,0x00,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10]
+               vsm4key4  268435456(%rbp,%r14,8), %xmm23, %xmm22
+
+// CHECK:      vsm4key4  291(%r8,%rax,4), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xc2,0x46,0x00,0xda,0xb4,0x80,0x23,0x01,0x00,0x00]
+               vsm4key4  291(%r8,%rax,4), %xmm23, %xmm22
+
+// CHECK:      vsm4key4  (%rip), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x00,0xda,0x35,0x00,0x00,0x00,0x00]
+               vsm4key4  (%rip), %xmm23, %xmm22
+
+// CHECK:      vsm4key4  -512(,%rbp,2), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x00,0xda,0x34,0x6d,0x00,0xfe,0xff,0xff]
+               vsm4key4  -512(,%rbp,2), %xmm23, %xmm22
+
+// CHECK:      vsm4key4  2032(%rcx), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x00,0xda,0x71,0x7f]
+               vsm4key4  2032(%rcx), %xmm23, %xmm22
+
+// CHECK:      vsm4key4  -2048(%rdx), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x00,0xda,0x72,0x80]
+               vsm4key4  -2048(%rdx), %xmm23, %xmm22
+// CHECK:      vsm4key4 %ymm24, %ymm23, %ymm22
+// CHECK: encoding: [0x62,0x82,0x46,0x20,0xda,0xf0]
+               vsm4key4 %ymm24, %ymm23, %ymm22
+
+// CHECK:      vsm4key4 %xmm24, %xmm23, %xmm22
+// CHECK: encoding: [0x62,0x82,0x46,0x00,0xda,0xf0]
+               vsm4key4 %xmm24, %xmm23, %xmm22
+
+// CHECK:      vsm4key4  268435456(%rbp,%r14,8), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xa2,0x46,0x20,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10]
+               vsm4key4  268435456(%rbp,%r14,8), %ymm23, %ymm22
+
+// CHECK:      vsm4key4  291(%r8,%rax,4), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xc2,0x46,0x20,0xda,0xb4,0x80,0x23,0x01,0x00,0x00]
+               vsm4key4  291(%r8,%rax,4), %ymm23, %ymm22
+
+// CHECK:      vsm4key4  (%rip), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x20,0xda,0x35,0x00,0x00,0x00,0x00]
+               vsm4key4  (%rip), %ymm23, %ymm22
+
+// CHECK:      vsm4key4  -1024(,%rbp,2), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x20,0xda,0x34,0x6d,0x00,0xfc,0xff,0xff]
+               vsm4key4  -1024(,%rbp,2), %ymm23, %ymm22
+
+// CHECK:      vsm4key4  4064(%rcx), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x20,0xda,0x71,0x7f]
+               vsm4key4  4064(%rcx), %ymm23, %ymm22
+
+// CHECK:      vsm4key4  -4096(%rdx), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x20,0xda,0x72,0x80]
+               vsm4key4  -4096(%rdx), %ymm23, %ymm22
+
+// CHECK:      vsm4key4  268435456(%rbp,%r14,8), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xa2,0x46,0x00,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10]
+               vsm4key4  268435456(%rbp,%r14,8), %xmm23, %xmm22
+
+// CHECK:      vsm4key4  291(%r8,%rax,4), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xc2,0x46,0x00,0xda,0xb4,0x80,0x23,0x01,0x00,0x00]
+               vsm4key4  291(%r8,%rax,4), %xmm23, %xmm22
+
+// CHECK:      vsm4key4  (%rip), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x00,0xda,0x35,0x00,0x00,0x00,0x00]
+               vsm4key4  (%rip), %xmm23, %xmm22
+
+// CHECK:      vsm4key4  -512(,%rbp,2), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x00,0xda,0x34,0x6d,0x00,0xfe,0xff,0xff]
+               vsm4key4  -512(,%rbp,2), %xmm23, %xmm22
+
+// CHECK:      vsm4key4  2032(%rcx), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x00,0xda,0x71,0x7f]
+               vsm4key4  2032(%rcx), %xmm23, %xmm22
+
+// CHECK:      vsm4key4  -2048(%rdx), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xe2,0x46,0x00,0xda,0x72,0x80]
+               vsm4key4  -2048(%rdx), %xmm23, %xmm22
+
+// CHECK:      vsm4rnds4 %ymm24, %ymm23, %ymm22
+// CHECK: encoding: [0x62,0x82,0x47,0x20,0xda,0xf0]
+               vsm4rnds4 %ymm24, %ymm23, %ymm22
+
+// CHECK:      vsm4rnds4 %xmm24, %xmm23, %xmm22
+// CHECK: encoding: [0x62,0x82,0x47,0x00,0xda,0xf0]
+               vsm4rnds4 %xmm24, %xmm23, %xmm22
+
+// CHECK:      vsm4rnds4  268435456(%rbp,%r14,8), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xa2,0x47,0x20,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10]
+               vsm4rnds4  268435456(%rbp,%r14,8), %ymm23, %ymm22
+
+// CHECK:      vsm4rnds4  291(%r8,%rax,4), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xc2,0x47,0x20,0xda,0xb4,0x80,0x23,0x01,0x00,0x00]
+               vsm4rnds4  291(%r8,%rax,4), %ymm23, %ymm22
+
+// CHECK:      vsm4rnds4  (%rip), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xe2,0x47,0x20,0xda,0x35,0x00,0x00,0x00,0x00]
+               vsm4rnds4  (%rip), %ymm23, %ymm22
+
+// CHECK:      vsm4rnds4  -1024(,%rbp,2), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xe2,0x47,0x20,0xda,0x34,0x6d,0x00,0xfc,0xff,0xff]
+               vsm4rnds4  -1024(,%rbp,2), %ymm23, %ymm22
+
+// CHECK:      vsm4rnds4  4064(%rcx), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xe2,0x47,0x20,0xda,0x71,0x7f]
+               vsm4rnds4  4064(%rcx), %ymm23, %ymm22
+
+// CHECK:      vsm4rnds4  -4096(%rdx), %ymm23, %ymm22
+// CHECK: encoding: [0x62,0xe2,0x47,0x20,0xda,0x72,0x80]
+               vsm4rnds4  -4096(%rdx), %ymm23, %ymm22
+
+// CHECK:      vsm4rnds4  268435456(%rbp,%r14,8), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xa2,0x47,0x00,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10]
+               vsm4rnds4  268435456(%rbp,%r14,8), %xmm23, %xmm22
+
+// CHECK:      vsm4rnds4  291(%r8,%rax,4), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xc2,0x47,0x00,0xda,0xb4,0x80,0x23,0x01,0x00,0x00]
+               vsm4rnds4  291(%r8,%rax,4), %xmm23, %xmm22
+
+// CHECK:      vsm4rnds4  (%rip), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xe2,0x47,0x00,0xda,0x35,0x00,0x00,0x00,0x00]
+               vsm4rnds4  (%rip), %xmm23, %xmm22
+
+// CHECK:      vsm4rnds4  -512(,%rbp,2), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xe2,0x47,0x00,0xda,0x34,0x6d,0x00,0xfe,0xff,0xff]
+               vsm4rnds4  -512(,%rbp,2), %xmm23, %xmm22
+
+// CHECK:      vsm4rnds4  2032(%rcx), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xe2,0x47,0x00,0xda,0x71,0x7f]
+               vsm4rnds4  2032(%rcx), %xmm23, %xmm22
+
+// CHECK:      vsm4rnds4  -2048(%rdx), %xmm23, %xmm22
+// CHECK: encoding: [0x62,0xe2,0x47,0x00,0xda,0x72,0x80]
+               vsm4rnds4  -2048(%rdx), %xmm23, %xmm22
diff --git a/llvm/test/MC/X86/sm4-evex-64-intel.s b/llvm/test/MC/X86/sm4-evex-64-intel.s
new file mode 100644
index 000000000000..3cc18cf4178e
--- /dev/null
+++ b/llvm/test/MC/X86/sm4-evex-64-intel.s
@@ -0,0 +1,169 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
+
+// CHECK:      vsm4key4 zmm22, zmm23, zmm24
+// CHECK: encoding: [0x62,0x82,0x46,0x40,0xda,0xf0]
+               vsm4key4 zmm22, zmm23, zmm24
+
+// CHECK:      vsm4key4 zmm22, zmm23, zmmword ptr [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0x62,0xa2,0x46,0x40,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10]
+               vsm4key4 zmm22, zmm23, zmmword ptr [rbp + 8*r14 + 268435456]
+
+// CHECK:      vsm4key4 zmm22, zmm23, zmmword ptr [r8 + 4*rax + 291]
+// CHECK: encoding: [0x62,0xc2,0x46,0x40,0xda,0xb4,0x80,0x23,0x01,0x00,0x00]
+               vsm4key4 zmm22, zmm23, zmmword ptr [r8 + 4*rax + 291]
+
+// CHECK:      vsm4key4 zmm22, zmm23, zmmword ptr [rip]
+// CHECK: encoding: [0x62,0xe2,0x46,0x40,0xda,0x35,0x00,0x00,0x00,0x00]
+               vsm4key4 zmm22, zmm23, zmmword ptr [rip]
+
+// CHECK:      vsm4key4 zmm22, zmm23, zmmword ptr [2*rbp - 2048]
+// CHECK: encoding: [0x62,0xe2,0x46,0x40,0xda,0x34,0x6d,0x00,0xf8,0xff,0xff]
+               vsm4key4 zmm22, zmm23, zmmword ptr [2*rbp - 2048]
+
+// CHECK:      vsm4key4 zmm22, zmm23, zmmword ptr [rcx + 8128]
+// CHECK: encoding: [0x62,0xe2,0x46,0x40,0xda,0x71,0x7f]
+               vsm4key4 zmm22, zmm23, zmmword ptr [rcx + 8128]
+
+// CHECK:      vsm4key4 zmm22, zmm23, zmmword ptr [rdx - 8192]
+// CHECK: encoding: [0x62,0xe2,0x46,0x40,0xda,0x72,0x80]
+               vsm4key4 zmm22, zmm23, zmmword ptr [rdx - 8192]
+
+// CHECK:      vsm4rnds4 zmm22, zmm23, zmm24
+// CHECK: encoding: [0x62,0x82,0x47,0x40,0xda,0xf0]
+               vsm4rnds4 zmm22, zmm23, zmm24
+
+// CHECK:      vsm4rnds4 zmm22, zmm23, zmmword ptr [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0x62,0xa2,0x47,0x40,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10]
+               vsm4rnds4 zmm22, zmm23, zmmword ptr [rbp + 8*r14 + 268435456]
+
+// CHECK:      vsm4rnds4 zmm22, zmm23, zmmword ptr [r8 + 4*rax + 291]
+// CHECK: encoding: [0x62,0xc2,0x47,0x40,0xda,0xb4,0x80,0x23,0x01,0x00,0x00]
+               vsm4rnds4 zmm22, zmm23, zmmword ptr [r8 + 4*rax + 291]
+
+// CHECK:      vsm4rnds4 zmm22, zmm23, zmmword ptr [rip]
+// CHECK: encoding: [0x62,0xe2,0x47,0x40,0xda,0x35,0x00,0x00,0x00,0x00]
+               vsm4rnds4 zmm22, zmm23, zmmword ptr [rip]
+
+// CHECK:      vsm4rnds4 zmm22, zmm23, zmmword ptr [2*rbp - 2048]
+// CHECK: encoding: [0x62,0xe2,0x47,0x40,0xda,0x34,0x6d,0x00,0xf8,0xff,0xff]
+               vsm4rnds4 zmm22, zmm23, zmmword ptr [2*rbp - 2048]
+
+// CHECK:      vsm4rnds4 zmm22, zmm23, zmmword ptr [rcx + 8128]
+// CHECK: encoding: [0x62,0xe2,0x47,0x40,0xda,0x71,0x7f]
+               vsm4rnds4 zmm22, zmm23, zmmword ptr [rcx + 8128]
+
+// CHECK:      vsm4rnds4 zmm22, zmm23, zmmword ptr [rdx - 8192]
+// CHECK: encoding: [0x62,0xe2,0x47,0x40,0xda,0x72,0x80]
+               vsm4rnds4 zmm22, zmm23, zmmword ptr [rdx - 8192]
+
+// CHECK:      vsm4key4 ymm22, ymm23, ymm24
+// CHECK: encoding: [0x62,0x82,0x46,0x20,0xda,0xf0]
+               vsm4key4 ymm22, ymm23, ymm24
+
+// CHECK:      vsm4key4 xmm22, xmm23, xmm24
+// CHECK: encoding: [0x62,0x82,0x46,0x00,0xda,0xf0]
+               vsm4key4 xmm22, xmm23, xmm24
+
+// CHECK:      vsm4key4 ymm22, ymm23, ymmword ptr [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0x62,0xa2,0x46,0x20,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10]
+               vsm4key4 ymm22, ymm23, ymmword ptr [rbp + 8*r14 + 268435456]
+
+// CHECK:      vsm4key4 ymm22, ymm23, ymmword ptr [r8 + 4*rax + 291]
+// CHECK: encoding: [0x62,0xc2,0x46,0x20,0xda,0xb4,0x80,0x23,0x01,0x00,0x00]
+               vsm4key4 ymm22, ymm23, ymmword ptr [r8 + 4*rax + 291]
+
+// CHECK:      vsm4key4 ymm22, ymm23, ymmword ptr [rip]
+// CHECK: encoding: [0x62,0xe2,0x46,0x20,0xda,0x35,0x00,0x00,0x00,0x00]
+               vsm4key4 ymm22, ymm23, ymmword ptr [rip]
+
+// CHECK:      vsm4key4 ymm22, ymm23, ymmword ptr [2*rbp - 1024]
+// CHECK: encoding: [0x62,0xe2,0x46,0x20,0xda,0x34,0x6d,0x00,0xfc,0xff,0xff]
+               vsm4key4 ymm22, ymm23, ymmword ptr [2*rbp - 1024]
+
+// CHECK:      vsm4key4 ymm22, ymm23, ymmword ptr [rcx + 4064]
+// CHECK: encoding: [0x62,0xe2,0x46,0x20,0xda,0x71,0x7f]
+               vsm4key4 ymm22, ymm23, ymmword ptr [rcx + 4064]
+
+// CHECK:      vsm4key4 ymm22, ymm23, ymmword ptr [rdx - 4096]
+// CHECK: encoding: [0x62,0xe2,0x46,0x20,0xda,0x72,0x80]
+               vsm4key4 ymm22, ymm23, ymmword ptr [rdx - 4096]
+
+// CHECK:      vsm4key4 xmm22, xmm23, xmmword ptr [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0x62,0xa2,0x46,0x00,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10]
+               vsm4key4 xmm22, xmm23, xmmword ptr [rbp + 8*r14 + 268435456]
+
+// CHECK:      vsm4key4 xmm22, xmm23, xmmword ptr [r8 + 4*rax + 291]
+// CHECK: encoding: [0x62,0xc2,0x46,0x00,0xda,0xb4,0x80,0x23,0x01,0x00,0x00]
+               vsm4key4 xmm22, xmm23, xmmword ptr [r8 + 4*rax + 291]
+
+// CHECK:      vsm4key4 xmm22, xmm23, xmmword ptr [rip]
+// CHECK: encoding: [0x62,0xe2,0x46,0x00,0xda,0x35,0x00,0x00,0x00,0x00]
+               vsm4key4 xmm22, xmm23, xmmword ptr [rip]
+
+// CHECK:      vsm4key4 xmm22, xmm23, xmmword ptr [2*rbp - 512]
+// CHECK: encoding: [0x62,0xe2,0x46,0x00,0xda,0x34,0x6d,0x00,0xfe,0xff,0xff]
+               vsm4key4 xmm22, xmm23, xmmword ptr [2*rbp - 512]
+
+// CHECK:      vsm4key4 xmm22, xmm23, xmmword ptr [rcx + 2032]
+// CHECK: encoding: [0x62,0xe2,0x46,0x00,0xda,0x71,0x7f]
+               vsm4key4 xmm22, xmm23, xmmword ptr [rcx + 2032]
+
+// CHECK:      vsm4key4 xmm22, xmm23, xmmword ptr [rdx - 2048]
+// CHECK: encoding: [0x62,0xe2,0x46,0x00,0xda,0x72,0x80]
+               vsm4key4 xmm22, xmm23, xmmword ptr [rdx - 2048]
+
+// CHECK:      vsm4rnds4 ymm22, ymm23, ymm24
+// CHECK: encoding: [0x62,0x82,0x47,0x20,0xda,0xf0]
+               vsm4rnds4 ymm22, ymm23, ymm24
+
+// CHECK:      vsm4rnds4 xmm22, xmm23, xmm24
+// CHECK: encoding: [0x62,0x82,0x47,0x00,0xda,0xf0]
+               vsm4rnds4 xmm22, xmm23, xmm24
+
+// CHECK:      vsm4rnds4 ymm22, ymm23, ymmword ptr [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0x62,0xa2,0x47,0x20,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10]
+               vsm4rnds4 ymm22, ymm23, ymmword ptr [rbp + 8*r14 + 268435456]
+
+// CHECK:      vsm4rnds4 ymm22, ymm23, ymmword ptr [r8 + 4*rax + 291]
+// CHECK: encoding: [0x62,0xc2,0x47,0x20,0xda,0xb4,0x80,0x23,0x01,0x00,0x00]
+               vsm4rnds4 ymm22, ymm23, ymmword ptr [r8 + 4*rax + 291]
+
+// CHECK:      vsm4rnds4 ymm22, ymm23, ymmword ptr [rip]
+// CHECK: encoding: [0x62,0xe2,0x47,0x20,0xda,0x35,0x00,0x00,0x00,0x00]
+               vsm4rnds4 ymm22, ymm23, ymmword ptr [rip]
+
+// CHECK:      vsm4rnds4 ymm22, ymm23, ymmword ptr [2*rbp - 1024]
+// CHECK: encoding: [0x62,0xe2,0x47,0x20,0xda,0x34,0x6d,0x00,0xfc,0xff,0xff]
+               vsm4rnds4 ymm22, ymm23, ymmword ptr [2*rbp - 1024]
+
+// CHECK:      vsm4rnds4 ymm22, ymm23, ymmword ptr [rcx + 4064]
+// CHECK: encoding: [0x62,0xe2,0x47,0x20,0xda,0x71,0x7f]
+               vsm4rnds4 ymm22, ymm23, ymmword ptr [rcx + 4064]
+
+// CHECK:      vsm4rnds4 ymm22, ymm23, ymmword ptr [rdx - 4096]
+// CHECK: encoding: [0x62,0xe2,0x47,0x20,0xda,0x72,0x80]
+               vsm4rnds4 ymm22, ymm23, ymmword ptr [rdx - 4096]
+
+// CHECK:      vsm4rnds4 xmm22, xmm23, xmmword ptr [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0x62,0xa2,0x47,0x00,0xda,0xb4,0xf5,0x00,0x00,0x00,0x10]
+               vsm4rnds4 xmm22, xmm23, xmmword ptr [rbp + 8*r14 + 268435456]
+
+// CHECK:      vsm4rnds4 xmm22, xmm23, xmmword ptr [r8 + 4*rax + 291]
+// CHECK: encoding: [0x62,0xc2,0x47,0x00,0xda,0xb4,0x80,0x23,0x01,0x00,0x00]
+               vsm4rnds4 xmm22, xmm23, xmmword ptr [r8 + 4*rax + 291]
+
+// CHECK:      vsm4rnds4 xmm22, xmm23, xmmword ptr [rip]
+// CHECK: encoding: [0x62,0xe2,0x47,0x00,0xda,0x35,0x00,0x00,0x00,0x00]
+               vsm4rnds4 xmm22, xmm23, xmmword ptr [rip]
+
+// CHECK:      vsm4rnds4 xmm22, xmm23, xmmword ptr [2*rbp - 512]
+// CHECK: encoding: [0x62,0xe2,0x47,0x00,0xda,0x34,0x6d,0x00,0xfe,0xff,0xff]
+               vsm4rnds4 xmm22, xmm23, xmmword ptr [2*rbp - 512]
+
+// CHECK:      vsm4rnds4 xmm22, xmm23, xmmword ptr [rcx + 2032]
+// CHECK: encoding: [0x62,0xe2,0x47,0x00,0xda,0x71,0x7f]
+               vsm4rnds4 xmm22, xmm23, xmmword ptr [rcx + 2032]
+
+// CHECK:      vsm4rnds4 xmm22, xmm23, xmmword ptr [rdx - 2048]
+// CHECK: encoding: [0x62,0xe2,0x47,0x00,0xda,0x72,0x80]
+               vsm4rnds4 xmm22, xmm23, xmmword ptr [rdx - 2048]
diff --git a/llvm/test/TableGen/x86-fold-tables.inc b/llvm/test/TableGen/x86-fold-tables.inc
index 85d9b02ac0cb..43c206fa0af6 100644
--- a/llvm/test/TableGen/x86-fold-tables.inc
+++ b/llvm/test/TableGen/x86-fold-tables.inc
@@ -4113,8 +4113,14 @@ static const X86FoldTableEntry Table2[] = {
   {X86::VSHUFPSZrri, X86::VSHUFPSZrmi, 0},
   {X86::VSHUFPSrri, X86::VSHUFPSrmi, 0},
   {X86::VSM4KEY4Yrr, X86::VSM4KEY4Yrm, 0},
+  {X86::VSM4KEY4Z128rr, X86::VSM4KEY4Z128rm, 0},
+  {X86::VSM4KEY4Z256rr, X86::VSM4KEY4Z256rm, 0},
+  {X86::VSM4KEY4Zrr, X86::VSM4KEY4Zrm, 0},
   {X86::VSM4KEY4rr, X86::VSM4KEY4rm, 0},
   {X86::VSM4RNDS4Yrr, X86::VSM4RNDS4Yrm, 0},
+  {X86::VSM4RNDS4Z128rr, X86::VSM4RNDS4Z128rm, 0},
+  {X86::VSM4RNDS4Z256rr, X86::VSM4RNDS4Z256rm, 0},
+  {X86::VSM4RNDS4Zrr, X86::VSM4RNDS4Zrm, 0},
   {X86::VSM4RNDS4rr, X86::VSM4RNDS4rm, 0},
   {X86::VSQRTNEPBF16Z128rkz, X86::VSQRTNEPBF16Z128mkz, 0},
   {X86::VSQRTNEPBF16Z256rkz, X86::VSQRTNEPBF16Z256mkz, 0},
-- 
GitLab


From a1d31caa8c53082d12f580122dcf2b2ff8285e78 Mon Sep 17 00:00:00 2001
From: Congcong Cai <congcongcai0907@163.com>
Date: Mon, 28 Oct 2024 11:36:37 +0800
Subject: [PATCH 274/693] [clang-tidy] fix false positive for implicit
 conversion of comparison result in C23 (#113639)

Fixed #111013
bool will be builtin type in C23 but comparison result in C is still
int.
It is no need to change this kind of implicit cast to explicit cast.
---
 .../readability/ImplicitBoolConversionCheck.cpp          | 8 ++++++++
 clang-tools-extra/docs/ReleaseNotes.rst                  | 3 ++-
 .../checkers/readability/implicit-bool-conversion.c      | 9 +++++++++
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp b/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
index 968a4a55a6d7..f9fd1d903e23 100644
--- a/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
@@ -10,6 +10,7 @@
 #include "../utils/FixItHintUtils.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Lex/Lexer.h"
 #include "clang/Tooling/FixIt.h"
 #include <queue>
@@ -26,6 +27,8 @@ AST_MATCHER(Stmt, isMacroExpansion) {
   return SM.isMacroBodyExpansion(Loc) || SM.isMacroArgExpansion(Loc);
 }
 
+AST_MATCHER(Stmt, isC23) { return Finder->getASTContext().getLangOpts().C23; }
+
 bool isNULLMacroExpansion(const Stmt *Statement, ASTContext &Context) {
   SourceManager &SM = Context.getSourceManager();
   const LangOptions &LO = Context.getLangOpts();
@@ -298,6 +301,11 @@ void ImplicitBoolConversionCheck::registerMatchers(MatchFinder *Finder) {
                          hasCastKind(CK_FloatingToBoolean),
                          hasCastKind(CK_PointerToBoolean),
                          hasCastKind(CK_MemberPointerToBoolean)),
+                   // Exclude cases of C23 comparison result.
+                   unless(allOf(isC23(),
+                                hasSourceExpression(ignoringParens(
+                                    binaryOperator(hasAnyOperatorName(
+                                        ">", ">=", "==", "!=", "<", "<=")))))),
                    // Exclude case of using if or while statements with variable
                    // declaration, e.g.:
                    //   if (int var = functionCall()) {}
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 876689c40fcd..4cc4c2146d7e 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -249,7 +249,8 @@ Changes in existing checks
 - Improved :doc:`readability-implicit-bool-conversion
   <clang-tidy/checks/readability/implicit-bool-conversion>` check
   by adding the option `UseUpperCaseLiteralSuffix` to select the
-  case of the literal suffix in fixes.
+  case of the literal suffix in fixes and fixing false positive for implicit
+  conversion of comparison result in C23.
 
 - Improved :doc:`readability-redundant-smartptr-get
   <clang-tidy/checks/readability/redundant-smartptr-get>` check to
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion.c b/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion.c
index f3dc32c10d64..0b231d10adf8 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion.c
@@ -304,6 +304,15 @@ void implicitConversionToBoolFromUnaryMinusAndZeroLiterals() {
   // CHECK-FIXES: functionTakingBool((-0.0) != 0.0);
 }
 
+void ignoreImplicitCastToBoolForComparisonResult() {
+  bool boolFromComparison0 = 1 != 0;
+  bool boolFromComparison1 = 1 == 0;
+  bool boolFromComparison2 = 1 > 0;
+  bool boolFromComparison3 = 1 >= 0;
+  bool boolFromComparison4 = 1 < 0;
+  bool boolFromComparison5 = 1 <= 0;
+}
+
 void ignoreExplicitCastsToBool() {
   int integer = 10;
   bool boolComingFromInt = (bool)integer;
-- 
GitLab


From c876d719ef5b10128eca6d8677068248b4831017 Mon Sep 17 00:00:00 2001
From: antangelo <contact@antangelo.com>
Date: Mon, 28 Oct 2024 00:25:56 -0400
Subject: [PATCH 275/693] [clang][NFC] Accept const NamedDecl pointer for
 getDepthAndIndex (#113857)

---
 clang/include/clang/Sema/SemaInternal.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/include/clang/Sema/SemaInternal.h b/clang/include/clang/Sema/SemaInternal.h
index d994d1819b44..41d05b2bfb07 100644
--- a/clang/include/clang/Sema/SemaInternal.h
+++ b/clang/include/clang/Sema/SemaInternal.h
@@ -58,7 +58,7 @@ inline InheritableAttr *getDLLAttr(Decl *D) {
 }
 
 /// Retrieve the depth and index of a template parameter.
-inline std::pair<unsigned, unsigned> getDepthAndIndex(NamedDecl *ND) {
+inline std::pair<unsigned, unsigned> getDepthAndIndex(const NamedDecl *ND) {
   if (const auto *TTP = dyn_cast<TemplateTypeParmDecl>(ND))
     return std::make_pair(TTP->getDepth(), TTP->getIndex());
 
-- 
GitLab


From 99b2feadcc770ea52f71dda6cf027373a6931c6c Mon Sep 17 00:00:00 2001
From: ZhangYin <zhangyin2018@iscas.ac.cn>
Date: Mon, 28 Oct 2024 12:40:09 +0800
Subject: [PATCH 276/693] [libc++] <experimental/simd> temporarily mark XFAIL
 for the armv7-unknown-linux-gnueabihf with simd unary test to pass the CI
 (#113641)

---
 .../test/std/experimental/simd/simd.class/simd_unary.pass.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libcxx/test/std/experimental/simd/simd.class/simd_unary.pass.cpp b/libcxx/test/std/experimental/simd/simd.class/simd_unary.pass.cpp
index f205ac971e5f..8cc9924a3cda 100644
--- a/libcxx/test/std/experimental/simd/simd.class/simd_unary.pass.cpp
+++ b/libcxx/test/std/experimental/simd/simd.class/simd_unary.pass.cpp
@@ -16,6 +16,10 @@
 //   clang: error: unable to execute command: Illegal instruction: 4
 // XFAIL: target=x86_64-apple-macosx13.7
 
+// FIXME: The following issue occurs on Windows to Armv7 Ubuntu Linux:
+//   Assertion failed: N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type"
+// XFAIL: target=armv7-unknown-linux-gnueabihf
+
 // <experimental/simd>
 //
 // [simd.class]
-- 
GitLab


From d3f70db51cbc0876937d404e96fbda04df793bd4 Mon Sep 17 00:00:00 2001
From: Freddy Ye <freddy.ye@intel.com>
Date: Mon, 28 Oct 2024 12:59:51 +0800
Subject: [PATCH 277/693] [X86][MC] Support instructions of MSR_IMM (#113524)

Ref.: https://cdrdv2.intel.com/v1/dl/getContent/671368
---
 llvm/docs/ReleaseNotes.md                     |  2 ++
 llvm/lib/Target/X86/X86InstrSystem.td         |  8 +++++-
 llvm/test/MC/Disassembler/X86/apx/msr-imm.txt | 18 +++++++++++++
 llvm/test/MC/Disassembler/X86/msrimm-64.txt   | 10 ++++++++
 llvm/test/MC/X86/apx/msrimm-att.s             | 25 +++++++++++++++++++
 llvm/test/MC/X86/apx/msrimm-intel.s           | 21 ++++++++++++++++
 llvm/test/MC/X86/msrimm-64-att.s              | 14 +++++++++++
 llvm/test/MC/X86/msrimm-64-intel.s            | 10 ++++++++
 8 files changed, 107 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/MC/Disassembler/X86/apx/msr-imm.txt
 create mode 100644 llvm/test/MC/Disassembler/X86/msrimm-64.txt
 create mode 100644 llvm/test/MC/X86/apx/msrimm-att.s
 create mode 100644 llvm/test/MC/X86/apx/msrimm-intel.s
 create mode 100644 llvm/test/MC/X86/msrimm-64-att.s
 create mode 100644 llvm/test/MC/X86/msrimm-64-intel.s

diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 7c7e687e9474..2580f09be3ad 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -221,6 +221,8 @@ Changes to the X86 Backend
 
 * Supported ISA of `SM4(EVEX)`.
 
+* Supported ISA of `MSR_IMM`.
+
 Changes to the OCaml bindings
 -----------------------------
 
diff --git a/llvm/lib/Target/X86/X86InstrSystem.td b/llvm/lib/Target/X86/X86InstrSystem.td
index e1573b37d4dc..dc701f1afc91 100644
--- a/llvm/lib/Target/X86/X86InstrSystem.td
+++ b/llvm/lib/Target/X86/X86InstrSystem.td
@@ -466,7 +466,10 @@ multiclass Urdwrmsr<Map rrmap, string suffix> {
                                 "urdmsr\t{$imm, $dst|$dst, $imm}",
                                 [(set GR64:$dst, (int_x86_urdmsr i64immSExt32_su:$imm))]>,
                            T_MAP7, VEX, XD, NoCD8;
-}
+    def RDMSRri#suffix  : Ii32<0xf6, MRM0r, (outs GR64:$dst), (ins i64i32imm:$imm),
+                                "rdmsr\t{$imm, $dst|$dst, $imm}", []>,
+                           T_MAP7, VEX, XD, NoCD8;
+  }
   let mayStore = 1 in {
     let OpMap = rrmap in
     def UWRMSRrr#suffix  : I<0xf8, MRMSrcReg, (outs), (ins GR64:$src1, GR64:$src2),
@@ -476,6 +479,9 @@ multiclass Urdwrmsr<Map rrmap, string suffix> {
                                 "uwrmsr\t{$src, $imm|$imm, $src}",
                                 [(int_x86_uwrmsr i64immSExt32_su:$imm, GR64:$src)]>,
                            T_MAP7, VEX, XS, NoCD8;
+    def WRMSRNSir#suffix  : Ii32<0xf6, MRM0r, (outs), (ins GR64:$src, i64i32imm:$imm),
+                                "wrmsrns\t{$src, $imm|$imm, $src}",
+                                []>, T_MAP7, VEX, XS, NoCD8;
   }
 }
 
diff --git a/llvm/test/MC/Disassembler/X86/apx/msr-imm.txt b/llvm/test/MC/Disassembler/X86/apx/msr-imm.txt
new file mode 100644
index 000000000000..63465bb7070e
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/apx/msr-imm.txt
@@ -0,0 +1,18 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64 | FileCheck %s --check-prefixes=ATT
+# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL
+
+# ATT:   rdmsr $123, %r9
+# INTEL: rdmsr r9, 123
+0x62,0xd7,0x7f,0x08,0xf6,0xc1,0x7b,0x00,0x00,0x00
+
+# ATT:   rdmsr $123, %r19
+# INTEL: rdmsr r19, 123
+0x62,0xff,0x7f,0x08,0xf6,0xc3,0x7b,0x00,0x00,0x00
+
+# ATT:   wrmsrns %r9, $123
+# INTEL: wrmsrns 123, r9
+0x62,0xd7,0x7e,0x08,0xf6,0xc1,0x7b,0x00,0x00,0x00
+
+# ATT:   wrmsrns %r19, $123
+# INTEL: wrmsrns 123, r19
+0x62,0xff,0x7e,0x08,0xf6,0xc3,0x7b,0x00,0x00,0x00
diff --git a/llvm/test/MC/Disassembler/X86/msrimm-64.txt b/llvm/test/MC/Disassembler/X86/msrimm-64.txt
new file mode 100644
index 000000000000..625d70d739cd
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/msrimm-64.txt
@@ -0,0 +1,10 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64 | FileCheck %s --check-prefixes=ATT
+# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL
+
+# ATT:   rdmsr $123, %r9
+# INTEL: rdmsr r9, 123
+0xc4,0xc7,0x7b,0xf6,0xc1,0x7b,0x00,0x00,0x00
+
+# ATT:   wrmsrns %r9, $123
+# INTEL: wrmsrns 123, r9
+0xc4,0xc7,0x7a,0xf6,0xc1,0x7b,0x00,0x00,0x00
diff --git a/llvm/test/MC/X86/apx/msrimm-att.s b/llvm/test/MC/X86/apx/msrimm-att.s
new file mode 100644
index 000000000000..e4259f19cb7b
--- /dev/null
+++ b/llvm/test/MC/X86/apx/msrimm-att.s
@@ -0,0 +1,25 @@
+# RUN: llvm-mc -triple x86_64 --show-encoding %s | FileCheck %s
+# RUN: not llvm-mc -triple i386 -show-encoding %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+# ERROR-COUNT-4: error:
+# ERROR-NOT: error:
+
+## rdmsr
+
+// CHECK: {evex} rdmsr $123, %r9
+// CHECK: encoding: [0x62,0xd7,0x7f,0x08,0xf6,0xc1,0x7b,0x00,0x00,0x00]
+          {evex} rdmsr $123, %r9
+
+// CHECK: rdmsr $123, %r19
+// CHECK: encoding: [0x62,0xff,0x7f,0x08,0xf6,0xc3,0x7b,0x00,0x00,0x00]
+          rdmsr $123, %r19
+
+## wrmsrns
+
+# CHECK: {evex}	wrmsrns %r9, $123
+# CHECK: encoding: [0x62,0xd7,0x7e,0x08,0xf6,0xc1,0x7b,0x00,0x00,0x00]
+         {evex}	wrmsrns %r9, $123
+
+# CHECK: wrmsrns %r19, $123
+# CHECK: encoding: [0x62,0xff,0x7e,0x08,0xf6,0xc3,0x7b,0x00,0x00,0x00]
+         wrmsrns %r19, $123
diff --git a/llvm/test/MC/X86/apx/msrimm-intel.s b/llvm/test/MC/X86/apx/msrimm-intel.s
new file mode 100644
index 000000000000..d7eab047dd0c
--- /dev/null
+++ b/llvm/test/MC/X86/apx/msrimm-intel.s
@@ -0,0 +1,21 @@
+# RUN: llvm-mc -triple x86_64 -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
+
+## urdmsr
+
+# CHECK: {evex}	rdmsr r9, 123
+# CHECK: encoding: [0x62,0xd7,0x7f,0x08,0xf6,0xc1,0x7b,0x00,0x00,0x00]
+         {evex}	rdmsr r9, 123
+
+# CHECK: rdmsr r19, 123
+# CHECK: encoding: [0x62,0xff,0x7f,0x08,0xf6,0xc3,0x7b,0x00,0x00,0x00]
+         rdmsr r19, 123
+
+## uwrmsr
+
+# CHECK: {evex}	wrmsrns 123, r9
+# CHECK: encoding: [0x62,0xd7,0x7e,0x08,0xf6,0xc1,0x7b,0x00,0x00,0x00]
+         {evex}	wrmsrns 123, r9
+
+# CHECK: wrmsrns 123, r19
+# CHECK: encoding: [0x62,0xff,0x7e,0x08,0xf6,0xc3,0x7b,0x00,0x00,0x00]
+         wrmsrns 123, r19
diff --git a/llvm/test/MC/X86/msrimm-64-att.s b/llvm/test/MC/X86/msrimm-64-att.s
new file mode 100644
index 000000000000..e69eb7ff29a6
--- /dev/null
+++ b/llvm/test/MC/X86/msrimm-64-att.s
@@ -0,0 +1,14 @@
+// RUN: llvm-mc -triple x86_64 --show-encoding %s | FileCheck %s
+// RUN: not llvm-mc -triple i386 -show-encoding %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+// ERROR-COUNT-2: error:
+// ERROR-NOT: error:
+
+// CHECK: rdmsr $123, %r9
+// CHECK: encoding: [0xc4,0xc7,0x7b,0xf6,0xc1,0x7b,0x00,0x00,0x00]
+          rdmsr $123, %r9
+
+// CHECK: wrmsrns %r9, $123
+// CHECK: encoding: [0xc4,0xc7,0x7a,0xf6,0xc1,0x7b,0x00,0x00,0x00]
+          wrmsrns %r9, $123
+
diff --git a/llvm/test/MC/X86/msrimm-64-intel.s b/llvm/test/MC/X86/msrimm-64-intel.s
new file mode 100644
index 000000000000..e1ae9c679123
--- /dev/null
+++ b/llvm/test/MC/X86/msrimm-64-intel.s
@@ -0,0 +1,10 @@
+// RUN: llvm-mc -triple x86_64 -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
+
+// CHECK: rdmsr r9, 123
+// CHECK: encoding: [0xc4,0xc7,0x7b,0xf6,0xc1,0x7b,0x00,0x00,0x00]
+          rdmsr r9, 123
+
+// CHECK: wrmsrns 123, r9
+// CHECK: encoding: [0xc4,0xc7,0x7a,0xf6,0xc1,0x7b,0x00,0x00,0x00]
+          wrmsrns 123, r9
+
-- 
GitLab


From 5d4a0d54b5269bad1410e6db957836fe98634069 Mon Sep 17 00:00:00 2001
From: David Majnemer <david.majnemer@gmail.com>
Date: Sun, 27 Oct 2024 23:51:20 +0000
Subject: [PATCH 278/693] [InstCombine] Teach takeLog2 about right shifts,
 truncation and bitwise-and

We left some easy opportunities for further simplifications.

log2(trunc(x)) is simply trunc(log2(x)). This is safe if we know that
trunc is NUW because it means that the truncation didn't drop any bits.
It is also safe if the caller is OK with zero as a possible answer.

log2(x >>u y) is simply `log2(x) - y`.

log2(x & y) is a funny one. It comes up when doing something like:
```
unsigned int f(unsigned int x, unsigned int y) {
  unsigned char a = 1u << x;
  return y / a;
}
```

LLVM would canonicalize this to:
```
  %shl = shl nuw i32 1, %x
  %conv1 = and i32 %shl, 255
  %div = udiv i32 %y, %conv1
```

In cases like these, we can ignore the mask entirely.
This is equivalent to `y >> x`.
---
 .../InstCombine/InstCombineMulDivRem.cpp      | 30 +++++++++++++
 llvm/test/Transforms/InstCombine/div.ll       | 43 +++++++++++++++++--
 llvm/test/Transforms/InstCombine/shift.ll     |  8 ++--
 3 files changed, 74 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index f4f3644acfe5..b9c165da906d 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -1427,6 +1427,18 @@ static Value *takeLog2(IRBuilderBase &Builder, Value *Op, unsigned Depth,
     if (Value *LogX = takeLog2(Builder, X, Depth, AssumeNonZero, DoFold))
       return IfFold([&]() { return Builder.CreateZExt(LogX, Op->getType()); });
 
+  // log2(trunc x) -> trunc log2(X)
+  // FIXME: Require one use?
+  if (match(Op, m_Trunc(m_Value(X)))) {
+    auto *TI = cast<TruncInst>(Op);
+    if (AssumeNonZero || TI->hasNoUnsignedWrap())
+      if (Value *LogX = takeLog2(Builder, X, Depth, AssumeNonZero, DoFold))
+        return IfFold([&]() {
+          return Builder.CreateTrunc(LogX, Op->getType(), "",
+                                     /*IsNUW=*/TI->hasNoUnsignedWrap());
+        });
+  }
+
   // log2(X << Y) -> log2(X) + Y
   // FIXME: Require one use unless X is 1?
   if (match(Op, m_Shl(m_Value(X), m_Value(Y)))) {
@@ -1437,6 +1449,24 @@ static Value *takeLog2(IRBuilderBase &Builder, Value *Op, unsigned Depth,
         return IfFold([&]() { return Builder.CreateAdd(LogX, Y); });
   }
 
+  // log2(X >>u Y) -> log2(X) - Y
+  // FIXME: Require one use?
+  if (match(Op, m_LShr(m_Value(X), m_Value(Y)))) {
+    auto *PEO = cast<PossiblyExactOperator>(Op);
+    if (AssumeNonZero || PEO->isExact())
+      if (Value *LogX = takeLog2(Builder, X, Depth, AssumeNonZero, DoFold))
+        return IfFold([&]() { return Builder.CreateSub(LogX, Y); });
+  }
+
+  // log2(X & Y) -> either log2(X) or log2(Y)
+  // This requires `AssumeNonZero` as `X & Y` may be zero when X != Y.
+  if (AssumeNonZero && match(Op, m_And(m_Value(X), m_Value(Y)))) {
+    if (Value *LogX = takeLog2(Builder, X, Depth, AssumeNonZero, DoFold))
+      return IfFold([&]() { return LogX; });
+    if (Value *LogY = takeLog2(Builder, Y, Depth, AssumeNonZero, DoFold))
+      return IfFold([&]() { return LogY; });
+  }
+
   // log2(Cond ? X : Y) -> Cond ? log2(X) : log2(Y)
   // FIXME: Require one use?
   if (SelectInst *SI = dyn_cast<SelectInst>(Op))
diff --git a/llvm/test/Transforms/InstCombine/div.ll b/llvm/test/Transforms/InstCombine/div.ll
index e8a25ff44d02..a91c9bfc91c4 100644
--- a/llvm/test/Transforms/InstCombine/div.ll
+++ b/llvm/test/Transforms/InstCombine/div.ll
@@ -429,9 +429,8 @@ define <2 x i32> @test31(<2 x i32> %x) {
 
 define i32 @test32(i32 %a, i32 %b) {
 ; CHECK-LABEL: @test32(
-; CHECK-NEXT:    [[SHL:%.*]] = shl i32 2, [[B:%.*]]
-; CHECK-NEXT:    [[DIV:%.*]] = lshr i32 [[SHL]], 2
-; CHECK-NEXT:    [[DIV2:%.*]] = udiv i32 [[A:%.*]], [[DIV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[B:%.*]], -1
+; CHECK-NEXT:    [[DIV2:%.*]] = lshr i32 [[A:%.*]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[DIV2]]
 ;
   %shl = shl i32 2, %b
@@ -1832,3 +1831,41 @@ define i32 @fold_disjoint_or_over_udiv(i32 %x) {
   %r = udiv i32 %or, 9
   ret i32 %r
 }
+
+define i8 @udiv_trunc_shl(i32 %x) {
+; CHECK-LABEL: @udiv_trunc_shl(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[UDIV1:%.*]] = lshr i8 8, [[TMP1]]
+; CHECK-NEXT:    ret i8 [[UDIV1]]
+;
+  %lshr = shl i32 1, %x
+  %trunc = trunc i32 %lshr to i8
+  %div = udiv i8 8, %trunc
+  ret i8 %div
+}
+
+define i32 @zext_udiv_trunc_lshr(i32 %x) {
+; CHECK-LABEL: @zext_udiv_trunc_lshr(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i8 5, [[TMP1]]
+; CHECK-NEXT:    [[UDIV1:%.*]] = lshr i8 8, [[TMP2]]
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext nneg i8 [[UDIV1]] to i32
+; CHECK-NEXT:    ret i32 [[ZEXT]]
+;
+  %lshr = lshr i32 32, %x
+  %trunc = trunc i32 %lshr to i8
+  %div = udiv i8 8, %trunc
+  %zext = zext i8 %div to i32
+  ret i32 %zext
+}
+
+define i32 @udiv_and_shl(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @udiv_and_shl(
+; CHECK-NEXT:    [[DIV1:%.*]] = lshr i32 [[C:%.*]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[DIV1]]
+;
+  %shl = shl i32 1, %a
+  %and = and i32 %b, %shl
+  %div = udiv i32 %c, %and
+  ret i32 %div
+}
diff --git a/llvm/test/Transforms/InstCombine/shift.ll b/llvm/test/Transforms/InstCombine/shift.ll
index 558f4ffbfcab..69f531e98f04 100644
--- a/llvm/test/Transforms/InstCombine/shift.ll
+++ b/llvm/test/Transforms/InstCombine/shift.ll
@@ -677,8 +677,8 @@ entry:
 
 define i32 @test42(i32 %a, i32 %b) {
 ; CHECK-LABEL: @test42(
-; CHECK-NEXT:    [[DIV:%.*]] = lshr exact i32 4096, [[B:%.*]]
-; CHECK-NEXT:    [[DIV2:%.*]] = udiv i32 [[A:%.*]], [[DIV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 12, [[B:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = lshr i32 [[A:%.*]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[DIV2]]
 ;
   %div = lshr i32 4096, %b    ; must be exact otherwise we'd divide by zero
@@ -688,8 +688,8 @@ define i32 @test42(i32 %a, i32 %b) {
 
 define <2 x i32> @test42vec(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: @test42vec(
-; CHECK-NEXT:    [[DIV:%.*]] = lshr exact <2 x i32> <i32 4096, i32 4096>, [[B:%.*]]
-; CHECK-NEXT:    [[DIV2:%.*]] = udiv <2 x i32> [[A:%.*]], [[DIV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i32> <i32 12, i32 12>, [[B:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = lshr <2 x i32> [[A:%.*]], [[TMP1]]
 ; CHECK-NEXT:    ret <2 x i32> [[DIV2]]
 ;
   %div = lshr <2 x i32> <i32 4096, i32 4096>, %b    ; must be exact otherwise we'd divide by zero
-- 
GitLab


From 819abe412dd554303cb932d6ec2200b9b9ebdd78 Mon Sep 17 00:00:00 2001
From: Serge Pavlov <sepavloff@gmail.com>
Date: Mon, 28 Oct 2024 14:07:32 +0700
Subject: [PATCH 279/693] [Test] Fix usage of constrained intrinsics (#113523)

Some tests contain errors in constrained intrinsic usage, such as missed
or extra type parameters, wrong type parameters order and some other.

---------

Co-authored-by: Andy Kaylor <andy_kaylor@yahoo.com>
---
 llvm/test/CodeGen/AArch64/cvt-fp-int-fp.ll    | 32 +++++------
 .../CodeGen/AArch64/fp-intrinsics-fp16.ll     | 16 +++---
 .../CodeGen/AArch64/fp-intrinsics-vector.ll   |  6 +-
 llvm/test/CodeGen/AArch64/fp-intrinsics.ll    | 48 ++++++++--------
 llvm/test/CodeGen/ARM/fp-intrinsics.ll        | 56 +++++++++----------
 .../ppcf128-constrained-fp-intrinsics.ll      | 28 +++++-----
 .../CodeGen/RISCV/double-intrinsics-strict.ll |  4 +-
 .../CodeGen/RISCV/float-intrinsics-strict.ll  |  4 +-
 .../RISCV/rvv/rvv-peephole-vmerge-vops.ll     |  6 +-
 llvm/test/CodeGen/X86/bfloat-constrained.ll   | 18 +++---
 .../CodeGen/X86/float-strict-powi-convert.ll  |  4 +-
 .../Transforms/EarlyCSE/defaultfp-strictfp.ll | 20 +++----
 .../Transforms/EarlyCSE/ebstrict-strictfp.ll  | 12 ++--
 .../Transforms/EarlyCSE/mixed-strictfp.ll     | 20 +++----
 .../Transforms/EarlyCSE/nonmixed-strictfp.ll  | 20 +++----
 .../Transforms/SCCP/strictfp-phis-fcmp.ll     | 26 ++++-----
 .../Transforms/SCCP/strictfp-phis-fcmps.ll    | 26 ++++-----
 17 files changed, 173 insertions(+), 173 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/cvt-fp-int-fp.ll b/llvm/test/CodeGen/AArch64/cvt-fp-int-fp.ll
index 049098ab2ae9..40684b0f3a25 100644
--- a/llvm/test/CodeGen/AArch64/cvt-fp-int-fp.ll
+++ b/llvm/test/CodeGen/AArch64/cvt-fp-int-fp.ll
@@ -131,7 +131,7 @@ define double @t1_strict(double %x) #0 {
 ; CHECK-NEXT:    ret
 entry:
   %conv = call i64 @llvm.experimental.constrained.fptosi.i64.f64(double %x, metadata !"fpexcept.strict") #0
-  %conv1 = call double @llvm.experimental.constrained.sitofp.i64.f64(i64 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %conv1 = call double @llvm.experimental.constrained.sitofp.f64.i64(i64 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret double %conv1
 }
 
@@ -143,7 +143,7 @@ define float @t2_strict(float %x) #0 {
 ; CHECK-NEXT:    ret
 entry:
   %conv = call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %x, metadata !"fpexcept.strict") #0
-  %conv1 = call float @llvm.experimental.constrained.sitofp.i32.f32(i32 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %conv1 = call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret float %conv1
 }
 
@@ -155,7 +155,7 @@ define half @t3_strict(half %x) #0 {
 ; CHECK-NEXT:    ret
 entry:
   %conv = call i32 @llvm.experimental.constrained.fptosi.i32.f16(half %x, metadata !"fpexcept.strict") #0
-  %conv1 = call half @llvm.experimental.constrained.sitofp.i32.f16(i32 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %conv1 = call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret half %conv1
 }
 
@@ -167,7 +167,7 @@ define double @t4_strict(double %x) #0 {
 ; CHECK-NEXT:    ret
 entry:
   %conv = call i64 @llvm.experimental.constrained.fptoui.i64.f64(double %x, metadata !"fpexcept.strict") #0
-  %conv1 = call double @llvm.experimental.constrained.uitofp.i64.f64(i64 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %conv1 = call double @llvm.experimental.constrained.uitofp.f64.i64(i64 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret double %conv1
 }
 
@@ -179,7 +179,7 @@ define float @t5_strict(float %x) #0 {
 ; CHECK-NEXT:    ret
 entry:
   %conv = call i32 @llvm.experimental.constrained.fptoui.i32.f32(float %x, metadata !"fpexcept.strict") #0
-  %conv1 = call float @llvm.experimental.constrained.uitofp.i32.f32(i32 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %conv1 = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret float %conv1
 }
 
@@ -191,7 +191,7 @@ define half @t6_strict(half %x) #0 {
 ; CHECK-NEXT:    ret
 entry:
   %conv = call i32 @llvm.experimental.constrained.fptoui.i32.f16(half %x, metadata !"fpexcept.strict") #0
-  %conv1 = call half @llvm.experimental.constrained.uitofp.i32.f16(i32 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %conv1 = call half @llvm.experimental.constrained.uitofp.f16.i32(i32 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret half %conv1
 }
 
@@ -216,7 +216,7 @@ define bfloat @t7_strict(bfloat %x) #0 {
 ; CHECK-NEXT:    ret
 entry:
   %conv = call i32 @llvm.experimental.constrained.fptosi.i32.bf16(bfloat %x, metadata !"fpexcept.strict") #0
-  %conv1 = call bfloat @llvm.experimental.constrained.sitofp.i32.bf16(i32 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %conv1 = call bfloat @llvm.experimental.constrained.sitofp.bf16.i32(i32 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret bfloat %conv1
 }
 
@@ -241,7 +241,7 @@ define bfloat @t8_strict(bfloat %x) #0 {
 ; CHECK-NEXT:    ret
 entry:
   %conv = call i32 @llvm.experimental.constrained.fptoui.i32.bf16(bfloat %x, metadata !"fpexcept.strict") #0
-  %conv1 = call bfloat @llvm.experimental.constrained.uitofp.i32.bf16(i32 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %conv1 = call bfloat @llvm.experimental.constrained.uitofp.bf16.i32(i32 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret bfloat %conv1
 }
 
@@ -255,11 +255,11 @@ declare i32 @llvm.experimental.constrained.fptosi.i32.f32(float, metadata)
 declare i32 @llvm.experimental.constrained.fptoui.i32.f32(float, metadata)
 declare i64 @llvm.experimental.constrained.fptosi.i64.f64(double, metadata)
 declare i64 @llvm.experimental.constrained.fptoui.i64.f64(double, metadata)
-declare bfloat @llvm.experimental.constrained.sitofp.i32.bf16(i32, metadata, metadata)
-declare bfloat @llvm.experimental.constrained.uitofp.i32.bf16(i32, metadata, metadata)
-declare half @llvm.experimental.constrained.sitofp.i32.f16(i32, metadata, metadata)
-declare half @llvm.experimental.constrained.uitofp.i32.f16(i32, metadata, metadata)
-declare float @llvm.experimental.constrained.sitofp.i32.f32(i32, metadata, metadata)
-declare float @llvm.experimental.constrained.uitofp.i32.f32(i32, metadata, metadata)
-declare double @llvm.experimental.constrained.sitofp.i64.f64(i64, metadata, metadata)
-declare double @llvm.experimental.constrained.uitofp.i64.f64(i64, metadata, metadata)
+declare bfloat @llvm.experimental.constrained.sitofp.bf16.i32(i32, metadata, metadata)
+declare bfloat @llvm.experimental.constrained.uitofp.bf16.i32(i32, metadata, metadata)
+declare half @llvm.experimental.constrained.sitofp.f16.i32(i32, metadata, metadata)
+declare half @llvm.experimental.constrained.uitofp.f16.i32(i32, metadata, metadata)
+declare float @llvm.experimental.constrained.sitofp.f32.i32(i32, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i32(i32, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i64(i64, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i64(i64, metadata, metadata)
diff --git a/llvm/test/CodeGen/AArch64/fp-intrinsics-fp16.ll b/llvm/test/CodeGen/AArch64/fp-intrinsics-fp16.ll
index 3aeefab52c6f..4cce06dce44c 100644
--- a/llvm/test/CodeGen/AArch64/fp-intrinsics-fp16.ll
+++ b/llvm/test/CodeGen/AArch64/fp-intrinsics-fp16.ll
@@ -595,7 +595,7 @@ define i32 @lrint_f16(half %x) #0 {
 ; CHECK-FP16-NEXT:    frintx h0, h0
 ; CHECK-FP16-NEXT:    fcvtzs w0, h0
 ; CHECK-FP16-NEXT:    ret
-  %val = call i32 @llvm.experimental.constrained.lrint.f16(half %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.lrint.i32.f16(half %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
@@ -612,7 +612,7 @@ define i64 @llrint_f16(half %x) #0 {
 ; CHECK-FP16-NEXT:    frintx h0, h0
 ; CHECK-FP16-NEXT:    fcvtzs x0, h0
 ; CHECK-FP16-NEXT:    ret
-  %val = call i64 @llvm.experimental.constrained.llrint.f16(half %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %val = call i64 @llvm.experimental.constrained.llrint.i64.f16(half %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret i64 %val
 }
 
@@ -693,7 +693,7 @@ define i32 @lround_f16(half %x) #0 {
 ; CHECK-FP16:       // %bb.0:
 ; CHECK-FP16-NEXT:    fcvtas w0, h0
 ; CHECK-FP16-NEXT:    ret
-  %val = call i32 @llvm.experimental.constrained.lround.f16(half %x, metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.lround.i32.f16(half %x, metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
@@ -708,7 +708,7 @@ define i64 @llround_f16(half %x) #0 {
 ; CHECK-FP16:       // %bb.0:
 ; CHECK-FP16-NEXT:    fcvtas x0, h0
 ; CHECK-FP16-NEXT:    ret
-  %val = call i64 @llvm.experimental.constrained.llround.f16(half %x, metadata !"fpexcept.strict") #0
+  %val = call i64 @llvm.experimental.constrained.llround.i64.f16(half %x, metadata !"fpexcept.strict") #0
   ret i64 %val
 }
 
@@ -1277,14 +1277,14 @@ declare half @llvm.experimental.constrained.exp.f16(half, metadata, metadata)
 declare half @llvm.experimental.constrained.exp2.f16(half, metadata, metadata)
 declare half @llvm.experimental.constrained.rint.f16(half, metadata, metadata)
 declare half @llvm.experimental.constrained.nearbyint.f16(half, metadata, metadata)
-declare i32 @llvm.experimental.constrained.lrint.f16(half, metadata, metadata)
-declare i64 @llvm.experimental.constrained.llrint.f16(half, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lrint.i32.f16(half, metadata, metadata)
+declare i64 @llvm.experimental.constrained.llrint.i64.f16(half, metadata, metadata)
 declare half @llvm.experimental.constrained.maxnum.f16(half, half, metadata)
 declare half @llvm.experimental.constrained.minnum.f16(half, half, metadata)
 declare half @llvm.experimental.constrained.ceil.f16(half, metadata)
 declare half @llvm.experimental.constrained.floor.f16(half, metadata)
-declare i32 @llvm.experimental.constrained.lround.f16(half, metadata)
-declare i64 @llvm.experimental.constrained.llround.f16(half, metadata)
+declare i32 @llvm.experimental.constrained.lround.i32.f16(half, metadata)
+declare i64 @llvm.experimental.constrained.llround.i64.f16(half, metadata)
 declare half @llvm.experimental.constrained.round.f16(half, metadata)
 declare half @llvm.experimental.constrained.roundeven.f16(half, metadata)
 declare half @llvm.experimental.constrained.trunc.f16(half, metadata)
diff --git a/llvm/test/CodeGen/AArch64/fp-intrinsics-vector.ll b/llvm/test/CodeGen/AArch64/fp-intrinsics-vector.ll
index 1a9ba9fd4a51..6147afba4e60 100644
--- a/llvm/test/CodeGen/AArch64/fp-intrinsics-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fp-intrinsics-vector.ll
@@ -279,7 +279,7 @@ define <4 x i1> @fcmps_v4f32(<4 x float> %x, <4 x float> %y) #0 {
 ; CHECK-NEXT:    xtn v0.4h, v4.4s
 ; CHECK-NEXT:    ret
 entry:
-  %val = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(<4 x float> %x, <4 x float> %y, metadata !"oeq", metadata !"fpexcept.strict")
+  %val = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(<4 x float> %x, <4 x float> %y, metadata !"oeq", metadata !"fpexcept.strict")
   ret <4 x i1> %val
 }
 
@@ -825,8 +825,8 @@ declare <4 x float> @llvm.experimental.constrained.floor.v4f32(<4 x float>, meta
 declare <4 x float> @llvm.experimental.constrained.round.v4f32(<4 x float>, metadata)
 declare <4 x float> @llvm.experimental.constrained.roundeven.v4f32(<4 x float>, metadata)
 declare <4 x float> @llvm.experimental.constrained.trunc.v4f32(<4 x float>, metadata)
-declare <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(<4 x float>, <4 x float>, metadata, metadata)
-declare <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(<4 x float>, <4 x float>, metadata, metadata)
+declare <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(<4 x float>, <4 x float>, metadata, metadata)
+declare <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(<4 x float>, <4 x float>, metadata, metadata)
 
 declare <2 x double> @llvm.experimental.constrained.fadd.v2f64(<2 x double>, <2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.fsub.v2f64(<2 x double>, <2 x double>, metadata, metadata)
diff --git a/llvm/test/CodeGen/AArch64/fp-intrinsics.ll b/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
index 1664fa3ce56a..fd3a0c320760 100644
--- a/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
@@ -262,7 +262,7 @@ define float @nearbyint_f32(float %x) #0 {
 ; CHECK: frintx [[REG:s[0-9]+]], s0
 ; CHECK: fcvtzs w0, [[REG]]
 define i32 @lrint_f32(float %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.lrint.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.lrint.i32.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
@@ -270,7 +270,7 @@ define i32 @lrint_f32(float %x) #0 {
 ; CHECK: frintx [[REG:s[0-9]+]], s0
 ; CHECK: fcvtzs x0, [[REG]]
 define i64 @llrint_f32(float %x) #0 {
-  %val = call i64 @llvm.experimental.constrained.llrint.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %val = call i64 @llvm.experimental.constrained.llrint.i64.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret i64 %val
 }
 
@@ -319,14 +319,14 @@ define float @floor_f32(float %x) #0 {
 ; CHECK-LABEL: lround_f32:
 ; CHECK: fcvtas w0, s0
 define i32 @lround_f32(float %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.lround.f32(float %x, metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.lround.i32.f32(float %x, metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
 ; CHECK-LABEL: llround_f32:
 ; CHECK: fcvtas x0, s0
 define i64 @llround_f32(float %x) #0 {
-  %val = call i64 @llvm.experimental.constrained.llround.f32(float %x, metadata !"fpexcept.strict") #0
+  %val = call i64 @llvm.experimental.constrained.llround.i64.f32(float %x, metadata !"fpexcept.strict") #0
   ret i64 %val
 }
 
@@ -802,7 +802,7 @@ define double @nearbyint_f64(double %x) #0 {
 ; CHECK: frintx [[REG:d[0-9]+]], d0
 ; CHECK: fcvtzs w0, [[REG]]
 define i32 @lrint_f64(double %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.lrint.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.lrint.i32.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
@@ -810,7 +810,7 @@ define i32 @lrint_f64(double %x) #0 {
 ; CHECK: frintx [[REG:d[0-9]+]], d0
 ; CHECK: fcvtzs x0, [[REG]]
 define i64 @llrint_f64(double %x) #0 {
-  %val = call i64 @llvm.experimental.constrained.llrint.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %val = call i64 @llvm.experimental.constrained.llrint.i64.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret i64 %val
 }
 
@@ -859,14 +859,14 @@ define double @floor_f64(double %x) #0 {
 ; CHECK-LABEL: lround_f64:
 ; CHECK: fcvtas w0, d0
 define i32 @lround_f64(double %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.lround.f64(double %x, metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.lround.i32.f64(double %x, metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
 ; CHECK-LABEL: llround_f64:
 ; CHECK: fcvtas x0, d0
 define i64 @llround_f64(double %x) #0 {
-  %val = call i64 @llvm.experimental.constrained.llround.f64(double %x, metadata !"fpexcept.strict") #0
+  %val = call i64 @llvm.experimental.constrained.llround.i64.f64(double %x, metadata !"fpexcept.strict") #0
   ret i64 %val
 }
 
@@ -1341,14 +1341,14 @@ define fp128 @nearbyint_f128(fp128 %x) #0 {
 ; CHECK-LABEL: lrint_f128:
 ; CHECK: bl lrintl
 define i32 @lrint_f128(fp128 %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.lrint.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.lrint.i32.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
 ; CHECK-LABEL: llrint_f128:
 ; CHECK: bl llrintl
 define i64 @llrint_f128(fp128 %x) #0 {
-  %val = call i64 @llvm.experimental.constrained.llrint.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %val = call i64 @llvm.experimental.constrained.llrint.i64.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret i64 %val
 }
 
@@ -1383,14 +1383,14 @@ define fp128 @floor_f128(fp128 %x) #0 {
 ; CHECK-LABEL: lround_f128:
 ; CHECK: bl lroundl
 define i32 @lround_f128(fp128 %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.lround.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.lround.i32.f128(fp128 %x, metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
 ; CHECK-LABEL: llround_f128:
 ; CHECK: bl llroundl
 define i64 @llround_f128(fp128 %x) #0 {
-  %val = call i64 @llvm.experimental.constrained.llround.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  %val = call i64 @llvm.experimental.constrained.llround.i64.f128(fp128 %x, metadata !"fpexcept.strict") #0
   ret i64 %val
 }
 
@@ -1795,16 +1795,16 @@ declare float @llvm.experimental.constrained.exp.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.exp2.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.rint.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.nearbyint.f32(float, metadata, metadata)
-declare i32 @llvm.experimental.constrained.lrint.f32(float, metadata, metadata)
-declare i64 @llvm.experimental.constrained.llrint.f32(float, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lrint.i32.f32(float, metadata, metadata)
+declare i64 @llvm.experimental.constrained.llrint.i64.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.maxnum.f32(float, float, metadata)
 declare float @llvm.experimental.constrained.minnum.f32(float, float, metadata)
 declare float @llvm.experimental.constrained.maximum.f32(float, float, metadata)
 declare float @llvm.experimental.constrained.minimum.f32(float, float, metadata)
 declare float @llvm.experimental.constrained.ceil.f32(float, metadata)
 declare float @llvm.experimental.constrained.floor.f32(float, metadata)
-declare i32 @llvm.experimental.constrained.lround.f32(float, metadata)
-declare i64 @llvm.experimental.constrained.llround.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.lround.i32.f32(float, metadata)
+declare i64 @llvm.experimental.constrained.llround.i64.f32(float, metadata)
 declare float @llvm.experimental.constrained.round.f32(float, metadata)
 declare float @llvm.experimental.constrained.roundeven.f32(float, metadata)
 declare float @llvm.experimental.constrained.trunc.f32(float, metadata)
@@ -1847,16 +1847,16 @@ declare double @llvm.experimental.constrained.exp.f64(double, metadata, metadata
 declare double @llvm.experimental.constrained.exp2.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.rint.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata)
-declare i32 @llvm.experimental.constrained.lrint.f64(double, metadata, metadata)
-declare i64 @llvm.experimental.constrained.llrint.f64(double, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lrint.i32.f64(double, metadata, metadata)
+declare i64 @llvm.experimental.constrained.llrint.i64.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.maxnum.f64(double, double, metadata)
 declare double @llvm.experimental.constrained.minnum.f64(double, double, metadata)
 declare double @llvm.experimental.constrained.maximum.f64(double, double, metadata)
 declare double @llvm.experimental.constrained.minimum.f64(double, double, metadata)
 declare double @llvm.experimental.constrained.ceil.f64(double, metadata)
 declare double @llvm.experimental.constrained.floor.f64(double, metadata)
-declare i32 @llvm.experimental.constrained.lround.f64(double, metadata)
-declare i64 @llvm.experimental.constrained.llround.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.lround.i32.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.llround.i64.f64(double, metadata)
 declare double @llvm.experimental.constrained.round.f64(double, metadata)
 declare double @llvm.experimental.constrained.roundeven.f64(double, metadata)
 declare double @llvm.experimental.constrained.trunc.f64(double, metadata)
@@ -1899,14 +1899,14 @@ declare fp128 @llvm.experimental.constrained.exp.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.exp2.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.rint.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.nearbyint.f128(fp128, metadata, metadata)
-declare i32 @llvm.experimental.constrained.lrint.f128(fp128, metadata, metadata)
-declare i64 @llvm.experimental.constrained.llrint.f128(fp128, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lrint.i32.f128(fp128, metadata, metadata)
+declare i64 @llvm.experimental.constrained.llrint.i64.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.maxnum.f128(fp128, fp128, metadata)
 declare fp128 @llvm.experimental.constrained.minnum.f128(fp128, fp128, metadata)
 declare fp128 @llvm.experimental.constrained.ceil.f128(fp128, metadata)
 declare fp128 @llvm.experimental.constrained.floor.f128(fp128, metadata)
-declare i32 @llvm.experimental.constrained.lround.f128(fp128, metadata)
-declare i64 @llvm.experimental.constrained.llround.f128(fp128, metadata)
+declare i32 @llvm.experimental.constrained.lround.i32.f128(fp128, metadata)
+declare i64 @llvm.experimental.constrained.llround.i64.f128(fp128, metadata)
 declare fp128 @llvm.experimental.constrained.round.f128(fp128, metadata)
 declare fp128 @llvm.experimental.constrained.trunc.f128(fp128, metadata)
 declare i1 @llvm.experimental.constrained.fcmps.f128(fp128, fp128, metadata, metadata)
diff --git a/llvm/test/CodeGen/ARM/fp-intrinsics.ll b/llvm/test/CodeGen/ARM/fp-intrinsics.ll
index e286eb3226e4..ca2dc701bd1f 100644
--- a/llvm/test/CodeGen/ARM/fp-intrinsics.ll
+++ b/llvm/test/CodeGen/ARM/fp-intrinsics.ll
@@ -68,7 +68,7 @@ define float @fma_f32(float %x, float %y, float %z) #0 {
 ; CHECK-NOSP: bl __aeabi_f2iz
 ; CHECK-SP: vcvt.s32.f32
 define i32 @fptosi_f32(float %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.fptosi.f32(float %x, metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %x, metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
@@ -79,9 +79,9 @@ define i32 @fptosi_f32(float %x) #0 {
 ; FIXME-CHECK-SP: vcvt.s32.f32
 define void @fptosi_f32_twice(float %arg, ptr %ptr) #0 {
 entry:
-  %conv = call i32 @llvm.experimental.constrained.fptosi.f32(float %arg, metadata !"fpexcept.strict") #0
+  %conv = call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %arg, metadata !"fpexcept.strict") #0
   store i32 %conv, ptr %ptr, align 4
-  %conv1 = call i32 @llvm.experimental.constrained.fptosi.f32(float %arg, metadata !"fpexcept.strict") #0
+  %conv1 = call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %arg, metadata !"fpexcept.strict") #0
   %idx = getelementptr inbounds i32, ptr %ptr, i32 1
   store i32 %conv1, ptr %idx, align 4
   ret void
@@ -91,7 +91,7 @@ entry:
 ; CHECK-NOSP: bl __aeabi_f2uiz
 ; FIXME-CHECK-SP: vcvt.u32.f32
 define i32 @fptoui_f32(float %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.fptoui.f32(float %x, metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.fptoui.i32.f32(float %x, metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
@@ -102,9 +102,9 @@ define i32 @fptoui_f32(float %x) #0 {
 ; FIXME-CHECK-SP: vcvt.u32.f32
 define void @fptoui_f32_twice(float %arg, ptr %ptr) #0 {
 entry:
-  %conv = call i32 @llvm.experimental.constrained.fptoui.f32(float %arg, metadata !"fpexcept.strict") #0
+  %conv = call i32 @llvm.experimental.constrained.fptoui.i32.f32(float %arg, metadata !"fpexcept.strict") #0
   store i32 %conv, ptr %ptr, align 4
-  %conv1 = call i32 @llvm.experimental.constrained.fptoui.f32(float %arg, metadata !"fpexcept.strict") #0
+  %conv1 = call i32 @llvm.experimental.constrained.fptoui.i32.f32(float %arg, metadata !"fpexcept.strict") #0
   %idx = getelementptr inbounds i32, ptr %ptr, i32 1
   store i32 %conv1, ptr %idx, align 4
   ret void
@@ -209,14 +209,14 @@ define float @nearbyint_f32(float %x) #0 {
 ; CHECK-LABEL: lrint_f32:
 ; CHECK: bl lrintf
 define i32 @lrint_f32(float %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.lrint.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.lrint.i32.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
 ; CHECK-LABEL: llrint_f32:
 ; CHECK: bl llrintf
 define i32 @llrint_f32(float %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.llrint.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.llrint.i32.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
@@ -259,14 +259,14 @@ define float @floor_f32(float %x) #0 {
 ; CHECK-LABEL: lround_f32:
 ; CHECK: bl lroundf
 define i32 @lround_f32(float %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.lround.f32(float %x, metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.lround.i32.f32(float %x, metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
 ; CHECK-LABEL: llround_f32:
 ; CHECK: bl llroundf
 define i32 @llround_f32(float %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.llround.f32(float %x, metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.llround.i32.f32(float %x, metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
@@ -562,7 +562,7 @@ define double @fma_f64(double %x, double %y, double %z) #0 {
 ; CHECK-NODP: bl __aeabi_d2iz
 ; CHECK-DP: vcvt.s32.f64
 define i32 @fptosi_f64(double %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.fptosi.f64(double %x, metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.fptosi.i32.f64(double %x, metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
@@ -570,7 +570,7 @@ define i32 @fptosi_f64(double %x) #0 {
 ; CHECK-NODP: bl __aeabi_d2uiz
 ; FIXME-CHECK-DP: vcvt.u32.f64
 define i32 @fptoui_f64(double %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.fptoui.f64(double %x, metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.fptoui.i32.f64(double %x, metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
@@ -673,14 +673,14 @@ define double @nearbyint_f64(double %x) #0 {
 ; CHECK-LABEL: lrint_f64:
 ; CHECK: bl lrint
 define i32 @lrint_f64(double %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.lrint.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.lrint.i32.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
 ; CHECK-LABEL: llrint_f64:
 ; CHECK: bl llrint
 define i32 @llrint_f64(double %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.llrint.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.llrint.i32.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
@@ -723,14 +723,14 @@ define double @floor_f64(double %x) #0 {
 ; CHECK-LABEL: lround_f64:
 ; CHECK: bl lround
 define i32 @lround_f64(double %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.lround.f64(double %x, metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.lround.i32.f64(double %x, metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
 ; CHECK-LABEL: llround_f64:
 ; CHECK: bl llround
 define i32 @llround_f64(double %x) #0 {
-  %val = call i32 @llvm.experimental.constrained.llround.f64(double %x, metadata !"fpexcept.strict") #0
+  %val = call i32 @llvm.experimental.constrained.llround.i32.f64(double %x, metadata !"fpexcept.strict") #0
   ret i32 %val
 }
 
@@ -1031,8 +1031,8 @@ declare float @llvm.experimental.constrained.fmul.f32(float, float, metadata, me
 declare float @llvm.experimental.constrained.fdiv.f32(float, float, metadata, metadata)
 declare float @llvm.experimental.constrained.frem.f32(float, float, metadata, metadata)
 declare float @llvm.experimental.constrained.fma.f32(float, float, float, metadata, metadata)
-declare i32 @llvm.experimental.constrained.fptosi.f32(float, metadata)
-declare i32 @llvm.experimental.constrained.fptoui.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.i32.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f32(float, metadata)
 declare float @llvm.experimental.constrained.sqrt.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.powi.f32(float, i32, metadata, metadata)
 declare float @llvm.experimental.constrained.sin.f32(float, metadata, metadata)
@@ -1046,14 +1046,14 @@ declare float @llvm.experimental.constrained.exp.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.exp2.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.rint.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.nearbyint.f32(float, metadata, metadata)
-declare i32 @llvm.experimental.constrained.lrint.f32(float, metadata, metadata)
-declare i32 @llvm.experimental.constrained.llrint.f32(float, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lrint.i32.f32(float, metadata, metadata)
+declare i32 @llvm.experimental.constrained.llrint.i32.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.maxnum.f32(float, float, metadata)
 declare float @llvm.experimental.constrained.minnum.f32(float, float, metadata)
 declare float @llvm.experimental.constrained.ceil.f32(float, metadata)
 declare float @llvm.experimental.constrained.floor.f32(float, metadata)
-declare i32 @llvm.experimental.constrained.lround.f32(float, metadata)
-declare i32 @llvm.experimental.constrained.llround.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.lround.i32.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.llround.i32.f32(float, metadata)
 declare float @llvm.experimental.constrained.round.f32(float, metadata)
 declare float @llvm.experimental.constrained.trunc.f32(float, metadata)
 declare i1 @llvm.experimental.constrained.fcmps.f32(float, float, metadata, metadata)
@@ -1065,8 +1065,8 @@ declare double @llvm.experimental.constrained.fmul.f64(double, double, metadata,
 declare double @llvm.experimental.constrained.fdiv.f64(double, double, metadata, metadata)
 declare double @llvm.experimental.constrained.frem.f64(double, double, metadata, metadata)
 declare double @llvm.experimental.constrained.fma.f64(double, double, double, metadata, metadata)
-declare i32 @llvm.experimental.constrained.fptosi.f64(double, metadata)
-declare i32 @llvm.experimental.constrained.fptoui.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.i32.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
 declare double @llvm.experimental.constrained.sqrt.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.powi.f64(double, i32, metadata, metadata)
 declare double @llvm.experimental.constrained.sin.f64(double, metadata, metadata)
@@ -1080,14 +1080,14 @@ declare double @llvm.experimental.constrained.exp.f64(double, metadata, metadata
 declare double @llvm.experimental.constrained.exp2.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.rint.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata)
-declare i32 @llvm.experimental.constrained.lrint.f64(double, metadata, metadata)
-declare i32 @llvm.experimental.constrained.llrint.f64(double, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lrint.i32.f64(double, metadata, metadata)
+declare i32 @llvm.experimental.constrained.llrint.i32.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.maxnum.f64(double, double, metadata)
 declare double @llvm.experimental.constrained.minnum.f64(double, double, metadata)
 declare double @llvm.experimental.constrained.ceil.f64(double, metadata)
 declare double @llvm.experimental.constrained.floor.f64(double, metadata)
-declare i32 @llvm.experimental.constrained.lround.f64(double, metadata)
-declare i32 @llvm.experimental.constrained.llround.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.lround.i32.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.llround.i32.f64(double, metadata)
 declare double @llvm.experimental.constrained.round.f64(double, metadata)
 declare double @llvm.experimental.constrained.trunc.f64(double, metadata)
 declare i1 @llvm.experimental.constrained.fcmps.f64(double, double, metadata, metadata)
diff --git a/llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll b/llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll
index 76f3dea5b775..3e8935e7d597 100644
--- a/llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll
+++ b/llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll
@@ -1083,7 +1083,7 @@ define float @test_fptrunc_ppc_fp128_f32(ppc_fp128 %first) #0 {
 ; PC64-NEXT:    frsp 1, 1
 ; PC64-NEXT:    blr
 entry:
-  %fptrunc = call float @llvm.experimental.constrained.fptrunc.ppcf128.f32(
+  %fptrunc = call float @llvm.experimental.constrained.fptrunc.f32.ppcf128.f32(
                     ppc_fp128 %first,
                     metadata !"round.dynamic",
                     metadata !"fpexcept.strict") #1
@@ -1103,7 +1103,7 @@ define double @test_fptrunc_ppc_fp128_f64(ppc_fp128 %first) #0 {
 ; PC64:       # %bb.0: # %entry
 ; PC64-NEXT:    blr
 entry:
-  %fptrunc = call double @llvm.experimental.constrained.fptrunc.ppcf128.f64(
+  %fptrunc = call double @llvm.experimental.constrained.fptrunc.f64.ppcf128(
                     ppc_fp128 %first,
                     metadata !"round.dynamic",
                     metadata !"fpexcept.strict") #1
@@ -1127,7 +1127,7 @@ define ppc_fp128 @test_fpext_ppc_fp128_f32(float %first) #0 {
 ; PC64-NEXT:    lfs 2, .LCPI26_0@toc@l(3)
 ; PC64-NEXT:    blr
 entry:
-  %fpext = call ppc_fp128 @llvm.experimental.constrained.fpext.f32.ppcf128(
+  %fpext = call ppc_fp128 @llvm.experimental.constrained.fpext.ppcf128.f32(
                     float %first,
                     metadata !"fpexcept.strict") #1
   ret ppc_fp128 %fpext
@@ -1150,7 +1150,7 @@ define ppc_fp128 @test_fpext_ppc_fp128_f64(double %first) #0 {
 ; PC64-NEXT:    lfs 2, .LCPI27_0@toc@l(3)
 ; PC64-NEXT:    blr
 entry:
-  %fpext = call ppc_fp128 @llvm.experimental.constrained.fpext.f64.ppcf128(
+  %fpext = call ppc_fp128 @llvm.experimental.constrained.fpext.ppcf128.f64(
                     double %first,
                     metadata !"fpexcept.strict") #1
   ret ppc_fp128 %fpext
@@ -1568,7 +1568,7 @@ define void @test_constrained_libcall_multichain(ptr %firstptr, ptr %result) #0
 ; PC64-NEXT:    mtlr 0
 ; PC64-NEXT:    blr
   %load = load float, ptr %firstptr
-  %first = call ppc_fp128 @llvm.experimental.constrained.fpext.f32.ppcf128(
+  %first = call ppc_fp128 @llvm.experimental.constrained.fpext.ppcf128.f32(
                     float %load,
                     metadata !"fpexcept.strict") #1
   store ppc_fp128 %first, ptr %result
@@ -1598,7 +1598,7 @@ define void @test_constrained_libcall_multichain(ptr %firstptr, ptr %result) #0
                     i32 2,
                     metadata !"round.dynamic",
                     metadata !"fpexcept.strict") #1
-  %tinypow = call float @llvm.experimental.constrained.fptrunc.ppcf128.f32(
+  %tinypow = call float @llvm.experimental.constrained.fptrunc.f32.ppcf128(
                     ppc_fp128 %powi,
                     metadata !"round.dynamic",
                     metadata !"fpexcept.strict") #1
@@ -2015,7 +2015,7 @@ define i1 @ppcq_to_s1(ppc_fp128 %a) {
 ; PC64-NEXT:    mtlr 0
 ; PC64-NEXT:    blr
 entry:
-  %conv = tail call i1 @llvm.experimental.constrained.fptosi.ppcf128.i1(ppc_fp128 %a, metadata !"fpexcept.strict") #1
+  %conv = tail call i1 @llvm.experimental.constrained.fptosi.i1.ppcf128(ppc_fp128 %a, metadata !"fpexcept.strict") #1
   ret i1 %conv
 }
 
@@ -2062,7 +2062,7 @@ define i1 @ppcq_to_u1(ppc_fp128 %a) {
 ; PC64-NEXT:    mtlr 0
 ; PC64-NEXT:    blr
 entry:
-  %conv = tail call i1 @llvm.experimental.constrained.fptoui.ppcf128.i1(ppc_fp128 %a, metadata !"fpexcept.strict") #1
+  %conv = tail call i1 @llvm.experimental.constrained.fptoui.i1.ppcf128(ppc_fp128 %a, metadata !"fpexcept.strict") #1
   ret i1 %conv
 }
 
@@ -2121,10 +2121,10 @@ declare ppc_fp128 @llvm.experimental.constrained.exp.ppcf128(ppc_fp128, metadata
 declare ppc_fp128 @llvm.experimental.constrained.exp2.ppcf128(ppc_fp128, metadata, metadata)
 declare ppc_fp128 @llvm.experimental.constrained.floor.ppcf128(ppc_fp128, metadata)
 declare ppc_fp128 @llvm.experimental.constrained.fma.ppcf128(ppc_fp128, ppc_fp128, ppc_fp128, metadata, metadata)
-declare ppc_fp128 @llvm.experimental.constrained.fpext.f32.ppcf128(float, metadata)
-declare ppc_fp128 @llvm.experimental.constrained.fpext.f64.ppcf128(double, metadata)
-declare float @llvm.experimental.constrained.fptrunc.ppcf128.f32(ppc_fp128, metadata, metadata)
-declare double @llvm.experimental.constrained.fptrunc.ppcf128.f64(ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.fpext.ppcf128.f32(float, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.fpext.ppcf128.f64(double, metadata)
+declare float @llvm.experimental.constrained.fptrunc.f32.ppcf128(ppc_fp128, metadata, metadata)
+declare double @llvm.experimental.constrained.fptrunc.f64.ppcf128(ppc_fp128, metadata, metadata)
 declare ppc_fp128 @llvm.experimental.constrained.log.ppcf128(ppc_fp128, metadata, metadata)
 declare ppc_fp128 @llvm.experimental.constrained.log10.ppcf128(ppc_fp128, metadata, metadata)
 declare ppc_fp128 @llvm.experimental.constrained.log2.ppcf128(ppc_fp128, metadata, metadata)
@@ -2144,10 +2144,10 @@ declare ppc_fp128 @llvm.experimental.constrained.tan.ppcf128(ppc_fp128, metadata
 declare ppc_fp128 @llvm.experimental.constrained.trunc.ppcf128(ppc_fp128, metadata)
 declare i64 @llvm.experimental.constrained.fptosi.i64.ppcf128(ppc_fp128, metadata)
 declare i32 @llvm.experimental.constrained.fptosi.i32.ppcf128(ppc_fp128, metadata)
-declare i1 @llvm.experimental.constrained.fptosi.ppcf128.i1(ppc_fp128, metadata)
+declare i1 @llvm.experimental.constrained.fptosi.i1.ppcf128(ppc_fp128, metadata)
 declare i64 @llvm.experimental.constrained.fptoui.i64.ppcf128(ppc_fp128, metadata)
 declare i32 @llvm.experimental.constrained.fptoui.i32.ppcf128(ppc_fp128, metadata)
-declare i1 @llvm.experimental.constrained.fptoui.ppcf128.i1(ppc_fp128, metadata)
+declare i1 @llvm.experimental.constrained.fptoui.i1.ppcf128(ppc_fp128, metadata)
 declare ppc_fp128 @llvm.experimental.constrained.sitofp.ppcf128.i32(i32, metadata, metadata)
 declare ppc_fp128 @llvm.experimental.constrained.uitofp.ppcf128.i32(i32, metadata, metadata)
 declare ppc_fp128 @llvm.experimental.constrained.sitofp.ppcf128.i64(i64, metadata, metadata)
diff --git a/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll b/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll
index 4cb6191e7322..7e5ea173e522 100644
--- a/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll
+++ b/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll
@@ -57,7 +57,7 @@ define double @sqrt_f64(double %a) nounwind strictfp {
   ret double %1
 }
 
-declare double @llvm.experimental.constrained.powi.f64.i32(double, i32, metadata, metadata)
+declare double @llvm.experimental.constrained.powi.f64(double, i32, metadata, metadata)
 
 define double @powi_f64(double %a, i32 %b) nounwind strictfp {
 ; RV32IFD-LABEL: powi_f64:
@@ -116,7 +116,7 @@ define double @powi_f64(double %a, i32 %b) nounwind strictfp {
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call double @llvm.experimental.constrained.powi.f64.i32(double %a, i32 %b, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  %1 = call double @llvm.experimental.constrained.powi.f64(double %a, i32 %b, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
   ret double %1
 }
 
diff --git a/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll b/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll
index cbd84634de11..7b2d38fefaac 100644
--- a/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll
+++ b/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll
@@ -52,7 +52,7 @@ define float @sqrt_f32(float %a) nounwind strictfp {
   ret float %1
 }
 
-declare float @llvm.experimental.constrained.powi.f32.i32(float, i32, metadata, metadata)
+declare float @llvm.experimental.constrained.powi.f32(float, i32, metadata, metadata)
 
 define float @powi_f32(float %a, i32 %b) nounwind strictfp {
 ; RV32IF-LABEL: powi_f32:
@@ -111,7 +111,7 @@ define float @powi_f32(float %a, i32 %b) nounwind strictfp {
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call float @llvm.experimental.constrained.powi.f32.i32(float %a, i32 %b, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  %1 = call float @llvm.experimental.constrained.powi.f32(float %a, i32 %b, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
   ret float %1
 }
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
index 6700920cebff..23ebfade6f6b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
@@ -357,11 +357,11 @@ define <vscale x 2 x float> @vpmerge_constrained_fadd(<vscale x 2 x float> %pass
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
 ; CHECK-NEXT:    vmerge.vvm v8, v8, v9, v0
 ; CHECK-NEXT:    ret
-  %a = call <vscale x 2 x float> @llvm.experimental.constrained.fadd(<vscale x 2 x float> %x, <vscale x 2 x float> %y, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  %a = call <vscale x 2 x float> @llvm.experimental.constrained.fadd.nxv2f32(<vscale x 2 x float> %x, <vscale x 2 x float> %y, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
   %b = call <vscale x 2 x float> @llvm.riscv.vmerge.nxv2f32.nxv2f32(<vscale x 2 x float> %passthru, <vscale x 2 x float> %passthru, <vscale x 2 x float> %a, <vscale x 2 x i1> %m, i64 %vl) strictfp
   ret <vscale x 2 x float> %b
 }
-declare <vscale x 2 x float> @llvm.experimental.constrained.fadd(<vscale x 2 x float>, <vscale x 2 x float>, metadata, metadata)
+declare <vscale x 2 x float> @llvm.experimental.constrained.fadd.nxv2f32(<vscale x 2 x float>, <vscale x 2 x float>, metadata, metadata)
 declare <vscale x 2 x float> @llvm.riscv.vmerge.nxv2f32.nxv2f32(<vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x i1>, i64)
 
 ; This shouldn't be folded because we need to preserve exceptions with
@@ -374,7 +374,7 @@ define <vscale x 2 x float> @vpmerge_constrained_fadd_vlmax(<vscale x 2 x float>
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, tu, ma
 ; CHECK-NEXT:    vmerge.vvm v8, v8, v9, v0
 ; CHECK-NEXT:    ret
-  %a = call <vscale x 2 x float> @llvm.experimental.constrained.fadd(<vscale x 2 x float> %x, <vscale x 2 x float> %y, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  %a = call <vscale x 2 x float> @llvm.experimental.constrained.fadd.nxv2f32(<vscale x 2 x float> %x, <vscale x 2 x float> %y, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
   %b = call <vscale x 2 x float> @llvm.riscv.vmerge.nxv2f32.nxv2f32(<vscale x 2 x float> %passthru, <vscale x 2 x float> %passthru, <vscale x 2 x float> %a, <vscale x 2 x i1> %m, i64 -1) strictfp
   ret <vscale x 2 x float> %b
 }
diff --git a/llvm/test/CodeGen/X86/bfloat-constrained.ll b/llvm/test/CodeGen/X86/bfloat-constrained.ll
index 0a8c4f20648b..081b1cebfc43 100644
--- a/llvm/test/CodeGen/X86/bfloat-constrained.ll
+++ b/llvm/test/CodeGen/X86/bfloat-constrained.ll
@@ -86,7 +86,7 @@ define void @float_to_bfloat(float %0) strictfp {
 ; X64-NEXT:    popq %rax
 ; X64-NEXT:    .cfi_def_cfa_offset 8
 ; X64-NEXT:    retq
-  %2 = tail call bfloat @llvm.experimental.constrained.fptrunc.bfloat.f32(float %0, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %2 = tail call bfloat @llvm.experimental.constrained.fptrunc.bf16.f32(float %0, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   store bfloat %2, ptr @a, align 2
   ret void
 }
@@ -115,7 +115,7 @@ define void @double_to_bfloat(double %0) strictfp {
 ; X64-NEXT:    popq %rax
 ; X64-NEXT:    .cfi_def_cfa_offset 8
 ; X64-NEXT:    retq
-  %2 = tail call bfloat @llvm.experimental.constrained.fptrunc.bfloat.f64(double %0, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %2 = tail call bfloat @llvm.experimental.constrained.fptrunc.bf16.f64(double %0, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   store bfloat %2, ptr @a, align 2
   ret void
 }
@@ -162,20 +162,20 @@ define void @add() strictfp {
 ; X64-NEXT:    .cfi_def_cfa_offset 8
 ; X64-NEXT:    retq
   %1 = load bfloat, ptr @a, align 2
-  %2 = tail call float @llvm.experimental.constrained.fpext.f32.bfloat(bfloat %1, metadata !"fpexcept.strict") #0
+  %2 = tail call float @llvm.experimental.constrained.fpext.f32.bf16(bfloat %1, metadata !"fpexcept.strict") #0
   %3 = load bfloat, ptr @b, align 2
-  %4 = tail call float @llvm.experimental.constrained.fpext.f32.bfloat(bfloat %3, metadata !"fpexcept.strict") #0
+  %4 = tail call float @llvm.experimental.constrained.fpext.f32.bf16(bfloat %3, metadata !"fpexcept.strict") #0
   %5 = tail call float @llvm.experimental.constrained.fadd.f32(float %2, float %4, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
-  %6 = tail call bfloat @llvm.experimental.constrained.fptrunc.bfloat.f32(float %5, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %6 = tail call bfloat @llvm.experimental.constrained.fptrunc.bf16.f32(float %5, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
   store bfloat %6, ptr @c, align 2
   ret void
 }
 
-declare float @llvm.experimental.constrained.fpext.f32.bfloat(bfloat, metadata)
-declare double @llvm.experimental.constrained.fpext.f64.bfloat(bfloat, metadata)
+declare float @llvm.experimental.constrained.fpext.f32.bf16(bfloat, metadata)
+declare double @llvm.experimental.constrained.fpext.f64.bf16(bfloat, metadata)
 declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata)
-declare bfloat @llvm.experimental.constrained.fptrunc.bfloat.f32(float, metadata, metadata)
-declare bfloat @llvm.experimental.constrained.fptrunc.bfloat.f64(double, metadata, metadata)
+declare bfloat @llvm.experimental.constrained.fptrunc.bf16.f32(float, metadata, metadata)
+declare bfloat @llvm.experimental.constrained.fptrunc.bf16.f64(double, metadata, metadata)
 
 attributes #0 = { strictfp }
 
diff --git a/llvm/test/CodeGen/X86/float-strict-powi-convert.ll b/llvm/test/CodeGen/X86/float-strict-powi-convert.ll
index 4d0cffc53d93..b39f5ec667ce 100644
--- a/llvm/test/CodeGen/X86/float-strict-powi-convert.ll
+++ b/llvm/test/CodeGen/X86/float-strict-powi-convert.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -mtriple=x86_64-pc-windows-msvc %s -o - | FileCheck %s -check-prefix=WIN
 ; RUN: llc -mtriple=x86_64-pc-linux %s -o -| FileCheck %s -check-prefix=UNIX
 
-declare float @llvm.experimental.constrained.powi.f32.i32(float, i32, metadata, metadata)
+declare float @llvm.experimental.constrained.powi.f32(float, i32, metadata, metadata)
 
 define float @powi_f64(float %a, i32 %b) nounwind strictfp {
 ; WIN-LABEL: powi_f64:
@@ -19,6 +19,6 @@ define float @powi_f64(float %a, i32 %b) nounwind strictfp {
 ; UNIX-NEXT:    callq __powisf2@PLT
 ; UNIX-NEXT:    popq %rax
 ; UNIX-NEXT:    retq
-  %1 = call float @llvm.experimental.constrained.powi.f32.i32(float %a, i32 %b, metadata !"round.tonearest", metadata !"fpexcept.ignore") strictfp
+  %1 = call float @llvm.experimental.constrained.powi.f32(float %a, i32 %b, metadata !"round.tonearest", metadata !"fpexcept.ignore") strictfp
   ret float %1
 }
diff --git a/llvm/test/Transforms/EarlyCSE/defaultfp-strictfp.ll b/llvm/test/Transforms/EarlyCSE/defaultfp-strictfp.ll
index bdfad0d6e44e..3871822c9dc1 100644
--- a/llvm/test/Transforms/EarlyCSE/defaultfp-strictfp.ll
+++ b/llvm/test/Transforms/EarlyCSE/defaultfp-strictfp.ll
@@ -246,8 +246,8 @@ define i1 @multiple_fcmp(double %a, double %b) #0 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @bar.i32(i32 [[TMP2]], i32 [[TMP2]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
-  %1 = call i1 @llvm.experimental.constrained.fcmp.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
-  %2 = call i1 @llvm.experimental.constrained.fcmp.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %1 = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %2 = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
   %3 = zext i1 %1 to i32
   %4 = zext i1 %2 to i32
   %5 = call i32 @bar.i32(i32 %3, i32 %4) #0
@@ -262,9 +262,9 @@ define i1 @multiple_fcmp_split(double %a, double %b) #0 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @bar.i32(i32 [[TMP2]], i32 [[TMP2]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
-  %1 = call i1 @llvm.experimental.constrained.fcmp.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %1 = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
   call void @arbitraryfunc() #0
-  %2 = call i1 @llvm.experimental.constrained.fcmp.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %2 = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
   %3 = zext i1 %1 to i32
   %4 = zext i1 %2 to i32
   %5 = call i32 @bar.i32(i32 %3, i32 %4) #0
@@ -278,8 +278,8 @@ define i1 @multiple_fcmps(double %a, double %b) #0 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @bar.i32(i32 [[TMP2]], i32 [[TMP2]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
-  %1 = call i1 @llvm.experimental.constrained.fcmps.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
-  %2 = call i1 @llvm.experimental.constrained.fcmps.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %1 = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %2 = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
   %3 = zext i1 %1 to i32
   %4 = zext i1 %2 to i32
   %5 = call i32 @bar.i32(i32 %3, i32 %4) #0
@@ -294,9 +294,9 @@ define i1 @multiple_fcmps_split(double %a, double %b) #0 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @bar.i32(i32 [[TMP2]], i32 [[TMP2]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
-  %1 = call i1 @llvm.experimental.constrained.fcmps.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %1 = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
   call void @arbitraryfunc() #0
-  %2 = call i1 @llvm.experimental.constrained.fcmps.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %2 = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
   %3 = zext i1 %1 to i32
   %4 = zext i1 %2 to i32
   %5 = call i32 @bar.i32(i32 %3, i32 %4) #0
@@ -318,5 +318,5 @@ declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
 declare double @llvm.experimental.constrained.uitofp.f64.i32(i32, metadata, metadata)
 declare i32 @llvm.experimental.constrained.fptosi.i32.f64(double, metadata)
 declare double @llvm.experimental.constrained.sitofp.f64.i32(i32, metadata, metadata)
-declare i1 @llvm.experimental.constrained.fcmp.i1.f64(double, double, metadata, metadata)
-declare i1 @llvm.experimental.constrained.fcmps.i1.f64(double, double, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f64(double, double, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f64(double, double, metadata, metadata)
diff --git a/llvm/test/Transforms/EarlyCSE/ebstrict-strictfp.ll b/llvm/test/Transforms/EarlyCSE/ebstrict-strictfp.ll
index fafc7ccbb38c..f2675ce7816a 100644
--- a/llvm/test/Transforms/EarlyCSE/ebstrict-strictfp.ll
+++ b/llvm/test/Transforms/EarlyCSE/ebstrict-strictfp.ll
@@ -132,8 +132,8 @@ define i1 @fcmp_strict(double %a, double %b) #0 {
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @bar.i32(i32 [[TMP3]], i32 [[TMP4]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
-  %1 = call i1 @llvm.experimental.constrained.fcmp.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
-  %2 = call i1 @llvm.experimental.constrained.fcmp.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %1 = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %2 = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
   %3 = zext i1 %1 to i32
   %4 = zext i1 %2 to i32
   %5 = call i32 @bar.i32(i32 %3, i32 %4) #0
@@ -149,8 +149,8 @@ define i1 @fcmps_strict(double %a, double %b) #0 {
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @bar.i32(i32 [[TMP3]], i32 [[TMP4]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
-  %1 = call i1 @llvm.experimental.constrained.fcmps.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
-  %2 = call i1 @llvm.experimental.constrained.fcmps.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %1 = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %2 = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
   %3 = zext i1 %1 to i32
   %4 = zext i1 %2 to i32
   %5 = call i32 @bar.i32(i32 %3, i32 %4) #0
@@ -172,5 +172,5 @@ declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
 declare double @llvm.experimental.constrained.uitofp.f64.i32(i32, metadata, metadata)
 declare i32 @llvm.experimental.constrained.fptosi.i32.f64(double, metadata)
 declare double @llvm.experimental.constrained.sitofp.f64.i32(i32, metadata, metadata)
-declare i1 @llvm.experimental.constrained.fcmp.i1.f64(double, double, metadata, metadata)
-declare i1 @llvm.experimental.constrained.fcmps.i1.f64(double, double, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f64(double, double, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f64(double, double, metadata, metadata)
diff --git a/llvm/test/Transforms/EarlyCSE/mixed-strictfp.ll b/llvm/test/Transforms/EarlyCSE/mixed-strictfp.ll
index f3b857ab2f48..b79f7018b8d0 100644
--- a/llvm/test/Transforms/EarlyCSE/mixed-strictfp.ll
+++ b/llvm/test/Transforms/EarlyCSE/mixed-strictfp.ll
@@ -339,8 +339,8 @@ define i1 @mixed_fcmp_maytrap(double %a, double %b) #0 {
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @bar.i32(i32 [[TMP3]], i32 [[TMP4]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
-  %1 = call i1 @llvm.experimental.constrained.fcmp.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
-  %2 = call i1 @llvm.experimental.constrained.fcmp.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.maytrap") #0
+  %1 = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %2 = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.maytrap") #0
   %3 = zext i1 %1 to i32
   %4 = zext i1 %2 to i32
   %5 = call i32 @bar.i32(i32 %3, i32 %4) #0
@@ -356,8 +356,8 @@ define i1 @mixed_fcmp_strict(double %a, double %b) #0 {
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @bar.i32(i32 [[TMP3]], i32 [[TMP4]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
-  %1 = call i1 @llvm.experimental.constrained.fcmp.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
-  %2 = call i1 @llvm.experimental.constrained.fcmp.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %1 = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %2 = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
   %3 = zext i1 %1 to i32
   %4 = zext i1 %2 to i32
   %5 = call i32 @bar.i32(i32 %3, i32 %4) #0
@@ -373,8 +373,8 @@ define i1 @mixed_fcmps_maytrap(double %a, double %b) #0 {
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @bar.i32(i32 [[TMP3]], i32 [[TMP4]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
-  %1 = call i1 @llvm.experimental.constrained.fcmps.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
-  %2 = call i1 @llvm.experimental.constrained.fcmps.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.maytrap") #0
+  %1 = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %2 = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.maytrap") #0
   %3 = zext i1 %1 to i32
   %4 = zext i1 %2 to i32
   %5 = call i32 @bar.i32(i32 %3, i32 %4) #0
@@ -390,8 +390,8 @@ define i1 @mixed_fcmps_strict(double %a, double %b) #0 {
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @bar.i32(i32 [[TMP3]], i32 [[TMP4]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
-  %1 = call i1 @llvm.experimental.constrained.fcmps.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
-  %2 = call i1 @llvm.experimental.constrained.fcmps.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %1 = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %2 = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
   %3 = zext i1 %1 to i32
   %4 = zext i1 %2 to i32
   %5 = call i32 @bar.i32(i32 %3, i32 %4) #0
@@ -413,5 +413,5 @@ declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
 declare double @llvm.experimental.constrained.uitofp.f64.i32(i32, metadata, metadata)
 declare i32 @llvm.experimental.constrained.fptosi.i32.f64(double, metadata)
 declare double @llvm.experimental.constrained.sitofp.f64.i32(i32, metadata, metadata)
-declare i1 @llvm.experimental.constrained.fcmp.i1.f64(double, double, metadata, metadata)
-declare i1 @llvm.experimental.constrained.fcmps.i1.f64(double, double, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f64(double, double, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f64(double, double, metadata, metadata)
diff --git a/llvm/test/Transforms/EarlyCSE/nonmixed-strictfp.ll b/llvm/test/Transforms/EarlyCSE/nonmixed-strictfp.ll
index 8772f208ebe4..3acf5597dfc3 100644
--- a/llvm/test/Transforms/EarlyCSE/nonmixed-strictfp.ll
+++ b/llvm/test/Transforms/EarlyCSE/nonmixed-strictfp.ll
@@ -313,8 +313,8 @@ define i1 @fcmp_defaultenv(double %a, double %b) #0 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @bar.i32(i32 [[TMP2]], i32 [[TMP2]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
-  %1 = call i1 @llvm.experimental.constrained.fcmp.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
-  %2 = call i1 @llvm.experimental.constrained.fcmp.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %1 = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %2 = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
   %3 = zext i1 %1 to i32
   %4 = zext i1 %2 to i32
   %5 = call i32 @bar.i32(i32 %3, i32 %4) #0
@@ -328,8 +328,8 @@ define i1 @fcmp_maytrap(double %a, double %b) #0 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @bar.i32(i32 [[TMP2]], i32 [[TMP2]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
-  %1 = call i1 @llvm.experimental.constrained.fcmp.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.maytrap") #0
-  %2 = call i1 @llvm.experimental.constrained.fcmp.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.maytrap") #0
+  %1 = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.maytrap") #0
+  %2 = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.maytrap") #0
   %3 = zext i1 %1 to i32
   %4 = zext i1 %2 to i32
   %5 = call i32 @bar.i32(i32 %3, i32 %4) #0
@@ -343,8 +343,8 @@ define i1 @fcmps_defaultenv(double %a, double %b) #0 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @bar.i32(i32 [[TMP2]], i32 [[TMP2]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
-  %1 = call i1 @llvm.experimental.constrained.fcmps.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
-  %2 = call i1 @llvm.experimental.constrained.fcmps.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %1 = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
+  %2 = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.ignore") #0
   %3 = zext i1 %1 to i32
   %4 = zext i1 %2 to i32
   %5 = call i32 @bar.i32(i32 %3, i32 %4) #0
@@ -358,8 +358,8 @@ define i1 @fcmps_maytrap(double %a, double %b) #0 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @bar.i32(i32 [[TMP2]], i32 [[TMP2]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
-  %1 = call i1 @llvm.experimental.constrained.fcmps.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.maytrap") #0
-  %2 = call i1 @llvm.experimental.constrained.fcmps.i1.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.maytrap") #0
+  %1 = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.maytrap") #0
+  %2 = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.maytrap") #0
   %3 = zext i1 %1 to i32
   %4 = zext i1 %2 to i32
   %5 = call i32 @bar.i32(i32 %3, i32 %4) #0
@@ -381,5 +381,5 @@ declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
 declare double @llvm.experimental.constrained.uitofp.f64.i32(i32, metadata, metadata)
 declare i32 @llvm.experimental.constrained.fptosi.i32.f64(double, metadata)
 declare double @llvm.experimental.constrained.sitofp.f64.i32(i32, metadata, metadata)
-declare i1 @llvm.experimental.constrained.fcmp.i1.f64(double, double, metadata, metadata)
-declare i1 @llvm.experimental.constrained.fcmps.i1.f64(double, double, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f64(double, double, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f64(double, double, metadata, metadata)
diff --git a/llvm/test/Transforms/SCCP/strictfp-phis-fcmp.ll b/llvm/test/Transforms/SCCP/strictfp-phis-fcmp.ll
index 3bf7d9578b56..a6c023a25608 100644
--- a/llvm/test/Transforms/SCCP/strictfp-phis-fcmp.ll
+++ b/llvm/test/Transforms/SCCP/strictfp-phis-fcmp.ll
@@ -19,7 +19,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 1.0, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmp.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.ignore") #0
+  %c = call i1 @llvm.experimental.constrained.fcmp.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.ignore") #0
 
   ret i1 %c
 }
@@ -42,7 +42,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 1.0, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmp.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.maytrap") #0
+  %c = call i1 @llvm.experimental.constrained.fcmp.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.maytrap") #0
 
   ret i1 %c
 }
@@ -66,7 +66,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 1.0, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmp.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %c = call i1 @llvm.experimental.constrained.fcmp.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.strict") #0
 
   ret i1 %c
 }
@@ -91,7 +91,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 2.0, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmp.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.ignore") #0
+  %c = call i1 @llvm.experimental.constrained.fcmp.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.ignore") #0
   ret i1 %c
 }
 
@@ -115,7 +115,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 2.0, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmp.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.maytrap") #0
+  %c = call i1 @llvm.experimental.constrained.fcmp.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.maytrap") #0
   ret i1 %c
 }
 
@@ -139,7 +139,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 2.0, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmp.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %c = call i1 @llvm.experimental.constrained.fcmp.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.strict") #0
   ret i1 %c
 }
 
@@ -163,7 +163,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ %f, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmp.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.ignore") #0
+  %c = call i1 @llvm.experimental.constrained.fcmp.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.ignore") #0
   ret i1 %c
 }
 
@@ -187,7 +187,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ %f, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmp.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.maytrap") #0
+  %c = call i1 @llvm.experimental.constrained.fcmp.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.maytrap") #0
   ret i1 %c
 }
 
@@ -211,7 +211,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ %f, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmp.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %c = call i1 @llvm.experimental.constrained.fcmp.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.strict") #0
   ret i1 %c
 }
 
@@ -236,7 +236,7 @@ dead:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 1.0, %if.true], [ %f, %dead ]
-  %c = call i1 @llvm.experimental.constrained.fcmp.i1.f32(float %p, float 1.0, metadata !"une", metadata !"fpexcept.ignore") #0
+  %c = call i1 @llvm.experimental.constrained.fcmp.f32(float %p, float 1.0, metadata !"une", metadata !"fpexcept.ignore") #0
   ret i1 %c
 }
 
@@ -261,7 +261,7 @@ dead:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 1.0, %if.true], [ %f, %dead ]
-  %c = call i1 @llvm.experimental.constrained.fcmp.i1.f32(float %p, float 1.0, metadata !"une", metadata !"fpexcept.maytrap") #0
+  %c = call i1 @llvm.experimental.constrained.fcmp.f32(float %p, float 1.0, metadata !"une", metadata !"fpexcept.maytrap") #0
   ret i1 %c
 }
 
@@ -288,11 +288,11 @@ dead:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 1.0, %if.true], [ %f, %dead ]
-  %c = call i1 @llvm.experimental.constrained.fcmp.i1.f32(float %p, float 1.0, metadata !"une", metadata !"fpexcept.strict") #0
+  %c = call i1 @llvm.experimental.constrained.fcmp.f32(float %p, float 1.0, metadata !"une", metadata !"fpexcept.strict") #0
   ret i1 %c
 }
 
 attributes #0 = { strictfp }
 
-declare i1 @llvm.experimental.constrained.fcmp.i1.f32(float, float, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f32(float, float, metadata, metadata)
 
diff --git a/llvm/test/Transforms/SCCP/strictfp-phis-fcmps.ll b/llvm/test/Transforms/SCCP/strictfp-phis-fcmps.ll
index 6db1f47ccca9..213293a78593 100644
--- a/llvm/test/Transforms/SCCP/strictfp-phis-fcmps.ll
+++ b/llvm/test/Transforms/SCCP/strictfp-phis-fcmps.ll
@@ -19,7 +19,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 1.0, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmps.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.ignore") #0
+  %c = call i1 @llvm.experimental.constrained.fcmps.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.ignore") #0
 
   ret i1 %c
 }
@@ -42,7 +42,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 1.0, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmps.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.maytrap") #0
+  %c = call i1 @llvm.experimental.constrained.fcmps.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.maytrap") #0
 
   ret i1 %c
 }
@@ -66,7 +66,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 1.0, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmps.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %c = call i1 @llvm.experimental.constrained.fcmps.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.strict") #0
 
   ret i1 %c
 }
@@ -91,7 +91,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 2.0, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmps.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.ignore") #0
+  %c = call i1 @llvm.experimental.constrained.fcmps.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.ignore") #0
   ret i1 %c
 }
 
@@ -115,7 +115,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 2.0, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmps.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.maytrap") #0
+  %c = call i1 @llvm.experimental.constrained.fcmps.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.maytrap") #0
   ret i1 %c
 }
 
@@ -139,7 +139,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 2.0, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmps.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %c = call i1 @llvm.experimental.constrained.fcmps.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.strict") #0
   ret i1 %c
 }
 
@@ -163,7 +163,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ %f, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmps.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.ignore") #0
+  %c = call i1 @llvm.experimental.constrained.fcmps.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.ignore") #0
   ret i1 %c
 }
 
@@ -187,7 +187,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ %f, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmps.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.maytrap") #0
+  %c = call i1 @llvm.experimental.constrained.fcmps.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.maytrap") #0
   ret i1 %c
 }
 
@@ -211,7 +211,7 @@ if.true:
 
 end:
   %p = phi float [ 1.0, %entry ], [ %f, %if.true]
-  %c = call i1 @llvm.experimental.constrained.fcmps.i1.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %c = call i1 @llvm.experimental.constrained.fcmps.f32(float %p, float 1.0, metadata !"ueq", metadata !"fpexcept.strict") #0
   ret i1 %c
 }
 
@@ -236,7 +236,7 @@ dead:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 1.0, %if.true], [ %f, %dead ]
-  %c = call i1 @llvm.experimental.constrained.fcmps.i1.f32(float %p, float 1.0, metadata !"une", metadata !"fpexcept.ignore") #0
+  %c = call i1 @llvm.experimental.constrained.fcmps.f32(float %p, float 1.0, metadata !"une", metadata !"fpexcept.ignore") #0
   ret i1 %c
 }
 
@@ -261,7 +261,7 @@ dead:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 1.0, %if.true], [ %f, %dead ]
-  %c = call i1 @llvm.experimental.constrained.fcmps.i1.f32(float %p, float 1.0, metadata !"une", metadata !"fpexcept.maytrap") #0
+  %c = call i1 @llvm.experimental.constrained.fcmps.f32(float %p, float 1.0, metadata !"une", metadata !"fpexcept.maytrap") #0
   ret i1 %c
 }
 
@@ -288,11 +288,11 @@ dead:
 
 end:
   %p = phi float [ 1.0, %entry ], [ 1.0, %if.true], [ %f, %dead ]
-  %c = call i1 @llvm.experimental.constrained.fcmps.i1.f32(float %p, float 1.0, metadata !"une", metadata !"fpexcept.strict") #0
+  %c = call i1 @llvm.experimental.constrained.fcmps.f32(float %p, float 1.0, metadata !"une", metadata !"fpexcept.strict") #0
   ret i1 %c
 }
 
 attributes #0 = { strictfp }
 
-declare i1 @llvm.experimental.constrained.fcmps.i1.f32(float, float, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f32(float, float, metadata, metadata)
 
-- 
GitLab


From 5155c38ceef019f3225c5eaeba6b986e2430752f Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Mon, 28 Oct 2024 15:09:20 +0800
Subject: [PATCH 280/693] [InstCombine] Don't check uses of constant exprs
 (#113684)

This patch skips constant expressions to avoid iterating over uses on
other functions.

Fix crash reported in
https://github.com/llvm/llvm-project/pull/105510#issuecomment-2437521147.
---
 .../InstCombine/InstructionCombining.cpp      |  4 +-
 llvm/test/Transforms/InstCombine/pr105510.ll  | 43 +++++++++++++++++++
 2 files changed, 46 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/InstCombine/pr105510.ll

diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 971ace2a4f47..73a470553178 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -3753,7 +3753,9 @@ Instruction *InstCombinerImpl::visitBranchInst(BranchInst &BI) {
   }
 
   // Replace all dominated uses of the condition with true/false
-  if (BI.getSuccessor(0) != BI.getSuccessor(1)) {
+  // Ignore constant expressions to avoid iterating over uses on other
+  // functions.
+  if (!isa<Constant>(Cond) && BI.getSuccessor(0) != BI.getSuccessor(1)) {
     for (auto &U : make_early_inc_range(Cond->uses())) {
       BasicBlockEdge Edge0(BI.getParent(), BI.getSuccessor(0));
       if (DT.dominates(Edge0, U)) {
diff --git a/llvm/test/Transforms/InstCombine/pr105510.ll b/llvm/test/Transforms/InstCombine/pr105510.ll
new file mode 100644
index 000000000000..844fa14ad991
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr105510.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+; Make sure we don't crash in this case.
+@g = global i32 0
+
+define i1 @foo() {
+; CHECK-LABEL: define i1 @foo() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 ptrtoint (ptr @g to i1), label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    ret i1 true
+; CHECK:       [[IF_ELSE]]:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  br i1 ptrtoint (ptr @g to i1), label %if.then, label %if.else
+
+if.then:
+  ret i1 true
+
+if.else:
+  ret i1 false
+}
+
+define i1 @bar() {
+; CHECK-LABEL: define i1 @bar() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 ptrtoint (ptr @g to i1), label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    ret i1 true
+; CHECK:       [[IF_ELSE]]:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  br i1 ptrtoint (ptr @g to i1), label %if.then, label %if.else
+
+if.then:
+  ret i1 true
+
+if.else:
+  ret i1 false
+}
-- 
GitLab


From fd85761208b4fe0569e3fde0d11297f7dcd0e48e Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Mon, 28 Oct 2024 15:15:49 +0800
Subject: [PATCH 281/693] [X86][BF16] Customize VSELECT for BF16 under
 AVX-NECONVERT (#113322)

Fixes: https://godbolt.org/z/9abGnE8zs
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       |   2 +
 .../CodeGen/X86/avxneconvert-intrinsics.ll    | 114 ++++++++++++++++++
 2 files changed, 116 insertions(+)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9d447959faf5..1c790f3813b7 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2393,6 +2393,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::bf16, Custom);
     for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
       setF16Action(VT, Expand);
+      if (!Subtarget.hasBF16())
+        setOperationAction(ISD::VSELECT, VT, Custom);
       setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
       setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
diff --git a/llvm/test/CodeGen/X86/avxneconvert-intrinsics.ll b/llvm/test/CodeGen/X86/avxneconvert-intrinsics.ll
index b311c8831457..ef87ac31fcf4 100644
--- a/llvm/test/CodeGen/X86/avxneconvert-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avxneconvert-intrinsics.ll
@@ -215,3 +215,117 @@ define <8 x bfloat> @test_int_x86_vcvtneps2bf16256(<8 x float> %A) {
 }
 declare <8 x bfloat> @llvm.x86.vcvtneps2bf16256(<8 x float> %A)
 
+define <8 x bfloat> @select(i8 %x, <8 x bfloat> %y) nounwind {
+; X64-LABEL: select:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovaps %xmm0, %xmm1 # encoding: [0xc5,0xf8,0x28,0xc8]
+; X64-NEXT:    movb %dil, %al # encoding: [0x40,0x88,0xf8]
+; X64-NEXT:    movb %al, -{{[0-9]+}}(%rsp) # encoding: [0x88,0x44,0x24,0xff]
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0xff]
+; X64-NEXT:    movl %eax, %ecx # encoding: [0x89,0xc1]
+; X64-NEXT:    andl $1, %ecx # encoding: [0x83,0xe1,0x01]
+; X64-NEXT:    negl %ecx # encoding: [0xf7,0xd9]
+; X64-NEXT:    vmovd %ecx, %xmm0 # encoding: [0xc5,0xf9,0x6e,0xc1]
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    movb %al, %cl # encoding: [0x88,0xc1]
+; X64-NEXT:    shrb %cl # encoding: [0xd0,0xe9]
+; X64-NEXT:    movzbl %cl, %ecx # encoding: [0x0f,0xb6,0xc9]
+; X64-NEXT:    andl $1, %ecx # encoding: [0x83,0xe1,0x01]
+; X64-NEXT:    negl %ecx # encoding: [0xf7,0xd9]
+; X64-NEXT:    vpinsrw $1, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x01]
+; X64-NEXT:    movb %al, %cl # encoding: [0x88,0xc1]
+; X64-NEXT:    shrb $2, %cl # encoding: [0xc0,0xe9,0x02]
+; X64-NEXT:    movzbl %cl, %ecx # encoding: [0x0f,0xb6,0xc9]
+; X64-NEXT:    andl $1, %ecx # encoding: [0x83,0xe1,0x01]
+; X64-NEXT:    negl %ecx # encoding: [0xf7,0xd9]
+; X64-NEXT:    vpinsrw $2, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x02]
+; X64-NEXT:    movb %al, %cl # encoding: [0x88,0xc1]
+; X64-NEXT:    shrb $3, %cl # encoding: [0xc0,0xe9,0x03]
+; X64-NEXT:    movzbl %cl, %ecx # encoding: [0x0f,0xb6,0xc9]
+; X64-NEXT:    andl $1, %ecx # encoding: [0x83,0xe1,0x01]
+; X64-NEXT:    negl %ecx # encoding: [0xf7,0xd9]
+; X64-NEXT:    vpinsrw $3, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x03]
+; X64-NEXT:    movb %al, %cl # encoding: [0x88,0xc1]
+; X64-NEXT:    shrb $4, %cl # encoding: [0xc0,0xe9,0x04]
+; X64-NEXT:    movzbl %cl, %ecx # encoding: [0x0f,0xb6,0xc9]
+; X64-NEXT:    andl $1, %ecx # encoding: [0x83,0xe1,0x01]
+; X64-NEXT:    negl %ecx # encoding: [0xf7,0xd9]
+; X64-NEXT:    vpinsrw $4, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x04]
+; X64-NEXT:    movb %al, %cl # encoding: [0x88,0xc1]
+; X64-NEXT:    shrb $5, %cl # encoding: [0xc0,0xe9,0x05]
+; X64-NEXT:    movzbl %cl, %ecx # encoding: [0x0f,0xb6,0xc9]
+; X64-NEXT:    andl $1, %ecx # encoding: [0x83,0xe1,0x01]
+; X64-NEXT:    negl %ecx # encoding: [0xf7,0xd9]
+; X64-NEXT:    vpinsrw $5, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x05]
+; X64-NEXT:    movb %al, %cl # encoding: [0x88,0xc1]
+; X64-NEXT:    shrb $6, %cl # encoding: [0xc0,0xe9,0x06]
+; X64-NEXT:    movzbl %cl, %ecx # encoding: [0x0f,0xb6,0xc9]
+; X64-NEXT:    andl $1, %ecx # encoding: [0x83,0xe1,0x01]
+; X64-NEXT:    negl %ecx # encoding: [0xf7,0xd9]
+; X64-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x06]
+; X64-NEXT:    shrb $7, %al # encoding: [0xc0,0xe8,0x07]
+; X64-NEXT:    movzbl %al, %eax # encoding: [0x0f,0xb6,0xc0]
+; X64-NEXT:    negl %eax # encoding: [0xf7,0xd8]
+; X64-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X64-NEXT:    vpandn %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xdf,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+;
+; X86-LABEL: select:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax # encoding: [0x50]
+; X86-NEXT:    vmovaps %xmm0, %xmm1 # encoding: [0xc5,0xf8,0x28,0xc8]
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al # encoding: [0x8a,0x44,0x24,0x08]
+; X86-NEXT:    movb %al, {{[0-9]+}}(%esp) # encoding: [0x88,0x44,0x24,0x03]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x03]
+; X86-NEXT:    movl %eax, %ecx # encoding: [0x89,0xc1]
+; X86-NEXT:    andl $1, %ecx # encoding: [0x83,0xe1,0x01]
+; X86-NEXT:    negl %ecx # encoding: [0xf7,0xd9]
+; X86-NEXT:    vmovd %ecx, %xmm0 # encoding: [0xc5,0xf9,0x6e,0xc1]
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    movb %al, %cl # encoding: [0x88,0xc1]
+; X86-NEXT:    shrb %cl # encoding: [0xd0,0xe9]
+; X86-NEXT:    movzbl %cl, %ecx # encoding: [0x0f,0xb6,0xc9]
+; X86-NEXT:    andl $1, %ecx # encoding: [0x83,0xe1,0x01]
+; X86-NEXT:    negl %ecx # encoding: [0xf7,0xd9]
+; X86-NEXT:    vpinsrw $1, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x01]
+; X86-NEXT:    movb %al, %cl # encoding: [0x88,0xc1]
+; X86-NEXT:    shrb $2, %cl # encoding: [0xc0,0xe9,0x02]
+; X86-NEXT:    movzbl %cl, %ecx # encoding: [0x0f,0xb6,0xc9]
+; X86-NEXT:    andl $1, %ecx # encoding: [0x83,0xe1,0x01]
+; X86-NEXT:    negl %ecx # encoding: [0xf7,0xd9]
+; X86-NEXT:    vpinsrw $2, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x02]
+; X86-NEXT:    movb %al, %cl # encoding: [0x88,0xc1]
+; X86-NEXT:    shrb $3, %cl # encoding: [0xc0,0xe9,0x03]
+; X86-NEXT:    movzbl %cl, %ecx # encoding: [0x0f,0xb6,0xc9]
+; X86-NEXT:    andl $1, %ecx # encoding: [0x83,0xe1,0x01]
+; X86-NEXT:    negl %ecx # encoding: [0xf7,0xd9]
+; X86-NEXT:    vpinsrw $3, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x03]
+; X86-NEXT:    movb %al, %cl # encoding: [0x88,0xc1]
+; X86-NEXT:    shrb $4, %cl # encoding: [0xc0,0xe9,0x04]
+; X86-NEXT:    movzbl %cl, %ecx # encoding: [0x0f,0xb6,0xc9]
+; X86-NEXT:    andl $1, %ecx # encoding: [0x83,0xe1,0x01]
+; X86-NEXT:    negl %ecx # encoding: [0xf7,0xd9]
+; X86-NEXT:    vpinsrw $4, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x04]
+; X86-NEXT:    movb %al, %cl # encoding: [0x88,0xc1]
+; X86-NEXT:    shrb $5, %cl # encoding: [0xc0,0xe9,0x05]
+; X86-NEXT:    movzbl %cl, %ecx # encoding: [0x0f,0xb6,0xc9]
+; X86-NEXT:    andl $1, %ecx # encoding: [0x83,0xe1,0x01]
+; X86-NEXT:    negl %ecx # encoding: [0xf7,0xd9]
+; X86-NEXT:    vpinsrw $5, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x05]
+; X86-NEXT:    movb %al, %cl # encoding: [0x88,0xc1]
+; X86-NEXT:    shrb $6, %cl # encoding: [0xc0,0xe9,0x06]
+; X86-NEXT:    movzbl %cl, %ecx # encoding: [0x0f,0xb6,0xc9]
+; X86-NEXT:    andl $1, %ecx # encoding: [0x83,0xe1,0x01]
+; X86-NEXT:    negl %ecx # encoding: [0xf7,0xd9]
+; X86-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x06]
+; X86-NEXT:    shrb $7, %al # encoding: [0xc0,0xe8,0x07]
+; X86-NEXT:    movzbl %al, %eax # encoding: [0x0f,0xb6,0xc0]
+; X86-NEXT:    negl %eax # encoding: [0xf7,0xd8]
+; X86-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X86-NEXT:    vpandn %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xdf,0xc1]
+; X86-NEXT:    popl %eax # encoding: [0x58]
+; X86-NEXT:    retl # encoding: [0xc3]
+  %1 = bitcast i8 %x to <8 x i1>
+  %2 = select <8 x i1> %1, <8 x bfloat> zeroinitializer, <8 x bfloat> %y
+  ret <8 x bfloat> %2
+}
-- 
GitLab


From e7f422d5c2ea05704eaab2cdd67a8b1ebc55e95b Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Mon, 28 Oct 2024 08:38:56 +0100
Subject: [PATCH 282/693] Add clang::lifetimebound annotation to ArrayRef
 constructors. (#113547)

This enables clang to detect more dangling issues.

```
ArrayRef<int> func() {
   constexpr int array[] = {...}; // oops, missing the static
   return array; // return a dangling reference, bomb.
}
```

See #113533.
---
 llvm/include/llvm/ADT/ArrayRef.h     | 15 +++++++++------
 llvm/include/llvm/Support/Compiler.h |  6 ++++++
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/ADT/ArrayRef.h b/llvm/include/llvm/ADT/ArrayRef.h
index bf6b55923b84..1139fd81cbd0 100644
--- a/llvm/include/llvm/ADT/ArrayRef.h
+++ b/llvm/include/llvm/ADT/ArrayRef.h
@@ -70,15 +70,16 @@ namespace llvm {
     /*implicit*/ ArrayRef(std::nullopt_t) {}
 
     /// Construct an ArrayRef from a single element.
-    /*implicit*/ ArrayRef(const T &OneElt)
-      : Data(&OneElt), Length(1) {}
+    /*implicit*/ ArrayRef(const T &OneElt LLVM_LIFETIME_BOUND)
+        : Data(&OneElt), Length(1) {}
 
     /// Construct an ArrayRef from a pointer and length.
-    constexpr /*implicit*/ ArrayRef(const T *data, size_t length)
+    constexpr /*implicit*/ ArrayRef(const T *data LLVM_LIFETIME_BOUND,
+                                    size_t length)
         : Data(data), Length(length) {}
 
     /// Construct an ArrayRef from a range.
-    constexpr ArrayRef(const T *begin, const T *end)
+    constexpr ArrayRef(const T *begin LLVM_LIFETIME_BOUND, const T *end)
         : Data(begin), Length(end - begin) {
       assert(begin <= end);
     }
@@ -103,7 +104,8 @@ namespace llvm {
 
     /// Construct an ArrayRef from a C array.
     template <size_t N>
-    /*implicit*/ constexpr ArrayRef(const T (&Arr)[N]) : Data(Arr), Length(N) {}
+    /*implicit*/ constexpr ArrayRef(const T (&Arr LLVM_LIFETIME_BOUND)[N])
+        : Data(Arr), Length(N) {}
 
     /// Construct an ArrayRef from a std::initializer_list.
 #if LLVM_GNUC_PREREQ(9, 0, 0)
@@ -113,7 +115,8 @@ namespace llvm {
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Winit-list-lifetime"
 #endif
-    constexpr /*implicit*/ ArrayRef(std::initializer_list<T> Vec)
+    constexpr /*implicit*/ ArrayRef(
+        std::initializer_list<T> Vec LLVM_LIFETIME_BOUND)
         : Data(Vec.begin() == Vec.end() ? (T *)nullptr : Vec.begin()),
           Length(Vec.size()) {}
 #if LLVM_GNUC_PREREQ(9, 0, 0)
diff --git a/llvm/include/llvm/Support/Compiler.h b/llvm/include/llvm/Support/Compiler.h
index 591e7647795b..f9c57b89f1f0 100644
--- a/llvm/include/llvm/Support/Compiler.h
+++ b/llvm/include/llvm/Support/Compiler.h
@@ -413,6 +413,12 @@
 #define LLVM_GSL_POINTER
 #endif
 
+#if LLVM_HAS_CPP_ATTRIBUTE(clang::lifetimebound)
+#define LLVM_LIFETIME_BOUND [[clang::lifetimebound]]
+#else
+#define LLVM_LIFETIME_BOUND
+#endif
+
 #if LLVM_HAS_CPP_ATTRIBUTE(nodiscard) >= 201907L
 #define LLVM_CTOR_NODISCARD [[nodiscard]]
 #else
-- 
GitLab


From 35f6cc6af09f48f9038fce632815a2ad6ffe8689 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Mon, 28 Oct 2024 07:42:33 +0000
Subject: [PATCH 283/693] [RISCV] Add the Sha extension (#113820)

This was introduced in the now-ratified RVA23 profile (and also added to
the RVA22 text) as a simple way of referring to H plus the set of
supervisor extensions required by RVA23.
https://github.com/riscv/riscv-profiles/blob/main/src/rva23-profile.adoc

This patch simply defines the extension. The next patch will adjust the
RVA23 profile to use it, and at that point I think we will be ready to
mark RVA23 as non-experimental.

Note that I haven't made it so if you enable all extensions that
constitute Sha, Sha is implied. Per #76893 (adding 'B'), the concern is
making this implication might break older external assemblers. Perhaps
this is less of a concern given the relative frequency of
`-march=${foo}_zba_zbb_zbs` vs the collection of H extensions. If we did
want to add that implication, we'd probably want to add it in a separate
patch so it can be easily reverted if found to cause problems.
---
 clang/test/Driver/print-supported-extensions-riscv.c | 1 +
 clang/test/Preprocessor/riscv-target-features.c      | 9 +++++++++
 llvm/docs/RISCVUsage.rst                             | 1 +
 llvm/docs/ReleaseNotes.md                            | 1 +
 llvm/lib/Target/RISCV/RISCVFeatures.td               | 7 +++++++
 llvm/test/CodeGen/RISCV/attributes.ll                | 4 ++++
 llvm/unittests/TargetParser/RISCVISAInfoTest.cpp     | 1 +
 7 files changed, 24 insertions(+)

diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index 342d6e921a5a..fc8a9c04667b 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -110,6 +110,7 @@
 // CHECK-NEXT:     zvl8192b             1.0       'Zvl' (Minimum Vector Length) 8192
 // CHECK-NEXT:     zhinx                1.0       'Zhinx' (Half Float in Integer)
 // CHECK-NEXT:     zhinxmin             1.0       'Zhinxmin' (Half Float in Integer Minimal)
+// CHECK-NEXT:     sha                  1.0       'Sha' (Augmented Hypervisor)
 // CHECK-NEXT:     shcounterenw         1.0       'Shcounterenw' (Support writeable hcounteren enable bit for any hpmcounter that is not read-only zero)
 // CHECK-NEXT:     shgatpa              1.0       'Sgatpa' (SvNNx4 mode supported for all modes supported by satp, as well as Bare)
 // CHECK-NEXT:     shtvala              1.0       'Shtvala' (htval provides all needed values)
diff --git a/clang/test/Preprocessor/riscv-target-features.c b/clang/test/Preprocessor/riscv-target-features.c
index 98ad564d2b84..6e586714af84 100644
--- a/clang/test/Preprocessor/riscv-target-features.c
+++ b/clang/test/Preprocessor/riscv-target-features.c
@@ -20,6 +20,7 @@
 // CHECK-NOT: __riscv_m {{.*$}}
 // CHECK-NOT: __riscv_mul {{.*$}}
 // CHECK-NOT: __riscv_muldiv {{.*$}}
+// CHECK-NOT: __riscv_sha {{.*$}}
 // CHECK-NOT: __riscv_shcounterenw {{.*$}}
 // CHECK-NOT: __riscv_shgatpa {{.*$}}
 // CHECK-NOT: __riscv_shtvala {{.*$}}
@@ -323,6 +324,14 @@
 // CHECK-M-EXT: __riscv_mul 1
 // CHECK-M-EXT: __riscv_muldiv 1
 
+// RUN: %clang --target=riscv32-unknown-linux-gnu \
+// RUN:   -march=rv32isha -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-SHCOUNTERENW-EXT %s
+// RUN: %clang --target=riscv64-unknown-linux-gnu \
+// RUN:   -march=rv64isha -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-SHCOUNTERENW-EXT %s
+// CHECK-SHA-EXT: __riscv_sha 1000000{{$}}
+
 // RUN: %clang --target=riscv32-unknown-linux-gnu \
 // RUN:   -march=rv32ishcounterenw -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-SHCOUNTERENW-EXT %s
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index ab58cdaa1b2f..6075a2289d47 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -119,6 +119,7 @@ on support follow.
      ``E``             Supported (`See note <#riscv-rve-note>`__)
      ``H``             Assembly Support
      ``M``             Supported
+     ``Sha``           Supported
      ``Shcounterenw``  Assembly Support (`See note <#riscv-profiles-extensions-note>`__)
      ``Shgatpa``       Assembly Support (`See note <#riscv-profiles-extensions-note>`__)
      ``Shtvala``       Assembly Support (`See note <#riscv-profiles-extensions-note>`__)
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 2580f09be3ad..f7215279940d 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -183,6 +183,7 @@ Changes to the RISC-V Backend
 * The `Zacas` extension is no longer marked as experimental.
 * The `Smmpm`, `Smnpm`, `Ssnpm`, `Supm`, and `Sspm` pointer masking extensions
   are no longer marked as experimental.
+* The `Sha` extension is now supported.
 
 Changes to the WebAssembly Backend
 ----------------------------------
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 778df542022f..559f0e5950ed 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -1029,6 +1029,13 @@ def FeatureStdExtSvpbmt
     : RISCVExtension<"svpbmt", 1, 0,
                      "'Svpbmt' (Page-Based Memory Types)">;
 
+def FeatureStdExtSha
+    : RISCVExtension<"sha", 1, 0,
+                     "'Sha' (Augmented Hypervisor)",
+                     [FeatureStdExtH, FeatureStdExtSsstateen, FeatureStdExtShcounterenw,
+                      FeatureStdExtShvstvala, FeatureStdExtShtvala, FeatureStdExtShvstvecd,
+                      FeatureStdExtShvsatpa, FeatureStdExtShgatpa]>;
+
 // Pointer Masking extensions
 
 // A supervisor-level extension that provides pointer masking for the next lower
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index 9be9ddd05ee2..4cc9ef2ea0d7 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -39,6 +39,7 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+zicbom %s -o - | FileCheck --check-prefixes=CHECK,RV32ZICBOM %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zicboz %s -o - | FileCheck --check-prefixes=CHECK,RV32ZICBOZ %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zicbop %s -o - | FileCheck --check-prefixes=CHECK,RV32ZICBOP %s
+; RUN: llc -mtriple=riscv32 -mattr=+sha %s -o - | FileCheck --check-prefixes=CHECK,RV32SHA %s
 ; RUN: llc -mtriple=riscv32 -mattr=+shcounterenw %s -o - | FileCheck --check-prefixes=CHECK,RV32SHCOUNTERENW %s
 ; RUN: llc -mtriple=riscv32 -mattr=+shgatpa %s -o - | FileCheck --check-prefixes=CHECK,RV32SHGATPA %s
 ; RUN: llc -mtriple=riscv32 -mattr=+shvsatpa %s -o - | FileCheck --check-prefixes=CHECK,RV32SHVSATPA %s
@@ -178,6 +179,7 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+zicbom %s -o - | FileCheck --check-prefixes=CHECK,RV64ZICBOM %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zicboz %s -o - | FileCheck --check-prefixes=CHECK,RV64ZICBOZ %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zicbop %s -o - | FileCheck --check-prefixes=CHECK,RV64ZICBOP %s
+; RUN: llc -mtriple=riscv64 -mattr=+sha %s -o - | FileCheck --check-prefixes=CHECK,RV64SHA %s
 ; RUN: llc -mtriple=riscv64 -mattr=+shcounterenw %s -o - | FileCheck --check-prefixes=CHECK,RV64SHCOUNTERENW %s
 ; RUN: llc -mtriple=riscv64 -mattr=+shgatpa %s -o - | FileCheck --check-prefixes=CHECK,RV64SHGATPA %s
 ; RUN: llc -mtriple=riscv64 -mattr=+shvsatpa %s -o - | FileCheck --check-prefixes=CHECK,RV64SHVSATPA %s
@@ -333,6 +335,7 @@
 ; RV32ZICBOM: .attribute 5, "rv32i2p1_zicbom1p0"
 ; RV32ZICBOZ: .attribute 5, "rv32i2p1_zicboz1p0"
 ; RV32ZICBOP: .attribute 5, "rv32i2p1_zicbop1p0"
+; RV32SHA: .attribute 5, "rv32i2p1_h1p0_sha1p0_shcounterenw1p0_shgatpa1p0_shtvala1p0_shvsatpa1p0_shvstvala1p0_shvstvecd1p0_ssstateen1p0"
 ; RV32SHCOUNTERENW: .attribute 5, "rv32i2p1_shcounterenw1p0"
 ; RV32SHGATPA: .attribute 5, "rv32i2p1_shgatpa1p0"
 ; RV32SHVSATPA: .attribute 5, "rv32i2p1_shvsatpa1p0"
@@ -474,6 +477,7 @@
 ; RV64ZAMA16B: .attribute 5, "rv64i2p1_zama16b1p0"
 ; RV64ZAWRS: .attribute 5, "rv64i2p1_zawrs1p0"
 ; RV64ZICBOP: .attribute 5, "rv64i2p1_zicbop1p0"
+; RV64SHA: .attribute 5, "rv64i2p1_h1p0_sha1p0_shcounterenw1p0_shgatpa1p0_shtvala1p0_shvsatpa1p0_shvstvala1p0_shvstvecd1p0_ssstateen1p0"
 ; RV64SHCOUNTERENW: .attribute 5, "rv64i2p1_shcounterenw1p0"
 ; RV64SHGATPA: .attribute 5, "rv64i2p1_shgatpa1p0"
 ; RV64SHVSATPA: .attribute 5, "rv64i2p1_shvsatpa1p0"
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 2e9c548ae872..e402b1a40de3 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -1055,6 +1055,7 @@ R"(All available -march extensions for RISC-V
     zvl8192b             1.0
     zhinx                1.0
     zhinxmin             1.0
+    sha                  1.0
     shcounterenw         1.0
     shgatpa              1.0
     shtvala              1.0
-- 
GitLab


From a4fd3dba6e285734bc635b0651a30dfeffedeada Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter@amd.com>
Date: Mon, 28 Oct 2024 09:04:19 +0100
Subject: [PATCH 284/693] [AMDGPU] Use wider loop lowering type for
 LowerMemIntrinsics (#112332)

When llvm.memcpy or llvm.memmove intrinsics are lowered as a loop in
LowerMemIntrinsics.cpp, the loop consists of a single load/store pair
per iteration. We can improve performance in some cases by emitting
multiple load/store pairs per iteration. This patch achieves that by
increasing the width of the loop lowering type in the GCN target and
letting legalization split the resulting too-wide access pairs into
multiple legal access pairs.

This change only affects lowered memcpys and memmoves with large (>=
1024 bytes) constant lengths. Smaller constant lengths are handled by
ISel directly; non-constant lengths would be slowed down by this change
if the dynamic length was smaller or slightly larger than what an
unrolled iteration copies.

The chosen default unroll factor is the result of microbenchmarks on
gfx1030. This change leads to speedups of 15-38% for global memory and
1.9-5.8x for scratch in these microbenchmarks.

Part of SWDEV-455845.
---
 .../AMDGPU/AMDGPUTargetTransformInfo.cpp      |    38 +-
 .../CodeGen/AMDGPU/GlobalISel/llvm.memcpy.ll  |   308 +-
 .../CodeGen/AMDGPU/lower-mem-intrinsics.ll    |   284 +-
 .../CodeGen/AMDGPU/memintrinsic-unroll.ll     | 16049 ++++++++++++++++
 4 files changed, 16445 insertions(+), 234 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 8f9495d83cde..5160851f8c44 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -75,6 +75,13 @@ static cl::opt<size_t> InlineMaxBB(
     cl::desc("Maximum number of BBs allowed in a function after inlining"
              " (compile time constraint)"));
 
+// This default unroll factor is based on microbenchmarks on gfx1030.
+static cl::opt<unsigned> MemcpyLoopUnroll(
+    "amdgpu-memcpy-loop-unroll",
+    cl::desc("Unroll factor (affecting 4x32-bit operations) to use for memory "
+             "operations when lowering memcpy as a loop"),
+    cl::init(16), cl::Hidden);
+
 static bool dependsOnLocalPhi(const Loop *L, const Value *Cond,
                               unsigned Depth = 0) {
   const Instruction *I = dyn_cast<Instruction>(Cond);
@@ -409,13 +416,8 @@ int64_t GCNTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
   return 1024;
 }
 
-// FIXME: Really we would like to issue multiple 128-bit loads and stores per
-// iteration. Should we report a larger size and let it legalize?
-//
 // FIXME: Should we use narrower types for local/region, or account for when
 // unaligned access is legal?
-//
-// FIXME: This could use fine tuning and microbenchmarks.
 Type *GCNTTIImpl::getMemcpyLoopLoweringType(
     LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
     unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
@@ -442,9 +444,22 @@ Type *GCNTTIImpl::getMemcpyLoopLoweringType(
     return FixedVectorType::get(Type::getInt32Ty(Context), 2);
   }
 
-  // Global memory works best with 16-byte accesses. Private memory will also
-  // hit this, although they'll be decomposed.
-  return FixedVectorType::get(Type::getInt32Ty(Context), 4);
+  // Global memory works best with 16-byte accesses.
+  // If the operation has a fixed known length that is large enough, it is
+  // worthwhile to return an even wider type and let legalization lower it into
+  // multiple accesses, effectively unrolling the memcpy loop. Private memory
+  // also hits this, although accesses may be decomposed.
+  //
+  // Don't unroll if Length is not a constant, since unrolling leads to worse
+  // performance for length values that are smaller or slightly larger than the
+  // total size of the type returned here. Mitigating that would require a more
+  // complex lowering for variable-length memcpy and memmove.
+  unsigned I32EltsInVector = 4;
+  if (MemcpyLoopUnroll > 0 && isa<ConstantInt>(Length))
+    return FixedVectorType::get(Type::getInt32Ty(Context),
+                                MemcpyLoopUnroll * I32EltsInVector);
+
+  return FixedVectorType::get(Type::getInt32Ty(Context), I32EltsInVector);
 }
 
 void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
@@ -452,7 +467,6 @@ void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
     unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
     Align SrcAlign, Align DestAlign,
     std::optional<uint32_t> AtomicCpySize) const {
-  assert(RemainingBytes < 16);
 
   if (AtomicCpySize)
     BaseT::getMemcpyLoopResidualLoweringType(
@@ -462,6 +476,12 @@ void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
   Align MinAlign = std::min(SrcAlign, DestAlign);
 
   if (MinAlign != Align(2)) {
+    Type *I32x4Ty = FixedVectorType::get(Type::getInt32Ty(Context), 4);
+    while (RemainingBytes >= 16) {
+      OpsOut.push_back(I32x4Ty);
+      RemainingBytes -= 16;
+    }
+
     Type *I64Ty = Type::getInt64Ty(Context);
     while (RemainingBytes >= 8) {
       OpsOut.push_back(I64Ty);
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memcpy.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memcpy.ll
index 7f23434c9dfd..75d4d8816fb3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memcpy.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memcpy.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-- -verify-machineinstrs -mem-intrinsic-expand-size=19 %s -o - | FileCheck -check-prefix=LOOP %s
-; RUN: llc -global-isel -mtriple=amdgcn-- -verify-machineinstrs -mem-intrinsic-expand-size=21 %s -o - | FileCheck -check-prefix=UNROLL %s
+; RUN: llc -global-isel -mtriple=amdgcn-- -verify-machineinstrs -amdgpu-memcpy-loop-unroll=2 -mem-intrinsic-expand-size=35 %s -o - | FileCheck -check-prefix=LOOP %s
+; RUN: llc -global-isel -mtriple=amdgcn-- -verify-machineinstrs -amdgpu-memcpy-loop-unroll=2 -mem-intrinsic-expand-size=37 %s -o - | FileCheck -check-prefix=UNROLL %s
 
 declare void @llvm.memcpy.p1.p1.i32(ptr addrspace(1), ptr addrspace(1), i32, i1 immarg)
 
@@ -14,104 +14,176 @@ define amdgpu_cs void @memcpy_p1i8(ptr addrspace(1) %dst, ptr addrspace(1) %src)
 ; LOOP-NEXT:    v_mov_b32_e32 v4, s0
 ; LOOP-NEXT:  .LBB0_1: ; %load-store-loop
 ; LOOP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; LOOP-NEXT:    s_waitcnt expcnt(2)
+; LOOP-NEXT:    v_add_i32_e32 v29, vcc, v2, v4
+; LOOP-NEXT:    v_addc_u32_e32 v30, vcc, v3, v5, vcc
+; LOOP-NEXT:    buffer_load_ubyte v24, v[29:30], s[0:3], 0 addr64
+; LOOP-NEXT:    buffer_load_ubyte v27, v[29:30], s[0:3], 0 addr64 offset:1
+; LOOP-NEXT:    buffer_load_ubyte v34, v[29:30], s[0:3], 0 addr64 offset:2
+; LOOP-NEXT:    buffer_load_ubyte v35, v[29:30], s[0:3], 0 addr64 offset:3
+; LOOP-NEXT:    buffer_load_ubyte v36, v[29:30], s[0:3], 0 addr64 offset:4
+; LOOP-NEXT:    buffer_load_ubyte v37, v[29:30], s[0:3], 0 addr64 offset:5
+; LOOP-NEXT:    buffer_load_ubyte v38, v[29:30], s[0:3], 0 addr64 offset:6
+; LOOP-NEXT:    buffer_load_ubyte v39, v[29:30], s[0:3], 0 addr64 offset:7
+; LOOP-NEXT:    buffer_load_ubyte v6, v[29:30], s[0:3], 0 addr64 offset:8
+; LOOP-NEXT:    buffer_load_ubyte v9, v[29:30], s[0:3], 0 addr64 offset:9
+; LOOP-NEXT:    buffer_load_ubyte v10, v[29:30], s[0:3], 0 addr64 offset:10
 ; LOOP-NEXT:    s_waitcnt expcnt(0)
-; LOOP-NEXT:    v_add_i32_e32 v6, vcc, v2, v4
-; LOOP-NEXT:    v_addc_u32_e32 v7, vcc, v3, v5, vcc
-; LOOP-NEXT:    v_add_i32_e32 v8, vcc, v0, v4
-; LOOP-NEXT:    v_addc_u32_e32 v9, vcc, v1, v5, vcc
-; LOOP-NEXT:    v_add_i32_e32 v4, vcc, 16, v4
-; LOOP-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; LOOP-NEXT:    buffer_load_ubyte v10, v[6:7], s[0:3], 0 addr64
-; LOOP-NEXT:    buffer_load_ubyte v11, v[6:7], s[0:3], 0 addr64 offset:1
-; LOOP-NEXT:    buffer_load_ubyte v12, v[6:7], s[0:3], 0 addr64 offset:2
-; LOOP-NEXT:    buffer_load_ubyte v13, v[6:7], s[0:3], 0 addr64 offset:3
-; LOOP-NEXT:    buffer_load_ubyte v14, v[6:7], s[0:3], 0 addr64 offset:4
-; LOOP-NEXT:    buffer_load_ubyte v15, v[6:7], s[0:3], 0 addr64 offset:5
-; LOOP-NEXT:    buffer_load_ubyte v16, v[6:7], s[0:3], 0 addr64 offset:6
-; LOOP-NEXT:    buffer_load_ubyte v17, v[6:7], s[0:3], 0 addr64 offset:7
-; LOOP-NEXT:    buffer_load_ubyte v18, v[6:7], s[0:3], 0 addr64 offset:8
-; LOOP-NEXT:    buffer_load_ubyte v19, v[6:7], s[0:3], 0 addr64 offset:9
-; LOOP-NEXT:    buffer_load_ubyte v20, v[6:7], s[0:3], 0 addr64 offset:10
-; LOOP-NEXT:    buffer_load_ubyte v21, v[6:7], s[0:3], 0 addr64 offset:11
-; LOOP-NEXT:    buffer_load_ubyte v22, v[6:7], s[0:3], 0 addr64 offset:12
-; LOOP-NEXT:    buffer_load_ubyte v23, v[6:7], s[0:3], 0 addr64 offset:13
-; LOOP-NEXT:    buffer_load_ubyte v24, v[6:7], s[0:3], 0 addr64 offset:14
-; LOOP-NEXT:    buffer_load_ubyte v6, v[6:7], s[0:3], 0 addr64 offset:15
-; LOOP-NEXT:    v_cmp_gt_u32_e32 vcc, 16, v4
+; LOOP-NEXT:    buffer_load_ubyte v11, v[29:30], s[0:3], 0 addr64 offset:11
+; LOOP-NEXT:    buffer_load_ubyte v7, v[29:30], s[0:3], 0 addr64 offset:12
+; LOOP-NEXT:    buffer_load_ubyte v13, v[29:30], s[0:3], 0 addr64 offset:13
+; LOOP-NEXT:    buffer_load_ubyte v14, v[29:30], s[0:3], 0 addr64 offset:14
+; LOOP-NEXT:    buffer_load_ubyte v15, v[29:30], s[0:3], 0 addr64 offset:15
+; LOOP-NEXT:    buffer_load_ubyte v8, v[29:30], s[0:3], 0 addr64 offset:16
+; LOOP-NEXT:    buffer_load_ubyte v17, v[29:30], s[0:3], 0 addr64 offset:17
+; LOOP-NEXT:    buffer_load_ubyte v18, v[29:30], s[0:3], 0 addr64 offset:18
+; LOOP-NEXT:    buffer_load_ubyte v19, v[29:30], s[0:3], 0 addr64 offset:19
+; LOOP-NEXT:    buffer_load_ubyte v12, v[29:30], s[0:3], 0 addr64 offset:20
+; LOOP-NEXT:    buffer_load_ubyte v21, v[29:30], s[0:3], 0 addr64 offset:21
+; LOOP-NEXT:    buffer_load_ubyte v22, v[29:30], s[0:3], 0 addr64 offset:22
+; LOOP-NEXT:    buffer_load_ubyte v23, v[29:30], s[0:3], 0 addr64 offset:23
+; LOOP-NEXT:    buffer_load_ubyte v16, v[29:30], s[0:3], 0 addr64 offset:24
+; LOOP-NEXT:    buffer_load_ubyte v25, v[29:30], s[0:3], 0 addr64 offset:25
+; LOOP-NEXT:    buffer_load_ubyte v26, v[29:30], s[0:3], 0 addr64 offset:26
+; LOOP-NEXT:    buffer_load_ubyte v28, v[29:30], s[0:3], 0 addr64 offset:27
+; LOOP-NEXT:    buffer_load_ubyte v20, v[29:30], s[0:3], 0 addr64 offset:28
+; LOOP-NEXT:    buffer_load_ubyte v31, v[29:30], s[0:3], 0 addr64 offset:29
+; LOOP-NEXT:    buffer_load_ubyte v32, v[29:30], s[0:3], 0 addr64 offset:30
+; LOOP-NEXT:    buffer_load_ubyte v33, v[29:30], s[0:3], 0 addr64 offset:31
 ; LOOP-NEXT:    s_waitcnt vmcnt(14)
-; LOOP-NEXT:    v_lshlrev_b32_e32 v7, 8, v11
+; LOOP-NEXT:    v_lshlrev_b32_e32 v27, 8, v27
+; LOOP-NEXT:    v_or_b32_e32 v24, v27, v24
+; LOOP-NEXT:    v_lshlrev_b32_e32 v27, 24, v35
+; LOOP-NEXT:    v_lshlrev_b32_e32 v29, 16, v34
+; LOOP-NEXT:    v_or_b32_e32 v27, v27, v29
+; LOOP-NEXT:    v_lshlrev_b32_e32 v29, 8, v37
+; LOOP-NEXT:    v_lshlrev_b32_e32 v30, 24, v39
+; LOOP-NEXT:    v_lshlrev_b32_e32 v34, 16, v38
+; LOOP-NEXT:    v_or_b32_e32 v29, v29, v36
+; LOOP-NEXT:    v_or_b32_e32 v30, v30, v34
+; LOOP-NEXT:    v_add_i32_e32 v34, vcc, v0, v4
+; LOOP-NEXT:    v_addc_u32_e32 v35, vcc, v1, v5, vcc
+; LOOP-NEXT:    v_add_i32_e32 v4, vcc, 32, v4
+; LOOP-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; LOOP-NEXT:    v_cmp_gt_u32_e32 vcc, 32, v4
+; LOOP-NEXT:    v_lshlrev_b32_e32 v9, 8, v9
+; LOOP-NEXT:    v_lshlrev_b32_e32 v11, 24, v11
+; LOOP-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; LOOP-NEXT:    v_lshlrev_b32_e32 v13, 8, v13
+; LOOP-NEXT:    v_lshlrev_b32_e32 v15, 24, v15
+; LOOP-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; LOOP-NEXT:    v_lshlrev_b32_e32 v17, 8, v17
 ; LOOP-NEXT:    s_waitcnt vmcnt(12)
-; LOOP-NEXT:    v_lshlrev_b32_e32 v11, 24, v13
-; LOOP-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; LOOP-NEXT:    v_lshlrev_b32_e32 v19, 24, v19
+; LOOP-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; LOOP-NEXT:    s_waitcnt vmcnt(10)
-; LOOP-NEXT:    v_lshlrev_b32_e32 v13, 8, v15
+; LOOP-NEXT:    v_lshlrev_b32_e32 v21, 8, v21
 ; LOOP-NEXT:    s_waitcnt vmcnt(8)
-; LOOP-NEXT:    v_lshlrev_b32_e32 v15, 24, v17
-; LOOP-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; LOOP-NEXT:    v_lshlrev_b32_e32 v23, 24, v23
+; LOOP-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
 ; LOOP-NEXT:    s_waitcnt vmcnt(6)
-; LOOP-NEXT:    v_lshlrev_b32_e32 v17, 8, v19
+; LOOP-NEXT:    v_lshlrev_b32_e32 v25, 8, v25
 ; LOOP-NEXT:    s_waitcnt vmcnt(4)
-; LOOP-NEXT:    v_lshlrev_b32_e32 v19, 24, v21
-; LOOP-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; LOOP-NEXT:    v_lshlrev_b32_e32 v28, 24, v28
+; LOOP-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
 ; LOOP-NEXT:    s_waitcnt vmcnt(2)
-; LOOP-NEXT:    v_lshlrev_b32_e32 v21, 8, v23
+; LOOP-NEXT:    v_lshlrev_b32_e32 v31, 8, v31
 ; LOOP-NEXT:    s_waitcnt vmcnt(0)
-; LOOP-NEXT:    v_lshlrev_b32_e32 v6, 24, v6
-; LOOP-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
-; LOOP-NEXT:    v_or_b32_e32 v7, v7, v10
-; LOOP-NEXT:    v_or_b32_e32 v10, v11, v12
-; LOOP-NEXT:    v_or_b32_e32 v11, v13, v14
-; LOOP-NEXT:    v_or_b32_e32 v12, v15, v16
-; LOOP-NEXT:    v_or_b32_e32 v13, v17, v18
-; LOOP-NEXT:    v_or_b32_e32 v14, v19, v20
-; LOOP-NEXT:    v_or_b32_e32 v15, v21, v22
-; LOOP-NEXT:    v_or_b32_e32 v6, v6, v23
+; LOOP-NEXT:    v_lshlrev_b32_e32 v33, 24, v33
+; LOOP-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; LOOP-NEXT:    v_or_b32_e32 v6, v9, v6
+; LOOP-NEXT:    v_or_b32_e32 v9, v11, v10
+; LOOP-NEXT:    v_or_b32_e32 v7, v13, v7
+; LOOP-NEXT:    v_or_b32_e32 v10, v15, v14
+; LOOP-NEXT:    v_or_b32_e32 v8, v17, v8
+; LOOP-NEXT:    v_or_b32_e32 v11, v19, v18
+; LOOP-NEXT:    v_or_b32_e32 v12, v21, v12
+; LOOP-NEXT:    v_or_b32_e32 v13, v23, v22
+; LOOP-NEXT:    v_or_b32_e32 v14, v25, v16
+; LOOP-NEXT:    v_or_b32_e32 v15, v28, v26
+; LOOP-NEXT:    v_or_b32_e32 v16, v31, v20
+; LOOP-NEXT:    v_or_b32_e32 v17, v33, v32
+; LOOP-NEXT:    v_or_b32_e32 v18, v27, v24
+; LOOP-NEXT:    v_or_b32_e32 v19, v30, v29
+; LOOP-NEXT:    v_or_b32_e32 v6, v9, v6
 ; LOOP-NEXT:    v_or_b32_e32 v7, v10, v7
-; LOOP-NEXT:    v_or_b32_e32 v10, v12, v11
-; LOOP-NEXT:    v_or_b32_e32 v11, v14, v13
-; LOOP-NEXT:    v_or_b32_e32 v6, v6, v15
-; LOOP-NEXT:    v_lshrrev_b32_e32 v12, 16, v7
-; LOOP-NEXT:    v_bfe_u32 v13, v7, 8, 8
-; LOOP-NEXT:    buffer_store_byte v7, v[8:9], s[0:3], 0 addr64
+; LOOP-NEXT:    v_or_b32_e32 v8, v11, v8
+; LOOP-NEXT:    v_or_b32_e32 v9, v13, v12
+; LOOP-NEXT:    v_or_b32_e32 v10, v15, v14
+; LOOP-NEXT:    v_or_b32_e32 v11, v17, v16
+; LOOP-NEXT:    v_lshrrev_b32_e32 v12, 16, v18
+; LOOP-NEXT:    v_bfe_u32 v13, v18, 8, 8
+; LOOP-NEXT:    buffer_store_byte v18, v[34:35], s[0:3], 0 addr64
+; LOOP-NEXT:    v_lshrrev_b32_e32 v14, 24, v18
+; LOOP-NEXT:    v_lshrrev_b32_e32 v15, 16, v19
+; LOOP-NEXT:    v_bfe_u32 v16, v19, 8, 8
+; LOOP-NEXT:    buffer_store_byte v19, v[34:35], s[0:3], 0 addr64 offset:4
+; LOOP-NEXT:    v_lshrrev_b32_e32 v17, 24, v19
+; LOOP-NEXT:    s_waitcnt expcnt(1)
+; LOOP-NEXT:    v_lshrrev_b32_e32 v18, 16, v6
+; LOOP-NEXT:    s_waitcnt expcnt(0)
+; LOOP-NEXT:    v_bfe_u32 v19, v6, 8, 8
+; LOOP-NEXT:    buffer_store_byte v6, v[34:35], s[0:3], 0 addr64 offset:8
+; LOOP-NEXT:    s_waitcnt expcnt(0)
+; LOOP-NEXT:    v_lshrrev_b32_e32 v6, 24, v6
+; LOOP-NEXT:    v_lshrrev_b32_e32 v20, 16, v7
+; LOOP-NEXT:    v_bfe_u32 v21, v7, 8, 8
+; LOOP-NEXT:    buffer_store_byte v7, v[34:35], s[0:3], 0 addr64 offset:12
 ; LOOP-NEXT:    s_waitcnt expcnt(0)
 ; LOOP-NEXT:    v_lshrrev_b32_e32 v7, 24, v7
-; LOOP-NEXT:    v_lshrrev_b32_e32 v14, 16, v10
-; LOOP-NEXT:    v_bfe_u32 v15, v10, 8, 8
-; LOOP-NEXT:    buffer_store_byte v10, v[8:9], s[0:3], 0 addr64 offset:4
+; LOOP-NEXT:    v_lshrrev_b32_e32 v22, 16, v8
+; LOOP-NEXT:    v_bfe_u32 v23, v8, 8, 8
+; LOOP-NEXT:    buffer_store_byte v8, v[34:35], s[0:3], 0 addr64 offset:16
+; LOOP-NEXT:    s_waitcnt expcnt(0)
+; LOOP-NEXT:    v_lshrrev_b32_e32 v8, 24, v8
+; LOOP-NEXT:    v_lshrrev_b32_e32 v24, 16, v9
+; LOOP-NEXT:    v_bfe_u32 v25, v9, 8, 8
+; LOOP-NEXT:    buffer_store_byte v9, v[34:35], s[0:3], 0 addr64 offset:20
+; LOOP-NEXT:    s_waitcnt expcnt(0)
+; LOOP-NEXT:    v_lshrrev_b32_e32 v9, 24, v9
+; LOOP-NEXT:    v_lshrrev_b32_e32 v26, 16, v10
+; LOOP-NEXT:    v_bfe_u32 v27, v10, 8, 8
+; LOOP-NEXT:    buffer_store_byte v10, v[34:35], s[0:3], 0 addr64 offset:24
 ; LOOP-NEXT:    s_waitcnt expcnt(0)
 ; LOOP-NEXT:    v_lshrrev_b32_e32 v10, 24, v10
-; LOOP-NEXT:    v_lshrrev_b32_e32 v16, 16, v11
-; LOOP-NEXT:    v_bfe_u32 v17, v11, 8, 8
-; LOOP-NEXT:    buffer_store_byte v11, v[8:9], s[0:3], 0 addr64 offset:8
+; LOOP-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; LOOP-NEXT:    v_bfe_u32 v29, v11, 8, 8
+; LOOP-NEXT:    buffer_store_byte v11, v[34:35], s[0:3], 0 addr64 offset:28
 ; LOOP-NEXT:    s_waitcnt expcnt(0)
 ; LOOP-NEXT:    v_lshrrev_b32_e32 v11, 24, v11
-; LOOP-NEXT:    v_lshrrev_b32_e32 v18, 16, v6
-; LOOP-NEXT:    v_bfe_u32 v19, v6, 8, 8
-; LOOP-NEXT:    buffer_store_byte v6, v[8:9], s[0:3], 0 addr64 offset:12
-; LOOP-NEXT:    s_waitcnt expcnt(0)
-; LOOP-NEXT:    v_lshrrev_b32_e32 v6, 24, v6
-; LOOP-NEXT:    buffer_store_byte v13, v[8:9], s[0:3], 0 addr64 offset:1
-; LOOP-NEXT:    buffer_store_byte v12, v[8:9], s[0:3], 0 addr64 offset:2
-; LOOP-NEXT:    buffer_store_byte v7, v[8:9], s[0:3], 0 addr64 offset:3
-; LOOP-NEXT:    buffer_store_byte v15, v[8:9], s[0:3], 0 addr64 offset:5
-; LOOP-NEXT:    buffer_store_byte v14, v[8:9], s[0:3], 0 addr64 offset:6
-; LOOP-NEXT:    buffer_store_byte v10, v[8:9], s[0:3], 0 addr64 offset:7
-; LOOP-NEXT:    buffer_store_byte v17, v[8:9], s[0:3], 0 addr64 offset:9
-; LOOP-NEXT:    buffer_store_byte v16, v[8:9], s[0:3], 0 addr64 offset:10
-; LOOP-NEXT:    buffer_store_byte v11, v[8:9], s[0:3], 0 addr64 offset:11
-; LOOP-NEXT:    buffer_store_byte v19, v[8:9], s[0:3], 0 addr64 offset:13
-; LOOP-NEXT:    buffer_store_byte v18, v[8:9], s[0:3], 0 addr64 offset:14
-; LOOP-NEXT:    buffer_store_byte v6, v[8:9], s[0:3], 0 addr64 offset:15
+; LOOP-NEXT:    buffer_store_byte v13, v[34:35], s[0:3], 0 addr64 offset:1
+; LOOP-NEXT:    buffer_store_byte v12, v[34:35], s[0:3], 0 addr64 offset:2
+; LOOP-NEXT:    buffer_store_byte v14, v[34:35], s[0:3], 0 addr64 offset:3
+; LOOP-NEXT:    buffer_store_byte v16, v[34:35], s[0:3], 0 addr64 offset:5
+; LOOP-NEXT:    buffer_store_byte v15, v[34:35], s[0:3], 0 addr64 offset:6
+; LOOP-NEXT:    buffer_store_byte v17, v[34:35], s[0:3], 0 addr64 offset:7
+; LOOP-NEXT:    buffer_store_byte v19, v[34:35], s[0:3], 0 addr64 offset:9
+; LOOP-NEXT:    buffer_store_byte v18, v[34:35], s[0:3], 0 addr64 offset:10
+; LOOP-NEXT:    buffer_store_byte v6, v[34:35], s[0:3], 0 addr64 offset:11
+; LOOP-NEXT:    buffer_store_byte v21, v[34:35], s[0:3], 0 addr64 offset:13
+; LOOP-NEXT:    buffer_store_byte v20, v[34:35], s[0:3], 0 addr64 offset:14
+; LOOP-NEXT:    buffer_store_byte v7, v[34:35], s[0:3], 0 addr64 offset:15
+; LOOP-NEXT:    buffer_store_byte v23, v[34:35], s[0:3], 0 addr64 offset:17
+; LOOP-NEXT:    buffer_store_byte v22, v[34:35], s[0:3], 0 addr64 offset:18
+; LOOP-NEXT:    buffer_store_byte v8, v[34:35], s[0:3], 0 addr64 offset:19
+; LOOP-NEXT:    buffer_store_byte v25, v[34:35], s[0:3], 0 addr64 offset:21
+; LOOP-NEXT:    buffer_store_byte v24, v[34:35], s[0:3], 0 addr64 offset:22
+; LOOP-NEXT:    buffer_store_byte v9, v[34:35], s[0:3], 0 addr64 offset:23
+; LOOP-NEXT:    buffer_store_byte v27, v[34:35], s[0:3], 0 addr64 offset:25
+; LOOP-NEXT:    buffer_store_byte v26, v[34:35], s[0:3], 0 addr64 offset:26
+; LOOP-NEXT:    buffer_store_byte v10, v[34:35], s[0:3], 0 addr64 offset:27
+; LOOP-NEXT:    buffer_store_byte v29, v[34:35], s[0:3], 0 addr64 offset:29
+; LOOP-NEXT:    buffer_store_byte v28, v[34:35], s[0:3], 0 addr64 offset:30
+; LOOP-NEXT:    buffer_store_byte v11, v[34:35], s[0:3], 0 addr64 offset:31
 ; LOOP-NEXT:    s_cbranch_vccnz .LBB0_1
 ; LOOP-NEXT:  ; %bb.2: ; %memcpy-split
 ; LOOP-NEXT:    s_mov_b32 s2, 0
 ; LOOP-NEXT:    s_mov_b32 s3, 0xf000
 ; LOOP-NEXT:    s_mov_b64 s[0:1], 0
-; LOOP-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:17
-; LOOP-NEXT:    buffer_load_ubyte v5, v[2:3], s[0:3], 0 addr64 offset:19
-; LOOP-NEXT:    s_waitcnt expcnt(0)
-; LOOP-NEXT:    buffer_load_ubyte v6, v[2:3], s[0:3], 0 addr64 offset:18
-; LOOP-NEXT:    buffer_load_ubyte v2, v[2:3], s[0:3], 0 addr64 offset:16
+; LOOP-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:33
+; LOOP-NEXT:    buffer_load_ubyte v5, v[2:3], s[0:3], 0 addr64 offset:35
+; LOOP-NEXT:    buffer_load_ubyte v6, v[2:3], s[0:3], 0 addr64 offset:34
+; LOOP-NEXT:    buffer_load_ubyte v2, v[2:3], s[0:3], 0 addr64 offset:32
 ; LOOP-NEXT:    s_waitcnt vmcnt(3)
 ; LOOP-NEXT:    v_lshlrev_b32_e32 v3, 8, v4
 ; LOOP-NEXT:    s_waitcnt vmcnt(2)
@@ -124,12 +196,12 @@ define amdgpu_cs void @memcpy_p1i8(ptr addrspace(1) %dst, ptr addrspace(1) %src)
 ; LOOP-NEXT:    v_or_b32_e32 v2, v3, v2
 ; LOOP-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
 ; LOOP-NEXT:    v_bfe_u32 v4, v2, 8, 8
-; LOOP-NEXT:    buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:16
+; LOOP-NEXT:    buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:32
 ; LOOP-NEXT:    s_waitcnt expcnt(0)
 ; LOOP-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
-; LOOP-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:17
-; LOOP-NEXT:    buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:18
-; LOOP-NEXT:    buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:19
+; LOOP-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:33
+; LOOP-NEXT:    buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:34
+; LOOP-NEXT:    buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:35
 ; LOOP-NEXT:    s_endpgm
 ;
 ; UNROLL-LABEL: memcpy_p1i8:
@@ -212,11 +284,75 @@ define amdgpu_cs void @memcpy_p1i8(ptr addrspace(1) %dst, ptr addrspace(1) %src)
 ; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:18
 ; UNROLL-NEXT:    s_waitcnt vmcnt(0)
 ; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:18
-; UNROLL-NEXT:    buffer_load_ubyte v2, v[2:3], s[0:3], 0 addr64 offset:19
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:19
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:19
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:20
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:20
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:21
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:21
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:22
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:22
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:23
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:23
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:24
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:24
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:25
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:25
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:26
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:26
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:27
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:27
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:28
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:28
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:29
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:29
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:30
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:30
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:31
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:31
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:32
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:32
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:33
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:33
+; UNROLL-NEXT:    s_waitcnt expcnt(0)
+; UNROLL-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:34
+; UNROLL-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:34
+; UNROLL-NEXT:    buffer_load_ubyte v2, v[2:3], s[0:3], 0 addr64 offset:35
 ; UNROLL-NEXT:    s_waitcnt vmcnt(0)
-; UNROLL-NEXT:    buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:19
+; UNROLL-NEXT:    buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:35
 ; UNROLL-NEXT:    s_endpgm
-  call void @llvm.memcpy.p1.p1.i32(ptr addrspace(1) %dst, ptr addrspace(1) %src, i32 20, i1 false)
+  call void @llvm.memcpy.p1.p1.i32(ptr addrspace(1) %dst, ptr addrspace(1) %src, i32 36, i1 false)
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
index a95f22507eec..ffe9e06c04ae 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
@@ -46,10 +46,10 @@ define amdgpu_kernel void @max_size_small_static_memcpy_caller0(ptr addrspace(1)
 ; ALL:       load-store-loop:
 ; ALL-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; ALL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; ALL-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP1]], align 1
+; ALL-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 1
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; ALL-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 1
-; ALL-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 16
+; ALL-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 1
+; ALL-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
 ; ALL-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1024
 ; ALL-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; ALL:       memcpy-split:
@@ -66,10 +66,10 @@ define amdgpu_kernel void @min_size_large_static_memcpy_caller0(ptr addrspace(1)
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP1]], align 1
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 1
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 1
-; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 16
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 1
+; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
 ; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
@@ -93,20 +93,20 @@ define amdgpu_kernel void @max_size_small_static_memmove_caller0(ptr addrspace(1
 ; ALL-NEXT:    br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
 ; ALL:       memmove_bwd_loop:
 ; ALL-NEXT:    [[TMP1:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 1024, [[TMP0:%.*]] ]
-; ALL-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP1]], 16
+; ALL-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP1]], 256
 ; ALL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP2]], align 1
+; ALL-NEXT:    [[ELEMENT:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP2]], align 1
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_INDEX]]
-; ALL-NEXT:    store <4 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
+; ALL-NEXT:    store <64 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
 ; ALL-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
 ; ALL-NEXT:    br i1 [[TMP4]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
 ; ALL:       memmove_fwd_loop:
 ; ALL-NEXT:    [[FWD_INDEX:%.*]] = phi i64 [ [[TMP7:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
 ; ALL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[FWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP5]], align 1
+; ALL-NEXT:    [[ELEMENT1:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP5]], align 1
 ; ALL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[FWD_INDEX]]
-; ALL-NEXT:    store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP6]], align 1
-; ALL-NEXT:    [[TMP7]] = add i64 [[FWD_INDEX]], 16
+; ALL-NEXT:    store <64 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP6]], align 1
+; ALL-NEXT:    [[TMP7]] = add i64 [[FWD_INDEX]], 256
 ; ALL-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[TMP7]], 1024
 ; ALL-NEXT:    br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
 ; ALL:       memmove_done:
@@ -128,20 +128,20 @@ define amdgpu_kernel void @min_size_large_static_memmove_caller0(ptr addrspace(1
 ; OPT-NEXT:    br label [[MEMMOVE_BWD_LOOP:%.*]]
 ; OPT:       memmove_bwd_loop:
 ; OPT-NEXT:    [[TMP4:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 1024, [[MEMMOVE_BWD_RESIDUAL]] ]
-; OPT-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP4]], 16
+; OPT-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP4]], 256
 ; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_INDEX]]
-; OPT-NEXT:    [[ELEMENT:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP5]], align 1
+; OPT-NEXT:    [[ELEMENT:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP5]], align 1
 ; OPT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP6]], align 1
+; OPT-NEXT:    store <64 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP6]], align 1
 ; OPT-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
 ; OPT-NEXT:    br i1 [[TMP7]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
 ; OPT:       memmove_fwd_loop:
 ; OPT-NEXT:    [[FWD_INDEX:%.*]] = phi i64 [ [[TMP10:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0:%.*]] ]
 ; OPT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[FWD_INDEX]]
-; OPT-NEXT:    [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP8]], align 1
+; OPT-NEXT:    [[ELEMENT1:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP8]], align 1
 ; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[FWD_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP9]], align 1
-; OPT-NEXT:    [[TMP10]] = add i64 [[FWD_INDEX]], 16
+; OPT-NEXT:    store <64 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP9]], align 1
+; OPT-NEXT:    [[TMP10]] = add i64 [[FWD_INDEX]], 256
 ; OPT-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 1024
 ; OPT-NEXT:    br i1 [[TMP11]], label [[MEMMOVE_FWD_RESIDUAL:%.*]], label [[MEMMOVE_FWD_LOOP]]
 ; OPT:       memmove_fwd_residual:
@@ -421,17 +421,30 @@ define amdgpu_kernel void @memcpy_multi_use_one_function_keep_small(ptr addrspac
 ; ALL-NEXT:    [[TMP15:%.*]] = icmp ult i64 [[TMP14]], [[TMP2]]
 ; ALL-NEXT:    br i1 [[TMP15]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
 ; ALL:       post-loop-memcpy-expansion:
-; ALL-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
-; ALL:       load-store-loop:
-; ALL-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[POST_LOOP_MEMCPY_EXPANSION]] ], [ [[TMP19:%.*]], [[LOAD_STORE_LOOP]] ]
-; ALL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[LOOP_INDEX]]
+; ALL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 0
 ; ALL-NEXT:    [[TMP17:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP16]], align 1
-; ALL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST1:%.*]], i64 [[LOOP_INDEX]]
+; ALL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST1:%.*]], i64 0
 ; ALL-NEXT:    store <4 x i32> [[TMP17]], ptr addrspace(1) [[TMP18]], align 1
-; ALL-NEXT:    [[TMP19]] = add i64 [[LOOP_INDEX]], 16
-; ALL-NEXT:    [[TMP20:%.*]] = icmp ult i64 [[TMP19]], 96
-; ALL-NEXT:    br i1 [[TMP20]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
-; ALL:       memcpy-split:
+; ALL-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 16
+; ALL-NEXT:    [[TMP19:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP33]], align 1
+; ALL-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST1]], i64 16
+; ALL-NEXT:    store <4 x i32> [[TMP19]], ptr addrspace(1) [[TMP20]], align 1
+; ALL-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 32
+; ALL-NEXT:    [[TMP35:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP34]], align 1
+; ALL-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST1]], i64 32
+; ALL-NEXT:    store <4 x i32> [[TMP35]], ptr addrspace(1) [[TMP36]], align 1
+; ALL-NEXT:    [[TMP37:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 48
+; ALL-NEXT:    [[TMP38:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP37]], align 1
+; ALL-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST1]], i64 48
+; ALL-NEXT:    store <4 x i32> [[TMP38]], ptr addrspace(1) [[TMP39]], align 1
+; ALL-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 64
+; ALL-NEXT:    [[TMP28:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP40]], align 1
+; ALL-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST1]], i64 64
+; ALL-NEXT:    store <4 x i32> [[TMP28]], ptr addrspace(1) [[TMP29]], align 1
+; ALL-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 80
+; ALL-NEXT:    [[TMP31:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP30]], align 1
+; ALL-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST1]], i64 80
+; ALL-NEXT:    store <4 x i32> [[TMP31]], ptr addrspace(1) [[TMP32]], align 1
 ; ALL-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 96
 ; ALL-NEXT:    [[TMP22:%.*]] = load i32, ptr addrspace(1) [[TMP21]], align 1
 ; ALL-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST1]], i64 96
@@ -456,10 +469,10 @@ define amdgpu_kernel void @memcpy_global_align4_global_align4_1028(ptr addrspace
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP1]], align 4
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 4
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
-; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 16
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
+; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
 ; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
@@ -479,10 +492,10 @@ define amdgpu_kernel void @memcpy_global_align4_global_align4_1025(ptr addrspace
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP1]], align 4
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 4
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
-; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 16
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
+; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
 ; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
@@ -502,10 +515,10 @@ define amdgpu_kernel void @memcpy_global_align4_global_align4_1026(ptr addrspace
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP1]], align 4
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 4
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
-; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 16
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
+; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
 ; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
@@ -525,10 +538,10 @@ define amdgpu_kernel void @memcpy_global_align4_global_align4_1032(ptr addrspace
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP1]], align 4
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 4
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
-; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 16
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
+; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
 ; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
@@ -548,10 +561,10 @@ define amdgpu_kernel void @memcpy_global_align4_global_align4_1034(ptr addrspace
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP1]], align 4
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 4
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
-; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 16
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
+; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
 ; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
@@ -575,10 +588,10 @@ define amdgpu_kernel void @memcpy_global_align4_global_align4_1035(ptr addrspace
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP1]], align 4
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 4
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
-; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 16
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
+; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
 ; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
@@ -606,10 +619,10 @@ define amdgpu_kernel void @memcpy_global_align4_global_align4_1036(ptr addrspace
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP1]], align 4
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 4
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
-; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 16
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
+; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
 ; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
@@ -633,10 +646,10 @@ define amdgpu_kernel void @memcpy_global_align4_global_align4_1039(ptr addrspace
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP1]], align 4
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 4
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
-; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 16
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
+; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
 ; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
@@ -691,10 +704,10 @@ define amdgpu_kernel void @memcpy_global_align4_global_align4_1027(ptr addrspace
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP1]], align 4
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 4
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
-; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 16
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
+; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
 ; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
@@ -764,10 +777,10 @@ define amdgpu_kernel void @memcpy_private_align4_private_align4_1027(ptr addrspa
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP1]], align 4
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(5) [[TMP1]], align 4
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(5) [[TMP3]], align 4
-; OPT-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 16
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(5) [[TMP3]], align 4
+; OPT-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 256
 ; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
@@ -814,10 +827,10 @@ define amdgpu_kernel void @memcpy_private_align1_private_align4_1027(ptr addrspa
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP1]], align 4
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(5) [[TMP1]], align 4
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(5) [[TMP3]], align 1
-; OPT-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 16
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(5) [[TMP3]], align 1
+; OPT-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 256
 ; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
@@ -864,10 +877,10 @@ define amdgpu_kernel void @memcpy_private_align4_private_align1_1027(ptr addrspa
 ; OPT:       load-store-loop:
 ; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP1]], align 1
+; OPT-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(5) [[TMP1]], align 1
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(5) [[TMP3]], align 4
-; OPT-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 16
+; OPT-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(5) [[TMP3]], align 4
+; OPT-NEXT:    [[TMP4]] = add i32 [[LOOP_INDEX]], 256
 ; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 1024
 ; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
@@ -1194,17 +1207,10 @@ define amdgpu_kernel void @memcpy_global_align4_global_align4_16(ptr addrspace(1
 ; MAX1024-NEXT:    ret void
 ;
 ; ALL-LABEL: @memcpy_global_align4_global_align4_16(
-; ALL-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
-; ALL:       load-store-loop:
-; ALL-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
-; ALL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
+; ALL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 0
 ; ALL-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP1]], align 4
-; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
+; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 0
 ; ALL-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 4
-; ALL-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 16
-; ALL-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 16
-; ALL-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
-; ALL:       memcpy-split:
 ; ALL-NEXT:    ret void
 ;
   call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) align 4 %dst, ptr addrspace(1) align 4 %src, i64 16, i1 false)
@@ -1326,20 +1332,20 @@ define amdgpu_kernel void @memmove_flat_align1_global_align1(ptr %dst, ptr addrs
 ; ALL-NEXT:    br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
 ; ALL:       memmove_bwd_loop:
 ; ALL-NEXT:    [[TMP2:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 256, [[TMP0:%.*]] ]
-; ALL-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP2]], 16
+; ALL-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP2]], 256
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP3]], align 1
+; ALL-NEXT:    [[ELEMENT:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP3]], align 1
 ; ALL-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[BWD_INDEX]]
-; ALL-NEXT:    store <4 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT:    store <64 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
 ; ALL-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
 ; ALL-NEXT:    br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
 ; ALL:       memmove_fwd_loop:
 ; ALL-NEXT:    [[FWD_INDEX:%.*]] = phi i64 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
 ; ALL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[FWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP6]], align 1
+; ALL-NEXT:    [[ELEMENT1:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP6]], align 1
 ; ALL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[FWD_INDEX]]
-; ALL-NEXT:    store <4 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
-; ALL-NEXT:    [[TMP8]] = add i64 [[FWD_INDEX]], 16
+; ALL-NEXT:    store <64 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
+; ALL-NEXT:    [[TMP8]] = add i64 [[FWD_INDEX]], 256
 ; ALL-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP8]], 256
 ; ALL-NEXT:    br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
 ; ALL:       memmove_done:
@@ -1360,20 +1366,20 @@ define amdgpu_kernel void @memmove_global_align1_flat_align1(ptr addrspace(1) %d
 ; ALL-NEXT:    br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
 ; ALL:       memmove_bwd_loop:
 ; ALL-NEXT:    [[TMP2:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 256, [[TMP0:%.*]] ]
-; ALL-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP2]], 16
+; ALL-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP2]], 256
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[BWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT:%.*]] = load <4 x i32>, ptr [[TMP3]], align 1
+; ALL-NEXT:    [[ELEMENT:%.*]] = load <64 x i32>, ptr [[TMP3]], align 1
 ; ALL-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_INDEX]]
-; ALL-NEXT:    store <4 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP4]], align 1
+; ALL-NEXT:    store <64 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP4]], align 1
 ; ALL-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
 ; ALL-NEXT:    br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
 ; ALL:       memmove_fwd_loop:
 ; ALL-NEXT:    [[FWD_INDEX:%.*]] = phi i64 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
 ; ALL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[FWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP6]], align 1
+; ALL-NEXT:    [[ELEMENT1:%.*]] = load <64 x i32>, ptr [[TMP6]], align 1
 ; ALL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[FWD_INDEX]]
-; ALL-NEXT:    store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP7]], align 1
-; ALL-NEXT:    [[TMP8]] = add i64 [[FWD_INDEX]], 16
+; ALL-NEXT:    store <64 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP7]], align 1
+; ALL-NEXT:    [[TMP8]] = add i64 [[FWD_INDEX]], 256
 ; ALL-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP8]], 256
 ; ALL-NEXT:    br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
 ; ALL:       memmove_done:
@@ -1394,20 +1400,20 @@ define amdgpu_kernel void @memmove_flat_align1_private_align1(ptr %dst, ptr addr
 ; ALL-NEXT:    br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
 ; ALL:       memmove_bwd_loop:
 ; ALL-NEXT:    [[TMP2:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 256, [[TMP0:%.*]] ]
-; ALL-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP2]], 16
+; ALL-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP2]], 256
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i64 [[BWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP3]], align 1
+; ALL-NEXT:    [[ELEMENT:%.*]] = load <64 x i32>, ptr addrspace(5) [[TMP3]], align 1
 ; ALL-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[BWD_INDEX]]
-; ALL-NEXT:    store <4 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT:    store <64 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
 ; ALL-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
 ; ALL-NEXT:    br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
 ; ALL:       memmove_fwd_loop:
 ; ALL-NEXT:    [[FWD_INDEX:%.*]] = phi i64 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
 ; ALL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i64 [[FWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP6]], align 1
+; ALL-NEXT:    [[ELEMENT1:%.*]] = load <64 x i32>, ptr addrspace(5) [[TMP6]], align 1
 ; ALL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[FWD_INDEX]]
-; ALL-NEXT:    store <4 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
-; ALL-NEXT:    [[TMP8]] = add i64 [[FWD_INDEX]], 16
+; ALL-NEXT:    store <64 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
+; ALL-NEXT:    [[TMP8]] = add i64 [[FWD_INDEX]], 256
 ; ALL-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP8]], 256
 ; ALL-NEXT:    br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
 ; ALL:       memmove_done:
@@ -1428,20 +1434,20 @@ define amdgpu_kernel void @memmove_private_align1_flat_align1(ptr addrspace(5) %
 ; ALL-NEXT:    br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
 ; ALL:       memmove_bwd_loop:
 ; ALL-NEXT:    [[TMP2:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 256, [[TMP0:%.*]] ]
-; ALL-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP2]], 16
+; ALL-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP2]], 256
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[BWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT:%.*]] = load <4 x i32>, ptr [[TMP3]], align 1
+; ALL-NEXT:    [[ELEMENT:%.*]] = load <64 x i32>, ptr [[TMP3]], align 1
 ; ALL-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i64 [[BWD_INDEX]]
-; ALL-NEXT:    store <4 x i32> [[ELEMENT]], ptr addrspace(5) [[TMP4]], align 1
+; ALL-NEXT:    store <64 x i32> [[ELEMENT]], ptr addrspace(5) [[TMP4]], align 1
 ; ALL-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
 ; ALL-NEXT:    br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
 ; ALL:       memmove_fwd_loop:
 ; ALL-NEXT:    [[FWD_INDEX:%.*]] = phi i64 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
 ; ALL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[FWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP6]], align 1
+; ALL-NEXT:    [[ELEMENT1:%.*]] = load <64 x i32>, ptr [[TMP6]], align 1
 ; ALL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i64 [[FWD_INDEX]]
-; ALL-NEXT:    store <4 x i32> [[ELEMENT1]], ptr addrspace(5) [[TMP7]], align 1
-; ALL-NEXT:    [[TMP8]] = add i64 [[FWD_INDEX]], 16
+; ALL-NEXT:    store <64 x i32> [[ELEMENT1]], ptr addrspace(5) [[TMP7]], align 1
+; ALL-NEXT:    [[TMP8]] = add i64 [[FWD_INDEX]], 256
 ; ALL-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP8]], 256
 ; ALL-NEXT:    br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
 ; ALL:       memmove_done:
@@ -1461,10 +1467,10 @@ define amdgpu_kernel void @memmove_private_align1_global_align1(ptr addrspace(5)
 ; ALL:       load-store-loop:
 ; ALL-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; ALL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; ALL-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP1]], align 1, !alias.scope [[META0:![0-9]+]]
+; ALL-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 1, !alias.scope [[META0:![0-9]+]]
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; ALL-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(5) [[TMP3]], align 1, !noalias [[META0]]
-; ALL-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 16
+; ALL-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(5) [[TMP3]], align 1, !noalias [[META0]]
+; ALL-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
 ; ALL-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 256
 ; ALL-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; ALL:       memcpy-split:
@@ -1484,10 +1490,10 @@ define amdgpu_kernel void @memmove_global_align1_private_align1(ptr addrspace(1)
 ; ALL:       load-store-loop:
 ; ALL-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; ALL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; ALL-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP1]], align 1, !alias.scope [[META3:![0-9]+]]
+; ALL-NEXT:    [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(5) [[TMP1]], align 1, !alias.scope [[META3:![0-9]+]]
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; ALL-NEXT:    store <4 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 1, !noalias [[META3]]
-; ALL-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 16
+; ALL-NEXT:    store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 1, !noalias [[META3]]
+; ALL-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
 ; ALL-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 256
 ; ALL-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; ALL:       memcpy-split:
@@ -2144,20 +2150,20 @@ define amdgpu_kernel void @memmove_private_align1_private_align1(ptr addrspace(5
 ; ALL-NEXT:    br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
 ; ALL:       memmove_bwd_loop:
 ; ALL-NEXT:    [[TMP1:%.*]] = phi i32 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 256, [[TMP0:%.*]] ]
-; ALL-NEXT:    [[BWD_INDEX]] = sub i32 [[TMP1]], 16
+; ALL-NEXT:    [[BWD_INDEX]] = sub i32 [[TMP1]], 256
 ; ALL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 [[BWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP2]], align 1
+; ALL-NEXT:    [[ELEMENT:%.*]] = load <64 x i32>, ptr addrspace(5) [[TMP2]], align 1
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 [[BWD_INDEX]]
-; ALL-NEXT:    store <4 x i32> [[ELEMENT]], ptr addrspace(5) [[TMP3]], align 1
+; ALL-NEXT:    store <64 x i32> [[ELEMENT]], ptr addrspace(5) [[TMP3]], align 1
 ; ALL-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[BWD_INDEX]], 0
 ; ALL-NEXT:    br i1 [[TMP4]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
 ; ALL:       memmove_fwd_loop:
 ; ALL-NEXT:    [[FWD_INDEX:%.*]] = phi i32 [ [[TMP7:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
 ; ALL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 [[FWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP5]], align 1
+; ALL-NEXT:    [[ELEMENT1:%.*]] = load <64 x i32>, ptr addrspace(5) [[TMP5]], align 1
 ; ALL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 [[FWD_INDEX]]
-; ALL-NEXT:    store <4 x i32> [[ELEMENT1]], ptr addrspace(5) [[TMP6]], align 1
-; ALL-NEXT:    [[TMP7]] = add i32 [[FWD_INDEX]], 16
+; ALL-NEXT:    store <64 x i32> [[ELEMENT1]], ptr addrspace(5) [[TMP6]], align 1
+; ALL-NEXT:    [[TMP7]] = add i32 [[FWD_INDEX]], 256
 ; ALL-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 256
 ; ALL-NEXT:    br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
 ; ALL:       memmove_done:
@@ -2231,27 +2237,27 @@ define amdgpu_kernel void @memmove_global_align4_static_residual_empty(ptr addrs
 ; OPT-NEXT:    [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(1) [[SRC:%.*]], [[DST:%.*]]
 ; OPT-NEXT:    br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
 ; OPT:       memmove_bwd_loop:
-; OPT-NEXT:    [[TMP1:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 1040, [[TMP0:%.*]] ]
-; OPT-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP1]], 16
+; OPT-NEXT:    [[TMP11:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 1280, [[TMP0:%.*]] ]
+; OPT-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP11]], 256
 ; OPT-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_INDEX]]
-; OPT-NEXT:    [[ELEMENT:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP2]], align 1
+; OPT-NEXT:    [[ELEMENT:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP2]], align 1
 ; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
+; OPT-NEXT:    store <64 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
 ; OPT-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
 ; OPT-NEXT:    br i1 [[TMP4]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
 ; OPT:       memmove_fwd_loop:
 ; OPT-NEXT:    [[FWD_INDEX:%.*]] = phi i64 [ [[TMP7:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
 ; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[FWD_INDEX]]
-; OPT-NEXT:    [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP5]], align 1
+; OPT-NEXT:    [[ELEMENT1:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP5]], align 1
 ; OPT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[FWD_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP6]], align 1
-; OPT-NEXT:    [[TMP7]] = add i64 [[FWD_INDEX]], 16
-; OPT-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[TMP7]], 1040
+; OPT-NEXT:    store <64 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP6]], align 1
+; OPT-NEXT:    [[TMP7]] = add i64 [[FWD_INDEX]], 256
+; OPT-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[TMP7]], 1280
 ; OPT-NEXT:    br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
 ; OPT:       memmove_done:
 ; OPT-NEXT:    ret void
 ;
-  call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) %dst, ptr addrspace(1) %src, i64 1040, i1 false)
+  call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) %dst, ptr addrspace(1) %src, i64 1280, i1 false)
   ret void
 }
 
@@ -2279,20 +2285,20 @@ define amdgpu_kernel void @memmove_global_align4_static_residual_full(ptr addrsp
 ; OPT-NEXT:    br label [[MEMMOVE_BWD_LOOP:%.*]]
 ; OPT:       memmove_bwd_loop:
 ; OPT-NEXT:    [[TMP13:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 1024, [[MEMMOVE_BWD_RESIDUAL]] ]
-; OPT-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP13]], 16
+; OPT-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP13]], 256
 ; OPT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_INDEX]]
-; OPT-NEXT:    [[ELEMENT:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP14]], align 1
+; OPT-NEXT:    [[ELEMENT:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP14]], align 1
 ; OPT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP15]], align 1
+; OPT-NEXT:    store <64 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP15]], align 1
 ; OPT-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
 ; OPT-NEXT:    br i1 [[TMP16]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
 ; OPT:       memmove_fwd_loop:
 ; OPT-NEXT:    [[FWD_INDEX:%.*]] = phi i64 [ [[TMP19:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0:%.*]] ]
 ; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[FWD_INDEX]]
-; OPT-NEXT:    [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP17]], align 1
+; OPT-NEXT:    [[ELEMENT1:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP17]], align 1
 ; OPT-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[FWD_INDEX]]
-; OPT-NEXT:    store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP18]], align 1
-; OPT-NEXT:    [[TMP19]] = add i64 [[FWD_INDEX]], 16
+; OPT-NEXT:    store <64 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP18]], align 1
+; OPT-NEXT:    [[TMP19]] = add i64 [[FWD_INDEX]], 256
 ; OPT-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[TMP19]], 1024
 ; OPT-NEXT:    br i1 [[TMP20]], label [[MEMMOVE_FWD_RESIDUAL:%.*]], label [[MEMMOVE_FWD_LOOP]]
 ; OPT:       memmove_fwd_residual:
@@ -2363,40 +2369,40 @@ entry:
 
 define amdgpu_kernel void @memmove_volatile(ptr addrspace(1) %dst, ptr addrspace(1) %src) #0 {
 ; MAX1024-LABEL: @memmove_volatile(
-; MAX1024-NEXT:    call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) [[DST:%.*]], ptr addrspace(1) [[SRC:%.*]], i64 64, i1 true)
+; MAX1024-NEXT:    call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) [[DST:%.*]], ptr addrspace(1) [[SRC:%.*]], i64 512, i1 true)
 ; MAX1024-NEXT:    ret void
 ;
 ; ALL-LABEL: @memmove_volatile(
 ; ALL-NEXT:    [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(1) [[SRC:%.*]], [[DST:%.*]]
 ; ALL-NEXT:    br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
 ; ALL:       memmove_bwd_loop:
-; ALL-NEXT:    [[TMP1:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 64, [[TMP0:%.*]] ]
-; ALL-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP1]], 16
+; ALL-NEXT:    [[TMP1:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 512, [[TMP0:%.*]] ]
+; ALL-NEXT:    [[BWD_INDEX]] = sub i64 [[TMP1]], 256
 ; ALL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT:%.*]] = load volatile <4 x i32>, ptr addrspace(1) [[TMP2]], align 1
+; ALL-NEXT:    [[ELEMENT:%.*]] = load volatile <64 x i32>, ptr addrspace(1) [[TMP2]], align 1
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_INDEX]]
-; ALL-NEXT:    store volatile <4 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
+; ALL-NEXT:    store volatile <64 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
 ; ALL-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
 ; ALL-NEXT:    br i1 [[TMP4]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
 ; ALL:       memmove_fwd_loop:
 ; ALL-NEXT:    [[FWD_INDEX:%.*]] = phi i64 [ [[TMP7:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
 ; ALL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[FWD_INDEX]]
-; ALL-NEXT:    [[ELEMENT1:%.*]] = load volatile <4 x i32>, ptr addrspace(1) [[TMP5]], align 1
+; ALL-NEXT:    [[ELEMENT1:%.*]] = load volatile <64 x i32>, ptr addrspace(1) [[TMP5]], align 1
 ; ALL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[FWD_INDEX]]
-; ALL-NEXT:    store volatile <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP6]], align 1
-; ALL-NEXT:    [[TMP7]] = add i64 [[FWD_INDEX]], 16
-; ALL-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[TMP7]], 64
+; ALL-NEXT:    store volatile <64 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP6]], align 1
+; ALL-NEXT:    [[TMP7]] = add i64 [[FWD_INDEX]], 256
+; ALL-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[TMP7]], 512
 ; ALL-NEXT:    br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
 ; ALL:       memmove_done:
 ; ALL-NEXT:    ret void
 ;
-  call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) %dst, ptr addrspace(1) %src, i64 64, i1 true)
+  call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) %dst, ptr addrspace(1) %src, i64 512, i1 true)
   ret void
 }
 
 define amdgpu_kernel void @memcpy_volatile(ptr addrspace(1) %dst, ptr addrspace(1) %src) #0 {
 ; MAX1024-LABEL: @memcpy_volatile(
-; MAX1024-NEXT:    call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) [[DST:%.*]], ptr addrspace(1) [[SRC:%.*]], i64 64, i1 true)
+; MAX1024-NEXT:    call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) [[DST:%.*]], ptr addrspace(1) [[SRC:%.*]], i64 512, i1 true)
 ; MAX1024-NEXT:    ret void
 ;
 ; ALL-LABEL: @memcpy_volatile(
@@ -2404,16 +2410,16 @@ define amdgpu_kernel void @memcpy_volatile(ptr addrspace(1) %dst, ptr addrspace(
 ; ALL:       load-store-loop:
 ; ALL-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
 ; ALL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; ALL-NEXT:    [[TMP2:%.*]] = load volatile <4 x i32>, ptr addrspace(1) [[TMP1]], align 1
+; ALL-NEXT:    [[TMP2:%.*]] = load volatile <64 x i32>, ptr addrspace(1) [[TMP1]], align 1
 ; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST:%.*]], i64 [[LOOP_INDEX]]
-; ALL-NEXT:    store volatile <4 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 1
-; ALL-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 16
-; ALL-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 64
+; ALL-NEXT:    store volatile <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 1
+; ALL-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 256
+; ALL-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 512
 ; ALL-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; ALL:       memcpy-split:
 ; ALL-NEXT:    ret void
 ;
-  call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dst, ptr addrspace(1) %src, i64 64, i1 true)
+  call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dst, ptr addrspace(1) %src, i64 512, i1 true)
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
new file mode 100644
index 000000000000..565fce0e7abd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
@@ -0,0 +1,16049 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 %s -o - | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 -mattr=-unaligned-access-mode %s -o - | FileCheck -check-prefix=ALIGNED %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 -amdgpu-memcpy-loop-unroll=3 %s -o - | FileCheck -check-prefix=UNROLL3 %s
+
+; For checking that LowerMemIntrinsics lowers memcpy and memmove with large
+; constant copy-sizes into loops with multiple load/store pairs.
+
+
+; memcpy for address spaces 0, 1, 4, 5
+
+define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0) align 1 readonly %src) {
+; CHECK-LABEL: memcpy_p0_p0_sz2048:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0
+; CHECK-NEXT:  .LBB0_1: ; %load-store-loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    v_add_co_u32 v96, vcc_lo, v2, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v97, vcc_lo, s5, v3, vcc_lo
+; CHECK-NEXT:    v_add_co_u32 v100, vcc_lo, v0, s4
+; CHECK-NEXT:    s_add_u32 s4, s4, 0x100
+; CHECK-NEXT:    s_clause 0xf
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[96:97] offset:224
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[96:97] offset:240
+; CHECK-NEXT:    flat_load_dwordx4 v[12:15], v[96:97] offset:192
+; CHECK-NEXT:    flat_load_dwordx4 v[16:19], v[96:97] offset:208
+; CHECK-NEXT:    flat_load_dwordx4 v[20:23], v[96:97] offset:160
+; CHECK-NEXT:    flat_load_dwordx4 v[24:27], v[96:97] offset:176
+; CHECK-NEXT:    flat_load_dwordx4 v[28:31], v[96:97] offset:128
+; CHECK-NEXT:    flat_load_dwordx4 v[32:35], v[96:97] offset:144
+; CHECK-NEXT:    flat_load_dwordx4 v[36:39], v[96:97] offset:96
+; CHECK-NEXT:    flat_load_dwordx4 v[48:51], v[96:97] offset:112
+; CHECK-NEXT:    flat_load_dwordx4 v[52:55], v[96:97] offset:64
+; CHECK-NEXT:    flat_load_dwordx4 v[64:67], v[96:97] offset:80
+; CHECK-NEXT:    flat_load_dwordx4 v[68:71], v[96:97] offset:32
+; CHECK-NEXT:    flat_load_dwordx4 v[80:83], v[96:97] offset:48
+; CHECK-NEXT:    flat_load_dwordx4 v[84:87], v[96:97]
+; CHECK-NEXT:    flat_load_dwordx4 v[96:99], v[96:97] offset:16
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v101, vcc_lo, s5, v1, vcc_lo
+; CHECK-NEXT:    s_addc_u32 s5, s5, 0
+; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[4:7] offset:224
+; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[8:11] offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[12:15] offset:192
+; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[16:19] offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[20:23] offset:160
+; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[24:27] offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[28:31] offset:128
+; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[32:35] offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[36:39] offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[48:51] offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[52:55] offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[64:67] offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[68:71] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[80:83] offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[84:87]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[96:99] offset:16
+; CHECK-NEXT:    v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
+; CHECK-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
+; CHECK-NEXT:    s_cbranch_vccnz .LBB0_1
+; CHECK-NEXT:  ; %bb.2: ; %memcpy-split
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+;
+; ALIGNED-LABEL: memcpy_p0_p0_sz2048:
+; ALIGNED:       ; %bb.0: ; %entry
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; ALIGNED-NEXT:    s_mov_b64 s[4:5], 0
+; ALIGNED-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; ALIGNED-NEXT:  .LBB0_1: ; %load-store-loop
+; ALIGNED-NEXT:    ; =>This Inner Loop Header: Depth=1
+; ALIGNED-NEXT:    v_add_co_u32 v24, vcc_lo, v2, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v25, vcc_lo, s5, v3, vcc_lo
+; ALIGNED-NEXT:    s_clause 0xf
+; ALIGNED-NEXT:    flat_load_dwordx4 v[16:19], v[24:25] offset:240
+; ALIGNED-NEXT:    flat_load_dwordx4 v[20:23], v[24:25] offset:224
+; ALIGNED-NEXT:    flat_load_dwordx4 v[4:7], v[24:25]
+; ALIGNED-NEXT:    flat_load_dwordx4 v[8:11], v[24:25] offset:16
+; ALIGNED-NEXT:    flat_load_dwordx4 v[12:15], v[24:25] offset:32
+; ALIGNED-NEXT:    flat_load_dwordx4 v[112:115], v[24:25] offset:48
+; ALIGNED-NEXT:    flat_load_dwordx4 v[116:119], v[24:25] offset:64
+; ALIGNED-NEXT:    flat_load_dwordx4 v[40:43], v[24:25] offset:80
+; ALIGNED-NEXT:    flat_load_dwordx4 v[26:29], v[24:25] offset:96
+; ALIGNED-NEXT:    flat_load_dwordx4 v[32:35], v[24:25] offset:112
+; ALIGNED-NEXT:    flat_load_dwordx4 v[44:47], v[24:25] offset:128
+; ALIGNED-NEXT:    flat_load_dwordx4 v[52:55], v[24:25] offset:144
+; ALIGNED-NEXT:    flat_load_dwordx4 v[66:69], v[24:25] offset:160
+; ALIGNED-NEXT:    flat_load_dwordx4 v[81:84], v[24:25] offset:176
+; ALIGNED-NEXT:    flat_load_dwordx4 v[96:99], v[24:25] offset:192
+; ALIGNED-NEXT:    flat_load_dwordx4 v[100:103], v[24:25] offset:208
+; ALIGNED-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
+; ALIGNED-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; ALIGNED-NEXT:    s_add_u32 s4, s4, 0x100
+; ALIGNED-NEXT:    s_addc_u32 s5, s5, 0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v31 offset:254
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v31 offset:252
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v30 offset:250
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v30 offset:248
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v25 offset:246
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v25 offset:244
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v24 offset:242
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v24 offset:240
+; ALIGNED-NEXT:    s_waitcnt lgkmcnt(22)
+; ALIGNED-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT:    v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v51 offset:238
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v51 offset:236
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v50 offset:234
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v50 offset:232
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v49 offset:230
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v49 offset:228
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v36 offset:226
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v36 offset:224
+; ALIGNED-NEXT:    s_waitcnt lgkmcnt(16)
+; ALIGNED-NEXT:    buffer_store_dword v100, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT:    buffer_store_dword v101, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT:    buffer_store_dword v102, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT:    buffer_store_dword v103, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v71, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT:    buffer_load_dword v70, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT:    buffer_load_dword v65, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT:    buffer_load_dword v64, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v31
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 8, v31
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v30
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 8, v30
+; ALIGNED-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v71 offset:222
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v71 offset:220
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v70 offset:218
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v70 offset:216
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v65 offset:214
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v65 offset:212
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v64 offset:210
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v64 offset:208
+; ALIGNED-NEXT:    buffer_store_dword v96, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT:    buffer_store_dword v97, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT:    buffer_store_dword v98, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT:    buffer_store_dword v99, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v87, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT:    buffer_load_dword v86, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT:    buffer_load_dword v85, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT:    buffer_load_dword v80, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v87 offset:206
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v87 offset:204
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v86 offset:202
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v86 offset:200
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v85 offset:198
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v85 offset:196
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v80 offset:194
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v80 offset:192
+; ALIGNED-NEXT:    buffer_store_dword v81, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT:    buffer_store_dword v82, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT:    buffer_store_dword v83, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT:    buffer_store_dword v84, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v101, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT:    buffer_load_dword v99, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT:    buffer_load_dword v96, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT:    buffer_load_dword v81, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v101 offset:190
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v101 offset:188
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v99 offset:186
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v99 offset:184
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v96 offset:182
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v96 offset:180
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v81 offset:178
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v81 offset:176
+; ALIGNED-NEXT:    buffer_store_dword v66, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT:    buffer_store_dword v67, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT:    buffer_store_dword v68, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT:    buffer_store_dword v69, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v100, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT:    buffer_load_dword v97, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT:    buffer_load_dword v82, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT:    buffer_load_dword v66, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v100 offset:174
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v100 offset:172
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v97 offset:170
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v97 offset:168
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v82 offset:166
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v82 offset:164
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v66 offset:162
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v66 offset:160
+; ALIGNED-NEXT:    buffer_store_dword v52, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT:    buffer_store_dword v53, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT:    buffer_store_dword v54, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT:    buffer_store_dword v55, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v98, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT:    buffer_load_dword v83, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT:    buffer_load_dword v67, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v98 offset:158
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v98 offset:156
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v83 offset:154
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v83 offset:152
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v67 offset:150
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v67 offset:148
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v52 offset:146
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v52 offset:144
+; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v84, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT:    buffer_load_dword v68, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v84 offset:142
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v84 offset:140
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v68 offset:138
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v68 offset:136
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v53 offset:134
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v53 offset:132
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v37 offset:130
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v37 offset:128
+; ALIGNED-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT:    buffer_store_dword v35, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v69, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v69 offset:126
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v69 offset:124
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v54 offset:122
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v54 offset:120
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v38 offset:118
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v38 offset:116
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v32 offset:114
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v32 offset:112
+; ALIGNED-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v55 offset:110
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v55 offset:108
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v39 offset:106
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v39 offset:104
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v33 offset:102
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v33 offset:100
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v26 offset:98
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v26 offset:96
+; ALIGNED-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v48 offset:94
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v48 offset:92
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v34 offset:90
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v34 offset:88
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v27 offset:86
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v27 offset:84
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v21 offset:82
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v21 offset:80
+; ALIGNED-NEXT:    buffer_store_dword v116, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT:    buffer_store_dword v117, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT:    buffer_store_dword v118, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT:    buffer_store_dword v119, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v35 offset:78
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v35 offset:76
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v28 offset:74
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v28 offset:72
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v22 offset:70
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v22 offset:68
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v19 offset:66
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v19 offset:64
+; ALIGNED-NEXT:    buffer_store_dword v112, off, s[0:3], s32 offset:256
+; ALIGNED-NEXT:    buffer_store_dword v113, off, s[0:3], s32 offset:260
+; ALIGNED-NEXT:    buffer_store_dword v114, off, s[0:3], s32 offset:264
+; ALIGNED-NEXT:    buffer_store_dword v115, off, s[0:3], s32 offset:268
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:268
+; ALIGNED-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:264
+; ALIGNED-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:260
+; ALIGNED-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:256
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 8, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 8, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v51
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 8, v51
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v50
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 8, v50
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v29 offset:62
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v29 offset:60
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v23 offset:58
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v23 offset:56
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v20 offset:54
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v20 offset:52
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v18 offset:50
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v18 offset:48
+; ALIGNED-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:272
+; ALIGNED-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:276
+; ALIGNED-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:280
+; ALIGNED-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:284
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:280
+; ALIGNED-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:284
+; ALIGNED-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:272
+; ALIGNED-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:276
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v15 offset:42
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v15 offset:40
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v14 offset:46
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v14 offset:44
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v13 offset:34
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v13 offset:32
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v12 offset:38
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v12 offset:36
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v11 offset:30
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v11 offset:28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v10 offset:26
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v10 offset:24
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v9 offset:22
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v9 offset:20
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v8 offset:18
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v8 offset:16
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v112 offset:247
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v65
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v102 offset:255
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v49
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 8, v49
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v31 offset:253
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 24, v36
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 8, v36
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v103 offset:251
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v71
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 8, v71
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v30 offset:249
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v70
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 8, v70
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 8, v65
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v25 offset:245
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 24, v64
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 8, v64
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v112 offset:215
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v67
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 8, v67
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v113 offset:243
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v87
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v87
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v24 offset:241
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 24, v86
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 8, v86
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v114 offset:239
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v85
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v85
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v51 offset:237
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 24, v80
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 8, v80
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v115 offset:235
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v101
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 8, v101
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v50 offset:233
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 24, v99
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 8, v99
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v102 offset:231
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v96
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v96, 8, v96
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v49 offset:229
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 24, v81
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 8, v81
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v31 offset:227
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 24, v100
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v100
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v36 offset:225
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 24, v97
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v97, 8, v97
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v103 offset:223
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v82
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 8, v82
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v71 offset:221
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 24, v66
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 8, v66
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v30 offset:219
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v98
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v98
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v70 offset:217
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 24, v83
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 8, v83
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v65 offset:213
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 24, v52
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v52, 8, v52
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v25 offset:211
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 24, v84
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 8, v84
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v64 offset:209
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 24, v68
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v67 offset:149
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 24, v8
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v8, 8, v8
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v68, 8, v68
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v113 offset:207
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v53
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v53, 8, v53
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v87 offset:205
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 24, v37
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v37, 8, v37
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v24 offset:203
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 24, v69
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v69, 8, v69
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v86 offset:201
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v54
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v54, 8, v54
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v114 offset:199
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v38
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v38, 8, v38
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v85 offset:197
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 24, v32
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v32, 8, v32
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v51 offset:195
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 24, v55
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v55, 8, v55
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v80 offset:193
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 24, v39
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v39, 8, v39
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v115 offset:191
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v33
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v33, 8, v33
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v101 offset:189
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 24, v26
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v26, 8, v26
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v50 offset:187
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 24, v48
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v48, 8, v48
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v99 offset:185
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 24, v34
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v34, 8, v34
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v102 offset:183
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v27
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v27, 8, v27
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v96 offset:181
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v96, 24, v21
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v21, 8, v21
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v49 offset:179
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 24, v35
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v35, 8, v35
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v81 offset:177
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 24, v28
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v28, 8, v28
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v31 offset:175
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 24, v22
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v22, 8, v22
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v100 offset:173
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 24, v19
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v19, 8, v19
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v36 offset:171
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 24, v29
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v29, 8, v29
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v97 offset:169
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v97, 24, v23
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v23, 8, v23
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v103 offset:167
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v20
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v20, 8, v20
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v82 offset:165
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 24, v18
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v18, 8, v18
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v71 offset:163
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 24, v15
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v15, 8, v15
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v66 offset:161
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 24, v14
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v14, 8, v14
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v30 offset:159
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v13
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v13, 8, v13
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v98 offset:157
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 24, v12
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v12, 8, v12
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v70 offset:155
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 24, v11
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v11, 8, v11
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v83 offset:153
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 24, v10
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v10, 8, v10
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v112 offset:151
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v9
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v9, 8, v9
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v65 offset:147
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v52 offset:145
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v25 offset:143
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v84 offset:141
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v64 offset:139
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v68 offset:137
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v113 offset:135
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v53 offset:133
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v87 offset:131
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v37 offset:129
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v24 offset:127
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v69 offset:125
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v86 offset:123
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v54 offset:121
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v114 offset:119
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v38 offset:117
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v85 offset:115
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v32 offset:113
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v51 offset:111
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v55 offset:109
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v80 offset:107
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v39 offset:105
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v115 offset:103
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v33 offset:101
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v101 offset:99
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v26 offset:97
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v50 offset:95
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v48 offset:93
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v99 offset:91
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v34 offset:89
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v102 offset:87
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v27 offset:85
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v96 offset:83
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v21 offset:81
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v49 offset:79
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v35 offset:77
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v81 offset:75
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v28 offset:73
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v31 offset:71
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v22 offset:69
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v100 offset:67
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v19 offset:65
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v36 offset:63
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v29 offset:61
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v97 offset:59
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v23 offset:57
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v103 offset:55
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v20 offset:53
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v82 offset:51
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v18 offset:49
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v71 offset:43
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v15 offset:41
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v66 offset:47
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v14 offset:45
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v30 offset:35
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v13 offset:33
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v98 offset:39
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v12 offset:37
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v70 offset:31
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v11 offset:29
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v83 offset:27
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v10 offset:25
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v112 offset:23
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v9 offset:21
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v67 offset:19
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v8 offset:17
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v7 offset:14
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v7 offset:12
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v6 offset:10
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v6 offset:8
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v5 offset:6
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v5 offset:4
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v4 offset:2
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v4
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v8, 24, v7
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v7, 8, v7
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v9, 24, v6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v6, 8, v6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v10, 24, v5
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v5, 8, v5
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v11, 24, v4
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v4, 8, v4
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v8 offset:15
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v7 offset:13
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v9 offset:11
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v6 offset:9
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v10 offset:7
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v5 offset:5
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v11 offset:3
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v4 offset:1
+; ALIGNED-NEXT:    s_cbranch_vccnz .LBB0_1
+; ALIGNED-NEXT:  ; %bb.2: ; %memcpy-split
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_dword v47, off, s[0:3], s32
+; ALIGNED-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4
+; ALIGNED-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8
+; ALIGNED-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12
+; ALIGNED-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16
+; ALIGNED-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20
+; ALIGNED-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24
+; ALIGNED-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; UNROLL3-LABEL: memcpy_p0_p0_sz2048:
+; UNROLL3:       ; %bb.0: ; %entry
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    s_mov_b64 s[4:5], 0
+; UNROLL3-NEXT:    .p2align 6
+; UNROLL3-NEXT:  .LBB0_1: ; %load-store-loop
+; UNROLL3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT:    v_add_co_u32 v12, vcc_lo, v2, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v13, vcc_lo, s5, v3, vcc_lo
+; UNROLL3-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; UNROLL3-NEXT:    s_add_u32 s4, s4, 48
+; UNROLL3-NEXT:    s_clause 0x2
+; UNROLL3-NEXT:    flat_load_dwordx4 v[4:7], v[12:13]
+; UNROLL3-NEXT:    flat_load_dwordx4 v[8:11], v[12:13] offset:16
+; UNROLL3-NEXT:    flat_load_dwordx4 v[12:15], v[12:13] offset:32
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; UNROLL3-NEXT:    s_addc_u32 s5, s5, 0
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[4:7]
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(2)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[8:11] offset:16
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(2)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[12:15] offset:32
+; UNROLL3-NEXT:    v_cmp_gt_u64_e64 s6, 0x7e0, s[4:5]
+; UNROLL3-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
+; UNROLL3-NEXT:    s_cbranch_vccnz .LBB0_1
+; UNROLL3-NEXT:  ; %bb.2: ; %memcpy-split
+; UNROLL3-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:2016
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:2016
+; UNROLL3-NEXT:    flat_load_dwordx4 v[2:5], v[2:3] offset:2032
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[2:5] offset:2032
+; UNROLL3-NEXT:    s_waitcnt lgkmcnt(0)
+; UNROLL3-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  tail call void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 2048, i1 false)
+  ret void
+}
+
+define void @memcpy_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {
+; CHECK-LABEL: memcpy_p1_p1_sz2048:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0
+; CHECK-NEXT:  .LBB1_1: ; %load-store-loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    v_add_co_u32 v96, vcc_lo, v2, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v97, vcc_lo, s5, v3, vcc_lo
+; CHECK-NEXT:    v_add_co_u32 v100, vcc_lo, v0, s4
+; CHECK-NEXT:    s_add_u32 s4, s4, 0x100
+; CHECK-NEXT:    s_clause 0xf
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[96:97], off offset:224
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[96:97], off offset:240
+; CHECK-NEXT:    global_load_dwordx4 v[12:15], v[96:97], off offset:192
+; CHECK-NEXT:    global_load_dwordx4 v[16:19], v[96:97], off offset:208
+; CHECK-NEXT:    global_load_dwordx4 v[20:23], v[96:97], off offset:160
+; CHECK-NEXT:    global_load_dwordx4 v[24:27], v[96:97], off offset:176
+; CHECK-NEXT:    global_load_dwordx4 v[28:31], v[96:97], off offset:128
+; CHECK-NEXT:    global_load_dwordx4 v[32:35], v[96:97], off offset:144
+; CHECK-NEXT:    global_load_dwordx4 v[36:39], v[96:97], off offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[48:51], v[96:97], off offset:112
+; CHECK-NEXT:    global_load_dwordx4 v[52:55], v[96:97], off offset:64
+; CHECK-NEXT:    global_load_dwordx4 v[64:67], v[96:97], off offset:80
+; CHECK-NEXT:    global_load_dwordx4 v[68:71], v[96:97], off offset:32
+; CHECK-NEXT:    global_load_dwordx4 v[80:83], v[96:97], off offset:48
+; CHECK-NEXT:    global_load_dwordx4 v[84:87], v[96:97], off
+; CHECK-NEXT:    global_load_dwordx4 v[96:99], v[96:97], off offset:16
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v101, vcc_lo, s5, v1, vcc_lo
+; CHECK-NEXT:    s_addc_u32 s5, s5, 0
+; CHECK-NEXT:    s_waitcnt vmcnt(15)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[4:7], off offset:224
+; CHECK-NEXT:    s_waitcnt vmcnt(14)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[8:11], off offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(13)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[12:15], off offset:192
+; CHECK-NEXT:    s_waitcnt vmcnt(12)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[16:19], off offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(11)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[20:23], off offset:160
+; CHECK-NEXT:    s_waitcnt vmcnt(10)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[24:27], off offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(9)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[28:31], off offset:128
+; CHECK-NEXT:    s_waitcnt vmcnt(8)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[32:35], off offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(7)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[36:39], off offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(6)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[48:51], off offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[52:55], off offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[64:67], off offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[68:71], off offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(2)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[80:83], off offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[84:87], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[96:99], off offset:16
+; CHECK-NEXT:    v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
+; CHECK-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
+; CHECK-NEXT:    s_cbranch_vccnz .LBB1_1
+; CHECK-NEXT:  ; %bb.2: ; %memcpy-split
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+;
+; ALIGNED-LABEL: memcpy_p1_p1_sz2048:
+; ALIGNED:       ; %bb.0: ; %entry
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; ALIGNED-NEXT:    s_mov_b64 s[4:5], 0
+; ALIGNED-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; ALIGNED-NEXT:  .LBB1_1: ; %load-store-loop
+; ALIGNED-NEXT:    ; =>This Inner Loop Header: Depth=1
+; ALIGNED-NEXT:    v_add_co_u32 v24, vcc_lo, v2, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v25, vcc_lo, s5, v3, vcc_lo
+; ALIGNED-NEXT:    s_clause 0xf
+; ALIGNED-NEXT:    global_load_dwordx4 v[16:19], v[24:25], off offset:240
+; ALIGNED-NEXT:    global_load_dwordx4 v[20:23], v[24:25], off offset:224
+; ALIGNED-NEXT:    global_load_dwordx4 v[4:7], v[24:25], off
+; ALIGNED-NEXT:    global_load_dwordx4 v[8:11], v[24:25], off offset:16
+; ALIGNED-NEXT:    global_load_dwordx4 v[12:15], v[24:25], off offset:32
+; ALIGNED-NEXT:    global_load_dwordx4 v[112:115], v[24:25], off offset:48
+; ALIGNED-NEXT:    global_load_dwordx4 v[116:119], v[24:25], off offset:64
+; ALIGNED-NEXT:    global_load_dwordx4 v[40:43], v[24:25], off offset:80
+; ALIGNED-NEXT:    global_load_dwordx4 v[26:29], v[24:25], off offset:96
+; ALIGNED-NEXT:    global_load_dwordx4 v[32:35], v[24:25], off offset:112
+; ALIGNED-NEXT:    global_load_dwordx4 v[44:47], v[24:25], off offset:128
+; ALIGNED-NEXT:    global_load_dwordx4 v[52:55], v[24:25], off offset:144
+; ALIGNED-NEXT:    global_load_dwordx4 v[66:69], v[24:25], off offset:160
+; ALIGNED-NEXT:    global_load_dwordx4 v[81:84], v[24:25], off offset:176
+; ALIGNED-NEXT:    global_load_dwordx4 v[96:99], v[24:25], off offset:192
+; ALIGNED-NEXT:    global_load_dwordx4 v[100:103], v[24:25], off offset:208
+; ALIGNED-NEXT:    s_waitcnt vmcnt(15)
+; ALIGNED-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; ALIGNED-NEXT:    s_add_u32 s4, s4, 0x100
+; ALIGNED-NEXT:    s_addc_u32 s5, s5, 0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v31, off offset:254
+; ALIGNED-NEXT:    global_store_byte v[16:17], v31, off offset:252
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v30, off offset:250
+; ALIGNED-NEXT:    global_store_byte v[16:17], v30, off offset:248
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v25, off offset:246
+; ALIGNED-NEXT:    global_store_byte v[16:17], v25, off offset:244
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v24, off offset:242
+; ALIGNED-NEXT:    global_store_byte v[16:17], v24, off offset:240
+; ALIGNED-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT:    v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v51, off offset:238
+; ALIGNED-NEXT:    global_store_byte v[16:17], v51, off offset:236
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v50, off offset:234
+; ALIGNED-NEXT:    global_store_byte v[16:17], v50, off offset:232
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v49, off offset:230
+; ALIGNED-NEXT:    global_store_byte v[16:17], v49, off offset:228
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v36, off offset:226
+; ALIGNED-NEXT:    global_store_byte v[16:17], v36, off offset:224
+; ALIGNED-NEXT:    buffer_store_dword v100, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT:    buffer_store_dword v101, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT:    buffer_store_dword v102, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT:    buffer_store_dword v103, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v71, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT:    buffer_load_dword v70, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT:    buffer_load_dword v65, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT:    buffer_load_dword v64, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v31
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 8, v31
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v30
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 8, v30
+; ALIGNED-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v71, off offset:222
+; ALIGNED-NEXT:    global_store_byte v[16:17], v71, off offset:220
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v70, off offset:218
+; ALIGNED-NEXT:    global_store_byte v[16:17], v70, off offset:216
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v65, off offset:214
+; ALIGNED-NEXT:    global_store_byte v[16:17], v65, off offset:212
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v64, off offset:210
+; ALIGNED-NEXT:    global_store_byte v[16:17], v64, off offset:208
+; ALIGNED-NEXT:    buffer_store_dword v96, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT:    buffer_store_dword v97, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT:    buffer_store_dword v98, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT:    buffer_store_dword v99, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v87, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT:    buffer_load_dword v86, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT:    buffer_load_dword v85, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT:    buffer_load_dword v80, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v87, off offset:206
+; ALIGNED-NEXT:    global_store_byte v[16:17], v87, off offset:204
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v86, off offset:202
+; ALIGNED-NEXT:    global_store_byte v[16:17], v86, off offset:200
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v85, off offset:198
+; ALIGNED-NEXT:    global_store_byte v[16:17], v85, off offset:196
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v80, off offset:194
+; ALIGNED-NEXT:    global_store_byte v[16:17], v80, off offset:192
+; ALIGNED-NEXT:    buffer_store_dword v81, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT:    buffer_store_dword v82, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT:    buffer_store_dword v83, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT:    buffer_store_dword v84, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v101, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT:    buffer_load_dword v99, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT:    buffer_load_dword v96, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT:    buffer_load_dword v81, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v101, off offset:190
+; ALIGNED-NEXT:    global_store_byte v[16:17], v101, off offset:188
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v99, off offset:186
+; ALIGNED-NEXT:    global_store_byte v[16:17], v99, off offset:184
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v96, off offset:182
+; ALIGNED-NEXT:    global_store_byte v[16:17], v96, off offset:180
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v81, off offset:178
+; ALIGNED-NEXT:    global_store_byte v[16:17], v81, off offset:176
+; ALIGNED-NEXT:    buffer_store_dword v66, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT:    buffer_store_dword v67, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT:    buffer_store_dword v68, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT:    buffer_store_dword v69, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v100, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT:    buffer_load_dword v97, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT:    buffer_load_dword v82, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT:    buffer_load_dword v66, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v100, off offset:174
+; ALIGNED-NEXT:    global_store_byte v[16:17], v100, off offset:172
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v97, off offset:170
+; ALIGNED-NEXT:    global_store_byte v[16:17], v97, off offset:168
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v82, off offset:166
+; ALIGNED-NEXT:    global_store_byte v[16:17], v82, off offset:164
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v66, off offset:162
+; ALIGNED-NEXT:    global_store_byte v[16:17], v66, off offset:160
+; ALIGNED-NEXT:    buffer_store_dword v52, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT:    buffer_store_dword v53, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT:    buffer_store_dword v54, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT:    buffer_store_dword v55, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v98, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT:    buffer_load_dword v83, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT:    buffer_load_dword v67, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v98, off offset:158
+; ALIGNED-NEXT:    global_store_byte v[16:17], v98, off offset:156
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v83, off offset:154
+; ALIGNED-NEXT:    global_store_byte v[16:17], v83, off offset:152
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v67, off offset:150
+; ALIGNED-NEXT:    global_store_byte v[16:17], v67, off offset:148
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v52, off offset:146
+; ALIGNED-NEXT:    global_store_byte v[16:17], v52, off offset:144
+; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v84, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT:    buffer_load_dword v68, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v84, off offset:142
+; ALIGNED-NEXT:    global_store_byte v[16:17], v84, off offset:140
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v68, off offset:138
+; ALIGNED-NEXT:    global_store_byte v[16:17], v68, off offset:136
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v53, off offset:134
+; ALIGNED-NEXT:    global_store_byte v[16:17], v53, off offset:132
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v37, off offset:130
+; ALIGNED-NEXT:    global_store_byte v[16:17], v37, off offset:128
+; ALIGNED-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT:    buffer_store_dword v35, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v69, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v69, off offset:126
+; ALIGNED-NEXT:    global_store_byte v[16:17], v69, off offset:124
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v54, off offset:122
+; ALIGNED-NEXT:    global_store_byte v[16:17], v54, off offset:120
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v38, off offset:118
+; ALIGNED-NEXT:    global_store_byte v[16:17], v38, off offset:116
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v32, off offset:114
+; ALIGNED-NEXT:    global_store_byte v[16:17], v32, off offset:112
+; ALIGNED-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v55, off offset:110
+; ALIGNED-NEXT:    global_store_byte v[16:17], v55, off offset:108
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v39, off offset:106
+; ALIGNED-NEXT:    global_store_byte v[16:17], v39, off offset:104
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v33, off offset:102
+; ALIGNED-NEXT:    global_store_byte v[16:17], v33, off offset:100
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v26, off offset:98
+; ALIGNED-NEXT:    global_store_byte v[16:17], v26, off offset:96
+; ALIGNED-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v48, off offset:94
+; ALIGNED-NEXT:    global_store_byte v[16:17], v48, off offset:92
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v34, off offset:90
+; ALIGNED-NEXT:    global_store_byte v[16:17], v34, off offset:88
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v27, off offset:86
+; ALIGNED-NEXT:    global_store_byte v[16:17], v27, off offset:84
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v21, off offset:82
+; ALIGNED-NEXT:    global_store_byte v[16:17], v21, off offset:80
+; ALIGNED-NEXT:    buffer_store_dword v116, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT:    buffer_store_dword v117, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT:    buffer_store_dword v118, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT:    buffer_store_dword v119, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v35, off offset:78
+; ALIGNED-NEXT:    global_store_byte v[16:17], v35, off offset:76
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v28, off offset:74
+; ALIGNED-NEXT:    global_store_byte v[16:17], v28, off offset:72
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v22, off offset:70
+; ALIGNED-NEXT:    global_store_byte v[16:17], v22, off offset:68
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v19, off offset:66
+; ALIGNED-NEXT:    global_store_byte v[16:17], v19, off offset:64
+; ALIGNED-NEXT:    buffer_store_dword v112, off, s[0:3], s32 offset:256
+; ALIGNED-NEXT:    buffer_store_dword v113, off, s[0:3], s32 offset:260
+; ALIGNED-NEXT:    buffer_store_dword v114, off, s[0:3], s32 offset:264
+; ALIGNED-NEXT:    buffer_store_dword v115, off, s[0:3], s32 offset:268
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:268
+; ALIGNED-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:264
+; ALIGNED-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:260
+; ALIGNED-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:256
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 8, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 8, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v51
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 8, v51
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v50
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 8, v50
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v29, off offset:62
+; ALIGNED-NEXT:    global_store_byte v[16:17], v29, off offset:60
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v23, off offset:58
+; ALIGNED-NEXT:    global_store_byte v[16:17], v23, off offset:56
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v20, off offset:54
+; ALIGNED-NEXT:    global_store_byte v[16:17], v20, off offset:52
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v18, off offset:50
+; ALIGNED-NEXT:    global_store_byte v[16:17], v18, off offset:48
+; ALIGNED-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:272
+; ALIGNED-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:276
+; ALIGNED-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:280
+; ALIGNED-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:284
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:280
+; ALIGNED-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:284
+; ALIGNED-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:272
+; ALIGNED-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:276
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v15, off offset:42
+; ALIGNED-NEXT:    global_store_byte v[16:17], v15, off offset:40
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v14, off offset:46
+; ALIGNED-NEXT:    global_store_byte v[16:17], v14, off offset:44
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v13, off offset:34
+; ALIGNED-NEXT:    global_store_byte v[16:17], v13, off offset:32
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v12, off offset:38
+; ALIGNED-NEXT:    global_store_byte v[16:17], v12, off offset:36
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v11, off offset:30
+; ALIGNED-NEXT:    global_store_byte v[16:17], v11, off offset:28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v10, off offset:26
+; ALIGNED-NEXT:    global_store_byte v[16:17], v10, off offset:24
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v9, off offset:22
+; ALIGNED-NEXT:    global_store_byte v[16:17], v9, off offset:20
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v8, off offset:18
+; ALIGNED-NEXT:    global_store_byte v[16:17], v8, off offset:16
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT:    global_store_byte v[16:17], v112, off offset:247
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v65
+; ALIGNED-NEXT:    global_store_byte v[16:17], v102, off offset:255
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v49
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 8, v49
+; ALIGNED-NEXT:    global_store_byte v[16:17], v31, off offset:253
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 24, v36
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 8, v36
+; ALIGNED-NEXT:    global_store_byte v[16:17], v103, off offset:251
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v71
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 8, v71
+; ALIGNED-NEXT:    global_store_byte v[16:17], v30, off offset:249
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v70
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 8, v70
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 8, v65
+; ALIGNED-NEXT:    global_store_byte v[16:17], v25, off offset:245
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 24, v64
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 8, v64
+; ALIGNED-NEXT:    global_store_byte v[16:17], v112, off offset:215
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v67
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 8, v67
+; ALIGNED-NEXT:    global_store_byte v[16:17], v113, off offset:243
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v87
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v87
+; ALIGNED-NEXT:    global_store_byte v[16:17], v24, off offset:241
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 24, v86
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 8, v86
+; ALIGNED-NEXT:    global_store_byte v[16:17], v114, off offset:239
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v85
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v85
+; ALIGNED-NEXT:    global_store_byte v[16:17], v51, off offset:237
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 24, v80
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 8, v80
+; ALIGNED-NEXT:    global_store_byte v[16:17], v115, off offset:235
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v101
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 8, v101
+; ALIGNED-NEXT:    global_store_byte v[16:17], v50, off offset:233
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 24, v99
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 8, v99
+; ALIGNED-NEXT:    global_store_byte v[16:17], v102, off offset:231
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v96
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v96, 8, v96
+; ALIGNED-NEXT:    global_store_byte v[16:17], v49, off offset:229
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 24, v81
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 8, v81
+; ALIGNED-NEXT:    global_store_byte v[16:17], v31, off offset:227
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 24, v100
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v100
+; ALIGNED-NEXT:    global_store_byte v[16:17], v36, off offset:225
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 24, v97
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v97, 8, v97
+; ALIGNED-NEXT:    global_store_byte v[16:17], v103, off offset:223
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v82
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 8, v82
+; ALIGNED-NEXT:    global_store_byte v[16:17], v71, off offset:221
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 24, v66
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 8, v66
+; ALIGNED-NEXT:    global_store_byte v[16:17], v30, off offset:219
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v98
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v98
+; ALIGNED-NEXT:    global_store_byte v[16:17], v70, off offset:217
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 24, v83
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 8, v83
+; ALIGNED-NEXT:    global_store_byte v[16:17], v65, off offset:213
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 24, v52
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v52, 8, v52
+; ALIGNED-NEXT:    global_store_byte v[16:17], v25, off offset:211
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 24, v84
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 8, v84
+; ALIGNED-NEXT:    global_store_byte v[16:17], v64, off offset:209
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 24, v68
+; ALIGNED-NEXT:    global_store_byte v[16:17], v67, off offset:149
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 24, v8
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v8, 8, v8
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v68, 8, v68
+; ALIGNED-NEXT:    global_store_byte v[16:17], v113, off offset:207
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v53
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v53, 8, v53
+; ALIGNED-NEXT:    global_store_byte v[16:17], v87, off offset:205
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 24, v37
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v37, 8, v37
+; ALIGNED-NEXT:    global_store_byte v[16:17], v24, off offset:203
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 24, v69
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v69, 8, v69
+; ALIGNED-NEXT:    global_store_byte v[16:17], v86, off offset:201
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v54
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v54, 8, v54
+; ALIGNED-NEXT:    global_store_byte v[16:17], v114, off offset:199
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v38
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v38, 8, v38
+; ALIGNED-NEXT:    global_store_byte v[16:17], v85, off offset:197
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 24, v32
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v32, 8, v32
+; ALIGNED-NEXT:    global_store_byte v[16:17], v51, off offset:195
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 24, v55
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v55, 8, v55
+; ALIGNED-NEXT:    global_store_byte v[16:17], v80, off offset:193
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 24, v39
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v39, 8, v39
+; ALIGNED-NEXT:    global_store_byte v[16:17], v115, off offset:191
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v33
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v33, 8, v33
+; ALIGNED-NEXT:    global_store_byte v[16:17], v101, off offset:189
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 24, v26
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v26, 8, v26
+; ALIGNED-NEXT:    global_store_byte v[16:17], v50, off offset:187
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 24, v48
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v48, 8, v48
+; ALIGNED-NEXT:    global_store_byte v[16:17], v99, off offset:185
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 24, v34
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v34, 8, v34
+; ALIGNED-NEXT:    global_store_byte v[16:17], v102, off offset:183
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v27
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v27, 8, v27
+; ALIGNED-NEXT:    global_store_byte v[16:17], v96, off offset:181
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v96, 24, v21
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v21, 8, v21
+; ALIGNED-NEXT:    global_store_byte v[16:17], v49, off offset:179
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 24, v35
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v35, 8, v35
+; ALIGNED-NEXT:    global_store_byte v[16:17], v81, off offset:177
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 24, v28
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v28, 8, v28
+; ALIGNED-NEXT:    global_store_byte v[16:17], v31, off offset:175
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 24, v22
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v22, 8, v22
+; ALIGNED-NEXT:    global_store_byte v[16:17], v100, off offset:173
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 24, v19
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v19, 8, v19
+; ALIGNED-NEXT:    global_store_byte v[16:17], v36, off offset:171
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 24, v29
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v29, 8, v29
+; ALIGNED-NEXT:    global_store_byte v[16:17], v97, off offset:169
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v97, 24, v23
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v23, 8, v23
+; ALIGNED-NEXT:    global_store_byte v[16:17], v103, off offset:167
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v20
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v20, 8, v20
+; ALIGNED-NEXT:    global_store_byte v[16:17], v82, off offset:165
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 24, v18
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v18, 8, v18
+; ALIGNED-NEXT:    global_store_byte v[16:17], v71, off offset:163
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 24, v15
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v15, 8, v15
+; ALIGNED-NEXT:    global_store_byte v[16:17], v66, off offset:161
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 24, v14
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v14, 8, v14
+; ALIGNED-NEXT:    global_store_byte v[16:17], v30, off offset:159
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v13
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v13, 8, v13
+; ALIGNED-NEXT:    global_store_byte v[16:17], v98, off offset:157
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 24, v12
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v12, 8, v12
+; ALIGNED-NEXT:    global_store_byte v[16:17], v70, off offset:155
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 24, v11
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v11, 8, v11
+; ALIGNED-NEXT:    global_store_byte v[16:17], v83, off offset:153
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 24, v10
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v10, 8, v10
+; ALIGNED-NEXT:    global_store_byte v[16:17], v112, off offset:151
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v9
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v9, 8, v9
+; ALIGNED-NEXT:    global_store_byte v[16:17], v65, off offset:147
+; ALIGNED-NEXT:    global_store_byte v[16:17], v52, off offset:145
+; ALIGNED-NEXT:    global_store_byte v[16:17], v25, off offset:143
+; ALIGNED-NEXT:    global_store_byte v[16:17], v84, off offset:141
+; ALIGNED-NEXT:    global_store_byte v[16:17], v64, off offset:139
+; ALIGNED-NEXT:    global_store_byte v[16:17], v68, off offset:137
+; ALIGNED-NEXT:    global_store_byte v[16:17], v113, off offset:135
+; ALIGNED-NEXT:    global_store_byte v[16:17], v53, off offset:133
+; ALIGNED-NEXT:    global_store_byte v[16:17], v87, off offset:131
+; ALIGNED-NEXT:    global_store_byte v[16:17], v37, off offset:129
+; ALIGNED-NEXT:    global_store_byte v[16:17], v24, off offset:127
+; ALIGNED-NEXT:    global_store_byte v[16:17], v69, off offset:125
+; ALIGNED-NEXT:    global_store_byte v[16:17], v86, off offset:123
+; ALIGNED-NEXT:    global_store_byte v[16:17], v54, off offset:121
+; ALIGNED-NEXT:    global_store_byte v[16:17], v114, off offset:119
+; ALIGNED-NEXT:    global_store_byte v[16:17], v38, off offset:117
+; ALIGNED-NEXT:    global_store_byte v[16:17], v85, off offset:115
+; ALIGNED-NEXT:    global_store_byte v[16:17], v32, off offset:113
+; ALIGNED-NEXT:    global_store_byte v[16:17], v51, off offset:111
+; ALIGNED-NEXT:    global_store_byte v[16:17], v55, off offset:109
+; ALIGNED-NEXT:    global_store_byte v[16:17], v80, off offset:107
+; ALIGNED-NEXT:    global_store_byte v[16:17], v39, off offset:105
+; ALIGNED-NEXT:    global_store_byte v[16:17], v115, off offset:103
+; ALIGNED-NEXT:    global_store_byte v[16:17], v33, off offset:101
+; ALIGNED-NEXT:    global_store_byte v[16:17], v101, off offset:99
+; ALIGNED-NEXT:    global_store_byte v[16:17], v26, off offset:97
+; ALIGNED-NEXT:    global_store_byte v[16:17], v50, off offset:95
+; ALIGNED-NEXT:    global_store_byte v[16:17], v48, off offset:93
+; ALIGNED-NEXT:    global_store_byte v[16:17], v99, off offset:91
+; ALIGNED-NEXT:    global_store_byte v[16:17], v34, off offset:89
+; ALIGNED-NEXT:    global_store_byte v[16:17], v102, off offset:87
+; ALIGNED-NEXT:    global_store_byte v[16:17], v27, off offset:85
+; ALIGNED-NEXT:    global_store_byte v[16:17], v96, off offset:83
+; ALIGNED-NEXT:    global_store_byte v[16:17], v21, off offset:81
+; ALIGNED-NEXT:    global_store_byte v[16:17], v49, off offset:79
+; ALIGNED-NEXT:    global_store_byte v[16:17], v35, off offset:77
+; ALIGNED-NEXT:    global_store_byte v[16:17], v81, off offset:75
+; ALIGNED-NEXT:    global_store_byte v[16:17], v28, off offset:73
+; ALIGNED-NEXT:    global_store_byte v[16:17], v31, off offset:71
+; ALIGNED-NEXT:    global_store_byte v[16:17], v22, off offset:69
+; ALIGNED-NEXT:    global_store_byte v[16:17], v100, off offset:67
+; ALIGNED-NEXT:    global_store_byte v[16:17], v19, off offset:65
+; ALIGNED-NEXT:    global_store_byte v[16:17], v36, off offset:63
+; ALIGNED-NEXT:    global_store_byte v[16:17], v29, off offset:61
+; ALIGNED-NEXT:    global_store_byte v[16:17], v97, off offset:59
+; ALIGNED-NEXT:    global_store_byte v[16:17], v23, off offset:57
+; ALIGNED-NEXT:    global_store_byte v[16:17], v103, off offset:55
+; ALIGNED-NEXT:    global_store_byte v[16:17], v20, off offset:53
+; ALIGNED-NEXT:    global_store_byte v[16:17], v82, off offset:51
+; ALIGNED-NEXT:    global_store_byte v[16:17], v18, off offset:49
+; ALIGNED-NEXT:    global_store_byte v[16:17], v71, off offset:43
+; ALIGNED-NEXT:    global_store_byte v[16:17], v15, off offset:41
+; ALIGNED-NEXT:    global_store_byte v[16:17], v66, off offset:47
+; ALIGNED-NEXT:    global_store_byte v[16:17], v14, off offset:45
+; ALIGNED-NEXT:    global_store_byte v[16:17], v30, off offset:35
+; ALIGNED-NEXT:    global_store_byte v[16:17], v13, off offset:33
+; ALIGNED-NEXT:    global_store_byte v[16:17], v98, off offset:39
+; ALIGNED-NEXT:    global_store_byte v[16:17], v12, off offset:37
+; ALIGNED-NEXT:    global_store_byte v[16:17], v70, off offset:31
+; ALIGNED-NEXT:    global_store_byte v[16:17], v11, off offset:29
+; ALIGNED-NEXT:    global_store_byte v[16:17], v83, off offset:27
+; ALIGNED-NEXT:    global_store_byte v[16:17], v10, off offset:25
+; ALIGNED-NEXT:    global_store_byte v[16:17], v112, off offset:23
+; ALIGNED-NEXT:    global_store_byte v[16:17], v9, off offset:21
+; ALIGNED-NEXT:    global_store_byte v[16:17], v67, off offset:19
+; ALIGNED-NEXT:    global_store_byte v[16:17], v8, off offset:17
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v7, off offset:14
+; ALIGNED-NEXT:    global_store_byte v[16:17], v7, off offset:12
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v6, off offset:10
+; ALIGNED-NEXT:    global_store_byte v[16:17], v6, off offset:8
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v5, off offset:6
+; ALIGNED-NEXT:    global_store_byte v[16:17], v5, off offset:4
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v4, off offset:2
+; ALIGNED-NEXT:    global_store_byte v[16:17], v4, off
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v8, 24, v7
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v7, 8, v7
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v9, 24, v6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v6, 8, v6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v10, 24, v5
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v5, 8, v5
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v11, 24, v4
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v4, 8, v4
+; ALIGNED-NEXT:    global_store_byte v[16:17], v8, off offset:15
+; ALIGNED-NEXT:    global_store_byte v[16:17], v7, off offset:13
+; ALIGNED-NEXT:    global_store_byte v[16:17], v9, off offset:11
+; ALIGNED-NEXT:    global_store_byte v[16:17], v6, off offset:9
+; ALIGNED-NEXT:    global_store_byte v[16:17], v10, off offset:7
+; ALIGNED-NEXT:    global_store_byte v[16:17], v5, off offset:5
+; ALIGNED-NEXT:    global_store_byte v[16:17], v11, off offset:3
+; ALIGNED-NEXT:    global_store_byte v[16:17], v4, off offset:1
+; ALIGNED-NEXT:    s_cbranch_vccnz .LBB1_1
+; ALIGNED-NEXT:  ; %bb.2: ; %memcpy-split
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_dword v47, off, s[0:3], s32
+; ALIGNED-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4
+; ALIGNED-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8
+; ALIGNED-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12
+; ALIGNED-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16
+; ALIGNED-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20
+; ALIGNED-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24
+; ALIGNED-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; UNROLL3-LABEL: memcpy_p1_p1_sz2048:
+; UNROLL3:       ; %bb.0: ; %entry
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    s_mov_b64 s[4:5], 0
+; UNROLL3-NEXT:    .p2align 6
+; UNROLL3-NEXT:  .LBB1_1: ; %load-store-loop
+; UNROLL3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT:    v_add_co_u32 v12, vcc_lo, v2, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v13, vcc_lo, s5, v3, vcc_lo
+; UNROLL3-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; UNROLL3-NEXT:    s_add_u32 s4, s4, 48
+; UNROLL3-NEXT:    s_clause 0x2
+; UNROLL3-NEXT:    global_load_dwordx4 v[4:7], v[12:13], off
+; UNROLL3-NEXT:    global_load_dwordx4 v[8:11], v[12:13], off offset:16
+; UNROLL3-NEXT:    global_load_dwordx4 v[12:15], v[12:13], off offset:32
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; UNROLL3-NEXT:    s_addc_u32 s5, s5, 0
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2)
+; UNROLL3-NEXT:    global_store_dwordx4 v[16:17], v[4:7], off
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    global_store_dwordx4 v[16:17], v[8:11], off offset:16
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    global_store_dwordx4 v[16:17], v[12:15], off offset:32
+; UNROLL3-NEXT:    v_cmp_gt_u64_e64 s6, 0x7e0, s[4:5]
+; UNROLL3-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
+; UNROLL3-NEXT:    s_cbranch_vccnz .LBB1_1
+; UNROLL3-NEXT:  ; %bb.2: ; %memcpy-split
+; UNROLL3-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:2016
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:2016
+; UNROLL3-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off offset:2032
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:2032
+; UNROLL3-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 2048, i1 false)
+  ret void
+}
+
+define void @memcpy_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4) align 1 readonly %src) {
+; CHECK-LABEL: memcpy_p0_p4_sz2048:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0
+; CHECK-NEXT:  .LBB2_1: ; %load-store-loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    v_add_co_u32 v96, vcc_lo, v2, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v97, vcc_lo, s5, v3, vcc_lo
+; CHECK-NEXT:    v_add_co_u32 v100, vcc_lo, v0, s4
+; CHECK-NEXT:    s_add_u32 s4, s4, 0x100
+; CHECK-NEXT:    s_clause 0xf
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[96:97], off offset:240
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[96:97], off offset:224
+; CHECK-NEXT:    global_load_dwordx4 v[12:15], v[96:97], off offset:208
+; CHECK-NEXT:    global_load_dwordx4 v[16:19], v[96:97], off offset:192
+; CHECK-NEXT:    global_load_dwordx4 v[20:23], v[96:97], off offset:176
+; CHECK-NEXT:    global_load_dwordx4 v[24:27], v[96:97], off offset:160
+; CHECK-NEXT:    global_load_dwordx4 v[28:31], v[96:97], off offset:144
+; CHECK-NEXT:    global_load_dwordx4 v[32:35], v[96:97], off offset:128
+; CHECK-NEXT:    global_load_dwordx4 v[36:39], v[96:97], off offset:112
+; CHECK-NEXT:    global_load_dwordx4 v[48:51], v[96:97], off offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[52:55], v[96:97], off offset:80
+; CHECK-NEXT:    global_load_dwordx4 v[64:67], v[96:97], off offset:64
+; CHECK-NEXT:    global_load_dwordx4 v[68:71], v[96:97], off offset:48
+; CHECK-NEXT:    global_load_dwordx4 v[80:83], v[96:97], off offset:32
+; CHECK-NEXT:    global_load_dwordx4 v[84:87], v[96:97], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[96:99], v[96:97], off
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v101, vcc_lo, s5, v1, vcc_lo
+; CHECK-NEXT:    s_addc_u32 s5, s5, 0
+; CHECK-NEXT:    s_waitcnt vmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[4:7] offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(14)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[8:11] offset:224
+; CHECK-NEXT:    s_waitcnt vmcnt(13)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[12:15] offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(12)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[16:19] offset:192
+; CHECK-NEXT:    s_waitcnt vmcnt(11)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[20:23] offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(10)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[24:27] offset:160
+; CHECK-NEXT:    s_waitcnt vmcnt(9)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[28:31] offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(8)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[32:35] offset:128
+; CHECK-NEXT:    s_waitcnt vmcnt(7)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[36:39] offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(6)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[48:51] offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[52:55] offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[64:67] offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[68:71] offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(2)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[80:83] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[84:87] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[96:99]
+; CHECK-NEXT:    v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
+; CHECK-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
+; CHECK-NEXT:    s_cbranch_vccnz .LBB2_1
+; CHECK-NEXT:  ; %bb.2: ; %memcpy-split
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+;
+; ALIGNED-LABEL: memcpy_p0_p4_sz2048:
+; ALIGNED:       ; %bb.0: ; %entry
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; ALIGNED-NEXT:    s_mov_b64 s[4:5], 0
+; ALIGNED-NEXT:  .LBB2_1: ; %load-store-loop
+; ALIGNED-NEXT:    ; =>This Inner Loop Header: Depth=1
+; ALIGNED-NEXT:    v_add_co_u32 v4, vcc_lo, v2, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, s5, v3, vcc_lo
+; ALIGNED-NEXT:    v_add_co_u32 v96, vcc_lo, v0, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v97, vcc_lo, s5, v1, vcc_lo
+; ALIGNED-NEXT:    s_clause 0xf
+; ALIGNED-NEXT:    global_load_dwordx4 v[98:101], v[4:5], off offset:240
+; ALIGNED-NEXT:    global_load_dwordx4 v[84:87], v[4:5], off offset:224
+; ALIGNED-NEXT:    global_load_dwordx4 v[80:83], v[4:5], off offset:208
+; ALIGNED-NEXT:    global_load_dwordx4 v[68:71], v[4:5], off offset:192
+; ALIGNED-NEXT:    global_load_dwordx4 v[64:67], v[4:5], off offset:176
+; ALIGNED-NEXT:    global_load_dwordx4 v[52:55], v[4:5], off offset:160
+; ALIGNED-NEXT:    global_load_dwordx4 v[48:51], v[4:5], off offset:144
+; ALIGNED-NEXT:    global_load_dwordx4 v[36:39], v[4:5], off offset:128
+; ALIGNED-NEXT:    global_load_dwordx4 v[32:35], v[4:5], off offset:112
+; ALIGNED-NEXT:    global_load_dwordx4 v[28:31], v[4:5], off offset:96
+; ALIGNED-NEXT:    global_load_dwordx4 v[24:27], v[4:5], off offset:80
+; ALIGNED-NEXT:    global_load_dwordx4 v[20:23], v[4:5], off offset:64
+; ALIGNED-NEXT:    global_load_dwordx4 v[16:19], v[4:5], off offset:48
+; ALIGNED-NEXT:    global_load_dwordx4 v[12:15], v[4:5], off offset:32
+; ALIGNED-NEXT:    global_load_dwordx4 v[8:11], v[4:5], off offset:16
+; ALIGNED-NEXT:    global_load_dwordx4 v[4:7], v[4:5], off
+; ALIGNED-NEXT:    s_add_u32 s4, s4, 0x100
+; ALIGNED-NEXT:    s_addc_u32 s5, s5, 0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(15)
+; ALIGNED-NEXT:    buffer_store_dword v100, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT:    buffer_store_dword v101, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT:    buffer_store_dword v99, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT:    buffer_store_dword v98, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v100 offset:250
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v101 offset:254
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:252
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:248
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v99 offset:246
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:244
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v98 offset:242
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:240
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v100
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v100
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v101
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 8, v101
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 24, v99
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v99
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 24, v98
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v98
+; ALIGNED-NEXT:    s_waitcnt vmcnt(14)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v86
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 8, v86
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:251
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v87
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:249
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v87
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v103 offset:255
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v85
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v102 offset:253
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 8, v85
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:247
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 24, v84
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:245
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v84
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:243
+; ALIGNED-NEXT:    s_waitcnt vmcnt(13)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 24, v82
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:241
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v82
+; ALIGNED-NEXT:    buffer_store_dword v86, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT:    buffer_store_dword v87, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT:    buffer_store_dword v85, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT:    buffer_store_dword v84, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v86 offset:234
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v87 offset:238
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v87 offset:236
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v86 offset:232
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v85 offset:230
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v85 offset:228
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v84 offset:226
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v84 offset:224
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 24, v83
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v83
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v81
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v81
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:235
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v80
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:233
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 8, v80
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:239
+; ALIGNED-NEXT:    s_waitcnt vmcnt(12)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v70
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:237
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v70
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v103 offset:231
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v71
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v102 offset:229
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 8, v71
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:227
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 24, v69
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:225
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v69
+; ALIGNED-NEXT:    buffer_store_dword v82, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT:    buffer_store_dword v83, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT:    buffer_store_dword v81, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT:    buffer_store_dword v80, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v82 offset:218
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v83 offset:222
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v83 offset:220
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v82 offset:216
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v81 offset:214
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v81 offset:212
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v80 offset:210
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v80 offset:208
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 24, v68
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 8, v68
+; ALIGNED-NEXT:    s_waitcnt vmcnt(11)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 24, v66
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 8, v66
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:219
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 24, v67
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:217
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v67
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v84 offset:223
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 24, v65
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v85 offset:221
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v65
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v86 offset:215
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v64
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v87 offset:213
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v64
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:211
+; ALIGNED-NEXT:    s_waitcnt vmcnt(10)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v54
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:209
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 8, v54
+; ALIGNED-NEXT:    buffer_store_dword v70, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT:    buffer_store_dword v71, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT:    buffer_store_dword v69, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT:    buffer_store_dword v68, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v70 offset:202
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v71 offset:206
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v71 offset:204
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v70 offset:200
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v69 offset:198
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v69 offset:196
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v68 offset:194
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v68 offset:192
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v69, 8, v55
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 24, v52
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 8, v52
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v68, 24, v55
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:203
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v53
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:201
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v53
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v103 offset:207
+; ALIGNED-NEXT:    s_waitcnt vmcnt(9)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v50
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v102 offset:205
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 8, v50
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:199
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 24, v51
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:197
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v51
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v80 offset:195
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 24, v49
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v81 offset:193
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 8, v49
+; ALIGNED-NEXT:    buffer_store_dword v66, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT:    buffer_store_dword v67, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT:    buffer_store_dword v65, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT:    buffer_store_dword v64, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v66 offset:186
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v67 offset:190
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v67 offset:188
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v66 offset:184
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v65 offset:182
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v65 offset:180
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v64 offset:178
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v64 offset:176
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 24, v48
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 8, v48
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v82 offset:187
+; ALIGNED-NEXT:    s_waitcnt vmcnt(8)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 24, v39
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 24, v38
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 8, v38
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v83 offset:185
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 8, v39
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:191
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 24, v37
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:189
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v37
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v84 offset:183
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 24, v36
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v85 offset:181
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v36
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v86 offset:179
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v34
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v87 offset:177
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v34
+; ALIGNED-NEXT:    buffer_store_dword v55, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT:    buffer_store_dword v54, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT:    buffer_store_dword v53, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT:    buffer_store_dword v52, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v54 offset:170
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v54 offset:168
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v55 offset:174
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v55 offset:172
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v52 offset:162
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v52 offset:160
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v53 offset:166
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v53 offset:164
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v53, 24, v35
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v54, 8, v35
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v55, 24, v33
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v116, 8, v33
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:171
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v32
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:169
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 8, v32
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v69 offset:173
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v69, 24, v31
+; ALIGNED-NEXT:    v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v70 offset:163
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 8, v31
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v71 offset:161
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 24, v29
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v68 offset:175
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v52, 24, v30
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v68, 8, v30
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:167
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:165
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v29
+; ALIGNED-NEXT:    buffer_store_dword v50, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT:    buffer_store_dword v51, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT:    buffer_store_dword v49, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT:    buffer_store_dword v48, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v50 offset:154
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v51 offset:158
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v51 offset:156
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v50 offset:152
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v49 offset:150
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v49 offset:148
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v48 offset:146
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v48 offset:144
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v48, 24, v28
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 8, v28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 24, v26
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v103 offset:155
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v102 offset:153
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:159
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:157
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v80 offset:151
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v81 offset:149
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v64 offset:147
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v65 offset:145
+; ALIGNED-NEXT:    buffer_store_dword v38, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT:    buffer_store_dword v39, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT:    buffer_store_dword v37, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT:    buffer_store_dword v36, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v38 offset:138
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v39 offset:142
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v39 offset:140
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v38 offset:136
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v37 offset:134
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v37 offset:132
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v36 offset:130
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v36 offset:128
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v82 offset:143
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 24, v18
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 8, v26
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v66 offset:139
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v67 offset:137
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v83 offset:141
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:135
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:133
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v84 offset:131
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v85 offset:129
+; ALIGNED-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT:    buffer_store_dword v35, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v34 offset:122
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v35 offset:126
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v35 offset:124
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v34 offset:120
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v33 offset:118
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v33 offset:116
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v32 offset:114
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v32 offset:112
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v34, 24, v14
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v86 offset:123
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v87 offset:121
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v53 offset:127
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v54 offset:125
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v55 offset:119
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v116 offset:117
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:115
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v10
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:113
+; ALIGNED-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v30 offset:106
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v31 offset:110
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v31 offset:108
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v30 offset:104
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v29 offset:102
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v29 offset:100
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v28 offset:98
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v28 offset:96
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v69 offset:111
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v69, 24, v6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v27
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 8, v18
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v35, 8, v14
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 8, v10
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v70 offset:109
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 8, v6
+; ALIGNED-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 8, v27
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 24, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 24, v19
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v15
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v28, 24, v11
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v71 offset:103
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 24, v7
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 24, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 8, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 24, v22
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 8, v22
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 24, v23
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v37, 8, v23
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v38, 24, v21
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v39, 8, v21
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 24, v20
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 8, v20
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v19
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 24, v17
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v17
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v32, 24, v16
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v33, 8, v16
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v15
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v53, 24, v13
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v54, 8, v13
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v55, 24, v12
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 8, v12
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v29, 8, v11
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v9
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 8, v9
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v52 offset:107
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v52, 24, v8
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v68 offset:105
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v68, 8, v8
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:101
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v7
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v48 offset:99
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v48, 24, v5
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v49 offset:97
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 8, v5
+; ALIGNED-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:8
+; ALIGNED-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:12
+; ALIGNED-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:4
+; ALIGNED-NEXT:    buffer_store_dword v24, off, s[0:3], s32
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v26 offset:90
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v27 offset:94
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v27 offset:92
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v26 offset:88
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v25 offset:86
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v25 offset:84
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v24 offset:82
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v24 offset:80
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 24, v4
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 8, v4
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v50 offset:91
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v51 offset:89
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v103 offset:95
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v102 offset:93
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:87
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:85
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v80 offset:83
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v81 offset:81
+; ALIGNED-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:24
+; ALIGNED-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:28
+; ALIGNED-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:20
+; ALIGNED-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:16
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v22 offset:74
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v23 offset:78
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v23 offset:76
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v22 offset:72
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v21 offset:70
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v21 offset:68
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v20 offset:66
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v20 offset:64
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v64 offset:75
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v65 offset:73
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v36 offset:79
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v37 offset:77
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v38 offset:71
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v39 offset:69
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v66 offset:67
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v67 offset:65
+; ALIGNED-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v82 offset:59
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v18 offset:58
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v83 offset:57
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v19 offset:62
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:63
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v19 offset:60
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:61
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v18 offset:56
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v84 offset:55
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v17 offset:54
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v85 offset:53
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v17 offset:52
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v32 offset:51
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v16 offset:50
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v33 offset:49
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v16 offset:48
+; ALIGNED-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v14 offset:42
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v34 offset:43
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v35 offset:41
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v15 offset:46
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v86 offset:47
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v15 offset:44
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v87 offset:45
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v14 offset:40
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v53 offset:39
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v13 offset:38
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v54 offset:37
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v13 offset:36
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v55 offset:35
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v12 offset:34
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:33
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v12 offset:32
+; ALIGNED-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v10 offset:26
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:27
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:25
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v11 offset:30
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v28 offset:31
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v11 offset:28
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v29 offset:29
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v10 offset:24
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v30 offset:23
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v9 offset:22
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v31 offset:21
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v9 offset:20
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v52 offset:19
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v8 offset:18
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v68 offset:17
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v8 offset:16
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v6 offset:10
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v69 offset:11
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v70 offset:9
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v7 offset:14
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v71 offset:15
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v7 offset:12
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:13
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v6 offset:8
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v48 offset:7
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v5 offset:6
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v49 offset:5
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v5 offset:4
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v24 offset:3
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v4 offset:2
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v25 offset:1
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v4
+; ALIGNED-NEXT:    s_cbranch_vccnz .LBB2_1
+; ALIGNED-NEXT:  ; %bb.2: ; %memcpy-split
+; ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; UNROLL3-LABEL: memcpy_p0_p4_sz2048:
+; UNROLL3:       ; %bb.0: ; %entry
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    s_mov_b64 s[4:5], 0
+; UNROLL3-NEXT:    .p2align 6
+; UNROLL3-NEXT:  .LBB2_1: ; %load-store-loop
+; UNROLL3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT:    v_add_co_u32 v12, vcc_lo, v2, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v13, vcc_lo, s5, v3, vcc_lo
+; UNROLL3-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; UNROLL3-NEXT:    s_add_u32 s4, s4, 48
+; UNROLL3-NEXT:    s_clause 0x2
+; UNROLL3-NEXT:    global_load_dwordx4 v[4:7], v[12:13], off offset:16
+; UNROLL3-NEXT:    global_load_dwordx4 v[8:11], v[12:13], off
+; UNROLL3-NEXT:    global_load_dwordx4 v[12:15], v[12:13], off offset:32
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; UNROLL3-NEXT:    s_addc_u32 s5, s5, 0
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[4:7] offset:16
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[8:11]
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[12:15] offset:32
+; UNROLL3-NEXT:    v_cmp_gt_u64_e64 s6, 0x7e0, s[4:5]
+; UNROLL3-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
+; UNROLL3-NEXT:    s_cbranch_vccnz .LBB2_1
+; UNROLL3-NEXT:  ; %bb.2: ; %memcpy-split
+; UNROLL3-NEXT:    s_clause 0x1
+; UNROLL3-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:2016
+; UNROLL3-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off offset:2032
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:2016
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[8:11] offset:2032
+; UNROLL3-NEXT:    s_waitcnt lgkmcnt(0)
+; UNROLL3-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  tail call void @llvm.memcpy.p0.p4.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 2048, i1 false)
+  ret void
+}
+
+define void @memcpy_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {
+; CHECK-LABEL: memcpy_p5_p5_sz2048:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0
+; CHECK-NEXT:  .LBB3_1: ; %load-store-loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_clause 0x3e
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:252
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:248
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:244
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:240
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:236
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:232
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:228
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:224
+; CHECK-NEXT:    buffer_load_dword v10, v1, s[0:3], 0 offen offset:220
+; CHECK-NEXT:    buffer_load_dword v11, v1, s[0:3], 0 offen offset:216
+; CHECK-NEXT:    buffer_load_dword v12, v1, s[0:3], 0 offen offset:212
+; CHECK-NEXT:    buffer_load_dword v13, v1, s[0:3], 0 offen offset:208
+; CHECK-NEXT:    buffer_load_dword v14, v1, s[0:3], 0 offen offset:204
+; CHECK-NEXT:    buffer_load_dword v15, v1, s[0:3], 0 offen offset:200
+; CHECK-NEXT:    buffer_load_dword v16, v1, s[0:3], 0 offen offset:196
+; CHECK-NEXT:    buffer_load_dword v17, v1, s[0:3], 0 offen offset:192
+; CHECK-NEXT:    buffer_load_dword v18, v1, s[0:3], 0 offen offset:188
+; CHECK-NEXT:    buffer_load_dword v19, v1, s[0:3], 0 offen offset:184
+; CHECK-NEXT:    buffer_load_dword v20, v1, s[0:3], 0 offen offset:180
+; CHECK-NEXT:    buffer_load_dword v21, v1, s[0:3], 0 offen offset:176
+; CHECK-NEXT:    buffer_load_dword v22, v1, s[0:3], 0 offen offset:172
+; CHECK-NEXT:    buffer_load_dword v23, v1, s[0:3], 0 offen offset:168
+; CHECK-NEXT:    buffer_load_dword v24, v1, s[0:3], 0 offen offset:164
+; CHECK-NEXT:    buffer_load_dword v25, v1, s[0:3], 0 offen offset:160
+; CHECK-NEXT:    buffer_load_dword v26, v1, s[0:3], 0 offen offset:156
+; CHECK-NEXT:    buffer_load_dword v27, v1, s[0:3], 0 offen offset:152
+; CHECK-NEXT:    buffer_load_dword v28, v1, s[0:3], 0 offen offset:148
+; CHECK-NEXT:    buffer_load_dword v29, v1, s[0:3], 0 offen offset:144
+; CHECK-NEXT:    buffer_load_dword v30, v1, s[0:3], 0 offen offset:140
+; CHECK-NEXT:    buffer_load_dword v31, v1, s[0:3], 0 offen offset:136
+; CHECK-NEXT:    buffer_load_dword v32, v1, s[0:3], 0 offen offset:132
+; CHECK-NEXT:    buffer_load_dword v33, v1, s[0:3], 0 offen offset:128
+; CHECK-NEXT:    buffer_load_dword v34, v1, s[0:3], 0 offen offset:124
+; CHECK-NEXT:    buffer_load_dword v35, v1, s[0:3], 0 offen offset:120
+; CHECK-NEXT:    buffer_load_dword v36, v1, s[0:3], 0 offen offset:116
+; CHECK-NEXT:    buffer_load_dword v37, v1, s[0:3], 0 offen offset:112
+; CHECK-NEXT:    buffer_load_dword v38, v1, s[0:3], 0 offen offset:108
+; CHECK-NEXT:    buffer_load_dword v39, v1, s[0:3], 0 offen offset:104
+; CHECK-NEXT:    buffer_load_dword v48, v1, s[0:3], 0 offen offset:100
+; CHECK-NEXT:    buffer_load_dword v49, v1, s[0:3], 0 offen offset:96
+; CHECK-NEXT:    buffer_load_dword v50, v1, s[0:3], 0 offen offset:92
+; CHECK-NEXT:    buffer_load_dword v51, v1, s[0:3], 0 offen offset:88
+; CHECK-NEXT:    buffer_load_dword v52, v1, s[0:3], 0 offen offset:84
+; CHECK-NEXT:    buffer_load_dword v53, v1, s[0:3], 0 offen offset:80
+; CHECK-NEXT:    buffer_load_dword v54, v1, s[0:3], 0 offen offset:76
+; CHECK-NEXT:    buffer_load_dword v55, v1, s[0:3], 0 offen offset:72
+; CHECK-NEXT:    buffer_load_dword v64, v1, s[0:3], 0 offen offset:68
+; CHECK-NEXT:    buffer_load_dword v65, v1, s[0:3], 0 offen offset:64
+; CHECK-NEXT:    buffer_load_dword v66, v1, s[0:3], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v67, v1, s[0:3], 0 offen offset:56
+; CHECK-NEXT:    buffer_load_dword v68, v1, s[0:3], 0 offen offset:52
+; CHECK-NEXT:    buffer_load_dword v69, v1, s[0:3], 0 offen offset:48
+; CHECK-NEXT:    buffer_load_dword v70, v1, s[0:3], 0 offen offset:44
+; CHECK-NEXT:    buffer_load_dword v71, v1, s[0:3], 0 offen offset:40
+; CHECK-NEXT:    buffer_load_dword v80, v1, s[0:3], 0 offen offset:36
+; CHECK-NEXT:    buffer_load_dword v81, v1, s[0:3], 0 offen offset:32
+; CHECK-NEXT:    buffer_load_dword v82, v1, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v83, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v84, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v85, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v86, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v87, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v96, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v97, v1, s[0:3], 0 offen
+; CHECK-NEXT:    s_add_u32 s4, s4, 0x100
+; CHECK-NEXT:    s_addc_u32 s5, s5, 0
+; CHECK-NEXT:    v_add_nc_u32_e32 v1, 0x100, v1
+; CHECK-NEXT:    v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
+; CHECK-NEXT:    s_waitcnt vmcnt(62)
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:252
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:248
+; CHECK-NEXT:    s_waitcnt vmcnt(61)
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:244
+; CHECK-NEXT:    s_waitcnt vmcnt(60)
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(59)
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:236
+; CHECK-NEXT:    s_waitcnt vmcnt(58)
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:232
+; CHECK-NEXT:    s_waitcnt vmcnt(57)
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:228
+; CHECK-NEXT:    s_waitcnt vmcnt(56)
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:224
+; CHECK-NEXT:    s_waitcnt vmcnt(55)
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:220
+; CHECK-NEXT:    s_waitcnt vmcnt(54)
+; CHECK-NEXT:    buffer_store_dword v11, v0, s[0:3], 0 offen offset:216
+; CHECK-NEXT:    s_waitcnt vmcnt(53)
+; CHECK-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen offset:212
+; CHECK-NEXT:    s_waitcnt vmcnt(52)
+; CHECK-NEXT:    buffer_store_dword v13, v0, s[0:3], 0 offen offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(51)
+; CHECK-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:204
+; CHECK-NEXT:    s_waitcnt vmcnt(50)
+; CHECK-NEXT:    buffer_store_dword v15, v0, s[0:3], 0 offen offset:200
+; CHECK-NEXT:    s_waitcnt vmcnt(49)
+; CHECK-NEXT:    buffer_store_dword v16, v0, s[0:3], 0 offen offset:196
+; CHECK-NEXT:    s_waitcnt vmcnt(48)
+; CHECK-NEXT:    buffer_store_dword v17, v0, s[0:3], 0 offen offset:192
+; CHECK-NEXT:    s_waitcnt vmcnt(47)
+; CHECK-NEXT:    buffer_store_dword v18, v0, s[0:3], 0 offen offset:188
+; CHECK-NEXT:    s_waitcnt vmcnt(46)
+; CHECK-NEXT:    buffer_store_dword v19, v0, s[0:3], 0 offen offset:184
+; CHECK-NEXT:    s_waitcnt vmcnt(45)
+; CHECK-NEXT:    buffer_store_dword v20, v0, s[0:3], 0 offen offset:180
+; CHECK-NEXT:    s_waitcnt vmcnt(44)
+; CHECK-NEXT:    buffer_store_dword v21, v0, s[0:3], 0 offen offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(43)
+; CHECK-NEXT:    buffer_store_dword v22, v0, s[0:3], 0 offen offset:172
+; CHECK-NEXT:    s_waitcnt vmcnt(42)
+; CHECK-NEXT:    buffer_store_dword v23, v0, s[0:3], 0 offen offset:168
+; CHECK-NEXT:    s_waitcnt vmcnt(41)
+; CHECK-NEXT:    buffer_store_dword v24, v0, s[0:3], 0 offen offset:164
+; CHECK-NEXT:    s_waitcnt vmcnt(40)
+; CHECK-NEXT:    buffer_store_dword v25, v0, s[0:3], 0 offen offset:160
+; CHECK-NEXT:    s_waitcnt vmcnt(39)
+; CHECK-NEXT:    buffer_store_dword v26, v0, s[0:3], 0 offen offset:156
+; CHECK-NEXT:    s_waitcnt vmcnt(38)
+; CHECK-NEXT:    buffer_store_dword v27, v0, s[0:3], 0 offen offset:152
+; CHECK-NEXT:    s_waitcnt vmcnt(37)
+; CHECK-NEXT:    buffer_store_dword v28, v0, s[0:3], 0 offen offset:148
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_dword v29, v0, s[0:3], 0 offen offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(35)
+; CHECK-NEXT:    buffer_store_dword v30, v0, s[0:3], 0 offen offset:140
+; CHECK-NEXT:    s_waitcnt vmcnt(34)
+; CHECK-NEXT:    buffer_store_dword v31, v0, s[0:3], 0 offen offset:136
+; CHECK-NEXT:    s_waitcnt vmcnt(33)
+; CHECK-NEXT:    buffer_store_dword v32, v0, s[0:3], 0 offen offset:132
+; CHECK-NEXT:    s_waitcnt vmcnt(32)
+; CHECK-NEXT:    buffer_store_dword v33, v0, s[0:3], 0 offen offset:128
+; CHECK-NEXT:    s_waitcnt vmcnt(31)
+; CHECK-NEXT:    buffer_store_dword v34, v0, s[0:3], 0 offen offset:124
+; CHECK-NEXT:    s_waitcnt vmcnt(30)
+; CHECK-NEXT:    buffer_store_dword v35, v0, s[0:3], 0 offen offset:120
+; CHECK-NEXT:    s_waitcnt vmcnt(29)
+; CHECK-NEXT:    buffer_store_dword v36, v0, s[0:3], 0 offen offset:116
+; CHECK-NEXT:    s_waitcnt vmcnt(28)
+; CHECK-NEXT:    buffer_store_dword v37, v0, s[0:3], 0 offen offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(27)
+; CHECK-NEXT:    buffer_store_dword v38, v0, s[0:3], 0 offen offset:108
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_dword v39, v0, s[0:3], 0 offen offset:104
+; CHECK-NEXT:    s_waitcnt vmcnt(25)
+; CHECK-NEXT:    buffer_store_dword v48, v0, s[0:3], 0 offen offset:100
+; CHECK-NEXT:    s_waitcnt vmcnt(24)
+; CHECK-NEXT:    buffer_store_dword v49, v0, s[0:3], 0 offen offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(23)
+; CHECK-NEXT:    buffer_store_dword v50, v0, s[0:3], 0 offen offset:92
+; CHECK-NEXT:    s_waitcnt vmcnt(22)
+; CHECK-NEXT:    buffer_store_dword v51, v0, s[0:3], 0 offen offset:88
+; CHECK-NEXT:    s_waitcnt vmcnt(21)
+; CHECK-NEXT:    buffer_store_dword v52, v0, s[0:3], 0 offen offset:84
+; CHECK-NEXT:    s_waitcnt vmcnt(20)
+; CHECK-NEXT:    buffer_store_dword v53, v0, s[0:3], 0 offen offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(19)
+; CHECK-NEXT:    buffer_store_dword v54, v0, s[0:3], 0 offen offset:76
+; CHECK-NEXT:    s_waitcnt vmcnt(18)
+; CHECK-NEXT:    buffer_store_dword v55, v0, s[0:3], 0 offen offset:72
+; CHECK-NEXT:    s_waitcnt vmcnt(17)
+; CHECK-NEXT:    buffer_store_dword v64, v0, s[0:3], 0 offen offset:68
+; CHECK-NEXT:    s_waitcnt vmcnt(16)
+; CHECK-NEXT:    buffer_store_dword v65, v0, s[0:3], 0 offen offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(15)
+; CHECK-NEXT:    buffer_store_dword v66, v0, s[0:3], 0 offen offset:60
+; CHECK-NEXT:    s_waitcnt vmcnt(14)
+; CHECK-NEXT:    buffer_store_dword v67, v0, s[0:3], 0 offen offset:56
+; CHECK-NEXT:    s_waitcnt vmcnt(13)
+; CHECK-NEXT:    buffer_store_dword v68, v0, s[0:3], 0 offen offset:52
+; CHECK-NEXT:    s_waitcnt vmcnt(12)
+; CHECK-NEXT:    buffer_store_dword v69, v0, s[0:3], 0 offen offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(11)
+; CHECK-NEXT:    buffer_store_dword v70, v0, s[0:3], 0 offen offset:44
+; CHECK-NEXT:    s_waitcnt vmcnt(10)
+; CHECK-NEXT:    buffer_store_dword v71, v0, s[0:3], 0 offen offset:40
+; CHECK-NEXT:    s_waitcnt vmcnt(9)
+; CHECK-NEXT:    buffer_store_dword v80, v0, s[0:3], 0 offen offset:36
+; CHECK-NEXT:    s_waitcnt vmcnt(8)
+; CHECK-NEXT:    buffer_store_dword v81, v0, s[0:3], 0 offen offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(7)
+; CHECK-NEXT:    buffer_store_dword v82, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(6)
+; CHECK-NEXT:    buffer_store_dword v83, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    buffer_store_dword v84, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    buffer_store_dword v85, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    buffer_store_dword v86, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    s_waitcnt vmcnt(2)
+; CHECK-NEXT:    buffer_store_dword v87, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    buffer_store_dword v96, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    buffer_store_dword v97, v0, s[0:3], 0 offen
+; CHECK-NEXT:    v_add_nc_u32_e32 v0, 0x100, v0
+; CHECK-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
+; CHECK-NEXT:    s_cbranch_vccnz .LBB3_1
+; CHECK-NEXT:  ; %bb.2: ; %memcpy-split
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+;
+; ALIGNED-LABEL: memcpy_p5_p5_sz2048:
+; ALIGNED:       ; %bb.0: ; %entry
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; ALIGNED-NEXT:    s_mov_b64 s[4:5], 0
+; ALIGNED-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v72, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v73, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v74, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v75, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v76, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v77, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v78, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v79, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v88, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v89, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v90, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v91, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v92, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v93, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v94, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v95, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v104, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v105, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v106, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v107, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v108, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v109, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v110, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v111, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v120, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v121, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v122, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v123, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v124, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v125, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v126, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v127, off, s[0:3], s32 ; 4-byte Folded Spill
+; ALIGNED-NEXT:  .LBB3_1: ; %load-store-loop
+; ALIGNED-NEXT:    ; =>This Inner Loop Header: Depth=1
+; ALIGNED-NEXT:    s_clause 0x34
+; ALIGNED-NEXT:    buffer_load_ubyte v116, v1, s[0:3], 0 offen offset:255
+; ALIGNED-NEXT:    buffer_load_ubyte v117, v1, s[0:3], 0 offen offset:254
+; ALIGNED-NEXT:    buffer_load_ubyte v118, v1, s[0:3], 0 offen offset:253
+; ALIGNED-NEXT:    buffer_load_ubyte v119, v1, s[0:3], 0 offen offset:252
+; ALIGNED-NEXT:    buffer_load_ubyte v40, v1, s[0:3], 0 offen offset:251
+; ALIGNED-NEXT:    buffer_load_ubyte v41, v1, s[0:3], 0 offen offset:250
+; ALIGNED-NEXT:    buffer_load_ubyte v42, v1, s[0:3], 0 offen offset:249
+; ALIGNED-NEXT:    buffer_load_ubyte v43, v1, s[0:3], 0 offen offset:248
+; ALIGNED-NEXT:    buffer_load_ubyte v44, v1, s[0:3], 0 offen offset:247
+; ALIGNED-NEXT:    buffer_load_ubyte v45, v1, s[0:3], 0 offen offset:246
+; ALIGNED-NEXT:    buffer_load_ubyte v46, v1, s[0:3], 0 offen offset:245
+; ALIGNED-NEXT:    buffer_load_ubyte v47, v1, s[0:3], 0 offen offset:244
+; ALIGNED-NEXT:    buffer_load_ubyte v56, v1, s[0:3], 0 offen offset:243
+; ALIGNED-NEXT:    buffer_load_ubyte v57, v1, s[0:3], 0 offen offset:242
+; ALIGNED-NEXT:    buffer_load_ubyte v58, v1, s[0:3], 0 offen offset:241
+; ALIGNED-NEXT:    buffer_load_ubyte v59, v1, s[0:3], 0 offen offset:240
+; ALIGNED-NEXT:    buffer_load_ubyte v60, v1, s[0:3], 0 offen offset:239
+; ALIGNED-NEXT:    buffer_load_ubyte v61, v1, s[0:3], 0 offen offset:238
+; ALIGNED-NEXT:    buffer_load_ubyte v62, v1, s[0:3], 0 offen offset:237
+; ALIGNED-NEXT:    buffer_load_ubyte v63, v1, s[0:3], 0 offen offset:236
+; ALIGNED-NEXT:    buffer_load_ubyte v72, v1, s[0:3], 0 offen offset:235
+; ALIGNED-NEXT:    buffer_load_ubyte v73, v1, s[0:3], 0 offen offset:234
+; ALIGNED-NEXT:    buffer_load_ubyte v74, v1, s[0:3], 0 offen offset:233
+; ALIGNED-NEXT:    buffer_load_ubyte v75, v1, s[0:3], 0 offen offset:232
+; ALIGNED-NEXT:    buffer_load_ubyte v76, v1, s[0:3], 0 offen offset:231
+; ALIGNED-NEXT:    buffer_load_ubyte v77, v1, s[0:3], 0 offen offset:230
+; ALIGNED-NEXT:    buffer_load_ubyte v78, v1, s[0:3], 0 offen offset:229
+; ALIGNED-NEXT:    buffer_load_ubyte v79, v1, s[0:3], 0 offen offset:228
+; ALIGNED-NEXT:    buffer_load_ubyte v88, v1, s[0:3], 0 offen offset:227
+; ALIGNED-NEXT:    buffer_load_ubyte v89, v1, s[0:3], 0 offen offset:226
+; ALIGNED-NEXT:    buffer_load_ubyte v90, v1, s[0:3], 0 offen offset:225
+; ALIGNED-NEXT:    buffer_load_ubyte v91, v1, s[0:3], 0 offen offset:224
+; ALIGNED-NEXT:    buffer_load_ubyte v92, v1, s[0:3], 0 offen offset:223
+; ALIGNED-NEXT:    buffer_load_ubyte v93, v1, s[0:3], 0 offen offset:222
+; ALIGNED-NEXT:    buffer_load_ubyte v94, v1, s[0:3], 0 offen offset:221
+; ALIGNED-NEXT:    buffer_load_ubyte v95, v1, s[0:3], 0 offen offset:220
+; ALIGNED-NEXT:    buffer_load_ubyte v104, v1, s[0:3], 0 offen offset:219
+; ALIGNED-NEXT:    buffer_load_ubyte v105, v1, s[0:3], 0 offen offset:218
+; ALIGNED-NEXT:    buffer_load_ubyte v106, v1, s[0:3], 0 offen offset:217
+; ALIGNED-NEXT:    buffer_load_ubyte v107, v1, s[0:3], 0 offen offset:216
+; ALIGNED-NEXT:    buffer_load_ubyte v108, v1, s[0:3], 0 offen offset:215
+; ALIGNED-NEXT:    buffer_load_ubyte v109, v1, s[0:3], 0 offen offset:214
+; ALIGNED-NEXT:    buffer_load_ubyte v110, v1, s[0:3], 0 offen offset:213
+; ALIGNED-NEXT:    buffer_load_ubyte v111, v1, s[0:3], 0 offen offset:212
+; ALIGNED-NEXT:    buffer_load_ubyte v120, v1, s[0:3], 0 offen offset:211
+; ALIGNED-NEXT:    buffer_load_ubyte v121, v1, s[0:3], 0 offen offset:210
+; ALIGNED-NEXT:    buffer_load_ubyte v122, v1, s[0:3], 0 offen offset:209
+; ALIGNED-NEXT:    buffer_load_ubyte v123, v1, s[0:3], 0 offen offset:208
+; ALIGNED-NEXT:    buffer_load_ubyte v124, v1, s[0:3], 0 offen offset:207
+; ALIGNED-NEXT:    buffer_load_ubyte v125, v1, s[0:3], 0 offen offset:206
+; ALIGNED-NEXT:    buffer_load_ubyte v126, v1, s[0:3], 0 offen offset:205
+; ALIGNED-NEXT:    buffer_load_ubyte v127, v1, s[0:3], 0 offen offset:204
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:203
+; ALIGNED-NEXT:    s_add_u32 s4, s4, 0x100
+; ALIGNED-NEXT:    s_addc_u32 s5, s5, 0
+; ALIGNED-NEXT:    v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
+; ALIGNED-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x3e
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:202
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:201
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:200
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:199
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:198
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:197
+; ALIGNED-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:196
+; ALIGNED-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:195
+; ALIGNED-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:194
+; ALIGNED-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:193
+; ALIGNED-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:192
+; ALIGNED-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:191
+; ALIGNED-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:190
+; ALIGNED-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:189
+; ALIGNED-NEXT:    buffer_load_ubyte v17, v1, s[0:3], 0 offen offset:188
+; ALIGNED-NEXT:    buffer_load_ubyte v18, v1, s[0:3], 0 offen offset:187
+; ALIGNED-NEXT:    buffer_load_ubyte v19, v1, s[0:3], 0 offen offset:186
+; ALIGNED-NEXT:    buffer_load_ubyte v20, v1, s[0:3], 0 offen offset:185
+; ALIGNED-NEXT:    buffer_load_ubyte v21, v1, s[0:3], 0 offen offset:184
+; ALIGNED-NEXT:    buffer_load_ubyte v22, v1, s[0:3], 0 offen offset:183
+; ALIGNED-NEXT:    buffer_load_ubyte v23, v1, s[0:3], 0 offen offset:182
+; ALIGNED-NEXT:    buffer_load_ubyte v24, v1, s[0:3], 0 offen offset:181
+; ALIGNED-NEXT:    buffer_load_ubyte v25, v1, s[0:3], 0 offen offset:180
+; ALIGNED-NEXT:    buffer_load_ubyte v26, v1, s[0:3], 0 offen offset:179
+; ALIGNED-NEXT:    buffer_load_ubyte v27, v1, s[0:3], 0 offen offset:178
+; ALIGNED-NEXT:    buffer_load_ubyte v28, v1, s[0:3], 0 offen offset:177
+; ALIGNED-NEXT:    buffer_load_ubyte v29, v1, s[0:3], 0 offen offset:176
+; ALIGNED-NEXT:    buffer_load_ubyte v30, v1, s[0:3], 0 offen offset:175
+; ALIGNED-NEXT:    buffer_load_ubyte v31, v1, s[0:3], 0 offen offset:174
+; ALIGNED-NEXT:    buffer_load_ubyte v32, v1, s[0:3], 0 offen offset:173
+; ALIGNED-NEXT:    buffer_load_ubyte v33, v1, s[0:3], 0 offen offset:172
+; ALIGNED-NEXT:    buffer_load_ubyte v34, v1, s[0:3], 0 offen offset:171
+; ALIGNED-NEXT:    buffer_load_ubyte v35, v1, s[0:3], 0 offen offset:170
+; ALIGNED-NEXT:    buffer_load_ubyte v36, v1, s[0:3], 0 offen offset:169
+; ALIGNED-NEXT:    buffer_load_ubyte v37, v1, s[0:3], 0 offen offset:168
+; ALIGNED-NEXT:    buffer_load_ubyte v38, v1, s[0:3], 0 offen offset:167
+; ALIGNED-NEXT:    buffer_load_ubyte v39, v1, s[0:3], 0 offen offset:166
+; ALIGNED-NEXT:    buffer_load_ubyte v48, v1, s[0:3], 0 offen offset:165
+; ALIGNED-NEXT:    buffer_load_ubyte v49, v1, s[0:3], 0 offen offset:164
+; ALIGNED-NEXT:    buffer_load_ubyte v50, v1, s[0:3], 0 offen offset:163
+; ALIGNED-NEXT:    buffer_load_ubyte v51, v1, s[0:3], 0 offen offset:162
+; ALIGNED-NEXT:    buffer_load_ubyte v52, v1, s[0:3], 0 offen offset:161
+; ALIGNED-NEXT:    buffer_load_ubyte v53, v1, s[0:3], 0 offen offset:160
+; ALIGNED-NEXT:    buffer_load_ubyte v54, v1, s[0:3], 0 offen offset:159
+; ALIGNED-NEXT:    buffer_load_ubyte v55, v1, s[0:3], 0 offen offset:158
+; ALIGNED-NEXT:    buffer_load_ubyte v64, v1, s[0:3], 0 offen offset:157
+; ALIGNED-NEXT:    buffer_load_ubyte v65, v1, s[0:3], 0 offen offset:156
+; ALIGNED-NEXT:    buffer_load_ubyte v66, v1, s[0:3], 0 offen offset:155
+; ALIGNED-NEXT:    buffer_load_ubyte v67, v1, s[0:3], 0 offen offset:154
+; ALIGNED-NEXT:    buffer_load_ubyte v68, v1, s[0:3], 0 offen offset:153
+; ALIGNED-NEXT:    buffer_load_ubyte v69, v1, s[0:3], 0 offen offset:152
+; ALIGNED-NEXT:    buffer_load_ubyte v70, v1, s[0:3], 0 offen offset:151
+; ALIGNED-NEXT:    buffer_load_ubyte v71, v1, s[0:3], 0 offen offset:150
+; ALIGNED-NEXT:    buffer_load_ubyte v80, v1, s[0:3], 0 offen offset:149
+; ALIGNED-NEXT:    buffer_load_ubyte v81, v1, s[0:3], 0 offen offset:148
+; ALIGNED-NEXT:    buffer_load_ubyte v82, v1, s[0:3], 0 offen offset:147
+; ALIGNED-NEXT:    buffer_load_ubyte v83, v1, s[0:3], 0 offen offset:146
+; ALIGNED-NEXT:    buffer_load_ubyte v84, v1, s[0:3], 0 offen offset:145
+; ALIGNED-NEXT:    buffer_load_ubyte v85, v1, s[0:3], 0 offen offset:144
+; ALIGNED-NEXT:    buffer_load_ubyte v86, v1, s[0:3], 0 offen offset:143
+; ALIGNED-NEXT:    buffer_load_ubyte v87, v1, s[0:3], 0 offen offset:142
+; ALIGNED-NEXT:    buffer_load_ubyte v96, v1, s[0:3], 0 offen offset:141
+; ALIGNED-NEXT:    buffer_load_ubyte v97, v1, s[0:3], 0 offen offset:140
+; ALIGNED-NEXT:    s_clause 0xa
+; ALIGNED-NEXT:    buffer_load_ubyte v98, v1, s[0:3], 0 offen offset:139
+; ALIGNED-NEXT:    buffer_load_ubyte v99, v1, s[0:3], 0 offen offset:138
+; ALIGNED-NEXT:    buffer_load_ubyte v100, v1, s[0:3], 0 offen offset:137
+; ALIGNED-NEXT:    buffer_load_ubyte v101, v1, s[0:3], 0 offen offset:136
+; ALIGNED-NEXT:    buffer_load_ubyte v102, v1, s[0:3], 0 offen offset:135
+; ALIGNED-NEXT:    buffer_load_ubyte v103, v1, s[0:3], 0 offen offset:134
+; ALIGNED-NEXT:    buffer_load_ubyte v112, v1, s[0:3], 0 offen offset:133
+; ALIGNED-NEXT:    buffer_load_ubyte v113, v1, s[0:3], 0 offen offset:132
+; ALIGNED-NEXT:    buffer_load_ubyte v114, v1, s[0:3], 0 offen offset:131
+; ALIGNED-NEXT:    buffer_load_ubyte v115, v1, s[0:3], 0 offen offset:130
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:129
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:128
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:127
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:126
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:125
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:124
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:123
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:122
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:121
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:120
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:119
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:118
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:117
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:116
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:115
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:114
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:113
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:112
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:111
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:110
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:109
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:108
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:107
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:106
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:105
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:104
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:103
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:102
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:101
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:100
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:99
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:98
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:97
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:96
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:95
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:94
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:93
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:92
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:91
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:90
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:89
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:88
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:87
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:86
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:85
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:84
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:83
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:82
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:81
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:80
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:79
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:78
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:77
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:76
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:75
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:74
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:73
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:72
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:71
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:70
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:69
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:68
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:67
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:66
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:65
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:64
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:63
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:62
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:61
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:60
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:59
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:58
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:57
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:56
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:55
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:54
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:53
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:52
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:51
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:50
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:49
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:48
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:47
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:46
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:45
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:44
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:43
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:42
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:41
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:40
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:39
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:38
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:37
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:36
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:35
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:34
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:33
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:32
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:31
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:30
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:29
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:27
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:26
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:25
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:24
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:23
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:22
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:21
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:20
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:19
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:18
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:17
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:16
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:15
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:14
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:13
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:12
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:11
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:10
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:9
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:8
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:7
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:6
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:5
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:4
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:3
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:2
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:1
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen
+; ALIGNED-NEXT:    v_add_nc_u32_e32 v1, 0x100, v1
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_byte v116, v0, s[0:3], 0 offen offset:255
+; ALIGNED-NEXT:    buffer_store_byte v117, v0, s[0:3], 0 offen offset:254
+; ALIGNED-NEXT:    buffer_store_byte v118, v0, s[0:3], 0 offen offset:253
+; ALIGNED-NEXT:    buffer_store_byte v119, v0, s[0:3], 0 offen offset:252
+; ALIGNED-NEXT:    buffer_store_byte v40, v0, s[0:3], 0 offen offset:251
+; ALIGNED-NEXT:    buffer_store_byte v41, v0, s[0:3], 0 offen offset:250
+; ALIGNED-NEXT:    buffer_store_byte v42, v0, s[0:3], 0 offen offset:249
+; ALIGNED-NEXT:    buffer_store_byte v43, v0, s[0:3], 0 offen offset:248
+; ALIGNED-NEXT:    buffer_store_byte v44, v0, s[0:3], 0 offen offset:247
+; ALIGNED-NEXT:    buffer_store_byte v45, v0, s[0:3], 0 offen offset:246
+; ALIGNED-NEXT:    buffer_store_byte v46, v0, s[0:3], 0 offen offset:245
+; ALIGNED-NEXT:    buffer_store_byte v47, v0, s[0:3], 0 offen offset:244
+; ALIGNED-NEXT:    buffer_store_byte v56, v0, s[0:3], 0 offen offset:243
+; ALIGNED-NEXT:    buffer_store_byte v57, v0, s[0:3], 0 offen offset:242
+; ALIGNED-NEXT:    buffer_store_byte v58, v0, s[0:3], 0 offen offset:241
+; ALIGNED-NEXT:    buffer_store_byte v59, v0, s[0:3], 0 offen offset:240
+; ALIGNED-NEXT:    buffer_store_byte v60, v0, s[0:3], 0 offen offset:239
+; ALIGNED-NEXT:    buffer_store_byte v61, v0, s[0:3], 0 offen offset:238
+; ALIGNED-NEXT:    buffer_store_byte v62, v0, s[0:3], 0 offen offset:237
+; ALIGNED-NEXT:    buffer_store_byte v63, v0, s[0:3], 0 offen offset:236
+; ALIGNED-NEXT:    buffer_store_byte v72, v0, s[0:3], 0 offen offset:235
+; ALIGNED-NEXT:    buffer_store_byte v73, v0, s[0:3], 0 offen offset:234
+; ALIGNED-NEXT:    buffer_store_byte v74, v0, s[0:3], 0 offen offset:233
+; ALIGNED-NEXT:    buffer_store_byte v75, v0, s[0:3], 0 offen offset:232
+; ALIGNED-NEXT:    buffer_store_byte v76, v0, s[0:3], 0 offen offset:231
+; ALIGNED-NEXT:    buffer_store_byte v77, v0, s[0:3], 0 offen offset:230
+; ALIGNED-NEXT:    buffer_store_byte v78, v0, s[0:3], 0 offen offset:229
+; ALIGNED-NEXT:    buffer_store_byte v79, v0, s[0:3], 0 offen offset:228
+; ALIGNED-NEXT:    buffer_store_byte v88, v0, s[0:3], 0 offen offset:227
+; ALIGNED-NEXT:    buffer_store_byte v89, v0, s[0:3], 0 offen offset:226
+; ALIGNED-NEXT:    buffer_store_byte v90, v0, s[0:3], 0 offen offset:225
+; ALIGNED-NEXT:    buffer_store_byte v91, v0, s[0:3], 0 offen offset:224
+; ALIGNED-NEXT:    buffer_store_byte v92, v0, s[0:3], 0 offen offset:223
+; ALIGNED-NEXT:    buffer_store_byte v93, v0, s[0:3], 0 offen offset:222
+; ALIGNED-NEXT:    buffer_store_byte v94, v0, s[0:3], 0 offen offset:221
+; ALIGNED-NEXT:    buffer_store_byte v95, v0, s[0:3], 0 offen offset:220
+; ALIGNED-NEXT:    buffer_store_byte v104, v0, s[0:3], 0 offen offset:219
+; ALIGNED-NEXT:    buffer_store_byte v105, v0, s[0:3], 0 offen offset:218
+; ALIGNED-NEXT:    buffer_store_byte v106, v0, s[0:3], 0 offen offset:217
+; ALIGNED-NEXT:    buffer_store_byte v107, v0, s[0:3], 0 offen offset:216
+; ALIGNED-NEXT:    buffer_store_byte v108, v0, s[0:3], 0 offen offset:215
+; ALIGNED-NEXT:    buffer_store_byte v109, v0, s[0:3], 0 offen offset:214
+; ALIGNED-NEXT:    buffer_store_byte v110, v0, s[0:3], 0 offen offset:213
+; ALIGNED-NEXT:    buffer_store_byte v111, v0, s[0:3], 0 offen offset:212
+; ALIGNED-NEXT:    buffer_store_byte v120, v0, s[0:3], 0 offen offset:211
+; ALIGNED-NEXT:    buffer_store_byte v121, v0, s[0:3], 0 offen offset:210
+; ALIGNED-NEXT:    buffer_store_byte v122, v0, s[0:3], 0 offen offset:209
+; ALIGNED-NEXT:    buffer_store_byte v123, v0, s[0:3], 0 offen offset:208
+; ALIGNED-NEXT:    buffer_store_byte v124, v0, s[0:3], 0 offen offset:207
+; ALIGNED-NEXT:    buffer_store_byte v125, v0, s[0:3], 0 offen offset:206
+; ALIGNED-NEXT:    buffer_store_byte v126, v0, s[0:3], 0 offen offset:205
+; ALIGNED-NEXT:    buffer_store_byte v127, v0, s[0:3], 0 offen offset:204
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:203
+; ALIGNED-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:202
+; ALIGNED-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:201
+; ALIGNED-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:200
+; ALIGNED-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:199
+; ALIGNED-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:198
+; ALIGNED-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:197
+; ALIGNED-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:196
+; ALIGNED-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:195
+; ALIGNED-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:194
+; ALIGNED-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:193
+; ALIGNED-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:192
+; ALIGNED-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:191
+; ALIGNED-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:190
+; ALIGNED-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:189
+; ALIGNED-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:188
+; ALIGNED-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:187
+; ALIGNED-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:186
+; ALIGNED-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:185
+; ALIGNED-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:184
+; ALIGNED-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:183
+; ALIGNED-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:182
+; ALIGNED-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:181
+; ALIGNED-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:180
+; ALIGNED-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:179
+; ALIGNED-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:178
+; ALIGNED-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:177
+; ALIGNED-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:176
+; ALIGNED-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:175
+; ALIGNED-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:174
+; ALIGNED-NEXT:    buffer_store_byte v32, v0, s[0:3], 0 offen offset:173
+; ALIGNED-NEXT:    buffer_store_byte v33, v0, s[0:3], 0 offen offset:172
+; ALIGNED-NEXT:    buffer_store_byte v34, v0, s[0:3], 0 offen offset:171
+; ALIGNED-NEXT:    buffer_store_byte v35, v0, s[0:3], 0 offen offset:170
+; ALIGNED-NEXT:    buffer_store_byte v36, v0, s[0:3], 0 offen offset:169
+; ALIGNED-NEXT:    buffer_store_byte v37, v0, s[0:3], 0 offen offset:168
+; ALIGNED-NEXT:    buffer_store_byte v38, v0, s[0:3], 0 offen offset:167
+; ALIGNED-NEXT:    buffer_store_byte v39, v0, s[0:3], 0 offen offset:166
+; ALIGNED-NEXT:    buffer_store_byte v48, v0, s[0:3], 0 offen offset:165
+; ALIGNED-NEXT:    buffer_store_byte v49, v0, s[0:3], 0 offen offset:164
+; ALIGNED-NEXT:    buffer_store_byte v50, v0, s[0:3], 0 offen offset:163
+; ALIGNED-NEXT:    buffer_store_byte v51, v0, s[0:3], 0 offen offset:162
+; ALIGNED-NEXT:    buffer_store_byte v52, v0, s[0:3], 0 offen offset:161
+; ALIGNED-NEXT:    buffer_store_byte v53, v0, s[0:3], 0 offen offset:160
+; ALIGNED-NEXT:    buffer_store_byte v54, v0, s[0:3], 0 offen offset:159
+; ALIGNED-NEXT:    buffer_store_byte v55, v0, s[0:3], 0 offen offset:158
+; ALIGNED-NEXT:    buffer_store_byte v64, v0, s[0:3], 0 offen offset:157
+; ALIGNED-NEXT:    buffer_store_byte v65, v0, s[0:3], 0 offen offset:156
+; ALIGNED-NEXT:    buffer_store_byte v66, v0, s[0:3], 0 offen offset:155
+; ALIGNED-NEXT:    buffer_store_byte v67, v0, s[0:3], 0 offen offset:154
+; ALIGNED-NEXT:    buffer_store_byte v68, v0, s[0:3], 0 offen offset:153
+; ALIGNED-NEXT:    buffer_store_byte v69, v0, s[0:3], 0 offen offset:152
+; ALIGNED-NEXT:    buffer_store_byte v70, v0, s[0:3], 0 offen offset:151
+; ALIGNED-NEXT:    buffer_store_byte v71, v0, s[0:3], 0 offen offset:150
+; ALIGNED-NEXT:    buffer_store_byte v80, v0, s[0:3], 0 offen offset:149
+; ALIGNED-NEXT:    buffer_store_byte v81, v0, s[0:3], 0 offen offset:148
+; ALIGNED-NEXT:    buffer_store_byte v82, v0, s[0:3], 0 offen offset:147
+; ALIGNED-NEXT:    buffer_store_byte v83, v0, s[0:3], 0 offen offset:146
+; ALIGNED-NEXT:    buffer_store_byte v84, v0, s[0:3], 0 offen offset:145
+; ALIGNED-NEXT:    buffer_store_byte v85, v0, s[0:3], 0 offen offset:144
+; ALIGNED-NEXT:    buffer_store_byte v86, v0, s[0:3], 0 offen offset:143
+; ALIGNED-NEXT:    buffer_store_byte v87, v0, s[0:3], 0 offen offset:142
+; ALIGNED-NEXT:    buffer_store_byte v96, v0, s[0:3], 0 offen offset:141
+; ALIGNED-NEXT:    buffer_store_byte v97, v0, s[0:3], 0 offen offset:140
+; ALIGNED-NEXT:    buffer_store_byte v98, v0, s[0:3], 0 offen offset:139
+; ALIGNED-NEXT:    buffer_store_byte v99, v0, s[0:3], 0 offen offset:138
+; ALIGNED-NEXT:    buffer_store_byte v100, v0, s[0:3], 0 offen offset:137
+; ALIGNED-NEXT:    buffer_store_byte v101, v0, s[0:3], 0 offen offset:136
+; ALIGNED-NEXT:    buffer_store_byte v102, v0, s[0:3], 0 offen offset:135
+; ALIGNED-NEXT:    buffer_store_byte v103, v0, s[0:3], 0 offen offset:134
+; ALIGNED-NEXT:    buffer_store_byte v112, v0, s[0:3], 0 offen offset:133
+; ALIGNED-NEXT:    buffer_store_byte v113, v0, s[0:3], 0 offen offset:132
+; ALIGNED-NEXT:    buffer_store_byte v114, v0, s[0:3], 0 offen offset:131
+; ALIGNED-NEXT:    buffer_store_byte v115, v0, s[0:3], 0 offen offset:130
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:129
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:128
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:127
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:126
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:125
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:124
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:123
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:122
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:121
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:120
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:119
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:118
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:117
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:116
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:115
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:114
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:113
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:112
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:111
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:110
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:109
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:108
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:107
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:106
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:105
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:104
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:103
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:102
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:101
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:100
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:99
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:98
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:97
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:96
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:95
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:94
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:93
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:92
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:91
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:90
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:89
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:88
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:87
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:86
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:85
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:84
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:83
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:82
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:81
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:80
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:79
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:78
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:77
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:76
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:75
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:74
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:73
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:72
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:71
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:70
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:69
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:68
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:67
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:66
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:65
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:64
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:63
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:62
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:61
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:60
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:59
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:58
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:57
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:56
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:55
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:54
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:53
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:52
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:51
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:50
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:49
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:48
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:47
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:46
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:45
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:44
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:43
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:42
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:41
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:40
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:39
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:38
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:37
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:36
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:35
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:34
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:33
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:32
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:31
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:30
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:29
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:28
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:27
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:26
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:25
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:24
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:23
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:22
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:21
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:20
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:19
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:18
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:17
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:16
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:15
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:14
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:13
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:12
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:11
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:10
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:9
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:8
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:7
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:6
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:5
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:4
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:3
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:2
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:1
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen
+; ALIGNED-NEXT:    v_add_nc_u32_e32 v0, 0x100, v0
+; ALIGNED-NEXT:    s_cbranch_vccnz .LBB3_1
+; ALIGNED-NEXT:  ; %bb.2: ; %memcpy-split
+; ALIGNED-NEXT:    s_clause 0x2f
+; ALIGNED-NEXT:    buffer_load_dword v127, off, s[0:3], s32
+; ALIGNED-NEXT:    buffer_load_dword v126, off, s[0:3], s32 offset:4
+; ALIGNED-NEXT:    buffer_load_dword v125, off, s[0:3], s32 offset:8
+; ALIGNED-NEXT:    buffer_load_dword v124, off, s[0:3], s32 offset:12
+; ALIGNED-NEXT:    buffer_load_dword v123, off, s[0:3], s32 offset:16
+; ALIGNED-NEXT:    buffer_load_dword v122, off, s[0:3], s32 offset:20
+; ALIGNED-NEXT:    buffer_load_dword v121, off, s[0:3], s32 offset:24
+; ALIGNED-NEXT:    buffer_load_dword v120, off, s[0:3], s32 offset:28
+; ALIGNED-NEXT:    buffer_load_dword v111, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT:    buffer_load_dword v110, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT:    buffer_load_dword v109, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT:    buffer_load_dword v108, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT:    buffer_load_dword v107, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT:    buffer_load_dword v106, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT:    buffer_load_dword v105, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT:    buffer_load_dword v104, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT:    buffer_load_dword v95, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT:    buffer_load_dword v94, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT:    buffer_load_dword v93, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT:    buffer_load_dword v92, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT:    buffer_load_dword v91, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT:    buffer_load_dword v90, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT:    buffer_load_dword v89, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT:    buffer_load_dword v88, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT:    buffer_load_dword v79, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT:    buffer_load_dword v78, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT:    buffer_load_dword v77, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT:    buffer_load_dword v76, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT:    buffer_load_dword v75, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT:    buffer_load_dword v74, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT:    buffer_load_dword v73, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT:    buffer_load_dword v72, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; UNROLL3-LABEL: memcpy_p5_p5_sz2048:
+; UNROLL3:       ; %bb.0: ; %entry
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    v_mov_b32_e32 v2, v1
+; UNROLL3-NEXT:    v_mov_b32_e32 v3, v0
+; UNROLL3-NEXT:    s_mov_b64 s[4:5], 0
+; UNROLL3-NEXT:  .LBB3_1: ; %load-store-loop
+; UNROLL3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT:    s_clause 0xb
+; UNROLL3-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:44
+; UNROLL3-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:40
+; UNROLL3-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:36
+; UNROLL3-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:32
+; UNROLL3-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:28
+; UNROLL3-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; UNROLL3-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:20
+; UNROLL3-NEXT:    buffer_load_dword v11, v2, s[0:3], 0 offen offset:16
+; UNROLL3-NEXT:    buffer_load_dword v12, v2, s[0:3], 0 offen offset:12
+; UNROLL3-NEXT:    buffer_load_dword v13, v2, s[0:3], 0 offen offset:8
+; UNROLL3-NEXT:    buffer_load_dword v14, v2, s[0:3], 0 offen offset:4
+; UNROLL3-NEXT:    buffer_load_dword v15, v2, s[0:3], 0 offen
+; UNROLL3-NEXT:    s_add_u32 s4, s4, 48
+; UNROLL3-NEXT:    s_addc_u32 s5, s5, 0
+; UNROLL3-NEXT:    v_add_nc_u32_e32 v2, 48, v2
+; UNROLL3-NEXT:    v_cmp_gt_u64_e64 s6, 0x7e0, s[4:5]
+; UNROLL3-NEXT:    s_waitcnt vmcnt(11)
+; UNROLL3-NEXT:    buffer_store_dword v4, v3, s[0:3], 0 offen offset:44
+; UNROLL3-NEXT:    s_waitcnt vmcnt(10)
+; UNROLL3-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen offset:40
+; UNROLL3-NEXT:    s_waitcnt vmcnt(9)
+; UNROLL3-NEXT:    buffer_store_dword v6, v3, s[0:3], 0 offen offset:36
+; UNROLL3-NEXT:    s_waitcnt vmcnt(8)
+; UNROLL3-NEXT:    buffer_store_dword v7, v3, s[0:3], 0 offen offset:32
+; UNROLL3-NEXT:    s_waitcnt vmcnt(7)
+; UNROLL3-NEXT:    buffer_store_dword v8, v3, s[0:3], 0 offen offset:28
+; UNROLL3-NEXT:    s_waitcnt vmcnt(6)
+; UNROLL3-NEXT:    buffer_store_dword v9, v3, s[0:3], 0 offen offset:24
+; UNROLL3-NEXT:    s_waitcnt vmcnt(5)
+; UNROLL3-NEXT:    buffer_store_dword v10, v3, s[0:3], 0 offen offset:20
+; UNROLL3-NEXT:    s_waitcnt vmcnt(4)
+; UNROLL3-NEXT:    buffer_store_dword v11, v3, s[0:3], 0 offen offset:16
+; UNROLL3-NEXT:    s_waitcnt vmcnt(3)
+; UNROLL3-NEXT:    buffer_store_dword v12, v3, s[0:3], 0 offen offset:12
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2)
+; UNROLL3-NEXT:    buffer_store_dword v13, v3, s[0:3], 0 offen offset:8
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    buffer_store_dword v14, v3, s[0:3], 0 offen offset:4
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    buffer_store_dword v15, v3, s[0:3], 0 offen
+; UNROLL3-NEXT:    v_add_nc_u32_e32 v3, 48, v3
+; UNROLL3-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
+; UNROLL3-NEXT:    s_cbranch_vccnz .LBB3_1
+; UNROLL3-NEXT:  ; %bb.2: ; %memcpy-split
+; UNROLL3-NEXT:    s_clause 0x3
+; UNROLL3-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:2028
+; UNROLL3-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:2024
+; UNROLL3-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:2020
+; UNROLL3-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:2016
+; UNROLL3-NEXT:    s_waitcnt vmcnt(3)
+; UNROLL3-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:2028
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2)
+; UNROLL3-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:2024
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:2020
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:2016
+; UNROLL3-NEXT:    s_clause 0x3
+; UNROLL3-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:2044
+; UNROLL3-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:2040
+; UNROLL3-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:2036
+; UNROLL3-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:2032
+; UNROLL3-NEXT:    s_waitcnt vmcnt(3)
+; UNROLL3-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:2044
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2)
+; UNROLL3-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:2040
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:2036
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:2032
+; UNROLL3-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 2048, i1 false)
+  ret void
+}
+
+define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {
+; CHECK-LABEL: memcpy_p0_p5_sz2048:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0
+; CHECK-NEXT:  .LBB4_1: ; %load-store-loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_clause 0x3e
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:32
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:36
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:40
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:44
+; CHECK-NEXT:    buffer_load_dword v11, v2, s[0:3], 0 offen offset:48
+; CHECK-NEXT:    buffer_load_dword v12, v2, s[0:3], 0 offen offset:52
+; CHECK-NEXT:    buffer_load_dword v13, v2, s[0:3], 0 offen offset:56
+; CHECK-NEXT:    buffer_load_dword v14, v2, s[0:3], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v18, v2, s[0:3], 0 offen offset:124
+; CHECK-NEXT:    buffer_load_dword v17, v2, s[0:3], 0 offen offset:120
+; CHECK-NEXT:    buffer_load_dword v16, v2, s[0:3], 0 offen offset:116
+; CHECK-NEXT:    buffer_load_dword v15, v2, s[0:3], 0 offen offset:112
+; CHECK-NEXT:    buffer_load_dword v22, v2, s[0:3], 0 offen offset:108
+; CHECK-NEXT:    buffer_load_dword v21, v2, s[0:3], 0 offen offset:104
+; CHECK-NEXT:    buffer_load_dword v20, v2, s[0:3], 0 offen offset:100
+; CHECK-NEXT:    buffer_load_dword v19, v2, s[0:3], 0 offen offset:96
+; CHECK-NEXT:    buffer_load_dword v26, v2, s[0:3], 0 offen offset:252
+; CHECK-NEXT:    buffer_load_dword v25, v2, s[0:3], 0 offen offset:248
+; CHECK-NEXT:    buffer_load_dword v24, v2, s[0:3], 0 offen offset:244
+; CHECK-NEXT:    buffer_load_dword v23, v2, s[0:3], 0 offen offset:240
+; CHECK-NEXT:    buffer_load_dword v30, v2, s[0:3], 0 offen offset:236
+; CHECK-NEXT:    buffer_load_dword v29, v2, s[0:3], 0 offen offset:232
+; CHECK-NEXT:    buffer_load_dword v28, v2, s[0:3], 0 offen offset:228
+; CHECK-NEXT:    buffer_load_dword v27, v2, s[0:3], 0 offen offset:224
+; CHECK-NEXT:    buffer_load_dword v34, v2, s[0:3], 0 offen offset:220
+; CHECK-NEXT:    buffer_load_dword v33, v2, s[0:3], 0 offen offset:216
+; CHECK-NEXT:    buffer_load_dword v32, v2, s[0:3], 0 offen offset:212
+; CHECK-NEXT:    buffer_load_dword v31, v2, s[0:3], 0 offen offset:208
+; CHECK-NEXT:    buffer_load_dword v38, v2, s[0:3], 0 offen offset:204
+; CHECK-NEXT:    buffer_load_dword v37, v2, s[0:3], 0 offen offset:200
+; CHECK-NEXT:    buffer_load_dword v36, v2, s[0:3], 0 offen offset:196
+; CHECK-NEXT:    buffer_load_dword v35, v2, s[0:3], 0 offen offset:192
+; CHECK-NEXT:    buffer_load_dword v51, v2, s[0:3], 0 offen offset:188
+; CHECK-NEXT:    buffer_load_dword v50, v2, s[0:3], 0 offen offset:184
+; CHECK-NEXT:    buffer_load_dword v49, v2, s[0:3], 0 offen offset:180
+; CHECK-NEXT:    buffer_load_dword v48, v2, s[0:3], 0 offen offset:176
+; CHECK-NEXT:    buffer_load_dword v55, v2, s[0:3], 0 offen offset:172
+; CHECK-NEXT:    buffer_load_dword v54, v2, s[0:3], 0 offen offset:168
+; CHECK-NEXT:    buffer_load_dword v53, v2, s[0:3], 0 offen offset:164
+; CHECK-NEXT:    buffer_load_dword v52, v2, s[0:3], 0 offen offset:160
+; CHECK-NEXT:    buffer_load_dword v67, v2, s[0:3], 0 offen offset:156
+; CHECK-NEXT:    buffer_load_dword v66, v2, s[0:3], 0 offen offset:152
+; CHECK-NEXT:    buffer_load_dword v65, v2, s[0:3], 0 offen offset:148
+; CHECK-NEXT:    buffer_load_dword v64, v2, s[0:3], 0 offen offset:144
+; CHECK-NEXT:    buffer_load_dword v71, v2, s[0:3], 0 offen offset:140
+; CHECK-NEXT:    buffer_load_dword v70, v2, s[0:3], 0 offen offset:136
+; CHECK-NEXT:    buffer_load_dword v69, v2, s[0:3], 0 offen offset:132
+; CHECK-NEXT:    buffer_load_dword v68, v2, s[0:3], 0 offen offset:128
+; CHECK-NEXT:    buffer_load_dword v83, v2, s[0:3], 0 offen offset:92
+; CHECK-NEXT:    buffer_load_dword v82, v2, s[0:3], 0 offen offset:88
+; CHECK-NEXT:    buffer_load_dword v81, v2, s[0:3], 0 offen offset:84
+; CHECK-NEXT:    buffer_load_dword v80, v2, s[0:3], 0 offen offset:80
+; CHECK-NEXT:    buffer_load_dword v87, v2, s[0:3], 0 offen offset:76
+; CHECK-NEXT:    buffer_load_dword v86, v2, s[0:3], 0 offen offset:72
+; CHECK-NEXT:    buffer_load_dword v85, v2, s[0:3], 0 offen offset:68
+; CHECK-NEXT:    buffer_load_dword v84, v2, s[0:3], 0 offen offset:64
+; CHECK-NEXT:    buffer_load_dword v96, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v97, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v98, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v99, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    v_add_co_u32 v100, vcc_lo, v0, s4
+; CHECK-NEXT:    s_add_u32 s4, s4, 0x100
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v101, vcc_lo, s5, v1, vcc_lo
+; CHECK-NEXT:    s_addc_u32 s5, s5, 0
+; CHECK-NEXT:    v_add_nc_u32_e32 v2, 0x100, v2
+; CHECK-NEXT:    v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
+; CHECK-NEXT:    s_waitcnt vmcnt(41)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[23:26] offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(37)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[27:30] offset:224
+; CHECK-NEXT:    s_waitcnt vmcnt(33)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[31:34] offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(29)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[35:38] offset:192
+; CHECK-NEXT:    s_waitcnt vmcnt(25)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[48:51] offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(21)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[52:55] offset:160
+; CHECK-NEXT:    s_waitcnt vmcnt(17)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[64:67] offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(13)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[68:71] offset:128
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[15:18] offset:112
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[19:22] offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(9)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[80:83] offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[84:87] offset:64
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[11:14] offset:48
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[7:10] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[3:6] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[96:99]
+; CHECK-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
+; CHECK-NEXT:    s_cbranch_vccnz .LBB4_1
+; CHECK-NEXT:  ; %bb.2: ; %memcpy-split
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+;
+; ALIGNED-LABEL: memcpy_p0_p5_sz2048:
+; ALIGNED:       ; %bb.0: ; %entry
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; ALIGNED-NEXT:    s_mov_b64 s[4:5], 0
+; ALIGNED-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v72, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v73, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v74, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v75, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v76, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v77, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v78, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v79, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v88, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v89, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v90, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v91, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v92, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v93, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v94, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v95, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v104, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v105, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v106, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v107, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v108, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v109, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v110, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v111, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v120, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v121, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v122, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v123, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v124, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v125, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v126, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v127, off, s[0:3], s32 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1224 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1228 ; 4-byte Folded Spill
+; ALIGNED-NEXT:  .LBB4_1: ; %load-store-loop
+; ALIGNED-NEXT:    ; =>This Inner Loop Header: Depth=1
+; ALIGNED-NEXT:    s_clause 0x39
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:20
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:21
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:22
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:23
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:24
+; ALIGNED-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:25
+; ALIGNED-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:26
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:30
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:31
+; ALIGNED-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:32
+; ALIGNED-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:33
+; ALIGNED-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:34
+; ALIGNED-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:29
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:28
+; ALIGNED-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:27
+; ALIGNED-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:35
+; ALIGNED-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:36
+; ALIGNED-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:37
+; ALIGNED-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:38
+; ALIGNED-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:39
+; ALIGNED-NEXT:    buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:40
+; ALIGNED-NEXT:    buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:41
+; ALIGNED-NEXT:    buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:42
+; ALIGNED-NEXT:    buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:43
+; ALIGNED-NEXT:    buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:44
+; ALIGNED-NEXT:    buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:45
+; ALIGNED-NEXT:    buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:46
+; ALIGNED-NEXT:    buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:47
+; ALIGNED-NEXT:    buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:48
+; ALIGNED-NEXT:    buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:49
+; ALIGNED-NEXT:    buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:50
+; ALIGNED-NEXT:    buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:51
+; ALIGNED-NEXT:    buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:52
+; ALIGNED-NEXT:    buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:53
+; ALIGNED-NEXT:    buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:54
+; ALIGNED-NEXT:    buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:55
+; ALIGNED-NEXT:    buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:56
+; ALIGNED-NEXT:    buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:57
+; ALIGNED-NEXT:    buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:58
+; ALIGNED-NEXT:    buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:60
+; ALIGNED-NEXT:    buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:61
+; ALIGNED-NEXT:    buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:62
+; ALIGNED-NEXT:    buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:63
+; ALIGNED-NEXT:    buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:64
+; ALIGNED-NEXT:    buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:65
+; ALIGNED-NEXT:    buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:66
+; ALIGNED-NEXT:    buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:59
+; ALIGNED-NEXT:    buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:67
+; ALIGNED-NEXT:    buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:68
+; ALIGNED-NEXT:    buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:69
+; ALIGNED-NEXT:    buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:70
+; ALIGNED-NEXT:    buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:71
+; ALIGNED-NEXT:    buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:76
+; ALIGNED-NEXT:    buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:77
+; ALIGNED-NEXT:    buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:78
+; ALIGNED-NEXT:    buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:79
+; ALIGNED-NEXT:    buffer_load_ubyte v127, v2, s[0:3], 0 offen offset:19
+; ALIGNED-NEXT:    buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:75
+; ALIGNED-NEXT:    s_waitcnt vmcnt(57)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(56)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(55)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(54)
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(53)
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(52)
+; ALIGNED-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(51)
+; ALIGNED-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(50)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(49)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(48)
+; ALIGNED-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v3
+; ALIGNED-NEXT:    s_waitcnt vmcnt(45)
+; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(44)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(43)
+; ALIGNED-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v9, 8, v5
+; ALIGNED-NEXT:    s_waitcnt vmcnt(41)
+; ALIGNED-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v8, 8, v6
+; ALIGNED-NEXT:    v_lshl_or_b32 v5, v10, 8, v7
+; ALIGNED-NEXT:    v_lshl_or_b32 v6, v11, 8, v12
+; ALIGNED-NEXT:    v_lshl_or_b32 v7, v15, 8, v14
+; ALIGNED-NEXT:    v_lshl_or_b32 v8, v19, 8, v17
+; ALIGNED-NEXT:    s_waitcnt vmcnt(40)
+; ALIGNED-NEXT:    v_lshl_or_b32 v9, v16, 8, v13
+; ALIGNED-NEXT:    s_waitcnt vmcnt(38)
+; ALIGNED-NEXT:    v_lshl_or_b32 v10, v20, 8, v18
+; ALIGNED-NEXT:    s_waitcnt vmcnt(36)
+; ALIGNED-NEXT:    v_lshl_or_b32 v11, v23, 8, v22
+; ALIGNED-NEXT:    s_waitcnt vmcnt(34)
+; ALIGNED-NEXT:    v_lshl_or_b32 v12, v28, 8, v25
+; ALIGNED-NEXT:    s_waitcnt vmcnt(32)
+; ALIGNED-NEXT:    v_lshl_or_b32 v13, v24, 8, v21
+; ALIGNED-NEXT:    s_waitcnt vmcnt(30)
+; ALIGNED-NEXT:    v_lshl_or_b32 v14, v27, 8, v26
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v6, 16, v5
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v8, 16, v7
+; ALIGNED-NEXT:    v_lshl_or_b32 v5, v10, 16, v9
+; ALIGNED-NEXT:    v_lshl_or_b32 v6, v12, 16, v11
+; ALIGNED-NEXT:    v_lshl_or_b32 v7, v14, 16, v13
+; ALIGNED-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(28)
+; ALIGNED-NEXT:    v_lshl_or_b32 v15, v31, 8, v30
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(26)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v34, 8, v33
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(24)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v37, 8, v32
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(22)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v36, 8, v35
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(17)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v50, 8, v38
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(15)
+; ALIGNED-NEXT:    v_lshl_or_b32 v5, v49, 8, v39
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v6, v51, 8, v48
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(11)
+; ALIGNED-NEXT:    v_lshl_or_b32 v7, v53, 8, v52
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v0, 16, v15
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 16, v1
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v5, 16, v4
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:85
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v7, 16, v6
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v55, 8, v29
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(11)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v67, 8, v66
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(9)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v64, 8, v54
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v68, 8, v65
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:86
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:82
+; ALIGNED-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v70, 8, v69
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:83
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:74
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v80, 8, v71
+; ALIGNED-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:73
+; ALIGNED-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:72
+; ALIGNED-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v37, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v35, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v36, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v38, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v50, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v39, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v49, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v48, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v51, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v52, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v53, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v55, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v66, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v67, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v54, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v64, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v65, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v68, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v69, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v70, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v71, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v80, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(8)
+; ALIGNED-NEXT:    buffer_store_dword v127, off, s[0:3], s32 offset:1152 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    buffer_store_dword v81, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:87
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v81, 8, v3
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:84
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:81
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:80
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v7
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:98
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v8, 8, v6
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:102
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:103
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v5, 8, v3
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:94
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:95
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:93
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:91
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:92
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v5, 8, v3
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:90
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:101
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:89
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:88
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v3
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:99
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:100
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:97
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:96
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v7
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:114
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v8, 8, v6
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:118
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:119
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v5, 8, v3
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:110
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:111
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:109
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:107
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:108
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v5, 8, v3
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:106
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:117
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:105
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:104
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v3
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:115
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:116
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:113
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:112
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v7
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:130
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v8, 8, v6
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:134
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:135
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:940 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v5, 8, v3
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:126
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:127
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:125
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:123
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:124
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:1012 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:1016 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:952 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:944 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v5, 8, v3
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:122
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:976 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:133
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:121
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:120
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:980 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1008 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v3
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:131
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:132
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:1004 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:129
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:128
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:996 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1000 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v7
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v8, 8, v6
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1020 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v5, 8, v3
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:142
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:143
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:141
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:139
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1028 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:140
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1036 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1040 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1032 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1024 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v5, 8, v3
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:138
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:1056 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:137
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1048 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:136
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1060 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1052 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1044 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:145
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1064 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:144
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1084 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1072 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x5
+; ALIGNED-NEXT:    buffer_load_ubyte v125, v2, s[0:3], 0 offen offset:146
+; ALIGNED-NEXT:    buffer_load_ubyte v126, v2, s[0:3], 0 offen offset:147
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:148
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:149
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:150
+; ALIGNED-NEXT:    buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:151
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v126, 8, v125
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1104 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:1112 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1116 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v123, 8, v5
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1124 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 8, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1132 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v121, v2, s[0:3], 0 offen offset:156
+; ALIGNED-NEXT:    buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:157
+; ALIGNED-NEXT:    buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:158
+; ALIGNED-NEXT:    buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:159
+; ALIGNED-NEXT:    buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:155
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v109, 8, v121
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v107, 8, v108
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1136 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v105, v2, s[0:3], 0 offen offset:152
+; ALIGNED-NEXT:    buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:153
+; ALIGNED-NEXT:    buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:154
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v93, 8, v105
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v106, 8, v91
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1144 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:160
+; ALIGNED-NEXT:    buffer_load_ubyte v78, v2, s[0:3], 0 offen offset:161
+; ALIGNED-NEXT:    buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:162
+; ALIGNED-NEXT:    buffer_load_ubyte v74, v2, s[0:3], 0 offen offset:163
+; ALIGNED-NEXT:    buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:164
+; ALIGNED-NEXT:    buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:165
+; ALIGNED-NEXT:    buffer_load_ubyte v76, v2, s[0:3], 0 offen offset:166
+; ALIGNED-NEXT:    buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:167
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v78, 8, v89
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v74, 8, v73
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v72, 8, v76
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1156 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v75, 8, v79
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1160 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:172
+; ALIGNED-NEXT:    buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:173
+; ALIGNED-NEXT:    buffer_load_ubyte v62, v2, s[0:3], 0 offen offset:174
+; ALIGNED-NEXT:    buffer_load_ubyte v60, v2, s[0:3], 0 offen offset:175
+; ALIGNED-NEXT:    buffer_load_ubyte v57, v2, s[0:3], 0 offen offset:171
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v61, 8, v63
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v60, 8, v62
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1164 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v59, v2, s[0:3], 0 offen offset:168
+; ALIGNED-NEXT:    buffer_load_ubyte v56, v2, s[0:3], 0 offen offset:169
+; ALIGNED-NEXT:    buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:170
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v56, 8, v59
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v57, 8, v47
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1168 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_ubyte v46, v2, s[0:3], 0 offen offset:176
+; ALIGNED-NEXT:    buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:177
+; ALIGNED-NEXT:    buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:178
+; ALIGNED-NEXT:    buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:179
+; ALIGNED-NEXT:    buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:180
+; ALIGNED-NEXT:    buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:181
+; ALIGNED-NEXT:    buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:182
+; ALIGNED-NEXT:    buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:183
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v43, 8, v46
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v40, 8, v119
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v118, 8, v42
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1172 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v41, 8, v45
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1176 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:188
+; ALIGNED-NEXT:    buffer_load_ubyte v115, v2, s[0:3], 0 offen offset:189
+; ALIGNED-NEXT:    buffer_load_ubyte v116, v2, s[0:3], 0 offen offset:190
+; ALIGNED-NEXT:    buffer_load_ubyte v114, v2, s[0:3], 0 offen offset:191
+; ALIGNED-NEXT:    buffer_load_ubyte v112, v2, s[0:3], 0 offen offset:187
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v115, 8, v117
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v114, 8, v116
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1180 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:184
+; ALIGNED-NEXT:    buffer_load_ubyte v103, v2, s[0:3], 0 offen offset:185
+; ALIGNED-NEXT:    buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:186
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v103, 8, v113
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v112, 8, v102
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1184 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_ubyte v100, v2, s[0:3], 0 offen offset:192
+; ALIGNED-NEXT:    buffer_load_ubyte v98, v2, s[0:3], 0 offen offset:193
+; ALIGNED-NEXT:    buffer_load_ubyte v87, v2, s[0:3], 0 offen offset:194
+; ALIGNED-NEXT:    buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:195
+; ALIGNED-NEXT:    buffer_load_ubyte v99, v2, s[0:3], 0 offen offset:196
+; ALIGNED-NEXT:    buffer_load_ubyte v97, v2, s[0:3], 0 offen offset:197
+; ALIGNED-NEXT:    buffer_load_ubyte v96, v2, s[0:3], 0 offen offset:198
+; ALIGNED-NEXT:    buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:199
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v98, 8, v100
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v86, 8, v87
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v85, 8, v96
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v97, 8, v99
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:204
+; ALIGNED-NEXT:    buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:205
+; ALIGNED-NEXT:    buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:206
+; ALIGNED-NEXT:    buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:207
+; ALIGNED-NEXT:    buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:203
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v81, 8, v83
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v80, 8, v82
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1196 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:200
+; ALIGNED-NEXT:    buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:201
+; ALIGNED-NEXT:    buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:202
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v69, 8, v70
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v71, 8, v68
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1200 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:212
+; ALIGNED-NEXT:    buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:213
+; ALIGNED-NEXT:    buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:214
+; ALIGNED-NEXT:    buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:215
+; ALIGNED-NEXT:    buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:211
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v54, 8, v67
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v52, 8, v65
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1204 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:216
+; ALIGNED-NEXT:    buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:217
+; ALIGNED-NEXT:    buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:218
+; ALIGNED-NEXT:    buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:219
+; ALIGNED-NEXT:    buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:220
+; ALIGNED-NEXT:    buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:221
+; ALIGNED-NEXT:    buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:222
+; ALIGNED-NEXT:    buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:223
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v53, 8, v66
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v48, 8, v49
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v51, 8, v64
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v39, 8, v50
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1208 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1212 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:208
+; ALIGNED-NEXT:    buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:209
+; ALIGNED-NEXT:    buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:210
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v36, 8, v38
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v55, 8, v37
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1216 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:224
+; ALIGNED-NEXT:    buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:225
+; ALIGNED-NEXT:    buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:226
+; ALIGNED-NEXT:    buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:227
+; ALIGNED-NEXT:    buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:228
+; ALIGNED-NEXT:    buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:229
+; ALIGNED-NEXT:    buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:230
+; ALIGNED-NEXT:    buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:231
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v33, 8, v35
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v30, 8, v29
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v31, 8, v34
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v28, 8, v32
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1220 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x17
+; ALIGNED-NEXT:    buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:236
+; ALIGNED-NEXT:    buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:237
+; ALIGNED-NEXT:    buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:238
+; ALIGNED-NEXT:    buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:239
+; ALIGNED-NEXT:    buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:235
+; ALIGNED-NEXT:    buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:232
+; ALIGNED-NEXT:    buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:233
+; ALIGNED-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:234
+; ALIGNED-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:240
+; ALIGNED-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:241
+; ALIGNED-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:242
+; ALIGNED-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:243
+; ALIGNED-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:244
+; ALIGNED-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:245
+; ALIGNED-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:246
+; ALIGNED-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:247
+; ALIGNED-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:252
+; ALIGNED-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:253
+; ALIGNED-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:254
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:255
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:251
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:248
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:249
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:250
+; ALIGNED-NEXT:    v_lshl_or_b32 v124, v4, 16, v3
+; ALIGNED-NEXT:    s_clause 0x5
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen
+; ALIGNED-NEXT:    buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:2
+; ALIGNED-NEXT:    buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:4
+; ALIGNED-NEXT:    buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:5
+; ALIGNED-NEXT:    buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:6
+; ALIGNED-NEXT:    buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:7
+; ALIGNED-NEXT:    s_waitcnt vmcnt(28)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v25, 8, v27
+; ALIGNED-NEXT:    s_waitcnt vmcnt(26)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v24, 8, v26
+; ALIGNED-NEXT:    s_waitcnt vmcnt(14)
+; ALIGNED-NEXT:    v_lshl_or_b32 v44, v12, 8, v16
+; ALIGNED-NEXT:    s_waitcnt vmcnt(10)
+; ALIGNED-NEXT:    v_lshl_or_b32 v58, v8, 8, v10
+; ALIGNED-NEXT:    v_lshl_or_b32 v104, v4, 16, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v21, 8, v22
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v23, 8, v20
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v88, off, s[0:3], s32 offset:1088 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v90, off, s[0:3], s32 offset:1096 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v92, off, s[0:3], s32 offset:1100 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v77, v4, 16, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v17, 8, v19
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v14, 8, v13
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v95, off, s[0:3], s32 offset:1108 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v101, v4, 16, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v15, 8, v18
+; ALIGNED-NEXT:    v_lshl_or_b32 v84, v44, 16, v4
+; ALIGNED-NEXT:    v_lshl_or_b32 v44, v9, 8, v11
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v58, 16, v44
+; ALIGNED-NEXT:    v_lshl_or_b32 v44, v5, 8, v6
+; ALIGNED-NEXT:    v_lshl_or_b32 v58, v7, 8, v1
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v58, 16, v44
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:1
+; ALIGNED-NEXT:    buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:3
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1068 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v94, off, s[0:3], s32 offset:1092 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:1076 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:1080 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v44, v44, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v58, v58, 8, v94
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v58, 16, v44
+; ALIGNED-NEXT:    v_lshl_or_b32 v44, v90, 8, v88
+; ALIGNED-NEXT:    v_lshl_or_b32 v58, v95, 8, v92
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1120 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v58, 16, v44
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:12
+; ALIGNED-NEXT:    buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:13
+; ALIGNED-NEXT:    buffer_load_ubyte v120, v2, s[0:3], 0 offen offset:14
+; ALIGNED-NEXT:    buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:15
+; ALIGNED-NEXT:    buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:11
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v44, v111, 8, v122
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v58, v110, 8, v120
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v58, 16, v44
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1140 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:8
+; ALIGNED-NEXT:    buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:9
+; ALIGNED-NEXT:    buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:10
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v44, v92, 8, v95
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v58, v94, 8, v90
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v58, 16, v44
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1148 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:16
+; ALIGNED-NEXT:    buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:18
+; ALIGNED-NEXT:    buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:17
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT:    buffer_store_dword v84, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT:    buffer_store_dword v101, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT:    v_add_nc_u32_e32 v2, 0x100, v2
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v127, 8, v44
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v127, v58, 8, v88
+; ALIGNED-NEXT:    v_lshl_or_b32 v127, v0, 16, v127
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1228 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_add_co_u32 v3, vcc_lo, v0, s4
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1224 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_add_u32 s4, s4, 0x100
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, s5, v0, vcc_lo
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v1 offset:250
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v7 offset:251
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v5 offset:249
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v8 offset:255
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v9 offset:253
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v10 offset:254
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v11 offset:252
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v6 offset:248
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v13 offset:242
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v14 offset:243
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v17 offset:241
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v12 offset:247
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v15 offset:245
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v16 offset:246
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v18 offset:244
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v19 offset:240
+; ALIGNED-NEXT:    buffer_store_dword v77, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT:    buffer_store_dword v104, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT:    buffer_store_dword v124, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1220 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_addc_u32 s5, s5, 0
+; ALIGNED-NEXT:    v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
+; ALIGNED-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v20 offset:234
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v23 offset:235
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v21 offset:233
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v24 offset:239
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v25 offset:237
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v26 offset:238
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v27 offset:236
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v22 offset:232
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v29 offset:226
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v30 offset:227
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v33 offset:225
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v28 offset:231
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v31 offset:229
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v32 offset:230
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v34 offset:228
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v35 offset:224
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1216 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1212 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1208 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1204 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v54 offset:213
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v52 offset:215
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v36 offset:209
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v55 offset:211
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v37 offset:210
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v65 offset:214
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v67 offset:212
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v49 offset:218
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v48 offset:219
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v53 offset:217
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v39 offset:223
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v51 offset:221
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v50 offset:222
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v64 offset:220
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v66 offset:216
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v38 offset:208
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1200 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1196 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v68 offset:202
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v71 offset:203
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v69 offset:201
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v80 offset:207
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v81 offset:205
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v82 offset:206
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v83 offset:204
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v70 offset:200
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v87 offset:194
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v86 offset:195
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v98 offset:193
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v85 offset:199
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v97 offset:197
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v96 offset:198
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v99 offset:196
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v100 offset:192
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1184 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1180 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:300
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1176 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1172 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v102 offset:186
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v112 offset:187
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v103 offset:185
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v114 offset:191
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v115 offset:189
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v116 offset:190
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v117 offset:188
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v113 offset:184
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v119 offset:178
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v40 offset:179
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v43 offset:177
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v118 offset:183
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v41 offset:181
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v42 offset:182
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v45 offset:180
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v46 offset:176
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1168 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1164 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1160 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:308
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1156 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:304
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v47 offset:170
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v57 offset:171
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v56 offset:169
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v60 offset:175
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v61 offset:173
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v62 offset:174
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v63 offset:172
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v59 offset:168
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v73 offset:162
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v74 offset:163
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v78 offset:161
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v72 offset:167
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v75 offset:165
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v76 offset:166
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v79 offset:164
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v89 offset:160
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1144 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:264
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1136 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1132 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1124 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:256
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v91 offset:154
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v106 offset:155
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v93 offset:153
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v107 offset:159
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v109 offset:157
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v108 offset:158
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v121 offset:156
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v105 offset:152
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v125 offset:146
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v126 offset:147
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1084 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:145
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v123 offset:151
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:149
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1116 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:150
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1104 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:148
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1072 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:144
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1064 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1020 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1060 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:138
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1056 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:139
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:137
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:143
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:141
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:142
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1024 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:140
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1044 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:136
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1004 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:130
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:996 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:131
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:992 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:129
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1016 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:135
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1008 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:133
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1012 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:134
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1000 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:132
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:128
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:360
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:968 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:364
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:948 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:356
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:352
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:980 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:122
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:976 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:123
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:972 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:121
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:127
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:125
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:126
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:944 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:124
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:120
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:114
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:115
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:113
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:119
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:117
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:118
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:116
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:112
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:376
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:380
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:372
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:368
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:106
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:107
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:105
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:111
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:109
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:110
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:108
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:104
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:98
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:99
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:97
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:103
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:101
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:102
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:100
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:96
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:328
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:332
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:324
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:90
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:91
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:89
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:95
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:93
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:94
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:92
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:88
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:82
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:83
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:81
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:87
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:85
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:86
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:84
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:80
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:344
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:348
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:340
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:336
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:74
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:75
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:73
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:79
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:77
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:78
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:76
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:72
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:66
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:67
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:65
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:71
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:69
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:70
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:68
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:64
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:424
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:428
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:420
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:416
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:61
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:58
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:59
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:57
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:63
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:62
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:60
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:56
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:53
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:50
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:51
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:49
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:55
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:54
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:52
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:48
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:444
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:440
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:436
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:432
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:43
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:42
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:41
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:40
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:47
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:46
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:45
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:44
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:35
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:34
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:33
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:32
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:39
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:38
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:37
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:36
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:392
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:396
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:388
+; ALIGNED-NEXT:    buffer_store_dword v127, off, s[0:3], s32 offset:384
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:26
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:27
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:25
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:31
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:29
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:30
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:28
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:24
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v44 offset:18
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1152 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:19
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v58 offset:17
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:23
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:21
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:22
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:20
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v88 offset:16
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1148 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:408
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1140 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:412
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1128 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:404
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1120 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:400
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v90 offset:10
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v94 offset:11
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v111 offset:13
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v92 offset:9
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v110 offset:15
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v120 offset:14
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v122 offset:12
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v95 offset:8
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1092 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:2
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1080 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:3
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1076 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:1
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1108 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:7
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1096 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:5
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1100 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:6
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1088 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:4
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1068 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0
+; ALIGNED-NEXT:    s_cbranch_vccnz .LBB4_1
+; ALIGNED-NEXT:  ; %bb.2: ; %memcpy-split
+; ALIGNED-NEXT:    s_clause 0x2f
+; ALIGNED-NEXT:    buffer_load_dword v127, off, s[0:3], s32
+; ALIGNED-NEXT:    buffer_load_dword v126, off, s[0:3], s32 offset:4
+; ALIGNED-NEXT:    buffer_load_dword v125, off, s[0:3], s32 offset:8
+; ALIGNED-NEXT:    buffer_load_dword v124, off, s[0:3], s32 offset:12
+; ALIGNED-NEXT:    buffer_load_dword v123, off, s[0:3], s32 offset:16
+; ALIGNED-NEXT:    buffer_load_dword v122, off, s[0:3], s32 offset:20
+; ALIGNED-NEXT:    buffer_load_dword v121, off, s[0:3], s32 offset:24
+; ALIGNED-NEXT:    buffer_load_dword v120, off, s[0:3], s32 offset:28
+; ALIGNED-NEXT:    buffer_load_dword v111, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT:    buffer_load_dword v110, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT:    buffer_load_dword v109, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT:    buffer_load_dword v108, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT:    buffer_load_dword v107, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT:    buffer_load_dword v106, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT:    buffer_load_dword v105, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT:    buffer_load_dword v104, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT:    buffer_load_dword v95, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT:    buffer_load_dword v94, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT:    buffer_load_dword v93, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT:    buffer_load_dword v92, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT:    buffer_load_dword v91, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT:    buffer_load_dword v90, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT:    buffer_load_dword v89, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT:    buffer_load_dword v88, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT:    buffer_load_dword v79, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT:    buffer_load_dword v78, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT:    buffer_load_dword v77, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT:    buffer_load_dword v76, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT:    buffer_load_dword v75, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT:    buffer_load_dword v74, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT:    buffer_load_dword v73, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT:    buffer_load_dword v72, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; UNROLL3-LABEL: memcpy_p0_p5_sz2048:
+; UNROLL3:       ; %bb.0: ; %entry
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    v_mov_b32_e32 v3, v2
+; UNROLL3-NEXT:    s_mov_b64 s[4:5], 0
+; UNROLL3-NEXT:    s_inst_prefetch 0x1
+; UNROLL3-NEXT:    .p2align 6
+; UNROLL3-NEXT:  .LBB4_1: ; %load-store-loop
+; UNROLL3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT:    s_clause 0xb
+; UNROLL3-NEXT:    buffer_load_dword v4, v3, s[0:3], 0 offen
+; UNROLL3-NEXT:    buffer_load_dword v5, v3, s[0:3], 0 offen offset:4
+; UNROLL3-NEXT:    buffer_load_dword v6, v3, s[0:3], 0 offen offset:8
+; UNROLL3-NEXT:    buffer_load_dword v7, v3, s[0:3], 0 offen offset:12
+; UNROLL3-NEXT:    buffer_load_dword v8, v3, s[0:3], 0 offen offset:16
+; UNROLL3-NEXT:    buffer_load_dword v9, v3, s[0:3], 0 offen offset:20
+; UNROLL3-NEXT:    buffer_load_dword v10, v3, s[0:3], 0 offen offset:24
+; UNROLL3-NEXT:    buffer_load_dword v11, v3, s[0:3], 0 offen offset:28
+; UNROLL3-NEXT:    buffer_load_dword v12, v3, s[0:3], 0 offen offset:32
+; UNROLL3-NEXT:    buffer_load_dword v13, v3, s[0:3], 0 offen offset:36
+; UNROLL3-NEXT:    buffer_load_dword v14, v3, s[0:3], 0 offen offset:40
+; UNROLL3-NEXT:    buffer_load_dword v15, v3, s[0:3], 0 offen offset:44
+; UNROLL3-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; UNROLL3-NEXT:    s_add_u32 s4, s4, 48
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; UNROLL3-NEXT:    s_addc_u32 s5, s5, 0
+; UNROLL3-NEXT:    v_add_nc_u32_e32 v3, 48, v3
+; UNROLL3-NEXT:    v_cmp_gt_u64_e64 s6, 0x7e0, s[4:5]
+; UNROLL3-NEXT:    s_waitcnt vmcnt(4)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[8:11] offset:16
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[4:7]
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[12:15] offset:32
+; UNROLL3-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
+; UNROLL3-NEXT:    s_cbranch_vccnz .LBB4_1
+; UNROLL3-NEXT:  ; %bb.2: ; %memcpy-split
+; UNROLL3-NEXT:    s_inst_prefetch 0x2
+; UNROLL3-NEXT:    s_clause 0x3
+; UNROLL3-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:2016
+; UNROLL3-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:2020
+; UNROLL3-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:2024
+; UNROLL3-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:2028
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:2016
+; UNROLL3-NEXT:    s_clause 0x3
+; UNROLL3-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:2032
+; UNROLL3-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:2036
+; UNROLL3-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:2040
+; UNROLL3-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:2044
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:2032
+; UNROLL3-NEXT:    s_waitcnt lgkmcnt(0)
+; UNROLL3-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  tail call void @llvm.memcpy.p0.p5.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 2048, i1 false)
+  ret void
+}
+
+
+; memmove for address spaces 0, 1, 4, 5
+
+define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0) align 1 readonly %src) {
+; CHECK-LABEL: memmove_p0_p0_sz2048:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s4, exec_lo
+; CHECK-NEXT:    v_cmpx_ge_u64_e64 v[2:3], v[0:1]
+; CHECK-NEXT:    s_xor_b32 s6, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execz .LBB5_3
+; CHECK-NEXT:  ; %bb.1: ; %memmove_fwd_loop.preheader
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0
+; CHECK-NEXT:  .LBB5_2: ; %memmove_fwd_loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    v_add_co_u32 v96, vcc_lo, v2, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v97, vcc_lo, s5, v3, vcc_lo
+; CHECK-NEXT:    v_add_co_u32 v100, vcc_lo, v0, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v101, vcc_lo, s5, v1, vcc_lo
+; CHECK-NEXT:    s_clause 0xf
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[96:97] offset:224
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[96:97] offset:240
+; CHECK-NEXT:    flat_load_dwordx4 v[12:15], v[96:97] offset:192
+; CHECK-NEXT:    flat_load_dwordx4 v[16:19], v[96:97] offset:208
+; CHECK-NEXT:    flat_load_dwordx4 v[20:23], v[96:97] offset:160
+; CHECK-NEXT:    flat_load_dwordx4 v[24:27], v[96:97] offset:176
+; CHECK-NEXT:    flat_load_dwordx4 v[28:31], v[96:97] offset:128
+; CHECK-NEXT:    flat_load_dwordx4 v[32:35], v[96:97] offset:144
+; CHECK-NEXT:    flat_load_dwordx4 v[36:39], v[96:97] offset:96
+; CHECK-NEXT:    flat_load_dwordx4 v[48:51], v[96:97] offset:112
+; CHECK-NEXT:    flat_load_dwordx4 v[52:55], v[96:97] offset:64
+; CHECK-NEXT:    flat_load_dwordx4 v[64:67], v[96:97] offset:80
+; CHECK-NEXT:    flat_load_dwordx4 v[68:71], v[96:97] offset:32
+; CHECK-NEXT:    flat_load_dwordx4 v[80:83], v[96:97] offset:48
+; CHECK-NEXT:    flat_load_dwordx4 v[84:87], v[96:97]
+; CHECK-NEXT:    flat_load_dwordx4 v[96:99], v[96:97] offset:16
+; CHECK-NEXT:    s_add_u32 s4, s4, 0x100
+; CHECK-NEXT:    s_addc_u32 s5, s5, 0
+; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[4:7] offset:224
+; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[8:11] offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[12:15] offset:192
+; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[16:19] offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[20:23] offset:160
+; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[24:27] offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[28:31] offset:128
+; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[32:35] offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[36:39] offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[48:51] offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[52:55] offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[64:67] offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[68:71] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[80:83] offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[84:87]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[96:99] offset:16
+; CHECK-NEXT:    s_cmp_lg_u64 s[4:5], 0x800
+; CHECK-NEXT:    s_cbranch_scc1 .LBB5_2
+; CHECK-NEXT:  .LBB5_3: ; %Flow5
+; CHECK-NEXT:    s_andn2_saveexec_b32 s8, s6
+; CHECK-NEXT:    s_cbranch_execz .LBB5_6
+; CHECK-NEXT:  ; %bb.4: ; %memmove_bwd_loop.preheader
+; CHECK-NEXT:    s_movk_i32 s6, 0xff00
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0x700
+; CHECK-NEXT:    s_mov_b32 s7, -1
+; CHECK-NEXT:  .LBB5_5: ; %memmove_bwd_loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    v_add_co_u32 v96, vcc_lo, v2, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v97, vcc_lo, s5, v3, vcc_lo
+; CHECK-NEXT:    v_add_co_u32 v100, vcc_lo, v0, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v101, vcc_lo, s5, v1, vcc_lo
+; CHECK-NEXT:    s_clause 0xf
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[96:97] offset:224
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[96:97] offset:240
+; CHECK-NEXT:    flat_load_dwordx4 v[12:15], v[96:97] offset:192
+; CHECK-NEXT:    flat_load_dwordx4 v[16:19], v[96:97] offset:208
+; CHECK-NEXT:    flat_load_dwordx4 v[20:23], v[96:97] offset:160
+; CHECK-NEXT:    flat_load_dwordx4 v[24:27], v[96:97] offset:176
+; CHECK-NEXT:    flat_load_dwordx4 v[28:31], v[96:97] offset:128
+; CHECK-NEXT:    flat_load_dwordx4 v[32:35], v[96:97] offset:144
+; CHECK-NEXT:    flat_load_dwordx4 v[36:39], v[96:97] offset:96
+; CHECK-NEXT:    flat_load_dwordx4 v[48:51], v[96:97] offset:112
+; CHECK-NEXT:    flat_load_dwordx4 v[52:55], v[96:97] offset:64
+; CHECK-NEXT:    flat_load_dwordx4 v[64:67], v[96:97] offset:80
+; CHECK-NEXT:    flat_load_dwordx4 v[68:71], v[96:97] offset:32
+; CHECK-NEXT:    flat_load_dwordx4 v[80:83], v[96:97] offset:48
+; CHECK-NEXT:    flat_load_dwordx4 v[84:87], v[96:97]
+; CHECK-NEXT:    flat_load_dwordx4 v[96:99], v[96:97] offset:16
+; CHECK-NEXT:    s_add_u32 s4, s4, 0xffffff00
+; CHECK-NEXT:    s_addc_u32 s5, s5, -1
+; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[4:7] offset:224
+; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[8:11] offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[12:15] offset:192
+; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[16:19] offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[20:23] offset:160
+; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[24:27] offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[28:31] offset:128
+; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[32:35] offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[36:39] offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[48:51] offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[52:55] offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[64:67] offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[68:71] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[80:83] offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[84:87]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[96:99] offset:16
+; CHECK-NEXT:    s_cmp_eq_u64 s[4:5], s[6:7]
+; CHECK-NEXT:    s_cbranch_scc0 .LBB5_5
+; CHECK-NEXT:  .LBB5_6: ; %Flow6
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+;
+; ALIGNED-LABEL: memmove_p0_p0_sz2048:
+; ALIGNED:       ; %bb.0: ; %entry
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_mov_b32 s4, exec_lo
+; ALIGNED-NEXT:    v_cmpx_ge_u64_e64 v[2:3], v[0:1]
+; ALIGNED-NEXT:    s_xor_b32 s6, exec_lo, s4
+; ALIGNED-NEXT:    s_cbranch_execz .LBB5_3
+; ALIGNED-NEXT:  ; %bb.1: ; %memmove_fwd_loop.preheader
+; ALIGNED-NEXT:    s_mov_b64 s[4:5], 0
+; ALIGNED-NEXT:  .LBB5_2: ; %memmove_fwd_loop
+; ALIGNED-NEXT:    ; =>This Inner Loop Header: Depth=1
+; ALIGNED-NEXT:    v_add_co_u32 v20, vcc_lo, v2, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v21, vcc_lo, s5, v3, vcc_lo
+; ALIGNED-NEXT:    s_clause 0xf
+; ALIGNED-NEXT:    flat_load_dwordx4 v[16:19], v[20:21] offset:240
+; ALIGNED-NEXT:    flat_load_dwordx4 v[22:25], v[20:21] offset:224
+; ALIGNED-NEXT:    flat_load_dwordx4 v[4:7], v[20:21]
+; ALIGNED-NEXT:    flat_load_dwordx4 v[8:11], v[20:21] offset:16
+; ALIGNED-NEXT:    flat_load_dwordx4 v[12:15], v[20:21] offset:32
+; ALIGNED-NEXT:    flat_load_dwordx4 v[98:101], v[20:21] offset:48
+; ALIGNED-NEXT:    flat_load_dwordx4 v[112:115], v[20:21] offset:64
+; ALIGNED-NEXT:    flat_load_dwordx4 v[82:85], v[20:21] offset:80
+; ALIGNED-NEXT:    flat_load_dwordx4 v[116:119], v[20:21] offset:96
+; ALIGNED-NEXT:    flat_load_dwordx4 v[66:69], v[20:21] offset:112
+; ALIGNED-NEXT:    flat_load_dwordx4 v[40:43], v[20:21] offset:128
+; ALIGNED-NEXT:    flat_load_dwordx4 v[50:53], v[20:21] offset:144
+; ALIGNED-NEXT:    flat_load_dwordx4 v[44:47], v[20:21] offset:160
+; ALIGNED-NEXT:    flat_load_dwordx4 v[34:37], v[20:21] offset:176
+; ALIGNED-NEXT:    flat_load_dwordx4 v[30:33], v[20:21] offset:192
+; ALIGNED-NEXT:    flat_load_dwordx4 v[26:29], v[20:21] offset:208
+; ALIGNED-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
+; ALIGNED-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; ALIGNED-NEXT:    s_add_u32 s4, s4, 0x100
+; ALIGNED-NEXT:    s_addc_u32 s5, s5, 0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v21 offset:254
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v21 offset:252
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v20 offset:250
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v20 offset:248
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v19 offset:246
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v19 offset:244
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v18 offset:242
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v18 offset:240
+; ALIGNED-NEXT:    s_waitcnt lgkmcnt(22)
+; ALIGNED-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v21
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v21, 8, v21
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v20
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v20, 8, v20
+; ALIGNED-NEXT:    s_cmp_lg_u64 s[4:5], 0x800
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v25 offset:238
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v25 offset:236
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v24 offset:234
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v24 offset:232
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v23 offset:230
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v23 offset:228
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v22 offset:226
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v22 offset:224
+; ALIGNED-NEXT:    s_waitcnt lgkmcnt(16)
+; ALIGNED-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v29 offset:222
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v29 offset:220
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v28 offset:218
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v28 offset:216
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v27 offset:214
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v27 offset:212
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v26 offset:210
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v26 offset:208
+; ALIGNED-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v33 offset:206
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v33 offset:204
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v32 offset:202
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v32 offset:200
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v31 offset:198
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v31 offset:196
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v30 offset:194
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v30 offset:192
+; ALIGNED-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT:    buffer_store_dword v35, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT:    buffer_store_dword v36, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT:    buffer_store_dword v37, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v37 offset:190
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v37 offset:188
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v36 offset:186
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v36 offset:184
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v35 offset:182
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v35 offset:180
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v34 offset:178
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v34 offset:176
+; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v49 offset:174
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v49 offset:172
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v48 offset:170
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v48 offset:168
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v39 offset:166
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v39 offset:164
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v38 offset:162
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v38 offset:160
+; ALIGNED-NEXT:    buffer_store_dword v50, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT:    buffer_store_dword v51, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT:    buffer_store_dword v52, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT:    buffer_store_dword v53, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v53 offset:158
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v53 offset:156
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v52 offset:154
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v52 offset:152
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v51 offset:150
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v51 offset:148
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v50 offset:146
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v50 offset:144
+; ALIGNED-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v65, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT:    buffer_load_dword v64, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v65 offset:142
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v65 offset:140
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v64 offset:138
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v64 offset:136
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v55 offset:134
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v55 offset:132
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v54 offset:130
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v54 offset:128
+; ALIGNED-NEXT:    buffer_store_dword v66, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT:    buffer_store_dword v67, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT:    buffer_store_dword v68, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT:    buffer_store_dword v69, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v69, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT:    buffer_load_dword v68, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT:    buffer_load_dword v67, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT:    buffer_load_dword v66, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v69 offset:126
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v69 offset:124
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v68 offset:122
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v68 offset:120
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v67 offset:118
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v67 offset:116
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v66 offset:114
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v66 offset:112
+; ALIGNED-NEXT:    buffer_store_dword v116, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT:    buffer_store_dword v117, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT:    buffer_store_dword v118, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT:    buffer_store_dword v119, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v81, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT:    buffer_load_dword v80, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT:    buffer_load_dword v71, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT:    buffer_load_dword v70, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v81 offset:110
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v81 offset:108
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v80 offset:106
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v80 offset:104
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v71 offset:102
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v71 offset:100
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v70 offset:98
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v70 offset:96
+; ALIGNED-NEXT:    buffer_store_dword v82, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT:    buffer_store_dword v83, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT:    buffer_store_dword v84, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT:    buffer_store_dword v85, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v85, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT:    buffer_load_dword v84, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT:    buffer_load_dword v83, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT:    buffer_load_dword v82, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v85 offset:94
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v85 offset:92
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v84 offset:90
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v84 offset:88
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v83 offset:86
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v83 offset:84
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v82 offset:82
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v82 offset:80
+; ALIGNED-NEXT:    buffer_store_dword v112, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT:    buffer_store_dword v113, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT:    buffer_store_dword v114, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT:    buffer_store_dword v115, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v97, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT:    buffer_load_dword v96, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT:    buffer_load_dword v87, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT:    buffer_load_dword v86, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v19
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v19, 8, v19
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v18
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v18, 8, v18
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 8, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 8, v24
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v97 offset:78
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v97 offset:76
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v96 offset:74
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v96 offset:72
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v87 offset:70
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v87 offset:68
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v86 offset:66
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v86 offset:64
+; ALIGNED-NEXT:    buffer_store_dword v98, off, s[0:3], s32 offset:256
+; ALIGNED-NEXT:    buffer_store_dword v99, off, s[0:3], s32 offset:260
+; ALIGNED-NEXT:    buffer_store_dword v100, off, s[0:3], s32 offset:264
+; ALIGNED-NEXT:    buffer_store_dword v101, off, s[0:3], s32 offset:268
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v101, off, s[0:3], s32 offset:268
+; ALIGNED-NEXT:    buffer_load_dword v100, off, s[0:3], s32 offset:264
+; ALIGNED-NEXT:    buffer_load_dword v99, off, s[0:3], s32 offset:260
+; ALIGNED-NEXT:    buffer_load_dword v98, off, s[0:3], s32 offset:256
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v101 offset:62
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v101 offset:60
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v100 offset:58
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v100 offset:56
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v99 offset:54
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v99 offset:52
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v98 offset:50
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v98 offset:48
+; ALIGNED-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:272
+; ALIGNED-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:276
+; ALIGNED-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:280
+; ALIGNED-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:284
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:280
+; ALIGNED-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:284
+; ALIGNED-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:272
+; ALIGNED-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:276
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v15 offset:42
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v15 offset:40
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v14 offset:46
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v14 offset:44
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v13 offset:34
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v13 offset:32
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v12 offset:38
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v12 offset:36
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v11 offset:30
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v11 offset:28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v10 offset:26
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v10 offset:24
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v9 offset:22
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v9 offset:20
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v8 offset:18
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v8 offset:16
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v112 offset:247
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v27
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v102 offset:255
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v23
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v23, 8, v23
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v21 offset:253
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v21, 24, v22
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v22, 8, v22
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v103 offset:251
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v29
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v29, 8, v29
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v20 offset:249
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v20, 24, v28
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v28, 8, v28
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v27, 8, v27
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v19 offset:245
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v19, 24, v26
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v26, 8, v26
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v112 offset:215
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v51
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 8, v51
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v113 offset:243
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v33
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v33, 8, v33
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v18 offset:241
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v18, 24, v32
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v32, 8, v32
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v114 offset:239
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v31
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 8, v31
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v25 offset:237
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 24, v30
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 8, v30
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v115 offset:235
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v37
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v37, 8, v37
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v24 offset:233
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 24, v36
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 8, v36
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v102 offset:231
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v35
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v35, 8, v35
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v23 offset:229
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v23, 24, v34
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v34, 8, v34
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v21 offset:227
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v21, 24, v49
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 8, v49
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v22 offset:225
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v22, 24, v48
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v48, 8, v48
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v103 offset:223
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v39
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v39, 8, v39
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v29 offset:221
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v29, 24, v38
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v38, 8, v38
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v20 offset:219
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v20, 24, v53
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v53, 8, v53
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v28 offset:217
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v28, 24, v52
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v52, 8, v52
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v27 offset:213
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v27, 24, v50
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 8, v50
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v19 offset:211
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v19, 24, v65
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 8, v65
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v26 offset:209
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v26, 24, v64
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v51 offset:149
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 24, v8
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v8, 8, v8
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 8, v64
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v113 offset:207
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v55
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v55, 8, v55
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v33 offset:205
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v33, 24, v54
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v54, 8, v54
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v18 offset:203
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v18, 24, v69
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v69, 8, v69
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v32 offset:201
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v32, 24, v68
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v68, 8, v68
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v114 offset:199
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v67
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 8, v67
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v31 offset:197
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 24, v66
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 8, v66
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v25 offset:195
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 24, v81
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 8, v81
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v30 offset:193
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v80
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 8, v80
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v115 offset:191
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v71
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 8, v71
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v37 offset:189
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v37, 24, v70
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 8, v70
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v24 offset:187
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 24, v85
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v85
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v36 offset:185
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 24, v84
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 8, v84
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v102 offset:183
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v83
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 8, v83
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v35 offset:181
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v35, 24, v82
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 8, v82
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v23 offset:179
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v23, 24, v97
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v97, 8, v97
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v34 offset:177
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v34, 24, v96
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v96, 8, v96
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v21 offset:175
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v21, 24, v87
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v87
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v49 offset:173
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 24, v86
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 8, v86
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v22 offset:171
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v22, 24, v101
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 8, v101
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v48 offset:169
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v48, 24, v100
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v100
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v103 offset:167
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v99
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 8, v99
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v39 offset:165
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v39, 24, v98
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v98
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v29 offset:163
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v29, 24, v15
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v15, 8, v15
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v38 offset:161
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v38, 24, v14
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v14, 8, v14
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v20 offset:159
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v20, 24, v13
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v13, 8, v13
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v53 offset:157
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v53, 24, v12
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v12, 8, v12
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v28 offset:155
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v28, 24, v11
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v11, 8, v11
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v52 offset:153
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v52, 24, v10
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v10, 8, v10
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v112 offset:151
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v9
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v9, 8, v9
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v27 offset:147
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v50 offset:145
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v19 offset:143
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v65 offset:141
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v26 offset:139
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v64 offset:137
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v113 offset:135
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v55 offset:133
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v33 offset:131
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v54 offset:129
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v18 offset:127
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v69 offset:125
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v32 offset:123
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v68 offset:121
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v114 offset:119
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v67 offset:117
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v31 offset:115
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v66 offset:113
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v25 offset:111
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v81 offset:109
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v30 offset:107
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v80 offset:105
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v115 offset:103
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v71 offset:101
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v37 offset:99
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v70 offset:97
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v24 offset:95
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v85 offset:93
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v36 offset:91
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v84 offset:89
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v102 offset:87
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v83 offset:85
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v35 offset:83
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v82 offset:81
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v23 offset:79
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v97 offset:77
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v34 offset:75
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v96 offset:73
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v21 offset:71
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v87 offset:69
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v49 offset:67
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v86 offset:65
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v22 offset:63
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v101 offset:61
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v48 offset:59
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v100 offset:57
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v103 offset:55
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v99 offset:53
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v39 offset:51
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v98 offset:49
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v29 offset:43
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v15 offset:41
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v38 offset:47
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v14 offset:45
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v20 offset:35
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v13 offset:33
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v53 offset:39
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v12 offset:37
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v28 offset:31
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v11 offset:29
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v52 offset:27
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v10 offset:25
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v112 offset:23
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v9 offset:21
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v51 offset:19
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v8 offset:17
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v7 offset:14
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v7 offset:12
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v6 offset:10
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v6 offset:8
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v5 offset:6
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v5 offset:4
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v4 offset:2
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v4
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v8, 24, v7
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v7, 8, v7
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v9, 24, v6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v6, 8, v6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v10, 24, v5
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v5, 8, v5
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v11, 24, v4
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v4, 8, v4
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v8 offset:15
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v7 offset:13
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v9 offset:11
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v6 offset:9
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v10 offset:7
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v5 offset:5
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v11 offset:3
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v4 offset:1
+; ALIGNED-NEXT:    s_cbranch_scc1 .LBB5_2
+; ALIGNED-NEXT:  .LBB5_3: ; %Flow5
+; ALIGNED-NEXT:    s_andn2_saveexec_b32 s8, s6
+; ALIGNED-NEXT:    s_cbranch_execz .LBB5_6
+; ALIGNED-NEXT:  ; %bb.4: ; %memmove_bwd_loop.preheader
+; ALIGNED-NEXT:    s_movk_i32 s6, 0xff00
+; ALIGNED-NEXT:    s_mov_b64 s[4:5], 0x700
+; ALIGNED-NEXT:    s_mov_b32 s7, -1
+; ALIGNED-NEXT:  .LBB5_5: ; %memmove_bwd_loop
+; ALIGNED-NEXT:    ; =>This Inner Loop Header: Depth=1
+; ALIGNED-NEXT:    v_add_co_u32 v24, vcc_lo, v2, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v25, vcc_lo, s5, v3, vcc_lo
+; ALIGNED-NEXT:    s_clause 0xf
+; ALIGNED-NEXT:    flat_load_dwordx4 v[16:19], v[24:25] offset:240
+; ALIGNED-NEXT:    flat_load_dwordx4 v[20:23], v[24:25] offset:224
+; ALIGNED-NEXT:    flat_load_dwordx4 v[4:7], v[24:25]
+; ALIGNED-NEXT:    flat_load_dwordx4 v[8:11], v[24:25] offset:16
+; ALIGNED-NEXT:    flat_load_dwordx4 v[12:15], v[24:25] offset:32
+; ALIGNED-NEXT:    flat_load_dwordx4 v[112:115], v[24:25] offset:48
+; ALIGNED-NEXT:    flat_load_dwordx4 v[116:119], v[24:25] offset:64
+; ALIGNED-NEXT:    flat_load_dwordx4 v[40:43], v[24:25] offset:80
+; ALIGNED-NEXT:    flat_load_dwordx4 v[26:29], v[24:25] offset:96
+; ALIGNED-NEXT:    flat_load_dwordx4 v[32:35], v[24:25] offset:112
+; ALIGNED-NEXT:    flat_load_dwordx4 v[44:47], v[24:25] offset:128
+; ALIGNED-NEXT:    flat_load_dwordx4 v[52:55], v[24:25] offset:144
+; ALIGNED-NEXT:    flat_load_dwordx4 v[66:69], v[24:25] offset:160
+; ALIGNED-NEXT:    flat_load_dwordx4 v[81:84], v[24:25] offset:176
+; ALIGNED-NEXT:    flat_load_dwordx4 v[96:99], v[24:25] offset:192
+; ALIGNED-NEXT:    flat_load_dwordx4 v[100:103], v[24:25] offset:208
+; ALIGNED-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
+; ALIGNED-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:320
+; ALIGNED-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:324
+; ALIGNED-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:328
+; ALIGNED-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:332
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:332
+; ALIGNED-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:328
+; ALIGNED-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:324
+; ALIGNED-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:320
+; ALIGNED-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; ALIGNED-NEXT:    s_add_u32 s4, s4, 0xffffff00
+; ALIGNED-NEXT:    s_addc_u32 s5, s5, -1
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v31 offset:254
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v31 offset:252
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v30 offset:250
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v30 offset:248
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v25 offset:246
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v25 offset:244
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v24 offset:242
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v24 offset:240
+; ALIGNED-NEXT:    s_waitcnt lgkmcnt(22)
+; ALIGNED-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:336
+; ALIGNED-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:340
+; ALIGNED-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:344
+; ALIGNED-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:348
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:348
+; ALIGNED-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:344
+; ALIGNED-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:340
+; ALIGNED-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:336
+; ALIGNED-NEXT:    s_cmp_eq_u64 s[4:5], s[6:7]
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v51 offset:238
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v51 offset:236
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v50 offset:234
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v50 offset:232
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v49 offset:230
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v49 offset:228
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v36 offset:226
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v36 offset:224
+; ALIGNED-NEXT:    s_waitcnt lgkmcnt(16)
+; ALIGNED-NEXT:    buffer_store_dword v100, off, s[0:3], s32 offset:288
+; ALIGNED-NEXT:    buffer_store_dword v101, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT:    buffer_store_dword v102, off, s[0:3], s32 offset:296
+; ALIGNED-NEXT:    buffer_store_dword v103, off, s[0:3], s32 offset:300
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v71, off, s[0:3], s32 offset:300
+; ALIGNED-NEXT:    buffer_load_dword v70, off, s[0:3], s32 offset:296
+; ALIGNED-NEXT:    buffer_load_dword v65, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT:    buffer_load_dword v64, off, s[0:3], s32 offset:288
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v31
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 8, v31
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v30
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 8, v30
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v71 offset:222
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v71 offset:220
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v70 offset:218
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v70 offset:216
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v65 offset:214
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v65 offset:212
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v64 offset:210
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v64 offset:208
+; ALIGNED-NEXT:    buffer_store_dword v96, off, s[0:3], s32 offset:304
+; ALIGNED-NEXT:    buffer_store_dword v97, off, s[0:3], s32 offset:308
+; ALIGNED-NEXT:    buffer_store_dword v98, off, s[0:3], s32 offset:312
+; ALIGNED-NEXT:    buffer_store_dword v99, off, s[0:3], s32 offset:316
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v87, off, s[0:3], s32 offset:316
+; ALIGNED-NEXT:    buffer_load_dword v86, off, s[0:3], s32 offset:312
+; ALIGNED-NEXT:    buffer_load_dword v85, off, s[0:3], s32 offset:308
+; ALIGNED-NEXT:    buffer_load_dword v80, off, s[0:3], s32 offset:304
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v87 offset:206
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v87 offset:204
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v86 offset:202
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v86 offset:200
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v85 offset:198
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v85 offset:196
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v80 offset:194
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v80 offset:192
+; ALIGNED-NEXT:    buffer_store_dword v81, off, s[0:3], s32 offset:384
+; ALIGNED-NEXT:    buffer_store_dword v82, off, s[0:3], s32 offset:388
+; ALIGNED-NEXT:    buffer_store_dword v83, off, s[0:3], s32 offset:392
+; ALIGNED-NEXT:    buffer_store_dword v84, off, s[0:3], s32 offset:396
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v101, off, s[0:3], s32 offset:396
+; ALIGNED-NEXT:    buffer_load_dword v99, off, s[0:3], s32 offset:392
+; ALIGNED-NEXT:    buffer_load_dword v96, off, s[0:3], s32 offset:388
+; ALIGNED-NEXT:    buffer_load_dword v81, off, s[0:3], s32 offset:384
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v101 offset:190
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v101 offset:188
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v99 offset:186
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v99 offset:184
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v96 offset:182
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v96 offset:180
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v81 offset:178
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v81 offset:176
+; ALIGNED-NEXT:    buffer_store_dword v66, off, s[0:3], s32 offset:400
+; ALIGNED-NEXT:    buffer_store_dword v67, off, s[0:3], s32 offset:404
+; ALIGNED-NEXT:    buffer_store_dword v68, off, s[0:3], s32 offset:408
+; ALIGNED-NEXT:    buffer_store_dword v69, off, s[0:3], s32 offset:412
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v100, off, s[0:3], s32 offset:412
+; ALIGNED-NEXT:    buffer_load_dword v97, off, s[0:3], s32 offset:408
+; ALIGNED-NEXT:    buffer_load_dword v82, off, s[0:3], s32 offset:404
+; ALIGNED-NEXT:    buffer_load_dword v66, off, s[0:3], s32 offset:400
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v100 offset:174
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v100 offset:172
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v97 offset:170
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v97 offset:168
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v82 offset:166
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v82 offset:164
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v66 offset:162
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v66 offset:160
+; ALIGNED-NEXT:    buffer_store_dword v52, off, s[0:3], s32 offset:352
+; ALIGNED-NEXT:    buffer_store_dword v53, off, s[0:3], s32 offset:356
+; ALIGNED-NEXT:    buffer_store_dword v54, off, s[0:3], s32 offset:360
+; ALIGNED-NEXT:    buffer_store_dword v55, off, s[0:3], s32 offset:364
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v98, off, s[0:3], s32 offset:364
+; ALIGNED-NEXT:    buffer_load_dword v83, off, s[0:3], s32 offset:360
+; ALIGNED-NEXT:    buffer_load_dword v67, off, s[0:3], s32 offset:356
+; ALIGNED-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:352
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v98 offset:158
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v98 offset:156
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v83 offset:154
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v83 offset:152
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v67 offset:150
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v67 offset:148
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v52 offset:146
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v52 offset:144
+; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:368
+; ALIGNED-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:372
+; ALIGNED-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:376
+; ALIGNED-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:380
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v84, off, s[0:3], s32 offset:380
+; ALIGNED-NEXT:    buffer_load_dword v68, off, s[0:3], s32 offset:376
+; ALIGNED-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:372
+; ALIGNED-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:368
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v84 offset:142
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v84 offset:140
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v68 offset:138
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v68 offset:136
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v53 offset:134
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v53 offset:132
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v37 offset:130
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v37 offset:128
+; ALIGNED-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:448
+; ALIGNED-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:452
+; ALIGNED-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:456
+; ALIGNED-NEXT:    buffer_store_dword v35, off, s[0:3], s32 offset:460
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v69, off, s[0:3], s32 offset:460
+; ALIGNED-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:456
+; ALIGNED-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:452
+; ALIGNED-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:448
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v69 offset:126
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v69 offset:124
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v54 offset:122
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v54 offset:120
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v38 offset:118
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v38 offset:116
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v32 offset:114
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v32 offset:112
+; ALIGNED-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:464
+; ALIGNED-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:468
+; ALIGNED-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:472
+; ALIGNED-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:476
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:476
+; ALIGNED-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:472
+; ALIGNED-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:468
+; ALIGNED-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:464
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v55 offset:110
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v55 offset:108
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v39 offset:106
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v39 offset:104
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v33 offset:102
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v33 offset:100
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v26 offset:98
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v26 offset:96
+; ALIGNED-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:416
+; ALIGNED-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:420
+; ALIGNED-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:424
+; ALIGNED-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:428
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:428
+; ALIGNED-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:424
+; ALIGNED-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:420
+; ALIGNED-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:416
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v48 offset:94
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v48 offset:92
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v34 offset:90
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v34 offset:88
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v27 offset:86
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v27 offset:84
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v21 offset:82
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v21 offset:80
+; ALIGNED-NEXT:    buffer_store_dword v116, off, s[0:3], s32 offset:432
+; ALIGNED-NEXT:    buffer_store_dword v117, off, s[0:3], s32 offset:436
+; ALIGNED-NEXT:    buffer_store_dword v118, off, s[0:3], s32 offset:440
+; ALIGNED-NEXT:    buffer_store_dword v119, off, s[0:3], s32 offset:444
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:444
+; ALIGNED-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:440
+; ALIGNED-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:436
+; ALIGNED-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:432
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v35 offset:78
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v35 offset:76
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v28 offset:74
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v28 offset:72
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v22 offset:70
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v22 offset:68
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v19 offset:66
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v19 offset:64
+; ALIGNED-NEXT:    buffer_store_dword v112, off, s[0:3], s32 offset:512
+; ALIGNED-NEXT:    buffer_store_dword v113, off, s[0:3], s32 offset:516
+; ALIGNED-NEXT:    buffer_store_dword v114, off, s[0:3], s32 offset:520
+; ALIGNED-NEXT:    buffer_store_dword v115, off, s[0:3], s32 offset:524
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:524
+; ALIGNED-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:520
+; ALIGNED-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:516
+; ALIGNED-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:512
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 8, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 8, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v51
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 8, v51
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v50
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 8, v50
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v29 offset:62
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v29 offset:60
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v23 offset:58
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v23 offset:56
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v20 offset:54
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v20 offset:52
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v18 offset:50
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v18 offset:48
+; ALIGNED-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:528
+; ALIGNED-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:532
+; ALIGNED-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:536
+; ALIGNED-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:540
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:536
+; ALIGNED-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:540
+; ALIGNED-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:528
+; ALIGNED-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:532
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v15 offset:42
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v15 offset:40
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v14 offset:46
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v14 offset:44
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v13 offset:34
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v13 offset:32
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v12 offset:38
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v12 offset:36
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:480
+; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:484
+; ALIGNED-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:488
+; ALIGNED-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:492
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:492
+; ALIGNED-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:488
+; ALIGNED-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:484
+; ALIGNED-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:480
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v11 offset:30
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v11 offset:28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v10 offset:26
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v10 offset:24
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v9 offset:22
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v9 offset:20
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v8 offset:18
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v8 offset:16
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:496
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:500
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:504
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:508
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:508
+; ALIGNED-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:504
+; ALIGNED-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:500
+; ALIGNED-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:496
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v112 offset:247
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v65
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v102 offset:255
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v49
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 8, v49
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v31 offset:253
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 24, v36
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 8, v36
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v103 offset:251
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v71
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 8, v71
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v30 offset:249
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v70
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 8, v70
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 8, v65
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v25 offset:245
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 24, v64
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 8, v64
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v112 offset:215
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v67
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 8, v67
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v113 offset:243
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v87
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v87
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v24 offset:241
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 24, v86
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 8, v86
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v114 offset:239
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v85
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v85
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v51 offset:237
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 24, v80
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 8, v80
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v115 offset:235
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v101
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 8, v101
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v50 offset:233
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 24, v99
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 8, v99
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v102 offset:231
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v96
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v96, 8, v96
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v49 offset:229
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 24, v81
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 8, v81
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v31 offset:227
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 24, v100
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v100
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v36 offset:225
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 24, v97
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v97, 8, v97
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v103 offset:223
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v82
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 8, v82
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v71 offset:221
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 24, v66
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 8, v66
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v30 offset:219
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v98
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v98
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v70 offset:217
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 24, v83
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 8, v83
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v65 offset:213
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 24, v52
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v52, 8, v52
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v25 offset:211
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 24, v84
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 8, v84
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v64 offset:209
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 24, v68
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v67 offset:149
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 24, v8
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v8, 8, v8
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v68, 8, v68
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v113 offset:207
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v53
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v53, 8, v53
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v87 offset:205
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 24, v37
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v37, 8, v37
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v24 offset:203
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 24, v69
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v69, 8, v69
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v86 offset:201
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v54
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v54, 8, v54
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v114 offset:199
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v38
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v38, 8, v38
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v85 offset:197
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 24, v32
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v32, 8, v32
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v51 offset:195
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 24, v55
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v55, 8, v55
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v80 offset:193
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 24, v39
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v39, 8, v39
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v115 offset:191
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v33
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v33, 8, v33
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v101 offset:189
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 24, v26
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v26, 8, v26
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v50 offset:187
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 24, v48
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v48, 8, v48
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v99 offset:185
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 24, v34
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v34, 8, v34
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v102 offset:183
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v27
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v27, 8, v27
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v96 offset:181
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v96, 24, v21
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v21, 8, v21
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v49 offset:179
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 24, v35
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v35, 8, v35
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v81 offset:177
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 24, v28
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v28, 8, v28
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v31 offset:175
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 24, v22
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v22, 8, v22
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v100 offset:173
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 24, v19
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v19, 8, v19
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v36 offset:171
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 24, v29
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v29, 8, v29
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v97 offset:169
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v97, 24, v23
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v23, 8, v23
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v103 offset:167
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v20
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v20, 8, v20
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v82 offset:165
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 24, v18
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v18, 8, v18
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v71 offset:163
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 24, v15
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v15, 8, v15
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v66 offset:161
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 24, v14
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v14, 8, v14
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v30 offset:159
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v13
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v13, 8, v13
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v98 offset:157
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 24, v12
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v12, 8, v12
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v70 offset:155
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 24, v11
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v11, 8, v11
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v83 offset:153
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 24, v10
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v10, 8, v10
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v112 offset:151
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v9
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v9, 8, v9
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v65 offset:147
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v52 offset:145
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v25 offset:143
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v84 offset:141
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v64 offset:139
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v68 offset:137
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v113 offset:135
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v53 offset:133
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v87 offset:131
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v37 offset:129
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v24 offset:127
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v69 offset:125
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v86 offset:123
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v54 offset:121
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v114 offset:119
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v38 offset:117
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v85 offset:115
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v32 offset:113
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v51 offset:111
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v55 offset:109
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v80 offset:107
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v39 offset:105
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v115 offset:103
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v33 offset:101
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v101 offset:99
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v26 offset:97
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v50 offset:95
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v48 offset:93
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v99 offset:91
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v34 offset:89
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v102 offset:87
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v27 offset:85
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v96 offset:83
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v21 offset:81
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v49 offset:79
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v35 offset:77
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v81 offset:75
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v28 offset:73
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v31 offset:71
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v22 offset:69
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v100 offset:67
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v19 offset:65
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v36 offset:63
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v29 offset:61
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v97 offset:59
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v23 offset:57
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v103 offset:55
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v20 offset:53
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v82 offset:51
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v18 offset:49
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v71 offset:43
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v15 offset:41
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v66 offset:47
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v14 offset:45
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v30 offset:35
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v13 offset:33
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v98 offset:39
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v12 offset:37
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v70 offset:31
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v11 offset:29
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v83 offset:27
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v10 offset:25
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v112 offset:23
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v9 offset:21
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v67 offset:19
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v8 offset:17
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v7 offset:14
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v7 offset:12
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v6 offset:10
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v6 offset:8
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v5 offset:6
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v5 offset:4
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[16:17], v4 offset:2
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v4
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v8, 24, v7
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v7, 8, v7
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v9, 24, v6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v6, 8, v6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v10, 24, v5
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v5, 8, v5
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v11, 24, v4
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v4, 8, v4
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v8 offset:15
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v7 offset:13
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v9 offset:11
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v6 offset:9
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v10 offset:7
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v5 offset:5
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v11 offset:3
+; ALIGNED-NEXT:    flat_store_byte v[16:17], v4 offset:1
+; ALIGNED-NEXT:    s_cbranch_scc0 .LBB5_5
+; ALIGNED-NEXT:  .LBB5_6: ; %Flow6
+; ALIGNED-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_dword v47, off, s[0:3], s32
+; ALIGNED-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4
+; ALIGNED-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8
+; ALIGNED-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12
+; ALIGNED-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16
+; ALIGNED-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20
+; ALIGNED-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24
+; ALIGNED-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; UNROLL3-LABEL: memmove_p0_p0_sz2048:
+; UNROLL3:       ; %bb.0: ; %entry
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    s_mov_b32 s4, exec_lo
+; UNROLL3-NEXT:    v_cmpx_ge_u64_e64 v[2:3], v[0:1]
+; UNROLL3-NEXT:    s_xor_b32 s6, exec_lo, s4
+; UNROLL3-NEXT:    s_cbranch_execz .LBB5_4
+; UNROLL3-NEXT:  ; %bb.1: ; %memmove_fwd_loop.preheader
+; UNROLL3-NEXT:    s_mov_b64 s[4:5], 0
+; UNROLL3-NEXT:    .p2align 6
+; UNROLL3-NEXT:  .LBB5_2: ; %memmove_fwd_loop
+; UNROLL3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT:    v_add_co_u32 v12, vcc_lo, v2, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v13, vcc_lo, s5, v3, vcc_lo
+; UNROLL3-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; UNROLL3-NEXT:    s_clause 0x2
+; UNROLL3-NEXT:    flat_load_dwordx4 v[4:7], v[12:13]
+; UNROLL3-NEXT:    flat_load_dwordx4 v[8:11], v[12:13] offset:16
+; UNROLL3-NEXT:    flat_load_dwordx4 v[12:15], v[12:13] offset:32
+; UNROLL3-NEXT:    s_add_u32 s4, s4, 48
+; UNROLL3-NEXT:    s_addc_u32 s5, s5, 0
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[4:7]
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(2)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[8:11] offset:16
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(2)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[12:15] offset:32
+; UNROLL3-NEXT:    s_cmp_lg_u64 s[4:5], 0x7e0
+; UNROLL3-NEXT:    s_cbranch_scc1 .LBB5_2
+; UNROLL3-NEXT:  ; %bb.3: ; %memmove_fwd_residual
+; UNROLL3-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:2016
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:2016
+; UNROLL3-NEXT:    flat_load_dwordx4 v[2:5], v[2:3] offset:2032
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[2:5] offset:2032
+; UNROLL3-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; UNROLL3-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; UNROLL3-NEXT:  .LBB5_4: ; %Flow3
+; UNROLL3-NEXT:    s_andn2_saveexec_b32 s8, s6
+; UNROLL3-NEXT:    s_cbranch_execz .LBB5_7
+; UNROLL3-NEXT:  ; %bb.5: ; %memmove_bwd_residual
+; UNROLL3-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:2032
+; UNROLL3-NEXT:    s_movk_i32 s6, 0xffd0
+; UNROLL3-NEXT:    s_mov_b64 s[4:5], 0x7b0
+; UNROLL3-NEXT:    s_mov_b32 s7, -1
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:2032
+; UNROLL3-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:2016
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:2016
+; UNROLL3-NEXT:    .p2align 6
+; UNROLL3-NEXT:  .LBB5_6: ; %memmove_bwd_loop
+; UNROLL3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT:    v_add_co_u32 v12, vcc_lo, v2, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v13, vcc_lo, s5, v3, vcc_lo
+; UNROLL3-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; UNROLL3-NEXT:    s_clause 0x2
+; UNROLL3-NEXT:    flat_load_dwordx4 v[4:7], v[12:13]
+; UNROLL3-NEXT:    flat_load_dwordx4 v[8:11], v[12:13] offset:16
+; UNROLL3-NEXT:    flat_load_dwordx4 v[12:15], v[12:13] offset:32
+; UNROLL3-NEXT:    s_add_u32 s4, s4, 0xffffffd0
+; UNROLL3-NEXT:    s_addc_u32 s5, s5, -1
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[4:7]
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(2)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[8:11] offset:16
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(2)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[12:15] offset:32
+; UNROLL3-NEXT:    s_cmp_eq_u64 s[4:5], s[6:7]
+; UNROLL3-NEXT:    s_cbranch_scc0 .LBB5_6
+; UNROLL3-NEXT:  .LBB5_7: ; %Flow4
+; UNROLL3-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; UNROLL3-NEXT:    s_waitcnt lgkmcnt(0)
+; UNROLL3-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  tail call void @llvm.memmove.p0.p0.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 2048, i1 false)
+  ret void
+}
+
+define void @memmove_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {
+; CHECK-LABEL: memmove_p1_p1_sz2048:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s4, exec_lo
+; CHECK-NEXT:    v_cmpx_ge_u64_e64 v[2:3], v[0:1]
+; CHECK-NEXT:    s_xor_b32 s6, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execz .LBB6_3
+; CHECK-NEXT:  ; %bb.1: ; %memmove_fwd_loop.preheader
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0
+; CHECK-NEXT:  .LBB6_2: ; %memmove_fwd_loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    v_add_co_u32 v96, vcc_lo, v2, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v97, vcc_lo, s5, v3, vcc_lo
+; CHECK-NEXT:    v_add_co_u32 v100, vcc_lo, v0, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v101, vcc_lo, s5, v1, vcc_lo
+; CHECK-NEXT:    s_clause 0xf
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[96:97], off offset:224
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[96:97], off offset:240
+; CHECK-NEXT:    global_load_dwordx4 v[12:15], v[96:97], off offset:192
+; CHECK-NEXT:    global_load_dwordx4 v[16:19], v[96:97], off offset:208
+; CHECK-NEXT:    global_load_dwordx4 v[20:23], v[96:97], off offset:160
+; CHECK-NEXT:    global_load_dwordx4 v[24:27], v[96:97], off offset:176
+; CHECK-NEXT:    global_load_dwordx4 v[28:31], v[96:97], off offset:128
+; CHECK-NEXT:    global_load_dwordx4 v[32:35], v[96:97], off offset:144
+; CHECK-NEXT:    global_load_dwordx4 v[36:39], v[96:97], off offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[48:51], v[96:97], off offset:112
+; CHECK-NEXT:    global_load_dwordx4 v[52:55], v[96:97], off offset:64
+; CHECK-NEXT:    global_load_dwordx4 v[64:67], v[96:97], off offset:80
+; CHECK-NEXT:    global_load_dwordx4 v[68:71], v[96:97], off offset:32
+; CHECK-NEXT:    global_load_dwordx4 v[80:83], v[96:97], off offset:48
+; CHECK-NEXT:    global_load_dwordx4 v[84:87], v[96:97], off
+; CHECK-NEXT:    global_load_dwordx4 v[96:99], v[96:97], off offset:16
+; CHECK-NEXT:    s_add_u32 s4, s4, 0x100
+; CHECK-NEXT:    s_addc_u32 s5, s5, 0
+; CHECK-NEXT:    s_waitcnt vmcnt(15)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[4:7], off offset:224
+; CHECK-NEXT:    s_waitcnt vmcnt(14)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[8:11], off offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(13)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[12:15], off offset:192
+; CHECK-NEXT:    s_waitcnt vmcnt(12)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[16:19], off offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(11)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[20:23], off offset:160
+; CHECK-NEXT:    s_waitcnt vmcnt(10)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[24:27], off offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(9)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[28:31], off offset:128
+; CHECK-NEXT:    s_waitcnt vmcnt(8)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[32:35], off offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(7)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[36:39], off offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(6)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[48:51], off offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[52:55], off offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[64:67], off offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[68:71], off offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(2)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[80:83], off offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[84:87], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[96:99], off offset:16
+; CHECK-NEXT:    s_cmp_lg_u64 s[4:5], 0x800
+; CHECK-NEXT:    s_cbranch_scc1 .LBB6_2
+; CHECK-NEXT:  .LBB6_3: ; %Flow9
+; CHECK-NEXT:    s_andn2_saveexec_b32 s8, s6
+; CHECK-NEXT:    s_cbranch_execz .LBB6_6
+; CHECK-NEXT:  ; %bb.4: ; %memmove_bwd_loop.preheader
+; CHECK-NEXT:    s_movk_i32 s6, 0xff00
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0x700
+; CHECK-NEXT:    s_mov_b32 s7, -1
+; CHECK-NEXT:  .LBB6_5: ; %memmove_bwd_loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    v_add_co_u32 v96, vcc_lo, v2, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v97, vcc_lo, s5, v3, vcc_lo
+; CHECK-NEXT:    v_add_co_u32 v100, vcc_lo, v0, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v101, vcc_lo, s5, v1, vcc_lo
+; CHECK-NEXT:    s_clause 0xf
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[96:97], off offset:224
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[96:97], off offset:240
+; CHECK-NEXT:    global_load_dwordx4 v[12:15], v[96:97], off offset:192
+; CHECK-NEXT:    global_load_dwordx4 v[16:19], v[96:97], off offset:208
+; CHECK-NEXT:    global_load_dwordx4 v[20:23], v[96:97], off offset:160
+; CHECK-NEXT:    global_load_dwordx4 v[24:27], v[96:97], off offset:176
+; CHECK-NEXT:    global_load_dwordx4 v[28:31], v[96:97], off offset:128
+; CHECK-NEXT:    global_load_dwordx4 v[32:35], v[96:97], off offset:144
+; CHECK-NEXT:    global_load_dwordx4 v[36:39], v[96:97], off offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[48:51], v[96:97], off offset:112
+; CHECK-NEXT:    global_load_dwordx4 v[52:55], v[96:97], off offset:64
+; CHECK-NEXT:    global_load_dwordx4 v[64:67], v[96:97], off offset:80
+; CHECK-NEXT:    global_load_dwordx4 v[68:71], v[96:97], off offset:32
+; CHECK-NEXT:    global_load_dwordx4 v[80:83], v[96:97], off offset:48
+; CHECK-NEXT:    global_load_dwordx4 v[84:87], v[96:97], off
+; CHECK-NEXT:    global_load_dwordx4 v[96:99], v[96:97], off offset:16
+; CHECK-NEXT:    s_add_u32 s4, s4, 0xffffff00
+; CHECK-NEXT:    s_addc_u32 s5, s5, -1
+; CHECK-NEXT:    s_waitcnt vmcnt(15)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[4:7], off offset:224
+; CHECK-NEXT:    s_waitcnt vmcnt(14)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[8:11], off offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(13)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[12:15], off offset:192
+; CHECK-NEXT:    s_waitcnt vmcnt(12)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[16:19], off offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(11)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[20:23], off offset:160
+; CHECK-NEXT:    s_waitcnt vmcnt(10)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[24:27], off offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(9)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[28:31], off offset:128
+; CHECK-NEXT:    s_waitcnt vmcnt(8)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[32:35], off offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(7)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[36:39], off offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(6)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[48:51], off offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[52:55], off offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[64:67], off offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[68:71], off offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(2)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[80:83], off offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[84:87], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v[100:101], v[96:99], off offset:16
+; CHECK-NEXT:    s_cmp_eq_u64 s[4:5], s[6:7]
+; CHECK-NEXT:    s_cbranch_scc0 .LBB6_5
+; CHECK-NEXT:  .LBB6_6: ; %Flow10
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+;
+; ALIGNED-LABEL: memmove_p1_p1_sz2048:
+; ALIGNED:       ; %bb.0: ; %entry
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_mov_b32 s4, exec_lo
+; ALIGNED-NEXT:    v_cmpx_ge_u64_e64 v[2:3], v[0:1]
+; ALIGNED-NEXT:    s_xor_b32 s6, exec_lo, s4
+; ALIGNED-NEXT:    s_cbranch_execz .LBB6_3
+; ALIGNED-NEXT:  ; %bb.1: ; %memmove_fwd_loop.preheader
+; ALIGNED-NEXT:    s_mov_b64 s[4:5], 0
+; ALIGNED-NEXT:  .LBB6_2: ; %memmove_fwd_loop
+; ALIGNED-NEXT:    ; =>This Inner Loop Header: Depth=1
+; ALIGNED-NEXT:    v_add_co_u32 v20, vcc_lo, v2, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v21, vcc_lo, s5, v3, vcc_lo
+; ALIGNED-NEXT:    s_clause 0xf
+; ALIGNED-NEXT:    global_load_dwordx4 v[16:19], v[20:21], off offset:240
+; ALIGNED-NEXT:    global_load_dwordx4 v[22:25], v[20:21], off offset:224
+; ALIGNED-NEXT:    global_load_dwordx4 v[4:7], v[20:21], off
+; ALIGNED-NEXT:    global_load_dwordx4 v[8:11], v[20:21], off offset:16
+; ALIGNED-NEXT:    global_load_dwordx4 v[12:15], v[20:21], off offset:32
+; ALIGNED-NEXT:    global_load_dwordx4 v[98:101], v[20:21], off offset:48
+; ALIGNED-NEXT:    global_load_dwordx4 v[112:115], v[20:21], off offset:64
+; ALIGNED-NEXT:    global_load_dwordx4 v[82:85], v[20:21], off offset:80
+; ALIGNED-NEXT:    global_load_dwordx4 v[116:119], v[20:21], off offset:96
+; ALIGNED-NEXT:    global_load_dwordx4 v[66:69], v[20:21], off offset:112
+; ALIGNED-NEXT:    global_load_dwordx4 v[40:43], v[20:21], off offset:128
+; ALIGNED-NEXT:    global_load_dwordx4 v[50:53], v[20:21], off offset:144
+; ALIGNED-NEXT:    global_load_dwordx4 v[44:47], v[20:21], off offset:160
+; ALIGNED-NEXT:    global_load_dwordx4 v[34:37], v[20:21], off offset:176
+; ALIGNED-NEXT:    global_load_dwordx4 v[30:33], v[20:21], off offset:192
+; ALIGNED-NEXT:    global_load_dwordx4 v[26:29], v[20:21], off offset:208
+; ALIGNED-NEXT:    s_waitcnt vmcnt(15)
+; ALIGNED-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; ALIGNED-NEXT:    s_add_u32 s4, s4, 0x100
+; ALIGNED-NEXT:    s_addc_u32 s5, s5, 0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v21, off offset:254
+; ALIGNED-NEXT:    global_store_byte v[16:17], v21, off offset:252
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v20, off offset:250
+; ALIGNED-NEXT:    global_store_byte v[16:17], v20, off offset:248
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v19, off offset:246
+; ALIGNED-NEXT:    global_store_byte v[16:17], v19, off offset:244
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v18, off offset:242
+; ALIGNED-NEXT:    global_store_byte v[16:17], v18, off offset:240
+; ALIGNED-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v21
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v21, 8, v21
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v20
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v20, 8, v20
+; ALIGNED-NEXT:    s_cmp_lg_u64 s[4:5], 0x800
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v25, off offset:238
+; ALIGNED-NEXT:    global_store_byte v[16:17], v25, off offset:236
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v24, off offset:234
+; ALIGNED-NEXT:    global_store_byte v[16:17], v24, off offset:232
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v23, off offset:230
+; ALIGNED-NEXT:    global_store_byte v[16:17], v23, off offset:228
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v22, off offset:226
+; ALIGNED-NEXT:    global_store_byte v[16:17], v22, off offset:224
+; ALIGNED-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v29, off offset:222
+; ALIGNED-NEXT:    global_store_byte v[16:17], v29, off offset:220
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v28, off offset:218
+; ALIGNED-NEXT:    global_store_byte v[16:17], v28, off offset:216
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v27, off offset:214
+; ALIGNED-NEXT:    global_store_byte v[16:17], v27, off offset:212
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v26, off offset:210
+; ALIGNED-NEXT:    global_store_byte v[16:17], v26, off offset:208
+; ALIGNED-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v33, off offset:206
+; ALIGNED-NEXT:    global_store_byte v[16:17], v33, off offset:204
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v32, off offset:202
+; ALIGNED-NEXT:    global_store_byte v[16:17], v32, off offset:200
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v31, off offset:198
+; ALIGNED-NEXT:    global_store_byte v[16:17], v31, off offset:196
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v30, off offset:194
+; ALIGNED-NEXT:    global_store_byte v[16:17], v30, off offset:192
+; ALIGNED-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT:    buffer_store_dword v35, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT:    buffer_store_dword v36, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT:    buffer_store_dword v37, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v37, off offset:190
+; ALIGNED-NEXT:    global_store_byte v[16:17], v37, off offset:188
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v36, off offset:186
+; ALIGNED-NEXT:    global_store_byte v[16:17], v36, off offset:184
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v35, off offset:182
+; ALIGNED-NEXT:    global_store_byte v[16:17], v35, off offset:180
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v34, off offset:178
+; ALIGNED-NEXT:    global_store_byte v[16:17], v34, off offset:176
+; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v49, off offset:174
+; ALIGNED-NEXT:    global_store_byte v[16:17], v49, off offset:172
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v48, off offset:170
+; ALIGNED-NEXT:    global_store_byte v[16:17], v48, off offset:168
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v39, off offset:166
+; ALIGNED-NEXT:    global_store_byte v[16:17], v39, off offset:164
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v38, off offset:162
+; ALIGNED-NEXT:    global_store_byte v[16:17], v38, off offset:160
+; ALIGNED-NEXT:    buffer_store_dword v50, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT:    buffer_store_dword v51, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT:    buffer_store_dword v52, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT:    buffer_store_dword v53, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v53, off offset:158
+; ALIGNED-NEXT:    global_store_byte v[16:17], v53, off offset:156
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v52, off offset:154
+; ALIGNED-NEXT:    global_store_byte v[16:17], v52, off offset:152
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v51, off offset:150
+; ALIGNED-NEXT:    global_store_byte v[16:17], v51, off offset:148
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v50, off offset:146
+; ALIGNED-NEXT:    global_store_byte v[16:17], v50, off offset:144
+; ALIGNED-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v65, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT:    buffer_load_dword v64, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v65, off offset:142
+; ALIGNED-NEXT:    global_store_byte v[16:17], v65, off offset:140
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v64, off offset:138
+; ALIGNED-NEXT:    global_store_byte v[16:17], v64, off offset:136
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v55, off offset:134
+; ALIGNED-NEXT:    global_store_byte v[16:17], v55, off offset:132
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v54, off offset:130
+; ALIGNED-NEXT:    global_store_byte v[16:17], v54, off offset:128
+; ALIGNED-NEXT:    buffer_store_dword v66, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT:    buffer_store_dword v67, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT:    buffer_store_dword v68, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT:    buffer_store_dword v69, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v69, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT:    buffer_load_dword v68, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT:    buffer_load_dword v67, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT:    buffer_load_dword v66, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v69, off offset:126
+; ALIGNED-NEXT:    global_store_byte v[16:17], v69, off offset:124
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v68, off offset:122
+; ALIGNED-NEXT:    global_store_byte v[16:17], v68, off offset:120
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v67, off offset:118
+; ALIGNED-NEXT:    global_store_byte v[16:17], v67, off offset:116
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v66, off offset:114
+; ALIGNED-NEXT:    global_store_byte v[16:17], v66, off offset:112
+; ALIGNED-NEXT:    buffer_store_dword v116, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT:    buffer_store_dword v117, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT:    buffer_store_dword v118, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT:    buffer_store_dword v119, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v81, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT:    buffer_load_dword v80, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT:    buffer_load_dword v71, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT:    buffer_load_dword v70, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v81, off offset:110
+; ALIGNED-NEXT:    global_store_byte v[16:17], v81, off offset:108
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v80, off offset:106
+; ALIGNED-NEXT:    global_store_byte v[16:17], v80, off offset:104
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v71, off offset:102
+; ALIGNED-NEXT:    global_store_byte v[16:17], v71, off offset:100
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v70, off offset:98
+; ALIGNED-NEXT:    global_store_byte v[16:17], v70, off offset:96
+; ALIGNED-NEXT:    buffer_store_dword v82, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT:    buffer_store_dword v83, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT:    buffer_store_dword v84, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT:    buffer_store_dword v85, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v85, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT:    buffer_load_dword v84, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT:    buffer_load_dword v83, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT:    buffer_load_dword v82, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v85, off offset:94
+; ALIGNED-NEXT:    global_store_byte v[16:17], v85, off offset:92
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v84, off offset:90
+; ALIGNED-NEXT:    global_store_byte v[16:17], v84, off offset:88
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v83, off offset:86
+; ALIGNED-NEXT:    global_store_byte v[16:17], v83, off offset:84
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v82, off offset:82
+; ALIGNED-NEXT:    global_store_byte v[16:17], v82, off offset:80
+; ALIGNED-NEXT:    buffer_store_dword v112, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT:    buffer_store_dword v113, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT:    buffer_store_dword v114, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT:    buffer_store_dword v115, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v97, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT:    buffer_load_dword v96, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT:    buffer_load_dword v87, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT:    buffer_load_dword v86, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v19
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v19, 8, v19
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v18
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v18, 8, v18
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 8, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 8, v24
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v97, off offset:78
+; ALIGNED-NEXT:    global_store_byte v[16:17], v97, off offset:76
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v96, off offset:74
+; ALIGNED-NEXT:    global_store_byte v[16:17], v96, off offset:72
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v87, off offset:70
+; ALIGNED-NEXT:    global_store_byte v[16:17], v87, off offset:68
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v86, off offset:66
+; ALIGNED-NEXT:    global_store_byte v[16:17], v86, off offset:64
+; ALIGNED-NEXT:    buffer_store_dword v98, off, s[0:3], s32 offset:256
+; ALIGNED-NEXT:    buffer_store_dword v99, off, s[0:3], s32 offset:260
+; ALIGNED-NEXT:    buffer_store_dword v100, off, s[0:3], s32 offset:264
+; ALIGNED-NEXT:    buffer_store_dword v101, off, s[0:3], s32 offset:268
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v101, off, s[0:3], s32 offset:268
+; ALIGNED-NEXT:    buffer_load_dword v100, off, s[0:3], s32 offset:264
+; ALIGNED-NEXT:    buffer_load_dword v99, off, s[0:3], s32 offset:260
+; ALIGNED-NEXT:    buffer_load_dword v98, off, s[0:3], s32 offset:256
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v101, off offset:62
+; ALIGNED-NEXT:    global_store_byte v[16:17], v101, off offset:60
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v100, off offset:58
+; ALIGNED-NEXT:    global_store_byte v[16:17], v100, off offset:56
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v99, off offset:54
+; ALIGNED-NEXT:    global_store_byte v[16:17], v99, off offset:52
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v98, off offset:50
+; ALIGNED-NEXT:    global_store_byte v[16:17], v98, off offset:48
+; ALIGNED-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:272
+; ALIGNED-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:276
+; ALIGNED-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:280
+; ALIGNED-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:284
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:280
+; ALIGNED-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:284
+; ALIGNED-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:272
+; ALIGNED-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:276
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v15, off offset:42
+; ALIGNED-NEXT:    global_store_byte v[16:17], v15, off offset:40
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v14, off offset:46
+; ALIGNED-NEXT:    global_store_byte v[16:17], v14, off offset:44
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v13, off offset:34
+; ALIGNED-NEXT:    global_store_byte v[16:17], v13, off offset:32
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v12, off offset:38
+; ALIGNED-NEXT:    global_store_byte v[16:17], v12, off offset:36
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v11, off offset:30
+; ALIGNED-NEXT:    global_store_byte v[16:17], v11, off offset:28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v10, off offset:26
+; ALIGNED-NEXT:    global_store_byte v[16:17], v10, off offset:24
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v9, off offset:22
+; ALIGNED-NEXT:    global_store_byte v[16:17], v9, off offset:20
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v8, off offset:18
+; ALIGNED-NEXT:    global_store_byte v[16:17], v8, off offset:16
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT:    global_store_byte v[16:17], v112, off offset:247
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v27
+; ALIGNED-NEXT:    global_store_byte v[16:17], v102, off offset:255
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v23
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v23, 8, v23
+; ALIGNED-NEXT:    global_store_byte v[16:17], v21, off offset:253
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v21, 24, v22
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v22, 8, v22
+; ALIGNED-NEXT:    global_store_byte v[16:17], v103, off offset:251
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v29
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v29, 8, v29
+; ALIGNED-NEXT:    global_store_byte v[16:17], v20, off offset:249
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v20, 24, v28
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v28, 8, v28
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v27, 8, v27
+; ALIGNED-NEXT:    global_store_byte v[16:17], v19, off offset:245
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v19, 24, v26
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v26, 8, v26
+; ALIGNED-NEXT:    global_store_byte v[16:17], v112, off offset:215
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v51
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 8, v51
+; ALIGNED-NEXT:    global_store_byte v[16:17], v113, off offset:243
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v33
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v33, 8, v33
+; ALIGNED-NEXT:    global_store_byte v[16:17], v18, off offset:241
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v18, 24, v32
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v32, 8, v32
+; ALIGNED-NEXT:    global_store_byte v[16:17], v114, off offset:239
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v31
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 8, v31
+; ALIGNED-NEXT:    global_store_byte v[16:17], v25, off offset:237
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 24, v30
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 8, v30
+; ALIGNED-NEXT:    global_store_byte v[16:17], v115, off offset:235
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v37
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v37, 8, v37
+; ALIGNED-NEXT:    global_store_byte v[16:17], v24, off offset:233
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 24, v36
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 8, v36
+; ALIGNED-NEXT:    global_store_byte v[16:17], v102, off offset:231
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v35
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v35, 8, v35
+; ALIGNED-NEXT:    global_store_byte v[16:17], v23, off offset:229
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v23, 24, v34
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v34, 8, v34
+; ALIGNED-NEXT:    global_store_byte v[16:17], v21, off offset:227
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v21, 24, v49
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 8, v49
+; ALIGNED-NEXT:    global_store_byte v[16:17], v22, off offset:225
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v22, 24, v48
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v48, 8, v48
+; ALIGNED-NEXT:    global_store_byte v[16:17], v103, off offset:223
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v39
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v39, 8, v39
+; ALIGNED-NEXT:    global_store_byte v[16:17], v29, off offset:221
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v29, 24, v38
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v38, 8, v38
+; ALIGNED-NEXT:    global_store_byte v[16:17], v20, off offset:219
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v20, 24, v53
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v53, 8, v53
+; ALIGNED-NEXT:    global_store_byte v[16:17], v28, off offset:217
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v28, 24, v52
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v52, 8, v52
+; ALIGNED-NEXT:    global_store_byte v[16:17], v27, off offset:213
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v27, 24, v50
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 8, v50
+; ALIGNED-NEXT:    global_store_byte v[16:17], v19, off offset:211
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v19, 24, v65
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 8, v65
+; ALIGNED-NEXT:    global_store_byte v[16:17], v26, off offset:209
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v26, 24, v64
+; ALIGNED-NEXT:    global_store_byte v[16:17], v51, off offset:149
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 24, v8
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v8, 8, v8
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 8, v64
+; ALIGNED-NEXT:    global_store_byte v[16:17], v113, off offset:207
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v55
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v55, 8, v55
+; ALIGNED-NEXT:    global_store_byte v[16:17], v33, off offset:205
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v33, 24, v54
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v54, 8, v54
+; ALIGNED-NEXT:    global_store_byte v[16:17], v18, off offset:203
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v18, 24, v69
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v69, 8, v69
+; ALIGNED-NEXT:    global_store_byte v[16:17], v32, off offset:201
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v32, 24, v68
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v68, 8, v68
+; ALIGNED-NEXT:    global_store_byte v[16:17], v114, off offset:199
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v67
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 8, v67
+; ALIGNED-NEXT:    global_store_byte v[16:17], v31, off offset:197
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 24, v66
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 8, v66
+; ALIGNED-NEXT:    global_store_byte v[16:17], v25, off offset:195
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 24, v81
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 8, v81
+; ALIGNED-NEXT:    global_store_byte v[16:17], v30, off offset:193
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v80
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 8, v80
+; ALIGNED-NEXT:    global_store_byte v[16:17], v115, off offset:191
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v71
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 8, v71
+; ALIGNED-NEXT:    global_store_byte v[16:17], v37, off offset:189
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v37, 24, v70
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 8, v70
+; ALIGNED-NEXT:    global_store_byte v[16:17], v24, off offset:187
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 24, v85
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v85
+; ALIGNED-NEXT:    global_store_byte v[16:17], v36, off offset:185
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 24, v84
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 8, v84
+; ALIGNED-NEXT:    global_store_byte v[16:17], v102, off offset:183
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v83
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 8, v83
+; ALIGNED-NEXT:    global_store_byte v[16:17], v35, off offset:181
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v35, 24, v82
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 8, v82
+; ALIGNED-NEXT:    global_store_byte v[16:17], v23, off offset:179
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v23, 24, v97
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v97, 8, v97
+; ALIGNED-NEXT:    global_store_byte v[16:17], v34, off offset:177
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v34, 24, v96
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v96, 8, v96
+; ALIGNED-NEXT:    global_store_byte v[16:17], v21, off offset:175
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v21, 24, v87
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v87
+; ALIGNED-NEXT:    global_store_byte v[16:17], v49, off offset:173
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 24, v86
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 8, v86
+; ALIGNED-NEXT:    global_store_byte v[16:17], v22, off offset:171
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v22, 24, v101
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 8, v101
+; ALIGNED-NEXT:    global_store_byte v[16:17], v48, off offset:169
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v48, 24, v100
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v100
+; ALIGNED-NEXT:    global_store_byte v[16:17], v103, off offset:167
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v99
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 8, v99
+; ALIGNED-NEXT:    global_store_byte v[16:17], v39, off offset:165
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v39, 24, v98
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v98
+; ALIGNED-NEXT:    global_store_byte v[16:17], v29, off offset:163
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v29, 24, v15
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v15, 8, v15
+; ALIGNED-NEXT:    global_store_byte v[16:17], v38, off offset:161
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v38, 24, v14
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v14, 8, v14
+; ALIGNED-NEXT:    global_store_byte v[16:17], v20, off offset:159
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v20, 24, v13
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v13, 8, v13
+; ALIGNED-NEXT:    global_store_byte v[16:17], v53, off offset:157
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v53, 24, v12
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v12, 8, v12
+; ALIGNED-NEXT:    global_store_byte v[16:17], v28, off offset:155
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v28, 24, v11
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v11, 8, v11
+; ALIGNED-NEXT:    global_store_byte v[16:17], v52, off offset:153
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v52, 24, v10
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v10, 8, v10
+; ALIGNED-NEXT:    global_store_byte v[16:17], v112, off offset:151
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v9
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v9, 8, v9
+; ALIGNED-NEXT:    global_store_byte v[16:17], v27, off offset:147
+; ALIGNED-NEXT:    global_store_byte v[16:17], v50, off offset:145
+; ALIGNED-NEXT:    global_store_byte v[16:17], v19, off offset:143
+; ALIGNED-NEXT:    global_store_byte v[16:17], v65, off offset:141
+; ALIGNED-NEXT:    global_store_byte v[16:17], v26, off offset:139
+; ALIGNED-NEXT:    global_store_byte v[16:17], v64, off offset:137
+; ALIGNED-NEXT:    global_store_byte v[16:17], v113, off offset:135
+; ALIGNED-NEXT:    global_store_byte v[16:17], v55, off offset:133
+; ALIGNED-NEXT:    global_store_byte v[16:17], v33, off offset:131
+; ALIGNED-NEXT:    global_store_byte v[16:17], v54, off offset:129
+; ALIGNED-NEXT:    global_store_byte v[16:17], v18, off offset:127
+; ALIGNED-NEXT:    global_store_byte v[16:17], v69, off offset:125
+; ALIGNED-NEXT:    global_store_byte v[16:17], v32, off offset:123
+; ALIGNED-NEXT:    global_store_byte v[16:17], v68, off offset:121
+; ALIGNED-NEXT:    global_store_byte v[16:17], v114, off offset:119
+; ALIGNED-NEXT:    global_store_byte v[16:17], v67, off offset:117
+; ALIGNED-NEXT:    global_store_byte v[16:17], v31, off offset:115
+; ALIGNED-NEXT:    global_store_byte v[16:17], v66, off offset:113
+; ALIGNED-NEXT:    global_store_byte v[16:17], v25, off offset:111
+; ALIGNED-NEXT:    global_store_byte v[16:17], v81, off offset:109
+; ALIGNED-NEXT:    global_store_byte v[16:17], v30, off offset:107
+; ALIGNED-NEXT:    global_store_byte v[16:17], v80, off offset:105
+; ALIGNED-NEXT:    global_store_byte v[16:17], v115, off offset:103
+; ALIGNED-NEXT:    global_store_byte v[16:17], v71, off offset:101
+; ALIGNED-NEXT:    global_store_byte v[16:17], v37, off offset:99
+; ALIGNED-NEXT:    global_store_byte v[16:17], v70, off offset:97
+; ALIGNED-NEXT:    global_store_byte v[16:17], v24, off offset:95
+; ALIGNED-NEXT:    global_store_byte v[16:17], v85, off offset:93
+; ALIGNED-NEXT:    global_store_byte v[16:17], v36, off offset:91
+; ALIGNED-NEXT:    global_store_byte v[16:17], v84, off offset:89
+; ALIGNED-NEXT:    global_store_byte v[16:17], v102, off offset:87
+; ALIGNED-NEXT:    global_store_byte v[16:17], v83, off offset:85
+; ALIGNED-NEXT:    global_store_byte v[16:17], v35, off offset:83
+; ALIGNED-NEXT:    global_store_byte v[16:17], v82, off offset:81
+; ALIGNED-NEXT:    global_store_byte v[16:17], v23, off offset:79
+; ALIGNED-NEXT:    global_store_byte v[16:17], v97, off offset:77
+; ALIGNED-NEXT:    global_store_byte v[16:17], v34, off offset:75
+; ALIGNED-NEXT:    global_store_byte v[16:17], v96, off offset:73
+; ALIGNED-NEXT:    global_store_byte v[16:17], v21, off offset:71
+; ALIGNED-NEXT:    global_store_byte v[16:17], v87, off offset:69
+; ALIGNED-NEXT:    global_store_byte v[16:17], v49, off offset:67
+; ALIGNED-NEXT:    global_store_byte v[16:17], v86, off offset:65
+; ALIGNED-NEXT:    global_store_byte v[16:17], v22, off offset:63
+; ALIGNED-NEXT:    global_store_byte v[16:17], v101, off offset:61
+; ALIGNED-NEXT:    global_store_byte v[16:17], v48, off offset:59
+; ALIGNED-NEXT:    global_store_byte v[16:17], v100, off offset:57
+; ALIGNED-NEXT:    global_store_byte v[16:17], v103, off offset:55
+; ALIGNED-NEXT:    global_store_byte v[16:17], v99, off offset:53
+; ALIGNED-NEXT:    global_store_byte v[16:17], v39, off offset:51
+; ALIGNED-NEXT:    global_store_byte v[16:17], v98, off offset:49
+; ALIGNED-NEXT:    global_store_byte v[16:17], v29, off offset:43
+; ALIGNED-NEXT:    global_store_byte v[16:17], v15, off offset:41
+; ALIGNED-NEXT:    global_store_byte v[16:17], v38, off offset:47
+; ALIGNED-NEXT:    global_store_byte v[16:17], v14, off offset:45
+; ALIGNED-NEXT:    global_store_byte v[16:17], v20, off offset:35
+; ALIGNED-NEXT:    global_store_byte v[16:17], v13, off offset:33
+; ALIGNED-NEXT:    global_store_byte v[16:17], v53, off offset:39
+; ALIGNED-NEXT:    global_store_byte v[16:17], v12, off offset:37
+; ALIGNED-NEXT:    global_store_byte v[16:17], v28, off offset:31
+; ALIGNED-NEXT:    global_store_byte v[16:17], v11, off offset:29
+; ALIGNED-NEXT:    global_store_byte v[16:17], v52, off offset:27
+; ALIGNED-NEXT:    global_store_byte v[16:17], v10, off offset:25
+; ALIGNED-NEXT:    global_store_byte v[16:17], v112, off offset:23
+; ALIGNED-NEXT:    global_store_byte v[16:17], v9, off offset:21
+; ALIGNED-NEXT:    global_store_byte v[16:17], v51, off offset:19
+; ALIGNED-NEXT:    global_store_byte v[16:17], v8, off offset:17
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v7, off offset:14
+; ALIGNED-NEXT:    global_store_byte v[16:17], v7, off offset:12
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v6, off offset:10
+; ALIGNED-NEXT:    global_store_byte v[16:17], v6, off offset:8
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v5, off offset:6
+; ALIGNED-NEXT:    global_store_byte v[16:17], v5, off offset:4
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v4, off offset:2
+; ALIGNED-NEXT:    global_store_byte v[16:17], v4, off
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v8, 24, v7
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v7, 8, v7
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v9, 24, v6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v6, 8, v6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v10, 24, v5
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v5, 8, v5
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v11, 24, v4
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v4, 8, v4
+; ALIGNED-NEXT:    global_store_byte v[16:17], v8, off offset:15
+; ALIGNED-NEXT:    global_store_byte v[16:17], v7, off offset:13
+; ALIGNED-NEXT:    global_store_byte v[16:17], v9, off offset:11
+; ALIGNED-NEXT:    global_store_byte v[16:17], v6, off offset:9
+; ALIGNED-NEXT:    global_store_byte v[16:17], v10, off offset:7
+; ALIGNED-NEXT:    global_store_byte v[16:17], v5, off offset:5
+; ALIGNED-NEXT:    global_store_byte v[16:17], v11, off offset:3
+; ALIGNED-NEXT:    global_store_byte v[16:17], v4, off offset:1
+; ALIGNED-NEXT:    s_cbranch_scc1 .LBB6_2
+; ALIGNED-NEXT:  .LBB6_3: ; %Flow9
+; ALIGNED-NEXT:    s_andn2_saveexec_b32 s8, s6
+; ALIGNED-NEXT:    s_cbranch_execz .LBB6_6
+; ALIGNED-NEXT:  ; %bb.4: ; %memmove_bwd_loop.preheader
+; ALIGNED-NEXT:    s_movk_i32 s6, 0xff00
+; ALIGNED-NEXT:    s_mov_b64 s[4:5], 0x700
+; ALIGNED-NEXT:    s_mov_b32 s7, -1
+; ALIGNED-NEXT:  .LBB6_5: ; %memmove_bwd_loop
+; ALIGNED-NEXT:    ; =>This Inner Loop Header: Depth=1
+; ALIGNED-NEXT:    v_add_co_u32 v24, vcc_lo, v2, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v25, vcc_lo, s5, v3, vcc_lo
+; ALIGNED-NEXT:    s_clause 0xf
+; ALIGNED-NEXT:    global_load_dwordx4 v[16:19], v[24:25], off offset:240
+; ALIGNED-NEXT:    global_load_dwordx4 v[20:23], v[24:25], off offset:224
+; ALIGNED-NEXT:    global_load_dwordx4 v[4:7], v[24:25], off
+; ALIGNED-NEXT:    global_load_dwordx4 v[8:11], v[24:25], off offset:16
+; ALIGNED-NEXT:    global_load_dwordx4 v[12:15], v[24:25], off offset:32
+; ALIGNED-NEXT:    global_load_dwordx4 v[112:115], v[24:25], off offset:48
+; ALIGNED-NEXT:    global_load_dwordx4 v[116:119], v[24:25], off offset:64
+; ALIGNED-NEXT:    global_load_dwordx4 v[40:43], v[24:25], off offset:80
+; ALIGNED-NEXT:    global_load_dwordx4 v[26:29], v[24:25], off offset:96
+; ALIGNED-NEXT:    global_load_dwordx4 v[32:35], v[24:25], off offset:112
+; ALIGNED-NEXT:    global_load_dwordx4 v[44:47], v[24:25], off offset:128
+; ALIGNED-NEXT:    global_load_dwordx4 v[52:55], v[24:25], off offset:144
+; ALIGNED-NEXT:    global_load_dwordx4 v[66:69], v[24:25], off offset:160
+; ALIGNED-NEXT:    global_load_dwordx4 v[81:84], v[24:25], off offset:176
+; ALIGNED-NEXT:    global_load_dwordx4 v[96:99], v[24:25], off offset:192
+; ALIGNED-NEXT:    global_load_dwordx4 v[100:103], v[24:25], off offset:208
+; ALIGNED-NEXT:    s_waitcnt vmcnt(15)
+; ALIGNED-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:320
+; ALIGNED-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:324
+; ALIGNED-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:328
+; ALIGNED-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:332
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:332
+; ALIGNED-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:328
+; ALIGNED-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:324
+; ALIGNED-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:320
+; ALIGNED-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; ALIGNED-NEXT:    s_add_u32 s4, s4, 0xffffff00
+; ALIGNED-NEXT:    s_addc_u32 s5, s5, -1
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v31, off offset:254
+; ALIGNED-NEXT:    global_store_byte v[16:17], v31, off offset:252
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v30, off offset:250
+; ALIGNED-NEXT:    global_store_byte v[16:17], v30, off offset:248
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v25, off offset:246
+; ALIGNED-NEXT:    global_store_byte v[16:17], v25, off offset:244
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v24, off offset:242
+; ALIGNED-NEXT:    global_store_byte v[16:17], v24, off offset:240
+; ALIGNED-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:336
+; ALIGNED-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:340
+; ALIGNED-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:344
+; ALIGNED-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:348
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:348
+; ALIGNED-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:344
+; ALIGNED-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:340
+; ALIGNED-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:336
+; ALIGNED-NEXT:    s_cmp_eq_u64 s[4:5], s[6:7]
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v51, off offset:238
+; ALIGNED-NEXT:    global_store_byte v[16:17], v51, off offset:236
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v50, off offset:234
+; ALIGNED-NEXT:    global_store_byte v[16:17], v50, off offset:232
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v49, off offset:230
+; ALIGNED-NEXT:    global_store_byte v[16:17], v49, off offset:228
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v36, off offset:226
+; ALIGNED-NEXT:    global_store_byte v[16:17], v36, off offset:224
+; ALIGNED-NEXT:    buffer_store_dword v100, off, s[0:3], s32 offset:288
+; ALIGNED-NEXT:    buffer_store_dword v101, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT:    buffer_store_dword v102, off, s[0:3], s32 offset:296
+; ALIGNED-NEXT:    buffer_store_dword v103, off, s[0:3], s32 offset:300
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v71, off, s[0:3], s32 offset:300
+; ALIGNED-NEXT:    buffer_load_dword v70, off, s[0:3], s32 offset:296
+; ALIGNED-NEXT:    buffer_load_dword v65, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT:    buffer_load_dword v64, off, s[0:3], s32 offset:288
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v31
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 8, v31
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v30
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 8, v30
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v71, off offset:222
+; ALIGNED-NEXT:    global_store_byte v[16:17], v71, off offset:220
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v70, off offset:218
+; ALIGNED-NEXT:    global_store_byte v[16:17], v70, off offset:216
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v65, off offset:214
+; ALIGNED-NEXT:    global_store_byte v[16:17], v65, off offset:212
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v64, off offset:210
+; ALIGNED-NEXT:    global_store_byte v[16:17], v64, off offset:208
+; ALIGNED-NEXT:    buffer_store_dword v96, off, s[0:3], s32 offset:304
+; ALIGNED-NEXT:    buffer_store_dword v97, off, s[0:3], s32 offset:308
+; ALIGNED-NEXT:    buffer_store_dword v98, off, s[0:3], s32 offset:312
+; ALIGNED-NEXT:    buffer_store_dword v99, off, s[0:3], s32 offset:316
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v87, off, s[0:3], s32 offset:316
+; ALIGNED-NEXT:    buffer_load_dword v86, off, s[0:3], s32 offset:312
+; ALIGNED-NEXT:    buffer_load_dword v85, off, s[0:3], s32 offset:308
+; ALIGNED-NEXT:    buffer_load_dword v80, off, s[0:3], s32 offset:304
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v87, off offset:206
+; ALIGNED-NEXT:    global_store_byte v[16:17], v87, off offset:204
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v86, off offset:202
+; ALIGNED-NEXT:    global_store_byte v[16:17], v86, off offset:200
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v85, off offset:198
+; ALIGNED-NEXT:    global_store_byte v[16:17], v85, off offset:196
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v80, off offset:194
+; ALIGNED-NEXT:    global_store_byte v[16:17], v80, off offset:192
+; ALIGNED-NEXT:    buffer_store_dword v81, off, s[0:3], s32 offset:384
+; ALIGNED-NEXT:    buffer_store_dword v82, off, s[0:3], s32 offset:388
+; ALIGNED-NEXT:    buffer_store_dword v83, off, s[0:3], s32 offset:392
+; ALIGNED-NEXT:    buffer_store_dword v84, off, s[0:3], s32 offset:396
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v101, off, s[0:3], s32 offset:396
+; ALIGNED-NEXT:    buffer_load_dword v99, off, s[0:3], s32 offset:392
+; ALIGNED-NEXT:    buffer_load_dword v96, off, s[0:3], s32 offset:388
+; ALIGNED-NEXT:    buffer_load_dword v81, off, s[0:3], s32 offset:384
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v101, off offset:190
+; ALIGNED-NEXT:    global_store_byte v[16:17], v101, off offset:188
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v99, off offset:186
+; ALIGNED-NEXT:    global_store_byte v[16:17], v99, off offset:184
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v96, off offset:182
+; ALIGNED-NEXT:    global_store_byte v[16:17], v96, off offset:180
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v81, off offset:178
+; ALIGNED-NEXT:    global_store_byte v[16:17], v81, off offset:176
+; ALIGNED-NEXT:    buffer_store_dword v66, off, s[0:3], s32 offset:400
+; ALIGNED-NEXT:    buffer_store_dword v67, off, s[0:3], s32 offset:404
+; ALIGNED-NEXT:    buffer_store_dword v68, off, s[0:3], s32 offset:408
+; ALIGNED-NEXT:    buffer_store_dword v69, off, s[0:3], s32 offset:412
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v100, off, s[0:3], s32 offset:412
+; ALIGNED-NEXT:    buffer_load_dword v97, off, s[0:3], s32 offset:408
+; ALIGNED-NEXT:    buffer_load_dword v82, off, s[0:3], s32 offset:404
+; ALIGNED-NEXT:    buffer_load_dword v66, off, s[0:3], s32 offset:400
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v100, off offset:174
+; ALIGNED-NEXT:    global_store_byte v[16:17], v100, off offset:172
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v97, off offset:170
+; ALIGNED-NEXT:    global_store_byte v[16:17], v97, off offset:168
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v82, off offset:166
+; ALIGNED-NEXT:    global_store_byte v[16:17], v82, off offset:164
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v66, off offset:162
+; ALIGNED-NEXT:    global_store_byte v[16:17], v66, off offset:160
+; ALIGNED-NEXT:    buffer_store_dword v52, off, s[0:3], s32 offset:352
+; ALIGNED-NEXT:    buffer_store_dword v53, off, s[0:3], s32 offset:356
+; ALIGNED-NEXT:    buffer_store_dword v54, off, s[0:3], s32 offset:360
+; ALIGNED-NEXT:    buffer_store_dword v55, off, s[0:3], s32 offset:364
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v98, off, s[0:3], s32 offset:364
+; ALIGNED-NEXT:    buffer_load_dword v83, off, s[0:3], s32 offset:360
+; ALIGNED-NEXT:    buffer_load_dword v67, off, s[0:3], s32 offset:356
+; ALIGNED-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:352
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v98, off offset:158
+; ALIGNED-NEXT:    global_store_byte v[16:17], v98, off offset:156
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v83, off offset:154
+; ALIGNED-NEXT:    global_store_byte v[16:17], v83, off offset:152
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v67, off offset:150
+; ALIGNED-NEXT:    global_store_byte v[16:17], v67, off offset:148
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v52, off offset:146
+; ALIGNED-NEXT:    global_store_byte v[16:17], v52, off offset:144
+; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:368
+; ALIGNED-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:372
+; ALIGNED-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:376
+; ALIGNED-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:380
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v84, off, s[0:3], s32 offset:380
+; ALIGNED-NEXT:    buffer_load_dword v68, off, s[0:3], s32 offset:376
+; ALIGNED-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:372
+; ALIGNED-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:368
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v84, off offset:142
+; ALIGNED-NEXT:    global_store_byte v[16:17], v84, off offset:140
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v68, off offset:138
+; ALIGNED-NEXT:    global_store_byte v[16:17], v68, off offset:136
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v53, off offset:134
+; ALIGNED-NEXT:    global_store_byte v[16:17], v53, off offset:132
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v37, off offset:130
+; ALIGNED-NEXT:    global_store_byte v[16:17], v37, off offset:128
+; ALIGNED-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:448
+; ALIGNED-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:452
+; ALIGNED-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:456
+; ALIGNED-NEXT:    buffer_store_dword v35, off, s[0:3], s32 offset:460
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v69, off, s[0:3], s32 offset:460
+; ALIGNED-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:456
+; ALIGNED-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:452
+; ALIGNED-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:448
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v69, off offset:126
+; ALIGNED-NEXT:    global_store_byte v[16:17], v69, off offset:124
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v54, off offset:122
+; ALIGNED-NEXT:    global_store_byte v[16:17], v54, off offset:120
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v38, off offset:118
+; ALIGNED-NEXT:    global_store_byte v[16:17], v38, off offset:116
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v32, off offset:114
+; ALIGNED-NEXT:    global_store_byte v[16:17], v32, off offset:112
+; ALIGNED-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:464
+; ALIGNED-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:468
+; ALIGNED-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:472
+; ALIGNED-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:476
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:476
+; ALIGNED-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:472
+; ALIGNED-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:468
+; ALIGNED-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:464
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v55, off offset:110
+; ALIGNED-NEXT:    global_store_byte v[16:17], v55, off offset:108
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v39, off offset:106
+; ALIGNED-NEXT:    global_store_byte v[16:17], v39, off offset:104
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v33, off offset:102
+; ALIGNED-NEXT:    global_store_byte v[16:17], v33, off offset:100
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v26, off offset:98
+; ALIGNED-NEXT:    global_store_byte v[16:17], v26, off offset:96
+; ALIGNED-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:416
+; ALIGNED-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:420
+; ALIGNED-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:424
+; ALIGNED-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:428
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:428
+; ALIGNED-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:424
+; ALIGNED-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:420
+; ALIGNED-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:416
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v48, off offset:94
+; ALIGNED-NEXT:    global_store_byte v[16:17], v48, off offset:92
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v34, off offset:90
+; ALIGNED-NEXT:    global_store_byte v[16:17], v34, off offset:88
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v27, off offset:86
+; ALIGNED-NEXT:    global_store_byte v[16:17], v27, off offset:84
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v21, off offset:82
+; ALIGNED-NEXT:    global_store_byte v[16:17], v21, off offset:80
+; ALIGNED-NEXT:    buffer_store_dword v116, off, s[0:3], s32 offset:432
+; ALIGNED-NEXT:    buffer_store_dword v117, off, s[0:3], s32 offset:436
+; ALIGNED-NEXT:    buffer_store_dword v118, off, s[0:3], s32 offset:440
+; ALIGNED-NEXT:    buffer_store_dword v119, off, s[0:3], s32 offset:444
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:444
+; ALIGNED-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:440
+; ALIGNED-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:436
+; ALIGNED-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:432
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v35, off offset:78
+; ALIGNED-NEXT:    global_store_byte v[16:17], v35, off offset:76
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v28, off offset:74
+; ALIGNED-NEXT:    global_store_byte v[16:17], v28, off offset:72
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v22, off offset:70
+; ALIGNED-NEXT:    global_store_byte v[16:17], v22, off offset:68
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v19, off offset:66
+; ALIGNED-NEXT:    global_store_byte v[16:17], v19, off offset:64
+; ALIGNED-NEXT:    buffer_store_dword v112, off, s[0:3], s32 offset:512
+; ALIGNED-NEXT:    buffer_store_dword v113, off, s[0:3], s32 offset:516
+; ALIGNED-NEXT:    buffer_store_dword v114, off, s[0:3], s32 offset:520
+; ALIGNED-NEXT:    buffer_store_dword v115, off, s[0:3], s32 offset:524
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:524
+; ALIGNED-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:520
+; ALIGNED-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:516
+; ALIGNED-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:512
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 8, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 8, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v51
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 8, v51
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v50
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 8, v50
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v29, off offset:62
+; ALIGNED-NEXT:    global_store_byte v[16:17], v29, off offset:60
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v23, off offset:58
+; ALIGNED-NEXT:    global_store_byte v[16:17], v23, off offset:56
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v20, off offset:54
+; ALIGNED-NEXT:    global_store_byte v[16:17], v20, off offset:52
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v18, off offset:50
+; ALIGNED-NEXT:    global_store_byte v[16:17], v18, off offset:48
+; ALIGNED-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:528
+; ALIGNED-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:532
+; ALIGNED-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:536
+; ALIGNED-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:540
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:536
+; ALIGNED-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:540
+; ALIGNED-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:528
+; ALIGNED-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:532
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v15, off offset:42
+; ALIGNED-NEXT:    global_store_byte v[16:17], v15, off offset:40
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v14, off offset:46
+; ALIGNED-NEXT:    global_store_byte v[16:17], v14, off offset:44
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v13, off offset:34
+; ALIGNED-NEXT:    global_store_byte v[16:17], v13, off offset:32
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v12, off offset:38
+; ALIGNED-NEXT:    global_store_byte v[16:17], v12, off offset:36
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:480
+; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:484
+; ALIGNED-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:488
+; ALIGNED-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:492
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:492
+; ALIGNED-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:488
+; ALIGNED-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:484
+; ALIGNED-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:480
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v11, off offset:30
+; ALIGNED-NEXT:    global_store_byte v[16:17], v11, off offset:28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v10, off offset:26
+; ALIGNED-NEXT:    global_store_byte v[16:17], v10, off offset:24
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v9, off offset:22
+; ALIGNED-NEXT:    global_store_byte v[16:17], v9, off offset:20
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v8, off offset:18
+; ALIGNED-NEXT:    global_store_byte v[16:17], v8, off offset:16
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:496
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:500
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:504
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:508
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:508
+; ALIGNED-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:504
+; ALIGNED-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:500
+; ALIGNED-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:496
+; ALIGNED-NEXT:    global_store_byte v[16:17], v112, off offset:247
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v65
+; ALIGNED-NEXT:    global_store_byte v[16:17], v102, off offset:255
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v49
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 8, v49
+; ALIGNED-NEXT:    global_store_byte v[16:17], v31, off offset:253
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 24, v36
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 8, v36
+; ALIGNED-NEXT:    global_store_byte v[16:17], v103, off offset:251
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v71
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 8, v71
+; ALIGNED-NEXT:    global_store_byte v[16:17], v30, off offset:249
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v70
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 8, v70
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 8, v65
+; ALIGNED-NEXT:    global_store_byte v[16:17], v25, off offset:245
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 24, v64
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 8, v64
+; ALIGNED-NEXT:    global_store_byte v[16:17], v112, off offset:215
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v67
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 8, v67
+; ALIGNED-NEXT:    global_store_byte v[16:17], v113, off offset:243
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v87
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v87
+; ALIGNED-NEXT:    global_store_byte v[16:17], v24, off offset:241
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 24, v86
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 8, v86
+; ALIGNED-NEXT:    global_store_byte v[16:17], v114, off offset:239
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v85
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v85
+; ALIGNED-NEXT:    global_store_byte v[16:17], v51, off offset:237
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 24, v80
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 8, v80
+; ALIGNED-NEXT:    global_store_byte v[16:17], v115, off offset:235
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v101
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 8, v101
+; ALIGNED-NEXT:    global_store_byte v[16:17], v50, off offset:233
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 24, v99
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 8, v99
+; ALIGNED-NEXT:    global_store_byte v[16:17], v102, off offset:231
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v96
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v96, 8, v96
+; ALIGNED-NEXT:    global_store_byte v[16:17], v49, off offset:229
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 24, v81
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 8, v81
+; ALIGNED-NEXT:    global_store_byte v[16:17], v31, off offset:227
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 24, v100
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v100
+; ALIGNED-NEXT:    global_store_byte v[16:17], v36, off offset:225
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 24, v97
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v97, 8, v97
+; ALIGNED-NEXT:    global_store_byte v[16:17], v103, off offset:223
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v82
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 8, v82
+; ALIGNED-NEXT:    global_store_byte v[16:17], v71, off offset:221
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 24, v66
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 8, v66
+; ALIGNED-NEXT:    global_store_byte v[16:17], v30, off offset:219
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v98
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v98
+; ALIGNED-NEXT:    global_store_byte v[16:17], v70, off offset:217
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 24, v83
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 8, v83
+; ALIGNED-NEXT:    global_store_byte v[16:17], v65, off offset:213
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 24, v52
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v52, 8, v52
+; ALIGNED-NEXT:    global_store_byte v[16:17], v25, off offset:211
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 24, v84
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 8, v84
+; ALIGNED-NEXT:    global_store_byte v[16:17], v64, off offset:209
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 24, v68
+; ALIGNED-NEXT:    global_store_byte v[16:17], v67, off offset:149
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 24, v8
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v8, 8, v8
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v68, 8, v68
+; ALIGNED-NEXT:    global_store_byte v[16:17], v113, off offset:207
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v53
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v53, 8, v53
+; ALIGNED-NEXT:    global_store_byte v[16:17], v87, off offset:205
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 24, v37
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v37, 8, v37
+; ALIGNED-NEXT:    global_store_byte v[16:17], v24, off offset:203
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 24, v69
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v69, 8, v69
+; ALIGNED-NEXT:    global_store_byte v[16:17], v86, off offset:201
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v54
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v54, 8, v54
+; ALIGNED-NEXT:    global_store_byte v[16:17], v114, off offset:199
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v38
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v38, 8, v38
+; ALIGNED-NEXT:    global_store_byte v[16:17], v85, off offset:197
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 24, v32
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v32, 8, v32
+; ALIGNED-NEXT:    global_store_byte v[16:17], v51, off offset:195
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 24, v55
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v55, 8, v55
+; ALIGNED-NEXT:    global_store_byte v[16:17], v80, off offset:193
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 24, v39
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v39, 8, v39
+; ALIGNED-NEXT:    global_store_byte v[16:17], v115, off offset:191
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 24, v33
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v33, 8, v33
+; ALIGNED-NEXT:    global_store_byte v[16:17], v101, off offset:189
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 24, v26
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v26, 8, v26
+; ALIGNED-NEXT:    global_store_byte v[16:17], v50, off offset:187
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 24, v48
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v48, 8, v48
+; ALIGNED-NEXT:    global_store_byte v[16:17], v99, off offset:185
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 24, v34
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v34, 8, v34
+; ALIGNED-NEXT:    global_store_byte v[16:17], v102, off offset:183
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v27
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v27, 8, v27
+; ALIGNED-NEXT:    global_store_byte v[16:17], v96, off offset:181
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v96, 24, v21
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v21, 8, v21
+; ALIGNED-NEXT:    global_store_byte v[16:17], v49, off offset:179
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 24, v35
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v35, 8, v35
+; ALIGNED-NEXT:    global_store_byte v[16:17], v81, off offset:177
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 24, v28
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v28, 8, v28
+; ALIGNED-NEXT:    global_store_byte v[16:17], v31, off offset:175
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 24, v22
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v22, 8, v22
+; ALIGNED-NEXT:    global_store_byte v[16:17], v100, off offset:173
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 24, v19
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v19, 8, v19
+; ALIGNED-NEXT:    global_store_byte v[16:17], v36, off offset:171
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 24, v29
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v29, 8, v29
+; ALIGNED-NEXT:    global_store_byte v[16:17], v97, off offset:169
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v97, 24, v23
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v23, 8, v23
+; ALIGNED-NEXT:    global_store_byte v[16:17], v103, off offset:167
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v20
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v20, 8, v20
+; ALIGNED-NEXT:    global_store_byte v[16:17], v82, off offset:165
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 24, v18
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v18, 8, v18
+; ALIGNED-NEXT:    global_store_byte v[16:17], v71, off offset:163
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 24, v15
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v15, 8, v15
+; ALIGNED-NEXT:    global_store_byte v[16:17], v66, off offset:161
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 24, v14
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v14, 8, v14
+; ALIGNED-NEXT:    global_store_byte v[16:17], v30, off offset:159
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v13
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v13, 8, v13
+; ALIGNED-NEXT:    global_store_byte v[16:17], v98, off offset:157
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 24, v12
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v12, 8, v12
+; ALIGNED-NEXT:    global_store_byte v[16:17], v70, off offset:155
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 24, v11
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v11, 8, v11
+; ALIGNED-NEXT:    global_store_byte v[16:17], v83, off offset:153
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 24, v10
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v10, 8, v10
+; ALIGNED-NEXT:    global_store_byte v[16:17], v112, off offset:151
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 24, v9
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v9, 8, v9
+; ALIGNED-NEXT:    global_store_byte v[16:17], v65, off offset:147
+; ALIGNED-NEXT:    global_store_byte v[16:17], v52, off offset:145
+; ALIGNED-NEXT:    global_store_byte v[16:17], v25, off offset:143
+; ALIGNED-NEXT:    global_store_byte v[16:17], v84, off offset:141
+; ALIGNED-NEXT:    global_store_byte v[16:17], v64, off offset:139
+; ALIGNED-NEXT:    global_store_byte v[16:17], v68, off offset:137
+; ALIGNED-NEXT:    global_store_byte v[16:17], v113, off offset:135
+; ALIGNED-NEXT:    global_store_byte v[16:17], v53, off offset:133
+; ALIGNED-NEXT:    global_store_byte v[16:17], v87, off offset:131
+; ALIGNED-NEXT:    global_store_byte v[16:17], v37, off offset:129
+; ALIGNED-NEXT:    global_store_byte v[16:17], v24, off offset:127
+; ALIGNED-NEXT:    global_store_byte v[16:17], v69, off offset:125
+; ALIGNED-NEXT:    global_store_byte v[16:17], v86, off offset:123
+; ALIGNED-NEXT:    global_store_byte v[16:17], v54, off offset:121
+; ALIGNED-NEXT:    global_store_byte v[16:17], v114, off offset:119
+; ALIGNED-NEXT:    global_store_byte v[16:17], v38, off offset:117
+; ALIGNED-NEXT:    global_store_byte v[16:17], v85, off offset:115
+; ALIGNED-NEXT:    global_store_byte v[16:17], v32, off offset:113
+; ALIGNED-NEXT:    global_store_byte v[16:17], v51, off offset:111
+; ALIGNED-NEXT:    global_store_byte v[16:17], v55, off offset:109
+; ALIGNED-NEXT:    global_store_byte v[16:17], v80, off offset:107
+; ALIGNED-NEXT:    global_store_byte v[16:17], v39, off offset:105
+; ALIGNED-NEXT:    global_store_byte v[16:17], v115, off offset:103
+; ALIGNED-NEXT:    global_store_byte v[16:17], v33, off offset:101
+; ALIGNED-NEXT:    global_store_byte v[16:17], v101, off offset:99
+; ALIGNED-NEXT:    global_store_byte v[16:17], v26, off offset:97
+; ALIGNED-NEXT:    global_store_byte v[16:17], v50, off offset:95
+; ALIGNED-NEXT:    global_store_byte v[16:17], v48, off offset:93
+; ALIGNED-NEXT:    global_store_byte v[16:17], v99, off offset:91
+; ALIGNED-NEXT:    global_store_byte v[16:17], v34, off offset:89
+; ALIGNED-NEXT:    global_store_byte v[16:17], v102, off offset:87
+; ALIGNED-NEXT:    global_store_byte v[16:17], v27, off offset:85
+; ALIGNED-NEXT:    global_store_byte v[16:17], v96, off offset:83
+; ALIGNED-NEXT:    global_store_byte v[16:17], v21, off offset:81
+; ALIGNED-NEXT:    global_store_byte v[16:17], v49, off offset:79
+; ALIGNED-NEXT:    global_store_byte v[16:17], v35, off offset:77
+; ALIGNED-NEXT:    global_store_byte v[16:17], v81, off offset:75
+; ALIGNED-NEXT:    global_store_byte v[16:17], v28, off offset:73
+; ALIGNED-NEXT:    global_store_byte v[16:17], v31, off offset:71
+; ALIGNED-NEXT:    global_store_byte v[16:17], v22, off offset:69
+; ALIGNED-NEXT:    global_store_byte v[16:17], v100, off offset:67
+; ALIGNED-NEXT:    global_store_byte v[16:17], v19, off offset:65
+; ALIGNED-NEXT:    global_store_byte v[16:17], v36, off offset:63
+; ALIGNED-NEXT:    global_store_byte v[16:17], v29, off offset:61
+; ALIGNED-NEXT:    global_store_byte v[16:17], v97, off offset:59
+; ALIGNED-NEXT:    global_store_byte v[16:17], v23, off offset:57
+; ALIGNED-NEXT:    global_store_byte v[16:17], v103, off offset:55
+; ALIGNED-NEXT:    global_store_byte v[16:17], v20, off offset:53
+; ALIGNED-NEXT:    global_store_byte v[16:17], v82, off offset:51
+; ALIGNED-NEXT:    global_store_byte v[16:17], v18, off offset:49
+; ALIGNED-NEXT:    global_store_byte v[16:17], v71, off offset:43
+; ALIGNED-NEXT:    global_store_byte v[16:17], v15, off offset:41
+; ALIGNED-NEXT:    global_store_byte v[16:17], v66, off offset:47
+; ALIGNED-NEXT:    global_store_byte v[16:17], v14, off offset:45
+; ALIGNED-NEXT:    global_store_byte v[16:17], v30, off offset:35
+; ALIGNED-NEXT:    global_store_byte v[16:17], v13, off offset:33
+; ALIGNED-NEXT:    global_store_byte v[16:17], v98, off offset:39
+; ALIGNED-NEXT:    global_store_byte v[16:17], v12, off offset:37
+; ALIGNED-NEXT:    global_store_byte v[16:17], v70, off offset:31
+; ALIGNED-NEXT:    global_store_byte v[16:17], v11, off offset:29
+; ALIGNED-NEXT:    global_store_byte v[16:17], v83, off offset:27
+; ALIGNED-NEXT:    global_store_byte v[16:17], v10, off offset:25
+; ALIGNED-NEXT:    global_store_byte v[16:17], v112, off offset:23
+; ALIGNED-NEXT:    global_store_byte v[16:17], v9, off offset:21
+; ALIGNED-NEXT:    global_store_byte v[16:17], v67, off offset:19
+; ALIGNED-NEXT:    global_store_byte v[16:17], v8, off offset:17
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v7, off offset:14
+; ALIGNED-NEXT:    global_store_byte v[16:17], v7, off offset:12
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v6, off offset:10
+; ALIGNED-NEXT:    global_store_byte v[16:17], v6, off offset:8
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v5, off offset:6
+; ALIGNED-NEXT:    global_store_byte v[16:17], v5, off offset:4
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    global_store_byte_d16_hi v[16:17], v4, off offset:2
+; ALIGNED-NEXT:    global_store_byte v[16:17], v4, off
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v8, 24, v7
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v7, 8, v7
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v9, 24, v6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v6, 8, v6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v10, 24, v5
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v5, 8, v5
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v11, 24, v4
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v4, 8, v4
+; ALIGNED-NEXT:    global_store_byte v[16:17], v8, off offset:15
+; ALIGNED-NEXT:    global_store_byte v[16:17], v7, off offset:13
+; ALIGNED-NEXT:    global_store_byte v[16:17], v9, off offset:11
+; ALIGNED-NEXT:    global_store_byte v[16:17], v6, off offset:9
+; ALIGNED-NEXT:    global_store_byte v[16:17], v10, off offset:7
+; ALIGNED-NEXT:    global_store_byte v[16:17], v5, off offset:5
+; ALIGNED-NEXT:    global_store_byte v[16:17], v11, off offset:3
+; ALIGNED-NEXT:    global_store_byte v[16:17], v4, off offset:1
+; ALIGNED-NEXT:    s_cbranch_scc0 .LBB6_5
+; ALIGNED-NEXT:  .LBB6_6: ; %Flow10
+; ALIGNED-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_dword v47, off, s[0:3], s32
+; ALIGNED-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4
+; ALIGNED-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8
+; ALIGNED-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12
+; ALIGNED-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16
+; ALIGNED-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20
+; ALIGNED-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24
+; ALIGNED-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; UNROLL3-LABEL: memmove_p1_p1_sz2048:
+; UNROLL3:       ; %bb.0: ; %entry
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    s_mov_b32 s4, exec_lo
+; UNROLL3-NEXT:    v_cmpx_ge_u64_e64 v[2:3], v[0:1]
+; UNROLL3-NEXT:    s_xor_b32 s6, exec_lo, s4
+; UNROLL3-NEXT:    s_cbranch_execz .LBB6_4
+; UNROLL3-NEXT:  ; %bb.1: ; %memmove_fwd_loop.preheader
+; UNROLL3-NEXT:    s_mov_b64 s[4:5], 0
+; UNROLL3-NEXT:    .p2align 6
+; UNROLL3-NEXT:  .LBB6_2: ; %memmove_fwd_loop
+; UNROLL3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT:    v_add_co_u32 v12, vcc_lo, v2, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v13, vcc_lo, s5, v3, vcc_lo
+; UNROLL3-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; UNROLL3-NEXT:    s_clause 0x2
+; UNROLL3-NEXT:    global_load_dwordx4 v[4:7], v[12:13], off
+; UNROLL3-NEXT:    global_load_dwordx4 v[8:11], v[12:13], off offset:16
+; UNROLL3-NEXT:    global_load_dwordx4 v[12:15], v[12:13], off offset:32
+; UNROLL3-NEXT:    s_add_u32 s4, s4, 48
+; UNROLL3-NEXT:    s_addc_u32 s5, s5, 0
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2)
+; UNROLL3-NEXT:    global_store_dwordx4 v[16:17], v[4:7], off
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    global_store_dwordx4 v[16:17], v[8:11], off offset:16
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    global_store_dwordx4 v[16:17], v[12:15], off offset:32
+; UNROLL3-NEXT:    s_cmp_lg_u64 s[4:5], 0x7e0
+; UNROLL3-NEXT:    s_cbranch_scc1 .LBB6_2
+; UNROLL3-NEXT:  ; %bb.3: ; %memmove_fwd_residual
+; UNROLL3-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:2016
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:2016
+; UNROLL3-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off offset:2032
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:2032
+; UNROLL3-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; UNROLL3-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; UNROLL3-NEXT:  .LBB6_4: ; %Flow7
+; UNROLL3-NEXT:    s_andn2_saveexec_b32 s8, s6
+; UNROLL3-NEXT:    s_cbranch_execz .LBB6_7
+; UNROLL3-NEXT:  ; %bb.5: ; %memmove_bwd_residual
+; UNROLL3-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:2032
+; UNROLL3-NEXT:    s_movk_i32 s6, 0xffd0
+; UNROLL3-NEXT:    s_mov_b64 s[4:5], 0x7b0
+; UNROLL3-NEXT:    s_mov_b32 s7, -1
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:2032
+; UNROLL3-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:2016
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:2016
+; UNROLL3-NEXT:    .p2align 6
+; UNROLL3-NEXT:  .LBB6_6: ; %memmove_bwd_loop
+; UNROLL3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT:    v_add_co_u32 v12, vcc_lo, v2, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v13, vcc_lo, s5, v3, vcc_lo
+; UNROLL3-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; UNROLL3-NEXT:    s_clause 0x2
+; UNROLL3-NEXT:    global_load_dwordx4 v[4:7], v[12:13], off
+; UNROLL3-NEXT:    global_load_dwordx4 v[8:11], v[12:13], off offset:16
+; UNROLL3-NEXT:    global_load_dwordx4 v[12:15], v[12:13], off offset:32
+; UNROLL3-NEXT:    s_add_u32 s4, s4, 0xffffffd0
+; UNROLL3-NEXT:    s_addc_u32 s5, s5, -1
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2)
+; UNROLL3-NEXT:    global_store_dwordx4 v[16:17], v[4:7], off
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    global_store_dwordx4 v[16:17], v[8:11], off offset:16
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    global_store_dwordx4 v[16:17], v[12:15], off offset:32
+; UNROLL3-NEXT:    s_cmp_eq_u64 s[4:5], s[6:7]
+; UNROLL3-NEXT:    s_cbranch_scc0 .LBB6_6
+; UNROLL3-NEXT:  .LBB6_7: ; %Flow8
+; UNROLL3-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; UNROLL3-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  tail call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 2048, i1 false)
+  ret void
+}
+
+define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4) align 1 readonly %src) {
+; CHECK-LABEL: memmove_p0_p4_sz2048:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s4, exec_lo
+; CHECK-NEXT:    v_cmpx_ge_u64_e64 v[2:3], v[0:1]
+; CHECK-NEXT:    s_xor_b32 s6, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execz .LBB7_3
+; CHECK-NEXT:  ; %bb.1: ; %memmove_fwd_loop.preheader
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0
+; CHECK-NEXT:  .LBB7_2: ; %memmove_fwd_loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    v_add_co_u32 v96, vcc_lo, v2, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v97, vcc_lo, s5, v3, vcc_lo
+; CHECK-NEXT:    v_add_co_u32 v100, vcc_lo, v0, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v101, vcc_lo, s5, v1, vcc_lo
+; CHECK-NEXT:    s_clause 0xf
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[96:97], off offset:240
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[96:97], off offset:224
+; CHECK-NEXT:    global_load_dwordx4 v[12:15], v[96:97], off offset:208
+; CHECK-NEXT:    global_load_dwordx4 v[16:19], v[96:97], off offset:192
+; CHECK-NEXT:    global_load_dwordx4 v[20:23], v[96:97], off offset:176
+; CHECK-NEXT:    global_load_dwordx4 v[24:27], v[96:97], off offset:160
+; CHECK-NEXT:    global_load_dwordx4 v[28:31], v[96:97], off offset:144
+; CHECK-NEXT:    global_load_dwordx4 v[32:35], v[96:97], off offset:128
+; CHECK-NEXT:    global_load_dwordx4 v[36:39], v[96:97], off offset:112
+; CHECK-NEXT:    global_load_dwordx4 v[48:51], v[96:97], off offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[52:55], v[96:97], off offset:80
+; CHECK-NEXT:    global_load_dwordx4 v[64:67], v[96:97], off offset:64
+; CHECK-NEXT:    global_load_dwordx4 v[68:71], v[96:97], off offset:48
+; CHECK-NEXT:    global_load_dwordx4 v[80:83], v[96:97], off offset:32
+; CHECK-NEXT:    global_load_dwordx4 v[84:87], v[96:97], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[96:99], v[96:97], off
+; CHECK-NEXT:    s_add_u32 s4, s4, 0x100
+; CHECK-NEXT:    s_addc_u32 s5, s5, 0
+; CHECK-NEXT:    s_waitcnt vmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[4:7] offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(14)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[8:11] offset:224
+; CHECK-NEXT:    s_waitcnt vmcnt(13)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[12:15] offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(12)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[16:19] offset:192
+; CHECK-NEXT:    s_waitcnt vmcnt(11)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[20:23] offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(10)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[24:27] offset:160
+; CHECK-NEXT:    s_waitcnt vmcnt(9)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[28:31] offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(8)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[32:35] offset:128
+; CHECK-NEXT:    s_waitcnt vmcnt(7)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[36:39] offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(6)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[48:51] offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[52:55] offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[64:67] offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[68:71] offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(2)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[80:83] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[84:87] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[96:99]
+; CHECK-NEXT:    s_cmp_lg_u64 s[4:5], 0x800
+; CHECK-NEXT:    s_cbranch_scc1 .LBB7_2
+; CHECK-NEXT:  .LBB7_3: ; %Flow6
+; CHECK-NEXT:    s_andn2_saveexec_b32 s8, s6
+; CHECK-NEXT:    s_cbranch_execz .LBB7_6
+; CHECK-NEXT:  ; %bb.4: ; %memmove_bwd_loop.preheader
+; CHECK-NEXT:    s_movk_i32 s6, 0xff00
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0x700
+; CHECK-NEXT:    s_mov_b32 s7, -1
+; CHECK-NEXT:  .LBB7_5: ; %memmove_bwd_loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    v_add_co_u32 v96, vcc_lo, v2, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v97, vcc_lo, s5, v3, vcc_lo
+; CHECK-NEXT:    v_add_co_u32 v100, vcc_lo, v0, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v101, vcc_lo, s5, v1, vcc_lo
+; CHECK-NEXT:    s_clause 0xf
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[96:97], off offset:240
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[96:97], off offset:224
+; CHECK-NEXT:    global_load_dwordx4 v[12:15], v[96:97], off offset:208
+; CHECK-NEXT:    global_load_dwordx4 v[16:19], v[96:97], off offset:192
+; CHECK-NEXT:    global_load_dwordx4 v[20:23], v[96:97], off offset:176
+; CHECK-NEXT:    global_load_dwordx4 v[24:27], v[96:97], off offset:160
+; CHECK-NEXT:    global_load_dwordx4 v[28:31], v[96:97], off offset:144
+; CHECK-NEXT:    global_load_dwordx4 v[32:35], v[96:97], off offset:128
+; CHECK-NEXT:    global_load_dwordx4 v[36:39], v[96:97], off offset:112
+; CHECK-NEXT:    global_load_dwordx4 v[48:51], v[96:97], off offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[52:55], v[96:97], off offset:80
+; CHECK-NEXT:    global_load_dwordx4 v[64:67], v[96:97], off offset:64
+; CHECK-NEXT:    global_load_dwordx4 v[68:71], v[96:97], off offset:48
+; CHECK-NEXT:    global_load_dwordx4 v[80:83], v[96:97], off offset:32
+; CHECK-NEXT:    global_load_dwordx4 v[84:87], v[96:97], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[96:99], v[96:97], off
+; CHECK-NEXT:    s_add_u32 s4, s4, 0xffffff00
+; CHECK-NEXT:    s_addc_u32 s5, s5, -1
+; CHECK-NEXT:    s_waitcnt vmcnt(15)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[4:7] offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(14)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[8:11] offset:224
+; CHECK-NEXT:    s_waitcnt vmcnt(13)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[12:15] offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(12)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[16:19] offset:192
+; CHECK-NEXT:    s_waitcnt vmcnt(11)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[20:23] offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(10)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[24:27] offset:160
+; CHECK-NEXT:    s_waitcnt vmcnt(9)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[28:31] offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(8)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[32:35] offset:128
+; CHECK-NEXT:    s_waitcnt vmcnt(7)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[36:39] offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(6)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[48:51] offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[52:55] offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[64:67] offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[68:71] offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(2)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[80:83] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[84:87] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[96:99]
+; CHECK-NEXT:    s_cmp_eq_u64 s[4:5], s[6:7]
+; CHECK-NEXT:    s_cbranch_scc0 .LBB7_5
+; CHECK-NEXT:  .LBB7_6: ; %Flow7
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+;
+; ALIGNED-LABEL: memmove_p0_p4_sz2048:
+; ALIGNED:       ; %bb.0: ; %entry
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; ALIGNED-NEXT:    s_mov_b32 s4, exec_lo
+; ALIGNED-NEXT:    v_cmpx_ge_u64_e64 v[2:3], v[0:1]
+; ALIGNED-NEXT:    s_xor_b32 s6, exec_lo, s4
+; ALIGNED-NEXT:    s_cbranch_execz .LBB7_3
+; ALIGNED-NEXT:  ; %bb.1: ; %memmove_fwd_loop.preheader
+; ALIGNED-NEXT:    s_mov_b64 s[4:5], 0
+; ALIGNED-NEXT:  .LBB7_2: ; %memmove_fwd_loop
+; ALIGNED-NEXT:    ; =>This Inner Loop Header: Depth=1
+; ALIGNED-NEXT:    v_add_co_u32 v4, vcc_lo, v2, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, s5, v3, vcc_lo
+; ALIGNED-NEXT:    v_add_co_u32 v96, vcc_lo, v0, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v97, vcc_lo, s5, v1, vcc_lo
+; ALIGNED-NEXT:    s_clause 0xf
+; ALIGNED-NEXT:    global_load_dwordx4 v[112:115], v[4:5], off offset:240
+; ALIGNED-NEXT:    global_load_dwordx4 v[84:87], v[4:5], off offset:224
+; ALIGNED-NEXT:    global_load_dwordx4 v[80:83], v[4:5], off offset:208
+; ALIGNED-NEXT:    global_load_dwordx4 v[68:71], v[4:5], off offset:192
+; ALIGNED-NEXT:    global_load_dwordx4 v[64:67], v[4:5], off offset:176
+; ALIGNED-NEXT:    global_load_dwordx4 v[52:55], v[4:5], off offset:160
+; ALIGNED-NEXT:    global_load_dwordx4 v[48:51], v[4:5], off offset:144
+; ALIGNED-NEXT:    global_load_dwordx4 v[36:39], v[4:5], off offset:128
+; ALIGNED-NEXT:    global_load_dwordx4 v[32:35], v[4:5], off offset:112
+; ALIGNED-NEXT:    global_load_dwordx4 v[28:31], v[4:5], off offset:96
+; ALIGNED-NEXT:    global_load_dwordx4 v[24:27], v[4:5], off offset:80
+; ALIGNED-NEXT:    global_load_dwordx4 v[20:23], v[4:5], off offset:64
+; ALIGNED-NEXT:    global_load_dwordx4 v[16:19], v[4:5], off offset:48
+; ALIGNED-NEXT:    global_load_dwordx4 v[12:15], v[4:5], off offset:32
+; ALIGNED-NEXT:    global_load_dwordx4 v[8:11], v[4:5], off offset:16
+; ALIGNED-NEXT:    global_load_dwordx4 v[4:7], v[4:5], off
+; ALIGNED-NEXT:    s_add_u32 s4, s4, 0x100
+; ALIGNED-NEXT:    s_addc_u32 s5, s5, 0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(15)
+; ALIGNED-NEXT:    buffer_store_dword v114, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT:    buffer_store_dword v115, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT:    buffer_store_dword v113, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT:    buffer_store_dword v112, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 24, v114
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 8, v114
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v114 offset:250
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v115 offset:254
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 24, v115
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:252
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 8, v115
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:248
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v113
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v113 offset:246
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 8, v113
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:244
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v112 offset:242
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:240
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v112
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v112
+; ALIGNED-NEXT:    s_waitcnt vmcnt(14)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v86
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 8, v86
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:251
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 24, v87
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:249
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 8, v87
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:255
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 24, v85
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:253
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 8, v85
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v102 offset:247
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v84
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v103 offset:245
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 8, v84
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:243
+; ALIGNED-NEXT:    s_waitcnt vmcnt(13)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v82
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:241
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v82
+; ALIGNED-NEXT:    buffer_store_dword v86, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT:    buffer_store_dword v87, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT:    buffer_store_dword v85, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT:    buffer_store_dword v84, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v86 offset:234
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v87 offset:238
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v87 offset:236
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v86 offset:232
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v85 offset:230
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v85 offset:228
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v84 offset:226
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v84 offset:224
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 24, v83
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v83
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v81
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v81
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:235
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v80
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:233
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 8, v80
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:239
+; ALIGNED-NEXT:    s_waitcnt vmcnt(12)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 24, v70
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:237
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 8, v70
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:231
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 24, v71
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:229
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 8, v71
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v102 offset:227
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v69
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v103 offset:225
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 8, v69
+; ALIGNED-NEXT:    buffer_store_dword v82, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT:    buffer_store_dword v83, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT:    buffer_store_dword v81, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT:    buffer_store_dword v80, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v82 offset:218
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v83 offset:222
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v83 offset:220
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v82 offset:216
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v81 offset:214
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v81 offset:212
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v80 offset:210
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v80 offset:208
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 24, v68
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 8, v68
+; ALIGNED-NEXT:    s_waitcnt vmcnt(11)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 24, v66
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 8, v66
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:219
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v67
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:217
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v67
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v84 offset:223
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 24, v65
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v85 offset:221
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v65
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v86 offset:215
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v64
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v87 offset:213
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v64
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:211
+; ALIGNED-NEXT:    s_waitcnt vmcnt(10)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v54
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:209
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 8, v54
+; ALIGNED-NEXT:    buffer_store_dword v70, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT:    buffer_store_dword v71, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT:    buffer_store_dword v69, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT:    buffer_store_dword v68, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v70 offset:202
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v71 offset:206
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v71 offset:204
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v70 offset:200
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v69 offset:198
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v69 offset:196
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v68 offset:194
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v68 offset:192
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 24, v52
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 8, v52
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:203
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 24, v53
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v68, 24, v55
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v69, 8, v55
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:201
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 8, v53
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:207
+; ALIGNED-NEXT:    s_waitcnt vmcnt(9)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 24, v50
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:205
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 8, v50
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v102 offset:199
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v51
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v103 offset:197
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 8, v51
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v80 offset:195
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 24, v49
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v81 offset:193
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 8, v49
+; ALIGNED-NEXT:    buffer_store_dword v66, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT:    buffer_store_dword v67, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT:    buffer_store_dword v65, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT:    buffer_store_dword v64, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v66 offset:186
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v67 offset:190
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v67 offset:188
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v66 offset:184
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v65 offset:182
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v65 offset:180
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v64 offset:178
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v64 offset:176
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 24, v48
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 8, v48
+; ALIGNED-NEXT:    s_waitcnt vmcnt(8)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 24, v38
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 8, v38
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v82 offset:187
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 24, v39
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v83 offset:185
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 8, v39
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:191
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v37
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:189
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v37
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v84 offset:183
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 24, v36
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v85 offset:181
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v36
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v86 offset:179
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v34
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v87 offset:177
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v34
+; ALIGNED-NEXT:    buffer_store_dword v55, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT:    buffer_store_dword v54, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT:    buffer_store_dword v53, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT:    buffer_store_dword v52, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v54 offset:170
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v54 offset:168
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v55 offset:174
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v55 offset:172
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v52 offset:162
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v52 offset:160
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v53 offset:166
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v53 offset:164
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v52, 24, v35
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v53, 8, v35
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v54, 24, v33
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v55, 8, v33
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:171
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v32
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:169
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 8, v32
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v70 offset:163
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 24, v31
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v71 offset:161
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 8, v31
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:167
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 24, v29
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v68 offset:175
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v68, 24, v30
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v69 offset:173
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v69, 8, v30
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:165
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 8, v29
+; ALIGNED-NEXT:    buffer_store_dword v50, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT:    buffer_store_dword v51, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT:    buffer_store_dword v49, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT:    buffer_store_dword v48, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v50 offset:154
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v51 offset:158
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v51 offset:156
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v50 offset:152
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v49 offset:150
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v49 offset:148
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v48 offset:146
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v48 offset:144
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v48, 24, v28
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 8, v28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 24, v26
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 8, v26
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:155
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:153
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v102 offset:159
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v103 offset:157
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v80 offset:151
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v81 offset:149
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v64 offset:147
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 24, v22
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v65 offset:145
+; ALIGNED-NEXT:    buffer_store_dword v38, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT:    buffer_store_dword v39, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT:    buffer_store_dword v37, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT:    buffer_store_dword v36, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v38 offset:138
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v39 offset:142
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v39 offset:140
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v38 offset:136
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v37 offset:134
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v37 offset:132
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v36 offset:130
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v36 offset:128
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v66 offset:139
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v67 offset:137
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v82 offset:143
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 24, v18
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v83 offset:141
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:135
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:133
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v84 offset:131
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v85 offset:129
+; ALIGNED-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT:    buffer_store_dword v35, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v34 offset:122
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v35 offset:126
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v35 offset:124
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v34 offset:120
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v33 offset:118
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v33 offset:116
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v32 offset:114
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v32 offset:112
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v34, 24, v14
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v86 offset:123
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v87 offset:121
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v52 offset:127
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v53 offset:125
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v54 offset:119
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v55 offset:117
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:115
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v10
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:113
+; ALIGNED-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v30 offset:106
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v31 offset:110
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v31 offset:108
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v30 offset:104
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v29 offset:102
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v29 offset:100
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v28 offset:98
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v28 offset:96
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v70 offset:111
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 24, v6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 24, v27
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 8, v22
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 8, v18
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v35, 8, v14
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 8, v10
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v71 offset:109
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 8, v6
+; ALIGNED-NEXT:    s_cmp_lg_u64 s[4:5], 0x800
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 8, v27
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 24, v23
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v19
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v15
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v28, 24, v11
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:103
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 24, v7
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 24, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 8, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 24, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 8, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v37, 8, v23
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v38, 24, v21
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v39, 8, v21
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 24, v20
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 8, v20
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v19
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 24, v17
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v17
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v32, 24, v16
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v33, 8, v16
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v15
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v52, 24, v13
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v53, 8, v13
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v54, 24, v12
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v55, 8, v12
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v29, 8, v11
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v9
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 8, v9
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v68 offset:107
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v68, 24, v8
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v69 offset:105
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v69, 8, v8
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:101
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 8, v7
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v48 offset:99
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v48, 24, v5
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v49 offset:97
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 8, v5
+; ALIGNED-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:8
+; ALIGNED-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:12
+; ALIGNED-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:4
+; ALIGNED-NEXT:    buffer_store_dword v24, off, s[0:3], s32
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v50 offset:91
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 24, v4
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v51 offset:89
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 8, v4
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v26 offset:90
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v27 offset:94
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:95
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v27 offset:92
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:93
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v26 offset:88
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v102 offset:87
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v25 offset:86
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v103 offset:85
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v25 offset:84
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v80 offset:83
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v24 offset:82
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v81 offset:81
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v24 offset:80
+; ALIGNED-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:24
+; ALIGNED-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:28
+; ALIGNED-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:20
+; ALIGNED-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:16
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v22 offset:74
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v64 offset:75
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v65 offset:73
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v23 offset:78
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v36 offset:79
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v23 offset:76
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v37 offset:77
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v22 offset:72
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v38 offset:71
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v21 offset:70
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v39 offset:69
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v21 offset:68
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v66 offset:67
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v20 offset:66
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v67 offset:65
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v20 offset:64
+; ALIGNED-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v18 offset:58
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v82 offset:59
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v83 offset:57
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v19 offset:62
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:63
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v19 offset:60
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:61
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v18 offset:56
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v84 offset:55
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v17 offset:54
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v85 offset:53
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v17 offset:52
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v32 offset:51
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v16 offset:50
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v33 offset:49
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v16 offset:48
+; ALIGNED-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v14 offset:42
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v34 offset:43
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v35 offset:41
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v15 offset:46
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v86 offset:47
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v15 offset:44
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v87 offset:45
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v14 offset:40
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v52 offset:39
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v13 offset:38
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v53 offset:37
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v13 offset:36
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v54 offset:35
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v12 offset:34
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v55 offset:33
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v12 offset:32
+; ALIGNED-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v10 offset:26
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:27
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:25
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v11 offset:30
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v28 offset:31
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v11 offset:28
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v29 offset:29
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v10 offset:24
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v30 offset:23
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v9 offset:22
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v31 offset:21
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v9 offset:20
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v68 offset:19
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v8 offset:18
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v69 offset:17
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v8 offset:16
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v6 offset:10
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v70 offset:11
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v71 offset:9
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v7 offset:14
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:15
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v7 offset:12
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:13
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v6 offset:8
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v48 offset:7
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v5 offset:6
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v49 offset:5
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v5 offset:4
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v50 offset:3
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v4 offset:2
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v51 offset:1
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v4
+; ALIGNED-NEXT:    s_cbranch_scc1 .LBB7_2
+; ALIGNED-NEXT:  .LBB7_3: ; %Flow6
+; ALIGNED-NEXT:    s_andn2_saveexec_b32 s8, s6
+; ALIGNED-NEXT:    s_cbranch_execz .LBB7_6
+; ALIGNED-NEXT:  ; %bb.4: ; %memmove_bwd_loop.preheader
+; ALIGNED-NEXT:    s_movk_i32 s6, 0xff00
+; ALIGNED-NEXT:    s_mov_b64 s[4:5], 0x700
+; ALIGNED-NEXT:    s_mov_b32 s7, -1
+; ALIGNED-NEXT:  .LBB7_5: ; %memmove_bwd_loop
+; ALIGNED-NEXT:    ; =>This Inner Loop Header: Depth=1
+; ALIGNED-NEXT:    v_add_co_u32 v4, vcc_lo, v2, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, s5, v3, vcc_lo
+; ALIGNED-NEXT:    v_add_co_u32 v96, vcc_lo, v0, s4
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v97, vcc_lo, s5, v1, vcc_lo
+; ALIGNED-NEXT:    s_clause 0xf
+; ALIGNED-NEXT:    global_load_dwordx4 v[98:101], v[4:5], off offset:240
+; ALIGNED-NEXT:    global_load_dwordx4 v[84:87], v[4:5], off offset:224
+; ALIGNED-NEXT:    global_load_dwordx4 v[80:83], v[4:5], off offset:208
+; ALIGNED-NEXT:    global_load_dwordx4 v[68:71], v[4:5], off offset:192
+; ALIGNED-NEXT:    global_load_dwordx4 v[64:67], v[4:5], off offset:176
+; ALIGNED-NEXT:    global_load_dwordx4 v[52:55], v[4:5], off offset:160
+; ALIGNED-NEXT:    global_load_dwordx4 v[48:51], v[4:5], off offset:144
+; ALIGNED-NEXT:    global_load_dwordx4 v[36:39], v[4:5], off offset:128
+; ALIGNED-NEXT:    global_load_dwordx4 v[32:35], v[4:5], off offset:112
+; ALIGNED-NEXT:    global_load_dwordx4 v[28:31], v[4:5], off offset:96
+; ALIGNED-NEXT:    global_load_dwordx4 v[24:27], v[4:5], off offset:80
+; ALIGNED-NEXT:    global_load_dwordx4 v[20:23], v[4:5], off offset:64
+; ALIGNED-NEXT:    global_load_dwordx4 v[16:19], v[4:5], off offset:48
+; ALIGNED-NEXT:    global_load_dwordx4 v[12:15], v[4:5], off offset:32
+; ALIGNED-NEXT:    global_load_dwordx4 v[8:11], v[4:5], off offset:16
+; ALIGNED-NEXT:    global_load_dwordx4 v[4:7], v[4:5], off
+; ALIGNED-NEXT:    s_add_u32 s4, s4, 0xffffff00
+; ALIGNED-NEXT:    s_addc_u32 s5, s5, -1
+; ALIGNED-NEXT:    s_waitcnt vmcnt(15)
+; ALIGNED-NEXT:    buffer_store_dword v100, off, s[0:3], s32 offset:424
+; ALIGNED-NEXT:    buffer_store_dword v101, off, s[0:3], s32 offset:428
+; ALIGNED-NEXT:    buffer_store_dword v99, off, s[0:3], s32 offset:420
+; ALIGNED-NEXT:    buffer_store_dword v98, off, s[0:3], s32 offset:416
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v100 offset:250
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v101 offset:254
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:252
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:248
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v99 offset:246
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:244
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v98 offset:242
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:240
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v100
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v100
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v101
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 8, v101
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 24, v99
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v99
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 24, v98
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v98
+; ALIGNED-NEXT:    s_waitcnt vmcnt(14)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v86
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 8, v86
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:251
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v87
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:249
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v87
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v103 offset:255
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v85
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v102 offset:253
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 8, v85
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:247
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 24, v84
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:245
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v84
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:243
+; ALIGNED-NEXT:    s_waitcnt vmcnt(13)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 24, v82
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:241
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v82
+; ALIGNED-NEXT:    buffer_store_dword v86, off, s[0:3], s32 offset:440
+; ALIGNED-NEXT:    buffer_store_dword v87, off, s[0:3], s32 offset:444
+; ALIGNED-NEXT:    buffer_store_dword v85, off, s[0:3], s32 offset:436
+; ALIGNED-NEXT:    buffer_store_dword v84, off, s[0:3], s32 offset:432
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v86 offset:234
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v87 offset:238
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v87 offset:236
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v86 offset:232
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v85 offset:230
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v85 offset:228
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v84 offset:226
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v84 offset:224
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 24, v83
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v83
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v81
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v81
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:235
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v80
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:233
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 8, v80
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:239
+; ALIGNED-NEXT:    s_waitcnt vmcnt(12)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v70
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:237
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v70
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v103 offset:231
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v71
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v102 offset:229
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 8, v71
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:227
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 24, v69
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:225
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v69
+; ALIGNED-NEXT:    buffer_store_dword v82, off, s[0:3], s32 offset:392
+; ALIGNED-NEXT:    buffer_store_dword v83, off, s[0:3], s32 offset:396
+; ALIGNED-NEXT:    buffer_store_dword v81, off, s[0:3], s32 offset:388
+; ALIGNED-NEXT:    buffer_store_dword v80, off, s[0:3], s32 offset:384
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v82 offset:218
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v83 offset:222
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v83 offset:220
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v82 offset:216
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v81 offset:214
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v81 offset:212
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v80 offset:210
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v80 offset:208
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 24, v68
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 8, v68
+; ALIGNED-NEXT:    s_waitcnt vmcnt(11)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 24, v66
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 8, v66
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:219
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 24, v67
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:217
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v67
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v84 offset:223
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 24, v65
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v85 offset:221
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v65
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v86 offset:215
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v64
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v87 offset:213
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v64
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:211
+; ALIGNED-NEXT:    s_waitcnt vmcnt(10)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v54
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:209
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 8, v54
+; ALIGNED-NEXT:    buffer_store_dword v70, off, s[0:3], s32 offset:408
+; ALIGNED-NEXT:    buffer_store_dword v71, off, s[0:3], s32 offset:412
+; ALIGNED-NEXT:    buffer_store_dword v69, off, s[0:3], s32 offset:404
+; ALIGNED-NEXT:    buffer_store_dword v68, off, s[0:3], s32 offset:400
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v70 offset:202
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v71 offset:206
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v71 offset:204
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v70 offset:200
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v69 offset:198
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v69 offset:196
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v68 offset:194
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v68 offset:192
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v69, 8, v55
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 24, v52
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 8, v52
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v68, 24, v55
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:203
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 24, v53
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:201
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v53
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v103 offset:207
+; ALIGNED-NEXT:    s_waitcnt vmcnt(9)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v50
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v102 offset:205
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 8, v50
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:199
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 24, v51
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:197
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v51
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v80 offset:195
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 24, v49
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v81 offset:193
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 8, v49
+; ALIGNED-NEXT:    buffer_store_dword v66, off, s[0:3], s32 offset:488
+; ALIGNED-NEXT:    buffer_store_dword v67, off, s[0:3], s32 offset:492
+; ALIGNED-NEXT:    buffer_store_dword v65, off, s[0:3], s32 offset:484
+; ALIGNED-NEXT:    buffer_store_dword v64, off, s[0:3], s32 offset:480
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v66 offset:186
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v67 offset:190
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v67 offset:188
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v66 offset:184
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v65 offset:182
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v65 offset:180
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v64 offset:178
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v64 offset:176
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 24, v48
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 8, v48
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v82 offset:187
+; ALIGNED-NEXT:    s_waitcnt vmcnt(8)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 24, v39
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 24, v38
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 8, v38
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v83 offset:185
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 8, v39
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:191
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 24, v37
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:189
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v37
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v84 offset:183
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 24, v36
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v85 offset:181
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v36
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v86 offset:179
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v34
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v87 offset:177
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v34
+; ALIGNED-NEXT:    buffer_store_dword v55, off, s[0:3], s32 offset:508
+; ALIGNED-NEXT:    buffer_store_dword v54, off, s[0:3], s32 offset:504
+; ALIGNED-NEXT:    buffer_store_dword v53, off, s[0:3], s32 offset:500
+; ALIGNED-NEXT:    buffer_store_dword v52, off, s[0:3], s32 offset:496
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v54 offset:170
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v54 offset:168
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v55 offset:174
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v55 offset:172
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v52 offset:162
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v52 offset:160
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v53 offset:166
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v53 offset:164
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v53, 24, v35
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v54, 8, v35
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v55, 24, v33
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v116, 8, v33
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:171
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v32
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:169
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 8, v32
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v69 offset:173
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v69, 24, v31
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v70 offset:163
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 8, v31
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v71 offset:161
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 24, v29
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v68 offset:175
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v52, 24, v30
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v68, 8, v30
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:167
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:165
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v29
+; ALIGNED-NEXT:    buffer_store_dword v50, off, s[0:3], s32 offset:456
+; ALIGNED-NEXT:    buffer_store_dword v51, off, s[0:3], s32 offset:460
+; ALIGNED-NEXT:    buffer_store_dword v49, off, s[0:3], s32 offset:452
+; ALIGNED-NEXT:    buffer_store_dword v48, off, s[0:3], s32 offset:448
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v50 offset:154
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v51 offset:158
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v51 offset:156
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v50 offset:152
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v49 offset:150
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v49 offset:148
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v48 offset:146
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v48 offset:144
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v48, 24, v28
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 8, v28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v50, 24, v26
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v103 offset:155
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v102 offset:153
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:159
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:157
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v80 offset:151
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v81 offset:149
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v64 offset:147
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v65 offset:145
+; ALIGNED-NEXT:    buffer_store_dword v38, off, s[0:3], s32 offset:472
+; ALIGNED-NEXT:    buffer_store_dword v39, off, s[0:3], s32 offset:476
+; ALIGNED-NEXT:    buffer_store_dword v37, off, s[0:3], s32 offset:468
+; ALIGNED-NEXT:    buffer_store_dword v36, off, s[0:3], s32 offset:464
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v38 offset:138
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v39 offset:142
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v39 offset:140
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v38 offset:136
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v37 offset:134
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v37 offset:132
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v36 offset:130
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v36 offset:128
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v82 offset:143
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v82, 24, v18
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v51, 8, v26
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v66 offset:139
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v67 offset:137
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v83 offset:141
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:135
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:133
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v84 offset:131
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v85 offset:129
+; ALIGNED-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:296
+; ALIGNED-NEXT:    buffer_store_dword v35, off, s[0:3], s32 offset:300
+; ALIGNED-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:288
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v34 offset:122
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v35 offset:126
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v35 offset:124
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v34 offset:120
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v33 offset:118
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v33 offset:116
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v32 offset:114
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v32 offset:112
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v34, 24, v14
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v86 offset:123
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v87 offset:121
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v53 offset:127
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v54 offset:125
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v55 offset:119
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v116 offset:117
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:115
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v114, 24, v10
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:113
+; ALIGNED-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:312
+; ALIGNED-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:316
+; ALIGNED-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:308
+; ALIGNED-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:304
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v30 offset:106
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v31 offset:110
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v31 offset:108
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v30 offset:104
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v29 offset:102
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v29 offset:100
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v28 offset:98
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v28 offset:96
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v69 offset:111
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v69, 24, v6
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v103, 24, v27
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v83, 8, v18
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v35, 8, v14
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v115, 8, v10
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v70 offset:109
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v70, 8, v6
+; ALIGNED-NEXT:    s_cmp_eq_u64 s[4:5], s[6:7]
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v102, 8, v27
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v101, 24, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v99, 24, v19
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v86, 24, v15
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v28, 24, v11
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v71 offset:103
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v71, 24, v7
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v100, 8, v25
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v80, 24, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v81, 8, v24
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v64, 24, v22
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v65, 8, v22
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v36, 24, v23
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v37, 8, v23
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v38, 24, v21
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v39, 8, v21
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v66, 24, v20
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v67, 8, v20
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v98, 8, v19
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v84, 24, v17
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v85, 8, v17
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v32, 24, v16
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v33, 8, v16
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v87, 8, v15
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v53, 24, v13
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v54, 8, v13
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v55, 24, v12
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v113, 8, v12
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v29, 8, v11
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v30, 24, v9
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v31, 8, v9
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v52 offset:107
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v52, 24, v8
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v68 offset:105
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v68, 8, v8
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:101
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v112, 8, v7
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v48 offset:99
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v48, 24, v5
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v49 offset:97
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v49, 8, v5
+; ALIGNED-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:264
+; ALIGNED-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:268
+; ALIGNED-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:260
+; ALIGNED-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:256
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v26 offset:90
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v27 offset:94
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v27 offset:92
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v26 offset:88
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v25 offset:86
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v25 offset:84
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v24 offset:82
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v24 offset:80
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v24, 24, v4
+; ALIGNED-NEXT:    v_lshrrev_b32_e32 v25, 8, v4
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v50 offset:91
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v51 offset:89
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v103 offset:95
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v102 offset:93
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v101 offset:87
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v100 offset:85
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v80 offset:83
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v81 offset:81
+; ALIGNED-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:280
+; ALIGNED-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:284
+; ALIGNED-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:276
+; ALIGNED-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:272
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v22 offset:74
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v23 offset:78
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v23 offset:76
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v22 offset:72
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v21 offset:70
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v21 offset:68
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v20 offset:66
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v20 offset:64
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v64 offset:75
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v65 offset:73
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v36 offset:79
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v37 offset:77
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v38 offset:71
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v39 offset:69
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v66 offset:67
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v67 offset:65
+; ALIGNED-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:360
+; ALIGNED-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:364
+; ALIGNED-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:356
+; ALIGNED-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:352
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v82 offset:59
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v18 offset:58
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v83 offset:57
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v19 offset:62
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v99 offset:63
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v19 offset:60
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v98 offset:61
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v18 offset:56
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v84 offset:55
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v17 offset:54
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v85 offset:53
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v17 offset:52
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v32 offset:51
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v16 offset:50
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v33 offset:49
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v16 offset:48
+; ALIGNED-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:376
+; ALIGNED-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:380
+; ALIGNED-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:372
+; ALIGNED-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:368
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v14 offset:42
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v34 offset:43
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v35 offset:41
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v15 offset:46
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v86 offset:47
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v15 offset:44
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v87 offset:45
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v14 offset:40
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v53 offset:39
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v13 offset:38
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v54 offset:37
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v13 offset:36
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v55 offset:35
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v12 offset:34
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v113 offset:33
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v12 offset:32
+; ALIGNED-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:328
+; ALIGNED-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:332
+; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:324
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:320
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v10 offset:26
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v114 offset:27
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v115 offset:25
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v11 offset:30
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v28 offset:31
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v11 offset:28
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v29 offset:29
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v10 offset:24
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v30 offset:23
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v9 offset:22
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v31 offset:21
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v9 offset:20
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v52 offset:19
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v8 offset:18
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v68 offset:17
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v8 offset:16
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:344
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:348
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:340
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:336
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v6 offset:10
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v69 offset:11
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v70 offset:9
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v7 offset:14
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v71 offset:15
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v7 offset:12
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v112 offset:13
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v6 offset:8
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v48 offset:7
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v5 offset:6
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v49 offset:5
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v5 offset:4
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v24 offset:3
+; ALIGNED-NEXT:    flat_store_byte_d16_hi v[96:97], v4 offset:2
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v25 offset:1
+; ALIGNED-NEXT:    flat_store_byte v[96:97], v4
+; ALIGNED-NEXT:    s_cbranch_scc0 .LBB7_5
+; ALIGNED-NEXT:  .LBB7_6: ; %Flow7
+; ALIGNED-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; UNROLL3-LABEL: memmove_p0_p4_sz2048:
+; UNROLL3:       ; %bb.0: ; %entry
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    s_mov_b32 s4, exec_lo
+; UNROLL3-NEXT:    v_cmpx_ge_u64_e64 v[2:3], v[0:1]
+; UNROLL3-NEXT:    s_xor_b32 s6, exec_lo, s4
+; UNROLL3-NEXT:    s_cbranch_execz .LBB7_4
+; UNROLL3-NEXT:  ; %bb.1: ; %memmove_fwd_loop.preheader
+; UNROLL3-NEXT:    s_mov_b64 s[4:5], 0
+; UNROLL3-NEXT:    .p2align 6
+; UNROLL3-NEXT:  .LBB7_2: ; %memmove_fwd_loop
+; UNROLL3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT:    v_add_co_u32 v12, vcc_lo, v2, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v13, vcc_lo, s5, v3, vcc_lo
+; UNROLL3-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; UNROLL3-NEXT:    s_clause 0x2
+; UNROLL3-NEXT:    global_load_dwordx4 v[4:7], v[12:13], off offset:16
+; UNROLL3-NEXT:    global_load_dwordx4 v[8:11], v[12:13], off
+; UNROLL3-NEXT:    global_load_dwordx4 v[12:15], v[12:13], off offset:32
+; UNROLL3-NEXT:    s_add_u32 s4, s4, 48
+; UNROLL3-NEXT:    s_addc_u32 s5, s5, 0
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[4:7] offset:16
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[8:11]
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[12:15] offset:32
+; UNROLL3-NEXT:    s_cmp_lg_u64 s[4:5], 0x7e0
+; UNROLL3-NEXT:    s_cbranch_scc1 .LBB7_2
+; UNROLL3-NEXT:  ; %bb.3: ; %memmove_fwd_residual
+; UNROLL3-NEXT:    s_clause 0x1
+; UNROLL3-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:2016
+; UNROLL3-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off offset:2032
+; UNROLL3-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:2016
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[8:11] offset:2032
+; UNROLL3-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; UNROLL3-NEXT:  .LBB7_4: ; %Flow4
+; UNROLL3-NEXT:    s_andn2_saveexec_b32 s8, s6
+; UNROLL3-NEXT:    s_cbranch_execz .LBB7_7
+; UNROLL3-NEXT:  ; %bb.5: ; %memmove_bwd_residual
+; UNROLL3-NEXT:    s_clause 0x1
+; UNROLL3-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:2032
+; UNROLL3-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off offset:2016
+; UNROLL3-NEXT:    s_movk_i32 s6, 0xffd0
+; UNROLL3-NEXT:    s_mov_b64 s[4:5], 0x7b0
+; UNROLL3-NEXT:    s_mov_b32 s7, -1
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:2032
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[8:11] offset:2016
+; UNROLL3-NEXT:    .p2align 6
+; UNROLL3-NEXT:  .LBB7_6: ; %memmove_bwd_loop
+; UNROLL3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT:    v_add_co_u32 v12, vcc_lo, v2, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v13, vcc_lo, s5, v3, vcc_lo
+; UNROLL3-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; UNROLL3-NEXT:    s_clause 0x2
+; UNROLL3-NEXT:    global_load_dwordx4 v[4:7], v[12:13], off offset:16
+; UNROLL3-NEXT:    global_load_dwordx4 v[8:11], v[12:13], off
+; UNROLL3-NEXT:    global_load_dwordx4 v[12:15], v[12:13], off offset:32
+; UNROLL3-NEXT:    s_add_u32 s4, s4, 0xffffffd0
+; UNROLL3-NEXT:    s_addc_u32 s5, s5, -1
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[4:7] offset:16
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[8:11]
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[12:15] offset:32
+; UNROLL3-NEXT:    s_cmp_eq_u64 s[4:5], s[6:7]
+; UNROLL3-NEXT:    s_cbranch_scc0 .LBB7_6
+; UNROLL3-NEXT:  .LBB7_7: ; %Flow5
+; UNROLL3-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; UNROLL3-NEXT:    s_waitcnt lgkmcnt(0)
+; UNROLL3-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  tail call void @llvm.memmove.p0.p4.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 2048, i1 false)
+  ret void
+}
+
+define void @memmove_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {
+; CHECK-LABEL: memmove_p5_p5_sz2048:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s4, exec_lo
+; CHECK-NEXT:    v_cmpx_ge_u32_e64 v1, v0
+; CHECK-NEXT:    s_xor_b32 s6, exec_lo, s4
+; CHECK-NEXT:    s_cbranch_execz .LBB8_3
+; CHECK-NEXT:  ; %bb.1: ; %memmove_fwd_loop.preheader
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0x800
+; CHECK-NEXT:  .LBB8_2: ; %memmove_fwd_loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_clause 0x3e
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:252
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:248
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:244
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:240
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:236
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:232
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:228
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:224
+; CHECK-NEXT:    buffer_load_dword v10, v1, s[0:3], 0 offen offset:220
+; CHECK-NEXT:    buffer_load_dword v11, v1, s[0:3], 0 offen offset:216
+; CHECK-NEXT:    buffer_load_dword v12, v1, s[0:3], 0 offen offset:212
+; CHECK-NEXT:    buffer_load_dword v13, v1, s[0:3], 0 offen offset:208
+; CHECK-NEXT:    buffer_load_dword v14, v1, s[0:3], 0 offen offset:204
+; CHECK-NEXT:    buffer_load_dword v15, v1, s[0:3], 0 offen offset:200
+; CHECK-NEXT:    buffer_load_dword v16, v1, s[0:3], 0 offen offset:196
+; CHECK-NEXT:    buffer_load_dword v17, v1, s[0:3], 0 offen offset:192
+; CHECK-NEXT:    buffer_load_dword v18, v1, s[0:3], 0 offen offset:188
+; CHECK-NEXT:    buffer_load_dword v19, v1, s[0:3], 0 offen offset:184
+; CHECK-NEXT:    buffer_load_dword v20, v1, s[0:3], 0 offen offset:180
+; CHECK-NEXT:    buffer_load_dword v21, v1, s[0:3], 0 offen offset:176
+; CHECK-NEXT:    buffer_load_dword v22, v1, s[0:3], 0 offen offset:172
+; CHECK-NEXT:    buffer_load_dword v23, v1, s[0:3], 0 offen offset:168
+; CHECK-NEXT:    buffer_load_dword v24, v1, s[0:3], 0 offen offset:164
+; CHECK-NEXT:    buffer_load_dword v25, v1, s[0:3], 0 offen offset:160
+; CHECK-NEXT:    buffer_load_dword v26, v1, s[0:3], 0 offen offset:156
+; CHECK-NEXT:    buffer_load_dword v27, v1, s[0:3], 0 offen offset:152
+; CHECK-NEXT:    buffer_load_dword v28, v1, s[0:3], 0 offen offset:148
+; CHECK-NEXT:    buffer_load_dword v29, v1, s[0:3], 0 offen offset:144
+; CHECK-NEXT:    buffer_load_dword v30, v1, s[0:3], 0 offen offset:140
+; CHECK-NEXT:    buffer_load_dword v31, v1, s[0:3], 0 offen offset:136
+; CHECK-NEXT:    buffer_load_dword v32, v1, s[0:3], 0 offen offset:132
+; CHECK-NEXT:    buffer_load_dword v33, v1, s[0:3], 0 offen offset:128
+; CHECK-NEXT:    buffer_load_dword v34, v1, s[0:3], 0 offen offset:124
+; CHECK-NEXT:    buffer_load_dword v35, v1, s[0:3], 0 offen offset:120
+; CHECK-NEXT:    buffer_load_dword v36, v1, s[0:3], 0 offen offset:116
+; CHECK-NEXT:    buffer_load_dword v37, v1, s[0:3], 0 offen offset:112
+; CHECK-NEXT:    buffer_load_dword v38, v1, s[0:3], 0 offen offset:108
+; CHECK-NEXT:    buffer_load_dword v39, v1, s[0:3], 0 offen offset:104
+; CHECK-NEXT:    buffer_load_dword v48, v1, s[0:3], 0 offen offset:100
+; CHECK-NEXT:    buffer_load_dword v49, v1, s[0:3], 0 offen offset:96
+; CHECK-NEXT:    buffer_load_dword v50, v1, s[0:3], 0 offen offset:92
+; CHECK-NEXT:    buffer_load_dword v51, v1, s[0:3], 0 offen offset:88
+; CHECK-NEXT:    buffer_load_dword v52, v1, s[0:3], 0 offen offset:84
+; CHECK-NEXT:    buffer_load_dword v53, v1, s[0:3], 0 offen offset:80
+; CHECK-NEXT:    buffer_load_dword v54, v1, s[0:3], 0 offen offset:76
+; CHECK-NEXT:    buffer_load_dword v55, v1, s[0:3], 0 offen offset:72
+; CHECK-NEXT:    buffer_load_dword v64, v1, s[0:3], 0 offen offset:68
+; CHECK-NEXT:    buffer_load_dword v65, v1, s[0:3], 0 offen offset:64
+; CHECK-NEXT:    buffer_load_dword v66, v1, s[0:3], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v67, v1, s[0:3], 0 offen offset:56
+; CHECK-NEXT:    buffer_load_dword v68, v1, s[0:3], 0 offen offset:52
+; CHECK-NEXT:    buffer_load_dword v69, v1, s[0:3], 0 offen offset:48
+; CHECK-NEXT:    buffer_load_dword v70, v1, s[0:3], 0 offen offset:44
+; CHECK-NEXT:    buffer_load_dword v71, v1, s[0:3], 0 offen offset:40
+; CHECK-NEXT:    buffer_load_dword v80, v1, s[0:3], 0 offen offset:36
+; CHECK-NEXT:    buffer_load_dword v81, v1, s[0:3], 0 offen offset:32
+; CHECK-NEXT:    buffer_load_dword v82, v1, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v83, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v84, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v85, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v86, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v87, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v96, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v97, v1, s[0:3], 0 offen
+; CHECK-NEXT:    v_add_nc_u32_e32 v1, 0x100, v1
+; CHECK-NEXT:    s_add_u32 s4, s4, 0xffffff00
+; CHECK-NEXT:    s_addc_u32 s5, s5, -1
+; CHECK-NEXT:    s_waitcnt vmcnt(62)
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:252
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:248
+; CHECK-NEXT:    s_waitcnt vmcnt(61)
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:244
+; CHECK-NEXT:    s_waitcnt vmcnt(60)
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(59)
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:236
+; CHECK-NEXT:    s_waitcnt vmcnt(58)
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:232
+; CHECK-NEXT:    s_waitcnt vmcnt(57)
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:228
+; CHECK-NEXT:    s_waitcnt vmcnt(56)
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:224
+; CHECK-NEXT:    s_waitcnt vmcnt(55)
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:220
+; CHECK-NEXT:    s_waitcnt vmcnt(54)
+; CHECK-NEXT:    buffer_store_dword v11, v0, s[0:3], 0 offen offset:216
+; CHECK-NEXT:    s_waitcnt vmcnt(53)
+; CHECK-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen offset:212
+; CHECK-NEXT:    s_waitcnt vmcnt(52)
+; CHECK-NEXT:    buffer_store_dword v13, v0, s[0:3], 0 offen offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(51)
+; CHECK-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:204
+; CHECK-NEXT:    s_waitcnt vmcnt(50)
+; CHECK-NEXT:    buffer_store_dword v15, v0, s[0:3], 0 offen offset:200
+; CHECK-NEXT:    s_waitcnt vmcnt(49)
+; CHECK-NEXT:    buffer_store_dword v16, v0, s[0:3], 0 offen offset:196
+; CHECK-NEXT:    s_waitcnt vmcnt(48)
+; CHECK-NEXT:    buffer_store_dword v17, v0, s[0:3], 0 offen offset:192
+; CHECK-NEXT:    s_waitcnt vmcnt(47)
+; CHECK-NEXT:    buffer_store_dword v18, v0, s[0:3], 0 offen offset:188
+; CHECK-NEXT:    s_waitcnt vmcnt(46)
+; CHECK-NEXT:    buffer_store_dword v19, v0, s[0:3], 0 offen offset:184
+; CHECK-NEXT:    s_waitcnt vmcnt(45)
+; CHECK-NEXT:    buffer_store_dword v20, v0, s[0:3], 0 offen offset:180
+; CHECK-NEXT:    s_waitcnt vmcnt(44)
+; CHECK-NEXT:    buffer_store_dword v21, v0, s[0:3], 0 offen offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(43)
+; CHECK-NEXT:    buffer_store_dword v22, v0, s[0:3], 0 offen offset:172
+; CHECK-NEXT:    s_waitcnt vmcnt(42)
+; CHECK-NEXT:    buffer_store_dword v23, v0, s[0:3], 0 offen offset:168
+; CHECK-NEXT:    s_waitcnt vmcnt(41)
+; CHECK-NEXT:    buffer_store_dword v24, v0, s[0:3], 0 offen offset:164
+; CHECK-NEXT:    s_waitcnt vmcnt(40)
+; CHECK-NEXT:    buffer_store_dword v25, v0, s[0:3], 0 offen offset:160
+; CHECK-NEXT:    s_waitcnt vmcnt(39)
+; CHECK-NEXT:    buffer_store_dword v26, v0, s[0:3], 0 offen offset:156
+; CHECK-NEXT:    s_waitcnt vmcnt(38)
+; CHECK-NEXT:    buffer_store_dword v27, v0, s[0:3], 0 offen offset:152
+; CHECK-NEXT:    s_waitcnt vmcnt(37)
+; CHECK-NEXT:    buffer_store_dword v28, v0, s[0:3], 0 offen offset:148
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_dword v29, v0, s[0:3], 0 offen offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(35)
+; CHECK-NEXT:    buffer_store_dword v30, v0, s[0:3], 0 offen offset:140
+; CHECK-NEXT:    s_waitcnt vmcnt(34)
+; CHECK-NEXT:    buffer_store_dword v31, v0, s[0:3], 0 offen offset:136
+; CHECK-NEXT:    s_waitcnt vmcnt(33)
+; CHECK-NEXT:    buffer_store_dword v32, v0, s[0:3], 0 offen offset:132
+; CHECK-NEXT:    s_waitcnt vmcnt(32)
+; CHECK-NEXT:    buffer_store_dword v33, v0, s[0:3], 0 offen offset:128
+; CHECK-NEXT:    s_waitcnt vmcnt(31)
+; CHECK-NEXT:    buffer_store_dword v34, v0, s[0:3], 0 offen offset:124
+; CHECK-NEXT:    s_waitcnt vmcnt(30)
+; CHECK-NEXT:    buffer_store_dword v35, v0, s[0:3], 0 offen offset:120
+; CHECK-NEXT:    s_waitcnt vmcnt(29)
+; CHECK-NEXT:    buffer_store_dword v36, v0, s[0:3], 0 offen offset:116
+; CHECK-NEXT:    s_waitcnt vmcnt(28)
+; CHECK-NEXT:    buffer_store_dword v37, v0, s[0:3], 0 offen offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(27)
+; CHECK-NEXT:    buffer_store_dword v38, v0, s[0:3], 0 offen offset:108
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_dword v39, v0, s[0:3], 0 offen offset:104
+; CHECK-NEXT:    s_waitcnt vmcnt(25)
+; CHECK-NEXT:    buffer_store_dword v48, v0, s[0:3], 0 offen offset:100
+; CHECK-NEXT:    s_waitcnt vmcnt(24)
+; CHECK-NEXT:    buffer_store_dword v49, v0, s[0:3], 0 offen offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(23)
+; CHECK-NEXT:    buffer_store_dword v50, v0, s[0:3], 0 offen offset:92
+; CHECK-NEXT:    s_waitcnt vmcnt(22)
+; CHECK-NEXT:    buffer_store_dword v51, v0, s[0:3], 0 offen offset:88
+; CHECK-NEXT:    s_waitcnt vmcnt(21)
+; CHECK-NEXT:    buffer_store_dword v52, v0, s[0:3], 0 offen offset:84
+; CHECK-NEXT:    s_waitcnt vmcnt(20)
+; CHECK-NEXT:    buffer_store_dword v53, v0, s[0:3], 0 offen offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(19)
+; CHECK-NEXT:    buffer_store_dword v54, v0, s[0:3], 0 offen offset:76
+; CHECK-NEXT:    s_waitcnt vmcnt(18)
+; CHECK-NEXT:    buffer_store_dword v55, v0, s[0:3], 0 offen offset:72
+; CHECK-NEXT:    s_waitcnt vmcnt(17)
+; CHECK-NEXT:    buffer_store_dword v64, v0, s[0:3], 0 offen offset:68
+; CHECK-NEXT:    s_waitcnt vmcnt(16)
+; CHECK-NEXT:    buffer_store_dword v65, v0, s[0:3], 0 offen offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(15)
+; CHECK-NEXT:    buffer_store_dword v66, v0, s[0:3], 0 offen offset:60
+; CHECK-NEXT:    s_waitcnt vmcnt(14)
+; CHECK-NEXT:    buffer_store_dword v67, v0, s[0:3], 0 offen offset:56
+; CHECK-NEXT:    s_waitcnt vmcnt(13)
+; CHECK-NEXT:    buffer_store_dword v68, v0, s[0:3], 0 offen offset:52
+; CHECK-NEXT:    s_waitcnt vmcnt(12)
+; CHECK-NEXT:    buffer_store_dword v69, v0, s[0:3], 0 offen offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(11)
+; CHECK-NEXT:    buffer_store_dword v70, v0, s[0:3], 0 offen offset:44
+; CHECK-NEXT:    s_waitcnt vmcnt(10)
+; CHECK-NEXT:    buffer_store_dword v71, v0, s[0:3], 0 offen offset:40
+; CHECK-NEXT:    s_waitcnt vmcnt(9)
+; CHECK-NEXT:    buffer_store_dword v80, v0, s[0:3], 0 offen offset:36
+; CHECK-NEXT:    s_waitcnt vmcnt(8)
+; CHECK-NEXT:    buffer_store_dword v81, v0, s[0:3], 0 offen offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(7)
+; CHECK-NEXT:    buffer_store_dword v82, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(6)
+; CHECK-NEXT:    buffer_store_dword v83, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    buffer_store_dword v84, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    buffer_store_dword v85, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    buffer_store_dword v86, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    s_waitcnt vmcnt(2)
+; CHECK-NEXT:    buffer_store_dword v87, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    buffer_store_dword v96, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    buffer_store_dword v97, v0, s[0:3], 0 offen
+; CHECK-NEXT:    v_add_nc_u32_e32 v0, 0x100, v0
+; CHECK-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; CHECK-NEXT:    s_cbranch_scc1 .LBB8_2
+; CHECK-NEXT:  .LBB8_3: ; %Flow18
+; CHECK-NEXT:    s_andn2_saveexec_b32 s6, s6
+; CHECK-NEXT:    s_cbranch_execz .LBB8_6
+; CHECK-NEXT:  ; %bb.4: ; %memmove_bwd_loop.preheader
+; CHECK-NEXT:    v_add_nc_u32_e32 v0, 0x700, v0
+; CHECK-NEXT:    v_add_nc_u32_e32 v1, 0x700, v1
+; CHECK-NEXT:    s_movk_i32 s4, 0xf800
+; CHECK-NEXT:    s_mov_b32 s5, -1
+; CHECK-NEXT:  .LBB8_5: ; %memmove_bwd_loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_clause 0x3e
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:252
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:248
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:244
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:240
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:236
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:232
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:228
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:224
+; CHECK-NEXT:    buffer_load_dword v10, v1, s[0:3], 0 offen offset:220
+; CHECK-NEXT:    buffer_load_dword v11, v1, s[0:3], 0 offen offset:216
+; CHECK-NEXT:    buffer_load_dword v12, v1, s[0:3], 0 offen offset:212
+; CHECK-NEXT:    buffer_load_dword v13, v1, s[0:3], 0 offen offset:208
+; CHECK-NEXT:    buffer_load_dword v14, v1, s[0:3], 0 offen offset:204
+; CHECK-NEXT:    buffer_load_dword v15, v1, s[0:3], 0 offen offset:200
+; CHECK-NEXT:    buffer_load_dword v16, v1, s[0:3], 0 offen offset:196
+; CHECK-NEXT:    buffer_load_dword v17, v1, s[0:3], 0 offen offset:192
+; CHECK-NEXT:    buffer_load_dword v18, v1, s[0:3], 0 offen offset:188
+; CHECK-NEXT:    buffer_load_dword v19, v1, s[0:3], 0 offen offset:184
+; CHECK-NEXT:    buffer_load_dword v20, v1, s[0:3], 0 offen offset:180
+; CHECK-NEXT:    buffer_load_dword v21, v1, s[0:3], 0 offen offset:176
+; CHECK-NEXT:    buffer_load_dword v22, v1, s[0:3], 0 offen offset:172
+; CHECK-NEXT:    buffer_load_dword v23, v1, s[0:3], 0 offen offset:168
+; CHECK-NEXT:    buffer_load_dword v24, v1, s[0:3], 0 offen offset:164
+; CHECK-NEXT:    buffer_load_dword v25, v1, s[0:3], 0 offen offset:160
+; CHECK-NEXT:    buffer_load_dword v26, v1, s[0:3], 0 offen offset:156
+; CHECK-NEXT:    buffer_load_dword v27, v1, s[0:3], 0 offen offset:152
+; CHECK-NEXT:    buffer_load_dword v28, v1, s[0:3], 0 offen offset:148
+; CHECK-NEXT:    buffer_load_dword v29, v1, s[0:3], 0 offen offset:144
+; CHECK-NEXT:    buffer_load_dword v30, v1, s[0:3], 0 offen offset:140
+; CHECK-NEXT:    buffer_load_dword v31, v1, s[0:3], 0 offen offset:136
+; CHECK-NEXT:    buffer_load_dword v32, v1, s[0:3], 0 offen offset:132
+; CHECK-NEXT:    buffer_load_dword v33, v1, s[0:3], 0 offen offset:128
+; CHECK-NEXT:    buffer_load_dword v34, v1, s[0:3], 0 offen offset:124
+; CHECK-NEXT:    buffer_load_dword v35, v1, s[0:3], 0 offen offset:120
+; CHECK-NEXT:    buffer_load_dword v36, v1, s[0:3], 0 offen offset:116
+; CHECK-NEXT:    buffer_load_dword v37, v1, s[0:3], 0 offen offset:112
+; CHECK-NEXT:    buffer_load_dword v38, v1, s[0:3], 0 offen offset:108
+; CHECK-NEXT:    buffer_load_dword v39, v1, s[0:3], 0 offen offset:104
+; CHECK-NEXT:    buffer_load_dword v48, v1, s[0:3], 0 offen offset:100
+; CHECK-NEXT:    buffer_load_dword v49, v1, s[0:3], 0 offen offset:96
+; CHECK-NEXT:    buffer_load_dword v50, v1, s[0:3], 0 offen offset:92
+; CHECK-NEXT:    buffer_load_dword v51, v1, s[0:3], 0 offen offset:88
+; CHECK-NEXT:    buffer_load_dword v52, v1, s[0:3], 0 offen offset:84
+; CHECK-NEXT:    buffer_load_dword v53, v1, s[0:3], 0 offen offset:80
+; CHECK-NEXT:    buffer_load_dword v54, v1, s[0:3], 0 offen offset:76
+; CHECK-NEXT:    buffer_load_dword v55, v1, s[0:3], 0 offen offset:72
+; CHECK-NEXT:    buffer_load_dword v64, v1, s[0:3], 0 offen offset:68
+; CHECK-NEXT:    buffer_load_dword v65, v1, s[0:3], 0 offen offset:64
+; CHECK-NEXT:    buffer_load_dword v66, v1, s[0:3], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v67, v1, s[0:3], 0 offen offset:56
+; CHECK-NEXT:    buffer_load_dword v68, v1, s[0:3], 0 offen offset:52
+; CHECK-NEXT:    buffer_load_dword v69, v1, s[0:3], 0 offen offset:48
+; CHECK-NEXT:    buffer_load_dword v70, v1, s[0:3], 0 offen offset:44
+; CHECK-NEXT:    buffer_load_dword v71, v1, s[0:3], 0 offen offset:40
+; CHECK-NEXT:    buffer_load_dword v80, v1, s[0:3], 0 offen offset:36
+; CHECK-NEXT:    buffer_load_dword v81, v1, s[0:3], 0 offen offset:32
+; CHECK-NEXT:    buffer_load_dword v82, v1, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v83, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v84, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v85, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v86, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v87, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v96, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v97, v1, s[0:3], 0 offen
+; CHECK-NEXT:    v_add_nc_u32_e32 v1, 0xffffff00, v1
+; CHECK-NEXT:    s_add_u32 s4, s4, 0x100
+; CHECK-NEXT:    s_addc_u32 s5, s5, 0
+; CHECK-NEXT:    s_waitcnt vmcnt(62)
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:252
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:248
+; CHECK-NEXT:    s_waitcnt vmcnt(61)
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:244
+; CHECK-NEXT:    s_waitcnt vmcnt(60)
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(59)
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:236
+; CHECK-NEXT:    s_waitcnt vmcnt(58)
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:232
+; CHECK-NEXT:    s_waitcnt vmcnt(57)
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:228
+; CHECK-NEXT:    s_waitcnt vmcnt(56)
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:224
+; CHECK-NEXT:    s_waitcnt vmcnt(55)
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:220
+; CHECK-NEXT:    s_waitcnt vmcnt(54)
+; CHECK-NEXT:    buffer_store_dword v11, v0, s[0:3], 0 offen offset:216
+; CHECK-NEXT:    s_waitcnt vmcnt(53)
+; CHECK-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen offset:212
+; CHECK-NEXT:    s_waitcnt vmcnt(52)
+; CHECK-NEXT:    buffer_store_dword v13, v0, s[0:3], 0 offen offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(51)
+; CHECK-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:204
+; CHECK-NEXT:    s_waitcnt vmcnt(50)
+; CHECK-NEXT:    buffer_store_dword v15, v0, s[0:3], 0 offen offset:200
+; CHECK-NEXT:    s_waitcnt vmcnt(49)
+; CHECK-NEXT:    buffer_store_dword v16, v0, s[0:3], 0 offen offset:196
+; CHECK-NEXT:    s_waitcnt vmcnt(48)
+; CHECK-NEXT:    buffer_store_dword v17, v0, s[0:3], 0 offen offset:192
+; CHECK-NEXT:    s_waitcnt vmcnt(47)
+; CHECK-NEXT:    buffer_store_dword v18, v0, s[0:3], 0 offen offset:188
+; CHECK-NEXT:    s_waitcnt vmcnt(46)
+; CHECK-NEXT:    buffer_store_dword v19, v0, s[0:3], 0 offen offset:184
+; CHECK-NEXT:    s_waitcnt vmcnt(45)
+; CHECK-NEXT:    buffer_store_dword v20, v0, s[0:3], 0 offen offset:180
+; CHECK-NEXT:    s_waitcnt vmcnt(44)
+; CHECK-NEXT:    buffer_store_dword v21, v0, s[0:3], 0 offen offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(43)
+; CHECK-NEXT:    buffer_store_dword v22, v0, s[0:3], 0 offen offset:172
+; CHECK-NEXT:    s_waitcnt vmcnt(42)
+; CHECK-NEXT:    buffer_store_dword v23, v0, s[0:3], 0 offen offset:168
+; CHECK-NEXT:    s_waitcnt vmcnt(41)
+; CHECK-NEXT:    buffer_store_dword v24, v0, s[0:3], 0 offen offset:164
+; CHECK-NEXT:    s_waitcnt vmcnt(40)
+; CHECK-NEXT:    buffer_store_dword v25, v0, s[0:3], 0 offen offset:160
+; CHECK-NEXT:    s_waitcnt vmcnt(39)
+; CHECK-NEXT:    buffer_store_dword v26, v0, s[0:3], 0 offen offset:156
+; CHECK-NEXT:    s_waitcnt vmcnt(38)
+; CHECK-NEXT:    buffer_store_dword v27, v0, s[0:3], 0 offen offset:152
+; CHECK-NEXT:    s_waitcnt vmcnt(37)
+; CHECK-NEXT:    buffer_store_dword v28, v0, s[0:3], 0 offen offset:148
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_dword v29, v0, s[0:3], 0 offen offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(35)
+; CHECK-NEXT:    buffer_store_dword v30, v0, s[0:3], 0 offen offset:140
+; CHECK-NEXT:    s_waitcnt vmcnt(34)
+; CHECK-NEXT:    buffer_store_dword v31, v0, s[0:3], 0 offen offset:136
+; CHECK-NEXT:    s_waitcnt vmcnt(33)
+; CHECK-NEXT:    buffer_store_dword v32, v0, s[0:3], 0 offen offset:132
+; CHECK-NEXT:    s_waitcnt vmcnt(32)
+; CHECK-NEXT:    buffer_store_dword v33, v0, s[0:3], 0 offen offset:128
+; CHECK-NEXT:    s_waitcnt vmcnt(31)
+; CHECK-NEXT:    buffer_store_dword v34, v0, s[0:3], 0 offen offset:124
+; CHECK-NEXT:    s_waitcnt vmcnt(30)
+; CHECK-NEXT:    buffer_store_dword v35, v0, s[0:3], 0 offen offset:120
+; CHECK-NEXT:    s_waitcnt vmcnt(29)
+; CHECK-NEXT:    buffer_store_dword v36, v0, s[0:3], 0 offen offset:116
+; CHECK-NEXT:    s_waitcnt vmcnt(28)
+; CHECK-NEXT:    buffer_store_dword v37, v0, s[0:3], 0 offen offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(27)
+; CHECK-NEXT:    buffer_store_dword v38, v0, s[0:3], 0 offen offset:108
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_dword v39, v0, s[0:3], 0 offen offset:104
+; CHECK-NEXT:    s_waitcnt vmcnt(25)
+; CHECK-NEXT:    buffer_store_dword v48, v0, s[0:3], 0 offen offset:100
+; CHECK-NEXT:    s_waitcnt vmcnt(24)
+; CHECK-NEXT:    buffer_store_dword v49, v0, s[0:3], 0 offen offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(23)
+; CHECK-NEXT:    buffer_store_dword v50, v0, s[0:3], 0 offen offset:92
+; CHECK-NEXT:    s_waitcnt vmcnt(22)
+; CHECK-NEXT:    buffer_store_dword v51, v0, s[0:3], 0 offen offset:88
+; CHECK-NEXT:    s_waitcnt vmcnt(21)
+; CHECK-NEXT:    buffer_store_dword v52, v0, s[0:3], 0 offen offset:84
+; CHECK-NEXT:    s_waitcnt vmcnt(20)
+; CHECK-NEXT:    buffer_store_dword v53, v0, s[0:3], 0 offen offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(19)
+; CHECK-NEXT:    buffer_store_dword v54, v0, s[0:3], 0 offen offset:76
+; CHECK-NEXT:    s_waitcnt vmcnt(18)
+; CHECK-NEXT:    buffer_store_dword v55, v0, s[0:3], 0 offen offset:72
+; CHECK-NEXT:    s_waitcnt vmcnt(17)
+; CHECK-NEXT:    buffer_store_dword v64, v0, s[0:3], 0 offen offset:68
+; CHECK-NEXT:    s_waitcnt vmcnt(16)
+; CHECK-NEXT:    buffer_store_dword v65, v0, s[0:3], 0 offen offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(15)
+; CHECK-NEXT:    buffer_store_dword v66, v0, s[0:3], 0 offen offset:60
+; CHECK-NEXT:    s_waitcnt vmcnt(14)
+; CHECK-NEXT:    buffer_store_dword v67, v0, s[0:3], 0 offen offset:56
+; CHECK-NEXT:    s_waitcnt vmcnt(13)
+; CHECK-NEXT:    buffer_store_dword v68, v0, s[0:3], 0 offen offset:52
+; CHECK-NEXT:    s_waitcnt vmcnt(12)
+; CHECK-NEXT:    buffer_store_dword v69, v0, s[0:3], 0 offen offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(11)
+; CHECK-NEXT:    buffer_store_dword v70, v0, s[0:3], 0 offen offset:44
+; CHECK-NEXT:    s_waitcnt vmcnt(10)
+; CHECK-NEXT:    buffer_store_dword v71, v0, s[0:3], 0 offen offset:40
+; CHECK-NEXT:    s_waitcnt vmcnt(9)
+; CHECK-NEXT:    buffer_store_dword v80, v0, s[0:3], 0 offen offset:36
+; CHECK-NEXT:    s_waitcnt vmcnt(8)
+; CHECK-NEXT:    buffer_store_dword v81, v0, s[0:3], 0 offen offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(7)
+; CHECK-NEXT:    buffer_store_dword v82, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(6)
+; CHECK-NEXT:    buffer_store_dword v83, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    buffer_store_dword v84, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    buffer_store_dword v85, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    buffer_store_dword v86, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    s_waitcnt vmcnt(2)
+; CHECK-NEXT:    buffer_store_dword v87, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    buffer_store_dword v96, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    buffer_store_dword v97, v0, s[0:3], 0 offen
+; CHECK-NEXT:    v_add_nc_u32_e32 v0, 0xffffff00, v0
+; CHECK-NEXT:    s_cmp_eq_u64 s[4:5], 0
+; CHECK-NEXT:    s_cbranch_scc0 .LBB8_5
+; CHECK-NEXT:  .LBB8_6: ; %Flow19
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s6
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+;
+; ALIGNED-LABEL: memmove_p5_p5_sz2048:
+; ALIGNED:       ; %bb.0: ; %entry
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v72, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v73, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v74, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v75, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v76, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v77, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v78, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v79, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v88, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v89, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v90, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v91, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v92, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v93, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v94, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v95, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v104, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v105, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v106, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v107, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v108, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v109, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v110, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v111, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v120, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v121, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v122, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v123, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v124, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v125, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v126, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v127, off, s[0:3], s32 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_mov_b32 s4, exec_lo
+; ALIGNED-NEXT:    v_cmpx_ge_u32_e64 v1, v0
+; ALIGNED-NEXT:    s_xor_b32 s6, exec_lo, s4
+; ALIGNED-NEXT:    s_cbranch_execz .LBB8_3
+; ALIGNED-NEXT:  ; %bb.1: ; %memmove_fwd_loop.preheader
+; ALIGNED-NEXT:    s_mov_b64 s[4:5], 0x800
+; ALIGNED-NEXT:  .LBB8_2: ; %memmove_fwd_loop
+; ALIGNED-NEXT:    ; =>This Inner Loop Header: Depth=1
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:255
+; ALIGNED-NEXT:    s_add_u32 s4, s4, 0xffffff00
+; ALIGNED-NEXT:    s_addc_u32 s5, s5, -1
+; ALIGNED-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:254
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:253
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:252
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:251
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:250
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:249
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:248
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x3e
+; ALIGNED-NEXT:    buffer_load_ubyte v123, v1, s[0:3], 0 offen offset:247
+; ALIGNED-NEXT:    buffer_load_ubyte v122, v1, s[0:3], 0 offen offset:246
+; ALIGNED-NEXT:    buffer_load_ubyte v121, v1, s[0:3], 0 offen offset:245
+; ALIGNED-NEXT:    buffer_load_ubyte v120, v1, s[0:3], 0 offen offset:244
+; ALIGNED-NEXT:    buffer_load_ubyte v111, v1, s[0:3], 0 offen offset:243
+; ALIGNED-NEXT:    buffer_load_ubyte v110, v1, s[0:3], 0 offen offset:242
+; ALIGNED-NEXT:    buffer_load_ubyte v109, v1, s[0:3], 0 offen offset:241
+; ALIGNED-NEXT:    buffer_load_ubyte v108, v1, s[0:3], 0 offen offset:240
+; ALIGNED-NEXT:    buffer_load_ubyte v107, v1, s[0:3], 0 offen offset:239
+; ALIGNED-NEXT:    buffer_load_ubyte v106, v1, s[0:3], 0 offen offset:238
+; ALIGNED-NEXT:    buffer_load_ubyte v105, v1, s[0:3], 0 offen offset:237
+; ALIGNED-NEXT:    buffer_load_ubyte v104, v1, s[0:3], 0 offen offset:236
+; ALIGNED-NEXT:    buffer_load_ubyte v95, v1, s[0:3], 0 offen offset:235
+; ALIGNED-NEXT:    buffer_load_ubyte v94, v1, s[0:3], 0 offen offset:234
+; ALIGNED-NEXT:    buffer_load_ubyte v93, v1, s[0:3], 0 offen offset:233
+; ALIGNED-NEXT:    buffer_load_ubyte v92, v1, s[0:3], 0 offen offset:232
+; ALIGNED-NEXT:    buffer_load_ubyte v91, v1, s[0:3], 0 offen offset:231
+; ALIGNED-NEXT:    buffer_load_ubyte v90, v1, s[0:3], 0 offen offset:230
+; ALIGNED-NEXT:    buffer_load_ubyte v89, v1, s[0:3], 0 offen offset:229
+; ALIGNED-NEXT:    buffer_load_ubyte v88, v1, s[0:3], 0 offen offset:228
+; ALIGNED-NEXT:    buffer_load_ubyte v79, v1, s[0:3], 0 offen offset:227
+; ALIGNED-NEXT:    buffer_load_ubyte v78, v1, s[0:3], 0 offen offset:226
+; ALIGNED-NEXT:    buffer_load_ubyte v77, v1, s[0:3], 0 offen offset:225
+; ALIGNED-NEXT:    buffer_load_ubyte v76, v1, s[0:3], 0 offen offset:224
+; ALIGNED-NEXT:    buffer_load_ubyte v75, v1, s[0:3], 0 offen offset:223
+; ALIGNED-NEXT:    buffer_load_ubyte v74, v1, s[0:3], 0 offen offset:222
+; ALIGNED-NEXT:    buffer_load_ubyte v73, v1, s[0:3], 0 offen offset:221
+; ALIGNED-NEXT:    buffer_load_ubyte v72, v1, s[0:3], 0 offen offset:220
+; ALIGNED-NEXT:    buffer_load_ubyte v63, v1, s[0:3], 0 offen offset:219
+; ALIGNED-NEXT:    buffer_load_ubyte v62, v1, s[0:3], 0 offen offset:218
+; ALIGNED-NEXT:    buffer_load_ubyte v61, v1, s[0:3], 0 offen offset:217
+; ALIGNED-NEXT:    buffer_load_ubyte v60, v1, s[0:3], 0 offen offset:216
+; ALIGNED-NEXT:    buffer_load_ubyte v59, v1, s[0:3], 0 offen offset:215
+; ALIGNED-NEXT:    buffer_load_ubyte v58, v1, s[0:3], 0 offen offset:214
+; ALIGNED-NEXT:    buffer_load_ubyte v57, v1, s[0:3], 0 offen offset:213
+; ALIGNED-NEXT:    buffer_load_ubyte v56, v1, s[0:3], 0 offen offset:212
+; ALIGNED-NEXT:    buffer_load_ubyte v47, v1, s[0:3], 0 offen offset:211
+; ALIGNED-NEXT:    buffer_load_ubyte v46, v1, s[0:3], 0 offen offset:210
+; ALIGNED-NEXT:    buffer_load_ubyte v45, v1, s[0:3], 0 offen offset:209
+; ALIGNED-NEXT:    buffer_load_ubyte v44, v1, s[0:3], 0 offen offset:208
+; ALIGNED-NEXT:    buffer_load_ubyte v43, v1, s[0:3], 0 offen offset:207
+; ALIGNED-NEXT:    buffer_load_ubyte v42, v1, s[0:3], 0 offen offset:206
+; ALIGNED-NEXT:    buffer_load_ubyte v41, v1, s[0:3], 0 offen offset:205
+; ALIGNED-NEXT:    buffer_load_ubyte v40, v1, s[0:3], 0 offen offset:204
+; ALIGNED-NEXT:    buffer_load_ubyte v119, v1, s[0:3], 0 offen offset:203
+; ALIGNED-NEXT:    buffer_load_ubyte v118, v1, s[0:3], 0 offen offset:202
+; ALIGNED-NEXT:    buffer_load_ubyte v117, v1, s[0:3], 0 offen offset:201
+; ALIGNED-NEXT:    buffer_load_ubyte v116, v1, s[0:3], 0 offen offset:200
+; ALIGNED-NEXT:    buffer_load_ubyte v115, v1, s[0:3], 0 offen offset:199
+; ALIGNED-NEXT:    buffer_load_ubyte v114, v1, s[0:3], 0 offen offset:198
+; ALIGNED-NEXT:    buffer_load_ubyte v113, v1, s[0:3], 0 offen offset:197
+; ALIGNED-NEXT:    buffer_load_ubyte v112, v1, s[0:3], 0 offen offset:196
+; ALIGNED-NEXT:    buffer_load_ubyte v103, v1, s[0:3], 0 offen offset:195
+; ALIGNED-NEXT:    buffer_load_ubyte v102, v1, s[0:3], 0 offen offset:194
+; ALIGNED-NEXT:    buffer_load_ubyte v101, v1, s[0:3], 0 offen offset:193
+; ALIGNED-NEXT:    buffer_load_ubyte v100, v1, s[0:3], 0 offen offset:192
+; ALIGNED-NEXT:    buffer_load_ubyte v99, v1, s[0:3], 0 offen offset:191
+; ALIGNED-NEXT:    buffer_load_ubyte v98, v1, s[0:3], 0 offen offset:190
+; ALIGNED-NEXT:    buffer_load_ubyte v97, v1, s[0:3], 0 offen offset:189
+; ALIGNED-NEXT:    buffer_load_ubyte v96, v1, s[0:3], 0 offen offset:188
+; ALIGNED-NEXT:    buffer_load_ubyte v87, v1, s[0:3], 0 offen offset:187
+; ALIGNED-NEXT:    buffer_load_ubyte v86, v1, s[0:3], 0 offen offset:186
+; ALIGNED-NEXT:    buffer_load_ubyte v85, v1, s[0:3], 0 offen offset:185
+; ALIGNED-NEXT:    s_clause 0x3a
+; ALIGNED-NEXT:    buffer_load_ubyte v84, v1, s[0:3], 0 offen offset:184
+; ALIGNED-NEXT:    buffer_load_ubyte v83, v1, s[0:3], 0 offen offset:183
+; ALIGNED-NEXT:    buffer_load_ubyte v82, v1, s[0:3], 0 offen offset:182
+; ALIGNED-NEXT:    buffer_load_ubyte v81, v1, s[0:3], 0 offen offset:181
+; ALIGNED-NEXT:    buffer_load_ubyte v80, v1, s[0:3], 0 offen offset:180
+; ALIGNED-NEXT:    buffer_load_ubyte v71, v1, s[0:3], 0 offen offset:179
+; ALIGNED-NEXT:    buffer_load_ubyte v70, v1, s[0:3], 0 offen offset:178
+; ALIGNED-NEXT:    buffer_load_ubyte v69, v1, s[0:3], 0 offen offset:177
+; ALIGNED-NEXT:    buffer_load_ubyte v68, v1, s[0:3], 0 offen offset:176
+; ALIGNED-NEXT:    buffer_load_ubyte v67, v1, s[0:3], 0 offen offset:175
+; ALIGNED-NEXT:    buffer_load_ubyte v66, v1, s[0:3], 0 offen offset:174
+; ALIGNED-NEXT:    buffer_load_ubyte v65, v1, s[0:3], 0 offen offset:173
+; ALIGNED-NEXT:    buffer_load_ubyte v64, v1, s[0:3], 0 offen offset:172
+; ALIGNED-NEXT:    buffer_load_ubyte v55, v1, s[0:3], 0 offen offset:171
+; ALIGNED-NEXT:    buffer_load_ubyte v54, v1, s[0:3], 0 offen offset:170
+; ALIGNED-NEXT:    buffer_load_ubyte v53, v1, s[0:3], 0 offen offset:169
+; ALIGNED-NEXT:    buffer_load_ubyte v52, v1, s[0:3], 0 offen offset:168
+; ALIGNED-NEXT:    buffer_load_ubyte v51, v1, s[0:3], 0 offen offset:167
+; ALIGNED-NEXT:    buffer_load_ubyte v50, v1, s[0:3], 0 offen offset:166
+; ALIGNED-NEXT:    buffer_load_ubyte v49, v1, s[0:3], 0 offen offset:165
+; ALIGNED-NEXT:    buffer_load_ubyte v48, v1, s[0:3], 0 offen offset:164
+; ALIGNED-NEXT:    buffer_load_ubyte v39, v1, s[0:3], 0 offen offset:163
+; ALIGNED-NEXT:    buffer_load_ubyte v38, v1, s[0:3], 0 offen offset:162
+; ALIGNED-NEXT:    buffer_load_ubyte v37, v1, s[0:3], 0 offen offset:161
+; ALIGNED-NEXT:    buffer_load_ubyte v36, v1, s[0:3], 0 offen offset:160
+; ALIGNED-NEXT:    buffer_load_ubyte v35, v1, s[0:3], 0 offen offset:159
+; ALIGNED-NEXT:    buffer_load_ubyte v34, v1, s[0:3], 0 offen offset:158
+; ALIGNED-NEXT:    buffer_load_ubyte v33, v1, s[0:3], 0 offen offset:157
+; ALIGNED-NEXT:    buffer_load_ubyte v32, v1, s[0:3], 0 offen offset:156
+; ALIGNED-NEXT:    buffer_load_ubyte v31, v1, s[0:3], 0 offen offset:155
+; ALIGNED-NEXT:    buffer_load_ubyte v30, v1, s[0:3], 0 offen offset:154
+; ALIGNED-NEXT:    buffer_load_ubyte v29, v1, s[0:3], 0 offen offset:153
+; ALIGNED-NEXT:    buffer_load_ubyte v28, v1, s[0:3], 0 offen offset:152
+; ALIGNED-NEXT:    buffer_load_ubyte v27, v1, s[0:3], 0 offen offset:151
+; ALIGNED-NEXT:    buffer_load_ubyte v26, v1, s[0:3], 0 offen offset:150
+; ALIGNED-NEXT:    buffer_load_ubyte v25, v1, s[0:3], 0 offen offset:149
+; ALIGNED-NEXT:    buffer_load_ubyte v24, v1, s[0:3], 0 offen offset:148
+; ALIGNED-NEXT:    buffer_load_ubyte v23, v1, s[0:3], 0 offen offset:147
+; ALIGNED-NEXT:    buffer_load_ubyte v22, v1, s[0:3], 0 offen offset:146
+; ALIGNED-NEXT:    buffer_load_ubyte v21, v1, s[0:3], 0 offen offset:145
+; ALIGNED-NEXT:    buffer_load_ubyte v20, v1, s[0:3], 0 offen offset:144
+; ALIGNED-NEXT:    buffer_load_ubyte v19, v1, s[0:3], 0 offen offset:143
+; ALIGNED-NEXT:    buffer_load_ubyte v18, v1, s[0:3], 0 offen offset:142
+; ALIGNED-NEXT:    buffer_load_ubyte v17, v1, s[0:3], 0 offen offset:141
+; ALIGNED-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:140
+; ALIGNED-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:139
+; ALIGNED-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:138
+; ALIGNED-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:137
+; ALIGNED-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:136
+; ALIGNED-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:135
+; ALIGNED-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:134
+; ALIGNED-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:133
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:132
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:131
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:130
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:129
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:128
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:127
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:126
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v127, v1, s[0:3], 0 offen offset:125
+; ALIGNED-NEXT:    buffer_load_ubyte v126, v1, s[0:3], 0 offen offset:124
+; ALIGNED-NEXT:    buffer_load_ubyte v125, v1, s[0:3], 0 offen offset:123
+; ALIGNED-NEXT:    buffer_load_ubyte v124, v1, s[0:3], 0 offen offset:122
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:121
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:120
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:119
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:118
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:117
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:116
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:115
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:114
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:113
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:112
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:111
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:110
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:109
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:108
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:107
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:106
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:105
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:104
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:103
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:102
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:101
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:100
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:99
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:98
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:97
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:96
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:95
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:94
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:93
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:92
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:91
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:90
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:89
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:88
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:87
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:86
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:85
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:84
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:83
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:82
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:81
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:80
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:79
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:78
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:77
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:76
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:75
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:74
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:73
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:72
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:71
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:70
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:69
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:68
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:67
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:66
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:65
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:64
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:63
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:62
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:61
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:60
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:59
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:58
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:57
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:56
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:55
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:54
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:53
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:52
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:51
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:50
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:49
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:48
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:47
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:46
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:45
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:44
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:43
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:42
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:41
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:40
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:39
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:38
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:37
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:36
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:35
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:34
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:33
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:32
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:31
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:30
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:29
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:27
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:26
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:25
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:24
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:23
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:22
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:21
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:20
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:19
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:18
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:17
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:16
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:15
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:14
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:13
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:12
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:11
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:10
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:9
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:8
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:7
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:6
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:5
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:4
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:3
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:2
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:1
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen
+; ALIGNED-NEXT:    v_add_nc_u32_e32 v1, 0x100, v1
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:255
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:254
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:253
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:252
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:251
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:250
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:249
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:248
+; ALIGNED-NEXT:    buffer_store_byte v123, v0, s[0:3], 0 offen offset:247
+; ALIGNED-NEXT:    buffer_store_byte v122, v0, s[0:3], 0 offen offset:246
+; ALIGNED-NEXT:    buffer_store_byte v121, v0, s[0:3], 0 offen offset:245
+; ALIGNED-NEXT:    buffer_store_byte v120, v0, s[0:3], 0 offen offset:244
+; ALIGNED-NEXT:    buffer_store_byte v111, v0, s[0:3], 0 offen offset:243
+; ALIGNED-NEXT:    buffer_store_byte v110, v0, s[0:3], 0 offen offset:242
+; ALIGNED-NEXT:    buffer_store_byte v109, v0, s[0:3], 0 offen offset:241
+; ALIGNED-NEXT:    buffer_store_byte v108, v0, s[0:3], 0 offen offset:240
+; ALIGNED-NEXT:    buffer_store_byte v107, v0, s[0:3], 0 offen offset:239
+; ALIGNED-NEXT:    buffer_store_byte v106, v0, s[0:3], 0 offen offset:238
+; ALIGNED-NEXT:    buffer_store_byte v105, v0, s[0:3], 0 offen offset:237
+; ALIGNED-NEXT:    buffer_store_byte v104, v0, s[0:3], 0 offen offset:236
+; ALIGNED-NEXT:    buffer_store_byte v95, v0, s[0:3], 0 offen offset:235
+; ALIGNED-NEXT:    buffer_store_byte v94, v0, s[0:3], 0 offen offset:234
+; ALIGNED-NEXT:    buffer_store_byte v93, v0, s[0:3], 0 offen offset:233
+; ALIGNED-NEXT:    buffer_store_byte v92, v0, s[0:3], 0 offen offset:232
+; ALIGNED-NEXT:    buffer_store_byte v91, v0, s[0:3], 0 offen offset:231
+; ALIGNED-NEXT:    buffer_store_byte v90, v0, s[0:3], 0 offen offset:230
+; ALIGNED-NEXT:    buffer_store_byte v89, v0, s[0:3], 0 offen offset:229
+; ALIGNED-NEXT:    buffer_store_byte v88, v0, s[0:3], 0 offen offset:228
+; ALIGNED-NEXT:    buffer_store_byte v79, v0, s[0:3], 0 offen offset:227
+; ALIGNED-NEXT:    buffer_store_byte v78, v0, s[0:3], 0 offen offset:226
+; ALIGNED-NEXT:    buffer_store_byte v77, v0, s[0:3], 0 offen offset:225
+; ALIGNED-NEXT:    buffer_store_byte v76, v0, s[0:3], 0 offen offset:224
+; ALIGNED-NEXT:    buffer_store_byte v75, v0, s[0:3], 0 offen offset:223
+; ALIGNED-NEXT:    buffer_store_byte v74, v0, s[0:3], 0 offen offset:222
+; ALIGNED-NEXT:    buffer_store_byte v73, v0, s[0:3], 0 offen offset:221
+; ALIGNED-NEXT:    buffer_store_byte v72, v0, s[0:3], 0 offen offset:220
+; ALIGNED-NEXT:    buffer_store_byte v63, v0, s[0:3], 0 offen offset:219
+; ALIGNED-NEXT:    buffer_store_byte v62, v0, s[0:3], 0 offen offset:218
+; ALIGNED-NEXT:    buffer_store_byte v61, v0, s[0:3], 0 offen offset:217
+; ALIGNED-NEXT:    buffer_store_byte v60, v0, s[0:3], 0 offen offset:216
+; ALIGNED-NEXT:    buffer_store_byte v59, v0, s[0:3], 0 offen offset:215
+; ALIGNED-NEXT:    buffer_store_byte v58, v0, s[0:3], 0 offen offset:214
+; ALIGNED-NEXT:    buffer_store_byte v57, v0, s[0:3], 0 offen offset:213
+; ALIGNED-NEXT:    buffer_store_byte v56, v0, s[0:3], 0 offen offset:212
+; ALIGNED-NEXT:    buffer_store_byte v47, v0, s[0:3], 0 offen offset:211
+; ALIGNED-NEXT:    buffer_store_byte v46, v0, s[0:3], 0 offen offset:210
+; ALIGNED-NEXT:    buffer_store_byte v45, v0, s[0:3], 0 offen offset:209
+; ALIGNED-NEXT:    buffer_store_byte v44, v0, s[0:3], 0 offen offset:208
+; ALIGNED-NEXT:    buffer_store_byte v43, v0, s[0:3], 0 offen offset:207
+; ALIGNED-NEXT:    buffer_store_byte v42, v0, s[0:3], 0 offen offset:206
+; ALIGNED-NEXT:    buffer_store_byte v41, v0, s[0:3], 0 offen offset:205
+; ALIGNED-NEXT:    buffer_store_byte v40, v0, s[0:3], 0 offen offset:204
+; ALIGNED-NEXT:    buffer_store_byte v119, v0, s[0:3], 0 offen offset:203
+; ALIGNED-NEXT:    buffer_store_byte v118, v0, s[0:3], 0 offen offset:202
+; ALIGNED-NEXT:    buffer_store_byte v117, v0, s[0:3], 0 offen offset:201
+; ALIGNED-NEXT:    buffer_store_byte v116, v0, s[0:3], 0 offen offset:200
+; ALIGNED-NEXT:    buffer_store_byte v115, v0, s[0:3], 0 offen offset:199
+; ALIGNED-NEXT:    buffer_store_byte v114, v0, s[0:3], 0 offen offset:198
+; ALIGNED-NEXT:    buffer_store_byte v113, v0, s[0:3], 0 offen offset:197
+; ALIGNED-NEXT:    buffer_store_byte v112, v0, s[0:3], 0 offen offset:196
+; ALIGNED-NEXT:    buffer_store_byte v103, v0, s[0:3], 0 offen offset:195
+; ALIGNED-NEXT:    buffer_store_byte v102, v0, s[0:3], 0 offen offset:194
+; ALIGNED-NEXT:    buffer_store_byte v101, v0, s[0:3], 0 offen offset:193
+; ALIGNED-NEXT:    buffer_store_byte v100, v0, s[0:3], 0 offen offset:192
+; ALIGNED-NEXT:    buffer_store_byte v99, v0, s[0:3], 0 offen offset:191
+; ALIGNED-NEXT:    buffer_store_byte v98, v0, s[0:3], 0 offen offset:190
+; ALIGNED-NEXT:    buffer_store_byte v97, v0, s[0:3], 0 offen offset:189
+; ALIGNED-NEXT:    buffer_store_byte v96, v0, s[0:3], 0 offen offset:188
+; ALIGNED-NEXT:    buffer_store_byte v87, v0, s[0:3], 0 offen offset:187
+; ALIGNED-NEXT:    buffer_store_byte v86, v0, s[0:3], 0 offen offset:186
+; ALIGNED-NEXT:    buffer_store_byte v85, v0, s[0:3], 0 offen offset:185
+; ALIGNED-NEXT:    buffer_store_byte v84, v0, s[0:3], 0 offen offset:184
+; ALIGNED-NEXT:    buffer_store_byte v83, v0, s[0:3], 0 offen offset:183
+; ALIGNED-NEXT:    buffer_store_byte v82, v0, s[0:3], 0 offen offset:182
+; ALIGNED-NEXT:    buffer_store_byte v81, v0, s[0:3], 0 offen offset:181
+; ALIGNED-NEXT:    buffer_store_byte v80, v0, s[0:3], 0 offen offset:180
+; ALIGNED-NEXT:    buffer_store_byte v71, v0, s[0:3], 0 offen offset:179
+; ALIGNED-NEXT:    buffer_store_byte v70, v0, s[0:3], 0 offen offset:178
+; ALIGNED-NEXT:    buffer_store_byte v69, v0, s[0:3], 0 offen offset:177
+; ALIGNED-NEXT:    buffer_store_byte v68, v0, s[0:3], 0 offen offset:176
+; ALIGNED-NEXT:    buffer_store_byte v67, v0, s[0:3], 0 offen offset:175
+; ALIGNED-NEXT:    buffer_store_byte v66, v0, s[0:3], 0 offen offset:174
+; ALIGNED-NEXT:    buffer_store_byte v65, v0, s[0:3], 0 offen offset:173
+; ALIGNED-NEXT:    buffer_store_byte v64, v0, s[0:3], 0 offen offset:172
+; ALIGNED-NEXT:    buffer_store_byte v55, v0, s[0:3], 0 offen offset:171
+; ALIGNED-NEXT:    buffer_store_byte v54, v0, s[0:3], 0 offen offset:170
+; ALIGNED-NEXT:    buffer_store_byte v53, v0, s[0:3], 0 offen offset:169
+; ALIGNED-NEXT:    buffer_store_byte v52, v0, s[0:3], 0 offen offset:168
+; ALIGNED-NEXT:    buffer_store_byte v51, v0, s[0:3], 0 offen offset:167
+; ALIGNED-NEXT:    buffer_store_byte v50, v0, s[0:3], 0 offen offset:166
+; ALIGNED-NEXT:    buffer_store_byte v49, v0, s[0:3], 0 offen offset:165
+; ALIGNED-NEXT:    buffer_store_byte v48, v0, s[0:3], 0 offen offset:164
+; ALIGNED-NEXT:    buffer_store_byte v39, v0, s[0:3], 0 offen offset:163
+; ALIGNED-NEXT:    buffer_store_byte v38, v0, s[0:3], 0 offen offset:162
+; ALIGNED-NEXT:    buffer_store_byte v37, v0, s[0:3], 0 offen offset:161
+; ALIGNED-NEXT:    buffer_store_byte v36, v0, s[0:3], 0 offen offset:160
+; ALIGNED-NEXT:    buffer_store_byte v35, v0, s[0:3], 0 offen offset:159
+; ALIGNED-NEXT:    buffer_store_byte v34, v0, s[0:3], 0 offen offset:158
+; ALIGNED-NEXT:    buffer_store_byte v33, v0, s[0:3], 0 offen offset:157
+; ALIGNED-NEXT:    buffer_store_byte v32, v0, s[0:3], 0 offen offset:156
+; ALIGNED-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:155
+; ALIGNED-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:154
+; ALIGNED-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:153
+; ALIGNED-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:152
+; ALIGNED-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:151
+; ALIGNED-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:150
+; ALIGNED-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:149
+; ALIGNED-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:148
+; ALIGNED-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:147
+; ALIGNED-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:146
+; ALIGNED-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:145
+; ALIGNED-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:144
+; ALIGNED-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:143
+; ALIGNED-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:142
+; ALIGNED-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:141
+; ALIGNED-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:140
+; ALIGNED-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:139
+; ALIGNED-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:138
+; ALIGNED-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:137
+; ALIGNED-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:136
+; ALIGNED-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:135
+; ALIGNED-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:134
+; ALIGNED-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:133
+; ALIGNED-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:132
+; ALIGNED-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:131
+; ALIGNED-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:130
+; ALIGNED-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:129
+; ALIGNED-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:128
+; ALIGNED-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:127
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:126
+; ALIGNED-NEXT:    buffer_store_byte v127, v0, s[0:3], 0 offen offset:125
+; ALIGNED-NEXT:    buffer_store_byte v126, v0, s[0:3], 0 offen offset:124
+; ALIGNED-NEXT:    buffer_store_byte v125, v0, s[0:3], 0 offen offset:123
+; ALIGNED-NEXT:    buffer_store_byte v124, v0, s[0:3], 0 offen offset:122
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:121
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:120
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:119
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:118
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:117
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:116
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:115
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:114
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:113
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:112
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:111
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:110
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:109
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:108
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:107
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:106
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:105
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:104
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:103
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:102
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:101
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:100
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:99
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:98
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:97
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:96
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:95
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:94
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:93
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:92
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:91
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:90
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:89
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:88
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:87
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:86
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:85
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:84
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:83
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:82
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:81
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:80
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:79
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:78
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:77
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:76
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:75
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:74
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:73
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:72
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:71
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:70
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:69
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:68
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:67
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:66
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:65
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:64
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:63
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:62
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:61
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:60
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:59
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:58
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:57
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:56
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:55
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:54
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:53
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:52
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:51
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:50
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:49
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:48
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:47
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:46
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:45
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:44
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:43
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:42
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:41
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:40
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:39
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:38
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:37
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:36
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:35
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:34
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:33
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:32
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:31
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:30
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:29
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:28
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:27
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:26
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:25
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:24
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:23
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:22
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:21
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:20
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:19
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:18
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:17
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:16
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:15
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:14
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:13
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:12
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:11
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:10
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:9
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:8
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:7
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:6
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:5
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:4
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:3
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:2
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:1
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen
+; ALIGNED-NEXT:    v_add_nc_u32_e32 v0, 0x100, v0
+; ALIGNED-NEXT:    s_cbranch_scc1 .LBB8_2
+; ALIGNED-NEXT:  .LBB8_3: ; %Flow18
+; ALIGNED-NEXT:    s_andn2_saveexec_b32 s6, s6
+; ALIGNED-NEXT:    s_cbranch_execz .LBB8_6
+; ALIGNED-NEXT:  ; %bb.4: ; %memmove_bwd_loop.preheader
+; ALIGNED-NEXT:    v_add_nc_u32_e32 v0, 0x700, v0
+; ALIGNED-NEXT:    v_add_nc_u32_e32 v1, 0x700, v1
+; ALIGNED-NEXT:    s_movk_i32 s4, 0xf800
+; ALIGNED-NEXT:    s_mov_b32 s5, -1
+; ALIGNED-NEXT:  .LBB8_5: ; %memmove_bwd_loop
+; ALIGNED-NEXT:    ; =>This Inner Loop Header: Depth=1
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:255
+; ALIGNED-NEXT:    s_add_u32 s4, s4, 0x100
+; ALIGNED-NEXT:    s_addc_u32 s5, s5, 0
+; ALIGNED-NEXT:    s_cmp_eq_u64 s[4:5], 0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:254
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:253
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:252
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:251
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:250
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:249
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:248
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:247
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:246
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:245
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:244
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:243
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:242
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:241
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:240
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:239
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:238
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:237
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:236
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:235
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:234
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:233
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:232
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:231
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:230
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:229
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:228
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:227
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:226
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:225
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:224
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:223
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:222
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:221
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:220
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:219
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:218
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:217
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:216
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:215
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:214
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:213
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:212
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:211
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:210
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:209
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:208
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:207
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:206
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:205
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:204
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:203
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:202
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:201
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:200
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x3e
+; ALIGNED-NEXT:    buffer_load_ubyte v115, v1, s[0:3], 0 offen offset:199
+; ALIGNED-NEXT:    buffer_load_ubyte v114, v1, s[0:3], 0 offen offset:198
+; ALIGNED-NEXT:    buffer_load_ubyte v113, v1, s[0:3], 0 offen offset:197
+; ALIGNED-NEXT:    buffer_load_ubyte v112, v1, s[0:3], 0 offen offset:196
+; ALIGNED-NEXT:    buffer_load_ubyte v103, v1, s[0:3], 0 offen offset:195
+; ALIGNED-NEXT:    buffer_load_ubyte v102, v1, s[0:3], 0 offen offset:194
+; ALIGNED-NEXT:    buffer_load_ubyte v101, v1, s[0:3], 0 offen offset:193
+; ALIGNED-NEXT:    buffer_load_ubyte v100, v1, s[0:3], 0 offen offset:192
+; ALIGNED-NEXT:    buffer_load_ubyte v99, v1, s[0:3], 0 offen offset:191
+; ALIGNED-NEXT:    buffer_load_ubyte v98, v1, s[0:3], 0 offen offset:190
+; ALIGNED-NEXT:    buffer_load_ubyte v97, v1, s[0:3], 0 offen offset:189
+; ALIGNED-NEXT:    buffer_load_ubyte v96, v1, s[0:3], 0 offen offset:188
+; ALIGNED-NEXT:    buffer_load_ubyte v87, v1, s[0:3], 0 offen offset:187
+; ALIGNED-NEXT:    buffer_load_ubyte v86, v1, s[0:3], 0 offen offset:186
+; ALIGNED-NEXT:    buffer_load_ubyte v85, v1, s[0:3], 0 offen offset:185
+; ALIGNED-NEXT:    buffer_load_ubyte v84, v1, s[0:3], 0 offen offset:184
+; ALIGNED-NEXT:    buffer_load_ubyte v83, v1, s[0:3], 0 offen offset:183
+; ALIGNED-NEXT:    buffer_load_ubyte v82, v1, s[0:3], 0 offen offset:182
+; ALIGNED-NEXT:    buffer_load_ubyte v81, v1, s[0:3], 0 offen offset:181
+; ALIGNED-NEXT:    buffer_load_ubyte v80, v1, s[0:3], 0 offen offset:180
+; ALIGNED-NEXT:    buffer_load_ubyte v71, v1, s[0:3], 0 offen offset:179
+; ALIGNED-NEXT:    buffer_load_ubyte v70, v1, s[0:3], 0 offen offset:178
+; ALIGNED-NEXT:    buffer_load_ubyte v69, v1, s[0:3], 0 offen offset:177
+; ALIGNED-NEXT:    buffer_load_ubyte v68, v1, s[0:3], 0 offen offset:176
+; ALIGNED-NEXT:    buffer_load_ubyte v67, v1, s[0:3], 0 offen offset:175
+; ALIGNED-NEXT:    buffer_load_ubyte v66, v1, s[0:3], 0 offen offset:174
+; ALIGNED-NEXT:    buffer_load_ubyte v65, v1, s[0:3], 0 offen offset:173
+; ALIGNED-NEXT:    buffer_load_ubyte v64, v1, s[0:3], 0 offen offset:172
+; ALIGNED-NEXT:    buffer_load_ubyte v55, v1, s[0:3], 0 offen offset:171
+; ALIGNED-NEXT:    buffer_load_ubyte v54, v1, s[0:3], 0 offen offset:170
+; ALIGNED-NEXT:    buffer_load_ubyte v53, v1, s[0:3], 0 offen offset:169
+; ALIGNED-NEXT:    buffer_load_ubyte v52, v1, s[0:3], 0 offen offset:168
+; ALIGNED-NEXT:    buffer_load_ubyte v51, v1, s[0:3], 0 offen offset:167
+; ALIGNED-NEXT:    buffer_load_ubyte v50, v1, s[0:3], 0 offen offset:166
+; ALIGNED-NEXT:    buffer_load_ubyte v49, v1, s[0:3], 0 offen offset:165
+; ALIGNED-NEXT:    buffer_load_ubyte v48, v1, s[0:3], 0 offen offset:164
+; ALIGNED-NEXT:    buffer_load_ubyte v39, v1, s[0:3], 0 offen offset:163
+; ALIGNED-NEXT:    buffer_load_ubyte v38, v1, s[0:3], 0 offen offset:162
+; ALIGNED-NEXT:    buffer_load_ubyte v37, v1, s[0:3], 0 offen offset:161
+; ALIGNED-NEXT:    buffer_load_ubyte v36, v1, s[0:3], 0 offen offset:160
+; ALIGNED-NEXT:    buffer_load_ubyte v35, v1, s[0:3], 0 offen offset:159
+; ALIGNED-NEXT:    buffer_load_ubyte v34, v1, s[0:3], 0 offen offset:158
+; ALIGNED-NEXT:    buffer_load_ubyte v33, v1, s[0:3], 0 offen offset:157
+; ALIGNED-NEXT:    buffer_load_ubyte v32, v1, s[0:3], 0 offen offset:156
+; ALIGNED-NEXT:    buffer_load_ubyte v31, v1, s[0:3], 0 offen offset:155
+; ALIGNED-NEXT:    buffer_load_ubyte v30, v1, s[0:3], 0 offen offset:154
+; ALIGNED-NEXT:    buffer_load_ubyte v29, v1, s[0:3], 0 offen offset:153
+; ALIGNED-NEXT:    buffer_load_ubyte v28, v1, s[0:3], 0 offen offset:152
+; ALIGNED-NEXT:    buffer_load_ubyte v27, v1, s[0:3], 0 offen offset:151
+; ALIGNED-NEXT:    buffer_load_ubyte v26, v1, s[0:3], 0 offen offset:150
+; ALIGNED-NEXT:    buffer_load_ubyte v25, v1, s[0:3], 0 offen offset:149
+; ALIGNED-NEXT:    buffer_load_ubyte v24, v1, s[0:3], 0 offen offset:148
+; ALIGNED-NEXT:    buffer_load_ubyte v23, v1, s[0:3], 0 offen offset:147
+; ALIGNED-NEXT:    buffer_load_ubyte v22, v1, s[0:3], 0 offen offset:146
+; ALIGNED-NEXT:    buffer_load_ubyte v21, v1, s[0:3], 0 offen offset:145
+; ALIGNED-NEXT:    buffer_load_ubyte v20, v1, s[0:3], 0 offen offset:144
+; ALIGNED-NEXT:    buffer_load_ubyte v19, v1, s[0:3], 0 offen offset:143
+; ALIGNED-NEXT:    buffer_load_ubyte v18, v1, s[0:3], 0 offen offset:142
+; ALIGNED-NEXT:    buffer_load_ubyte v17, v1, s[0:3], 0 offen offset:141
+; ALIGNED-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:140
+; ALIGNED-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:139
+; ALIGNED-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:138
+; ALIGNED-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:137
+; ALIGNED-NEXT:    s_clause 0xa
+; ALIGNED-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:136
+; ALIGNED-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:135
+; ALIGNED-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:134
+; ALIGNED-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:133
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:132
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:131
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:130
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:129
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:128
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:127
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:126
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x34
+; ALIGNED-NEXT:    buffer_load_ubyte v127, v1, s[0:3], 0 offen offset:125
+; ALIGNED-NEXT:    buffer_load_ubyte v126, v1, s[0:3], 0 offen offset:124
+; ALIGNED-NEXT:    buffer_load_ubyte v125, v1, s[0:3], 0 offen offset:123
+; ALIGNED-NEXT:    buffer_load_ubyte v124, v1, s[0:3], 0 offen offset:122
+; ALIGNED-NEXT:    buffer_load_ubyte v123, v1, s[0:3], 0 offen offset:121
+; ALIGNED-NEXT:    buffer_load_ubyte v122, v1, s[0:3], 0 offen offset:120
+; ALIGNED-NEXT:    buffer_load_ubyte v121, v1, s[0:3], 0 offen offset:119
+; ALIGNED-NEXT:    buffer_load_ubyte v120, v1, s[0:3], 0 offen offset:118
+; ALIGNED-NEXT:    buffer_load_ubyte v111, v1, s[0:3], 0 offen offset:117
+; ALIGNED-NEXT:    buffer_load_ubyte v110, v1, s[0:3], 0 offen offset:116
+; ALIGNED-NEXT:    buffer_load_ubyte v109, v1, s[0:3], 0 offen offset:115
+; ALIGNED-NEXT:    buffer_load_ubyte v108, v1, s[0:3], 0 offen offset:114
+; ALIGNED-NEXT:    buffer_load_ubyte v107, v1, s[0:3], 0 offen offset:113
+; ALIGNED-NEXT:    buffer_load_ubyte v106, v1, s[0:3], 0 offen offset:112
+; ALIGNED-NEXT:    buffer_load_ubyte v105, v1, s[0:3], 0 offen offset:111
+; ALIGNED-NEXT:    buffer_load_ubyte v104, v1, s[0:3], 0 offen offset:110
+; ALIGNED-NEXT:    buffer_load_ubyte v95, v1, s[0:3], 0 offen offset:109
+; ALIGNED-NEXT:    buffer_load_ubyte v94, v1, s[0:3], 0 offen offset:108
+; ALIGNED-NEXT:    buffer_load_ubyte v93, v1, s[0:3], 0 offen offset:107
+; ALIGNED-NEXT:    buffer_load_ubyte v92, v1, s[0:3], 0 offen offset:106
+; ALIGNED-NEXT:    buffer_load_ubyte v91, v1, s[0:3], 0 offen offset:105
+; ALIGNED-NEXT:    buffer_load_ubyte v90, v1, s[0:3], 0 offen offset:104
+; ALIGNED-NEXT:    buffer_load_ubyte v89, v1, s[0:3], 0 offen offset:103
+; ALIGNED-NEXT:    buffer_load_ubyte v88, v1, s[0:3], 0 offen offset:102
+; ALIGNED-NEXT:    buffer_load_ubyte v79, v1, s[0:3], 0 offen offset:101
+; ALIGNED-NEXT:    buffer_load_ubyte v78, v1, s[0:3], 0 offen offset:100
+; ALIGNED-NEXT:    buffer_load_ubyte v77, v1, s[0:3], 0 offen offset:99
+; ALIGNED-NEXT:    buffer_load_ubyte v76, v1, s[0:3], 0 offen offset:98
+; ALIGNED-NEXT:    buffer_load_ubyte v75, v1, s[0:3], 0 offen offset:97
+; ALIGNED-NEXT:    buffer_load_ubyte v74, v1, s[0:3], 0 offen offset:96
+; ALIGNED-NEXT:    buffer_load_ubyte v73, v1, s[0:3], 0 offen offset:95
+; ALIGNED-NEXT:    buffer_load_ubyte v72, v1, s[0:3], 0 offen offset:94
+; ALIGNED-NEXT:    buffer_load_ubyte v63, v1, s[0:3], 0 offen offset:93
+; ALIGNED-NEXT:    buffer_load_ubyte v62, v1, s[0:3], 0 offen offset:92
+; ALIGNED-NEXT:    buffer_load_ubyte v61, v1, s[0:3], 0 offen offset:91
+; ALIGNED-NEXT:    buffer_load_ubyte v60, v1, s[0:3], 0 offen offset:90
+; ALIGNED-NEXT:    buffer_load_ubyte v59, v1, s[0:3], 0 offen offset:89
+; ALIGNED-NEXT:    buffer_load_ubyte v58, v1, s[0:3], 0 offen offset:88
+; ALIGNED-NEXT:    buffer_load_ubyte v57, v1, s[0:3], 0 offen offset:87
+; ALIGNED-NEXT:    buffer_load_ubyte v56, v1, s[0:3], 0 offen offset:86
+; ALIGNED-NEXT:    buffer_load_ubyte v47, v1, s[0:3], 0 offen offset:85
+; ALIGNED-NEXT:    buffer_load_ubyte v46, v1, s[0:3], 0 offen offset:84
+; ALIGNED-NEXT:    buffer_load_ubyte v45, v1, s[0:3], 0 offen offset:83
+; ALIGNED-NEXT:    buffer_load_ubyte v44, v1, s[0:3], 0 offen offset:82
+; ALIGNED-NEXT:    buffer_load_ubyte v43, v1, s[0:3], 0 offen offset:81
+; ALIGNED-NEXT:    buffer_load_ubyte v42, v1, s[0:3], 0 offen offset:80
+; ALIGNED-NEXT:    buffer_load_ubyte v41, v1, s[0:3], 0 offen offset:79
+; ALIGNED-NEXT:    buffer_load_ubyte v40, v1, s[0:3], 0 offen offset:78
+; ALIGNED-NEXT:    buffer_load_ubyte v119, v1, s[0:3], 0 offen offset:77
+; ALIGNED-NEXT:    buffer_load_ubyte v118, v1, s[0:3], 0 offen offset:76
+; ALIGNED-NEXT:    buffer_load_ubyte v117, v1, s[0:3], 0 offen offset:75
+; ALIGNED-NEXT:    buffer_load_ubyte v116, v1, s[0:3], 0 offen offset:74
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:73
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:72
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:71
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:70
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:69
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:68
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:67
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:66
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:65
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:64
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:63
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:62
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:61
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:60
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:59
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:58
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:57
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:56
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:55
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:54
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:53
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:52
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:51
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:50
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:49
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:48
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:47
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:46
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:45
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:44
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:43
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:42
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:41
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:40
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:39
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:38
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:37
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:36
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:35
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:34
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:33
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:32
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:31
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:30
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:29
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:27
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:26
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:25
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:24
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:23
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:22
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:21
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:20
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:19
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:18
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:17
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:16
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:15
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:14
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:13
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:12
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:11
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:10
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:9
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:8
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:7
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:6
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:5
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:4
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:3
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:2
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:1
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen
+; ALIGNED-NEXT:    v_add_nc_u32_e32 v1, 0xffffff00, v1
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:255
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:254
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:253
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:252
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:251
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:250
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:249
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:248
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:247
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:246
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:245
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:244
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:243
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:242
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:241
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:240
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:239
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:238
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:237
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:236
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:235
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:234
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:233
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:232
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:231
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:230
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:229
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:228
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:227
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:226
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:225
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:224
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:223
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:222
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:221
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:220
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:219
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:218
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:217
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:216
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:215
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:214
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:213
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:212
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:211
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:210
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:209
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:208
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:207
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:206
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:205
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:204
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:203
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:202
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:201
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:200
+; ALIGNED-NEXT:    buffer_store_byte v115, v0, s[0:3], 0 offen offset:199
+; ALIGNED-NEXT:    buffer_store_byte v114, v0, s[0:3], 0 offen offset:198
+; ALIGNED-NEXT:    buffer_store_byte v113, v0, s[0:3], 0 offen offset:197
+; ALIGNED-NEXT:    buffer_store_byte v112, v0, s[0:3], 0 offen offset:196
+; ALIGNED-NEXT:    buffer_store_byte v103, v0, s[0:3], 0 offen offset:195
+; ALIGNED-NEXT:    buffer_store_byte v102, v0, s[0:3], 0 offen offset:194
+; ALIGNED-NEXT:    buffer_store_byte v101, v0, s[0:3], 0 offen offset:193
+; ALIGNED-NEXT:    buffer_store_byte v100, v0, s[0:3], 0 offen offset:192
+; ALIGNED-NEXT:    buffer_store_byte v99, v0, s[0:3], 0 offen offset:191
+; ALIGNED-NEXT:    buffer_store_byte v98, v0, s[0:3], 0 offen offset:190
+; ALIGNED-NEXT:    buffer_store_byte v97, v0, s[0:3], 0 offen offset:189
+; ALIGNED-NEXT:    buffer_store_byte v96, v0, s[0:3], 0 offen offset:188
+; ALIGNED-NEXT:    buffer_store_byte v87, v0, s[0:3], 0 offen offset:187
+; ALIGNED-NEXT:    buffer_store_byte v86, v0, s[0:3], 0 offen offset:186
+; ALIGNED-NEXT:    buffer_store_byte v85, v0, s[0:3], 0 offen offset:185
+; ALIGNED-NEXT:    buffer_store_byte v84, v0, s[0:3], 0 offen offset:184
+; ALIGNED-NEXT:    buffer_store_byte v83, v0, s[0:3], 0 offen offset:183
+; ALIGNED-NEXT:    buffer_store_byte v82, v0, s[0:3], 0 offen offset:182
+; ALIGNED-NEXT:    buffer_store_byte v81, v0, s[0:3], 0 offen offset:181
+; ALIGNED-NEXT:    buffer_store_byte v80, v0, s[0:3], 0 offen offset:180
+; ALIGNED-NEXT:    buffer_store_byte v71, v0, s[0:3], 0 offen offset:179
+; ALIGNED-NEXT:    buffer_store_byte v70, v0, s[0:3], 0 offen offset:178
+; ALIGNED-NEXT:    buffer_store_byte v69, v0, s[0:3], 0 offen offset:177
+; ALIGNED-NEXT:    buffer_store_byte v68, v0, s[0:3], 0 offen offset:176
+; ALIGNED-NEXT:    buffer_store_byte v67, v0, s[0:3], 0 offen offset:175
+; ALIGNED-NEXT:    buffer_store_byte v66, v0, s[0:3], 0 offen offset:174
+; ALIGNED-NEXT:    buffer_store_byte v65, v0, s[0:3], 0 offen offset:173
+; ALIGNED-NEXT:    buffer_store_byte v64, v0, s[0:3], 0 offen offset:172
+; ALIGNED-NEXT:    buffer_store_byte v55, v0, s[0:3], 0 offen offset:171
+; ALIGNED-NEXT:    buffer_store_byte v54, v0, s[0:3], 0 offen offset:170
+; ALIGNED-NEXT:    buffer_store_byte v53, v0, s[0:3], 0 offen offset:169
+; ALIGNED-NEXT:    buffer_store_byte v52, v0, s[0:3], 0 offen offset:168
+; ALIGNED-NEXT:    buffer_store_byte v51, v0, s[0:3], 0 offen offset:167
+; ALIGNED-NEXT:    buffer_store_byte v50, v0, s[0:3], 0 offen offset:166
+; ALIGNED-NEXT:    buffer_store_byte v49, v0, s[0:3], 0 offen offset:165
+; ALIGNED-NEXT:    buffer_store_byte v48, v0, s[0:3], 0 offen offset:164
+; ALIGNED-NEXT:    buffer_store_byte v39, v0, s[0:3], 0 offen offset:163
+; ALIGNED-NEXT:    buffer_store_byte v38, v0, s[0:3], 0 offen offset:162
+; ALIGNED-NEXT:    buffer_store_byte v37, v0, s[0:3], 0 offen offset:161
+; ALIGNED-NEXT:    buffer_store_byte v36, v0, s[0:3], 0 offen offset:160
+; ALIGNED-NEXT:    buffer_store_byte v35, v0, s[0:3], 0 offen offset:159
+; ALIGNED-NEXT:    buffer_store_byte v34, v0, s[0:3], 0 offen offset:158
+; ALIGNED-NEXT:    buffer_store_byte v33, v0, s[0:3], 0 offen offset:157
+; ALIGNED-NEXT:    buffer_store_byte v32, v0, s[0:3], 0 offen offset:156
+; ALIGNED-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:155
+; ALIGNED-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:154
+; ALIGNED-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:153
+; ALIGNED-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:152
+; ALIGNED-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:151
+; ALIGNED-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:150
+; ALIGNED-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:149
+; ALIGNED-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:148
+; ALIGNED-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:147
+; ALIGNED-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:146
+; ALIGNED-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:145
+; ALIGNED-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:144
+; ALIGNED-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:143
+; ALIGNED-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:142
+; ALIGNED-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:141
+; ALIGNED-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:140
+; ALIGNED-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:139
+; ALIGNED-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:138
+; ALIGNED-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:137
+; ALIGNED-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:136
+; ALIGNED-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:135
+; ALIGNED-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:134
+; ALIGNED-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:133
+; ALIGNED-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:132
+; ALIGNED-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:131
+; ALIGNED-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:130
+; ALIGNED-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:129
+; ALIGNED-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:128
+; ALIGNED-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:127
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:126
+; ALIGNED-NEXT:    buffer_store_byte v127, v0, s[0:3], 0 offen offset:125
+; ALIGNED-NEXT:    buffer_store_byte v126, v0, s[0:3], 0 offen offset:124
+; ALIGNED-NEXT:    buffer_store_byte v125, v0, s[0:3], 0 offen offset:123
+; ALIGNED-NEXT:    buffer_store_byte v124, v0, s[0:3], 0 offen offset:122
+; ALIGNED-NEXT:    buffer_store_byte v123, v0, s[0:3], 0 offen offset:121
+; ALIGNED-NEXT:    buffer_store_byte v122, v0, s[0:3], 0 offen offset:120
+; ALIGNED-NEXT:    buffer_store_byte v121, v0, s[0:3], 0 offen offset:119
+; ALIGNED-NEXT:    buffer_store_byte v120, v0, s[0:3], 0 offen offset:118
+; ALIGNED-NEXT:    buffer_store_byte v111, v0, s[0:3], 0 offen offset:117
+; ALIGNED-NEXT:    buffer_store_byte v110, v0, s[0:3], 0 offen offset:116
+; ALIGNED-NEXT:    buffer_store_byte v109, v0, s[0:3], 0 offen offset:115
+; ALIGNED-NEXT:    buffer_store_byte v108, v0, s[0:3], 0 offen offset:114
+; ALIGNED-NEXT:    buffer_store_byte v107, v0, s[0:3], 0 offen offset:113
+; ALIGNED-NEXT:    buffer_store_byte v106, v0, s[0:3], 0 offen offset:112
+; ALIGNED-NEXT:    buffer_store_byte v105, v0, s[0:3], 0 offen offset:111
+; ALIGNED-NEXT:    buffer_store_byte v104, v0, s[0:3], 0 offen offset:110
+; ALIGNED-NEXT:    buffer_store_byte v95, v0, s[0:3], 0 offen offset:109
+; ALIGNED-NEXT:    buffer_store_byte v94, v0, s[0:3], 0 offen offset:108
+; ALIGNED-NEXT:    buffer_store_byte v93, v0, s[0:3], 0 offen offset:107
+; ALIGNED-NEXT:    buffer_store_byte v92, v0, s[0:3], 0 offen offset:106
+; ALIGNED-NEXT:    buffer_store_byte v91, v0, s[0:3], 0 offen offset:105
+; ALIGNED-NEXT:    buffer_store_byte v90, v0, s[0:3], 0 offen offset:104
+; ALIGNED-NEXT:    buffer_store_byte v89, v0, s[0:3], 0 offen offset:103
+; ALIGNED-NEXT:    buffer_store_byte v88, v0, s[0:3], 0 offen offset:102
+; ALIGNED-NEXT:    buffer_store_byte v79, v0, s[0:3], 0 offen offset:101
+; ALIGNED-NEXT:    buffer_store_byte v78, v0, s[0:3], 0 offen offset:100
+; ALIGNED-NEXT:    buffer_store_byte v77, v0, s[0:3], 0 offen offset:99
+; ALIGNED-NEXT:    buffer_store_byte v76, v0, s[0:3], 0 offen offset:98
+; ALIGNED-NEXT:    buffer_store_byte v75, v0, s[0:3], 0 offen offset:97
+; ALIGNED-NEXT:    buffer_store_byte v74, v0, s[0:3], 0 offen offset:96
+; ALIGNED-NEXT:    buffer_store_byte v73, v0, s[0:3], 0 offen offset:95
+; ALIGNED-NEXT:    buffer_store_byte v72, v0, s[0:3], 0 offen offset:94
+; ALIGNED-NEXT:    buffer_store_byte v63, v0, s[0:3], 0 offen offset:93
+; ALIGNED-NEXT:    buffer_store_byte v62, v0, s[0:3], 0 offen offset:92
+; ALIGNED-NEXT:    buffer_store_byte v61, v0, s[0:3], 0 offen offset:91
+; ALIGNED-NEXT:    buffer_store_byte v60, v0, s[0:3], 0 offen offset:90
+; ALIGNED-NEXT:    buffer_store_byte v59, v0, s[0:3], 0 offen offset:89
+; ALIGNED-NEXT:    buffer_store_byte v58, v0, s[0:3], 0 offen offset:88
+; ALIGNED-NEXT:    buffer_store_byte v57, v0, s[0:3], 0 offen offset:87
+; ALIGNED-NEXT:    buffer_store_byte v56, v0, s[0:3], 0 offen offset:86
+; ALIGNED-NEXT:    buffer_store_byte v47, v0, s[0:3], 0 offen offset:85
+; ALIGNED-NEXT:    buffer_store_byte v46, v0, s[0:3], 0 offen offset:84
+; ALIGNED-NEXT:    buffer_store_byte v45, v0, s[0:3], 0 offen offset:83
+; ALIGNED-NEXT:    buffer_store_byte v44, v0, s[0:3], 0 offen offset:82
+; ALIGNED-NEXT:    buffer_store_byte v43, v0, s[0:3], 0 offen offset:81
+; ALIGNED-NEXT:    buffer_store_byte v42, v0, s[0:3], 0 offen offset:80
+; ALIGNED-NEXT:    buffer_store_byte v41, v0, s[0:3], 0 offen offset:79
+; ALIGNED-NEXT:    buffer_store_byte v40, v0, s[0:3], 0 offen offset:78
+; ALIGNED-NEXT:    buffer_store_byte v119, v0, s[0:3], 0 offen offset:77
+; ALIGNED-NEXT:    buffer_store_byte v118, v0, s[0:3], 0 offen offset:76
+; ALIGNED-NEXT:    buffer_store_byte v117, v0, s[0:3], 0 offen offset:75
+; ALIGNED-NEXT:    buffer_store_byte v116, v0, s[0:3], 0 offen offset:74
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:73
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:72
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:71
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:70
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:69
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:68
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:67
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:66
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:65
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:64
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:63
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:62
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:61
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:60
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:59
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:58
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:57
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:56
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:55
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:54
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:53
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:52
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:51
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:50
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:49
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:48
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:47
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:46
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:45
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:44
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:43
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:42
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:41
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:40
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:39
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:38
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:37
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:36
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:35
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:34
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:33
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:32
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:31
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:30
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:29
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:28
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:27
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:26
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:25
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:24
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:23
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:22
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:21
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:20
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:19
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:18
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:17
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:16
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:15
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:14
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:13
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:12
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:11
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:10
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:9
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:8
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:7
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:6
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:5
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:4
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:3
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:2
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:1
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen
+; ALIGNED-NEXT:    v_add_nc_u32_e32 v0, 0xffffff00, v0
+; ALIGNED-NEXT:    s_cbranch_scc0 .LBB8_5
+; ALIGNED-NEXT:  .LBB8_6: ; %Flow19
+; ALIGNED-NEXT:    s_or_b32 exec_lo, exec_lo, s6
+; ALIGNED-NEXT:    s_clause 0x2f
+; ALIGNED-NEXT:    buffer_load_dword v127, off, s[0:3], s32
+; ALIGNED-NEXT:    buffer_load_dword v126, off, s[0:3], s32 offset:4
+; ALIGNED-NEXT:    buffer_load_dword v125, off, s[0:3], s32 offset:8
+; ALIGNED-NEXT:    buffer_load_dword v124, off, s[0:3], s32 offset:12
+; ALIGNED-NEXT:    buffer_load_dword v123, off, s[0:3], s32 offset:16
+; ALIGNED-NEXT:    buffer_load_dword v122, off, s[0:3], s32 offset:20
+; ALIGNED-NEXT:    buffer_load_dword v121, off, s[0:3], s32 offset:24
+; ALIGNED-NEXT:    buffer_load_dword v120, off, s[0:3], s32 offset:28
+; ALIGNED-NEXT:    buffer_load_dword v111, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT:    buffer_load_dword v110, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT:    buffer_load_dword v109, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT:    buffer_load_dword v108, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT:    buffer_load_dword v107, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT:    buffer_load_dword v106, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT:    buffer_load_dword v105, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT:    buffer_load_dword v104, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT:    buffer_load_dword v95, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT:    buffer_load_dword v94, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT:    buffer_load_dword v93, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT:    buffer_load_dword v92, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT:    buffer_load_dword v91, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT:    buffer_load_dword v90, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT:    buffer_load_dword v89, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT:    buffer_load_dword v88, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT:    buffer_load_dword v79, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT:    buffer_load_dword v78, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT:    buffer_load_dword v77, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT:    buffer_load_dword v76, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT:    buffer_load_dword v75, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT:    buffer_load_dword v74, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT:    buffer_load_dword v73, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT:    buffer_load_dword v72, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; UNROLL3-LABEL: memmove_p5_p5_sz2048:
+; UNROLL3:       ; %bb.0: ; %entry
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    s_mov_b32 s4, exec_lo
+; UNROLL3-NEXT:    v_cmpx_ge_u32_e64 v1, v0
+; UNROLL3-NEXT:    s_xor_b32 s6, exec_lo, s4
+; UNROLL3-NEXT:    s_cbranch_execz .LBB8_4
+; UNROLL3-NEXT:  ; %bb.1: ; %memmove_fwd_loop.preheader
+; UNROLL3-NEXT:    v_mov_b32_e32 v2, v1
+; UNROLL3-NEXT:    v_mov_b32_e32 v3, v0
+; UNROLL3-NEXT:    s_mov_b64 s[4:5], 0x7e0
+; UNROLL3-NEXT:  .LBB8_2: ; %memmove_fwd_loop
+; UNROLL3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT:    s_clause 0xb
+; UNROLL3-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:44
+; UNROLL3-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:40
+; UNROLL3-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:36
+; UNROLL3-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:32
+; UNROLL3-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:28
+; UNROLL3-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; UNROLL3-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:20
+; UNROLL3-NEXT:    buffer_load_dword v11, v2, s[0:3], 0 offen offset:16
+; UNROLL3-NEXT:    buffer_load_dword v12, v2, s[0:3], 0 offen offset:12
+; UNROLL3-NEXT:    buffer_load_dword v13, v2, s[0:3], 0 offen offset:8
+; UNROLL3-NEXT:    buffer_load_dword v14, v2, s[0:3], 0 offen offset:4
+; UNROLL3-NEXT:    buffer_load_dword v15, v2, s[0:3], 0 offen
+; UNROLL3-NEXT:    v_add_nc_u32_e32 v2, 48, v2
+; UNROLL3-NEXT:    s_add_u32 s4, s4, 0xffffffd0
+; UNROLL3-NEXT:    s_addc_u32 s5, s5, -1
+; UNROLL3-NEXT:    s_waitcnt vmcnt(11)
+; UNROLL3-NEXT:    buffer_store_dword v4, v3, s[0:3], 0 offen offset:44
+; UNROLL3-NEXT:    s_waitcnt vmcnt(10)
+; UNROLL3-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen offset:40
+; UNROLL3-NEXT:    s_waitcnt vmcnt(9)
+; UNROLL3-NEXT:    buffer_store_dword v6, v3, s[0:3], 0 offen offset:36
+; UNROLL3-NEXT:    s_waitcnt vmcnt(8)
+; UNROLL3-NEXT:    buffer_store_dword v7, v3, s[0:3], 0 offen offset:32
+; UNROLL3-NEXT:    s_waitcnt vmcnt(7)
+; UNROLL3-NEXT:    buffer_store_dword v8, v3, s[0:3], 0 offen offset:28
+; UNROLL3-NEXT:    s_waitcnt vmcnt(6)
+; UNROLL3-NEXT:    buffer_store_dword v9, v3, s[0:3], 0 offen offset:24
+; UNROLL3-NEXT:    s_waitcnt vmcnt(5)
+; UNROLL3-NEXT:    buffer_store_dword v10, v3, s[0:3], 0 offen offset:20
+; UNROLL3-NEXT:    s_waitcnt vmcnt(4)
+; UNROLL3-NEXT:    buffer_store_dword v11, v3, s[0:3], 0 offen offset:16
+; UNROLL3-NEXT:    s_waitcnt vmcnt(3)
+; UNROLL3-NEXT:    buffer_store_dword v12, v3, s[0:3], 0 offen offset:12
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2)
+; UNROLL3-NEXT:    buffer_store_dword v13, v3, s[0:3], 0 offen offset:8
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    buffer_store_dword v14, v3, s[0:3], 0 offen offset:4
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    buffer_store_dword v15, v3, s[0:3], 0 offen
+; UNROLL3-NEXT:    v_add_nc_u32_e32 v3, 48, v3
+; UNROLL3-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; UNROLL3-NEXT:    s_cbranch_scc1 .LBB8_2
+; UNROLL3-NEXT:  ; %bb.3: ; %memmove_fwd_residual
+; UNROLL3-NEXT:    s_clause 0x3
+; UNROLL3-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:2028
+; UNROLL3-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:2024
+; UNROLL3-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:2020
+; UNROLL3-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:2016
+; UNROLL3-NEXT:    s_waitcnt vmcnt(3)
+; UNROLL3-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:2028
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2)
+; UNROLL3-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:2024
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:2020
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:2016
+; UNROLL3-NEXT:    s_clause 0x3
+; UNROLL3-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:2044
+; UNROLL3-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:2040
+; UNROLL3-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:2036
+; UNROLL3-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:2032
+; UNROLL3-NEXT:    s_waitcnt vmcnt(3)
+; UNROLL3-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:2044
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2)
+; UNROLL3-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:2040
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:2036
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:2032
+; UNROLL3-NEXT:    ; implicit-def: $vgpr1
+; UNROLL3-NEXT:    ; implicit-def: $vgpr0
+; UNROLL3-NEXT:  .LBB8_4: ; %Flow16
+; UNROLL3-NEXT:    s_andn2_saveexec_b32 s6, s6
+; UNROLL3-NEXT:    s_cbranch_execz .LBB8_7
+; UNROLL3-NEXT:  ; %bb.5: ; %memmove_bwd_residual
+; UNROLL3-NEXT:    s_clause 0x3
+; UNROLL3-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:2044
+; UNROLL3-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:2040
+; UNROLL3-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:2036
+; UNROLL3-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:2032
+; UNROLL3-NEXT:    s_movk_i32 s4, 0xf820
+; UNROLL3-NEXT:    s_mov_b32 s5, -1
+; UNROLL3-NEXT:    s_waitcnt vmcnt(3)
+; UNROLL3-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:2044
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2)
+; UNROLL3-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:2040
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:2036
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:2032
+; UNROLL3-NEXT:    s_clause 0x3
+; UNROLL3-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:2028
+; UNROLL3-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:2024
+; UNROLL3-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:2020
+; UNROLL3-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:2016
+; UNROLL3-NEXT:    v_add_nc_u32_e32 v2, 0x7b0, v0
+; UNROLL3-NEXT:    v_add_nc_u32_e32 v1, 0x7b0, v1
+; UNROLL3-NEXT:    s_waitcnt vmcnt(3)
+; UNROLL3-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:2028
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2)
+; UNROLL3-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:2024
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:2020
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:2016
+; UNROLL3-NEXT:  .LBB8_6: ; %memmove_bwd_loop
+; UNROLL3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT:    s_clause 0xb
+; UNROLL3-NEXT:    buffer_load_dword v0, v1, s[0:3], 0 offen offset:44
+; UNROLL3-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:40
+; UNROLL3-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:36
+; UNROLL3-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:32
+; UNROLL3-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:28
+; UNROLL3-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:24
+; UNROLL3-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:20
+; UNROLL3-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:16
+; UNROLL3-NEXT:    buffer_load_dword v10, v1, s[0:3], 0 offen offset:12
+; UNROLL3-NEXT:    buffer_load_dword v11, v1, s[0:3], 0 offen offset:8
+; UNROLL3-NEXT:    buffer_load_dword v12, v1, s[0:3], 0 offen offset:4
+; UNROLL3-NEXT:    buffer_load_dword v13, v1, s[0:3], 0 offen
+; UNROLL3-NEXT:    v_subrev_nc_u32_e32 v1, 48, v1
+; UNROLL3-NEXT:    s_add_u32 s4, s4, 48
+; UNROLL3-NEXT:    s_addc_u32 s5, s5, 0
+; UNROLL3-NEXT:    s_waitcnt vmcnt(11)
+; UNROLL3-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen offset:44
+; UNROLL3-NEXT:    s_waitcnt vmcnt(10)
+; UNROLL3-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen offset:40
+; UNROLL3-NEXT:    s_waitcnt vmcnt(9)
+; UNROLL3-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen offset:36
+; UNROLL3-NEXT:    s_waitcnt vmcnt(8)
+; UNROLL3-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen offset:32
+; UNROLL3-NEXT:    s_waitcnt vmcnt(7)
+; UNROLL3-NEXT:    buffer_store_dword v6, v2, s[0:3], 0 offen offset:28
+; UNROLL3-NEXT:    s_waitcnt vmcnt(6)
+; UNROLL3-NEXT:    buffer_store_dword v7, v2, s[0:3], 0 offen offset:24
+; UNROLL3-NEXT:    s_waitcnt vmcnt(5)
+; UNROLL3-NEXT:    buffer_store_dword v8, v2, s[0:3], 0 offen offset:20
+; UNROLL3-NEXT:    s_waitcnt vmcnt(4)
+; UNROLL3-NEXT:    buffer_store_dword v9, v2, s[0:3], 0 offen offset:16
+; UNROLL3-NEXT:    s_waitcnt vmcnt(3)
+; UNROLL3-NEXT:    buffer_store_dword v10, v2, s[0:3], 0 offen offset:12
+; UNROLL3-NEXT:    s_waitcnt vmcnt(2)
+; UNROLL3-NEXT:    buffer_store_dword v11, v2, s[0:3], 0 offen offset:8
+; UNROLL3-NEXT:    s_waitcnt vmcnt(1)
+; UNROLL3-NEXT:    buffer_store_dword v12, v2, s[0:3], 0 offen offset:4
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    buffer_store_dword v13, v2, s[0:3], 0 offen
+; UNROLL3-NEXT:    v_subrev_nc_u32_e32 v2, 48, v2
+; UNROLL3-NEXT:    s_cmp_eq_u64 s[4:5], 0
+; UNROLL3-NEXT:    s_cbranch_scc0 .LBB8_6
+; UNROLL3-NEXT:  .LBB8_7: ; %Flow17
+; UNROLL3-NEXT:    s_or_b32 exec_lo, exec_lo, s6
+; UNROLL3-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  tail call void @llvm.memmove.p5.p5.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 2048, i1 false)
+  ret void
+}
+
+define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {
+; CHECK-LABEL: memmove_p0_p5_sz2048:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0
+; CHECK-NEXT:    s_mov_b32 s6, exec_lo
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, -1, v0, vcc_lo
+; CHECK-NEXT:    v_cmpx_ge_u32_e64 v2, v3
+; CHECK-NEXT:    s_xor_b32 s6, exec_lo, s6
+; CHECK-NEXT:    s_cbranch_execz .LBB9_2
+; CHECK-NEXT:  .LBB9_1: ; %memmove_fwd_loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_clause 0x3e
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:32
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:36
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:40
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:44
+; CHECK-NEXT:    buffer_load_dword v11, v2, s[0:3], 0 offen offset:48
+; CHECK-NEXT:    buffer_load_dword v12, v2, s[0:3], 0 offen offset:52
+; CHECK-NEXT:    buffer_load_dword v13, v2, s[0:3], 0 offen offset:56
+; CHECK-NEXT:    buffer_load_dword v14, v2, s[0:3], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v18, v2, s[0:3], 0 offen offset:92
+; CHECK-NEXT:    buffer_load_dword v17, v2, s[0:3], 0 offen offset:88
+; CHECK-NEXT:    buffer_load_dword v16, v2, s[0:3], 0 offen offset:84
+; CHECK-NEXT:    buffer_load_dword v15, v2, s[0:3], 0 offen offset:80
+; CHECK-NEXT:    buffer_load_dword v22, v2, s[0:3], 0 offen offset:124
+; CHECK-NEXT:    buffer_load_dword v21, v2, s[0:3], 0 offen offset:120
+; CHECK-NEXT:    buffer_load_dword v20, v2, s[0:3], 0 offen offset:116
+; CHECK-NEXT:    buffer_load_dword v19, v2, s[0:3], 0 offen offset:112
+; CHECK-NEXT:    buffer_load_dword v26, v2, s[0:3], 0 offen offset:108
+; CHECK-NEXT:    buffer_load_dword v25, v2, s[0:3], 0 offen offset:104
+; CHECK-NEXT:    buffer_load_dword v24, v2, s[0:3], 0 offen offset:100
+; CHECK-NEXT:    buffer_load_dword v23, v2, s[0:3], 0 offen offset:96
+; CHECK-NEXT:    buffer_load_dword v30, v2, s[0:3], 0 offen offset:156
+; CHECK-NEXT:    buffer_load_dword v29, v2, s[0:3], 0 offen offset:152
+; CHECK-NEXT:    buffer_load_dword v28, v2, s[0:3], 0 offen offset:148
+; CHECK-NEXT:    buffer_load_dword v27, v2, s[0:3], 0 offen offset:144
+; CHECK-NEXT:    buffer_load_dword v34, v2, s[0:3], 0 offen offset:188
+; CHECK-NEXT:    buffer_load_dword v33, v2, s[0:3], 0 offen offset:184
+; CHECK-NEXT:    buffer_load_dword v32, v2, s[0:3], 0 offen offset:180
+; CHECK-NEXT:    buffer_load_dword v31, v2, s[0:3], 0 offen offset:176
+; CHECK-NEXT:    buffer_load_dword v38, v2, s[0:3], 0 offen offset:172
+; CHECK-NEXT:    buffer_load_dword v37, v2, s[0:3], 0 offen offset:168
+; CHECK-NEXT:    buffer_load_dword v36, v2, s[0:3], 0 offen offset:164
+; CHECK-NEXT:    buffer_load_dword v35, v2, s[0:3], 0 offen offset:160
+; CHECK-NEXT:    buffer_load_dword v51, v2, s[0:3], 0 offen offset:220
+; CHECK-NEXT:    buffer_load_dword v50, v2, s[0:3], 0 offen offset:216
+; CHECK-NEXT:    buffer_load_dword v49, v2, s[0:3], 0 offen offset:212
+; CHECK-NEXT:    buffer_load_dword v48, v2, s[0:3], 0 offen offset:208
+; CHECK-NEXT:    buffer_load_dword v55, v2, s[0:3], 0 offen offset:252
+; CHECK-NEXT:    buffer_load_dword v54, v2, s[0:3], 0 offen offset:248
+; CHECK-NEXT:    buffer_load_dword v53, v2, s[0:3], 0 offen offset:244
+; CHECK-NEXT:    buffer_load_dword v52, v2, s[0:3], 0 offen offset:240
+; CHECK-NEXT:    buffer_load_dword v67, v2, s[0:3], 0 offen offset:236
+; CHECK-NEXT:    buffer_load_dword v66, v2, s[0:3], 0 offen offset:232
+; CHECK-NEXT:    buffer_load_dword v65, v2, s[0:3], 0 offen offset:228
+; CHECK-NEXT:    buffer_load_dword v64, v2, s[0:3], 0 offen offset:224
+; CHECK-NEXT:    buffer_load_dword v71, v2, s[0:3], 0 offen offset:204
+; CHECK-NEXT:    buffer_load_dword v70, v2, s[0:3], 0 offen offset:200
+; CHECK-NEXT:    buffer_load_dword v69, v2, s[0:3], 0 offen offset:196
+; CHECK-NEXT:    buffer_load_dword v68, v2, s[0:3], 0 offen offset:192
+; CHECK-NEXT:    buffer_load_dword v83, v2, s[0:3], 0 offen offset:140
+; CHECK-NEXT:    buffer_load_dword v82, v2, s[0:3], 0 offen offset:136
+; CHECK-NEXT:    buffer_load_dword v81, v2, s[0:3], 0 offen offset:132
+; CHECK-NEXT:    buffer_load_dword v80, v2, s[0:3], 0 offen offset:128
+; CHECK-NEXT:    buffer_load_dword v87, v2, s[0:3], 0 offen offset:76
+; CHECK-NEXT:    buffer_load_dword v86, v2, s[0:3], 0 offen offset:72
+; CHECK-NEXT:    buffer_load_dword v85, v2, s[0:3], 0 offen offset:68
+; CHECK-NEXT:    buffer_load_dword v84, v2, s[0:3], 0 offen offset:64
+; CHECK-NEXT:    buffer_load_dword v96, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v97, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v98, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v99, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    v_add_co_u32 v100, vcc_lo, v0, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v101, vcc_lo, s5, v1, vcc_lo
+; CHECK-NEXT:    s_add_u32 s4, s4, 0x100
+; CHECK-NEXT:    v_add_nc_u32_e32 v2, 0x100, v2
+; CHECK-NEXT:    s_addc_u32 s5, s5, 0
+; CHECK-NEXT:    s_waitcnt vmcnt(20)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[52:55] offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(16)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[64:67] offset:224
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[48:51] offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(12)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[68:71] offset:192
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[31:34] offset:176
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[35:38] offset:160
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[27:30] offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(8)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[80:83] offset:128
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[19:22] offset:112
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[23:26] offset:96
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[15:18] offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[84:87] offset:64
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[11:14] offset:48
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[7:10] offset:32
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[3:6] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[96:99]
+; CHECK-NEXT:    s_cmp_lg_u64 s[4:5], 0x800
+; CHECK-NEXT:    s_cbranch_scc1 .LBB9_1
+; CHECK-NEXT:  .LBB9_2: ; %Flow10
+; CHECK-NEXT:    s_andn2_saveexec_b32 s8, s6
+; CHECK-NEXT:    s_cbranch_execz .LBB9_5
+; CHECK-NEXT:  ; %bb.3: ; %memmove_bwd_loop.preheader
+; CHECK-NEXT:    v_add_nc_u32_e32 v2, 0x700, v2
+; CHECK-NEXT:    s_movk_i32 s6, 0xff00
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0x700
+; CHECK-NEXT:    s_mov_b32 s7, -1
+; CHECK-NEXT:  .LBB9_4: ; %memmove_bwd_loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_clause 0x3e
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:32
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:36
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:40
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:44
+; CHECK-NEXT:    buffer_load_dword v11, v2, s[0:3], 0 offen offset:48
+; CHECK-NEXT:    buffer_load_dword v12, v2, s[0:3], 0 offen offset:52
+; CHECK-NEXT:    buffer_load_dword v13, v2, s[0:3], 0 offen offset:56
+; CHECK-NEXT:    buffer_load_dword v14, v2, s[0:3], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v18, v2, s[0:3], 0 offen offset:124
+; CHECK-NEXT:    buffer_load_dword v17, v2, s[0:3], 0 offen offset:120
+; CHECK-NEXT:    buffer_load_dword v16, v2, s[0:3], 0 offen offset:116
+; CHECK-NEXT:    buffer_load_dword v15, v2, s[0:3], 0 offen offset:112
+; CHECK-NEXT:    buffer_load_dword v22, v2, s[0:3], 0 offen offset:108
+; CHECK-NEXT:    buffer_load_dword v21, v2, s[0:3], 0 offen offset:104
+; CHECK-NEXT:    buffer_load_dword v20, v2, s[0:3], 0 offen offset:100
+; CHECK-NEXT:    buffer_load_dword v19, v2, s[0:3], 0 offen offset:96
+; CHECK-NEXT:    buffer_load_dword v26, v2, s[0:3], 0 offen offset:252
+; CHECK-NEXT:    buffer_load_dword v25, v2, s[0:3], 0 offen offset:248
+; CHECK-NEXT:    buffer_load_dword v24, v2, s[0:3], 0 offen offset:244
+; CHECK-NEXT:    buffer_load_dword v23, v2, s[0:3], 0 offen offset:240
+; CHECK-NEXT:    buffer_load_dword v30, v2, s[0:3], 0 offen offset:236
+; CHECK-NEXT:    buffer_load_dword v29, v2, s[0:3], 0 offen offset:232
+; CHECK-NEXT:    buffer_load_dword v28, v2, s[0:3], 0 offen offset:228
+; CHECK-NEXT:    buffer_load_dword v27, v2, s[0:3], 0 offen offset:224
+; CHECK-NEXT:    buffer_load_dword v34, v2, s[0:3], 0 offen offset:220
+; CHECK-NEXT:    buffer_load_dword v33, v2, s[0:3], 0 offen offset:216
+; CHECK-NEXT:    buffer_load_dword v32, v2, s[0:3], 0 offen offset:212
+; CHECK-NEXT:    buffer_load_dword v31, v2, s[0:3], 0 offen offset:208
+; CHECK-NEXT:    buffer_load_dword v38, v2, s[0:3], 0 offen offset:204
+; CHECK-NEXT:    buffer_load_dword v37, v2, s[0:3], 0 offen offset:200
+; CHECK-NEXT:    buffer_load_dword v36, v2, s[0:3], 0 offen offset:196
+; CHECK-NEXT:    buffer_load_dword v35, v2, s[0:3], 0 offen offset:192
+; CHECK-NEXT:    buffer_load_dword v51, v2, s[0:3], 0 offen offset:188
+; CHECK-NEXT:    buffer_load_dword v50, v2, s[0:3], 0 offen offset:184
+; CHECK-NEXT:    buffer_load_dword v49, v2, s[0:3], 0 offen offset:180
+; CHECK-NEXT:    buffer_load_dword v48, v2, s[0:3], 0 offen offset:176
+; CHECK-NEXT:    buffer_load_dword v55, v2, s[0:3], 0 offen offset:172
+; CHECK-NEXT:    buffer_load_dword v54, v2, s[0:3], 0 offen offset:168
+; CHECK-NEXT:    buffer_load_dword v53, v2, s[0:3], 0 offen offset:164
+; CHECK-NEXT:    buffer_load_dword v52, v2, s[0:3], 0 offen offset:160
+; CHECK-NEXT:    buffer_load_dword v67, v2, s[0:3], 0 offen offset:156
+; CHECK-NEXT:    buffer_load_dword v66, v2, s[0:3], 0 offen offset:152
+; CHECK-NEXT:    buffer_load_dword v65, v2, s[0:3], 0 offen offset:148
+; CHECK-NEXT:    buffer_load_dword v64, v2, s[0:3], 0 offen offset:144
+; CHECK-NEXT:    buffer_load_dword v71, v2, s[0:3], 0 offen offset:140
+; CHECK-NEXT:    buffer_load_dword v70, v2, s[0:3], 0 offen offset:136
+; CHECK-NEXT:    buffer_load_dword v69, v2, s[0:3], 0 offen offset:132
+; CHECK-NEXT:    buffer_load_dword v68, v2, s[0:3], 0 offen offset:128
+; CHECK-NEXT:    buffer_load_dword v83, v2, s[0:3], 0 offen offset:92
+; CHECK-NEXT:    buffer_load_dword v82, v2, s[0:3], 0 offen offset:88
+; CHECK-NEXT:    buffer_load_dword v81, v2, s[0:3], 0 offen offset:84
+; CHECK-NEXT:    buffer_load_dword v80, v2, s[0:3], 0 offen offset:80
+; CHECK-NEXT:    buffer_load_dword v87, v2, s[0:3], 0 offen offset:76
+; CHECK-NEXT:    buffer_load_dword v86, v2, s[0:3], 0 offen offset:72
+; CHECK-NEXT:    buffer_load_dword v85, v2, s[0:3], 0 offen offset:68
+; CHECK-NEXT:    buffer_load_dword v84, v2, s[0:3], 0 offen offset:64
+; CHECK-NEXT:    buffer_load_dword v96, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v97, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v98, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v99, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    v_add_co_u32 v100, vcc_lo, v0, s4
+; CHECK-NEXT:    v_add_co_ci_u32_e32 v101, vcc_lo, s5, v1, vcc_lo
+; CHECK-NEXT:    v_add_nc_u32_e32 v2, 0xffffff00, v2
+; CHECK-NEXT:    s_add_u32 s4, s4, 0xffffff00
+; CHECK-NEXT:    s_addc_u32 s5, s5, -1
+; CHECK-NEXT:    s_waitcnt vmcnt(41)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[23:26] offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(37)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[27:30] offset:224
+; CHECK-NEXT:    s_waitcnt vmcnt(33)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[31:34] offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(29)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[35:38] offset:192
+; CHECK-NEXT:    s_waitcnt vmcnt(25)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[48:51] offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(21)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[52:55] offset:160
+; CHECK-NEXT:    s_waitcnt vmcnt(17)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[64:67] offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(13)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[68:71] offset:128
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[15:18] offset:112
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[19:22] offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(9)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[80:83] offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[84:87] offset:64
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[11:14] offset:48
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[7:10] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[3:6] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[96:99]
+; CHECK-NEXT:    s_cmp_eq_u64 s[4:5], s[6:7]
+; CHECK-NEXT:    s_cbranch_scc0 .LBB9_4
+; CHECK-NEXT:  .LBB9_5: ; %Flow11
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+;
+; ALIGNED-LABEL: memmove_p0_p5_sz2048:
+; ALIGNED:       ; %bb.0: ; %entry
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v72, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v73, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v74, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v75, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v76, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v77, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v78, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v79, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v88, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v89, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v90, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v91, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v92, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v93, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v94, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v95, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v104, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v105, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v106, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v107, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v108, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v109, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v110, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v111, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v120, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v121, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v122, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v123, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v124, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v125, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v126, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v127, off, s[0:3], s32 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_mov_b64 s[4:5], 0
+; ALIGNED-NEXT:    s_mov_b32 s6, exec_lo
+; ALIGNED-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc_lo
+; ALIGNED-NEXT:    v_cmpx_ge_u32_e64 v2, v0
+; ALIGNED-NEXT:    s_xor_b32 s6, exec_lo, s6
+; ALIGNED-NEXT:    s_cbranch_execz .LBB9_2
+; ALIGNED-NEXT:  .LBB9_1: ; %memmove_fwd_loop
+; ALIGNED-NEXT:    ; =>This Inner Loop Header: Depth=1
+; ALIGNED-NEXT:    s_clause 0x39
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:20
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:21
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:22
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:23
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:24
+; ALIGNED-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:25
+; ALIGNED-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:26
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:30
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:31
+; ALIGNED-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:32
+; ALIGNED-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:33
+; ALIGNED-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:34
+; ALIGNED-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:29
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:28
+; ALIGNED-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:27
+; ALIGNED-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:35
+; ALIGNED-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:36
+; ALIGNED-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:37
+; ALIGNED-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:38
+; ALIGNED-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:39
+; ALIGNED-NEXT:    buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:40
+; ALIGNED-NEXT:    buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:41
+; ALIGNED-NEXT:    buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:42
+; ALIGNED-NEXT:    buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:43
+; ALIGNED-NEXT:    buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:44
+; ALIGNED-NEXT:    buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:45
+; ALIGNED-NEXT:    buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:46
+; ALIGNED-NEXT:    buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:47
+; ALIGNED-NEXT:    buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:48
+; ALIGNED-NEXT:    buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:49
+; ALIGNED-NEXT:    buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:50
+; ALIGNED-NEXT:    buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:51
+; ALIGNED-NEXT:    buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:52
+; ALIGNED-NEXT:    buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:53
+; ALIGNED-NEXT:    buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:54
+; ALIGNED-NEXT:    buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:55
+; ALIGNED-NEXT:    buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:56
+; ALIGNED-NEXT:    buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:57
+; ALIGNED-NEXT:    buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:58
+; ALIGNED-NEXT:    buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:62
+; ALIGNED-NEXT:    buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:63
+; ALIGNED-NEXT:    buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:64
+; ALIGNED-NEXT:    buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:65
+; ALIGNED-NEXT:    buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:66
+; ALIGNED-NEXT:    buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:61
+; ALIGNED-NEXT:    buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:60
+; ALIGNED-NEXT:    buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:59
+; ALIGNED-NEXT:    buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:67
+; ALIGNED-NEXT:    buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:68
+; ALIGNED-NEXT:    buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:69
+; ALIGNED-NEXT:    buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:70
+; ALIGNED-NEXT:    buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:71
+; ALIGNED-NEXT:    buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:76
+; ALIGNED-NEXT:    buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:77
+; ALIGNED-NEXT:    buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:78
+; ALIGNED-NEXT:    buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:79
+; ALIGNED-NEXT:    buffer_load_ubyte v127, v2, s[0:3], 0 offen offset:19
+; ALIGNED-NEXT:    buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:75
+; ALIGNED-NEXT:    s_waitcnt vmcnt(57)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(56)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(55)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(54)
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(53)
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(52)
+; ALIGNED-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(51)
+; ALIGNED-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(50)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(49)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(48)
+; ALIGNED-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v3
+; ALIGNED-NEXT:    s_waitcnt vmcnt(45)
+; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(44)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(43)
+; ALIGNED-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v9, 8, v5
+; ALIGNED-NEXT:    s_waitcnt vmcnt(41)
+; ALIGNED-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v8, 8, v6
+; ALIGNED-NEXT:    v_lshl_or_b32 v5, v10, 8, v7
+; ALIGNED-NEXT:    v_lshl_or_b32 v6, v11, 8, v12
+; ALIGNED-NEXT:    v_lshl_or_b32 v7, v15, 8, v14
+; ALIGNED-NEXT:    v_lshl_or_b32 v8, v19, 8, v17
+; ALIGNED-NEXT:    s_waitcnt vmcnt(40)
+; ALIGNED-NEXT:    v_lshl_or_b32 v9, v16, 8, v13
+; ALIGNED-NEXT:    s_waitcnt vmcnt(38)
+; ALIGNED-NEXT:    v_lshl_or_b32 v10, v20, 8, v18
+; ALIGNED-NEXT:    s_waitcnt vmcnt(36)
+; ALIGNED-NEXT:    v_lshl_or_b32 v11, v23, 8, v22
+; ALIGNED-NEXT:    s_waitcnt vmcnt(34)
+; ALIGNED-NEXT:    v_lshl_or_b32 v12, v27, 8, v25
+; ALIGNED-NEXT:    s_waitcnt vmcnt(32)
+; ALIGNED-NEXT:    v_lshl_or_b32 v13, v24, 8, v21
+; ALIGNED-NEXT:    s_waitcnt vmcnt(30)
+; ALIGNED-NEXT:    v_lshl_or_b32 v14, v28, 8, v26
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v6, 16, v5
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v8, 16, v7
+; ALIGNED-NEXT:    v_lshl_or_b32 v5, v10, 16, v9
+; ALIGNED-NEXT:    v_lshl_or_b32 v6, v12, 16, v11
+; ALIGNED-NEXT:    v_lshl_or_b32 v7, v14, 16, v13
+; ALIGNED-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(28)
+; ALIGNED-NEXT:    v_lshl_or_b32 v15, v30, 8, v29
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(26)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v32, 8, v34
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(24)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v36, 8, v31
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(22)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v35, 8, v33
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(12)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v48, 8, v37
+; ALIGNED-NEXT:    v_lshl_or_b32 v5, v39, 8, v38
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v6, v50, 8, v49
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(11)
+; ALIGNED-NEXT:    v_lshl_or_b32 v7, v51, 8, v52
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v0, 16, v15
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 16, v1
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v5, 16, v4
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:85
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v7, 16, v6
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v54, 8, v53
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(11)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v55, 8, v65
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(9)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v66, 8, v64
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v68, 8, v67
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:86
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:82
+; ALIGNED-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v70, 8, v69
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:83
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:74
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v80, 8, v71
+; ALIGNED-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:73
+; ALIGNED-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:996 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:72
+; ALIGNED-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v36, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v35, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v38, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v39, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v48, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v37, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v49, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v50, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v52, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v51, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v53, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v54, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v65, off, s[0:3], s32 offset:944 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v55, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v64, off, s[0:3], s32 offset:940 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v66, off, s[0:3], s32 offset:952 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v67, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v68, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v69, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v70, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v71, off, s[0:3], s32 offset:976 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v80, off, s[0:3], s32 offset:980 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    buffer_store_dword v81, off, s[0:3], s32 offset:1000 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v127, off, s[0:3], s32 offset:1404 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:87
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1032 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:1036 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:1024 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:1020 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1004 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v81, 8, v3
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:84
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:81
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1008 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:80
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:1040 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1028 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1016 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1012 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v7
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:98
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v8, 8, v6
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:102
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:103
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1044 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v5, 8, v3
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:94
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:95
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:93
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:91
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1052 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:92
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:1116 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:1120 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1060 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1064 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1056 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1048 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v5, 8, v3
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:90
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:1080 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:101
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:89
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1076 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:88
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1084 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1112 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1072 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1068 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v3
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:99
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:100
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:1104 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:97
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1088 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:96
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:1100 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1108 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1096 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1092 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v7
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:114
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v8, 8, v6
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:118
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:119
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1124 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v5, 8, v3
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:110
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:111
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:109
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:107
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1132 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:108
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:1196 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:1200 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1140 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1144 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1136 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v5, 8, v3
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:106
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:1160 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:117
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:105
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1156 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:104
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1164 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1192 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1152 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1148 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v3
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:115
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:116
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:1188 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:113
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1168 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:112
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:1180 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1184 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1176 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1172 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v7
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:130
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v8, 8, v6
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:134
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:135
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1204 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v5, 8, v3
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:126
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:127
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:125
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:123
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1212 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:124
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:1280 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:1288 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1220 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1224 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1216 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1208 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v5, 8, v3
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:122
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:1240 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:133
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:121
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1232 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:120
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1244 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1276 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1236 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1228 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v3
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:131
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:132
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:1272 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:129
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1248 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:128
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:1264 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1268 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1256 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1252 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v7
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v8, 8, v6
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1312 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v5, 8, v3
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:142
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:143
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:141
+; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:139
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1328 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:140
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1336 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1340 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1332 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1324 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v5, 8, v3
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:138
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:1364 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:137
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1352 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:136
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1376 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1356 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1344 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1384 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:144
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1396 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x3
+; ALIGNED-NEXT:    buffer_load_ubyte v124, v2, s[0:3], 0 offen offset:145
+; ALIGNED-NEXT:    buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:146
+; ALIGNED-NEXT:    buffer_load_ubyte v120, v2, s[0:3], 0 offen offset:147
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:148
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v124, 8, v0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v120, 8, v111
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1400 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v121, v2, s[0:3], 0 offen offset:149
+; ALIGNED-NEXT:    buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:150
+; ALIGNED-NEXT:    buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:151
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1408 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v121, 8, v3
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v109, 8, v122
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1412 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:156
+; ALIGNED-NEXT:    buffer_load_ubyte v105, v2, s[0:3], 0 offen offset:157
+; ALIGNED-NEXT:    buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:158
+; ALIGNED-NEXT:    buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:159
+; ALIGNED-NEXT:    buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:155
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v105, 8, v108
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v104, 8, v107
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:152
+; ALIGNED-NEXT:    buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:153
+; ALIGNED-NEXT:    buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:154
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v92, 8, v93
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v95, 8, v90
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1420 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:160
+; ALIGNED-NEXT:    buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:161
+; ALIGNED-NEXT:    buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:162
+; ALIGNED-NEXT:    buffer_load_ubyte v74, v2, s[0:3], 0 offen offset:163
+; ALIGNED-NEXT:    buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:164
+; ALIGNED-NEXT:    buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:165
+; ALIGNED-NEXT:    buffer_load_ubyte v76, v2, s[0:3], 0 offen offset:166
+; ALIGNED-NEXT:    buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:167
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v79, 8, v89
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v74, 8, v73
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v72, 8, v76
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1424 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v75, 8, v88
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1428 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:172
+; ALIGNED-NEXT:    buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:173
+; ALIGNED-NEXT:    buffer_load_ubyte v62, v2, s[0:3], 0 offen offset:174
+; ALIGNED-NEXT:    buffer_load_ubyte v60, v2, s[0:3], 0 offen offset:175
+; ALIGNED-NEXT:    buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:171
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v61, 8, v63
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v60, 8, v62
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1432 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v59, v2, s[0:3], 0 offen offset:168
+; ALIGNED-NEXT:    buffer_load_ubyte v57, v2, s[0:3], 0 offen offset:169
+; ALIGNED-NEXT:    buffer_load_ubyte v56, v2, s[0:3], 0 offen offset:170
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v57, 8, v59
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v58, 8, v56
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1436 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:176
+; ALIGNED-NEXT:    buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:177
+; ALIGNED-NEXT:    buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:178
+; ALIGNED-NEXT:    buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:179
+; ALIGNED-NEXT:    buffer_load_ubyte v46, v2, s[0:3], 0 offen offset:180
+; ALIGNED-NEXT:    buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:181
+; ALIGNED-NEXT:    buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:182
+; ALIGNED-NEXT:    buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:183
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v45, 8, v47
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v42, 8, v41
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v40, 8, v44
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1440 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v43, 8, v46
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1444 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:188
+; ALIGNED-NEXT:    buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:189
+; ALIGNED-NEXT:    buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:190
+; ALIGNED-NEXT:    buffer_load_ubyte v116, v2, s[0:3], 0 offen offset:191
+; ALIGNED-NEXT:    buffer_load_ubyte v114, v2, s[0:3], 0 offen offset:187
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v117, 8, v119
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v116, 8, v118
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1448 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v115, v2, s[0:3], 0 offen offset:184
+; ALIGNED-NEXT:    buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:185
+; ALIGNED-NEXT:    buffer_load_ubyte v112, v2, s[0:3], 0 offen offset:186
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v113, 8, v115
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v114, 8, v112
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1452 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:192
+; ALIGNED-NEXT:    buffer_load_ubyte v100, v2, s[0:3], 0 offen offset:193
+; ALIGNED-NEXT:    buffer_load_ubyte v97, v2, s[0:3], 0 offen offset:194
+; ALIGNED-NEXT:    buffer_load_ubyte v96, v2, s[0:3], 0 offen offset:195
+; ALIGNED-NEXT:    buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:196
+; ALIGNED-NEXT:    buffer_load_ubyte v99, v2, s[0:3], 0 offen offset:197
+; ALIGNED-NEXT:    buffer_load_ubyte v98, v2, s[0:3], 0 offen offset:198
+; ALIGNED-NEXT:    buffer_load_ubyte v87, v2, s[0:3], 0 offen offset:199
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v100, 8, v102
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v96, 8, v97
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v87, 8, v98
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1456 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v99, 8, v101
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1460 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:204
+; ALIGNED-NEXT:    buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:205
+; ALIGNED-NEXT:    buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:206
+; ALIGNED-NEXT:    buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:207
+; ALIGNED-NEXT:    buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:203
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v83, 8, v85
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v82, 8, v84
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1464 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:200
+; ALIGNED-NEXT:    buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:201
+; ALIGNED-NEXT:    buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:202
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v70, 8, v80
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v81, 8, v69
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1468 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:212
+; ALIGNED-NEXT:    buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:213
+; ALIGNED-NEXT:    buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:214
+; ALIGNED-NEXT:    buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:215
+; ALIGNED-NEXT:    buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:211
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v54, 8, v68
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v53, 8, v66
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1472 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:216
+; ALIGNED-NEXT:    buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:217
+; ALIGNED-NEXT:    buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:218
+; ALIGNED-NEXT:    buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:219
+; ALIGNED-NEXT:    buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:220
+; ALIGNED-NEXT:    buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:221
+; ALIGNED-NEXT:    buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:222
+; ALIGNED-NEXT:    buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:223
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v64, 8, v67
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v50, 8, v49
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v51, 8, v65
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v48, 8, v52
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1476 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1480 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:208
+; ALIGNED-NEXT:    buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:209
+; ALIGNED-NEXT:    buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:210
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v37, 8, v39
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v55, 8, v38
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1484 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:224
+; ALIGNED-NEXT:    buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:225
+; ALIGNED-NEXT:    buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:226
+; ALIGNED-NEXT:    buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:227
+; ALIGNED-NEXT:    buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:228
+; ALIGNED-NEXT:    buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:229
+; ALIGNED-NEXT:    buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:230
+; ALIGNED-NEXT:    buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:231
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v34, 8, v36
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v30, 8, v31
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v33, 8, v35
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v29, 8, v32
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1488 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x17
+; ALIGNED-NEXT:    buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:236
+; ALIGNED-NEXT:    buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:237
+; ALIGNED-NEXT:    buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:238
+; ALIGNED-NEXT:    buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:239
+; ALIGNED-NEXT:    buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:235
+; ALIGNED-NEXT:    buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:232
+; ALIGNED-NEXT:    buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:233
+; ALIGNED-NEXT:    buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:234
+; ALIGNED-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:240
+; ALIGNED-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:241
+; ALIGNED-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:242
+; ALIGNED-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:243
+; ALIGNED-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:244
+; ALIGNED-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:245
+; ALIGNED-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:246
+; ALIGNED-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:247
+; ALIGNED-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:252
+; ALIGNED-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:253
+; ALIGNED-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:254
+; ALIGNED-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:255
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:251
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:248
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:249
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:250
+; ALIGNED-NEXT:    v_lshl_or_b32 v110, v4, 16, v3
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:3
+; ALIGNED-NEXT:    buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:4
+; ALIGNED-NEXT:    buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:5
+; ALIGNED-NEXT:    buffer_load_ubyte v125, v2, s[0:3], 0 offen offset:6
+; ALIGNED-NEXT:    s_waitcnt vmcnt(27)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v27, 8, v28
+; ALIGNED-NEXT:    s_waitcnt vmcnt(25)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v25, 8, v26
+; ALIGNED-NEXT:    s_waitcnt vmcnt(13)
+; ALIGNED-NEXT:    v_lshl_or_b32 v77, v13, 8, v16
+; ALIGNED-NEXT:    s_waitcnt vmcnt(9)
+; ALIGNED-NEXT:    v_lshl_or_b32 v91, v9, 8, v10
+; ALIGNED-NEXT:    v_lshl_or_b32 v94, v4, 16, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v22, 8, v24
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v23, 8, v21
+; ALIGNED-NEXT:    v_lshl_or_b32 v78, v4, 16, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v18, 8, v20
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v14, 8, v15
+; ALIGNED-NEXT:    v_lshl_or_b32 v103, v4, 16, v3
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:7
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v17, 8, v19
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1292 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v106, off, s[0:3], s32 offset:1296 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v123, off, s[0:3], s32 offset:1304 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v125, off, s[0:3], s32 offset:1308 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v86, v77, 16, v4
+; ALIGNED-NEXT:    v_lshl_or_b32 v77, v11, 8, v12
+; ALIGNED-NEXT:    v_lshl_or_b32 v71, v91, 16, v77
+; ALIGNED-NEXT:    v_lshl_or_b32 v77, v6, 8, v8
+; ALIGNED-NEXT:    v_lshl_or_b32 v91, v7, 8, v5
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v91, 16, v77
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:1
+; ALIGNED-NEXT:    buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:2
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1260 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1320 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v77, off, s[0:3], s32 offset:1284 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v91, off, s[0:3], s32 offset:1300 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v77, v77, 8, v1
+; ALIGNED-NEXT:    v_lshl_or_b32 v91, v0, 8, v91
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:12
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v91, 16, v77
+; ALIGNED-NEXT:    v_lshl_or_b32 v77, v123, 8, v106
+; ALIGNED-NEXT:    v_lshl_or_b32 v91, v3, 8, v125
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:13
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1316 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v91, 16, v77
+; ALIGNED-NEXT:    buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:15
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1348 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:14
+; ALIGNED-NEXT:    buffer_load_ubyte v126, v2, s[0:3], 0 offen offset:11
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1360 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v77, v3, 8, v1
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1380 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v91, off, s[0:3], s32 offset:1372 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v91, v91, 8, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1368 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v91, 16, v77
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1388 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:8
+; ALIGNED-NEXT:    buffer_load_ubyte v125, v2, s[0:3], 0 offen offset:9
+; ALIGNED-NEXT:    buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:10
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v77, v125, 8, v1
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v91, v126, 8, v123
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v91, 16, v77
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1392 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:16
+; ALIGNED-NEXT:    buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:18
+; ALIGNED-NEXT:    buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:17
+; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT:    buffer_store_dword v71, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT:    buffer_store_dword v86, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT:    buffer_store_dword v103, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:704
+; ALIGNED-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:708
+; ALIGNED-NEXT:    v_add_nc_u32_e32 v2, 0x100, v2
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v127, 8, v77
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshl_or_b32 v127, v91, 8, v106
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_add_co_u32 v3, vcc_lo, v3, s4
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, s5, v4, vcc_lo
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v5 offset:250
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v7 offset:251
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v6 offset:249
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v9 offset:255
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v11 offset:253
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v10 offset:254
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v12 offset:252
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v8 offset:248
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v15 offset:242
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v14 offset:243
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v18 offset:241
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v13 offset:247
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v17 offset:245
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v16 offset:246
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v19 offset:244
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v20 offset:240
+; ALIGNED-NEXT:    buffer_store_dword v78, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT:    buffer_store_dword v94, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT:    buffer_store_dword v110, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT:    v_lshl_or_b32 v127, v0, 16, v127
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1488 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_add_u32 s4, s4, 0x100
+; ALIGNED-NEXT:    s_addc_u32 s5, s5, 0
+; ALIGNED-NEXT:    s_cmp_lg_u64 s[4:5], 0x800
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v21 offset:234
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v23 offset:235
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v22 offset:233
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v25 offset:239
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v27 offset:237
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v26 offset:238
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v28 offset:236
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v24 offset:232
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v31 offset:226
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v30 offset:227
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v34 offset:225
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v29 offset:231
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v33 offset:229
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v32 offset:230
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v35 offset:228
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v36 offset:224
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1484 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1480 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1476 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1472 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v54 offset:213
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v53 offset:215
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v37 offset:209
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v55 offset:211
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v38 offset:210
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v66 offset:214
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v68 offset:212
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v49 offset:218
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v50 offset:219
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v64 offset:217
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v48 offset:223
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v51 offset:221
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v52 offset:222
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v65 offset:220
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v67 offset:216
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v39 offset:208
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1468 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1464 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1460 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1456 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v69 offset:202
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v81 offset:203
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v70 offset:201
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v82 offset:207
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v83 offset:205
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v84 offset:206
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v85 offset:204
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v80 offset:200
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v97 offset:194
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v96 offset:195
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v100 offset:193
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v87 offset:199
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v99 offset:197
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v98 offset:198
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v101 offset:196
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v102 offset:192
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1452 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1448 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:300
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1444 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1440 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v112 offset:186
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v114 offset:187
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v113 offset:185
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v116 offset:191
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v117 offset:189
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v118 offset:190
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v119 offset:188
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v115 offset:184
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v41 offset:178
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v42 offset:179
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v45 offset:177
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v40 offset:183
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v43 offset:181
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v44 offset:182
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v46 offset:180
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v47 offset:176
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1436 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1432 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1428 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:308
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1424 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:304
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v56 offset:170
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v58 offset:171
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v57 offset:169
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v60 offset:175
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v61 offset:173
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v62 offset:174
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v63 offset:172
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v59 offset:168
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v73 offset:162
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v74 offset:163
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v79 offset:161
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v72 offset:167
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v75 offset:165
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v76 offset:166
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v88 offset:164
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v89 offset:160
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1420 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:264
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1416 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1412 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1408 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:256
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v90 offset:154
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v95 offset:155
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v92 offset:153
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v104 offset:159
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v105 offset:157
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v107 offset:158
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v108 offset:156
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v93 offset:152
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v111 offset:146
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v120 offset:147
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v124 offset:145
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v109 offset:151
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v121 offset:149
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v122 offset:150
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1400 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:148
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1396 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:144
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1384 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1352 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1328 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1312 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1376 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:138
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1364 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:139
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1356 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:137
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1340 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:143
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1332 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:141
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1336 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:142
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1324 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:140
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1344 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:136
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1272 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:130
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1264 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:131
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1256 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:129
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1288 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:135
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1276 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:133
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1280 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:134
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1268 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:132
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1252 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:128
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1248 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:360
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1232 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:364
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1212 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:356
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1204 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:352
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1244 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:122
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1240 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:123
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1236 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:121
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1224 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:127
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1216 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:125
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1220 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:126
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1208 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:124
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1228 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:120
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:114
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1180 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:115
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1176 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:113
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1200 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:119
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:117
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1196 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:118
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1184 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:116
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1172 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:112
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1168 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:376
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1156 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:380
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1132 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:372
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1124 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:368
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1164 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:106
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1160 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:107
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1152 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:105
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1144 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:111
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1136 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:109
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1140 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:110
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1128 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:108
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1148 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:104
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1104 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:98
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1100 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:99
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1096 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:97
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1120 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:103
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:101
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1116 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:102
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1108 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:100
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1092 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:96
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1088 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:328
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1076 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:332
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:324
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1044 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1084 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:90
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1080 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:91
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1072 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:89
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1064 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:95
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1056 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:93
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1060 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:94
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:92
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1068 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:88
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1024 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:82
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1020 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:83
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1016 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:81
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:87
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:85
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:86
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:84
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1012 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:80
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1008 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:344
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:996 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:348
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:340
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:972 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:336
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1004 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:74
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1000 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:75
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:992 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:73
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:980 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:79
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:968 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:77
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:976 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:78
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:76
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:72
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:944 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:66
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:67
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:65
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:71
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:69
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:70
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:68
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:64
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:948 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:424
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:428
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:420
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:416
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:61
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:58
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:59
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:57
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:63
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:62
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:60
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:56
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:53
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:50
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:51
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:49
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:55
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:54
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:52
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:48
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:444
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:440
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:436
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:432
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:43
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:42
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:41
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:40
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:47
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:46
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:45
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:44
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:35
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:34
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:33
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:32
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:39
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:38
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:37
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:36
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:392
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:396
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:388
+; ALIGNED-NEXT:    buffer_store_dword v127, off, s[0:3], s32 offset:384
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:26
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:27
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:25
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:31
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:29
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:30
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:28
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:24
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v77 offset:18
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1404 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:19
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v91 offset:17
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:23
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:21
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:22
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:20
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v106 offset:16
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1392 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:408
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1388 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:412
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1348 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:404
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1316 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:400
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v123 offset:10
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v126 offset:11
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1380 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:13
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v125 offset:9
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1372 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:15
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1368 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:14
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1360 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:12
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v1 offset:8
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1300 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:2
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1292 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:3
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1284 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:1
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1320 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:7
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1304 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:5
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1308 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:6
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1296 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:4
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1260 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v0
+; ALIGNED-NEXT:    s_cbranch_scc1 .LBB9_1
+; ALIGNED-NEXT:  .LBB9_2: ; %Flow10
+; ALIGNED-NEXT:    s_andn2_saveexec_b32 s8, s6
+; ALIGNED-NEXT:    s_cbranch_execz .LBB9_5
+; ALIGNED-NEXT:  ; %bb.3: ; %memmove_bwd_loop.preheader
+; ALIGNED-NEXT:    v_add_nc_u32_e32 v4, 0x700, v2
+; ALIGNED-NEXT:    s_movk_i32 s6, 0xff00
+; ALIGNED-NEXT:    s_mov_b64 s[4:5], 0x700
+; ALIGNED-NEXT:    s_mov_b32 s7, -1
+; ALIGNED-NEXT:  .LBB9_4: ; %memmove_bwd_loop
+; ALIGNED-NEXT:    ; =>This Inner Loop Header: Depth=1
+; ALIGNED-NEXT:    s_clause 0x3a
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen offset:20
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:21
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:22
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v4, s[0:3], 0 offen offset:23
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v4, s[0:3], 0 offen offset:24
+; ALIGNED-NEXT:    buffer_load_ubyte v10, v4, s[0:3], 0 offen offset:25
+; ALIGNED-NEXT:    buffer_load_ubyte v12, v4, s[0:3], 0 offen offset:26
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v4, s[0:3], 0 offen offset:30
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v4, s[0:3], 0 offen offset:31
+; ALIGNED-NEXT:    buffer_load_ubyte v14, v4, s[0:3], 0 offen offset:32
+; ALIGNED-NEXT:    buffer_load_ubyte v15, v4, s[0:3], 0 offen offset:33
+; ALIGNED-NEXT:    buffer_load_ubyte v17, v4, s[0:3], 0 offen offset:34
+; ALIGNED-NEXT:    buffer_load_ubyte v9, v4, s[0:3], 0 offen offset:29
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v4, s[0:3], 0 offen offset:28
+; ALIGNED-NEXT:    buffer_load_ubyte v11, v4, s[0:3], 0 offen offset:27
+; ALIGNED-NEXT:    buffer_load_ubyte v19, v4, s[0:3], 0 offen offset:35
+; ALIGNED-NEXT:    buffer_load_ubyte v13, v4, s[0:3], 0 offen offset:36
+; ALIGNED-NEXT:    buffer_load_ubyte v16, v4, s[0:3], 0 offen offset:37
+; ALIGNED-NEXT:    buffer_load_ubyte v18, v4, s[0:3], 0 offen offset:38
+; ALIGNED-NEXT:    buffer_load_ubyte v20, v4, s[0:3], 0 offen offset:39
+; ALIGNED-NEXT:    buffer_load_ubyte v22, v4, s[0:3], 0 offen offset:40
+; ALIGNED-NEXT:    buffer_load_ubyte v23, v4, s[0:3], 0 offen offset:41
+; ALIGNED-NEXT:    buffer_load_ubyte v25, v4, s[0:3], 0 offen offset:42
+; ALIGNED-NEXT:    buffer_load_ubyte v28, v4, s[0:3], 0 offen offset:43
+; ALIGNED-NEXT:    buffer_load_ubyte v21, v4, s[0:3], 0 offen offset:44
+; ALIGNED-NEXT:    buffer_load_ubyte v24, v4, s[0:3], 0 offen offset:45
+; ALIGNED-NEXT:    buffer_load_ubyte v26, v4, s[0:3], 0 offen offset:46
+; ALIGNED-NEXT:    buffer_load_ubyte v27, v4, s[0:3], 0 offen offset:47
+; ALIGNED-NEXT:    buffer_load_ubyte v30, v4, s[0:3], 0 offen offset:48
+; ALIGNED-NEXT:    buffer_load_ubyte v31, v4, s[0:3], 0 offen offset:49
+; ALIGNED-NEXT:    buffer_load_ubyte v33, v4, s[0:3], 0 offen offset:50
+; ALIGNED-NEXT:    buffer_load_ubyte v34, v4, s[0:3], 0 offen offset:51
+; ALIGNED-NEXT:    buffer_load_ubyte v32, v4, s[0:3], 0 offen offset:52
+; ALIGNED-NEXT:    buffer_load_ubyte v37, v4, s[0:3], 0 offen offset:53
+; ALIGNED-NEXT:    buffer_load_ubyte v35, v4, s[0:3], 0 offen offset:54
+; ALIGNED-NEXT:    buffer_load_ubyte v36, v4, s[0:3], 0 offen offset:55
+; ALIGNED-NEXT:    buffer_load_ubyte v48, v4, s[0:3], 0 offen offset:56
+; ALIGNED-NEXT:    buffer_load_ubyte v51, v4, s[0:3], 0 offen offset:57
+; ALIGNED-NEXT:    buffer_load_ubyte v52, v4, s[0:3], 0 offen offset:58
+; ALIGNED-NEXT:    buffer_load_ubyte v38, v4, s[0:3], 0 offen offset:60
+; ALIGNED-NEXT:    buffer_load_ubyte v50, v4, s[0:3], 0 offen offset:61
+; ALIGNED-NEXT:    buffer_load_ubyte v39, v4, s[0:3], 0 offen offset:62
+; ALIGNED-NEXT:    buffer_load_ubyte v49, v4, s[0:3], 0 offen offset:63
+; ALIGNED-NEXT:    buffer_load_ubyte v29, v4, s[0:3], 0 offen offset:64
+; ALIGNED-NEXT:    buffer_load_ubyte v55, v4, s[0:3], 0 offen offset:65
+; ALIGNED-NEXT:    buffer_load_ubyte v66, v4, s[0:3], 0 offen offset:66
+; ALIGNED-NEXT:    buffer_load_ubyte v53, v4, s[0:3], 0 offen offset:59
+; ALIGNED-NEXT:    buffer_load_ubyte v67, v4, s[0:3], 0 offen offset:67
+; ALIGNED-NEXT:    buffer_load_ubyte v54, v4, s[0:3], 0 offen offset:68
+; ALIGNED-NEXT:    buffer_load_ubyte v64, v4, s[0:3], 0 offen offset:69
+; ALIGNED-NEXT:    buffer_load_ubyte v65, v4, s[0:3], 0 offen offset:70
+; ALIGNED-NEXT:    buffer_load_ubyte v68, v4, s[0:3], 0 offen offset:71
+; ALIGNED-NEXT:    buffer_load_ubyte v69, v4, s[0:3], 0 offen offset:76
+; ALIGNED-NEXT:    buffer_load_ubyte v70, v4, s[0:3], 0 offen offset:77
+; ALIGNED-NEXT:    buffer_load_ubyte v71, v4, s[0:3], 0 offen offset:78
+; ALIGNED-NEXT:    buffer_load_ubyte v80, v4, s[0:3], 0 offen offset:79
+; ALIGNED-NEXT:    buffer_load_ubyte v126, v4, s[0:3], 0 offen offset:19
+; ALIGNED-NEXT:    buffer_load_ubyte v81, v4, s[0:3], 0 offen offset:75
+; ALIGNED-NEXT:    buffer_load_ubyte v125, v4, s[0:3], 0 offen offset:151
+; ALIGNED-NEXT:    s_waitcnt vmcnt(58)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(57)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(56)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(55)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(54)
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(53)
+; ALIGNED-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(52)
+; ALIGNED-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(51)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(50)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(49)
+; ALIGNED-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 8, v2
+; ALIGNED-NEXT:    s_waitcnt vmcnt(46)
+; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(45)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(44)
+; ALIGNED-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v9, 8, v5
+; ALIGNED-NEXT:    s_waitcnt vmcnt(42)
+; ALIGNED-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v8, 8, v6
+; ALIGNED-NEXT:    v_lshl_or_b32 v5, v10, 8, v7
+; ALIGNED-NEXT:    v_lshl_or_b32 v6, v11, 8, v12
+; ALIGNED-NEXT:    v_lshl_or_b32 v7, v15, 8, v14
+; ALIGNED-NEXT:    v_lshl_or_b32 v8, v19, 8, v17
+; ALIGNED-NEXT:    s_waitcnt vmcnt(41)
+; ALIGNED-NEXT:    v_lshl_or_b32 v9, v16, 8, v13
+; ALIGNED-NEXT:    s_waitcnt vmcnt(39)
+; ALIGNED-NEXT:    v_lshl_or_b32 v10, v20, 8, v18
+; ALIGNED-NEXT:    s_waitcnt vmcnt(37)
+; ALIGNED-NEXT:    v_lshl_or_b32 v11, v23, 8, v22
+; ALIGNED-NEXT:    s_waitcnt vmcnt(35)
+; ALIGNED-NEXT:    v_lshl_or_b32 v12, v28, 8, v25
+; ALIGNED-NEXT:    s_waitcnt vmcnt(33)
+; ALIGNED-NEXT:    v_lshl_or_b32 v13, v24, 8, v21
+; ALIGNED-NEXT:    s_waitcnt vmcnt(31)
+; ALIGNED-NEXT:    v_lshl_or_b32 v14, v27, 8, v26
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 16, v2
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v6, 16, v5
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v8, 16, v7
+; ALIGNED-NEXT:    v_lshl_or_b32 v5, v10, 16, v9
+; ALIGNED-NEXT:    v_lshl_or_b32 v6, v12, 16, v11
+; ALIGNED-NEXT:    v_lshl_or_b32 v7, v14, 16, v13
+; ALIGNED-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(29)
+; ALIGNED-NEXT:    v_lshl_or_b32 v15, v31, 8, v30
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(27)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v34, 8, v33
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(25)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v37, 8, v32
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(23)
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v36, 8, v35
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(18)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v50, 8, v38
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(16)
+; ALIGNED-NEXT:    v_lshl_or_b32 v5, v49, 8, v39
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v6, v51, 8, v48
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(12)
+; ALIGNED-NEXT:    v_lshl_or_b32 v7, v53, 8, v52
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v0, 16, v15
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v5, 16, v3
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v4, s[0:3], 0 offen offset:85
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v7, 16, v6
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v55, 8, v29
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(12)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v67, 8, v66
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(10)
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v64, 8, v54
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(8)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v68, 8, v65
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v4, s[0:3], 0 offen offset:86
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v4, s[0:3], 0 offen offset:82
+; ALIGNED-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 16, v2
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:976 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(8)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v70, 8, v69
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v4, s[0:3], 0 offen offset:83
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:74
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(8)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v80, 8, v71
+; ALIGNED-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:73
+; ALIGNED-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen offset:72
+; ALIGNED-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v37, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v35, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v36, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v38, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v50, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v39, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v49, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v48, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v51, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v52, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v53, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v55, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v66, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v67, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v54, off, s[0:3], s32 offset:940 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v64, off, s[0:3], s32 offset:944 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v65, off, s[0:3], s32 offset:952 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v68, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v69, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v70, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v71, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v80, off, s[0:3], s32 offset:980 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(9)
+; ALIGNED-NEXT:    buffer_store_dword v126, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(8)
+; ALIGNED-NEXT:    buffer_store_dword v81, off, s[0:3], s32 offset:1000 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v4, s[0:3], 0 offen offset:87
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1032 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:1036 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:1028 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1020 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:1004 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:996 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v81, 8, v2
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:84
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:81
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1008 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen offset:80
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:1040 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:1024 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1016 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1012 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 8, v7
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v4, s[0:3], 0 offen offset:98
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v8, 8, v6
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v4, s[0:3], 0 offen offset:102
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v4, s[0:3], 0 offen offset:103
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1044 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v5, 8, v2
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:94
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v4, s[0:3], 0 offen offset:95
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:93
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v4, s[0:3], 0 offen offset:91
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1052 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen offset:92
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:1116 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:1120 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:1060 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1064 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1056 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1048 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v5, 8, v2
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:90
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1080 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v4, s[0:3], 0 offen offset:101
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:89
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1072 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen offset:88
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:1084 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1112 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1076 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1068 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 8, v2
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v4, s[0:3], 0 offen offset:99
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:100
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:1108 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:97
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1088 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen offset:96
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1100 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:1104 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1096 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1092 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 8, v7
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v4, s[0:3], 0 offen offset:114
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v8, 8, v6
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v4, s[0:3], 0 offen offset:118
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v4, s[0:3], 0 offen offset:119
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1124 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v5, 8, v2
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:110
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v4, s[0:3], 0 offen offset:111
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:109
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v4, s[0:3], 0 offen offset:107
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1132 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen offset:108
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:1196 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:1200 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:1140 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1144 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1136 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v5, 8, v2
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:106
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1160 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v4, s[0:3], 0 offen offset:117
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:105
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1152 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen offset:104
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:1164 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1192 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1156 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1148 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 8, v2
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v4, s[0:3], 0 offen offset:115
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:116
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:1188 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:113
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1168 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen offset:112
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1180 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:1184 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1176 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1172 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 8, v7
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v4, s[0:3], 0 offen offset:130
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v8, 8, v6
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v4, s[0:3], 0 offen offset:134
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v4, s[0:3], 0 offen offset:135
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1204 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v5, 8, v2
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:126
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v4, s[0:3], 0 offen offset:127
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:125
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v4, s[0:3], 0 offen offset:123
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1212 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen offset:124
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:1276 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:1280 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:1220 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1224 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1216 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1208 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v5, 8, v2
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:122
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1240 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v4, s[0:3], 0 offen offset:133
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:121
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1232 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen offset:120
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:1244 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1272 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1236 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1228 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 8, v2
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v4, s[0:3], 0 offen offset:131
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:132
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:1268 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:129
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1248 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen offset:128
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1260 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:1264 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1256 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1252 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 8, v7
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v4, s[0:3], 0 offen offset:146
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v8, 8, v6
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v4, s[0:3], 0 offen offset:150
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1284 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v5, 8, v2
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:142
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v4, s[0:3], 0 offen offset:143
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:141
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v4, s[0:3], 0 offen offset:139
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1292 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen offset:140
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:1372 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:1300 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1304 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1296 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1288 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v5, 8, v2
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:138
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1320 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v4, s[0:3], 0 offen offset:149
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:137
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1312 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen offset:136
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:1324 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1368 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1316 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1308 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 8, v2
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_ubyte v3, v4, s[0:3], 0 offen offset:147
+; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:148
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:1356 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:145
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1328 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen offset:144
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1348 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:1352 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:1340 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1332 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 8, v7
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v125, 8, v6
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1384 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v5, 8, v2
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1392 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v122, v4, s[0:3], 0 offen offset:156
+; ALIGNED-NEXT:    buffer_load_ubyte v111, v4, s[0:3], 0 offen offset:157
+; ALIGNED-NEXT:    buffer_load_ubyte v120, v4, s[0:3], 0 offen offset:158
+; ALIGNED-NEXT:    buffer_load_ubyte v109, v4, s[0:3], 0 offen offset:159
+; ALIGNED-NEXT:    buffer_load_ubyte v106, v4, s[0:3], 0 offen offset:155
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v111, 8, v122
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v109, 8, v120
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1400 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v105, v4, s[0:3], 0 offen offset:152
+; ALIGNED-NEXT:    buffer_load_ubyte v94, v4, s[0:3], 0 offen offset:153
+; ALIGNED-NEXT:    buffer_load_ubyte v92, v4, s[0:3], 0 offen offset:154
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v94, 8, v105
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v106, 8, v92
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1408 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_ubyte v89, v4, s[0:3], 0 offen offset:160
+; ALIGNED-NEXT:    buffer_load_ubyte v79, v4, s[0:3], 0 offen offset:161
+; ALIGNED-NEXT:    buffer_load_ubyte v73, v4, s[0:3], 0 offen offset:162
+; ALIGNED-NEXT:    buffer_load_ubyte v74, v4, s[0:3], 0 offen offset:163
+; ALIGNED-NEXT:    buffer_load_ubyte v88, v4, s[0:3], 0 offen offset:164
+; ALIGNED-NEXT:    buffer_load_ubyte v75, v4, s[0:3], 0 offen offset:165
+; ALIGNED-NEXT:    buffer_load_ubyte v77, v4, s[0:3], 0 offen offset:166
+; ALIGNED-NEXT:    buffer_load_ubyte v72, v4, s[0:3], 0 offen offset:167
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v79, 8, v89
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v74, 8, v73
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v72, 8, v77
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1420 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v75, 8, v88
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1424 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v63, v4, s[0:3], 0 offen offset:172
+; ALIGNED-NEXT:    buffer_load_ubyte v61, v4, s[0:3], 0 offen offset:173
+; ALIGNED-NEXT:    buffer_load_ubyte v62, v4, s[0:3], 0 offen offset:174
+; ALIGNED-NEXT:    buffer_load_ubyte v60, v4, s[0:3], 0 offen offset:175
+; ALIGNED-NEXT:    buffer_load_ubyte v58, v4, s[0:3], 0 offen offset:171
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v61, 8, v63
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v60, 8, v62
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1428 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v59, v4, s[0:3], 0 offen offset:168
+; ALIGNED-NEXT:    buffer_load_ubyte v56, v4, s[0:3], 0 offen offset:169
+; ALIGNED-NEXT:    buffer_load_ubyte v47, v4, s[0:3], 0 offen offset:170
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v56, 8, v59
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v58, 8, v47
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1432 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_ubyte v46, v4, s[0:3], 0 offen offset:176
+; ALIGNED-NEXT:    buffer_load_ubyte v44, v4, s[0:3], 0 offen offset:177
+; ALIGNED-NEXT:    buffer_load_ubyte v119, v4, s[0:3], 0 offen offset:178
+; ALIGNED-NEXT:    buffer_load_ubyte v40, v4, s[0:3], 0 offen offset:179
+; ALIGNED-NEXT:    buffer_load_ubyte v45, v4, s[0:3], 0 offen offset:180
+; ALIGNED-NEXT:    buffer_load_ubyte v41, v4, s[0:3], 0 offen offset:181
+; ALIGNED-NEXT:    buffer_load_ubyte v42, v4, s[0:3], 0 offen offset:182
+; ALIGNED-NEXT:    buffer_load_ubyte v118, v4, s[0:3], 0 offen offset:183
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v44, 8, v46
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v40, 8, v119
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v118, 8, v42
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1436 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v41, 8, v45
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1440 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v117, v4, s[0:3], 0 offen offset:188
+; ALIGNED-NEXT:    buffer_load_ubyte v115, v4, s[0:3], 0 offen offset:189
+; ALIGNED-NEXT:    buffer_load_ubyte v116, v4, s[0:3], 0 offen offset:190
+; ALIGNED-NEXT:    buffer_load_ubyte v114, v4, s[0:3], 0 offen offset:191
+; ALIGNED-NEXT:    buffer_load_ubyte v112, v4, s[0:3], 0 offen offset:187
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v115, 8, v117
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v114, 8, v116
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1444 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v113, v4, s[0:3], 0 offen offset:184
+; ALIGNED-NEXT:    buffer_load_ubyte v103, v4, s[0:3], 0 offen offset:185
+; ALIGNED-NEXT:    buffer_load_ubyte v102, v4, s[0:3], 0 offen offset:186
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v103, 8, v113
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v112, 8, v102
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1448 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_ubyte v100, v4, s[0:3], 0 offen offset:192
+; ALIGNED-NEXT:    buffer_load_ubyte v98, v4, s[0:3], 0 offen offset:193
+; ALIGNED-NEXT:    buffer_load_ubyte v87, v4, s[0:3], 0 offen offset:194
+; ALIGNED-NEXT:    buffer_load_ubyte v86, v4, s[0:3], 0 offen offset:195
+; ALIGNED-NEXT:    buffer_load_ubyte v99, v4, s[0:3], 0 offen offset:196
+; ALIGNED-NEXT:    buffer_load_ubyte v97, v4, s[0:3], 0 offen offset:197
+; ALIGNED-NEXT:    buffer_load_ubyte v96, v4, s[0:3], 0 offen offset:198
+; ALIGNED-NEXT:    buffer_load_ubyte v85, v4, s[0:3], 0 offen offset:199
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v98, 8, v100
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v86, 8, v87
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v2
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v97, 8, v99
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v85, 8, v96
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1452 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v2
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1456 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v83, v4, s[0:3], 0 offen offset:204
+; ALIGNED-NEXT:    buffer_load_ubyte v81, v4, s[0:3], 0 offen offset:205
+; ALIGNED-NEXT:    buffer_load_ubyte v82, v4, s[0:3], 0 offen offset:206
+; ALIGNED-NEXT:    buffer_load_ubyte v80, v4, s[0:3], 0 offen offset:207
+; ALIGNED-NEXT:    buffer_load_ubyte v71, v4, s[0:3], 0 offen offset:203
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v81, 8, v83
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v80, 8, v82
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v2
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1460 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v70, v4, s[0:3], 0 offen offset:200
+; ALIGNED-NEXT:    buffer_load_ubyte v69, v4, s[0:3], 0 offen offset:201
+; ALIGNED-NEXT:    buffer_load_ubyte v68, v4, s[0:3], 0 offen offset:202
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v69, 8, v70
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v71, 8, v68
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v2
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1464 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v67, v4, s[0:3], 0 offen offset:212
+; ALIGNED-NEXT:    buffer_load_ubyte v54, v4, s[0:3], 0 offen offset:213
+; ALIGNED-NEXT:    buffer_load_ubyte v65, v4, s[0:3], 0 offen offset:214
+; ALIGNED-NEXT:    buffer_load_ubyte v52, v4, s[0:3], 0 offen offset:215
+; ALIGNED-NEXT:    buffer_load_ubyte v55, v4, s[0:3], 0 offen offset:211
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v54, 8, v67
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v52, 8, v65
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v2
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1468 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_ubyte v66, v4, s[0:3], 0 offen offset:216
+; ALIGNED-NEXT:    buffer_load_ubyte v53, v4, s[0:3], 0 offen offset:217
+; ALIGNED-NEXT:    buffer_load_ubyte v49, v4, s[0:3], 0 offen offset:218
+; ALIGNED-NEXT:    buffer_load_ubyte v48, v4, s[0:3], 0 offen offset:219
+; ALIGNED-NEXT:    buffer_load_ubyte v64, v4, s[0:3], 0 offen offset:220
+; ALIGNED-NEXT:    buffer_load_ubyte v51, v4, s[0:3], 0 offen offset:221
+; ALIGNED-NEXT:    buffer_load_ubyte v50, v4, s[0:3], 0 offen offset:222
+; ALIGNED-NEXT:    buffer_load_ubyte v39, v4, s[0:3], 0 offen offset:223
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v53, 8, v66
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v48, 8, v49
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v2
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v51, 8, v64
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v39, 8, v50
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1472 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v2
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1476 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v38, v4, s[0:3], 0 offen offset:208
+; ALIGNED-NEXT:    buffer_load_ubyte v36, v4, s[0:3], 0 offen offset:209
+; ALIGNED-NEXT:    buffer_load_ubyte v37, v4, s[0:3], 0 offen offset:210
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v36, 8, v38
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v55, 8, v37
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v2
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1480 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x7
+; ALIGNED-NEXT:    buffer_load_ubyte v35, v4, s[0:3], 0 offen offset:224
+; ALIGNED-NEXT:    buffer_load_ubyte v33, v4, s[0:3], 0 offen offset:225
+; ALIGNED-NEXT:    buffer_load_ubyte v29, v4, s[0:3], 0 offen offset:226
+; ALIGNED-NEXT:    buffer_load_ubyte v30, v4, s[0:3], 0 offen offset:227
+; ALIGNED-NEXT:    buffer_load_ubyte v34, v4, s[0:3], 0 offen offset:228
+; ALIGNED-NEXT:    buffer_load_ubyte v31, v4, s[0:3], 0 offen offset:229
+; ALIGNED-NEXT:    buffer_load_ubyte v32, v4, s[0:3], 0 offen offset:230
+; ALIGNED-NEXT:    buffer_load_ubyte v28, v4, s[0:3], 0 offen offset:231
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v33, 8, v35
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v30, 8, v29
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v2
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v31, 8, v34
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v28, 8, v32
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1484 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x17
+; ALIGNED-NEXT:    buffer_load_ubyte v27, v4, s[0:3], 0 offen offset:236
+; ALIGNED-NEXT:    buffer_load_ubyte v25, v4, s[0:3], 0 offen offset:237
+; ALIGNED-NEXT:    buffer_load_ubyte v26, v4, s[0:3], 0 offen offset:238
+; ALIGNED-NEXT:    buffer_load_ubyte v24, v4, s[0:3], 0 offen offset:239
+; ALIGNED-NEXT:    buffer_load_ubyte v23, v4, s[0:3], 0 offen offset:235
+; ALIGNED-NEXT:    buffer_load_ubyte v22, v4, s[0:3], 0 offen offset:232
+; ALIGNED-NEXT:    buffer_load_ubyte v21, v4, s[0:3], 0 offen offset:233
+; ALIGNED-NEXT:    buffer_load_ubyte v20, v4, s[0:3], 0 offen offset:234
+; ALIGNED-NEXT:    buffer_load_ubyte v19, v4, s[0:3], 0 offen offset:240
+; ALIGNED-NEXT:    buffer_load_ubyte v17, v4, s[0:3], 0 offen offset:241
+; ALIGNED-NEXT:    buffer_load_ubyte v13, v4, s[0:3], 0 offen offset:242
+; ALIGNED-NEXT:    buffer_load_ubyte v14, v4, s[0:3], 0 offen offset:243
+; ALIGNED-NEXT:    buffer_load_ubyte v18, v4, s[0:3], 0 offen offset:244
+; ALIGNED-NEXT:    buffer_load_ubyte v15, v4, s[0:3], 0 offen offset:245
+; ALIGNED-NEXT:    buffer_load_ubyte v16, v4, s[0:3], 0 offen offset:246
+; ALIGNED-NEXT:    buffer_load_ubyte v12, v4, s[0:3], 0 offen offset:247
+; ALIGNED-NEXT:    buffer_load_ubyte v11, v4, s[0:3], 0 offen offset:252
+; ALIGNED-NEXT:    buffer_load_ubyte v9, v4, s[0:3], 0 offen offset:253
+; ALIGNED-NEXT:    buffer_load_ubyte v10, v4, s[0:3], 0 offen offset:254
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v4, s[0:3], 0 offen offset:255
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v4, s[0:3], 0 offen offset:251
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v4, s[0:3], 0 offen offset:248
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v4, s[0:3], 0 offen offset:249
+; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:250
+; ALIGNED-NEXT:    v_lshl_or_b32 v123, v3, 16, v2
+; ALIGNED-NEXT:    buffer_load_ubyte v0, v4, s[0:3], 0 offen
+; ALIGNED-NEXT:    s_waitcnt vmcnt(23)
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v25, 8, v27
+; ALIGNED-NEXT:    s_waitcnt vmcnt(21)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v24, 8, v26
+; ALIGNED-NEXT:    s_waitcnt vmcnt(9)
+; ALIGNED-NEXT:    v_lshl_or_b32 v43, v12, 8, v16
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
+; ALIGNED-NEXT:    v_lshl_or_b32 v57, v8, 8, v10
+; ALIGNED-NEXT:    v_lshl_or_b32 v104, v3, 16, v2
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v21, 8, v22
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v23, 8, v20
+; ALIGNED-NEXT:    v_lshl_or_b32 v76, v3, 16, v2
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v17, 8, v19
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v14, 8, v13
+; ALIGNED-NEXT:    v_lshl_or_b32 v101, v3, 16, v2
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v15, 8, v18
+; ALIGNED-NEXT:    v_lshl_or_b32 v84, v43, 16, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v43, v9, 8, v11
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v57, 16, v43
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshl_or_b32 v43, v5, 8, v6
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v57, v7, 8, v1
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v57, 16, v43
+; ALIGNED-NEXT:    buffer_load_ubyte v43, v4, s[0:3], 0 offen offset:1
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1336 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:1344 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x5
+; ALIGNED-NEXT:    buffer_load_ubyte v127, v4, s[0:3], 0 offen offset:2
+; ALIGNED-NEXT:    buffer_load_ubyte v57, v4, s[0:3], 0 offen offset:3
+; ALIGNED-NEXT:    buffer_load_ubyte v78, v4, s[0:3], 0 offen offset:4
+; ALIGNED-NEXT:    buffer_load_ubyte v90, v4, s[0:3], 0 offen offset:5
+; ALIGNED-NEXT:    buffer_load_ubyte v91, v4, s[0:3], 0 offen offset:6
+; ALIGNED-NEXT:    buffer_load_ubyte v124, v4, s[0:3], 0 offen offset:7
+; ALIGNED-NEXT:    v_lshl_or_b32 v43, v43, 8, v0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:1360 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v57, v57, 8, v127
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    buffer_store_dword v78, off, s[0:3], s32 offset:1364 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    buffer_store_dword v90, off, s[0:3], s32 offset:1376 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    buffer_store_dword v91, off, s[0:3], s32 offset:1380 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v57, 16, v43
+; ALIGNED-NEXT:    v_lshl_or_b32 v43, v90, 8, v78
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v57, v124, 8, v91
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1388 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v57, 16, v43
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1396 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x4
+; ALIGNED-NEXT:    buffer_load_ubyte v121, v4, s[0:3], 0 offen offset:12
+; ALIGNED-NEXT:    buffer_load_ubyte v107, v4, s[0:3], 0 offen offset:13
+; ALIGNED-NEXT:    buffer_load_ubyte v110, v4, s[0:3], 0 offen offset:14
+; ALIGNED-NEXT:    buffer_load_ubyte v108, v4, s[0:3], 0 offen offset:15
+; ALIGNED-NEXT:    buffer_load_ubyte v93, v4, s[0:3], 0 offen offset:11
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v43, v107, 8, v121
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v57, v108, 8, v110
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v57, 16, v43
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1404 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v95, v4, s[0:3], 0 offen offset:8
+; ALIGNED-NEXT:    buffer_load_ubyte v91, v4, s[0:3], 0 offen offset:9
+; ALIGNED-NEXT:    buffer_load_ubyte v90, v4, s[0:3], 0 offen offset:10
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_lshl_or_b32 v43, v91, 8, v95
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_lshl_or_b32 v57, v93, 8, v90
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v57, 16, v43
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1412 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v78, v4, s[0:3], 0 offen offset:16
+; ALIGNED-NEXT:    buffer_load_ubyte v43, v4, s[0:3], 0 offen offset:18
+; ALIGNED-NEXT:    buffer_load_ubyte v57, v4, s[0:3], 0 offen offset:17
+; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:488
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:492
+; ALIGNED-NEXT:    buffer_store_dword v84, off, s[0:3], s32 offset:484
+; ALIGNED-NEXT:    buffer_store_dword v101, off, s[0:3], s32 offset:480
+; ALIGNED-NEXT:    s_clause 0x1
+; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:704
+; ALIGNED-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:708
+; ALIGNED-NEXT:    v_add_nc_u32_e32 v4, 0xffffff00, v4
+; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v126, 8, v43
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshl_or_b32 v126, v57, 8, v78
+; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; ALIGNED-NEXT:    v_add_co_u32 v2, vcc_lo, v2, s4
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, s5, v3, vcc_lo
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v1 offset:250
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v7 offset:251
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v5 offset:249
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v8 offset:255
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v9 offset:253
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v10 offset:254
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v11 offset:252
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v6 offset:248
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v13 offset:242
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v14 offset:243
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v17 offset:241
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v12 offset:247
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v15 offset:245
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v16 offset:246
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v18 offset:244
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v19 offset:240
+; ALIGNED-NEXT:    buffer_store_dword v76, off, s[0:3], s32 offset:504
+; ALIGNED-NEXT:    buffer_store_dword v104, off, s[0:3], s32 offset:508
+; ALIGNED-NEXT:    buffer_store_dword v123, off, s[0:3], s32 offset:500
+; ALIGNED-NEXT:    v_lshl_or_b32 v126, v0, 16, v126
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1484 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_add_u32 s4, s4, 0xffffff00
+; ALIGNED-NEXT:    s_addc_u32 s5, s5, -1
+; ALIGNED-NEXT:    s_cmp_eq_u64 s[4:5], s[6:7]
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:496
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v20 offset:234
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v23 offset:235
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v21 offset:233
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v24 offset:239
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v25 offset:237
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v26 offset:238
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v27 offset:236
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v22 offset:232
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v29 offset:226
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v30 offset:227
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v33 offset:225
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v28 offset:231
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v31 offset:229
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v32 offset:230
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v34 offset:228
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v35 offset:224
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1480 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:448
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1476 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:460
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1472 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:456
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1468 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:452
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v54 offset:213
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v52 offset:215
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v36 offset:209
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v55 offset:211
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v37 offset:210
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v65 offset:214
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v67 offset:212
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v49 offset:218
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v48 offset:219
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v53 offset:217
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v39 offset:223
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v51 offset:221
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v50 offset:222
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v64 offset:220
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v66 offset:216
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v38 offset:208
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1464 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:472
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1460 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:476
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1456 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:468
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1452 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:464
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v68 offset:202
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v71 offset:203
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v69 offset:201
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v80 offset:207
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v81 offset:205
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v82 offset:206
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v83 offset:204
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v70 offset:200
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v87 offset:194
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v86 offset:195
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v98 offset:193
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v85 offset:199
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v97 offset:197
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v96 offset:198
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v99 offset:196
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v100 offset:192
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1448 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:552
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1444 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:556
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1440 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:548
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1436 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v102 offset:186
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v112 offset:187
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v103 offset:185
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v114 offset:191
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v115 offset:189
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v116 offset:190
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v117 offset:188
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v113 offset:184
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v119 offset:178
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v40 offset:179
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v44 offset:177
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v118 offset:183
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v41 offset:181
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v42 offset:182
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v45 offset:180
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v46 offset:176
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1432 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:568
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1428 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:572
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1424 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1420 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:560
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v47 offset:170
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v58 offset:171
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v56 offset:169
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v60 offset:175
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v61 offset:173
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v62 offset:174
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v63 offset:172
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v59 offset:168
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v73 offset:162
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v74 offset:163
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v79 offset:161
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v72 offset:167
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v75 offset:165
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v77 offset:166
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v88 offset:164
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v89 offset:160
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1408 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:520
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1400 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:524
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1392 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:516
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1384 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:512
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v92 offset:154
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v106 offset:155
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v94 offset:153
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v109 offset:159
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v111 offset:157
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v120 offset:158
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v122 offset:156
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v105 offset:152
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1356 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:146
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1348 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:147
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1340 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:145
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v125 offset:151
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1368 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:149
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1372 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:150
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1352 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:148
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1332 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:144
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1328 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:536
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1312 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:540
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1292 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1284 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:528
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1324 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:138
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1320 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:139
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1316 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:137
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1304 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:143
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1296 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:141
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1300 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:142
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1288 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:140
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1308 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:136
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1268 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:130
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1260 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:131
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1256 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:129
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1280 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:135
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1272 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:133
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1276 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:134
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1264 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:132
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1252 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:128
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1248 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1232 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1212 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1204 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:608
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1244 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:122
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1240 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:123
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1236 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:121
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1224 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:127
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1216 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:125
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1220 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:126
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1208 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:124
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1228 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:120
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:114
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1180 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:115
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1176 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:113
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1200 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:119
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:117
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1196 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:118
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1184 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:116
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1172 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:112
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1168 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:632
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1152 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:636
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1132 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1124 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:624
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1164 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:106
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1160 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:107
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1156 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:105
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1144 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:111
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1136 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:109
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1140 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:110
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1128 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:108
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1148 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:104
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1108 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:98
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1100 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:99
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1096 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:97
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1120 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:103
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:101
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1116 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:102
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1104 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:100
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1092 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:96
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1088 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1072 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:588
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1044 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1084 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:90
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1080 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:91
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1076 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:89
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1064 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:95
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1056 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:93
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1060 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:94
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:92
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1068 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:88
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:82
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1020 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:83
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1016 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:81
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:87
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:85
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:86
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1024 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:84
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1012 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:80
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1008 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:600
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:992 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:604
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:596
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:976 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:592
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1004 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:74
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1000 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:75
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:996 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:73
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:980 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:79
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:968 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:77
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:972 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:78
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:76
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:72
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:66
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:948 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:67
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:65
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:71
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:944 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:69
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:70
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:68
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:64
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:684
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:676
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:61
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:58
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:59
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:57
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:63
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:62
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:60
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:56
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:53
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:50
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:51
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:49
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:55
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:54
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:52
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:48
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:43
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:42
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:41
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:40
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:47
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:46
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:45
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:44
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:35
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:34
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:33
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:32
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:39
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:38
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:37
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:36
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:648
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:652
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:644
+; ALIGNED-NEXT:    buffer_store_dword v126, off, s[0:3], s32 offset:640
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:26
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:27
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:25
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:31
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:29
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:30
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:28
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:24
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v43 offset:18
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1416 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:19
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v57 offset:17
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:23
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:21
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:22
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:20
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v78 offset:16
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1412 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1404 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:668
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1396 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1388 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v90 offset:10
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v93 offset:11
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v107 offset:13
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v91 offset:9
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v108 offset:15
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v110 offset:14
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v121 offset:12
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v95 offset:8
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v127 offset:2
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1360 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:3
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1344 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:1
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v124 offset:7
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1376 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:5
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1380 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:6
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1364 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:4
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1336 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v0
+; ALIGNED-NEXT:    s_cbranch_scc0 .LBB9_4
+; ALIGNED-NEXT:  .LBB9_5: ; %Flow11
+; ALIGNED-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; ALIGNED-NEXT:    s_clause 0x2f
+; ALIGNED-NEXT:    buffer_load_dword v127, off, s[0:3], s32
+; ALIGNED-NEXT:    buffer_load_dword v126, off, s[0:3], s32 offset:4
+; ALIGNED-NEXT:    buffer_load_dword v125, off, s[0:3], s32 offset:8
+; ALIGNED-NEXT:    buffer_load_dword v124, off, s[0:3], s32 offset:12
+; ALIGNED-NEXT:    buffer_load_dword v123, off, s[0:3], s32 offset:16
+; ALIGNED-NEXT:    buffer_load_dword v122, off, s[0:3], s32 offset:20
+; ALIGNED-NEXT:    buffer_load_dword v121, off, s[0:3], s32 offset:24
+; ALIGNED-NEXT:    buffer_load_dword v120, off, s[0:3], s32 offset:28
+; ALIGNED-NEXT:    buffer_load_dword v111, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT:    buffer_load_dword v110, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT:    buffer_load_dword v109, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT:    buffer_load_dword v108, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT:    buffer_load_dword v107, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT:    buffer_load_dword v106, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT:    buffer_load_dword v105, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT:    buffer_load_dword v104, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT:    buffer_load_dword v95, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT:    buffer_load_dword v94, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT:    buffer_load_dword v93, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT:    buffer_load_dword v92, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT:    buffer_load_dword v91, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT:    buffer_load_dword v90, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT:    buffer_load_dword v89, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT:    buffer_load_dword v88, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT:    buffer_load_dword v79, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT:    buffer_load_dword v78, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT:    buffer_load_dword v77, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT:    buffer_load_dword v76, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT:    buffer_load_dword v75, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT:    buffer_load_dword v74, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT:    buffer_load_dword v73, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT:    buffer_load_dword v72, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; UNROLL3-LABEL: memmove_p0_p5_sz2048:
+; UNROLL3:       ; %bb.0: ; %entry
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; UNROLL3-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; UNROLL3-NEXT:    s_mov_b64 s[4:5], 0
+; UNROLL3-NEXT:    s_mov_b32 s6, exec_lo
+; UNROLL3-NEXT:    v_cndmask_b32_e32 v3, -1, v0, vcc_lo
+; UNROLL3-NEXT:    v_cmpx_ge_u32_e64 v2, v3
+; UNROLL3-NEXT:    s_xor_b32 s6, exec_lo, s6
+; UNROLL3-NEXT:    s_cbranch_execz .LBB9_4
+; UNROLL3-NEXT:  ; %bb.1: ; %memmove_fwd_loop.preheader
+; UNROLL3-NEXT:    v_mov_b32_e32 v3, v2
+; UNROLL3-NEXT:    s_inst_prefetch 0x1
+; UNROLL3-NEXT:    .p2align 6
+; UNROLL3-NEXT:  .LBB9_2: ; %memmove_fwd_loop
+; UNROLL3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT:    s_clause 0xb
+; UNROLL3-NEXT:    buffer_load_dword v4, v3, s[0:3], 0 offen
+; UNROLL3-NEXT:    buffer_load_dword v5, v3, s[0:3], 0 offen offset:4
+; UNROLL3-NEXT:    buffer_load_dword v6, v3, s[0:3], 0 offen offset:8
+; UNROLL3-NEXT:    buffer_load_dword v7, v3, s[0:3], 0 offen offset:12
+; UNROLL3-NEXT:    buffer_load_dword v8, v3, s[0:3], 0 offen offset:16
+; UNROLL3-NEXT:    buffer_load_dword v9, v3, s[0:3], 0 offen offset:20
+; UNROLL3-NEXT:    buffer_load_dword v10, v3, s[0:3], 0 offen offset:24
+; UNROLL3-NEXT:    buffer_load_dword v11, v3, s[0:3], 0 offen offset:28
+; UNROLL3-NEXT:    buffer_load_dword v12, v3, s[0:3], 0 offen offset:32
+; UNROLL3-NEXT:    buffer_load_dword v13, v3, s[0:3], 0 offen offset:36
+; UNROLL3-NEXT:    buffer_load_dword v14, v3, s[0:3], 0 offen offset:40
+; UNROLL3-NEXT:    buffer_load_dword v15, v3, s[0:3], 0 offen offset:44
+; UNROLL3-NEXT:    v_add_co_u32 v16, vcc_lo, v0, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, s5, v1, vcc_lo
+; UNROLL3-NEXT:    s_add_u32 s4, s4, 48
+; UNROLL3-NEXT:    v_add_nc_u32_e32 v3, 48, v3
+; UNROLL3-NEXT:    s_addc_u32 s5, s5, 0
+; UNROLL3-NEXT:    s_waitcnt vmcnt(4)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[8:11] offset:16
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[4:7]
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[16:17], v[12:15] offset:32
+; UNROLL3-NEXT:    s_cmp_lg_u64 s[4:5], 0x7e0
+; UNROLL3-NEXT:    s_cbranch_scc1 .LBB9_2
+; UNROLL3-NEXT:  ; %bb.3: ; %memmove_fwd_residual
+; UNROLL3-NEXT:    s_inst_prefetch 0x2
+; UNROLL3-NEXT:    s_clause 0x3
+; UNROLL3-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:2016
+; UNROLL3-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:2020
+; UNROLL3-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:2024
+; UNROLL3-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:2028
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:2016
+; UNROLL3-NEXT:    s_clause 0x3
+; UNROLL3-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:2032
+; UNROLL3-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:2036
+; UNROLL3-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:2040
+; UNROLL3-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:2044
+; UNROLL3-NEXT:    ; implicit-def: $vgpr2
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:2032
+; UNROLL3-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; UNROLL3-NEXT:  .LBB9_4: ; %Flow8
+; UNROLL3-NEXT:    s_andn2_saveexec_b32 s8, s6
+; UNROLL3-NEXT:    s_cbranch_execz .LBB9_7
+; UNROLL3-NEXT:  ; %bb.5: ; %memmove_bwd_residual
+; UNROLL3-NEXT:    s_clause 0x3
+; UNROLL3-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:2032
+; UNROLL3-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:2036
+; UNROLL3-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:2040
+; UNROLL3-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:2044
+; UNROLL3-NEXT:    s_movk_i32 s6, 0xffd0
+; UNROLL3-NEXT:    s_mov_b64 s[4:5], 0x7b0
+; UNROLL3-NEXT:    s_mov_b32 s7, -1
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:2032
+; UNROLL3-NEXT:    s_clause 0x3
+; UNROLL3-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:2016
+; UNROLL3-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:2020
+; UNROLL3-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:2024
+; UNROLL3-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:2028
+; UNROLL3-NEXT:    v_add_nc_u32_e32 v2, 0x7b0, v2
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:2016
+; UNROLL3-NEXT:    s_inst_prefetch 0x1
+; UNROLL3-NEXT:    .p2align 6
+; UNROLL3-NEXT:  .LBB9_6: ; %memmove_bwd_loop
+; UNROLL3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT:    s_clause 0xb
+; UNROLL3-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; UNROLL3-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; UNROLL3-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; UNROLL3-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; UNROLL3-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; UNROLL3-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; UNROLL3-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; UNROLL3-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:28
+; UNROLL3-NEXT:    buffer_load_dword v11, v2, s[0:3], 0 offen offset:32
+; UNROLL3-NEXT:    buffer_load_dword v12, v2, s[0:3], 0 offen offset:36
+; UNROLL3-NEXT:    buffer_load_dword v13, v2, s[0:3], 0 offen offset:40
+; UNROLL3-NEXT:    buffer_load_dword v14, v2, s[0:3], 0 offen offset:44
+; UNROLL3-NEXT:    v_add_co_u32 v15, vcc_lo, v0, s4
+; UNROLL3-NEXT:    v_add_co_ci_u32_e32 v16, vcc_lo, s5, v1, vcc_lo
+; UNROLL3-NEXT:    v_subrev_nc_u32_e32 v2, 48, v2
+; UNROLL3-NEXT:    s_add_u32 s4, s4, 0xffffffd0
+; UNROLL3-NEXT:    s_addc_u32 s5, s5, -1
+; UNROLL3-NEXT:    s_waitcnt vmcnt(4)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[15:16], v[7:10] offset:16
+; UNROLL3-NEXT:    flat_store_dwordx4 v[15:16], v[3:6]
+; UNROLL3-NEXT:    s_waitcnt vmcnt(0)
+; UNROLL3-NEXT:    flat_store_dwordx4 v[15:16], v[11:14] offset:32
+; UNROLL3-NEXT:    s_cmp_eq_u64 s[4:5], s[6:7]
+; UNROLL3-NEXT:    s_cbranch_scc0 .LBB9_6
+; UNROLL3-NEXT:  .LBB9_7: ; %Flow9
+; UNROLL3-NEXT:    s_inst_prefetch 0x2
+; UNROLL3-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; UNROLL3-NEXT:    s_waitcnt lgkmcnt(0)
+; UNROLL3-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  tail call void @llvm.memmove.p0.p5.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 2048, i1 false)
+  ret void
+}
+
+
+declare void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) noalias nocapture writeonly, ptr addrspace(0) noalias nocapture readonly, i64, i1 immarg) #2
+declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noalias nocapture writeonly, ptr addrspace(1) noalias nocapture readonly, i64, i1 immarg) #2
+declare void @llvm.memcpy.p0.p4.i64(ptr addrspace(0) noalias nocapture writeonly, ptr addrspace(4) noalias nocapture readonly, i64, i1 immarg) #2
+declare void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noalias nocapture writeonly, ptr addrspace(5) noalias nocapture readonly, i64, i1 immarg) #2
+
+declare void @llvm.memcpy.p0.p5.i64(ptr addrspace(0) noalias nocapture writeonly, ptr addrspace(5) noalias nocapture readonly, i64, i1 immarg) #2
+
+declare void @llvm.memmove.p0.p0.i64(ptr addrspace(0) nocapture writeonly, ptr addrspace(0) nocapture readonly, i64, i1 immarg) #2
+declare void @llvm.memmove.p1.p1.i64(ptr addrspace(1) nocapture writeonly, ptr addrspace(1) nocapture readonly, i64, i1 immarg) #2
+declare void @llvm.memmove.p0.p4.i64(ptr addrspace(0) nocapture writeonly, ptr addrspace(4) nocapture readonly, i64, i1 immarg) #2
+declare void @llvm.memmove.p5.p5.i64(ptr addrspace(5) nocapture writeonly, ptr addrspace(5) nocapture readonly, i64, i1 immarg) #2
+
+declare void @llvm.memmove.p0.p5.i64(ptr addrspace(0) nocapture writeonly, ptr addrspace(5) nocapture readonly, i64, i1 immarg) #2
+
+attributes #2 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
-- 
GitLab


From 86f76c3b171f95fd0560339f2ad0f4449277cf8d Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles@arm.com>
Date: Mon, 28 Oct 2024 08:22:38 +0000
Subject: [PATCH 285/693] [AArch64][Libunwind] Add Support for FEAT_PAuthLR
 DWARF Instruction (#112171)

As part of FEAT_PAuthLR, a new DWARF Frame Instruction was introduced,
`DW_CFA_AARCH64_negate_ra_state_with_pc`. This instructs Libunwind that
the PC has been used with the signing instruction. This change includes
three commits
- Libunwind support for the newly introduced DWARF Instruction
- CodeGen Support for the DWARF Instructions
- Reversing the changes made in #96377. Due to
`DW_CFA_AARCH64_negate_ra_state_with_pc`'s requirements to be placed
immediately after the signing instruction, this would mean the CFI
Instruction location was not consistent with the generated location when
not using FEAT_PAuthLR. The commit reverses the changes and makes the
location consistent across the different branch protection options.
While this does have a code size effect, this is a negligible one.

For the ABI information, see here:
https://github.com/ARM-software/abi-aa/blob/853286c7ab66048e4b819682ce17f567b77a0291/aadwarf64/aadwarf64.rst#id23
---
 libunwind/src/DwarfInstructions.hpp           |  59 ++++--
 libunwind/src/DwarfParser.hpp                 |  21 +++
 libunwind/src/dwarf2.h                        |  58 +++---
 llvm/include/llvm/BinaryFormat/Dwarf.def      |   1 +
 llvm/include/llvm/MC/MCDwarf.h                |   7 +
 llvm/include/llvm/MC/MCStreamer.h             |   1 +
 .../CodeGen/AsmPrinter/AsmPrinterDwarf.cpp    |   3 +
 llvm/lib/CodeGen/CFIInstrInserter.cpp         |   1 +
 llvm/lib/CodeGen/MIRParser/MILexer.cpp        |   2 +
 llvm/lib/CodeGen/MIRParser/MILexer.h          |   1 +
 llvm/lib/CodeGen/MIRParser/MIParser.cpp       |   5 +
 llvm/lib/CodeGen/MachineOperand.cpp           |   4 +
 llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp  |  24 +++
 llvm/lib/MC/MCAsmStreamer.cpp                 |   7 +
 llvm/lib/MC/MCDwarf.cpp                       |   4 +
 llvm/lib/MC/MCStreamer.cpp                    |  10 ++
 .../Target/AArch64/AArch64FrameLowering.cpp   |   5 +-
 .../lib/Target/AArch64/AArch64PointerAuth.cpp |  74 ++++----
 .../AArch64/AsmParser/AArch64AsmParser.cpp    |  10 ++
 .../machine-outliner-retaddr-sign-cfi.ll      |   3 +-
 ...tliner-retaddr-sign-diff-scope-same-key.ll |   6 +-
 .../machine-outliner-retaddr-sign-non-leaf.ll | 121 +++++++++++--
 .../machine-outliner-retaddr-sign-regsave.mir |   3 +-
 ...tliner-retaddr-sign-same-scope-diff-key.ll | 139 +++++++++++---
 ...machine-outliner-retaddr-sign-subtarget.ll |   9 +-
 .../machine-outliner-retaddr-sign-thunk.ll    | 169 ++++++++++++++----
 .../AArch64/pacbti-llvm-generated-funcs-2.ll  |   3 +-
 ...sign-return-address-cfi-negate-ra-state.ll |  11 +-
 .../AArch64/sign-return-address-pauth-lr.ll   | 162 +++++++++++------
 .../CodeGen/AArch64/sign-return-address.ll    |  18 +-
 .../MIR/AArch64/return-address-signing.mir    |  23 +++
 .../MC/AArch64/directives-case_insensitive.s  |   2 +
 .../test/MC/AArch64/negate_ra_state_with_pc.s |   7 +
 .../DebugInfo/DWARF/DWARFDebugFrameTest.cpp   |   1 +
 34 files changed, 733 insertions(+), 241 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/negate_ra_state_with_pc.s

diff --git a/libunwind/src/DwarfInstructions.hpp b/libunwind/src/DwarfInstructions.hpp
index bd9ece60ee58..e7be0d6d5d63 100644
--- a/libunwind/src/DwarfInstructions.hpp
+++ b/libunwind/src/DwarfInstructions.hpp
@@ -74,8 +74,10 @@ private:
     __builtin_unreachable();
   }
 #if defined(_LIBUNWIND_TARGET_AARCH64)
-  static bool getRA_SIGN_STATE(A &addressSpace, R registers, pint_t cfa,
-                               PrologInfo &prolog);
+  static bool isReturnAddressSigned(A &addressSpace, R registers, pint_t cfa,
+                                    PrologInfo &prolog);
+  static bool isReturnAddressSignedWithPC(A &addressSpace, R registers,
+                                          pint_t cfa, PrologInfo &prolog);
 #endif
 };
 
@@ -173,8 +175,9 @@ v128 DwarfInstructions<A, R>::getSavedVectorRegister(
 }
 #if defined(_LIBUNWIND_TARGET_AARCH64)
 template <typename A, typename R>
-bool DwarfInstructions<A, R>::getRA_SIGN_STATE(A &addressSpace, R registers,
-                                               pint_t cfa, PrologInfo &prolog) {
+bool DwarfInstructions<A, R>::isReturnAddressSigned(A &addressSpace,
+                                                    R registers, pint_t cfa,
+                                                    PrologInfo &prolog) {
   pint_t raSignState;
   auto regloc = prolog.savedRegisters[UNW_AARCH64_RA_SIGN_STATE];
   if (regloc.location == CFI_Parser<A>::kRegisterUnused)
@@ -185,6 +188,22 @@ bool DwarfInstructions<A, R>::getRA_SIGN_STATE(A &addressSpace, R registers,
   // Only bit[0] is meaningful.
   return raSignState & 0x01;
 }
+
+template <typename A, typename R>
+bool DwarfInstructions<A, R>::isReturnAddressSignedWithPC(A &addressSpace,
+                                                          R registers,
+                                                          pint_t cfa,
+                                                          PrologInfo &prolog) {
+  pint_t raSignState;
+  auto regloc = prolog.savedRegisters[UNW_AARCH64_RA_SIGN_STATE];
+  if (regloc.location == CFI_Parser<A>::kRegisterUnused)
+    raSignState = static_cast<pint_t>(regloc.value);
+  else
+    raSignState = getSavedRegister(addressSpace, registers, cfa, regloc);
+
+  // Only bit[1] is meaningful.
+  return raSignState & 0x02;
+}
 #endif
 
 template <typename A, typename R>
@@ -288,7 +307,7 @@ int DwarfInstructions<A, R>::stepWithDwarf(A &addressSpace, pint_t pc,
       // restored. autia1716 is used instead of autia as autia1716 assembles
       // to a NOP on pre-v8.3a architectures.
       if ((R::getArch() == REGISTERS_ARM64) &&
-          getRA_SIGN_STATE(addressSpace, registers, cfa, prolog) &&
+          isReturnAddressSigned(addressSpace, registers, cfa, prolog) &&
           returnAddress != 0) {
 #if !defined(_LIBUNWIND_IS_NATIVE_ONLY)
         return UNW_ECROSSRASIGNING;
@@ -296,13 +315,29 @@ int DwarfInstructions<A, R>::stepWithDwarf(A &addressSpace, pint_t pc,
         register unsigned long long x17 __asm("x17") = returnAddress;
         register unsigned long long x16 __asm("x16") = cfa;
 
-        // These are the autia1716/autib1716 instructions. The hint instructions
-        // are used here as gcc does not assemble autia1716/autib1716 for pre
-        // armv8.3a targets.
-        if (cieInfo.addressesSignedWithBKey)
-          asm("hint 0xe" : "+r"(x17) : "r"(x16)); // autib1716
-        else
-          asm("hint 0xc" : "+r"(x17) : "r"(x16)); // autia1716
+        // We use the hint versions of the authentication instructions below to
+        // ensure they're assembled by the compiler even for targets with no
+        // FEAT_PAuth/FEAT_PAuth_LR support.
+        if (isReturnAddressSignedWithPC(addressSpace, registers, cfa, prolog)) {
+          register unsigned long long x15 __asm("x15") =
+              prolog.ptrAuthDiversifier;
+          if (cieInfo.addressesSignedWithBKey) {
+            asm("hint 0x27\n\t" // pacm
+                "hint 0xe"
+                : "+r"(x17)
+                : "r"(x16), "r"(x15)); // autib1716
+          } else {
+            asm("hint 0x27\n\t" // pacm
+                "hint 0xc"
+                : "+r"(x17)
+                : "r"(x16), "r"(x15)); // autia1716
+          }
+        } else {
+          if (cieInfo.addressesSignedWithBKey)
+            asm("hint 0xe" : "+r"(x17) : "r"(x16)); // autib1716
+          else
+            asm("hint 0xc" : "+r"(x17) : "r"(x16)); // autia1716
+        }
         returnAddress = x17;
 #endif
       }
diff --git a/libunwind/src/DwarfParser.hpp b/libunwind/src/DwarfParser.hpp
index 0682942ce137..7e85025dd054 100644
--- a/libunwind/src/DwarfParser.hpp
+++ b/libunwind/src/DwarfParser.hpp
@@ -91,6 +91,9 @@ public:
     int64_t           cfaExpression;      // CFA = expression
     uint32_t          spExtraArgSize;
     RegisterLocation  savedRegisters[kMaxRegisterNumber + 1];
+#if defined(_LIBUNWIND_TARGET_AARCH64)
+    pint_t ptrAuthDiversifier;
+#endif
     enum class InitializeTime { kLazy, kNormal };
 
     // When saving registers, this data structure is lazily initialized.
@@ -799,6 +802,24 @@ bool CFI_Parser<A>::parseFDEInstructions(A &addressSpace,
         }
         break;
 
+#if defined(_LIBUNWIND_TARGET_AARCH64)
+      case DW_CFA_AARCH64_negate_ra_state_with_pc: {
+        int64_t value =
+            results->savedRegisters[UNW_AARCH64_RA_SIGN_STATE].value ^ 0x3;
+        results->setRegisterValue(UNW_AARCH64_RA_SIGN_STATE, value,
+                                  initialState);
+        // When calculating the value of the PC, it is assumed that the CFI
+        // instruction is placed before the signing instruction, however it is
+        // placed after. Because of this, we need to take into account the CFI
+        // instruction is one instruction call later than expected, and reduce
+        // the PC value by 4 bytes to compensate.
+        results->ptrAuthDiversifier = fdeInfo.pcStart + codeOffset - 0x4;
+        _LIBUNWIND_TRACE_DWARF(
+            "DW_CFA_AARCH64_negate_ra_state_with_pc(pc=0x%" PRIx64 ")\n",
+            static_cast<uint64_t>(results->ptrAuthDiversifier));
+      } break;
+#endif
+
 #else
         (void)arch;
 #endif
diff --git a/libunwind/src/dwarf2.h b/libunwind/src/dwarf2.h
index 174277d5a795..68ad88234720 100644
--- a/libunwind/src/dwarf2.h
+++ b/libunwind/src/dwarf2.h
@@ -18,43 +18,43 @@
 
 // DWARF unwind instructions
 enum {
-  DW_CFA_nop                 = 0x0,
-  DW_CFA_set_loc             = 0x1,
-  DW_CFA_advance_loc1        = 0x2,
-  DW_CFA_advance_loc2        = 0x3,
-  DW_CFA_advance_loc4        = 0x4,
-  DW_CFA_offset_extended     = 0x5,
-  DW_CFA_restore_extended    = 0x6,
-  DW_CFA_undefined           = 0x7,
-  DW_CFA_same_value          = 0x8,
-  DW_CFA_register            = 0x9,
-  DW_CFA_remember_state      = 0xA,
-  DW_CFA_restore_state       = 0xB,
-  DW_CFA_def_cfa             = 0xC,
-  DW_CFA_def_cfa_register    = 0xD,
-  DW_CFA_def_cfa_offset      = 0xE,
-  DW_CFA_def_cfa_expression  = 0xF,
-  DW_CFA_expression         = 0x10,
+  DW_CFA_nop = 0x0,
+  DW_CFA_set_loc = 0x1,
+  DW_CFA_advance_loc1 = 0x2,
+  DW_CFA_advance_loc2 = 0x3,
+  DW_CFA_advance_loc4 = 0x4,
+  DW_CFA_offset_extended = 0x5,
+  DW_CFA_restore_extended = 0x6,
+  DW_CFA_undefined = 0x7,
+  DW_CFA_same_value = 0x8,
+  DW_CFA_register = 0x9,
+  DW_CFA_remember_state = 0xA,
+  DW_CFA_restore_state = 0xB,
+  DW_CFA_def_cfa = 0xC,
+  DW_CFA_def_cfa_register = 0xD,
+  DW_CFA_def_cfa_offset = 0xE,
+  DW_CFA_def_cfa_expression = 0xF,
+  DW_CFA_expression = 0x10,
   DW_CFA_offset_extended_sf = 0x11,
-  DW_CFA_def_cfa_sf         = 0x12,
-  DW_CFA_def_cfa_offset_sf  = 0x13,
-  DW_CFA_val_offset         = 0x14,
-  DW_CFA_val_offset_sf      = 0x15,
-  DW_CFA_val_expression     = 0x16,
-  DW_CFA_advance_loc        = 0x40, // high 2 bits are 0x1, lower 6 bits are delta
-  DW_CFA_offset             = 0x80, // high 2 bits are 0x2, lower 6 bits are register
-  DW_CFA_restore            = 0xC0, // high 2 bits are 0x3, lower 6 bits are register
+  DW_CFA_def_cfa_sf = 0x12,
+  DW_CFA_def_cfa_offset_sf = 0x13,
+  DW_CFA_val_offset = 0x14,
+  DW_CFA_val_offset_sf = 0x15,
+  DW_CFA_val_expression = 0x16,
+  DW_CFA_advance_loc = 0x40, // high 2 bits are 0x1, lower 6 bits are delta
+  DW_CFA_offset = 0x80,      // high 2 bits are 0x2, lower 6 bits are register
+  DW_CFA_restore = 0xC0,     // high 2 bits are 0x3, lower 6 bits are register
 
   // GNU extensions
-  DW_CFA_GNU_window_save              = 0x2D,
-  DW_CFA_GNU_args_size                = 0x2E,
+  DW_CFA_GNU_window_save = 0x2D,
+  DW_CFA_GNU_args_size = 0x2E,
   DW_CFA_GNU_negative_offset_extended = 0x2F,
 
   // AARCH64 extensions
-  DW_CFA_AARCH64_negate_ra_state      = 0x2D
+  DW_CFA_AARCH64_negate_ra_state_with_pc = 0x2C,
+  DW_CFA_AARCH64_negate_ra_state = 0x2D
 };
 
-
 // FSF exception handling Pointer-Encoding constants
 // Used in CFI augmentation by GCC
 enum {
diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.def b/llvm/include/llvm/BinaryFormat/Dwarf.def
index d55947fc5103..9336f2a454ae 100644
--- a/llvm/include/llvm/BinaryFormat/Dwarf.def
+++ b/llvm/include/llvm/BinaryFormat/Dwarf.def
@@ -1238,6 +1238,7 @@ HANDLE_DW_CFA(0x16, val_expression)
 // Vendor extensions:
 HANDLE_DW_CFA_PRED(0x1d, MIPS_advance_loc8, SELECT_MIPS64)
 HANDLE_DW_CFA_PRED(0x2d, GNU_window_save, SELECT_SPARC)
+HANDLE_DW_CFA_PRED(0x2c, AARCH64_negate_ra_state_with_pc, SELECT_AARCH64)
 HANDLE_DW_CFA_PRED(0x2d, AARCH64_negate_ra_state, SELECT_AARCH64)
 HANDLE_DW_CFA_PRED(0x2e, GNU_args_size, SELECT_X86)
 // Heterogeneous Debugging Extension defined at
diff --git a/llvm/include/llvm/MC/MCDwarf.h b/llvm/include/llvm/MC/MCDwarf.h
index bea79545d1ab..1392336968e7 100644
--- a/llvm/include/llvm/MC/MCDwarf.h
+++ b/llvm/include/llvm/MC/MCDwarf.h
@@ -515,6 +515,7 @@ public:
     OpRegister,
     OpWindowSave,
     OpNegateRAState,
+    OpNegateRAStateWithPC,
     OpGnuArgsSize,
     OpLabel,
   };
@@ -642,6 +643,12 @@ public:
     return MCCFIInstruction(OpNegateRAState, L, 0, INT64_C(0), Loc);
   }
 
+  /// .cfi_negate_ra_state_with_pc AArch64 negate RA state with PC.
+  static MCCFIInstruction createNegateRAStateWithPC(MCSymbol *L,
+                                                    SMLoc Loc = {}) {
+    return MCCFIInstruction(OpNegateRAStateWithPC, L, 0, INT64_C(0), Loc);
+  }
+
   /// .cfi_restore says that the rule for Register is now the same as it
   /// was at the beginning of the function, after all initial instructions added
   /// by .cfi_startproc were executed.
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index 707aecc5dc57..a376ba810ba5 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -1022,6 +1022,7 @@ public:
                                SMLoc Loc = {});
   virtual void emitCFIWindowSave(SMLoc Loc = {});
   virtual void emitCFINegateRAState(SMLoc Loc = {});
+  virtual void emitCFINegateRAStateWithPC(SMLoc Loc = {});
   virtual void emitCFILabelDirective(SMLoc Loc, StringRef Name);
 
   virtual void emitWinCFIStartProc(const MCSymbol *Symbol, SMLoc Loc = SMLoc());
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
index 21d0d070c247..daad82d26da6 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
@@ -236,6 +236,9 @@ void AsmPrinter::emitCFIInstruction(const MCCFIInstruction &Inst) const {
   case MCCFIInstruction::OpNegateRAState:
     OutStreamer->emitCFINegateRAState(Loc);
     break;
+  case MCCFIInstruction::OpNegateRAStateWithPC:
+    OutStreamer->emitCFINegateRAStateWithPC(Loc);
+    break;
   case MCCFIInstruction::OpSameValue:
     OutStreamer->emitCFISameValue(Inst.getRegister(), Loc);
     break;
diff --git a/llvm/lib/CodeGen/CFIInstrInserter.cpp b/llvm/lib/CodeGen/CFIInstrInserter.cpp
index f5bedc7b8ecd..4217ec6a1cca 100644
--- a/llvm/lib/CodeGen/CFIInstrInserter.cpp
+++ b/llvm/lib/CodeGen/CFIInstrInserter.cpp
@@ -260,6 +260,7 @@ void CFIInstrInserter::calculateOutgoingCFAInfo(MBBCFAInfo &MBBInfo) {
       case MCCFIInstruction::OpEscape:
       case MCCFIInstruction::OpWindowSave:
       case MCCFIInstruction::OpNegateRAState:
+      case MCCFIInstruction::OpNegateRAStateWithPC:
       case MCCFIInstruction::OpGnuArgsSize:
       case MCCFIInstruction::OpLabel:
         break;
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.cpp b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
index 0809f88fde56..5a3806ce5733 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
@@ -238,6 +238,8 @@ static MIToken::TokenKind getIdentifierKind(StringRef Identifier) {
       .Case("window_save", MIToken::kw_cfi_window_save)
       .Case("negate_ra_sign_state",
             MIToken::kw_cfi_aarch64_negate_ra_sign_state)
+      .Case("negate_ra_sign_state_with_pc",
+            MIToken::kw_cfi_aarch64_negate_ra_sign_state_with_pc)
       .Case("blockaddress", MIToken::kw_blockaddress)
       .Case("intrinsic", MIToken::kw_intrinsic)
       .Case("target-index", MIToken::kw_target_index)
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.h b/llvm/lib/CodeGen/MIRParser/MILexer.h
index 22547483a8a8..3931da3eaae1 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.h
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.h
@@ -96,6 +96,7 @@ struct MIToken {
     kw_cfi_undefined,
     kw_cfi_window_save,
     kw_cfi_aarch64_negate_ra_sign_state,
+    kw_cfi_aarch64_negate_ra_sign_state_with_pc,
     kw_blockaddress,
     kw_intrinsic,
     kw_target_index,
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index 7aaa0f409d5e..45847b5830da 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -2576,6 +2576,10 @@ bool MIParser::parseCFIOperand(MachineOperand &Dest) {
   case MIToken::kw_cfi_aarch64_negate_ra_sign_state:
     CFIIndex = MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
     break;
+  case MIToken::kw_cfi_aarch64_negate_ra_sign_state_with_pc:
+    CFIIndex =
+        MF.addFrameInst(MCCFIInstruction::createNegateRAStateWithPC(nullptr));
+    break;
   case MIToken::kw_cfi_escape: {
     std::string Values;
     if (parseCFIEscapeValues(Values))
@@ -2931,6 +2935,7 @@ bool MIParser::parseMachineOperand(const unsigned OpCode, const unsigned OpIdx,
   case MIToken::kw_cfi_undefined:
   case MIToken::kw_cfi_window_save:
   case MIToken::kw_cfi_aarch64_negate_ra_sign_state:
+  case MIToken::kw_cfi_aarch64_negate_ra_sign_state_with_pc:
     return parseCFIOperand(Dest);
   case MIToken::kw_blockaddress:
     return parseBlockAddressOperand(Dest);
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index c0e004555de9..620779e2dec7 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -768,6 +768,10 @@ static void printCFI(raw_ostream &OS, const MCCFIInstruction &CFI,
     if (MCSymbol *Label = CFI.getLabel())
       MachineOperand::printSymbol(OS, *Label);
     break;
+  case MCCFIInstruction::OpNegateRAStateWithPC:
+    OS << "negate_ra_sign_state_with_pc ";
+    if (MCSymbol *Label = CFI.getLabel())
+      MachineOperand::printSymbol(OS, *Label);
   default:
     // TODO: Print the other CFI Operations.
     OS << "<unserializable cfi directive>";
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
index aff26824dda1..96cb86ad4c37 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
@@ -288,6 +288,7 @@ Error CFIProgram::parse(DWARFDataExtractor Data, uint64_t *Offset,
     case DW_CFA_remember_state:
     case DW_CFA_restore_state:
     case DW_CFA_GNU_window_save:
+    case DW_CFA_AARCH64_negate_ra_state_with_pc:
       // No operands
       addInstruction(Opcode);
       break;
@@ -666,6 +667,28 @@ Error UnwindTable::parseRows(const CFIProgram &CFIP, UnwindRow &Row,
       }
       break;
 
+    case dwarf::DW_CFA_AARCH64_negate_ra_state_with_pc: {
+      constexpr uint32_t AArch64DWARFPAuthRaState = 34;
+      auto LRLoc = Row.getRegisterLocations().getRegisterLocation(
+          AArch64DWARFPAuthRaState);
+      if (LRLoc) {
+        if (LRLoc->getLocation() == UnwindLocation::Constant) {
+          // Toggle the constant value of bits[1:0] from 0 to 1 or 1 to 0.
+          LRLoc->setConstant(LRLoc->getConstant() ^ 0x3);
+        } else {
+          return createStringError(
+              errc::invalid_argument,
+              "%s encountered when existing rule for this register is not "
+              "a constant",
+              CFIP.callFrameString(Inst.Opcode).str().c_str());
+        }
+      } else {
+        Row.getRegisterLocations().setRegisterLocation(
+            AArch64DWARFPAuthRaState, UnwindLocation::createIsConstant(0x3));
+      }
+      break;
+    }
+
     case dwarf::DW_CFA_undefined: {
       llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
       if (!RegNum)
@@ -847,6 +870,7 @@ CFIProgram::getOperandTypes() {
   DECLARE_OP0(DW_CFA_remember_state);
   DECLARE_OP0(DW_CFA_restore_state);
   DECLARE_OP0(DW_CFA_GNU_window_save);
+  DECLARE_OP0(DW_CFA_AARCH64_negate_ra_state_with_pc);
   DECLARE_OP1(DW_CFA_GNU_args_size, OT_Offset);
   DECLARE_OP0(DW_CFA_nop);
 
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 31b519a3e5c5..b9ad0b4eac9c 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -373,6 +373,7 @@ public:
                        SMLoc Loc) override;
   void emitCFIWindowSave(SMLoc Loc) override;
   void emitCFINegateRAState(SMLoc Loc) override;
+  void emitCFINegateRAStateWithPC(SMLoc Loc) override;
   void emitCFIReturnColumn(int64_t Register) override;
   void emitCFILabelDirective(SMLoc Loc, StringRef Name) override;
 
@@ -2145,6 +2146,12 @@ void MCAsmStreamer::emitCFINegateRAState(SMLoc Loc) {
   EmitEOL();
 }
 
+void MCAsmStreamer::emitCFINegateRAStateWithPC(SMLoc Loc) {
+  MCStreamer::emitCFINegateRAStateWithPC(Loc);
+  OS << "\t.cfi_negate_ra_state_with_pc";
+  EmitEOL();
+}
+
 void MCAsmStreamer::emitCFIReturnColumn(int64_t Register) {
   MCStreamer::emitCFIReturnColumn(Register);
   OS << "\t.cfi_return_column ";
diff --git a/llvm/lib/MC/MCDwarf.cpp b/llvm/lib/MC/MCDwarf.cpp
index 8ff097f29aeb..e058358fb8ad 100644
--- a/llvm/lib/MC/MCDwarf.cpp
+++ b/llvm/lib/MC/MCDwarf.cpp
@@ -1381,6 +1381,10 @@ void FrameEmitterImpl::emitCFIInstruction(const MCCFIInstruction &Instr) {
     Streamer.emitInt8(dwarf::DW_CFA_AARCH64_negate_ra_state);
     return;
 
+  case MCCFIInstruction::OpNegateRAStateWithPC:
+    Streamer.emitInt8(dwarf::DW_CFA_AARCH64_negate_ra_state_with_pc);
+    return;
+
   case MCCFIInstruction::OpUndefined: {
     unsigned Reg = Instr.getRegister();
     Streamer.emitInt8(dwarf::DW_CFA_undefined);
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index 13b162768578..5474db1315f1 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -688,6 +688,16 @@ void MCStreamer::emitCFINegateRAState(SMLoc Loc) {
   CurFrame->Instructions.push_back(Instruction);
 }
 
+void MCStreamer::emitCFINegateRAStateWithPC(SMLoc Loc) {
+  MCSymbol *Label = emitCFILabel();
+  MCCFIInstruction Instruction =
+      MCCFIInstruction::createNegateRAStateWithPC(Label, Loc);
+  MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
+  if (!CurFrame)
+    return;
+  CurFrame->Instructions.push_back(Instruction);
+}
+
 void MCStreamer::emitCFIReturnColumn(int64_t Register) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index bbf2f2677954..275070b332ac 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -702,7 +702,10 @@ void AArch64FrameLowering::resetCFIToInitialState(
 
   // Flip the RA sign state.
   if (MFI.shouldSignReturnAddress(MF)) {
-    CFIIndex = MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
+    auto CFIInst = MFI.branchProtectionPAuthLR()
+                       ? MCCFIInstruction::createNegateRAStateWithPC(nullptr)
+                       : MCCFIInstruction::createNegateRAState(nullptr);
+    CFIIndex = MF.addFrameInst(CFIInst);
     BuildMI(MBB, InsertPt, DL, CFIDesc).addCFIIndex(CFIIndex);
   }
 
diff --git a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
index 92ab4b5c3d25..2e5688cf6002 100644
--- a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
@@ -71,6 +71,18 @@ FunctionPass *llvm::createAArch64PointerAuthPass() {
 
 char AArch64PointerAuth::ID = 0;
 
+static void emitPACSymOffsetIntoX16(const TargetInstrInfo &TII,
+                                    MachineBasicBlock &MBB,
+                                    MachineBasicBlock::iterator I, DebugLoc DL,
+                                    MCSymbol *PACSym) {
+  BuildMI(MBB, I, DL, TII.get(AArch64::ADRP), AArch64::X16)
+      .addSym(PACSym, AArch64II::MO_PAGE);
+  BuildMI(MBB, I, DL, TII.get(AArch64::ADDXri), AArch64::X16)
+      .addReg(AArch64::X16)
+      .addSym(PACSym, AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
+      .addImm(0);
+}
+
 // Where PAuthLR support is not known at compile time, it is supported using
 // PACM. PACM is in the hint space so has no effect when PAuthLR is not
 // supported by the hardware, but will alter the behaviour of PACI*SP, AUTI*SP
@@ -81,12 +93,10 @@ static void BuildPACM(const AArch64Subtarget &Subtarget, MachineBasicBlock &MBB,
   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
   auto &MFnI = *MBB.getParent()->getInfo<AArch64FunctionInfo>();
 
-  // ADR X16,<address_of_PACIASP>
+  // Offset to PAC*SP using ADRP + ADD.
   if (PACSym) {
     assert(Flags == MachineInstr::FrameDestroy);
-    BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADR))
-        .addReg(AArch64::X16, RegState::Define)
-        .addSym(PACSym);
+    emitPACSymOffsetIntoX16(*TII, MBB, MBBI, DL, PACSym);
   }
 
   // Only emit PACM if -mbranch-protection has +pc and the target does not
@@ -95,12 +105,31 @@ static void BuildPACM(const AArch64Subtarget &Subtarget, MachineBasicBlock &MBB,
     BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACM)).setMIFlag(Flags);
 }
 
+static void emitPACCFI(const AArch64Subtarget &Subtarget,
+                       MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+                       DebugLoc DL, MachineInstr::MIFlag Flags, bool EmitCFI) {
+  if (!EmitCFI)
+    return;
+
+  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
+  auto &MF = *MBB.getParent();
+  auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
+
+  auto CFIInst = MFnI.branchProtectionPAuthLR()
+                     ? MCCFIInstruction::createNegateRAStateWithPC(nullptr)
+                     : MCCFIInstruction::createNegateRAState(nullptr);
+
+  unsigned CFIIndex = MF.addFrameInst(CFIInst);
+  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+      .addCFIIndex(CFIIndex)
+      .setMIFlags(Flags);
+}
+
 void AArch64PointerAuth::signLR(MachineFunction &MF,
                                 MachineBasicBlock::iterator MBBI) const {
   auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
   bool UseBKey = MFnI.shouldSignWithBKey();
   bool EmitCFI = MFnI.needsDwarfUnwindInfo(MF);
-  bool EmitAsyncCFI = MFnI.needsAsyncDwarfUnwindInfo(MF);
   bool NeedsWinCFI = MF.hasWinCFI();
 
   MachineBasicBlock &MBB = *MBBI->getParent();
@@ -128,6 +157,7 @@ void AArch64PointerAuth::signLR(MachineFunction &MF,
                                                : AArch64::PACIASPPC))
         .setMIFlag(MachineInstr::FrameSetup)
         ->setPreInstrSymbol(MF, MFnI.getSigningInstrLabel());
+    emitPACCFI(*Subtarget, MBB, MBBI, DL, MachineInstr::FrameSetup, EmitCFI);
   } else {
     BuildPACM(*Subtarget, MBB, MBBI, DL, MachineInstr::FrameSetup);
     BuildMI(MBB, MBBI, DL,
@@ -135,27 +165,10 @@ void AArch64PointerAuth::signLR(MachineFunction &MF,
                                                : AArch64::PACIASP))
         .setMIFlag(MachineInstr::FrameSetup)
         ->setPreInstrSymbol(MF, MFnI.getSigningInstrLabel());
+    emitPACCFI(*Subtarget, MBB, MBBI, DL, MachineInstr::FrameSetup, EmitCFI);
   }
 
-  if (EmitCFI) {
-    if (!EmitAsyncCFI) {
-      // Reduce the size of the generated call frame information for synchronous
-      // CFI by bundling the new CFI instruction with others in the prolog, so
-      // that no additional DW_CFA_advance_loc is needed.
-      for (auto I = MBBI; I != MBB.end(); ++I) {
-        if (I->getOpcode() == TargetOpcode::CFI_INSTRUCTION &&
-            I->getFlag(MachineInstr::FrameSetup)) {
-          MBBI = I;
-          break;
-        }
-      }
-    }
-    unsigned CFIIndex =
-        MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
-    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlags(MachineInstr::FrameSetup);
-  } else if (NeedsWinCFI) {
+  if (!EmitCFI && NeedsWinCFI) {
     BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PACSignLR))
         .setMIFlag(MachineInstr::FrameSetup);
   }
@@ -190,6 +203,7 @@ void AArch64PointerAuth::authenticateLR(
       !MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
     if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
       assert(PACSym && "No PAC instruction to refer to");
+      emitPACSymOffsetIntoX16(*TII, MBB, MBBI, DL, PACSym);
       BuildMI(MBB, TI, DL,
               TII->get(UseBKey ? AArch64::RETABSPPCi : AArch64::RETAASPPCi))
           .addSym(PACSym)
@@ -205,24 +219,22 @@ void AArch64PointerAuth::authenticateLR(
   } else {
     if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
       assert(PACSym && "No PAC instruction to refer to");
+      emitPACSymOffsetIntoX16(*TII, MBB, MBBI, DL, PACSym);
       BuildMI(MBB, MBBI, DL,
               TII->get(UseBKey ? AArch64::AUTIBSPPCi : AArch64::AUTIASPPCi))
           .addSym(PACSym)
           .setMIFlag(MachineInstr::FrameDestroy);
+      emitPACCFI(*Subtarget, MBB, MBBI, DL, MachineInstr::FrameDestroy,
+                 EmitAsyncCFI);
     } else {
       BuildPACM(*Subtarget, MBB, MBBI, DL, MachineInstr::FrameDestroy, PACSym);
       BuildMI(MBB, MBBI, DL,
               TII->get(UseBKey ? AArch64::AUTIBSP : AArch64::AUTIASP))
           .setMIFlag(MachineInstr::FrameDestroy);
+      emitPACCFI(*Subtarget, MBB, MBBI, DL, MachineInstr::FrameDestroy,
+                 EmitAsyncCFI);
     }
 
-    if (EmitAsyncCFI) {
-      unsigned CFIIndex =
-          MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
-      BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlags(MachineInstr::FrameDestroy);
-    }
     if (NeedsWinCFI) {
       BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PACSignLR))
           .setMIFlag(MachineInstr::FrameDestroy);
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index dfc5e04110cf..8e267e658622 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -195,6 +195,7 @@ private:
   bool parseDirectiveReq(StringRef Name, SMLoc L);
   bool parseDirectiveUnreq(SMLoc L);
   bool parseDirectiveCFINegateRAState();
+  bool parseDirectiveCFINegateRAStateWithPC();
   bool parseDirectiveCFIBKeyFrame();
   bool parseDirectiveCFIMTETaggedFrame();
 
@@ -6975,6 +6976,8 @@ bool AArch64AsmParser::ParseDirective(AsmToken DirectiveID) {
     parseDirectiveInst(Loc);
   else if (IDVal == ".cfi_negate_ra_state")
     parseDirectiveCFINegateRAState();
+  else if (IDVal == ".cfi_negate_ra_state_with_pc")
+    parseDirectiveCFINegateRAStateWithPC();
   else if (IDVal == ".cfi_b_key_frame")
     parseDirectiveCFIBKeyFrame();
   else if (IDVal == ".cfi_mte_tagged_frame")
@@ -7425,6 +7428,13 @@ bool AArch64AsmParser::parseDirectiveCFINegateRAState() {
   return false;
 }
 
+bool AArch64AsmParser::parseDirectiveCFINegateRAStateWithPC() {
+  if (parseEOL())
+    return true;
+  getStreamer().emitCFINegateRAStateWithPC();
+  return false;
+}
+
 /// parseDirectiveCFIBKeyFrame
 /// ::= .cfi_b_key
 bool AArch64AsmParser::parseDirectiveCFIBKeyFrame() {
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-cfi.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-cfi.ll
index c64b3842aa5b..4bbbe4017631 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-cfi.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-cfi.ll
@@ -11,8 +11,7 @@ define void @a() "sign-return-address"="all" "sign-return-address-key"="b_key" {
 ; CHECK-NEXT:          .cfi_b_key_frame
 ; V8A-NEXT:            hint #27
 ; V83A-NEXT:           pacibsp
-; CHECK:               .cfi_negate_ra_state
-; CHECK-NEXT:          .cfi_def_cfa_offset
+; CHECK-NEXT:          .cfi_negate_ra_state
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
   %3 = alloca i32, align 4
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-diff-scope-same-key.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-diff-scope-same-key.ll
index 3221815da33c..6a11bef08c74 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-diff-scope-same-key.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-diff-scope-same-key.ll
@@ -7,8 +7,7 @@ define void @a() "sign-return-address"="all" {
 ; CHECK-LABEL:      a:                                     // @a
 ; V8A:              hint #25
 ; V83A:             paciasp
-; CHECK:            .cfi_negate_ra_state
-; CHECK-NEXT:       .cfi_def_cfa_offset
+; CHECK-NEXT:      .cfi_negate_ra_state
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
   %3 = alloca i32, align 4
@@ -55,8 +54,7 @@ define void @c() "sign-return-address"="all" {
 ; CHECK-LABEL:         c:              // @c
 ; V8A:                 hint #25
 ; V83A:                paciasp
-; CHECK:              .cfi_negate_ra_state
-; CHECK-NEXT:         .cfi_def_cfa_offset
+; CHECK-NEXT          .cfi_negate_ra_state
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
   %3 = alloca i32, align 4
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-non-leaf.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-non-leaf.ll
index d43b74b9451a..1e7224683c6c 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-non-leaf.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-non-leaf.ll
@@ -1,15 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple aarch64 %s -o - | \
 ; RUN:   FileCheck %s --check-prefixes CHECK,V8A
 ; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple aarch64 -mattr=+v8.3a %s -o - | \
 ; RUN:   FileCheck %s --check-prefixes CHECK,V83A
 
 define i64 @a(i64 %x) "sign-return-address"="non-leaf" "sign-return-address-key"="b_key" {
-; CHECK-LABEL:      a:                                     // @a
-; CHECK:                .cfi_b_key_frame
-; V8A-NEXT:             hint #27
-; V83A-NEXT:            pacibsp
-; CHECK:                .cfi_negate_ra_state
-; CHECK-NEXT:           .cfi_def_cfa_offset
+; V8A-LABEL: a:
+; V8A:       // %bb.0:
+; V8A-NEXT:    .cfi_b_key_frame
+; V8A-NEXT:    hint #27
+; V8A-NEXT:    .cfi_negate_ra_state
+; V8A-NEXT:    sub sp, sp, #32
+; V8A-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; V8A-NEXT:    .cfi_def_cfa_offset 32
+; V8A-NEXT:    .cfi_offset w30, -16
+; V8A-NEXT:    bl OUTLINED_FUNCTION_0
+; V8A-NEXT:    //APP
+; V8A-NEXT:    mov x30, x0
+; V8A-NEXT:    //NO_APP
+; V8A-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; V8A-NEXT:    add sp, sp, #32
+; V8A-NEXT:    hint #31
+; V8A-NEXT:    ret
+;
+; V83A-LABEL: a:
+; V83A:       // %bb.0:
+; V83A-NEXT:    .cfi_b_key_frame
+; V83A-NEXT:    pacibsp
+; V83A-NEXT:    .cfi_negate_ra_state
+; V83A-NEXT:    sub sp, sp, #32
+; V83A-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; V83A-NEXT:    .cfi_def_cfa_offset 32
+; V83A-NEXT:    .cfi_offset w30, -16
+; V83A-NEXT:    bl OUTLINED_FUNCTION_0
+; V83A-NEXT:    //APP
+; V83A-NEXT:    mov x30, x0
+; V83A-NEXT:    //NO_APP
+; V83A-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; V83A-NEXT:    add sp, sp, #32
+; V83A-NEXT:    retab
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
   %3 = alloca i32, align 4
@@ -27,12 +56,40 @@ define i64 @a(i64 %x) "sign-return-address"="non-leaf" "sign-return-address-key"
 }
 
 define i64 @b(i64 %x) "sign-return-address"="non-leaf" "sign-return-address-key"="b_key" {
-; CHECK-LABEL:      b:                                     // @b
-; CHECK:                .cfi_b_key_frame
-; V8A-NEXT:             hint #27
-; V83A-NEXT:            pacibsp
-; CHECK:                .cfi_negate_ra_state
-; CHECK-NEXT:           .cfi_def_cfa_offset
+; V8A-LABEL: b:
+; V8A:       // %bb.0:
+; V8A-NEXT:    .cfi_b_key_frame
+; V8A-NEXT:    hint #27
+; V8A-NEXT:    .cfi_negate_ra_state
+; V8A-NEXT:    sub sp, sp, #32
+; V8A-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; V8A-NEXT:    .cfi_def_cfa_offset 32
+; V8A-NEXT:    .cfi_offset w30, -16
+; V8A-NEXT:    bl OUTLINED_FUNCTION_0
+; V8A-NEXT:    //APP
+; V8A-NEXT:    mov x30, x0
+; V8A-NEXT:    //NO_APP
+; V8A-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; V8A-NEXT:    add sp, sp, #32
+; V8A-NEXT:    hint #31
+; V8A-NEXT:    ret
+;
+; V83A-LABEL: b:
+; V83A:       // %bb.0:
+; V83A-NEXT:    .cfi_b_key_frame
+; V83A-NEXT:    pacibsp
+; V83A-NEXT:    .cfi_negate_ra_state
+; V83A-NEXT:    sub sp, sp, #32
+; V83A-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; V83A-NEXT:    .cfi_def_cfa_offset 32
+; V83A-NEXT:    .cfi_offset w30, -16
+; V83A-NEXT:    bl OUTLINED_FUNCTION_0
+; V83A-NEXT:    //APP
+; V83A-NEXT:    mov x30, x0
+; V83A-NEXT:    //NO_APP
+; V83A-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; V83A-NEXT:    add sp, sp, #32
+; V83A-NEXT:    retab
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
   %3 = alloca i32, align 4
@@ -50,12 +107,40 @@ define i64 @b(i64 %x) "sign-return-address"="non-leaf" "sign-return-address-key"
 }
 
 define i64 @c(i64 %x) "sign-return-address"="non-leaf" "sign-return-address-key"="b_key" {
-; CHECK-LABEL:      c:                                     // @c
-; CHECK:                .cfi_b_key_frame
-; V8A-NEXT:             hint #27
-; V83A-NEXT:            pacibsp
-; CHECK:                .cfi_negate_ra_state
-; CHECK-NEXT:           .cfi_def_cfa_offset
+; V8A-LABEL: c:
+; V8A:       // %bb.0:
+; V8A-NEXT:    .cfi_b_key_frame
+; V8A-NEXT:    hint #27
+; V8A-NEXT:    .cfi_negate_ra_state
+; V8A-NEXT:    sub sp, sp, #32
+; V8A-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; V8A-NEXT:    .cfi_def_cfa_offset 32
+; V8A-NEXT:    .cfi_offset w30, -16
+; V8A-NEXT:    bl OUTLINED_FUNCTION_0
+; V8A-NEXT:    //APP
+; V8A-NEXT:    mov x30, x0
+; V8A-NEXT:    //NO_APP
+; V8A-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; V8A-NEXT:    add sp, sp, #32
+; V8A-NEXT:    hint #31
+; V8A-NEXT:    ret
+;
+; V83A-LABEL: c:
+; V83A:       // %bb.0:
+; V83A-NEXT:    .cfi_b_key_frame
+; V83A-NEXT:    pacibsp
+; V83A-NEXT:    .cfi_negate_ra_state
+; V83A-NEXT:    sub sp, sp, #32
+; V83A-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; V83A-NEXT:    .cfi_def_cfa_offset 32
+; V83A-NEXT:    .cfi_offset w30, -16
+; V83A-NEXT:    bl OUTLINED_FUNCTION_0
+; V83A-NEXT:    //APP
+; V83A-NEXT:    mov x30, x0
+; V83A-NEXT:    //NO_APP
+; V83A-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; V83A-NEXT:    add sp, sp, #32
+; V83A-NEXT:    retab
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
   %3 = alloca i32, align 4
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-regsave.mir b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-regsave.mir
index ba27d1c681e3..9a983cbd6714 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-regsave.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-regsave.mir
@@ -82,8 +82,7 @@ body:             |
 # CHECK:          bb.0:
 # CHECK:            frame-setup EMITBKEY
 # CHECK-NEXT:       frame-setup PACIBSP implicit-def $lr, implicit $lr, implicit $sp
-# CHECK:            frame-setup CFI_INSTRUCTION negate_ra_sign_state
-# CHECK-NEXT:       frame-setup CFI_INSTRUCTION
+# CHECK-NEXT:       frame-setup CFI_INSTRUCTION negate_ra_sign_state
 # CHECK-NOT:        OUTLINED_FUNCTION_
 # CHECK:          bb.1:
 # CHECK-NOT:        OUTLINED_FUNCTION_
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-diff-key.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-diff-key.ll
index 8c36ab4d8f40..87771f5de4f6 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-diff-key.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-diff-key.ll
@@ -1,14 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple aarch64 %s -o - | \
 ; RUN:   FileCheck %s --check-prefixes CHECK,V8A
 ; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple aarch64 -mattr=+v8.3a %s -o - | \
 ; RUN:   FileCheck %s --check-prefixes CHECK,V83A
 
 define void @a() "sign-return-address"="all" {
-; CHECK-LABEL:      a:                                     // @a
-; V8A:              hint #25
-; V83A:             paciasp
-; CHECK:            .cfi_negate_ra_state
-; CHECK-NEXT:       .cfi_def_cfa_offset
+; V8A-LABEL: a:
+; V8A:       // %bb.0:
+; V8A-NEXT:    hint #25
+; V8A-NEXT:    .cfi_negate_ra_state
+; V8A-NEXT:    sub sp, sp, #32
+; V8A-NEXT:    .cfi_def_cfa_offset 32
+; V8A-NEXT:    mov w8, #1 // =0x1
+; V8A-NEXT:    mov w9, #2 // =0x2
+; V8A-NEXT:    stp w9, w8, [sp, #24]
+; V8A-NEXT:    mov w9, #3 // =0x3
+; V8A-NEXT:    mov w8, #4 // =0x4
+; V8A-NEXT:    stp w8, w9, [sp, #16]
+; V8A-NEXT:    mov w9, #5 // =0x5
+; V8A-NEXT:    mov w8, #6 // =0x6
+; V8A-NEXT:    stp w8, w9, [sp, #8]
+; V8A-NEXT:    add sp, sp, #32
+; V8A-NEXT:    hint #29
+; V8A-NEXT:    ret
+;
+; V83A-LABEL: a:
+; V83A:       // %bb.0:
+; V83A-NEXT:    paciasp
+; V83A-NEXT:    .cfi_negate_ra_state
+; V83A-NEXT:    sub sp, sp, #32
+; V83A-NEXT:    .cfi_def_cfa_offset 32
+; V83A-NEXT:    mov w8, #1 // =0x1
+; V83A-NEXT:    mov w9, #2 // =0x2
+; V83A-NEXT:    stp w9, w8, [sp, #24]
+; V83A-NEXT:    mov w9, #3 // =0x3
+; V83A-NEXT:    mov w8, #4 // =0x4
+; V83A-NEXT:    stp w8, w9, [sp, #16]
+; V83A-NEXT:    mov w9, #5 // =0x5
+; V83A-NEXT:    mov w8, #6 // =0x6
+; V83A-NEXT:    stp w8, w9, [sp, #8]
+; V83A-NEXT:    add sp, sp, #32
+; V83A-NEXT:    retaa
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
   %3 = alloca i32, align 4
@@ -21,19 +53,48 @@ define void @a() "sign-return-address"="all" {
   store i32 4, ptr %4, align 4
   store i32 5, ptr %5, align 4
   store i32 6, ptr %6, align 4
-; V8A:            hint #29
-; V83A:           retaa
   ret void
-; CHECK:          .cfi_endproc
 }
 
 define void @b() "sign-return-address"="all" "sign-return-address-key"="b_key" {
-; CHECK-LABEL:      b:                                     // @b
-; CHECK:            .cfi_b_key_frame
-; V8A-NEXT:         hint #27
-; V83A-NEXT:        pacibsp
-; CHECK:            .cfi_negate_ra_state
-; CHECK-NEXT:       .cfi_def_cfa_offset
+; V8A-LABEL: b:
+; V8A:       // %bb.0:
+; V8A-NEXT:    .cfi_b_key_frame
+; V8A-NEXT:    hint #27
+; V8A-NEXT:    .cfi_negate_ra_state
+; V8A-NEXT:    sub sp, sp, #32
+; V8A-NEXT:    .cfi_def_cfa_offset 32
+; V8A-NEXT:    mov w8, #1 // =0x1
+; V8A-NEXT:    mov w9, #2 // =0x2
+; V8A-NEXT:    stp w9, w8, [sp, #24]
+; V8A-NEXT:    mov w9, #3 // =0x3
+; V8A-NEXT:    mov w8, #4 // =0x4
+; V8A-NEXT:    stp w8, w9, [sp, #16]
+; V8A-NEXT:    mov w9, #5 // =0x5
+; V8A-NEXT:    mov w8, #6 // =0x6
+; V8A-NEXT:    stp w8, w9, [sp, #8]
+; V8A-NEXT:    add sp, sp, #32
+; V8A-NEXT:    hint #31
+; V8A-NEXT:    ret
+;
+; V83A-LABEL: b:
+; V83A:       // %bb.0:
+; V83A-NEXT:    .cfi_b_key_frame
+; V83A-NEXT:    pacibsp
+; V83A-NEXT:    .cfi_negate_ra_state
+; V83A-NEXT:    sub sp, sp, #32
+; V83A-NEXT:    .cfi_def_cfa_offset 32
+; V83A-NEXT:    mov w8, #1 // =0x1
+; V83A-NEXT:    mov w9, #2 // =0x2
+; V83A-NEXT:    stp w9, w8, [sp, #24]
+; V83A-NEXT:    mov w9, #3 // =0x3
+; V83A-NEXT:    mov w8, #4 // =0x4
+; V83A-NEXT:    stp w8, w9, [sp, #16]
+; V83A-NEXT:    mov w9, #5 // =0x5
+; V83A-NEXT:    mov w8, #6 // =0x6
+; V83A-NEXT:    stp w8, w9, [sp, #8]
+; V83A-NEXT:    add sp, sp, #32
+; V83A-NEXT:    retab
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
   %3 = alloca i32, align 4
@@ -46,19 +107,46 @@ define void @b() "sign-return-address"="all" "sign-return-address-key"="b_key" {
   store i32 4, ptr %4, align 4
   store i32 5, ptr %5, align 4
   store i32 6, ptr %6, align 4
-; V8A-NOT:          hint #29
-; V83A-NOT:         autiasp
-; V83A-NOT:         retaa
   ret void
-; CHECK:            .cfi_endproc
 }
 
 define void @c() "sign-return-address"="all" {
-; CHECK-LABEL:      c:                                     // @c
-; V8A:              hint #25
-; V83A:             paciasp
-; CHECK:            .cfi_negate_ra_state
-; CHECK-NEXT:       .cfi_def_cfa_offset
+; V8A-LABEL: c:
+; V8A:       // %bb.0:
+; V8A-NEXT:    hint #25
+; V8A-NEXT:    .cfi_negate_ra_state
+; V8A-NEXT:    sub sp, sp, #32
+; V8A-NEXT:    .cfi_def_cfa_offset 32
+; V8A-NEXT:    mov w8, #1 // =0x1
+; V8A-NEXT:    mov w9, #2 // =0x2
+; V8A-NEXT:    stp w9, w8, [sp, #24]
+; V8A-NEXT:    mov w9, #3 // =0x3
+; V8A-NEXT:    mov w8, #4 // =0x4
+; V8A-NEXT:    stp w8, w9, [sp, #16]
+; V8A-NEXT:    mov w9, #5 // =0x5
+; V8A-NEXT:    mov w8, #6 // =0x6
+; V8A-NEXT:    stp w8, w9, [sp, #8]
+; V8A-NEXT:    add sp, sp, #32
+; V8A-NEXT:    hint #29
+; V8A-NEXT:    ret
+;
+; V83A-LABEL: c:
+; V83A:       // %bb.0:
+; V83A-NEXT:    paciasp
+; V83A-NEXT:    .cfi_negate_ra_state
+; V83A-NEXT:    sub sp, sp, #32
+; V83A-NEXT:    .cfi_def_cfa_offset 32
+; V83A-NEXT:    mov w8, #1 // =0x1
+; V83A-NEXT:    mov w9, #2 // =0x2
+; V83A-NEXT:    stp w9, w8, [sp, #24]
+; V83A-NEXT:    mov w9, #3 // =0x3
+; V83A-NEXT:    mov w8, #4 // =0x4
+; V83A-NEXT:    stp w8, w9, [sp, #16]
+; V83A-NEXT:    mov w9, #5 // =0x5
+; V83A-NEXT:    mov w8, #6 // =0x6
+; V83A-NEXT:    stp w8, w9, [sp, #8]
+; V83A-NEXT:    add sp, sp, #32
+; V83A-NEXT:    retaa
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
   %3 = alloca i32, align 4
@@ -71,11 +159,10 @@ define void @c() "sign-return-address"="all" {
   store i32 4, ptr %4, align 4
   store i32 5, ptr %5, align 4
   store i32 6, ptr %6, align 4
-; V8A:            hint #29
-; V83A:           retaa
   ret void
-; CHECK:          .cfi_endproc
 }
 
 ; CHECK-NOT:      OUTLINED_FUNCTION_0:
 ; CHECK-NOT:      // -- Begin function
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-subtarget.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-subtarget.ll
index d5ef94e90099..a7ea32952f3b 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-subtarget.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-subtarget.ll
@@ -10,8 +10,7 @@ define void @a() #0 {
 ; CHECK:            // %bb.0:
 ; CHECK-NEXT:               .cfi_b_key_frame
 ; CHECK-NEXT:               pacibsp
-; CHECK:                    .cfi_negate_ra_state
-; CHECK-NEXT:               .cfi_def_cfa_offset
+; CHECK-NEXT:               .cfi_negate_ra_state
 ; CHECK-NOT:                OUTLINED_FUNCTION_
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
@@ -35,8 +34,7 @@ define void @b() #0 {
 ; CHECK:            // %bb.0:
 ; CHECK-NEXT:               .cfi_b_key_frame
 ; CHECK-NEXT:               pacibsp
-; CHECK:                    .cfi_negate_ra_state
-; CHECK-NEXT:               .cfi_def_cfa_offset
+; CHECK-NEXT:               .cfi_negate_ra_state
 ; CHECK-NOT:                OUTLINED_FUNCTION_
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
@@ -60,8 +58,7 @@ define void @c() #1 {
 ; CHECK:            // %bb.0:
 ; CHECK-NEXT:               .cfi_b_key_frame
 ; CHECK-NEXT:               hint #27
-; CHECK:                    .cfi_negate_ra_state
-; CHECK-NEXT:               .cfi_def_cfa_offset
+; CHECK-NEXT:               .cfi_negate_ra_state
 ; CHECK-NOT:                OUTLINED_FUNCTION_
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll
index 3e361111b545..da68ea5bf0db 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple aarch64-arm-linux-gnu --enable-machine-outliner -outliner-leaf-descendants=false \
 ; RUN:   -verify-machineinstrs %s -o - | FileCheck --check-prefixes CHECK,V8A %s
 ; RUN: llc -mtriple aarch64 -enable-machine-outliner -outliner-leaf-descendants=false \
@@ -7,15 +8,38 @@
 declare i32 @thunk_called_fn(i32, i32, i32, i32)
 
 define i32 @a() #0 {
-; CHECK-LABEL:  a:                                      // @a
-; CHECK:        // %bb.0:                               // %entry
-; V8A-NEXT:         hint #25
-; V83A-NEXT:        paciasp
-; CHECK:            .cfi_negate_ra_state
-; CHECK-NEXT:       .cfi_def_cfa_offset
-; V8A:              hint #29
-; V8A-NEXT:         ret
-; V83A:             retaa
+; V8A-LABEL: a:
+; V8A:       // %bb.0: // %entry
+; V8A-NEXT:    hint #25
+; V8A-NEXT:    .cfi_negate_ra_state
+; V8A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; V8A-NEXT:    .cfi_def_cfa_offset 16
+; V8A-NEXT:    .cfi_offset w30, -16
+; V8A-NEXT:    mov w0, #1 // =0x1
+; V8A-NEXT:    mov w1, #2 // =0x2
+; V8A-NEXT:    mov w2, #3 // =0x3
+; V8A-NEXT:    mov w3, #4 // =0x4
+; V8A-NEXT:    bl thunk_called_fn
+; V8A-NEXT:    add w0, w0, #8
+; V8A-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; V8A-NEXT:    hint #29
+; V8A-NEXT:    ret
+;
+; V83A-LABEL: a:
+; V83A:       // %bb.0: // %entry
+; V83A-NEXT:    paciasp
+; V83A-NEXT:    .cfi_negate_ra_state
+; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; V83A-NEXT:    .cfi_def_cfa_offset 16
+; V83A-NEXT:    .cfi_offset w30, -16
+; V83A-NEXT:    mov w0, #1 // =0x1
+; V83A-NEXT:    mov w1, #2 // =0x2
+; V83A-NEXT:    mov w2, #3 // =0x3
+; V83A-NEXT:    mov w3, #4 // =0x4
+; V83A-NEXT:    bl thunk_called_fn
+; V83A-NEXT:    add w0, w0, #8
+; V83A-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; V83A-NEXT:    retaa
 entry:
   %call = tail call i32 @thunk_called_fn(i32 1, i32 2, i32 3, i32 4)
   %cx = add i32 %call, 8
@@ -23,15 +47,38 @@ entry:
 }
 
 define i32 @b() #0 {
-; CHECK-LABEL:  b:                                      // @b
-; CHECK:        // %bb.0:                               // %entry
-; V8A-NEXT:         hint #25
-; V83A-NEXT:        paciasp
-; CHECK:            .cfi_negate_ra_state
-; CHECK-NEXT:       .cfi_def_cfa_offset
-; V8A:              hint #29
-; V8A-NEXT:         ret
-; V83A:             retaa
+; V8A-LABEL: b:
+; V8A:       // %bb.0: // %entry
+; V8A-NEXT:    hint #25
+; V8A-NEXT:    .cfi_negate_ra_state
+; V8A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; V8A-NEXT:    .cfi_def_cfa_offset 16
+; V8A-NEXT:    .cfi_offset w30, -16
+; V8A-NEXT:    mov w0, #1 // =0x1
+; V8A-NEXT:    mov w1, #2 // =0x2
+; V8A-NEXT:    mov w2, #3 // =0x3
+; V8A-NEXT:    mov w3, #4 // =0x4
+; V8A-NEXT:    bl thunk_called_fn
+; V8A-NEXT:    add w0, w0, #88
+; V8A-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; V8A-NEXT:    hint #29
+; V8A-NEXT:    ret
+;
+; V83A-LABEL: b:
+; V83A:       // %bb.0: // %entry
+; V83A-NEXT:    paciasp
+; V83A-NEXT:    .cfi_negate_ra_state
+; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; V83A-NEXT:    .cfi_def_cfa_offset 16
+; V83A-NEXT:    .cfi_offset w30, -16
+; V83A-NEXT:    mov w0, #1 // =0x1
+; V83A-NEXT:    mov w1, #2 // =0x2
+; V83A-NEXT:    mov w2, #3 // =0x3
+; V83A-NEXT:    mov w3, #4 // =0x4
+; V83A-NEXT:    bl thunk_called_fn
+; V83A-NEXT:    add w0, w0, #88
+; V83A-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; V83A-NEXT:    retaa
 entry:
   %call = tail call i32 @thunk_called_fn(i32 1, i32 2, i32 3, i32 4)
   %cx = add i32 %call, 88
@@ -39,15 +86,40 @@ entry:
 }
 
 define hidden i32 @c(ptr %fptr) #0 {
-; CHECK-LABEL:  c:                                      // @c
-; CHECK:        // %bb.0:                               // %entry
-; V8A-NEXT:         hint #25
-; V83A-NEXT:        paciasp
-; CHECK:            .cfi_negate_ra_state
-; CHECK-NEXT:       .cfi_def_cfa_offset
-; V8A:              hint #29
-; V8A-NEXT:         ret
-; V83A:             retaa
+; V8A-LABEL: c:
+; V8A:       // %bb.0: // %entry
+; V8A-NEXT:    hint #25
+; V8A-NEXT:    .cfi_negate_ra_state
+; V8A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; V8A-NEXT:    .cfi_def_cfa_offset 16
+; V8A-NEXT:    .cfi_offset w30, -16
+; V8A-NEXT:    mov x8, x0
+; V8A-NEXT:    mov w0, #1 // =0x1
+; V8A-NEXT:    mov w1, #2 // =0x2
+; V8A-NEXT:    mov w2, #3 // =0x3
+; V8A-NEXT:    mov w3, #4 // =0x4
+; V8A-NEXT:    blr x8
+; V8A-NEXT:    add w0, w0, #8
+; V8A-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; V8A-NEXT:    hint #29
+; V8A-NEXT:    ret
+;
+; V83A-LABEL: c:
+; V83A:       // %bb.0: // %entry
+; V83A-NEXT:    paciasp
+; V83A-NEXT:    .cfi_negate_ra_state
+; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; V83A-NEXT:    .cfi_def_cfa_offset 16
+; V83A-NEXT:    .cfi_offset w30, -16
+; V83A-NEXT:    mov x8, x0
+; V83A-NEXT:    mov w0, #1 // =0x1
+; V83A-NEXT:    mov w1, #2 // =0x2
+; V83A-NEXT:    mov w2, #3 // =0x3
+; V83A-NEXT:    mov w3, #4 // =0x4
+; V83A-NEXT:    blr x8
+; V83A-NEXT:    add w0, w0, #8
+; V83A-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; V83A-NEXT:    retaa
 entry:
   %call = tail call i32 %fptr(i32 1, i32 2, i32 3, i32 4)
   %add = add nsw i32 %call, 8
@@ -55,15 +127,40 @@ entry:
 }
 
 define hidden i32 @d(ptr %fptr) #0 {
-; CHECK-LABEL:  d:                                      // @d
-; CHECK:        // %bb.0:                               // %entry
-; V8A-NEXT:         hint #25
-; V83A-NEXT:        paciasp
-; CHECK:            .cfi_negate_ra_state
-; CHECK-NEXT:       .cfi_def_cfa_offset
-; V8A:              hint #29
-; V8A-NEXT:         ret
-; V83A:             retaa
+; V8A-LABEL: d:
+; V8A:       // %bb.0: // %entry
+; V8A-NEXT:    hint #25
+; V8A-NEXT:    .cfi_negate_ra_state
+; V8A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; V8A-NEXT:    .cfi_def_cfa_offset 16
+; V8A-NEXT:    .cfi_offset w30, -16
+; V8A-NEXT:    mov x8, x0
+; V8A-NEXT:    mov w0, #1 // =0x1
+; V8A-NEXT:    mov w1, #2 // =0x2
+; V8A-NEXT:    mov w2, #3 // =0x3
+; V8A-NEXT:    mov w3, #4 // =0x4
+; V8A-NEXT:    blr x8
+; V8A-NEXT:    add w0, w0, #88
+; V8A-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; V8A-NEXT:    hint #29
+; V8A-NEXT:    ret
+;
+; V83A-LABEL: d:
+; V83A:       // %bb.0: // %entry
+; V83A-NEXT:    paciasp
+; V83A-NEXT:    .cfi_negate_ra_state
+; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; V83A-NEXT:    .cfi_def_cfa_offset 16
+; V83A-NEXT:    .cfi_offset w30, -16
+; V83A-NEXT:    mov x8, x0
+; V83A-NEXT:    mov w0, #1 // =0x1
+; V83A-NEXT:    mov w1, #2 // =0x2
+; V83A-NEXT:    mov w2, #3 // =0x3
+; V83A-NEXT:    mov w3, #4 // =0x4
+; V83A-NEXT:    blr x8
+; V83A-NEXT:    add w0, w0, #88
+; V83A-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; V83A-NEXT:    retaa
 entry:
   %call = tail call i32 %fptr(i32 1, i32 2, i32 3, i32 4)
   %add = add nsw i32 %call, 88
diff --git a/llvm/test/CodeGen/AArch64/pacbti-llvm-generated-funcs-2.ll b/llvm/test/CodeGen/AArch64/pacbti-llvm-generated-funcs-2.ll
index 0969ec246399..373c4969a940 100644
--- a/llvm/test/CodeGen/AArch64/pacbti-llvm-generated-funcs-2.ll
+++ b/llvm/test/CodeGen/AArch64/pacbti-llvm-generated-funcs-2.ll
@@ -35,8 +35,7 @@ entry:
 ;; CHECK-LABEL: __llvm_gcov_writeout:
 ;; CHECK:       .cfi_b_key_frame
 ;; CHECK-NEXT:  pacibsp
-;; CHECK:       .cfi_negate_ra_state
-;; CHECK-NEXT:  .cfi_def_cfa_offset
+;; CHECK-NEXT:  .cfi_negate_ra_state
 
 define internal void @__llvm_gcov_reset() unnamed_addr #2 {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/sign-return-address-cfi-negate-ra-state.ll b/llvm/test/CodeGen/AArch64/sign-return-address-cfi-negate-ra-state.ll
index eb224bbbd601..4d4b7c215b97 100644
--- a/llvm/test/CodeGen/AArch64/sign-return-address-cfi-negate-ra-state.ll
+++ b/llvm/test/CodeGen/AArch64/sign-return-address-cfi-negate-ra-state.ll
@@ -10,8 +10,8 @@ define dso_local i32 @_Z3fooi(i32 %x) #0 {
 ; CHECK-V8A-LABEL: _Z3fooi:
 ; CHECK-V8A:       // %bb.0: // %entry
 ; CHECK-V8A-NEXT:    hint #25
-; CHECK-V8A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-V8A-NEXT:    .cfi_negate_ra_state
+; CHECK-V8A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-V8A-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-V8A-NEXT:    .cfi_offset w30, -16
 ; CHECK-V8A-NEXT:    str w0, [sp, #8]
@@ -28,8 +28,8 @@ define dso_local i32 @_Z3fooi(i32 %x) #0 {
 ; CHECK-V83A-LABEL: _Z3fooi:
 ; CHECK-V83A:       // %bb.0: // %entry
 ; CHECK-V83A-NEXT:    paciasp
-; CHECK-V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-V83A-NEXT:    .cfi_negate_ra_state
+; CHECK-V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-V83A-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-V83A-NEXT:    .cfi_offset w30, -16
 ; CHECK-V83A-NEXT:    str w0, [sp, #8]
@@ -144,8 +144,8 @@ define hidden noundef i32 @baz_sync(i32 noundef %a) #0 uwtable(sync) {
 ; CHECK-V8A-LABEL: baz_sync:
 ; CHECK-V8A:       // %bb.0: // %entry
 ; CHECK-V8A-NEXT:    hint #25
-; CHECK-V8A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-V8A-NEXT:    .cfi_negate_ra_state
+; CHECK-V8A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-V8A-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-V8A-NEXT:    .cfi_offset w30, -16
 ; CHECK-V8A-NEXT:    cbz w0, .LBB2_2
@@ -165,8 +165,8 @@ define hidden noundef i32 @baz_sync(i32 noundef %a) #0 uwtable(sync) {
 ; CHECK-V83A-LABEL: baz_sync:
 ; CHECK-V83A:       // %bb.0: // %entry
 ; CHECK-V83A-NEXT:    paciasp
-; CHECK-V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-V83A-NEXT:    .cfi_negate_ra_state
+; CHECK-V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-V83A-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-V83A-NEXT:    .cfi_offset w30, -16
 ; CHECK-V83A-NEXT:    cbz w0, .LBB2_2
@@ -229,7 +229,6 @@ attributes #0 = { "sign-return-address"="all" }
 ; CHECK-DUMP:   DW_CFA_restore_state:
 ; CHECK-DUMP:   DW_CFA_AARCH64_negate_ra_state:
 
-; CHECK-DUMP: CFA=WSP{{$}}
 ;; First DW_CFA_AARCH64_negate_ra_state:
 ; CHECK-DUMP: reg34=1
 ;; Second DW_CFA_AARCH64_negate_ra_state:
@@ -238,7 +237,7 @@ attributes #0 = { "sign-return-address"="all" }
 ; CHECK-DUMP: reg34=1
 ;; Third DW_CFA_AARCH64_negate_ra_state:
 ; CHECK-DUMP: reg34=0
-; CHECK-DUMP-NOT: reg34=
+; CHECK-DUMP-NOT: reg34=1
 
 ; baz_sync
 ; CHECK-DUMP-LABEL: FDE
diff --git a/llvm/test/CodeGen/AArch64/sign-return-address-pauth-lr.ll b/llvm/test/CodeGen/AArch64/sign-return-address-pauth-lr.ll
index 3d133e02106b..fa689d2b9d7f 100644
--- a/llvm/test/CodeGen/AArch64/sign-return-address-pauth-lr.ll
+++ b/llvm/test/CodeGen/AArch64/sign-return-address-pauth-lr.ll
@@ -62,8 +62,9 @@ define i32 @leaf_sign_all(i32 %x) "branch-protection-pauth-lr" "sign-return-addr
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:  .Ltmp0:
 ; COMPAT-NEXT:    hint #25
-; COMPAT-NEXT:    .cfi_negate_ra_state
-; COMPAT-NEXT:    adr x16, .Ltmp0
+; COMPAT-NEXT:    .cfi_negate_ra_state_with_pc
+; COMPAT-NEXT:    adrp x16, .Ltmp0
+; COMPAT-NEXT:    add x16, x16, :lo12:.Ltmp0
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:    hint #29
 ; COMPAT-NEXT:    ret
@@ -73,8 +74,9 @@ define i32 @leaf_sign_all(i32 %x) "branch-protection-pauth-lr" "sign-return-addr
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:  .Ltmp0:
 ; V83A-NEXT:    paciasp
-; V83A-NEXT:    .cfi_negate_ra_state
-; V83A-NEXT:    adr x16, .Ltmp0
+; V83A-NEXT:    .cfi_negate_ra_state_with_pc
+; V83A-NEXT:    adrp x16, .Ltmp0
+; V83A-NEXT:    add x16, x16, :lo12:.Ltmp0
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:    retaa
 ;
@@ -82,7 +84,9 @@ define i32 @leaf_sign_all(i32 %x) "branch-protection-pauth-lr" "sign-return-addr
 ; PAUTHLR:       // %bb.0:
 ; PAUTHLR-NEXT:  .Ltmp0:
 ; PAUTHLR-NEXT:    paciasppc
-; PAUTHLR-NEXT:    .cfi_negate_ra_state
+; PAUTHLR-NEXT:    .cfi_negate_ra_state_with_pc
+; PAUTHLR-NEXT:    adrp x16, .Ltmp0
+; PAUTHLR-NEXT:    add x16, x16, :lo12:.Ltmp0
 ; PAUTHLR-NEXT:    retaasppc .Ltmp0
   ret i32 %x
 }
@@ -93,15 +97,16 @@ define i64 @leaf_clobbers_lr(i64 %x) "branch-protection-pauth-lr" "sign-return-a
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:  .Ltmp1:
 ; COMPAT-NEXT:    hint #25
+; COMPAT-NEXT:    .cfi_negate_ra_state_with_pc
 ; COMPAT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; COMPAT-NEXT:    .cfi_negate_ra_state
 ; COMPAT-NEXT:    .cfi_def_cfa_offset 16
 ; COMPAT-NEXT:    .cfi_offset w30, -16
 ; COMPAT-NEXT:    //APP
 ; COMPAT-NEXT:    mov x30, x0
 ; COMPAT-NEXT:    //NO_APP
 ; COMPAT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; COMPAT-NEXT:    adr x16, .Ltmp1
+; COMPAT-NEXT:    adrp x16, .Ltmp1
+; COMPAT-NEXT:    add x16, x16, :lo12:.Ltmp1
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:    hint #29
 ; COMPAT-NEXT:    ret
@@ -111,15 +116,16 @@ define i64 @leaf_clobbers_lr(i64 %x) "branch-protection-pauth-lr" "sign-return-a
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:  .Ltmp1:
 ; V83A-NEXT:    paciasp
+; V83A-NEXT:    .cfi_negate_ra_state_with_pc
 ; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; V83A-NEXT:    .cfi_negate_ra_state
 ; V83A-NEXT:    .cfi_def_cfa_offset 16
 ; V83A-NEXT:    .cfi_offset w30, -16
 ; V83A-NEXT:    //APP
 ; V83A-NEXT:    mov x30, x0
 ; V83A-NEXT:    //NO_APP
 ; V83A-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; V83A-NEXT:    adr x16, .Ltmp1
+; V83A-NEXT:    adrp x16, .Ltmp1
+; V83A-NEXT:    add x16, x16, :lo12:.Ltmp1
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:    retaa
 ;
@@ -127,14 +133,16 @@ define i64 @leaf_clobbers_lr(i64 %x) "branch-protection-pauth-lr" "sign-return-a
 ; PAUTHLR:       // %bb.0:
 ; PAUTHLR-NEXT:  .Ltmp1:
 ; PAUTHLR-NEXT:    paciasppc
+; PAUTHLR-NEXT:    .cfi_negate_ra_state_with_pc
 ; PAUTHLR-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; PAUTHLR-NEXT:    .cfi_negate_ra_state
 ; PAUTHLR-NEXT:    .cfi_def_cfa_offset 16
 ; PAUTHLR-NEXT:    .cfi_offset w30, -16
 ; PAUTHLR-NEXT:    //APP
 ; PAUTHLR-NEXT:    mov x30, x0
 ; PAUTHLR-NEXT:    //NO_APP
 ; PAUTHLR-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; PAUTHLR-NEXT:    adrp x16, .Ltmp1
+; PAUTHLR-NEXT:    add x16, x16, :lo12:.Ltmp1
 ; PAUTHLR-NEXT:    retaasppc .Ltmp1
   call void asm sideeffect "mov x30, $0", "r,~{lr}"(i64 %x) #1
   ret i64 %x
@@ -148,13 +156,14 @@ define i32 @non_leaf_sign_all(i32 %x) "branch-protection-pauth-lr" "sign-return-
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:  .Ltmp2:
 ; COMPAT-NEXT:    hint #25
+; COMPAT-NEXT:    .cfi_negate_ra_state_with_pc
 ; COMPAT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; COMPAT-NEXT:    .cfi_negate_ra_state
 ; COMPAT-NEXT:    .cfi_def_cfa_offset 16
 ; COMPAT-NEXT:    .cfi_offset w30, -16
 ; COMPAT-NEXT:    bl foo
 ; COMPAT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; COMPAT-NEXT:    adr x16, .Ltmp2
+; COMPAT-NEXT:    adrp x16, .Ltmp2
+; COMPAT-NEXT:    add x16, x16, :lo12:.Ltmp2
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:    hint #29
 ; COMPAT-NEXT:    ret
@@ -164,13 +173,14 @@ define i32 @non_leaf_sign_all(i32 %x) "branch-protection-pauth-lr" "sign-return-
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:  .Ltmp2:
 ; V83A-NEXT:    paciasp
+; V83A-NEXT:    .cfi_negate_ra_state_with_pc
 ; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; V83A-NEXT:    .cfi_negate_ra_state
 ; V83A-NEXT:    .cfi_def_cfa_offset 16
 ; V83A-NEXT:    .cfi_offset w30, -16
 ; V83A-NEXT:    bl foo
 ; V83A-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; V83A-NEXT:    adr x16, .Ltmp2
+; V83A-NEXT:    adrp x16, .Ltmp2
+; V83A-NEXT:    add x16, x16, :lo12:.Ltmp2
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:    retaa
 ;
@@ -178,12 +188,14 @@ define i32 @non_leaf_sign_all(i32 %x) "branch-protection-pauth-lr" "sign-return-
 ; PAUTHLR:       // %bb.0:
 ; PAUTHLR-NEXT:  .Ltmp2:
 ; PAUTHLR-NEXT:    paciasppc
+; PAUTHLR-NEXT:    .cfi_negate_ra_state_with_pc
 ; PAUTHLR-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; PAUTHLR-NEXT:    .cfi_negate_ra_state
 ; PAUTHLR-NEXT:    .cfi_def_cfa_offset 16
 ; PAUTHLR-NEXT:    .cfi_offset w30, -16
 ; PAUTHLR-NEXT:    bl foo
 ; PAUTHLR-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; PAUTHLR-NEXT:    adrp x16, .Ltmp2
+; PAUTHLR-NEXT:    add x16, x16, :lo12:.Ltmp2
 ; PAUTHLR-NEXT:    retaasppc .Ltmp2
   %call = call i32 @foo(i32 %x)
   ret i32 %call
@@ -195,13 +207,14 @@ define i32 @non_leaf_sign_non_leaf(i32 %x) "branch-protection-pauth-lr" "sign-re
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:  .Ltmp3:
 ; COMPAT-NEXT:    hint #25
+; COMPAT-NEXT:    .cfi_negate_ra_state_with_pc
 ; COMPAT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; COMPAT-NEXT:    .cfi_negate_ra_state
 ; COMPAT-NEXT:    .cfi_def_cfa_offset 16
 ; COMPAT-NEXT:    .cfi_offset w30, -16
 ; COMPAT-NEXT:    bl foo
 ; COMPAT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; COMPAT-NEXT:    adr x16, .Ltmp3
+; COMPAT-NEXT:    adrp x16, .Ltmp3
+; COMPAT-NEXT:    add x16, x16, :lo12:.Ltmp3
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:    hint #29
 ; COMPAT-NEXT:    ret
@@ -211,13 +224,14 @@ define i32 @non_leaf_sign_non_leaf(i32 %x) "branch-protection-pauth-lr" "sign-re
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:  .Ltmp3:
 ; V83A-NEXT:    paciasp
+; V83A-NEXT:    .cfi_negate_ra_state_with_pc
 ; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; V83A-NEXT:    .cfi_negate_ra_state
 ; V83A-NEXT:    .cfi_def_cfa_offset 16
 ; V83A-NEXT:    .cfi_offset w30, -16
 ; V83A-NEXT:    bl foo
 ; V83A-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; V83A-NEXT:    adr x16, .Ltmp3
+; V83A-NEXT:    adrp x16, .Ltmp3
+; V83A-NEXT:    add x16, x16, :lo12:.Ltmp3
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:    retaa
 ;
@@ -225,12 +239,14 @@ define i32 @non_leaf_sign_non_leaf(i32 %x) "branch-protection-pauth-lr" "sign-re
 ; PAUTHLR:       // %bb.0:
 ; PAUTHLR-NEXT:  .Ltmp3:
 ; PAUTHLR-NEXT:    paciasppc
+; PAUTHLR-NEXT:    .cfi_negate_ra_state_with_pc
 ; PAUTHLR-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; PAUTHLR-NEXT:    .cfi_negate_ra_state
 ; PAUTHLR-NEXT:    .cfi_def_cfa_offset 16
 ; PAUTHLR-NEXT:    .cfi_offset w30, -16
 ; PAUTHLR-NEXT:    bl foo
 ; PAUTHLR-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; PAUTHLR-NEXT:    adrp x16, .Ltmp3
+; PAUTHLR-NEXT:    add x16, x16, :lo12:.Ltmp3
 ; PAUTHLR-NEXT:    retaasppc .Ltmp3
   %call = call i32 @foo(i32 %x)
   ret i32 %call
@@ -245,13 +261,14 @@ define i32 @non_leaf_scs(i32 %x) "branch-protection-pauth-lr" "sign-return-addre
 ; CHECK-NEXT:    hint #39
 ; CHECK-NEXT:  .Ltmp4:
 ; CHECK-NEXT:    paciasp
+; CHECK-NEXT:    .cfi_negate_ra_state_with_pc
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_negate_ra_state
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl foo
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT:    adr x16, .Ltmp4
+; CHECK-NEXT:    adrp x16, .Ltmp4
+; CHECK-NEXT:    add x16, x16, :lo12:.Ltmp4
 ; CHECK-NEXT:    hint #39
 ; CHECK-NEXT:    autiasp
 ; CHECK-NEXT:    ldr x30, [x18, #-8]!
@@ -263,12 +280,14 @@ define i32 @non_leaf_scs(i32 %x) "branch-protection-pauth-lr" "sign-return-addre
 ; PAUTHLR-NEXT:    .cfi_escape 0x16, 0x12, 0x02, 0x82, 0x78 //
 ; PAUTHLR-NEXT:  .Ltmp4:
 ; PAUTHLR-NEXT:    paciasppc
+; PAUTHLR-NEXT:    .cfi_negate_ra_state_with_pc
 ; PAUTHLR-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; PAUTHLR-NEXT:    .cfi_negate_ra_state
 ; PAUTHLR-NEXT:    .cfi_def_cfa_offset 16
 ; PAUTHLR-NEXT:    .cfi_offset w30, -16
 ; PAUTHLR-NEXT:    bl foo
 ; PAUTHLR-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; PAUTHLR-NEXT:    adrp x16, .Ltmp4
+; PAUTHLR-NEXT:    add x16, x16, :lo12:.Ltmp4
 ; PAUTHLR-NEXT:    autiasppc .Ltmp4
 ; PAUTHLR-NEXT:    ldr x30, [x18, #-8]!
 ; PAUTHLR-NEXT:    ret
@@ -282,8 +301,9 @@ define i32 @leaf_sign_all_v83(i32 %x) "branch-protection-pauth-lr" "sign-return-
 ; CHECK-NEXT:    hint #39
 ; CHECK-NEXT:  .Ltmp5:
 ; CHECK-NEXT:    paciasp
-; CHECK-NEXT:    .cfi_negate_ra_state
-; CHECK-NEXT:    adr x16, .Ltmp5
+; CHECK-NEXT:    .cfi_negate_ra_state_with_pc
+; CHECK-NEXT:    adrp x16, .Ltmp5
+; CHECK-NEXT:    add x16, x16, :lo12:.Ltmp5
 ; CHECK-NEXT:    hint #39
 ; CHECK-NEXT:    retaa
 ;
@@ -291,7 +311,9 @@ define i32 @leaf_sign_all_v83(i32 %x) "branch-protection-pauth-lr" "sign-return-
 ; PAUTHLR:       // %bb.0:
 ; PAUTHLR-NEXT:  .Ltmp5:
 ; PAUTHLR-NEXT:    paciasppc
-; PAUTHLR-NEXT:    .cfi_negate_ra_state
+; PAUTHLR-NEXT:    .cfi_negate_ra_state_with_pc
+; PAUTHLR-NEXT:    adrp x16, .Ltmp5
+; PAUTHLR-NEXT:    add x16, x16, :lo12:.Ltmp5
 ; PAUTHLR-NEXT:    retaasppc .Ltmp5
   ret i32 %x
 }
@@ -304,15 +326,16 @@ define fastcc void @spill_lr_and_tail_call(i64 %x) "branch-protection-pauth-lr"
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:  .Ltmp6:
 ; COMPAT-NEXT:    hint #25
+; COMPAT-NEXT:    .cfi_negate_ra_state_with_pc
 ; COMPAT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; COMPAT-NEXT:    .cfi_negate_ra_state
 ; COMPAT-NEXT:    .cfi_def_cfa_offset 16
 ; COMPAT-NEXT:    .cfi_offset w30, -16
 ; COMPAT-NEXT:    //APP
 ; COMPAT-NEXT:    mov x30, x0
 ; COMPAT-NEXT:    //NO_APP
 ; COMPAT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; COMPAT-NEXT:    adr x16, .Ltmp6
+; COMPAT-NEXT:    adrp x16, .Ltmp6
+; COMPAT-NEXT:    add x16, x16, :lo12:.Ltmp6
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:    hint #29
 ; COMPAT-NEXT:    b bar
@@ -322,15 +345,16 @@ define fastcc void @spill_lr_and_tail_call(i64 %x) "branch-protection-pauth-lr"
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:  .Ltmp6:
 ; V83A-NEXT:    paciasp
+; V83A-NEXT:    .cfi_negate_ra_state_with_pc
 ; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; V83A-NEXT:    .cfi_negate_ra_state
 ; V83A-NEXT:    .cfi_def_cfa_offset 16
 ; V83A-NEXT:    .cfi_offset w30, -16
 ; V83A-NEXT:    //APP
 ; V83A-NEXT:    mov x30, x0
 ; V83A-NEXT:    //NO_APP
 ; V83A-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; V83A-NEXT:    adr x16, .Ltmp6
+; V83A-NEXT:    adrp x16, .Ltmp6
+; V83A-NEXT:    add x16, x16, :lo12:.Ltmp6
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:    autiasp
 ; V83A-NEXT:    b bar
@@ -339,14 +363,16 @@ define fastcc void @spill_lr_and_tail_call(i64 %x) "branch-protection-pauth-lr"
 ; PAUTHLR:       // %bb.0:
 ; PAUTHLR-NEXT:  .Ltmp6:
 ; PAUTHLR-NEXT:    paciasppc
+; PAUTHLR-NEXT:    .cfi_negate_ra_state_with_pc
 ; PAUTHLR-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; PAUTHLR-NEXT:    .cfi_negate_ra_state
 ; PAUTHLR-NEXT:    .cfi_def_cfa_offset 16
 ; PAUTHLR-NEXT:    .cfi_offset w30, -16
 ; PAUTHLR-NEXT:    //APP
 ; PAUTHLR-NEXT:    mov x30, x0
 ; PAUTHLR-NEXT:    //NO_APP
 ; PAUTHLR-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; PAUTHLR-NEXT:    adrp x16, .Ltmp6
+; PAUTHLR-NEXT:    add x16, x16, :lo12:.Ltmp6
 ; PAUTHLR-NEXT:    autiasppc .Ltmp6
 ; PAUTHLR-NEXT:    b bar
   call void asm sideeffect "mov x30, $0", "r,~{lr}"(i64 %x) #1
@@ -360,8 +386,9 @@ define i32 @leaf_sign_all_a_key(i32 %x) "branch-protection-pauth-lr" "sign-retur
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:  .Ltmp7:
 ; COMPAT-NEXT:    hint #25
-; COMPAT-NEXT:    .cfi_negate_ra_state
-; COMPAT-NEXT:    adr x16, .Ltmp7
+; COMPAT-NEXT:    .cfi_negate_ra_state_with_pc
+; COMPAT-NEXT:    adrp x16, .Ltmp7
+; COMPAT-NEXT:    add x16, x16, :lo12:.Ltmp7
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:    hint #29
 ; COMPAT-NEXT:    ret
@@ -371,8 +398,9 @@ define i32 @leaf_sign_all_a_key(i32 %x) "branch-protection-pauth-lr" "sign-retur
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:  .Ltmp7:
 ; V83A-NEXT:    paciasp
-; V83A-NEXT:    .cfi_negate_ra_state
-; V83A-NEXT:    adr x16, .Ltmp7
+; V83A-NEXT:    .cfi_negate_ra_state_with_pc
+; V83A-NEXT:    adrp x16, .Ltmp7
+; V83A-NEXT:    add x16, x16, :lo12:.Ltmp7
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:    retaa
 ;
@@ -380,7 +408,9 @@ define i32 @leaf_sign_all_a_key(i32 %x) "branch-protection-pauth-lr" "sign-retur
 ; PAUTHLR:       // %bb.0:
 ; PAUTHLR-NEXT:  .Ltmp7:
 ; PAUTHLR-NEXT:    paciasppc
-; PAUTHLR-NEXT:    .cfi_negate_ra_state
+; PAUTHLR-NEXT:    .cfi_negate_ra_state_with_pc
+; PAUTHLR-NEXT:    adrp x16, .Ltmp7
+; PAUTHLR-NEXT:    add x16, x16, :lo12:.Ltmp7
 ; PAUTHLR-NEXT:    retaasppc .Ltmp7
   ret i32 %x
 }
@@ -392,8 +422,9 @@ define i32 @leaf_sign_all_b_key(i32 %x) "branch-protection-pauth-lr" "sign-retur
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:  .Ltmp8:
 ; COMPAT-NEXT:    hint #27
-; COMPAT-NEXT:    .cfi_negate_ra_state
-; COMPAT-NEXT:    adr x16, .Ltmp8
+; COMPAT-NEXT:    .cfi_negate_ra_state_with_pc
+; COMPAT-NEXT:    adrp x16, .Ltmp8
+; COMPAT-NEXT:    add x16, x16, :lo12:.Ltmp8
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:    hint #31
 ; COMPAT-NEXT:    ret
@@ -404,8 +435,9 @@ define i32 @leaf_sign_all_b_key(i32 %x) "branch-protection-pauth-lr" "sign-retur
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:  .Ltmp8:
 ; V83A-NEXT:    pacibsp
-; V83A-NEXT:    .cfi_negate_ra_state
-; V83A-NEXT:    adr x16, .Ltmp8
+; V83A-NEXT:    .cfi_negate_ra_state_with_pc
+; V83A-NEXT:    adrp x16, .Ltmp8
+; V83A-NEXT:    add x16, x16, :lo12:.Ltmp8
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:    retab
 ;
@@ -414,7 +446,9 @@ define i32 @leaf_sign_all_b_key(i32 %x) "branch-protection-pauth-lr" "sign-retur
 ; PAUTHLR-NEXT:    .cfi_b_key_frame
 ; PAUTHLR-NEXT:  .Ltmp8:
 ; PAUTHLR-NEXT:    pacibsppc
-; PAUTHLR-NEXT:    .cfi_negate_ra_state
+; PAUTHLR-NEXT:    .cfi_negate_ra_state_with_pc
+; PAUTHLR-NEXT:    adrp x16, .Ltmp8
+; PAUTHLR-NEXT:    add x16, x16, :lo12:.Ltmp8
 ; PAUTHLR-NEXT:    retabsppc .Ltmp8
   ret i32 %x
 }
@@ -426,8 +460,9 @@ define i32 @leaf_sign_all_v83_b_key(i32 %x) "branch-protection-pauth-lr" "sign-r
 ; CHECK-NEXT:    hint #39
 ; CHECK-NEXT:  .Ltmp9:
 ; CHECK-NEXT:    pacibsp
-; CHECK-NEXT:    .cfi_negate_ra_state
-; CHECK-NEXT:    adr x16, .Ltmp9
+; CHECK-NEXT:    .cfi_negate_ra_state_with_pc
+; CHECK-NEXT:    adrp x16, .Ltmp9
+; CHECK-NEXT:    add x16, x16, :lo12:.Ltmp9
 ; CHECK-NEXT:    hint #39
 ; CHECK-NEXT:    retab
 ;
@@ -436,7 +471,9 @@ define i32 @leaf_sign_all_v83_b_key(i32 %x) "branch-protection-pauth-lr" "sign-r
 ; PAUTHLR-NEXT:    .cfi_b_key_frame
 ; PAUTHLR-NEXT:  .Ltmp9:
 ; PAUTHLR-NEXT:    pacibsppc
-; PAUTHLR-NEXT:    .cfi_negate_ra_state
+; PAUTHLR-NEXT:    .cfi_negate_ra_state_with_pc
+; PAUTHLR-NEXT:    adrp x16, .Ltmp9
+; PAUTHLR-NEXT:    add x16, x16, :lo12:.Ltmp9
 ; PAUTHLR-NEXT:    retabsppc .Ltmp9
   ret i32 %x
 }
@@ -449,8 +486,9 @@ define i32 @leaf_sign_all_a_key_bti(i32 %x) "branch-protection-pauth-lr" "sign-r
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:  .Ltmp10:
 ; COMPAT-NEXT:    hint #25
-; COMPAT-NEXT:    .cfi_negate_ra_state
-; COMPAT-NEXT:    adr x16, .Ltmp10
+; COMPAT-NEXT:    .cfi_negate_ra_state_with_pc
+; COMPAT-NEXT:    adrp x16, .Ltmp10
+; COMPAT-NEXT:    add x16, x16, :lo12:.Ltmp10
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:    hint #29
 ; COMPAT-NEXT:    ret
@@ -461,8 +499,9 @@ define i32 @leaf_sign_all_a_key_bti(i32 %x) "branch-protection-pauth-lr" "sign-r
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:  .Ltmp10:
 ; V83A-NEXT:    paciasp
-; V83A-NEXT:    .cfi_negate_ra_state
-; V83A-NEXT:    adr x16, .Ltmp10
+; V83A-NEXT:    .cfi_negate_ra_state_with_pc
+; V83A-NEXT:    adrp x16, .Ltmp10
+; V83A-NEXT:    add x16, x16, :lo12:.Ltmp10
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:    retaa
 ;
@@ -471,7 +510,9 @@ define i32 @leaf_sign_all_a_key_bti(i32 %x) "branch-protection-pauth-lr" "sign-r
 ; PAUTHLR-NEXT:    bti c
 ; PAUTHLR-NEXT:  .Ltmp10:
 ; PAUTHLR-NEXT:    paciasppc
-; PAUTHLR-NEXT:    .cfi_negate_ra_state
+; PAUTHLR-NEXT:    .cfi_negate_ra_state_with_pc
+; PAUTHLR-NEXT:    adrp x16, .Ltmp10
+; PAUTHLR-NEXT:    add x16, x16, :lo12:.Ltmp10
 ; PAUTHLR-NEXT:    retaasppc .Ltmp10
   ret i32 %x
 }
@@ -485,8 +526,9 @@ define i32 @leaf_sign_all_b_key_bti(i32 %x) "branch-protection-pauth-lr" "sign-r
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:  .Ltmp11:
 ; COMPAT-NEXT:    hint #27
-; COMPAT-NEXT:    .cfi_negate_ra_state
-; COMPAT-NEXT:    adr x16, .Ltmp11
+; COMPAT-NEXT:    .cfi_negate_ra_state_with_pc
+; COMPAT-NEXT:    adrp x16, .Ltmp11
+; COMPAT-NEXT:    add x16, x16, :lo12:.Ltmp11
 ; COMPAT-NEXT:    hint #39
 ; COMPAT-NEXT:    hint #31
 ; COMPAT-NEXT:    ret
@@ -498,8 +540,9 @@ define i32 @leaf_sign_all_b_key_bti(i32 %x) "branch-protection-pauth-lr" "sign-r
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:  .Ltmp11:
 ; V83A-NEXT:    pacibsp
-; V83A-NEXT:    .cfi_negate_ra_state
-; V83A-NEXT:    adr x16, .Ltmp11
+; V83A-NEXT:    .cfi_negate_ra_state_with_pc
+; V83A-NEXT:    adrp x16, .Ltmp11
+; V83A-NEXT:    add x16, x16, :lo12:.Ltmp11
 ; V83A-NEXT:    hint #39
 ; V83A-NEXT:    retab
 ;
@@ -509,7 +552,9 @@ define i32 @leaf_sign_all_b_key_bti(i32 %x) "branch-protection-pauth-lr" "sign-r
 ; PAUTHLR-NEXT:    .cfi_b_key_frame
 ; PAUTHLR-NEXT:  .Ltmp11:
 ; PAUTHLR-NEXT:    pacibsppc
-; PAUTHLR-NEXT:    .cfi_negate_ra_state
+; PAUTHLR-NEXT:    .cfi_negate_ra_state_with_pc
+; PAUTHLR-NEXT:    adrp x16, .Ltmp11
+; PAUTHLR-NEXT:    add x16, x16, :lo12:.Ltmp11
 ; PAUTHLR-NEXT:    retabsppc .Ltmp11
   ret i32 %x
 }
@@ -523,8 +568,9 @@ define i32 @leaf_sign_all_v83_b_key_bti(i32 %x) "branch-protection-pauth-lr" "si
 ; CHECK-NEXT:    hint #39
 ; CHECK-NEXT:  .Ltmp12:
 ; CHECK-NEXT:    pacibsp
-; CHECK-NEXT:    .cfi_negate_ra_state
-; CHECK-NEXT:    adr x16, .Ltmp12
+; CHECK-NEXT:    .cfi_negate_ra_state_with_pc
+; CHECK-NEXT:    adrp x16, .Ltmp12
+; CHECK-NEXT:    add x16, x16, :lo12:.Ltmp12
 ; CHECK-NEXT:    hint #39
 ; CHECK-NEXT:    retab
 ;
@@ -534,7 +580,9 @@ define i32 @leaf_sign_all_v83_b_key_bti(i32 %x) "branch-protection-pauth-lr" "si
 ; PAUTHLR-NEXT:    .cfi_b_key_frame
 ; PAUTHLR-NEXT:  .Ltmp12:
 ; PAUTHLR-NEXT:    pacibsppc
-; PAUTHLR-NEXT:    .cfi_negate_ra_state
+; PAUTHLR-NEXT:    .cfi_negate_ra_state_with_pc
+; PAUTHLR-NEXT:    adrp x16, .Ltmp12
+; PAUTHLR-NEXT:    add x16, x16, :lo12:.Ltmp12
 ; PAUTHLR-NEXT:    retabsppc .Ltmp12
   ret i32 %x
 }
diff --git a/llvm/test/CodeGen/AArch64/sign-return-address.ll b/llvm/test/CodeGen/AArch64/sign-return-address.ll
index c33463eb96a6..dafe0d71ceb5 100644
--- a/llvm/test/CodeGen/AArch64/sign-return-address.ll
+++ b/llvm/test/CodeGen/AArch64/sign-return-address.ll
@@ -46,8 +46,8 @@ define i64 @leaf_clobbers_lr(i64 %x) "sign-return-address"="non-leaf"  {
 ; COMPAT-LABEL: leaf_clobbers_lr:
 ; COMPAT:       // %bb.0:
 ; COMPAT-NEXT:    hint #25
-; COMPAT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; COMPAT-NEXT:    .cfi_negate_ra_state
+; COMPAT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; COMPAT-NEXT:    .cfi_def_cfa_offset 16
 ; COMPAT-NEXT:    .cfi_offset w30, -16
 ; COMPAT-NEXT:    //APP
@@ -60,8 +60,8 @@ define i64 @leaf_clobbers_lr(i64 %x) "sign-return-address"="non-leaf"  {
 ; V83A-LABEL: leaf_clobbers_lr:
 ; V83A:       // %bb.0:
 ; V83A-NEXT:    paciasp
-; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; V83A-NEXT:    .cfi_negate_ra_state
+; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; V83A-NEXT:    .cfi_def_cfa_offset 16
 ; V83A-NEXT:    .cfi_offset w30, -16
 ; V83A-NEXT:    //APP
@@ -79,8 +79,8 @@ define i32 @non_leaf_sign_all(i32 %x) "sign-return-address"="all" {
 ; COMPAT-LABEL: non_leaf_sign_all:
 ; COMPAT:       // %bb.0:
 ; COMPAT-NEXT:    hint #25
-; COMPAT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; COMPAT-NEXT:    .cfi_negate_ra_state
+; COMPAT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; COMPAT-NEXT:    .cfi_def_cfa_offset 16
 ; COMPAT-NEXT:    .cfi_offset w30, -16
 ; COMPAT-NEXT:    bl foo
@@ -91,8 +91,8 @@ define i32 @non_leaf_sign_all(i32 %x) "sign-return-address"="all" {
 ; V83A-LABEL: non_leaf_sign_all:
 ; V83A:       // %bb.0:
 ; V83A-NEXT:    paciasp
-; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; V83A-NEXT:    .cfi_negate_ra_state
+; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; V83A-NEXT:    .cfi_def_cfa_offset 16
 ; V83A-NEXT:    .cfi_offset w30, -16
 ; V83A-NEXT:    bl foo
@@ -106,8 +106,8 @@ define i32 @non_leaf_sign_non_leaf(i32 %x) "sign-return-address"="non-leaf"  {
 ; COMPAT-LABEL: non_leaf_sign_non_leaf:
 ; COMPAT:       // %bb.0:
 ; COMPAT-NEXT:    hint #25
-; COMPAT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; COMPAT-NEXT:    .cfi_negate_ra_state
+; COMPAT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; COMPAT-NEXT:    .cfi_def_cfa_offset 16
 ; COMPAT-NEXT:    .cfi_offset w30, -16
 ; COMPAT-NEXT:    bl foo
@@ -118,8 +118,8 @@ define i32 @non_leaf_sign_non_leaf(i32 %x) "sign-return-address"="non-leaf"  {
 ; V83A-LABEL: non_leaf_sign_non_leaf:
 ; V83A:       // %bb.0:
 ; V83A-NEXT:    paciasp
-; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; V83A-NEXT:    .cfi_negate_ra_state
+; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; V83A-NEXT:    .cfi_def_cfa_offset 16
 ; V83A-NEXT:    .cfi_offset w30, -16
 ; V83A-NEXT:    bl foo
@@ -136,8 +136,8 @@ define i32 @non_leaf_scs(i32 %x) "sign-return-address"="non-leaf" shadowcallstac
 ; CHECK-NEXT:    str x30, [x18], #8
 ; CHECK-NEXT:    .cfi_escape 0x16, 0x12, 0x02, 0x82, 0x78 //
 ; CHECK-NEXT:    paciasp
-; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_negate_ra_state
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl foo
@@ -164,8 +164,8 @@ define fastcc void @spill_lr_and_tail_call(i64 %x) "sign-return-address"="all" {
 ; COMPAT-LABEL: spill_lr_and_tail_call:
 ; COMPAT:       // %bb.0:
 ; COMPAT-NEXT:    hint #25
-; COMPAT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; COMPAT-NEXT:    .cfi_negate_ra_state
+; COMPAT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; COMPAT-NEXT:    .cfi_def_cfa_offset 16
 ; COMPAT-NEXT:    .cfi_offset w30, -16
 ; COMPAT-NEXT:    //APP
@@ -178,8 +178,8 @@ define fastcc void @spill_lr_and_tail_call(i64 %x) "sign-return-address"="all" {
 ; V83A-LABEL: spill_lr_and_tail_call:
 ; V83A:       // %bb.0:
 ; V83A-NEXT:    paciasp
-; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; V83A-NEXT:    .cfi_negate_ra_state
+; V83A-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; V83A-NEXT:    .cfi_def_cfa_offset 16
 ; V83A-NEXT:    .cfi_offset w30, -16
 ; V83A-NEXT:    //APP
diff --git a/llvm/test/CodeGen/MIR/AArch64/return-address-signing.mir b/llvm/test/CodeGen/MIR/AArch64/return-address-signing.mir
index a63bb8452ebb..d2b063a05713 100644
--- a/llvm/test/CodeGen/MIR/AArch64/return-address-signing.mir
+++ b/llvm/test/CodeGen/MIR/AArch64/return-address-signing.mir
@@ -12,6 +12,11 @@
   entry:
     ret i32 2
   }
+
+  define dso_local i32 @foobar() "sign-return-address"="all" "branch-protection-pauth-lr"="true" {
+  entry:
+    ret i32 2
+  }
 ...
 ---
 #CHECK: foo
@@ -46,3 +51,21 @@ body:             |
     RET_ReallyLR implicit killed $w0
 
 ...
+---
+#CHECK: foobar
+name:            foobar
+alignment:       4
+tracksRegLiveness: true
+frameInfo:
+  maxCallFrameSize: 0
+#CHECK:    frame-setup PACM
+#CHECK:    frame-setup PACIASP implicit-def $lr, implicit $lr, implicit $sp, pre-instr-symbol <mcsymbol >
+#CHECK:    frame-setup CFI_INSTRUCTION negate_ra_sign_state_with_pc
+#CHECK:    frame-destroy PACM
+#CHECK:    frame-destroy AUTIASP implicit-def $lr, implicit $lr, implicit $sp
+body:             |
+  bb.0.entry:
+    $w0 = MOVi32imm 2
+    RET_ReallyLR implicit killed $w0
+
+...
diff --git a/llvm/test/MC/AArch64/directives-case_insensitive.s b/llvm/test/MC/AArch64/directives-case_insensitive.s
index be92e00cfad1..35a90a1bffea 100644
--- a/llvm/test/MC/AArch64/directives-case_insensitive.s
+++ b/llvm/test/MC/AArch64/directives-case_insensitive.s
@@ -32,10 +32,12 @@ fred .REQ x5
 
 .CFI_STARTPROC
 .CFI_NEGATE_RA_STATE
+.CFI_NEGATE_RA_STATE_WITH_PC
 .CFI_B_KEY_FRAME
 .CFI_ENDPROC
 // CHECK: .cfi_startproc
 // CHECK: .cfi_negate_ra_state
+// CHECK: .cfi_negate_ra_state_with_pc
 // CHECK: .cfi_b_key_frame
 // CHECK: .cfi_endproc
 
diff --git a/llvm/test/MC/AArch64/negate_ra_state_with_pc.s b/llvm/test/MC/AArch64/negate_ra_state_with_pc.s
new file mode 100644
index 000000000000..44b8ab2df9a9
--- /dev/null
+++ b/llvm/test/MC/AArch64/negate_ra_state_with_pc.s
@@ -0,0 +1,7 @@
+//RUN: llvm-mc  -triple=aarch64-arm-none-eabi -o - %s | FileCheck %s
+
+// CHECK: .cfi_negate_ra_state_with_pc
+foo:
+  .cfi_startproc
+  .cfi_negate_ra_state_with_pc
+  .cfi_endproc
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp
index 17fb18fc6b4d..2be656547c92 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp
@@ -174,6 +174,7 @@ TEST(DWARFDebugFrame, InvalidCFIOpcodesTest) {
       dwarf::DW_CFA_MIPS_advance_loc8,
       dwarf::DW_CFA_GNU_window_save,
       dwarf::DW_CFA_AARCH64_negate_ra_state,
+      dwarf::DW_CFA_AARCH64_negate_ra_state_with_pc,
       dwarf::DW_CFA_GNU_args_size};
 
   dwarf::CIE TestCIE = createCIE(/*IsDWARF64=*/false,
-- 
GitLab


From d7ca703eab7997814de425eaa4fd888563d78831 Mon Sep 17 00:00:00 2001
From: Thomas Fransham <tfransham@gmail.com>
Date: Mon, 28 Oct 2024 08:47:40 +0000
Subject: [PATCH 286/693] Add explicit symbol visibility macros to
 InstrProfData.inc (#110732)

Add explicit symbol visibility macros to InstrProfData.inc

Annotating these symbols will fix missing symbols for InstrProfTest when
using shared library builds on windows with explicit visibility macros
enabled.
Add a empty fallback definition for LLVM_ABI macro so the code works in
compiler-rt.

This is part of the work to enable LLVM_BUILD_LLVM_DYLIB and plugins on
window.

```
llvm\lld-link : error : undefined symbol: public: void ValueProfData::deserializeTo(InstrProfRecord&,  InstrProfSymtab*)
>>> referenced by unittests\ProfileData\InstrProfTest.cpp:1372 void ValueProfileReadWriteTest_value_prof_data_read_write_Test::TestBody()
```
---
 compiler-rt/include/profile/InstrProfData.inc   | 13 +++++++++++--
 llvm/include/llvm/ProfileData/InstrProfData.inc | 13 +++++++++++--
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index c66b0465a0b5..08ecaf0ed9fa 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -62,6 +62,15 @@
 #define INSTR_PROF_VISIBILITY
 #endif
 
+/* This include is needed for symbol visibility macros used on
+ * ValueProfRecord\ValueProfData so there functions are exported from the
+ * LLVM shared library on windows. */
+#ifdef __cplusplus
+#include "llvm/Support/Compiler.h"
+#else
+#define LLVM_ABI
+#endif
+
 // clang-format off:consider re-enabling clang-format if auto-formatted C macros
 // are readable (e.g., after `issue #82426` is fixed)
 /* INSTR_PROF_DATA start. */
@@ -373,7 +382,7 @@ INSTR_PROF_SECT_ENTRY(IPSK_covinit, \
  * This is the header of the data structure that defines the on-disk
  * layout of the value profile data of a particular kind for one function.
  */
-typedef struct ValueProfRecord {
+typedef struct LLVM_ABI ValueProfRecord {
   /* The kind of the value profile record. */
   uint32_t Kind;
   /*
@@ -423,7 +432,7 @@ typedef struct ValueProfRecord {
  * Per-function header/control data structure for value profiling
  * data in indexed format.
  */
-typedef struct ValueProfData {
+typedef struct LLVM_ABI ValueProfData {
   /*
    * Total size in bytes including this field. It must be a multiple
    * of sizeof(uint64_t).
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index c66b0465a0b5..08ecaf0ed9fa 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -62,6 +62,15 @@
 #define INSTR_PROF_VISIBILITY
 #endif
 
+/* This include is needed for symbol visibility macros used on
+ * ValueProfRecord\ValueProfData so there functions are exported from the
+ * LLVM shared library on windows. */
+#ifdef __cplusplus
+#include "llvm/Support/Compiler.h"
+#else
+#define LLVM_ABI
+#endif
+
 // clang-format off:consider re-enabling clang-format if auto-formatted C macros
 // are readable (e.g., after `issue #82426` is fixed)
 /* INSTR_PROF_DATA start. */
@@ -373,7 +382,7 @@ INSTR_PROF_SECT_ENTRY(IPSK_covinit, \
  * This is the header of the data structure that defines the on-disk
  * layout of the value profile data of a particular kind for one function.
  */
-typedef struct ValueProfRecord {
+typedef struct LLVM_ABI ValueProfRecord {
   /* The kind of the value profile record. */
   uint32_t Kind;
   /*
@@ -423,7 +432,7 @@ typedef struct ValueProfRecord {
  * Per-function header/control data structure for value profiling
  * data in indexed format.
  */
-typedef struct ValueProfData {
+typedef struct LLVM_ABI ValueProfData {
   /*
    * Total size in bytes including this field. It must be a multiple
    * of sizeof(uint64_t).
-- 
GitLab


From 14171b0b13534e10ac57d13d072970616e17391b Mon Sep 17 00:00:00 2001
From: Thomas Fransham <tfransham@gmail.com>
Date: Mon, 28 Oct 2024 08:52:22 +0000
Subject: [PATCH 287/693] [Remarks] Fix missing include in
 HotnessThresholdParser for Expected (#113755)

The use of Expected<> requires llvm/Support/Error.h to be included.
---
 llvm/include/llvm/Remarks/HotnessThresholdParser.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/include/llvm/Remarks/HotnessThresholdParser.h b/llvm/include/llvm/Remarks/HotnessThresholdParser.h
index 4cd0d2dff2fe..374992de5987 100644
--- a/llvm/include/llvm/Remarks/HotnessThresholdParser.h
+++ b/llvm/include/llvm/Remarks/HotnessThresholdParser.h
@@ -16,6 +16,7 @@
 #define LLVM_REMARKS_HOTNESSTHRESHOLDPARSER_H
 
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Error.h"
 #include <optional>
 
 namespace llvm {
-- 
GitLab


From 3d6923dbac16741736a841abee05f35bf7b0379a Mon Sep 17 00:00:00 2001
From: vabridgers <58314289+vabridgers@users.noreply.github.com>
Date: Mon, 28 Oct 2024 03:53:36 -0500
Subject: [PATCH 288/693] RFC: [clang-tidy] [analyzer] Move nondeterministic
 pointer usage check to tidy (#110471)

This change moves the `alpha.nondeterministic.PointerSorting` and
`alpha.nondeterministic.PointerIteration` static analyzer checkers to a
single `clang-tidy` check. Those checkers were implemented as simple
`clang-tidy` check-like code, wrapped in the static analyzer framework.
The documentation was updated to describe what the checks can and cannot
do, and testing was completed on a broad set of open-source projects.

Co-authored-by: Vince Bridgers <vince.a.bridgers@ericsson.com>
---
 .../bugprone/BugproneTidyModule.cpp           |   3 +
 .../clang-tidy/bugprone/CMakeLists.txt        |   1 +
 ...eterministicPointerIterationOrderCheck.cpp |  79 +++++++++
 ...ndeterministicPointerIterationOrderCheck.h |  39 +++++
 clang-tools-extra/docs/ReleaseNotes.rst       |   6 +
 ...ndeterministic-pointer-iteration-order.rst |  44 +++++
 .../docs/clang-tidy/checks/list.rst           |   1 +
 .../system-header-simulator/sim_algorithm     |  31 ++++
 .../system-header-simulator/sim_c++config.h   |  11 ++
 .../sim_initializer_list                      |  39 +++++
 .../system-header-simulator/sim_iterator_base |  22 +++
 .../Inputs/system-header-simulator/sim_map    |  34 ++++
 .../Inputs/system-header-simulator/sim_set    |  44 +++++
 .../system-header-simulator/sim_stl_pair      |  32 ++++
 .../system-header-simulator/sim_type_traits   |  19 +++
 .../system-header-simulator/sim_unordered_map |  33 ++++
 .../system-header-simulator/sim_unordered_set |  35 ++++
 .../Inputs/system-header-simulator/sim_vector | 150 ++++++++++++++++++
 ...ndeterministic-pointer-iteration-order.cpp |  84 ++++++++++
 clang/docs/ReleaseNotes.rst                   |   6 +
 clang/docs/analyzer/checkers.rst              |  31 ----
 .../clang/StaticAnalyzer/Checkers/Checkers.td |  18 ---
 .../StaticAnalyzer/Checkers/CMakeLists.txt    |   2 -
 .../Checkers/PointerIterationChecker.cpp      | 101 ------------
 .../Checkers/PointerSortingChecker.cpp        | 115 --------------
 clang/test/Analysis/ptr-iter.cpp              |  28 ----
 clang/test/Analysis/ptr-sort.cpp              |  36 -----
 27 files changed, 713 insertions(+), 331 deletions(-)
 create mode 100644 clang-tools-extra/clang-tidy/bugprone/NondeterministicPointerIterationOrderCheck.cpp
 create mode 100644 clang-tools-extra/clang-tidy/bugprone/NondeterministicPointerIterationOrderCheck.h
 create mode 100644 clang-tools-extra/docs/clang-tidy/checks/bugprone/nondeterministic-pointer-iteration-order.rst
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_algorithm
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_c++config.h
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_initializer_list
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_iterator_base
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_map
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_set
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_stl_pair
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_type_traits
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_unordered_map
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_unordered_set
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_vector
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/bugprone/nondeterministic-pointer-iteration-order.cpp
 delete mode 100644 clang/lib/StaticAnalyzer/Checkers/PointerIterationChecker.cpp
 delete mode 100644 clang/lib/StaticAnalyzer/Checkers/PointerSortingChecker.cpp
 delete mode 100644 clang/test/Analysis/ptr-iter.cpp
 delete mode 100644 clang/test/Analysis/ptr-sort.cpp

diff --git a/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp b/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
index 9120c4b6c0d9..33ac65e715ce 100644
--- a/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
@@ -49,6 +49,7 @@
 #include "MultipleStatementMacroCheck.h"
 #include "NoEscapeCheck.h"
 #include "NonZeroEnumToBoolConversionCheck.h"
+#include "NondeterministicPointerIterationOrderCheck.h"
 #include "NotNullTerminatedResultCheck.h"
 #include "OptionalValueConversionCheck.h"
 #include "ParentVirtualCallCheck.h"
@@ -174,6 +175,8 @@ public:
         "bugprone-multiple-new-in-one-expression");
     CheckFactories.registerCheck<MultipleStatementMacroCheck>(
         "bugprone-multiple-statement-macro");
+    CheckFactories.registerCheck<NondeterministicPointerIterationOrderCheck>(
+        "bugprone-nondeterministic-pointer-iteration-order");
     CheckFactories.registerCheck<OptionalValueConversionCheck>(
         "bugprone-optional-value-conversion");
     CheckFactories.registerCheck<PointerArithmeticOnPolymorphicObjectCheck>(
diff --git a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
index f0667bbfdd87..b0a2318acc05 100644
--- a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
@@ -45,6 +45,7 @@ add_clang_library(clangTidyBugproneModule STATIC
   MultipleNewInOneExpressionCheck.cpp
   MultipleStatementMacroCheck.cpp
   NoEscapeCheck.cpp
+  NondeterministicPointerIterationOrderCheck.cpp
   NonZeroEnumToBoolConversionCheck.cpp
   NotNullTerminatedResultCheck.cpp
   OptionalValueConversionCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/bugprone/NondeterministicPointerIterationOrderCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/NondeterministicPointerIterationOrderCheck.cpp
new file mode 100644
index 000000000000..0797376d0aa9
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/bugprone/NondeterministicPointerIterationOrderCheck.cpp
@@ -0,0 +1,79 @@
+//===----- NondeterministicPointerIterationOrderCheck.cpp - clang-tidy ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NondeterministicPointerIterationOrderCheck.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/Lex/Lexer.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang::tidy::bugprone {
+
+void NondeterministicPointerIterationOrderCheck::registerMatchers(
+    MatchFinder *Finder) {
+
+  auto LoopVariable = varDecl(hasType(
+      qualType(hasCanonicalType(anyOf(referenceType(), pointerType())))));
+
+  auto RangeInit = declRefExpr(to(varDecl(
+      hasType(recordDecl(hasAnyName("std::unordered_set", "std::unordered_map",
+                                    "std::unordered_multiset",
+                                    "std::unordered_multimap"))
+                  .bind("recorddecl")))));
+
+  Finder->addMatcher(cxxForRangeStmt(hasLoopVariable(LoopVariable),
+                                     hasRangeInit(RangeInit.bind("rangeinit")))
+                         .bind("cxxForRangeStmt"),
+                     this);
+
+  auto SortFuncM = callee(functionDecl(hasAnyName(
+      "std::is_sorted", "std::nth_element", "std::sort", "std::partial_sort",
+      "std::partition", "std::stable_partition", "std::stable_sort")));
+
+  auto IteratesPointerEltsM = hasArgument(
+      0,
+      cxxMemberCallExpr(on(hasType(cxxRecordDecl(has(fieldDecl(hasType(qualType(
+          hasCanonicalType(pointsTo(hasCanonicalType(pointerType()))))))))))));
+
+  Finder->addMatcher(
+      callExpr(allOf(SortFuncM, IteratesPointerEltsM)).bind("sortsemantic"),
+      this);
+}
+
+void NondeterministicPointerIterationOrderCheck::check(
+    const MatchFinder::MatchResult &Result) {
+  const auto *ForRangePointers =
+      Result.Nodes.getNodeAs<CXXForRangeStmt>("cxxForRangeStmt");
+
+  if ((ForRangePointers) && !(ForRangePointers->getBeginLoc().isMacroID())) {
+    const auto *RangeInit = Result.Nodes.getNodeAs<Stmt>("rangeinit");
+    if (const auto *ClassTemplate =
+            Result.Nodes.getNodeAs<ClassTemplateSpecializationDecl>(
+                "recorddecl")) {
+      const TemplateArgumentList &TemplateArgs =
+          ClassTemplate->getTemplateArgs();
+      const llvm::StringRef AlgoName = ClassTemplate->getName();
+      const bool IsAlgoArgPointer =
+          TemplateArgs[0].getAsType()->isPointerType();
+
+      if (IsAlgoArgPointer) {
+        SourceRange R = RangeInit->getSourceRange();
+        diag(R.getBegin(), "iteration of pointers is nondeterministic") << R;
+      }
+    }
+    return;
+  }
+  const auto *SortPointers = Result.Nodes.getNodeAs<Stmt>("sortsemantic");
+
+  if ((SortPointers) && !(SortPointers->getBeginLoc().isMacroID())) {
+    SourceRange R = SortPointers->getSourceRange();
+    diag(R.getBegin(), "sorting pointers is nondeterministic") << R;
+  }
+}
+
+} // namespace clang::tidy::bugprone
diff --git a/clang-tools-extra/clang-tidy/bugprone/NondeterministicPointerIterationOrderCheck.h b/clang-tools-extra/clang-tidy/bugprone/NondeterministicPointerIterationOrderCheck.h
new file mode 100644
index 000000000000..698872fefca9
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/bugprone/NondeterministicPointerIterationOrderCheck.h
@@ -0,0 +1,39 @@
+//=== NondeterministicPointerIterationOrderCheck.h - clang-tidy -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_NONDETERMINISTIC_POINTER_ITERATION_ORDER_CHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_NONDETERMINISTIC_POINTER_ITERATION_ORDER_CHECK_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang::tidy::bugprone {
+
+/// Finds nondeterministic usages of pointers in unordered containers. The
+/// check also finds calls to sorting-like algorithms on a container of
+/// pointers.
+///
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/bugprone/nondeterministic-pointer-iteration-order.html
+class NondeterministicPointerIterationOrderCheck : public ClangTidyCheck {
+public:
+  NondeterministicPointerIterationOrderCheck(StringRef Name,
+                                             ClangTidyContext *Context)
+      : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  std::optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
+};
+
+} // namespace clang::tidy::bugprone
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_NONDETERMINISTIC_POINTER_ITERATION_ORDER_CHECK_H
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 4cc4c2146d7e..7eb2ee511a05 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -119,6 +119,12 @@ New checks
   Warns about code that tries to cast between pointers by means of
   ``std::bit_cast`` or ``memcpy``.
 
+- New :doc:`bugprone-nondeterministic-pointer-iteration-order
+  <clang-tidy/checks/bugprone/nondeterministic-pointer-iteration-order>`
+  check.
+
+  Finds nondeterministic usages of pointers in unordered containers.
+
 - New :doc:`bugprone-tagged-union-member-count
   <clang-tidy/checks/bugprone/tagged-union-member-count>` check.
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/nondeterministic-pointer-iteration-order.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/nondeterministic-pointer-iteration-order.rst
new file mode 100644
index 000000000000..41be0bf1c677
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/nondeterministic-pointer-iteration-order.rst
@@ -0,0 +1,44 @@
+.. title:: clang-tidy - bugprone-nondeterministic-pointer-iteration-order
+
+bugprone-nondeterministic-pointer-iteration-order
+=================================================
+
+Finds nondeterministic usages of pointers in unordered containers.
+
+One canonical example is iteration across a container of pointers.
+
+.. code-block:: c++
+
+  {
+    int a = 1, b = 2;
+    std::unordered_set<int *> UnorderedPtrSet = {&a, &b};
+    for (auto i : UnorderedPtrSet)
+      f(i);
+  }
+
+Another such example is sorting a container of pointers.
+
+.. code-block:: c++
+
+  {
+    int a = 1, b = 2;
+    std::vector<int *> VectorOfPtr = {&a, &b};
+    std::sort(VectorOfPtr.begin(), VectorOfPtr.end());
+  }
+
+Iteration of a containers of pointers may present the order of different
+pointers differently across different runs of a program. In some cases this
+may be acceptable behavior, in others this may be unexpected behavior. This
+check is advisory for this reason.
+
+This check only detects range-based for loops over unordered sets and maps. It
+also detects calls sorting-like algorithms on containers holding pointers.
+Other similar usages will not be found and are false negatives.
+
+Limitations:
+
+* This check currently does not check if a nondeterministic iteration order is
+  likely to be a mistake, and instead marks all such iterations as bugprone.
+* std::reference_wrapper is not considered yet.
+* Only for loops are considered, other iterators can be included in
+  improvements.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index 0082234f5ed3..d731b13fc0df 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -115,6 +115,7 @@ Clang-Tidy Checks
    :doc:`bugprone-multiple-new-in-one-expression <bugprone/multiple-new-in-one-expression>`,
    :doc:`bugprone-multiple-statement-macro <bugprone/multiple-statement-macro>`,
    :doc:`bugprone-no-escape <bugprone/no-escape>`,
+   :doc:`bugprone-nondeterministic-pointer-iteration-order <bugprone/nondeterministic-pointer-iteration-order>`,
    :doc:`bugprone-non-zero-enum-to-bool-conversion <bugprone/non-zero-enum-to-bool-conversion>`,
    :doc:`bugprone-not-null-terminated-result <bugprone/not-null-terminated-result>`, "Yes"
    :doc:`bugprone-optional-value-conversion <bugprone/optional-value-conversion>`, "Yes"
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_algorithm b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_algorithm
new file mode 100644
index 000000000000..6dbca55a8e36
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_algorithm
@@ -0,0 +1,31 @@
+#ifndef _SIM_ALGORITHM
+#define _SIM_ALGORITHM
+
+#pragma clang system_header
+
+namespace std {
+
+template<class ForwardIt>
+bool is_sorted(ForwardIt first, ForwardIt last);
+
+template <class RandomIt>
+void nth_element(RandomIt first, RandomIt nth, RandomIt last);
+
+template<class RandomIt>
+void partial_sort(RandomIt first, RandomIt middle, RandomIt last);
+
+template<class RandomIt>
+void sort (RandomIt first, RandomIt last);
+
+template<class RandomIt>
+void stable_sort(RandomIt first, RandomIt last);
+
+template<class BidirIt, class UnaryPredicate>
+BidirIt partition(BidirIt first, BidirIt last, UnaryPredicate p);
+
+template<class BidirIt, class UnaryPredicate>
+BidirIt stable_partition(BidirIt first, BidirIt last, UnaryPredicate p);
+
+} // namespace std
+
+#endif // _SIM_ALGORITHM
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_c++config.h b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_c++config.h
new file mode 100644
index 000000000000..ba98e0cc2208
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_c++config.h
@@ -0,0 +1,11 @@
+#ifndef _SIM_CPP_CONFIG_H
+#define _SIM_CPP_CONFIG_H
+
+#pragma clang system_header
+
+typedef unsigned char uint8_t;
+
+typedef __typeof__(sizeof(int)) size_t;
+typedef __typeof__((char*)0-(char*)0) ptrdiff_t;
+
+#endif // _SIM_CPP_CONFIG_H
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_initializer_list b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_initializer_list
new file mode 100644
index 000000000000..e4d9d534b3bd
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_initializer_list
@@ -0,0 +1,39 @@
+#ifndef _INITIALIZER_LIST
+#define _INITIALIZER_LIST
+
+#pragma clang system_header
+#
+#include "sim_c++config.h" // size_t
+
+namespace std {
+
+template <class _E>
+class initializer_list  {
+  const _E* __begin_;
+  size_t    __size_;
+
+  initializer_list(const _E* __b, size_t __s)
+    : __begin_(__b),
+      __size_(__s)
+  {}
+
+public:
+  typedef _E        value_type;
+  typedef const _E& reference;
+  typedef const _E& const_reference;
+  typedef size_t    size_type;
+
+  typedef const _E* iterator;
+  typedef const _E* const_iterator;
+
+  initializer_list() : __begin_(0), __size_(0) {}
+
+  size_t    size()  const {return __size_;}
+  const _E* begin() const {return __begin_;}
+  const _E* end()   const {return __begin_ + __size_;}
+
+}; // class initializer_list
+
+} // namespace std
+
+#endif // _INITIALIZER_LIST
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_iterator_base b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_iterator_base
new file mode 100644
index 000000000000..3b205d1722c9
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_iterator_base
@@ -0,0 +1,22 @@
+#ifndef _SIM_ITERATOR_BASE
+#define _SIM_ITERATOR_BASE
+
+namespace std {
+
+struct input_iterator_tag { };
+struct output_iterator_tag { };
+struct forward_iterator_tag : public input_iterator_tag { };
+struct bidirectional_iterator_tag : public forward_iterator_tag { };
+struct random_access_iterator_tag : public bidirectional_iterator_tag { };
+
+template <typename Iterator> struct iterator_traits {
+  typedef typename Iterator::difference_type difference_type;
+  typedef typename Iterator::value_type value_type;
+  typedef typename Iterator::pointer pointer;
+  typedef typename Iterator::reference reference;
+  typedef typename Iterator::iterator_category iterator_category;
+};
+
+} // namespace std
+
+#endif // _SIM_ITERATOR_BASE
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_map b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_map
new file mode 100644
index 000000000000..8c57f5c71f88
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_map
@@ -0,0 +1,34 @@
+
+#ifndef _SIM_MAP
+#define _SIM_MAP
+
+#pragma clang system_header
+#include "sim_stl_pair"
+
+namespace std {
+
+template <typename Key, typename Value>
+class map {
+  public:
+    using value_type = pair<Key, Value>;
+    map();
+    map(initializer_list<pair<Key, Value>> initList);
+    value_type& operator[](const Key& key);
+    value_type& operator[](Key&& key);
+    class iterator {
+    public:
+      iterator(Key *key): ptr(key) {}
+      iterator& operator++() { ++ptr; return *this; }
+      bool operator!=(const iterator &other) const { return ptr != other.ptr; }
+      const Key &operator*() const { return *ptr; }
+    private:
+      Key *ptr;
+    };
+    Key *val;
+    iterator begin() const { return iterator(val); }
+    iterator end() const { return iterator(val + 1); }
+};
+
+} // namespace std
+
+#endif // _SIM_MAP
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_set b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_set
new file mode 100644
index 000000000000..f2f700955389
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_set
@@ -0,0 +1,44 @@
+
+#ifndef _SIM_SET
+#define _SIM_SET
+
+#pragma clang system_header
+#include "sim_initializer_list"
+
+namespace std {
+
+template< class T = void >
+struct less;
+
+template< class T >
+struct allocator;
+
+template< class Key >
+struct hash;
+
+template<
+  class Key,
+  class Compare = std::less<Key>,
+  class Alloc = std::allocator<Key>
+> class set {
+  public:
+    set(initializer_list<Key> __list) {}
+
+    class iterator {
+    public:
+      iterator(Key *key): ptr(key) {}
+      iterator& operator++() { ++ptr; return *this; }
+      bool operator!=(const iterator &other) const { return ptr != other.ptr; }
+      const Key &operator*() const { return *ptr; }
+    private:
+      Key *ptr;
+    };
+
+    Key *val;
+    iterator begin() const { return iterator(val); }
+    iterator end() const { return iterator(val + 1); }
+};
+
+} // namespace std
+
+#endif // _SIM_SET
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_stl_pair b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_stl_pair
new file mode 100644
index 000000000000..d244bb363b86
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_stl_pair
@@ -0,0 +1,32 @@
+#ifndef _SIM_STL_PAIR
+#define _SIM_STL_PAIR
+
+#pragma clang system_header
+
+#include "sim_type_traits"
+
+namespace std {
+
+template <class T1, class T2>
+struct pair {
+  T1 first;
+  T2 second;
+
+  pair() : first(), second() {}
+  pair(const T1 &a, const T2 &b) : first(a), second(b) {}
+
+  template<class U1, class U2>
+  pair(const pair<U1, U2> &other) : first(other.first),
+                                      second(other.second) {}
+};
+
+template <typename T1, typename T2>
+pair<typename remove_reference<T1>::type, typename remove_reference<T2>::type>
+make_pair(T1 &&, T2 &&) {
+  return {};
+};
+
+} // namespace std
+
+#endif // _SIM_STL_PAIR
+
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_type_traits b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_type_traits
new file mode 100644
index 000000000000..f066767c4d98
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_type_traits
@@ -0,0 +1,19 @@
+
+#ifndef _SIM_TYPE_TRAITS
+#define _SIM_TYPE_TRAITS
+
+#pragma clang system_header
+namespace std {
+
+template< class T > struct remove_reference      {typedef T type;};
+template< class T > struct remove_reference<T&>  {typedef T type;};
+template< class T > struct remove_reference<T&&> {typedef T type;};
+
+template<typename T> typename remove_reference<T>::type&& move(T&& a);
+
+template< class T >
+using remove_reference_t = typename remove_reference<T>::type;
+
+} // namespace std
+
+#endif // _SIM_TYPE_TRAITS
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_unordered_map b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_unordered_map
new file mode 100644
index 000000000000..fabd8e7fd2d7
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_unordered_map
@@ -0,0 +1,33 @@
+#ifndef _SIM_UNORDERED_MAP
+#define _SIM_UNORDERED_MAP
+
+#pragma clang system_header
+#include "sim_initializer_list"
+
+namespace std {
+
+template <typename Key, typename Value>
+class unordered_map {
+public:
+  using value_type = pair<Key, Value>;
+  unordered_map();
+  unordered_map(initializer_list<pair<Key, Value>> initList);
+  value_type& operator[](const Key& key);
+  value_type& operator[](Key&& key);
+  class iterator {
+  public:
+    iterator(Key *key): ptr(key) {}
+    iterator& operator++() { ++ptr; return *this; }
+    bool operator!=(const iterator &other) const { return ptr != other.ptr; }
+    const Key &operator*() const { return *ptr; }
+  private:
+    Key *ptr;
+  };
+  Key *val;
+  iterator begin() const { return iterator(val); }
+  iterator end() const { return iterator(val + 1); }
+};
+
+} // namespace std
+
+#endif // _SIM_UNORDERED_MAP
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_unordered_set b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_unordered_set
new file mode 100644
index 000000000000..a077507bbdcb
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_unordered_set
@@ -0,0 +1,35 @@
+#ifndef _SIM_UNORDERED_SET
+#define _SIM_UNORDERED_SET
+
+#pragma clang system_header
+#include "sim_initializer_list"
+
+namespace std {
+
+template<
+  class Key,
+  class Hash = std::hash<Key>,
+  class Compare = std::less<Key>,
+  class Alloc = std::allocator<Key>
+> class unordered_set {
+  public:
+    unordered_set(initializer_list<Key> __list) {}
+
+    class iterator {
+    public:
+      iterator(Key *key): ptr(key) {}
+      iterator& operator++() { ++ptr; return *this; }
+      bool operator!=(const iterator &other) const { return ptr != other.ptr; }
+      const Key &operator*() const { return *ptr; }
+    private:
+      Key *ptr;
+    };
+
+    Key *val;
+    iterator begin() const { return iterator(val); }
+    iterator end() const { return iterator(val + 1); }
+};
+
+} // namespace std
+
+#endif // _SIM_UNORDERED_SET
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_vector b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_vector
new file mode 100644
index 000000000000..dfa9abfb8863
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/system-header-simulator/sim_vector
@@ -0,0 +1,150 @@
+#ifndef _SIM_VECTOR
+#define _SIM_VECTOR
+
+#pragma clang system_header
+
+#include "sim_iterator_base"
+
+namespace std {
+
+template <typename T, typename Ptr, typename Ref> struct __vector_iterator {
+  typedef __vector_iterator<T, T *, T &> iterator;
+  typedef __vector_iterator<T, const T *, const T &> const_iterator;
+
+  typedef ptrdiff_t difference_type;
+  typedef T value_type;
+  typedef Ptr pointer;
+  typedef Ref reference;
+  typedef std::random_access_iterator_tag iterator_category;
+
+  __vector_iterator(const Ptr p = 0) : ptr(p) {}
+  __vector_iterator(const iterator &rhs): ptr(rhs.base()) {}
+  __vector_iterator<T, Ptr, Ref>& operator++() { ++ ptr; return *this; }
+  __vector_iterator<T, Ptr, Ref> operator++(int) {
+    auto tmp = *this;
+    ++ ptr;
+    return tmp;
+  }
+  __vector_iterator<T, Ptr, Ref> operator--() { -- ptr; return *this; }
+  __vector_iterator<T, Ptr, Ref> operator--(int) {
+    auto tmp = *this; -- ptr;
+    return tmp;
+  }
+  __vector_iterator<T, Ptr, Ref> operator+(difference_type n) {
+    return ptr + n;
+  }
+  friend __vector_iterator<T, Ptr, Ref> operator+(
+      difference_type n,
+      const __vector_iterator<T, Ptr, Ref> &iter) {
+    return n + iter.ptr;
+  }
+  __vector_iterator<T, Ptr, Ref> operator-(difference_type n) {
+    return ptr - n;
+  }
+  __vector_iterator<T, Ptr, Ref> operator+=(difference_type n) {
+    return ptr += n;
+  }
+  __vector_iterator<T, Ptr, Ref> operator-=(difference_type n) {
+    return ptr -= n;
+  }
+
+  template<typename U, typename Ptr2, typename Ref2>
+  difference_type operator-(const __vector_iterator<U, Ptr2, Ref2> &rhs);
+
+  Ref operator*() const { return *ptr; }
+  Ptr operator->() const { return ptr; }
+
+  Ref operator[](difference_type n) {
+    return *(ptr+n);
+  }
+
+  bool operator==(const iterator &rhs) const { return ptr == rhs.ptr; }
+  bool operator==(const const_iterator &rhs) const { return ptr == rhs.ptr; }
+
+  bool operator!=(const iterator &rhs) const { return ptr != rhs.ptr; }
+  bool operator!=(const const_iterator &rhs) const { return ptr != rhs.ptr; }
+
+  const Ptr& base() const { return ptr; }
+
+private:
+  Ptr ptr;
+};
+
+template<typename T>
+class vector {
+  T *_start;
+  T *_finish;
+  T *_end_of_storage;
+
+public:
+  typedef T value_type;
+  typedef size_t size_type;
+  typedef __vector_iterator<T, T *, T &> iterator;
+  typedef __vector_iterator<T, const T *, const T &> const_iterator;
+
+  vector() : _start(0), _finish(0), _end_of_storage(0) {}
+  template <typename InputIterator>
+  vector(InputIterator first, InputIterator last);
+  vector(const vector &other);
+  vector(vector &&other);
+  ~vector();
+
+  size_t size() const {
+    return size_t(_finish - _start);
+  }
+
+  vector& operator=(const vector &other);
+  vector& operator=(vector &&other);
+  vector& operator=(std::initializer_list<T> ilist);
+
+  void assign(size_type count, const T &value);
+  template <typename InputIterator >
+  void assign(InputIterator first, InputIterator last);
+  void assign(std::initializer_list<T> ilist);
+
+  void clear();
+
+  void push_back(const T &value);
+  void push_back(T &&value);
+  template<class... Args>
+  void emplace_back(Args&&... args);
+  void pop_back();
+
+  iterator insert(const_iterator position, const value_type &val);
+  iterator insert(const_iterator position, size_type n,
+                  const value_type &val);
+  template <typename InputIterator>
+  iterator insert(const_iterator position, InputIterator first,
+                  InputIterator last);
+  iterator insert(const_iterator position, value_type &&val);
+  iterator insert(const_iterator position, initializer_list<value_type> il);
+
+  template <class... Args>
+  iterator emplace(const_iterator position, Args&&... args);
+
+  iterator erase(const_iterator position);
+  iterator erase(const_iterator first, const_iterator last);
+
+  T &operator[](size_t n) {
+    return _start[n];
+  }
+
+  const T &operator[](size_t n) const {
+    return _start[n];
+  }
+
+  iterator begin() { return iterator(_start); }
+  const_iterator begin() const { return const_iterator(_start); }
+  const_iterator cbegin() const { return const_iterator(_start); }
+  iterator end() { return iterator(_finish); }
+  const_iterator end() const { return const_iterator(_finish); }
+  const_iterator cend() const { return const_iterator(_finish); }
+  T& front() { return *begin(); }
+  const T& front() const { return *begin(); }
+  T& back() { return *(end() - 1); }
+  const T& back() const { return *(end() - 1); }
+};
+
+} // namespace std
+
+#endif // _SIM_VECTOR
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/nondeterministic-pointer-iteration-order.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/nondeterministic-pointer-iteration-order.cpp
new file mode 100644
index 000000000000..91853874d0af
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/nondeterministic-pointer-iteration-order.cpp
@@ -0,0 +1,84 @@
+// RUN: %check_clang_tidy %s bugprone-nondeterministic-pointer-iteration-order %t -- -- -I%S -std=c++!4
+
+#include "Inputs/system-header-simulator/sim_set"
+#include "Inputs/system-header-simulator/sim_unordered_set"
+#include "Inputs/system-header-simulator/sim_map"
+#include "Inputs/system-header-simulator/sim_unordered_map"
+#include "Inputs/system-header-simulator/sim_vector"
+#include "Inputs/system-header-simulator/sim_algorithm"
+
+template<class T>
+void f(T x);
+
+void PointerIteration() {
+  int a = 1, b = 2;
+  std::set<int> OrderedIntSet = {a, b};
+  std::set<int *> OrderedPtrSet = {&a, &b};
+  std::unordered_set<int> UnorderedIntSet = {a, b};
+  std::unordered_set<int *> UnorderedPtrSet = {&a, &b};
+  std::map<int, int> IntMap = { std::make_pair(a,a), std::make_pair(b,b) };
+  std::map<int*, int*> PtrMap = { std::make_pair(&a,&a), std::make_pair(&b,&b) };
+  std::unordered_map<int, int> IntUnorderedMap = { std::make_pair(a,a), std::make_pair(b,b) };
+  std::unordered_map<int*, int*> PtrUnorderedMap = { std::make_pair(&a,&a), std::make_pair(&b,&b) };
+
+  for (auto i : OrderedIntSet) // no-warning
+    f(i);
+
+  for (auto i : OrderedPtrSet) // no-warning
+    f(i);
+
+  for (auto i : UnorderedIntSet) // no-warning
+    f(i);
+
+  for (auto i : UnorderedPtrSet)
+    f(i);
+  // CHECK-MESSAGES: :[[@LINE-2]]:17: warning: iteration of pointers is nondeterministic
+
+  for (auto &i : UnorderedPtrSet)
+    f(i);
+  // CHECK-MESSAGES: :[[@LINE-2]]:18: warning: iteration of pointers is nondeterministic
+
+  for (auto &i : IntMap) // no-warning
+    f(i);
+
+  for (auto &i : PtrMap) // no-warning
+    f(i);
+
+  for (auto &i : IntUnorderedMap) // no-warning
+    f(i);
+
+  for (auto &i : PtrUnorderedMap)
+    f(i);
+  // CHECK-MESSAGES: :[[@LINE-2]]:18: warning: iteration of pointers is nondeterministic
+}
+
+bool g (int *x) { return true; }
+bool h (int x) { return true; }
+
+void PointerSorting() {
+  int a = 1, b = 2, c = 3;
+  std::vector<int> V1 = {a, b};
+  std::vector<int *> V2 = {&a, &b};
+
+  std::is_sorted(V1.begin(), V1.end());                    // no-warning
+  std::nth_element(V1.begin(), V1.begin() + 1, V1.end());  // no-warning
+  std::partial_sort(V1.begin(), V1.begin() + 1, V1.end()); // no-warning
+  std::sort(V1.begin(), V1.end());                         // no-warning
+  std::stable_sort(V1.begin(), V1.end());                  // no-warning
+  std::partition(V1.begin(), V1.end(), h);                 // no-warning
+  std::stable_partition(V1.begin(), V1.end(), h);          // no-warning
+  std::is_sorted(V2.begin(), V2.end());
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: sorting pointers is nondeterministic
+  std::nth_element(V2.begin(), V2.begin() + 1, V2.end());
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: sorting pointers is nondeterministic
+  std::partial_sort(V2.begin(), V2.begin() + 1, V2.end());
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: sorting pointers is nondeterministic
+  std::sort(V2.begin(), V2.end());
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: sorting pointers is nondeterministic
+  std::stable_sort(V2.begin(), V2.end());
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: sorting pointers is nondeterministic
+  std::partition(V2.begin(), V2.end(), g);
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: sorting pointers is nondeterministic
+  std::stable_partition(V2.begin(), V2.end(), g);
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: sorting pointers is nondeterministic
+}
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 31ee4f7e516f..920a2369f964 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -786,6 +786,12 @@ Moved checkers
   To detect too large arguments passed to malloc, consider using the checker
   ``alpha.taint.TaintedAlloc``.
 
+- The checkers ``alpha.nondeterministic.PointerSorting`` and
+  ``alpha.nondeterministic.PointerIteration`` were moved to a new bugprone
+  checker named ``bugprone-nondeterministic-pointer-iteration-order``. The
+  original checkers were implemented only using AST matching and make more
+  sense as a single clang-tidy check.
+
 .. _release-notes-sanitizers:
 
 Sanitizers
diff --git a/clang/docs/analyzer/checkers.rst b/clang/docs/analyzer/checkers.rst
index 58dbd686a6dc..87b03438e6e0 100644
--- a/clang/docs/analyzer/checkers.rst
+++ b/clang/docs/analyzer/checkers.rst
@@ -3447,37 +3447,6 @@ Limitations:
 
      More details at the corresponding `GitHub issue <https://github.com/llvm/llvm-project/issues/43459>`_.
 
-.. _alpha-nondeterminism-PointerIteration:
-
-alpha.nondeterminism.PointerIteration (C++)
-"""""""""""""""""""""""""""""""""""""""""""
-Check for non-determinism caused by iterating unordered containers of pointers.
-
-.. code-block:: c
-
- void test() {
-  int a = 1, b = 2;
-  std::unordered_set<int *> UnorderedPtrSet = {&a, &b};
-
-  for (auto i : UnorderedPtrSet) // warn
-    f(i);
- }
-
-.. _alpha-nondeterminism-PointerSorting:
-
-alpha.nondeterminism.PointerSorting (C++)
-"""""""""""""""""""""""""""""""""""""""""
-Check for non-determinism caused by sorting of pointers.
-
-.. code-block:: c
-
- void test() {
-  int a = 1, b = 2;
-  std::vector<int *> V = {&a, &b};
-  std::sort(V.begin(), V.end()); // warn
- }
-
-
 alpha.WebKit
 ^^^^^^^^^^^^
 
diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
index 349040c15eeb..9a6b35c1b9f7 100644
--- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
+++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
@@ -118,8 +118,6 @@ def Debug : Package<"debug">, Hidden;
 
 def CloneDetectionAlpha : Package<"clone">, ParentPackage<Alpha>;
 
-def NonDeterminismAlpha : Package<"nondeterminism">, ParentPackage<Alpha>;
-
 def Fuchsia : Package<"fuchsia">;
 def FuchsiaAlpha : Package<"fuchsia">, ParentPackage<Alpha>;
 
@@ -1711,22 +1709,6 @@ def TaintedDivChecker: Checker<"TaintedDiv">,
 
 } // end "optin.taint"
 
-//===----------------------------------------------------------------------===//
-// NonDeterminism checkers.
-//===----------------------------------------------------------------------===//
-
-let ParentPackage = NonDeterminismAlpha in {
-
-def PointerIterationChecker : Checker<"PointerIteration">,
-  HelpText<"Checks for non-determinism caused by iteration of unordered containers of pointers">,
-  Documentation<HasDocumentation>;
-
-def PointerSortingChecker : Checker<"PointerSorting">,
-  HelpText<"Check for non-determinism caused by sorting of pointers">,
-  Documentation<HasDocumentation>;
-
-} // end alpha.nondeterminism
-
 //===----------------------------------------------------------------------===//
 // Fuchsia checkers.
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
index 6da3665ab9a4..62aa5ff7f002 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
+++ b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
@@ -91,8 +91,6 @@ add_clang_library(clangStaticAnalyzerCheckers
   OSObjectCStyleCast.cpp
   PaddingChecker.cpp
   PointerArithChecker.cpp
-  PointerIterationChecker.cpp
-  PointerSortingChecker.cpp
   PointerSubChecker.cpp
   PthreadLockChecker.cpp
   PutenvStackArrayChecker.cpp
diff --git a/clang/lib/StaticAnalyzer/Checkers/PointerIterationChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/PointerIterationChecker.cpp
deleted file mode 100644
index 895b2160b76a..000000000000
--- a/clang/lib/StaticAnalyzer/Checkers/PointerIterationChecker.cpp
+++ /dev/null
@@ -1,101 +0,0 @@
-//== PointerIterationChecker.cpp ------------------------------- -*- C++ -*--=//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines PointerIterationChecker which checks for non-determinism
-// caused due to iteration of unordered containers of pointer elements.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/ASTMatchers/ASTMatchFinder.h"
-#include "clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h"
-#include "clang/StaticAnalyzer/Core/Checker.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
-
-using namespace clang;
-using namespace ento;
-using namespace ast_matchers;
-
-namespace {
-
-// ID of a node at which the diagnostic would be emitted.
-constexpr llvm::StringLiteral WarnAtNode = "iter";
-
-class PointerIterationChecker : public Checker<check::ASTCodeBody> {
-public:
-  void checkASTCodeBody(const Decl *D,
-                        AnalysisManager &AM,
-                        BugReporter &BR) const;
-};
-
-static void emitDiagnostics(const BoundNodes &Match, const Decl *D,
-                            BugReporter &BR, AnalysisManager &AM,
-                            const PointerIterationChecker *Checker) {
-  auto *ADC = AM.getAnalysisDeclContext(D);
-
-  const auto *MarkedStmt = Match.getNodeAs<Stmt>(WarnAtNode);
-  assert(MarkedStmt);
-
-  auto Range = MarkedStmt->getSourceRange();
-  auto Location = PathDiagnosticLocation::createBegin(MarkedStmt,
-                                                      BR.getSourceManager(),
-                                                      ADC);
-  std::string Diagnostics;
-  llvm::raw_string_ostream OS(Diagnostics);
-  OS << "Iteration of pointer-like elements "
-     << "can result in non-deterministic ordering";
-
-  BR.EmitBasicReport(ADC->getDecl(), Checker,
-                     "Iteration of pointer-like elements", "Non-determinism",
-                     Diagnostics, Location, Range);
-}
-
-// Assumption: Iteration of ordered containers of pointers is deterministic.
-
-// TODO: Currently, we only check for std::unordered_set. Other unordered
-// containers like std::unordered_map also need to be handled.
-
-// TODO: Currently, we do not check what the for loop does with the iterated
-// pointer values. Not all iterations may cause non-determinism. For example,
-// counting or summing up the elements should not be non-deterministic.
-
-auto matchUnorderedIterWithPointers() -> decltype(decl()) {
-
-  auto UnorderedContainerM = declRefExpr(to(varDecl(hasType(
-                               recordDecl(hasName("std::unordered_set")
-                             )))));
-
-  auto PointerTypeM = varDecl(hasType(hasCanonicalType(pointerType())));
-
-  auto PointerIterM = stmt(cxxForRangeStmt(
-                             hasLoopVariable(PointerTypeM),
-                             hasRangeInit(UnorderedContainerM)
-                      )).bind(WarnAtNode);
-
-  return decl(forEachDescendant(PointerIterM));
-}
-
-void PointerIterationChecker::checkASTCodeBody(const Decl *D,
-                                             AnalysisManager &AM,
-                                             BugReporter &BR) const {
-  auto MatcherM = matchUnorderedIterWithPointers();
-
-  auto Matches = match(MatcherM, *D, AM.getASTContext());
-  for (const auto &Match : Matches)
-    emitDiagnostics(Match, D, BR, AM, this);
-}
-
-} // end of anonymous namespace
-
-void ento::registerPointerIterationChecker(CheckerManager &Mgr) {
-  Mgr.registerChecker<PointerIterationChecker>();
-}
-
-bool ento::shouldRegisterPointerIterationChecker(const CheckerManager &mgr) {
-  const LangOptions &LO = mgr.getLangOpts();
-  return LO.CPlusPlus;
-}
diff --git a/clang/lib/StaticAnalyzer/Checkers/PointerSortingChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/PointerSortingChecker.cpp
deleted file mode 100644
index 25d87f4acfc9..000000000000
--- a/clang/lib/StaticAnalyzer/Checkers/PointerSortingChecker.cpp
+++ /dev/null
@@ -1,115 +0,0 @@
-//== PointerSortingChecker.cpp --------------------------------- -*- C++ -*--=//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines PointerSortingChecker which checks for non-determinism
-// caused due to sorting containers with pointer-like elements.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/ASTMatchers/ASTMatchFinder.h"
-#include "clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h"
-#include "clang/StaticAnalyzer/Core/Checker.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
-
-using namespace clang;
-using namespace ento;
-using namespace ast_matchers;
-
-namespace {
-
-// ID of a node at which the diagnostic would be emitted.
-constexpr llvm::StringLiteral WarnAtNode = "sort";
-
-class PointerSortingChecker : public Checker<check::ASTCodeBody> {
-public:
-  void checkASTCodeBody(const Decl *D,
-                        AnalysisManager &AM,
-                        BugReporter &BR) const;
-};
-
-static void emitDiagnostics(const BoundNodes &Match, const Decl *D,
-                            BugReporter &BR, AnalysisManager &AM,
-                            const PointerSortingChecker *Checker) {
-  auto *ADC = AM.getAnalysisDeclContext(D);
-
-  const auto *MarkedStmt = Match.getNodeAs<CallExpr>(WarnAtNode);
-  assert(MarkedStmt);
-
-  auto Range = MarkedStmt->getSourceRange();
-  auto Location = PathDiagnosticLocation::createBegin(MarkedStmt,
-                                                      BR.getSourceManager(),
-                                                      ADC);
-  std::string Diagnostics;
-  llvm::raw_string_ostream OS(Diagnostics);
-  OS << "Sorting pointer-like elements "
-     << "can result in non-deterministic ordering";
-
-  BR.EmitBasicReport(ADC->getDecl(), Checker,
-                     "Sorting of pointer-like elements", "Non-determinism",
-                     OS.str(), Location, Range);
-}
-
-decltype(auto) callsName(const char *FunctionName) {
-  return callee(functionDecl(hasName(FunctionName)));
-}
-
-// FIXME: Currently we simply check if std::sort is used with pointer-like
-// elements. This approach can have a big false positive rate. Using std::sort,
-// std::unique and then erase is common technique for deduplicating a container
-// (which in some cases might even be quicker than using, let's say std::set).
-// In case a container contains arbitrary memory addresses (e.g. multiple
-// things give different stuff but might give the same thing multiple times)
-// which we don't want to do things with more than once, we might use
-// sort-unique-erase and the sort call will emit a report.
-auto matchSortWithPointers() -> decltype(decl()) {
-  // Match any of these function calls.
-  auto SortFuncM = anyOf(
-                     callsName("std::is_sorted"),
-                     callsName("std::nth_element"),
-                     callsName("std::partial_sort"),
-                     callsName("std::partition"),
-                     callsName("std::sort"),
-                     callsName("std::stable_partition"),
-                     callsName("std::stable_sort")
-                    );
-
-  // Match only if the container has pointer-type elements.
-  auto IteratesPointerEltsM = hasArgument(0,
-                                hasType(cxxRecordDecl(has(
-                                  fieldDecl(hasType(hasCanonicalType(
-                                    pointsTo(hasCanonicalType(pointerType()))
-                                  )))
-                              ))));
-
-  auto PointerSortM = traverse(
-      TK_AsIs,
-      stmt(callExpr(allOf(SortFuncM, IteratesPointerEltsM))).bind(WarnAtNode));
-
-  return decl(forEachDescendant(PointerSortM));
-}
-
-void PointerSortingChecker::checkASTCodeBody(const Decl *D,
-                                             AnalysisManager &AM,
-                                             BugReporter &BR) const {
-  auto MatcherM = matchSortWithPointers();
-
-  auto Matches = match(MatcherM, *D, AM.getASTContext());
-  for (const auto &Match : Matches)
-    emitDiagnostics(Match, D, BR, AM, this);
-}
-
-} // end of anonymous namespace
-
-void ento::registerPointerSortingChecker(CheckerManager &Mgr) {
-  Mgr.registerChecker<PointerSortingChecker>();
-}
-
-bool ento::shouldRegisterPointerSortingChecker(const CheckerManager &mgr) {
-  const LangOptions &LO = mgr.getLangOpts();
-  return LO.CPlusPlus;
-}
diff --git a/clang/test/Analysis/ptr-iter.cpp b/clang/test/Analysis/ptr-iter.cpp
deleted file mode 100644
index a94288cd1c8c..000000000000
--- a/clang/test/Analysis/ptr-iter.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-// RUN: %clang_analyze_cc1 %s -std=c++14 -analyzer-output=text -verify \
-// RUN: -analyzer-checker=core,alpha.nondeterminism.PointerIteration
-
-#include "Inputs/system-header-simulator-cxx.h"
-
-template<class T>
-void f(T x);
-
-void PointerIteration() {
-  int a = 1, b = 2;
-  std::set<int> OrderedIntSet = {a, b};
-  std::set<int *> OrderedPtrSet = {&a, &b};
-  std::unordered_set<int> UnorderedIntSet = {a, b};
-  std::unordered_set<int *> UnorderedPtrSet = {&a, &b};
-
-  for (auto i : OrderedIntSet) // no-warning
-    f(i);
-
-  for (auto i : OrderedPtrSet) // no-warning
-    f(i);
-
-  for (auto i : UnorderedIntSet) // no-warning
-    f(i);
-
-  for (auto i : UnorderedPtrSet) // expected-warning {{Iteration of pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerIteration]
-// expected-note@-1 {{Iteration of pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerIteration]
-    f(i);
-}
diff --git a/clang/test/Analysis/ptr-sort.cpp b/clang/test/Analysis/ptr-sort.cpp
deleted file mode 100644
index d238b390bdc2..000000000000
--- a/clang/test/Analysis/ptr-sort.cpp
+++ /dev/null
@@ -1,36 +0,0 @@
-// RUN: %clang_analyze_cc1 %s -std=c++14 -analyzer-output=text -verify \
-// RUN: -analyzer-checker=core,alpha.nondeterminism.PointerSorting
-
-#include "Inputs/system-header-simulator-cxx.h"
-
-bool f(int x) { return true; }
-bool g(int *x) { return true; }
-
-void PointerSorting() {
-  int a = 1, b = 2;
-  std::vector<int> V1 = {a, b};
-  std::vector<int *> V2 = {&a, &b};
-
-  std::is_sorted(V1.begin(), V1.end());                    // no-warning
-  std::nth_element(V1.begin(), V1.begin() + 1, V1.end());  // no-warning
-  std::partial_sort(V1.begin(), V1.begin() + 1, V1.end()); // no-warning
-  std::sort(V1.begin(), V1.end());                         // no-warning
-  std::stable_sort(V1.begin(), V1.end());                  // no-warning
-  std::partition(V1.begin(), V1.end(), f);                 // no-warning
-  std::stable_partition(V1.begin(), V1.end(), g);          // no-warning
-
-  std::is_sorted(V2.begin(), V2.end()); // expected-warning {{Sorting pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerSorting]
-  // expected-note@-1 {{Sorting pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerSorting]
-  std::nth_element(V2.begin(), V2.begin() + 1, V2.end()); // expected-warning {{Sorting pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerSorting]
-  // expected-note@-1 {{Sorting pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerSorting]
-  std::partial_sort(V2.begin(), V2.begin() + 1, V2.end()); // expected-warning {{Sorting pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerSorting]
-  // expected-note@-1 {{Sorting pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerSorting]
-  std::sort(V2.begin(), V2.end()); // expected-warning {{Sorting pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerSorting]
-  // expected-note@-1 {{Sorting pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerSorting]
-  std::stable_sort(V2.begin(), V2.end()); // expected-warning {{Sorting pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerSorting]
-  // expected-note@-1 {{Sorting pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerSorting]
-  std::partition(V2.begin(), V2.end(), f); // expected-warning {{Sorting pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerSorting]
-  // expected-note@-1 {{Sorting pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerSorting]
-  std::stable_partition(V2.begin(), V2.end(), g); // expected-warning {{Sorting pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerSorting]
-  // expected-note@-1 {{Sorting pointer-like elements can result in non-deterministic ordering}} [alpha.nondeterminism.PointerSorting]
-}
-- 
GitLab


From f6b513a7857cbcdb7df93079916e6bb8a00bd0b8 Mon Sep 17 00:00:00 2001
From: Vassil Vassilev <v.g.vassilev@gmail.com>
Date: Mon, 28 Oct 2024 08:57:57 +0000
Subject: [PATCH 289/693] Revert "Add explicit symbol visibility macros to
 InstrProfData.inc (#110732)"

This reverts commit d7ca703eab7997814de425eaa4fd888563d78831 in llvm/llvm-project#110732
---
 compiler-rt/include/profile/InstrProfData.inc   | 13 ++-----------
 llvm/include/llvm/ProfileData/InstrProfData.inc | 13 ++-----------
 2 files changed, 4 insertions(+), 22 deletions(-)

diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index 08ecaf0ed9fa..c66b0465a0b5 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -62,15 +62,6 @@
 #define INSTR_PROF_VISIBILITY
 #endif
 
-/* This include is needed for symbol visibility macros used on
- * ValueProfRecord\ValueProfData so there functions are exported from the
- * LLVM shared library on windows. */
-#ifdef __cplusplus
-#include "llvm/Support/Compiler.h"
-#else
-#define LLVM_ABI
-#endif
-
 // clang-format off:consider re-enabling clang-format if auto-formatted C macros
 // are readable (e.g., after `issue #82426` is fixed)
 /* INSTR_PROF_DATA start. */
@@ -382,7 +373,7 @@ INSTR_PROF_SECT_ENTRY(IPSK_covinit, \
  * This is the header of the data structure that defines the on-disk
  * layout of the value profile data of a particular kind for one function.
  */
-typedef struct LLVM_ABI ValueProfRecord {
+typedef struct ValueProfRecord {
   /* The kind of the value profile record. */
   uint32_t Kind;
   /*
@@ -432,7 +423,7 @@ typedef struct LLVM_ABI ValueProfRecord {
  * Per-function header/control data structure for value profiling
  * data in indexed format.
  */
-typedef struct LLVM_ABI ValueProfData {
+typedef struct ValueProfData {
   /*
    * Total size in bytes including this field. It must be a multiple
    * of sizeof(uint64_t).
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index 08ecaf0ed9fa..c66b0465a0b5 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -62,15 +62,6 @@
 #define INSTR_PROF_VISIBILITY
 #endif
 
-/* This include is needed for symbol visibility macros used on
- * ValueProfRecord\ValueProfData so there functions are exported from the
- * LLVM shared library on windows. */
-#ifdef __cplusplus
-#include "llvm/Support/Compiler.h"
-#else
-#define LLVM_ABI
-#endif
-
 // clang-format off:consider re-enabling clang-format if auto-formatted C macros
 // are readable (e.g., after `issue #82426` is fixed)
 /* INSTR_PROF_DATA start. */
@@ -382,7 +373,7 @@ INSTR_PROF_SECT_ENTRY(IPSK_covinit, \
  * This is the header of the data structure that defines the on-disk
  * layout of the value profile data of a particular kind for one function.
  */
-typedef struct LLVM_ABI ValueProfRecord {
+typedef struct ValueProfRecord {
   /* The kind of the value profile record. */
   uint32_t Kind;
   /*
@@ -432,7 +423,7 @@ typedef struct LLVM_ABI ValueProfRecord {
  * Per-function header/control data structure for value profiling
  * data in indexed format.
  */
-typedef struct LLVM_ABI ValueProfData {
+typedef struct ValueProfData {
   /*
    * Total size in bytes including this field. It must be a multiple
    * of sizeof(uint64_t).
-- 
GitLab


From dff114b3565e4c981fcb40f24f72a0cb426294fe Mon Sep 17 00:00:00 2001
From: Oliver Stannard <oliver.stannard@arm.com>
Date: Mon, 28 Oct 2024 10:01:06 +0100
Subject: [PATCH 290/693] [ARM] Optimise non-ABI frame pointers (#110286)

With -fomit-frame-pointer, even if we set up a frame pointer for other
reasons (e.g. variable-sized or over-aligned stack allocations), we
don't need to create an ABI-compliant frame record. This means that we
can save all of the general-purpose registers in one push, instead of
splitting it to ensure that the frame pointer and link register are
adjacent on the stack, saving two instructions per function.
---
 llvm/lib/Target/ARM/ARMFrameLowering.cpp      |  11 +
 llvm/lib/Target/ARM/ARMSubtarget.cpp          |  30 +-
 llvm/lib/Target/ARM/ARMSubtarget.h            |   4 +
 llvm/test/CodeGen/Thumb2/bti-pac-replace-2.ll |  47 +-
 llvm/test/CodeGen/Thumb2/pacbti-m-basic.ll    | 160 +++--
 .../CodeGen/Thumb2/pacbti-m-frame-chain.ll    | 617 +++++++++++++++---
 .../Thumb2/pacbti-m-indirect-tail-call.ll     |  36 +-
 .../CodeGen/Thumb2/pacbti-m-outliner-3.ll     | 126 ++--
 .../CodeGen/Thumb2/pacbti-m-outliner-4.ll     | 198 +++---
 .../test/CodeGen/Thumb2/pacbti-m-overalign.ll |  64 +-
 .../test/CodeGen/Thumb2/pacbti-m-stack-arg.ll |   9 +-
 .../test/CodeGen/Thumb2/pacbti-m-varargs-1.ll |  73 ++-
 .../test/CodeGen/Thumb2/pacbti-m-varargs-2.ll |  85 ++-
 llvm/test/CodeGen/Thumb2/pacbti-m-vla.ll      | 110 +++-
 14 files changed, 1089 insertions(+), 481 deletions(-)

diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
index 4f366dcffcd2..a1d131103239 100644
--- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
@@ -3003,6 +3003,17 @@ bool ARMFrameLowering::assignCalleeSavedSpillSlots(
       // on the stack.
       CSI.insert(CSI.begin(), CalleeSavedInfo(ARM::R12));
       break;
+    case ARMSubtarget::NoSplit:
+      assert(!MF.getTarget().Options.DisableFramePointerElim(MF) &&
+             "ABI-required frame pointers need a CSR split when signing return "
+             "address.");
+      CSI.insert(find_if(CSI,
+                         [=](const auto &CS) {
+                           Register Reg = CS.getReg();
+                           return Reg != ARM::LR;
+                         }),
+                 CalleeSavedInfo(ARM::R12));
+      break;
     default:
       llvm_unreachable("Unexpected CSR split with return address signing");
     }
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.cpp b/llvm/lib/Target/ARM/ARMSubtarget.cpp
index 9adfb1fab5f0..cec44acc5443 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.cpp
+++ b/llvm/lib/Target/ARM/ARMSubtarget.cpp
@@ -492,17 +492,16 @@ ARMSubtarget::getPushPopSplitVariation(const MachineFunction &MF) const {
   const std::vector<CalleeSavedInfo> CSI =
       MF.getFrameInfo().getCalleeSavedInfo();
 
-  // Returns SplitR7 if the frame setup must be split into two separate pushes
-  // of r0-r7,lr and another containing r8-r11 (+r12 if necessary). This is
-  // always required on Thumb1-only targets, as the push and pop instructions
-  // can't access the high registers. This is also required when R7 is the frame
-  // pointer and frame pointer elimiination is disabled, or branch signing is
-  // enabled and AAPCS is disabled.
-  if ((MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress() &&
-       !createAAPCSFrameChain()) ||
-      (getFramePointerReg() == ARM::R7 &&
-       MF.getTarget().Options.DisableFramePointerElim(MF)) ||
-      isThumb1Only())
+  // Thumb1 always splits the pushes at R7, because the Thumb1 push instruction
+  // cannot use high registers except for lr.
+  if (isThumb1Only())
+    return SplitR7;
+
+  // If R7 is the frame pointer, we must split at R7 to ensure that the
+  // previous frame pointer (R7) and return address (LR) are adjacent on the
+  // stack, to form a valid frame record.
+  if (getFramePointerReg() == ARM::R7 &&
+      MF.getTarget().Options.FramePointerIsReserved(MF))
     return SplitR7;
 
   // Returns SplitR11WindowsSEH when the stack pointer needs to be
@@ -515,11 +514,12 @@ ARMSubtarget::getPushPopSplitVariation(const MachineFunction &MF) const {
       (MFI.hasVarSizedObjects() || getRegisterInfo()->hasStackRealignment(MF)))
     return SplitR11WindowsSEH;
 
-  // Returns R11SplitAAPCSBranchSigning if R11 and lr are not adjacent to each
-  // other in the list of callee saved registers in a frame, and branch
-  // signing is enabled.
+  // Returns SplitR11AAPCSSignRA when the frame pointer is R11, requiring R11
+  // and LR to be adjacent on the stack, and branch signing is enabled,
+  // requiring R12 to be on the stack.
   if (MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress() &&
-      getFramePointerReg() == ARM::R11)
+      getFramePointerReg() == ARM::R11 &&
+      MF.getTarget().Options.FramePointerIsReserved(MF))
     return SplitR11AAPCSSignRA;
   return NoSplit;
 }
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h
index 214c5f1b45e5..2f7af05a259f 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -95,6 +95,10 @@ public:
     /// push {r0-r7, lr}
     /// push {r8-r12}
     /// vpush {d8-d15}
+    /// Note that Thumb1 changes this layout when the frame pointer is R11,
+    /// using a longer sequence of instructions because R11 can't be used by a
+    /// Thumb1 push instruction. This doesn't currently have a separate enum
+    /// value, and is handled entriely within Thumb1FrameLowering::emitPrologue.
     SplitR7,
 
     /// When the stack frame size is not known (because of variable-sized
diff --git a/llvm/test/CodeGen/Thumb2/bti-pac-replace-2.ll b/llvm/test/CodeGen/Thumb2/bti-pac-replace-2.ll
index c309d992b95a..a0e6f9bf9b30 100644
--- a/llvm/test/CodeGen/Thumb2/bti-pac-replace-2.ll
+++ b/llvm/test/CodeGen/Thumb2/bti-pac-replace-2.ll
@@ -1,9 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc --force-dwarf-frame-section %s -o - | FileCheck %s
 ; RUN: llc --filetype=obj %s -o - | llvm-readelf -u - | FileCheck %s --check-prefix=UNWIND
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 target triple = "thumbv8.1m.main-arm-unknown-eabi"
 
+; Check the function starts with `pacbti` and correct unwind info is emitted
 define hidden i32 @_Z1fi(i32 %x) "sign-return-address"="non-leaf" "sign-return-address-key"="a_key" "branch-target-enforcement" {
+; CHECK-LABEL: _Z1fi:
+; CHECK:         .cfi_sections .debug_frame
+; CHECK-NEXT:    .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pacbti r12, lr, sp
+; CHECK-NEXT:    .save {r7, ra_auth_code, lr}
+; CHECK-NEXT:    push.w {r7, r12, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    .cfi_offset lr, -4
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -8
+; CHECK-NEXT:    .cfi_offset r7, -12
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    adds r0, #1
+; CHECK-NEXT:    bl _Z1gi
+; CHECK-NEXT:    subs r0, #1
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop.w {r7, r12, lr}
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    bx lr
 entry:
   %add = add nsw i32 %x, 1
   %call = tail call i32 @_Z1gi(i32 %add)
@@ -13,26 +36,10 @@ entry:
 
 declare dso_local i32 @_Z1gi(i32)
 
-; Check the function starts with `pacbti` and correct unwind info is emitted
-; CHECK-LABEL: _Z1fi:
-; ...
-; CHECK:       pacbti   r12, lr, sp
-; CHECK-NEXT:  .save    {r7, lr}
-; CHECK-NEXT:  push     {r7, lr}
-; CHECK-NEXT:  .cfi_def_cfa_offset 8
-; CHECK-NEXT:  .cfi_offset lr, -4
-; CHECK-NEXT:  .cfi_offset r7, -8
-; CHECK-NEXT:  .save   {ra_auth_code}
-; CHECK-NEXT:  str     r12, [sp, #-4]!
-; CHECK-NEXT:  .cfi_def_cfa_offset 12
-; CHECK-NEXT:  .cfi_offset ra_auth_code, -12
-; CHECK-NEXT:  .pad    #4
-; CHECK-NEXT:  sub     sp, #4
-; CHECK-NEXT:  .cfi_def_cfa_offset 16
-; ...
-
 ; UNWIND-LABEL: Opcodes [
 ; UNWIND-NEXT:  0x00      ; vsp = vsp + 4
+; UNWIND-NEXT:  0x80 0x08 ; pop {r7}
 ; UNWIND-NEXT:  0xB4      ; pop ra_auth_code
-; UNWIND-NEXT:  0x84 0x08 ; pop {r7, lr}
-; UNWIND-NEXT:  0xB0      ; finish
+; UNWIND-NEXT:  0x84 0x00 ; pop {lr}
+
+
diff --git a/llvm/test/CodeGen/Thumb2/pacbti-m-basic.ll b/llvm/test/CodeGen/Thumb2/pacbti-m-basic.ll
index 0ae46cb8879e..31f8ecddcb98 100644
--- a/llvm/test/CodeGen/Thumb2/pacbti-m-basic.ll
+++ b/llvm/test/CodeGen/Thumb2/pacbti-m-basic.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc --force-dwarf-frame-section %s -o - | FileCheck %s
 ; RUN: llc --filetype=obj %s -o - | llvm-readelf -s --unwind - | FileCheck %s --check-prefix=UNWIND
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
@@ -16,6 +17,27 @@ target triple = "thumbv8m.main-none-none-eabi"
 ; }
 
 define hidden i32 @f0(i32 %x) local_unnamed_addr "sign-return-address"="non-leaf" {
+; CHECK-LABEL: f0:
+; CHECK:         .cfi_sections .debug_frame
+; CHECK-NEXT:    .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    .save {r7, ra_auth_code, lr}
+; CHECK-NEXT:    push.w {r7, r12, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    .cfi_offset lr, -4
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -8
+; CHECK-NEXT:    .cfi_offset r7, -12
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    subs r0, #1
+; CHECK-NEXT:    bl g
+; CHECK-NEXT:    adds r0, #1
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop.w {r7, r12, lr}
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    bx lr
 entry:
   %sub = add nsw i32 %x, -1
   %call = tail call i32 @g(i32 %sub)
@@ -23,27 +45,28 @@ entry:
   ret i32 %add
 }
 
-; CHECK-LABEL: f0:
-; CHECK:       pac     r12, lr, sp
-; CHECK-NEXT:  .save   {r7, lr}
-; CHECK-NEXT:  push    {r7, lr}
-; CHECK-NEXT: .cfi_def_cfa_offset 8
-; CHECK-NEXT: .cfi_offset lr, -4
-; CHECK-NEXT: .cfi_offset r7, -8
-; CHECK-NEXT: .save   {ra_auth_code}
-; CHECK-NEXT:  str     r12, [sp, #-4]!
-; CHECK-NEXT: .cfi_def_cfa_offset 12
-; CHECK-NEXT: .cfi_offset ra_auth_code, -12
-; CHECK-NEXT: .pad    #4
-; CHECK-NEXT:  sub     sp, #4
-; ...
-; CHECK:       add     sp, #4
-; CHECK-NEXT:  ldr     r12, [sp], #4
-; CHECK-NEXT:  pop.w   {r7, lr}
-; CHECK-NEXT:  aut     r12, lr, sp
-; CHECK-NEXT:  bx      lr
-
 define hidden i32 @f1(i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: f1:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    vstr fpcxtns, [sp, #-4]!
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    .save {r7, ra_auth_code, lr}
+; CHECK-NEXT:    push.w {r7, r12, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset lr, -8
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -12
+; CHECK-NEXT:    .cfi_offset r7, -16
+; CHECK-NEXT:    subs r0, #1
+; CHECK-NEXT:    bl g
+; CHECK-NEXT:    adds r0, #1
+; CHECK-NEXT:    pop.w {r7, r12, lr}
+; CHECK-NEXT:    vscclrm {s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, s15, vpr}
+; CHECK-NEXT:    vldr fpcxtns, [sp], #4
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    clrm {r1, r2, r3, r12, apsr}
+; CHECK-NEXT:    bxns lr
 entry:
   %sub = add nsw i32 %x, -1
   %call = tail call i32 @g(i32 %sub)
@@ -51,44 +74,55 @@ entry:
   ret i32 %add
 }
 
-; CHECK-LABEL: f1:
-; CHECK:       pac     r12, lr, sp
-; CHECK-NEXT:  vstr    fpcxtns, [sp, #-4]!
-; CHECK-NEXT:  .cfi_def_cfa_offset 4
-; CHECK-NEXT:  .save    {r7, lr}
-; CHECK-NEXT:  push    {r7, lr}
-; CHECK:       vldr    fpcxtns, [sp], #4
-; CHECK:       aut     r12, lr, sp
-
 define hidden i32 @f2(i32 %x) local_unnamed_addr #1 {
+; CHECK-LABEL: f2:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    .save {r7, ra_auth_code, lr}
+; CHECK-NEXT:    push.w {r7, r12, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    .cfi_offset lr, -4
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -8
+; CHECK-NEXT:    .cfi_offset r7, -12
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    subs r0, #1
+; CHECK-NEXT:    bl g
+; CHECK-NEXT:    adds r0, #1
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop.w {r7, r12, lr}
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    mrs r12, control
+; CHECK-NEXT:    tst.w r12, #8
+; CHECK-NEXT:    beq .LBB2_2
+; CHECK-NEXT:  @ %bb.1: @ %entry
+; CHECK-NEXT:    vmrs r12, fpscr
+; CHECK-NEXT:    vmov d0, lr, lr
+; CHECK-NEXT:    vmov d1, lr, lr
+; CHECK-NEXT:    vmov d2, lr, lr
+; CHECK-NEXT:    vmov d3, lr, lr
+; CHECK-NEXT:    vmov d4, lr, lr
+; CHECK-NEXT:    vmov d5, lr, lr
+; CHECK-NEXT:    vmov d6, lr, lr
+; CHECK-NEXT:    vmov d7, lr, lr
+; CHECK-NEXT:    bic r12, r12, #159
+; CHECK-NEXT:    bic r12, r12, #4026531840
+; CHECK-NEXT:    vmsr fpscr, r12
+; CHECK-NEXT:  .LBB2_2: @ %entry
+; CHECK-NEXT:    mov r1, lr
+; CHECK-NEXT:    mov r2, lr
+; CHECK-NEXT:    mov r3, lr
+; CHECK-NEXT:    mov r12, lr
+; CHECK-NEXT:    msr apsr_nzcvq, lr
+; CHECK-NEXT:    bxns lr
 entry:
   %sub = add nsw i32 %x, -1
   %call = tail call i32 @g(i32 %sub)
   %add = add nsw i32 %call, 1
   ret i32 %add
 }
-; CHECK-LABEL: f2:
-; CHECK:       pac    r12, lr, sp
-; CHECK-NEXT:  .save  {r7, lr}
-; CHECK-NEXT:  push   {r7, lr}
-; CHECK-NEXT: .cfi_def_cfa_offset 8
-; CHECK-NEXT: .cfi_offset lr, -4
-; CHECK-NEXT: .cfi_offset r7, -8
-; CHECK-NEXT:  .save  {ra_auth_code}
-; CHECK-NEXT:  str    r12, [sp, #-4]!
-; CHECK-NEXT: .cfi_def_cfa_offset 12
-; CHECK-NEXT: .cfi_offset ra_auth_code, -12
-; CHECK-NEXT:  .pad   #4
-; CHECK-NEXT:  sub    sp, #4
-; CHECK-NEXT:  .cfi_def_cfa_offset 16
-; ...
-; CHECK:       add    sp, #4
-; CHECK-NEXT:  ldr    r12, [sp], #4
-; CHECK-NEXT:  pop.w  {r7, lr}
-; CHECK-NEXT:  aut    r12, lr, sp
-; CHECK-NEXT:  mrs    r12, control
-; ...
-; CHECK:       bxns    lr
 
 declare dso_local i32 @g(i32) local_unnamed_addr
 
@@ -103,22 +137,22 @@ attributes #1 = { "sign-return-address"="non-leaf" "cmse_nonsecure_entry" "targe
 
 ; UNWIND-LABEL: FunctionAddress: 0x0
 ; UNWIND:       0x00      ; vsp = vsp + 4
+; UNWIND-NEXT:  0x80 0x08 ; pop {r7}
 ; UNWIND-NEXT:  0xB4      ; pop ra_auth_code
-; UNWIND-NEXT:  0x84 0x08 ; pop {r7, lr}
-; UNWIND-NEXT:  0xB0      ; finish
-; UNWIND-NEXT:  0xB0      ; finish
+; UNWIND-NEXT:  0x84 0x00 ; pop {lr}
 
-; UNWIND-LABEL: FunctionAddress: 0x24
-; UNWIND:       0xB4      ; pop ra_auth_code
-; UNWIND-NEXT:  0x84 0x08 ; pop {r7, lr}
 
-; UNWIND-LABEL: FunctionAddress: 0x54
+; UNWIND-LABEL: FunctionAddress: 0x1E
+; UNWIND:       0x80 0x08 ; pop {r7}
+; UNWIND-NEXT:  0xB4      ; pop ra_auth_code
+; UNWIND-NEXT:  0x84 0x00 ; pop {lr}
+
+; UNWIND-LABEL: FunctionAddress: 0x48
 ; UNWIND:       0x00      ; vsp = vsp + 4
+; UNWIND-NEXT:  0x80 0x08 ; pop {r7}
 ; UNWIND-NEXT:  0xB4      ; pop ra_auth_code
-; UNWIND-NEXT:  0x84 0x08 ; pop {r7, lr}
-; UNWIND-NEXT:  0xB0      ; finish
-; UNWIND-NEXT:  0xB0      ; finish
+; UNWIND-NEXT:  0x84 0x00 ; pop {lr}
 
 ; UNWIND-LABEL: 00000001 {{.*}} f0
-; UNWIND-LABEL: 00000025 {{.*}} f1
-; UNWIND-LABEL: 00000055 {{.*}} f2
+; UNWIND-LABEL: 0000001f {{.*}} f1
+; UNWIND-LABEL: 00000049 {{.*}} f2
diff --git a/llvm/test/CodeGen/Thumb2/pacbti-m-frame-chain.ll b/llvm/test/CodeGen/Thumb2/pacbti-m-frame-chain.ll
index 8bcf87130c54..7e9258002f61 100644
--- a/llvm/test/CodeGen/Thumb2/pacbti-m-frame-chain.ll
+++ b/llvm/test/CodeGen/Thumb2/pacbti-m-frame-chain.ll
@@ -1,32 +1,87 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=thumbv8.1m.main-none-eabi < %s --force-dwarf-frame-section -frame-pointer=all -mattr=+aapcs-frame-chain | FileCheck %s
+; RUN: llc -mtriple=thumbv8.1m.main-none-eabi < %s --force-dwarf-frame-section -frame-pointer=none                          | FileCheck %s --check-prefix=R7
+; RUN: llc -mtriple=thumbv8.1m.main-none-eabi < %s --force-dwarf-frame-section -frame-pointer=reserved                      | FileCheck %s --check-prefix=R7-RES
+; RUN: llc -mtriple=thumbv8.1m.main-none-eabi < %s --force-dwarf-frame-section -frame-pointer=all                           | FileCheck %s --check-prefix=R7-ABI
+; RUN: llc -mtriple=thumbv8.1m.main-none-eabi < %s --force-dwarf-frame-section -frame-pointer=none     -mattr=+aapcs-frame-chain | FileCheck %s --check-prefix=R11
+; RUN: llc -mtriple=thumbv8.1m.main-none-eabi < %s --force-dwarf-frame-section -frame-pointer=reserved -mattr=+aapcs-frame-chain | FileCheck %s --check-prefix=R11-RES
+; RUN: llc -mtriple=thumbv8.1m.main-none-eabi < %s --force-dwarf-frame-section -frame-pointer=all      -mattr=+aapcs-frame-chain | FileCheck %s --check-prefix=R11-ABI
 
 ; int test1() {
 ;     return 0;
 ; }
 define i32 @test1() "sign-return-address"="non-leaf" {
-; CHECK-LABEL: test1:
-; CHECK:         .cfi_sections .debug_frame
-; CHECK-NEXT:    .cfi_startproc
-; CHECK-NEXT:  @ %bb.0: @ %entry
-; CHECK-NEXT:    pac r12, lr, sp
-; CHECK-NEXT:    .save {ra_auth_code}
-; CHECK-NEXT:    str r12, [sp, #-4]!
-; CHECK-NEXT:    .cfi_def_cfa_offset 4
-; CHECK-NEXT:    .cfi_offset ra_auth_code, -4
-; CHECK-NEXT:    .save {r11, lr}
-; CHECK-NEXT:    push.w {r11, lr}
-; CHECK-NEXT:    .cfi_def_cfa_offset 12
-; CHECK-NEXT:    .cfi_offset lr, -8
-; CHECK-NEXT:    .cfi_offset r11, -12
-; CHECK-NEXT:    .setfp r11, sp
-; CHECK-NEXT:    mov r11, sp
-; CHECK-NEXT:    .cfi_def_cfa_register r11
-; CHECK-NEXT:    movs r0, #0
-; CHECK-NEXT:    pop.w {r11, lr}
-; CHECK-NEXT:    ldr r12, [sp], #4
-; CHECK-NEXT:    aut r12, lr, sp
-; CHECK-NEXT:    bx lr
+; R7-LABEL: test1:
+; R7:         .cfi_sections .debug_frame
+; R7-NEXT:    .cfi_startproc
+; R7-NEXT:  @ %bb.0: @ %entry
+; R7-NEXT:    movs r0, #0
+; R7-NEXT:    bx lr
+;
+; R7-RES-LABEL: test1:
+; R7-RES:         .cfi_sections .debug_frame
+; R7-RES-NEXT:    .cfi_startproc
+; R7-RES-NEXT:  @ %bb.0: @ %entry
+; R7-RES-NEXT:    movs r0, #0
+; R7-RES-NEXT:    bx lr
+;
+; R7-ABI-LABEL: test1:
+; R7-ABI:         .cfi_sections .debug_frame
+; R7-ABI-NEXT:    .cfi_startproc
+; R7-ABI-NEXT:  @ %bb.0: @ %entry
+; R7-ABI-NEXT:    pac r12, lr, sp
+; R7-ABI-NEXT:    .save {r7, lr}
+; R7-ABI-NEXT:    push {r7, lr}
+; R7-ABI-NEXT:    .cfi_def_cfa_offset 8
+; R7-ABI-NEXT:    .cfi_offset lr, -4
+; R7-ABI-NEXT:    .cfi_offset r7, -8
+; R7-ABI-NEXT:    .setfp r7, sp
+; R7-ABI-NEXT:    mov r7, sp
+; R7-ABI-NEXT:    .cfi_def_cfa_register r7
+; R7-ABI-NEXT:    .save {ra_auth_code}
+; R7-ABI-NEXT:    str r12, [sp, #-4]!
+; R7-ABI-NEXT:    .cfi_offset ra_auth_code, -12
+; R7-ABI-NEXT:    movs r0, #0
+; R7-ABI-NEXT:    ldr r12, [sp], #4
+; R7-ABI-NEXT:    pop.w {r7, lr}
+; R7-ABI-NEXT:    aut r12, lr, sp
+; R7-ABI-NEXT:    bx lr
+;
+; R11-LABEL: test1:
+; R11:         .cfi_sections .debug_frame
+; R11-NEXT:    .cfi_startproc
+; R11-NEXT:  @ %bb.0: @ %entry
+; R11-NEXT:    movs r0, #0
+; R11-NEXT:    bx lr
+;
+; R11-RES-LABEL: test1:
+; R11-RES:         .cfi_sections .debug_frame
+; R11-RES-NEXT:    .cfi_startproc
+; R11-RES-NEXT:  @ %bb.0: @ %entry
+; R11-RES-NEXT:    movs r0, #0
+; R11-RES-NEXT:    bx lr
+;
+; R11-ABI-LABEL: test1:
+; R11-ABI:         .cfi_sections .debug_frame
+; R11-ABI-NEXT:    .cfi_startproc
+; R11-ABI-NEXT:  @ %bb.0: @ %entry
+; R11-ABI-NEXT:    pac r12, lr, sp
+; R11-ABI-NEXT:    .save {ra_auth_code}
+; R11-ABI-NEXT:    str r12, [sp, #-4]!
+; R11-ABI-NEXT:    .cfi_def_cfa_offset 4
+; R11-ABI-NEXT:    .cfi_offset ra_auth_code, -4
+; R11-ABI-NEXT:    .save {r11, lr}
+; R11-ABI-NEXT:    push.w {r11, lr}
+; R11-ABI-NEXT:    .cfi_def_cfa_offset 12
+; R11-ABI-NEXT:    .cfi_offset lr, -8
+; R11-ABI-NEXT:    .cfi_offset r11, -12
+; R11-ABI-NEXT:    .setfp r11, sp
+; R11-ABI-NEXT:    mov r11, sp
+; R11-ABI-NEXT:    .cfi_def_cfa_register r11
+; R11-ABI-NEXT:    movs r0, #0
+; R11-ABI-NEXT:    pop.w {r11, lr}
+; R11-ABI-NEXT:    ldr r12, [sp], #4
+; R11-ABI-NEXT:    aut r12, lr, sp
+; R11-ABI-NEXT:    bx lr
 entry:
     ret i32 0
 }
@@ -36,37 +91,191 @@ entry:
 ;   bar(a);
 ; }
 define dso_local void @test2(i32 noundef %n) "sign-return-address"="non-leaf" {
-; CHECK-LABEL: test2:
-; CHECK:         .cfi_startproc
-; CHECK-NEXT:  @ %bb.0: @ %entry
-; CHECK-NEXT:    pac r12, lr, sp
-; CHECK-NEXT:    .save {r4, r7, ra_auth_code}
-; CHECK-NEXT:    push.w {r4, r7, r12}
-; CHECK-NEXT:    .cfi_def_cfa_offset 12
-; CHECK-NEXT:    .cfi_offset ra_auth_code, -4
-; CHECK-NEXT:    .cfi_offset r7, -8
-; CHECK-NEXT:    .cfi_offset r4, -12
-; CHECK-NEXT:    .save {r11, lr}
-; CHECK-NEXT:    push.w {r11, lr}
-; CHECK-NEXT:    .cfi_def_cfa_offset 20
-; CHECK-NEXT:    .cfi_offset lr, -16
-; CHECK-NEXT:    .cfi_offset r11, -20
-; CHECK-NEXT:    .setfp r11, sp
-; CHECK-NEXT:    mov r11, sp
-; CHECK-NEXT:    .cfi_def_cfa_register r11
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
-; CHECK-NEXT:    movs r1, #7
-; CHECK-NEXT:    add.w r0, r1, r0, lsl #2
-; CHECK-NEXT:    bic r0, r0, #7
-; CHECK-NEXT:    sub.w r0, sp, r0
-; CHECK-NEXT:    mov sp, r0
-; CHECK-NEXT:    bl take_ptr
-; CHECK-NEXT:    mov sp, r11
-; CHECK-NEXT:    pop.w {r11, lr}
-; CHECK-NEXT:    pop.w {r4, r7, r12}
-; CHECK-NEXT:    aut r12, lr, sp
-; CHECK-NEXT:    bx lr
+; R7-LABEL: test2:
+; R7:         .cfi_startproc
+; R7-NEXT:  @ %bb.0: @ %entry
+; R7-NEXT:    pac r12, lr, sp
+; R7-NEXT:    .save {r4, r6, r7, ra_auth_code, lr}
+; R7-NEXT:    push.w {r4, r6, r7, r12, lr}
+; R7-NEXT:    .cfi_def_cfa_offset 20
+; R7-NEXT:    .cfi_offset lr, -4
+; R7-NEXT:    .cfi_offset ra_auth_code, -8
+; R7-NEXT:    .cfi_offset r7, -12
+; R7-NEXT:    .cfi_offset r6, -16
+; R7-NEXT:    .cfi_offset r4, -20
+; R7-NEXT:    .setfp r7, sp, #8
+; R7-NEXT:    add r7, sp, #8
+; R7-NEXT:    .cfi_def_cfa r7, 12
+; R7-NEXT:    .pad #4
+; R7-NEXT:    sub sp, #4
+; R7-NEXT:    movs r1, #7
+; R7-NEXT:    add.w r0, r1, r0, lsl #2
+; R7-NEXT:    bic r0, r0, #7
+; R7-NEXT:    sub.w r0, sp, r0
+; R7-NEXT:    mov sp, r0
+; R7-NEXT:    bl take_ptr
+; R7-NEXT:    sub.w r4, r7, #8
+; R7-NEXT:    mov sp, r4
+; R7-NEXT:    pop.w {r4, r6, r7, r12, lr}
+; R7-NEXT:    aut r12, lr, sp
+; R7-NEXT:    bx lr
+;
+; R7-RES-LABEL: test2:
+; R7-RES:         .cfi_startproc
+; R7-RES-NEXT:  @ %bb.0: @ %entry
+; R7-RES-NEXT:    pac r12, lr, sp
+; R7-RES-NEXT:    .save {r4, r6, r7, lr}
+; R7-RES-NEXT:    push {r4, r6, r7, lr}
+; R7-RES-NEXT:    .cfi_def_cfa_offset 16
+; R7-RES-NEXT:    .cfi_offset lr, -4
+; R7-RES-NEXT:    .cfi_offset r7, -8
+; R7-RES-NEXT:    .cfi_offset r6, -12
+; R7-RES-NEXT:    .cfi_offset r4, -16
+; R7-RES-NEXT:    .setfp r7, sp, #8
+; R7-RES-NEXT:    add r7, sp, #8
+; R7-RES-NEXT:    .cfi_def_cfa r7, 8
+; R7-RES-NEXT:    .save {ra_auth_code}
+; R7-RES-NEXT:    str r12, [sp, #-4]!
+; R7-RES-NEXT:    .cfi_offset ra_auth_code, -20
+; R7-RES-NEXT:    .pad #4
+; R7-RES-NEXT:    sub sp, #4
+; R7-RES-NEXT:    movs r1, #7
+; R7-RES-NEXT:    add.w r0, r1, r0, lsl #2
+; R7-RES-NEXT:    bic r0, r0, #7
+; R7-RES-NEXT:    sub.w r0, sp, r0
+; R7-RES-NEXT:    mov sp, r0
+; R7-RES-NEXT:    bl take_ptr
+; R7-RES-NEXT:    sub.w r4, r7, #12
+; R7-RES-NEXT:    mov sp, r4
+; R7-RES-NEXT:    ldr r12, [sp], #4
+; R7-RES-NEXT:    pop.w {r4, r6, r7, lr}
+; R7-RES-NEXT:    aut r12, lr, sp
+; R7-RES-NEXT:    bx lr
+;
+; R7-ABI-LABEL: test2:
+; R7-ABI:         .cfi_startproc
+; R7-ABI-NEXT:  @ %bb.0: @ %entry
+; R7-ABI-NEXT:    pac r12, lr, sp
+; R7-ABI-NEXT:    .save {r4, r6, r7, lr}
+; R7-ABI-NEXT:    push {r4, r6, r7, lr}
+; R7-ABI-NEXT:    .cfi_def_cfa_offset 16
+; R7-ABI-NEXT:    .cfi_offset lr, -4
+; R7-ABI-NEXT:    .cfi_offset r7, -8
+; R7-ABI-NEXT:    .cfi_offset r6, -12
+; R7-ABI-NEXT:    .cfi_offset r4, -16
+; R7-ABI-NEXT:    .setfp r7, sp, #8
+; R7-ABI-NEXT:    add r7, sp, #8
+; R7-ABI-NEXT:    .cfi_def_cfa r7, 8
+; R7-ABI-NEXT:    .save {ra_auth_code}
+; R7-ABI-NEXT:    str r12, [sp, #-4]!
+; R7-ABI-NEXT:    .cfi_offset ra_auth_code, -20
+; R7-ABI-NEXT:    .pad #4
+; R7-ABI-NEXT:    sub sp, #4
+; R7-ABI-NEXT:    movs r1, #7
+; R7-ABI-NEXT:    add.w r0, r1, r0, lsl #2
+; R7-ABI-NEXT:    bic r0, r0, #7
+; R7-ABI-NEXT:    sub.w r0, sp, r0
+; R7-ABI-NEXT:    mov sp, r0
+; R7-ABI-NEXT:    bl take_ptr
+; R7-ABI-NEXT:    sub.w r4, r7, #12
+; R7-ABI-NEXT:    mov sp, r4
+; R7-ABI-NEXT:    ldr r12, [sp], #4
+; R7-ABI-NEXT:    pop.w {r4, r6, r7, lr}
+; R7-ABI-NEXT:    aut r12, lr, sp
+; R7-ABI-NEXT:    bx lr
+;
+; R11-LABEL: test2:
+; R11:         .cfi_startproc
+; R11-NEXT:  @ %bb.0: @ %entry
+; R11-NEXT:    pac r12, lr, sp
+; R11-NEXT:    .save {r4, r7, r11, ra_auth_code, lr}
+; R11-NEXT:    push.w {r4, r7, r11, r12, lr}
+; R11-NEXT:    .cfi_def_cfa_offset 20
+; R11-NEXT:    .cfi_offset lr, -4
+; R11-NEXT:    .cfi_offset ra_auth_code, -8
+; R11-NEXT:    .cfi_offset r11, -12
+; R11-NEXT:    .cfi_offset r7, -16
+; R11-NEXT:    .cfi_offset r4, -20
+; R11-NEXT:    .setfp r11, sp, #8
+; R11-NEXT:    add.w r11, sp, #8
+; R11-NEXT:    .cfi_def_cfa r11, 12
+; R11-NEXT:    .pad #4
+; R11-NEXT:    sub sp, #4
+; R11-NEXT:    movs r1, #7
+; R11-NEXT:    add.w r0, r1, r0, lsl #2
+; R11-NEXT:    bic r0, r0, #7
+; R11-NEXT:    sub.w r0, sp, r0
+; R11-NEXT:    mov sp, r0
+; R11-NEXT:    bl take_ptr
+; R11-NEXT:    sub.w r4, r11, #8
+; R11-NEXT:    mov sp, r4
+; R11-NEXT:    pop.w {r4, r7, r11, r12, lr}
+; R11-NEXT:    aut r12, lr, sp
+; R11-NEXT:    bx lr
+;
+; R11-RES-LABEL: test2:
+; R11-RES:         .cfi_startproc
+; R11-RES-NEXT:  @ %bb.0: @ %entry
+; R11-RES-NEXT:    pac r12, lr, sp
+; R11-RES-NEXT:    .save {r4, r7, ra_auth_code}
+; R11-RES-NEXT:    push.w {r4, r7, r12}
+; R11-RES-NEXT:    .cfi_def_cfa_offset 12
+; R11-RES-NEXT:    .cfi_offset ra_auth_code, -4
+; R11-RES-NEXT:    .cfi_offset r7, -8
+; R11-RES-NEXT:    .cfi_offset r4, -12
+; R11-RES-NEXT:    .save {r11, lr}
+; R11-RES-NEXT:    push.w {r11, lr}
+; R11-RES-NEXT:    .cfi_def_cfa_offset 20
+; R11-RES-NEXT:    .cfi_offset lr, -16
+; R11-RES-NEXT:    .cfi_offset r11, -20
+; R11-RES-NEXT:    .setfp r11, sp
+; R11-RES-NEXT:    mov r11, sp
+; R11-RES-NEXT:    .cfi_def_cfa_register r11
+; R11-RES-NEXT:    .pad #4
+; R11-RES-NEXT:    sub sp, #4
+; R11-RES-NEXT:    movs r1, #7
+; R11-RES-NEXT:    add.w r0, r1, r0, lsl #2
+; R11-RES-NEXT:    bic r0, r0, #7
+; R11-RES-NEXT:    sub.w r0, sp, r0
+; R11-RES-NEXT:    mov sp, r0
+; R11-RES-NEXT:    bl take_ptr
+; R11-RES-NEXT:    mov sp, r11
+; R11-RES-NEXT:    pop.w {r11, lr}
+; R11-RES-NEXT:    pop.w {r4, r7, r12}
+; R11-RES-NEXT:    aut r12, lr, sp
+; R11-RES-NEXT:    bx lr
+;
+; R11-ABI-LABEL: test2:
+; R11-ABI:         .cfi_startproc
+; R11-ABI-NEXT:  @ %bb.0: @ %entry
+; R11-ABI-NEXT:    pac r12, lr, sp
+; R11-ABI-NEXT:    .save {r4, r7, ra_auth_code}
+; R11-ABI-NEXT:    push.w {r4, r7, r12}
+; R11-ABI-NEXT:    .cfi_def_cfa_offset 12
+; R11-ABI-NEXT:    .cfi_offset ra_auth_code, -4
+; R11-ABI-NEXT:    .cfi_offset r7, -8
+; R11-ABI-NEXT:    .cfi_offset r4, -12
+; R11-ABI-NEXT:    .save {r11, lr}
+; R11-ABI-NEXT:    push.w {r11, lr}
+; R11-ABI-NEXT:    .cfi_def_cfa_offset 20
+; R11-ABI-NEXT:    .cfi_offset lr, -16
+; R11-ABI-NEXT:    .cfi_offset r11, -20
+; R11-ABI-NEXT:    .setfp r11, sp
+; R11-ABI-NEXT:    mov r11, sp
+; R11-ABI-NEXT:    .cfi_def_cfa_register r11
+; R11-ABI-NEXT:    .pad #4
+; R11-ABI-NEXT:    sub sp, #4
+; R11-ABI-NEXT:    movs r1, #7
+; R11-ABI-NEXT:    add.w r0, r1, r0, lsl #2
+; R11-ABI-NEXT:    bic r0, r0, #7
+; R11-ABI-NEXT:    sub.w r0, sp, r0
+; R11-ABI-NEXT:    mov sp, r0
+; R11-ABI-NEXT:    bl take_ptr
+; R11-ABI-NEXT:    mov sp, r11
+; R11-ABI-NEXT:    pop.w {r11, lr}
+; R11-ABI-NEXT:    pop.w {r4, r7, r12}
+; R11-ABI-NEXT:    aut r12, lr, sp
+; R11-ABI-NEXT:    bx lr
 entry:
   %vla = alloca i32, i32 %n, align 4
   call void @take_ptr(ptr noundef nonnull %vla)
@@ -81,49 +290,263 @@ entry:
 ;     knr();
 ; }
 define void @test3(i32 noundef %c, float noundef %e, i32 noundef %z) "sign-return-address"="non-leaf" {
-; CHECK-LABEL: test3:
-; CHECK:         .cfi_startproc
-; CHECK-NEXT:  @ %bb.0: @ %entry
-; CHECK-NEXT:    pac r12, lr, sp
-; CHECK-NEXT:    .save {r4, r5, r6, r7, ra_auth_code}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r12}
-; CHECK-NEXT:    .cfi_def_cfa_offset 20
-; CHECK-NEXT:    .cfi_offset ra_auth_code, -4
-; CHECK-NEXT:    .cfi_offset r7, -8
-; CHECK-NEXT:    .cfi_offset r6, -12
-; CHECK-NEXT:    .cfi_offset r5, -16
-; CHECK-NEXT:    .cfi_offset r4, -20
-; CHECK-NEXT:    .save {r11, lr}
-; CHECK-NEXT:    push.w {r11, lr}
-; CHECK-NEXT:    .cfi_def_cfa_offset 28
-; CHECK-NEXT:    .cfi_offset lr, -24
-; CHECK-NEXT:    .cfi_offset r11, -28
-; CHECK-NEXT:    .setfp r11, sp
-; CHECK-NEXT:    mov r11, sp
-; CHECK-NEXT:    .cfi_def_cfa_register r11
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r5, r2
-; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    blne knr
-; CHECK-NEXT:    adds r0, r5, #7
-; CHECK-NEXT:    bic r0, r0, #7
-; CHECK-NEXT:    sub.w r0, sp, r0
-; CHECK-NEXT:    mov sp, r0
-; CHECK-NEXT:    bl take_ptr
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    bl __aeabi_fcmpeq
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    bleq knr
-; CHECK-NEXT:    mov sp, r11
-; CHECK-NEXT:    pop.w {r11, lr}
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r12}
-; CHECK-NEXT:    aut r12, lr, sp
-; CHECK-NEXT:    bx lr
+; R7-LABEL: test3:
+; R7:         .cfi_startproc
+; R7-NEXT:  @ %bb.0: @ %entry
+; R7-NEXT:    pac r12, lr, sp
+; R7-NEXT:    .save {r4, r5, r6, r7, r8, ra_auth_code, lr}
+; R7-NEXT:    push.w {r4, r5, r6, r7, r8, r12, lr}
+; R7-NEXT:    .cfi_def_cfa_offset 28
+; R7-NEXT:    .cfi_offset lr, -4
+; R7-NEXT:    .cfi_offset ra_auth_code, -8
+; R7-NEXT:    .cfi_offset r8, -12
+; R7-NEXT:    .cfi_offset r7, -16
+; R7-NEXT:    .cfi_offset r6, -20
+; R7-NEXT:    .cfi_offset r5, -24
+; R7-NEXT:    .cfi_offset r4, -28
+; R7-NEXT:    .setfp r7, sp, #12
+; R7-NEXT:    add r7, sp, #12
+; R7-NEXT:    .cfi_def_cfa r7, 16
+; R7-NEXT:    .pad #4
+; R7-NEXT:    sub sp, #4
+; R7-NEXT:    cmp r0, #0
+; R7-NEXT:    mov r5, r2
+; R7-NEXT:    mov r4, r1
+; R7-NEXT:    it ne
+; R7-NEXT:    blne knr
+; R7-NEXT:    adds r0, r5, #7
+; R7-NEXT:    bic r0, r0, #7
+; R7-NEXT:    sub.w r0, sp, r0
+; R7-NEXT:    mov sp, r0
+; R7-NEXT:    bl take_ptr
+; R7-NEXT:    mov r0, r4
+; R7-NEXT:    movs r1, #0
+; R7-NEXT:    bl __aeabi_fcmpeq
+; R7-NEXT:    cmp r0, #0
+; R7-NEXT:    it eq
+; R7-NEXT:    bleq knr
+; R7-NEXT:    sub.w r4, r7, #12
+; R7-NEXT:    mov sp, r4
+; R7-NEXT:    pop.w {r4, r5, r6, r7, r8, r12, lr}
+; R7-NEXT:    aut r12, lr, sp
+; R7-NEXT:    bx lr
+;
+; R7-RES-LABEL: test3:
+; R7-RES:         .cfi_startproc
+; R7-RES-NEXT:  @ %bb.0: @ %entry
+; R7-RES-NEXT:    pac r12, lr, sp
+; R7-RES-NEXT:    .save {r4, r5, r6, r7, lr}
+; R7-RES-NEXT:    push {r4, r5, r6, r7, lr}
+; R7-RES-NEXT:    .cfi_def_cfa_offset 20
+; R7-RES-NEXT:    .cfi_offset lr, -4
+; R7-RES-NEXT:    .cfi_offset r7, -8
+; R7-RES-NEXT:    .cfi_offset r6, -12
+; R7-RES-NEXT:    .cfi_offset r5, -16
+; R7-RES-NEXT:    .cfi_offset r4, -20
+; R7-RES-NEXT:    .setfp r7, sp, #12
+; R7-RES-NEXT:    add r7, sp, #12
+; R7-RES-NEXT:    .cfi_def_cfa r7, 8
+; R7-RES-NEXT:    .save {r8, ra_auth_code}
+; R7-RES-NEXT:    push.w {r8, r12}
+; R7-RES-NEXT:    .cfi_offset ra_auth_code, -24
+; R7-RES-NEXT:    .cfi_offset r8, -28
+; R7-RES-NEXT:    .pad #4
+; R7-RES-NEXT:    sub sp, #4
+; R7-RES-NEXT:    cmp r0, #0
+; R7-RES-NEXT:    mov r5, r2
+; R7-RES-NEXT:    mov r4, r1
+; R7-RES-NEXT:    it ne
+; R7-RES-NEXT:    blne knr
+; R7-RES-NEXT:    adds r0, r5, #7
+; R7-RES-NEXT:    bic r0, r0, #7
+; R7-RES-NEXT:    sub.w r0, sp, r0
+; R7-RES-NEXT:    mov sp, r0
+; R7-RES-NEXT:    bl take_ptr
+; R7-RES-NEXT:    mov r0, r4
+; R7-RES-NEXT:    movs r1, #0
+; R7-RES-NEXT:    bl __aeabi_fcmpeq
+; R7-RES-NEXT:    cmp r0, #0
+; R7-RES-NEXT:    it eq
+; R7-RES-NEXT:    bleq knr
+; R7-RES-NEXT:    sub.w r4, r7, #20
+; R7-RES-NEXT:    mov sp, r4
+; R7-RES-NEXT:    pop.w {r8, r12}
+; R7-RES-NEXT:    pop.w {r4, r5, r6, r7, lr}
+; R7-RES-NEXT:    aut r12, lr, sp
+; R7-RES-NEXT:    bx lr
+;
+; R7-ABI-LABEL: test3:
+; R7-ABI:         .cfi_startproc
+; R7-ABI-NEXT:  @ %bb.0: @ %entry
+; R7-ABI-NEXT:    pac r12, lr, sp
+; R7-ABI-NEXT:    .save {r4, r5, r6, r7, lr}
+; R7-ABI-NEXT:    push {r4, r5, r6, r7, lr}
+; R7-ABI-NEXT:    .cfi_def_cfa_offset 20
+; R7-ABI-NEXT:    .cfi_offset lr, -4
+; R7-ABI-NEXT:    .cfi_offset r7, -8
+; R7-ABI-NEXT:    .cfi_offset r6, -12
+; R7-ABI-NEXT:    .cfi_offset r5, -16
+; R7-ABI-NEXT:    .cfi_offset r4, -20
+; R7-ABI-NEXT:    .setfp r7, sp, #12
+; R7-ABI-NEXT:    add r7, sp, #12
+; R7-ABI-NEXT:    .cfi_def_cfa r7, 8
+; R7-ABI-NEXT:    .save {r8, ra_auth_code}
+; R7-ABI-NEXT:    push.w {r8, r12}
+; R7-ABI-NEXT:    .cfi_offset ra_auth_code, -24
+; R7-ABI-NEXT:    .cfi_offset r8, -28
+; R7-ABI-NEXT:    .pad #4
+; R7-ABI-NEXT:    sub sp, #4
+; R7-ABI-NEXT:    cmp r0, #0
+; R7-ABI-NEXT:    mov r5, r2
+; R7-ABI-NEXT:    mov r4, r1
+; R7-ABI-NEXT:    it ne
+; R7-ABI-NEXT:    blne knr
+; R7-ABI-NEXT:    adds r0, r5, #7
+; R7-ABI-NEXT:    bic r0, r0, #7
+; R7-ABI-NEXT:    sub.w r0, sp, r0
+; R7-ABI-NEXT:    mov sp, r0
+; R7-ABI-NEXT:    bl take_ptr
+; R7-ABI-NEXT:    mov r0, r4
+; R7-ABI-NEXT:    movs r1, #0
+; R7-ABI-NEXT:    bl __aeabi_fcmpeq
+; R7-ABI-NEXT:    cmp r0, #0
+; R7-ABI-NEXT:    it eq
+; R7-ABI-NEXT:    bleq knr
+; R7-ABI-NEXT:    sub.w r4, r7, #20
+; R7-ABI-NEXT:    mov sp, r4
+; R7-ABI-NEXT:    pop.w {r8, r12}
+; R7-ABI-NEXT:    pop.w {r4, r5, r6, r7, lr}
+; R7-ABI-NEXT:    aut r12, lr, sp
+; R7-ABI-NEXT:    bx lr
+;
+; R11-LABEL: test3:
+; R11:         .cfi_startproc
+; R11-NEXT:  @ %bb.0: @ %entry
+; R11-NEXT:    pac r12, lr, sp
+; R11-NEXT:    .save {r4, r5, r6, r7, r11, ra_auth_code, lr}
+; R11-NEXT:    push.w {r4, r5, r6, r7, r11, r12, lr}
+; R11-NEXT:    .cfi_def_cfa_offset 28
+; R11-NEXT:    .cfi_offset lr, -4
+; R11-NEXT:    .cfi_offset ra_auth_code, -8
+; R11-NEXT:    .cfi_offset r11, -12
+; R11-NEXT:    .cfi_offset r7, -16
+; R11-NEXT:    .cfi_offset r6, -20
+; R11-NEXT:    .cfi_offset r5, -24
+; R11-NEXT:    .cfi_offset r4, -28
+; R11-NEXT:    .setfp r11, sp, #16
+; R11-NEXT:    add.w r11, sp, #16
+; R11-NEXT:    .cfi_def_cfa r11, 12
+; R11-NEXT:    .pad #4
+; R11-NEXT:    sub sp, #4
+; R11-NEXT:    cmp r0, #0
+; R11-NEXT:    mov r5, r2
+; R11-NEXT:    mov r4, r1
+; R11-NEXT:    it ne
+; R11-NEXT:    blne knr
+; R11-NEXT:    adds r0, r5, #7
+; R11-NEXT:    bic r0, r0, #7
+; R11-NEXT:    sub.w r0, sp, r0
+; R11-NEXT:    mov sp, r0
+; R11-NEXT:    bl take_ptr
+; R11-NEXT:    mov r0, r4
+; R11-NEXT:    movs r1, #0
+; R11-NEXT:    bl __aeabi_fcmpeq
+; R11-NEXT:    cmp r0, #0
+; R11-NEXT:    it eq
+; R11-NEXT:    bleq knr
+; R11-NEXT:    sub.w r4, r11, #16
+; R11-NEXT:    mov sp, r4
+; R11-NEXT:    pop.w {r4, r5, r6, r7, r11, r12, lr}
+; R11-NEXT:    aut r12, lr, sp
+; R11-NEXT:    bx lr
+;
+; R11-RES-LABEL: test3:
+; R11-RES:         .cfi_startproc
+; R11-RES-NEXT:  @ %bb.0: @ %entry
+; R11-RES-NEXT:    pac r12, lr, sp
+; R11-RES-NEXT:    .save {r4, r5, r6, r7, ra_auth_code}
+; R11-RES-NEXT:    push.w {r4, r5, r6, r7, r12}
+; R11-RES-NEXT:    .cfi_def_cfa_offset 20
+; R11-RES-NEXT:    .cfi_offset ra_auth_code, -4
+; R11-RES-NEXT:    .cfi_offset r7, -8
+; R11-RES-NEXT:    .cfi_offset r6, -12
+; R11-RES-NEXT:    .cfi_offset r5, -16
+; R11-RES-NEXT:    .cfi_offset r4, -20
+; R11-RES-NEXT:    .save {r11, lr}
+; R11-RES-NEXT:    push.w {r11, lr}
+; R11-RES-NEXT:    .cfi_def_cfa_offset 28
+; R11-RES-NEXT:    .cfi_offset lr, -24
+; R11-RES-NEXT:    .cfi_offset r11, -28
+; R11-RES-NEXT:    .setfp r11, sp
+; R11-RES-NEXT:    mov r11, sp
+; R11-RES-NEXT:    .cfi_def_cfa_register r11
+; R11-RES-NEXT:    .pad #4
+; R11-RES-NEXT:    sub sp, #4
+; R11-RES-NEXT:    cmp r0, #0
+; R11-RES-NEXT:    mov r5, r2
+; R11-RES-NEXT:    mov r4, r1
+; R11-RES-NEXT:    it ne
+; R11-RES-NEXT:    blne knr
+; R11-RES-NEXT:    adds r0, r5, #7
+; R11-RES-NEXT:    bic r0, r0, #7
+; R11-RES-NEXT:    sub.w r0, sp, r0
+; R11-RES-NEXT:    mov sp, r0
+; R11-RES-NEXT:    bl take_ptr
+; R11-RES-NEXT:    mov r0, r4
+; R11-RES-NEXT:    movs r1, #0
+; R11-RES-NEXT:    bl __aeabi_fcmpeq
+; R11-RES-NEXT:    cmp r0, #0
+; R11-RES-NEXT:    it eq
+; R11-RES-NEXT:    bleq knr
+; R11-RES-NEXT:    mov sp, r11
+; R11-RES-NEXT:    pop.w {r11, lr}
+; R11-RES-NEXT:    pop.w {r4, r5, r6, r7, r12}
+; R11-RES-NEXT:    aut r12, lr, sp
+; R11-RES-NEXT:    bx lr
+;
+; R11-ABI-LABEL: test3:
+; R11-ABI:         .cfi_startproc
+; R11-ABI-NEXT:  @ %bb.0: @ %entry
+; R11-ABI-NEXT:    pac r12, lr, sp
+; R11-ABI-NEXT:    .save {r4, r5, r6, r7, ra_auth_code}
+; R11-ABI-NEXT:    push.w {r4, r5, r6, r7, r12}
+; R11-ABI-NEXT:    .cfi_def_cfa_offset 20
+; R11-ABI-NEXT:    .cfi_offset ra_auth_code, -4
+; R11-ABI-NEXT:    .cfi_offset r7, -8
+; R11-ABI-NEXT:    .cfi_offset r6, -12
+; R11-ABI-NEXT:    .cfi_offset r5, -16
+; R11-ABI-NEXT:    .cfi_offset r4, -20
+; R11-ABI-NEXT:    .save {r11, lr}
+; R11-ABI-NEXT:    push.w {r11, lr}
+; R11-ABI-NEXT:    .cfi_def_cfa_offset 28
+; R11-ABI-NEXT:    .cfi_offset lr, -24
+; R11-ABI-NEXT:    .cfi_offset r11, -28
+; R11-ABI-NEXT:    .setfp r11, sp
+; R11-ABI-NEXT:    mov r11, sp
+; R11-ABI-NEXT:    .cfi_def_cfa_register r11
+; R11-ABI-NEXT:    .pad #4
+; R11-ABI-NEXT:    sub sp, #4
+; R11-ABI-NEXT:    cmp r0, #0
+; R11-ABI-NEXT:    mov r5, r2
+; R11-ABI-NEXT:    mov r4, r1
+; R11-ABI-NEXT:    it ne
+; R11-ABI-NEXT:    blne knr
+; R11-ABI-NEXT:    adds r0, r5, #7
+; R11-ABI-NEXT:    bic r0, r0, #7
+; R11-ABI-NEXT:    sub.w r0, sp, r0
+; R11-ABI-NEXT:    mov sp, r0
+; R11-ABI-NEXT:    bl take_ptr
+; R11-ABI-NEXT:    mov r0, r4
+; R11-ABI-NEXT:    movs r1, #0
+; R11-ABI-NEXT:    bl __aeabi_fcmpeq
+; R11-ABI-NEXT:    cmp r0, #0
+; R11-ABI-NEXT:    it eq
+; R11-ABI-NEXT:    bleq knr
+; R11-ABI-NEXT:    mov sp, r11
+; R11-ABI-NEXT:    pop.w {r11, lr}
+; R11-ABI-NEXT:    pop.w {r4, r5, r6, r7, r12}
+; R11-ABI-NEXT:    aut r12, lr, sp
+; R11-ABI-NEXT:    bx lr
 entry:
   %tobool.not = icmp eq i32 %c, 0
   br i1 %tobool.not, label %if.end, label %if.then
diff --git a/llvm/test/CodeGen/Thumb2/pacbti-m-indirect-tail-call.ll b/llvm/test/CodeGen/Thumb2/pacbti-m-indirect-tail-call.ll
index 4dfac252e231..615af15e8b56 100644
--- a/llvm/test/CodeGen/Thumb2/pacbti-m-indirect-tail-call.ll
+++ b/llvm/test/CodeGen/Thumb2/pacbti-m-indirect-tail-call.ll
@@ -1,11 +1,30 @@
-; RUN: llc %s -o - | FileCheck %s --check-prefix=CHECK1
-; RUN: llc %s -o - | FileCheck %s --check-prefix=CHECK2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc %s -o - | FileCheck %s
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 target triple = "thumbv8.1m.main-arm-unknown-eabi"
 
 @p = hidden local_unnamed_addr global ptr null, align 4
 
 define hidden i32 @f(i32 %a, i32 %b, i32 %c, i32 %d) local_unnamed_addr #0 {
+; CHECK-LABEL: f:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    .save {r4, r5, r6, r7, ra_auth_code, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r12, lr}
+; CHECK-NEXT:    mov r7, r3
+; CHECK-NEXT:    mov r5, r2
+; CHECK-NEXT:    mov r6, r1
+; CHECK-NEXT:    bl g
+; CHECK-NEXT:    movw r1, :lower16:p
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    movt r1, :upper16:p
+; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    ldr r4, [r1]
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    blx r4
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r12, lr}
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    bx lr
 entry:
   %call = tail call i32 @g(i32 %a) #0
   %0 = load ptr, ptr @p, align 4
@@ -13,19 +32,6 @@ entry:
   ret i32 %call1
 }
 
-; CHECK1-LABEL: f
-; ...
-; CHECK1:       aut r12, lr, sp
-; CHECK1-NOT:   bx r12
-
-; CHECK2-LABEL: f
-; ...
-; CHECK2:       blx r4
-; CHECK2-NEXT:  ldr r12, [sp], #4
-; CHECK2-NEXT:  pop.w {r4, r5, r6, r7, lr}
-; CHECK2-NEXT:  aut r12, lr, sp
-; CHECK2-NEXT:  bx lr
-
 declare dso_local i32 @g(i32) local_unnamed_addr #0
 
 attributes #0 = { nounwind "sign-return-address"="non-leaf"}
diff --git a/llvm/test/CodeGen/Thumb2/pacbti-m-outliner-3.ll b/llvm/test/CodeGen/Thumb2/pacbti-m-outliner-3.ll
index 1b13e06546f1..d02d4b51d73b 100644
--- a/llvm/test/CodeGen/Thumb2/pacbti-m-outliner-3.ll
+++ b/llvm/test/CodeGen/Thumb2/pacbti-m-outliner-3.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc --force-dwarf-frame-section %s -o - | FileCheck %s
 ; RUN: llc --filetype=obj %s -o - | llvm-readelf -s --unwind - | FileCheck %s --check-prefix=UNWIND
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
@@ -21,12 +22,43 @@ target triple = "thumbv7m-arm-none-eabi"
 ; }
 
 define hidden i32 @h(i32 %a, i32 %b) local_unnamed_addr #0 {
+; CHECK-LABEL: h:
+; CHECK:         .cfi_sections .debug_frame
+; CHECK-NEXT:    .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    add r0, r1
+; CHECK-NEXT:    bx lr
 entry:
   %add = add nsw i32 %b, %a
   ret i32 %add
 }
 
 define hidden i32 @f(i32 %a, i32 %b, i32 %c, i32 %d) local_unnamed_addr #0 {
+; CHECK-LABEL: f:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    .save {r4, r5, r6, ra_auth_code, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    push.w {r3, r4, r5, r6, r12, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    .cfi_offset lr, -4
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -8
+; CHECK-NEXT:    .cfi_offset r6, -12
+; CHECK-NEXT:    .cfi_offset r5, -16
+; CHECK-NEXT:    .cfi_offset r4, -20
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    bmi .LBB1_2
+; CHECK-NEXT:  @ %bb.1: @ %if.end
+; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    b .LBB1_3
+; CHECK-NEXT:  .LBB1_2:
+; CHECK-NEXT:    mov.w r0, #-1
+; CHECK-NEXT:  .LBB1_3: @ %return
+; CHECK-NEXT:    pop.w {r3, r4, r5, r6, r12, lr}
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp slt i32 %a, 0
   br i1 %cmp, label %return, label %if.end
@@ -48,34 +80,32 @@ return:                                           ; preds = %entry, %if.end
   ret i32 %retval.0
 }
 
-; CHECK-LABEL: f:
-; ...
-; CHECK:       pac    r12, lr, sp
-; CHECK-NEXT:  .save  {r4, r5, r6, lr}
-; CHECK-NEXT:  push   {r4, r5, r6, lr}
-; CHECK-NEXT:  .cfi_def_cfa_offset 16
-; CHECK-NEXT:  .cfi_offset lr, -4
-; CHECK-NEXT:  .cfi_offset r6, -8
-; CHECK-NEXT:  .cfi_offset r5, -12
-; CHECK-NEXT:  .cfi_offset r4, -16
-; CHECK-NEXT:  .save  {ra_auth_code}
-; CHECK-NEXT:  str    r12, [sp, #-4]!
-; CHECK-NEXT:  .cfi_def_cfa_offset 20
-; CHECK-NEXT:  .cfi_offset ra_auth_code, -20
-; CHECK-NEXT:  .pad    #4
-; CHECK-NEXT:  sub     sp, #4
-; CHECK-NEXT:  .cfi_def_cfa_offset 24
-; ...
-; CHECK:        bl  OUTLINED_FUNCTION_0
-; ...
-; CHECK:        add    sp, #4
-; CHECK-NEXT:   ldr    r12, [sp], #4
-; CHECK-NEXT:   pop.w  {r4, r5, r6, lr}
-; CHECK-NEXT:   aut    r12, lr, sp
-; CHECK-NEXT:   bx     lr
-
-
 define hidden i32 @g(i32 %a, i32 %b, i32 %c, i32 %d) local_unnamed_addr #0 {
+; CHECK-LABEL: g:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    .save {r4, r5, r6, ra_auth_code, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    push.w {r3, r4, r5, r6, r12, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    .cfi_offset lr, -4
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -8
+; CHECK-NEXT:    .cfi_offset r6, -12
+; CHECK-NEXT:    .cfi_offset r5, -16
+; CHECK-NEXT:    .cfi_offset r4, -20
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    bmi .LBB2_2
+; CHECK-NEXT:  @ %bb.1: @ %if.end
+; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
+; CHECK-NEXT:    adds r0, #1
+; CHECK-NEXT:    b .LBB2_3
+; CHECK-NEXT:  .LBB2_2:
+; CHECK-NEXT:    mov.w r0, #-1
+; CHECK-NEXT:  .LBB2_3: @ %return
+; CHECK-NEXT:    pop.w {r3, r4, r5, r6, r12, lr}
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp slt i32 %a, 0
   br i1 %cmp, label %return, label %if.end
@@ -96,30 +126,6 @@ return:                                           ; preds = %entry, %if.end
   %retval.0 = phi i32 [ %add3, %if.end ], [ -1, %entry ]
   ret i32 %retval.0
 }
-; CHECK-LABEL: g:
-; CHECK:       pac    r12, lr, sp
-; CHECK-NEXT:  .save  {r4, r5, r6, lr}
-; CHECK-NEXT:  push   {r4, r5, r6, lr}
-; CHECK-NEXT:  .cfi_def_cfa_offset 16
-; CHECK-NEXT:  .cfi_offset lr, -4
-; CHECK-NEXT:  .cfi_offset r6, -8
-; CHECK-NEXT:  .cfi_offset r5, -12
-; CHECK-NEXT:  .cfi_offset r4, -16
-; CHECK-NEXT:  .save  {ra_auth_code}
-; CHECK-NEXT:  str    r12, [sp, #-4]!
-; CHECK-NEXT:  .cfi_def_cfa_offset 20
-; CHECK-NEXT:  .cfi_offset ra_auth_code, -20
-; CHECK-NEXT:  .pad   #4
-; CHECK-NEXT:  sub    sp, #4
-; CHECK-NEXT:  .cfi_def_cfa_offset 24
-; ...
-; CHECK:        bl  OUTLINED_FUNCTION_0
-; ...
-; CHECK:       add    sp, #4
-; CHECK-NEXT:  ldr    r12, [sp], #4
-; CHECK-NEXT:  pop.w  {r4, r5, r6, lr}
-; CHECK-NEXT:  aut    r12, lr, sp
-; CHECK-NEXT:  bx     lr
 
 ; CHECK-LABEL: OUTLINED_FUNCTION_0:
 ; CHECK:       pac    r12, lr, sp
@@ -147,18 +153,20 @@ attributes #0 = { minsize noinline norecurse nounwind optsize readnone uwtable "
 
 ; UNWIND-LABEL: FunctionAddress: 0x4
 ; UNWIND:       0x00      ; vsp = vsp + 4
+; UNWIND-NEXT:  0xA2      ; pop {r4, r5, r6}
 ; UNWIND-NEXT:  0xB4      ; pop ra_auth_code
-; UNWIND-NEXT:  0xAA      ; pop {r4, r5, r6, lr}
+; UNWIND-NEXT:  0x84 0x00 ; pop {lr}
 
-; UNWIND-LABEL: FunctionAddress: 0x30
+; UNWIND-LABEL: FunctionAddress: 0x26
 ; UNWIND:       0x00      ; vsp = vsp + 4
+; UNWIND-NEXT:  0xA2      ; pop {r4, r5, r6}
 ; UNWIND-NEXT:  0xB4      ; pop ra_auth_code
-; UNWIND-NEXT:  0xAA      ; pop {r4, r5, r6, lr}
+; UNWIND-NEXT:  0x84 0x00 ; pop {lr}
 
-; UNWIND-LABEL: FunctionAddress: 0x5C
+; UNWIND-LABEL: FunctionAddress: 0x48
 ; UNWIND:       0xB4      ; pop ra_auth_code
-; UNWIND:       0x84 0x00 ; pop {lr}
- 
-; UNWIND-LABEL: 0000005d {{.*}} OUTLINED_FUNCTION_0
+; UNWIND-NEXT:  0x84 0x00 ; pop {lr}
+
+; UNWIND-LABEL: 00000049 {{.*}} OUTLINED_FUNCTION_0
 ; UNWIND-LABEL: 00000005 {{.*}} f
-; UNWIND-LABEL: 00000031 {{.*}} g
+; UNWIND-LABEL: 00000027 {{.*}} g
diff --git a/llvm/test/CodeGen/Thumb2/pacbti-m-outliner-4.ll b/llvm/test/CodeGen/Thumb2/pacbti-m-outliner-4.ll
index 38c23977b623..8777d517c4ba 100644
--- a/llvm/test/CodeGen/Thumb2/pacbti-m-outliner-4.ll
+++ b/llvm/test/CodeGen/Thumb2/pacbti-m-outliner-4.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc --force-dwarf-frame-section %s -o - | FileCheck %s
 ; RUN: llc --filetype=obj %s -o - | llvm-readelf -s --unwind - | FileCheck %s --check-prefix=UNWIND
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
@@ -27,6 +28,37 @@ target triple = "thumbv7m-arm-none-eabi"
 @_ZTIi = external dso_local constant ptr
 
 define hidden i32 @_Z1hii(i32 %a, i32 %b) local_unnamed_addr #0 {
+; CHECK-LABEL: _Z1hii:
+; CHECK:         .cfi_sections .debug_frame
+; CHECK-NEXT:    .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    .save {r7, ra_auth_code, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    push.w {r6, r7, r12, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset lr, -4
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -8
+; CHECK-NEXT:    .cfi_offset r7, -12
+; CHECK-NEXT:    cmp.w r0, #-1
+; CHECK-NEXT:    ble .LBB0_2
+; CHECK-NEXT:  @ %bb.1: @ %if.end
+; CHECK-NEXT:    add r0, r1
+; CHECK-NEXT:    pop.w {r3, r7, r12, lr}
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:  .LBB0_2: @ %if.then
+; CHECK-NEXT:    movs r0, #4
+; CHECK-NEXT:    bl __cxa_allocate_exception
+; CHECK-NEXT:    movs r1, #1
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    ldr r1, .LCPI0_0
+; CHECK-NEXT:    bl __cxa_throw
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long _ZTIi
 entry:
   %cmp = icmp slt i32 %a, 0
   br i1 %cmp, label %if.then, label %if.end
@@ -42,31 +74,40 @@ if.end:                                           ; preds = %entry
   ret i32 %add
 }
 
-; CHECK-LABEL: _Z1hii:
-; ...
-; CHECK:    pac    r12, lr, sp
-; CHECK-NEXT:    .save    {r7, lr}
-; CHECK-NEXT:    push    {r7, lr}
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
-; CHECK-NEXT:    .cfi_offset lr, -4
-; CHECK-NEXT:    .cfi_offset r7, -8
-; CHECK-NEXT:    .save    {ra_auth_code}
-; CHECK-NEXT:    str    r12, [sp, #-4]!
-; CHECK-NEXT:    .cfi_def_cfa_offset 12
-; CHECK-NEXT:    .cfi_offset ra_auth_code, -12
-; CHECK-NEXT:    .pad    #4
-; CHECK-NEXT:    sub    sp, #4
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; ...
-; CHECK-NOT: pac
-; CHECK: aut
-; CHECK:     .cfi_endproc
-
 declare dso_local ptr @__cxa_allocate_exception(i32) local_unnamed_addr
 
 declare dso_local void @__cxa_throw(ptr, ptr, ptr) local_unnamed_addr
 
 define hidden i32 @_Z1fiiii(i32 %a, i32 %b, i32 %c, i32 %d) local_unnamed_addr #0 {
+; CHECK-LABEL: _Z1fiiii:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    .save {r4, r5, r6, ra_auth_code, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    push.w {r3, r4, r5, r6, r12, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    .cfi_offset lr, -4
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -8
+; CHECK-NEXT:    .cfi_offset r6, -12
+; CHECK-NEXT:    .cfi_offset r5, -16
+; CHECK-NEXT:    .cfi_offset r4, -20
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    bmi .LBB1_2
+; CHECK-NEXT:  @ %bb.1: @ %if.end
+; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
+; CHECK-NEXT:    adds r1, r0, r6
+; CHECK-NEXT:    muls r0, r1, r0
+; CHECK-NEXT:    adds r1, r4, r5
+; CHECK-NEXT:    sdiv r0, r0, r1
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    b .LBB1_3
+; CHECK-NEXT:  .LBB1_2:
+; CHECK-NEXT:    mov.w r0, #-1
+; CHECK-NEXT:  .LBB1_3: @ %return
+; CHECK-NEXT:    pop.w {r3, r4, r5, r6, r12, lr}
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp slt i32 %a, 0
   br i1 %cmp, label %return, label %if.end
@@ -85,35 +126,36 @@ return:                                           ; preds = %entry, %if.end
   ret i32 %retval.0
 }
 
-; CHECK-LABEL: _Z1fiiii:
-; ...
-; CHECK:    pac    r12, lr, sp
-; CHECK-NEXT:    .save    {r4, r5, r6, lr}
-; CHECK-NEXT:    push    {r4, r5, r6, lr}
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset lr, -4
-; CHECK-NEXT:    .cfi_offset r6, -8
-; CHECK-NEXT:    .cfi_offset r5, -12
-; CHECK-NEXT:    .cfi_offset r4, -16
-; CHECK-NEXT:    .save    {ra_auth_code}
-; CHECK-NEXT:    str    r12, [sp, #-4]!
-; CHECK-NEXT:    .cfi_def_cfa_offset 20
-; CHECK-NEXT:    .cfi_offset ra_auth_code, -20
-; CHECK-NEXT:    .pad    #4
-; CHECK-NEXT:    sub    sp, #4
-; CHECK-NEXT:    .cfi_def_cfa_offset 24
-; ...
-; CHECK:    bl	OUTLINED_FUNCTION_0
-; ...
-; CHECK:    add    sp, #4
-; CHECK-NEXT:    ldr    r12, [sp], #4
-; CHECK-NEXT:    pop.w    {r4, r5, r6, lr}
-; CHECK-NEXT:    aut    r12, lr, sp
-; CHECK-NEXT:    bx    lr
-
-
-
 define hidden i32 @_Z1giiii(i32 %a, i32 %b, i32 %c, i32 %d) local_unnamed_addr #0 {
+; CHECK-LABEL: _Z1giiii:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    .save {r4, r5, r6, ra_auth_code, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    push.w {r3, r4, r5, r6, r12, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    .cfi_offset lr, -4
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -8
+; CHECK-NEXT:    .cfi_offset r6, -12
+; CHECK-NEXT:    .cfi_offset r5, -16
+; CHECK-NEXT:    .cfi_offset r4, -20
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    bmi .LBB2_2
+; CHECK-NEXT:  @ %bb.1: @ %if.end
+; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
+; CHECK-NEXT:    adds r1, r0, r6
+; CHECK-NEXT:    muls r0, r1, r0
+; CHECK-NEXT:    adds r1, r4, r5
+; CHECK-NEXT:    sdiv r0, r0, r1
+; CHECK-NEXT:    adds r0, #1
+; CHECK-NEXT:    b .LBB2_3
+; CHECK-NEXT:  .LBB2_2:
+; CHECK-NEXT:    mov.w r0, #-1
+; CHECK-NEXT:  .LBB2_3: @ %return
+; CHECK-NEXT:    pop.w {r3, r4, r5, r6, r12, lr}
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp slt i32 %a, 0
   br i1 %cmp, label %return, label %if.end
@@ -132,33 +174,6 @@ return:                                           ; preds = %entry, %if.end
   ret i32 %retval.0
 }
 
-; CHECK-LABEL: _Z1giiii:
-; ...
-; CHECK:    pac    r12, lr, sp
-; CHECK-NEXT:    .save    {r4, r5, r6, lr}
-; CHECK-NEXT:    push    {r4, r5, r6, lr}
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset lr, -4
-; CHECK-NEXT:    .cfi_offset r6, -8
-; CHECK-NEXT:    .cfi_offset r5, -12
-; CHECK-NEXT:    .cfi_offset r4, -16
-; CHECK-NEXT:    .save    {ra_auth_code}
-; CHECK-NEXT:    str    r12, [sp, #-4]!
-; CHECK-NEXT:    .cfi_def_cfa_offset 20
-; CHECK-NEXT:    .cfi_offset ra_auth_code, -20
-; CHECK-NEXT:    .pad    #4
-; CHECK-NEXT:    sub    sp, #4
-; CHECK-NEXT:    .cfi_def_cfa_offset 24
-; ...
-; CHECK:    bl	OUTLINED_FUNCTION_0
-; ...
-; CHECK:    add    sp, #4
-; CHECK-NEXT:    ldr    r12, [sp], #4
-; CHECK-NEXT:    pop.w    {r4, r5, r6, lr}
-; CHECK-NEXT:    aut    r12, lr, sp
-; CHECK-NEXT:    bx    lr
-
-
 ; CHEK-LABEL: OUTLINED_FUNCTION_0:
 ; CHECK-NOT: pac
 ; CHECK-NOT: aut
@@ -177,32 +192,31 @@ attributes #2 = { noreturn "sign-return-address"="non-leaf" }
 
 
 ; UNWIND-LABEL: FunctionAddress: 0x0
-; UNWIND:       Opcodes
+; UNWIND: Opcodes [
 ; UNWIND-NEXT:  0x00      ; vsp = vsp + 4
+; UNWIND-NEXT:  0x80 0x08 ; pop {r7}
 ; UNWIND-NEXT:  0xB4      ; pop ra_auth_code
-; UNWIND-NEXT:  0x84 0x08 ; pop {r7, lr}
-; UNWIND-NEXT:  0xB0      ; finish
-; UNWIND-NEXT:  0xB0      ; finish
+; UNWIND-NEXT:  0x84 0x00 ; pop {lr}
 
-; UNWIND-LABEL: FunctionAddress: 0x3C
-; UNWIND:       Opcodes
+; UNWIND-LABEL: FunctionAddress: 0x30
+; UNWIND: Opcodes [
 ; UNWIND-NEXT:  0x00      ; vsp = vsp + 4
+; UNWIND-NEXT:  0xA2      ; pop {r4, r5, r6}
 ; UNWIND-NEXT:  0xB4      ; pop ra_auth_code
-; UNWIND-NEXT:  0xAA      ; pop {r4, r5, r6, lr}
+; UNWIND-NEXT:  0x84 0x00 ; pop {lr}
 
-; UNWIND-LABEL: FunctionAddress: 0x72
-; UNWIND:       Opcodes
+; UNWIND-LABEL: FunctionAddress: 0x5C
+; UNWIND: Opcodes [
 ; UNWIND-NEXT:  0x00      ; vsp = vsp + 4
+; UNWIND-NEXT:  0xA2      ; pop {r4, r5, r6}
 ; UNWIND-NEXT:  0xB4      ; pop ra_auth_code
-; UNWIND-NEXT:  0xAA      ; pop {r4, r5, r6, lr}
+; UNWIND-NEXT:  0x84 0x00 ; pop {lr}
 
-; UNWIND-LABEL: FunctionAddress: 0xA8
-; UNWIND:       Opcodes
-; UNWIND-NEXT:  0xB0      ; finish
-; UNWIND-NEXT:  0xB0      ; finish
+; UNWIND-LABEL: FunctionAddress: 0x88
+; UNWIND: Opcodes [
 ; UNWIND-NEXT:  0xB0      ; finish
 
-; UNWIND: 000000a9 {{.*}} OUTLINED_FUNCTION_0
+; UNWIND: 00000089 {{.*}} OUTLINED_FUNCTION_0
 ; UWNIND: 00000001 {{.*}} _Z1hii
-; UWNIND: 0000003d {{.*}} _Z1fiiii
-; UWNIND: 00000073 {{.*}} _Z1giiii
+; UWNIND: 00000031 {{.*}} _Z1fiiii
+; UWNIND: 0000005d {{.*}} _Z1giiii
diff --git a/llvm/test/CodeGen/Thumb2/pacbti-m-overalign.ll b/llvm/test/CodeGen/Thumb2/pacbti-m-overalign.ll
index 5dce6752c065..5354303a034d 100644
--- a/llvm/test/CodeGen/Thumb2/pacbti-m-overalign.ll
+++ b/llvm/test/CodeGen/Thumb2/pacbti-m-overalign.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc --force-dwarf-frame-section %s -o - | FileCheck %s
 ; RUN: llc --filetype=obj %s -o - | llvm-readelf --unwind - | FileCheck %s --check-prefix=UNWIND
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
@@ -15,6 +16,39 @@ target triple = "thumbv8.1m.main-arm-none-eabi"
 ; }
 
 define hidden i32 @_Z1fv() local_unnamed_addr "sign-return-address"="non-leaf" {
+; CHECK-LABEL: _Z1fv:
+; CHECK:         .cfi_sections .debug_frame
+; CHECK-NEXT:    .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    .save {r4, r6, r7, ra_auth_code, lr}
+; CHECK-NEXT:    push.w {r4, r6, r7, r12, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 20
+; CHECK-NEXT:    .cfi_offset lr, -4
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -8
+; CHECK-NEXT:    .cfi_offset r7, -12
+; CHECK-NEXT:    .cfi_offset r6, -16
+; CHECK-NEXT:    .cfi_offset r4, -20
+; CHECK-NEXT:    .setfp r7, sp, #8
+; CHECK-NEXT:    add r7, sp, #8
+; CHECK-NEXT:    .cfi_def_cfa r7, 12
+; CHECK-NEXT:    .pad #44
+; CHECK-NEXT:    sub sp, #44
+; CHECK-NEXT:    mov r4, sp
+; CHECK-NEXT:    bfc r4, #0, #5
+; CHECK-NEXT:    mov sp, r4
+; CHECK-NEXT:    mov r1, sp
+; CHECK-NEXT:    movs r0, #4
+; CHECK-NEXT:    bl _Z1giPi
+; CHECK-NEXT:    ldm.w sp, {r0, r1, r2, r3}
+; CHECK-NEXT:    sub.w r4, r7, #8
+; CHECK-NEXT:    add r0, r1
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    add r0, r3
+; CHECK-NEXT:    mov sp, r4
+; CHECK-NEXT:    pop.w {r4, r6, r7, r12, lr}
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    bx lr
 entry:
   %a = alloca [4 x i32], align 32
   %call = call i32 @_Z1giPi(i32 4, ptr nonnull %a)
@@ -31,29 +65,6 @@ entry:
   ret i32 %add.3
 }
 
-; CHECK-LABEL: _Z1fv:
-; CHECK:      pac     r12, lr, sp
-; CHECK:      .save   {r4, r6, r7, lr}
-; CHECK-NEXT: push    {r4, r6, r7, lr}
-; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: .cfi_offset lr, -4
-; CHECK-NEXT: .cfi_offset r7, -8
-; CHECK-NEXT: .cfi_offset r6, -12
-; CHECK-NEXT: .cfi_offset r4, -16
-; CHECK-NEXT: .setfp  r7, sp, #8
-; CHECK-NEXT: add     r7, sp, #8
-; CHECK-NEXT: .cfi_def_cfa r7, 8
-; CHECK-NEXT: .save   {ra_auth_code}
-; CHECK-NEXT: str     r12, [sp, #-4]!
-; CHECK-NEXT: .cfi_offset ra_auth_code, -20
-; CHECK-NEXT: .pad    #44
-; CHECK-NEXT: sub     sp, #44
-; CHECK:      ldr     r12, [sp], #4
-; CHECK-NEXT: pop.w   {r4, r6, r7, lr}
-; CHECK-NEXT: aut     r12, lr, sp
-; CHECK-NEXT: bx      lr
-
-
 declare dso_local i32 @_Z1giPi(i32, ptr) local_unnamed_addr
 
 !llvm.module.flags = !{!0, !1, !2}
@@ -64,6 +75,7 @@ declare dso_local i32 @_Z1giPi(i32, ptr) local_unnamed_addr
 
 ; UNWIND-LABEL:        FunctionAddress: 0x0
 ; UNWIND:          0x97      ; vsp = r7
-; UNWIND:          0x42      ; vsp = vsp - 12
-; UNWIND:          0xB4      ; pop ra_auth_code
-; UNWIND:          0x84 0x0D ; pop {r4, r6, r7, lr}
+; UNWIND-NEXT:     0x41      ; vsp = vsp - 8
+; UNWIND-NEXT:     0x80 0x0D ; pop {r4, r6, r7}
+; UNWIND-NEXT:     0xB4      ; pop ra_auth_code
+; UNWIND-NEXT:     0x84 0x00 ; pop {lr}
diff --git a/llvm/test/CodeGen/Thumb2/pacbti-m-stack-arg.ll b/llvm/test/CodeGen/Thumb2/pacbti-m-stack-arg.ll
index cae38b5e4a5a..c0c32de509b7 100644
--- a/llvm/test/CodeGen/Thumb2/pacbti-m-stack-arg.ll
+++ b/llvm/test/CodeGen/Thumb2/pacbti-m-stack-arg.ll
@@ -19,17 +19,14 @@ define i32 @test_non_leaf(i32 %arg0, i32 %arg1, i32 %arg2, i32 %arg3, i32 %x) "s
 ; CHECK-LABEL: test_non_leaf:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    pac r12, lr, sp
-; CHECK-NEXT:    .save {r7, lr}
-; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .save {ra_auth_code}
-; CHECK-NEXT:    str r12, [sp, #-4]!
+; CHECK-NEXT:    .save {r7, ra_auth_code, lr}
+; CHECK-NEXT:    push.w {r7, r12, lr}
 ; CHECK-NEXT:    .pad #4
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    bl otherfn
 ; CHECK-NEXT:    ldr r0, [sp, #16]
 ; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    ldr r12, [sp], #4
-; CHECK-NEXT:    pop.w {r7, lr}
+; CHECK-NEXT:    pop.w {r7, r12, lr}
 ; CHECK-NEXT:    aut r12, lr, sp
 ; CHECK-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/Thumb2/pacbti-m-varargs-1.ll b/llvm/test/CodeGen/Thumb2/pacbti-m-varargs-1.ll
index d027c9e8c7b5..2b7abfabf703 100644
--- a/llvm/test/CodeGen/Thumb2/pacbti-m-varargs-1.ll
+++ b/llvm/test/CodeGen/Thumb2/pacbti-m-varargs-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc --force-dwarf-frame-section %s -o - | FileCheck %s
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 target triple = "thumbv8.1m.main-arm-none-eabi"
@@ -5,6 +6,50 @@ target triple = "thumbv8.1m.main-arm-none-eabi"
 %"struct.std::__va_list" = type { ptr }
 
 define hidden i32 @_Z1fiz(i32 %n, ...) local_unnamed_addr #0 {
+; CHECK-LABEL: _Z1fiz:
+; CHECK:         .cfi_sections .debug_frame
+; CHECK-NEXT:    .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    .pad #12
+; CHECK-NEXT:    sub sp, #12
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    .save {r7, ra_auth_code, lr}
+; CHECK-NEXT:    push.w {r7, r12, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    .cfi_offset lr, -16
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -20
+; CHECK-NEXT:    .cfi_offset r7, -24
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .cfi_def_cfa_offset 28
+; CHECK-NEXT:    add.w r12, sp, #16
+; CHECK-NEXT:    cmp r0, #1
+; CHECK-NEXT:    stm.w r12, {r1, r2, r3}
+; CHECK-NEXT:    add r1, sp, #16
+; CHECK-NEXT:    str r1, [sp]
+; CHECK-NEXT:    blt .LBB0_3
+; CHECK-NEXT:  @ %bb.1: @ %for.body.lr.ph
+; CHECK-NEXT:    ldr r1, [sp]
+; CHECK-NEXT:    dls lr, r0
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    adds r1, #4
+; CHECK-NEXT:  .LBB0_2: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    str r1, [sp]
+; CHECK-NEXT:    ldr r2, [r1, #-4]
+; CHECK-NEXT:    adds r1, #4
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    le lr, .LBB0_2
+; CHECK-NEXT:    b .LBB0_4
+; CHECK-NEXT:  .LBB0_3:
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:  .LBB0_4: @ %for.cond.cleanup
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop.w {r7, r12, lr}
+; CHECK-NEXT:    add sp, #12
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    bx lr
 entry:
   %ap = alloca %"struct.std::__va_list", align 4
   call void @llvm.va_start(ptr nonnull %ap)
@@ -33,34 +78,6 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
 }
 
-; CHECK-LABEL: _Z1fiz:
-; CHECK:      pac    r12, lr, sp
-; CHECK-NEXT: .pad    #12
-; CHECK-NEXT: sub    sp, #12
-; CHECK-NEXT: .cfi_def_cfa_offset 12
-; CHECK-NEXT:  .save    {r7, lr}
-; CHECK-NEXT: push    {r7, lr}
-; CHECK-NEXT: .cfi_def_cfa_offset 20
-; CHECK-NEXT: .cfi_offset lr, -16
-; CHECK-NEXT: .cfi_offset r7, -20
-; CHECK-NEXT: .save    {ra_auth_code}
-; CHECK-NEXT: str    r12, [sp, #-4]!
-; CHECK-NEXT: .cfi_def_cfa_offset 24
-; CHECK-NEXT: .cfi_offset ra_auth_code, -24
-; CHECK-NEXT: .pad    #4
-; CHECK-NEXT: sub    sp, #4
-; CHECK-NEXT: .cfi_def_cfa_offset 28
-; ...
-; CHECK:      add.w r[[N:[0-9]*]], sp, #16
-; CHECK:      stm.w r[[N]], {r1, r2, r3}
-; ...
-; CHECK:      add    sp, #4
-; CHECK-NEXT: ldr    r12, [sp], #4
-; CHECK-NEXT: pop.w    {r7, lr}
-; CHECK-NEXT: add    sp, #12
-; CHECK-NEXT: aut    r12, lr, sp
-; CHECK-NEXT: bx    lr
-
 declare void @llvm.va_start(ptr) #1
 declare void @llvm.va_end(ptr) #1
 
diff --git a/llvm/test/CodeGen/Thumb2/pacbti-m-varargs-2.ll b/llvm/test/CodeGen/Thumb2/pacbti-m-varargs-2.ll
index 8019cd5b6109..03b769f256bc 100644
--- a/llvm/test/CodeGen/Thumb2/pacbti-m-varargs-2.ll
+++ b/llvm/test/CodeGen/Thumb2/pacbti-m-varargs-2.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc --force-dwarf-frame-section %s -o - | FileCheck %s
 ; RUN: llc --filetype=obj %s -o - | llvm-readelf --unwind - | FileCheck %s --check-prefix=UNWIND
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
@@ -19,6 +20,50 @@ target triple = "thumbv8.1m.main-arm-none-eabi"
 %"struct.std::__va_list" = type { ptr }
 
 define hidden i32 @_Z1fiz(i32 %n, ...) local_unnamed_addr #0 {
+; CHECK-LABEL: _Z1fiz:
+; CHECK:         .cfi_sections .debug_frame
+; CHECK-NEXT:    .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    .pad #12
+; CHECK-NEXT:    sub sp, #12
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    .save {r4, r5, r7, ra_auth_code, lr}
+; CHECK-NEXT:    push.w {r4, r5, r7, r12, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, -16
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -20
+; CHECK-NEXT:    .cfi_offset r7, -24
+; CHECK-NEXT:    .cfi_offset r5, -28
+; CHECK-NEXT:    .cfi_offset r4, -32
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    add r0, sp, #28
+; CHECK-NEXT:    movs r5, #0
+; CHECK-NEXT:    cmp r4, #1
+; CHECK-NEXT:    stm r0!, {r1, r2, r3}
+; CHECK-NEXT:    add r0, sp, #28
+; CHECK-NEXT:    str r0, [sp, #4]
+; CHECK-NEXT:    blt .LBB0_2
+; CHECK-NEXT:  .LBB0_1: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r0, [sp, #4]
+; CHECK-NEXT:    adds r1, r0, #4
+; CHECK-NEXT:    str r1, [sp, #4]
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    bl _Z1gi
+; CHECK-NEXT:    add r5, r0
+; CHECK-NEXT:    subs r4, #1
+; CHECK-NEXT:    bne .LBB0_1
+; CHECK-NEXT:  .LBB0_2: @ %for.cond.cleanup
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop.w {r4, r5, r7, r12, lr}
+; CHECK-NEXT:    add sp, #12
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    bx lr
 entry:
   %ap = alloca %"struct.std::__va_list", align 4
   call void @llvm.va_start(ptr nonnull %ap)
@@ -47,36 +92,6 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
 }
 
-; CHECK-LABEL: _Z1fiz:
-; CHECK:       pac    r12, lr, sp
-; CHECK-NEXT:  .pad   #12
-; CHECK-NEXT:  sub    sp, #12
-; CHECK-NEXT:  .cfi_def_cfa_offset 12
-; CHECK-NEXT:  .save   {r4, r5, r7, lr}
-; CHECK-NEXT:  push    {r4, r5, r7, lr}
-; CHECK-NEXT:  .cfi_def_cfa_offset 28
-; CHECK-NEXT:  .cfi_offset lr, -16
-; CHECK-NEXT:  .cfi_offset r7, -20
-; CHECK-NEXT:  .cfi_offset r5, -24
-; CHECK-NEXT:  .cfi_offset r4, -28
-; CHECK-NEXT:  .save  {ra_auth_code}
-; CHECK-NEXT:  str    r12, [sp, #-4]!
-; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: .cfi_offset ra_auth_code, -32
-; CHECK-NEXT:  .pad   #8
-; CHECK-NEXT:  sub    sp, #8
-; CHECK-NEXT: .cfi_def_cfa_offset 40
-; ...
-; CHECK:       add    r[[N:[0-9]*]], sp, #28
-; CHECK:       stm    r[[N]]!, {r1, r2, r3}
-; ...
-; CHECK:       add    sp, #8
-; CHECK-NEXT:  ldr    r12, [sp], #4
-; CHECK-NEXT:  pop.w  {r4, r5, r7, lr}
-; CHECK-NEXT:  add    sp, #12
-; CHECK-NEXT:  aut    r12, lr, sp
-; CHECK-NEXT:  bx     lr
-
 declare void @llvm.va_start(ptr) #1
 declare void @llvm.va_end(ptr) #1
 
@@ -92,7 +107,9 @@ attributes #1 = { nounwind "sign-return-address"="non-leaf"}
 !2 = !{i32 8, !"sign-return-address-all", i32 0}
 
 ; UNWIND-LABEL: FunctionAddress
-; UNWIND:       0x01      ; vsp = vsp + 8
-; UNWIND-NEXT:  0xB4      ; pop ra_auth_code
-; UNWIND-NEXT:  0x84 0x0B ; pop {r4, r5, r7, lr}
-; UNWIND-NEXT:  0x02      ; vsp = vsp + 12
+; UNWIND:      0x01      ; vsp = vsp + 8
+; UNWIND-NEXT: 0x80 0x0B ; pop {r4, r5, r7}
+; UNWIND-NEXT: 0xB4      ; pop ra_auth_code
+; UNWIND-NEXT: 0x84 0x00 ; pop {lr}
+; UNWIND-NEXT: 0x02      ; vsp = vsp + 12
+
diff --git a/llvm/test/CodeGen/Thumb2/pacbti-m-vla.ll b/llvm/test/CodeGen/Thumb2/pacbti-m-vla.ll
index c1d17a7587be..5eb5990be7c1 100644
--- a/llvm/test/CodeGen/Thumb2/pacbti-m-vla.ll
+++ b/llvm/test/CodeGen/Thumb2/pacbti-m-vla.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc --force-dwarf-frame-section %s -o - | FileCheck %s
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 target triple = "thumbv8.1m.main-arm-none-eabi"
@@ -14,6 +15,89 @@ target triple = "thumbv8.1m.main-arm-none-eabi"
 ; }
 
 define hidden i32 @f(i32 %n) local_unnamed_addr #0 {
+; CHECK-LABEL: f:
+; CHECK:         .cfi_sections .debug_frame
+; CHECK-NEXT:    .cfi_startproc
+; CHECK-NEXT:  @ %bb.0: @ %entry
+; CHECK-NEXT:    pac r12, lr, sp
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, ra_auth_code, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r12, lr}
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, -4
+; CHECK-NEXT:    .cfi_offset ra_auth_code, -8
+; CHECK-NEXT:    .cfi_offset r9, -12
+; CHECK-NEXT:    .cfi_offset r8, -16
+; CHECK-NEXT:    .cfi_offset r7, -20
+; CHECK-NEXT:    .cfi_offset r6, -24
+; CHECK-NEXT:    .cfi_offset r5, -28
+; CHECK-NEXT:    .cfi_offset r4, -32
+; CHECK-NEXT:    .setfp r7, sp, #12
+; CHECK-NEXT:    add r7, sp, #12
+; CHECK-NEXT:    .cfi_def_cfa r7, 20
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    movs r0, #7
+; CHECK-NEXT:    add.w r0, r0, r5, lsl #2
+; CHECK-NEXT:    bic r0, r0, #7
+; CHECK-NEXT:    sub.w r4, sp, r0
+; CHECK-NEXT:    mov sp, r4
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    bl g
+; CHECK-NEXT:    cmp r5, #1
+; CHECK-NEXT:    blt .LBB0_3
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    subs r0, r5, #1
+; CHECK-NEXT:    and r12, r5, #3
+; CHECK-NEXT:    cmp r0, #3
+; CHECK-NEXT:    bhs .LBB0_4
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    b .LBB0_6
+; CHECK-NEXT:  .LBB0_3:
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    b .LBB0_9
+; CHECK-NEXT:  .LBB0_4: @ %for.body.preheader.new
+; CHECK-NEXT:    bic r0, r5, #3
+; CHECK-NEXT:    movs r2, #1
+; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    sub.w r3, r4, #16
+; CHECK-NEXT:    add.w lr, r2, r0, lsr #2
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:  .LBB0_5: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r5, [r3, #16]!
+; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    add r0, r5
+; CHECK-NEXT:    ldrd r5, r1, [r3, #4]
+; CHECK-NEXT:    ldr r6, [r3, #12]
+; CHECK-NEXT:    add r0, r5
+; CHECK-NEXT:    add r0, r1
+; CHECK-NEXT:    add r0, r6
+; CHECK-NEXT:    le lr, .LBB0_5
+; CHECK-NEXT:  .LBB0_6: @ %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    beq .LBB0_9
+; CHECK-NEXT:  @ %bb.7: @ %for.body.epil
+; CHECK-NEXT:    ldr.w r3, [r4, r2, lsl #2]
+; CHECK-NEXT:    cmp.w r12, #1
+; CHECK-NEXT:    add r0, r3
+; CHECK-NEXT:    beq .LBB0_9
+; CHECK-NEXT:  @ %bb.8: @ %for.body.epil.1
+; CHECK-NEXT:    add.w r2, r4, r2, lsl #2
+; CHECK-NEXT:    cmp.w r12, #2
+; CHECK-NEXT:    ldr r1, [r2, #4]
+; CHECK-NEXT:    add r0, r1
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    ldrne r1, [r2, #8]
+; CHECK-NEXT:    addne r0, r1
+; CHECK-NEXT:  .LBB0_9: @ %for.cond.cleanup
+; CHECK-NEXT:    sub.w r4, r7, #12
+; CHECK-NEXT:    mov sp, r4
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r12, lr}
+; CHECK-NEXT:    aut r12, lr, sp
+; CHECK-NEXT:    bx lr
 entry:
   %vla = alloca i32, i32 %n, align 4
   %call = call i32 @g(i32 %n, ptr nonnull %vla) #0
@@ -88,32 +172,6 @@ for.body.epil.2:                                  ; preds = %for.body.epil.1
   br label %for.cond.cleanup
 }
 
-; CHECK-LABEL: f:
-; CHECK:       pac    r12, lr, sp
-; CHECK-NEXT: .save   {r4, r5, r6, r7, lr}
-; CHECK-NEXT: push    {r4, r5, r6, r7, lr}
-; CHECK-NEXT: .cfi_def_cfa_offset 20
-; CHECK-NEXT: .cfi_offset lr, -4
-; CHECK-NEXT: .cfi_offset r7, -8
-; CHECK-NEXT: .cfi_offset r6, -12
-; CHECK-NEXT: .cfi_offset r5, -16
-; CHECK-NEXT: .cfi_offset r4, -20
-; CHECK-NEXT: .setfp r7, sp, #12
-; CHECK-NEXT: add    r7, sp, #12
-; CHECK-NEXT: .cfi_def_cfa r7, 8
-; CHECK-NEXT: .save    {r8, r9, ra_auth_code}
-; CHECK-NEXT: push.w   {r8, r9, r12}
-; CHECK-NEXT: .cfi_offset ra_auth_code, -24
-; CHECK-NEXT: .cfi_offset r9, -28
-; CHECK-NEXT: .cfi_offset r8, -32
-; ...
-; CHECK:      sub.w  r[[N:[0-9]*]], r7, #24
-; CHECK-NEXT: mov    sp, r[[N]]
-; CHECK-NEXT: pop.w  {r8, r9, r12}
-; CHECK-NEXT: pop.w  {r4, r5, r6, r7, lr}
-; CHECK-NEXT: aut    r12, lr, sp
-; CHECK-NEXT: bx     lr
-
 declare dso_local i32 @g(i32, ptr) local_unnamed_addr #0
 
 attributes #0 = { nounwind "sign-return-address"="non-leaf"}
-- 
GitLab


From 933a56674e75ed372e000758378b7981e5b4d387 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles@arm.com>
Date: Mon, 28 Oct 2024 09:08:48 +0000
Subject: [PATCH 291/693] [PAuthLR] Add Missing Break Statement for
 MachineOperand Switch Statement (#113883)

There was a missing break, which led to an unannotated fallthrough when
merging #112171. This has caused sanitizer builds to fail.

This adds the missing break in the switch statement to ensure that the
fallthrough does not occur.
---
 llvm/lib/CodeGen/MachineOperand.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index 620779e2dec7..d9e5e9d9d1e4 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -772,6 +772,7 @@ static void printCFI(raw_ostream &OS, const MCCFIInstruction &CFI,
     OS << "negate_ra_sign_state_with_pc ";
     if (MCSymbol *Label = CFI.getLabel())
       MachineOperand::printSymbol(OS, *Label);
+    break;
   default:
     // TODO: Print the other CFI Operations.
     OS << "<unserializable cfi directive>";
-- 
GitLab


From 43a5719d9f54fb482d523a6e313d9b9b9af82379 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Mon, 28 Oct 2024 09:22:09 +0000
Subject: [PATCH 292/693] [RISCV] Use Sha extension in RVA23S64 profile
 (#113823)

In the ratified version of the RVA23S64 definition, the Sha extension is
now used to group together the set of hypervisor related extensions.

<https://github.com/riscv/riscv-profiles/blob/main/src/rva23-profile.adoc>
---
 llvm/lib/Target/RISCV/RISCVProfiles.td | 9 +--------
 llvm/test/CodeGen/RISCV/attributes.ll  | 2 +-
 2 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVProfiles.td b/llvm/lib/Target/RISCV/RISCVProfiles.td
index ce7d1973989f..78c076fdb0b2 100644
--- a/llvm/lib/Target/RISCV/RISCVProfiles.td
+++ b/llvm/lib/Target/RISCV/RISCVProfiles.td
@@ -82,14 +82,7 @@ defvar RVA23S64BaseFeatures = !listconcat(RVA22S64BaseFeatures,
                                            FeatureStdExtSscofpmf,
                                            FeatureStdExtSsnpm,
                                            FeatureStdExtSsu64xl,
-                                           FeatureStdExtH,
-                                           FeatureStdExtSsstateen,
-                                           FeatureStdExtShcounterenw,
-                                           FeatureStdExtShvstvala,
-                                           FeatureStdExtShtvala,
-                                           FeatureStdExtShvstvecd,
-                                           FeatureStdExtShvsatpa,
-                                           FeatureStdExtShgatpa]);
+                                           FeatureStdExtSha]);
 defvar RVA23S64Features = !listconcat(RVA23U64Features,
                                       RVA23S64BaseFeatures);
 
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index 4cc9ef2ea0d7..5c835befd6f5 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -583,7 +583,7 @@
 ; RVA22U64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicsr2p0_zihintpause2p0_zihpm2p0_zmmul1p0_za64rs1p0_zfhmin1p0_zba1p0_zbb1p0_zbs1p0_zkt1p0"
 ; RVA22S64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicsr2p0_zifencei2p0_zihintpause2p0_zihpm2p0_zmmul1p0_za64rs1p0_zfhmin1p0_zba1p0_zbb1p0_zbs1p0_zkt1p0_ssccptr1p0_sscounterenw1p0_sstvala1p0_sstvecd1p0_svade1p0_svbare1p0_svinval1p0_svpbmt1p0"
 ; RVA23U64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_v1p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicond1p0_zicsr2p0_zihintntl1p0_zihintpause2p0_zihpm2p0_zimop1p0_zmmul1p0_za64rs1p0_zawrs1p0_zfa1p0_zfhmin1p0_zca1p0_zcb1p0_zcmop1p0_zba1p0_zbb1p0_zbs1p0_zkt1p0_zvbb1p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvfhmin1p0_zvkb1p0_zvkt1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0_supm1p0"
-; RVA23S64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_v1p0_h1p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicond1p0_zicsr2p0_zifencei2p0_zihintntl1p0_zihintpause2p0_zihpm2p0_zimop1p0_zmmul1p0_za64rs1p0_zawrs1p0_zfa1p0_zfhmin1p0_zca1p0_zcb1p0_zcmop1p0_zba1p0_zbb1p0_zbs1p0_zkt1p0_zvbb1p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvfhmin1p0_zvkb1p0_zvkt1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0_shcounterenw1p0_shgatpa1p0_shtvala1p0_shvsatpa1p0_shvstvala1p0_shvstvecd1p0_ssccptr1p0_sscofpmf1p0_sscounterenw1p0_ssnpm1p0_ssstateen1p0_sstc1p0_sstvala1p0_sstvecd1p0_ssu64xl1p0_supm1p0_svade1p0_svbare1p0_svinval1p0_svnapot1p0_svpbmt1p0"
+; RVA23S64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_v1p0_h1p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicond1p0_zicsr2p0_zifencei2p0_zihintntl1p0_zihintpause2p0_zihpm2p0_zimop1p0_zmmul1p0_za64rs1p0_zawrs1p0_zfa1p0_zfhmin1p0_zca1p0_zcb1p0_zcmop1p0_zba1p0_zbb1p0_zbs1p0_zkt1p0_zvbb1p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvfhmin1p0_zvkb1p0_zvkt1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0_sha1p0_shcounterenw1p0_shgatpa1p0_shtvala1p0_shvsatpa1p0_shvstvala1p0_shvstvecd1p0_ssccptr1p0_sscofpmf1p0_sscounterenw1p0_ssnpm1p0_ssstateen1p0_sstc1p0_sstvala1p0_sstvecd1p0_ssu64xl1p0_supm1p0_svade1p0_svbare1p0_svinval1p0_svnapot1p0_svpbmt1p0"
 ; RVB23U64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicond1p0_zicsr2p0_zihintntl1p0_zihintpause2p0_zihpm2p0_zimop1p0_zmmul1p0_za64rs1p0_zawrs1p0_zfa1p0_zca1p0_zcb1p0_zcmop1p0_zba1p0_zbb1p0_zbs1p0_zkt1p0"
 ; RVB23S64: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicond1p0_zicsr2p0_zifencei2p0_zihintntl1p0_zihintpause2p0_zihpm2p0_zimop1p0_zmmul1p0_za64rs1p0_zawrs1p0_zfa1p0_zca1p0_zcb1p0_zcmop1p0_zba1p0_zbb1p0_zbs1p0_zkt1p0_ssccptr1p0_sscofpmf1p0_sscounterenw1p0_sstc1p0_sstvala1p0_sstvecd1p0_ssu64xl1p0_svade1p0_svbare1p0_svinval1p0_svnapot1p0_svpbmt1p0"
 ; RVM23U32: .attribute 5, "rv32i2p1_m2p0_zicbop1p0_zicond1p0_zicsr2p0_zihintntl1p0_zihintpause2p0_zimop1p0_zmmul1p0_zca1p0_zcb1p0_zce1p0_zcmop1p0_zcmp1p0_zcmt1p0_zba1p0_zbb1p0_zbs1p0"
-- 
GitLab


From 96f5c683500eb2d7f7c3984e3a056315c50c4662 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Mon, 28 Oct 2024 09:37:06 +0000
Subject: [PATCH 293/693] [RISCV] Lower @llvm.experimental.vector.compress for
 zvfhmin/zvfbfmin (#113770)

This is a follow up to #113291 and handles f16/bf16 with zvfhmin and
zvfbmin.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |   8 +-
 .../RISCV/rvv/fixed-vectors-compress-fp.ll    |  90 +++++++++++-
 .../test/CodeGen/RISCV/rvv/vector-compress.ll | 134 +++++++++++++++++-
 3 files changed, 224 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 69112d868bff..af7a39b2580a 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1081,7 +1081,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::CONCAT_VECTORS,
                           ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
                           ISD::VECTOR_DEINTERLEAVE, ISD::VECTOR_INTERLEAVE,
-                          ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE},
+                          ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE,
+                          ISD::VECTOR_COMPRESS},
                          VT, Custom);
       MVT EltVT = VT.getVectorElementType();
       if (isTypeLegal(EltVT))
@@ -1333,7 +1334,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         setOperationAction(ISD::UNDEF, VT, Custom);
 
         setOperationAction({ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE,
-                            ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR},
+                            ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
+                            ISD::VECTOR_COMPRESS},
                            VT, Custom);
 
         // FIXME: mload, mstore, mgather, mscatter, vp_load/store,
@@ -1440,8 +1442,6 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
              ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
              ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
             VT, Custom);
-
-        setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
       }
 
       // Custom-legalize bitcasts from fixed-length vectors to scalar types.
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compress-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compress-fp.ll
index 8f1ff7ed4a11..3069d6011075 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compress-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compress-fp.ll
@@ -1,6 +1,92 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh,+zvfbfmin < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh,+zvfbfmin < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfhmin,+zvfbfmin < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfhmin,+zvfbfmin < %s | FileCheck %s
+
+define <1 x bfloat> @vector_compress_v1bf16(<1 x bfloat> %v, <1 x i1> %mask) {
+; CHECK-LABEL: vector_compress_v1bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; CHECK-NEXT:    vcompress.vm v9, v8, v0
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %ret = call <1 x bfloat> @llvm.experimental.vector.compress.v1bf16(<1 x bfloat> %v, <1 x i1> %mask, <1 x bfloat> undef)
+  ret <1 x bfloat> %ret
+}
+
+define <1 x bfloat> @vector_compress_v1bf16_passthru(<1 x bfloat> %passthru, <1 x bfloat> %v, <1 x i1> %mask) {
+; CHECK-LABEL: vector_compress_v1bf16_passthru:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, tu, ma
+; CHECK-NEXT:    vcompress.vm v8, v9, v0
+; CHECK-NEXT:    ret
+  %ret = call <1 x bfloat> @llvm.experimental.vector.compress.v1bf16(<1 x bfloat> %v, <1 x i1> %mask, <1 x bfloat> %passthru)
+  ret <1 x bfloat> %ret
+}
+
+define <2 x bfloat> @vector_compress_v2bf16(<2 x bfloat> %v, <2 x i1> %mask) {
+; CHECK-LABEL: vector_compress_v2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vcompress.vm v9, v8, v0
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %ret = call <2 x bfloat> @llvm.experimental.vector.compress.v2bf16(<2 x bfloat> %v, <2 x i1> %mask, <2 x bfloat> undef)
+  ret <2 x bfloat> %ret
+}
+
+define <2 x bfloat> @vector_compress_v2bf16_passthru(<2 x bfloat> %passthru, <2 x bfloat> %v, <2 x i1> %mask) {
+; CHECK-LABEL: vector_compress_v2bf16_passthru:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, tu, ma
+; CHECK-NEXT:    vcompress.vm v8, v9, v0
+; CHECK-NEXT:    ret
+  %ret = call <2 x bfloat> @llvm.experimental.vector.compress.v2bf16(<2 x bfloat> %v, <2 x i1> %mask, <2 x bfloat> %passthru)
+  ret <2 x bfloat> %ret
+}
+
+define <4 x bfloat> @vector_compress_v4bf16(<4 x bfloat> %v, <4 x i1> %mask) {
+; CHECK-LABEL: vector_compress_v4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vcompress.vm v9, v8, v0
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %ret = call <4 x bfloat> @llvm.experimental.vector.compress.v4bf16(<4 x bfloat> %v, <4 x i1> %mask, <4 x bfloat> undef)
+  ret <4 x bfloat> %ret
+}
+
+define <4 x bfloat> @vector_compress_v4bf16_passthru(<4 x bfloat> %passthru, <4 x bfloat> %v, <4 x i1> %mask) {
+; CHECK-LABEL: vector_compress_v4bf16_passthru:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, tu, ma
+; CHECK-NEXT:    vcompress.vm v8, v9, v0
+; CHECK-NEXT:    ret
+  %ret = call <4 x bfloat> @llvm.experimental.vector.compress.v4bf16(<4 x bfloat> %v, <4 x i1> %mask, <4 x bfloat> %passthru)
+  ret <4 x bfloat> %ret
+}
+
+define <8 x bfloat> @vector_compress_v8bf16(<8 x bfloat> %v, <8 x i1> %mask) {
+; CHECK-LABEL: vector_compress_v8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vcompress.vm v9, v8, v0
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+  %ret = call <8 x bfloat> @llvm.experimental.vector.compress.v8bf16(<8 x bfloat> %v, <8 x i1> %mask, <8 x bfloat> undef)
+  ret <8 x bfloat> %ret
+}
+
+define <8 x bfloat> @vector_compress_v8bf16_passthru(<8 x bfloat> %passthru, <8 x bfloat> %v, <8 x i1> %mask) {
+; CHECK-LABEL: vector_compress_v8bf16_passthru:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, tu, ma
+; CHECK-NEXT:    vcompress.vm v8, v9, v0
+; CHECK-NEXT:    ret
+  %ret = call <8 x bfloat> @llvm.experimental.vector.compress.v8bf16(<8 x bfloat> %v, <8 x i1> %mask, <8 x bfloat> %passthru)
+  ret <8 x bfloat> %ret
+}
 
 define <1 x half> @vector_compress_v1f16(<1 x half> %v, <1 x i1> %mask) {
 ; CHECK-LABEL: vector_compress_v1f16:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-compress.ll b/llvm/test/CodeGen/RISCV/rvv/vector-compress.ll
index 85d72ad2fe9c..7516a72a92bc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-compress.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-compress.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -verify-machineinstrs -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh %s -o - | FileCheck %s
-; RUN: llc -verify-machineinstrs -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh,+zvfbfmin < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh,+zvfbfmin < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfhmin,+zvfbfmin < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfhmin,+zvfbfmin < %s | FileCheck %s
 
 ; Vector compress for i8 type
 
@@ -472,6 +474,134 @@ define <vscale x 8 x i64> @vector_compress_nxv8i64_passthru(<vscale x 8 x i64> %
   ret <vscale x 8 x i64> %ret
 }
 
+; Vector compress for bf16 type
+
+define <vscale x 1 x bfloat> @vector_compress_nxv1bf16(<vscale x 1 x bfloat> %data, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: vector_compress_nxv1bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vcompress.vm v9, v8, v0
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %ret = call <vscale x 1 x bfloat> @llvm.experimental.vector.compress.nxv1bf16(<vscale x 1 x bfloat> %data, <vscale x 1 x i1> %mask, <vscale x 1 x bfloat> undef)
+  ret <vscale x 1 x bfloat> %ret
+}
+
+define <vscale x 1 x bfloat> @vector_compress_nxv1bf16_passthru(<vscale x 1 x bfloat> %passthru, <vscale x 1 x bfloat> %data, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: vector_compress_nxv1bf16_passthru:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, tu, ma
+; CHECK-NEXT:    vcompress.vm v8, v9, v0
+; CHECK-NEXT:    ret
+  %ret = call <vscale x 1 x bfloat> @llvm.experimental.vector.compress.nxv1bf16(<vscale x 1 x bfloat> %data, <vscale x 1 x i1> %mask, <vscale x 1 x bfloat> %passthru)
+  ret <vscale x 1 x bfloat> %ret
+}
+
+define <vscale x 2 x bfloat> @vector_compress_nxv2bf16(<vscale x 2 x bfloat> %data, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: vector_compress_nxv2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vcompress.vm v9, v8, v0
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %ret = call <vscale x 2 x bfloat> @llvm.experimental.vector.compress.nxv2bf16(<vscale x 2 x bfloat> %data, <vscale x 2 x i1> %mask, <vscale x 2 x bfloat> undef)
+  ret <vscale x 2 x bfloat> %ret
+}
+
+define <vscale x 2 x bfloat> @vector_compress_nxv2bf16_passthru(<vscale x 2 x bfloat> %passthru, <vscale x 2 x bfloat> %data, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: vector_compress_nxv2bf16_passthru:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, tu, ma
+; CHECK-NEXT:    vcompress.vm v8, v9, v0
+; CHECK-NEXT:    ret
+  %ret = call <vscale x 2 x bfloat> @llvm.experimental.vector.compress.nxv2bf16(<vscale x 2 x bfloat> %data, <vscale x 2 x i1> %mask, <vscale x 2 x bfloat> %passthru)
+  ret <vscale x 2 x bfloat> %ret
+}
+
+define <vscale x 4 x bfloat> @vector_compress_nxv4bf16(<vscale x 4 x bfloat> %data, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: vector_compress_nxv4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vcompress.vm v9, v8, v0
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+  %ret = call <vscale x 4 x bfloat> @llvm.experimental.vector.compress.nxv4bf16(<vscale x 4 x bfloat> %data, <vscale x 4 x i1> %mask, <vscale x 4 x bfloat> undef)
+  ret <vscale x 4 x bfloat> %ret
+}
+
+define <vscale x 4 x bfloat> @vector_compress_nxv4bf16_passthru(<vscale x 4 x bfloat> %passthru, <vscale x 4 x bfloat> %data, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: vector_compress_nxv4bf16_passthru:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, tu, ma
+; CHECK-NEXT:    vcompress.vm v8, v9, v0
+; CHECK-NEXT:    ret
+  %ret = call <vscale x 4 x bfloat> @llvm.experimental.vector.compress.nxv4bf16(<vscale x 4 x bfloat> %data, <vscale x 4 x i1> %mask, <vscale x 4 x bfloat> %passthru)
+  ret <vscale x 4 x bfloat> %ret
+}
+
+define <vscale x 8 x bfloat> @vector_compress_nxv8bf16(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %mask) {
+; CHECK-LABEL: vector_compress_nxv8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vcompress.vm v10, v8, v0
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %ret = call <vscale x 8 x bfloat> @llvm.experimental.vector.compress.nxv8bf16(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %mask, <vscale x 8 x bfloat> undef)
+  ret <vscale x 8 x bfloat> %ret
+}
+
+define <vscale x 8 x bfloat> @vector_compress_nxv8bf16_passthru(<vscale x 8 x bfloat> %passthru, <vscale x 8 x bfloat> %data, <vscale x 8 x i1> %mask) {
+; CHECK-LABEL: vector_compress_nxv8bf16_passthru:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, tu, ma
+; CHECK-NEXT:    vcompress.vm v8, v10, v0
+; CHECK-NEXT:    ret
+  %ret = call <vscale x 8 x bfloat> @llvm.experimental.vector.compress.nxv8bf16(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %mask, <vscale x 8 x bfloat> %passthru)
+  ret <vscale x 8 x bfloat> %ret
+}
+
+define <vscale x 16 x bfloat> @vector_compress_nxv16bf16(<vscale x 16 x bfloat> %data, <vscale x 16 x i1> %mask) {
+; CHECK-LABEL: vector_compress_nxv16bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vcompress.vm v12, v8, v0
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %ret = call <vscale x 16 x bfloat> @llvm.experimental.vector.compress.nxv16bf16(<vscale x 16 x bfloat> %data, <vscale x 16 x i1> %mask, <vscale x 16 x bfloat> undef)
+  ret <vscale x 16 x bfloat> %ret
+}
+
+define <vscale x 16 x bfloat> @vector_compress_nxv16bf16_passthru(<vscale x 16 x bfloat> %passthru, <vscale x 16 x bfloat> %data, <vscale x 16 x i1> %mask) {
+; CHECK-LABEL: vector_compress_nxv16bf16_passthru:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, tu, ma
+; CHECK-NEXT:    vcompress.vm v8, v12, v0
+; CHECK-NEXT:    ret
+  %ret = call <vscale x 16 x bfloat> @llvm.experimental.vector.compress.nxv16bf16(<vscale x 16 x bfloat> %data, <vscale x 16 x i1> %mask, <vscale x 16 x bfloat> %passthru)
+  ret <vscale x 16 x bfloat> %ret
+}
+
+define <vscale x 32 x bfloat> @vector_compress_nxv32bf16(<vscale x 32 x bfloat> %data, <vscale x 32 x i1> %mask) {
+; CHECK-LABEL: vector_compress_nxv32bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vcompress.vm v16, v8, v0
+; CHECK-NEXT:    vmv.v.v v8, v16
+; CHECK-NEXT:    ret
+  %ret = call <vscale x 32 x bfloat> @llvm.experimental.vector.compress.nxv32bf16(<vscale x 32 x bfloat> %data, <vscale x 32 x i1> %mask, <vscale x 32 x bfloat> undef)
+  ret <vscale x 32 x bfloat> %ret
+}
+
+define <vscale x 32 x bfloat> @vector_compress_nxv32bf16_passthru(<vscale x 32 x bfloat> %passthru, <vscale x 32 x bfloat> %data, <vscale x 32 x i1> %mask) {
+; CHECK-LABEL: vector_compress_nxv32bf16_passthru:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m8, tu, ma
+; CHECK-NEXT:    vcompress.vm v8, v16, v0
+; CHECK-NEXT:    ret
+  %ret = call <vscale x 32 x bfloat> @llvm.experimental.vector.compress.nxv32bf16(<vscale x 32 x bfloat> %data, <vscale x 32 x i1> %mask, <vscale x 32 x bfloat> %passthru)
+  ret <vscale x 32 x bfloat> %ret
+}
+
 ; Vector compress for f16 type
 
 define <vscale x 1 x half> @vector_compress_nxv1f16(<vscale x 1 x half> %data, <vscale x 1 x i1> %mask) {
-- 
GitLab


From fa4790e404843cff4370cf259b93e51e8341469d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mirko=20Brku=C5=A1anin?= <Mirko.Brkusanin@amd.com>
Date: Mon, 28 Oct 2024 10:43:18 +0100
Subject: [PATCH 294/693] [AMDGPU][MC] Fix disassembler for VIMAGE when
 non-first vaddr is v0 (#113569)

For disassembler tables we use *V1_V4* variants for VIMAGE and then
remove unused vaddr fields. *V1_V1* variant, which has every vaddr
field other than vaddr0 set to 0, was also enabled and caused confusion
when decoding cases which used v0 (whose encoded value is 0)
---
 llvm/lib/Target/AMDGPU/MIMGInstructions.td           | 10 ++++------
 llvm/test/MC/AMDGPU/gfx12_asm_vimage.s               | 12 ++++++++++++
 .../MC/Disassembler/AMDGPU/gfx12_dasm_vimage.txt     | 12 ++++++++++++
 3 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
index e0d1cde28452..2f342365c3a5 100644
--- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
@@ -567,8 +567,7 @@ multiclass MIMG_NoSampler_Src_Helper <mimgopc op, string asm,
           def _V1_gfx12 : VSAMPLE_Sampler_gfx12<op, asm, dst_rc, 1>;
       }
       else {
-        def _V1_gfx12 : VIMAGE_NoSampler_gfx12<op, asm, dst_rc, 1,
-                                               !if(enableDisasm, "GFX12", "")>;
+        def _V1_gfx12 : VIMAGE_NoSampler_gfx12<op, asm, dst_rc, 1>;
       }
     }
   }
@@ -789,8 +788,7 @@ multiclass MIMG_Store_Addr_Helper <mimgopc op, string asm,
         }
       }
       if op.HAS_GFX12 then {
-        def _V1_gfx12 : VIMAGE_Store_gfx12 <op, asm, data_rc, 1,
-                                            !if(enableDisasm, "GFX12", "")>;
+        def _V1_gfx12 : VIMAGE_Store_gfx12 <op, asm, data_rc, 1>;
       }
     }
     let VAddrDwords = 2 in {
@@ -1017,9 +1015,9 @@ multiclass MIMG_Atomic_Addr_Helper_m <mimgopc op, string asm,
       }
       if op.HAS_GFX12 then {
         if !empty(renamed) then
-          def _V1_gfx12 : VIMAGE_Atomic_gfx12 <op, asm, data_rc, 1, enableDasm>;
+          def _V1_gfx12 : VIMAGE_Atomic_gfx12 <op, asm, data_rc, 1>;
         else
-          def _V1_gfx12 : VIMAGE_Atomic_gfx12_Renamed <op, renamed, data_rc, 1, enableDasm>;
+          def _V1_gfx12 : VIMAGE_Atomic_gfx12_Renamed <op, renamed, data_rc, 1>;
       }
     }
     let VAddrDwords = 2 in {
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vimage.s b/llvm/test/MC/AMDGPU/gfx12_asm_vimage.s
index 39010883a3c0..196d75db4260 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vimage.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vimage.s
@@ -155,6 +155,9 @@ image_load v[0:2], [v4, v5], s[8:15] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY th:TH_LO
 image_load v[0:2], [v4, v5], s[8:15] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY th:TH_LOAD_HT scope:SCOPE_SE r128 a16 tfe d16
 // GFX12: encoding: [0x75,0x00,0xc0,0xd3,0x00,0x10,0xa4,0x00,0x04,0x05,0x00,0x00]
 
+image_load v[4:7], [v1, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+// GFX12: encoding: [0x01,0x00,0xc0,0xd3,0x04,0x08,0x00,0x00,0x01,0x00,0x00,0x00]
+
 image_load_mip v[252:255], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D
 // GFX12: encoding: [0x00,0x40,0xc0,0xd3,0xfc,0x00,0x00,0x00,0x00,0x01,0x00,0x00]
 
@@ -402,6 +405,9 @@ image_store v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_STORE_NT_WB
 image_store v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_STORE_BYPASS scope:SCOPE_SYS
 // GFX12: encoding: [0x00,0x80,0x41,0xd0,0x00,0x00,0x3c,0x00,0x00,0x00,0x00,0x00]
 
+image_store v[1:4], [v2, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+// GFX12: encoding: [0x01,0x80,0xc1,0xd3,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00]
+
 image_store_mip v[252:255], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D
 // GFX12: encoding: [0x00,0xc0,0xc1,0xd3,0xfc,0x00,0x00,0x00,0x00,0x01,0x00,0x00]
 
@@ -559,6 +565,9 @@ image_atomic_swap v[3:4], [v4, v5], s[8:15] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA a1
 image_atomic_swap v[254:255], [v4, v5], s[96:103] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY a16
 // GFX12: encoding: [0x47,0x80,0xc2,0xd0,0xfe,0xc0,0x00,0x00,0x04,0x05,0x00,0x00]
 
+image_atomic_swap v1, [v2, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX12: encoding: [0x01,0x80,0x42,0xd0,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00]
+
 image_atomic_cmpswap v[0:1], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D
 // GFX12: encoding: [0x00,0xc0,0xc2,0xd0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
 
@@ -613,6 +622,9 @@ image_atomic_add_uint v[254:255], [v4, v5, v6, v7], s[96:103] dmask:0x3 dim:SQ_R
 image_atomic_add_uint v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_NT
 // GFX12: encoding: [0x00,0x00,0x43,0xd0,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00]
 
+image_atomic_add_uint v1, [v2, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX12: encoding: [0x01,0x00,0x43,0xd0,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00]
+
 image_atomic_sub_uint v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
 // GFX12: encoding: [0x00,0x40,0x43,0xd0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage.txt
index aa49caacb4fc..08e9bef8cf67 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage.txt
@@ -157,6 +157,9 @@
 # GFX12: image_load v[0:2], [v4, v5], s[8:15] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY th:TH_LOAD_HT scope:SCOPE_SE r128 a16 tfe d16 ; encoding: [0x75,0x00,0xc0,0xd3,0x00,0x10,0xa4,0x00,0x04,0x05,0x00,0x00]
 0x75,0x00,0xc0,0xd3,0x00,0x10,0xa4,0x00,0x04,0x05,0x00,0x00
 
+# GFX12: image_load v[4:7], [v1, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x01,0x00,0xc0,0xd3,0x04,0x08,0x00,0x00,0x01,0x00,0x00,0x00]
+0x01,0x00,0xc0,0xd3,0x04,0x08,0x00,0x00,0x01,0x00,0x00,0x00
+
 # GFX12: image_load_mip v[252:255], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x40,0xc0,0xd3,0xfc,0x00,0x00,0x00,0x00,0x01,0x00,0x00]
 0x00,0x40,0xc0,0xd3,0xfc,0x00,0x00,0x00,0x00,0x01,0x00,0x00
 
@@ -403,6 +406,9 @@
 # GFX12: image_store v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x00,0x80,0x41,0xd0,0x00,0x00,0x3c,0x00,0x00,0x00,0x00,0x00]
 0x00,0x80,0x41,0xd0,0x00,0x00,0x3c,0x00,0x00,0x00,0x00,0x00
 
+# GFX12: image_store v[1:4], [v2, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x01,0x80,0xc1,0xd3,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00]
+0x01,0x80,0xc1,0xd3,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00
+
 # GFX12: image_store_mip v[252:255], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0xc0,0xc1,0xd3,0xfc,0x00,0x00,0x00,0x00,0x01,0x00,0x00]
 0x00,0xc0,0xc1,0xd3,0xfc,0x00,0x00,0x00,0x00,0x01,0x00,0x00
 
@@ -559,6 +565,9 @@
 # GFX12: image_atomic_swap v[254:255], [v4, v5], s[96:103] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY a16 ; encoding: [0x47,0x80,0xc2,0xd0,0xfe,0xc0,0x00,0x00,0x04,0x05,0x00,0x00]
 0x47,0x80,0xc2,0xd0,0xfe,0xc0,0x00,0x00,0x04,0x05,0x00,0x00
 
+# GFX12: image_atomic_swap v1, [v2, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D ; encoding: [0x01,0x80,0x42,0xd0,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00]
+0x01,0x80,0x42,0xd0,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00
+
 # GFX12: image_atomic_cmpswap v[0:1], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0xc0,0xc2,0xd0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
 0x00,0xc0,0xc2,0xd0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
 
@@ -613,6 +622,9 @@
 # GFX12: image_atomic_add_uint v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_NT ; encoding: [0x00,0x00,0x43,0xd0,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00]
 0x00,0x00,0x43,0xd0,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00
 
+# GFX12: image_atomic_add_uint v1, [v2, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D ; encoding: [0x01,0x00,0x43,0xd0,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00]
+0x01,0x00,0x43,0xd0,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00
+
 # GFX12: image_atomic_sub_uint v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x40,0x43,0xd0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
 0x00,0x40,0x43,0xd0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
 
-- 
GitLab


From f78610af3feb88f0e1edb2482dc77490fb4cad77 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Mon, 28 Oct 2024 17:45:08 +0800
Subject: [PATCH 295/693] [InstCombine] Add function attribute
 `instcombine-no-verify-fixpoint` (#113822)

This patch introduces a function attribute
`instcombine-no-verify-fixpoint` to avoids disabling fix-point
verification for unrelated tests in the same file.
Address comment
https://github.com/llvm/llvm-project/pull/112642#discussion_r1804714387.
---
 .../Transforms/InstCombine/InstructionCombining.cpp  | 12 ++++++++----
 .../ValueTracking/numsignbits-from-assume.ll         |  5 +----
 .../Transforms/InstCombine/2007-10-31-RangeCrash.ll  |  7 ++++---
 llvm/test/Transforms/InstCombine/cast_phi.ll         |  4 ++--
 .../InstCombine/constant-fold-iteration.ll           |  5 +++--
 llvm/test/Transforms/InstCombine/icmp-or.ll          |  6 +++---
 llvm/test/Transforms/InstCombine/pr55228.ll          |  4 ++--
 llvm/test/Transforms/InstCombine/shift.ll            | 10 +++++-----
 llvm/test/Transforms/InstCombine/sink_instruction.ll | 11 +++++------
 llvm/test/Transforms/InstCombine/zext-or-icmp.ll     |  4 ++--
 llvm/test/Transforms/PGOProfile/chr.ll               | 10 +++++-----
 .../SimpleLoopUnswitch/2007-08-01-LCSSA.ll           |  6 +++---
 12 files changed, 43 insertions(+), 41 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 73a470553178..2a54390c0f18 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -5464,6 +5464,8 @@ static bool combineInstructionsOverFunction(
     BranchProbabilityInfo *BPI, ProfileSummaryInfo *PSI,
     const InstCombineOptions &Opts) {
   auto &DL = F.getDataLayout();
+  bool VerifyFixpoint = Opts.VerifyFixpoint &&
+                        !F.hasFnAttribute("instcombine-no-verify-fixpoint");
 
   /// Builder - This is an IRBuilder that automatically inserts new
   /// instructions into the worklist when they are created.
@@ -5488,7 +5490,7 @@ static bool combineInstructionsOverFunction(
   while (true) {
     ++Iteration;
 
-    if (Iteration > Opts.MaxIterations && !Opts.VerifyFixpoint) {
+    if (Iteration > Opts.MaxIterations && !VerifyFixpoint) {
       LLVM_DEBUG(dbgs() << "\n\n[IC] Iteration limit #" << Opts.MaxIterations
                         << " on " << F.getName()
                         << " reached; stopping without verifying fixpoint\n");
@@ -5510,9 +5512,11 @@ static bool combineInstructionsOverFunction(
     MadeIRChange = true;
     if (Iteration > Opts.MaxIterations) {
       report_fatal_error(
-          "Instruction Combining did not reach a fixpoint after " +
-              Twine(Opts.MaxIterations) + " iterations. " +
-              "Use 'instcombine<no-verify-fixpoint>' to suppress this error.",
+          "Instruction Combining on " + Twine(F.getName()) +
+              " did not reach a fixpoint after " + Twine(Opts.MaxIterations) +
+              " iterations. " +
+              "Use 'instcombine<no-verify-fixpoint>' or function attribute "
+              "'instcombine-no-verify-fixpoint' to suppress this error.",
           /*GenCrashDiag=*/false);
     }
   }
diff --git a/llvm/test/Analysis/ValueTracking/numsignbits-from-assume.ll b/llvm/test/Analysis/ValueTracking/numsignbits-from-assume.ll
index 95ac98532da6..5beb0c7cadfb 100644
--- a/llvm/test/Analysis/ValueTracking/numsignbits-from-assume.ll
+++ b/llvm/test/Analysis/ValueTracking/numsignbits-from-assume.ll
@@ -1,8 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes='instcombine<no-verify-fixpoint>' -S | FileCheck %s
-
-; FIXME: This does not currently reach a fix point, because an assume can only
-; be propagated backwards after its argument has been simplified.
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 define i32 @computeNumSignBits_add1(i32 %in) {
 ; CHECK-LABEL: @computeNumSignBits_add1(
diff --git a/llvm/test/Transforms/InstCombine/2007-10-31-RangeCrash.ll b/llvm/test/Transforms/InstCombine/2007-10-31-RangeCrash.ll
index 8b472aa5af09..b5ae08e1daa3 100644
--- a/llvm/test/Transforms/InstCombine/2007-10-31-RangeCrash.ll
+++ b/llvm/test/Transforms/InstCombine/2007-10-31-RangeCrash.ll
@@ -1,13 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt < %s -S -passes='instcombine<no-verify-fixpoint>' | FileCheck %s
+; RUN: opt < %s -S -passes=instcombine | FileCheck %s
 
 ; We do not reach a fixpoint, because we first have to infer nsw on the IV add,
 ; and could eliminate the icmp slt afterwards, but don't revisit it.
 
 target datalayout = "E-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f128:64:128"
 
-define i32 @test() {
-; CHECK-LABEL: define i32 @test() {
+define i32 @test() "instcombine-no-verify-fixpoint" {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 true, label [[BB_I:%.*]], label [[CALCULATECOLORSPECIFICBLACKLEVEL_EXIT:%.*]]
 ; CHECK:       bb.i:
diff --git a/llvm/test/Transforms/InstCombine/cast_phi.ll b/llvm/test/Transforms/InstCombine/cast_phi.ll
index 2819b7d05f7b..6b05edc31deb 100644
--- a/llvm/test/Transforms/InstCombine/cast_phi.ll
+++ b/llvm/test/Transforms/InstCombine/cast_phi.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes="instcombine<no-verify-fixpoint>" -S | FileCheck %s
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 target datalayout = "n32:64"
 
@@ -309,7 +309,7 @@ exit:
   ret i64 %r
 }
 
-define i8 @trunc_in_loop_exit_block() {
+define i8 @trunc_in_loop_exit_block() "instcombine-no-verify-fixpoint" {
 ; CHECK-LABEL: @trunc_in_loop_exit_block(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
diff --git a/llvm/test/Transforms/InstCombine/constant-fold-iteration.ll b/llvm/test/Transforms/InstCombine/constant-fold-iteration.ll
index 7f800f614c47..ed4fcc6ecaac 100644
--- a/llvm/test/Transforms/InstCombine/constant-fold-iteration.ll
+++ b/llvm/test/Transforms/InstCombine/constant-fold-iteration.ll
@@ -1,11 +1,12 @@
-; RUN: opt < %s -passes='instcombine<no-verify-fixpoint>' -S -debug 2>&1 | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instcombine -S -debug 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
 
 ; This test disables fixpoint verification, because that would cause a second
 ; iteration for verification.
 
-define i32 @a() nounwind readnone {
+define i32 @a() nounwind readnone "instcombine-no-verify-fixpoint" {
 entry:
   %cmp = icmp eq i32 0, ptrtoint (ptr @a to i32)
   %ext = zext i1 %cmp to i32
diff --git a/llvm/test/Transforms/InstCombine/icmp-or.ll b/llvm/test/Transforms/InstCombine/icmp-or.ll
index 36b3216196f8..56115f6d7d34 100644
--- a/llvm/test/Transforms/InstCombine/icmp-or.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-or.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes='instcombine<no-verify-fixpoint>' -S | FileCheck %s
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 declare void @use(i8)
 
@@ -432,7 +432,7 @@ define i1 @icmp_or_xor_2_ne_fail(i64 %x1, i64 %y1, i64 %x2, i64 %y2) {
 ; simplify `%cmp_1 = icmp eq i64 %xor, 0`, `%xor = xor i64 %x1, %y1`
 ; has one use which allows for complete simplification (rooted on
 ; `%or1 = or i1 %cmp, %cmp_1` so we don't end up adding it back).
-define i1 @icmp_or_xor_2_3_fail(i64 %x1, i64 %y1, i64 %x2, i64 %y2) {
+define i1 @icmp_or_xor_2_3_fail(i64 %x1, i64 %y1, i64 %x2, i64 %y2) "instcombine-no-verify-fixpoint" {
 ; CHECK-LABEL: @icmp_or_xor_2_3_fail(
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i64 [[X1:%.*]], [[Y1:%.*]]
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i64 [[X2:%.*]], [[Y2:%.*]]
@@ -453,7 +453,7 @@ define i1 @icmp_or_xor_2_3_fail(i64 %x1, i64 %y1, i64 %x2, i64 %y2) {
 
 ; negative test - xor multiuse
 
-define i1 @icmp_or_xor_2_4_fail(i64 %x1, i64 %y1, i64 %x2, i64 %y2) {
+define i1 @icmp_or_xor_2_4_fail(i64 %x1, i64 %y1, i64 %x2, i64 %y2) "instcombine-no-verify-fixpoint" {
 ; CHECK-LABEL: @icmp_or_xor_2_4_fail(
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i64 [[X1:%.*]], [[Y1:%.*]]
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i64 [[X2:%.*]], [[Y2:%.*]]
diff --git a/llvm/test/Transforms/InstCombine/pr55228.ll b/llvm/test/Transforms/InstCombine/pr55228.ll
index 5e34c074346e..c959bf16bcb5 100644
--- a/llvm/test/Transforms/InstCombine/pr55228.ll
+++ b/llvm/test/Transforms/InstCombine/pr55228.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -passes='instcombine<no-verify-fixpoint>' < %s | FileCheck %s
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
 
 ; This does not reach a fixpoint, because the global initializer is not in
 ; folded form. This will not happen if preceded by a GlobalOpt run.
@@ -9,7 +9,7 @@ target datalayout = "p:8:8"
 @g = external global i8
 @c = constant ptr getelementptr inbounds (i8, ptr @g, i64 1)
 
-define i1 @test(ptr %p) {
+define i1 @test(ptr %p) "instcombine-no-verify-fixpoint" {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[P:%.*]], getelementptr inbounds (i8, ptr @g, i64 1)
 ; CHECK-NEXT:    ret i1 [[CMP]]
diff --git a/llvm/test/Transforms/InstCombine/shift.ll b/llvm/test/Transforms/InstCombine/shift.ll
index 69f531e98f04..986e1073c638 100644
--- a/llvm/test/Transforms/InstCombine/shift.ll
+++ b/llvm/test/Transforms/InstCombine/shift.ll
@@ -1,8 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes='instcombine<no-verify-fixpoint>' -S | FileCheck %s
-
-; The fuzzer-generated @ashr_out_of_range test case does not reach a fixpoint,
-; because a logical and it not relaxed to a bitwise and in one iteration.
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 declare void @use(i64)
 declare void @use_i32(i32)
@@ -1735,9 +1732,12 @@ define i177 @lshr_out_of_range2(i177 %Y, ptr %A2, ptr %ptr) {
   ret i177 %B1
 }
 
+; The fuzzer-generated @ashr_out_of_range test case does not reach a fixpoint,
+; because a logical and it not relaxed to a bitwise and in one iteration.
+
 ; OSS Fuzz #5032
 ; https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=5032
-define void @ashr_out_of_range(ptr %A) {
+define void @ashr_out_of_range(ptr %A) "instcombine-no-verify-fixpoint" {
 ; CHECK-LABEL: @ashr_out_of_range(
 ; CHECK-NEXT:    [[L:%.*]] = load i177, ptr [[A:%.*]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i177 [[L]], -1
diff --git a/llvm/test/Transforms/InstCombine/sink_instruction.ll b/llvm/test/Transforms/InstCombine/sink_instruction.ll
index dac40852c4bd..cb9a3069ca5f 100644
--- a/llvm/test/Transforms/InstCombine/sink_instruction.ll
+++ b/llvm/test/Transforms/InstCombine/sink_instruction.ll
@@ -1,9 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes='instcombine<no-verify-fixpoint>' -S < %s | FileCheck %s
-
-; We fail to reach a fixpoint, because sunk instructions get revisited too
-; early. In @test2 the sunk add is revisited before the dominating condition
-; is visited and added to the DomConditionCache.
+; RUN: opt -passes=instcombine -S < %s | FileCheck %s
 
 ;; This tests that the instructions in the entry blocks are sunk into each
 ;; arm of the 'if'.
@@ -31,9 +27,12 @@ endif:          ; preds = %entry
   ret i32 %tmp.2
 }
 
+; We fail to reach a fixpoint, because sunk instructions get revisited too
+; early. In @test2 the sunk add is revisited before the dominating condition
+; is visited and added to the DomConditionCache.
 
 ;; PHI use, sink divide before call.
-define i32 @test2(i32 %x) nounwind ssp {
+define i32 @test2(i32 %x) nounwind ssp "instcombine-no-verify-fixpoint" {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[BB:%.*]]
diff --git a/llvm/test/Transforms/InstCombine/zext-or-icmp.ll b/llvm/test/Transforms/InstCombine/zext-or-icmp.ll
index acf547b55722..feb4be9e3705 100644
--- a/llvm/test/Transforms/InstCombine/zext-or-icmp.ll
+++ b/llvm/test/Transforms/InstCombine/zext-or-icmp.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes='instcombine<no-verify-fixpoint>' -S | FileCheck %s
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 define i8 @zext_or_icmp_icmp(i8 %a, i8 %b) {
 ; CHECK-LABEL: @zext_or_icmp_icmp(
@@ -168,7 +168,7 @@ define i32 @PR49475(i32 %x, i16 %y) {
 
 ; This would infinite-loop.
 
-define i8 @PR49475_infloop(i32 %t0, i16 %insert, i64 %e, i8 %i162) {
+define i8 @PR49475_infloop(i32 %t0, i16 %insert, i64 %e, i8 %i162) "instcombine-no-verify-fixpoint" {
 ; CHECK-LABEL: @PR49475_infloop(
 ; CHECK-NEXT:    [[B2:%.*]] = icmp eq i16 [[INSERT:%.*]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[T0:%.*]], 1
diff --git a/llvm/test/Transforms/PGOProfile/chr.ll b/llvm/test/Transforms/PGOProfile/chr.ll
index 34e39fe37979..46f9a2bde7a2 100644
--- a/llvm/test/Transforms/PGOProfile/chr.ll
+++ b/llvm/test/Transforms/PGOProfile/chr.ll
@@ -1,8 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes='require<profile-summary>,function(chr,instcombine<no-verify-fixpoint>,simplifycfg)' -S | FileCheck %s
-
-; FIXME: This does not currently reach a fix point, because we don't make use
-; of a freeze that is pushed up the instruction chain later.
+; RUN: opt < %s -passes='require<profile-summary>,function(chr,instcombine,simplifycfg)' -S | FileCheck %s
 
 declare void @foo()
 declare void @bar()
@@ -1910,6 +1907,9 @@ bb4:
   ret i32 %v13
 }
 
+; FIXME: This does not currently reach a fix point, because we don't make use
+; of a freeze that is pushed up the instruction chain later.
+
 ; Test the case where two scopes share a common instruction to hoist (%cmp.i).
 ; Two scopes would hoist it to their hoist points, but since the outer scope
 ; hoists (entry/bb6-9) it first to its hoist point, it'd be wrong (causing bad
@@ -1928,7 +1928,7 @@ bb4:
 ;     foo();
 ;  }
 ;  return 45;
-define i32 @test_chr_21(i64 %i, i64 %k, i64 %j) !prof !14 {
+define i32 @test_chr_21(i64 %i, i64 %k, i64 %j) "instcombine-no-verify-fixpoint" !prof !14 {
 ; CHECK-LABEL: @test_chr_21(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[I_FR:%.*]] = freeze i64 [[I:%.*]]
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-LCSSA.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-LCSSA.ll
index 9e4b33d4c4d5..fb342322b2da 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-LCSSA.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-LCSSA.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt < %s -S -passes='loop(simple-loop-unswitch),instcombine<no-verify-fixpoint>' -verify-memoryssa | FileCheck %s
+; RUN: opt < %s -S -passes='loop(simple-loop-unswitch),instcombine' -verify-memoryssa | FileCheck %s
 
 ; We do not reach a fixpoint, because we first have to infer nsw on the IV add,
 ; and could eliminate the icmp slt afterwards, but don't revisit it.
@@ -8,9 +8,9 @@
 
 declare i32 @strcmp(ptr, ptr)
 
-define i32 @_ZN9Generator6strregEPKc(ptr %this, ptr %s) {
+define i32 @_ZN9Generator6strregEPKc(ptr %this, ptr %s) "instcombine-no-verify-fixpoint" {
 ; CHECK-LABEL: define i32 @_ZN9Generator6strregEPKc(
-; CHECK-SAME: ptr [[THIS:%.*]], ptr [[S:%.*]]) {
+; CHECK-SAME: ptr [[THIS:%.*]], ptr [[S:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP122:%.*]] = icmp eq ptr [[S]], null
 ; CHECK-NEXT:    br label [[BB184:%.*]]
-- 
GitLab


From 09c258ef6a2fcca2161488b214d53ef39891fa22 Mon Sep 17 00:00:00 2001
From: Adrian Vogelsgesang <avogelsgesang@salesforce.com>
Date: Mon, 28 Oct 2024 11:01:57 +0100
Subject: [PATCH 296/693] [NFC][lldb-dap] Clean-up includes (#113839)

This commit cleans up the includes in the `lldb-dap` subfolder. The main
motivation was that I got annoyed by `clangd` always complaining about
unused includes while working on lldb-dap.
---
 lldb/tools/lldb-dap/Breakpoint.cpp            |  1 +
 lldb/tools/lldb-dap/Breakpoint.h              |  1 +
 lldb/tools/lldb-dap/BreakpointBase.cpp        |  3 +--
 lldb/tools/lldb-dap/BreakpointBase.h          |  2 --
 lldb/tools/lldb-dap/DAP.cpp                   |  5 ++++-
 lldb/tools/lldb-dap/DAP.h                     | 19 -------------------
 lldb/tools/lldb-dap/FifoFiles.cpp             |  6 +-----
 lldb/tools/lldb-dap/FifoFiles.h               |  4 +---
 lldb/tools/lldb-dap/FunctionBreakpoint.cpp    |  1 +
 lldb/tools/lldb-dap/IOStream.cpp              |  1 -
 lldb/tools/lldb-dap/IOStream.h                |  2 --
 lldb/tools/lldb-dap/InstructionBreakpoint.cpp |  1 +
 lldb/tools/lldb-dap/InstructionBreakpoint.h   |  1 -
 lldb/tools/lldb-dap/JSONUtils.cpp             |  6 +-----
 lldb/tools/lldb-dap/LLDBUtils.cpp             |  2 ++
 lldb/tools/lldb-dap/OutputRedirector.h        |  2 --
 lldb/tools/lldb-dap/RunInTerminal.cpp         |  5 +----
 lldb/tools/lldb-dap/RunInTerminal.h           |  4 +++-
 lldb/tools/lldb-dap/SourceBreakpoint.cpp      |  1 +
 lldb/tools/lldb-dap/lldb-dap.cpp              | 10 +++++-----
 20 files changed, 24 insertions(+), 53 deletions(-)

diff --git a/lldb/tools/lldb-dap/Breakpoint.cpp b/lldb/tools/lldb-dap/Breakpoint.cpp
index 0c33d4b114d7..9ea7a42ca85a 100644
--- a/lldb/tools/lldb-dap/Breakpoint.cpp
+++ b/lldb/tools/lldb-dap/Breakpoint.cpp
@@ -9,6 +9,7 @@
 #include "Breakpoint.h"
 #include "DAP.h"
 #include "JSONUtils.h"
+#include "lldb/API/SBBreakpointLocation.h"
 #include "llvm/ADT/StringExtras.h"
 
 using namespace lldb_dap;
diff --git a/lldb/tools/lldb-dap/Breakpoint.h b/lldb/tools/lldb-dap/Breakpoint.h
index 47a9d9c59ae2..ee9d3736d619 100644
--- a/lldb/tools/lldb-dap/Breakpoint.h
+++ b/lldb/tools/lldb-dap/Breakpoint.h
@@ -10,6 +10,7 @@
 #define LLDB_TOOLS_LLDB_DAP_BREAKPOINT_H
 
 #include "BreakpointBase.h"
+#include "lldb/API/SBBreakpoint.h"
 
 namespace lldb_dap {
 
diff --git a/lldb/tools/lldb-dap/BreakpointBase.cpp b/lldb/tools/lldb-dap/BreakpointBase.cpp
index 519729f5519f..f3cb06a3562d 100644
--- a/lldb/tools/lldb-dap/BreakpointBase.cpp
+++ b/lldb/tools/lldb-dap/BreakpointBase.cpp
@@ -7,8 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "BreakpointBase.h"
-#include "DAP.h"
-#include "llvm/ADT/StringExtras.h"
+#include "JSONUtils.h"
 
 using namespace lldb_dap;
 
diff --git a/lldb/tools/lldb-dap/BreakpointBase.h b/lldb/tools/lldb-dap/BreakpointBase.h
index 5a04bb201615..79301480e0e5 100644
--- a/lldb/tools/lldb-dap/BreakpointBase.h
+++ b/lldb/tools/lldb-dap/BreakpointBase.h
@@ -9,10 +9,8 @@
 #ifndef LLDB_TOOLS_LLDB_DAP_BREAKPOINTBASE_H
 #define LLDB_TOOLS_LLDB_DAP_BREAKPOINTBASE_H
 
-#include "lldb/API/SBBreakpoint.h"
 #include "llvm/Support/JSON.h"
 #include <string>
-#include <vector>
 
 namespace lldb_dap {
 
diff --git a/lldb/tools/lldb-dap/DAP.cpp b/lldb/tools/lldb-dap/DAP.cpp
index 68559e382006..283392270ba2 100644
--- a/lldb/tools/lldb-dap/DAP.cpp
+++ b/lldb/tools/lldb-dap/DAP.cpp
@@ -10,11 +10,14 @@
 #include <cstdarg>
 #include <fstream>
 #include <mutex>
-#include <sstream>
 
 #include "DAP.h"
+#include "JSONUtils.h"
 #include "LLDBUtils.h"
 #include "lldb/API/SBCommandInterpreter.h"
+#include "lldb/API/SBLanguageRuntime.h"
+#include "lldb/API/SBListener.h"
+#include "lldb/API/SBStream.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/FormatVariadic.h"
 
diff --git a/lldb/tools/lldb-dap/DAP.h b/lldb/tools/lldb-dap/DAP.h
index acc10ade75fd..dab4ce44ab20 100644
--- a/lldb/tools/lldb-dap/DAP.h
+++ b/lldb/tools/lldb-dap/DAP.h
@@ -9,16 +9,10 @@
 #ifndef LLDB_TOOLS_LLDB_DAP_DAP_H
 #define LLDB_TOOLS_LLDB_DAP_DAP_H
 
-#include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
-
-#include <atomic>
-#include <condition_variable>
 #include <cstdio>
-#include <future>
 #include <iosfwd>
 #include <map>
 #include <optional>
-#include <set>
 #include <thread>
 
 #include "llvm/ADT/DenseMap.h"
@@ -30,24 +24,12 @@
 #include "llvm/Support/raw_ostream.h"
 
 #include "lldb/API/SBAttachInfo.h"
-#include "lldb/API/SBBreakpoint.h"
-#include "lldb/API/SBBreakpointLocation.h"
 #include "lldb/API/SBCommandInterpreter.h"
 #include "lldb/API/SBCommandReturnObject.h"
-#include "lldb/API/SBCommunication.h"
 #include "lldb/API/SBDebugger.h"
 #include "lldb/API/SBEvent.h"
 #include "lldb/API/SBFormat.h"
-#include "lldb/API/SBHostOS.h"
-#include "lldb/API/SBInstruction.h"
-#include "lldb/API/SBInstructionList.h"
-#include "lldb/API/SBLanguageRuntime.h"
 #include "lldb/API/SBLaunchInfo.h"
-#include "lldb/API/SBLineEntry.h"
-#include "lldb/API/SBListener.h"
-#include "lldb/API/SBProcess.h"
-#include "lldb/API/SBStream.h"
-#include "lldb/API/SBStringList.h"
 #include "lldb/API/SBTarget.h"
 #include "lldb/API/SBThread.h"
 
@@ -56,7 +38,6 @@
 #include "IOStream.h"
 #include "InstructionBreakpoint.h"
 #include "ProgressEvent.h"
-#include "RunInTerminal.h"
 #include "SourceBreakpoint.h"
 
 #define VARREF_LOCALS (int64_t)1
diff --git a/lldb/tools/lldb-dap/FifoFiles.cpp b/lldb/tools/lldb-dap/FifoFiles.cpp
index 9a6423f79471..1f1bba80bd3b 100644
--- a/lldb/tools/lldb-dap/FifoFiles.cpp
+++ b/lldb/tools/lldb-dap/FifoFiles.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "FifoFiles.h"
+#include "JSONUtils.h"
 
 #if !defined(_WIN32)
 #include <sys/stat.h>
@@ -18,11 +19,6 @@
 #include <fstream>
 #include <future>
 #include <optional>
-#include <thread>
-
-#include "llvm/Support/FileSystem.h"
-
-#include "lldb/lldb-defines.h"
 
 using namespace llvm;
 
diff --git a/lldb/tools/lldb-dap/FifoFiles.h b/lldb/tools/lldb-dap/FifoFiles.h
index 02a97cd5cbbd..633ebeb2aedd 100644
--- a/lldb/tools/lldb-dap/FifoFiles.h
+++ b/lldb/tools/lldb-dap/FifoFiles.h
@@ -9,10 +9,8 @@
 #ifndef LLDB_TOOLS_LLDB_DAP_FIFOFILES_H
 #define LLDB_TOOLS_LLDB_DAP_FIFOFILES_H
 
-#include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
 #include "llvm/Support/Error.h"
-
-#include "JSONUtils.h"
+#include "llvm/Support/JSON.h"
 
 #include <chrono>
 
diff --git a/lldb/tools/lldb-dap/FunctionBreakpoint.cpp b/lldb/tools/lldb-dap/FunctionBreakpoint.cpp
index 21743bf90870..216c685f633d 100644
--- a/lldb/tools/lldb-dap/FunctionBreakpoint.cpp
+++ b/lldb/tools/lldb-dap/FunctionBreakpoint.cpp
@@ -8,6 +8,7 @@
 
 #include "FunctionBreakpoint.h"
 #include "DAP.h"
+#include "JSONUtils.h"
 
 namespace lldb_dap {
 
diff --git a/lldb/tools/lldb-dap/IOStream.cpp b/lldb/tools/lldb-dap/IOStream.cpp
index 96e9a1ed4953..d2e8ec40b0a7 100644
--- a/lldb/tools/lldb-dap/IOStream.cpp
+++ b/lldb/tools/lldb-dap/IOStream.cpp
@@ -18,7 +18,6 @@
 
 #include <fstream>
 #include <string>
-#include <vector>
 
 using namespace lldb_dap;
 
diff --git a/lldb/tools/lldb-dap/IOStream.h b/lldb/tools/lldb-dap/IOStream.h
index b62502419182..57d5fd458b71 100644
--- a/lldb/tools/lldb-dap/IOStream.h
+++ b/lldb/tools/lldb-dap/IOStream.h
@@ -9,8 +9,6 @@
 #ifndef LLDB_TOOLS_LLDB_DAP_IOSTREAM_H
 #define LLDB_TOOLS_LLDB_DAP_IOSTREAM_H
 
-#include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
-
 #if defined(_WIN32)
 // We need to #define NOMINMAX in order to skip `min()` and `max()` macro
 // definitions that conflict with other system headers.
diff --git a/lldb/tools/lldb-dap/InstructionBreakpoint.cpp b/lldb/tools/lldb-dap/InstructionBreakpoint.cpp
index de4f6f5d8671..e3a8460bb7b3 100644
--- a/lldb/tools/lldb-dap/InstructionBreakpoint.cpp
+++ b/lldb/tools/lldb-dap/InstructionBreakpoint.cpp
@@ -9,6 +9,7 @@
 
 #include "InstructionBreakpoint.h"
 #include "DAP.h"
+#include "JSONUtils.h"
 
 namespace lldb_dap {
 
diff --git a/lldb/tools/lldb-dap/InstructionBreakpoint.h b/lldb/tools/lldb-dap/InstructionBreakpoint.h
index cf1516f46e95..53912af46ca1 100644
--- a/lldb/tools/lldb-dap/InstructionBreakpoint.h
+++ b/lldb/tools/lldb-dap/InstructionBreakpoint.h
@@ -11,7 +11,6 @@
 #define LLDB_TOOLS_LLDB_DAP_INSTRUCTIONBREAKPOINT_H
 
 #include "Breakpoint.h"
-#include "llvm/ADT/StringRef.h"
 
 namespace lldb_dap {
 
diff --git a/lldb/tools/lldb-dap/JSONUtils.cpp b/lldb/tools/lldb-dap/JSONUtils.cpp
index e42a6d9d6998..fd09e4ae505e 100644
--- a/lldb/tools/lldb-dap/JSONUtils.cpp
+++ b/lldb/tools/lldb-dap/JSONUtils.cpp
@@ -6,25 +6,21 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <algorithm>
 #include <iomanip>
 #include <optional>
 #include <sstream>
 #include <string.h>
 
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/FormatAdapters.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/ScopedPrinter.h"
 
-#include "lldb/API/SBBreakpoint.h"
-#include "lldb/API/SBBreakpointLocation.h"
 #include "lldb/API/SBDeclaration.h"
+#include "lldb/API/SBStream.h"
 #include "lldb/API/SBStringList.h"
 #include "lldb/API/SBStructuredData.h"
 #include "lldb/API/SBValue.h"
-#include "lldb/Host/PosixApi.h"
 
 #include "DAP.h"
 #include "ExceptionBreakpoint.h"
diff --git a/lldb/tools/lldb-dap/LLDBUtils.cpp b/lldb/tools/lldb-dap/LLDBUtils.cpp
index b38833c0fdb6..2ffcba7dff4f 100644
--- a/lldb/tools/lldb-dap/LLDBUtils.cpp
+++ b/lldb/tools/lldb-dap/LLDBUtils.cpp
@@ -8,6 +8,8 @@
 
 #include "LLDBUtils.h"
 #include "DAP.h"
+#include "JSONUtils.h"
+#include "lldb/API/SBStringList.h"
 
 #include <mutex>
 
diff --git a/lldb/tools/lldb-dap/OutputRedirector.h b/lldb/tools/lldb-dap/OutputRedirector.h
index dba51016775b..e26d1648b104 100644
--- a/lldb/tools/lldb-dap/OutputRedirector.h
+++ b/lldb/tools/lldb-dap/OutputRedirector.h
@@ -9,8 +9,6 @@
 #ifndef LLDB_TOOLS_LLDB_DAP_OUTPUT_REDIRECTOR_H
 #define LLDB_TOOLS_LLDB_DAP_OUTPUT_REDIRECTOR_H
 
-#include <thread>
-
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
 
diff --git a/lldb/tools/lldb-dap/RunInTerminal.cpp b/lldb/tools/lldb-dap/RunInTerminal.cpp
index ad019b8a56a4..4fe09e2885a8 100644
--- a/lldb/tools/lldb-dap/RunInTerminal.cpp
+++ b/lldb/tools/lldb-dap/RunInTerminal.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "RunInTerminal.h"
+#include "JSONUtils.h"
 
 #if !defined(_WIN32)
 #include <sys/stat.h>
@@ -15,14 +16,10 @@
 #endif
 
 #include <chrono>
-#include <fstream>
 #include <future>
-#include <thread>
 
 #include "llvm/Support/FileSystem.h"
 
-#include "lldb/lldb-defines.h"
-
 using namespace llvm;
 
 namespace lldb_dap {
diff --git a/lldb/tools/lldb-dap/RunInTerminal.h b/lldb/tools/lldb-dap/RunInTerminal.h
index 2fbe3acbb408..b20f8beb6071 100644
--- a/lldb/tools/lldb-dap/RunInTerminal.h
+++ b/lldb/tools/lldb-dap/RunInTerminal.h
@@ -10,9 +10,11 @@
 #define LLDB_TOOLS_LLDB_DAP_RUNINTERMINAL_H
 
 #include "FifoFiles.h"
+#include "lldb/API/SBError.h"
 
 #include <future>
-#include <thread>
+#include <memory>
+#include <string>
 
 namespace lldb_dap {
 
diff --git a/lldb/tools/lldb-dap/SourceBreakpoint.cpp b/lldb/tools/lldb-dap/SourceBreakpoint.cpp
index f5dd1346cb9e..d1a3a5bedb0a 100644
--- a/lldb/tools/lldb-dap/SourceBreakpoint.cpp
+++ b/lldb/tools/lldb-dap/SourceBreakpoint.cpp
@@ -8,6 +8,7 @@
 
 #include "SourceBreakpoint.h"
 #include "DAP.h"
+#include "JSONUtils.h"
 
 namespace lldb_dap {
 
diff --git a/lldb/tools/lldb-dap/lldb-dap.cpp b/lldb/tools/lldb-dap/lldb-dap.cpp
index f70b0d3d4cbe..a2f7be2b214e 100644
--- a/lldb/tools/lldb-dap/lldb-dap.cpp
+++ b/lldb/tools/lldb-dap/lldb-dap.cpp
@@ -7,9 +7,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "DAP.h"
+#include "FifoFiles.h"
+#include "RunInTerminal.h"
 #include "Watchpoint.h"
 #include "lldb/API/SBDeclaration.h"
+#include "lldb/API/SBInstruction.h"
+#include "lldb/API/SBListener.h"
 #include "lldb/API/SBMemoryRegionInfo.h"
+#include "lldb/API/SBStringList.h"
 #include "llvm/Support/Base64.h"
 
 #include <cassert>
@@ -43,17 +48,12 @@
 
 #include <algorithm>
 #include <array>
-#include <chrono>
-#include <fstream>
 #include <map>
 #include <memory>
-#include <mutex>
 #include <set>
-#include <sstream>
 #include <thread>
 #include <vector>
 
-#include "lldb/API/SBEnvironment.h"
 #include "lldb/API/SBStream.h"
 #include "lldb/Host/Config.h"
 #include "llvm/ADT/ArrayRef.h"
-- 
GitLab


From e6fcf349eff5be326e1923567646edf99f69d297 Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Mon, 28 Oct 2024 18:28:52 +0800
Subject: [PATCH 297/693] [clang-tidy] Remove unused variable (NFC)

/llvm-project/clang-tools-extra/clang-tidy/bugprone/NondeterministicPointerIterationOrderCheck.cpp:60:29:
error: unused variable 'AlgoName' [-Werror,-Wunused-variable]
      const llvm::StringRef AlgoName = ClassTemplate->getName();
                            ^
1 error generated.
---
 .../bugprone/NondeterministicPointerIterationOrderCheck.cpp      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/clang-tools-extra/clang-tidy/bugprone/NondeterministicPointerIterationOrderCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/NondeterministicPointerIterationOrderCheck.cpp
index 0797376d0aa9..22ecd6896146 100644
--- a/clang-tools-extra/clang-tidy/bugprone/NondeterministicPointerIterationOrderCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/NondeterministicPointerIterationOrderCheck.cpp
@@ -57,7 +57,6 @@ void NondeterministicPointerIterationOrderCheck::check(
                 "recorddecl")) {
       const TemplateArgumentList &TemplateArgs =
           ClassTemplate->getTemplateArgs();
-      const llvm::StringRef AlgoName = ClassTemplate->getName();
       const bool IsAlgoArgPointer =
           TemplateArgs[0].getAsType()->isPointerType();
 
-- 
GitLab


From f5d8a485e216996b076369b4d66c41a70fe34ea4 Mon Sep 17 00:00:00 2001
From: Brandon Wu <brandon.wu@sifive.com>
Date: Mon, 28 Oct 2024 18:30:21 +0800
Subject: [PATCH 298/693] [RISCV] Fix typo in UserGuides.rst. NFC (#113861)

---
 llvm/docs/UserGuides.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/docs/UserGuides.rst b/llvm/docs/UserGuides.rst
index 86101ffbd9ca..0b204d512876 100644
--- a/llvm/docs/UserGuides.rst
+++ b/llvm/docs/UserGuides.rst
@@ -286,7 +286,7 @@ Additional Topics
    DirectX runtime.
 
 :doc:`RISCVUsage`
-   This document describes using the RISCV-V target.
+   This document describes using the RISC-V target.
 
 :doc:`RISCV/RISCVVectorExtension`
    This document describes how the RISC-V Vector extension can be expressed in LLVM IR and how code is generated for it in the backend.
-- 
GitLab


From 09160a98218e7f1038e06c5d8e704a826ed0ae13 Mon Sep 17 00:00:00 2001
From: Jonas Paulsson <paulson1@linux.ibm.com>
Date: Mon, 28 Oct 2024 11:32:39 +0100
Subject: [PATCH 299/693] [SystemZ] Silence compiler warning (#113894)

Use SystemZ::NoRegister instead of 0 in
SystemZTargetLowering::getRegisterByName().
---
 llvm/lib/Target/SystemZ/SystemZISelLowering.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 3e05f3b0180a..1fa2dbfb26fc 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -1400,9 +1400,11 @@ SystemZTargetLowering::getRegisterByName(const char *RegName, LLT VT,
                                          const MachineFunction &MF) const {
   Register Reg =
       StringSwitch<Register>(RegName)
-          .Case("r4", Subtarget.isTargetXPLINK64() ? SystemZ::R4D : 0)
-          .Case("r15", Subtarget.isTargetELF() ? SystemZ::R15D : 0)
-          .Default(0);
+          .Case("r4", Subtarget.isTargetXPLINK64() ? SystemZ::R4D
+                                                   : SystemZ::NoRegister)
+          .Case("r15",
+                Subtarget.isTargetELF() ? SystemZ::R15D : SystemZ::NoRegister)
+          .Default(SystemZ::NoRegister);
 
   if (Reg)
     return Reg;
-- 
GitLab


From 64148944c56f50a50383c79d5b6d074d8608f2a4 Mon Sep 17 00:00:00 2001
From: SpencerAbson <Spencer.Abson@arm.com>
Date: Mon, 28 Oct 2024 10:41:07 +0000
Subject: [PATCH 300/693] [AArch64] Add assembly/disassembly for zeroing SVE2
 integer instructions (#113473)

This patch adds assembly/disassembly for the following SVE2.2
instructions

    -  SQABS  (zeroing)
    -  SQNEG  (zeroing)
    -  URECPE (zeroing)
    -  USQRTE (zeroing)

- Refactor the existing merging forms to remove the now redundant bit 17
argument.
- In accordance with:
https://developer.arm.com/documentation/ddi0602/latest/
---
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 14 +++--
 llvm/lib/Target/AArch64/SVEInstrFormats.td    | 52 ++++++++++++++-----
 llvm/test/MC/AArch64/SVE2/sqabs-diagnostics.s |  2 +-
 llvm/test/MC/AArch64/SVE2/sqneg-diagnostics.s |  2 +-
 .../test/MC/AArch64/SVE2/urecpe-diagnostics.s |  2 +-
 .../MC/AArch64/SVE2/ursqrte-diagnostics.s     |  2 +-
 .../MC/AArch64/SVE2p2/sqabs_z-diagnostics.s   | 37 +++++++++++++
 llvm/test/MC/AArch64/SVE2p2/sqabs_z.s         | 39 ++++++++++++++
 .../MC/AArch64/SVE2p2/sqneg_z-diagnostics.s   | 37 +++++++++++++
 llvm/test/MC/AArch64/SVE2p2/sqneg_z.s         | 39 ++++++++++++++
 .../MC/AArch64/SVE2p2/urecpe_z-diagnostics.s  | 47 +++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/urecpe_z.s        | 33 ++++++++++++
 .../MC/AArch64/SVE2p2/ursqrte_z-diagnostics.s | 47 +++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/ursqrte_z.s       | 33 ++++++++++++
 14 files changed, 366 insertions(+), 20 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/sqabs_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/sqabs_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/sqneg_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/sqneg_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/urecpe_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/urecpe_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/ursqrte_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/ursqrte_z.s

diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index bf6e25438633..dc96b249c4e4 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -3586,10 +3586,10 @@ let Predicates = [HasSVE2orSME] in {
   defm UMINP_ZPmZ : sve2_int_arith_pred<0b101111, "uminp", int_aarch64_sve_uminp>;
 
   // SVE2 integer unary operations (predicated)
-  defm URECPE_ZPmZ  : sve2_int_un_pred_arit_s<0b000, "urecpe",  int_aarch64_sve_urecpe>;
-  defm URSQRTE_ZPmZ : sve2_int_un_pred_arit_s<0b001, "ursqrte", int_aarch64_sve_ursqrte>;
-  defm SQABS_ZPmZ   : sve2_int_un_pred_arit<0b100,   "sqabs",   int_aarch64_sve_sqabs>;
-  defm SQNEG_ZPmZ   : sve2_int_un_pred_arit<0b101,   "sqneg",   int_aarch64_sve_sqneg>;
+  defm URECPE_ZPmZ  : sve2_int_un_pred_arit_s<0b00, "urecpe",  int_aarch64_sve_urecpe>;
+  defm URSQRTE_ZPmZ : sve2_int_un_pred_arit_s<0b01, "ursqrte", int_aarch64_sve_ursqrte>;
+  defm SQABS_ZPmZ   : sve2_int_un_pred_arit<  0b10, "sqabs",   int_aarch64_sve_sqabs>;
+  defm SQNEG_ZPmZ   : sve2_int_un_pred_arit<  0b11, "sqneg",   int_aarch64_sve_sqneg>;
 
   // SVE2 saturating add/subtract
   defm SQADD_ZPmZ  : sve2_int_arith_pred<0b110000, "sqadd",  int_aarch64_sve_sqadd>;
@@ -4236,6 +4236,12 @@ let Predicates = [HasSVE2p2orSME2p2] in {
   // Signed integer base 2 logarithm of fp value, zeroing predicate
   defm FLOGB_ZPzZ : sve_fp_z2op_p_zd_d_flogb<"flogb">;
 
+  // SVE2 integer unary operations, zeroing predicate
+  def URECPE_ZPzZ  : sve2_int_un_pred_arit_z<0b10, 0b00, "urecpe", ZPR32>;
+  def URSQRTE_ZPzZ : sve2_int_un_pred_arit_z<0b10, 0b01, "ursqrte", ZPR32>;
+  defm SQABS_ZPzZ  : sve2_int_un_pred_arit_z<0b10, "sqabs">;
+  defm SQNEG_ZPzZ  : sve2_int_un_pred_arit_z<0b11, "sqneg">;
+
   // Floating point round to integral fp value in integer size range
   // Merging
   defm FRINT32Z_ZPmZ : sve_fp_2op_p_zd_frint<0b00, "frint32z">;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index af5c96eb5c8c..02ee0fe92445 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -3964,7 +3964,7 @@ multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm, SDPatternOperato
   def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>;
 }
 
-class sve2_int_un_pred_arit<bits<2> sz, bit Q, bits<2> opc,
+class sve2_int_un_pred_arit<bits<2> sz, bits<2> opc,
                             string asm, ZPRRegOp zprty>
 : I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn),
   asm, "\t$Zd, $Pg/m, $Zn",
@@ -3976,23 +3976,44 @@ class sve2_int_un_pred_arit<bits<2> sz, bit Q, bits<2> opc,
   let Inst{31-24} = 0b01000100;
   let Inst{23-22} = sz;
   let Inst{21-20} = 0b00;
-  let Inst{19}    = Q;
-  let Inst{18}    = 0b0;
-  let Inst{17-16} = opc;
+  let Inst{19}    = opc{1};
+  let Inst{18-17} = 0b00;
+  let Inst{16}    = opc{0};
   let Inst{15-13} = 0b101;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
-
   let Constraints = "$Zd = $_Zd";
   let DestructiveInstType = DestructiveUnaryPassthru;
   let ElementSize = zprty.ElementSize;
   let hasSideEffects = 0;
 }
 
-multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm,
+class sve2_int_un_pred_arit_z<bits<2> sz, bits<2> opc,
+                              string asm, ZPRRegOp zprty>
+: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn),
+  asm, "\t$Zd, $Pg/z, $Zn",
+  "",
+  []>, Sched<[]> {
+  bits<3> Pg;
+  bits<5> Zd;
+  bits<5> Zn;
+  let Inst{31-24} = 0b01000100;
+  let Inst{23-22} = sz;
+  let Inst{21-20} = 0b00;
+  let Inst{19}    = opc{1};
+  let Inst{18-17} = 0b01;
+  let Inst{16}    = opc{0};
+  let Inst{15-13} = 0b101;
+  let Inst{12-10} = Pg;
+  let Inst{9-5}   = Zn;
+  let Inst{4-0}   = Zd;
+  let hasSideEffects = 0;
+}
+
+multiclass sve2_int_un_pred_arit_s<bits<2> opc, string asm,
                                    SDPatternOperator op> {
-  def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>,
+  def _S : sve2_int_un_pred_arit<0b10, opc, asm, ZPR32>,
            SVEPseudo2Instr<NAME # _S, 1>;
 
   def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;
@@ -4002,14 +4023,14 @@ multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm,
   defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1,  nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;
 }
 
-multiclass sve2_int_un_pred_arit<bits<3> opc, string asm, SDPatternOperator op> {
-  def _B : sve2_int_un_pred_arit<0b00, opc{2}, opc{1-0}, asm, ZPR8>,
+multiclass sve2_int_un_pred_arit<bits<2> opc, string asm, SDPatternOperator op> {
+  def _B : sve2_int_un_pred_arit<0b00, opc, asm, ZPR8>,
            SVEPseudo2Instr<NAME # _B, 1>;
-  def _H : sve2_int_un_pred_arit<0b01, opc{2}, opc{1-0}, asm, ZPR16>,
+  def _H : sve2_int_un_pred_arit<0b01, opc, asm, ZPR16>,
            SVEPseudo2Instr<NAME # _H, 1>;
-  def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>,
+  def _S : sve2_int_un_pred_arit<0b10, opc, asm, ZPR32>,
            SVEPseudo2Instr<NAME # _S, 1>;
-  def _D : sve2_int_un_pred_arit<0b11, opc{2}, opc{1-0}, asm, ZPR64>,
+  def _D : sve2_int_un_pred_arit<0b11, opc, asm, ZPR64>,
            SVEPseudo2Instr<NAME # _D, 1>;
 
   def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
@@ -4028,6 +4049,13 @@ multiclass sve2_int_un_pred_arit<bits<3> opc, string asm, SDPatternOperator op>
   defm : SVE_3_Op_Undef_Pat<nxv2i64, op, nxv2i64, nxv2i1,  nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;
 }
 
+multiclass sve2_int_un_pred_arit_z<bits<2> opc, string asm> {
+  def _B : sve2_int_un_pred_arit_z<0b00, opc, asm, ZPR8>;
+  def _H : sve2_int_un_pred_arit_z<0b01, opc, asm, ZPR16>;
+  def _S : sve2_int_un_pred_arit_z<0b10, opc, asm, ZPR32>;
+  def _D : sve2_int_un_pred_arit_z<0b11, opc, asm, ZPR64>;
+}
+
 //===----------------------------------------------------------------------===//
 // SVE2 Widening Integer Arithmetic Group
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AArch64/SVE2/sqabs-diagnostics.s b/llvm/test/MC/AArch64/SVE2/sqabs-diagnostics.s
index 8b3a136a7d74..7dd268dd7cdd 100644
--- a/llvm/test/MC/AArch64/SVE2/sqabs-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE2/sqabs-diagnostics.s
@@ -4,7 +4,7 @@
 // Invalid predicate
 
 sqabs z0.s, p0/z, z1.s
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: sqabs z0.s, p0/z, z1.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE2/sqneg-diagnostics.s b/llvm/test/MC/AArch64/SVE2/sqneg-diagnostics.s
index 7b0f5722b94a..372adad0427c 100644
--- a/llvm/test/MC/AArch64/SVE2/sqneg-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE2/sqneg-diagnostics.s
@@ -4,7 +4,7 @@
 // Invalid predicate
 
 sqneg z0.s, p0/z, z1.s
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: sqneg z0.s, p0/z, z1.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE2/urecpe-diagnostics.s b/llvm/test/MC/AArch64/SVE2/urecpe-diagnostics.s
index f04538494cd6..73bb6cecffa5 100644
--- a/llvm/test/MC/AArch64/SVE2/urecpe-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE2/urecpe-diagnostics.s
@@ -4,7 +4,7 @@
 // Invalid predicate
 
 urecpe z0.s, p0/z, z1.s
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: urecpe z0.s, p0/z, z1.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE2/ursqrte-diagnostics.s b/llvm/test/MC/AArch64/SVE2/ursqrte-diagnostics.s
index 2190ff1ebd82..0c6746a4a7c1 100644
--- a/llvm/test/MC/AArch64/SVE2/ursqrte-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE2/ursqrte-diagnostics.s
@@ -4,7 +4,7 @@
 // Invalid predicate
 
 ursqrte z0.s, p0/z, z1.s
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: ursqrte z0.s, p0/z, z1.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE2p2/sqabs_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/sqabs_z-diagnostics.s
new file mode 100644
index 000000000000..8c00a74c69c8
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/sqabs_z-diagnostics.s
@@ -0,0 +1,37 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+sqabs     z31.b, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: sqabs     z31.b, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sqabs     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: sqabs     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+sqabs     z31.b, p8/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: sqabs     z31.b, p8/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.h, p0/z, z7.h
+sqabs z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: sqabs z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+sqabs z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: sqabs z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/sqabs_z.s b/llvm/test/MC/AArch64/SVE2p2/sqabs_z.s
new file mode 100644
index 000000000000..c583423fac6e
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/sqabs_z.s
@@ -0,0 +1,39 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+sqabs   z0.b, p0/z, z0.b  // 01000100-00001010-10100000-00000000
+// CHECK-INST: sqabs   z0.b, p0/z, z0.b
+// CHECK-ENCODING: [0x00,0xa0,0x0a,0x44]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 440aa000 <unknown>
+
+sqabs   z21.h, p5/z, z10.h  // 01000100-01001010-10110101-01010101
+// CHECK-INST: sqabs   z21.h, p5/z, z10.h
+// CHECK-ENCODING: [0x55,0xb5,0x4a,0x44]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 444ab555 <unknown>
+
+sqabs   z23.s, p3/z, z13.s  // 01000100-10001010-10101101-10110111
+// CHECK-INST: sqabs   z23.s, p3/z, z13.s
+// CHECK-ENCODING: [0xb7,0xad,0x8a,0x44]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 448aadb7 <unknown>
+
+sqabs   z31.d, p7/z, z31.d  // 01000100-11001010-10111111-11111111
+// CHECK-INST: sqabs   z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xca,0x44]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 44cabfff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/sqneg_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/sqneg_z-diagnostics.s
new file mode 100644
index 000000000000..576633c0ab1b
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/sqneg_z-diagnostics.s
@@ -0,0 +1,37 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+sqneg     z31.b, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: sqneg     z31.b, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sqneg     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: sqneg     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+sqneg     z31.b, p8/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: sqneg     z31.b, p8/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.h, p0/z, z7.h
+sqneg z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: sqneg z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+sqneg z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: sqneg z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/sqneg_z.s b/llvm/test/MC/AArch64/SVE2p2/sqneg_z.s
new file mode 100644
index 000000000000..287211fd3ff5
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/sqneg_z.s
@@ -0,0 +1,39 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+sqneg   z0.b, p0/z, z0.b  // 01000100-00001011-10100000-00000000
+// CHECK-INST: sqneg   z0.b, p0/z, z0.b
+// CHECK-ENCODING: [0x00,0xa0,0x0b,0x44]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 440ba000 <unknown>
+
+sqneg   z23.h, p3/z, z13.h  // 01000100-01001011-10101101-10110111
+// CHECK-INST: sqneg   z23.h, p3/z, z13.h
+// CHECK-ENCODING: [0xb7,0xad,0x4b,0x44]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 444badb7 <unknown>
+
+sqneg   z21.s, p5/z, z10.s  // 01000100-10001011-10110101-01010101
+// CHECK-INST: sqneg   z21.s, p5/z, z10.s
+// CHECK-ENCODING: [0x55,0xb5,0x8b,0x44]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 448bb555 <unknown>
+
+sqneg   z31.d, p7/z, z31.d  // 01000100-11001011-10111111-11111111
+// CHECK-INST: sqneg   z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xcb,0x44]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 44cbbfff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/urecpe_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/urecpe_z-diagnostics.s
new file mode 100644
index 000000000000..9da2a7e096f3
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/urecpe_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid element width
+
+urecpe     z31.b, p7/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: urecpe     z31.b, p7/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+urecpe     z31.h, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: urecpe     z31.h, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+urecpe     z31.s, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: urecpe     z31.s, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+urecpe     z31.d, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: urecpe     z31.d, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+urecpe z0.s, p8/z, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: urecpe z0.s, p8/z, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.s, p0/z, z7.s
+urecpe z0.s, p0/z, z3.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: urecpe z0.s, p0/z, z3.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+urecpe z0.s, p0/z, z3.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: urecpe z0.s, p0/z, z3.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/urecpe_z.s b/llvm/test/MC/AArch64/SVE2p2/urecpe_z.s
new file mode 100644
index 000000000000..6b3ec79c9886
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/urecpe_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+urecpe  z0.s, p0/z, z0.s  // 01000100-10000010-10100000-00000000
+// CHECK-INST: urecpe  z0.s, p0/z, z0.s
+// CHECK-ENCODING: [0x00,0xa0,0x82,0x44]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 4482a000 <unknown>
+
+urecpe  z23.s, p3/z, z13.s  // 01000100-10000010-10101101-10110111
+// CHECK-INST: urecpe  z23.s, p3/z, z13.s
+// CHECK-ENCODING: [0xb7,0xad,0x82,0x44]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 4482adb7 <unknown>
+
+urecpe  z31.s, p7/z, z31.s  // 01000100-10000010-10111111-11111111
+// CHECK-INST: urecpe  z31.s, p7/z, z31.s
+// CHECK-ENCODING: [0xff,0xbf,0x82,0x44]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 4482bfff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/ursqrte_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/ursqrte_z-diagnostics.s
new file mode 100644
index 000000000000..1b6cf8d78e65
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/ursqrte_z-diagnostics.s
@@ -0,0 +1,47 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid element width
+
+ursqrte     z31.b, p7/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: ursqrte     z31.b, p7/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ursqrte     z31.h, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: ursqrte     z31.h, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ursqrte     z31.s, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: ursqrte     z31.s, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ursqrte     z31.d, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: ursqrte     z31.d, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+ursqrte z0.s, p8/z, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: ursqrte z0.s, p8/z, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.s, p0/z, z7.s
+ursqrte z0.s, p0/z, z3.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: ursqrte z0.s, p0/z, z3.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+ursqrte z0.s, p0/z, z3.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: ursqrte z0.s, p0/z, z3.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/ursqrte_z.s b/llvm/test/MC/AArch64/SVE2p2/ursqrte_z.s
new file mode 100644
index 000000000000..097d82bebd43
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/ursqrte_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+ursqrte z0.s, p0/z, z0.s  // 01000100-10000011-10100000-00000000
+// CHECK-INST: ursqrte z0.s, p0/z, z0.s
+// CHECK-ENCODING: [0x00,0xa0,0x83,0x44]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 4483a000 <unknown>
+
+ursqrte z21.s, p5/z, z10.s  // 01000100-10000011-10110101-01010101
+// CHECK-INST: ursqrte z21.s, p5/z, z10.s
+// CHECK-ENCODING: [0x55,0xb5,0x83,0x44]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 4483b555 <unknown>
+
+ursqrte z31.s, p7/z, z31.s  // 01000100-10000011-10111111-11111111
+// CHECK-INST: ursqrte z31.s, p7/z, z31.s
+// CHECK-ENCODING: [0xff,0xbf,0x83,0x44]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 4483bfff <unknown>
\ No newline at end of file
-- 
GitLab


From f5ff3a560fe247206814792a7f8efef8215f7689 Mon Sep 17 00:00:00 2001
From: Discookie <viktor.cseh@ericsson.com>
Date: Mon, 28 Oct 2024 11:19:22 +0000
Subject: [PATCH 301/693] [clang-tidy] Do not emit file path for anonymous
 enums in `readability-enum-initial-value` check (#112496)

Previously the name of anonymous enums in the check were `enum 'enum
(unnamed at /full/path/to/file.c:1:1)'`, which breaks reproducibility of
clang-tidy reports when the analyzed project is in a different folder.
---
 .../readability/EnumInitialValueCheck.cpp     | 24 ++++++++++++-------
 clang-tools-extra/docs/ReleaseNotes.rst       |  3 ++-
 .../checkers/readability/enum-initial-value.c | 12 +++++++++-
 3 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp b/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp
index 1cb95c2b2347..e0b993968179 100644
--- a/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp
@@ -123,6 +123,13 @@ AST_MATCHER(EnumDecl, hasSequentialInitialValues) {
   return !AllEnumeratorsArePowersOfTwo;
 }
 
+std::string getName(const EnumDecl *Decl) {
+  if (!Decl->getDeclName())
+    return "<unnamed>";
+
+  return Decl->getQualifiedNameAsString();
+}
+
 } // namespace
 
 EnumInitialValueCheck::EnumInitialValueCheck(StringRef Name,
@@ -160,10 +167,11 @@ void EnumInitialValueCheck::registerMatchers(MatchFinder *Finder) {
 void EnumInitialValueCheck::check(const MatchFinder::MatchResult &Result) {
   if (const auto *Enum = Result.Nodes.getNodeAs<EnumDecl>("inconsistent")) {
     DiagnosticBuilder Diag =
-        diag(Enum->getBeginLoc(),
-             "initial values in enum %0 are not consistent, consider explicit "
-             "initialization of all, none or only the first enumerator")
-        << Enum;
+        diag(
+            Enum->getBeginLoc(),
+            "initial values in enum '%0' are not consistent, consider explicit "
+            "initialization of all, none or only the first enumerator")
+        << getName(Enum);
     for (const EnumConstantDecl *ECD : Enum->enumerators())
       if (ECD->getInitExpr() == nullptr) {
         const SourceLocation EndLoc = Lexer::getLocForEndOfToken(
@@ -183,16 +191,16 @@ void EnumInitialValueCheck::check(const MatchFinder::MatchResult &Result) {
     if (Loc.isInvalid() || Loc.isMacroID())
       return;
     DiagnosticBuilder Diag = diag(Loc, "zero initial value for the first "
-                                       "enumerator in %0 can be disregarded")
-                             << Enum;
+                                       "enumerator in '%0' can be disregarded")
+                             << getName(Enum);
     cleanInitialValue(Diag, ECD, *Result.SourceManager, getLangOpts());
     return;
   }
   if (const auto *Enum = Result.Nodes.getNodeAs<EnumDecl>("sequential")) {
     DiagnosticBuilder Diag =
         diag(Enum->getBeginLoc(),
-             "sequential initial value in %0 can be ignored")
-        << Enum;
+             "sequential initial value in '%0' can be ignored")
+        << getName(Enum);
     for (const EnumConstantDecl *ECD : llvm::drop_begin(Enum->enumerators()))
       cleanInitialValue(Diag, ECD, *Result.SourceManager, getLangOpts());
     return;
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 7eb2ee511a05..54118e5f92f4 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -249,7 +249,8 @@ Changes in existing checks
 
 - Improved :doc:`readability-enum-initial-value
   <clang-tidy/checks/readability/enum-initial-value>` check by only issuing
-  diagnostics for the definition of an ``enum``, and by fixing a typo in the
+  diagnostics for the definition of an ``enum``, by not emitting a redundant
+  file path for anonymous enums in the diagnostic, and by fixing a typo in the
   diagnostic.
 
 - Improved :doc:`readability-implicit-bool-conversion
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.c b/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.c
index b9a34d0683d7..54108585f030 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.c
@@ -53,6 +53,17 @@ enum EMacro2 {
   // CHECK-FIXES: EMacro2_c = 3,
 };
 
+
+enum {
+  // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: initial values in enum '<unnamed>' are not consistent
+  // CHECK-MESSAGES-ENABLE: :[[@LINE-2]]:1: warning: initial values in enum '<unnamed>' are not consistent
+  EAnonymous_a = 1,
+  EAnonymous_b,
+  // CHECK-FIXES: EAnonymous_b = 2,
+  EAnonymous_c = 3,
+};
+
+
 enum EnumZeroFirstInitialValue {
   EnumZeroFirstInitialValue_0 = 0,
   // CHECK-MESSAGES-ENABLE: :[[@LINE-1]]:3: warning: zero initial value for the first enumerator in 'EnumZeroFirstInitialValue' can be disregarded
@@ -114,4 +125,3 @@ enum WithFwdDeclSequential : int {
   EFS2 = 4,
   // CHECK-FIXES-ENABLE: EFS2 ,
 };
-
-- 
GitLab


From 1df5c943439b050dbc34e7a1c88c4d8ec90d26b5 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Mon, 28 Oct 2024 11:22:38 +0000
Subject: [PATCH 302/693] [AArch64] Implement FP8 floating-point mode helper
 intrinsics (#100608)

Implement FP8 mode helper intrinsics (as inline functions) as
specified in ACLE 2024Q3 "14.2 Helper intrinsics"

https://github.com/ARM-software/acle/releases/download/r2024Q3/acle-2024Q3.pdf
---
 clang/test/CodeGen/aarch64-fpm-helpers.c | 165 +++++++++++++++++++++++
 clang/utils/TableGen/NeonEmitter.cpp     |  54 ++++++++
 2 files changed, 219 insertions(+)
 create mode 100644 clang/test/CodeGen/aarch64-fpm-helpers.c

diff --git a/clang/test/CodeGen/aarch64-fpm-helpers.c b/clang/test/CodeGen/aarch64-fpm-helpers.c
new file mode 100644
index 000000000000..4bced01d5c71
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-fpm-helpers.c
@@ -0,0 +1,165 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+
+// RUN: %clang_cc1 -O2 -triple aarch64 -emit-llvm -x c   -DUSE_NEON_H  %s -o - | FileCheck %s
+// RUN: %clang_cc1 -O2 -triple aarch64 -emit-llvm -x c   -DUSE_SVE_H   %s -o - | FileCheck %s
+// RUN: %clang_cc1 -O2 -triple aarch64 -emit-llvm -x c   -DUSE_SME_H   %s -o - | FileCheck %s
+// RUN: %clang_cc1 -O2 -triple aarch64 -emit-llvm -x c++ -DUSE_NEON_H  %s -o - | FileCheck %s
+// RUN: %clang_cc1 -O2 -triple aarch64 -emit-llvm -x c++ -DUSE_SVE_H   %s -o - | FileCheck %s
+// RUN: %clang_cc1 -O2 -triple aarch64 -emit-llvm -x c++ -DUSE_SME_H   %s -o - | FileCheck %s
+
+// REQUIRES: aarch64-registered-target
+
+#ifdef USE_NEON_H
+#include "arm_neon.h"
+#endif
+
+#ifdef USE_SVE_H
+#include "arm_sve.h"
+#endif
+
+#ifdef USE_SME_H
+#include "arm_sme.h"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define INIT_ZERO 0
+#define INIT_ONES 0xffffffffffffffffU
+
+// CHECK-LABEL: define dso_local noundef i64 @test_init(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 0
+//
+fpm_t test_init() { return __arm_fpm_init(); }
+
+// CHECK-LABEL: define dso_local noundef i64 @test_src1_1(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 -8
+//
+fpm_t test_src1_1() {
+  return __arm_set_fpm_src1_format(INIT_ONES, __ARM_FPM_E5M2);
+}
+
+// CHECK-LABEL: define dso_local noundef i64 @test_src1_2(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 1
+//
+fpm_t test_src1_2() {
+  return __arm_set_fpm_src1_format(INIT_ZERO, __ARM_FPM_E4M3);
+}
+
+// CHECK-LABEL: define dso_local noundef i64 @test_src2_1(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 -57
+//
+fpm_t test_src2_1() {
+  return __arm_set_fpm_src2_format(INIT_ONES, __ARM_FPM_E5M2);
+}
+
+// CHECK-LABEL: define dso_local noundef i64 @test_src2_2(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 8
+//
+fpm_t test_src2_2() {
+  return __arm_set_fpm_src2_format(INIT_ZERO, __ARM_FPM_E4M3);
+}
+
+// CHECK-LABEL: define dso_local noundef i64 @test_dst1_1(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 -449
+//
+fpm_t test_dst1_1() {
+  return __arm_set_fpm_dst_format(INIT_ONES, __ARM_FPM_E5M2);
+}
+
+// CHECK-LABEL: define dso_local noundef i64 @test_dst2_2(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 64
+//
+fpm_t test_dst2_2() {
+  return __arm_set_fpm_dst_format(INIT_ZERO, __ARM_FPM_E4M3);
+}
+
+// CHECK-LABEL: define dso_local noundef i64 @test_of_mul_1(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 -16385
+//
+fpm_t test_of_mul_1() {
+  return __arm_set_fpm_overflow_mul(INIT_ONES, __ARM_FPM_INFNAN);
+}
+
+// CHECK-LABEL: define dso_local noundef i64 @test_of_mul_2(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 16384
+//
+fpm_t test_of_mul_2() {
+  return __arm_set_fpm_overflow_mul(INIT_ZERO, __ARM_FPM_SATURATE);
+}
+
+// CHECK-LABEL: define dso_local noundef i64 @test_of_cvt_1(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 -32769
+//
+fpm_t test_of_cvt_1() {
+  return __arm_set_fpm_overflow_cvt(INIT_ONES, __ARM_FPM_INFNAN);
+}
+
+// CHECK-LABEL: define dso_local noundef i64 @test_of_cvt_2(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 32768
+//
+fpm_t test_of_cvt_2() {
+  return __arm_set_fpm_overflow_cvt(INIT_ZERO, __ARM_FPM_SATURATE);
+}
+
+// CHECK-LABEL: define dso_local noundef i64 @test_lscale(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 8323072
+//
+fpm_t test_lscale() { return __arm_set_fpm_lscale(INIT_ZERO, 127); }
+
+// CHECK-LABEL: define dso_local noundef i64 @test_lscale2(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 270582939648
+//
+fpm_t test_lscale2() { return __arm_set_fpm_lscale2(INIT_ZERO, 63); }
+
+// CHECK-LABEL: define dso_local noundef range(i64 0, 4294967296) i64 @test_nscale_1(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 2147483648
+//
+fpm_t test_nscale_1() { return __arm_set_fpm_nscale(INIT_ZERO, -128); }
+
+// CHECK-LABEL: define dso_local noundef range(i64 0, 4294967296) i64 @test_nscale_2(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 2130706432
+//
+fpm_t test_nscale_2() { return __arm_set_fpm_nscale(INIT_ZERO, 127); }
+
+// CHECK-LABEL: define dso_local noundef range(i64 0, 4294967296) i64 @test_nscale_3(
+// CHECK-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret i64 4278190080
+//
+fpm_t test_nscale_3() { return __arm_set_fpm_nscale(INIT_ZERO, -1); }
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index 8caba774649a..c6d82646b40d 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -2594,6 +2594,60 @@ void NeonEmitter::runVectorTypes(raw_ostream &OS) {
   OS << "typedef double float64_t;\n";
   OS << "#endif\n\n";
 
+  OS << R"(
+typedef uint64_t fpm_t;
+
+enum __ARM_FPM_FORMAT { __ARM_FPM_E5M2, __ARM_FPM_E4M3 };
+
+enum __ARM_FPM_OVERFLOW { __ARM_FPM_INFNAN, __ARM_FPM_SATURATE };
+
+static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
+__arm_fpm_init(void) {
+  return 0;
+}
+
+static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
+__arm_set_fpm_src1_format(fpm_t __fpm, enum __ARM_FPM_FORMAT __format) {
+  return (__fpm & ~7ull) | (fpm_t)__format;
+}
+
+static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
+__arm_set_fpm_src2_format(fpm_t __fpm, enum __ARM_FPM_FORMAT __format) {
+  return (__fpm & ~0x38ull) | ((fpm_t)__format << 3u);
+}
+
+static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
+__arm_set_fpm_dst_format(fpm_t __fpm, enum __ARM_FPM_FORMAT __format) {
+  return (__fpm & ~0x1c0ull) | ((fpm_t)__format << 6u);
+}
+
+static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
+__arm_set_fpm_overflow_mul(fpm_t __fpm, enum __ARM_FPM_OVERFLOW __behaviour) {
+  return (__fpm & ~0x4000ull) | ((fpm_t)__behaviour << 14u);
+}
+
+static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
+__arm_set_fpm_overflow_cvt(fpm_t __fpm, enum __ARM_FPM_OVERFLOW __behaviour) {
+  return (__fpm & ~0x8000ull) | ((fpm_t)__behaviour << 15u);
+}
+
+static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
+__arm_set_fpm_lscale(fpm_t __fpm, uint64_t __scale) {
+  return (__fpm & ~0x7f0000ull) | (__scale << 16u);
+}
+
+static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
+__arm_set_fpm_nscale(fpm_t __fpm, int64_t __scale) {
+  return (__fpm & ~0xff000000ull) | (((fpm_t)__scale & 0xffu) << 24u);
+}
+
+static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
+__arm_set_fpm_lscale2(fpm_t __fpm, uint64_t __scale) {
+  return (uint32_t)__fpm | (__scale << 32u);
+}
+
+)";
+
   emitNeonTypeDefs("cQcsQsiQilQlUcQUcUsQUsUiQUiUlQUlhQhfQfdQd", OS);
 
   emitNeonTypeDefs("bQb", OS);
-- 
GitLab


From 0cbccb13d6757b0ea7f2a7f29bb598e1935bcf37 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Mon, 28 Oct 2024 11:31:18 +0000
Subject: [PATCH 303/693] [RISCV] Remove support for pre-RA vsetvli insertion
 (#110796)

Now that LLVM 19.1.0 has been out for a while with post-vector-RA
vsetvli insertion enabled by default, this proposes to remove the flag
that restores the old pre-RA behaviour so we only have one configuration
going forward.

That flag was mainly meant as a fallback in case users ran into issues,
but I haven't seen anything reported so far.
---
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp     |  11 +-
 llvm/lib/Target/RISCV/RISCVTargetMachine.cpp |  20 +-
 llvm/test/CodeGen/RISCV/rvv/remat.ll         | 468 +++++--------------
 3 files changed, 133 insertions(+), 366 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 3989a966edfd..a3963fadf3e4 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -179,17 +179,10 @@ bool RISCVInstrInfo::isReallyTriviallyReMaterializable(
   case RISCV::VMV_S_X:
   case RISCV::VFMV_S_F:
   case RISCV::VID_V:
-    if (MI.getOperand(1).isUndef() &&
-        /* After RISCVInsertVSETVLI most pseudos will have implicit uses on vl
-           and vtype.  Make sure we only rematerialize before RISCVInsertVSETVLI
-           i.e. -riscv-vsetvl-after-rvv-regalloc=true */
-        !MI.hasRegisterImplicitUseOperand(RISCV::VTYPE))
-      return true;
-    break;
+    return MI.getOperand(1).isUndef();
   default:
-    break;
+    return TargetInstrInfo::isReallyTriviallyReMaterializable(MI);
   }
-  return TargetInstrInfo::isReallyTriviallyReMaterializable(MI);
 }
 
 static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 089dc6c52919..72d74d2d79b1 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -99,11 +99,6 @@ static cl::opt<bool> EnableMISchedLoadStoreClustering(
     cl::desc("Enable load and store clustering in the machine scheduler"),
     cl::init(true));
 
-static cl::opt<bool> EnableVSETVLIAfterRVVRegAlloc(
-    "riscv-vsetvl-after-rvv-regalloc", cl::Hidden,
-    cl::desc("Insert vsetvls after vector register allocation"),
-    cl::init(true));
-
 static cl::opt<bool>
     EnableVLOptimizer("riscv-enable-vl-optimizer",
                       cl::desc("Enable the RISC-V VL Optimizer pass"),
@@ -413,8 +408,7 @@ FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
 
 bool RISCVPassConfig::addRegAssignAndRewriteFast() {
   addPass(createRVVRegAllocPass(false));
-  if (EnableVSETVLIAfterRVVRegAlloc)
-    addPass(createRISCVInsertVSETVLIPass());
+  addPass(createRISCVInsertVSETVLIPass());
   if (TM->getOptLevel() != CodeGenOptLevel::None &&
       EnableRISCVDeadRegisterElimination)
     addPass(createRISCVDeadRegisterDefinitionsPass());
@@ -424,8 +418,7 @@ bool RISCVPassConfig::addRegAssignAndRewriteFast() {
 bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
   addPass(createRVVRegAllocPass(true));
   addPass(createVirtRegRewriter(false));
-  if (EnableVSETVLIAfterRVVRegAlloc)
-    addPass(createRISCVInsertVSETVLIPass());
+  addPass(createRISCVInsertVSETVLIPass());
   if (TM->getOptLevel() != CodeGenOptLevel::None &&
       EnableRISCVDeadRegisterElimination)
     addPass(createRISCVDeadRegisterDefinitionsPass());
@@ -575,15 +568,6 @@ void RISCVPassConfig::addPreRegAlloc() {
   addPass(createRISCVInsertReadWriteCSRPass());
   addPass(createRISCVInsertWriteVXRMPass());
   addPass(createRISCVLandingPadSetupPass());
-
-  // Run RISCVInsertVSETVLI after PHI elimination. On O1 and above do it after
-  // register coalescing so needVSETVLIPHI doesn't need to look through COPYs.
-  if (!EnableVSETVLIAfterRVVRegAlloc) {
-    if (TM->getOptLevel() == CodeGenOptLevel::None)
-      insertPass(&PHIEliminationID, &RISCVInsertVSETVLIID);
-    else
-      insertPass(&RegisterCoalescerID, &RISCVInsertVSETVLIID);
-  }
 }
 
 void RISCVPassConfig::addFastRegAlloc() {
diff --git a/llvm/test/CodeGen/RISCV/rvv/remat.ll b/llvm/test/CodeGen/RISCV/rvv/remat.ll
index 4f58ccb5188d..64c59769546f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/remat.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/remat.ll
@@ -1,53 +1,23 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,POSTRA
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v -riscv-vsetvl-after-rvv-regalloc=false -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,PRERA
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v -verify-machineinstrs | FileCheck %s
 
 define void @vid(ptr %p) {
-; POSTRA-LABEL: vid:
-; POSTRA:       # %bb.0:
-; POSTRA-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; POSTRA-NEXT:    vid.v v8
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    vl8re64.v v16, (a0)
-; POSTRA-NEXT:    vl8re64.v v24, (a0)
-; POSTRA-NEXT:    vl8re64.v v0, (a0)
-; POSTRA-NEXT:    vl8re64.v v8, (a0)
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    vs8r.v v0, (a0)
-; POSTRA-NEXT:    vs8r.v v24, (a0)
-; POSTRA-NEXT:    vs8r.v v16, (a0)
-; POSTRA-NEXT:    vid.v v8
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    ret
-;
-; PRERA-LABEL: vid:
-; PRERA:       # %bb.0:
-; PRERA-NEXT:    addi sp, sp, -16
-; PRERA-NEXT:    .cfi_def_cfa_offset 16
-; PRERA-NEXT:    csrr a1, vlenb
-; PRERA-NEXT:    slli a1, a1, 3
-; PRERA-NEXT:    sub sp, sp, a1
-; PRERA-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; PRERA-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; PRERA-NEXT:    vid.v v8
-; PRERA-NEXT:    vs8r.v v8, (a0)
-; PRERA-NEXT:    vl8re64.v v16, (a0)
-; PRERA-NEXT:    addi a1, sp, 16
-; PRERA-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; PRERA-NEXT:    vl8re64.v v24, (a0)
-; PRERA-NEXT:    vl8re64.v v0, (a0)
-; PRERA-NEXT:    vl8re64.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v0, (a0)
-; PRERA-NEXT:    vs8r.v v24, (a0)
-; PRERA-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; PRERA-NEXT:    vs8r.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v8, (a0)
-; PRERA-NEXT:    csrr a0, vlenb
-; PRERA-NEXT:    slli a0, a0, 3
-; PRERA-NEXT:    add sp, sp, a0
-; PRERA-NEXT:    addi sp, sp, 16
-; PRERA-NEXT:    ret
+; CHECK-LABEL: vid:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vl8re64.v v16, (a0)
+; CHECK-NEXT:    vl8re64.v v24, (a0)
+; CHECK-NEXT:    vl8re64.v v0, (a0)
+; CHECK-NEXT:    vl8re64.v v8, (a0)
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vs8r.v v0, (a0)
+; CHECK-NEXT:    vs8r.v v24, (a0)
+; CHECK-NEXT:    vs8r.v v16, (a0)
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    ret
   %vid = call <vscale x 8 x i64> @llvm.riscv.vid.nxv8i64(<vscale x 8 x i64> poison, i64 -1)
   store volatile <vscale x 8 x i64> %vid, ptr %p
 
@@ -111,51 +81,22 @@ define void @vid_passthru(ptr %p, <vscale x 8 x i64> %v) {
 }
 
 define void @vmv.v.i(ptr %p) {
-; POSTRA-LABEL: vmv.v.i:
-; POSTRA:       # %bb.0:
-; POSTRA-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; POSTRA-NEXT:    vmv.v.i v8, 1
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    vl8re64.v v16, (a0)
-; POSTRA-NEXT:    vl8re64.v v24, (a0)
-; POSTRA-NEXT:    vl8re64.v v0, (a0)
-; POSTRA-NEXT:    vl8re64.v v8, (a0)
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    vs8r.v v0, (a0)
-; POSTRA-NEXT:    vs8r.v v24, (a0)
-; POSTRA-NEXT:    vs8r.v v16, (a0)
-; POSTRA-NEXT:    vmv.v.i v8, 1
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    ret
-;
-; PRERA-LABEL: vmv.v.i:
-; PRERA:       # %bb.0:
-; PRERA-NEXT:    addi sp, sp, -16
-; PRERA-NEXT:    .cfi_def_cfa_offset 16
-; PRERA-NEXT:    csrr a1, vlenb
-; PRERA-NEXT:    slli a1, a1, 3
-; PRERA-NEXT:    sub sp, sp, a1
-; PRERA-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; PRERA-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; PRERA-NEXT:    vmv.v.i v8, 1
-; PRERA-NEXT:    vs8r.v v8, (a0)
-; PRERA-NEXT:    vl8re64.v v16, (a0)
-; PRERA-NEXT:    addi a1, sp, 16
-; PRERA-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; PRERA-NEXT:    vl8re64.v v24, (a0)
-; PRERA-NEXT:    vl8re64.v v0, (a0)
-; PRERA-NEXT:    vl8re64.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v0, (a0)
-; PRERA-NEXT:    vs8r.v v24, (a0)
-; PRERA-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; PRERA-NEXT:    vs8r.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v8, (a0)
-; PRERA-NEXT:    csrr a0, vlenb
-; PRERA-NEXT:    slli a0, a0, 3
-; PRERA-NEXT:    add sp, sp, a0
-; PRERA-NEXT:    addi sp, sp, 16
-; PRERA-NEXT:    ret
+; CHECK-LABEL: vmv.v.i:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 1
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vl8re64.v v16, (a0)
+; CHECK-NEXT:    vl8re64.v v24, (a0)
+; CHECK-NEXT:    vl8re64.v v0, (a0)
+; CHECK-NEXT:    vl8re64.v v8, (a0)
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vs8r.v v0, (a0)
+; CHECK-NEXT:    vs8r.v v24, (a0)
+; CHECK-NEXT:    vs8r.v v16, (a0)
+; CHECK-NEXT:    vmv.v.i v8, 1
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    ret
   %vmv.v.i = call <vscale x 8 x i64> @llvm.riscv.vmv.v.x.nxv8i64(<vscale x 8 x i64> poison, i64 1, i64 -1)
   store volatile <vscale x 8 x i64> %vmv.v.i, ptr %p
 
@@ -172,66 +113,35 @@ define void @vmv.v.i(ptr %p) {
   ret void
 }
 
-; The live range of %x needs extended down to the use of vmv.v.x at the end of
-; the block.
 define void @vmv.v.x_needs_extended(ptr %p, i64 %x) {
-; POSTRA-LABEL: vmv.v.x_needs_extended:
-; POSTRA:       # %bb.0:
-; POSTRA-NEXT:    addi sp, sp, -16
-; POSTRA-NEXT:    .cfi_def_cfa_offset 16
-; POSTRA-NEXT:    csrr a2, vlenb
-; POSTRA-NEXT:    slli a2, a2, 3
-; POSTRA-NEXT:    sub sp, sp, a2
-; POSTRA-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; POSTRA-NEXT:    vsetvli a2, zero, e64, m8, ta, ma
-; POSTRA-NEXT:    vmv.v.x v8, a1
-; POSTRA-NEXT:    addi a1, sp, 16
-; POSTRA-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    vl8re64.v v16, (a0)
-; POSTRA-NEXT:    vl8re64.v v24, (a0)
-; POSTRA-NEXT:    vl8re64.v v0, (a0)
-; POSTRA-NEXT:    vl8re64.v v8, (a0)
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    vs8r.v v0, (a0)
-; POSTRA-NEXT:    vs8r.v v24, (a0)
-; POSTRA-NEXT:    vs8r.v v16, (a0)
-; POSTRA-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    csrr a0, vlenb
-; POSTRA-NEXT:    slli a0, a0, 3
-; POSTRA-NEXT:    add sp, sp, a0
-; POSTRA-NEXT:    addi sp, sp, 16
-; POSTRA-NEXT:    ret
-;
-; PRERA-LABEL: vmv.v.x_needs_extended:
-; PRERA:       # %bb.0:
-; PRERA-NEXT:    addi sp, sp, -16
-; PRERA-NEXT:    .cfi_def_cfa_offset 16
-; PRERA-NEXT:    csrr a2, vlenb
-; PRERA-NEXT:    slli a2, a2, 3
-; PRERA-NEXT:    sub sp, sp, a2
-; PRERA-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; PRERA-NEXT:    vsetvli a2, zero, e64, m8, ta, ma
-; PRERA-NEXT:    vmv.v.x v8, a1
-; PRERA-NEXT:    vs8r.v v8, (a0)
-; PRERA-NEXT:    vl8re64.v v16, (a0)
-; PRERA-NEXT:    addi a1, sp, 16
-; PRERA-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; PRERA-NEXT:    vl8re64.v v24, (a0)
-; PRERA-NEXT:    vl8re64.v v0, (a0)
-; PRERA-NEXT:    vl8re64.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v0, (a0)
-; PRERA-NEXT:    vs8r.v v24, (a0)
-; PRERA-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; PRERA-NEXT:    vs8r.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v8, (a0)
-; PRERA-NEXT:    csrr a0, vlenb
-; PRERA-NEXT:    slli a0, a0, 3
-; PRERA-NEXT:    add sp, sp, a0
-; PRERA-NEXT:    addi sp, sp, 16
-; PRERA-NEXT:    ret
+; CHECK-LABEL: vmv.v.x_needs_extended:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a2, vlenb
+; CHECK-NEXT:    slli a2, a2, 3
+; CHECK-NEXT:    sub sp, sp, a2
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a2, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vmv.v.x v8, a1
+; CHECK-NEXT:    addi a1, sp, 16
+; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vl8re64.v v16, (a0)
+; CHECK-NEXT:    vl8re64.v v24, (a0)
+; CHECK-NEXT:    vl8re64.v v0, (a0)
+; CHECK-NEXT:    vl8re64.v v8, (a0)
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vs8r.v v0, (a0)
+; CHECK-NEXT:    vs8r.v v24, (a0)
+; CHECK-NEXT:    vs8r.v v16, (a0)
+; CHECK-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
   %vmv.v.x = call <vscale x 8 x i64> @llvm.riscv.vmv.v.x.nxv8i64(<vscale x 8 x i64> poison, i64 %x, i64 -1)
   store volatile <vscale x 8 x i64> %vmv.v.x, ptr %p
 
@@ -249,53 +159,23 @@ define void @vmv.v.x_needs_extended(ptr %p, i64 %x) {
 }
 
 define void @vmv.v.x_live(ptr %p, i64 %x) {
-; POSTRA-LABEL: vmv.v.x_live:
-; POSTRA:       # %bb.0:
-; POSTRA-NEXT:    vsetvli a2, zero, e64, m8, ta, ma
-; POSTRA-NEXT:    vmv.v.x v8, a1
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    vl8re64.v v16, (a0)
-; POSTRA-NEXT:    vl8re64.v v24, (a0)
-; POSTRA-NEXT:    vl8re64.v v0, (a0)
-; POSTRA-NEXT:    vl8re64.v v8, (a0)
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    vs8r.v v0, (a0)
-; POSTRA-NEXT:    vs8r.v v24, (a0)
-; POSTRA-NEXT:    vs8r.v v16, (a0)
-; POSTRA-NEXT:    vmv.v.x v8, a1
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    sd a1, 0(a0)
-; POSTRA-NEXT:    ret
-;
-; PRERA-LABEL: vmv.v.x_live:
-; PRERA:       # %bb.0:
-; PRERA-NEXT:    addi sp, sp, -16
-; PRERA-NEXT:    .cfi_def_cfa_offset 16
-; PRERA-NEXT:    csrr a2, vlenb
-; PRERA-NEXT:    slli a2, a2, 3
-; PRERA-NEXT:    sub sp, sp, a2
-; PRERA-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; PRERA-NEXT:    vsetvli a2, zero, e64, m8, ta, ma
-; PRERA-NEXT:    vmv.v.x v8, a1
-; PRERA-NEXT:    vs8r.v v8, (a0)
-; PRERA-NEXT:    vl8re64.v v16, (a0)
-; PRERA-NEXT:    addi a2, sp, 16
-; PRERA-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
-; PRERA-NEXT:    vl8re64.v v24, (a0)
-; PRERA-NEXT:    vl8re64.v v0, (a0)
-; PRERA-NEXT:    vl8re64.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v0, (a0)
-; PRERA-NEXT:    vs8r.v v24, (a0)
-; PRERA-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
-; PRERA-NEXT:    vs8r.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v8, (a0)
-; PRERA-NEXT:    sd a1, 0(a0)
-; PRERA-NEXT:    csrr a0, vlenb
-; PRERA-NEXT:    slli a0, a0, 3
-; PRERA-NEXT:    add sp, sp, a0
-; PRERA-NEXT:    addi sp, sp, 16
-; PRERA-NEXT:    ret
+; CHECK-LABEL: vmv.v.x_live:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a2, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vmv.v.x v8, a1
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vl8re64.v v16, (a0)
+; CHECK-NEXT:    vl8re64.v v24, (a0)
+; CHECK-NEXT:    vl8re64.v v0, (a0)
+; CHECK-NEXT:    vl8re64.v v8, (a0)
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vs8r.v v0, (a0)
+; CHECK-NEXT:    vs8r.v v24, (a0)
+; CHECK-NEXT:    vs8r.v v16, (a0)
+; CHECK-NEXT:    vmv.v.x v8, a1
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    sd a1, 0(a0)
+; CHECK-NEXT:    ret
   %vmv.v.x = call <vscale x 8 x i64> @llvm.riscv.vmv.v.x.nxv8i64(<vscale x 8 x i64> poison, i64 %x, i64 -1)
   store volatile <vscale x 8 x i64> %vmv.v.x, ptr %p
 
@@ -314,53 +194,23 @@ define void @vmv.v.x_live(ptr %p, i64 %x) {
 }
 
 define void @vfmv.v.f(ptr %p, double %x) {
-; POSTRA-LABEL: vfmv.v.f:
-; POSTRA:       # %bb.0:
-; POSTRA-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; POSTRA-NEXT:    vfmv.v.f v8, fa0
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    vl8re64.v v16, (a0)
-; POSTRA-NEXT:    vl8re64.v v24, (a0)
-; POSTRA-NEXT:    vl8re64.v v0, (a0)
-; POSTRA-NEXT:    vl8re64.v v8, (a0)
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    vs8r.v v0, (a0)
-; POSTRA-NEXT:    vs8r.v v24, (a0)
-; POSTRA-NEXT:    vs8r.v v16, (a0)
-; POSTRA-NEXT:    vfmv.v.f v8, fa0
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    fsd fa0, 0(a0)
-; POSTRA-NEXT:    ret
-;
-; PRERA-LABEL: vfmv.v.f:
-; PRERA:       # %bb.0:
-; PRERA-NEXT:    addi sp, sp, -16
-; PRERA-NEXT:    .cfi_def_cfa_offset 16
-; PRERA-NEXT:    csrr a1, vlenb
-; PRERA-NEXT:    slli a1, a1, 3
-; PRERA-NEXT:    sub sp, sp, a1
-; PRERA-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; PRERA-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; PRERA-NEXT:    vfmv.v.f v8, fa0
-; PRERA-NEXT:    vs8r.v v8, (a0)
-; PRERA-NEXT:    vl8re64.v v16, (a0)
-; PRERA-NEXT:    addi a1, sp, 16
-; PRERA-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; PRERA-NEXT:    vl8re64.v v24, (a0)
-; PRERA-NEXT:    vl8re64.v v0, (a0)
-; PRERA-NEXT:    vl8re64.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v0, (a0)
-; PRERA-NEXT:    vs8r.v v24, (a0)
-; PRERA-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; PRERA-NEXT:    vs8r.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v8, (a0)
-; PRERA-NEXT:    fsd fa0, 0(a0)
-; PRERA-NEXT:    csrr a0, vlenb
-; PRERA-NEXT:    slli a0, a0, 3
-; PRERA-NEXT:    add sp, sp, a0
-; PRERA-NEXT:    addi sp, sp, 16
-; PRERA-NEXT:    ret
+; CHECK-LABEL: vfmv.v.f:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vfmv.v.f v8, fa0
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vl8re64.v v16, (a0)
+; CHECK-NEXT:    vl8re64.v v24, (a0)
+; CHECK-NEXT:    vl8re64.v v0, (a0)
+; CHECK-NEXT:    vl8re64.v v8, (a0)
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vs8r.v v0, (a0)
+; CHECK-NEXT:    vs8r.v v24, (a0)
+; CHECK-NEXT:    vs8r.v v16, (a0)
+; CHECK-NEXT:    vfmv.v.f v8, fa0
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    fsd fa0, 0(a0)
+; CHECK-NEXT:    ret
   %vfmv.v.f = call <vscale x 8 x double> @llvm.riscv.vfmv.v.f.nxv8f64(<vscale x 8 x double> poison, double %x, i64 -1)
   store volatile <vscale x 8 x double> %vfmv.v.f, ptr %p
 
@@ -379,53 +229,23 @@ define void @vfmv.v.f(ptr %p, double %x) {
 }
 
 define void @vmv.s.x(ptr %p, i64 %x) {
-; POSTRA-LABEL: vmv.s.x:
-; POSTRA:       # %bb.0:
-; POSTRA-NEXT:    vsetvli a2, zero, e64, m1, ta, ma
-; POSTRA-NEXT:    vmv.s.x v8, a1
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    vl8re64.v v16, (a0)
-; POSTRA-NEXT:    vl8re64.v v24, (a0)
-; POSTRA-NEXT:    vl8re64.v v0, (a0)
-; POSTRA-NEXT:    vl8re64.v v8, (a0)
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    vs8r.v v0, (a0)
-; POSTRA-NEXT:    vs8r.v v24, (a0)
-; POSTRA-NEXT:    vs8r.v v16, (a0)
-; POSTRA-NEXT:    vmv.s.x v8, a1
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    sd a1, 0(a0)
-; POSTRA-NEXT:    ret
-;
-; PRERA-LABEL: vmv.s.x:
-; PRERA:       # %bb.0:
-; PRERA-NEXT:    addi sp, sp, -16
-; PRERA-NEXT:    .cfi_def_cfa_offset 16
-; PRERA-NEXT:    csrr a2, vlenb
-; PRERA-NEXT:    slli a2, a2, 3
-; PRERA-NEXT:    sub sp, sp, a2
-; PRERA-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; PRERA-NEXT:    vsetvli a2, zero, e64, m1, ta, ma
-; PRERA-NEXT:    vmv.s.x v8, a1
-; PRERA-NEXT:    vs8r.v v8, (a0)
-; PRERA-NEXT:    vl8re64.v v16, (a0)
-; PRERA-NEXT:    addi a2, sp, 16
-; PRERA-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
-; PRERA-NEXT:    vl8re64.v v24, (a0)
-; PRERA-NEXT:    vl8re64.v v0, (a0)
-; PRERA-NEXT:    vl8re64.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v0, (a0)
-; PRERA-NEXT:    vs8r.v v24, (a0)
-; PRERA-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
-; PRERA-NEXT:    vs8r.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v8, (a0)
-; PRERA-NEXT:    sd a1, 0(a0)
-; PRERA-NEXT:    csrr a0, vlenb
-; PRERA-NEXT:    slli a0, a0, 3
-; PRERA-NEXT:    add sp, sp, a0
-; PRERA-NEXT:    addi sp, sp, 16
-; PRERA-NEXT:    ret
+; CHECK-LABEL: vmv.s.x:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a2, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vmv.s.x v8, a1
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vl8re64.v v16, (a0)
+; CHECK-NEXT:    vl8re64.v v24, (a0)
+; CHECK-NEXT:    vl8re64.v v0, (a0)
+; CHECK-NEXT:    vl8re64.v v8, (a0)
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vs8r.v v0, (a0)
+; CHECK-NEXT:    vs8r.v v24, (a0)
+; CHECK-NEXT:    vs8r.v v16, (a0)
+; CHECK-NEXT:    vmv.s.x v8, a1
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    sd a1, 0(a0)
+; CHECK-NEXT:    ret
   %vmv.s.x = call <vscale x 8 x i64> @llvm.riscv.vmv.s.x.nxv8i64(<vscale x 8 x i64> poison, i64 %x, i64 -1)
   store volatile <vscale x 8 x i64> %vmv.s.x, ptr %p
 
@@ -444,53 +264,23 @@ define void @vmv.s.x(ptr %p, i64 %x) {
 }
 
 define void @vfmv.s.f(ptr %p, double %x) {
-; POSTRA-LABEL: vfmv.s.f:
-; POSTRA:       # %bb.0:
-; POSTRA-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; POSTRA-NEXT:    vfmv.s.f v8, fa0
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    vl8re64.v v16, (a0)
-; POSTRA-NEXT:    vl8re64.v v24, (a0)
-; POSTRA-NEXT:    vl8re64.v v0, (a0)
-; POSTRA-NEXT:    vl8re64.v v8, (a0)
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    vs8r.v v0, (a0)
-; POSTRA-NEXT:    vs8r.v v24, (a0)
-; POSTRA-NEXT:    vs8r.v v16, (a0)
-; POSTRA-NEXT:    vfmv.s.f v8, fa0
-; POSTRA-NEXT:    vs8r.v v8, (a0)
-; POSTRA-NEXT:    fsd fa0, 0(a0)
-; POSTRA-NEXT:    ret
-;
-; PRERA-LABEL: vfmv.s.f:
-; PRERA:       # %bb.0:
-; PRERA-NEXT:    addi sp, sp, -16
-; PRERA-NEXT:    .cfi_def_cfa_offset 16
-; PRERA-NEXT:    csrr a1, vlenb
-; PRERA-NEXT:    slli a1, a1, 3
-; PRERA-NEXT:    sub sp, sp, a1
-; PRERA-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; PRERA-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; PRERA-NEXT:    vfmv.s.f v8, fa0
-; PRERA-NEXT:    vs8r.v v8, (a0)
-; PRERA-NEXT:    vl8re64.v v16, (a0)
-; PRERA-NEXT:    addi a1, sp, 16
-; PRERA-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; PRERA-NEXT:    vl8re64.v v24, (a0)
-; PRERA-NEXT:    vl8re64.v v0, (a0)
-; PRERA-NEXT:    vl8re64.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v0, (a0)
-; PRERA-NEXT:    vs8r.v v24, (a0)
-; PRERA-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; PRERA-NEXT:    vs8r.v v16, (a0)
-; PRERA-NEXT:    vs8r.v v8, (a0)
-; PRERA-NEXT:    fsd fa0, 0(a0)
-; PRERA-NEXT:    csrr a0, vlenb
-; PRERA-NEXT:    slli a0, a0, 3
-; PRERA-NEXT:    add sp, sp, a0
-; PRERA-NEXT:    addi sp, sp, 16
-; PRERA-NEXT:    ret
+; CHECK-LABEL: vfmv.s.f:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vl8re64.v v16, (a0)
+; CHECK-NEXT:    vl8re64.v v24, (a0)
+; CHECK-NEXT:    vl8re64.v v0, (a0)
+; CHECK-NEXT:    vl8re64.v v8, (a0)
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    vs8r.v v0, (a0)
+; CHECK-NEXT:    vs8r.v v24, (a0)
+; CHECK-NEXT:    vs8r.v v16, (a0)
+; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    fsd fa0, 0(a0)
+; CHECK-NEXT:    ret
   %vfmv.s.f = call <vscale x 8 x double> @llvm.riscv.vfmv.s.f.nxv8f64(<vscale x 8 x double> poison, double %x, i64 -1)
   store volatile <vscale x 8 x double> %vfmv.s.f, ptr %p
 
-- 
GitLab


From d6d4569dd90261bc8cdb3d1c3c2204cde1fe566f Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 28 Oct 2024 11:45:31 +0000
Subject: [PATCH 304/693] Fix MSVC "signed/unsigned mismatch" warnings. NFC.

---
 clang/lib/CodeGen/CGBuiltin.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 911eec48bcb2..a57c95d5b966 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -19055,7 +19055,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
       Args.push_back(llvm::PoisonValue::get(IntTy));
     for (unsigned I = 0; I != E->getNumArgs(); ++I) {
       llvm::Value *V = EmitScalarOrConstFoldImmArg(ICEArguments, I, E);
-      if (I <= !InsertOld && Size < 32) {
+      if (I <= (InsertOld ? 0u : 1u) && Size < 32) {
         if (!DataTy->isIntegerTy())
           V = Builder.CreateBitCast(
               V, llvm::IntegerType::get(Builder.getContext(), Size));
-- 
GitLab


From 056cf936a730a72db7ad9da8b475814c6bbe03b7 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 28 Oct 2024 11:52:44 +0000
Subject: [PATCH 305/693] [DAG] Fold (and X, (bswap/bitreverse (not Y))) ->
 (and X, (not (bswap/bitreverse Y))) (#112547)

On ANDNOT capable targets we can always do this profitably, without ANDNOT we only attempt this if we don't introduce an additional NOT

Fixes #112425
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  11 +
 llvm/test/CodeGen/X86/andnot-patterns.ll      | 683 +++++++++++++-----
 2 files changed, 505 insertions(+), 189 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index ad2d2ede302a..b800204d9175 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -7353,6 +7353,17 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
   if (SDValue R = foldLogicOfShifts(N, N1, N0, DAG))
     return R;
 
+  // Fold (and X, (bswap (not Y))) -> (and X, (not (bswap Y)))
+  // Fold (and X, (bitreverse (not Y))) -> (and X, (not (bitreverse Y)))
+  SDValue X, Y, NotY;
+  for (unsigned Opc : {ISD::BSWAP, ISD::BITREVERSE})
+    if (sd_match(N,
+                 m_And(m_Value(X), m_OneUse(m_UnaryOp(Opc, m_Value(NotY))))) &&
+        sd_match(NotY, m_Not(m_Value(Y))) &&
+        (TLI.hasAndNot(SDValue(N, 0)) || NotY->hasOneUse()))
+      return DAG.getNode(ISD::AND, DL, VT, X,
+                         DAG.getNOT(DL, DAG.getNode(Opc, DL, VT, Y), VT));
+
   // Masking the negated extension of a boolean is just the zero-extended
   // boolean:
   // and (sub 0, zext(bool X)), 1 --> zext(bool X)
diff --git a/llvm/test/CodeGen/X86/andnot-patterns.ll b/llvm/test/CodeGen/X86/andnot-patterns.ll
index 101e4ed008f7..1df29f0b12d1 100644
--- a/llvm/test/CodeGen/X86/andnot-patterns.ll
+++ b/llvm/test/CodeGen/X86/andnot-patterns.ll
@@ -7,6 +7,7 @@
 ; TODO - PR112425 - attempt to reconstruct andnot patterns through bitwise-agnostic operations
 
 declare void @use_i64(i64)
+declare void @use_i32(i32)
 
 ;
 ; Fold (and X, (rotl (not Y), Z))) -> (and X, (not (rotl Y, Z)))
@@ -132,8 +133,8 @@ define i8 @andnot_rotl_i8(i8 %a0, i8 %a1, i8 %a2) nounwind {
   ret i8 %and
 }
 
-define i64 @andnot_rotl_i64_multiuse(i64 %a0, i64 %a1, i64 %a2) nounwind {
-; X86-LABEL: andnot_rotl_i64_multiuse:
+define i64 @andnot_rotl_i64_multiuse_rot(i64 %a0, i64 %a1, i64 %a2) nounwind {
+; X86-LABEL: andnot_rotl_i64_multiuse_rot:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
@@ -171,7 +172,7 @@ define i64 @andnot_rotl_i64_multiuse(i64 %a0, i64 %a1, i64 %a2) nounwind {
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: andnot_rotl_i64_multiuse:
+; X64-LABEL: andnot_rotl_i64_multiuse_rot:
 ; X64:       # %bb.0:
 ; X64-NEXT:    pushq %rbx
 ; X64-NEXT:    movq %rdx, %rcx
@@ -316,30 +317,84 @@ define i8 @andnot_rotr_i8(i8 %a0, i8 %a1, i8 %a2) nounwind {
   ret i8 %and
 }
 
-;
-; Fold (and X, (bswap (not Y)))) -> (and X, (not (bswap Y)))
-;
-
-define i64 @andnot_bswap_i64(i64 %a0, i64 %a1) nounwind {
-; X86-LABEL: andnot_bswap_i64:
+define i32 @andnot_rotr_i32_multiuse_not(i32 %a0, i32 %a1, i32 %a2) nounwind {
+; X86-LABEL: andnot_rotr_i32_multiuse_not:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    notl %eax
-; X86-NEXT:    notl %edx
-; X86-NEXT:    bswapl %edx
-; X86-NEXT:    bswapl %eax
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    rorl %cl, %esi
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll use_i32@PLT
+; X86-NEXT:    addl $4, %esp
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: andnot_bswap_i64:
+; X64-LABEL: andnot_rotr_i32_multiuse_not:
 ; X64:       # %bb.0:
-; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    notq %rax
-; X64-NEXT:    bswapq %rax
-; X64-NEXT:    andq %rdi, %rax
+; X64-NEXT:    pushq %rbx
+; X64-NEXT:    movl %edx, %ecx
+; X64-NEXT:    notl %esi
+; X64-NEXT:    movl %esi, %ebx
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    rorl %cl, %ebx
+; X64-NEXT:    andl %edi, %ebx
+; X64-NEXT:    movl %esi, %edi
+; X64-NEXT:    callq use_i32@PLT
+; X64-NEXT:    movl %ebx, %eax
+; X64-NEXT:    popq %rbx
 ; X64-NEXT:    retq
+  %not = xor i32 %a1, -1
+  %rot = tail call i32 @llvm.fshr.i32(i32 %not, i32 %not, i32 %a2)
+  %and = and i32 %rot, %a0
+  call void @use_i32(i32 %not)
+  ret i32 %and
+}
+
+;
+; Fold (and X, (bswap (not Y)))) -> (and X, (not (bswap Y)))
+;
+
+define i64 @andnot_bswap_i64(i64 %a0, i64 %a1) nounwind {
+; X86-NOBMI-LABEL: andnot_bswap_i64:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    bswapl %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    bswapl %edx
+; X86-NOBMI-NEXT:    notl %edx
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI-LABEL: andnot_bswap_i64:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    bswapl %eax
+; X86-BMI-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
+; X86-BMI-NEXT:    bswapl %ecx
+; X86-BMI-NEXT:    andnl {{[0-9]+}}(%esp), %ecx, %edx
+; X86-BMI-NEXT:    retl
+;
+; X64-NOBMI-LABEL: andnot_bswap_i64:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movq %rsi, %rax
+; X64-NOBMI-NEXT:    bswapq %rax
+; X64-NOBMI-NEXT:    notq %rax
+; X64-NOBMI-NEXT:    andq %rdi, %rax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_bswap_i64:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    bswapq %rsi
+; X64-BMI-NEXT:    andnq %rdi, %rsi, %rax
+; X64-BMI-NEXT:    retq
   %not = xor i64 %a1, -1
   %bswap = tail call i64 @llvm.bswap.i64(i64 %not)
   %and = and i64 %bswap, %a0
@@ -347,21 +402,34 @@ define i64 @andnot_bswap_i64(i64 %a0, i64 %a1) nounwind {
 }
 
 define i32 @andnot_bswap_i32(i32 %a0, i32 %a1) nounwind {
-; X86-LABEL: andnot_bswap_i32:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    notl %eax
-; X86-NEXT:    bswapl %eax
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    retl
+; X86-NOBMI-LABEL: andnot_bswap_i32:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    bswapl %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    retl
 ;
-; X64-LABEL: andnot_bswap_i32:
-; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    notl %eax
-; X64-NEXT:    bswapl %eax
-; X64-NEXT:    andl %edi, %eax
-; X64-NEXT:    retq
+; X86-BMI-LABEL: andnot_bswap_i32:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    bswapl %eax
+; X86-BMI-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
+; X86-BMI-NEXT:    retl
+;
+; X64-NOBMI-LABEL: andnot_bswap_i32:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %eax
+; X64-NOBMI-NEXT:    bswapl %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_bswap_i32:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    bswapl %esi
+; X64-BMI-NEXT:    andnl %edi, %esi, %eax
+; X64-BMI-NEXT:    retq
   %not = xor i32 %a1, -1
   %bswap = tail call i32 @llvm.bswap.i32(i32 %not)
   %and = and i32 %bswap, %a0
@@ -371,101 +439,273 @@ define i32 @andnot_bswap_i32(i32 %a0, i32 %a1) nounwind {
 define i16 @andnot_bswap_i16(i16 %a0, i16 %a1) nounwind {
 ; X86-LABEL: andnot_bswap_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    notl %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    rolw $8, %ax
+; X86-NEXT:    notl %eax
 ; X86-NEXT:    andw {{[0-9]+}}(%esp), %ax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: andnot_bswap_i16:
-; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    notl %eax
-; X64-NEXT:    rolw $8, %ax
-; X64-NEXT:    andl %edi, %eax
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    retq
+; X64-NOBMI-LABEL: andnot_bswap_i16:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %eax
+; X64-NOBMI-NEXT:    rolw $8, %ax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_bswap_i16:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    rolw $8, %si
+; X64-BMI-NEXT:    andnl %edi, %esi, %eax
+; X64-BMI-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-BMI-NEXT:    retq
   %not = xor i16 %a1, -1
   %bswap = tail call i16 @llvm.bswap.i16(i16 %not)
   %and = and i16 %bswap, %a0
   ret i16 %and
 }
 
-;
-; Fold (and X, (bitreverse (not Y)))) -> (and X, (not (bitreverse Y)))
-;
-
-define i64 @andnot_bitreverse_i64(i64 %a0, i64 %a1) nounwind {
-; X86-LABEL: andnot_bitreverse_i64:
+define i32 @andnot_bswap_i32_multiuse_bswap(i32 %a0, i32 %a1) nounwind {
+; X86-LABEL: andnot_bswap_i32_multiuse_bswap:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    notl %eax
-; X86-NEXT:    notl %ecx
-; X86-NEXT:    bswapl %ecx
-; X86-NEXT:    movl %ecx, %edx
-; X86-NEXT:    andl $252645135, %edx # imm = 0xF0F0F0F
-; X86-NEXT:    shll $4, %edx
-; X86-NEXT:    shrl $4, %ecx
-; X86-NEXT:    andl $252645135, %ecx # imm = 0xF0F0F0F
-; X86-NEXT:    orl %edx, %ecx
-; X86-NEXT:    movl %ecx, %edx
-; X86-NEXT:    andl $858993459, %edx # imm = 0x33333333
-; X86-NEXT:    shrl $2, %ecx
-; X86-NEXT:    andl $858993459, %ecx # imm = 0x33333333
-; X86-NEXT:    leal (%ecx,%edx,4), %ecx
-; X86-NEXT:    movl %ecx, %edx
-; X86-NEXT:    andl $1431655765, %edx # imm = 0x55555555
-; X86-NEXT:    shrl %ecx
-; X86-NEXT:    andl $1431655765, %ecx # imm = 0x55555555
-; X86-NEXT:    leal (%ecx,%edx,2), %edx
 ; X86-NEXT:    bswapl %eax
-; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    andl $252645135, %ecx # imm = 0xF0F0F0F
-; X86-NEXT:    shll $4, %ecx
-; X86-NEXT:    shrl $4, %eax
-; X86-NEXT:    andl $252645135, %eax # imm = 0xF0F0F0F
-; X86-NEXT:    orl %ecx, %eax
-; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    andl $858993459, %ecx # imm = 0x33333333
-; X86-NEXT:    shrl $2, %eax
-; X86-NEXT:    andl $858993459, %eax # imm = 0x33333333
-; X86-NEXT:    leal (%eax,%ecx,4), %eax
-; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    andl $1431655765, %ecx # imm = 0x55555555
-; X86-NEXT:    shrl %eax
-; X86-NEXT:    andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT:    leal (%eax,%ecx,2), %eax
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    andl %eax, %esi
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll use_i32@PLT
+; X86-NEXT:    addl $4, %esp
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: andnot_bitreverse_i64:
+; X64-LABEL: andnot_bswap_i32_multiuse_bswap:
 ; X64:       # %bb.0:
-; X64-NEXT:    notq %rsi
-; X64-NEXT:    bswapq %rsi
-; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    shrq $4, %rax
-; X64-NEXT:    movabsq $1085102592571150095, %rcx # imm = 0xF0F0F0F0F0F0F0F
-; X64-NEXT:    andq %rcx, %rax
-; X64-NEXT:    andq %rcx, %rsi
-; X64-NEXT:    shlq $4, %rsi
-; X64-NEXT:    orq %rax, %rsi
-; X64-NEXT:    movabsq $3689348814741910323, %rax # imm = 0x3333333333333333
-; X64-NEXT:    movq %rsi, %rcx
-; X64-NEXT:    andq %rax, %rcx
-; X64-NEXT:    shrq $2, %rsi
-; X64-NEXT:    andq %rax, %rsi
-; X64-NEXT:    leaq (%rsi,%rcx,4), %rax
-; X64-NEXT:    movabsq $6148914691236517205, %rcx # imm = 0x5555555555555555
-; X64-NEXT:    movq %rax, %rdx
-; X64-NEXT:    andq %rcx, %rdx
-; X64-NEXT:    shrq %rax
-; X64-NEXT:    andq %rcx, %rax
-; X64-NEXT:    leaq (%rax,%rdx,2), %rax
-; X64-NEXT:    andq %rdi, %rax
+; X64-NEXT:    pushq %rbx
+; X64-NEXT:    movl %edi, %ebx
+; X64-NEXT:    notl %esi
+; X64-NEXT:    bswapl %esi
+; X64-NEXT:    andl %esi, %ebx
+; X64-NEXT:    movl %esi, %edi
+; X64-NEXT:    callq use_i32@PLT
+; X64-NEXT:    movl %ebx, %eax
+; X64-NEXT:    popq %rbx
 ; X64-NEXT:    retq
+  %not = xor i32 %a1, -1
+  %bswap = tail call i32 @llvm.bswap.i32(i32 %not)
+  %and = and i32 %bswap, %a0
+  call void @use_i32(i32 %bswap)
+  ret i32 %and
+}
+
+define i32 @andnot_bswap_i32_multiuse_not(i32 %a0, i32 %a1) nounwind {
+; X86-NOBMI-LABEL: andnot_bswap_i32_multiuse_not:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    movl %eax, %esi
+; X86-NOBMI-NEXT:    bswapl %esi
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    pushl %eax
+; X86-NOBMI-NEXT:    calll use_i32@PLT
+; X86-NOBMI-NEXT:    addl $4, %esp
+; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI-LABEL: andnot_bswap_i32_multiuse_not:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    pushl %esi
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    movl %eax, %ecx
+; X86-BMI-NEXT:    notl %ecx
+; X86-BMI-NEXT:    bswapl %eax
+; X86-BMI-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %esi
+; X86-BMI-NEXT:    pushl %ecx
+; X86-BMI-NEXT:    calll use_i32@PLT
+; X86-BMI-NEXT:    addl $4, %esp
+; X86-BMI-NEXT:    movl %esi, %eax
+; X86-BMI-NEXT:    popl %esi
+; X86-BMI-NEXT:    retl
+;
+; X64-NOBMI-LABEL: andnot_bswap_i32_multiuse_not:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    pushq %rbx
+; X64-NOBMI-NEXT:    notl %esi
+; X64-NOBMI-NEXT:    movl %esi, %ebx
+; X64-NOBMI-NEXT:    bswapl %ebx
+; X64-NOBMI-NEXT:    andl %edi, %ebx
+; X64-NOBMI-NEXT:    movl %esi, %edi
+; X64-NOBMI-NEXT:    callq use_i32@PLT
+; X64-NOBMI-NEXT:    movl %ebx, %eax
+; X64-NOBMI-NEXT:    popq %rbx
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_bswap_i32_multiuse_not:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    pushq %rbx
+; X64-BMI-NEXT:    movl %esi, %eax
+; X64-BMI-NEXT:    notl %eax
+; X64-BMI-NEXT:    bswapl %esi
+; X64-BMI-NEXT:    andnl %edi, %esi, %ebx
+; X64-BMI-NEXT:    movl %eax, %edi
+; X64-BMI-NEXT:    callq use_i32@PLT
+; X64-BMI-NEXT:    movl %ebx, %eax
+; X64-BMI-NEXT:    popq %rbx
+; X64-BMI-NEXT:    retq
+  %not = xor i32 %a1, -1
+  %bswap = tail call i32 @llvm.bswap.i32(i32 %not)
+  %and = and i32 %bswap, %a0
+  call void @use_i32(i32 %not)
+  ret i32 %and
+}
+
+;
+; Fold (and X, (bitreverse (not Y)))) -> (and X, (not (bitreverse Y)))
+;
+
+define i64 @andnot_bitreverse_i64(i64 %a0, i64 %a1) nounwind {
+; X86-NOBMI-LABEL: andnot_bitreverse_i64:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    bswapl %eax
+; X86-NOBMI-NEXT:    movl %eax, %edx
+; X86-NOBMI-NEXT:    andl $252645135, %edx # imm = 0xF0F0F0F
+; X86-NOBMI-NEXT:    shll $4, %edx
+; X86-NOBMI-NEXT:    shrl $4, %eax
+; X86-NOBMI-NEXT:    andl $252645135, %eax # imm = 0xF0F0F0F
+; X86-NOBMI-NEXT:    orl %edx, %eax
+; X86-NOBMI-NEXT:    movl %eax, %edx
+; X86-NOBMI-NEXT:    andl $858993459, %edx # imm = 0x33333333
+; X86-NOBMI-NEXT:    shrl $2, %eax
+; X86-NOBMI-NEXT:    andl $858993459, %eax # imm = 0x33333333
+; X86-NOBMI-NEXT:    leal (%eax,%edx,4), %eax
+; X86-NOBMI-NEXT:    movl %eax, %edx
+; X86-NOBMI-NEXT:    andl $1431655765, %edx # imm = 0x55555555
+; X86-NOBMI-NEXT:    shrl %eax
+; X86-NOBMI-NEXT:    andl $1431655765, %eax # imm = 0x55555555
+; X86-NOBMI-NEXT:    leal (%eax,%edx,2), %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    bswapl %ecx
+; X86-NOBMI-NEXT:    movl %ecx, %edx
+; X86-NOBMI-NEXT:    andl $252645135, %edx # imm = 0xF0F0F0F
+; X86-NOBMI-NEXT:    shll $4, %edx
+; X86-NOBMI-NEXT:    shrl $4, %ecx
+; X86-NOBMI-NEXT:    andl $252645135, %ecx # imm = 0xF0F0F0F
+; X86-NOBMI-NEXT:    orl %edx, %ecx
+; X86-NOBMI-NEXT:    movl %ecx, %edx
+; X86-NOBMI-NEXT:    andl $858993459, %edx # imm = 0x33333333
+; X86-NOBMI-NEXT:    shrl $2, %ecx
+; X86-NOBMI-NEXT:    andl $858993459, %ecx # imm = 0x33333333
+; X86-NOBMI-NEXT:    leal (%ecx,%edx,4), %ecx
+; X86-NOBMI-NEXT:    movl %ecx, %edx
+; X86-NOBMI-NEXT:    andl $1431655765, %edx # imm = 0x55555555
+; X86-NOBMI-NEXT:    shrl %ecx
+; X86-NOBMI-NEXT:    andl $1431655765, %ecx # imm = 0x55555555
+; X86-NOBMI-NEXT:    leal (%ecx,%edx,2), %edx
+; X86-NOBMI-NEXT:    notl %edx
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI-LABEL: andnot_bitreverse_i64:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    bswapl %eax
+; X86-BMI-NEXT:    movl %eax, %edx
+; X86-BMI-NEXT:    andl $252645135, %edx # imm = 0xF0F0F0F
+; X86-BMI-NEXT:    shll $4, %edx
+; X86-BMI-NEXT:    shrl $4, %eax
+; X86-BMI-NEXT:    andl $252645135, %eax # imm = 0xF0F0F0F
+; X86-BMI-NEXT:    orl %edx, %eax
+; X86-BMI-NEXT:    movl %eax, %edx
+; X86-BMI-NEXT:    andl $858993459, %edx # imm = 0x33333333
+; X86-BMI-NEXT:    shrl $2, %eax
+; X86-BMI-NEXT:    andl $858993459, %eax # imm = 0x33333333
+; X86-BMI-NEXT:    leal (%eax,%edx,4), %eax
+; X86-BMI-NEXT:    movl %eax, %edx
+; X86-BMI-NEXT:    andl $1431655765, %edx # imm = 0x55555555
+; X86-BMI-NEXT:    shrl %eax
+; X86-BMI-NEXT:    andl $1431655765, %eax # imm = 0x55555555
+; X86-BMI-NEXT:    leal (%eax,%edx,2), %eax
+; X86-BMI-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
+; X86-BMI-NEXT:    bswapl %ecx
+; X86-BMI-NEXT:    movl %ecx, %edx
+; X86-BMI-NEXT:    andl $252645135, %edx # imm = 0xF0F0F0F
+; X86-BMI-NEXT:    shll $4, %edx
+; X86-BMI-NEXT:    shrl $4, %ecx
+; X86-BMI-NEXT:    andl $252645135, %ecx # imm = 0xF0F0F0F
+; X86-BMI-NEXT:    orl %edx, %ecx
+; X86-BMI-NEXT:    movl %ecx, %edx
+; X86-BMI-NEXT:    andl $858993459, %edx # imm = 0x33333333
+; X86-BMI-NEXT:    shrl $2, %ecx
+; X86-BMI-NEXT:    andl $858993459, %ecx # imm = 0x33333333
+; X86-BMI-NEXT:    leal (%ecx,%edx,4), %ecx
+; X86-BMI-NEXT:    movl %ecx, %edx
+; X86-BMI-NEXT:    andl $1431655765, %edx # imm = 0x55555555
+; X86-BMI-NEXT:    shrl %ecx
+; X86-BMI-NEXT:    andl $1431655765, %ecx # imm = 0x55555555
+; X86-BMI-NEXT:    leal (%ecx,%edx,2), %ecx
+; X86-BMI-NEXT:    andnl {{[0-9]+}}(%esp), %ecx, %edx
+; X86-BMI-NEXT:    retl
+;
+; X64-NOBMI-LABEL: andnot_bitreverse_i64:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    bswapq %rsi
+; X64-NOBMI-NEXT:    movq %rsi, %rax
+; X64-NOBMI-NEXT:    shrq $4, %rax
+; X64-NOBMI-NEXT:    movabsq $1085102592571150095, %rcx # imm = 0xF0F0F0F0F0F0F0F
+; X64-NOBMI-NEXT:    andq %rcx, %rax
+; X64-NOBMI-NEXT:    andq %rcx, %rsi
+; X64-NOBMI-NEXT:    shlq $4, %rsi
+; X64-NOBMI-NEXT:    orq %rax, %rsi
+; X64-NOBMI-NEXT:    movabsq $3689348814741910323, %rax # imm = 0x3333333333333333
+; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    andq %rax, %rcx
+; X64-NOBMI-NEXT:    shrq $2, %rsi
+; X64-NOBMI-NEXT:    andq %rax, %rsi
+; X64-NOBMI-NEXT:    leaq (%rsi,%rcx,4), %rax
+; X64-NOBMI-NEXT:    movabsq $6148914691236517205, %rcx # imm = 0x5555555555555555
+; X64-NOBMI-NEXT:    movq %rax, %rdx
+; X64-NOBMI-NEXT:    andq %rcx, %rdx
+; X64-NOBMI-NEXT:    shrq %rax
+; X64-NOBMI-NEXT:    andq %rcx, %rax
+; X64-NOBMI-NEXT:    leaq (%rax,%rdx,2), %rax
+; X64-NOBMI-NEXT:    notq %rax
+; X64-NOBMI-NEXT:    andq %rdi, %rax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_bitreverse_i64:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    bswapq %rsi
+; X64-BMI-NEXT:    movq %rsi, %rax
+; X64-BMI-NEXT:    shrq $4, %rax
+; X64-BMI-NEXT:    movabsq $1085102592571150095, %rcx # imm = 0xF0F0F0F0F0F0F0F
+; X64-BMI-NEXT:    andq %rcx, %rax
+; X64-BMI-NEXT:    andq %rcx, %rsi
+; X64-BMI-NEXT:    shlq $4, %rsi
+; X64-BMI-NEXT:    orq %rax, %rsi
+; X64-BMI-NEXT:    movabsq $3689348814741910323, %rax # imm = 0x3333333333333333
+; X64-BMI-NEXT:    movq %rsi, %rcx
+; X64-BMI-NEXT:    andq %rax, %rcx
+; X64-BMI-NEXT:    shrq $2, %rsi
+; X64-BMI-NEXT:    andq %rax, %rsi
+; X64-BMI-NEXT:    leaq (%rsi,%rcx,4), %rax
+; X64-BMI-NEXT:    movabsq $6148914691236517205, %rcx # imm = 0x5555555555555555
+; X64-BMI-NEXT:    movq %rax, %rdx
+; X64-BMI-NEXT:    andq %rcx, %rdx
+; X64-BMI-NEXT:    shrq %rax
+; X64-BMI-NEXT:    andq %rcx, %rax
+; X64-BMI-NEXT:    leaq (%rax,%rdx,2), %rax
+; X64-BMI-NEXT:    andnq %rdi, %rax, %rax
+; X64-BMI-NEXT:    retq
   %not = xor i64 %a1, -1
   %bitrev = tail call i64 @llvm.bitreverse.i64(i64 %not)
   %and = and i64 %bitrev, %a0
@@ -473,53 +713,99 @@ define i64 @andnot_bitreverse_i64(i64 %a0, i64 %a1) nounwind {
 }
 
 define i32 @andnot_bitreverse_i32(i32 %a0, i32 %a1) nounwind {
-; X86-LABEL: andnot_bitreverse_i32:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    notl %eax
-; X86-NEXT:    bswapl %eax
-; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    andl $252645135, %ecx # imm = 0xF0F0F0F
-; X86-NEXT:    shll $4, %ecx
-; X86-NEXT:    shrl $4, %eax
-; X86-NEXT:    andl $252645135, %eax # imm = 0xF0F0F0F
-; X86-NEXT:    orl %ecx, %eax
-; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    andl $858993459, %ecx # imm = 0x33333333
-; X86-NEXT:    shrl $2, %eax
-; X86-NEXT:    andl $858993459, %eax # imm = 0x33333333
-; X86-NEXT:    leal (%eax,%ecx,4), %eax
-; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    andl $1431655765, %ecx # imm = 0x55555555
-; X86-NEXT:    shrl %eax
-; X86-NEXT:    andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT:    leal (%eax,%ecx,2), %eax
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    retl
+; X86-NOBMI-LABEL: andnot_bitreverse_i32:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    bswapl %eax
+; X86-NOBMI-NEXT:    movl %eax, %ecx
+; X86-NOBMI-NEXT:    andl $252645135, %ecx # imm = 0xF0F0F0F
+; X86-NOBMI-NEXT:    shll $4, %ecx
+; X86-NOBMI-NEXT:    shrl $4, %eax
+; X86-NOBMI-NEXT:    andl $252645135, %eax # imm = 0xF0F0F0F
+; X86-NOBMI-NEXT:    orl %ecx, %eax
+; X86-NOBMI-NEXT:    movl %eax, %ecx
+; X86-NOBMI-NEXT:    andl $858993459, %ecx # imm = 0x33333333
+; X86-NOBMI-NEXT:    shrl $2, %eax
+; X86-NOBMI-NEXT:    andl $858993459, %eax # imm = 0x33333333
+; X86-NOBMI-NEXT:    leal (%eax,%ecx,4), %eax
+; X86-NOBMI-NEXT:    movl %eax, %ecx
+; X86-NOBMI-NEXT:    andl $1431655765, %ecx # imm = 0x55555555
+; X86-NOBMI-NEXT:    shrl %eax
+; X86-NOBMI-NEXT:    andl $1431655765, %eax # imm = 0x55555555
+; X86-NOBMI-NEXT:    leal (%eax,%ecx,2), %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    retl
 ;
-; X64-LABEL: andnot_bitreverse_i32:
-; X64:       # %bb.0:
-; X64-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-NEXT:    notl %esi
-; X64-NEXT:    bswapl %esi
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    andl $252645135, %eax # imm = 0xF0F0F0F
-; X64-NEXT:    shll $4, %eax
-; X64-NEXT:    shrl $4, %esi
-; X64-NEXT:    andl $252645135, %esi # imm = 0xF0F0F0F
-; X64-NEXT:    orl %eax, %esi
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    andl $858993459, %eax # imm = 0x33333333
-; X64-NEXT:    shrl $2, %esi
-; X64-NEXT:    andl $858993459, %esi # imm = 0x33333333
-; X64-NEXT:    leal (%rsi,%rax,4), %eax
-; X64-NEXT:    movl %eax, %ecx
-; X64-NEXT:    andl $1431655765, %ecx # imm = 0x55555555
-; X64-NEXT:    shrl %eax
-; X64-NEXT:    andl $1431655765, %eax # imm = 0x55555555
-; X64-NEXT:    leal (%rax,%rcx,2), %eax
-; X64-NEXT:    andl %edi, %eax
-; X64-NEXT:    retq
+; X86-BMI-LABEL: andnot_bitreverse_i32:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    bswapl %eax
+; X86-BMI-NEXT:    movl %eax, %ecx
+; X86-BMI-NEXT:    andl $252645135, %ecx # imm = 0xF0F0F0F
+; X86-BMI-NEXT:    shll $4, %ecx
+; X86-BMI-NEXT:    shrl $4, %eax
+; X86-BMI-NEXT:    andl $252645135, %eax # imm = 0xF0F0F0F
+; X86-BMI-NEXT:    orl %ecx, %eax
+; X86-BMI-NEXT:    movl %eax, %ecx
+; X86-BMI-NEXT:    andl $858993459, %ecx # imm = 0x33333333
+; X86-BMI-NEXT:    shrl $2, %eax
+; X86-BMI-NEXT:    andl $858993459, %eax # imm = 0x33333333
+; X86-BMI-NEXT:    leal (%eax,%ecx,4), %eax
+; X86-BMI-NEXT:    movl %eax, %ecx
+; X86-BMI-NEXT:    andl $1431655765, %ecx # imm = 0x55555555
+; X86-BMI-NEXT:    shrl %eax
+; X86-BMI-NEXT:    andl $1431655765, %eax # imm = 0x55555555
+; X86-BMI-NEXT:    leal (%eax,%ecx,2), %eax
+; X86-BMI-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
+; X86-BMI-NEXT:    retl
+;
+; X64-NOBMI-LABEL: andnot_bitreverse_i32:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-NOBMI-NEXT:    bswapl %esi
+; X64-NOBMI-NEXT:    movl %esi, %eax
+; X64-NOBMI-NEXT:    andl $252645135, %eax # imm = 0xF0F0F0F
+; X64-NOBMI-NEXT:    shll $4, %eax
+; X64-NOBMI-NEXT:    shrl $4, %esi
+; X64-NOBMI-NEXT:    andl $252645135, %esi # imm = 0xF0F0F0F
+; X64-NOBMI-NEXT:    orl %eax, %esi
+; X64-NOBMI-NEXT:    movl %esi, %eax
+; X64-NOBMI-NEXT:    andl $858993459, %eax # imm = 0x33333333
+; X64-NOBMI-NEXT:    shrl $2, %esi
+; X64-NOBMI-NEXT:    andl $858993459, %esi # imm = 0x33333333
+; X64-NOBMI-NEXT:    leal (%rsi,%rax,4), %eax
+; X64-NOBMI-NEXT:    movl %eax, %ecx
+; X64-NOBMI-NEXT:    andl $1431655765, %ecx # imm = 0x55555555
+; X64-NOBMI-NEXT:    shrl %eax
+; X64-NOBMI-NEXT:    andl $1431655765, %eax # imm = 0x55555555
+; X64-NOBMI-NEXT:    leal (%rax,%rcx,2), %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_bitreverse_i32:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-BMI-NEXT:    bswapl %esi
+; X64-BMI-NEXT:    movl %esi, %eax
+; X64-BMI-NEXT:    andl $252645135, %eax # imm = 0xF0F0F0F
+; X64-BMI-NEXT:    shll $4, %eax
+; X64-BMI-NEXT:    shrl $4, %esi
+; X64-BMI-NEXT:    andl $252645135, %esi # imm = 0xF0F0F0F
+; X64-BMI-NEXT:    orl %eax, %esi
+; X64-BMI-NEXT:    movl %esi, %eax
+; X64-BMI-NEXT:    andl $858993459, %eax # imm = 0x33333333
+; X64-BMI-NEXT:    shrl $2, %esi
+; X64-BMI-NEXT:    andl $858993459, %esi # imm = 0x33333333
+; X64-BMI-NEXT:    leal (%rsi,%rax,4), %eax
+; X64-BMI-NEXT:    movl %eax, %ecx
+; X64-BMI-NEXT:    andl $1431655765, %ecx # imm = 0x55555555
+; X64-BMI-NEXT:    shrl %eax
+; X64-BMI-NEXT:    andl $1431655765, %eax # imm = 0x55555555
+; X64-BMI-NEXT:    leal (%rax,%rcx,2), %eax
+; X64-BMI-NEXT:    andnl %edi, %eax, %eax
+; X64-BMI-NEXT:    retq
   %not = xor i32 %a1, -1
   %bitrev = tail call i32 @llvm.bitreverse.i32(i32 %not)
   %and = and i32 %bitrev, %a0
@@ -529,8 +815,7 @@ define i32 @andnot_bitreverse_i32(i32 %a0, i32 %a1) nounwind {
 define i16 @andnot_bitreverse_i16(i16 %a0, i16 %a1) nounwind {
 ; X86-LABEL: andnot_bitreverse_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    notl %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    rolw $8, %ax
 ; X86-NEXT:    movl %eax, %ecx
 ; X86-NEXT:    andl $3855, %ecx # imm = 0xF0F
@@ -548,34 +833,59 @@ define i16 @andnot_bitreverse_i16(i16 %a0, i16 %a1) nounwind {
 ; X86-NEXT:    shrl %eax
 ; X86-NEXT:    andl $21845, %eax # imm = 0x5555
 ; X86-NEXT:    leal (%eax,%ecx,2), %eax
+; X86-NEXT:    notl %eax
 ; X86-NEXT:    andw {{[0-9]+}}(%esp), %ax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: andnot_bitreverse_i16:
-; X64:       # %bb.0:
-; X64-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-NEXT:    notl %esi
-; X64-NEXT:    rolw $8, %si
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    andl $3855, %eax # imm = 0xF0F
-; X64-NEXT:    shll $4, %eax
-; X64-NEXT:    shrl $4, %esi
-; X64-NEXT:    andl $3855, %esi # imm = 0xF0F
-; X64-NEXT:    orl %eax, %esi
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    andl $13107, %eax # imm = 0x3333
-; X64-NEXT:    shrl $2, %esi
-; X64-NEXT:    andl $13107, %esi # imm = 0x3333
-; X64-NEXT:    leal (%rsi,%rax,4), %eax
-; X64-NEXT:    movl %eax, %ecx
-; X64-NEXT:    andl $21845, %ecx # imm = 0x5555
-; X64-NEXT:    shrl %eax
-; X64-NEXT:    andl $21845, %eax # imm = 0x5555
-; X64-NEXT:    leal (%rax,%rcx,2), %eax
-; X64-NEXT:    andl %edi, %eax
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    retq
+; X64-NOBMI-LABEL: andnot_bitreverse_i16:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-NOBMI-NEXT:    rolw $8, %si
+; X64-NOBMI-NEXT:    movl %esi, %eax
+; X64-NOBMI-NEXT:    andl $3855, %eax # imm = 0xF0F
+; X64-NOBMI-NEXT:    shll $4, %eax
+; X64-NOBMI-NEXT:    shrl $4, %esi
+; X64-NOBMI-NEXT:    andl $3855, %esi # imm = 0xF0F
+; X64-NOBMI-NEXT:    orl %eax, %esi
+; X64-NOBMI-NEXT:    movl %esi, %eax
+; X64-NOBMI-NEXT:    andl $13107, %eax # imm = 0x3333
+; X64-NOBMI-NEXT:    shrl $2, %esi
+; X64-NOBMI-NEXT:    andl $13107, %esi # imm = 0x3333
+; X64-NOBMI-NEXT:    leal (%rsi,%rax,4), %eax
+; X64-NOBMI-NEXT:    movl %eax, %ecx
+; X64-NOBMI-NEXT:    andl $21845, %ecx # imm = 0x5555
+; X64-NOBMI-NEXT:    shrl %eax
+; X64-NOBMI-NEXT:    andl $21845, %eax # imm = 0x5555
+; X64-NOBMI-NEXT:    leal (%rax,%rcx,2), %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_bitreverse_i16:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-BMI-NEXT:    rolw $8, %si
+; X64-BMI-NEXT:    movl %esi, %eax
+; X64-BMI-NEXT:    andl $3855, %eax # imm = 0xF0F
+; X64-BMI-NEXT:    shll $4, %eax
+; X64-BMI-NEXT:    shrl $4, %esi
+; X64-BMI-NEXT:    andl $3855, %esi # imm = 0xF0F
+; X64-BMI-NEXT:    orl %eax, %esi
+; X64-BMI-NEXT:    movl %esi, %eax
+; X64-BMI-NEXT:    andl $13107, %eax # imm = 0x3333
+; X64-BMI-NEXT:    shrl $2, %esi
+; X64-BMI-NEXT:    andl $13107, %esi # imm = 0x3333
+; X64-BMI-NEXT:    leal (%rsi,%rax,4), %eax
+; X64-BMI-NEXT:    movl %eax, %ecx
+; X64-BMI-NEXT:    andl $21845, %ecx # imm = 0x5555
+; X64-BMI-NEXT:    shrl %eax
+; X64-BMI-NEXT:    andl $21845, %eax # imm = 0x5555
+; X64-BMI-NEXT:    leal (%rax,%rcx,2), %eax
+; X64-BMI-NEXT:    andnl %edi, %eax, %eax
+; X64-BMI-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-BMI-NEXT:    retq
   %not = xor i16 %a1, -1
   %bitrev = tail call i16 @llvm.bitreverse.i16(i16 %not)
   %and = and i16 %bitrev, %a0
@@ -586,7 +896,6 @@ define i8 @andnot_bitreverse_i8(i8 %a0, i8 %a1) nounwind {
 ; X86-LABEL: andnot_bitreverse_i8:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    notb %al
 ; X86-NEXT:    rolb $4, %al
 ; X86-NEXT:    movl %eax, %ecx
 ; X86-NEXT:    andb $51, %cl
@@ -600,12 +909,12 @@ define i8 @andnot_bitreverse_i8(i8 %a0, i8 %a1) nounwind {
 ; X86-NEXT:    shrb %al
 ; X86-NEXT:    andb $85, %al
 ; X86-NEXT:    orb %cl, %al
+; X86-NEXT:    notb %al
 ; X86-NEXT:    andb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: andnot_bitreverse_i8:
 ; X64:       # %bb.0:
-; X64-NEXT:    notb %sil
 ; X64-NEXT:    rolb $4, %sil
 ; X64-NEXT:    movl %esi, %eax
 ; X64-NEXT:    andb $51, %al
@@ -619,6 +928,7 @@ define i8 @andnot_bitreverse_i8(i8 %a0, i8 %a1) nounwind {
 ; X64-NEXT:    shrb %al
 ; X64-NEXT:    andb $85, %al
 ; X64-NEXT:    orb %cl, %al
+; X64-NEXT:    notb %al
 ; X64-NEXT:    andb %dil, %al
 ; X64-NEXT:    retq
   %not = xor i8 %a1, -1
@@ -626,8 +936,3 @@ define i8 @andnot_bitreverse_i8(i8 %a0, i8 %a1) nounwind {
   %and = and i8 %bitrev, %a0
   ret i8 %and
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; X64-BMI: {{.*}}
-; X64-NOBMI: {{.*}}
-; X86-BMI: {{.*}}
-; X86-NOBMI: {{.*}}
-- 
GitLab


From 75c75fc16e8a202668cfa71404724117b22336b9 Mon Sep 17 00:00:00 2001
From: dong-miao <601183878@qq.com>
Date: Mon, 28 Oct 2024 04:54:51 -0700
Subject: [PATCH 306/693] [RISCV]Add svvptc extension (#113882)

---
 clang/test/Driver/print-supported-extensions-riscv.c | 1 +
 clang/test/Preprocessor/riscv-target-features.c      | 9 +++++++++
 llvm/docs/RISCVUsage.rst                             | 1 +
 llvm/docs/ReleaseNotes.md                            | 2 +-
 llvm/lib/Target/RISCV/RISCVFeatures.td               | 4 ++++
 llvm/test/CodeGen/RISCV/attributes.ll                | 4 ++++
 llvm/test/MC/RISCV/attribute-arch.s                  | 3 +++
 llvm/unittests/TargetParser/RISCVISAInfoTest.cpp     | 1 +
 8 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index fc8a9c04667b..4d93144724df 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -147,6 +147,7 @@
 // CHECK-NEXT:     svinval              1.0       'Svinval' (Fine-Grained Address-Translation Cache Invalidation)
 // CHECK-NEXT:     svnapot              1.0       'Svnapot' (NAPOT Translation Contiguity)
 // CHECK-NEXT:     svpbmt               1.0       'Svpbmt' (Page-Based Memory Types)
+// CHECK-NEXT:     svvptc               1.0       'svvptc' (Obviating Memory-Management Instructions after Marking PTEs Valid)
 // CHECK-NEXT:     xcvalu               1.0       'XCValu' (CORE-V ALU Operations)
 // CHECK-NEXT:     xcvbi                1.0       'XCVbi' (CORE-V Immediate Branching)
 // CHECK-NEXT:     xcvbitmanip          1.0       'XCVbitmanip' (CORE-V Bit Manipulation)
diff --git a/clang/test/Preprocessor/riscv-target-features.c b/clang/test/Preprocessor/riscv-target-features.c
index 6e586714af84..597325ffa5e4 100644
--- a/clang/test/Preprocessor/riscv-target-features.c
+++ b/clang/test/Preprocessor/riscv-target-features.c
@@ -52,6 +52,7 @@
 // CHECK-NOT: __riscv_svinval {{.*$}}
 // CHECK-NOT: __riscv_svnapot {{.*$}}
 // CHECK-NOT: __riscv_svpbmt {{.*$}}
+// CHECK-NOT: __riscv_svvptc {{.*$}}
 // CHECK-NOT: __riscv_v {{.*$}}
 // CHECK-NOT: __riscv_v_elen {{.*$}}
 // CHECK-NOT: __riscv_v_elen_fp {{.*$}}
@@ -516,6 +517,14 @@
 // RUN:   -o - | FileCheck --check-prefix=CHECK-SVPBMT-EXT %s
 // CHECK-SVPBMT-EXT: __riscv_svpbmt 1000000{{$}}
 
+// RUN: %clang --target=riscv32-unknown-linux-gnu \
+// RUN:   -march=rv32isvvptc -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-SVVPTC-EXT %s
+// RUN: %clang --target=riscv64-unknown-linux-gnu \
+// RUN:   -march=rv64isvvptc -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-SVVPTC-EXT %s
+// CHECK-SVVPTC-EXT: __riscv_svvptc 1000000{{$}}
+
 // RUN: %clang --target=riscv32-unknown-linux-gnu \
 // RUN:   -march=rv32iv1p0 -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-V-EXT %s
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index 6075a2289d47..dae6f7c46cc4 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -156,6 +156,7 @@ on support follow.
      ``Svinval``       Assembly Support
      ``Svnapot``       Assembly Support
      ``Svpbmt``        Supported
+     ``Svvptc``        Supported
      ``V``             Supported
      ``Za128rs``       Supported (`See note <#riscv-profiles-extensions-note>`__)
      ``Za64rs``        Supported (`See note <#riscv-profiles-extensions-note>`__)
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index f7215279940d..22a32d110855 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -178,7 +178,7 @@ Changes to the RISC-V Backend
   means Zve32x and Zve32f will also require Zvl64b. The prior support was
   largely untested.
 * The `Zvbc32e` and `Zvkgs` extensions are now supported experimentally.
-* Added `Smctr` and `Ssctr` extensions.
+* Added `Smctr`, `Ssctr` and `Svvptc` extensions.
 * `-mcpu=syntacore-scr7` was added.
 * The `Zacas` extension is no longer marked as experimental.
 * The `Smmpm`, `Smnpm`, `Ssnpm`, `Supm`, and `Sspm` pointer masking extensions
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 559f0e5950ed..608782d7839a 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -1029,6 +1029,10 @@ def FeatureStdExtSvpbmt
     : RISCVExtension<"svpbmt", 1, 0,
                      "'Svpbmt' (Page-Based Memory Types)">;
 
+def FeatureStdExtSvvptc
+    : RISCVExtension<"svvptc", 1, 0,
+                     "'svvptc' (Obviating Memory-Management Instructions after Marking PTEs Valid)">;                    
+
 def FeatureStdExtSha
     : RISCVExtension<"sha", 1, 0,
                      "'Sha' (Augmented Hypervisor)",
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index 5c835befd6f5..e5b308a17266 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -62,6 +62,7 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+svbare %s -o - | FileCheck --check-prefixes=CHECK,RV32SVBARE %s
 ; RUN: llc -mtriple=riscv32 -mattr=+svnapot %s -o - | FileCheck --check-prefixes=CHECK,RV32SVNAPOT %s
 ; RUN: llc -mtriple=riscv32 -mattr=+svpbmt %s -o - | FileCheck --check-prefixes=CHECK,RV32SVPBMT %s
+; RUN: llc -mtriple=riscv32 -mattr=+svvptc %s -o - | FileCheck --check-prefixes=CHECK,RV32SVVPTC %s
 ; RUN: llc -mtriple=riscv32 -mattr=+svinval %s -o - | FileCheck --check-prefixes=CHECK,RV32SVINVAL %s
 ; RUN: llc -mtriple=riscv32 -mattr=+xcvalu %s -o - | FileCheck --check-prefix=RV32XCVALU %s
 ; RUN: llc -mtriple=riscv32 -mattr=+xcvbitmanip %s -o - | FileCheck --check-prefix=RV32XCVBITMANIP %s
@@ -202,6 +203,7 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+svbare %s -o - | FileCheck --check-prefixes=CHECK,RV64SVBARE %s
 ; RUN: llc -mtriple=riscv64 -mattr=+svnapot %s -o - | FileCheck --check-prefixes=CHECK,RV64SVNAPOT %s
 ; RUN: llc -mtriple=riscv64 -mattr=+svpbmt %s -o - | FileCheck --check-prefixes=CHECK,RV64SVPBMT %s
+; RUN: llc -mtriple=riscv64 -mattr=+svvptc %s -o - | FileCheck --check-prefixes=CHECK,RV64SVVPTC %s
 ; RUN: llc -mtriple=riscv64 -mattr=+svinval %s -o - | FileCheck --check-prefixes=CHECK,RV64SVINVAL %s
 ; RUN: llc -mtriple=riscv64 -mattr=+xventanacondops %s -o - | FileCheck --check-prefixes=CHECK,RV64XVENTANACONDOPS %s
 ; RUN: llc -mtriple=riscv64 -mattr=+xsfvfwmaccqqq %s -o - | FileCheck --check-prefix=RV64XSFVFWMACCQQQ %s
@@ -358,6 +360,7 @@
 ; RV32SVBARE: .attribute 5, "rv32i2p1_svbare1p0"
 ; RV32SVNAPOT: .attribute 5, "rv32i2p1_svnapot1p0"
 ; RV32SVPBMT: .attribute 5, "rv32i2p1_svpbmt1p0"
+; RV32SVVPTC: .attribute 5, "rv32i2p1_svvptc1p0"
 ; RV32SVINVAL: .attribute 5, "rv32i2p1_svinval1p0"
 ; RV32XCVALU: .attribute 5, "rv32i2p1_xcvalu1p0"
 ; RV32XCVBITMANIP: .attribute 5, "rv32i2p1_xcvbitmanip1p0"
@@ -500,6 +503,7 @@
 ; RV64SVBARE: .attribute 5, "rv64i2p1_svbare1p0"
 ; RV64SVNAPOT: .attribute 5, "rv64i2p1_svnapot1p0"
 ; RV64SVPBMT: .attribute 5, "rv64i2p1_svpbmt1p0"
+; RV64SVVPTC: .attribute 5, "rv64i2p1_svvptc1p0"
 ; RV64SVINVAL: .attribute 5, "rv64i2p1_svinval1p0"
 ; RV64XVENTANACONDOPS: .attribute 5, "rv64i2p1_xventanacondops1p0"
 ; RV64XSFVFWMACCQQQ: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zve32f1p0_zve32x1p0_zvfbfmin1p0_zvl32b1p0_xsfvfwmaccqqq1p0"
diff --git a/llvm/test/MC/RISCV/attribute-arch.s b/llvm/test/MC/RISCV/attribute-arch.s
index 2332bcacc99d..a744a660a707 100644
--- a/llvm/test/MC/RISCV/attribute-arch.s
+++ b/llvm/test/MC/RISCV/attribute-arch.s
@@ -375,6 +375,9 @@
 .attribute arch, "rv32i_svbare1p0"
 # CHECK: attribute      5, "rv32i2p1_svbare1p0"
 
+.attribute arch, "rv32i_svvptc1p0"
+# CHECK: attribute      5, "rv32i2p1_svvptc1p0"
+
 .attribute arch, "rv32i_zfbfmin1p0"
 # CHECK: .attribute     5, "rv32i2p1_f2p2_zicsr2p0_zfbfmin1p0"
 
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index e402b1a40de3..30b5f3ce3cb0 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -1092,6 +1092,7 @@ R"(All available -march extensions for RISC-V
     svinval              1.0
     svnapot              1.0
     svpbmt               1.0
+    svvptc               1.0
     xcvalu               1.0
     xcvbi                1.0
     xcvbitmanip          1.0
-- 
GitLab


From 233e64d8e4211e6c3c94a986c28f9993da2b8de0 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 28 Oct 2024 11:56:40 +0000
Subject: [PATCH 307/693] [gn build] Port 0be1883c36fc

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 0586704850a5..1b73f37b0b1b 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -430,6 +430,8 @@ if (current_toolchain == default_toolchain) {
       "__filesystem/recursive_directory_iterator.h",
       "__filesystem/space_info.h",
       "__filesystem/u8path.h",
+      "__flat_map/flat_map.h",
+      "__flat_map/sorted_unique.h",
       "__format/buffer.h",
       "__format/concepts.h",
       "__format/container_adaptor.h",
-- 
GitLab


From cbaecb5d299c48ea23319704445c1ee8405f4c04 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 28 Oct 2024 11:56:42 +0000
Subject: [PATCH 308/693] [gn build] Port 3d6923dbac16

---
 .../secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn  | 1 +
 .../gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn   | 4 +---
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
index f23bccf9d511..61e4f8da3c04 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
@@ -56,6 +56,7 @@ static_library("bugprone") {
     "MultipleStatementMacroCheck.cpp",
     "NoEscapeCheck.cpp",
     "NonZeroEnumToBoolConversionCheck.cpp",
+    "NondeterministicPointerIterationOrderCheck.cpp",
     "NotNullTerminatedResultCheck.cpp",
     "OptionalValueConversionCheck.cpp",
     "ParentVirtualCallCheck.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
index 7a6c360e88c1..fe211726eb5d 100644
--- a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
@@ -99,8 +99,6 @@ static_library("Checkers") {
     "ObjCUnusedIVarsChecker.cpp",
     "PaddingChecker.cpp",
     "PointerArithChecker.cpp",
-    "PointerIterationChecker.cpp",
-    "PointerSortingChecker.cpp",
     "PointerSubChecker.cpp",
     "PthreadLockChecker.cpp",
     "PutenvStackArrayChecker.cpp",
@@ -141,8 +139,8 @@ static_library("Checkers") {
     "VforkChecker.cpp",
     "VirtualCallChecker.cpp",
     "WebKit/ASTUtils.cpp",
-    "WebKit/RawPtrRefMemberChecker.cpp",
     "WebKit/PtrTypesSemantics.cpp",
+    "WebKit/RawPtrRefMemberChecker.cpp",
     "WebKit/RefCntblBaseVirtualDtorChecker.cpp",
     "WebKit/UncountedCallArgsChecker.cpp",
     "WebKit/UncountedLambdaCapturesChecker.cpp",
-- 
GitLab


From 595ec4e4be7de4258dd346e606b4a03a5eb0ce8a Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 28 Oct 2024 11:56:42 +0000
Subject: [PATCH 309/693] [gn build] Port 5aa1275d03b6

---
 llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
index 0a97bcf59112..b47189accd13 100644
--- a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
@@ -293,6 +293,7 @@ copy("Headers") {
     "shaintrin.h",
     "sifive_vector.h",
     "sm3intrin.h",
+    "sm4evexintrin.h",
     "sm4intrin.h",
     "smmintrin.h",
     "stdalign.h",
-- 
GitLab


From 2a4bab3fac64833321287d9ba17dcad5255d0ab4 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 28 Oct 2024 11:56:43 +0000
Subject: [PATCH 310/693] [gn build] Port cfde4fbccf5d

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 1b73f37b0b1b..28d41cda5aa8 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -571,7 +571,6 @@ if (current_toolchain == default_toolchain) {
       "__locale_dir/locale_base_api/fuchsia.h",
       "__locale_dir/locale_base_api/ibm.h",
       "__locale_dir/locale_base_api/musl.h",
-      "__locale_dir/locale_base_api/newlib.h",
       "__locale_dir/locale_base_api/openbsd.h",
       "__locale_dir/locale_base_api/win32.h",
       "__locale_dir/locale_guard.h",
-- 
GitLab


From 1164bd774783f33419e56cf10fdb87fbeab2b185 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 28 Oct 2024 11:56:44 +0000
Subject: [PATCH 311/693] [gn build] Port e146c1867e8d

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 28d41cda5aa8..8121e34dcf6e 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -566,8 +566,10 @@ if (current_toolchain == default_toolchain) {
       "__locale",
       "__locale_dir/locale_base_api.h",
       "__locale_dir/locale_base_api/android.h",
+      "__locale_dir/locale_base_api/apple.h",
       "__locale_dir/locale_base_api/bsd_locale_defaults.h",
       "__locale_dir/locale_base_api/bsd_locale_fallbacks.h",
+      "__locale_dir/locale_base_api/freebsd.h",
       "__locale_dir/locale_base_api/fuchsia.h",
       "__locale_dir/locale_base_api/ibm.h",
       "__locale_dir/locale_base_api/musl.h",
-- 
GitLab


From ba7555e640ea7fe341e19e0c1fffe5960b7c0d5b Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Mon, 28 Oct 2024 12:56:47 +0000
Subject: [PATCH 312/693] [RISCV] Mark the RVA23S64 and RVA23U64 profiles as
 non-experimental (#113826)

All of the extensions used by these profile are themselves
non-experimental, and RVA23 was just ratified

<https://riscv.org/announcements/2024/10/risc-v-announces-ratification-of-the-rva23-profile-standard/>.

<https://github.com/riscv/riscv-profiles/blob/main/src/rva23-profile.adoc>

We lack a way of expressing `Ss1p13` (supervisor architecture 1.13), but
this is a problem we have for RVA22 (Ss1p12) and RVA20 (Ss1p11) so I
don't feel it's a blocker.
---
 clang/test/Driver/print-supported-extensions-riscv.c |  4 ++--
 clang/test/Driver/riscv-profiles.c                   | 10 +++++-----
 llvm/docs/RISCVUsage.rst                             |  4 ++--
 llvm/docs/ReleaseNotes.md                            |  1 +
 llvm/lib/Target/RISCV/RISCVProfiles.td               |  4 ++--
 llvm/test/CodeGen/RISCV/attributes.ll                |  4 ++--
 llvm/unittests/TargetParser/RISCVISAInfoTest.cpp     |  8 ++++----
 7 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index 4d93144724df..e39847b9c31a 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -191,12 +191,12 @@
 // CHECK-NEXT:     rva20u64
 // CHECK-NEXT:     rva22s64
 // CHECK-NEXT:     rva22u64
+// CHECK-NEXT:     rva23s64
+// CHECK-NEXT:     rva23u64
 // CHECK-NEXT:     rvi20u32
 // CHECK-NEXT:     rvi20u64
 // CHECK-EMPTY:
 // CHECK-NEXT: Experimental Profiles
-// CHECK-NEXT:     rva23s64
-// CHECK-NEXT:     rva23u64
 // CHECK-NEXT:     rvb23s64
 // CHECK-NEXT:     rvb23u64
 // CHECK-NEXT:     rvm23u32
diff --git a/clang/test/Driver/riscv-profiles.c b/clang/test/Driver/riscv-profiles.c
index 42e23cf57c88..d85ac8baf4ed 100644
--- a/clang/test/Driver/riscv-profiles.c
+++ b/clang/test/Driver/riscv-profiles.c
@@ -111,7 +111,7 @@
 // RVA22S64: "-target-feature" "+svinval"
 // RVA22S64: "-target-feature" "+svpbmt"
 
-// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva23u64 -menable-experimental-extensions \
+// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva23u64 \
 // RUN:   | FileCheck -check-prefix=RVA23U64 %s
 // RVA23U64: "-target-feature" "+m"
 // RVA23U64: "-target-feature" "+a"
@@ -148,7 +148,7 @@
 // RVA23U64: "-target-feature" "+zvfhmin"
 // RVA23U64: "-target-feature" "+zvkt"
 
-// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva23s64 -menable-experimental-extensions \
+// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva23s64 \
 // RUN:   | FileCheck -check-prefix=RVA23S64 %s
 // RVA23S64: "-target-feature" "+m"
 // RVA23S64: "-target-feature" "+a"
@@ -323,6 +323,6 @@
 // RUN: not %clang --target=riscv64 -### -c %s 2>&1 -march=rva22u64zfa | FileCheck -check-prefix=INVALID-ADDITIONAL %s
 // INVALID-ADDITIONAL: error: invalid arch name 'rva22u64zfa', additional extensions must be after separator '_'
 
-// RUN: not %clang --target=riscv64 -### -c %s 2>&1 -march=rva23u64 | FileCheck -check-prefix=EXPERIMENTAL-NOFLAG %s
-// EXPERIMENTAL-NOFLAG: error: invalid arch name 'rva23u64'
-// EXPERIMENTAL-NOFLAG: requires '-menable-experimental-extensions' for profile 'rva23u64'
+// RUN: not %clang --target=riscv32 -### -c %s 2>&1 -march=rvm23u32 | FileCheck -check-prefix=EXPERIMENTAL-NOFLAG %s
+// EXPERIMENTAL-NOFLAG: error: invalid arch name 'rvm23u32'
+// EXPERIMENTAL-NOFLAG: requires '-menable-experimental-extensions' for profile 'rvm23u32'
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index dae6f7c46cc4..04f2c357766d 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -82,6 +82,8 @@ ISA naming string. Currently supported profiles:
 * ``rva20s64``
 * ``rva22u64``
 * ``rva22s64``
+* ``rva23u64``
+* ``rva23s64``
 
 Note that you can also append additional extension names to be enabled, e.g.
 ``rva20u64_zicond`` will enable the ``zicond`` extension in addition to those
@@ -91,8 +93,6 @@ Profiles that are not yet ratified cannot be used unless
 ``-menable-experimental-extensions`` (or equivalent for other tools) is
 specified. This applies to the following profiles:
 
-* ``rva23u64``
-* ``rva23s64``
 * ``rvb23u64``
 * ``rvb23s64``
 * ``rvm23u32``
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 22a32d110855..ac7a795daf79 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -184,6 +184,7 @@ Changes to the RISC-V Backend
 * The `Smmpm`, `Smnpm`, `Ssnpm`, `Supm`, and `Sspm` pointer masking extensions
   are no longer marked as experimental.
 * The `Sha` extension is now supported.
+* The RVA23U64 and RVA23S64 profiles are no longer marked as experimental.
 
 Changes to the WebAssembly Backend
 ----------------------------------
diff --git a/llvm/lib/Target/RISCV/RISCVProfiles.td b/llvm/lib/Target/RISCV/RISCVProfiles.td
index 78c076fdb0b2..ea0fe08abd7a 100644
--- a/llvm/lib/Target/RISCV/RISCVProfiles.td
+++ b/llvm/lib/Target/RISCV/RISCVProfiles.td
@@ -161,8 +161,8 @@ def RVA20U64 : RISCVProfile<"rva20u64", RVA20U64Features>;
 def RVA20S64 : RISCVProfile<"rva20s64", RVA20S64Features>;
 def RVA22U64 : RISCVProfile<"rva22u64", RVA22U64Features>;
 def RVA22S64 : RISCVProfile<"rva22s64", RVA22S64Features>;
-def RVA23U64 : RISCVExperimentalProfile<"rva23u64", RVA23U64Features>;
-def RVA23S64 : RISCVExperimentalProfile<"rva23s64", RVA23S64Features>;
+def RVA23U64 : RISCVProfile<"rva23u64", RVA23U64Features>;
+def RVA23S64 : RISCVProfile<"rva23s64", RVA23S64Features>;
 def RVB23U64 : RISCVExperimentalProfile<"rvb23u64", RVB23U64Features>;
 def RVB23S64 : RISCVExperimentalProfile<"rvb23s64", RVB23S64Features>;
 def RVM23U32 : RISCVExperimentalProfile<"rvm23u32", RVM23U32Features>;
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index e5b308a17266..2545c7075e4c 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -291,8 +291,8 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+rva20s64 %s -o - | FileCheck --check-prefix=RVA20S64 %s
 ; RUN: llc -mtriple=riscv64 -mattr=+rva22u64 %s -o - | FileCheck --check-prefix=RVA22U64 %s
 ; RUN: llc -mtriple=riscv64 -mattr=+rva22s64 %s -o - | FileCheck --check-prefix=RVA22S64 %s
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-rva23u64 %s -o - | FileCheck --check-prefix=RVA23U64 %s
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-rva23s64 %s -o - | FileCheck --check-prefix=RVA23S64 %s
+; RUN: llc -mtriple=riscv64 -mattr=+rva23u64 %s -o - | FileCheck --check-prefix=RVA23U64 %s
+; RUN: llc -mtriple=riscv64 -mattr=+rva23s64 %s -o - | FileCheck --check-prefix=RVA23S64 %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-rvb23u64 %s -o - | FileCheck --check-prefix=RVB23U64 %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-rvb23s64 %s -o - | FileCheck --check-prefix=RVB23S64 %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-rvm23u32 %s -o - | FileCheck --check-prefix=RVM23U32 %s
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 30b5f3ce3cb0..48792ad0265f 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -774,8 +774,8 @@ TEST(ParseArchString,
 TEST(ParseArchString,
      RejectsExperimentalProfilesIfEnableExperimentalExtensionsNotSet) {
   EXPECT_EQ(
-      toString(RISCVISAInfo::parseArchString("rva23u64", false).takeError()),
-      "requires '-menable-experimental-extensions' for profile 'rva23u64'");
+      toString(RISCVISAInfo::parseArchString("rvm23u32", false).takeError()),
+      "requires '-menable-experimental-extensions' for profile 'rvm23u32'");
 }
 
 TEST(ToFeatures, IIsDroppedAndExperimentalExtensionsArePrefixed) {
@@ -1136,12 +1136,12 @@ Supported Profiles
     rva20u64
     rva22s64
     rva22u64
+    rva23s64
+    rva23u64
     rvi20u32
     rvi20u64
 
 Experimental Profiles
-    rva23s64
-    rva23u64
     rvb23s64
     rvb23u64
     rvm23u32
-- 
GitLab


From ddd463be7edc3f3d03a6e88917fff66ef84210e9 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Mon, 28 Oct 2024 13:01:22 +0000
Subject: [PATCH 313/693] [AArch64] Add getStreamingHazardSize() to
 AArch64Subtarget (#113679)

This is defined by the `-aarch64-streaming-hazard-size` option or its
alias `-aarch64-stack-hazard-size` (the original name). It has been
renamed to be more general as this option will (for the time being) be
used to detect if the current target has streaming mode memory hazards.

---------

Co-authored-by: Hari Limaye <hari.limaye@arm.com>
---
 .../Target/AArch64/AArch64FrameLowering.cpp   | 30 +++++++++++--------
 llvm/lib/Target/AArch64/AArch64Subtarget.cpp  | 11 +++++++
 llvm/lib/Target/AArch64/AArch64Subtarget.h    |  5 ++++
 3 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 275070b332ac..dfaa36f7f512 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -126,14 +126,15 @@
 // and the SME unit try to access the same area of memory, including if the
 // access is to an area of the stack. To try to alleviate this we attempt to
 // introduce extra padding into the stack frame between FP and GPR accesses,
-// controlled by the StackHazardSize option. Without changing the layout of the
-// stack frame in the diagram above, a stack object of size StackHazardSize is
-// added between GPR and FPR CSRs. Another is added to the stack objects
-// section, and stack objects are sorted so that FPR > Hazard padding slot >
-// GPRs (where possible). Unfortunately some things are not handled well (VLA
-// area, arguments on the stack, object with both GPR and FPR accesses), but if
-// those are controlled by the user then the entire stack frame becomes GPR at
-// the start/end with FPR in the middle, surrounded by Hazard padding.
+// controlled by the aarch64-stack-hazard-size option. Without changing the
+// layout of the stack frame in the diagram above, a stack object of size
+// aarch64-stack-hazard-size is added between GPR and FPR CSRs. Another is added
+// to the stack objects section, and stack objects are sorted so that FPR >
+// Hazard padding slot > GPRs (where possible). Unfortunately some things are
+// not handled well (VLA area, arguments on the stack, objects with both GPR and
+// FPR accesses), but if those are controlled by the user then the entire stack
+// frame becomes GPR at the start/end with FPR in the middle, surrounded by
+// Hazard padding.
 //
 // An example of the prologue:
 //
@@ -273,9 +274,6 @@ cl::opt<bool> EnableHomogeneousPrologEpilog(
     cl::desc("Emit homogeneous prologue and epilogue for the size "
              "optimization (default = off)"));
 
-// Stack hazard padding size. 0 = disabled.
-static cl::opt<unsigned> StackHazardSize("aarch64-stack-hazard-size",
-                                         cl::init(0), cl::Hidden);
 // Stack hazard size for analysis remarks. StackHazardSize takes precedence.
 static cl::opt<unsigned>
     StackHazardRemarkSize("aarch64-stack-hazard-remark-size", cl::init(0),
@@ -1617,6 +1615,10 @@ static bool isTargetWindows(const MachineFunction &MF) {
   return MF.getSubtarget<AArch64Subtarget>().isTargetWindows();
 }
 
+static unsigned getStackHazardSize(const MachineFunction &MF) {
+  return MF.getSubtarget<AArch64Subtarget>().getStreamingHazardSize();
+}
+
 // Convenience function to determine whether I is an SVE callee save.
 static bool IsSVECalleeSave(MachineBasicBlock::iterator I) {
   switch (I->getOpcode()) {
@@ -2988,6 +2990,7 @@ static void computeCalleeSaveRegisterPairs(
   bool IsWindows = isTargetWindows(MF);
   bool NeedsWinCFI = needsWinCFI(MF);
   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
+  unsigned StackHazardSize = getStackHazardSize(MF);
   MachineFrameInfo &MFI = MF.getFrameInfo();
   CallingConv::ID CC = MF.getFunction().getCallingConv();
   unsigned Count = CSI.size();
@@ -3615,6 +3618,7 @@ static std::optional<int> getLdStFrameID(const MachineInstr &MI,
 // which can be used to determine if any hazard padding is needed.
 void AArch64FrameLowering::determineStackHazardSlot(
     MachineFunction &MF, BitVector &SavedRegs) const {
+  unsigned StackHazardSize = getStackHazardSize(MF);
   if (StackHazardSize == 0 || StackHazardSize % 16 != 0 ||
       MF.getInfo<AArch64FunctionInfo>()->hasStackHazardSlotIndex())
     return;
@@ -3805,7 +3809,7 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
   // StackHazardSize if so.
   determineStackHazardSlot(MF, SavedRegs);
   if (AFI->hasStackHazardSlotIndex())
-    CSStackSize += StackHazardSize;
+    CSStackSize += getStackHazardSize(MF);
 
   // Save number of saved regs, so we can easily update CSStackSize later.
   unsigned NumSavedRegs = SavedRegs.count();
@@ -3920,6 +3924,7 @@ bool AArch64FrameLowering::assignCalleeSavedSpillSlots(
     std::vector<CalleeSavedInfo> &CSI, unsigned &MinCSFrameIndex,
     unsigned &MaxCSFrameIndex) const {
   bool NeedsWinCFI = needsWinCFI(MF);
+  unsigned StackHazardSize = getStackHazardSize(MF);
   // To match the canonical windows frame layout, reverse the list of
   // callee saved registers to get them laid out by PrologEpilogInserter
   // in the right order. (PrologEpilogInserter allocates stack objects top
@@ -5154,6 +5159,7 @@ void AArch64FrameLowering::emitRemarks(
   if (Attrs.hasNonStreamingInterfaceAndBody())
     return;
 
+  unsigned StackHazardSize = getStackHazardSize(MF);
   const uint64_t HazardSize =
       (StackHazardSize) ? StackHazardSize : StackHazardRemarkSize;
 
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
index 32db1e8c2477..7fb2a961e031 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -76,6 +76,16 @@ static cl::opt<unsigned> AArch64MinimumJumpTableEntries(
     "aarch64-min-jump-table-entries", cl::init(13), cl::Hidden,
     cl::desc("Set minimum number of entries to use a jump table on AArch64"));
 
+static cl::opt<unsigned> AArch64StreamingHazardSize(
+    "aarch64-streaming-hazard-size",
+    cl::desc("Hazard size for streaming mode memory accesses. 0 = disabled."),
+    cl::init(0), cl::Hidden);
+
+static cl::alias AArch64StreamingStackHazardSize(
+    "aarch64-stack-hazard-size",
+    cl::desc("alias for -aarch64-streaming-hazard-size"),
+    cl::aliasopt(AArch64StreamingHazardSize));
+
 unsigned AArch64Subtarget::getVectorInsertExtractBaseCost() const {
   if (OverrideVectorInsertExtractBaseCost.getNumOccurrences() > 0)
     return OverrideVectorInsertExtractBaseCost;
@@ -333,6 +343,7 @@ AArch64Subtarget::AArch64Subtarget(const Triple &TT, StringRef CPU,
       CustomCallSavedXRegs(AArch64::GPR64commonRegClass.getNumRegs()),
       IsLittle(LittleEndian), IsStreaming(IsStreaming),
       IsStreamingCompatible(IsStreamingCompatible),
+      StreamingHazardSize(AArch64StreamingHazardSize),
       MinSVEVectorSizeInBits(MinSVEVectorSizeInBitsOverride),
       MaxSVEVectorSizeInBits(MaxSVEVectorSizeInBitsOverride), TargetTriple(TT),
       InstrInfo(initializeSubtargetDependencies(FS, CPU, TuneCPU, HasMinSize)),
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index 9856415361e5..50adb7cbf69a 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -84,6 +84,7 @@ protected:
 
   bool IsStreaming;
   bool IsStreamingCompatible;
+  unsigned StreamingHazardSize;
   unsigned MinSVEVectorSizeInBits;
   unsigned MaxSVEVectorSizeInBits;
   unsigned VScaleForTuning = 2;
@@ -172,6 +173,10 @@ public:
   /// Returns true if the function has a streaming-compatible body.
   bool isStreamingCompatible() const { return IsStreamingCompatible; }
 
+  /// Returns the size of memory region that if accessed by both the CPU and
+  /// the SME unit could result in a hazard. 0 = disabled.
+  unsigned getStreamingHazardSize() const { return StreamingHazardSize; }
+
   /// Returns true if the target has NEON and the function at runtime is known
   /// to have NEON enabled (e.g. the function is known not to be in streaming-SVE
   /// mode, which disables NEON instructions).
-- 
GitLab


From 9090430d4176fa260b8da46b7b983b3760d452be Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Mon, 28 Oct 2024 14:28:07 +0100
Subject: [PATCH 314/693] Add clang::lifetimebound annotation to StringRef
 constructors. (#113878)

Adding the lifetimebound annotation to the ArrayRef's array constructor
can enable us to detect the following use-after-free issues:

```
llvm::StringRef TestZoneName() {
   char test[] = "foo"; // oops, missing static
   return test; // use-after-free.
}
```

See #113533
---
 llvm/include/llvm/ADT/StringRef.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h
index f879bbf7164f..0dcd4d90086e 100644
--- a/llvm/include/llvm/ADT/StringRef.h
+++ b/llvm/include/llvm/ADT/StringRef.h
@@ -81,7 +81,7 @@ namespace llvm {
     StringRef(std::nullptr_t) = delete;
 
     /// Construct a string ref from a cstring.
-    /*implicit*/ constexpr StringRef(const char *Str)
+    /*implicit*/ constexpr StringRef(const char *Str LLVM_LIFETIME_BOUND)
         : View(Str, Str ?
     // GCC 7 doesn't have constexpr char_traits. Fall back to __builtin_strlen.
 #if defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 8
@@ -93,7 +93,8 @@ namespace llvm {
     }
 
     /// Construct a string ref from a pointer and length.
-    /*implicit*/ constexpr StringRef(const char *data, size_t length)
+    /*implicit*/ constexpr StringRef(const char *data LLVM_LIFETIME_BOUND,
+                                     size_t length)
         : View(data, length) {}
 
     /// Construct a string ref from an std::string.
-- 
GitLab


From b1ede8fcb45d91092f5afe6c88d7a548f14ed848 Mon Sep 17 00:00:00 2001
From: Guillaume Chatelet <gchatelet@google.com>
Date: Mon, 28 Oct 2024 14:43:10 +0100
Subject: [PATCH 315/693] [reland][libc][bazel] Enable software prefetching for
 memcpy (#113886)

This will affect only Bazel configuration for now.
This is a reland of #108939 which has been reverted because of codegen
issues fixed by https://github.com/llvm/llvm-project/pull/113161.
---
 .../bazel/llvm-project-overlay/libc/libc_configure_options.bzl  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl b/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl
index f65da9e98226..96d7fa86e9dd 100644
--- a/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl
@@ -24,7 +24,7 @@ LIBC_CONFIGURE_OPTIONS = [
     # Documentation in libc/src/string/memory_utils/...
     # "LIBC_COPT_MEMCPY_USE_EMBEDDED_TINY",
     # "LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE",
-    # "LIBC_COPT_MEMCPY_X86_USE_SOFTWARE_PREFETCHING",
+    "LIBC_COPT_MEMCPY_X86_USE_SOFTWARE_PREFETCHING",
     "LIBC_COPT_MEMSET_X86_USE_SOFTWARE_PREFETCHING",
 
     # Documentation in libc/docs/dev/printf_behavior.rst
-- 
GitLab


From ce0368eb8417f2d369499bb98b1f0ccbe2219598 Mon Sep 17 00:00:00 2001
From: SpencerAbson <Spencer.Abson@arm.com>
Date: Mon, 28 Oct 2024 13:55:16 +0000
Subject: [PATCH 316/693] [AArch64] Add assembly/disassembly for PMLAL/PMULL
 instructions (#113564)

This patch adds assembly/disassembly for the following SVE_AES2
instructions

    -  PMLAL
    -  PMULL
- In accordance with:
https://developer.arm.com/documentation/ddi0602/latest/
---
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |  4 ++
 llvm/lib/Target/AArch64/SVEInstrFormats.td    | 40 +++++++++++++++++++
 .../MC/AArch64/SVE2p1/pmlal-diagnostics.s     | 37 +++++++++++++++++
 llvm/test/MC/AArch64/SVE2p1/pmlal.s           | 33 +++++++++++++++
 .../MC/AArch64/SVE2p1/pmull-diagnostics.s     | 37 +++++++++++++++++
 llvm/test/MC/AArch64/SVE2p1/pmull.s           | 33 +++++++++++++++
 6 files changed, 184 insertions(+)
 create mode 100644 llvm/test/MC/AArch64/SVE2p1/pmlal-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p1/pmlal.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p1/pmull-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p1/pmull.s

diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index dc96b249c4e4..11c64df2eb92 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -3928,6 +3928,10 @@ let Predicates = [HasSVEAES2, HasSVE2p1orSSVE_AES] in {
   def AESD_4ZZI_B    : sve_crypto_binary_multi4<0b0100, "aesd">;
   def AESEMC_4ZZI_B  : sve_crypto_binary_multi4<0b1000, "aesemc">;
   def AESDMIC_4ZZI_B : sve_crypto_binary_multi4<0b1100, "aesdimc">;
+
+  // SVE_AES2 multi-vector polynomial multiply
+  def PMLAL_2ZZZ_Q : sve_crypto_pmlal_multi<"pmlal">;
+  def PMULL_2ZZZ_Q : sve_crypto_pmull_multi<"pmull">;
 } // End HasSVEAES2, HasSVE2p1orSSVE_AES
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 02ee0fe92445..72cbad17bc04 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -8879,6 +8879,46 @@ class sve_crypto_binary_multi4<bits<4> opc, string asm>
   let hasSideEffects = 0;
 }
 
+class sve_crypto_pmlal_multi<string asm>
+: I<(outs ZZ_q_mul_r:$Zda),
+    (ins ZZ_q_mul_r:$_Zda, ZPR64:$Zn, ZPR64:$Zm),
+  asm,
+  "\t$Zda, $Zn, $Zm",
+  "",
+  []>, Sched<[]> {
+  bits<5> Zm;
+  bits<5> Zn;
+  bits<4> Zda;
+  let Inst{31-21} = 0b01000101001;
+  let Inst{20-16} = Zm;
+  let Inst{15-10} = 0b111111;
+  let Inst{9-5}   = Zn;
+  let Inst{4-1}   = Zda;
+  let Inst{0}     = 0b0;
+
+  let Constraints = "$Zda = $_Zda";
+  let hasSideEffects = 0;
+}
+
+class sve_crypto_pmull_multi<string asm>
+: I<(outs ZZ_q_mul_r:$Zd),
+    (ins ZPR64:$Zn, ZPR64:$Zm),
+  asm,
+  "\t$Zd, $Zn, $Zm",
+  "",
+  []>, Sched<[]> {
+  bits<5> Zm;
+  bits<5> Zn;
+  bits<4> Zd;
+  let Inst{31-21} = 0b01000101001;
+  let Inst{20-16} = Zm;
+  let Inst{15-10} = 0b111110;
+  let Inst{9-5}   = Zn;
+  let Inst{4-1}   = Zd;
+  let Inst{0}     = 0b0;
+  let hasSideEffects = 0;
+}
+
 //===----------------------------------------------------------------------===//
 // SVE BFloat16 Group
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AArch64/SVE2p1/pmlal-diagnostics.s b/llvm/test/MC/AArch64/SVE2p1/pmlal-diagnostics.s
new file mode 100644
index 000000000000..61c2b6eff969
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p1/pmlal-diagnostics.s
@@ -0,0 +1,37 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1,+sve-aes2 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid vector list
+
+pmlal   {z0.q-z2.q}, z0.d, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: pmlal   {z0.q-z2.q}, z0.d, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+pmlal   {z0.q-z0.q}, z0.d, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+// CHECK-NEXT: pmlal   {z0.q-z0.q}, z0.d, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+pmlal   {z1.q-z2.q}, z0.d, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+// CHECK-NEXT: pmlal   {z1.q-z2.q}, z0.d, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+pmlal   {z0.d-z1.d}, z0.d, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: pmlal   {z0.d-z1.d}, z0.d, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid single source vectors
+
+pmlal   {z0.q-z1.q}, z0.s, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: pmlal   {z0.q-z1.q}, z0.s, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+pmlal   {z0.q-z1.q}, z0.d, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: pmlal   {z0.q-z1.q}, z0.d, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p1/pmlal.s b/llvm/test/MC/AArch64/SVE2p1/pmlal.s
new file mode 100644
index 000000000000..0420b230956c
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p1/pmlal.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+ssve-aes < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve-aes2,+sve2p1 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sve-aes2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve-aes2,+sve2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+pmlal   {z0.q-z1.q}, z0.d, z0.d  // 01000101-00100000-11111100-00000000
+// CHECK-INST: pmlal   { z0.q, z1.q }, z0.d, z0.d
+// CHECK-ENCODING: [0x00,0xfc,0x20,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4520fc00 <unknown>
+
+pmlal   {z22.q-z23.q}, z13.d, z8.d  // 01000101-00101000-11111101-10110110
+// CHECK-INST: pmlal   { z22.q, z23.q }, z13.d, z8.d
+// CHECK-ENCODING: [0xb6,0xfd,0x28,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4528fdb6 <unknown>
+
+pmlal   {z30.q-z31.q}, z31.d, z31.d  // 01000101-00111111-11111111-11111110
+// CHECK-INST: pmlal   { z30.q, z31.q }, z31.d, z31.d
+// CHECK-ENCODING: [0xfe,0xff,0x3f,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453ffffe <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p1/pmull-diagnostics.s b/llvm/test/MC/AArch64/SVE2p1/pmull-diagnostics.s
new file mode 100644
index 000000000000..3aaef0cddf4a
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p1/pmull-diagnostics.s
@@ -0,0 +1,37 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1,+sve-aes2 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid vector list
+
+pmull   {z0.q-z2.q}, z0.d, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: pmull   {z0.q-z2.q}, z0.d, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+pmull   {z0.q-z0.q}, z0.d, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+// CHECK-NEXT: pmull   {z0.q-z0.q}, z0.d, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+pmull   {z1.q-z2.q}, z0.d, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+// CHECK-NEXT: pmull   {z1.q-z2.q}, z0.d, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+pmull   {z0.d-z1.d}, z0.d, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: pmull   {z0.d-z1.d}, z0.d, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid single source vectors
+
+pmull   {z0.q-z1.q}, z0.s, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: pmull   {z0.q-z1.q}, z0.s, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+pmull   {z0.q-z1.q}, z0.d, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: pmull   {z0.q-z1.q}, z0.d, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p1/pmull.s b/llvm/test/MC/AArch64/SVE2p1/pmull.s
new file mode 100644
index 000000000000..9c3ee16401c1
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p1/pmull.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+ssve-aes < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve-aes2,+sve2p1 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sve-aes2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-aes2,+sve2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve-aes2,+sve2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+pmull   {z0.q-z1.q}, z0.d, z0.d  // 01000101-00100000-11111000-00000000
+// CHECK-INST: pmull   { z0.q, z1.q }, z0.d, z0.d
+// CHECK-ENCODING: [0x00,0xf8,0x20,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4520f800 <unknown>
+
+pmull   {z22.q-z23.q}, z13.d, z8.d  // 01000101-00101000-11111001-10110110
+// CHECK-INST: pmull   { z22.q, z23.q }, z13.d, z8.d
+// CHECK-ENCODING: [0xb6,0xf9,0x28,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 4528f9b6 <unknown>
+
+pmull   {z30.q-z31.q}, z31.d, z31.d  // 01000101-00111111-11111011-11111110
+// CHECK-INST: pmull   { z30.q, z31.q }, z31.d, z31.d
+// CHECK-ENCODING: [0xfe,0xfb,0x3f,0x45]
+// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453ffbfe <unknown>
\ No newline at end of file
-- 
GitLab


From 7152bf3bc805b8d9b1873058ab0a084d7b6079d6 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Mon, 28 Oct 2024 06:37:24 -0700
Subject: [PATCH 317/693] [SLP]Do not create new vector node if scalars fully
 overlap with the existing one

If the list of scalars vectorized as the part of the same vector node,
no need to generate vector node again, it will be handled as part of
overlapping matching.

Fixes #113810
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  9 +-
 .../full-overlap-non-schedulable.ll           | 93 +++++++++++++++++++
 2 files changed, 100 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/full-overlap-non-schedulable.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 2afd02dae3a8..268546fe99e1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7947,8 +7947,13 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
           Nodes.insert(E);
         SmallPtrSet<Value *, 8> Values(VL.begin(), VL.end());
         if (any_of(Nodes, [&](const TreeEntry *E) {
-              return all_of(E->Scalars,
-                            [&](Value *V) { return Values.contains(V); });
+              if (all_of(E->Scalars,
+                         [&](Value *V) { return Values.contains(V); }))
+                return true;
+              SmallPtrSet<Value *, 8> EValues(E->Scalars.begin(),
+                                              E->Scalars.end());
+              return (
+                  all_of(VL, [&](Value *V) { return EValues.contains(V); }));
             })) {
           LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
           if (TryToFindDuplicates(S))
diff --git a/llvm/test/Transforms/SLPVectorizer/full-overlap-non-schedulable.ll b/llvm/test/Transforms/SLPVectorizer/full-overlap-non-schedulable.ll
new file mode 100644
index 000000000000..dbd91199c24e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/full-overlap-non-schedulable.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer < %s | FileCheck %s
+
+define void @test(ptr %p1, ptr %0, i32 %1, i1 %c1, ptr %p2) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[P1:%.*]], ptr [[TMP0:%.*]], i32 [[TMP1:%.*]], i1 [[C1:%.*]], ptr [[P2:%.*]]) {
+; CHECK-NEXT:  [[TOP:.*:]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP0]], i64 8
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x ptr> [[TMP3]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, <4 x ptr> [[TMP4]], <4 x i64> <i64 8, i64 12, i64 16, i64 20>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x ptr> [[TMP5]], i32 2
+; CHECK-NEXT:    br i1 [[C1]], label %[[L42:.*]], label %[[L41:.*]]
+; CHECK:       [[L41]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <4 x ptr> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[TMP7]], <4 x i32> zeroinitializer, <4 x i32> [[TMP8]]
+; CHECK-NEXT:    br label %[[L112:.*]]
+; CHECK:       [[L42]]:
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[DOTNOT280:%.*]] = icmp eq i32 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 0>, i32 [[TMP1]], i32 2
+; CHECK-NEXT:    br i1 [[DOTNOT280]], label %[[L112]], label %[[L47:.*]]
+; CHECK:       [[L47]]:
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x ptr> [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x ptr> [[TMP5]], <4 x ptr> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq <2 x ptr> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = load <2 x i32>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP17:%.*]] = select <2 x i1> [[TMP15]], <2 x i32> zeroinitializer, <2 x i32> [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>, i32 [[TMP13]], i32 1
+; CHECK-NEXT:    [[TMP19:%.*]] = call <4 x i32> @llvm.vector.insert.v4i32.v2i32(<4 x i32> [[TMP18]], <2 x i32> [[TMP17]], i64 2)
+; CHECK-NEXT:    br label %[[L112]]
+; CHECK:       [[L112]]:
+; CHECK-NEXT:    [[TMP20:%.*]] = phi <4 x i32> [ [[TMP19]], %[[L47]] ], [ [[TMP9]], %[[L41]] ], [ [[TMP11]], %[[L42]] ]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i32> [[TMP20]], i32 0
+; CHECK-NEXT:    store i32 [[TMP21]], ptr [[P2]], align 4
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x i32> [[TMP20]], i32 1
+; CHECK-NEXT:    store i32 [[TMP22]], ptr [[P1]], align 4
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <4 x i32> [[TMP20]], i32 2
+; CHECK-NEXT:    store i32 [[TMP23]], ptr [[P2]], align 4
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <4 x i32> [[TMP20]], i32 3
+; CHECK-NEXT:    store i32 [[TMP24]], ptr [[P1]], align 4
+; CHECK-NEXT:    ret void
+;
+top:
+  %2 = getelementptr i8, ptr %0, i64 8
+  %3 = getelementptr i8, ptr %0, i64 12
+  %4 = getelementptr i8, ptr %0, i64 16
+  %5 = getelementptr i8, ptr %0, i64 20
+  br i1 %c1, label %L42, label %L41
+
+L41:
+  %.not276 = icmp eq ptr %2, null
+  %6 = load i32, ptr %2, align 4
+  %7 = select i1 %.not276, i32 0, i32 %6
+  %.not277 = icmp eq ptr %3, null
+  %8 = load i32, ptr %3, align 4
+  %9 = select i1 %.not277, i32 0, i32 %8
+  %.not278 = icmp eq ptr %4, null
+  %10 = load i32, ptr %4, align 4
+  %11 = select i1 %.not278, i32 0, i32 %10
+  %.not279 = icmp eq ptr %5, null
+  %12 = load i32, ptr %5, align 4
+  %13 = select i1 %.not279, i32 0, i32 %12
+  br label %L112
+
+L42:
+  %14 = load i32, ptr %2, align 4
+  %.not280 = icmp eq i32 %14, 0
+  br i1 %.not280, label %L112, label %L47
+
+L47:
+  %15 = load i32, ptr %3, align 4
+  %.not282 = icmp eq ptr %4, null
+  %16 = load i32, ptr %4, align 4
+  %17 = select i1 %.not282, i32 0, i32 %16
+  %.not283 = icmp eq ptr %5, null
+  %18 = load i32, ptr %5, align 4
+  %19 = select i1 %.not283, i32 0, i32 %18
+  br label %L112
+
+L112:
+  %value_phi13336 = phi i32 [ %19, %L47 ], [ %13, %L41 ], [ 0, %L42 ]
+  %value_phi12335 = phi i32 [ %17, %L47 ], [ %11, %L41 ], [ %1, %L42 ]
+  %value_phi11334 = phi i32 [ %15, %L47 ], [ %9, %L41 ], [ 0, %L42 ]
+  %value_phi10333 = phi i32 [ 0, %L47 ], [ %7, %L41 ], [ 0, %L42 ]
+  store i32 %value_phi10333, ptr %p2, align 4
+  store i32 %value_phi11334, ptr %p1, align 4
+  store i32 %value_phi12335, ptr %p2, align 4
+  store i32 %value_phi13336, ptr %p1, align 4
+  ret void
+}
-- 
GitLab


From d4c41804175e9cb37266c410cafe9caaac1819ca Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Mon, 28 Oct 2024 07:17:19 -0700
Subject: [PATCH 318/693] [Clang] Add a flag to include GPU startup files
 (#112025)

Summary:
The C library for GPUs provides the ability to target regular C/C++
programs by providing the C library and a file containing kernels that
call the `main` function. This is mostly used for unit tests, this patch
provides a quick way to add them without needing to know the paths. I
currently do this explicitly, but according to the libc++ contributors
we don't want to need to specify these paths manually. See the
discussion in https://github.com/llvm/llvm-project/pull/104515.

I just default to `lib/` if the target-specific one isn't found because
the linker will handle giving a reasonable error message if it's not
found. Basically the use-case looks like this.

```console
$ clang test.c --target=amdgcn-amd-amdhsa -mcpu=native -startfiles -stdlib
$ amdhsa-loader a.out
PASS!
```
---
 clang/include/clang/Driver/Options.td    |  4 ++++
 clang/lib/Driver/ToolChains/AMDGPU.cpp   | 11 +++++++++++
 clang/lib/Driver/ToolChains/Cuda.cpp     | 11 +++++++++++
 clang/test/Driver/amdgpu-toolchain.c     |  4 ++++
 clang/test/Driver/cuda-cross-compiling.c |  8 ++++++++
 5 files changed, 38 insertions(+)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 5df6ddd5e6a0..7b28e8b4c31e 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5633,6 +5633,7 @@ def noprebind : Flag<["-"], "noprebind">;
 def noprofilelib : Flag<["-"], "noprofilelib">;
 def noseglinkedit : Flag<["-"], "noseglinkedit">;
 def nostartfiles : Flag<["-"], "nostartfiles">, Group<Link_Group>;
+def startfiles : Flag<["-"], "startfiles">, Group<Link_Group>;
 def nostdinc : Flag<["-"], "nostdinc">,
   Visibility<[ClangOption, CLOption, DXCOption]>, Group<IncludePath_Group>,
   HelpText<"Disable both standard system #include directories and builtin #include directories">;
@@ -5645,6 +5646,9 @@ def nostdincxx : Flag<["-"], "nostdinc++">, Visibility<[ClangOption, CC1Option]>
 def nostdlib : Flag<["-"], "nostdlib">,
   Visibility<[ClangOption, CLOption, FlangOption, DXCOption]>,
   Group<Link_Group>;
+def stdlib : Flag<["-"], "stdlib">,
+  Visibility<[ClangOption, CLOption, FlangOption, DXCOption]>,
+  Group<Link_Group>;
 def nostdlibxx : Flag<["-"], "nostdlib++">;
 def object : Flag<["-"], "object">;
 def o : JoinedOrSeparate<["-"], "o">,
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index 2c85d21ebd73..a8061ffd9321 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -648,6 +648,17 @@ void amdgpu::Linker::ConstructJob(Compilation &C, const JobAction &JA,
         Args.MakeArgString("-plugin-opt=-mattr=" + llvm::join(Features, ",")));
   }
 
+  if (Args.hasArg(options::OPT_stdlib))
+    CmdArgs.append({"-lc", "-lm"});
+  if (Args.hasArg(options::OPT_startfiles)) {
+    std::optional<std::string> IncludePath = getToolChain().getStdlibPath();
+    if (!IncludePath)
+      IncludePath = "/lib";
+    SmallString<128> P(*IncludePath);
+    llvm::sys::path::append(P, "crt1.o");
+    CmdArgs.push_back(Args.MakeArgString(P));
+  }
+
   CmdArgs.push_back("-o");
   CmdArgs.push_back(Output.getFilename());
   C.addCommand(std::make_unique<Command>(
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index 412b379304b1..ddd5ea248ca0 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -643,6 +643,17 @@ void NVPTX::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   llvm::sys::path::append(DefaultLibPath, CLANG_INSTALL_LIBDIR_BASENAME);
   CmdArgs.push_back(Args.MakeArgString(Twine("-L") + DefaultLibPath));
 
+  if (Args.hasArg(options::OPT_stdlib))
+    CmdArgs.append({"-lc", "-lm"});
+  if (Args.hasArg(options::OPT_startfiles)) {
+    std::optional<std::string> IncludePath = getToolChain().getStdlibPath();
+    if (!IncludePath)
+      IncludePath = "/lib";
+    SmallString<128> P(*IncludePath);
+    llvm::sys::path::append(P, "crt1.o");
+    CmdArgs.push_back(Args.MakeArgString(P));
+  }
+
   C.addCommand(std::make_unique<Command>(
       JA, *this,
       ResponseFileSupport{ResponseFileSupport::RF_Full, llvm::sys::WEM_UTF8,
diff --git a/clang/test/Driver/amdgpu-toolchain.c b/clang/test/Driver/amdgpu-toolchain.c
index b60d31bae627..c1c5aa8e90e6 100644
--- a/clang/test/Driver/amdgpu-toolchain.c
+++ b/clang/test/Driver/amdgpu-toolchain.c
@@ -32,3 +32,7 @@
 // RUN: %clang -### --target=amdgcn-amd-amdhsa -mcpu=gfx906 -nogpulib \
 // RUN:   -r %s 2>&1 | FileCheck -check-prefixes=RELO %s
 // RELO-NOT: -shared
+
+// RUN: %clang -target amdgcn-amd-amdhsa -march=gfx90a -stdlib -startfiles \
+// RUN:   -nogpulib -nogpuinc -### %s 2>&1 | FileCheck -check-prefix=STARTUP %s
+// STARTUP: ld.lld{{.*}}"-lc" "-lm" "{{.*}}crt1.o"
diff --git a/clang/test/Driver/cuda-cross-compiling.c b/clang/test/Driver/cuda-cross-compiling.c
index 126e9e9fc83d..baf370483003 100644
--- a/clang/test/Driver/cuda-cross-compiling.c
+++ b/clang/test/Driver/cuda-cross-compiling.c
@@ -105,3 +105,11 @@
 // RUN:   | FileCheck -check-prefix=FEATURE %s
 
 // FEATURE: clang-nvlink-wrapper{{.*}}"--plugin-opt=-mattr=+ptx63"
+
+//
+// Test including the libc startup files and libc
+//
+// RUN: %clang -target nvptx64-nvidia-cuda -march=sm_61 -stdlib -startfiles \
+// RUN:   -nogpulib -nogpuinc -### %s 2>&1 | FileCheck -check-prefix=STARTUP %s
+
+// STARTUP: clang-nvlink-wrapper{{.*}}"-lc" "-lm" "{{.*}}crt1.o"
-- 
GitLab


From 42eb54b7743df421af10ebe14b67bb79b46ecabb Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Mon, 28 Oct 2024 07:17:50 -0700
Subject: [PATCH 319/693] [Clang] Put offloading globals in the
 `.llvm.rodata.offloading` section (#111890)

Summary:
For our offloading entries, we currently store all the string names of
kernels that the runtime will need to load from the target executable.
These are available via pointer in the `__tgt_offload_entry` struct,
however this makes it difficult to obtain from the object itself. This
patch simply puts the strings in a named section so they can be easily
queried.

The motivation behind this is that when the linker wrapper is doing
linking, it wants to know which kernels the host executable is calling.
We *could* get this already via the `.relaomp_offloading_entires`
section and trawling through the string table, but that's quite annoying
and not portable. The follow-up to this should be to make the linker
wrapper get a list of all used symbols the device link job should count
as "needed" so we can handle static linking more directly.
---
 clang/test/CodeGenCUDA/offloading-entries.cu | 65 ++++++++++++++------
 llvm/lib/Frontend/Offloading/Utility.cpp     |  8 +++
 2 files changed, 53 insertions(+), 20 deletions(-)

diff --git a/clang/test/CodeGenCUDA/offloading-entries.cu b/clang/test/CodeGenCUDA/offloading-entries.cu
index ec21f018607f..259e3324e8ac 100644
--- a/clang/test/CodeGenCUDA/offloading-entries.cu
+++ b/clang/test/CodeGenCUDA/offloading-entries.cu
@@ -15,48 +15,48 @@
 #include "Inputs/cuda.h"
 
 //.
-// CUDA: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00"
+// CUDA: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00", section ".llvm.rodata.offloading", align 1
 // CUDA: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z18__device_stub__foov, ptr @.offloading.entry_name, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries", align 1
-// CUDA: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00"
+// CUDA: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00", section ".llvm.rodata.offloading", align 1
 // CUDA: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z21__device_stub__kernelv, ptr @.offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries", align 1
-// CUDA: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00"
+// CUDA: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00", section ".llvm.rodata.offloading", align 1
 // CUDA: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "cuda_offloading_entries", align 1
-// CUDA: @.offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00"
+// CUDA: @.offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00", section ".llvm.rodata.offloading", align 1
 // CUDA: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.offloading.entry_name.3, i64 4, i32 2, i32 1 }, section "cuda_offloading_entries", align 1
-// CUDA: @.offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00"
+// CUDA: @.offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00", section ".llvm.rodata.offloading", align 1
 // CUDA: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.offloading.entry_name.4, i64 4, i32 3, i32 1 }, section "cuda_offloading_entries", align 1
 //.
-// HIP: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00"
+// HIP: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00", section ".llvm.rodata.offloading", align 1
 // HIP: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z3foov, ptr @.offloading.entry_name, i64 0, i32 0, i32 0 }, section "hip_offloading_entries", align 1
-// HIP: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00"
+// HIP: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00", section ".llvm.rodata.offloading", align 1
 // HIP: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z6kernelv, ptr @.offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "hip_offloading_entries", align 1
-// HIP: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00"
+// HIP: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00", section ".llvm.rodata.offloading", align 1
 // HIP: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "hip_offloading_entries", align 1
-// HIP: @.offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00"
+// HIP: @.offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00", section ".llvm.rodata.offloading", align 1
 // HIP: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.offloading.entry_name.3, i64 4, i32 2, i32 1 }, section "hip_offloading_entries", align 1
-// HIP: @.offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00"
+// HIP: @.offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00", section ".llvm.rodata.offloading", align 1
 // HIP: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.offloading.entry_name.4, i64 4, i32 3, i32 1 }, section "hip_offloading_entries", align 1
 //.
-// CUDA-COFF: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00"
+// CUDA-COFF: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00", section ".llvm.rodata.offloading", align 1
 // CUDA-COFF: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z18__device_stub__foov, ptr @.offloading.entry_name, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1
-// CUDA-COFF: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00"
+// CUDA-COFF: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00", section ".llvm.rodata.offloading", align 1
 // CUDA-COFF: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z21__device_stub__kernelv, ptr @.offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1
-// CUDA-COFF: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00"
+// CUDA-COFF: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00", section ".llvm.rodata.offloading", align 1
 // CUDA-COFF: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1
-// CUDA-COFF: @.offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00"
+// CUDA-COFF: @.offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00", section ".llvm.rodata.offloading", align 1
 // CUDA-COFF: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.offloading.entry_name.3, i64 4, i32 2, i32 1 }, section "cuda_offloading_entries$OE", align 1
-// CUDA-COFF: @.offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00"
+// CUDA-COFF: @.offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00", section ".llvm.rodata.offloading", align 1
 // CUDA-COFF: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.offloading.entry_name.4, i64 4, i32 3, i32 1 }, section "cuda_offloading_entries$OE", align 1
 //.
-// HIP-COFF: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00"
+// HIP-COFF: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00", section ".llvm.rodata.offloading", align 1
 // HIP-COFF: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z3foov, ptr @.offloading.entry_name, i64 0, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1
-// HIP-COFF: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00"
+// HIP-COFF: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00", section ".llvm.rodata.offloading", align 1
 // HIP-COFF: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z6kernelv, ptr @.offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1
-// HIP-COFF: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00"
+// HIP-COFF: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00", section ".llvm.rodata.offloading", align 1
 // HIP-COFF: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1
-// HIP-COFF: @.offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00"
+// HIP-COFF: @.offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00", section ".llvm.rodata.offloading", align 1
 // HIP-COFF: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.offloading.entry_name.3, i64 4, i32 2, i32 1 }, section "hip_offloading_entries$OE", align 1
-// HIP-COFF: @.offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00"
+// HIP-COFF: @.offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00", section ".llvm.rodata.offloading", align 1
 // HIP-COFF: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.offloading.entry_name.4, i64 4, i32 3, i32 1 }, section "hip_offloading_entries$OE", align 1
 //.
 // CUDA-LABEL: @_Z18__device_stub__foov(
@@ -137,3 +137,28 @@ template <typename T, int dim = 1, int mode = 0>
 struct __attribute__((device_builtin_texture_type)) texture : public textureReference {};
 
 texture<void> tex;
+//.
+// CUDA: [[META0:![0-9]+]] = !{ptr @.offloading.entry_name}
+// CUDA: [[META1:![0-9]+]] = !{ptr @.offloading.entry_name.1}
+// CUDA: [[META2:![0-9]+]] = !{ptr @.offloading.entry_name.2}
+// CUDA: [[META3:![0-9]+]] = !{ptr @.offloading.entry_name.3}
+// CUDA: [[META4:![0-9]+]] = !{ptr @.offloading.entry_name.4}
+//.
+// HIP: [[META0:![0-9]+]] = !{ptr @.offloading.entry_name}
+// HIP: [[META1:![0-9]+]] = !{ptr @.offloading.entry_name.1}
+// HIP: [[META2:![0-9]+]] = !{ptr @.offloading.entry_name.2}
+// HIP: [[META3:![0-9]+]] = !{ptr @.offloading.entry_name.3}
+// HIP: [[META4:![0-9]+]] = !{ptr @.offloading.entry_name.4}
+//.
+// CUDA-COFF: [[META0:![0-9]+]] = !{ptr @.offloading.entry_name}
+// CUDA-COFF: [[META1:![0-9]+]] = !{ptr @.offloading.entry_name.1}
+// CUDA-COFF: [[META2:![0-9]+]] = !{ptr @.offloading.entry_name.2}
+// CUDA-COFF: [[META3:![0-9]+]] = !{ptr @.offloading.entry_name.3}
+// CUDA-COFF: [[META4:![0-9]+]] = !{ptr @.offloading.entry_name.4}
+//.
+// HIP-COFF: [[META0:![0-9]+]] = !{ptr @.offloading.entry_name}
+// HIP-COFF: [[META1:![0-9]+]] = !{ptr @.offloading.entry_name.1}
+// HIP-COFF: [[META2:![0-9]+]] = !{ptr @.offloading.entry_name.2}
+// HIP-COFF: [[META3:![0-9]+]] = !{ptr @.offloading.entry_name.3}
+// HIP-COFF: [[META4:![0-9]+]] = !{ptr @.offloading.entry_name.4}
+//.
diff --git a/llvm/lib/Frontend/Offloading/Utility.cpp b/llvm/lib/Frontend/Offloading/Utility.cpp
index 010c0bfd3be7..7a0a7afcfcb5 100644
--- a/llvm/lib/Frontend/Offloading/Utility.cpp
+++ b/llvm/lib/Frontend/Offloading/Utility.cpp
@@ -53,7 +53,15 @@ offloading::getOffloadingEntryInitializer(Module &M, Constant *Addr,
   auto *Str =
       new GlobalVariable(M, AddrName->getType(), /*isConstant=*/true,
                          GlobalValue::InternalLinkage, AddrName, Prefix);
+  StringRef SectionName = ".llvm.rodata.offloading";
   Str->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
+  Str->setSection(SectionName);
+  Str->setAlignment(Align(1));
+
+  // Make a metadata node for these constants so it can be queried from IR.
+  NamedMDNode *MD = M.getOrInsertNamedMetadata("llvm.offloading.symbols");
+  Metadata *MDVals[] = {ConstantAsMetadata::get(Str)};
+  MD->addOperand(llvm::MDNode::get(M.getContext(), MDVals));
 
   // Construct the offloading entry.
   Constant *EntryData[] = {
-- 
GitLab


From bd6ab32e6eb642f2b0b15be8c7c2a668192f07d8 Mon Sep 17 00:00:00 2001
From: Yusuke MINATO <minato.yusuke@fujitsu.com>
Date: Mon, 28 Oct 2024 23:19:20 +0900
Subject: [PATCH 320/693] Revert "[flang] Integrate the option
 -flang-experimental-integer-overflow into -fno-wrapv" (#113901)

Reverts llvm/llvm-project#110063 due to the performance regression on
503.bwaves_r in SPEC2017.
---
 clang/include/clang/Driver/Options.td         |   4 +
 clang/lib/Driver/ToolChains/Flang.cpp         |   1 +
 flang/include/flang/Lower/LoweringOptions.def |   5 +
 .../flang/Optimizer/Transforms/Passes.h       |   4 +-
 .../flang/Optimizer/Transforms/Passes.td      |   2 +-
 flang/include/flang/Tools/CrossToolHelpers.h  |   2 +-
 flang/lib/Frontend/CompilerInvocation.cpp     |   6 +
 flang/lib/Frontend/FrontendActions.cpp        |   4 +-
 flang/lib/Lower/Bridge.cpp                    |   2 +-
 flang/lib/Lower/IO.cpp                        |   2 +-
 flang/lib/Optimizer/Passes/Pipelines.cpp      |   6 +-
 .../Transforms/ControlFlowConverter.cpp       |   8 +-
 flang/test/Driver/frontend-forwarding.f90     |   2 +
 .../Fir/convert-to-llvm-openmp-and-fir.fir    |  10 +-
 flang/test/Fir/loop01.fir                     | 230 +++++++++++++++++-
 flang/test/Fir/loop02.fir                     |   4 +-
 flang/test/Lower/HLFIR/goto-do-body.f90       |   4 +-
 .../OpenMP/parallel-private-clause-fixes.f90  |   4 +-
 ...oop-reduction-allocatable-array-minmax.f90 |   4 +-
 flang/test/Lower/OpenMP/wsloop-variable.f90   |   4 +-
 flang/test/Lower/array-character.f90          |   2 +-
 .../test/Lower/array-derived-assignments.f90  |   2 +-
 flang/test/Lower/array-derived.f90            |   2 +-
 .../array-elemental-calls-char-byval.f90      |   2 +-
 .../test/Lower/array-elemental-calls-char.f90 |   2 +-
 .../Lower/array-expression-assumed-size.f90   |   2 +-
 flang/test/Lower/array-expression-slice-1.f90 |   2 +-
 flang/test/Lower/array-substring.f90          |  46 +++-
 flang/test/Lower/array-temp.f90               |   2 +-
 flang/test/Lower/components.f90               |   2 +-
 flang/test/Lower/do_loop.f90                  |  73 ++++--
 flang/test/Lower/do_loop_unstructured.f90     | 203 +++++++++++++++-
 flang/test/Lower/goto-do-body.f90             |   4 +-
 flang/test/Lower/host-associated.f90          |   2 +-
 flang/test/Lower/infinite_loop.f90            |  41 +++-
 flang/test/Lower/io-implied-do-fixes.f90      |  49 +++-
 flang/test/Lower/loops2.f90                   |   2 +-
 flang/test/Lower/mixed_loops.f90              |   6 +-
 flang/test/Lower/vector-subscript-io.f90      |  34 +--
 flang/tools/bbc/bbc.cpp                       |   9 +-
 40 files changed, 676 insertions(+), 119 deletions(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 7b28e8b4c31e..f86e90e14c47 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -6791,6 +6791,10 @@ def flang_deprecated_no_hlfir : Flag<["-"], "flang-deprecated-no-hlfir">,
   Flags<[HelpHidden]>, Visibility<[FlangOption, FC1Option]>,
   HelpText<"Do not use HLFIR lowering (deprecated)">;
 
+def flang_experimental_integer_overflow : Flag<["-"], "flang-experimental-integer-overflow">,
+  Flags<[HelpHidden]>, Visibility<[FlangOption, FC1Option]>,
+  HelpText<"Add nsw flag to internal operations such as do-variable increment (experimental)">;
+
 //===----------------------------------------------------------------------===//
 // FLangOption + CoreOption + NoXarchOption
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 366cadc2e547..a9d2b7a4dc48 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -148,6 +148,7 @@ void Flang::addCodegenOptions(const ArgList &Args,
 
   Args.addAllArgs(CmdArgs, {options::OPT_flang_experimental_hlfir,
                             options::OPT_flang_deprecated_no_hlfir,
+                            options::OPT_flang_experimental_integer_overflow,
                             options::OPT_fno_ppc_native_vec_elem_order,
                             options::OPT_fppc_native_vec_elem_order});
 }
diff --git a/flang/include/flang/Lower/LoweringOptions.def b/flang/include/flang/Lower/LoweringOptions.def
index 0b22e54b648e..231de533fbd3 100644
--- a/flang/include/flang/Lower/LoweringOptions.def
+++ b/flang/include/flang/Lower/LoweringOptions.def
@@ -38,5 +38,10 @@ ENUM_LOWERINGOPT(Underscoring, unsigned, 1, 1)
 /// (i.e. wraps around as two's complement). Off by default.
 ENUM_LOWERINGOPT(IntegerWrapAround, unsigned, 1, 0)
 
+/// If true, add nsw flags to loop variable increments.
+/// Off by default.
+/// TODO: integrate this option with the above
+ENUM_LOWERINGOPT(NSWOnLoopVarInc, unsigned, 1, 0)
+
 #undef LOWERINGOPT
 #undef ENUM_LOWERINGOPT
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
index e1453cfa374b..e8f0a8444a31 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.h
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -66,7 +66,7 @@ namespace fir {
 std::unique_ptr<mlir::Pass> createAffineDemotionPass();
 std::unique_ptr<mlir::Pass>
 createArrayValueCopyPass(fir::ArrayValueCopyOptions options = {});
-std::unique_ptr<mlir::Pass> createCFGConversionPassWithoutNSW();
+std::unique_ptr<mlir::Pass> createCFGConversionPassWithNSW();
 std::unique_ptr<mlir::Pass> createMemDataFlowOptPass();
 std::unique_ptr<mlir::Pass> createPromoteToAffinePass();
 std::unique_ptr<mlir::Pass>
@@ -83,7 +83,7 @@ createVScaleAttrPass(std::pair<unsigned, unsigned> vscaleAttr);
 
 void populateCfgConversionRewrites(mlir::RewritePatternSet &patterns,
                                    bool forceLoopToExecuteOnce = false,
-                                   bool setNSW = true);
+                                   bool setNSW = false);
 
 // declarative passes
 #define GEN_PASS_REGISTRATION
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
index 54b43adbfc6b..a41f0f348f27 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -153,7 +153,7 @@ def CFGConversion : Pass<"cfg-conversion"> {
            /*default=*/"false",
            "force the body of a loop to execute at least once">,
     Option<"setNSW", "set-nsw", "bool",
-           /*default=*/"true",
+           /*default=*/"false",
            "set nsw on loop variable increment">
   ];
 }
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index 1626970600ef..df4b21ada058 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -122,7 +122,7 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
   bool NoSignedZerosFPMath =
       false; ///< Set no-signed-zeros-fp-math attribute for functions.
   bool UnsafeFPMath = false; ///< Set unsafe-fp-math attribute for functions.
-  bool NSWOnLoopVarInc = true; ///< Add nsw flag to loop variable increments.
+  bool NSWOnLoopVarInc = false; ///< Add nsw flag to loop variable increments.
 };
 
 struct OffloadModuleOpts {
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 5da5236af2b0..94d3d1154178 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1350,6 +1350,12 @@ bool CompilerInvocation::createFromArgs(
     invoc.loweringOpts.setNoPPCNativeVecElemOrder(true);
   }
 
+  // -flang-experimental-integer-overflow
+  if (args.hasArg(
+          clang::driver::options::OPT_flang_experimental_integer_overflow)) {
+    invoc.loweringOpts.setNSWOnLoopVarInc(true);
+  }
+
   // Preserve all the remark options requested, i.e. -Rpass, -Rpass-missed or
   // -Rpass-analysis. This will be used later when processing and outputting the
   // remarks generated by LLVM in ExecuteCompilerInvocation.cpp.
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index 3ea242315484..f2e460fc53a6 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -828,8 +828,8 @@ void CodeGenAction::generateLLVMIR() {
     config.VScaleMax = vsr->second;
   }
 
-  if (ci.getInvocation().getLoweringOpts().getIntegerWrapAround())
-    config.NSWOnLoopVarInc = false;
+  if (ci.getInvocation().getLoweringOpts().getNSWOnLoopVarInc())
+    config.NSWOnLoopVarInc = true;
 
   // Create the pass pipeline
   fir::createMLIRToLLVMPassPipeline(pm, config, getCurrentFile());
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index a3bd1ace11da..877fe122265d 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -2271,7 +2271,7 @@ private:
     assert(!incrementLoopNestInfo.empty() && "empty loop nest");
     mlir::Location loc = toLocation();
     mlir::arith::IntegerOverflowFlags flags{};
-    if (!getLoweringOptions().getIntegerWrapAround())
+    if (getLoweringOptions().getNSWOnLoopVarInc())
       flags = bitEnumSet(flags, mlir::arith::IntegerOverflowFlags::nsw);
     auto iofAttr = mlir::arith::IntegerOverflowFlagsAttr::get(
         builder->getContext(), flags);
diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index b534c81a605a..1894b0cfd1be 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -929,7 +929,7 @@ static void genIoLoop(Fortran::lower::AbstractConverter &converter,
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   mlir::Location loc = converter.getCurrentLocation();
   mlir::arith::IntegerOverflowFlags flags{};
-  if (!converter.getLoweringOptions().getIntegerWrapAround())
+  if (converter.getLoweringOptions().getNSWOnLoopVarInc())
     flags = bitEnumSet(flags, mlir::arith::IntegerOverflowFlags::nsw);
   auto iofAttr =
       mlir::arith::IntegerOverflowFlagsAttr::get(builder.getContext(), flags);
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 091b7c4c164e..3c139f7e9340 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -35,11 +35,11 @@ void addCanonicalizerPassWithoutRegionSimplification(mlir::OpPassManager &pm) {
 void addCfgConversionPass(mlir::PassManager &pm,
                           const MLIRToLLVMPassPipelineConfig &config) {
   if (config.NSWOnLoopVarInc)
+    addNestedPassToAllTopLevelOperationsConditionally(
+        pm, disableCfgConversion, fir::createCFGConversionPassWithNSW);
+  else
     addNestedPassToAllTopLevelOperationsConditionally(pm, disableCfgConversion,
                                                       fir::createCFGConversion);
-  else
-    addNestedPassToAllTopLevelOperationsConditionally(
-        pm, disableCfgConversion, fir::createCFGConversionPassWithoutNSW);
 }
 
 void addAVC(mlir::PassManager &pm, const llvm::OptimizationLevel &optLevel) {
diff --git a/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp b/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
index 411bf7f364a6..3b79d6d311b7 100644
--- a/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
+++ b/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
@@ -332,6 +332,8 @@ class CfgConversion : public fir::impl::CFGConversionBase<CfgConversion> {
 public:
   using CFGConversionBase<CfgConversion>::CFGConversionBase;
 
+  CfgConversion(bool setNSW) { this->setNSW = setNSW; }
+
   void runOnOperation() override {
     auto *context = &this->getContext();
     mlir::RewritePatternSet patterns(context);
@@ -364,8 +366,6 @@ void fir::populateCfgConversionRewrites(mlir::RewritePatternSet &patterns,
       patterns.getContext(), forceLoopToExecuteOnce, setNSW);
 }
 
-std::unique_ptr<mlir::Pass> fir::createCFGConversionPassWithoutNSW() {
-  fir::CFGConversionOptions options;
-  options.setNSW = false;
-  return fir::createCFGConversion(options);
+std::unique_ptr<mlir::Pass> fir::createCFGConversionPassWithNSW() {
+  return std::make_unique<CfgConversion>(true);
 }
diff --git a/flang/test/Driver/frontend-forwarding.f90 b/flang/test/Driver/frontend-forwarding.f90
index 55a74ccf4046..ff2d66095214 100644
--- a/flang/test/Driver/frontend-forwarding.f90
+++ b/flang/test/Driver/frontend-forwarding.f90
@@ -20,6 +20,7 @@
 ! RUN:     -fversion-loops-for-stride \
 ! RUN:     -flang-experimental-hlfir \
 ! RUN:     -flang-deprecated-no-hlfir \
+! RUN:     -flang-experimental-integer-overflow \
 ! RUN:     -fno-ppc-native-vector-element-order \
 ! RUN:     -fppc-native-vector-element-order \
 ! RUN:     -mllvm -print-before-all \
@@ -51,6 +52,7 @@
 ! CHECK: "-fversion-loops-for-stride"
 ! CHECK: "-flang-experimental-hlfir"
 ! CHECK: "-flang-deprecated-no-hlfir"
+! CHECK: "-flang-experimental-integer-overflow"
 ! CHECK: "-fno-ppc-native-vector-element-order"
 ! CHECK: "-fppc-native-vector-element-order"
 ! CHECK: "-Rpass"
diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index 00f8e6e6cc9a..335877e7c9a8 100644
--- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -358,10 +358,10 @@ func.func @_QPopenmp_target_data_region() {
       %9 = arith.subi %8, %c1_i64 : i64
       %10 = fir.coordinate_of %0, %9 : (!fir.ref<!fir.array<1024xi32>>, i64) -> !fir.ref<i32>
       fir.store %6 to %10 : !fir.ref<i32>
-      %11 = arith.addi %arg0, %c1 overflow<nsw> : index
+      %11 = arith.addi %arg0, %c1 : index
       %12 = fir.convert %c1 : (index) -> i32
       %13 = fir.load %1 : !fir.ref<i32>
-      %14 = arith.addi %13, %12 overflow<nsw> : i32
+      %14 = arith.addi %13, %12 : i32
       fir.result %11, %14 : index, i32
     }
     fir.store %5#1 to %1 : !fir.ref<i32>
@@ -404,11 +404,11 @@ func.func @_QPopenmp_target_data_region() {
 // CHECK:             %[[VAL_21:.*]] = llvm.sub %[[VAL_19]], %[[VAL_20]]  : i64
 // CHECK:             %[[VAL_22:.*]] = llvm.getelementptr %[[VAL_1]][0, %[[VAL_21]]] : (!llvm.ptr, i64) -> !llvm.ptr
 // CHECK:             llvm.store %[[VAL_17]], %[[VAL_22]] : i32, !llvm.ptr
-// CHECK:             %[[VAL_23:.*]] = llvm.add %[[VAL_12]], %[[VAL_8]] overflow<nsw> : i64
+// CHECK:             %[[VAL_23:.*]] = llvm.add %[[VAL_12]], %[[VAL_8]]  : i64
 // CHECK:             %[[VAL_24:.*]] = llvm.trunc %[[VAL_8]] : i64 to i32
 // CHECK:             %[[VAL_25:.*]] = llvm.load %[[VAL_3]] : !llvm.ptr -> i32
-// CHECK:             %[[VAL_26:.*]] = llvm.add %[[VAL_25]], %[[VAL_24]] overflow<nsw> : i32
-// CHECK:             %[[VAL_27:.*]] = llvm.add %[[VAL_12]], %[[VAL_8]] overflow<nsw> : i64
+// CHECK:             %[[VAL_26:.*]] = llvm.add %[[VAL_25]], %[[VAL_24]]  : i32
+// CHECK:             %[[VAL_27:.*]] = llvm.add %[[VAL_12]], %[[VAL_8]]  : i64
 // CHECK:             %[[VAL_28:.*]] = llvm.mlir.constant(1 : index) : i64
 // CHECK:             %[[VAL_29:.*]] = llvm.sub %[[VAL_14]], %[[VAL_28]]  : i64
 // CHECK:             llvm.br ^bb1(%[[VAL_27]], %[[VAL_26]], %[[VAL_29]] : i64, i32, i64)
diff --git a/flang/test/Fir/loop01.fir b/flang/test/Fir/loop01.fir
index 30d10b9bbdb9..c1cbb522c378 100644
--- a/flang/test/Fir/loop01.fir
+++ b/flang/test/Fir/loop01.fir
@@ -1,7 +1,5 @@
 // RUN: fir-opt --split-input-file --cfg-conversion %s | FileCheck %s
-// RUN: fir-opt --split-input-file --cfg-conversion="set-nsw=false" %s | FileCheck %s --check-prefix=NO-NSW
-
-// NO-NSW-NOT: overflow<nsw>
+// RUN: fir-opt --split-input-file --cfg-conversion="set-nsw=true" %s | FileCheck %s --check-prefix=NSW
 
 func.func @x(%lb : index, %ub : index, %step : index, %b : i1, %addr : !fir.ref<index>) {
   fir.do_loop %iv = %lb to %ub step %step unordered {
@@ -37,7 +35,7 @@ func.func private @f2() -> i1
 // CHECK:       fir.store %[[VAL_12]] to %[[VAL_4]] : !fir.ref<index>
 // CHECK:       br ^bb5
 // CHECK:     ^bb5:
-// CHECK:       %[[VAL_13:.*]] = arith.addi %[[VAL_8]], %[[VAL_2]] overflow<nsw> : index
+// CHECK:       %[[VAL_13:.*]] = arith.addi %[[VAL_8]], %[[VAL_2]] : index
 // CHECK:       %[[VAL_14:.*]] = arith.constant 1 : index
 // CHECK:       %[[VAL_15:.*]] = arith.subi %[[VAL_9]], %[[VAL_14]] : index
 // CHECK:       br ^bb1(%[[VAL_13]], %[[VAL_15]] : index, index)
@@ -46,6 +44,34 @@ func.func private @f2() -> i1
 // CHECK:     }
 // CHECK:     func private @f2() -> i1
 
+// NSW:     func @x(%[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: index, %[[VAL_3:.*]]: i1, %[[VAL_4:.*]]: !fir.ref<index>) {
+// NSW:       %[[VAL_5:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
+// NSW:       %[[VAL_6:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] : index
+// NSW:       %[[VAL_7:.*]] = arith.divsi %[[VAL_6]], %[[VAL_2]] : index
+// NSW:       br ^bb1(%[[VAL_0]], %[[VAL_7]] : index, index)
+// NSW:     ^bb1(%[[VAL_8:.*]]: index, %[[VAL_9:.*]]: index):
+// NSW:       %[[VAL_10:.*]] = arith.constant 0 : index
+// NSW:       %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_10]] : index
+// NSW:       cond_br %[[VAL_11]], ^bb2, ^bb6
+// NSW:     ^bb2:
+// NSW:       cond_br %[[VAL_3]], ^bb3, ^bb4
+// NSW:     ^bb3:
+// NSW:       fir.store %[[VAL_8]] to %[[VAL_4]] : !fir.ref<index>
+// NSW:       br ^bb5
+// NSW:     ^bb4:
+// NSW:       %[[VAL_12:.*]] = arith.constant 0 : index
+// NSW:       fir.store %[[VAL_12]] to %[[VAL_4]] : !fir.ref<index>
+// NSW:       br ^bb5
+// NSW:     ^bb5:
+// NSW:       %[[VAL_13:.*]] = arith.addi %[[VAL_8]], %[[VAL_2]] overflow<nsw> : index
+// NSW:       %[[VAL_14:.*]] = arith.constant 1 : index
+// NSW:       %[[VAL_15:.*]] = arith.subi %[[VAL_9]], %[[VAL_14]] : index
+// NSW:       br ^bb1(%[[VAL_13]], %[[VAL_15]] : index, index)
+// NSW:     ^bb6:
+// NSW:       return
+// NSW:     }
+// NSW:     func private @f2() -> i1
+
 // -----
 
 func.func @x2(%lo : index, %up : index, %ok : i1) {
@@ -75,13 +101,36 @@ func.func private @f3(i16)
 // CHECK:     cond_br %[[VAL_14]], ^bb2, ^bb3
 // CHECK:   ^bb2:
 // CHECK:     %[[VAL_15:.*]] = fir.call @f2() : () -> i1
-// CHECK:     %[[VAL_16:.*]] = arith.addi %[[VAL_4]], %[[VAL_3]] overflow<nsw> : index
+// CHECK:     %[[VAL_16:.*]] = arith.addi %[[VAL_4]], %[[VAL_3]] : index
 // CHECK:     br ^bb1(%[[VAL_16]], %[[VAL_15]] : index, i1)
 // CHECK:   ^bb3:
 // CHECK:     return
 // CHECK:   }
 // CHECK:   func private @f3(i16)
 
+// NSW:   func @x2(%[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: i1) {
+// NSW:     %[[VAL_3:.*]] = arith.constant 1 : index
+// NSW:     br ^bb1(%[[VAL_0]], %[[VAL_2]] : index, i1)
+// NSW:   ^bb1(%[[VAL_4:.*]]: index, %[[VAL_5:.*]]: i1):
+// NSW:     %[[VAL_6:.*]] = arith.constant 0 : index
+// NSW:     %[[VAL_7:.*]] = arith.cmpi slt, %[[VAL_6]], %[[VAL_3]] : index
+// NSW:     %[[VAL_8:.*]] = arith.cmpi sle, %[[VAL_4]], %[[VAL_1]] : index
+// NSW:     %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_3]], %[[VAL_6]] : index
+// NSW:     %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_4]] : index
+// NSW:     %[[VAL_11:.*]] = arith.andi %[[VAL_7]], %[[VAL_8]] : i1
+// NSW:     %[[VAL_12:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
+// NSW:     %[[VAL_13:.*]] = arith.ori %[[VAL_11]], %[[VAL_12]] : i1
+// NSW:     %[[VAL_14:.*]] = arith.andi %[[VAL_5]], %[[VAL_13]] : i1
+// NSW:     cond_br %[[VAL_14]], ^bb2, ^bb3
+// NSW:   ^bb2:
+// NSW:     %[[VAL_15:.*]] = fir.call @f2() : () -> i1
+// NSW:     %[[VAL_16:.*]] = arith.addi %[[VAL_4]], %[[VAL_3]] overflow<nsw> : index
+// NSW:     br ^bb1(%[[VAL_16]], %[[VAL_15]] : index, i1)
+// NSW:   ^bb3:
+// NSW:     return
+// NSW:   }
+// NSW:   func private @f3(i16)
+
 // -----
 
 // do_loop with an extra loop-carried value
@@ -110,7 +159,7 @@ func.func @x3(%lo : index, %up : index) -> i1 {
 // CHECK:           cond_br %[[VAL_11]], ^bb2, ^bb3
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_12:.*]] = fir.call @f2() : () -> i1
-// CHECK:           %[[VAL_13:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] overflow<nsw> : index
+// CHECK:           %[[VAL_13:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] : index
 // CHECK:           %[[VAL_14:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_15:.*]] = arith.subi %[[VAL_9]], %[[VAL_14]] : index
 // CHECK:           br ^bb1(%[[VAL_13]], %[[VAL_12]], %[[VAL_15]] : index, i1, index)
@@ -118,6 +167,29 @@ func.func @x3(%lo : index, %up : index) -> i1 {
 // CHECK:           return %[[VAL_8]] : i1
 // CHECK:         }
 
+// NSW-LABEL:   func @x3(
+// NSW-SAME:             %[[VAL_0:.*]]: index,
+// NSW-SAME:             %[[VAL_1:.*]]: index) -> i1 {
+// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_3:.*]] = arith.constant true
+// NSW:           %[[VAL_4:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
+// NSW:           %[[VAL_5:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] : index
+// NSW:           %[[VAL_6:.*]] = arith.divsi %[[VAL_5]], %[[VAL_2]] : index
+// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]], %[[VAL_6]] : index, i1, index)
+// NSW:         ^bb1(%[[VAL_7:.*]]: index, %[[VAL_8:.*]]: i1, %[[VAL_9:.*]]: index):
+// NSW:           %[[VAL_10:.*]] = arith.constant 0 : index
+// NSW:           %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_10]] : index
+// NSW:           cond_br %[[VAL_11]], ^bb2, ^bb3
+// NSW:         ^bb2:
+// NSW:           %[[VAL_12:.*]] = fir.call @f2() : () -> i1
+// NSW:           %[[VAL_13:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] overflow<nsw> : index
+// NSW:           %[[VAL_14:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_15:.*]] = arith.subi %[[VAL_9]], %[[VAL_14]] : index
+// NSW:           br ^bb1(%[[VAL_13]], %[[VAL_12]], %[[VAL_15]] : index, i1, index)
+// NSW:         ^bb3:
+// NSW:           return %[[VAL_8]] : i1
+// NSW:         }
+
 // -----
 
 // iterate_while with an extra loop-carried value
@@ -155,7 +227,7 @@ func.func private @f4(i32) -> i1
 // CHECK:           cond_br %[[VAL_16]], ^bb2, ^bb3
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_17:.*]] = fir.call @f2() : () -> i1
-// CHECK:           %[[VAL_18:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] overflow<nsw> : index
+// CHECK:           %[[VAL_18:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] : index
 // CHECK:           br ^bb1(%[[VAL_18]], %[[VAL_6]], %[[VAL_17]] : index, i1, i1)
 // CHECK:         ^bb3:
 // CHECK:           %[[VAL_19:.*]] = arith.andi %[[VAL_6]], %[[VAL_7]] : i1
@@ -163,6 +235,34 @@ func.func private @f4(i32) -> i1
 // CHECK:         }
 // CHECK:         func private @f4(i32) -> i1
 
+// NSW-LABEL:   func @y3(
+// NSW-SAME:             %[[VAL_0:.*]]: index,
+// NSW-SAME:             %[[VAL_1:.*]]: index) -> i1 {
+// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_3:.*]] = arith.constant true
+// NSW:           %[[VAL_4:.*]] = fir.call @f2() : () -> i1
+// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]], %[[VAL_4]] : index, i1, i1)
+// NSW:         ^bb1(%[[VAL_5:.*]]: index, %[[VAL_6:.*]]: i1, %[[VAL_7:.*]]: i1):
+// NSW:           %[[VAL_8:.*]] = arith.constant 0 : index
+// NSW:           %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_8]], %[[VAL_2]] : index
+// NSW:           %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_5]], %[[VAL_1]] : index
+// NSW:           %[[VAL_11:.*]] = arith.cmpi slt, %[[VAL_2]], %[[VAL_8]] : index
+// NSW:           %[[VAL_12:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_5]] : index
+// NSW:           %[[VAL_13:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
+// NSW:           %[[VAL_14:.*]] = arith.andi %[[VAL_11]], %[[VAL_12]] : i1
+// NSW:           %[[VAL_15:.*]] = arith.ori %[[VAL_13]], %[[VAL_14]] : i1
+// NSW:           %[[VAL_16:.*]] = arith.andi %[[VAL_6]], %[[VAL_15]] : i1
+// NSW:           cond_br %[[VAL_16]], ^bb2, ^bb3
+// NSW:         ^bb2:
+// NSW:           %[[VAL_17:.*]] = fir.call @f2() : () -> i1
+// NSW:           %[[VAL_18:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] overflow<nsw> : index
+// NSW:           br ^bb1(%[[VAL_18]], %[[VAL_6]], %[[VAL_17]] : index, i1, i1)
+// NSW:         ^bb3:
+// NSW:           %[[VAL_19:.*]] = arith.andi %[[VAL_6]], %[[VAL_7]] : i1
+// NSW:           return %[[VAL_19]] : i1
+// NSW:         }
+// NSW:         func private @f4(i32) -> i1
+
 // -----
 
 // do_loop that returns the final value of the induction
@@ -191,7 +291,7 @@ func.func @x4(%lo : index, %up : index) -> index {
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_6]] : (index) -> i32
 // CHECK:           %[[VAL_11:.*]] = fir.call @f4(%[[VAL_10]]) : (i32) -> i1
-// CHECK:           %[[VAL_12:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] overflow<nsw> : index
+// CHECK:           %[[VAL_12:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] : index
 // CHECK:           %[[VAL_13:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_14:.*]] = arith.subi %[[VAL_7]], %[[VAL_13]] : index
 // CHECK:           br ^bb1(%[[VAL_12]], %[[VAL_14]] : index, index)
@@ -199,6 +299,29 @@ func.func @x4(%lo : index, %up : index) -> index {
 // CHECK:           return %[[VAL_6]] : index
 // CHECK:         }
 
+// NSW-LABEL:   func @x4(
+// NSW-SAME:             %[[VAL_0:.*]]: index,
+// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
+// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_3:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
+// NSW:           %[[VAL_4:.*]] = arith.addi %[[VAL_3]], %[[VAL_2]] : index
+// NSW:           %[[VAL_5:.*]] = arith.divsi %[[VAL_4]], %[[VAL_2]] : index
+// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_5]] : index, index)
+// NSW:         ^bb1(%[[VAL_6:.*]]: index, %[[VAL_7:.*]]: index):
+// NSW:           %[[VAL_8:.*]] = arith.constant 0 : index
+// NSW:           %[[VAL_9:.*]] = arith.cmpi sgt, %[[VAL_7]], %[[VAL_8]] : index
+// NSW:           cond_br %[[VAL_9]], ^bb2, ^bb3
+// NSW:         ^bb2:
+// NSW:           %[[VAL_10:.*]] = fir.convert %[[VAL_6]] : (index) -> i32
+// NSW:           %[[VAL_11:.*]] = fir.call @f4(%[[VAL_10]]) : (i32) -> i1
+// NSW:           %[[VAL_12:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] overflow<nsw> : index
+// NSW:           %[[VAL_13:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_14:.*]] = arith.subi %[[VAL_7]], %[[VAL_13]] : index
+// NSW:           br ^bb1(%[[VAL_12]], %[[VAL_14]] : index, index)
+// NSW:         ^bb3:
+// NSW:           return %[[VAL_6]] : index
+// NSW:         }
+
 // -----
 
 // iterate_while that returns the final value of both inductions
@@ -233,12 +356,38 @@ func.func @y4(%lo : index, %up : index) -> index {
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
 // CHECK:           %[[VAL_16:.*]] = fir.call @f4(%[[VAL_15]]) : (i32) -> i1
-// CHECK:           %[[VAL_17:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] overflow<nsw> : index
+// CHECK:           %[[VAL_17:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] : index
 // CHECK:           br ^bb1(%[[VAL_17]], %[[VAL_16]] : index, i1)
 // CHECK:         ^bb3:
 // CHECK:           return %[[VAL_4]] : index
 // CHECK:         }
 
+// NSW-LABEL:   func @y4(
+// NSW-SAME:             %[[VAL_0:.*]]: index,
+// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
+// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_3:.*]] = arith.constant true
+// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]] : index, i1)
+// NSW:         ^bb1(%[[VAL_4:.*]]: index, %[[VAL_5:.*]]: i1):
+// NSW:           %[[VAL_6:.*]] = arith.constant 0 : index
+// NSW:           %[[VAL_7:.*]] = arith.cmpi slt, %[[VAL_6]], %[[VAL_2]] : index
+// NSW:           %[[VAL_8:.*]] = arith.cmpi sle, %[[VAL_4]], %[[VAL_1]] : index
+// NSW:           %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_2]], %[[VAL_6]] : index
+// NSW:           %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_4]] : index
+// NSW:           %[[VAL_11:.*]] = arith.andi %[[VAL_7]], %[[VAL_8]] : i1
+// NSW:           %[[VAL_12:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
+// NSW:           %[[VAL_13:.*]] = arith.ori %[[VAL_11]], %[[VAL_12]] : i1
+// NSW:           %[[VAL_14:.*]] = arith.andi %[[VAL_5]], %[[VAL_13]] : i1
+// NSW:           cond_br %[[VAL_14]], ^bb2, ^bb3
+// NSW:         ^bb2:
+// NSW:           %[[VAL_15:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
+// NSW:           %[[VAL_16:.*]] = fir.call @f4(%[[VAL_15]]) : (i32) -> i1
+// NSW:           %[[VAL_17:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] overflow<nsw> : index
+// NSW:           br ^bb1(%[[VAL_17]], %[[VAL_16]] : index, i1)
+// NSW:         ^bb3:
+// NSW:           return %[[VAL_4]] : index
+// NSW:         }
+
 // -----
 
 // do_loop that returns the final induction value
@@ -271,7 +420,7 @@ func.func @x5(%lo : index, %up : index) -> index {
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_12:.*]] = fir.call @f2() : () -> i1
 // CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i1) -> i16
-// CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] overflow<nsw> : index
+// CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] : index
 // CHECK:           %[[VAL_15:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_16:.*]] = arith.subi %[[VAL_9]], %[[VAL_15]] : index
 // CHECK:           br ^bb1(%[[VAL_14]], %[[VAL_13]], %[[VAL_16]] : index, i16, index)
@@ -280,6 +429,31 @@ func.func @x5(%lo : index, %up : index) -> index {
 // CHECK:           return %[[VAL_7]] : index
 // CHECK:         }
 
+// NSW-LABEL:   func @x5(
+// NSW-SAME:             %[[VAL_0:.*]]: index,
+// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
+// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_3:.*]] = arith.constant 42 : i16
+// NSW:           %[[VAL_4:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
+// NSW:           %[[VAL_5:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] : index
+// NSW:           %[[VAL_6:.*]] = arith.divsi %[[VAL_5]], %[[VAL_2]] : index
+// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]], %[[VAL_6]] : index, i16, index)
+// NSW:         ^bb1(%[[VAL_7:.*]]: index, %[[VAL_8:.*]]: i16, %[[VAL_9:.*]]: index):
+// NSW:           %[[VAL_10:.*]] = arith.constant 0 : index
+// NSW:           %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_10]] : index
+// NSW:           cond_br %[[VAL_11]], ^bb2, ^bb3
+// NSW:         ^bb2:
+// NSW:           %[[VAL_12:.*]] = fir.call @f2() : () -> i1
+// NSW:           %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i1) -> i16
+// NSW:           %[[VAL_14:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] overflow<nsw> : index
+// NSW:           %[[VAL_15:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_16:.*]] = arith.subi %[[VAL_9]], %[[VAL_15]] : index
+// NSW:           br ^bb1(%[[VAL_14]], %[[VAL_13]], %[[VAL_16]] : index, i16, index)
+// NSW:         ^bb3:
+// NSW:           fir.call @f3(%[[VAL_8]]) : (i16) -> ()
+// NSW:           return %[[VAL_7]] : index
+// NSW:         }
+
 // -----
 
 // iterate_while that returns the both induction values
@@ -322,7 +496,7 @@ func.func @y5(%lo : index, %up : index) -> index {
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_17:.*]] = fir.call @f2() : () -> i1
 // CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (i1) -> i16
-// CHECK:           %[[VAL_19:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] overflow<nsw> : index
+// CHECK:           %[[VAL_19:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] : index
 // CHECK:           br ^bb1(%[[VAL_19]], %[[VAL_17]], %[[VAL_18]] : index, i1, i16)
 // CHECK:         ^bb3:
 // CHECK:           cond_br %[[VAL_6]], ^bb4, ^bb5
@@ -334,3 +508,37 @@ func.func @y5(%lo : index, %up : index) -> index {
 // CHECK:           fir.call @f3(%[[VAL_7]]) : (i16) -> ()
 // CHECK:           return %[[VAL_5]] : index
 // CHECK:         }
+
+// NSW-LABEL:   func @y5(
+// NSW-SAME:             %[[VAL_0:.*]]: index,
+// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
+// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_3:.*]] = arith.constant 42 : i16
+// NSW:           %[[VAL_4:.*]] = arith.constant true
+// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_4]], %[[VAL_3]] : index, i1, i16)
+// NSW:         ^bb1(%[[VAL_5:.*]]: index, %[[VAL_6:.*]]: i1, %[[VAL_7:.*]]: i16):
+// NSW:           %[[VAL_8:.*]] = arith.constant 0 : index
+// NSW:           %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_8]], %[[VAL_2]] : index
+// NSW:           %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_5]], %[[VAL_1]] : index
+// NSW:           %[[VAL_11:.*]] = arith.cmpi slt, %[[VAL_2]], %[[VAL_8]] : index
+// NSW:           %[[VAL_12:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_5]] : index
+// NSW:           %[[VAL_13:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
+// NSW:           %[[VAL_14:.*]] = arith.andi %[[VAL_11]], %[[VAL_12]] : i1
+// NSW:           %[[VAL_15:.*]] = arith.ori %[[VAL_13]], %[[VAL_14]] : i1
+// NSW:           %[[VAL_16:.*]] = arith.andi %[[VAL_6]], %[[VAL_15]] : i1
+// NSW:           cond_br %[[VAL_16]], ^bb2, ^bb3
+// NSW:         ^bb2:
+// NSW:           %[[VAL_17:.*]] = fir.call @f2() : () -> i1
+// NSW:           %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (i1) -> i16
+// NSW:           %[[VAL_19:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] overflow<nsw> : index
+// NSW:           br ^bb1(%[[VAL_19]], %[[VAL_17]], %[[VAL_18]] : index, i1, i16)
+// NSW:         ^bb3:
+// NSW:           cond_br %[[VAL_6]], ^bb4, ^bb5
+// NSW:         ^bb4:
+// NSW:           %[[VAL_20:.*]] = arith.constant 0 : i32
+// NSW:           %[[VAL_21:.*]] = fir.call @f4(%[[VAL_20]]) : (i32) -> i1
+// NSW:           br ^bb5
+// NSW:         ^bb5:
+// NSW:           fir.call @f3(%[[VAL_7]]) : (i16) -> ()
+// NSW:           return %[[VAL_5]] : index
+// NSW:         }
diff --git a/flang/test/Fir/loop02.fir b/flang/test/Fir/loop02.fir
index fb209a9dfeb4..50948e0e7aa6 100644
--- a/flang/test/Fir/loop02.fir
+++ b/flang/test/Fir/loop02.fir
@@ -31,7 +31,7 @@ func.func private @y(%addr : !fir.ref<index>)
 // CHECK:           cond_br %[[VAL_13]], ^bb2, ^bb3
 // CHECK:         ^bb2:
 // CHECK:           fir.call @y(%[[VAL_0]]) : (!fir.ref<index>) -> ()
-// CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_10]], %[[VAL_2]] overflow<nsw> : index
+// CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_10]], %[[VAL_2]] : index
 // CHECK:           %[[VAL_15:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_16:.*]] = arith.subi %[[VAL_11]], %[[VAL_15]] : index
 // CHECK:           br ^bb1(%[[VAL_14]], %[[VAL_16]] : index, index)
@@ -54,7 +54,7 @@ func.func private @y(%addr : !fir.ref<index>)
 // NOOPT:           cond_br %[[VAL_9]], ^bb2, ^bb3
 // NOOPT:         ^bb2:
 // NOOPT:           fir.call @y(%[[VAL_0]]) : (!fir.ref<index>) -> ()
-// NOOPT:           %[[VAL_10:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] overflow<nsw> : index
+// NOOPT:           %[[VAL_10:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] : index
 // NOOPT:           %[[VAL_11:.*]] = arith.constant 1 : index
 // NOOPT:           %[[VAL_12:.*]] = arith.subi %[[VAL_7]], %[[VAL_11]] : index
 // NOOPT:           br ^bb1(%[[VAL_10]], %[[VAL_12]] : index, index)
diff --git a/flang/test/Lower/HLFIR/goto-do-body.f90 b/flang/test/Lower/HLFIR/goto-do-body.f90
index 5f5b09ccb8f7..383b839e591e 100644
--- a/flang/test/Lower/HLFIR/goto-do-body.f90
+++ b/flang/test/Lower/HLFIR/goto-do-body.f90
@@ -40,7 +40,7 @@ subroutine sub1()
 ! CHECK:    %[[TMP5:.*]] = arith.subi %[[TMP4]], %[[C1]] : i32
 ! CHECK:    fir.store %[[TMP5]] to %[[TRIP]] : !fir.ref<i32>
 ! CHECK:    %[[TMP6:.*]] = fir.load %[[I]]#1 : !fir.ref<i32>
-! CHECK:    %[[TMP7:.*]] = arith.addi %[[TMP6]], %[[C1]] overflow<nsw> : i32
+! CHECK:    %[[TMP7:.*]] = arith.addi %[[TMP6]], %[[C1]] : i32
 ! CHECK:    fir.store %[[TMP7]] to %[[I]]#1 : !fir.ref<i32>
 ! CHECK:    cf.br ^[[HEADER]]
   end do
@@ -104,7 +104,7 @@ subroutine sub2()
 ! CHECK:    fir.store %[[TMP9]] to %[[TRIP]] : !fir.ref<i32>
 ! CHECK:    %[[TMP10:.*]] = fir.load %[[I]]#1 : !fir.ref<i32>
 ! CHECK:    %[[STEP_VAL:.*]] = fir.load %[[STEP_VAR]] : !fir.ref<i32>
-! CHECK:    %[[TMP11:.*]] = arith.addi %[[TMP10]], %[[STEP_VAL]] overflow<nsw> : i32
+! CHECK:    %[[TMP11:.*]] = arith.addi %[[TMP10]], %[[STEP_VAL]] : i32
 ! CHECK:    fir.store %[[TMP11]] to %[[I]]#1 : !fir.ref<i32>
 ! CHECK:    cf.br ^[[HEADER]]
   end do
diff --git a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90 b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
index 99323e69113b..5e76e8ff1663 100644
--- a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
+++ b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
@@ -82,10 +82,10 @@
 ! CHECK:                 %[[VAL_15:.*]] = fir.load %[[PRIV_J_DECL]]#0 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_16:.*]] = arith.addi %[[LOAD]], %[[VAL_15]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_16]] to %[[PRIV_X_DECL]]#0 : i32, !fir.ref<i32>
-! CHECK:                 %[[VAL_17:.*]] = arith.addi %[[VAL_13]], %[[VAL_11]] overflow<nsw> : index
+! CHECK:                 %[[VAL_17:.*]] = arith.addi %[[VAL_13]], %[[VAL_11]] : index
 ! CHECK:                 %[[STEPCAST:.*]] = fir.convert %[[VAL_11]] : (index) -> i32
 ! CHECK:                 %[[IVLOAD:.*]] = fir.load %[[PRIV_J_DECL]]#1 : !fir.ref<i32>
-! CHECK:                 %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST]] overflow<nsw> :
+! CHECK:                 %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST]]
 ! CHECK:                 fir.result %[[VAL_17]], %[[IVINC]] : index, i32
 ! CHECK:               }
 ! CHECK:               fir.store %[[VAL_12]]#1 to %[[PRIV_J_DECL]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
index ce45d09d77a2..a49eba69ff38 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
@@ -206,10 +206,10 @@ end program
 ! CHECK:             %[[VAL_48:.*]] = fir.convert %[[VAL_47]] : (i32) -> i64
 ! CHECK:             %[[VAL_49:.*]] = hlfir.designate %[[VAL_46]] (%[[VAL_48]])  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, i64) -> !fir.ref<i32>
 ! CHECK:             hlfir.assign %[[VAL_45]] to %[[VAL_49]] : i32, !fir.ref<i32>
-! CHECK:             %[[VAL_50:.*]] = arith.addi %[[VAL_43]], %[[VAL_40]] overflow<nsw> : index
+! CHECK:             %[[VAL_50:.*]] = arith.addi %[[VAL_43]], %[[VAL_40]] : index
 ! CHECK:             %[[VAL_51:.*]] = fir.convert %[[VAL_40]] : (index) -> i32
 ! CHECK:             %[[VAL_52:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref<i32>
-! CHECK:             %[[VAL_53:.*]] = arith.addi %[[VAL_52]], %[[VAL_51]] overflow<nsw> : i32
+! CHECK:             %[[VAL_53:.*]] = arith.addi %[[VAL_52]], %[[VAL_51]] : i32
 ! CHECK:             fir.result %[[VAL_50]], %[[VAL_53]] : index, i32
 ! CHECK:           }
 ! CHECK:           fir.store %[[VAL_54:.*]]#1 to %[[VAL_3]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-variable.f90 b/flang/test/Lower/OpenMP/wsloop-variable.f90
index cc77ce754d97..8d235c10fa1d 100644
--- a/flang/test/Lower/OpenMP/wsloop-variable.f90
+++ b/flang/test/Lower/OpenMP/wsloop-variable.f90
@@ -150,10 +150,10 @@ subroutine wsloop_variable_sub
 !CHECK:                 %[[VAL_42:.*]] = arith.addi %[[VAL_40]], %[[VAL_41]] : i64
 !CHECK:                 %[[VAL_43:.*]] = fir.convert %[[VAL_42]] : (i64) -> f32
 !CHECK:                 hlfir.assign %[[VAL_43]] to %[[VAL_21]]#0 : f32, !fir.ref<f32>
-!CHECK:                 %[[VAL_44:.*]] = arith.addi %[[VAL_37]], %[[VAL_34]] overflow<nsw> : index
+!CHECK:                 %[[VAL_44:.*]] = arith.addi %[[VAL_37]], %[[VAL_34]] : index
 !CHECK:                 %[[VAL_45:.*]] = fir.convert %[[VAL_34]] : (index) -> i64
 !CHECK:                 %[[VAL_46:.*]] = fir.load %[[VAL_17]]#1 : !fir.ref<i64>
-!CHECK:                 %[[VAL_47:.*]] = arith.addi %[[VAL_46]], %[[VAL_45]] overflow<nsw> : i64
+!CHECK:                 %[[VAL_47:.*]] = arith.addi %[[VAL_46]], %[[VAL_45]] : i64
 !CHECK:                 fir.result %[[VAL_44]], %[[VAL_47]] : index, i64
 !CHECK:               }
 !CHECK:               fir.store %[[VAL_48:.*]]#1 to %[[VAL_17]]#1 : !fir.ref<i64>
diff --git a/flang/test/Lower/array-character.f90 b/flang/test/Lower/array-character.f90
index 53adc5c02958..c93ef4be3082 100644
--- a/flang/test/Lower/array-character.f90
+++ b/flang/test/Lower/array-character.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -hlfir=false -fwrapv %s -o - | fir-opt --canonicalize --cse | FileCheck %s
+! RUN: bbc -hlfir=false %s -o - | fir-opt --canonicalize --cse | FileCheck %s
 
 ! CHECK-LABEL: func @_QPissue(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.boxchar<1>{{.*}}, %[[VAL_1:.*]]: !fir.boxchar<1>{{.*}}) {
diff --git a/flang/test/Lower/array-derived-assignments.f90 b/flang/test/Lower/array-derived-assignments.f90
index f4e51271d593..71e61f651302 100644
--- a/flang/test/Lower/array-derived-assignments.f90
+++ b/flang/test/Lower/array-derived-assignments.f90
@@ -1,5 +1,5 @@
 ! Test derived type assignment lowering inside array expression
-! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false %s -o - | FileCheck %s
 
 module array_derived_assign
   type simple_copy
diff --git a/flang/test/Lower/array-derived.f90 b/flang/test/Lower/array-derived.f90
index a0c55f5d8825..b5eb7621c90f 100644
--- a/flang/test/Lower/array-derived.f90
+++ b/flang/test/Lower/array-derived.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false %s -o - | FileCheck %s
 
 module cs
   type r
diff --git a/flang/test/Lower/array-elemental-calls-char-byval.f90 b/flang/test/Lower/array-elemental-calls-char-byval.f90
index 682191fc7895..c321614e7fc5 100644
--- a/flang/test/Lower/array-elemental-calls-char-byval.f90
+++ b/flang/test/Lower/array-elemental-calls-char-byval.f90
@@ -1,6 +1,6 @@
 ! Test lowering of elemental calls with character argument
 ! with the VALUE attribute.
-! RUN: bbc -hlfir=false -fwrapv -o - %s | FileCheck %s
+! RUN: bbc -hlfir=false -o - %s | FileCheck %s
 
 
 module char_elem_byval
diff --git a/flang/test/Lower/array-elemental-calls-char.f90 b/flang/test/Lower/array-elemental-calls-char.f90
index 00e2f8e8f9c1..603cc677805f 100644
--- a/flang/test/Lower/array-elemental-calls-char.f90
+++ b/flang/test/Lower/array-elemental-calls-char.f90
@@ -1,6 +1,6 @@
 ! Test lowering of elemental calls with character argument
 ! without the VALUE attribute.
-! RUN: bbc -hlfir=false -fwrapv -o - %s | FileCheck %s
+! RUN: bbc -hlfir=false -o - %s | FileCheck %s
 
 module char_elem
 
diff --git a/flang/test/Lower/array-expression-assumed-size.f90 b/flang/test/Lower/array-expression-assumed-size.f90
index 2fbf315aff11..ae35da951538 100644
--- a/flang/test/Lower/array-expression-assumed-size.f90
+++ b/flang/test/Lower/array-expression-assumed-size.f90
@@ -1,5 +1,5 @@
 ! RUN: bbc --emit-fir -hlfir=false %s -o - | FileCheck %s
-! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck --check-prefix=PostOpt %s
+! RUN: bbc -hlfir=false %s -o - | FileCheck --check-prefix=PostOpt %s
 
 
 subroutine assumed_size_test(a)
diff --git a/flang/test/Lower/array-expression-slice-1.f90 b/flang/test/Lower/array-expression-slice-1.f90
index b597814bc0d9..152450902432 100644
--- a/flang/test/Lower/array-expression-slice-1.f90
+++ b/flang/test/Lower/array-expression-slice-1.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -hlfir=false -fwrapv -o - --outline-intrinsics %s | FileCheck %s
+! RUN: bbc -hlfir=false -o - --outline-intrinsics %s | FileCheck %s
 
 ! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "p"} {
 ! CHECK-DAG:         %[[VAL_0:.*]] = arith.constant 10 : index
diff --git a/flang/test/Lower/array-substring.f90 b/flang/test/Lower/array-substring.f90
index 02101039120e..2e283997e3e0 100644
--- a/flang/test/Lower/array-substring.f90
+++ b/flang/test/Lower/array-substring.f90
@@ -1,7 +1,5 @@
 ! RUN: bbc -hlfir=false %s -o - | FileCheck %s
-! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s --check-prefix=NO-NSW
-
-! NO-NSW-NOT: overflow<nsw>
+! RUN: bbc -hlfir=false -integer-overflow %s -o - | FileCheck %s --check-prefix=NSW
 
 ! CHECK-LABEL: func @_QPtest(
 ! CHECK-SAME:     %[[VAL_0:.*]]: !fir.boxchar<1>{{.*}}) -> !fir.array<1x!fir.logical<4>> {
@@ -34,9 +32,8 @@
 ! CHECK:         %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:         %[[VAL_27:.*]] = fir.array_coor %[[VAL_8]](%[[VAL_9]]) %[[VAL_15]] : (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.shape<1>, index) -> !fir.ref<!fir.logical<4>>
 ! CHECK:         fir.store %[[VAL_26]] to %[[VAL_27]] : !fir.ref<!fir.logical<4>>
-! CHECK:         %[[VAL_15_NSW:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] overflow<nsw> : index
 ! CHECK:         %[[VAL_28:.*]] = arith.subi %[[VAL_13]], %[[VAL_1]] : index
-! CHECK:         br ^bb1(%[[VAL_15_NSW]], %[[VAL_28]] : index, index)
+! CHECK:         br ^bb1(%[[VAL_15]], %[[VAL_28]] : index, index)
 ! CHECK:       ^bb3:
 ! CHECK:         %[[VAL_29:.*]] = fir.load %[[VAL_8]] : !fir.ref<!fir.array<1x!fir.logical<4>>>
 ! CHECK:         return %[[VAL_29]] : !fir.array<1x!fir.logical<4>>
@@ -49,3 +46,42 @@ function test(C)
 
   test = C(1:1)(1:8) == (/'ABCDabcd'/) 
 end function test
+
+! NSW-LABEL: func @_QPtest(
+! NSW-SAME:     %[[VAL_0:.*]]: !fir.boxchar<1>{{.*}}) -> !fir.array<1x!fir.logical<4>> {
+! NSW-DAG:         %[[VAL_1:.*]] = arith.constant 1 : index
+! NSW-DAG:         %[[VAL_2:.*]] = arith.constant 0 : index
+! NSW-DAG:         %[[VAL_3:.*]] = arith.constant 0 : i32
+! NSW-DAG:         %[[VAL_4:.*]] = arith.constant 8 : index
+! NSW:         %[[VAL_6:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! NSW:         %[[VAL_7:.*]] = fir.convert %[[VAL_6]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<1x!fir.char<1,12>>>
+! NSW:         %[[VAL_8:.*]] = fir.alloca !fir.array<1x!fir.logical<4>> {bindc_name = "test", uniq_name = "_QFtestEtest"}
+! NSW:         %[[VAL_9:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
+! NSW:         %[[VAL_10:.*]] = fir.slice %[[VAL_1]], %[[VAL_1]], %[[VAL_1]] : (index, index, index) -> !fir.slice<1>
+! NSW:         %[[VAL_11:.*]] = fir.address_of(@_QQ{{.*}}) : !fir.ref<!fir.array<1x!fir.char<1,8>>>
+! NSW:         br ^bb1(%[[VAL_2]], %[[VAL_1]] : index, index)
+! NSW:       ^bb1(%[[VAL_12:.*]]: index, %[[VAL_13:.*]]: index):
+! NSW:         %[[VAL_14:.*]] = arith.cmpi sgt, %[[VAL_13]], %[[VAL_2]] : index
+! NSW:         cond_br %[[VAL_14]], ^bb2, ^bb3
+! NSW:       ^bb2:
+! NSW:         %[[VAL_15:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] : index
+! NSW:         %[[VAL_16:.*]] = fir.array_coor %[[VAL_7]](%[[VAL_9]]) {{\[}}%[[VAL_10]]] %[[VAL_15]] : (!fir.ref<!fir.array<1x!fir.char<1,12>>>, !fir.shape<1>, !fir.slice<1>, index) -> !fir.ref<!fir.char<1,12>>
+! NSW:         %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (!fir.ref<!fir.char<1,12>>) -> !fir.ref<!fir.array<12x!fir.char<1>>>
+! NSW:         %[[VAL_18:.*]] = fir.coordinate_of %[[VAL_17]], %[[VAL_2]] : (!fir.ref<!fir.array<12x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
+! NSW:         %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<!fir.char<1,?>>
+! NSW:         %[[VAL_20:.*]] = fir.array_coor %[[VAL_11]](%[[VAL_9]]) %[[VAL_15]] : (!fir.ref<!fir.array<1x!fir.char<1,8>>>, !fir.shape<1>, index) -> !fir.ref<!fir.char<1,8>>
+! NSW:         %[[VAL_21:.*]] = fir.convert %[[VAL_19]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
+! NSW:         %[[VAL_22:.*]] = fir.convert %[[VAL_20]] : (!fir.ref<!fir.char<1,8>>) -> !fir.ref<i8>
+! NSW:         %[[VAL_23:.*]] = fir.convert %[[VAL_4]] : (index) -> i64
+! NSW:         %[[VAL_24:.*]] = fir.call @_FortranACharacterCompareScalar1(%[[VAL_21]], %[[VAL_22]], %[[VAL_23]], %[[VAL_23]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64, i64) -> i32
+! NSW:         %[[VAL_25:.*]] = arith.cmpi eq, %[[VAL_24]], %[[VAL_3]] : i32
+! NSW:         %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
+! NSW:         %[[VAL_27:.*]] = fir.array_coor %[[VAL_8]](%[[VAL_9]]) %[[VAL_15]] : (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.shape<1>, index) -> !fir.ref<!fir.logical<4>>
+! NSW:         fir.store %[[VAL_26]] to %[[VAL_27]] : !fir.ref<!fir.logical<4>>
+! NSW:         %[[VAL_15_NSW:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] overflow<nsw> : index
+! NSW:         %[[VAL_28:.*]] = arith.subi %[[VAL_13]], %[[VAL_1]] : index
+! NSW:         br ^bb1(%[[VAL_15_NSW]], %[[VAL_28]] : index, index)
+! NSW:       ^bb3:
+! NSW:         %[[VAL_29:.*]] = fir.load %[[VAL_8]] : !fir.ref<!fir.array<1x!fir.logical<4>>>
+! NSW:         return %[[VAL_29]] : !fir.array<1x!fir.logical<4>>
+! NSW:       }
diff --git a/flang/test/Lower/array-temp.f90 b/flang/test/Lower/array-temp.f90
index 718aef84a4e8..10c5ee91d44b 100644
--- a/flang/test/Lower/array-temp.f90
+++ b/flang/test/Lower/array-temp.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPss1()
 subroutine ss1
diff --git a/flang/test/Lower/components.f90 b/flang/test/Lower/components.f90
index 28e836c5d104..e1582a8a31e0 100644
--- a/flang/test/Lower/components.f90
+++ b/flang/test/Lower/components.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false %s -o - | FileCheck %s
 
 module components_test
   type t1
diff --git a/flang/test/Lower/do_loop.f90 b/flang/test/Lower/do_loop.f90
index 5d8343b8d68a..a46e6c947391 100644
--- a/flang/test/Lower/do_loop.f90
+++ b/flang/test/Lower/do_loop.f90
@@ -1,17 +1,17 @@
 ! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false -o - %s | FileCheck %s
 ! RUN: %flang_fc1 -mllvm --use-desc-for-alloc=false -emit-fir -flang-deprecated-no-hlfir -o - %s | FileCheck %s
-! RUN: %flang_fc1 -mllvm --use-desc-for-alloc=false -emit-fir -flang-deprecated-no-hlfir -fwrapv -o - %s | FileCheck %s --check-prefix=NO-NSW
+! RUN: %flang_fc1 -mllvm --use-desc-for-alloc=false -emit-fir -flang-deprecated-no-hlfir -flang-experimental-integer-overflow -o - %s | FileCheck %s --check-prefix=NSW
 
 ! Simple tests for structured ordered loops with loop-control.
 ! Tests the structure of the loop, storage to index variable and return and 
 ! storage of the final value of the index variable.
 
-! NO-NSW-NOT: overflow<nsw>
-
 ! Test a simple loop with the final value of the index variable read outside the loop
 ! CHECK-LABEL: simple_loop
+! NSW-LABEL:   simple_loop
 subroutine simple_loop
   ! CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_loopEi"}
+  ! NSW:   %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_loopEi"}
   integer :: i
 
   ! CHECK: %[[C1:.*]] = arith.constant 1 : i32
@@ -21,14 +21,18 @@ subroutine simple_loop
   ! CHECK: %[[C1:.*]] = arith.constant 1 : index
   ! CHECK: %[[LB:.*]] = fir.convert %[[C1_CVT]] : (index) -> i32
   ! CHECK: %[[LI_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
+  ! NSW:   %[[LI_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[C1_CVT]] to %[[C5_CVT]] step %[[C1]]
   ! CHECK-SAME: iter_args(%[[IV:.*]] = %[[LB]]) -> (index, i32) {
   do i=1,5
   ! CHECK:   fir.store %[[IV]] to %[[I_REF]] : !fir.ref<i32>
-  ! CHECK:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[C1]] overflow<nsw> : index
+  ! CHECK:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[C1]] : index
+  ! NSW:     %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[C1:.*]] overflow<nsw> : index
   ! CHECK:   %[[STEPCAST:.*]] = fir.convert %[[C1]] : (index) -> i32
   ! CHECK:   %[[IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-  ! CHECK:   %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST]] overflow<nsw> : i32
+  ! NSW:     %[[IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
+  ! CHECK:   %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST]] : i32
+  ! NSW:     %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST:.*]] overflow<nsw> : i32
   ! CHECK:  fir.result %[[LI_NEXT]], %[[IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -40,11 +44,14 @@ end subroutine
 
 ! Test a 2-nested loop with a body composed of a reduction. Values are read from a 2d array.
 ! CHECK-LABEL: nested_loop
+! NSW-LABEL:   nested_loop
 subroutine nested_loop
   ! CHECK: %[[ARR_REF:.*]] = fir.alloca !fir.array<5x5xi32> {bindc_name = "arr", uniq_name = "_QFnested_loopEarr"}
   ! CHECK: %[[ASUM_REF:.*]] = fir.alloca i32 {bindc_name = "asum", uniq_name = "_QFnested_loopEasum"}
   ! CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_loopEi"}
+  ! NSW:   %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_loopEi"}
   ! CHECK: %[[J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_loopEj"}
+  ! NSW:   %[[J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_loopEj"}
   integer :: asum, arr(5,5)
   integer :: i, j
   asum = 0
@@ -55,6 +62,7 @@ subroutine nested_loop
   ! CHECK: %[[ST_I:.*]] = arith.constant 1 : index
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[S_I_CVT]] : (index) -> i32
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
+  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[S_I_CVT]] to %[[E_I_CVT]] step %[[ST_I]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do i=1,5
@@ -66,6 +74,7 @@ subroutine nested_loop
     ! CHECK: %[[ST_J:.*]] = arith.constant 1 : index
     ! CHECK: %[[J_LB:.*]] = fir.convert %[[S_J_CVT]] : (index) -> i32
     ! CHECK: %[[J_RES:.*]]:2 = fir.do_loop %[[LJ:[^ ]*]] =
+    ! NSW:   %[[J_RES:.*]]:2 = fir.do_loop %[[LJ:[^ ]*]] =
     ! CHECK-SAME: %[[S_J_CVT]] to %[[E_J_CVT]] step %[[ST_J]]
     ! CHECK-SAME: iter_args(%[[J_IV:.*]] = %[[J_LB]]) -> (index, i32) {
     do j=1,5
@@ -84,18 +93,24 @@ subroutine nested_loop
       ! CHECK: %[[ASUM_NEW:.*]] = arith.addi %[[ASUM]], %[[ARR_VAL]] : i32
       ! CHECK: fir.store %[[ASUM_NEW]] to %[[ASUM_REF]] : !fir.ref<i32>
       asum = asum + arr(i,j)
-      ! CHECK: %[[LJ_NEXT:.*]] = arith.addi %[[LJ]], %[[ST_J]] overflow<nsw> : index
+      ! CHECK: %[[LJ_NEXT:.*]] = arith.addi %[[LJ]], %[[ST_J]] : index
+      ! NSW:   %[[LJ_NEXT:.*]] = arith.addi %[[LJ]], %[[ST_J:.*]] overflow<nsw> : index
       ! CHECK: %[[J_STEPCAST:.*]] = fir.convert %[[ST_J]] : (index) -> i32
       ! CHECK: %[[J_IVLOAD:.*]] = fir.load %[[J_REF]] : !fir.ref<i32>
-      ! CHECK: %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] overflow<nsw> : i32
+      ! NSW:   %[[J_IVLOAD:.*]] = fir.load %[[J_REF]] : !fir.ref<i32>
+      ! CHECK: %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] : i32
+      ! NSW:   %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST:.*]] overflow<nsw> : i32
       ! CHECK: fir.result %[[LJ_NEXT]], %[[J_IVINC]] : index, i32
     ! CHECK: }
     end do
     ! CHECK: fir.store %[[J_RES]]#1 to %[[J_REF]] : !fir.ref<i32>
-    ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_I]] overflow<nsw> : index
+    ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_I]] : index
+    ! NSW:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_I:.*]] overflow<nsw> : index
     ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[ST_I]] : (index) -> i32
     ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-    ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i32
+    ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
+    ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
+    ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
     ! CHECK: fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -104,9 +119,11 @@ end subroutine
 
 ! Test a downcounting loop
 ! CHECK-LABEL: down_counting_loop
+! NSW-LABEL:   down_counting_loop
 subroutine down_counting_loop()
   integer :: i
   ! CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFdown_counting_loopEi"}
+  ! NSW:   %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFdown_counting_loopEi"}
 
   ! CHECK: %[[C5:.*]] = arith.constant 5 : i32
   ! CHECK: %[[C5_CVT:.*]] = fir.convert %[[C5]] : (i32) -> index
@@ -116,14 +133,18 @@ subroutine down_counting_loop()
   ! CHECK: %[[CMINUS1_STEP_CVT:.*]] = fir.convert %[[CMINUS1]] : (i32) -> index
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[C5_CVT]] : (index) -> i32
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
+  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[C5_CVT]] to %[[C1_CVT]] step %[[CMINUS1_STEP_CVT]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do i=5,1,-1
   ! CHECK: fir.store %[[I_IV]] to %[[I_REF]] : !fir.ref<i32>
-  ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[CMINUS1_STEP_CVT]] overflow<nsw> : index
+  ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[CMINUS1_STEP_CVT]] : index
+  ! NSW:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[CMINUS1_STEP_CVT:.*]] overflow<nsw> : index
   ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[CMINUS1_STEP_CVT]] : (index) -> i32
   ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-  ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i32
+  ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
+  ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
+  ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
   ! CHECK: fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -132,6 +153,7 @@ end subroutine
 
 ! Test a general loop with a variable step
 ! CHECK-LABEL: loop_with_variable_step
+! NSW-LABEL:   loop_with_variable_step
 ! CHECK-SAME: (%[[S_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "s"}, %[[E_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "e"}, %[[ST_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "st"}) {
 subroutine loop_with_variable_step(s,e,st)
   integer :: s, e, st
@@ -144,14 +166,18 @@ subroutine loop_with_variable_step(s,e,st)
   ! CHECK: %[[ST_CVT:.*]] = fir.convert %[[ST]] : (i32) -> index
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[S_CVT]] : (index) -> i32
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
+  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[S_CVT]] to %[[E_CVT]] step %[[ST_CVT]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do i=s,e,st
   ! CHECK:  fir.store %[[I_IV]] to %[[I_REF]] : !fir.ref<i32>
-  ! CHECK:  %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] overflow<nsw> : index
+  ! CHECK:  %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] : index
+  ! NSW:    %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT:.*]] overflow<nsw> : index
   ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[ST_CVT]] : (index) -> i32
   ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-  ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i32
+  ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
+  ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
+  ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
   ! CHECK:  fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -160,11 +186,13 @@ end subroutine
 
 ! Test usage of pointer variables as index, start, end and step variables
 ! CHECK-LABEL: loop_with_pointer_variables
+! NSW-LABEL:   loop_with_pointer_variables
 ! CHECK-SAME: (%[[S_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "s", fir.target}, %[[E_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "e", fir.target}, %[[ST_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "st", fir.target}) {
 subroutine loop_with_pointer_variables(s,e,st)
 ! CHECK:  %[[E_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEeptr.addr"}
 ! CHECK:  %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", fir.target, uniq_name = "_QFloop_with_pointer_variablesEi"}
 ! CHECK:  %[[I_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEiptr.addr"}
+! NSW:    %[[I_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEiptr.addr"}
 ! CHECK:  %[[S_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEsptr.addr"}
 ! CHECK:  %[[ST_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEstptr.addr"}
   integer, target :: i
@@ -185,6 +213,7 @@ subroutine loop_with_pointer_variables(s,e,st)
   stptr => st
 
 ! CHECK:  %[[I_PTR:.*]] = fir.load %[[I_PTR_REF]] : !fir.ref<!fir.ptr<i32>>
+! NSW:    %[[I_PTR:.*]] = fir.load %[[I_PTR_REF]] : !fir.ref<!fir.ptr<i32>>
 ! CHECK:  %[[S_PTR:.*]] = fir.load %[[S_PTR_REF]] : !fir.ref<!fir.ptr<i32>>
 ! CHECK:  %[[S:.*]] = fir.load %[[S_PTR]] : !fir.ptr<i32>
 ! CHECK:  %[[S_CVT:.*]] = fir.convert %[[S]] : (i32) -> index
@@ -196,14 +225,18 @@ subroutine loop_with_pointer_variables(s,e,st)
 ! CHECK:  %[[ST_CVT:.*]] = fir.convert %[[ST]] : (i32) -> index
 ! CHECK:  %[[I_LB:.*]] = fir.convert %[[S_CVT]] : (index) -> i32
 ! CHECK:  %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
+! NSW:    %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
 ! CHECK-SAME: %[[S_CVT]] to %[[E_CVT]] step %[[ST_CVT]]
 ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do iptr=sptr,eptr,stptr
 ! CHECK:    fir.store %[[I_IV]] to %[[I_PTR]] : !fir.ptr<i32>
-! CHECK:    %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] overflow<nsw> : index
+! CHECK:    %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] : index
+! NSW:      %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT:.*]] overflow<nsw> : index
 ! CHECK:    %[[I_STEPCAST:.*]] = fir.convert %[[ST_CVT]] : (index) -> i32
 ! CHECK:    %[[I_IVLOAD:.*]] = fir.load %[[I_PTR]] : !fir.ptr<i32>
-! CHECK:    %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i32
+! NSW:      %[[I_IVLOAD:.*]] = fir.load %[[I_PTR]] : !fir.ptr<i32>
+! CHECK:    %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
+! NSW:      %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
 ! CHECK:    fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   end do
 ! CHECK:  }
@@ -212,9 +245,11 @@ end subroutine
 
 ! Test usage of non-default integer kind for loop control and loop index variable
 ! CHECK-LABEL: loop_with_non_default_integer
+! NSW-LABEL:   loop_with_non_default_integer
 ! CHECK-SAME: (%[[S_REF:.*]]: !fir.ref<i64> {fir.bindc_name = "s"}, %[[E_REF:.*]]: !fir.ref<i64> {fir.bindc_name = "e"}, %[[ST_REF:.*]]: !fir.ref<i64> {fir.bindc_name = "st"}) {
 subroutine loop_with_non_default_integer(s,e,st)
   ! CHECK: %[[I_REF:.*]] = fir.alloca i64 {bindc_name = "i", uniq_name = "_QFloop_with_non_default_integerEi"}
+  ! NSW:   %[[I_REF:.*]] = fir.alloca i64 {bindc_name = "i", uniq_name = "_QFloop_with_non_default_integerEi"}
   integer(kind=8):: i
   ! CHECK: %[[S:.*]] = fir.load %[[S_REF]] : !fir.ref<i64>
   ! CHECK: %[[S_CVT:.*]] = fir.convert %[[S]] : (i64) -> index
@@ -226,14 +261,18 @@ subroutine loop_with_non_default_integer(s,e,st)
 
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[S_CVT]] : (index) -> i64
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
+  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[S_CVT]] to %[[E_CVT]] step %[[ST_CVT]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i64) {
   do i=s,e,st
     ! CHECK: fir.store %[[I_IV]] to %[[I_REF]] : !fir.ref<i64>
-    ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] overflow<nsw> : index
+    ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] : index
+    ! NSW:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT:.*]] overflow<nsw> : index
     ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[ST_CVT]] : (index) -> i64
     ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i64>
-    ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i64
+    ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i64>
+    ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i64
+    ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i64
     ! CHECK: fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i64
   end do
   ! CHECK: }
diff --git a/flang/test/Lower/do_loop_unstructured.f90 b/flang/test/Lower/do_loop_unstructured.f90
index d8890b2d0926..e1a669e09c9a 100644
--- a/flang/test/Lower/do_loop_unstructured.f90
+++ b/flang/test/Lower/do_loop_unstructured.f90
@@ -1,11 +1,9 @@
 ! RUN: bbc -emit-fir -hlfir=false -o - %s | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -o - %s | FileCheck %s
-! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fwrapv -o - %s | FileCheck %s --check-prefix=NO-NSW
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -flang-experimental-integer-overflow -o - %s | FileCheck %s --check-prefix=NSW
 
 ! Tests for unstructured loops.
 
-! NO-NSW-NOT: overflow<nsw>
-
 ! Test a simple unstructured loop. Test for the existence of,
 ! -> The initialization of the trip-count and loop-variable
 ! -> The branch to the body or the exit inside the header
@@ -41,12 +39,42 @@ end subroutine
 ! CHECK:   fir.store %[[TRIP_VAR_NEXT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR:.*]] = fir.load %[[LOOP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   %[[STEP_ONE_2:.*]] = arith.constant 1 : i32
-! CHECK:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_ONE_2]] overflow<nsw> : i32
+! CHECK:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_ONE_2]] : i32
 ! CHECK:   fir.store %[[LOOP_VAR_NEXT]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER]]
 ! CHECK: ^[[EXIT]]:
 ! CHECK:   return
 
+! NSW-LABEL: simple_unstructured
+! NSW:   %[[TRIP_VAR_REF:.*]] = fir.alloca i32
+! NSW:   %[[LOOP_VAR_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_unstructuredEi"}
+! NSW:   %[[ONE:.*]] = arith.constant 1 : i32
+! NSW:   %[[HUNDRED:.*]] = arith.constant 100 : i32
+! NSW:   %[[STEP_ONE:.*]] = arith.constant 1 : i32
+! NSW:   %[[TMP1:.*]] = arith.subi %[[HUNDRED]], %[[ONE]] : i32
+! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[STEP_ONE]] : i32
+! NSW:   %[[TRIP_COUNT:.*]] = arith.divsi %[[TMP2]], %[[STEP_ONE]] : i32
+! NSW:   fir.store %[[TRIP_COUNT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   fir.store %[[ONE]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER:.*]]
+! NSW: ^[[HEADER]]:
+! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[ZERO:.*]] = arith.constant 0 : i32
+! NSW:   %[[COND:.*]] = arith.cmpi sgt, %[[TRIP_VAR]], %[[ZERO]] : i32
+! NSW:   cf.cond_br %[[COND]], ^[[BODY:.*]], ^[[EXIT:.*]]
+! NSW: ^[[BODY]]:
+! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
+! NSW:   %[[TRIP_VAR_NEXT:.*]] = arith.subi %[[TRIP_VAR]], %[[ONE_1]] : i32
+! NSW:   fir.store %[[TRIP_VAR_NEXT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[LOOP_VAR:.*]] = fir.load %[[LOOP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[STEP_ONE_2:.*]] = arith.constant 1 : i32
+! NSW:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_ONE_2]] overflow<nsw> : i32
+! NSW:   fir.store %[[LOOP_VAR_NEXT]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER]]
+! NSW: ^[[EXIT]]:
+! NSW:   return
+
 ! Test an unstructured loop with a step. Mostly similar to the previous one.
 ! Only difference is a non-unit step.
 subroutine simple_unstructured_with_step()
@@ -80,12 +108,42 @@ end subroutine
 ! CHECK:   fir.store %[[TRIP_VAR_NEXT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR:.*]] = fir.load %[[LOOP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   %[[STEP_2:.*]] = arith.constant 2 : i32
-! CHECK:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_2]] overflow<nsw> : i32
+! CHECK:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_2]] : i32
 ! CHECK:   fir.store %[[LOOP_VAR_NEXT]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER]]
 ! CHECK: ^[[EXIT]]:
 ! CHECK:   return
 
+! NSW-LABEL: simple_unstructured_with_step
+! NSW:   %[[TRIP_VAR_REF:.*]] = fir.alloca i32
+! NSW:   %[[LOOP_VAR_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_unstructured_with_stepEi"}
+! NSW:   %[[ONE:.*]] = arith.constant 1 : i32
+! NSW:   %[[HUNDRED:.*]] = arith.constant 100 : i32
+! NSW:   %[[STEP:.*]] = arith.constant 2 : i32
+! NSW:   %[[TMP1:.*]] = arith.subi %[[HUNDRED]], %[[ONE]] : i32
+! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[STEP]] : i32
+! NSW:   %[[TRIP_COUNT:.*]] = arith.divsi %[[TMP2]], %[[STEP]] : i32
+! NSW:   fir.store %[[TRIP_COUNT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   fir.store %[[ONE]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER:.*]]
+! NSW: ^[[HEADER]]:
+! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[ZERO:.*]] = arith.constant 0 : i32
+! NSW:   %[[COND:.*]] = arith.cmpi sgt, %[[TRIP_VAR]], %[[ZERO]] : i32
+! NSW:   cf.cond_br %[[COND]], ^[[BODY:.*]], ^[[EXIT:.*]]
+! NSW: ^[[BODY]]:
+! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
+! NSW:   %[[TRIP_VAR_NEXT:.*]] = arith.subi %[[TRIP_VAR]], %[[ONE_1]] : i32
+! NSW:   fir.store %[[TRIP_VAR_NEXT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[LOOP_VAR:.*]] = fir.load %[[LOOP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[STEP_2:.*]] = arith.constant 2 : i32
+! NSW:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_2]] overflow<nsw> : i32
+! NSW:   fir.store %[[LOOP_VAR_NEXT]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER]]
+! NSW: ^[[EXIT]]:
+! NSW:   return
+
 ! Test a three nested unstructured loop. Three nesting is the basic case where
 ! we have loops that are neither innermost or outermost.
 subroutine nested_unstructured()
@@ -157,7 +215,7 @@ end subroutine
 ! CHECK:   fir.store %[[TRIP_VAR_K_NEXT]] to %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR_K:.*]] = fir.load %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
 ! CHECK:   %[[K_STEP_2:.*]] = arith.constant 1 : i32
-! CHECK:   %[[LOOP_VAR_K_NEXT:.*]] = arith.addi %[[LOOP_VAR_K]], %[[K_STEP_2]] overflow<nsw> : i32
+! CHECK:   %[[LOOP_VAR_K_NEXT:.*]] = arith.addi %[[LOOP_VAR_K]], %[[K_STEP_2]] : i32
 ! CHECK:   fir.store %[[LOOP_VAR_K_NEXT]] to %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER_K]]
 ! CHECK: ^[[EXIT_K]]:
@@ -167,7 +225,7 @@ end subroutine
 ! CHECK:   fir.store %[[TRIP_VAR_J_NEXT]] to %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR_J:.*]] = fir.load %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
 ! CHECK:   %[[J_STEP_2:.*]] = arith.constant 1 : i32
-! CHECK:   %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %[[J_STEP_2]] overflow<nsw> : i32
+! CHECK:   %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %[[J_STEP_2]] : i32
 ! CHECK:   fir.store %[[LOOP_VAR_J_NEXT]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER_J]]
 ! CHECK: ^[[EXIT_J]]:
@@ -177,12 +235,96 @@ end subroutine
 ! CHECK:   fir.store %[[TRIP_VAR_I_NEXT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR_I:.*]] = fir.load %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[I_STEP_2:.*]] = arith.constant 1 : i32
-! CHECK:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] overflow<nsw> : i32
+! CHECK:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] : i32
 ! CHECK:   fir.store %[[LOOP_VAR_I_NEXT]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER_I]]
 ! CHECK: ^[[EXIT_I]]:
 ! CHECK:   return
 
+! NSW-LABEL: nested_unstructured
+! NSW:   %[[TRIP_VAR_K_REF:.*]] = fir.alloca i32
+! NSW:   %[[TRIP_VAR_J_REF:.*]] = fir.alloca i32
+! NSW:   %[[TRIP_VAR_I_REF:.*]] = fir.alloca i32
+! NSW:   %[[LOOP_VAR_I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_unstructuredEi"}
+! NSW:   %[[LOOP_VAR_J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_unstructuredEj"}
+! NSW:   %[[LOOP_VAR_K_REF:.*]] = fir.alloca i32 {bindc_name = "k", uniq_name = "_QFnested_unstructuredEk"}
+! NSW:   %[[I_START:.*]] = arith.constant 1 : i32
+! NSW:   %[[I_END:.*]] = arith.constant 100 : i32
+! NSW:   %[[I_STEP:.*]] = arith.constant 1 : i32
+! NSW:   %[[TMP1:.*]] = arith.subi %[[I_END]], %[[I_START]] : i32
+! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[I_STEP]] : i32
+! NSW:   %[[TRIP_COUNT_I:.*]] = arith.divsi %[[TMP2]], %[[I_STEP]] : i32
+! NSW:   fir.store %[[TRIP_COUNT_I]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   fir.store %[[I_START]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER_I:.*]]
+! NSW: ^[[HEADER_I]]:
+! NSW:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[ZERO_1:.*]] = arith.constant 0 : i32
+! NSW:   %[[COND_I:.*]] = arith.cmpi sgt, %[[TRIP_VAR_I]], %[[ZERO_1]] : i32
+! NSW:   cf.cond_br %[[COND_I]], ^[[BODY_I:.*]], ^[[EXIT_I:.*]]
+! NSW: ^[[BODY_I]]:
+! NSW:   %[[J_START:.*]] = arith.constant 1 : i32
+! NSW:   %[[J_END:.*]] = arith.constant 200 : i32
+! NSW:   %[[J_STEP:.*]] = arith.constant 1 : i32
+! NSW:   %[[TMP3:.*]] = arith.subi %[[J_END]], %[[J_START]] : i32
+! NSW:   %[[TMP4:.*]] = arith.addi %[[TMP3]], %[[J_STEP]] : i32
+! NSW:   %[[TRIP_COUNT_J:.*]] = arith.divsi %[[TMP4]], %[[J_STEP]] : i32
+! NSW:   fir.store %[[TRIP_COUNT_J]] to %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:   fir.store %[[J_START]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER_J:.*]]
+! NSW: ^[[HEADER_J]]:
+! NSW:   %[[TRIP_VAR_J:.*]] = fir.load %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:   %[[ZERO_2:.*]] = arith.constant 0 : i32
+! NSW:   %[[COND_J:.*]] = arith.cmpi sgt, %[[TRIP_VAR_J]], %[[ZERO_2]] : i32
+! NSW:   cf.cond_br %[[COND_J]], ^[[BODY_J:.*]], ^[[EXIT_J:.*]]
+! NSW: ^[[BODY_J]]:
+! NSW:   %[[K_START:.*]] = arith.constant 1 : i32
+! NSW:   %[[K_END:.*]] = arith.constant 300 : i32
+! NSW:   %[[K_STEP:.*]] = arith.constant 1 : i32
+! NSW:   %[[TMP3:.*]] = arith.subi %[[K_END]], %[[K_START]] : i32
+! NSW:   %[[TMP4:.*]] = arith.addi %[[TMP3]], %[[K_STEP]] : i32
+! NSW:   %[[TRIP_COUNT_K:.*]] = arith.divsi %[[TMP4]], %[[K_STEP]] : i32
+! NSW:   fir.store %[[TRIP_COUNT_K]] to %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
+! NSW:   fir.store %[[K_START]] to %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER_K:.*]]
+! NSW: ^[[HEADER_K]]:
+! NSW:   %[[TRIP_VAR_K:.*]] = fir.load %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
+! NSW:   %[[ZERO_2:.*]] = arith.constant 0 : i32
+! NSW:   %[[COND_K:.*]] = arith.cmpi sgt, %[[TRIP_VAR_K]], %[[ZERO_2]] : i32
+! NSW:   cf.cond_br %[[COND_K]], ^[[BODY_K:.*]], ^[[EXIT_K:.*]]
+! NSW: ^[[BODY_K]]:
+! NSW:   %[[TRIP_VAR_K:.*]] = fir.load %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
+! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
+! NSW:   %[[TRIP_VAR_K_NEXT:.*]] = arith.subi %[[TRIP_VAR_K]], %[[ONE_1]] : i32
+! NSW:   fir.store %[[TRIP_VAR_K_NEXT]] to %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
+! NSW:   %[[LOOP_VAR_K:.*]] = fir.load %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
+! NSW:   %[[K_STEP_2:.*]] = arith.constant 1 : i32
+! NSW:   %[[LOOP_VAR_K_NEXT:.*]] = arith.addi %[[LOOP_VAR_K]], %[[K_STEP_2]] overflow<nsw> : i32
+! NSW:   fir.store %[[LOOP_VAR_K_NEXT]] to %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER_K]]
+! NSW: ^[[EXIT_K]]:
+! NSW:   %[[TRIP_VAR_J:.*]] = fir.load %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
+! NSW:   %[[TRIP_VAR_J_NEXT:.*]] = arith.subi %[[TRIP_VAR_J]], %[[ONE_1]] : i32
+! NSW:   fir.store %[[TRIP_VAR_J_NEXT]] to %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:   %[[LOOP_VAR_J:.*]] = fir.load %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:   %[[J_STEP_2:.*]] = arith.constant 1 : i32
+! NSW:   %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %[[J_STEP_2]] overflow<nsw> : i32
+! NSW:   fir.store %[[LOOP_VAR_J_NEXT]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER_J]]
+! NSW: ^[[EXIT_J]]:
+! NSW:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
+! NSW:   %[[TRIP_VAR_I_NEXT:.*]] = arith.subi %[[TRIP_VAR_I]], %[[ONE_1]] : i32
+! NSW:   fir.store %[[TRIP_VAR_I_NEXT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[LOOP_VAR_I:.*]] = fir.load %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[I_STEP_2:.*]] = arith.constant 1 : i32
+! NSW:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] overflow<nsw> : i32
+! NSW:   fir.store %[[LOOP_VAR_I_NEXT]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER_I]]
+! NSW: ^[[EXIT_I]]:
+! NSW:   return
+
 ! Test the existence of a structured loop inside an unstructured loop.
 ! Only minimal checks are inserted for the structured loop.
 subroutine nested_structured_in_unstructured()
@@ -217,9 +359,9 @@ end subroutine
 ! CHECK-SAME: %{{.*}} to %{{.*}} step %[[ST:[^ ]*]]
 ! CHECK-SAME: iter_args(%[[J_IV:.*]] = %{{.*}}) -> (index, i32) {
 ! CHECK:     fir.store %[[J_IV]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! CHECK:     %[[J_INDEX_NEXT:.*]] = arith.addi %[[J_INDEX]], %[[ST]] overflow<nsw> : index
+! CHECK:     %[[J_INDEX_NEXT:.*]] = arith.addi %[[J_INDEX]], %[[ST]] : index
 ! CHECK:     %[[LOOP_VAR_J:.*]] = fir.load %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! CHECK:     %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %{{[^ ]*}} overflow<nsw> : i32
+! CHECK:     %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %{{[^ ]*}} : i32
 ! CHECK:   }
 ! CHECK:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[C1_3:.*]] = arith.constant 1 : i32
@@ -227,8 +369,47 @@ end subroutine
 ! CHECK:   fir.store %[[TRIP_VAR_I_NEXT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR_I:.*]] = fir.load %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[I_STEP_2:.*]] = arith.constant 1 : i32
-! CHECK:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] overflow<nsw> : i32
+! CHECK:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] : i32
 ! CHECK:   fir.store %[[LOOP_VAR_I_NEXT]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER]]
 ! CHECK: ^[[EXIT]]:
 ! CHECK:   return
+
+! NSW-LABEL: nested_structured_in_unstructured
+! NSW:   %[[TRIP_VAR_I_REF:.*]] = fir.alloca i32
+! NSW:   %[[LOOP_VAR_I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_structured_in_unstructuredEi"}
+! NSW:   %[[LOOP_VAR_J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_structured_in_unstructuredEj"}
+! NSW:   %[[I_START:.*]] = arith.constant 1 : i32
+! NSW:   %[[I_END:.*]] = arith.constant 100 : i32
+! NSW:   %[[I_STEP:.*]] = arith.constant 1 : i32
+! NSW:   %[[TMP1:.*]] = arith.subi %[[I_END]], %[[I_START]] : i32
+! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[I_STEP]] : i32
+! NSW:   %[[TRIP_COUNT:.*]] = arith.divsi %[[TMP2]], %[[I_STEP]] : i32
+! NSW:   fir.store %[[TRIP_COUNT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   fir.store %[[I_START]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER:.*]]
+! NSW: ^[[HEADER]]:
+! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[ZERO:.*]] = arith.constant 0 : i32
+! NSW:   %[[COND:.*]] = arith.cmpi sgt, %[[TRIP_VAR]], %[[ZERO]] : i32
+! NSW:   cf.cond_br %[[COND]], ^[[BODY:.*]], ^[[EXIT:.*]]
+! NSW: ^[[BODY]]:
+! NSW:   %{{.*}} = fir.do_loop %[[J_INDEX:[^ ]*]] =
+! NSW-SAME: %{{.*}} to %{{.*}} step %[[ST:[^ ]*]]
+! NSW-SAME: iter_args(%[[J_IV:.*]] = %{{.*}}) -> (index, i32) {
+! NSW:     fir.store %[[J_IV]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:     %[[J_INDEX_NEXT:.*]] = arith.addi %[[J_INDEX]], %[[ST]] overflow<nsw> : index
+! NSW:     %[[LOOP_VAR_J:.*]] = fir.load %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:     %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %{{[^ ]*}} overflow<nsw> : i32
+! NSW:   }
+! NSW:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[C1_3:.*]] = arith.constant 1 : i32
+! NSW:   %[[TRIP_VAR_I_NEXT:.*]] = arith.subi %[[TRIP_VAR_I]], %[[C1_3]] : i32
+! NSW:   fir.store %[[TRIP_VAR_I_NEXT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[LOOP_VAR_I:.*]] = fir.load %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[I_STEP_2:.*]] = arith.constant 1 : i32
+! NSW:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] overflow<nsw> : i32
+! NSW:   fir.store %[[LOOP_VAR_I_NEXT]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER]]
+! NSW: ^[[EXIT]]:
+! NSW:   return
diff --git a/flang/test/Lower/goto-do-body.f90 b/flang/test/Lower/goto-do-body.f90
index 89e4a7a64a87..910e55f1839f 100644
--- a/flang/test/Lower/goto-do-body.f90
+++ b/flang/test/Lower/goto-do-body.f90
@@ -48,7 +48,7 @@ subroutine sub1()
 ! CHECK:    fir.store %[[TMP8]] to %[[TRIP]] : !fir.ref<i32>
 ! CHECK:    %[[TMP9:.*]] = fir.load %[[I]] : !fir.ref<i32>
 ! CHECK:    %[[C1_4:.*]] = arith.constant 1 : i32
-! CHECK:    %[[TMP10:.*]] = arith.addi %[[TMP9]], %[[C1_4]] overflow<nsw> : i32
+! CHECK:    %[[TMP10:.*]] = arith.addi %[[TMP9]], %[[C1_4]] : i32
 ! CHECK:    fir.store %[[TMP10]] to %[[I]] : !fir.ref<i32>
 ! CHECK:    cf.br ^[[HEADER]]
   end do
@@ -115,7 +115,7 @@ subroutine sub2()
 ! CHECK:    fir.store %[[TMP10]] to %[[TRIP]] : !fir.ref<i32>
 ! CHECK:    %[[TMP11:.*]] = fir.load %[[I]] : !fir.ref<i32>
 ! CHECK:    %[[STEP_VAL:.*]] = fir.load %[[STEP_VAR]] : !fir.ref<i32>
-! CHECK:    %[[TMP12:.*]] = arith.addi %[[TMP11]], %[[STEP_VAL]] overflow<nsw> : i32
+! CHECK:    %[[TMP12:.*]] = arith.addi %[[TMP11]], %[[STEP_VAL]] : i32
 ! CHECK:    fir.store %[[TMP12]] to %[[I]] : !fir.ref<i32>
 ! CHECK:    cf.br ^[[HEADER]]
   end do
diff --git a/flang/test/Lower/host-associated.f90 b/flang/test/Lower/host-associated.f90
index 33acdff1bb74..9b4269df7bfc 100644
--- a/flang/test/Lower/host-associated.f90
+++ b/flang/test/Lower/host-associated.f90
@@ -1,5 +1,5 @@
 ! Test internal procedure host association lowering.
-! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false %s -o - | FileCheck %s
 
 ! -----------------------------------------------------------------------------
 !     Test non character intrinsic scalars
diff --git a/flang/test/Lower/infinite_loop.f90 b/flang/test/Lower/infinite_loop.f90
index de0bee779c5b..6942dda8d7a2 100644
--- a/flang/test/Lower/infinite_loop.f90
+++ b/flang/test/Lower/infinite_loop.f90
@@ -1,11 +1,9 @@
 ! RUN: bbc -emit-fir -hlfir=false -o - %s | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -o - %s | FileCheck %s
-! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fwrapv -o - %s | FileCheck %s --check-prefix=NO-NSW
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -flang-experimental-integer-overflow -o - %s | FileCheck %s --check-prefix=NSW
 
 ! Tests for infinite loop.
 
-! NO-NSW-NOT: overflow<nsw>
-
 subroutine empty_infinite()
   do
   end do
@@ -98,10 +96,10 @@ end subroutine
 ! CHECK-SAME: %[[C1_INDEX]] to %[[C10_INDEX]] step %[[C1_1]]
 ! CHECK-SAME: iter_args(%[[J_IV:.*]] = %[[J_LB]]) -> (index, i32) {
 ! CHECK:    fir.store %[[J_IV]] to %[[J_REF]] : !fir.ref<i32>
-! CHECK:    %[[J_NEXT:.*]] = arith.addi %[[J]], %[[C1_1]] overflow<nsw> : index
+! CHECK:    %[[J_NEXT:.*]] = arith.addi %[[J]], %[[C1_1]] : index
 ! CHECK:    %[[J_STEPCAST:.*]] = fir.convert %[[C1_1]] : (index) -> i32
 ! CHECK:    %[[J_IVLOAD:.*]] = fir.load %[[J_REF]] : !fir.ref<i32>
-! CHECK:    %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] overflow<nsw> : i32
+! CHECK:    %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] : i32
 ! CHECK:    fir.result %[[J_NEXT]], %[[J_IVINC]] : index, i32
 ! CHECK:  }
 ! CHECK:  fir.store %[[J_FINAL]]#1 to %[[J_REF]] : !fir.ref<i32>
@@ -109,6 +107,39 @@ end subroutine
 ! CHECK: ^[[RETURN]]:
 ! CHECK:   return
 
+! NSW-LABEL: structured_loop_in_infinite
+! NSW-SAME: %[[I_REF:.*]]: !fir.ref<i32>
+! NSW:  %[[J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFstructured_loop_in_infiniteEj"}
+! NSW:  cf.br ^[[BODY1:.*]]
+! NSW: ^[[BODY1]]:
+! NSW:  %[[I:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
+! NSW:  %[[C100:.*]] = arith.constant 100 : i32
+! NSW:  %[[COND:.*]] = arith.cmpi sgt, %[[I]], %[[C100]] : i32
+! NSW:  cf.cond_br %[[COND]], ^[[EXIT:.*]], ^[[BODY2:.*]]
+! NSW: ^[[EXIT]]:
+! NSW:  cf.br ^[[RETURN:.*]]
+! NSW: ^[[BODY2:.*]]:
+! NSW:  %[[C1:.*]] = arith.constant 1 : i32
+! NSW:  %[[C1_INDEX:.*]] = fir.convert %[[C1]] : (i32) -> index
+! NSW:  %[[C10:.*]] = arith.constant 10 : i32
+! NSW:  %[[C10_INDEX:.*]] = fir.convert %[[C10]] : (i32) -> index
+! NSW:  %[[C1_1:.*]] = arith.constant 1 : index
+! NSW:  %[[J_LB:.*]] = fir.convert %[[C1_INDEX]] : (index) -> i32
+! NSW:  %[[J_FINAL:.*]]:2 = fir.do_loop %[[J:[^ ]*]] =
+! NSW-SAME: %[[C1_INDEX]] to %[[C10_INDEX]] step %[[C1_1]]
+! NSW-SAME: iter_args(%[[J_IV:.*]] = %[[J_LB]]) -> (index, i32) {
+! NSW:    fir.store %[[J_IV]] to %[[J_REF]] : !fir.ref<i32>
+! NSW:    %[[J_NEXT:.*]] = arith.addi %[[J]], %[[C1_1]] overflow<nsw> : index
+! NSW:    %[[J_STEPCAST:.*]] = fir.convert %[[C1_1]] : (index) -> i32
+! NSW:    %[[J_IVLOAD:.*]] = fir.load %[[J_REF]] : !fir.ref<i32>
+! NSW:    %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] overflow<nsw> : i32
+! NSW:    fir.result %[[J_NEXT]], %[[J_IVINC]] : index, i32
+! NSW:  }
+! NSW:  fir.store %[[J_FINAL]]#1 to %[[J_REF]] : !fir.ref<i32>
+! NSW:  cf.br ^[[BODY1]]
+! NSW: ^[[RETURN]]:
+! NSW:   return
+
 subroutine empty_infinite_in_while(i)
   integer :: i
   do while (i .gt. 50)
diff --git a/flang/test/Lower/io-implied-do-fixes.f90 b/flang/test/Lower/io-implied-do-fixes.f90
index cd4fd43e0519..a6c115fa80de 100644
--- a/flang/test/Lower/io-implied-do-fixes.f90
+++ b/flang/test/Lower/io-implied-do-fixes.f90
@@ -1,20 +1,30 @@
 ! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
-! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false -fwrapv %s -o - | FileCheck %s --check-prefix=NO-NSW
+! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false -integer-overflow %s -o - | FileCheck %s --check-prefix=NSW
 ! UNSUPPORTED: system-windows
 
-! NO-NSW-NOT: overflow<nsw>
-
 ! CHECK-LABEL: func @_QPido1
 ! CHECK: %[[J_REF_ADDR:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFido1Eiptr.addr"}
 ! CHECK: %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.ptr<i32>>
 ! CHECK: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
 ! CHECK:   %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
 ! CHECK:   fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.ptr<i32>
-! CHECK:   %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
+! CHECK:   %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} : index
 ! CHECK:   fir.result %[[J_VAL_NEXT]] : index
 ! CHECK: }
 ! CHECK: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
 ! CHECK: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.ptr<i32>
+
+! NSW-LABEL: func @_QPido1
+! NSW: %[[J_REF_ADDR:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFido1Eiptr.addr"}
+! NSW: %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.ptr<i32>>
+! NSW: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
+! NSW:   %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
+! NSW:   fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.ptr<i32>
+! NSW:   %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
+! NSW:   fir.result %[[J_VAL_NEXT]] : index
+! NSW: }
+! NSW: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
+! NSW: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.ptr<i32>
 subroutine ido1
   integer, pointer :: iptr
   integer, target :: itgt
@@ -28,11 +38,23 @@ end subroutine
 ! CHECK: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
 ! CHECK: %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
 ! CHECK: fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
-! CHECK: %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
+! CHECK: %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} : index
 ! CHECK: fir.result %[[J_VAL_NEXT]] : index
 ! CHECK: }
 ! CHECK: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
 ! CHECK: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32>
+
+! NSW-LABEL: func @_QPido2
+! NSW: %[[J_REF_ADDR:.*]] = fir.alloca !fir.heap<i32> {uniq_name = "_QFido2Eiptr.addr"}
+! NSW: %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.heap<i32>>
+! NSW: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
+! NSW: %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
+! NSW: fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
+! NSW: %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
+! NSW: fir.result %[[J_VAL_NEXT]] : index
+! NSW: }
+! NSW: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
+! NSW: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32>
 subroutine ido2
   integer, allocatable :: iptr
   allocate(iptr)
@@ -47,12 +69,27 @@ end subroutine
 ! CHECK:    fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
 ! CHECK:    %[[RES:.*]] = fir.if %[[OK]] -> (i1) {
 ! CHECK:    }
-! CHECK:   %[[J_VAL_INC:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
+! CHECK:   %[[J_VAL_INC:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} : index
 ! CHECK:   %[[J_VAL_NEXT:.*]] = arith.select %[[RES]], %[[J_VAL_INC]], %[[J_VAL]] : index
 ! CHECK:   fir.result %[[J_VAL_NEXT]], %[[RES]] : index, i1
 ! CHECK:  }
 ! CHECK:  %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]]#0 : (index) -> i32
 ! CHECK:  fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32
+
+! NSW-LABEL: func @_QPido3
+! NSW:  %[[J_REF_ADDR:.*]] = fir.alloca !fir.heap<i32> {uniq_name = "_QFido3Ej.addr"}
+! NSW:  %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.heap<i32>>
+! NSW:  %[[J_VAL_FINAL:.*]]:2 = fir.iterate_while (%[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}}) and (%[[OK:.*]] = {{.*}}) -> (index, i1) {
+! NSW:    %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
+! NSW:    fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
+! NSW:    %[[RES:.*]] = fir.if %[[OK]] -> (i1) {
+! NSW:    }
+! NSW:   %[[J_VAL_INC:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
+! NSW:   %[[J_VAL_NEXT:.*]] = arith.select %[[RES]], %[[J_VAL_INC]], %[[J_VAL]] : index
+! NSW:   fir.result %[[J_VAL_NEXT]], %[[RES]] : index, i1
+! NSW:  }
+! NSW:  %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]]#0 : (index) -> i32
+! NSW:  fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32
 subroutine ido3
   integer, allocatable :: j
   allocate(j)
diff --git a/flang/test/Lower/loops2.f90 b/flang/test/Lower/loops2.f90
index 60a6bf6c5311..0a587234a991 100644
--- a/flang/test/Lower/loops2.f90
+++ b/flang/test/Lower/loops2.f90
@@ -107,7 +107,7 @@ contains
 ! CHECK:       ^bb4:
 ! CHECK:         %[[VAL_20:.*]] = fir.load %[[VAL_3]] : !fir.ptr<i32>
 ! CHECK:         %[[VAL_21:.*]] = arith.constant 1 : i32
-! CHECK:         %[[VAL_22:.*]] = arith.addi %[[VAL_20]], %[[VAL_21]] overflow<nsw> : i32
+! CHECK:         %[[VAL_22:.*]] = arith.addi %[[VAL_20]], %[[VAL_21]] : i32
 ! CHECK:         fir.store %[[VAL_22]] to %[[VAL_3]] : !fir.ptr<i32>
 ! CHECK:         br ^bb1
 ! CHECK:       ^bb5:
diff --git a/flang/test/Lower/mixed_loops.f90 b/flang/test/Lower/mixed_loops.f90
index 991fd7aa82bb..1aa0225129be 100644
--- a/flang/test/Lower/mixed_loops.f90
+++ b/flang/test/Lower/mixed_loops.f90
@@ -53,7 +53,7 @@ subroutine while_inside_do_loop
   ! CHECK: fir.store %[[TDEC]] to %[[T_REF]]
   ! CHECK: %[[I3:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
   ! CHECK: %[[C1_2:.*]] = arith.constant 1 : i32
-  ! CHECK: %[[IINC:.*]] = arith.addi %[[I3]], %[[C1_2]] overflow<nsw> : i32
+  ! CHECK: %[[IINC:.*]] = arith.addi %[[I3]], %[[C1_2]] : i32
   ! CHECK: fir.store %[[IINC]] to %[[I_REF]] : !fir.ref<i32>
   ! CHECK: br ^[[HDR1]]
   end do
@@ -100,10 +100,10 @@ subroutine do_inside_while_loop
         ! CHECK-DAG: %[[C2:.*]] = arith.constant 2 : i32
         ! CHECK: %[[JINC:.*]] = arith.muli %[[C2]], %[[J2]] : i32
         ! CHECK: fir.store %[[JINC]] to %[[J_REF]] : !fir.ref<i32>
-        ! CHECK: %[[IINC:.*]] = arith.addi %[[IDX]], %[[C1]] overflow<nsw> : index
+        ! CHECK: %[[IINC:.*]] = arith.addi %[[IDX]], %[[C1]] : index
         ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[C1]] : (index) -> i32
         ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-        ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i32
+        ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
         ! CHECK: fir.result %[[IINC]], %[[I_IVINC]] : index, i32
       do i=8,13
         j=j*2
diff --git a/flang/test/Lower/vector-subscript-io.f90 b/flang/test/Lower/vector-subscript-io.f90
index 372130fd0990..129e3ee1206c 100644
--- a/flang/test/Lower/vector-subscript-io.f90
+++ b/flang/test/Lower/vector-subscript-io.f90
@@ -30,7 +30,7 @@ subroutine simple(x, y)
 ! CHECK:   %[[VAL_19:.*]] = fir.array_coor %[[VAL_20]](%[[VAL_10]]) {{\[}}%[[VAL_11]]] %[[VAL_18]] : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, !fir.slice<1>, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_21:.*]] = fir.convert %[[VAL_19]] : (!fir.ref<i32>) -> !fir.ref<i64>
 ! CHECK:   %[[VAL_22:.*]] = fir.call @_FortranAioInputInteger(%[[VAL_9]], %[[VAL_21]], %[[VAL_3]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
-! CHECK:   %[[VAL_23:.*]] = arith.addi %[[VAL_12]], %[[VAL_6]] overflow<nsw> : index
+! CHECK:   %[[VAL_23:.*]] = arith.addi %[[VAL_12]], %[[VAL_6]] : index
 ! CHECK:   %[[VAL_24:.*]] = arith.subi %[[VAL_13]], %[[VAL_6]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_23]], %[[VAL_24]] : index, index)
 ! CHECK: ^bb3:
@@ -79,7 +79,7 @@ subroutine only_once(x)
 ! CHECK:   %[[VAL_49:.*]] = fir.convert %[[VAL_48]] : (i32) -> index
 ! CHECK:   %[[VAL_50:.*]] = fir.array_coor %[[VAL_51]] {{\[}}%[[VAL_42]]] %[[VAL_46]], %[[VAL_49]] : (!fir.box<!fir.array<?x?xf32>>, !fir.slice<2>, index, index) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_52:.*]] = fir.call @_FortranAioInputReal32(%[[VAL_34]], %[[VAL_50]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
-! CHECK:   %[[VAL_53:.*]] = arith.addi %[[VAL_43]], %[[VAL_30]] overflow<nsw> : index
+! CHECK:   %[[VAL_53:.*]] = arith.addi %[[VAL_43]], %[[VAL_30]] : index
 ! CHECK:   %[[VAL_54:.*]] = arith.subi %[[VAL_44]], %[[VAL_30]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_53]], %[[VAL_54]] : index, index)
 ! CHECK: ^bb3:
@@ -122,7 +122,7 @@ subroutine with_assumed_shapes(x, y)
 ! CHECK:   %[[VAL_77:.*]] = fir.array_coor %[[VAL_78]] {{\[}}%[[VAL_70]]] %[[VAL_76]] : (!fir.box<!fir.array<?xi32>>, !fir.slice<1>, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_79:.*]] = fir.convert %[[VAL_77]] : (!fir.ref<i32>) -> !fir.ref<i64>
 ! CHECK:   %[[VAL_80:.*]] = fir.call @_FortranAioInputInteger(%[[VAL_67]], %[[VAL_79]], %[[VAL_62]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
-! CHECK:   %[[VAL_81:.*]] = arith.addi %[[VAL_71]], %[[VAL_64]] overflow<nsw> : index
+! CHECK:   %[[VAL_81:.*]] = arith.addi %[[VAL_71]], %[[VAL_64]] : index
 ! CHECK:   %[[VAL_82:.*]] = arith.subi %[[VAL_72]], %[[VAL_64]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_81]], %[[VAL_82]] : index, index)
 ! CHECK: ^bb3:
@@ -162,7 +162,7 @@ subroutine lower_bounds(x, y)
 ! CHECK:   %[[VAL_107:.*]] = fir.array_coor %[[VAL_108]](%[[VAL_97]]) {{\[}}%[[VAL_99]]] %[[VAL_91]], %[[VAL_106]] : (!fir.ref<!fir.array<4x6xi32>>, !fir.shapeshift<2>, !fir.slice<2>, index, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_109:.*]] = fir.convert %[[VAL_107]] : (!fir.ref<i32>) -> !fir.ref<i64>
 ! CHECK:   %[[VAL_110:.*]] = fir.call @_FortranAioInputInteger(%[[VAL_96]], %[[VAL_109]], %[[VAL_90]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
-! CHECK:   %[[VAL_111:.*]] = arith.addi %[[VAL_100]], %[[VAL_93]] overflow<nsw> : index
+! CHECK:   %[[VAL_111:.*]] = arith.addi %[[VAL_100]], %[[VAL_93]] : index
 ! CHECK:   %[[VAL_112:.*]] = arith.subi %[[VAL_101]], %[[VAL_93]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_111]], %[[VAL_112]] : index, index)
 ! CHECK: ^bb3:
@@ -202,11 +202,11 @@ subroutine two_vectors(x, y1, y2)
 ! CHECK:   %[[VAL_138:.*]] = fir.convert %[[VAL_137]] : (i32) -> index
 ! CHECK:   %[[VAL_139:.*]] = fir.array_coor %[[VAL_140]](%[[VAL_123]]) {{\[}}%[[VAL_124]]] %[[VAL_134]], %[[VAL_138]] : (!fir.ref<!fir.array<4x4xf32>>, !fir.shape<2>, !fir.slice<2>, index, index) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_141:.*]] = fir.call @_FortranAioInputReal32(%[[VAL_122]], %[[VAL_139]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
-! CHECK:   %[[VAL_142:.*]] = arith.addi %[[VAL_128]], %[[VAL_119]] overflow<nsw> : index
+! CHECK:   %[[VAL_142:.*]] = arith.addi %[[VAL_128]], %[[VAL_119]] : index
 ! CHECK:   %[[VAL_143:.*]] = arith.subi %[[VAL_129]], %[[VAL_119]] : index
 ! CHECK:   cf.br ^bb2(%[[VAL_142]], %[[VAL_143]] : index, index)
 ! CHECK: ^bb4:
-! CHECK:   %[[VAL_144:.*]] = arith.addi %[[VAL_125]], %[[VAL_119]] overflow<nsw> : index
+! CHECK:   %[[VAL_144:.*]] = arith.addi %[[VAL_125]], %[[VAL_119]] : index
 ! CHECK:   %[[VAL_145:.*]] = arith.subi %[[VAL_126]], %[[VAL_119]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_144]], %[[VAL_145]] : index, index)
 ! CHECK: ^bb5:
@@ -245,11 +245,11 @@ subroutine triplets_and_vector(x, y)
 ! CHECK:   %[[VAL_169:.*]] = fir.array_coor %[[VAL_170]](%[[VAL_157]]) {{\[}}%[[VAL_158]]] %[[VAL_162]], %[[VAL_168]] : (!fir.ref<!fir.array<4x4xcomplex<f32>>>, !fir.shape<2>, !fir.slice<2>, index, index) -> !fir.ref<complex<f32>>
 ! CHECK:   %[[VAL_171:.*]] = fir.convert %[[VAL_169]] : (!fir.ref<complex<f32>>) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_172:.*]] = fir.call @_FortranAioInputComplex32(%[[VAL_156]], %[[VAL_171]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
-! CHECK:   %[[VAL_173:.*]] = arith.addi %[[VAL_162]], %[[VAL_153]] overflow<nsw> : index
+! CHECK:   %[[VAL_173:.*]] = arith.addi %[[VAL_162]], %[[VAL_153]] : index
 ! CHECK:   %[[VAL_174:.*]] = arith.subi %[[VAL_163]], %[[VAL_153]] : index
 ! CHECK:   cf.br ^bb2(%[[VAL_173]], %[[VAL_174]] : index, index)
 ! CHECK: ^bb4:
-! CHECK:   %[[VAL_175:.*]] = arith.addi %[[VAL_159]], %[[VAL_153]] overflow<nsw> : index
+! CHECK:   %[[VAL_175:.*]] = arith.addi %[[VAL_159]], %[[VAL_153]] : index
 ! CHECK:   %[[VAL_176:.*]] = arith.subi %[[VAL_160]], %[[VAL_153]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_175]], %[[VAL_176]] : index, index)
 ! CHECK: ^bb5:
@@ -287,7 +287,7 @@ subroutine simple_char(x, y)
 ! CHECK:   %[[VAL_200:.*]] = fir.convert %[[VAL_199]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
 ! CHECK:   %[[VAL_201:.*]] = fir.convert %[[VAL_184]]#1 : (index) -> i64
 ! CHECK:   %[[VAL_202:.*]] = fir.call @_FortranAioInputAscii(%[[VAL_189]], %[[VAL_200]], %[[VAL_201]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64) -> i1
-! CHECK:   %[[VAL_203:.*]] = arith.addi %[[VAL_192]], %[[VAL_183]] overflow<nsw> : index
+! CHECK:   %[[VAL_203:.*]] = arith.addi %[[VAL_192]], %[[VAL_183]] : index
 ! CHECK:   %[[VAL_204:.*]] = arith.subi %[[VAL_193]], %[[VAL_183]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_203]], %[[VAL_204]] : index, index)
 ! CHECK: ^bb3:
@@ -333,7 +333,7 @@ subroutine substring(x, y, i, j)
 ! CHECK:   %[[VAL_238:.*]] = fir.convert %[[VAL_233]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
 ! CHECK:   %[[VAL_239:.*]] = fir.convert %[[VAL_237]] : (index) -> i64
 ! CHECK:   %[[VAL_240:.*]] = fir.call @_FortranAioInputAscii(%[[VAL_213]], %[[VAL_238]], %[[VAL_239]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64) -> i1
-! CHECK:   %[[VAL_241:.*]] = arith.addi %[[VAL_221]], %[[VAL_210]] overflow<nsw> : index
+! CHECK:   %[[VAL_241:.*]] = arith.addi %[[VAL_221]], %[[VAL_210]] : index
 ! CHECK:   %[[VAL_242:.*]] = arith.subi %[[VAL_222]], %[[VAL_210]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_241]], %[[VAL_242]] : index, index)
 ! CHECK: ^bb3:
@@ -366,7 +366,7 @@ subroutine complex_part(z, y)
 ! CHECK:   %[[VAL_260:.*]] = fir.convert %[[VAL_259]] : (i32) -> index
 ! CHECK:   %[[VAL_261:.*]] = fir.array_coor %[[VAL_262]] {{\[}}%[[VAL_254]]] %[[VAL_260]] : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.slice<1>, index) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_263:.*]] = fir.call @_FortranAioInputReal32(%[[VAL_251]], %[[VAL_261]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
-! CHECK:   %[[VAL_264:.*]] = arith.addi %[[VAL_255]], %[[VAL_248]] overflow<nsw> : index
+! CHECK:   %[[VAL_264:.*]] = arith.addi %[[VAL_255]], %[[VAL_248]] : index
 ! CHECK:   %[[VAL_265:.*]] = arith.subi %[[VAL_256]], %[[VAL_248]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_264]], %[[VAL_265]] : index, index)
 ! CHECK: ^bb3:
@@ -414,7 +414,7 @@ subroutine simple_derived(x, y)
 ! CHECK:   %[[VAL_288:.*]] = fir.embox %[[VAL_286]] : (!fir.ref<!fir.type<_QMderived_typesTt{i:i32,c:!fir.char<1,2>}>>) -> !fir.box<!fir.type<_QMderived_typesTt{i:i32,c:!fir.char<1,2>}>>
 ! CHECK:   %[[VAL_289:.*]] = fir.convert %[[VAL_288]] : (!fir.box<!fir.type<_QMderived_typesTt{i:i32,c:!fir.char<1,2>}>>) -> !fir.box<none>
 ! CHECK:   %[[VAL_290:.*]] = fir.call @_FortranAioInputDerivedType(%[[VAL_276]], %[[VAL_289]], {{.*}}) {{.*}}: (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
-! CHECK:   %[[VAL_291:.*]] = arith.addi %[[VAL_279]], %[[VAL_273]] overflow<nsw> : index
+! CHECK:   %[[VAL_291:.*]] = arith.addi %[[VAL_279]], %[[VAL_273]] : index
 ! CHECK:   %[[VAL_292:.*]] = arith.subi %[[VAL_280]], %[[VAL_273]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_291]], %[[VAL_292]] : index, index)
 ! CHECK: ^bb3:
@@ -463,11 +463,11 @@ subroutine with_path(b, i)
 ! CHECK:   %[[VAL_325:.*]] = fir.array_coor %[[VAL_326:.*]](%[[VAL_313]]) {{\[}}%[[VAL_315]]] %[[VAL_301]], %[[VAL_324]], %[[VAL_316]] : (!fir.box<!fir.array<?x?x?x!fir.type<_QMderived_typesTt2{a:!fir.array<5x5x!fir.type<_QMderived_typesTt{i:i32,c:!fir.char<1,2>}>>}>>>, !fir.shift<3>, !fir.slice<3>, index, index, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_327:.*]] = fir.convert %[[VAL_325]] : (!fir.ref<i32>) -> !fir.ref<i64>
 ! CHECK:   %[[VAL_328:.*]] = fir.call @_FortranAioInputInteger(%[[VAL_308]], %[[VAL_327]], %[[VAL_302]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
-! CHECK:   %[[VAL_329:.*]] = arith.addi %[[VAL_319]], %[[VAL_305]] overflow<nsw> : index
+! CHECK:   %[[VAL_329:.*]] = arith.addi %[[VAL_319]], %[[VAL_305]] : index
 ! CHECK:   %[[VAL_330:.*]] = arith.subi %[[VAL_320]], %[[VAL_305]] : index
 ! CHECK:   cf.br ^bb2(%[[VAL_329]], %[[VAL_330]] : index, index)
 ! CHECK: ^bb4:
-! CHECK:   %[[VAL_331:.*]] = arith.addi %[[VAL_316]], %[[VAL_305]] overflow<nsw> : index
+! CHECK:   %[[VAL_331:.*]] = arith.addi %[[VAL_316]], %[[VAL_305]] : index
 ! CHECK:   %[[VAL_332:.*]] = arith.subi %[[VAL_317]], %[[VAL_305]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_331]], %[[VAL_332]] : index, index)
 ! CHECK: ^bb5:
@@ -505,7 +505,7 @@ subroutine simple_iostat(x, y, j, stat)
 ! CHECK:   %[[VAL_355:.*]] = fir.convert %[[VAL_354]] : (i32) -> index
 ! CHECK:   %[[VAL_356:.*]] = fir.array_coor %[[VAL_357]] {{\[}}%[[VAL_347]]] %[[VAL_355]] : (!fir.box<!fir.array<?xf32>>, !fir.slice<1>, index) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_358:.*]] = fir.call @_FortranAioInputReal32(%[[VAL_343]], %[[VAL_356]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
-! CHECK:   %[[VAL_359:.*]] = arith.addi %[[VAL_349]], %[[VAL_338]] overflow<nsw> : index
+! CHECK:   %[[VAL_359:.*]] = arith.addi %[[VAL_349]], %[[VAL_338]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_359]], %[[VAL_358]] : index, i1)
 ! CHECK: ^bb3:
 ! CHECK:   cf.cond_br %[[VAL_350]], ^bb4, ^bb5
@@ -568,10 +568,10 @@ subroutine iostat_in_io_loop(k, j, stat)
 ! CHECK:   %[[VAL_399:.*]] = fir.array_coor %[[VAL_400]](%[[VAL_387]]) {{\[}}%[[VAL_389]]] %[[VAL_394]], %[[VAL_398]] : (!fir.ref<!fir.array<3x5xi32>>, !fir.shape<2>, !fir.slice<2>, index, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_401:.*]] = fir.convert %[[VAL_399]] : (!fir.ref<i32>) -> !fir.ref<i64>
 ! CHECK:   %[[VAL_402:.*]] = fir.call @_FortranAioInputInteger(%[[VAL_378]], %[[VAL_401]], %[[VAL_374]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
-! CHECK:   %[[VAL_403:.*]] = arith.addi %[[VAL_390]], %[[VAL_371]] overflow<nsw> : index
+! CHECK:   %[[VAL_403:.*]] = arith.addi %[[VAL_390]], %[[VAL_371]] : index
 ! CHECK:   cf.br ^bb4(%[[VAL_403]], %[[VAL_402]] : index, i1)
 ! CHECK: ^bb6(%[[VAL_404:.*]]: i1):
-! CHECK:   %[[VAL_405:.*]] = arith.addi %[[VAL_380]], %[[VAL_371]] overflow<nsw> : index
+! CHECK:   %[[VAL_405:.*]] = arith.addi %[[VAL_380]], %[[VAL_371]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_405]], %[[VAL_404]] : index, i1)
 ! CHECK: ^bb7:
 ! CHECK:   %[[VAL_406:.*]] = fir.convert %[[VAL_380]] : (index) -> i32
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index af38bf272b02..fe5e36f704c7 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -234,6 +234,12 @@ static llvm::cl::opt<bool> integerWrapAround(
     llvm::cl::desc("Treat signed integer overflow as two's complement"),
     llvm::cl::init(false));
 
+// TODO: integrate this option with the above
+static llvm::cl::opt<bool>
+    setNSW("integer-overflow",
+           llvm::cl::desc("add nsw flag to internal operations"),
+           llvm::cl::init(false));
+
 #define FLANG_EXCLUDE_CODEGEN
 #include "flang/Optimizer/Passes/CommandLineOpts.h"
 #include "flang/Optimizer/Passes/Pipelines.h"
@@ -375,6 +381,7 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
   loweringOptions.setNoPPCNativeVecElemOrder(enableNoPPCNativeVecElemOrder);
   loweringOptions.setLowerToHighLevelFIR(useHLFIR || emitHLFIR);
   loweringOptions.setIntegerWrapAround(integerWrapAround);
+  loweringOptions.setNSWOnLoopVarInc(setNSW);
   std::vector<Fortran::lower::EnvironmentDefault> envDefaults = {};
   Fortran::frontend::TargetOptions targetOpts;
   Fortran::frontend::CodeGenOptions cgOpts;
@@ -460,7 +467,7 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
 
     // Add O2 optimizer pass pipeline.
     MLIRToLLVMPassPipelineConfig config(llvm::OptimizationLevel::O2);
-    config.NSWOnLoopVarInc = !integerWrapAround;
+    config.NSWOnLoopVarInc = setNSW;
     fir::registerDefaultInlinerPass(config);
     fir::createDefaultFIROptimizerPassPipeline(pm, config);
   }
-- 
GitLab


From 1f6741c1645954b1f4b2fbca470a20081f5e75af Mon Sep 17 00:00:00 2001
From: Jan Voung <jvoung@google.com>
Date: Mon, 28 Oct 2024 10:28:02 -0400
Subject: [PATCH 321/693] [clang][dataflow] Don't clear cached field state if
 field is const (#113698)

... in the unchecked optional access model.
---
 .../Models/UncheckedOptionalAccessModel.cpp   |  8 +++++--
 .../UncheckedOptionalAccessModelTest.cpp      | 23 ++++++++++++++++++-
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp b/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
index b0bd8274405d..31ae2b94f5b6 100644
--- a/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
@@ -601,10 +601,14 @@ void handleNonConstMemberCall(const CallExpr *CE,
                               dataflow::RecordStorageLocation *RecordLoc,
                               const MatchFinder::MatchResult &Result,
                               LatticeTransferState &State) {
-  // When a non-const member function is called, reset some state.
   if (RecordLoc != nullptr) {
+    // When a non-const member function is called, clear all (non-const)
+    // optional fields of the receiver. Const-qualified fields can't be
+    // changed (at least, not without UB).
     for (const auto &[Field, FieldLoc] : RecordLoc->children()) {
-      if (isSupportedOptionalType(Field->getType())) {
+      QualType FieldType = Field->getType();
+      if (!FieldType.isConstQualified() &&
+          isSupportedOptionalType(Field->getType())) {
         auto *FieldRecordLoc = cast_or_null<RecordStorageLocation>(FieldLoc);
         if (FieldRecordLoc) {
           setHasValue(*FieldRecordLoc, State.Env.makeAtomicBoolValue(),
diff --git a/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp b/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
index 22fe347c4255..5b64eaca0e10 100644
--- a/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
@@ -2167,7 +2167,7 @@ TEST_P(UncheckedOptionalAccessTest, OptionalReturnedFromFuntionCall) {
   )");
 }
 
-TEST_P(UncheckedOptionalAccessTest, OptionalFieldModified) {
+TEST_P(UncheckedOptionalAccessTest, NonConstMethodMayClearOptionalField) {
   ExpectDiagnosticsFor(
       R"(
     #include "unchecked_optional_access_test.h"
@@ -2187,6 +2187,27 @@ TEST_P(UncheckedOptionalAccessTest, OptionalFieldModified) {
   )");
 }
 
+TEST_P(UncheckedOptionalAccessTest,
+       NonConstMethodMayNotClearConstOptionalField) {
+  ExpectDiagnosticsFor(
+      R"(
+    #include "unchecked_optional_access_test.h"
+
+    struct Foo {
+      const $ns::$optional<std::string> opt;
+      void clear();
+    };
+
+    void target(Foo& foo) {
+      if (foo.opt) {
+        foo.opt.value();
+        foo.clear();
+        foo.opt.value();
+      }
+    }
+  )");
+}
+
 TEST_P(UncheckedOptionalAccessTest, StdSwap) {
   ExpectDiagnosticsFor(
       R"(
-- 
GitLab


From d83485a080341688cbb9d5c07a3839bd97f9001e Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Mon, 28 Oct 2024 18:48:23 +0400
Subject: [PATCH 322/693] [clang][NFC] Add test for CWG issues about linkage in
 cross-TU context (#113736)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

[CWG279](https://cplusplus.github.io/CWG/issues/279.html) Correspondence
of "names for linkage purposes"
====================

[P1787R6](https://wg21.link/p1787r6):
> CWG1884, CWG279, and CWG338 are resolved by defining entity identity
explicitly.

In both cases discussed in the issue, declaration matching succeeds per
[[basic.link]/8](https://eel.is/c++draft/basic.link#8), thanks to the
> considering declarations of unnamed types to introduce their names for
linkage purposes, if any ([dcl.typedef], [dcl.enum])

and

> they both declare names with external linkage

parts. Which means that both pairs of declarations of `S` and `S2`
declare the same entity.
[[basic.link]/11](https://eel.is/c++draft/basic.link#11) is also
satisfied, because in all cases type is declared.

Then cases diverge. `S` is a pair of a forward declaration and a
definition, which means it's not a subject to
[[basic.def.odr]/15](https://eel.is/c++draft/basic.def.odr#15), hence
it's well-formed. Whereas `S2` is a pair of two definitions, so they are
subjects to
[[basic.def.odr]/15](https://eel.is/c++draft/basic.def.odr#15), which
they do not satisfy due to the violation of the following requirement:

> — Each such definition shall consist of the same sequence of tokens,
<...>

Because per [[class.pre/2]](https://eel.is/c++draft/class#pre-2), class
definition is class-specifier, which includes class-head, but class-head
is different between definitions.



[CWG338](https://cplusplus.github.io/CWG/issues/338.html) Enumerator
name with linkage used as class name in other translation unit
====================

[P1787R6](https://wg21.link/p1787r6):
> CWG1884, CWG279, and CWG338 are resolved by defining entity identity
explicitly.

I believe this CWG issue points to the same underlying issue with old
[basic.link]/9 as CWG1884 (see below), so I'm calling it a duplicate of
CWG1884. Cases described there are a part of an extensive CWG1884 test.
Also worth noting that enumerators don't have linkage these days.



[CWG1884](https://cplusplus.github.io/CWG/issues/1898.html) Unclear
requirements for same-named external-linkage entities
====================

[P1787R6](https://wg21.link/p1787r6):
> CWG1884, CWG279, and CWG338 are resolved by defining entity identity
explicitly.

[basic.link]/9 quoted in the issue is now split into several pieces.

> Two names that are the same (6.1 [basic.pre])

I believe this is corresponding declarations now, defined in
[[basic.scope.scope]/4](https://eel.is/c++draft/basic.scope#scope-4).

> and that are declared in different scopes shall denote the same
variable, function, type, enumerator, template or namespace if

This is covered by
[[basic.link]/11](https://eel.is/c++draft/basic.link#11) after it's
determined by [[basic.link]/8](https://eel.is/c++draft/basic.link#8)
that two declarations declare the same entity.

> — both names have external linkage or else both names have internal
linkage and are declared in the same translation unit; and
> — both names refer to members of the same namespace or to members, not
by inheritance, of the same class; and

Most of this is covered by
[[basic.link]/8](https://eel.is/c++draft/basic.link#8).

> — when both names denote functions, the parameter-type-lists of the
functions (9.3.4.6 [dcl.fct]) are identical; and
> — when both names denote function templates, the signatures (13.7.7.2
[temp.over.link]) are the same.

This is corresponding overloads now, defined in
[[basic.scope.scope]/4](https://eel.is/c++draft/basic.scope#scope-4).

> Among other things, it should be clarified that "declared in" refers
to the namespace of which the name is a member, not the lexical scope in
which the declaration appears (which affects friend declarations,
block-scope extern declarations, and elaborated-type-specifiers).

This is addressed by "have the same target scope" part of
[[basic.link]/8](https://eel.is/c++draft/basic.link#8).

The tests basically test
[[basic.link]/11](https://eel.is/c++draft/basic.link#11) from the
following standpoint:
> The intent is that this rule prevents declaring a name with extenal
linkage to be, for instance, a type in one translation unit and a
namespace in a different translation unit.

See the comment in the beginning of the test for details on testing
approach. Reviewers are advised to check compiler output for any
surprises, but if they are in a hurry, they can search for `FIXME`,
`OK`, and `#cwg1884` to see the most of irregularities in Clang behavior
across the test (not all of them are incorrect behavior, though).
---
 clang/test/CXX/drs/cwg1884.cpp | 643 +++++++++++++++++++++++++++++++++
 clang/test/CXX/drs/cwg18xx.cpp |   2 +
 clang/test/CXX/drs/cwg279.cpp  |  53 +++
 clang/test/CXX/drs/cwg2xx.cpp  |   2 +
 clang/test/CXX/drs/cwg3xx.cpp  |   2 +
 clang/www/cxx_dr_status.html   |   6 +-
 6 files changed, 705 insertions(+), 3 deletions(-)
 create mode 100644 clang/test/CXX/drs/cwg1884.cpp
 create mode 100644 clang/test/CXX/drs/cwg279.cpp

diff --git a/clang/test/CXX/drs/cwg1884.cpp b/clang/test/CXX/drs/cwg1884.cpp
new file mode 100644
index 000000000000..c4f76baa3933
--- /dev/null
+++ b/clang/test/CXX/drs/cwg1884.cpp
@@ -0,0 +1,643 @@
+// RUN: rm -rf %t
+// RUN: mkdir %t
+// RUN: split-file --leading-lines %s %t
+// RUN: %clang_cc1 -std=c++20 -pedantic-errors -fexceptions -fcxx-exceptions %t/cwg1884_A.cppm -triple x86_64-unknown-unknown -emit-module-interface -o %t/cwg1884_A.pcm
+// RUN: %clang_cc1 -std=c++20 -verify=since-cxx20 -pedantic-errors -fexceptions -fcxx-exceptions -triple x86_64-unknown-unknown %t/cwg1884.cpp -fmodule-file=cwg1884_A=%t/cwg1884_A.pcm
+// RUN: %clang_cc1 -std=c++23 -pedantic-errors -fexceptions -fcxx-exceptions %t/cwg1884_A.cppm -triple x86_64-unknown-unknown -emit-module-interface -o %t/cwg1884_A.pcm
+// RUN: %clang_cc1 -std=c++23 -verify=since-cxx20 -pedantic-errors -fexceptions -fcxx-exceptions -triple x86_64-unknown-unknown %t/cwg1884.cpp -fmodule-file=cwg1884_A=%t/cwg1884_A.pcm
+// RUN: %clang_cc1 -std=c++2c -pedantic-errors -fexceptions -fcxx-exceptions %t/cwg1884_A.cppm -triple x86_64-unknown-unknown -emit-module-interface -o %t/cwg1884_A.pcm
+// RUN: %clang_cc1 -std=c++2c -verify=since-cxx20 -pedantic-errors -fexceptions -fcxx-exceptions -triple x86_64-unknown-unknown %t/cwg1884.cpp -fmodule-file=cwg1884_A=%t/cwg1884_A.pcm
+
+// cwg1884: partial
+// Cases b11, e11, g3, g4 are problematic, but we handle the other 101 cases fine.
+
+// _N4993_.[basic.link]/11:
+// For any two declarations of an entity E:
+//   — If one declares E to be a variable or function,
+//     the other shall declare E as one of the same type.
+//   — If one declares E to be an enumerator, the other shall do so.
+//   — If one declares E to be a namespace, the other shall do so.
+//   — If one declares E to be a type,
+//     the other shall declare E to be a type of the same kind (9.2.9.5).
+//   — If one declares E to be a class template,
+//     the other shall do so with the same kind and an equivalent template-head (13.7.7.2).
+//     [Note 5 : The declarations can supply different default template arguments. — end note]
+//   — If one declares E to be a function template or a (partial specialization of a) variable template,
+//     the other shall declare E to be one with an equivalent template-head and type.
+//   — If one declares E to be an alias template,
+//     the other shall declare E to be one with an equivalent template-head and defining-type-id.
+//   — If one declares E to be a concept, the other shall do so.
+// Types are compared after all adjustments of types (during which typedefs (9.2.4) are replaced by their definitions);
+// declarations for an array object can specify array types that differ by the presence or absence of a major array bound (9.3.4.5).
+// No diagnostic is required if neither declaration is reachable from the other.
+
+// The structure of the test is the following. First, module cwg1884_A
+// provides all (significant) kinds of entities, each named 'a' through 'h', and copies of them.
+// Then the .cpp file does MxN kind of testing, where it tests one kind of entity against every other kind.
+
+//--- cwg1884_A.cppm
+export module cwg1884_A;
+
+export {
+int a1;
+int a2;
+int a3;
+int a4;
+int a5;
+int a6;
+int a7;
+int a8;
+int a9;
+int a10;
+int a11;
+void b1();
+void b2();
+void b3();
+void b4();
+void b5();
+void b6();
+void b7();
+void b8();
+void b9();
+void b10();
+void b11();
+enum E {
+  c1,
+  c2, 
+  c3,
+  c4,
+  c5,
+  c6,
+  c7,
+  c8,
+  c9,
+  c10
+};
+namespace d1 {}
+namespace d2 {}
+namespace d3 {}
+namespace d4 {}
+namespace d5 {}
+namespace d6 {}
+namespace d7 {}
+namespace d8 {}
+namespace d9 {}
+namespace d10 {}
+struct e1;
+struct e2;
+struct e3;
+struct e4;
+struct e5;
+struct e6;
+struct e7;
+struct e8;
+struct e9;
+struct e10;
+struct e11;
+struct e12;
+struct e13;
+template <typename>
+class f1;
+template <typename>
+class f2;
+template <typename>
+class f3;
+template <typename>
+class f4;
+template <typename>
+class f5;
+template <typename>
+class f6;
+template <typename>
+class f7;
+template <typename>
+class f8;
+template <typename>
+class f9;
+template <typename>
+class f10;
+template <typename>
+class f11;
+template <typename>
+void g1(int);
+template <typename>
+void g2(int);
+template <typename>
+void g3(int);
+template <typename>
+void g4(int);
+template <typename>
+void g5(int);
+template <typename>
+void g6(int);
+template <typename>
+void g7(int);
+template <typename>
+void g8(int);
+template <typename>
+void g9(int);
+template <typename>
+void g10(int);
+template <typename, typename>
+int h1;
+template <typename, typename>
+int h2;
+template <typename, typename>
+int h3;
+template <typename, typename>
+int h4;
+template <typename, typename>
+int h5;
+template <typename, typename>
+int h6;
+template <typename, typename>
+int h7;
+template <typename, typename>
+int h8;
+template <typename, typename>
+int h9;
+template <typename, typename>
+int h10;
+template <typename>
+using i1 = int;
+template <typename>
+using i2 = int;
+template <typename>
+using i3 = int;
+template <typename>
+using i4 = int;
+template <typename>
+using i5 = int;
+template <typename>
+using i6 = int;
+template <typename>
+using i7 = int;
+template <typename>
+using i8 = int;
+template <typename>
+using i9 = int;
+template <typename>
+using i10 = int;
+template <typename>
+using i11 = int;
+template <typename>
+concept j1 = true;
+template <typename>
+concept j2 = true;
+template <typename>
+concept j3 = true;
+template <typename>
+concept j4 = true;
+template <typename>
+concept j5 = true;
+template <typename>
+concept j6 = true;
+template <typename>
+concept j7 = true;
+template <typename>
+concept j8 = true;
+template <typename>
+concept j9 = true;
+template <typename>
+concept j10 = true;
+template <typename>
+concept j11 = true;
+} // export
+
+
+//--- cwg1884.cpp
+import cwg1884_A;
+
+// FIXME: we don't diagnose several cases we should be. They are marked with MISSING prefix.
+
+// Part A: matching against `int a;`
+// ---------------------------------
+
+void a1();
+// since-cxx20-error@-1 {{redefinition of 'a1' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:42 {{previous definition is here}}
+enum Ea {
+  a2
+  // since-cxx20-error@-1 {{redefinition of 'a2'}}
+  //   since-cxx20-note@cwg1884_A.cppm:43 {{previous definition is here}}
+};
+namespace a3 {}
+// since-cxx20-error@-1 {{redefinition of 'a3' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:44 {{previous definition is here}}
+struct a4;
+// @-1 OK, types and variables do not correspond
+template <typename>
+class a5;
+// since-cxx20-error@-1 {{redefinition of 'a5' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:46 {{previous definition is here}}
+template <typename>
+void a6(int);
+// since-cxx20-error@-1 {{redefinition of 'a6' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:47 {{previous definition is here}}
+template <typename, typename>
+int a7;
+// since-cxx20-error@-1 {{redefinition of 'a7' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:48 {{previous definition is here}}
+template <typename T>
+int a8<T, int>;
+// since-cxx20-error@-1 {{redefinition of 'a8' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:49 {{previous definition is here}}
+// since-cxx20-error@-3 {{expected ';' after top level declarator}}
+template <typename>
+using a9 = int;
+// since-cxx20-error@-1 {{redefinition of 'a9' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:50 {{previous definition is here}}
+template <typename>
+concept a10 = true;
+// since-cxx20-error@-1 {{redefinition of 'a10' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:51 {{previous definition is here}}
+// For variables, type has to match as well.
+long a11;
+// since-cxx20-error@-1 {{redefinition of 'a11' with a different type: 'long' vs 'int'}}
+//   since-cxx20-note@cwg1884_A.cppm:52 {{previous definition is here}}
+
+
+// Part B: matching against `void b();`
+// ------------------------------------
+
+int b1;
+// since-cxx20-error@-1 {{redefinition of 'b1' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:53 {{previous definition is here}}
+enum Eb {
+  b2
+  // since-cxx20-error@-1 {{redefinition of 'b2'}}
+  //   since-cxx20-note@cwg1884_A.cppm:54 {{previous definition is here}}
+};
+namespace b3 {} // #cwg1884-namespace-b
+// since-cxx20-error@-1 {{redefinition of 'b3' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:55 {{previous definition is here}}
+struct b4;
+// @-1 OK, types and functions do not correspond
+template <typename>
+class b5;
+// since-cxx20-error@-1 {{redefinition of 'b5' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:57 {{previous definition is here}}
+template <typename>
+void b6(int);
+// @-1 OK, a non-corresponding overload
+template <typename, typename>
+int b7;
+// since-cxx20-error@-1 {{redefinition of 'b7' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:59 {{previous definition is here}}
+template <typename T>
+int b8<T, int>;
+// since-cxx20-error@-1 {{no variable template matches partial specialization}}
+template <typename>
+using b9 = int;
+// since-cxx20-error@-1 {{redefinition of 'b9' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:61 {{previous definition is here}}
+template <typename>
+concept b10 = true;
+// since-cxx20-error@-1 {{redefinition of 'b10' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:62 {{previous definition is here}}
+// For functions, type has to match as well.
+// FIXME: we should be loud and clear here about type mismatch, like we do in `a11` case.
+int b11();
+// since-cxx20-error@-1 {{declaration of 'b11' in the global module follows declaration in module cwg1884_A}}
+//   since-cxx20-note@cwg1884_A.cppm:63 {{previous declaration is here}}
+
+
+// Part C: matching against `enum E { c };`
+// ----------------------------------------
+
+int c1;
+// since-cxx20-error@-1 {{redefinition of 'c1' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:65 {{previous definition is here}}
+void c2();
+// since-cxx20-error@-1 {{redefinition of 'c2' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:66 {{previous definition is here}}
+namespace c3 {}
+// since-cxx20-error@-1 {{redefinition of 'c3' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:67 {{previous definition is here}}
+struct c4;
+// @-1 OK, types and enumerators do not correspond
+template <typename>
+class c5;
+// since-cxx20-error@-1 {{redefinition of 'c5' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:69 {{previous definition is here}}
+template <typename>
+void c6(int);
+// since-cxx20-error@-1 {{redefinition of 'c6' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:70 {{previous definition is here}}
+template <typename, typename>
+int c7;
+// since-cxx20-error@-1 {{redefinition of 'c7' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:71 {{previous definition is here}}
+template <typename T>
+int c8<T, int>;
+// since-cxx20-error@-1 {{redefinition of 'c8' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:72 {{previous definition is here}}
+// since-cxx20-error@-3 {{expected ';' after top level declarator}}
+template <typename>
+using c9 = int;
+// since-cxx20-error@-1 {{redefinition of 'c9' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:73 {{previous definition is here}}
+template <typename>
+concept c10 = true;
+// since-cxx20-error@-1 {{redefinition of 'c10' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:74 {{previous definition is here}}
+
+
+// Part D: matching against `namespace d {};`
+// ------------------------------------------
+
+int d1;
+// since-cxx20-error@-1 {{redefinition of 'd1' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:76 {{previous definition is here}}
+void d2();
+// since-cxx20-error@-1 {{redefinition of 'd2' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:77 {{previous definition is here}}
+enum Ed {
+  d3
+  // since-cxx20-error@-1 {{redefinition of 'd3'}}
+  //   since-cxx20-note@cwg1884_A.cppm:78 {{previous definition is here}}
+};
+struct d4;
+// since-cxx20-error@-1 {{redefinition of 'd4' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:79 {{previous definition is here}}
+template <typename>
+class d5;
+// since-cxx20-error@-1 {{redefinition of 'd5' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:80 {{previous definition is here}}
+template <typename>
+void d6(int);
+// since-cxx20-error@-1 {{redefinition of 'd6' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:81 {{previous definition is here}}
+template <typename, typename>
+int d7;
+// since-cxx20-error@-1 {{redefinition of 'd7' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:82 {{previous definition is here}}
+template <typename T>
+int d8<T, int>;
+// since-cxx20-error@-1 {{redefinition of 'd8' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:83 {{previous definition is here}}
+// since-cxx20-error@-3 {{expected ';' after top level declarator}}
+template <typename>
+using d9 = int;
+// since-cxx20-error@-1 {{redefinition of 'd9' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:84 {{previous definition is here}}
+template <typename>
+concept d10 = true;
+// since-cxx20-error@-1 {{redefinition of 'd10' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:85 {{previous definition is here}}
+
+
+// Part E: matching against `struct e;`
+// ------------------------------------
+
+int e1;
+// @-1 OK, types and variables do not correspond
+void e2();
+// @-1 OK, types and functions do not correspond
+enum Ee {
+  e3
+  // @-1 OK, types and enumerators do not correspond
+};
+namespace e4 {}
+// since-cxx20-error@-1 {{redefinition of 'e4' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:89 {{previous definition is here}}
+template <typename>
+class e5;
+// since-cxx20-error@-1 {{redefinition of 'e5' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:90 {{previous definition is here}}
+template <typename>
+void e6(int);
+// @-1 OK, types and function templates do not correspond
+template <typename, typename>
+int e7;
+// since-cxx20-error@-1 {{redefinition of 'e7' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:92 {{previous definition is here}}
+template <typename T>
+int e8<T, int>;
+// since-cxx20-error@-1 {{redefinition of 'e8' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:93 {{previous definition is here}}
+// since-cxx20-error@-3 {{expected ';' after top level declarator}}
+template <typename>
+using e9 = int;
+// since-cxx20-error@-1 {{redefinition of 'e9' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:94 {{previous definition is here}}
+template <typename>
+concept e10 = true;
+// since-cxx20-error@-1 {{redefinition of 'e10' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:95 {{previous definition is here}}
+// FIXME: the following forward declaration is well-formed.
+//        Agreement on 'struct' vs 'class' is not required per [dcl.type.elab]/7.
+class e11;
+// since-cxx20-error@-1 {{declaration of 'e11' in the global module follows declaration in module cwg1884_A}}
+//   since-cxx20-note@cwg1884_A.cppm:96 {{previous declaration is here}}
+union e12;
+// since-cxx20-error@-1 {{use of 'e12' with tag type that does not match previous declaration}}
+//   since-cxx20-note@cwg1884_A.cppm:97 {{previous use is here}}
+// since-cxx20-error@-3 {{declaration of 'e12' in the global module follows declaration in module cwg1884_A}}
+//   since-cxx20-note@cwg1884_A.cppm:97 {{previous declaration is here}}
+enum e13 {};
+// since-cxx20-error@-1 {{use of 'e13' with tag type that does not match previous declaration}}
+//   since-cxx20-note@cwg1884_A.cppm:98 {{previous use is here}}
+
+
+// Part F: matching against `template <typename> class f;`
+// -------------------------------------------------------
+
+int f1;
+// since-cxx20-error@-1 {{redefinition of 'f1' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:100 {{previous definition is here}}
+void f2();
+// since-cxx20-error@-1 {{redefinition of 'f2' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:102 {{previous definition is here}}
+enum Ef {
+  f3
+  // since-cxx20-error@-1 {{redefinition of 'f3'}}
+  //   since-cxx20-note@cwg1884_A.cppm:104 {{previous definition is here}}
+};
+namespace f4 {}
+// since-cxx20-error@-1 {{redefinition of 'f4' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:106 {{previous definition is here}}
+struct f5;
+// since-cxx20-error@-1 {{redefinition of 'f5' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:108 {{previous definition is here}}
+template <typename>
+void f6(int);
+// since-cxx20-error@-1 {{redefinition of 'f6' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:110 {{previous definition is here}}
+template <typename, typename>
+int f7;
+// since-cxx20-error@-1 {{redefinition of 'f7' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:112 {{previous definition is here}}
+template <typename T>
+int f8<T, int>;
+// since-cxx20-error@-1 {{no variable template matches partial specialization}}
+template <typename>
+using f9 = int;
+// since-cxx20-error@-1 {{redefinition of 'f9' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:116 {{previous definition is here}}
+template <typename>
+concept f10 = true;
+// since-cxx20-error@-1 {{redefinition of 'f10' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:118 {{previous definition is here}}
+
+
+// Part G: matching against `template <typename> void g(int);`
+// -----------------------------------------------------------
+
+int g1;
+// since-cxx20-error@-1 {{redefinition of 'g1' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:122 {{previous definition is here}}
+void g2();
+// @-1 OK, a non-corresponding overload
+enum Eg {
+  g3
+  // MISSING-since-cxx20-error@-1 {{redefinition of 'g3'}}
+  //   MISSING-since-cxx20-note@cwg1884_A.cppm:126 {{previous definition is here}}
+};
+namespace g4 {}
+// MISSING-since-cxx20-error@-1 {{redefinition of 'g4' as different kind of symbol}}
+//   MISSING-since-cxx20-note@cwg1884_A.cppm:128 {{previous definition is here}}
+struct g5;
+// @-1 OK, types and function templates do not correspond
+template <typename>
+class g6;
+// since-cxx20-error@-1 {{redefinition of 'g6' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:132 {{previous definition is here}}
+template <typename, typename>
+int g7;
+// since-cxx20-error@-1 {{redefinition of 'g7' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:134 {{previous definition is here}}
+template <typename T>
+int g8<T, int>;
+// since-cxx20-error@-1 {{no variable template matches specialization; did you mean to use 'g8' as function template instead?}}
+template <typename>
+using g9 = int;
+// since-cxx20-error@-1 {{redefinition of 'g9' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:138 {{previous definition is here}}
+template <typename>
+concept g10 = true;
+// since-cxx20-error@-1 {{redefinition of 'g10' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:140 {{previous definition is here}}
+
+
+// Part H: matching against `template <typename, typename> int h;`
+// ---------------------------------------------------------------
+
+int h1;
+// since-cxx20-error@-1 {{redefinition of 'h1' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:142 {{previous definition is here}}
+void h2();
+// since-cxx20-error@-1 {{redefinition of 'h2' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:144 {{previous definition is here}}
+enum Eh {
+  h3
+  // since-cxx20-error@-1 {{redefinition of 'h3'}}
+  //   since-cxx20-note@cwg1884_A.cppm:146 {{previous definition is here}}
+};
+namespace h4 {}
+// since-cxx20-error@-1 {{redefinition of 'h4' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:148 {{previous definition is here}}
+struct h5;
+// since-cxx20-error@-1 {{redefinition of 'h5' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:150 {{previous definition is here}}
+template <typename>
+class h6;
+// since-cxx20-error@-1 {{redefinition of 'h6' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:152 {{previous definition is here}}
+template <typename>
+void h7(int);
+// since-cxx20-error@-1 {{redefinition of 'h7' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:154 {{previous definition is here}}
+template <typename T>
+int h8<T, int>;
+// @-1 OK, partial specialization
+template <typename>
+using h9 = int;
+// since-cxx20-error@-1 {{redefinition of 'h9' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:158 {{previous definition is here}}
+template <typename>
+concept h10 = true;
+// since-cxx20-error@-1 {{redefinition of 'h10' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:160 {{previous definition is here}}
+
+
+// Part I: matching against `template <typename> using i = int;`
+// -------------------------------------------------------------
+
+int i1;
+// since-cxx20-error@-1 {{redefinition of 'i1' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:162 {{previous definition is here}}
+void i2();
+// since-cxx20-error@-1 {{redefinition of 'i2' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:164 {{previous definition is here}}
+enum Ei {
+  i3
+  // since-cxx20-error@-1 {{redefinition of 'i3'}}
+  //   since-cxx20-note@cwg1884_A.cppm:166 {{previous definition is here}}
+};
+namespace i4 {}
+// since-cxx20-error@-1 {{redefinition of 'i4' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:168 {{previous definition is here}}
+struct i5;
+// since-cxx20-error@-1 {{redefinition of 'i5' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:170 {{previous definition is here}}
+template <typename>
+class i6;
+// since-cxx20-error@-1 {{redefinition of 'i6' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:172 {{previous definition is here}}
+template <typename>
+void i7(int);
+// since-cxx20-error@-1 {{redefinition of 'i7' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:174 {{previous definition is here}}
+template <typename, typename>
+int i8;
+// since-cxx20-error@-1 {{redefinition of 'i8' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:176 {{previous definition is here}}
+template <typename T>
+int i9<T, int>;
+// since-cxx20-error@-1 {{no variable template matches partial specialization}}
+template <typename>
+concept i10 = true;
+// since-cxx20-error@-1 {{redefinition of 'i10' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:180 {{previous definition is here}}
+
+
+// Part J: matching against `template <typename> concept j = true;`
+// ----------------------------------------------------------------
+
+int j1;
+// since-cxx20-error@-1 {{redefinition of 'j1' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:184 {{previous definition is here}}
+void j2();
+// since-cxx20-error@-1 {{redefinition of 'j2' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:186 {{previous definition is here}}
+enum Ej {
+  j3
+  // since-cxx20-error@-1 {{redefinition of 'j3'}}
+  //   since-cxx20-note@cwg1884_A.cppm:188 {{previous definition is here}}
+};
+namespace j4 {}
+// since-cxx20-error@-1 {{redefinition of 'j4' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:190 {{previous definition is here}}
+struct j5;
+// since-cxx20-error@-1 {{redefinition of 'j5' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:192 {{previous definition is here}}
+template <typename>
+class j6;
+// since-cxx20-error@-1 {{redefinition of 'j6' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:194 {{previous definition is here}}
+template <typename>
+void j7(int);
+// since-cxx20-error@-1 {{redefinition of 'j7' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:196 {{previous definition is here}}
+template <typename, typename>
+int j8;
+// since-cxx20-error@-1 {{redefinition of 'j8' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:198 {{previous definition is here}}
+template <typename T>
+int j9<T, int>;
+// since-cxx20-error@-1 {{no variable template matches partial specialization}}
+template <typename>
+using j10 = int;
+// since-cxx20-error@-1 {{redefinition of 'j10' as different kind of symbol}}
+//   since-cxx20-note@cwg1884_A.cppm:202 {{previous definition is here}}
diff --git a/clang/test/CXX/drs/cwg18xx.cpp b/clang/test/CXX/drs/cwg18xx.cpp
index b059492637bd..0fd2cd6b2d87 100644
--- a/clang/test/CXX/drs/cwg18xx.cpp
+++ b/clang/test/CXX/drs/cwg18xx.cpp
@@ -547,6 +547,8 @@ namespace cwg1881 { // cwg1881: 7
   static_assert(!__is_standard_layout(D), "");
 }
 
+// cwg1884 is in cwg1884.cpp
+
 namespace cwg1890 { // cwg1890: no drafting 2018-06-04
 // FIXME: current consensus for CWG2335 is that the examples are well-formed.
 namespace ex1 {
diff --git a/clang/test/CXX/drs/cwg279.cpp b/clang/test/CXX/drs/cwg279.cpp
new file mode 100644
index 000000000000..3c63486cc0dd
--- /dev/null
+++ b/clang/test/CXX/drs/cwg279.cpp
@@ -0,0 +1,53 @@
+// RUN: rm -rf %t
+// RUN: mkdir %t
+// RUN: split-file --leading-lines %s %t
+// RUN: %clang_cc1 -std=c++20 -pedantic-errors -fexceptions -fcxx-exceptions %t/cwg279_A.cppm -triple x86_64-unknown-unknown -emit-module-interface -o %t/cwg279_A.pcm
+// RUN: %clang_cc1 -std=c++20 -verify=since-cxx20 -pedantic-errors -fexceptions -fcxx-exceptions -triple x86_64-unknown-unknown %t/cwg279.cpp -fmodule-file=cwg279_A=%t/cwg279_A.pcm
+// RUN: %clang_cc1 -std=c++23 -pedantic-errors -fexceptions -fcxx-exceptions %t/cwg279_A.cppm -triple x86_64-unknown-unknown -emit-module-interface -o %t/cwg279_A.pcm
+// RUN: %clang_cc1 -std=c++23 -verify=since-cxx20 -pedantic-errors -fexceptions -fcxx-exceptions -triple x86_64-unknown-unknown %t/cwg279.cpp -fmodule-file=cwg279_A=%t/cwg279_A.pcm
+// RUN: %clang_cc1 -std=c++2c -pedantic-errors -fexceptions -fcxx-exceptions %t/cwg279_A.cppm -triple x86_64-unknown-unknown -emit-module-interface -o %t/cwg279_A.pcm
+// RUN: %clang_cc1 -std=c++2c -verify=since-cxx20 -pedantic-errors -fexceptions -fcxx-exceptions -triple x86_64-unknown-unknown %t/cwg279.cpp -fmodule-file=cwg279_A=%t/cwg279_A.pcm
+
+// cwg279: no
+
+//--- cwg279_A.cppm
+export module cwg279_A;
+
+export {
+struct S; // #cwg279-S
+extern S *q; // #cwg279-q
+
+struct S2 {}; // #cwg279-S2
+extern S2 *q2; // #cwg279-q2
+
+struct S3 {}; // #cwg279-S3
+extern S3 *q3; // #cwg279-q3
+} // export
+
+//--- cwg279.cpp
+import cwg279_A;
+
+// FIXME: We should use markers instead. They are less fragile,
+//        but -verify doesn't support them across modules yet.
+// FIXME: This is well-formed. Previous "definition" is actually just a declaration.
+typedef struct {} S;
+// since-cxx20-error@-1 {{typedef redefinition with different types ('struct S' vs 'S')}}
+//   since-cxx20-note@cwg279_A.cppm:17 {{previous definition is here}}
+extern S *q;
+// since-cxx20-error@-1 {{declaration of 'q' in the global module follows declaration in module cwg279_A}}
+//   since-cxx20-note@cwg279_A.cppm:18 {{previous declaration is here}}
+
+typedef struct {} S2;
+// since-cxx20-error@-1 {{typedef redefinition with different types ('struct S2' vs 'S2')}}
+//   since-cxx20-note@cwg279_A.cppm:20 {{previous definition is here}}
+extern S2 *q2;
+// since-cxx20-error@-1 {{declaration of 'q2' in the global module follows declaration in module cwg279_A}}
+//   since-cxx20-note@cwg279_A.cppm:21 {{previous declaration is here}}
+
+// FIXME: This is well-formed, because [basic.def.odr]/15 is satisfied.
+struct S3 {};
+// since-cxx20-error@-1 {{redefinition of 'S3'}}
+//   since-cxx20-note@cwg279_A.cppm:23 {{previous definition is here}}
+extern S3 *q3;
+// since-cxx20-error@-1 {{declaration of 'q3' in the global module follows declaration in module cwg279_A}}
+//   since-cxx20-note@cwg279_A.cppm:24 {{previous declaration is here}}
diff --git a/clang/test/CXX/drs/cwg2xx.cpp b/clang/test/CXX/drs/cwg2xx.cpp
index 926cb1959602..ec37b420880e 100644
--- a/clang/test/CXX/drs/cwg2xx.cpp
+++ b/clang/test/CXX/drs/cwg2xx.cpp
@@ -1032,6 +1032,8 @@ namespace cwg277 { // cwg277: 3.1
   static_assert(__enable_constant_folding(!intp()), "");
 }
 
+// cwg279 is in cwg279.cpp
+
 namespace cwg280 { // cwg280: 2.9
   typedef void f0();
   typedef void f1(int);
diff --git a/clang/test/CXX/drs/cwg3xx.cpp b/clang/test/CXX/drs/cwg3xx.cpp
index f20054c3701b..10c8d86ed16a 100644
--- a/clang/test/CXX/drs/cwg3xx.cpp
+++ b/clang/test/CXX/drs/cwg3xx.cpp
@@ -637,6 +637,8 @@ namespace cwg337 { // cwg337: yes
   struct B { virtual ~B() = 0; };
 }
 
+// cwg338: dup 1884
+
 namespace cwg339 { // cwg339: 2.8
   template <int I> struct A { static const int value = I; };
 
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 6640ed477a24..186f7cc0ace5 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -1721,7 +1721,7 @@ of class templates</td>
     <td><a href="https://cplusplus.github.io/CWG/issues/279.html">279</a></td>
     <td>CD6</td>
     <td>Correspondence of "names for linkage purposes"</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="none" align="center">No</td>
   </tr>
   <tr id="280">
     <td><a href="https://cplusplus.github.io/CWG/issues/280.html">280</a></td>
@@ -2075,7 +2075,7 @@ of class templates</td>
     <td><a href="https://cplusplus.github.io/CWG/issues/338.html">338</a></td>
     <td>CD6</td>
     <td>Enumerator name with linkage used as class name in other translation unit</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="partial" align="center">Duplicate of <a href="#1884">1884</a></td>
   </tr>
   <tr id="339">
     <td><a href="https://cplusplus.github.io/CWG/issues/339.html">339</a></td>
@@ -11131,7 +11131,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/1884.html">1884</a></td>
     <td>CD6</td>
     <td>Unclear requirements for same-named external-linkage entities</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="partial" align="center">Partial</td>
   </tr>
   <tr id="1885">
     <td><a href="https://cplusplus.github.io/CWG/issues/1885.html">1885</a></td>
-- 
GitLab


From 40363d506db8ab3382339dc4463372dcfcf931b7 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Mon, 28 Oct 2024 14:50:40 +0000
Subject: [PATCH 323/693] [RISCV] Add cost model tests for fp rounding ops for
 bf16. NFC

---
 llvm/test/Analysis/CostModel/RISCV/fround.ll | 890 +++++++++++++------
 1 file changed, 598 insertions(+), 292 deletions(-)

diff --git a/llvm/test/Analysis/CostModel/RISCV/fround.ll b/llvm/test/Analysis/CostModel/RISCV/fround.ll
index b4740f223eca..c6826760a45b 100644
--- a/llvm/test/Analysis/CostModel/RISCV/fround.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/fround.ll
@@ -1,30 +1,50 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+f,+d,+zvfh | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+f,+d,+zvfhmin | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+f,+d,+zvfh,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+f,+d,+zvfhmin,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 define void @floor() {
 ; CHECK-LABEL: 'floor'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = call float @llvm.floor.f32(float undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %2 = call <2 x float> @llvm.floor.v2f32(<2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %3 = call <4 x float> @llvm.floor.v4f32(<4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %4 = call <8 x float> @llvm.floor.v8f32(<8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %5 = call <16 x float> @llvm.floor.v16f32(<16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %6 = call <vscale x 1 x float> @llvm.floor.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %7 = call <vscale x 2 x float> @llvm.floor.nxv2f32(<vscale x 2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %8 = call <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %9 = call <vscale x 8 x float> @llvm.floor.nxv8f32(<vscale x 8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %10 = call <vscale x 16 x float> @llvm.floor.nxv16f32(<vscale x 16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %11 = call double @llvm.floor.f64(double undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %12 = call <2 x double> @llvm.floor.v2f64(<2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %13 = call <4 x double> @llvm.floor.v4f64(<4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %14 = call <8 x double> @llvm.floor.v8f64(<8 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %15 = call <16 x double> @llvm.floor.v16f64(<16 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %16 = call <vscale x 1 x double> @llvm.floor.nxv1f64(<vscale x 1 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %17 = call <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %18 = call <vscale x 4 x double> @llvm.floor.nxv4f64(<vscale x 4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %19 = call <vscale x 8 x double> @llvm.floor.nxv8f64(<vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = call bfloat @llvm.floor.bf16(bfloat undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x bfloat> @llvm.floor.v2bf16(<2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x bfloat> @llvm.floor.v4bf16(<4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x bfloat> @llvm.floor.v16bf16(<16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x bfloat> @llvm.floor.nxv1bf16(<vscale x 1 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x bfloat> @llvm.floor.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x bfloat> @llvm.floor.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x bfloat> @llvm.floor.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call <vscale x 16 x bfloat> @llvm.floor.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %11 = call float @llvm.floor.f32(float undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %12 = call <2 x float> @llvm.floor.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %13 = call <4 x float> @llvm.floor.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %14 = call <8 x float> @llvm.floor.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %15 = call <16 x float> @llvm.floor.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %16 = call <vscale x 1 x float> @llvm.floor.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %17 = call <vscale x 2 x float> @llvm.floor.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %18 = call <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %19 = call <vscale x 8 x float> @llvm.floor.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %20 = call <vscale x 16 x float> @llvm.floor.nxv16f32(<vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %21 = call double @llvm.floor.f64(double undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %22 = call <2 x double> @llvm.floor.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %23 = call <4 x double> @llvm.floor.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %24 = call <8 x double> @llvm.floor.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %25 = call <16 x double> @llvm.floor.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %26 = call <vscale x 1 x double> @llvm.floor.nxv1f64(<vscale x 1 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %27 = call <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %28 = call <vscale x 4 x double> @llvm.floor.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %29 = call <vscale x 8 x double> @llvm.floor.nxv8f64(<vscale x 8 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call bfloat @llvm.floor.bf16(bfloat undef)
+  call <2 x bfloat> @llvm.floor.v2bf16(<2 x bfloat> undef)
+  call <4 x bfloat> @llvm.floor.v4bf16(<4 x bfloat> undef)
+  call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> undef)
+  call <16 x bfloat> @llvm.floor.v16bf16(<16 x bfloat> undef)
+  call <vscale x 1 x bfloat> @llvm.floor.nvx1bf16(<vscale x 1 x bfloat> undef)
+  call <vscale x 2 x bfloat> @llvm.floor.nvx2bf16(<vscale x 2 x bfloat> undef)
+  call <vscale x 4 x bfloat> @llvm.floor.nvx4bf16(<vscale x 4 x bfloat> undef)
+  call <vscale x 8 x bfloat> @llvm.floor.nvx8bf16(<vscale x 8 x bfloat> undef)
+  call <vscale x 16 x bfloat> @llvm.floor.nvx16bf16(<vscale x 16 x bfloat> undef)
   call float @llvm.floor.f32(float undef)
   call <2 x float> @llvm.floor.v2f32(<2 x float> undef)
   call <4 x float> @llvm.floor.v4f32(<4 x float> undef)
@@ -89,27 +109,47 @@ define void @floor_fp16() {
 
 define void @ceil() {
 ; CHECK-LABEL: 'ceil'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = call float @llvm.ceil.f32(float undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %2 = call <2 x float> @llvm.ceil.v2f32(<2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %3 = call <4 x float> @llvm.ceil.v4f32(<4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %4 = call <8 x float> @llvm.ceil.v8f32(<8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %5 = call <16 x float> @llvm.ceil.v16f32(<16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %6 = call <vscale x 1 x float> @llvm.ceil.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %7 = call <vscale x 2 x float> @llvm.ceil.nxv2f32(<vscale x 2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %8 = call <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %9 = call <vscale x 8 x float> @llvm.ceil.nxv8f32(<vscale x 8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %10 = call <vscale x 16 x float> @llvm.ceil.nxv16f32(<vscale x 16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %11 = call double @llvm.ceil.f64(double undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %12 = call <2 x double> @llvm.ceil.v2f64(<2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %13 = call <4 x double> @llvm.ceil.v4f64(<4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %14 = call <8 x double> @llvm.ceil.v8f64(<8 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %15 = call <16 x double> @llvm.ceil.v16f64(<16 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %16 = call <vscale x 1 x double> @llvm.ceil.nxv1f64(<vscale x 1 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %17 = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %18 = call <vscale x 4 x double> @llvm.ceil.nxv4f64(<vscale x 4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %19 = call <vscale x 8 x double> @llvm.ceil.nxv8f64(<vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = call bfloat @llvm.ceil.bf16(bfloat undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x bfloat> @llvm.ceil.v2bf16(<2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x bfloat> @llvm.ceil.v4bf16(<4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x bfloat> @llvm.ceil.v8bf16(<8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x bfloat> @llvm.ceil.v16bf16(<16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x bfloat> @llvm.ceil.nxv1bf16(<vscale x 1 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x bfloat> @llvm.ceil.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x bfloat> @llvm.ceil.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x bfloat> @llvm.ceil.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call <vscale x 16 x bfloat> @llvm.ceil.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %11 = call float @llvm.ceil.f32(float undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %12 = call <2 x float> @llvm.ceil.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %13 = call <4 x float> @llvm.ceil.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %14 = call <8 x float> @llvm.ceil.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %15 = call <16 x float> @llvm.ceil.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %16 = call <vscale x 1 x float> @llvm.ceil.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %17 = call <vscale x 2 x float> @llvm.ceil.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %18 = call <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %19 = call <vscale x 8 x float> @llvm.ceil.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %20 = call <vscale x 16 x float> @llvm.ceil.nxv16f32(<vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %21 = call double @llvm.ceil.f64(double undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %22 = call <2 x double> @llvm.ceil.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %23 = call <4 x double> @llvm.ceil.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %24 = call <8 x double> @llvm.ceil.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %25 = call <16 x double> @llvm.ceil.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %26 = call <vscale x 1 x double> @llvm.ceil.nxv1f64(<vscale x 1 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %27 = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %28 = call <vscale x 4 x double> @llvm.ceil.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %29 = call <vscale x 8 x double> @llvm.ceil.nxv8f64(<vscale x 8 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call bfloat @llvm.ceil.bf16(bfloat undef)
+  call <2 x bfloat> @llvm.ceil.v2bf16(<2 x bfloat> undef)
+  call <4 x bfloat> @llvm.ceil.v4bf16(<4 x bfloat> undef)
+  call <8 x bfloat> @llvm.ceil.v8bf16(<8 x bfloat> undef)
+  call <16 x bfloat> @llvm.ceil.v16bf16(<16 x bfloat> undef)
+  call <vscale x 1 x bfloat> @llvm.ceil.nvx1bf16(<vscale x 1 x bfloat> undef)
+  call <vscale x 2 x bfloat> @llvm.ceil.nvx2bf16(<vscale x 2 x bfloat> undef)
+  call <vscale x 4 x bfloat> @llvm.ceil.nvx4bf16(<vscale x 4 x bfloat> undef)
+  call <vscale x 8 x bfloat> @llvm.ceil.nvx8bf16(<vscale x 8 x bfloat> undef)
+  call <vscale x 16 x bfloat> @llvm.ceil.nvx16bf16(<vscale x 16 x bfloat> undef)
   call float @llvm.ceil.f32(float undef)
   call <2 x float> @llvm.ceil.v2f32(<2 x float> undef)
   call <4 x float> @llvm.ceil.v4f32(<4 x float> undef)
@@ -174,27 +214,47 @@ define void @ceil_fp16() {
 
 define void @trunc() {
 ; CHECK-LABEL: 'trunc'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = call float @llvm.trunc.f32(float undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %2 = call <2 x float> @llvm.trunc.v2f32(<2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %3 = call <4 x float> @llvm.trunc.v4f32(<4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %4 = call <8 x float> @llvm.trunc.v8f32(<8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %5 = call <16 x float> @llvm.trunc.v16f32(<16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %6 = call <vscale x 1 x float> @llvm.trunc.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %7 = call <vscale x 2 x float> @llvm.trunc.nxv2f32(<vscale x 2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %8 = call <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %9 = call <vscale x 8 x float> @llvm.trunc.nxv8f32(<vscale x 8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <vscale x 16 x float> @llvm.trunc.nxv16f32(<vscale x 16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %11 = call double @llvm.trunc.f64(double undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <2 x double> @llvm.trunc.v2f64(<2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <4 x double> @llvm.trunc.v4f64(<4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <8 x double> @llvm.trunc.v8f64(<8 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <16 x double> @llvm.trunc.v16f64(<16 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 1 x double> @llvm.trunc.nxv1f64(<vscale x 1 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %18 = call <vscale x 4 x double> @llvm.trunc.nxv4f64(<vscale x 4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %19 = call <vscale x 8 x double> @llvm.trunc.nxv8f64(<vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = call bfloat @llvm.trunc.bf16(bfloat undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x bfloat> @llvm.trunc.v2bf16(<2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x bfloat> @llvm.trunc.v4bf16(<4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x bfloat> @llvm.trunc.v8bf16(<8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x bfloat> @llvm.trunc.v16bf16(<16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x bfloat> @llvm.trunc.nxv1bf16(<vscale x 1 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x bfloat> @llvm.trunc.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x bfloat> @llvm.trunc.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x bfloat> @llvm.trunc.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call <vscale x 16 x bfloat> @llvm.trunc.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %11 = call float @llvm.trunc.f32(float undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <2 x float> @llvm.trunc.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <4 x float> @llvm.trunc.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <8 x float> @llvm.trunc.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <16 x float> @llvm.trunc.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 1 x float> @llvm.trunc.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 2 x float> @llvm.trunc.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %18 = call <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %19 = call <vscale x 8 x float> @llvm.trunc.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %20 = call <vscale x 16 x float> @llvm.trunc.nxv16f32(<vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %21 = call double @llvm.trunc.f64(double undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %22 = call <2 x double> @llvm.trunc.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %23 = call <4 x double> @llvm.trunc.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %24 = call <8 x double> @llvm.trunc.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %25 = call <16 x double> @llvm.trunc.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %26 = call <vscale x 1 x double> @llvm.trunc.nxv1f64(<vscale x 1 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %27 = call <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %28 = call <vscale x 4 x double> @llvm.trunc.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %29 = call <vscale x 8 x double> @llvm.trunc.nxv8f64(<vscale x 8 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call bfloat @llvm.trunc.bf16(bfloat undef)
+  call <2 x bfloat> @llvm.trunc.v2bf16(<2 x bfloat> undef)
+  call <4 x bfloat> @llvm.trunc.v4bf16(<4 x bfloat> undef)
+  call <8 x bfloat> @llvm.trunc.v8bf16(<8 x bfloat> undef)
+  call <16 x bfloat> @llvm.trunc.v16bf16(<16 x bfloat> undef)
+  call <vscale x 1 x bfloat> @llvm.trunc.nvx1bf16(<vscale x 1 x bfloat> undef)
+  call <vscale x 2 x bfloat> @llvm.trunc.nvx2bf16(<vscale x 2 x bfloat> undef)
+  call <vscale x 4 x bfloat> @llvm.trunc.nvx4bf16(<vscale x 4 x bfloat> undef)
+  call <vscale x 8 x bfloat> @llvm.trunc.nvx8bf16(<vscale x 8 x bfloat> undef)
+  call <vscale x 16 x bfloat> @llvm.trunc.nvx16bf16(<vscale x 16 x bfloat> undef)
   call float @llvm.trunc.f32(float undef)
   call <2 x float> @llvm.trunc.v2f32(<2 x float> undef)
   call <4 x float> @llvm.trunc.v4f32(<4 x float> undef)
@@ -259,27 +319,47 @@ define void @trunc_fp16() {
 
 define void @rint() {
 ; CHECK-LABEL: 'rint'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = call float @llvm.rint.f32(float undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %2 = call <2 x float> @llvm.rint.v2f32(<2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %3 = call <4 x float> @llvm.rint.v4f32(<4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %4 = call <8 x float> @llvm.rint.v8f32(<8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %5 = call <16 x float> @llvm.rint.v16f32(<16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %6 = call <vscale x 1 x float> @llvm.rint.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %7 = call <vscale x 2 x float> @llvm.rint.nxv2f32(<vscale x 2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %8 = call <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %9 = call <vscale x 8 x float> @llvm.rint.nxv8f32(<vscale x 8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <vscale x 16 x float> @llvm.rint.nxv16f32(<vscale x 16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %11 = call double @llvm.rint.f64(double undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <2 x double> @llvm.rint.v2f64(<2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <4 x double> @llvm.rint.v4f64(<4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <8 x double> @llvm.rint.v8f64(<8 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <16 x double> @llvm.rint.v16f64(<16 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 1 x double> @llvm.rint.nxv1f64(<vscale x 1 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %18 = call <vscale x 4 x double> @llvm.rint.nxv4f64(<vscale x 4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %19 = call <vscale x 8 x double> @llvm.rint.nxv8f64(<vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = call bfloat @llvm.rint.bf16(bfloat undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x bfloat> @llvm.rint.v2bf16(<2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x bfloat> @llvm.rint.v4bf16(<4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x bfloat> @llvm.rint.v8bf16(<8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x bfloat> @llvm.rint.v16bf16(<16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x bfloat> @llvm.rint.nxv1bf16(<vscale x 1 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x bfloat> @llvm.rint.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x bfloat> @llvm.rint.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x bfloat> @llvm.rint.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call <vscale x 16 x bfloat> @llvm.rint.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %11 = call float @llvm.rint.f32(float undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <2 x float> @llvm.rint.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <4 x float> @llvm.rint.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <8 x float> @llvm.rint.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <16 x float> @llvm.rint.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 1 x float> @llvm.rint.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 2 x float> @llvm.rint.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %18 = call <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %19 = call <vscale x 8 x float> @llvm.rint.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %20 = call <vscale x 16 x float> @llvm.rint.nxv16f32(<vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %21 = call double @llvm.rint.f64(double undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %22 = call <2 x double> @llvm.rint.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %23 = call <4 x double> @llvm.rint.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %24 = call <8 x double> @llvm.rint.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %25 = call <16 x double> @llvm.rint.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %26 = call <vscale x 1 x double> @llvm.rint.nxv1f64(<vscale x 1 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %27 = call <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %28 = call <vscale x 4 x double> @llvm.rint.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %29 = call <vscale x 8 x double> @llvm.rint.nxv8f64(<vscale x 8 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call bfloat @llvm.rint.bf16(bfloat undef)
+  call <2 x bfloat> @llvm.rint.v2bf16(<2 x bfloat> undef)
+  call <4 x bfloat> @llvm.rint.v4bf16(<4 x bfloat> undef)
+  call <8 x bfloat> @llvm.rint.v8bf16(<8 x bfloat> undef)
+  call <16 x bfloat> @llvm.rint.v16bf16(<16 x bfloat> undef)
+  call <vscale x 1 x bfloat> @llvm.rint.nvx1bf16(<vscale x 1 x bfloat> undef)
+  call <vscale x 2 x bfloat> @llvm.rint.nvx2bf16(<vscale x 2 x bfloat> undef)
+  call <vscale x 4 x bfloat> @llvm.rint.nvx4bf16(<vscale x 4 x bfloat> undef)
+  call <vscale x 8 x bfloat> @llvm.rint.nvx8bf16(<vscale x 8 x bfloat> undef)
+  call <vscale x 16 x bfloat> @llvm.rint.nvx16bf16(<vscale x 16 x bfloat> undef)
   call float @llvm.rint.f32(float undef)
   call <2 x float> @llvm.rint.v2f32(<2 x float> undef)
   call <4 x float> @llvm.rint.v4f32(<4 x float> undef)
@@ -344,27 +424,47 @@ define void @rint_fp16() {
 
 define void @lrint() {
 ; CHECK-LABEL: 'lrint'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call i64 @llvm.lrint.i64.f32(float undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x i64> @llvm.lrint.v8i64.v8f32(<8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x i64> @llvm.lrint.v16i64.v16f32(<16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x i64> @llvm.lrint.nxv1i64.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2f32(<vscale x 2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4f32(<vscale x 4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8f32(<vscale x 8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16f32(<vscale x 16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call i64 @llvm.lrint.i64.f64(double undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <16 x i64> @llvm.lrint.v16i64.v16f64(<16 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 1 x i64> @llvm.lrint.nxv1i64.nxv1f64(<vscale x 1 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2f64(<vscale x 2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4f64(<vscale x 4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8f64(<vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call i64 @llvm.lrint.i64.bf16(bfloat undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x i64> @llvm.lrint.v2i64.v2bf16(<2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x i64> @llvm.lrint.v4i64.v4bf16(<4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x i64> @llvm.lrint.v8i64.v8bf16(<8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x i64> @llvm.lrint.v16i64.v16bf16(<16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x i64> @llvm.lrint.nxv1i64.nxv1bf16(<vscale x 1 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call i64 @llvm.lrint.i64.f32(float undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x i64> @llvm.lrint.v8i64.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <16 x i64> @llvm.lrint.v16i64.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 1 x i64> @llvm.lrint.nxv1i64.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16f32(<vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call i64 @llvm.lrint.i64.f64(double undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = call <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = call <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = call <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %25 = call <16 x i64> @llvm.lrint.v16i64.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = call <vscale x 1 x i64> @llvm.lrint.nxv1i64.nxv1f64(<vscale x 1 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8f64(<vscale x 8 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call i64 @llvm.lrint.i64.bf16(bfloat undef)
+  call <2 x i64> @llvm.lrint.v2i64.v2bf16(<2 x bfloat> undef)
+  call <4 x i64> @llvm.lrint.v4i64.v4bf16(<4 x bfloat> undef)
+  call <8 x i64> @llvm.lrint.v8i64.v8bf16(<8 x bfloat> undef)
+  call <16 x i64> @llvm.lrint.v16i64.v16bf16(<16 x bfloat> undef)
+  call <vscale x 1 x i64> @llvm.lrint.nvx1i64.nvx1bf16(<vscale x 1 x bfloat> undef)
+  call <vscale x 2 x i64> @llvm.lrint.nvx2i64.nvx2bf16(<vscale x 2 x bfloat> undef)
+  call <vscale x 4 x i64> @llvm.lrint.nvx4i64.nvx4bf16(<vscale x 4 x bfloat> undef)
+  call <vscale x 8 x i64> @llvm.lrint.nvx8i64.nvx8bf16(<vscale x 8 x bfloat> undef)
+  call <vscale x 16 x i64> @llvm.lrint.nvx16i64.nvx16bf16(<vscale x 16 x bfloat> undef)
   call i64 @llvm.lrint.i64.f32(float undef)
   call <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float> undef)
   call <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float> undef)
@@ -416,27 +516,47 @@ define void @lrint_fp16() {
 
 define void @llrint() {
 ; CHECK-LABEL: 'llrint'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call i64 @llvm.llrint.i64.f32(float undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f32(<vscale x 2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f32(<vscale x 4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f32(<vscale x 8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f32(<vscale x 16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call i64 @llvm.llrint.i64.f64(double undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <16 x i64> @llvm.llrint.v16i64.v16f64(<16 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f64(<vscale x 1 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f64(<vscale x 2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f64(<vscale x 4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f64(<vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call i64 @llvm.llrint.i64.bf16(bfloat undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x i64> @llvm.llrint.v2i64.v2bf16(<2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x i64> @llvm.llrint.v4i64.v4bf16(<4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x i64> @llvm.llrint.v8i64.v8bf16(<8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x i64> @llvm.llrint.v16i64.v16bf16(<16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1bf16(<vscale x 1 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call i64 @llvm.llrint.i64.f32(float undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f32(<vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call i64 @llvm.llrint.i64.f64(double undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = call <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = call <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = call <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %25 = call <16 x i64> @llvm.llrint.v16i64.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f64(<vscale x 1 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f64(<vscale x 8 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call i64 @llvm.llrint.i64.bf16(bfloat undef)
+  call <2 x i64> @llvm.llrint.v2i64.v2bf16(<2 x bfloat> undef)
+  call <4 x i64> @llvm.llrint.v4i64.v4bf16(<4 x bfloat> undef)
+  call <8 x i64> @llvm.llrint.v8i64.v8bf16(<8 x bfloat> undef)
+  call <16 x i64> @llvm.llrint.v16i64.v16bf16(<16 x bfloat> undef)
+  call <vscale x 1 x i64> @llvm.llrint.nvx1i64.nvx1bf16(<vscale x 1 x bfloat> undef)
+  call <vscale x 2 x i64> @llvm.llrint.nvx2i64.nvx2bf16(<vscale x 2 x bfloat> undef)
+  call <vscale x 4 x i64> @llvm.llrint.nvx4i64.nvx4bf16(<vscale x 4 x bfloat> undef)
+  call <vscale x 8 x i64> @llvm.llrint.nvx8i64.nvx8bf16(<vscale x 8 x bfloat> undef)
+  call <vscale x 16 x i64> @llvm.llrint.nvx16i64.nvx16bf16(<vscale x 16 x bfloat> undef)
   call i64 @llvm.llrint.i64.f32(float undef)
   call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> undef)
   call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> undef)
@@ -488,27 +608,47 @@ define void @llrint_fp16() {
 
 define void @nearbyint() {
 ; CHECK-LABEL: 'nearbyint'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %1 = call float @llvm.nearbyint.f32(float undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %2 = call <2 x float> @llvm.nearbyint.v2f32(<2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %3 = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %4 = call <8 x float> @llvm.nearbyint.v8f32(<8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %5 = call <16 x float> @llvm.nearbyint.v16f32(<16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %6 = call <vscale x 1 x float> @llvm.nearbyint.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %7 = call <vscale x 2 x float> @llvm.nearbyint.nxv2f32(<vscale x 2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %8 = call <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %9 = call <vscale x 8 x float> @llvm.nearbyint.nxv8f32(<vscale x 8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %10 = call <vscale x 16 x float> @llvm.nearbyint.nxv16f32(<vscale x 16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %11 = call double @llvm.nearbyint.f64(double undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %12 = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %13 = call <4 x double> @llvm.nearbyint.v4f64(<4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %14 = call <8 x double> @llvm.nearbyint.v8f64(<8 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %15 = call <16 x double> @llvm.nearbyint.v16f64(<16 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %16 = call <vscale x 1 x double> @llvm.nearbyint.nxv1f64(<vscale x 1 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %17 = call <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %18 = call <vscale x 4 x double> @llvm.nearbyint.nxv4f64(<vscale x 4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %19 = call <vscale x 8 x double> @llvm.nearbyint.nxv8f64(<vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %1 = call bfloat @llvm.nearbyint.bf16(bfloat undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x bfloat> @llvm.nearbyint.v2bf16(<2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x bfloat> @llvm.nearbyint.v4bf16(<4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x bfloat> @llvm.nearbyint.v8bf16(<8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x bfloat> @llvm.nearbyint.v16bf16(<16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x bfloat> @llvm.nearbyint.nxv1bf16(<vscale x 1 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x bfloat> @llvm.nearbyint.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x bfloat> @llvm.nearbyint.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x bfloat> @llvm.nearbyint.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call <vscale x 16 x bfloat> @llvm.nearbyint.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %11 = call float @llvm.nearbyint.f32(float undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %12 = call <2 x float> @llvm.nearbyint.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %13 = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %14 = call <8 x float> @llvm.nearbyint.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %15 = call <16 x float> @llvm.nearbyint.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %16 = call <vscale x 1 x float> @llvm.nearbyint.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %17 = call <vscale x 2 x float> @llvm.nearbyint.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %18 = call <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %19 = call <vscale x 8 x float> @llvm.nearbyint.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %20 = call <vscale x 16 x float> @llvm.nearbyint.nxv16f32(<vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %21 = call double @llvm.nearbyint.f64(double undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %22 = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %23 = call <4 x double> @llvm.nearbyint.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %24 = call <8 x double> @llvm.nearbyint.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %25 = call <16 x double> @llvm.nearbyint.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %26 = call <vscale x 1 x double> @llvm.nearbyint.nxv1f64(<vscale x 1 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %27 = call <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %28 = call <vscale x 4 x double> @llvm.nearbyint.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %29 = call <vscale x 8 x double> @llvm.nearbyint.nxv8f64(<vscale x 8 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call bfloat @llvm.nearbyint.bf16(bfloat undef)
+  call <2 x bfloat> @llvm.nearbyint.v2bf16(<2 x bfloat> undef)
+  call <4 x bfloat> @llvm.nearbyint.v4bf16(<4 x bfloat> undef)
+  call <8 x bfloat> @llvm.nearbyint.v8bf16(<8 x bfloat> undef)
+  call <16 x bfloat> @llvm.nearbyint.v16bf16(<16 x bfloat> undef)
+  call <vscale x 1 x bfloat> @llvm.nearbyint.nvx1bf16(<vscale x 1 x bfloat> undef)
+  call <vscale x 2 x bfloat> @llvm.nearbyint.nvx2bf16(<vscale x 2 x bfloat> undef)
+  call <vscale x 4 x bfloat> @llvm.nearbyint.nvx4bf16(<vscale x 4 x bfloat> undef)
+  call <vscale x 8 x bfloat> @llvm.nearbyint.nvx8bf16(<vscale x 8 x bfloat> undef)
+  call <vscale x 16 x bfloat> @llvm.nearbyint.nvx16bf16(<vscale x 16 x bfloat> undef)
   call float @llvm.nearbyint.f32(float undef)
   call <2 x float> @llvm.nearbyint.v2f32(<2 x float> undef)
   call <4 x float> @llvm.nearbyint.v4f32(<4 x float> undef)
@@ -573,27 +713,47 @@ define void @nearbyint_fp16() {
 
 define void @round() {
 ; CHECK-LABEL: 'round'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = call float @llvm.round.f32(float undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %2 = call <2 x float> @llvm.round.v2f32(<2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %3 = call <4 x float> @llvm.round.v4f32(<4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %4 = call <8 x float> @llvm.round.v8f32(<8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %5 = call <16 x float> @llvm.round.v16f32(<16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %6 = call <vscale x 1 x float> @llvm.round.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %7 = call <vscale x 2 x float> @llvm.round.nxv2f32(<vscale x 2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %8 = call <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %9 = call <vscale x 8 x float> @llvm.round.nxv8f32(<vscale x 8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %10 = call <vscale x 16 x float> @llvm.round.nxv16f32(<vscale x 16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %11 = call double @llvm.round.f64(double undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %12 = call <2 x double> @llvm.round.v2f64(<2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %13 = call <4 x double> @llvm.round.v4f64(<4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %14 = call <8 x double> @llvm.round.v8f64(<8 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %15 = call <16 x double> @llvm.round.v16f64(<16 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %16 = call <vscale x 1 x double> @llvm.round.nxv1f64(<vscale x 1 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %17 = call <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %18 = call <vscale x 4 x double> @llvm.round.nxv4f64(<vscale x 4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %19 = call <vscale x 8 x double> @llvm.round.nxv8f64(<vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = call bfloat @llvm.round.bf16(bfloat undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x bfloat> @llvm.round.v2bf16(<2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x bfloat> @llvm.round.v4bf16(<4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x bfloat> @llvm.round.v8bf16(<8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x bfloat> @llvm.round.v16bf16(<16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x bfloat> @llvm.round.nxv1bf16(<vscale x 1 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x bfloat> @llvm.round.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x bfloat> @llvm.round.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x bfloat> @llvm.round.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call <vscale x 16 x bfloat> @llvm.round.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %11 = call float @llvm.round.f32(float undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %12 = call <2 x float> @llvm.round.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %13 = call <4 x float> @llvm.round.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %14 = call <8 x float> @llvm.round.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %15 = call <16 x float> @llvm.round.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %16 = call <vscale x 1 x float> @llvm.round.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %17 = call <vscale x 2 x float> @llvm.round.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %18 = call <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %19 = call <vscale x 8 x float> @llvm.round.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %20 = call <vscale x 16 x float> @llvm.round.nxv16f32(<vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %21 = call double @llvm.round.f64(double undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %22 = call <2 x double> @llvm.round.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %23 = call <4 x double> @llvm.round.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %24 = call <8 x double> @llvm.round.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %25 = call <16 x double> @llvm.round.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %26 = call <vscale x 1 x double> @llvm.round.nxv1f64(<vscale x 1 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %27 = call <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %28 = call <vscale x 4 x double> @llvm.round.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %29 = call <vscale x 8 x double> @llvm.round.nxv8f64(<vscale x 8 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call bfloat @llvm.round.bf16(bfloat undef)
+  call <2 x bfloat> @llvm.round.v2bf16(<2 x bfloat> undef)
+  call <4 x bfloat> @llvm.round.v4bf16(<4 x bfloat> undef)
+  call <8 x bfloat> @llvm.round.v8bf16(<8 x bfloat> undef)
+  call <16 x bfloat> @llvm.round.v16bf16(<16 x bfloat> undef)
+  call <vscale x 1 x bfloat> @llvm.round.nvx1bf16(<vscale x 1 x bfloat> undef)
+  call <vscale x 2 x bfloat> @llvm.round.nvx2bf16(<vscale x 2 x bfloat> undef)
+  call <vscale x 4 x bfloat> @llvm.round.nvx4bf16(<vscale x 4 x bfloat> undef)
+  call <vscale x 8 x bfloat> @llvm.round.nvx8bf16(<vscale x 8 x bfloat> undef)
+  call <vscale x 16 x bfloat> @llvm.round.nvx16bf16(<vscale x 16 x bfloat> undef)
   call float @llvm.round.f32(float undef)
   call <2 x float> @llvm.round.v2f32(<2 x float> undef)
   call <4 x float> @llvm.round.v4f32(<4 x float> undef)
@@ -658,27 +818,47 @@ define void @round_fp16() {
 
 define void @roundeven() {
 ; CHECK-LABEL: 'roundeven'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = call float @llvm.roundeven.f32(float undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %2 = call <2 x float> @llvm.roundeven.v2f32(<2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %3 = call <4 x float> @llvm.roundeven.v4f32(<4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %4 = call <8 x float> @llvm.roundeven.v8f32(<8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %5 = call <16 x float> @llvm.roundeven.v16f32(<16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %6 = call <vscale x 1 x float> @llvm.roundeven.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %7 = call <vscale x 2 x float> @llvm.roundeven.nxv2f32(<vscale x 2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %8 = call <vscale x 4 x float> @llvm.roundeven.nxv4f32(<vscale x 4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %9 = call <vscale x 8 x float> @llvm.roundeven.nxv8f32(<vscale x 8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %10 = call <vscale x 16 x float> @llvm.roundeven.nxv16f32(<vscale x 16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %11 = call double @llvm.roundeven.f64(double undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %12 = call <2 x double> @llvm.roundeven.v2f64(<2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %13 = call <4 x double> @llvm.roundeven.v4f64(<4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %14 = call <8 x double> @llvm.roundeven.v8f64(<8 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %15 = call <16 x double> @llvm.roundeven.v16f64(<16 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %16 = call <vscale x 1 x double> @llvm.roundeven.nxv1f64(<vscale x 1 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %17 = call <vscale x 2 x double> @llvm.roundeven.nxv2f64(<vscale x 2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %18 = call <vscale x 4 x double> @llvm.roundeven.nxv4f64(<vscale x 4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %19 = call <vscale x 8 x double> @llvm.roundeven.nxv8f64(<vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = call bfloat @llvm.roundeven.bf16(bfloat undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x bfloat> @llvm.roundeven.v2bf16(<2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x bfloat> @llvm.roundeven.v4bf16(<4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x bfloat> @llvm.roundeven.v8bf16(<8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x bfloat> @llvm.roundeven.v16bf16(<16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x bfloat> @llvm.roundeven.nxv1bf16(<vscale x 1 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x bfloat> @llvm.roundeven.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x bfloat> @llvm.roundeven.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x bfloat> @llvm.roundeven.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call <vscale x 16 x bfloat> @llvm.roundeven.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %11 = call float @llvm.roundeven.f32(float undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %12 = call <2 x float> @llvm.roundeven.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %13 = call <4 x float> @llvm.roundeven.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %14 = call <8 x float> @llvm.roundeven.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %15 = call <16 x float> @llvm.roundeven.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %16 = call <vscale x 1 x float> @llvm.roundeven.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %17 = call <vscale x 2 x float> @llvm.roundeven.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %18 = call <vscale x 4 x float> @llvm.roundeven.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %19 = call <vscale x 8 x float> @llvm.roundeven.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %20 = call <vscale x 16 x float> @llvm.roundeven.nxv16f32(<vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %21 = call double @llvm.roundeven.f64(double undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %22 = call <2 x double> @llvm.roundeven.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %23 = call <4 x double> @llvm.roundeven.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %24 = call <8 x double> @llvm.roundeven.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %25 = call <16 x double> @llvm.roundeven.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %26 = call <vscale x 1 x double> @llvm.roundeven.nxv1f64(<vscale x 1 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %27 = call <vscale x 2 x double> @llvm.roundeven.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %28 = call <vscale x 4 x double> @llvm.roundeven.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %29 = call <vscale x 8 x double> @llvm.roundeven.nxv8f64(<vscale x 8 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call bfloat @llvm.roundeven.bf16(bfloat undef)
+  call <2 x bfloat> @llvm.roundeven.v2bf16(<2 x bfloat> undef)
+  call <4 x bfloat> @llvm.roundeven.v4bf16(<4 x bfloat> undef)
+  call <8 x bfloat> @llvm.roundeven.v8bf16(<8 x bfloat> undef)
+  call <16 x bfloat> @llvm.roundeven.v16bf16(<16 x bfloat> undef)
+  call <vscale x 1 x bfloat> @llvm.roundeven.nvx1bf16(<vscale x 1 x bfloat> undef)
+  call <vscale x 2 x bfloat> @llvm.roundeven.nvx2bf16(<vscale x 2 x bfloat> undef)
+  call <vscale x 4 x bfloat> @llvm.roundeven.nvx4bf16(<vscale x 4 x bfloat> undef)
+  call <vscale x 8 x bfloat> @llvm.roundeven.nvx8bf16(<vscale x 8 x bfloat> undef)
+  call <vscale x 16 x bfloat> @llvm.roundeven.nvx16bf16(<vscale x 16 x bfloat> undef)
   call float @llvm.roundeven.f32(float undef)
   call <2 x float> @llvm.roundeven.v2f32(<2 x float> undef)
   call <4 x float> @llvm.roundeven.v4f32(<4 x float> undef)
@@ -743,25 +923,43 @@ define void @roundeven_fp16() {
 
 define void @vp_ceil() {
 ; CHECK-LABEL: 'vp_ceil'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %1 = call <2 x float> @llvm.vp.ceil.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %2 = call <4 x float> @llvm.vp.ceil.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %3 = call <8 x float> @llvm.vp.ceil.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %4 = call <16 x float> @llvm.vp.ceil.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %5 = call <vscale x 1 x float> @llvm.vp.ceil.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %6 = call <vscale x 2 x float> @llvm.vp.ceil.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %7 = call <vscale x 4 x float> @llvm.vp.ceil.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %8 = call <vscale x 8 x float> @llvm.vp.ceil.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %9 = call <vscale x 16 x float> @llvm.vp.ceil.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <2 x double> @llvm.vp.ceil.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %11 = call <4 x double> @llvm.vp.ceil.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <8 x double> @llvm.vp.ceil.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <16 x double> @llvm.vp.ceil.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <vscale x 1 x double> @llvm.vp.ceil.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <vscale x 2 x double> @llvm.vp.ceil.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 4 x double> @llvm.vp.ceil.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 8 x double> @llvm.vp.ceil.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call <2 x bfloat> @llvm.vp.ceil.v2bf16(<2 x bfloat> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <4 x bfloat> @llvm.vp.ceil.v4bf16(<4 x bfloat> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <8 x bfloat> @llvm.vp.ceil.v8bf16(<8 x bfloat> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <16 x bfloat> @llvm.vp.ceil.v16bf16(<16 x bfloat> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <vscale x 1 x bfloat> @llvm.vp.ceil.nxv1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x bfloat> @llvm.vp.ceil.nxv2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x bfloat> @llvm.vp.ceil.nxv4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x bfloat> @llvm.vp.ceil.nxv8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x bfloat> @llvm.vp.ceil.nxv16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <2 x float> @llvm.vp.ceil.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %11 = call <4 x float> @llvm.vp.ceil.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <8 x float> @llvm.vp.ceil.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <16 x float> @llvm.vp.ceil.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <vscale x 1 x float> @llvm.vp.ceil.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <vscale x 2 x float> @llvm.vp.ceil.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 4 x float> @llvm.vp.ceil.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 8 x float> @llvm.vp.ceil.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %18 = call <vscale x 16 x float> @llvm.vp.ceil.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %19 = call <2 x double> @llvm.vp.ceil.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %20 = call <4 x double> @llvm.vp.ceil.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %21 = call <8 x double> @llvm.vp.ceil.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %22 = call <16 x double> @llvm.vp.ceil.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %23 = call <vscale x 1 x double> @llvm.vp.ceil.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %24 = call <vscale x 2 x double> @llvm.vp.ceil.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %25 = call <vscale x 4 x double> @llvm.vp.ceil.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %26 = call <vscale x 8 x double> @llvm.vp.ceil.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call <2 x bfloat> @llvm.vp.ceil.v2bf16(<2 x bfloat> undef, <2 x i1> undef, i32 undef)
+  call <4 x bfloat> @llvm.vp.ceil.v4bf16(<4 x bfloat> undef, <4 x i1> undef, i32 undef)
+  call <8 x bfloat> @llvm.vp.ceil.v8bf16(<8 x bfloat> undef, <8 x i1> undef, i32 undef)
+  call <16 x bfloat> @llvm.vp.ceil.v16bf16(<16 x bfloat> undef, <16 x i1> undef, i32 undef)
+  call <vscale x 1 x bfloat> @llvm.vp.ceil.nvx1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x i1> undef, i32 undef)
+  call <vscale x 2 x bfloat> @llvm.vp.ceil.nvx2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x i1> undef, i32 undef)
+  call <vscale x 4 x bfloat> @llvm.vp.ceil.nvx4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x i1> undef, i32 undef)
+  call <vscale x 8 x bfloat> @llvm.vp.ceil.nvx8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> undef, i32 undef)
+  call <vscale x 16 x bfloat> @llvm.vp.ceil.nvx16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x i1> undef, i32 undef)
   call <2 x float> @llvm.vp.ceil.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
   call <4 x float> @llvm.vp.ceil.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
   call <8 x float> @llvm.vp.ceil.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
@@ -821,25 +1019,43 @@ define void @vp_ceil_f16() {
 
 define void @vp_floor() {
 ; CHECK-LABEL: 'vp_floor'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %1 = call <2 x float> @llvm.vp.floor.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %2 = call <4 x float> @llvm.vp.floor.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %3 = call <8 x float> @llvm.vp.floor.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %4 = call <16 x float> @llvm.vp.floor.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %5 = call <vscale x 1 x float> @llvm.vp.floor.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %6 = call <vscale x 2 x float> @llvm.vp.floor.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %7 = call <vscale x 4 x float> @llvm.vp.floor.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %8 = call <vscale x 8 x float> @llvm.vp.floor.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %9 = call <vscale x 16 x float> @llvm.vp.floor.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <2 x double> @llvm.vp.floor.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %11 = call <4 x double> @llvm.vp.floor.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <8 x double> @llvm.vp.floor.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <16 x double> @llvm.vp.floor.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <vscale x 1 x double> @llvm.vp.floor.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <vscale x 2 x double> @llvm.vp.floor.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 4 x double> @llvm.vp.floor.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 8 x double> @llvm.vp.floor.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call <2 x bfloat> @llvm.vp.floor.v2bf16(<2 x bfloat> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <4 x bfloat> @llvm.vp.floor.v4bf16(<4 x bfloat> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <8 x bfloat> @llvm.vp.floor.v8bf16(<8 x bfloat> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <16 x bfloat> @llvm.vp.floor.v16bf16(<16 x bfloat> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <vscale x 1 x bfloat> @llvm.vp.floor.nxv1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x bfloat> @llvm.vp.floor.nxv2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x bfloat> @llvm.vp.floor.nxv4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x bfloat> @llvm.vp.floor.nxv8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x bfloat> @llvm.vp.floor.nxv16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <2 x float> @llvm.vp.floor.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %11 = call <4 x float> @llvm.vp.floor.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <8 x float> @llvm.vp.floor.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <16 x float> @llvm.vp.floor.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <vscale x 1 x float> @llvm.vp.floor.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <vscale x 2 x float> @llvm.vp.floor.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 4 x float> @llvm.vp.floor.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 8 x float> @llvm.vp.floor.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %18 = call <vscale x 16 x float> @llvm.vp.floor.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %19 = call <2 x double> @llvm.vp.floor.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %20 = call <4 x double> @llvm.vp.floor.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %21 = call <8 x double> @llvm.vp.floor.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %22 = call <16 x double> @llvm.vp.floor.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %23 = call <vscale x 1 x double> @llvm.vp.floor.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %24 = call <vscale x 2 x double> @llvm.vp.floor.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %25 = call <vscale x 4 x double> @llvm.vp.floor.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %26 = call <vscale x 8 x double> @llvm.vp.floor.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call <2 x bfloat> @llvm.vp.floor.v2bf16(<2 x bfloat> undef, <2 x i1> undef, i32 undef)
+  call <4 x bfloat> @llvm.vp.floor.v4bf16(<4 x bfloat> undef, <4 x i1> undef, i32 undef)
+  call <8 x bfloat> @llvm.vp.floor.v8bf16(<8 x bfloat> undef, <8 x i1> undef, i32 undef)
+  call <16 x bfloat> @llvm.vp.floor.v16bf16(<16 x bfloat> undef, <16 x i1> undef, i32 undef)
+  call <vscale x 1 x bfloat> @llvm.vp.floor.nvx1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x i1> undef, i32 undef)
+  call <vscale x 2 x bfloat> @llvm.vp.floor.nvx2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x i1> undef, i32 undef)
+  call <vscale x 4 x bfloat> @llvm.vp.floor.nvx4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x i1> undef, i32 undef)
+  call <vscale x 8 x bfloat> @llvm.vp.floor.nvx8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> undef, i32 undef)
+  call <vscale x 16 x bfloat> @llvm.vp.floor.nvx16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x i1> undef, i32 undef)
   call <2 x float> @llvm.vp.floor.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
   call <4 x float> @llvm.vp.floor.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
   call <8 x float> @llvm.vp.floor.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
@@ -899,25 +1115,43 @@ define void @vp_floor_f16() {
 
 define void @vp_round() {
 ; CHECK-LABEL: 'vp_round'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %1 = call <2 x float> @llvm.vp.round.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %2 = call <4 x float> @llvm.vp.round.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %3 = call <8 x float> @llvm.vp.round.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %4 = call <16 x float> @llvm.vp.round.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %5 = call <vscale x 1 x float> @llvm.vp.round.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %6 = call <vscale x 2 x float> @llvm.vp.round.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %7 = call <vscale x 4 x float> @llvm.vp.round.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %8 = call <vscale x 8 x float> @llvm.vp.round.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %9 = call <vscale x 16 x float> @llvm.vp.round.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <2 x double> @llvm.vp.round.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %11 = call <4 x double> @llvm.vp.round.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <8 x double> @llvm.vp.round.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <16 x double> @llvm.vp.round.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <vscale x 1 x double> @llvm.vp.round.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <vscale x 2 x double> @llvm.vp.round.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 4 x double> @llvm.vp.round.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 8 x double> @llvm.vp.round.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call <2 x bfloat> @llvm.vp.round.v2bf16(<2 x bfloat> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <4 x bfloat> @llvm.vp.round.v4bf16(<4 x bfloat> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <8 x bfloat> @llvm.vp.round.v8bf16(<8 x bfloat> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <16 x bfloat> @llvm.vp.round.v16bf16(<16 x bfloat> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <vscale x 1 x bfloat> @llvm.vp.round.nxv1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x bfloat> @llvm.vp.round.nxv2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x bfloat> @llvm.vp.round.nxv4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x bfloat> @llvm.vp.round.nxv8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x bfloat> @llvm.vp.round.nxv16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <2 x float> @llvm.vp.round.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %11 = call <4 x float> @llvm.vp.round.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <8 x float> @llvm.vp.round.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <16 x float> @llvm.vp.round.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <vscale x 1 x float> @llvm.vp.round.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <vscale x 2 x float> @llvm.vp.round.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 4 x float> @llvm.vp.round.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 8 x float> @llvm.vp.round.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %18 = call <vscale x 16 x float> @llvm.vp.round.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %19 = call <2 x double> @llvm.vp.round.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %20 = call <4 x double> @llvm.vp.round.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %21 = call <8 x double> @llvm.vp.round.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %22 = call <16 x double> @llvm.vp.round.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %23 = call <vscale x 1 x double> @llvm.vp.round.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %24 = call <vscale x 2 x double> @llvm.vp.round.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %25 = call <vscale x 4 x double> @llvm.vp.round.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %26 = call <vscale x 8 x double> @llvm.vp.round.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call <2 x bfloat> @llvm.vp.round.v2bf16(<2 x bfloat> undef, <2 x i1> undef, i32 undef)
+  call <4 x bfloat> @llvm.vp.round.v4bf16(<4 x bfloat> undef, <4 x i1> undef, i32 undef)
+  call <8 x bfloat> @llvm.vp.round.v8bf16(<8 x bfloat> undef, <8 x i1> undef, i32 undef)
+  call <16 x bfloat> @llvm.vp.round.v16bf16(<16 x bfloat> undef, <16 x i1> undef, i32 undef)
+  call <vscale x 1 x bfloat> @llvm.vp.round.nvx1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x i1> undef, i32 undef)
+  call <vscale x 2 x bfloat> @llvm.vp.round.nvx2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x i1> undef, i32 undef)
+  call <vscale x 4 x bfloat> @llvm.vp.round.nvx4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x i1> undef, i32 undef)
+  call <vscale x 8 x bfloat> @llvm.vp.round.nvx8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> undef, i32 undef)
+  call <vscale x 16 x bfloat> @llvm.vp.round.nvx16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x i1> undef, i32 undef)
   call <2 x float> @llvm.vp.round.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
   call <4 x float> @llvm.vp.round.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
   call <8 x float> @llvm.vp.round.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
@@ -977,25 +1211,43 @@ define void @vp_round_f16() {
 
 define void @vp_roundeven() {
 ; CHECK-LABEL: 'vp_roundeven'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %1 = call <2 x float> @llvm.vp.roundeven.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %2 = call <4 x float> @llvm.vp.roundeven.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %3 = call <8 x float> @llvm.vp.roundeven.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %4 = call <16 x float> @llvm.vp.roundeven.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %5 = call <vscale x 1 x float> @llvm.vp.roundeven.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %6 = call <vscale x 2 x float> @llvm.vp.roundeven.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %7 = call <vscale x 4 x float> @llvm.vp.roundeven.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %8 = call <vscale x 8 x float> @llvm.vp.roundeven.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %9 = call <vscale x 16 x float> @llvm.vp.roundeven.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <2 x double> @llvm.vp.roundeven.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %11 = call <4 x double> @llvm.vp.roundeven.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <8 x double> @llvm.vp.roundeven.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <16 x double> @llvm.vp.roundeven.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <vscale x 1 x double> @llvm.vp.roundeven.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <vscale x 2 x double> @llvm.vp.roundeven.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 4 x double> @llvm.vp.roundeven.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 8 x double> @llvm.vp.roundeven.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call <2 x bfloat> @llvm.vp.roundeven.v2bf16(<2 x bfloat> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <4 x bfloat> @llvm.vp.roundeven.v4bf16(<4 x bfloat> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <8 x bfloat> @llvm.vp.roundeven.v8bf16(<8 x bfloat> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <16 x bfloat> @llvm.vp.roundeven.v16bf16(<16 x bfloat> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <vscale x 1 x bfloat> @llvm.vp.roundeven.nxv1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x bfloat> @llvm.vp.roundeven.nxv2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x bfloat> @llvm.vp.roundeven.nxv4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x bfloat> @llvm.vp.roundeven.nxv8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x bfloat> @llvm.vp.roundeven.nxv16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <2 x float> @llvm.vp.roundeven.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %11 = call <4 x float> @llvm.vp.roundeven.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <8 x float> @llvm.vp.roundeven.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <16 x float> @llvm.vp.roundeven.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <vscale x 1 x float> @llvm.vp.roundeven.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <vscale x 2 x float> @llvm.vp.roundeven.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 4 x float> @llvm.vp.roundeven.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 8 x float> @llvm.vp.roundeven.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %18 = call <vscale x 16 x float> @llvm.vp.roundeven.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %19 = call <2 x double> @llvm.vp.roundeven.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %20 = call <4 x double> @llvm.vp.roundeven.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %21 = call <8 x double> @llvm.vp.roundeven.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %22 = call <16 x double> @llvm.vp.roundeven.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %23 = call <vscale x 1 x double> @llvm.vp.roundeven.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %24 = call <vscale x 2 x double> @llvm.vp.roundeven.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %25 = call <vscale x 4 x double> @llvm.vp.roundeven.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %26 = call <vscale x 8 x double> @llvm.vp.roundeven.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call <2 x bfloat> @llvm.vp.roundeven.v2bf16(<2 x bfloat> undef, <2 x i1> undef, i32 undef)
+  call <4 x bfloat> @llvm.vp.roundeven.v4bf16(<4 x bfloat> undef, <4 x i1> undef, i32 undef)
+  call <8 x bfloat> @llvm.vp.roundeven.v8bf16(<8 x bfloat> undef, <8 x i1> undef, i32 undef)
+  call <16 x bfloat> @llvm.vp.roundeven.v16bf16(<16 x bfloat> undef, <16 x i1> undef, i32 undef)
+  call <vscale x 1 x bfloat> @llvm.vp.roundeven.nvx1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x i1> undef, i32 undef)
+  call <vscale x 2 x bfloat> @llvm.vp.roundeven.nvx2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x i1> undef, i32 undef)
+  call <vscale x 4 x bfloat> @llvm.vp.roundeven.nvx4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x i1> undef, i32 undef)
+  call <vscale x 8 x bfloat> @llvm.vp.roundeven.nvx8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> undef, i32 undef)
+  call <vscale x 16 x bfloat> @llvm.vp.roundeven.nvx16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x i1> undef, i32 undef)
   call <2 x float> @llvm.vp.roundeven.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
   call <4 x float> @llvm.vp.roundeven.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
   call <8 x float> @llvm.vp.roundeven.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
@@ -1055,25 +1307,43 @@ define void @vp_roundeven_f16() {
 
 define void @vp_roundtozero() {
 ; CHECK-LABEL: 'vp_roundtozero'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %1 = call <2 x float> @llvm.vp.roundtozero.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %2 = call <4 x float> @llvm.vp.roundtozero.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %3 = call <8 x float> @llvm.vp.roundtozero.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %4 = call <16 x float> @llvm.vp.roundtozero.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %5 = call <vscale x 1 x float> @llvm.vp.roundtozero.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %6 = call <vscale x 2 x float> @llvm.vp.roundtozero.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %7 = call <vscale x 4 x float> @llvm.vp.roundtozero.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %8 = call <vscale x 8 x float> @llvm.vp.roundtozero.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %9 = call <vscale x 16 x float> @llvm.vp.roundtozero.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <2 x double> @llvm.vp.roundtozero.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %11 = call <4 x double> @llvm.vp.roundtozero.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <8 x double> @llvm.vp.roundtozero.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <16 x double> @llvm.vp.roundtozero.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <vscale x 1 x double> @llvm.vp.roundtozero.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <vscale x 2 x double> @llvm.vp.roundtozero.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 4 x double> @llvm.vp.roundtozero.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 8 x double> @llvm.vp.roundtozero.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call <2 x bfloat> @llvm.vp.roundtozero.v2bf16(<2 x bfloat> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <4 x bfloat> @llvm.vp.roundtozero.v4bf16(<4 x bfloat> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <8 x bfloat> @llvm.vp.roundtozero.v8bf16(<8 x bfloat> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <16 x bfloat> @llvm.vp.roundtozero.v16bf16(<16 x bfloat> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <vscale x 1 x bfloat> @llvm.vp.roundtozero.nxv1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x bfloat> @llvm.vp.roundtozero.nxv2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x bfloat> @llvm.vp.roundtozero.nxv4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x bfloat> @llvm.vp.roundtozero.nxv8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x bfloat> @llvm.vp.roundtozero.nxv16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <2 x float> @llvm.vp.roundtozero.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %11 = call <4 x float> @llvm.vp.roundtozero.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <8 x float> @llvm.vp.roundtozero.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <16 x float> @llvm.vp.roundtozero.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <vscale x 1 x float> @llvm.vp.roundtozero.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <vscale x 2 x float> @llvm.vp.roundtozero.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 4 x float> @llvm.vp.roundtozero.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 8 x float> @llvm.vp.roundtozero.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %18 = call <vscale x 16 x float> @llvm.vp.roundtozero.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %19 = call <2 x double> @llvm.vp.roundtozero.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %20 = call <4 x double> @llvm.vp.roundtozero.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %21 = call <8 x double> @llvm.vp.roundtozero.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %22 = call <16 x double> @llvm.vp.roundtozero.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %23 = call <vscale x 1 x double> @llvm.vp.roundtozero.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %24 = call <vscale x 2 x double> @llvm.vp.roundtozero.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %25 = call <vscale x 4 x double> @llvm.vp.roundtozero.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %26 = call <vscale x 8 x double> @llvm.vp.roundtozero.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call <2 x bfloat> @llvm.vp.roundtozero.v2bf16(<2 x bfloat> undef, <2 x i1> undef, i32 undef)
+  call <4 x bfloat> @llvm.vp.roundtozero.v4bf16(<4 x bfloat> undef, <4 x i1> undef, i32 undef)
+  call <8 x bfloat> @llvm.vp.roundtozero.v8bf16(<8 x bfloat> undef, <8 x i1> undef, i32 undef)
+  call <16 x bfloat> @llvm.vp.roundtozero.v16bf16(<16 x bfloat> undef, <16 x i1> undef, i32 undef)
+  call <vscale x 1 x bfloat> @llvm.vp.roundtozero.nvx1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x i1> undef, i32 undef)
+  call <vscale x 2 x bfloat> @llvm.vp.roundtozero.nvx2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x i1> undef, i32 undef)
+  call <vscale x 4 x bfloat> @llvm.vp.roundtozero.nvx4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x i1> undef, i32 undef)
+  call <vscale x 8 x bfloat> @llvm.vp.roundtozero.nvx8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> undef, i32 undef)
+  call <vscale x 16 x bfloat> @llvm.vp.roundtozero.nvx16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x i1> undef, i32 undef)
   call <2 x float> @llvm.vp.roundtozero.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
   call <4 x float> @llvm.vp.roundtozero.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
   call <8 x float> @llvm.vp.roundtozero.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
@@ -1133,25 +1403,43 @@ define void @vp_roundtozero_f16() {
 
 define void @vp_rint() {
 ; CHECK-LABEL: 'vp_rint'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %1 = call <2 x float> @llvm.vp.rint.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %2 = call <4 x float> @llvm.vp.rint.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %3 = call <8 x float> @llvm.vp.rint.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %4 = call <16 x float> @llvm.vp.rint.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %5 = call <vscale x 1 x float> @llvm.vp.rint.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %6 = call <vscale x 2 x float> @llvm.vp.rint.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %7 = call <vscale x 4 x float> @llvm.vp.rint.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %8 = call <vscale x 8 x float> @llvm.vp.rint.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %9 = call <vscale x 16 x float> @llvm.vp.rint.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %10 = call <2 x double> @llvm.vp.rint.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %11 = call <4 x double> @llvm.vp.rint.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %12 = call <8 x double> @llvm.vp.rint.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %13 = call <16 x double> @llvm.vp.rint.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %14 = call <vscale x 1 x double> @llvm.vp.rint.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %15 = call <vscale x 2 x double> @llvm.vp.rint.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %16 = call <vscale x 4 x double> @llvm.vp.rint.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %17 = call <vscale x 8 x double> @llvm.vp.rint.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call <2 x bfloat> @llvm.vp.rint.v2bf16(<2 x bfloat> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <4 x bfloat> @llvm.vp.rint.v4bf16(<4 x bfloat> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <8 x bfloat> @llvm.vp.rint.v8bf16(<8 x bfloat> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <16 x bfloat> @llvm.vp.rint.v16bf16(<16 x bfloat> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <vscale x 1 x bfloat> @llvm.vp.rint.nxv1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x bfloat> @llvm.vp.rint.nxv2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x bfloat> @llvm.vp.rint.nxv4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x bfloat> @llvm.vp.rint.nxv8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x bfloat> @llvm.vp.rint.nxv16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %10 = call <2 x float> @llvm.vp.rint.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %11 = call <4 x float> @llvm.vp.rint.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %12 = call <8 x float> @llvm.vp.rint.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %13 = call <16 x float> @llvm.vp.rint.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %14 = call <vscale x 1 x float> @llvm.vp.rint.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %15 = call <vscale x 2 x float> @llvm.vp.rint.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %16 = call <vscale x 4 x float> @llvm.vp.rint.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %17 = call <vscale x 8 x float> @llvm.vp.rint.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %18 = call <vscale x 16 x float> @llvm.vp.rint.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %19 = call <2 x double> @llvm.vp.rint.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %20 = call <4 x double> @llvm.vp.rint.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %21 = call <8 x double> @llvm.vp.rint.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %22 = call <16 x double> @llvm.vp.rint.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %23 = call <vscale x 1 x double> @llvm.vp.rint.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %24 = call <vscale x 2 x double> @llvm.vp.rint.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %25 = call <vscale x 4 x double> @llvm.vp.rint.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %26 = call <vscale x 8 x double> @llvm.vp.rint.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call <2 x bfloat> @llvm.vp.rint.v2bf16(<2 x bfloat> undef, <2 x i1> undef, i32 undef)
+  call <4 x bfloat> @llvm.vp.rint.v4bf16(<4 x bfloat> undef, <4 x i1> undef, i32 undef)
+  call <8 x bfloat> @llvm.vp.rint.v8bf16(<8 x bfloat> undef, <8 x i1> undef, i32 undef)
+  call <16 x bfloat> @llvm.vp.rint.v16bf16(<16 x bfloat> undef, <16 x i1> undef, i32 undef)
+  call <vscale x 1 x bfloat> @llvm.vp.rint.nvx1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x i1> undef, i32 undef)
+  call <vscale x 2 x bfloat> @llvm.vp.rint.nvx2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x i1> undef, i32 undef)
+  call <vscale x 4 x bfloat> @llvm.vp.rint.nvx4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x i1> undef, i32 undef)
+  call <vscale x 8 x bfloat> @llvm.vp.rint.nvx8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> undef, i32 undef)
+  call <vscale x 16 x bfloat> @llvm.vp.rint.nvx16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x i1> undef, i32 undef)
   call <2 x float> @llvm.vp.rint.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
   call <4 x float> @llvm.vp.rint.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
   call <8 x float> @llvm.vp.rint.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
@@ -1211,25 +1499,43 @@ define void @vp_rint_f16() {
 
 define void @vp_nearbyint() {
 ; CHECK-LABEL: 'vp_nearbyint'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %1 = call <2 x float> @llvm.vp.nearbyint.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %2 = call <4 x float> @llvm.vp.nearbyint.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %3 = call <8 x float> @llvm.vp.nearbyint.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %4 = call <16 x float> @llvm.vp.nearbyint.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %5 = call <vscale x 1 x float> @llvm.vp.nearbyint.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %6 = call <vscale x 2 x float> @llvm.vp.nearbyint.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %7 = call <vscale x 4 x float> @llvm.vp.nearbyint.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %8 = call <vscale x 8 x float> @llvm.vp.nearbyint.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %9 = call <vscale x 16 x float> @llvm.vp.nearbyint.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <2 x double> @llvm.vp.nearbyint.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %11 = call <4 x double> @llvm.vp.nearbyint.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <8 x double> @llvm.vp.nearbyint.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <16 x double> @llvm.vp.nearbyint.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <vscale x 1 x double> @llvm.vp.nearbyint.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <vscale x 2 x double> @llvm.vp.nearbyint.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 4 x double> @llvm.vp.nearbyint.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 8 x double> @llvm.vp.nearbyint.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call <2 x bfloat> @llvm.vp.nearbyint.v2bf16(<2 x bfloat> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <4 x bfloat> @llvm.vp.nearbyint.v4bf16(<4 x bfloat> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <8 x bfloat> @llvm.vp.nearbyint.v8bf16(<8 x bfloat> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <16 x bfloat> @llvm.vp.nearbyint.v16bf16(<16 x bfloat> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <vscale x 1 x bfloat> @llvm.vp.nearbyint.nxv1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x bfloat> @llvm.vp.nearbyint.nxv2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x bfloat> @llvm.vp.nearbyint.nxv4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x bfloat> @llvm.vp.nearbyint.nxv8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x bfloat> @llvm.vp.nearbyint.nxv16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call <2 x float> @llvm.vp.nearbyint.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %11 = call <4 x float> @llvm.vp.nearbyint.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %12 = call <8 x float> @llvm.vp.nearbyint.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %13 = call <16 x float> @llvm.vp.nearbyint.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = call <vscale x 1 x float> @llvm.vp.nearbyint.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %15 = call <vscale x 2 x float> @llvm.vp.nearbyint.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %16 = call <vscale x 4 x float> @llvm.vp.nearbyint.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = call <vscale x 8 x float> @llvm.vp.nearbyint.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %18 = call <vscale x 16 x float> @llvm.vp.nearbyint.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %19 = call <2 x double> @llvm.vp.nearbyint.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %20 = call <4 x double> @llvm.vp.nearbyint.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %21 = call <8 x double> @llvm.vp.nearbyint.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %22 = call <16 x double> @llvm.vp.nearbyint.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %23 = call <vscale x 1 x double> @llvm.vp.nearbyint.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %24 = call <vscale x 2 x double> @llvm.vp.nearbyint.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %25 = call <vscale x 4 x double> @llvm.vp.nearbyint.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %26 = call <vscale x 8 x double> @llvm.vp.nearbyint.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call <2 x bfloat> @llvm.vp.nearbyint.v2bf16(<2 x bfloat> undef, <2 x i1> undef, i32 undef)
+  call <4 x bfloat> @llvm.vp.nearbyint.v4bf16(<4 x bfloat> undef, <4 x i1> undef, i32 undef)
+  call <8 x bfloat> @llvm.vp.nearbyint.v8bf16(<8 x bfloat> undef, <8 x i1> undef, i32 undef)
+  call <16 x bfloat> @llvm.vp.nearbyint.v16bf16(<16 x bfloat> undef, <16 x i1> undef, i32 undef)
+  call <vscale x 1 x bfloat> @llvm.vp.nearbyint.nvx1bf16(<vscale x 1 x bfloat> undef, <vscale x 1 x i1> undef, i32 undef)
+  call <vscale x 2 x bfloat> @llvm.vp.nearbyint.nvx2bf16(<vscale x 2 x bfloat> undef, <vscale x 2 x i1> undef, i32 undef)
+  call <vscale x 4 x bfloat> @llvm.vp.nearbyint.nvx4bf16(<vscale x 4 x bfloat> undef, <vscale x 4 x i1> undef, i32 undef)
+  call <vscale x 8 x bfloat> @llvm.vp.nearbyint.nvx8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> undef, i32 undef)
+  call <vscale x 16 x bfloat> @llvm.vp.nearbyint.nvx16bf16(<vscale x 16 x bfloat> undef, <vscale x 16 x i1> undef, i32 undef)
   call <2 x float> @llvm.vp.nearbyint.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
   call <4 x float> @llvm.vp.nearbyint.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
   call <8 x float> @llvm.vp.nearbyint.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-- 
GitLab


From c9f01f699cc55929b18befc0fa34d70630fc9074 Mon Sep 17 00:00:00 2001
From: Sushant Gokhale <sgokhale@nvidia.com>
Date: Mon, 28 Oct 2024 20:37:41 +0530
Subject: [PATCH 324/693] [SLP][AArch64][NFC] Add more tests for SLP
 vectorization of div (#113876)

Currently, we dont have much tests that show SLP outcome for integer
divisions. This patch adds tests for same.

In certain scenarios, for Neon, vectorization is profitable. An attempt
would be made in future to improve the cost-model for the same.
---
 .../Transforms/SLPVectorizer/AArch64/div.ll   | 553 ++++++++++++++++++
 1 file changed, 553 insertions(+)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/div.ll

diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/div.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/div.ll
new file mode 100644
index 000000000000..e972955e26cb
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/div.ll
@@ -0,0 +1,553 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=aarch64 -passes=slp-vectorizer -mattr=-sve -S < %s | FileCheck %s --check-prefixes=CHECK,NO-SVE
+; RUN: opt -mtriple=aarch64 -passes=slp-vectorizer -mattr=+sve -S < %s | FileCheck %s --check-prefixes=CHECK,SVE
+
+define <2 x i8> @slp_v2i8_Op1_Op2_unknown(<2 x i8> %a, <2 x i8> %b)
+; NO-SVE-LABEL: define <2 x i8> @slp_v2i8_Op1_Op2_unknown(
+; NO-SVE-SAME: <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; NO-SVE-NEXT:    [[A0:%.*]] = extractelement <2 x i8> [[A]], i32 0
+; NO-SVE-NEXT:    [[A1:%.*]] = extractelement <2 x i8> [[A]], i32 1
+; NO-SVE-NEXT:    [[B0:%.*]] = extractelement <2 x i8> [[B]], i32 0
+; NO-SVE-NEXT:    [[B1:%.*]] = extractelement <2 x i8> [[B]], i32 1
+; NO-SVE-NEXT:    [[TMP1:%.*]] = sdiv i8 [[A0]], [[B0]]
+; NO-SVE-NEXT:    [[TMP2:%.*]] = sdiv i8 [[A1]], [[B1]]
+; NO-SVE-NEXT:    [[R0:%.*]] = insertelement <2 x i8> poison, i8 [[TMP1]], i32 0
+; NO-SVE-NEXT:    [[R1:%.*]] = insertelement <2 x i8> [[R0]], i8 [[TMP2]], i32 1
+; NO-SVE-NEXT:    ret <2 x i8> [[R1]]
+;
+; SVE-LABEL: define <2 x i8> @slp_v2i8_Op1_Op2_unknown(
+; SVE-SAME: <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; SVE-NEXT:    [[TMP1:%.*]] = sdiv <2 x i8> [[A]], [[B]]
+; SVE-NEXT:    ret <2 x i8> [[TMP1]]
+;
+{
+  %a0 = extractelement <2 x i8> %a, i32 0
+  %a1 = extractelement <2 x i8> %a, i32 1
+  %b0 = extractelement <2 x i8> %b, i32 0
+  %b1 = extractelement <2 x i8> %b, i32 1
+  %1 = sdiv i8 %a0, %b0
+  %2 = sdiv i8 %a1, %b1
+  %r0 = insertelement <2 x i8> poison, i8 %1, i32 0
+  %r1 = insertelement <2 x i8> %r0, i8 %2, i32 1
+  ret <2 x i8> %r1
+}
+
+define <2 x i16> @slp_v2i16_Op1_Op2_unknown(<2 x i16> %a, <2 x i16> %b)
+; NO-SVE-LABEL: define <2 x i16> @slp_v2i16_Op1_Op2_unknown(
+; NO-SVE-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) #[[ATTR0]] {
+; NO-SVE-NEXT:    [[A0:%.*]] = extractelement <2 x i16> [[A]], i32 0
+; NO-SVE-NEXT:    [[A1:%.*]] = extractelement <2 x i16> [[A]], i32 1
+; NO-SVE-NEXT:    [[B0:%.*]] = extractelement <2 x i16> [[B]], i32 0
+; NO-SVE-NEXT:    [[B1:%.*]] = extractelement <2 x i16> [[B]], i32 1
+; NO-SVE-NEXT:    [[TMP1:%.*]] = sdiv i16 [[A0]], [[B0]]
+; NO-SVE-NEXT:    [[TMP2:%.*]] = sdiv i16 [[A1]], [[B1]]
+; NO-SVE-NEXT:    [[R0:%.*]] = insertelement <2 x i16> poison, i16 [[TMP1]], i32 0
+; NO-SVE-NEXT:    [[R1:%.*]] = insertelement <2 x i16> [[R0]], i16 [[TMP2]], i32 1
+; NO-SVE-NEXT:    ret <2 x i16> [[R1]]
+;
+; SVE-LABEL: define <2 x i16> @slp_v2i16_Op1_Op2_unknown(
+; SVE-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) #[[ATTR0]] {
+; SVE-NEXT:    [[TMP1:%.*]] = sdiv <2 x i16> [[A]], [[B]]
+; SVE-NEXT:    ret <2 x i16> [[TMP1]]
+;
+{
+  %a0 = extractelement <2 x i16> %a, i32 0
+  %a1 = extractelement <2 x i16> %a, i32 1
+  %b0 = extractelement <2 x i16> %b, i32 0
+  %b1 = extractelement <2 x i16> %b, i32 1
+  %1 = sdiv i16 %a0, %b0
+  %2 = sdiv i16 %a1, %b1
+  %r0 = insertelement <2 x i16> poison, i16 %1, i32 0
+  %r1 = insertelement <2 x i16> %r0, i16 %2, i32 1
+  ret <2 x i16> %r1
+}
+
+define <2 x i32> @slp_v2i32_Op1_Op2_unknown(<2 x i32> %a, <2 x i32> %b)
+; NO-SVE-LABEL: define <2 x i32> @slp_v2i32_Op1_Op2_unknown(
+; NO-SVE-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]]) #[[ATTR0]] {
+; NO-SVE-NEXT:    [[A0:%.*]] = extractelement <2 x i32> [[A]], i32 0
+; NO-SVE-NEXT:    [[A1:%.*]] = extractelement <2 x i32> [[A]], i32 1
+; NO-SVE-NEXT:    [[B0:%.*]] = extractelement <2 x i32> [[B]], i32 0
+; NO-SVE-NEXT:    [[B1:%.*]] = extractelement <2 x i32> [[B]], i32 1
+; NO-SVE-NEXT:    [[TMP1:%.*]] = sdiv i32 [[A0]], [[B0]]
+; NO-SVE-NEXT:    [[TMP2:%.*]] = sdiv i32 [[A1]], [[B1]]
+; NO-SVE-NEXT:    [[R0:%.*]] = insertelement <2 x i32> poison, i32 [[TMP1]], i32 0
+; NO-SVE-NEXT:    [[R1:%.*]] = insertelement <2 x i32> [[R0]], i32 [[TMP2]], i32 1
+; NO-SVE-NEXT:    ret <2 x i32> [[R1]]
+;
+; SVE-LABEL: define <2 x i32> @slp_v2i32_Op1_Op2_unknown(
+; SVE-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]]) #[[ATTR0]] {
+; SVE-NEXT:    [[TMP1:%.*]] = sdiv <2 x i32> [[A]], [[B]]
+; SVE-NEXT:    ret <2 x i32> [[TMP1]]
+;
+{
+  %a0 = extractelement <2 x i32> %a, i32 0
+  %a1 = extractelement <2 x i32> %a, i32 1
+  %b0 = extractelement <2 x i32> %b, i32 0
+  %b1 = extractelement <2 x i32> %b, i32 1
+  %1 = sdiv i32 %a0, %b0
+  %2 = sdiv i32 %a1, %b1
+  %r0 = insertelement <2 x i32> poison, i32 %1, i32 0
+  %r1 = insertelement <2 x i32> %r0, i32 %2, i32 1
+  ret <2 x i32> %r1
+}
+
+define <2 x i64> @slp_v2i64_Op1_Op2_unknown(<2 x i64> %a, <2 x i64> %b)
+; NO-SVE-LABEL: define <2 x i64> @slp_v2i64_Op1_Op2_unknown(
+; NO-SVE-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) #[[ATTR0]] {
+; NO-SVE-NEXT:    [[A0:%.*]] = extractelement <2 x i64> [[A]], i32 0
+; NO-SVE-NEXT:    [[A1:%.*]] = extractelement <2 x i64> [[A]], i32 1
+; NO-SVE-NEXT:    [[B0:%.*]] = extractelement <2 x i64> [[B]], i32 0
+; NO-SVE-NEXT:    [[B1:%.*]] = extractelement <2 x i64> [[B]], i32 1
+; NO-SVE-NEXT:    [[TMP1:%.*]] = sdiv i64 [[A0]], [[B0]]
+; NO-SVE-NEXT:    [[TMP2:%.*]] = sdiv i64 [[A1]], [[B1]]
+; NO-SVE-NEXT:    [[R0:%.*]] = insertelement <2 x i64> poison, i64 [[TMP1]], i32 0
+; NO-SVE-NEXT:    [[R1:%.*]] = insertelement <2 x i64> [[R0]], i64 [[TMP2]], i32 1
+; NO-SVE-NEXT:    ret <2 x i64> [[R1]]
+;
+; SVE-LABEL: define <2 x i64> @slp_v2i64_Op1_Op2_unknown(
+; SVE-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) #[[ATTR0]] {
+; SVE-NEXT:    [[TMP1:%.*]] = sdiv <2 x i64> [[A]], [[B]]
+; SVE-NEXT:    ret <2 x i64> [[TMP1]]
+;
+{
+  %a0 = extractelement <2 x i64> %a, i32 0
+  %a1 = extractelement <2 x i64> %a, i32 1
+  %b0 = extractelement <2 x i64> %b, i32 0
+  %b1 = extractelement <2 x i64> %b, i32 1
+  %1 = sdiv i64 %a0, %b0
+  %2 = sdiv i64 %a1, %b1
+  %r0 = insertelement <2 x i64> poison, i64 %1, i32 0
+  %r1 = insertelement <2 x i64> %r0, i64 %2, i32 1
+  ret <2 x i64> %r1
+}
+
+define <4 x i8> @slp_v4i8_Op1_Op2_unknown(<4 x i8> %a, <4 x i8> %b)
+; NO-SVE-LABEL: define <4 x i8> @slp_v4i8_Op1_Op2_unknown(
+; NO-SVE-SAME: <4 x i8> [[A:%.*]], <4 x i8> [[B:%.*]]) #[[ATTR0]] {
+; NO-SVE-NEXT:    [[A0:%.*]] = extractelement <4 x i8> [[A]], i32 0
+; NO-SVE-NEXT:    [[A1:%.*]] = extractelement <4 x i8> [[A]], i32 1
+; NO-SVE-NEXT:    [[A2:%.*]] = extractelement <4 x i8> [[A]], i32 2
+; NO-SVE-NEXT:    [[A3:%.*]] = extractelement <4 x i8> [[A]], i32 3
+; NO-SVE-NEXT:    [[B0:%.*]] = extractelement <4 x i8> [[B]], i32 0
+; NO-SVE-NEXT:    [[B1:%.*]] = extractelement <4 x i8> [[B]], i32 1
+; NO-SVE-NEXT:    [[B2:%.*]] = extractelement <4 x i8> [[B]], i32 2
+; NO-SVE-NEXT:    [[B3:%.*]] = extractelement <4 x i8> [[B]], i32 3
+; NO-SVE-NEXT:    [[TMP1:%.*]] = sdiv i8 [[A0]], [[B0]]
+; NO-SVE-NEXT:    [[TMP2:%.*]] = sdiv i8 [[A1]], [[B1]]
+; NO-SVE-NEXT:    [[TMP3:%.*]] = sdiv i8 [[A2]], [[B2]]
+; NO-SVE-NEXT:    [[TMP4:%.*]] = sdiv i8 [[A3]], [[B3]]
+; NO-SVE-NEXT:    [[R0:%.*]] = insertelement <4 x i8> poison, i8 [[TMP1]], i32 0
+; NO-SVE-NEXT:    [[R1:%.*]] = insertelement <4 x i8> [[R0]], i8 [[TMP2]], i32 1
+; NO-SVE-NEXT:    [[R2:%.*]] = insertelement <4 x i8> [[R1]], i8 [[TMP3]], i32 2
+; NO-SVE-NEXT:    [[R3:%.*]] = insertelement <4 x i8> [[R2]], i8 [[TMP4]], i32 3
+; NO-SVE-NEXT:    ret <4 x i8> [[R3]]
+;
+; SVE-LABEL: define <4 x i8> @slp_v4i8_Op1_Op2_unknown(
+; SVE-SAME: <4 x i8> [[A:%.*]], <4 x i8> [[B:%.*]]) #[[ATTR0]] {
+; SVE-NEXT:    [[TMP1:%.*]] = sdiv <4 x i8> [[A]], [[B]]
+; SVE-NEXT:    ret <4 x i8> [[TMP1]]
+;
+{
+  %a0 = extractelement <4 x i8> %a, i32 0
+  %a1 = extractelement <4 x i8> %a, i32 1
+  %a2 = extractelement <4 x i8> %a, i32 2
+  %a3 = extractelement <4 x i8> %a, i32 3
+  %b0 = extractelement <4 x i8> %b, i32 0
+  %b1 = extractelement <4 x i8> %b, i32 1
+  %b2 = extractelement <4 x i8> %b, i32 2
+  %b3 = extractelement <4 x i8> %b, i32 3
+  %1 = sdiv i8 %a0, %b0
+  %2 = sdiv i8 %a1, %b1
+  %3 = sdiv i8 %a2, %b2
+  %4 = sdiv i8 %a3, %b3
+  %r0 = insertelement <4 x i8> poison, i8 %1, i32 0
+  %r1 = insertelement <4 x i8> %r0, i8 %2, i32 1
+  %r2 = insertelement <4 x i8> %r1, i8 %3, i32 2
+  %r3 = insertelement <4 x i8> %r2, i8 %4, i32 3
+  ret <4 x i8> %r3
+}
+
+define <4 x i16> @slp_v4i16_Op1_Op2_unknown(<4 x i16> %a, <4 x i16> %b)
+; NO-SVE-LABEL: define <4 x i16> @slp_v4i16_Op1_Op2_unknown(
+; NO-SVE-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]]) #[[ATTR0]] {
+; NO-SVE-NEXT:    [[A0:%.*]] = extractelement <4 x i16> [[A]], i32 0
+; NO-SVE-NEXT:    [[A1:%.*]] = extractelement <4 x i16> [[A]], i32 1
+; NO-SVE-NEXT:    [[A2:%.*]] = extractelement <4 x i16> [[A]], i32 2
+; NO-SVE-NEXT:    [[A3:%.*]] = extractelement <4 x i16> [[A]], i32 3
+; NO-SVE-NEXT:    [[B0:%.*]] = extractelement <4 x i16> [[B]], i32 0
+; NO-SVE-NEXT:    [[B1:%.*]] = extractelement <4 x i16> [[B]], i32 1
+; NO-SVE-NEXT:    [[B2:%.*]] = extractelement <4 x i16> [[B]], i32 2
+; NO-SVE-NEXT:    [[B3:%.*]] = extractelement <4 x i16> [[B]], i32 3
+; NO-SVE-NEXT:    [[TMP1:%.*]] = sdiv i16 [[A0]], [[B0]]
+; NO-SVE-NEXT:    [[TMP2:%.*]] = sdiv i16 [[A1]], [[B1]]
+; NO-SVE-NEXT:    [[TMP3:%.*]] = sdiv i16 [[A2]], [[B2]]
+; NO-SVE-NEXT:    [[TMP4:%.*]] = sdiv i16 [[A3]], [[B3]]
+; NO-SVE-NEXT:    [[R0:%.*]] = insertelement <4 x i16> poison, i16 [[TMP1]], i32 0
+; NO-SVE-NEXT:    [[R1:%.*]] = insertelement <4 x i16> [[R0]], i16 [[TMP2]], i32 1
+; NO-SVE-NEXT:    [[R2:%.*]] = insertelement <4 x i16> [[R1]], i16 [[TMP3]], i32 2
+; NO-SVE-NEXT:    [[R3:%.*]] = insertelement <4 x i16> [[R2]], i16 [[TMP4]], i32 3
+; NO-SVE-NEXT:    ret <4 x i16> [[R3]]
+;
+; SVE-LABEL: define <4 x i16> @slp_v4i16_Op1_Op2_unknown(
+; SVE-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]]) #[[ATTR0]] {
+; SVE-NEXT:    [[TMP1:%.*]] = sdiv <4 x i16> [[A]], [[B]]
+; SVE-NEXT:    ret <4 x i16> [[TMP1]]
+;
+{
+  %a0 = extractelement <4 x i16> %a, i32 0
+  %a1 = extractelement <4 x i16> %a, i32 1
+  %a2 = extractelement <4 x i16> %a, i32 2
+  %a3 = extractelement <4 x i16> %a, i32 3
+  %b0 = extractelement <4 x i16> %b, i32 0
+  %b1 = extractelement <4 x i16> %b, i32 1
+  %b2 = extractelement <4 x i16> %b, i32 2
+  %b3 = extractelement <4 x i16> %b, i32 3
+  %1 = sdiv i16 %a0, %b0
+  %2 = sdiv i16 %a1, %b1
+  %3 = sdiv i16 %a2, %b2
+  %4 = sdiv i16 %a3, %b3
+  %r0 = insertelement <4 x i16> poison, i16 %1, i32 0
+  %r1 = insertelement <4 x i16> %r0, i16 %2, i32 1
+  %r2 = insertelement <4 x i16> %r1, i16 %3, i32 2
+  %r3 = insertelement <4 x i16> %r2, i16 %4, i32 3
+  ret <4 x i16> %r3
+}
+
+define <4 x i32> @slp_v4i32_Op1_Op2_unknown(<4 x i32> %a, <4 x i32> %b)
+; NO-SVE-LABEL: define <4 x i32> @slp_v4i32_Op1_Op2_unknown(
+; NO-SVE-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; NO-SVE-NEXT:    [[A0:%.*]] = extractelement <4 x i32> [[A]], i32 0
+; NO-SVE-NEXT:    [[A1:%.*]] = extractelement <4 x i32> [[A]], i32 1
+; NO-SVE-NEXT:    [[A2:%.*]] = extractelement <4 x i32> [[A]], i32 2
+; NO-SVE-NEXT:    [[A3:%.*]] = extractelement <4 x i32> [[A]], i32 3
+; NO-SVE-NEXT:    [[B0:%.*]] = extractelement <4 x i32> [[B]], i32 0
+; NO-SVE-NEXT:    [[B1:%.*]] = extractelement <4 x i32> [[B]], i32 1
+; NO-SVE-NEXT:    [[B2:%.*]] = extractelement <4 x i32> [[B]], i32 2
+; NO-SVE-NEXT:    [[B3:%.*]] = extractelement <4 x i32> [[B]], i32 3
+; NO-SVE-NEXT:    [[TMP1:%.*]] = sdiv i32 [[A0]], [[B0]]
+; NO-SVE-NEXT:    [[TMP2:%.*]] = sdiv i32 [[A1]], [[B1]]
+; NO-SVE-NEXT:    [[TMP3:%.*]] = sdiv i32 [[A2]], [[B2]]
+; NO-SVE-NEXT:    [[TMP4:%.*]] = sdiv i32 [[A3]], [[B3]]
+; NO-SVE-NEXT:    [[R0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i32 0
+; NO-SVE-NEXT:    [[R1:%.*]] = insertelement <4 x i32> [[R0]], i32 [[TMP2]], i32 1
+; NO-SVE-NEXT:    [[R2:%.*]] = insertelement <4 x i32> [[R1]], i32 [[TMP3]], i32 2
+; NO-SVE-NEXT:    [[R3:%.*]] = insertelement <4 x i32> [[R2]], i32 [[TMP4]], i32 3
+; NO-SVE-NEXT:    ret <4 x i32> [[R3]]
+;
+; SVE-LABEL: define <4 x i32> @slp_v4i32_Op1_Op2_unknown(
+; SVE-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; SVE-NEXT:    [[TMP1:%.*]] = sdiv <4 x i32> [[A]], [[B]]
+; SVE-NEXT:    ret <4 x i32> [[TMP1]]
+;
+{
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %b0 = extractelement <4 x i32> %b, i32 0
+  %b1 = extractelement <4 x i32> %b, i32 1
+  %b2 = extractelement <4 x i32> %b, i32 2
+  %b3 = extractelement <4 x i32> %b, i32 3
+  %1 = sdiv i32 %a0, %b0
+  %2 = sdiv i32 %a1, %b1
+  %3 = sdiv i32 %a2, %b2
+  %4 = sdiv i32 %a3, %b3
+  %r0 = insertelement <4 x i32> poison, i32 %1, i32 0
+  %r1 = insertelement <4 x i32> %r0, i32 %2, i32 1
+  %r2 = insertelement <4 x i32> %r1, i32 %3, i32 2
+  %r3 = insertelement <4 x i32> %r2, i32 %4, i32 3
+  ret <4 x i32> %r3
+}
+
+define <8 x i8> @slp_v8i8_Op1_Op2_unknown(<8 x i8> %a, <8 x i8> %b)
+; NO-SVE-LABEL: define <8 x i8> @slp_v8i8_Op1_Op2_unknown(
+; NO-SVE-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
+; NO-SVE-NEXT:    [[A0:%.*]] = extractelement <8 x i8> [[A]], i32 0
+; NO-SVE-NEXT:    [[A1:%.*]] = extractelement <8 x i8> [[A]], i32 1
+; NO-SVE-NEXT:    [[A2:%.*]] = extractelement <8 x i8> [[A]], i32 2
+; NO-SVE-NEXT:    [[A3:%.*]] = extractelement <8 x i8> [[A]], i32 3
+; NO-SVE-NEXT:    [[A4:%.*]] = extractelement <8 x i8> [[A]], i32 4
+; NO-SVE-NEXT:    [[A5:%.*]] = extractelement <8 x i8> [[A]], i32 5
+; NO-SVE-NEXT:    [[A6:%.*]] = extractelement <8 x i8> [[A]], i32 6
+; NO-SVE-NEXT:    [[A7:%.*]] = extractelement <8 x i8> [[A]], i32 7
+; NO-SVE-NEXT:    [[B0:%.*]] = extractelement <8 x i8> [[B]], i32 0
+; NO-SVE-NEXT:    [[B1:%.*]] = extractelement <8 x i8> [[B]], i32 1
+; NO-SVE-NEXT:    [[B2:%.*]] = extractelement <8 x i8> [[B]], i32 2
+; NO-SVE-NEXT:    [[B3:%.*]] = extractelement <8 x i8> [[B]], i32 3
+; NO-SVE-NEXT:    [[B4:%.*]] = extractelement <8 x i8> [[B]], i32 4
+; NO-SVE-NEXT:    [[B5:%.*]] = extractelement <8 x i8> [[B]], i32 5
+; NO-SVE-NEXT:    [[B6:%.*]] = extractelement <8 x i8> [[B]], i32 6
+; NO-SVE-NEXT:    [[B7:%.*]] = extractelement <8 x i8> [[B]], i32 7
+; NO-SVE-NEXT:    [[TMP1:%.*]] = sdiv i8 [[A0]], [[B0]]
+; NO-SVE-NEXT:    [[TMP2:%.*]] = sdiv i8 [[A1]], [[B1]]
+; NO-SVE-NEXT:    [[TMP3:%.*]] = sdiv i8 [[A2]], [[B2]]
+; NO-SVE-NEXT:    [[TMP4:%.*]] = sdiv i8 [[A3]], [[B3]]
+; NO-SVE-NEXT:    [[TMP5:%.*]] = sdiv i8 [[A4]], [[B4]]
+; NO-SVE-NEXT:    [[TMP6:%.*]] = sdiv i8 [[A5]], [[B5]]
+; NO-SVE-NEXT:    [[TMP7:%.*]] = sdiv i8 [[A6]], [[B6]]
+; NO-SVE-NEXT:    [[TMP8:%.*]] = sdiv i8 [[A7]], [[B7]]
+; NO-SVE-NEXT:    [[R0:%.*]] = insertelement <8 x i8> poison, i8 [[TMP1]], i32 0
+; NO-SVE-NEXT:    [[R1:%.*]] = insertelement <8 x i8> [[R0]], i8 [[TMP2]], i32 1
+; NO-SVE-NEXT:    [[R2:%.*]] = insertelement <8 x i8> [[R1]], i8 [[TMP3]], i32 2
+; NO-SVE-NEXT:    [[R3:%.*]] = insertelement <8 x i8> [[R2]], i8 [[TMP4]], i32 3
+; NO-SVE-NEXT:    [[R4:%.*]] = insertelement <8 x i8> [[R3]], i8 [[TMP5]], i32 4
+; NO-SVE-NEXT:    [[R5:%.*]] = insertelement <8 x i8> [[R4]], i8 [[TMP6]], i32 5
+; NO-SVE-NEXT:    [[R6:%.*]] = insertelement <8 x i8> [[R5]], i8 [[TMP7]], i32 6
+; NO-SVE-NEXT:    [[R7:%.*]] = insertelement <8 x i8> [[R6]], i8 [[TMP8]], i32 7
+; NO-SVE-NEXT:    ret <8 x i8> [[R3]]
+;
+; SVE-LABEL: define <8 x i8> @slp_v8i8_Op1_Op2_unknown(
+; SVE-SAME: <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
+; SVE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SVE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i8> [[B]], <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SVE-NEXT:    [[TMP3:%.*]] = sdiv <4 x i8> [[TMP1]], [[TMP2]]
+; SVE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SVE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i8> [[B]], <8 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SVE-NEXT:    [[TMP6:%.*]] = sdiv <4 x i8> [[TMP4]], [[TMP5]]
+; SVE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SVE-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i8> [[TMP6]], <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SVE-NEXT:    [[R71:%.*]] = shufflevector <8 x i8> [[TMP7]], <8 x i8> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SVE-NEXT:    ret <8 x i8> [[TMP7]]
+;
+{
+  %a0 = extractelement <8 x i8> %a, i32 0
+  %a1 = extractelement <8 x i8> %a, i32 1
+  %a2 = extractelement <8 x i8> %a, i32 2
+  %a3 = extractelement <8 x i8> %a, i32 3
+  %a4 = extractelement <8 x i8> %a, i32 4
+  %a5 = extractelement <8 x i8> %a, i32 5
+  %a6 = extractelement <8 x i8> %a, i32 6
+  %a7 = extractelement <8 x i8> %a, i32 7
+  %b0 = extractelement <8 x i8> %b, i32 0
+  %b1 = extractelement <8 x i8> %b, i32 1
+  %b2 = extractelement <8 x i8> %b, i32 2
+  %b3 = extractelement <8 x i8> %b, i32 3
+  %b4 = extractelement <8 x i8> %b, i32 4
+  %b5 = extractelement <8 x i8> %b, i32 5
+  %b6 = extractelement <8 x i8> %b, i32 6
+  %b7 = extractelement <8 x i8> %b, i32 7
+  %1 = sdiv i8 %a0, %b0
+  %2 = sdiv i8 %a1, %b1
+  %3 = sdiv i8 %a2, %b2
+  %4 = sdiv i8 %a3, %b3
+  %5 = sdiv i8 %a4, %b4
+  %6 = sdiv i8 %a5, %b5
+  %7 = sdiv i8 %a6, %b6
+  %8 = sdiv i8 %a7, %b7
+  %r0 = insertelement <8 x i8> poison, i8 %1, i32 0
+  %r1 = insertelement <8 x i8> %r0, i8 %2, i32 1
+  %r2 = insertelement <8 x i8> %r1, i8 %3, i32 2
+  %r3 = insertelement <8 x i8> %r2, i8 %4, i32 3
+  %r4 = insertelement <8 x i8> %r3, i8 %5, i32 4
+  %r5 = insertelement <8 x i8> %r4, i8 %6, i32 5
+  %r6 = insertelement <8 x i8> %r5, i8 %7, i32 6
+  %r7 = insertelement <8 x i8> %r6, i8 %8, i32 7
+  ret <8 x i8> %r3
+}
+
+define <8 x i16> @slp_v8i16_Op1_Op2_unknown(<8 x i16> %a, <8 x i16> %b)
+; NO-SVE-LABEL: define <8 x i16> @slp_v8i16_Op1_Op2_unknown(
+; NO-SVE-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]]) #[[ATTR0]] {
+; NO-SVE-NEXT:    [[A0:%.*]] = extractelement <8 x i16> [[A]], i32 0
+; NO-SVE-NEXT:    [[A1:%.*]] = extractelement <8 x i16> [[A]], i32 1
+; NO-SVE-NEXT:    [[A2:%.*]] = extractelement <8 x i16> [[A]], i32 2
+; NO-SVE-NEXT:    [[A3:%.*]] = extractelement <8 x i16> [[A]], i32 3
+; NO-SVE-NEXT:    [[A4:%.*]] = extractelement <8 x i16> [[A]], i32 4
+; NO-SVE-NEXT:    [[A5:%.*]] = extractelement <8 x i16> [[A]], i32 5
+; NO-SVE-NEXT:    [[A6:%.*]] = extractelement <8 x i16> [[A]], i32 6
+; NO-SVE-NEXT:    [[A7:%.*]] = extractelement <8 x i16> [[A]], i32 7
+; NO-SVE-NEXT:    [[B0:%.*]] = extractelement <8 x i16> [[B]], i32 0
+; NO-SVE-NEXT:    [[B1:%.*]] = extractelement <8 x i16> [[B]], i32 1
+; NO-SVE-NEXT:    [[B2:%.*]] = extractelement <8 x i16> [[B]], i32 2
+; NO-SVE-NEXT:    [[B3:%.*]] = extractelement <8 x i16> [[B]], i32 3
+; NO-SVE-NEXT:    [[B4:%.*]] = extractelement <8 x i16> [[B]], i32 4
+; NO-SVE-NEXT:    [[B5:%.*]] = extractelement <8 x i16> [[B]], i32 5
+; NO-SVE-NEXT:    [[B6:%.*]] = extractelement <8 x i16> [[B]], i32 6
+; NO-SVE-NEXT:    [[B7:%.*]] = extractelement <8 x i16> [[B]], i32 7
+; NO-SVE-NEXT:    [[TMP1:%.*]] = sdiv i16 [[A0]], [[B0]]
+; NO-SVE-NEXT:    [[TMP2:%.*]] = sdiv i16 [[A1]], [[B1]]
+; NO-SVE-NEXT:    [[TMP3:%.*]] = sdiv i16 [[A2]], [[B2]]
+; NO-SVE-NEXT:    [[TMP4:%.*]] = sdiv i16 [[A3]], [[B3]]
+; NO-SVE-NEXT:    [[TMP5:%.*]] = sdiv i16 [[A4]], [[B4]]
+; NO-SVE-NEXT:    [[TMP6:%.*]] = sdiv i16 [[A5]], [[B5]]
+; NO-SVE-NEXT:    [[TMP7:%.*]] = sdiv i16 [[A6]], [[B6]]
+; NO-SVE-NEXT:    [[TMP8:%.*]] = sdiv i16 [[A7]], [[B7]]
+; NO-SVE-NEXT:    [[R0:%.*]] = insertelement <8 x i16> poison, i16 [[TMP1]], i32 0
+; NO-SVE-NEXT:    [[R1:%.*]] = insertelement <8 x i16> [[R0]], i16 [[TMP2]], i32 1
+; NO-SVE-NEXT:    [[R2:%.*]] = insertelement <8 x i16> [[R1]], i16 [[TMP3]], i32 2
+; NO-SVE-NEXT:    [[R3:%.*]] = insertelement <8 x i16> [[R2]], i16 [[TMP4]], i32 3
+; NO-SVE-NEXT:    [[R4:%.*]] = insertelement <8 x i16> [[R3]], i16 [[TMP5]], i32 4
+; NO-SVE-NEXT:    [[R5:%.*]] = insertelement <8 x i16> [[R4]], i16 [[TMP6]], i32 5
+; NO-SVE-NEXT:    [[R6:%.*]] = insertelement <8 x i16> [[R5]], i16 [[TMP7]], i32 6
+; NO-SVE-NEXT:    [[R7:%.*]] = insertelement <8 x i16> [[R6]], i16 [[TMP8]], i32 7
+; NO-SVE-NEXT:    ret <8 x i16> [[R3]]
+;
+; SVE-LABEL: define <8 x i16> @slp_v8i16_Op1_Op2_unknown(
+; SVE-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]]) #[[ATTR0]] {
+; SVE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SVE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SVE-NEXT:    [[TMP3:%.*]] = sdiv <4 x i16> [[TMP1]], [[TMP2]]
+; SVE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SVE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SVE-NEXT:    [[TMP6:%.*]] = sdiv <4 x i16> [[TMP4]], [[TMP5]]
+; SVE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i16> [[TMP3]], <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SVE-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i16> [[TMP6]], <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SVE-NEXT:    [[R71:%.*]] = shufflevector <8 x i16> [[TMP7]], <8 x i16> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SVE-NEXT:    ret <8 x i16> [[TMP7]]
+;
+{
+  %a0 = extractelement <8 x i16> %a, i32 0
+  %a1 = extractelement <8 x i16> %a, i32 1
+  %a2 = extractelement <8 x i16> %a, i32 2
+  %a3 = extractelement <8 x i16> %a, i32 3
+  %a4 = extractelement <8 x i16> %a, i32 4
+  %a5 = extractelement <8 x i16> %a, i32 5
+  %a6 = extractelement <8 x i16> %a, i32 6
+  %a7 = extractelement <8 x i16> %a, i32 7
+  %b0 = extractelement <8 x i16> %b, i32 0
+  %b1 = extractelement <8 x i16> %b, i32 1
+  %b2 = extractelement <8 x i16> %b, i32 2
+  %b3 = extractelement <8 x i16> %b, i32 3
+  %b4 = extractelement <8 x i16> %b, i32 4
+  %b5 = extractelement <8 x i16> %b, i32 5
+  %b6 = extractelement <8 x i16> %b, i32 6
+  %b7 = extractelement <8 x i16> %b, i32 7
+  %1 = sdiv i16 %a0, %b0
+  %2 = sdiv i16 %a1, %b1
+  %3 = sdiv i16 %a2, %b2
+  %4 = sdiv i16 %a3, %b3
+  %5 = sdiv i16 %a4, %b4
+  %6 = sdiv i16 %a5, %b5
+  %7 = sdiv i16 %a6, %b6
+  %8 = sdiv i16 %a7, %b7
+  %r0 = insertelement <8 x i16> poison, i16 %1, i32 0
+  %r1 = insertelement <8 x i16> %r0, i16 %2, i32 1
+  %r2 = insertelement <8 x i16> %r1, i16 %3, i32 2
+  %r3 = insertelement <8 x i16> %r2, i16 %4, i32 3
+  %r4 = insertelement <8 x i16> %r3, i16 %5, i32 4
+  %r5 = insertelement <8 x i16> %r4, i16 %6, i32 5
+  %r6 = insertelement <8 x i16> %r5, i16 %7, i32 6
+  %r7 = insertelement <8 x i16> %r6, i16 %8, i32 7
+  ret <8 x i16> %r3
+}
+
+define <4 x i32> @slp_v4i32_Op1_unknown_Op2_const(<4 x i32> %a)
+; NO-SVE-LABEL: define <4 x i32> @slp_v4i32_Op1_unknown_Op2_const(
+; NO-SVE-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; NO-SVE-NEXT:    [[A0:%.*]] = extractelement <4 x i32> [[A]], i32 0
+; NO-SVE-NEXT:    [[A1:%.*]] = extractelement <4 x i32> [[A]], i32 1
+; NO-SVE-NEXT:    [[A2:%.*]] = extractelement <4 x i32> [[A]], i32 2
+; NO-SVE-NEXT:    [[A3:%.*]] = extractelement <4 x i32> [[A]], i32 3
+; NO-SVE-NEXT:    [[TMP1:%.*]] = sdiv i32 [[A0]], 1
+; NO-SVE-NEXT:    [[TMP2:%.*]] = sdiv i32 [[A1]], 3
+; NO-SVE-NEXT:    [[TMP3:%.*]] = sdiv i32 [[A2]], 5
+; NO-SVE-NEXT:    [[TMP4:%.*]] = sdiv i32 [[A3]], 7
+; NO-SVE-NEXT:    [[R0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i32 0
+; NO-SVE-NEXT:    [[R1:%.*]] = insertelement <4 x i32> [[R0]], i32 [[TMP2]], i32 1
+; NO-SVE-NEXT:    [[R2:%.*]] = insertelement <4 x i32> [[R1]], i32 [[TMP3]], i32 2
+; NO-SVE-NEXT:    [[R3:%.*]] = insertelement <4 x i32> [[R2]], i32 [[TMP4]], i32 3
+; NO-SVE-NEXT:    ret <4 x i32> [[R3]]
+;
+; SVE-LABEL: define <4 x i32> @slp_v4i32_Op1_unknown_Op2_const(
+; SVE-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; SVE-NEXT:    [[TMP1:%.*]] = sdiv <4 x i32> [[A]], <i32 1, i32 3, i32 5, i32 7>
+; SVE-NEXT:    ret <4 x i32> [[TMP1]]
+;
+{
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %1 = sdiv i32 %a0, 1
+  %2 = sdiv i32 %a1, 3
+  %3 = sdiv i32 %a2, 5
+  %4 = sdiv i32 %a3, 7
+  %r0 = insertelement <4 x i32> poison, i32 %1, i32 0
+  %r1 = insertelement <4 x i32> %r0, i32 %2, i32 1
+  %r2 = insertelement <4 x i32> %r1, i32 %3, i32 2
+  %r3 = insertelement <4 x i32> %r2, i32 %4, i32 3
+  ret <4 x i32> %r3
+}
+
+define <4 x i32> @slp_v4i32_Op1_unknown_Op2_uniform_const(<4 x i32> %a)
+; CHECK-LABEL: define <4 x i32> @slp_v4i32_Op1_unknown_Op2_uniform_const(
+; CHECK-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv <4 x i32> [[A]], <i32 5, i32 5, i32 5, i32 5>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+{
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %1 = sdiv i32 %a0, 5
+  %2 = sdiv i32 %a1, 5
+  %3 = sdiv i32 %a2, 5
+  %4 = sdiv i32 %a3, 5
+  %r0 = insertelement <4 x i32> poison, i32 %1, i32 0
+  %r1 = insertelement <4 x i32> %r0, i32 %2, i32 1
+  %r2 = insertelement <4 x i32> %r1, i32 %3, i32 2
+  %r3 = insertelement <4 x i32> %r2, i32 %4, i32 3
+  ret <4 x i32> %r3
+}
+
+define <4 x i32> @slp_v4i32_Op1_unknown_Op2_uniform_const_pow2(<4 x i32> %a)
+; CHECK-LABEL: define <4 x i32> @slp_v4i32_Op1_unknown_Op2_uniform_const_pow2(
+; CHECK-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv <4 x i32> [[A]], <i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+{
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %1 = sdiv i32 %a0, 4
+  %2 = sdiv i32 %a1, 4
+  %3 = sdiv i32 %a2, 4
+  %4 = sdiv i32 %a3, 4
+  %r0 = insertelement <4 x i32> poison, i32 %1, i32 0
+  %r1 = insertelement <4 x i32> %r0, i32 %2, i32 1
+  %r2 = insertelement <4 x i32> %r1, i32 %3, i32 2
+  %r3 = insertelement <4 x i32> %r2, i32 %4, i32 3
+  ret <4 x i32> %r3
+}
+
+define <4 x i32> @slp_v4i32_Op1_unknown_Op2_const_pow2(<4 x i32> %a)
+; NO-SVE-LABEL: define <4 x i32> @slp_v4i32_Op1_unknown_Op2_const_pow2(
+; NO-SVE-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; NO-SVE-NEXT:    [[A0:%.*]] = extractelement <4 x i32> [[A]], i32 0
+; NO-SVE-NEXT:    [[A1:%.*]] = extractelement <4 x i32> [[A]], i32 1
+; NO-SVE-NEXT:    [[A2:%.*]] = extractelement <4 x i32> [[A]], i32 2
+; NO-SVE-NEXT:    [[A3:%.*]] = extractelement <4 x i32> [[A]], i32 3
+; NO-SVE-NEXT:    [[TMP1:%.*]] = sdiv i32 [[A0]], 1
+; NO-SVE-NEXT:    [[TMP2:%.*]] = sdiv i32 [[A1]], 2
+; NO-SVE-NEXT:    [[TMP3:%.*]] = sdiv i32 [[A2]], 4
+; NO-SVE-NEXT:    [[TMP4:%.*]] = sdiv i32 [[A3]], 8
+; NO-SVE-NEXT:    [[R0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i32 0
+; NO-SVE-NEXT:    [[R1:%.*]] = insertelement <4 x i32> [[R0]], i32 [[TMP2]], i32 1
+; NO-SVE-NEXT:    [[R2:%.*]] = insertelement <4 x i32> [[R1]], i32 [[TMP3]], i32 2
+; NO-SVE-NEXT:    [[R3:%.*]] = insertelement <4 x i32> [[R2]], i32 [[TMP4]], i32 3
+; NO-SVE-NEXT:    ret <4 x i32> [[R3]]
+;
+; SVE-LABEL: define <4 x i32> @slp_v4i32_Op1_unknown_Op2_const_pow2(
+; SVE-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; SVE-NEXT:    [[TMP1:%.*]] = sdiv <4 x i32> [[A]], <i32 1, i32 2, i32 4, i32 8>
+; SVE-NEXT:    ret <4 x i32> [[TMP1]]
+;
+{
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %1 = sdiv i32 %a0, 1
+  %2 = sdiv i32 %a1, 2
+  %3 = sdiv i32 %a2, 4
+  %4 = sdiv i32 %a3, 8
+  %r0 = insertelement <4 x i32> poison, i32 %1, i32 0
+  %r1 = insertelement <4 x i32> %r0, i32 %2, i32 1
+  %r2 = insertelement <4 x i32> %r1, i32 %3, i32 2
+  %r3 = insertelement <4 x i32> %r2, i32 %4, i32 3
+  ret <4 x i32> %r3
+}
-- 
GitLab


From 3ac75ee8ecbe3bb1d1907adf48731bb6dc2c9918 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Mon, 28 Oct 2024 16:25:58 +0100
Subject: [PATCH 325/693] Revert "[runtimes] Allow building against an
 installed LLVM tree" (#113920)

Reverts llvm/llvm-project#86209

This patch breaks running tests locally, which is extremely disruptive
to libc++ development.
---
 compiler-rt/cmake/Modules/AddCompilerRT.cmake |  1 -
 compiler-rt/test/hwasan/lit.cfg.py            |  9 ------
 compiler-rt/test/lit.common.configured.in     |  1 -
 runtimes/CMakeLists.txt                       | 29 +++++++------------
 4 files changed, 10 insertions(+), 30 deletions(-)

diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
index b2f33d1a961c..e3d81d241b10 100644
--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake
+++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
@@ -773,7 +773,6 @@ function(configure_compiler_rt_lit_site_cfg input output)
 
   string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} COMPILER_RT_RESOLVED_TEST_COMPILER ${COMPILER_RT_TEST_COMPILER})
   string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} COMPILER_RT_RESOLVED_OUTPUT_DIR ${COMPILER_RT_OUTPUT_DIR})
-  string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} COMPILER_RT_RESOLVED_EXEC_OUTPUT_DIR ${COMPILER_RT_EXEC_OUTPUT_DIR})
   string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR ${output_dir})
 
   configure_lit_site_cfg(${input} ${output})
diff --git a/compiler-rt/test/hwasan/lit.cfg.py b/compiler-rt/test/hwasan/lit.cfg.py
index bbf23e683240..594f3294a84a 100644
--- a/compiler-rt/test/hwasan/lit.cfg.py
+++ b/compiler-rt/test/hwasan/lit.cfg.py
@@ -2,9 +2,6 @@
 
 import os
 
-from lit.llvm import llvm_config
-from lit.llvm.subst import ToolSubst, FindTool
-
 # Setup config name.
 config.name = "HWAddressSanitizer" + getattr(config, "name_suffix", "default")
 
@@ -77,12 +74,6 @@ config.substitutions.append(
     ("%env_hwasan_opts=", "env HWASAN_OPTIONS=" + default_hwasan_opts_str)
 )
 
-# Ensure that we can use hwasan_symbolize from the expected location
-llvm_config.add_tool_substitutions(
-    [ToolSubst("hwasan_symbolize", unresolved="fatal")],
-    search_dirs=[config.compiler_rt_bindir],
-)
-
 # Default test suffixes.
 config.suffixes = [".c", ".cpp"]
 
diff --git a/compiler-rt/test/lit.common.configured.in b/compiler-rt/test/lit.common.configured.in
index 050792b6b262..66935c358afe 100644
--- a/compiler-rt/test/lit.common.configured.in
+++ b/compiler-rt/test/lit.common.configured.in
@@ -28,7 +28,6 @@ set_default("python_executable", "@Python3_EXECUTABLE@")
 set_default("compiler_rt_debug", @COMPILER_RT_DEBUG_PYBOOL@)
 set_default("compiler_rt_intercept_libdispatch", @COMPILER_RT_INTERCEPT_LIBDISPATCH_PYBOOL@)
 set_default("compiler_rt_output_dir", "@COMPILER_RT_RESOLVED_OUTPUT_DIR@")
-set_default("compiler_rt_bindir", "@COMPILER_RT_RESOLVED_EXEC_OUTPUT_DIR@")
 set_default("compiler_rt_libdir", "@COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR@")
 set_default("emulator", "@COMPILER_RT_EMULATOR@")
 set_default("asan_shadow_scale", "@COMPILER_RT_ASAN_SHADOW_SCALE@")
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 67368dcedb3b..830165c799c2 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -239,23 +239,6 @@ foreach(entry ${runtimes})
 endforeach()
 
 if(LLVM_INCLUDE_TESTS)
-  # If built with the runtimes build (rooted at runtimes/CMakeLists.txt), we
-  # won't have llvm-lit. If built with the bootstrapping build (rooted at
-  # llvm/CMakeLists.txt), the top-level llvm CMake invocation already generated
-  # the llvm-lit script.
-  if (NOT HAVE_LLVM_LIT)
-    # Add lit before adding any runtimes since their CMake tests configuration
-    # might depend on lit being present.
-    set(LLVM_LIT_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/bin)
-    add_subdirectory(${LLVM_MAIN_SRC_DIR}/utils/llvm-lit
-                     ${CMAKE_CURRENT_BINARY_DIR}/llvm-lit)
-    # Ensure that the testsuites use the local lit rather than
-    # ${LLVM_INSTALL_DIR}/bin/llvm-lit (which may not exist if LLVM_BINARY_DIR
-    # points at an installed LLVM tree rather than a build tree).
-    get_llvm_lit_path(_base_dir _file_name)
-    set(LLVM_EXTERNAL_LIT "${_base_dir}/${_file_name}" CACHE STRING "Command used to spawn lit" FORCE)
-  endif()
-
   set(LIT_ARGS_DEFAULT "-sv --show-xfail --show-unsupported")
   if (MSVC OR XCODE)
     set(LIT_ARGS_DEFAULT "${LIT_ARGS_DEFAULT} --no-progress-bar")
@@ -289,6 +272,14 @@ if(LLVM_INCLUDE_TESTS)
   # and we know the total set of lit testsuites.
   umbrella_lit_testsuite_end(check-runtimes)
 
+  if (NOT HAVE_LLVM_LIT)
+    # If built by manually invoking cmake on this directory, we don't have
+    # llvm-lit. If invoked via llvm/runtimes, the toplevel llvm cmake
+    # invocation already generated the llvm-lit script.
+    add_subdirectory(${LLVM_MAIN_SRC_DIR}/utils/llvm-lit
+                     ${CMAKE_CURRENT_BINARY_DIR}/llvm-lit)
+  endif()
+
   get_property(LLVM_RUNTIMES_LIT_TESTSUITES GLOBAL PROPERTY LLVM_RUNTIMES_LIT_TESTSUITES)
   string(REPLACE ";" "\n" LLVM_RUNTIMES_LIT_TESTSUITES "${LLVM_RUNTIMES_LIT_TESTSUITES}")
   file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/lit.tests ${LLVM_RUNTIMES_LIT_TESTSUITES})
@@ -318,10 +309,10 @@ if(SUB_COMPONENTS)
   if(LLVM_RUNTIMES_TARGET)
     configure_file(
       ${CMAKE_CURRENT_SOURCE_DIR}/Components.cmake.in
-      ${CMAKE_CURRENT_BINARY_DIR}/runtimes/${LLVM_RUNTIMES_TARGET}/Components.cmake)
+      ${LLVM_BINARY_DIR}/runtimes/${LLVM_RUNTIMES_TARGET}/Components.cmake)
   else()
     configure_file(
       ${CMAKE_CURRENT_SOURCE_DIR}/Components.cmake.in
-      ${CMAKE_CURRENT_BINARY_DIR}/runtimes/Components.cmake)
+      ${LLVM_BINARY_DIR}/runtimes/Components.cmake)
   endif()
 endif()
-- 
GitLab


From 7d1e98c7d34e2f4be1812a2f00f1d0464f29e542 Mon Sep 17 00:00:00 2001
From: Abhina Sree <Abhina.Sreeskantharajan@ibm.com>
Date: Mon, 28 Oct 2024 11:34:57 -0400
Subject: [PATCH 326/693] [AIX][SystemZ][z/OS] Disable test for AIX, z/OS due
 to missing DWARF sections (#113910)

This patch disables the testcase for AIX and z/OS due to incomplete
DWARF support.
---
 llvm/test/DebugInfo/Generic/debug-names-accel-table-types.ll | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/test/DebugInfo/Generic/debug-names-accel-table-types.ll b/llvm/test/DebugInfo/Generic/debug-names-accel-table-types.ll
index e88afe1b4c51..2a6c89dcab59 100644
--- a/llvm/test/DebugInfo/Generic/debug-names-accel-table-types.ll
+++ b/llvm/test/DebugInfo/Generic/debug-names-accel-table-types.ll
@@ -1,3 +1,4 @@
+; XFAIL: target={{.*}}-aix{{.*}}, target={{.*}}-zos{{.*}}
 ; RUN: %llc_dwarf -debugger-tune=lldb -accel-tables=Dwarf -filetype=obj -o %t < %s
 ; RUN: llvm-dwarfdump %t | FileCheck %s
 ; RUN: llvm-dwarfdump -debug-names %t | FileCheck --check-prefix=SAME-NAME %s
-- 
GitLab


From 53f7f8eccabd6e3383edfeec312bf8671a89bc66 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Mon, 28 Oct 2024 15:43:14 +0000
Subject: [PATCH 327/693] [Clang][AArch64] Fix Pure Scalables Types argument
 passing and return (#112747)

Pure Scalable Types are defined in AAPCS64 here:

https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#pure-scalable-types-psts

And should be passed according to Rule C.7 here:

https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#682parameter-passing-rules

This part of the ABI is completely unimplemented in Clang, instead it
treats PSTs sometimes as HFAs/HVAs, sometime as general composite types.

This patch implements the rules for passing PSTs by employing the
`CoerceAndExpand` method and extending it to:
* allow array types in the `coerceToType`; Now only `[N x i8]` are
considered padding.
* allow mismatch between the elements of the `coerceToType` and the
elements of the `unpaddedCoerceToType`; AArch64 uses this to map
fixed-length vector types to SVE vector types.

Corectly passing a PST argument needs a decision in Clang about whether
to pass it in memory or registers or, equivalently, whether to use the
`Indirect` or `Expand/CoerceAndExpand` method. It was considered
relatively harder (or not practically possible) to make that decision in
the AArch64 backend.
Hence this patch implements the register counting from AAPCS64 (cf.
`NSRN`, `NPRN`) to guide the Clang's decision.
---
 clang/include/clang/CodeGen/CGFunctionInfo.h  |  16 +-
 clang/lib/CodeGen/CGCall.cpp                  |  85 +++-
 clang/lib/CodeGen/Targets/AArch64.cpp         | 374 ++++++++++++--
 .../aarch64-pure-scalable-args-empty-union.c  |  39 ++
 .../test/CodeGen/aarch64-pure-scalable-args.c | 461 ++++++++++++++++++
 5 files changed, 897 insertions(+), 78 deletions(-)
 create mode 100644 clang/test/CodeGen/aarch64-pure-scalable-args-empty-union.c
 create mode 100644 clang/test/CodeGen/aarch64-pure-scalable-args.c

diff --git a/clang/include/clang/CodeGen/CGFunctionInfo.h b/clang/include/clang/CodeGen/CGFunctionInfo.h
index d19f84d19887..9d785d878b61 100644
--- a/clang/include/clang/CodeGen/CGFunctionInfo.h
+++ b/clang/include/clang/CodeGen/CGFunctionInfo.h
@@ -271,12 +271,8 @@ public:
     // in the unpadded type.
     unsigned unpaddedIndex = 0;
     for (auto eltType : coerceToType->elements()) {
-      if (isPaddingForCoerceAndExpand(eltType)) continue;
-      if (unpaddedStruct) {
-        assert(unpaddedStruct->getElementType(unpaddedIndex) == eltType);
-      } else {
-        assert(unpaddedIndex == 0 && unpaddedCoerceToType == eltType);
-      }
+      if (isPaddingForCoerceAndExpand(eltType))
+        continue;
       unpaddedIndex++;
     }
 
@@ -295,12 +291,8 @@ public:
   }
 
   static bool isPaddingForCoerceAndExpand(llvm::Type *eltType) {
-    if (eltType->isArrayTy()) {
-      assert(eltType->getArrayElementType()->isIntegerTy(8));
-      return true;
-    } else {
-      return false;
-    }
+    return eltType->isArrayTy() &&
+           eltType->getArrayElementType()->isIntegerTy(8);
   }
 
   Kind getKind() const { return TheKind; }
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 1949b4ceb7f2..64e60f0616d7 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -1410,6 +1410,30 @@ static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
   return addr;
 }
 
+static std::pair<llvm::Value *, bool>
+CoerceScalableToFixed(CodeGenFunction &CGF, llvm::FixedVectorType *ToTy,
+                      llvm::ScalableVectorType *FromTy, llvm::Value *V,
+                      StringRef Name = "") {
+  // If we are casting a scalable i1 predicate vector to a fixed i8
+  // vector, first bitcast the source.
+  if (FromTy->getElementType()->isIntegerTy(1) &&
+      FromTy->getElementCount().isKnownMultipleOf(8) &&
+      ToTy->getElementType() == CGF.Builder.getInt8Ty()) {
+    FromTy = llvm::ScalableVectorType::get(
+        ToTy->getElementType(),
+        FromTy->getElementCount().getKnownMinValue() / 8);
+    V = CGF.Builder.CreateBitCast(V, FromTy);
+  }
+  if (FromTy->getElementType() == ToTy->getElementType()) {
+    llvm::Value *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty);
+
+    V->setName(Name + ".coerce");
+    V = CGF.Builder.CreateExtractVector(ToTy, V, Zero, "cast.fixed");
+    return {V, true};
+  }
+  return {V, false};
+}
+
 namespace {
 
 /// Encapsulates information about the way function arguments from
@@ -3196,26 +3220,14 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
       // a VLAT at the function boundary and the types match up, use
       // llvm.vector.extract to convert back to the original VLST.
       if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(ConvertType(Ty))) {
-        llvm::Value *Coerced = Fn->getArg(FirstIRArg);
+        llvm::Value *ArgVal = Fn->getArg(FirstIRArg);
         if (auto *VecTyFrom =
-                dyn_cast<llvm::ScalableVectorType>(Coerced->getType())) {
-          // If we are casting a scalable i1 predicate vector to a fixed i8
-          // vector, bitcast the source and use a vector extract.
-          if (VecTyFrom->getElementType()->isIntegerTy(1) &&
-              VecTyFrom->getElementCount().isKnownMultipleOf(8) &&
-              VecTyTo->getElementType() == Builder.getInt8Ty()) {
-            VecTyFrom = llvm::ScalableVectorType::get(
-                VecTyTo->getElementType(),
-                VecTyFrom->getElementCount().getKnownMinValue() / 8);
-            Coerced = Builder.CreateBitCast(Coerced, VecTyFrom);
-          }
-          if (VecTyFrom->getElementType() == VecTyTo->getElementType()) {
-            llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty);
-
+                dyn_cast<llvm::ScalableVectorType>(ArgVal->getType())) {
+          auto [Coerced, Extracted] = CoerceScalableToFixed(
+              *this, VecTyTo, VecTyFrom, ArgVal, Arg->getName());
+          if (Extracted) {
             assert(NumIRArgs == 1);
-            Coerced->setName(Arg->getName() + ".coerce");
-            ArgVals.push_back(ParamValue::forDirect(Builder.CreateExtractVector(
-                VecTyTo, Coerced, Zero, "cast.fixed")));
+            ArgVals.push_back(ParamValue::forDirect(Coerced));
             break;
           }
         }
@@ -3326,16 +3338,33 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
       ArgVals.push_back(ParamValue::forIndirect(alloca));
 
       auto coercionType = ArgI.getCoerceAndExpandType();
+      auto unpaddedCoercionType = ArgI.getUnpaddedCoerceAndExpandType();
+      auto *unpaddedStruct = dyn_cast<llvm::StructType>(unpaddedCoercionType);
+
       alloca = alloca.withElementType(coercionType);
 
       unsigned argIndex = FirstIRArg;
+      unsigned unpaddedIndex = 0;
       for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
         llvm::Type *eltType = coercionType->getElementType(i);
         if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
           continue;
 
         auto eltAddr = Builder.CreateStructGEP(alloca, i);
-        auto elt = Fn->getArg(argIndex++);
+        llvm::Value *elt = Fn->getArg(argIndex++);
+
+        auto paramType = unpaddedStruct
+                             ? unpaddedStruct->getElementType(unpaddedIndex++)
+                             : unpaddedCoercionType;
+
+        if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(eltType)) {
+          if (auto *VecTyFrom = dyn_cast<llvm::ScalableVectorType>(paramType)) {
+            bool Extracted;
+            std::tie(elt, Extracted) = CoerceScalableToFixed(
+                *this, VecTyTo, VecTyFrom, elt, elt->getName());
+            assert(Extracted && "Unexpected scalable to fixed vector coercion");
+          }
+        }
         Builder.CreateStore(elt, eltAddr);
       }
       assert(argIndex == FirstIRArg + NumIRArgs);
@@ -3930,17 +3959,24 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
 
   case ABIArgInfo::CoerceAndExpand: {
     auto coercionType = RetAI.getCoerceAndExpandType();
+    auto unpaddedCoercionType = RetAI.getUnpaddedCoerceAndExpandType();
+    auto *unpaddedStruct = dyn_cast<llvm::StructType>(unpaddedCoercionType);
 
     // Load all of the coerced elements out into results.
     llvm::SmallVector<llvm::Value*, 4> results;
     Address addr = ReturnValue.withElementType(coercionType);
+    unsigned unpaddedIndex = 0;
     for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
       auto coercedEltType = coercionType->getElementType(i);
       if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType))
         continue;
 
       auto eltAddr = Builder.CreateStructGEP(addr, i);
-      auto elt = Builder.CreateLoad(eltAddr);
+      llvm::Value *elt = CreateCoercedLoad(
+          eltAddr,
+          unpaddedStruct ? unpaddedStruct->getElementType(unpaddedIndex++)
+                         : unpaddedCoercionType,
+          *this);
       results.push_back(elt);
     }
 
@@ -5472,6 +5508,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     case ABIArgInfo::CoerceAndExpand: {
       auto coercionType = ArgInfo.getCoerceAndExpandType();
       auto layout = CGM.getDataLayout().getStructLayout(coercionType);
+      auto unpaddedCoercionType = ArgInfo.getUnpaddedCoerceAndExpandType();
+      auto *unpaddedStruct = dyn_cast<llvm::StructType>(unpaddedCoercionType);
 
       llvm::Value *tempSize = nullptr;
       Address addr = Address::invalid();
@@ -5502,11 +5540,16 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       addr = addr.withElementType(coercionType);
 
       unsigned IRArgPos = FirstIRArg;
+      unsigned unpaddedIndex = 0;
       for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
         llvm::Type *eltType = coercionType->getElementType(i);
         if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) continue;
         Address eltAddr = Builder.CreateStructGEP(addr, i);
-        llvm::Value *elt = Builder.CreateLoad(eltAddr);
+        llvm::Value *elt = CreateCoercedLoad(
+            eltAddr,
+            unpaddedStruct ? unpaddedStruct->getElementType(unpaddedIndex++)
+                           : unpaddedCoercionType,
+            *this);
         if (ArgHasMaybeUndefAttr)
           elt = Builder.CreateFreeze(elt);
         IRCallArgs[IRArgPos++] = elt;
diff --git a/clang/lib/CodeGen/Targets/AArch64.cpp b/clang/lib/CodeGen/Targets/AArch64.cpp
index ec617eec6719..a80411971b60 100644
--- a/clang/lib/CodeGen/Targets/AArch64.cpp
+++ b/clang/lib/CodeGen/Targets/AArch64.cpp
@@ -34,10 +34,17 @@ private:
   AArch64ABIKind getABIKind() const { return Kind; }
   bool isDarwinPCS() const { return Kind == AArch64ABIKind::DarwinPCS; }
 
-  ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadic) const;
-  ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadic,
-                                  unsigned CallingConvention) const;
-  ABIArgInfo coerceIllegalVector(QualType Ty) const;
+  ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadicFn) const;
+  ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadicFn,
+                                  bool IsNamedArg, unsigned CallingConvention,
+                                  unsigned &NSRN, unsigned &NPRN) const;
+  llvm::Type *convertFixedToScalableVectorType(const VectorType *VT) const;
+  ABIArgInfo coerceIllegalVector(QualType Ty, unsigned &NSRN,
+                                 unsigned &NPRN) const;
+  ABIArgInfo coerceAndExpandPureScalableAggregate(
+      QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
+      const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
+      unsigned &NPRN) const;
   bool isHomogeneousAggregateBaseType(QualType Ty) const override;
   bool isHomogeneousAggregateSmallEnough(const Type *Ty,
                                          uint64_t Members) const override;
@@ -45,14 +52,26 @@ private:
 
   bool isIllegalVectorType(QualType Ty) const;
 
+  bool passAsPureScalableType(QualType Ty, unsigned &NV, unsigned &NP,
+                              SmallVectorImpl<llvm::Type *> &CoerceToSeq) const;
+
+  void flattenType(llvm::Type *Ty,
+                   SmallVectorImpl<llvm::Type *> &Flattened) const;
+
   void computeInfo(CGFunctionInfo &FI) const override {
     if (!::classifyReturnType(getCXXABI(), FI, *this))
       FI.getReturnInfo() =
           classifyReturnType(FI.getReturnType(), FI.isVariadic());
 
-    for (auto &it : FI.arguments())
-      it.info = classifyArgumentType(it.type, FI.isVariadic(),
-                                     FI.getCallingConvention());
+    unsigned ArgNo = 0;
+    unsigned NSRN = 0, NPRN = 0;
+    for (auto &it : FI.arguments()) {
+      const bool IsNamedArg =
+          !FI.isVariadic() || ArgNo < FI.getRequiredArgs().getNumRequiredArgs();
+      ++ArgNo;
+      it.info = classifyArgumentType(it.type, FI.isVariadic(), IsNamedArg,
+                                     FI.getCallingConvention(), NSRN, NPRN);
+    }
   }
 
   RValue EmitDarwinVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
@@ -201,65 +220,83 @@ void WindowsAArch64TargetCodeGenInfo::setTargetAttributes(
 }
 }
 
-ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty) const {
-  assert(Ty->isVectorType() && "expected vector type!");
+llvm::Type *
+AArch64ABIInfo::convertFixedToScalableVectorType(const VectorType *VT) const {
+  assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
 
-  const auto *VT = Ty->castAs<VectorType>();
   if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
-    assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
     assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
                BuiltinType::UChar &&
            "unexpected builtin type for SVE predicate!");
-    return ABIArgInfo::getDirect(llvm::ScalableVectorType::get(
-        llvm::Type::getInt1Ty(getVMContext()), 16));
+    return llvm::ScalableVectorType::get(llvm::Type::getInt1Ty(getVMContext()),
+                                         16);
   }
 
   if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
-    assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
-
     const auto *BT = VT->getElementType()->castAs<BuiltinType>();
-    llvm::ScalableVectorType *ResType = nullptr;
     switch (BT->getKind()) {
     default:
       llvm_unreachable("unexpected builtin type for SVE vector!");
+
     case BuiltinType::SChar:
     case BuiltinType::UChar:
-      ResType = llvm::ScalableVectorType::get(
+      return llvm::ScalableVectorType::get(
           llvm::Type::getInt8Ty(getVMContext()), 16);
-      break;
+
     case BuiltinType::Short:
     case BuiltinType::UShort:
-      ResType = llvm::ScalableVectorType::get(
+      return llvm::ScalableVectorType::get(
           llvm::Type::getInt16Ty(getVMContext()), 8);
-      break;
+
     case BuiltinType::Int:
     case BuiltinType::UInt:
-      ResType = llvm::ScalableVectorType::get(
+      return llvm::ScalableVectorType::get(
           llvm::Type::getInt32Ty(getVMContext()), 4);
-      break;
+
     case BuiltinType::Long:
     case BuiltinType::ULong:
-      ResType = llvm::ScalableVectorType::get(
+      return llvm::ScalableVectorType::get(
           llvm::Type::getInt64Ty(getVMContext()), 2);
-      break;
+
     case BuiltinType::Half:
-      ResType = llvm::ScalableVectorType::get(
+      return llvm::ScalableVectorType::get(
           llvm::Type::getHalfTy(getVMContext()), 8);
-      break;
+
     case BuiltinType::Float:
-      ResType = llvm::ScalableVectorType::get(
+      return llvm::ScalableVectorType::get(
           llvm::Type::getFloatTy(getVMContext()), 4);
-      break;
+
     case BuiltinType::Double:
-      ResType = llvm::ScalableVectorType::get(
+      return llvm::ScalableVectorType::get(
           llvm::Type::getDoubleTy(getVMContext()), 2);
-      break;
+
     case BuiltinType::BFloat16:
-      ResType = llvm::ScalableVectorType::get(
+      return llvm::ScalableVectorType::get(
           llvm::Type::getBFloatTy(getVMContext()), 8);
-      break;
     }
-    return ABIArgInfo::getDirect(ResType);
+  }
+
+  llvm_unreachable("expected fixed-length SVE vector");
+}
+
+ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty, unsigned &NSRN,
+                                               unsigned &NPRN) const {
+  assert(Ty->isVectorType() && "expected vector type!");
+
+  const auto *VT = Ty->castAs<VectorType>();
+  if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
+    assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
+    assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
+               BuiltinType::UChar &&
+           "unexpected builtin type for SVE predicate!");
+    NPRN = std::min(NPRN + 1, 4u);
+    return ABIArgInfo::getDirect(llvm::ScalableVectorType::get(
+        llvm::Type::getInt1Ty(getVMContext()), 16));
+  }
+
+  if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
+    NSRN = std::min(NSRN + 1, 8u);
+    return ABIArgInfo::getDirect(convertFixedToScalableVectorType(VT));
   }
 
   uint64_t Size = getContext().getTypeSize(Ty);
@@ -273,26 +310,54 @@ ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty) const {
     return ABIArgInfo::getDirect(ResType);
   }
   if (Size == 64) {
+    NSRN = std::min(NSRN + 1, 8u);
     auto *ResType =
         llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 2);
     return ABIArgInfo::getDirect(ResType);
   }
   if (Size == 128) {
+    NSRN = std::min(NSRN + 1, 8u);
     auto *ResType =
         llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 4);
     return ABIArgInfo::getDirect(ResType);
   }
+
   return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
 }
 
-ABIArgInfo
-AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadic,
-                                     unsigned CallingConvention) const {
+ABIArgInfo AArch64ABIInfo::coerceAndExpandPureScalableAggregate(
+    QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
+    const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
+    unsigned &NPRN) const {
+  if (!IsNamedArg || NSRN + NVec > 8 || NPRN + NPred > 4)
+    return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
+  NSRN += NVec;
+  NPRN += NPred;
+
+  llvm::Type *UnpaddedCoerceToType =
+      UnpaddedCoerceToSeq.size() == 1
+          ? UnpaddedCoerceToSeq[0]
+          : llvm::StructType::get(CGT.getLLVMContext(), UnpaddedCoerceToSeq,
+                                  true);
+
+  SmallVector<llvm::Type *> CoerceToSeq;
+  flattenType(CGT.ConvertType(Ty), CoerceToSeq);
+  auto *CoerceToType =
+      llvm::StructType::get(CGT.getLLVMContext(), CoerceToSeq, false);
+
+  return ABIArgInfo::getCoerceAndExpand(CoerceToType, UnpaddedCoerceToType);
+}
+
+ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn,
+                                                bool IsNamedArg,
+                                                unsigned CallingConvention,
+                                                unsigned &NSRN,
+                                                unsigned &NPRN) const {
   Ty = useFirstFieldIfTransparentUnion(Ty);
 
   // Handle illegal vector types here.
   if (isIllegalVectorType(Ty))
-    return coerceIllegalVector(Ty);
+    return coerceIllegalVector(Ty, NSRN, NPRN);
 
   if (!isAggregateTypeForABI(Ty)) {
     // Treat an enum type as its underlying type.
@@ -303,6 +368,36 @@ AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadic,
       if (EIT->getNumBits() > 128)
         return getNaturalAlignIndirect(Ty, false);
 
+    if (Ty->isVectorType())
+      NSRN = std::min(NSRN + 1, 8u);
+    else if (const auto *BT = Ty->getAs<BuiltinType>()) {
+      if (BT->isFloatingPoint())
+        NSRN = std::min(NSRN + 1, 8u);
+      else {
+        switch (BT->getKind()) {
+        case BuiltinType::MFloat8x8:
+        case BuiltinType::MFloat8x16:
+          NSRN = std::min(NSRN + 1, 8u);
+          break;
+        case BuiltinType::SveBool:
+        case BuiltinType::SveCount:
+          NPRN = std::min(NPRN + 1, 4u);
+          break;
+        case BuiltinType::SveBoolx2:
+          NPRN = std::min(NPRN + 2, 4u);
+          break;
+        case BuiltinType::SveBoolx4:
+          NPRN = std::min(NPRN + 4, 4u);
+          break;
+        default:
+          if (BT->isSVESizelessBuiltinType())
+            NSRN = std::min(
+                NSRN + getContext().getBuiltinVectorTypeInfo(BT).NumVectors,
+                8u);
+        }
+      }
+    }
+
     return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS()
                 ? ABIArgInfo::getExtend(Ty, CGT.ConvertType(Ty))
                 : ABIArgInfo::getDirect());
@@ -335,10 +430,11 @@ AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadic,
   uint64_t Members = 0;
   bool IsWin64 = Kind == AArch64ABIKind::Win64 ||
                  CallingConvention == llvm::CallingConv::Win64;
-  bool IsWinVariadic = IsWin64 && IsVariadic;
+  bool IsWinVariadic = IsWin64 && IsVariadicFn;
   // In variadic functions on Windows, all composite types are treated alike,
   // no special handling of HFAs/HVAs.
   if (!IsWinVariadic && isHomogeneousAggregate(Ty, Base, Members)) {
+    NSRN = std::min(NSRN + Members, uint64_t(8));
     if (Kind != AArch64ABIKind::AAPCS)
       return ABIArgInfo::getDirect(
           llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members));
@@ -353,6 +449,17 @@ AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadic,
         nullptr, true, Align);
   }
 
+  // In AAPCS named arguments of a Pure Scalable Type are passed expanded in
+  // registers, or indirectly if there are not enough registers.
+  if (Kind == AArch64ABIKind::AAPCS) {
+    unsigned NVec = 0, NPred = 0;
+    SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
+    if (passAsPureScalableType(Ty, NVec, NPred, UnpaddedCoerceToSeq) &&
+        (NVec + NPred) > 0)
+      return coerceAndExpandPureScalableAggregate(
+          Ty, IsNamedArg, NVec, NPred, UnpaddedCoerceToSeq, NSRN, NPRN);
+  }
+
   // Aggregates <= 16 bytes are passed directly in registers or on the stack.
   if (Size <= 128) {
     // On RenderScript, coerce Aggregates <= 16 bytes to an integer array of
@@ -383,14 +490,16 @@ AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadic,
 }
 
 ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,
-                                              bool IsVariadic) const {
+                                              bool IsVariadicFn) const {
   if (RetTy->isVoidType())
     return ABIArgInfo::getIgnore();
 
   if (const auto *VT = RetTy->getAs<VectorType>()) {
     if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
-        VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)
-      return coerceIllegalVector(RetTy);
+        VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
+      unsigned NSRN = 0, NPRN = 0;
+      return coerceIllegalVector(RetTy, NSRN, NPRN);
+    }
   }
 
   // Large vector types should be returned via memory.
@@ -419,10 +528,24 @@ ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,
   uint64_t Members = 0;
   if (isHomogeneousAggregate(RetTy, Base, Members) &&
       !(getTarget().getTriple().getArch() == llvm::Triple::aarch64_32 &&
-        IsVariadic))
+        IsVariadicFn))
     // Homogeneous Floating-point Aggregates (HFAs) are returned directly.
     return ABIArgInfo::getDirect();
 
+  // In AAPCS return values of a Pure Scalable type are treated as a single
+  // named argument and passed expanded in registers, or indirectly if there are
+  // not enough registers.
+  if (Kind == AArch64ABIKind::AAPCS) {
+    unsigned NSRN = 0, NPRN = 0;
+    unsigned NVec = 0, NPred = 0;
+    SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
+    if (passAsPureScalableType(RetTy, NVec, NPred, UnpaddedCoerceToSeq) &&
+        (NVec + NPred) > 0)
+      return coerceAndExpandPureScalableAggregate(
+          RetTy, /* IsNamedArg */ true, NVec, NPred, UnpaddedCoerceToSeq, NSRN,
+          NPRN);
+  }
+
   // Aggregates <= 16 bytes are returned directly in registers or on the stack.
   if (Size <= 128) {
     // On RenderScript, coerce Aggregates <= 16 bytes to an integer array of
@@ -508,9 +631,15 @@ bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
   // but with the difference that any floating-point type is allowed,
   // including __fp16.
   if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
-    if (BT->isFloatingPoint())
+    if (BT->isFloatingPoint() || BT->getKind() == BuiltinType::MFloat8x16 ||
+        BT->getKind() == BuiltinType::MFloat8x8)
       return true;
   } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
+    if (auto Kind = VT->getVectorKind();
+        Kind == VectorKind::SveFixedLengthData ||
+        Kind == VectorKind::SveFixedLengthPredicate)
+      return false;
+
     unsigned VecSize = getContext().getTypeSize(VT);
     if (VecSize == 64 || VecSize == 128)
       return true;
@@ -533,11 +662,166 @@ bool AArch64ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate()
   return true;
 }
 
+// Check if a type needs to be passed in registers as a Pure Scalable Type (as
+// defined by AAPCS64). Return the number of data vectors and the number of
+// predicate vectors in the type, into `NVec` and `NPred`, respectively. Upon
+// return `CoerceToSeq` contains an expanded sequence of LLVM IR types, one
+// element for each non-composite member. For practical purposes, limit the
+// length of `CoerceToSeq` to about 12 (the maximum that could possibly fit
+// in registers) and return false, the effect of which will be to  pass the
+// argument under the rules for a large (> 128 bytes) composite.
+bool AArch64ABIInfo::passAsPureScalableType(
+    QualType Ty, unsigned &NVec, unsigned &NPred,
+    SmallVectorImpl<llvm::Type *> &CoerceToSeq) const {
+  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
+    uint64_t NElt = AT->getZExtSize();
+    if (NElt == 0)
+      return false;
+
+    unsigned NV = 0, NP = 0;
+    SmallVector<llvm::Type *> EltCoerceToSeq;
+    if (!passAsPureScalableType(AT->getElementType(), NV, NP, EltCoerceToSeq))
+      return false;
+
+    if (CoerceToSeq.size() + NElt * EltCoerceToSeq.size() > 12)
+      return false;
+
+    for (uint64_t I = 0; I < NElt; ++I)
+      llvm::copy(EltCoerceToSeq, std::back_inserter(CoerceToSeq));
+
+    NVec += NElt * NV;
+    NPred += NElt * NP;
+    return true;
+  }
+
+  if (const RecordType *RT = Ty->getAs<RecordType>()) {
+    // If the record cannot be passed in registers, then it's not a PST.
+    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
+        RAA != CGCXXABI::RAA_Default)
+      return false;
+
+    // Pure scalable types are never unions and never contain unions.
+    const RecordDecl *RD = RT->getDecl();
+    if (RD->isUnion())
+      return false;
+
+    // If this is a C++ record, check the bases.
+    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
+      for (const auto &I : CXXRD->bases()) {
+        if (isEmptyRecord(getContext(), I.getType(), true))
+          continue;
+        if (!passAsPureScalableType(I.getType(), NVec, NPred, CoerceToSeq))
+          return false;
+      }
+    }
+
+    // Check members.
+    for (const auto *FD : RD->fields()) {
+      QualType FT = FD->getType();
+      if (isEmptyField(getContext(), FD, /* AllowArrays */ true))
+        continue;
+      if (!passAsPureScalableType(FT, NVec, NPred, CoerceToSeq))
+        return false;
+    }
+
+    return true;
+  }
+
+  const auto *VT = Ty->getAs<VectorType>();
+  if (!VT)
+    return false;
+
+  if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
+    ++NPred;
+    if (CoerceToSeq.size() + 1 > 12)
+      return false;
+    CoerceToSeq.push_back(convertFixedToScalableVectorType(VT));
+    return true;
+  }
+
+  if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
+    ++NVec;
+    if (CoerceToSeq.size() + 1 > 12)
+      return false;
+    CoerceToSeq.push_back(convertFixedToScalableVectorType(VT));
+    return true;
+  }
+
+  if (!VT->isBuiltinType())
+    return false;
+
+  switch (cast<BuiltinType>(VT)->getKind()) {
+#define SVE_VECTOR_TYPE(Name, MangledName, Id, SingletonId)                    \
+  case BuiltinType::Id:                                                        \
+    ++NVec;                                                                    \
+    break;
+#define SVE_PREDICATE_TYPE(Name, MangledName, Id, SingletonId)                 \
+  case BuiltinType::Id:                                                        \
+    ++NPred;                                                                   \
+    break;
+#define SVE_TYPE(Name, Id, SingletonId)
+#include "clang/Basic/AArch64SVEACLETypes.def"
+  default:
+    return false;
+  }
+
+  ASTContext::BuiltinVectorTypeInfo Info =
+      getContext().getBuiltinVectorTypeInfo(cast<BuiltinType>(Ty));
+  assert(Info.NumVectors > 0 && Info.NumVectors <= 4 &&
+         "Expected 1, 2, 3 or 4 vectors!");
+  auto VTy = llvm::ScalableVectorType::get(CGT.ConvertType(Info.ElementType),
+                                           Info.EC.getKnownMinValue());
+
+  if (CoerceToSeq.size() + Info.NumVectors > 12)
+    return false;
+  std::fill_n(std::back_inserter(CoerceToSeq), Info.NumVectors, VTy);
+
+  return true;
+}
+
+// Expand an LLVM IR type into a sequence with a element for each non-struct,
+// non-array member of the type, with the exception of the padding types, which
+// are retained.
+void AArch64ABIInfo::flattenType(
+    llvm::Type *Ty, SmallVectorImpl<llvm::Type *> &Flattened) const {
+
+  if (ABIArgInfo::isPaddingForCoerceAndExpand(Ty)) {
+    Flattened.push_back(Ty);
+    return;
+  }
+
+  if (const auto *AT = dyn_cast<llvm::ArrayType>(Ty)) {
+    uint64_t NElt = AT->getNumElements();
+    if (NElt == 0)
+      return;
+
+    SmallVector<llvm::Type *> EltFlattened;
+    flattenType(AT->getElementType(), EltFlattened);
+
+    for (uint64_t I = 0; I < NElt; ++I)
+      llvm::copy(EltFlattened, std::back_inserter(Flattened));
+    return;
+  }
+
+  if (const auto *ST = dyn_cast<llvm::StructType>(Ty)) {
+    for (auto *ET : ST->elements())
+      flattenType(ET, Flattened);
+    return;
+  }
+
+  Flattened.push_back(Ty);
+}
+
 RValue AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
                                       CodeGenFunction &CGF, AArch64ABIKind Kind,
                                       AggValueSlot Slot) const {
-  ABIArgInfo AI = classifyArgumentType(Ty, /*IsVariadic=*/true,
-                                       CGF.CurFnInfo->getCallingConvention());
+  // These numbers are not used for variadic arguments, hence it doesn't matter
+  // they don't retain their values across multiple calls to
+  // `classifyArgumentType` here.
+  unsigned NSRN = 0, NPRN = 0;
+  ABIArgInfo AI =
+      classifyArgumentType(Ty, /*IsVariadicFn=*/true, /* IsNamedArg */ false,
+                           CGF.CurFnInfo->getCallingConvention(), NSRN, NPRN);
   // Empty records are ignored for parameter passing purposes.
   if (AI.isIgnore())
     return Slot.asRValue();
diff --git a/clang/test/CodeGen/aarch64-pure-scalable-args-empty-union.c b/clang/test/CodeGen/aarch64-pure-scalable-args-empty-union.c
new file mode 100644
index 000000000000..546910068c78
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-pure-scalable-args-empty-union.c
@@ -0,0 +1,39 @@
+// RUN: %clang_cc1        -O3 -triple aarch64 -target-feature +sve -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK-C
+// RUN: %clang_cc1 -x c++ -O3 -triple aarch64 -target-feature +sve -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK-CXX
+
+typedef __SVFloat32_t fvec32 __attribute__((arm_sve_vector_bits(128)));
+
+// PST containing an empty union: when compiled as C pass it in registers,
+// when compiled as C++ - in memory.
+typedef struct {
+  fvec32 x[4];
+  union {} u;
+} S0;
+
+#ifdef __cplusplus
+extern "C"
+#endif
+void use0(S0);
+
+void f0(S0 *p) {
+  use0(*p);
+}
+// CHECK-C:   declare void @use0(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
+// CHECK-CXX: declare void @use0(ptr noundef)
+
+#ifdef __cplusplus
+
+// PST containing an empty union with `[[no_unique_address]]`` - pass in registers.
+typedef struct {
+   fvec32 x[4];
+   [[no_unique_address]]
+   union {} u;
+} S1;
+
+extern "C" void use1(S1);
+void f1(S1 *p) {
+  use1(*p);
+}
+// CHECK-CXX: declare void @use1(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
+
+#endif // __cplusplus
diff --git a/clang/test/CodeGen/aarch64-pure-scalable-args.c b/clang/test/CodeGen/aarch64-pure-scalable-args.c
new file mode 100644
index 000000000000..851159ada767
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-pure-scalable-args.c
@@ -0,0 +1,461 @@
+// RUN: %clang_cc1 -O3 -triple aarch64                                  -target-feature +sve -target-feature +sve2p1 -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-AAPCS
+// RUN: %clang_cc1 -O3 -triple arm64-apple-ios7.0 -target-abi darwinpcs -target-feature +sve -target-feature +sve2p1 -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-DARWIN
+// RUN: %clang_cc1 -O3 -triple aarch64-linux-gnu                        -target-feature +sve -target-feature +sve2p1 -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-AAPCS
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_neon.h>
+#include <arm_sve.h>
+#include <stdarg.h>
+
+typedef svfloat32_t fvec32 __attribute__((arm_sve_vector_bits(128)));
+typedef svfloat64_t fvec64 __attribute__((arm_sve_vector_bits(128)));
+typedef svbool_t bvec __attribute__((arm_sve_vector_bits(128)));
+typedef svmfloat8_t mfvec8 __attribute__((arm_sve_vector_bits(128)));
+
+typedef struct {
+    float f[4];
+} HFA;
+
+typedef struct {
+    mfloat8x16_t f[4];
+} HVA;
+
+// Pure Scalable Type, needs 4 Z-regs, 2 P-regs
+typedef struct {
+     bvec a;
+     fvec64 x;
+     fvec32 y[2];
+     mfvec8 z;
+     bvec b;
+} PST;
+
+// Pure Scalable Type, 1 Z-reg
+typedef struct {
+    fvec32 x;
+} SmallPST;
+
+// Big PST, does not fit in registers.
+typedef struct {
+    struct {
+        bvec a;
+        fvec32 x[4];
+    } u[2];
+    fvec64 v;
+} BigPST;
+
+// A small aggregate type
+typedef struct  {
+    char data[16];
+} SmallAgg;
+
+// CHECK: %struct.PST = type { <2 x i8>, <2 x double>, [2 x <4 x float>], <16 x i8>, <2 x i8> }
+
+// Test argument passing of Pure Scalable Types by examining the generated
+// LLVM IR function declarations. A PST argument in C/C++ should map to:
+//   a) an `ptr` argument, if passed indirectly through memory
+//   b) a series of scalable vector arguments, if passed via registers
+
+// Simple argument passing, PST expanded into registers.
+//   a    -> p0
+//   b    -> p1
+//   x    -> q0
+//   y[0] -> q1
+//   y[1] -> q2
+//   z    -> q3
+void test_argpass_simple(PST *p) {
+    void argpass_simple_callee(PST);
+    argpass_simple_callee(*p);
+}
+// CHECK-AAPCS:      define dso_local void @test_argpass_simple(ptr nocapture noundef readonly %p)
+// CHECK-AAPCS-NEXT: entry:
+// CHECK-AAPCS-NEXT: %0 = load <2 x i8>, ptr %p, align 16
+// CHECK-AAPCS-NEXT: %cast.scalable = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> undef, <2 x i8> %0, i64 0)
+// CHECK-AAPCS-NEXT: %1 = bitcast <vscale x 2 x i8> %cast.scalable to <vscale x 16 x i1>
+// CHECK-AAPCS-NEXT: %2 = getelementptr inbounds nuw i8, ptr %p, i64 16
+// CHECK-AAPCS-NEXT: %3 = load <2 x double>, ptr %2, align 16
+// CHECK-AAPCS-NEXT: %cast.scalable1 = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double> undef, <2 x double> %3, i64 0)
+// CHECK-AAPCS-NEXT: %4 = getelementptr inbounds nuw i8, ptr %p, i64 32
+// CHECK-AAPCS-NEXT: %5 = load <4 x float>, ptr %4, align 16
+// CHECK-AAPCS-NEXT: %cast.scalable2 = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> %5, i64 0)
+// CHECK-AAPCS-NEXT: %6 = getelementptr inbounds nuw i8, ptr %p, i64 48
+// CHECK-AAPCS-NEXT: %7 = load <4 x float>, ptr %6, align 16
+// CHECK-AAPCS-NEXT: %cast.scalable3 = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> %7, i64 0)
+// CHECK-AAPCS-NEXT: %8 = getelementptr inbounds nuw i8, ptr %p, i64 64
+// CHECK-AAPCS-NEXT: %9 = load <16 x i8>, ptr %8, align 16
+// CHECK-AAPCS-NEXT: %cast.scalable4 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> %9, i64 0)
+// CHECK-AAPCS-NEXT: %10 = getelementptr inbounds nuw i8, ptr %p, i64 80
+// CHECK-AAPCS-NEXT: %11 = load <2 x i8>, ptr %10, align 16
+// CHECK-AAPCS-NEXT: %cast.scalable5 = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> undef, <2 x i8> %11, i64 0)
+// CHECK-AAPCS-NEXT: %12 = bitcast <vscale x 2 x i8> %cast.scalable5 to <vscale x 16 x i1>
+// CHECK-AAPCS-NEXT: tail call void @argpass_simple_callee(<vscale x 16 x i1> %1, <vscale x 2 x double> %cast.scalable1, <vscale x 4 x float> %cast.scalable2, <vscale x 4 x float> %cast.scalable3, <vscale x 16 x i8> %cast.scalable4, <vscale x 16 x i1> %12)
+// CHECK-AAPCS-NEXT: ret void
+
+// CHECK-AAPCS:  declare void @argpass_simple_callee(<vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @argpass_simple_callee(ptr noundef)
+
+// Boundary case of using the last available Z-reg, PST expanded.
+//   0.0  -> d0-d3
+//   a    -> p0
+//   b    -> p1
+//   x    -> q4
+//   y[0] -> q5
+//   y[1] -> q6
+//   z    -> q7
+void test_argpass_last_z(PST *p) {
+    void argpass_last_z_callee(double, double, double, double, PST);
+    argpass_last_z_callee(.0, .0, .0, .0, *p);
+}
+// CHECK-AAPCS:  declare void @argpass_last_z_callee(double noundef, double noundef, double noundef, double noundef, <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @argpass_last_z_callee(double noundef, double noundef, double noundef, double noundef, ptr noundef)
+
+
+// Like the above, but using a tuple type to occupy some registers.
+//   x    -> z0.d-z3.d
+//   a    -> p0
+//   b    -> p1
+//   x    -> q4
+//   y[0] -> q5
+//   y[1] -> q6
+//   z    -> q7
+void test_argpass_last_z_tuple(PST *p, svfloat64x4_t x) {
+  void argpass_last_z_tuple_callee(svfloat64x4_t, PST);
+  argpass_last_z_tuple_callee(x, *p);
+}
+// CHECK-AAPCS:  declare void @argpass_last_z_tuple_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @argpass_last_z_tuple_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, ptr noundef)
+
+
+// Boundary case of using the last available P-reg, PST expanded.
+//   false -> p0-p1
+//   a     -> p2
+//   b     -> p3
+//   x     -> q0
+//   y[0]  -> q1
+//   y[1]  -> q2
+//   z     -> q3
+void test_argpass_last_p(PST *p) {
+    void argpass_last_p_callee(svbool_t, svcount_t, PST);
+    argpass_last_p_callee(svpfalse(), svpfalse_c(), *p);
+}
+// CHECK-AAPCS:  declare void @argpass_last_p_callee(<vscale x 16 x i1>, target("aarch64.svcount"), <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @argpass_last_p_callee(<vscale x 16 x i1>, target("aarch64.svcount"), ptr noundef)
+
+
+// Not enough Z-regs, push PST to memory and pass a pointer, Z-regs and
+// P-regs still available for other arguments
+//   u     -> z0
+//   v     -> q1
+//   w     -> q2
+//   0.0   -> d3-d4
+//   1     -> w0
+//   *p    -> memory, address -> x1
+//   2     -> w2
+//   3.0   -> d5
+//   true  -> p0
+void test_argpass_no_z(PST *p, double dummy, svmfloat8_t u, int8x16_t v, mfloat8x16_t w) {
+    void argpass_no_z_callee(svmfloat8_t, int8x16_t, mfloat8x16_t, double, double, int, PST, int, double, svbool_t);
+    argpass_no_z_callee(u, v, w, .0, .0, 1, *p, 2, 3.0, svptrue_b64());
+}
+// CHECK: declare void @argpass_no_z_callee(<vscale x 16 x i8>, <16 x i8> noundef, <16 x i8>, double noundef, double noundef, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+
+
+// Like the above, using a tuple to occupy some registers.
+//   x     -> z0.d-z3.d
+//   0.0   -> d4
+//   1     -> w0
+//   *p    -> memory, address -> x1
+//   2     -> w2
+//   3.0   -> d5
+//   true  -> p0
+void test_argpass_no_z_tuple_f64(PST *p, float dummy, svfloat64x4_t x) {
+  void argpass_no_z_tuple_f64_callee(svfloat64x4_t, double, int, PST, int,
+                                     double, svbool_t);
+  argpass_no_z_tuple_f64_callee(x, .0, 1, *p, 2, 3.0, svptrue_b64());
+}
+// CHECK: declare void @argpass_no_z_tuple_f64_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, double noundef, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+
+
+// Likewise, using a different tuple.
+//   x     -> z0.d-z3.d
+//   0.0   -> d4
+//   1     -> w0
+//   *p    -> memory, address -> x1
+//   2     -> w2
+//   3.0   -> d5
+//   true  -> p0
+void test_argpass_no_z_tuple_mfp8(PST *p, float dummy, svmfloat8x4_t x) {
+  void argpass_no_z_tuple_mfp8_callee(svmfloat8x4_t, double, int, PST, int,
+                                      double, svbool_t);
+  argpass_no_z_tuple_mfp8_callee(x, .0, 1, *p, 2, 3.0, svptrue_b64());
+}
+// CHECK: declare void @argpass_no_z_tuple_mfp8_callee(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, double noundef, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+
+
+// Not enough Z-regs (consumed by a HFA), PST passed indirectly
+//   0.0  -> d0
+//   *h   -> s1-s4
+//   1    -> w0
+//   *p   -> memory, address -> x1
+//   p    -> x1
+//   2    -> w2
+//   true -> p0
+void test_argpass_no_z_hfa(HFA *h, PST *p) {
+    void argpass_no_z_hfa_callee(double, HFA, int, PST, int, svbool_t);
+    argpass_no_z_hfa_callee(.0, *h, 1, *p, 2, svptrue_b64());
+}
+// CHECK-AAPCS:  declare void @argpass_no_z_hfa_callee(double noundef, [4 x float] alignstack(8), i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @argpass_no_z_hfa_callee(double noundef, [4 x float], i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
+
+// Not enough Z-regs (consumed by a HVA), PST passed indirectly
+//   0.0  -> d0
+//   *h   -> s1-s4
+//   1    -> w0
+//   *p   -> memory, address -> x1
+//   p    -> x1
+//   2    -> w2
+//   true -> p0
+void test_argpass_no_z_hva(HVA *h, PST *p) {
+    void argpass_no_z_hva_callee(double, HVA, int, PST, int, svbool_t);
+    argpass_no_z_hva_callee(.0, *h, 1, *p, 2, svptrue_b64());
+}
+// CHECK-AAPCS:  declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>] alignstack(16), i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>], i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
+
+// Not enough P-regs, PST passed indirectly, Z-regs and P-regs still available.
+//   true -> p0-p2
+//   1    -> w0
+//   *p   -> memory, address -> x1
+//   2    -> w2
+//   3.0  -> d0
+//   true -> p3
+void test_argpass_no_p(PST *p) {
+    void argpass_no_p_callee(svbool_t, svbool_t, svbool_t, int, PST, int, double, svbool_t);
+    argpass_no_p_callee(svptrue_b8(), svptrue_b16(), svptrue_b32(), 1, *p, 2, 3.0, svptrue_b64());
+}
+// CHECK: declare void @argpass_no_p_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+
+
+// Like above, using a tuple to occupy some registers.
+// P-regs still available.
+//   v    -> p0-p1
+//   u    -> p2
+//   1    -> w0
+//   *p   -> memory, address -> x1
+//   2    -> w2
+//   3.0  -> d0
+//   true -> p3
+void test_argpass_no_p_tuple(PST *p, svbool_t u, svboolx2_t v) {
+  void argpass_no_p_tuple_callee(svboolx2_t, svbool_t, int, PST, int, double,
+                                 svbool_t);
+  argpass_no_p_tuple_callee(v, u, 1, *p, 2, 3.0, svptrue_b64());
+}
+// CHECK: declare void @argpass_no_p_tuple_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+
+
+// HFAs go back-to-back to memory, afterwards Z-regs not available, PST passed indirectly.
+//   0.0   -> d0-d3
+//   *h    -> memory
+//   *p    -> memory, address -> x0
+//   *h    -> memory
+//   false -> p0
+void test_after_hfa(HFA *h, PST *p) {
+    void after_hfa_callee(double, double, double, double, double, HFA, PST, HFA, svbool_t);
+    after_hfa_callee(.0, .0, .0, .0, .0, *h, *p, *h, svpfalse());
+}
+// CHECK-AAPCS:  declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float] alignstack(8), ptr noundef, [4 x float] alignstack(8), <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float], ptr noundef, [4 x float], <vscale x 16 x i1>)
+
+// Small PST, not enough registers, passed indirectly, unlike other small
+// aggregates.
+//   *s  -> x0-x1
+//   0.0 -> d0-d7
+//   *p  -> memory, address -> x2
+//   1.0 -> memory
+//   2.0 -> memory (next to the above)
+void test_small_pst(SmallPST *p, SmallAgg *s) {
+    void small_pst_callee(SmallAgg, double, double, double, double, double, double, double, double, double, SmallPST, double);
+    small_pst_callee(*s, .0, .0, .0, .0, .0, .0, .0, .0, 1.0, *p, 2.0);
+}
+// CHECK-AAPCS:  declare void @small_pst_callee([2 x i64], double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, ptr noundef, double noundef)
+// CHECK-DARWIN: declare void @small_pst_callee([2 x i64], double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, i128, double noundef)
+
+
+// Simple return, PST expanded to registers
+//   p->a    -> p0
+//   p->x    -> q0
+//   p->y[0] -> q1
+//   p->y[1] -> q2
+//   p->z    -> q3
+//   p->b    -> p1
+PST test_return(PST *p) {
+    return *p;
+}
+// CHECK-AAPCS:  define dso_local <{ <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1> }> @test_return(ptr
+// CHECK-DARWIN: define void @test_return(ptr dead_on_unwind noalias nocapture writable writeonly sret(%struct.PST) align 16 %agg.result, ptr nocapture noundef readonly %p)
+
+// Corner case of 1-element aggregate
+//   p->x -> q0
+SmallPST test_return_small_pst(SmallPST *p) {
+    return *p;
+}
+// CHECK-AAPCS:  define dso_local <vscale x 4 x float> @test_return_small_pst(ptr
+// CHECK-DARWIN: define i128 @test_return_small_pst(ptr nocapture noundef readonly %p)
+
+
+// Big PST, returned indirectly
+//   *p -> *x8
+BigPST test_return_big_pst(BigPST *p) {
+    return *p;
+}
+// CHECK-AAPCS:  define dso_local void @test_return_big_pst(ptr dead_on_unwind noalias nocapture writable writeonly sret(%struct.BigPST) align 16 %agg.result, ptr nocapture noundef readonly %p)
+// CHECK-DARWIN: define void @test_return_big_pst(ptr dead_on_unwind noalias nocapture writable writeonly sret(%struct.BigPST) align 16 %agg.result, ptr nocapture noundef readonly %p)
+
+// Variadic arguments are unnamed, PST passed indirectly.
+// (Passing SVE types to a variadic function currently unsupported by
+// the AArch64 backend)
+//   p->a    -> p0
+//   p->x    -> q0
+//   p->y[0] -> q1
+//   p->y[1] -> q2
+//   p->z    -> q3
+//   p->b    -> p1
+//   *q -> memory, address -> x1
+void test_pass_variadic(PST *p, PST *q) {
+    void pass_variadic_callee(PST, ...);
+    pass_variadic_callee(*p, *q);
+}
+// CHECK-AAPCS: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp, ptr noundef nonnull align 16 dereferenceable(96) %q, i64 96, i1 false)
+// CHECK-AAPCS: call void (<vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>, ...) @pass_variadic_callee(<vscale x 16 x i1> %1, <vscale x 2 x double> %cast.scalable1, <vscale x 4 x float> %cast.scalable2, <vscale x 4 x float> %cast.scalable3, <vscale x 16 x i8> %cast.scalable4, <vscale x 16 x i1> %12, ptr noundef nonnull %byval-temp)
+
+// CHECK-DARWIN: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp, ptr noundef nonnull align 16 dereferenceable(96) %p, i64 96, i1 false)
+// CHECK-DARWIN: call void @llvm.lifetime.start.p0(i64 96, ptr nonnull %byval-temp1)
+// CHECK-DARWIN: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp1, ptr noundef nonnull align 16 dereferenceable(96) %q, i64 96, i1 false)
+// CHECK-DARWIN: call void (ptr, ...) @pass_variadic_callee(ptr noundef nonnull %byval-temp, ptr noundef nonnull %byval-temp1)
+
+
+// Test passing a small PST, still passed indirectly, despite being <= 128 bits
+void test_small_pst_variadic(SmallPST *p) {
+    void small_pst_variadic_callee(int, ...);
+    small_pst_variadic_callee(0, *p);
+}
+// CHECK-AAPCS: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(16) %byval-temp, ptr noundef nonnull align 16 dereferenceable(16) %p, i64 16, i1 false)
+// CHECK-AAPCS: call void (i32, ...) @small_pst_variadic_callee(i32 noundef 0, ptr noundef nonnull %byval-temp)
+
+// CHECK-DARWIN: %0 = load i128, ptr %p, align 16
+// CHECK-DARWIN: tail call void (i32, ...) @small_pst_variadic_callee(i32 noundef 0, i128 %0)
+
+// Test handling of a PST argument when passed in registers, from the callee side.
+void test_argpass_callee_side(PST v) {
+    void use(PST *p);
+    use(&v);
+}
+// CHECK-AAPCS:      define dso_local void @test_argpass_callee_side(<vscale x 16 x i1> %0, <vscale x 2 x double> %.coerce1, <vscale x 4 x float> %.coerce3, <vscale x 4 x float> %.coerce5, <vscale x 16 x i8> %.coerce7, <vscale x 16 x i1> %1)
+// CHECK-AAPCS-NEXT: entry:
+// CHECK-AAPCS-NEXT:   %v = alloca %struct.PST, align 16
+// CHECK-AAPCS-NEXT:   %.coerce = bitcast <vscale x 16 x i1> %0 to <vscale x 2 x i8>
+// CHECK-AAPCS-NEXT:   %cast.fixed = tail call <2 x i8> @llvm.vector.extract.v2i8.nxv2i8(<vscale x 2 x i8> %.coerce, i64 0)
+// CHECK-AAPCS-NEXT:   store <2 x i8> %cast.fixed, ptr %v, align 16
+// CHECK-AAPCS-NEXT:   %2 = getelementptr inbounds nuw i8, ptr %v, i64 16
+// CHECK-AAPCS-NEXT:   %cast.fixed2 = tail call <2 x double> @llvm.vector.extract.v2f64.nxv2f64(<vscale x 2 x double> %.coerce1, i64 0)
+// CHECK-AAPCS-NEXT:   store <2 x double> %cast.fixed2, ptr %2, align 16
+// CHECK-AAPCS-NEXT:   %3 = getelementptr inbounds nuw i8, ptr %v, i64 32
+// CHECK-AAPCS-NEXT:   %cast.fixed4 = tail call <4 x float> @llvm.vector.extract.v4f32.nxv4f32(<vscale x 4 x float> %.coerce3, i64 0)
+// CHECK-AAPCS-NEXT:   store <4 x float> %cast.fixed4, ptr %3, align 16
+// CHECK-AAPCS-NEXT:   %4 = getelementptr inbounds nuw i8, ptr %v, i64 48
+// CHECK-AAPCS-NEXT:   %cast.fixed6 = tail call <4 x float> @llvm.vector.extract.v4f32.nxv4f32(<vscale x 4 x float> %.coerce5, i64 0)
+// CHECK-AAPCS-NEXT:   store <4 x float> %cast.fixed6, ptr %4, align 16
+// CHECK-AAPCS-NEXT:   %5 = getelementptr inbounds nuw i8, ptr %v, i64 64
+// CHECK-AAPCS-NEXT:   %cast.fixed8 = tail call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> %.coerce7, i64 0)
+// CHECK-AAPCS-NEXT:   store <16 x i8> %cast.fixed8, ptr %5, align 16
+// CHECK-AAPCS-NEXT:   %6 = getelementptr inbounds nuw i8, ptr %v, i64 80
+// CHECK-AAPCS-NEXT:   %.coerce9 = bitcast <vscale x 16 x i1> %1 to <vscale x 2 x i8>
+// CHECK-AAPCS-NEXT:   %cast.fixed10 = tail call <2 x i8> @llvm.vector.extract.v2i8.nxv2i8(<vscale x 2 x i8> %.coerce9, i64 0)
+// CHECK-AAPCS-NEXT:   store <2 x i8> %cast.fixed10, ptr %6, align 16
+// CHECK-AAPCS-NEXT:   call void @use(ptr noundef nonnull %v)
+// CHECK-AAPCS-NEXT:   ret void
+// CHECK-AAPCS-NEXT: }
+
+// Test va_arg operation
+#ifdef __cplusplus
+ extern "C"
+#endif
+void test_va_arg(int n, ...) {
+     va_list ap;
+     va_start(ap, n);  
+     PST v = va_arg(ap, PST);
+     va_end(ap);
+
+     void use1(bvec, fvec32);
+     use1(v.a, v.y[1]);
+}
+// CHECK-AAPCS: define dso_local void @test_va_arg(i32 noundef %n, ...)
+// CHECK-AAPCS-NEXT: entry:
+// CHECK-AAPCS-NEXT:   %ap = alloca %struct.__va_list, align 8
+// CHECK-AAPCS-NEXT:   call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %ap)
+// CHECK-AAPCS-NEXT:   call void @llvm.va_start.p0(ptr nonnull %ap)
+// CHECK-AAPCS-NEXT:   %gr_offs_p = getelementptr inbounds nuw i8, ptr %ap, i64 24
+// CHECK-AAPCS-NEXT:   %gr_offs = load i32, ptr %gr_offs_p, align 8
+// CHECK-AAPCS-NEXT:   %0 = icmp sgt i32 %gr_offs, -1
+// CHECK-AAPCS-NEXT:   br i1 %0, label %vaarg.on_stack, label %vaarg.maybe_reg
+// CHECK-AAPCS-EMPTY:
+// CHECK-AAPCS-NEXT: vaarg.maybe_reg:                                  ; preds = %entry
+
+// Increment by 8, size of the pointer to the argument value, not size of the argument value itself.
+
+// CHECK-AAPCS-NEXT:   %new_reg_offs = add nsw i32 %gr_offs, 8
+// CHECK-AAPCS-NEXT:   store i32 %new_reg_offs, ptr %gr_offs_p, align 8
+// CHECK-AAPCS-NEXT:   %inreg = icmp ult i32 %gr_offs, -7
+// CHECK-AAPCS-NEXT:   br i1 %inreg, label %vaarg.in_reg, label %vaarg.on_stack
+// CHECK-AAPCS-EMPTY:
+// CHECK-AAPCS-NEXT: vaarg.in_reg:                                     ; preds = %vaarg.maybe_reg
+// CHECK-AAPCS-NEXT:   %reg_top_p = getelementptr inbounds nuw i8, ptr %ap, i64 8
+// CHECK-AAPCS-NEXT:   %reg_top = load ptr, ptr %reg_top_p, align 8
+// CHECK-AAPCS-NEXT:   %1 = sext i32 %gr_offs to i64
+// CHECK-AAPCS-NEXT:   %2 = getelementptr inbounds i8, ptr %reg_top, i64 %1
+// CHECK-AAPCS-NEXT:   br label %vaarg.end
+// CHECK-AAPCS-EMPTY:
+// CHECK-AAPCS-NEXT: vaarg.on_stack:                                   ; preds = %vaarg.maybe_reg, %entry
+// CHECK-AAPCS-NEXT:   %stack = load ptr, ptr %ap, align 8
+// CHECK-AAPCS-NEXT:   %new_stack = getelementptr inbounds i8, ptr %stack, i64 8
+// CHECK-AAPCS-NEXT:   store ptr %new_stack, ptr %ap, align 8
+// CHECK-AAPCS-NEXT:   br label %vaarg.end
+// CHECK-AAPCS-EMPTY:
+// CHECK-AAPCS-NEXT: vaarg.end:                                        ; preds = %vaarg.on_stack, %vaarg.in_reg
+// CHECK-AAPCS-NEXT:   %vaargs.addr = phi ptr [ %2, %vaarg.in_reg ], [ %stack, %vaarg.on_stack ]
+
+// Extra indirection, for a composite passed indirectly.
+// CHECK-AAPCS-NEXT:   %vaarg.addr = load ptr, ptr %vaargs.addr, align 8
+
+// CHECK-AAPCS-NEXT:   %v.sroa.0.0.copyload = load <2 x i8>, ptr %vaarg.addr, align 16
+// CHECK-AAPCS-NEXT:   %v.sroa.43.0.vaarg.addr.sroa_idx = getelementptr inbounds i8, ptr %vaarg.addr, i64 48
+// CHECK-AAPCS-NEXT:   %v.sroa.43.0.copyload = load <4 x float>, ptr %v.sroa.43.0.vaarg.addr.sroa_idx, align 16
+// CHECK-AAPCS-NEXT:   call void @llvm.va_end.p0(ptr nonnull %ap)
+// CHECK-AAPCS-NEXT:   %cast.scalable = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> undef, <2 x i8> %v.sroa.0.0.copyload, i64 0)
+// CHECK-AAPCS-NEXT:   %3 = bitcast <vscale x 2 x i8> %cast.scalable to <vscale x 16 x i1>
+// CHECK-AAPCS-NEXT:   %cast.scalable2 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> %v.sroa.43.0.copyload, i64 0)
+// CHECK-AAPCS-NEXT:   call void @use1(<vscale x 16 x i1> noundef %3, <vscale x 4 x float> noundef %cast.scalable2)
+// CHECK-AAPCS-NEXT:   call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %ap)
+// CHECK-AAPCS-NEXT:   ret void
+// CHECK-AAPCS-NEXT: }
+
+// CHECK-DARWIN: define void @test_va_arg(i32 noundef %n, ...)
+// CHECK-DARWIN-NEXT: entry:
+// CHECK-DARWIN-NEXT:   %ap = alloca ptr, align 8
+// CHECK-DARWIN-NEXT:   call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %ap)
+// CHECK-DARWIN-NEXT:   call void @llvm.va_start.p0(ptr nonnull %ap)
+// CHECK-DARWIN-NEXT:   %argp.cur = load ptr, ptr %ap, align 8
+// CHECK-DARWIN-NEXT:   %argp.next = getelementptr inbounds i8, ptr %argp.cur, i64 8
+// CHECK-DARWIN-NEXT:   store ptr %argp.next, ptr %ap, align 8
+// CHECK-DARWIN-NEXT:   %0 = load ptr, ptr %argp.cur, align 8
+// CHECK-DARWIN-NEXT:   %v.sroa.0.0.copyload = load <2 x i8>, ptr %0, align 16
+// CHECK-DARWIN-NEXT:   %v.sroa.43.0..sroa_idx = getelementptr inbounds i8, ptr %0, i64 48
+// CHECK-DARWIN-NEXT:   %v.sroa.43.0.copyload = load <4 x float>, ptr %v.sroa.43.0..sroa_idx, align 16
+// CHECK-DARWIN-NEXT:   call void @llvm.va_end.p0(ptr nonnull %ap)
+// CHECK-DARWIN-NEXT:   %cast.scalable = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> undef, <2 x i8> %v.sroa.0.0.copyload, i64 0)
+// CHECK-DARWIN-NEXT:   %1 = bitcast <vscale x 2 x i8> %cast.scalable to <vscale x 16 x i1>
+// CHECK-DARWIN-NEXT:   %cast.scalable2 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> %v.sroa.43.0.copyload, i64 0)
+// CHECK-DARWIN-NEXT:   call void @use1(<vscale x 16 x i1> noundef %1, <vscale x 4 x float> noundef %cast.scalable2)
+// CHECK-DARWIN-NEXT:   call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %ap)
+// CHECK-DARWIN-NEXT:   ret void
+// CHECK-DARWIN-NEXT: }
-- 
GitLab


From b27acebe63b128feb3bb9c3c62d77f235d2e6a6e Mon Sep 17 00:00:00 2001
From: Brox Chen <guochen2@amd.com>
Date: Mon, 28 Oct 2024 11:57:00 -0400
Subject: [PATCH 328/693] [AMDGPU][True16][test] update VOPC/VOPCX test for
 true16/fake16 (#112829)

This is a non-functional change

update GFX11/GFX12 VOPC/VOPCX asm/dasm test for true16/fake16:

1. duplicate files to be true16/fake16 by adding
"-mattr=+real-true16/-mattr=-real-true16" while true16 test file will be
updated to true16 format when the true16 instructions are supported
2. sort "*t16_err.s" and "*t16_promote.s" tests to alphabetic order.
tests to alphabetic order. This is for the upcoming true16 mc changes,
and mainly trying to help repo maintainer to resolve conflicts in the
tests quickly. A script is proposed to help for the sorting
https://github.com/llvm/llvm-project/pull/111769. Since these two files
are t16 only, it should not create conflicts in downstream branches
3. add `-filetype=null` to seperate stdout and stderr to avoid
disordered output from llvm-mc
---
 .../AMDGPU/gfx11_asm_vop3_from_vopc-fake16.s  | 10486 +++++++++++++++
 .../test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s |     8 +-
 llvm/test/MC/AMDGPU/gfx11_asm_vopc-fake16.s   | 10948 ++++++++++++++++
 llvm/test/MC/AMDGPU/gfx11_asm_vopc.s          |     8 +-
 .../MC/AMDGPU/gfx11_asm_vopc_dpp16-fake16.s   |  7172 ++++++++++
 llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp16.s    |     8 +-
 .../MC/AMDGPU/gfx11_asm_vopc_dpp8-fake16.s    |  1540 +++
 llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp8.s     |     8 +-
 llvm/test/MC/AMDGPU/gfx11_asm_vopc_t16_err.s  |  2962 ++---
 .../MC/AMDGPU/gfx11_asm_vopc_t16_promote.s    |  2622 ++--
 llvm/test/MC/AMDGPU/gfx11_asm_vopcx-fake16.s  |  4106 ++++++
 llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s         |     4 +-
 .../MC/AMDGPU/gfx11_asm_vopcx_dpp16-fake16.s  |  2690 ++++
 llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp16.s   |     4 +-
 .../MC/AMDGPU/gfx11_asm_vopcx_dpp8-fake16.s   |   578 +
 llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp8.s    |     4 +-
 llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_err.s |   718 +-
 .../MC/AMDGPU/gfx11_asm_vopcx_t16_promote.s   |   718 +-
 llvm/test/MC/AMDGPU/gfx12_asm_vop3c-fake16.s  |  8695 ++++++++++++
 llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s         |     8 +-
 llvm/test/MC/AMDGPU/gfx12_asm_vopc-fake16.s   |  9076 +++++++++++++
 llvm/test/MC/AMDGPU/gfx12_asm_vopc.s          |     8 +-
 .../MC/AMDGPU/gfx12_asm_vopc_dpp16-fake16.s   |  6052 +++++++++
 llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp16.s    |     8 +-
 .../MC/AMDGPU/gfx12_asm_vopc_dpp8-fake16.s    |  1300 ++
 llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp8.s     |     8 +-
 llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_err.s  |  2693 ++--
 .../MC/AMDGPU/gfx12_asm_vopc_t16_promote.s    |  3881 +++---
 llvm/test/MC/AMDGPU/gfx12_asm_vopcx-fake16.s  |  3404 +++++
 llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s         |     4 +-
 .../MC/AMDGPU/gfx12_asm_vopcx_dpp16-fake16.s  |  2270 ++++
 llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16.s   |     4 +-
 .../MC/AMDGPU/gfx12_asm_vopcx_dpp8-fake16.s   |   488 +
 llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8.s    |     4 +-
 llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_err.s |   647 +-
 .../MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s   |   647 +-
 .../Disassembler/AMDGPU/gfx11_dasm_vopc.txt   |     6 +-
 .../AMDGPU/gfx11_dasm_vopc_dpp16.txt          |     6 +-
 .../AMDGPU/gfx11_dasm_vopc_dpp8.txt           |     6 +-
 .../Disassembler/AMDGPU/gfx11_dasm_vopcx.txt  |     6 +-
 .../AMDGPU/gfx11_dasm_vopcx_dpp16.txt         |     6 +-
 .../AMDGPU/gfx11_dasm_vopcx_dpp8.txt          |     6 +-
 .../Disassembler/AMDGPU/gfx12_dasm_vopc.txt   |     6 +-
 .../AMDGPU/gfx12_dasm_vopc_dpp16.txt          |     6 +-
 .../AMDGPU/gfx12_dasm_vopc_dpp8.txt           |     6 +-
 .../Disassembler/AMDGPU/gfx12_dasm_vopcx.txt  |     6 +-
 .../AMDGPU/gfx12_dasm_vopcx_dpp16.txt         |     6 +-
 .../AMDGPU/gfx12_dasm_vopcx_dpp8.txt          |     6 +-
 48 files changed, 76343 insertions(+), 7510 deletions(-)
 create mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vopc-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp16-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp8-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vopcx-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp16-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp8-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx12_asm_vop3c-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx12_asm_vopc-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp16-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp8-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx12_asm_vopcx-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8-fake16.s

diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc-fake16.s
new file mode 100644
index 000000000000..faa2b1f97699
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc-fake16.s
@@ -0,0 +1,10486 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_cmp_class_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, v255, v2
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0xff,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, s1, v2
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x01,0x04,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, s105, v255
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x69,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, vcc_lo, s2
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x6a,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, vcc_hi, s105
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x6b,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, ttmp15, ttmp15
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x7b,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, m0, src_scc
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x7d,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x7d,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x7d,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x7d,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x7d,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], v1, 0.5
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x01,0xe1,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], v255, v2
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0xff,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], s1, v2
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x01,0x04,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], s105, v255
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x69,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], vcc_lo, s2
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x6a,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], vcc_hi, s105
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x6b,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], ttmp15, ttmp15
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x7b,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], m0, src_scc
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x7d,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x7d,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x7d,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x7d,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 null, -|0xfe0b|, vcc_hi
+// GFX11: encoding: [0x7c,0x01,0x7d,0xd4,0xff,0xd6,0x00,0x20,0x0b,0xfe,0x00,0x00]
+
+v_cmp_class_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x7e,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x7e,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x7e,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x7e,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x7e,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x7e,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x7e,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 null, -|0xaf123456|, vcc_hi
+// GFX11: encoding: [0x7c,0x01,0x7e,0xd4,0xff,0xd6,0x00,0x20,0x56,0x34,0x12,0xaf]
+
+v_cmp_class_f64_e64 s5, v[1:2], v2
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, v[1:2], v255
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, v[1:2], s2
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x01,0x05,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, v[1:2], s105
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x01,0xd3,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, v[254:255], ttmp15
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0xfe,0xf7,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, s[2:3], vcc_hi
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x02,0xd6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, s[104:105], vcc_lo
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x68,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, vcc, m0
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x6a,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, ttmp[14:15], exec_hi
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x7a,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, exec, exec_lo
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x7e,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s105, null, null
+// W32: encoding: [0x69,0x00,0x7f,0xd4,0x7c,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x7f,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 vcc_hi, 0.5, 0.5
+// W32: encoding: [0x6b,0x00,0x7f,0xd4,0xf0,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 ttmp15, -|src_scc|, src_scc
+// W32: encoding: [0x7b,0x01,0x7f,0xd4,0xfd,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], v[1:2], v2
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], v[1:2], v255
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], v[1:2], s2
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x01,0x05,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], v[1:2], s105
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x01,0xd3,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], v[254:255], ttmp15
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0xfe,0xf7,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], s[2:3], vcc_hi
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x02,0xd6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], s[104:105], vcc_lo
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x68,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], vcc, m0
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x6a,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], ttmp[14:15], exec_hi
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x7a,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], exec, exec_lo
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x7e,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], null, null
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x7c,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x7f,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 vcc, 0.5, 0.5
+// W64: encoding: [0x6a,0x00,0x7f,0xd4,0xf0,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 ttmp[14:15], -|src_scc|, src_scc
+// W64: encoding: [0x7a,0x01,0x7f,0xd4,0xfd,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 null, 0xaf123456, 0xaf123456
+// GFX11: encoding: [0x7c,0x00,0x7f,0xd4,0xff,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_eq_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x02,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x02,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x02,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x02,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x02,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x02,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x02,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x02,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x02,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x02,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x02,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_eq_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x12,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x12,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x12,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x12,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x12,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x12,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x12,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x12,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x12,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x12,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x12,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_eq_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x22,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x22,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x22,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x22,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x22,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x22,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x22,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x22,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x22,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x22,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x22,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x22,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x22,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x22,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x22,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x22,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x22,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x22,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x22,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x22,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x22,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x22,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x22,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_eq_i16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x32,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x32,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x32,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x32,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x32,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x32,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x32,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x32,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 null, 0xfe0b, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x32,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_eq_i32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x42,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x42,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x42,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x42,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x42,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x42,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x42,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x42,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x42,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_eq_i64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x52,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x52,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x52,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x52,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x52,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x52,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x52,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x52,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x52,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x52,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x52,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x52,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x52,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x52,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x52,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_eq_u16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x3a,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x3a,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x3a,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x3a,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x3a,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x3a,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x3a,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 null, 0xfe0b, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x3a,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_eq_u32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x4a,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x4a,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x4a,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x4a,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x4a,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x4a,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x4a,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x4a,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_eq_u64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x5a,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x5a,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x5a,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x5a,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x5a,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x5a,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x5a,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x5a,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x5a,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x5a,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x5a,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x5a,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x5a,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x5a,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x5a,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_f_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x00,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x00,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x00,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x00,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x00,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x00,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x00,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x00,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x00,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x00,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x00,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x00,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x00,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x00,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x00,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x00,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x00,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x00,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x00,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x00,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x00,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x00,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x00,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x00,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x00,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x00,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x00,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x00,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x00,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_f_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x10,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x10,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x10,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x10,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x10,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x10,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x10,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x10,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x10,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x10,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x10,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x10,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x10,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x10,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x10,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x10,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x10,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x10,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x10,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x10,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x10,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x10,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x10,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x10,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x10,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x10,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x10,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x10,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x10,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_f_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x20,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x20,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x20,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x20,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x20,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x20,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x20,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x20,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x20,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x20,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x20,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x20,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x20,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x20,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x20,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x20,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x20,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x20,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x20,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x20,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x20,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x20,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x20,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_f_i32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x40,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x40,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x40,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x40,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x40,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x40,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x40,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x40,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x40,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x40,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x40,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x40,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x40,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x40,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x40,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x40,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x40,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x40,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x40,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x40,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x40,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x40,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x40,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x40,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x40,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x40,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x40,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x40,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x40,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_f_i64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x50,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x50,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x50,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x50,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x50,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x50,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x50,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x50,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x50,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x50,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x50,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x50,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x50,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x50,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x50,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x50,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x50,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x50,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x50,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x50,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x50,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x50,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_i64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x50,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_f_u32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x48,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x48,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x48,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x48,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x48,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x48,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x48,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x48,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x48,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x48,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x48,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x48,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x48,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x48,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x48,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x48,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x48,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x48,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x48,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x48,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x48,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x48,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x48,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x48,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x48,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x48,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x48,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x48,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x48,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_f_u64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x58,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x58,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x58,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x58,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x58,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x58,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x58,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x58,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x58,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x58,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x58,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x58,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x58,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x58,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x58,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x58,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x58,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x58,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x58,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x58,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x58,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x58,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_f_u64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x58,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ge_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x06,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x06,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x06,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x06,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x06,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x06,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x06,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x06,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x06,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x06,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x06,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_ge_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x16,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x16,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x16,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x16,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x16,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x16,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x16,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x16,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x16,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x16,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x16,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_ge_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x26,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x26,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x26,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x26,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x26,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x26,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x26,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x26,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x26,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x26,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x26,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x26,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x26,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x26,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x26,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x26,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x26,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x26,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x26,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x26,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x26,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x26,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x26,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_ge_i16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x36,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x36,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x36,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x36,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x36,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x36,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x36,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x36,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 null, 0xfe0b, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x36,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_ge_i32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x46,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x46,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x46,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x46,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x46,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x46,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x46,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x46,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x46,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ge_i64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x56,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x56,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x56,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x56,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x56,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x56,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x56,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x56,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x56,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x56,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x56,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x56,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x56,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x56,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x56,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ge_u16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x3e,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x3e,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x3e,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x3e,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x3e,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x3e,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x3e,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 null, 0xfe0b, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x3e,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_ge_u32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x4e,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x4e,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x4e,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x4e,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x4e,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x4e,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x4e,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x4e,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ge_u64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x5e,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x5e,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x5e,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x5e,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x5e,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x5e,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x5e,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x5e,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x5e,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x5e,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x5e,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x5e,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x5e,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x5e,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x5e,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_gt_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x04,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x04,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x04,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x04,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x04,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x04,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x04,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x04,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x04,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x04,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x04,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_gt_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x14,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x14,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x14,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x14,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x14,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x14,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x14,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x14,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x14,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x14,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x14,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_gt_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x24,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x24,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x24,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x24,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x24,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x24,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x24,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x24,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x24,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x24,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x24,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x24,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x24,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x24,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x24,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x24,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x24,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x24,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x24,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x24,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x24,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x24,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x24,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_gt_i16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x34,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x34,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x34,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x34,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x34,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x34,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x34,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x34,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 null, 0xfe0b, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x34,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_gt_i32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x44,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x44,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x44,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x44,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x44,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x44,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x44,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x44,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x44,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_gt_i64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x54,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x54,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x54,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x54,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x54,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x54,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x54,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x54,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x54,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x54,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x54,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x54,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x54,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x54,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x54,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_gt_u16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x3c,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x3c,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x3c,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x3c,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x3c,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x3c,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x3c,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 null, 0xfe0b, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x3c,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_gt_u32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x4c,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x4c,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x4c,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x4c,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x4c,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x4c,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x4c,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x4c,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_gt_u64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x5c,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x5c,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x5c,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x5c,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x5c,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x5c,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x5c,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x5c,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x5c,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x5c,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x5c,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x5c,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x5c,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x5c,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x5c,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_le_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x03,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x03,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x03,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x03,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x03,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x03,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x03,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x03,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x03,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x03,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x03,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_le_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x13,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x13,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x13,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x13,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x13,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x13,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x13,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x13,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x13,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x13,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x13,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_le_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x23,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x23,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x23,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x23,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x23,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x23,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x23,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x23,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x23,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x23,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x23,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x23,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x23,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x23,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x23,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x23,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x23,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x23,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x23,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x23,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x23,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x23,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x23,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_le_i16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x33,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x33,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x33,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x33,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x33,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x33,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x33,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x33,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 null, 0xfe0b, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x33,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_le_i32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x43,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x43,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x43,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x43,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x43,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x43,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x43,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x43,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x43,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_le_i64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x53,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x53,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x53,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x53,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x53,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x53,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x53,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x53,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x53,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x53,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x53,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x53,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x53,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x53,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x53,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_le_u16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x3b,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x3b,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x3b,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x3b,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x3b,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x3b,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x3b,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 null, 0xfe0b, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x3b,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_le_u32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x4b,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x4b,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x4b,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x4b,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x4b,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x4b,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x4b,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x4b,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_le_u64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x5b,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x5b,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x5b,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x5b,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x5b,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x5b,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x5b,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x5b,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x5b,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x5b,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x5b,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x5b,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x5b,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x5b,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x5b,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_lg_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x05,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x05,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x05,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x05,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x05,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x05,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x05,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x05,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x05,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x05,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x05,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_lg_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x15,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x15,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x15,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x15,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x15,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x15,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x15,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x15,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x15,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x15,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x15,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_lg_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x25,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x25,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x25,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x25,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x25,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x25,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x25,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x25,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x25,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x25,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x25,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x25,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x25,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x25,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x25,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x25,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x25,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x25,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x25,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x25,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x25,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x25,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x25,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_lt_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x01,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x01,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x01,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x01,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x01,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x01,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x01,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x01,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x01,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x01,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x01,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_lt_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x11,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x11,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x11,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x11,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x11,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x11,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x11,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x11,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x11,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x11,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x11,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_lt_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x21,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x21,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x21,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x21,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x21,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x21,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x21,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x21,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x21,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x21,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x21,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x21,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x21,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x21,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x21,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x21,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x21,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x21,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x21,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x21,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x21,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x21,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x21,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_lt_i16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x31,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x31,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x31,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x31,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x31,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x31,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x31,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x31,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 null, 0xfe0b, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x31,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_lt_i32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x41,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x41,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x41,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x41,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x41,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x41,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x41,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x41,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x41,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_lt_i64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x51,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x51,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x51,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x51,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x51,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x51,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x51,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x51,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x51,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x51,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x51,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x51,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x51,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x51,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x51,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_lt_u16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x39,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x39,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x39,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x39,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x39,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x39,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x39,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x39,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 null, 0xfe0b, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x39,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_lt_u32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x49,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x49,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x49,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x49,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x49,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x49,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x49,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x49,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x49,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_lt_u64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x59,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x59,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x59,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x59,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x59,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x59,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x59,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x59,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x59,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x59,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x59,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x59,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x59,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x59,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x59,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ne_i16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x35,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x35,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x35,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x35,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x35,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x35,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x35,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x35,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 null, 0xfe0b, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x35,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_ne_i32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x45,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x45,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x45,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x45,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x45,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x45,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x45,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x45,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x45,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ne_i64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x55,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x55,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x55,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x55,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x55,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x55,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x55,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x55,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x55,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x55,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x55,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x55,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x55,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x55,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x55,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ne_u16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x3d,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x3d,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x3d,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x3d,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x3d,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x3d,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x3d,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 null, 0xfe0b, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x3d,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_ne_u32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x4d,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x4d,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x4d,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x4d,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x4d,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x4d,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x4d,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x4d,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ne_u64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x5d,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x5d,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x5d,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x5d,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x5d,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x5d,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x5d,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x5d,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x5d,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x5d,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x5d,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x5d,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x5d,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x5d,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x5d,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_neq_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x0d,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x0d,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x0d,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x0d,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x0d,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x0d,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x0d,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x0d,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x0d,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x0d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_neq_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x1d,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x1d,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x1d,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x1d,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x1d,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x1d,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x1d,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x1d,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x1d,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x1d,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_neq_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x2d,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x2d,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x2d,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x2d,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x2d,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x2d,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x2d,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x2d,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x2d,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x2d,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x2d,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x2d,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x2d,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x2d,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x2d,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x2d,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x2d,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x2d,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x2d,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x2d,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x2d,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x2d,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x2d,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_nge_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x09,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x09,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x09,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x09,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x09,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x09,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x09,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x09,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x09,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x09,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x09,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_nge_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x19,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x19,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x19,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x19,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x19,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x19,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x19,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x19,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x19,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x19,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x19,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_nge_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x29,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x29,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x29,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x29,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x29,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x29,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x29,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x29,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x29,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x29,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x29,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x29,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x29,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x29,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x29,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x29,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x29,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x29,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x29,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x29,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x29,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x29,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x29,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_ngt_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x0b,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x0b,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x0b,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x0b,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x0b,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x0b,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x0b,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x0b,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x0b,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x0b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_ngt_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x1b,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x1b,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x1b,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x1b,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x1b,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x1b,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x1b,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x1b,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x1b,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x1b,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_ngt_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x2b,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x2b,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x2b,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x2b,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x2b,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x2b,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x2b,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x2b,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x2b,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x2b,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x2b,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x2b,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x2b,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x2b,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x2b,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x2b,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x2b,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x2b,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x2b,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x2b,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x2b,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x2b,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x2b,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_nle_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x0c,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x0c,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x0c,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x0c,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x0c,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x0c,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x0c,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x0c,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x0c,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x0c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_nle_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x1c,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x1c,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x1c,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x1c,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x1c,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x1c,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x1c,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x1c,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x1c,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x1c,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_nle_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x2c,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x2c,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x2c,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x2c,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x2c,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x2c,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x2c,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x2c,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x2c,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x2c,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x2c,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x2c,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x2c,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x2c,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x2c,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x2c,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x2c,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x2c,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x2c,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x2c,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x2c,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x2c,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x2c,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_nlg_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x0a,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x0a,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x0a,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x0a,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x0a,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x0a,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x0a,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x0a,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x0a,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x0a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_nlg_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x1a,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x1a,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x1a,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x1a,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x1a,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x1a,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x1a,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x1a,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x1a,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x1a,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_nlg_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x2a,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x2a,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x2a,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x2a,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x2a,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x2a,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x2a,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x2a,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x2a,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x2a,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x2a,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x2a,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x2a,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x2a,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x2a,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x2a,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x2a,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x2a,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x2a,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x2a,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x2a,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x2a,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x2a,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_nlt_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x0e,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x0e,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x0e,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x0e,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x0e,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x0e,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x0e,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x0e,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x0e,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x0e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_nlt_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x1e,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x1e,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x1e,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x1e,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x1e,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x1e,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x1e,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x1e,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x1e,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x1e,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_nlt_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x2e,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x2e,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x2e,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x2e,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x2e,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x2e,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x2e,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x2e,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x2e,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x2e,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x2e,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x2e,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x2e,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x2e,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x2e,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x2e,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x2e,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x2e,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x2e,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x2e,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x2e,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x2e,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x2e,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_o_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x07,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x07,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x07,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x07,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x07,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x07,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x07,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x07,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x07,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x07,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x07,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_o_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x17,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x17,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x17,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x17,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x17,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x17,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x17,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x17,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x17,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x17,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x17,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_o_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x27,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x27,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x27,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x27,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x27,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x27,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x27,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x27,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x27,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x27,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x27,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x27,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x27,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x27,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x27,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x27,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x27,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x27,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x27,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x27,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x27,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x27,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x27,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_t_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x0f,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x0f,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x0f,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x0f,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x0f,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x0f,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x0f,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x0f,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x0f,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x0f,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_t_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x1f,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x1f,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x1f,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x1f,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x1f,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x1f,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x1f,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x1f,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x1f,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x1f,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_t_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x2f,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x2f,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x2f,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x2f,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x2f,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x2f,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x2f,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x2f,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x2f,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x2f,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x2f,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x2f,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x2f,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x2f,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x2f,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x2f,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x2f,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x2f,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x2f,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x2f,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x2f,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x2f,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x2f,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_t_i32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x47,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x47,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x47,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x47,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x47,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x47,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x47,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x47,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x47,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x47,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x47,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x47,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x47,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x47,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x47,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x47,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x47,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x47,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x47,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x47,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x47,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x47,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x47,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x47,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x47,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x47,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x47,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x47,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x47,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_t_i64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x57,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x57,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x57,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x57,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x57,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x57,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x57,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x57,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x57,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x57,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x57,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x57,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x57,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x57,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x57,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x57,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x57,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x57,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x57,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x57,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x57,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x57,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_i64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x57,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_t_u32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x4f,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x4f,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x4f,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x4f,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x4f,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x4f,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x4f,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x4f,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x4f,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x4f,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x4f,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x4f,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x4f,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x4f,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x4f,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x4f,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x4f,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x4f,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x4f,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x4f,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x4f,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x4f,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x4f,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x4f,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x4f,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x4f,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x4f,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x4f,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u32_e64 null, 0xaf123456, vcc_hi
+// GFX11: encoding: [0x7c,0x00,0x4f,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_t_u64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x5f,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x5f,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x5f,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x5f,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x5f,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x5f,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x5f,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x5f,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x5f,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x5f,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x5f,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x5f,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x5f,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x5f,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x5f,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x5f,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x5f,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x5f,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x5f,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x5f,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x5f,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x5f,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_t_u64_e64 null, 0xaf123456, vcc
+// GFX11: encoding: [0x7c,0x00,0x5f,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_tru_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x0f,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x0f,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x0f,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x0f,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x0f,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x0f,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x0f,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x0f,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x0f,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x0f,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x0f,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x0f,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_tru_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x1f,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x1f,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x1f,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x1f,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x1f,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x1f,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x1f,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x1f,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x1f,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x1f,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x1f,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x1f,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_tru_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x2f,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x2f,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x2f,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x2f,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x2f,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x2f,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x2f,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x2f,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x2f,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x2f,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x2f,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x2f,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x2f,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x2f,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x2f,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x2f,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x2f,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x2f,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x2f,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x2f,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x2f,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x2f,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_tru_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x2f,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_u_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x08,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x08,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x08,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x08,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x08,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x08,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x08,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x08,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x08,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x08,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x08,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_u_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x18,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x18,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x18,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x18,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x18,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x18,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x18,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x18,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x18,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x18,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: encoding: [0x7c,0x83,0x18,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_u_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x28,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x28,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x28,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x28,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x28,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x28,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x28,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x28,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x28,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x28,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x28,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x28,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x28,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x28,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x28,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x28,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x28,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x28,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x28,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x28,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x28,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x28,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX11: encoding: [0x7c,0x82,0x28,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s
index d0e79c0aa344..faa2b1f97699 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s
@@ -1,7 +1,7 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_cmp_class_f16_e64 s5, v1, v2
 // W32: encoding: [0x05,0x00,0x7d,0xd4,0x01,0x05,0x02,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopc-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopc-fake16.s
new file mode 100644
index 000000000000..e896511e6e5c
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopc-fake16.s
@@ -0,0 +1,10948 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefix=W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefix=W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_cmp_class_f16_e32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0xfa,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0xfa,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0xfd,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0xfd,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, v[1:2], v2
+// W32: encoding: [0x01,0x05,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, v[254:255], v2
+// W32: encoding: [0xfe,0x05,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, s[2:3], v2
+// W32: encoding: [0x02,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, s[104:105], v2
+// W32: encoding: [0x68,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, vcc, v2
+// W32: encoding: [0x6a,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, ttmp[14:15], v2
+// W32: encoding: [0x7a,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, exec, v2
+// W32: encoding: [0x7e,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0xff,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, v[1:2], v2
+// W64: encoding: [0x01,0x05,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, v[254:255], v2
+// W64: encoding: [0xfe,0x05,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, s[2:3], v2
+// W64: encoding: [0x02,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, s[104:105], v2
+// W64: encoding: [0x68,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, vcc, v2
+// W64: encoding: [0x6a,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, ttmp[14:15], v2
+// W64: encoding: [0x7a,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, exec, v2
+// W64: encoding: [0x7e,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0xff,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x04,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x04,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x25,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x25,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x45,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x45,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x64,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x64,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x85,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x85,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xa5,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xa5,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x74,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x74,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x95,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x95,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xb5,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xb5,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x00,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x00,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x00,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x00,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x00,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x00,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x00,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x00,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x00,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x00,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x00,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x00,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x00,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x00,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x00,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x00,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x00,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x00,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x00,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x00,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x00,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x00,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x00,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x00,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x00,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x00,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x00,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x00,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x00,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x00,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x20,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x20,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x20,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x20,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x20,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x20,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x20,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x20,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x20,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x20,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x20,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x20,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x20,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x20,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x21,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x20,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x20,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x20,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x20,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x20,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x20,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x20,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x20,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x20,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x20,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x20,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x20,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x20,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x20,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x21,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x40,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x40,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x40,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x40,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x40,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x40,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x40,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x40,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x40,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x40,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x40,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x41,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x40,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x40,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x40,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x40,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x40,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x40,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x40,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x40,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x40,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x40,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x40,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x41,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x80,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x80,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x80,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x80,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x80,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x80,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x80,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x80,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x80,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x80,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x80,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x80,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x80,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x80,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x81,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x80,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x80,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x80,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x80,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x80,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x80,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x80,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x80,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x80,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x80,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x80,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x80,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x80,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x80,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x81,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xa0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xa0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xa0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xa0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xa0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xa0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xa0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xa0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xa0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xa0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xa0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xa1,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xa0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xa0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xa0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xa0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xa0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xa0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xa0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xa0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xa0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xa0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xa0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xa1,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x90,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x90,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x90,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x90,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x90,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x90,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x90,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x90,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x90,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x90,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x90,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x90,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x90,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x90,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x91,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x90,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x90,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x90,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x90,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x90,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x90,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x90,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x90,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x90,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x90,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x90,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x90,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x90,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x90,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x91,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xb0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xb0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xb0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xb0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xb0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xb0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xb0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xb0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xb0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xb0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xb0,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xb1,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xb0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xb0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xb0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xb0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xb0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xb0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xb0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xb0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xb0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xb0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xb0,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xb1,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x0c,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x0c,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x2d,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x2d,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x4d,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x4d,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x6c,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x6c,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x8d,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x8d,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xad,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xad,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x7c,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x7c,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x9d,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x9d,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xbd,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xbd,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x08,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x08,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x29,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x29,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x49,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x49,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x68,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x68,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x89,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x89,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xa9,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xa9,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x78,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x78,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x99,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x99,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xb9,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xb9,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x06,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x06,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x27,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x27,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x47,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x47,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x66,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x66,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x87,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x87,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xa7,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xa7,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x76,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x76,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x97,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x97,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xb7,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xb7,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x0a,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x0a,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x2b,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x2b,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x4b,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x4b,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x02,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x02,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x23,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x23,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x43,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x43,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x62,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x62,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x83,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x83,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xa3,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xa3,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x72,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x72,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x93,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x93,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xb3,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xb3,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x6a,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x6a,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x8b,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x8b,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xab,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xab,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x7a,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x7a,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x9b,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x9b,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xbb,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xbb,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x1a,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x1a,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x3b,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x3b,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x5b,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x5b,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x12,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x12,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x33,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x33,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x53,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x53,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x16,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x16,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x37,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x37,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x57,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x57,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x18,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x18,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x39,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x39,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x59,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x59,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x14,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x14,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x35,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x35,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x55,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x55,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x1c,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x1c,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x3d,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x3d,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x5d,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x5d,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x0e,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x0e,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x2f,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x2f,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x4f,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x4f,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x1e,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x1e,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x3f,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x3f,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x5f,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x5f,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x8e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x8e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x8e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x8e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x8e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x8e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x8e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x8e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x8e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x8e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x8e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x8e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x8e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x8e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x8f,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x8e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x8e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x8e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x8e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x8e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x8e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x8e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x8e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x8e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x8e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x8e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x8e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x8e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x8e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x8f,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xae,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xae,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xae,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xae,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xae,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xae,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xae,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xae,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xae,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xae,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xae,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xaf,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xae,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xae,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xae,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xae,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xae,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xae,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xae,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xae,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xae,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xae,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xae,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xaf,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x9e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x9e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x9e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x9e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x9e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x9e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x9e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x9e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x9e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x9e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x9e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x9e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x9e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x9e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x9f,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x9e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x9e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x9e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x9e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x9e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x9e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x9e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x9e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x9e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x9e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x9e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x9e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x9e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x9e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x9f,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xbe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xbe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xbe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xbe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xbe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xbe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xbe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xbe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xbe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xbe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xbe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xbf,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xbe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xbe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xbe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xbe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xbe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xbe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xbe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xbe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xbe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xbe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xbe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xbf,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x1e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x1e,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x1e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x1e,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x3e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x3f,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x3e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x3f,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x5e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x5f,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x5e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x5f,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x10,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x10,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x31,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x31,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x51,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x51,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopc.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopc.s
index 5349362b8fba..05ed37c612ba 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopc.s
@@ -1,7 +1,7 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefix=W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefix=W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefix=W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_cmp_class_f16_e32 vcc_lo, v1, v2
 // W32: encoding: [0x01,0x05,0xfa,0x7c]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp16-fake16.s
new file mode 100644
index 000000000000..aa315a16b983
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp16-fake16.s
@@ -0,0 +1,7172 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefix=W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefix=W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_cmp_class_f16_dpp vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, -|v127|, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0xfa,0x7c,0x7f,0x6f,0x35,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, -|v127|, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0xfa,0x7c,0x7f,0x6f,0x35,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0xfd,0x7c,0xff,0x6f,0x35,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0xfd,0x7c,0xff,0x6f,0x35,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x04,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x04,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x25,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x25,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x64,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x64,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x85,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x85,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x74,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x74,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x95,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x95,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x00,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x00,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x00,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x00,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x00,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x20,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x21,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x20,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x20,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x21,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x80,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x81,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x80,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x80,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x81,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x90,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x91,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x90,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x90,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x91,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x0c,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x0c,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x2d,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x2d,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x6c,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x6c,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x8d,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x8d,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x7c,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x7c,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x9d,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x9d,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x08,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x08,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x29,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x29,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x68,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x68,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x89,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x89,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x78,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x78,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x99,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x99,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x06,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x06,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x27,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x27,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x66,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x66,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x87,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x87,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x76,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x76,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x97,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x97,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x0a,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x0a,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x2b,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x2b,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x02,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x02,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x23,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x23,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x62,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x62,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x83,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x83,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x72,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x72,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x93,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x93,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x6a,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x6a,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x8b,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x8b,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x7a,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x7a,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x9b,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x9b,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x1a,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x1a,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x3b,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x3b,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x12,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x12,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x33,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x33,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x16,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x16,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x37,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x37,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x18,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x18,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x39,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x39,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x14,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x14,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x35,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x35,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x1c,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x1c,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x3d,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x3d,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x0e,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x0e,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x2f,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x2f,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x1e,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x1e,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x3f,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x3f,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x8f,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x8e,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x8f,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x9f,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x9e,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x9f,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x1e,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x1e,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x1e,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x3f,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x3e,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x3f,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x10,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x10,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x31,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x31,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp16.s
index 1299d02c3c0a..bc77f0c1967d 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp16.s
@@ -1,7 +1,7 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefix=W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefix=W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefix=W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_cmp_class_f16_dpp vcc_lo, v1, v2 quad_perm:[3,2,1,0]
 // W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp8-fake16.s
new file mode 100644
index 000000000000..6f2f9e6704de
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp8-fake16.s
@@ -0,0 +1,1540 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefix=W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefix=W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_cmp_class_f16_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0xfa,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0xfa,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0xfc,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0xfc,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0xfd,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0xfc,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0xfc,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0xfd,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x04,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x04,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x04,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x04,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x04,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x04,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x24,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x24,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x25,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x24,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x24,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x25,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x64,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x64,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x64,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x64,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x64,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x64,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x84,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x84,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x85,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x84,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x84,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x85,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x74,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x74,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x74,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x74,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x74,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x74,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x94,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x94,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x95,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x94,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x94,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x95,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x00,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x00,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x00,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x00,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x00,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x00,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x20,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x20,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x21,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x20,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x20,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x21,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x80,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x80,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x81,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x80,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x80,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_i32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x81,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x90,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x90,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x91,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x90,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x90,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_f_u32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x91,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x0c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x0c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x0c,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x0c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x0c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x0c,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x2c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x2c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x2d,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x2c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x2c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x2d,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x6c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x6c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x6c,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x6c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x6c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x6c,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x8c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x8c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x8d,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x8c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x8c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x8d,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x7c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x7c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x7c,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x7c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x7c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x7c,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x9c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x9c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x9d,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x9c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x9c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x9d,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x08,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x08,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x08,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x08,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x08,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x08,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x28,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x28,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x29,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x28,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x28,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x29,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x68,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x68,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x68,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x68,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x68,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x68,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x88,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x88,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x89,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x88,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x88,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x89,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x78,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x78,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x78,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x78,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x78,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x78,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x98,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x98,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x99,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x98,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x98,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x99,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x06,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x06,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x06,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x06,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x06,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x06,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x26,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x26,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x27,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x26,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x26,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x27,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x66,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x66,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x66,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x66,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x66,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x66,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x86,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x86,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x87,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x86,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x86,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x87,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x76,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x76,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x76,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x76,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x76,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x76,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x96,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x96,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x97,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x96,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x96,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x97,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x0a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x0a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x0a,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x0a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x0a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x0a,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x2a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x2a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x2b,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x2a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x2a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x2b,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x02,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x02,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x02,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x02,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x02,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x02,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x22,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x22,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x23,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x22,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x22,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x23,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x62,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x62,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x62,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x62,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x62,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x62,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x82,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x82,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x83,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x82,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x82,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x83,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x72,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x72,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x72,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x72,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x72,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x72,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x92,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x92,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x93,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x92,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x92,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x93,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x6a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x6a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x6a,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x6a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x6a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x6a,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x8a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x8a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x8b,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x8a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x8a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x8b,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x7a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x7a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x7a,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x7a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x7a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x7a,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x9a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x9a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x9b,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x9a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x9a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x9b,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x1a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x1a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x1a,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x1a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x1a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x1a,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x3a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x3a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x3b,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x3a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x3a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x3b,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x12,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x12,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x12,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x12,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x12,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x12,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x32,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x32,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x33,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x32,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x32,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x33,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x16,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x16,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x16,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x16,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x16,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x16,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x36,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x36,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x37,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x36,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x36,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x37,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x18,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x18,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x18,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x18,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x18,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x18,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x38,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x38,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x39,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x38,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x38,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x39,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x14,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x14,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x14,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x14,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x14,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x14,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x34,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x34,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x35,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x34,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x34,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x35,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x1c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x1c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x1c,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x1c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x1c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x1c,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x3c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x3c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x3d,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x3c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x3c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x3d,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x0e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x0e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x0e,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x0e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x0e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x0e,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x2e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x2e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x2f,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x2e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x2e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x2f,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x1e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x1e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x1e,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x1e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x1e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x1e,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x3e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x3e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x3f,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x3e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x3e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x3f,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x8e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x8e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x8f,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x8e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x8e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_i32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x8f,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x9e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x9e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x9f,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x9e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x9e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_t_u32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x9f,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x1e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x1e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x1e,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x1e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x1e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x1e,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x3e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x3e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x3f,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x3e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x3e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_tru_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x3f,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x10,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x10,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x10,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x10,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x10,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x10,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x30,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x30,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x31,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x30,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x30,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x31,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp8.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp8.s
index 9f10a29791ad..1c333a0c9091 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_dpp8.s
@@ -1,7 +1,7 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefix=W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefix=W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefix=W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_cmp_class_f16_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // W32: encoding: [0xe9,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopc_t16_err.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_t16_err.s
index 889293b1a0f2..4b97d276d556 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopc_t16_err.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_t16_err.s
@@ -1,1973 +1,1973 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=error: %s
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort --version 5
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=error: %s
 
 v_cmp_class_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_class_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_class_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_class_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_class_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_class_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_class_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_class_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_class_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:40: error: invalid operand for instruction
+
+v_cmp_class_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:40: error: invalid operand for instruction
+
+v_cmp_class_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_class_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:38: error: invalid operand for instruction
 
 v_cmp_class_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_class_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_eq_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_eq_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_eq_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_eq_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_eq_i16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_eq_i16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_eq_i16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_eq_i16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_eq_u16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_eq_u16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_eq_u16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_eq_u16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_f_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
+
+v_cmp_f_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
 v_cmp_f_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_f_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_f_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
+
+v_cmp_f_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
 v_cmp_f_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_f_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_f_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_f_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
 
 v_cmp_f_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_f_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_f_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_f_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_f_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
 
 v_cmp_f_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_f_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ge_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ge_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ge_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ge_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ge_i16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ge_i16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ge_i16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ge_i16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ge_u16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ge_u16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ge_u16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ge_u16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_gt_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_gt_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_gt_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_gt_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_gt_i16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_gt_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
-v_cmp_gt_i16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_gt_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
-v_cmp_gt_i16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_gt_i16_e32 vcc, v127, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_gt_i16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_gt_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc, vcc_hi, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16_e32 vcc, vcc_lo, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16_e32 vcc_lo, v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_gt_i16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_gt_i16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_gt_u16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_gt_u16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_gt_u16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_gt_u16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_le_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_le_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_le_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_le_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_le_i16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_le_i16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_le_i16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_le_i16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_le_u16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_le_u16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_le_u16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_le_u16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lg_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lg_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lg_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lg_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lg_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lg_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lg_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lg_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lt_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lt_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lt_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lt_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lt_i16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lt_i16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lt_i16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lt_i16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lt_u16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lt_u16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lt_u16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lt_u16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_ne_i16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_ne_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
-v_cmp_ne_i16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_ne_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc, v127, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc, vcc_hi, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ne_i16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ne_i16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ne_u16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ne_u16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ne_u16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ne_u16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_neq_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_neq_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_neq_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_neq_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_neq_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_neq_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_neq_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_neq_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nge_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nge_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nge_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nge_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nge_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nge_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nge_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nge_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ngt_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_ngt_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_ngt_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ngt_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ngt_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_ngt_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_ngt_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ngt_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nle_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nle_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nle_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nle_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nle_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nle_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nle_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nle_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlg_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nlg_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nlg_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlg_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlg_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nlg_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nlg_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlg_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlt_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nlt_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nlt_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlt_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlt_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nlt_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nlt_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlt_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_o_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
 v_cmp_o_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
 v_cmp_o_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_o_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_o_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
 
 v_cmp_o_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
 
 v_cmp_o_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_o_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_t_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmp_t_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmp_t_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cmp_t_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_t_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_t_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_t_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_t_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_u_f16_e32 vcc, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_u_f16_e32 vcc, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_u_f16_e32 vcc, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_u_f16_e32 vcc_lo, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_u_f16_e32 vcc_lo, v127, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_u_f16_e32 vcc_lo, vcc_hi, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_u_f16_e32 vcc_lo, vcc_lo, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_class_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_class_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_eq_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_eq_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_eq_i16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_eq_i16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_eq_u16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_eq_u16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_f_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_f_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_i16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_i16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_u16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_u16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_i16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_i16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_u16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_u16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_i16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_i16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_u16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_u16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lg_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lg_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_i16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_i16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_u16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_u16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ne_i16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ne_i16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ne_u16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ne_u16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_neq_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_neq_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nge_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nge_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ngt_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ngt_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nle_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nle_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nlg_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nlg_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nlt_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nlt_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_o_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_o_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_t_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_t_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_u_f16_e32 vcc_lo, v128, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_class_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_f_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_f_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_f_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_f_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_t_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_t_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_t_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_t_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_f_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_f_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_t_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_t_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_f_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_f_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_f_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_f_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_t_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc, v127, v255
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
 v_cmp_t_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_t_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_t_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_tru_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmp_class_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmp_class_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_cmp_eq_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_cmp_eq_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_cmp_eq_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc, vcc_hi, v255
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cmp_eq_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc, vcc_lo, v255
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cmp_eq_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc_lo, v1, v255
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmp_eq_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmp_f_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmp_f_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc_lo, v127, v255
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmp_ge_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmp_ge_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmp_ge_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmp_ge_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmp_ge_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cmp_ge_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc_lo, vcc_hi, v255
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
-v_cmp_gt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_t_f16_e32 vcc_lo, vcc_lo, v255
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
-v_cmp_gt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc, v1, v255
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmp_gt_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmp_gt_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmp_gt_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc, v127, v255
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cmp_gt_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cmp_le_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cmp_le_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmp_le_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmp_le_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmp_le_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc, vcc_hi, v255
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
-v_cmp_le_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc, vcc_lo, v255
+// GFX11: :[[@LINE-1]]:32: error: invalid operand for instruction
 
-v_cmp_lg_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc_lo, v1, v255
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmp_lg_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmp_lt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmp_lt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc_lo, v127, v255
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
-v_cmp_lt_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
-v_cmp_lt_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
-v_cmp_lt_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmp_lt_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmp_ne_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmp_ne_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc_lo, vcc_hi, v255
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
-v_cmp_ne_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_tru_f16_e32 vcc_lo, vcc_lo, v255
+// GFX11: :[[@LINE-1]]:35: error: invalid operand for instruction
 
-v_cmp_ne_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc, v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_neq_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmp_neq_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmp_nge_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc, v127, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_nge_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
-v_cmp_ngt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:33: error: invalid operand for instruction
 
-v_cmp_ngt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_nle_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmp_nle_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmp_nlg_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc, vcc_hi, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_nlg_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc, vcc_lo, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_nlt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc_lo, v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_nlt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
 
-v_cmp_o_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
 
-v_cmp_o_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc_lo, v127, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_t_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
-v_cmp_t_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
-v_cmp_tru_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc_lo, v128, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_tru_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
 
-v_cmp_u_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:34: error: invalid operand for instruction
 
-v_cmp_u_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc_lo, vcc_hi, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
+v_cmp_u_f16_e32 vcc_lo, vcc_lo, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopc_t16_promote.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_t16_promote.s
index 75f20b0c7f0c..49a3f8ad63e7 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopc_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_t16_promote.s
@@ -1,1973 +1,1973 @@
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 %s
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 %s
 
 v_cmp_class_f16 vcc, v1, v255
-// GFX11: v_cmp_class_f16_e64
+// GFX11: v_cmp_class_f16_e64 vcc, v1, v255       ; encoding: [0x6a,0x00,0x7d,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_class_f16 vcc, v127, v255
-// GFX11: v_cmp_class_f16_e64
+v_cmp_class_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_class_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x7d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_class_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_class_f16_e64
+v_cmp_class_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_class_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_class_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_class_f16_e64
+v_cmp_class_f16 vcc, v127, v255
+// GFX11: v_cmp_class_f16_e64 vcc, v127, v255     ; encoding: [0x6a,0x00,0x7d,0xd4,0x7f,0xff,0x03,0x00]
 
 v_cmp_class_f16 vcc, v127, v255
-// GFX11: v_cmp_class_f16_e64
+// GFX11: v_cmp_class_f16_e64 vcc, v127, v255     ; encoding: [0x6a,0x00,0x7d,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_class_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_class_f16_e64
+v_cmp_class_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_class_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x7d,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_class_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_class_f16_e64
+v_cmp_class_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_class_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x7d,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_eq_f16 vcc, v1, v255
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_class_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_class_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7d,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_eq_f16 vcc, v127, v255
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_class_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_class_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7d,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_eq_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_class_f16 vcc, v128, v2
+// GFX11: v_cmp_class_f16_e64 vcc, v128, v2       ; encoding: [0x6a,0x00,0x7d,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_eq_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_class_f16 vcc, v128, v2
+// GFX11: v_cmp_class_f16_e64 vcc, v128, v2       ; encoding: [0x6a,0x00,0x7d,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_eq_f16 vcc, v1, v255
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_class_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_class_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_eq_f16 vcc, v127, v255
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_class_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_class_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_eq_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_class_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_class_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_eq_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_class_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_class_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_eq_i16 vcc, v1, v255
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_class_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_class_f16_e64 vcc, vcc_hi, v255   ; encoding: [0x6a,0x00,0x7d,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_eq_i16 vcc, v127, v255
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_class_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_class_f16_e64 vcc, vcc_hi, v255   ; encoding: [0x6a,0x00,0x7d,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_eq_i16 vcc, vcc_hi, v255
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_class_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_class_f16_e64 vcc, vcc_lo, v255   ; encoding: [0x6a,0x00,0x7d,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_eq_i16 vcc, vcc_lo, v255
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_class_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_class_f16_e64 vcc, vcc_lo, v255   ; encoding: [0x6a,0x00,0x7d,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_eq_i16 vcc, v1, v255
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_eq_f16 vcc, v1, v255
+// GFX11: v_cmp_eq_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x02,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_eq_i16 vcc, v127, v255
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_eq_f16 vcc, v1, v255
+// GFX11: v_cmp_eq_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x02,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_eq_i16 vcc, vcc_hi, v255
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_eq_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x02,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_eq_i16 vcc, vcc_lo, v255
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_eq_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x02,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_eq_u16 vcc, v1, v255
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_eq_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x02,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_eq_u16 vcc, v127, v255
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_eq_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x02,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_eq_u16 vcc, vcc_hi, v255
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_eq_f16 vcc, v127, v255
+// GFX11: v_cmp_eq_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x02,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_eq_u16 vcc, vcc_lo, v255
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_eq_f16 vcc, v127, v255
+// GFX11: v_cmp_eq_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x02,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_eq_u16 vcc, v1, v255
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_eq_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x02,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_eq_u16 vcc, v127, v255
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_eq_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x02,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_eq_u16 vcc, vcc_hi, v255
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_eq_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x02,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_eq_u16 vcc, vcc_lo, v255
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_eq_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x02,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_f_f16 vcc, v1, v255
-// GFX11: v_cmp_f_f16_e64
+v_cmp_eq_f16 vcc, v128, v2
+// GFX11: v_cmp_eq_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x02,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_f_f16 vcc, v127, v255
-// GFX11: v_cmp_f_f16_e64
+v_cmp_eq_f16 vcc, v128, v2
+// GFX11: v_cmp_eq_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x02,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_f_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_f_f16_e64
+v_cmp_eq_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x02,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_f_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_f_f16_e64
+v_cmp_eq_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x02,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_f_f16 vcc, v1, v255
-// GFX11: v_cmp_f_f16_e64
+v_cmp_eq_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_f_f16 vcc, v127, v255
-// GFX11: v_cmp_f_f16_e64
+v_cmp_eq_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_f_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_f_f16_e64
+v_cmp_eq_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_eq_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x02,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_f_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_f_f16_e64
+v_cmp_eq_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_eq_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x02,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ge_f16 vcc, v1, v255
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_eq_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_eq_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x02,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_ge_f16 vcc, v127, v255
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_eq_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_eq_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x02,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_ge_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_eq_i16 vcc, v1, v255
+// GFX11: v_cmp_eq_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x32,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_ge_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_eq_i16 vcc, v1, v255
+// GFX11: v_cmp_eq_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x32,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_ge_f16 vcc, v1, v255
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_eq_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x32,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_ge_f16 vcc, v127, v255
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_eq_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x32,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_ge_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_eq_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x32,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_ge_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_eq_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x32,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_ge_i16 vcc, v1, v255
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_eq_i16 vcc, v127, v255
+// GFX11: v_cmp_eq_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x32,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_ge_i16 vcc, v127, v255
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_eq_i16 vcc, v127, v255
+// GFX11: v_cmp_eq_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x32,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_ge_i16 vcc, vcc_hi, v255
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_eq_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x32,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_ge_i16 vcc, vcc_lo, v255
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_eq_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x32,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_ge_i16 vcc, v1, v255
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_eq_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x32,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ge_i16 vcc, v127, v255
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_eq_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x32,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ge_i16 vcc, vcc_hi, v255
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_eq_i16 vcc, v128, v2
+// GFX11: v_cmp_eq_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x32,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ge_i16 vcc, vcc_lo, v255
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_eq_i16 vcc, v128, v2
+// GFX11: v_cmp_eq_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x32,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ge_u16 vcc, v1, v255
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_eq_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x32,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_ge_u16 vcc, v127, v255
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_eq_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x32,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_ge_u16 vcc, vcc_hi, v255
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_eq_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_ge_u16 vcc, vcc_lo, v255
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_eq_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_ge_u16 vcc, v1, v255
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_eq_i16 vcc, vcc_hi, v255
+// GFX11: v_cmp_eq_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x32,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ge_u16 vcc, v127, v255
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_eq_i16 vcc, vcc_hi, v255
+// GFX11: v_cmp_eq_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x32,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ge_u16 vcc, vcc_hi, v255
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_eq_i16 vcc, vcc_lo, v255
+// GFX11: v_cmp_eq_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x32,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_ge_u16 vcc, vcc_lo, v255
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_eq_i16 vcc, vcc_lo, v255
+// GFX11: v_cmp_eq_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x32,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_gt_f16 vcc, v1, v255
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_eq_u16 vcc, v1, v255
+// GFX11: v_cmp_eq_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x3a,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_gt_f16 vcc, v127, v255
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_eq_u16 vcc, v1, v255
+// GFX11: v_cmp_eq_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x3a,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_gt_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_eq_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_gt_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_eq_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_gt_f16 vcc, v1, v255
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_eq_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_gt_f16 vcc, v127, v255
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_eq_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_gt_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_eq_u16 vcc, v127, v255
+// GFX11: v_cmp_eq_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x3a,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_gt_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_eq_u16 vcc, v127, v255
+// GFX11: v_cmp_eq_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x3a,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_gt_i16 vcc, v1, v255
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_eq_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3a,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_gt_i16 vcc, v127, v255
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_eq_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3a,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_gt_i16 vcc, vcc_hi, v255
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_eq_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3a,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_gt_i16 vcc, vcc_lo, v255
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_eq_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3a,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_gt_i16 vcc, v1, v255
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_eq_u16 vcc, v128, v2
+// GFX11: v_cmp_eq_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x3a,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_gt_i16 vcc, v127, v255
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_eq_u16 vcc, v128, v2
+// GFX11: v_cmp_eq_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x3a,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_gt_i16 vcc, vcc_hi, v255
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_eq_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3a,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_gt_i16 vcc, vcc_lo, v255
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_eq_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_eq_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3a,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_gt_u16 vcc, v1, v255
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_eq_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_gt_u16 vcc, v127, v255
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_eq_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_eq_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_gt_u16 vcc, vcc_hi, v255
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_eq_u16 vcc, vcc_hi, v255
+// GFX11: v_cmp_eq_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x3a,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_gt_u16 vcc, vcc_lo, v255
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_eq_u16 vcc, vcc_hi, v255
+// GFX11: v_cmp_eq_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x3a,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_gt_u16 vcc, v1, v255
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_eq_u16 vcc, vcc_lo, v255
+// GFX11: v_cmp_eq_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x3a,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_gt_u16 vcc, v127, v255
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_eq_u16 vcc, vcc_lo, v255
+// GFX11: v_cmp_eq_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x3a,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_gt_u16 vcc, vcc_hi, v255
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_f_f16 vcc, v1, v255
+// GFX11: v_cmp_f_f16_e64 vcc, v1, v255           ; encoding: [0x6a,0x00,0x00,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_gt_u16 vcc, vcc_lo, v255
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_f_f16 vcc, v1, v255
+// GFX11: v_cmp_f_f16_e64 vcc, v1, v255           ; encoding: [0x6a,0x00,0x00,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_le_f16 vcc, v1, v255
-// GFX11: v_cmp_le_f16_e64
+v_cmp_f_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_f_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x00,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_le_f16 vcc, v127, v255
-// GFX11: v_cmp_le_f16_e64
+v_cmp_f_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_f_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x00,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_le_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_le_f16_e64
+v_cmp_f_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_f_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x00,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_le_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_le_f16_e64
+v_cmp_f_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_f_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x00,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_le_f16 vcc, v1, v255
-// GFX11: v_cmp_le_f16_e64
+v_cmp_f_f16 vcc, v127, v255
+// GFX11: v_cmp_f_f16_e64 vcc, v127, v255         ; encoding: [0x6a,0x00,0x00,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_le_f16 vcc, v127, v255
-// GFX11: v_cmp_le_f16_e64
+v_cmp_f_f16 vcc, v127, v255
+// GFX11: v_cmp_f_f16_e64 vcc, v127, v255         ; encoding: [0x6a,0x00,0x00,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_le_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_le_f16_e64
+v_cmp_f_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_f_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x00,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_le_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_le_f16_e64
+v_cmp_f_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_f_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x00,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_le_i16 vcc, v1, v255
-// GFX11: v_cmp_le_i16_e64
+v_cmp_f_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_f_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x00,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_le_i16 vcc, v127, v255
-// GFX11: v_cmp_le_i16_e64
+v_cmp_f_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_f_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x00,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_le_i16 vcc, vcc_hi, v255
-// GFX11: v_cmp_le_i16_e64
+v_cmp_f_f16 vcc, v128, v2
+// GFX11: v_cmp_f_f16_e64 vcc, v128, v2           ; encoding: [0x6a,0x00,0x00,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_le_i16 vcc, vcc_lo, v255
-// GFX11: v_cmp_le_i16_e64
+v_cmp_f_f16 vcc, v128, v2
+// GFX11: v_cmp_f_f16_e64 vcc, v128, v2           ; encoding: [0x6a,0x00,0x00,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_le_i16 vcc, v1, v255
-// GFX11: v_cmp_le_i16_e64
+v_cmp_f_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_f_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x00,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_le_i16 vcc, v127, v255
-// GFX11: v_cmp_le_i16_e64
+v_cmp_f_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_f_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x00,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_le_i16 vcc, vcc_hi, v255
-// GFX11: v_cmp_le_i16_e64
+v_cmp_f_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_f_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_le_i16 vcc, vcc_lo, v255
-// GFX11: v_cmp_le_i16_e64
+v_cmp_f_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_f_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_le_u16 vcc, v1, v255
-// GFX11: v_cmp_le_u16_e64
+v_cmp_f_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_f_f16_e64 vcc, vcc_hi, v255       ; encoding: [0x6a,0x00,0x00,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_le_u16 vcc, v127, v255
-// GFX11: v_cmp_le_u16_e64
+v_cmp_f_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_f_f16_e64 vcc, vcc_hi, v255       ; encoding: [0x6a,0x00,0x00,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_le_u16 vcc, vcc_hi, v255
-// GFX11: v_cmp_le_u16_e64
+v_cmp_f_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_f_f16_e64 vcc, vcc_lo, v255       ; encoding: [0x6a,0x00,0x00,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_le_u16 vcc, vcc_lo, v255
-// GFX11: v_cmp_le_u16_e64
+v_cmp_f_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_f_f16_e64 vcc, vcc_lo, v255       ; encoding: [0x6a,0x00,0x00,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_le_u16 vcc, v1, v255
-// GFX11: v_cmp_le_u16_e64
+v_cmp_ge_f16 vcc, v1, v255
+// GFX11: v_cmp_ge_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x06,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_le_u16 vcc, v127, v255
-// GFX11: v_cmp_le_u16_e64
+v_cmp_ge_f16 vcc, v1, v255
+// GFX11: v_cmp_ge_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x06,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_le_u16 vcc, vcc_hi, v255
-// GFX11: v_cmp_le_u16_e64
+v_cmp_ge_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x06,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_le_u16 vcc, vcc_lo, v255
-// GFX11: v_cmp_le_u16_e64
+v_cmp_ge_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x06,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_lg_f16 vcc, v1, v255
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_ge_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x06,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_lg_f16 vcc, v127, v255
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_ge_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x06,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_lg_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_ge_f16 vcc, v127, v255
+// GFX11: v_cmp_ge_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x06,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_lg_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_ge_f16 vcc, v127, v255
+// GFX11: v_cmp_ge_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x06,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_lg_f16 vcc, v1, v255
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_ge_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x06,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_lg_f16 vcc, v127, v255
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_ge_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x06,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_lg_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_ge_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x06,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_lg_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_ge_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x06,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_lt_f16 vcc, v1, v255
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_ge_f16 vcc, v128, v2
+// GFX11: v_cmp_ge_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x06,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_lt_f16 vcc, v127, v255
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_ge_f16 vcc, v128, v2
+// GFX11: v_cmp_ge_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x06,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_lt_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_ge_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x06,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_lt_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_ge_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x06,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_lt_f16 vcc, v1, v255
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_ge_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_lt_f16 vcc, v127, v255
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_ge_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_lt_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_ge_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_ge_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x06,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_lt_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_ge_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_ge_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x06,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_lt_i16 vcc, v1, v255
-// GFX11: v_cmp_lt_i16_e64
+v_cmp_ge_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_ge_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x06,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_lt_i16 vcc, v127, v255
-// GFX11: v_cmp_lt_i16_e64
+v_cmp_ge_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_ge_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x06,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_lt_i16 vcc, vcc_hi, v255
-// GFX11: v_cmp_lt_i16_e64
+v_cmp_ge_i16 vcc, v1, v255
+// GFX11: v_cmp_ge_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x36,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_lt_i16 vcc, vcc_lo, v255
-// GFX11: v_cmp_lt_i16_e64
-
-v_cmp_lt_i16 vcc, v1, v255
-// GFX11: v_cmp_lt_i16_e64
-
-v_cmp_lt_i16 vcc, v127, v255
-// GFX11: v_cmp_lt_i16_e64
-
-v_cmp_lt_i16 vcc, vcc_hi, v255
-// GFX11: v_cmp_lt_i16_e64
-
-v_cmp_lt_i16 vcc, vcc_lo, v255
-// GFX11: v_cmp_lt_i16_e64
+v_cmp_ge_i16 vcc, v1, v255
+// GFX11: v_cmp_ge_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x36,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_lt_u16 vcc, v1, v255
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_ge_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x36,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_lt_u16 vcc, v127, v255
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_ge_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x36,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_lt_u16 vcc, vcc_hi, v255
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_ge_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x36,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_lt_u16 vcc, vcc_lo, v255
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_ge_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x36,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_lt_u16 vcc, v1, v255
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_ge_i16 vcc, v127, v255
+// GFX11: v_cmp_ge_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x36,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_lt_u16 vcc, v127, v255
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_ge_i16 vcc, v127, v255
+// GFX11: v_cmp_ge_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x36,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_lt_u16 vcc, vcc_hi, v255
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_ge_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x36,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_lt_u16 vcc, vcc_lo, v255
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_ge_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x36,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_ne_i16 vcc, v1, v255
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_ge_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x36,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ne_i16 vcc, v127, v255
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_ge_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x36,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ne_i16 vcc, vcc_hi, v255
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_ge_i16 vcc, v128, v2
+// GFX11: v_cmp_ge_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x36,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ne_i16 vcc, vcc_lo, v255
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_ge_i16 vcc, v128, v2
+// GFX11: v_cmp_ge_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x36,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ne_i16 vcc, v1, v255
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_ge_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x36,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_ne_i16 vcc, v127, v255
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_ge_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x36,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_ne_i16 vcc, vcc_hi, v255
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_ge_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_ne_i16 vcc, vcc_lo, v255
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_ge_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_ne_u16 vcc, v1, v255
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_ge_i16 vcc, vcc_hi, v255
+// GFX11: v_cmp_ge_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x36,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ne_u16 vcc, v127, v255
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_ge_i16 vcc, vcc_hi, v255
+// GFX11: v_cmp_ge_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x36,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ne_u16 vcc, vcc_hi, v255
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_ge_i16 vcc, vcc_lo, v255
+// GFX11: v_cmp_ge_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x36,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_ne_u16 vcc, vcc_lo, v255
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_ge_i16 vcc, vcc_lo, v255
+// GFX11: v_cmp_ge_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x36,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_ne_u16 vcc, v1, v255
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_ge_u16 vcc, v1, v255
+// GFX11: v_cmp_ge_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x3e,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_ne_u16 vcc, v127, v255
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_ge_u16 vcc, v1, v255
+// GFX11: v_cmp_ge_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x3e,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_ne_u16 vcc, vcc_hi, v255
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_ge_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_ne_u16 vcc, vcc_lo, v255
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_ge_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_neq_f16 vcc, v1, v255
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_ge_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_neq_f16 vcc, v127, v255
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_ge_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_neq_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_ge_u16 vcc, v127, v255
+// GFX11: v_cmp_ge_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x3e,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_neq_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_ge_u16 vcc, v127, v255
+// GFX11: v_cmp_ge_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x3e,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_neq_f16 vcc, v1, v255
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_ge_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3e,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_neq_f16 vcc, v127, v255
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_ge_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3e,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_neq_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_ge_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3e,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_neq_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_ge_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3e,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_nge_f16 vcc, v1, v255
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_ge_u16 vcc, v128, v2
+// GFX11: v_cmp_ge_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x3e,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_nge_f16 vcc, v127, v255
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_ge_u16 vcc, v128, v2
+// GFX11: v_cmp_ge_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x3e,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_nge_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_ge_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3e,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_nge_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_ge_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ge_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3e,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_nge_f16 vcc, v1, v255
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_ge_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_nge_f16 vcc, v127, v255
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_ge_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ge_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_nge_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_ge_u16 vcc, vcc_hi, v255
+// GFX11: v_cmp_ge_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x3e,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_nge_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_ge_u16 vcc, vcc_hi, v255
+// GFX11: v_cmp_ge_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x3e,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ngt_f16 vcc, v1, v255
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_ge_u16 vcc, vcc_lo, v255
+// GFX11: v_cmp_ge_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x3e,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_ngt_f16 vcc, v127, v255
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_ge_u16 vcc, vcc_lo, v255
+// GFX11: v_cmp_ge_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x3e,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_ngt_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_gt_f16 vcc, v1, v255
+// GFX11: v_cmp_gt_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x04,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_ngt_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_gt_f16 vcc, v1, v255
+// GFX11: v_cmp_gt_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x04,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_ngt_f16 vcc, v1, v255
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_gt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x04,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_ngt_f16 vcc, v127, v255
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_gt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x04,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_ngt_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_gt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x04,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_ngt_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_gt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x04,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_nle_f16 vcc, v1, v255
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_gt_f16 vcc, v127, v255
+// GFX11: v_cmp_gt_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x04,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_nle_f16 vcc, v127, v255
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_gt_f16 vcc, v127, v255
+// GFX11: v_cmp_gt_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x04,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_nle_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_gt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x04,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_nle_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_gt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x04,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_nle_f16 vcc, v1, v255
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_gt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x04,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_nle_f16 vcc, v127, v255
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_gt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x04,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_nle_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_gt_f16 vcc, v128, v2
+// GFX11: v_cmp_gt_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x04,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_nle_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_gt_f16 vcc, v128, v2
+// GFX11: v_cmp_gt_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x04,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_nlg_f16 vcc, v1, v255
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_gt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x04,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_nlg_f16 vcc, v127, v255
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_gt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x04,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_nlg_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_gt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_nlg_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_gt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_nlg_f16 vcc, v1, v255
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_gt_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_gt_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x04,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_nlg_f16 vcc, v127, v255
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_gt_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_gt_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x04,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_nlg_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_gt_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_gt_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x04,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_nlg_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_gt_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_gt_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x04,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_nlt_f16 vcc, v1, v255
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_gt_i16 vcc, v1, v255
+// GFX11: v_cmp_gt_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x34,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_nlt_f16 vcc, v127, v255
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_gt_i16 vcc, v1, v255
+// GFX11: v_cmp_gt_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x34,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_nlt_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_gt_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x34,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_nlt_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_gt_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x34,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_nlt_f16 vcc, v1, v255
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_gt_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x34,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_nlt_f16 vcc, v127, v255
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_gt_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x34,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_nlt_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_gt_i16 vcc, v127, v255
+// GFX11: v_cmp_gt_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x34,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_nlt_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_gt_i16 vcc, v127, v255
+// GFX11: v_cmp_gt_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x34,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_o_f16 vcc, v1, v255
-// GFX11: v_cmp_o_f16_e64
+v_cmp_gt_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x34,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_o_f16 vcc, v127, v255
-// GFX11: v_cmp_o_f16_e64
+v_cmp_gt_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x34,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_o_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_o_f16_e64
+v_cmp_gt_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x34,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_o_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_o_f16_e64
+v_cmp_gt_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x34,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_o_f16 vcc, v1, v255
-// GFX11: v_cmp_o_f16_e64
+v_cmp_gt_i16 vcc, v128, v2
+// GFX11: v_cmp_gt_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x34,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_o_f16 vcc, v127, v255
-// GFX11: v_cmp_o_f16_e64
+v_cmp_gt_i16 vcc, v128, v2
+// GFX11: v_cmp_gt_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x34,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_o_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_o_f16_e64
+v_cmp_gt_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x34,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_o_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_o_f16_e64
+v_cmp_gt_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x34,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_t_f16 vcc, v1, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_t_f16 vcc, v127, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_t_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_i16 vcc, vcc_hi, v255
+// GFX11: v_cmp_gt_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x34,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_t_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_i16 vcc, vcc_hi, v255
+// GFX11: v_cmp_gt_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x34,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_t_f16 vcc, v1, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_i16 vcc, vcc_lo, v255
+// GFX11: v_cmp_gt_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x34,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_t_f16 vcc, v127, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_i16 vcc, vcc_lo, v255
+// GFX11: v_cmp_gt_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x34,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_t_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_u16 vcc, v1, v255
+// GFX11: v_cmp_gt_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x3c,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_t_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_u16 vcc, v1, v255
+// GFX11: v_cmp_gt_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x3c,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_tru_f16 vcc, v1, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_tru_f16 vcc, v127, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_tru_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_tru_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_tru_f16 vcc, v1, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_u16 vcc, v127, v255
+// GFX11: v_cmp_gt_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x3c,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_tru_f16 vcc, v127, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_u16 vcc, v127, v255
+// GFX11: v_cmp_gt_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x3c,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_tru_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3c,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_tru_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_t_f16_e64
+v_cmp_gt_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3c,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_u_f16 vcc, v1, v255
-// GFX11: v_cmp_u_f16_e64
+v_cmp_gt_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3c,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_u_f16 vcc, v127, v255
-// GFX11: v_cmp_u_f16_e64
+v_cmp_gt_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3c,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_u_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_u_f16_e64
+v_cmp_gt_u16 vcc, v128, v2
+// GFX11: v_cmp_gt_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x3c,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_u_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_u_f16_e64
+v_cmp_gt_u16 vcc, v128, v2
+// GFX11: v_cmp_gt_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x3c,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_u_f16 vcc, v1, v255
-// GFX11: v_cmp_u_f16_e64
+v_cmp_gt_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3c,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_u_f16 vcc, v127, v255
-// GFX11: v_cmp_u_f16_e64
+v_cmp_gt_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_gt_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3c,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_u_f16 vcc, vcc_hi, v255
-// GFX11: v_cmp_u_f16_e64
+v_cmp_gt_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_u_f16 vcc, vcc_lo, v255
-// GFX11: v_cmp_u_f16_e64
+v_cmp_gt_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_gt_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_class_f16 vcc, v128, v2
-// GFX11: v_cmp_class_f16_e64
+v_cmp_gt_u16 vcc, vcc_hi, v255
+// GFX11: v_cmp_gt_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x3c,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_class_f16 vcc, v128, v2
-// GFX11: v_cmp_class_f16_e64
+v_cmp_gt_u16 vcc, vcc_hi, v255
+// GFX11: v_cmp_gt_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x3c,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_eq_f16 vcc, v128, v2
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_gt_u16 vcc, vcc_lo, v255
+// GFX11: v_cmp_gt_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x3c,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_eq_f16 vcc, v128, v2
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_gt_u16 vcc, vcc_lo, v255
+// GFX11: v_cmp_gt_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x3c,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_eq_i16 vcc, v128, v2
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_le_f16 vcc, v1, v255
+// GFX11: v_cmp_le_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x03,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_eq_i16 vcc, v128, v2
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_le_f16 vcc, v1, v255
+// GFX11: v_cmp_le_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x03,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_eq_u16 vcc, v128, v2
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_le_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x03,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_eq_u16 vcc, v128, v2
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_le_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x03,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_f_f16 vcc, v128, v2
-// GFX11: v_cmp_f_f16_e64
+v_cmp_le_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x03,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_f_f16 vcc, v128, v2
-// GFX11: v_cmp_f_f16_e64
+v_cmp_le_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x03,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_ge_f16 vcc, v128, v2
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_le_f16 vcc, v127, v255
+// GFX11: v_cmp_le_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x03,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_ge_f16 vcc, v128, v2
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_le_f16 vcc, v127, v255
+// GFX11: v_cmp_le_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x03,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_ge_i16 vcc, v128, v2
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_le_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x03,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_ge_i16 vcc, v128, v2
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_le_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x03,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_ge_u16 vcc, v128, v2
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_le_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x03,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ge_u16 vcc, v128, v2
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_le_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x03,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_gt_f16 vcc, v128, v2
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_le_f16 vcc, v128, v2
+// GFX11: v_cmp_le_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x03,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_gt_f16 vcc, v128, v2
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_le_f16 vcc, v128, v2
+// GFX11: v_cmp_le_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x03,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_gt_i16 vcc, v128, v2
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_le_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x03,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_gt_i16 vcc, v128, v2
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_le_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x03,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_gt_u16 vcc, v128, v2
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_le_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_gt_u16 vcc, v128, v2
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_le_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_le_f16 vcc, v128, v2
-// GFX11: v_cmp_le_f16_e64
+v_cmp_le_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_le_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x03,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_le_f16 vcc, v128, v2
-// GFX11: v_cmp_le_f16_e64
+v_cmp_le_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_le_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x03,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_le_i16 vcc, v128, v2
-// GFX11: v_cmp_le_i16_e64
+v_cmp_le_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_le_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x03,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_le_i16 vcc, v128, v2
-// GFX11: v_cmp_le_i16_e64
+v_cmp_le_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_le_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x03,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_le_u16 vcc, v128, v2
-// GFX11: v_cmp_le_u16_e64
+v_cmp_le_i16 vcc, v1, v255
+// GFX11: v_cmp_le_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x33,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_le_u16 vcc, v128, v2
-// GFX11: v_cmp_le_u16_e64
+v_cmp_le_i16 vcc, v1, v255
+// GFX11: v_cmp_le_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x33,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_lg_f16 vcc, v128, v2
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_le_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x33,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_lg_f16 vcc, v128, v2
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_le_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x33,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_lt_f16 vcc, v128, v2
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_le_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x33,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_lt_f16 vcc, v128, v2
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_le_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x33,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_lt_i16 vcc, v128, v2
-// GFX11: v_cmp_lt_i16_e64
+v_cmp_le_i16 vcc, v127, v255
+// GFX11: v_cmp_le_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x33,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_lt_i16 vcc, v128, v2
-// GFX11: v_cmp_lt_i16_e64
+v_cmp_le_i16 vcc, v127, v255
+// GFX11: v_cmp_le_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x33,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_lt_u16 vcc, v128, v2
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_le_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x33,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_lt_u16 vcc, v128, v2
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_le_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x33,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_ne_i16 vcc, v128, v2
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_le_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x33,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ne_i16 vcc, v128, v2
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_le_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x33,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ne_u16 vcc, v128, v2
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_le_i16 vcc, v128, v2
+// GFX11: v_cmp_le_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x33,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ne_u16 vcc, v128, v2
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_le_i16 vcc, v128, v2
+// GFX11: v_cmp_le_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x33,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_neq_f16 vcc, v128, v2
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_le_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x33,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_neq_f16 vcc, v128, v2
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_le_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x33,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_nge_f16 vcc, v128, v2
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_le_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_nge_f16 vcc, v128, v2
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_le_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_ngt_f16 vcc, v128, v2
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_le_i16 vcc, vcc_hi, v255
+// GFX11: v_cmp_le_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x33,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ngt_f16 vcc, v128, v2
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_le_i16 vcc, vcc_hi, v255
+// GFX11: v_cmp_le_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x33,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_nle_f16 vcc, v128, v2
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_le_i16 vcc, vcc_lo, v255
+// GFX11: v_cmp_le_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x33,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_nle_f16 vcc, v128, v2
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_le_i16 vcc, vcc_lo, v255
+// GFX11: v_cmp_le_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x33,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_nlg_f16 vcc, v128, v2
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_le_u16 vcc, v1, v255
+// GFX11: v_cmp_le_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x3b,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_nlg_f16 vcc, v128, v2
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_le_u16 vcc, v1, v255
+// GFX11: v_cmp_le_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x3b,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_nlt_f16 vcc, v128, v2
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_le_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_nlt_f16 vcc, v128, v2
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_le_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_o_f16 vcc, v128, v2
-// GFX11: v_cmp_o_f16_e64
+v_cmp_le_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_o_f16 vcc, v128, v2
-// GFX11: v_cmp_o_f16_e64
+v_cmp_le_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_t_f16 vcc, v128, v2
-// GFX11: v_cmp_t_f16_e64
+v_cmp_le_u16 vcc, v127, v255
+// GFX11: v_cmp_le_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x3b,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_t_f16 vcc, v128, v2
-// GFX11: v_cmp_t_f16_e64
+v_cmp_le_u16 vcc, v127, v255
+// GFX11: v_cmp_le_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x3b,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_tru_f16 vcc, v128, v2
-// GFX11: v_cmp_t_f16_e64
+v_cmp_le_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3b,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_tru_f16 vcc, v128, v2
-// GFX11: v_cmp_t_f16_e64
+v_cmp_le_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3b,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_u_f16 vcc, v128, v2
-// GFX11: v_cmp_u_f16_e64
+v_cmp_le_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3b,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_u_f16 vcc, v128, v2
-// GFX11: v_cmp_u_f16_e64
+v_cmp_le_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3b,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_class_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_class_f16_e64
+v_cmp_le_u16 vcc, v128, v2
+// GFX11: v_cmp_le_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x3b,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_class_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_class_f16_e64
+v_cmp_le_u16 vcc, v128, v2
+// GFX11: v_cmp_le_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x3b,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_class_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_class_f16_e64
+v_cmp_le_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3b,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_eq_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_le_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_le_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3b,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_eq_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_le_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_eq_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_le_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_le_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_eq_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_le_u16 vcc, vcc_hi, v255
+// GFX11: v_cmp_le_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x3b,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_eq_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_le_u16 vcc, vcc_hi, v255
+// GFX11: v_cmp_le_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x3b,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_eq_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_le_u16 vcc, vcc_lo, v255
+// GFX11: v_cmp_le_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x3b,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_eq_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_le_u16 vcc, vcc_lo, v255
+// GFX11: v_cmp_le_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x3b,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_eq_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_lg_f16 vcc, v1, v255
+// GFX11: v_cmp_lg_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x05,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_eq_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_lg_f16 vcc, v1, v255
+// GFX11: v_cmp_lg_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x05,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_eq_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_lg_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lg_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x05,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_eq_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_lg_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lg_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x05,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_eq_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_lg_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lg_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x05,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_f_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_f_f16_e64
+v_cmp_lg_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lg_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x05,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_f_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_f_f16_e64
+v_cmp_lg_f16 vcc, v127, v255
+// GFX11: v_cmp_lg_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x05,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_f_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_f_f16_e64
+v_cmp_lg_f16 vcc, v127, v255
+// GFX11: v_cmp_lg_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x05,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_f_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_f_f16_e64
+v_cmp_lg_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lg_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x05,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_ge_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_lg_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lg_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x05,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_ge_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_lg_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lg_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x05,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ge_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_lg_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lg_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x05,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ge_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_lg_f16 vcc, v128, v2
+// GFX11: v_cmp_lg_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x05,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ge_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_lg_f16 vcc, v128, v2
+// GFX11: v_cmp_lg_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x05,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ge_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_lg_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lg_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x05,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_ge_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_lg_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lg_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x05,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_ge_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_lg_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lg_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_ge_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_lg_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lg_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_ge_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_lg_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_lg_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x05,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ge_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_lg_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_lg_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x05,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ge_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_lg_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_lg_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x05,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_gt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_lg_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_lg_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x05,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_gt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_lt_f16 vcc, v1, v255
+// GFX11: v_cmp_lt_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x01,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_gt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_lt_f16 vcc, v1, v255
+// GFX11: v_cmp_lt_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x01,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_gt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_lt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x01,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_gt_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_lt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x01,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_gt_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_lt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x01,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_gt_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_lt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x01,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_gt_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_lt_f16 vcc, v127, v255
+// GFX11: v_cmp_lt_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x01,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_gt_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_lt_f16 vcc, v127, v255
+// GFX11: v_cmp_lt_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x01,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_gt_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_lt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x01,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_gt_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_lt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x01,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_gt_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_lt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x01,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_le_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_f16_e64
+v_cmp_lt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x01,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_le_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_f16_e64
+v_cmp_lt_f16 vcc, v128, v2
+// GFX11: v_cmp_lt_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x01,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_le_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_f16_e64
+v_cmp_lt_f16 vcc, v128, v2
+// GFX11: v_cmp_lt_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x01,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_le_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_f16_e64
+v_cmp_lt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x01,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_le_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_i16_e64
+v_cmp_lt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x01,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_le_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_i16_e64
+v_cmp_lt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_le_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_i16_e64
+v_cmp_lt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_le_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_i16_e64
+v_cmp_lt_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_lt_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x01,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_le_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_u16_e64
+v_cmp_lt_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_lt_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x01,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_le_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_u16_e64
+v_cmp_lt_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_lt_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x01,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_le_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_u16_e64
+v_cmp_lt_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_lt_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x01,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_le_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_u16_e64
+v_cmp_lt_i16 vcc, v1, v255
+// GFX11: v_cmp_lt_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x31,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_lg_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_lt_i16 vcc, v1, v255
+// GFX11: v_cmp_lt_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x31,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_lg_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_lt_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x31,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_lg_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_lt_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x31,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_lg_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_lt_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x31,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_lt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_lt_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x31,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_lt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_lt_i16 vcc, v127, v255
+// GFX11: v_cmp_lt_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x31,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_lt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_lt_i16 vcc, v127, v255
+// GFX11: v_cmp_lt_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x31,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_lt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_lt_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x31,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_lt_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_i16_e64
+v_cmp_lt_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x31,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
 v_cmp_lt_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_i16_e64
-
-v_cmp_lt_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_i16_e64
+// GFX11: v_cmp_lt_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x31,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
 v_cmp_lt_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_i16_e64
+// GFX11: v_cmp_lt_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x31,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_lt_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_lt_i16 vcc, v128, v2
+// GFX11: v_cmp_lt_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x31,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_lt_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_lt_i16 vcc, v128, v2
+// GFX11: v_cmp_lt_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x31,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_lt_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_lt_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x31,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_lt_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_lt_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x31,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_ne_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_lt_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_ne_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_lt_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_ne_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_lt_i16 vcc, vcc_hi, v255
+// GFX11: v_cmp_lt_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x31,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ne_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_lt_i16 vcc, vcc_hi, v255
+// GFX11: v_cmp_lt_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x31,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ne_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_lt_i16 vcc, vcc_lo, v255
+// GFX11: v_cmp_lt_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x31,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_ne_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_lt_i16 vcc, vcc_lo, v255
+// GFX11: v_cmp_lt_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x31,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_ne_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_lt_u16 vcc, v1, v255
+// GFX11: v_cmp_lt_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x39,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_ne_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_lt_u16 vcc, v1, v255
+// GFX11: v_cmp_lt_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x39,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_neq_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_lt_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x39,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_neq_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_lt_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x39,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_neq_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_lt_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x39,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_neq_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_lt_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x39,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_nge_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_lt_u16 vcc, v127, v255
+// GFX11: v_cmp_lt_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x39,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_nge_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_lt_u16 vcc, v127, v255
+// GFX11: v_cmp_lt_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x39,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_nge_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_lt_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x39,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_nge_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_lt_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x39,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_ngt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_lt_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x39,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ngt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_lt_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x39,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ngt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_lt_u16 vcc, v128, v2
+// GFX11: v_cmp_lt_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x39,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ngt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_lt_u16 vcc, v128, v2
+// GFX11: v_cmp_lt_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x39,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_nle_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_lt_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x39,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_nle_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_lt_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_lt_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x39,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_nle_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_lt_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_nle_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_lt_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_lt_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_nlg_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_lt_u16 vcc, vcc_hi, v255
+// GFX11: v_cmp_lt_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x39,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_nlg_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_lt_u16 vcc, vcc_hi, v255
+// GFX11: v_cmp_lt_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x39,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_nlg_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_lt_u16 vcc, vcc_lo, v255
+// GFX11: v_cmp_lt_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x39,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_nlg_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_lt_u16 vcc, vcc_lo, v255
+// GFX11: v_cmp_lt_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x39,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_nlt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_ne_i16 vcc, v1, v255
+// GFX11: v_cmp_ne_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x35,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_nlt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_ne_i16 vcc, v1, v255
+// GFX11: v_cmp_ne_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x35,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_nlt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_ne_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ne_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x35,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_nlt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_ne_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ne_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x35,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_o_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_o_f16_e64
+v_cmp_ne_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ne_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x35,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_o_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_o_f16_e64
+v_cmp_ne_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ne_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x35,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_o_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_o_f16_e64
+v_cmp_ne_i16 vcc, v127, v255
+// GFX11: v_cmp_ne_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x35,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_o_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_o_f16_e64
+v_cmp_ne_i16 vcc, v127, v255
+// GFX11: v_cmp_ne_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x35,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_t_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_ne_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ne_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x35,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_t_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_ne_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ne_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x35,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_t_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_ne_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ne_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x35,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_t_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_ne_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ne_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x35,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_tru_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_ne_i16 vcc, v128, v2
+// GFX11: v_cmp_ne_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x35,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_tru_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_ne_i16 vcc, v128, v2
+// GFX11: v_cmp_ne_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x35,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_tru_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_ne_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ne_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x35,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_tru_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_ne_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ne_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x35,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_u_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_u_f16_e64
+v_cmp_ne_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ne_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_u_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_u_f16_e64
+v_cmp_ne_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ne_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_u_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_u_f16_e64
+v_cmp_ne_i16 vcc, vcc_hi, v255
+// GFX11: v_cmp_ne_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x35,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_u_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_u_f16_e64
+v_cmp_ne_i16 vcc, vcc_hi, v255
+// GFX11: v_cmp_ne_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x35,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_class_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_class_f16_e64
+v_cmp_ne_i16 vcc, vcc_lo, v255
+// GFX11: v_cmp_ne_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x35,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_class_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_class_f16_e64
+v_cmp_ne_i16 vcc, vcc_lo, v255
+// GFX11: v_cmp_ne_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x35,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_eq_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_ne_u16 vcc, v1, v255
+// GFX11: v_cmp_ne_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x3d,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_eq_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_ne_u16 vcc, v1, v255
+// GFX11: v_cmp_ne_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x3d,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_eq_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_ne_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ne_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_eq_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_ne_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ne_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_eq_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_ne_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ne_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_eq_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_ne_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ne_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_f_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_f_f16_e64
+v_cmp_ne_u16 vcc, v127, v255
+// GFX11: v_cmp_ne_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x3d,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_f_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_f_f16_e64
+v_cmp_ne_u16 vcc, v127, v255
+// GFX11: v_cmp_ne_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x3d,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_ge_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_ne_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ne_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3d,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_ge_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_ne_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ne_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3d,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_ge_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_ne_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ne_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3d,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ge_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_ne_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ne_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3d,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ge_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_ne_u16 vcc, v128, v2
+// GFX11: v_cmp_ne_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x3d,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ge_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_ne_u16 vcc, v128, v2
+// GFX11: v_cmp_ne_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x3d,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_gt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_ne_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ne_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3d,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_gt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_ne_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ne_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3d,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_gt_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_ne_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ne_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_gt_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_ne_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ne_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_gt_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_ne_u16 vcc, vcc_hi, v255
+// GFX11: v_cmp_ne_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x3d,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_gt_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_ne_u16 vcc, vcc_hi, v255
+// GFX11: v_cmp_ne_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x3d,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_le_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_f16_e64
+v_cmp_ne_u16 vcc, vcc_lo, v255
+// GFX11: v_cmp_ne_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x3d,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_le_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_f16_e64
+v_cmp_ne_u16 vcc, vcc_lo, v255
+// GFX11: v_cmp_ne_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x3d,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_le_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_i16_e64
+v_cmp_neq_f16 vcc, v1, v255
+// GFX11: v_cmp_neq_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x0d,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_le_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_i16_e64
+v_cmp_neq_f16 vcc, v1, v255
+// GFX11: v_cmp_neq_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x0d,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_le_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_u16_e64
+v_cmp_neq_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_neq_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_le_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_le_u16_e64
+v_cmp_neq_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_neq_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_lg_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_neq_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_neq_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_lg_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_neq_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_neq_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_lt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_neq_f16 vcc, v127, v255
+// GFX11: v_cmp_neq_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x0d,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_lt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_neq_f16 vcc, v127, v255
+// GFX11: v_cmp_neq_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x0d,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_lt_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_i16_e64
+v_cmp_neq_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_neq_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0d,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_lt_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_i16_e64
+v_cmp_neq_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_neq_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0d,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_lt_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_neq_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_neq_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0d,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_lt_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_neq_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_neq_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0d,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ne_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_neq_f16 vcc, v128, v2
+// GFX11: v_cmp_neq_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x0d,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ne_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_neq_f16 vcc, v128, v2
+// GFX11: v_cmp_neq_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x0d,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ne_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_neq_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_neq_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0d,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_ne_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_neq_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_neq_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0d,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
 v_cmp_neq_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_neq_f16_e64
+// GFX11: v_cmp_neq_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
 v_cmp_neq_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_neq_f16_e64
+// GFX11: v_cmp_neq_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_nge_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_neq_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_neq_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x0d,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_nge_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_neq_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_neq_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x0d,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ngt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_neq_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_neq_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x0d,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_ngt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_neq_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_neq_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x0d,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_nle_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_nge_f16 vcc, v1, v255
+// GFX11: v_cmp_nge_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x09,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_nle_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_nge_f16 vcc, v1, v255
+// GFX11: v_cmp_nge_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x09,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_nlg_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_nge_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nge_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x09,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_nlg_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_nge_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nge_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x09,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_nlt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_nge_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nge_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x09,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_nlt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_nge_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nge_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x09,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_o_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_o_f16_e64
+v_cmp_nge_f16 vcc, v127, v255
+// GFX11: v_cmp_nge_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x09,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_o_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_o_f16_e64
+v_cmp_nge_f16 vcc, v127, v255
+// GFX11: v_cmp_nge_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x09,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_t_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_nge_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nge_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x09,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_t_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_nge_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nge_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x09,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_tru_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_nge_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nge_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x09,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_tru_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_nge_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nge_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x09,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_u_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_u_f16_e64
+v_cmp_nge_f16 vcc, v128, v2
+// GFX11: v_cmp_nge_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x09,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_u_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmp_u_f16_e64
+v_cmp_nge_f16 vcc, v128, v2
+// GFX11: v_cmp_nge_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x09,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_class_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_class_f16_e64
+v_cmp_nge_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nge_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x09,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_class_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_class_f16_e64
+v_cmp_nge_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nge_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x09,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_class_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_class_f16_e64
+v_cmp_nge_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nge_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_eq_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_nge_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nge_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_eq_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_nge_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_nge_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x09,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_eq_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_nge_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_nge_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x09,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_eq_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_nge_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_nge_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x09,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_eq_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_nge_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_nge_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x09,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_eq_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_ngt_f16 vcc, v1, v255
+// GFX11: v_cmp_ngt_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x0b,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_eq_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_ngt_f16 vcc, v1, v255
+// GFX11: v_cmp_ngt_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x0b,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_eq_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_ngt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ngt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_eq_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_ngt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ngt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_eq_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_ngt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ngt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_eq_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_ngt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ngt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_eq_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_ngt_f16 vcc, v127, v255
+// GFX11: v_cmp_ngt_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x0b,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_f_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_f_f16_e64
+v_cmp_ngt_f16 vcc, v127, v255
+// GFX11: v_cmp_ngt_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x0b,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_f_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_f_f16_e64
+v_cmp_ngt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ngt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0b,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_f_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_f_f16_e64
+v_cmp_ngt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ngt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0b,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_f_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_f_f16_e64
+v_cmp_ngt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ngt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0b,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ge_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_ngt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ngt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0b,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ge_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_ngt_f16 vcc, v128, v2
+// GFX11: v_cmp_ngt_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x0b,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ge_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_ngt_f16 vcc, v128, v2
+// GFX11: v_cmp_ngt_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x0b,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ge_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_ngt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ngt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0b,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_ge_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_ngt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_ngt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0b,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_ge_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_ngt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ngt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_ge_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_ngt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_ngt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_ge_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_ngt_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_ngt_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x0b,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ge_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_ngt_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_ngt_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x0b,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ge_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_ngt_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_ngt_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x0b,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_ge_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_ngt_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_ngt_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x0b,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_ge_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_nle_f16 vcc, v1, v255
+// GFX11: v_cmp_nle_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x0c,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_gt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_nle_f16 vcc, v1, v255
+// GFX11: v_cmp_nle_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x0c,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_gt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_nle_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nle_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_gt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_nle_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nle_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_gt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_nle_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nle_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_gt_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_nle_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nle_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_gt_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_nle_f16 vcc, v127, v255
+// GFX11: v_cmp_nle_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x0c,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_gt_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_nle_f16 vcc, v127, v255
+// GFX11: v_cmp_nle_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x0c,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_gt_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_nle_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nle_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0c,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_gt_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_nle_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nle_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0c,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_gt_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_nle_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nle_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0c,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_gt_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_nle_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nle_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0c,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_gt_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_nle_f16 vcc, v128, v2
+// GFX11: v_cmp_nle_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x0c,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_le_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_f16_e64
+v_cmp_nle_f16 vcc, v128, v2
+// GFX11: v_cmp_nle_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x0c,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_le_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_f16_e64
+v_cmp_nle_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nle_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0c,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_le_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_f16_e64
+v_cmp_nle_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nle_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0c,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_le_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_f16_e64
+v_cmp_nle_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nle_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_le_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_i16_e64
+v_cmp_nle_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nle_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_le_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_i16_e64
+v_cmp_nle_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_nle_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x0c,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_le_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_i16_e64
+v_cmp_nle_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_nle_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x0c,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_le_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_i16_e64
+v_cmp_nle_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_nle_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x0c,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_le_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_u16_e64
+v_cmp_nle_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_nle_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x0c,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_le_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_u16_e64
+v_cmp_nlg_f16 vcc, v1, v255
+// GFX11: v_cmp_nlg_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x0a,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_le_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_u16_e64
+v_cmp_nlg_f16 vcc, v1, v255
+// GFX11: v_cmp_nlg_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x0a,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_le_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_u16_e64
+v_cmp_nlg_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nlg_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_lg_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_nlg_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nlg_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_lg_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_nlg_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nlg_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_lg_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_nlg_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nlg_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_lg_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_nlg_f16 vcc, v127, v255
+// GFX11: v_cmp_nlg_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x0a,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_lt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_nlg_f16 vcc, v127, v255
+// GFX11: v_cmp_nlg_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x0a,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_lt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_nlg_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nlg_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0a,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_lt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_nlg_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nlg_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0a,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_lt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_nlg_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nlg_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0a,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_lt_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_i16_e64
+v_cmp_nlg_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nlg_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0a,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_lt_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_i16_e64
+v_cmp_nlg_f16 vcc, v128, v2
+// GFX11: v_cmp_nlg_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x0a,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_lt_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_i16_e64
+v_cmp_nlg_f16 vcc, v128, v2
+// GFX11: v_cmp_nlg_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x0a,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_lt_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_i16_e64
+v_cmp_nlg_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nlg_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0a,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_lt_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_nlg_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nlg_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0a,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_lt_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_nlg_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nlg_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_lt_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_nlg_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nlg_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_lt_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_nlg_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_nlg_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x0a,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ne_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_nlg_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_nlg_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x0a,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_ne_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_nlg_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_nlg_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x0a,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_ne_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_nlg_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_nlg_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x0a,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_ne_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_nlt_f16 vcc, v1, v255
+// GFX11: v_cmp_nlt_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x0e,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_ne_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_nlt_f16 vcc, v1, v255
+// GFX11: v_cmp_nlt_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x0e,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_ne_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_nlt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nlt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_ne_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_nlt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nlt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_ne_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_nlt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nlt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_neq_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_nlt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nlt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_neq_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_nlt_f16 vcc, v127, v255
+// GFX11: v_cmp_nlt_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x0e,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_neq_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_nlt_f16 vcc, v127, v255
+// GFX11: v_cmp_nlt_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x0e,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_neq_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_nlt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nlt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0e,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_nge_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_nlt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nlt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0e,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_nge_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_nlt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nlt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0e,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_nge_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_nlt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nlt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0e,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_nge_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_nlt_f16 vcc, v128, v2
+// GFX11: v_cmp_nlt_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x0e,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ngt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_nlt_f16 vcc, v128, v2
+// GFX11: v_cmp_nlt_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x0e,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ngt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_nlt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nlt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0e,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_ngt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_nlt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_nlt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0e,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_ngt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_nlt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nlt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_nle_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_nlt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_nlt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_nle_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_nlt_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_nlt_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x0e,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_nle_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_nlt_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_nlt_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x0e,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_nle_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_nlt_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_nlt_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x0e,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_nlg_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_nlt_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_nlt_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x0e,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_nlg_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_o_f16 vcc, v1, v255
+// GFX11: v_cmp_o_f16_e64 vcc, v1, v255           ; encoding: [0x6a,0x00,0x07,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_nlg_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_o_f16 vcc, v1, v255
+// GFX11: v_cmp_o_f16_e64 vcc, v1, v255           ; encoding: [0x6a,0x00,0x07,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_nlg_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_o_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_o_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x07,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_nlt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_o_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_o_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x07,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_nlt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_o_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_o_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x07,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_nlt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_o_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_o_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x07,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_nlt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_o_f16 vcc, v127, v255
+// GFX11: v_cmp_o_f16_e64 vcc, v127, v255         ; encoding: [0x6a,0x00,0x07,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_o_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_o_f16_e64
+v_cmp_o_f16 vcc, v127, v255
+// GFX11: v_cmp_o_f16_e64 vcc, v127, v255         ; encoding: [0x6a,0x00,0x07,0xd4,0x7f,0xff,0x03,0x00]
 
 v_cmp_o_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_o_f16_e64
-
-v_cmp_o_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_o_f16_e64
+// GFX11: v_cmp_o_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x07,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
 v_cmp_o_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_o_f16_e64
+// GFX11: v_cmp_o_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x07,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_t_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_o_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_o_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x07,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_t_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_o_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_o_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x07,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_t_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_o_f16 vcc, v128, v2
+// GFX11: v_cmp_o_f16_e64 vcc, v128, v2           ; encoding: [0x6a,0x00,0x07,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_t_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_o_f16 vcc, v128, v2
+// GFX11: v_cmp_o_f16_e64 vcc, v128, v2           ; encoding: [0x6a,0x00,0x07,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_tru_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_o_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_o_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x07,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_tru_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_o_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_o_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x07,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_tru_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_o_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_o_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_tru_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_o_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_o_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_u_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_u_f16_e64
+v_cmp_o_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_o_f16_e64 vcc, vcc_hi, v255       ; encoding: [0x6a,0x00,0x07,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_u_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_u_f16_e64
+v_cmp_o_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_o_f16_e64 vcc, vcc_hi, v255       ; encoding: [0x6a,0x00,0x07,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_u_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_u_f16_e64
+v_cmp_o_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_o_f16_e64 vcc, vcc_lo, v255       ; encoding: [0x6a,0x00,0x07,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_u_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_u_f16_e64
+v_cmp_o_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_o_f16_e64 vcc, vcc_lo, v255       ; encoding: [0x6a,0x00,0x07,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_class_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_class_f16_e64
+v_cmp_t_f16 vcc, v1, v255
+// GFX11: v_cmp_t_f16_e64 vcc, v1, v255           ; encoding: [0x6a,0x00,0x0f,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_class_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_class_f16_e64
+v_cmp_t_f16 vcc, v1, v255
+// GFX11: v_cmp_t_f16_e64 vcc, v1, v255           ; encoding: [0x6a,0x00,0x0f,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_eq_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_t_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0f,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_eq_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_f16_e64
+v_cmp_t_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0f,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_eq_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_t_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0f,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_eq_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_i16_e64
+v_cmp_t_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0f,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_eq_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_t_f16 vcc, v127, v255
+// GFX11: v_cmp_t_f16_e64 vcc, v127, v255         ; encoding: [0x6a,0x00,0x0f,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_eq_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_eq_u16_e64
+v_cmp_t_f16 vcc, v127, v255
+// GFX11: v_cmp_t_f16_e64 vcc, v127, v255         ; encoding: [0x6a,0x00,0x0f,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_f_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_f_f16_e64
+v_cmp_t_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0f,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_f_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_f_f16_e64
+v_cmp_t_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0f,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_ge_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_t_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0f,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ge_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_f16_e64
+v_cmp_t_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0f,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_ge_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_t_f16 vcc, v128, v2
+// GFX11: v_cmp_t_f16_e64 vcc, v128, v2           ; encoding: [0x6a,0x00,0x0f,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ge_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_i16_e64
+v_cmp_t_f16 vcc, v128, v2
+// GFX11: v_cmp_t_f16_e64 vcc, v128, v2           ; encoding: [0x6a,0x00,0x0f,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ge_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_t_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0f,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_ge_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ge_u16_e64
+v_cmp_t_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0f,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_gt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_t_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_gt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_f16_e64
+v_cmp_t_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_gt_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_t_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_t_f16_e64 vcc, vcc_hi, v255       ; encoding: [0x6a,0x00,0x0f,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_gt_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_i16_e64
+v_cmp_t_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_t_f16_e64 vcc, vcc_hi, v255       ; encoding: [0x6a,0x00,0x0f,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_gt_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_t_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_t_f16_e64 vcc, vcc_lo, v255       ; encoding: [0x6a,0x00,0x0f,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_gt_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_gt_u16_e64
+v_cmp_t_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_t_f16_e64 vcc, vcc_lo, v255       ; encoding: [0x6a,0x00,0x0f,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_le_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_f16_e64
+v_cmp_tru_f16 vcc, v1, v255
+// GFX11: v_cmp_t_f16_e64 vcc, v1, v255           ; encoding: [0x6a,0x00,0x0f,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_le_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_f16_e64
+v_cmp_tru_f16 vcc, v1, v255
+// GFX11: v_cmp_t_f16_e64 vcc, v1, v255           ; encoding: [0x6a,0x00,0x0f,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_le_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_i16_e64
+v_cmp_tru_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0f,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_le_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_i16_e64
+v_cmp_tru_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0f,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_le_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_u16_e64
+v_cmp_tru_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0f,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_le_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_le_u16_e64
+v_cmp_tru_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0f,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_lg_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_tru_f16 vcc, v127, v255
+// GFX11: v_cmp_t_f16_e64 vcc, v127, v255         ; encoding: [0x6a,0x00,0x0f,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_lg_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lg_f16_e64
+v_cmp_tru_f16 vcc, v127, v255
+// GFX11: v_cmp_t_f16_e64 vcc, v127, v255         ; encoding: [0x6a,0x00,0x0f,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_lt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_tru_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0f,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_lt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_f16_e64
+v_cmp_tru_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0f,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_lt_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_i16_e64
+v_cmp_tru_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0f,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_lt_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_i16_e64
+v_cmp_tru_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0f,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_lt_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_tru_f16 vcc, v128, v2
+// GFX11: v_cmp_t_f16_e64 vcc, v128, v2           ; encoding: [0x6a,0x00,0x0f,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_lt_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_lt_u16_e64
+v_cmp_tru_f16 vcc, v128, v2
+// GFX11: v_cmp_t_f16_e64 vcc, v128, v2           ; encoding: [0x6a,0x00,0x0f,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_ne_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_tru_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0f,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_ne_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ne_i16_e64
+v_cmp_tru_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0f,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
-v_cmp_ne_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_tru_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_ne_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ne_u16_e64
+v_cmp_tru_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_t_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
 
-v_cmp_neq_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_tru_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_t_f16_e64 vcc, vcc_hi, v255       ; encoding: [0x6a,0x00,0x0f,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_neq_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_neq_f16_e64
+v_cmp_tru_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_t_f16_e64 vcc, vcc_hi, v255       ; encoding: [0x6a,0x00,0x0f,0xd4,0x6b,0xfe,0x03,0x00]
 
-v_cmp_nge_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_tru_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_t_f16_e64 vcc, vcc_lo, v255       ; encoding: [0x6a,0x00,0x0f,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_nge_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nge_f16_e64
+v_cmp_tru_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_t_f16_e64 vcc, vcc_lo, v255       ; encoding: [0x6a,0x00,0x0f,0xd4,0x6a,0xfe,0x03,0x00]
 
-v_cmp_ngt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_u_f16 vcc, v1, v255
+// GFX11: v_cmp_u_f16_e64 vcc, v1, v255           ; encoding: [0x6a,0x00,0x08,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_ngt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_ngt_f16_e64
+v_cmp_u_f16 vcc, v1, v255
+// GFX11: v_cmp_u_f16_e64 vcc, v1, v255           ; encoding: [0x6a,0x00,0x08,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmp_nle_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_u_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_u_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x08,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_nle_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nle_f16_e64
+v_cmp_u_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_u_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x08,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmp_nlg_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_u_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_u_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x08,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_nlg_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nlg_f16_e64
+v_cmp_u_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_u_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x08,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmp_nlt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_u_f16 vcc, v127, v255
+// GFX11: v_cmp_u_f16_e64 vcc, v127, v255         ; encoding: [0x6a,0x00,0x08,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_nlt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_nlt_f16_e64
+v_cmp_u_f16 vcc, v127, v255
+// GFX11: v_cmp_u_f16_e64 vcc, v127, v255         ; encoding: [0x6a,0x00,0x08,0xd4,0x7f,0xff,0x03,0x00]
 
-v_cmp_o_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_o_f16_e64
+v_cmp_u_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_u_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x08,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_o_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_o_f16_e64
+v_cmp_u_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_u_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x08,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
 
-v_cmp_t_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_u_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_u_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x08,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_t_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_u_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_u_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x08,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cmp_tru_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_u_f16 vcc, v128, v2
+// GFX11: v_cmp_u_f16_e64 vcc, v128, v2           ; encoding: [0x6a,0x00,0x08,0xd4,0x80,0x05,0x02,0x00]
 
-v_cmp_tru_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_t_f16_e64
+v_cmp_u_f16 vcc, v128, v2
+// GFX11: v_cmp_u_f16_e64 vcc, v128, v2           ; encoding: [0x6a,0x00,0x08,0xd4,0x80,0x05,0x02,0x00]
 
 v_cmp_u_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_u_f16_e64
+// GFX11: v_cmp_u_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x08,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
 
 v_cmp_u_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmp_u_f16_e64
+// GFX11: v_cmp_u_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x08,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+
+v_cmp_u_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_u_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+
+v_cmp_u_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_u_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+
+v_cmp_u_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_u_f16_e64 vcc, vcc_hi, v255       ; encoding: [0x6a,0x00,0x08,0xd4,0x6b,0xfe,0x03,0x00]
+
+v_cmp_u_f16 vcc, vcc_hi, v255
+// GFX11: v_cmp_u_f16_e64 vcc, vcc_hi, v255       ; encoding: [0x6a,0x00,0x08,0xd4,0x6b,0xfe,0x03,0x00]
+
+v_cmp_u_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_u_f16_e64 vcc, vcc_lo, v255       ; encoding: [0x6a,0x00,0x08,0xd4,0x6a,0xfe,0x03,0x00]
 
+v_cmp_u_f16 vcc, vcc_lo, v255
+// GFX11: v_cmp_u_f16_e64 vcc, vcc_lo, v255       ; encoding: [0x6a,0x00,0x08,0xd4,0x6a,0xfe,0x03,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx-fake16.s
new file mode 100644
index 000000000000..33a5e7c140b5
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx-fake16.s
@@ -0,0 +1,4106 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+
+v_cmpx_class_f16_e32 v1, v2
+// GFX11: encoding: [0x01,0x05,0xfa,0x7d]
+
+v_cmpx_class_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0xfa,0x7d]
+
+v_cmpx_class_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0xfa,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_class_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0xfc,0x7d]
+
+v_cmpx_class_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0xfc,0x7d]
+
+v_cmpx_class_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xfd,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_class_f64 v[1:2], v2
+// GFX11: encoding: [0x01,0x05,0xfe,0x7d]
+
+v_cmpx_class_f64 v[254:255], v2
+// GFX11: encoding: [0xfe,0x05,0xfe,0x7d]
+
+v_cmpx_class_f64 s[2:3], v2
+// GFX11: encoding: [0x02,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 s[104:105], v2
+// GFX11: encoding: [0x68,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 vcc, v2
+// GFX11: encoding: [0x6a,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 ttmp[14:15], v2
+// GFX11: encoding: [0x7a,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 exec, v2
+// GFX11: encoding: [0x7e,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 null, v2
+// GFX11: encoding: [0x7c,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 -1, v2
+// GFX11: encoding: [0xc1,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x04,0x7d]
+
+v_cmpx_eq_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x04,0x7d]
+
+v_cmpx_eq_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x04,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x24,0x7d]
+
+v_cmpx_eq_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x24,0x7d]
+
+v_cmpx_eq_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x25,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x44,0x7d]
+
+v_cmpx_eq_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x44,0x7d]
+
+v_cmpx_eq_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x45,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_i16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x64,0x7d]
+
+v_cmpx_eq_i16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x64,0x7d]
+
+v_cmpx_eq_i16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x64,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_i32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x84,0x7d]
+
+v_cmpx_eq_i32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x84,0x7d]
+
+v_cmpx_eq_i32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x85,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_i64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xa4,0x7d]
+
+v_cmpx_eq_i64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xa4,0x7d]
+
+v_cmpx_eq_i64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xa5,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_u16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x74,0x7d]
+
+v_cmpx_eq_u16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x74,0x7d]
+
+v_cmpx_eq_u16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x74,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_u32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x94,0x7d]
+
+v_cmpx_eq_u32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x94,0x7d]
+
+v_cmpx_eq_u32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x95,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_u64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xb4,0x7d]
+
+v_cmpx_eq_u64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xb4,0x7d]
+
+v_cmpx_eq_u64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xb5,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x00,0x7d]
+
+v_cmpx_f_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x00,0x7d]
+
+v_cmpx_f_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x00,0x7d]
+
+v_cmpx_f_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x00,0x7d]
+
+v_cmpx_f_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x00,0x7d]
+
+v_cmpx_f_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x00,0x7d]
+
+v_cmpx_f_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x00,0x7d]
+
+v_cmpx_f_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x00,0x7d]
+
+v_cmpx_f_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x00,0x7d]
+
+v_cmpx_f_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x00,0x7d]
+
+v_cmpx_f_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x00,0x7d]
+
+v_cmpx_f_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x00,0x7d]
+
+v_cmpx_f_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x00,0x7d]
+
+v_cmpx_f_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x00,0x7d]
+
+v_cmpx_f_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x00,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_f_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x20,0x7d]
+
+v_cmpx_f_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x20,0x7d]
+
+v_cmpx_f_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x20,0x7d]
+
+v_cmpx_f_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x20,0x7d]
+
+v_cmpx_f_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x20,0x7d]
+
+v_cmpx_f_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x20,0x7d]
+
+v_cmpx_f_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x20,0x7d]
+
+v_cmpx_f_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x20,0x7d]
+
+v_cmpx_f_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x20,0x7d]
+
+v_cmpx_f_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x20,0x7d]
+
+v_cmpx_f_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x20,0x7d]
+
+v_cmpx_f_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x20,0x7d]
+
+v_cmpx_f_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x20,0x7d]
+
+v_cmpx_f_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x20,0x7d]
+
+v_cmpx_f_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x21,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x40,0x7d]
+
+v_cmpx_f_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x40,0x7d]
+
+v_cmpx_f_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x40,0x7d]
+
+v_cmpx_f_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x40,0x7d]
+
+v_cmpx_f_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x40,0x7d]
+
+v_cmpx_f_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x40,0x7d]
+
+v_cmpx_f_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x40,0x7d]
+
+v_cmpx_f_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x40,0x7d]
+
+v_cmpx_f_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x40,0x7d]
+
+v_cmpx_f_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x40,0x7d]
+
+v_cmpx_f_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x40,0x7d]
+
+v_cmpx_f_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x41,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_i32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x80,0x7d]
+
+v_cmpx_f_i32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x80,0x7d]
+
+v_cmpx_f_i32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x80,0x7d]
+
+v_cmpx_f_i32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x80,0x7d]
+
+v_cmpx_f_i32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x80,0x7d]
+
+v_cmpx_f_i32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x80,0x7d]
+
+v_cmpx_f_i32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x80,0x7d]
+
+v_cmpx_f_i32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x80,0x7d]
+
+v_cmpx_f_i32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x80,0x7d]
+
+v_cmpx_f_i32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x80,0x7d]
+
+v_cmpx_f_i32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x80,0x7d]
+
+v_cmpx_f_i32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x80,0x7d]
+
+v_cmpx_f_i32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x80,0x7d]
+
+v_cmpx_f_i32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x80,0x7d]
+
+v_cmpx_f_i32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x81,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_i64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xa0,0x7d]
+
+v_cmpx_f_i64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xa0,0x7d]
+
+v_cmpx_f_i64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xa0,0x7d]
+
+v_cmpx_f_i64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xa0,0x7d]
+
+v_cmpx_f_i64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xa0,0x7d]
+
+v_cmpx_f_i64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xa0,0x7d]
+
+v_cmpx_f_i64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xa0,0x7d]
+
+v_cmpx_f_i64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xa0,0x7d]
+
+v_cmpx_f_i64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xa0,0x7d]
+
+v_cmpx_f_i64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xa0,0x7d]
+
+v_cmpx_f_i64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xa0,0x7d]
+
+v_cmpx_f_i64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xa1,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_u32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x90,0x7d]
+
+v_cmpx_f_u32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x90,0x7d]
+
+v_cmpx_f_u32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x90,0x7d]
+
+v_cmpx_f_u32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x90,0x7d]
+
+v_cmpx_f_u32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x90,0x7d]
+
+v_cmpx_f_u32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x90,0x7d]
+
+v_cmpx_f_u32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x90,0x7d]
+
+v_cmpx_f_u32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x90,0x7d]
+
+v_cmpx_f_u32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x90,0x7d]
+
+v_cmpx_f_u32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x90,0x7d]
+
+v_cmpx_f_u32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x90,0x7d]
+
+v_cmpx_f_u32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x90,0x7d]
+
+v_cmpx_f_u32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x90,0x7d]
+
+v_cmpx_f_u32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x90,0x7d]
+
+v_cmpx_f_u32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x91,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_u64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xb0,0x7d]
+
+v_cmpx_f_u64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xb0,0x7d]
+
+v_cmpx_f_u64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xb0,0x7d]
+
+v_cmpx_f_u64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xb0,0x7d]
+
+v_cmpx_f_u64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xb0,0x7d]
+
+v_cmpx_f_u64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xb0,0x7d]
+
+v_cmpx_f_u64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xb0,0x7d]
+
+v_cmpx_f_u64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xb0,0x7d]
+
+v_cmpx_f_u64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xb0,0x7d]
+
+v_cmpx_f_u64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xb0,0x7d]
+
+v_cmpx_f_u64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xb0,0x7d]
+
+v_cmpx_f_u64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xb1,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x0c,0x7d]
+
+v_cmpx_ge_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x0c,0x7d]
+
+v_cmpx_ge_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x0c,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x2c,0x7d]
+
+v_cmpx_ge_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x2c,0x7d]
+
+v_cmpx_ge_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x2d,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x4c,0x7d]
+
+v_cmpx_ge_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x4c,0x7d]
+
+v_cmpx_ge_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x4d,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_i16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x6c,0x7d]
+
+v_cmpx_ge_i16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x6c,0x7d]
+
+v_cmpx_ge_i16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x6c,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_i32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x8c,0x7d]
+
+v_cmpx_ge_i32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x8c,0x7d]
+
+v_cmpx_ge_i32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x8d,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_i64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xac,0x7d]
+
+v_cmpx_ge_i64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xac,0x7d]
+
+v_cmpx_ge_i64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xad,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_u16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x7c,0x7d]
+
+v_cmpx_ge_u16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x7c,0x7d]
+
+v_cmpx_ge_u16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x7c,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_u32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x9c,0x7d]
+
+v_cmpx_ge_u32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x9c,0x7d]
+
+v_cmpx_ge_u32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x9d,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_u64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xbc,0x7d]
+
+v_cmpx_ge_u64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xbc,0x7d]
+
+v_cmpx_ge_u64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xbd,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x08,0x7d]
+
+v_cmpx_gt_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x08,0x7d]
+
+v_cmpx_gt_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x08,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x28,0x7d]
+
+v_cmpx_gt_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x28,0x7d]
+
+v_cmpx_gt_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x29,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x48,0x7d]
+
+v_cmpx_gt_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x48,0x7d]
+
+v_cmpx_gt_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x49,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_i16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x68,0x7d]
+
+v_cmpx_gt_i16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x68,0x7d]
+
+v_cmpx_gt_i16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x68,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_i32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x88,0x7d]
+
+v_cmpx_gt_i32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x88,0x7d]
+
+v_cmpx_gt_i32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x89,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_i64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xa8,0x7d]
+
+v_cmpx_gt_i64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xa8,0x7d]
+
+v_cmpx_gt_i64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xa9,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_u16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x78,0x7d]
+
+v_cmpx_gt_u16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x78,0x7d]
+
+v_cmpx_gt_u16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x78,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_u32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x98,0x7d]
+
+v_cmpx_gt_u32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x98,0x7d]
+
+v_cmpx_gt_u32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x99,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_u64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xb8,0x7d]
+
+v_cmpx_gt_u64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xb8,0x7d]
+
+v_cmpx_gt_u64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xb9,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x06,0x7d]
+
+v_cmpx_le_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x06,0x7d]
+
+v_cmpx_le_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x06,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x26,0x7d]
+
+v_cmpx_le_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x26,0x7d]
+
+v_cmpx_le_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x27,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x46,0x7d]
+
+v_cmpx_le_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x46,0x7d]
+
+v_cmpx_le_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x47,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_i16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x66,0x7d]
+
+v_cmpx_le_i16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x66,0x7d]
+
+v_cmpx_le_i16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x66,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_i32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x86,0x7d]
+
+v_cmpx_le_i32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x86,0x7d]
+
+v_cmpx_le_i32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x87,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_i64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xa6,0x7d]
+
+v_cmpx_le_i64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xa6,0x7d]
+
+v_cmpx_le_i64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xa7,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_u16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x76,0x7d]
+
+v_cmpx_le_u16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x76,0x7d]
+
+v_cmpx_le_u16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x76,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_u32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x96,0x7d]
+
+v_cmpx_le_u32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x96,0x7d]
+
+v_cmpx_le_u32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x97,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_u64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xb6,0x7d]
+
+v_cmpx_le_u64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xb6,0x7d]
+
+v_cmpx_le_u64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xb7,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lg_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x7d]
+
+v_cmpx_lg_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x0a,0x7d]
+
+v_cmpx_lg_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x0a,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lg_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x2a,0x7d]
+
+v_cmpx_lg_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x2a,0x7d]
+
+v_cmpx_lg_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x2b,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lg_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x4a,0x7d]
+
+v_cmpx_lg_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x4a,0x7d]
+
+v_cmpx_lg_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x4b,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x02,0x7d]
+
+v_cmpx_lt_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x02,0x7d]
+
+v_cmpx_lt_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x02,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x22,0x7d]
+
+v_cmpx_lt_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x22,0x7d]
+
+v_cmpx_lt_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x23,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x42,0x7d]
+
+v_cmpx_lt_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x42,0x7d]
+
+v_cmpx_lt_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x43,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_i16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x62,0x7d]
+
+v_cmpx_lt_i16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x62,0x7d]
+
+v_cmpx_lt_i16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x62,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_i32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x82,0x7d]
+
+v_cmpx_lt_i32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x82,0x7d]
+
+v_cmpx_lt_i32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x83,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_i64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xa2,0x7d]
+
+v_cmpx_lt_i64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xa2,0x7d]
+
+v_cmpx_lt_i64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xa3,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_u16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x72,0x7d]
+
+v_cmpx_lt_u16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x72,0x7d]
+
+v_cmpx_lt_u16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x72,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_u32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x92,0x7d]
+
+v_cmpx_lt_u32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x92,0x7d]
+
+v_cmpx_lt_u32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x93,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_u64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xb2,0x7d]
+
+v_cmpx_lt_u64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xb2,0x7d]
+
+v_cmpx_lt_u64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xb3,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_i16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x6a,0x7d]
+
+v_cmpx_ne_i16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x6a,0x7d]
+
+v_cmpx_ne_i16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x6a,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ne_i32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x8a,0x7d]
+
+v_cmpx_ne_i32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x8a,0x7d]
+
+v_cmpx_ne_i32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x8b,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_i64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xaa,0x7d]
+
+v_cmpx_ne_i64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xaa,0x7d]
+
+v_cmpx_ne_i64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xab,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_u16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x7a,0x7d]
+
+v_cmpx_ne_u16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x7a,0x7d]
+
+v_cmpx_ne_u16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x7a,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ne_u32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x9a,0x7d]
+
+v_cmpx_ne_u32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x9a,0x7d]
+
+v_cmpx_ne_u32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x9b,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_u64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xba,0x7d]
+
+v_cmpx_ne_u64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xba,0x7d]
+
+v_cmpx_ne_u64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xbb,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_neq_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x1a,0x7d]
+
+v_cmpx_neq_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x1a,0x7d]
+
+v_cmpx_neq_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x1a,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_neq_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x3a,0x7d]
+
+v_cmpx_neq_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x3a,0x7d]
+
+v_cmpx_neq_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x3b,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_neq_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x5a,0x7d]
+
+v_cmpx_neq_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x5a,0x7d]
+
+v_cmpx_neq_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x5b,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nge_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x12,0x7d]
+
+v_cmpx_nge_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x12,0x7d]
+
+v_cmpx_nge_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x12,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nge_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x32,0x7d]
+
+v_cmpx_nge_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x32,0x7d]
+
+v_cmpx_nge_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x33,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nge_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x52,0x7d]
+
+v_cmpx_nge_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x52,0x7d]
+
+v_cmpx_nge_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x53,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ngt_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x16,0x7d]
+
+v_cmpx_ngt_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x16,0x7d]
+
+v_cmpx_ngt_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x16,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ngt_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x36,0x7d]
+
+v_cmpx_ngt_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x36,0x7d]
+
+v_cmpx_ngt_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x37,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ngt_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x56,0x7d]
+
+v_cmpx_ngt_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x56,0x7d]
+
+v_cmpx_ngt_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x57,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nle_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x18,0x7d]
+
+v_cmpx_nle_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x18,0x7d]
+
+v_cmpx_nle_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x18,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nle_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x38,0x7d]
+
+v_cmpx_nle_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x38,0x7d]
+
+v_cmpx_nle_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x39,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nle_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x58,0x7d]
+
+v_cmpx_nle_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x58,0x7d]
+
+v_cmpx_nle_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x59,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlg_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x14,0x7d]
+
+v_cmpx_nlg_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x14,0x7d]
+
+v_cmpx_nlg_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x14,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nlg_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x34,0x7d]
+
+v_cmpx_nlg_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x34,0x7d]
+
+v_cmpx_nlg_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x35,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlg_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x54,0x7d]
+
+v_cmpx_nlg_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x54,0x7d]
+
+v_cmpx_nlg_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x55,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlt_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x1c,0x7d]
+
+v_cmpx_nlt_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x1c,0x7d]
+
+v_cmpx_nlt_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x1c,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nlt_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x3c,0x7d]
+
+v_cmpx_nlt_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x3c,0x7d]
+
+v_cmpx_nlt_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x3d,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlt_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x5c,0x7d]
+
+v_cmpx_nlt_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x5c,0x7d]
+
+v_cmpx_nlt_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x5d,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_o_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x0e,0x7d]
+
+v_cmpx_o_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x0e,0x7d]
+
+v_cmpx_o_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x0e,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_o_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x2e,0x7d]
+
+v_cmpx_o_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x2e,0x7d]
+
+v_cmpx_o_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x2f,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_o_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x4e,0x7d]
+
+v_cmpx_o_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x4e,0x7d]
+
+v_cmpx_o_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x4f,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x1e,0x7d]
+
+v_cmpx_t_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x1e,0x7d]
+
+v_cmpx_t_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x1e,0x7d]
+
+v_cmpx_t_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x1e,0x7d]
+
+v_cmpx_t_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x1e,0x7d]
+
+v_cmpx_t_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x1e,0x7d]
+
+v_cmpx_t_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x1e,0x7d]
+
+v_cmpx_t_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x1e,0x7d]
+
+v_cmpx_t_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x1e,0x7d]
+
+v_cmpx_t_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x1e,0x7d]
+
+v_cmpx_t_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x1e,0x7d]
+
+v_cmpx_t_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x1e,0x7d]
+
+v_cmpx_t_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x1e,0x7d]
+
+v_cmpx_t_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x1e,0x7d]
+
+v_cmpx_t_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x1e,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_t_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x3e,0x7d]
+
+v_cmpx_t_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x3e,0x7d]
+
+v_cmpx_t_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x3e,0x7d]
+
+v_cmpx_t_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x3e,0x7d]
+
+v_cmpx_t_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x3e,0x7d]
+
+v_cmpx_t_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x3e,0x7d]
+
+v_cmpx_t_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x3e,0x7d]
+
+v_cmpx_t_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x3e,0x7d]
+
+v_cmpx_t_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x3e,0x7d]
+
+v_cmpx_t_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x3e,0x7d]
+
+v_cmpx_t_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x3e,0x7d]
+
+v_cmpx_t_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x3e,0x7d]
+
+v_cmpx_t_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x3e,0x7d]
+
+v_cmpx_t_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x3e,0x7d]
+
+v_cmpx_t_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x3f,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x5e,0x7d]
+
+v_cmpx_t_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x5e,0x7d]
+
+v_cmpx_t_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x5e,0x7d]
+
+v_cmpx_t_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x5e,0x7d]
+
+v_cmpx_t_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x5e,0x7d]
+
+v_cmpx_t_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x5e,0x7d]
+
+v_cmpx_t_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x5e,0x7d]
+
+v_cmpx_t_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x5e,0x7d]
+
+v_cmpx_t_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x5e,0x7d]
+
+v_cmpx_t_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x5e,0x7d]
+
+v_cmpx_t_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x5e,0x7d]
+
+v_cmpx_t_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x5f,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_i32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x8e,0x7d]
+
+v_cmpx_t_i32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x8e,0x7d]
+
+v_cmpx_t_i32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x8e,0x7d]
+
+v_cmpx_t_i32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x8e,0x7d]
+
+v_cmpx_t_i32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x8e,0x7d]
+
+v_cmpx_t_i32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x8e,0x7d]
+
+v_cmpx_t_i32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x8e,0x7d]
+
+v_cmpx_t_i32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x8e,0x7d]
+
+v_cmpx_t_i32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x8e,0x7d]
+
+v_cmpx_t_i32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x8e,0x7d]
+
+v_cmpx_t_i32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x8e,0x7d]
+
+v_cmpx_t_i32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x8e,0x7d]
+
+v_cmpx_t_i32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x8e,0x7d]
+
+v_cmpx_t_i32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x8e,0x7d]
+
+v_cmpx_t_i32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x8f,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_i64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xae,0x7d]
+
+v_cmpx_t_i64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xae,0x7d]
+
+v_cmpx_t_i64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xae,0x7d]
+
+v_cmpx_t_i64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xae,0x7d]
+
+v_cmpx_t_i64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xae,0x7d]
+
+v_cmpx_t_i64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xae,0x7d]
+
+v_cmpx_t_i64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xae,0x7d]
+
+v_cmpx_t_i64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xae,0x7d]
+
+v_cmpx_t_i64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xae,0x7d]
+
+v_cmpx_t_i64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xae,0x7d]
+
+v_cmpx_t_i64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xae,0x7d]
+
+v_cmpx_t_i64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xaf,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_u32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x9e,0x7d]
+
+v_cmpx_t_u32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x9e,0x7d]
+
+v_cmpx_t_u32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x9e,0x7d]
+
+v_cmpx_t_u32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x9e,0x7d]
+
+v_cmpx_t_u32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x9e,0x7d]
+
+v_cmpx_t_u32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x9e,0x7d]
+
+v_cmpx_t_u32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x9e,0x7d]
+
+v_cmpx_t_u32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x9e,0x7d]
+
+v_cmpx_t_u32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x9e,0x7d]
+
+v_cmpx_t_u32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x9e,0x7d]
+
+v_cmpx_t_u32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x9e,0x7d]
+
+v_cmpx_t_u32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x9e,0x7d]
+
+v_cmpx_t_u32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x9e,0x7d]
+
+v_cmpx_t_u32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x9e,0x7d]
+
+v_cmpx_t_u32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x9f,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_u64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0xbe,0x7d]
+
+v_cmpx_t_u64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0xbe,0x7d]
+
+v_cmpx_t_u64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0xbe,0x7d]
+
+v_cmpx_t_u64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0xbe,0x7d]
+
+v_cmpx_t_u64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0xbe,0x7d]
+
+v_cmpx_t_u64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0xbe,0x7d]
+
+v_cmpx_t_u64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0xbe,0x7d]
+
+v_cmpx_t_u64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0xbe,0x7d]
+
+v_cmpx_t_u64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0xbe,0x7d]
+
+v_cmpx_t_u64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0xbe,0x7d]
+
+v_cmpx_t_u64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0xbe,0x7d]
+
+v_cmpx_t_u64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0xbf,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_tru_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x1e,0x7d]
+
+v_cmpx_tru_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x1e,0x7d]
+
+v_cmpx_tru_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x1e,0x7d]
+
+v_cmpx_tru_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x1e,0x7d]
+
+v_cmpx_tru_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x1e,0x7d]
+
+v_cmpx_tru_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x1e,0x7d]
+
+v_cmpx_tru_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x1e,0x7d]
+
+v_cmpx_tru_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x1e,0x7d]
+
+v_cmpx_tru_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x1e,0x7d]
+
+v_cmpx_tru_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x1e,0x7d]
+
+v_cmpx_tru_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x1e,0x7d]
+
+v_cmpx_tru_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x1e,0x7d]
+
+v_cmpx_tru_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x1e,0x7d]
+
+v_cmpx_tru_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x1e,0x7d]
+
+v_cmpx_tru_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x1e,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_tru_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x3e,0x7d]
+
+v_cmpx_tru_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x3e,0x7d]
+
+v_cmpx_tru_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x3e,0x7d]
+
+v_cmpx_tru_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x3e,0x7d]
+
+v_cmpx_tru_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x3e,0x7d]
+
+v_cmpx_tru_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x3e,0x7d]
+
+v_cmpx_tru_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x3e,0x7d]
+
+v_cmpx_tru_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x3e,0x7d]
+
+v_cmpx_tru_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x3e,0x7d]
+
+v_cmpx_tru_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x3e,0x7d]
+
+v_cmpx_tru_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x3e,0x7d]
+
+v_cmpx_tru_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x3e,0x7d]
+
+v_cmpx_tru_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x3e,0x7d]
+
+v_cmpx_tru_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x3e,0x7d]
+
+v_cmpx_tru_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x3f,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_tru_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x5e,0x7d]
+
+v_cmpx_tru_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x5e,0x7d]
+
+v_cmpx_tru_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x5e,0x7d]
+
+v_cmpx_tru_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x5e,0x7d]
+
+v_cmpx_tru_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x5e,0x7d]
+
+v_cmpx_tru_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x5e,0x7d]
+
+v_cmpx_tru_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x5e,0x7d]
+
+v_cmpx_tru_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x5e,0x7d]
+
+v_cmpx_tru_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x5e,0x7d]
+
+v_cmpx_tru_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x5e,0x7d]
+
+v_cmpx_tru_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x5e,0x7d]
+
+v_cmpx_tru_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x5f,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_u_f16 v1, v2
+// GFX11: encoding: [0x01,0x05,0x10,0x7d]
+
+v_cmpx_u_f16 v127, v2
+// GFX11: encoding: [0x7f,0x05,0x10,0x7d]
+
+v_cmpx_u_f16 s1, v2
+// GFX11: encoding: [0x01,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 s105, v2
+// GFX11: encoding: [0x69,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 null, v2
+// GFX11: encoding: [0x7c,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0x10,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_u_f32 v1, v2
+// GFX11: encoding: [0x01,0x05,0x30,0x7d]
+
+v_cmpx_u_f32 v255, v2
+// GFX11: encoding: [0xff,0x05,0x30,0x7d]
+
+v_cmpx_u_f32 s1, v2
+// GFX11: encoding: [0x01,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 s105, v2
+// GFX11: encoding: [0x69,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 m0, v2
+// GFX11: encoding: [0x7d,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 null, v2
+// GFX11: encoding: [0x7c,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 -1, v2
+// GFX11: encoding: [0xc1,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0x31,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_u_f64 v[1:2], v[2:3]
+// GFX11: encoding: [0x01,0x05,0x50,0x7d]
+
+v_cmpx_u_f64 v[254:255], v[2:3]
+// GFX11: encoding: [0xfe,0x05,0x50,0x7d]
+
+v_cmpx_u_f64 s[2:3], v[2:3]
+// GFX11: encoding: [0x02,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 s[104:105], v[2:3]
+// GFX11: encoding: [0x68,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 vcc, v[2:3]
+// GFX11: encoding: [0x6a,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 ttmp[14:15], v[2:3]
+// GFX11: encoding: [0x7a,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 exec, v[2:3]
+// GFX11: encoding: [0x7e,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 null, v[2:3]
+// GFX11: encoding: [0x7c,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 -1, v[2:3]
+// GFX11: encoding: [0xc1,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 0.5, v[2:3]
+// GFX11: encoding: [0xf0,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 src_scc, v[2:3]
+// GFX11: encoding: [0xfd,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 0xaf123456, v[254:255]
+// GFX11: encoding: [0xff,0xfc,0x51,0x7d,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s
index e90bb80f098c..17e60c08f6a7 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
 
 v_cmpx_class_f16_e32 v1, v2
 // GFX11: encoding: [0x01,0x05,0xfa,0x7d]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp16-fake16.s
new file mode 100644
index 000000000000..897eefe1fcbe
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp16-fake16.s
@@ -0,0 +1,2690 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+
+v_cmpx_class_f16_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_class_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_class_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_class_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_class_f16 -|v127|, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xfa,0x7d,0x7f,0x6f,0x35,0x30]
+
+v_cmpx_class_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_class_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_class_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_class_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_class_f32 -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xfd,0x7d,0xff,0x6f,0x35,0x30]
+
+v_cmpx_eq_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x04,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x04,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_eq_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x24,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x25,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_eq_i16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_i16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x64,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_i16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_i16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x64,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_eq_i32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_i32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x84,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_i32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_i32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x85,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_eq_u16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_u16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x74,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_u16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_u16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x74,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_eq_u32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_u32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x94,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x95,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_f_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x00,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_f_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x00,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_f_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x00,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_f_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x00,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_f_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x00,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_f_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x00,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_f_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x00,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_f_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x00,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_f_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x00,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_f_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x00,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_f_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x00,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_f_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x00,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_f_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x00,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_f_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x00,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_f_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x20,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_f_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x20,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_f_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x20,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_f_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x20,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_f_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x20,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_f_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x20,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_f_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x20,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_f_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x20,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_f_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x20,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_f_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x20,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_f_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x20,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_f_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x20,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_f_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x20,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_f_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x21,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_f_i32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x80,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_f_i32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x80,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_f_i32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x80,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_f_i32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x80,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_f_i32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x80,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_f_i32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x80,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_f_i32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x80,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_f_i32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x80,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_f_i32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x80,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_f_i32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x80,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_f_i32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x80,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_f_i32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x80,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_f_i32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x80,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_f_i32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x81,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_f_u32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x90,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_f_u32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x90,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_f_u32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x90,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_f_u32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x90,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_f_u32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x90,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_f_u32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x90,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_f_u32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x90,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_f_u32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x90,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_f_u32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x90,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_f_u32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x90,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_f_u32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x90,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_f_u32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x90,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_f_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x90,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_f_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x91,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ge_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x0c,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_ge_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x2d,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_ge_i16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_i16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_i16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_i16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x6c,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_ge_i32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_i32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_i32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_i32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x8d,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ge_u16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_u16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_u16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_u16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x7c,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_ge_u32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_u32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x9d,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_gt_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x08,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x08,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_gt_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x28,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x29,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_gt_i16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_i16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x68,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_i16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_i16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x68,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_gt_i32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_i32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x88,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_i32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_i32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x89,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_gt_u16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_u16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x78,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_u16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_u16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x78,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_gt_u32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_u32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x98,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x99,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_le_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x06,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x06,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_le_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x26,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x27,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_le_i16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_i16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x66,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_i16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_i16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_i16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x66,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_le_i32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_i32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x86,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_i32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_i32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_i32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x87,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_le_u16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_u16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x76,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_u16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_u16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_u16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x76,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_le_u32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_u32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x96,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_u32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x97,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_lg_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x0a,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_lg_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lg_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lg_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x2b,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_lt_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x02,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x02,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_lt_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x22,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x23,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_lt_i16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_i16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x62,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_i16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_i16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x62,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_lt_i32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_i32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x82,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_i32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_i32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x83,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_lt_u16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_u16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x72,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_u16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_u16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x72,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_lt_u32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_u32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x92,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x93,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ne_i16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ne_i16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ne_i16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ne_i16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x6a,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_ne_i32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ne_i32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ne_i32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ne_i32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x8b,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ne_u16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ne_u16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ne_u16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ne_u16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x7a,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_ne_u32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ne_u32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ne_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ne_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x9b,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_neq_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_neq_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_neq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_neq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x1a,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_neq_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_neq_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_neq_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_neq_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x3b,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_nge_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x12,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x12,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_nge_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nge_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x32,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nge_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nge_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x33,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_ngt_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x16,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ngt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ngt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x16,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_ngt_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x36,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ngt_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ngt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x37,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_nle_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nle_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x18,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nle_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nle_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x18,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_nle_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nle_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x38,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nle_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nle_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x39,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_nlg_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x14,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nlg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nlg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x14,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_nlg_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x34,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nlg_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nlg_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x35,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_nlt_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlt_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nlt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nlt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x1c,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_nlt_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlt_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nlt_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nlt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x3d,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_o_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_o_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_o_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_o_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_o_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x0e,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_o_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_o_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_o_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_o_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_o_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x2f,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_t_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_t_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_t_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_t_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_t_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_t_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_t_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_t_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_t_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_t_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_t_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_t_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_t_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_t_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x1e,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_t_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_t_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_t_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_t_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_t_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_t_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_t_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_t_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_t_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_t_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_t_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_t_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_t_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_t_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x3f,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_t_i32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x8e,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_t_i32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x8e,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_t_i32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x8e,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_t_i32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x8e,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_t_i32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x8e,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_t_i32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x8e,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_t_i32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x8e,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_t_i32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x8e,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_t_i32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x8e,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_t_i32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x8e,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_t_i32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x8e,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_t_i32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x8e,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_t_i32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x8e,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_t_i32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x8f,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_t_u32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x9e,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_t_u32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x9e,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_t_u32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x9e,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_t_u32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x9e,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_t_u32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x9e,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_t_u32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x9e,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_t_u32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x9e,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_t_u32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x9e,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_t_u32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x9e,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_t_u32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x9e,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_t_u32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x9e,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_t_u32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x9e,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_t_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x9e,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_t_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x9f,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_tru_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_tru_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_tru_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_tru_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_tru_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_tru_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_tru_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_tru_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_tru_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_tru_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_tru_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_tru_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_tru_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x1e,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_tru_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x1e,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_tru_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_tru_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_tru_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_tru_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_tru_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_tru_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_tru_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_tru_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_tru_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_tru_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_tru_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_tru_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_tru_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x3e,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_tru_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x3f,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_u_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_u_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x10,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_u_f16 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_u_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_u_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x10,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_u_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_u_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x30,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_u_f32 v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_u_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_u_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0x31,0x7d,0xff,0x6f,0xf5,0x30]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp16.s
index d8fc1d3e2b3c..e46661df84a1 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp16.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
 
 v_cmpx_class_f16_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX11: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp8-fake16.s
new file mode 100644
index 000000000000..e66da32fe032
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp8-fake16.s
@@ -0,0 +1,578 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+
+v_cmpx_class_f16_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xfa,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_class_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0xfc,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0xfc,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xfd,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x04,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_eq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x24,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x24,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x25,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_eq_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x64,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x64,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_i16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x64,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_eq_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x84,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x84,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_i32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x85,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_eq_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x74,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x74,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_u16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x74,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_eq_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x94,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x94,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x95,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_f_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x00,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x00,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x00,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_f_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x20,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x20,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x21,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_f_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x80,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x80,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_i32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x81,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_f_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x90,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x90,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x91,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x0c,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_ge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x2c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x2c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x2d,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_ge_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x6c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x6c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_i16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x6c,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_ge_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x8c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x8c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_i32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x8d,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_ge_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x7c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x7c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_u16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x7c,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_ge_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x9c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x9c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x9d,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x08,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_gt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x28,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x28,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x29,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_gt_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x68,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x68,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_i16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x68,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_gt_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x88,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x88,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_i32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x89,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_gt_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x78,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x78,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_u16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x78,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_gt_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x98,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x98,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x99,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x06,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_le_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x26,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x26,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x27,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_le_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x66,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x66,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_i16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x66,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_le_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x86,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x86,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_i32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x87,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_le_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x76,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x76,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_u16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x76,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_le_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x96,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x96,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x97,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x0a,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_lg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x2a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x2a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x2b,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_lt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x02,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x02,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x02,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_lt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x22,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x22,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x23,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_lt_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x62,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x62,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_i16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x62,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_lt_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x82,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x82,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_i32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x83,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_lt_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x72,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x72,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_u16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x72,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_lt_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x92,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x92,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x93,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_ne_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x6a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x6a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_i16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x6a,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_ne_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x8a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x8a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_i32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x8b,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_ne_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x7a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x7a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_u16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x7a,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_ne_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x9a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x9a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x9b,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x1a,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_neq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x3a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x3a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x3b,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x12,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_nge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x32,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x32,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x33,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x16,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_ngt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x36,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x36,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x37,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x18,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_nle_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x38,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x38,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x39,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x14,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_nlg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x34,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x34,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x35,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x1c,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_nlt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x3c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x3c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x3d,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x0e,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_o_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x2e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x2e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x2f,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_t_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x1e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x1e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x1e,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_t_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x3e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x3e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x3f,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_t_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x8e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x8e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_i32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x8f,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_t_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x9e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x9e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x9f,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_tru_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x1e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_tru_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x1e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_tru_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x1e,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_tru_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x3e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_tru_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x3e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_tru_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x3f,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x10,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_u_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x30,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x30,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0x31,0x7d,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp8.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp8.s
index 9db7e48809ee..5062f901d2aa 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_dpp8.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
 
 v_cmpx_class_f16_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: encoding: [0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_err.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_err.s
index c37d15b7abc7..ec628dd94f36 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_err.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_err.s
@@ -1,542 +1,542 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=error: %s
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort --version 5
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=error: %s
 
 v_cmpx_class_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_eq_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_class_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmpx_eq_i16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_class_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmpx_eq_u16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_class_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_f_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_class_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_class_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmpx_ge_i16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ge_u16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_i16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_gt_u16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_le_i16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_i16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_le_u16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_i16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lt_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_i16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_lt_i16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lt_u16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_i16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ne_i16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_u16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ne_u16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_u16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_u16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ngt_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_u16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_f_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_nlt_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_f_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_f_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmpx_t_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_f_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_tru_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_f_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_f_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmpx_class_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_eq_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_eq_i16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_eq_u16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_f_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ge_i16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_i16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ge_u16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_i16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_i16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_i16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_gt_u16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_i16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_le_i16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_u16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_le_u16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_u16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lt_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_u16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_lt_i16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lt_u16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_u16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ne_i16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ne_u16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ngt_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_i16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_nlt_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_i16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_t_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_i16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_tru_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_i16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_class_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_u16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_eq_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_eq_i16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_u16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_eq_u16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_u16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_f_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_u16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ge_i16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ge_u16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_i16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_gt_u16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_le_i16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_i16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_le_u16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_i16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_i16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_lt_i16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lt_u16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_i16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ne_i16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_u16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ne_u16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_u16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_u16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ngt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_u16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_nlt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_t_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_tru_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_class_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_eq_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_eq_i16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_eq_u16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_f_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ge_i16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_i16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ge_u16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_i16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_i16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_i16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_gt_u16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_i16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_le_i16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_u16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_le_u16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_u16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_u16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_lt_i16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
 v_cmpx_lt_u16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ne_i16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmpx_ne_u16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_i16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_neq_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_i16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ngt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_i16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_nle_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_i16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nlt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_u16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_o_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_t_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_u16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_tru_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_u16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_u_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_class_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_u16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_eq_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_eq_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_f_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ge_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_ge_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_ge_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_gt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_gt_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_gt_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_le_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_le_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_le_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_lg_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_lt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_lt_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_lt_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_ne_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_ne_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_neq_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_ngt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_nle_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_nlt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_o_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_t_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_tru_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_u_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_class_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_eq_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_eq_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_f_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ge_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_ge_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_ge_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_gt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmpx_gt_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmpx_gt_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_le_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmpx_le_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmpx_le_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_t_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_t_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_cmpx_lt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_t_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_cmpx_lt_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_t_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_lt_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_t_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_ne_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_t_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:18: error: invalid operand for instruction
 
-v_cmpx_ne_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_tru_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_tru_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_tru_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cmpx_ngt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_tru_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_tru_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_tru_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:20: error: invalid operand for instruction
 
-v_cmpx_nlt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_o_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmpx_t_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmpx_tru_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmpx_u_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
+v_cmpx_u_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_promote.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_promote.s
index a279bd381848..3bbdf3d3a903 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_promote.s
@@ -1,542 +1,542 @@
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 %s
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 %s
 
 v_cmpx_class_f16 v1, v255
-// GFX11: v_cmpx_class_f16_e64
+// GFX11: v_cmpx_class_f16_e64 v1, v255           ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_eq_f16 v1, v255
-// GFX11: v_cmpx_eq_f16_e64
+v_cmpx_class_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xfd,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_i16 v1, v255
-// GFX11: v_cmpx_eq_i16_e64
+v_cmpx_class_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_eq_u16 v1, v255
-// GFX11: v_cmpx_eq_u16_e64
+v_cmpx_class_f16 v255, v2
+// GFX11: v_cmpx_class_f16_e64 v255, v2           ; encoding: [0x7e,0x00,0xfd,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_f_f16 v1, v255
-// GFX11: v_cmpx_f_f16_e64
+v_cmpx_class_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_class_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xfd,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v1, v255
-// GFX11: v_cmpx_ge_f16_e64
+v_cmpx_class_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_class_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ge_i16 v1, v255
-// GFX11: v_cmpx_ge_i16_e64
+v_cmpx_eq_f16 v1, v255
+// GFX11: v_cmpx_eq_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ge_u16 v1, v255
-// GFX11: v_cmpx_ge_u16_e64
+v_cmpx_eq_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v1, v255
-// GFX11: v_cmpx_gt_f16_e64
+v_cmpx_eq_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_gt_i16 v1, v255
-// GFX11: v_cmpx_gt_i16_e64
+v_cmpx_eq_f16 v255, v2
+// GFX11: v_cmpx_eq_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_gt_u16 v1, v255
-// GFX11: v_cmpx_gt_u16_e64
+v_cmpx_eq_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_le_f16 v1, v255
-// GFX11: v_cmpx_le_f16_e64
+v_cmpx_eq_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_le_i16 v1, v255
-// GFX11: v_cmpx_le_i16_e64
+v_cmpx_eq_i16 v1, v255
+// GFX11: v_cmpx_eq_i16_e64 v1, v255              ; encoding: [0x7e,0x00,0xb2,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_le_u16 v1, v255
-// GFX11: v_cmpx_le_u16_e64
+v_cmpx_eq_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb2,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v1, v255
-// GFX11: v_cmpx_lg_f16_e64
+v_cmpx_eq_i16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_lt_f16 v1, v255
-// GFX11: v_cmpx_lt_f16_e64
+v_cmpx_eq_i16 v255, v2
+// GFX11: v_cmpx_eq_i16_e64 v255, v2              ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_lt_i16 v1, v255
-// GFX11: v_cmpx_lt_i16_e64
+v_cmpx_eq_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_i16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb2,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_lt_u16 v1, v255
-// GFX11: v_cmpx_lt_u16_e64
+v_cmpx_eq_i16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_i16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ne_i16 v1, v255
-// GFX11: v_cmpx_ne_i16_e64
+v_cmpx_eq_u16 v1, v255
+// GFX11: v_cmpx_eq_u16_e64 v1, v255              ; encoding: [0x7e,0x00,0xba,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ne_u16 v1, v255
-// GFX11: v_cmpx_ne_u16_e64
+v_cmpx_eq_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xba,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v1, v255
-// GFX11: v_cmpx_neq_f16_e64
+v_cmpx_eq_u16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nge_f16 v1, v255
-// GFX11: v_cmpx_nge_f16_e64
+v_cmpx_eq_u16 v255, v2
+// GFX11: v_cmpx_eq_u16_e64 v255, v2              ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_ngt_f16 v1, v255
-// GFX11: v_cmpx_ngt_f16_e64
+v_cmpx_eq_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_u16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xba,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nle_f16 v1, v255
-// GFX11: v_cmpx_nle_f16_e64
+v_cmpx_eq_u16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_u16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nlg_f16 v1, v255
-// GFX11: v_cmpx_nlg_f16_e64
+v_cmpx_f_f16 v1, v255
+// GFX11: v_cmpx_f_f16_e64 v1, v255               ; encoding: [0x7e,0x00,0x80,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_nlt_f16 v1, v255
-// GFX11: v_cmpx_nlt_f16_e64
+v_cmpx_f_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x80,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16 v1, v255
-// GFX11: v_cmpx_o_f16_e64
+v_cmpx_f_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_t_f16 v1, v255
-// GFX11: v_cmpx_t_f16_e64
+v_cmpx_f_f16 v255, v2
+// GFX11: v_cmpx_f_f16_e64 v255, v2               ; encoding: [0x7e,0x00,0x80,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_tru_f16 v1, v255
-// GFX11: v_cmpx_t_f16_e64
+v_cmpx_f_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x80,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_u_f16 v1, v255
-// GFX11: v_cmpx_u_f16_e64
+v_cmpx_f_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_class_f16 v255, v2
-// GFX11: v_cmpx_class_f16_e64
+v_cmpx_ge_f16 v1, v255
+// GFX11: v_cmpx_ge_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_eq_f16 v255, v2
-// GFX11: v_cmpx_eq_f16_e64
+v_cmpx_ge_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_i16 v255, v2
-// GFX11: v_cmpx_eq_i16_e64
+v_cmpx_ge_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_eq_u16 v255, v2
-// GFX11: v_cmpx_eq_u16_e64
+v_cmpx_ge_f16 v255, v2
+// GFX11: v_cmpx_ge_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_f_f16 v255, v2
-// GFX11: v_cmpx_f_f16_e64
+v_cmpx_ge_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v255, v2
-// GFX11: v_cmpx_ge_f16_e64
+v_cmpx_ge_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ge_i16 v255, v2
-// GFX11: v_cmpx_ge_i16_e64
+v_cmpx_ge_i16 v1, v255
+// GFX11: v_cmpx_ge_i16_e64 v1, v255              ; encoding: [0x7e,0x00,0xb6,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ge_u16 v255, v2
-// GFX11: v_cmpx_ge_u16_e64
+v_cmpx_ge_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb6,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v255, v2
-// GFX11: v_cmpx_gt_f16_e64
+v_cmpx_ge_i16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_gt_i16 v255, v2
-// GFX11: v_cmpx_gt_i16_e64
+v_cmpx_ge_i16 v255, v2
+// GFX11: v_cmpx_ge_i16_e64 v255, v2              ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_gt_u16 v255, v2
-// GFX11: v_cmpx_gt_u16_e64
+v_cmpx_ge_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_i16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb6,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_le_f16 v255, v2
-// GFX11: v_cmpx_le_f16_e64
+v_cmpx_ge_i16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_i16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_le_i16 v255, v2
-// GFX11: v_cmpx_le_i16_e64
+v_cmpx_ge_u16 v1, v255
+// GFX11: v_cmpx_ge_u16_e64 v1, v255              ; encoding: [0x7e,0x00,0xbe,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_le_u16 v255, v2
-// GFX11: v_cmpx_le_u16_e64
+v_cmpx_ge_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbe,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v255, v2
-// GFX11: v_cmpx_lg_f16_e64
+v_cmpx_ge_u16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_lt_f16 v255, v2
-// GFX11: v_cmpx_lt_f16_e64
+v_cmpx_ge_u16 v255, v2
+// GFX11: v_cmpx_ge_u16_e64 v255, v2              ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_lt_i16 v255, v2
-// GFX11: v_cmpx_lt_i16_e64
+v_cmpx_ge_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_u16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbe,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_lt_u16 v255, v2
-// GFX11: v_cmpx_lt_u16_e64
+v_cmpx_ge_u16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_u16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ne_i16 v255, v2
-// GFX11: v_cmpx_ne_i16_e64
+v_cmpx_gt_f16 v1, v255
+// GFX11: v_cmpx_gt_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ne_u16 v255, v2
-// GFX11: v_cmpx_ne_u16_e64
+v_cmpx_gt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v255, v2
-// GFX11: v_cmpx_neq_f16_e64
+v_cmpx_gt_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nge_f16 v255, v2
-// GFX11: v_cmpx_nge_f16_e64
+v_cmpx_gt_f16 v255, v2
+// GFX11: v_cmpx_gt_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_ngt_f16 v255, v2
-// GFX11: v_cmpx_ngt_f16_e64
+v_cmpx_gt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nle_f16 v255, v2
-// GFX11: v_cmpx_nle_f16_e64
+v_cmpx_gt_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nlg_f16 v255, v2
-// GFX11: v_cmpx_nlg_f16_e64
+v_cmpx_gt_i16 v1, v255
+// GFX11: v_cmpx_gt_i16_e64 v1, v255              ; encoding: [0x7e,0x00,0xb4,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_nlt_f16 v255, v2
-// GFX11: v_cmpx_nlt_f16_e64
+v_cmpx_gt_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb4,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16 v255, v2
-// GFX11: v_cmpx_o_f16_e64
+v_cmpx_gt_i16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_t_f16 v255, v2
-// GFX11: v_cmpx_t_f16_e64
+v_cmpx_gt_i16 v255, v2
+// GFX11: v_cmpx_gt_i16_e64 v255, v2              ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_tru_f16 v255, v2
-// GFX11: v_cmpx_t_f16_e64
+v_cmpx_gt_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_i16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb4,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_u_f16 v255, v2
-// GFX11: v_cmpx_u_f16_e64
+v_cmpx_gt_i16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_i16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_class_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_class_f16_e64
+v_cmpx_gt_u16 v1, v255
+// GFX11: v_cmpx_gt_u16_e64 v1, v255              ; encoding: [0x7e,0x00,0xbc,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_eq_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_eq_f16_e64
+v_cmpx_gt_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbc,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_i16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_eq_i16_e64
+v_cmpx_gt_u16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_eq_u16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_eq_u16_e64
+v_cmpx_gt_u16 v255, v2
+// GFX11: v_cmpx_gt_u16_e64 v255, v2              ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_f_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_f_f16_e64
+v_cmpx_gt_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_u16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbc,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ge_f16_e64
+v_cmpx_gt_u16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_u16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ge_i16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ge_i16_e64
+v_cmpx_le_f16 v1, v255
+// GFX11: v_cmpx_le_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ge_u16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ge_u16_e64
+v_cmpx_le_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_gt_f16_e64
+v_cmpx_le_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_gt_i16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_gt_i16_e64
+v_cmpx_le_f16 v255, v2
+// GFX11: v_cmpx_le_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_gt_u16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_gt_u16_e64
+v_cmpx_le_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_le_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_le_f16_e64
+v_cmpx_le_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_le_i16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_le_i16_e64
+v_cmpx_le_i16 v1, v255
+// GFX11: v_cmpx_le_i16_e64 v1, v255              ; encoding: [0x7e,0x00,0xb3,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_le_u16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_le_u16_e64
+v_cmpx_le_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb3,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_lg_f16_e64
+v_cmpx_le_i16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_lt_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_lt_f16_e64
+v_cmpx_le_i16 v255, v2
+// GFX11: v_cmpx_le_i16_e64 v255, v2              ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_lt_i16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_lt_i16_e64
+v_cmpx_le_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_i16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb3,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_lt_u16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_lt_u16_e64
+v_cmpx_le_i16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_i16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ne_i16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ne_i16_e64
+v_cmpx_le_u16 v1, v255
+// GFX11: v_cmpx_le_u16_e64 v1, v255              ; encoding: [0x7e,0x00,0xbb,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ne_u16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ne_u16_e64
+v_cmpx_le_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbb,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_neq_f16_e64
+v_cmpx_le_u16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nge_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nge_f16_e64
+v_cmpx_le_u16 v255, v2
+// GFX11: v_cmpx_le_u16_e64 v255, v2              ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_ngt_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ngt_f16_e64
+v_cmpx_le_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_u16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbb,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nle_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nle_f16_e64
+v_cmpx_le_u16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_u16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nlg_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nlg_f16_e64
+v_cmpx_lg_f16 v1, v255
+// GFX11: v_cmpx_lg_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_nlt_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nlt_f16_e64
+v_cmpx_lg_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_o_f16_e64
+v_cmpx_lg_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_t_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64
+v_cmpx_lg_f16 v255, v2
+// GFX11: v_cmpx_lg_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_tru_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64
+v_cmpx_lg_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_u_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_u_f16_e64
+v_cmpx_lg_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_class_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_class_f16_e64
+v_cmpx_lt_f16 v1, v255
+// GFX11: v_cmpx_lt_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x81,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_eq_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_eq_f16_e64
+v_cmpx_lt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lt_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x81,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_i16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_eq_i16_e64
+v_cmpx_lt_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lt_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_eq_u16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_eq_u16_e64
+v_cmpx_lt_f16 v255, v2
+// GFX11: v_cmpx_lt_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x81,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_f_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_f_f16_e64
+v_cmpx_lt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lt_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x81,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ge_f16_e64
+v_cmpx_lt_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lt_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ge_i16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ge_i16_e64
+v_cmpx_lt_i16 v1, v255
+// GFX11: v_cmpx_lt_i16_e64 v1, v255              ; encoding: [0x7e,0x00,0xb1,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ge_u16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ge_u16_e64
+v_cmpx_lt_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb1,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_gt_f16_e64
+v_cmpx_lt_i16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_gt_i16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_gt_i16_e64
+v_cmpx_lt_i16 v255, v2
+// GFX11: v_cmpx_lt_i16_e64 v255, v2              ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_gt_u16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_gt_u16_e64
+v_cmpx_lt_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lt_i16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb1,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_le_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_le_f16_e64
+v_cmpx_lt_i16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lt_i16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_le_i16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_le_i16_e64
+v_cmpx_lt_u16 v1, v255
+// GFX11: v_cmpx_lt_u16_e64 v1, v255              ; encoding: [0x7e,0x00,0xb9,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_le_u16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_le_u16_e64
+v_cmpx_lt_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb9,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_lg_f16_e64
+v_cmpx_lt_u16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_lt_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_lt_f16_e64
+v_cmpx_lt_u16 v255, v2
+// GFX11: v_cmpx_lt_u16_e64 v255, v2              ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_lt_i16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_lt_i16_e64
+v_cmpx_lt_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lt_u16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb9,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
 v_cmpx_lt_u16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_lt_u16_e64
+// GFX11: v_cmpx_lt_u16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ne_i16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ne_i16_e64
-
-v_cmpx_ne_u16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ne_u16_e64
+v_cmpx_ne_i16 v1, v255
+// GFX11: v_cmpx_ne_i16_e64 v1, v255              ; encoding: [0x7e,0x00,0xb5,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_neq_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_neq_f16_e64
+v_cmpx_ne_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb5,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_nge_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nge_f16_e64
+v_cmpx_ne_i16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ngt_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_ngt_f16_e64
+v_cmpx_ne_i16 v255, v2
+// GFX11: v_cmpx_ne_i16_e64 v255, v2              ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nle_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nle_f16_e64
+v_cmpx_ne_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ne_i16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb5,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nlg_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nlg_f16_e64
+v_cmpx_ne_i16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ne_i16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nlt_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_nlt_f16_e64
+v_cmpx_ne_u16 v1, v255
+// GFX11: v_cmpx_ne_u16_e64 v1, v255              ; encoding: [0x7e,0x00,0xbd,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_o_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_o_f16_e64
+v_cmpx_ne_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbd,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_t_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64
+v_cmpx_ne_u16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_tru_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64
+v_cmpx_ne_u16 v255, v2
+// GFX11: v_cmpx_ne_u16_e64 v255, v2              ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_u_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_u_f16_e64
+v_cmpx_ne_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ne_u16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbd,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_class_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_class_f16_e64
+v_cmpx_ne_u16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ne_u16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_eq_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_eq_f16_e64
+v_cmpx_neq_f16 v1, v255
+// GFX11: v_cmpx_neq_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_eq_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_eq_i16_e64
+v_cmpx_neq_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_eq_u16_e64
+v_cmpx_neq_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_f_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_f_f16_e64
+v_cmpx_neq_f16 v255, v2
+// GFX11: v_cmpx_neq_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_ge_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ge_f16_e64
+v_cmpx_neq_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_ge_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ge_i16_e64
+v_cmpx_neq_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ge_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ge_u16_e64
+v_cmpx_nge_f16 v1, v255
+// GFX11: v_cmpx_nge_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_gt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_gt_f16_e64
+v_cmpx_nge_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_gt_i16_e64
+v_cmpx_nge_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_gt_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_gt_u16_e64
+v_cmpx_nge_f16 v255, v2
+// GFX11: v_cmpx_nge_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_le_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_le_f16_e64
+v_cmpx_nge_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_le_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_le_i16_e64
+v_cmpx_nge_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_le_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_le_u16_e64
+v_cmpx_ngt_f16 v1, v255
+// GFX11: v_cmpx_ngt_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_lg_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_lg_f16_e64
+v_cmpx_ngt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_lt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_lt_f16_e64
+v_cmpx_ngt_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_lt_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_lt_i16_e64
+v_cmpx_ngt_f16 v255, v2
+// GFX11: v_cmpx_ngt_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_lt_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_lt_u16_e64
+v_cmpx_ngt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_ne_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ne_i16_e64
+v_cmpx_ngt_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ne_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ne_u16_e64
+v_cmpx_nle_f16 v1, v255
+// GFX11: v_cmpx_nle_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_neq_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_neq_f16_e64
+v_cmpx_nle_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_nge_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nge_f16_e64
+v_cmpx_nle_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ngt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ngt_f16_e64
+v_cmpx_nle_f16 v255, v2
+// GFX11: v_cmpx_nle_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nle_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nle_f16_e64
+v_cmpx_nle_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nlg_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nlg_f16_e64
+v_cmpx_nle_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nlt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nlt_f16_e64
+v_cmpx_nlg_f16 v1, v255
+// GFX11: v_cmpx_nlg_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_o_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_o_f16_e64
+v_cmpx_nlg_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_t_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64
+v_cmpx_nlg_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_tru_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64
+v_cmpx_nlg_f16 v255, v2
+// GFX11: v_cmpx_nlg_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_u_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_u_f16_e64
+v_cmpx_nlg_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_class_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_class_f16_e64
+v_cmpx_nlg_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_eq_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_eq_f16_e64
+v_cmpx_nlt_f16 v1, v255
+// GFX11: v_cmpx_nlt_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_eq_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_eq_i16_e64
+v_cmpx_nlt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_eq_u16_e64
+v_cmpx_nlt_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_f_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_f_f16_e64
+v_cmpx_nlt_f16 v255, v2
+// GFX11: v_cmpx_nlt_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_ge_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ge_f16_e64
+v_cmpx_nlt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_ge_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ge_i16_e64
+v_cmpx_nlt_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ge_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ge_u16_e64
+v_cmpx_o_f16 v1, v255
+// GFX11: v_cmpx_o_f16_e64 v1, v255               ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_gt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_gt_f16_e64
+v_cmpx_o_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_gt_i16_e64
+v_cmpx_o_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_gt_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_gt_u16_e64
+v_cmpx_o_f16 v255, v2
+// GFX11: v_cmpx_o_f16_e64 v255, v2               ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_le_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_le_f16_e64
+v_cmpx_o_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_le_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_le_i16_e64
+v_cmpx_o_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_le_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_le_u16_e64
+v_cmpx_t_f16 v1, v255
+// GFX11: v_cmpx_t_f16_e64 v1, v255               ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_lg_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_lg_f16_e64
+v_cmpx_t_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_lt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_lt_f16_e64
+v_cmpx_t_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_lt_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_lt_i16_e64
+v_cmpx_t_f16 v255, v2
+// GFX11: v_cmpx_t_f16_e64 v255, v2               ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_lt_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_lt_u16_e64
+v_cmpx_t_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_ne_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ne_i16_e64
+v_cmpx_t_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ne_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ne_u16_e64
+v_cmpx_tru_f16 v1, v255
+// GFX11: v_cmpx_t_f16_e64 v1, v255               ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_neq_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_neq_f16_e64
+v_cmpx_tru_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_nge_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nge_f16_e64
+v_cmpx_tru_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ngt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_ngt_f16_e64
+v_cmpx_tru_f16 v255, v2
+// GFX11: v_cmpx_t_f16_e64 v255, v2               ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nle_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nle_f16_e64
+v_cmpx_tru_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nlg_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nlg_f16_e64
+v_cmpx_tru_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nlt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_nlt_f16_e64
+v_cmpx_u_f16 v1, v255
+// GFX11: v_cmpx_u_f16_e64 v1, v255               ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_o_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_o_f16_e64
+v_cmpx_u_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_t_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64
+v_cmpx_u_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_tru_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_t_f16_e64
+v_cmpx_u_f16 v255, v2
+// GFX11: v_cmpx_u_f16_e64 v255, v2               ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_u_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_u_f16_e64
+// GFX11: v_cmpx_u_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
+v_cmpx_u_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3c-fake16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c-fake16.s
new file mode 100644
index 000000000000..76db94023fc9
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c-fake16.s
@@ -0,0 +1,8695 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_cmp_class_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, v255, v2
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0xff,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, s1, v2
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x01,0x04,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, s105, v255
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x69,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, vcc_lo, s2
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x6a,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, vcc_hi, s105
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x6b,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, ttmp15, ttmp15
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x7b,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, m0, src_scc
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x7d,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x7d,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x7d,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x7d,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x7d,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x7d,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], v255, v2
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0xff,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], s1, v2
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x01,0x04,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], s105, v255
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x69,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], vcc_lo, s2
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x6a,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], vcc_hi, s105
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x6b,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], ttmp15, ttmp15
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x7b,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], m0, src_scc
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x7d,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x7d,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x7d,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x7d,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x7d,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 null, -|0xfe0b|, vcc_hi
+// GFX12: encoding: [0x7c,0x01,0x7d,0xd4,0xff,0xd6,0x00,0x20,0x0b,0xfe,0x00,0x00]
+
+v_cmp_class_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x7e,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x7e,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x7e,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x7e,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x7e,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x7e,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x7e,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x7e,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x7e,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f32_e64 null, -|0xaf123456|, vcc_hi
+// GFX12: encoding: [0x7c,0x01,0x7e,0xd4,0xff,0xd6,0x00,0x20,0x56,0x34,0x12,0xaf]
+
+v_cmp_class_f64_e64 s5, v[1:2], v2
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, v[1:2], v255
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, v[1:2], s2
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x01,0x05,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, v[1:2], s105
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x01,0xd3,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, v[254:255], ttmp15
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0xfe,0xf7,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, s[2:3], vcc_hi
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x02,0xd6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, s[104:105], vcc_lo
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x68,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, vcc, m0
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x6a,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, ttmp[14:15], exec_hi
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x7a,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s5, exec, exec_lo
+// W32: encoding: [0x05,0x00,0x7f,0xd4,0x7e,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s105, null, null
+// W32: encoding: [0x69,0x00,0x7f,0xd4,0x7c,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x7f,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 vcc_hi, 0.5, 0.5
+// W32: encoding: [0x6b,0x00,0x7f,0xd4,0xf0,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 ttmp15, -|src_scc|, src_scc
+// W32: encoding: [0x7b,0x01,0x7f,0xd4,0xfd,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], v[1:2], v2
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], v[1:2], v255
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], v[1:2], s2
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x01,0x05,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], v[1:2], s105
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x01,0xd3,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], v[254:255], ttmp15
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0xfe,0xf7,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], s[2:3], vcc_hi
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x02,0xd6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], s[104:105], vcc_lo
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x68,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], vcc, m0
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x6a,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], ttmp[14:15], exec_hi
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x7a,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], exec, exec_lo
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x7e,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[10:11], null, null
+// W64: encoding: [0x0a,0x00,0x7f,0xd4,0x7c,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x7f,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 vcc, 0.5, 0.5
+// W64: encoding: [0x6a,0x00,0x7f,0xd4,0xf0,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 ttmp[14:15], -|src_scc|, src_scc
+// W64: encoding: [0x7a,0x01,0x7f,0xd4,0xfd,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_class_f64_e64 null, 0xaf123456, 0xaf123456
+// GFX12: encoding: [0x7c,0x00,0x7f,0xd4,0xff,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_eq_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x02,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x02,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x02,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x02,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x02,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x02,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x02,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x02,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x02,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x02,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x02,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x02,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x02,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_eq_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x12,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x12,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x12,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x12,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x12,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x12,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x12,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x12,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x12,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x12,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x12,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x12,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x12,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_eq_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x22,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x22,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x22,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x22,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x22,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x22,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x22,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x22,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x22,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x22,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x22,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x22,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x22,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x22,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x22,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x22,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x22,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x22,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x22,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x22,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x22,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x22,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX12: encoding: [0x7c,0x82,0x22,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_eq_i16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x32,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x32,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x32,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x32,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x32,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x32,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x32,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x32,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64 null, 0xfe0b, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x32,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_eq_i32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x42,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x42,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x42,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x42,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x42,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x42,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x42,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x42,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x42,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x42,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64 null, 0xaf123456, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x42,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_eq_i64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x52,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x52,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x52,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x52,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x52,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x52,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x52,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x52,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x52,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x52,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x52,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x52,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x52,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x52,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x52,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_i64_e64 null, 0xaf123456, vcc
+// GFX12: encoding: [0x7c,0x00,0x52,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_eq_u16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x3a,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x3a,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x3a,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x3a,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x3a,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x3a,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x3a,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64 null, 0xfe0b, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x3a,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_eq_u32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x4a,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x4a,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x4a,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x4a,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x4a,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x4a,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x4a,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x4a,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x4a,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64 null, 0xaf123456, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x4a,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_eq_u64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x5a,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x5a,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x5a,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x5a,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x5a,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x5a,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x5a,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x5a,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x5a,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x5a,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x5a,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x5a,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x5a,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x5a,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x5a,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_eq_u64_e64 null, 0xaf123456, vcc
+// GFX12: encoding: [0x7c,0x00,0x5a,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ge_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x06,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x06,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x06,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x06,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x06,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x06,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x06,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x06,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x06,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x06,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x06,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x06,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x06,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_ge_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x16,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x16,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x16,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x16,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x16,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x16,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x16,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x16,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x16,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x16,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x16,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x16,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x16,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_ge_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x26,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x26,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x26,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x26,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x26,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x26,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x26,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x26,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x26,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x26,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x26,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x26,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x26,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x26,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x26,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x26,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x26,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x26,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x26,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x26,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x26,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x26,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX12: encoding: [0x7c,0x82,0x26,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_ge_i16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x36,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x36,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x36,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x36,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x36,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x36,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x36,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x36,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64 null, 0xfe0b, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x36,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_ge_i32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x46,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x46,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x46,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x46,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x46,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x46,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x46,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x46,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x46,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x46,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64 null, 0xaf123456, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x46,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ge_i64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x56,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x56,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x56,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x56,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x56,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x56,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x56,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x56,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x56,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x56,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x56,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x56,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x56,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x56,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x56,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_i64_e64 null, 0xaf123456, vcc
+// GFX12: encoding: [0x7c,0x00,0x56,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ge_u16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x3e,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x3e,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x3e,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x3e,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x3e,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x3e,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x3e,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64 null, 0xfe0b, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x3e,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_ge_u32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x4e,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x4e,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x4e,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x4e,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x4e,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x4e,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x4e,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x4e,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x4e,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64 null, 0xaf123456, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x4e,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ge_u64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x5e,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x5e,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x5e,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x5e,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x5e,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x5e,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x5e,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x5e,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x5e,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x5e,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x5e,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x5e,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x5e,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x5e,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x5e,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ge_u64_e64 null, 0xaf123456, vcc
+// GFX12: encoding: [0x7c,0x00,0x5e,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_gt_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x04,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x04,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x04,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x04,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x04,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x04,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x04,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x04,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x04,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x04,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x04,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x04,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x04,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_gt_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x14,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x14,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x14,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x14,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x14,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x14,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x14,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x14,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x14,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x14,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x14,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x14,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x14,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_gt_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x24,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x24,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x24,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x24,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x24,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x24,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x24,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x24,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x24,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x24,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x24,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x24,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x24,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x24,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x24,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x24,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x24,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x24,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x24,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x24,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x24,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x24,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX12: encoding: [0x7c,0x82,0x24,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_gt_i16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x34,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x34,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x34,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x34,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x34,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x34,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x34,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x34,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64 null, 0xfe0b, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x34,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_gt_i32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x44,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x44,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x44,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x44,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x44,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x44,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x44,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x44,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x44,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x44,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64 null, 0xaf123456, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x44,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_gt_i64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x54,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x54,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x54,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x54,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x54,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x54,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x54,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x54,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x54,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x54,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x54,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x54,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x54,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x54,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x54,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_i64_e64 null, 0xaf123456, vcc
+// GFX12: encoding: [0x7c,0x00,0x54,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_gt_u16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x3c,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x3c,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x3c,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x3c,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x3c,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x3c,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x3c,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64 null, 0xfe0b, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x3c,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_gt_u32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x4c,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x4c,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x4c,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x4c,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x4c,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x4c,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x4c,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x4c,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x4c,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64 null, 0xaf123456, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x4c,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_gt_u64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x5c,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x5c,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x5c,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x5c,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x5c,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x5c,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x5c,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x5c,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x5c,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x5c,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x5c,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x5c,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x5c,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x5c,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x5c,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_gt_u64_e64 null, 0xaf123456, vcc
+// GFX12: encoding: [0x7c,0x00,0x5c,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_le_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x03,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x03,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x03,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x03,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x03,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x03,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x03,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x03,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x03,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x03,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x03,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x03,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x03,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_le_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x13,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x13,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x13,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x13,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x13,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x13,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x13,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x13,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x13,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x13,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x13,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x13,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x13,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_le_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x23,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x23,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x23,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x23,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x23,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x23,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x23,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x23,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x23,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x23,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x23,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x23,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x23,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x23,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x23,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x23,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x23,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x23,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x23,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x23,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x23,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x23,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX12: encoding: [0x7c,0x82,0x23,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_le_i16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x33,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x33,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x33,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x33,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x33,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x33,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x33,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x33,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i16_e64 null, 0xfe0b, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x33,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_le_i32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x43,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x43,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x43,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x43,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x43,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x43,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x43,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x43,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x43,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x43,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i32_e64 null, 0xaf123456, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x43,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_le_i64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x53,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x53,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x53,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x53,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x53,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x53,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x53,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x53,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x53,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x53,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x53,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x53,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x53,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x53,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x53,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_i64_e64 null, 0xaf123456, vcc
+// GFX12: encoding: [0x7c,0x00,0x53,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_le_u16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x3b,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x3b,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x3b,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x3b,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x3b,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x3b,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x3b,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u16_e64 null, 0xfe0b, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x3b,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_le_u32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x4b,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x4b,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x4b,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x4b,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x4b,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x4b,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x4b,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x4b,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x4b,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u32_e64 null, 0xaf123456, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x4b,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_le_u64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x5b,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x5b,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x5b,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x5b,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x5b,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x5b,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x5b,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x5b,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x5b,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x5b,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x5b,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x5b,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x5b,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x5b,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x5b,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_le_u64_e64 null, 0xaf123456, vcc
+// GFX12: encoding: [0x7c,0x00,0x5b,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_lg_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x05,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x05,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x05,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x05,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x05,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x05,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x05,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x05,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x05,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x05,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x05,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x05,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x05,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_lg_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x15,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x15,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x15,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x15,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x15,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x15,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x15,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x15,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x15,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x15,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x15,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x15,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x15,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_lg_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x25,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x25,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x25,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x25,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x25,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x25,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x25,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x25,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x25,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x25,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x25,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x25,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x25,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x25,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x25,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x25,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x25,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x25,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x25,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x25,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x25,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x25,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lg_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX12: encoding: [0x7c,0x82,0x25,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_lt_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x01,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x01,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x01,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x01,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x01,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x01,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x01,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x01,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x01,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x01,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x01,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x01,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x01,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_lt_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x11,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x11,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x11,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x11,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x11,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x11,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x11,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x11,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x11,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x11,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x11,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x11,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x11,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_lt_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x21,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x21,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x21,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x21,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x21,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x21,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x21,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x21,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x21,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x21,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x21,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x21,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x21,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x21,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x21,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x21,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x21,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x21,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x21,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x21,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x21,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x21,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX12: encoding: [0x7c,0x82,0x21,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_lt_i16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x31,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x31,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x31,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x31,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x31,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x31,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x31,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x31,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64 null, 0xfe0b, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x31,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_lt_i32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x41,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x41,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x41,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x41,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x41,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x41,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x41,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x41,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x41,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x41,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64 null, 0xaf123456, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x41,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_lt_i64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x51,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x51,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x51,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x51,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x51,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x51,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x51,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x51,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x51,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x51,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x51,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x51,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x51,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x51,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x51,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_i64_e64 null, 0xaf123456, vcc
+// GFX12: encoding: [0x7c,0x00,0x51,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_lt_u16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x39,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x39,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x39,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x39,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x39,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x39,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x39,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x39,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64 null, 0xfe0b, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x39,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_lt_u32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x49,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x49,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x49,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x49,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x49,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x49,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x49,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x49,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x49,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x49,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64 null, 0xaf123456, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x49,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_lt_u64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x59,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x59,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x59,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x59,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x59,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x59,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x59,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x59,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x59,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x59,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x59,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x59,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x59,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x59,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x59,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_lt_u64_e64 null, 0xaf123456, vcc
+// GFX12: encoding: [0x7c,0x00,0x59,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ne_i16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x35,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x35,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x35,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x35,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x35,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x35,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x35,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x35,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64 null, 0xfe0b, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x35,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_ne_i32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x45,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x45,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x45,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x45,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x45,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x45,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x45,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x45,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x45,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x45,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64 null, 0xaf123456, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x45,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ne_i64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x55,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x55,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x55,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x55,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x55,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x55,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x55,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x55,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x55,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x55,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x55,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x55,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x55,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x55,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x55,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_i64_e64 null, 0xaf123456, vcc
+// GFX12: encoding: [0x7c,0x00,0x55,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ne_u16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x3d,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x3d,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x3d,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x3d,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x3d,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x3d,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x3d,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64 null, 0xfe0b, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x3d,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmp_ne_u32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s5, exec_hi, null
+// W32: encoding: [0x05,0x00,0x4d,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x4d,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x4d,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 vcc_hi, 0.5, m0
+// W32: encoding: [0x6b,0x00,0x4d,0xd4,0xf0,0xfa,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 ttmp15, src_scc, vcc_lo
+// W32: encoding: [0x7b,0x00,0x4d,0xd4,0xfd,0xd4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], exec_hi, null
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x4d,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x4d,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 vcc, 0.5, m0
+// W64: encoding: [0x6a,0x00,0x4d,0xd4,0xf0,0xfa,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 ttmp[14:15], src_scc, vcc_lo
+// W64: encoding: [0x7a,0x00,0x4d,0xd4,0xfd,0xd4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64 null, 0xaf123456, vcc_hi
+// GFX12: encoding: [0x7c,0x00,0x4d,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_ne_u64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x5d,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x5d,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x5d,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x5d,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x5d,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x5d,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s5, exec, src_scc
+// W32: encoding: [0x05,0x00,0x5d,0xd4,0x7e,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x5d,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x5d,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x5d,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 ttmp15, src_scc, exec
+// W32: encoding: [0x7b,0x00,0x5d,0xd4,0xfd,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], exec, src_scc
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0x7e,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x5d,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x5d,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x5d,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 ttmp[14:15], src_scc, exec
+// W64: encoding: [0x7a,0x00,0x5d,0xd4,0xfd,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ne_u64_e64 null, 0xaf123456, vcc
+// GFX12: encoding: [0x7c,0x00,0x5d,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmp_neq_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x0d,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x0d,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x0d,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x0d,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x0d,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x0d,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x0d,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x0d,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x0d,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x0d,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x0d,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x0d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_neq_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x1d,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x1d,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x1d,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x1d,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x1d,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x1d,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x1d,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x1d,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x1d,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x1d,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x1d,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x1d,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_neq_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x2d,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x2d,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x2d,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x2d,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x2d,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x2d,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x2d,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x2d,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x2d,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x2d,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x2d,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x2d,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x2d,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x2d,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x2d,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x2d,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x2d,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x2d,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x2d,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x2d,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x2d,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x2d,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_neq_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX12: encoding: [0x7c,0x82,0x2d,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_nge_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x09,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x09,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x09,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x09,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x09,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x09,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x09,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x09,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x09,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x09,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x09,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x09,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x09,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_nge_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x19,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x19,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x19,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x19,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x19,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x19,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x19,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x19,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x19,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x19,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x19,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x19,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x19,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_nge_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x29,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x29,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x29,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x29,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x29,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x29,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x29,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x29,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x29,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x29,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x29,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x29,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x29,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x29,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x29,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x29,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x29,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x29,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x29,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x29,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x29,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x29,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nge_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX12: encoding: [0x7c,0x82,0x29,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_ngt_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x0b,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x0b,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x0b,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x0b,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x0b,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x0b,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x0b,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x0b,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x0b,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x0b,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x0b,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x0b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_ngt_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x1b,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x1b,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x1b,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x1b,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x1b,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x1b,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x1b,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x1b,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x1b,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x1b,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x1b,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x1b,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_ngt_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x2b,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x2b,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x2b,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x2b,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x2b,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x2b,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x2b,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x2b,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x2b,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x2b,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x2b,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x2b,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x2b,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x2b,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x2b,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x2b,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x2b,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x2b,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x2b,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x2b,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x2b,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x2b,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_ngt_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX12: encoding: [0x7c,0x82,0x2b,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_nle_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x0c,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x0c,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x0c,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x0c,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x0c,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x0c,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x0c,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x0c,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x0c,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x0c,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x0c,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x0c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_nle_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x1c,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x1c,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x1c,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x1c,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x1c,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x1c,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x1c,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x1c,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x1c,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x1c,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x1c,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x1c,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_nle_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x2c,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x2c,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x2c,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x2c,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x2c,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x2c,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x2c,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x2c,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x2c,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x2c,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x2c,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x2c,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x2c,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x2c,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x2c,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x2c,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x2c,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x2c,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x2c,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x2c,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x2c,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x2c,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nle_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX12: encoding: [0x7c,0x82,0x2c,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_nlg_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x0a,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x0a,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x0a,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x0a,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x0a,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x0a,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x0a,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x0a,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x0a,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x0a,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x0a,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x0a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_nlg_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x1a,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x1a,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x1a,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x1a,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x1a,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x1a,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x1a,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x1a,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x1a,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x1a,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x1a,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x1a,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_nlg_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x2a,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x2a,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x2a,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x2a,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x2a,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x2a,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x2a,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x2a,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x2a,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x2a,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x2a,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x2a,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x2a,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x2a,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x2a,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x2a,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x2a,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x2a,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x2a,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x2a,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x2a,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x2a,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlg_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX12: encoding: [0x7c,0x82,0x2a,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_nlt_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x0e,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x0e,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x0e,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x0e,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x0e,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x0e,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x0e,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x0e,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x0e,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x0e,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x0e,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x0e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_nlt_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x1e,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x1e,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x1e,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x1e,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x1e,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x1e,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x1e,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x1e,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x1e,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x1e,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x1e,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x1e,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_nlt_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x2e,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x2e,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x2e,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x2e,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x2e,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x2e,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x2e,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x2e,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x2e,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x2e,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x2e,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x2e,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x2e,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x2e,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x2e,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x2e,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x2e,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x2e,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x2e,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x2e,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x2e,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x2e,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_nlt_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX12: encoding: [0x7c,0x82,0x2e,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_o_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x07,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x07,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x07,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x07,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x07,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x07,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x07,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x07,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x07,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x07,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x07,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x07,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x07,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_o_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x17,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x17,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x17,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x17,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x17,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x17,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x17,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x17,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x17,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x17,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x17,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x17,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x17,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_o_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x27,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x27,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x27,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x27,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x27,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x27,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x27,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x27,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x27,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x27,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x27,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x27,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x27,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x27,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x27,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x27,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x27,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x27,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x27,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x27,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x27,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x27,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_o_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX12: encoding: [0x7c,0x82,0x27,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmp_u_f16_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x08,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, vcc_hi, 0xfe0b
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x08,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x08,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x08,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x08,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x08,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x08,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], vcc_hi, 0xfe0b
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x08,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x08,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x08,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x08,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x08,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f16_e64 null, -|0xfe0b|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x08,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmp_u_f32_e64 s5, v1, v2
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, v255, v255
+// W32: encoding: [0x05,0x00,0x18,0xd4,0xff,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, s1, s2
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x01,0x04,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, s105, s105
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x69,0xd2,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, vcc_lo, ttmp15
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x6a,0xf6,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, vcc_hi, 0xaf123456
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, ttmp15, src_scc
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x7b,0xfa,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, m0, 0.5
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x7d,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, exec_lo, -1
+// W32: encoding: [0x05,0x00,0x18,0xd4,0x7e,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s5, |exec_hi|, null
+// W32: encoding: [0x05,0x01,0x18,0xd4,0x7f,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s105, null, exec_lo
+// W32: encoding: [0x69,0x00,0x18,0xd4,0x7c,0xfc,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 vcc_lo, -1, exec_hi
+// W32: encoding: [0x6a,0x00,0x18,0xd4,0xc1,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 vcc_hi, 0.5, -m0
+// W32: encoding: [0x6b,0x00,0x18,0xd4,0xf0,0xfa,0x00,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 ttmp15, -src_scc, |vcc_lo|
+// W32: encoding: [0x7b,0x02,0x18,0xd4,0xfd,0xd4,0x00,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], v1, v2
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], v255, v255
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0xff,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], s1, s2
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x01,0x04,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], s105, s105
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x69,0xd2,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], vcc_lo, ttmp15
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x6a,0xf6,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], vcc_hi, 0xaf123456
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], ttmp15, src_scc
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x7b,0xfa,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], m0, 0.5
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x7d,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], exec_lo, -1
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x7e,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], |exec_hi|, null
+// W64: encoding: [0x0a,0x01,0x18,0xd4,0x7f,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[10:11], null, exec_lo
+// W64: encoding: [0x0a,0x00,0x18,0xd4,0x7c,0xfc,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 s[104:105], -1, exec_hi
+// W64: encoding: [0x68,0x00,0x18,0xd4,0xc1,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 vcc, 0.5, -m0
+// W64: encoding: [0x6a,0x00,0x18,0xd4,0xf0,0xfa,0x00,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 ttmp[14:15], -src_scc, |vcc_lo|
+// W64: encoding: [0x7a,0x02,0x18,0xd4,0xfd,0xd4,0x00,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f32_e64 null, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: encoding: [0x7c,0x83,0x18,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmp_u_f64_e64 s5, v[1:2], v[2:3]
+// W32: encoding: [0x05,0x00,0x28,0xd4,0x01,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s5, v[254:255], v[254:255]
+// W32: encoding: [0x05,0x00,0x28,0xd4,0xfe,0xfd,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s5, s[2:3], s[4:5]
+// W32: encoding: [0x05,0x00,0x28,0xd4,0x02,0x08,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s5, s[104:105], s[104:105]
+// W32: encoding: [0x05,0x00,0x28,0xd4,0x68,0xd0,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s5, vcc, ttmp[14:15]
+// W32: encoding: [0x05,0x00,0x28,0xd4,0x6a,0xf4,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s5, ttmp[14:15], 0xaf123456
+// W32: encoding: [0x05,0x00,0x28,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s5, -|exec|, src_scc
+// W32: encoding: [0x05,0x01,0x28,0xd4,0x7e,0xfa,0x01,0x20]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s105, null, 0.5
+// W32: encoding: [0x69,0x00,0x28,0xd4,0x7c,0xe0,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 vcc_lo, -1, -1
+// W32: encoding: [0x6a,0x00,0x28,0xd4,0xc1,0x82,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 vcc_hi, 0.5, null
+// W32: encoding: [0x6b,0x00,0x28,0xd4,0xf0,0xf8,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 ttmp15, -|src_scc|, -|exec|
+// W32: encoding: [0x7b,0x03,0x28,0xd4,0xfd,0xfc,0x00,0x60]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], v[1:2], v[2:3]
+// W64: encoding: [0x0a,0x00,0x28,0xd4,0x01,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], v[254:255], v[254:255]
+// W64: encoding: [0x0a,0x00,0x28,0xd4,0xfe,0xfd,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], s[2:3], s[4:5]
+// W64: encoding: [0x0a,0x00,0x28,0xd4,0x02,0x08,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], s[104:105], s[104:105]
+// W64: encoding: [0x0a,0x00,0x28,0xd4,0x68,0xd0,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], vcc, ttmp[14:15]
+// W64: encoding: [0x0a,0x00,0x28,0xd4,0x6a,0xf4,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], ttmp[14:15], 0xaf123456
+// W64: encoding: [0x0a,0x00,0x28,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], -|exec|, src_scc
+// W64: encoding: [0x0a,0x01,0x28,0xd4,0x7e,0xfa,0x01,0x20]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[10:11], null, 0.5
+// W64: encoding: [0x0a,0x00,0x28,0xd4,0x7c,0xe0,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 s[104:105], -1, -1
+// W64: encoding: [0x68,0x00,0x28,0xd4,0xc1,0x82,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 vcc, 0.5, null
+// W64: encoding: [0x6a,0x00,0x28,0xd4,0xf0,0xf8,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 ttmp[14:15], -|src_scc|, -|exec|
+// W64: encoding: [0x7a,0x03,0x28,0xd4,0xfd,0xfc,0x00,0x60]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cmp_u_f64_e64 null, 0xaf123456, -|vcc| clamp
+// GFX12: encoding: [0x7c,0x82,0x28,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s
index 98dbbf6cff44..76db94023fc9 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s
@@ -1,7 +1,7 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_cmp_class_f16_e64 s5, v1, v2
 // W32: encoding: [0x05,0x00,0x7d,0xd4,0x01,0x05,0x02,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc-fake16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc-fake16.s
new file mode 100644
index 000000000000..ebac9fed9250
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc-fake16.s
@@ -0,0 +1,9076 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefix=W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefix=W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_cmp_class_f16_e32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0xfa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0xfa,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0xfa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0xfa,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0xfc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0xfd,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0xfc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0xfd,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, v[1:2], v2
+// W32: encoding: [0x01,0x05,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, v[254:255], v2
+// W32: encoding: [0xfe,0x05,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, s[2:3], v2
+// W32: encoding: [0x02,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, s[104:105], v2
+// W32: encoding: [0x68,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, vcc, v2
+// W32: encoding: [0x6a,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, ttmp[14:15], v2
+// W32: encoding: [0x7a,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, exec, v2
+// W32: encoding: [0x7e,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0xfe,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0xff,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, v[1:2], v2
+// W64: encoding: [0x01,0x05,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, v[254:255], v2
+// W64: encoding: [0xfe,0x05,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, s[2:3], v2
+// W64: encoding: [0x02,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, s[104:105], v2
+// W64: encoding: [0x68,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, vcc, v2
+// W64: encoding: [0x6a,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, ttmp[14:15], v2
+// W64: encoding: [0x7a,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, exec, v2
+// W64: encoding: [0x7e,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0xfe,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f64 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0xff,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x04,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x04,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x04,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x04,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x24,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x25,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x24,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x25,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x44,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x45,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x44,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x45,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x64,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x64,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x64,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x64,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x84,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x85,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x84,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x85,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xa4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xa5,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xa4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xa5,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x74,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x74,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x74,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x74,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x94,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x95,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x94,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x95,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xb4,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xb5,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xb4,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xb5,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x0c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x0c,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x0c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x0c,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x2c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x2d,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x2c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x2d,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x4c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x4d,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x4c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x4d,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x6c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x6c,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x6c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x6c,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x8c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x8d,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x8c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x8d,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xac,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xad,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xac,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xad,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x7c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x7c,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x7c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x7c,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x9c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x9d,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x9c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x9d,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xbc,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xbd,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xbc,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xbd,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x08,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x08,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x08,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x08,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x28,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x29,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x28,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x29,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x48,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x49,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x48,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x49,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x68,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x68,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x68,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x68,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x88,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x89,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x88,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x89,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xa8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xa9,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xa8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xa9,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x78,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x78,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x78,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x78,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x98,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x99,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x98,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x99,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xb8,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xb9,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xb8,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xb9,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x06,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x06,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x06,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x06,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x26,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x27,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x26,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x27,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x46,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x47,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x46,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x47,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x66,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x66,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x66,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x66,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x86,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x87,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x86,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x87,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xa6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xa7,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xa6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xa7,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x76,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x76,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x76,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x76,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x96,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x97,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x96,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x97,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xb6,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xb7,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xb6,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xb7,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x0a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x0a,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x0a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x0a,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x2a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x2b,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x2a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x2b,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x4a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x4b,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x4a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x4b,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x02,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x02,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x02,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x02,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x22,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x23,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x22,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x23,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x42,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x43,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x42,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x43,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x62,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x62,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x62,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x62,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x82,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x83,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x82,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x83,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xa2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xa3,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xa2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xa3,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x72,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x72,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x72,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x72,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x92,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x93,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x92,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x93,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xb2,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xb3,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xb2,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xb3,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x6a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x6a,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x6a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x6a,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x8a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x8b,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x8a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x8b,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xaa,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xab,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xaa,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xab,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x7a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x7a,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x7a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x7a,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x9a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x9b,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x9a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x9b,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0xba,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0xbb,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0xba,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0xbb,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x1a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x1a,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x1a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x1a,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x3a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x3b,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x3a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x3b,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x5a,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x5b,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x5a,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x5b,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x12,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x12,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x12,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x12,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x32,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x33,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x32,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x33,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x52,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x53,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x52,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x53,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x16,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x16,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x16,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x16,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x36,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x37,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x36,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x37,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x56,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x57,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x56,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x57,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x18,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x18,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x18,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x18,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x38,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x39,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x38,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x39,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x58,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x59,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x58,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x59,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x14,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x14,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x14,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x14,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x34,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x35,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x34,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x35,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x54,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x55,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x54,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x55,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x1c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x1c,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x1c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x1c,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x3c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x3d,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x3c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x3d,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x5c,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x5d,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x5c,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x5d,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x0e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x0e,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x0e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x0e,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x2e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x2f,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x2e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x2f,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x4e,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x4f,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x4e,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x4f,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v127, v2
+// W32: encoding: [0x7f,0x05,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x10,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, 0xfe0b, v127
+// W32: encoding: [0xff,0xfe,0x10,0x7c,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v127, v2
+// W64: encoding: [0x7f,0x05,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x10,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, 0xfe0b, v127
+// W64: encoding: [0xff,0xfe,0x10,0x7c,0x0b,0xfe,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2
+// W32: encoding: [0x01,0x05,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v255, v2
+// W32: encoding: [0xff,0x05,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, s1, v2
+// W32: encoding: [0x01,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, s105, v2
+// W32: encoding: [0x69,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, vcc_lo, v2
+// W32: encoding: [0x6a,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, vcc_hi, v2
+// W32: encoding: [0x6b,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, ttmp15, v2
+// W32: encoding: [0x7b,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, m0, v2
+// W32: encoding: [0x7d,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, exec_lo, v2
+// W32: encoding: [0x7e,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, exec_hi, v2
+// W32: encoding: [0x7f,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, null, v2
+// W32: encoding: [0x7c,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, -1, v2
+// W32: encoding: [0xc1,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, 0.5, v2
+// W32: encoding: [0xf0,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, src_scc, v2
+// W32: encoding: [0xfd,0x04,0x30,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, 0xaf123456, v255
+// W32: encoding: [0xff,0xfe,0x31,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2
+// W64: encoding: [0x01,0x05,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v255, v2
+// W64: encoding: [0xff,0x05,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, s1, v2
+// W64: encoding: [0x01,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, s105, v2
+// W64: encoding: [0x69,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, vcc_lo, v2
+// W64: encoding: [0x6a,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, vcc_hi, v2
+// W64: encoding: [0x6b,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, ttmp15, v2
+// W64: encoding: [0x7b,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, m0, v2
+// W64: encoding: [0x7d,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, exec_lo, v2
+// W64: encoding: [0x7e,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, exec_hi, v2
+// W64: encoding: [0x7f,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, null, v2
+// W64: encoding: [0x7c,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, -1, v2
+// W64: encoding: [0xc1,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, 0.5, v2
+// W64: encoding: [0xf0,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, src_scc, v2
+// W64: encoding: [0xfd,0x04,0x30,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, 0xaf123456, v255
+// W64: encoding: [0xff,0xfe,0x31,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, v[1:2], v[2:3]
+// W32: encoding: [0x01,0x05,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, v[254:255], v[2:3]
+// W32: encoding: [0xfe,0x05,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, s[2:3], v[2:3]
+// W32: encoding: [0x02,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, s[104:105], v[2:3]
+// W32: encoding: [0x68,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, vcc, v[2:3]
+// W32: encoding: [0x6a,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, ttmp[14:15], v[2:3]
+// W32: encoding: [0x7a,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, exec, v[2:3]
+// W32: encoding: [0x7e,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, null, v[2:3]
+// W32: encoding: [0x7c,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, -1, v[2:3]
+// W32: encoding: [0xc1,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, 0.5, v[2:3]
+// W32: encoding: [0xf0,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, src_scc, v[2:3]
+// W32: encoding: [0xfd,0x04,0x50,0x7c]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc_lo, 0xaf123456, v[254:255]
+// W32: encoding: [0xff,0xfc,0x51,0x7c,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, v[1:2], v[2:3]
+// W64: encoding: [0x01,0x05,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, v[254:255], v[2:3]
+// W64: encoding: [0xfe,0x05,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, s[2:3], v[2:3]
+// W64: encoding: [0x02,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, s[104:105], v[2:3]
+// W64: encoding: [0x68,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, vcc, v[2:3]
+// W64: encoding: [0x6a,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, ttmp[14:15], v[2:3]
+// W64: encoding: [0x7a,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, exec, v[2:3]
+// W64: encoding: [0x7e,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, null, v[2:3]
+// W64: encoding: [0x7c,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, -1, v[2:3]
+// W64: encoding: [0xc1,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, 0.5, v[2:3]
+// W64: encoding: [0xf0,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, src_scc, v[2:3]
+// W64: encoding: [0xfd,0x04,0x50,0x7c]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f64 vcc, 0xaf123456, v[254:255]
+// W64: encoding: [0xff,0xfc,0x51,0x7c,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s
index c9241ebd161d..4ae4f74ad219 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s
@@ -1,7 +1,7 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefix=W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefix=W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefix=W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_cmp_class_f16_e32 vcc_lo, v1, v2
 // W32: encoding: [0x01,0x05,0xfa,0x7c]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp16-fake16.s
new file mode 100644
index 000000000000..13c4f89cb702
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp16-fake16.s
@@ -0,0 +1,6052 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefix=W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefix=W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_cmp_class_f16_dpp vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, -|v127|, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0xfa,0x7c,0x7f,0x6f,0x35,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, -|v127|, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0xfa,0x7c,0x7f,0x6f,0x35,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0xfd,0x7c,0xff,0x6f,0x35,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0xfd,0x7c,0xff,0x6f,0x35,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x04,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x04,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x04,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x25,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x24,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x25,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x64,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x64,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x64,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x85,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x84,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x85,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x74,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x74,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x74,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x95,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x94,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x95,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x0c,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x0c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x0c,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x2d,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x2c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x2d,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x6c,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x6c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x6c,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x8d,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x8c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x8d,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x7c,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x7c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x7c,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x9d,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x9c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x9d,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x08,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x08,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x08,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x29,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x28,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x29,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x68,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x68,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x68,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x89,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x88,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x89,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x78,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x78,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x78,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x99,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x98,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x99,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x06,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x06,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x06,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x27,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x27,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x66,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x66,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x66,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x87,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x86,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x87,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x76,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x76,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x76,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x97,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x96,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x97,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x0a,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x0a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x0a,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x2b,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x2a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x2b,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x02,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x02,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x02,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x23,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x22,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x23,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x62,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x62,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x62,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x83,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x82,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x83,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x72,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x72,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x72,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x93,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x92,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x93,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x6a,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x6a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x6a,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x8b,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x8a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x8b,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x7a,0x7c,0x7f,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x7a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x7a,0x7c,0x7f,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x9b,0x7c,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x9a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x9b,0x7c,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x1a,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x1a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x1a,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x3b,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x3a,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x3b,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x12,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x12,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x12,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x33,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x32,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x33,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x16,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x16,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x16,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x37,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x36,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x37,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x18,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x18,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x18,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x39,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x38,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x39,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x14,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x14,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x14,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x35,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x34,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x35,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x1c,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x1c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x1c,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x3d,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x3c,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x3d,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x0e,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x0e,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x0e,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x2f,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x2e,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x2f,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x10,0x7c,0x7f,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x10,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x10,0x7c,0x7f,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_mirror
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_half_mirror
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_shl:1
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_shl:15
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_shr:1
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_shr:15
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_ror:1
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_ror:15
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0x31,0x7c,0xff,0x6f,0xf5,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_mirror
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_half_mirror
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_shl:1
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_shl:15
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_shr:1
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_shr:15
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_ror:1
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_ror:15
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x30,0x7c,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0x31,0x7c,0xff,0x6f,0xf5,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp16.s
index 0c3a38626fa6..0c36108cb0cb 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp16.s
@@ -1,7 +1,7 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefix=W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefix=W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefix=W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_cmp_class_f16_dpp vcc_lo, v1, v2 quad_perm:[3,2,1,0]
 // W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp8-fake16.s
new file mode 100644
index 000000000000..87305ec913d1
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp8-fake16.s
@@ -0,0 +1,1300 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefix=W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefix=W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_cmp_class_f16_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0xfa,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0xfa,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0xfc,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0xfc,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0xfd,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0xfc,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0xfc,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0xfd,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x04,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x04,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x04,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x04,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x04,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x04,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x24,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x24,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x25,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x24,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x24,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x25,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x64,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x64,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x64,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x64,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x64,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x64,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x84,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x84,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x85,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x84,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x84,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x85,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x74,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x74,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x74,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x74,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x74,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x74,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x94,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x94,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x95,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x94,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x94,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x95,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x0c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x0c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x0c,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x0c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x0c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x0c,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x2c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x2c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x2d,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x2c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x2c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x2d,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x6c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x6c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x6c,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x6c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x6c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x6c,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x8c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x8c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x8d,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x8c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x8c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x8d,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x7c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x7c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x7c,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x7c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x7c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x7c,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x9c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x9c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x9d,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x9c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x9c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x9d,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x08,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x08,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x08,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x08,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x08,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x08,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x28,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x28,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x29,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x28,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x28,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x29,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x68,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x68,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x68,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x68,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x68,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x68,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x88,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x88,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x89,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x88,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x88,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x89,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x78,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x78,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x78,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x78,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x78,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x78,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x98,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x98,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x99,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x98,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x98,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x99,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x06,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x06,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x06,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x06,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x06,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x06,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x26,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x26,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x27,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x26,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x26,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x27,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x66,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x66,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x66,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x66,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x66,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x66,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x86,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x86,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x87,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x86,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x86,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x87,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x76,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x76,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x76,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x76,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x76,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x76,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x96,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x96,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x97,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x96,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x96,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x97,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x0a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x0a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x0a,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x0a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x0a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x0a,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x2a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x2a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x2b,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x2a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x2a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x2b,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x02,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x02,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x02,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x02,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x02,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x02,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x22,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x22,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x23,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x22,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x22,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x23,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x62,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x62,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x62,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x62,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x62,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x62,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x82,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x82,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x83,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x82,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x82,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x83,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x72,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x72,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x72,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x72,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x72,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x72,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x92,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x92,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x93,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x92,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x92,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x93,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x6a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x6a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x6a,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x6a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x6a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x6a,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x8a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x8a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x8b,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x8a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x8a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x8b,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x7a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x7a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x7a,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x7a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x7a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x7a,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x9a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x9a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x9b,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x9a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x9a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x9b,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x1a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x1a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x1a,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x1a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x1a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x1a,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x3a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x3a,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x3b,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x3a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x3a,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x3b,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x12,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x12,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x12,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x12,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x12,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x12,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x32,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x32,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x33,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x32,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x32,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x33,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x16,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x16,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x16,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x16,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x16,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x16,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x36,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x36,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x37,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x36,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x36,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x37,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x18,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x18,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x18,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x18,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x18,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x18,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x38,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x38,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x39,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x38,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x38,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x39,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x14,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x14,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x14,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x14,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x14,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x14,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x34,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x34,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x35,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x34,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x34,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x35,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x1c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x1c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x1c,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x1c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x1c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x1c,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x3c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x3c,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x3d,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x3c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x3c,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x3d,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x0e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x0e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x0e,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x0e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x0e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x0e,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x2e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x2e,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x2f,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x2e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x2e,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x2f,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x10,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x10,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc_lo, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x10,0x7c,0x7f,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x10,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x10,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x10,0x7c,0x7f,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x30,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x30,0x7c,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc_lo, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0x31,0x7c,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x30,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x30,0x7c,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f32 vcc, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0x31,0x7c,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp8.s
index ceecbc660d06..2a4095f99d83 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp8.s
@@ -1,7 +1,7 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefix=W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefix=W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefix=W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_cmp_class_f16_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // W32: encoding: [0xe9,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_err.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_err.s
index ebdd8adc0adf..e603e7388a68 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_err.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_err.s
@@ -1,1774 +1,1775 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort --version 5
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s
 
 v_cmp_class_f16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_class_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_class_f16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_class_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_class_f16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_class_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_class_f16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_class_f16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_class_f16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:40: error: invalid operand for instruction
+
+v_cmp_class_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:40: error: invalid operand for instruction
+
+v_cmp_class_f16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_class_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:38: error: invalid operand for instruction
 
 v_cmp_class_f16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_class_f16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_f16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_eq_f16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_eq_f16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_f16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_f16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_eq_f16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_eq_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_eq_f16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_f16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_eq_i16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_eq_i16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_eq_i16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_eq_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_eq_i16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_eq_u16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_eq_u16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_eq_u16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_eq_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_eq_u16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_f16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ge_f16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ge_f16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_f16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_f16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ge_f16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ge_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ge_f16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_f16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ge_i16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ge_i16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ge_i16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ge_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ge_i16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ge_u16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ge_u16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ge_u16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ge_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ge_u16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_f16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_gt_f16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_gt_f16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_f16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_f16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_gt_f16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_gt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_gt_f16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_f16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_gt_i16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_gt_i16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_gt_i16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_gt_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_gt_i16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_gt_u16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_gt_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
-v_cmp_gt_u16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_gt_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
-v_cmp_gt_u16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_gt_u16_e32 vcc, v127, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_gt_u16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_gt_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc, vcc_hi, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16_e32 vcc, vcc_lo, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16_e32 vcc_lo, v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_gt_u16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_gt_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_gt_u16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_f16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_le_f16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_le_f16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_f16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_f16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_le_f16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_le_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_le_f16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_f16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_le_i16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_le_i16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_le_i16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_le_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_le_i16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_le_u16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_le_u16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_le_u16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_le_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_le_u16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lg_f16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lg_f16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lg_f16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lg_f16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lg_f16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lg_f16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lg_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lg_f16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lg_f16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_f16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lt_f16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lt_f16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_f16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_f16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lt_f16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lt_f16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_f16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lt_i16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lt_i16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lt_i16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lt_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lt_i16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lt_u16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_lt_u16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lt_u16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_lt_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_lt_u16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ne_i16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
 v_cmp_ne_i16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ne_i16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ne_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ne_i16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_ne_u16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_ne_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
 
-v_cmp_ne_u16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_ne_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc, v127, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc, vcc_hi, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ne_u16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:37: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ne_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
 
 v_cmp_ne_u16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_neq_f16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_neq_f16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_neq_f16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_neq_f16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_neq_f16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_neq_f16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_neq_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_neq_f16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_neq_f16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nge_f16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nge_f16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nge_f16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nge_f16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nge_f16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nge_f16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nge_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nge_f16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nge_f16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ngt_f16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_ngt_f16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_ngt_f16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ngt_f16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ngt_f16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_ngt_f16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_ngt_f16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ngt_f16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nle_f16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nle_f16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nle_f16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nle_f16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nle_f16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nle_f16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nle_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nle_f16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nle_f16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlg_f16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nlg_f16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nlg_f16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlg_f16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlg_f16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nlg_f16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nlg_f16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlg_f16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlt_f16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nlt_f16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:35: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
 
 v_cmp_nlt_f16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlt_f16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlt_f16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nlt_f16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:38: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
 
 v_cmp_nlt_f16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlt_f16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_o_f16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
 v_cmp_o_f16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
 v_cmp_o_f16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_o_f16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_o_f16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
 
 v_cmp_o_f16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_cmp_o_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
 
 v_cmp_o_f16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_o_f16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_u_f16_e32 vcc, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
+
+v_cmp_u_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
 v_cmp_u_f16_e32 vcc, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_u_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:33: error: invalid operand for instruction
+
+v_cmp_u_f16_e32 vcc, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
+
+v_cmp_u_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
 v_cmp_u_f16_e32 vcc, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_u_f16_e32 vcc, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_u_f16_e32 vcc_lo, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_u_f16_e32 vcc_lo, v127, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
 
-v_cmp_u_f16_e32 vcc_lo, vcc_hi, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
 
-v_cmp_u_f16_e32 vcc_lo, vcc_lo, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16_e32 vcc_lo, v127, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_class_f16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
 
-v_cmp_class_f16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:36: error: invalid operand for instruction
 
-v_cmp_eq_f16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16_e32 vcc_lo, v128, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_eq_f16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
 
-v_cmp_eq_i16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:34: error: invalid operand for instruction
 
-v_cmp_eq_i16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16_e32 vcc_lo, vcc_hi, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_eq_u16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_eq_u16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_f16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_f16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_i16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_i16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_u16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_u16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_f16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_f16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_i16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_i16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_u16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_u16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_f16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_f16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_i16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_i16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_u16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_u16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lg_f16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lg_f16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_f16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_f16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_i16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_i16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_u16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_u16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ne_i16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ne_i16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ne_u16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ne_u16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_neq_f16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_neq_f16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nge_f16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nge_f16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ngt_f16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ngt_f16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nle_f16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nle_f16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nlg_f16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nlg_f16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nlt_f16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nlt_f16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_o_f16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_o_f16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_u_f16_e32 vcc, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_u_f16_e32 vcc_lo, v128, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_class_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16_e32 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16_e32 vcc_lo, vcc_lo, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_promote.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_promote.s
index 65c0a3c874ef..e56c46bb5544 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_promote.s
@@ -1,2368 +1,2369 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefix=W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort --version 5
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefix=W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefix=W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 %s 2>&1 > /dev/null | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 %s 2>&1 > /dev/null | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_cmp_class_f16 vcc, v1, v255
-// W64: v_cmp_class_f16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_class_f16_e64 vcc, v1, v255       ; encoding: [0x6a,0x00,0x7d,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_class_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x7d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
+
+v_cmp_class_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_class_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc, v127, v255
-// W64: v_cmp_class_f16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_class_f16_e64 vcc, v127, v255     ; encoding: [0x6a,0x00,0x7d,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_class_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x7d,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
+
+v_cmp_class_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_class_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7d,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
+
+v_cmp_class_f16 vcc, v128, v2
+// W64: v_cmp_class_f16_e64 vcc, v128, v2       ; encoding: [0x6a,0x00,0x7d,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_class_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
+
+v_cmp_class_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_class_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc, vcc_hi, v255
-// W64: v_cmp_class_f16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_class_f16_e64 vcc, vcc_hi, v255   ; encoding: [0x6a,0x00,0x7d,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_class_f16 vcc, vcc_lo, v255
-// W64: v_cmp_class_f16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_class_f16_e64 vcc, vcc_lo, v255   ; encoding: [0x6a,0x00,0x7d,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_class_f16 vcc_lo, v127, v255
-// W32: v_cmp_class_f16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_class_f16_e64 vcc_lo, v127, v255  ; encoding: [0x6a,0x00,0x7d,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_class_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x7d,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
+
+v_cmp_class_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_class_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7d,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
+
+v_cmp_class_f16 vcc_lo, v128, v2
+// W32: v_cmp_class_f16_e64 vcc_lo, v128, v2    ; encoding: [0x6a,0x00,0x7d,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_class_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_class_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
+
+v_cmp_class_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_class_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:17: error: invalid operand for instruction
 
 v_cmp_class_f16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_class_f16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_class_f16_e64 vcc_lo, vcc_hi, v255 ; encoding: [0x6a,0x00,0x7d,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_class_f16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_class_f16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_class_f16_e64 vcc_lo, vcc_lo, v255 ; encoding: [0x6a,0x00,0x7d,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_f16 vcc, v1, v255
-// W64: v_cmp_eq_f16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_eq_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x02,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x02,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_eq_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x02,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_eq_f16 vcc, v127, v255
-// W64: v_cmp_eq_f16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_eq_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x02,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x02,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_eq_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x02,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_f16 vcc, v128, v2
+// W64: v_cmp_eq_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x02,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x02,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_eq_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_eq_f16 vcc, vcc_hi, v255
-// W64: v_cmp_eq_f16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_eq_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x02,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_f16 vcc, vcc_lo, v255
-// W64: v_cmp_eq_f16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_eq_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x02,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_f16 vcc_lo, v1, v255
-// W32: v_cmp_eq_f16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_eq_f16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x02,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x02,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_eq_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x02,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_eq_f16 vcc_lo, v127, v255
-// W32: v_cmp_eq_f16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_eq_f16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x02,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x02,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_eq_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x02,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_f16 vcc_lo, v128, v2
+// W32: v_cmp_eq_f16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x02,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x02,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_eq_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_eq_f16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_eq_f16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_eq_f16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x02,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_f16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_eq_f16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_eq_f16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x02,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, v1, v255
-// W64: v_cmp_eq_i16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_eq_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x32,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x32,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_eq_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x32,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_eq_i16 vcc, v127, v255
-// W64: v_cmp_eq_i16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_eq_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x32,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x32,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_eq_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x32,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_i16 vcc, v128, v2
+// W64: v_cmp_eq_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x32,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x32,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_eq_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_eq_i16 vcc, vcc_hi, v255
-// W64: v_cmp_eq_i16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_eq_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x32,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, vcc_lo, v255
-// W64: v_cmp_eq_i16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_eq_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x32,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, v1, v255
-// W32: v_cmp_eq_i16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_eq_i16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x32,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_i16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x32,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_i16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_eq_i16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x32,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_eq_i16 vcc_lo, v127, v255
-// W32: v_cmp_eq_i16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_eq_i16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x32,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_i16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x32,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_i16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_eq_i16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x32,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_i16 vcc_lo, v128, v2
+// W32: v_cmp_eq_i16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x32,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_i16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_i16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x32,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_i16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_eq_i16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_eq_i16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_eq_i16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_eq_i16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x32,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_eq_i16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_eq_i16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x32,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, v1, v255
-// W64: v_cmp_eq_u16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_eq_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x3a,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_eq_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_eq_u16 vcc, v127, v255
-// W64: v_cmp_eq_u16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_eq_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x3a,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3a,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_eq_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3a,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_u16 vcc, v128, v2
+// W64: v_cmp_eq_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x3a,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3a,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_eq_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_eq_u16 vcc, vcc_hi, v255
-// W64: v_cmp_eq_u16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_eq_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x3a,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, vcc_lo, v255
-// W64: v_cmp_eq_u16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_eq_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x3a,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, v1, v255
-// W32: v_cmp_eq_u16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_eq_u16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x3a,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_u16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_u16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_eq_u16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_eq_u16 vcc_lo, v127, v255
-// W32: v_cmp_eq_u16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_eq_u16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x3a,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_u16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3a,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_u16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_eq_u16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3a,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_u16 vcc_lo, v128, v2
+// W32: v_cmp_eq_u16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x3a,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_eq_u16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_u16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3a,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_eq_u16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_eq_u16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_eq_u16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_eq_u16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_eq_u16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x3a,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_eq_u16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_eq_u16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x3a,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_f16 vcc, v1, v255
-// W64: v_cmp_ge_f16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ge_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x06,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x06,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_ge_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x06,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ge_f16 vcc, v127, v255
-// W64: v_cmp_ge_f16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ge_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x06,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x06,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_ge_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x06,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_f16 vcc, v128, v2
+// W64: v_cmp_ge_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x06,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x06,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ge_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ge_f16 vcc, vcc_hi, v255
-// W64: v_cmp_ge_f16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ge_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x06,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_f16 vcc, vcc_lo, v255
-// W64: v_cmp_ge_f16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ge_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x06,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_f16 vcc_lo, v1, v255
-// W32: v_cmp_ge_f16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ge_f16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x06,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x06,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_ge_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x06,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ge_f16 vcc_lo, v127, v255
-// W32: v_cmp_ge_f16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ge_f16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x06,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x06,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_ge_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x06,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_f16 vcc_lo, v128, v2
+// W32: v_cmp_ge_f16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x06,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x06,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ge_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ge_f16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_ge_f16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ge_f16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x06,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_f16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_ge_f16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ge_f16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x06,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, v1, v255
-// W64: v_cmp_ge_i16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ge_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x36,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x36,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_ge_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x36,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ge_i16 vcc, v127, v255
-// W64: v_cmp_ge_i16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ge_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x36,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x36,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_ge_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x36,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_i16 vcc, v128, v2
+// W64: v_cmp_ge_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x36,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x36,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ge_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ge_i16 vcc, vcc_hi, v255
-// W64: v_cmp_ge_i16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ge_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x36,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, vcc_lo, v255
-// W64: v_cmp_ge_i16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ge_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x36,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, v1, v255
-// W32: v_cmp_ge_i16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ge_i16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x36,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_i16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x36,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_i16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_ge_i16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x36,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ge_i16 vcc_lo, v127, v255
-// W32: v_cmp_ge_i16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ge_i16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x36,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_i16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x36,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_i16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_ge_i16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x36,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_i16 vcc_lo, v128, v2
+// W32: v_cmp_ge_i16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x36,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_i16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_i16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x36,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_i16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ge_i16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ge_i16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_ge_i16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ge_i16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x36,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_ge_i16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ge_i16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x36,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, v1, v255
-// W64: v_cmp_ge_u16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ge_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x3e,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_ge_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ge_u16 vcc, v127, v255
-// W64: v_cmp_ge_u16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ge_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x3e,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3e,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_ge_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3e,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_u16 vcc, v128, v2
+// W64: v_cmp_ge_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x3e,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3e,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ge_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ge_u16 vcc, vcc_hi, v255
-// W64: v_cmp_ge_u16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ge_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x3e,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, vcc_lo, v255
-// W64: v_cmp_ge_u16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ge_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x3e,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, v1, v255
-// W32: v_cmp_ge_u16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ge_u16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x3e,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_u16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_u16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_ge_u16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ge_u16 vcc_lo, v127, v255
-// W32: v_cmp_ge_u16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ge_u16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x3e,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_u16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3e,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_u16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_ge_u16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3e,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_u16 vcc_lo, v128, v2
+// W32: v_cmp_ge_u16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x3e,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ge_u16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_u16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3e,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ge_u16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ge_u16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ge_u16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_ge_u16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ge_u16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x3e,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_ge_u16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ge_u16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x3e,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_f16 vcc, v1, v255
-// W64: v_cmp_gt_f16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_gt_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x04,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x04,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_gt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x04,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_gt_f16 vcc, v127, v255
-// W64: v_cmp_gt_f16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_gt_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x04,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x04,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_gt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x04,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_f16 vcc, v128, v2
+// W64: v_cmp_gt_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x04,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x04,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_gt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_gt_f16 vcc, vcc_hi, v255
-// W64: v_cmp_gt_f16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_gt_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x04,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_f16 vcc, vcc_lo, v255
-// W64: v_cmp_gt_f16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_gt_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x04,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_f16 vcc_lo, v1, v255
-// W32: v_cmp_gt_f16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_gt_f16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x04,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x04,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_gt_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x04,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_gt_f16 vcc_lo, v127, v255
-// W32: v_cmp_gt_f16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_gt_f16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x04,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x04,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_gt_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x04,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_f16 vcc_lo, v128, v2
+// W32: v_cmp_gt_f16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x04,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x04,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_gt_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_gt_f16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_gt_f16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_gt_f16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x04,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_f16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_gt_f16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_gt_f16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x04,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, v1, v255
-// W64: v_cmp_gt_i16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_gt_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x34,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x34,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_gt_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x34,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_gt_i16 vcc, v127, v255
-// W64: v_cmp_gt_i16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_gt_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x34,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x34,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_gt_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x34,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_i16 vcc, v128, v2
+// W64: v_cmp_gt_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x34,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x34,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_gt_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_gt_i16 vcc, vcc_hi, v255
-// W64: v_cmp_gt_i16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_gt_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x34,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, vcc_lo, v255
-// W64: v_cmp_gt_i16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_gt_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x34,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, v1, v255
-// W32: v_cmp_gt_i16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_gt_i16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x34,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_i16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x34,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_i16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_gt_i16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x34,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_gt_i16 vcc_lo, v127, v255
-// W32: v_cmp_gt_i16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_gt_i16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x34,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_i16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x34,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_i16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_gt_i16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x34,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_i16 vcc_lo, v128, v2
+// W32: v_cmp_gt_i16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x34,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_i16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_i16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x34,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_i16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_gt_i16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_gt_i16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_gt_i16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_gt_i16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x34,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_gt_i16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_gt_i16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x34,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc, v1, v255
-// W64: v_cmp_gt_u16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_gt_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x3c,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_gt_u16 vcc, v127, v255
-// W64: v_cmp_gt_u16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_gt_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
-v_cmp_gt_u16 vcc, vcc_hi, v255
-// W64: v_cmp_gt_u16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_gt_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_gt_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
-v_cmp_gt_u16 vcc, vcc_lo, v255
-// W64: v_cmp_gt_u16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_gt_u16 vcc, v127, v255
+// W64: v_cmp_gt_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x3c,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_gt_u16 vcc_lo, v1, v255
-// W32: v_cmp_gt_u16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_gt_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3c,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_gt_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3c,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_u16 vcc, v128, v2
+// W64: v_cmp_gt_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x3c,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3c,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_gt_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_u16 vcc, vcc_hi, v255
+// W64: v_cmp_gt_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x3c,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc, vcc_lo, v255
+// W64: v_cmp_gt_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x3c,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v255
+// W32: v_cmp_gt_u16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x3c,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_u16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_u16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_gt_u16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_gt_u16 vcc_lo, v127, v255
-// W32: v_cmp_gt_u16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_gt_u16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x3c,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_u16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3c,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_u16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_gt_u16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3c,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_u16 vcc_lo, v128, v2
+// W32: v_cmp_gt_u16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x3c,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_gt_u16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_u16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3c,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_gt_u16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_gt_u16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_gt_u16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_gt_u16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_gt_u16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x3c,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_gt_u16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_gt_u16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x3c,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_f16 vcc, v1, v255
-// W64: v_cmp_le_f16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_le_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x03,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x03,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_le_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x03,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_le_f16 vcc, v127, v255
-// W64: v_cmp_le_f16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_le_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x03,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x03,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_le_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x03,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_f16 vcc, v128, v2
+// W64: v_cmp_le_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x03,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x03,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_le_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_le_f16 vcc, vcc_hi, v255
-// W64: v_cmp_le_f16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_le_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x03,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_f16 vcc, vcc_lo, v255
-// W64: v_cmp_le_f16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_le_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x03,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_f16 vcc_lo, v1, v255
-// W32: v_cmp_le_f16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_le_f16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x03,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x03,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_le_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x03,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_le_f16 vcc_lo, v127, v255
-// W32: v_cmp_le_f16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_le_f16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x03,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x03,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_le_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x03,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_f16 vcc_lo, v128, v2
+// W32: v_cmp_le_f16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x03,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x03,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_le_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_le_f16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_le_f16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_le_f16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x03,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_f16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_le_f16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_le_f16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x03,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, v1, v255
-// W64: v_cmp_le_i16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_le_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x33,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x33,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_le_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x33,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_le_i16 vcc, v127, v255
-// W64: v_cmp_le_i16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_le_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x33,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x33,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_le_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x33,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_i16 vcc, v128, v2
+// W64: v_cmp_le_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x33,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x33,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_le_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_le_i16 vcc, vcc_hi, v255
-// W64: v_cmp_le_i16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_le_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x33,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, vcc_lo, v255
-// W64: v_cmp_le_i16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_le_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x33,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, v1, v255
-// W32: v_cmp_le_i16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_le_i16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x33,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_i16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x33,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_i16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_le_i16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x33,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_le_i16 vcc_lo, v127, v255
-// W32: v_cmp_le_i16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_le_i16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x33,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_i16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x33,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_i16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_le_i16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x33,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_i16 vcc_lo, v128, v2
+// W32: v_cmp_le_i16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x33,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_i16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_i16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x33,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_i16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_le_i16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_le_i16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_le_i16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_le_i16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x33,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_le_i16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_le_i16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x33,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, v1, v255
-// W64: v_cmp_le_u16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_le_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x3b,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_le_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_le_u16 vcc, v127, v255
-// W64: v_cmp_le_u16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_le_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x3b,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3b,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_le_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3b,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_u16 vcc, v128, v2
+// W64: v_cmp_le_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x3b,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3b,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_le_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_le_u16 vcc, vcc_hi, v255
-// W64: v_cmp_le_u16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_le_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x3b,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, vcc_lo, v255
-// W64: v_cmp_le_u16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_le_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x3b,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, v1, v255
-// W32: v_cmp_le_u16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_le_u16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x3b,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_u16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_u16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_le_u16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_le_u16 vcc_lo, v127, v255
-// W32: v_cmp_le_u16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_le_u16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x3b,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_u16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3b,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_u16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_le_u16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3b,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_u16 vcc_lo, v128, v2
+// W32: v_cmp_le_u16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x3b,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_le_u16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_u16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3b,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_le_u16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_le_u16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_le_u16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_le_u16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_le_u16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x3b,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_le_u16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_le_u16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x3b,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lg_f16 vcc, v1, v255
-// W64: v_cmp_lg_f16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lg_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x05,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lg_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x05,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lg_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_lg_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x05,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lg_f16 vcc, v127, v255
-// W64: v_cmp_lg_f16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lg_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x05,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lg_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x05,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lg_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_lg_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x05,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lg_f16 vcc, v128, v2
+// W64: v_cmp_lg_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x05,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lg_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x05,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lg_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_lg_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lg_f16 vcc, vcc_hi, v255
-// W64: v_cmp_lg_f16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lg_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x05,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lg_f16 vcc, vcc_lo, v255
-// W64: v_cmp_lg_f16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lg_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x05,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lg_f16 vcc_lo, v1, v255
-// W32: v_cmp_lg_f16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lg_f16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x05,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lg_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x05,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lg_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_lg_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x05,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lg_f16 vcc_lo, v127, v255
-// W32: v_cmp_lg_f16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lg_f16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x05,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lg_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x05,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lg_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_lg_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x05,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lg_f16 vcc_lo, v128, v2
+// W32: v_cmp_lg_f16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x05,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lg_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lg_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x05,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lg_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_lg_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lg_f16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_lg_f16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lg_f16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x05,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lg_f16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_lg_f16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lg_f16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x05,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_f16 vcc, v1, v255
-// W64: v_cmp_lt_f16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lt_f16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x01,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x01,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_lt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x01,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lt_f16 vcc, v127, v255
-// W64: v_cmp_lt_f16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lt_f16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x01,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x01,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_lt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x01,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_f16 vcc, v128, v2
+// W64: v_cmp_lt_f16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x01,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x01,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_lt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lt_f16 vcc, vcc_hi, v255
-// W64: v_cmp_lt_f16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lt_f16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x01,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_f16 vcc, vcc_lo, v255
-// W64: v_cmp_lt_f16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lt_f16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x01,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_f16 vcc_lo, v1, v255
-// W32: v_cmp_lt_f16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lt_f16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x01,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x01,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_lt_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x01,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lt_f16 vcc_lo, v127, v255
-// W32: v_cmp_lt_f16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lt_f16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x01,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x01,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_lt_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x01,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_f16 vcc_lo, v128, v2
+// W32: v_cmp_lt_f16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x01,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x01,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_lt_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lt_f16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_lt_f16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lt_f16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x01,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_f16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_lt_f16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lt_f16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x01,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, v1, v255
-// W64: v_cmp_lt_i16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lt_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x31,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x31,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_lt_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x31,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lt_i16 vcc, v127, v255
-// W64: v_cmp_lt_i16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lt_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x31,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x31,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_lt_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x31,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_i16 vcc, v128, v2
+// W64: v_cmp_lt_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x31,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x31,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_lt_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lt_i16 vcc, vcc_hi, v255
-// W64: v_cmp_lt_i16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lt_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x31,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, vcc_lo, v255
-// W64: v_cmp_lt_i16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lt_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x31,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, v1, v255
-// W32: v_cmp_lt_i16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lt_i16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x31,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_i16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x31,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_i16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_lt_i16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x31,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lt_i16 vcc_lo, v127, v255
-// W32: v_cmp_lt_i16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lt_i16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x31,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_i16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x31,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_i16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_lt_i16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x31,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_i16 vcc_lo, v128, v2
+// W32: v_cmp_lt_i16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x31,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_i16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_i16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x31,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_i16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_lt_i16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lt_i16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_lt_i16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lt_i16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x31,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_lt_i16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lt_i16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x31,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, v1, v255
-// W64: v_cmp_lt_u16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lt_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x39,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x39,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_lt_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x39,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lt_u16 vcc, v127, v255
-// W64: v_cmp_lt_u16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lt_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x39,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x39,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_lt_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x39,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_u16 vcc, v128, v2
+// W64: v_cmp_lt_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x39,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x39,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_lt_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lt_u16 vcc, vcc_hi, v255
-// W64: v_cmp_lt_u16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lt_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x39,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, vcc_lo, v255
-// W64: v_cmp_lt_u16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_lt_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x39,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, v1, v255
-// W32: v_cmp_lt_u16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lt_u16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x39,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_u16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x39,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_u16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_lt_u16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x39,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lt_u16 vcc_lo, v127, v255
-// W32: v_cmp_lt_u16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lt_u16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x39,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_u16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x39,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_u16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_lt_u16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x39,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_u16 vcc_lo, v128, v2
+// W32: v_cmp_lt_u16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x39,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_lt_u16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_u16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x39,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_lt_u16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_lt_u16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_lt_u16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_lt_u16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lt_u16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x39,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_lt_u16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_lt_u16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x39,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, v1, v255
-// W64: v_cmp_ne_i16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ne_i16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x35,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x35,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_ne_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x35,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ne_i16 vcc, v127, v255
-// W64: v_cmp_ne_i16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ne_i16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x35,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x35,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_ne_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x35,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_i16 vcc, v128, v2
+// W64: v_cmp_ne_i16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x35,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x35,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ne_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ne_i16 vcc, vcc_hi, v255
-// W64: v_cmp_ne_i16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ne_i16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x35,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, vcc_lo, v255
-// W64: v_cmp_ne_i16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ne_i16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x35,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, v1, v255
-// W32: v_cmp_ne_i16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ne_i16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x35,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_i16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x35,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_i16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_ne_i16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x35,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ne_i16 vcc_lo, v127, v255
-// W32: v_cmp_ne_i16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ne_i16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x35,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_i16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x35,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_i16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_ne_i16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x35,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_i16 vcc_lo, v128, v2
+// W32: v_cmp_ne_i16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x35,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_i16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_i16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x35,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_i16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ne_i16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ne_i16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_ne_i16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ne_i16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x35,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_ne_i16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ne_i16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x35,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, v1, v255
-// W64: v_cmp_ne_u16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ne_u16_e64 vcc, v1, v255          ; encoding: [0x6a,0x00,0x3d,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_ne_u16 vcc, v127, v255
-// W64: v_cmp_ne_u16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_ne_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
-v_cmp_ne_u16 vcc, vcc_hi, v255
-// W64: v_cmp_ne_u16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_ne_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_ne_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_u16 vcc, v127, v255
+// W64: v_cmp_ne_u16_e64 vcc, v127, v255        ; encoding: [0x6a,0x00,0x3d,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3d,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_ne_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3d,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_u16 vcc, v128, v2
+// W64: v_cmp_ne_u16_e64 vcc, v128, v2          ; encoding: [0x6a,0x00,0x3d,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3d,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ne_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_u16 vcc, vcc_hi, v255
+// W64: v_cmp_ne_u16_e64 vcc, vcc_hi, v255      ; encoding: [0x6a,0x00,0x3d,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, vcc_lo, v255
-// W64: v_cmp_ne_u16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ne_u16_e64 vcc, vcc_lo, v255      ; encoding: [0x6a,0x00,0x3d,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, v1, v255
-// W32: v_cmp_ne_u16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ne_u16_e64 vcc_lo, v1, v255       ; encoding: [0x6a,0x00,0x3d,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_u16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_u16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_ne_u16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ne_u16 vcc_lo, v127, v255
-// W32: v_cmp_ne_u16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ne_u16_e64 vcc_lo, v127, v255     ; encoding: [0x6a,0x00,0x3d,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_u16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3d,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_u16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_ne_u16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3d,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_u16 vcc_lo, v128, v2
+// W32: v_cmp_ne_u16_e64 vcc_lo, v128, v2       ; encoding: [0x6a,0x00,0x3d,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ne_u16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_u16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3d,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
+
+v_cmp_ne_u16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ne_u16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:14: error: invalid operand for instruction
 
 v_cmp_ne_u16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_ne_u16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ne_u16_e64 vcc_lo, vcc_hi, v255   ; encoding: [0x6a,0x00,0x3d,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_ne_u16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ne_u16_e64 vcc_lo, vcc_lo, v255   ; encoding: [0x6a,0x00,0x3d,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_neq_f16 vcc, v1, v255
-// W64: v_cmp_neq_f16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_neq_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x0d,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_neq_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_neq_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_neq_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_neq_f16 vcc, v127, v255
-// W64: v_cmp_neq_f16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_neq_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x0d,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_neq_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0d,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_neq_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_neq_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0d,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_neq_f16 vcc, v128, v2
+// W64: v_cmp_neq_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x0d,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_neq_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0d,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_neq_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_neq_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_neq_f16 vcc, vcc_hi, v255
-// W64: v_cmp_neq_f16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_neq_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x0d,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_neq_f16 vcc, vcc_lo, v255
-// W64: v_cmp_neq_f16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_neq_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x0d,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_neq_f16 vcc_lo, v1, v255
-// W32: v_cmp_neq_f16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_neq_f16_e64 vcc_lo, v1, v255      ; encoding: [0x6a,0x00,0x0d,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_neq_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_neq_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_neq_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_neq_f16 vcc_lo, v127, v255
-// W32: v_cmp_neq_f16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_neq_f16_e64 vcc_lo, v127, v255    ; encoding: [0x6a,0x00,0x0d,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_neq_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0d,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_neq_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_neq_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0d,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_neq_f16 vcc_lo, v128, v2
+// W32: v_cmp_neq_f16_e64 vcc_lo, v128, v2      ; encoding: [0x6a,0x00,0x0d,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_neq_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_neq_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0d,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_neq_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_neq_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_neq_f16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_neq_f16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_neq_f16_e64 vcc_lo, vcc_hi, v255  ; encoding: [0x6a,0x00,0x0d,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_neq_f16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_neq_f16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_neq_f16_e64 vcc_lo, vcc_lo, v255  ; encoding: [0x6a,0x00,0x0d,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nge_f16 vcc, v1, v255
-// W64: v_cmp_nge_f16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nge_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x09,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nge_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x09,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nge_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_nge_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x09,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nge_f16 vcc, v127, v255
-// W64: v_cmp_nge_f16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nge_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x09,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nge_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x09,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nge_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_nge_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x09,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nge_f16 vcc, v128, v2
+// W64: v_cmp_nge_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x09,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nge_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x09,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nge_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_nge_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nge_f16 vcc, vcc_hi, v255
-// W64: v_cmp_nge_f16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nge_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x09,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nge_f16 vcc, vcc_lo, v255
-// W64: v_cmp_nge_f16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nge_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x09,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nge_f16 vcc_lo, v1, v255
-// W32: v_cmp_nge_f16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nge_f16_e64 vcc_lo, v1, v255      ; encoding: [0x6a,0x00,0x09,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nge_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x09,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nge_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_nge_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x09,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nge_f16 vcc_lo, v127, v255
-// W32: v_cmp_nge_f16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nge_f16_e64 vcc_lo, v127, v255    ; encoding: [0x6a,0x00,0x09,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nge_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x09,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nge_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_nge_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x09,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nge_f16 vcc_lo, v128, v2
+// W32: v_cmp_nge_f16_e64 vcc_lo, v128, v2      ; encoding: [0x6a,0x00,0x09,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nge_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nge_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x09,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nge_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_nge_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nge_f16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_nge_f16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nge_f16_e64 vcc_lo, vcc_hi, v255  ; encoding: [0x6a,0x00,0x09,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nge_f16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_nge_f16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nge_f16_e64 vcc_lo, vcc_lo, v255  ; encoding: [0x6a,0x00,0x09,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ngt_f16 vcc, v1, v255
-// W64: v_cmp_ngt_f16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ngt_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x0b,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ngt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_ngt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_ngt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_ngt_f16 vcc, v127, v255
-// W64: v_cmp_ngt_f16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ngt_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x0b,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ngt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0b,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_ngt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_ngt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0b,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_ngt_f16 vcc, v128, v2
+// W64: v_cmp_ngt_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x0b,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ngt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0b,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_ngt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ngt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_ngt_f16 vcc, vcc_hi, v255
-// W64: v_cmp_ngt_f16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ngt_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x0b,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ngt_f16 vcc, vcc_lo, v255
-// W64: v_cmp_ngt_f16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_ngt_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x0b,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ngt_f16 vcc_lo, v1, v255
-// W32: v_cmp_ngt_f16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ngt_f16_e64 vcc_lo, v1, v255      ; encoding: [0x6a,0x00,0x0b,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ngt_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_ngt_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_ngt_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_ngt_f16 vcc_lo, v127, v255
-// W32: v_cmp_ngt_f16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ngt_f16_e64 vcc_lo, v127, v255    ; encoding: [0x6a,0x00,0x0b,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ngt_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0b,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_ngt_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_ngt_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0b,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_ngt_f16 vcc_lo, v128, v2
+// W32: v_cmp_ngt_f16_e64 vcc_lo, v128, v2      ; encoding: [0x6a,0x00,0x0b,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_ngt_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ngt_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0b,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_ngt_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ngt_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_ngt_f16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_ngt_f16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ngt_f16_e64 vcc_lo, vcc_hi, v255  ; encoding: [0x6a,0x00,0x0b,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_ngt_f16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_ngt_f16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_ngt_f16_e64 vcc_lo, vcc_lo, v255  ; encoding: [0x6a,0x00,0x0b,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nle_f16 vcc, v1, v255
-// W64: v_cmp_nle_f16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nle_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x0c,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nle_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nle_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_nle_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nle_f16 vcc, v127, v255
-// W64: v_cmp_nle_f16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nle_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x0c,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nle_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0c,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nle_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_nle_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0c,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nle_f16 vcc, v128, v2
+// W64: v_cmp_nle_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x0c,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nle_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0c,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nle_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_nle_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nle_f16 vcc, vcc_hi, v255
-// W64: v_cmp_nle_f16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nle_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x0c,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nle_f16 vcc, vcc_lo, v255
-// W64: v_cmp_nle_f16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nle_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x0c,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nle_f16 vcc_lo, v1, v255
-// W32: v_cmp_nle_f16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nle_f16_e64 vcc_lo, v1, v255      ; encoding: [0x6a,0x00,0x0c,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nle_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nle_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_nle_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nle_f16 vcc_lo, v127, v255
-// W32: v_cmp_nle_f16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nle_f16_e64 vcc_lo, v127, v255    ; encoding: [0x6a,0x00,0x0c,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nle_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0c,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nle_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_nle_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0c,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nle_f16 vcc_lo, v128, v2
+// W32: v_cmp_nle_f16_e64 vcc_lo, v128, v2      ; encoding: [0x6a,0x00,0x0c,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nle_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nle_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0c,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nle_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_nle_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nle_f16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_nle_f16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nle_f16_e64 vcc_lo, vcc_hi, v255  ; encoding: [0x6a,0x00,0x0c,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nle_f16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_nle_f16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nle_f16_e64 vcc_lo, vcc_lo, v255  ; encoding: [0x6a,0x00,0x0c,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlg_f16 vcc, v1, v255
-// W64: v_cmp_nlg_f16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nlg_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x0a,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlg_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlg_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_nlg_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nlg_f16 vcc, v127, v255
-// W64: v_cmp_nlg_f16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nlg_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x0a,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlg_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0a,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlg_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_nlg_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0a,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlg_f16 vcc, v128, v2
+// W64: v_cmp_nlg_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x0a,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlg_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0a,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlg_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_nlg_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nlg_f16 vcc, vcc_hi, v255
-// W64: v_cmp_nlg_f16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nlg_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x0a,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlg_f16 vcc, vcc_lo, v255
-// W64: v_cmp_nlg_f16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nlg_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x0a,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlg_f16 vcc_lo, v1, v255
-// W32: v_cmp_nlg_f16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nlg_f16_e64 vcc_lo, v1, v255      ; encoding: [0x6a,0x00,0x0a,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlg_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlg_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_nlg_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nlg_f16 vcc_lo, v127, v255
-// W32: v_cmp_nlg_f16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nlg_f16_e64 vcc_lo, v127, v255    ; encoding: [0x6a,0x00,0x0a,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlg_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0a,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlg_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_nlg_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0a,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlg_f16 vcc_lo, v128, v2
+// W32: v_cmp_nlg_f16_e64 vcc_lo, v128, v2      ; encoding: [0x6a,0x00,0x0a,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlg_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlg_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0a,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlg_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_nlg_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nlg_f16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_nlg_f16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nlg_f16_e64 vcc_lo, vcc_hi, v255  ; encoding: [0x6a,0x00,0x0a,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlg_f16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_nlg_f16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nlg_f16_e64 vcc_lo, vcc_lo, v255  ; encoding: [0x6a,0x00,0x0a,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlt_f16 vcc, v1, v255
-// W64: v_cmp_nlt_f16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nlt_f16_e64 vcc, v1, v255         ; encoding: [0x6a,0x00,0x0e,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_nlt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nlt_f16 vcc, v127, v255
-// W64: v_cmp_nlt_f16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nlt_f16_e64 vcc, v127, v255       ; encoding: [0x6a,0x00,0x0e,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0e,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_nlt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0e,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlt_f16 vcc, v128, v2
+// W64: v_cmp_nlt_f16_e64 vcc, v128, v2         ; encoding: [0x6a,0x00,0x0e,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0e,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_nlt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nlt_f16 vcc, vcc_hi, v255
-// W64: v_cmp_nlt_f16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nlt_f16_e64 vcc, vcc_hi, v255     ; encoding: [0x6a,0x00,0x0e,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlt_f16 vcc, vcc_lo, v255
-// W64: v_cmp_nlt_f16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_nlt_f16_e64 vcc, vcc_lo, v255     ; encoding: [0x6a,0x00,0x0e,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlt_f16 vcc_lo, v1, v255
-// W32: v_cmp_nlt_f16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nlt_f16_e64 vcc_lo, v1, v255      ; encoding: [0x6a,0x00,0x0e,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlt_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlt_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_nlt_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nlt_f16 vcc_lo, v127, v255
-// W32: v_cmp_nlt_f16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nlt_f16_e64 vcc_lo, v127, v255    ; encoding: [0x6a,0x00,0x0e,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlt_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0e,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlt_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_nlt_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0e,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlt_f16 vcc_lo, v128, v2
+// W32: v_cmp_nlt_f16_e64 vcc_lo, v128, v2      ; encoding: [0x6a,0x00,0x0e,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_nlt_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlt_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0e,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
+
+v_cmp_nlt_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_nlt_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:15: error: invalid operand for instruction
 
 v_cmp_nlt_f16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_nlt_f16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nlt_f16_e64 vcc_lo, vcc_hi, v255  ; encoding: [0x6a,0x00,0x0e,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_nlt_f16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_nlt_f16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_nlt_f16_e64 vcc_lo, vcc_lo, v255  ; encoding: [0x6a,0x00,0x0e,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_o_f16 vcc, v1, v255
-// W64: v_cmp_o_f16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_o_f16_e64 vcc, v1, v255           ; encoding: [0x6a,0x00,0x07,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_o_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x07,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
+
+v_cmp_o_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_o_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x07,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
 
 v_cmp_o_f16 vcc, v127, v255
-// W64: v_cmp_o_f16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_o_f16_e64 vcc, v127, v255         ; encoding: [0x6a,0x00,0x07,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_o_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x07,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
+
+v_cmp_o_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_o_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x07,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
+
+v_cmp_o_f16 vcc, v128, v2
+// W64: v_cmp_o_f16_e64 vcc, v128, v2           ; encoding: [0x6a,0x00,0x07,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_o_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x07,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
+
+v_cmp_o_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_o_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
 
 v_cmp_o_f16 vcc, vcc_hi, v255
-// W64: v_cmp_o_f16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_o_f16_e64 vcc, vcc_hi, v255       ; encoding: [0x6a,0x00,0x07,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_o_f16 vcc, vcc_lo, v255
-// W64: v_cmp_o_f16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_o_f16_e64 vcc, vcc_lo, v255       ; encoding: [0x6a,0x00,0x07,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_o_f16 vcc_lo, v1, v255
-// W32: v_cmp_o_f16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_o_f16_e64 vcc_lo, v1, v255        ; encoding: [0x6a,0x00,0x07,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_o_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x07,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
+
+v_cmp_o_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_o_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x07,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
 
 v_cmp_o_f16 vcc_lo, v127, v255
-// W32: v_cmp_o_f16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_o_f16_e64 vcc_lo, v127, v255      ; encoding: [0x6a,0x00,0x07,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_o_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x07,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
+
+v_cmp_o_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_o_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x07,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
+
+v_cmp_o_f16 vcc_lo, v128, v2
+// W32: v_cmp_o_f16_e64 vcc_lo, v128, v2        ; encoding: [0x6a,0x00,0x07,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_o_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_o_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x07,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
+
+v_cmp_o_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_o_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
 
 v_cmp_o_f16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_o_f16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_o_f16_e64 vcc_lo, vcc_hi, v255    ; encoding: [0x6a,0x00,0x07,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_o_f16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_o_f16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_o_f16_e64 vcc_lo, vcc_lo, v255    ; encoding: [0x6a,0x00,0x07,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_u_f16 vcc, v1, v255
-// W64: v_cmp_u_f16_e64 vcc, v1, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_u_f16_e64 vcc, v1, v255           ; encoding: [0x6a,0x00,0x08,0xd4,0x01,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_u_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x08,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
+
+v_cmp_u_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_u_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x08,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
 
 v_cmp_u_f16 vcc, v127, v255
-// W64: v_cmp_u_f16_e64 vcc, v127, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_u_f16_e64 vcc, v127, v255         ; encoding: [0x6a,0x00,0x08,0xd4,0x7f,0xff,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_u_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x08,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
+
+v_cmp_u_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
+// W64: v_cmp_u_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x08,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
+
+v_cmp_u_f16 vcc, v128, v2
+// W64: v_cmp_u_f16_e64 vcc, v128, v2           ; encoding: [0x6a,0x00,0x08,0xd4,0x80,0x05,0x02,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
+
+v_cmp_u_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_u_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x08,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
+
+v_cmp_u_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_u_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
 
 v_cmp_u_f16 vcc, vcc_hi, v255
-// W64: v_cmp_u_f16_e64 vcc, vcc_hi, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_u_f16_e64 vcc, vcc_hi, v255       ; encoding: [0x6a,0x00,0x08,0xd4,0x6b,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_u_f16 vcc, vcc_lo, v255
-// W64: v_cmp_u_f16_e64 vcc, vcc_lo, v255
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W64: v_cmp_u_f16_e64 vcc, vcc_lo, v255       ; encoding: [0x6a,0x00,0x08,0xd4,0x6a,0xfe,0x03,0x00]
+// W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
 v_cmp_u_f16 vcc_lo, v1, v255
-// W32: v_cmp_u_f16_e64 vcc_lo, v1, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// W32: v_cmp_u_f16_e64 vcc_lo, v1, v255        ; encoding: [0x6a,0x00,0x08,0xd4,0x01,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_u_f16 vcc_lo, v127, v255
-// W32: v_cmp_u_f16_e64 vcc_lo, v127, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_u_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x08,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
 
-v_cmp_u_f16 vcc_lo, vcc_hi, v255
-// W32: v_cmp_u_f16_e64 vcc_lo, vcc_hi, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_u_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x08,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
 
-v_cmp_u_f16 vcc_lo, vcc_lo, v255
-// W32: v_cmp_u_f16_e64 vcc_lo, vcc_lo, v255
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16 vcc_lo, v127, v255
+// W32: v_cmp_u_f16_e64 vcc_lo, v127, v255      ; encoding: [0x6a,0x00,0x08,0xd4,0x7f,0xff,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_class_f16 vcc, v128, v2
-// W64: v_cmp_class_f16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_u_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x08,0xd4,0xe9,0xfe,0x03,0x00,0x7f,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
 
-v_cmp_class_f16 vcc_lo, v128, v2
-// W32: v_cmp_class_f16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
+// W32: v_cmp_u_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x08,0xd4,0xfa,0xfe,0x03,0x00,0x7f,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
 
-v_cmp_eq_f16 vcc, v128, v2
-// W64: v_cmp_eq_f16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16 vcc_lo, v128, v2
+// W32: v_cmp_u_f16_e64 vcc_lo, v128, v2        ; encoding: [0x6a,0x00,0x08,0xd4,0x80,0x05,0x02,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_eq_f16 vcc_lo, v128, v2
-// W32: v_cmp_eq_f16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_u_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x08,0xd4,0xe9,0x04,0x02,0x00,0x80,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
 
-v_cmp_eq_i16 vcc, v128, v2
-// W64: v_cmp_eq_i16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_u_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x80,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:13: error: invalid operand for instruction
 
-v_cmp_eq_i16 vcc_lo, v128, v2
-// W32: v_cmp_eq_i16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmp_u_f16 vcc_lo, vcc_hi, v255
+// W32: v_cmp_u_f16_e64 vcc_lo, vcc_hi, v255    ; encoding: [0x6a,0x00,0x08,0xd4,0x6b,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_cmp_eq_u16 vcc, v128, v2
-// W64: v_cmp_eq_u16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_eq_u16 vcc_lo, v128, v2
-// W32: v_cmp_eq_u16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_f16 vcc, v128, v2
-// W64: v_cmp_ge_f16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_f16 vcc_lo, v128, v2
-// W32: v_cmp_ge_f16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_i16 vcc, v128, v2
-// W64: v_cmp_ge_i16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_i16 vcc_lo, v128, v2
-// W32: v_cmp_ge_i16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_u16 vcc, v128, v2
-// W64: v_cmp_ge_u16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ge_u16 vcc_lo, v128, v2
-// W32: v_cmp_ge_u16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_f16 vcc, v128, v2
-// W64: v_cmp_gt_f16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_f16 vcc_lo, v128, v2
-// W32: v_cmp_gt_f16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_i16 vcc, v128, v2
-// W64: v_cmp_gt_i16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_i16 vcc_lo, v128, v2
-// W32: v_cmp_gt_i16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_u16 vcc, v128, v2
-// W64: v_cmp_gt_u16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_gt_u16 vcc_lo, v128, v2
-// W32: v_cmp_gt_u16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_f16 vcc, v128, v2
-// W64: v_cmp_le_f16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_f16 vcc_lo, v128, v2
-// W32: v_cmp_le_f16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_i16 vcc, v128, v2
-// W64: v_cmp_le_i16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_i16 vcc_lo, v128, v2
-// W32: v_cmp_le_i16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_u16 vcc, v128, v2
-// W64: v_cmp_le_u16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_le_u16 vcc_lo, v128, v2
-// W32: v_cmp_le_u16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lg_f16 vcc, v128, v2
-// W64: v_cmp_lg_f16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lg_f16 vcc_lo, v128, v2
-// W32: v_cmp_lg_f16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_f16 vcc, v128, v2
-// W64: v_cmp_lt_f16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_f16 vcc_lo, v128, v2
-// W32: v_cmp_lt_f16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_i16 vcc, v128, v2
-// W64: v_cmp_lt_i16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_i16 vcc_lo, v128, v2
-// W32: v_cmp_lt_i16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_u16 vcc, v128, v2
-// W64: v_cmp_lt_u16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_lt_u16 vcc_lo, v128, v2
-// W32: v_cmp_lt_u16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ne_i16 vcc, v128, v2
-// W64: v_cmp_ne_i16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ne_i16 vcc_lo, v128, v2
-// W32: v_cmp_ne_i16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ne_u16 vcc, v128, v2
-// W64: v_cmp_ne_u16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ne_u16 vcc_lo, v128, v2
-// W32: v_cmp_ne_u16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_neq_f16 vcc, v128, v2
-// W64: v_cmp_neq_f16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_neq_f16 vcc_lo, v128, v2
-// W32: v_cmp_neq_f16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nge_f16 vcc, v128, v2
-// W64: v_cmp_nge_f16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nge_f16 vcc_lo, v128, v2
-// W32: v_cmp_nge_f16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ngt_f16 vcc, v128, v2
-// W64: v_cmp_ngt_f16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_ngt_f16 vcc_lo, v128, v2
-// W32: v_cmp_ngt_f16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nle_f16 vcc, v128, v2
-// W64: v_cmp_nle_f16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nle_f16 vcc_lo, v128, v2
-// W32: v_cmp_nle_f16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nlg_f16 vcc, v128, v2
-// W64: v_cmp_nlg_f16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nlg_f16 vcc_lo, v128, v2
-// W32: v_cmp_nlg_f16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nlt_f16 vcc, v128, v2
-// W64: v_cmp_nlt_f16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_nlt_f16 vcc_lo, v128, v2
-// W32: v_cmp_nlt_f16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_o_f16 vcc, v128, v2
-// W64: v_cmp_o_f16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_o_f16 vcc_lo, v128, v2
-// W32: v_cmp_o_f16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_u_f16 vcc, v128, v2
-// W64: v_cmp_u_f16_e64 vcc, v128, v2
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_u_f16 vcc_lo, v128, v2
-// W32: v_cmp_u_f16_e64 vcc_lo, v128, v2
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmp_class_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_class_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_class_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_class_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_eq_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_eq_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_eq_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_eq_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_eq_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_eq_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_eq_i16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_eq_i16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_eq_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_eq_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_eq_u16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_eq_u16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_ge_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_ge_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_ge_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_ge_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_ge_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_ge_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_ge_i16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_ge_i16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_ge_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_ge_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_ge_u16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_ge_u16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_gt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_gt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_gt_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_gt_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_gt_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_gt_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_gt_i16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_gt_i16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_gt_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_gt_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_gt_u16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_gt_u16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_le_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_le_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_le_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_le_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_le_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_le_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_le_i16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_le_i16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_le_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_le_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_le_u16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_le_u16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_lg_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_lg_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_lg_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_lg_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_lt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_lt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_lt_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_lt_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_lt_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_lt_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_lt_i16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_lt_i16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_lt_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_lt_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_lt_u16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_lt_u16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_ne_i16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_ne_i16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_ne_i16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_ne_i16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_ne_u16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_ne_u16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_ne_u16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_ne_u16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_neq_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_neq_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_neq_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_neq_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_nge_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_nge_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_nge_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_nge_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_ngt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_ngt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_ngt_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_ngt_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_nle_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_nle_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_nle_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_nle_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_nlg_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_nlg_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_nlg_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_nlg_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_nlt_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_nlt_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_nlt_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_nlt_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_o_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_o_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_o_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_o_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16 vcc, v1, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_u_f16_e64_dpp vcc, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16 vcc, v127, v255 quad_perm:[3,2,1,0]
-// W64: v_cmp_u_f16_e64_dpp vcc, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16 vcc_lo, v1, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_u_f16_e64_dpp vcc_lo, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16 vcc_lo, v127, v255 quad_perm:[3,2,1,0]
-// W32: v_cmp_u_f16_e64_dpp vcc_lo, v127, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_class_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_class_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_eq_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_eq_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_eq_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_eq_i16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_eq_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_eq_u16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_ge_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_ge_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_ge_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_ge_i16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_ge_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_ge_u16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_gt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_gt_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_gt_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_gt_i16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_gt_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_gt_u16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_le_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_le_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_le_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_le_i16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_le_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_le_u16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_lg_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_lg_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_lt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_lt_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_lt_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_lt_i16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_lt_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_lt_u16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_ne_i16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_ne_i16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_ne_u16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_ne_u16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_neq_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_neq_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_nge_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_nge_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_ngt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_ngt_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_nle_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_nle_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_nlg_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_nlg_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_nlt_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_nlt_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_o_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_o_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16 vcc, v128, v2 quad_perm:[3,2,1,0]
-// W64: v_cmp_u_f16_e64_dpp vcc, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16 vcc_lo, v128, v2 quad_perm:[3,2,1,0]
-// W32: v_cmp_u_f16_e64_dpp vcc_lo, v128, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_class_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_class_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_class_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_eq_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_eq_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_eq_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_eq_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_eq_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_eq_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_eq_i16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_eq_i16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_eq_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_eq_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_eq_u16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_eq_u16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ge_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ge_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ge_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ge_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ge_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ge_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ge_i16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ge_i16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ge_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ge_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ge_u16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ge_u16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_gt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_gt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_gt_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_gt_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_gt_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_gt_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_gt_i16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_gt_i16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_gt_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_gt_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_gt_u16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_gt_u16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_le_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_le_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_le_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_le_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_le_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_le_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_le_i16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_le_i16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_le_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_le_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_le_u16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_le_u16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_lg_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_lg_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_lg_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_lg_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_lt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_lt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_lt_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_lt_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_lt_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_lt_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_lt_i16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_lt_i16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_lt_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_lt_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_lt_u16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_lt_u16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ne_i16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ne_i16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ne_i16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ne_i16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ne_u16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ne_u16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ne_u16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ne_u16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_neq_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_neq_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_neq_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_neq_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_nge_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_nge_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_nge_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_nge_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ngt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ngt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ngt_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ngt_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_nle_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_nle_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_nle_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_nle_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_nlg_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_nlg_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_nlg_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_nlg_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_nlt_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_nlt_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_nlt_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_nlt_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_o_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_o_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_o_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_o_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16 vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_u_f16_e64_dpp vcc, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16 vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_u_f16_e64_dpp vcc, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16 vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_u_f16_e64_dpp vcc_lo, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16 vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_u_f16_e64_dpp vcc_lo, v127, v255 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_class_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_class_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_class_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_eq_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_eq_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_eq_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_i16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_eq_i16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_eq_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_eq_u16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_eq_u16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ge_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ge_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ge_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_i16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ge_i16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ge_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ge_u16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ge_u16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_gt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_gt_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_gt_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_i16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_gt_i16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_gt_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_gt_u16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_gt_u16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_le_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_le_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_le_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_i16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_le_i16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_le_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_le_u16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_le_u16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_lg_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lg_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_lg_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_lt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_lt_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_lt_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_i16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_lt_i16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_lt_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_lt_u16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_lt_u16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ne_i16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_i16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ne_i16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ne_u16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ne_u16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ne_u16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_neq_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_neq_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_neq_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_nge_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nge_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_nge_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_ngt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_ngt_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_ngt_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_nle_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nle_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_nle_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_nlg_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlg_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_nlg_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_nlt_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_nlt_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_nlt_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_o_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_o_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_o_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16 vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64: v_cmp_u_f16_e64_dpp vcc, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cmp_u_f16 vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W32: v_cmp_u_f16_e64_dpp vcc_lo, v128, v2 dpp8:[7,6,5,4,3,2,1,0]
-// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmp_u_f16 vcc_lo, vcc_lo, v255
+// W32: v_cmp_u_f16_e64 vcc_lo, vcc_lo, v255    ; encoding: [0x6a,0x00,0x08,0xd4,0x6a,0xfe,0x03,0x00]
+// W64-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx-fake16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx-fake16.s
new file mode 100644
index 000000000000..a5b673494f13
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx-fake16.s
@@ -0,0 +1,3404 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+
+v_cmpx_class_f16_e32 v1, v2
+// GFX12: encoding: [0x01,0x05,0xfa,0x7d]
+
+v_cmpx_class_f16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0xfa,0x7d]
+
+v_cmpx_class_f16 s1, v2
+// GFX12: encoding: [0x01,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 s105, v2
+// GFX12: encoding: [0x69,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 null, v2
+// GFX12: encoding: [0x7c,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0xfa,0x7d]
+
+v_cmpx_class_f16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0xfa,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_class_f32 v1, v2
+// GFX12: encoding: [0x01,0x05,0xfc,0x7d]
+
+v_cmpx_class_f32 v255, v2
+// GFX12: encoding: [0xff,0x05,0xfc,0x7d]
+
+v_cmpx_class_f32 s1, v2
+// GFX12: encoding: [0x01,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 s105, v2
+// GFX12: encoding: [0x69,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 null, v2
+// GFX12: encoding: [0x7c,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0xfc,0x7d]
+
+v_cmpx_class_f32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xfd,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_class_f64 v[1:2], v2
+// GFX12: encoding: [0x01,0x05,0xfe,0x7d]
+
+v_cmpx_class_f64 v[254:255], v2
+// GFX12: encoding: [0xfe,0x05,0xfe,0x7d]
+
+v_cmpx_class_f64 s[2:3], v2
+// GFX12: encoding: [0x02,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 s[104:105], v2
+// GFX12: encoding: [0x68,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 vcc, v2
+// GFX12: encoding: [0x6a,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 ttmp[14:15], v2
+// GFX12: encoding: [0x7a,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 exec, v2
+// GFX12: encoding: [0x7e,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 null, v2
+// GFX12: encoding: [0x7c,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 -1, v2
+// GFX12: encoding: [0xc1,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0xfe,0x7d]
+
+v_cmpx_class_f64 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_f16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x04,0x7d]
+
+v_cmpx_eq_f16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x04,0x7d]
+
+v_cmpx_eq_f16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x04,0x7d]
+
+v_cmpx_eq_f16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x04,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_f32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x24,0x7d]
+
+v_cmpx_eq_f32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x24,0x7d]
+
+v_cmpx_eq_f32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x24,0x7d]
+
+v_cmpx_eq_f32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x25,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_f64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0x44,0x7d]
+
+v_cmpx_eq_f64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x44,0x7d]
+
+v_cmpx_eq_f64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x44,0x7d]
+
+v_cmpx_eq_f64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0x45,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_i16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x64,0x7d]
+
+v_cmpx_eq_i16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x64,0x7d]
+
+v_cmpx_eq_i16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x64,0x7d]
+
+v_cmpx_eq_i16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x64,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_i32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x84,0x7d]
+
+v_cmpx_eq_i32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x84,0x7d]
+
+v_cmpx_eq_i32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x84,0x7d]
+
+v_cmpx_eq_i32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x85,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_i64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0xa4,0x7d]
+
+v_cmpx_eq_i64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0xa4,0x7d]
+
+v_cmpx_eq_i64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0xa4,0x7d]
+
+v_cmpx_eq_i64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xa5,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_u16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x74,0x7d]
+
+v_cmpx_eq_u16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x74,0x7d]
+
+v_cmpx_eq_u16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x74,0x7d]
+
+v_cmpx_eq_u16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x74,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_u32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x94,0x7d]
+
+v_cmpx_eq_u32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x94,0x7d]
+
+v_cmpx_eq_u32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x94,0x7d]
+
+v_cmpx_eq_u32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x95,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_u64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0xb4,0x7d]
+
+v_cmpx_eq_u64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0xb4,0x7d]
+
+v_cmpx_eq_u64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0xb4,0x7d]
+
+v_cmpx_eq_u64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xb5,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_f16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x0c,0x7d]
+
+v_cmpx_ge_f16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x0c,0x7d]
+
+v_cmpx_ge_f16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0c,0x7d]
+
+v_cmpx_ge_f16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x0c,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_f32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x2c,0x7d]
+
+v_cmpx_ge_f32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x2c,0x7d]
+
+v_cmpx_ge_f32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x2c,0x7d]
+
+v_cmpx_ge_f32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x2d,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_f64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0x4c,0x7d]
+
+v_cmpx_ge_f64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x4c,0x7d]
+
+v_cmpx_ge_f64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x4c,0x7d]
+
+v_cmpx_ge_f64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0x4d,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_i16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x6c,0x7d]
+
+v_cmpx_ge_i16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x6c,0x7d]
+
+v_cmpx_ge_i16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x6c,0x7d]
+
+v_cmpx_ge_i16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x6c,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_i32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x8c,0x7d]
+
+v_cmpx_ge_i32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x8c,0x7d]
+
+v_cmpx_ge_i32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x8c,0x7d]
+
+v_cmpx_ge_i32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x8d,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_i64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0xac,0x7d]
+
+v_cmpx_ge_i64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0xac,0x7d]
+
+v_cmpx_ge_i64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0xac,0x7d]
+
+v_cmpx_ge_i64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xad,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_u16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x7c,0x7d]
+
+v_cmpx_ge_u16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x7c,0x7d]
+
+v_cmpx_ge_u16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x7c,0x7d]
+
+v_cmpx_ge_u16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x7c,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_u32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x9c,0x7d]
+
+v_cmpx_ge_u32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x9c,0x7d]
+
+v_cmpx_ge_u32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x9c,0x7d]
+
+v_cmpx_ge_u32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x9d,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_u64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0xbc,0x7d]
+
+v_cmpx_ge_u64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0xbc,0x7d]
+
+v_cmpx_ge_u64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0xbc,0x7d]
+
+v_cmpx_ge_u64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xbd,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_f16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x08,0x7d]
+
+v_cmpx_gt_f16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x08,0x7d]
+
+v_cmpx_gt_f16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x08,0x7d]
+
+v_cmpx_gt_f16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x08,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_f32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x28,0x7d]
+
+v_cmpx_gt_f32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x28,0x7d]
+
+v_cmpx_gt_f32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x28,0x7d]
+
+v_cmpx_gt_f32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x29,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_f64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0x48,0x7d]
+
+v_cmpx_gt_f64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x48,0x7d]
+
+v_cmpx_gt_f64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x48,0x7d]
+
+v_cmpx_gt_f64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0x49,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_i16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x68,0x7d]
+
+v_cmpx_gt_i16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x68,0x7d]
+
+v_cmpx_gt_i16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x68,0x7d]
+
+v_cmpx_gt_i16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x68,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_i32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x88,0x7d]
+
+v_cmpx_gt_i32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x88,0x7d]
+
+v_cmpx_gt_i32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x88,0x7d]
+
+v_cmpx_gt_i32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x89,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_i64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0xa8,0x7d]
+
+v_cmpx_gt_i64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0xa8,0x7d]
+
+v_cmpx_gt_i64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0xa8,0x7d]
+
+v_cmpx_gt_i64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xa9,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_u16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x78,0x7d]
+
+v_cmpx_gt_u16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x78,0x7d]
+
+v_cmpx_gt_u16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x78,0x7d]
+
+v_cmpx_gt_u16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x78,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_u32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x98,0x7d]
+
+v_cmpx_gt_u32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x98,0x7d]
+
+v_cmpx_gt_u32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x98,0x7d]
+
+v_cmpx_gt_u32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x99,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_u64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0xb8,0x7d]
+
+v_cmpx_gt_u64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0xb8,0x7d]
+
+v_cmpx_gt_u64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0xb8,0x7d]
+
+v_cmpx_gt_u64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xb9,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_f16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x06,0x7d]
+
+v_cmpx_le_f16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x06,0x7d]
+
+v_cmpx_le_f16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x06,0x7d]
+
+v_cmpx_le_f16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x06,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_f32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x26,0x7d]
+
+v_cmpx_le_f32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x26,0x7d]
+
+v_cmpx_le_f32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x26,0x7d]
+
+v_cmpx_le_f32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x27,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_f64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0x46,0x7d]
+
+v_cmpx_le_f64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x46,0x7d]
+
+v_cmpx_le_f64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x46,0x7d]
+
+v_cmpx_le_f64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0x47,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_i16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x66,0x7d]
+
+v_cmpx_le_i16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x66,0x7d]
+
+v_cmpx_le_i16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x66,0x7d]
+
+v_cmpx_le_i16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x66,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_i32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x86,0x7d]
+
+v_cmpx_le_i32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x86,0x7d]
+
+v_cmpx_le_i32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x86,0x7d]
+
+v_cmpx_le_i32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x87,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_i64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0xa6,0x7d]
+
+v_cmpx_le_i64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0xa6,0x7d]
+
+v_cmpx_le_i64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0xa6,0x7d]
+
+v_cmpx_le_i64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xa7,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_u16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x76,0x7d]
+
+v_cmpx_le_u16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x76,0x7d]
+
+v_cmpx_le_u16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x76,0x7d]
+
+v_cmpx_le_u16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x76,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_u32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x96,0x7d]
+
+v_cmpx_le_u32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x96,0x7d]
+
+v_cmpx_le_u32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x96,0x7d]
+
+v_cmpx_le_u32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x97,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_u64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0xb6,0x7d]
+
+v_cmpx_le_u64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0xb6,0x7d]
+
+v_cmpx_le_u64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0xb6,0x7d]
+
+v_cmpx_le_u64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xb7,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lg_f16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x7d]
+
+v_cmpx_lg_f16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x0a,0x7d]
+
+v_cmpx_lg_f16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x7d]
+
+v_cmpx_lg_f16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x0a,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lg_f32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x2a,0x7d]
+
+v_cmpx_lg_f32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x2a,0x7d]
+
+v_cmpx_lg_f32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x2a,0x7d]
+
+v_cmpx_lg_f32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x2b,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lg_f64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0x4a,0x7d]
+
+v_cmpx_lg_f64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x4a,0x7d]
+
+v_cmpx_lg_f64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x4a,0x7d]
+
+v_cmpx_lg_f64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0x4b,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_f16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x02,0x7d]
+
+v_cmpx_lt_f16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x02,0x7d]
+
+v_cmpx_lt_f16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x02,0x7d]
+
+v_cmpx_lt_f16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x02,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_f32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x22,0x7d]
+
+v_cmpx_lt_f32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x22,0x7d]
+
+v_cmpx_lt_f32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x22,0x7d]
+
+v_cmpx_lt_f32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x23,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_f64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0x42,0x7d]
+
+v_cmpx_lt_f64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x42,0x7d]
+
+v_cmpx_lt_f64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x42,0x7d]
+
+v_cmpx_lt_f64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0x43,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_i16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x62,0x7d]
+
+v_cmpx_lt_i16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x62,0x7d]
+
+v_cmpx_lt_i16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x62,0x7d]
+
+v_cmpx_lt_i16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x62,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_i32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x82,0x7d]
+
+v_cmpx_lt_i32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x82,0x7d]
+
+v_cmpx_lt_i32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x82,0x7d]
+
+v_cmpx_lt_i32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x83,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_i64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0xa2,0x7d]
+
+v_cmpx_lt_i64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0xa2,0x7d]
+
+v_cmpx_lt_i64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0xa2,0x7d]
+
+v_cmpx_lt_i64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xa3,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_u16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x72,0x7d]
+
+v_cmpx_lt_u16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x72,0x7d]
+
+v_cmpx_lt_u16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x72,0x7d]
+
+v_cmpx_lt_u16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x72,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_u32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x92,0x7d]
+
+v_cmpx_lt_u32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x92,0x7d]
+
+v_cmpx_lt_u32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x92,0x7d]
+
+v_cmpx_lt_u32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x93,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_u64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0xb2,0x7d]
+
+v_cmpx_lt_u64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0xb2,0x7d]
+
+v_cmpx_lt_u64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0xb2,0x7d]
+
+v_cmpx_lt_u64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xb3,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_i16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x6a,0x7d]
+
+v_cmpx_ne_i16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x6a,0x7d]
+
+v_cmpx_ne_i16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x6a,0x7d]
+
+v_cmpx_ne_i16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x6a,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ne_i32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x8a,0x7d]
+
+v_cmpx_ne_i32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x8a,0x7d]
+
+v_cmpx_ne_i32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x8a,0x7d]
+
+v_cmpx_ne_i32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x8b,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_i64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0xaa,0x7d]
+
+v_cmpx_ne_i64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0xaa,0x7d]
+
+v_cmpx_ne_i64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0xaa,0x7d]
+
+v_cmpx_ne_i64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xab,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_u16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x7a,0x7d]
+
+v_cmpx_ne_u16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x7a,0x7d]
+
+v_cmpx_ne_u16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x7a,0x7d]
+
+v_cmpx_ne_u16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x7a,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ne_u32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x9a,0x7d]
+
+v_cmpx_ne_u32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x9a,0x7d]
+
+v_cmpx_ne_u32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x9a,0x7d]
+
+v_cmpx_ne_u32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x9b,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_u64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0xba,0x7d]
+
+v_cmpx_ne_u64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0xba,0x7d]
+
+v_cmpx_ne_u64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0xba,0x7d]
+
+v_cmpx_ne_u64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xbb,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_neq_f16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x1a,0x7d]
+
+v_cmpx_neq_f16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x1a,0x7d]
+
+v_cmpx_neq_f16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x1a,0x7d]
+
+v_cmpx_neq_f16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x1a,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_neq_f32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x3a,0x7d]
+
+v_cmpx_neq_f32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x3a,0x7d]
+
+v_cmpx_neq_f32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x3a,0x7d]
+
+v_cmpx_neq_f32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x3b,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_neq_f64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0x5a,0x7d]
+
+v_cmpx_neq_f64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x5a,0x7d]
+
+v_cmpx_neq_f64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x5a,0x7d]
+
+v_cmpx_neq_f64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0x5b,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nge_f16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x12,0x7d]
+
+v_cmpx_nge_f16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x12,0x7d]
+
+v_cmpx_nge_f16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x12,0x7d]
+
+v_cmpx_nge_f16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x12,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nge_f32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x32,0x7d]
+
+v_cmpx_nge_f32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x32,0x7d]
+
+v_cmpx_nge_f32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x32,0x7d]
+
+v_cmpx_nge_f32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x33,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nge_f64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0x52,0x7d]
+
+v_cmpx_nge_f64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x52,0x7d]
+
+v_cmpx_nge_f64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x52,0x7d]
+
+v_cmpx_nge_f64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0x53,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ngt_f16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x16,0x7d]
+
+v_cmpx_ngt_f16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x16,0x7d]
+
+v_cmpx_ngt_f16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x16,0x7d]
+
+v_cmpx_ngt_f16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x16,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ngt_f32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x36,0x7d]
+
+v_cmpx_ngt_f32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x36,0x7d]
+
+v_cmpx_ngt_f32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x36,0x7d]
+
+v_cmpx_ngt_f32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x37,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ngt_f64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0x56,0x7d]
+
+v_cmpx_ngt_f64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x56,0x7d]
+
+v_cmpx_ngt_f64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x56,0x7d]
+
+v_cmpx_ngt_f64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0x57,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nle_f16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x18,0x7d]
+
+v_cmpx_nle_f16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x18,0x7d]
+
+v_cmpx_nle_f16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x18,0x7d]
+
+v_cmpx_nle_f16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x18,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nle_f32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x38,0x7d]
+
+v_cmpx_nle_f32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x38,0x7d]
+
+v_cmpx_nle_f32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x38,0x7d]
+
+v_cmpx_nle_f32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x39,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nle_f64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0x58,0x7d]
+
+v_cmpx_nle_f64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x58,0x7d]
+
+v_cmpx_nle_f64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x58,0x7d]
+
+v_cmpx_nle_f64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0x59,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlg_f16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x14,0x7d]
+
+v_cmpx_nlg_f16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x14,0x7d]
+
+v_cmpx_nlg_f16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x14,0x7d]
+
+v_cmpx_nlg_f16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x14,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nlg_f32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x34,0x7d]
+
+v_cmpx_nlg_f32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x34,0x7d]
+
+v_cmpx_nlg_f32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x34,0x7d]
+
+v_cmpx_nlg_f32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x35,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlg_f64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0x54,0x7d]
+
+v_cmpx_nlg_f64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x54,0x7d]
+
+v_cmpx_nlg_f64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x54,0x7d]
+
+v_cmpx_nlg_f64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0x55,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlt_f16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x1c,0x7d]
+
+v_cmpx_nlt_f16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x1c,0x7d]
+
+v_cmpx_nlt_f16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x1c,0x7d]
+
+v_cmpx_nlt_f16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x1c,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nlt_f32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x3c,0x7d]
+
+v_cmpx_nlt_f32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x3c,0x7d]
+
+v_cmpx_nlt_f32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x3c,0x7d]
+
+v_cmpx_nlt_f32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x3d,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlt_f64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0x5c,0x7d]
+
+v_cmpx_nlt_f64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x5c,0x7d]
+
+v_cmpx_nlt_f64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x5c,0x7d]
+
+v_cmpx_nlt_f64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0x5d,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_o_f16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x0e,0x7d]
+
+v_cmpx_o_f16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x0e,0x7d]
+
+v_cmpx_o_f16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0e,0x7d]
+
+v_cmpx_o_f16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x0e,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_o_f32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x2e,0x7d]
+
+v_cmpx_o_f32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x2e,0x7d]
+
+v_cmpx_o_f32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x2e,0x7d]
+
+v_cmpx_o_f32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x2f,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_o_f64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0x4e,0x7d]
+
+v_cmpx_o_f64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x4e,0x7d]
+
+v_cmpx_o_f64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x4e,0x7d]
+
+v_cmpx_o_f64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0x4f,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_u_f16 v1, v2
+// GFX12: encoding: [0x01,0x05,0x10,0x7d]
+
+v_cmpx_u_f16 v127, v2
+// GFX12: encoding: [0x7f,0x05,0x10,0x7d]
+
+v_cmpx_u_f16 s1, v2
+// GFX12: encoding: [0x01,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 s105, v2
+// GFX12: encoding: [0x69,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 null, v2
+// GFX12: encoding: [0x7c,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x10,0x7d]
+
+v_cmpx_u_f16 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0x10,0x7d,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_u_f32 v1, v2
+// GFX12: encoding: [0x01,0x05,0x30,0x7d]
+
+v_cmpx_u_f32 v255, v2
+// GFX12: encoding: [0xff,0x05,0x30,0x7d]
+
+v_cmpx_u_f32 s1, v2
+// GFX12: encoding: [0x01,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 s105, v2
+// GFX12: encoding: [0x69,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 m0, v2
+// GFX12: encoding: [0x7d,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 null, v2
+// GFX12: encoding: [0x7c,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 -1, v2
+// GFX12: encoding: [0xc1,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x30,0x7d]
+
+v_cmpx_u_f32 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0x31,0x7d,0x56,0x34,0x12,0xaf]
+
+v_cmpx_u_f64 v[1:2], v[2:3]
+// GFX12: encoding: [0x01,0x05,0x50,0x7d]
+
+v_cmpx_u_f64 v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x50,0x7d]
+
+v_cmpx_u_f64 s[2:3], v[2:3]
+// GFX12: encoding: [0x02,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x50,0x7d]
+
+v_cmpx_u_f64 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0x51,0x7d,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s
index 4c5a8e638e3f..8c01cf4fbce2 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
 
 v_cmpx_class_f16_e32 v1, v2
 // GFX12: encoding: [0x01,0x05,0xfa,0x7d]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16-fake16.s
new file mode 100644
index 000000000000..d4e8069f8798
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16-fake16.s
@@ -0,0 +1,2270 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+
+v_cmpx_class_f16_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_class_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_class_f16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_class_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_class_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_class_f16 -|v127|, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xfa,0x7d,0x7f,0x6f,0x35,0x30]
+
+v_cmpx_class_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_class_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_class_f32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_class_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_class_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_class_f32 -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xfd,0x7d,0xff,0x6f,0x35,0x30]
+
+v_cmpx_eq_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x04,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x04,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x04,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_eq_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x24,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x24,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x25,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_eq_i16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_i16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x64,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_i16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_i16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x64,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_i16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x64,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_eq_i32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_i32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x84,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_i32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_i32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x84,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_i32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x85,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_eq_u16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_u16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x74,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_u16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_u16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x74,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_u16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x74,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_eq_u32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_u32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x94,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_u32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x94,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x95,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ge_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x0c,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_ge_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x2c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x2d,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_ge_i16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_i16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_i16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_i16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x6c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_i16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x6c,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_ge_i32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_i32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_i32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_i32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x8c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_i32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x8d,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ge_u16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_u16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_u16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_u16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x7c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_u16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x7c,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_ge_u32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_u32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_u32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x9c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x9d,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_gt_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x08,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x08,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x08,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_gt_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x28,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x28,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x29,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_gt_i16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_i16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x68,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_i16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_i16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x68,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_i16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x68,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_gt_i32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_i32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x88,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_i32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_i32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x88,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_i32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x89,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_gt_u16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_u16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x78,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_u16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_u16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x78,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_u16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x78,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_gt_u32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_u32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x98,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_u32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x98,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x99,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_le_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x06,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_f16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x06,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x06,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_le_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x26,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_f32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x26,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x27,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_le_i16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_i16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x66,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_i16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_i16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_i16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x66,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_i16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x66,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_le_i32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_i32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x86,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_i32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_i32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_i32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x86,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_i32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x87,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_le_u16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_u16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x76,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_u16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_u16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_u16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x76,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_u16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x76,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_le_u32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_u32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x96,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_u32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_u32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x96,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x97,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_lg_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x0a,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_lg_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lg_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lg_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x2a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lg_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x2b,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_lt_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x02,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x02,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x02,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_lt_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x22,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x22,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x23,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_lt_i16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_i16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x62,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_i16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_i16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x62,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_i16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x62,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_lt_i32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_i32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x82,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_i32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_i32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x82,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_i32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x83,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_lt_u16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_u16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x72,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_u16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_u16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x72,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_u16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x72,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_lt_u32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_u32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x92,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_u32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x92,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x93,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ne_i16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ne_i16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ne_i16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ne_i16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x6a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ne_i16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x6a,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_ne_i32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ne_i32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ne_i32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ne_i32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x8a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ne_i32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x8b,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ne_u16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ne_u16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ne_u16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ne_u16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x7a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ne_u16 v127, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x7a,0x7d,0x7f,0x6f,0x05,0x30]
+
+v_cmpx_ne_u32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ne_u32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ne_u32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ne_u32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x9a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ne_u32 v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x9b,0x7d,0xff,0x6f,0x05,0x30]
+
+v_cmpx_neq_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_neq_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_neq_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_neq_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x1a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_neq_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x1a,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_neq_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_neq_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_neq_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_neq_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x3a,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_neq_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x3b,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_nge_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x12,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nge_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nge_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x12,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nge_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x12,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_nge_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nge_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x32,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nge_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nge_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x32,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nge_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x33,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_ngt_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x16,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ngt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ngt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x16,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ngt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x16,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_ngt_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x36,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_ngt_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ngt_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x36,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_ngt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x37,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_nle_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nle_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x18,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nle_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nle_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x18,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nle_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x18,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_nle_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nle_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x38,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nle_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nle_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x38,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nle_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x39,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_nlg_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x14,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nlg_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nlg_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x14,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nlg_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x14,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_nlg_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x34,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nlg_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nlg_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x34,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nlg_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x35,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_nlt_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlt_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nlt_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nlt_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x1c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nlt_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x1c,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_nlt_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlt_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_nlt_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nlt_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x3c,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_nlt_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x3d,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_o_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_o_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_o_f16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_o_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_o_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0e,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_o_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x0e,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_o_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_o_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_o_f32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_o_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_o_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x2e,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_o_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x2f,0x7d,0xff,0x6f,0xf5,0x30]
+
+v_cmpx_u_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_u_f16 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x10,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_u_f16 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_u_f16 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_u_f16 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x10,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_u_f16 -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x10,0x7d,0x7f,0x6f,0xf5,0x30]
+
+v_cmpx_u_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_u_f32 v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x30,0x7d,0x01,0xe4,0x00,0xff]
+
+v_cmpx_u_f32 v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x40,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x41,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x01,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x0f,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x11,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x1f,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x21,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x2f,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x50,0x01,0xff]
+
+v_cmpx_u_f32 v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x5f,0x01,0x01]
+
+v_cmpx_u_f32 v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x30,0x7d,0x01,0x60,0x09,0x13]
+
+v_cmpx_u_f32 -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0x31,0x7d,0xff,0x6f,0xf5,0x30]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16.s
index 9c29f5bcd714..2dc2ecfbe9ba 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
 
 v_cmpx_class_f16_dpp v1, v2 quad_perm:[3,2,1,0]
 // GFX12: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8-fake16.s
new file mode 100644
index 000000000000..067a1e2a65d2
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8-fake16.s
@@ -0,0 +1,488 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+
+v_cmpx_class_f16_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xfa,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_class_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0xfc,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0xfc,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xfd,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x04,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x04,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_eq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x24,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x24,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x25,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_eq_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x64,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x64,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_i16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x64,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_eq_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x84,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x84,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_i32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x85,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_eq_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x74,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x74,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_u16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x74,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_eq_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x94,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x94,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x95,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x0c,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_ge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x2c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x2c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x2d,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_ge_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x6c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x6c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_i16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x6c,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_ge_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x8c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x8c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_i32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x8d,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_ge_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x7c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x7c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_u16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x7c,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_ge_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x9c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x9c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x9d,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x08,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x08,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_gt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x28,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x28,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x29,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_gt_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x68,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x68,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_i16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x68,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_gt_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x88,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x88,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_i32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x89,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_gt_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x78,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x78,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_u16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x78,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_gt_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x98,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x98,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x99,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x06,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x06,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_le_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x26,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x26,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x27,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_le_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x66,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x66,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_i16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x66,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_le_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x86,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x86,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_i32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x87,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_le_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x76,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x76,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_u16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x76,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_le_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x96,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x96,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x97,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x0a,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_lg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x2a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x2a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x2b,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_lt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x02,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x02,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x02,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_lt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x22,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x22,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x23,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_lt_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x62,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x62,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_i16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x62,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_lt_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x82,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x82,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_i32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x83,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_lt_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x72,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x72,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_u16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x72,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_lt_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x92,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x92,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x93,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_ne_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x6a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_i16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x6a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_i16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x6a,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_ne_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x8a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_i32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x8a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_i32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x8b,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_ne_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x7a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_u16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x7a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_u16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x7a,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_ne_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x9a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_u32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x9a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_u32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x9b,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x1a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x1a,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_neq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x3a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x3a,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x3b,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x12,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x12,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_nge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x32,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x32,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x33,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x16,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x16,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_ngt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x36,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x36,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x37,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x18,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x18,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_nle_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x38,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x38,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x39,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x14,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x14,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_nlg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x34,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x34,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x35,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x1c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x1c,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_nlt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x3c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x3c,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x3d,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x0e,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_o_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x2e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x2e,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x2f,0x7d,0xff,0x00,0x00,0x00]
+
+v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x10,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x10,0x7d,0x7f,0x00,0x00,0x00]
+
+v_cmpx_u_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x30,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x30,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0x31,0x7d,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8.s
index 0f82932a9e34..a679d693a595 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
 
 v_cmpx_class_f16_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: encoding: [0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_err.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_err.s
index 8d6bff152101..5019324d174b 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_err.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_err.s
@@ -1,487 +1,488 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort --version 5
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s
 
 v_cmpx_class_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_eq_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmpx_eq_i16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_cmpx_eq_u16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_class_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_class_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmpx_ge_i16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_class_f16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ge_u16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_class_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_class_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cmpx_gt_i16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_gt_u16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_le_i16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_le_u16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lt_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_i16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_lt_i16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lt_u16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_i16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ne_i16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_i16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ne_u16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_i16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_u16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ngt_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_u16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_u16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_nlt_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_eq_u16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_f16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_class_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_eq_i16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_f16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_eq_u16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ge_i16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_i16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ge_u16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_i16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_i16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_i16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_gt_u16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_i16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_le_i16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_u16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_le_u16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_u16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lt_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_u16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_lt_i16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lt_u16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_ge_u16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ne_i16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ne_u16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ngt_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_i16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_nlt_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_i16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cmpx_gt_i16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_class_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_i16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_eq_i16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_u16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_eq_u16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_u16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ge_i16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_u16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ge_u16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_gt_u16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_i16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_gt_u16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
 v_cmpx_le_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_le_i16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_le_u16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_i16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_lt_i16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lt_u16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_i16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ne_i16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_i16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ne_u16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_i16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_u16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ngt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_u16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_u16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_nlt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_le_u16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_class_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_eq_i16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_eq_u16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lg_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ge_i16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_f16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ge_u16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_i16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_f16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_gt_u16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_le_i16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_i16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_le_u16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_i16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_i16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_lt_i16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_lt_u16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_i16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ne_i16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_u16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ne_u16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_u16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_u16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ngt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_lt_u16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_i16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_nlt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_i16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_i16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_class_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_i16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_eq_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_u16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_eq_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_u16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_ge_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_u16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ge_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ne_u16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cmpx_gt_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_gt_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_le_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_le_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_neq_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_lt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_lt_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_lt_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_ne_i16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ne_u16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nge_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmpx_ngt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_nlt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_ngt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_u_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_class_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_eq_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_eq_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_eq_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_ge_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nle_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_ge_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ge_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_gt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_gt_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_gt_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_le_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlg_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_le_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_le_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_lg_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_lt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_lt_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_lt_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_nlt_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_cmpx_ne_i16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ne_u16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmpx_neq_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmpx_nge_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_ngt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmpx_nle_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_o_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmpx_nlg_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cmpx_nlt_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_cmpx_o_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cmpx_u_f16_e32 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_cmpx_u_f16_e32 v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cmpx_u_f16_e32 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_cmpx_u_f16_e32 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s
index 9d7b60b91730..4f462861e3a0 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s
@@ -1,487 +1,488 @@
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 %s
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 %s
 
 v_cmpx_class_f16 v1, v255
-// GFX12: v_cmpx_class_f16_e64
+// GFX12: v_cmpx_class_f16_e64 v1, v255           ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_eq_f16 v1, v255
-// GFX12: v_cmpx_eq_f16_e64
-
-v_cmpx_eq_i16 v1, v255
-// GFX12: v_cmpx_eq_i16_e64
-
-v_cmpx_eq_u16 v1, v255
-// GFX12: v_cmpx_eq_u16_e64
+v_cmpx_class_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_class_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xfd,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v1, v255
-// GFX12: v_cmpx_ge_f16_e64
+v_cmpx_class_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_class_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ge_i16 v1, v255
-// GFX12: v_cmpx_ge_i16_e64
+v_cmpx_class_f16 v255, v2
+// GFX12: v_cmpx_class_f16_e64 v255, v2           ; encoding: [0x7e,0x00,0xfd,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_ge_u16 v1, v255
-// GFX12: v_cmpx_ge_u16_e64
+v_cmpx_class_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_class_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xfd,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v1, v255
-// GFX12: v_cmpx_gt_f16_e64
+v_cmpx_class_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_class_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_gt_i16 v1, v255
-// GFX12: v_cmpx_gt_i16_e64
+v_cmpx_eq_f16 v1, v255
+// GFX12: v_cmpx_eq_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_gt_u16 v1, v255
-// GFX12: v_cmpx_gt_u16_e64
+v_cmpx_eq_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_le_f16 v1, v255
-// GFX12: v_cmpx_le_f16_e64
+v_cmpx_eq_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_le_i16 v1, v255
-// GFX12: v_cmpx_le_i16_e64
+v_cmpx_eq_f16 v255, v2
+// GFX12: v_cmpx_eq_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_le_u16 v1, v255
-// GFX12: v_cmpx_le_u16_e64
+v_cmpx_eq_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v1, v255
-// GFX12: v_cmpx_lg_f16_e64
+v_cmpx_eq_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_eq_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_lt_f16 v1, v255
-// GFX12: v_cmpx_lt_f16_e64
+v_cmpx_eq_i16 v1, v255
+// GFX12: v_cmpx_eq_i16_e64 v1, v255              ; encoding: [0x7e,0x00,0xb2,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_lt_i16 v1, v255
-// GFX12: v_cmpx_lt_i16_e64
+v_cmpx_eq_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_eq_i16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb2,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_lt_u16 v1, v255
-// GFX12: v_cmpx_lt_u16_e64
+v_cmpx_eq_i16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_eq_i16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ne_i16 v1, v255
-// GFX12: v_cmpx_ne_i16_e64
+v_cmpx_eq_i16 v255, v2
+// GFX12: v_cmpx_eq_i16_e64 v255, v2              ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_ne_u16 v1, v255
-// GFX12: v_cmpx_ne_u16_e64
+v_cmpx_eq_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_eq_i16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb2,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v1, v255
-// GFX12: v_cmpx_neq_f16_e64
+v_cmpx_eq_i16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_eq_i16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nge_f16 v1, v255
-// GFX12: v_cmpx_nge_f16_e64
+v_cmpx_eq_u16 v1, v255
+// GFX12: v_cmpx_eq_u16_e64 v1, v255              ; encoding: [0x7e,0x00,0xba,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ngt_f16 v1, v255
-// GFX12: v_cmpx_ngt_f16_e64
+v_cmpx_eq_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_eq_u16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xba,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_nle_f16 v1, v255
-// GFX12: v_cmpx_nle_f16_e64
+v_cmpx_eq_u16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_eq_u16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nlg_f16 v1, v255
-// GFX12: v_cmpx_nlg_f16_e64
+v_cmpx_eq_u16 v255, v2
+// GFX12: v_cmpx_eq_u16_e64 v255, v2              ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nlt_f16 v1, v255
-// GFX12: v_cmpx_nlt_f16_e64
+v_cmpx_eq_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_eq_u16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xba,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_o_f16 v1, v255
-// GFX12: v_cmpx_o_f16_e64
+v_cmpx_eq_u16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_eq_u16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_u_f16 v1, v255
-// GFX12: v_cmpx_u_f16_e64
+v_cmpx_ge_f16 v1, v255
+// GFX12: v_cmpx_ge_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_class_f16 v255, v2
-// GFX12: v_cmpx_class_f16_e64
+v_cmpx_ge_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16 v255, v2
-// GFX12: v_cmpx_eq_f16_e64
+v_cmpx_ge_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_eq_i16 v255, v2
-// GFX12: v_cmpx_eq_i16_e64
+v_cmpx_ge_f16 v255, v2
+// GFX12: v_cmpx_ge_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_eq_u16 v255, v2
-// GFX12: v_cmpx_eq_u16_e64
+v_cmpx_ge_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v255, v2
-// GFX12: v_cmpx_ge_f16_e64
+v_cmpx_ge_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ge_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ge_i16 v255, v2
-// GFX12: v_cmpx_ge_i16_e64
+v_cmpx_ge_i16 v1, v255
+// GFX12: v_cmpx_ge_i16_e64 v1, v255              ; encoding: [0x7e,0x00,0xb6,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ge_u16 v255, v2
-// GFX12: v_cmpx_ge_u16_e64
+v_cmpx_ge_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ge_i16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb6,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v255, v2
-// GFX12: v_cmpx_gt_f16_e64
+v_cmpx_ge_i16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ge_i16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_gt_i16 v255, v2
-// GFX12: v_cmpx_gt_i16_e64
+v_cmpx_ge_i16 v255, v2
+// GFX12: v_cmpx_ge_i16_e64 v255, v2              ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_gt_u16 v255, v2
-// GFX12: v_cmpx_gt_u16_e64
+v_cmpx_ge_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ge_i16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb6,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_le_f16 v255, v2
-// GFX12: v_cmpx_le_f16_e64
+v_cmpx_ge_i16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ge_i16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_le_i16 v255, v2
-// GFX12: v_cmpx_le_i16_e64
+v_cmpx_ge_u16 v1, v255
+// GFX12: v_cmpx_ge_u16_e64 v1, v255              ; encoding: [0x7e,0x00,0xbe,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_le_u16 v255, v2
-// GFX12: v_cmpx_le_u16_e64
+v_cmpx_ge_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ge_u16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbe,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v255, v2
-// GFX12: v_cmpx_lg_f16_e64
+v_cmpx_ge_u16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ge_u16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_lt_f16 v255, v2
-// GFX12: v_cmpx_lt_f16_e64
+v_cmpx_ge_u16 v255, v2
+// GFX12: v_cmpx_ge_u16_e64 v255, v2              ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_lt_i16 v255, v2
-// GFX12: v_cmpx_lt_i16_e64
+v_cmpx_ge_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ge_u16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbe,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_lt_u16 v255, v2
-// GFX12: v_cmpx_lt_u16_e64
+v_cmpx_ge_u16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ge_u16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ne_i16 v255, v2
-// GFX12: v_cmpx_ne_i16_e64
+v_cmpx_gt_f16 v1, v255
+// GFX12: v_cmpx_gt_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ne_u16 v255, v2
-// GFX12: v_cmpx_ne_u16_e64
+v_cmpx_gt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v255, v2
-// GFX12: v_cmpx_neq_f16_e64
+v_cmpx_gt_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nge_f16 v255, v2
-// GFX12: v_cmpx_nge_f16_e64
+v_cmpx_gt_f16 v255, v2
+// GFX12: v_cmpx_gt_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_ngt_f16 v255, v2
-// GFX12: v_cmpx_ngt_f16_e64
+v_cmpx_gt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nle_f16 v255, v2
-// GFX12: v_cmpx_nle_f16_e64
+v_cmpx_gt_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_gt_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nlg_f16 v255, v2
-// GFX12: v_cmpx_nlg_f16_e64
+v_cmpx_gt_i16 v1, v255
+// GFX12: v_cmpx_gt_i16_e64 v1, v255              ; encoding: [0x7e,0x00,0xb4,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_nlt_f16 v255, v2
-// GFX12: v_cmpx_nlt_f16_e64
+v_cmpx_gt_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_gt_i16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb4,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16 v255, v2
-// GFX12: v_cmpx_o_f16_e64
+v_cmpx_gt_i16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_gt_i16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_u_f16 v255, v2
-// GFX12: v_cmpx_u_f16_e64
+v_cmpx_gt_i16 v255, v2
+// GFX12: v_cmpx_gt_i16_e64 v255, v2              ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_class_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_class_f16_e64
+v_cmpx_gt_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_gt_i16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb4,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_eq_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_eq_f16_e64
+v_cmpx_gt_i16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_gt_i16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_eq_i16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_eq_i16_e64
+v_cmpx_gt_u16 v1, v255
+// GFX12: v_cmpx_gt_u16_e64 v1, v255              ; encoding: [0x7e,0x00,0xbc,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_eq_u16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_eq_u16_e64
+v_cmpx_gt_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_gt_u16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbc,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ge_f16_e64
+v_cmpx_gt_u16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_gt_u16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ge_i16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ge_i16_e64
+v_cmpx_gt_u16 v255, v2
+// GFX12: v_cmpx_gt_u16_e64 v255, v2              ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_ge_u16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ge_u16_e64
+v_cmpx_gt_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_gt_u16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbc,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_gt_f16_e64
+v_cmpx_gt_u16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_gt_u16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_gt_i16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_gt_i16_e64
+v_cmpx_le_f16 v1, v255
+// GFX12: v_cmpx_le_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_gt_u16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_gt_u16_e64
+v_cmpx_le_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
 v_cmpx_le_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_le_f16_e64
+// GFX12: v_cmpx_le_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_le_i16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_le_i16_e64
+v_cmpx_le_f16 v255, v2
+// GFX12: v_cmpx_le_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_le_u16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_le_u16_e64
+v_cmpx_le_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_lg_f16_e64
+v_cmpx_le_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_le_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_lt_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_lt_f16_e64
+v_cmpx_le_i16 v1, v255
+// GFX12: v_cmpx_le_i16_e64 v1, v255              ; encoding: [0x7e,0x00,0xb3,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_lt_i16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_lt_i16_e64
+v_cmpx_le_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_le_i16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb3,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_lt_u16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_lt_u16_e64
+v_cmpx_le_i16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_le_i16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ne_i16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ne_i16_e64
+v_cmpx_le_i16 v255, v2
+// GFX12: v_cmpx_le_i16_e64 v255, v2              ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_ne_u16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ne_u16_e64
+v_cmpx_le_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_le_i16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb3,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_neq_f16_e64
+v_cmpx_le_i16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_le_i16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nge_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nge_f16_e64
+v_cmpx_le_u16 v1, v255
+// GFX12: v_cmpx_le_u16_e64 v1, v255              ; encoding: [0x7e,0x00,0xbb,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ngt_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ngt_f16_e64
+v_cmpx_le_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_le_u16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbb,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_nle_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nle_f16_e64
+v_cmpx_le_u16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_le_u16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nlg_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlg_f16_e64
+v_cmpx_le_u16 v255, v2
+// GFX12: v_cmpx_le_u16_e64 v255, v2              ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nlt_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlt_f16_e64
+v_cmpx_le_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_le_u16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbb,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_o_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_o_f16_e64
+v_cmpx_le_u16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_le_u16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_u_f16 v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_u_f16_e64
+v_cmpx_lg_f16 v1, v255
+// GFX12: v_cmpx_lg_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_class_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_class_f16_e64
+v_cmpx_lg_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_eq_f16_e64
+v_cmpx_lg_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_eq_i16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_eq_i16_e64
+v_cmpx_lg_f16 v255, v2
+// GFX12: v_cmpx_lg_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_eq_u16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_eq_u16_e64
+v_cmpx_lg_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ge_f16_e64
+v_cmpx_lg_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lg_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ge_i16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ge_i16_e64
+v_cmpx_lt_f16 v1, v255
+// GFX12: v_cmpx_lt_f16_e64 v1, v255              ; encoding: [0x7e,0x00,0x81,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ge_u16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ge_u16_e64
+v_cmpx_lt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lt_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x81,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_gt_f16_e64
+v_cmpx_lt_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lt_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_gt_i16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_gt_i16_e64
+v_cmpx_lt_f16 v255, v2
+// GFX12: v_cmpx_lt_f16_e64 v255, v2              ; encoding: [0x7e,0x00,0x81,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_gt_u16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_gt_u16_e64
+v_cmpx_lt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lt_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x81,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_le_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_le_f16_e64
+v_cmpx_lt_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lt_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_le_i16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_le_i16_e64
+v_cmpx_lt_i16 v1, v255
+// GFX12: v_cmpx_lt_i16_e64 v1, v255              ; encoding: [0x7e,0x00,0xb1,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_le_u16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_le_u16_e64
+v_cmpx_lt_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lt_i16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb1,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_lg_f16_e64
+v_cmpx_lt_i16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lt_i16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_lt_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_lt_f16_e64
+v_cmpx_lt_i16 v255, v2
+// GFX12: v_cmpx_lt_i16_e64 v255, v2              ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_lt_i16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_lt_i16_e64
+v_cmpx_lt_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lt_i16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb1,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_lt_u16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_lt_u16_e64
+v_cmpx_lt_i16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lt_i16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ne_i16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ne_i16_e64
+v_cmpx_lt_u16 v1, v255
+// GFX12: v_cmpx_lt_u16_e64 v1, v255              ; encoding: [0x7e,0x00,0xb9,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ne_u16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ne_u16_e64
+v_cmpx_lt_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lt_u16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb9,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_neq_f16_e64
+v_cmpx_lt_u16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lt_u16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nge_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nge_f16_e64
+v_cmpx_lt_u16 v255, v2
+// GFX12: v_cmpx_lt_u16_e64 v255, v2              ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_ngt_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_ngt_f16_e64
+v_cmpx_lt_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_lt_u16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb9,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nle_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nle_f16_e64
+v_cmpx_lt_u16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_lt_u16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nlg_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlg_f16_e64
+v_cmpx_ne_i16 v1, v255
+// GFX12: v_cmpx_ne_i16_e64 v1, v255              ; encoding: [0x7e,0x00,0xb5,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_nlt_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_nlt_f16_e64
+v_cmpx_ne_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ne_i16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb5,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_o_f16_e64
+v_cmpx_ne_i16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ne_i16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_u_f16 v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_cmpx_u_f16_e64
+v_cmpx_ne_i16 v255, v2
+// GFX12: v_cmpx_ne_i16_e64 v255, v2              ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_class_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_class_f16_e64
+v_cmpx_ne_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ne_i16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb5,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_eq_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_eq_f16_e64
+v_cmpx_ne_i16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ne_i16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_eq_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_eq_i16_e64
+v_cmpx_ne_u16 v1, v255
+// GFX12: v_cmpx_ne_u16_e64 v1, v255              ; encoding: [0x7e,0x00,0xbd,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_eq_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_eq_u16_e64
+v_cmpx_ne_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ne_u16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbd,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ge_f16_e64
+v_cmpx_ne_u16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ne_u16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ge_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ge_i16_e64
+v_cmpx_ne_u16 v255, v2
+// GFX12: v_cmpx_ne_u16_e64 v255, v2              ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_ge_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ge_u16_e64
+v_cmpx_ne_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ne_u16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbd,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_gt_f16_e64
+v_cmpx_ne_u16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ne_u16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_gt_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_gt_i16_e64
+v_cmpx_neq_f16 v1, v255
+// GFX12: v_cmpx_neq_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_gt_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_gt_u16_e64
+v_cmpx_neq_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_le_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_le_f16_e64
+v_cmpx_neq_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_le_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_le_i16_e64
+v_cmpx_neq_f16 v255, v2
+// GFX12: v_cmpx_neq_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_le_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_le_u16_e64
+v_cmpx_neq_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_lg_f16_e64
+v_cmpx_neq_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_neq_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_lt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_lt_f16_e64
+v_cmpx_nge_f16 v1, v255
+// GFX12: v_cmpx_nge_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_lt_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_lt_i16_e64
+v_cmpx_nge_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_lt_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_lt_u16_e64
+v_cmpx_nge_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_ne_i16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ne_i16_e64
+v_cmpx_nge_f16 v255, v2
+// GFX12: v_cmpx_nge_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_ne_u16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ne_u16_e64
+v_cmpx_nge_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_neq_f16_e64
+v_cmpx_nge_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nge_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nge_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nge_f16_e64
+v_cmpx_ngt_f16 v1, v255
+// GFX12: v_cmpx_ngt_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ngt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ngt_f16_e64
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_nle_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nle_f16_e64
+v_cmpx_ngt_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nlg_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nlg_f16_e64
+v_cmpx_ngt_f16 v255, v2
+// GFX12: v_cmpx_ngt_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_nlt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nlt_f16_e64
+v_cmpx_ngt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_o_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_o_f16_e64
+v_cmpx_ngt_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_ngt_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_u_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_u_f16_e64
+v_cmpx_nle_f16 v1, v255
+// GFX12: v_cmpx_nle_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_class_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_class_f16_e64
+v_cmpx_nle_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_eq_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_eq_f16_e64
+v_cmpx_nle_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_eq_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_eq_i16_e64
+v_cmpx_nle_f16 v255, v2
+// GFX12: v_cmpx_nle_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_eq_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_eq_u16_e64
+v_cmpx_nle_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_ge_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ge_f16_e64
+v_cmpx_nle_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nle_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ge_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ge_i16_e64
+v_cmpx_nlg_f16 v1, v255
+// GFX12: v_cmpx_nlg_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ge_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ge_u16_e64
+v_cmpx_nlg_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_gt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_gt_f16_e64
+v_cmpx_nlg_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_gt_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_gt_i16_e64
+v_cmpx_nlg_f16 v255, v2
+// GFX12: v_cmpx_nlg_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_gt_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_gt_u16_e64
+v_cmpx_nlg_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_le_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_le_f16_e64
+v_cmpx_nlg_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlg_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_le_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_le_i16_e64
+v_cmpx_nlt_f16 v1, v255
+// GFX12: v_cmpx_nlt_f16_e64 v1, v255             ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_le_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_le_u16_e64
+v_cmpx_nlt_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_lg_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_lg_f16_e64
+v_cmpx_nlt_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_lt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_lt_f16_e64
+v_cmpx_nlt_f16 v255, v2
+// GFX12: v_cmpx_nlt_f16_e64 v255, v2             ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_lt_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_lt_i16_e64
+v_cmpx_nlt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_lt_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_lt_u16_e64
+v_cmpx_nlt_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_nlt_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_ne_i16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ne_i16_e64
+v_cmpx_o_f16 v1, v255
+// GFX12: v_cmpx_o_f16_e64 v1, v255               ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_ne_u16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ne_u16_e64
+v_cmpx_o_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_neq_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_neq_f16_e64
+v_cmpx_o_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_cmpx_nge_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nge_f16_e64
+v_cmpx_o_f16 v255, v2
+// GFX12: v_cmpx_o_f16_e64 v255, v2               ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0x05,0x02,0x00]
 
-v_cmpx_ngt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_ngt_f16_e64
+v_cmpx_o_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_cmpx_nle_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nle_f16_e64
+v_cmpx_o_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_o_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_cmpx_nlg_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nlg_f16_e64
+v_cmpx_u_f16 v1, v255
+// GFX12: v_cmpx_u_f16_e64 v1, v255               ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0xff,0x03,0x00]
 
-v_cmpx_nlt_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_nlt_f16_e64
+v_cmpx_u_f16 v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_o_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_o_f16_e64
+v_cmpx_u_f16 v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_u_f16 v255, v2
+// GFX12: v_cmpx_u_f16_e64 v255, v2               ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_u_f16 v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cmpx_u_f16_e64
+// GFX12: v_cmpx_u_f16_e64_dpp v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_cmpx_u_f16 v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cmpx_u_f16_e64_dpp v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopc.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopc.txt
index 29e39c9e60ec..b87c7bb9b268 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopc.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopc.txt
@@ -1,5 +1,7 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
 
 # W32: v_cmp_class_f16_e32 vcc_lo, v1, v2        ; encoding: [0x01,0x05,0xfa,0x7c]
 # W64: v_cmp_class_f16_e32 vcc, v1, v2           ; encoding: [0x01,0x05,0xfa,0x7c]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopc_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopc_dpp16.txt
index 460b222d0b7d..40735cef0c53 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopc_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopc_dpp16.txt
@@ -1,5 +1,7 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefixes=W32
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefixes=W64
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefixes=W32
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefixes=W64
 
 # W32: v_cmp_class_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1b,0x00,0xff]
 # W64: v_cmp_class_f16 vcc, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopc_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopc_dpp8.txt
index 18f9db15c980..00d5106cc90a 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopc_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopc_dpp8.txt
@@ -1,5 +1,7 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
 
 # W32: v_cmp_class_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05]
 # W64: v_cmp_class_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx.txt
index 863d747e9c0b..c6019b7fdfa7 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx.txt
@@ -1,5 +1,7 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
 
 # GFX11: v_cmpx_class_f16_e32 v1, v2             ; encoding: [0x01,0x05,0xfa,0x7d]
 0x01,0x05,0xfa,0x7d
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp16.txt
index e88d666fb3f9..d3f92d035818 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp16.txt
@@ -1,5 +1,7 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
 
 # GFX11: v_cmpx_class_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp8.txt
index 6c51c9ba5a24..3c5b243e497f 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopcx_dpp8.txt
@@ -1,5 +1,7 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX11
 
 # GFX11: v_cmpx_class_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05]
 0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc.txt
index 320e85238e36..8fff403b502a 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc.txt
@@ -1,5 +1,7 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
 
 # W32: v_cmp_class_f16_e32 vcc_lo, v1, v2        ; encoding: [0x01,0x05,0xfa,0x7c]
 # W64: v_cmp_class_f16_e32 vcc, v1, v2           ; encoding: [0x01,0x05,0xfa,0x7c]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp16.txt
index be40d5fc8af9..a840f0a9c2be 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp16.txt
@@ -1,5 +1,7 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
 
 # W32: v_cmp_class_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1b,0x00,0xff]
 # W64: v_cmp_class_f16 vcc, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp8.txt
index 3fb7bef6c18a..0300ff215c35 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp8.txt
@@ -1,5 +1,7 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64
 
 # W32: v_cmp_class_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05]
 # W64: v_cmp_class_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx.txt
index 5a3b1f183ebf..74213ba162ae 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx.txt
@@ -1,5 +1,7 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
 
 # GFX12: v_cmpx_class_f16_e32 v1, v2             ; encoding: [0x01,0x05,0xfa,0x7d]
 0x01,0x05,0xfa,0x7d
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp16.txt
index 704a17f8b091..fe9ef4f9e90d 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp16.txt
@@ -1,5 +1,7 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
 
 # GFX12: v_cmpx_class_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp8.txt
index 0b030b9e316e..53f15e8ae431 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp8.txt
@@ -1,5 +1,7 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12
 
 # GFX12: v_cmpx_class_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05]
 0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05
-- 
GitLab


From 670512b5c32217e37796fd8d42101ac24cdb4a8d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 28 Oct 2024 14:37:44 +0000
Subject: [PATCH 329/693] [AArch64] Regenerate srem-lkk.ll to add missing asm
 comments

Reduces diff in #112588
---
 llvm/test/CodeGen/AArch64/srem-lkk.ll | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/srem-lkk.ll b/llvm/test/CodeGen/AArch64/srem-lkk.ll
index 5ff178937ebb..d9f91449dffb 100644
--- a/llvm/test/CodeGen/AArch64/srem-lkk.ll
+++ b/llvm/test/CodeGen/AArch64/srem-lkk.ll
@@ -4,14 +4,14 @@
 define i32 @fold_srem_positive_odd(i32 %x) {
 ; CHECK-LABEL: fold_srem_positive_odd:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #37253
+; CHECK-NEXT:    mov w8, #37253 // =0x9185
 ; CHECK-NEXT:    movk w8, #44150, lsl #16
 ; CHECK-NEXT:    smull x8, w0, w8
 ; CHECK-NEXT:    lsr x8, x8, #32
 ; CHECK-NEXT:    add w8, w8, w0
 ; CHECK-NEXT:    asr w9, w8, #6
 ; CHECK-NEXT:    add w8, w9, w8, lsr #31
-; CHECK-NEXT:    mov w9, #95
+; CHECK-NEXT:    mov w9, #95 // =0x5f
 ; CHECK-NEXT:    msub w0, w8, w9, w0
 ; CHECK-NEXT:    ret
   %1 = srem i32 %x, 95
@@ -22,13 +22,13 @@ define i32 @fold_srem_positive_odd(i32 %x) {
 define i32 @fold_srem_positive_even(i32 %x) {
 ; CHECK-LABEL: fold_srem_positive_even:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #36849
+; CHECK-NEXT:    mov w8, #36849 // =0x8ff1
 ; CHECK-NEXT:    movk w8, #15827, lsl #16
 ; CHECK-NEXT:    smull x8, w0, w8
 ; CHECK-NEXT:    lsr x9, x8, #63
 ; CHECK-NEXT:    asr x8, x8, #40
 ; CHECK-NEXT:    add w8, w8, w9
-; CHECK-NEXT:    mov w9, #1060
+; CHECK-NEXT:    mov w9, #1060 // =0x424
 ; CHECK-NEXT:    msub w0, w8, w9, w0
 ; CHECK-NEXT:    ret
   %1 = srem i32 %x, 1060
@@ -39,13 +39,13 @@ define i32 @fold_srem_positive_even(i32 %x) {
 define i32 @fold_srem_negative_odd(i32 %x) {
 ; CHECK-LABEL: fold_srem_negative_odd:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #65445
+; CHECK-NEXT:    mov w8, #65445 // =0xffa5
 ; CHECK-NEXT:    movk w8, #42330, lsl #16
 ; CHECK-NEXT:    smull x8, w0, w8
 ; CHECK-NEXT:    lsr x9, x8, #63
 ; CHECK-NEXT:    asr x8, x8, #40
 ; CHECK-NEXT:    add w8, w8, w9
-; CHECK-NEXT:    mov w9, #-723
+; CHECK-NEXT:    mov w9, #-723 // =0xfffffd2d
 ; CHECK-NEXT:    msub w0, w8, w9, w0
 ; CHECK-NEXT:    ret
   %1 = srem i32 %x, -723
@@ -56,13 +56,13 @@ define i32 @fold_srem_negative_odd(i32 %x) {
 define i32 @fold_srem_negative_even(i32 %x) {
 ; CHECK-LABEL: fold_srem_negative_even:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #62439
+; CHECK-NEXT:    mov w8, #62439 // =0xf3e7
 ; CHECK-NEXT:    movk w8, #64805, lsl #16
 ; CHECK-NEXT:    smull x8, w0, w8
 ; CHECK-NEXT:    lsr x9, x8, #63
 ; CHECK-NEXT:    asr x8, x8, #40
 ; CHECK-NEXT:    add w8, w8, w9
-; CHECK-NEXT:    mov w9, #-22981
+; CHECK-NEXT:    mov w9, #-22981 // =0xffffa63b
 ; CHECK-NEXT:    msub w0, w8, w9, w0
 ; CHECK-NEXT:    ret
   %1 = srem i32 %x, -22981
@@ -74,14 +74,14 @@ define i32 @fold_srem_negative_even(i32 %x) {
 define i32 @combine_srem_sdiv(i32 %x) {
 ; CHECK-LABEL: combine_srem_sdiv:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #37253
+; CHECK-NEXT:    mov w8, #37253 // =0x9185
 ; CHECK-NEXT:    movk w8, #44150, lsl #16
 ; CHECK-NEXT:    smull x8, w0, w8
 ; CHECK-NEXT:    lsr x8, x8, #32
 ; CHECK-NEXT:    add w8, w8, w0
 ; CHECK-NEXT:    asr w9, w8, #6
 ; CHECK-NEXT:    add w8, w9, w8, lsr #31
-; CHECK-NEXT:    mov w9, #95
+; CHECK-NEXT:    mov w9, #95 // =0x5f
 ; CHECK-NEXT:    msub w9, w8, w9, w0
 ; CHECK-NEXT:    add w0, w9, w8
 ; CHECK-NEXT:    ret
@@ -95,14 +95,14 @@ define i32 @combine_srem_sdiv(i32 %x) {
 define i64 @dont_fold_srem_i64(i64 %x) {
 ; CHECK-LABEL: dont_fold_srem_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x8, #58849
+; CHECK-NEXT:    mov x8, #58849 // =0xe5e1
 ; CHECK-NEXT:    movk x8, #48148, lsl #16
 ; CHECK-NEXT:    movk x8, #33436, lsl #32
 ; CHECK-NEXT:    movk x8, #21399, lsl #48
 ; CHECK-NEXT:    smulh x8, x0, x8
 ; CHECK-NEXT:    asr x9, x8, #5
 ; CHECK-NEXT:    add x8, x9, x8, lsr #63
-; CHECK-NEXT:    mov w9, #98
+; CHECK-NEXT:    mov w9, #98 // =0x62
 ; CHECK-NEXT:    msub x0, x8, x9, x0
 ; CHECK-NEXT:    ret
   %1 = srem i64 %x, 98
-- 
GitLab


From 5ac3f3c45cd4d2934d36565eae7e33a629ff7b22 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 28 Oct 2024 09:06:12 -0700
Subject: [PATCH 330/693] [RISCV] Add DestEEW = EEW1 to VMADC. (#113013)

It was present on VMSBC but not VMADC. Reorder the instructions to avoid
duplicate 'let' statements.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoV.td | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
index 6f7d14d5503b..29759132c47d 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
@@ -1142,13 +1142,11 @@ defm VSEXT_VF2 : VALU_MV_VS2<"vsext.vf2", 0b010010, 0b00111>;
 
 // Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
 defm VADC_V : VALUm_IV_V_X_I<"vadc", 0b010000>;
-let Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint in {
-defm VMADC_V : VALUm_IV_V_X_I<"vmadc", 0b010001>;
-defm VMADC_V : VALUNoVm_IV_V_X_I<"vmadc", 0b010001>;
-} // Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint
 defm VSBC_V : VALUm_IV_V_X<"vsbc", 0b010010>;
 let Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint,
     DestEEW = EEW1 in {
+defm VMADC_V : VALUm_IV_V_X_I<"vmadc", 0b010001>;
+defm VMADC_V : VALUNoVm_IV_V_X_I<"vmadc", 0b010001>;
 defm VMSBC_V : VALUm_IV_V_X<"vmsbc", 0b010011>;
 defm VMSBC_V : VALUNoVm_IV_V_X<"vmsbc", 0b010011>;
 } // Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint, DestEEW = EEW1
-- 
GitLab


From eb53d08bce52d4503cf38e26bedfcbd0fed9cd3a Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman@google.com>
Date: Mon, 28 Oct 2024 09:07:14 -0700
Subject: [PATCH 331/693] [llvm-exegesis] Add Pfm Counters for SapphireRapids
 (#113847)

This patch adds the appropriate hookups in X86PfmCounters.td for
SapphireRapids. This is mostly to fix errors when some of my jobs that
only really need dummy counters get scheduled on sapphire rapids
machines, but figured I might as well do it properly while here. I do
not have hardware access to test this currently, but this matches
exactly with what is in the libpfm source code.
---
 llvm/lib/Target/X86/X86PfmCounters.td         | 16 ++++++++++++++++
 llvm/lib/Target/X86/X86SchedSapphireRapids.td |  6 ++++++
 2 files changed, 22 insertions(+)

diff --git a/llvm/lib/Target/X86/X86PfmCounters.td b/llvm/lib/Target/X86/X86PfmCounters.td
index c30e989cdc2a..38d8d19091e0 100644
--- a/llvm/lib/Target/X86/X86PfmCounters.td
+++ b/llvm/lib/Target/X86/X86PfmCounters.td
@@ -220,6 +220,22 @@ def AlderLakePfmCounters : ProcPfmCounters {
 }
 def : PfmCountersBinding<"alderlake", AlderLakePfmCounters>;
 
+def SapphireRapidsPfmCounters : ProcPfmCounters {
+  let CycleCounter = UnhaltedCoreCyclesPfmCounter;
+  let UopsCounter = UopsIssuedPfmCounter;
+  let IssueCounters = [
+    PfmIssueCounter<"SPRPort00", "uops_dispatched_port:port_0">,
+    PfmIssueCounter<"SPRPort01", "uops_dispatched_port:port_1">,
+    PfmIssueCounter<"SPRPort02_03_10", "uops_dispatched_port:port_2_3_10">,
+    PfmIssueCounter<"SPRPort04_09", "uops_dispatched_port:port_4_9">,
+    PfmIssueCounter<"SPRPort05_11", "uops_dispatched_port:port_5_11">,
+    PfmIssueCounter<"SPRPort06", "uops_dispatched_port:port_6">,
+    PfmIssueCounter<"SPRPort07_08", "uops_dispatched_port:port_7_8">,
+  ];
+  let ValidationCounters = DefaultIntelPfmValidationCounters;
+}
+def : PfmCountersBinding<"sapphirerapids", SapphireRapidsPfmCounters>;
+
 // AMD X86 Counters.
 defvar DefaultAMDPfmValidationCounters = [
   PfmValidationCounter<InstructionRetired, "RETIRED_INSTRUCTIONS">,
diff --git a/llvm/lib/Target/X86/X86SchedSapphireRapids.td b/llvm/lib/Target/X86/X86SchedSapphireRapids.td
index 6e292da4e293..b0ebe70c31fd 100644
--- a/llvm/lib/Target/X86/X86SchedSapphireRapids.td
+++ b/llvm/lib/Target/X86/X86SchedSapphireRapids.td
@@ -59,6 +59,8 @@ def SPRPort01_05          : ProcResGroup<[SPRPort01, SPRPort05]>;
 def SPRPort01_05_10       : ProcResGroup<[SPRPort01, SPRPort05, SPRPort10]>;
 def SPRPort02_03          : ProcResGroup<[SPRPort02, SPRPort03]>;
 def SPRPort02_03_11       : ProcResGroup<[SPRPort02, SPRPort03, SPRPort11]>;
+def SPRPort02_03_10       : ProcResGroup<[SPRPort02, SPRPort03, SPRPort10]>;
+def SPRPort05_11          : ProcResGroup<[SPRPort05, SPRPort11]>;
 def SPRPort07_08          : ProcResGroup<[SPRPort07, SPRPort08]>;
 
 // EU has 112 reservation stations.
@@ -78,6 +80,10 @@ def SPRPort02_03_07_08_11 : ProcResGroup<[SPRPort02, SPRPort03, SPRPort07,
   let BufferSize = 72;
 }
 
+def SPRPortAny : ProcResGroup<[SPRPort00, SPRPort01, SPRPort02, SPRPort03,
+                               SPRPort04, SPRPort05, SPRPort06, SPRPort07,
+                               SPRPort08, SPRPort09, SPRPort10, SPRPort11]>;
+
 // Integer loads are 5 cycles, so ReadAfterLd registers needn't be available
 // until 5 cycles after the memory operand.
 def : ReadAdvance<ReadAfterLd, 5>;
-- 
GitLab


From 80f38fbdcfa96316908bb4ac3481d9eac6abf60e Mon Sep 17 00:00:00 2001
From: Boaz Brickner <brickner@google.com>
Date: Mon, 28 Oct 2024 17:10:07 +0100
Subject: [PATCH 332/693] [clang] [NFC] Deduplicate the logic between
 StringMapEntry.first() and StringMapEntry.getKey() (#113735)

---
 llvm/include/llvm/ADT/StringMapEntry.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/llvm/include/llvm/ADT/StringMapEntry.h b/llvm/include/llvm/ADT/StringMapEntry.h
index 98b51cc1aebd..d93af5aedc39 100644
--- a/llvm/include/llvm/ADT/StringMapEntry.h
+++ b/llvm/include/llvm/ADT/StringMapEntry.h
@@ -116,9 +116,7 @@ public:
     return reinterpret_cast<const char *>(this + 1);
   }
 
-  StringRef first() const {
-    return StringRef(getKeyData(), this->getKeyLength());
-  }
+  StringRef first() const { return getKey(); }
 
   /// Create a StringMapEntry for the specified key construct the value using
   /// \p InitiVals.
-- 
GitLab


From 7a710110fcb2ad5d903ec41ba6a63193cf03edc0 Mon Sep 17 00:00:00 2001
From: Petr Kurapov <petr.a.kurapov@intel.com>
Date: Mon, 28 Oct 2024 17:12:12 +0100
Subject: [PATCH 333/693] =?UTF-8?q?[MLIR][Vector]=20Remove=20unused=20and?=
 =?UTF-8?q?=20unimplemented=20Vector=5FWarpExecuteOnLa=E2=80=A6=20(#112338?=
 =?UTF-8?q?)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…ne0Op builder

Removing the declaration instead of implementing the builder as
discussed in #110106
---
 mlir/include/mlir/Dialect/Vector/IR/VectorOps.td | 1 -
 1 file changed, 1 deletion(-)

diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index e859270cf9a5..474f4ccf4891 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -3090,7 +3090,6 @@ def Vector_WarpExecuteOnLane0Op : Vector_Op<"warp_execute_on_lane_0",
 
   let skipDefaultBuilders = 1;
   let builders = [
-    OpBuilder<(ins "Value":$laneid, "int64_t":$warpSize)>,
     OpBuilder<(ins "TypeRange":$resultTypes, "Value":$laneid,
                    "int64_t":$warpSize)>,
     // `blockArgTypes` are different than `args` types as they are they
-- 
GitLab


From 106259510f6a7a3824dd34f78a77ead150dd2154 Mon Sep 17 00:00:00 2001
From: CarolineConcatto <caroline.concatto@arm.com>
Date: Mon, 28 Oct 2024 16:36:02 +0000
Subject: [PATCH 334/693] =?UTF-8?q?[AArch64]Add=20convert=20and=20multiply?=
 =?UTF-8?q?-add=20SIMD&FP=20assembly/disassembly=20in=E2=80=A6=20(#113296)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…structions

This patch adds the following instructions:
Conversion between floating-point and integer:
  FCVT{AS, AU, MS, MU, NS, NU, PS, PU, ZS, ZU}
  {S,U}CVTF
Advanced SIMD three-register extension:
  FMMLA

According to https://developer.arm.com/documentation/ddi0602

Co-authored-by: Marian Lukac marian.lukac@arm.com
Co-authored-by: Spencer Abson spencer.abson@arm.com
---
 .../lib/Target/AArch64/AArch64InstrFormats.td | 109 ++++++--
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |  25 +-
 llvm/test/MC/AArch64/FP8/fmmla-diagnostics.s  |  46 ++++
 llvm/test/MC/AArch64/FP8/fmmla.s              |  25 ++
 llvm/test/MC/AArch64/armv9.6a-cvtf.s          |  61 +++++
 llvm/test/MC/AArch64/armv9.6a-fcvt.s          | 253 ++++++++++++++++++
 .../test/MC/AArch64/directive-arch-negative.s |  18 ++
 llvm/test/MC/AArch64/directive-arch.s         |  12 +
 .../directive-arch_extension-negative.s       |  18 ++
 .../MC/AArch64/directive-arch_extension.s     |  12 +
 llvm/test/MC/AArch64/directive-cpu.s          |  12 +
 llvm/test/MC/AArch64/neon-diagnostics.s       | 108 +++++---
 12 files changed, 637 insertions(+), 62 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/FP8/fmmla-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/FP8/fmmla.s
 create mode 100644 llvm/test/MC/AArch64/armv9.6a-cvtf.s
 create mode 100644 llvm/test/MC/AArch64/armv9.6a-fcvt.s

diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 9dd417314fbb..837d737b2858 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -5234,6 +5234,32 @@ multiclass FPToIntegerUnscaled<bits<2> rmode, bits<3> opcode, string asm,
   }
 }
 
+multiclass FPToIntegerSIMDScalar<bits<2> rmode, bits<3> opcode, string asm> {
+  // double-precision to 32-bit SIMD/FPR
+  def SDr :  BaseFPToIntegerUnscaled<0b01, rmode, opcode, FPR64, FPR32, asm,
+                                     []> {
+    let Inst{31} = 0; // 32-bit FPR flag
+  }
+
+  // half-precision to 32-bit SIMD/FPR
+  def SHr :  BaseFPToIntegerUnscaled<0b11, rmode, opcode, FPR16, FPR32, asm,
+                                     []> {
+    let Inst{31} = 0; // 32-bit FPR flag
+  }
+
+  // half-precision to 64-bit SIMD/FPR
+  def DHr :  BaseFPToIntegerUnscaled<0b11, rmode, opcode, FPR16, FPR64, asm,
+                                     []> {
+    let Inst{31} = 1; // 64-bit FPR flag
+  }
+
+  // single-precision to 64-bit SIMD/FPR
+  def DSr :  BaseFPToIntegerUnscaled<0b00, rmode, opcode, FPR32, FPR64, asm,
+                                     []> {
+    let Inst{31} = 1; // 64-bit FPR flag
+  }
+}
+
 multiclass FPToIntegerScaled<bits<2> rmode, bits<3> opcode, string asm,
                              SDPatternOperator OpN> {
   // Scaled half-precision to 32-bit
@@ -5295,7 +5321,7 @@ multiclass FPToIntegerScaled<bits<2> rmode, bits<3> opcode, string asm,
 //---
 
 let mayStore = 0, mayLoad = 0, hasSideEffects = 0, mayRaiseFPException = 1, Uses = [FPCR] in
-class BaseIntegerToFP<bit isUnsigned,
+class BaseIntegerToFP<bits<2> rmode, bits<3> opcode,
                       RegisterClass srcType, RegisterClass dstType,
                       Operand immType, string asm, list<dag> pattern>
     : I<(outs dstType:$Rd), (ins srcType:$Rn, immType:$scale),
@@ -5305,15 +5331,16 @@ class BaseIntegerToFP<bit isUnsigned,
   bits<5> Rn;
   bits<6> scale;
   let Inst{30-24} = 0b0011110;
-  let Inst{21-17} = 0b00001;
-  let Inst{16}    = isUnsigned;
+  let Inst{21}    = 0b0;
+  let Inst{20-19} = rmode;
+  let Inst{18-16} = opcode;
   let Inst{15-10} = scale;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Rd;
 }
 
 let mayRaiseFPException = 1, Uses = [FPCR] in
-class BaseIntegerToFPUnscaled<bit isUnsigned,
+class BaseIntegerToFPUnscaled<bits<2> rmode, bits<3> opcode,
                       RegisterClass srcType, RegisterClass dstType,
                       ValueType dvt, string asm, SDPatternOperator node>
     : I<(outs dstType:$Rd), (ins srcType:$Rn),
@@ -5323,49 +5350,50 @@ class BaseIntegerToFPUnscaled<bit isUnsigned,
   bits<5> Rn;
   bits<6> scale;
   let Inst{30-24} = 0b0011110;
-  let Inst{21-17} = 0b10001;
-  let Inst{16}    = isUnsigned;
+  let Inst{21}    = 0b1;
+  let Inst{20-19} = rmode;
+  let Inst{18-16} = opcode;
   let Inst{15-10} = 0b000000;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Rd;
 }
 
-multiclass IntegerToFP<bit isUnsigned, string asm, SDPatternOperator node> {
+multiclass IntegerToFP<bits<2> rmode, bits<3> opcode, string asm, SDPatternOperator node> {
   // Unscaled
-  def UWHri: BaseIntegerToFPUnscaled<isUnsigned, GPR32, FPR16, f16, asm, node> {
+  def UWHri: BaseIntegerToFPUnscaled<rmode, opcode, GPR32, FPR16, f16, asm, node> {
     let Inst{31} = 0; // 32-bit GPR flag
     let Inst{23-22} = 0b11; // 16-bit FPR flag
     let Predicates = [HasFullFP16];
   }
 
-  def UWSri: BaseIntegerToFPUnscaled<isUnsigned, GPR32, FPR32, f32, asm, node> {
+  def UWSri: BaseIntegerToFPUnscaled<rmode, opcode, GPR32, FPR32, f32, asm, node> {
     let Inst{31} = 0; // 32-bit GPR flag
     let Inst{23-22} = 0b00; // 32-bit FPR flag
   }
 
-  def UWDri: BaseIntegerToFPUnscaled<isUnsigned, GPR32, FPR64, f64, asm, node> {
+  def UWDri: BaseIntegerToFPUnscaled<rmode, opcode, GPR32, FPR64, f64, asm, node> {
     let Inst{31} = 0; // 32-bit GPR flag
     let Inst{23-22} = 0b01; // 64-bit FPR flag
   }
 
-  def UXHri: BaseIntegerToFPUnscaled<isUnsigned, GPR64, FPR16, f16, asm, node> {
+  def UXHri: BaseIntegerToFPUnscaled<rmode, opcode, GPR64, FPR16, f16, asm, node> {
     let Inst{31} = 1; // 64-bit GPR flag
     let Inst{23-22} = 0b11; // 16-bit FPR flag
     let Predicates = [HasFullFP16];
   }
 
-  def UXSri: BaseIntegerToFPUnscaled<isUnsigned, GPR64, FPR32, f32, asm, node> {
+  def UXSri: BaseIntegerToFPUnscaled<rmode, opcode, GPR64, FPR32, f32, asm, node> {
     let Inst{31} = 1; // 64-bit GPR flag
     let Inst{23-22} = 0b00; // 32-bit FPR flag
   }
 
-  def UXDri: BaseIntegerToFPUnscaled<isUnsigned, GPR64, FPR64, f64, asm, node> {
+  def UXDri: BaseIntegerToFPUnscaled<rmode, opcode, GPR64, FPR64, f64, asm, node> {
     let Inst{31} = 1; // 64-bit GPR flag
     let Inst{23-22} = 0b01; // 64-bit FPR flag
   }
 
   // Scaled
-  def SWHri: BaseIntegerToFP<isUnsigned, GPR32, FPR16, fixedpoint_recip_f16_i32, asm,
+  def SWHri: BaseIntegerToFP<rmode, opcode, GPR32, FPR16, fixedpoint_recip_f16_i32, asm,
                              [(set (f16 FPR16:$Rd),
                                    (fmul (node GPR32:$Rn),
                                          fixedpoint_recip_f16_i32:$scale))]> {
@@ -5375,7 +5403,7 @@ multiclass IntegerToFP<bit isUnsigned, string asm, SDPatternOperator node> {
     let Predicates = [HasFullFP16];
   }
 
-  def SWSri: BaseIntegerToFP<isUnsigned, GPR32, FPR32, fixedpoint_recip_f32_i32, asm,
+  def SWSri: BaseIntegerToFP<rmode, opcode, GPR32, FPR32, fixedpoint_recip_f32_i32, asm,
                              [(set FPR32:$Rd,
                                    (fmul (node GPR32:$Rn),
                                          fixedpoint_recip_f32_i32:$scale))]> {
@@ -5384,7 +5412,7 @@ multiclass IntegerToFP<bit isUnsigned, string asm, SDPatternOperator node> {
     let scale{5} = 1;
   }
 
-  def SWDri: BaseIntegerToFP<isUnsigned, GPR32, FPR64, fixedpoint_recip_f64_i32, asm,
+  def SWDri: BaseIntegerToFP<rmode, opcode, GPR32, FPR64, fixedpoint_recip_f64_i32, asm,
                              [(set FPR64:$Rd,
                                    (fmul (node GPR32:$Rn),
                                          fixedpoint_recip_f64_i32:$scale))]> {
@@ -5393,7 +5421,7 @@ multiclass IntegerToFP<bit isUnsigned, string asm, SDPatternOperator node> {
     let scale{5} = 1;
   }
 
-  def SXHri: BaseIntegerToFP<isUnsigned, GPR64, FPR16, fixedpoint_recip_f16_i64, asm,
+  def SXHri: BaseIntegerToFP<rmode, opcode, GPR64, FPR16, fixedpoint_recip_f16_i64, asm,
                              [(set (f16 FPR16:$Rd),
                                    (fmul (node GPR64:$Rn),
                                          fixedpoint_recip_f16_i64:$scale))]> {
@@ -5402,7 +5430,7 @@ multiclass IntegerToFP<bit isUnsigned, string asm, SDPatternOperator node> {
     let Predicates = [HasFullFP16];
   }
 
-  def SXSri: BaseIntegerToFP<isUnsigned, GPR64, FPR32, fixedpoint_recip_f32_i64, asm,
+  def SXSri: BaseIntegerToFP<rmode, opcode, GPR64, FPR32, fixedpoint_recip_f32_i64, asm,
                              [(set FPR32:$Rd,
                                    (fmul (node GPR64:$Rn),
                                          fixedpoint_recip_f32_i64:$scale))]> {
@@ -5410,7 +5438,7 @@ multiclass IntegerToFP<bit isUnsigned, string asm, SDPatternOperator node> {
     let Inst{23-22} = 0b00; // 32-bit FPR flag
   }
 
-  def SXDri: BaseIntegerToFP<isUnsigned, GPR64, FPR64, fixedpoint_recip_f64_i64, asm,
+  def SXDri: BaseIntegerToFP<rmode, opcode, GPR64, FPR64, fixedpoint_recip_f64_i64, asm,
                              [(set FPR64:$Rd,
                                    (fmul (node GPR64:$Rn),
                                          fixedpoint_recip_f64_i64:$scale))]> {
@@ -5419,6 +5447,32 @@ multiclass IntegerToFP<bit isUnsigned, string asm, SDPatternOperator node> {
   }
 }
 
+multiclass IntegerToFPSIMDScalar<bits<2> rmode, bits<3> opcode, string asm, SDPatternOperator node = null_frag> {
+  // 32-bit to half-precision
+  def HSr: BaseIntegerToFPUnscaled<rmode, opcode, FPR32, FPR16, f16, asm, node> {
+    let Inst{31} = 0; // 32-bit FPR flag
+    let Inst{23-22} = 0b11; // 16-bit FPR flag
+  }
+
+  // 32-bit to double-precision
+  def DSr: BaseIntegerToFPUnscaled<rmode, opcode, FPR32, FPR64, f64, asm, node> {
+    let Inst{31} = 0; // 32-bit FPR flag
+    let Inst{23-22} = 0b01; // 64-bit FPR flag
+  }
+
+  // 64-bit to half-precision
+  def HDr: BaseIntegerToFPUnscaled<rmode, opcode, FPR64, FPR16, f16, asm, node> {
+    let Inst{31} = 1; // 64-bit FPR flag
+    let Inst{23-22} = 0b11; // 16-bit FPR flag
+  }
+
+  // 64-bit to single-precision
+  def SDr: BaseIntegerToFPUnscaled<rmode, opcode, FPR64, FPR32, f32, asm, node> {
+    let Inst{31} = 1; // 64-bit FPR flag
+    let Inst{23-22} = 0b00; // 32-bit FPR flag
+  }
+}
+
 //---
 // Unscaled integer <-> floating point conversion (i.e. FMOV)
 //---
@@ -13126,3 +13180,20 @@ multiclass AtomicFPStore<bit R, bits<3> op0, string asm> {
   def S : BaseAtomicFPStore<FPR32, 0b10, R, op0, asm>;
   def H : BaseAtomicFPStore<FPR16, 0b01, R, op0, asm>;
 }
+
+class BaseSIMDThreeSameVectorFP8MatrixMul<string asm, bits<2> size, string kind>
+  : BaseSIMDThreeSameVectorTied<1, 1, {size, 0}, 0b11101,
+                                V128, asm, ".16b", []> {
+  let AsmString = !strconcat(asm, "{\t$Rd", kind, ", $Rn", ".16b",
+                                    ", $Rm", ".16b", "}");
+}
+
+multiclass SIMDThreeSameVectorFP8MatrixMul<string asm>{
+    def v8f16: BaseSIMDThreeSameVectorFP8MatrixMul<asm, 0b00, ".8h">{
+      let Predicates = [HasNEON, HasF8F16MM];
+    }
+    def v4f32: BaseSIMDThreeSameVectorFP8MatrixMul<asm, 0b10, ".4s">{
+      let Predicates = [HasNEON, HasF8F32MM];
+    }
+}
+
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index fe3c8578b52a..6194de2d56b6 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -4838,6 +4838,19 @@ defm FCVTZU : FPToIntegerUnscaled<0b11, 0b001, "fcvtzu", any_fp_to_uint>;
 defm FCVTZS : FPToIntegerScaled<0b11, 0b000, "fcvtzs", any_fp_to_sint>;
 defm FCVTZU : FPToIntegerScaled<0b11, 0b001, "fcvtzu", any_fp_to_uint>;
 
+let Predicates = [HasNEON, HasFPRCVT] in{
+  defm FCVTAS : FPToIntegerSIMDScalar<0b11, 0b010, "fcvtas">;
+  defm FCVTAU : FPToIntegerSIMDScalar<0b11, 0b011, "fcvtau">;
+  defm FCVTMS : FPToIntegerSIMDScalar<0b10, 0b100, "fcvtms">;
+  defm FCVTMU : FPToIntegerSIMDScalar<0b10, 0b101, "fcvtmu">;
+  defm FCVTNS : FPToIntegerSIMDScalar<0b01, 0b010, "fcvtns">;
+  defm FCVTNU : FPToIntegerSIMDScalar<0b01, 0b011, "fcvtnu">;
+  defm FCVTPS : FPToIntegerSIMDScalar<0b10, 0b010, "fcvtps">;
+  defm FCVTPU : FPToIntegerSIMDScalar<0b10, 0b011, "fcvtpu">;
+  defm FCVTZS : FPToIntegerSIMDScalar<0b10, 0b110, "fcvtzs">;
+  defm FCVTZU : FPToIntegerSIMDScalar<0b10, 0b111, "fcvtzu">;
+}
+
 // AArch64's FCVT instructions saturate when out of range.
 multiclass FPToIntegerSatPats<SDNode to_int_sat, SDNode to_int_sat_gi, string INST> {
   let Predicates = [HasFullFP16] in {
@@ -4996,8 +5009,13 @@ def : Pat<(i64 (any_llround f64:$Rn)),
 // Scaled integer to floating point conversion instructions.
 //===----------------------------------------------------------------------===//
 
-defm SCVTF : IntegerToFP<0, "scvtf", any_sint_to_fp>;
-defm UCVTF : IntegerToFP<1, "ucvtf", any_uint_to_fp>;
+defm SCVTF : IntegerToFP<0b00, 0b010, "scvtf", any_sint_to_fp>;
+defm UCVTF : IntegerToFP<0b00, 0b011, "ucvtf", any_uint_to_fp>;
+
+let Predicates = [HasNEON, HasFPRCVT] in {
+  defm SCVTF : IntegerToFPSIMDScalar<0b11, 0b100, "scvtf">;
+  defm UCVTF : IntegerToFPSIMDScalar<0b11, 0b101, "ucvtf">;
+}
 
 def : Pat<(f16 (fdiv (f16 (any_sint_to_fp (i32 GPR32:$Rn))), fixedpoint_f16_i32:$scale)),
           (SCVTFSWHri GPR32:$Rn, fixedpoint_f16_i32:$scale)>;
@@ -10547,6 +10565,9 @@ let Predicates = [HasLSFE] in {
   def STBFMINNML : BaseAtomicFPStore<FPR16, 0b00, 0b1, 0b111, "stbfminnml">;
 }
 
+let Uses = [FPMR, FPCR] in
+defm FMMLA : SIMDThreeSameVectorFP8MatrixMul<"fmmla">;
+
 include "AArch64InstrAtomics.td"
 include "AArch64SVEInstrInfo.td"
 include "AArch64SMEInstrInfo.td"
diff --git a/llvm/test/MC/AArch64/FP8/fmmla-diagnostics.s b/llvm/test/MC/AArch64/FP8/fmmla-diagnostics.s
new file mode 100644
index 000000000000..cf8d21658124
--- /dev/null
+++ b/llvm/test/MC/AArch64/FP8/fmmla-diagnostics.s
@@ -0,0 +1,46 @@
+// RUN: not llvm-mc -triple=aarch64 -mattr=+f8f16mm,+f8f32mm  2>&1 < %s| FileCheck %s
+
+fmmla v0.4h, v1.16b, v2.16b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: fmmla v0.4h, v1.16b, v2.16b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmmla v0.8s, v1.16b, v2.16b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid vector kind qualifier
+// CHECK-NEXT: fmmla v0.8s, v1.16b, v2.16b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmmla v0.4s, v1.4s, v2.4s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: fmmla v0.4s, v1.4s, v2.4s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmmla v0.8h, v1.8h, v2.8h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: fmmla v0.8h, v1.8h, v2.8h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmmla v0.16b, v1.16b, v2.16b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: fmmla v0.16b, v1.16b, v2.16b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmmla v0.d, v1.16b, v2.16b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: fmmla v0.d, v1.16b, v2.16b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmmla v0.2d, v1.16b, v2.16b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: fmmla v0.2d, v1.16b, v2.16b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmmla v0.8h, v1.8b, v2.8b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: fmmla v0.8h, v1.8b, v2.8b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmmla v0.4s, v1.8b, v2.8b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: fmmla v0.4s, v1.8b, v2.8b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/FP8/fmmla.s b/llvm/test/MC/AArch64/FP8/fmmla.s
new file mode 100644
index 000000000000..922f4c9d918c
--- /dev/null
+++ b/llvm/test/MC/AArch64/FP8/fmmla.s
@@ -0,0 +1,25 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+f8f16mm,+f8f32mm  < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+f8f16mm,+f8f32mm  < %s \
+// RUN:        | llvm-objdump -d --mattr=+f8f16mm,+f8f32mm  - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+f8f16mm,+f8f32mm  < %s \
+// RUN:        | llvm-objdump -d  --no-print-imm-hex --mattr=-f8f16mm,-f8f32mm - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+f8f16mm,+f8f32mm  < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+f8f16mm,+f8f32mm  -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+fmmla v0.8h, v1.16b, v2.16b
+// CHECK-INST: fmmla v0.8h, v1.16b, v2.16b
+// CHECK-ENCODING: [0x20,0xec,0x02,0x6e]
+// CHECK-ERROR: instruction requires: f8f16mm
+// CHECK-UNKNOWN: 6e02ec20 <unknown>
+
+fmmla v0.4s, v1.16b, v2.16b
+// CHECK-INST: fmmla v0.4s, v1.16b, v2.16b
+// CHECK-ENCODING: [0x20,0xec,0x82,0x6e]
+// CHECK-ERROR: instruction requires: f8f32mm
+// CHECK-UNKNOWN: 6e82ec20 <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/armv9.6a-cvtf.s b/llvm/test/MC/AArch64/armv9.6a-cvtf.s
new file mode 100644
index 000000000000..6858d3896af5
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv9.6a-cvtf.s
@@ -0,0 +1,61 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+fprcvt < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+fprcvt < %s \
+// RUN:        | llvm-objdump -d --mattr=+fprcvt - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+fprcvt < %s \
+// RUN:        | llvm-objdump -d  --no-print-imm-hex --mattr=-fprcvt - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+fprcvt < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+fprcvt -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+scvtf d1, s2
+// CHECK-INST: scvtf d1, s2
+// CHECK-ENCODING: [0x41,0x00,0x7c,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1e7c0041 <unknown>
+
+scvtf h1, s2
+// CHECK-INST: scvtf h1, s2
+// CHECK-ENCODING: [0x41,0x00,0xfc,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1efc0041 <unknown>
+
+scvtf h2, d0
+// CHECK-INST: scvtf h2, d0
+// CHECK-ENCODING: [0x02,0x00,0xfc,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9efc0002 <unknown>
+
+scvtf s3, d4
+// CHECK-INST: scvtf s3, d4
+// CHECK-ENCODING: [0x83,0x00,0x3c,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9e3c0083 <unknown>
+
+ucvtf d1, s2
+// CHECK-INST: ucvtf d1, s2
+// CHECK-ENCODING: [0x41,0x00,0x7d,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1e7d0041 <unknown>
+
+ucvtf h1, s2
+// CHECK-INST: ucvtf h1, s2
+// CHECK-ENCODING: [0x41,0x00,0xfd,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1efd0041 <unknown>
+
+ucvtf h2, d0
+// CHECK-INST: ucvtf h2, d0
+// CHECK-ENCODING: [0x02,0x00,0xfd,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9efd0002 <unknown>
+
+ucvtf s3, d4
+// CHECK-INST: ucvtf s3, d4
+// CHECK-ENCODING: [0x83,0x00,0x3d,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9e3d0083 <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/armv9.6a-fcvt.s b/llvm/test/MC/AArch64/armv9.6a-fcvt.s
new file mode 100644
index 000000000000..b14ec93563f5
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv9.6a-fcvt.s
@@ -0,0 +1,253 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+fprcvt < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+fprcvt < %s \
+// RUN:        | llvm-objdump -d --mattr=+fprcvt - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+fprcvt < %s \
+// RUN:        | llvm-objdump -d  --no-print-imm-hex --mattr=-fprcvt - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+fprcvt < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+fprcvt -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+fcvtas s0, d1
+// CHECK-INST: fcvtas s0, d1
+// CHECK-ENCODING: [0x20,0x00,0x7a,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1e7a0020 <unknown>
+
+fcvtas s1, h2
+// CHECK-INST: fcvtas s1, h2
+// CHECK-ENCODING: [0x41,0x00,0xfa,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1efa0041 <unknown>
+
+fcvtas d3, h4
+// CHECK-INST: fcvtas d3, h4
+// CHECK-ENCODING: [0x83,0x00,0xfa,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9efa0083 <unknown>
+
+fcvtas d0, s5
+// CHECK-INST: fcvtas d0, s5
+// CHECK-ENCODING: [0xa0,0x00,0x3a,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9e3a00a0 <unknown>
+
+fcvtau s0, d1
+// CHECK-INST: fcvtau s0, d1
+// CHECK-ENCODING: [0x20,0x00,0x7b,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1e7b0020 <unknown>
+
+fcvtau s1, h2
+// CHECK-INST: fcvtau s1, h2
+// CHECK-ENCODING: [0x41,0x00,0xfb,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1efb0041 <unknown>
+
+fcvtau d3, h4
+// CHECK-INST: fcvtau d3, h4
+// CHECK-ENCODING: [0x83,0x00,0xfb,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9efb0083 <unknown>
+
+fcvtau d0, s5
+// CHECK-INST: fcvtau d0, s5
+// CHECK-ENCODING: [0xa0,0x00,0x3b,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9e3b00a0 <unknown>
+
+fcvtms s0, d1
+// CHECK-INST: fcvtms s0, d1
+// CHECK-ENCODING: [0x20,0x00,0x74,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1e740020 <unknown>
+
+fcvtms s1, h2
+// CHECK-INST: fcvtms s1, h2
+// CHECK-ENCODING: [0x41,0x00,0xf4,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1ef40041 <unknown>
+
+fcvtms d3, h4
+// CHECK-INST: fcvtms d3, h4
+// CHECK-ENCODING: [0x83,0x00,0xf4,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9ef40083 <unknown>
+
+fcvtms d0, s5
+// CHECK-INST: fcvtms d0, s5
+// CHECK-ENCODING: [0xa0,0x00,0x34,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9e3400a0 <unknown>
+
+fcvtmu s0, d1
+// CHECK-INST: fcvtmu s0, d1
+// CHECK-ENCODING: [0x20,0x00,0x75,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1e750020 <unknown>
+
+fcvtmu s1, h2
+// CHECK-INST: fcvtmu s1, h2
+// CHECK-ENCODING: [0x41,0x00,0xf5,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1ef50041 <unknown>
+
+fcvtmu d3, h4
+// CHECK-INST: fcvtmu d3, h4
+// CHECK-ENCODING: [0x83,0x00,0xf5,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9ef50083 <unknown>
+
+fcvtmu d0, s5
+// CHECK-INST: fcvtmu d0, s5
+// CHECK-ENCODING: [0xa0,0x00,0x35,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9e3500a0 <unknown>
+
+fcvtns s0, d1
+// CHECK-INST: fcvtns s0, d1
+// CHECK-ENCODING: [0x20,0x00,0x6a,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1e6a0020 <unknown>
+
+fcvtns s1, h2
+// CHECK-INST: fcvtns s1, h2
+// CHECK-ENCODING: [0x41,0x00,0xea,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1eea0041 <unknown>
+
+fcvtns d3, h4
+// CHECK-INST: fcvtns d3, h4
+// CHECK-ENCODING: [0x83,0x00,0xea,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9eea0083 <unknown>
+
+fcvtns d0, s5
+// CHECK-INST: fcvtns d0, s5
+// CHECK-ENCODING: [0xa0,0x00,0x2a,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9e2a00a0 <unknown>
+
+fcvtnu s0, d1
+// CHECK-INST: fcvtnu s0, d1
+// CHECK-ENCODING: [0x20,0x00,0x6b,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1e6b0020 <unknown>
+
+fcvtnu s1, h2
+// CHECK-INST: fcvtnu s1, h2
+// CHECK-ENCODING: [0x41,0x00,0xeb,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1eeb0041 <unknown>
+
+fcvtnu d3, h4
+// CHECK-INST: fcvtnu d3, h4
+// CHECK-ENCODING: [0x83,0x00,0xeb,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9eeb0083 <unknown>
+
+fcvtnu d0, s5
+// CHECK-INST: fcvtnu d0, s5
+// CHECK-ENCODING: [0xa0,0x00,0x2b,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9e2b00a0 <unknown>
+
+fcvtps s0, d1
+// CHECK-INST: fcvtps s0, d1
+// CHECK-ENCODING: [0x20,0x00,0x72,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1e720020 <unknown>
+
+fcvtps s1, h2
+// CHECK-INST: fcvtps s1, h2
+// CHECK-ENCODING: [0x41,0x00,0xf2,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1ef20041 <unknown>
+
+fcvtps d3, h4
+// CHECK-INST: fcvtps d3, h4
+// CHECK-ENCODING: [0x83,0x00,0xf2,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9ef20083 <unknown>
+
+fcvtps d0, s5
+// CHECK-INST: fcvtps d0, s5
+// CHECK-ENCODING: [0xa0,0x00,0x32,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9e3200a0 <unknown>
+
+fcvtpu s0, d1
+// CHECK-INST: fcvtpu s0, d1
+// CHECK-ENCODING: [0x20,0x00,0x73,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1e730020 <unknown>
+
+fcvtpu s1, h2
+// CHECK-INST: fcvtpu s1, h2
+// CHECK-ENCODING: [0x41,0x00,0xf3,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1ef30041 <unknown>
+
+fcvtpu d3, h4
+// CHECK-INST: fcvtpu d3, h4
+// CHECK-ENCODING: [0x83,0x00,0xf3,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9ef30083 <unknown>
+
+fcvtpu d0, s5
+// CHECK-INST: fcvtpu d0, s5
+// CHECK-ENCODING: [0xa0,0x00,0x33,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9e3300a0 <unknown>
+
+fcvtzs s0, d1
+// CHECK-INST: fcvtzs s0, d1
+// CHECK-ENCODING: [0x20,0x00,0x76,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1e760020 <unknown>
+
+fcvtzs s1, h2
+// CHECK-INST: fcvtzs s1, h2
+// CHECK-ENCODING: [0x41,0x00,0xf6,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1ef60041 <unknown>
+
+fcvtzs d3, h4
+// CHECK-INST: fcvtzs d3, h4
+// CHECK-ENCODING: [0x83,0x00,0xf6,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9ef60083 <unknown>
+
+fcvtzs d0, s5
+// CHECK-INST: fcvtzs d0, s5
+// CHECK-ENCODING: [0xa0,0x00,0x36,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9e3600a0 <unknown>
+
+fcvtzu s0, d1
+// CHECK-INST: fcvtzu s0, d1
+// CHECK-ENCODING: [0x20,0x00,0x77,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1e770020 <unknown>
+
+fcvtzu s1, h2
+// CHECK-INST: fcvtzu s1, h2
+// CHECK-ENCODING: [0x41,0x00,0xf7,0x1e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 1ef70041 <unknown>
+
+fcvtzu d3, h4
+// CHECK-INST: fcvtzu d3, h4
+// CHECK-ENCODING: [0x83,0x00,0xf7,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9ef70083 <unknown>
+
+fcvtzu d0, s5
+// CHECK-INST: fcvtzu d0, s5
+// CHECK-ENCODING: [0xa0,0x00,0x37,0x9e]
+// CHECK-ERROR: instruction requires: fprcvt
+// CHECK-UNKNOWN: 9e3700a0 <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/directive-arch-negative.s b/llvm/test/MC/AArch64/directive-arch-negative.s
index 19b48ea66bfe..4c17c5609712 100644
--- a/llvm/test/MC/AArch64/directive-arch-negative.s
+++ b/llvm/test/MC/AArch64/directive-arch-negative.s
@@ -61,3 +61,21 @@
         cbhi x5, x5, #1020
 # CHECK: error: instruction requires: cmpbr
 # CHECK-NEXT:   cbhi x5, x5, #1020
+
+	.arch armv9.6.-a+nofprcvt
+        scvtf d1, s2
+
+# CHECK: error: instruction requires: fprcvt
+# CHECK-NEXT:   scvtf d1, s2
+
+	.arch armv9.6.-a+nof8f16mm
+        fmmla v0.8h, v1.16b, v2.16b
+
+# CHECK: error: instruction requires: f8f16mm
+# CHECK-NEXT:   fmmla v0.8h, v1.16b, v2.16b
+
+	.arch armv9.6.-a+nof8f32mm
+        fmmla v0.4s, v1.16b, v2.16b
+
+# CHECK: error: instruction requires: f8f32mm
+# CHECK-NEXT:   fmmla v0.4s, v1.16b, v2.16b
diff --git a/llvm/test/MC/AArch64/directive-arch.s b/llvm/test/MC/AArch64/directive-arch.s
index 8d9c0cef7536..ba605cc5d1a6 100644
--- a/llvm/test/MC/AArch64/directive-arch.s
+++ b/llvm/test/MC/AArch64/directive-arch.s
@@ -26,3 +26,15 @@
 	cbne x5, #31, lbl
 # CHECK:        cbne x5, #31, lbl
 
+
+	.arch armv9-a+fprcvt
+	scvtf h1, s2
+# CHECK:        scvtf h1, s2
+
+	.arch armv9-a+f8f16mm
+	fmmla v0.8h, v1.16b, v2.16b
+# CHECK:        fmmla v0.8h, v1.16b, v2.16b
+
+	.arch armv9-a+f8f32mm
+	fmmla v0.4s, v1.16b, v2.16b
+# CHECK:        fmmla v0.4s, v1.16b, v2.16b
diff --git a/llvm/test/MC/AArch64/directive-arch_extension-negative.s b/llvm/test/MC/AArch64/directive-arch_extension-negative.s
index 363989d7b9b2..63da153c1a6f 100644
--- a/llvm/test/MC/AArch64/directive-arch_extension-negative.s
+++ b/llvm/test/MC/AArch64/directive-arch_extension-negative.s
@@ -237,3 +237,21 @@ cbhi x5, x5, #1020
 // CHECK: [[@LINE-1]]:1: error: instruction requires: cmpbr
 // CHECK-NEXT: cbhi x5, x5, #1020
 
+
+.arch_extension fprcvt
+.arch_extension nofprcvt
+fcvtmu s0, d1
+// CHECK: [[@LINE-1]]:1: error: instruction requires: fprcvt
+// CHECK-NEXT: fcvtmu s0, d1
+
+.arch_extension f8f16mm
+.arch_extension nof8f16mm
+fmmla v2.8h, v1.16b, v0.16b
+// CHECK: [[@LINE-1]]:1: error: instruction requires: f8f16mm
+// CHECK-NEXT: fmmla v2.8h, v1.16b, v0.16b
+
+.arch_extension f8f32mm
+.arch_extension nof8f32mm
+fmmla v2.4s, v1.16b, v0.16b
+// CHECK: [[@LINE-1]]:1: error: instruction requires: f8f32mm
+// CHECK-NEXT: fmmla v2.4s, v1.16b, v0.16b
diff --git a/llvm/test/MC/AArch64/directive-arch_extension.s b/llvm/test/MC/AArch64/directive-arch_extension.s
index 8a0e1ac471ce..b8e8696c7abb 100644
--- a/llvm/test/MC/AArch64/directive-arch_extension.s
+++ b/llvm/test/MC/AArch64/directive-arch_extension.s
@@ -189,3 +189,15 @@ msr SSBS, #1
 .arch_extension tme
 tstart x0
 // CHECK: tstart x0
+
+.arch_extension fprcvt
+fcvtns s0, d1
+// CHECK: fcvtns s0, d1
+
+.arch_extension f8f16mm
+fmmla v1.8h, v2.16b, v3.16b
+// CHECK: fmmla v1.8h, v2.16b, v3.16b
+
+.arch_extension f8f32mm
+fmmla v1.4s, v2.16b, v3.16b
+// CHECK: fmmla v1.4s, v2.16b, v3.16b
diff --git a/llvm/test/MC/AArch64/directive-cpu.s b/llvm/test/MC/AArch64/directive-cpu.s
index e3d7b1cd75e5..1a0a0bd0c513 100644
--- a/llvm/test/MC/AArch64/directive-cpu.s
+++ b/llvm/test/MC/AArch64/directive-cpu.s
@@ -39,3 +39,15 @@ sha512h q0, q1, v2.2d
 .cpu generic+sm4
 sm4e v2.4s, v15.4s
 // CHECK: sm4e  v2.4s, v15.4s
+
+.cpu generic+fprcvt
+scvtf d1, s2
+// CHECK: scvtf d1, s2
+
+.cpu generic+f8f16mm
+fmmla v0.8h, v1.16b, v2.16b
+// CHECK: fmmla v0.8h, v1.16b, v2.16b
+
+.cpu generic+f8f32mm
+fmmla v0.4s, v1.16b, v2.16b
+// CHECK: fmmla v0.4s, v1.16b, v2.16b
diff --git a/llvm/test/MC/AArch64/neon-diagnostics.s b/llvm/test/MC/AArch64/neon-diagnostics.s
index 9a0445131ddf..6863a89bbe18 100644
--- a/llvm/test/MC/AArch64/neon-diagnostics.s
+++ b/llvm/test/MC/AArch64/neon-diagnostics.s
@@ -1,4 +1,4 @@
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -mattr=+neon < %s 2> %t
+// RUN: not llvm-mc -triple aarch64-none-linux-gnu -mattr=+neon,+fprcvt < %s 2> %t
 // RUN: FileCheck --check-prefix=CHECK-ERROR < %t %s
 
 //------------------------------------------------------------------------------
@@ -5176,6 +5176,32 @@
 // CHECK-ERROR:        ucvtf d21, s14, #64
 // CHECK-ERROR:                   ^
 
+//----------------------------------------------------------------------
+// Scalar Signed Integer Convert To Floating-Point
+//---------------------------------------------------------------------
+
+    scvtf d0, h0
+    scvtf s0, h0
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR:         scvtf d0, h0
+// CHECK-ERROR:                   ^
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR:         scvtf s0, h0
+// CHECK-ERROR:                   ^
+
+//----------------------------------------------------------------------
+// Scalar Unsigned Integer Convert To Floating-Point
+//---------------------------------------------------------------------
+
+    ucvtf d0, h0
+    ucvtf s0, h0
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR:         ucvtf d0, h0
+// CHECK-ERROR:                   ^
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR:         ucvtf s0, h0
+// CHECK-ERROR:                   ^
+
 //------------------------------------------------------------------------------
 // Element reverse
 //------------------------------------------------------------------------------
@@ -6943,14 +6969,14 @@
 // With Ties To Away
 //----------------------------------------------------------------------
 
-    fcvtas s0, d0
-    fcvtas d0, s0
+    fcvtas h0, d0
+    fcvtas h0, s0
 
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtas s0, d0
+// CHECK-ERROR:        fcvtas h0, d0
 // CHECK-ERROR:                   ^
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtas d0, s0
+// CHECK-ERROR:        fcvtas h0, s0
 // CHECK-ERROR:                   ^
 
 //----------------------------------------------------------------------
@@ -6958,14 +6984,14 @@
 // Nearest With Ties To Away
 //----------------------------------------------------------------------
 
-    fcvtau s0, d0
-    fcvtau d0, s0
+    fcvtau h0, d0
+    fcvtau h0, s0
 
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtau s0, d0
+// CHECK-ERROR:        fcvtau h0, d0
 // CHECK-ERROR:                   ^
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtau d0, s0
+// CHECK-ERROR:        fcvtau h0, s0
 // CHECK-ERROR:                   ^
 
 //----------------------------------------------------------------------
@@ -6973,14 +6999,14 @@
 // Minus Infinity
 //----------------------------------------------------------------------
 
-    fcvtms s0, d0
-    fcvtms d0, s0
+    fcvtms h0, d0
+    fcvtms h0, s0
 
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtms s0, d0
+// CHECK-ERROR:        fcvtms h0, d0
 // CHECK-ERROR:                   ^
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtms d0, s0
+// CHECK-ERROR:        fcvtms h0, s0
 // CHECK-ERROR:                   ^
 
 //----------------------------------------------------------------------
@@ -6988,14 +7014,14 @@
 // Minus Infinity
 //----------------------------------------------------------------------
 
-    fcvtmu s0, d0
-    fcvtmu d0, s0
+    fcvtmu h0, d0
+    fcvtmu h0, s0
 
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtmu s0, d0
+// CHECK-ERROR:        fcvtmu h0, d0
 // CHECK-ERROR:                   ^
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtmu d0, s0
+// CHECK-ERROR:        fcvtmu h0, s0
 // CHECK-ERROR:                   ^
 
 //----------------------------------------------------------------------
@@ -7003,14 +7029,14 @@
 // With Ties To Even
 //----------------------------------------------------------------------
 
-    fcvtns s0, d0
-    fcvtns d0, s0
+    fcvtns h0, d0
+    fcvtns h0, s0
 
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtns s0, d0
+// CHECK-ERROR:        fcvtns h0, d0
 // CHECK-ERROR:                   ^
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtns d0, s0
+// CHECK-ERROR:        fcvtns h0, s0
 // CHECK-ERROR:                   ^
 
 //----------------------------------------------------------------------
@@ -7018,14 +7044,14 @@
 // Nearest With Ties To Even
 //----------------------------------------------------------------------
 
-    fcvtnu s0, d0
-    fcvtnu d0, s0
+    fcvtnu h0, d0
+    fcvtnu h0, s0
 
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtnu s0, d0
+// CHECK-ERROR:        fcvtnu h0, d0
 // CHECK-ERROR:                   ^
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtnu d0, s0
+// CHECK-ERROR:        fcvtnu h0, s0
 // CHECK-ERROR:                   ^
 
 //----------------------------------------------------------------------
@@ -7033,14 +7059,14 @@
 // Positive Infinity
 //----------------------------------------------------------------------
 
-    fcvtps s0, d0
-    fcvtps d0, s0
+    fcvtps h0, d0
+    fcvtps h0, s0
 
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtps s0, d0
+// CHECK-ERROR:        fcvtps h0, d0
 // CHECK-ERROR:                   ^
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtps d0, s0
+// CHECK-ERROR:        fcvtps h0, s0
 // CHECK-ERROR:                   ^
 
 //----------------------------------------------------------------------
@@ -7048,28 +7074,28 @@
 // Positive Infinity
 //----------------------------------------------------------------------
 
-    fcvtpu s0, d0
-    fcvtpu d0, s0
+    fcvtpu h0, d0
+    fcvtpu h0, s0
 
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtpu s0, d0
+// CHECK-ERROR:        fcvtpu h0, d0
 // CHECK-ERROR:                   ^
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtpu d0, s0
+// CHECK-ERROR:        fcvtpu h0, s0
 // CHECK-ERROR:                   ^
 
 //----------------------------------------------------------------------
 // Scalar Floating-point Convert To Signed Integer, Rounding Toward Zero
 //----------------------------------------------------------------------
 
-    fcvtzs s0, d0
-    fcvtzs d0, s0
+    fcvtzs h0, d0
+    fcvtzs h0, s0
 
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtzs s0, d0
+// CHECK-ERROR:        fcvtzs h0, d0
 // CHECK-ERROR:                   ^
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtzs d0, s0
+// CHECK-ERROR:        fcvtzs h0, s0
 // CHECK-ERROR:                   ^
 
 //----------------------------------------------------------------------
@@ -7077,14 +7103,14 @@
 // Zero
 //----------------------------------------------------------------------
 
-    fcvtzu s0, d0
-    fcvtzu d0, s0
+    fcvtzu h0, d0
+    fcvtzu h0, s0
 
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtzu s0, d0
+// CHECK-ERROR:        fcvtzu h0, d0
 // CHECK-ERROR:                   ^
 // CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR:        fcvtzu d0, s0
+// CHECK-ERROR:        fcvtzu h0, s0
 // CHECK-ERROR:                   ^
 
 //----------------------------------------------------------------------
-- 
GitLab


From ab5d3c9d359d84e454d54e8d91b5c834c42c5a47 Mon Sep 17 00:00:00 2001
From: Min-Yih Hsu <min.hsu@sifive.com>
Date: Mon, 28 Oct 2024 09:37:54 -0700
Subject: [PATCH 335/693] [RISCV] Assign different scheduling classes for
 VMADC/VMSBC (#113009)

Split the scheduling classes of VMADC/VMSBC away from that of VADC/VSBC.
Because the former are technically mask-producing instructions rather
than normal vector arithmetics, which might have different performance
characteristics on some processors.

This is effectively NFC.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoV.td      | 18 +++++++++++------
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    | 20 +++++++++----------
 llvm/lib/Target/RISCV/RISCVSchedSiFive7.td    |  3 +++
 llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td |  3 +++
 llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td |  3 +++
 llvm/lib/Target/RISCV/RISCVScheduleV.td       |  6 ++++++
 6 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
index 29759132c47d..4e8619c5ec23 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
@@ -630,31 +630,37 @@ multiclass VMRG_IV_V_X_I<string opcodestr, bits<6> funct6> {
 }
 
 multiclass VALUm_IV_V_X<string opcodestr, bits<6> funct6> {
+  // if LSB of funct6 is 1, it's a mask-producing instruction that
+  // uses a different scheduling class.
+  defvar WritePrefix = !if(funct6{0}, "WriteVICALUM", "WriteVICALU");
   def VM : VALUmVV<funct6, OPIVV, opcodestr # ".vvm">,
-           SchedBinaryMC<"WriteVICALUV", "ReadVICALUV", "ReadVICALUV">;
+           SchedBinaryMC<WritePrefix#"V", "ReadVICALUV", "ReadVICALUV">;
   def XM : VALUmVX<funct6, OPIVX, opcodestr # ".vxm">,
-           SchedBinaryMC<"WriteVICALUX", "ReadVICALUV", "ReadVICALUX">;
+           SchedBinaryMC<WritePrefix#"X", "ReadVICALUV", "ReadVICALUX">;
 }
 
 multiclass VALUm_IV_V_X_I<string opcodestr, bits<6> funct6>
     : VALUm_IV_V_X<opcodestr, funct6> {
+  // if LSB of funct6 is 1, it's a mask-producing instruction that
+  // uses a different scheduling class.
+  defvar WriteSched = !if(funct6{0}, "WriteVICALUMI", "WriteVICALUI");
   def IM : VALUmVI<funct6, opcodestr # ".vim">,
-           SchedUnaryMC<"WriteVICALUI", "ReadVICALUV">;
+           SchedUnaryMC<WriteSched, "ReadVICALUV">;
 }
 
 multiclass VALUNoVm_IV_V_X<string opcodestr, bits<6> funct6> {
   def V : VALUVVNoVm<funct6, OPIVV, opcodestr # ".vv">,
-          SchedBinaryMC<"WriteVICALUV", "ReadVICALUV", "ReadVICALUV",
+          SchedBinaryMC<"WriteVICALUMV", "ReadVICALUV", "ReadVICALUV",
                         forceMasked=0>;
   def X : VALUVXNoVm<funct6, OPIVX, opcodestr # ".vx">,
-          SchedBinaryMC<"WriteVICALUX", "ReadVICALUV", "ReadVICALUX",
+          SchedBinaryMC<"WriteVICALUMX", "ReadVICALUV", "ReadVICALUX",
                         forceMasked=0>;
 }
 
 multiclass VALUNoVm_IV_V_X_I<string opcodestr, bits<6> funct6>
    : VALUNoVm_IV_V_X<opcodestr, funct6> {
   def I : VALUVINoVm<funct6, opcodestr # ".vi">,
-          SchedUnaryMC<"WriteVICALUI", "ReadVICALUV", forceMasked=0>;
+          SchedUnaryMC<"WriteVICALUMI", "ReadVICALUV", forceMasked=0>;
 }
 
 multiclass VALU_FV_F<string opcodestr, bits<6> funct6> {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 6b308bc8c9aa..af4f653f57af 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -3072,13 +3072,13 @@ multiclass VPseudoVCALUM_VM_XM_IM {
     defvar mx = m.MX;
     defm "" : VPseudoBinaryV_VM<m, CarryOut=1, CarryIn=1, Constraint=constraint,
                                 Commutable=1, TargetConstraintType=2>,
-              SchedBinary<"WriteVICALUV", "ReadVICALUV", "ReadVICALUV", mx, forceMasked=1,
+              SchedBinary<"WriteVICALUMV", "ReadVICALUV", "ReadVICALUV", mx, forceMasked=1,
                           forcePassthruRead=true>;
     defm "" : VPseudoBinaryV_XM<m, CarryOut=1, CarryIn=1, Constraint=constraint, TargetConstraintType=2>,
-              SchedBinary<"WriteVICALUX", "ReadVICALUV", "ReadVICALUX", mx, forceMasked=1,
+              SchedBinary<"WriteVICALUMX", "ReadVICALUV", "ReadVICALUX", mx, forceMasked=1,
                           forcePassthruRead=true>;
     defm "" : VPseudoBinaryV_IM<m, CarryOut=1, CarryIn=1, Constraint=constraint, TargetConstraintType=2>,
-              SchedUnary<"WriteVICALUI", "ReadVICALUV", mx, forceMasked=1,
+              SchedUnary<"WriteVICALUMI", "ReadVICALUV", mx, forceMasked=1,
                           forcePassthruRead=true>;
   }
 }
@@ -3089,11 +3089,11 @@ multiclass VPseudoVCALUM_VM_XM {
     defvar mx = m.MX;
     defm "" : VPseudoBinaryV_VM<m, CarryOut=1, CarryIn=1, Constraint=constraint,
                                 TargetConstraintType=2>,
-              SchedBinary<"WriteVICALUV", "ReadVICALUV", "ReadVICALUV", mx, forceMasked=1,
+              SchedBinary<"WriteVICALUMV", "ReadVICALUV", "ReadVICALUV", mx, forceMasked=1,
                           forcePassthruRead=true>;
     defm "" : VPseudoBinaryV_XM<m, CarryOut=1, CarryIn=1, Constraint=constraint,
                                 TargetConstraintType=2>,
-              SchedBinary<"WriteVICALUX", "ReadVICALUV", "ReadVICALUX", mx, forceMasked=1,
+              SchedBinary<"WriteVICALUMX", "ReadVICALUV", "ReadVICALUX", mx, forceMasked=1,
                           forcePassthruRead=true>;
   }
 }
@@ -3104,13 +3104,13 @@ multiclass VPseudoVCALUM_V_X_I {
     defvar mx = m.MX;
     defm "" : VPseudoBinaryV_VM<m, CarryOut=1, CarryIn=0, Constraint=constraint,
                                 Commutable=1, TargetConstraintType=2>,
-              SchedBinary<"WriteVICALUV", "ReadVICALUV", "ReadVICALUV", mx,
+              SchedBinary<"WriteVICALUMV", "ReadVICALUV", "ReadVICALUV", mx,
                           forcePassthruRead=true>;
     defm "" : VPseudoBinaryV_XM<m, CarryOut=1, CarryIn=0, Constraint=constraint, TargetConstraintType=2>,
-              SchedBinary<"WriteVICALUX", "ReadVICALUV", "ReadVICALUX", mx,
+              SchedBinary<"WriteVICALUMX", "ReadVICALUV", "ReadVICALUX", mx,
                           forcePassthruRead=true>;
     defm "" : VPseudoBinaryV_IM<m, CarryOut=1, CarryIn=0, Constraint=constraint>,
-              SchedUnary<"WriteVICALUI", "ReadVICALUV", mx,
+              SchedUnary<"WriteVICALUMI", "ReadVICALUV", mx,
                           forcePassthruRead=true>;
   }
 }
@@ -3120,10 +3120,10 @@ multiclass VPseudoVCALUM_V_X {
   foreach m = MxList in {
     defvar mx = m.MX;
     defm "" : VPseudoBinaryV_VM<m, CarryOut=1, CarryIn=0, Constraint=constraint, TargetConstraintType=2>,
-              SchedBinary<"WriteVICALUV", "ReadVICALUV", "ReadVICALUV", mx,
+              SchedBinary<"WriteVICALUMV", "ReadVICALUV", "ReadVICALUV", mx,
                           forcePassthruRead=true>;
     defm "" : VPseudoBinaryV_XM<m, CarryOut=1, CarryIn=0, Constraint=constraint, TargetConstraintType=2>,
-              SchedBinary<"WriteVICALUX", "ReadVICALUV", "ReadVICALUX", mx,
+              SchedBinary<"WriteVICALUMX", "ReadVICALUV", "ReadVICALUX", mx,
                           forcePassthruRead=true>;
   }
 }
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index 24cbe1531c01..d07ee393bbcf 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -631,6 +631,9 @@ foreach mx = SchedMxList in {
     defm "" : LMULWriteResMX<"WriteVICALUV",    [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVICALUX",    [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVICALUI",    [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICALUMV",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICALUMX",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICALUMI",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVShiftV",    [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVShiftX",    [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVShiftI",    [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td
index 6926184e9239..7a54d2fe1080 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td
@@ -467,6 +467,9 @@ foreach mx = SchedMxList in {
     defm "" : LMULWriteResMX<"WriteVICALUV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVICALUX",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVICALUI",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICALUMV",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICALUMX",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICALUMI",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVICmpV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVICmpX",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVICmpI",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
index 59972d781a31..c685a6d2b094 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
@@ -403,6 +403,9 @@ foreach mx = SchedMxList in {
     defm "" : LMULWriteResMX<"WriteVICALUV",  [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVICALUX",  [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVICALUI",  [SiFiveP600VectorArith], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICALUMV", [SiFiveP600VectorArith], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICALUMX", [SiFiveP600VectorArith], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICALUMI", [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVICmpV",   [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVICmpX",   [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVICmpI",   [SiFiveP600VectorArith], mx, IsWorstCase>;
diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td
index ee041ea142b9..6b9f1dd32189 100644
--- a/llvm/lib/Target/RISCV/RISCVScheduleV.td
+++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td
@@ -364,6 +364,9 @@ defm "" : LMULSchedWrites<"WriteVExtV">;
 defm "" : LMULSchedWrites<"WriteVICALUV">;
 defm "" : LMULSchedWrites<"WriteVICALUX">;
 defm "" : LMULSchedWrites<"WriteVICALUI">;
+defm "" : LMULSchedWrites<"WriteVICALUMV">;
+defm "" : LMULSchedWrites<"WriteVICALUMX">;
+defm "" : LMULSchedWrites<"WriteVICALUMI">;
 // 11.6. Vector Single-Width Bit Shift Instructions
 defm "" : LMULSchedWrites<"WriteVShiftV">;
 defm "" : LMULSchedWrites<"WriteVShiftX">;
@@ -856,6 +859,9 @@ defm "" : LMULWriteRes<"WriteVExtV", []>;
 defm "" : LMULWriteRes<"WriteVICALUV", []>;
 defm "" : LMULWriteRes<"WriteVICALUX", []>;
 defm "" : LMULWriteRes<"WriteVICALUI", []>;
+defm "" : LMULWriteRes<"WriteVICALUMV", []>;
+defm "" : LMULWriteRes<"WriteVICALUMX", []>;
+defm "" : LMULWriteRes<"WriteVICALUMI", []>;
 defm "" : LMULWriteRes<"WriteVShiftV", []>;
 defm "" : LMULWriteRes<"WriteVShiftX", []>;
 defm "" : LMULWriteRes<"WriteVShiftI", []>;
-- 
GitLab


From 92412c106f5275b4b385f7c2d882008181de2854 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 28 Oct 2024 09:40:15 -0700
Subject: [PATCH 336/693] [llvm-objdump] Handle -M for --macho

--macho -d uses the `parseInputMachO` code path, which does not handle
-M. Add -M handling for --macho as well.

Close #61019

Pull Request: https://github.com/llvm/llvm-project/pull/113795
---
 .../tools/llvm-objdump/MachO/AArch64/aliases.s    | 15 +++++++++++++++
 llvm/tools/llvm-objdump/MachODump.cpp             |  4 ++++
 llvm/tools/llvm-objdump/llvm-objdump.cpp          |  2 +-
 llvm/tools/llvm-objdump/llvm-objdump.h            |  1 +
 4 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/tools/llvm-objdump/MachO/AArch64/aliases.s

diff --git a/llvm/test/tools/llvm-objdump/MachO/AArch64/aliases.s b/llvm/test/tools/llvm-objdump/MachO/AArch64/aliases.s
new file mode 100644
index 000000000000..d8b71348a8e4
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/MachO/AArch64/aliases.s
@@ -0,0 +1,15 @@
+# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %s -o %t
+# RUN: llvm-objdump --macho -d -M no-aliases %t | FileCheck %s
+# RUN: llvm-objdump --macho -d --disassembler-options=no-aliases %t | FileCheck %s
+
+# CHECK: orr w1, wzr, w2
+
+# RUN: llvm-objdump --macho -d %t | FileCheck %s --check-prefix=ALIAS
+
+# ALIAS: mov w1, w2
+
+# RUN: not llvm-objdump --macho -d -M unknown %t 2>&1 | FileCheck %s -DFILE=%t --check-prefix=ERR
+
+# ERR: error: '[[FILE]]': unrecognized disassembler option: unknown
+
+mov w1, w2
diff --git a/llvm/tools/llvm-objdump/MachODump.cpp b/llvm/tools/llvm-objdump/MachODump.cpp
index b8afb560d2ae..ab6f65cd41a3 100644
--- a/llvm/tools/llvm-objdump/MachODump.cpp
+++ b/llvm/tools/llvm-objdump/MachODump.cpp
@@ -7330,6 +7330,10 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
   // comment causing different diffs with the 'C' disassembler library API.
   // IP->setCommentStream(CommentStream);
 
+  for (StringRef Opt : DisassemblerOptions)
+    if (!IP->applyTargetSpecificCLOption(Opt))
+      reportError(Filename, "unrecognized disassembler option: " + Opt);
+
   // Set up separate thumb disassembler if needed.
   std::unique_ptr<const MCRegisterInfo> ThumbMRI;
   std::unique_ptr<const MCAsmInfo> ThumbAsmInfo;
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp
index 8073c898b8a1..86ba9193dff2 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.cpp
+++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp
@@ -305,11 +305,11 @@ bool objdump::ArchiveHeaders;
 bool objdump::Demangle;
 bool objdump::Disassemble;
 bool objdump::DisassembleAll;
+std::vector<std::string> objdump::DisassemblerOptions;
 bool objdump::SymbolDescription;
 bool objdump::TracebackTable;
 static std::vector<std::string> DisassembleSymbols;
 static bool DisassembleZeroes;
-static std::vector<std::string> DisassemblerOptions;
 static ColorOutput DisassemblyColor;
 DIDumpType objdump::DwarfDumpType;
 static bool DynamicRelocations;
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.h b/llvm/tools/llvm-objdump/llvm-objdump.h
index 7778cf6c2784..debaedd33429 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.h
+++ b/llvm/tools/llvm-objdump/llvm-objdump.h
@@ -50,6 +50,7 @@ extern DebugVarsFormat DbgVariables;
 extern bool Demangle;
 extern bool Disassemble;
 extern bool DisassembleAll;
+extern std::vector<std::string> DisassemblerOptions;
 extern DIDumpType DwarfDumpType;
 extern std::vector<std::string> FilterSections;
 extern bool LeadingAddr;
-- 
GitLab


From 6ab26eab4f1e06f2da7b3183c55666ad57f8866e Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellis.sparky.hoag@gmail.com>
Date: Mon, 28 Oct 2024 09:45:03 -0700
Subject: [PATCH 337/693] Check hasOptSize() in shouldOptimizeForSize()
 (#112626)

---
 llvm/lib/CodeGen/BranchFolding.cpp                |  7 ++-----
 llvm/lib/CodeGen/CodeGenPrepare.cpp               |  9 +++------
 llvm/lib/CodeGen/ExpandMemCmp.cpp                 |  3 +--
 llvm/lib/CodeGen/GlobalISel/Utils.cpp             |  4 +---
 llvm/lib/CodeGen/LiveIntervals.cpp                |  3 +--
 llvm/lib/CodeGen/MachineBlockPlacement.cpp        |  5 +----
 llvm/lib/CodeGen/MachineCombiner.cpp              |  6 +-----
 llvm/lib/CodeGen/MachineSizeOpts.cpp              |  8 +++++++-
 llvm/lib/CodeGen/SelectOptimize.cpp               |  4 ++--
 llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp    |  3 +--
 llvm/lib/CodeGen/TailDuplicator.cpp               |  4 +---
 llvm/lib/CodeGen/TargetLoweringBase.cpp           |  1 -
 llvm/lib/Target/X86/X86FixupBWInsts.cpp           |  3 +--
 llvm/lib/Target/X86/X86OptimizeLEAs.cpp           |  4 +---
 llvm/lib/Target/X86/X86PadShortFunction.cpp       |  4 +---
 .../lib/Transforms/IPO/FunctionSpecialization.cpp |  3 +--
 llvm/lib/Transforms/Scalar/ConstantHoisting.cpp   |  3 +--
 .../lib/Transforms/Scalar/LoopLoadElimination.cpp |  7 ++-----
 llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp    | 15 +++++----------
 llvm/lib/Transforms/Utils/SizeOpts.cpp            |  4 ++++
 .../Vectorize/LoopVectorizationLegality.cpp       |  7 ++-----
 21 files changed, 39 insertions(+), 68 deletions(-)

diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp
index 1dc278586f11..f8de13650680 100644
--- a/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/llvm/lib/CodeGen/BranchFolding.cpp
@@ -645,11 +645,8 @@ ProfitableToMerge(MachineBasicBlock *MBB1, MachineBasicBlock *MBB2,
   // we don't have to split a block.  At worst we will be introducing 1 new
   // branch instruction, which is likely to be smaller than the 2
   // instructions that would be deleted in the merge.
-  MachineFunction *MF = MBB1->getParent();
-  bool OptForSize =
-      MF->getFunction().hasOptSize() ||
-      (llvm::shouldOptimizeForSize(MBB1, PSI, &MBBFreqInfo) &&
-       llvm::shouldOptimizeForSize(MBB2, PSI, &MBBFreqInfo));
+  bool OptForSize = llvm::shouldOptimizeForSize(MBB1, PSI, &MBBFreqInfo) &&
+                    llvm::shouldOptimizeForSize(MBB2, PSI, &MBBFreqInfo);
   return EffectiveTailLen >= 2 && OptForSize &&
          (FullBlockTail1 || FullBlockTail2);
 }
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 67a359015114..5224a6c8d1a3 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -612,7 +612,6 @@ bool CodeGenPrepare::_run(Function &F) {
       // bypassSlowDivision may create new BBs, but we don't want to reapply the
       // optimization to those blocks.
       BasicBlock *Next = BB->getNextNode();
-      // F.hasOptSize is already checked in the outer if statement.
       if (!llvm::shouldOptimizeForSize(BB, PSI, BFI.get()))
         EverMadeChange |= bypassSlowDivision(BB, BypassWidths);
       BB = Next;
@@ -2608,7 +2607,7 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, ModifyDT &ModifiedDT) {
   // cold block.  This interacts with our handling for loads and stores to
   // ensure that we can fold all uses of a potential addressing computation
   // into their uses.  TODO: generalize this to work over profiling data
-  if (CI->hasFnAttr(Attribute::Cold) && !OptSize &&
+  if (CI->hasFnAttr(Attribute::Cold) &&
       !llvm::shouldOptimizeForSize(BB, PSI, BFI.get()))
     for (auto &Arg : CI->args()) {
       if (!Arg->getType()->isPointerTy())
@@ -5505,9 +5504,7 @@ static bool FindAllMemoryUses(
       if (CI->hasFnAttr(Attribute::Cold)) {
         // If this is a cold call, we can sink the addressing calculation into
         // the cold path.  See optimizeCallInst
-        bool OptForSize =
-            OptSize || llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI);
-        if (!OptForSize)
+        if (!llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI))
           continue;
       }
 
@@ -7402,7 +7399,7 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
     SelectKind = TargetLowering::ScalarValSelect;
 
   if (TLI->isSelectSupported(SelectKind) &&
-      (!isFormingBranchFromSelectProfitable(TTI, TLI, SI) || OptSize ||
+      (!isFormingBranchFromSelectProfitable(TTI, TLI, SI) ||
        llvm::shouldOptimizeForSize(SI->getParent(), PSI, BFI.get())))
     return false;
 
diff --git a/llvm/lib/CodeGen/ExpandMemCmp.cpp b/llvm/lib/CodeGen/ExpandMemCmp.cpp
index 6d626de0b4e6..1de01e402e59 100644
--- a/llvm/lib/CodeGen/ExpandMemCmp.cpp
+++ b/llvm/lib/CodeGen/ExpandMemCmp.cpp
@@ -852,8 +852,7 @@ static bool expandMemCmp(CallInst *CI, const TargetTransformInfo *TTI,
   // available load sizes.
   const bool IsUsedForZeroCmp =
       IsBCmp || isOnlyUsedInZeroEqualityComparison(CI);
-  bool OptForSize = CI->getFunction()->hasOptSize() ||
-                    llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI);
+  bool OptForSize = llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI);
   auto Options = TTI->enableMemCmpExpansion(OptForSize,
                                             IsUsedForZeroCmp);
   if (!Options) return false;
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 722ceea29c95..513a49b4fc2e 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -1621,9 +1621,7 @@ int64_t llvm::getICmpTrueVal(const TargetLowering &TLI, bool IsVector,
 
 bool llvm::shouldOptForSize(const MachineBasicBlock &MBB,
                             ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) {
-  const auto &F = MBB.getParent()->getFunction();
-  return F.hasOptSize() || F.hasMinSize() ||
-         llvm::shouldOptimizeForSize(MBB.getBasicBlock(), PSI, BFI);
+  return llvm::shouldOptimizeForSize(MBB.getBasicBlock(), PSI, BFI);
 }
 
 void llvm::saveUsesAndErase(MachineInstr &MI, MachineRegisterInfo &MRI,
diff --git a/llvm/lib/CodeGen/LiveIntervals.cpp b/llvm/lib/CodeGen/LiveIntervals.cpp
index 21a316cf99a2..a0b6bf445fa8 100644
--- a/llvm/lib/CodeGen/LiveIntervals.cpp
+++ b/llvm/lib/CodeGen/LiveIntervals.cpp
@@ -890,8 +890,7 @@ float LiveIntervals::getSpillWeight(bool isDef, bool isUse,
   const auto *MF = MBB->getParent();
   // When optimizing for size we only consider the codesize impact of spilling
   // the register, not the runtime impact.
-  if (PSI && (MF->getFunction().hasOptSize() ||
-              llvm::shouldOptimizeForSize(MF, PSI, MBFI)))
+  if (PSI && llvm::shouldOptimizeForSize(MF, PSI, MBFI))
     return Weight;
   return Weight * MBFI->getBlockFreqRelativeToEntryBlock(MBB);
 }
diff --git a/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
index dd5220b4599f..d1dced9ef28d 100644
--- a/llvm/lib/CodeGen/MachineBlockPlacement.cpp
+++ b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
@@ -2189,9 +2189,7 @@ MachineBlockPlacement::findBestLoopTop(const MachineLoop &L,
   // i.e. when the layout predecessor does not fallthrough to the loop header.
   // In practice this never happens though: there always seems to be a preheader
   // that can fallthrough and that is also placed before the header.
-  bool OptForSize = F->getFunction().hasOptSize() ||
-                    llvm::shouldOptimizeForSize(L.getHeader(), PSI, MBFI.get());
-  if (OptForSize)
+  if (llvm::shouldOptimizeForSize(L.getHeader(), PSI, MBFI.get()))
     return L.getHeader();
 
   MachineBasicBlock *OldTop = nullptr;
@@ -3511,7 +3509,6 @@ bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &MF) {
   initTailDupThreshold();
 
   const bool OptForSize =
-      MF.getFunction().hasOptSize() ||
       llvm::shouldOptimizeForSize(&MF, PSI, &MBFI->getMBFI());
   // Determine whether to use ext-tsp for perf/size optimization. The method
   // is beneficial only for instances with at least 3 basic blocks and it can be
diff --git a/llvm/lib/CodeGen/MachineCombiner.cpp b/llvm/lib/CodeGen/MachineCombiner.cpp
index 5bfc1d63ac37..141cc1f35d66 100644
--- a/llvm/lib/CodeGen/MachineCombiner.cpp
+++ b/llvm/lib/CodeGen/MachineCombiner.cpp
@@ -77,9 +77,6 @@ class MachineCombiner : public MachineFunctionPass {
 
   TargetSchedModel TSchedModel;
 
-  /// True if optimizing for code size.
-  bool OptSize = false;
-
 public:
   static char ID;
   MachineCombiner() : MachineFunctionPass(ID) {
@@ -571,7 +568,7 @@ bool MachineCombiner::combineInstructions(MachineBasicBlock *MBB) {
   SparseSet<LiveRegUnit> RegUnits;
   RegUnits.setUniverse(TRI->getNumRegUnits());
 
-  bool OptForSize = OptSize || llvm::shouldOptimizeForSize(MBB, PSI, MBFI);
+  bool OptForSize = llvm::shouldOptimizeForSize(MBB, PSI, MBFI);
 
   bool DoRegPressureReduce =
       TII->shouldReduceRegisterPressure(MBB, &RegClassInfo);
@@ -733,7 +730,6 @@ bool MachineCombiner::runOnMachineFunction(MachineFunction &MF) {
          &getAnalysis<LazyMachineBlockFrequencyInfoPass>().getBFI() :
          nullptr;
   TraceEnsemble = nullptr;
-  OptSize = MF.getFunction().hasOptSize();
   RegClassInfo.runOnMachineFunction(MF);
 
   LLVM_DEBUG(dbgs() << getPassName() << ": " << MF.getName() << '\n');
diff --git a/llvm/lib/CodeGen/MachineSizeOpts.cpp b/llvm/lib/CodeGen/MachineSizeOpts.cpp
index 53bed7397d09..4d458f2c2e24 100644
--- a/llvm/lib/CodeGen/MachineSizeOpts.cpp
+++ b/llvm/lib/CodeGen/MachineSizeOpts.cpp
@@ -28,6 +28,8 @@ bool llvm::shouldOptimizeForSize(const MachineFunction *MF,
                                  ProfileSummaryInfo *PSI,
                                  const MachineBlockFrequencyInfo *MBFI,
                                  PGSOQueryType QueryType) {
+  if (MF->getFunction().hasOptSize())
+    return true;
   return shouldFuncOptimizeForSizeImpl(MF, PSI, MBFI, QueryType);
 }
 
@@ -36,6 +38,8 @@ bool llvm::shouldOptimizeForSize(const MachineBasicBlock *MBB,
                                  const MachineBlockFrequencyInfo *MBFI,
                                  PGSOQueryType QueryType) {
   assert(MBB);
+  if (MBB->getParent()->getFunction().hasOptSize())
+    return true;
   return shouldOptimizeForSizeImpl(MBB, PSI, MBFI, QueryType);
 }
 
@@ -44,7 +48,9 @@ bool llvm::shouldOptimizeForSize(const MachineBasicBlock *MBB,
                                  MBFIWrapper *MBFIW,
                                  PGSOQueryType QueryType) {
   assert(MBB);
-  if (!PSI || !MBFIW)
+  if (MBB->getParent()->getFunction().hasOptSize())
+    return true;
+  if (!MBFIW)
     return false;
   BlockFrequency BlockFreq = MBFIW->getBlockFreq(MBB);
   return shouldOptimizeForSizeImpl(BlockFreq, PSI, &MBFIW->getMBFI(),
diff --git a/llvm/lib/CodeGen/SelectOptimize.cpp b/llvm/lib/CodeGen/SelectOptimize.cpp
index 61341e1f2d04..55b0eb71ac11 100644
--- a/llvm/lib/CodeGen/SelectOptimize.cpp
+++ b/llvm/lib/CodeGen/SelectOptimize.cpp
@@ -431,7 +431,7 @@ PreservedAnalyses SelectOptimizeImpl::run(Function &F,
   BFI = &FAM.getResult<BlockFrequencyAnalysis>(F);
 
   // When optimizing for size, selects are preferable over branches.
-  if (F.hasOptSize() || llvm::shouldOptimizeForSize(&F, PSI, BFI))
+  if (llvm::shouldOptimizeForSize(&F, PSI, BFI))
     return PreservedAnalyses::all();
 
   LI = &FAM.getResult<LoopAnalysis>(F);
@@ -467,7 +467,7 @@ bool SelectOptimizeImpl::runOnFunction(Function &F, Pass &P) {
   TSchedModel.init(TSI);
 
   // When optimizing for size, selects are preferable over branches.
-  if (F.hasOptSize() || llvm::shouldOptimizeForSize(&F, PSI, BFI))
+  if (llvm::shouldOptimizeForSize(&F, PSI, BFI))
     return false;
 
   return optimizeSelects(F);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 0770355ec18c..1a86b3b51234 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1370,8 +1370,7 @@ SelectionDAG::~SelectionDAG() {
 }
 
 bool SelectionDAG::shouldOptForSize() const {
-  return MF->getFunction().hasOptSize() ||
-      llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI);
+  return llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI);
 }
 
 void SelectionDAG::allnodes_clear() {
diff --git a/llvm/lib/CodeGen/TailDuplicator.cpp b/llvm/lib/CodeGen/TailDuplicator.cpp
index c5fa4e6211a6..3f2e1511d403 100644
--- a/llvm/lib/CodeGen/TailDuplicator.cpp
+++ b/llvm/lib/CodeGen/TailDuplicator.cpp
@@ -586,13 +586,11 @@ bool TailDuplicator::shouldTailDuplicate(bool IsSimple,
   // duplicate only one, because one branch instruction can be eliminated to
   // compensate for the duplication.
   unsigned MaxDuplicateCount;
-  bool OptForSize = MF->getFunction().hasOptSize() ||
-                    llvm::shouldOptimizeForSize(&TailBB, PSI, MBFI);
   if (TailDupSize == 0)
     MaxDuplicateCount = TailDuplicateSize;
   else
     MaxDuplicateCount = TailDupSize;
-  if (OptForSize)
+  if (llvm::shouldOptimizeForSize(&TailBB, PSI, MBFI))
     MaxDuplicateCount = 1;
 
   // If the block to be duplicated ends in an unanalyzable fallthrough, don't
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 7a28f7892cbf..cab0ed235774 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1633,7 +1633,6 @@ bool TargetLoweringBase::isSuitableForJumpTable(const SwitchInst *SI,
   // performed in findJumpTable() in SelectionDAGBuiler and
   // getEstimatedNumberOfCaseClusters() in BasicTTIImpl.
   const bool OptForSize =
-      SI->getParent()->getParent()->hasOptSize() ||
       llvm::shouldOptimizeForSize(SI->getParent(), PSI, BFI);
   const unsigned MinDensity = getMinimumJumpTableDensity(OptForSize);
   const unsigned MaxJumpTableSize = getMaximumJumpTableSize();
diff --git a/llvm/lib/Target/X86/X86FixupBWInsts.cpp b/llvm/lib/Target/X86/X86FixupBWInsts.cpp
index a0c91d4e3c3d..fe2c8fff5775 100644
--- a/llvm/lib/Target/X86/X86FixupBWInsts.cpp
+++ b/llvm/lib/Target/X86/X86FixupBWInsts.cpp
@@ -443,8 +443,7 @@ void FixupBWInstPass::processBasicBlock(MachineFunction &MF,
   // We run after PEI, so we need to AddPristinesAndCSRs.
   LiveUnits.addLiveOuts(MBB);
 
-  OptForSize = MF.getFunction().hasOptSize() ||
-               llvm::shouldOptimizeForSize(&MBB, PSI, MBFI);
+  OptForSize = llvm::shouldOptimizeForSize(&MBB, PSI, MBFI);
 
   for (MachineInstr &MI : llvm::reverse(MBB)) {
     if (MachineInstr *NewMI = tryReplaceInstr(&MI, MBB))
diff --git a/llvm/lib/Target/X86/X86OptimizeLEAs.cpp b/llvm/lib/Target/X86/X86OptimizeLEAs.cpp
index 3172896a8f60..280eaf04f23c 100644
--- a/llvm/lib/Target/X86/X86OptimizeLEAs.cpp
+++ b/llvm/lib/Target/X86/X86OptimizeLEAs.cpp
@@ -741,9 +741,7 @@ bool X86OptimizeLEAPass::runOnMachineFunction(MachineFunction &MF) {
 
     // Remove redundant address calculations. Do it only for -Os/-Oz since only
     // a code size gain is expected from this part of the pass.
-    bool OptForSize = MF.getFunction().hasOptSize() ||
-                      llvm::shouldOptimizeForSize(&MBB, PSI, MBFI);
-    if (OptForSize)
+    if (llvm::shouldOptimizeForSize(&MBB, PSI, MBFI))
       Changed |= removeRedundantAddrCalc(LEAs);
   }
 
diff --git a/llvm/lib/Target/X86/X86PadShortFunction.cpp b/llvm/lib/Target/X86/X86PadShortFunction.cpp
index bb59cee8badb..50d63e196d1d 100644
--- a/llvm/lib/Target/X86/X86PadShortFunction.cpp
+++ b/llvm/lib/Target/X86/X86PadShortFunction.cpp
@@ -132,9 +132,7 @@ bool PadShortFunc::runOnMachineFunction(MachineFunction &MF) {
     MachineBasicBlock *MBB = ReturnBB.first;
     unsigned Cycles = ReturnBB.second;
 
-    // Function::hasOptSize is already checked above.
-    bool OptForSize = llvm::shouldOptimizeForSize(MBB, PSI, MBFI);
-    if (OptForSize)
+    if (llvm::shouldOptimizeForSize(MBB, PSI, MBFI))
       continue;
 
     if (Cycles < Threshold) {
diff --git a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
index 20249a20a37e..48971e9a4635 100644
--- a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
@@ -1001,8 +1001,7 @@ bool FunctionSpecializer::isCandidateFunction(Function *F) {
     return false;
 
   // If we're optimizing the function for size, we shouldn't specialize it.
-  if (F->hasOptSize() ||
-      shouldOptimizeForSize(F, nullptr, nullptr, PGSOQueryType::IRPass))
+  if (shouldOptimizeForSize(F, nullptr, nullptr, PGSOQueryType::IRPass))
     return false;
 
   // Exit if the function is not executable. There's no point in specializing
diff --git a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
index 4a6dedc93d30..9b913e5c2a04 100644
--- a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
@@ -953,8 +953,7 @@ bool ConstantHoistingPass::runImpl(Function &Fn, TargetTransformInfo &TTI,
   this->Ctx = &Fn.getContext();
   this->Entry = &Entry;
   this->PSI = PSI;
-  this->OptForSize = Entry.getParent()->hasOptSize() ||
-                     llvm::shouldOptimizeForSize(Entry.getParent(), PSI, BFI,
+  this->OptForSize = llvm::shouldOptimizeForSize(Entry.getParent(), PSI, BFI,
                                                  PGSOQueryType::IRPass);
 
   // Collect all constant candidates.
diff --git a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
index db82f75bad5f..9b4a19106d39 100644
--- a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
@@ -586,11 +586,8 @@ public:
       }
 
       auto *HeaderBB = L->getHeader();
-      auto *F = HeaderBB->getParent();
-      bool OptForSize = F->hasOptSize() ||
-                        llvm::shouldOptimizeForSize(HeaderBB, PSI, BFI,
-                                                    PGSOQueryType::IRPass);
-      if (OptForSize) {
+      if (llvm::shouldOptimizeForSize(HeaderBB, PSI, BFI,
+                                      PGSOQueryType::IRPass)) {
         LLVM_DEBUG(
             dbgs() << "Versioning is needed but not allowed when optimizing "
                       "for size.\n");
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 79e91ad097cf..d85e0d994660 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -1413,8 +1413,7 @@ Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilderBase &B) {
     return nullptr;
   }
 
-  bool OptForSize = CI->getFunction()->hasOptSize() ||
-                    llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI,
+  bool OptForSize = llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI,
                                                 PGSOQueryType::IRPass);
 
   // If the char is variable but the input str and length are not we can turn
@@ -3482,10 +3481,8 @@ Value *LibCallSimplifier::optimizeSPrintFString(CallInst *CI,
       return B.CreateIntCast(PtrDiff, CI->getType(), false);
     }
 
-    bool OptForSize = CI->getFunction()->hasOptSize() ||
-                      llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI,
-                                                  PGSOQueryType::IRPass);
-    if (OptForSize)
+    if (llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI,
+                                    PGSOQueryType::IRPass))
       return nullptr;
 
     Value *Len = emitStrLen(CI->getArgOperand(2), B, DL, TLI);
@@ -3795,10 +3792,8 @@ Value *LibCallSimplifier::optimizeFPuts(CallInst *CI, IRBuilderBase &B) {
 
   // Don't rewrite fputs to fwrite when optimising for size because fwrite
   // requires more arguments and thus extra MOVs are required.
-  bool OptForSize = CI->getFunction()->hasOptSize() ||
-                    llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI,
-                                                PGSOQueryType::IRPass);
-  if (OptForSize)
+  if (llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI,
+                                  PGSOQueryType::IRPass))
     return nullptr;
 
   // We can't optimize if return value is used.
diff --git a/llvm/lib/Transforms/Utils/SizeOpts.cpp b/llvm/lib/Transforms/Utils/SizeOpts.cpp
index 09c4c1c3c511..7c95e7e6b996 100644
--- a/llvm/lib/Transforms/Utils/SizeOpts.cpp
+++ b/llvm/lib/Transforms/Utils/SizeOpts.cpp
@@ -99,6 +99,8 @@ struct BasicBlockBFIAdapter {
 bool llvm::shouldOptimizeForSize(const Function *F, ProfileSummaryInfo *PSI,
                                  BlockFrequencyInfo *BFI,
                                  PGSOQueryType QueryType) {
+  if (F->hasOptSize())
+    return true;
   return shouldFuncOptimizeForSizeImpl(F, PSI, BFI, QueryType);
 }
 
@@ -106,5 +108,7 @@ bool llvm::shouldOptimizeForSize(const BasicBlock *BB, ProfileSummaryInfo *PSI,
                                  BlockFrequencyInfo *BFI,
                                  PGSOQueryType QueryType) {
   assert(BB);
+  if (BB->getParent()->hasOptSize())
+    return true;
   return shouldOptimizeForSizeImpl(BB, PSI, BFI, QueryType);
 }
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 43be72f0f34d..f1568781252c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -460,11 +460,8 @@ int LoopVectorizationLegality::isConsecutivePtr(Type *AccessTy,
   const auto &Strides =
     LAI ? LAI->getSymbolicStrides() : DenseMap<Value *, const SCEV *>();
 
-  Function *F = TheLoop->getHeader()->getParent();
-  bool OptForSize = F->hasOptSize() ||
-                    llvm::shouldOptimizeForSize(TheLoop->getHeader(), PSI, BFI,
-                                                PGSOQueryType::IRPass);
-  bool CanAddPredicate = !OptForSize;
+  bool CanAddPredicate = !llvm::shouldOptimizeForSize(
+      TheLoop->getHeader(), PSI, BFI, PGSOQueryType::IRPass);
   int Stride = getPtrStride(PSE, AccessTy, Ptr, TheLoop, Strides,
                             CanAddPredicate, false).value_or(0);
   if (Stride == 1 || Stride == -1)
-- 
GitLab


From 6827a00d4dfe1fa7d479337b6192602744055686 Mon Sep 17 00:00:00 2001
From: SharonXSharon <xiaoranxu.nju@gmail.com>
Date: Mon, 28 Oct 2024 09:47:21 -0700
Subject: [PATCH 338/693] [lld][InstrProf] Do not use cstring offset hashes in
 function order for compression (#113606)

---
 lld/MachO/BPSectionOrderer.cpp | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/lld/MachO/BPSectionOrderer.cpp b/lld/MachO/BPSectionOrderer.cpp
index 07b44d48d659..5db2242a35ef 100644
--- a/lld/MachO/BPSectionOrderer.cpp
+++ b/lld/MachO/BPSectionOrderer.cpp
@@ -53,11 +53,8 @@ getRelocHash(const Reloc &reloc,
     kind = ("Section " + Twine(static_cast<uint8_t>(isec->kind()))).str();
   if (auto *sym = reloc.referent.dyn_cast<Symbol *>()) {
     kind += (" Symbol " + Twine(static_cast<uint8_t>(sym->kind()))).str();
-    if (auto *d = dyn_cast<Defined>(sym)) {
-      if (isa_and_nonnull<CStringInputSection>(isec))
-        return getRelocHash(kind, 0, isec->getOffset(d->value), reloc.addend);
+    if (auto *d = dyn_cast<Defined>(sym))
       return getRelocHash(kind, sectionIdx.value_or(0), d->value, reloc.addend);
-    }
   }
   return getRelocHash(kind, sectionIdx.value_or(0), 0, reloc.addend);
 }
-- 
GitLab


From af7c58b7ea853ef34462ce97739203e2da3c5894 Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Mon, 28 Oct 2024 12:48:42 -0400
Subject: [PATCH 339/693] Remove support for RenderScript (#112916)

See
https://discourse.llvm.org/t/rfc-deprecate-and-eventually-remove-renderscript-support/81284
for the RFC
---
 clang/docs/ReleaseNotes.rst                   |   6 +
 clang/include/clang/Basic/Attr.td             |   9 --
 clang/include/clang/Basic/AttrDocs.td         |  15 --
 clang/include/clang/Basic/LangOptions.def     |   1 -
 clang/include/clang/Basic/LangStandard.h      |   1 -
 clang/include/clang/Basic/TargetInfo.h        |   6 -
 clang/include/clang/Driver/Options.td         |   5 +-
 clang/include/clang/Driver/Types.def          |   1 -
 clang/lib/Basic/LangOptions.cpp               |   2 -
 clang/lib/Basic/LangStandards.cpp             |   4 -
 clang/lib/Basic/TargetInfo.cpp                |   1 -
 clang/lib/Basic/Targets.cpp                   |   6 -
 clang/lib/Basic/Targets/AArch64.cpp           |  16 --
 clang/lib/Basic/Targets/AArch64.h             |  11 --
 clang/lib/Basic/Targets/ARM.cpp               |  16 --
 clang/lib/Basic/Targets/ARM.h                 |  11 --
 clang/lib/CodeGen/ABIInfoImpl.cpp             |  10 --
 clang/lib/CodeGen/ABIInfoImpl.h               |  17 ---
 clang/lib/CodeGen/CGDebugInfo.cpp             |   2 -
 clang/lib/CodeGen/Targets/AArch64.cpp         |  11 --
 clang/lib/CodeGen/Targets/ARM.cpp             |  11 --
 clang/lib/Driver/Types.cpp                    |   2 -
 .../Serialization/SymbolGraphSerializer.cpp   |   1 -
 clang/lib/Frontend/ASTUnit.cpp                |   2 -
 clang/lib/Frontend/CompilerInvocation.cpp     |   7 -
 clang/lib/Frontend/FrontendActions.cpp        |   1 -
 .../CodeGen/debug-info-renderscript-tag.rs    |   3 -
 clang/test/CodeGen/fp16-ops.c                 |   2 -
 clang/test/CodeGen/renderscript.c             | 140 ------------------
 clang/test/Driver/renderscript.rs             |   3 -
 clang/test/Driver/unknown-std.c               |   1 -
 ...a-attribute-supported-attributes-list.test |   1 -
 .../predefined-macros-no-warnings.c           |   2 -
 clang/test/Sema/renderscript.rs               |  25 ----
 clang/www/index.html                          |   4 +-
 35 files changed, 10 insertions(+), 346 deletions(-)
 delete mode 100644 clang/test/CodeGen/debug-info-renderscript-tag.rs
 delete mode 100644 clang/test/CodeGen/renderscript.c
 delete mode 100644 clang/test/Driver/renderscript.rs
 delete mode 100644 clang/test/Sema/renderscript.rs

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 920a2369f964..9515e96ffd01 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -46,6 +46,12 @@ code bases.
 
 - The ``clang-rename`` tool has been removed.
 
+- Removed support for RenderScript targets. This technology is
+  `officially deprecated <https://developer.android.com/guide/topics/renderscript/compute>`_
+  and users are encouraged to
+  `migrate to Vulkan <https://developer.android.com/guide/topics/renderscript/migrate>`_
+  or other options.
+
 C/C++ Language Potentially Breaking Changes
 -------------------------------------------
 
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 0259b6e40ca9..47c93b48175f 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -411,7 +411,6 @@ def SYCL : LangOpt<"SYCLIsDevice">;
 def COnly : LangOpt<"", "!LangOpts.CPlusPlus">;
 def CPlusPlus : LangOpt<"CPlusPlus">;
 def OpenCL : LangOpt<"OpenCL">;
-def RenderScript : LangOpt<"RenderScript">;
 def ObjC : LangOpt<"ObjC">;
 def BlocksSupported : LangOpt<"Blocks">;
 def ObjCAutoRefCount : LangOpt<"ObjCAutoRefCount">;
@@ -1629,14 +1628,6 @@ def OpenCLNoSVM : Attr {
   let ASTNode = 0;
 }
 
-def RenderScriptKernel : Attr {
-  let Spellings = [GNU<"kernel">];
-  let Subjects = SubjectList<[Function]>;
-  let Documentation = [RenderScriptKernelAttributeDocs];
-  let LangOpts = [RenderScript];
-  let SimpleHandler = 1;
-}
-
 def Deprecated : InheritableAttr {
   let Spellings = [GCC<"deprecated">, Declspec<"deprecated">,
                    CXX11<"","deprecated", 201309>,
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index ee8126cadae2..7a130c434e73 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -5831,21 +5831,6 @@ provided with the regular ``visibility`` attribute.
   }];
 }
 
-def RenderScriptKernelAttributeDocs : Documentation {
-  let Category = DocCatFunction;
-  let Content = [{
-``__attribute__((kernel))`` is used to mark a ``kernel`` function in
-RenderScript.
-
-In RenderScript, ``kernel`` functions are used to express data-parallel
-computations. The RenderScript runtime efficiently parallelizes ``kernel``
-functions to run on computational resources such as multi-core CPUs and GPUs.
-See the RenderScript_ documentation for more information.
-
-.. _RenderScript: https://developer.android.com/guide/topics/renderscript/compute.html
-  }];
-}
-
 def XRayDocs : Documentation {
   let Category = DocCatFunction;
   let Heading = "xray_always_instrument, xray_never_instrument, xray_log_args";
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 68db400c22e6..942fc557c5b9 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -280,7 +280,6 @@ LANGOPT(OpenMPNoNestedParallelism  , 1, 0, "Assume that no thread in a parallel
 LANGOPT(OpenMPOffloadMandatory  , 1, 0, "Assert that offloading is mandatory and do not create a host fallback.")
 LANGOPT(OpenMPForceUSM     , 1, 0, "Enable OpenMP unified shared memory mode via compiler.")
 LANGOPT(NoGPULib  , 1, 0, "Indicate a build without the standard GPU libraries.")
-LANGOPT(RenderScript      , 1, 0, "RenderScript")
 
 LANGOPT(HLSL, 1, 0, "HLSL")
 ENUM_LANGOPT(HLSLVersion, HLSLLangStd, 16, HLSL_Unset, "HLSL Version")
diff --git a/clang/include/clang/Basic/LangStandard.h b/clang/include/clang/Basic/LangStandard.h
index 56a0d2c95e2b..49412232c9c5 100644
--- a/clang/include/clang/Basic/LangStandard.h
+++ b/clang/include/clang/Basic/LangStandard.h
@@ -39,7 +39,6 @@ enum class Language : uint8_t {
   OpenCL,
   OpenCLCXX,
   CUDA,
-  RenderScript,
   HIP,
   HLSL,
   ///@}
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index 17262d5968b1..25eda907d20a 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -262,9 +262,6 @@ protected:
   LLVM_PREFERRED_TYPE(bool)
   unsigned HasBuiltinMSVaList : 1;
 
-  LLVM_PREFERRED_TYPE(bool)
-  unsigned IsRenderScriptTarget : 1;
-
   LLVM_PREFERRED_TYPE(bool)
   unsigned HasAArch64SVETypes : 1;
 
@@ -1031,9 +1028,6 @@ public:
   /// available on this target.
   bool hasBuiltinMSVaList() const { return HasBuiltinMSVaList; }
 
-  /// Returns true for RenderScript.
-  bool isRenderScriptTarget() const { return IsRenderScriptTarget; }
-
   /// Returns whether or not the AArch64 SVE built-in types are
   /// available on this target.
   bool hasAArch64SVETypes() const { return HasAArch64SVETypes; }
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index f86e90e14c47..69ae0553507b 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -627,7 +627,6 @@ defvar c23 = LangOpts<"C23">;
 defvar lang_std = LangOpts<"LangStd">;
 defvar open_cl = LangOpts<"OpenCL">;
 defvar cuda = LangOpts<"CUDA">;
-defvar render_script = LangOpts<"RenderScript">;
 defvar hip = LangOpts<"HIP">;
 defvar gnu_mode = LangOpts<"GNUMode">;
 defvar asm_preprocessor = LangOpts<"AsmPreprocessor">;
@@ -8118,11 +8117,11 @@ def vtordisp_mode_EQ : Joined<["-"], "vtordisp-mode=">,
 def fnative_half_type: Flag<["-"], "fnative-half-type">,
   HelpText<"Use the native half type for __fp16 instead of promoting to float">,
   MarshallingInfoFlag<LangOpts<"NativeHalfType">>,
-  ImpliedByAnyOf<[open_cl.KeyPath, render_script.KeyPath]>;
+  ImpliedByAnyOf<[open_cl.KeyPath]>;
 def fnative_half_arguments_and_returns : Flag<["-"], "fnative-half-arguments-and-returns">,
   HelpText<"Use the native __fp16 type for arguments and returns (and skip ABI-specific lowering)">,
   MarshallingInfoFlag<LangOpts<"NativeHalfArgsAndReturns">>,
-  ImpliedByAnyOf<[open_cl.KeyPath, render_script.KeyPath, hlsl.KeyPath, hip.KeyPath]>;
+  ImpliedByAnyOf<[open_cl.KeyPath, hlsl.KeyPath, hip.KeyPath]>;
 def fdefault_calling_conv_EQ : Joined<["-"], "fdefault-calling-conv=">,
   HelpText<"Set default calling convention">,
   Values<"cdecl,fastcall,stdcall,vectorcall,regcall,rtdcall">,
diff --git a/clang/include/clang/Driver/Types.def b/clang/include/clang/Driver/Types.def
index af186c5df692..214c5e7a789f 100644
--- a/clang/include/clang/Driver/Types.def
+++ b/clang/include/clang/Driver/Types.def
@@ -55,7 +55,6 @@ TYPE("c++",                      CXX,          PP_CXX,          "cpp",    phases
 TYPE("objective-c++-cpp-output", PP_ObjCXX,    INVALID,         "mii",    phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("objc++-cpp-output",        PP_ObjCXX_Alias, INVALID,      "mii",    phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("objective-c++",            ObjCXX,       PP_ObjCXX,       "mm",     phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
-TYPE("renderscript",             RenderScript, PP_C,            "rs",     phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("hlsl",                     HLSL,         PP_CXX,          "hlsl",   phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble)
 
 // C family input files to precompile.
diff --git a/clang/lib/Basic/LangOptions.cpp b/clang/lib/Basic/LangOptions.cpp
index da3216ae03af..94caf6a3897b 100644
--- a/clang/lib/Basic/LangOptions.cpp
+++ b/clang/lib/Basic/LangOptions.cpp
@@ -203,8 +203,6 @@ void LangOptions::setLangDefaults(LangOptions &Opts, Language Lang,
     Opts.setDefaultFPContractMode(LangOptions::FPM_Fast);
   }
 
-  Opts.RenderScript = Lang == Language::RenderScript;
-
   // OpenCL, C++ and C23 have bool, true, false keywords.
   Opts.Bool = Opts.OpenCL || Opts.CPlusPlus || Opts.C23;
 
diff --git a/clang/lib/Basic/LangStandards.cpp b/clang/lib/Basic/LangStandards.cpp
index 214567a53efe..c49d095018b2 100644
--- a/clang/lib/Basic/LangStandards.cpp
+++ b/clang/lib/Basic/LangStandards.cpp
@@ -37,8 +37,6 @@ StringRef clang::languageToString(Language L) {
     return "OpenCLC++";
   case Language::CUDA:
     return "CUDA";
-  case Language::RenderScript:
-    return "RenderScript";
   case Language::HIP:
     return "HIP";
   case Language::HLSL:
@@ -114,8 +112,6 @@ LangStandard::Kind clang::getDefaultLanguageStandard(clang::Language Lang,
   case Language::CUDA:
   case Language::HIP:
     return LangStandard::lang_gnucxx17;
-  case Language::RenderScript:
-    return LangStandard::lang_c99;
   case Language::HLSL:
     return LangStandard::lang_hlsl202x;
   }
diff --git a/clang/lib/Basic/TargetInfo.cpp b/clang/lib/Basic/TargetInfo.cpp
index 145ca545854d..86befb1cbc74 100644
--- a/clang/lib/Basic/TargetInfo.cpp
+++ b/clang/lib/Basic/TargetInfo.cpp
@@ -154,7 +154,6 @@ TargetInfo::TargetInfo(const llvm::Triple &T) : Triple(T) {
   SSERegParmMax = 0;
   HasAlignMac68kSupport = false;
   HasBuiltinMSVaList = false;
-  IsRenderScriptTarget = false;
   HasAArch64SVETypes = false;
   HasRISCVVTypes = false;
   AllowAMDGPUUnsafeFPAtomics = false;
diff --git a/clang/lib/Basic/Targets.cpp b/clang/lib/Basic/Targets.cpp
index 4917ef015941..0021d33c45d7 100644
--- a/clang/lib/Basic/Targets.cpp
+++ b/clang/lib/Basic/Targets.cpp
@@ -710,12 +710,6 @@ std::unique_ptr<TargetInfo> AllocateTarget(const llvm::Triple &Triple,
 
   case llvm::Triple::dxil:
     return std::make_unique<DirectXTargetInfo>(Triple, Opts);
-  case llvm::Triple::renderscript32:
-    return std::make_unique<LinuxTargetInfo<RenderScript32TargetInfo>>(Triple,
-                                                                       Opts);
-  case llvm::Triple::renderscript64:
-    return std::make_unique<LinuxTargetInfo<RenderScript64TargetInfo>>(Triple,
-                                                                       Opts);
 
   case llvm::Triple::ve:
     return std::make_unique<LinuxTargetInfo<VETargetInfo>>(Triple, Opts);
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 3dbba2b4d25b..a0f94d5d3154 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -1723,19 +1723,3 @@ TargetInfo::BuiltinVaListKind
 DarwinAArch64TargetInfo::getBuiltinVaListKind() const {
   return TargetInfo::CharPtrBuiltinVaList;
 }
-
-// 64-bit RenderScript is aarch64
-RenderScript64TargetInfo::RenderScript64TargetInfo(const llvm::Triple &Triple,
-                                                   const TargetOptions &Opts)
-    : AArch64leTargetInfo(llvm::Triple("aarch64", Triple.getVendorName(),
-                                       Triple.getOSName(),
-                                       Triple.getEnvironmentName()),
-                          Opts) {
-  IsRenderScriptTarget = true;
-}
-
-void RenderScript64TargetInfo::getTargetDefines(const LangOptions &Opts,
-                                                MacroBuilder &Builder) const {
-  Builder.defineMacro("__RENDERSCRIPT__");
-  AArch64leTargetInfo::getTargetDefines(Opts, Builder);
-}
diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h
index 16a02e102e04..ea3e4015d842 100644
--- a/clang/lib/Basic/Targets/AArch64.h
+++ b/clang/lib/Basic/Targets/AArch64.h
@@ -319,17 +319,6 @@ public:
                     MacroBuilder &Builder) const override;
 };
 
-// 64-bit RenderScript is aarch64
-class LLVM_LIBRARY_VISIBILITY RenderScript64TargetInfo
-    : public AArch64leTargetInfo {
-public:
-  RenderScript64TargetInfo(const llvm::Triple &Triple,
-                           const TargetOptions &Opts);
-
-  void getTargetDefines(const LangOptions &Opts,
-                        MacroBuilder &Builder) const override;
-};
-
 } // namespace targets
 } // namespace clang
 
diff --git a/clang/lib/Basic/Targets/ARM.cpp b/clang/lib/Basic/Targets/ARM.cpp
index c87300bf2d60..370444057b42 100644
--- a/clang/lib/Basic/Targets/ARM.cpp
+++ b/clang/lib/Basic/Targets/ARM.cpp
@@ -1498,19 +1498,3 @@ void DarwinARMTargetInfo::getOSDefines(const LangOptions &Opts,
                                        MacroBuilder &Builder) const {
   getDarwinDefines(Builder, Opts, Triple, PlatformName, PlatformMinVersion);
 }
-
-RenderScript32TargetInfo::RenderScript32TargetInfo(const llvm::Triple &Triple,
-                                                   const TargetOptions &Opts)
-    : ARMleTargetInfo(llvm::Triple("armv7", Triple.getVendorName(),
-                                   Triple.getOSName(),
-                                   Triple.getEnvironmentName()),
-                      Opts) {
-  IsRenderScriptTarget = true;
-  LongWidth = LongAlign = 64;
-}
-
-void RenderScript32TargetInfo::getTargetDefines(const LangOptions &Opts,
-                                                MacroBuilder &Builder) const {
-  Builder.defineMacro("__RENDERSCRIPT__");
-  ARMleTargetInfo::getTargetDefines(Opts, Builder);
-}
diff --git a/clang/lib/Basic/Targets/ARM.h b/clang/lib/Basic/Targets/ARM.h
index df9855a52e61..55ecb99d82d8 100644
--- a/clang/lib/Basic/Targets/ARM.h
+++ b/clang/lib/Basic/Targets/ARM.h
@@ -310,17 +310,6 @@ public:
   DarwinARMTargetInfo(const llvm::Triple &Triple, const TargetOptions &Opts);
 };
 
-// 32-bit RenderScript is armv7 with width and align of 'long' set to 8-bytes
-class LLVM_LIBRARY_VISIBILITY RenderScript32TargetInfo
-    : public ARMleTargetInfo {
-public:
-  RenderScript32TargetInfo(const llvm::Triple &Triple,
-                           const TargetOptions &Opts);
-
-  void getTargetDefines(const LangOptions &Opts,
-                        MacroBuilder &Builder) const override;
-};
-
 } // namespace targets
 } // namespace clang
 
diff --git a/clang/lib/CodeGen/ABIInfoImpl.cpp b/clang/lib/CodeGen/ABIInfoImpl.cpp
index be91b85e3a81..79300df15d0e 100644
--- a/clang/lib/CodeGen/ABIInfoImpl.cpp
+++ b/clang/lib/CodeGen/ABIInfoImpl.cpp
@@ -80,16 +80,6 @@ RValue DefaultABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
       Slot);
 }
 
-ABIArgInfo CodeGen::coerceToIntArray(QualType Ty, ASTContext &Context,
-                                     llvm::LLVMContext &LLVMContext) {
-  // Alignment and Size are measured in bits.
-  const uint64_t Size = Context.getTypeSize(Ty);
-  const uint64_t Alignment = Context.getTypeAlign(Ty);
-  llvm::Type *IntType = llvm::Type::getIntNTy(LLVMContext, Alignment);
-  const uint64_t NumElements = (Size + Alignment - 1) / Alignment;
-  return ABIArgInfo::getDirect(llvm::ArrayType::get(IntType, NumElements));
-}
-
 void CodeGen::AssignToArrayRange(CodeGen::CGBuilderTy &Builder,
                                  llvm::Value *Array, llvm::Value *Value,
                                  unsigned FirstIndex, unsigned LastIndex) {
diff --git a/clang/lib/CodeGen/ABIInfoImpl.h b/clang/lib/CodeGen/ABIInfoImpl.h
index 2a3ef6b8a6c9..d9d79c6a55dd 100644
--- a/clang/lib/CodeGen/ABIInfoImpl.h
+++ b/clang/lib/CodeGen/ABIInfoImpl.h
@@ -33,23 +33,6 @@ public:
                    AggValueSlot Slot) const override;
 };
 
-// Helper for coercing an aggregate argument or return value into an integer
-// array of the same size (including padding) and alignment.  This alternate
-// coercion happens only for the RenderScript ABI and can be removed after
-// runtimes that rely on it are no longer supported.
-//
-// RenderScript assumes that the size of the argument / return value in the IR
-// is the same as the size of the corresponding qualified type. This helper
-// coerces the aggregate type into an array of the same size (including
-// padding).  This coercion is used in lieu of expansion of struct members or
-// other canonical coercions that return a coerced-type of larger size.
-//
-// Ty          - The argument / return value type
-// Context     - The associated ASTContext
-// LLVMContext - The associated LLVMContext
-ABIArgInfo coerceToIntArray(QualType Ty, ASTContext &Context,
-                            llvm::LLVMContext &LLVMContext);
-
 void AssignToArrayRange(CodeGen::CGBuilderTy &Builder, llvm::Value *Array,
                         llvm::Value *Value, unsigned FirstIndex,
                         unsigned LastIndex);
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 5fd6cfa63e6e..ad64abe7cd40 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -624,8 +624,6 @@ void CGDebugInfo::CreateCompileUnit() {
   } else if (LO.OpenCL && (!CGM.getCodeGenOpts().DebugStrictDwarf ||
                            CGM.getCodeGenOpts().DwarfVersion >= 5)) {
     LangTag = llvm::dwarf::DW_LANG_OpenCL;
-  } else if (LO.RenderScript) {
-    LangTag = llvm::dwarf::DW_LANG_GOOGLE_RenderScript;
   } else if (LO.C11 && !(CGO.DebugStrictDwarf && CGO.DwarfVersion < 5)) {
       LangTag = llvm::dwarf::DW_LANG_C11;
   } else if (LO.C99) {
diff --git a/clang/lib/CodeGen/Targets/AArch64.cpp b/clang/lib/CodeGen/Targets/AArch64.cpp
index a80411971b60..9320c6ef06ef 100644
--- a/clang/lib/CodeGen/Targets/AArch64.cpp
+++ b/clang/lib/CodeGen/Targets/AArch64.cpp
@@ -462,11 +462,6 @@ ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn,
 
   // Aggregates <= 16 bytes are passed directly in registers or on the stack.
   if (Size <= 128) {
-    // On RenderScript, coerce Aggregates <= 16 bytes to an integer array of
-    // same size and alignment.
-    if (getTarget().isRenderScriptTarget()) {
-      return coerceToIntArray(Ty, getContext(), getVMContext());
-    }
     unsigned Alignment;
     if (Kind == AArch64ABIKind::AAPCS) {
       Alignment = getContext().getTypeUnadjustedAlign(Ty);
@@ -548,12 +543,6 @@ ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,
 
   // Aggregates <= 16 bytes are returned directly in registers or on the stack.
   if (Size <= 128) {
-    // On RenderScript, coerce Aggregates <= 16 bytes to an integer array of
-    // same size and alignment.
-    if (getTarget().isRenderScriptTarget()) {
-      return coerceToIntArray(RetTy, getContext(), getVMContext());
-    }
-
     if (Size <= 64 && getDataLayout().isLittleEndian()) {
       // Composite types are returned in lower bits of a 64-bit register for LE,
       // and in higher bits for BE. However, integer types are always returned
diff --git a/clang/lib/CodeGen/Targets/ARM.cpp b/clang/lib/CodeGen/Targets/ARM.cpp
index 49ac1a76e767..2d858fa2f3c3 100644
--- a/clang/lib/CodeGen/Targets/ARM.cpp
+++ b/clang/lib/CodeGen/Targets/ARM.cpp
@@ -420,12 +420,6 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic,
                                    /*Realign=*/TyAlign > ABIAlign);
   }
 
-  // On RenderScript, coerce Aggregates <= 64 bytes to an integer array of
-  // same size and alignment.
-  if (getTarget().isRenderScriptTarget()) {
-    return coerceToIntArray(Ty, getContext(), getVMContext());
-  }
-
   // Otherwise, pass by coercing to a structure of the appropriate size.
   llvm::Type* ElemTy;
   unsigned SizeRegs;
@@ -609,11 +603,6 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, bool isVariadic,
   // are returned indirectly.
   uint64_t Size = getContext().getTypeSize(RetTy);
   if (Size <= 32) {
-    // On RenderScript, coerce Aggregates <= 4 bytes to an integer array of
-    // same size and alignment.
-    if (getTarget().isRenderScriptTarget()) {
-      return coerceToIntArray(RetTy, getContext(), getVMContext());
-    }
     if (getDataLayout().isBigEndian())
       // Return in 32 bit integer integer type (as if loaded by LDR, AAPCS 5.4)
       return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
diff --git a/clang/lib/Driver/Types.cpp b/clang/lib/Driver/Types.cpp
index 3de45b00b4d0..b14aac0f0ce0 100644
--- a/clang/lib/Driver/Types.cpp
+++ b/clang/lib/Driver/Types.cpp
@@ -201,7 +201,6 @@ bool types::isDerivedFromC(ID Id) {
   case TY_PP_ObjCXX:
   case TY_PP_ObjCXX_Alias:
   case TY_ObjCXX:
-  case TY_RenderScript:
   case TY_PP_CHeader:
   case TY_CHeader:
   case TY_CLHeader:
@@ -328,7 +327,6 @@ types::ID types::lookupTypeForExtension(llvm::StringRef Ext) {
       .Case("ll", TY_LLVM_IR)
       .Case("mi", TY_PP_ObjC)
       .Case("mm", TY_ObjCXX)
-      .Case("rs", TY_RenderScript)
       .Case("adb", TY_Ada)
       .Case("ads", TY_Ada)
       .Case("asm", TY_PP_Asm)
diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
index 030509d37875..c730c062b6a1 100644
--- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
+++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
@@ -213,7 +213,6 @@ StringRef getLanguageName(Language Lang) {
   case Language::OpenCL:
   case Language::OpenCLCXX:
   case Language::CUDA:
-  case Language::RenderScript:
   case Language::HIP:
   case Language::HLSL:
 
diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp
index bffff0d27af3..4aec928f9eb0 100644
--- a/clang/lib/Frontend/ASTUnit.cpp
+++ b/clang/lib/Frontend/ASTUnit.cpp
@@ -2699,8 +2699,6 @@ InputKind ASTUnit::getInputKind() const {
     Lang = Language::OpenCL;
   else if (LangOpts.CUDA)
     Lang = Language::CUDA;
-  else if (LangOpts.RenderScript)
-    Lang = Language::RenderScript;
   else if (LangOpts.CPlusPlus)
     Lang = LangOpts.ObjC ? Language::ObjCXX : Language::CXX;
   else
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index db7c791059a3..d8261e12b08b 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -2846,9 +2846,6 @@ static void GenerateFrontendArgs(const FrontendOptions &Opts,
     case Language::ObjCXX:
       Lang = "objective-c++";
       break;
-    case Language::RenderScript:
-      Lang = "renderscript";
-      break;
     case Language::Asm:
       Lang = "assembler-with-cpp";
       break;
@@ -3071,7 +3068,6 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
                 .Case("c++", Language::CXX)
                 .Case("objective-c", Language::ObjC)
                 .Case("objective-c++", Language::ObjCXX)
-                .Case("renderscript", Language::RenderScript)
                 .Case("hlsl", Language::HLSL)
                 .Default(Language::Unknown);
 
@@ -3499,7 +3495,6 @@ static bool IsInputCompatibleWithStandard(InputKind IK,
 
   case Language::C:
   case Language::ObjC:
-  case Language::RenderScript:
     return S.getLanguage() == Language::C;
 
   case Language::OpenCL:
@@ -3551,8 +3546,6 @@ static StringRef GetInputKindName(InputKind IK) {
     return "C++ for OpenCL";
   case Language::CUDA:
     return "CUDA";
-  case Language::RenderScript:
-    return "RenderScript";
   case Language::HIP:
     return "HIP";
 
diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp
index 64f90c493c10..e943f143d4c1 100644
--- a/clang/lib/Frontend/FrontendActions.cpp
+++ b/clang/lib/Frontend/FrontendActions.cpp
@@ -1108,7 +1108,6 @@ void PrintPreambleAction::ExecuteAction() {
   case Language::Unknown:
   case Language::Asm:
   case Language::LLVM_IR:
-  case Language::RenderScript:
     // We can't do anything with these.
     return;
   }
diff --git a/clang/test/CodeGen/debug-info-renderscript-tag.rs b/clang/test/CodeGen/debug-info-renderscript-tag.rs
deleted file mode 100644
index ded650d9660b..000000000000
--- a/clang/test/CodeGen/debug-info-renderscript-tag.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-// RUN: %clang -emit-llvm -S -g %s -o - | FileCheck %s
-
-// CHECK: !DICompileUnit(language: DW_LANG_GOOGLE_RenderScript{{.*}})
diff --git a/clang/test/CodeGen/fp16-ops.c b/clang/test/CodeGen/fp16-ops.c
index bfa2a2f7f6c8..4c206690a751 100644
--- a/clang/test/CodeGen/fp16-ops.c
+++ b/clang/test/CodeGen/fp16-ops.c
@@ -6,8 +6,6 @@
 // RUN:   | FileCheck %s --check-prefix=NATIVE-HALF
 // RUN: %clang_cc1 -emit-llvm -o - -triple aarch64 -fnative-half-type %s \
 // RUN:   | FileCheck %s --check-prefix=NATIVE-HALF
-// RUN: %clang_cc1 -emit-llvm -o - -x renderscript %s \
-// RUN:   | FileCheck %s --check-prefix=NATIVE-HALF
 typedef unsigned cond_t;
 typedef __fp16 float16_t;
 
diff --git a/clang/test/CodeGen/renderscript.c b/clang/test/CodeGen/renderscript.c
deleted file mode 100644
index 1629665c1ffb..000000000000
--- a/clang/test/CodeGen/renderscript.c
+++ /dev/null
@@ -1,140 +0,0 @@
-// RUN: %clang_cc1 %s -triple=renderscript32-none-linux-gnueabi -emit-llvm -o - -Werror | FileCheck %s -check-prefix=CHECK-RS32
-// RUN: %clang_cc1 %s -triple=renderscript64-none-linux-android -emit-llvm -o - -Werror | FileCheck %s -check-prefix=CHECK-RS64
-// RUN: %clang_cc1 %s -triple=armv7-none-linux-gnueabi -emit-llvm -o - -Werror | FileCheck %s -check-prefix=CHECK-ARM
-
-// Ensure that the bitcode has the correct triple
-// CHECK-RS32: target triple = "armv7-none-linux-gnueabi"
-// CHECK-RS64: target triple = "aarch64-none-linux-android"
-// CHECK-ARM: target triple = "armv7-none-linux-gnueabi"
-
-// Ensure that long data type has 8-byte size and alignment in RenderScript
-#ifdef __RENDERSCRIPT__
-#define LONG_WIDTH_AND_ALIGN 8
-#else
-#define LONG_WIDTH_AND_ALIGN 4
-#endif
-
-_Static_assert(sizeof(long) == LONG_WIDTH_AND_ALIGN, "sizeof long is wrong");
-_Static_assert(_Alignof(long) == LONG_WIDTH_AND_ALIGN, "sizeof long is wrong");
-
-// CHECK-RS32: i64 @test_long(i64 noundef %v)
-// CHECK-RS64: i64 @test_long(i64 noundef %v)
-// CHECK-ARM: i32 @test_long(i32 noundef %v)
-long test_long(long v) {
-  return v + 1;
-}
-
-// =============================================================================
-// Test coercion of aggregate argument or return value into integer arrays
-// =============================================================================
-
-// =============================================================================
-// aggregate parameter <= 4 bytes: coerced to [a x iNN] for both 32-bit and
-// 64-bit RenderScript
-// ==============================================================================
-
-typedef struct {char c1, c2, c3; } sChar3;
-typedef struct {short s; char c;} sShortChar;
-
-// CHECK-RS32: void @argChar3([3 x i8] %s.coerce)
-// CHECK-RS64: void @argChar3([3 x i8] %s.coerce)
-void argChar3(sChar3 s) {}
-
-// CHECK-RS32: void @argShortChar([2 x i16] %s.coerce)
-// CHECK-RS64: void @argShortChar([2 x i16] %s.coerce)
-void argShortChar(sShortChar s) {}
-
-// =============================================================================
-// aggregate return value <= 4 bytes: coerced to [a x iNN] for both 32-bit and
-// 64-bit RenderScript
-// =============================================================================
-
-// CHECK-RS32: [3 x i8] @retChar3()
-// CHECK-RS64: [3 x i8] @retChar3()
-sChar3 retChar3(void) { sChar3 r; return r; }
-
-// CHECK-RS32: [2 x i16] @retShortChar()
-// CHECK-RS64: [2 x i16] @retShortChar()
-sShortChar retShortChar(void) { sShortChar r; return r; }
-
-// =============================================================================
-// aggregate parameter <= 16 bytes: coerced to [a x iNN] for both 32-bit and
-// 64-bit RenderScript
-// =============================================================================
-
-typedef struct {short s1; char c; short s2; } sShortCharShort;
-typedef struct {int i; short s; char c; } sIntShortChar;
-typedef struct {long l; int i; } sLongInt;
-
-// CHECK-RS32: void @argShortCharShort([3 x i16] %s.coerce)
-// CHECK-RS64: void @argShortCharShort([3 x i16] %s.coerce)
-void argShortCharShort(sShortCharShort s) {}
-
-// CHECK-RS32: void @argIntShortChar([2 x i32] %s.coerce)
-// CHECK-RS64: void @argIntShortChar([2 x i32] %s.coerce)
-void argIntShortChar(sIntShortChar s) {}
-
-// CHECK-RS32: void @argLongInt([2 x i64] %s.coerce)
-// CHECK-RS64: void @argLongInt([2 x i64] %s.coerce)
-void argLongInt(sLongInt s) {}
-
-// =============================================================================
-// aggregate return value <= 16 bytes: returned on stack for 32-bit RenderScript
-// and coerced to [a x iNN] for 64-bit RenderScript
-// =============================================================================
-
-// CHECK-RS32: void @retShortCharShort(ptr dead_on_unwind noalias writable sret(%struct.sShortCharShort) align 2 %agg.result)
-// CHECK-RS64: [3 x i16] @retShortCharShort()
-sShortCharShort retShortCharShort(void) { sShortCharShort r; return r; }
-
-// CHECK-RS32: void @retIntShortChar(ptr dead_on_unwind noalias writable sret(%struct.sIntShortChar) align 4 %agg.result)
-// CHECK-RS64: [2 x i32] @retIntShortChar()
-sIntShortChar retIntShortChar(void) { sIntShortChar r; return r; }
-
-// CHECK-RS32: void @retLongInt(ptr dead_on_unwind noalias writable sret(%struct.sLongInt) align 8 %agg.result)
-// CHECK-RS64: [2 x i64] @retLongInt()
-sLongInt retLongInt(void) { sLongInt r; return r; }
-
-// =============================================================================
-// aggregate parameter <= 64 bytes: coerced to [a x iNN] for 32-bit RenderScript
-// and passed on the stack for 64-bit RenderScript
-// =============================================================================
-
-typedef struct {int i1, i2, i3, i4, i5; } sInt5;
-typedef struct {long l1, l2; char c; } sLong2Char;
-
-// CHECK-RS32: void @argInt5([5 x i32] %s.coerce)
-// CHECK-RS64: void @argInt5(ptr noundef %s)
-void argInt5(sInt5 s) {}
-
-// CHECK-RS32: void @argLong2Char([3 x i64] %s.coerce)
-// CHECK-RS64: void @argLong2Char(ptr noundef %s)
-void argLong2Char(sLong2Char s) {}
-
-// =============================================================================
-// aggregate return value <= 64 bytes: returned on stack for both 32-bit and
-// 64-bit RenderScript
-// =============================================================================
-
-// CHECK-RS32: void @retInt5(ptr dead_on_unwind noalias writable sret(%struct.sInt5) align 4 %agg.result)
-// CHECK-RS64: void @retInt5(ptr dead_on_unwind noalias writable sret(%struct.sInt5) align 4 %agg.result)
-sInt5 retInt5(void) { sInt5 r; return r;}
-
-// CHECK-RS32: void @retLong2Char(ptr dead_on_unwind noalias writable sret(%struct.sLong2Char) align 8 %agg.result)
-// CHECK-RS64: void @retLong2Char(ptr dead_on_unwind noalias writable sret(%struct.sLong2Char) align 8 %agg.result)
-sLong2Char retLong2Char(void) { sLong2Char r; return r;}
-
-// =============================================================================
-// aggregate parameters and return values > 64 bytes: passed and returned on the
-// stack for both 32-bit and 64-bit RenderScript
-// =============================================================================
-
-typedef struct {long l1, l2, l3, l4, l5, l6, l7, l8, l9; } sLong9;
-
-// CHECK-RS32: void @argLong9(ptr noundef byval(%struct.sLong9) align 8 %s)
-// CHECK-RS64: void @argLong9(ptr noundef %s)
-void argLong9(sLong9 s) {}
-
-// CHECK-RS32: void @retLong9(ptr dead_on_unwind noalias writable sret(%struct.sLong9) align 8 %agg.result)
-// CHECK-RS64: void @retLong9(ptr dead_on_unwind noalias writable sret(%struct.sLong9) align 8 %agg.result)
-sLong9 retLong9(void) { sLong9 r; return r; }
diff --git a/clang/test/Driver/renderscript.rs b/clang/test/Driver/renderscript.rs
deleted file mode 100644
index 84f5dc4de777..000000000000
--- a/clang/test/Driver/renderscript.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-// RUN: %clang -### 2>&1 %s | FileCheck %s
-
-// CHECK: "-x" "renderscript"
diff --git a/clang/test/Driver/unknown-std.c b/clang/test/Driver/unknown-std.c
index 8f9047b2230a..332d587ddd4a 100644
--- a/clang/test/Driver/unknown-std.c
+++ b/clang/test/Driver/unknown-std.c
@@ -4,7 +4,6 @@
 
 // RUN: not %clang %s -std=foobar -c 2>&1 | FileCheck --match-full-lines %s
 // RUN: not %clang -x objective-c %s -std=foobar -c 2>&1 | FileCheck --match-full-lines %s
-// RUN: not %clang -x renderscript %s -std=foobar -c 2>&1 | FileCheck --match-full-lines %s
 
 // CHECK: error: invalid value 'foobar' in '-std=foobar'
 // CHECK-NEXT: note: use 'c89', 'c90', or 'iso9899:1990' for 'ISO C 1990' standard
diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index 914f94c08a9f..e28b0775410c 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -173,7 +173,6 @@
 // CHECK-NEXT: RandomizeLayout (SubjectMatchRule_record)
 // CHECK-NEXT: ReadOnlyPlacement (SubjectMatchRule_record)
 // CHECK-NEXT: ReleaseHandle (SubjectMatchRule_variable_is_parameter)
-// CHECK-NEXT: RenderScriptKernel (SubjectMatchRule_function)
 // CHECK-NEXT: ReqdWorkGroupSize (SubjectMatchRule_function)
 // CHECK-NEXT: Restrict (SubjectMatchRule_function)
 // CHECK-NEXT: ReturnTypestate (SubjectMatchRule_function, SubjectMatchRule_variable_is_parameter)
diff --git a/clang/test/Preprocessor/predefined-macros-no-warnings.c b/clang/test/Preprocessor/predefined-macros-no-warnings.c
index d44b99a2b192..4e3e29ccfa8a 100644
--- a/clang/test/Preprocessor/predefined-macros-no-warnings.c
+++ b/clang/test/Preprocessor/predefined-macros-no-warnings.c
@@ -179,8 +179,6 @@
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple wasm64-wasi
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple wasm64-emscripten
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple dxil
-// RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple renderscript32
-// RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple renderscript64
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple ve
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple csky
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple csky-linux
diff --git a/clang/test/Sema/renderscript.rs b/clang/test/Sema/renderscript.rs
deleted file mode 100644
index 8fa0d4389e7f..000000000000
--- a/clang/test/Sema/renderscript.rs
+++ /dev/null
@@ -1,25 +0,0 @@
-// RUN: %clang_cc1 -fsyntax-only -verify -x renderscript -D__RENDERSCRIPT__ %s
-// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fsyntax-only -verify -x c %s
-// REQUIRES: x86-registered-target
-
-#ifndef __RENDERSCRIPT__
-// expected-warning@+2 {{'kernel' attribute ignored}}
-#endif
-void __attribute__((kernel)) kernel(void) {}
-
-#ifndef __RENDERSCRIPT__
-// expected-warning@+4 {{'kernel' attribute ignored}}
-#else
-// expected-warning@+2 {{'kernel' attribute only applies to functions}}
-#endif
-int __attribute__((kernel)) global;
-
-#ifndef __RENDERSCRIPT__
-// expected-error@+2 {{function return value cannot have __fp16 type; did you forget * ?}}
-#endif
-__fp16 fp16_return(void);
-
-#ifndef __RENDERSCRIPT__
-// expected-error@+2 {{parameters cannot have __fp16 type; did you forget * ?}}
-#endif
-void fp16_arg(__fp16 p);
diff --git a/clang/www/index.html b/clang/www/index.html
index 95bbfa86172b..465a6a6d29dc 100755
--- a/clang/www/index.html
+++ b/clang/www/index.html
@@ -16,8 +16,8 @@
   <!--*********************************************************************-->
 
   <p>The Clang project provides a language front-end and tooling infrastructure
-  for languages in the C language family (C, C++, Objective C/C++, OpenCL,
-  CUDA, and RenderScript) for the <a href="https://www.llvm.org/">LLVM</a>
+  for languages in the C language family (C, C++, Objective C/C++, OpenCL, and
+  CUDA) for the <a href="https://www.llvm.org/">LLVM</a>
   project. Both a GCC-compatible compiler driver (<tt>clang</tt>) and an
   MSVC-compatible compiler driver (<tt>clang-cl.exe</tt>) are provided. You
   can <a href="get_started.html">get and build</a> the source today.</p>
-- 
GitLab


From f14743794587db102c6d1b20f9c87a1ac20decfd Mon Sep 17 00:00:00 2001
From: jimingham <jingham@apple.com>
Date: Mon, 28 Oct 2024 10:01:57 -0700
Subject: [PATCH 340/693] Add the ability to break on call-site locations,
 improve inline stepping (#112939)

Previously lldb didn't support setting breakpoints on call site
locations. This patch adds that ability.

It would be very slow if we did this by searching all the debug
information for every inlined subroutine record looking for a call-site
match, so I added one restriction to the call-site support. This change
will find all call sites for functions that also supply at least one
line to the regular line table. That way we can use the fact that the
line table search will move the location to that subsequent line (but
only within the same function). When we find an actually moved source
line match, we can search in the function that contained that line table
entry for the call-site, and set the breakpoint location back to that.

When I started writing tests for this new ability, it quickly became
obvious that our support for virtual inline stepping was pretty buggy.
We didn't print the right file & line number for the breakpoint, and we
didn't set the position in the "virtual inlined stack" correctly when we
hit the breakpoint. We also didn't step through the inlined frames
correctly. There was code to try to detect the right inlined stack
position, but it had been refactored a while back with the comment that
it was super confusing and the refactor was supposed to make it clearer,
but the refactor didn't work either.

That code was made much clearer by abstracting the job of "handling the
stack readjustment" to the various StopInfo's. Previously, there was a
big (and buggy) switch over stop info's. Moving the responsibility to
the stop info made this code much easier to reason about.

We also had no tests for virtual inlined stepping (our inlined stepping
test was actually written specifically to avoid the formation of a
virtual inlined stack... So I also added tests for that along with the
tests for setting the call-site breakpoints.
---
 .../lldb/Breakpoint/BreakpointLocation.h      |  36 ++++
 lldb/include/lldb/Breakpoint/BreakpointSite.h |   5 +
 lldb/include/lldb/Core/Declaration.h          |   6 +-
 lldb/include/lldb/Target/StopInfo.h           |  12 ++
 .../lldb/Target/ThreadPlanStepInRange.h       |   4 +-
 lldb/source/Breakpoint/BreakpointLocation.cpp |  63 ++++++-
 lldb/source/Breakpoint/BreakpointResolver.cpp |  15 ++
 lldb/source/Breakpoint/BreakpointSite.cpp     |  17 ++
 lldb/source/Core/Declaration.cpp              |   5 +-
 lldb/source/Symbol/Block.cpp                  |   2 +-
 lldb/source/Symbol/CompileUnit.cpp            | 111 +++++++++++-
 lldb/source/Target/StackFrameList.cpp         | 171 ++++++------------
 lldb/source/Target/StopInfo.cpp               |  55 ++++++
 lldb/source/Target/Thread.cpp                 |   8 +
 lldb/source/Target/ThreadPlanStepInRange.cpp  |  24 ++-
 .../source/Target/ThreadPlanStepOverRange.cpp |   2 +-
 .../inline-stepping/TestInlineStepping.py     |  63 +++++++
 .../inline-stepping/calling.cpp               |  25 +++
 18 files changed, 493 insertions(+), 131 deletions(-)

diff --git a/lldb/include/lldb/Breakpoint/BreakpointLocation.h b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
index cca00335bc3c..3592291bb2d0 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointLocation.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
@@ -11,10 +11,12 @@
 
 #include <memory>
 #include <mutex>
+#include <optional>
 
 #include "lldb/Breakpoint/BreakpointOptions.h"
 #include "lldb/Breakpoint/StoppointHitCounter.h"
 #include "lldb/Core/Address.h"
+#include "lldb/Symbol/LineEntry.h"
 #include "lldb/Utility/UserID.h"
 #include "lldb/lldb-private.h"
 
@@ -282,6 +284,25 @@ public:
   /// Returns the breakpoint location ID.
   lldb::break_id_t GetID() const { return m_loc_id; }
 
+  /// Set the line entry that should be shown to users for this location.
+  /// It is up to the caller to verify that this is a valid entry to show.
+  /// The current use of this is to distinguish among line entries from a
+  /// virtual inlined call stack that all share the same address.
+  /// The line entry must have the same start address as the address for this
+  /// location.
+  bool SetPreferredLineEntry(const LineEntry &line_entry) {
+    if (m_address == line_entry.range.GetBaseAddress()) {
+      m_preferred_line_entry = line_entry;
+      return true;
+    }
+    assert(0 && "Tried to set a preferred line entry with a different address");
+    return false;
+  }
+
+  const std::optional<LineEntry> GetPreferredLineEntry() {
+    return m_preferred_line_entry;
+  }
+
 protected:
   friend class BreakpointSite;
   friend class BreakpointLocationList;
@@ -306,6 +327,16 @@ protected:
   /// If it returns false we should continue, otherwise stop.
   bool IgnoreCountShouldStop();
 
+  /// If this location knows that the virtual stack frame it represents is
+  /// not frame 0, return the suggested stack frame instead.  This will happen
+  /// when the location's address contains a "virtual inlined call stack" and
+  /// the breakpoint was set on a file & line that are not at the bottom of that
+  /// stack.  For now we key off the "preferred line entry" - looking for that
+  /// in the blocks that start with the stop PC.
+  /// This version of the API doesn't take an "inlined" parameter because it
+  /// only changes frames in the inline stack.
+  std::optional<uint32_t> GetSuggestedStackFrameIndex();
+
 private:
   void SwapLocation(lldb::BreakpointLocationSP swap_from);
 
@@ -369,6 +400,11 @@ private:
   lldb::break_id_t m_loc_id; ///< Breakpoint location ID.
   StoppointHitCounter m_hit_counter; ///< Number of times this breakpoint
                                      /// location has been hit.
+  /// If this exists, use it to print the stop description rather than the
+  /// LineEntry m_address resolves to directly.  Use this for instance when the
+  /// location was given somewhere in the virtual inlined call stack since the
+  /// Address always resolves to the lowest entry in the stack.
+  std::optional<LineEntry> m_preferred_line_entry;
 
   void SetShouldResolveIndirectFunctions(bool do_resolve) {
     m_should_resolve_indirect_functions = do_resolve;
diff --git a/lldb/include/lldb/Breakpoint/BreakpointSite.h b/lldb/include/lldb/Breakpoint/BreakpointSite.h
index 17b76d51c1ae..7b3f7be23639 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointSite.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointSite.h
@@ -170,6 +170,11 @@ public:
   /// \see lldb::DescriptionLevel
   void GetDescription(Stream *s, lldb::DescriptionLevel level);
 
+  // This runs through all the breakpoint locations owning this site and returns
+  // the greatest of their suggested stack frame indexes.  This only handles
+  // inlined stack changes.
+  std::optional<uint32_t> GetSuggestedStackFrameIndex();
+
   /// Tell whether a breakpoint has a location at this site.
   ///
   /// \param[in] bp_id
diff --git a/lldb/include/lldb/Core/Declaration.h b/lldb/include/lldb/Core/Declaration.h
index 4a0e9047b546..c864b88c6b32 100644
--- a/lldb/include/lldb/Core/Declaration.h
+++ b/lldb/include/lldb/Core/Declaration.h
@@ -84,10 +84,14 @@ public:
   /// \param[in] declaration
   ///     The const Declaration object to compare with.
   ///
+  /// \param[in] full
+  ///     Same meaning as Full in FileSpec::Equal.  True means an empty
+  ///     directory is not equal to a specified one, false means it is equal.
+  ///
   /// \return
   ///     Returns \b true if \b declaration is at the same file and
   ///     line, \b false otherwise.
-  bool FileAndLineEqual(const Declaration &declaration) const;
+  bool FileAndLineEqual(const Declaration &declaration, bool full) const;
 
   /// Dump a description of this object to a Stream.
   ///
diff --git a/lldb/include/lldb/Target/StopInfo.h b/lldb/include/lldb/Target/StopInfo.h
index fae90364deaf..45beac129e86 100644
--- a/lldb/include/lldb/Target/StopInfo.h
+++ b/lldb/include/lldb/Target/StopInfo.h
@@ -77,6 +77,18 @@ public:
       m_description.clear();
   }
 
+  /// This gives the StopInfo a chance to suggest a stack frame to select.
+  /// Passing true for inlined_stack will request changes to the inlined
+  /// call stack.  Passing false will request changes to the real stack
+  /// frame.  The inlined stack gets adjusted before we call into the thread
+  /// plans so they can reason based on the correct values.  The real stack
+  /// adjustment is handled after the frame recognizers get a chance to adjust
+  /// the frame.
+  virtual std::optional<uint32_t>
+  GetSuggestedStackFrameIndex(bool inlined_stack) {
+    return {};
+  }
+
   virtual bool IsValidForOperatingSystemThread(Thread &thread) { return true; }
 
   /// A Continue operation can result in a false stop event
diff --git a/lldb/include/lldb/Target/ThreadPlanStepInRange.h b/lldb/include/lldb/Target/ThreadPlanStepInRange.h
index f9ef87942a7c..9da8370ef1c9 100644
--- a/lldb/include/lldb/Target/ThreadPlanStepInRange.h
+++ b/lldb/include/lldb/Target/ThreadPlanStepInRange.h
@@ -80,8 +80,8 @@ private:
   bool m_step_past_prologue; // FIXME: For now hard-coded to true, we could put
                              // a switch in for this if there's
                              // demand for that.
-  bool m_virtual_step; // true if we've just done a "virtual step", i.e. just
-                       // moved the inline stack depth.
+  LazyBool m_virtual_step;   // true if we've just done a "virtual step", i.e.
+                             // just moved the inline stack depth.
   ConstString m_step_into_target;
   ThreadPlanStepInRange(const ThreadPlanStepInRange &) = delete;
   const ThreadPlanStepInRange &
diff --git a/lldb/source/Breakpoint/BreakpointLocation.cpp b/lldb/source/Breakpoint/BreakpointLocation.cpp
index ad9057c8141e..c7ea50407ae1 100644
--- a/lldb/source/Breakpoint/BreakpointLocation.cpp
+++ b/lldb/source/Breakpoint/BreakpointLocation.cpp
@@ -508,8 +508,20 @@ void BreakpointLocation::GetDescription(Stream *s,
         s->PutCString("re-exported target = ");
       else
         s->PutCString("where = ");
+
+      // If there's a preferred line entry for printing, use that.
+      bool show_function_info = true;
+      if (auto preferred = GetPreferredLineEntry()) {
+        sc.line_entry = *preferred;
+        // FIXME: We're going to get the function name wrong when the preferred
+        // line entry is not the lowest one.  For now, just leave the function
+        // out in this case, but we really should also figure out how to easily
+        // fake the function name here.
+        show_function_info = false;
+      }
       sc.DumpStopContext(s, m_owner.GetTarget().GetProcessSP().get(), m_address,
-                         false, true, false, true, true, true);
+                         false, true, false, show_function_info,
+                         show_function_info, show_function_info);
     } else {
       if (sc.module_sp) {
         s->EOL();
@@ -537,7 +549,10 @@ void BreakpointLocation::GetDescription(Stream *s,
         if (sc.line_entry.line > 0) {
           s->EOL();
           s->Indent("location = ");
-          sc.line_entry.DumpStopContext(s, true);
+          if (auto preferred = GetPreferredLineEntry())
+            preferred->DumpStopContext(s, true);
+          else
+            sc.line_entry.DumpStopContext(s, true);
         }
 
       } else {
@@ -656,6 +671,50 @@ void BreakpointLocation::SendBreakpointLocationChangedEvent(
   }
 }
 
+std::optional<uint32_t> BreakpointLocation::GetSuggestedStackFrameIndex() {
+  auto preferred_opt = GetPreferredLineEntry();
+  if (!preferred_opt)
+    return {};
+  LineEntry preferred = *preferred_opt;
+  SymbolContext sc;
+  if (!m_address.CalculateSymbolContext(&sc))
+    return {};
+  // Don't return anything special if frame 0 is the preferred line entry.
+  // We not really telling the stack frame list to do anything special in that
+  // case.
+  if (!LineEntry::Compare(sc.line_entry, preferred))
+    return {};
+
+  if (!sc.block)
+    return {};
+
+  // Blocks have their line info in Declaration form, so make one here:
+  Declaration preferred_decl(preferred.GetFile(), preferred.line,
+                             preferred.column);
+
+  uint32_t depth = 0;
+  Block *inlined_block = sc.block->GetContainingInlinedBlock();
+  while (inlined_block) {
+    // If we've moved to a block that this isn't the start of, that's not
+    // our inlining info or call site, so we can stop here.
+    Address start_address;
+    if (!inlined_block->GetStartAddress(start_address) ||
+        start_address != m_address)
+      return {};
+
+    const InlineFunctionInfo *info = inlined_block->GetInlinedFunctionInfo();
+    if (info) {
+      if (preferred_decl == info->GetDeclaration())
+        return depth;
+      if (preferred_decl == info->GetCallSite())
+        return depth + 1;
+    }
+    inlined_block = inlined_block->GetInlinedParent();
+    depth++;
+  }
+  return {};
+}
+
 void BreakpointLocation::SwapLocation(BreakpointLocationSP swap_from) {
   m_address = swap_from->m_address;
   m_should_resolve_indirect_functions =
diff --git a/lldb/source/Breakpoint/BreakpointResolver.cpp b/lldb/source/Breakpoint/BreakpointResolver.cpp
index 8307689c7640..9643602d78c7 100644
--- a/lldb/source/Breakpoint/BreakpointResolver.cpp
+++ b/lldb/source/Breakpoint/BreakpointResolver.cpp
@@ -340,6 +340,21 @@ void BreakpointResolver::AddLocation(SearchFilter &filter,
   }
 
   BreakpointLocationSP bp_loc_sp(AddLocation(line_start));
+  // If the address that we resolved the location to returns a different
+  // LineEntry from the one in the incoming SC, we're probably dealing with an
+  // inlined call site, so set that as the preferred LineEntry:
+  LineEntry resolved_entry;
+  if (!skipped_prologue && bp_loc_sp &&
+      line_start.CalculateSymbolContextLineEntry(resolved_entry) &&
+      LineEntry::Compare(resolved_entry, sc.line_entry)) {
+    // FIXME: The function name will also be wrong here.  Do we need to record
+    // that as well, or can we figure that out again when we report this
+    // breakpoint location.
+    if (!bp_loc_sp->SetPreferredLineEntry(sc.line_entry)) {
+      LLDB_LOG(log, "Tried to add a preferred line entry that didn't have the "
+                    "same address as this location's address.");
+    }
+  }
   if (log && bp_loc_sp && !GetBreakpoint()->IsInternal()) {
     StreamString s;
     bp_loc_sp->GetDescription(&s, lldb::eDescriptionLevelVerbose);
diff --git a/lldb/source/Breakpoint/BreakpointSite.cpp b/lldb/source/Breakpoint/BreakpointSite.cpp
index 3ca93f908e30..9700a57d3346 100644
--- a/lldb/source/Breakpoint/BreakpointSite.cpp
+++ b/lldb/source/Breakpoint/BreakpointSite.cpp
@@ -87,6 +87,23 @@ void BreakpointSite::GetDescription(Stream *s, lldb::DescriptionLevel level) {
   m_constituents.GetDescription(s, level);
 }
 
+std::optional<uint32_t> BreakpointSite::GetSuggestedStackFrameIndex() {
+
+  std::optional<uint32_t> result;
+  std::lock_guard<std::recursive_mutex> guard(m_constituents_mutex);
+  for (BreakpointLocationSP loc_sp : m_constituents.BreakpointLocations()) {
+    std::optional<uint32_t> loc_frame_index =
+        loc_sp->GetSuggestedStackFrameIndex();
+    if (loc_frame_index) {
+      if (result)
+        result = std::max(*loc_frame_index, *result);
+      else
+        result = loc_frame_index;
+    }
+  }
+  return result;
+}
+
 bool BreakpointSite::IsInternal() const { return m_constituents.IsInternal(); }
 
 uint8_t *BreakpointSite::GetTrapOpcodeBytes() { return &m_trap_opcode[0]; }
diff --git a/lldb/source/Core/Declaration.cpp b/lldb/source/Core/Declaration.cpp
index 579a3999d14e..a485c4b9ba48 100644
--- a/lldb/source/Core/Declaration.cpp
+++ b/lldb/source/Core/Declaration.cpp
@@ -70,8 +70,9 @@ int Declaration::Compare(const Declaration &a, const Declaration &b) {
   return 0;
 }
 
-bool Declaration::FileAndLineEqual(const Declaration &declaration) const {
-  int file_compare = FileSpec::Compare(this->m_file, declaration.m_file, true);
+bool Declaration::FileAndLineEqual(const Declaration &declaration,
+                                   bool full) const {
+  int file_compare = FileSpec::Compare(this->m_file, declaration.m_file, full);
   return file_compare == 0 && this->m_line == declaration.m_line;
 }
 
diff --git a/lldb/source/Symbol/Block.cpp b/lldb/source/Symbol/Block.cpp
index f7d9c0d2d330..5c7772a6db78 100644
--- a/lldb/source/Symbol/Block.cpp
+++ b/lldb/source/Symbol/Block.cpp
@@ -230,7 +230,7 @@ Block *Block::GetContainingInlinedBlockWithCallSite(
     const auto *function_info = inlined_block->GetInlinedFunctionInfo();
 
     if (function_info &&
-        function_info->GetCallSite().FileAndLineEqual(find_call_site))
+        function_info->GetCallSite().FileAndLineEqual(find_call_site, true))
       return inlined_block;
     inlined_block = inlined_block->GetInlinedParent();
   }
diff --git a/lldb/source/Symbol/CompileUnit.cpp b/lldb/source/Symbol/CompileUnit.cpp
index db8f8ce6bcbc..f0f7e40ae70d 100644
--- a/lldb/source/Symbol/CompileUnit.cpp
+++ b/lldb/source/Symbol/CompileUnit.cpp
@@ -251,7 +251,10 @@ void CompileUnit::ResolveSymbolContext(
     SymbolContextItem resolve_scope, SymbolContextList &sc_list,
     RealpathPrefixes *realpath_prefixes) {
   const FileSpec file_spec = src_location_spec.GetFileSpec();
-  const uint32_t line = src_location_spec.GetLine().value_or(0);
+  const uint32_t line =
+      src_location_spec.GetLine().value_or(LLDB_INVALID_LINE_NUMBER);
+  const uint32_t column_num =
+      src_location_spec.GetColumn().value_or(LLDB_INVALID_COLUMN_NUMBER);
   const bool check_inlines = src_location_spec.GetCheckInlines();
 
   // First find all of the file indexes that match our "file_spec". If
@@ -312,6 +315,112 @@ void CompileUnit::ResolveSymbolContext(
         0, file_indexes, src_location_spec, &line_entry);
   }
 
+  // If we didn't manage to find a breakpoint that matched the line number
+  // requested, that might be because it is only an inline call site, and
+  // doesn't have a line entry in the line table.  Scan for that here.
+  //
+  // We are making the assumption that if there was an inlined function it will
+  // contribute at least 1 non-call-site entry to the line table.  That's handy
+  // because we don't move line breakpoints over function boundaries, so if we
+  // found a hit, and there were also a call site entry, it would have to be in
+  // the function containing the PC of the line table match.  That way we can
+  // limit the call site search to that function.
+  // We will miss functions that ONLY exist as a call site entry.
+
+  if (line_entry.IsValid() &&
+      (line_entry.line != line || line_entry.column != column_num) &&
+      resolve_scope & eSymbolContextLineEntry && check_inlines) {
+    // We don't move lines over function boundaries, so the address in the
+    // line entry will be the in function that contained the line that might
+    // be a CallSite, and we can just iterate over that function to find any
+    // inline records, and dig up their call sites.
+    Address start_addr = line_entry.range.GetBaseAddress();
+    Function *function = start_addr.CalculateSymbolContextFunction();
+
+    Declaration sought_decl(file_spec, line, column_num);
+    // We use this recursive function to descend the block structure looking
+    // for a block that has this Declaration as in it's CallSite info.
+    // This function recursively scans the sibling blocks of the incoming
+    // block parameter.
+    std::function<void(Block &)> examine_block =
+        [&sought_decl, &sc_list, &src_location_spec, resolve_scope,
+         &examine_block](Block &block) -> void {
+      // Iterate over the sibling child blocks of the incoming block.
+      Block *sibling_block = block.GetFirstChild();
+      while (sibling_block) {
+        // We only have to descend through the regular blocks, looking for
+        // immediate inlines, since those are the only ones that will have this
+        // callsite.
+        const InlineFunctionInfo *inline_info =
+            sibling_block->GetInlinedFunctionInfo();
+        if (inline_info) {
+          // If this is the call-site we are looking for, record that:
+          // We need to be careful because the call site from the debug info
+          // will generally have a column, but the user might not have specified
+          // it.
+          Declaration found_decl = inline_info->GetCallSite();
+          uint32_t sought_column = sought_decl.GetColumn();
+          if (found_decl.FileAndLineEqual(sought_decl, false) &&
+              (sought_column == LLDB_INVALID_COLUMN_NUMBER ||
+               sought_column == found_decl.GetColumn())) {
+            // If we found a call site, it belongs not in this inlined block,
+            // but in the parent block that inlined it.
+            Address parent_start_addr;
+            if (sibling_block->GetParent()->GetStartAddress(
+                    parent_start_addr)) {
+              SymbolContext sc;
+              parent_start_addr.CalculateSymbolContext(&sc, resolve_scope);
+              // Now swap out the line entry for the one we found.
+              LineEntry call_site_line = sc.line_entry;
+              call_site_line.line = found_decl.GetLine();
+              call_site_line.column = found_decl.GetColumn();
+              bool matches_spec = true;
+              // If the user asked for an exact match, we need to make sure the
+              // call site we found actually matches the location.
+              if (src_location_spec.GetExactMatch()) {
+                matches_spec = false;
+                if ((src_location_spec.GetFileSpec() ==
+                     sc.line_entry.GetFile()) &&
+                    (src_location_spec.GetLine() &&
+                     *src_location_spec.GetLine() == call_site_line.line) &&
+                    (src_location_spec.GetColumn() &&
+                     *src_location_spec.GetColumn() == call_site_line.column))
+                  matches_spec = true;
+              }
+              if (matches_spec &&
+                  sibling_block->GetRangeAtIndex(0, call_site_line.range)) {
+                SymbolContext call_site_sc(sc.target_sp, sc.module_sp,
+                                           sc.comp_unit, sc.function, sc.block,
+                                           &call_site_line, sc.symbol);
+                sc_list.Append(call_site_sc);
+              }
+            }
+          }
+        }
+
+        // Descend into the child blocks:
+        examine_block(*sibling_block);
+        // Now go to the next sibling:
+        sibling_block = sibling_block->GetSibling();
+      }
+    };
+
+    if (function) {
+      // We don't need to examine the function block, it can't be inlined.
+      Block &func_block = function->GetBlock(true);
+      examine_block(func_block);
+    }
+    // If we found entries here, we are done.  We only get here because we
+    // didn't find an exact line entry for this line & column, but if we found
+    // an exact match from the call site info that's strictly better than
+    // continuing to look for matches further on in the file.
+    // FIXME: Should I also do this for "call site line exists between the
+    // given line number and the later line we found in the line table"?  That's
+    // a closer approximation to our general sliding algorithm.
+    if (sc_list.GetSize())
+      return;
+  }
+
   // If "exact == true", then "found_line" will be the same as "line". If
   // "exact == false", the "found_line" will be the closest line entry
   // with a line number greater than "line" and we will use this for our
diff --git a/lldb/source/Target/StackFrameList.cpp b/lldb/source/Target/StackFrameList.cpp
index 3849ec5ed178..94a381edd5e2 100644
--- a/lldb/source/Target/StackFrameList.cpp
+++ b/lldb/source/Target/StackFrameList.cpp
@@ -85,121 +85,32 @@ void StackFrameList::ResetCurrentInlinedDepth() {
     return;
 
   std::lock_guard<std::recursive_mutex> guard(m_mutex);
-  
-  GetFramesUpTo(0, DoNotAllowInterruption);
-  if (m_frames.empty())
-    return;
-  if (!m_frames[0]->IsInlined()) {
-    m_current_inlined_depth = UINT32_MAX;
-    m_current_inlined_pc = LLDB_INVALID_ADDRESS;
-    Log *log = GetLog(LLDBLog::Step);
-    if (log && log->GetVerbose())
-      LLDB_LOGF(
-          log,
-          "ResetCurrentInlinedDepth: Invalidating current inlined depth.\n");
-    return;
-  }
 
-  // We only need to do something special about inlined blocks when we are
-  // at the beginning of an inlined function:
-  // FIXME: We probably also have to do something special if the PC is at
-  // the END of an inlined function, which coincides with the end of either
-  // its containing function or another inlined function.
-
-  Block *block_ptr = m_frames[0]->GetFrameBlock();
-  if (!block_ptr)
-    return;
+  m_current_inlined_pc = LLDB_INVALID_ADDRESS;
+  m_current_inlined_depth = UINT32_MAX;
 
-  Address pc_as_address;
-  lldb::addr_t curr_pc = m_thread.GetRegisterContext()->GetPC();
-  pc_as_address.SetLoadAddress(curr_pc, &(m_thread.GetProcess()->GetTarget()));
-  AddressRange containing_range;
-  if (!block_ptr->GetRangeContainingAddress(pc_as_address, containing_range) ||
-      pc_as_address != containing_range.GetBaseAddress())
-    return;
-
-  // If we got here because of a breakpoint hit, then set the inlined depth
-  // depending on where the breakpoint was set. If we got here because of a
-  // crash, then set the inlined depth to the deepest most block.  Otherwise,
-  // we stopped here naturally as the result of a step, so set ourselves in the
-  // containing frame of the whole set of nested inlines, so the user can then
-  // "virtually" step into the frames one by one, or next over the whole mess.
-  // Note: We don't have to handle being somewhere in the middle of the stack
-  // here, since ResetCurrentInlinedDepth doesn't get called if there is a
-  // valid inlined depth set.
   StopInfoSP stop_info_sp = m_thread.GetStopInfo();
   if (!stop_info_sp)
     return;
-  switch (stop_info_sp->GetStopReason()) {
-  case eStopReasonWatchpoint:
-  case eStopReasonException:
-  case eStopReasonExec:
-  case eStopReasonFork:
-  case eStopReasonVFork:
-  case eStopReasonVForkDone:
-  case eStopReasonSignal:
-    // In all these cases we want to stop in the deepest frame.
-    m_current_inlined_pc = curr_pc;
-    m_current_inlined_depth = 0;
-    break;
-  case eStopReasonBreakpoint: {
-    // FIXME: Figure out what this break point is doing, and set the inline
-    // depth appropriately.  Be careful to take into account breakpoints that
-    // implement step over prologue, since that should do the default
-    // calculation. For now, if the breakpoints corresponding to this hit are
-    // all internal, I set the stop location to the top of the inlined stack,
-    // since that will make things like stepping over prologues work right.
-    // But if there are any non-internal breakpoints I do to the bottom of the
-    // stack, since that was the old behavior.
-    uint32_t bp_site_id = stop_info_sp->GetValue();
-    BreakpointSiteSP bp_site_sp(
-        m_thread.GetProcess()->GetBreakpointSiteList().FindByID(bp_site_id));
-    bool all_internal = true;
-    if (bp_site_sp) {
-      uint32_t num_owners = bp_site_sp->GetNumberOfConstituents();
-      for (uint32_t i = 0; i < num_owners; i++) {
-        Breakpoint &bp_ref =
-            bp_site_sp->GetConstituentAtIndex(i)->GetBreakpoint();
-        if (!bp_ref.IsInternal()) {
-          all_internal = false;
-        }
-      }
-    }
-    if (!all_internal) {
-      m_current_inlined_pc = curr_pc;
-      m_current_inlined_depth = 0;
-      break;
-    }
-  }
-    [[fallthrough]];
-  default: {
-    // Otherwise, we should set ourselves at the container of the inlining, so
-    // that the user can descend into them. So first we check whether we have
-    // more than one inlined block sharing this PC:
-    int num_inlined_functions = 0;
-
-    for (Block *container_ptr = block_ptr->GetInlinedParent();
-         container_ptr != nullptr;
-         container_ptr = container_ptr->GetInlinedParent()) {
-      if (!container_ptr->GetRangeContainingAddress(pc_as_address,
-                                                    containing_range))
-        break;
-      if (pc_as_address != containing_range.GetBaseAddress())
-        break;
 
-      num_inlined_functions++;
-    }
-    m_current_inlined_pc = curr_pc;
-    m_current_inlined_depth = num_inlined_functions + 1;
-    Log *log = GetLog(LLDBLog::Step);
+  bool inlined = true;
+  auto inline_depth = stop_info_sp->GetSuggestedStackFrameIndex(inlined);
+  // We're only adjusting the inlined stack here.
+  Log *log = GetLog(LLDBLog::Step);
+  if (inline_depth) {
+    m_current_inlined_depth = *inline_depth;
+    m_current_inlined_pc = m_thread.GetRegisterContext()->GetPC();
+
     if (log && log->GetVerbose())
       LLDB_LOGF(log,
                 "ResetCurrentInlinedDepth: setting inlined "
                 "depth: %d 0x%" PRIx64 ".\n",
-                m_current_inlined_depth, curr_pc);
-
-    break;
-  }
+                m_current_inlined_depth, m_current_inlined_pc);
+  } else {
+    if (log && log->GetVerbose())
+      LLDB_LOGF(
+          log,
+          "ResetCurrentInlinedDepth: Invalidating current inlined depth.\n");
   }
 }
 
@@ -816,19 +727,48 @@ void StackFrameList::SelectMostRelevantFrame() {
 
   RecognizedStackFrameSP recognized_frame_sp = frame_sp->GetRecognizedFrame();
 
-  if (!recognized_frame_sp) {
-    LLDB_LOG(log, "Frame #0 not recognized");
-    return;
+  if (recognized_frame_sp) {
+    if (StackFrameSP most_relevant_frame_sp =
+            recognized_frame_sp->GetMostRelevantFrame()) {
+      LLDB_LOG(log, "Found most relevant frame at index {0}",
+               most_relevant_frame_sp->GetFrameIndex());
+      SetSelectedFrame(most_relevant_frame_sp.get());
+      return;
+    }
   }
+  LLDB_LOG(log, "Frame #0 not recognized");
 
-  if (StackFrameSP most_relevant_frame_sp =
-          recognized_frame_sp->GetMostRelevantFrame()) {
-    LLDB_LOG(log, "Found most relevant frame at index {0}",
-             most_relevant_frame_sp->GetFrameIndex());
-    SetSelectedFrame(most_relevant_frame_sp.get());
-  } else {
-    LLDB_LOG(log, "No relevant frame!");
+  // If this thread has a non-trivial StopInof, then let it suggest
+  // a most relevant frame:
+  StopInfoSP stop_info_sp = m_thread.GetStopInfo();
+  uint32_t stack_idx = 0;
+  bool found_relevant = false;
+  if (stop_info_sp) {
+    // Here we're only asking the stop info if it wants to adjust the real stack
+    // index.  We have to ask about the m_inlined_stack_depth in
+    // Thread::ShouldStop since the plans need to reason with that info.
+    bool inlined = false;
+    std::optional<uint32_t> stack_opt =
+        stop_info_sp->GetSuggestedStackFrameIndex(inlined);
+    if (stack_opt) {
+      stack_idx = *stack_opt;
+      found_relevant = true;
+    }
   }
+
+  frame_sp = GetFrameAtIndex(stack_idx);
+  if (!frame_sp)
+    LLDB_LOG(log, "Stop info suggested relevant frame {0} but it didn't exist",
+             stack_idx);
+  else if (found_relevant)
+    LLDB_LOG(log, "Setting selected frame from stop info to {0}", stack_idx);
+  // Note, we don't have to worry about "inlined" frames here, because we've
+  // already calculated the inlined frame in Thread::ShouldStop, and
+  // SetSelectedFrame will take care of that adjustment for us.
+  SetSelectedFrame(frame_sp.get());
+
+  if (!found_relevant)
+    LLDB_LOG(log, "No relevant frame!");
 }
 
 uint32_t StackFrameList::GetSelectedFrameIndex(
@@ -841,6 +781,7 @@ uint32_t StackFrameList::GetSelectedFrameIndex(
     // isn't set, then don't force a selection here, just return 0.
     if (!select_most_relevant)
       return 0;
+    // If the inlined stack frame is set, then use that:
     m_selected_frame_idx = 0;
   }
   return *m_selected_frame_idx;
diff --git a/lldb/source/Target/StopInfo.cpp b/lldb/source/Target/StopInfo.cpp
index 60aa65ed38c7..f6387d47504e 100644
--- a/lldb/source/Target/StopInfo.cpp
+++ b/lldb/source/Target/StopInfo.cpp
@@ -15,6 +15,7 @@
 #include "lldb/Breakpoint/WatchpointResource.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Expression/UserExpression.h"
+#include "lldb/Symbol/Block.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/StopInfo.h"
 #include "lldb/Target/Target.h"
@@ -246,6 +247,22 @@ public:
     return m_description.c_str();
   }
 
+  std::optional<uint32_t>
+  GetSuggestedStackFrameIndex(bool inlined_stack) override {
+    if (!inlined_stack)
+      return {};
+
+    ThreadSP thread_sp(m_thread_wp.lock());
+    if (!thread_sp)
+      return {};
+    BreakpointSiteSP bp_site_sp(
+        thread_sp->GetProcess()->GetBreakpointSiteList().FindByID(m_value));
+    if (!bp_site_sp)
+      return {};
+
+    return bp_site_sp->GetSuggestedStackFrameIndex();
+  }
+
 protected:
   bool ShouldStop(Event *event_ptr) override {
     // This just reports the work done by PerformAction or the synchronous
@@ -1164,6 +1181,44 @@ public:
     else
       return m_description.c_str();
   }
+
+  std::optional<uint32_t>
+  GetSuggestedStackFrameIndex(bool inlined_stack) override {
+    // Trace only knows how to adjust inlined stacks:
+    if (!inlined_stack)
+      return {};
+
+    ThreadSP thread_sp = GetThread();
+    StackFrameSP frame_0_sp = thread_sp->GetStackFrameAtIndex(0);
+    if (!frame_0_sp)
+      return {};
+    if (!frame_0_sp->IsInlined())
+      return {};
+    Block *block_ptr = frame_0_sp->GetFrameBlock();
+    if (!block_ptr)
+      return {};
+    Address pc_address = frame_0_sp->GetFrameCodeAddress();
+    AddressRange containing_range;
+    if (!block_ptr->GetRangeContainingAddress(pc_address, containing_range) ||
+        pc_address != containing_range.GetBaseAddress())
+      return {};
+
+    int num_inlined_functions = 0;
+
+    for (Block *container_ptr = block_ptr->GetInlinedParent();
+         container_ptr != nullptr;
+         container_ptr = container_ptr->GetInlinedParent()) {
+      if (!container_ptr->GetRangeContainingAddress(pc_address,
+                                                    containing_range))
+        break;
+      if (pc_address != containing_range.GetBaseAddress())
+        break;
+
+      num_inlined_functions++;
+    }
+    inlined_stack = true;
+    return num_inlined_functions + 1;
+  }
 };
 
 // StopInfoException
diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp
index 8373cdc36268..735295e6f259 100644
--- a/lldb/source/Target/Thread.cpp
+++ b/lldb/source/Target/Thread.cpp
@@ -619,6 +619,14 @@ void Thread::WillStop() {
 
 void Thread::SetupForResume() {
   if (GetResumeState() != eStateSuspended) {
+    // First check whether this thread is going to "actually" resume at all.
+    // For instance, if we're stepping from one level to the next of an
+    // virtual inlined call stack, we just change the inlined call stack index
+    // without actually running this thread.  In that case, for this thread we
+    // shouldn't push a step over breakpoint plan or do that work.
+    if (GetCurrentPlan()->IsVirtualStep())
+      return;
+
     // If we're at a breakpoint push the step-over breakpoint plan.  Do this
     // before telling the current plan it will resume, since we might change
     // what the current plan is.
diff --git a/lldb/source/Target/ThreadPlanStepInRange.cpp b/lldb/source/Target/ThreadPlanStepInRange.cpp
index 567dcc26d0d3..325a70619908 100644
--- a/lldb/source/Target/ThreadPlanStepInRange.cpp
+++ b/lldb/source/Target/ThreadPlanStepInRange.cpp
@@ -41,7 +41,7 @@ ThreadPlanStepInRange::ThreadPlanStepInRange(
                           "Step Range stepping in", thread, range, addr_context,
                           stop_others),
       ThreadPlanShouldStopHere(this), m_step_past_prologue(true),
-      m_virtual_step(false), m_step_into_target(step_into_target) {
+      m_virtual_step(eLazyBoolCalculate), m_step_into_target(step_into_target) {
   SetCallbacks();
   SetFlagsToDefault();
   SetupAvoidNoDebug(step_in_avoids_code_without_debug_info,
@@ -149,7 +149,7 @@ bool ThreadPlanStepInRange::ShouldStop(Event *event_ptr) {
       m_sub_plan_sp.reset();
   }
 
-  if (m_virtual_step) {
+  if (m_virtual_step == eLazyBoolYes) {
     // If we've just completed a virtual step, all we need to do is check for a
     // ShouldStopHere plan, and otherwise we're done.
     // FIXME - This can be both a step in and a step out.  Probably should
@@ -431,7 +431,7 @@ bool ThreadPlanStepInRange::DoPlanExplainsStop(Event *event_ptr) {
 
   bool return_value = false;
 
-  if (m_virtual_step) {
+  if (m_virtual_step == eLazyBoolYes) {
     return_value = true;
   } else {
     StopInfoSP stop_info_sp = GetPrivateStopInfo();
@@ -460,10 +460,13 @@ bool ThreadPlanStepInRange::DoPlanExplainsStop(Event *event_ptr) {
 
 bool ThreadPlanStepInRange::DoWillResume(lldb::StateType resume_state,
                                          bool current_plan) {
-  m_virtual_step = false;
+  m_virtual_step = eLazyBoolCalculate;
   if (resume_state == eStateStepping && current_plan) {
     Thread &thread = GetThread();
     // See if we are about to step over a virtual inlined call.
+    // But if we already know we're virtual stepping, don't decrement the
+    // inlined depth again...
+
     bool step_without_resume = thread.DecrementCurrentInlinedDepth();
     if (step_without_resume) {
       Log *log = GetLog(LLDBLog::Step);
@@ -476,11 +479,20 @@ bool ThreadPlanStepInRange::DoWillResume(lldb::StateType resume_state,
       // FIXME: Maybe it would be better to create a InlineStep stop reason, but
       // then
       // the whole rest of the world would have to handle that stop reason.
-      m_virtual_step = true;
+      m_virtual_step = eLazyBoolYes;
     }
     return !step_without_resume;
   }
   return true;
 }
 
-bool ThreadPlanStepInRange::IsVirtualStep() { return m_virtual_step; }
+bool ThreadPlanStepInRange::IsVirtualStep() {
+  if (m_virtual_step == eLazyBoolCalculate) {
+    Thread &thread = GetThread();
+    if (thread.GetCurrentInlinedDepth() == UINT32_MAX)
+      m_virtual_step = eLazyBoolNo;
+    else
+      m_virtual_step = eLazyBoolYes;
+  }
+  return m_virtual_step == eLazyBoolYes;
+}
diff --git a/lldb/source/Target/ThreadPlanStepOverRange.cpp b/lldb/source/Target/ThreadPlanStepOverRange.cpp
index ef5b4b5c434d..643ee827c865 100644
--- a/lldb/source/Target/ThreadPlanStepOverRange.cpp
+++ b/lldb/source/Target/ThreadPlanStepOverRange.cpp
@@ -402,7 +402,7 @@ bool ThreadPlanStepOverRange::DoWillResume(lldb::StateType resume_state,
       if (in_inlined_stack) {
         Log *log = GetLog(LLDBLog::Step);
         LLDB_LOGF(log,
-                  "ThreadPlanStepInRange::DoWillResume: adjusting range to "
+                  "ThreadPlanStepOverRange::DoWillResume: adjusting range to "
                   "the frame at inlined depth %d.",
                   thread.GetCurrentInlinedDepth());
         StackFrameSP stack_sp = thread.GetStackFrameAtIndex(0);
diff --git a/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py b/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
index 752c3a9cbd28..f52e0f0fd5bc 100644
--- a/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
+++ b/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
@@ -32,6 +32,12 @@ class TestInlineStepping(TestBase):
         self.build()
         self.step_in_template()
 
+    @add_test_categories(["pyapi"])
+    def test_virtual_inline_stepping(self):
+        """Test stepping through a virtual inlined call stack"""
+        self.build()
+        self.virtual_inline_stepping()
+
     def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)
@@ -357,3 +363,60 @@ class TestInlineStepping(TestBase):
 
         step_sequence = [["// In max_value specialized", "into"]]
         self.run_step_sequence(step_sequence)
+
+    def run_to_call_site_and_step(self, source_regex, func_name, start_pos):
+        main_spec = lldb.SBFileSpec("calling.cpp")
+        # Set the breakpoint by file and line, not sourced regex because
+        # we want to make sure we can set breakpoints on call sites:
+        call_site_line_num = line_number(self.main_source, source_regex)
+        target, process, thread, bkpt = lldbutil.run_to_line_breakpoint(
+            self, main_spec, call_site_line_num
+        )
+
+        # Make sure that the location is at the call site (run_to_line_breakpoint already asserted
+        # that there's one location.):
+        bkpt_loc = bkpt.location[0]
+        strm = lldb.SBStream()
+        result = bkpt_loc.GetDescription(strm, lldb.eDescriptionLevelFull)
+
+        self.assertTrue(result, "Got a location description")
+        desc = strm.GetData()
+        self.assertIn(f"calling.cpp:{call_site_line_num}", desc, "Right line listed")
+        # We don't get the function name right yet - so we omit it in printing.
+        # Turn on this test when that is working.
+        # self.assertIn(func_name, desc, "Right function listed")
+
+        pc = thread.frame[0].pc
+        for i in range(start_pos, 3):
+            thread.StepInto()
+            frame_0 = thread.frame[0]
+
+            trivial_line_num = line_number(
+                self.main_source, f"In caller_trivial_inline_{i}."
+            )
+            self.assertEqual(
+                frame_0.line_entry.line,
+                trivial_line_num,
+                f"Stepped into the caller_trivial_inline_{i}",
+            )
+            if pc != frame_0.pc:
+                # If we get here, we stepped to the expected line number, but
+                # the compiler on this system has decided to insert an instruction
+                # between the call site of an inlined function with no arguments,
+                # returning void, and its immediate call to another void inlined function
+                # with no arguments.  We aren't going to be testing virtual inline
+                # stepping for this function...
+                break
+
+        process.Kill()
+        target.Clear()
+
+    def virtual_inline_stepping(self):
+        """Use the Python API's to step through a virtual inlined stack"""
+        self.run_to_call_site_and_step("At caller_trivial_inline_1", "main", 1)
+        self.run_to_call_site_and_step(
+            "In caller_trivial_inline_1", "caller_trivial_inline_1", 2
+        )
+        self.run_to_call_site_and_step(
+            "In caller_trivial_inline_2", "caller_trivial_inline_2", 3
+        )
diff --git a/lldb/test/API/functionalities/inline-stepping/calling.cpp b/lldb/test/API/functionalities/inline-stepping/calling.cpp
index 49179ce7c978..d7ee56b3c079 100644
--- a/lldb/test/API/functionalities/inline-stepping/calling.cpp
+++ b/lldb/test/API/functionalities/inline-stepping/calling.cpp
@@ -13,6 +13,12 @@ int called_by_inline_ref (int &value);
 inline void inline_trivial_1 () __attribute__((always_inline));
 inline void inline_trivial_2 () __attribute__((always_inline));
 
+// These three should share the same initial pc so we can test
+// virtual inline stepping.
+inline void caller_trivial_inline_1() __attribute__((always_inline));
+inline void caller_trivial_inline_2() __attribute__((always_inline));
+inline void caller_trivial_inline_3() __attribute__((always_inline));
+
 void caller_trivial_1 ();
 void caller_trivial_2 ();
 
@@ -79,6 +85,23 @@ caller_trivial_2 ()
     inline_value += 1;  // At increment in caller_trivial_2.
 }
 
+// When you call caller_trivial_inline_1, the inlined call-site
+// should share a PC with all three of the following inlined
+// functions, so we can exercise "virtual inline stepping".
+void caller_trivial_inline_1() {
+  caller_trivial_inline_2(); // In caller_trivial_inline_1.
+  inline_value += 1;
+}
+
+void caller_trivial_inline_2() {
+  caller_trivial_inline_3(); // In caller_trivial_inline_2.
+  inline_value += 1;
+}
+
+void caller_trivial_inline_3() {
+  inline_value += 1; // In caller_trivial_inline_3.
+}
+
 void
 called_by_inline_trivial ()
 {
@@ -132,5 +155,7 @@ main (int argc, char **argv)
     max_value(123, 456);                                // Call max_value template
     max_value(std::string("abc"), std::string("0022")); // Call max_value specialized
 
+    caller_trivial_inline_1(); // At caller_trivial_inline_1.
+
     return 0;            // About to return from main.
 }
-- 
GitLab


From e517cfc531886bf6ed64b4e7109bb3141ac7f430 Mon Sep 17 00:00:00 2001
From: Lei Wang <wlei@fb.com>
Date: Mon, 28 Oct 2024 10:13:45 -0700
Subject: [PATCH 341/693] [InstrPGO] Support cold function coverage
 instrumentation (#109837)

This patch adds support for cold function coverage instrumentation based
on sampling PGO counts. The major motivation is to detect dead functions
for the services that are optimized with sampling PGO. If a function is
covered by sampling profile count (e.g., those with an entry count > 0),
we choose to skip instrumenting those functions, which significantly
reduces the instrumentation overhead.

More details about the implementation and flags:
- Added a flag `--pgo-instrument-cold-function-only` in
`PGOInstrumentation.cpp` as the main switch to control skipping the
instrumentation.
- Built the extra instrumentation passes(a bundle of passes in
`addPGOInstrPasses`) under sampling PGO pipeline. This is controlled by
`--instrument-cold-function-only-path` flag.
- Added a driver flag `-fprofile-generate-cold-function-coverage`:
- 1) Config the flags in one place, i,e. adding
`--instrument-cold-function-only-path=<...>` and
`--pgo-function-entry-coverage`. Note that the instrumentation file path
is passed through `--instrument-sample-cold-function-path`, because we
cannot use the `PGOOptions.ProfileFile` as it's already used by
`-fprofile-sample-use=<...>`.
- 2) makes linker to link `compiler_rt.profile` lib(see
[ToolChain.cpp#L1125-L1131](https://github.com/llvm/llvm-project/blob/main/clang/lib/Driver/ToolChain.cpp#L1125-L1131)
).
- Added a flag(`--pgo-cold-instrument-entry-threshold`) to config entry
count to determine cold function.

Overall, the full command is like:

```
clang++ -O2 -fprofile-generate-cold-function-coverage=<...> -fprofile-sample-use=<...>  code.cc -o code
```
---
 clang/include/clang/Driver/Options.td         |  6 ++++
 clang/lib/Driver/ToolChain.cpp                |  4 ++-
 clang/lib/Driver/ToolChains/Clang.cpp         | 20 +++++++++++
 .../test/CodeGen/pgo-cold-function-coverage.c | 19 ++++++++++
 ...fprofile-generate-cold-function-coverage.c |  8 +++++
 llvm/lib/Passes/PassBuilderPipelines.cpp      | 17 ++++++++-
 .../Instrumentation/PGOInstrumentation.cpp    | 19 ++++++++++
 .../PGOProfile/instr-gen-cold-function.ll     | 35 +++++++++++++++++++
 8 files changed, 126 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/CodeGen/pgo-cold-function-coverage.c
 create mode 100644 clang/test/Driver/fprofile-generate-cold-function-coverage.c
 create mode 100644 llvm/test/Transforms/PGOProfile/instr-gen-cold-function.ll

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 69ae0553507b..1ddf488b8bf4 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1786,6 +1786,12 @@ defm debug_info_for_profiling : BoolFOption<"debug-info-for-profiling",
   PosFlag<SetTrue, [], [ClangOption, CC1Option],
           "Emit extra debug info to make sample profile more accurate">,
   NegFlag<SetFalse>>;
+def fprofile_generate_cold_function_coverage : Flag<["-"], "fprofile-generate-cold-function-coverage">, 
+    Group<f_Group>, Visibility<[ClangOption, CLOption]>,
+    HelpText<"Generate instrumented code to collect coverage info for cold functions into default.profraw file (overridden by '=' form of option or LLVM_PROFILE_FILE env var)">;
+def fprofile_generate_cold_function_coverage_EQ : Joined<["-"], "fprofile-generate-cold-function-coverage=">, 
+    Group<f_Group>, Visibility<[ClangOption, CLOption]>, MetaVarName<"<directory>">,
+    HelpText<"Generate instrumented code to collect coverage info for cold functions into <directory>/default.profraw (overridden by LLVM_PROFILE_FILE env var)">; 
 def fprofile_instr_generate : Flag<["-"], "fprofile-instr-generate">,
     Group<f_Group>, Visibility<[ClangOption, CLOption]>,
     HelpText<"Generate instrumented code to collect execution counts into default.profraw file (overridden by '=' form of option or LLVM_PROFILE_FILE env var)">;
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 4df317709508..34de0043ca01 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -897,7 +897,9 @@ bool ToolChain::needsProfileRT(const ArgList &Args) {
          Args.hasArg(options::OPT_fprofile_instr_generate) ||
          Args.hasArg(options::OPT_fprofile_instr_generate_EQ) ||
          Args.hasArg(options::OPT_fcreate_profile) ||
-         Args.hasArg(options::OPT_forder_file_instrumentation);
+         Args.hasArg(options::OPT_forder_file_instrumentation) ||
+         Args.hasArg(options::OPT_fprofile_generate_cold_function_coverage) ||
+         Args.hasArg(options::OPT_fprofile_generate_cold_function_coverage_EQ);
 }
 
 bool ToolChain::needsGCovInstrumentation(const llvm::opt::ArgList &Args) {
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 04b3832327a9..4c6f508f1f24 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -632,6 +632,26 @@ static void addPGOAndCoverageFlags(const ToolChain &TC, Compilation &C,
     }
   }
 
+  if (auto *ColdFuncCoverageArg = Args.getLastArg(
+          options::OPT_fprofile_generate_cold_function_coverage,
+          options::OPT_fprofile_generate_cold_function_coverage_EQ)) {
+    SmallString<128> Path(
+        ColdFuncCoverageArg->getOption().matches(
+            options::OPT_fprofile_generate_cold_function_coverage_EQ)
+            ? ColdFuncCoverageArg->getValue()
+            : "");
+    llvm::sys::path::append(Path, "default_%m.profraw");
+    // FIXME: Idealy the file path should be passed through
+    // `-fprofile-instrument-path=`(InstrProfileOutput), however, this field is
+    // shared with other profile use path(see PGOOptions), we need to refactor
+    // PGOOptions to make it work.
+    CmdArgs.push_back("-mllvm");
+    CmdArgs.push_back(Args.MakeArgString(
+        Twine("--instrument-cold-function-only-path=") + Path));
+    CmdArgs.push_back("-mllvm");
+    CmdArgs.push_back("--pgo-function-entry-coverage");
+  }
+
   Arg *PGOGenArg = nullptr;
   if (PGOGenerateArg) {
     assert(!CSPGOGenerateArg);
diff --git a/clang/test/CodeGen/pgo-cold-function-coverage.c b/clang/test/CodeGen/pgo-cold-function-coverage.c
new file mode 100644
index 000000000000..fd1e1e7e14cd
--- /dev/null
+++ b/clang/test/CodeGen/pgo-cold-function-coverage.c
@@ -0,0 +1,19 @@
+// Test -fprofile-generate-cold-function-coverage 
+
+// RUN: rm -rf %t && split-file %s %t
+// RUN: %clang -O2 -fprofile-generate-cold-function-coverage=/xxx/yyy/ -fprofile-sample-accurate -fprofile-sample-use=%t/pgo-cold-func.prof  -S -emit-llvm -o - %t/pgo-cold-func.c | FileCheck %s
+
+// CHECK: @__llvm_profile_filename = {{.*}} c"/xxx/yyy/default_%m.profraw\00"
+
+// CHECK: store i8 0, ptr @__profc_bar, align 1
+// CHECK-NOT: @__profc_foo 
+
+//--- pgo-cold-func.prof
+foo:1:1
+ 1: 1
+
+//--- pgo-cold-func.c
+int bar(int x) { return x;}
+int foo(int x) { 
+    return x;
+}
diff --git a/clang/test/Driver/fprofile-generate-cold-function-coverage.c b/clang/test/Driver/fprofile-generate-cold-function-coverage.c
new file mode 100644
index 000000000000..9b2f46423f34
--- /dev/null
+++ b/clang/test/Driver/fprofile-generate-cold-function-coverage.c
@@ -0,0 +1,8 @@
+// RUN: %clang -### -c -fprofile-generate-cold-function-coverage %s 2>&1 | FileCheck %s
+// CHECK: "--instrument-cold-function-only-path=default_%m.profraw" 
+// CHECK: "--pgo-function-entry-coverage"
+// CHECK-NOT:  "-fprofile-instrument"
+// CHECK-NOT:  "-fprofile-instrument-path=
+
+// RUN: %clang -### -c -fprofile-generate-cold-function-coverage=dir %s 2>&1 | FileCheck %s --check-prefix=CHECK-EQ
+// CHECK-EQ: "--instrument-cold-function-only-path=dir{{/|\\\\}}default_%m.profraw" 
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 17710eb94b6d..488554c84c1c 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -296,7 +296,12 @@ static cl::opt<bool> UseLoopVersioningLICM(
     "enable-loop-versioning-licm", cl::init(false), cl::Hidden,
     cl::desc("Enable the experimental Loop Versioning LICM pass"));
 
+static cl::opt<std::string> InstrumentColdFuncOnlyPath(
+    "instrument-cold-function-only-path", cl::init(""),
+    cl::desc("File path for cold function only instrumentation"), cl::Hidden);
+
 extern cl::opt<std::string> UseCtxProfile;
+extern cl::opt<bool> PGOInstrumentColdFunctionOnly;
 
 namespace llvm {
 extern cl::opt<bool> EnableMemProfContextDisambiguation;
@@ -1182,8 +1187,13 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
   const bool IsCtxProfUse =
       !UseCtxProfile.empty() && Phase == ThinOrFullLTOPhase::ThinLTOPreLink;
 
+  // Enable cold function coverage instrumentation if
+  // InstrumentColdFuncOnlyPath is provided.
+  const bool IsColdFuncOnlyInstrGen = PGOInstrumentColdFunctionOnly =
+      IsPGOPreLink && !InstrumentColdFuncOnlyPath.empty();
+
   if (IsPGOInstrGen || IsPGOInstrUse || IsMemprofUse || IsCtxProfGen ||
-      IsCtxProfUse)
+      IsCtxProfUse || IsColdFuncOnlyInstrGen)
     addPreInlinerPasses(MPM, Level, Phase);
 
   // Add all the requested passes for instrumentation PGO, if requested.
@@ -1205,6 +1215,11 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
       return MPM;
     addPostPGOLoopRotation(MPM, Level);
     MPM.addPass(PGOCtxProfLoweringPass());
+  } else if (IsColdFuncOnlyInstrGen) {
+    addPGOInstrPasses(
+        MPM, Level, /* RunProfileGen */ true, /* IsCS */ false,
+        /* AtomicCounterUpdate */ false, InstrumentColdFuncOnlyPath,
+        /* ProfileRemappingFile */ "", IntrusiveRefCntPtr<vfs::FileSystem>());
   }
 
   if (IsPGOInstrGen || IsPGOInstrUse || IsCtxProfGen)
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index bceb6135cc1f..4d8141431a0c 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -319,6 +319,20 @@ static cl::opt<unsigned> PGOFunctionCriticalEdgeThreshold(
     cl::desc("Do not instrument functions with the number of critical edges "
              " greater than this threshold."));
 
+static cl::opt<uint64_t> PGOColdInstrumentEntryThreshold(
+    "pgo-cold-instrument-entry-threshold", cl::init(0), cl::Hidden,
+    cl::desc("For cold function instrumentation, skip instrumenting functions "
+             "whose entry count is above the given value."));
+
+static cl::opt<bool> PGOTreatUnknownAsCold(
+    "pgo-treat-unknown-as-cold", cl::init(false), cl::Hidden,
+    cl::desc("For cold function instrumentation, treat count unknown(e.g. "
+             "unprofiled) functions as cold."));
+
+cl::opt<bool> PGOInstrumentColdFunctionOnly(
+    "pgo-instrument-cold-function-only", cl::init(false), cl::Hidden,
+    cl::desc("Enable cold function only instrumentation."));
+
 extern cl::opt<unsigned> MaxNumVTableAnnotations;
 
 namespace llvm {
@@ -1897,6 +1911,11 @@ static bool skipPGOGen(const Function &F) {
     return true;
   if (F.getInstructionCount() < PGOFunctionSizeThreshold)
     return true;
+  if (PGOInstrumentColdFunctionOnly) {
+    if (auto EntryCount = F.getEntryCount())
+      return EntryCount->getCount() > PGOColdInstrumentEntryThreshold;
+    return !PGOTreatUnknownAsCold;
+  }
   return false;
 }
 
diff --git a/llvm/test/Transforms/PGOProfile/instr-gen-cold-function.ll b/llvm/test/Transforms/PGOProfile/instr-gen-cold-function.ll
new file mode 100644
index 000000000000..ab8cf8c01081
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/instr-gen-cold-function.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s  --passes=pgo-instr-gen -pgo-instrument-cold-function-only -pgo-function-entry-coverage -S  | FileCheck --check-prefixes=COLD %s
+; RUN: opt < %s  --passes=pgo-instr-gen -pgo-instrument-cold-function-only -pgo-function-entry-coverage -pgo-cold-instrument-entry-threshold=1 -S  | FileCheck --check-prefixes=ENTRY-COUNT %s
+; RUN: opt < %s  --passes=pgo-instr-gen -pgo-instrument-cold-function-only -pgo-function-entry-coverage -pgo-treat-unknown-as-cold -S  | FileCheck --check-prefixes=UNKNOWN-FUNC %s
+
+; COLD: call void @llvm.instrprof.cover(ptr @__profn_foo, i64  [[#]], i32 1, i32 0)
+; COLD-NOT: __profn_main
+; COLD-NOT: __profn_bar
+
+; ENTRY-COUNT: call void @llvm.instrprof.cover(ptr @__profn_foo, i64  [[#]], i32 1, i32 0)
+; ENTRY-COUNT: call void @llvm.instrprof.cover(ptr @__profn_main, i64 [[#]], i32 1, i32 0)
+
+; UNKNOWN-FUNC: call void @llvm.instrprof.cover(ptr @__profn_bar, i64  [[#]], i32 1, i32 0)
+; UNKNOWN-FUNC: call void @llvm.instrprof.cover(ptr @__profn_foo, i64  [[#]], i32 1, i32 0)
+
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @bar() {
+entry:
+  ret void
+}
+
+define void @foo() !prof !0 {
+entry:
+  ret void
+}
+
+define i32 @main() !prof !1 {
+entry:
+  ret i32 0
+}
+
+!0 = !{!"function_entry_count", i64 0}
+!1 = !{!"function_entry_count", i64 1}
-- 
GitLab


From 98e3075df992636fa42aafde96748d1d5c834688 Mon Sep 17 00:00:00 2001
From: Steven Perron <stevenperron@google.com>
Date: Mon, 28 Oct 2024 13:25:04 -0400
Subject: [PATCH 342/693] [HLSL][SPIRV] Add convergence tokens to entry point
 wrapper (#112757)

Inlining currently assumes that either all function use controled
convergence or none of them do. This is why we need to have the entry
point wrapper use controled convergence.


https://github.com/llvm/llvm-project/blob/c85611e8583e6392d56075ebdfa60893b6284813/llvm/lib/Transforms/Utils/InlineFunction.cpp#L2431-L2439
---
 clang/lib/CodeGen/CGHLSLRuntime.cpp           | 41 +++++++++++++++++--
 clang/lib/CodeGen/CGHLSLRuntime.h             |  1 +
 .../CodeGenHLSL/convergence/entry.point.hlsl  | 11 +++++
 3 files changed, 49 insertions(+), 4 deletions(-)
 create mode 100644 clang/test/CodeGenHLSL/convergence/entry.point.hlsl

diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp
index 2cce2936fe5a..06558ce796f2 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.cpp
+++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -404,6 +404,16 @@ void CGHLSLRuntime::emitEntryFunction(const FunctionDecl *FD,
   BasicBlock *BB = BasicBlock::Create(Ctx, "entry", EntryFn);
   IRBuilder<> B(BB);
   llvm::SmallVector<Value *> Args;
+
+  SmallVector<OperandBundleDef, 1> OB;
+  if (CGM.shouldEmitConvergenceTokens()) {
+    assert(EntryFn->isConvergent());
+    llvm::Value *I = B.CreateIntrinsic(
+        llvm::Intrinsic::experimental_convergence_entry, {}, {});
+    llvm::Value *bundleArgs[] = {I};
+    OB.emplace_back("convergencectrl", bundleArgs);
+  }
+
   // FIXME: support struct parameters where semantics are on members.
   // See: https://github.com/llvm/llvm-project/issues/57874
   unsigned SRetOffset = 0;
@@ -419,7 +429,7 @@ void CGHLSLRuntime::emitEntryFunction(const FunctionDecl *FD,
     Args.push_back(emitInputSemantic(B, *PD, Param.getType()));
   }
 
-  CallInst *CI = B.CreateCall(FunctionCallee(Fn), Args);
+  CallInst *CI = B.CreateCall(FunctionCallee(Fn), Args, OB);
   CI->setCallingConv(Fn->getCallingConv());
   // FIXME: Handle codegen for return type semantics.
   // See: https://github.com/llvm/llvm-project/issues/57875
@@ -474,14 +484,22 @@ void CGHLSLRuntime::generateGlobalCtorDtorCalls() {
   for (auto &F : M.functions()) {
     if (!F.hasFnAttribute("hlsl.shader"))
       continue;
-    IRBuilder<> B(&F.getEntryBlock(), F.getEntryBlock().begin());
+    auto *Token = getConvergenceToken(F.getEntryBlock());
+    Instruction *IP = &*F.getEntryBlock().begin();
+    SmallVector<OperandBundleDef, 1> OB;
+    if (Token) {
+      llvm::Value *bundleArgs[] = {Token};
+      OB.emplace_back("convergencectrl", bundleArgs);
+      IP = Token->getNextNode();
+    }
+    IRBuilder<> B(IP);
     for (auto *Fn : CtorFns)
-      B.CreateCall(FunctionCallee(Fn));
+      B.CreateCall(FunctionCallee(Fn), {}, OB);
 
     // Insert global dtors before the terminator of the last instruction
     B.SetInsertPoint(F.back().getTerminator());
     for (auto *Fn : DtorFns)
-      B.CreateCall(FunctionCallee(Fn));
+      B.CreateCall(FunctionCallee(Fn), {}, OB);
   }
 
   // No need to keep global ctors/dtors for non-lib profile after call to
@@ -579,3 +597,18 @@ llvm::Function *CGHLSLRuntime::createResourceBindingInitFn() {
   Builder.CreateRetVoid();
   return InitResBindingsFunc;
 }
+
+llvm::Instruction *CGHLSLRuntime::getConvergenceToken(BasicBlock &BB) {
+  if (!CGM.shouldEmitConvergenceTokens())
+    return nullptr;
+
+  auto E = BB.end();
+  for (auto I = BB.begin(); I != E; ++I) {
+    auto *II = dyn_cast<llvm::IntrinsicInst>(&*I);
+    if (II && llvm::isConvergenceControlIntrinsic(II->getIntrinsicID())) {
+      return II;
+    }
+  }
+  llvm_unreachable("Convergence token should have been emitted.");
+  return nullptr;
+}
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h
index ff7df41b5c62..cd533cad84e9 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -143,6 +143,7 @@ public:
 
   bool needsResourceBindingInitFn();
   llvm::Function *createResourceBindingInitFn();
+  llvm::Instruction *getConvergenceToken(llvm::BasicBlock &BB);
 
 private:
   void addBufferResourceAnnotation(llvm::GlobalVariable *GV,
diff --git a/clang/test/CodeGenHLSL/convergence/entry.point.hlsl b/clang/test/CodeGenHLSL/convergence/entry.point.hlsl
new file mode 100644
index 000000000000..337a9ad5026c
--- /dev/null
+++ b/clang/test/CodeGenHLSL/convergence/entry.point.hlsl
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 -triple spirv-pc-vulkan-compute -finclude-default-header -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s
+
+// CHECK-LABEL: define void @main()
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[token:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT: call spir_func void @_Z4mainv() [ "convergencectrl"(token [[token]]) ]
+
+[numthreads(1,1,1)]
+void main() {
+}
+
-- 
GitLab


From 97fb21ac1d6bc528b61a555356457ff2129dfde1 Mon Sep 17 00:00:00 2001
From: Chris Apple <cja-private@pm.me>
Date: Mon, 28 Oct 2024 10:33:29 -0700
Subject: [PATCH 343/693] [rtsan] Intercept aligned_alloc on all versions of
 OSX if available on build machine (#112780)

---
 .../tests/rtsan_test_interceptors_posix.cpp   | 17 +++++++++-----
 .../sanitizer_platform_interceptors.h         | 22 ++++++++++++++++++-
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
index 6233c3e91800..38274485c29f 100644
--- a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
+++ b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
@@ -122,13 +122,20 @@ TEST(TestRtsanInterceptors, VallocDiesWhenRealtime) {
   ExpectNonRealtimeSurvival(Func);
 }
 
-#if SANITIZER_INTERCEPT_ALIGNED_ALLOC
+#if __has_builtin(__builtin_available) && SANITIZER_APPLE
+#define ALIGNED_ALLOC_AVAILABLE() (__builtin_available(macOS 10.15, *))
+#else
+// We are going to assume this is true until we hit systems where it isn't
+#define ALIGNED_ALLOC_AVAILABLE() (true)
+#endif
+
 TEST(TestRtsanInterceptors, AlignedAllocDiesWhenRealtime) {
-  auto Func = []() { EXPECT_NE(nullptr, aligned_alloc(16, 32)); };
-  ExpectRealtimeDeath(Func, "aligned_alloc");
-  ExpectNonRealtimeSurvival(Func);
+  if (ALIGNED_ALLOC_AVAILABLE()) {
+    auto Func = []() { EXPECT_NE(nullptr, aligned_alloc(16, 32)); };
+    ExpectRealtimeDeath(Func, "aligned_alloc");
+    ExpectNonRealtimeSurvival(Func);
+  }
 }
-#endif
 
 // free_sized and free_aligned_sized (both C23) are not yet supported
 TEST(TestRtsanInterceptors, FreeDiesWhenRealtime) {
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
index 6959a6d52d60..3fd6b595ef19 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
@@ -84,6 +84,25 @@
 #define SI_NOT_MAC 1
 #endif
 
+#if SANITIZER_APPLE
+#  include <Availability.h>
+
+// aligned_alloc was introduced in OSX 10.15
+// Linking will fail when using an older SDK
+#  if defined(__MAC_10_15)
+// macOS 10.15 is greater than our minimal deployment target.  To ensure we
+// generate a weak reference so the dylib continues to work on older
+// systems, we need to forward declare the intercepted function as "weak
+// imports".
+SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
+                                          __sanitizer::usize __size);
+#    define SI_MAC_SDK_10_15_AVAILABLE 1
+#  else
+#    define SI_MAC_SDK_10_15_AVAILABLE 0
+#  endif  // defined(__MAC_10_15)
+
+#endif  // SANITIZER_APPLE
+
 #if SANITIZER_IOS
 #define SI_IOS 1
 #else
@@ -500,7 +519,8 @@
 #define SANITIZER_INTERCEPT_PVALLOC (SI_GLIBC || SI_ANDROID)
 #define SANITIZER_INTERCEPT_CFREE (SI_GLIBC && !SANITIZER_RISCV64)
 #define SANITIZER_INTERCEPT_REALLOCARRAY SI_POSIX
-#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC)
+#define SANITIZER_INTERCEPT_ALIGNED_ALLOC \
+  (!SI_MAC || SI_MAC_SDK_10_15_AVAILABLE)
 #define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD)
 #define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_WCSLEN 1
-- 
GitLab


From 31a6dbe941b25aadd6cbf3829d1a96973968bf11 Mon Sep 17 00:00:00 2001
From: Jacek Caban <jacek@codeweavers.com>
Date: Mon, 28 Oct 2024 18:44:23 +0100
Subject: [PATCH 344/693] [LLD][COFF] Add Support for ARM64EC pseudo
 relocations (#113832)

---
 lld/COFF/Chunks.cpp                        | 16 +++----
 lld/test/COFF/autoimport-arm64ec-data.test | 56 ++++++++++++++++++++++
 2 files changed, 63 insertions(+), 9 deletions(-)
 create mode 100644 lld/test/COFF/autoimport-arm64ec-data.test

diff --git a/lld/COFF/Chunks.cpp b/lld/COFF/Chunks.cpp
index c6986681dffe..33fb20ffeaf3 100644
--- a/lld/COFF/Chunks.cpp
+++ b/lld/COFF/Chunks.cpp
@@ -570,8 +570,7 @@ void SectionChunk::getBaserels(std::vector<Baserel> *res) {
 // another DLL) This returns the size the relocation is supposed to update,
 // in bits, or 0 if the relocation cannot be handled as a runtime pseudo
 // relocation.
-static int getRuntimePseudoRelocSize(uint16_t type,
-                                     llvm::COFF::MachineTypes machine) {
+static int getRuntimePseudoRelocSize(uint16_t type, Triple::ArchType arch) {
   // Relocations that either contain an absolute address, or a plain
   // relative offset, since the runtime pseudo reloc implementation
   // adds 8/16/32/64 bit values to a memory address.
@@ -597,8 +596,8 @@ static int getRuntimePseudoRelocSize(uint16_t type,
   // the image, or temporarily changed at runtime with VirtualProtect.
   // Since this only operates on direct address values, it doesn't work for
   // ARM/ARM64 relocations, other than the plain ADDR32/ADDR64 relocations.
-  switch (machine) {
-  case AMD64:
+  switch (arch) {
+  case Triple::x86_64:
     switch (type) {
     case IMAGE_REL_AMD64_ADDR64:
       return 64;
@@ -613,7 +612,7 @@ static int getRuntimePseudoRelocSize(uint16_t type,
     default:
       return 0;
     }
-  case I386:
+  case Triple::x86:
     switch (type) {
     case IMAGE_REL_I386_DIR32:
     case IMAGE_REL_I386_REL32:
@@ -621,14 +620,14 @@ static int getRuntimePseudoRelocSize(uint16_t type,
     default:
       return 0;
     }
-  case ARMNT:
+  case Triple::thumb:
     switch (type) {
     case IMAGE_REL_ARM_ADDR32:
       return 32;
     default:
       return 0;
     }
-  case ARM64:
+  case Triple::aarch64:
     switch (type) {
     case IMAGE_REL_ARM64_ADDR64:
       return 64;
@@ -661,8 +660,7 @@ void SectionChunk::getRuntimePseudoRelocs(
     // alive. Thus such dangling references in DWARF sections are expected.
     if (!target->getChunk())
       continue;
-    int sizeInBits =
-        getRuntimePseudoRelocSize(rel.Type, file->ctx.config.machine);
+    int sizeInBits = getRuntimePseudoRelocSize(rel.Type, getArch());
     if (sizeInBits == 0) {
       error("unable to automatically import from " + target->getName() +
             " with relocation type " +
diff --git a/lld/test/COFF/autoimport-arm64ec-data.test b/lld/test/COFF/autoimport-arm64ec-data.test
new file mode 100644
index 000000000000..4d71b55f651a
--- /dev/null
+++ b/lld/test/COFF/autoimport-arm64ec-data.test
@@ -0,0 +1,56 @@
+# REQUIRES: aarch64, x86
+RUN: split-file %s %t.dir && cd %t.dir
+
+RUN: llvm-lib -machine:arm64ec -out:libtest.a -def:test.def
+RUN: llvm-mc -triple=arm64ec-windows-gnu arm64ec.s -filetype=obj -o arm64ec.obj
+RUN: llvm-mc -triple=arm64ec-windows-gnu x86_64.s -filetype=obj -o x86_64.obj
+
+RUN: lld-link -machine:arm64ec -out:out.dll -dll -noentry x86_64.obj arm64ec.obj libtest.a -lldmingw
+
+RUN: llvm-readobj --coff-imports out.dll | FileCheck -check-prefix=IMPORTS %s
+RUN: llvm-objdump -s out.dll | FileCheck --check-prefix=CONTENTS %s
+
+IMPORTS:      Import {
+IMPORTS-NEXT:   Name: test.dll
+IMPORTS-NEXT:   ImportLookupTableRVA: 0x40E0
+IMPORTS-NEXT:   ImportAddressTableRVA: 0x3000
+IMPORTS-NEXT:   Symbol: variable (0)
+IMPORTS-NEXT: }
+
+Runtime pseudo relocation list header at 0x401c, consisting of 0x0, 0x0, 0x1.
+The first runtime pseudo relocation is from an x86_64 object file, with import
+from 0x3000, applied at 0x7000 with a size of 32 bits. The second pseudo
+relocation is from an ARM64EC object file, with import from 0x3000, applied
+at 0x7008 with a size of 32 bits.
+
+CONTENTS: Contents of section .rdata:
+CONTENTS:  180004010 00200000 10200000 00200000 00000000
+CONTENTS:  180004020 00000000 01000000 00300000 00700000
+CONTENTS:  180004030 40000000 00300000 08700000 40000000
+
+CONTENTS:      Contents of section .test:
+CONTENTS-NEXT:  180007000 00300080 01000000 00300080 01000000
+CONTENTS-NEXT:  180007010 1c400080 01000000 40400080 01000000
+
+#--- arm64ec.s
+    .text
+    .global "#_pei386_runtime_relocator"
+"#_pei386_runtime_relocator":
+    ret
+
+    .weak_anti_dep _pei386_runtime_relocator
+.set _pei386_runtime_relocator,"#_pei386_runtime_relocator"
+
+    .section .test,"dr"
+    .quad variable
+    .quad __RUNTIME_PSEUDO_RELOC_LIST__
+    .quad __RUNTIME_PSEUDO_RELOC_LIST_END__
+
+#--- x86_64.s
+    .section .test,"dr"
+    .quad variable
+
+#--- test.def
+LIBRARY test.dll
+EXPORTS
+    variable DATA
-- 
GitLab


From 4cf128512be5d7e41d8b8b5a12eec47a64af36ea Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Mon, 28 Oct 2024 13:55:57 -0400
Subject: [PATCH 345/693] [NFC][AMDGPU] Use C++17 structured bindings as much
 as possible (#113939)

This only changes `llvm/lib/Target/AMDGPU/SIISelLowering.cpp`.
There are five uses of `std::tie` remaining because they can't be
replaced with
C++17 structured bindings.
---
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 72 +++++++----------------
 1 file changed, 22 insertions(+), 50 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 803eb86e0898..52ca38aca5c7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1948,13 +1948,9 @@ SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
   const DataLayout &DL = DAG.getDataLayout();
   MachineFunction &MF = DAG.getMachineFunction();
   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
-
-  const ArgDescriptor *InputPtrReg;
-  const TargetRegisterClass *RC;
-  LLT ArgTy;
   MVT PtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS);
 
-  std::tie(InputPtrReg, RC, ArgTy) =
+  auto [InputPtrReg, RC, ArgTy] =
       Info->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
 
   // We may not have the kernarg segment argument if we have no kernel
@@ -3335,25 +3331,18 @@ void SITargetLowering::passSpecialInputs(
   // clang-format on
 
   for (auto Attr : ImplicitAttrs) {
-    const ArgDescriptor *OutgoingArg;
-    const TargetRegisterClass *ArgRC;
-    LLT ArgTy;
-
     AMDGPUFunctionArgInfo::PreloadedValue InputID = Attr.first;
 
     // If the callee does not use the attribute value, skip copying the value.
     if (CLI.CB->hasFnAttr(Attr.second))
       continue;
 
-    std::tie(OutgoingArg, ArgRC, ArgTy) =
+    const auto [OutgoingArg, ArgRC, ArgTy] =
         CalleeArgInfo->getPreloadedValue(InputID);
     if (!OutgoingArg)
       continue;
 
-    const ArgDescriptor *IncomingArg;
-    const TargetRegisterClass *IncomingArgRC;
-    LLT Ty;
-    std::tie(IncomingArg, IncomingArgRC, Ty) =
+    const auto [IncomingArg, IncomingArgRC, Ty] =
         CallerArgInfo.getPreloadedValue(InputID);
     assert(IncomingArgRC == ArgRC);
 
@@ -3396,11 +3385,8 @@ void SITargetLowering::passSpecialInputs(
 
   // Pack workitem IDs into a single register or pass it as is if already
   // packed.
-  const ArgDescriptor *OutgoingArg;
-  const TargetRegisterClass *ArgRC;
-  LLT Ty;
 
-  std::tie(OutgoingArg, ArgRC, Ty) =
+  auto [OutgoingArg, ArgRC, Ty] =
       CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
   if (!OutgoingArg)
     std::tie(OutgoingArg, ArgRC, Ty) =
@@ -4460,15 +4446,13 @@ SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI,
 
   MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
 
-  MachineBasicBlock *LoopBB;
-  MachineBasicBlock *RemainderBB;
   const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
 
   // Apparently kill flags are only valid if the def is in the same block?
   if (MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::data0))
     Src->setIsKill(false);
 
-  std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, *BB, true);
+  auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, *BB, true);
 
   MachineBasicBlock::iterator I = LoopBB->end();
 
@@ -4628,9 +4612,7 @@ loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI,
       .addReg(Exec);
   // clang-format on
 
-  MachineBasicBlock *LoopBB;
-  MachineBasicBlock *RemainderBB;
-  std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, MBB, false);
+  auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, MBB, false);
 
   const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
 
@@ -5755,8 +5737,7 @@ SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op,
          VT == MVT::v16f16 || VT == MVT::v8f32 || VT == MVT::v16f32 ||
          VT == MVT::v32f32 || VT == MVT::v32i16 || VT == MVT::v32f16);
 
-  SDValue Lo, Hi;
-  std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
+  auto [Lo, Hi] = DAG.SplitVectorOperand(Op.getNode(), 0);
 
   SDLoc SL(Op);
   SDValue OpLo = DAG.getNode(Opc, SL, Lo.getValueType(), Lo, Op->getFlags());
@@ -5776,10 +5757,8 @@ SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op,
          VT == MVT::v16f16 || VT == MVT::v8f32 || VT == MVT::v16f32 ||
          VT == MVT::v32f32 || VT == MVT::v32i16 || VT == MVT::v32f16);
 
-  SDValue Lo0, Hi0;
-  std::tie(Lo0, Hi0) = DAG.SplitVectorOperand(Op.getNode(), 0);
-  SDValue Lo1, Hi1;
-  std::tie(Lo1, Hi1) = DAG.SplitVectorOperand(Op.getNode(), 1);
+  auto [Lo0, Hi0] = DAG.SplitVectorOperand(Op.getNode(), 0);
+  auto [Lo1, Hi1] = DAG.SplitVectorOperand(Op.getNode(), 1);
 
   SDLoc SL(Op);
 
@@ -5802,15 +5781,13 @@ SDValue SITargetLowering::splitTernaryVectorOp(SDValue Op,
          VT == MVT::v4bf16 || VT == MVT::v8bf16 || VT == MVT::v16bf16 ||
          VT == MVT::v32bf16);
 
-  SDValue Lo0, Hi0;
   SDValue Op0 = Op.getOperand(0);
-  std::tie(Lo0, Hi0) = Op0.getValueType().isVector()
-                           ? DAG.SplitVectorOperand(Op.getNode(), 0)
-                           : std::pair(Op0, Op0);
-  SDValue Lo1, Hi1;
-  std::tie(Lo1, Hi1) = DAG.SplitVectorOperand(Op.getNode(), 1);
-  SDValue Lo2, Hi2;
-  std::tie(Lo2, Hi2) = DAG.SplitVectorOperand(Op.getNode(), 2);
+  auto [Lo0, Hi0] = Op0.getValueType().isVector()
+                        ? DAG.SplitVectorOperand(Op.getNode(), 0)
+                        : std::pair(Op0, Op0);
+
+  auto [Lo1, Hi1] = DAG.SplitVectorOperand(Op.getNode(), 1);
+  auto [Lo2, Hi2] = DAG.SplitVectorOperand(Op.getNode(), 2);
 
   SDLoc SL(Op);
   auto ResVT = DAG.GetSplitDestVTs(VT);
@@ -7427,8 +7404,7 @@ SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
 
   if (VecSize == 128 || VecSize == 256 || VecSize == 512) {
     SDValue Lo, Hi;
-    EVT LoVT, HiVT;
-    std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
+    auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VecVT);
 
     if (VecSize == 128) {
       SDValue V2 = DAG.getBitcast(MVT::v2i64, Vec);
@@ -10459,9 +10435,8 @@ SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
     // size.
     switch (Subtarget->getMaxPrivateElementSize()) {
     case 4: {
-      SDValue Ops[2];
-      std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(Load, DAG);
-      return DAG.getMergeValues(Ops, DL);
+      auto [Op0, Op1] = scalarizeVectorLoad(Load, DAG);
+      return DAG.getMergeValues({Op0, Op1}, DL);
     }
     case 8:
       if (NumElements > 2)
@@ -10493,9 +10468,8 @@ SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
 
   if (!allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
                                       MemVT, *Load->getMemOperand())) {
-    SDValue Ops[2];
-    std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(Load, DAG);
-    return DAG.getMergeValues(Ops, DL);
+    auto [Op0, Op1] = expandUnalignedLoad(Load, DAG);
+    return DAG.getMergeValues({Op0, Op1}, DL);
   }
 
   return SDValue();
@@ -12534,8 +12508,7 @@ SDValue SITargetLowering::performOrCombine(SDNode *N,
     EVT SrcVT = ExtSrc.getValueType();
     if (SrcVT == MVT::i32) {
       SDLoc SL(N);
-      SDValue LowLHS, HiBits;
-      std::tie(LowLHS, HiBits) = split64BitValue(LHS, DAG);
+      auto [LowLHS, HiBits] = split64BitValue(LHS, DAG);
       SDValue LowOr = DAG.getNode(ISD::OR, SL, MVT::i32, LowLHS, ExtSrc);
 
       DCI.AddToWorklist(LowOr.getNode());
@@ -13870,8 +13843,7 @@ SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
       getMad64_32(DAG, SL, MVT::i64, MulLHSLo, MulRHSLo, AddRHS, MulSignedLo);
 
   if (!MulSignedLo && (!MulLHSUnsigned32 || !MulRHSUnsigned32)) {
-    SDValue AccumLo, AccumHi;
-    std::tie(AccumLo, AccumHi) = DAG.SplitScalar(Accum, SL, MVT::i32, MVT::i32);
+    auto [AccumLo, AccumHi] = DAG.SplitScalar(Accum, SL, MVT::i32, MVT::i32);
 
     if (!MulLHSUnsigned32) {
       auto MulLHSHi =
-- 
GitLab


From f23bdbbaff5b89b1c102a155d062fc32f99d4a92 Mon Sep 17 00:00:00 2001
From: tf2spi <misomosispi@gmail.com>
Date: Mon, 28 Oct 2024 13:59:53 -0400
Subject: [PATCH 346/693] Add DILabel functions for LLVM-C (#112840)

Addresses #112799
---
 llvm/include/llvm-c/DebugInfo.h               | 46 +++++++++++++++++++
 llvm/lib/IR/DebugInfo.cpp                     | 41 +++++++++++++++++
 .../Bindings/llvm-c/debug_info_new_format.ll  | 22 +++++----
 llvm/tools/llvm-c-test/debuginfo.c            | 21 ++++++++-
 4 files changed, 120 insertions(+), 10 deletions(-)

diff --git a/llvm/include/llvm-c/DebugInfo.h b/llvm/include/llvm-c/DebugInfo.h
index 6d8891e70577..f7d81636f4dd 100644
--- a/llvm/include/llvm-c/DebugInfo.h
+++ b/llvm/include/llvm-c/DebugInfo.h
@@ -1415,6 +1415,52 @@ LLVMMetadataRef LLVMInstructionGetDebugLoc(LLVMValueRef Inst);
  */
 void LLVMInstructionSetDebugLoc(LLVMValueRef Inst, LLVMMetadataRef Loc);
 
+/**
+ * Create a new descriptor for a label
+ *
+ * \param Builder         The DIBuilder.
+ * \param Scope           The scope to create the label in.
+ * \param Name            Variable name.
+ * \param NameLen         Length of variable name.
+ * \param File            The file to create the label in.
+ * \param LineNo          Line Number.
+ * \param AlwaysPreserve  Preserve the label regardless of optimization.
+ *
+ * @see llvm::DIBuilder::createLabel()
+ */
+LLVMMetadataRef LLVMDIBuilderCreateLabel(
+    LLVMDIBuilderRef Builder,
+    LLVMMetadataRef Context, const char *Name, size_t NameLen,
+    LLVMMetadataRef File, unsigned LineNo, LLVMBool AlwaysPreserve);
+
+/**
+ * Insert a new llvm.dbg.label intrinsic call
+ *
+ * \param Builder         The DIBuilder.
+ * \param LabelInfo       The Label's debug info descriptor
+ * \param Location        The debug info location
+ * \param InsertBefore    Location for the new intrinsic.
+ *
+ * @see llvm::DIBuilder::insertLabel()
+ */
+LLVMDbgRecordRef LLVMDIBuilderInsertLabelBefore(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef LabelInfo,
+    LLVMMetadataRef Location, LLVMValueRef InsertBefore);
+
+/**
+ * Insert a new llvm.dbg.label intrinsic call
+ *
+ * \param Builder         The DIBuilder.
+ * \param LabelInfo       The Label's debug info descriptor
+ * \param Location        The debug info location
+ * \param InsertAtEnd     Location for the new intrinsic.
+ *
+ * @see llvm::DIBuilder::insertLabel()
+ */
+LLVMDbgRecordRef LLVMDIBuilderInsertLabelAtEnd(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef LabelInfo,
+    LLVMMetadataRef Location, LLVMBasicBlockRef InsertAtEnd);
+
 /**
  * Obtain the enumerated type of a Metadata instance.
  *
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index 50b29ae4f416..e20a0f053481 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -1799,6 +1799,47 @@ void LLVMInstructionSetDebugLoc(LLVMValueRef Inst, LLVMMetadataRef Loc) {
     unwrap<Instruction>(Inst)->setDebugLoc(DebugLoc());
 }
 
+LLVMMetadataRef LLVMDIBuilderCreateLabel(
+    LLVMDIBuilderRef Builder,
+    LLVMMetadataRef Context, const char *Name, size_t NameLen,
+    LLVMMetadataRef File, unsigned LineNo, LLVMBool AlwaysPreserve) {
+  return wrap(unwrap(Builder)->createLabel(
+    unwrapDI<DIScope>(Context), StringRef(Name, NameLen),
+    unwrapDI<DIFile>(File), LineNo, AlwaysPreserve));
+}
+
+LLVMDbgRecordRef LLVMDIBuilderInsertLabelBefore(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef LabelInfo,
+    LLVMMetadataRef Location, LLVMValueRef InsertBefore) {
+  DbgInstPtr DbgInst = unwrap(Builder)->insertLabel(
+    unwrapDI<DILabel>(LabelInfo), unwrapDI<DILocation>(Location),
+    unwrap<Instruction>(InsertBefore));
+  // This assert will fail if the module is in the old debug info format.
+  // This function should only be called if the module is in the new
+  // debug info format.
+  // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes,
+  // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info.
+  assert(isa<DbgRecord *>(DbgInst) &&
+         "Function unexpectedly in old debug info format");
+  return wrap(cast<DbgRecord *>(DbgInst));
+}
+
+LLVMDbgRecordRef LLVMDIBuilderInsertLabelAtEnd(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef LabelInfo,
+    LLVMMetadataRef Location, LLVMBasicBlockRef InsertAtEnd) {
+  DbgInstPtr DbgInst = unwrap(Builder)->insertLabel(
+    unwrapDI<DILabel>(LabelInfo), unwrapDI<DILocation>(Location),
+    unwrap(InsertAtEnd));
+  // This assert will fail if the module is in the old debug info format.
+  // This function should only be called if the module is in the new
+  // debug info format.
+  // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes,
+  // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info.
+  assert(isa<DbgRecord *>(DbgInst) &&
+         "Function unexpectedly in old debug info format");
+  return wrap(cast<DbgRecord *>(DbgInst));
+}
+
 LLVMMetadataKind LLVMGetMetadataKind(LLVMMetadataRef Metadata) {
   switch(unwrap(Metadata)->getMetadataID()) {
 #define HANDLE_METADATA_LEAF(CLASS) \
diff --git a/llvm/test/Bindings/llvm-c/debug_info_new_format.ll b/llvm/test/Bindings/llvm-c/debug_info_new_format.ll
index e7f537aa4f1a..a3f4bb2421fa 100644
--- a/llvm/test/Bindings/llvm-c/debug_info_new_format.ll
+++ b/llvm/test/Bindings/llvm-c/debug_info_new_format.ll
@@ -6,15 +6,18 @@
 
 ; CHECK:      define i64 @foo(i64 %0, i64 %1, <10 x i64> %2) !dbg !31 {
 ; CHECK-NEXT: entry:
-; CHECK-NEXT:     #dbg_declare(i64 0, !38, !DIExpression(), !44)
-; CHECK-NEXT:     #dbg_declare(i64 0, !39, !DIExpression(), !44)
-; CHECK-NEXT:     #dbg_declare(i64 0, !40, !DIExpression(), !44)
+; CHECK-NEXT:     #dbg_declare(i64 0, !38, !DIExpression(), !45)
+; CHECK-NEXT:     #dbg_declare(i64 0, !39, !DIExpression(), !45)
+; CHECK-NEXT:     #dbg_declare(i64 0, !40, !DIExpression(), !45)
+; CHECK-NEXT:     #dbg_label(!46, !45)
+; CHECK-NEXT:   br label %vars
+; CHECK-NEXT:     #dbg_label(!47, !45)
 ; CHECK-NEXT:   br label %vars
 ; CHECK:      vars:
 ; CHECK-NEXT:   %p1 = phi i64 [ 0, %entry ]
 ; CHECK-NEXT:   %p2 = phi i64 [ 0, %entry ]
-; CHECK-NEXT:     #dbg_value(i64 0, !41, !DIExpression(DW_OP_constu, 0, DW_OP_stack_value), !45)
-; CHECK-NEXT:     #dbg_value(i64 1, !43, !DIExpression(DW_OP_constu, 1, DW_OP_stack_value), !45)
+; CHECK-NEXT:     #dbg_value(i64 0, !41, !DIExpression(DW_OP_constu, 0, DW_OP_stack_value), !48)
+; CHECK-NEXT:     #dbg_value(i64 1, !43, !DIExpression(DW_OP_constu, 1, DW_OP_stack_value), !48)
 ; CHECK-NEXT:   %a = add i64 %p1, %p2
 ; CHECK-NEXT:   ret i64 0
 ; CHECK-NEXT: }
@@ -60,12 +63,15 @@
 ; CHECK-NEXT: !34 = !DICompositeType(tag: DW_TAG_array_type, baseType: !6, size: 640, flags: DIFlagVector, elements: !35)
 ; CHECK-NEXT: !35 = !{!36}
 ; CHECK-NEXT: !36 = !DISubrange(count: 10, lowerBound: 0)
-; CHECK-NEXT: !37 = !{!38, !39, !40, !41, !43}
+; CHECK-NEXT: !37 = !{!38, !39, !40, !41, !43, !44}
 ; CHECK-NEXT: !38 = !DILocalVariable(name: "a", arg: 1, scope: !31, file: !1, line: 42, type: !6)
 ; CHECK-NEXT: !39 = !DILocalVariable(name: "b", arg: 2, scope: !31, file: !1, line: 42, type: !6)
 ; CHECK-NEXT: !40 = !DILocalVariable(name: "c", arg: 3, scope: !31, file: !1, line: 42, type: !34)
 ; CHECK-NEXT: !41 = !DILocalVariable(name: "d", scope: !42, file: !1, line: 43, type: !6)
 ; CHECK-NEXT: !42 = distinct !DILexicalBlock(scope: !31, file: !1, line: 42)
 ; CHECK-NEXT: !43 = !DILocalVariable(name: "e", scope: !42, file: !1, line: 44, type: !6)
-; CHECK-NEXT: !44 = !DILocation(line: 42, scope: !31)
-; CHECK-NEXT: !45 = !DILocation(line: 43, scope: !31)
+; CHECK-NEXT: !44 = !DILabel(scope: !31, name: "label3", file: !1, line: 42)
+; CHECK-NEXT: !45 = !DILocation(line: 42, scope: !31)
+; CHECK-NEXT: !46 = !DILabel(scope: !31, name: "label1", file: !1, line: 42)
+; CHECK-NEXT: !47 = !DILabel(scope: !31, name: "label2", file: !1, line: 42)
+; CHECK-NEXT: !48 = !DILocation(line: 43, scope: !31)
diff --git a/llvm/tools/llvm-c-test/debuginfo.c b/llvm/tools/llvm-c-test/debuginfo.c
index 942cbe5e5900..baf4ddfcc9a3 100644
--- a/llvm/tools/llvm-c-test/debuginfo.c
+++ b/llvm/tools/llvm-c-test/debuginfo.c
@@ -163,6 +163,11 @@ int llvm_test_dibuilder(void) {
 
   LLVMSetSubprogram(FooFunction, FunctionMetadata);
 
+  LLVMMetadataRef FooLabel1 = LLVMDIBuilderCreateLabel(DIB, FunctionMetadata,
+    "label1", 6, File, 42, false);
+  LLVMDIBuilderInsertLabelAtEnd(DIB, FooLabel1, FooParamLocation,
+    FooEntryBlock);
+
   LLVMMetadataRef FooLexicalBlock =
     LLVMDIBuilderCreateLexicalBlock(DIB, FunctionMetadata, File, 42, 0);
 
@@ -210,8 +215,6 @@ int llvm_test_dibuilder(void) {
   LLVMAddNamedMetadataOperand(
       M, "EnumTest", LLVMMetadataAsValue(LLVMGetModuleContext(M), EnumTest));
 
-  LLVMDIBuilderFinalize(DIB);
-
   // Using the new debug format, debug records get attached to instructions.
   // Insert a `br` and `ret` now to absorb the debug records which are
   // currently "trailing", meaning that they're associated with a block
@@ -221,6 +224,20 @@ int llvm_test_dibuilder(void) {
   LLVMPositionBuilderAtEnd(Builder, FooEntryBlock);
   // Build `br label %vars` in entry.
   LLVMBuildBr(Builder, FooVarBlock);
+
+  // Build another br for the sake of testing labels.
+  LLVMMetadataRef FooLabel2 = LLVMDIBuilderCreateLabel(DIB, FunctionMetadata,
+    "label2", 6, File, 42, false);
+  LLVMDIBuilderInsertLabelBefore(DIB, FooLabel2, FooParamLocation,
+    LLVMBuildBr(Builder, FooVarBlock));
+  // label3 will be emitted, but label4 won't be emitted
+  // because label3 is AlwaysPreserve and label4 is not.
+  LLVMDIBuilderCreateLabel(DIB, FunctionMetadata,
+    "label3", 6, File, 42, true);
+  LLVMDIBuilderCreateLabel(DIB, FunctionMetadata,
+    "label4", 6, File, 42, false);
+  LLVMDIBuilderFinalize(DIB);
+
   // Build `ret i64 0` in vars.
   LLVMPositionBuilderAtEnd(Builder, FooVarBlock);
   LLVMTypeRef I64 = LLVMInt64TypeInContext(Ctx);
-- 
GitLab


From 19c0a74ad6baa9eb38dbe0a20af7c67999c41821 Mon Sep 17 00:00:00 2001
From: Zequan Wu <zequanwu@google.com>
Date: Mon, 28 Oct 2024 11:13:00 -0700
Subject: [PATCH 347/693] [lldb] Fix lldb windows build breakage from
 https://github.com/llvm/llvm-project/pull/113839.

---
 lldb/tools/lldb-dap/JSONUtils.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lldb/tools/lldb-dap/JSONUtils.cpp b/lldb/tools/lldb-dap/JSONUtils.cpp
index fd09e4ae505e..97fe6b4f9f05 100644
--- a/lldb/tools/lldb-dap/JSONUtils.cpp
+++ b/lldb/tools/lldb-dap/JSONUtils.cpp
@@ -21,6 +21,7 @@
 #include "lldb/API/SBStringList.h"
 #include "lldb/API/SBStructuredData.h"
 #include "lldb/API/SBValue.h"
+#include "lldb/Host/PosixApi.h"
 
 #include "DAP.h"
 #include "ExceptionBreakpoint.h"
-- 
GitLab


From da1a16ae10177494c7cae929bec987e90a160403 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Mon, 28 Oct 2024 11:24:27 -0700
Subject: [PATCH 348/693] [clang][modules] Preserve the module map that allowed
 inferring (#113389)

With inferred modules, the dependency scanner takes care to replace the
fake "__inferred_module.map" path with the file that allowed the module
to be inferred. However, this only worked when such a module was
imported directly in the TU. Whenever such module got loaded
transitively, the scanner would fail to perform the replacement. This is
caused by the fact that PCM files are lossy and drop this information.

This patch makes sure that PCMs include this file for each submodule (in
the `SUBMODULE_DEFINITION` record), fixes one existing test with an
incorrect assertion, and does a little drive-by refactoring of
`ModuleMap`.
---
 .../include/clang/Serialization/ASTBitCodes.h  |  4 ++--
 clang/include/clang/Serialization/ASTReader.h  |  2 ++
 clang/lib/Frontend/FrontendAction.cpp          |  1 -
 clang/lib/Lex/ModuleMap.cpp                    | 11 ++++-------
 clang/lib/Serialization/ASTReader.cpp          |  5 +++--
 clang/lib/Serialization/ASTWriter.cpp          |  8 ++++++++
 .../DependencyScanning/ModuleDepCollector.cpp  | 18 +++++++-----------
 clang/test/ClangScanDeps/link-libraries.c      |  7 +++----
 8 files changed, 29 insertions(+), 27 deletions(-)

diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 3ddbc5fcd26c..b6193866fc71 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -44,7 +44,7 @@ namespace serialization {
 /// Version 4 of AST files also requires that the version control branch and
 /// revision match exactly, since there is no backward compatibility of
 /// AST files at this time.
-const unsigned VERSION_MAJOR = 31;
+const unsigned VERSION_MAJOR = 32;
 
 /// AST file minor version number supported by this version of
 /// Clang.
@@ -54,7 +54,7 @@ const unsigned VERSION_MAJOR = 31;
 /// for the previous version could still support reading the new
 /// version by ignoring new kinds of subblocks), this number
 /// should be increased.
-const unsigned VERSION_MINOR = 1;
+const unsigned VERSION_MINOR = 0;
 
 /// An ID number that refers to an identifier in an AST file.
 ///
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index b476a40ebd2c..070c1c9a54f4 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2335,6 +2335,8 @@ public:
   /// Translate a FileID from another module file's FileID space into ours.
   FileID TranslateFileID(ModuleFile &F, FileID FID) const {
     assert(FID.ID >= 0 && "Reading non-local FileID.");
+    if (FID.isInvalid())
+      return FID;
     return FileID::get(F.SLocEntryBaseID + FID.ID - 1);
   }
 
diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp
index 8264bd702fe4..9a50e7453eb6 100644
--- a/clang/lib/Frontend/FrontendAction.cpp
+++ b/clang/lib/Frontend/FrontendAction.cpp
@@ -534,7 +534,6 @@ static Module *prepareToBuildModule(CompilerInstance &CI,
     }
     if (*OriginalModuleMap != CI.getSourceManager().getFileEntryRefForID(
                                   CI.getSourceManager().getMainFileID())) {
-      M->IsInferred = true;
       auto FileCharacter =
           M->IsSystem ? SrcMgr::C_System_ModuleMap : SrcMgr::C_User_ModuleMap;
       FileID OriginalModuleMapFID = CI.getSourceManager().getOrCreateFileID(
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index bc76a54abd95..201ab91cf68c 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -657,8 +657,7 @@ ModuleMap::findOrCreateModuleForHeaderInUmbrellaDir(FileEntryRef File) {
             llvm::sys::path::stem(SkippedDir.getName()), NameBuf);
         Result = findOrCreateModule(Name, Result, /*IsFramework=*/false,
                                     Explicit).first;
-        InferredModuleAllowedBy[Result] = UmbrellaModuleMap;
-        Result->IsInferred = true;
+        setInferredModuleAllowedBy(Result, UmbrellaModuleMap);
 
         // Associate the module and the directory.
         UmbrellaDirs[SkippedDir] = Result;
@@ -675,8 +674,7 @@ ModuleMap::findOrCreateModuleForHeaderInUmbrellaDir(FileEntryRef File) {
                          llvm::sys::path::stem(File.getName()), NameBuf);
       Result = findOrCreateModule(Name, Result, /*IsFramework=*/false,
                                   Explicit).first;
-      InferredModuleAllowedBy[Result] = UmbrellaModuleMap;
-      Result->IsInferred = true;
+      setInferredModuleAllowedBy(Result, UmbrellaModuleMap);
       Result->addTopHeader(File);
 
       // If inferred submodules export everything they import, add a
@@ -1097,8 +1095,7 @@ Module *ModuleMap::inferFrameworkModule(DirectoryEntryRef FrameworkDir,
   Module *Result = new (ModulesAlloc.Allocate())
       Module(ModuleConstructorTag{}, ModuleName, SourceLocation(), Parent,
              /*IsFramework=*/true, /*IsExplicit=*/false, NumCreatedModules++);
-  InferredModuleAllowedBy[Result] = ModuleMapFID;
-  Result->IsInferred = true;
+  setInferredModuleAllowedBy(Result, ModuleMapFID);
   if (!Parent) {
     if (LangOpts.CurrentModule == ModuleName)
       SourceModule = Result;
@@ -1346,7 +1343,7 @@ ModuleMap::getModuleMapFileForUniquing(const Module *M) const {
 }
 
 void ModuleMap::setInferredModuleAllowedBy(Module *M, FileID ModMapFID) {
-  assert(M->IsInferred && "module not inferred");
+  M->IsInferred = true;
   InferredModuleAllowedBy[M] = ModMapFID;
 }
 
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 7d9170e7f0b4..2419ed84e68a 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -5813,6 +5813,7 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F,
       SubmoduleID Parent = getGlobalSubmoduleID(F, Record[Idx++]);
       Module::ModuleKind Kind = (Module::ModuleKind)Record[Idx++];
       SourceLocation DefinitionLoc = ReadSourceLocation(F, Record[Idx++]);
+      FileID InferredAllowedBy = ReadFileID(F, Record, Idx);
       bool IsFramework = Record[Idx++];
       bool IsExplicit = Record[Idx++];
       bool IsSystem = Record[Idx++];
@@ -5834,8 +5835,6 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F,
           ModMap.findOrCreateModule(Name, ParentModule, IsFramework, IsExplicit)
               .first;
 
-      // FIXME: Call ModMap.setInferredModuleAllowedBy()
-
       SubmoduleID GlobalIndex = GlobalID - NUM_PREDEF_SUBMODULE_IDS;
       if (GlobalIndex >= SubmodulesLoaded.size() ||
           SubmodulesLoaded[GlobalIndex])
@@ -5866,6 +5865,8 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F,
       CurrentModule->DefinitionLoc = DefinitionLoc;
       CurrentModule->Signature = F.Signature;
       CurrentModule->IsFromModuleFile = true;
+      if (InferredAllowedBy.isValid())
+        ModMap.setInferredModuleAllowedBy(CurrentModule, InferredAllowedBy);
       CurrentModule->IsSystem = IsSystem || CurrentModule->IsSystem;
       CurrentModule->IsExternC = IsExternC;
       CurrentModule->InferSubmodules = InferSubmodules;
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index c09a41f4d140..569c688f793d 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -2914,6 +2914,7 @@ void ASTWriter::WriteSubmodules(Module *WritingModule) {
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Parent
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // Kind
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Definition location
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Inferred allowed by
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // IsFramework
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // IsExplicit
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // IsSystem
@@ -3018,6 +3019,12 @@ void ASTWriter::WriteSubmodules(Module *WritingModule) {
     SourceLocationEncoding::RawLocEncoding DefinitionLoc =
         getRawSourceLocationEncoding(getAdjustedLocation(Mod->DefinitionLoc));
 
+    ModuleMap &ModMap = PP->getHeaderSearchInfo().getModuleMap();
+    FileID UnadjustedInferredFID;
+    if (Mod->IsInferred)
+      UnadjustedInferredFID = ModMap.getModuleMapFileIDForUniquing(Mod);
+    int InferredFID = getAdjustedFileID(UnadjustedInferredFID).getOpaqueValue();
+
     // Emit the definition of the block.
     {
       RecordData::value_type Record[] = {SUBMODULE_DEFINITION,
@@ -3025,6 +3032,7 @@ void ASTWriter::WriteSubmodules(Module *WritingModule) {
                                          ParentID,
                                          (RecordData::value_type)Mod->Kind,
                                          DefinitionLoc,
+                                         (RecordData::value_type)InferredFID,
                                          Mod->IsFramework,
                                          Mod->IsExplicit,
                                          Mod->IsSystem,
diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
index 77f9d07175c2..637416cd1fc6 100644
--- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -587,9 +587,7 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   ModuleMap &ModMapInfo =
       MDC.ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap();
 
-  OptionalFileEntryRef ModuleMap = ModMapInfo.getModuleMapFileForUniquing(M);
-
-  if (ModuleMap) {
+  if (auto ModuleMap = ModMapInfo.getModuleMapFileForUniquing(M)) {
     SmallString<128> Path = ModuleMap->getNameAsRequested();
     ModMapInfo.canonicalizeModuleMapPath(Path);
     MD.ClangModuleMapFile = std::string(Path);
@@ -601,15 +599,13 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   MDC.ScanInstance.getASTReader()->visitInputFileInfos(
       *MF, /*IncludeSystem=*/true,
       [&](const serialization::InputFileInfo &IFI, bool IsSystem) {
-        // __inferred_module.map is the result of the way in which an implicit
-        // module build handles inferred modules. It adds an overlay VFS with
-        // this file in the proper directory and relies on the rest of Clang to
-        // handle it like normal. With explicitly built modules we don't need
-        // to play VFS tricks, so replace it with the correct module map.
-        if (StringRef(IFI.Filename).ends_with("__inferred_module.map")) {
-          MDC.addFileDep(MD, ModuleMap->getName());
+        // The __inferred_module.map file is an insignificant implementation
+        // detail of implicitly-built modules. The PCM will also report the
+        // actual on-disk module map file that allowed inferring the module,
+        // which is what we need for building the module explicitly
+        // Let's ignore this file.
+        if (StringRef(IFI.Filename).ends_with("__inferred_module.map"))
           return;
-        }
         MDC.addFileDep(MD, IFI.Filename);
       });
 
diff --git a/clang/test/ClangScanDeps/link-libraries.c b/clang/test/ClangScanDeps/link-libraries.c
index c09691d2356e..bc0b0c546ea0 100644
--- a/clang/test/ClangScanDeps/link-libraries.c
+++ b/clang/test/ClangScanDeps/link-libraries.c
@@ -39,14 +39,13 @@ module transitive {
 // CHECK-NEXT:   "modules": [
 // CHECK-NEXT:     {
 // CHECK-NEXT:       "clang-module-deps": [],
-// CHECK-NEXT:       "clang-modulemap-file": "{{.*}}/__inferred_module.map",
+// CHECK-NEXT:       "clang-modulemap-file": "[[PREFIX]]/Inputs/frameworks/module.modulemap",
 // CHECK-NEXT:       "command-line": [
 // CHECK:            ],
 // CHECK-NEXT:       "context-hash": "{{.*}}",
 // CHECK-NEXT:       "file-deps": [
-// CHECK-NEXT:         "{{.*}}/Framework.h"
-// CHECK-NEXT:         "{{.*}}/__inferred_module.map"
-// CHECK-NEXT:         "{{.*}}/module.modulemap"
+// CHECK-NEXT:         "[[PREFIX]]/Inputs/frameworks/Framework.framework/Headers/Framework.h"
+// CHECK-NEXT:         "[[PREFIX]]/Inputs/frameworks/module.modulemap"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "link-libraries": [
 // CHECK-NEXT:         {
-- 
GitLab


From cafd3e10c39a2bfc81eac33ee56a706476e676a9 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Mon, 28 Oct 2024 11:30:30 -0700
Subject: [PATCH 349/693] [BOLT][test] Fix NFC check with
 pre-aggregated-perf.test (#113944)

NFC checks have been failing starting with
https://lab.llvm.org/buildbot/#/builders/92/builds/8567.

NFC testing wrapper (llvm-bolt-wrapper) replaces the call of `perf2bolt`
with `llvm-bolt --aggregate-only --ignore-build-id`.

`show-density` is automatically enabled for perf2bolt only but not for
`llvm-bolt --aggregate-only`. Add the flag to the test to work around
the issue.

Test Plan:
```
cd build
../llvm-project/bolt/utils/nfc-check-setup.py --switch-back --verbose
bin/llvm-lit -a tools/bolt/test/X86/pre-aggregated-perf.test
```
---
 bolt/test/X86/pre-aggregated-perf.test | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bolt/test/X86/pre-aggregated-perf.test b/bolt/test/X86/pre-aggregated-perf.test
index 3242ba22f591..cf745ca7bf7b 100644
--- a/bolt/test/X86/pre-aggregated-perf.test
+++ b/bolt/test/X86/pre-aggregated-perf.test
@@ -11,6 +11,7 @@ REQUIRES: system-linux
 
 RUN: yaml2obj %p/Inputs/blarge.yaml &> %t.exe
 RUN: perf2bolt %t.exe -o %t --pa -p %p/Inputs/pre-aggregated.txt -w %t.new \
+RUN:   --show-density \
 RUN:   --profile-density-threshold=9 --profile-density-cutoff-hot=970000 \
 RUN:   --profile-use-dfs | FileCheck %s --check-prefix=CHECK-P2B
 
@@ -18,6 +19,7 @@ CHECK-P2B: BOLT-INFO: 4 out of 7 functions in the binary (57.1%) have non-empty
 CHECK-P2B: BOLT-INFO: Functions with density >= 21.7 account for 97.00% total sample counts.
 
 RUN: perf2bolt %t.exe -o %t --pa -p %p/Inputs/pre-aggregated.txt -w %t.new \
+RUN:   --show-density \
 RUN:   --profile-density-cutoff-hot=970000 \
 RUN:   --profile-use-dfs 2>&1 | FileCheck %s --check-prefix=CHECK-WARNING
 
-- 
GitLab


From e0a02fdb459f3126fbc40cf376f4a3871652ae49 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Mon, 28 Oct 2024 11:43:28 -0700
Subject: [PATCH 350/693] [NFC][Clang][TableGen] Use StringRef in
 FlattenedSpelling (#113809)

- Change FlattenedSpelling to use StringRef instead of std::String.
- Use range for loops and enumerate().
- Use ArrayRef<> instead of std::vector reference as function arguments.
- Use {} for all if/else branch bodies if one of them uses it.
---
 clang/utils/TableGen/ClangAttrEmitter.cpp | 245 ++++++++++------------
 1 file changed, 109 insertions(+), 136 deletions(-)

diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp
index 4890d249c6d8..cf9c70a93e5d 100644
--- a/clang/utils/TableGen/ClangAttrEmitter.cpp
+++ b/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -49,40 +49,38 @@ using namespace llvm;
 namespace {
 
 class FlattenedSpelling {
-  std::string V, N, NS;
+  StringRef V, N, NS;
   bool K = false;
   const Record &OriginalSpelling;
 
 public:
-  FlattenedSpelling(const std::string &Variety, const std::string &Name,
-                    const std::string &Namespace, bool KnownToGCC,
-                    const Record &OriginalSpelling)
+  FlattenedSpelling(StringRef Variety, StringRef Name, StringRef Namespace,
+                    bool KnownToGCC, const Record &OriginalSpelling)
       : V(Variety), N(Name), NS(Namespace), K(KnownToGCC),
         OriginalSpelling(OriginalSpelling) {}
   explicit FlattenedSpelling(const Record &Spelling)
-      : V(std::string(Spelling.getValueAsString("Variety"))),
-        N(std::string(Spelling.getValueAsString("Name"))),
-        OriginalSpelling(Spelling) {
+      : V(Spelling.getValueAsString("Variety")),
+        N(Spelling.getValueAsString("Name")), OriginalSpelling(Spelling) {
     assert(V != "GCC" && V != "Clang" &&
            "Given a GCC spelling, which means this hasn't been flattened!");
     if (V == "CXX11" || V == "C23" || V == "Pragma")
-      NS = std::string(Spelling.getValueAsString("Namespace"));
+      NS = Spelling.getValueAsString("Namespace");
   }
 
-  const std::string &variety() const { return V; }
-  const std::string &name() const { return N; }
-  const std::string &nameSpace() const { return NS; }
+  StringRef variety() const { return V; }
+  StringRef name() const { return N; }
+  StringRef nameSpace() const { return NS; }
   bool knownToGCC() const { return K; }
   const Record &getSpellingRecord() const { return OriginalSpelling; }
 };
 
 struct FlattenedSpellingInfo {
-  FlattenedSpellingInfo(std::string Syntax, std::string Scope,
-                        std::string TargetTest, uint32_t ArgMask)
+  FlattenedSpellingInfo(StringRef Syntax, StringRef Scope,
+                        const std::string &TargetTest, uint32_t ArgMask)
       : Syntax(Syntax), Scope(Scope), TargetTest(TargetTest), ArgMask(ArgMask) {
   }
-  std::string Syntax;
-  std::string Scope;
+  StringRef Syntax;
+  StringRef Scope;
   std::string TargetTest;
   uint32_t ArgMask;
 };
@@ -105,17 +103,18 @@ GetFlattenedSpellings(const Record &Attr) {
     StringRef Variety = Spelling->getValueAsString("Variety");
     StringRef Name = Spelling->getValueAsString("Name");
     if (Variety == "GCC") {
-      Ret.emplace_back("GNU", std::string(Name), "", true, *Spelling);
-      Ret.emplace_back("CXX11", std::string(Name), "gnu", true, *Spelling);
+      Ret.emplace_back("GNU", Name, "", true, *Spelling);
+      Ret.emplace_back("CXX11", Name, "gnu", true, *Spelling);
       if (Spelling->getValueAsBit("AllowInC"))
-        Ret.emplace_back("C23", std::string(Name), "gnu", true, *Spelling);
+        Ret.emplace_back("C23", Name, "gnu", true, *Spelling);
     } else if (Variety == "Clang") {
-      Ret.emplace_back("GNU", std::string(Name), "", false, *Spelling);
-      Ret.emplace_back("CXX11", std::string(Name), "clang", false, *Spelling);
+      Ret.emplace_back("GNU", Name, "", false, *Spelling);
+      Ret.emplace_back("CXX11", Name, "clang", false, *Spelling);
       if (Spelling->getValueAsBit("AllowInC"))
-        Ret.emplace_back("C23", std::string(Name), "clang", false, *Spelling);
-    } else
+        Ret.emplace_back("C23", Name, "clang", false, *Spelling);
+    } else {
       Ret.push_back(FlattenedSpelling(*Spelling));
+    }
   }
 
   return Ret;
@@ -1554,7 +1553,7 @@ static void writeAvailabilityValue(raw_ostream &OS) {
      << "  OS << \"";
 }
 
-static void writeDeprecatedAttrValue(raw_ostream &OS, std::string &Variety) {
+static void writeDeprecatedAttrValue(raw_ostream &OS, StringRef Variety) {
   OS << "\\\"\" << getMessage() << \"\\\"\";\n";
   // Only GNU deprecated has an optional fixit argument at the second position.
   if (Variety == "GNU")
@@ -1577,9 +1576,12 @@ static void writeGetSpellingFunction(const Record &R, raw_ostream &OS) {
         "    llvm_unreachable(\"Unknown attribute spelling!\");\n"
         "    return \"(No spelling)\";\n";
 
-  for (unsigned I = 0; I < Spellings.size(); ++I)
-    OS << "  case " << I << ":\n"
-          "    return \"" << Spellings[I].name() << "\";\n";
+  for (const auto &[Idx, S] : enumerate(Spellings)) {
+    // clang-format off
+    OS << "  case " << Idx << ":\n"
+          "    return \"" << S.name() << "\";\n";
+    // clang-format on
+  }
   // End of the switch statement.
   OS << "  }\n";
   // End of the getSpelling function.
@@ -1607,14 +1609,14 @@ writePrettyPrintFunction(const Record &R,
      << "    llvm_unreachable(\"Unknown attribute spelling!\");\n"
      << "    break;\n";
 
-  for (unsigned I = 0; I < Spellings.size(); ++ I) {
+  for (const auto &[Idx, S] : enumerate(Spellings)) {
     SmallString<16> Prefix;
     SmallString<8> Suffix;
     // The actual spelling of the name and namespace (if applicable)
     // of an attribute without considering prefix and suffix.
     SmallString<64> Spelling;
-    std::string Name = Spellings[I].name();
-    std::string Variety = Spellings[I].variety();
+    StringRef Name = S.name();
+    StringRef Variety = S.variety();
 
     if (Variety == "GNU") {
       Prefix = "__attribute__((";
@@ -1622,7 +1624,7 @@ writePrettyPrintFunction(const Record &R,
     } else if (Variety == "CXX11" || Variety == "C23") {
       Prefix = "[[";
       Suffix = "]]";
-      std::string Namespace = Spellings[I].nameSpace();
+      StringRef Namespace = S.nameSpace();
       if (!Namespace.empty()) {
         Spelling += Namespace;
         Spelling += "::";
@@ -1639,7 +1641,7 @@ writePrettyPrintFunction(const Record &R,
     } else if (Variety == "Pragma") {
       Prefix = "#pragma ";
       Suffix = "\n";
-      std::string Namespace = Spellings[I].nameSpace();
+      StringRef Namespace = S.nameSpace();
       if (!Namespace.empty()) {
         Spelling += Namespace;
         Spelling += " ";
@@ -1653,7 +1655,7 @@ writePrettyPrintFunction(const Record &R,
 
     Spelling += Name;
 
-    OS << "  case " << I << " : {\n"
+    OS << "  case " << Idx << " : {\n"
        << "    OS << \"" << Prefix << Spelling << "\";\n";
 
     if (Variety == "Pragma") {
@@ -1724,24 +1726,17 @@ writePrettyPrintFunction(const Record &R,
 }
 
 /// Return the index of a spelling in a spelling list.
-static unsigned
-getSpellingListIndex(const std::vector<FlattenedSpelling> &SpellingList,
-                     const FlattenedSpelling &Spelling) {
+static unsigned getSpellingListIndex(ArrayRef<FlattenedSpelling> SpellingList,
+                                     const FlattenedSpelling &Spelling) {
   assert(!SpellingList.empty() && "Spelling list is empty!");
 
-  for (unsigned Index = 0; Index < SpellingList.size(); ++Index) {
-    const FlattenedSpelling &S = SpellingList[Index];
-    if (S.variety() != Spelling.variety())
-      continue;
-    if (S.nameSpace() != Spelling.nameSpace())
-      continue;
-    if (S.name() != Spelling.name())
-      continue;
-
-    return Index;
+  for (const auto &[Index, S] : enumerate(SpellingList)) {
+    if (S.variety() == Spelling.variety() &&
+        S.nameSpace() == Spelling.nameSpace() && S.name() == Spelling.name())
+      return Index;
   }
 
-  llvm_unreachable("Unknown spelling!");
+  PrintFatalError("Unknown spelling: " + Spelling.name());
 }
 
 static void writeAttrAccessorDefinition(const Record &R, raw_ostream &OS) {
@@ -1802,15 +1797,15 @@ CreateSemanticSpellings(const std::vector<FlattenedSpelling> &Spellings,
          "AttributeCommonInfo");
   for (auto I = Spellings.begin(), E = Spellings.end(); I != E; ++I, ++Idx) {
     const FlattenedSpelling &S = *I;
-    const std::string &Variety = S.variety();
-    const std::string &Spelling = S.name();
-    const std::string &Namespace = S.nameSpace();
+    StringRef Variety = S.variety();
+    StringRef Spelling = S.name();
+    StringRef Namespace = S.nameSpace();
     std::string EnumName;
 
-    EnumName += (Variety + "_");
+    EnumName += Variety;
+    EnumName += "_";
     if (!Namespace.empty())
-      EnumName += (NormalizeNameForSpellingComparison(Namespace).str() +
-      "_");
+      EnumName += NormalizeNameForSpellingComparison(Namespace).str() + "_";
     EnumName += NormalizeNameForSpellingComparison(Spelling);
 
     // Even if the name is not unique, this spelling index corresponds to a
@@ -1837,7 +1832,7 @@ CreateSemanticSpellings(const std::vector<FlattenedSpelling> &Spellings,
   return Ret;
 }
 
-void WriteSemanticSpellingSwitch(const std::string &VarName,
+void WriteSemanticSpellingSwitch(StringRef VarName,
                                  const SemanticSpellingMap &Map,
                                  raw_ostream &OS) {
   OS << "  switch (" << VarName << ") {\n    default: "
@@ -2378,30 +2373,22 @@ void PragmaClangAttributeSupport::generateParsingHelpers(raw_ostream &OS) {
 }
 
 template <typename Fn> static void forEachSpelling(const Record &Attr, Fn &&F) {
-  std::vector<FlattenedSpelling> Spellings = GetFlattenedSpellings(Attr);
-  for (const FlattenedSpelling &S : Spellings) {
+  for (const FlattenedSpelling &S : GetFlattenedSpellings(Attr)) {
     F(S);
   }
 }
 
-std::map<std::string, std::vector<const Record *>> NameToAttrsMap;
+std::map<StringRef, std::vector<const Record *>> NameToAttrsMap;
 
 /// Build a map from the attribute name to the Attrs that use that name. If more
 /// than one Attr use a name, the arguments could be different so a more complex
 /// check is needed in the generated switch.
 void generateNameToAttrsMap(const RecordKeeper &Records) {
   for (const auto *A : Records.getAllDerivedDefinitions("Attr")) {
-    std::vector<FlattenedSpelling> Spellings = GetFlattenedSpellings(*A);
-    for (const auto &S : Spellings) {
-      auto It = NameToAttrsMap.find(S.name());
-      if (It != NameToAttrsMap.end()) {
-        if (none_of(It->second, [&](const Record *R) { return R == A; }))
-          It->second.emplace_back(A);
-      } else {
-        std::vector<const Record *> V;
-        V.emplace_back(A);
-        NameToAttrsMap.insert(std::make_pair(S.name(), V));
-      }
+    for (const FlattenedSpelling &S : GetFlattenedSpellings(*A)) {
+      auto [It, Inserted] = NameToAttrsMap.try_emplace(S.name());
+      if (Inserted || !is_contained(It->second, A))
+        It->second.emplace_back(A);
     }
   }
 }
@@ -2410,7 +2397,7 @@ void generateNameToAttrsMap(const RecordKeeper &Records) {
 /// attribute has the same name. Store the info in a map that can be processed
 /// after all attributes are seen.
 static void generateFlattenedSpellingInfo(const Record &Attr,
-                                          std::map<std::string, FSIVecTy> &Map,
+                                          std::map<StringRef, FSIVecTy> &Map,
                                           uint32_t ArgMask = 0) {
   std::string TargetTest;
   if (Attr.isSubClassOf("TargetSpecificAttr") &&
@@ -2421,24 +2408,17 @@ static void generateFlattenedSpellingInfo(const Record &Attr,
   }
 
   forEachSpelling(Attr, [&](const FlattenedSpelling &S) {
-    auto It = Map.find(S.name());
-    if (It != Map.end()) {
-      It->second.emplace_back(S.variety(), S.nameSpace(), TargetTest, ArgMask);
-    } else {
-      FSIVecTy V;
-      V.emplace_back(S.variety(), S.nameSpace(), TargetTest, ArgMask);
-      Map.insert(std::make_pair(S.name(), V));
-    }
+    Map[S.name()].emplace_back(S.variety(), S.nameSpace(), TargetTest, ArgMask);
   });
 }
 
-static bool nameAppliesToOneAttribute(std::string Name) {
+static bool nameAppliesToOneAttribute(StringRef Name) {
   auto It = NameToAttrsMap.find(Name);
   assert(It != NameToAttrsMap.end());
   return It->second.size() == 1;
 }
 
-static bool emitIfSimpleValue(std::string Name, uint32_t ArgMask,
+static bool emitIfSimpleValue(StringRef Name, uint32_t ArgMask,
                               raw_ostream &OS) {
   if (nameAppliesToOneAttribute(Name)) {
     OS << ".Case(\"" << Name << "\", ";
@@ -2463,15 +2443,13 @@ static void emitSingleCondition(const FlattenedSpellingInfo &FSI,
   OS << ")";
 }
 
-static void emitStringSwitchCases(std::map<std::string, FSIVecTy> &Map,
+static void emitStringSwitchCases(std::map<StringRef, FSIVecTy> &Map,
                                   raw_ostream &OS) {
-  for (const auto &P : Map) {
-    if (emitIfSimpleValue(P.first, P.second[0].ArgMask, OS))
+  for (const auto &[Name, Vec] : Map) {
+    if (emitIfSimpleValue(Name, Vec[0].ArgMask, OS))
       continue;
 
     // Not simple, build expressions for each case.
-    StringRef Name = P.first;
-    const FSIVecTy &Vec = P.second;
     OS << ".Case(\"" << Name << "\", ";
     for (unsigned I = 0, E = Vec.size(); I < E; ++I) {
       emitSingleCondition(Vec[I], OS);
@@ -2498,7 +2476,7 @@ static bool isTypeArgument(const Record *Arg) {
 static void emitClangAttrTypeArgList(const RecordKeeper &Records,
                                      raw_ostream &OS) {
   OS << "#if defined(CLANG_ATTR_TYPE_ARG_LIST)\n";
-  std::map<std::string, FSIVecTy> FSIMap;
+  std::map<StringRef, FSIVecTy> FSIMap;
   for (const auto *Attr : Records.getAllDerivedDefinitions("Attr")) {
     // Determine whether the first argument is a type.
     std::vector<const Record *> Args = Attr->getValueAsListOfDefs("Args");
@@ -2518,7 +2496,7 @@ static void emitClangAttrTypeArgList(const RecordKeeper &Records,
 static void emitClangAttrArgContextList(const RecordKeeper &Records,
                                         raw_ostream &OS) {
   OS << "#if defined(CLANG_ATTR_ARG_CONTEXT_LIST)\n";
-  std::map<std::string, FSIVecTy> FSIMap;
+  std::map<StringRef, FSIVecTy> FSIMap;
   ParsedAttrMap Attrs = getParsedAttrList(Records);
   for (const auto &I : Attrs) {
     const Record &Attr = *I.second;
@@ -2576,7 +2554,7 @@ static bool isVariadicStringLiteralArgument(const Record *Arg) {
 static void emitClangAttrVariadicIdentifierArgList(const RecordKeeper &Records,
                                                    raw_ostream &OS) {
   OS << "#if defined(CLANG_ATTR_VARIADIC_IDENTIFIER_ARG_LIST)\n";
-  std::map<std::string, FSIVecTy> FSIMap;
+  std::map<StringRef, FSIVecTy> FSIMap;
   for (const auto *A : Records.getAllDerivedDefinitions("Attr")) {
     // Determine whether the first argument is a variadic identifier.
     std::vector<const Record *> Args = A->getValueAsListOfDefs("Args");
@@ -2609,7 +2587,7 @@ emitClangAttrUnevaluatedStringLiteralList(const RecordKeeper &Records,
     return Bits;
   };
 
-  std::map<std::string, FSIVecTy> FSIMap;
+  std::map<StringRef, FSIVecTy> FSIMap;
   for (const auto *Attr : Records.getAllDerivedDefinitions("Attr")) {
     // Determine whether there are any string arguments.
     uint32_t ArgMask = MakeMask(Attr->getValueAsListOfDefs("Args"));
@@ -2625,7 +2603,7 @@ emitClangAttrUnevaluatedStringLiteralList(const RecordKeeper &Records,
 static void emitClangAttrIdentifierArgList(const RecordKeeper &Records,
                                            raw_ostream &OS) {
   OS << "#if defined(CLANG_ATTR_IDENTIFIER_ARG_LIST)\n";
-  std::map<std::string, FSIVecTy> FSIMap;
+  std::map<StringRef, FSIVecTy> FSIMap;
   for (const auto *Attr : Records.getAllDerivedDefinitions("Attr")) {
     // Determine whether the first argument is an identifier.
     std::vector<const Record *> Args = Attr->getValueAsListOfDefs("Args");
@@ -2641,7 +2619,7 @@ static void emitClangAttrIdentifierArgList(const RecordKeeper &Records,
 static void emitClangAttrStrictIdentifierArgList(const RecordKeeper &Records,
                                                  raw_ostream &OS) {
   OS << "#if defined(CLANG_ATTR_STRICT_IDENTIFIER_ARG_LIST)\n";
-  std::map<std::string, FSIVecTy> FSIMap;
+  std::map<StringRef, FSIVecTy> FSIMap;
   for (const auto *Attr : Records.getAllDerivedDefinitions("Attr")) {
     if (!Attr->getValueAsBit("StrictEnumParameters"))
       continue;
@@ -2665,7 +2643,7 @@ static bool keywordThisIsaIdentifierInArgument(const Record *Arg) {
 static void emitClangAttrThisIsaIdentifierArgList(const RecordKeeper &Records,
                                                   raw_ostream &OS) {
   OS << "#if defined(CLANG_ATTR_THIS_ISA_IDENTIFIER_ARG_LIST)\n";
-  std::map<std::string, FSIVecTy> FSIMap;
+  std::map<StringRef, FSIVecTy> FSIMap;
   for (const auto *A : Records.getAllDerivedDefinitions("Attr")) {
     // Determine whether the first argument is a variadic identifier.
     std::vector<const Record *> Args = A->getValueAsListOfDefs("Args");
@@ -2681,7 +2659,7 @@ static void emitClangAttrAcceptsExprPack(const RecordKeeper &Records,
                                          raw_ostream &OS) {
   OS << "#if defined(CLANG_ATTR_ACCEPTS_EXPR_PACK)\n";
   ParsedAttrMap Attrs = getParsedAttrList(Records);
-  std::map<std::string, FSIVecTy> FSIMap;
+  std::map<StringRef, FSIVecTy> FSIMap;
   for (const auto &I : Attrs) {
     const Record &Attr = *I.second;
 
@@ -3678,9 +3656,8 @@ static bool GenerateTargetSpecificAttrChecks(const Record *R,
 }
 
 static void GenerateHasAttrSpellingStringSwitch(
-    const std::vector<std::pair<const Record *, FlattenedSpelling>> &Attrs,
-    raw_ostream &OS, const std::string &Variety,
-    const std::string &Scope = "") {
+    ArrayRef<std::pair<const Record *, FlattenedSpelling>> Attrs,
+    raw_ostream &OS, StringRef Variety, StringRef Scope = "") {
   for (const auto &[Attr, Spelling] : Attrs) {
     // C++11-style attributes have specific version information associated with
     // them. If the attribute has no scope, the version information must not
@@ -3783,18 +3760,15 @@ void EmitClangAttrHasAttrImpl(const RecordKeeper &Records, raw_ostream &OS) {
 
   // Separate all of the attributes out into four group: generic, C++11, GNU,
   // and declspecs. Then generate a big switch statement for each of them.
-  std::vector<std::pair<const Record *, FlattenedSpelling>> Declspec, Microsoft,
-      GNU, Pragma, HLSLAnnotation;
-  std::map<std::string,
-           std::vector<std::pair<const Record *, FlattenedSpelling>>>
-      CXX, C23;
+  using PairTy = std::pair<const Record *, FlattenedSpelling>;
+  std::vector<PairTy> Declspec, Microsoft, GNU, Pragma, HLSLAnnotation;
+  std::map<StringRef, std::vector<PairTy>> CXX, C23;
 
   // Walk over the list of all attributes, and split them out based on the
   // spelling variety.
   for (auto *R : Records.getAllDerivedDefinitions("Attr")) {
-    std::vector<FlattenedSpelling> Spellings = GetFlattenedSpellings(*R);
-    for (const auto &SI : Spellings) {
-      const std::string &Variety = SI.variety();
+    for (const FlattenedSpelling &SI : GetFlattenedSpellings(*R)) {
+      StringRef Variety = SI.variety();
       if (Variety == "GNU")
         GNU.emplace_back(R, SI);
       else if (Variety == "Declspec")
@@ -3829,22 +3803,16 @@ void EmitClangAttrHasAttrImpl(const RecordKeeper &Records, raw_ostream &OS) {
   OS << "case AttributeCommonInfo::Syntax::AS_HLSLAnnotation:\n";
   OS << "  return llvm::StringSwitch<int>(Name)\n";
   GenerateHasAttrSpellingStringSwitch(HLSLAnnotation, OS, "HLSLAnnotation");
-  auto fn = [&OS](const char *Spelling,
-                  const std::map<
-                      std::string,
-                      std::vector<std::pair<const Record *, FlattenedSpelling>>>
-                      &List) {
+  auto fn = [&OS](StringRef Spelling,
+                  const std::map<StringRef, std::vector<PairTy>> &Map) {
     OS << "case AttributeCommonInfo::Syntax::AS_" << Spelling << ": {\n";
     // C++11-style attributes are further split out based on the Scope.
-    for (auto I = List.cbegin(), E = List.cend(); I != E; ++I) {
-      if (I != List.cbegin())
-        OS << " else ";
-      if (I->first.empty())
-        OS << "if (ScopeName == \"\") {\n";
-      else
-        OS << "if (ScopeName == \"" << I->first << "\") {\n";
+    ListSeparator LS(" else ");
+    for (const auto &[Scope, List] : Map) {
+      OS << LS;
+      OS << "if (ScopeName == \"" << Scope << "\") {\n";
       OS << "  return llvm::StringSwitch<int>(Name)\n";
-      GenerateHasAttrSpellingStringSwitch(I->second, OS, Spelling, I->first);
+      GenerateHasAttrSpellingStringSwitch(List, OS, Spelling, Scope);
       OS << "}";
     }
     OS << "\n} break;\n";
@@ -4424,7 +4392,7 @@ static void GenerateMutualExclusionsChecks(const Record &Attr,
     MergeStmtOS << "      auto Iter = llvm::find_if(C, [](const Attr *Check) "
                 << "{ return isa<";
     interleave(
-        StmtAttrs, [&](const std::string &Name) { MergeStmtOS << Name; },
+        StmtAttrs, [&](StringRef Name) { MergeStmtOS << Name; },
         [&] { MergeStmtOS << ", "; });
     MergeStmtOS << ">(Check); });\n";
     MergeStmtOS << "      if (Iter != C.end()) {\n";
@@ -4719,10 +4687,10 @@ void EmitClangAttrParsedAttrImpl(const RecordKeeper &Records, raw_ostream &OS) {
       OS << "static constexpr ParsedAttrInfo::Spelling " << I->first
          << "Spellings[] = {\n";
       for (const auto &S : Spellings) {
-        const std::string &RawSpelling = S.name();
+        StringRef RawSpelling = S.name();
         std::string Spelling;
         if (!S.nameSpace().empty())
-          Spelling += S.nameSpace() + "::";
+          Spelling += S.nameSpace().str() + "::";
         if (S.variety() == "GNU")
           Spelling += NormalizeGNUAttrSpelling(RawSpelling);
         else
@@ -4841,7 +4809,7 @@ void EmitClangAttrParsedAttrKinds(const RecordKeeper &Records,
 
   std::vector<StringMatcher::StringPair> GNU, Declspec, Microsoft, CXX11,
       Keywords, Pragma, C23, HLSLAnnotation;
-  std::set<std::string> Seen;
+  std::set<StringRef> Seen;
   for (const auto *A : Records.getAllDerivedDefinitions("Attr")) {
     const Record &Attr = *A;
 
@@ -4861,38 +4829,41 @@ void EmitClangAttrParsedAttrKinds(const RecordKeeper &Records,
       std::string AttrName;
       if (Attr.isSubClassOf("TargetSpecificAttr") &&
           !Attr.isValueUnset("ParseKind")) {
-        AttrName = std::string(Attr.getValueAsString("ParseKind"));
-        if (!Seen.insert(AttrName).second)
+        StringRef ParseKind = Attr.getValueAsString("ParseKind");
+        if (!Seen.insert(ParseKind).second)
           continue;
-      } else
-        AttrName = NormalizeAttrName(StringRef(Attr.getName())).str();
+        AttrName = ParseKind.str();
+      } else {
+        AttrName = NormalizeAttrName(Attr.getName()).str();
+      }
 
       std::vector<FlattenedSpelling> Spellings = GetFlattenedSpellings(Attr);
       for (const auto &S : Spellings) {
-        const std::string &RawSpelling = S.name();
+        StringRef RawSpelling = S.name();
         std::vector<StringMatcher::StringPair> *Matches = nullptr;
         std::string Spelling;
-        const std::string &Variety = S.variety();
+        StringRef Variety = S.variety();
         if (Variety == "CXX11") {
           Matches = &CXX11;
           if (!S.nameSpace().empty())
-            Spelling += S.nameSpace() + "::";
+            Spelling += S.nameSpace().str() + "::";
         } else if (Variety == "C23") {
           Matches = &C23;
           if (!S.nameSpace().empty())
-            Spelling += S.nameSpace() + "::";
-        } else if (Variety == "GNU")
+            Spelling += S.nameSpace().str() + "::";
+        } else if (Variety == "GNU") {
           Matches = &GNU;
-        else if (Variety == "Declspec")
+        } else if (Variety == "Declspec") {
           Matches = &Declspec;
-        else if (Variety == "Microsoft")
+        } else if (Variety == "Microsoft") {
           Matches = &Microsoft;
-        else if (Variety == "Keyword")
+        } else if (Variety == "Keyword") {
           Matches = &Keywords;
-        else if (Variety == "Pragma")
+        } else if (Variety == "Pragma") {
           Matches = &Pragma;
-        else if (Variety == "HLSLAnnotation")
+        } else if (Variety == "HLSLAnnotation") {
           Matches = &HLSLAnnotation;
+        }
 
         assert(Matches && "Unsupported spelling variety found");
 
@@ -5073,14 +5044,16 @@ public:
             .Case("Pragma", SpellingKind::Pragma)
             .Case("HLSLAnnotation", SpellingKind::HLSLAnnotation);
     std::string Name;
-    if (!Spelling.nameSpace().empty()) {
+    StringRef NameSpace = Spelling.nameSpace();
+    if (!NameSpace.empty()) {
+      Name = NameSpace;
       switch (Kind) {
       case SpellingKind::CXX11:
       case SpellingKind::C23:
-        Name = Spelling.nameSpace() + "::";
+        Name += "::";
         break;
       case SpellingKind::Pragma:
-        Name = Spelling.nameSpace() + " ";
+        Name = " ";
         break;
       default:
         PrintFatalError(Attr.getLoc(), "Unexpected namespace in spelling");
-- 
GitLab


From 6c6351ee350589c8e6bcd69c3255374a714d87d0 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Mon, 28 Oct 2024 11:47:59 -0700
Subject: [PATCH 351/693] [clang][modules] Optimize construction and usage of
 the submodule index (#113391)

This patch avoids eagerly populating the submodule index on `Module`
construction. The `StringMap` allocation shows up in my profiles of
`clang-scan-deps`, while the index is not necessary most of the time. We
still construct it on-demand.

Moreover, this patch avoids performing qualified submodule lookup in
`ASTReader` whenever we're serializing a module graph whose top-level
module is unknown. This is pointless, since that's guaranteed to never
find any existing submodules anyway.

This speeds up `clang-scan-deps` by ~0.5% on my workload.
---
 clang/include/clang/Basic/Module.h    |  3 +--
 clang/include/clang/Lex/ModuleMap.h   | 13 +++++++-----
 clang/lib/Basic/Module.cpp            | 12 ++++++-----
 clang/lib/Lex/ModuleMap.cpp           | 29 ++++++++++++++++-----------
 clang/lib/Serialization/ASTReader.cpp | 14 ++++++++-----
 5 files changed, 42 insertions(+), 29 deletions(-)

diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h
index 1ab3b5e5f815..dd384c1d76c5 100644
--- a/clang/include/clang/Basic/Module.h
+++ b/clang/include/clang/Basic/Module.h
@@ -227,7 +227,7 @@ private:
 
   /// A mapping from the submodule name to the index into the
   /// \c SubModules vector at which that submodule resides.
-  llvm::StringMap<unsigned> SubModuleIndex;
+  mutable llvm::StringMap<unsigned> SubModuleIndex;
 
   /// The AST file if this is a top-level module which has a
   /// corresponding serialized AST file, or null otherwise.
@@ -612,7 +612,6 @@ public:
   void setParent(Module *M) {
     assert(!Parent);
     Parent = M;
-    Parent->SubModuleIndex[Name] = Parent->SubModules.size();
     Parent->SubModules.push_back(this);
   }
 
diff --git a/clang/include/clang/Lex/ModuleMap.h b/clang/include/clang/Lex/ModuleMap.h
index 75b567a347cb..5ee152e4213a 100644
--- a/clang/include/clang/Lex/ModuleMap.h
+++ b/clang/include/clang/Lex/ModuleMap.h
@@ -541,11 +541,14 @@ public:
   ///
   /// \param IsExplicit Whether this is an explicit submodule.
   ///
-  /// \returns The found or newly-created module, along with a boolean value
-  /// that will be true if the module is newly-created.
-  std::pair<Module *, bool> findOrCreateModule(StringRef Name, Module *Parent,
-                                               bool IsFramework,
-                                               bool IsExplicit);
+  /// \returns The found or newly-created module.
+  Module *findOrCreateModule(StringRef Name, Module *Parent, bool IsFramework,
+                             bool IsExplicit);
+  /// Create new submodule, assuming it does not exist. This function can only
+  /// be called when it is guaranteed that this submodule does not exist yet.
+  /// The parameters have same semantics as \c ModuleMap::findOrCreateModule.
+  Module *createModule(StringRef Name, Module *Parent, bool IsFramework,
+                       bool IsExplicit);
 
   /// Create a global module fragment for a C++ module unit.
   ///
diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp
index a7a3f6b37efe..330108d5b3e4 100644
--- a/clang/lib/Basic/Module.cpp
+++ b/clang/lib/Basic/Module.cpp
@@ -54,7 +54,6 @@ Module::Module(ModuleConstructorTag, StringRef Name,
     NoUndeclaredIncludes = Parent->NoUndeclaredIncludes;
     ModuleMapIsPrivate = Parent->ModuleMapIsPrivate;
 
-    Parent->SubModuleIndex[Name] = Parent->SubModules.size();
     Parent->SubModules.push_back(this);
   }
 }
@@ -351,11 +350,14 @@ void Module::markUnavailable(bool Unimportable) {
 }
 
 Module *Module::findSubmodule(StringRef Name) const {
-  llvm::StringMap<unsigned>::const_iterator Pos = SubModuleIndex.find(Name);
-  if (Pos == SubModuleIndex.end())
-    return nullptr;
+  // Add new submodules into the index.
+  for (unsigned I = SubModuleIndex.size(), E = SubModules.size(); I != E; ++I)
+    SubModuleIndex[SubModules[I]->Name] = I;
 
-  return SubModules[Pos->getValue()];
+  if (auto It = SubModuleIndex.find(Name); It != SubModuleIndex.end())
+    return SubModules[It->second];
+
+  return nullptr;
 }
 
 Module *Module::getGlobalModuleFragment() const {
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index 201ab91cf68c..10774429a217 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -655,8 +655,8 @@ ModuleMap::findOrCreateModuleForHeaderInUmbrellaDir(FileEntryRef File) {
         SmallString<32> NameBuf;
         StringRef Name = sanitizeFilenameAsIdentifier(
             llvm::sys::path::stem(SkippedDir.getName()), NameBuf);
-        Result = findOrCreateModule(Name, Result, /*IsFramework=*/false,
-                                    Explicit).first;
+        Result =
+            findOrCreateModule(Name, Result, /*IsFramework=*/false, Explicit);
         setInferredModuleAllowedBy(Result, UmbrellaModuleMap);
 
         // Associate the module and the directory.
@@ -672,8 +672,8 @@ ModuleMap::findOrCreateModuleForHeaderInUmbrellaDir(FileEntryRef File) {
       SmallString<32> NameBuf;
       StringRef Name = sanitizeFilenameAsIdentifier(
                          llvm::sys::path::stem(File.getName()), NameBuf);
-      Result = findOrCreateModule(Name, Result, /*IsFramework=*/false,
-                                  Explicit).first;
+      Result =
+          findOrCreateModule(Name, Result, /*IsFramework=*/false, Explicit);
       setInferredModuleAllowedBy(Result, UmbrellaModuleMap);
       Result->addTopHeader(File);
 
@@ -857,15 +857,21 @@ Module *ModuleMap::lookupModuleQualified(StringRef Name, Module *Context) const{
   return Context->findSubmodule(Name);
 }
 
-std::pair<Module *, bool> ModuleMap::findOrCreateModule(StringRef Name,
-                                                        Module *Parent,
-                                                        bool IsFramework,
-                                                        bool IsExplicit) {
+Module *ModuleMap::findOrCreateModule(StringRef Name, Module *Parent,
+                                      bool IsFramework, bool IsExplicit) {
   // Try to find an existing module with this name.
   if (Module *Sub = lookupModuleQualified(Name, Parent))
-    return std::make_pair(Sub, false);
+    return Sub;
 
   // Create a new module with this name.
+  return createModule(Name, Parent, IsFramework, IsExplicit);
+}
+
+Module *ModuleMap::createModule(StringRef Name, Module *Parent,
+                                bool IsFramework, bool IsExplicit) {
+  assert(lookupModuleQualified(Name, Parent) == nullptr &&
+         "Creating duplicate submodule");
+
   Module *Result = new (ModulesAlloc.Allocate())
       Module(ModuleConstructorTag{}, Name, SourceLocation(), Parent,
              IsFramework, IsExplicit, NumCreatedModules++);
@@ -875,7 +881,7 @@ std::pair<Module *, bool> ModuleMap::findOrCreateModule(StringRef Name,
     Modules[Name] = Result;
     ModuleScopeIDs[Result] = CurrentModuleScopeID;
   }
-  return std::make_pair(Result, true);
+  return Result;
 }
 
 Module *ModuleMap::createGlobalModuleFragmentForModuleUnit(SourceLocation Loc,
@@ -2124,8 +2130,7 @@ void ModuleMapParser::parseModuleDecl() {
         Map.createShadowedModule(ModuleName, Framework, ShadowingModule);
   } else {
     ActiveModule =
-        Map.findOrCreateModule(ModuleName, ActiveModule, Framework, Explicit)
-            .first;
+        Map.findOrCreateModule(ModuleName, ActiveModule, Framework, Explicit);
   }
 
   ActiveModule->DefinitionLoc = ModuleNameLoc;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 2419ed84e68a..74a79ac54bb4 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -5756,6 +5756,13 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F,
     return Err;
 
   ModuleMap &ModMap = PP.getHeaderSearchInfo().getModuleMap();
+  bool KnowsTopLevelModule = ModMap.findModule(F.ModuleName) != nullptr;
+  // If we don't know the top-level module, there's no point in doing qualified
+  // lookup of its submodules; it won't find anything anywhere within this tree.
+  // Let's skip that and avoid some string lookups.
+  auto CreateModule = !KnowsTopLevelModule ? &ModuleMap::createModule
+                                           : &ModuleMap::findOrCreateModule;
+
   bool First = true;
   Module *CurrentModule = nullptr;
   RecordData Record;
@@ -5829,11 +5836,8 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F,
       if (Parent)
         ParentModule = getSubmodule(Parent);
 
-      // Retrieve this (sub)module from the module map, creating it if
-      // necessary.
-      CurrentModule =
-          ModMap.findOrCreateModule(Name, ParentModule, IsFramework, IsExplicit)
-              .first;
+      CurrentModule = std::invoke(CreateModule, &ModMap, Name, ParentModule,
+                                  IsFramework, IsExplicit);
 
       SubmoduleID GlobalIndex = GlobalID - NUM_PREDEF_SUBMODULE_IDS;
       if (GlobalIndex >= SubmodulesLoaded.size() ||
-- 
GitLab


From 5a5b78a84e7214796410265139ecf0266b1fd216 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 28 Oct 2024 18:51:10 +0000
Subject: [PATCH 352/693] [AArch64][GlobalISel] Lower aarch64.neon.smull/umull
 intrinsics.

As with other nodes, we can convert these into G_UMULL and G_SMULL aarch64
instructions.
---
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |  46 ++-
 llvm/test/CodeGen/AArch64/aarch64-smull.ll    | 313 +++++++++++++-----
 2 files changed, 258 insertions(+), 101 deletions(-)

diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index dd65dbe594a6..6024027afaf6 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -1536,6 +1536,14 @@ bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
 
 bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
                                              MachineInstr &MI) const {
+  auto LowerBinOp = [&MI](unsigned Opcode) {
+    MachineIRBuilder MIB(MI);
+    MIB.buildInstr(Opcode, {MI.getOperand(0)},
+                   {MI.getOperand(2), MI.getOperand(3)});
+    MI.eraseFromParent();
+    return true;
+  };
+
   Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
   switch (IntrinsicID) {
   case Intrinsic::vacopy: {
@@ -1675,37 +1683,25 @@ bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
     return true;
   }
   case Intrinsic::aarch64_neon_smax:
+    return LowerBinOp(TargetOpcode::G_SMAX);
   case Intrinsic::aarch64_neon_smin:
+    return LowerBinOp(TargetOpcode::G_SMIN);
   case Intrinsic::aarch64_neon_umax:
+    return LowerBinOp(TargetOpcode::G_UMAX);
   case Intrinsic::aarch64_neon_umin:
+    return LowerBinOp(TargetOpcode::G_UMIN);
   case Intrinsic::aarch64_neon_fmax:
+    return LowerBinOp(TargetOpcode::G_FMAXIMUM);
   case Intrinsic::aarch64_neon_fmin:
+    return LowerBinOp(TargetOpcode::G_FMINIMUM);
   case Intrinsic::aarch64_neon_fmaxnm:
-  case Intrinsic::aarch64_neon_fminnm: {
-    MachineIRBuilder MIB(MI);
-    if (IntrinsicID == Intrinsic::aarch64_neon_smax)
-      MIB.buildSMax(MI.getOperand(0), MI.getOperand(2), MI.getOperand(3));
-    else if (IntrinsicID == Intrinsic::aarch64_neon_smin)
-      MIB.buildSMin(MI.getOperand(0), MI.getOperand(2), MI.getOperand(3));
-    else if (IntrinsicID == Intrinsic::aarch64_neon_umax)
-      MIB.buildUMax(MI.getOperand(0), MI.getOperand(2), MI.getOperand(3));
-    else if (IntrinsicID == Intrinsic::aarch64_neon_umin)
-      MIB.buildUMin(MI.getOperand(0), MI.getOperand(2), MI.getOperand(3));
-    else if (IntrinsicID == Intrinsic::aarch64_neon_fmax)
-      MIB.buildInstr(TargetOpcode::G_FMAXIMUM, {MI.getOperand(0)},
-                     {MI.getOperand(2), MI.getOperand(3)});
-    else if (IntrinsicID == Intrinsic::aarch64_neon_fmin)
-      MIB.buildInstr(TargetOpcode::G_FMINIMUM, {MI.getOperand(0)},
-                     {MI.getOperand(2), MI.getOperand(3)});
-    else if (IntrinsicID == Intrinsic::aarch64_neon_fmaxnm)
-      MIB.buildInstr(TargetOpcode::G_FMAXNUM, {MI.getOperand(0)},
-                     {MI.getOperand(2), MI.getOperand(3)});
-    else if (IntrinsicID == Intrinsic::aarch64_neon_fminnm)
-      MIB.buildInstr(TargetOpcode::G_FMINNUM, {MI.getOperand(0)},
-                     {MI.getOperand(2), MI.getOperand(3)});
-    MI.eraseFromParent();
-    return true;
-  }
+    return LowerBinOp(TargetOpcode::G_FMAXNUM);
+  case Intrinsic::aarch64_neon_fminnm:
+    return LowerBinOp(TargetOpcode::G_FMINNUM);
+  case Intrinsic::aarch64_neon_smull:
+    return LowerBinOp(AArch64::G_UMULL);
+  case Intrinsic::aarch64_neon_umull:
+    return LowerBinOp(AArch64::G_SMULL);
   case Intrinsic::vector_reverse:
     // TODO: Add support for vector_reverse
     return false;
diff --git a/llvm/test/CodeGen/AArch64/aarch64-smull.ll b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
index d677526bab00..11397703b444 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-smull.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
@@ -4,16 +4,7 @@
 ; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 ; CHECK-GI:       warning: Instruction selection used fallback path for pmlsl2_v8i16_uzp1
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for smlsl2_v8i16_uzp1
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for umlsl2_v8i16_uzp1
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for smlsl2_v4i32_uzp1
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for umlsl2_v4i32_uzp1
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for pmlsl_pmlsl2_v8i16_uzp1
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for smlsl_smlsl2_v8i16_uzp1
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for umlsl_umlsl2_v8i16_uzp1
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for smlsl_smlsl2_v4i32_uzp1
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for umlsl_umlsl2_v4i32_uzp1
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for do_stuff
 
 define <8 x i16> @smull_v8i8_v8i16(ptr %A, ptr %B) nounwind {
 ; CHECK-LABEL: smull_v8i8_v8i16:
@@ -2025,13 +2016,30 @@ define void @pmlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3) {
 }
 
 define void @smlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3) {
-; CHECK-LABEL: smlsl2_v8i16_uzp1:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr q2, [x1, #16]
-; CHECK-NEXT:    uzp1 v2.16b, v0.16b, v2.16b
-; CHECK-NEXT:    smlsl2 v1.8h, v0.16b, v2.16b
-; CHECK-NEXT:    str q1, [x0]
-; CHECK-NEXT:    ret
+; CHECK-NEON-LABEL: smlsl2_v8i16_uzp1:
+; CHECK-NEON:       // %bb.0:
+; CHECK-NEON-NEXT:    ldr q2, [x1, #16]
+; CHECK-NEON-NEXT:    uzp1 v2.16b, v0.16b, v2.16b
+; CHECK-NEON-NEXT:    smlsl2 v1.8h, v0.16b, v2.16b
+; CHECK-NEON-NEXT:    str q1, [x0]
+; CHECK-NEON-NEXT:    ret
+;
+; CHECK-SVE-LABEL: smlsl2_v8i16_uzp1:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    ldr q2, [x1, #16]
+; CHECK-SVE-NEXT:    uzp1 v2.16b, v0.16b, v2.16b
+; CHECK-SVE-NEXT:    smlsl2 v1.8h, v0.16b, v2.16b
+; CHECK-SVE-NEXT:    str q1, [x0]
+; CHECK-SVE-NEXT:    ret
+;
+; CHECK-GI-LABEL: smlsl2_v8i16_uzp1:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr q2, [x1, #16]
+; CHECK-GI-NEXT:    mov d0, v0.d[1]
+; CHECK-GI-NEXT:    xtn v2.8b, v2.8h
+; CHECK-GI-NEXT:    umlsl v1.8h, v0.8b, v2.8b
+; CHECK-GI-NEXT:    str q1, [x0]
+; CHECK-GI-NEXT:    ret
   %5 = getelementptr inbounds i32, ptr %3, i64 4
   %6 = load <8 x i16>, ptr %5, align 4
   %7 = trunc <8 x i16> %6 to <8 x i8>
@@ -2043,13 +2051,30 @@ define void @smlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3) {
 }
 
 define void @umlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3) {
-; CHECK-LABEL: umlsl2_v8i16_uzp1:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr q2, [x1, #16]
-; CHECK-NEXT:    uzp1 v2.16b, v0.16b, v2.16b
-; CHECK-NEXT:    umlsl2 v1.8h, v0.16b, v2.16b
-; CHECK-NEXT:    str q1, [x0]
-; CHECK-NEXT:    ret
+; CHECK-NEON-LABEL: umlsl2_v8i16_uzp1:
+; CHECK-NEON:       // %bb.0:
+; CHECK-NEON-NEXT:    ldr q2, [x1, #16]
+; CHECK-NEON-NEXT:    uzp1 v2.16b, v0.16b, v2.16b
+; CHECK-NEON-NEXT:    umlsl2 v1.8h, v0.16b, v2.16b
+; CHECK-NEON-NEXT:    str q1, [x0]
+; CHECK-NEON-NEXT:    ret
+;
+; CHECK-SVE-LABEL: umlsl2_v8i16_uzp1:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    ldr q2, [x1, #16]
+; CHECK-SVE-NEXT:    uzp1 v2.16b, v0.16b, v2.16b
+; CHECK-SVE-NEXT:    umlsl2 v1.8h, v0.16b, v2.16b
+; CHECK-SVE-NEXT:    str q1, [x0]
+; CHECK-SVE-NEXT:    ret
+;
+; CHECK-GI-LABEL: umlsl2_v8i16_uzp1:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr q2, [x1, #16]
+; CHECK-GI-NEXT:    mov d0, v0.d[1]
+; CHECK-GI-NEXT:    xtn v2.8b, v2.8h
+; CHECK-GI-NEXT:    smlsl v1.8h, v0.8b, v2.8b
+; CHECK-GI-NEXT:    str q1, [x0]
+; CHECK-GI-NEXT:    ret
   %5 = getelementptr inbounds i32, ptr %3, i64 4
   %6 = load <8 x i16>, ptr %5, align 4
   %7 = trunc <8 x i16> %6 to <8 x i8>
@@ -2061,13 +2086,30 @@ define void @umlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3) {
 }
 
 define void @smlsl2_v4i32_uzp1(<8 x i16> %0, <4 x i32> %1, ptr %2, ptr %3) {
-; CHECK-LABEL: smlsl2_v4i32_uzp1:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr q2, [x1, #16]
-; CHECK-NEXT:    uzp1 v2.8h, v0.8h, v2.8h
-; CHECK-NEXT:    smlsl2 v1.4s, v0.8h, v2.8h
-; CHECK-NEXT:    str q1, [x0]
-; CHECK-NEXT:    ret
+; CHECK-NEON-LABEL: smlsl2_v4i32_uzp1:
+; CHECK-NEON:       // %bb.0:
+; CHECK-NEON-NEXT:    ldr q2, [x1, #16]
+; CHECK-NEON-NEXT:    uzp1 v2.8h, v0.8h, v2.8h
+; CHECK-NEON-NEXT:    smlsl2 v1.4s, v0.8h, v2.8h
+; CHECK-NEON-NEXT:    str q1, [x0]
+; CHECK-NEON-NEXT:    ret
+;
+; CHECK-SVE-LABEL: smlsl2_v4i32_uzp1:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    ldr q2, [x1, #16]
+; CHECK-SVE-NEXT:    uzp1 v2.8h, v0.8h, v2.8h
+; CHECK-SVE-NEXT:    smlsl2 v1.4s, v0.8h, v2.8h
+; CHECK-SVE-NEXT:    str q1, [x0]
+; CHECK-SVE-NEXT:    ret
+;
+; CHECK-GI-LABEL: smlsl2_v4i32_uzp1:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr q2, [x1, #16]
+; CHECK-GI-NEXT:    mov d0, v0.d[1]
+; CHECK-GI-NEXT:    xtn v2.4h, v2.4s
+; CHECK-GI-NEXT:    umlsl v1.4s, v0.4h, v2.4h
+; CHECK-GI-NEXT:    str q1, [x0]
+; CHECK-GI-NEXT:    ret
   %5 = getelementptr inbounds i32, ptr %3, i64 4
   %6 = load <4 x i32>, ptr %5, align 4
   %7 = trunc <4 x i32> %6 to <4 x i16>
@@ -2079,13 +2121,30 @@ define void @smlsl2_v4i32_uzp1(<8 x i16> %0, <4 x i32> %1, ptr %2, ptr %3) {
 }
 
 define void @umlsl2_v4i32_uzp1(<8 x i16> %0, <4 x i32> %1, ptr %2, ptr %3) {
-; CHECK-LABEL: umlsl2_v4i32_uzp1:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr q2, [x1, #16]
-; CHECK-NEXT:    uzp1 v2.8h, v0.8h, v2.8h
-; CHECK-NEXT:    umlsl2 v1.4s, v0.8h, v2.8h
-; CHECK-NEXT:    str q1, [x0]
-; CHECK-NEXT:    ret
+; CHECK-NEON-LABEL: umlsl2_v4i32_uzp1:
+; CHECK-NEON:       // %bb.0:
+; CHECK-NEON-NEXT:    ldr q2, [x1, #16]
+; CHECK-NEON-NEXT:    uzp1 v2.8h, v0.8h, v2.8h
+; CHECK-NEON-NEXT:    umlsl2 v1.4s, v0.8h, v2.8h
+; CHECK-NEON-NEXT:    str q1, [x0]
+; CHECK-NEON-NEXT:    ret
+;
+; CHECK-SVE-LABEL: umlsl2_v4i32_uzp1:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    ldr q2, [x1, #16]
+; CHECK-SVE-NEXT:    uzp1 v2.8h, v0.8h, v2.8h
+; CHECK-SVE-NEXT:    umlsl2 v1.4s, v0.8h, v2.8h
+; CHECK-SVE-NEXT:    str q1, [x0]
+; CHECK-SVE-NEXT:    ret
+;
+; CHECK-GI-LABEL: umlsl2_v4i32_uzp1:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr q2, [x1, #16]
+; CHECK-GI-NEXT:    mov d0, v0.d[1]
+; CHECK-GI-NEXT:    xtn v2.4h, v2.4s
+; CHECK-GI-NEXT:    smlsl v1.4s, v0.4h, v2.4h
+; CHECK-GI-NEXT:    str q1, [x0]
+; CHECK-GI-NEXT:    ret
   %5 = getelementptr inbounds i32, ptr %3, i64 4
   %6 = load <4 x i32>, ptr %5, align 4
   %7 = trunc <4 x i32> %6 to <4 x i16>
@@ -2124,14 +2183,35 @@ entry:
 }
 
 define void @smlsl_smlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3, i32 %4) {
-; CHECK-LABEL: smlsl_smlsl2_v8i16_uzp1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp q2, q3, [x1]
-; CHECK-NEXT:    uzp1 v2.16b, v2.16b, v3.16b
-; CHECK-NEXT:    smlsl v1.8h, v0.8b, v2.8b
-; CHECK-NEXT:    smlsl2 v1.8h, v0.16b, v2.16b
-; CHECK-NEXT:    str q1, [x0]
-; CHECK-NEXT:    ret
+; CHECK-NEON-LABEL: smlsl_smlsl2_v8i16_uzp1:
+; CHECK-NEON:       // %bb.0: // %entry
+; CHECK-NEON-NEXT:    ldp q2, q3, [x1]
+; CHECK-NEON-NEXT:    uzp1 v2.16b, v2.16b, v3.16b
+; CHECK-NEON-NEXT:    smlsl v1.8h, v0.8b, v2.8b
+; CHECK-NEON-NEXT:    smlsl2 v1.8h, v0.16b, v2.16b
+; CHECK-NEON-NEXT:    str q1, [x0]
+; CHECK-NEON-NEXT:    ret
+;
+; CHECK-SVE-LABEL: smlsl_smlsl2_v8i16_uzp1:
+; CHECK-SVE:       // %bb.0: // %entry
+; CHECK-SVE-NEXT:    ldp q2, q3, [x1]
+; CHECK-SVE-NEXT:    uzp1 v2.16b, v2.16b, v3.16b
+; CHECK-SVE-NEXT:    smlsl v1.8h, v0.8b, v2.8b
+; CHECK-SVE-NEXT:    smlsl2 v1.8h, v0.16b, v2.16b
+; CHECK-SVE-NEXT:    str q1, [x0]
+; CHECK-SVE-NEXT:    ret
+;
+; CHECK-GI-LABEL: smlsl_smlsl2_v8i16_uzp1:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldp q4, q2, [x1]
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    xtn v2.8b, v2.8h
+; CHECK-GI-NEXT:    xtn v4.8b, v4.8h
+; CHECK-GI-NEXT:    umull v2.8h, v3.8b, v2.8b
+; CHECK-GI-NEXT:    umlal v2.8h, v0.8b, v4.8b
+; CHECK-GI-NEXT:    sub v0.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT:    str q0, [x0]
+; CHECK-GI-NEXT:    ret
 entry:
   %5 = load <8 x i16>, ptr %3, align 4
   %6 = trunc <8 x i16> %5 to <8 x i8>
@@ -2149,14 +2229,35 @@ entry:
 }
 
 define void @umlsl_umlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3, i32 %4) {
-; CHECK-LABEL: umlsl_umlsl2_v8i16_uzp1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp q2, q3, [x1]
-; CHECK-NEXT:    uzp1 v2.16b, v2.16b, v3.16b
-; CHECK-NEXT:    umlsl v1.8h, v0.8b, v2.8b
-; CHECK-NEXT:    umlsl2 v1.8h, v0.16b, v2.16b
-; CHECK-NEXT:    str q1, [x0]
-; CHECK-NEXT:    ret
+; CHECK-NEON-LABEL: umlsl_umlsl2_v8i16_uzp1:
+; CHECK-NEON:       // %bb.0: // %entry
+; CHECK-NEON-NEXT:    ldp q2, q3, [x1]
+; CHECK-NEON-NEXT:    uzp1 v2.16b, v2.16b, v3.16b
+; CHECK-NEON-NEXT:    umlsl v1.8h, v0.8b, v2.8b
+; CHECK-NEON-NEXT:    umlsl2 v1.8h, v0.16b, v2.16b
+; CHECK-NEON-NEXT:    str q1, [x0]
+; CHECK-NEON-NEXT:    ret
+;
+; CHECK-SVE-LABEL: umlsl_umlsl2_v8i16_uzp1:
+; CHECK-SVE:       // %bb.0: // %entry
+; CHECK-SVE-NEXT:    ldp q2, q3, [x1]
+; CHECK-SVE-NEXT:    uzp1 v2.16b, v2.16b, v3.16b
+; CHECK-SVE-NEXT:    umlsl v1.8h, v0.8b, v2.8b
+; CHECK-SVE-NEXT:    umlsl2 v1.8h, v0.16b, v2.16b
+; CHECK-SVE-NEXT:    str q1, [x0]
+; CHECK-SVE-NEXT:    ret
+;
+; CHECK-GI-LABEL: umlsl_umlsl2_v8i16_uzp1:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldp q4, q2, [x1]
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    xtn v2.8b, v2.8h
+; CHECK-GI-NEXT:    xtn v4.8b, v4.8h
+; CHECK-GI-NEXT:    smull v2.8h, v3.8b, v2.8b
+; CHECK-GI-NEXT:    smlal v2.8h, v0.8b, v4.8b
+; CHECK-GI-NEXT:    sub v0.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT:    str q0, [x0]
+; CHECK-GI-NEXT:    ret
 entry:
   %5 = load <8 x i16>, ptr %3, align 4
   %6 = trunc <8 x i16> %5 to <8 x i8>
@@ -2174,14 +2275,35 @@ entry:
 }
 
 define void @smlsl_smlsl2_v4i32_uzp1(<8 x i16> %0, <4 x i32> %1, ptr %2, ptr %3, i32 %4) {
-; CHECK-LABEL: smlsl_smlsl2_v4i32_uzp1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp q2, q3, [x1]
-; CHECK-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
-; CHECK-NEXT:    smlsl v1.4s, v0.4h, v2.4h
-; CHECK-NEXT:    smlsl2 v1.4s, v0.8h, v2.8h
-; CHECK-NEXT:    str q1, [x0]
-; CHECK-NEXT:    ret
+; CHECK-NEON-LABEL: smlsl_smlsl2_v4i32_uzp1:
+; CHECK-NEON:       // %bb.0: // %entry
+; CHECK-NEON-NEXT:    ldp q2, q3, [x1]
+; CHECK-NEON-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-NEON-NEXT:    smlsl v1.4s, v0.4h, v2.4h
+; CHECK-NEON-NEXT:    smlsl2 v1.4s, v0.8h, v2.8h
+; CHECK-NEON-NEXT:    str q1, [x0]
+; CHECK-NEON-NEXT:    ret
+;
+; CHECK-SVE-LABEL: smlsl_smlsl2_v4i32_uzp1:
+; CHECK-SVE:       // %bb.0: // %entry
+; CHECK-SVE-NEXT:    ldp q2, q3, [x1]
+; CHECK-SVE-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-SVE-NEXT:    smlsl v1.4s, v0.4h, v2.4h
+; CHECK-SVE-NEXT:    smlsl2 v1.4s, v0.8h, v2.8h
+; CHECK-SVE-NEXT:    str q1, [x0]
+; CHECK-SVE-NEXT:    ret
+;
+; CHECK-GI-LABEL: smlsl_smlsl2_v4i32_uzp1:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldp q4, q2, [x1]
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    xtn v2.4h, v2.4s
+; CHECK-GI-NEXT:    xtn v4.4h, v4.4s
+; CHECK-GI-NEXT:    umull v2.4s, v3.4h, v2.4h
+; CHECK-GI-NEXT:    umlal v2.4s, v0.4h, v4.4h
+; CHECK-GI-NEXT:    sub v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    str q0, [x0]
+; CHECK-GI-NEXT:    ret
 entry:
   %5 = load <4 x i32>, ptr %3, align 4
   %6 = trunc <4 x i32> %5 to <4 x i16>
@@ -2199,14 +2321,35 @@ entry:
 }
 
 define void @umlsl_umlsl2_v4i32_uzp1(<8 x i16> %0, <4 x i32> %1, ptr %2, ptr %3, i32 %4) {
-; CHECK-LABEL: umlsl_umlsl2_v4i32_uzp1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp q2, q3, [x1]
-; CHECK-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
-; CHECK-NEXT:    umlsl v1.4s, v0.4h, v2.4h
-; CHECK-NEXT:    umlsl2 v1.4s, v0.8h, v2.8h
-; CHECK-NEXT:    str q1, [x0]
-; CHECK-NEXT:    ret
+; CHECK-NEON-LABEL: umlsl_umlsl2_v4i32_uzp1:
+; CHECK-NEON:       // %bb.0: // %entry
+; CHECK-NEON-NEXT:    ldp q2, q3, [x1]
+; CHECK-NEON-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-NEON-NEXT:    umlsl v1.4s, v0.4h, v2.4h
+; CHECK-NEON-NEXT:    umlsl2 v1.4s, v0.8h, v2.8h
+; CHECK-NEON-NEXT:    str q1, [x0]
+; CHECK-NEON-NEXT:    ret
+;
+; CHECK-SVE-LABEL: umlsl_umlsl2_v4i32_uzp1:
+; CHECK-SVE:       // %bb.0: // %entry
+; CHECK-SVE-NEXT:    ldp q2, q3, [x1]
+; CHECK-SVE-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-SVE-NEXT:    umlsl v1.4s, v0.4h, v2.4h
+; CHECK-SVE-NEXT:    umlsl2 v1.4s, v0.8h, v2.8h
+; CHECK-SVE-NEXT:    str q1, [x0]
+; CHECK-SVE-NEXT:    ret
+;
+; CHECK-GI-LABEL: umlsl_umlsl2_v4i32_uzp1:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldp q4, q2, [x1]
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    xtn v2.4h, v2.4s
+; CHECK-GI-NEXT:    xtn v4.4h, v4.4s
+; CHECK-GI-NEXT:    smull v2.4s, v3.4h, v2.4h
+; CHECK-GI-NEXT:    smlal v2.4s, v0.4h, v4.4h
+; CHECK-GI-NEXT:    sub v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    str q0, [x0]
+; CHECK-GI-NEXT:    ret
 entry:
   %5 = load <4 x i32>, ptr %3, align 4
   %6 = trunc <4 x i32> %5 to <4 x i16>
@@ -2224,13 +2367,31 @@ entry:
 }
 
 define <2 x i32> @do_stuff(<2 x i64> %0, <2 x i64> %1) {
-; CHECK-LABEL: do_stuff:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v0.4s
-; CHECK-NEXT:    smull2 v0.2d, v1.4s, v0.4s
-; CHECK-NEXT:    xtn v0.2s, v0.2d
-; CHECK-NEXT:    add v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    ret
+; CHECK-NEON-LABEL: do_stuff:
+; CHECK-NEON:       // %bb.0:
+; CHECK-NEON-NEXT:    uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEON-NEXT:    smull2 v0.2d, v1.4s, v0.4s
+; CHECK-NEON-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEON-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-NEON-NEXT:    ret
+;
+; CHECK-SVE-LABEL: do_stuff:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-SVE-NEXT:    smull2 v0.2d, v1.4s, v0.4s
+; CHECK-SVE-NEXT:    xtn v0.2s, v0.2d
+; CHECK-SVE-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-SVE-NEXT:    ret
+;
+; CHECK-GI-LABEL: do_stuff:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    ext v2.16b, v1.16b, v2.16b, #8
+; CHECK-GI-NEXT:    umull v0.2d, v2.2s, v0.2s
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
   %bc.1 = bitcast <2 x i64> %1 to <4 x i32>
   %trunc.0 = trunc <2 x i64> %0 to <2 x i32>
   %shuff.hi = shufflevector <4 x i32> %bc.1, <4 x i32> zeroinitializer, <2 x i32> <i32 2, i32 3>
-- 
GitLab


From b54bc104ea87e301816b450ee117d2d864c7d82d Mon Sep 17 00:00:00 2001
From: jimingham <jingham@apple.com>
Date: Mon, 28 Oct 2024 11:52:32 -0700
Subject: [PATCH 353/693] =?UTF-8?q?Revert=20"Add=20the=20ability=20to=20br?=
 =?UTF-8?q?eak=20on=20call-site=20locations,=20improve=20inli=E2=80=A6=20(?=
 =?UTF-8?q?#113947)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…ne stepping (#112939)"

This was breaking some gdb-remote packet counting tests on the bots. I
can't see how this patch could cause that breakage, but I'm reverting to
figure that out.

This reverts commit f14743794587db102c6d1b20f9c87a1ac20decfd.
---
 .../lldb/Breakpoint/BreakpointLocation.h      |  36 ----
 lldb/include/lldb/Breakpoint/BreakpointSite.h |   5 -
 lldb/include/lldb/Core/Declaration.h          |   6 +-
 lldb/include/lldb/Target/StopInfo.h           |  12 --
 .../lldb/Target/ThreadPlanStepInRange.h       |   4 +-
 lldb/source/Breakpoint/BreakpointLocation.cpp |  63 +------
 lldb/source/Breakpoint/BreakpointResolver.cpp |  15 --
 lldb/source/Breakpoint/BreakpointSite.cpp     |  17 --
 lldb/source/Core/Declaration.cpp              |   5 +-
 lldb/source/Symbol/Block.cpp                  |   2 +-
 lldb/source/Symbol/CompileUnit.cpp            | 111 +-----------
 lldb/source/Target/StackFrameList.cpp         | 171 ++++++++++++------
 lldb/source/Target/StopInfo.cpp               |  55 ------
 lldb/source/Target/Thread.cpp                 |   8 -
 lldb/source/Target/ThreadPlanStepInRange.cpp  |  24 +--
 .../source/Target/ThreadPlanStepOverRange.cpp |   2 +-
 .../inline-stepping/TestInlineStepping.py     |  63 -------
 .../inline-stepping/calling.cpp               |  25 ---
 18 files changed, 131 insertions(+), 493 deletions(-)

diff --git a/lldb/include/lldb/Breakpoint/BreakpointLocation.h b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
index 3592291bb2d0..cca00335bc3c 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointLocation.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
@@ -11,12 +11,10 @@
 
 #include <memory>
 #include <mutex>
-#include <optional>
 
 #include "lldb/Breakpoint/BreakpointOptions.h"
 #include "lldb/Breakpoint/StoppointHitCounter.h"
 #include "lldb/Core/Address.h"
-#include "lldb/Symbol/LineEntry.h"
 #include "lldb/Utility/UserID.h"
 #include "lldb/lldb-private.h"
 
@@ -284,25 +282,6 @@ public:
   /// Returns the breakpoint location ID.
   lldb::break_id_t GetID() const { return m_loc_id; }
 
-  /// Set the line entry that should be shown to users for this location.
-  /// It is up to the caller to verify that this is a valid entry to show.
-  /// The current use of this is to distinguish among line entries from a
-  /// virtual inlined call stack that all share the same address.
-  /// The line entry must have the same start address as the address for this
-  /// location.
-  bool SetPreferredLineEntry(const LineEntry &line_entry) {
-    if (m_address == line_entry.range.GetBaseAddress()) {
-      m_preferred_line_entry = line_entry;
-      return true;
-    }
-    assert(0 && "Tried to set a preferred line entry with a different address");
-    return false;
-  }
-
-  const std::optional<LineEntry> GetPreferredLineEntry() {
-    return m_preferred_line_entry;
-  }
-
 protected:
   friend class BreakpointSite;
   friend class BreakpointLocationList;
@@ -327,16 +306,6 @@ protected:
   /// If it returns false we should continue, otherwise stop.
   bool IgnoreCountShouldStop();
 
-  /// If this location knows that the virtual stack frame it represents is
-  /// not frame 0, return the suggested stack frame instead.  This will happen
-  /// when the location's address contains a "virtual inlined call stack" and
-  /// the breakpoint was set on a file & line that are not at the bottom of that
-  /// stack.  For now we key off the "preferred line entry" - looking for that
-  /// in the blocks that start with the stop PC.
-  /// This version of the API doesn't take an "inlined" parameter because it
-  /// only changes frames in the inline stack.
-  std::optional<uint32_t> GetSuggestedStackFrameIndex();
-
 private:
   void SwapLocation(lldb::BreakpointLocationSP swap_from);
 
@@ -400,11 +369,6 @@ private:
   lldb::break_id_t m_loc_id; ///< Breakpoint location ID.
   StoppointHitCounter m_hit_counter; ///< Number of times this breakpoint
                                      /// location has been hit.
-  /// If this exists, use it to print the stop description rather than the
-  /// LineEntry m_address resolves to directly.  Use this for instance when the
-  /// location was given somewhere in the virtual inlined call stack since the
-  /// Address always resolves to the lowest entry in the stack.
-  std::optional<LineEntry> m_preferred_line_entry;
 
   void SetShouldResolveIndirectFunctions(bool do_resolve) {
     m_should_resolve_indirect_functions = do_resolve;
diff --git a/lldb/include/lldb/Breakpoint/BreakpointSite.h b/lldb/include/lldb/Breakpoint/BreakpointSite.h
index 7b3f7be23639..17b76d51c1ae 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointSite.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointSite.h
@@ -170,11 +170,6 @@ public:
   /// \see lldb::DescriptionLevel
   void GetDescription(Stream *s, lldb::DescriptionLevel level);
 
-  // This runs through all the breakpoint locations owning this site and returns
-  // the greatest of their suggested stack frame indexes.  This only handles
-  // inlined stack changes.
-  std::optional<uint32_t> GetSuggestedStackFrameIndex();
-
   /// Tell whether a breakpoint has a location at this site.
   ///
   /// \param[in] bp_id
diff --git a/lldb/include/lldb/Core/Declaration.h b/lldb/include/lldb/Core/Declaration.h
index c864b88c6b32..4a0e9047b546 100644
--- a/lldb/include/lldb/Core/Declaration.h
+++ b/lldb/include/lldb/Core/Declaration.h
@@ -84,14 +84,10 @@ public:
   /// \param[in] declaration
   ///     The const Declaration object to compare with.
   ///
-  /// \param[in] full
-  ///     Same meaning as Full in FileSpec::Equal.  True means an empty
-  ///     directory is not equal to a specified one, false means it is equal.
-  ///
   /// \return
   ///     Returns \b true if \b declaration is at the same file and
   ///     line, \b false otherwise.
-  bool FileAndLineEqual(const Declaration &declaration, bool full) const;
+  bool FileAndLineEqual(const Declaration &declaration) const;
 
   /// Dump a description of this object to a Stream.
   ///
diff --git a/lldb/include/lldb/Target/StopInfo.h b/lldb/include/lldb/Target/StopInfo.h
index 45beac129e86..fae90364deaf 100644
--- a/lldb/include/lldb/Target/StopInfo.h
+++ b/lldb/include/lldb/Target/StopInfo.h
@@ -77,18 +77,6 @@ public:
       m_description.clear();
   }
 
-  /// This gives the StopInfo a chance to suggest a stack frame to select.
-  /// Passing true for inlined_stack will request changes to the inlined
-  /// call stack.  Passing false will request changes to the real stack
-  /// frame.  The inlined stack gets adjusted before we call into the thread
-  /// plans so they can reason based on the correct values.  The real stack
-  /// adjustment is handled after the frame recognizers get a chance to adjust
-  /// the frame.
-  virtual std::optional<uint32_t>
-  GetSuggestedStackFrameIndex(bool inlined_stack) {
-    return {};
-  }
-
   virtual bool IsValidForOperatingSystemThread(Thread &thread) { return true; }
 
   /// A Continue operation can result in a false stop event
diff --git a/lldb/include/lldb/Target/ThreadPlanStepInRange.h b/lldb/include/lldb/Target/ThreadPlanStepInRange.h
index 9da8370ef1c9..f9ef87942a7c 100644
--- a/lldb/include/lldb/Target/ThreadPlanStepInRange.h
+++ b/lldb/include/lldb/Target/ThreadPlanStepInRange.h
@@ -80,8 +80,8 @@ private:
   bool m_step_past_prologue; // FIXME: For now hard-coded to true, we could put
                              // a switch in for this if there's
                              // demand for that.
-  LazyBool m_virtual_step;   // true if we've just done a "virtual step", i.e.
-                             // just moved the inline stack depth.
+  bool m_virtual_step; // true if we've just done a "virtual step", i.e. just
+                       // moved the inline stack depth.
   ConstString m_step_into_target;
   ThreadPlanStepInRange(const ThreadPlanStepInRange &) = delete;
   const ThreadPlanStepInRange &
diff --git a/lldb/source/Breakpoint/BreakpointLocation.cpp b/lldb/source/Breakpoint/BreakpointLocation.cpp
index c7ea50407ae1..ad9057c8141e 100644
--- a/lldb/source/Breakpoint/BreakpointLocation.cpp
+++ b/lldb/source/Breakpoint/BreakpointLocation.cpp
@@ -508,20 +508,8 @@ void BreakpointLocation::GetDescription(Stream *s,
         s->PutCString("re-exported target = ");
       else
         s->PutCString("where = ");
-
-      // If there's a preferred line entry for printing, use that.
-      bool show_function_info = true;
-      if (auto preferred = GetPreferredLineEntry()) {
-        sc.line_entry = *preferred;
-        // FIXME: We're going to get the function name wrong when the preferred
-        // line entry is not the lowest one.  For now, just leave the function
-        // out in this case, but we really should also figure out how to easily
-        // fake the function name here.
-        show_function_info = false;
-      }
       sc.DumpStopContext(s, m_owner.GetTarget().GetProcessSP().get(), m_address,
-                         false, true, false, show_function_info,
-                         show_function_info, show_function_info);
+                         false, true, false, true, true, true);
     } else {
       if (sc.module_sp) {
         s->EOL();
@@ -549,10 +537,7 @@ void BreakpointLocation::GetDescription(Stream *s,
         if (sc.line_entry.line > 0) {
           s->EOL();
           s->Indent("location = ");
-          if (auto preferred = GetPreferredLineEntry())
-            preferred->DumpStopContext(s, true);
-          else
-            sc.line_entry.DumpStopContext(s, true);
+          sc.line_entry.DumpStopContext(s, true);
         }
 
       } else {
@@ -671,50 +656,6 @@ void BreakpointLocation::SendBreakpointLocationChangedEvent(
   }
 }
 
-std::optional<uint32_t> BreakpointLocation::GetSuggestedStackFrameIndex() {
-  auto preferred_opt = GetPreferredLineEntry();
-  if (!preferred_opt)
-    return {};
-  LineEntry preferred = *preferred_opt;
-  SymbolContext sc;
-  if (!m_address.CalculateSymbolContext(&sc))
-    return {};
-  // Don't return anything special if frame 0 is the preferred line entry.
-  // We not really telling the stack frame list to do anything special in that
-  // case.
-  if (!LineEntry::Compare(sc.line_entry, preferred))
-    return {};
-
-  if (!sc.block)
-    return {};
-
-  // Blocks have their line info in Declaration form, so make one here:
-  Declaration preferred_decl(preferred.GetFile(), preferred.line,
-                             preferred.column);
-
-  uint32_t depth = 0;
-  Block *inlined_block = sc.block->GetContainingInlinedBlock();
-  while (inlined_block) {
-    // If we've moved to a block that this isn't the start of, that's not
-    // our inlining info or call site, so we can stop here.
-    Address start_address;
-    if (!inlined_block->GetStartAddress(start_address) ||
-        start_address != m_address)
-      return {};
-
-    const InlineFunctionInfo *info = inlined_block->GetInlinedFunctionInfo();
-    if (info) {
-      if (preferred_decl == info->GetDeclaration())
-        return depth;
-      if (preferred_decl == info->GetCallSite())
-        return depth + 1;
-    }
-    inlined_block = inlined_block->GetInlinedParent();
-    depth++;
-  }
-  return {};
-}
-
 void BreakpointLocation::SwapLocation(BreakpointLocationSP swap_from) {
   m_address = swap_from->m_address;
   m_should_resolve_indirect_functions =
diff --git a/lldb/source/Breakpoint/BreakpointResolver.cpp b/lldb/source/Breakpoint/BreakpointResolver.cpp
index 9643602d78c7..8307689c7640 100644
--- a/lldb/source/Breakpoint/BreakpointResolver.cpp
+++ b/lldb/source/Breakpoint/BreakpointResolver.cpp
@@ -340,21 +340,6 @@ void BreakpointResolver::AddLocation(SearchFilter &filter,
   }
 
   BreakpointLocationSP bp_loc_sp(AddLocation(line_start));
-  // If the address that we resolved the location to returns a different
-  // LineEntry from the one in the incoming SC, we're probably dealing with an
-  // inlined call site, so set that as the preferred LineEntry:
-  LineEntry resolved_entry;
-  if (!skipped_prologue && bp_loc_sp &&
-      line_start.CalculateSymbolContextLineEntry(resolved_entry) &&
-      LineEntry::Compare(resolved_entry, sc.line_entry)) {
-    // FIXME: The function name will also be wrong here.  Do we need to record
-    // that as well, or can we figure that out again when we report this
-    // breakpoint location.
-    if (!bp_loc_sp->SetPreferredLineEntry(sc.line_entry)) {
-      LLDB_LOG(log, "Tried to add a preferred line entry that didn't have the "
-                    "same address as this location's address.");
-    }
-  }
   if (log && bp_loc_sp && !GetBreakpoint()->IsInternal()) {
     StreamString s;
     bp_loc_sp->GetDescription(&s, lldb::eDescriptionLevelVerbose);
diff --git a/lldb/source/Breakpoint/BreakpointSite.cpp b/lldb/source/Breakpoint/BreakpointSite.cpp
index 9700a57d3346..3ca93f908e30 100644
--- a/lldb/source/Breakpoint/BreakpointSite.cpp
+++ b/lldb/source/Breakpoint/BreakpointSite.cpp
@@ -87,23 +87,6 @@ void BreakpointSite::GetDescription(Stream *s, lldb::DescriptionLevel level) {
   m_constituents.GetDescription(s, level);
 }
 
-std::optional<uint32_t> BreakpointSite::GetSuggestedStackFrameIndex() {
-
-  std::optional<uint32_t> result;
-  std::lock_guard<std::recursive_mutex> guard(m_constituents_mutex);
-  for (BreakpointLocationSP loc_sp : m_constituents.BreakpointLocations()) {
-    std::optional<uint32_t> loc_frame_index =
-        loc_sp->GetSuggestedStackFrameIndex();
-    if (loc_frame_index) {
-      if (result)
-        result = std::max(*loc_frame_index, *result);
-      else
-        result = loc_frame_index;
-    }
-  }
-  return result;
-}
-
 bool BreakpointSite::IsInternal() const { return m_constituents.IsInternal(); }
 
 uint8_t *BreakpointSite::GetTrapOpcodeBytes() { return &m_trap_opcode[0]; }
diff --git a/lldb/source/Core/Declaration.cpp b/lldb/source/Core/Declaration.cpp
index a485c4b9ba48..579a3999d14e 100644
--- a/lldb/source/Core/Declaration.cpp
+++ b/lldb/source/Core/Declaration.cpp
@@ -70,9 +70,8 @@ int Declaration::Compare(const Declaration &a, const Declaration &b) {
   return 0;
 }
 
-bool Declaration::FileAndLineEqual(const Declaration &declaration,
-                                   bool full) const {
-  int file_compare = FileSpec::Compare(this->m_file, declaration.m_file, full);
+bool Declaration::FileAndLineEqual(const Declaration &declaration) const {
+  int file_compare = FileSpec::Compare(this->m_file, declaration.m_file, true);
   return file_compare == 0 && this->m_line == declaration.m_line;
 }
 
diff --git a/lldb/source/Symbol/Block.cpp b/lldb/source/Symbol/Block.cpp
index 5c7772a6db78..f7d9c0d2d330 100644
--- a/lldb/source/Symbol/Block.cpp
+++ b/lldb/source/Symbol/Block.cpp
@@ -230,7 +230,7 @@ Block *Block::GetContainingInlinedBlockWithCallSite(
     const auto *function_info = inlined_block->GetInlinedFunctionInfo();
 
     if (function_info &&
-        function_info->GetCallSite().FileAndLineEqual(find_call_site, true))
+        function_info->GetCallSite().FileAndLineEqual(find_call_site))
       return inlined_block;
     inlined_block = inlined_block->GetInlinedParent();
   }
diff --git a/lldb/source/Symbol/CompileUnit.cpp b/lldb/source/Symbol/CompileUnit.cpp
index f0f7e40ae70d..db8f8ce6bcbc 100644
--- a/lldb/source/Symbol/CompileUnit.cpp
+++ b/lldb/source/Symbol/CompileUnit.cpp
@@ -251,10 +251,7 @@ void CompileUnit::ResolveSymbolContext(
     SymbolContextItem resolve_scope, SymbolContextList &sc_list,
     RealpathPrefixes *realpath_prefixes) {
   const FileSpec file_spec = src_location_spec.GetFileSpec();
-  const uint32_t line =
-      src_location_spec.GetLine().value_or(LLDB_INVALID_LINE_NUMBER);
-  const uint32_t column_num =
-      src_location_spec.GetColumn().value_or(LLDB_INVALID_COLUMN_NUMBER);
+  const uint32_t line = src_location_spec.GetLine().value_or(0);
   const bool check_inlines = src_location_spec.GetCheckInlines();
 
   // First find all of the file indexes that match our "file_spec". If
@@ -315,112 +312,6 @@ void CompileUnit::ResolveSymbolContext(
         0, file_indexes, src_location_spec, &line_entry);
   }
 
-  // If we didn't manage to find a breakpoint that matched the line number
-  // requested, that might be because it is only an inline call site, and
-  // doesn't have a line entry in the line table.  Scan for that here.
-  //
-  // We are making the assumption that if there was an inlined function it will
-  // contribute at least 1 non-call-site entry to the line table.  That's handy
-  // because we don't move line breakpoints over function boundaries, so if we
-  // found a hit, and there were also a call site entry, it would have to be in
-  // the function containing the PC of the line table match.  That way we can
-  // limit the call site search to that function.
-  // We will miss functions that ONLY exist as a call site entry.
-
-  if (line_entry.IsValid() &&
-      (line_entry.line != line || line_entry.column != column_num) &&
-      resolve_scope & eSymbolContextLineEntry && check_inlines) {
-    // We don't move lines over function boundaries, so the address in the
-    // line entry will be the in function that contained the line that might
-    // be a CallSite, and we can just iterate over that function to find any
-    // inline records, and dig up their call sites.
-    Address start_addr = line_entry.range.GetBaseAddress();
-    Function *function = start_addr.CalculateSymbolContextFunction();
-
-    Declaration sought_decl(file_spec, line, column_num);
-    // We use this recursive function to descend the block structure looking
-    // for a block that has this Declaration as in it's CallSite info.
-    // This function recursively scans the sibling blocks of the incoming
-    // block parameter.
-    std::function<void(Block &)> examine_block =
-        [&sought_decl, &sc_list, &src_location_spec, resolve_scope,
-         &examine_block](Block &block) -> void {
-      // Iterate over the sibling child blocks of the incoming block.
-      Block *sibling_block = block.GetFirstChild();
-      while (sibling_block) {
-        // We only have to descend through the regular blocks, looking for
-        // immediate inlines, since those are the only ones that will have this
-        // callsite.
-        const InlineFunctionInfo *inline_info =
-            sibling_block->GetInlinedFunctionInfo();
-        if (inline_info) {
-          // If this is the call-site we are looking for, record that:
-          // We need to be careful because the call site from the debug info
-          // will generally have a column, but the user might not have specified
-          // it.
-          Declaration found_decl = inline_info->GetCallSite();
-          uint32_t sought_column = sought_decl.GetColumn();
-          if (found_decl.FileAndLineEqual(sought_decl, false) &&
-              (sought_column == LLDB_INVALID_COLUMN_NUMBER ||
-               sought_column == found_decl.GetColumn())) {
-            // If we found a call site, it belongs not in this inlined block,
-            // but in the parent block that inlined it.
-            Address parent_start_addr;
-            if (sibling_block->GetParent()->GetStartAddress(
-                    parent_start_addr)) {
-              SymbolContext sc;
-              parent_start_addr.CalculateSymbolContext(&sc, resolve_scope);
-              // Now swap out the line entry for the one we found.
-              LineEntry call_site_line = sc.line_entry;
-              call_site_line.line = found_decl.GetLine();
-              call_site_line.column = found_decl.GetColumn();
-              bool matches_spec = true;
-              // If the user asked for an exact match, we need to make sure the
-              // call site we found actually matches the location.
-              if (src_location_spec.GetExactMatch()) {
-                matches_spec = false;
-                if ((src_location_spec.GetFileSpec() ==
-                     sc.line_entry.GetFile()) &&
-                    (src_location_spec.GetLine() &&
-                     *src_location_spec.GetLine() == call_site_line.line) &&
-                    (src_location_spec.GetColumn() &&
-                     *src_location_spec.GetColumn() == call_site_line.column))
-                  matches_spec = true;
-              }
-              if (matches_spec &&
-                  sibling_block->GetRangeAtIndex(0, call_site_line.range)) {
-                SymbolContext call_site_sc(sc.target_sp, sc.module_sp,
-                                           sc.comp_unit, sc.function, sc.block,
-                                           &call_site_line, sc.symbol);
-                sc_list.Append(call_site_sc);
-              }
-            }
-          }
-        }
-
-        // Descend into the child blocks:
-        examine_block(*sibling_block);
-        // Now go to the next sibling:
-        sibling_block = sibling_block->GetSibling();
-      }
-    };
-
-    if (function) {
-      // We don't need to examine the function block, it can't be inlined.
-      Block &func_block = function->GetBlock(true);
-      examine_block(func_block);
-    }
-    // If we found entries here, we are done.  We only get here because we
-    // didn't find an exact line entry for this line & column, but if we found
-    // an exact match from the call site info that's strictly better than
-    // continuing to look for matches further on in the file.
-    // FIXME: Should I also do this for "call site line exists between the
-    // given line number and the later line we found in the line table"?  That's
-    // a closer approximation to our general sliding algorithm.
-    if (sc_list.GetSize())
-      return;
-  }
-
   // If "exact == true", then "found_line" will be the same as "line". If
   // "exact == false", the "found_line" will be the closest line entry
   // with a line number greater than "line" and we will use this for our
diff --git a/lldb/source/Target/StackFrameList.cpp b/lldb/source/Target/StackFrameList.cpp
index 94a381edd5e2..3849ec5ed178 100644
--- a/lldb/source/Target/StackFrameList.cpp
+++ b/lldb/source/Target/StackFrameList.cpp
@@ -85,32 +85,121 @@ void StackFrameList::ResetCurrentInlinedDepth() {
     return;
 
   std::lock_guard<std::recursive_mutex> guard(m_mutex);
+  
+  GetFramesUpTo(0, DoNotAllowInterruption);
+  if (m_frames.empty())
+    return;
+  if (!m_frames[0]->IsInlined()) {
+    m_current_inlined_depth = UINT32_MAX;
+    m_current_inlined_pc = LLDB_INVALID_ADDRESS;
+    Log *log = GetLog(LLDBLog::Step);
+    if (log && log->GetVerbose())
+      LLDB_LOGF(
+          log,
+          "ResetCurrentInlinedDepth: Invalidating current inlined depth.\n");
+    return;
+  }
 
-  m_current_inlined_pc = LLDB_INVALID_ADDRESS;
-  m_current_inlined_depth = UINT32_MAX;
+  // We only need to do something special about inlined blocks when we are
+  // at the beginning of an inlined function:
+  // FIXME: We probably also have to do something special if the PC is at
+  // the END of an inlined function, which coincides with the end of either
+  // its containing function or another inlined function.
+
+  Block *block_ptr = m_frames[0]->GetFrameBlock();
+  if (!block_ptr)
+    return;
 
+  Address pc_as_address;
+  lldb::addr_t curr_pc = m_thread.GetRegisterContext()->GetPC();
+  pc_as_address.SetLoadAddress(curr_pc, &(m_thread.GetProcess()->GetTarget()));
+  AddressRange containing_range;
+  if (!block_ptr->GetRangeContainingAddress(pc_as_address, containing_range) ||
+      pc_as_address != containing_range.GetBaseAddress())
+    return;
+
+  // If we got here because of a breakpoint hit, then set the inlined depth
+  // depending on where the breakpoint was set. If we got here because of a
+  // crash, then set the inlined depth to the deepest most block.  Otherwise,
+  // we stopped here naturally as the result of a step, so set ourselves in the
+  // containing frame of the whole set of nested inlines, so the user can then
+  // "virtually" step into the frames one by one, or next over the whole mess.
+  // Note: We don't have to handle being somewhere in the middle of the stack
+  // here, since ResetCurrentInlinedDepth doesn't get called if there is a
+  // valid inlined depth set.
   StopInfoSP stop_info_sp = m_thread.GetStopInfo();
   if (!stop_info_sp)
     return;
+  switch (stop_info_sp->GetStopReason()) {
+  case eStopReasonWatchpoint:
+  case eStopReasonException:
+  case eStopReasonExec:
+  case eStopReasonFork:
+  case eStopReasonVFork:
+  case eStopReasonVForkDone:
+  case eStopReasonSignal:
+    // In all these cases we want to stop in the deepest frame.
+    m_current_inlined_pc = curr_pc;
+    m_current_inlined_depth = 0;
+    break;
+  case eStopReasonBreakpoint: {
+    // FIXME: Figure out what this break point is doing, and set the inline
+    // depth appropriately.  Be careful to take into account breakpoints that
+    // implement step over prologue, since that should do the default
+    // calculation. For now, if the breakpoints corresponding to this hit are
+    // all internal, I set the stop location to the top of the inlined stack,
+    // since that will make things like stepping over prologues work right.
+    // But if there are any non-internal breakpoints I do to the bottom of the
+    // stack, since that was the old behavior.
+    uint32_t bp_site_id = stop_info_sp->GetValue();
+    BreakpointSiteSP bp_site_sp(
+        m_thread.GetProcess()->GetBreakpointSiteList().FindByID(bp_site_id));
+    bool all_internal = true;
+    if (bp_site_sp) {
+      uint32_t num_owners = bp_site_sp->GetNumberOfConstituents();
+      for (uint32_t i = 0; i < num_owners; i++) {
+        Breakpoint &bp_ref =
+            bp_site_sp->GetConstituentAtIndex(i)->GetBreakpoint();
+        if (!bp_ref.IsInternal()) {
+          all_internal = false;
+        }
+      }
+    }
+    if (!all_internal) {
+      m_current_inlined_pc = curr_pc;
+      m_current_inlined_depth = 0;
+      break;
+    }
+  }
+    [[fallthrough]];
+  default: {
+    // Otherwise, we should set ourselves at the container of the inlining, so
+    // that the user can descend into them. So first we check whether we have
+    // more than one inlined block sharing this PC:
+    int num_inlined_functions = 0;
+
+    for (Block *container_ptr = block_ptr->GetInlinedParent();
+         container_ptr != nullptr;
+         container_ptr = container_ptr->GetInlinedParent()) {
+      if (!container_ptr->GetRangeContainingAddress(pc_as_address,
+                                                    containing_range))
+        break;
+      if (pc_as_address != containing_range.GetBaseAddress())
+        break;
 
-  bool inlined = true;
-  auto inline_depth = stop_info_sp->GetSuggestedStackFrameIndex(inlined);
-  // We're only adjusting the inlined stack here.
-  Log *log = GetLog(LLDBLog::Step);
-  if (inline_depth) {
-    m_current_inlined_depth = *inline_depth;
-    m_current_inlined_pc = m_thread.GetRegisterContext()->GetPC();
-
+      num_inlined_functions++;
+    }
+    m_current_inlined_pc = curr_pc;
+    m_current_inlined_depth = num_inlined_functions + 1;
+    Log *log = GetLog(LLDBLog::Step);
     if (log && log->GetVerbose())
       LLDB_LOGF(log,
                 "ResetCurrentInlinedDepth: setting inlined "
                 "depth: %d 0x%" PRIx64 ".\n",
-                m_current_inlined_depth, m_current_inlined_pc);
-  } else {
-    if (log && log->GetVerbose())
-      LLDB_LOGF(
-          log,
-          "ResetCurrentInlinedDepth: Invalidating current inlined depth.\n");
+                m_current_inlined_depth, curr_pc);
+
+    break;
+  }
   }
 }
 
@@ -727,48 +816,19 @@ void StackFrameList::SelectMostRelevantFrame() {
 
   RecognizedStackFrameSP recognized_frame_sp = frame_sp->GetRecognizedFrame();
 
-  if (recognized_frame_sp) {
-    if (StackFrameSP most_relevant_frame_sp =
-            recognized_frame_sp->GetMostRelevantFrame()) {
-      LLDB_LOG(log, "Found most relevant frame at index {0}",
-               most_relevant_frame_sp->GetFrameIndex());
-      SetSelectedFrame(most_relevant_frame_sp.get());
-      return;
-    }
-  }
-  LLDB_LOG(log, "Frame #0 not recognized");
-
-  // If this thread has a non-trivial StopInof, then let it suggest
-  // a most relevant frame:
-  StopInfoSP stop_info_sp = m_thread.GetStopInfo();
-  uint32_t stack_idx = 0;
-  bool found_relevant = false;
-  if (stop_info_sp) {
-    // Here we're only asking the stop info if it wants to adjust the real stack
-    // index.  We have to ask about the m_inlined_stack_depth in
-    // Thread::ShouldStop since the plans need to reason with that info.
-    bool inlined = false;
-    std::optional<uint32_t> stack_opt =
-        stop_info_sp->GetSuggestedStackFrameIndex(inlined);
-    if (stack_opt) {
-      stack_idx = *stack_opt;
-      found_relevant = true;
-    }
+  if (!recognized_frame_sp) {
+    LLDB_LOG(log, "Frame #0 not recognized");
+    return;
   }
 
-  frame_sp = GetFrameAtIndex(stack_idx);
-  if (!frame_sp)
-    LLDB_LOG(log, "Stop info suggested relevant frame {0} but it didn't exist",
-             stack_idx);
-  else if (found_relevant)
-    LLDB_LOG(log, "Setting selected frame from stop info to {0}", stack_idx);
-  // Note, we don't have to worry about "inlined" frames here, because we've
-  // already calculated the inlined frame in Thread::ShouldStop, and
-  // SetSelectedFrame will take care of that adjustment for us.
-  SetSelectedFrame(frame_sp.get());
-
-  if (!found_relevant)
+  if (StackFrameSP most_relevant_frame_sp =
+          recognized_frame_sp->GetMostRelevantFrame()) {
+    LLDB_LOG(log, "Found most relevant frame at index {0}",
+             most_relevant_frame_sp->GetFrameIndex());
+    SetSelectedFrame(most_relevant_frame_sp.get());
+  } else {
     LLDB_LOG(log, "No relevant frame!");
+  }
 }
 
 uint32_t StackFrameList::GetSelectedFrameIndex(
@@ -781,7 +841,6 @@ uint32_t StackFrameList::GetSelectedFrameIndex(
     // isn't set, then don't force a selection here, just return 0.
     if (!select_most_relevant)
       return 0;
-    // If the inlined stack frame is set, then use that:
     m_selected_frame_idx = 0;
   }
   return *m_selected_frame_idx;
diff --git a/lldb/source/Target/StopInfo.cpp b/lldb/source/Target/StopInfo.cpp
index f6387d47504e..60aa65ed38c7 100644
--- a/lldb/source/Target/StopInfo.cpp
+++ b/lldb/source/Target/StopInfo.cpp
@@ -15,7 +15,6 @@
 #include "lldb/Breakpoint/WatchpointResource.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Expression/UserExpression.h"
-#include "lldb/Symbol/Block.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/StopInfo.h"
 #include "lldb/Target/Target.h"
@@ -247,22 +246,6 @@ public:
     return m_description.c_str();
   }
 
-  std::optional<uint32_t>
-  GetSuggestedStackFrameIndex(bool inlined_stack) override {
-    if (!inlined_stack)
-      return {};
-
-    ThreadSP thread_sp(m_thread_wp.lock());
-    if (!thread_sp)
-      return {};
-    BreakpointSiteSP bp_site_sp(
-        thread_sp->GetProcess()->GetBreakpointSiteList().FindByID(m_value));
-    if (!bp_site_sp)
-      return {};
-
-    return bp_site_sp->GetSuggestedStackFrameIndex();
-  }
-
 protected:
   bool ShouldStop(Event *event_ptr) override {
     // This just reports the work done by PerformAction or the synchronous
@@ -1181,44 +1164,6 @@ public:
     else
       return m_description.c_str();
   }
-
-  std::optional<uint32_t>
-  GetSuggestedStackFrameIndex(bool inlined_stack) override {
-    // Trace only knows how to adjust inlined stacks:
-    if (!inlined_stack)
-      return {};
-
-    ThreadSP thread_sp = GetThread();
-    StackFrameSP frame_0_sp = thread_sp->GetStackFrameAtIndex(0);
-    if (!frame_0_sp)
-      return {};
-    if (!frame_0_sp->IsInlined())
-      return {};
-    Block *block_ptr = frame_0_sp->GetFrameBlock();
-    if (!block_ptr)
-      return {};
-    Address pc_address = frame_0_sp->GetFrameCodeAddress();
-    AddressRange containing_range;
-    if (!block_ptr->GetRangeContainingAddress(pc_address, containing_range) ||
-        pc_address != containing_range.GetBaseAddress())
-      return {};
-
-    int num_inlined_functions = 0;
-
-    for (Block *container_ptr = block_ptr->GetInlinedParent();
-         container_ptr != nullptr;
-         container_ptr = container_ptr->GetInlinedParent()) {
-      if (!container_ptr->GetRangeContainingAddress(pc_address,
-                                                    containing_range))
-        break;
-      if (pc_address != containing_range.GetBaseAddress())
-        break;
-
-      num_inlined_functions++;
-    }
-    inlined_stack = true;
-    return num_inlined_functions + 1;
-  }
 };
 
 // StopInfoException
diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp
index 735295e6f259..8373cdc36268 100644
--- a/lldb/source/Target/Thread.cpp
+++ b/lldb/source/Target/Thread.cpp
@@ -619,14 +619,6 @@ void Thread::WillStop() {
 
 void Thread::SetupForResume() {
   if (GetResumeState() != eStateSuspended) {
-    // First check whether this thread is going to "actually" resume at all.
-    // For instance, if we're stepping from one level to the next of an
-    // virtual inlined call stack, we just change the inlined call stack index
-    // without actually running this thread.  In that case, for this thread we
-    // shouldn't push a step over breakpoint plan or do that work.
-    if (GetCurrentPlan()->IsVirtualStep())
-      return;
-
     // If we're at a breakpoint push the step-over breakpoint plan.  Do this
     // before telling the current plan it will resume, since we might change
     // what the current plan is.
diff --git a/lldb/source/Target/ThreadPlanStepInRange.cpp b/lldb/source/Target/ThreadPlanStepInRange.cpp
index 325a70619908..567dcc26d0d3 100644
--- a/lldb/source/Target/ThreadPlanStepInRange.cpp
+++ b/lldb/source/Target/ThreadPlanStepInRange.cpp
@@ -41,7 +41,7 @@ ThreadPlanStepInRange::ThreadPlanStepInRange(
                           "Step Range stepping in", thread, range, addr_context,
                           stop_others),
       ThreadPlanShouldStopHere(this), m_step_past_prologue(true),
-      m_virtual_step(eLazyBoolCalculate), m_step_into_target(step_into_target) {
+      m_virtual_step(false), m_step_into_target(step_into_target) {
   SetCallbacks();
   SetFlagsToDefault();
   SetupAvoidNoDebug(step_in_avoids_code_without_debug_info,
@@ -149,7 +149,7 @@ bool ThreadPlanStepInRange::ShouldStop(Event *event_ptr) {
       m_sub_plan_sp.reset();
   }
 
-  if (m_virtual_step == eLazyBoolYes) {
+  if (m_virtual_step) {
     // If we've just completed a virtual step, all we need to do is check for a
     // ShouldStopHere plan, and otherwise we're done.
     // FIXME - This can be both a step in and a step out.  Probably should
@@ -431,7 +431,7 @@ bool ThreadPlanStepInRange::DoPlanExplainsStop(Event *event_ptr) {
 
   bool return_value = false;
 
-  if (m_virtual_step == eLazyBoolYes) {
+  if (m_virtual_step) {
     return_value = true;
   } else {
     StopInfoSP stop_info_sp = GetPrivateStopInfo();
@@ -460,13 +460,10 @@ bool ThreadPlanStepInRange::DoPlanExplainsStop(Event *event_ptr) {
 
 bool ThreadPlanStepInRange::DoWillResume(lldb::StateType resume_state,
                                          bool current_plan) {
-  m_virtual_step = eLazyBoolCalculate;
+  m_virtual_step = false;
   if (resume_state == eStateStepping && current_plan) {
     Thread &thread = GetThread();
     // See if we are about to step over a virtual inlined call.
-    // But if we already know we're virtual stepping, don't decrement the
-    // inlined depth again...
-
     bool step_without_resume = thread.DecrementCurrentInlinedDepth();
     if (step_without_resume) {
       Log *log = GetLog(LLDBLog::Step);
@@ -479,20 +476,11 @@ bool ThreadPlanStepInRange::DoWillResume(lldb::StateType resume_state,
       // FIXME: Maybe it would be better to create a InlineStep stop reason, but
       // then
       // the whole rest of the world would have to handle that stop reason.
-      m_virtual_step = eLazyBoolYes;
+      m_virtual_step = true;
     }
     return !step_without_resume;
   }
   return true;
 }
 
-bool ThreadPlanStepInRange::IsVirtualStep() {
-  if (m_virtual_step == eLazyBoolCalculate) {
-    Thread &thread = GetThread();
-    if (thread.GetCurrentInlinedDepth() == UINT32_MAX)
-      m_virtual_step = eLazyBoolNo;
-    else
-      m_virtual_step = eLazyBoolYes;
-  }
-  return m_virtual_step == eLazyBoolYes;
-}
+bool ThreadPlanStepInRange::IsVirtualStep() { return m_virtual_step; }
diff --git a/lldb/source/Target/ThreadPlanStepOverRange.cpp b/lldb/source/Target/ThreadPlanStepOverRange.cpp
index 643ee827c865..ef5b4b5c434d 100644
--- a/lldb/source/Target/ThreadPlanStepOverRange.cpp
+++ b/lldb/source/Target/ThreadPlanStepOverRange.cpp
@@ -402,7 +402,7 @@ bool ThreadPlanStepOverRange::DoWillResume(lldb::StateType resume_state,
       if (in_inlined_stack) {
         Log *log = GetLog(LLDBLog::Step);
         LLDB_LOGF(log,
-                  "ThreadPlanStepOverRange::DoWillResume: adjusting range to "
+                  "ThreadPlanStepInRange::DoWillResume: adjusting range to "
                   "the frame at inlined depth %d.",
                   thread.GetCurrentInlinedDepth());
         StackFrameSP stack_sp = thread.GetStackFrameAtIndex(0);
diff --git a/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py b/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
index f52e0f0fd5bc..752c3a9cbd28 100644
--- a/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
+++ b/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
@@ -32,12 +32,6 @@ class TestInlineStepping(TestBase):
         self.build()
         self.step_in_template()
 
-    @add_test_categories(["pyapi"])
-    def test_virtual_inline_stepping(self):
-        """Test stepping through a virtual inlined call stack"""
-        self.build()
-        self.virtual_inline_stepping()
-
     def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)
@@ -363,60 +357,3 @@ class TestInlineStepping(TestBase):
 
         step_sequence = [["// In max_value specialized", "into"]]
         self.run_step_sequence(step_sequence)
-
-    def run_to_call_site_and_step(self, source_regex, func_name, start_pos):
-        main_spec = lldb.SBFileSpec("calling.cpp")
-        # Set the breakpoint by file and line, not sourced regex because
-        # we want to make sure we can set breakpoints on call sites:
-        call_site_line_num = line_number(self.main_source, source_regex)
-        target, process, thread, bkpt = lldbutil.run_to_line_breakpoint(
-            self, main_spec, call_site_line_num
-        )
-
-        # Make sure that the location is at the call site (run_to_line_breakpoint already asserted
-        # that there's one location.):
-        bkpt_loc = bkpt.location[0]
-        strm = lldb.SBStream()
-        result = bkpt_loc.GetDescription(strm, lldb.eDescriptionLevelFull)
-
-        self.assertTrue(result, "Got a location description")
-        desc = strm.GetData()
-        self.assertIn(f"calling.cpp:{call_site_line_num}", desc, "Right line listed")
-        # We don't get the function name right yet - so we omit it in printing.
-        # Turn on this test when that is working.
-        # self.assertIn(func_name, desc, "Right function listed")
-
-        pc = thread.frame[0].pc
-        for i in range(start_pos, 3):
-            thread.StepInto()
-            frame_0 = thread.frame[0]
-
-            trivial_line_num = line_number(
-                self.main_source, f"In caller_trivial_inline_{i}."
-            )
-            self.assertEqual(
-                frame_0.line_entry.line,
-                trivial_line_num,
-                f"Stepped into the caller_trivial_inline_{i}",
-            )
-            if pc != frame_0.pc:
-                # If we get here, we stepped to the expected line number, but
-                # the compiler on this system has decided to insert an instruction
-                # between the call site of an inlined function with no arguments,
-                # returning void, and its immediate call to another void inlined function
-                # with no arguments.  We aren't going to be testing virtual inline
-                # stepping for this function...
-                break
-
-        process.Kill()
-        target.Clear()
-
-    def virtual_inline_stepping(self):
-        """Use the Python API's to step through a virtual inlined stack"""
-        self.run_to_call_site_and_step("At caller_trivial_inline_1", "main", 1)
-        self.run_to_call_site_and_step(
-            "In caller_trivial_inline_1", "caller_trivial_inline_1", 2
-        )
-        self.run_to_call_site_and_step(
-            "In caller_trivial_inline_2", "caller_trivial_inline_2", 3
-        )
diff --git a/lldb/test/API/functionalities/inline-stepping/calling.cpp b/lldb/test/API/functionalities/inline-stepping/calling.cpp
index d7ee56b3c079..49179ce7c978 100644
--- a/lldb/test/API/functionalities/inline-stepping/calling.cpp
+++ b/lldb/test/API/functionalities/inline-stepping/calling.cpp
@@ -13,12 +13,6 @@ int called_by_inline_ref (int &value);
 inline void inline_trivial_1 () __attribute__((always_inline));
 inline void inline_trivial_2 () __attribute__((always_inline));
 
-// These three should share the same initial pc so we can test
-// virtual inline stepping.
-inline void caller_trivial_inline_1() __attribute__((always_inline));
-inline void caller_trivial_inline_2() __attribute__((always_inline));
-inline void caller_trivial_inline_3() __attribute__((always_inline));
-
 void caller_trivial_1 ();
 void caller_trivial_2 ();
 
@@ -85,23 +79,6 @@ caller_trivial_2 ()
     inline_value += 1;  // At increment in caller_trivial_2.
 }
 
-// When you call caller_trivial_inline_1, the inlined call-site
-// should share a PC with all three of the following inlined
-// functions, so we can exercise "virtual inline stepping".
-void caller_trivial_inline_1() {
-  caller_trivial_inline_2(); // In caller_trivial_inline_1.
-  inline_value += 1;
-}
-
-void caller_trivial_inline_2() {
-  caller_trivial_inline_3(); // In caller_trivial_inline_2.
-  inline_value += 1;
-}
-
-void caller_trivial_inline_3() {
-  inline_value += 1; // In caller_trivial_inline_3.
-}
-
 void
 called_by_inline_trivial ()
 {
@@ -155,7 +132,5 @@ main (int argc, char **argv)
     max_value(123, 456);                                // Call max_value template
     max_value(std::string("abc"), std::string("0022")); // Call max_value specialized
 
-    caller_trivial_inline_1(); // At caller_trivial_inline_1.
-
     return 0;            // About to return from main.
 }
-- 
GitLab


From 8274be509ed9e07188a8a64d95907a46cbe8e657 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 28 Oct 2024 18:53:38 +0000
Subject: [PATCH 354/693] [AArch64] Remove header dependencies of
 AArch64ISelLowering.h. NFC

This patch aims to reduce the include used by AArch64ISelLowering, allowing it
to be included by unittests so that they can reference the AArch64ISD nodes.
It:
 - Moves the inclusion of AArch64SMEAttributes.h to the uses.
 - Moves LowerPtrAuthGlobalAddressStatically to a static function, so that
   AArch64PACKey is not required in the header.
 - Moves the definitions of getExceptionPointerRegister to the cpp file, to
   remove the reference of AArch64::X0.
---
 llvm/lib/Target/AArch64/AArch64FastISel.cpp   |  1 +
 .../Target/AArch64/AArch64FrameLowering.cpp   |  1 +
 .../Target/AArch64/AArch64ISelLowering.cpp    | 21 +++++++++++++++++--
 llvm/lib/Target/AArch64/AArch64ISelLowering.h | 17 ++-------------
 .../AArch64/AArch64SelectionDAGInfo.cpp       |  2 ++
 .../AArch64/AArch64TargetTransformInfo.cpp    |  1 +
 .../AArch64/GISel/AArch64CallLowering.cpp     |  1 +
 7 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
index cbf38f2c57a3..6c874fcabcc3 100644
--- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
@@ -19,6 +19,7 @@
 #include "AArch64Subtarget.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "Utils/AArch64BaseInfo.h"
+#include "Utils/AArch64SMEAttributes.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/DenseMap.h"
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index dfaa36f7f512..9af6429c5cae 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -212,6 +212,7 @@
 #include "AArch64TargetMachine.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "MCTargetDesc/AArch64MCTargetDesc.h"
+#include "Utils/AArch64SMEAttributes.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 4c0cd1ac3d45..32ba2866ac81 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -19,6 +19,7 @@
 #include "AArch64Subtarget.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "Utils/AArch64BaseInfo.h"
+#include "Utils/AArch64SMEAttributes.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -10082,9 +10083,9 @@ SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op,
 // Thus, it's only used for ptrauth references to extern_weak to avoid null
 // checks.
 
-SDValue AArch64TargetLowering::LowerPtrAuthGlobalAddressStatically(
+static SDValue LowerPtrAuthGlobalAddressStatically(
     SDValue TGA, SDLoc DL, EVT VT, AArch64PACKey::ID KeyC,
-    SDValue Discriminator, SDValue AddrDiscriminator, SelectionDAG &DAG) const {
+    SDValue Discriminator, SDValue AddrDiscriminator, SelectionDAG &DAG) {
   const auto *TGN = cast<GlobalAddressSDNode>(TGA.getNode());
   assert(TGN->getGlobal()->hasExternalWeakLinkage());
 
@@ -27574,6 +27575,22 @@ AArch64TargetLowering::getSafeStackPointerLocation(IRBuilderBase &IRB) const {
   return TargetLowering::getSafeStackPointerLocation(IRB);
 }
 
+/// If a physical register, this returns the register that receives the
+/// exception address on entry to an EH pad.
+Register AArch64TargetLowering::getExceptionPointerRegister(
+    const Constant *PersonalityFn) const {
+  // FIXME: This is a guess. Has this been defined yet?
+  return AArch64::X0;
+}
+
+/// If a physical register, this returns the register that receives the
+/// exception typeid on entry to a landing pad.
+Register AArch64TargetLowering::getExceptionSelectorRegister(
+    const Constant *PersonalityFn) const {
+  // FIXME: This is a guess. Has this been defined yet?
+  return AArch64::X1;
+}
+
 bool AArch64TargetLowering::isMaskAndCmp0FoldingBeneficial(
     const Instruction &AndI) const {
   // Only sink 'and' mask to cmp use block if it is masking a single bit, since
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 160cd18ca53b..d696355bb062 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -14,8 +14,6 @@
 #ifndef LLVM_LIB_TARGET_AARCH64_AARCH64ISELLOWERING_H
 #define LLVM_LIB_TARGET_AARCH64_AARCH64ISELLOWERING_H
 
-#include "AArch64.h"
-#include "Utils/AArch64SMEAttributes.h"
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/SelectionDAG.h"
@@ -830,18 +828,12 @@ public:
   /// If a physical register, this returns the register that receives the
   /// exception address on entry to an EH pad.
   Register
-  getExceptionPointerRegister(const Constant *PersonalityFn) const override {
-    // FIXME: This is a guess. Has this been defined yet?
-    return AArch64::X0;
-  }
+  getExceptionPointerRegister(const Constant *PersonalityFn) const override;
 
   /// If a physical register, this returns the register that receives the
   /// exception typeid on entry to a landing pad.
   Register
-  getExceptionSelectorRegister(const Constant *PersonalityFn) const override {
-    // FIXME: This is a guess. Has this been defined yet?
-    return AArch64::X1;
-  }
+  getExceptionSelectorRegister(const Constant *PersonalityFn) const override;
 
   bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
 
@@ -1132,11 +1124,6 @@ private:
                                  SelectionDAG &DAG) const;
   SDValue LowerWindowsGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerPtrAuthGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
-  SDValue LowerPtrAuthGlobalAddressStatically(SDValue TGA, SDLoc DL, EVT VT,
-                                              AArch64PACKey::ID Key,
-                                              SDValue Discriminator,
-                                              SDValue AddrDiscriminator,
-                                              SelectionDAG &DAG) const;
   SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
index 19ef6f4fb32e..525538db8036 100644
--- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
@@ -11,6 +11,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "AArch64TargetMachine.h"
+#include "Utils/AArch64SMEAttributes.h"
+
 using namespace llvm;
 
 #define DEBUG_TYPE "aarch64-selectiondag-info"
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index ff3c69f7e10c..71f9bbbbc350 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -10,6 +10,7 @@
 #include "AArch64ExpandImm.h"
 #include "AArch64PerfectShuffle.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
+#include "Utils/AArch64SMEAttributes.h"
 #include "llvm/Analysis/IVDescriptors.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
index 6cbfb018b318..065858c42894 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@@ -18,6 +18,7 @@
 #include "AArch64MachineFunctionInfo.h"
 #include "AArch64RegisterInfo.h"
 #include "AArch64Subtarget.h"
+#include "Utils/AArch64SMEAttributes.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/ObjCARCUtil.h"
-- 
GitLab


From 66bbbf2e951a486f972a5a54378e6791193ade60 Mon Sep 17 00:00:00 2001
From: Jan Voung <jvoung@google.com>
Date: Mon, 28 Oct 2024 15:13:29 -0400
Subject: [PATCH 355/693] [clang][dataflow] Cache accessors returning pointers
 in bugprone-unchecked-optional-access (#113922)

Previously, we covered returning refs, or copies of optional, and bools.
Now cover returning pointers (to any type).
This is useful for cases like operator-> of smart pointers.
Addresses more of issue llvm#58510
---
 .../Models/UncheckedOptionalAccessModel.h     |  8 +++
 .../Models/UncheckedOptionalAccessModel.cpp   | 20 +++++-
 .../UncheckedOptionalAccessModelTest.cpp      | 72 ++++++++++++++++---
 3 files changed, 87 insertions(+), 13 deletions(-)

diff --git a/clang/include/clang/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.h b/clang/include/clang/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.h
index 9d81cacb5073..713494178b97 100644
--- a/clang/include/clang/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.h
+++ b/clang/include/clang/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.h
@@ -37,6 +37,14 @@ struct UncheckedOptionalAccessModelOptions {
   /// can't identify when their results are used safely (across calls),
   /// resulting in false positives in all such cases. Note: this option does not
   /// cover access through `operator[]`.
+  /// FIXME: we currently cache and equate the result of const accessors
+  /// returning pointers, so cover the case of operator-> followed by
+  /// operator->, which covers the common case of smart pointers. We also cover
+  /// some limited cases of returning references (if return type is an optional
+  /// type), so cover some cases of operator* followed by operator*. We don't
+  /// cover mixing operator-> and operator*. Once we are confident in this const
+  /// accessor caching, we shouldn't need the IgnoreSmartPointerDereference
+  /// option anymore.
   bool IgnoreSmartPointerDereference = false;
 };
 
diff --git a/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp b/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
index 31ae2b94f5b6..da5dda063344 100644
--- a/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
@@ -338,6 +338,11 @@ auto isZeroParamConstMemberCall() {
       callee(cxxMethodDecl(parameterCountIs(0), isConst())));
 }
 
+auto isZeroParamConstMemberOperatorCall() {
+  return cxxOperatorCallExpr(
+      callee(cxxMethodDecl(parameterCountIs(0), isConst())));
+}
+
 auto isNonConstMemberCall() {
   return cxxMemberCallExpr(callee(cxxMethodDecl(unless(isConst()))));
 }
@@ -572,9 +577,10 @@ void handleConstMemberCall(const CallExpr *CE,
     return;
   }
 
-  // Cache if the const method returns a boolean type.
+  // Cache if the const method returns a boolean or pointer type.
   // We may decide to cache other return types in the future.
-  if (RecordLoc != nullptr && CE->getType()->isBooleanType()) {
+  if (RecordLoc != nullptr &&
+      (CE->getType()->isBooleanType() || CE->getType()->isPointerType())) {
     Value *Val = State.Lattice.getOrCreateConstMethodReturnValue(*RecordLoc, CE,
                                                                  State.Env);
     if (Val == nullptr)
@@ -597,6 +603,14 @@ void transferValue_ConstMemberCall(const CXXMemberCallExpr *MCE,
       MCE, dataflow::getImplicitObjectLocation(*MCE, State.Env), Result, State);
 }
 
+void transferValue_ConstMemberOperatorCall(
+    const CXXOperatorCallExpr *OCE, const MatchFinder::MatchResult &Result,
+    LatticeTransferState &State) {
+  auto *RecordLoc = cast_or_null<dataflow::RecordStorageLocation>(
+      State.Env.getStorageLocation(*OCE->getArg(0)));
+  handleConstMemberCall(OCE, RecordLoc, Result, State);
+}
+
 void handleNonConstMemberCall(const CallExpr *CE,
                               dataflow::RecordStorageLocation *RecordLoc,
                               const MatchFinder::MatchResult &Result,
@@ -1020,6 +1034,8 @@ auto buildTransferMatchSwitch() {
       // const accessor calls
       .CaseOfCFGStmt<CXXMemberCallExpr>(isZeroParamConstMemberCall(),
                                         transferValue_ConstMemberCall)
+      .CaseOfCFGStmt<CXXOperatorCallExpr>(isZeroParamConstMemberOperatorCall(),
+                                          transferValue_ConstMemberOperatorCall)
       // non-const member calls that may modify the state of an object.
       .CaseOfCFGStmt<CXXMemberCallExpr>(isNonConstMemberCall(),
                                         transferValue_NonConstMemberCall)
diff --git a/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp b/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
index 5b64eaca0e10..de16f6be8eed 100644
--- a/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
@@ -1282,28 +1282,35 @@ static raw_ostream &operator<<(raw_ostream &OS,
 class UncheckedOptionalAccessTest
     : public ::testing::TestWithParam<OptionalTypeIdentifier> {
 protected:
-  void ExpectDiagnosticsFor(std::string SourceCode) {
-    ExpectDiagnosticsFor(SourceCode, ast_matchers::hasName("target"));
+  void ExpectDiagnosticsFor(std::string SourceCode,
+                            bool IgnoreSmartPointerDereference = true) {
+    ExpectDiagnosticsFor(SourceCode, ast_matchers::hasName("target"),
+                         IgnoreSmartPointerDereference);
   }
 
-  void ExpectDiagnosticsForLambda(std::string SourceCode) {
+  void ExpectDiagnosticsForLambda(std::string SourceCode,
+                                  bool IgnoreSmartPointerDereference = true) {
     ExpectDiagnosticsFor(
-        SourceCode, ast_matchers::hasDeclContext(
-                        ast_matchers::cxxRecordDecl(ast_matchers::isLambda())));
+        SourceCode,
+        ast_matchers::hasDeclContext(
+            ast_matchers::cxxRecordDecl(ast_matchers::isLambda())),
+        IgnoreSmartPointerDereference);
   }
 
   template <typename FuncDeclMatcher>
-  void ExpectDiagnosticsFor(std::string SourceCode,
-                            FuncDeclMatcher FuncMatcher) {
+  void ExpectDiagnosticsFor(std::string SourceCode, FuncDeclMatcher FuncMatcher,
+                            bool IgnoreSmartPointerDereference = true) {
     // Run in C++17 and C++20 mode to cover differences in the AST between modes
     // (e.g. C++20 can contain `CXXRewrittenBinaryOperator`).
     for (const char *CxxMode : {"-std=c++17", "-std=c++20"})
-      ExpectDiagnosticsFor(SourceCode, FuncMatcher, CxxMode);
+      ExpectDiagnosticsFor(SourceCode, FuncMatcher, CxxMode,
+                           IgnoreSmartPointerDereference);
   }
 
   template <typename FuncDeclMatcher>
   void ExpectDiagnosticsFor(std::string SourceCode, FuncDeclMatcher FuncMatcher,
-                            const char *CxxMode) {
+                            const char *CxxMode,
+                            bool IgnoreSmartPointerDereference) {
     ReplaceAllOccurrences(SourceCode, "$ns", GetParam().NamespaceName);
     ReplaceAllOccurrences(SourceCode, "$optional", GetParam().TypeName);
 
@@ -1328,8 +1335,7 @@ protected:
       template <typename T>
       T Make();
     )");
-    UncheckedOptionalAccessModelOptions Options{
-        /*IgnoreSmartPointerDereference=*/true};
+    UncheckedOptionalAccessModelOptions Options{IgnoreSmartPointerDereference};
     std::vector<SourceLocation> Diagnostics;
     llvm::Error Error = checkDataflow<UncheckedOptionalAccessModel>(
         AnalysisInputs<UncheckedOptionalAccessModel>(
@@ -3721,6 +3727,50 @@ TEST_P(UncheckedOptionalAccessTest, ConstByValueAccessorWithModInBetween) {
   )cc");
 }
 
+TEST_P(UncheckedOptionalAccessTest, ConstPointerAccessor) {
+  ExpectDiagnosticsFor(R"cc(
+     #include "unchecked_optional_access_test.h"
+
+    struct A {
+      $ns::$optional<int> x;
+    };
+
+    struct MyUniquePtr {
+      A* operator->() const;
+    };
+
+    void target(MyUniquePtr p) {
+      if (p->x) {
+        *p->x;
+      }
+    }
+  )cc",
+                       /*IgnoreSmartPointerDereference=*/false);
+}
+
+TEST_P(UncheckedOptionalAccessTest, ConstPointerAccessorWithModInBetween) {
+  ExpectDiagnosticsFor(R"cc(
+    #include "unchecked_optional_access_test.h"
+
+    struct A {
+      $ns::$optional<int> x;
+    };
+
+    struct MyUniquePtr {
+      A* operator->() const;
+      void reset(A*);
+    };
+
+    void target(MyUniquePtr p) {
+      if (p->x) {
+        p.reset(nullptr);
+        *p->x;  // [[unsafe]]
+      }
+    }
+  )cc",
+                       /*IgnoreSmartPointerDereference=*/false);
+}
+
 TEST_P(UncheckedOptionalAccessTest, ConstBoolAccessor) {
   ExpectDiagnosticsFor(R"cc(
     #include "unchecked_optional_access_test.h"
-- 
GitLab


From 0d0abb351b5fcf49ccc46eba8b7f2a1f353a05a6 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 28 Oct 2024 20:14:08 +0100
Subject: [PATCH 356/693] [VPlan] Use ResumePhi to create reduction resume
 phis. (#110004)

Use VPInstruction::ResumePhi to create phi nodes for reduction resume
values in the scalar preheader, similar to how ResumePhis are used for
first-order recurrence resume values after 9a5a8731e77.

This allows simplifying createAndCollectMergePhiForReduction to only
collect reduction resume phis when vectorizing epilogue loops and adding
extra incoming edges from the main vector loop. Updating phis for the
epilogue vector loops requires special attention, because additional
incoming values from the bypass blocks need to be added.

PR: https://github.com/llvm/llvm-project/pull/110004
---
 .../Transforms/Vectorize/LoopVectorize.cpp    | 130 +++++++++---------
 .../RISCV/vplan-vp-intrinsics-reduction.ll    |   9 ++
 ...-order-recurrence-sink-replicate-region.ll |   2 +
 .../LoopVectorize/vplan-printing.ll           |   9 ++
 4 files changed, 88 insertions(+), 62 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 88086f24dfdc..778d928252e0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7562,67 +7562,62 @@ static void addRuntimeUnrollDisableMetaData(Loop *L) {
   }
 }
 
-// Check if \p RedResult is a ComputeReductionResult instruction, and if it is
-// create a merge phi node for it.
-static void createAndCollectMergePhiForReduction(
-    VPInstruction *RedResult,
-    VPTransformState &State, Loop *OrigLoop, BasicBlock *LoopMiddleBlock,
-    bool VectorizingEpilogue) {
-  if (!RedResult ||
-      RedResult->getOpcode() != VPInstruction::ComputeReductionResult)
+// If \p R is a ComputeReductionResult when vectorizing the epilog loop,
+// fix the reduction's scalar PHI node by adding the incoming value from the
+// main vector loop.
+static void fixReductionScalarResumeWhenVectorizingEpilog(
+    VPRecipeBase *R, VPTransformState &State, BasicBlock *LoopMiddleBlock) {
+  auto *EpiRedResult = dyn_cast<VPInstruction>(R);
+  if (!EpiRedResult ||
+      EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult)
     return;
 
-  auto *PhiR = cast<VPReductionPHIRecipe>(RedResult->getOperand(0));
-  const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
-
-  Value *FinalValue = State.get(RedResult, VPLane(VPLane::getFirstLane()));
-  auto *ResumePhi =
-      dyn_cast<PHINode>(PhiR->getStartValue()->getUnderlyingValue());
-  if (VectorizingEpilogue && RecurrenceDescriptor::isAnyOfRecurrenceKind(
-                                 RdxDesc.getRecurrenceKind())) {
-    auto *Cmp = cast<ICmpInst>(PhiR->getStartValue()->getUnderlyingValue());
-    assert(Cmp->getPredicate() == CmpInst::ICMP_NE);
-    assert(Cmp->getOperand(1) == RdxDesc.getRecurrenceStartValue());
-    ResumePhi = cast<PHINode>(Cmp->getOperand(0));
-  }
-  assert((!VectorizingEpilogue || ResumePhi) &&
-         "when vectorizing the epilogue loop, we need a resume phi from main "
-         "vector loop");
-
-  // TODO: bc.merge.rdx should not be created here, instead it should be
-  // modeled in VPlan.
-  BasicBlock *LoopScalarPreHeader = OrigLoop->getLoopPreheader();
-  // Create a phi node that merges control-flow from the backedge-taken check
-  // block and the middle block.
-  auto *BCBlockPhi =
-      PHINode::Create(FinalValue->getType(), 2, "bc.merge.rdx",
-                      LoopScalarPreHeader->getTerminator()->getIterator());
-
-  // If we are fixing reductions in the epilogue loop then we should already
-  // have created a bc.merge.rdx Phi after the main vector body. Ensure that
-  // we carry over the incoming values correctly.
+  auto *EpiRedHeaderPhi =
+      cast<VPReductionPHIRecipe>(EpiRedResult->getOperand(0));
+  const RecurrenceDescriptor &RdxDesc =
+      EpiRedHeaderPhi->getRecurrenceDescriptor();
+  Value *MainResumeValue =
+      EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
+  if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
+          RdxDesc.getRecurrenceKind())) {
+    auto *Cmp = cast<ICmpInst>(MainResumeValue);
+    assert(Cmp->getPredicate() == CmpInst::ICMP_NE &&
+           "AnyOf expected to start with ICMP_NE");
+    assert(Cmp->getOperand(1) == RdxDesc.getRecurrenceStartValue() &&
+           "AnyOf expected to start by comparing main resume value to original "
+           "start value");
+    MainResumeValue = Cmp->getOperand(0);
+  }
+  PHINode *MainResumePhi = cast<PHINode>(MainResumeValue);
+
+  // When fixing reductions in the epilogue loop we should already have
+  // created a bc.merge.rdx Phi after the main vector body. Ensure that we carry
+  // over the incoming values correctly.
+  using namespace VPlanPatternMatch;
+  auto IsResumePhi = [](VPUser *U) {
+    return match(
+        U, m_VPInstruction<VPInstruction::ResumePhi>(m_VPValue(), m_VPValue()));
+  };
+  assert(count_if(EpiRedResult->users(), IsResumePhi) == 1 &&
+         "ResumePhi must have a single user");
+  auto *EpiResumePhiVPI =
+      cast<VPInstruction>(*find_if(EpiRedResult->users(), IsResumePhi));
+  auto *EpiResumePhi = cast<PHINode>(State.get(EpiResumePhiVPI, true));
+  BasicBlock *LoopScalarPreHeader = EpiResumePhi->getParent();
+  bool Updated = false;
   for (auto *Incoming : predecessors(LoopScalarPreHeader)) {
-    if (Incoming == LoopMiddleBlock)
-      BCBlockPhi->addIncoming(FinalValue, Incoming);
-    else if (ResumePhi && is_contained(ResumePhi->blocks(), Incoming))
-      BCBlockPhi->addIncoming(ResumePhi->getIncomingValueForBlock(Incoming),
-                              Incoming);
-    else
-      BCBlockPhi->addIncoming(RdxDesc.getRecurrenceStartValue(), Incoming);
+    if (is_contained(MainResumePhi->blocks(), Incoming)) {
+      assert(EpiResumePhi->getIncomingValueForBlock(Incoming) ==
+                 RdxDesc.getRecurrenceStartValue() &&
+             "Trying to reset unexpected value");
+      assert(!Updated && "Should update at most 1 incoming value");
+      EpiResumePhi->setIncomingValueForBlock(
+          Incoming, MainResumePhi->getIncomingValueForBlock(Incoming));
+      Updated = true;
+    }
   }
-
-  auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
-  // TODO: This fixup should instead be modeled in VPlan.
-  // Fix the scalar loop reduction variable with the incoming reduction sum
-  // from the vector body and from the backedge value.
-  int IncomingEdgeBlockIdx =
-      OrigPhi->getBasicBlockIndex(OrigLoop->getLoopLatch());
-  assert(IncomingEdgeBlockIdx >= 0 && "Invalid block index");
-  // Pick the other block.
-  int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1);
-  OrigPhi->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi);
-  Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
-  OrigPhi->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst);
+  assert(Updated && "Must update EpiResumePhi.");
+  (void)Updated;
 }
 
 DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
@@ -7713,11 +7708,11 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   // 2.5 Collect reduction resume values.
   auto *ExitVPBB =
       cast<VPBasicBlock>(BestVPlan.getVectorLoopRegion()->getSingleSuccessor());
-  for (VPRecipeBase &R : *ExitVPBB) {
-    createAndCollectMergePhiForReduction(
-        dyn_cast<VPInstruction>(&R), State, OrigLoop,
-        State.CFG.VPBB2IRBB[ExitVPBB], VectorizingEpilogue);
-  }
+  if (VectorizingEpilogue)
+    for (VPRecipeBase &R : *ExitVPBB) {
+      fixReductionScalarResumeWhenVectorizingEpilog(
+          &R, State, State.CFG.VPBB2IRBB[ExitVPBB]);
+    }
 
   // 2.6. Maintain Loop Hints
   // Keep all loop hints from the original loop on the vector loop (we'll
@@ -9518,6 +9513,17 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
         });
     FinalReductionResult->insertBefore(*MiddleVPBB, IP);
 
+    // Order is strict: if there are multiple successors, the first is the exit
+    // block, second is the scalar preheader.
+    VPBasicBlock *ScalarPHVPBB =
+        cast<VPBasicBlock>(MiddleVPBB->getSuccessors().back());
+    VPBuilder ScalarPHBuilder(ScalarPHVPBB);
+    auto *ResumePhiRecipe = ScalarPHBuilder.createNaryOp(
+        VPInstruction::ResumePhi, {FinalReductionResult, PhiR->getStartValue()},
+        {}, "bc.merge.rdx");
+    auto *RedPhi = cast<PHINode>(PhiR->getUnderlyingInstr());
+    Plan->addLiveOut(RedPhi, ResumePhiRecipe);
+
     // Adjust AnyOf reductions; replace the reduction phi for the selected value
     // with a boolean reduction phi node to check if the condition is true in
     // any iteration. The final value is selected by the final
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
index 1326751a847d..59db6c197ef8 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -65,7 +65,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT: No successors
 ; IF-EVL-INLOOP-EMPTY:
 ; IF-EVL-INLOOP-NEXT: scalar.ph:
+; IF-EVL-INLOOP-NEXT:   EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
 ; IF-EVL-INLOOP-NEXT: No successors
+; IF-EVL-INLOOP-EMPTY:
+; IF-EVL-INLOOP-NEXT: Live-out i32 %rdx = vp<[[RED_RESUME]]>
 ; IF-EVL-INLOOP-NEXT: }
 ;
 
@@ -104,7 +107,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-NEXT: No successors
 ; NO-VP-OUTLOOP-EMPTY:
 ; NO-VP-OUTLOOP-NEXT: scalar.ph:
+; NO-VP-OUTLOOP-NEXT:   EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
 ; NO-VP-OUTLOOP-NEXT: No successors
+; NO-VP-OUTLOOP-EMPTY:
+; NO-VP-OUTLOOP-NEXT: Live-out i32 %rdx = vp<[[RED_RESUME]]>
 ; NO-VP-OUTLOOP-NEXT: }
 ;
 
@@ -143,7 +149,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-NEXT: No successors
 ; NO-VP-INLOOP-EMPTY:
 ; NO-VP-INLOOP-NEXT: scalar.ph:
+; NO-VP-INLOOP-NEXT:   EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
 ; NO-VP-INLOOP-NEXT: No successors
+; NO-VP-INLOOP-EMPTY:
+; NO-VP-INLOOP-NEXT: Live-out i32 %rdx = vp<[[RED_RESUME]]>
 ; NO-VP-INLOOP-NEXT: }
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index 8e56614a2e3d..b05980bef1b3 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -232,9 +232,11 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:   EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
+; CHECK-NEXT:   EMIT vp<[[RESUME_RED:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<1234>
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: Live-out i32 %recur = vp<[[RESUME_1_P]]>
+; CHECK-NEXT: Live-out i32 %and.red = vp<[[RESUME_RED]]>
 ; CHECK-NEXT: }
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index 0dde507d08be..224729529566 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -165,7 +165,10 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
+; CHECK-NEXT:   EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00>
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: Live-out float %red = vp<[[RED_RESUME]]>
 ; CHECK-NEXT: }
 ;
 entry:
@@ -221,7 +224,10 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
+; CHECK-NEXT:   EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00>
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: Live-out float %red = vp<[[RED_RESUME]]>
 ; CHECK-NEXT: }
 ;
 entry:
@@ -447,7 +453,10 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
+; CHECK-NEXT:   EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00>
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: Live-out float %sum.07 = vp<[[RED_RESUME]]>
 ; CHECK-NEXT:}
 
 entry:
-- 
GitLab


From ad5b9441f949716570e89fcb27b76e9bfb4b7f70 Mon Sep 17 00:00:00 2001
From: Chris Apple <cja-private@pm.me>
Date: Mon, 28 Oct 2024 12:25:12 -0700
Subject: [PATCH 357/693] [rtsan][asan] NFC Fix hyperlink to CMake doc
 (#113931)

---
 clang/docs/AddressSanitizer.rst  | 2 +-
 clang/docs/RealtimeSanitizer.rst | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/docs/AddressSanitizer.rst b/clang/docs/AddressSanitizer.rst
index 76fdf5599505..d937cbfdf583 100644
--- a/clang/docs/AddressSanitizer.rst
+++ b/clang/docs/AddressSanitizer.rst
@@ -26,7 +26,7 @@ Typical slowdown introduced by AddressSanitizer is **2x**.
 How to build
 ============
 
-Build LLVM/Clang with `CMake <https://llvm.org/docs/CMake.html>` and enable
+Build LLVM/Clang with `CMake <https://llvm.org/docs/CMake.html>`_ and enable
 the ``compiler-rt`` runtime. An example CMake configuration that will allow
 for the use/testing of AddressSanitizer:
 
diff --git a/clang/docs/RealtimeSanitizer.rst b/clang/docs/RealtimeSanitizer.rst
index b09162cd99f4..41b8bbb33baf 100644
--- a/clang/docs/RealtimeSanitizer.rst
+++ b/clang/docs/RealtimeSanitizer.rst
@@ -21,7 +21,7 @@ The runtime slowdown introduced by RealtimeSanitizer is negligible.
 How to build
 ============
 
-Build LLVM/Clang with `CMake <https://llvm.org/docs/CMake.html>` and enable the
+Build LLVM/Clang with `CMake <https://llvm.org/docs/CMake.html>`_ and enable the
 ``compiler-rt`` runtime. An example CMake configuration that will allow for the
 use/testing of RealtimeSanitizer:
 
-- 
GitLab


From 19131c7f36e047898ea954ee5a187ac62f2ab09b Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Mon, 28 Oct 2024 12:49:28 -0700
Subject: [PATCH 358/693] [clang][modules][lldb] Fix build after #113391

Instead of changing the return type of `ModuleMap::findOrCreateModule`, this patch adds a counterpart that only returns `Module *` and thus has the same signature as `createModule()`, which is important in `ASTReader`.
---
 clang/include/clang/Lex/ModuleMap.h   | 14 +++++++++++---
 clang/lib/Lex/ModuleMap.cpp           | 23 +++++++++++++----------
 clang/lib/Serialization/ASTReader.cpp |  5 +++--
 3 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/clang/include/clang/Lex/ModuleMap.h b/clang/include/clang/Lex/ModuleMap.h
index 5ee152e4213a..53e9e0ec83dd 100644
--- a/clang/include/clang/Lex/ModuleMap.h
+++ b/clang/include/clang/Lex/ModuleMap.h
@@ -541,9 +541,17 @@ public:
   ///
   /// \param IsExplicit Whether this is an explicit submodule.
   ///
-  /// \returns The found or newly-created module.
-  Module *findOrCreateModule(StringRef Name, Module *Parent, bool IsFramework,
-                             bool IsExplicit);
+  /// \returns The found or newly-created module, along with a boolean value
+  /// that will be true if the module is newly-created.
+  std::pair<Module *, bool> findOrCreateModule(StringRef Name, Module *Parent,
+                                               bool IsFramework,
+                                               bool IsExplicit);
+  /// Call \c ModuleMap::findOrCreateModule and throw away the information
+  /// whether the module was found or created.
+  Module *findOrCreateModuleFirst(StringRef Name, Module *Parent,
+                                  bool IsFramework, bool IsExplicit) {
+    return findOrCreateModule(Name, Parent, IsFramework, IsExplicit).first;
+  }
   /// Create new submodule, assuming it does not exist. This function can only
   /// be called when it is guaranteed that this submodule does not exist yet.
   /// The parameters have same semantics as \c ModuleMap::findOrCreateModule.
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index 10774429a217..dc9d2bfd5629 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -655,8 +655,8 @@ ModuleMap::findOrCreateModuleForHeaderInUmbrellaDir(FileEntryRef File) {
         SmallString<32> NameBuf;
         StringRef Name = sanitizeFilenameAsIdentifier(
             llvm::sys::path::stem(SkippedDir.getName()), NameBuf);
-        Result =
-            findOrCreateModule(Name, Result, /*IsFramework=*/false, Explicit);
+        Result = findOrCreateModuleFirst(Name, Result, /*IsFramework=*/false,
+                                         Explicit);
         setInferredModuleAllowedBy(Result, UmbrellaModuleMap);
 
         // Associate the module and the directory.
@@ -672,8 +672,8 @@ ModuleMap::findOrCreateModuleForHeaderInUmbrellaDir(FileEntryRef File) {
       SmallString<32> NameBuf;
       StringRef Name = sanitizeFilenameAsIdentifier(
                          llvm::sys::path::stem(File.getName()), NameBuf);
-      Result =
-          findOrCreateModule(Name, Result, /*IsFramework=*/false, Explicit);
+      Result = findOrCreateModuleFirst(Name, Result, /*IsFramework=*/false,
+                                       Explicit);
       setInferredModuleAllowedBy(Result, UmbrellaModuleMap);
       Result->addTopHeader(File);
 
@@ -857,14 +857,17 @@ Module *ModuleMap::lookupModuleQualified(StringRef Name, Module *Context) const{
   return Context->findSubmodule(Name);
 }
 
-Module *ModuleMap::findOrCreateModule(StringRef Name, Module *Parent,
-                                      bool IsFramework, bool IsExplicit) {
+std::pair<Module *, bool> ModuleMap::findOrCreateModule(StringRef Name,
+                                                        Module *Parent,
+                                                        bool IsFramework,
+                                                        bool IsExplicit) {
   // Try to find an existing module with this name.
   if (Module *Sub = lookupModuleQualified(Name, Parent))
-    return Sub;
+    return std::make_pair(Sub, false);
 
   // Create a new module with this name.
-  return createModule(Name, Parent, IsFramework, IsExplicit);
+  Module *M = createModule(Name, Parent, IsFramework, IsExplicit);
+  return std::make_pair(M, true);
 }
 
 Module *ModuleMap::createModule(StringRef Name, Module *Parent,
@@ -2129,8 +2132,8 @@ void ModuleMapParser::parseModuleDecl() {
     ActiveModule =
         Map.createShadowedModule(ModuleName, Framework, ShadowingModule);
   } else {
-    ActiveModule =
-        Map.findOrCreateModule(ModuleName, ActiveModule, Framework, Explicit);
+    ActiveModule = Map.findOrCreateModuleFirst(ModuleName, ActiveModule,
+                                               Framework, Explicit);
   }
 
   ActiveModule->DefinitionLoc = ModuleNameLoc;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 74a79ac54bb4..8d8f9378cfea 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -5760,8 +5760,9 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F,
   // If we don't know the top-level module, there's no point in doing qualified
   // lookup of its submodules; it won't find anything anywhere within this tree.
   // Let's skip that and avoid some string lookups.
-  auto CreateModule = !KnowsTopLevelModule ? &ModuleMap::createModule
-                                           : &ModuleMap::findOrCreateModule;
+  auto CreateModule = !KnowsTopLevelModule
+                          ? &ModuleMap::createModule
+                          : &ModuleMap::findOrCreateModuleFirst;
 
   bool First = true;
   Module *CurrentModule = nullptr;
-- 
GitLab


From 67bcce21415c7f687c28eb727c40b27924335f5a Mon Sep 17 00:00:00 2001
From: Igor Kudrin <ikudrin@accesssoftek.com>
Date: Tue, 29 Oct 2024 03:14:42 +0700
Subject: [PATCH 359/693] [CFI][LowerTypeTests] Fix indirect call with alias
 (#106185)

Motivation example:
```
> cat test.cpp
extern "C" [[gnu::weak]] void f() {}
void alias() __attribute__((alias("f")));
int main() { auto p = alias; p(); }
> clang test.cpp -fsanitize=cfi-icall -flto=thin -fuse-ld=lld
> ./a.out
[1]    1868 illegal hardware instruction  ./a.out
```

If the address of a function was only taken through its alias, the
function was not considered exported and therefore was not included
in the CFI jumptable. This resulted in `@llvm.type.test()` being lowered
to `false`, and consequently the indirect call to the function was
eventually optimized to `ubsantrap()`.
---
 llvm/include/llvm/IR/ModuleSummaryIndexYAML.h | 102 +++++++++++++-----
 llvm/lib/Transforms/IPO/LowerTypeTests.cpp    |   6 +-
 .../LowerTypeTests/cfi-icall-alias.ll         |  54 ++++++++++
 3 files changed, 133 insertions(+), 29 deletions(-)
 create mode 100644 llvm/test/Transforms/LowerTypeTests/cfi-icall-alias.ll

diff --git a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h
index 902d1305c818..d12bc260f5cf 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h
@@ -135,10 +135,14 @@ template <> struct MappingTraits<TypeIdSummary> {
   }
 };
 
-struct FunctionSummaryYaml {
+struct GlobalValueSummaryYaml {
+  // Commonly used fields
   unsigned Linkage, Visibility;
   bool NotEligibleToImport, Live, IsLocal, CanAutoHide;
   unsigned ImportType;
+  // Fields for AliasSummary
+  std::optional<uint64_t> Aliasee;
+  // Fields for FunctionSummary
   std::vector<uint64_t> Refs;
   std::vector<uint64_t> TypeTests;
   std::vector<FunctionSummary::VFuncId> TypeTestAssumeVCalls,
@@ -176,8 +180,8 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(FunctionSummary::ConstVCall)
 namespace llvm {
 namespace yaml {
 
-template <> struct MappingTraits<FunctionSummaryYaml> {
-  static void mapping(IO &io, FunctionSummaryYaml& summary) {
+template <> struct MappingTraits<GlobalValueSummaryYaml> {
+  static void mapping(IO &io, GlobalValueSummaryYaml &summary) {
     io.mapOptional("Linkage", summary.Linkage);
     io.mapOptional("Visibility", summary.Visibility);
     io.mapOptional("NotEligibleToImport", summary.NotEligibleToImport);
@@ -185,6 +189,7 @@ template <> struct MappingTraits<FunctionSummaryYaml> {
     io.mapOptional("Local", summary.IsLocal);
     io.mapOptional("CanAutoHide", summary.CanAutoHide);
     io.mapOptional("ImportType", summary.ImportType);
+    io.mapOptional("Aliasee", summary.Aliasee);
     io.mapOptional("Refs", summary.Refs);
     io.mapOptional("TypeTests", summary.TypeTests);
     io.mapOptional("TypeTestAssumeVCalls", summary.TypeTestAssumeVCalls);
@@ -199,7 +204,7 @@ template <> struct MappingTraits<FunctionSummaryYaml> {
 } // End yaml namespace
 } // End llvm namespace
 
-LLVM_YAML_IS_SEQUENCE_VECTOR(FunctionSummaryYaml)
+LLVM_YAML_IS_SEQUENCE_VECTOR(GlobalValueSummaryYaml)
 
 namespace llvm {
 namespace yaml {
@@ -207,61 +212,99 @@ namespace yaml {
 // FIXME: Add YAML mappings for the rest of the module summary.
 template <> struct CustomMappingTraits<GlobalValueSummaryMapTy> {
   static void inputOne(IO &io, StringRef Key, GlobalValueSummaryMapTy &V) {
-    std::vector<FunctionSummaryYaml> FSums;
-    io.mapRequired(Key.str().c_str(), FSums);
+    std::vector<GlobalValueSummaryYaml> GVSums;
+    io.mapRequired(Key.str().c_str(), GVSums);
     uint64_t KeyInt;
     if (Key.getAsInteger(0, KeyInt)) {
       io.setError("key not an integer");
       return;
     }
     auto &Elem = V.try_emplace(KeyInt, /*IsAnalysis=*/false).first->second;
-    for (auto &FSum : FSums) {
+    for (auto &GVSum : GVSums) {
+      GlobalValueSummary::GVFlags GVFlags(
+          static_cast<GlobalValue::LinkageTypes>(GVSum.Linkage),
+          static_cast<GlobalValue::VisibilityTypes>(GVSum.Visibility),
+          GVSum.NotEligibleToImport, GVSum.Live, GVSum.IsLocal,
+          GVSum.CanAutoHide,
+          static_cast<GlobalValueSummary::ImportKind>(GVSum.ImportType));
+      if (GVSum.Aliasee) {
+        auto ASum = std::make_unique<AliasSummary>(GVFlags);
+        if (!V.count(*GVSum.Aliasee))
+          V.emplace(*GVSum.Aliasee, /*IsAnalysis=*/false);
+        ValueInfo AliaseeVI(/*IsAnalysis=*/false, &*V.find(*GVSum.Aliasee));
+        // Note: Aliasee cannot be filled until all summaries are loaded.
+        // This is done in fixAliaseeLinks() which is called in
+        // MappingTraits<ModuleSummaryIndex>::mapping().
+        ASum->setAliasee(AliaseeVI, /*Aliasee=*/nullptr);
+        Elem.SummaryList.push_back(std::move(ASum));
+        continue;
+      }
       SmallVector<ValueInfo, 0> Refs;
-      Refs.reserve(FSum.Refs.size());
-      for (auto &RefGUID : FSum.Refs) {
+      Refs.reserve(GVSum.Refs.size());
+      for (auto &RefGUID : GVSum.Refs) {
         auto It = V.try_emplace(RefGUID, /*IsAnalysis=*/false).first;
         Refs.push_back(ValueInfo(/*IsAnalysis=*/false, &*It));
       }
       Elem.SummaryList.push_back(std::make_unique<FunctionSummary>(
-          GlobalValueSummary::GVFlags(
-              static_cast<GlobalValue::LinkageTypes>(FSum.Linkage),
-              static_cast<GlobalValue::VisibilityTypes>(FSum.Visibility),
-              FSum.NotEligibleToImport, FSum.Live, FSum.IsLocal,
-              FSum.CanAutoHide,
-              static_cast<GlobalValueSummary::ImportKind>(FSum.ImportType)),
-          /*NumInsts=*/0, FunctionSummary::FFlags{}, std::move(Refs),
-          SmallVector<FunctionSummary::EdgeTy, 0>{}, std::move(FSum.TypeTests),
-          std::move(FSum.TypeTestAssumeVCalls),
-          std::move(FSum.TypeCheckedLoadVCalls),
-          std::move(FSum.TypeTestAssumeConstVCalls),
-          std::move(FSum.TypeCheckedLoadConstVCalls),
+          GVFlags, /*NumInsts=*/0, FunctionSummary::FFlags{}, std::move(Refs),
+          SmallVector<FunctionSummary::EdgeTy, 0>{}, std::move(GVSum.TypeTests),
+          std::move(GVSum.TypeTestAssumeVCalls),
+          std::move(GVSum.TypeCheckedLoadVCalls),
+          std::move(GVSum.TypeTestAssumeConstVCalls),
+          std::move(GVSum.TypeCheckedLoadConstVCalls),
           ArrayRef<FunctionSummary::ParamAccess>{}, ArrayRef<CallsiteInfo>{},
           ArrayRef<AllocInfo>{}));
     }
   }
   static void output(IO &io, GlobalValueSummaryMapTy &V) {
     for (auto &P : V) {
-      std::vector<FunctionSummaryYaml> FSums;
+      std::vector<GlobalValueSummaryYaml> GVSums;
       for (auto &Sum : P.second.SummaryList) {
         if (auto *FSum = dyn_cast<FunctionSummary>(Sum.get())) {
           std::vector<uint64_t> Refs;
           Refs.reserve(FSum->refs().size());
           for (auto &VI : FSum->refs())
             Refs.push_back(VI.getGUID());
-          FSums.push_back(FunctionSummaryYaml{
+          GVSums.push_back(GlobalValueSummaryYaml{
               FSum->flags().Linkage, FSum->flags().Visibility,
               static_cast<bool>(FSum->flags().NotEligibleToImport),
               static_cast<bool>(FSum->flags().Live),
               static_cast<bool>(FSum->flags().DSOLocal),
               static_cast<bool>(FSum->flags().CanAutoHide),
-              FSum->flags().ImportType, Refs, FSum->type_tests(),
-              FSum->type_test_assume_vcalls(), FSum->type_checked_load_vcalls(),
+              FSum->flags().ImportType, /*Aliasee=*/std::nullopt, Refs,
+              FSum->type_tests(), FSum->type_test_assume_vcalls(),
+              FSum->type_checked_load_vcalls(),
               FSum->type_test_assume_const_vcalls(),
               FSum->type_checked_load_const_vcalls()});
-          }
+        } else if (auto *ASum = dyn_cast<AliasSummary>(Sum.get());
+                   ASum && ASum->hasAliasee()) {
+          GVSums.push_back(GlobalValueSummaryYaml{
+              ASum->flags().Linkage, ASum->flags().Visibility,
+              static_cast<bool>(ASum->flags().NotEligibleToImport),
+              static_cast<bool>(ASum->flags().Live),
+              static_cast<bool>(ASum->flags().DSOLocal),
+              static_cast<bool>(ASum->flags().CanAutoHide),
+              ASum->flags().ImportType,
+              /*Aliasee=*/ASum->getAliaseeGUID()});
+        }
+      }
+      if (!GVSums.empty())
+        io.mapRequired(llvm::utostr(P.first).c_str(), GVSums);
+    }
+  }
+  static void fixAliaseeLinks(GlobalValueSummaryMapTy &V) {
+    for (auto &P : V) {
+      for (auto &Sum : P.second.SummaryList) {
+        if (auto *Alias = dyn_cast<AliasSummary>(Sum.get())) {
+          ValueInfo AliaseeVI = Alias->getAliaseeVI();
+          auto AliaseeSL = AliaseeVI.getSummaryList();
+          if (AliaseeSL.empty()) {
+            ValueInfo EmptyVI;
+            Alias->setAliasee(EmptyVI, nullptr);
+          } else
+            Alias->setAliasee(AliaseeVI, AliaseeSL[0].get());
+        }
       }
-      if (!FSums.empty())
-        io.mapRequired(llvm::utostr(P.first).c_str(), FSums);
     }
   }
 };
@@ -281,6 +324,9 @@ template <> struct CustomMappingTraits<TypeIdSummaryMapTy> {
 template <> struct MappingTraits<ModuleSummaryIndex> {
   static void mapping(IO &io, ModuleSummaryIndex& index) {
     io.mapOptional("GlobalValueMap", index.GlobalValueMap);
+    if (!io.outputting())
+      CustomMappingTraits<GlobalValueSummaryMapTy>::fixAliaseeLinks(
+          index.GlobalValueMap);
     io.mapOptional("TypeIdMap", index.TypeIdMap);
     io.mapOptional("WithGlobalValueDeadStripping",
                    index.WithGlobalValueDeadStripping);
diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index 3fcfc6a87677..6ba371069bb2 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -2083,8 +2083,12 @@ bool LowerTypeTestsModule::lower() {
     for (auto &I : *ExportSummary)
       for (auto &GVS : I.second.SummaryList)
         if (GVS->isLive())
-          for (const auto &Ref : GVS->refs())
+          for (const auto &Ref : GVS->refs()) {
             AddressTaken.insert(Ref.getGUID());
+            for (auto &RefGVS : Ref.getSummaryList())
+              if (auto Alias = dyn_cast<AliasSummary>(RefGVS.get()))
+                AddressTaken.insert(Alias->getAliaseeGUID());
+          }
 
     NamedMDNode *CfiFunctionsMD = M.getNamedMetadata("cfi.functions");
     if (CfiFunctionsMD) {
diff --git a/llvm/test/Transforms/LowerTypeTests/cfi-icall-alias.ll b/llvm/test/Transforms/LowerTypeTests/cfi-icall-alias.ll
new file mode 100644
index 000000000000..0c5324ee96c9
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/cfi-icall-alias.ll
@@ -0,0 +1,54 @@
+;; Check that if the address of a weak function is only taken through an alias,
+;; it is still added to a list of exported functions and @llvm.type.test() is
+;; lowered to an actual check against the generated CFI jumptable.
+
+RUN: rm -rf %t.dir && split-file %s %t.dir && cd %t.dir
+RUN: opt test.ll --thinlto-bc --thinlto-split-lto-unit -o test.bc
+RUN: llvm-modextract test.bc -n 0 -o test0.bc
+RUN: llvm-modextract test.bc -n 1 -o test1.bc
+
+;; Check that a CFI jumptable is generated.
+RUN: opt test1.bc -passes=lowertypetests -lowertypetests-read-summary=in.yaml \
+RUN:   -lowertypetests-summary-action=export -lowertypetests-write-summary=exported.yaml \
+RUN:   -S -o - | FileCheck %s --check-prefix=REGULAR
+REGULAR: @__typeid__ZTSFvvE_global_addr = hidden alias i8, ptr @.cfi.jumptable
+REGULAR: @f = alias void (), ptr @.cfi.jumptable
+REGULAR: define private void @.cfi.jumptable()
+
+;; CHECK that @llvm.type.test() is lowered to an actual check.
+RUN: opt test0.bc -passes=lowertypetests -lowertypetests-read-summary=exported.yaml \
+RUN:   -lowertypetests-summary-action=import -S -o - | FileCheck %s --check-prefix=THIN
+THIN:      define i1 @test() {
+THIN-NEXT:   %1 = icmp eq i64 ptrtoint (ptr @alias to i64), ptrtoint (ptr @__typeid__ZTSFvvE_global_addr to i64)
+THIN-NEXT:   ret i1 %1
+THIN-NEXT: }
+
+;--- test.ll
+target triple = "x86_64-pc-linux-gnu"
+
+@alias = alias void(), ptr @f
+
+define weak void @f() !type !0 {
+  ret void
+}
+
+define i1 @test() {
+  %1 = call i1 @llvm.type.test(ptr nonnull @alias, metadata !"_ZTSFvvE")
+  ret i1 %1
+}
+
+declare i1 @llvm.type.test(ptr, metadata)
+
+!0 = !{i64 0, !"_ZTSFvvE"}
+;--- in.yaml
+---
+GlobalValueMap:
+  8346051122425466633: # guid("test")
+    - Live: true
+      Refs: [5833419078793185394] # guid("alias")
+      TypeTests: [9080559750644022485] # guid("_ZTSFvvE")
+  5833419078793185394: # guid("alias")
+    - Aliasee: 14740650423002898831 # guid("f")
+  14740650423002898831: # guid("f")
+    -
+...
-- 
GitLab


From 7bd8a165f95123e390f9cbb0a6a5e60d835a4461 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 28 Oct 2024 13:20:46 -0700
Subject: [PATCH 360/693] [X86] Don't allow '+f' as an inline asm constraint.
 (#113871)

f cannot be used as an output constraint. We already errored for '=f'
but not '+f'.

Fixes #113692.
---
 clang/lib/Basic/Targets/X86.cpp | 2 +-
 clang/test/Sema/asm.c           | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index d067ec218b52..700c2f9a5dbd 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -1465,7 +1465,7 @@ bool X86TargetInfo::validateAsmConstraint(
     }
   case 'f': // Any x87 floating point stack register.
     // Constraint 'f' cannot be used for output operands.
-    if (Info.ConstraintStr[0] == '=')
+    if (Info.ConstraintStr[0] == '=' || Info.ConstraintStr[0] == '+')
       return false;
     Info.setAllowsRegister();
     return true;
diff --git a/clang/test/Sema/asm.c b/clang/test/Sema/asm.c
index 6cd95c71604d..28ef3ec6ce09 100644
--- a/clang/test/Sema/asm.c
+++ b/clang/test/Sema/asm.c
@@ -204,6 +204,12 @@ double f_output_constraint(void) {
   return result;
 }
 
+double f_output_constraint_2(void) {
+  double result;
+  __asm("foo1": "+f" (result)); // expected-error {{invalid output constraint '+f' in asm}}
+  return result;
+}
+
 void fn1(void) {
   int l;
   __asm__(""
-- 
GitLab


From 481bce018ea8872277f79102842eaf8a55f634a2 Mon Sep 17 00:00:00 2001
From: joaosaffran <126493771+joaosaffran@users.noreply.github.com>
Date: Mon, 28 Oct 2024 13:26:59 -0700
Subject: [PATCH 361/693] Adding splitdouble HLSL function (#109331)

- Adding hlsl `splitdouble` intrinsics
- Adding DXIL lowering
- Adding SPIRV lowering
- Adding test

Fixes: #108901

---------

Co-authored-by: Joao Saffran <jderezende@microsoft.com>
---
 clang/include/clang/Basic/Builtins.td         |  6 ++
 clang/lib/CodeGen/CGBuiltin.cpp               | 82 +++++++++++++++++
 clang/lib/CodeGen/CGCall.cpp                  | 14 +--
 clang/lib/CodeGen/CGExpr.cpp                  | 13 ++-
 clang/lib/CodeGen/CodeGenFunction.h           | 10 +-
 clang/lib/Headers/hlsl/hlsl_intrinsics.h      | 18 ++++
 clang/lib/Sema/SemaHLSL.cpp                   | 74 +++++++++++----
 .../CodeGenHLSL/builtins/splitdouble.hlsl     | 91 +++++++++++++++++++
 .../test/SemaHLSL/BuiltIns/asuint-errors.hlsl | 30 ++++++
 .../SemaHLSL/BuiltIns/splitdouble-errors.hlsl | 76 ++++++++++++++++
 llvm/lib/Target/DirectX/DXIL.td               | 10 ++
 llvm/lib/Target/DirectX/DXILOpBuilder.cpp     | 13 +++
 llvm/lib/Target/DirectX/DXILOpBuilder.h       |  4 +
 llvm/lib/Target/DirectX/DXILOpLowering.cpp    | 55 +++++++++++
 llvm/test/CodeGen/DirectX/split-double.ll     | 45 ---------
 llvm/test/CodeGen/DirectX/splitdouble.ll      | 76 ++++++++++++++++
 .../SPIRV/hlsl-intrinsics/splitdouble.ll      | 40 ++++++++
 17 files changed, 581 insertions(+), 76 deletions(-)
 create mode 100644 clang/test/CodeGenHLSL/builtins/splitdouble.hlsl
 create mode 100644 clang/test/SemaHLSL/BuiltIns/splitdouble-errors.hlsl
 delete mode 100644 llvm/test/CodeGen/DirectX/split-double.ll
 create mode 100644 llvm/test/CodeGen/DirectX/splitdouble.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/splitdouble.ll

diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 90475a361bb8..9bd67e0cefeb 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -4871,6 +4871,12 @@ def HLSLRadians : LangBuiltin<"HLSL_LANG"> {
   let Prototype = "void(...)";
 }
 
+def HLSLSplitDouble: LangBuiltin<"HLSL_LANG"> {
+  let Spellings = ["__builtin_hlsl_elementwise_splitdouble"];
+  let Attributes = [NoThrow, Const];
+  let Prototype = "void(...)";
+}
+
 // Builtins for XRay.
 def XRayCustomEvent : Builtin {
   let Spellings = ["__xray_customevent"];
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index a57c95d5b966..65d7f5c54a19 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -17,6 +17,7 @@
 #include "CGObjCRuntime.h"
 #include "CGOpenCLRuntime.h"
 #include "CGRecordLayout.h"
+#include "CGValue.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
 #include "ConstantEmitter.h"
@@ -25,8 +26,10 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
+#include "clang/AST/Expr.h"
 #include "clang/AST/OSLog.h"
 #include "clang/AST/OperationKinds.h"
+#include "clang/AST/Type.h"
 #include "clang/Basic/TargetBuiltins.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/TargetOptions.h"
@@ -67,6 +70,7 @@
 #include "llvm/TargetParser/X86TargetParser.h"
 #include <optional>
 #include <sstream>
+#include <utility>
 
 using namespace clang;
 using namespace CodeGen;
@@ -95,6 +99,76 @@ static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size,
   I->addAnnotationMetadata("auto-init");
 }
 
+static Value *handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF) {
+  Value *Op0 = CGF->EmitScalarExpr(E->getArg(0));
+  const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(E->getArg(1));
+  const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(E->getArg(2));
+
+  CallArgList Args;
+  LValue Op1TmpLValue =
+      CGF->EmitHLSLOutArgExpr(OutArg1, Args, OutArg1->getType());
+  LValue Op2TmpLValue =
+      CGF->EmitHLSLOutArgExpr(OutArg2, Args, OutArg2->getType());
+
+  if (CGF->getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee())
+    Args.reverseWritebacks();
+
+  Value *LowBits = nullptr;
+  Value *HighBits = nullptr;
+
+  if (CGF->CGM.getTarget().getTriple().isDXIL()) {
+
+    llvm::Type *RetElementTy = CGF->Int32Ty;
+    if (auto *Op0VecTy = E->getArg(0)->getType()->getAs<clang::VectorType>())
+      RetElementTy = llvm::VectorType::get(
+          CGF->Int32Ty, ElementCount::getFixed(Op0VecTy->getNumElements()));
+    auto *RetTy = llvm::StructType::get(RetElementTy, RetElementTy);
+
+    CallInst *CI = CGF->Builder.CreateIntrinsic(
+        RetTy, Intrinsic::dx_splitdouble, {Op0}, nullptr, "hlsl.splitdouble");
+
+    LowBits = CGF->Builder.CreateExtractValue(CI, 0);
+    HighBits = CGF->Builder.CreateExtractValue(CI, 1);
+
+  } else {
+    // For Non DXIL targets we generate the instructions.
+
+    if (!Op0->getType()->isVectorTy()) {
+      FixedVectorType *DestTy = FixedVectorType::get(CGF->Int32Ty, 2);
+      Value *Bitcast = CGF->Builder.CreateBitCast(Op0, DestTy);
+
+      LowBits = CGF->Builder.CreateExtractElement(Bitcast, (uint64_t)0);
+      HighBits = CGF->Builder.CreateExtractElement(Bitcast, 1);
+    } else {
+      int NumElements = 1;
+      if (const auto *VecTy =
+              E->getArg(0)->getType()->getAs<clang::VectorType>())
+        NumElements = VecTy->getNumElements();
+
+      FixedVectorType *Uint32VecTy =
+          FixedVectorType::get(CGF->Int32Ty, NumElements * 2);
+      Value *Uint32Vec = CGF->Builder.CreateBitCast(Op0, Uint32VecTy);
+      if (NumElements == 1) {
+        LowBits = CGF->Builder.CreateExtractElement(Uint32Vec, (uint64_t)0);
+        HighBits = CGF->Builder.CreateExtractElement(Uint32Vec, 1);
+      } else {
+        SmallVector<int> EvenMask, OddMask;
+        for (int I = 0, E = NumElements; I != E; ++I) {
+          EvenMask.push_back(I * 2);
+          OddMask.push_back(I * 2 + 1);
+        }
+        LowBits = CGF->Builder.CreateShuffleVector(Uint32Vec, EvenMask);
+        HighBits = CGF->Builder.CreateShuffleVector(Uint32Vec, OddMask);
+      }
+    }
+  }
+  CGF->Builder.CreateStore(LowBits, Op1TmpLValue.getAddress());
+  auto *LastInst =
+      CGF->Builder.CreateStore(HighBits, Op2TmpLValue.getAddress());
+  CGF->EmitWritebacks(Args);
+  return LastInst;
+}
+
 /// getBuiltinLibFunction - Given a builtin id for a function like
 /// "__builtin_fabsf", return a Function* for "fabsf".
 llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
@@ -18959,6 +19033,14 @@ case Builtin::BI__builtin_hlsl_elementwise_isinf: {
         CGM.getHLSLRuntime().getRadiansIntrinsic(), ArrayRef<Value *>{Op0},
         nullptr, "hlsl.radians");
   }
+  case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {
+
+    assert((E->getArg(0)->getType()->hasFloatingRepresentation() &&
+            E->getArg(1)->getType()->hasUnsignedIntegerRepresentation() &&
+            E->getArg(2)->getType()->hasUnsignedIntegerRepresentation()) &&
+           "asuint operands types mismatch");
+    return handleHlslSplitdouble(E, this);
+  }
   }
   return nullptr;
 }
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 64e60f0616d7..8f4f5d3ed816 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -40,6 +40,7 @@
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Type.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include <optional>
 using namespace clang;
@@ -4243,12 +4244,6 @@ static void emitWriteback(CodeGenFunction &CGF,
     CGF.EmitBlock(contBB);
 }
 
-static void emitWritebacks(CodeGenFunction &CGF,
-                           const CallArgList &args) {
-  for (const auto &I : args.writebacks())
-    emitWriteback(CGF, I);
-}
-
 static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
                                             const CallArgList &CallArgs) {
   ArrayRef<CallArgList::CallArgCleanup> Cleanups =
@@ -4717,6 +4712,11 @@ void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
   IsUsed = true;
 }
 
+void CodeGenFunction::EmitWritebacks(const CallArgList &args) {
+  for (const auto &I : args.writebacks())
+    emitWriteback(*this, I);
+}
+
 void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
                                   QualType type) {
   DisableDebugLocationUpdates Dis(*this, E);
@@ -5940,7 +5940,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   // Emit any call-associated writebacks immediately.  Arguably this
   // should happen after any return-value munging.
   if (CallArgs.hasWritebacks())
-    emitWritebacks(*this, CallArgs);
+    EmitWritebacks(CallArgs);
 
   // The stack cleanup for inalloca arguments has to run out of the normal
   // lexical order, so deactivate it and run it manually here.
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index e0ea65bcaf36..e90e8da3e9f1 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -5460,9 +5460,8 @@ LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
   return getOrCreateOpaqueLValueMapping(e);
 }
 
-void CodeGenFunction::EmitHLSLOutArgExpr(const HLSLOutArgExpr *E,
-                                         CallArgList &Args, QualType Ty) {
-
+std::pair<LValue, LValue>
+CodeGenFunction::EmitHLSLOutArgLValues(const HLSLOutArgExpr *E, QualType Ty) {
   // Emitting the casted temporary through an opaque value.
   LValue BaseLV = EmitLValue(E->getArgLValue());
   OpaqueValueMappingData::bind(*this, E->getOpaqueArgLValue(), BaseLV);
@@ -5476,6 +5475,13 @@ void CodeGenFunction::EmitHLSLOutArgExpr(const HLSLOutArgExpr *E,
                                TempLV);
 
   OpaqueValueMappingData::bind(*this, E->getCastedTemporary(), TempLV);
+  return std::make_pair(BaseLV, TempLV);
+}
+
+LValue CodeGenFunction::EmitHLSLOutArgExpr(const HLSLOutArgExpr *E,
+                                           CallArgList &Args, QualType Ty) {
+
+  auto [BaseLV, TempLV] = EmitHLSLOutArgLValues(E, Ty);
 
   llvm::Value *Addr = TempLV.getAddress().getBasePointer();
   llvm::Type *ElTy = ConvertTypeForMem(TempLV.getType());
@@ -5488,6 +5494,7 @@ void CodeGenFunction::EmitHLSLOutArgExpr(const HLSLOutArgExpr *E,
   Args.addWriteback(BaseLV, TmpAddr, nullptr, E->getWritebackCast(),
                     LifetimeSize);
   Args.add(RValue::get(TmpAddr, *this), Ty);
+  return TempLV;
 }
 
 LValue
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 750a6cc24bad..3ff4458fb320 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -4296,8 +4296,11 @@ public:
   LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
   LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e);
   LValue EmitHLSLArrayAssignLValue(const BinaryOperator *E);
-  void EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args,
-                          QualType Ty);
+
+  std::pair<LValue, LValue> EmitHLSLOutArgLValues(const HLSLOutArgExpr *E,
+                                                  QualType Ty);
+  LValue EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args,
+                            QualType Ty);
 
   Address EmitExtVectorElementLValue(LValue V);
 
@@ -5147,6 +5150,9 @@ public:
                            SourceLocation ArgLoc, AbstractCallee AC,
                            unsigned ParmNum);
 
+  /// EmitWriteback - Emit callbacks for function.
+  void EmitWritebacks(const CallArgList &Args);
+
   /// EmitCallArg - Emit a single call argument.
   void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);
 
diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index 30dce60b3ff7..8ade4b27f360 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -438,6 +438,24 @@ template <typename T> constexpr uint asuint(T F) {
   return __detail::bit_cast<uint, T>(F);
 }
 
+//===----------------------------------------------------------------------===//
+// asuint splitdouble builtins
+//===----------------------------------------------------------------------===//
+
+/// \fn void asuint(double D, out uint lowbits, out int highbits)
+/// \brief Split and interprets the lowbits and highbits of double D into uints.
+/// \param D The input double.
+/// \param lowbits The output lowbits of D.
+/// \param highbits The output highbits of D.
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_splitdouble)
+void asuint(double, out uint, out uint);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_splitdouble)
+void asuint(double2, out uint2, out uint2);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_splitdouble)
+void asuint(double3, out uint3, out uint3);
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_splitdouble)
+void asuint(double4, out uint4, out uint4);
+
 //===----------------------------------------------------------------------===//
 // atan builtins
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 1f6c5b8d4561..a472538236e2 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -1698,18 +1698,27 @@ static bool CheckVectorElementCallArgs(Sema *S, CallExpr *TheCall) {
   return true;
 }
 
-static bool CheckArgsTypesAreCorrect(
+bool CheckArgTypeIsCorrect(
+    Sema *S, Expr *Arg, QualType ExpectedType,
+    llvm::function_ref<bool(clang::QualType PassedType)> Check) {
+  QualType PassedType = Arg->getType();
+  if (Check(PassedType)) {
+    if (auto *VecTyA = PassedType->getAs<VectorType>())
+      ExpectedType = S->Context.getVectorType(
+          ExpectedType, VecTyA->getNumElements(), VecTyA->getVectorKind());
+    S->Diag(Arg->getBeginLoc(), diag::err_typecheck_convert_incompatible)
+        << PassedType << ExpectedType << 1 << 0 << 0;
+    return true;
+  }
+  return false;
+}
+
+bool CheckAllArgTypesAreCorrect(
     Sema *S, CallExpr *TheCall, QualType ExpectedType,
     llvm::function_ref<bool(clang::QualType PassedType)> Check) {
   for (unsigned i = 0; i < TheCall->getNumArgs(); ++i) {
-    QualType PassedType = TheCall->getArg(i)->getType();
-    if (Check(PassedType)) {
-      if (auto *VecTyA = PassedType->getAs<VectorType>())
-        ExpectedType = S->Context.getVectorType(
-            ExpectedType, VecTyA->getNumElements(), VecTyA->getVectorKind());
-      S->Diag(TheCall->getArg(0)->getBeginLoc(),
-              diag::err_typecheck_convert_incompatible)
-          << PassedType << ExpectedType << 1 << 0 << 0;
+    Expr *Arg = TheCall->getArg(i);
+    if (CheckArgTypeIsCorrect(S, Arg, ExpectedType, Check)) {
       return true;
     }
   }
@@ -1720,8 +1729,8 @@ static bool CheckAllArgsHaveFloatRepresentation(Sema *S, CallExpr *TheCall) {
   auto checkAllFloatTypes = [](clang::QualType PassedType) -> bool {
     return !PassedType->hasFloatingRepresentation();
   };
-  return CheckArgsTypesAreCorrect(S, TheCall, S->Context.FloatTy,
-                                  checkAllFloatTypes);
+  return CheckAllArgTypesAreCorrect(S, TheCall, S->Context.FloatTy,
+                                    checkAllFloatTypes);
 }
 
 static bool CheckFloatOrHalfRepresentations(Sema *S, CallExpr *TheCall) {
@@ -1732,8 +1741,19 @@ static bool CheckFloatOrHalfRepresentations(Sema *S, CallExpr *TheCall) {
             : PassedType;
     return !BaseType->isHalfType() && !BaseType->isFloat32Type();
   };
-  return CheckArgsTypesAreCorrect(S, TheCall, S->Context.FloatTy,
-                                  checkFloatorHalf);
+  return CheckAllArgTypesAreCorrect(S, TheCall, S->Context.FloatTy,
+                                    checkFloatorHalf);
+}
+
+static bool CheckModifiableLValue(Sema *S, CallExpr *TheCall,
+                                  unsigned ArgIndex) {
+  auto *Arg = TheCall->getArg(ArgIndex);
+  SourceLocation OrigLoc = Arg->getExprLoc();
+  if (Arg->IgnoreCasts()->isModifiableLvalue(S->Context, &OrigLoc) ==
+      Expr::MLV_Valid)
+    return false;
+  S->Diag(OrigLoc, diag::error_hlsl_inout_lvalue) << Arg << 0;
+  return true;
 }
 
 static bool CheckNoDoubleVectors(Sema *S, CallExpr *TheCall) {
@@ -1742,24 +1762,24 @@ static bool CheckNoDoubleVectors(Sema *S, CallExpr *TheCall) {
       return VecTy->getElementType()->isDoubleType();
     return false;
   };
-  return CheckArgsTypesAreCorrect(S, TheCall, S->Context.FloatTy,
-                                  checkDoubleVector);
+  return CheckAllArgTypesAreCorrect(S, TheCall, S->Context.FloatTy,
+                                    checkDoubleVector);
 }
 static bool CheckFloatingOrIntRepresentation(Sema *S, CallExpr *TheCall) {
   auto checkAllSignedTypes = [](clang::QualType PassedType) -> bool {
     return !PassedType->hasIntegerRepresentation() &&
            !PassedType->hasFloatingRepresentation();
   };
-  return CheckArgsTypesAreCorrect(S, TheCall, S->Context.IntTy,
-                                  checkAllSignedTypes);
+  return CheckAllArgTypesAreCorrect(S, TheCall, S->Context.IntTy,
+                                    checkAllSignedTypes);
 }
 
 static bool CheckUnsignedIntRepresentation(Sema *S, CallExpr *TheCall) {
   auto checkAllUnsignedTypes = [](clang::QualType PassedType) -> bool {
     return !PassedType->hasUnsignedIntegerRepresentation();
   };
-  return CheckArgsTypesAreCorrect(S, TheCall, S->Context.UnsignedIntTy,
-                                  checkAllUnsignedTypes);
+  return CheckAllArgTypesAreCorrect(S, TheCall, S->Context.UnsignedIntTy,
+                                    checkAllUnsignedTypes);
 }
 
 static void SetElementTypeAsReturnType(Sema *S, CallExpr *TheCall,
@@ -2074,6 +2094,22 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
       return true;
     break;
   }
+  case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {
+    if (SemaRef.checkArgCount(TheCall, 3))
+      return true;
+
+    if (CheckScalarOrVector(&SemaRef, TheCall, SemaRef.Context.DoubleTy, 0) ||
+        CheckScalarOrVector(&SemaRef, TheCall, SemaRef.Context.UnsignedIntTy,
+                            1) ||
+        CheckScalarOrVector(&SemaRef, TheCall, SemaRef.Context.UnsignedIntTy,
+                            2))
+      return true;
+
+    if (CheckModifiableLValue(&SemaRef, TheCall, 1) ||
+        CheckModifiableLValue(&SemaRef, TheCall, 2))
+      return true;
+    break;
+  }
   case Builtin::BI__builtin_elementwise_acos:
   case Builtin::BI__builtin_elementwise_asin:
   case Builtin::BI__builtin_elementwise_atan:
diff --git a/clang/test/CodeGenHLSL/builtins/splitdouble.hlsl b/clang/test/CodeGenHLSL/builtins/splitdouble.hlsl
new file mode 100644
index 000000000000..a883c9d5cc35
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/splitdouble.hlsl
@@ -0,0 +1,91 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple dxil-pc-shadermodel6.3-library %s -fnative-half-type -emit-llvm -O1 -o - | FileCheck %s
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple spirv-vulkan-library %s -fnative-half-type -emit-llvm -O0 -o - | FileCheck %s --check-prefix=SPIRV
+
+
+
+// CHECK: define {{.*}} i32 {{.*}}test_scalar{{.*}}(double {{.*}} [[VALD:%.*]])
+// CHECK:      [[VALRET:%.*]] = {{.*}} call { i32, i32 } @llvm.dx.splitdouble.i32(double [[VALD]])
+// CHECK-NEXT: extractvalue { i32, i32 } [[VALRET]], 0
+// CHECK-NEXT: extractvalue { i32, i32 } [[VALRET]], 1
+//
+// SPIRV: define spir_func {{.*}} i32 {{.*}}test_scalar{{.*}}(double {{.*}} [[VALD:%.*]])
+// SPIRV-NOT:  @llvm.dx.splitdouble.i32
+// SPIRV:      [[LOAD:%.*]] = load double, ptr [[VALD]].addr, align 8
+// SPIRV-NEXT: [[CAST:%.*]] = bitcast double [[LOAD]] to <2 x i32>
+// SPIRV-NEXT: extractelement <2 x i32> [[CAST]], i64 0
+// SPIRV-NEXT: extractelement <2 x i32> [[CAST]], i64 1
+uint test_scalar(double D) {
+  uint A, B;
+  asuint(D, A, B);
+  return A + B;
+}
+
+// CHECK: define {{.*}} <1 x i32> {{.*}}test_double1{{.*}}(<1 x double> {{.*}} [[VALD:%.*]])
+// CHECK:      [[TRUNC:%.*]] = extractelement <1 x double> %D, i64 0
+// CHECK-NEXT: [[VALRET:%.*]] = {{.*}} call { i32, i32 } @llvm.dx.splitdouble.i32(double [[TRUNC]])
+// CHECK-NEXT: extractvalue { i32, i32 } [[VALRET]], 0
+// CHECK-NEXT: extractvalue { i32, i32 } [[VALRET]], 1
+//
+// SPIRV: define spir_func {{.*}} <1 x i32> {{.*}}test_double1{{.*}}(<1 x double> {{.*}} [[VALD:%.*]])
+// SPIRV-NOT:  @llvm.dx.splitdouble.i32
+// SPIRV:      [[LOAD:%.*]] = load <1 x double>, ptr [[VALD]].addr, align 8
+// SPIRV-NEXT: [[TRUNC:%.*]] = extractelement <1 x double> [[LOAD]], i64 0
+// SPIRV-NEXT: [[CAST:%.*]] = bitcast double [[TRUNC]] to <2 x i32>
+// SPIRV-NEXT: extractelement <2 x i32> [[CAST]], i64 0
+// SPIRV-NEXT: extractelement <2 x i32> [[CAST]], i64 1
+uint1 test_double1(double1 D) {
+  uint A, B;
+  asuint(D, A, B);
+  return A + B;
+}
+
+// CHECK: define {{.*}} <2 x i32> {{.*}}test_vector2{{.*}}(<2 x double> {{.*}} [[VALD:%.*]])
+// CHECK:      [[VALRET:%.*]] = {{.*}} call { <2 x i32>, <2 x i32> } @llvm.dx.splitdouble.v2i32(<2 x double> [[VALD]])
+// CHECK-NEXT: extractvalue { <2 x i32>, <2 x i32> } [[VALRET]], 0
+// CHECK-NEXT: extractvalue { <2 x i32>, <2 x i32> } [[VALRET]], 1
+//
+// SPIRV: define spir_func {{.*}} <2 x i32> {{.*}}test_vector2{{.*}}(<2 x double> {{.*}} [[VALD:%.*]])
+// SPIRV-NOT:  @llvm.dx.splitdouble.i32
+// SPIRV:      [[LOAD:%.*]] = load <2 x double>, ptr [[VALD]].addr, align 16
+// SPIRV-NEXT: [[CAST1:%.*]] = bitcast <2 x double> [[LOAD]] to <4 x i32>
+// SPIRV-NEXT: [[SHUF1:%.*]] = shufflevector <4 x i32> [[CAST1]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+// SPIRV-NEXT: [[SHUF2:%.*]] = shufflevector <4 x i32> [[CAST1]], <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+uint2 test_vector2(double2 D) {
+  uint2 A, B;
+  asuint(D, A, B);
+  return A + B;
+}
+
+// CHECK: define {{.*}} <3 x i32> {{.*}}test_vector3{{.*}}(<3 x double> {{.*}} [[VALD:%.*]])
+// CHECK:      [[VALRET:%.*]] = {{.*}} call { <3 x i32>, <3 x i32> } @llvm.dx.splitdouble.v3i32(<3 x double> [[VALD]])
+// CHECK-NEXT: extractvalue { <3 x i32>, <3 x i32> } [[VALRET]], 0
+// CHECK-NEXT: extractvalue { <3 x i32>, <3 x i32> } [[VALRET]], 1
+//
+// SPIRV: define spir_func {{.*}} <3 x i32> {{.*}}test_vector3{{.*}}(<3 x double> {{.*}} [[VALD:%.*]])
+// SPIRV-NOT:  @llvm.dx.splitdouble.i32
+// SPIRV:      [[LOAD:%.*]] = load <3 x double>, ptr [[VALD]].addr, align 32
+// SPIRV-NEXT: [[CAST1:%.*]] = bitcast <3 x double> [[LOAD]] to <6 x i32>
+// SPIRV-NEXT: [[SHUF1:%.*]] = shufflevector <6 x i32> [[CAST1]], <6 x i32> poison, <3 x i32> <i32 0, i32 2, i32 4>
+// SPIRV-NEXT: [[SHUF2:%.*]] = shufflevector <6 x i32> [[CAST1]], <6 x i32> poison, <3 x i32> <i32 1, i32 3, i32 5>
+uint3 test_vector3(double3 D) {
+  uint3 A, B;
+  asuint(D, A, B);
+  return A + B;
+}
+
+// CHECK: define {{.*}} <4 x i32> {{.*}}test_vector4{{.*}}(<4 x double> {{.*}} [[VALD:%.*]])
+// CHECK:      [[VALRET:%.*]] = {{.*}} call { <4 x i32>, <4 x i32> } @llvm.dx.splitdouble.v4i32(<4 x double> [[VALD]])
+// CHECK-NEXT: extractvalue { <4 x i32>, <4 x i32> } [[VALRET]], 0
+// CHECK-NEXT: extractvalue { <4 x i32>, <4 x i32> } [[VALRET]], 1
+//
+// SPIRV: define spir_func {{.*}} <4 x i32> {{.*}}test_vector4{{.*}}(<4 x double> {{.*}} [[VALD:%.*]])
+// SPIRV-NOT: @llvm.dx.splitdouble.i32
+// SPIRV:      [[LOAD:%.*]] = load <4 x double>, ptr [[VALD]].addr, align 32
+// SPIRV-NEXT: [[CAST1:%.*]] = bitcast <4 x double> [[LOAD]] to <8 x i32>
+// SPIRV-NEXT: [[SHUF1:%.*]] = shufflevector <8 x i32> [[CAST1]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+// SPIRV-NEXT: [[SHUF2:%.*]] = shufflevector <8 x i32> [[CAST1]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+uint4 test_vector4(double4 D) {
+  uint4 A, B;
+  asuint(D, A, B);
+  return A + B;
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/asuint-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/asuint-errors.hlsl
index 8c56fdddb1c2..4adb0555c35b 100644
--- a/clang/test/SemaHLSL/BuiltIns/asuint-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/asuint-errors.hlsl
@@ -6,6 +6,10 @@ uint4 test_asuint_too_many_arg(float p0, float p1) {
   // expected-error@-1 {{no matching function for call to 'asuint'}}
   // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires single argument 'V', but 2 arguments were provided}}
   // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires single argument 'F', but 2 arguments were provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 3 arguments, but 2 were provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 3 arguments, but 2 were provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 3 arguments, but 2 were provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 3 arguments, but 2 were provided}}
 }
 
 uint test_asuint_double(double p1) {
@@ -23,3 +27,29 @@ uint test_asuint_half(half p1) {
     // expected-note@hlsl/hlsl_detail.h:* {{candidate template ignored: could not match 'vector<half, N>' against 'half'}}
     // expected-note@hlsl/hlsl_detail.h:* {{candidate template ignored: substitution failure [with U = uint, T = half]: no type named 'Type'}}
 }
+
+void test_asuint_first_arg_const(double D) {
+  const uint A = 0;
+  uint B;
+  asuint(D, A, B);
+ // expected-error@hlsl/hlsl_intrinsics.h:* {{read-only variable is not assignable}} 
+}
+
+void test_asuint_second_arg_const(double D) {
+  const uint A = 0;
+  uint B;
+  asuint(D, B, A);
+ // expected-error@hlsl/hlsl_intrinsics.h:* {{read-only variable is not assignable}} 
+}
+
+void test_asuint_imidiate_value(double D) {
+  uint B;
+  asuint(D, B, 1);
+ // expected-error@-1 {{cannot bind non-lvalue argument 1 to out paramemter}} 
+}
+
+void test_asuint_expr(double D) {
+  uint B;
+  asuint(D, B, B + 1);
+ // expected-error@-1 {{cannot bind non-lvalue argument B + 1 to out paramemter}} 
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/splitdouble-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/splitdouble-errors.hlsl
new file mode 100644
index 000000000000..18d2b692b335
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/splitdouble-errors.hlsl
@@ -0,0 +1,76 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -verify
+
+void test_no_second_arg(double D) {
+  __builtin_hlsl_elementwise_splitdouble(D);
+ // expected-error@-1 {{too few arguments to function call, expected 3, have 1}} 
+}
+
+void test_no_third_arg(double D) {
+  uint A;
+  __builtin_hlsl_elementwise_splitdouble(D, A);
+ // expected-error@-1 {{too few arguments to function call, expected 3, have 2}} 
+}
+
+void test_too_many_arg(double D) {
+  uint A, B, C;
+  __builtin_hlsl_elementwise_splitdouble(D, A, B, C);
+ // expected-error@-1 {{too many arguments to function call, expected 3, have 4}} 
+}
+
+void test_first_arg_type_mismatch(bool3 D) {
+  uint3 A, B;
+  __builtin_hlsl_elementwise_splitdouble(D, A, B);
+ // expected-error@-1 {{invalid operand of type 'bool3' (aka 'vector<bool, 3>') where 'double' or a vector of such type is required}} 
+}
+
+void test_second_arg_type_mismatch(double D) {
+  bool A;
+  uint B;
+  __builtin_hlsl_elementwise_splitdouble(D, A, B);
+ // expected-error@-1 {{invalid operand of type 'bool' where 'unsigned int' or a vector of such type is required}} 
+}
+
+void test_third_arg_type_mismatch(double D) {
+  bool A;
+  uint B;
+  __builtin_hlsl_elementwise_splitdouble(D, B, A);
+ // expected-error@-1 {{invalid operand of type 'bool' where 'unsigned int' or a vector of such type is required}} 
+}
+
+void test_const_second_arg(double D) {
+  const uint A = 1;
+  uint B;
+  __builtin_hlsl_elementwise_splitdouble(D, A, B);
+ // expected-error@-1 {{cannot bind non-lvalue argument A to out paramemter}} 
+}
+
+void test_const_third_arg(double D) {
+  uint A;
+  const uint B = 1;
+  __builtin_hlsl_elementwise_splitdouble(D, A, B);
+ // expected-error@-1 {{cannot bind non-lvalue argument B to out paramemter}} 
+}
+
+void test_number_second_arg(double D) {
+  uint B;
+  __builtin_hlsl_elementwise_splitdouble(D, (uint)1, B);
+ // expected-error@-1 {{cannot bind non-lvalue argument (uint)1 to out paramemter}} 
+}
+
+void test_number_third_arg(double D) {
+  uint B;
+  __builtin_hlsl_elementwise_splitdouble(D, B, (uint)1);
+ // expected-error@-1 {{cannot bind non-lvalue argument (uint)1 to out paramemter}} 
+}
+
+void test_expr_second_arg(double D) {
+  uint B;
+  __builtin_hlsl_elementwise_splitdouble(D, B+1, B);
+ // expected-error@-1 {{cannot bind non-lvalue argument B + 1 to out paramemter}} 
+}
+
+void test_expr_third_arg(double D) {
+  uint B;
+  __builtin_hlsl_elementwise_splitdouble(D, B, B+1);
+ // expected-error@-1 {{cannot bind non-lvalue argument B + 1 to out paramemter}} 
+}
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 147b32b1ca99..68ae5de06423 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -47,6 +47,7 @@ def ResRetInt32Ty : DXILOpParamType;
 def HandleTy : DXILOpParamType;
 def ResBindTy : DXILOpParamType;
 def ResPropsTy : DXILOpParamType;
+def SplitDoubleTy : DXILOpParamType;
 
 class DXILOpClass;
 
@@ -779,6 +780,15 @@ def FlattenedThreadIdInGroup :  DXILOp<96, flattenedThreadIdInGroup> {
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
 
+def SplitDouble :  DXILOp<102, splitDouble> {
+  let Doc = "Splits a double into 2 uints";
+  let arguments = [OverloadTy];
+  let result = SplitDoubleTy;
+  let overloads = [Overloads<DXIL1_0, [DoubleTy]>];
+  let stages = [Stages<DXIL1_0, [all_stages]>];
+  let attributes = [Attributes<DXIL1_0, [ReadNone]>];
+}
+
 def AnnotateHandle : DXILOp<217, annotateHandle> {
   let Doc = "annotate handle with resource properties";
   let arguments = [HandleTy, ResPropsTy];
diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
index 7719d6b10791..5d5bb3eacace 100644
--- a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
@@ -229,6 +229,13 @@ static StructType *getResPropsType(LLVMContext &Context) {
   return StructType::create({Int32Ty, Int32Ty}, "dx.types.ResourceProperties");
 }
 
+static StructType *getSplitDoubleType(LLVMContext &Context) {
+  if (auto *ST = StructType::getTypeByName(Context, "dx.types.splitdouble"))
+    return ST;
+  Type *Int32Ty = Type::getInt32Ty(Context);
+  return StructType::create({Int32Ty, Int32Ty}, "dx.types.splitdouble");
+}
+
 static Type *getTypeFromOpParamType(OpParamType Kind, LLVMContext &Ctx,
                                     Type *OverloadTy) {
   switch (Kind) {
@@ -266,6 +273,8 @@ static Type *getTypeFromOpParamType(OpParamType Kind, LLVMContext &Ctx,
     return getResBindType(Ctx);
   case OpParamType::ResPropsTy:
     return getResPropsType(Ctx);
+  case OpParamType::SplitDoubleTy:
+    return getSplitDoubleType(Ctx);
   }
   llvm_unreachable("Invalid parameter kind");
   return nullptr;
@@ -467,6 +476,10 @@ StructType *DXILOpBuilder::getResRetType(Type *ElementTy) {
   return ::getResRetType(ElementTy);
 }
 
+StructType *DXILOpBuilder::getSplitDoubleType(LLVMContext &Context) {
+  return ::getSplitDoubleType(Context);
+}
+
 StructType *DXILOpBuilder::getHandleType() {
   return ::getHandleType(IRB.getContext());
 }
diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.h b/llvm/lib/Target/DirectX/DXILOpBuilder.h
index 037ae3822cfb..df5a0240870f 100644
--- a/llvm/lib/Target/DirectX/DXILOpBuilder.h
+++ b/llvm/lib/Target/DirectX/DXILOpBuilder.h
@@ -49,6 +49,10 @@ public:
 
   /// Get a `%dx.types.ResRet` type with the given element type.
   StructType *getResRetType(Type *ElementTy);
+
+  /// Get the `%dx.types.splitdouble` type.
+  StructType *getSplitDoubleType(LLVMContext &Context);
+
   /// Get the `%dx.types.Handle` type.
   StructType *getHandleType();
 
diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index c62ba8c21d67..f7722d770747 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -17,6 +17,7 @@
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsDirectX.h"
 #include "llvm/IR/Module.h"
@@ -128,6 +129,30 @@ public:
     });
   }
 
+  [[nodiscard]] bool replaceFunctionWithNamedStructOp(
+      Function &F, dxil::OpCode DXILOp, Type *NewRetTy,
+      llvm::function_ref<Error(CallInst *CI, CallInst *Op)> ReplaceUses) {
+    bool IsVectorArgExpansion = isVectorArgExpansion(F);
+    return replaceFunction(F, [&](CallInst *CI) -> Error {
+      SmallVector<Value *> Args;
+      OpBuilder.getIRB().SetInsertPoint(CI);
+      if (IsVectorArgExpansion) {
+        SmallVector<Value *> NewArgs = argVectorFlatten(CI, OpBuilder.getIRB());
+        Args.append(NewArgs.begin(), NewArgs.end());
+      } else
+        Args.append(CI->arg_begin(), CI->arg_end());
+
+      Expected<CallInst *> OpCall =
+          OpBuilder.tryCreateOp(DXILOp, Args, CI->getName(), NewRetTy);
+      if (Error E = OpCall.takeError())
+        return E;
+      if (Error E = ReplaceUses(CI, *OpCall))
+        return E;
+
+      return Error::success();
+    });
+  }
+
   /// Create a cast between a `target("dx")` type and `dx.types.Handle`, which
   /// is intended to be removed by the end of lowering. This is used to allow
   /// lowering of ops which need to change their return or argument types in a
@@ -263,6 +288,26 @@ public:
     return lowerToBindAndAnnotateHandle(F);
   }
 
+  Error replaceSplitDoubleCallUsages(CallInst *Intrin, CallInst *Op) {
+    for (Use &U : make_early_inc_range(Intrin->uses())) {
+      if (auto *EVI = dyn_cast<ExtractValueInst>(U.getUser())) {
+
+        if (EVI->getNumIndices() != 1)
+          return createStringError(std::errc::invalid_argument,
+                                   "Splitdouble has only 2 elements");
+        EVI->setOperand(0, Op);
+      } else {
+        return make_error<StringError>(
+            "Splitdouble use is not ExtractValueInst",
+            inconvertibleErrorCode());
+      }
+    }
+
+    Intrin->eraseFromParent();
+
+    return Error::success();
+  }
+
   /// Replace uses of \c Intrin with the values in the `dx.ResRet` of \c Op.
   /// Since we expect to be post-scalarization, make an effort to avoid vectors.
   Error replaceResRetUses(CallInst *Intrin, CallInst *Op, bool HasCheckBit) {
@@ -488,6 +533,16 @@ public:
       case Intrinsic::dx_typedBufferStore:
         HasErrors |= lowerTypedBufferStore(F);
         break;
+      // TODO: this can be removed when
+      // https://github.com/llvm/llvm-project/issues/113192 is fixed
+      case Intrinsic::dx_splitdouble:
+        HasErrors |= replaceFunctionWithNamedStructOp(
+            F, OpCode::SplitDouble,
+            OpBuilder.getSplitDoubleType(M.getContext()),
+            [&](CallInst *CI, CallInst *Op) {
+              return replaceSplitDoubleCallUsages(CI, Op);
+            });
+        break;
       }
       Updated = true;
     }
diff --git a/llvm/test/CodeGen/DirectX/split-double.ll b/llvm/test/CodeGen/DirectX/split-double.ll
deleted file mode 100644
index 759590fa5627..000000000000
--- a/llvm/test/CodeGen/DirectX/split-double.ll
+++ /dev/null
@@ -1,45 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes='function(scalarizer)' -S -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
-
-define void @test_vector_double_split_void(<2 x double> noundef %d) {
-; CHECK-LABEL: define void @test_vector_double_split_void(
-; CHECK-SAME: <2 x double> noundef [[D:%.*]]) {
-; CHECK-NEXT:    [[D_I0:%.*]] = extractelement <2 x double> [[D]], i64 0
-; CHECK-NEXT:    [[HLSL_ASUINT_I0:%.*]] = call { i32, i32 } @llvm.dx.splitdouble.i32(double [[D_I0]])
-; CHECK-NEXT:    [[D_I1:%.*]] = extractelement <2 x double> [[D]], i64 1
-; CHECK-NEXT:    [[HLSL_ASUINT_I1:%.*]] = call { i32, i32 } @llvm.dx.splitdouble.i32(double [[D_I1]])
-; CHECK-NEXT:    ret void
-;
-  %hlsl.asuint = call { <2 x i32>, <2 x i32> }  @llvm.dx.splitdouble.v2i32(<2 x double> %d)
-  ret void
-}
-
-define noundef <3 x i32> @test_vector_double_split(<3 x double> noundef %d) {
-; CHECK-LABEL: define noundef <3 x i32> @test_vector_double_split(
-; CHECK-SAME: <3 x double> noundef [[D:%.*]]) {
-; CHECK-NEXT:    [[D_I0:%.*]] = extractelement <3 x double> [[D]], i64 0
-; CHECK-NEXT:    [[HLSL_ASUINT_I0:%.*]] = call { i32, i32 } @llvm.dx.splitdouble.i32(double [[D_I0]])
-; CHECK-NEXT:    [[D_I1:%.*]] = extractelement <3 x double> [[D]], i64 1
-; CHECK-NEXT:    [[HLSL_ASUINT_I1:%.*]] = call { i32, i32 } @llvm.dx.splitdouble.i32(double [[D_I1]])
-; CHECK-NEXT:    [[D_I2:%.*]] = extractelement <3 x double> [[D]], i64 2
-; CHECK-NEXT:    [[HLSL_ASUINT_I2:%.*]] = call { i32, i32 } @llvm.dx.splitdouble.i32(double [[D_I2]])
-; CHECK-NEXT:    [[DOTELEM0:%.*]] = extractvalue { i32, i32 } [[HLSL_ASUINT_I0]], 0
-; CHECK-NEXT:    [[DOTELEM01:%.*]] = extractvalue { i32, i32 } [[HLSL_ASUINT_I1]], 0
-; CHECK-NEXT:    [[DOTELEM02:%.*]] = extractvalue { i32, i32 } [[HLSL_ASUINT_I2]], 0
-; CHECK-NEXT:    [[DOTELEM1:%.*]] = extractvalue { i32, i32 } [[HLSL_ASUINT_I0]], 1
-; CHECK-NEXT:    [[DOTELEM13:%.*]] = extractvalue { i32, i32 } [[HLSL_ASUINT_I1]], 1
-; CHECK-NEXT:    [[DOTELEM14:%.*]] = extractvalue { i32, i32 } [[HLSL_ASUINT_I2]], 1
-; CHECK-NEXT:    [[DOTI0:%.*]] = add i32 [[DOTELEM0]], [[DOTELEM1]]
-; CHECK-NEXT:    [[DOTI1:%.*]] = add i32 [[DOTELEM01]], [[DOTELEM13]]
-; CHECK-NEXT:    [[DOTI2:%.*]] = add i32 [[DOTELEM02]], [[DOTELEM14]]
-; CHECK-NEXT:    [[DOTUPTO015:%.*]] = insertelement <3 x i32> poison, i32 [[DOTI0]], i64 0
-; CHECK-NEXT:    [[DOTUPTO116:%.*]] = insertelement <3 x i32> [[DOTUPTO015]], i32 [[DOTI1]], i64 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <3 x i32> [[DOTUPTO116]], i32 [[DOTI2]], i64 2
-; CHECK-NEXT:    ret <3 x i32> [[TMP1]]
-;
-  %hlsl.asuint = call { <3 x i32>, <3 x i32> }  @llvm.dx.splitdouble.v3i32(<3 x double> %d)
-  %1 = extractvalue { <3 x i32>, <3 x i32> } %hlsl.asuint, 0
-  %2 = extractvalue { <3 x i32>, <3 x i32> } %hlsl.asuint, 1
-  %3 = add <3 x i32> %1, %2
-  ret <3 x i32> %3
-}
diff --git a/llvm/test/CodeGen/DirectX/splitdouble.ll b/llvm/test/CodeGen/DirectX/splitdouble.ll
new file mode 100644
index 000000000000..1443ba626925
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/splitdouble.ll
@@ -0,0 +1,76 @@
+; RUN: opt -passes='function(scalarizer)' -S -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,NOLOWER
+; RUN: opt -passes='function(scalarizer),module(dxil-op-lower)' -S -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,WITHLOWER
+
+define i32 @test_scalar(double noundef %D) {
+; CHECK-LABEL: define i32 @test_scalar(
+; CHECK-SAME: double noundef [[D:%.*]]) {
+; NOLOWER-NEXT:    [[HLSL_ASUINT_I0:%.*]] = call { i32, i32 } @llvm.dx.splitdouble.i32(double [[D]])
+; WITHLOWER-NEXT:  [[HLSL_ASUINT_I0:%.*]] = call %dx.types.splitdouble @dx.op.splitDouble.f64(i32 102, double [[D]])
+; NOLOWER-NEXT:    [[EV1:%.*]] = extractvalue { i32, i32 } [[HLSL_ASUINT_I0]], 0
+; NOLOWER-NEXT:    [[EV2:%.*]] = extractvalue { i32, i32 } [[HLSL_ASUINT_I0]], 1
+; WITHLOWER-NEXT:  [[EV1:%.*]] = extractvalue %dx.types.splitdouble [[HLSL_ASUINT_I0]], 0
+; WITHLOWER-NEXT:  [[EV2:%.*]] = extractvalue %dx.types.splitdouble [[HLSL_ASUINT_I0]], 1
+; CHECK-NEXT:      [[ADD:%.*]] = add i32 [[EV1]], [[EV2]]
+; CHECK-NEXT:      ret i32 [[ADD]]
+;
+  %hlsl.splitdouble = call { i32, i32 } @llvm.dx.splitdouble.i32(double %D)
+  %1 = extractvalue { i32, i32 } %hlsl.splitdouble, 0
+  %2 = extractvalue { i32, i32 } %hlsl.splitdouble, 1
+  %add = add i32 %1, %2
+  ret i32 %add
+}
+
+
+define void @test_vector_double_split_void(<2 x double> noundef %d) {
+; CHECK-LABEL: define void @test_vector_double_split_void(
+; CHECK-SAME: <2 x double> noundef [[D:%.*]]) {
+; CHECK-NEXT:      [[D_I0:%.*]] = extractelement <2 x double> [[D]], i64 0
+; NOLOWER-NEXT:    [[HLSL_ASUINT_I0:%.*]] = call { i32, i32 } @llvm.dx.splitdouble.i32(double [[D_I0]])
+; WITHLOWER-NEXT:  [[HLSL_ASUINT_I0:%.*]] = call %dx.types.splitdouble @dx.op.splitDouble.f64(i32 102, double [[D_I0]])
+; CHECK-NEXT:      [[D_I1:%.*]] = extractelement <2 x double> [[D]], i64 1
+; NOLOWER-NEXT:    [[HLSL_ASUINT_I1:%.*]] = call { i32, i32 } @llvm.dx.splitdouble.i32(double [[D_I1]])
+; WITHLOWER-NEXT:  [[HLSL_ASUINT_I1:%.*]] = call %dx.types.splitdouble @dx.op.splitDouble.f64(i32 102, double [[D_I1]])
+; CHECK-NEXT:      ret void
+;
+  %hlsl.asuint = call { <2 x i32>, <2 x i32> }  @llvm.dx.splitdouble.v2i32(<2 x double> %d)
+  ret void
+}
+
+define noundef <3 x i32> @test_vector_double_split(<3 x double> noundef %d) {
+; CHECK-LABEL: define noundef <3 x i32> @test_vector_double_split(
+; CHECK-SAME: <3 x double> noundef [[D:%.*]]) {
+; CHECK-NEXT:      [[D_I0:%.*]] = extractelement <3 x double> [[D]], i64 0
+; NOLOWER-NEXT:    [[HLSL_ASUINT_I0:%.*]] = call { i32, i32 } @llvm.dx.splitdouble.i32(double [[D_I0]])
+; WITHLOWER-NEXT:  [[HLSL_ASUINT_I0:%.*]] = call %dx.types.splitdouble @dx.op.splitDouble.f64(i32 102, double [[D_I0]])
+; CHECK-NEXT:      [[D_I1:%.*]] = extractelement <3 x double> [[D]], i64 1
+; NOLOWER-NEXT:    [[HLSL_ASUINT_I1:%.*]] = call { i32, i32 } @llvm.dx.splitdouble.i32(double [[D_I1]])
+; WITHLOWER-NEXT:  [[HLSL_ASUINT_I1:%.*]] = call %dx.types.splitdouble @dx.op.splitDouble.f64(i32 102, double [[D_I1]])
+; CHECK-NEXT:      [[D_I2:%.*]] = extractelement <3 x double> [[D]], i64 2
+; NOLOWER-NEXT:    [[HLSL_ASUINT_I2:%.*]] = call { i32, i32 } @llvm.dx.splitdouble.i32(double [[D_I2]])
+; WITHLOWER-NEXT:  [[HLSL_ASUINT_I2:%.*]] = call %dx.types.splitdouble @dx.op.splitDouble.f64(i32 102, double [[D_I2]])
+; NOLOWER-NEXT:    [[DOTELEM0:%.*]] = extractvalue { i32, i32 } [[HLSL_ASUINT_I0]], 0
+; WITHLOWER-NEXT:  [[DOTELEM0:%.*]] = extractvalue %dx.types.splitdouble [[HLSL_ASUINT_I0]], 0
+; NOLOWER-NEXT:    [[DOTELEM01:%.*]] = extractvalue { i32, i32 } [[HLSL_ASUINT_I1]], 0
+; WITHLOWER-NEXT:  [[DOTELEM01:%.*]] = extractvalue %dx.types.splitdouble [[HLSL_ASUINT_I1]], 0
+; NOLOWER-NEXT:    [[DOTELEM02:%.*]] = extractvalue { i32, i32 } [[HLSL_ASUINT_I2]], 0
+; WITHLOWER-NEXT:  [[DOTELEM02:%.*]] = extractvalue %dx.types.splitdouble [[HLSL_ASUINT_I2]], 0
+; NOLOWER-NEXT:    [[DOTELEM1:%.*]] = extractvalue { i32, i32 } [[HLSL_ASUINT_I0]], 1
+; WITHLOWER-NEXT:  [[DOTELEM1:%.*]] = extractvalue %dx.types.splitdouble [[HLSL_ASUINT_I0]], 1
+; NOLOWER-NEXT:    [[DOTELEM13:%.*]] = extractvalue { i32, i32 } [[HLSL_ASUINT_I1]], 1
+; WITHLOWER-NEXT:  [[DOTELEM13:%.*]] = extractvalue %dx.types.splitdouble [[HLSL_ASUINT_I1]], 1
+; NOLOWER-NEXT:    [[DOTELEM14:%.*]] = extractvalue { i32, i32 } [[HLSL_ASUINT_I2]], 1
+; WITHLOWER-NEXT:  [[DOTELEM14:%.*]] = extractvalue %dx.types.splitdouble [[HLSL_ASUINT_I2]], 1
+; CHECK-NEXT:      [[DOTI0:%.*]] = add i32 [[DOTELEM0]], [[DOTELEM1]]
+; CHECK-NEXT:      [[DOTI1:%.*]] = add i32 [[DOTELEM01]], [[DOTELEM13]]
+; CHECK-NEXT:      [[DOTI2:%.*]] = add i32 [[DOTELEM02]], [[DOTELEM14]]
+; CHECK-NEXT:      [[DOTUPTO015:%.*]] = insertelement <3 x i32> poison, i32 [[DOTI0]], i64 0
+; CHECK-NEXT:      [[DOTUPTO116:%.*]] = insertelement <3 x i32> [[DOTUPTO015]], i32 [[DOTI1]], i64 1
+; CHECK-NEXT:      [[TMP1:%.*]] = insertelement <3 x i32> [[DOTUPTO116]], i32 [[DOTI2]], i64 2
+; CHECK-NEXT:      ret <3 x i32> [[TMP1]]
+;
+  %hlsl.asuint = call { <3 x i32>, <3 x i32> }  @llvm.dx.splitdouble.v3i32(<3 x double> %d)
+  %1 = extractvalue { <3 x i32>, <3 x i32> } %hlsl.asuint, 0
+  %2 = extractvalue { <3 x i32>, <3 x i32> } %hlsl.asuint, 1
+  %3 = add <3 x i32> %1, %2
+  ret <3 x i32> %3
+}
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/splitdouble.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/splitdouble.ll
new file mode 100644
index 000000000000..d18b16b843c3
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/splitdouble.ll
@@ -0,0 +1,40 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; Make sure lowering is correctly generating spirv code.
+
+; CHECK-DAG: %[[#double:]] = OpTypeFloat 64
+; CHECK-DAG: %[[#vec_2_double:]] = OpTypeVector %[[#double]] 2
+; CHECK-DAG: %[[#int_32:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#vec_2_int_32:]] = OpTypeVector %[[#int_32]] 2
+; CHECK-DAG: %[[#vec_4_int_32:]] = OpTypeVector %[[#int_32]] 4
+
+
+define spir_func noundef i32 @test_scalar(double noundef %D) local_unnamed_addr {
+entry:
+  ; CHECK-LABEL: ; -- Begin function test_scalar
+  ; CHECK: %[[#param:]] = OpFunctionParameter %[[#double]]
+  ; CHECK: %[[#bitcast:]] = OpBitcast %[[#vec_2_int_32]] %[[#param]]
+  %0 = bitcast double %D to <2 x i32>
+  ; CHECK: %[[#]] = OpCompositeExtract %[[#int_32]] %[[#bitcast]] 0
+  %1 = extractelement <2 x i32> %0, i64 0
+  ; CHECK: %[[#]] = OpCompositeExtract %[[#int_32]] %[[#bitcast]] 1
+  %2 = extractelement <2 x i32> %0, i64 1
+  %add = add i32 %1, %2
+  ret i32 %add
+}
+
+
+define spir_func noundef <2 x i32> @test_vector(<2 x double> noundef %D) local_unnamed_addr {
+entry:
+  ; CHECK-LABEL: ; -- Begin function test_vector
+  ; CHECK: %[[#param:]] = OpFunctionParameter %[[#vec_2_double]]
+  ; CHECK: %[[#CAST1:]] = OpBitcast %[[#vec_4_int_32]] %[[#param]]
+  ; CHECK: %[[#SHUFF2:]] = OpVectorShuffle %[[#vec_2_int_32]] %[[#CAST1]] %[[#]] 0 2
+  ; CHECK: %[[#SHUFF3:]] = OpVectorShuffle %[[#vec_2_int_32]] %[[#CAST1]] %[[#]] 1 3
+  %0 = bitcast <2 x double> %D to <4 x i32>
+  %1 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+  %2 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+  %add = add <2 x i32> %1, %2
+  ret <2 x i32> %add
+}
-- 
GitLab


From abc49cc19463970d5523d7d3332e4c1f83bc2ef7 Mon Sep 17 00:00:00 2001
From: Job Henandez Lara <jobhdezlara93@gmail.com>
Date: Mon, 28 Oct 2024 13:29:16 -0700
Subject: [PATCH 362/693] [libc] remove #include <fcntl.h> and add proxy or
 type (#113836)

---
 libc/hdr/CMakeLists.txt                       |  3 ++
 libc/hdr/fcntl_macros.h                       |  2 +-
 libc/hdr/fcntl_overlay.h                      | 37 +++++++++++++++++++
 libc/hdr/types/CMakeLists.txt                 | 11 ++++++
 libc/hdr/types/mode_t.h                       | 22 +++++++++++
 libc/src/__support/File/linux/CMakeLists.txt  |  4 +-
 libc/src/__support/File/linux/dir.cpp         |  2 +-
 libc/src/__support/File/linux/file.cpp        |  2 +-
 .../__support/threads/linux/CMakeLists.txt    |  2 +-
 libc/src/__support/threads/linux/thread.cpp   |  2 +-
 libc/src/fcntl/creat.h                        |  2 +-
 libc/src/fcntl/linux/CMakeLists.txt           |  8 ++--
 libc/src/fcntl/linux/creat.cpp                |  2 +-
 libc/src/fcntl/linux/open.cpp                 |  2 +-
 libc/src/fcntl/linux/openat.cpp               |  2 +-
 libc/src/fcntl/open.h                         |  2 +-
 libc/src/fcntl/openat.h                       |  2 +-
 libc/src/spawn/linux/CMakeLists.txt           |  2 +-
 libc/src/spawn/linux/posix_spawn.cpp          |  2 +-
 libc/src/stdio/linux/CMakeLists.txt           |  2 +-
 libc/src/stdio/linux/remove.cpp               |  2 +-
 libc/src/sys/stat/linux/CMakeLists.txt        | 11 +++---
 libc/src/sys/stat/linux/chmod.cpp             |  2 +-
 libc/src/sys/stat/linux/fchmod.cpp            |  2 +-
 libc/src/sys/stat/linux/fstat.cpp             |  2 +-
 libc/src/sys/stat/linux/lstat.cpp             |  2 +-
 libc/src/sys/stat/linux/mkdir.cpp             |  2 +-
 libc/src/sys/stat/linux/stat.cpp              |  2 +-
 libc/src/unistd/linux/CMakeLists.txt          | 21 ++++++-----
 libc/src/unistd/linux/access.cpp              |  2 +-
 libc/src/unistd/linux/dup2.cpp                |  2 +-
 libc/src/unistd/linux/link.cpp                |  2 +-
 libc/src/unistd/linux/linkat.cpp              |  2 +-
 libc/src/unistd/linux/readlink.cpp            |  2 +-
 libc/src/unistd/linux/readlinkat.cpp          |  2 +-
 libc/src/unistd/linux/rmdir.cpp               |  2 +-
 libc/src/unistd/linux/symlink.cpp             |  2 +-
 libc/src/unistd/linux/symlinkat.cpp           |  2 +-
 libc/src/unistd/linux/unlink.cpp              |  2 +-
 libc/src/unistd/linux/unlinkat.cpp            |  2 +-
 libc/test/src/fcntl/CMakeLists.txt            |  2 +-
 libc/test/src/fcntl/openat_test.cpp           |  2 +-
 libc/test/src/sys/sendfile/CMakeLists.txt     |  2 +-
 libc/test/src/sys/sendfile/sendfile_test.cpp  |  2 +-
 libc/test/src/sys/stat/CMakeLists.txt         | 14 +++----
 libc/test/src/sys/stat/chmod_test.cpp         |  2 +-
 libc/test/src/sys/stat/fchmod_test.cpp        |  2 +-
 libc/test/src/sys/stat/fchmodat_test.cpp      |  2 +-
 libc/test/src/sys/stat/fstat_test.cpp         |  2 +-
 libc/test/src/sys/stat/lstat_test.cpp         |  2 +-
 libc/test/src/sys/stat/mkdirat_test.cpp       |  2 +-
 libc/test/src/sys/stat/stat_test.cpp          |  2 +-
 libc/test/src/unistd/CMakeLists.txt           |  9 +++--
 libc/test/src/unistd/chdir_test.cpp           |  2 +-
 libc/test/src/unistd/fchdir_test.cpp          |  2 +-
 libc/test/src/unistd/readlinkat_test.cpp      |  2 +-
 libc/test/src/unistd/rmdir_test.cpp           |  2 +-
 libc/test/src/unistd/syscall_test.cpp         |  2 +-
 58 files changed, 156 insertions(+), 80 deletions(-)
 create mode 100644 libc/hdr/fcntl_overlay.h
 create mode 100644 libc/hdr/types/mode_t.h

diff --git a/libc/hdr/CMakeLists.txt b/libc/hdr/CMakeLists.txt
index 13dc892978bb..80545ee4b359 100644
--- a/libc/hdr/CMakeLists.txt
+++ b/libc/hdr/CMakeLists.txt
@@ -51,10 +51,13 @@ add_proxy_header_library(
     libc.include.llvm-libc-macros.generic_error_number_macros
 )
 
+add_header_library(fcntl_overlay HDRS fcntl_overlay.h)
 add_proxy_header_library(
   fcntl_macros
   HDRS
     fcntl_macros.h
+  DEPENDS
+    .fcntl_overlay
   FULL_BUILD_DEPENDS
     libc.include.llvm-libc-macros.fcntl_macros
     libc.include.fcntl
diff --git a/libc/hdr/fcntl_macros.h b/libc/hdr/fcntl_macros.h
index 828cb984c0cb..3a1ddeb0a2da 100644
--- a/libc/hdr/fcntl_macros.h
+++ b/libc/hdr/fcntl_macros.h
@@ -15,7 +15,7 @@
 
 #else // Overlay mode
 
-#include <fcntl.h>
+#include "hdr/fcntl_overlay.h"
 
 #endif // LLVM_LIBC_FULL_BUILD
 
diff --git a/libc/hdr/fcntl_overlay.h b/libc/hdr/fcntl_overlay.h
new file mode 100644
index 000000000000..c1cc98b0ebb2
--- /dev/null
+++ b/libc/hdr/fcntl_overlay.h
@@ -0,0 +1,37 @@
+//===-- Including fcntl.h in overlay mode ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_FCNTL_OVERLAY_H
+#define LLVM_LIBC_HDR_FCNTL_OVERLAY_H
+
+#ifdef LIBC_FULL_BUILD
+#error "This header should only be included in overlay mode"
+#endif
+
+// Overlay mode
+
+// glibc <fcntl.h> header might provide extern inline definitions for few
+// functions, causing external alias errors.  They are guarded by
+// `__USE_FORTIFY_LEVEL`, which will be temporarily disabled
+// with `_FORTIFY_SOURCE`.
+
+#ifdef __USE_FORTIFY_LEVEL
+#define LIBC_OLD_USE_FORTIFY_LEVEL __USE_FORTIFY_LEVEL
+#undef __USE_FORTIFY_LEVEL
+#define __USE_FORTIFY_LEVEL 0
+#endif
+
+#include <fcntl.h>
+
+#ifdef LIBC_OLD_USE_FORTIFY_LEVEL
+#undef __USE_FORTIFY_LEVEL
+#define __USE_FORTIFY_LEVEL LIBC_OLD_USE_FORTIFY_LEVEL
+#undef LIBC_OLD_USE_FORTIFY_LEVEL
+#endif
+
+#endif // LLVM_LIBC_HDR_FCNTL_OVERLAY_H
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index fab5245816bb..e45979857d79 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -46,6 +46,17 @@ add_proxy_header_library(
     libc.include.llvm-libc-types.struct_timespec
 )
 
+add_proxy_header_library(
+  mode_t
+  HDRS
+    mode_t.h
+  DEPENDS
+    ../fcntl_overlay
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.mode_t
+    libc.include.fcntl
+)
+
 add_proxy_header_library(
   fenv_t
   HDRS
diff --git a/libc/hdr/types/mode_t.h b/libc/hdr/types/mode_t.h
new file mode 100644
index 000000000000..abbbdb0a09d7
--- /dev/null
+++ b/libc/hdr/types/mode_t.h
@@ -0,0 +1,22 @@
+//===-- Definition of macros from mode_t.h --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_MODE_T_H
+#define LLVM_LIBC_HDR_MODE_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/mode_t.h"
+
+#else // Overlay mode
+
+#include "hdr/fcntl_overlay.h"
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_MODE_T_H
diff --git a/libc/src/__support/File/linux/CMakeLists.txt b/libc/src/__support/File/linux/CMakeLists.txt
index 5abbf11b3671..84e3d5608361 100644
--- a/libc/src/__support/File/linux/CMakeLists.txt
+++ b/libc/src/__support/File/linux/CMakeLists.txt
@@ -7,7 +7,7 @@ add_object_library(
     file.h
     lseekImpl.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.sys_syscall
     libc.include.sys_stat
     libc.src.__support.CPP.new
@@ -55,7 +55,7 @@ add_object_library(
   SRCS
     dir.cpp
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
     libc.src.__support.error_or
diff --git a/libc/src/__support/File/linux/dir.cpp b/libc/src/__support/File/linux/dir.cpp
index fc90ff097e46..5fe44fa8297b 100644
--- a/libc/src/__support/File/linux/dir.cpp
+++ b/libc/src/__support/File/linux/dir.cpp
@@ -12,7 +12,7 @@
 #include "src/__support/error_or.h"
 #include "src/__support/macros/config.h"
 
-#include <fcntl.h>       // For open flags
+#include "hdr/fcntl_macros.h" // For open flags
 #include <sys/syscall.h> // For syscall numbers
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/__support/File/linux/file.cpp b/libc/src/__support/File/linux/file.cpp
index 22292336f300..824c1f200e8c 100644
--- a/libc/src/__support/File/linux/file.cpp
+++ b/libc/src/__support/File/linux/file.cpp
@@ -18,7 +18,7 @@
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h" // For error macros
 
-#include <fcntl.h>       // For mode_t and other flags to the open syscall
+#include "hdr/fcntl_macros.h" // For mode_t and other flags to the open syscall
 #include <sys/stat.h>    // For S_IS*, S_IF*, and S_IR* flags.
 #include <sys/syscall.h> // For syscall numbers
 
diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt
index b6796f40adce..fa11458f99b6 100644
--- a/libc/src/__support/threads/linux/CMakeLists.txt
+++ b/libc/src/__support/threads/linux/CMakeLists.txt
@@ -79,7 +79,7 @@ add_object_library(
     .futex_utils
     libc.config.app_h
     libc.include.sys_syscall
-    libc.include.fcntl
+    libc.hdr.fcntl_macros  
     libc.src.errno.errno
     libc.src.__support.CPP.atomic
     libc.src.__support.CPP.stringstream
diff --git a/libc/src/__support/threads/linux/thread.cpp b/libc/src/__support/threads/linux/thread.cpp
index ee3f63fa3cde..c531d74c5335 100644
--- a/libc/src/__support/threads/linux/thread.cpp
+++ b/libc/src/__support/threads/linux/thread.cpp
@@ -22,7 +22,7 @@
 #include <arm_acle.h>
 #endif
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <linux/param.h> // For EXEC_PAGESIZE.
 #include <linux/prctl.h> // For PR_SET_NAME
 #include <linux/sched.h> // For CLONE_* flags.
diff --git a/libc/src/fcntl/creat.h b/libc/src/fcntl/creat.h
index e180e17c2578..3e00427638a3 100644
--- a/libc/src/fcntl/creat.h
+++ b/libc/src/fcntl/creat.h
@@ -9,8 +9,8 @@
 #ifndef LLVM_LIBC_SRC_FCNTL_CREAT_H
 #define LLVM_LIBC_SRC_FCNTL_CREAT_H
 
+#include "hdr/fcntl_macros.h"
 #include "src/__support/macros/config.h"
-#include <fcntl.h>
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/fcntl/linux/CMakeLists.txt b/libc/src/fcntl/linux/CMakeLists.txt
index ee8ae63b8cf0..ecfb2cdd3f33 100644
--- a/libc/src/fcntl/linux/CMakeLists.txt
+++ b/libc/src/fcntl/linux/CMakeLists.txt
@@ -5,7 +5,7 @@ add_entrypoint_object(
   HDRS
     ../creat.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.src.__support.OSUtil.osutil
     libc.src.errno.errno
 )
@@ -17,7 +17,7 @@ add_entrypoint_object(
   HDRS
     ../fcntl.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.src.__support.OSUtil.osutil
 )
 
@@ -28,7 +28,7 @@ add_entrypoint_object(
   HDRS
     ../open.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.types.mode_t
     libc.src.__support.OSUtil.osutil
     libc.src.errno.errno
 )
@@ -40,7 +40,7 @@ add_entrypoint_object(
   HDRS
     ../openat.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.types.mode_t
     libc.src.__support.OSUtil.osutil
     libc.src.errno.errno
 )
diff --git a/libc/src/fcntl/linux/creat.cpp b/libc/src/fcntl/linux/creat.cpp
index 2c5b5d736a3b..23abae243aed 100644
--- a/libc/src/fcntl/linux/creat.cpp
+++ b/libc/src/fcntl/linux/creat.cpp
@@ -13,7 +13,7 @@
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/fcntl/linux/open.cpp b/libc/src/fcntl/linux/open.cpp
index 79b7b2b32c88..9f3d035388e8 100644
--- a/libc/src/fcntl/linux/open.cpp
+++ b/libc/src/fcntl/linux/open.cpp
@@ -13,7 +13,7 @@
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
 
-#include <fcntl.h>
+#include "hdr/types/mode_t.h"
 #include <stdarg.h>
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/libc/src/fcntl/linux/openat.cpp b/libc/src/fcntl/linux/openat.cpp
index 0862082c22eb..6063d9c00ad6 100644
--- a/libc/src/fcntl/linux/openat.cpp
+++ b/libc/src/fcntl/linux/openat.cpp
@@ -13,7 +13,7 @@
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
 
-#include <fcntl.h>
+#include "hdr/types/mode_t.h"
 #include <stdarg.h>
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/libc/src/fcntl/open.h b/libc/src/fcntl/open.h
index 19bb53c2e320..11f0ae537953 100644
--- a/libc/src/fcntl/open.h
+++ b/libc/src/fcntl/open.h
@@ -9,8 +9,8 @@
 #ifndef LLVM_LIBC_SRC_FCNTL_OPEN_H
 #define LLVM_LIBC_SRC_FCNTL_OPEN_H
 
+#include "hdr/fcntl_macros.h"
 #include "src/__support/macros/config.h"
-#include <fcntl.h>
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/fcntl/openat.h b/libc/src/fcntl/openat.h
index d09791a84f73..051c8a2304dc 100644
--- a/libc/src/fcntl/openat.h
+++ b/libc/src/fcntl/openat.h
@@ -9,8 +9,8 @@
 #ifndef LLVM_LIBC_SRC_FCNTL_OPENAT_H
 #define LLVM_LIBC_SRC_FCNTL_OPENAT_H
 
+#include "hdr/fcntl_macros.h"
 #include "src/__support/macros/config.h"
-#include <fcntl.h>
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/spawn/linux/CMakeLists.txt b/libc/src/spawn/linux/CMakeLists.txt
index 9ef3a9d18b0c..5f7ab4c43165 100644
--- a/libc/src/spawn/linux/CMakeLists.txt
+++ b/libc/src/spawn/linux/CMakeLists.txt
@@ -5,7 +5,7 @@ add_entrypoint_object(
   HDRS
     ../posix_spawn.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.types.mode_t
     libc.include.spawn
     libc.include.sys_syscall
     libc.include.signal
diff --git a/libc/src/spawn/linux/posix_spawn.cpp b/libc/src/spawn/linux/posix_spawn.cpp
index 4c0469b3ce38..d6caf8b374a0 100644
--- a/libc/src/spawn/linux/posix_spawn.cpp
+++ b/libc/src/spawn/linux/posix_spawn.cpp
@@ -14,7 +14,7 @@
 #include "src/__support/macros/config.h"
 #include "src/spawn/file_actions.h"
 
-#include <fcntl.h>
+#include "hdr/types/mode_t.h"
 #include <signal.h> // For SIGCHLD
 #include <spawn.h>
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/libc/src/stdio/linux/CMakeLists.txt b/libc/src/stdio/linux/CMakeLists.txt
index d6241e1ca043..e81642dc6f01 100644
--- a/libc/src/stdio/linux/CMakeLists.txt
+++ b/libc/src/stdio/linux/CMakeLists.txt
@@ -5,7 +5,7 @@ add_entrypoint_object(
   HDRS
     ../remove.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.unistd
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
diff --git a/libc/src/stdio/linux/remove.cpp b/libc/src/stdio/linux/remove.cpp
index 9e299aaf43e4..dbb4491d0e6c 100644
--- a/libc/src/stdio/linux/remove.cpp
+++ b/libc/src/stdio/linux/remove.cpp
@@ -11,9 +11,9 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "hdr/fcntl_macros.h" // For AT_* macros.
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
-#include <fcntl.h>       // For AT_* macros.
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/sys/stat/linux/CMakeLists.txt b/libc/src/sys/stat/linux/CMakeLists.txt
index 415d2fa5c877..7c9496b6b6e8 100644
--- a/libc/src/sys/stat/linux/CMakeLists.txt
+++ b/libc/src/sys/stat/linux/CMakeLists.txt
@@ -5,7 +5,7 @@ add_entrypoint_object(
   HDRS
     ../chmod.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.types.mode_t
     libc.include.sys_stat
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
@@ -19,6 +19,7 @@ add_entrypoint_object(
   HDRS
     ../fchmod.h
   DEPENDS
+    libc.hdr.types.mode_t
     libc.include.sys_stat
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
@@ -45,7 +46,7 @@ add_entrypoint_object(
   HDRS
     ../mkdir.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.types.mode_t
     libc.include.sys_stat
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
@@ -84,7 +85,7 @@ add_entrypoint_object(
     ../stat.h
   DEPENDS
     .kernel_statx
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.sys_stat
     libc.src.errno.errno
 )
@@ -97,7 +98,7 @@ add_entrypoint_object(
     ../lstat.h
   DEPENDS
     .kernel_statx
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.sys_stat
     libc.src.errno.errno
 )
@@ -110,7 +111,7 @@ add_entrypoint_object(
     ../fstat.h
   DEPENDS
     .kernel_statx
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.sys_stat
     libc.src.errno.errno
 )
diff --git a/libc/src/sys/stat/linux/chmod.cpp b/libc/src/sys/stat/linux/chmod.cpp
index c91cabb514a8..9d2860391818 100644
--- a/libc/src/sys/stat/linux/chmod.cpp
+++ b/libc/src/sys/stat/linux/chmod.cpp
@@ -11,9 +11,9 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "hdr/types/mode_t.h"
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
-#include <fcntl.h>
 #include <sys/stat.h>
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/libc/src/sys/stat/linux/fchmod.cpp b/libc/src/sys/stat/linux/fchmod.cpp
index 7b6c7b7091a8..0d6fd359169a 100644
--- a/libc/src/sys/stat/linux/fchmod.cpp
+++ b/libc/src/sys/stat/linux/fchmod.cpp
@@ -11,9 +11,9 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "hdr/types/mode_t.h"
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
-#include <fcntl.h>
 #include <sys/stat.h>
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/libc/src/sys/stat/linux/fstat.cpp b/libc/src/sys/stat/linux/fstat.cpp
index 411aa47bcda2..35cf8f08f782 100644
--- a/libc/src/sys/stat/linux/fstat.cpp
+++ b/libc/src/sys/stat/linux/fstat.cpp
@@ -13,7 +13,7 @@
 
 #include "src/__support/common.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/stat.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/sys/stat/linux/lstat.cpp b/libc/src/sys/stat/linux/lstat.cpp
index 5a6eff068d1d..354c5b6e029a 100644
--- a/libc/src/sys/stat/linux/lstat.cpp
+++ b/libc/src/sys/stat/linux/lstat.cpp
@@ -14,7 +14,7 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/stat.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/sys/stat/linux/mkdir.cpp b/libc/src/sys/stat/linux/mkdir.cpp
index 527c3d2058d2..bd6efef858c7 100644
--- a/libc/src/sys/stat/linux/mkdir.cpp
+++ b/libc/src/sys/stat/linux/mkdir.cpp
@@ -11,9 +11,9 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "hdr/types/mode_t.h"
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
-#include <fcntl.h>
 #include <sys/stat.h>
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/libc/src/sys/stat/linux/stat.cpp b/libc/src/sys/stat/linux/stat.cpp
index c5149e6e3c88..de9cdb197d68 100644
--- a/libc/src/sys/stat/linux/stat.cpp
+++ b/libc/src/sys/stat/linux/stat.cpp
@@ -13,7 +13,7 @@
 
 #include "src/__support/common.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/stat.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/unistd/linux/CMakeLists.txt b/libc/src/unistd/linux/CMakeLists.txt
index 9b0d752cefbd..472438ca72e4 100644
--- a/libc/src/unistd/linux/CMakeLists.txt
+++ b/libc/src/unistd/linux/CMakeLists.txt
@@ -5,6 +5,7 @@ add_entrypoint_object(
   HDRS
     ../access.h
   DEPENDS
+    libc.hdr.fcntl_macros
     libc.include.unistd
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
@@ -57,7 +58,7 @@ add_entrypoint_object(
   HDRS
     ../dup2.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.unistd
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
@@ -254,7 +255,7 @@ add_entrypoint_object(
   HDRS
     ../link.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.unistd
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
@@ -268,7 +269,7 @@ add_entrypoint_object(
   HDRS
     ../linkat.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.unistd
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
@@ -377,7 +378,7 @@ add_entrypoint_object(
   HDRS
     ../rmdir.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.unistd
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
@@ -391,7 +392,7 @@ add_entrypoint_object(
   HDRS
     ../readlink.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.unistd
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
@@ -405,7 +406,7 @@ add_entrypoint_object(
   HDRS
     ../readlinkat.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.unistd
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
@@ -419,7 +420,7 @@ add_entrypoint_object(
   HDRS
     ../symlink.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.unistd
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
@@ -433,7 +434,7 @@ add_entrypoint_object(
   HDRS
     ../symlinkat.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.unistd
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
@@ -485,7 +486,7 @@ add_entrypoint_object(
   HDRS
     ../unlink.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.unistd
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
@@ -499,7 +500,7 @@ add_entrypoint_object(
   HDRS
     ../unlinkat.h
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.unistd
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
diff --git a/libc/src/unistd/linux/access.cpp b/libc/src/unistd/linux/access.cpp
index e9ad74989b05..2f7ebbcdf9e8 100644
--- a/libc/src/unistd/linux/access.cpp
+++ b/libc/src/unistd/linux/access.cpp
@@ -11,9 +11,9 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "hdr/fcntl_macros.h"
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
-#include <fcntl.h>
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/unistd/linux/dup2.cpp b/libc/src/unistd/linux/dup2.cpp
index 51a19a71a7d8..c7c7c1a8ca78 100644
--- a/libc/src/unistd/linux/dup2.cpp
+++ b/libc/src/unistd/linux/dup2.cpp
@@ -11,9 +11,9 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "hdr/fcntl_macros.h"
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
-#include <fcntl.h>
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/unistd/linux/link.cpp b/libc/src/unistd/linux/link.cpp
index 37ca58eab109..477806a70df7 100644
--- a/libc/src/unistd/linux/link.cpp
+++ b/libc/src/unistd/linux/link.cpp
@@ -13,7 +13,7 @@
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/unistd/linux/linkat.cpp b/libc/src/unistd/linux/linkat.cpp
index fcd6a5f75a19..40f68cc90c48 100644
--- a/libc/src/unistd/linux/linkat.cpp
+++ b/libc/src/unistd/linux/linkat.cpp
@@ -11,9 +11,9 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "hdr/fcntl_macros.h"
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
-#include <fcntl.h>
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/unistd/linux/readlink.cpp b/libc/src/unistd/linux/readlink.cpp
index 7b1524500440..2055e6b3400f 100644
--- a/libc/src/unistd/linux/readlink.cpp
+++ b/libc/src/unistd/linux/readlink.cpp
@@ -11,9 +11,9 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "hdr/fcntl_macros.h"
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
-#include <fcntl.h>
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/unistd/linux/readlinkat.cpp b/libc/src/unistd/linux/readlinkat.cpp
index 19a9ff9fbeb7..e5e4d0d39bc9 100644
--- a/libc/src/unistd/linux/readlinkat.cpp
+++ b/libc/src/unistd/linux/readlinkat.cpp
@@ -11,9 +11,9 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "hdr/fcntl_macros.h"
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
-#include <fcntl.h>
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/unistd/linux/rmdir.cpp b/libc/src/unistd/linux/rmdir.cpp
index 8974468ebcf1..075af12af64c 100644
--- a/libc/src/unistd/linux/rmdir.cpp
+++ b/libc/src/unistd/linux/rmdir.cpp
@@ -13,7 +13,7 @@
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/unistd/linux/symlink.cpp b/libc/src/unistd/linux/symlink.cpp
index 5efd4df85eda..9e1b2886ea0f 100644
--- a/libc/src/unistd/linux/symlink.cpp
+++ b/libc/src/unistd/linux/symlink.cpp
@@ -13,7 +13,7 @@
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/unistd/linux/symlinkat.cpp b/libc/src/unistd/linux/symlinkat.cpp
index 63d2e6d1507a..bcf2d0f8cc05 100644
--- a/libc/src/unistd/linux/symlinkat.cpp
+++ b/libc/src/unistd/linux/symlinkat.cpp
@@ -11,9 +11,9 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "hdr/fcntl_macros.h"
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
-#include <fcntl.h>
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/unistd/linux/unlink.cpp b/libc/src/unistd/linux/unlink.cpp
index de7cae8b826e..72d8e2398e3d 100644
--- a/libc/src/unistd/linux/unlink.cpp
+++ b/libc/src/unistd/linux/unlink.cpp
@@ -11,9 +11,9 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "hdr/fcntl_macros.h"
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
-#include <fcntl.h>
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/unistd/linux/unlinkat.cpp b/libc/src/unistd/linux/unlinkat.cpp
index e794f242b945..4ed20f542f17 100644
--- a/libc/src/unistd/linux/unlinkat.cpp
+++ b/libc/src/unistd/linux/unlinkat.cpp
@@ -13,7 +13,7 @@
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/test/src/fcntl/CMakeLists.txt b/libc/test/src/fcntl/CMakeLists.txt
index 48048b7fe886..b522fef7439d 100644
--- a/libc/test/src/fcntl/CMakeLists.txt
+++ b/libc/test/src/fcntl/CMakeLists.txt
@@ -42,7 +42,7 @@ add_libc_unittest(
   SRCS
     openat_test.cpp
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.src.errno.errno
     libc.src.fcntl.open
     libc.src.fcntl.openat
diff --git a/libc/test/src/fcntl/openat_test.cpp b/libc/test/src/fcntl/openat_test.cpp
index 9dafd125224a..547359eb9f7a 100644
--- a/libc/test/src/fcntl/openat_test.cpp
+++ b/libc/test/src/fcntl/openat_test.cpp
@@ -14,7 +14,7 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 
 TEST(LlvmLibcUniStd, OpenAndReadTest) {
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
diff --git a/libc/test/src/sys/sendfile/CMakeLists.txt b/libc/test/src/sys/sendfile/CMakeLists.txt
index 82efaa147bd8..ceaa4accdd06 100644
--- a/libc/test/src/sys/sendfile/CMakeLists.txt
+++ b/libc/test/src/sys/sendfile/CMakeLists.txt
@@ -9,7 +9,7 @@ add_libc_unittest(
   SRCS
     sendfile_test.cpp
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.sys_stat
     libc.src.errno.errno 
     libc.src.fcntl.open
diff --git a/libc/test/src/sys/sendfile/sendfile_test.cpp b/libc/test/src/sys/sendfile/sendfile_test.cpp
index 59025438a246..a658212ddb72 100644
--- a/libc/test/src/sys/sendfile/sendfile_test.cpp
+++ b/libc/test/src/sys/sendfile/sendfile_test.cpp
@@ -17,7 +17,7 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/stat.h>
 
 namespace cpp = LIBC_NAMESPACE::cpp;
diff --git a/libc/test/src/sys/stat/CMakeLists.txt b/libc/test/src/sys/stat/CMakeLists.txt
index 877a129b627d..dd3d0932755b 100644
--- a/libc/test/src/sys/stat/CMakeLists.txt
+++ b/libc/test/src/sys/stat/CMakeLists.txt
@@ -9,7 +9,7 @@ add_libc_unittest(
   SRCS
     chmod_test.cpp
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.sys_stat
     libc.src.errno.errno
     libc.src.fcntl.open
@@ -25,7 +25,7 @@ add_libc_unittest(
   SRCS
     fchmodat_test.cpp
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.sys_stat
     libc.src.errno.errno
     libc.src.fcntl.open
@@ -41,7 +41,7 @@ add_libc_unittest(
   SRCS
     fchmod_test.cpp
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.sys_stat
     libc.src.errno.errno
     libc.src.fcntl.open
@@ -57,7 +57,7 @@ add_libc_unittest(
   SRCS
     mkdirat_test.cpp
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.sys_stat
     libc.src.errno.errno
     libc.src.sys.stat.mkdirat
@@ -71,7 +71,7 @@ add_libc_unittest(
   SRCS
     stat_test.cpp
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.sys_stat
     libc.src.errno.errno
     libc.src.sys.stat.stat
@@ -87,7 +87,7 @@ add_libc_unittest(
   SRCS
     lstat_test.cpp
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.sys_stat
     libc.src.errno.errno
     libc.src.sys.stat.lstat
@@ -103,7 +103,7 @@ add_libc_unittest(
   SRCS
     fstat_test.cpp
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.sys_stat
     libc.src.errno.errno
     libc.src.sys.stat.fstat
diff --git a/libc/test/src/sys/stat/chmod_test.cpp b/libc/test/src/sys/stat/chmod_test.cpp
index c688996615ce..83ab0f45b6f0 100644
--- a/libc/test/src/sys/stat/chmod_test.cpp
+++ b/libc/test/src/sys/stat/chmod_test.cpp
@@ -14,7 +14,7 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/stat.h>
 
 TEST(LlvmLibcChmodTest, ChangeAndOpen) {
diff --git a/libc/test/src/sys/stat/fchmod_test.cpp b/libc/test/src/sys/stat/fchmod_test.cpp
index 91c0f68b8708..03eb79d95ddd 100644
--- a/libc/test/src/sys/stat/fchmod_test.cpp
+++ b/libc/test/src/sys/stat/fchmod_test.cpp
@@ -14,7 +14,7 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/stat.h>
 
 TEST(LlvmLibcChmodTest, ChangeAndOpen) {
diff --git a/libc/test/src/sys/stat/fchmodat_test.cpp b/libc/test/src/sys/stat/fchmodat_test.cpp
index c43ef8ae1331..09970b6e0fb1 100644
--- a/libc/test/src/sys/stat/fchmodat_test.cpp
+++ b/libc/test/src/sys/stat/fchmodat_test.cpp
@@ -14,7 +14,7 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/stat.h>
 
 TEST(LlvmLibcFchmodatTest, ChangeAndOpen) {
diff --git a/libc/test/src/sys/stat/fstat_test.cpp b/libc/test/src/sys/stat/fstat_test.cpp
index 1379eae26a47..34c675d1a4e2 100644
--- a/libc/test/src/sys/stat/fstat_test.cpp
+++ b/libc/test/src/sys/stat/fstat_test.cpp
@@ -14,7 +14,7 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/stat.h>
 
 TEST(LlvmLibcFStatTest, CreatAndReadMode) {
diff --git a/libc/test/src/sys/stat/lstat_test.cpp b/libc/test/src/sys/stat/lstat_test.cpp
index b44b3d1a59ce..a723d5ae2e29 100644
--- a/libc/test/src/sys/stat/lstat_test.cpp
+++ b/libc/test/src/sys/stat/lstat_test.cpp
@@ -14,7 +14,7 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/stat.h>
 
 TEST(LlvmLibcLStatTest, CreatAndReadMode) {
diff --git a/libc/test/src/sys/stat/mkdirat_test.cpp b/libc/test/src/sys/stat/mkdirat_test.cpp
index cbacc16b402d..85e013de234e 100644
--- a/libc/test/src/sys/stat/mkdirat_test.cpp
+++ b/libc/test/src/sys/stat/mkdirat_test.cpp
@@ -11,7 +11,7 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 
 TEST(LlvmLibcMkdiratTest, CreateAndRemove) {
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
diff --git a/libc/test/src/sys/stat/stat_test.cpp b/libc/test/src/sys/stat/stat_test.cpp
index baf363382022..0ddd8baaec1c 100644
--- a/libc/test/src/sys/stat/stat_test.cpp
+++ b/libc/test/src/sys/stat/stat_test.cpp
@@ -14,7 +14,7 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/stat.h>
 
 TEST(LlvmLibcStatTest, CreatAndReadMode) {
diff --git a/libc/test/src/unistd/CMakeLists.txt b/libc/test/src/unistd/CMakeLists.txt
index e03e56b3cf8a..ce936cebad42 100644
--- a/libc/test/src/unistd/CMakeLists.txt
+++ b/libc/test/src/unistd/CMakeLists.txt
@@ -24,11 +24,12 @@ add_libc_unittest(
   SRCS
     chdir_test.cpp
   DEPENDS
+    libc.hdr.fcntl_macros
     libc.include.unistd
     libc.src.errno.errno
-    libc.src.fcntl.open
     libc.src.unistd.chdir
     libc.src.unistd.close
+    libc.src.fcntl.open 
     libc.test.UnitTest.ErrnoSetterMatcher
 )
 
@@ -223,7 +224,7 @@ add_libc_unittest(
   SRCS
     rmdir_test.cpp
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.src.errno.errno
     libc.src.sys.stat.mkdir
     libc.src.unistd.rmdir
@@ -262,7 +263,7 @@ add_libc_unittest(
   SRCS
     readlinkat_test.cpp
   DEPENDS
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.unistd
     libc.src.errno.errno
     libc.src.unistd.readlinkat
@@ -410,7 +411,7 @@ add_libc_unittest(
     syscall_test.cpp
   DEPENDS
     libc.include.unistd
-    libc.include.fcntl
+    libc.hdr.fcntl_macros
     libc.include.sys_syscall
     libc.src.errno.errno 
     libc.src.unistd.__llvm_libc_syscall
diff --git a/libc/test/src/unistd/chdir_test.cpp b/libc/test/src/unistd/chdir_test.cpp
index 51dc7bb15d3e..e1bdcd77119f 100644
--- a/libc/test/src/unistd/chdir_test.cpp
+++ b/libc/test/src/unistd/chdir_test.cpp
@@ -13,7 +13,7 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 
 TEST(LlvmLibcChdirTest, ChangeAndOpen) {
   // The idea of this test is that we will first open an existing test file
diff --git a/libc/test/src/unistd/fchdir_test.cpp b/libc/test/src/unistd/fchdir_test.cpp
index ae88e1f22ed6..0e39fde17c67 100644
--- a/libc/test/src/unistd/fchdir_test.cpp
+++ b/libc/test/src/unistd/fchdir_test.cpp
@@ -13,7 +13,7 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 
 TEST(LlvmLibcChdirTest, ChangeAndOpen) {
   // The idea of this test is that we will first open an existing test file
diff --git a/libc/test/src/unistd/readlinkat_test.cpp b/libc/test/src/unistd/readlinkat_test.cpp
index 1fa683b02b5b..9e4bb9af02e7 100644
--- a/libc/test/src/unistd/readlinkat_test.cpp
+++ b/libc/test/src/unistd/readlinkat_test.cpp
@@ -15,7 +15,7 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 
 namespace cpp = LIBC_NAMESPACE::cpp;
 
diff --git a/libc/test/src/unistd/rmdir_test.cpp b/libc/test/src/unistd/rmdir_test.cpp
index 93cb0f3f53c1..4f4cd94c5cf0 100644
--- a/libc/test/src/unistd/rmdir_test.cpp
+++ b/libc/test/src/unistd/rmdir_test.cpp
@@ -12,7 +12,7 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 
 TEST(LlvmLibcRmdirTest, CreateAndRemove) {
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
diff --git a/libc/test/src/unistd/syscall_test.cpp b/libc/test/src/unistd/syscall_test.cpp
index cee29bd9afa3..f6cc3eab9aab 100644
--- a/libc/test/src/unistd/syscall_test.cpp
+++ b/libc/test/src/unistd/syscall_test.cpp
@@ -11,7 +11,7 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include <fcntl.h>
+#include "hdr/fcntl_macros.h"
 #include <sys/stat.h>    // For S_* flags.
 #include <sys/syscall.h> // For syscall numbers.
 #include <unistd.h>
-- 
GitLab


From 9d9b1ba951cb2c30a192aa1157ef1c022d6d145b Mon Sep 17 00:00:00 2001
From: Kai Nacke <kai.peter.nacke@ibm.com>
Date: Mon, 28 Oct 2024 16:29:40 -0400
Subject: [PATCH 363/693] [ORC] Fix JITLinkRedirectableSymbolManager
 construction to avoid crash (#113492)

Check `AnonymousPtrCreator` and `PtrJumpStubCreator` before creating the
JITLinkRedirectableSymbolManager object. This simplifies construction, and
avoids premature registration as a resource manager in the failure case.
-- 
GitLab


From 70d61f6de71bfe5ee870efc9b3e98db37273f17d Mon Sep 17 00:00:00 2001
From: Renaud Kauffmann <rkauffmann@nvidia.com>
Date: Mon, 28 Oct 2024 13:34:37 -0700
Subject: [PATCH 364/693] [flang][cuda] Adding runtime call to
 CUFRegisterVariable (#113952)

---
 flang/include/flang/Runtime/CUDA/registration.h | 5 +++++
 flang/runtime/CUDA/registration.cpp             | 8 ++++++++
 2 files changed, 13 insertions(+)

diff --git a/flang/include/flang/Runtime/CUDA/registration.h b/flang/include/flang/Runtime/CUDA/registration.h
index 009715613e29..5237069a4c73 100644
--- a/flang/include/flang/Runtime/CUDA/registration.h
+++ b/flang/include/flang/Runtime/CUDA/registration.h
@@ -11,6 +11,7 @@
 
 #include "flang/Runtime/entry-names.h"
 #include <cstddef>
+#include <cstdint>
 
 namespace Fortran::runtime::cuda {
 
@@ -23,6 +24,10 @@ void *RTDECL(CUFRegisterModule)(void *data);
 void RTDECL(CUFRegisterFunction)(
     void **module, const char *fctSym, char *fctName);
 
+/// Register a device variable.
+void RTDECL(CUFRegisterVariable)(
+    void **module, char *varSym, const char *varName, int64_t size);
+
 } // extern "C"
 
 } // namespace Fortran::runtime::cuda
diff --git a/flang/runtime/CUDA/registration.cpp b/flang/runtime/CUDA/registration.cpp
index 20d274c4d8d1..b7b6ef389bff 100644
--- a/flang/runtime/CUDA/registration.cpp
+++ b/flang/runtime/CUDA/registration.cpp
@@ -21,6 +21,9 @@ extern void __cudaRegisterFatBinaryEnd(void *);
 extern void __cudaRegisterFunction(void **fatCubinHandle, const char *hostFun,
     char *deviceFun, const char *deviceName, int thread_limit, uint3 *tid,
     uint3 *bid, dim3 *bDim, dim3 *gDim, int *wSize);
+extern void __cudaRegisterVar(void **fatCubinHandle, char *hostVar,
+    const char *deviceAddress, const char *deviceName, int ext, size_t size,
+    int constant, int global);
 
 void *RTDECL(CUFRegisterModule)(void *data) {
   void **fatHandle{__cudaRegisterFatBinary(data)};
@@ -34,6 +37,11 @@ void RTDEF(CUFRegisterFunction)(
       (uint3 *)0, (dim3 *)0, (dim3 *)0, (int *)0);
 }
 
+void RTDEF(CUFRegisterVariable)(
+    void **module, char *varSym, const char *varName, int64_t size) {
+  __cudaRegisterVar(module, varSym, varName, varName, 0, size, 0, 0);
+}
+
 } // extern "C"
 
 } // namespace Fortran::runtime::cuda
-- 
GitLab


From 71315698c91d0cda054b903da0594ca6f072c350 Mon Sep 17 00:00:00 2001
From: serge-sans-paille <sguelton@mozilla.com>
Date: Mon, 28 Oct 2024 20:40:52 +0000
Subject: [PATCH 365/693] [clang] Warn about memset/memcpy to
 NonTriviallyCopyable types (#111434)

This implements a warning that's similar to what GCC does in that
context: both memcpy and memset require their first and second operand
to be trivially copyable, let's warn if that's not the case.
---
 clang/docs/ReleaseNotes.rst                   |  5 ++
 .../clang/Basic/DiagnosticSemaKinds.td        |  4 ++
 clang/lib/Sema/SemaChecking.cpp               | 18 +++++
 clang/test/SemaCXX/constexpr-string.cpp       |  2 +
 clang/test/SemaCXX/warn-memaccess.cpp         | 68 +++++++++++++++++++
 .../__memory/uninitialized_algorithms.h       |  3 +-
 libcxx/test/std/utilities/expected/types.h    |  6 +-
 libcxx/test/support/min_allocator.h           |  4 +-
 8 files changed, 104 insertions(+), 6 deletions(-)
 create mode 100644 clang/test/SemaCXX/warn-memaccess.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 9515e96ffd01..424f02ef08d7 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -322,6 +322,11 @@ Modified Compiler Flags
   to utilize these vector libraries. The behavior for all other vector function
   libraries remains unchanged.
 
+- The ``-Wnontrivial-memaccess`` warning has been updated to also warn about
+  passing non-trivially-copyable destrination parameter to ``memcpy``,
+  ``memset`` and similar functions for which it is a documented undefined
+  behavior.
+
 Removed Compiler Flags
 -------------------------
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 9b9bdd7c800e..34ff49d7238a 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -795,6 +795,10 @@ def warn_cstruct_memaccess : Warning<
   "%1 call is a pointer to record %2 that is not trivial to "
   "%select{primitive-default-initialize|primitive-copy}3">,
   InGroup<NonTrivialMemaccess>;
+def warn_cxxstruct_memaccess : Warning<
+  "first argument in call to "
+  "%0 is a pointer to non-trivially copyable type %1">,
+  InGroup<NonTrivialMemaccess>;
 def note_nontrivial_field : Note<
   "field is non-trivial to %select{copy|default-initialize}0">;
 def err_non_trivial_c_union_in_invalid_context : Error<
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 27b274d74ce7..d027e4c6dfdb 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -8899,18 +8899,36 @@ void Sema::CheckMemaccessArguments(const CallExpr *Call,
           << ArgIdx << FnName << PointeeTy
           << Call->getCallee()->getSourceRange());
     else if (const auto *RT = PointeeTy->getAs<RecordType>()) {
+
+      bool IsTriviallyCopyableCXXRecord =
+          RT->desugar().isTriviallyCopyableType(Context);
+
       if ((BId == Builtin::BImemset || BId == Builtin::BIbzero) &&
           RT->getDecl()->isNonTrivialToPrimitiveDefaultInitialize()) {
         DiagRuntimeBehavior(Dest->getExprLoc(), Dest,
                             PDiag(diag::warn_cstruct_memaccess)
                                 << ArgIdx << FnName << PointeeTy << 0);
         SearchNonTrivialToInitializeField::diag(PointeeTy, Dest, *this);
+      } else if ((BId == Builtin::BImemset || BId == Builtin::BIbzero) &&
+                 !IsTriviallyCopyableCXXRecord && ArgIdx == 0) {
+        // FIXME: Limiting this warning to dest argument until we decide
+        // whether it's valid for source argument too.
+        DiagRuntimeBehavior(Dest->getExprLoc(), Dest,
+                            PDiag(diag::warn_cxxstruct_memaccess)
+                                << FnName << PointeeTy);
       } else if ((BId == Builtin::BImemcpy || BId == Builtin::BImemmove) &&
                  RT->getDecl()->isNonTrivialToPrimitiveCopy()) {
         DiagRuntimeBehavior(Dest->getExprLoc(), Dest,
                             PDiag(diag::warn_cstruct_memaccess)
                                 << ArgIdx << FnName << PointeeTy << 1);
         SearchNonTrivialToCopyField::diag(PointeeTy, Dest, *this);
+      } else if ((BId == Builtin::BImemcpy || BId == Builtin::BImemmove) &&
+                 !IsTriviallyCopyableCXXRecord && ArgIdx == 0) {
+        // FIXME: Limiting this warning to dest argument until we decide
+        // whether it's valid for source argument too.
+        DiagRuntimeBehavior(Dest->getExprLoc(), Dest,
+                            PDiag(diag::warn_cxxstruct_memaccess)
+                                << FnName << PointeeTy);
       } else {
         continue;
       }
diff --git a/clang/test/SemaCXX/constexpr-string.cpp b/clang/test/SemaCXX/constexpr-string.cpp
index c456740ef755..5448365489a5 100644
--- a/clang/test/SemaCXX/constexpr-string.cpp
+++ b/clang/test/SemaCXX/constexpr-string.cpp
@@ -670,6 +670,8 @@ namespace MemcpyEtc {
   constexpr bool test_address_of_incomplete_struct_type() { // expected-error {{never produces a constant}}
     struct Incomplete;
     extern Incomplete x, y;
+    // expected-warning@+2 {{first argument in call to '__builtin_memcpy' is a pointer to non-trivially copyable type 'Incomplete'}}
+    // expected-note@+1 {{explicitly cast the pointer to silence this warning}}
     __builtin_memcpy(&x, &x, 4);
     // expected-note@-1 2{{cannot constant evaluate 'memcpy' between objects of incomplete type 'Incomplete'}}
     return true;
diff --git a/clang/test/SemaCXX/warn-memaccess.cpp b/clang/test/SemaCXX/warn-memaccess.cpp
new file mode 100644
index 000000000000..b4b7f6a6905b
--- /dev/null
+++ b/clang/test/SemaCXX/warn-memaccess.cpp
@@ -0,0 +1,68 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 -Wnontrivial-memaccess %s
+
+extern "C" void *bzero(void *, unsigned);
+extern "C" void *memset(void *, int, unsigned);
+extern "C" void *memmove(void *s1, const void *s2, unsigned n);
+extern "C" void *memcpy(void *s1, const void *s2, unsigned n);
+
+class TriviallyCopyable {};
+class NonTriviallyCopyable { NonTriviallyCopyable(const NonTriviallyCopyable&);};
+
+void test_bzero(TriviallyCopyable* tc,
+                 NonTriviallyCopyable *ntc) {
+  // OK
+  bzero(tc, sizeof(*tc));
+
+  // expected-warning@+2{{first argument in call to 'bzero' is a pointer to non-trivially copyable type 'NonTriviallyCopyable'}}
+  // expected-note@+1{{explicitly cast the pointer to silence this warning}}
+  bzero(ntc, sizeof(*ntc));
+
+  // OK
+  bzero((void*)ntc, sizeof(*ntc));
+}
+
+void test_memset(TriviallyCopyable* tc,
+                 NonTriviallyCopyable *ntc) {
+  // OK
+  memset(tc, 0, sizeof(*tc));
+
+  // expected-warning@+2{{first argument in call to 'memset' is a pointer to non-trivially copyable type 'NonTriviallyCopyable'}}
+  // expected-note@+1{{explicitly cast the pointer to silence this warning}}
+  memset(ntc, 0, sizeof(*ntc));
+
+  // OK
+  memset((void*)ntc, 0, sizeof(*ntc));
+}
+
+
+void test_memcpy(TriviallyCopyable* tc0, TriviallyCopyable* tc1,
+                 NonTriviallyCopyable *ntc0, NonTriviallyCopyable *ntc1) {
+  // OK
+  memcpy(tc0, tc1, sizeof(*tc0));
+
+  // expected-warning@+2{{first argument in call to 'memcpy' is a pointer to non-trivially copyable type 'NonTriviallyCopyable'}}
+  // expected-note@+1{{explicitly cast the pointer to silence this warning}}
+  memcpy(ntc0, ntc1, sizeof(*ntc0));
+
+  // ~ OK
+  memcpy((void*)ntc0, ntc1, sizeof(*ntc0));
+
+  // OK
+  memcpy((void*)ntc0, (void*)ntc1, sizeof(*ntc0));
+}
+
+void test_memmove(TriviallyCopyable* tc0, TriviallyCopyable* tc1,
+                 NonTriviallyCopyable *ntc0, NonTriviallyCopyable *ntc1) {
+  // OK
+  memmove(tc0, tc1, sizeof(*tc0));
+
+  // expected-warning@+2{{first argument in call to 'memmove' is a pointer to non-trivially copyable type 'NonTriviallyCopyable'}}
+  // expected-note@+1{{explicitly cast the pointer to silence this warning}}
+  memmove(ntc0, ntc1, sizeof(*ntc0));
+
+  // ~ OK
+  memmove((void*)ntc0, ntc1, sizeof(*ntc0));
+
+  // OK
+  memmove((void*)ntc0, (void*)ntc1, sizeof(*ntc0));
+}
diff --git a/libcxx/include/__memory/uninitialized_algorithms.h b/libcxx/include/__memory/uninitialized_algorithms.h
index 54af1fa1a1cc..3fa948ecc43c 100644
--- a/libcxx/include/__memory/uninitialized_algorithms.h
+++ b/libcxx/include/__memory/uninitialized_algorithms.h
@@ -638,7 +638,8 @@ __uninitialized_allocator_relocate(_Alloc& __alloc, _Tp* __first, _Tp* __last, _
     __guard.__complete();
     std::__allocator_destroy(__alloc, __first, __last);
   } else {
-    __builtin_memcpy(__result, __first, sizeof(_Tp) * (__last - __first));
+    // Casting to void* to suppress clang complaining that this is technically UB.
+    __builtin_memcpy(static_cast<void*>(__result), __first, sizeof(_Tp) * (__last - __first));
   }
 }
 
diff --git a/libcxx/test/std/utilities/expected/types.h b/libcxx/test/std/utilities/expected/types.h
index 2b6983fb399c..df73ebdfe495 100644
--- a/libcxx/test/std/utilities/expected/types.h
+++ b/libcxx/test/std/utilities/expected/types.h
@@ -162,7 +162,7 @@ template <int Constant>
 struct TailClobberer {
   constexpr TailClobberer() noexcept {
     if (!std::is_constant_evaluated()) {
-      std::memset(this, Constant, sizeof(*this));
+      std::memset(static_cast<void*>(this), Constant, sizeof(*this));
     }
     // Always set `b` itself to `false` so that the comparison works.
     b = false;
@@ -245,7 +245,7 @@ struct BoolWithPadding {
   constexpr explicit BoolWithPadding() noexcept : BoolWithPadding(false) {}
   constexpr BoolWithPadding(bool val) noexcept {
     if (!std::is_constant_evaluated()) {
-      std::memset(this, 0, sizeof(*this));
+      std::memset(static_cast<void*>(this), 0, sizeof(*this));
     }
     val_ = val;
   }
@@ -268,7 +268,7 @@ struct IntWithoutPadding {
   constexpr explicit IntWithoutPadding() noexcept : IntWithoutPadding(0) {}
   constexpr IntWithoutPadding(int val) noexcept {
     if (!std::is_constant_evaluated()) {
-      std::memset(this, 0, sizeof(*this));
+      std::memset(static_cast<void*>(this), 0, sizeof(*this));
     }
     val_ = val;
   }
diff --git a/libcxx/test/support/min_allocator.h b/libcxx/test/support/min_allocator.h
index 13ee98289c36..18f51f807264 100644
--- a/libcxx/test/support/min_allocator.h
+++ b/libcxx/test/support/min_allocator.h
@@ -465,14 +465,14 @@ public:
   TEST_CONSTEXPR_CXX20 T* allocate(std::size_t n) {
     T* memory = std::allocator<T>().allocate(n);
     if (!TEST_IS_CONSTANT_EVALUATED)
-      std::memset(memory, 0, sizeof(T) * n);
+      std::memset(static_cast<void*>(memory), 0, sizeof(T) * n);
 
     return memory;
   }
 
   TEST_CONSTEXPR_CXX20 void deallocate(T* p, std::size_t n) {
     if (!TEST_IS_CONSTANT_EVALUATED)
-      DoNotOptimize(std::memset(p, 0, sizeof(T) * n));
+      DoNotOptimize(std::memset(static_cast<void*>(p), 0, sizeof(T) * n));
     std::allocator<T>().deallocate(p, n);
   }
 
-- 
GitLab


From 03dcefe08ecb68a3fedb7e9de6277df77371e9fc Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Mon, 28 Oct 2024 13:47:31 -0700
Subject: [PATCH 366/693] [libc] Fix leftover `LIBC_NAMESPACE` after porting it
 (#113960)

Summary:
There are a few of these leftover, they should all use the
`LIBC_NAMESPACE_DECL` version because that implies visibility.
---
 libc/src/math/cbrt.h                | 6 ++++--
 libc/src/stdio/gpu/fprintf.cpp      | 4 ++--
 libc/src/stdio/gpu/printf.cpp       | 4 ++--
 libc/src/stdio/gpu/vfprintf.cpp     | 4 ++--
 libc/src/stdio/gpu/vfprintf_utils.h | 5 +++--
 libc/src/stdio/gpu/vprintf.cpp      | 4 ++--
 libc/src/stdio/vsscanf.h            | 6 ++++--
 7 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/libc/src/math/cbrt.h b/libc/src/math/cbrt.h
index a7d5fe80e57b..8cf7d9b221df 100644
--- a/libc/src/math/cbrt.h
+++ b/libc/src/math/cbrt.h
@@ -9,10 +9,12 @@
 #ifndef LLVM_LIBC_SRC_MATH_CBRT_H
 #define LLVM_LIBC_SRC_MATH_CBRT_H
 
-namespace LIBC_NAMESPACE {
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
 
 double cbrt(double x);
 
-} // namespace LIBC_NAMESPACE
+} // namespace LIBC_NAMESPACE_DECL
 
 #endif // LLVM_LIBC_SRC_MATH_CBRT_H
diff --git a/libc/src/stdio/gpu/fprintf.cpp b/libc/src/stdio/gpu/fprintf.cpp
index 6222589cc4ba..46196d7d2b10 100644
--- a/libc/src/stdio/gpu/fprintf.cpp
+++ b/libc/src/stdio/gpu/fprintf.cpp
@@ -16,7 +16,7 @@
 
 #include <stdarg.h>
 
-namespace LIBC_NAMESPACE {
+namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, fprintf,
                    (::FILE *__restrict stream, const char *__restrict format,
@@ -29,4 +29,4 @@ LLVM_LIBC_FUNCTION(int, fprintf,
   return ret_val;
 }
 
-} // namespace LIBC_NAMESPACE
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdio/gpu/printf.cpp b/libc/src/stdio/gpu/printf.cpp
index d9903193ef16..be1885fd6801 100644
--- a/libc/src/stdio/gpu/printf.cpp
+++ b/libc/src/stdio/gpu/printf.cpp
@@ -15,7 +15,7 @@
 
 #include <stdarg.h>
 
-namespace LIBC_NAMESPACE {
+namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, printf, (const char *__restrict format, ...)) {
   va_list vlist;
@@ -26,4 +26,4 @@ LLVM_LIBC_FUNCTION(int, printf, (const char *__restrict format, ...)) {
   return ret_val;
 }
 
-} // namespace LIBC_NAMESPACE
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdio/gpu/vfprintf.cpp b/libc/src/stdio/gpu/vfprintf.cpp
index 961cfa48579e..c92685f48c72 100644
--- a/libc/src/stdio/gpu/vfprintf.cpp
+++ b/libc/src/stdio/gpu/vfprintf.cpp
@@ -14,7 +14,7 @@
 #include "src/errno/libc_errno.h"
 #include "src/stdio/gpu/vfprintf_utils.h"
 
-namespace LIBC_NAMESPACE {
+namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, vfprintf,
                    (::FILE *__restrict stream, const char *__restrict format,
@@ -24,4 +24,4 @@ LLVM_LIBC_FUNCTION(int, vfprintf,
   return ret_val;
 }
 
-} // namespace LIBC_NAMESPACE
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdio/gpu/vfprintf_utils.h b/libc/src/stdio/gpu/vfprintf_utils.h
index 93ce1649869f..5010ee16d960 100644
--- a/libc/src/stdio/gpu/vfprintf_utils.h
+++ b/libc/src/stdio/gpu/vfprintf_utils.h
@@ -9,10 +9,11 @@
 #include "hdr/types/FILE.h"
 #include "src/__support/RPC/rpc_client.h"
 #include "src/__support/arg_list.h"
+#include "src/__support/macros/config.h"
 #include "src/stdio/gpu/file.h"
 #include "src/string/string_utils.h"
 
-namespace LIBC_NAMESPACE {
+namespace LIBC_NAMESPACE_DECL {
 
 template <uint16_t opcode>
 LIBC_INLINE int vfprintf_impl(::FILE *__restrict file,
@@ -82,4 +83,4 @@ LIBC_INLINE int vfprintf_internal(::FILE *__restrict stream,
 #endif
 }
 
-} // namespace LIBC_NAMESPACE
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdio/gpu/vprintf.cpp b/libc/src/stdio/gpu/vprintf.cpp
index 2bb74d7f017b..54012f307184 100644
--- a/libc/src/stdio/gpu/vprintf.cpp
+++ b/libc/src/stdio/gpu/vprintf.cpp
@@ -13,7 +13,7 @@
 #include "src/errno/libc_errno.h"
 #include "src/stdio/gpu/vfprintf_utils.h"
 
-namespace LIBC_NAMESPACE {
+namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, vprintf,
                    (const char *__restrict format, va_list vlist)) {
@@ -22,4 +22,4 @@ LLVM_LIBC_FUNCTION(int, vprintf,
   return ret_val;
 }
 
-} // namespace LIBC_NAMESPACE
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdio/vsscanf.h b/libc/src/stdio/vsscanf.h
index 992c44d3d95b..c57b1743e477 100644
--- a/libc/src/stdio/vsscanf.h
+++ b/libc/src/stdio/vsscanf.h
@@ -9,12 +9,14 @@
 #ifndef LLVM_LIBC_SRC_STDIO_VSSCANF_H
 #define LLVM_LIBC_SRC_STDIO_VSSCANF_H
 
+#include "src/__support/macros/config.h"
+
 #include <stdarg.h>
 
-namespace LIBC_NAMESPACE {
+namespace LIBC_NAMESPACE_DECL {
 
 int vsscanf(const char *s, const char *format, va_list vlist);
 
-} // namespace LIBC_NAMESPACE
+} // namespace LIBC_NAMESPACE_DECL
 
 #endif // LLVM_LIBC_SRC_STDIO_VSSCANF_H
-- 
GitLab


From 39303e24b6f628f3c080f1b54bd12383a55b9b3a Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Mon, 28 Oct 2024 14:00:31 -0700
Subject: [PATCH 367/693] [clang][deps] Improve timing output (#113726)

This patch adds the number of executed instructions into the timing
output, which provides more stable results compared to wall or process
time.

The format itself is also tweaked so that it's more amenable for direct
import into a spreadsheet editor.
---
 clang/test/ClangScanDeps/print-timing.c       |  3 ++-
 clang/tools/clang-scan-deps/ClangScanDeps.cpp | 13 +++++++++----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/clang/test/ClangScanDeps/print-timing.c b/clang/test/ClangScanDeps/print-timing.c
index f27df1ebf732..fa2a433b9553 100644
--- a/clang/test/ClangScanDeps/print-timing.c
+++ b/clang/test/ClangScanDeps/print-timing.c
@@ -3,7 +3,8 @@
 
 // RUN: clang-scan-deps -compilation-database %t/cdb.json -print-timing > %t/result.json 2>%t/errs
 // RUN: cat %t/errs | FileCheck %s
-// CHECK: clang-scan-deps timing: {{[0-9]+}}.{{[0-9][0-9]}}s wall, {{[0-9]+}}.{{[0-9][0-9]}}s process
+// CHECK:      wall time [s]              process time [s]           instruction count
+// CHECK-NEXT: {{[0-9]+}}.{{([0-9]{4})}}  {{[0-9]+}}.{{([0-9]{4})}}  {{[0-9]+}}
 
 //--- cdb.json
 []
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index 7d36cee7a22b..f474b1346b1b 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -1080,10 +1080,15 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
                  << NumExistsCalls << " exists() calls\n"
                  << NumIsLocalCalls << " isLocal() calls\n";
 
-  if (PrintTiming)
-    llvm::errs() << llvm::format(
-        "clang-scan-deps timing: %0.2fs wall, %0.2fs process\n",
-        T.getTotalTime().getWallTime(), T.getTotalTime().getProcessTime());
+  if (PrintTiming) {
+    llvm::errs() << "wall time [s]\t"
+                 << "process time [s]\t"
+                 << "instruction count\n";
+    const llvm::TimeRecord &R = T.getTotalTime();
+    llvm::errs() << llvm::format("%0.4f", R.getWallTime()) << "\t"
+                 << llvm::format("%0.4f", R.getProcessTime()) << "\t"
+                 << llvm::format("%llu", R.getInstructionsExecuted()) << "\n";
+  }
 
   if (RoundTripArgs)
     if (FD && FD->roundTripCommands(llvm::errs()))
-- 
GitLab


From 5ea694816b569e010854a861ad58502c056d5a39 Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Mon, 28 Oct 2024 14:05:58 -0700
Subject: [PATCH 368/693] [SandboxVec][Legality] Check opcodes and types
 (#113741)

---
 .../Vectorize/SandboxVectorizer/VecUtils.h    | 12 ++++--
 .../Vectorize/SandboxVectorizer/Legality.cpp  | 20 +++++++++-
 .../SandboxVectorizer/CMakeLists.txt          |  1 +
 .../SandboxVectorizer/LegalityTest.cpp        | 27 +++++++++++++-
 .../SandboxVectorizer/VecUtilsTest.cpp        | 37 +++++++++++++++++++
 5 files changed, 92 insertions(+), 5 deletions(-)
 create mode 100644 llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp

diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
index 64f57edb3848..9577e8ef7b37 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
@@ -12,7 +12,11 @@
 #ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
 #define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
 
-class Utils {
+#include "llvm/SandboxIR/Type.h"
+
+namespace llvm::sandboxir {
+
+class VecUtils {
 public:
   /// \Returns the number of elements in \p Ty. That is the number of lanes if a
   /// fixed vector or 1 if scalar. ScalableVectors have unknown size and
@@ -25,6 +29,8 @@ public:
   static Type *getElementType(Type *Ty) {
     return Ty->isVectorTy() ? cast<FixedVectorType>(Ty)->getElementType() : Ty;
   }
-}
+};
+
+} // namespace llvm::sandboxir
 
-#endif LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
index e4546c2f9811..fcfb11c669fa 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
@@ -11,6 +11,7 @@
 #include "llvm/SandboxIR/Utils.h"
 #include "llvm/SandboxIR/Value.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
 
 namespace llvm::sandboxir {
 
@@ -26,7 +27,24 @@ void LegalityResult::dump() const {
 std::optional<ResultReason>
 LegalityAnalysis::notVectorizableBasedOnOpcodesAndTypes(
     ArrayRef<Value *> Bndl) {
-  // TODO: Unimplemented.
+  auto *I0 = cast<Instruction>(Bndl[0]);
+  auto Opcode = I0->getOpcode();
+  // If they have different opcodes, then we cannot form a vector (for now).
+  if (any_of(drop_begin(Bndl), [Opcode](Value *V) {
+        return cast<Instruction>(V)->getOpcode() != Opcode;
+      }))
+    return ResultReason::DiffOpcodes;
+
+  // If not the same scalar type, Pack. This will accept scalars and vectors as
+  // long as the element type is the same.
+  Type *ElmTy0 = VecUtils::getElementType(Utils::getExpectedType(I0));
+  if (any_of(drop_begin(Bndl), [ElmTy0](Value *V) {
+        return VecUtils::getElementType(Utils::getExpectedType(V)) != ElmTy0;
+      }))
+    return ResultReason::DiffTypes;
+
+  // TODO: Missing checks
+
   return std::nullopt;
 }
 
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/CMakeLists.txt b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/CMakeLists.txt
index 24512cb0225e..df689767b772 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/CMakeLists.txt
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/CMakeLists.txt
@@ -13,4 +13,5 @@ add_llvm_unittest(SandboxVectorizerTests
   LegalityTest.cpp
   SchedulerTest.cpp
   SeedCollectorTest.cpp	
+  VecUtilsTest.cpp
 )
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
index 56c6bf5f1ef1..51f445c8d1d0 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
@@ -29,13 +29,17 @@ struct LegalityTest : public testing::Test {
 
 TEST_F(LegalityTest, Legality) {
   parseIR(C, R"IR(
-define void @foo(ptr %ptr) {
+define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg) {
   %gep0 = getelementptr float, ptr %ptr, i32 0
   %gep1 = getelementptr float, ptr %ptr, i32 1
+  %gep3 = getelementptr float, ptr %ptr, i32 3
   %ld0 = load float, ptr %gep0
   %ld1 = load float, ptr %gep0
   store float %ld0, ptr %gep0
   store float %ld1, ptr %gep1
+  store <2 x float> %vec2, ptr %gep1
+  store <3 x float> %vec3, ptr %gep3
+  store i8 %arg, ptr %gep1
   ret void
 }
 )IR");
@@ -46,10 +50,14 @@ define void @foo(ptr %ptr) {
   auto It = BB->begin();
   [[maybe_unused]] auto *Gep0 = cast<sandboxir::GetElementPtrInst>(&*It++);
   [[maybe_unused]] auto *Gep1 = cast<sandboxir::GetElementPtrInst>(&*It++);
+  [[maybe_unused]] auto *Gep3 = cast<sandboxir::GetElementPtrInst>(&*It++);
   [[maybe_unused]] auto *Ld0 = cast<sandboxir::LoadInst>(&*It++);
   [[maybe_unused]] auto *Ld1 = cast<sandboxir::LoadInst>(&*It++);
   auto *St0 = cast<sandboxir::StoreInst>(&*It++);
   auto *St1 = cast<sandboxir::StoreInst>(&*It++);
+  auto *StVec2 = cast<sandboxir::StoreInst>(&*It++);
+  auto *StVec3 = cast<sandboxir::StoreInst>(&*It++);
+  auto *StI8 = cast<sandboxir::StoreInst>(&*It++);
 
   sandboxir::LegalityAnalysis Legality;
   const auto &Result = Legality.canVectorize({St0, St1});
@@ -62,6 +70,23 @@ define void @foo(ptr %ptr) {
     EXPECT_EQ(cast<sandboxir::Pack>(Result).getReason(),
               sandboxir::ResultReason::NotInstructions);
   }
+  {
+    // Check DiffOpcodes
+    const auto &Result = Legality.canVectorize({St0, Ld0});
+    EXPECT_TRUE(isa<sandboxir::Pack>(Result));
+    EXPECT_EQ(cast<sandboxir::Pack>(Result).getReason(),
+              sandboxir::ResultReason::DiffOpcodes);
+  }
+  {
+    // Check DiffTypes
+    EXPECT_TRUE(isa<sandboxir::Widen>(Legality.canVectorize({St0, StVec2})));
+    EXPECT_TRUE(isa<sandboxir::Widen>(Legality.canVectorize({StVec2, StVec3})));
+
+    const auto &Result = Legality.canVectorize({St0, StI8});
+    EXPECT_TRUE(isa<sandboxir::Pack>(Result));
+    EXPECT_EQ(cast<sandboxir::Pack>(Result).getReason(),
+              sandboxir::ResultReason::DiffTypes);
+  }
 }
 
 #ifndef NDEBUG
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
new file mode 100644
index 000000000000..e0b082849643
--- /dev/null
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
@@ -0,0 +1,37 @@
+//===- VecUtilsTest.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/SandboxIR/Context.h"
+#include "llvm/SandboxIR/Type.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+struct VecUtilsTest : public testing::Test {
+  LLVMContext C;
+};
+
+TEST_F(VecUtilsTest, GetNumElements) {
+  sandboxir::Context Ctx(C);
+  auto *ElemTy = sandboxir::Type::getInt32Ty(Ctx);
+  EXPECT_EQ(sandboxir::VecUtils::getNumElements(ElemTy), 1);
+  auto *VTy = sandboxir::FixedVectorType::get(ElemTy, 2);
+  EXPECT_EQ(sandboxir::VecUtils::getNumElements(VTy), 2);
+  auto *VTy1 = sandboxir::FixedVectorType::get(ElemTy, 1);
+  EXPECT_EQ(sandboxir::VecUtils::getNumElements(VTy1), 1);
+}
+
+TEST_F(VecUtilsTest, GetElementType) {
+  sandboxir::Context Ctx(C);
+  auto *ElemTy = sandboxir::Type::getInt32Ty(Ctx);
+  EXPECT_EQ(sandboxir::VecUtils::getElementType(ElemTy), ElemTy);
+  auto *VTy = sandboxir::FixedVectorType::get(ElemTy, 2);
+  EXPECT_EQ(sandboxir::VecUtils::getElementType(VTy), ElemTy);
+}
-- 
GitLab


From 09a4bcf1a549eea738bda74b2b7dc0f5c8309310 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Mon, 28 Oct 2024 16:06:22 -0500
Subject: [PATCH 369/693] [flang][OpenMP] Update handling of DEPEND clause
 (#113620)

Parse the locator list in OmpDependClause as an OmpObjectList (instead
of a list of Designators). When a common block appears in the locator
list, show an informative message.
Implement resolving symbols in DependSinkVec in a dedicated visitor
instead of having a visitor for OmpDependClause.
Resolve unresolved names common blocks in OmpObjectList.

Minor changes to the code organization:
- rename OmpDependenceType to OmpTaskDependenceType (to follow 5.2
terminology),
- rename Depend::WithLocators to Depend::DepType,
- add comments with more detailed spec references to parse-tree.h.

---------

Co-authored-by: Kiran Chandramohan <kiran.chandramohan@arm.com>
---
 flang/examples/FeatureList/FeatureList.cpp    |  4 +--
 .../FlangOmpReport/FlangOmpReportVisitor.cpp  |  4 +--
 .../FlangOmpReport/FlangOmpReportVisitor.h    |  2 +-
 flang/include/flang/Parser/dump-parse-tree.h  |  4 +--
 flang/include/flang/Parser/parse-tree.h       | 32 +++++++++++++------
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    | 10 +++---
 flang/lib/Lower/OpenMP/Clauses.cpp            | 18 ++++-------
 flang/lib/Parser/openmp-parsers.cpp           | 10 +++---
 flang/lib/Parser/unparse.cpp                  |  6 ++--
 flang/lib/Semantics/check-omp-structure.cpp   | 24 ++++++++------
 flang/lib/Semantics/resolve-directives.cpp    | 27 ++++++++++------
 flang/test/Semantics/OpenMP/depend04.f90      | 10 ++++++
 llvm/include/llvm/Frontend/OpenMP/ClauseT.h   |  4 +--
 13 files changed, 93 insertions(+), 62 deletions(-)
 create mode 100644 flang/test/Semantics/OpenMP/depend04.f90

diff --git a/flang/examples/FeatureList/FeatureList.cpp b/flang/examples/FeatureList/FeatureList.cpp
index 9fce67e61ed3..62f8d39a8aba 100644
--- a/flang/examples/FeatureList/FeatureList.cpp
+++ b/flang/examples/FeatureList/FeatureList.cpp
@@ -473,8 +473,8 @@ public:
   READ_FEATURE(OmpDependClause::InOut)
   READ_FEATURE(OmpDependClause::Sink)
   READ_FEATURE(OmpDependClause::Source)
-  READ_FEATURE(OmpDependenceType)
-  READ_FEATURE(OmpDependenceType::Type)
+  READ_FEATURE(OmpTaskDependenceType)
+  READ_FEATURE(OmpTaskDependenceType::Type)
   READ_FEATURE(OmpDependSinkVec)
   READ_FEATURE(OmpDependSinkVecLength)
   READ_FEATURE(OmpEndAllocators)
diff --git a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
index 5d3c5cd72eef..d28ed0534d60 100644
--- a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
+++ b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
@@ -222,9 +222,9 @@ void OpenMPCounterVisitor::Post(const OmpLinearModifier::Type &c) {
   clauseDetails +=
       "modifier=" + std::string{OmpLinearModifier::EnumToString(c)} + ";";
 }
-void OpenMPCounterVisitor::Post(const OmpDependenceType::Type &c) {
+void OpenMPCounterVisitor::Post(const OmpTaskDependenceType::Type &c) {
   clauseDetails +=
-      "type=" + std::string{OmpDependenceType::EnumToString(c)} + ";";
+      "type=" + std::string{OmpTaskDependenceType::EnumToString(c)} + ";";
 }
 void OpenMPCounterVisitor::Post(const OmpMapClause::Type &c) {
   clauseDetails += "type=" + std::string{OmpMapClause::EnumToString(c)} + ";";
diff --git a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h
index 380534ebbfd7..68c52db46e2f 100644
--- a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h
+++ b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h
@@ -73,7 +73,7 @@ struct OpenMPCounterVisitor {
   void Post(const OmpDeviceTypeClause::Type &c);
   void Post(const OmpScheduleModifierType::ModType &c);
   void Post(const OmpLinearModifier::Type &c);
-  void Post(const OmpDependenceType::Type &c);
+  void Post(const OmpTaskDependenceType::Type &c);
   void Post(const OmpMapClause::Type &c);
   void Post(const OmpScheduleClause::ScheduleType &c);
   void Post(const OmpIfClause::DirectiveNameModifier &c);
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index ccdfe980f6f3..31ad1b7c6ce5 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -513,8 +513,8 @@ public:
   NODE(OmpDependClause, InOut)
   NODE(OmpDependClause, Sink)
   NODE(OmpDependClause, Source)
-  NODE(parser, OmpDependenceType)
-  NODE_ENUM(OmpDependenceType, Type)
+  NODE(parser, OmpTaskDependenceType)
+  NODE_ENUM(OmpTaskDependenceType, Type)
   NODE(parser, OmpDependSinkVec)
   NODE(parser, OmpDependSinkVecLength)
   NODE(parser, OmpEndAllocators)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 2a312e29a3a4..506a470c5557 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3439,6 +3439,18 @@ struct OmpObject {
 
 WRAPPER_CLASS(OmpObjectList, std::list<OmpObject>);
 
+// Ref: [4.5:169-170], [5.0:254-256], [5.1:287-289], [5.2:321]
+//
+// task-dependence-type -> // "dependence-type" in 5.1 and before
+//    IN | OUT | INOUT |        // since 4.5
+//    SOURCE | SINK |           // since 4.5, until 5.1
+//    MUTEXINOUTSET | DEPOBJ |  // since 5.0
+//    INOUTSET                  // since 5.2
+struct OmpTaskDependenceType {
+  ENUM_CLASS(Type, In, Out, Inout, Source, Sink)
+  WRAPPER_CLASS_BOILERPLATE(OmpTaskDependenceType, Type);
+};
+
 // [5.0] 2.1.6 iterator-specifier -> type-declaration-stmt = subscript-triple
 //             iterator-modifier -> iterator-specifier-list
 struct OmpIteratorSpecifier {
@@ -3534,27 +3546,27 @@ struct OmpDependSinkVecLength {
   std::tuple<DefinedOperator, ScalarIntConstantExpr> t;
 };
 
-// 2.13.9 depend-vec -> iterator [+/- depend-vec-length],...,iterator[...]
+// 2.13.9 depend-vec -> induction-variable [depend-vec-length], ...
 struct OmpDependSinkVec {
   TUPLE_CLASS_BOILERPLATE(OmpDependSinkVec);
   std::tuple<Name, std::optional<OmpDependSinkVecLength>> t;
 };
 
-// 2.13.9 depend-type -> IN | OUT | INOUT | SOURCE | SINK
-struct OmpDependenceType {
-  ENUM_CLASS(Type, In, Out, Inout, Source, Sink)
-  WRAPPER_CLASS_BOILERPLATE(OmpDependenceType, Type);
-};
-
-// 2.13.9 depend-clause -> DEPEND (((IN | OUT | INOUT) : variable-name-list) |
-//                                 SOURCE | SINK : depend-vec)
+// Ref: [4.5:169-170], [5.0:255-256], [5.1:288-289], [5.2:323-324]
+//
+// depend-clause ->
+//    DEPEND(SOURCE) |                               // since 4.5, until 5.1
+//    DEPEND(SINK: depend-vec) |                     // since 4.5, until 5.1
+//    DEPEND([depend-modifier,]dependence-type: locator-list)   // since 4.5
+//
+// depend-modifier -> iterator-modifier              // since 5.0
 struct OmpDependClause {
   UNION_CLASS_BOILERPLATE(OmpDependClause);
   EMPTY_CLASS(Source);
   WRAPPER_CLASS(Sink, std::list<OmpDependSinkVec>);
   struct InOut {
     TUPLE_CLASS_BOILERPLATE(InOut);
-    std::tuple<OmpDependenceType, std::list<Designator>> t;
+    std::tuple<OmpTaskDependenceType, OmpObjectList> t;
   };
   std::variant<Source, Sink, InOut> u;
 };
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index fbc031f3a93d..8fb0dd4a1ec3 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -798,11 +798,11 @@ bool ClauseProcessor::processDepend(mlir::omp::DependClauseOps &result) const {
   return findRepeatableClause<omp::clause::Depend>(
       [&](const omp::clause::Depend &clause, const parser::CharBlock &) {
         using Depend = omp::clause::Depend;
-        assert(std::holds_alternative<Depend::WithLocators>(clause.u) &&
-               "Only the modern form is handled at the moment");
-        auto &modern = std::get<Depend::WithLocators>(clause.u);
-        auto kind = std::get<Depend::TaskDependenceType>(modern.t);
-        auto &objects = std::get<omp::ObjectList>(modern.t);
+        assert(std::holds_alternative<Depend::DepType>(clause.u) &&
+               "Only the form with dependence type is handled at the moment");
+        auto &depType = std::get<Depend::DepType>(clause.u);
+        auto kind = std::get<Depend::TaskDependenceType>(depType.t);
+        auto &objects = std::get<omp::ObjectList>(depType.t);
 
         mlir::omp::ClauseTaskDependAttr dependTypeOperand =
             genDependKindAttr(firOpBuilder, kind);
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 3bd89b543288..b1fa52751fbd 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -555,7 +555,7 @@ Depend make(const parser::OmpClause::Depend &inp,
   using Iteration = Doacross::Vector::value_type; // LoopIterationT
 
   CLAUSET_ENUM_CONVERT( //
-      convert1, parser::OmpDependenceType::Type, Depend::TaskDependenceType,
+      convert1, parser::OmpTaskDependenceType::Type, Depend::TaskDependenceType,
       // clang-format off
       MS(In,     In)
       MS(Out,    Out)
@@ -593,17 +593,13 @@ Depend make(const parser::OmpClause::Depend &inp,
             return Doacross{{/*DependenceType=*/Doacross::DependenceType::Sink,
                              /*Vector=*/makeList(s.v, convert2)}};
           },
-          // Depend::WithLocators
+          // Depend::DepType
           [&](const wrapped::InOut &s) -> Variant {
-            auto &t0 = std::get<parser::OmpDependenceType>(s.t);
-            auto &t1 = std::get<std::list<parser::Designator>>(s.t);
-            auto convert4 = [&](const parser::Designator &t) {
-              return makeObject(t, semaCtx);
-            };
-            return Depend::WithLocators{
-                {/*TaskDependenceType=*/convert1(t0.v),
-                 /*Iterator=*/std::nullopt,
-                 /*LocatorList=*/makeList(t1, convert4)}};
+            auto &t0 = std::get<parser::OmpTaskDependenceType>(s.t);
+            auto &t1 = std::get<parser::OmpObjectList>(s.t);
+            return Depend::DepType{{/*TaskDependenceType=*/convert1(t0.v),
+                                    /*Iterator=*/std::nullopt,
+                                    /*LocatorList=*/makeObjects(t1, semaCtx)}};
           },
       },
       inp.v.u)};
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index ae0c351fed56..3ca4e93a6c9b 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -365,10 +365,10 @@ TYPE_PARSER(construct<OmpDependSinkVecLength>(
 TYPE_PARSER(
     construct<OmpDependSinkVec>(name, maybe(Parser<OmpDependSinkVecLength>{})))
 
-TYPE_PARSER(
-    construct<OmpDependenceType>("IN"_id >> pure(OmpDependenceType::Type::In) ||
-        "INOUT" >> pure(OmpDependenceType::Type::Inout) ||
-        "OUT" >> pure(OmpDependenceType::Type::Out)))
+TYPE_PARSER(construct<OmpTaskDependenceType>(
+    "IN"_id >> pure(OmpTaskDependenceType::Type::In) ||
+    "INOUT" >> pure(OmpTaskDependenceType::Type::Inout) ||
+    "OUT" >> pure(OmpTaskDependenceType::Type::Out)))
 
 TYPE_CONTEXT_PARSER("Omp Depend clause"_en_US,
     construct<OmpDependClause>(construct<OmpDependClause::Sink>(
@@ -376,7 +376,7 @@ TYPE_CONTEXT_PARSER("Omp Depend clause"_en_US,
         construct<OmpDependClause>(
             construct<OmpDependClause::Source>("SOURCE"_tok)) ||
         construct<OmpDependClause>(construct<OmpDependClause::InOut>(
-            Parser<OmpDependenceType>{}, ":" >> nonemptyList(designator))))
+            Parser<OmpTaskDependenceType>{}, ":" >> Parser<OmpObjectList>{})))
 
 // 2.15.3.7 LINEAR (linear-list: linear-step)
 //          linear-list -> list | modifier(list)
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index ba4155469073..39fcb61609e3 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2216,9 +2216,9 @@ public:
   }
   void Unparse(const OmpDependClause::InOut &x) {
     Put("(");
-    Walk(std::get<OmpDependenceType>(x.t));
+    Walk(std::get<OmpTaskDependenceType>(x.t));
     Put(":");
-    Walk(std::get<std::list<Designator>>(x.t), ",");
+    Walk(std::get<OmpObjectList>(x.t));
     Put(")");
   }
   bool Pre(const OmpDependClause &x) {
@@ -2829,7 +2829,7 @@ public:
       OmpLastprivateClause, LastprivateModifier) // OMP lastprivate-modifier
   WALK_NESTED_ENUM(OmpScheduleModifierType, ModType) // OMP schedule-modifier
   WALK_NESTED_ENUM(OmpLinearModifier, Type) // OMP linear-modifier
-  WALK_NESTED_ENUM(OmpDependenceType, Type) // OMP dependence-type
+  WALK_NESTED_ENUM(OmpTaskDependenceType, Type) // OMP task-dependence-type
   WALK_NESTED_ENUM(OmpScheduleClause, ScheduleType) // OMP schedule-type
   WALK_NESTED_ENUM(OmpDeviceClause, DeviceModifier) // OMP device modifier
   WALK_NESTED_ENUM(OmpDeviceTypeClause, Type) // OMP DEVICE_TYPE
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 455322d610d6..599cc61a83bf 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -3288,15 +3288,21 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Depend &x) {
         parser::ToUpperCaseLetters(getDirectiveName(GetContext().directive)));
   }
   if (const auto *inOut{std::get_if<parser::OmpDependClause::InOut>(&x.v.u)}) {
-    const auto &designators{std::get<std::list<parser::Designator>>(inOut->t)};
-    for (const auto &ele : designators) {
-      if (const auto *dataRef{std::get_if<parser::DataRef>(&ele.u)}) {
-        CheckDependList(*dataRef);
-        if (const auto *arr{
-                std::get_if<common::Indirection<parser::ArrayElement>>(
-                    &dataRef->u)}) {
-          CheckArraySection(arr->value(), GetLastName(*dataRef),
-              llvm::omp::Clause::OMPC_depend);
+    for (const auto &object : std::get<parser::OmpObjectList>(inOut->t).v) {
+      if (const auto *name{std::get_if<parser::Name>(&object.u)}) {
+        context_.Say(GetContext().clauseSource,
+            "Common block name ('%s') cannot appear in a DEPEND "
+            "clause"_err_en_US,
+            name->ToString());
+      } else if (auto *designator{std::get_if<parser::Designator>(&object.u)}) {
+        if (auto *dataRef{std::get_if<parser::DataRef>(&designator->u)}) {
+          CheckDependList(*dataRef);
+          if (const auto *arr{
+                  std::get_if<common::Indirection<parser::ArrayElement>>(
+                      &dataRef->u)}) {
+            CheckArraySection(arr->value(), GetLastName(*dataRef),
+                llvm::omp::Clause::OMPC_depend);
+          }
         }
       }
     }
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 979570a7d410..014b7987a658 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -435,6 +435,20 @@ public:
   bool Pre(const parser::OpenMPAllocatorsConstruct &);
   void Post(const parser::OpenMPAllocatorsConstruct &);
 
+  void Post(const parser::OmpObjectList &x) {
+    // The objects from OMP clauses should have already been resolved,
+    // except common blocks (the ResolveNamesVisitor does not visit
+    // parser::Name, those are dealt with as members of other structures).
+    // Iterate over elements of x, and resolve any common blocks that
+    // are still unresolved.
+    for (const parser::OmpObject &obj : x.v) {
+      auto *name{std::get_if<parser::Name>(&obj.u)};
+      if (name && !name->symbol) {
+        Resolve(*name, currScope().MakeCommonBlock(name->source));
+      }
+    }
+  }
+
   // 2.15.3 Data-Sharing Attribute Clauses
   void Post(const parser::OmpDefaultClause &);
   bool Pre(const parser::OmpClause::Shared &x) {
@@ -531,16 +545,9 @@ public:
     return false;
   }
 
-  bool Pre(const parser::OmpDependClause &x) {
-    if (const auto *dependSink{
-            std::get_if<parser::OmpDependClause::Sink>(&x.u)}) {
-      const auto &dependSinkVec{dependSink->v};
-      for (const auto &dependSinkElement : dependSinkVec) {
-        const auto &name{std::get<parser::Name>(dependSinkElement.t)};
-        ResolveName(&name);
-      }
-    }
-    return false;
+  void Post(const parser::OmpDependSinkVec &x) {
+    const auto &name{std::get<parser::Name>(x.t)};
+    ResolveName(&name);
   }
 
   bool Pre(const parser::OmpClause::UseDevicePtr &x) {
diff --git a/flang/test/Semantics/OpenMP/depend04.f90 b/flang/test/Semantics/OpenMP/depend04.f90
new file mode 100644
index 000000000000..8bdddb017d2c
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/depend04.f90
@@ -0,0 +1,10 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=50
+
+subroutine f00
+  integer :: x
+  common /cc/ x
+!ERROR: Common block name ('cc') cannot appear in a DEPEND clause
+  !$omp task depend(in: /cc/)
+  x = 0
+  !$omp end task
+end
diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
index ac34ddafc5e7..2a890905dc63 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -503,7 +503,7 @@ struct DependT {
   using LocatorList = ObjectListT<I, E>;
   using TaskDependenceType = tomp::type::TaskDependenceType;
 
-  struct WithLocators { // Modern form
+  struct DepType { // The form with task dependence type.
     using TupleTrait = std::true_type;
     // Empty LocatorList means "omp_all_memory".
     std::tuple<TaskDependenceType, OPT(Iterator), LocatorList> t;
@@ -511,7 +511,7 @@ struct DependT {
 
   using Doacross = DoacrossT<T, I, E>;
   using UnionTrait = std::true_type;
-  std::variant<Doacross, WithLocators> u; // Doacross form is legacy
+  std::variant<Doacross, DepType> u; // Doacross form is legacy
 };
 
 // V5.2: [3.5] `destroy` clause
-- 
GitLab


From c5edecbb4bfe08997819ff84712e3e22ddd04490 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 28 Oct 2024 21:36:48 +0000
Subject: [PATCH 370/693] [X86] Regenerate scmp/ucmp test checks with vpternlog
 comments

---
 llvm/test/CodeGen/X86/scmp.ll | 2 +-
 llvm/test/CodeGen/X86/ucmp.ll | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/test/CodeGen/X86/scmp.ll b/llvm/test/CodeGen/X86/scmp.ll
index 0746a07d2cdf..5ae5caf3e88b 100644
--- a/llvm/test/CodeGen/X86/scmp.ll
+++ b/llvm/test/CodeGen/X86/scmp.ll
@@ -848,7 +848,7 @@ define <16 x i32> @scmp_wide_vec_result(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; AVX512-NEXT:    vpcmpgtb %xmm0, %xmm1, %k1
 ; AVX512-NEXT:    vpcmpgtb %xmm1, %xmm0, %k2
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm0 {%k2} {z} = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = -1
 ; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
 ; AVX512-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/ucmp.ll b/llvm/test/CodeGen/X86/ucmp.ll
index cd643cb8d637..6a52acfe2fb3 100644
--- a/llvm/test/CodeGen/X86/ucmp.ll
+++ b/llvm/test/CodeGen/X86/ucmp.ll
@@ -819,7 +819,7 @@ define <16 x i32> @ucmp_wide_vec_result(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; AVX512-NEXT:    vpcmpltub %xmm1, %xmm0, %k1
 ; AVX512-NEXT:    vpcmpnleub %xmm1, %xmm0, %k2
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm0 {%k2} {z} = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = -1
 ; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
 ; AVX512-NEXT:    retq
 ;
-- 
GitLab


From 7b663bd9179a205d5a65a34e447fbeffcb43c194 Mon Sep 17 00:00:00 2001
From: Job Henandez Lara <jobhdezlara93@gmail.com>
Date: Mon, 28 Oct 2024 14:49:36 -0700
Subject: [PATCH 371/693] [libc] Fix the remaining fcntl.h proxy header
 includes. (#113961)

---
 libc/src/stdio/linux/CMakeLists.txt               | 1 +
 libc/src/stdio/linux/rename.cpp                   | 2 +-
 libc/src/sys/mman/linux/CMakeLists.txt            | 3 +--
 libc/src/sys/mman/linux/shm_open.cpp              | 2 +-
 libc/src/sys/mman/shm_open.h                      | 2 +-
 libc/src/sys/stat/linux/CMakeLists.txt            | 2 ++
 libc/src/sys/stat/linux/chmod.cpp                 | 1 +
 libc/src/sys/stat/linux/mkdir.cpp                 | 1 +
 libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp | 2 +-
 9 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/libc/src/stdio/linux/CMakeLists.txt b/libc/src/stdio/linux/CMakeLists.txt
index e81642dc6f01..1b2fcb33ce54 100644
--- a/libc/src/stdio/linux/CMakeLists.txt
+++ b/libc/src/stdio/linux/CMakeLists.txt
@@ -22,6 +22,7 @@ add_entrypoint_object(
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
     libc.src.errno.errno
+    libc.hdr.fcntl_macros
 )
 
 add_entrypoint_object(
diff --git a/libc/src/stdio/linux/rename.cpp b/libc/src/stdio/linux/rename.cpp
index 69fd22720ed1..fbcb29be48f4 100644
--- a/libc/src/stdio/linux/rename.cpp
+++ b/libc/src/stdio/linux/rename.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/rename.h"
-#include "include/llvm-libc-macros/linux/fcntl-macros.h"
+#include "hdr/fcntl_macros.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
diff --git a/libc/src/sys/mman/linux/CMakeLists.txt b/libc/src/sys/mman/linux/CMakeLists.txt
index 11188254cfbd..47c16f79bc8d 100644
--- a/libc/src/sys/mman/linux/CMakeLists.txt
+++ b/libc/src/sys/mman/linux/CMakeLists.txt
@@ -187,8 +187,7 @@ add_entrypoint_object(
     ../shm_open.h
   DEPENDS
     libc.src.fcntl.open
-    libc.include.llvm-libc-macros.fcntl_macros
-    libc.include.llvm-libc-types.mode_t
+    libc.hdr.types.mode_t
     .shm_common
 )
 
diff --git a/libc/src/sys/mman/linux/shm_open.cpp b/libc/src/sys/mman/linux/shm_open.cpp
index d235e57aefde..11de482272d0 100644
--- a/libc/src/sys/mman/linux/shm_open.cpp
+++ b/libc/src/sys/mman/linux/shm_open.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/sys/mman/shm_open.h"
-#include "llvm-libc-macros/fcntl-macros.h"
+#include "hdr/types/mode_t.h"
 #include "src/__support/macros/config.h"
 #include "src/fcntl/open.h"
 #include "src/sys/mman/linux/shm_common.h"
diff --git a/libc/src/sys/mman/shm_open.h b/libc/src/sys/mman/shm_open.h
index c890304aa4ac..1872dd30cb6f 100644
--- a/libc/src/sys/mman/shm_open.h
+++ b/libc/src/sys/mman/shm_open.h
@@ -9,8 +9,8 @@
 #ifndef LLVM_LIBC_SRC_SYS_MMAN_SHM_OPEN_H
 #define LLVM_LIBC_SRC_SYS_MMAN_SHM_OPEN_H
 
+#include "hdr/types/mode_t.h"
 #include "src/__support/macros/config.h"
-#include <llvm-libc-types/mode_t.h>
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/sys/stat/linux/CMakeLists.txt b/libc/src/sys/stat/linux/CMakeLists.txt
index 7c9496b6b6e8..9aeb14636c2c 100644
--- a/libc/src/sys/stat/linux/CMakeLists.txt
+++ b/libc/src/sys/stat/linux/CMakeLists.txt
@@ -6,6 +6,7 @@ add_entrypoint_object(
     ../chmod.h
   DEPENDS
     libc.hdr.types.mode_t
+    libc.hdr.fcntl_macros
     libc.include.sys_stat
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
@@ -47,6 +48,7 @@ add_entrypoint_object(
     ../mkdir.h
   DEPENDS
     libc.hdr.types.mode_t
+    libc.hdr.fcntl_macros
     libc.include.sys_stat
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
diff --git a/libc/src/sys/stat/linux/chmod.cpp b/libc/src/sys/stat/linux/chmod.cpp
index 9d2860391818..57d5bae6b819 100644
--- a/libc/src/sys/stat/linux/chmod.cpp
+++ b/libc/src/sys/stat/linux/chmod.cpp
@@ -11,6 +11,7 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "hdr/fcntl_macros.h"
 #include "hdr/types/mode_t.h"
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
diff --git a/libc/src/sys/stat/linux/mkdir.cpp b/libc/src/sys/stat/linux/mkdir.cpp
index bd6efef858c7..b319b5c8393d 100644
--- a/libc/src/sys/stat/linux/mkdir.cpp
+++ b/libc/src/sys/stat/linux/mkdir.cpp
@@ -11,6 +11,7 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "hdr/fcntl_macros.h"
 #include "hdr/types/mode_t.h"
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
diff --git a/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp b/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp
index 8cb5f867453e..2f3e0b96ff09 100644
--- a/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp
+++ b/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp
@@ -1,4 +1,4 @@
-#include "llvm-libc-macros/linux/fcntl-macros.h"
+#include "hdr/fcntl_macros.h"
 #include "src/__support/macros/config.h"
 #include "src/fcntl/open.h"
 #include "src/sys/statvfs/fstatvfs.h"
-- 
GitLab


From 82cb22e735be24acf5aac594e519935fc43f0aec Mon Sep 17 00:00:00 2001
From: Thomas Preud'homme <thomas.preudhomme@arm.com>
Date: Mon, 28 Oct 2024 21:50:23 +0000
Subject: [PATCH 372/693] [MLIR] Add missing MLIRLLVMDialect dep to
 MLIRMathToLibm (#113563)

This fixes the following failure when doing a clean build (in particular
no .ninja* lying around) of lib/libMLIRMathToLibm.a only:
```
In file included from llvm/include/llvm/IR/Module.h:22,
                 from mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h:37,
                 from mlir/lib/Conversion/MathToLibm/MathToLibm.cpp:13
llvm/include/llvm/IR/Attributes.h:90:14: fatal error: llvm/IR/Attributes.inc: No such file or directory
```
---
 mlir/lib/Conversion/MathToLibm/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mlir/lib/Conversion/MathToLibm/CMakeLists.txt b/mlir/lib/Conversion/MathToLibm/CMakeLists.txt
index 61c46e9bfe25..0a4eb97474f3 100644
--- a/mlir/lib/Conversion/MathToLibm/CMakeLists.txt
+++ b/mlir/lib/Conversion/MathToLibm/CMakeLists.txt
@@ -14,6 +14,7 @@ add_mlir_conversion_library(MLIRMathToLibm
   MLIRArithDialect
   MLIRDialectUtils
   MLIRFuncDialect
+  MLIRLLVMDialect
   MLIRMathDialect
   MLIRPass
   MLIRTransformUtils
-- 
GitLab


From 7db4cacfd72b64de8460509d07a22b9142df85f6 Mon Sep 17 00:00:00 2001
From: Thomas Preud'homme <thomas.preudhomme@arm.com>
Date: Mon, 28 Oct 2024 21:53:39 +0000
Subject: [PATCH 373/693] [MLIR] Add missing MLIRLLVMDialect dep to
 MLIRLinalgToStandard (#113561)

This fixes the following failure when doing a clean build (in particular
no .ninja* lying around) of lib/libMLIRLinalgToStandard.a only:
```
In file included from llvm/include/llvm/IR/Module.h:22,
                 from mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h:37,
                 from mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp:13:
llvm/include/llvm/IR/Attributes.h:90:14: fatal error: llvm/IR/Attributes.inc: No such file or directory
```
---
 mlir/lib/Conversion/LinalgToStandard/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mlir/lib/Conversion/LinalgToStandard/CMakeLists.txt b/mlir/lib/Conversion/LinalgToStandard/CMakeLists.txt
index cbe85789b29a..7fc4af540318 100644
--- a/mlir/lib/Conversion/LinalgToStandard/CMakeLists.txt
+++ b/mlir/lib/Conversion/LinalgToStandard/CMakeLists.txt
@@ -15,6 +15,7 @@ add_mlir_conversion_library(MLIRLinalgToStandard
   MLIRIR
   MLIRLinalgDialect
   MLIRLinalgTransforms
+  MLIRLLVMDialect
   MLIRMemRefDialect
   MLIRPass
   MLIRSCFDialect
-- 
GitLab


From 474234a09655e57b7a4270150f0926db77e864b4 Mon Sep 17 00:00:00 2001
From: Felipe de Azevedo Piovezan <fpiovezan@apple.com>
Date: Mon, 28 Oct 2024 19:02:07 -0300
Subject: [PATCH 374/693] [debugserver] Mark ASAN memory regions as "heap"
 (#113968)

This memory type is currently not handled, but it makes sense to mark it
as a heap allocation in requests asking for memory region info.
---
 lldb/tools/debugserver/source/MacOSX/MachVMRegion.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lldb/tools/debugserver/source/MacOSX/MachVMRegion.cpp b/lldb/tools/debugserver/source/MacOSX/MachVMRegion.cpp
index 60d4c3bc293a..97908b4acaf2 100644
--- a/lldb/tools/debugserver/source/MacOSX/MachVMRegion.cpp
+++ b/lldb/tools/debugserver/source/MacOSX/MachVMRegion.cpp
@@ -208,7 +208,8 @@ std::vector<std::string> MachVMRegion::GetMemoryTypes() const {
       m_data.user_tag == VM_MEMORY_MALLOC_LARGE_REUSABLE ||
       m_data.user_tag == VM_MEMORY_MALLOC_HUGE ||
       m_data.user_tag == VM_MEMORY_REALLOC ||
-      m_data.user_tag == VM_MEMORY_SBRK) {
+      m_data.user_tag == VM_MEMORY_SBRK ||
+      m_data.user_tag == VM_MEMORY_SANITIZER) {
     types.push_back("heap");
     if (m_data.user_tag == VM_MEMORY_MALLOC_TINY) {
       types.push_back("malloc-tiny");
-- 
GitLab


From 9f69da35e2e5438d0c042f76277fff397f6a1505 Mon Sep 17 00:00:00 2001
From: Prabhuk <prabhukr@google.com>
Date: Mon, 28 Oct 2024 15:24:47 -0700
Subject: [PATCH 375/693] [NFC][compiler-rt] Add missing header include
 (#113951)

Include `cstdlib` which was originally included transitively but the
changes to `vector` in libcpp breaks new builds due to missing cstdlib
header for `abort()` function call.
---
 compiler-rt/include/fuzzer/FuzzedDataProvider.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/compiler-rt/include/fuzzer/FuzzedDataProvider.h b/compiler-rt/include/fuzzer/FuzzedDataProvider.h
index 5903ed837917..e57b95b6304a 100644
--- a/compiler-rt/include/fuzzer/FuzzedDataProvider.h
+++ b/compiler-rt/include/fuzzer/FuzzedDataProvider.h
@@ -18,6 +18,7 @@
 #include <climits>
 #include <cstddef>
 #include <cstdint>
+#include <cstdlib>
 #include <cstring>
 #include <initializer_list>
 #include <limits>
-- 
GitLab


From bf4b31ad54f6bf298210c201a0afe50469ebe98e Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Mon, 28 Oct 2024 15:32:20 -0700
Subject: [PATCH 376/693] [SandboxVec][Legality] Check Fastmath flags (#113967)

---
 .../Vectorize/SandboxVectorizer/Legality.h       |  3 +++
 .../Vectorize/SandboxVectorizer/Legality.cpp     | 12 ++++++++++++
 .../Vectorize/SandboxVectorizer/LegalityTest.cpp | 16 +++++++++++++++-
 3 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
index d4b0b54375b0..49dcec26dbc5 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
@@ -31,6 +31,7 @@ enum class ResultReason {
   NotInstructions,
   DiffOpcodes,
   DiffTypes,
+  DiffMathFlags,
 };
 
 #ifndef NDEBUG
@@ -53,6 +54,8 @@ struct ToStr {
       return "DiffOpcodes";
     case ResultReason::DiffTypes:
       return "DiffTypes";
+    case ResultReason::DiffMathFlags:
+      return "DiffMathFlags";
     }
     llvm_unreachable("Unknown ResultReason enum");
   }
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
index fcfb11c669fa..346d8a90589f 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
 #include "llvm/SandboxIR/Instruction.h"
+#include "llvm/SandboxIR/Operator.h"
 #include "llvm/SandboxIR/Utils.h"
 #include "llvm/SandboxIR/Value.h"
 #include "llvm/Support/Debug.h"
@@ -43,6 +44,17 @@ LegalityAnalysis::notVectorizableBasedOnOpcodesAndTypes(
       }))
     return ResultReason::DiffTypes;
 
+  // TODO: Allow vectorization of instrs with different flags as long as we
+  // change them to the least common one.
+  // For now pack if differnt FastMathFlags.
+  if (isa<FPMathOperator>(I0)) {
+    FastMathFlags FMF0 = cast<Instruction>(Bndl[0])->getFastMathFlags();
+    if (any_of(drop_begin(Bndl), [FMF0](auto *V) {
+          return cast<Instruction>(V)->getFastMathFlags() != FMF0;
+        }))
+      return ResultReason::DiffMathFlags;
+  }
+
   // TODO: Missing checks
 
   return std::nullopt;
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
index 51f445c8d1d0..aaa8e96de6d1 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
@@ -29,7 +29,7 @@ struct LegalityTest : public testing::Test {
 
 TEST_F(LegalityTest, Legality) {
   parseIR(C, R"IR(
-define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg) {
+define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg, float %farg0, float %farg1) {
   %gep0 = getelementptr float, ptr %ptr, i32 0
   %gep1 = getelementptr float, ptr %ptr, i32 1
   %gep3 = getelementptr float, ptr %ptr, i32 3
@@ -40,6 +40,8 @@ define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg) {
   store <2 x float> %vec2, ptr %gep1
   store <3 x float> %vec3, ptr %gep3
   store i8 %arg, ptr %gep1
+  %fadd0 = fadd float %farg0, %farg0
+  %fadd1 = fadd fast float %farg1, %farg1
   ret void
 }
 )IR");
@@ -58,6 +60,8 @@ define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg) {
   auto *StVec2 = cast<sandboxir::StoreInst>(&*It++);
   auto *StVec3 = cast<sandboxir::StoreInst>(&*It++);
   auto *StI8 = cast<sandboxir::StoreInst>(&*It++);
+  auto *FAdd0 = cast<sandboxir::BinaryOperator>(&*It++);
+  auto *FAdd1 = cast<sandboxir::BinaryOperator>(&*It++);
 
   sandboxir::LegalityAnalysis Legality;
   const auto &Result = Legality.canVectorize({St0, St1});
@@ -87,6 +91,13 @@ define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg) {
     EXPECT_EQ(cast<sandboxir::Pack>(Result).getReason(),
               sandboxir::ResultReason::DiffTypes);
   }
+  {
+    // Check DiffMathFlags
+    const auto &Result = Legality.canVectorize({FAdd0, FAdd1});
+    EXPECT_TRUE(isa<sandboxir::Pack>(Result));
+    EXPECT_EQ(cast<sandboxir::Pack>(Result).getReason(),
+              sandboxir::ResultReason::DiffMathFlags);
+  }
 }
 
 #ifndef NDEBUG
@@ -110,5 +121,8 @@ TEST_F(LegalityTest, LegalityResultDump) {
   EXPECT_TRUE(Matches(Legality.createLegalityResult<sandboxir::Pack>(
                           sandboxir::ResultReason::DiffTypes),
                       "Pack Reason: DiffTypes"));
+  EXPECT_TRUE(Matches(Legality.createLegalityResult<sandboxir::Pack>(
+                          sandboxir::ResultReason::DiffMathFlags),
+                      "Pack Reason: DiffMathFlags"));
 }
 #endif // NDEBUG
-- 
GitLab


From e873b415a2bfbbc2e5e2e70d77b61b7883cbf949 Mon Sep 17 00:00:00 2001
From: Job Henandez Lara <jobhdezlara93@gmail.com>
Date: Mon, 28 Oct 2024 15:32:45 -0700
Subject: [PATCH 377/693] [libc] add the rest of the hdr/fcntl_macro.h headers
 (#113972)

---
 libc/src/fcntl/linux/CMakeLists.txt  | 1 +
 libc/src/fcntl/linux/open.cpp        | 1 +
 libc/src/spawn/linux/CMakeLists.txt  | 1 +
 libc/src/spawn/linux/posix_spawn.cpp | 1 +
 4 files changed, 4 insertions(+)

diff --git a/libc/src/fcntl/linux/CMakeLists.txt b/libc/src/fcntl/linux/CMakeLists.txt
index ecfb2cdd3f33..580db16cd413 100644
--- a/libc/src/fcntl/linux/CMakeLists.txt
+++ b/libc/src/fcntl/linux/CMakeLists.txt
@@ -29,6 +29,7 @@ add_entrypoint_object(
     ../open.h
   DEPENDS
     libc.hdr.types.mode_t
+    libc.hdr.fcntl_macros
     libc.src.__support.OSUtil.osutil
     libc.src.errno.errno
 )
diff --git a/libc/src/fcntl/linux/open.cpp b/libc/src/fcntl/linux/open.cpp
index 9f3d035388e8..8b699ecdd204 100644
--- a/libc/src/fcntl/linux/open.cpp
+++ b/libc/src/fcntl/linux/open.cpp
@@ -13,6 +13,7 @@
 #include "src/__support/macros/config.h"
 #include "src/errno/libc_errno.h"
 
+#include "hdr/fcntl_macros.h"
 #include "hdr/types/mode_t.h"
 #include <stdarg.h>
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/libc/src/spawn/linux/CMakeLists.txt b/libc/src/spawn/linux/CMakeLists.txt
index 5f7ab4c43165..26148fe1c76d 100644
--- a/libc/src/spawn/linux/CMakeLists.txt
+++ b/libc/src/spawn/linux/CMakeLists.txt
@@ -6,6 +6,7 @@ add_entrypoint_object(
     ../posix_spawn.h
   DEPENDS
     libc.hdr.types.mode_t
+    libc.hdr.fcntl_macros
     libc.include.spawn
     libc.include.sys_syscall
     libc.include.signal
diff --git a/libc/src/spawn/linux/posix_spawn.cpp b/libc/src/spawn/linux/posix_spawn.cpp
index d6caf8b374a0..fe82ba260148 100644
--- a/libc/src/spawn/linux/posix_spawn.cpp
+++ b/libc/src/spawn/linux/posix_spawn.cpp
@@ -14,6 +14,7 @@
 #include "src/__support/macros/config.h"
 #include "src/spawn/file_actions.h"
 
+#include "hdr/fcntl_macros.h"
 #include "hdr/types/mode_t.h"
 #include <signal.h> // For SIGCHLD
 #include <spawn.h>
-- 
GitLab


From 0eb5c9d2ef8d932eef84d4db8aef3dd512f80277 Mon Sep 17 00:00:00 2001
From: Renaud Kauffmann <rkauffmann@nvidia.com>
Date: Mon, 28 Oct 2024 15:34:27 -0700
Subject: [PATCH 378/693] [flang][cuda] Copying device globals in the gpu
 module (#113955)

---
 .../Optimizer/Transforms/CUFDeviceGlobal.cpp  | 27 +++++++++++++++++++
 flang/test/Fir/CUDA/cuda-device-global.f90    | 13 +++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 flang/test/Fir/CUDA/cuda-device-global.f90

diff --git a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
index a4761f24f16d..dc39be8574f8 100644
--- a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
@@ -11,6 +11,7 @@
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
+#include "flang/Optimizer/Transforms/CUFCommon.h"
 #include "flang/Runtime/CUDA/common.h"
 #include "flang/Runtime/allocatable.h"
 #include "mlir/IR/SymbolTable.h"
@@ -58,6 +59,32 @@ public:
       prepareImplicitDeviceGlobals(funcOp, symTable);
       return mlir::WalkResult::advance();
     });
+
+    // Copying the device global variable into the gpu module
+    mlir::SymbolTable parentSymTable(mod);
+    auto gpuMod =
+        parentSymTable.lookup<mlir::gpu::GPUModuleOp>(cudaDeviceModuleName);
+    if (gpuMod) {
+      mlir::SymbolTable gpuSymTable(gpuMod);
+      for (auto globalOp : mod.getOps<fir::GlobalOp>()) {
+        auto attr = globalOp.getDataAttrAttr();
+        if (!attr)
+          continue;
+        switch (attr.getValue()) {
+        case cuf::DataAttribute::Device:
+        case cuf::DataAttribute::Constant:
+        case cuf::DataAttribute::Managed: {
+          auto globalName{globalOp.getSymbol().getValue()};
+          if (gpuSymTable.lookup<fir::GlobalOp>(globalName)) {
+            break;
+          }
+          gpuSymTable.insert(globalOp->clone());
+        } break;
+        default:
+          break;
+        }
+      }
+    }
   }
 };
 } // namespace
diff --git a/flang/test/Fir/CUDA/cuda-device-global.f90 b/flang/test/Fir/CUDA/cuda-device-global.f90
new file mode 100644
index 000000000000..c83a938d5af2
--- /dev/null
+++ b/flang/test/Fir/CUDA/cuda-device-global.f90
@@ -0,0 +1,13 @@
+
+// RUN: fir-opt --split-input-file --cuf-device-global %s | FileCheck %s
+
+
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module} {
+  fir.global @_QMmtestsEn(dense<[3, 4, 5, 6, 7]> : tensor<5xi32>) {data_attr = #cuf.cuda<device>} : !fir.array<5xi32>
+
+  gpu.module @cuda_device_mod [#nvvm.target] {
+  }
+}
+
+// CHECK: gpu.module @cuda_device_mod [#nvvm.target] 
+// CHECK-NEXT: fir.global @_QMmtestsEn(dense<[3, 4, 5, 6, 7]> : tensor<5xi32>) {data_attr = #cuf.cuda<device>} : !fir.array<5xi32>
-- 
GitLab


From 36c119490630846c1fa0f427cc60837fd7b40a28 Mon Sep 17 00:00:00 2001
From: Matthias Braun <matze@braunis.de>
Date: Mon, 28 Oct 2024 15:34:56 -0700
Subject: [PATCH 379/693] Remove optimization flags from clang codegen tests
 (#113714)

- Remove an -O3 flag from a couple of clang x86 codegen tests so the
  tests do not need to be updated when optimizations in LLVM change.
- Change the tests to use utils/update_cc_test_checks.sh
- Change from apple/darwin triples to generic x86_64-- and
  i386-- because it was not relevant to the test but
  `update_cc_test_checks` seems to be unable to handle platforms that
  prepend `_` to function names.
---
 clang/test/CodeGen/X86/avx-cmp-builtins.c     | 104 ++++--
 clang/test/CodeGen/X86/avx-shuffle-builtins.c | 302 +++++++++++++-----
 clang/test/CodeGen/X86/sse.c                  |  58 +++-
 3 files changed, 348 insertions(+), 116 deletions(-)

diff --git a/clang/test/CodeGen/X86/avx-cmp-builtins.c b/clang/test/CodeGen/X86/avx-cmp-builtins.c
index c4e3c7ccd549..2e4a383a6b3f 100644
--- a/clang/test/CodeGen/X86/avx-cmp-builtins.c
+++ b/clang/test/CodeGen/X86/avx-cmp-builtins.c
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -O3 -triple=x86_64-apple-darwin -target-feature +avx -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -O3 -triple=i386-apple-darwin -target-feature +avx -emit-llvm -o - | FileCheck %s
-// FIXME: The shufflevector instructions in test_cmpgt_sd are relying on O3 here.
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-- -target-feature +avx -disable-O0-optnone -emit-llvm -o - | opt -S -passes=mem2reg | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-- -target-feature +avx -disable-O0-optnone -emit-llvm -o - | opt -S -passes=mem2reg | FileCheck %s
 
 
 #include <immintrin.h>
@@ -9,62 +9,124 @@
 // Test LLVM IR codegen of cmpXY instructions
 //
 
+// CHECK-LABEL: define dso_local <2 x double> @test_cmp_sd(
+// CHECK-SAME: <2 x double> noundef [[A:%.*]], <2 x double> noundef [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> [[A]], <2 x double> [[B]], i8 13)
+// CHECK-NEXT:    ret <2 x double> [[TMP0]]
+//
 __m128d test_cmp_sd(__m128d a, __m128d b) {
   // Expects that the third argument in LLVM IR is immediate expression
-  // CHECK: @llvm.x86.sse2.cmp.sd({{.*}}, i8 13)
   return _mm_cmp_sd(a, b, _CMP_GE_OS);
 }
 
+// CHECK-LABEL: define dso_local <4 x float> @test_cmp_ss(
+// CHECK-SAME: <4 x float> noundef [[A:%.*]], <4 x float> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> [[A]], <4 x float> [[B]], i8 13)
+// CHECK-NEXT:    ret <4 x float> [[TMP0]]
+//
 __m128 test_cmp_ss(__m128 a, __m128 b) {
   // Expects that the third argument in LLVM IR is immediate expression
-  // CHECK: @llvm.x86.sse.cmp.ss({{.*}}, i8 13)
   return _mm_cmp_ss(a, b, _CMP_GE_OS);
 }
 
+// CHECK-LABEL: define dso_local <4 x float> @test_cmpgt_ss(
+// CHECK-SAME: <4 x float> noundef [[A:%.*]], <4 x float> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> [[B]], <4 x float> [[A]], i8 1)
+// CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <4 x float> [[A]], <4 x float> [[TMP0]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    ret <4 x float> [[SHUFFLE_I]]
+//
 __m128 test_cmpgt_ss(__m128 a, __m128 b) {
-  // CHECK: @llvm.x86.sse.cmp.ss({{.*}}, i8 1)
-  // CHECK: shufflevector <{{.*}}, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   return _mm_cmpgt_ss(a, b);
 }
 
+// CHECK-LABEL: define dso_local <4 x float> @test_cmpge_ss(
+// CHECK-SAME: <4 x float> noundef [[A:%.*]], <4 x float> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> [[B]], <4 x float> [[A]], i8 2)
+// CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <4 x float> [[A]], <4 x float> [[TMP0]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    ret <4 x float> [[SHUFFLE_I]]
+//
 __m128 test_cmpge_ss(__m128 a, __m128 b) {
-  // CHECK: @llvm.x86.sse.cmp.ss({{.*}}, i8 2)
-  // CHECK: shufflevector <{{.*}}, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   return _mm_cmpge_ss(a, b);
 }
 
+// CHECK-LABEL: define dso_local <4 x float> @test_cmpngt_ss(
+// CHECK-SAME: <4 x float> noundef [[A:%.*]], <4 x float> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> [[B]], <4 x float> [[A]], i8 5)
+// CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <4 x float> [[A]], <4 x float> [[TMP0]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    ret <4 x float> [[SHUFFLE_I]]
+//
 __m128 test_cmpngt_ss(__m128 a, __m128 b) {
-  // CHECK: @llvm.x86.sse.cmp.ss({{.*}}, i8 5)
-  // CHECK: shufflevector <{{.*}}, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   return _mm_cmpngt_ss(a, b);
 }
 
+// CHECK-LABEL: define dso_local <4 x float> @test_cmpnge_ss(
+// CHECK-SAME: <4 x float> noundef [[A:%.*]], <4 x float> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> [[B]], <4 x float> [[A]], i8 6)
+// CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <4 x float> [[A]], <4 x float> [[TMP0]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    ret <4 x float> [[SHUFFLE_I]]
+//
 __m128 test_cmpnge_ss(__m128 a, __m128 b) {
-  // CHECK: @llvm.x86.sse.cmp.ss({{.*}}, i8 6)
-  // CHECK: shufflevector <{{.*}}, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   return _mm_cmpnge_ss(a, b);
 }
 
+// CHECK-LABEL: define dso_local <2 x double> @test_cmpgt_sd(
+// CHECK-SAME: <2 x double> noundef [[A:%.*]], <2 x double> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> [[B]], <2 x double> [[A]], i8 1)
+// CHECK-NEXT:    [[VECEXT_I:%.*]] = extractelement <2 x double> [[TMP0]], i32 0
+// CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <2 x double> poison, double [[VECEXT_I]], i32 0
+// CHECK-NEXT:    [[VECEXT1_I:%.*]] = extractelement <2 x double> [[A]], i32 1
+// CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <2 x double> [[VECINIT_I]], double [[VECEXT1_I]], i32 1
+// CHECK-NEXT:    ret <2 x double> [[VECINIT2_I]]
+//
 __m128d test_cmpgt_sd(__m128d a, __m128d b) {
-  // CHECK: @llvm.x86.sse2.cmp.sd({{.*}}, i8 1)
-  // CHECK: shufflevector <{{.*}}, <2 x i32> <i32 0, i32 3>
   return _mm_cmpgt_sd(a, b);
 }
 
+// CHECK-LABEL: define dso_local <2 x double> @test_cmpge_sd(
+// CHECK-SAME: <2 x double> noundef [[A:%.*]], <2 x double> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> [[B]], <2 x double> [[A]], i8 2)
+// CHECK-NEXT:    [[VECEXT_I:%.*]] = extractelement <2 x double> [[TMP0]], i32 0
+// CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <2 x double> poison, double [[VECEXT_I]], i32 0
+// CHECK-NEXT:    [[VECEXT1_I:%.*]] = extractelement <2 x double> [[A]], i32 1
+// CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <2 x double> [[VECINIT_I]], double [[VECEXT1_I]], i32 1
+// CHECK-NEXT:    ret <2 x double> [[VECINIT2_I]]
+//
 __m128d test_cmpge_sd(__m128d a, __m128d b) {
-  // CHECK: @llvm.x86.sse2.cmp.sd({{.*}}, i8 2)
-  // CHECK: shufflevector <{{.*}}, <2 x i32> <i32 0, i32 3>
   return _mm_cmpge_sd(a, b);
 }
 
+// CHECK-LABEL: define dso_local <2 x double> @test_cmpngt_sd(
+// CHECK-SAME: <2 x double> noundef [[A:%.*]], <2 x double> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> [[B]], <2 x double> [[A]], i8 5)
+// CHECK-NEXT:    [[VECEXT_I:%.*]] = extractelement <2 x double> [[TMP0]], i32 0
+// CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <2 x double> poison, double [[VECEXT_I]], i32 0
+// CHECK-NEXT:    [[VECEXT1_I:%.*]] = extractelement <2 x double> [[A]], i32 1
+// CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <2 x double> [[VECINIT_I]], double [[VECEXT1_I]], i32 1
+// CHECK-NEXT:    ret <2 x double> [[VECINIT2_I]]
+//
 __m128d test_cmpngt_sd(__m128d a, __m128d b) {
-  // CHECK: @llvm.x86.sse2.cmp.sd({{.*}}, i8 5)
-  // CHECK: shufflevector <{{.*}}, <2 x i32> <i32 0, i32 3>
   return _mm_cmpngt_sd(a, b);
 }
 
+// CHECK-LABEL: define dso_local <2 x double> @test_cmpnge_sd(
+// CHECK-SAME: <2 x double> noundef [[A:%.*]], <2 x double> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> [[B]], <2 x double> [[A]], i8 6)
+// CHECK-NEXT:    [[VECEXT_I:%.*]] = extractelement <2 x double> [[TMP0]], i32 0
+// CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <2 x double> poison, double [[VECEXT_I]], i32 0
+// CHECK-NEXT:    [[VECEXT1_I:%.*]] = extractelement <2 x double> [[A]], i32 1
+// CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <2 x double> [[VECINIT_I]], double [[VECEXT1_I]], i32 1
+// CHECK-NEXT:    ret <2 x double> [[VECINIT2_I]]
+//
 __m128d test_cmpnge_sd(__m128d a, __m128d b) {
-  // CHECK: @llvm.x86.sse2.cmp.sd({{.*}}, i8 6)
-  // CHECK: shufflevector <{{.*}}, <2 x i32> <i32 0, i32 3>
   return _mm_cmpnge_sd(a, b);
 }
diff --git a/clang/test/CodeGen/X86/avx-shuffle-builtins.c b/clang/test/CodeGen/X86/avx-shuffle-builtins.c
index d184d28f3e07..1c05fa436983 100644
--- a/clang/test/CodeGen/X86/avx-shuffle-builtins.c
+++ b/clang/test/CodeGen/X86/avx-shuffle-builtins.c
@@ -1,7 +1,7 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
 // REQUIRES: x86-registered-target
-// RUN: %clang_cc1 -ffreestanding %s -O3 -triple=x86_64-apple-darwin -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,X64
-// RUN: %clang_cc1 -ffreestanding %s -O3 -triple=i386-apple-darwin -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,X86
-// FIXME: This is testing optimized generation of shuffle instructions and should be fixed.
+// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-- -target-feature +avx -disable-O0-optnone -emit-llvm -o - | opt -S -passes=mem2reg | FileCheck %s
+// RUN: %clang_cc1 -ffreestanding %s -triple=i386-- -target-feature +avx -disable-O0-optnone -emit-llvm -o - | opt -S -passes=mem2reg | FileCheck %s
 
 
 #include <immintrin.h>
@@ -10,201 +10,341 @@
 // Test LLVM IR codegen of shuffle instructions, checking if the masks are correct
 //
 
+// CHECK-LABEL: define dso_local <8 x float> @x(
+// CHECK-SAME: <8 x float> noundef [[A:%.*]], <8 x float> noundef [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[SHUFP:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 3, i32 2, i32 8, i32 11, i32 7, i32 6, i32 12, i32 15>
+// CHECK-NEXT:    ret <8 x float> [[SHUFP]]
+//
 __m256 x(__m256 a, __m256 b) {
-  // CHECK-LABEL: x
-  // CHECK: shufflevector{{.*}}<i32 3, i32 2, i32 8, i32 11, i32 7, i32 6, i32 12, i32 15>
   return _mm256_shuffle_ps(a, b, 203);
 }
 
+// CHECK-LABEL: define dso_local <2 x double> @test_mm_permute_pd(
+// CHECK-SAME: <2 x double> noundef [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[PERMIL:%.*]] = shufflevector <2 x double> [[A]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+// CHECK-NEXT:    ret <2 x double> [[PERMIL]]
+//
 __m128d test_mm_permute_pd(__m128d a) {
-  // CHECK-LABEL: test_mm_permute_pd
-  // CHECK: shufflevector{{.*}}<i32 1, i32 0>
   return _mm_permute_pd(a, 1);
 }
 
+// CHECK-LABEL: define dso_local <4 x double> @test_mm256_permute_pd(
+// CHECK-SAME: <4 x double> noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[PERMIL:%.*]] = shufflevector <4 x double> [[A]], <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+// CHECK-NEXT:    ret <4 x double> [[PERMIL]]
+//
 __m256d test_mm256_permute_pd(__m256d a) {
-  // CHECK-LABEL: test_mm256_permute_pd
-  // CHECK: shufflevector{{.*}}<i32 1, i32 0, i32 3, i32 2>
   return _mm256_permute_pd(a, 5);
 }
 
+// CHECK-LABEL: define dso_local <4 x float> @test_mm_permute_ps(
+// CHECK-SAME: <4 x float> noundef [[A:%.*]]) #[[ATTR1]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[PERMIL:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-NEXT:    ret <4 x float> [[PERMIL]]
+//
 __m128 test_mm_permute_ps(__m128 a) {
-  // CHECK-LABEL: test_mm_permute_ps
-  // CHECK: shufflevector{{.*}}<i32 3, i32 2, i32 1, i32 0>
   return _mm_permute_ps(a, 0x1b);
 }
 
-// Test case for PR12401
+// CHECK-LABEL: define dso_local <4 x float> @test_mm_permute_ps2(
+// CHECK-SAME: <4 x float> noundef [[A:%.*]]) #[[ATTR1]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[PERMIL:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <4 x i32> <i32 2, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    ret <4 x float> [[PERMIL]]
+//
 __m128 test_mm_permute_ps2(__m128 a) {
-  // CHECK-LABEL: test_mm_permute_ps2
-  // CHECK: shufflevector{{.*}}<i32 2, i32 1, i32 2, i32 3>
   return _mm_permute_ps(a, 0xe6);
 }
 
+// CHECK-LABEL: define dso_local <8 x float> @test_mm256_permute_ps(
+// CHECK-SAME: <8 x float> noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[PERMIL:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+// CHECK-NEXT:    ret <8 x float> [[PERMIL]]
+//
 __m256 test_mm256_permute_ps(__m256 a) {
-  // CHECK-LABEL: test_mm256_permute_ps
-  // CHECK: shufflevector{{.*}}<i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
   return _mm256_permute_ps(a, 0x1b);
 }
 
+// CHECK-LABEL: define dso_local <4 x double> @test_mm256_permute2f128_pd(
+// CHECK-SAME: <4 x double> noundef [[A:%.*]], <4 x double> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VPERM:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+// CHECK-NEXT:    ret <4 x double> [[VPERM]]
+//
 __m256d test_mm256_permute2f128_pd(__m256d a, __m256d b) {
-  // CHECK-LABEL: test_mm256_permute2f128_pd
-  // CHECK: shufflevector{{.*}}<i32 2, i32 3, i32 6, i32 7> 
   return _mm256_permute2f128_pd(a, b, 0x31);
 }
 
+// CHECK-LABEL: define dso_local <8 x float> @test_mm256_permute2f128_ps(
+// CHECK-SAME: <8 x float> noundef [[A:%.*]], <8 x float> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VPERM:%.*]] = shufflevector <8 x float> [[B]], <8 x float> [[A]], <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
+// CHECK-NEXT:    ret <8 x float> [[VPERM]]
+//
 __m256 test_mm256_permute2f128_ps(__m256 a, __m256 b) {
-  // CHECK-LABEL: test_mm256_permute2f128_ps
-  // CHECK: shufflevector{{.*}}<i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
   return _mm256_permute2f128_ps(a, b, 0x13);
 }
 
+// CHECK-LABEL: define dso_local <4 x i64> @test_mm256_permute2f128_si256(
+// CHECK-SAME: <4 x i64> noundef [[A:%.*]], <4 x i64> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i64> [[A]] to <8 x i32>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[B]] to <8 x i32>
+// CHECK-NEXT:    [[VPERM:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+// CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i32> [[VPERM]] to <4 x i64>
+// CHECK-NEXT:    ret <4 x i64> [[TMP2]]
+//
 __m256i test_mm256_permute2f128_si256(__m256i a, __m256i b) {
-  // CHECK-LABEL: test_mm256_permute2f128_si256
-  // CHECK: shufflevector{{.*}}<i32 0, i32 1, i32 4, i32 5>
   return _mm256_permute2f128_si256(a, b, 0x20);
 }
 
-__m128
-test_mm_broadcast_ss(float const *__a) {
-  // CHECK-LABEL: test_mm_broadcast_ss
-  // CHECK: insertelement <4 x float> {{.*}}, i64 0
-  // CHECK: shufflevector <4 x float> {{.*}}, <4 x float> poison, <4 x i32> zeroinitializer
+// CHECK-LABEL: define dso_local <4 x float> @test_mm_broadcast_ss(
+// CHECK-SAME: ptr noundef [[__A:%.*]]) #[[ATTR1]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[__A]], align 1
+// CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i32 0
+// CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <4 x float> [[VECINIT_I]], float [[TMP0]], i32 1
+// CHECK-NEXT:    [[VECINIT3_I:%.*]] = insertelement <4 x float> [[VECINIT2_I]], float [[TMP0]], i32 2
+// CHECK-NEXT:    [[VECINIT4_I:%.*]] = insertelement <4 x float> [[VECINIT3_I]], float [[TMP0]], i32 3
+// CHECK-NEXT:    ret <4 x float> [[VECINIT4_I]]
+//
+__m128 test_mm_broadcast_ss(float const *__a) {
   return _mm_broadcast_ss(__a);
 }
 
-__m256d
-test_mm256_broadcast_sd(double const *__a) {
-  // CHECK-LABEL: test_mm256_broadcast_sd
-  // CHECK: insertelement <4 x double> {{.*}}, i64 0
-  // CHECK: shufflevector <4 x double> {{.*}}, <4 x double> poison, <4 x i32> zeroinitializer
+// CHECK-LABEL: define dso_local <4 x double> @test_mm256_broadcast_sd(
+// CHECK-SAME: ptr noundef [[__A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[__A]], align 1
+// CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x double> poison, double [[TMP0]], i32 0
+// CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <4 x double> [[VECINIT_I]], double [[TMP0]], i32 1
+// CHECK-NEXT:    [[VECINIT3_I:%.*]] = insertelement <4 x double> [[VECINIT2_I]], double [[TMP0]], i32 2
+// CHECK-NEXT:    [[VECINIT4_I:%.*]] = insertelement <4 x double> [[VECINIT3_I]], double [[TMP0]], i32 3
+// CHECK-NEXT:    ret <4 x double> [[VECINIT4_I]]
+//
+__m256d test_mm256_broadcast_sd(double const *__a) {
   return _mm256_broadcast_sd(__a);
 }
 
-__m256
-test_mm256_broadcast_ss(float const *__a) {
-  // CHECK-LABEL: test_mm256_broadcast_ss
-  // CHECK: insertelement <8 x float> {{.*}}, i64 0
-  // CHECK: shufflevector <8 x float> {{.*}}, <8 x float> poison, <8 x i32> zeroinitializer
+// CHECK-LABEL: define dso_local <8 x float> @test_mm256_broadcast_ss(
+// CHECK-SAME: ptr noundef [[__A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[__A]], align 1
+// CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x float> poison, float [[TMP0]], i32 0
+// CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <8 x float> [[VECINIT_I]], float [[TMP0]], i32 1
+// CHECK-NEXT:    [[VECINIT3_I:%.*]] = insertelement <8 x float> [[VECINIT2_I]], float [[TMP0]], i32 2
+// CHECK-NEXT:    [[VECINIT4_I:%.*]] = insertelement <8 x float> [[VECINIT3_I]], float [[TMP0]], i32 3
+// CHECK-NEXT:    [[VECINIT5_I:%.*]] = insertelement <8 x float> [[VECINIT4_I]], float [[TMP0]], i32 4
+// CHECK-NEXT:    [[VECINIT6_I:%.*]] = insertelement <8 x float> [[VECINIT5_I]], float [[TMP0]], i32 5
+// CHECK-NEXT:    [[VECINIT7_I:%.*]] = insertelement <8 x float> [[VECINIT6_I]], float [[TMP0]], i32 6
+// CHECK-NEXT:    [[VECINIT8_I:%.*]] = insertelement <8 x float> [[VECINIT7_I]], float [[TMP0]], i32 7
+// CHECK-NEXT:    ret <8 x float> [[VECINIT8_I]]
+//
+__m256 test_mm256_broadcast_ss(float const *__a) {
   return _mm256_broadcast_ss(__a);
 }
 
 // Make sure we have the correct mask for each insertf128 case.
 
+// CHECK-LABEL: define dso_local <8 x float> @test_mm256_insertf128_ps_0(
+// CHECK-SAME: <8 x float> noundef [[A:%.*]], <4 x float> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[WIDEN:%.*]] = shufflevector <4 x float> [[B]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+// CHECK-NEXT:    [[INSERT:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[WIDEN]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+// CHECK-NEXT:    ret <8 x float> [[INSERT]]
+//
 __m256 test_mm256_insertf128_ps_0(__m256 a, __m128 b) {
-  // CHECK-LABEL: test_mm256_insertf128_ps_0
-  // CHECK: shufflevector{{.*}}<i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
   return _mm256_insertf128_ps(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local <4 x double> @test_mm256_insertf128_pd_0(
+// CHECK-SAME: <4 x double> noundef [[A:%.*]], <2 x double> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[WIDEN:%.*]] = shufflevector <2 x double> [[B]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    [[INSERT:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[WIDEN]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+// CHECK-NEXT:    ret <4 x double> [[INSERT]]
+//
 __m256d test_mm256_insertf128_pd_0(__m256d a, __m128d b) {
-  // CHECK-LABEL: test_mm256_insertf128_pd_0
-  // CHECK: shufflevector{{.*}}<i32 0, i32 1, i32 6, i32 7>
   return _mm256_insertf128_pd(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local <4 x i64> @test_mm256_insertf128_si256_0(
+// CHECK-SAME: <4 x i64> noundef [[A:%.*]], <2 x i64> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i64> [[A]] to <8 x i32>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[B]] to <4 x i32>
+// CHECK-NEXT:    [[WIDEN:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+// CHECK-NEXT:    [[INSERT:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> [[WIDEN]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+// CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i32> [[INSERT]] to <4 x i64>
+// CHECK-NEXT:    ret <4 x i64> [[TMP2]]
+//
 __m256i test_mm256_insertf128_si256_0(__m256i a, __m128i b) {
-  // CHECK-LABEL: test_mm256_insertf128_si256_0
-  // X64: shufflevector{{.*}}<i32 0, i32 1, i32 6, i32 7>
-  // X86: shufflevector{{.*}}<i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
   return _mm256_insertf128_si256(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local <8 x float> @test_mm256_insertf128_ps_1(
+// CHECK-SAME: <8 x float> noundef [[A:%.*]], <4 x float> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[WIDEN:%.*]] = shufflevector <4 x float> [[B]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+// CHECK-NEXT:    [[INSERT:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[WIDEN]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+// CHECK-NEXT:    ret <8 x float> [[INSERT]]
+//
 __m256 test_mm256_insertf128_ps_1(__m256 a, __m128 b) {
-  // CHECK-LABEL: test_mm256_insertf128_ps_1
-  // CHECK: shufflevector{{.*}}<i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
   return _mm256_insertf128_ps(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local <4 x double> @test_mm256_insertf128_pd_1(
+// CHECK-SAME: <4 x double> noundef [[A:%.*]], <2 x double> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[WIDEN:%.*]] = shufflevector <2 x double> [[B]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    [[INSERT:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[WIDEN]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+// CHECK-NEXT:    ret <4 x double> [[INSERT]]
+//
 __m256d test_mm256_insertf128_pd_1(__m256d a, __m128d b) {
-  // CHECK-LABEL: test_mm256_insertf128_pd_1
-  // CHECK: shufflevector{{.*}}<i32 0, i32 1, i32 4, i32 5>
   return _mm256_insertf128_pd(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local <4 x i64> @test_mm256_insertf128_si256_1(
+// CHECK-SAME: <4 x i64> noundef [[A:%.*]], <2 x i64> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i64> [[A]] to <8 x i32>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[B]] to <4 x i32>
+// CHECK-NEXT:    [[WIDEN:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+// CHECK-NEXT:    [[INSERT:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> [[WIDEN]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+// CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i32> [[INSERT]] to <4 x i64>
+// CHECK-NEXT:    ret <4 x i64> [[TMP2]]
+//
 __m256i test_mm256_insertf128_si256_1(__m256i a, __m128i b) {
-  // CHECK-LABEL: test_mm256_insertf128_si256_1
-  // X64: shufflevector{{.*}}<i32 0, i32 1, i32 4, i32 5>
-  // X86: shufflevector{{.*}}<i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
   return _mm256_insertf128_si256(a, b, 1);
 }
 
 // Make sure we have the correct mask for each extractf128 case.
 
+// CHECK-LABEL: define dso_local <4 x float> @test_mm256_extractf128_ps_0(
+// CHECK-SAME: <8 x float> noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    ret <4 x float> [[EXTRACT]]
+//
 __m128 test_mm256_extractf128_ps_0(__m256 a) {
-  // X64-LABEL: test_mm256_extractf128_ps_0
-  // X64: shufflevector{{.*}}<i32 0, i32 1, i32 2, i32 3>
-  //
-  // X86-LABEL: test_mm256_extractf128_ps_0
-  // X86: shufflevector{{.*}}<i32 0, i32 1>
   return _mm256_extractf128_ps(a, 0);
 }
 
+// CHECK-LABEL: define dso_local <2 x double> @test_mm256_extractf128_pd_0(
+// CHECK-SAME: <4 x double> noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <4 x double> [[A]], <4 x double> poison, <2 x i32> <i32 0, i32 1>
+// CHECK-NEXT:    ret <2 x double> [[EXTRACT]]
+//
 __m128d test_mm256_extractf128_pd_0(__m256d a) {
-  // CHECK-LABEL: test_mm256_extractf128_pd_0
-  // CHECK: shufflevector{{.*}}<i32 0, i32 1>
   return _mm256_extractf128_pd(a, 0);
 }
 
+// CHECK-LABEL: define dso_local <2 x i64> @test_mm256_extractf128_si256_0(
+// CHECK-SAME: <4 x i64> noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i64> [[A]] to <8 x i32>
+// CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[EXTRACT]] to <2 x i64>
+// CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
 __m128i test_mm256_extractf128_si256_0(__m256i a) {
-  // CHECK-LABEL: test_mm256_extractf128_si256_0
-  // CHECK: shufflevector{{.*}}<i32 0, i32 1>
   return _mm256_extractf128_si256(a, 0);
 }
 
+// CHECK-LABEL: define dso_local <4 x float> @test_mm256_extractf128_ps_1(
+// CHECK-SAME: <8 x float> noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+// CHECK-NEXT:    ret <4 x float> [[EXTRACT]]
+//
 __m128 test_mm256_extractf128_ps_1(__m256 a) {
-  // X64-LABEL: test_mm256_extractf128_ps_1
-  // X64: shufflevector{{.*}}<i32 4, i32 5, i32 6, i32 7>
-  //
-  // X86-LABEL: test_mm256_extractf128_ps_1
-  // X86: shufflevector{{.*}}<i32 2, i32 3>
   return _mm256_extractf128_ps(a, 1);
 }
 
+// CHECK-LABEL: define dso_local <2 x double> @test_mm256_extractf128_pd_1(
+// CHECK-SAME: <4 x double> noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <4 x double> [[A]], <4 x double> poison, <2 x i32> <i32 2, i32 3>
+// CHECK-NEXT:    ret <2 x double> [[EXTRACT]]
+//
 __m128d test_mm256_extractf128_pd_1(__m256d a) {
-  // CHECK-LABEL: test_mm256_extractf128_pd_1
-  // CHECK: shufflevector{{.*}}<i32 2, i32 3>
   return _mm256_extractf128_pd(a, 1);
 }
 
+// CHECK-LABEL: define dso_local <2 x i64> @test_mm256_extractf128_si256_1(
+// CHECK-SAME: <4 x i64> noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i64> [[A]] to <8 x i32>
+// CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[EXTRACT]] to <2 x i64>
+// CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
 __m128i test_mm256_extractf128_si256_1(__m256i a) {
-  // CHECK-LABEL: test_mm256_extractf128_si256_1
-  // CHECK: shufflevector{{.*}}<i32 2, i32 3>
   return _mm256_extractf128_si256(a, 1);
 }
 
+// CHECK-LABEL: define dso_local <8 x float> @test_mm256_set_m128(
+// CHECK-SAME: <4 x float> noundef [[HI:%.*]], <4 x float> noundef [[LO:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <4 x float> [[LO]], <4 x float> [[HI]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+// CHECK-NEXT:    ret <8 x float> [[SHUFFLE_I]]
+//
 __m256 test_mm256_set_m128(__m128 hi, __m128 lo) {
-  // CHECK-LABEL: test_mm256_set_m128
-  // CHECK: shufflevector{{.*}}<i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   return _mm256_set_m128(hi, lo);
 }
 
+// CHECK-LABEL: define dso_local <4 x double> @test_mm256_set_m128d(
+// CHECK-SAME: <2 x double> noundef [[HI:%.*]], <2 x double> noundef [[LO:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <2 x double> [[LO]], <2 x double> [[HI]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    ret <4 x double> [[SHUFFLE_I]]
+//
 __m256d test_mm256_set_m128d(__m128d hi, __m128d lo) {
-  // CHECK-LABEL: test_mm256_set_m128d
-  // CHECK: shufflevector{{.*}}<i32 0, i32 1, i32 2, i32 3>
   return _mm256_set_m128d(hi, lo);
 }
 
+// CHECK-LABEL: define dso_local <4 x i64> @test_mm256_set_m128i(
+// CHECK-SAME: <2 x i64> noundef [[HI:%.*]], <2 x i64> noundef [[LO:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <2 x i64> [[LO]], <2 x i64> [[HI]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    ret <4 x i64> [[SHUFFLE_I]]
+//
 __m256i test_mm256_set_m128i(__m128i hi, __m128i lo) {
-  // CHECK-LABEL: test_mm256_set_m128i
-  // CHECK: shufflevector{{.*}}<i32 0, i32 1, i32 2, i32 3>  
   return _mm256_set_m128i(hi, lo);
 }
 
+// CHECK-LABEL: define dso_local <8 x float> @test_mm256_setr_m128(
+// CHECK-SAME: <4 x float> noundef [[HI:%.*]], <4 x float> noundef [[LO:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <4 x float> [[LO]], <4 x float> [[HI]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+// CHECK-NEXT:    ret <8 x float> [[SHUFFLE_I_I]]
+//
 __m256 test_mm256_setr_m128(__m128 hi, __m128 lo) {
-  // CHECK-LABEL: test_mm256_setr_m128
-  // CHECK: shufflevector{{.*}}<i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   return _mm256_setr_m128(lo, hi);
 }
 
+// CHECK-LABEL: define dso_local <4 x double> @test_mm256_setr_m128d(
+// CHECK-SAME: <2 x double> noundef [[HI:%.*]], <2 x double> noundef [[LO:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x double> [[LO]], <2 x double> [[HI]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    ret <4 x double> [[SHUFFLE_I_I]]
+//
 __m256d test_mm256_setr_m128d(__m128d hi, __m128d lo) {
-  // CHECK-LABEL: test_mm256_setr_m128d
-  // CHECK: shufflevector{{.*}}<i32 0, i32 1, i32 2, i32 3>
   return _mm256_setr_m128d(lo, hi);
 }
 
+// CHECK-LABEL: define dso_local <4 x i64> @test_mm256_setr_m128i(
+// CHECK-SAME: <2 x i64> noundef [[HI:%.*]], <2 x i64> noundef [[LO:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[LO]], <2 x i64> [[HI]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    ret <4 x i64> [[SHUFFLE_I_I]]
+//
 __m256i test_mm256_setr_m128i(__m128i hi, __m128i lo) {
-  // CHECK-LABEL: test_mm256_setr_m128i
-  // CHECK: shufflevector{{.*}}<i32 0, i32 1, i32 2, i32 3>
   return _mm256_setr_m128i(lo, hi);
 }
diff --git a/clang/test/CodeGen/X86/sse.c b/clang/test/CodeGen/X86/sse.c
index a75b8dc77e86..017bdd7846fa 100644
--- a/clang/test/CodeGen/X86/sse.c
+++ b/clang/test/CodeGen/X86/sse.c
@@ -1,42 +1,72 @@
-// RUN: %clang_cc1 -ffreestanding -O3 -triple x86_64-apple-macosx10.8.0 -target-feature +sse4.1 -emit-llvm %s -o - | FileCheck %s
-// FIXME: This test currently depends on optimization - it should be rewritten to avoid it.
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1 -ffreestanding -triple x86_64-- -target-feature +sse4.1 -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | FileCheck %s
 
 
 #include <emmintrin.h>
 
 // Byte-shifts look reversed due to xmm register layout
+// CHECK-LABEL: define dso_local <2 x i64> @test_mm_slli_si128(
+// CHECK-SAME: <2 x i64> noundef [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[CAST:%.*]] = bitcast <2 x i64> [[A]] to <16 x i8>
+// CHECK-NEXT:    [[PSLLDQ:%.*]] = shufflevector <16 x i8> zeroinitializer, <16 x i8> [[CAST]], <16 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26>
+// CHECK-NEXT:    [[CAST1:%.*]] = bitcast <16 x i8> [[PSLLDQ]] to <2 x i64>
+// CHECK-NEXT:    ret <2 x i64> [[CAST1]]
+//
 __m128i test_mm_slli_si128(__m128i a) {
-  // CHECK-LABEL: @test_mm_slli_si128
-  // CHECK: shufflevector <16 x i8> <{{.*}}, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i8> {{.*}}, <16 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26>
   return _mm_slli_si128(a, 5);
 }
 
+// CHECK-LABEL: define dso_local <2 x i64> @test_mm_slli_si128_0(
+// CHECK-SAME: <2 x i64> noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[CAST:%.*]] = bitcast <2 x i64> [[A]] to <16 x i8>
+// CHECK-NEXT:    [[PSLLDQ:%.*]] = shufflevector <16 x i8> zeroinitializer, <16 x i8> [[CAST]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+// CHECK-NEXT:    [[CAST1:%.*]] = bitcast <16 x i8> [[PSLLDQ]] to <2 x i64>
+// CHECK-NEXT:    ret <2 x i64> [[CAST1]]
+//
 __m128i test_mm_slli_si128_0(__m128i a) {
-  // CHECK-LABEL: @test_mm_slli_si128_0
-  // CHECK-NOT: shufflevector
   return _mm_slli_si128(a, 0);
 }
 
+// CHECK-LABEL: define dso_local <2 x i64> @test_mm_slli_si128_16(
+// CHECK-SAME: <2 x i64> noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <2 x i64> zeroinitializer
+//
 __m128i test_mm_slli_si128_16(__m128i a) {
-  // CHECK-LABEL: @test_mm_slli_si128_16
-  // CHECK-NOT: shufflevector
   return _mm_slli_si128(a, 16);
 }
 
+// CHECK-LABEL: define dso_local <2 x i64> @test_mm_srli_si128(
+// CHECK-SAME: <2 x i64> noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[CAST:%.*]] = bitcast <2 x i64> [[A]] to <16 x i8>
+// CHECK-NEXT:    [[PSRLDQ:%.*]] = shufflevector <16 x i8> [[CAST]], <16 x i8> zeroinitializer, <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>
+// CHECK-NEXT:    [[CAST1:%.*]] = bitcast <16 x i8> [[PSRLDQ]] to <2 x i64>
+// CHECK-NEXT:    ret <2 x i64> [[CAST1]]
+//
 __m128i test_mm_srli_si128(__m128i a) {
-  // CHECK-LABEL: @test_mm_srli_si128
-  // CHECK: shufflevector <16 x i8> {{.*}}, <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, {{.*}}>, <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>
   return _mm_srli_si128(a, 5);
 }
 
+// CHECK-LABEL: define dso_local <2 x i64> @test_mm_srli_si128_0(
+// CHECK-SAME: <2 x i64> noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[CAST:%.*]] = bitcast <2 x i64> [[A]] to <16 x i8>
+// CHECK-NEXT:    [[PSRLDQ:%.*]] = shufflevector <16 x i8> [[CAST]], <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+// CHECK-NEXT:    [[CAST1:%.*]] = bitcast <16 x i8> [[PSRLDQ]] to <2 x i64>
+// CHECK-NEXT:    ret <2 x i64> [[CAST1]]
+//
 __m128i test_mm_srli_si128_0(__m128i a) {
-  // CHECK-LABEL: @test_mm_srli_si128_0
-  // CHECK-NOT: shufflevector
   return _mm_srli_si128(a, 0);
 }
 
+// CHECK-LABEL: define dso_local <2 x i64> @test_mm_srli_si128_16(
+// CHECK-SAME: <2 x i64> noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret <2 x i64> zeroinitializer
+//
 __m128i test_mm_srli_si128_16(__m128i a) {
-  // CHECK-LABEL: @test_mm_srli_si128_16
-  // CHECK-NOT: shufflevector
   return _mm_srli_si128(a, 16);
 }
-- 
GitLab


From 5903c6af44256e0bba77b7a69d608aa3ccc6a0dd Mon Sep 17 00:00:00 2001
From: Matthias Braun <matze@braunis.de>
Date: Mon, 28 Oct 2024 15:35:17 -0700
Subject: [PATCH 380/693] InstCombine: Fold shufflevector(select) and
 shufflevector(phi) (#113746)

- Transform `shufflevector(select(c, x, y), C)` to
  `select(c, shufflevector(x, C), shufflevector(y, C))` by re-using
  the `FoldOpIntoSelect` helper.
- Transform `shufflevector(phi(x, y), C)` to
  `phi(shufflevector(x, C), shufflevector(y, C))` by re-using the
  `foldOpInotPhi` helper.
---
 .../InstCombine/InstCombineVectorOps.cpp      | 11 +++++
 .../Transforms/InstCombine/vec_shuffle.ll     | 44 +++++++++++++++++++
 2 files changed, 55 insertions(+)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index d68ae64f08aa..75e7c1c97018 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -2900,6 +2900,17 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
   if (Instruction *I = foldIdentityPaddedShuffles(SVI))
     return I;
 
+  if (match(RHS, m_Constant())) {
+    if (auto *SI = dyn_cast<SelectInst>(LHS)) {
+      if (Instruction *I = FoldOpIntoSelect(SVI, SI))
+        return I;
+    }
+    if (auto *PN = dyn_cast<PHINode>(LHS)) {
+      if (Instruction *I = foldOpIntoPhi(SVI, PN))
+        return I;
+    }
+  }
+
   if (match(RHS, m_Poison()) && canEvaluateShuffled(LHS, Mask)) {
     Value *V = evaluateInDifferentElementOrder(LHS, Mask, Builder);
     return replaceInstUsesWith(SVI, V);
diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
index 75a84e51279b..d050cf10849e 100644
--- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll
+++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
@@ -2377,3 +2377,47 @@ define <2 x i32> @not_splat_shuffle2(i32 %x) {
   %shuf = shufflevector <2 x i32> %vec, <2 x i32> undef, <2 x i32> <i32 1, i32 3>
   ret <2 x i32> %shuf
 }
+define <2 x i32> @foldselect0(i1 %c) {
+; CHECK-LABEL: @foldselect0(
+; CHECK-NEXT:    [[SHUF:%.*]] = select i1 [[C:%.*]], <2 x i32> <i32 7, i32 42>, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    ret <2 x i32> [[SHUF]]
+;
+  %sel = select i1 %c, <2 x i32> <i32 42, i32 7>, <2 x i32> <i32 0, i32 1>
+  %shuf = shufflevector <2 x i32> %sel, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+  ret <2 x i32> %shuf
+}
+
+declare i1 @cond()
+declare <4 x i32> @value()
+
+define <4 x i32> @foldphi1() {
+; CHECK-LABEL: @foldphi1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[V:%.*]] = phi <4 x i32> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[XOR:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[VAL:%.*]] = call <4 x i32> @value()
+; CHECK-NEXT:    [[XOR]] = xor <4 x i32> [[V]], [[VAL]]
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[SHUF1:%.*]] = shufflevector <4 x i32> [[XOR]], <4 x i32> poison, <4 x i32> <i32 3, i32 0, i32 1, i32 2>
+; CHECK-NEXT:    ret <4 x i32> [[SHUF1]]
+;
+entry:
+  br label %loop
+
+loop:
+  %v = phi <4 x i32> [zeroinitializer, %entry], [%shuf1, %loop]
+
+  %shuf0 = shufflevector <4 x i32> %v, <4 x i32> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %val = call <4 x i32> @value()
+  %xor = xor <4 x i32> %shuf0, %val
+  %shuf1 = shufflevector <4 x i32> %xor, <4 x i32> poison, <4 x i32> <i32 3, i32 0, i32 1, i32 2>
+
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret <4 x i32> %shuf1
+}
-- 
GitLab


From 034cae42b130760587770b6a001b70c2a01bdfe9 Mon Sep 17 00:00:00 2001
From: Doug Wyatt <doug@sonosphere.com>
Date: Mon, 28 Oct 2024 15:36:37 -0700
Subject: [PATCH 381/693] [clang] Add clang/docs/FunctionEffectAnalysis.rst.
 (#109855)

Follow-on from #99656, which introduces 2nd pass caller/callee analysis
for function effects.

Wrote a new documentation page, derived directly from the RFC posted to
LLVM Discourse earlier this year.

---------

Co-authored-by: Doug Wyatt <dwyatt@apple.com>
Co-authored-by: Sirraide <aeternalmail@gmail.com>
---
 clang/docs/FunctionEffectAnalysis.rst | 535 ++++++++++++++++++++++++++
 clang/docs/index.rst                  |   1 +
 2 files changed, 536 insertions(+)
 create mode 100644 clang/docs/FunctionEffectAnalysis.rst

diff --git a/clang/docs/FunctionEffectAnalysis.rst b/clang/docs/FunctionEffectAnalysis.rst
new file mode 100644
index 000000000000..f286aec4f5b9
--- /dev/null
+++ b/clang/docs/FunctionEffectAnalysis.rst
@@ -0,0 +1,535 @@
+========================
+Function Effect Analysis
+========================
+
+.. contents::
+  :depth: 3
+  :local:
+
+
+Introduction
+============
+
+Clang Function Effect Analysis is a language extension which can warn about "unsafe"
+constructs. The feature is currently tailored for the Performance Constraint attributes
+``nonblocking`` and ``nonallocating``; functions with these attributes are verified as not
+containing any language constructs or calls to other functions which violate the constraint.
+(See :doc:`AttributeReference`.)
+
+
+The ``nonblocking`` and ``nonallocating`` attributes
+====================================================
+
+Attribute syntax
+----------------
+
+The ``nonblocking`` and ``nonallocating`` attributes apply to function types, allowing them to be
+attached to functions, blocks, function pointers, lambdas, and member functions.
+
+.. code-block:: c++
+
+  // Functions
+  void nonblockingFunction() [[clang::nonblocking]];
+  void nonallocatingFunction() [[clang::nonallocating]];
+
+  // Function pointers
+  void (*nonblockingFunctionPtr)() [[clang::nonblocking]];
+
+  // Typedefs, type aliases.
+  typedef void (*NBFunctionPtrTypedef)() [[clang::nonblocking]];
+  using NBFunctionPtrTypeAlias_gnu = __attribute__((nonblocking)) void (*)();
+  using NBFunctionPtrTypeAlias_std = void (*)() [[clang::nonblocking]];
+
+  // C++ methods
+  struct Struct {
+    void NBMethod() [[clang::nonblocking]];
+  };
+
+  // C++ lambdas
+  auto nbLambda = []() [[clang::nonblocking]] {};
+
+  // Blocks
+  void (^nbBlock)() = ^() [[clang::nonblocking]] {};
+
+The attribute applies only to the function itself. In particular, it does not apply to any nested
+functions or declarations, such as blocks, lambdas, and local classes.
+
+This document uses the C++/C23 syntax ``[[clang::nonblocking]]``, since it parallels the placement 
+of the ``noexcept`` specifier, and the attributes have other similarities to ``noexcept``. The GNU
+``__attribute__((nonblocking))`` syntax is also supported. Note that it requires a different 
+placement on a C++ type alias.
+
+Like ``noexcept``, ``nonblocking`` and ``nonallocating`` have an optional argument, a compile-time
+constant boolean expression. By default, the argument is ``true``, so ``[[clang::nonblocking]]``
+is equivalent to ``[[clang::nonblocking(true)]]``, and declares the function type as never blocking.
+
+
+Attribute semantics
+-------------------
+
+Together with ``noexcept``, the ``nonallocating`` and ``nonblocking`` attributes define an ordered
+series of performance constraints. From weakest to strongest:
+
+- ``noexcept`` (as per the C++ standard): The function type will never throw an exception.
+- ``nonallocating``: The function type will never allocate memory on the heap or throw an
+  exception.
+- ``nonblocking``: The function type will never block on a lock, allocate memory on the heap,
+  or throw an exception.
+
+``nonblocking`` includes the ``nonallocating`` guarantee. 
+
+While ``nonblocking`` and ``nonallocating`` are conceptually a superset of ``noexcept``, neither
+attribute implicitly specifies ``noexcept``. Further, ``noexcept`` has a specified runtime behavior of 
+aborting if an exception is thrown, while the ``nonallocating`` and ``nonblocking`` attributes are
+mainly for compile-time analysis and have no runtime behavior, except in code built
+with Clang's :doc:`RealtimeSanitizer`. Nonetheless, Clang emits a
+warning if, in C++, a function is declared ``nonblocking`` or ``nonallocating`` without
+``noexcept``. This diagnostic is controlled by ``-Wperf-constraint-implies-noexcept``.
+
+``nonblocking(true)`` and ``nonallocating(true)`` apply to function *types*, and by extension, to
+function-like declarations. When applied to a declaration with a body, the compiler verifies the
+function, as described in the section "Analysis and warnings", below.
+
+``blocking`` and ``allocating`` are synonyms for ``nonblocking(false)`` and
+``nonallocating(false)``, respectively. They can be used on a function-like declaration to
+explicitly disable any potential inference of ``nonblocking`` or ``nonallocating`` during
+verification. (Inference is described later in this document). ``nonblocking(false)`` and
+``nonallocating(false)`` are legal, but superfluous  when applied to a function *type*
+that is not part of a declarator: ``float (int) [[nonblocking(false)]]`` and 
+``float (int)`` are identical types.
+
+For functions with no explicit performance constraint, the worst is assumed: the function
+allocates memory and potentially blocks, unless it can be inferred otherwise. This is detailed in the
+discussion of verification.
+
+The following example describes the meanings of all permutations of the two attributes and arguments:
+
+.. code-block:: c++
+
+  void nb1_na1() [[clang::nonblocking(true)]] [[clang::nonallocating(true)]];
+  // Valid; nonallocating(true) is superfluous but doesn't contradict the guarantee.
+
+  void nb1_na0() [[clang::nonblocking(true)]] [[clang::nonallocating(false)]];
+  // error: 'allocating' and 'nonblocking' attributes are not compatible
+
+  void nb0_na1() [[clang::nonblocking(false)]] [[clang::nonallocating(true)]];
+  // Valid; the function does not allocate memory, but may lock for other reasons.
+
+  void nb0_na0() [[clang::nonblocking(false)]] [[clang::nonallocating(false)]];
+  // Valid.
+
+
+Type conversions
+----------------
+
+A performance constraint can be removed or weakened via an implicit conversion. An attempt to add
+or strengthen a performance constraint is unsafe and results in a warning. The rules for this
+are comparable to that for ``noexcept`` in C++17 and later.
+
+.. code-block:: c++
+
+  void unannotated();
+  void nonblocking() [[clang::nonblocking]];
+  void nonallocating() [[clang::nonallocating]];
+
+  void example()
+  {
+    // It's fine to remove a performance constraint.
+    void (*fp_plain)();
+    fp_plain = unannotated;
+    fp_plain = nonblocking;
+    fp_plain = nonallocating;
+
+    // Adding/spoofing nonblocking is unsafe.
+    void (*fp_nonblocking)() [[clang::nonblocking]];
+    fp_nonblocking = nullptr;
+    fp_nonblocking = nonblocking;
+    fp_nonblocking = unannotated;
+    // ^ warning: attribute 'nonblocking' should not be added via type conversion
+    fp_nonblocking = nonallocating;
+    // ^ warning: attribute 'nonblocking' should not be added via type conversion
+
+    // Adding/spoofing nonallocating is unsafe.
+    void (*fp_nonallocating)() [[clang::nonallocating]];
+    fp_nonallocating = nullptr;
+    fp_nonallocating = nonallocating;
+    fp_nonallocating = nonblocking; // no warning because nonblocking includes nonallocating 
+    fp_nonallocating = unannotated;
+    // ^ warning: attribute 'nonallocating' should not be added via type conversion
+  }
+
+Virtual methods
+---------------
+
+In C++, when a virtual method has a performance constraint, overriding methods in
+subclasses inherit the constraint.
+
+.. code-block:: c++
+
+  struct Base {
+    virtual void unsafe();
+    virtual void safe() noexcept [[clang::nonblocking]];
+  };
+
+  struct Derived : public Base {
+    void unsafe() [[clang::nonblocking]] override;
+    // It's okay for an overridden method to be more constrained
+
+    void safe() noexcept override;
+    // This method is implicitly declared `nonblocking`, inherited from Base.
+  };
+
+Redeclarations, overloads, and name mangling
+--------------------------------------------
+
+The ``nonblocking`` and ``nonallocating`` attributes, like ``noexcept``, do not factor into
+argument-dependent lookup and overloaded functions/methods.
+
+First, consider that ``noexcept`` is integral to a function's type:
+
+.. code-block:: c++
+
+  void f1(int);
+  void f1(int) noexcept;
+  // error: exception specification in declaration does not match previous
+  //   declaration
+
+Unlike ``noexcept``, a redeclaration of ``f2`` with an added or stronger performance constraint is
+legal and propagates the attribute to the previous declaration:
+
+.. code-block:: c++
+
+  int f2();
+  int f2() [[clang::nonblocking]]; // redeclaration with stronger constraint is OK.
+
+This greatly eases adoption by making it possible to annotate functions in external libraries
+without modifying library headers.
+
+A redeclaration with a removed or weaker performance constraint produces a warning, paralleling
+the behavior of ``noexcept``:
+
+.. code-block:: c++
+
+  int f2() { return 42; }
+  // warning: attribute 'nonblocking' on function does not match previous declaration
+
+In C++14, the following two declarations of `f3` are identical (a single function). In C++17 they
+are separate overloads:
+
+.. code-block:: c++
+
+  void f3(void (*)());
+  void f3(void (*)() noexcept);
+
+Similarly, the following two declarations of `f4` are separate overloads. This pattern may pose
+difficulties due to ambiguity:
+
+.. code-block:: c++
+
+  void f4(void (*)());
+  void f4(void (*)() [[clang::nonblocking]]);
+
+The attributes have no effect on the mangling of function and method names.
+
+Objective-C
+-----------
+
+The attributes are currently unsupported on Objective-C methods.
+
+Analysis and warnings
+=====================
+
+Constraints
+-----------
+
+Functions declared ``nonallocating`` or ``nonblocking``, when defined, are verified according to the
+following rules. Such functions:
+
+1. May not allocate or deallocate memory on the heap. The analysis follows the calls to
+   ``operator new`` and ``operator delete`` generated by the ``new`` and ``delete`` keywords, and
+   treats them like any other function call. The global ``operator new`` and ``operator delete``
+   aren't declared ``nonblocking`` or ``nonallocating`` and so they are considered unsafe. (This
+   is correct because most memory allocators are not lock-free. Note that the placement form of
+   ``operator new`` is implemented inline in libc++'s ``<new>`` header, and is verifiably
+   ``nonblocking``, since it merely casts the supplied pointer to the result type.)
+
+2. May not throw or catch exceptions. To throw, the compiler must allocate the exception on the
+   heap. (Also, many subclasses of ``std::exception`` allocate a string). Exceptions are
+   deallocated when caught.
+
+3. May not make any indirect function call, via a virtual method, function pointer, or
+   pointer-to-member function, unless the target is explicitly declared with the same
+   ``nonblocking`` or ``nonallocating`` attribute (or stronger).
+
+4. May not make direct calls to any other function, with the following exceptions:
+
+  a. The callee is also explicitly declared with the same ``nonblocking`` or ``nonallocating``
+     attribute (or stronger).
+  b. The callee is defined in the same translation unit as the caller, does not have the ``false``
+     form of the required attribute, and can be verified to have the same attribute or stronger,
+     according to these same rules.
+  c. The callee is a built-in function that is known not to block or allocate.
+  d. The callee is declared ``noreturn`` and, if compiling C++, the callee is also declared
+     ``noexcept``. This special case excludes functions such as ``abort()`` and ``std::terminate()``
+     from the analysis. (The reason for requiring ``noexcept`` in C++ is that a function declared
+     ``noreturn`` could be a wrapper for ``throw``.)
+
+5. May not invoke or access an Objective-C method or property, since ``objc_msgSend()`` calls into 
+   the Objective-C runtime, which may allocate memory or otherwise block.
+
+6. May not access thread-local variables. Typically, thread-local variables are allocated on the
+   heap when first accessed.
+
+Functions declared ``nonblocking`` have an additional constraint:
+
+7. May not declare static local variables (e.g. Meyers singletons). The compiler generates a lock
+   protecting the initialization of the variable.
+
+Violations of any of these rules result in warnings, in the ``-Wfunction-effects`` category:
+
+.. code-block:: c++
+
+  void notInline();
+
+  void example() [[clang::nonblocking]]
+  {
+    auto* x = new int;
+    // warning: function with 'nonblocking' attribute must not allocate or deallocate
+    //   memory
+
+    if (x == nullptr) {
+      static Logger* logger = createLogger();
+      // warning: function with 'nonblocking' attribute must not have static local variables
+
+      throw std::runtime_warning{ "null" };
+      // warning: 'nonblocking" function 'example' must not throw exceptions
+    }
+    notInline();
+    // warning: 'function with 'nonblocking' attribute must not call non-'nonblocking' function
+    //   'notInline'
+    // note (on notInline()): declaration cannot be inferred 'nonblocking' because it has no
+    //   definition in this translation unit
+  }
+
+Inferring ``nonblocking`` or ``nonallocating``
+----------------------------------------------
+
+In the absence of a ``nonblocking`` or ``nonallocating`` attribute (whether ``true`` or ``false``),
+a function that is called from a performance-constrained function may be analyzed to
+infer whether it has a desired attribute. This analysis happens when the function is not a virtual
+method, and it has a visible definition within the current translation unit (i.e. its body can be
+traversed).
+
+.. code-block:: c++
+
+  void notInline();
+  int implicitlySafe() { return 42; }
+  void implicitlyUnsafe() { notInline(); }
+
+  void example() [[clang::nonblocking]]
+  {
+    int x = implicitlySafe(); // OK
+    implicitlyUnsafe();
+    // warning: function with 'nonblocking' attribute must not call non-'nonblocking' function
+    //   'implicitlyUnsafe'
+    // note (on implicitlyUnsafe): function cannot be inferred 'nonblocking' because it calls
+    //   non-'nonblocking' function 'notInline'
+    // note (on notInline()): declaration cannot be inferred 'nonblocking' because it has no
+    //   definition in this translation unit
+  }
+
+Lambdas and blocks
+------------------
+
+As mentioned earlier, the performance constraint attributes apply only to a single function and not
+to any code nested inside it, including blocks, lambdas, and local classes. It is possible for a
+nonblocking function to schedule the execution of a blocking lambda on another thread. Similarly, a
+blocking function may create a ``nonblocking`` lambda for use in a realtime context.
+
+Operations which create, destroy, copy, and move lambdas and blocks are analyzed in terms of the
+underlying function calls. For example, the creation of a lambda with captures generates a function
+call to an anonymous struct's constructor, passing the captures as parameters.
+
+Implicit function calls in the AST
+----------------------------------
+
+The ``nonblocking`` / ``nonallocating`` analysis occurs at the Sema phase of analysis in Clang.
+During Sema, there are some constructs which will eventually become function calls, but do not
+appear as function calls in the AST. For example, ``auto* foo = new Foo;`` becomes a declaration
+containing a ``CXXNewExpr`` which is understood as a function call to the global ``operator new``
+(in this example), and a ``CXXConstructExpr``, which, for analysis purposes, is a function call to
+``Foo``'s constructor. Most gaps in the analysis would be due to incomplete knowledge of AST
+constructs which become function calls.
+
+Disabling diagnostics
+---------------------
+
+Function effect diagnostics are controlled by ``-Wfunction-effects``.
+
+A construct like this can be used to exempt code from the checks described here:
+
+.. code-block:: c++
+
+  #define NONBLOCKING_UNSAFE(...)                                    \
+    _Pragma("clang diagnostic push")                                 \
+    _Pragma("clang diagnostic ignored \"-Wunknown-warning-option\"") \
+    _Pragma("clang diagnostic ignored \"-Wfunction-effects\"")       \
+    __VA_ARGS__                                                      \
+    _Pragma("clang diagnostic pop")
+
+Disabling the diagnostic allows for:
+
+- constructs which do block, but which in practice are used in ways to avoid unbounded blocking,
+  e.g. a thread pool with semaphores to coordinate multiple realtime threads;
+- using libraries which are safe but not yet annotated;
+- incremental adoption in a large codebase.
+
+Adoption
+========
+
+There are a few common issues that arise when adopting the ``nonblocking`` and ``nonallocating``
+attributes.
+
+C++ exceptions
+--------------
+
+Exceptions pose a challenge to the adoption of the performance constraints. Common library functions
+which throw exceptions include:
+
++----------------------------------+-----------------------------------------------------------------------+
+| Method                           | Alternative                                                           |
++==================================+=======================================================================+
+| ``std::vector<T>::at()``         | ``operator[](size_t)``, after verifying that the index is in range.   |
++----------------------------------+-----------------------------------------------------------------------+
+| ``std::optional<T>::value()``    | ``operator*``, after checking ``has_value()`` or ``operator bool()``. |
++----------------------------------+-----------------------------------------------------------------------+
+| ``std::expected<T, E>::value()`` | Same as for ``std::optional<T>::value()``.                            |
++----------------------------------+-----------------------------------------------------------------------+
+
+
+``std::function<R(Args...)>``
+-----------------------------
+
+``std::function<R(Args...)>`` is generally incompatible with ``nonblocking`` and ``nonallocating``
+code, because a typical implementation may allocate heap memory in the constructor.
+
+Alternatives:
+
+- ``std::function_ref`` (available in C++26 or as ``llvm::function_ref``). This is appropriate and
+  optimal when a functor's lifetime does not need to extend past the function that created it.
+
+- ``inplace_function`` from WG14. This solves the allocation problem by giving the functor wrapper
+  a fixed size known at compile time and using an inline buffer.
+
+While these alternatives both address the heap allocation of ``std::function``, they are still
+obstacles to ``nonblocking/nonallocating`` verification, for reasons detailed in the next section.
+
+
+Interactions with type-erasure techniques
+-----------------------------------------
+
+``std::function<R(Args...)>`` illustrates a common C++ type-erasure technique. Using template
+argument deduction, it decomposes a function type into its return and parameter types. Additional
+components of the function type, including ``noexcept``, ``nonblocking``, ``nonallocating``, and any
+other attributes, are discarded.
+
+Standard library support for these components of a function type is not immediately forthcoming.
+
+Code can work around this limitation in either of two ways:
+
+1. Avoid abstractions like ``std::function`` and instead work directly with the original lambda type.
+
+2. Create a specialized alternative, e.g. ``nonblocking_function_ref<R(Args...)>`` where all function
+   pointers used in the implementation and its interface are ``nonblocking``.
+
+As an example of the first approach, when using a lambda as a *Callable* template parameter, the
+attribute is preserved:
+
+.. code-block:: c++
+
+  std::sort(vec.begin(), vec.end(),
+    [](const Elem& a, const Elem& b) [[clang::nonblocking]] { return a.mem < b.mem; });
+
+Here, the type of the ``Compare`` template parameter is an anonymous class generated from the
+lambda, with an ``operator()`` method holding the ``nonblocking`` attribute.
+
+A complication arises when a *Callable* template parameter, instead of being a lambda or class
+implementing ``operator()``, is a function pointer:
+
+.. code-block:: c++
+
+  static bool compare_elems(const Elem& a, const Elem& b) [[clang::nonblocking]] {
+    return a.mem < b.mem; };
+
+  std::sort(vec.begin(), vec.end(), compare_elems);
+
+Here, the type of ``compare_elems`` is decomposed to ``bool(const Elem&, const Elem&)``, without
+``nonblocking``, when forming the template parameter. This can be solved using the second approach,
+creating a specialized alternative which explicitly requires the attribute. In this case, it's
+possible to use a small wrapper to transform the function pointer into a functor:
+
+.. code-block:: c++
+
+  template <typename>
+  class nonblocking_fp;
+
+  template <typename R, typename... Args>
+  class nonblocking_fp<R(Args...)> {
+  public:
+    using impl_t = R (*)(Args...) [[clang::nonblocking]];
+
+  private:
+    impl_t mImpl{ nullptr_t };
+  public:
+    nonblocking_fp() = default;
+    nonblocking_fp(impl_t f) : mImpl{ f } {}
+
+    R operator()(Args... args) const
+    {
+      return mImpl(std::forward<Args>(args)...);
+    }
+  };
+
+  // deduction guide (like std::function's)
+  template< class R, class... ArgTypes >
+  nonblocking_fp( R(*)(ArgTypes...) ) -> nonblocking_fp<R(ArgTypes...)>;
+
+  // --
+
+  // Wrap the function pointer in a functor which preserves ``nonblocking``.
+  std::sort(vec.begin(), vec.end(), nonblocking_fp{ compare_elems });
+
+Now, the ``nonblocking`` attribute of ``compare_elems`` is verified when it is converted to a
+``nonblocking`` function pointer, as the argument to ``nonblocking_fp``'s constructor. The template
+parameter is the functor class ``nonblocking_fp``.
+
+
+Static local variables
+----------------------
+
+Static local variables are often used for lazily-constructed globals (Meyers singletons). Beyond the
+compiler's use of a lock to ensure thread-safe initialization, it is dangerously easy to
+inadvertently trigger initialization, involving heap allocation, from a ``nonblocking`` or
+``nonallocating`` context.
+
+Generally, such singletons need to be replaced by globals, and care must be taken to ensure their
+initialization before they are used from ``nonblocking`` or ``nonallocating`` contexts.
+
+
+Annotating libraries
+--------------------
+
+It can be surprising that the analysis does not depend on knowledge of any primitives; it simply
+assumes the worst, that all function calls are unsafe unless explicitly marked as safe or able to be
+inferred as safe. With ``nonblocking``, this appears to suffice for all but the most primitive of
+spinlocks.
+
+At least for an operating system's C functions, it is possible to define an override header which
+redeclares safe common functions (e.g. ``pthread_self()``) with the addition of ``nonblocking``.
+This may help in adopting the feature incrementally.
+
+It also helps that many of the functions in the standard C libraries (notably ``<math.h>``)
+are treated as built-in functions by Clang, which the diagnosis understands to be safe.
+
+Much of the C++ standard library consists of inline templated functions which work well with
+inference. A small number of primitives may need explicit ``nonblocking/nonallocating`` attributes.
diff --git a/clang/docs/index.rst b/clang/docs/index.rst
index 0f6fb36c4d33..1096432813fa 100644
--- a/clang/docs/index.rst
+++ b/clang/docs/index.rst
@@ -27,6 +27,7 @@ Using Clang as a Compiler
    ThreadSafetyAnalysis
    SafeBuffers
    DataFlowAnalysisIntro
+   FunctionEffectAnalysis
    AddressSanitizer
    ThreadSanitizer
    MemorySanitizer
-- 
GitLab


From 902acde34198bb11cc758dcf3aee00eb1cb09ceb Mon Sep 17 00:00:00 2001
From: David Majnemer <david.majnemer@gmail.com>
Date: Mon, 28 Oct 2024 16:08:28 +0000
Subject: [PATCH 382/693] [InstCombine] Optimize away certain additions using
 modular arithmetic

We can turn:
```
  %add = add i8 %arg, C1
  %and = and i8 %add, C2
  %cmp = icmp eq i1 %and, C3
```

into:
```
  %and = and i8 %arg, C2
  %cmp = icmp eq i1 %and, (C3 - C1) & C2
```

This is only worth doing if the sequence is the sole user of the addition
operation.
---
 .../InstCombine/InstCombineCompares.cpp       | 16 ++++
 .../Transforms/InstCombine/and-compare.ll     | 84 +++++++++++++++++++
 2 files changed, 100 insertions(+)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 338e9772c7cc..6bb39cabb098 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -1976,6 +1976,22 @@ Instruction *InstCombinerImpl::foldICmpAndConstant(ICmpInst &Cmp,
     return new ICmpInst(Pred, LShr, Constant::getNullValue(LShr->getType()));
   }
 
+  // (icmp eq/ne (and (add A, Addend), Msk), C)
+  //    -> (icmp eq/ne (and A, Msk), (and (sub C, Addend), Msk))
+  {
+    Value *A;
+    const APInt *Addend, *Msk;
+    if (match(And, m_And(m_OneUse(m_Add(m_Value(A), m_APInt(Addend))),
+                         m_APInt(Msk))) &&
+        Msk->isMask() && C.ule(*Msk)) {
+      APInt NewComperand = (C - *Addend) & *Msk;
+      Value* MaskA = Builder.CreateAnd(A, ConstantInt::get(A->getType(), *Msk));
+      return new ICmpInst(
+          Pred, MaskA,
+          Constant::getIntegerValue(MaskA->getType(), NewComperand));
+    }
+  }
+
   return nullptr;
 }
 
diff --git a/llvm/test/Transforms/InstCombine/and-compare.ll b/llvm/test/Transforms/InstCombine/and-compare.ll
index 5a9767a64a2c..9f8d3e317acc 100644
--- a/llvm/test/Transforms/InstCombine/and-compare.ll
+++ b/llvm/test/Transforms/InstCombine/and-compare.ll
@@ -172,3 +172,87 @@ define i1 @test_ne_cp2_other_okay2(i8 %x, i8 %yy) {
   %r = icmp ne i8 %and_x_y, %and_x_neg_y
   ret i1 %r
 }
+
+define i1 @test_eq_0_and_15_add_1(i8 %a) {
+; CHECK-LABEL: @test_eq_0_and_15_add_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[A:%.*]], 15
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[TMP0]], 15
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %add = add i8 %a, 1
+  %and = and i8 %add, 15
+  %cmp = icmp eq i8 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @test_ne_0_and_15_add_1(i8 %a) {
+; CHECK-LABEL: @test_ne_0_and_15_add_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[A:%.*]], 15
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[TMP0]], 15
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %add = add i8 %a, 1
+  %and = and i8 %add, 15
+  %cmp = icmp ne i8 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @test_eq_0_and_15_add_3(i8 %a) {
+; CHECK-LABEL: @test_eq_0_and_15_add_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[A:%.*]], 15
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[TMP0]], 13
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %add = add i8 %a, 3
+  %and = and i8 %add, 15
+  %cmp = icmp eq i8 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @test_ne_0_and_15_add_3(i8 %a) {
+; CHECK-LABEL: @test_ne_0_and_15_add_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[A:%.*]], 15
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[TMP0]], 13
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %add = add i8 %a, 3
+  %and = and i8 %add, 15
+  %cmp = icmp ne i8 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @test_eq_11_and_15_add_10(i8 %a) {
+; CHECK-LABEL: @test_eq_11_and_15_add_10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[A:%.*]], 15
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[TMP0]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %add = add i8 %a, 10
+  %and = and i8 %add, 15
+  %cmp = icmp eq i8 %and, 11
+  ret i1 %cmp
+}
+
+define i1 @test_ne_11_and_15_add_10(i8 %a) {
+; CHECK-LABEL: @test_ne_11_and_15_add_10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[A:%.*]], 15
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[TMP0]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %add = add i8 %a, 10
+  %and = and i8 %add, 15
+  %cmp = icmp ne i8 %and, 11
+  ret i1 %cmp
+}
-- 
GitLab


From 757d0e4764fffcd4e60338147c5f5456e2534395 Mon Sep 17 00:00:00 2001
From: Igor Kudrin <ikudrin@accesssoftek.com>
Date: Tue, 29 Oct 2024 06:13:32 +0700
Subject: [PATCH 383/693] Revert "[CFI][LowerTypeTests] Fix indirect call with
 alias" (#113978)

Reverts llvm/llvm-project#106185

This is breaking Sanitizer bots:
https://lab.llvm.org/buildbot/#/builders/66/builds/5449/steps/8/logs/stdio
---
 llvm/include/llvm/IR/ModuleSummaryIndexYAML.h | 102 +++++-------------
 llvm/lib/Transforms/IPO/LowerTypeTests.cpp    |   6 +-
 .../LowerTypeTests/cfi-icall-alias.ll         |  54 ----------
 3 files changed, 29 insertions(+), 133 deletions(-)
 delete mode 100644 llvm/test/Transforms/LowerTypeTests/cfi-icall-alias.ll

diff --git a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h
index d12bc260f5cf..902d1305c818 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h
@@ -135,14 +135,10 @@ template <> struct MappingTraits<TypeIdSummary> {
   }
 };
 
-struct GlobalValueSummaryYaml {
-  // Commonly used fields
+struct FunctionSummaryYaml {
   unsigned Linkage, Visibility;
   bool NotEligibleToImport, Live, IsLocal, CanAutoHide;
   unsigned ImportType;
-  // Fields for AliasSummary
-  std::optional<uint64_t> Aliasee;
-  // Fields for FunctionSummary
   std::vector<uint64_t> Refs;
   std::vector<uint64_t> TypeTests;
   std::vector<FunctionSummary::VFuncId> TypeTestAssumeVCalls,
@@ -180,8 +176,8 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(FunctionSummary::ConstVCall)
 namespace llvm {
 namespace yaml {
 
-template <> struct MappingTraits<GlobalValueSummaryYaml> {
-  static void mapping(IO &io, GlobalValueSummaryYaml &summary) {
+template <> struct MappingTraits<FunctionSummaryYaml> {
+  static void mapping(IO &io, FunctionSummaryYaml& summary) {
     io.mapOptional("Linkage", summary.Linkage);
     io.mapOptional("Visibility", summary.Visibility);
     io.mapOptional("NotEligibleToImport", summary.NotEligibleToImport);
@@ -189,7 +185,6 @@ template <> struct MappingTraits<GlobalValueSummaryYaml> {
     io.mapOptional("Local", summary.IsLocal);
     io.mapOptional("CanAutoHide", summary.CanAutoHide);
     io.mapOptional("ImportType", summary.ImportType);
-    io.mapOptional("Aliasee", summary.Aliasee);
     io.mapOptional("Refs", summary.Refs);
     io.mapOptional("TypeTests", summary.TypeTests);
     io.mapOptional("TypeTestAssumeVCalls", summary.TypeTestAssumeVCalls);
@@ -204,7 +199,7 @@ template <> struct MappingTraits<GlobalValueSummaryYaml> {
 } // End yaml namespace
 } // End llvm namespace
 
-LLVM_YAML_IS_SEQUENCE_VECTOR(GlobalValueSummaryYaml)
+LLVM_YAML_IS_SEQUENCE_VECTOR(FunctionSummaryYaml)
 
 namespace llvm {
 namespace yaml {
@@ -212,99 +207,61 @@ namespace yaml {
 // FIXME: Add YAML mappings for the rest of the module summary.
 template <> struct CustomMappingTraits<GlobalValueSummaryMapTy> {
   static void inputOne(IO &io, StringRef Key, GlobalValueSummaryMapTy &V) {
-    std::vector<GlobalValueSummaryYaml> GVSums;
-    io.mapRequired(Key.str().c_str(), GVSums);
+    std::vector<FunctionSummaryYaml> FSums;
+    io.mapRequired(Key.str().c_str(), FSums);
     uint64_t KeyInt;
     if (Key.getAsInteger(0, KeyInt)) {
       io.setError("key not an integer");
       return;
     }
     auto &Elem = V.try_emplace(KeyInt, /*IsAnalysis=*/false).first->second;
-    for (auto &GVSum : GVSums) {
-      GlobalValueSummary::GVFlags GVFlags(
-          static_cast<GlobalValue::LinkageTypes>(GVSum.Linkage),
-          static_cast<GlobalValue::VisibilityTypes>(GVSum.Visibility),
-          GVSum.NotEligibleToImport, GVSum.Live, GVSum.IsLocal,
-          GVSum.CanAutoHide,
-          static_cast<GlobalValueSummary::ImportKind>(GVSum.ImportType));
-      if (GVSum.Aliasee) {
-        auto ASum = std::make_unique<AliasSummary>(GVFlags);
-        if (!V.count(*GVSum.Aliasee))
-          V.emplace(*GVSum.Aliasee, /*IsAnalysis=*/false);
-        ValueInfo AliaseeVI(/*IsAnalysis=*/false, &*V.find(*GVSum.Aliasee));
-        // Note: Aliasee cannot be filled until all summaries are loaded.
-        // This is done in fixAliaseeLinks() which is called in
-        // MappingTraits<ModuleSummaryIndex>::mapping().
-        ASum->setAliasee(AliaseeVI, /*Aliasee=*/nullptr);
-        Elem.SummaryList.push_back(std::move(ASum));
-        continue;
-      }
+    for (auto &FSum : FSums) {
       SmallVector<ValueInfo, 0> Refs;
-      Refs.reserve(GVSum.Refs.size());
-      for (auto &RefGUID : GVSum.Refs) {
+      Refs.reserve(FSum.Refs.size());
+      for (auto &RefGUID : FSum.Refs) {
         auto It = V.try_emplace(RefGUID, /*IsAnalysis=*/false).first;
         Refs.push_back(ValueInfo(/*IsAnalysis=*/false, &*It));
       }
       Elem.SummaryList.push_back(std::make_unique<FunctionSummary>(
-          GVFlags, /*NumInsts=*/0, FunctionSummary::FFlags{}, std::move(Refs),
-          SmallVector<FunctionSummary::EdgeTy, 0>{}, std::move(GVSum.TypeTests),
-          std::move(GVSum.TypeTestAssumeVCalls),
-          std::move(GVSum.TypeCheckedLoadVCalls),
-          std::move(GVSum.TypeTestAssumeConstVCalls),
-          std::move(GVSum.TypeCheckedLoadConstVCalls),
+          GlobalValueSummary::GVFlags(
+              static_cast<GlobalValue::LinkageTypes>(FSum.Linkage),
+              static_cast<GlobalValue::VisibilityTypes>(FSum.Visibility),
+              FSum.NotEligibleToImport, FSum.Live, FSum.IsLocal,
+              FSum.CanAutoHide,
+              static_cast<GlobalValueSummary::ImportKind>(FSum.ImportType)),
+          /*NumInsts=*/0, FunctionSummary::FFlags{}, std::move(Refs),
+          SmallVector<FunctionSummary::EdgeTy, 0>{}, std::move(FSum.TypeTests),
+          std::move(FSum.TypeTestAssumeVCalls),
+          std::move(FSum.TypeCheckedLoadVCalls),
+          std::move(FSum.TypeTestAssumeConstVCalls),
+          std::move(FSum.TypeCheckedLoadConstVCalls),
           ArrayRef<FunctionSummary::ParamAccess>{}, ArrayRef<CallsiteInfo>{},
           ArrayRef<AllocInfo>{}));
     }
   }
   static void output(IO &io, GlobalValueSummaryMapTy &V) {
     for (auto &P : V) {
-      std::vector<GlobalValueSummaryYaml> GVSums;
+      std::vector<FunctionSummaryYaml> FSums;
       for (auto &Sum : P.second.SummaryList) {
         if (auto *FSum = dyn_cast<FunctionSummary>(Sum.get())) {
           std::vector<uint64_t> Refs;
           Refs.reserve(FSum->refs().size());
           for (auto &VI : FSum->refs())
             Refs.push_back(VI.getGUID());
-          GVSums.push_back(GlobalValueSummaryYaml{
+          FSums.push_back(FunctionSummaryYaml{
               FSum->flags().Linkage, FSum->flags().Visibility,
               static_cast<bool>(FSum->flags().NotEligibleToImport),
               static_cast<bool>(FSum->flags().Live),
               static_cast<bool>(FSum->flags().DSOLocal),
               static_cast<bool>(FSum->flags().CanAutoHide),
-              FSum->flags().ImportType, /*Aliasee=*/std::nullopt, Refs,
-              FSum->type_tests(), FSum->type_test_assume_vcalls(),
-              FSum->type_checked_load_vcalls(),
+              FSum->flags().ImportType, Refs, FSum->type_tests(),
+              FSum->type_test_assume_vcalls(), FSum->type_checked_load_vcalls(),
               FSum->type_test_assume_const_vcalls(),
               FSum->type_checked_load_const_vcalls()});
-        } else if (auto *ASum = dyn_cast<AliasSummary>(Sum.get());
-                   ASum && ASum->hasAliasee()) {
-          GVSums.push_back(GlobalValueSummaryYaml{
-              ASum->flags().Linkage, ASum->flags().Visibility,
-              static_cast<bool>(ASum->flags().NotEligibleToImport),
-              static_cast<bool>(ASum->flags().Live),
-              static_cast<bool>(ASum->flags().DSOLocal),
-              static_cast<bool>(ASum->flags().CanAutoHide),
-              ASum->flags().ImportType,
-              /*Aliasee=*/ASum->getAliaseeGUID()});
-        }
-      }
-      if (!GVSums.empty())
-        io.mapRequired(llvm::utostr(P.first).c_str(), GVSums);
-    }
-  }
-  static void fixAliaseeLinks(GlobalValueSummaryMapTy &V) {
-    for (auto &P : V) {
-      for (auto &Sum : P.second.SummaryList) {
-        if (auto *Alias = dyn_cast<AliasSummary>(Sum.get())) {
-          ValueInfo AliaseeVI = Alias->getAliaseeVI();
-          auto AliaseeSL = AliaseeVI.getSummaryList();
-          if (AliaseeSL.empty()) {
-            ValueInfo EmptyVI;
-            Alias->setAliasee(EmptyVI, nullptr);
-          } else
-            Alias->setAliasee(AliaseeVI, AliaseeSL[0].get());
-        }
+          }
       }
+      if (!FSums.empty())
+        io.mapRequired(llvm::utostr(P.first).c_str(), FSums);
     }
   }
 };
@@ -324,9 +281,6 @@ template <> struct CustomMappingTraits<TypeIdSummaryMapTy> {
 template <> struct MappingTraits<ModuleSummaryIndex> {
   static void mapping(IO &io, ModuleSummaryIndex& index) {
     io.mapOptional("GlobalValueMap", index.GlobalValueMap);
-    if (!io.outputting())
-      CustomMappingTraits<GlobalValueSummaryMapTy>::fixAliaseeLinks(
-          index.GlobalValueMap);
     io.mapOptional("TypeIdMap", index.TypeIdMap);
     io.mapOptional("WithGlobalValueDeadStripping",
                    index.WithGlobalValueDeadStripping);
diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index 6ba371069bb2..3fcfc6a87677 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -2083,12 +2083,8 @@ bool LowerTypeTestsModule::lower() {
     for (auto &I : *ExportSummary)
       for (auto &GVS : I.second.SummaryList)
         if (GVS->isLive())
-          for (const auto &Ref : GVS->refs()) {
+          for (const auto &Ref : GVS->refs())
             AddressTaken.insert(Ref.getGUID());
-            for (auto &RefGVS : Ref.getSummaryList())
-              if (auto Alias = dyn_cast<AliasSummary>(RefGVS.get()))
-                AddressTaken.insert(Alias->getAliaseeGUID());
-          }
 
     NamedMDNode *CfiFunctionsMD = M.getNamedMetadata("cfi.functions");
     if (CfiFunctionsMD) {
diff --git a/llvm/test/Transforms/LowerTypeTests/cfi-icall-alias.ll b/llvm/test/Transforms/LowerTypeTests/cfi-icall-alias.ll
deleted file mode 100644
index 0c5324ee96c9..000000000000
--- a/llvm/test/Transforms/LowerTypeTests/cfi-icall-alias.ll
+++ /dev/null
@@ -1,54 +0,0 @@
-;; Check that if the address of a weak function is only taken through an alias,
-;; it is still added to a list of exported functions and @llvm.type.test() is
-;; lowered to an actual check against the generated CFI jumptable.
-
-RUN: rm -rf %t.dir && split-file %s %t.dir && cd %t.dir
-RUN: opt test.ll --thinlto-bc --thinlto-split-lto-unit -o test.bc
-RUN: llvm-modextract test.bc -n 0 -o test0.bc
-RUN: llvm-modextract test.bc -n 1 -o test1.bc
-
-;; Check that a CFI jumptable is generated.
-RUN: opt test1.bc -passes=lowertypetests -lowertypetests-read-summary=in.yaml \
-RUN:   -lowertypetests-summary-action=export -lowertypetests-write-summary=exported.yaml \
-RUN:   -S -o - | FileCheck %s --check-prefix=REGULAR
-REGULAR: @__typeid__ZTSFvvE_global_addr = hidden alias i8, ptr @.cfi.jumptable
-REGULAR: @f = alias void (), ptr @.cfi.jumptable
-REGULAR: define private void @.cfi.jumptable()
-
-;; CHECK that @llvm.type.test() is lowered to an actual check.
-RUN: opt test0.bc -passes=lowertypetests -lowertypetests-read-summary=exported.yaml \
-RUN:   -lowertypetests-summary-action=import -S -o - | FileCheck %s --check-prefix=THIN
-THIN:      define i1 @test() {
-THIN-NEXT:   %1 = icmp eq i64 ptrtoint (ptr @alias to i64), ptrtoint (ptr @__typeid__ZTSFvvE_global_addr to i64)
-THIN-NEXT:   ret i1 %1
-THIN-NEXT: }
-
-;--- test.ll
-target triple = "x86_64-pc-linux-gnu"
-
-@alias = alias void(), ptr @f
-
-define weak void @f() !type !0 {
-  ret void
-}
-
-define i1 @test() {
-  %1 = call i1 @llvm.type.test(ptr nonnull @alias, metadata !"_ZTSFvvE")
-  ret i1 %1
-}
-
-declare i1 @llvm.type.test(ptr, metadata)
-
-!0 = !{i64 0, !"_ZTSFvvE"}
-;--- in.yaml
----
-GlobalValueMap:
-  8346051122425466633: # guid("test")
-    - Live: true
-      Refs: [5833419078793185394] # guid("alias")
-      TypeTests: [9080559750644022485] # guid("_ZTSFvvE")
-  5833419078793185394: # guid("alias")
-    - Aliasee: 14740650423002898831 # guid("f")
-  14740650423002898831: # guid("f")
-    -
-...
-- 
GitLab


From 8e6856e27859c90c5337a8328848b0959fe409fe Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Mon, 28 Oct 2024 16:15:39 -0700
Subject: [PATCH 384/693] [Clang][TableGen] Use StringRef::str() instead of
 std::string() cast (#113645)

Use `StringRef::str()` instead of std::string(StringRef) to cast from
StringRef to std::string.
---
 clang/utils/TableGen/ClangAttrEmitter.cpp     | 65 +++++++++----------
 .../ClangCommentCommandInfoEmitter.cpp        |  2 +-
 ...mentHTMLNamedCharacterReferenceEmitter.cpp |  2 +-
 .../TableGen/ClangCommentHTMLTagsEmitter.cpp  |  4 +-
 4 files changed, 33 insertions(+), 40 deletions(-)

diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp
index cf9c70a93e5d..3031d81b3df7 100644
--- a/clang/utils/TableGen/ClangAttrEmitter.cpp
+++ b/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -122,9 +122,7 @@ GetFlattenedSpellings(const Record &Attr) {
 
 static std::string ReadPCHRecord(StringRef type) {
   return StringSwitch<std::string>(type)
-      .EndsWith("Decl *", "Record.readDeclAs<" +
-                              std::string(type.data(), 0, type.size() - 1) +
-                              ">()")
+      .EndsWith("Decl *", "Record.readDeclAs<" + type.drop_back().str() + ">()")
       .Case("TypeSourceInfo *", "Record.readTypeSourceInfo()")
       .Case("Expr *", "Record.readExpr()")
       .Case("IdentifierInfo *", "Record.readIdentifier()")
@@ -145,18 +143,16 @@ static StringRef getStorageType(StringRef type) {
 static std::string WritePCHRecord(StringRef type, StringRef name) {
   return "Record." +
          StringSwitch<std::string>(type)
-             .EndsWith("Decl *", "AddDeclRef(" + std::string(name) + ");\n")
+             .EndsWith("Decl *", "AddDeclRef(" + name.str() + ");\n")
              .Case("TypeSourceInfo *",
-                   "AddTypeSourceInfo(" + std::string(name) + ");\n")
-             .Case("Expr *", "AddStmt(" + std::string(name) + ");\n")
+                   "AddTypeSourceInfo(" + name.str() + ");\n")
+             .Case("Expr *", "AddStmt(" + name.str() + ");\n")
              .Case("IdentifierInfo *",
-                   "AddIdentifierRef(" + std::string(name) + ");\n")
-             .Case("StringRef", "AddString(" + std::string(name) + ");\n")
-             .Case("ParamIdx",
-                   "push_back(" + std::string(name) + ".serialize());\n")
-             .Case("OMPTraitInfo *",
-                   "writeOMPTraitInfo(" + std::string(name) + ");\n")
-             .Default("push_back(" + std::string(name) + ");\n");
+                   "AddIdentifierRef(" + name.str() + ");\n")
+             .Case("StringRef", "AddString(" + name.str() + ");\n")
+             .Case("ParamIdx", "push_back(" + name.str() + ".serialize());\n")
+             .Case("OMPTraitInfo *", "writeOMPTraitInfo(" + name.str() + ");\n")
+             .Default("push_back(" + name.str() + ");\n");
 }
 
 // Normalize attribute name by removing leading and trailing
@@ -197,7 +193,7 @@ static ParsedAttrMap getParsedAttrList(const RecordKeeper &Records,
       std::string AN;
       if (Attr->isSubClassOf("TargetSpecificAttr") &&
           !Attr->isValueUnset("ParseKind")) {
-        AN = std::string(Attr->getValueAsString("ParseKind"));
+        AN = Attr->getValueAsString("ParseKind").str();
 
         // If this attribute has already been handled, it does not need to be
         // handled again.
@@ -225,7 +221,7 @@ namespace {
 
   public:
     Argument(StringRef Arg, StringRef Attr)
-        : lowerName(std::string(Arg)), upperName(lowerName), attrName(Attr),
+        : lowerName(Arg.str()), upperName(lowerName), attrName(Attr),
           isOpt(false), Fake(false) {
       if (!lowerName.empty()) {
         lowerName[0] = std::tolower(lowerName[0]);
@@ -331,8 +327,7 @@ namespace {
 
     void writePCHWrite(raw_ostream &OS) const override {
       OS << "    "
-         << WritePCHRecord(type,
-                           "SA->get" + std::string(getUpperName()) + "()");
+         << WritePCHRecord(type, "SA->get" + getUpperName().str() + "()");
     }
 
     std::string getIsOmitted() const override {
@@ -698,12 +693,12 @@ namespace {
     VariadicArgument(const Record &Arg, StringRef Attr, std::string T)
         : Argument(Arg, Attr), Type(std::move(T)),
           ArgName(getLowerName().str() + "_"), ArgSizeName(ArgName + "Size"),
-          RangeName(std::string(getLowerName())) {}
+          RangeName(getLowerName().str()) {}
 
     VariadicArgument(StringRef Arg, StringRef Attr, std::string T)
         : Argument(Arg, Attr), Type(std::move(T)),
           ArgName(getLowerName().str() + "_"), ArgSizeName(ArgName + "Size"),
-          RangeName(std::string(getLowerName())) {}
+          RangeName(getLowerName().str()) {}
 
     const std::string &getType() const { return Type; }
     const std::string &getArgName() const { return ArgName; }
@@ -792,8 +787,8 @@ namespace {
       // If we can't store the values in the current type (if it's something
       // like StringRef), store them in a different type and convert the
       // container afterwards.
-      std::string StorageType = std::string(getStorageType(getType()));
-      std::string StorageName = std::string(getLowerName());
+      std::string StorageType = getStorageType(getType()).str();
+      std::string StorageName = getLowerName().str();
       if (StorageType != getType()) {
         StorageName += "Storage";
         OS << "    SmallVector<" << StorageType << ", 4> "
@@ -1081,8 +1076,7 @@ namespace {
 
   public:
     VariadicEnumArgument(const Record &Arg, StringRef Attr)
-        : VariadicArgument(Arg, Attr,
-                           std::string(Arg.getValueAsString("Type"))),
+        : VariadicArgument(Arg, Attr, Arg.getValueAsString("Type").str()),
           values(Arg.getValueAsListOfStrings("Values")),
           enums(Arg.getValueAsListOfStrings("Enums")),
           uniques(uniqueEnumsInOrder(enums)),
@@ -1437,7 +1431,7 @@ namespace {
     void writePCHWrite(raw_ostream &OS) const override {
       OS << "    "
          << WritePCHRecord(getType(),
-                           "SA->get" + std::string(getUpperName()) + "Loc()");
+                           "SA->get" + getUpperName().str() + "Loc()");
     }
   };
 
@@ -1766,11 +1760,10 @@ static void writeAttrAccessorDefinition(const Record &R, raw_ostream &OS) {
 static bool
 SpellingNamesAreCommon(const std::vector<FlattenedSpelling>& Spellings) {
   assert(!Spellings.empty() && "An empty list of spellings was provided");
-  std::string FirstName =
-      std::string(NormalizeNameForSpellingComparison(Spellings.front().name()));
+  StringRef FirstName =
+      NormalizeNameForSpellingComparison(Spellings.front().name());
   for (const auto &Spelling : drop_begin(Spellings)) {
-    std::string Name =
-        std::string(NormalizeNameForSpellingComparison(Spelling.name()));
+    StringRef Name = NormalizeNameForSpellingComparison(Spelling.name());
     if (Name != FirstName)
       return false;
   }
@@ -1985,7 +1978,7 @@ struct AttributeSubjectMatchRule {
   }
 
   std::string getSpelling() const {
-    std::string Result = std::string(MetaSubject->getValueAsString("Name"));
+    std::string Result = MetaSubject->getValueAsString("Name").str();
     if (isSubRule()) {
       Result += '(';
       if (isNegatedSubRule())
@@ -2728,7 +2721,7 @@ static void emitAttributes(const RecordKeeper &Records, raw_ostream &OS,
     for (const auto &[R, _] : reverse(Supers)) {
       if (R->getName() != "TargetSpecificAttr" &&
           R->getName() != "DeclOrTypeAttr" && SuperName.empty())
-        SuperName = std::string(R->getName());
+        SuperName = R->getName().str();
       if (R->getName() == "InheritableAttr")
         Inheritable = true;
     }
@@ -4054,9 +4047,9 @@ static void emitArgInfo(const Record &R, raw_ostream &OS) {
 }
 
 static std::string GetDiagnosticSpelling(const Record &R) {
-  std::string Ret = std::string(R.getValueAsString("DiagSpelling"));
+  StringRef Ret = R.getValueAsString("DiagSpelling");
   if (!Ret.empty())
-    return Ret;
+    return Ret.str();
 
   // If we couldn't find the DiagSpelling in this object, we can check to see
   // if the object is one that has a base, and if it is, loop up to the Base
@@ -4089,7 +4082,7 @@ static std::string CalculateDiagnostic(const Record &S) {
       SmallVector<StringRef, 2> Frags;
       SplitString(V, Frags, ",");
       for (auto Str : Frags) {
-        DiagList.push_back(std::string(Str.trim()));
+        DiagList.push_back(Str.trim().str());
       }
     }
   }
@@ -4120,7 +4113,7 @@ static std::string CalculateDiagnostic(const Record &S) {
 }
 
 static std::string GetSubjectWithSuffix(const Record *R) {
-  const std::string &B = std::string(R->getName());
+  const std::string B = R->getName().str();
   if (B == "DeclBase")
     return "Decl";
   return B + "Decl";
@@ -5107,7 +5100,7 @@ GetAttributeHeadingAndSpellings(const Record &Documentation,
                     "documented");
 
   // Determine the heading to be used for this attribute.
-  std::string Heading = std::string(Documentation.getValueAsString("Heading"));
+  std::string Heading = Documentation.getValueAsString("Heading").str();
   if (Heading.empty()) {
     // If there's only one spelling, we can simply use that.
     if (Spellings.size() == 1)
@@ -5117,7 +5110,7 @@ GetAttributeHeadingAndSpellings(const Record &Documentation,
       for (auto I = Spellings.begin(), E = Spellings.end();
            I != E; ++I) {
         std::string Spelling =
-            std::string(NormalizeNameForSpellingComparison(I->name()));
+            NormalizeNameForSpellingComparison(I->name()).str();
         Uniques.insert(Spelling);
       }
       // If the semantic map has only one spelling, that is sufficient for our
diff --git a/clang/utils/TableGen/ClangCommentCommandInfoEmitter.cpp b/clang/utils/TableGen/ClangCommentCommandInfoEmitter.cpp
index 1a2503dcf660..45a97425ef92 100644
--- a/clang/utils/TableGen/ClangCommentCommandInfoEmitter.cpp
+++ b/clang/utils/TableGen/ClangCommentCommandInfoEmitter.cpp
@@ -63,7 +63,7 @@ void clang::EmitClangCommentCommandInfo(const RecordKeeper &Records,
   std::vector<StringMatcher::StringPair> Matches;
   for (size_t i = 0, e = Tags.size(); i != e; ++i) {
     const Record &Tag = *Tags[i];
-    std::string Name = std::string(Tag.getValueAsString("Name"));
+    std::string Name = Tag.getValueAsString("Name").str();
     std::string Return;
     raw_string_ostream(Return) << "return &Commands[" << i << "];";
     Matches.emplace_back(std::move(Name), std::move(Return));
diff --git a/clang/utils/TableGen/ClangCommentHTMLNamedCharacterReferenceEmitter.cpp b/clang/utils/TableGen/ClangCommentHTMLNamedCharacterReferenceEmitter.cpp
index bd75b3f6b652..2d615760814e 100644
--- a/clang/utils/TableGen/ClangCommentHTMLNamedCharacterReferenceEmitter.cpp
+++ b/clang/utils/TableGen/ClangCommentHTMLNamedCharacterReferenceEmitter.cpp
@@ -51,7 +51,7 @@ void clang::EmitClangCommentHTMLNamedCharacterReferences(
   std::vector<StringMatcher::StringPair> NameToUTF8;
   SmallString<32> CLiteral;
   for (const Record *Tag : Records.getAllDerivedDefinitions("NCR")) {
-    std::string Spelling = std::string(Tag->getValueAsString("Spelling"));
+    std::string Spelling = Tag->getValueAsString("Spelling").str();
     uint64_t CodePoint = Tag->getValueAsInt("CodePoint");
     CLiteral.clear();
     CLiteral.append("return ");
diff --git a/clang/utils/TableGen/ClangCommentHTMLTagsEmitter.cpp b/clang/utils/TableGen/ClangCommentHTMLTagsEmitter.cpp
index a457315bc62c..7d65cfe0d3f5 100644
--- a/clang/utils/TableGen/ClangCommentHTMLTagsEmitter.cpp
+++ b/clang/utils/TableGen/ClangCommentHTMLTagsEmitter.cpp
@@ -24,7 +24,7 @@ void clang::EmitClangCommentHTMLTags(const RecordKeeper &Records,
   ArrayRef<const Record *> Tags = Records.getAllDerivedDefinitions("Tag");
   std::vector<StringMatcher::StringPair> Matches;
   for (const Record *Tag : Tags) {
-    Matches.emplace_back(std::string(Tag->getValueAsString("Spelling")),
+    Matches.emplace_back(Tag->getValueAsString("Spelling").str(),
                          "return true;");
   }
 
@@ -42,7 +42,7 @@ void clang::EmitClangCommentHTMLTagsProperties(const RecordKeeper &Records,
   std::vector<StringMatcher::StringPair> MatchesEndTagOptional;
   std::vector<StringMatcher::StringPair> MatchesEndTagForbidden;
   for (const Record *Tag : Tags) {
-    std::string Spelling = std::string(Tag->getValueAsString("Spelling"));
+    std::string Spelling = Tag->getValueAsString("Spelling").str();
     StringMatcher::StringPair Match(Spelling, "return true;");
     if (Tag->getValueAsBit("EndTagOptional"))
       MatchesEndTagOptional.push_back(Match);
-- 
GitLab


From 7c554265ce0b94059f216dcab643055e98c8f439 Mon Sep 17 00:00:00 2001
From: Chris Apple <cja-private@pm.me>
Date: Mon, 28 Oct 2024 16:55:27 -0700
Subject: [PATCH 385/693] Revert "[rtsan] Intercept aligned_alloc on all
 versions of OSX if available on build machine (#112780)" (#113982)

This reverts commit 97fb21ac1d6bc528b61a555356457ff2129dfde1.

Due to issue brought up in #112780

> Unfortunately this breaks the build on our (automerger) bots, which
have -mmacosx-version-min=10.13 and also
-Werror=unguarded-availability-new . I was thinking about patching it
via wrapping in __builtin_available check (which I believe is the right
one to use, as it should match the -mmacosx-version-min ) - but can't
actually think of a quick fix, due to interceptors being defined via C
macros.

>
llvm-project/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp:475:21:
error: 'aligned_alloc' is only available on macOS 10.15 or newer
[-Werror,-Wunguarded-availability-new] 475 | INTERCEPTOR(void *,
aligned_alloc, SIZE_T alignment, SIZE_T size) {
---
 .../tests/rtsan_test_interceptors_posix.cpp   | 17 +++++---------
 .../sanitizer_platform_interceptors.h         | 22 +------------------
 2 files changed, 6 insertions(+), 33 deletions(-)

diff --git a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
index 38274485c29f..6233c3e91800 100644
--- a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
+++ b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
@@ -122,20 +122,13 @@ TEST(TestRtsanInterceptors, VallocDiesWhenRealtime) {
   ExpectNonRealtimeSurvival(Func);
 }
 
-#if __has_builtin(__builtin_available) && SANITIZER_APPLE
-#define ALIGNED_ALLOC_AVAILABLE() (__builtin_available(macOS 10.15, *))
-#else
-// We are going to assume this is true until we hit systems where it isn't
-#define ALIGNED_ALLOC_AVAILABLE() (true)
-#endif
-
+#if SANITIZER_INTERCEPT_ALIGNED_ALLOC
 TEST(TestRtsanInterceptors, AlignedAllocDiesWhenRealtime) {
-  if (ALIGNED_ALLOC_AVAILABLE()) {
-    auto Func = []() { EXPECT_NE(nullptr, aligned_alloc(16, 32)); };
-    ExpectRealtimeDeath(Func, "aligned_alloc");
-    ExpectNonRealtimeSurvival(Func);
-  }
+  auto Func = []() { EXPECT_NE(nullptr, aligned_alloc(16, 32)); };
+  ExpectRealtimeDeath(Func, "aligned_alloc");
+  ExpectNonRealtimeSurvival(Func);
 }
+#endif
 
 // free_sized and free_aligned_sized (both C23) are not yet supported
 TEST(TestRtsanInterceptors, FreeDiesWhenRealtime) {
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
index 3fd6b595ef19..6959a6d52d60 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
@@ -84,25 +84,6 @@
 #define SI_NOT_MAC 1
 #endif
 
-#if SANITIZER_APPLE
-#  include <Availability.h>
-
-// aligned_alloc was introduced in OSX 10.15
-// Linking will fail when using an older SDK
-#  if defined(__MAC_10_15)
-// macOS 10.15 is greater than our minimal deployment target.  To ensure we
-// generate a weak reference so the dylib continues to work on older
-// systems, we need to forward declare the intercepted function as "weak
-// imports".
-SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
-                                          __sanitizer::usize __size);
-#    define SI_MAC_SDK_10_15_AVAILABLE 1
-#  else
-#    define SI_MAC_SDK_10_15_AVAILABLE 0
-#  endif  // defined(__MAC_10_15)
-
-#endif  // SANITIZER_APPLE
-
 #if SANITIZER_IOS
 #define SI_IOS 1
 #else
@@ -519,8 +500,7 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
 #define SANITIZER_INTERCEPT_PVALLOC (SI_GLIBC || SI_ANDROID)
 #define SANITIZER_INTERCEPT_CFREE (SI_GLIBC && !SANITIZER_RISCV64)
 #define SANITIZER_INTERCEPT_REALLOCARRAY SI_POSIX
-#define SANITIZER_INTERCEPT_ALIGNED_ALLOC \
-  (!SI_MAC || SI_MAC_SDK_10_15_AVAILABLE)
+#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC)
 #define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD)
 #define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_WCSLEN 1
-- 
GitLab


From b46a0482f9e4c0ee82b38da794b20f8f1a76f044 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Tue, 29 Oct 2024 00:56:03 +0100
Subject: [PATCH 386/693] [clang][bytecode] Implement
 __builtin_arithmetic_fence (#113937)

---
 clang/lib/AST/ByteCode/InterpBuiltin.cpp   | 14 ++++++++++++++
 clang/test/Sema/arithmetic-fence-builtin.c |  5 +++++
 2 files changed, 19 insertions(+)

diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index 10e33c14f4b4..b00d2a1768b6 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -1670,6 +1670,15 @@ static bool interp__builtin_operator_delete(InterpState &S, CodePtr OpPC,
       S, OpPC, *AllocForm, DynamicAllocator::Form::Operator, BlockDesc, Source);
 }
 
+static bool interp__builtin_arithmetic_fence(InterpState &S, CodePtr OpPC,
+                                             const InterpFrame *Frame,
+                                             const Function *Func,
+                                             const CallExpr *Call) {
+  const Floating &Arg0 = S.Stk.peek<Floating>();
+  S.Stk.push<Floating>(Arg0);
+  return true;
+}
+
 bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
                       const CallExpr *Call, uint32_t BuiltinID) {
   const InterpFrame *Frame = S.Current;
@@ -2111,6 +2120,11 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
       return false;
     break;
 
+  case Builtin::BI__arithmetic_fence:
+    if (!interp__builtin_arithmetic_fence(S, OpPC, Frame, F, Call))
+      return false;
+    break;
+
   default:
     S.FFDiag(S.Current->getLocation(OpPC),
              diag::note_invalid_subexpr_in_const_expr)
diff --git a/clang/test/Sema/arithmetic-fence-builtin.c b/clang/test/Sema/arithmetic-fence-builtin.c
index a1941970edb5..55867ffb5e01 100644
--- a/clang/test/Sema/arithmetic-fence-builtin.c
+++ b/clang/test/Sema/arithmetic-fence-builtin.c
@@ -1,8 +1,13 @@
 // RUN: %clang_cc1 -triple i386-pc-linux-gnu -emit-llvm -o - -verify -x c++ %s
+// RUN: %clang_cc1 -triple i386-pc-linux-gnu -emit-llvm -o - -verify -x c++ %s -fexperimental-new-constant-interpreter
 // RUN: %clang_cc1 -triple ppc64le -DPPC     -emit-llvm -o - -verify -x c++ %s
+// RUN: %clang_cc1 -triple ppc64le -DPPC     -emit-llvm -o - -verify -x c++ %s -fexperimental-new-constant-interpreter
 // RUN: not %clang_cc1 -triple ppc64le -DPPC     -emit-llvm -o - -x c++ %s \
 // RUN:            -fprotect-parens 2>&1 | FileCheck -check-prefix=PPC %s
+// RUN: not %clang_cc1 -triple ppc64le -DPPC     -emit-llvm -o - -x c++ %s -fexperimental-new-constant-interpreter \
+// RUN:            -fprotect-parens 2>&1 | FileCheck -check-prefix=PPC %s
 // RUN: %clang_cc1 -triple spir64 -emit-llvm -o - -verify -x c++ %s
+// RUN: %clang_cc1 -triple spir64 -emit-llvm -o - -verify -x c++ %s -fexperimental-new-constant-interpreter
 #ifndef PPC
 int v;
 template <typename T> T addT(T a, T b) {
-- 
GitLab


From 1549a0c183ee337a6de4c3933e10828808c6a094 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Tue, 29 Oct 2024 09:10:30 +0900
Subject: [PATCH 387/693] [mlir][SCF] Remove `scf-bufferize` pass (#113840)

The dialect conversion-based bufferization passes have been migrated to
One-Shot Bufferize about two years ago. To clean up the code base, this
commit removes the `scf-bufferize` pass, one of the few remaining parts
of the old infrastructure. Most bufferization passes have already been
removed.

Note for LLVM integration: If you depend on this pass, migrate to
One-Shot Bufferize or copy the pass to your codebase.
---
 mlir/docs/Bufferization.md                    | 17 +------
 .../mlir/Dialect/SCF/Transforms/Passes.h      |  3 --
 .../mlir/Dialect/SCF/Transforms/Passes.td     |  7 ---
 .../BufferizableOpInterfaceImpl.cpp           |  6 ++-
 mlir/lib/Dialect/SCF/Transforms/Bufferize.cpp | 47 -------------------
 .../lib/Dialect/SCF/Transforms/CMakeLists.txt |  1 -
 mlir/test/Dialect/SCF/bufferize.mlir          | 34 +++++++++++---
 7 files changed, 32 insertions(+), 83 deletions(-)
 delete mode 100644 mlir/lib/Dialect/SCF/Transforms/Bufferize.cpp

diff --git a/mlir/docs/Bufferization.md b/mlir/docs/Bufferization.md
index d5a426e09e7c..7d38ebb38535 100644
--- a/mlir/docs/Bufferization.md
+++ b/mlir/docs/Bufferization.md
@@ -579,7 +579,6 @@ The code, slightly simplified and annotated, is reproduced here:
   // Partial bufferization passes.
   pm.addPass(createTensorConstantBufferizePass());
   pm.addNestedPass<func::FuncOp>(createTCPBufferizePass()); // Bufferizes the downstream `tcp` dialect.
-  pm.addNestedPass<func::FuncOp>(createSCFBufferizePass());
   pm.addNestedPass<func::FuncOp>(createLinalgBufferizePass());
   pm.addNestedPass<func::FuncOp>(createTensorBufferizePass());
   pm.addPass(createFuncBufferizePass());
@@ -596,7 +595,7 @@ must be module passes because they make changes to the top-level module.
 
 The bulk of the bufferization work is done by the function passes. Most of these
 passes are provided as part of the upstream MLIR distribution and bufferize
-their respective dialects (e.g. `scf-bufferize` bufferizes the `scf` dialect).
+their respective dialects (e.g. `abc-bufferize` bufferizes the `abc` dialect).
 The `tcp-bufferize` pass is an exception -- it is a partial bufferization pass
 used to bufferize the downstream `tcp` dialect, and fits in perfectly with all
 the other passes provided upstream.
@@ -694,20 +693,6 @@ which helps with this in general.
 
 ### Other partial bufferization examples
 
--   `scf-bufferize`
-    ([code](https://github.com/llvm/llvm-project/blob/bc8acf2ce8ad6e8c9b1d97b2e02d3f4ad26e1d9d/mlir/lib/Dialect/SCF/Transforms/Bufferize.cpp#L1),
-    [test](https://github.com/llvm/llvm-project/blob/bc8acf2ce8ad6e8c9b1d97b2e02d3f4ad26e1d9d/mlir/test/Dialect/SCF/bufferize.mlir#L1))
-
-    -   Bufferizes ops from the `scf` dialect.
-    -   This is an example of how to bufferize ops that implement
-        `RegionBranchOpInterface` (that is, they use regions to represent
-        control flow).
-    -   The bulk of the work is done by
-        `lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp`
-        ([code](https://github.com/llvm/llvm-project/blob/daaaed6bb89044ac58a23f1bb1ccdd12342a5a58/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp#L1)),
-        which is well-commented and covers how to correctly convert ops that
-        contain regions.
-
 -   `func-bufferize`
     ([code](https://github.com/llvm/llvm-project/blob/2f5715dc78328215d51d5664c72c632a6dac1046/mlir/lib/Dialect/Func/Transforms/FuncBufferize.cpp#L1),
     [test](https://github.com/llvm/llvm-project/blob/2f5715dc78328215d51d5664c72c632a6dac1046/mlir/test/Dialect/Func/func-bufferize.mlir#L1))
diff --git a/mlir/include/mlir/Dialect/SCF/Transforms/Passes.h b/mlir/include/mlir/Dialect/SCF/Transforms/Passes.h
index fb8411418ff9..b70599df6f50 100644
--- a/mlir/include/mlir/Dialect/SCF/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/SCF/Transforms/Passes.h
@@ -20,9 +20,6 @@ namespace mlir {
 #define GEN_PASS_DECL
 #include "mlir/Dialect/SCF/Transforms/Passes.h.inc"
 
-/// Creates a pass that bufferizes the SCF dialect.
-std::unique_ptr<Pass> createSCFBufferizePass();
-
 /// Creates a pass that specializes for loop for unrolling and
 /// vectorization.
 std::unique_ptr<Pass> createForLoopSpecializationPass();
diff --git a/mlir/include/mlir/Dialect/SCF/Transforms/Passes.td b/mlir/include/mlir/Dialect/SCF/Transforms/Passes.td
index 53d1ae10dc87..6e5ef96c450a 100644
--- a/mlir/include/mlir/Dialect/SCF/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/SCF/Transforms/Passes.td
@@ -11,13 +11,6 @@
 
 include "mlir/Pass/PassBase.td"
 
-def SCFBufferize : Pass<"scf-bufferize"> {
-  let summary = "Bufferize the scf dialect.";
-  let constructor = "mlir::createSCFBufferizePass()";
-  let dependentDialects = ["bufferization::BufferizationDialect",
-                           "memref::MemRefDialect"];
-}
-
 // Note: Making these canonicalization patterns would require a dependency
 // of the SCF dialect on the Affine/Tensor/MemRef dialects or vice versa.
 def SCFForLoopCanonicalization
diff --git a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
index cf40443ff383..779c41a22e9e 100644
--- a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -649,7 +649,8 @@ struct ForOpInterface
     if (failed(bufferizableOp.resolveTensorOpOperandConflicts(rewriter, state)))
       return failure();
 
-    if (!state.getOptions().enforceAliasingInvariants)
+    if (!state.getOptions().enforceAliasingInvariants ||
+        state.getOptions().copyBeforeWrite)
       return success();
 
     // According to the `getAliasing...` implementations, a bufferized OpResult
@@ -889,7 +890,8 @@ struct WhileOpInterface
     if (failed(bufferizableOp.resolveTensorOpOperandConflicts(rewriter, state)))
       return failure();
 
-    if (!state.getOptions().enforceAliasingInvariants)
+    if (!state.getOptions().enforceAliasingInvariants ||
+        state.getOptions().copyBeforeWrite)
       return success();
 
     // According to the `getAliasing...` implementations, a bufferized OpResult
diff --git a/mlir/lib/Dialect/SCF/Transforms/Bufferize.cpp b/mlir/lib/Dialect/SCF/Transforms/Bufferize.cpp
deleted file mode 100644
index 21c618ab633f..000000000000
--- a/mlir/lib/Dialect/SCF/Transforms/Bufferize.cpp
+++ /dev/null
@@ -1,47 +0,0 @@
-//===- Bufferize.cpp - scf bufferize pass ---------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/SCF/Transforms/Passes.h"
-
-#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
-#include "mlir/Dialect/Bufferization/Transforms/Bufferize.h"
-#include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Dialect/SCF/Transforms/Patterns.h"
-#include "mlir/Transforms/DialectConversion.h"
-
-namespace mlir {
-#define GEN_PASS_DEF_SCFBUFFERIZE
-#include "mlir/Dialect/SCF/Transforms/Passes.h.inc"
-} // namespace mlir
-
-using namespace mlir;
-using namespace mlir::scf;
-
-namespace {
-struct SCFBufferizePass : public impl::SCFBufferizeBase<SCFBufferizePass> {
-  void runOnOperation() override {
-    auto *func = getOperation();
-    auto *context = &getContext();
-
-    bufferization::BufferizeTypeConverter typeConverter;
-    RewritePatternSet patterns(context);
-    ConversionTarget target(*context);
-
-    bufferization::populateBufferizeMaterializationLegality(target);
-    populateSCFStructuralTypeConversionsAndLegality(typeConverter, patterns,
-                                                    target);
-    if (failed(applyPartialConversion(func, target, std::move(patterns))))
-      return signalPassFailure();
-  };
-};
-} // namespace
-
-std::unique_ptr<Pass> mlir::createSCFBufferizePass() {
-  return std::make_unique<SCFBufferizePass>();
-}
diff --git a/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt b/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt
index 5dc7c60792b9..e99b5d0cc26f 100644
--- a/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt
@@ -1,7 +1,6 @@
 add_mlir_dialect_library(MLIRSCFTransforms
   BufferDeallocationOpInterfaceImpl.cpp
   BufferizableOpInterfaceImpl.cpp
-  Bufferize.cpp
   ForallToFor.cpp
   ForallToParallel.cpp
   ForToWhile.cpp
diff --git a/mlir/test/Dialect/SCF/bufferize.mlir b/mlir/test/Dialect/SCF/bufferize.mlir
index ff1612310255..53fcee692226 100644
--- a/mlir/test/Dialect/SCF/bufferize.mlir
+++ b/mlir/test/Dialect/SCF/bufferize.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -scf-bufferize | FileCheck %s
+// RUN: mlir-opt %s -one-shot-bufferize="dialect-filter=scf,bufferization copy-before-write unknown-type-conversion=identity-layout-map" -split-input-file | FileCheck %s
 
 // CHECK-LABEL:   func @if(
 // CHECK-SAME:             %[[PRED:.*]]: i1,
@@ -23,15 +23,21 @@ func.func @if(%pred: i1, %true_val: tensor<?xf32>, %false_val: tensor<?xf32>) ->
   return %0 : tensor<?xf32>
 }
 
+// -----
+
 // CHECK-LABEL:   func @for(
 // CHECK-SAME:              %[[TENSOR:.*]]: tensor<f32>,
 // CHECK-SAME:              %[[LB:.*]]: index, %[[UB:.*]]: index,
 // CHECK-SAME:              %[[STEP:.*]]: index) -> tensor<f32> {
 // CHECK:           %[[MEMREF:.*]] = bufferization.to_memref %[[TENSOR]] : memref<f32>
-// CHECK:           %[[RESULT_MEMREF:.*]] = scf.for %[[VAL_6:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[ITER:.*]] = %[[MEMREF]]) -> (memref<f32>) {
+// Note: scf.for iter_args always bufferize to a memory write. This could be
+// optimized by analyzing the loop body.
+// CHECK:           %[[MEMREF_COPY:.*]] = memref.alloc()
+// CHECK:           memref.copy %[[MEMREF]], %[[MEMREF_COPY]]
+// CHECK:           %[[RESULT_MEMREF:.*]] = scf.for %{{.*}} = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[ITER:.*]] = %[[MEMREF_COPY]]) -> (memref<f32>) {
 // CHECK:             scf.yield %[[ITER]] : memref<f32>
 // CHECK:           } {some_attr}
-// CHECK:           %[[VAL_8:.*]] = bufferization.to_tensor %[[VAL_9:.*]] : memref<f32>
+// CHECK:           %[[VAL_8:.*]] = bufferization.to_tensor %[[RESULT_MEMREF]] : memref<f32>
 // CHECK:           return %[[VAL_8]] : tensor<f32>
 // CHECK:         }
 func.func @for(%arg0: tensor<f32>, %lb: index, %ub: index, %step: index) -> tensor<f32> {
@@ -41,6 +47,8 @@ func.func @for(%arg0: tensor<f32>, %lb: index, %ub: index, %step: index) -> tens
   return %ret : tensor<f32>
 }
 
+// -----
+
 // Check whether this converts at all.
 //
 // It would previously fail altogether.
@@ -57,17 +65,23 @@ func.func @if_correct_recursive_legalization_behavior(%pred: i1, %tensor: tensor
   return %0 : tensor<f32>
 }
 
+// -----
+
 // CHECK-LABEL:   func @for_correct_recursive_legalization_behavior(
 // CHECK-SAME:                                                      %[[TENSOR:.*]]: tensor<f32>,
 // CHECK-SAME:                                                      %[[INDEX:.*]]: index) -> tensor<f32> {
 // CHECK:           %[[MEMREF:.*]] = bufferization.to_memref %[[TENSOR]] : memref<f32>
-// CHECK:           %[[RESULT:.*]] = scf.for %[[IV:.*]] = %[[INDEX]] to %[[INDEX]] step %[[INDEX]] iter_args(%[[MEMREF_ITER:.*]] = %[[MEMREF]]) -> (memref<f32>) {
+// Note: scf.for iter_args always bufferize to a memory write. This could be
+// optimized by analyzing the loop body.
+// CHECK:           %[[MEMREF_COPY:.*]] = memref.alloc()
+// CHECK:           memref.copy %[[MEMREF]], %[[MEMREF_COPY]]
+// CHECK:           %[[RESULT:.*]] = scf.for %{{.*}} = %[[INDEX]] to %[[INDEX]] step %[[INDEX]] iter_args(%[[MEMREF_ITER:.*]] = %[[MEMREF_COPY]]) -> (memref<f32>) {
 // CHECK:             %[[TENSOR_ITER:.*]] = bufferization.to_tensor %[[MEMREF_ITER]] : memref<f32>
 // CHECK:             %[[TENSOR_MUNGED:.*]] = "test.munge_tensor"(%[[TENSOR_ITER]]) : (tensor<f32>) -> tensor<f32>
 // CHECK:             %[[MEMREF_MUNGED:.*]] = bufferization.to_memref %[[TENSOR_MUNGED]] : memref<f32>
 // CHECK:             scf.yield %[[MEMREF_MUNGED]] : memref<f32>
 // CHECK:           }
-// CHECK:           %[[TENSOR:.*]] = bufferization.to_tensor %[[RESULT:.*]] : memref<f32>
+// CHECK:           %[[TENSOR:.*]] = bufferization.to_tensor %[[RESULT]] : memref<f32>
 // CHECK:           return %[[TENSOR]] : tensor<f32>
 // CHECK:         }
 func.func @for_correct_recursive_legalization_behavior(%arg0: tensor<f32>, %index: index) -> tensor<f32> {
@@ -78,11 +92,17 @@ func.func @for_correct_recursive_legalization_behavior(%arg0: tensor<f32>, %inde
   return %ret : tensor<f32>
 }
 
+// -----
+
 // CHECK-LABEL:   func @bufferize_while(
 // CHECK-SAME: %[[ARG0:.*]]: i64, %[[ARG1:.*]]: i64, %[[ARG2:.*]]: tensor<f32>
 // CHECK: %[[M:.*]] = bufferization.to_memref %[[ARG2]] : memref<f32>
-// CHECK: %[[RES1:.*]]:3 = scf.while (%{{.*}} = %[[ARG0]], %{{.*}} = %[[M]]) : (i64, memref<f32>) -> (i64, i64, memref<f32>)
-// CHECK: scf.condition(%{{.*}}) %{{.*}}, %{{.*}}, %{{.*}} : i64, i64, memref<f32>
+// Note: scf.while iter_args always bufferize to a memory write. This could be
+// optimized by analyzing the loop body.
+// CHECK:           %[[MEMREF_COPY:.*]] = memref.alloc()
+// CHECK:           memref.copy %[[M]], %[[MEMREF_COPY]]
+// CHECK: %[[RES1:.*]]:3 = scf.while (%{{.*}} = %[[ARG0]], %[[ITER:.*]] = %[[MEMREF_COPY]]) : (i64, memref<f32>) -> (i64, i64, memref<f32>)
+// CHECK: scf.condition(%{{.*}}) %{{.*}}, %{{.*}}, %[[ITER]] : i64, i64, memref<f32>
 // CHECK: ^bb0(%{{.*}}: i64, %{{.*}}: i64, %{{.*}}: memref<f32>):
 // CHECK: scf.yield %{{.*}}, %{{.*}} : i64, memref<f32>
 // CHECK:  %[[RES2:.*]] = bufferization.to_tensor %[[RES1]]#2 : memref<f32>
-- 
GitLab


From 6233346895abfb57782511cddc263d439fdd537b Mon Sep 17 00:00:00 2001
From: Chengjun <chengjunp@Nvidia.com>
Date: Mon, 28 Oct 2024 17:22:48 -0700
Subject: [PATCH 388/693] [GenericCycle] Add a Cache for getExitBlocks in
 GenericCycle (#112290)

In `UniformityAnalysis`, we need to get the exit blocks of cycles in the
`DivergencePropagator` and currently, we have to do a search for the
exit blocks every time. In this change, we add a cache of the results in
the `GenericCycle` so that it can save the compile time. By testing, for
some large cases, this can save about 60% compile time in the
`UniformityAnalysis`.
---
 llvm/include/llvm/ADT/GenericCycleImpl.h |  9 +++++++++
 llvm/include/llvm/ADT/GenericCycleInfo.h | 21 +++++++++++++++++++--
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h
index 3d2c5f428835..41ba8bf8fde1 100644
--- a/llvm/include/llvm/ADT/GenericCycleImpl.h
+++ b/llvm/include/llvm/ADT/GenericCycleImpl.h
@@ -47,6 +47,11 @@ bool GenericCycle<ContextT>::contains(const GenericCycle *C) const {
 template <typename ContextT>
 void GenericCycle<ContextT>::getExitBlocks(
     SmallVectorImpl<BlockT *> &TmpStorage) const {
+  if (!ExitBlocksCache.empty()) {
+    TmpStorage = ExitBlocksCache;
+    return;
+  }
+
   TmpStorage.clear();
 
   size_t NumExitBlocks = 0;
@@ -65,6 +70,7 @@ void GenericCycle<ContextT>::getExitBlocks(
 
     TmpStorage.resize(NumExitBlocks);
   }
+  ExitBlocksCache.append(TmpStorage.begin(), TmpStorage.end());
 }
 
 template <typename ContextT>
@@ -298,6 +304,8 @@ void GenericCycleInfo<ContextT>::moveTopLevelCycleToNewParent(CycleT *NewParent,
   for (auto &It : BlockMapTopLevel)
     if (It.second == Child)
       It.second = NewParent;
+  NewParent->clearCache();
+  Child->clearCache();
 }
 
 template <typename ContextT>
@@ -316,6 +324,7 @@ void GenericCycleInfo<ContextT>::addBlockToCycle(BlockT *Block, CycleT *Cycle) {
   }
 
   BlockMapTopLevel.try_emplace(Block, Cycle);
+  Cycle->clearCache();
 }
 
 /// \brief Main function of the cycle info computations.
diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h
index 8c2fa0490e63..b8b6e3e9967a 100644
--- a/llvm/include/llvm/ADT/GenericCycleInfo.h
+++ b/llvm/include/llvm/ADT/GenericCycleInfo.h
@@ -74,16 +74,27 @@ private:
   ///       always have the same depth.
   unsigned Depth = 0;
 
+  /// Cache for the results of GetExitBlocks
+  mutable SmallVector<BlockT *, 4> ExitBlocksCache;
+
   void clear() {
     Entries.clear();
     Children.clear();
     Blocks.clear();
     Depth = 0;
     ParentCycle = nullptr;
+    clearCache();
+  }
+
+  void appendEntry(BlockT *Block) {
+    Entries.push_back(Block);
+    clearCache();
   }
 
-  void appendEntry(BlockT *Block) { Entries.push_back(Block); }
-  void appendBlock(BlockT *Block) { Blocks.insert(Block); }
+  void appendBlock(BlockT *Block) {
+    Blocks.insert(Block);
+    clearCache();
+  }
 
   GenericCycle(const GenericCycle &) = delete;
   GenericCycle &operator=(const GenericCycle &) = delete;
@@ -102,6 +113,11 @@ public:
     return Entries;
   }
 
+  /// Clear the cache of the cycle.
+  /// This should be run in all non-const function in GenericCycle
+  /// and GenericCycleInfo.
+  void clearCache() const { ExitBlocksCache.clear(); }
+
   /// \brief Return whether \p Block is an entry block of the cycle.
   bool isEntry(const BlockT *Block) const {
     return is_contained(Entries, Block);
@@ -112,6 +128,7 @@ public:
     assert(contains(Block));
     Entries.clear();
     Entries.push_back(Block);
+    clearCache();
   }
 
   /// \brief Return whether \p Block is contained in the cycle.
-- 
GitLab


From 61353cc1f65f02477eedeebcb08e9193cbd53305 Mon Sep 17 00:00:00 2001
From: Eisuke Kawashima <e.kawaschima+github@gmail.com>
Date: Tue, 29 Oct 2024 09:32:36 +0900
Subject: [PATCH 389/693] [compiler-rt] Fix invalid escape sequences in python
 files (#94030)

\d, \( and \) are not valid escape sequences; since python 3.12
they give SyntaxWarning, and will raise SyntaxError in future.
https://docs.python.org/3.12/whatsnew/3.12.html#other-language-changes

r"\(\d\)" and "\\(\\d\\)" are equivalent but the former is the shorter.

Co-authored-by: Eisuke Kawashima <e-kwsm@users.noreply.github.com>
---
 compiler-rt/lib/asan/scripts/asan_symbolize.py  | 4 ++--
 compiler-rt/lib/hwasan/scripts/hwasan_symbolize | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/compiler-rt/lib/asan/scripts/asan_symbolize.py b/compiler-rt/lib/asan/scripts/asan_symbolize.py
index b08769614aeb..058a1614b55e 100755
--- a/compiler-rt/lib/asan/scripts/asan_symbolize.py
+++ b/compiler-rt/lib/asan/scripts/asan_symbolize.py
@@ -316,7 +316,7 @@ class DarwinSymbolizer(Symbolizer):
         #   * For C functions atos omits parentheses and argument types.
         #   * For C++ functions the function name (i.e., `foo` above) may contain
         #     templates which may contain parentheses.
-        match = re.match("^(.*) \(in (.*)\) \((.*:\d*)\)$", atos_line)
+        match = re.match(r"^(.*) \(in (.*)\) \((.*:\d*)\)$", atos_line)
         logging.debug("atos_line: %s", atos_line)
         if match:
             function_name = match.group(1)
@@ -541,7 +541,7 @@ class SymbolizationLoop(object):
         # names in the regex because it could be an
         # Objective-C or C++ demangled name.
         stack_trace_line_format = (
-            "^( *#([0-9]+) *)(0x[0-9a-f]+) *(?:in *.+)? *\((.*)\+(0x[0-9a-f]+)\)"
+            r"^( *#([0-9]+) *)(0x[0-9a-f]+) *(?:in *.+)? *\((.*)\+(0x[0-9a-f]+)\)"
         )
         match = re.match(stack_trace_line_format, line)
         if not match:
diff --git a/compiler-rt/lib/hwasan/scripts/hwasan_symbolize b/compiler-rt/lib/hwasan/scripts/hwasan_symbolize
index d1b2857ccd81..efca6b82809b 100755
--- a/compiler-rt/lib/hwasan/scripts/hwasan_symbolize
+++ b/compiler-rt/lib/hwasan/scripts/hwasan_symbolize
@@ -316,7 +316,7 @@ class Symbolizer:
       self.__last_access_tag = int(match.group(2), 16)
 
   def process_tag_dump_line(self, line, ignore_tags=False):
-    m = re.match(r'.*?(0x[0-9a-f]+):' + '([ ]*[\[ ][0-9a-f][0-9a-f]\]?)' * 16, line)
+    m = re.match(r'.*?(0x[0-9a-f]+):' + r'([ ]*[\[ ][0-9a-f][0-9a-f]\]?)' * 16, line)
     if m is None:
       return False
     addr = m.group(1)
-- 
GitLab


From 1ceccbb0dd9d8539fec2213566fe6cc2a05b7993 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Mon, 28 Oct 2024 17:33:53 -0700
Subject: [PATCH 390/693] VirtRegRewriter: Add implicit register defs for live
 out undef lanes (#112679)

If an undef subregister def is live into another block, we need to
maintain a physreg def to track the liveness of those lanes. This
would manifest a verifier error after branch folding, when the cloned
tail block use no longer had a def.

We need to detect interference with other assigned intervals to avoid
clobbering the undef lanes defined in other intervals, since the undef
def didn't count as interference. This is pretty ugly and adds a new
dependency on LiveRegMatrix, keeping it live for one more pass. It also
adds a lot of implicit operand spam (we really should have a better
representation for this).

There is a missing verifier check for this situation. Added an xfailed
test that demonstrates this. We may also be able to revert the changes
in 47d3cbcf842a036c20c3f1c74255cdfc213f41c2.

It might be better to insert an IMPLICIT_DEF before the instruction
rather than using the implicit-def operand.

Fixes #98474
---
 llvm/include/llvm/CodeGen/LiveRegMatrix.h     |  10 +
 llvm/lib/CodeGen/LiveRegMatrix.cpp            |  35 ++
 llvm/lib/CodeGen/VirtRegMap.cpp               |  68 ++++
 .../branch-folding-implicit-def-subreg.ll     |  65 ++--
 ...nfloop-subrange-spill-inspect-subrange.mir |   4 +-
 .../CodeGen/AMDGPU/infloop-subrange-spill.mir |   4 +-
 ...sue98474-assigned-physreg-interference.mir |  55 +++
 ...474-need-live-out-undef-subregister-def.ll |  42 ++
 ...egrewriter-live-out-undef-subregisters.mir | 363 ++++++++++++++++++
 ...ssing-def-liveout-physical-subregister.mir |  36 ++
 10 files changed, 643 insertions(+), 39 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/issue98474-need-live-out-undef-subregister-def.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir
 create mode 100644 llvm/test/MachineVerifier/AMDGPU/issue98474-missing-def-liveout-physical-subregister.mir

diff --git a/llvm/include/llvm/CodeGen/LiveRegMatrix.h b/llvm/include/llvm/CodeGen/LiveRegMatrix.h
index 84050bf17073..486392ca3c49 100644
--- a/llvm/include/llvm/CodeGen/LiveRegMatrix.h
+++ b/llvm/include/llvm/CodeGen/LiveRegMatrix.h
@@ -118,6 +118,16 @@ public:
   /// the segment [Start, End).
   bool checkInterference(SlotIndex Start, SlotIndex End, MCRegister PhysReg);
 
+  /// Check for interference in the segment [Start, End) that may prevent
+  /// assignment to PhysReg, like checkInterference. Returns a lane mask of
+  /// which lanes of the physical register interfere in the segment [Start, End)
+  /// of some other interval already assigned to PhysReg.
+  ///
+  /// If this function returns LaneBitmask::getNone(), PhysReg is completely
+  /// free at the segment [Start, End).
+  LaneBitmask checkInterferenceLanes(SlotIndex Start, SlotIndex End,
+                                     MCRegister PhysReg);
+
   /// Assign VirtReg to PhysReg.
   /// This will mark VirtReg's live range as occupied in the LiveRegMatrix and
   /// update VirtRegMap. The live range is expected to be available in PhysReg.
diff --git a/llvm/lib/CodeGen/LiveRegMatrix.cpp b/llvm/lib/CodeGen/LiveRegMatrix.cpp
index a57233cf37da..bc8c59381a40 100644
--- a/llvm/lib/CodeGen/LiveRegMatrix.cpp
+++ b/llvm/lib/CodeGen/LiveRegMatrix.cpp
@@ -244,6 +244,41 @@ bool LiveRegMatrix::checkInterference(SlotIndex Start, SlotIndex End,
   return false;
 }
 
+LaneBitmask LiveRegMatrix::checkInterferenceLanes(SlotIndex Start,
+                                                  SlotIndex End,
+                                                  MCRegister PhysReg) {
+  // Construct artificial live range containing only one segment [Start, End).
+  VNInfo valno(0, Start);
+  LiveRange::Segment Seg(Start, End, &valno);
+  LiveRange LR;
+  LR.addSegment(Seg);
+
+  LaneBitmask InterferingLanes;
+
+  // Check for interference with that segment
+  for (MCRegUnitMaskIterator MCRU(PhysReg, TRI); MCRU.isValid(); ++MCRU) {
+    auto [Unit, Lanes] = *MCRU;
+    // LR is stack-allocated. LiveRegMatrix caches queries by a key that
+    // includes the address of the live range. If (for the same reg unit) this
+    // checkInterference overload is called twice, without any other query()
+    // calls in between (on heap-allocated LiveRanges)  - which would invalidate
+    // the cached query - the LR address seen the second time may well be the
+    // same as that seen the first time, while the Start/End/valno may not - yet
+    // the same cached result would be fetched. To avoid that, we don't cache
+    // this query.
+    //
+    // FIXME: the usability of the Query API needs to be improved to avoid
+    // subtle bugs due to query identity. Avoiding caching, for example, would
+    // greatly simplify things.
+    LiveIntervalUnion::Query Q;
+    Q.reset(UserTag, LR, Matrix[Unit]);
+    if (Q.checkInterference())
+      InterferingLanes |= Lanes;
+  }
+
+  return InterferingLanes;
+}
+
 Register LiveRegMatrix::getOneVReg(unsigned PhysReg) const {
   const LiveInterval *VRegInterval = nullptr;
   for (MCRegUnit Unit : TRI->regunits(PhysReg)) {
diff --git a/llvm/lib/CodeGen/VirtRegMap.cpp b/llvm/lib/CodeGen/VirtRegMap.cpp
index 46253b1743d9..26a12512c87b 100644
--- a/llvm/lib/CodeGen/VirtRegMap.cpp
+++ b/llvm/lib/CodeGen/VirtRegMap.cpp
@@ -21,6 +21,7 @@
 #include "llvm/CodeGen/LiveDebugVariables.h"
 #include "llvm/CodeGen/LiveInterval.h"
 #include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/LiveRegMatrix.h"
 #include "llvm/CodeGen/LiveStacks.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -203,6 +204,7 @@ class VirtRegRewriter : public MachineFunctionPass {
   MachineRegisterInfo *MRI = nullptr;
   SlotIndexes *Indexes = nullptr;
   LiveIntervals *LIS = nullptr;
+  LiveRegMatrix *LRM = nullptr;
   VirtRegMap *VRM = nullptr;
   LiveDebugVariables *DebugVars = nullptr;
   DenseSet<Register> RewriteRegs;
@@ -215,6 +217,9 @@ class VirtRegRewriter : public MachineFunctionPass {
   void handleIdentityCopy(MachineInstr &MI);
   void expandCopyBundle(MachineInstr &MI) const;
   bool subRegLiveThrough(const MachineInstr &MI, MCRegister SuperPhysReg) const;
+  LaneBitmask liveOutUndefPhiLanesForUndefSubregDef(
+      const LiveInterval &LI, const MachineBasicBlock &MBB, unsigned SubReg,
+      MCPhysReg PhysReg, const MachineInstr &MI) const;
 
 public:
   static char ID;
@@ -247,6 +252,7 @@ INITIALIZE_PASS_BEGIN(VirtRegRewriter, "virtregrewriter",
 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LiveDebugVariables)
+INITIALIZE_PASS_DEPENDENCY(LiveRegMatrixWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(LiveStacks)
 INITIALIZE_PASS_DEPENDENCY(VirtRegMapWrapperLegacy)
 INITIALIZE_PASS_END(VirtRegRewriter, "virtregrewriter",
@@ -262,6 +268,7 @@ void VirtRegRewriter::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<LiveStacks>();
   AU.addPreserved<LiveStacks>();
   AU.addRequired<VirtRegMapWrapperLegacy>();
+  AU.addRequired<LiveRegMatrixWrapperLegacy>();
 
   if (!ClearVirtRegs)
     AU.addPreserved<LiveDebugVariables>();
@@ -276,6 +283,7 @@ bool VirtRegRewriter::runOnMachineFunction(MachineFunction &fn) {
   MRI = &MF->getRegInfo();
   Indexes = &getAnalysis<SlotIndexesWrapperPass>().getSI();
   LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
+  LRM = &getAnalysis<LiveRegMatrixWrapperLegacy>().getLRM();
   VRM = &getAnalysis<VirtRegMapWrapperLegacy>().getVRM();
   DebugVars = &getAnalysis<LiveDebugVariables>();
   LLVM_DEBUG(dbgs() << "********** REWRITE VIRTUAL REGISTERS **********\n"
@@ -548,6 +556,40 @@ bool VirtRegRewriter::subRegLiveThrough(const MachineInstr &MI,
   return false;
 }
 
+/// Compute a lanemask for undef lanes which need to be preserved out of the
+/// defining block for a register assignment for a subregister def. \p PhysReg
+/// is assigned to \p LI, which is the main range.
+LaneBitmask VirtRegRewriter::liveOutUndefPhiLanesForUndefSubregDef(
+    const LiveInterval &LI, const MachineBasicBlock &MBB, unsigned SubReg,
+    MCPhysReg PhysReg, const MachineInstr &MI) const {
+  LaneBitmask UndefMask = ~TRI->getSubRegIndexLaneMask(SubReg);
+  LaneBitmask LiveOutUndefLanes;
+
+  for (const LiveInterval::SubRange &SR : LI.subranges()) {
+    // Figure out which lanes are undef live into a successor.
+    LaneBitmask NeedImpDefLanes = UndefMask & SR.LaneMask;
+    if (NeedImpDefLanes.any() && !LIS->isLiveOutOfMBB(SR, &MBB)) {
+      for (const MachineBasicBlock *Succ : MBB.successors()) {
+        if (LIS->isLiveInToMBB(SR, Succ))
+          LiveOutUndefLanes |= NeedImpDefLanes;
+      }
+    }
+  }
+
+  SlotIndex MIIndex = LIS->getInstructionIndex(MI);
+  SlotIndex BeforeMIUses = MIIndex.getBaseIndex();
+  LaneBitmask InterferingLanes =
+      LRM->checkInterferenceLanes(BeforeMIUses, MIIndex.getRegSlot(), PhysReg);
+  LiveOutUndefLanes &= ~InterferingLanes;
+
+  LLVM_DEBUG(if (LiveOutUndefLanes.any()) {
+    dbgs() << "Need live out undef defs for " << printReg(PhysReg)
+           << LiveOutUndefLanes << " from " << printMBBReference(MBB) << '\n';
+  });
+
+  return LiveOutUndefLanes;
+}
+
 void VirtRegRewriter::rewrite() {
   bool NoSubRegLiveness = !MRI->subRegLivenessEnabled();
   SmallVector<Register, 8> SuperDeads;
@@ -602,6 +644,32 @@ void VirtRegRewriter::rewrite() {
                 MO.setIsUndef(true);
             } else if (!MO.isDead()) {
               assert(MO.isDef());
+              if (MO.isUndef()) {
+                const LiveInterval &LI = LIS->getInterval(VirtReg);
+
+                LaneBitmask LiveOutUndefLanes =
+                    liveOutUndefPhiLanesForUndefSubregDef(LI, *MBBI, SubReg,
+                                                          PhysReg, MI);
+                if (LiveOutUndefLanes.any()) {
+                  SmallVector<unsigned, 16> CoveringIndexes;
+
+                  // TODO: Just use one super register def if none of the lanes
+                  // are needed?
+                  if (!TRI->getCoveringSubRegIndexes(
+                          *MRI, MRI->getRegClass(VirtReg), LiveOutUndefLanes,
+                          CoveringIndexes))
+                    llvm_unreachable(
+                        "cannot represent required subregister defs");
+
+                  // Try to represent the minimum needed live out def as a
+                  // sequence of subregister defs.
+                  //
+                  // FIXME: It would be better if we could directly represent
+                  // liveness with a lanemask instead of spamming operands.
+                  for (unsigned SubIdx : CoveringIndexes)
+                    SuperDefs.push_back(TRI->getSubReg(PhysReg, SubIdx));
+                }
+              }
             }
           }
 
diff --git a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
index 862543299239..055e9850de3d 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
@@ -38,24 +38,19 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $sgpr30_sgpr31 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, renamable $sgpr26_sgpr27, implicit-def dead $scc
-  ; GFX90A-NEXT:   $vgpr22 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   $vgpr10 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   $vgpr24 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   $vgpr18 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   $vgpr20 = IMPLICIT_DEF
   ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.59, implicit $vcc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.2:
   ; GFX90A-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr12, $sgpr13, $sgpr14, $vgpr22, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6, $sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr24_sgpr25, $sgpr26_sgpr27, $sgpr30_sgpr31, $sgpr42_sgpr43, $sgpr54, $sgpr55, $sgpr16_sgpr17_sgpr18, $sgpr18_sgpr19, $sgpr20_sgpr21_sgpr22, $vgpr2, $vgpr3, $vgpr10, $vgpr24, $vgpr18, $vgpr20
+  ; GFX90A-NEXT:   liveins: $sgpr12, $sgpr13, $sgpr14, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6, $sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr24_sgpr25, $sgpr26_sgpr27, $sgpr30_sgpr31, $sgpr42_sgpr43, $sgpr54, $sgpr55, $sgpr16_sgpr17_sgpr18, $sgpr18_sgpr19, $sgpr20_sgpr21_sgpr22, $vgpr2, $vgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $sgpr15 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $sgpr23 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr19 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr21 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr23 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr25 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
+  ; GFX90A-NEXT:   renamable $vgpr19 = IMPLICIT_DEF implicit-def $vgpr18
+  ; GFX90A-NEXT:   renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
+  ; GFX90A-NEXT:   renamable $vgpr23 = IMPLICIT_DEF implicit-def $vgpr22
+  ; GFX90A-NEXT:   renamable $vgpr25 = IMPLICIT_DEF implicit-def $vgpr24
   ; GFX90A-NEXT:   renamable $sgpr28_sgpr29 = S_MOV_B64 0
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.3.Flow17:
@@ -111,8 +106,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr52 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
+  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
   ; GFX90A-NEXT:   renamable $sgpr15 = IMPLICIT_DEF
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.6.Flow20:
@@ -395,8 +390,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr52 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
+  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
   ; GFX90A-NEXT:   renamable $sgpr15 = IMPLICIT_DEF
   ; GFX90A-NEXT:   $sgpr30_sgpr31 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.37, implicit $exec
@@ -434,8 +429,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr52 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
+  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
   ; GFX90A-NEXT:   renamable $sgpr15 = IMPLICIT_DEF
   ; GFX90A-NEXT:   $sgpr36_sgpr37 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.39, implicit $exec
@@ -484,8 +479,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr52 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
+  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
   ; GFX90A-NEXT:   renamable $sgpr15 = IMPLICIT_DEF
   ; GFX90A-NEXT:   $sgpr38_sgpr39 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.41, implicit $exec
@@ -535,8 +530,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr52 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
+  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
   ; GFX90A-NEXT:   renamable $sgpr15 = IMPLICIT_DEF
   ; GFX90A-NEXT:   $sgpr40_sgpr41 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.47, implicit $exec
@@ -589,8 +584,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr52 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
+  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
   ; GFX90A-NEXT:   renamable $sgpr15 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_MOV_B64 0
   ; GFX90A-NEXT: {{  $}}
@@ -643,8 +638,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr52 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
+  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
   ; GFX90A-NEXT:   renamable $sgpr15 = IMPLICIT_DEF
   ; GFX90A-NEXT:   $sgpr16_sgpr17 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.43, implicit $exec
@@ -689,8 +684,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr52 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
+  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
   ; GFX90A-NEXT:   renamable $sgpr15 = IMPLICIT_DEF
   ; GFX90A-NEXT:   S_BRANCH %bb.45
   ; GFX90A-NEXT: {{  $}}
@@ -719,8 +714,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr52 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
+  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
   ; GFX90A-NEXT:   renamable $sgpr15 = IMPLICIT_DEF
   ; GFX90A-NEXT:   S_BRANCH %bb.46
   ; GFX90A-NEXT: {{  $}}
@@ -748,8 +743,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr52 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
+  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
   ; GFX90A-NEXT:   renamable $sgpr15 = IMPLICIT_DEF
   ; GFX90A-NEXT:   S_BRANCH %bb.62
   ; GFX90A-NEXT: {{  $}}
@@ -773,8 +768,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr52 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
+  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
   ; GFX90A-NEXT:   renamable $sgpr15 = IMPLICIT_DEF
   ; GFX90A-NEXT:   $sgpr58_sgpr59 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.53, implicit $exec
@@ -880,8 +875,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr52 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
+  ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
   ; GFX90A-NEXT:   $sgpr50_sgpr51 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.57, implicit $exec
   ; GFX90A-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
index 6603f2ef7ade..7421a2e10c3b 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
@@ -32,7 +32,7 @@ body:             |
   ; CHECK-NEXT:   dead undef [[DEF2:%[0-9]+]].sub0:vreg_64 = IMPLICIT_DEF
   ; CHECK-NEXT:   renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s512), align 32, addrspace 4)
   ; CHECK-NEXT:   SI_SPILL_S512_SAVE killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s512) into %stack.0, align 4, addrspace 5)
-  ; CHECK-NEXT:   renamable $sgpr24 = IMPLICIT_DEF
+  ; CHECK-NEXT:   renamable $sgpr24 = IMPLICIT_DEF implicit-def $sgpr25
   ; CHECK-NEXT:   renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM undef renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s512), align 32, addrspace 4)
   ; CHECK-NEXT:   $exec = S_MOV_B64_term undef renamable $sgpr4_sgpr5
   ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.6, implicit $exec
@@ -82,7 +82,7 @@ body:             |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = SI_SPILL_S512_RESTORE %stack.0, implicit $exec, implicit $sgpr32 :: (load (s512) from %stack.0, align 4, addrspace 5)
   ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
-  ; CHECK-NEXT:   renamable $sgpr25 = COPY undef renamable $sgpr24
+  ; CHECK-NEXT:   renamable $sgpr25 = COPY undef renamable $sgpr24, implicit-def $sgpr24
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.7, implicit undef $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.6
   ; CHECK-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
index fa95f4c13417..8ae6c2795589 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
@@ -30,7 +30,7 @@ body:             |
   ; CHECK-NEXT:   dead renamable $sgpr5 = IMPLICIT_DEF
   ; CHECK-NEXT:   dead undef [[DEF3:%[0-9]+]].sub1:vreg_64 = IMPLICIT_DEF
   ; CHECK-NEXT:   dead renamable $sgpr5 = IMPLICIT_DEF
-  ; CHECK-NEXT:   renamable $sgpr24 = IMPLICIT_DEF
+  ; CHECK-NEXT:   renamable $sgpr24 = IMPLICIT_DEF implicit-def $sgpr25
   ; CHECK-NEXT:   renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM undef renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s512), align 32, addrspace 4)
   ; CHECK-NEXT:   $exec = S_MOV_B64_term undef renamable $sgpr4_sgpr5
   ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.5, implicit $exec
@@ -78,7 +78,7 @@ body:             |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX8_IMM undef renamable $sgpr4_sgpr5, 32, 0 :: (invariant load (s256), addrspace 4)
   ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, undef renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
-  ; CHECK-NEXT:   renamable $sgpr25 = COPY undef renamable $sgpr24
+  ; CHECK-NEXT:   renamable $sgpr25 = COPY undef renamable $sgpr24, implicit-def $sgpr24
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.6, implicit undef $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.5
   ; CHECK-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir b/llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir
new file mode 100644
index 000000000000..786ce4020383
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir
@@ -0,0 +1,55 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -start-before=greedy,2 -stop-after=tailduplication -verify-machineinstrs -o - %s | FileCheck %s
+
+---
+name:            undef_subreg_def_live_out_tailduplicate_vreg96_undef_sub1_sub2_assigned_physreg_interference
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
+  stackPtrOffsetReg: '$sgpr32'
+body:             |
+  ; CHECK-LABEL: name: undef_subreg_def_live_out_tailduplicate_vreg96_undef_sub1_sub2_assigned_physreg_interference
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $sgpr0, $vgpr2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+  ; CHECK-NEXT:   S_CBRANCH_SCC0 %bb.2, implicit killed $scc
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   liveins: $vgpr2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $vgpr3 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr4_vgpr5
+  ; CHECK-NEXT:   EXP 0, killed renamable $vgpr3, renamable $vgpr4, renamable $vgpr5, killed renamable $vgpr2, 0, 0, 0, implicit $exec
+  ; CHECK-NEXT:   S_ENDPGM 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $vgpr2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $sgpr4_sgpr5_sgpr6_sgpr7
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr3_vgpr4_vgpr5 = BUFFER_LOAD_FORMAT_XYZ_IDXEN killed renamable $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s96), align 16, addrspace 8)
+  ; CHECK-NEXT:   EXP 0, killed renamable $vgpr3, renamable $vgpr4, renamable $vgpr5, killed renamable $vgpr2, 0, 0, 0, implicit $exec
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $sgpr0, $vgpr2
+
+    %2:vgpr_32 = COPY $vgpr2
+    S_CMP_EQ_U32 killed $sgpr0, 0, implicit-def $scc
+    S_CBRANCH_SCC0 %bb.2, implicit killed $scc
+
+  bb.1:
+    undef %0.sub0:vreg_96 = V_MOV_B32_e32 0, implicit $exec
+    S_BRANCH %bb.3
+
+  bb.2:
+    S_NOP 0, implicit-def $sgpr4_sgpr5_sgpr6_sgpr7
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %0:vreg_96 = BUFFER_LOAD_FORMAT_XYZ_IDXEN killed %1, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s96), addrspace 8)
+
+  bb.3:
+    EXP 0, killed %0.sub0, killed %0.sub1, killed %0.sub2, %2:vgpr_32, 0, 0, 0, implicit $exec
+    S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/issue98474-need-live-out-undef-subregister-def.ll b/llvm/test/CodeGen/AMDGPU/issue98474-need-live-out-undef-subregister-def.ll
new file mode 100644
index 000000000000..7caa563d8b29
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/issue98474-need-live-out-undef-subregister-def.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -verify-machineinstrs -o - %s | FileCheck %s
+
+; Check for verifier error after tail duplication. An implicit_def of
+; a subregsiter is needed to maintain liveness after assignment.
+
+define amdgpu_vs void @test(i32 inreg %cmp, i32 %e0) {
+; CHECK-LABEL: test:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_cmp_eq_u32 s0, 0
+; CHECK-NEXT:    s_mov_b32 s0, 0
+; CHECK-NEXT:    s_cbranch_scc1 .LBB0_2
+; CHECK-NEXT:  ; %bb.1: ; %load
+; CHECK-NEXT:    s_mov_b32 s1, s0
+; CHECK-NEXT:    s_mov_b32 s2, s0
+; CHECK-NEXT:    s_mov_b32 s3, s0
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    buffer_load_format_xy v[1:2], v1, s[0:3], 0 idxen
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    exp mrt0 v0, v1, v2, v0
+; CHECK-NEXT:    s_endpgm
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    exp mrt0 v0, v1, v2, v0
+; CHECK-NEXT:    s_endpgm
+entry:
+  %cond = icmp eq i32 %cmp, 0
+  br i1 %cond, label %end, label %load
+
+load:
+  %data1 = call <2 x i32> @llvm.amdgcn.struct.buffer.load.format.v2i32(<4 x i32> zeroinitializer, i32 0, i32 0, i32 0, i32 0)
+  %e1 = extractelement <2 x i32> %data1, i32 0
+  %e2 = extractelement <2 x i32> %data1, i32 1
+  br label %end
+
+end:
+  %out1 = phi i32 [ 0, %entry ], [ %e1, %load ]
+  %out2 = phi i32 [ poison, %entry ], [ %e2, %load ]
+  call void @llvm.amdgcn.exp.i32(i32 0, i32 15, i32 %e0, i32 %out1, i32 %out2, i32 %e0, i1 false, i1 false)
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir b/llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir
new file mode 100644
index 000000000000..86b6c5982b4c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir
@@ -0,0 +1,363 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -start-before=greedy,2 -stop-after=tailduplication -verify-machineinstrs -o - %s | FileCheck %s
+
+# The partial def of %0 introduces a live out undef def of %0.sub1
+# into bb.3. We need to maintain this liveness with an explicit def of
+# the physical subregister. Without this, a verifier error would
+# appear after tail duplication.
+
+---
+name:            undef_subreg_def_live_out_tailduplicate_vreg64_undef_sub1
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
+  stackPtrOffsetReg: '$sgpr32'
+body:             |
+  ; CHECK-LABEL: name: undef_subreg_def_live_out_tailduplicate_vreg64_undef_sub1
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $sgpr0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+  ; CHECK-NEXT:   S_CBRANCH_SCC0 %bb.2, implicit killed $scc
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr1
+  ; CHECK-NEXT:   EXP 0, killed renamable $vgpr0, renamable $vgpr1, undef renamable $vgpr0, undef renamable $vgpr0, 0, 0, 0, implicit $exec
+  ; CHECK-NEXT:   S_ENDPGM 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $sgpr4_sgpr5_sgpr6_sgpr7
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = BUFFER_LOAD_FORMAT_XY_IDXEN killed renamable $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), addrspace 8)
+  ; CHECK-NEXT:   EXP 0, killed renamable $vgpr0, renamable $vgpr1, undef renamable $vgpr0, undef renamable $vgpr0, 0, 0, 0, implicit $exec
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $sgpr0
+
+    S_CMP_EQ_U32 killed $sgpr0, 0, implicit-def $scc
+    S_CBRANCH_SCC0 %bb.2, implicit killed $scc
+
+  bb.1:
+    undef %0.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    S_BRANCH %bb.3
+
+  bb.2:
+    S_NOP 0, implicit-def $sgpr4_sgpr5_sgpr6_sgpr7
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %0:vreg_64 = BUFFER_LOAD_FORMAT_XY_IDXEN killed %1, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), addrspace 8)
+
+  bb.3:
+    EXP 0, killed %0.sub0, killed %0.sub1, undef %2:vgpr_32, undef %2:vgpr_32, 0, 0, 0, implicit $exec
+    S_ENDPGM 0
+
+...
+
+---
+name:            undef_subreg_def_live_out_tailduplicate_vreg96_undef_sub1_sub2
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
+  stackPtrOffsetReg: '$sgpr32'
+body:             |
+  ; CHECK-LABEL: name: undef_subreg_def_live_out_tailduplicate_vreg96_undef_sub1_sub2
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $sgpr0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+  ; CHECK-NEXT:   S_CBRANCH_SCC0 %bb.2, implicit killed $scc
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr1_vgpr2
+  ; CHECK-NEXT:   EXP 0, killed renamable $vgpr0, renamable $vgpr1, renamable $vgpr2, undef renamable $vgpr0, 0, 0, 0, implicit $exec
+  ; CHECK-NEXT:   S_ENDPGM 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $sgpr4_sgpr5_sgpr6_sgpr7
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1_vgpr2 = BUFFER_LOAD_FORMAT_XYZ_IDXEN killed renamable $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s96), align 16, addrspace 8)
+  ; CHECK-NEXT:   EXP 0, killed renamable $vgpr0, renamable $vgpr1, renamable $vgpr2, undef renamable $vgpr0, 0, 0, 0, implicit $exec
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $sgpr0
+
+    S_CMP_EQ_U32 killed $sgpr0, 0, implicit-def $scc
+    S_CBRANCH_SCC0 %bb.2, implicit killed $scc
+
+  bb.1:
+    undef %0.sub0:vreg_96 = V_MOV_B32_e32 0, implicit $exec
+    S_BRANCH %bb.3
+
+  bb.2:
+    S_NOP 0, implicit-def $sgpr4_sgpr5_sgpr6_sgpr7
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %0:vreg_96 = BUFFER_LOAD_FORMAT_XYZ_IDXEN killed %1, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s96), addrspace 8)
+
+  bb.3:
+    EXP 0, killed %0.sub0, killed %0.sub1, killed %0.sub2, undef %2:vgpr_32, 0, 0, 0, implicit $exec
+    S_ENDPGM 0
+
+...
+
+---
+name:            undef_subreg_def_live_out_tailduplicate_vreg96_undef_sub0_sub2
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
+  stackPtrOffsetReg: '$sgpr32'
+body:             |
+  ; CHECK-LABEL: name: undef_subreg_def_live_out_tailduplicate_vreg96_undef_sub0_sub2
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $sgpr0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+  ; CHECK-NEXT:   S_CBRANCH_SCC0 %bb.2, implicit killed $scc
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   renamable $vgpr1 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr2, implicit-def $vgpr0
+  ; CHECK-NEXT:   EXP 0, killed renamable $vgpr0, renamable $vgpr1, renamable $vgpr2, undef renamable $vgpr0, 0, 0, 0, implicit $exec
+  ; CHECK-NEXT:   S_ENDPGM 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $sgpr4_sgpr5_sgpr6_sgpr7
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1_vgpr2 = BUFFER_LOAD_FORMAT_XYZ_IDXEN killed renamable $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s96), align 16, addrspace 8)
+  ; CHECK-NEXT:   EXP 0, killed renamable $vgpr0, renamable $vgpr1, renamable $vgpr2, undef renamable $vgpr0, 0, 0, 0, implicit $exec
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $sgpr0
+
+    S_CMP_EQ_U32 killed $sgpr0, 0, implicit-def $scc
+    S_CBRANCH_SCC0 %bb.2, implicit killed $scc
+
+  bb.1:
+    undef %0.sub1:vreg_96 = V_MOV_B32_e32 0, implicit $exec
+    S_BRANCH %bb.3
+
+  bb.2:
+    S_NOP 0, implicit-def $sgpr4_sgpr5_sgpr6_sgpr7
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %0:vreg_96 = BUFFER_LOAD_FORMAT_XYZ_IDXEN killed %1, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s96), addrspace 8)
+
+  bb.3:
+    EXP 0, killed %0.sub0, killed %0.sub1, killed %0.sub2, undef %2:vgpr_32, 0, 0, 0, implicit $exec
+    S_ENDPGM 0
+
+...
+
+# Test another use of the value before the block end.
+---
+name:            undef_subreg_def_live_out_tailduplicate_vreg64_undef_sub1_undef_use_in_def_block
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
+  stackPtrOffsetReg: '$sgpr32'
+body:             |
+  ; CHECK-LABEL: name: undef_subreg_def_live_out_tailduplicate_vreg64_undef_sub1_undef_use_in_def_block
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $sgpr0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+  ; CHECK-NEXT:   S_CBRANCH_SCC0 %bb.2, implicit killed $scc
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr1
+  ; CHECK-NEXT:   S_NOP 0, implicit renamable $vgpr0_vgpr1
+  ; CHECK-NEXT:   EXP 0, killed renamable $vgpr0, renamable $vgpr1, undef renamable $vgpr0, undef renamable $vgpr0, 0, 0, 0, implicit $exec
+  ; CHECK-NEXT:   S_ENDPGM 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $sgpr4_sgpr5_sgpr6_sgpr7
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = BUFFER_LOAD_FORMAT_XY_IDXEN killed renamable $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), addrspace 8)
+  ; CHECK-NEXT:   EXP 0, killed renamable $vgpr0, renamable $vgpr1, undef renamable $vgpr0, undef renamable $vgpr0, 0, 0, 0, implicit $exec
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $sgpr0
+
+    S_CMP_EQ_U32 killed $sgpr0, 0, implicit-def $scc
+    S_CBRANCH_SCC0 %bb.2, implicit killed $scc
+
+  bb.1:
+    undef %0.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    S_NOP 0, implicit %0
+    S_BRANCH %bb.3
+
+  bb.2:
+    S_NOP 0, implicit-def $sgpr4_sgpr5_sgpr6_sgpr7
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %0:vreg_64 = BUFFER_LOAD_FORMAT_XY_IDXEN killed %1, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), addrspace 8)
+
+  bb.3:
+    EXP 0, killed %0.sub0, killed %0.sub1, undef %2:vgpr_32, undef %2:vgpr_32, 0, 0, 0, implicit $exec
+    S_ENDPGM 0
+
+...
+
+# The undef subregister is not live out, no implicit def should be added for it
+---
+name:            undef_subreg_def_live_out_tailduplicate_vreg64_undef_sub1_no_phi_use
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
+  stackPtrOffsetReg: '$sgpr32'
+body:             |
+  ; CHECK-LABEL: name: undef_subreg_def_live_out_tailduplicate_vreg64_undef_sub1_no_phi_use
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $sgpr0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+  ; CHECK-NEXT:   S_CBRANCH_SCC0 %bb.2, implicit killed $scc
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   EXP 0, killed renamable $vgpr0, renamable $vgpr0, undef renamable $vgpr0, undef renamable $vgpr0, 0, 0, 0, implicit $exec
+  ; CHECK-NEXT:   S_ENDPGM 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $sgpr4_sgpr5_sgpr6_sgpr7
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = BUFFER_LOAD_FORMAT_XY_IDXEN killed renamable $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), addrspace 8)
+  ; CHECK-NEXT:   EXP 0, killed renamable $vgpr0, renamable $vgpr0, undef renamable $vgpr0, undef renamable $vgpr0, 0, 0, 0, implicit $exec
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $sgpr0
+
+    S_CMP_EQ_U32 killed $sgpr0, 0, implicit-def $scc
+    S_CBRANCH_SCC0 %bb.2, implicit killed $scc
+
+  bb.1:
+    undef %0.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    S_BRANCH %bb.3
+
+  bb.2:
+    S_NOP 0, implicit-def $sgpr4_sgpr5_sgpr6_sgpr7
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %0:vreg_64 = BUFFER_LOAD_FORMAT_XY_IDXEN killed %1, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), addrspace 8)
+
+  bb.3:
+    EXP 0, killed %0.sub0, killed %0.sub0, undef %2:vgpr_32, undef %2:vgpr_32, 0, 0, 0, implicit $exec
+    S_ENDPGM 0
+
+...
+
+# In bb.2, %0 should be assigned to vgpr0_vgpr1. Make sure the value
+# copied from $vgpr0 into %3 isn't clobbered by the undef phi def for
+# %0.sub1.
+---
+name:            assigned_physreg_subregister_interference
+tracksRegLiveness: true
+frameInfo:
+  adjustsStack: true
+machineFunctionInfo:
+  scratchRSrcReg:  '$sgpr0_sgpr1_sgpr2_sgpr3'
+  frameOffsetReg:  '$sgpr33'
+  stackPtrOffsetReg: '$sgpr32'
+  wwmReservedRegs:
+    - '$vgpr63'
+body:             |
+  ; CHECK-LABEL: name: assigned_physreg_subregister_interference
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $sgpr30, $sgpr31, $sgpr34, $sgpr35, $sgpr36, $sgpr37, $vgpr0, $vgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $sgpr4_sgpr5 = S_OR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFSET killed $vgpr40, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, implicit $exec :: (store (s32) into %stack.0, addrspace 5)
+  ; CHECK-NEXT:   $exec = S_MOV_B64 killed $sgpr4_sgpr5
+  ; CHECK-NEXT:   $vgpr40 = SI_SPILL_S32_TO_VGPR $sgpr30, 0, $vgpr40
+  ; CHECK-NEXT:   $vgpr40 = SI_SPILL_S32_TO_VGPR $sgpr31, 1, $vgpr40
+  ; CHECK-NEXT:   $vgpr40 = SI_SPILL_S32_TO_VGPR $sgpr34, 2, $vgpr40
+  ; CHECK-NEXT:   $vgpr40 = SI_SPILL_S32_TO_VGPR $sgpr35, 3, $vgpr40
+  ; CHECK-NEXT:   $vgpr40 = SI_SPILL_S32_TO_VGPR $sgpr36, 4, $vgpr40
+  ; CHECK-NEXT:   $vgpr40 = SI_SPILL_S32_TO_VGPR $sgpr37, 5, $vgpr40
+  ; CHECK-NEXT:   renamable $sgpr34_sgpr35 = S_MOV_B64 $exec
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $sgpr34_sgpr35, $vgpr0_vgpr1:0x000000000000000F
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $sgpr4 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr5 = V_READFIRSTLANE_B32 $vgpr1, implicit $exec
+  ; CHECK-NEXT:   renamable $vcc = V_CMP_EQ_U64_e64 $sgpr4_sgpr5, killed $vgpr0_vgpr1, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr36_sgpr37 = S_AND_SAVEEXEC_B64 killed renamable $vcc, implicit-def $exec, implicit-def dead $scc, implicit $exec
+  ; CHECK-NEXT:   dead $sgpr30_sgpr31 = noconvergent SI_CALL killed renamable $sgpr4_sgpr5, 0, csr_amdgpu, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit-def $vgpr0
+  ; CHECK-NEXT:   renamable $vgpr1 = COPY $vgpr0, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 123, implicit $exec
+  ; CHECK-NEXT:   $exec = S_XOR_B64 $exec, renamable $sgpr36_sgpr37, implicit-def dead $scc
+  ; CHECK-NEXT:   S_CBRANCH_EXECNZ %bb.1, implicit $exec
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $vgpr1, $sgpr34_sgpr35
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $exec = COPY renamable $sgpr34_sgpr35
+  ; CHECK-NEXT:   renamable $vgpr0 = V_ADD_U32_e32 1, killed $vgpr1, implicit $exec
+  ; CHECK-NEXT:   $sgpr37 = SI_RESTORE_S32_FROM_VGPR $vgpr40, 5
+  ; CHECK-NEXT:   $sgpr36 = SI_RESTORE_S32_FROM_VGPR $vgpr40, 4
+  ; CHECK-NEXT:   $sgpr35 = SI_RESTORE_S32_FROM_VGPR $vgpr40, 3
+  ; CHECK-NEXT:   $sgpr34 = SI_RESTORE_S32_FROM_VGPR $vgpr40, 2
+  ; CHECK-NEXT:   $sgpr31 = SI_RESTORE_S32_FROM_VGPR $vgpr40, 1
+  ; CHECK-NEXT:   $sgpr30 = SI_RESTORE_S32_FROM_VGPR $vgpr40, 0
+  ; CHECK-NEXT:   $sgpr4_sgpr5 = S_OR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
+  ; CHECK-NEXT:   $vgpr40 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, implicit $exec :: (load (s32) from %stack.0, addrspace 5)
+  ; CHECK-NEXT:   $exec = S_MOV_B64 killed $sgpr4_sgpr5
+  ; CHECK-NEXT:   SI_RETURN implicit $vgpr0
+  bb.0:
+    liveins: $sgpr30, $sgpr31, $sgpr34, $sgpr35, $sgpr36, $sgpr37, $vgpr0, $vgpr1, $vgpr63
+
+    $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr30, 0, $vgpr63
+    $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr31, 1, $vgpr63
+    $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr34, 2, $vgpr63
+    $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr35, 3, $vgpr63
+    $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr36, 4, $vgpr63
+    $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr37, 5, $vgpr63
+    undef %0.sub0:vreg_64 = COPY $vgpr0
+    %0.sub1:vreg_64 = COPY $vgpr1
+    ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+    renamable $sgpr34_sgpr35 = S_MOV_B64 $exec
+
+  bb.1:
+    liveins: $vgpr63, $sgpr34_sgpr35
+
+    renamable $sgpr4 = V_READFIRSTLANE_B32 %0.sub0, implicit $exec
+    renamable $sgpr5 = V_READFIRSTLANE_B32 %0.sub1, implicit $exec
+    renamable $vcc = V_CMP_EQ_U64_e64 $sgpr4_sgpr5, %0, implicit $exec
+    renamable $sgpr36_sgpr37 = S_AND_SAVEEXEC_B64 killed renamable $vcc, implicit-def $exec, implicit-def dead $scc, implicit $exec
+
+  bb.2:
+    liveins: $vgpr63, $sgpr4_sgpr5:0x000000000000000F, $sgpr34_sgpr35, $sgpr36_sgpr37
+
+    dead $sgpr30_sgpr31 = noconvergent SI_CALL killed renamable $sgpr4_sgpr5, 0, csr_amdgpu, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit-def $vgpr0
+    %3:vgpr_32 = COPY $vgpr0
+    undef %0.sub0:vreg_64 = V_MOV_B32_e32 123, implicit $exec
+    $exec = S_XOR_B64_term $exec, killed renamable $sgpr36_sgpr37, implicit-def dead $scc
+    S_CBRANCH_EXECNZ %bb.1, implicit $exec
+
+  bb.3:
+    liveins: $vgpr63, $sgpr34_sgpr35
+
+    $exec = S_MOV_B64_term killed renamable $sgpr34_sgpr35
+
+  bb.4:
+    liveins: $vgpr63
+
+    ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+    %6:vgpr_32 = V_ADD_U32_e32 1, %3, implicit $exec
+    $vgpr0 = COPY %6
+    $sgpr37 = SI_RESTORE_S32_FROM_VGPR $vgpr63, 5
+    $sgpr36 = SI_RESTORE_S32_FROM_VGPR $vgpr63, 4
+    $sgpr35 = SI_RESTORE_S32_FROM_VGPR $vgpr63, 3
+    $sgpr34 = SI_RESTORE_S32_FROM_VGPR $vgpr63, 2
+    $sgpr31 = SI_RESTORE_S32_FROM_VGPR $vgpr63, 1
+    $sgpr30 = SI_RESTORE_S32_FROM_VGPR $vgpr63, 0
+    SI_RETURN implicit $vgpr0
+
+...
diff --git a/llvm/test/MachineVerifier/AMDGPU/issue98474-missing-def-liveout-physical-subregister.mir b/llvm/test/MachineVerifier/AMDGPU/issue98474-missing-def-liveout-physical-subregister.mir
new file mode 100644
index 000000000000..892a4298bbdb
--- /dev/null
+++ b/llvm/test/MachineVerifier/AMDGPU/issue98474-missing-def-liveout-physical-subregister.mir
@@ -0,0 +1,36 @@
+# XFAIL: *
+# RUN: not --crash llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -run-pass=none -filetype=null %s
+
+# FIXME: This should fail the machine verifier. There is a missing def
+# of $vgpr2 in bb.1, which is needed since it's live into bb.3
+
+---
+name: missing_live_out_subreg_def
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+
+    S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+    S_CBRANCH_SCC0 %bb.2, implicit killed $scc
+
+  bb.1:
+    liveins: $vgpr0
+
+    renamable $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+    S_BRANCH %bb.3
+
+  bb.2:
+    liveins: $vgpr0
+
+    renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+    renamable $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+    renamable $vgpr1_vgpr2 = BUFFER_LOAD_FORMAT_XY_IDXEN killed renamable $vgpr1, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), addrspace 8)
+
+  bb.3:
+    liveins: $vgpr0, $vgpr1_vgpr2
+
+    EXP 0, killed renamable $vgpr0, killed renamable $vgpr1, renamable $vgpr2, renamable $vgpr0, 0, 0, 0, implicit $exec
+    S_ENDPGM 0
+
+...
-- 
GitLab


From a461869db3bdc372203c9a7b8326d66a626f80d9 Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Mon, 28 Oct 2024 18:00:52 -0700
Subject: [PATCH 391/693] [SandboxIR][Pass] Implement Analyses class (#113962)

The Analyses class provides a way to pass around commonly used Analyses
to SandboxIR passes throught `runOnFunction()` and `runOnRegion()`
functions.
---
 llvm/include/llvm/SandboxIR/Pass.h            | 27 ++++++++++++++---
 llvm/include/llvm/SandboxIR/PassManager.h     |  4 +--
 .../SandboxVectorizer/Passes/BottomUpVec.h    |  2 +-
 .../SandboxVectorizer/Passes/NullPass.h       |  2 +-
 .../Passes/PrintInstructionCount.h            |  2 +-
 .../Passes/RegionsFromMetadata.h              |  2 +-
 .../SandboxVectorizer/SandboxVectorizer.h     |  2 ++
 llvm/lib/SandboxIR/PassManager.cpp            |  8 ++---
 .../SandboxVectorizer/Passes/BottomUpVec.cpp  |  2 +-
 .../Passes/RegionsFromMetadata.cpp            |  4 +--
 .../SandboxVectorizer/SandboxVectorizer.cpp   |  4 ++-
 llvm/unittests/SandboxIR/PassTest.cpp         | 30 +++++++++----------
 12 files changed, 56 insertions(+), 33 deletions(-)

diff --git a/llvm/include/llvm/SandboxIR/Pass.h b/llvm/include/llvm/SandboxIR/Pass.h
index 5ed9d7442ee7..fee6bd9e779f 100644
--- a/llvm/include/llvm/SandboxIR/Pass.h
+++ b/llvm/include/llvm/SandboxIR/Pass.h
@@ -12,11 +12,29 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 
-namespace llvm::sandboxir {
+namespace llvm {
+
+class ScalarEvolution;
+
+namespace sandboxir {
 
 class Function;
 class Region;
 
+class Analyses {
+  ScalarEvolution *SE = nullptr;
+
+  Analyses() = default;
+
+public:
+  Analyses(ScalarEvolution &SE) : SE(&SE) {}
+
+public:
+  ScalarEvolution &getScalarEvolution() const { return *SE; }
+  /// For use by unit tests.
+  static Analyses emptyForTesting() { return Analyses(); }
+};
+
 /// The base class of a Sandbox IR Pass.
 class Pass {
 protected:
@@ -52,7 +70,7 @@ public:
   /// \p Name can't contain any spaces or start with '-'.
   FunctionPass(StringRef Name) : Pass(Name) {}
   /// \Returns true if it modifies \p F.
-  virtual bool runOnFunction(Function &F) = 0;
+  virtual bool runOnFunction(Function &F, const Analyses &A) = 0;
 };
 
 /// A pass that runs on a sandbox::Region.
@@ -61,9 +79,10 @@ public:
   /// \p Name can't contain any spaces or start with '-'.
   RegionPass(StringRef Name) : Pass(Name) {}
   /// \Returns true if it modifies \p R.
-  virtual bool runOnRegion(Region &R) = 0;
+  virtual bool runOnRegion(Region &R, const Analyses &A) = 0;
 };
 
-} // namespace llvm::sandboxir
+} // namespace sandboxir
+} // namespace llvm
 
 #endif // LLVM_SANDBOXIR_PASS_H
diff --git a/llvm/include/llvm/SandboxIR/PassManager.h b/llvm/include/llvm/SandboxIR/PassManager.h
index e8221996bc8f..77154cc71434 100644
--- a/llvm/include/llvm/SandboxIR/PassManager.h
+++ b/llvm/include/llvm/SandboxIR/PassManager.h
@@ -208,7 +208,7 @@ public:
   FunctionPassManager(StringRef Name, StringRef Pipeline,
                       CreatePassFunc CreatePass)
       : PassManager(Name, Pipeline, CreatePass) {}
-  bool runOnFunction(Function &F) final;
+  bool runOnFunction(Function &F, const Analyses &A) final;
 };
 
 class RegionPassManager final : public PassManager<RegionPass, RegionPass> {
@@ -217,7 +217,7 @@ public:
   RegionPassManager(StringRef Name, StringRef Pipeline,
                     CreatePassFunc CreatePass)
       : PassManager(Name, Pipeline, CreatePass) {}
-  bool runOnRegion(Region &R) final;
+  bool runOnRegion(Region &R, const Analyses &A) final;
 };
 
 } // namespace llvm::sandboxir
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
index 5cd47efd6b34..2b0b3f8192c0 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
@@ -33,7 +33,7 @@ class BottomUpVec final : public FunctionPass {
 
 public:
   BottomUpVec(StringRef Pipeline);
-  bool runOnFunction(Function &F) final;
+  bool runOnFunction(Function &F, const Analyses &A) final;
   void printPipeline(raw_ostream &OS) const final {
     OS << getName() << "\n";
     RPM.printPipeline(OS);
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/NullPass.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/NullPass.h
index 75b9f4252015..1025379770ba 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/NullPass.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/NullPass.h
@@ -11,7 +11,7 @@ class Region;
 class NullPass final : public RegionPass {
 public:
   NullPass() : RegionPass("null") {}
-  bool runOnRegion(Region &R) final { return false; }
+  bool runOnRegion(Region &R, const Analyses &A) final { return false; }
 };
 
 } // namespace llvm::sandboxir
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PrintInstructionCount.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PrintInstructionCount.h
index 9d88bc828038..cd11d4c14892 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PrintInstructionCount.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PrintInstructionCount.h
@@ -12,7 +12,7 @@ namespace llvm::sandboxir {
 class PrintInstructionCount final : public RegionPass {
 public:
   PrintInstructionCount() : RegionPass("null") {}
-  bool runOnRegion(Region &R) final {
+  bool runOnRegion(Region &R, const Analyses &A) final {
     outs() << "InstructionCount: " << std::distance(R.begin(), R.end()) << "\n";
     return false;
   }
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.h
index 3d82a61c9015..3d738ac8917e 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.h
@@ -26,7 +26,7 @@ class RegionsFromMetadata final : public FunctionPass {
 
 public:
   RegionsFromMetadata(StringRef Pipeline);
-  bool runOnFunction(Function &F) final;
+  bool runOnFunction(Function &F, const Analyses &A) final;
   void printPipeline(raw_ostream &OS) const final {
     OS << getName() << "\n";
     RPM.printPipeline(OS);
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.h
index b83744cf9e6c..03867df3d980 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.h
@@ -10,6 +10,7 @@
 
 #include <memory>
 
+#include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/SandboxIR/PassManager.h"
 
@@ -19,6 +20,7 @@ class TargetTransformInfo;
 
 class SandboxVectorizerPass : public PassInfoMixin<SandboxVectorizerPass> {
   TargetTransformInfo *TTI = nullptr;
+  ScalarEvolution *SE = nullptr;
 
   // A pipeline of SandboxIR function passes run by the vectorizer.
   sandboxir::FunctionPassManager FPM;
diff --git a/llvm/lib/SandboxIR/PassManager.cpp b/llvm/lib/SandboxIR/PassManager.cpp
index 3a1cfa1d367a..aaa49e0f6912 100644
--- a/llvm/lib/SandboxIR/PassManager.cpp
+++ b/llvm/lib/SandboxIR/PassManager.cpp
@@ -10,20 +10,20 @@
 
 namespace llvm::sandboxir {
 
-bool FunctionPassManager::runOnFunction(Function &F) {
+bool FunctionPassManager::runOnFunction(Function &F, const Analyses &A) {
   bool Change = false;
   for (auto &Pass : Passes) {
-    Change |= Pass->runOnFunction(F);
+    Change |= Pass->runOnFunction(F, A);
     // TODO: run the verifier.
   }
   // TODO: Check ChangeAll against hashes before/after.
   return Change;
 }
 
-bool RegionPassManager::runOnRegion(Region &R) {
+bool RegionPassManager::runOnRegion(Region &R, const Analyses &A) {
   bool Change = false;
   for (auto &Pass : Passes) {
-    Change |= Pass->runOnRegion(R);
+    Change |= Pass->runOnRegion(R, A);
     // TODO: run the verifier.
   }
   // TODO: Check ChangeAll against hashes before/after.
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index ede41cd661b5..66d631edfc40 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -59,7 +59,7 @@ void BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl) {
 
 void BottomUpVec::tryVectorize(ArrayRef<Value *> Bndl) { vectorizeRec(Bndl); }
 
-bool BottomUpVec::runOnFunction(Function &F) {
+bool BottomUpVec::runOnFunction(Function &F, const Analyses &A) {
   Change = false;
   // TODO: Start from innermost BBs first
   for (auto &BB : F) {
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.cpp
index 5887d5e8bc26..8e3f5b77429c 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.cpp
@@ -17,11 +17,11 @@ RegionsFromMetadata::RegionsFromMetadata(StringRef Pipeline)
     : FunctionPass("regions-from-metadata"),
       RPM("rpm", Pipeline, SandboxVectorizerPassBuilder::createRegionPass) {}
 
-bool RegionsFromMetadata::runOnFunction(Function &F) {
+bool RegionsFromMetadata::runOnFunction(Function &F, const Analyses &A) {
   SmallVector<std::unique_ptr<sandboxir::Region>> Regions =
       sandboxir::Region::createRegionsFromMD(F);
   for (auto &R : Regions) {
-    RPM.runOnRegion(*R);
+    RPM.runOnRegion(*R, A);
   }
   return false;
 }
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
index c68f9482e337..96d825ed852f 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
@@ -51,6 +51,7 @@ SandboxVectorizerPass::~SandboxVectorizerPass() = default;
 PreservedAnalyses SandboxVectorizerPass::run(Function &F,
                                              FunctionAnalysisManager &AM) {
   TTI = &AM.getResult<TargetIRAnalysis>(F);
+  SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
 
   bool Changed = runImpl(F);
   if (!Changed)
@@ -82,5 +83,6 @@ bool SandboxVectorizerPass::runImpl(Function &LLVMF) {
   // Create SandboxIR for LLVMF and run BottomUpVec on it.
   sandboxir::Context Ctx(LLVMF.getContext());
   sandboxir::Function &F = *Ctx.createFunction(&LLVMF);
-  return FPM.runOnFunction(F);
+  sandboxir::Analyses A(*SE);
+  return FPM.runOnFunction(F, A);
 }
diff --git a/llvm/unittests/SandboxIR/PassTest.cpp b/llvm/unittests/SandboxIR/PassTest.cpp
index 866bd8233d80..751aedefd8fe 100644
--- a/llvm/unittests/SandboxIR/PassTest.cpp
+++ b/llvm/unittests/SandboxIR/PassTest.cpp
@@ -46,7 +46,7 @@ define void @foo() {
 
   public:
     TestPass(unsigned &BBCnt) : FunctionPass("test-pass"), BBCnt(BBCnt) {}
-    bool runOnFunction(Function &F) final {
+    bool runOnFunction(Function &F, const Analyses &A) final {
       for ([[maybe_unused]] auto &BB : F)
         ++BBCnt;
       return false;
@@ -59,7 +59,7 @@ define void @foo() {
   // Check classof().
   EXPECT_TRUE(llvm::isa<FunctionPass>(TPass));
   // Check runOnFunction();
-  TPass.runOnFunction(*F);
+  TPass.runOnFunction(*F, Analyses::emptyForTesting());
   EXPECT_EQ(BBCnt, 1u);
 #ifndef NDEBUG
   {
@@ -80,7 +80,7 @@ define void @foo() {
   class TestNamePass final : public FunctionPass {
   public:
     TestNamePass(llvm::StringRef Name) : FunctionPass(Name) {}
-    bool runOnFunction(Function &F) { return false; }
+    bool runOnFunction(Function &F, const Analyses &A) { return false; }
   };
   EXPECT_DEATH(TestNamePass("white space"), ".*whitespace.*");
   EXPECT_DEATH(TestNamePass("-dash"), ".*start with.*");
@@ -106,7 +106,7 @@ define i8 @foo(i8 %v0, i8 %v1) {
   public:
     TestPass(unsigned &InstCount)
         : RegionPass("test-pass"), InstCount(InstCount) {}
-    bool runOnRegion(Region &R) final {
+    bool runOnRegion(Region &R, const Analyses &A) final {
       for ([[maybe_unused]] auto &Inst : R) {
         ++InstCount;
       }
@@ -121,7 +121,7 @@ define i8 @foo(i8 %v0, i8 %v1) {
   llvm::SmallVector<std::unique_ptr<Region>> Regions =
       Region::createRegionsFromMD(*F);
   ASSERT_EQ(Regions.size(), 1u);
-  TPass.runOnRegion(*Regions[0]);
+  TPass.runOnRegion(*Regions[0], Analyses::emptyForTesting());
   EXPECT_EQ(InstCount, 2u);
 #ifndef NDEBUG
   {
@@ -142,7 +142,7 @@ define i8 @foo(i8 %v0, i8 %v1) {
   class TestNamePass final : public RegionPass {
   public:
     TestNamePass(llvm::StringRef Name) : RegionPass(Name) {}
-    bool runOnRegion(Region &F) { return false; }
+    bool runOnRegion(Region &F, const Analyses &A) { return false; }
   };
   EXPECT_DEATH(TestNamePass("white space"), ".*whitespace.*");
   EXPECT_DEATH(TestNamePass("-dash"), ".*start with.*");
@@ -161,7 +161,7 @@ define void @foo() {
 
   public:
     TestPass1(unsigned &BBCnt) : FunctionPass("test-pass1"), BBCnt(BBCnt) {}
-    bool runOnFunction(Function &F) final {
+    bool runOnFunction(Function &F, const Analyses &A) final {
       for ([[maybe_unused]] auto &BB : F)
         ++BBCnt;
       return false;
@@ -172,7 +172,7 @@ define void @foo() {
 
   public:
     TestPass2(unsigned &BBCnt) : FunctionPass("test-pass2"), BBCnt(BBCnt) {}
-    bool runOnFunction(Function &F) final {
+    bool runOnFunction(Function &F, const Analyses &A) final {
       for ([[maybe_unused]] auto &BB : F)
         ++BBCnt;
       return false;
@@ -185,7 +185,7 @@ define void @foo() {
   FPM.addPass(std::make_unique<TestPass1>(BBCnt1));
   FPM.addPass(std::make_unique<TestPass2>(BBCnt2));
   // Check runOnFunction().
-  FPM.runOnFunction(*F);
+  FPM.runOnFunction(*F, Analyses::emptyForTesting());
   EXPECT_EQ(BBCnt1, 1u);
   EXPECT_EQ(BBCnt2, 1u);
 #ifndef NDEBUG
@@ -216,7 +216,7 @@ define i8 @foo(i8 %v0, i8 %v1) {
   public:
     TestPass1(unsigned &InstCount)
         : RegionPass("test-pass1"), InstCount(InstCount) {}
-    bool runOnRegion(Region &R) final {
+    bool runOnRegion(Region &R, const Analyses &A) final {
       for ([[maybe_unused]] auto &Inst : R)
         ++InstCount;
       return false;
@@ -228,7 +228,7 @@ define i8 @foo(i8 %v0, i8 %v1) {
   public:
     TestPass2(unsigned &InstCount)
         : RegionPass("test-pass2"), InstCount(InstCount) {}
-    bool runOnRegion(Region &R) final {
+    bool runOnRegion(Region &R, const Analyses &A) final {
       for ([[maybe_unused]] auto &Inst : R)
         ++InstCount;
       return false;
@@ -244,7 +244,7 @@ define i8 @foo(i8 %v0, i8 %v1) {
   llvm::SmallVector<std::unique_ptr<Region>> Regions =
       Region::createRegionsFromMD(*F);
   ASSERT_EQ(Regions.size(), 1u);
-  RPM.runOnRegion(*Regions[0]);
+  RPM.runOnRegion(*Regions[0], Analyses::emptyForTesting());
   EXPECT_EQ(InstCount1, 2u);
   EXPECT_EQ(InstCount2, 2u);
 #ifndef NDEBUG
@@ -270,7 +270,7 @@ define void @f() {
   public:
     FooPass(std::string &Str, llvm::StringRef Args)
         : FunctionPass("foo-pass"), Str(Str), Args(Args.str()) {}
-    bool runOnFunction(Function &F) final {
+    bool runOnFunction(Function &F, const Analyses &A) final {
       Str += "foo<" + Args + ">";
       return false;
     }
@@ -282,7 +282,7 @@ define void @f() {
   public:
     BarPass(std::string &Str, llvm::StringRef Args)
         : FunctionPass("bar-pass"), Str(Str), Args(Args.str()) {}
-    bool runOnFunction(Function &F) final {
+    bool runOnFunction(Function &F, const Analyses &A) final {
       Str += "bar<" + Args + ">";
       return false;
     }
@@ -302,7 +302,7 @@ define void @f() {
   FunctionPassManager FPM("test-fpm");
   FPM.setPassPipeline("foo<abc>,bar<nested1<nested2<nested3>>>,foo",
                       CreatePass);
-  FPM.runOnFunction(*F);
+  FPM.runOnFunction(*F, Analyses::emptyForTesting());
   EXPECT_EQ(Str, "foo<abc>bar<nested1<nested2<nested3>>>foo<>");
 
   // A second call to setPassPipeline will trigger an assertion in debug mode.
-- 
GitLab


From 6128ff6630762310f6ae4eb61adda02cb4ad5260 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Tue, 22 Oct 2024 17:17:30 -0700
Subject: [PATCH 392/693] [JITLink][MachO] Add convenience functions for
 default text/data sections.

The getMachODefaultTextSection and getMachODefaultRWDataSection functions
return the "__TEXT,__text" and "__DATA,__data" sections respectively, creating
empty sections if the default sections are not already present in the graph.
These functions can be used by utilities that want to add code or data to these
standard sections (e.g. these functions can be used to supply the section
argument to the createAnonymousPointerJumpStub and
createPointerJumpStubBlock functions in the various targets).
---
 .../llvm/ExecutionEngine/JITLink/MachO.h      |  21 ++++
 .../Orc/Shared/MachOObjectFormat.h            |   1 +
 .../Orc/Shared/MachOObjectFormat.cpp          |   1 +
 .../ExecutionEngine/JITLink/CMakeLists.txt    |   3 +-
 .../ExecutionEngine/JITLink/JITLinkMocks.cpp  |  73 -----------
 .../JITLink/JITLinkTestUtils.cpp              | 114 ++++++++++++++++++
 .../{JITLinkMocks.h => JITLinkTestUtils.h}    |  12 +-
 .../JITLink/LinkGraphTests.cpp                |  43 +------
 .../JITLink/MachOLinkGraphTests.cpp           |  35 ++++++
 .../JITLink/MemoryManagerErrorTests.cpp       |   2 +-
 10 files changed, 184 insertions(+), 121 deletions(-)
 delete mode 100644 llvm/unittests/ExecutionEngine/JITLink/JITLinkMocks.cpp
 create mode 100644 llvm/unittests/ExecutionEngine/JITLink/JITLinkTestUtils.cpp
 rename llvm/unittests/ExecutionEngine/JITLink/{JITLinkMocks.h => JITLinkTestUtils.h} (95%)
 create mode 100644 llvm/unittests/ExecutionEngine/JITLink/MachOLinkGraphTests.cpp

diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/MachO.h b/llvm/include/llvm/ExecutionEngine/JITLink/MachO.h
index b8432c4d26c6..bb8da0ab9db2 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/MachO.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/MachO.h
@@ -14,6 +14,7 @@
 #define LLVM_EXECUTIONENGINE_JITLINK_MACHO_H
 
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
+#include "llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h"
 
 namespace llvm {
 namespace jitlink {
@@ -33,6 +34,26 @@ createLinkGraphFromMachOObject(MemoryBufferRef ObjectBuffer);
 void link_MachO(std::unique_ptr<LinkGraph> G,
                 std::unique_ptr<JITLinkContext> Ctx);
 
+/// Get a pointer to the standard MachO data section (creates an empty
+/// section with RW- permissions and standard lifetime if one does not
+/// already exist).
+inline Section &getMachODefaultRWDataSection(LinkGraph &G) {
+  if (auto *DataSec = G.findSectionByName(orc::MachODataDataSectionName))
+    return *DataSec;
+  return G.createSection(orc::MachODataDataSectionName,
+                         orc::MemProt::Read | orc::MemProt::Write);
+}
+
+/// Get a pointer to the standard MachO text section (creates an empty
+/// section with R-X permissions and standard lifetime if one does not
+/// already exist).
+inline Section &getMachODefaultTextSection(LinkGraph &G) {
+  if (auto *TextSec = G.findSectionByName(orc::MachOTextTextSectionName))
+    return *TextSec;
+  return G.createSection(orc::MachOTextTextSectionName,
+                         orc::MemProt::Read | orc::MemProt::Exec);
+}
+
 } // end namespace jitlink
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h
index f886203f8e3f..b927dfbce992 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h
@@ -49,6 +49,7 @@ extern StringRef MachOSwift5TypesSectionName;
 extern StringRef MachOSwift5TypeRefSectionName;
 extern StringRef MachOSwift5FieldMetadataSectionName;
 extern StringRef MachOSwift5EntrySectionName;
+extern StringRef MachOTextTextSectionName;
 extern StringRef MachOThreadBSSSectionName;
 extern StringRef MachOThreadDataSectionName;
 extern StringRef MachOThreadVarsSectionName;
diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/MachOObjectFormat.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/MachOObjectFormat.cpp
index 7f4c2934d026..11e8eb7bc3a1 100644
--- a/llvm/lib/ExecutionEngine/Orc/Shared/MachOObjectFormat.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Shared/MachOObjectFormat.cpp
@@ -42,6 +42,7 @@ StringRef MachOSwift5TypesSectionName = "__TEXT,__swift5_types";
 StringRef MachOSwift5TypeRefSectionName = "__TEXT,__swift5_typeref";
 StringRef MachOSwift5FieldMetadataSectionName = "__TEXT,__swift5_fieldmd";
 StringRef MachOSwift5EntrySectionName = "__TEXT,__swift5_entry";
+StringRef MachOTextTextSectionName = "__TEXT,__text";
 StringRef MachOThreadBSSSectionName = "__DATA,__thread_bss";
 StringRef MachOThreadDataSectionName = "__DATA,__thread_data";
 StringRef MachOThreadVarsSectionName = "__DATA,__thread_vars";
diff --git a/llvm/unittests/ExecutionEngine/JITLink/CMakeLists.txt b/llvm/unittests/ExecutionEngine/JITLink/CMakeLists.txt
index 82d277309307..d1c7b799880a 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/CMakeLists.txt
+++ b/llvm/unittests/ExecutionEngine/JITLink/CMakeLists.txt
@@ -11,8 +11,9 @@ add_llvm_unittest(JITLinkTests
     AArch32Tests.cpp
     AArch32ErrorTests.cpp
     EHFrameSupportTests.cpp
-    JITLinkMocks.cpp
+    JITLinkTestUtils.cpp
     LinkGraphTests.cpp
+    MachOLinkGraphTests.cpp
     MemoryManagerErrorTests.cpp
     StubsTests.cpp
   )
diff --git a/llvm/unittests/ExecutionEngine/JITLink/JITLinkMocks.cpp b/llvm/unittests/ExecutionEngine/JITLink/JITLinkMocks.cpp
deleted file mode 100644
index c40ce7adb0b5..000000000000
--- a/llvm/unittests/ExecutionEngine/JITLink/JITLinkMocks.cpp
+++ /dev/null
@@ -1,73 +0,0 @@
-//===--------- JITLinkMocks.cpp - Mock APIs for JITLink unit tests --------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "JITLinkMocks.h"
-#include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h"
-
-#include "llvm/Testing/Support/Error.h"
-#include "gtest/gtest.h"
-
-using namespace llvm;
-using namespace llvm::orc;
-using namespace llvm::jitlink;
-
-void lookupResolveEverythingToNull(
-    const llvm::jitlink::JITLinkContext::LookupMap &Symbols,
-    std::unique_ptr<llvm::jitlink::JITLinkAsyncLookupContinuation> LC) {
-  llvm::orc::ExecutorAddr Null;
-  llvm::jitlink::AsyncLookupResult Result;
-  for (auto &KV : Symbols)
-    Result[KV.first] = {Null, llvm::JITSymbolFlags::Exported};
-  LC->run(std::move(Result));
-}
-
-void lookupErrorOut(
-    const llvm::jitlink::JITLinkContext::LookupMap &Symbols,
-    std::unique_ptr<llvm::jitlink::JITLinkAsyncLookupContinuation> LC) {
-  LC->run(llvm::make_error<llvm::StringError>("Lookup failed",
-                                              llvm::inconvertibleErrorCode()));
-}
-
-std::unique_ptr<MockJITLinkContext> makeMockContext(
-    llvm::unique_function<void(llvm::Error)> HandleFailed,
-    llvm::unique_function<void(MockJITLinkMemoryManager &)> SetupMemMgr,
-    llvm::unique_function<void(MockJITLinkContext &)> SetupContext) {
-  auto MemMgr = std::make_unique<MockJITLinkMemoryManager>();
-  SetupMemMgr(*MemMgr);
-  auto Ctx = std::make_unique<MockJITLinkContext>(std::move(MemMgr),
-                                                  std::move(HandleFailed));
-  SetupContext(*Ctx);
-  return Ctx;
-}
-
-void defaultMemMgrSetup(MockJITLinkMemoryManager &) {}
-void defaultCtxSetup(MockJITLinkContext &) {}
-
-TEST(JITLinkMocks, SmokeTest) {
-  // Check that the testing infrastructure defaults can "link" a graph
-  // successfully.
-  auto G = std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"), 8,
-                                       llvm::endianness::little,
-                                       getGenericEdgeKindName);
-
-  ArrayRef<char> Content = "hello, world!";
-  auto &Sec =
-      G->createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
-  orc::ExecutorAddr B1Addr(0x1000);
-  auto &B = G->createContentBlock(Sec, Content, B1Addr, 8, 0);
-  G->addDefinedSymbol(B, 4, "S", 4, Linkage::Strong, Scope::Default, false,
-                      false);
-
-  Error Err = Error::success();
-  auto Ctx =
-      makeMockContext(JoinErrorsInto(Err), defaultMemMgrSetup, defaultCtxSetup);
-
-  link_MachO_x86_64(std::move(G), std::move(Ctx));
-
-  EXPECT_THAT_ERROR(std::move(Err), Succeeded());
-}
diff --git a/llvm/unittests/ExecutionEngine/JITLink/JITLinkTestUtils.cpp b/llvm/unittests/ExecutionEngine/JITLink/JITLinkTestUtils.cpp
new file mode 100644
index 000000000000..9a7878edab50
--- /dev/null
+++ b/llvm/unittests/ExecutionEngine/JITLink/JITLinkTestUtils.cpp
@@ -0,0 +1,114 @@
+//===------- JITLinkTestUtils.cpp - Utilities for JITLink unit tests ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "JITLinkTestUtils.h"
+#include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h"
+
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::orc;
+using namespace llvm::jitlink;
+
+static const char BlockContentBytes[] = {
+    0x54, 0x68, 0x65, 0x72, 0x65, 0x20, 0x77, 0x61, 0x73, 0x20, 0x6d, 0x6f,
+    0x76, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x20, 0x61, 0x74, 0x20, 0x74, 0x68,
+    0x65, 0x20, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2c, 0x20, 0x66,
+    0x6f, 0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x77, 0x6f, 0x72, 0x64, 0x20,
+    0x68, 0x61, 0x64, 0x20, 0x70, 0x61, 0x73, 0x73, 0x65, 0x64, 0x20, 0x61,
+    0x72, 0x6f, 0x75, 0x6e, 0x64, 0x0a, 0x54, 0x68, 0x61, 0x74, 0x20, 0x74,
+    0x68, 0x65, 0x20, 0x63, 0x6f, 0x6c, 0x74, 0x20, 0x66, 0x72, 0x6f, 0x6d,
+    0x20, 0x4f, 0x6c, 0x64, 0x20, 0x52, 0x65, 0x67, 0x72, 0x65, 0x74, 0x20,
+    0x68, 0x61, 0x64, 0x20, 0x67, 0x6f, 0x74, 0x20, 0x61, 0x77, 0x61, 0x79,
+    0x2c, 0x0a, 0x41, 0x6e, 0x64, 0x20, 0x68, 0x61, 0x64, 0x20, 0x6a, 0x6f,
+    0x69, 0x6e, 0x65, 0x64, 0x20, 0x74, 0x68, 0x65, 0x20, 0x77, 0x69, 0x6c,
+    0x64, 0x20, 0x62, 0x75, 0x73, 0x68, 0x20, 0x68, 0x6f, 0x72, 0x73, 0x65,
+    0x73, 0x20, 0x2d, 0x2d, 0x20, 0x68, 0x65, 0x20, 0x77, 0x61, 0x73, 0x20,
+    0x77, 0x6f, 0x72, 0x74, 0x68, 0x20, 0x61, 0x20, 0x74, 0x68, 0x6f, 0x75,
+    0x73, 0x61, 0x6e, 0x64, 0x20, 0x70, 0x6f, 0x75, 0x6e, 0x64, 0x2c, 0x0a,
+    0x53, 0x6f, 0x20, 0x61, 0x6c, 0x6c, 0x20, 0x74, 0x68, 0x65, 0x20, 0x63,
+    0x72, 0x61, 0x63, 0x6b, 0x73, 0x20, 0x68, 0x61, 0x64, 0x20, 0x67, 0x61,
+    0x74, 0x68, 0x65, 0x72, 0x65, 0x64, 0x20, 0x74, 0x6f, 0x20, 0x74, 0x68,
+    0x65, 0x20, 0x66, 0x72, 0x61, 0x79, 0x2e, 0x0a, 0x41, 0x6c, 0x6c, 0x20,
+    0x74, 0x68, 0x65, 0x20, 0x74, 0x72, 0x69, 0x65, 0x64, 0x20, 0x61, 0x6e,
+    0x64, 0x20, 0x6e, 0x6f, 0x74, 0x65, 0x64, 0x20, 0x72, 0x69, 0x64, 0x65,
+    0x72, 0x73, 0x20, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x74, 0x68, 0x65, 0x20,
+    0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x6e, 0x65, 0x61,
+    0x72, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x66, 0x61, 0x72, 0x0a, 0x48, 0x61,
+    0x64, 0x20, 0x6d, 0x75, 0x73, 0x74, 0x65, 0x72, 0x65, 0x64, 0x20, 0x61,
+    0x74, 0x20, 0x74, 0x68, 0x65, 0x20, 0x68, 0x6f, 0x6d, 0x65, 0x73, 0x74,
+    0x65, 0x61, 0x64, 0x20, 0x6f, 0x76, 0x65, 0x72, 0x6e, 0x69, 0x67, 0x68,
+    0x74, 0x2c, 0x0a, 0x46, 0x6f, 0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x62,
+    0x75, 0x73, 0x68, 0x6d, 0x65, 0x6e, 0x20, 0x6c, 0x6f, 0x76, 0x65, 0x20,
+    0x68, 0x61, 0x72, 0x64, 0x20, 0x72, 0x69, 0x64, 0x69, 0x6e, 0x67, 0x20,
+    0x77, 0x68, 0x65, 0x72, 0x65, 0x20, 0x74, 0x68, 0x65, 0x20, 0x77, 0x69,
+    0x6c, 0x64, 0x20, 0x62, 0x75, 0x73, 0x68, 0x20, 0x68, 0x6f, 0x72, 0x73,
+    0x65, 0x73, 0x20, 0x61, 0x72, 0x65, 0x2c, 0x0a, 0x41, 0x6e, 0x64, 0x20,
+    0x74, 0x68, 0x65, 0x20, 0x73, 0x74, 0x6f, 0x63, 0x6b, 0x2d, 0x68, 0x6f,
+    0x72, 0x73, 0x65, 0x20, 0x73, 0x6e, 0x75, 0x66, 0x66, 0x73, 0x20, 0x74,
+    0x68, 0x65, 0x20, 0x62, 0x61, 0x74, 0x74, 0x6c, 0x65, 0x20, 0x77, 0x69,
+    0x74, 0x68, 0x20, 0x64, 0x65, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x00};
+
+ArrayRef<char> BlockContent(BlockContentBytes);
+
+void lookupResolveEverythingToNull(
+    const llvm::jitlink::JITLinkContext::LookupMap &Symbols,
+    std::unique_ptr<llvm::jitlink::JITLinkAsyncLookupContinuation> LC) {
+  llvm::orc::ExecutorAddr Null;
+  llvm::jitlink::AsyncLookupResult Result;
+  for (auto &KV : Symbols)
+    Result[KV.first] = {Null, llvm::JITSymbolFlags::Exported};
+  LC->run(std::move(Result));
+}
+
+void lookupErrorOut(
+    const llvm::jitlink::JITLinkContext::LookupMap &Symbols,
+    std::unique_ptr<llvm::jitlink::JITLinkAsyncLookupContinuation> LC) {
+  LC->run(llvm::make_error<llvm::StringError>("Lookup failed",
+                                              llvm::inconvertibleErrorCode()));
+}
+
+std::unique_ptr<MockJITLinkContext> makeMockContext(
+    llvm::unique_function<void(llvm::Error)> HandleFailed,
+    llvm::unique_function<void(MockJITLinkMemoryManager &)> SetupMemMgr,
+    llvm::unique_function<void(MockJITLinkContext &)> SetupContext) {
+  auto MemMgr = std::make_unique<MockJITLinkMemoryManager>();
+  SetupMemMgr(*MemMgr);
+  auto Ctx = std::make_unique<MockJITLinkContext>(std::move(MemMgr),
+                                                  std::move(HandleFailed));
+  SetupContext(*Ctx);
+  return Ctx;
+}
+
+void defaultMemMgrSetup(MockJITLinkMemoryManager &) {}
+void defaultCtxSetup(MockJITLinkContext &) {}
+
+TEST(JITLinkMocks, SmokeTest) {
+  // Check that the testing infrastructure defaults can "link" a graph
+  // successfully.
+  auto G = std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"), 8,
+                                       llvm::endianness::little,
+                                       getGenericEdgeKindName);
+
+  ArrayRef<char> Content = "hello, world!";
+  auto &Sec =
+      G->createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
+  orc::ExecutorAddr B1Addr(0x1000);
+  auto &B = G->createContentBlock(Sec, Content, B1Addr, 8, 0);
+  G->addDefinedSymbol(B, 4, "S", 4, Linkage::Strong, Scope::Default, false,
+                      false);
+
+  Error Err = Error::success();
+  auto Ctx =
+      makeMockContext(JoinErrorsInto(Err), defaultMemMgrSetup, defaultCtxSetup);
+
+  link_MachO_x86_64(std::move(G), std::move(Ctx));
+
+  EXPECT_THAT_ERROR(std::move(Err), Succeeded());
+}
diff --git a/llvm/unittests/ExecutionEngine/JITLink/JITLinkMocks.h b/llvm/unittests/ExecutionEngine/JITLink/JITLinkTestUtils.h
similarity index 95%
rename from llvm/unittests/ExecutionEngine/JITLink/JITLinkMocks.h
rename to llvm/unittests/ExecutionEngine/JITLink/JITLinkTestUtils.h
index 8c1e3ff2c77d..dc077f900d19 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/JITLinkMocks.h
+++ b/llvm/unittests/ExecutionEngine/JITLink/JITLinkTestUtils.h
@@ -1,4 +1,4 @@
-//===----- JITLinkMocks.h - Mock APIs for JITLink unit tests ----*- C++ -*-===//
+//===--- JITLinkTestUtils.h - Utilities for JITLink unit tests --*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,12 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Mock APIs for JITLink unit tests.
+// Utilities for JITLink unit tests.
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_UNITTESTS_EXECUTIONENGINE_JITLINK_JITLINKMOCKS_H
-#define LLVM_UNITTESTS_EXECUTIONENGINE_JITLINK_JITLINKMOCKS_H
+#ifndef LLVM_UNITTESTS_EXECUTIONENGINE_JITLINK_JITLINKTESTUTILS_H
+#define LLVM_UNITTESTS_EXECUTIONENGINE_JITLINK_JITLINKTESTUTILS_H
 
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 
@@ -225,4 +225,6 @@ private:
   llvm::Error &Err;
 };
 
-#endif // LLVM_UNITTESTS_EXECUTIONENGINE_JITLINK_JITLINKMOCKS_H
+extern llvm::ArrayRef<char> BlockContent;
+
+#endif // LLVM_UNITTESTS_EXECUTIONENGINE_JITLINK_JITLINKTESTUTILS_H
diff --git a/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp b/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
index 5eea21184619..32d917d75d5c 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "JITLinkTestUtils.h"
+
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
@@ -17,47 +19,6 @@
 using namespace llvm;
 using namespace llvm::jitlink;
 
-static const char BlockContentBytes[] = {
-    0x54, 0x68, 0x65, 0x72, 0x65, 0x20, 0x77, 0x61, 0x73, 0x20, 0x6d, 0x6f,
-    0x76, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x20, 0x61, 0x74, 0x20, 0x74, 0x68,
-    0x65, 0x20, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2c, 0x20, 0x66,
-    0x6f, 0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x77, 0x6f, 0x72, 0x64, 0x20,
-    0x68, 0x61, 0x64, 0x20, 0x70, 0x61, 0x73, 0x73, 0x65, 0x64, 0x20, 0x61,
-    0x72, 0x6f, 0x75, 0x6e, 0x64, 0x0a, 0x54, 0x68, 0x61, 0x74, 0x20, 0x74,
-    0x68, 0x65, 0x20, 0x63, 0x6f, 0x6c, 0x74, 0x20, 0x66, 0x72, 0x6f, 0x6d,
-    0x20, 0x4f, 0x6c, 0x64, 0x20, 0x52, 0x65, 0x67, 0x72, 0x65, 0x74, 0x20,
-    0x68, 0x61, 0x64, 0x20, 0x67, 0x6f, 0x74, 0x20, 0x61, 0x77, 0x61, 0x79,
-    0x2c, 0x0a, 0x41, 0x6e, 0x64, 0x20, 0x68, 0x61, 0x64, 0x20, 0x6a, 0x6f,
-    0x69, 0x6e, 0x65, 0x64, 0x20, 0x74, 0x68, 0x65, 0x20, 0x77, 0x69, 0x6c,
-    0x64, 0x20, 0x62, 0x75, 0x73, 0x68, 0x20, 0x68, 0x6f, 0x72, 0x73, 0x65,
-    0x73, 0x20, 0x2d, 0x2d, 0x20, 0x68, 0x65, 0x20, 0x77, 0x61, 0x73, 0x20,
-    0x77, 0x6f, 0x72, 0x74, 0x68, 0x20, 0x61, 0x20, 0x74, 0x68, 0x6f, 0x75,
-    0x73, 0x61, 0x6e, 0x64, 0x20, 0x70, 0x6f, 0x75, 0x6e, 0x64, 0x2c, 0x0a,
-    0x53, 0x6f, 0x20, 0x61, 0x6c, 0x6c, 0x20, 0x74, 0x68, 0x65, 0x20, 0x63,
-    0x72, 0x61, 0x63, 0x6b, 0x73, 0x20, 0x68, 0x61, 0x64, 0x20, 0x67, 0x61,
-    0x74, 0x68, 0x65, 0x72, 0x65, 0x64, 0x20, 0x74, 0x6f, 0x20, 0x74, 0x68,
-    0x65, 0x20, 0x66, 0x72, 0x61, 0x79, 0x2e, 0x0a, 0x41, 0x6c, 0x6c, 0x20,
-    0x74, 0x68, 0x65, 0x20, 0x74, 0x72, 0x69, 0x65, 0x64, 0x20, 0x61, 0x6e,
-    0x64, 0x20, 0x6e, 0x6f, 0x74, 0x65, 0x64, 0x20, 0x72, 0x69, 0x64, 0x65,
-    0x72, 0x73, 0x20, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x74, 0x68, 0x65, 0x20,
-    0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x6e, 0x65, 0x61,
-    0x72, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x66, 0x61, 0x72, 0x0a, 0x48, 0x61,
-    0x64, 0x20, 0x6d, 0x75, 0x73, 0x74, 0x65, 0x72, 0x65, 0x64, 0x20, 0x61,
-    0x74, 0x20, 0x74, 0x68, 0x65, 0x20, 0x68, 0x6f, 0x6d, 0x65, 0x73, 0x74,
-    0x65, 0x61, 0x64, 0x20, 0x6f, 0x76, 0x65, 0x72, 0x6e, 0x69, 0x67, 0x68,
-    0x74, 0x2c, 0x0a, 0x46, 0x6f, 0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x62,
-    0x75, 0x73, 0x68, 0x6d, 0x65, 0x6e, 0x20, 0x6c, 0x6f, 0x76, 0x65, 0x20,
-    0x68, 0x61, 0x72, 0x64, 0x20, 0x72, 0x69, 0x64, 0x69, 0x6e, 0x67, 0x20,
-    0x77, 0x68, 0x65, 0x72, 0x65, 0x20, 0x74, 0x68, 0x65, 0x20, 0x77, 0x69,
-    0x6c, 0x64, 0x20, 0x62, 0x75, 0x73, 0x68, 0x20, 0x68, 0x6f, 0x72, 0x73,
-    0x65, 0x73, 0x20, 0x61, 0x72, 0x65, 0x2c, 0x0a, 0x41, 0x6e, 0x64, 0x20,
-    0x74, 0x68, 0x65, 0x20, 0x73, 0x74, 0x6f, 0x63, 0x6b, 0x2d, 0x68, 0x6f,
-    0x72, 0x73, 0x65, 0x20, 0x73, 0x6e, 0x75, 0x66, 0x66, 0x73, 0x20, 0x74,
-    0x68, 0x65, 0x20, 0x62, 0x61, 0x74, 0x74, 0x6c, 0x65, 0x20, 0x77, 0x69,
-    0x74, 0x68, 0x20, 0x64, 0x65, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x2e, 0x00};
-
-static ArrayRef<char> BlockContent(BlockContentBytes);
-
 TEST(LinkGraphTest, Construction) {
   // Check that LinkGraph construction works as expected.
   LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
diff --git a/llvm/unittests/ExecutionEngine/JITLink/MachOLinkGraphTests.cpp b/llvm/unittests/ExecutionEngine/JITLink/MachOLinkGraphTests.cpp
new file mode 100644
index 000000000000..be922275be26
--- /dev/null
+++ b/llvm/unittests/ExecutionEngine/JITLink/MachOLinkGraphTests.cpp
@@ -0,0 +1,35 @@
+//===------ LinkGraphTests.cpp - Unit tests for core JITLink classes ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "JITLinkTestUtils.h"
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ExecutionEngine/JITLink/JITLink.h"
+#include "llvm/ExecutionEngine/JITLink/MachO.h"
+
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::jitlink;
+
+TEST(MachOLinkGraphTest, GetStandardSections) {
+  // Check that LinkGraph construction works as expected.
+  LinkGraph G("foo", Triple("arm64-apple-darwin"), 8, llvm::endianness::little,
+              getGenericEdgeKindName);
+
+  auto &Data = getMachODefaultRWDataSection(G);
+  EXPECT_TRUE(Data.empty());
+  EXPECT_EQ(Data.getName(), orc::MachODataDataSectionName);
+  EXPECT_EQ(Data.getMemProt(), orc::MemProt::Read | orc::MemProt::Write);
+
+  auto &Text = getMachODefaultTextSection(G);
+  EXPECT_TRUE(Text.empty());
+  EXPECT_EQ(Text.getName(), orc::MachOTextTextSectionName);
+  EXPECT_EQ(Text.getMemProt(), orc::MemProt::Read | orc::MemProt::Exec);
+}
diff --git a/llvm/unittests/ExecutionEngine/JITLink/MemoryManagerErrorTests.cpp b/llvm/unittests/ExecutionEngine/JITLink/MemoryManagerErrorTests.cpp
index f0f3dd117c6f..2b303f7a8c1a 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/MemoryManagerErrorTests.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/MemoryManagerErrorTests.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "JITLinkMocks.h"
+#include "JITLinkTestUtils.h"
 #include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h"
 
 #include "llvm/Testing/Support/Error.h"
-- 
GitLab


From 0c1c37bfbed08c9d4e414a10f46cbed9a3e4c870 Mon Sep 17 00:00:00 2001
From: c8ef <c8ef@outlook.com>
Date: Tue, 29 Oct 2024 10:08:38 +0800
Subject: [PATCH 393/693] [TLI] Add support for the `tgamma` libcall. (#113791)

This patch adds the `tgamma` libcall.
---
 .../llvm/Analysis/TargetLibraryInfo.def       | 15 ++++++++++++++
 llvm/lib/Transforms/Utils/BuildLibCalls.cpp   |  3 +++
 .../Transforms/InferFunctionAttrs/annotate.ll |  9 +++++++++
 .../tools/llvm-tli-checker/ps4-tli-check.yaml | 20 +++++++++++++++----
 .../Analysis/TargetLibraryInfoTest.cpp        |  3 +++
 5 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.def b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
index f890e2b9ec4c..3e23e398f6a7 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.def
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
@@ -1140,6 +1140,21 @@ TLI_DEFINE_ENUM_INTERNAL(erfl)
 TLI_DEFINE_STRING_INTERNAL("erfl")
 TLI_DEFINE_SIG_INTERNAL(LDbl, LDbl)
 
+/// double tgamma(double x);
+TLI_DEFINE_ENUM_INTERNAL(tgamma)
+TLI_DEFINE_STRING_INTERNAL("tgamma")
+TLI_DEFINE_SIG_INTERNAL(Dbl, Dbl)
+
+/// float tgammaf(float x);
+TLI_DEFINE_ENUM_INTERNAL(tgammaf)
+TLI_DEFINE_STRING_INTERNAL("tgammaf")
+TLI_DEFINE_SIG_INTERNAL(Flt, Flt)
+
+/// long double tgammal(long double x);
+TLI_DEFINE_ENUM_INTERNAL(tgammal)
+TLI_DEFINE_STRING_INTERNAL("tgammal")
+TLI_DEFINE_SIG_INTERNAL(LDbl, LDbl)
+
 /// int execl(const char *path, const char *arg, ...);
 TLI_DEFINE_ENUM_INTERNAL(execl)
 TLI_DEFINE_STRING_INTERNAL("execl")
diff --git a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
index 13323604eb51..5fd4fd78c28a 100644
--- a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -1179,6 +1179,9 @@ bool llvm::inferNonMandatoryLibFuncAttrs(Function &F,
   case LibFunc_erf:
   case LibFunc_erff:
   case LibFunc_erfl:
+  case LibFunc_tgamma:
+  case LibFunc_tgammaf:
+  case LibFunc_tgammal:
   case LibFunc_exp:
   case LibFunc_expf:
   case LibFunc_expl:
diff --git a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
index 3e9b2d94efda..d8266f4c6703 100644
--- a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
+++ b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
@@ -667,6 +667,15 @@ declare float @logf(float)
 ; CHECK: declare x86_fp80 @logl(x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare x86_fp80 @logl(x86_fp80)
 
+; CHECK: declare double @tgamma(double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+declare double @tgamma(double)
+
+; CHECK: declare float @tgammaf(float) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+declare float @tgammaf(float)
+
+; CHECK: declare x86_fp80 @tgammal(x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+declare x86_fp80 @tgammal(x86_fp80)
+
 ; CHECK: declare noundef i32 @lstat(ptr nocapture noundef readonly, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
 declare i32 @lstat(ptr, ptr)
 
diff --git a/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml b/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
index 20e7e15e3efb..408b9c399342 100644
--- a/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
+++ b/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
@@ -34,7 +34,7 @@
 #
 # CHECK: << Total TLI yes SDK no:  18
 # CHECK: >> Total TLI no  SDK yes: 0
-# CHECK: == Total TLI yes SDK yes: 265
+# CHECK: == Total TLI yes SDK yes: 268
 #
 # WRONG_DETAIL: << TLI yes SDK no : '_ZdaPv' aka operator delete[](void*)
 # WRONG_DETAIL: >> TLI no  SDK yes: '_ZdaPvj' aka operator delete[](void*, unsigned int)
@@ -48,14 +48,14 @@
 # WRONG_DETAIL: << TLI yes SDK no : 'fminimum_numl'
 # WRONG_SUMMARY: << Total TLI yes SDK no:  19{{$}}
 # WRONG_SUMMARY: >> Total TLI no  SDK yes: 1{{$}}
-# WRONG_SUMMARY: == Total TLI yes SDK yes: 264
+# WRONG_SUMMARY: == Total TLI yes SDK yes: 267
 #
 ## The -COUNT suffix doesn't care if there are too many matches, so check
 ## the exact count first; the two directives should add up to that.
 ## Yes, this means additions to TLI will fail this test, but the argument
 ## to -COUNT can't be an expression.
-# AVAIL: TLI knows 516 symbols, 283 available
-# AVAIL-COUNT-283: {{^}} available
+# AVAIL: TLI knows 519 symbols, 286 available
+# AVAIL-COUNT-286: {{^}} available
 # AVAIL-NOT:       {{^}} available
 # UNAVAIL-COUNT-233: not available
 # UNAVAIL-NOT:       not available
@@ -390,6 +390,18 @@ DynamicSymbols:
     Type:            STT_FUNC
     Section:         .text
     Binding:         STB_GLOBAL
+  - Name:            tgamma
+    Type:            STT_FUNC
+    Section:         .text
+    Binding:         STB_GLOBAL
+  - Name:            tgammaf
+    Type:            STT_FUNC
+    Section:         .text
+    Binding:         STB_GLOBAL
+  - Name:            tgammal
+    Type:            STT_FUNC
+    Section:         .text
+    Binding:         STB_GLOBAL
   - Name:            exp
     Type:            STT_FUNC
     Section:         .text
diff --git a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
index 346940384aff..98f8989d4e6e 100644
--- a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
+++ b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
@@ -303,6 +303,9 @@ TEST_F(TargetLibraryInfoTest, ValidProto) {
       "declare double @erf(double)\n"
       "declare float @erff(float)\n"
       "declare x86_fp80 @erfl(x86_fp80)\n"
+      "declare double @tgamma(double)\n"
+      "declare float @tgammaf(float)\n"
+      "declare x86_fp80 @tgammal(x86_fp80)\n"
       "declare i32 @printf(i8*, ...)\n"
       "declare i32 @putc(i32, %struct*)\n"
       "declare i32 @putc_unlocked(i32, %struct*)\n"
-- 
GitLab


From 18311093abe6481388a0d963d58438d743b47569 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Tue, 29 Oct 2024 10:39:07 +0800
Subject: [PATCH 394/693] [InstCombine] Do not fold `shufflevector(select)` if
 the select condition is a vector (#113993)

Since `shufflevector` is not element-wise, we cannot do fold it into
select when the select condition is a vector.
For shufflevector that doesn't change the length, it doesn't crash, but
it is still a miscompilation: https://alive2.llvm.org/ce/z/s8saCx

Fixes https://github.com/llvm/llvm-project/issues/113986.
---
 .../InstCombine/InstCombineVectorOps.cpp      |  8 +++++--
 .../Transforms/InstCombine/vec_shuffle.ll     | 24 +++++++++++++++++++
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index 75e7c1c97018..454fe5a91d37 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -2902,8 +2902,12 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
 
   if (match(RHS, m_Constant())) {
     if (auto *SI = dyn_cast<SelectInst>(LHS)) {
-      if (Instruction *I = FoldOpIntoSelect(SVI, SI))
-        return I;
+      // We cannot do this fold for elementwise select since ShuffleVector is
+      // not elementwise.
+      if (SI->getCondition()->getType()->isIntegerTy()) {
+        if (Instruction *I = FoldOpIntoSelect(SVI, SI))
+          return I;
+      }
     }
     if (auto *PN = dyn_cast<PHINode>(LHS)) {
       if (Instruction *I = foldOpIntoPhi(SVI, PN))
diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
index d050cf10849e..39a9db02eef2 100644
--- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll
+++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
@@ -2387,6 +2387,30 @@ define <2 x i32> @foldselect0(i1 %c) {
   ret <2 x i32> %shuf
 }
 
+; Make sure we do not crash in this case.
+define <4 x float> @shuf_larger_length_vec_select(<2 x i1> %cond) {
+; CHECK-LABEL: @shuf_larger_length_vec_select(
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[COND:%.*]], <2 x float> zeroinitializer, <2 x float> <float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x float> [[SEL]], <2 x float> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[SHUF]]
+;
+  %sel = select <2 x i1> %cond, <2 x float> zeroinitializer, <2 x float> splat(float 1.000000e+00)
+  %shuf = shufflevector <2 x float> %sel, <2 x float> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x float> %shuf
+}
+
+; Make sure we do not fold in this case.
+define <4 x i32> @shuf_same_length_vec_select(<4 x i1> %cond) {
+; CHECK-LABEL: @shuf_same_length_vec_select(
+; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> [[COND:%.*]], <4 x i32> <i32 poison, i32 1, i32 2, i32 3>, <4 x i32> <i32 poison, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[SEL]], <4 x i32> <i32 poison, i32 9, i32 poison, i32 poison>, <4 x i32> <i32 2, i32 1, i32 3, i32 5>
+; CHECK-NEXT:    ret <4 x i32> [[SHUF]]
+;
+  %sel = select <4 x i1> %cond, <4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuf = shufflevector <4 x i32> %sel, <4 x i32> <i32 8, i32 9, i32 10, i32 11>, <4 x i32> <i32 2, i32 1, i32 3, i32 5>
+  ret <4 x i32> %shuf
+}
+
 declare i1 @cond()
 declare <4 x i32> @value()
 
-- 
GitLab


From 635c344dfb3227f80c76dfbee9d6bf44ef742675 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 28 Oct 2024 19:59:00 -0700
Subject: [PATCH 395/693] [X86] Add vector_compress patterns with a zero vector
 passthru. (#113970)

We can use the kz form to automatically zero the extra elements.

Fixes #113263.
---
 llvm/lib/Target/X86/X86InstrAVX512.td    |  3 ++
 llvm/test/CodeGen/X86/vector-compress.ll | 56 ++++++++++++++++++++++++
 2 files changed, 59 insertions(+)

diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index 98c31867e6b2..32c4ebc331f1 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -10549,6 +10549,9 @@ multiclass compress_by_vec_width_lowering<X86VectorVTInfo _, string Name> {
   def : Pat<(_.VT (vector_compress _.RC:$src, _.KRCWM:$mask, undef)),
             (!cast<Instruction>(Name#_.ZSuffix#rrkz)
                             _.KRCWM:$mask, _.RC:$src)>;
+  def : Pat<(_.VT (vector_compress _.RC:$src, _.KRCWM:$mask, _.ImmAllZerosV)),
+            (!cast<Instruction>(Name#_.ZSuffix#rrkz)
+                            _.KRCWM:$mask, _.RC:$src)>;
   def : Pat<(_.VT (vector_compress _.RC:$src, _.KRCWM:$mask, _.RC:$passthru)),
               (!cast<Instruction>(Name#_.ZSuffix#rrk)
                             _.RC:$passthru, _.KRCWM:$mask, _.RC:$src)>;
diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll
index 2b963ab896cc..f8c076db65de 100644
--- a/llvm/test/CodeGen/X86/vector-compress.ll
+++ b/llvm/test/CodeGen/X86/vector-compress.ll
@@ -1211,3 +1211,59 @@ define <3 x i3> @test_compress_narrow_illegal_element_type(<3 x i3> %vec, <3 x i
     %out = call <3 x i3> @llvm.experimental.vector.compress(<3 x i3> %vec, <3 x i1> %mask, <3 x i3> undef)
     ret <3 x i3> %out
 }
+
+define <4 x i32> @test_compress_v4i32_zero_passthru(<4 x i32> %vec, <4 x i1> %mask) {
+; AVX2-LABEL: test_compress_v4i32_zero_passthru:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vmovaps %xmm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    vmovss %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    vmovd %xmm1, %eax
+; AVX2-NEXT:    andl $1, %eax
+; AVX2-NEXT:    vextractps $1, %xmm0, -24(%rsp,%rax,4)
+; AVX2-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX2-NEXT:    andl $1, %ecx
+; AVX2-NEXT:    addq %rax, %rcx
+; AVX2-NEXT:    vextractps $2, %xmm0, -24(%rsp,%rcx,4)
+; AVX2-NEXT:    vpextrd $2, %xmm1, %eax
+; AVX2-NEXT:    andl $1, %eax
+; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX2-NEXT:    andl $1, %ecx
+; AVX2-NEXT:    addq %rax, %rcx
+; AVX2-NEXT:    # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT:    andl $3, %eax
+; AVX2-NEXT:    vextractps $3, %xmm0, -24(%rsp,%rax,4)
+; AVX2-NEXT:    xorl %eax, %eax
+; AVX2-NEXT:    cmpq $3, %rcx
+; AVX2-NEXT:    movl $3, %edx
+; AVX2-NEXT:    cmovbq %rcx, %rdx
+; AVX2-NEXT:    vextractps $3, %xmm0, %ecx
+; AVX2-NEXT:    cmovbel %eax, %ecx
+; AVX2-NEXT:    movl %ecx, -24(%rsp,%rdx,4)
+; AVX2-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: test_compress_v4i32_zero_passthru:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
+; AVX512F-NEXT:    vpcompressd %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: test_compress_v4i32_zero_passthru:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX512VL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512VL-NEXT:    vpcompressd %xmm0, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    retq
+    %out = call <4 x i32> @llvm.experimental.vector.compress(<4 x i32> %vec, <4 x i1> %mask, <4 x i32> zeroinitializer)
+    ret <4 x i32> %out
+}
-- 
GitLab


From 00ca2071e08f3a82171e564618981906a15e8dca Mon Sep 17 00:00:00 2001
From: Jerry Sun <105613447+jerryyiransun@users.noreply.github.com>
Date: Mon, 28 Oct 2024 23:26:10 -0400
Subject: [PATCH 396/693] [TableGen] [NFC] Remove unused includes in TableGen
 BE (#113725)

split PR as requested from
https://github.com/llvm/llvm-project/pull/113318.

Removes unused imports in TableGen BE
---
 llvm/utils/TableGen/Attributes.cpp                | 1 -
 llvm/utils/TableGen/CTagsEmitter.cpp              | 2 +-
 llvm/utils/TableGen/DXILEmitter.cpp               | 1 -
 llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp | 2 --
 llvm/utils/TableGen/GlobalISelEmitter.cpp         | 1 -
 llvm/utils/TableGen/IntrinsicEmitter.cpp          | 1 -
 llvm/utils/TableGen/OptionParserEmitter.cpp       | 1 -
 llvm/utils/TableGen/PseudoLoweringEmitter.cpp     | 1 -
 llvm/utils/TableGen/VTEmitter.cpp                 | 1 -
 9 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/llvm/utils/TableGen/Attributes.cpp b/llvm/utils/TableGen/Attributes.cpp
index 138275356dc9..66ba25c6dcc8 100644
--- a/llvm/utils/TableGen/Attributes.cpp
+++ b/llvm/utils/TableGen/Attributes.cpp
@@ -9,7 +9,6 @@
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
-#include <vector>
 using namespace llvm;
 
 #define DEBUG_TYPE "attr-enum"
diff --git a/llvm/utils/TableGen/CTagsEmitter.cpp b/llvm/utils/TableGen/CTagsEmitter.cpp
index 3718486ff7ad..413d8f5dbcff 100644
--- a/llvm/utils/TableGen/CTagsEmitter.cpp
+++ b/llvm/utils/TableGen/CTagsEmitter.cpp
@@ -17,7 +17,7 @@
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
-#include <algorithm>
+#include <tuple>
 #include <vector>
 using namespace llvm;
 
diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp
index 0598baea9be7..467a6163ae3b 100644
--- a/llvm/utils/TableGen/DXILEmitter.cpp
+++ b/llvm/utils/TableGen/DXILEmitter.cpp
@@ -14,7 +14,6 @@
 #include "Basic/SequenceToOffsetTable.h"
 #include "Common/CodeGenTarget.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSet.h"
diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
index 424f1ccb067f..149ba7a1d903 100644
--- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
@@ -40,9 +40,7 @@
 #include "Common/SubtargetFeatureInfo.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/EquivalenceClasses.h"
-#include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/MapVector.h"
-#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSet.h"
diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index e866bd983e04..859310906af4 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -38,7 +38,6 @@
 #include "Common/GlobalISel/GlobalISelMatchTable.h"
 #include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h"
 #include "Common/InfoByHwMode.h"
-#include "Common/SubtargetFeatureInfo.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGenTypes/LowLevelType.h"
 #include "llvm/CodeGenTypes/MachineValueType.h"
diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp
index 1968e7eac21e..070d7522a97b 100644
--- a/llvm/utils/TableGen/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp
@@ -29,7 +29,6 @@
 #include <array>
 #include <cassert>
 #include <cctype>
-#include <limits>
 #include <map>
 #include <optional>
 #include <string>
diff --git a/llvm/utils/TableGen/OptionParserEmitter.cpp b/llvm/utils/TableGen/OptionParserEmitter.cpp
index 2872762cc7fd..cd7a140bb231 100644
--- a/llvm/utils/TableGen/OptionParserEmitter.cpp
+++ b/llvm/utils/TableGen/OptionParserEmitter.cpp
@@ -15,7 +15,6 @@
 #include "llvm/TableGen/TableGenBackend.h"
 #include <cstring>
 #include <map>
-#include <memory>
 
 using namespace llvm;
 
diff --git a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
index d2d2bd91445a..bcbc6ea20751 100644
--- a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
+++ b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
@@ -17,7 +17,6 @@
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TGTimer.h"
 #include "llvm/TableGen/TableGenBackend.h"
-#include <vector>
 using namespace llvm;
 
 #define DEBUG_TYPE "pseudo-lowering"
diff --git a/llvm/utils/TableGen/VTEmitter.cpp b/llvm/utils/TableGen/VTEmitter.cpp
index 8f4bcd5fccc7..4cbc7abd699d 100644
--- a/llvm/utils/TableGen/VTEmitter.cpp
+++ b/llvm/utils/TableGen/VTEmitter.cpp
@@ -10,7 +10,6 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
-#include <array>
 #include <cassert>
 #include <map>
 using namespace llvm;
-- 
GitLab


From 6588073724d3241d90663e45154d806a28bce95a Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Tue, 29 Oct 2024 13:19:43 +0900
Subject: [PATCH 397/693] [mlir][func] Fix incorrect API usage in
 `FuncOpConversion` (#113977)

This commit fixes a case of incorrect dialect conversion API usage
during `FuncOpConversion`. `replaceAllUsesExcept` (same as
`replaceAllUsesWith`) is currently not supported in a dialect
conversion. `replaceUsesOfBlockArgument` should be used instead. It
sometimes works anyway (like in this case), but that's just because of
the way we insert materializations.

This commit is in preparation of merging the 1:1 and 1:N dialect
conversion drivers. (At that point, the current use of
`replaceAllUsesExcept` will no longer work.)
---
 mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 27c43e0daad0..c046ea1b824f 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -273,7 +273,7 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
 static void restoreByValRefArgumentType(
     ConversionPatternRewriter &rewriter, const LLVMTypeConverter &typeConverter,
     ArrayRef<std::optional<NamedAttribute>> byValRefNonPtrAttrs,
-    LLVM::LLVMFuncOp funcOp) {
+    ArrayRef<BlockArgument> oldBlockArgs, LLVM::LLVMFuncOp funcOp) {
   // Nothing to do for function declarations.
   if (funcOp.isExternal())
     return;
@@ -281,8 +281,8 @@ static void restoreByValRefArgumentType(
   ConversionPatternRewriter::InsertionGuard guard(rewriter);
   rewriter.setInsertionPointToStart(&funcOp.getFunctionBody().front());
 
-  for (const auto &[arg, byValRefAttr] :
-       llvm::zip(funcOp.getArguments(), byValRefNonPtrAttrs)) {
+  for (const auto &[arg, oldArg, byValRefAttr] :
+       llvm::zip(funcOp.getArguments(), oldBlockArgs, byValRefNonPtrAttrs)) {
     // Skip argument if no `llvm.byval` or `llvm.byref` attribute.
     if (!byValRefAttr)
       continue;
@@ -295,7 +295,7 @@ static void restoreByValRefArgumentType(
         cast<TypeAttr>(byValRefAttr->getValue()).getValue());
 
     auto valueArg = rewriter.create<LLVM::LoadOp>(arg.getLoc(), resTy, arg);
-    rewriter.replaceAllUsesExcept(arg, valueArg, valueArg);
+    rewriter.replaceUsesOfBlockArgument(oldArg, valueArg);
   }
 }
 
@@ -309,6 +309,10 @@ mlir::convertFuncOpToLLVMFuncOp(FunctionOpInterface funcOp,
     return rewriter.notifyMatchFailure(
         funcOp, "Only support FunctionOpInterface with FunctionType");
 
+  // Keep track of the entry block arguments. They will be needed later.
+  SmallVector<BlockArgument> oldBlockArgs =
+      llvm::to_vector(funcOp.getArguments());
+
   // Convert the original function arguments. They are converted using the
   // LLVMTypeConverter provided to this legalization pattern.
   auto varargsAttr = funcOp->getAttrOfType<BoolAttr>(varargsAttrName);
@@ -438,7 +442,7 @@ mlir::convertFuncOpToLLVMFuncOp(FunctionOpInterface funcOp,
   // pointee type in the function body when converting `llvm.byval`/`llvm.byref`
   // function arguments.
   restoreByValRefArgumentType(rewriter, converter, byValRefNonPtrAttrs,
-                              newFuncOp);
+                              oldBlockArgs, newFuncOp);
 
   if (!shouldUseBarePtrCallConv(funcOp, &converter)) {
     if (funcOp->getAttrOfType<UnitAttr>(
-- 
GitLab


From 828467a54e156cbb04820ae47df32c45fbc75fc0 Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic@gmail.com>
Date: Tue, 29 Oct 2024 14:15:43 +0900
Subject: [PATCH 398/693] Fix warnings introduced in #111434
 [-Wnontrivial-memaccess]

---
 lld/ELF/Symbols.h                            | 2 +-
 llvm/include/llvm/Support/Endian.h           | 2 +-
 llvm/lib/ExecutionEngine/ExecutionEngine.cpp | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lld/ELF/Symbols.h b/lld/ELF/Symbols.h
index 339f32e05f16..8c9c9a56cfbc 100644
--- a/lld/ELF/Symbols.h
+++ b/lld/ELF/Symbols.h
@@ -75,7 +75,7 @@ public:
 
   // The default copy constructor is deleted due to atomic flags. Define one for
   // places where no atomic is needed.
-  Symbol(const Symbol &o) { memcpy(this, &o, sizeof(o)); }
+  Symbol(const Symbol &o) { memcpy(static_cast<void *>(this), &o, sizeof(o)); }
 
 protected:
   const char *nameData;
diff --git a/llvm/include/llvm/Support/Endian.h b/llvm/include/llvm/Support/Endian.h
index 5831fe66a1f7..f86ea8901ae4 100644
--- a/llvm/include/llvm/Support/Endian.h
+++ b/llvm/include/llvm/Support/Endian.h
@@ -58,7 +58,7 @@ template <typename value_type, std::size_t alignment = unaligned>
 [[nodiscard]] inline value_type read(const void *memory, endianness endian) {
   value_type ret;
 
-  memcpy(&ret,
+  memcpy(static_cast<void *>(&ret),
          LLVM_ASSUME_ALIGNED(
              memory, (detail::PickAlignment<value_type, alignment>::value)),
          sizeof(value_type));
diff --git a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
index f09975331bba..42622ea12152 100644
--- a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -1056,7 +1056,7 @@ void ExecutionEngine::StoreValueToMemory(const GenericValue &Val,
     *((double*)Ptr) = Val.DoubleVal;
     break;
   case Type::X86_FP80TyID:
-    memcpy(Ptr, Val.IntVal.getRawData(), 10);
+    memcpy(static_cast<void *>(Ptr), Val.IntVal.getRawData(), 10);
     break;
   case Type::PointerTyID:
     // Ensure 64 bit target pointers are fully initialized on 32 bit hosts.
-- 
GitLab


From d3b98559be72682da45df73522173cb315912f6f Mon Sep 17 00:00:00 2001
From: Peng Liu <winner245@hotmail.com>
Date: Tue, 29 Oct 2024 01:29:37 -0400
Subject: [PATCH 399/693] Add exception guard for constructor vector(n, x, a)
 (#113086)

Added exception guard to the `vector(n, x, a)` constructor to enhance
exception safety. This change ensures that the `vector(n, x, a)`
constructor is consistent with other constructors, such as `vector(n)`,
`vector(n, x)`, `vector(n, a)`, in terms of exception safety.
---
 libcxx/include/__vector/vector.h                          | 2 ++
 .../sequences/vector/vector.cons/exceptions.pass.cpp      | 8 ++++++++
 2 files changed, 10 insertions(+)

diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h
index 7889e8c2201a..844e5d6a2105 100644
--- a/libcxx/include/__vector/vector.h
+++ b/libcxx/include/__vector/vector.h
@@ -165,10 +165,12 @@ public:
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
   vector(size_type __n, const value_type& __x, const allocator_type& __a)
       : __alloc_(__a) {
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
     if (__n > 0) {
       __vallocate(__n);
       __construct_at_end(__n, __x);
     }
+    __guard.__complete();
   }
 
   template <class _InputIterator,
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/exceptions.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/exceptions.pass.cpp
index 3ef5aeecc1b0..c9c1bac2fb4a 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/exceptions.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/exceptions.pass.cpp
@@ -139,6 +139,14 @@ int main(int, char**) {
   check_new_delete_called();
 #endif  // TEST_STD_VER >= 14
 
+  try { // Throw in vector(size_type, value_type, const allocator_type&) from the type
+    int throw_after = 1;
+    ThrowingT v(throw_after);
+    std::vector<ThrowingT> vec(1, v, std::allocator<ThrowingT>());
+  } catch (int) {
+  }
+  check_new_delete_called();
+
   try { // Throw in vector(InputIterator, InputIterator) from input iterator
     std::vector<int> vec((Iterator<std::input_iterator_tag>()), Iterator<std::input_iterator_tag>(2));
   } catch (int) {
-- 
GitLab


From 3f4468faaa9525ad615118675c3c68938f4a8d5f Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 28 Oct 2024 22:37:47 -0700
Subject: [PATCH 400/693] [RISCV] Teach expandRV32ZdinxStore to handle
 memoperand not being present. (#113981)

I received a report that the outliner drops memoperands and causes this
code to crash. Handle this by only copying the memoperand if it exists.

Similar for expandRV32ZdinxLoad
---
 .../Target/RISCV/RISCVExpandPseudoInsts.cpp   | 89 ++++++++++---------
 1 file changed, 47 insertions(+), 42 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
index 5dcec078856e..eb3e1a1fe9fd 100644
--- a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
@@ -320,34 +320,37 @@ bool RISCVExpandPseudo::expandRV32ZdinxStore(MachineBasicBlock &MBB,
   Register Hi =
       TRI->getSubReg(MBBI->getOperand(0).getReg(), RISCV::sub_gpr_odd);
 
-  assert(MBBI->hasOneMemOperand() && "Expected mem operand");
-  MachineMemOperand *OldMMO = MBBI->memoperands().front();
-  MachineFunction *MF = MBB.getParent();
-  MachineMemOperand *MMOLo = MF->getMachineMemOperand(OldMMO, 0, 4);
-  MachineMemOperand *MMOHi = MF->getMachineMemOperand(OldMMO, 4, 4);
-
-  BuildMI(MBB, MBBI, DL, TII->get(RISCV::SW))
-      .addReg(Lo, getKillRegState(MBBI->getOperand(0).isKill()))
-      .addReg(MBBI->getOperand(1).getReg())
-      .add(MBBI->getOperand(2))
-      .setMemRefs(MMOLo);
+  auto MIBLo = BuildMI(MBB, MBBI, DL, TII->get(RISCV::SW))
+                   .addReg(Lo, getKillRegState(MBBI->getOperand(0).isKill()))
+                   .addReg(MBBI->getOperand(1).getReg())
+                   .add(MBBI->getOperand(2));
 
+  MachineInstrBuilder MIBHi;
   if (MBBI->getOperand(2).isGlobal() || MBBI->getOperand(2).isCPI()) {
     assert(MBBI->getOperand(2).getOffset() % 8 == 0);
     MBBI->getOperand(2).setOffset(MBBI->getOperand(2).getOffset() + 4);
-    BuildMI(MBB, MBBI, DL, TII->get(RISCV::SW))
-        .addReg(Hi, getKillRegState(MBBI->getOperand(0).isKill()))
-        .add(MBBI->getOperand(1))
-        .add(MBBI->getOperand(2))
-        .setMemRefs(MMOHi);
+    MIBHi = BuildMI(MBB, MBBI, DL, TII->get(RISCV::SW))
+                .addReg(Hi, getKillRegState(MBBI->getOperand(0).isKill()))
+                .add(MBBI->getOperand(1))
+                .add(MBBI->getOperand(2));
   } else {
     assert(isInt<12>(MBBI->getOperand(2).getImm() + 4));
-    BuildMI(MBB, MBBI, DL, TII->get(RISCV::SW))
-        .addReg(Hi, getKillRegState(MBBI->getOperand(0).isKill()))
-        .add(MBBI->getOperand(1))
-        .addImm(MBBI->getOperand(2).getImm() + 4)
-        .setMemRefs(MMOHi);
+    MIBHi = BuildMI(MBB, MBBI, DL, TII->get(RISCV::SW))
+                .addReg(Hi, getKillRegState(MBBI->getOperand(0).isKill()))
+                .add(MBBI->getOperand(1))
+                .addImm(MBBI->getOperand(2).getImm() + 4);
+  }
+
+  if (!MBBI->memoperands_empty()) {
+    assert(MBBI->hasOneMemOperand() && "Expected mem operand");
+    MachineMemOperand *OldMMO = MBBI->memoperands().front();
+    MachineFunction *MF = MBB.getParent();
+    MachineMemOperand *MMOLo = MF->getMachineMemOperand(OldMMO, 0, 4);
+    MachineMemOperand *MMOHi = MF->getMachineMemOperand(OldMMO, 4, 4);
+    MIBLo.setMemRefs(MMOLo);
+    MIBHi.setMemRefs(MMOHi);
   }
+
   MBBI->eraseFromParent();
   return true;
 }
@@ -364,46 +367,48 @@ bool RISCVExpandPseudo::expandRV32ZdinxLoad(MachineBasicBlock &MBB,
   Register Hi =
       TRI->getSubReg(MBBI->getOperand(0).getReg(), RISCV::sub_gpr_odd);
 
-  assert(MBBI->hasOneMemOperand() && "Expected mem operand");
-  MachineMemOperand *OldMMO = MBBI->memoperands().front();
-  MachineFunction *MF = MBB.getParent();
-  MachineMemOperand *MMOLo = MF->getMachineMemOperand(OldMMO, 0, 4);
-  MachineMemOperand *MMOHi = MF->getMachineMemOperand(OldMMO, 4, 4);
+  MachineInstrBuilder MIBLo, MIBHi;
 
   // If the register of operand 1 is equal to the Lo register, then swap the
   // order of loading the Lo and Hi statements.
   bool IsOp1EqualToLo = Lo == MBBI->getOperand(1).getReg();
   // Order: Lo, Hi
   if (!IsOp1EqualToLo) {
-    BuildMI(MBB, MBBI, DL, TII->get(RISCV::LW), Lo)
-        .addReg(MBBI->getOperand(1).getReg())
-        .add(MBBI->getOperand(2))
-        .setMemRefs(MMOLo);
+    MIBLo = BuildMI(MBB, MBBI, DL, TII->get(RISCV::LW), Lo)
+                .addReg(MBBI->getOperand(1).getReg())
+                .add(MBBI->getOperand(2));
   }
 
   if (MBBI->getOperand(2).isGlobal() || MBBI->getOperand(2).isCPI()) {
     auto Offset = MBBI->getOperand(2).getOffset();
     assert(Offset % 8 == 0);
     MBBI->getOperand(2).setOffset(Offset + 4);
-    BuildMI(MBB, MBBI, DL, TII->get(RISCV::LW), Hi)
-        .addReg(MBBI->getOperand(1).getReg())
-        .add(MBBI->getOperand(2))
-        .setMemRefs(MMOHi);
+    MIBHi = BuildMI(MBB, MBBI, DL, TII->get(RISCV::LW), Hi)
+                .addReg(MBBI->getOperand(1).getReg())
+                .add(MBBI->getOperand(2));
     MBBI->getOperand(2).setOffset(Offset);
   } else {
     assert(isInt<12>(MBBI->getOperand(2).getImm() + 4));
-    BuildMI(MBB, MBBI, DL, TII->get(RISCV::LW), Hi)
-        .addReg(MBBI->getOperand(1).getReg())
-        .addImm(MBBI->getOperand(2).getImm() + 4)
-        .setMemRefs(MMOHi);
+    MIBHi = BuildMI(MBB, MBBI, DL, TII->get(RISCV::LW), Hi)
+                .addReg(MBBI->getOperand(1).getReg())
+                .addImm(MBBI->getOperand(2).getImm() + 4);
   }
 
   // Order: Hi, Lo
   if (IsOp1EqualToLo) {
-    BuildMI(MBB, MBBI, DL, TII->get(RISCV::LW), Lo)
-        .addReg(MBBI->getOperand(1).getReg())
-        .add(MBBI->getOperand(2))
-        .setMemRefs(MMOLo);
+    MIBLo = BuildMI(MBB, MBBI, DL, TII->get(RISCV::LW), Lo)
+                .addReg(MBBI->getOperand(1).getReg())
+                .add(MBBI->getOperand(2));
+  }
+
+  if (!MBBI->memoperands_empty()) {
+    assert(MBBI->hasOneMemOperand() && "Expected mem operand");
+    MachineMemOperand *OldMMO = MBBI->memoperands().front();
+    MachineFunction *MF = MBB.getParent();
+    MachineMemOperand *MMOLo = MF->getMachineMemOperand(OldMMO, 0, 4);
+    MachineMemOperand *MMOHi = MF->getMachineMemOperand(OldMMO, 4, 4);
+    MIBLo.setMemRefs(MMOLo);
+    MIBHi.setMemRefs(MMOHi);
   }
 
   MBBI->eraseFromParent();
-- 
GitLab


From 48adfaf3b290d97260eabb53254de9ada313cd0e Mon Sep 17 00:00:00 2001
From: Sylvestre Ledru <sylvestre@debian.org>
Date: Tue, 29 Oct 2024 08:05:33 +0100
Subject: [PATCH 401/693] doc: remove trailing whitespaces

---
 clang/docs/FunctionEffectAnalysis.rst | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/clang/docs/FunctionEffectAnalysis.rst b/clang/docs/FunctionEffectAnalysis.rst
index f286aec4f5b9..3f2c4db7bad0 100644
--- a/clang/docs/FunctionEffectAnalysis.rst
+++ b/clang/docs/FunctionEffectAnalysis.rst
@@ -54,9 +54,9 @@ attached to functions, blocks, function pointers, lambdas, and member functions.
 The attribute applies only to the function itself. In particular, it does not apply to any nested
 functions or declarations, such as blocks, lambdas, and local classes.
 
-This document uses the C++/C23 syntax ``[[clang::nonblocking]]``, since it parallels the placement 
+This document uses the C++/C23 syntax ``[[clang::nonblocking]]``, since it parallels the placement
 of the ``noexcept`` specifier, and the attributes have other similarities to ``noexcept``. The GNU
-``__attribute__((nonblocking))`` syntax is also supported. Note that it requires a different 
+``__attribute__((nonblocking))`` syntax is also supported. Note that it requires a different
 placement on a C++ type alias.
 
 Like ``noexcept``, ``nonblocking`` and ``nonallocating`` have an optional argument, a compile-time
@@ -76,10 +76,10 @@ series of performance constraints. From weakest to strongest:
 - ``nonblocking``: The function type will never block on a lock, allocate memory on the heap,
   or throw an exception.
 
-``nonblocking`` includes the ``nonallocating`` guarantee. 
+``nonblocking`` includes the ``nonallocating`` guarantee.
 
 While ``nonblocking`` and ``nonallocating`` are conceptually a superset of ``noexcept``, neither
-attribute implicitly specifies ``noexcept``. Further, ``noexcept`` has a specified runtime behavior of 
+attribute implicitly specifies ``noexcept``. Further, ``noexcept`` has a specified runtime behavior of
 aborting if an exception is thrown, while the ``nonallocating`` and ``nonblocking`` attributes are
 mainly for compile-time analysis and have no runtime behavior, except in code built
 with Clang's :doc:`RealtimeSanitizer`. Nonetheless, Clang emits a
@@ -95,7 +95,7 @@ function, as described in the section "Analysis and warnings", below.
 explicitly disable any potential inference of ``nonblocking`` or ``nonallocating`` during
 verification. (Inference is described later in this document). ``nonblocking(false)`` and
 ``nonallocating(false)`` are legal, but superfluous  when applied to a function *type*
-that is not part of a declarator: ``float (int) [[nonblocking(false)]]`` and 
+that is not part of a declarator: ``float (int) [[nonblocking(false)]]`` and
 ``float (int)`` are identical types.
 
 For functions with no explicit performance constraint, the worst is assumed: the function
@@ -153,7 +153,7 @@ are comparable to that for ``noexcept`` in C++17 and later.
     void (*fp_nonallocating)() [[clang::nonallocating]];
     fp_nonallocating = nullptr;
     fp_nonallocating = nonallocating;
-    fp_nonallocating = nonblocking; // no warning because nonblocking includes nonallocating 
+    fp_nonallocating = nonblocking; // no warning because nonblocking includes nonallocating
     fp_nonallocating = unannotated;
     // ^ warning: attribute 'nonallocating' should not be added via type conversion
   }
@@ -274,7 +274,7 @@ following rules. Such functions:
      from the analysis. (The reason for requiring ``noexcept`` in C++ is that a function declared
      ``noreturn`` could be a wrapper for ``throw``.)
 
-5. May not invoke or access an Objective-C method or property, since ``objc_msgSend()`` calls into 
+5. May not invoke or access an Objective-C method or property, since ``objc_msgSend()`` calls into
    the Objective-C runtime, which may allocate memory or otherwise block.
 
 6. May not access thread-local variables. Typically, thread-local variables are allocated on the
-- 
GitLab


From 59085e9c3f522b59e512f1651058a9f98c739088 Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic@gmail.com>
Date: Tue, 29 Oct 2024 16:16:26 +0900
Subject: [PATCH 402/693] Fix more lld warnings introduced in #111434
 [-Wnontrivial-memaccess]

---
 lld/ELF/Arch/ARM.cpp    | 2 +-
 lld/ELF/SymbolTable.cpp | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lld/ELF/Arch/ARM.cpp b/lld/ELF/Arch/ARM.cpp
index be3f80337aae..271e2e27a269 100644
--- a/lld/ELF/Arch/ARM.cpp
+++ b/lld/ELF/Arch/ARM.cpp
@@ -1207,7 +1207,7 @@ template <class ELFT> void ObjFile<ELFT>::importCmseSymbols() {
     Defined *sym = reinterpret_cast<Defined *>(make<SymbolUnion>());
 
     // Initialize symbol fields.
-    memset(sym, 0, sizeof(Symbol));
+    memset(static_cast<void *>(sym), 0, sizeof(Symbol));
     sym->setName(CHECK(eSyms[i].getName(stringTable), this));
     sym->value = eSym.st_value;
     sym->size = eSym.st_size;
diff --git a/lld/ELF/SymbolTable.cpp b/lld/ELF/SymbolTable.cpp
index 674b1ef983f8..8878acdc43e8 100644
--- a/lld/ELF/SymbolTable.cpp
+++ b/lld/ELF/SymbolTable.cpp
@@ -56,7 +56,7 @@ void SymbolTable::wrap(Symbol *sym, Symbol *real, Symbol *wrap) {
   // alias for sym, but that could degrade the user experience of some tools
   // that can print out only one symbol for each location: sym is a preferred
   // name than real, but they might print out real instead.
-  memcpy(real, sym, sizeof(SymbolUnion));
+  memcpy(static_cast<void *>(real), sym, sizeof(SymbolUnion));
   real->isUsedInRegularObj = false;
 }
 
@@ -87,7 +87,7 @@ Symbol *SymbolTable::insert(StringRef name) {
   symVector.push_back(sym);
 
   // *sym was not initialized by a constructor. Initialize all Symbol fields.
-  memset(sym, 0, sizeof(Symbol));
+  memset(static_cast<void *>(sym), 0, sizeof(Symbol));
   sym->setName(name);
   sym->partition = 1;
   sym->versionId = VER_NDX_GLOBAL;
-- 
GitLab


From 66fc81cb430c724dc8b4b8d2d029101cc1d6808a Mon Sep 17 00:00:00 2001
From: Takuto Ikuta <tikuta@google.com>
Date: Tue, 29 Oct 2024 16:30:06 +0900
Subject: [PATCH 403/693] fix missing include for `abort` in
 `FuzzedDataProvider.h` (#113872)

This is to fix build with newer libc++ in chromium.

ref: https://crbug.com/375980422
-- 
GitLab


From 335e68d8bce5ad3f5d6471c0ec1423211c71c0f0 Mon Sep 17 00:00:00 2001
From: Jesse Huang <jesse.huang@sifive.com>
Date: Tue, 29 Oct 2024 00:47:49 -0700
Subject: [PATCH 404/693] [Clang][RISCV] Support -fcf-protection=return for
 RISC-V (#112477)

Enables the support of `-fcf-protection=return` on RISC-V, which
requires Zicfiss. It also adds a string attribute "hw-shadow-stack"
to every function if the option is set on RISC-V
---
 clang/lib/Basic/Targets/RISCV.h                 | 7 +++++++
 clang/lib/CodeGen/Targets/RISCV.cpp             | 7 +++++--
 clang/test/CodeGen/RISCV/attr-hw-shadow-stack.c | 9 +++++++++
 3 files changed, 21 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/CodeGen/RISCV/attr-hw-shadow-stack.c

diff --git a/clang/lib/Basic/Targets/RISCV.h b/clang/lib/Basic/Targets/RISCV.h
index bf40edb8683b..3b418585ab4a 100644
--- a/clang/lib/Basic/Targets/RISCV.h
+++ b/clang/lib/Basic/Targets/RISCV.h
@@ -141,6 +141,13 @@ public:
     return true;
   }
 
+  bool
+  checkCFProtectionReturnSupported(DiagnosticsEngine &Diags) const override {
+    if (ISAInfo->hasExtension("zicfiss"))
+      return true;
+    return TargetInfo::checkCFProtectionReturnSupported(Diags);
+  }
+
   CFBranchLabelSchemeKind getDefaultCFBranchLabelScheme() const override {
     return CFBranchLabelSchemeKind::FuncSig;
   }
diff --git a/clang/lib/CodeGen/Targets/RISCV.cpp b/clang/lib/CodeGen/Targets/RISCV.cpp
index fd72fe673b9b..b04e436c665f 100644
--- a/clang/lib/CodeGen/Targets/RISCV.cpp
+++ b/clang/lib/CodeGen/Targets/RISCV.cpp
@@ -594,6 +594,11 @@ public:
     const auto *FD = dyn_cast_or_null<FunctionDecl>(D);
     if (!FD) return;
 
+    auto *Fn = cast<llvm::Function>(GV);
+
+    if (CGM.getCodeGenOpts().CFProtectionReturn)
+      Fn->addFnAttr("hw-shadow-stack");
+
     const auto *Attr = FD->getAttr<RISCVInterruptAttr>();
     if (!Attr)
       return;
@@ -604,8 +609,6 @@ public:
     case RISCVInterruptAttr::machine: Kind = "machine"; break;
     }
 
-    auto *Fn = cast<llvm::Function>(GV);
-
     Fn->addFnAttr("interrupt", Kind);
   }
 };
diff --git a/clang/test/CodeGen/RISCV/attr-hw-shadow-stack.c b/clang/test/CodeGen/RISCV/attr-hw-shadow-stack.c
new file mode 100644
index 000000000000..cabff7e598eb
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/attr-hw-shadow-stack.c
@@ -0,0 +1,9 @@
+// RUN: %clang_cc1 -triple riscv64 -target-feature +experimental-zicfiss -emit-llvm -o - %s -fcf-protection=return | FileCheck %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +experimental-zicfiss -emit-llvm -o - %s | FileCheck -check-prefix=NOSHADOWSTACK %s
+// RUN: %clang_cc1 -triple riscv32 -target-feature +experimental-zicfiss -emit-llvm -o - %s -fcf-protection=return | FileCheck %s
+// RUN: %clang_cc1 -triple riscv32 -target-feature +experimental-zicfiss -emit-llvm -o - %s | FileCheck -check-prefix=NOSHADOWSTACK %s
+
+int foo(int *a) { return *a; }
+
+// CHECK: attributes {{.*}}"hw-shadow-stack"{{.*}}
+// NOSHADOWSTACK-NOT: attributes {{.*}}"hw-shadow-stack"{{.*}}
-- 
GitLab


From 7544d3af0e285ecd2fa28698621dd3125f749b2d Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Tue, 29 Oct 2024 07:57:34 +0000
Subject: [PATCH 405/693] [RISCV] Mark RVB23U64 and RVB23S64 as
 non-experimental (#113918)

The specification was recently ratified

<https://github.com/riscv/riscv-profiles/blob/main/src/rvb23-profile.adoc>.
---
 clang/test/Driver/print-supported-extensions-riscv.c | 4 ++--
 clang/test/Driver/riscv-profiles.c                   | 4 ++--
 llvm/docs/RISCVUsage.rst                             | 4 ++--
 llvm/docs/ReleaseNotes.md                            | 3 ++-
 llvm/lib/Target/RISCV/RISCVProfiles.td               | 4 ++--
 llvm/test/CodeGen/RISCV/attributes.ll                | 4 ++--
 llvm/unittests/TargetParser/RISCVISAInfoTest.cpp     | 4 ++--
 7 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index e39847b9c31a..68acde65a74b 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -193,12 +193,12 @@
 // CHECK-NEXT:     rva22u64
 // CHECK-NEXT:     rva23s64
 // CHECK-NEXT:     rva23u64
+// CHECK-NEXT:     rvb23s64
+// CHECK-NEXT:     rvb23u64
 // CHECK-NEXT:     rvi20u32
 // CHECK-NEXT:     rvi20u64
 // CHECK-EMPTY:
 // CHECK-NEXT: Experimental Profiles
-// CHECK-NEXT:     rvb23s64
-// CHECK-NEXT:     rvb23u64
 // CHECK-NEXT:     rvm23u32
 // CHECK-EMPTY:
 // CHECK-NEXT: Use -march to specify the target's extension.
diff --git a/clang/test/Driver/riscv-profiles.c b/clang/test/Driver/riscv-profiles.c
index d85ac8baf4ed..2b4d19422874 100644
--- a/clang/test/Driver/riscv-profiles.c
+++ b/clang/test/Driver/riscv-profiles.c
@@ -207,7 +207,7 @@
 // RVA23S64: "-target-feature" "+svnapot"
 // RVA23S64: "-target-feature" "+svpbmt"
 
-// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rvb23u64 -menable-experimental-extensions \
+// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rvb23u64 \
 // RUN:   | FileCheck -check-prefix=RVB23U64 %s
 // RVB23U64: "-target-feature" "+m"
 // RVB23U64: "-target-feature" "+a"
@@ -239,7 +239,7 @@
 // RVB23U64: "-target-feature" "+zbs"
 // RVB23U64: "-target-feature" "+zkt"
 
-// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rvb23s64 -menable-experimental-extensions \
+// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rvb23s64 \
 // RUN:   | FileCheck -check-prefix=RVB23S64 %s
 // RVB23S64: "-target-feature" "+m"
 // RVB23S64: "-target-feature" "+a"
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index 04f2c357766d..f6f2eb45c49c 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -84,6 +84,8 @@ ISA naming string. Currently supported profiles:
 * ``rva22s64``
 * ``rva23u64``
 * ``rva23s64``
+* ``rvb23u64``
+* ``rvb23s64``
 
 Note that you can also append additional extension names to be enabled, e.g.
 ``rva20u64_zicond`` will enable the ``zicond`` extension in addition to those
@@ -93,8 +95,6 @@ Profiles that are not yet ratified cannot be used unless
 ``-menable-experimental-extensions`` (or equivalent for other tools) is
 specified. This applies to the following profiles:
 
-* ``rvb23u64``
-* ``rvb23s64``
 * ``rvm23u32``
 
 .. _riscv-extensions:
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index ac7a795daf79..92a45d845f1d 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -184,7 +184,8 @@ Changes to the RISC-V Backend
 * The `Smmpm`, `Smnpm`, `Ssnpm`, `Supm`, and `Sspm` pointer masking extensions
   are no longer marked as experimental.
 * The `Sha` extension is now supported.
-* The RVA23U64 and RVA23S64 profiles are no longer marked as experimental.
+* The RVA23U64, RVA23S64, RVB23U64, and RVB23S64 profiles are no longer marked
+  as experimental.
 
 Changes to the WebAssembly Backend
 ----------------------------------
diff --git a/llvm/lib/Target/RISCV/RISCVProfiles.td b/llvm/lib/Target/RISCV/RISCVProfiles.td
index ea0fe08abd7a..cbf2a2eddf38 100644
--- a/llvm/lib/Target/RISCV/RISCVProfiles.td
+++ b/llvm/lib/Target/RISCV/RISCVProfiles.td
@@ -163,6 +163,6 @@ def RVA22U64 : RISCVProfile<"rva22u64", RVA22U64Features>;
 def RVA22S64 : RISCVProfile<"rva22s64", RVA22S64Features>;
 def RVA23U64 : RISCVProfile<"rva23u64", RVA23U64Features>;
 def RVA23S64 : RISCVProfile<"rva23s64", RVA23S64Features>;
-def RVB23U64 : RISCVExperimentalProfile<"rvb23u64", RVB23U64Features>;
-def RVB23S64 : RISCVExperimentalProfile<"rvb23s64", RVB23S64Features>;
+def RVB23U64 : RISCVProfile<"rvb23u64", RVB23U64Features>;
+def RVB23S64 : RISCVProfile<"rvb23s64", RVB23S64Features>;
 def RVM23U32 : RISCVExperimentalProfile<"rvm23u32", RVM23U32Features>;
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index 2545c7075e4c..c03108c0617e 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -293,8 +293,8 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+rva22s64 %s -o - | FileCheck --check-prefix=RVA22S64 %s
 ; RUN: llc -mtriple=riscv64 -mattr=+rva23u64 %s -o - | FileCheck --check-prefix=RVA23U64 %s
 ; RUN: llc -mtriple=riscv64 -mattr=+rva23s64 %s -o - | FileCheck --check-prefix=RVA23S64 %s
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-rvb23u64 %s -o - | FileCheck --check-prefix=RVB23U64 %s
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-rvb23s64 %s -o - | FileCheck --check-prefix=RVB23S64 %s
+; RUN: llc -mtriple=riscv64 -mattr=+rvb23u64 %s -o - | FileCheck --check-prefix=RVB23U64 %s
+; RUN: llc -mtriple=riscv64 -mattr=+rvb23s64 %s -o - | FileCheck --check-prefix=RVB23S64 %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-rvm23u32 %s -o - | FileCheck --check-prefix=RVM23U32 %s
 
 ; CHECK: .attribute 4, 16
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 48792ad0265f..a1d493e12fda 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -1138,12 +1138,12 @@ Supported Profiles
     rva22u64
     rva23s64
     rva23u64
+    rvb23s64
+    rvb23u64
     rvi20u32
     rvi20u64
 
 Experimental Profiles
-    rvb23s64
-    rvb23u64
     rvm23u32
 
 Use -march to specify the target's extension.
-- 
GitLab


From d4197f3ac1bbdd0665599bf3843e865d13af18ab Mon Sep 17 00:00:00 2001
From: CarolineConcatto <caroline.concatto@arm.com>
Date: Tue, 29 Oct 2024 09:09:13 +0000
Subject: [PATCH 406/693] [LLVM][AArch64] Add assembly/disassembly for
 MUL/BFMUL SME instructions (#113535)

According to https://developer.arm.com/documentation/ddi0602

Co-authored-by: Momchil-Velikov Momchil.Velikov@arm.com
---
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |  10 +
 llvm/lib/Target/AArch64/SMEInstrFormats.td    | 110 ++++++++
 llvm/test/MC/AArch64/SME2/bfmul-diagnostics.s | 111 ++++++++
 llvm/test/MC/AArch64/SME2/bfmul.s             |  92 ++++++
 .../test/MC/AArch64/SME2p2/fmul-diagnostics.s | 112 ++++++++
 llvm/test/MC/AArch64/SME2p2/fmul.s            | 261 ++++++++++++++++++
 6 files changed, 696 insertions(+)
 create mode 100644 llvm/test/MC/AArch64/SME2/bfmul-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SME2/bfmul.s
 create mode 100644 llvm/test/MC/AArch64/SME2p2/fmul-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SME2p2/fmul.s

diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index b763aa15a7c3..76e0501a5cc2 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -1007,6 +1007,11 @@ let Predicates = [HasSME2p2] in {
 
   defm BFMOP4A : sme2_bfmop4as_widening<0, "bfmop4a">;
   defm BFMOP4S : sme2_bfmop4as_widening<1, "bfmop4s">;
+
+  defm FMUL_2ZZ  : sme2_multi2_fmul_sm<"fmul">;
+  defm FMUL_2Z2Z : sme2_multi2_fmul_mm< "fmul">;
+  defm FMUL_4ZZ  : sme2_multi4_fmul_sm<"fmul">;
+  defm FMUL_4Z4Z : sme2_multi4_fmul_mm< "fmul">;
 } // [HasSME2p2]
 
 let Predicates = [HasSME2p2, HasSMEB16B16] in {
@@ -1024,3 +1029,8 @@ let Predicates = [HasSME2p2, HasSMEF8F16], Uses = [FPMR, FPCR] in {
 let Predicates = [HasSME2p2, HasSMEF16F16] in {
   def FTMOPA_M2ZZZI_HtoH : sme_tmopa_16b<0b10001, ZZ_h_mul_r, ZPR16, "ftmopa">;
 } // [HasSME2p2, HasSMEF16F16]
+
+let Predicates = [HasSME2, HasSVEBFSCALE] in {
+  defm BFMUL : sme2_bfmul_single<"bfmul">;
+  defm BFMUL : sme2_bfmul_multi<"bfmul">;
+} //[HasSME2, HasSVEBFSCALE]
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index 4cfe18eddf48..a05c5206320f 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -5225,3 +5225,113 @@ multiclass sme2_bfmop4as_widening<bit S, string mnemonic> {
   // Multiple vectors
   def _M2Z2Z_S : sme2_bf16_fp32_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi>;
 }
+
+class sme2_multi2_fmul_sm<bits<2> size, string mnemonic, RegisterOperand vector_ty, RegisterOperand zpr_ty>
+    : I<(outs vector_ty:$Zd), (ins vector_ty:$Zn, zpr_ty:$Zm),
+        mnemonic, "\t$Zd, $Zn, $Zm",
+        "", []>, Sched<[]> {
+  bits<4> Zd;
+  bits<4> Zn;
+  bits<4> Zm;
+
+  let Inst{31-24} = 0b11000001;
+  let Inst{23-22} = size;
+  let Inst{21}    = 0b1;
+  let Inst{20-17} = Zm;
+  let Inst{16-10} = 0b0111010;
+  let Inst{9-6}   = Zn;
+  let Inst{5}     = 0b0;
+  let Inst{4-1}   = Zd;
+  let Inst{0}     = 0b0;
+}
+
+multiclass sme2_multi2_fmul_sm<string mnemonic> {
+  def _H : sme2_multi2_fmul_sm<0b01, mnemonic, ZZ_h_mul_r, ZPR4b16>;
+  def _S : sme2_multi2_fmul_sm<0b10, mnemonic, ZZ_s_mul_r, ZPR4b32>;
+  def _D : sme2_multi2_fmul_sm<0b11, mnemonic, ZZ_d_mul_r, ZPR4b64>;
+}
+
+class sme2_multi4_fmul_sm<bits<2> size, string mnemonic, RegisterOperand vector_ty, RegisterOperand zpr_ty>
+    : I<(outs vector_ty:$Zd), (ins vector_ty:$Zn, zpr_ty:$Zm),
+        mnemonic, "\t$Zd, $Zn, $Zm",
+        "", []>, Sched<[]> {
+  bits<3> Zd;
+  bits<3> Zn;
+  bits<4> Zm;
+
+  let Inst{31-24} = 0b11000001;
+  let Inst{23-22} = size;
+  let Inst{21}    = 0b1;
+  let Inst{20-17} = Zm;
+  let Inst{16-10} = 0b1111010;
+  let Inst{9-7}   = Zn;
+  let Inst{6-5}   = 0b00;
+  let Inst{4-2}   = Zd;
+  let Inst{1-0}   = 0b00;
+}
+
+multiclass sme2_multi4_fmul_sm<string mnemonic> {
+  def _H : sme2_multi4_fmul_sm<0b01, mnemonic, ZZZZ_h_mul_r, ZPR4b16>;
+  def _S : sme2_multi4_fmul_sm<0b10, mnemonic, ZZZZ_s_mul_r, ZPR4b32>;
+  def _D : sme2_multi4_fmul_sm<0b11, mnemonic, ZZZZ_d_mul_r, ZPR4b64>;
+}
+
+multiclass sme2_bfmul_single<string mnemonic> {
+  def _2ZZ  : sme2_multi2_fmul_sm<0b00, mnemonic, ZZ_h_mul_r,   ZPR4b16>;
+  def _4ZZ  : sme2_multi4_fmul_sm<0b00, mnemonic, ZZZZ_h_mul_r, ZPR4b16>;
+}
+
+class sme2_multi2_fmul_mm<bits<2> size, string mnemonic, RegisterOperand vector_ty>
+    : I<(outs vector_ty:$Zd), (ins vector_ty:$Zn, vector_ty:$Zm),
+        mnemonic, "\t$Zd, $Zn, $Zm",
+        "", []>, Sched<[]> {
+  bits<4> Zd;
+  bits<4> Zn;
+  bits<4> Zm;
+
+  let Inst{31-24} = 0b11000001;
+  let Inst{23-22} = size;
+  let Inst{21}    = 0b1;
+  let Inst{20-17} = Zm;
+  let Inst{16-10} = 0b0111001;
+  let Inst{9-6}   = Zn;
+  let Inst{5}     = 0b0;
+  let Inst{4-1}   = Zd;
+  let Inst{0}     = 0b0;
+}
+
+multiclass sme2_multi2_fmul_mm<string mnemonic> {
+  def _H : sme2_multi2_fmul_mm<0b01, mnemonic, ZZ_h_mul_r>;
+  def _S : sme2_multi2_fmul_mm<0b10, mnemonic, ZZ_s_mul_r>;
+  def _D : sme2_multi2_fmul_mm<0b11, mnemonic, ZZ_d_mul_r>;
+}
+
+class sme2_multi4_fmul_mm<bits<2> size, string mnemonic, RegisterOperand vector_ty>
+    : I<(outs vector_ty:$Zd), (ins vector_ty:$Zn, vector_ty:$Zm),
+        mnemonic, "\t$Zd, $Zn, $Zm",
+        "", []>, Sched<[]> {
+  bits<3> Zd;
+  bits<3> Zn;
+  bits<3> Zm;
+
+  let Inst{31-24} = 0b11000001;
+  let Inst{23-22} = size;
+  let Inst{21}    = 0b1;
+  let Inst{20-18} = Zm;
+  let Inst{17-10} = 0b01111001;
+  let Inst{9-7}   = Zn;
+  let Inst{6-5}   = 0b00;
+  let Inst{4-2}   = Zd;
+  let Inst{1-0}   = 0b00;
+}
+
+multiclass sme2_multi4_fmul_mm<string mnemonic> {
+  def _H : sme2_multi4_fmul_mm<0b01, mnemonic, ZZZZ_h_mul_r>;
+  def _S : sme2_multi4_fmul_mm<0b10, mnemonic, ZZZZ_s_mul_r>;
+  def _D : sme2_multi4_fmul_mm<0b11, mnemonic, ZZZZ_d_mul_r>;
+}
+
+multiclass sme2_bfmul_multi<string mnemonic> {
+  def _2Z2Z : sme2_multi2_fmul_mm<0b00, mnemonic, ZZ_h_mul_r>;
+  def _4Z4Z : sme2_multi4_fmul_mm<0b00, mnemonic, ZZZZ_h_mul_r>;
+}
diff --git a/llvm/test/MC/AArch64/SME2/bfmul-diagnostics.s b/llvm/test/MC/AArch64/SME2/bfmul-diagnostics.s
new file mode 100644
index 000000000000..c28cc5cd426d
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2/bfmul-diagnostics.s
@@ -0,0 +1,111 @@
+// RUN: not llvm-mc -triple=aarch64 -mattr=+sme2,+sve-bfscale 2>&1 < %s| FileCheck %s
+
+// Multiple and single, 2 regs
+
+bfmul   {z0.s-z1.s}, {z0.h-z1.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmul   {z1.h-z2.h}, {z0.h-z1.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+
+bfmul   {z0.h-z2.h}, {z0.h-z1.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmul   {z0.h-z1.h}, {z0.s-z1.s}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmul   {z0.h-z1.h}, {z1.h-z2.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+
+bfmul   {z0.h-z1.h}, {z0.h-z2.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmul   {z0.h-z1.h}, {z0.h-z1.h}, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.h..z15.h
+
+bfmul   {z0.h-z1.h}, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.h..z15.h
+
+// Multiple and single, 4 regs
+
+bfmul   {z0.s-z3.s}, {z0.h-z3.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmul   {z1.h-z4.h}, {z0.h-z3.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 4 consecutive SVE vectors, where the first vector is a multiple of 4 and with matching element types
+
+bfmul   {z0.h-z4.h}, {z0.h-z3.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+
+bfmul   {z0.h-z3.h}, {z0.s-z3.s}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmul   {z0.h-z3.h}, {z1.h-z4.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 4 consecutive SVE vectors, where the first vector is a multiple of 4 and with matching element types
+
+bfmul   {z0.h-z3.h}, {z0.h-z4.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+
+bfmul   {z0.h-z3.h}, {z0.h-z3.h}, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.h..z15.h
+
+bfmul   {z0.h-z3.h}, {z0.h-z3.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.h..z15.h
+
+// Multiple, 2 regs
+
+bfmul   {z0.s-z1.s}, {z0.h-z1.h}, {z0.h-z1.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmul   {z1.h-z2.h}, {z0.h-z1.h}, {z0.h-z1.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+
+bfmul   {z0.h-z2.h}, {z0.h-z1.h}, {z0.h-z1.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmul   {z0.h-z1.h}, {z0.s-z1.s}, {z0.h-z1.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmul   {z0.h-z1.h}, {z1.h-z2.h}, {z0.h-z1.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+
+bfmul   {z0.h-z1.h}, {z0.h-z2.h}, {z0.h-z1.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmul   {z0.h-z1.h}, {z0.h-z1.h}, {z0.s-z1.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmul   {z0.h-z1.h}, {z0.h-z1.h}, {z1.h-z2.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+
+bfmul   {z0.h-z1.h}, {z0.h-z1.h}, {z0.h-z2.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+// Multiple, 4 regs
+
+bfmul   {z0.s-z3.s}, {z0.h-z3.h}, {z0.h-z3.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmul   {z1.h-z4.h}, {z0.h-z3.h}, {z0.h-z3.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 4 consecutive SVE vectors, where the first vector is a multiple of 4 and with matching element types
+
+bfmul   {z0.h-z4.h}, {z0.h-z3.h}, {z0.h-z3.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+
+bfmul   {z0.h-z3.h}, {z0.s-z3.s}, {z0.h-z3.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmul   {z0.h-z3.h}, {z1.h-z4.h}, {z0.h-z3.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 4 consecutive SVE vectors, where the first vector is a multiple of 4 and with matching element types
+
+bfmul   {z0.h-z3.h}, {z0.h-z4.h}, {z0.h-z3.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+
+bfmul   {z0.h-z3.h}, {z0.h-z3.h}, {z0.s-z3.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmul   {z0.h-z3.h}, {z0.h-z3.h}, {z1.h-z4.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 4 consecutive SVE vectors, where the first vector is a multiple of 4 and with matching element types
+
+bfmul   {z0.h-z3.h}, {z0.h-z3.h}, {z0.h-z4.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
diff --git a/llvm/test/MC/AArch64/SME2/bfmul.s b/llvm/test/MC/AArch64/SME2/bfmul.s
new file mode 100644
index 000000000000..10a43848c738
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2/bfmul.s
@@ -0,0 +1,92 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2,+sve-bfscale < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2,+sve-bfscale < %s \
+// RUN:        | llvm-objdump -d --mattr=+sme2,+sve-bfscale - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2,+sve-bfscale < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2,+sve-bfscale < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2,+sve-bfscale -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// Multiple and single, 2 regs
+
+bfmul   {z0.h-z1.h}, {z0.h-z1.h}, z0.h  // 11000001-00100000-11101000-00000000
+// CHECK-INST: bfmul   { z0.h, z1.h }, { z0.h, z1.h }, z0.h
+// CHECK-ENCODING: [0x00,0xe8,0x20,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c120e800 <unknown>
+
+bfmul   {z20.h-z21.h}, {z10.h-z11.h}, z10.h  // 11000001-00110100-11101001-01010100
+// CHECK-INST: bfmul   { z20.h, z21.h }, { z10.h, z11.h }, z10.h
+// CHECK-ENCODING: [0x54,0xe9,0x34,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c134e954 <unknown>
+
+bfmul   {z30.h-z31.h}, {z30.h-z31.h}, z15.h  // 11000001-00111110-11101011-11011110
+// CHECK-INST: bfmul   { z30.h, z31.h }, { z30.h, z31.h }, z15.h
+// CHECK-ENCODING: [0xde,0xeb,0x3e,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c13eebde <unknown>
+
+// Multiple and single, 4 regs
+
+bfmul   {z0.h-z3.h}, {z0.h-z3.h}, z0.h  // 11000001-00100001-11101000-00000000
+// CHECK-INST: bfmul   { z0.h - z3.h }, { z0.h - z3.h }, z0.h
+// CHECK-ENCODING: [0x00,0xe8,0x21,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c121e800 <unknown>
+
+bfmul   {z20.h-z23.h}, {z8.h-z11.h}, z10.h  // 11000001-00110101-11101001-00010100
+// CHECK-INST: bfmul   { z20.h - z23.h }, { z8.h - z11.h }, z10.h
+// CHECK-ENCODING: [0x14,0xe9,0x35,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c135e914 <unknown>
+
+bfmul   {z28.h-z31.h}, {z28.h-z31.h}, z15.h  // 11000001-00111111-11101011-10011100
+// CHECK-INST: bfmul   { z28.h - z31.h }, { z28.h - z31.h }, z15.h
+// CHECK-ENCODING: [0x9c,0xeb,0x3f,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c13feb9c <unknown>
+
+// Multiple, 2 regs
+bfmul   {z0.h-z1.h}, {z0.h-z1.h}, {z0.h-z1.h}  // 11000001-00100000-11100100-00000000
+// CHECK-INST: bfmul   { z0.h, z1.h }, { z0.h, z1.h }, { z0.h, z1.h }
+// CHECK-ENCODING: [0x00,0xe4,0x20,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c120e400 <unknown>
+
+bfmul   {z20.h-z21.h}, {z10.h-z11.h}, {z20.h-z21.h}  // 11000001-00110100-11100101-01010100
+// CHECK-INST: bfmul   { z20.h, z21.h }, { z10.h, z11.h }, { z20.h, z21.h }
+// CHECK-ENCODING: [0x54,0xe5,0x34,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c134e554 <unknown>
+
+bfmul   {z30.h-z31.h}, {z30.h-z31.h}, {z30.h-z31.h}  // 11000001-00111110-11100111-11011110
+// CHECK-INST: bfmul   { z30.h, z31.h }, { z30.h, z31.h }, { z30.h, z31.h }
+// CHECK-ENCODING: [0xde,0xe7,0x3e,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c13ee7de <unknown>
+
+// Multiple, 4 regs
+
+bfmul   {z0.h-z3.h}, {z0.h-z3.h}, {z0.h-z3.h}  // 11000001-00100001-11100100-00000000
+// CHECK-INST: bfmul   { z0.h - z3.h }, { z0.h - z3.h }, { z0.h - z3.h }
+// CHECK-ENCODING: [0x00,0xe4,0x21,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c121e400 <unknown>
+
+bfmul   {z20.h-z23.h}, {z8.h-z11.h}, {z20.h-z23.h}  // 11000001-00110101-11100101-00010100
+// CHECK-INST: bfmul   { z20.h - z23.h }, { z8.h - z11.h }, { z20.h - z23.h }
+// CHECK-ENCODING: [0x14,0xe5,0x35,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c135e514 <unknown>
+
+bfmul   {z28.h-z31.h}, {z28.h-z31.h}, {z28.h-z31.h}  // 11000001-00111101-11100111-10011100
+// CHECK-INST: bfmul   { z28.h - z31.h }, { z28.h - z31.h }, { z28.h - z31.h }
+// CHECK-ENCODING: [0x9c,0xe7,0x3d,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c13de79c <unknown>
diff --git a/llvm/test/MC/AArch64/SME2p2/fmul-diagnostics.s b/llvm/test/MC/AArch64/SME2p2/fmul-diagnostics.s
new file mode 100644
index 000000000000..2fdd3f82adc1
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/fmul-diagnostics.s
@@ -0,0 +1,112 @@
+
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 2>&1 < %s| FileCheck %s
+
+// Multiple and single, 2 regs
+
+fmul    {z0.b-z1.b}, {z0.h-z1.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z1.s-z2.s}, {z0.s-z1.s}, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+
+fmul    {z0.d-z2.d}, {z0.d-z1.d}, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z0.h-z1.h}, {z0.b-z1.b}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z0.s-z1.s}, {z1.s-z2.s}, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+
+fmul    {z0.d-z1.d}, {z0.d-z2.d}, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z0.h-z1.h}, {z0.h-z1.h}, z0.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.h..z15.h
+
+fmul    {z0.s-z1.s}, {z0.s-z1.s}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.s..z15.s
+
+// Multiple and single, 4 regs
+
+fmul    {z0.b-z3.b}, {z0.h-z3.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z1.s-z3.s}, {z0.h-z3.h}, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z0.d-z4.d}, {z0.d-z3.d}, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+
+fmul    {z0.h-z3.h}, {z0.b-z3.b}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z0.s-z3.s}, {z1.s-z3.s}, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z0.d-z3.d}, {z0.d-z4.d}, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+
+fmul    {z0.h-z3.h}, {z0.h-z3.h}, z0.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.h..z15.h
+
+fmul    {z0.s-z3.s}, {z0.s-z3.s}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.s..z15.s
+
+// Multiple, 2 regs
+
+fmul    {z0.b-z1.b}, {z0.h-z1.h}, {z0.h-z1.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z1.s-z2.s}, {z0.s-z1.s}, {z0.s-z1.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+
+fmul    {z0.d-z2.d}, {z0.d-z1.d}, {z0.d-z1.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z0.h-z1.h}, {z0.b-z1.b}, {z0.h-z1.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z0.s-z1.s}, {z1.s-z2.s}, {z0.s-z1.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+
+fmul    {z0.d-z1.d}, {z0.d-z2.d}, {z0.d-z1.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z0.h-z1.h}, {z0.h-z1.h}, {z0.b-z1.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z0.s-z1.s}, {z0.s-z1.s}, {z1.s-z2.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+
+fmul    {z0.d-z1.d}, {z0.d-z1.d}, {z0.d-z2.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+// Multiple, 4 regs
+
+fmul    {z0.b-z3.b}, {z0.h-z3.h}, {z0.h-z3.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z1.s-z3.s}, {z0.s-z3.s}, {z0.s-z3.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z0.d-z4.d}, {z0.d-z3.d}, {z0.d-z3.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+
+fmul    {z0.h-z3.h}, {z0.b-z3.b}, {z0.h-z3.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z0.s-z3.s}, {z1.s-z3.s}, {z0.s-z3.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z0.d-z3.d}, {z0.d-z4.d}, {z0.d-z3.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+
+fmul    {z0.h-z3.h}, {z0.h-z3.h}, {z0.b-z3.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z0.s-z3.s}, {z0.s-z3.s}, {z1.s-z3.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmul    {z0.d-z3.d}, {z0.d-z3.d}, {z0.d-z4.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
diff --git a/llvm/test/MC/AArch64/SME2p2/fmul.s b/llvm/test/MC/AArch64/SME2p2/fmul.s
new file mode 100644
index 000000000000..ec6f523867ce
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/fmul.s
@@ -0,0 +1,261 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sme2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2p2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// Multiple and single, 2 regs
+
+// 16-bit elements
+
+fmul    {z0.h-z1.h}, {z0.h-z1.h}, z0.h  // 11000001-01100000-11101000-00000000
+// CHECK-INST: fmul    { z0.h, z1.h }, { z0.h, z1.h }, z0.h
+// CHECK-ENCODING: [0x00,0xe8,0x60,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c160e800 <unknown>
+
+fmul    {z20.h-z21.h}, {z10.h-z11.h}, z10.h  // 11000001-01110100-11101001-01010100
+// CHECK-INST: fmul    { z20.h, z21.h }, { z10.h, z11.h }, z10.h
+// CHECK-ENCODING: [0x54,0xe9,0x74,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c174e954 <unknown>
+
+fmul    {z30.h-z31.h}, {z30.h-z31.h}, z15.h  // 11000001-01111110-11101011-11011110
+// CHECK-INST: fmul    { z30.h, z31.h }, { z30.h, z31.h }, z15.h
+// CHECK-ENCODING: [0xde,0xeb,0x7e,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c17eebde <unknown>
+
+// 32-bit elements
+
+fmul    {z0.s-z1.s}, {z0.s-z1.s}, z0.s  // 11000001-10100000-11101000-00000000
+// CHECK-INST: fmul    { z0.s, z1.s }, { z0.s, z1.s }, z0.s
+// CHECK-ENCODING: [0x00,0xe8,0xa0,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1a0e800 <unknown>
+
+fmul    {z20.s-z21.s}, {z10.s-z11.s}, z10.s  // 11000001-10110100-11101001-01010100
+// CHECK-INST: fmul    { z20.s, z21.s }, { z10.s, z11.s }, z10.s
+// CHECK-ENCODING: [0x54,0xe9,0xb4,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1b4e954 <unknown>
+
+fmul    {z30.s-z31.s}, {z30.s-z31.s}, z15.s  // 11000001-10111110-11101011-11011110
+// CHECK-INST: fmul    { z30.s, z31.s }, { z30.s, z31.s }, z15.s
+// CHECK-ENCODING: [0xde,0xeb,0xbe,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1beebde <unknown>
+
+// 64-bit elements
+
+fmul    {z0.d-z1.d}, {z0.d-z1.d}, z0.d  // 11000001-11100000-11101000-00000000
+// CHECK-INST: fmul    { z0.d, z1.d }, { z0.d, z1.d }, z0.d
+// CHECK-ENCODING: [0x00,0xe8,0xe0,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1e0e800 <unknown>
+
+fmul    {z20.d-z21.d}, {z10.d-z11.d}, z10.d  // 11000001-11110100-11101001-01010100
+// CHECK-INST: fmul    { z20.d, z21.d }, { z10.d, z11.d }, z10.d
+// CHECK-ENCODING: [0x54,0xe9,0xf4,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1f4e954 <unknown>
+
+fmul    {z30.d-z31.d}, {z30.d-z31.d}, z15.d  // 11000001-11111110-11101011-11011110
+// CHECK-INST: fmul    { z30.d, z31.d }, { z30.d, z31.d }, z15.d
+// CHECK-ENCODING: [0xde,0xeb,0xfe,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1feebde <unknown>
+
+// Multiple and single, 4 regs
+
+// 16-bit elements
+
+fmul    {z0.h-z3.h}, {z0.h-z3.h}, z0.h  // 11000001-01100001-11101000-00000000
+// CHECK-INST: fmul    { z0.h - z3.h }, { z0.h - z3.h }, z0.h
+// CHECK-ENCODING: [0x00,0xe8,0x61,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c161e800 <unknown>
+
+fmul    {z20.h-z23.h}, {z8.h-z11.h}, z10.h  // 11000001-01110101-11101001-00010100
+// CHECK-INST: fmul    { z20.h - z23.h }, { z8.h - z11.h }, z10.h
+// CHECK-ENCODING: [0x14,0xe9,0x75,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c175e914 <unknown>
+
+fmul    {z28.h-z31.h}, {z28.h-z31.h}, z15.h  // 11000001-01111111-11101011-10011100
+// CHECK-INST: fmul    { z28.h - z31.h }, { z28.h - z31.h }, z15.h
+// CHECK-ENCODING: [0x9c,0xeb,0x7f,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c17feb9c <unknown>
+
+// 32-bit elements
+
+fmul    {z0.s-z3.s}, {z0.s-z3.s}, z0.s  // 11000001-10100001-11101000-00000000
+// CHECK-INST: fmul    { z0.s - z3.s }, { z0.s - z3.s }, z0.s
+// CHECK-ENCODING: [0x00,0xe8,0xa1,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1a1e800 <unknown>
+
+fmul    {z20.s-z23.s}, {z8.s-z11.s}, z10.s  // 11000001-10110101-11101001-00010100
+// CHECK-INST: fmul    { z20.s - z23.s }, { z8.s - z11.s }, z10.s
+// CHECK-ENCODING: [0x14,0xe9,0xb5,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1b5e914 <unknown>
+
+fmul    {z28.s-z31.s}, {z28.s-z31.s}, z15.s  // 11000001-10111111-11101011-10011100
+// CHECK-INST: fmul    { z28.s - z31.s }, { z28.s - z31.s }, z15.s
+// CHECK-ENCODING: [0x9c,0xeb,0xbf,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1bfeb9c <unknown>
+
+// 64-bit elements
+
+fmul    {z0.d-z3.d}, {z0.d-z3.d}, z0.d  // 11000001-11100001-11101000-00000000
+// CHECK-INST: fmul    { z0.d - z3.d }, { z0.d - z3.d }, z0.d
+// CHECK-ENCODING: [0x00,0xe8,0xe1,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1e1e800 <unknown>
+
+fmul    {z20.d-z23.d}, {z8.d-z11.d}, z10.d  // 11000001-11110101-11101001-00010100
+// CHECK-INST: fmul    { z20.d - z23.d }, { z8.d - z11.d }, z10.d
+// CHECK-ENCODING: [0x14,0xe9,0xf5,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1f5e914 <unknown>
+
+fmul    {z28.d-z31.d}, {z28.d-z31.d}, z15.d  // 11000001-11111111-11101011-10011100
+// CHECK-INST: fmul    { z28.d - z31.d }, { z28.d - z31.d }, z15.d
+// CHECK-ENCODING: [0x9c,0xeb,0xff,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1ffeb9c <unknown>
+
+// Multiple, 2 regs
+
+// 16-bit elements
+
+fmul    {z0.h-z1.h}, {z0.h-z1.h}, {z0.h-z1.h}  // 11000001-01100000-11100100-00000000
+// CHECK-INST: fmul    { z0.h, z1.h }, { z0.h, z1.h }, { z0.h, z1.h }
+// CHECK-ENCODING: [0x00,0xe4,0x60,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c160e400 <unknown>
+
+fmul    {z20.h-z21.h}, {z10.h-z11.h}, {z20.h-z21.h}  // 11000001-01110100-11100101-01010100
+// CHECK-INST: fmul    { z20.h, z21.h }, { z10.h, z11.h }, { z20.h, z21.h }
+// CHECK-ENCODING: [0x54,0xe5,0x74,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c174e554 <unknown>
+
+fmul    {z30.h-z31.h}, {z30.h-z31.h}, {z30.h-z31.h}  // 11000001-01111110-11100111-11011110
+// CHECK-INST: fmul    { z30.h, z31.h }, { z30.h, z31.h }, { z30.h, z31.h }
+// CHECK-ENCODING: [0xde,0xe7,0x7e,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c17ee7de <unknown>
+
+// 32-bit elememnts
+
+fmul    {z0.s-z1.s}, {z0.s-z1.s}, {z0.s-z1.s}  // 11000001-10100000-11100100-00000000
+// CHECK-INST: fmul    { z0.s, z1.s }, { z0.s, z1.s }, { z0.s, z1.s }
+// CHECK-ENCODING: [0x00,0xe4,0xa0,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1a0e400 <unknown>
+
+fmul    {z20.s-z21.s}, {z10.s-z11.s}, {z20.s-z21.s}  // 11000001-10110100-11100101-01010100
+// CHECK-INST: fmul    { z20.s, z21.s }, { z10.s, z11.s }, { z20.s, z21.s }
+// CHECK-ENCODING: [0x54,0xe5,0xb4,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1b4e554 <unknown>
+
+fmul    {z30.s-z31.s}, {z30.s-z31.s}, {z30.s-z31.s}  // 11000001-10111110-11100111-11011110
+// CHECK-INST: fmul    { z30.s, z31.s }, { z30.s, z31.s }, { z30.s, z31.s }
+// CHECK-ENCODING: [0xde,0xe7,0xbe,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1bee7de <unknown>
+
+// 64-bit elements
+
+fmul    {z0.d-z1.d}, {z0.d-z1.d}, {z0.d-z1.d}  // 11000001-11100000-11100100-00000000
+// CHECK-INST: fmul    { z0.d, z1.d }, { z0.d, z1.d }, { z0.d, z1.d }
+// CHECK-ENCODING: [0x00,0xe4,0xe0,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1e0e400 <unknown>
+
+fmul    {z20.d-z21.d}, {z10.d-z11.d}, {z20.d-z21.d}  // 11000001-11110100-11100101-01010100
+// CHECK-INST: fmul    { z20.d, z21.d }, { z10.d, z11.d }, { z20.d, z21.d }
+// CHECK-ENCODING: [0x54,0xe5,0xf4,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1f4e554 <unknown>
+
+fmul    {z30.d-z31.d}, {z30.d-z31.d}, {z30.d-z31.d}  // 11000001-11111110-11100111-11011110
+// CHECK-INST: fmul    { z30.d, z31.d }, { z30.d, z31.d }, { z30.d, z31.d }
+// CHECK-ENCODING: [0xde,0xe7,0xfe,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1fee7de <unknown>
+
+// Multiple, 4 regs
+
+// 16-bit elements
+
+fmul    {z0.h-z3.h}, {z0.h-z3.h}, {z0.h-z3.h}  // 11000001-01100001-11100100-00000000
+// CHECK-INST: fmul    { z0.h - z3.h }, { z0.h - z3.h }, { z0.h - z3.h }
+// CHECK-ENCODING: [0x00,0xe4,0x61,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c161e400 <unknown>
+
+fmul    {z20.h-z23.h}, {z8.h-z11.h}, {z20.h-z23.h}  // 11000001-01110101-11100101-00010100
+// CHECK-INST: fmul    { z20.h - z23.h }, { z8.h - z11.h }, { z20.h - z23.h }
+// CHECK-ENCODING: [0x14,0xe5,0x75,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c175e514 <unknown>
+
+fmul    {z28.h-z31.h}, {z28.h-z31.h}, {z28.h-z31.h}  // 11000001-01111101-11100111-10011100
+// CHECK-INST: fmul    { z28.h - z31.h }, { z28.h - z31.h }, { z28.h - z31.h }
+// CHECK-ENCODING: [0x9c,0xe7,0x7d,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c17de79c <unknown>
+
+// 32-bit elements
+
+fmul    {z0.s-z3.s}, {z0.s-z3.s}, {z0.s-z3.s}  // 11000001-10100001-11100100-00000000
+// CHECK-INST: fmul    { z0.s - z3.s }, { z0.s - z3.s }, { z0.s - z3.s }
+// CHECK-ENCODING: [0x00,0xe4,0xa1,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1a1e400 <unknown>
+
+fmul    {z20.s-z23.s}, {z8.s-z11.s}, {z20.s-z23.s}  // 11000001-10110101-11100101-00010100
+// CHECK-INST: fmul    { z20.s - z23.s }, { z8.s - z11.s }, { z20.s - z23.s }
+// CHECK-ENCODING: [0x14,0xe5,0xb5,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1b5e514 <unknown>
+
+fmul    {z28.s-z31.s}, {z28.s-z31.s}, {z28.s-z31.s}  // 11000001-10111101-11100111-10011100
+// CHECK-INST: fmul    { z28.s - z31.s }, { z28.s - z31.s }, { z28.s - z31.s }
+// CHECK-ENCODING: [0x9c,0xe7,0xbd,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1bde79c <unknown>
+
+// 64-bit elements
+
+fmul    {z0.d-z3.d}, {z0.d-z3.d}, {z0.d-z3.d}  // 11000001-11100001-11100100-00000000
+// CHECK-INST: fmul    { z0.d - z3.d }, { z0.d - z3.d }, { z0.d - z3.d }
+// CHECK-ENCODING: [0x00,0xe4,0xe1,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1e1e400 <unknown>
+
+fmul    {z20.d-z23.d}, {z8.d-z11.d}, {z20.d-z23.d}  // 11000001-11110101-11100101-00010100
+// CHECK-INST: fmul    { z20.d - z23.d }, { z8.d - z11.d }, { z20.d - z23.d }
+// CHECK-ENCODING: [0x14,0xe5,0xf5,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1f5e514 <unknown>
+
+fmul    {z28.d-z31.d}, {z28.d-z31.d}, {z28.d-z31.d}  // 11000001-11111101-11100111-10011100
+// CHECK-INST: fmul    { z28.d - z31.d }, { z28.d - z31.d }, { z28.d - z31.d }
+// CHECK-ENCODING: [0x9c,0xe7,0xfd,0xc1]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: c1fde79c <unknown>
-- 
GitLab


From 8d38fbf2f027c72332c8ba03ff0ff0f83b4dcf02 Mon Sep 17 00:00:00 2001
From: CarolineConcatto <caroline.concatto@arm.com>
Date: Tue, 29 Oct 2024 09:09:55 +0000
Subject: [PATCH 407/693] =?UTF-8?q?[LLVM][AArch64]=20Add=20assembly/disass?=
 =?UTF-8?q?embly=20for=20SVE=20Integer=20Unary=20Arithm=E2=80=A6=20(#11367?=
 =?UTF-8?q?0)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…etic Predicated instructions

This patch adds the following instructions:

SVE bitwise unary operations (predicated)
CLS, CLZ, CNT, CNOT, FABS, FNEG, NOT

SVE integer unary operations (predicated)
SXT{B,H,W}, UXT{B,H,W}, ABS ,NEG

SVE2 integer unary operations (predicated)
URECPE, URSQRTE, SQABS, SQNEG

According to https://developer.arm.com/documentation/ddi0602

Co-authored-by: Spencer Abson Spencer.Abson@arm.com
---
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |  53 ++--
 llvm/lib/Target/AArch64/SVEInstrFormats.td    |  76 +++++-
 ...unary_arithmetic_predicated_z-diagnotics.s | 249 ++++++++++++++++++
 .../SVE2p2/unary_arithmetic_predicated_z.s    | 225 ++++++++++++++++
 4 files changed, 576 insertions(+), 27 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/unary_arithmetic_predicated_z-diagnotics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/unary_arithmetic_predicated_z.s

diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 11c64df2eb92..2b69903b133f 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -647,22 +647,22 @@ let Predicates = [HasSVEorSME] in {
   defm SDOT_ZZZI : sve_intx_dot_by_indexed_elem<0b0, "sdot", int_aarch64_sve_sdot_lane>;
   defm UDOT_ZZZI : sve_intx_dot_by_indexed_elem<0b1, "udot", int_aarch64_sve_udot_lane>;
 
-  defm SXTB_ZPmZ : sve_int_un_pred_arit_0_h<0b000, "sxtb", AArch64sxt_mt>;
-  defm UXTB_ZPmZ : sve_int_un_pred_arit_0_h<0b001, "uxtb", AArch64uxt_mt>;
-  defm SXTH_ZPmZ : sve_int_un_pred_arit_0_w<0b010, "sxth", AArch64sxt_mt>;
-  defm UXTH_ZPmZ : sve_int_un_pred_arit_0_w<0b011, "uxth", AArch64uxt_mt>;
-  defm SXTW_ZPmZ : sve_int_un_pred_arit_0_d<0b100, "sxtw", AArch64sxt_mt>;
-  defm UXTW_ZPmZ : sve_int_un_pred_arit_0_d<0b101, "uxtw", AArch64uxt_mt>;
-  defm ABS_ZPmZ  : sve_int_un_pred_arit_0<  0b110, "abs",  AArch64abs_mt>;
-  defm NEG_ZPmZ  : sve_int_un_pred_arit_0<  0b111, "neg",  AArch64neg_mt>;
-
-  defm CLS_ZPmZ  : sve_int_un_pred_arit_1<   0b000, "cls",  AArch64cls_mt>;
-  defm CLZ_ZPmZ  : sve_int_un_pred_arit_1<   0b001, "clz",  AArch64clz_mt>;
-  defm CNT_ZPmZ  : sve_int_un_pred_arit_1<   0b010, "cnt",  AArch64cnt_mt>;
-  defm CNOT_ZPmZ : sve_int_un_pred_arit_1<   0b011, "cnot", AArch64cnot_mt>;
-  defm NOT_ZPmZ  : sve_int_un_pred_arit_1<   0b110, "not",  AArch64not_mt>;
-  defm FABS_ZPmZ : sve_int_un_pred_arit_1_fp<0b100, "fabs", AArch64fabs_mt>;
-  defm FNEG_ZPmZ : sve_int_un_pred_arit_1_fp<0b101, "fneg", AArch64fneg_mt>;
+  defm SXTB_ZPmZ : sve_int_un_pred_arit_h<0b000, "sxtb", AArch64sxt_mt>;
+  defm UXTB_ZPmZ : sve_int_un_pred_arit_h<0b001, "uxtb", AArch64uxt_mt>;
+  defm SXTH_ZPmZ : sve_int_un_pred_arit_w<0b010, "sxth", AArch64sxt_mt>;
+  defm UXTH_ZPmZ : sve_int_un_pred_arit_w<0b011, "uxth", AArch64uxt_mt>;
+  defm SXTW_ZPmZ : sve_int_un_pred_arit_d<0b100, "sxtw", AArch64sxt_mt>;
+  defm UXTW_ZPmZ : sve_int_un_pred_arit_d<0b101, "uxtw", AArch64uxt_mt>;
+  defm ABS_ZPmZ  : sve_int_un_pred_arit<  0b110, "abs",  AArch64abs_mt>;
+  defm NEG_ZPmZ  : sve_int_un_pred_arit<  0b111, "neg",  AArch64neg_mt>;
+
+  defm CLS_ZPmZ  : sve_int_un_pred_arit_bitwise<   0b000, "cls",  AArch64cls_mt>;
+  defm CLZ_ZPmZ  : sve_int_un_pred_arit_bitwise<   0b001, "clz",  AArch64clz_mt>;
+  defm CNT_ZPmZ  : sve_int_un_pred_arit_bitwise<   0b010, "cnt",  AArch64cnt_mt>;
+  defm CNOT_ZPmZ : sve_int_un_pred_arit_bitwise<   0b011, "cnot", AArch64cnot_mt>;
+  defm NOT_ZPmZ  : sve_int_un_pred_arit_bitwise<   0b110, "not",  AArch64not_mt>;
+  defm FABS_ZPmZ : sve_int_un_pred_arit_bitwise_fp<0b100, "fabs", AArch64fabs_mt>;
+  defm FNEG_ZPmZ : sve_int_un_pred_arit_bitwise_fp<0b101, "fneg", AArch64fneg_mt>;
 
   foreach VT = [nxv2bf16, nxv4bf16, nxv8bf16] in {
     // No dedicated instruction, so just clear the sign bit.
@@ -4271,6 +4271,27 @@ let Predicates = [HasSVE2p2orSME2p2] in {
   // Floating-point square root, zeroing predicate
   defm FSQRT_ZPZz  : sve_fp_z2op_p_zd_hsd<0b01101, "fsqrt">;
 
+  // SVE2p2 integer unary arithmetic (bitwise), zeroing predicate
+  defm CLS_ZPzZ  : sve_int_un_pred_arit_bitwise_z<0b000, "cls">;
+  defm CLZ_ZPzZ  : sve_int_un_pred_arit_bitwise_z<0b001, "clz">;
+  defm CNT_ZPzZ  : sve_int_un_pred_arit_bitwise_z<0b010, "cnt">;
+  defm CNOT_ZPzZ : sve_int_un_pred_arit_bitwise_z<0b011, "cnot">;
+  defm NOT_ZPzZ  : sve_int_un_pred_arit_bitwise_z<0b110, "not">;
+
+  // floating point
+  defm FABS_ZPzZ : sve_int_un_pred_arit_bitwise_fp_z<0b100, "fabs">;
+  defm FNEG_ZPzZ : sve_int_un_pred_arit_bitwise_fp_z<0b101, "fneg">;
+
+  // SVE2p2 integer unary arithmetic, zeroing predicate
+  defm SXTB_ZPzZ  : sve_int_un_pred_arit_h_z<0b000, "sxtb">;
+  defm UXTB_ZPzZ  : sve_int_un_pred_arit_h_z<0b001, "uxtb">;
+  defm SXTH_ZPzZ  : sve_int_un_pred_arit_w_z<0b010, "sxth">;
+  defm UXTH_ZPzZ  : sve_int_un_pred_arit_w_z<0b011, "uxth">;
+  defm ABS_ZPzZ   : sve_int_un_pred_arit_z<  0b110, "abs">;
+  defm NEG_ZPzZ   : sve_int_un_pred_arit_z<  0b111, "neg">;
+  def SXTW_ZPzZ_D : sve_int_un_pred_arit_z<0b11, 0b1000, "sxtw", ZPR64>;
+  def UXTW_ZPzZ_D : sve_int_un_pred_arit_z<0b11, 0b1010, "uxtw", ZPR64>;
+
 } // End HasSME2p2orSVE2p2
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 72cbad17bc04..31312e00b919 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -4685,8 +4685,30 @@ class sve_int_un_pred_arit<bits<2> sz8_64, bits<4> opc,
   let hasSideEffects = 0;
 }
 
-multiclass sve_int_un_pred_arit_0<bits<3> opc, string asm,
-                                  SDPatternOperator op> {
+class sve_int_un_pred_arit_z<bits<2> sz8_64, bits<4> opc,
+                            string asm, ZPRRegOp zprty>
+: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn),
+  asm, "\t$Zd, $Pg/z, $Zn",
+  "",
+  []>, Sched<[]> {
+  bits<3> Pg;
+  bits<5> Zd;
+  bits<5> Zn;
+  let Inst{31-24} = 0b00000100;
+  let Inst{23-22} = sz8_64;
+  let Inst{21-20} = 0b00;
+  let Inst{19}    = opc{0};
+  let Inst{18-16} = opc{3-1};
+  let Inst{15-13} = 0b101;
+  let Inst{12-10} = Pg;
+  let Inst{9-5}   = Zn;
+  let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
+}
+
+multiclass sve_int_un_pred_arit<bits<3> opc, string asm,
+                                SDPatternOperator op> {
   def _B : sve_int_un_pred_arit<0b00, { opc, 0b0 }, asm, ZPR8>,
            SVEPseudo2Instr<NAME # _B, 1>;
   def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>,
@@ -4712,8 +4734,15 @@ multiclass sve_int_un_pred_arit_0<bits<3> opc, string asm,
   defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;
 }
 
-multiclass sve_int_un_pred_arit_0_h<bits<3> opc, string asm,
-                                    SDPatternOperator op> {
+multiclass sve_int_un_pred_arit_z<bits<3> opc, string asm> {
+  def _B : sve_int_un_pred_arit_z<0b00, { opc, 0b0 }, asm, ZPR8>;
+  def _H : sve_int_un_pred_arit_z<0b01, { opc, 0b0 }, asm, ZPR16>;
+  def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b0 }, asm, ZPR32>;
+  def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b0 }, asm, ZPR64>;
+}
+
+multiclass sve_int_un_pred_arit_h<bits<3> opc, string asm,
+                                  SDPatternOperator op> {
   def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>,
            SVEPseudo2Instr<NAME # _H, 1>;
   def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>,
@@ -4734,8 +4763,14 @@ multiclass sve_int_un_pred_arit_0_h<bits<3> opc, string asm,
   defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i8, !cast<Pseudo>(NAME # _D_UNDEF)>;
 }
 
-multiclass sve_int_un_pred_arit_0_w<bits<3> opc, string asm,
-                                    SDPatternOperator op> {
+multiclass sve_int_un_pred_arit_h_z<bits<3> opc, string asm> {
+  def _H : sve_int_un_pred_arit_z<0b01, { opc, 0b0 }, asm, ZPR16>;
+  def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b0 }, asm, ZPR32>;
+  def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b0 }, asm, ZPR64>;
+}
+
+multiclass sve_int_un_pred_arit_w<bits<3> opc, string asm,
+                                  SDPatternOperator op> {
   def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>,
            SVEPseudo2Instr<NAME # _S, 1>;
   def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>,
@@ -4751,8 +4786,13 @@ multiclass sve_int_un_pred_arit_0_w<bits<3> opc, string asm,
   defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i16, !cast<Pseudo>(NAME # _D_UNDEF)>;
 }
 
-multiclass sve_int_un_pred_arit_0_d<bits<3> opc, string asm,
-                                    SDPatternOperator op> {
+multiclass sve_int_un_pred_arit_w_z<bits<3> opc, string asm> {
+  def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b0 }, asm, ZPR32>;
+  def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b0 }, asm, ZPR64>;
+}
+
+multiclass sve_int_un_pred_arit_d<bits<3> opc, string asm,
+                                  SDPatternOperator op> {
   def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>,
            SVEPseudo2Instr<NAME # _D, 1>;
 
@@ -4763,8 +4803,8 @@ multiclass sve_int_un_pred_arit_0_d<bits<3> opc, string asm,
   defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i32, !cast<Pseudo>(NAME # _D_UNDEF)>;
 }
 
-multiclass sve_int_un_pred_arit_1<bits<3> opc, string asm,
-                                  SDPatternOperator op> {
+multiclass sve_int_un_pred_arit_bitwise<bits<3> opc, string asm,
+                                        SDPatternOperator op> {
   def _B : sve_int_un_pred_arit<0b00, { opc, 0b1 }, asm, ZPR8>,
            SVEPseudo2Instr<NAME # _B, 1>;
   def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>,
@@ -4790,7 +4830,15 @@ multiclass sve_int_un_pred_arit_1<bits<3> opc, string asm,
   defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;
 }
 
-multiclass sve_int_un_pred_arit_1_fp<bits<3> opc, string asm, SDPatternOperator op> {
+multiclass sve_int_un_pred_arit_bitwise_z<bits<3> opc, string asm> {
+  def _B : sve_int_un_pred_arit_z<0b00, { opc, 0b1 }, asm, ZPR8>;
+  def _H : sve_int_un_pred_arit_z<0b01, { opc, 0b1 }, asm, ZPR16>;
+  def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b1 }, asm, ZPR32>;
+  def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b1 }, asm, ZPR64>;
+}
+
+multiclass sve_int_un_pred_arit_bitwise_fp<bits<3> opc, string asm,
+                                           SDPatternOperator op> {
   def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>,
            SVEPseudo2Instr<NAME # _H, 1>;
   def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>,
@@ -4817,6 +4865,12 @@ multiclass sve_int_un_pred_arit_1_fp<bits<3> opc, string asm, SDPatternOperator
   defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Pseudo>(NAME # _D_UNDEF)>;
 }
 
+multiclass sve_int_un_pred_arit_bitwise_fp_z<bits<3> opc, string asm> {
+  def _H : sve_int_un_pred_arit_z<0b01, { opc, 0b1 }, asm, ZPR16>;
+  def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b1 }, asm, ZPR32>;
+  def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b1 }, asm, ZPR64>;
+}
+
 //===----------------------------------------------------------------------===//
 // SVE Integer Wide Immediate - Unpredicated Group
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AArch64/SVE2p2/unary_arithmetic_predicated_z-diagnotics.s b/llvm/test/MC/AArch64/SVE2p2/unary_arithmetic_predicated_z-diagnotics.s
new file mode 100644
index 000000000000..b44039fae464
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/unary_arithmetic_predicated_z-diagnotics.s
@@ -0,0 +1,249 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid element width
+abs     z31.b, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: abs     z31.b, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+cls     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: cls     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+clz     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: clz     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+cnot     z31.b, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: cnot     z31.b, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+cnt     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: cnt     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fabs     z31.h, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fabs     z31.h, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fneg     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fneg     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+neg     z31.s, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: neg     z31.s, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+not     z31.b, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: not     z31.b, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sxtb     z31.h, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: sxtb     z31.h, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sxth     z31.s, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: sxth     z31.s, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sxtw     z31.d, p7/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: sxtw     z31.d, p7/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uxtb     z31.s, p7/z, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: uxtb     z31.s, p7/z, z31.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uxth     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: uxth     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uxtw     z31.d, p7/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: uxtw     z31.d, p7/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate
+
+abs     z31.s, p8/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: abs     z31.s, p8/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+cls     z31.b, p8/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: cls     z31.b, p8/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+clz     z31.b, p8/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: clz     z31.b, p8/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+cnot     z31.b, p8/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: cnot     z31.b, p8/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+cnt     z31.b, p8/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: cnt     z31.b, p8/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fabs    z31.h, p8/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: fabs    z31.h, p8/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fneg    z31.h, p8/z, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: fneg    z31.h, p8/z, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+neg     z31.s, p8/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: neg     z31.s, p8/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+not     z31.b, p8/z, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: not     z31.b, p8/z, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sxtb     z31.s, p8/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: sxtb     z31.s, p8/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sxth z0.s, p8/z, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: sxth z0.s, p8/z, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sxtw z0.d, p8/z, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: sxtw z0.d, p8/z, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uxtb     z31.s, p8/z, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: uxtb     z31.s, p8/z, z31.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uxth z0.s, p8/z, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: uxth z0.s, p8/z, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uxtw z0.d, p8/z, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: uxtw z0.d, p8/z, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.d, p0/z, z7.d
+abs z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: abs z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+cls z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: cls z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0.h, p0/z, z7.h
+clz z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: clz z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+cnot z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: cnot z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0.h, p0/z, z7.h
+cnt z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: cnt z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+fabs z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: fabs z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0.h, p0/z, z7.h
+fneg z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: fneg z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+neg z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: neg z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0.h, p0/z, z7.h
+not z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: not z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+sxtb z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: sxtb z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0.s, p0/z, z7.s
+sxth z0.s, p0/z, z3.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: sxth z0.s, p0/z, z3.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+sxtw z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: sxtw z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+uxtb z0.h, p0/z, z3.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: uxtb z0.h, p0/z, z3.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0.s, p0/z, z7.s
+uxth z0.s, p0/z, z3.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: uxth z0.s, p0/z, z3.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+uxtw z0.d, p0/z, z3.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: uxtw z0.d, p0/z, z3.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE2p2/unary_arithmetic_predicated_z.s b/llvm/test/MC/AArch64/SVE2p2/unary_arithmetic_predicated_z.s
new file mode 100644
index 000000000000..c460602f31ae
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/unary_arithmetic_predicated_z.s
@@ -0,0 +1,225 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// ABS
+
+abs     z0.b, p0/z, z0.b  // 00000100-00000110-10100000-00000000
+// CHECK-INST: abs     z0.b, p0/z, z0.b
+// CHECK-ENCODING: [0x00,0xa0,0x06,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 0406a000 <unknown>
+
+abs     z31.d, p7/z, z31.d  // 00000100-11000110-10111111-11111111
+// CHECK-INST: abs     z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xc6,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04c6bfff <unknown>
+
+
+// CLS
+
+cls     z0.b, p0/z, z0.b  // 00000100-00001000-10100000-00000000
+// CHECK-INST: cls     z0.b, p0/z, z0.b
+// CHECK-ENCODING: [0x00,0xa0,0x08,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 0408a000 <unknown>
+
+clz     z31.d, p7/z, z31.d  // 00000100-11001001-10111111-11111111
+// CHECK-INST: clz     z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xc9,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04c9bfff <unknown>
+
+// CLZ
+
+clz     z0.b, p0/z, z0.b  // 00000100-00001001-10100000-00000000
+// CHECK-INST: clz     z0.b, p0/z, z0.b
+// CHECK-ENCODING: [0x00,0xa0,0x09,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 0409a000 <unknown>
+
+clz     z31.d, p7/z, z31.d  // 00000100-11001001-10111111-11111111
+// CHECK-INST: clz     z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xc9,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04c9bfff <unknown>
+
+// CNOT
+
+cnot    z0.b, p0/z, z0.b  // 00000100-00001011-10100000-00000000
+// CHECK-INST: cnot    z0.b, p0/z, z0.b
+// CHECK-ENCODING: [0x00,0xa0,0x0b,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 040ba000 <unknown>
+
+cnot    z31.d, p7/z, z31.d  // 00000100-11001011-10111111-11111111
+// CHECK-INST: cnot    z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xcb,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04cbbfff <unknown>
+
+// CNT
+
+cnt     z0.b, p0/z, z0.b  // 00000100-00001010-10100000-00000000
+// CHECK-INST: cnt     z0.b, p0/z, z0.b
+// CHECK-ENCODING: [0x00,0xa0,0x0a,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 040aa000 <unknown>
+
+cnt     z31.d, p7/z, z31.d  // 00000100-11001010-10111111-11111111
+// CHECK-INST: cnt     z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xca,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04cabfff <unknown>
+
+
+// FABS
+
+fabs    z0.h, p0/z, z0.h  // 00000100-01001100-10100000-00000000
+// CHECK-INST: fabs    z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xa0,0x4c,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 044ca000 <unknown>
+
+fabs    z31.d, p7/z, z31.d  // 00000100-11001100-10111111-11111111
+// CHECK-INST: fabs    z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xcc,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04ccbfff <unknown>
+
+// FNEG
+
+fneg    z0.h, p0/z, z0.h  // 00000100-01001101-10100000-00000000
+// CHECK-INST: fneg    z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xa0,0x4d,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 044da000 <unknown>
+
+fneg    z31.d, p7/z, z31.d  // 00000100-11001101-10111111-11111111
+// CHECK-INST: fneg    z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xcd,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04cdbfff <unknown>
+
+// NEG
+
+neg     z0.b, p0/z, z0.b  // 00000100-00000111-10100000-00000000
+// CHECK-INST: neg     z0.b, p0/z, z0.b
+// CHECK-ENCODING: [0x00,0xa0,0x07,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 0407a000 <unknown>
+
+neg     z31.d, p7/z, z31.d  // 00000100-11000111-10111111-11111111
+// CHECK-INST: neg     z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xc7,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04c7bfff <unknown>
+
+//NOT
+
+not     z0.b, p0/z, z0.b  // 00000100-00001110-10100000-00000000
+// CHECK-INST: not     z0.b, p0/z, z0.b
+// CHECK-ENCODING: [0x00,0xa0,0x0e,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 040ea000 <unknown>
+
+not     z31.d, p7/z, z31.d  // 00000100-11001110-10111111-11111111
+// CHECK-INST: not     z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xce,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04cebfff <unknown>
+
+// SXTB
+
+sxtb    z0.h, p0/z, z0.h  // 00000100-01000000-10100000-00000000
+// CHECK-INST: sxtb    z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xa0,0x40,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 0440a000 <unknown>
+
+sxtb    z31.d, p7/z, z31.d  // 00000100-11000000-10111111-11111111
+// CHECK-INST: sxtb    z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xc0,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04c0bfff <unknown>
+
+// SXTH
+
+sxth    z0.s, p0/z, z0.s  // 00000100-10000010-10100000-00000000
+// CHECK-INST: sxth    z0.s, p0/z, z0.s
+// CHECK-ENCODING: [0x00,0xa0,0x82,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 0482a000 <unknown>
+
+sxth    z31.d, p7/z, z31.d  // 00000100-11000010-10111111-11111111
+// CHECK-INST: sxth    z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xc2,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04c2bfff <unknown>
+
+// SXTW
+
+sxtw    z0.d, p0/z, z0.d  // 00000100-11000100-10100000-00000000
+// CHECK-INST: sxtw    z0.d, p0/z, z0.d
+// CHECK-ENCODING: [0x00,0xa0,0xc4,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04c4a000 <unknown>
+
+sxtw    z31.d, p7/z, z31.d  // 00000100-11000100-10111111-11111111
+// CHECK-INST: sxtw    z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xc4,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04c4bfff <unknown>
+
+// UXTB
+
+uxtb    z0.h, p0/z, z0.h  // 00000100-01000001-10100000-00000000
+// CHECK-INST: uxtb    z0.h, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xa0,0x41,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 0441a000 <unknown>
+
+uxtb    z31.d, p7/z, z31.d  // 00000100-11000001-10111111-11111111
+// CHECK-INST: uxtb    z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xc1,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04c1bfff <unknown>
+
+uxth    z0.s, p0/z, z0.s  // 00000100-10000011-10100000-00000000
+// CHECK-INST: uxth    z0.s, p0/z, z0.s
+// CHECK-ENCODING: [0x00,0xa0,0x83,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 0483a000 <unknown>
+
+uxth    z31.d, p7/z, z31.d  // 00000100-11000011-10111111-11111111
+// CHECK-INST: uxth    z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xc3,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04c3bfff <unknown>
+
+// UXTW
+
+uxtw    z0.d, p0/z, z0.d  // 00000100-11000101-10100000-00000000
+// CHECK-INST: uxtw    z0.d, p0/z, z0.d
+// CHECK-ENCODING: [0x00,0xa0,0xc5,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04c5a000 <unknown>
+
+uxtw    z31.d, p7/z, z31.d  // 00000100-11000101-10111111-11111111
+// CHECK-INST: uxtw    z31.d, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xbf,0xc5,0x04]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 04c5bfff <unknown>
-- 
GitLab


From 05b6c2e4b933e7a3606899c72067c92b6077287b Mon Sep 17 00:00:00 2001
From: "Oleksandr T." <oleksandr.tarasiuk@outlook.com>
Date: Tue, 29 Oct 2024 11:44:01 +0200
Subject: [PATCH 408/693] [Clang] fix range calculation for conditionals with
 throw expressions (#112081)

Fixes #111854

---

The issue arises when `GetExprRange` encounters a `ConditionalOperator`
with a `CXXThrowExpr`

```md
ConditionalOperator 0x1108658e0 'int'
|-CXXBoolLiteralExpr 0x110865878 '_Bool' true
|-CXXThrowExpr 0x1108658a8 'void'
| `-IntegerLiteral 0x110865888 'int' 0
`-IntegerLiteral 0x1108658c0 'int' 0
```


https://github.com/llvm/llvm-project/blob/ed3d05178274890fb804f43ae1bcdfd33b5fd8f0/clang/lib/Sema/SemaChecking.cpp#L9628-L9631

The current behavior causes the `GetExprRange` to proceed with the throw
expression (`CO->getTrueExpr()`/`void` type)
---
 clang/docs/ReleaseNotes.rst             |   3 +-
 clang/lib/Sema/SemaChecking.cpp         | 216 ++++++++++++++----------
 clang/test/SemaCXX/conditional-expr.cpp |   7 +
 3 files changed, 140 insertions(+), 86 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 424f02ef08d7..a39ffc8366dd 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -571,7 +571,8 @@ Bug Fixes to C++ Support
 - Clang incorrectly considered a class with an anonymous union member to not be
   const-default-constructible even if a union member has a default member initializer.
   (#GH95854).
-- Fixed an assertion failure when evaluating an invalid expression in an array initializer (#GH112140)
+- Fixed an assertion failure when evaluating an invalid expression in an array initializer. (#GH112140)
+- Fixed an assertion failure in range calculations for conditional throw expressions. (#GH111854)
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index d027e4c6dfdb..3308b898a5b6 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -9592,16 +9592,23 @@ static QualType GetExprType(const Expr *E) {
   return Ty;
 }
 
-/// Pseudo-evaluate the given integer expression, estimating the
-/// range of values it might take.
+/// Attempts to estimate an approximate range for the given integer expression.
+/// Returns a range if successful, otherwise it returns \c std::nullopt if a
+/// reliable estimation cannot be determined.
 ///
 /// \param MaxWidth The width to which the value will be truncated.
-/// \param Approximate If \c true, return a likely range for the result: in
-///        particular, assume that arithmetic on narrower types doesn't leave
-///        those types. If \c false, return a range including all possible
-///        result values.
-static IntRange GetExprRange(ASTContext &C, const Expr *E, unsigned MaxWidth,
-                             bool InConstantContext, bool Approximate) {
+/// \param InConstantContext If \c true, interpret the expression within a
+///        constant context.
+/// \param Approximate If \c true, provide a likely range of values by assuming
+///        that arithmetic on narrower types remains within those types.
+///        If \c false, return a range that includes all possible values
+///        resulting from the expression.
+/// \returns A range of values that the expression might take, or
+///          std::nullopt if a reliable estimation cannot be determined.
+static std::optional<IntRange> TryGetExprRange(ASTContext &C, const Expr *E,
+                                               unsigned MaxWidth,
+                                               bool InConstantContext,
+                                               bool Approximate) {
   E = E->IgnoreParens();
 
   // Try a full evaluation first.
@@ -9614,8 +9621,8 @@ static IntRange GetExprRange(ASTContext &C, const Expr *E, unsigned MaxWidth,
   // being of the new, wider type.
   if (const auto *CE = dyn_cast<ImplicitCastExpr>(E)) {
     if (CE->getCastKind() == CK_NoOp || CE->getCastKind() == CK_LValueToRValue)
-      return GetExprRange(C, CE->getSubExpr(), MaxWidth, InConstantContext,
-                          Approximate);
+      return TryGetExprRange(C, CE->getSubExpr(), MaxWidth, InConstantContext,
+                             Approximate);
 
     IntRange OutputTypeRange = IntRange::forValueOfType(C, GetExprType(CE));
 
@@ -9626,40 +9633,52 @@ static IntRange GetExprRange(ASTContext &C, const Expr *E, unsigned MaxWidth,
     if (!isIntegerCast)
       return OutputTypeRange;
 
-    IntRange SubRange = GetExprRange(C, CE->getSubExpr(),
-                                     std::min(MaxWidth, OutputTypeRange.Width),
-                                     InConstantContext, Approximate);
+    std::optional<IntRange> SubRange = TryGetExprRange(
+        C, CE->getSubExpr(), std::min(MaxWidth, OutputTypeRange.Width),
+        InConstantContext, Approximate);
+    if (!SubRange)
+      return std::nullopt;
 
     // Bail out if the subexpr's range is as wide as the cast type.
-    if (SubRange.Width >= OutputTypeRange.Width)
+    if (SubRange->Width >= OutputTypeRange.Width)
       return OutputTypeRange;
 
     // Otherwise, we take the smaller width, and we're non-negative if
     // either the output type or the subexpr is.
-    return IntRange(SubRange.Width,
-                    SubRange.NonNegative || OutputTypeRange.NonNegative);
+    return IntRange(SubRange->Width,
+                    SubRange->NonNegative || OutputTypeRange.NonNegative);
   }
 
   if (const auto *CO = dyn_cast<ConditionalOperator>(E)) {
     // If we can fold the condition, just take that operand.
     bool CondResult;
     if (CO->getCond()->EvaluateAsBooleanCondition(CondResult, C))
-      return GetExprRange(C,
-                          CondResult ? CO->getTrueExpr() : CO->getFalseExpr(),
-                          MaxWidth, InConstantContext, Approximate);
+      return TryGetExprRange(
+          C, CondResult ? CO->getTrueExpr() : CO->getFalseExpr(), MaxWidth,
+          InConstantContext, Approximate);
 
     // Otherwise, conservatively merge.
-    // GetExprRange requires an integer expression, but a throw expression
+    // TryGetExprRange requires an integer expression, but a throw expression
     // results in a void type.
-    Expr *E = CO->getTrueExpr();
-    IntRange L = E->getType()->isVoidType()
-                     ? IntRange{0, true}
-                     : GetExprRange(C, E, MaxWidth, InConstantContext, Approximate);
-    E = CO->getFalseExpr();
-    IntRange R = E->getType()->isVoidType()
-                     ? IntRange{0, true}
-                     : GetExprRange(C, E, MaxWidth, InConstantContext, Approximate);
-    return IntRange::join(L, R);
+    Expr *TrueExpr = CO->getTrueExpr();
+    if (TrueExpr->getType()->isVoidType())
+      return std::nullopt;
+
+    std::optional<IntRange> L =
+        TryGetExprRange(C, TrueExpr, MaxWidth, InConstantContext, Approximate);
+    if (!L)
+      return std::nullopt;
+
+    Expr *FalseExpr = CO->getFalseExpr();
+    if (FalseExpr->getType()->isVoidType())
+      return std::nullopt;
+
+    std::optional<IntRange> R =
+        TryGetExprRange(C, FalseExpr, MaxWidth, InConstantContext, Approximate);
+    if (!R)
+      return std::nullopt;
+
+    return IntRange::join(*L, *R);
   }
 
   if (const auto *BO = dyn_cast<BinaryOperator>(E)) {
@@ -9696,8 +9715,8 @@ static IntRange GetExprRange(ASTContext &C, const Expr *E, unsigned MaxWidth,
     // been coerced to the LHS type.
     case BO_Assign:
       // TODO: bitfields?
-      return GetExprRange(C, BO->getRHS(), MaxWidth, InConstantContext,
-                          Approximate);
+      return TryGetExprRange(C, BO->getRHS(), MaxWidth, InConstantContext,
+                             Approximate);
 
     // Operations with opaque sources are black-listed.
     case BO_PtrMemD:
@@ -9729,18 +9748,20 @@ static IntRange GetExprRange(ASTContext &C, const Expr *E, unsigned MaxWidth,
     // Right shift by a constant can narrow its left argument.
     case BO_Shr:
     case BO_ShrAssign: {
-      IntRange L = GetExprRange(C, BO->getLHS(), MaxWidth, InConstantContext,
-                                Approximate);
+      std::optional<IntRange> L = TryGetExprRange(
+          C, BO->getLHS(), MaxWidth, InConstantContext, Approximate);
+      if (!L)
+        return std::nullopt;
 
       // If the shift amount is a positive constant, drop the width by
       // that much.
       if (std::optional<llvm::APSInt> shift =
               BO->getRHS()->getIntegerConstantExpr(C)) {
         if (shift->isNonNegative()) {
-          if (shift->uge(L.Width))
-            L.Width = (L.NonNegative ? 0 : 1);
+          if (shift->uge(L->Width))
+            L->Width = (L->NonNegative ? 0 : 1);
           else
-            L.Width -= shift->getZExtValue();
+            L->Width -= shift->getZExtValue();
         }
       }
 
@@ -9749,8 +9770,8 @@ static IntRange GetExprRange(ASTContext &C, const Expr *E, unsigned MaxWidth,
 
     // Comma acts as its right operand.
     case BO_Comma:
-      return GetExprRange(C, BO->getRHS(), MaxWidth, InConstantContext,
-                          Approximate);
+      return TryGetExprRange(C, BO->getRHS(), MaxWidth, InConstantContext,
+                             Approximate);
 
     case BO_Add:
       if (!Approximate)
@@ -9774,26 +9795,31 @@ static IntRange GetExprRange(ASTContext &C, const Expr *E, unsigned MaxWidth,
     case BO_Div: {
       // Don't 'pre-truncate' the operands.
       unsigned opWidth = C.getIntWidth(GetExprType(E));
-      IntRange L = GetExprRange(C, BO->getLHS(), opWidth, InConstantContext,
-                                Approximate);
+      std::optional<IntRange> L = TryGetExprRange(
+          C, BO->getLHS(), opWidth, InConstantContext, Approximate);
+      if (!L)
+        return std::nullopt;
 
       // If the divisor is constant, use that.
       if (std::optional<llvm::APSInt> divisor =
               BO->getRHS()->getIntegerConstantExpr(C)) {
         unsigned log2 = divisor->logBase2(); // floor(log_2(divisor))
-        if (log2 >= L.Width)
-          L.Width = (L.NonNegative ? 0 : 1);
+        if (log2 >= L->Width)
+          L->Width = (L->NonNegative ? 0 : 1);
         else
-          L.Width = std::min(L.Width - log2, MaxWidth);
+          L->Width = std::min(L->Width - log2, MaxWidth);
         return L;
       }
 
       // Otherwise, just use the LHS's width.
       // FIXME: This is wrong if the LHS could be its minimal value and the RHS
       // could be -1.
-      IntRange R = GetExprRange(C, BO->getRHS(), opWidth, InConstantContext,
-                                Approximate);
-      return IntRange(L.Width, L.NonNegative && R.NonNegative);
+      std::optional<IntRange> R = TryGetExprRange(
+          C, BO->getRHS(), opWidth, InConstantContext, Approximate);
+      if (!R)
+        return std::nullopt;
+
+      return IntRange(L->Width, L->NonNegative && R->NonNegative);
     }
 
     case BO_Rem:
@@ -9810,11 +9836,17 @@ static IntRange GetExprRange(ASTContext &C, const Expr *E, unsigned MaxWidth,
     // performed the computation.
     QualType T = GetExprType(E);
     unsigned opWidth = C.getIntWidth(T);
-    IntRange L =
-        GetExprRange(C, BO->getLHS(), opWidth, InConstantContext, Approximate);
-    IntRange R =
-        GetExprRange(C, BO->getRHS(), opWidth, InConstantContext, Approximate);
-    IntRange C = Combine(L, R);
+    std::optional<IntRange> L = TryGetExprRange(C, BO->getLHS(), opWidth,
+                                                InConstantContext, Approximate);
+    if (!L)
+      return std::nullopt;
+
+    std::optional<IntRange> R = TryGetExprRange(C, BO->getRHS(), opWidth,
+                                                InConstantContext, Approximate);
+    if (!R)
+      return std::nullopt;
+
+    IntRange C = Combine(*L, *R);
     C.NonNegative |= T->isUnsignedIntegerOrEnumerationType();
     C.Width = std::min(C.Width, MaxWidth);
     return C;
@@ -9832,26 +9864,30 @@ static IntRange GetExprRange(ASTContext &C, const Expr *E, unsigned MaxWidth,
       return IntRange::forValueOfType(C, GetExprType(E));
 
     default:
-      return GetExprRange(C, UO->getSubExpr(), MaxWidth, InConstantContext,
-                          Approximate);
+      return TryGetExprRange(C, UO->getSubExpr(), MaxWidth, InConstantContext,
+                             Approximate);
     }
   }
 
   if (const auto *OVE = dyn_cast<OpaqueValueExpr>(E))
-    return GetExprRange(C, OVE->getSourceExpr(), MaxWidth, InConstantContext,
-                        Approximate);
+    return TryGetExprRange(C, OVE->getSourceExpr(), MaxWidth, InConstantContext,
+                           Approximate);
 
   if (const auto *BitField = E->getSourceBitField())
     return IntRange(BitField->getBitWidthValue(C),
                     BitField->getType()->isUnsignedIntegerOrEnumerationType());
 
+  if (GetExprType(E)->isVoidType())
+    return std::nullopt;
+
   return IntRange::forValueOfType(C, GetExprType(E));
 }
 
-static IntRange GetExprRange(ASTContext &C, const Expr *E,
-                             bool InConstantContext, bool Approximate) {
-  return GetExprRange(C, E, C.getIntWidth(GetExprType(E)), InConstantContext,
-                      Approximate);
+static std::optional<IntRange> TryGetExprRange(ASTContext &C, const Expr *E,
+                                               bool InConstantContext,
+                                               bool Approximate) {
+  return TryGetExprRange(C, E, C.getIntWidth(GetExprType(E)), InConstantContext,
+                         Approximate);
 }
 
 /// Checks whether the given value, which currently has the given
@@ -10096,8 +10132,10 @@ static bool CheckTautologicalComparison(Sema &S, BinaryOperator *E,
       S.Context.hasSameUnqualifiedType(Constant->getType(), Other->getType()))
     return false;
 
-  IntRange OtherValueRange = GetExprRange(
+  std::optional<IntRange> OtherValueRange = TryGetExprRange(
       S.Context, Other, S.isConstantEvaluatedContext(), /*Approximate=*/false);
+  if (!OtherValueRange)
+    return false;
 
   QualType OtherT = Other->getType();
   if (const auto *AT = OtherT->getAs<AtomicType>())
@@ -10115,11 +10153,11 @@ static bool CheckTautologicalComparison(Sema &S, BinaryOperator *E,
   bool OtherIsBooleanDespiteType =
       !OtherT->isBooleanType() && Other->isKnownToHaveBooleanValue();
   if (OtherIsBooleanDespiteType || IsObjCSignedCharBool)
-    OtherTypeRange = OtherValueRange = IntRange::forBoolType();
+    OtherTypeRange = *OtherValueRange = IntRange::forBoolType();
 
   // Check if all values in the range of possible values of this expression
   // lead to the same comparison outcome.
-  PromotedRange OtherPromotedValueRange(OtherValueRange, Value.getBitWidth(),
+  PromotedRange OtherPromotedValueRange(*OtherValueRange, Value.getBitWidth(),
                                         Value.isUnsigned());
   auto Cmp = OtherPromotedValueRange.compare(Value);
   auto Result = PromotedRange::constantValue(E->getOpcode(), Cmp, RhsConstant);
@@ -10143,7 +10181,7 @@ static bool CheckTautologicalComparison(Sema &S, BinaryOperator *E,
 
   // Don't warn if the non-constant operand actually always evaluates to the
   // same value.
-  if (!TautologicalTypeCompare && OtherValueRange.Width == 0)
+  if (!TautologicalTypeCompare && OtherValueRange->Width == 0)
     return false;
 
   // Suppress the diagnostic for an in-range comparison if the constant comes
@@ -10182,7 +10220,7 @@ static bool CheckTautologicalComparison(Sema &S, BinaryOperator *E,
 
   if (!TautologicalTypeCompare) {
     S.Diag(E->getOperatorLoc(), diag::warn_tautological_compare_value_range)
-        << RhsConstant << OtherValueRange.Width << OtherValueRange.NonNegative
+        << RhsConstant << OtherValueRange->Width << OtherValueRange->NonNegative
         << E->getOpcodeStr() << OS.str() << *Result
         << E->getLHS()->getSourceRange() << E->getRHS()->getSourceRange();
     return true;
@@ -10312,9 +10350,11 @@ static void AnalyzeComparison(Sema &S, BinaryOperator *E) {
   }
 
   // Otherwise, calculate the effective range of the signed operand.
-  IntRange signedRange =
-      GetExprRange(S.Context, signedOperand, S.isConstantEvaluatedContext(),
-                   /*Approximate=*/true);
+  std::optional<IntRange> signedRange =
+      TryGetExprRange(S.Context, signedOperand, S.isConstantEvaluatedContext(),
+                      /*Approximate=*/true);
+  if (!signedRange)
+    return;
 
   // Go ahead and analyze implicit conversions in the operands.  Note
   // that we skip the implicit conversions on both sides.
@@ -10322,7 +10362,7 @@ static void AnalyzeComparison(Sema &S, BinaryOperator *E) {
   AnalyzeImplicitConversions(S, RHS, E->getOperatorLoc());
 
   // If the signed range is non-negative, -Wsign-compare won't fire.
-  if (signedRange.NonNegative)
+  if (signedRange->NonNegative)
     return;
 
   // For (in)equality comparisons, if the unsigned operand is a
@@ -10331,15 +10371,17 @@ static void AnalyzeComparison(Sema &S, BinaryOperator *E) {
   // change the result of the comparison.
   if (E->isEqualityOp()) {
     unsigned comparisonWidth = S.Context.getIntWidth(T);
-    IntRange unsignedRange =
-        GetExprRange(S.Context, unsignedOperand, S.isConstantEvaluatedContext(),
-                     /*Approximate=*/true);
+    std::optional<IntRange> unsignedRange = TryGetExprRange(
+        S.Context, unsignedOperand, S.isConstantEvaluatedContext(),
+        /*Approximate=*/true);
+    if (!unsignedRange)
+      return;
 
     // We should never be unable to prove that the unsigned operand is
     // non-negative.
-    assert(unsignedRange.NonNegative && "unsigned range includes negative?");
+    assert(unsignedRange->NonNegative && "unsigned range includes negative?");
 
-    if (unsignedRange.Width < comparisonWidth)
+    if (unsignedRange->Width < comparisonWidth)
       return;
   }
 
@@ -11146,10 +11188,12 @@ void Sema::CheckImplicitConversion(Expr *E, QualType T, SourceLocation CC,
   if (SourceBT && TargetBT && SourceBT->isIntegerType() &&
       TargetBT->isFloatingType() && !IsListInit) {
     // Determine the number of precision bits in the source integer type.
-    IntRange SourceRange =
-        GetExprRange(Context, E, isConstantEvaluatedContext(),
-                     /*Approximate=*/true);
-    unsigned int SourcePrecision = SourceRange.Width;
+    std::optional<IntRange> SourceRange =
+        TryGetExprRange(Context, E, isConstantEvaluatedContext(),
+                        /*Approximate=*/true);
+    if (!SourceRange)
+      return;
+    unsigned int SourcePrecision = SourceRange->Width;
 
     // Determine the number of precision bits in the
     // target floating point type.
@@ -11212,14 +11256,16 @@ void Sema::CheckImplicitConversion(Expr *E, QualType T, SourceLocation CC,
         E, Diag(CC, diag::warn_impcast_int_to_objc_signed_char_bool)
                << E->getType());
   }
+  std::optional<IntRange> LikelySourceRange = TryGetExprRange(
+      Context, E, isConstantEvaluatedContext(), /*Approximate=*/true);
+  if (!LikelySourceRange)
+    return;
 
   IntRange SourceTypeRange =
       IntRange::forTargetOfCanonicalType(Context, Source);
-  IntRange LikelySourceRange = GetExprRange(
-      Context, E, isConstantEvaluatedContext(), /*Approximate=*/true);
   IntRange TargetRange = IntRange::forTargetOfCanonicalType(Context, Target);
 
-  if (LikelySourceRange.Width > TargetRange.Width) {
+  if (LikelySourceRange->Width > TargetRange.Width) {
     // If the source is a constant, use a default-on diagnostic.
     // TODO: this should happen for bitfield stores, too.
     Expr::EvalResult Result;
@@ -11266,8 +11312,8 @@ void Sema::CheckImplicitConversion(Expr *E, QualType T, SourceLocation CC,
         }
   }
 
-  if (TargetRange.Width == LikelySourceRange.Width &&
-      !TargetRange.NonNegative && LikelySourceRange.NonNegative &&
+  if (TargetRange.Width == LikelySourceRange->Width &&
+      !TargetRange.NonNegative && LikelySourceRange->NonNegative &&
       Source->isSignedIntegerType()) {
     // Warn when doing a signed to signed conversion, warn if the positive
     // source value is exactly the width of the target type, which will
@@ -11293,9 +11339,9 @@ void Sema::CheckImplicitConversion(Expr *E, QualType T, SourceLocation CC,
   }
 
   if ((!isa<EnumType>(Target) || !isa<EnumType>(Source)) &&
-      ((TargetRange.NonNegative && !LikelySourceRange.NonNegative) ||
-       (!TargetRange.NonNegative && LikelySourceRange.NonNegative &&
-        LikelySourceRange.Width == TargetRange.Width))) {
+      ((TargetRange.NonNegative && !LikelySourceRange->NonNegative) ||
+       (!TargetRange.NonNegative && LikelySourceRange->NonNegative &&
+        LikelySourceRange->Width == TargetRange.Width))) {
     if (SourceMgr.isInSystemMacro(CC))
       return;
 
diff --git a/clang/test/SemaCXX/conditional-expr.cpp b/clang/test/SemaCXX/conditional-expr.cpp
index 01effaa18932..8f17555fd806 100644
--- a/clang/test/SemaCXX/conditional-expr.cpp
+++ b/clang/test/SemaCXX/conditional-expr.cpp
@@ -429,3 +429,10 @@ void g() {
   long e = a = b ? throw 0 : throw 1;
 }
 } // namespace PR46484
+
+namespace GH111854 {
+void f() {
+  (true ? throw 0 : 0) <= 0;  // expected-warning {{relational comparison result unused}}
+  (false ? 0 : throw 0) <= 0; // expected-warning {{relational comparison result unused}}
+}
+}
-- 
GitLab


From dfb60bb9193d78d0980193e1ade715cffbb55af8 Mon Sep 17 00:00:00 2001
From: Rohit Aggarwal <44664450+rohitaggarwal007@users.noreply.github.com>
Date: Tue, 29 Oct 2024 15:39:55 +0530
Subject: [PATCH 409/693] Adding more vector calls for -fveclib=AMDLIBM
 (#109662)

AMD has it's own implementation of vector calls.
New vector calls are introduced in the library for exp10, log10, sincos and finite asin/acos
Please refer [https://github.com/amd/aocl-libm-ose]

---------

Co-authored-by: Rohit Aggarwal <Rohit.Aggarwal@amd.com>
---
 .../include/llvm/Analysis/TargetLibraryInfo.h |   1 +
 .../llvm/Analysis/TargetTransformInfoImpl.h   |   3 +-
 llvm/include/llvm/Analysis/VecFuncs.def       |  24 ++
 llvm/lib/Analysis/ValueTracking.cpp           |   4 +
 llvm/lib/Analysis/VectorUtils.cpp             |   1 +
 .../LoopVectorize/X86/amdlibm-calls-finite.ll | 275 ++++++++++++------
 .../LoopVectorize/X86/amdlibm-calls.ll        | 216 ++++++++++++++
 llvm/test/Transforms/Util/add-TLI-mappings.ll |  27 +-
 8 files changed, 455 insertions(+), 96 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
index aeb8de3973f7..5347c64e43e7 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
@@ -416,6 +416,7 @@ public:
     case LibFunc_cos:          case LibFunc_cosf:       case LibFunc_cosl:
     case LibFunc_cosh:         case LibFunc_coshf:      case LibFunc_coshl:
     case LibFunc_exp2:         case LibFunc_exp2f:      case LibFunc_exp2l:
+    case LibFunc_exp10:        case LibFunc_exp10f:     case LibFunc_exp10l:
     case LibFunc_fabs:         case LibFunc_fabsf:      case LibFunc_fabsl:
     case LibFunc_floor:        case LibFunc_floorf:     case LibFunc_floorl:
     case LibFunc_fmax:         case LibFunc_fmaxf:      case LibFunc_fmaxl:
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 0b7792f89a05..317c13917c0c 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -177,7 +177,8 @@ public:
         Name == "sinh"  || Name == "sinhf"  || Name == "sinhl" ||
         Name == "cosh"  || Name == "coshf"  || Name == "coshl" ||
         Name == "tanh"  || Name == "tanhf"  || Name == "tanhl" ||
-        Name == "sqrt" || Name == "sqrtf" || Name == "sqrtl")
+        Name == "sqrt"  || Name == "sqrtf"  || Name == "sqrtl" ||
+        Name == "exp10"  || Name == "exp10l"  || Name == "exp10f")
       return false;
     // clang-format on
     // These are all likely to be optimized into something smaller.
diff --git a/llvm/include/llvm/Analysis/VecFuncs.def b/llvm/include/llvm/Analysis/VecFuncs.def
index c4586894e3e4..71ad3a35eb3f 100644
--- a/llvm/include/llvm/Analysis/VecFuncs.def
+++ b/llvm/include/llvm/Analysis/VecFuncs.def
@@ -1328,14 +1328,17 @@ TLI_DEFINE_VECFUNC("llvm.log2.f64", "amd_vrd2_log2", FIXED(2), NOMASK, "_ZGV_LLV
 TLI_DEFINE_VECFUNC("llvm.log2.f64", "amd_vrd4_log2", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("llvm.log2.f64", "amd_vrd8_log2", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
 
+TLI_DEFINE_VECFUNC("log10", "amd_vrd2_log10", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("log10f", "amd_vrs16_log10f", FIXED(16), NOMASK, "_ZGV_LLVM_N16v")
 TLI_DEFINE_VECFUNC("log10f", "amd_vrs8_log10f", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
 TLI_DEFINE_VECFUNC("log10f", "amd_vrs4_log10f", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 
+TLI_DEFINE_VECFUNC("__log10_finite", "amd_vrd2_log10", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("__log10f_finite", "amd_vrs16_log10f", FIXED(16), NOMASK, "_ZGV_LLVM_N16v")
 TLI_DEFINE_VECFUNC("__log10f_finite", "amd_vrs8_log10f", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
 TLI_DEFINE_VECFUNC("__log10f_finite", "amd_vrs4_log10f", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 
+TLI_DEFINE_VECFUNC("llvm.log10.f64", "amd_vrd2_log10", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("llvm.log10.f32", "amd_vrs16_log10f", FIXED(16), NOMASK, "_ZGV_LLVM_N16v")
 TLI_DEFINE_VECFUNC("llvm.log10.f32", "amd_vrs8_log10f", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
 TLI_DEFINE_VECFUNC("llvm.log10.f32", "amd_vrs4_log10f", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
@@ -1350,6 +1353,12 @@ TLI_DEFINE_VECFUNC("erf", "amd_vrd8_erf", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
 TLI_DEFINE_VECFUNC("exp10", "amd_vrd2_exp10", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("exp10f", "amd_vrs4_exp10f", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 
+TLI_DEFINE_VECFUNC("__exp10_finite", "amd_vrd2_exp10", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("__exp10f_finite", "amd_vrs4_exp10f", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
+
+TLI_DEFINE_VECFUNC("llvm.exp10.f64", "amd_vrd2_exp10", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.exp10.f32", "amd_vrs4_exp10f", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
+
 TLI_DEFINE_VECFUNC("expm1", "amd_vrd2_expm1", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("expm1f", "amd_vrs4_expm1f", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 
@@ -1380,10 +1389,19 @@ TLI_DEFINE_VECFUNC("llvm.asin.f32", "amd_vrs4_asinf", FIXED(4), NOMASK, "_ZGV_LL
 TLI_DEFINE_VECFUNC("llvm.asin.f32", "amd_vrs8_asinf", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
 TLI_DEFINE_VECFUNC("llvm.asin.f32", "amd_vrs16_asinf", FIXED(16), NOMASK, "_ZGV_LLVM_N16v")
 
+TLI_DEFINE_VECFUNC("__asin_finite", "amd_vrd8_asin", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
+TLI_DEFINE_VECFUNC("__asinf_finite", "amd_vrs4_asinf", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("__asinf_finite", "amd_vrs8_asinf", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
+TLI_DEFINE_VECFUNC("__asinf_finite", "amd_vrs16_asinf", FIXED(16), NOMASK, "_ZGV_LLVM_N16v")
+
 TLI_DEFINE_VECFUNC("acosf", "amd_vrs4_acosf", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("acosf", "amd_vrs8_acosf", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
 TLI_DEFINE_VECFUNC("acosf", "amd_vrs16_acosf", FIXED(16), NOMASK, "_ZGV_LLVM_N16v")
 
+TLI_DEFINE_VECFUNC("__acosf_finite", "amd_vrs4_acosf", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("__acosf_finite", "amd_vrs8_acosf", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
+TLI_DEFINE_VECFUNC("__acosf_finite", "amd_vrs16_acosf", FIXED(16), NOMASK, "_ZGV_LLVM_N16v")
+
 TLI_DEFINE_VECFUNC("llvm.acos.f32", "amd_vrs16_acosf", FIXED(16), NOMASK, "_ZGV_LLVM_N16v")
 TLI_DEFINE_VECFUNC("llvm.acos.f32", "amd_vrs8_acosf", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
 TLI_DEFINE_VECFUNC("llvm.acos.f32", "amd_vrs4_acosf", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
@@ -1421,6 +1439,12 @@ TLI_DEFINE_VECFUNC("llvm.tanh.f32", "amd_vrs16_tanhf", FIXED(16), NOMASK, "_ZGV_
 TLI_DEFINE_VECFUNC("cbrt", "amd_vrd2_cbrt", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("cbrtf", "amd_vrs4_cbrtf", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 
+TLI_DEFINE_VECFUNC("sincos", "amd_vrd4_sincos", FIXED(4), NOMASK, "_ZGV_LLVM_N4vl8l8")
+TLI_DEFINE_VECFUNC("sincos", "amd_vrd8_sincos", FIXED(8), NOMASK, "_ZGV_LLVM_N8vl8l8")
+
+TLI_DEFINE_VECFUNC("sincosf", "amd_vrs4_sincosf", FIXED(4), NOMASK, "_ZGV_LLVM_N4vl4l4")
+TLI_DEFINE_VECFUNC("sincosf", "amd_vrs8_sincosf", FIXED(8), NOMASK, "_ZGV_LLVM_N8vl4l4")
+TLI_DEFINE_VECFUNC("sincosf", "amd_vrs16_sincosf", FIXED(16), NOMASK, "_ZGV_LLVM_N16vl4l4")
 #else
 #error "Must choose which vector library functions are to be defined."
 #endif
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index e9ed8b3c862b..aa5142f33624 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -4241,6 +4241,10 @@ Intrinsic::ID llvm::getIntrinsicForCallSite(const CallBase &CB,
   case LibFunc_exp2f:
   case LibFunc_exp2l:
     return Intrinsic::exp2;
+  case LibFunc_exp10:
+  case LibFunc_exp10f:
+  case LibFunc_exp10l:
+    return Intrinsic::exp10;
   case LibFunc_log:
   case LibFunc_logf:
   case LibFunc_logl:
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 37c443011719..cd5cf0443541 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -76,6 +76,7 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) {
   case Intrinsic::cosh:
   case Intrinsic::tanh:
   case Intrinsic::exp:
+  case Intrinsic::exp10:
   case Intrinsic::exp2:
   case Intrinsic::log:
   case Intrinsic::log10:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls-finite.ll b/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls-finite.ll
index 54bb9352f3c8..9899eded7380 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls-finite.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls-finite.ll
@@ -7,12 +7,10 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-declare float @__expf_finite(float) #0
-
+define void @exp_f32(ptr nocapture %varray) {
 ; CHECK-LABEL: @exp_f32
 ; CHECK: <4 x float> @amd_vrs4_expf
 ; CHECK: ret
-define void @exp_f32(ptr nocapture %varray) {
 entry:
   br label %for.body
 
@@ -25,23 +23,16 @@ for.body:                                         ; preds = %for.body, %entry
   store float %call, ptr %arrayidx, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 1000
-  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
 
 for.end:                                          ; preds = %for.body
   ret void
 }
 
-!1 = distinct !{!1, !2, !3}
-!2 = !{!"llvm.loop.vectorize.width", i32 4}
-!3 = !{!"llvm.loop.vectorize.enable", i1 true}
-
-
-declare double @__exp_finite(double) #0
-
+define void @exp_f64(ptr nocapture %varray) {
 ; CHECK-LABEL: @exp_f64
 ; CHECK: <4 x double> @amd_vrd4_exp
 ; CHECK: ret
-define void @exp_f64(ptr nocapture %varray) {
 entry:
   br label %for.body
 
@@ -54,25 +45,16 @@ for.body:                                         ; preds = %for.body, %entry
   store double %call, ptr %arrayidx, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 1000
-  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !11
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
 
 for.end:                                          ; preds = %for.body
   ret void
 }
 
-!11 = distinct !{!11, !12, !13}
-!12 = !{!"llvm.loop.vectorize.width", i32 4}
-!13 = !{!"llvm.loop.vectorize.enable", i1 true}
-
-
-
-
-declare float @__logf_finite(float) #0
-
+define void @log_f32(ptr nocapture %varray) {
 ; CHECK-LABEL: @log_f32
 ; CHECK: <4 x float> @amd_vrs4_logf
 ; CHECK: ret
-define void @log_f32(ptr nocapture %varray) {
 entry:
   br label %for.body
 
@@ -85,23 +67,16 @@ for.body:                                         ; preds = %for.body, %entry
   store float %call, ptr %arrayidx, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 1000
-  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !21
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
 
 for.end:                                          ; preds = %for.body
   ret void
 }
 
-!21 = distinct !{!21, !22, !23}
-!22 = !{!"llvm.loop.vectorize.width", i32 4}
-!23 = !{!"llvm.loop.vectorize.enable", i1 true}
-
-
-declare double @__log_finite(double) #0
-
+define void @log_f64(ptr nocapture %varray) {
 ; CHECK-LABEL: @log_f64
 ; CHECK: <4 x double> @amd_vrd4_log
 ; CHECK: ret
-define void @log_f64(ptr nocapture %varray) {
 entry:
   br label %for.body
 
@@ -114,23 +89,16 @@ for.body:                                         ; preds = %for.body, %entry
   store double %call, ptr %arrayidx, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 1000
-  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !31
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
 
 for.end:                                          ; preds = %for.body
   ret void
 }
 
-!31 = distinct !{!31, !32, !33}
-!32 = !{!"llvm.loop.vectorize.width", i32 4}
-!33 = !{!"llvm.loop.vectorize.enable", i1 true}
-
-
-declare float @__powf_finite(float, float) #0
-
+define void @pow_f32(ptr nocapture %varray, ptr nocapture readonly %exp) {
 ; CHECK-LABEL: @pow_f32
 ; CHECK: <4 x float> @amd_vrs4_powf
 ; CHECK: ret
-define void @pow_f32(ptr nocapture %varray, ptr nocapture readonly %exp) {
 entry:
   br label %for.body
 
@@ -145,23 +113,16 @@ for.body:                                         ; preds = %for.body, %entry
   store float %tmp2, ptr %arrayidx2, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 1000
-  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !41
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
 
 for.end:                                          ; preds = %for.body
   ret void
 }
 
-!41 = distinct !{!41, !42, !43}
-!42 = !{!"llvm.loop.vectorize.width", i32 4}
-!43 = !{!"llvm.loop.vectorize.enable", i1 true}
-
-
-declare double @__pow_finite(double, double) #0
-
+define void @pow_f64(ptr nocapture %varray, ptr nocapture readonly %exp) {
 ; CHECK-LABEL: @pow_f64
 ; CHECK: <4 x double> @amd_vrd4_pow
 ; CHECK: ret
-define void @pow_f64(ptr nocapture %varray, ptr nocapture readonly %exp) {
 entry:
   br label %for.body
 
@@ -176,18 +137,12 @@ for.body:                                         ; preds = %for.body, %entry
   store double %tmp2, ptr %arrayidx2, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 1000
-  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !51
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
 
 for.end:                                          ; preds = %for.body
   ret void
 }
 
-!51 = distinct !{!51, !52, !53}
-!52 = !{!"llvm.loop.vectorize.width", i32 4}
-!53 = !{!"llvm.loop.vectorize.enable", i1 true}
-
-declare float @__exp2f_finite(float) #0
-
 define void @exp2f_finite(ptr nocapture %varray) {
 ; CHECK-LABEL: @exp2f_finite(
 ; CHECK:    call <4 x float> @amd_vrs4_exp2f(<4 x float> %{{.*}})
@@ -205,18 +160,12 @@ for.body:
   store float %call, ptr %arrayidx, align 4
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond = icmp eq i64 %iv.next, 1000
-  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !61
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
 
 for.end:
   ret void
 }
 
-!61 = distinct !{!61, !62, !63}
-!62 = !{!"llvm.loop.vectorize.width", i32 4}
-!63 = !{!"llvm.loop.vectorize.enable", i1 true}
-
-declare double @__exp2_finite(double) #0
-
 define void @exp2_finite(ptr nocapture %varray) {
 ; CHECK-LABEL: @exp2_finite(
 ; CHECK:    call <4 x double> @amd_vrd4_exp2(<4 x double> {{.*}})
@@ -234,22 +183,16 @@ for.body:
   store double %call, ptr %arrayidx, align 4
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond = icmp eq i64 %iv.next, 1000
-  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !71
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
 
 for.end:
   ret void
 }
 
-!71 = distinct !{!71, !72, !73}
-!72 = !{!"llvm.loop.vectorize.width", i32 4}
-!73 = !{!"llvm.loop.vectorize.enable", i1 true}
-
-declare float @__log2f_finite(float) #0
-
+define void @log2_f32(ptr nocapture %varray) {
 ; CHECK-LABEL: @log2_f32
 ; CHECK: <4 x float> @amd_vrs4_log2f
 ; CHECK: ret
-define void @log2_f32(ptr nocapture %varray) {
 entry:
   br label %for.body
 
@@ -262,23 +205,16 @@ for.body:                                         ; preds = %for.body, %entry
   store float %call, ptr %arrayidx, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 1000
-  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !21
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
 
 for.end:                                          ; preds = %for.body
   ret void
 }
 
-!81 = distinct !{!21, !22, !23}
-!82 = !{!"llvm.loop.vectorize.width", i32 4}
-!83 = !{!"llvm.loop.vectorize.enable", i1 true}
-
-
-declare double @__log2_finite(double) #0
-
+define void @log2_f64(ptr nocapture %varray) {
 ; CHECK-LABEL: @log2_f64
 ; CHECK: <4 x double> @amd_vrd4_log2
 ; CHECK: ret
-define void @log2_f64(ptr nocapture %varray) {
 entry:
   br label %for.body
 
@@ -291,22 +227,16 @@ for.body:                                         ; preds = %for.body, %entry
   store double %call, ptr %arrayidx, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 1000
-  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !31
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
 
 for.end:                                          ; preds = %for.body
   ret void
 }
 
-!91 = distinct !{!31, !32, !33}
-!92 = !{!"llvm.loop.vectorize.width", i32 4}
-!93 = !{!"llvm.loop.vectorize.enable", i1 true}
-
-declare float @__log10f_finite(float) #0
-
+define void @log10_f32(ptr nocapture %varray) {
 ; CHECK-LABEL: @log10_f32
 ; CHECK: <4 x float> @amd_vrs4_log10f
 ; CHECK: ret
-define void @log10_f32(ptr nocapture %varray) {
 entry:
   br label %for.body
 
@@ -319,14 +249,173 @@ for.body:                                         ; preds = %for.body, %entry
   store float %call, ptr %arrayidx, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 1000
-  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !21
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @log10_finite(ptr nocapture %varray) {
+; CHECK-LABEL: @log10_finite(
+; CHECK:    call <2 x double> @amd_vrd2_log10(<2 x double> {{.*}})
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @__log10_finite(double %conv)
+  %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv
+  store double %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1
+
+for.end:
+  ret void
+}
+
+define void @exp10_finite(ptr nocapture %varray) {
+; CHECK-LABEL: @exp10_finite(
+; CHECK:    call <2 x double> @amd_vrd2_exp10(<2 x double> {{.*}})
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @__exp10_finite(double %conv)
+  %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv
+  store double %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1
+
+for.end:
+  ret void
+}
+
+define void @exp10_f32(ptr nocapture %varray) {
+; CHECK-LABEL: @exp10_f32
+; CHECK: <4 x float> @amd_vrs4_exp10f
+; CHECK: ret
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %tmp = trunc i64 %indvars.iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call fast float @__exp10f_finite(float %conv)
+  %arrayidx = getelementptr inbounds float, ptr %varray, i64 %indvars.iv
+  store float %call, ptr %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
 
 for.end:                                          ; preds = %for.body
   ret void
 }
 
-!101 = distinct !{!21, !22, !23}
-!102 = !{!"llvm.loop.vectorize.width", i32 4}
-!103 = !{!"llvm.loop.vectorize.enable", i1 true}
+define void @asin_finite(ptr nocapture %varray) {
+; CHECK-LABEL: @asin_finite(
+; CHECK:    call <8 x double> @amd_vrd8_asin(<8 x double> {{.*}})
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
 
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @__asin_finite(double %conv)
+  %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv
+  store double %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !7
+
+for.end:
+  ret void
+}
 
+define void @asinf_finite(ptr nocapture %varray) {
+; CHECK-LABEL: @asinf_finite
+; CHECK: <4 x float> @amd_vrs4_asinf
+; CHECK: ret
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %tmp = trunc i64 %indvars.iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call fast float @__asinf_finite(float %conv)
+  %arrayidx = getelementptr inbounds float, ptr %varray, i64 %indvars.iv
+  store float %call, ptr %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @acosf_finite(ptr nocapture %varray) {
+; CHECK-LABEL: @acosf_finite
+; CHECK: <4 x float> @amd_vrs4_acosf
+; CHECK: ret
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %tmp = trunc i64 %indvars.iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call fast float @__acosf_finite(float %conv)
+  %arrayidx = getelementptr inbounds float, ptr %varray, i64 %indvars.iv
+  store float %call, ptr %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+!1 = distinct !{!1, !2, !3}
+!2 = !{!"llvm.loop.vectorize.width", i32 2}
+!3 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+!4 = distinct !{!4, !5, !6}
+!5 = !{!"llvm.loop.vectorize.width", i32 4}
+!6 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+!7 = distinct !{!7, !8, !9}
+!8 = !{!"llvm.loop.vectorize.width", i32 8}
+!9 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+declare float @__expf_finite(float) #0
+declare double @__exp_finite(double) #0
+declare double @__log_finite(double) #0
+declare float @__logf_finite(float) #0
+declare float @__powf_finite(float, float) #0
+declare double @__pow_finite(double, double) #0
+declare float @__exp2f_finite(float) #0
+declare double @__exp2_finite(double) #0
+declare float @__log2f_finite(float) #0
+declare double @__log2_finite(double) #0
+declare float @__log10f_finite(float) #0
+declare double @__log10_finite(double) #0
+declare double @__exp10_finite(double) #0
+declare float @__exp10f_finite(float) #0
+declare double @__asin_finite(double) #0
+declare float @__asinf_finite(float) #0
+declare float @__acosf_finite(float) #0
\ No newline at end of file
diff --git a/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls.ll b/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls.ll
index 4acc7fe7eacc..4ced0372e5da 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls.ll
@@ -1444,6 +1444,32 @@ for.end:
   ret void
 }
 
+define void @log10_f64(ptr nocapture %varray) {
+; CHECK-LABEL: @log10_f64(
+; CHECK-VF2:    [[TMP5:%.*]] = call <2 x double> @amd_vrd2_log10(<2 x double> [[TMP4:%.*]])
+; CHECK-VF4:    [[TMP5:%.*]] = call <4 x double> @llvm.log10.v4f64(<4 x double> [[TMP4:%.*]])
+; CHECK-VF8:    [[TMP5:%.*]] = call <8 x double> @llvm.log10.v8f64(<8 x double> [[TMP4:%.*]])
+; CHECK-VF16:   [[TMP5:%.*]] = call <16 x double> @llvm.log10.v16f64(<16 x double> [[TMP4:%.*]])
+; CHECK:        ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @log10(double %conv)
+  %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv
+  store double %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
 define void @log10_f32(ptr nocapture %varray) {
 ; CHECK-LABEL: @log10_f32(
 ; CHECK-VF2:    [[TMP5:%.*]] = call <2 x float> @llvm.log10.v2f32(<2 x float> [[TMP4:%.*]])
@@ -1470,6 +1496,32 @@ for.end:
   ret void
 }
 
+define void @log10_f64_intrinsic(ptr nocapture %varray) {
+; CHECK-LABEL: @log10_f64_intrinsic(
+; CHECK-VF2:    [[TMP5:%.*]] = call <2 x double> @amd_vrd2_log10(<2 x double> [[TMP4:%.*]])
+; CHECK-VF4:    [[TMP5:%.*]] = call <4 x double> @llvm.log10.v4f64(<4 x double> [[TMP4:%.*]])
+; CHECK-VF8:    [[TMP5:%.*]] = call <8 x double> @llvm.log10.v8f64(<8 x double> [[TMP4:%.*]])
+; CHECK-VF16:   [[TMP5:%.*]] = call <16 x double> @llvm.log10.v16f64(<16 x double> [[TMP4:%.*]])
+; CHECK:        ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.log10.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv
+  store double %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
 define void @log10_f32_intrinsic(ptr nocapture %varray) {
 ; CHECK-LABEL: @log10_f32_intrinsic(
 ; CHECK-VF2:    [[TMP5:%.*]] = call <2 x float> @llvm.log10.v2f32(<2 x float> [[TMP4:%.*]])
@@ -1600,4 +1652,168 @@ for.end:
   ret void
 }
 
+define void @exp10_f64(ptr nocapture %varray) {
+; CHECK-LABEL: @exp10_f64(
+; CHECK-VF2:    [[TMP5:%.*]] = call <2 x double> @amd_vrd2_exp10(<2 x double> [[TMP4:%.*]])
+; CHECK-VF4:    call <4 x double> @llvm.exp10.v4f64(<4 x double> [[TMP4:%.*]])
+; CHECK-VF8:    call <8 x double> @llvm.exp10.v8f64(<8 x double> [[TMP4:%.*]])
+; CHECK-VF16:    call <16 x double> @llvm.exp10.v16f64(<16 x double> [[TMP4:%.*]])
+; CHECK:        ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @exp10(double %conv)
+  %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv
+  store double %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @exp10_f32(ptr nocapture %varray) {
+; CHECK-LABEL: @exp10_f32(
+; CHECK-VF2:    [[TMP5:%.*]] = call <2 x float> @llvm.exp10.v2f32(<2 x float> [[TMP4:%.*]])
+; CHECK-VF4:    [[TMP5:%.*]] = call <4 x float> @amd_vrs4_exp10f(<4 x float> [[TMP4:%.*]])
+; CHECK-VF8:    [[TMP5:%.*]] = call <8 x float> @llvm.exp10.v8f32(<8 x float> [[TMP4:%.*]])
+; CHECK-VF16:   [[TMP5:%.*]] = call <16 x float> @llvm.exp10.v16f32(<16 x float> [[TMP4:%.*]])
+; CHECK:        ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @exp10f(float %conv)
+  %arrayidx = getelementptr inbounds float, ptr %varray, i64 %iv
+  store float %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @exp10_f64_intrinsic(ptr nocapture %varray) {
+; CHECK-LABEL: @exp10_f64_intrinsic(
+; CHECK-VF2:    [[TMP5:%.*]] = call <2 x double> @amd_vrd2_exp10(<2 x double> [[TMP4:%.*]])
+; CHECK-VF4:    [[TMP5:%.*]] = call <4 x double> @llvm.exp10.v4f64(<4 x double> [[TMP4:%.*]])
+; CHECK-VF8:    [[TMP5:%.*]] = call <8 x double> @llvm.exp10.v8f64(<8 x double> [[TMP4:%.*]])
+; CHECK-VF16:    [[TMP5:%.*]] = call <16 x double> @llvm.exp10.v16f64(<16 x double> [[TMP4:%.*]])
+; CHECK:        ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.exp10.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv
+  store double %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @exp10_f32_intrinsic(ptr nocapture %varray) {
+; CHECK-LABEL: @exp10_f32_intrinsic(
+; CHECK-VF2:    [[TMP5:%.*]] = call <2 x float> @llvm.exp10.v2f32(<2 x float> [[TMP4:%.*]])
+; CHECK-VF4:    [[TMP5:%.*]] = call <4 x float> @amd_vrs4_exp10f(<4 x float> [[TMP4:%.*]])
+; CHECK-VF8:    [[TMP5:%.*]] = call <8 x float> @llvm.exp10.v8f32(<8 x float> [[TMP4:%.*]])
+; CHECK-VF16:   [[TMP5:%.*]] = call <16 x float> @llvm.exp10.v16f32(<16 x float> [[TMP4:%.*]])
+; CHECK:        ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.exp10.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, ptr %varray, i64 %iv
+  store float %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+
+define void @sincos_f64(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: define void @sincos_f64
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]])
+; CHECK-VF2-NOT:    call void @amd_vrd2_sincos(<2 x double> [[WIDE_LOAD:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]])
+; CHECK-VF4-NOT:    call void @amd_vrd4_sincos(<4 x double> [[WIDE_LOAD:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]])
+; CHECK-VF8-NOT:    call void @amd_vrd8_sincos(<8 x double> [[WIDE_LOAD:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]])
+; CHECK:        ret void
+; 
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %gepa = getelementptr double, ptr %a, i64 %indvars.iv
+  %num = load double, ptr %gepa, align 8
+  %gepb = getelementptr double, ptr %b, i64 %indvars.iv
+  %gepc = getelementptr double, ptr %c, i64 %indvars.iv
+  call void @sincos(double %num, ptr %gepb, ptr %gepc)
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+define void @sincos_f32(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: define void @sincos_f32
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]])
+; CHECK-VF4-NOT:    call void @amd_vrs4_sincosf(<4 x float> [[WIDE_LOAD:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]])
+; CHECK-VF8-NOT:    call void @amd_vrs8_sincosf(<8 x float> [[WIDE_LOAD:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]])
+; CHECK-VF16-NOT:    call void @amd_vrs16_sincosf(<16 x float> [[WIDE_LOAD:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]])
+; CHECK:        ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %gepa = getelementptr float, ptr %a, i64 %indvars.iv
+  %num = load float, ptr %gepa, align 8
+  %gepb = getelementptr float, ptr %b, i64 %indvars.iv
+  %gepc = getelementptr float, ptr %c, i64 %indvars.iv
+  call void @sincosf(float %num, ptr %gepb, ptr %gepc)
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
 attributes #0 = { nounwind readnone }
+
+declare double @exp10(double) #0
+declare float @exp10f(float) #0
+declare double @llvm.exp10.f64(double) #0
+declare float @llvm.exp10.f32(float) #0
+declare void @sincos(double, ptr, ptr)
+declare void @sincosf(float, ptr, ptr)
diff --git a/llvm/test/Transforms/Util/add-TLI-mappings.ll b/llvm/test/Transforms/Util/add-TLI-mappings.ll
index 4e4b81e89a32..76cccbd6f39c 100644
--- a/llvm/test/Transforms/Util/add-TLI-mappings.ll
+++ b/llvm/test/Transforms/Util/add-TLI-mappings.ll
@@ -14,10 +14,15 @@
 ; SVML-SAME:          ptr @__svml_log10f4,
 ; SVML-SAME:          ptr @__svml_log10f8,
 ; SVML-SAME:          ptr @__svml_log10f16
-; AMDLIBM-SAME:     [6 x ptr] [
+; AMDLIBM-SAME:     [11 x ptr] [
 ; AMDLIBM-SAME:       ptr @amd_vrd2_sin,
 ; AMDLIBM-SAME:       ptr @amd_vrd4_sin,
 ; AMDLIBM-SAME:       ptr @amd_vrd8_sin,
+; AMDLIBM-SAME:       ptr @amd_vrd4_sincos,
+; AMDLIBM-SAME:       ptr @amd_vrd8_sincos,
+; AMDLIBM-SAME:       ptr @amd_vrs4_sincosf,
+; AMDLIBM-SAME:       ptr @amd_vrs8_sincosf,
+; AMDLIBM-SAME:       ptr @amd_vrs16_sincosf
 ; AMDLIBM-SAME:       ptr @amd_vrs4_log10f,
 ; AMDLIBM-SAME:       ptr @amd_vrs8_log10f,
 ; AMDLIBM-SAME:       ptr @amd_vrs16_log10f
@@ -106,6 +111,7 @@ define void @sincos_f64(double %in, ptr %sin, ptr %cos) {
 ; COMMON-LABEL: @sincos_f64(
 ; SLEEFGNUABI:  call void @sincos(double %{{.*}}, ptr %{{.*}}, ptr %{{.*}}) #[[SINCOS:[0-9]+]]
 ; ARMPL:        call void @sincos(double %{{.*}}, ptr %{{.*}}, ptr %{{.*}}) #[[SINCOS:[0-9]+]]
+; AMDLIBM:        call void @sincos(double %{{.*}}, ptr %{{.*}}, ptr %{{.*}}) #[[SINCOS:[0-9]+]]
   call void @sincos(double %in, ptr %sin, ptr %cos)
   ret void
 }
@@ -116,6 +122,7 @@ define void @sincos_f32(float %in, ptr %sin, ptr %cos) {
 ; COMMON-LABEL: @sincos_f32(
 ; SLEEFGNUABI:  call void @sincosf(float %{{.*}}, ptr %{{.*}}, ptr %{{.*}}) #[[SINCOSF:[0-9]+]]
 ; ARMPL:        call void @sincosf(float %{{.*}}, ptr %{{.*}}, ptr %{{.*}}) #[[SINCOSF:[0-9]+]]
+; AMDLIBM:        call void @sincosf(float %{{.*}}, ptr %{{.*}}, ptr %{{.*}}) #[[SINCOSF:[0-9]+]]
   call void @sincosf(float %in, ptr %sin, ptr %cos)
   ret void
 }
@@ -145,7 +152,7 @@ declare void @sincospif(float, ptr, ptr) #0
 define float @call_llvm.log10.f32(float %in) {
 ; COMMON-LABEL: @call_llvm.log10.f32(
 ; SVML:         call float @llvm.log10.f32(float %{{.*}})
-; AMDLIBM:      call float @llvm.log10.f32(float %{{.*}})
+; AMDLIBM:      call float @llvm.log10.f32(float %{{.*}}) #[[LOG10:[0-9]+]]
 ; LIBMVEC-X86:  call float @llvm.log10.f32(float %{{.*}})
 ; MASSV:        call float @llvm.log10.f32(float %{{.*}}) #[[LOG10:[0-9]+]]
 ; ACCELERATE:   call float @llvm.log10.f32(float %{{.*}}) #[[LOG10:[0-9]+]]
@@ -171,6 +178,11 @@ declare float @llvm.log10.f32(float) #0
 ; AMDLIBM: declare <2 x double> @amd_vrd2_sin(<2 x double>)
 ; AMDLIBM: declare <4 x double> @amd_vrd4_sin(<4 x double>)
 ; AMDLIBM: declare <8 x double> @amd_vrd8_sin(<8 x double>)
+; AMDLIBM: declare void @amd_vrd4_sincos(<4 x double>, ptr, ptr)
+; AMDLIBM: declare void @amd_vrd8_sincos(<8 x double>, ptr, ptr)
+; AMDLIBM: declare void @amd_vrs4_sincosf(<4 x float>, ptr, ptr)
+; AMDLIBM: declare void @amd_vrs8_sincosf(<8 x float>, ptr, ptr)
+; AMDLIBM: declare void @amd_vrs16_sincosf(<16 x float>, ptr, ptr)
 ; AMDLIBM: declare <4 x float> @amd_vrs4_log10f(<4 x float>)
 ; AMDLIBM: declare <8 x float> @amd_vrs8_log10f(<8 x float>)
 ; AMDLIBM: declare <16 x float> @amd_vrs16_log10f(<16 x float>)
@@ -228,6 +240,17 @@ attributes #0 = { nounwind readnone }
 ; AMDLIBM-SAME:   "_ZGV_LLVM_N2v_sin(amd_vrd2_sin),
 ; AMDLIBM-SAME:   _ZGV_LLVM_N4v_sin(amd_vrd4_sin),
 ; AMDLIBM-SAME:   _ZGV_LLVM_N8v_sin(amd_vrd8_sin)" }
+; AMDLIBM:      attributes #[[SINCOS]] = { "vector-function-abi-variant"=
+; AMDLIBM-SAME:   "_ZGV_LLVM_N4vl8l8_sincos(amd_vrd4_sincos),
+; AMDLIBM-SAME:   _ZGV_LLVM_N8vl8l8_sincos(amd_vrd8_sincos)" }
+; AMDLIBM:      attributes #[[SINCOSF]] = { "vector-function-abi-variant"=
+; AMDLIBM-SAME:   "_ZGV_LLVM_N4vl4l4_sincosf(amd_vrs4_sincosf),
+; AMDLIBM-SAME:   _ZGV_LLVM_N8vl4l4_sincosf(amd_vrs8_sincosf),
+; AMDLIBM-SAME:   _ZGV_LLVM_N16vl4l4_sincosf(amd_vrs16_sincosf)" }
+; AMDLIBM:      attributes #[[LOG10]] = { "vector-function-abi-variant"=
+; AMDLIBM-SAME:   "_ZGV_LLVM_N4v_llvm.log10.f32(amd_vrs4_log10f),
+; AMDLIBM-SAME:   _ZGV_LLVM_N8v_llvm.log10.f32(amd_vrs8_log10f),
+; AMDLIBM-SAME:   _ZGV_LLVM_N16v_llvm.log10.f32(amd_vrs16_log10f)" }
 
 ; MASSV:      attributes #[[SIN]] = { "vector-function-abi-variant"=
 ; MASSV-SAME:   "_ZGV_LLVM_N2v_sin(__sind2)" }
-- 
GitLab


From a393c92f5df141d464bb17cc82f2344866cea1de Mon Sep 17 00:00:00 2001
From: Edd Dawson <edd.dawson@sony.com>
Date: Tue, 29 Oct 2024 10:16:58 +0000
Subject: [PATCH 410/693] [PS5][Driver] Update default linking options when
 `-r` omitted. (#113595)

Until now, these options have been hardcoded as downstream patches in
lld. Add them to the driver so that the private patches can be removed.

PS5 only. On PS4, the proprietary linker will continue to perform the
equivalent behaviours itself.

SIE tracker: TOOLCHAIN-16704
---
 clang/lib/Driver/ToolChains/PS4CPU.cpp | 14 +++++++++
 clang/test/Driver/ps5-linker.c         | 41 ++++++++++++++++----------
 2 files changed, 40 insertions(+), 15 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/PS4CPU.cpp b/clang/lib/Driver/ToolChains/PS4CPU.cpp
index a50333223ff5..719bba41436a 100644
--- a/clang/lib/Driver/ToolChains/PS4CPU.cpp
+++ b/clang/lib/Driver/ToolChains/PS4CPU.cpp
@@ -250,6 +250,20 @@ void tools::PS5cpu::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-pie");
 
   if (!Relocatable) {
+    CmdArgs.push_back("--eh-frame-hdr");
+    CmdArgs.push_back("--hash-style=sysv");
+
+    // Add a build-id by default to allow the PlayStation symbol server to
+    // index the symbols. `uuid` is the cheapest fool-proof method.
+    // (The non-determinism and alternative methods are noted in the downstream
+    // PlayStation docs).
+    CmdArgs.push_back("--build-id=uuid");
+
+    // All references are expected to be resolved at static link time for both
+    // executables and dynamic libraries. This has been the default linking
+    // behaviour for numerous PlayStation generations.
+    CmdArgs.push_back("--unresolved-symbols=report-all");
+
     // Lazy binding of PLTs is not supported on PlayStation. They are placed in
     // the RelRo segment.
     CmdArgs.push_back("-z");
diff --git a/clang/test/Driver/ps5-linker.c b/clang/test/Driver/ps5-linker.c
index d18309a65072..2080f4dc91a7 100644
--- a/clang/test/Driver/ps5-linker.c
+++ b/clang/test/Driver/ps5-linker.c
@@ -14,21 +14,32 @@
 // CHECK-NO-PIE-NOT: "-pie"
 // CHECK-SHARED: "--shared"
 
-// Test the driver passes PlayStation-specific -z options to the linker.
-
-// RUN: %clang --target=x86_64-sie-ps5 %s -### 2>&1 | FileCheck --check-prefixes=CHECK-Z %s
-
-// CHECK-Z: {{ld(\.exe)?}}"
-// CHECK-Z-SAME: "-z" "now"
-// CHECK-Z-SAME: "-z" "start-stop-visibility=hidden"
-// CHECK-Z-SAME: "-z" "dead-reloc-in-nonalloc=.debug_*=0xffffffffffffffff"
-// CHECK-Z-SAME: "-z" "dead-reloc-in-nonalloc=.debug_ranges=0xfffffffffffffffe"
-// CHECK-Z-SAME: "-z" "dead-reloc-in-nonalloc=.debug_loc=0xfffffffffffffffe"
-
-// RUN: %clang --target=x86_64-sie-ps5 -r %s -### 2>&1 | FileCheck --check-prefixes=CHECK-NO-Z %s
-
-// CHECK-NO-Z: {{ld(\.exe)?}}"
-// CHECK-NO-Z-NOT: "-z"
+// Test the driver passes PlayStation-specific options to the linker that are
+// appropriate for the type of output. Many options don't apply for relocatable
+// output (-r).
+
+// RUN: %clang --target=x86_64-sie-ps5 %s -### 2>&1 | FileCheck --check-prefixes=CHECK-EXE %s
+// RUN: %clang --target=x86_64-sie-ps5 %s -shared -### 2>&1 | FileCheck --check-prefixes=CHECK-EXE %s
+// RUN: %clang --target=x86_64-sie-ps5 %s -static -### 2>&1 | FileCheck --check-prefixes=CHECK-EXE %s
+// RUN: %clang --target=x86_64-sie-ps5 %s -r -### 2>&1 | FileCheck --check-prefixes=CHECK-NO-EXE %s
+
+// CHECK-EXE: {{ld(\.exe)?}}"
+// CHECK-EXE-SAME: "--eh-frame-hdr"
+// CHECK-EXE-SAME: "--hash-style=sysv"
+// CHECK-EXE-SAME: "--build-id=uuid"
+// CHECK-EXE-SAME: "--unresolved-symbols=report-all"
+// CHECK-EXE-SAME: "-z" "now"
+// CHECK-EXE-SAME: "-z" "start-stop-visibility=hidden"
+// CHECK-EXE-SAME: "-z" "dead-reloc-in-nonalloc=.debug_*=0xffffffffffffffff"
+// CHECK-EXE-SAME: "-z" "dead-reloc-in-nonalloc=.debug_ranges=0xfffffffffffffffe"
+// CHECK-EXE-SAME: "-z" "dead-reloc-in-nonalloc=.debug_loc=0xfffffffffffffffe"
+
+// CHECK-NO-EXE: {{ld(\.exe)?}}"
+// CHECK-NO-EXE-NOT: "--eh-frame-hdr"
+// CHECK-NO-EXE-NOT: "--hash-style
+// CHECK-NO-EXE-NOT: "--build-id
+// CHECK-NO-EXE-NOT: "--unresolved-symbols
+// CHECK-NO-EXE-NOT: "-z"
 
 // Test that -static is forwarded to the linker
 
-- 
GitLab


From a8398bd81770a2801ec083fd2cd8a19140fe92a9 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Tue, 29 Oct 2024 10:33:17 +0000
Subject: [PATCH 411/693] [llvm][docs] Update list of llvm-lit options

Fixes #62899

In this commit I have updated the list of options
to include any missing options and re-rordered
some of them to match the order in lit's --help.

Where there was a larger description in this document
I've used that instead of the --help description.

This *does not* include --use-unique-output-file-name
as this was only added recently and we are still
debating whether it will be kept.
---
 llvm/docs/CommandGuide/lit.rst | 162 +++++++++++++++++++++++----------
 1 file changed, 113 insertions(+), 49 deletions(-)

diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index bd1cfbbade51..af8a1a08be53 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -57,7 +57,11 @@ GENERAL OPTIONS
 
 .. option:: -h, --help
 
- Show the :program:`lit` help message.
+ Show the :program:`lit` help message and exit.
+
+.. option:: --version
+
+ Show :program:`lit`'s version number and exit.
 
 .. option:: -j N, --workers=N
 
@@ -108,23 +112,51 @@ OUTPUT OPTIONS
 
  Enable -v, but for all tests not just failed tests.
 
+.. option:: -o PATH, --output PATH
+
+ Write test results to the provided path.
+
 .. option:: --no-progress-bar
 
  Do not use curses based progress bar.
 
+.. option:: --show-excluded
+
+ Show excluded tests.
+
+.. option:: --show-skipped
+
+ Show skipped tests.
+
 .. option:: --show-unsupported
 
- Show the names of unsupported tests.
+ Show unsupported tests.
+
+.. option:: --show-pass
+
+ Show passed tests.
+
+.. option:: --show-flakypass
+
+ Show passed with retry tests.
 
 .. option:: --show-xfail
 
- Show the names of tests that were expected to fail.
+ Show expectedly failed tests.
 
 .. _execution-options:
 
 EXECUTION OPTIONS
 -----------------
 
+.. option:: --gtest-sharding
+
+ Enable sharding for GoogleTest format.
+
+.. option:: --no-gtest-sharding
+
+ Disable sharding for GoogleTest format.
+
 .. option:: --path=PATH
 
  Specify an additional ``PATH`` to use when searching for executables in tests.
@@ -139,11 +171,6 @@ EXECUTION OPTIONS
  "``valgrind``" feature that can be used to conditionally disable (or expect
  failure in) certain tests.
 
-.. option:: --vg-arg=ARG
-
- When :option:`--vg` is used, specify an additional argument to pass to
- :program:`valgrind` itself.
-
 .. option:: --vg-leak
 
  When :option:`--vg` is used, enable memory leak checks.  When this option is
@@ -151,9 +178,59 @@ EXECUTION OPTIONS
  feature that can be used to conditionally disable (or expect failure in)
  certain tests.
 
+.. option:: --vg-arg=ARG
+
+ When :option:`--vg` is used, specify an additional argument to pass to
+ :program:`valgrind` itself.
+
+.. option:: --no-execute
+
+ Don't execute any tests (assume that they pass).
+
+.. option:: --xunit-xml-output XUNIT_XML_OUTPUT
+
+ Write XUnit-compatible XML test reports to the specified file.
+
+.. option:: --resultdb-output RESULTDB_OUTPUT
+
+ Write LuCI ResultDB compatible JSON to the specified file.
+
+.. option:: --time-trace-output TIME_TRACE_OUTPUT
+
+ Write Chrome tracing compatible JSON to the specified file
+
+.. option:: --timeout MAXINDIVIDUALTESTTIME
+
+ Maximum time to spend running a single test (in seconds). 0 means no time
+ limit. [Default: 0]
+
+.. option:: --timeout=N
+
+ Spend at most ``N`` seconds (approximately) running each individual test.
+ ``0`` means no time limit, and ``0`` is the default. Note that this is not an
+ alias for :option:`--max-time`; the two are different kinds of maximums.
+
+.. option:: --max-failures MAX_FAILURES
+
+ Stop execution after the given number of failures.
+
+.. option:: --allow-empty-runs
+
+ Do not fail the run if all tests are filtered out.
+
+.. option:: --per-test-coverage
+
+ Emit the necessary test coverage data, divided per test case (involves
+ setting a unique value to LLVM_PROFILE_FILE for each RUN). The coverage
+ data files will be emitted in the directory specified by ``config.test_exec_root``.
+
+.. option:: --ignore-fail
+
+ Exit with status zero even if some tests fail.
+
 .. option:: --skip-test-time-recording
 
- Disable tracking the wall time individual tests take to execute.
+ Do not track elapsed wall time for each test.
 
 .. option:: --time-tests
 
@@ -161,10 +238,6 @@ EXECUTION OPTIONS
  in the summary output.  This is useful for determining which tests in a test
  suite take the most time to execute.
 
-.. option:: --ignore-fail
-
- Exit with status zero even if some tests fail.
-
 .. _selection-options:
 
 SELECTION OPTIONS
@@ -178,23 +251,6 @@ The timing data is stored in the `test_exec_root` in a file named
 `.lit_test_times.txt`. If this file does not exist, then `lit` checks the
 `test_source_root` for the file to optionally accelerate clean builds.
 
-.. option:: --shuffle
-
- Run the tests in a random order, not failing/slowest first. Deprecated,
- use :option:`--order` instead.
-
-.. option:: --per-test-coverage
-
- Emit the necessary test coverage data, divided per test case (involves
- setting a unique value to LLVM_PROFILE_FILE for each RUN). The coverage
- data files will be emitted in the directory specified by `config.test_exec_root`.
-
-.. option:: --max-failures N
-
- Stop execution after the given number ``N`` of failures.
- An integer argument should be passed on the command line
- prior to execution.
-
 .. option:: --max-tests=N
 
  Run at most ``N`` tests and then terminate.
@@ -205,16 +261,6 @@ The timing data is stored in the `test_exec_root` in a file named
  Note that this is not an alias for :option:`--timeout`; the two are
  different kinds of maximums.
 
-.. option:: --num-shards=M
-
- Divide the set of selected tests into ``M`` equal-sized subsets or
- "shards", and run only one of them.  Must be used with the
- ``--run-shard=N`` option, which selects the shard to run. The environment
- variable ``LIT_NUM_SHARDS`` can also be used in place of this
- option. These two options provide a coarse mechanism for partitioning large
- testsuites, for parallel execution on separate machines (say in a large
- testing farm).
-
 .. option:: --order={lexical,random,smart}
 
  Define the order in which tests are run. The supported values are:
@@ -228,18 +274,14 @@ The timing data is stored in the `test_exec_root` in a file named
    tests, all in descending execution time order. This is the default as it
    optimizes concurrency.
 
-.. option:: --run-shard=N
+.. option:: --shuffle
 
- Select which shard to run, assuming the ``--num-shards=M`` option was
- provided. The two options must be used together, and the value of ``N``
- must be in the range ``1..M``. The environment variable
- ``LIT_RUN_SHARD`` can also be used in place of this option.
+ Run the tests in a random order, not failing/slowest first. Deprecated,
+ use :option:`--order` instead.
 
-.. option:: --timeout=N
+.. option:: -i, --incremental
 
- Spend at most ``N`` seconds (approximately) running each individual test.
- ``0`` means no time limit, and ``0`` is the default. Note that this is not an
- alias for :option:`--max-time`; the two are different kinds of maximums.
+ Run failed tests first (DEPRECATED: use ``--order=smart``).
 
 .. option:: --filter=REGEXP
 
@@ -297,6 +339,23 @@ The timing data is stored in the `test_exec_root` in a file named
   primary purpose is to suppress an ``XPASS`` result without modifying a test
   case that uses the ``XFAIL`` directive.
 
+.. option:: --num-shards=M
+
+ Divide the set of selected tests into ``M`` equal-sized subsets or
+ "shards", and run only one of them.  Must be used with the
+ ``--run-shard=N`` option, which selects the shard to run. The environment
+ variable ``LIT_NUM_SHARDS`` can also be used in place of this
+ option. These two options provide a coarse mechanism for partitioning large
+ testsuites, for parallel execution on separate machines (say in a large
+ testing farm).
+
+.. option:: --run-shard=N
+
+ Select which shard to run, assuming the ``--num-shards=M`` option was
+ provided. The two options must be used together, and the value of ``N``
+ must be in the range ``1..M``. The environment variable
+ ``LIT_RUN_SHARD`` can also be used in place of this option.
+
 ADDITIONAL OPTIONS
 ------------------
 
@@ -313,6 +372,11 @@ ADDITIONAL OPTIONS
 
  List all of the discovered tests and exit.
 
+.. option:: --show-used-features
+
+ Show all features used in the test suite (in ``XFAIL``, ``UNSUPPORTED`` and
+ ``REQUIRES``) and exit.
+
 EXIT STATUS
 -----------
 
-- 
GitLab


From 7395ef5419a6438f0c48685bf00b7f151178743d Mon Sep 17 00:00:00 2001
From: wldfngrs <wldfngrs@gmail.com>
Date: Tue, 29 Oct 2024 11:39:57 +0100
Subject: [PATCH 412/693] [libc][math][c23] Add cospif16 function (#113001)

Implementation of `cos` for half precision floating point inputs scaled
by pi (i.e., `cospi`), correctly rounded for all rounding modes.

---------

Co-authored-by: OverMighty <its.overmighty@gmail.com>
---
 libc/config/linux/aarch64/entrypoints.txt  |  1 +
 libc/config/linux/x86_64/entrypoints.txt   |  1 +
 libc/docs/math/index.rst                   |  2 +-
 libc/newhdrgen/yaml/math.yaml              |  7 ++
 libc/src/math/CMakeLists.txt               |  1 +
 libc/src/math/cospif16.h                   | 21 ++++++
 libc/src/math/generic/CMakeLists.txt       | 40 +++++++++--
 libc/src/math/generic/cospif16.cpp         | 81 ++++++++++++++++++++++
 libc/src/math/generic/sincosf16_utils.h    | 77 ++++++++++++++++++++
 libc/src/math/generic/sinpif16.cpp         | 78 +++------------------
 libc/test/src/math/CMakeLists.txt          | 11 +++
 libc/test/src/math/cospif16_test.cpp       | 40 +++++++++++
 libc/test/src/math/smoke/CMakeLists.txt    | 11 +++
 libc/test/src/math/smoke/cospif16_test.cpp | 44 ++++++++++++
 libc/utils/MPFRWrapper/MPFRUtils.cpp       | 20 ++----
 15 files changed, 347 insertions(+), 88 deletions(-)
 create mode 100644 libc/src/math/cospif16.h
 create mode 100644 libc/src/math/generic/cospif16.cpp
 create mode 100644 libc/src/math/generic/sincosf16_utils.h
 create mode 100644 libc/test/src/math/cospif16_test.cpp
 create mode 100644 libc/test/src/math/smoke/cospif16_test.cpp

diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 71c6e874429f..b3f94a581c8a 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -607,6 +607,7 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.canonicalizef16
     libc.src.math.ceilf16
     libc.src.math.copysignf16
+    libc.src.math.cospif16
     # TODO: aarch64 bug
     # Please see https://github.com/llvm/llvm-project/pull/100632#issuecomment-2258772681
     # libc.src.math.expf16
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 9bc63edf06f2..a2fb97d04584 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -611,6 +611,7 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.ceilf16
     libc.src.math.copysignf16
     libc.src.math.coshf16
+    libc.src.math.cospif16
     libc.src.math.exp10f16
     libc.src.math.exp10m1f16
     libc.src.math.exp2f16
diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst
index ce4df92393ce..a50e054622e1 100644
--- a/libc/docs/math/index.rst
+++ b/libc/docs/math/index.rst
@@ -280,7 +280,7 @@ Higher Math Functions
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | cosh      | |check|          |                 |                        | |check|              |                        | 7.12.5.4               | F.10.2.4                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| cospi     | |check|          |                 |                        |                      |                        | 7.12.4.12              | F.10.1.12                  |
+| cospi     | |check|          |                 |                        | |check|              |                        | 7.12.4.12              | F.10.1.12                  |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | dsqrt     | N/A              | N/A             |   |check|              | N/A                  |       |check|\*        | 7.12.14.6              | F.10.11                    |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
diff --git a/libc/newhdrgen/yaml/math.yaml b/libc/newhdrgen/yaml/math.yaml
index fe07803cff06..3cc4b599c777 100644
--- a/libc/newhdrgen/yaml/math.yaml
+++ b/libc/newhdrgen/yaml/math.yaml
@@ -206,6 +206,13 @@ functions:
     return_type: float
     arguments:
       - type: float
+  - name: cospif16
+    standards:
+      - stdc
+    return_type: _Float16
+    arguments:
+      - type: _Float16
+    guard: LIBC_TYPES_HAS_FLOAT16
   - name: coshf16
     standards:
       - stdc
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index cb4817348cbb..80c1867d2116 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -95,6 +95,7 @@ add_math_entrypoint_object(coshf)
 add_math_entrypoint_object(coshf16)
 
 add_math_entrypoint_object(cospif)
+add_math_entrypoint_object(cospif16)
 
 add_math_entrypoint_object(daddl)
 add_math_entrypoint_object(daddf128)
diff --git a/libc/src/math/cospif16.h b/libc/src/math/cospif16.h
new file mode 100644
index 000000000000..ef9625dfed45
--- /dev/null
+++ b/libc/src/math/cospif16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for cospif16 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_COSPIF16_H
+#define LLVM_LIBC_SRC_MATH_COSPIF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+float16 cospif16(float16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_SINPIF16_H
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 35e7347b9136..ca27759d3212 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -351,6 +351,17 @@ add_header_library(
     libc.src.__support.common
 )
 
+add_header_library(
+  sincosf16_utils
+  HDRS
+    sincosf16_utils.h
+  DEPENDS
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.polyeval
+    libc.src.__support.FPUtil.nearest_integer
+    libc.src.__support.common  
+)
+
 add_header_library(
   sincos_eval
   HDRS
@@ -422,6 +433,25 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  cospif16
+  SRCS
+    cospif16.cpp
+  HDRS
+    ../cospif16.h
+  DEPENDS
+    .sincosf16_utils
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.cast
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.macros.optimization
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   sin
   SRCS
@@ -535,14 +565,14 @@ add_entrypoint_object(
   HDRS
     ../sinpif16.h
   DEPENDS
-    libc.src.__support.common
+    .sincosf16_utils
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
     libc.src.__support.FPUtil.cast
     libc.src.__support.FPUtil.fenv_impl
-    libc.src.__support.FPUtil.fp_bits 
+    libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.multiply_add
-    libc.src.__support.FPUtil.nearest_integer
-    libc.src.__support.FPUtil.polyeval
-    libc.src.__support.macros.properties.types
+    libc.src.__support.macros.optimization
   COMPILE_OPTIONS
     -O3
 )
diff --git a/libc/src/math/generic/cospif16.cpp b/libc/src/math/generic/cospif16.cpp
new file mode 100644
index 000000000000..dd8c7ab6afa3
--- /dev/null
+++ b/libc/src/math/generic/cospif16.cpp
@@ -0,0 +1,81 @@
+//===-- Half-precision cospif function ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/cospif16.h"
+#include "hdr/errno_macros.h"
+#include "hdr/fenv_macros.h"
+#include "sincosf16_utils.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/cast.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/macros/optimization.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(float16, cospif16, (float16 x)) {
+  using FPBits = typename fputil::FPBits<float16>;
+  FPBits xbits(x);
+
+  uint16_t x_u = xbits.uintval();
+  uint16_t x_abs = x_u & 0x7fff;
+  float xf = x;
+
+  // Range reduction:
+  // For |x| > 1/32, we perform range reduction as follows:
+  // Find k and y such that:
+  //   x = (k + y) * 1/32
+  //   k is an integer
+  //   |y| < 0.5
+  //
+  // This is done by performing:
+  //   k = round(x * 32)
+  //   y = x * 32 - k
+  //
+  // Once k and y are computed, we then deduce the answer by the sine of sum
+  // formula:
+  //   cos(x * pi) = cos((k + y) * pi/32)
+  //               = cos(k * pi/32) * cos(y * pi/32) +
+  //                 sin(y * pi/32) * sin(k * pi/32)
+
+  // For signed zeros
+  if (LIBC_UNLIKELY(x_abs == 0U))
+    return fputil::cast<float16>(1.0f);
+
+  // Numbers greater or equal to 2^10 are integers, or infinity, or NaN
+  if (LIBC_UNLIKELY(x_abs >= 0x6400)) {
+    if (LIBC_UNLIKELY(x_abs <= 0x67FF))
+      return fputil::cast<float16>((x_abs & 0x1) ? -1.0f : 1.0f);
+
+    // Check for NaN or infintiy values
+    if (LIBC_UNLIKELY(x_abs >= 0x7c00)) {
+      // If value is equal to infinity
+      if (x_abs == 0x7c00) {
+        fputil::set_errno_if_required(EDOM);
+        fputil::raise_except_if_required(FE_INVALID);
+      }
+
+      return x + FPBits::quiet_nan().get_val();
+    }
+
+    return fputil::cast<float16>(1.0f);
+  }
+
+  float sin_k, cos_k, sin_y, cosm1_y;
+  sincospif16_eval(xf, sin_k, cos_k, sin_y, cosm1_y);
+
+  if (LIBC_UNLIKELY(sin_y == 0 && cos_k == 0))
+    return fputil::cast<float16>(0.0f);
+
+  // Since, cosm1_y = cos_y - 1, therefore:
+  // 	cos(x * pi) = cos_k(cosm1_y) + cos_k - sin_k * sin_y
+  return fputil::cast<float16>(fputil::multiply_add(
+      cos_k, cosm1_y, fputil::multiply_add(-sin_k, sin_y, cos_k)));
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/sincosf16_utils.h b/libc/src/math/generic/sincosf16_utils.h
new file mode 100644
index 000000000000..83511755a56c
--- /dev/null
+++ b/libc/src/math/generic/sincosf16_utils.h
@@ -0,0 +1,77 @@
+//===-- Collection of utils for sinf16/cosf16 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_GENERIC_SINCOSF16_UTILS_H
+#define LLVM_LIBC_SRC_MATH_GENERIC_SINCOSF16_UTILS_H
+
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/PolyEval.h"
+#include "src/__support/FPUtil/nearest_integer.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+// Lookup table for sin(k * pi / 32) with k = 0, ..., 63.
+// Table is generated with Sollya as follows:
+// > display = hexadecimmal;
+// > for k from 0 to 63 do { round(sin(k * pi/32), SG, RN); };
+constexpr float SIN_K_PI_OVER_32[64] = {
+    0x0.0p0,        0x1.917a6cp-4,  0x1.8f8b84p-3,  0x1.294062p-2,
+    0x1.87de2ap-2,  0x1.e2b5d4p-2,  0x1.1c73b4p-1,  0x1.44cf32p-1,
+    0x1.6a09e6p-1,  0x1.8bc806p-1,  0x1.a9b662p-1,  0x1.c38b3p-1,
+    0x1.d906bcp-1,  0x1.e9f416p-1,  0x1.f6297cp-1,  0x1.fd88dap-1,
+    0x1p0,          0x1.fd88dap-1,  0x1.f6297cp-1,  0x1.e9f416p-1,
+    0x1.d906bcp-1,  0x1.c38b3p-1,   0x1.a9b662p-1,  0x1.8bc806p-1,
+    0x1.6a09e6p-1,  0x1.44cf32p-1,  0x1.1c73b4p-1,  0x1.e2b5d4p-2,
+    0x1.87de2ap-2,  0x1.294062p-2,  0x1.8f8b84p-3,  0x1.917a6cp-4,
+    0x0.0p0,        -0x1.917a6cp-4, -0x1.8f8b84p-3, -0x1.294062p-2,
+    -0x1.87de2ap-2, -0x1.e2b5d4p-2, -0x1.1c73b4p-1, -0x1.44cf32p-1,
+    -0x1.6a09e6p-1, -0x1.8bc806p-1, -0x1.a9b662p-1, -0x1.c38b3p-1,
+    -0x1.d906bcp-1, -0x1.e9f416p-1, -0x1.f6297ep-1, -0x1.fd88dap-1,
+    -0x1p0,         -0x1.fd88dap-1, -0x1.f6297cp-1, -0x1.e9f416p-1,
+    -0x1.d906bcp-1, -0x1.c38b3p-1,  -0x1.a9b662p-1, -0x1.8bc806p-1,
+    -0x1.6a09e6p-1, -0x1.44cf32p-1, -0x1.1c73b4p-1, -0x1.e2b5d4p-2,
+    -0x1.87de2ap-2, -0x1.294062p-2, -0x1.8f8b84p-3, -0x1.917a6cp-4};
+
+LIBC_INLINE int32_t range_reduction_sincospif16(float x, float &y) {
+  float kf = fputil::nearest_integer(x * 32);
+  y = fputil::multiply_add<float>(x, 32.0, -kf);
+
+  return static_cast<int32_t>(kf);
+}
+
+LIBC_INLINE void sincospif16_eval(float xf, float &sin_k, float &cos_k,
+                                  float &sin_y, float &cosm1_y) {
+  float y;
+  int32_t k = range_reduction_sincospif16(xf, y);
+
+  sin_k = SIN_K_PI_OVER_32[k & 63];
+  cos_k = SIN_K_PI_OVER_32[(k + 16) & 63];
+
+  // Recall, after range reduction, -0.5 <= y <= 0.5. For very small values of
+  // y, calculating sin(y * p/32) can be inaccurate. Generating a polynomial for
+  // sin(y * p/32)/y instead significantly reduces the relative errors.
+  float ysq = y * y;
+
+  // Degree-6 minimax even polynomial for sin(y*pi/32)/y generated by Sollya
+  // with:
+  // > Q = fpminimax(sin(y * pi/32)/y, [|0, 2, 4, 6|], [|SG...|], [0, 0.5]);
+  sin_y = y * fputil::polyeval(ysq, 0x1.921fb6p-4f, -0x1.4aeabcp-13f,
+                               0x1.a03354p-21f, -0x1.ad02d2p-20f);
+
+  // Degree-6 minimax even polynomial for cos(y*pi/32) generated by Sollya
+  // with:
+  // > P = fpminimax(cos(y * pi/32), [|0, 2, 4, 6|],[|1, SG...|], [0, 0.5]);
+  cosm1_y = ysq * fputil::polyeval(ysq, -0x1.3bd3ccp-8f, 0x1.03a61ap-18f,
+                                   0x1.a6f7a2p-29f);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_GENERIC_SINCOSF16_UTILS_H
diff --git a/libc/src/math/generic/sinpif16.cpp b/libc/src/math/generic/sinpif16.cpp
index 17cca583e0c0..51ea595653b4 100644
--- a/libc/src/math/generic/sinpif16.cpp
+++ b/libc/src/math/generic/sinpif16.cpp
@@ -7,52 +7,23 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/sinpif16.h"
+#include "hdr/errno_macros.h"
+#include "hdr/fenv_macros.h"
+#include "sincosf16_utils.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/PolyEval.h"
 #include "src/__support/FPUtil/cast.h"
 #include "src/__support/FPUtil/multiply_add.h"
-#include "src/__support/FPUtil/nearest_integer.h"
-#include "src/__support/common.h"
-#include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-// Lookup table for sin(k * pi / 32) with k = 0, ..., 63.
-// Table is generated with Sollya as follows:
-// > display = hexadecimmal;
-// > for k from 0 to 63 do { round(sin(k * pi/32), SG, RN); };
-static constexpr float SIN_K_PI_OVER_32[64] = {
-    0x0.0p0,        0x1.917a6cp-4,  0x1.8f8b84p-3,  0x1.294062p-2,
-    0x1.87de2ap-2,  0x1.e2b5d4p-2,  0x1.1c73b4p-1,  0x1.44cf32p-1,
-    0x1.6a09e6p-1,  0x1.8bc806p-1,  0x1.a9b662p-1,  0x1.c38b3p-1,
-    0x1.d906bcp-1,  0x1.e9f416p-1,  0x1.f6297cp-1,  0x1.fd88dap-1,
-    0x1p0,          0x1.fd88dap-1,  0x1.f6297cp-1,  0x1.e9f416p-1,
-    0x1.d906bcp-1,  0x1.c38b3p-1,   0x1.a9b662p-1,  0x1.8bc806p-1,
-    0x1.6a09e6p-1,  0x1.44cf32p-1,  0x1.1c73b4p-1,  0x1.e2b5d4p-2,
-    0x1.87de2ap-2,  0x1.294062p-2,  0x1.8f8b84p-3,  0x1.917a6cp-4,
-    0x0.0p0,        -0x1.917a6cp-4, -0x1.8f8b84p-3, -0x1.294062p-2,
-    -0x1.87de2ap-2, -0x1.e2b5d4p-2, -0x1.1c73b4p-1, -0x1.44cf32p-1,
-    -0x1.6a09e6p-1, -0x1.8bc806p-1, -0x1.a9b662p-1, -0x1.c38b3p-1,
-    -0x1.d906bcp-1, -0x1.e9f416p-1, -0x1.f6297ep-1, -0x1.fd88dap-1,
-    -0x1p0,         -0x1.fd88dap-1, -0x1.f6297cp-1, -0x1.e9f416p-1,
-    -0x1.d906bcp-1, -0x1.c38b3p-1,  -0x1.a9b662p-1, -0x1.8bc806p-1,
-    -0x1.6a09e6p-1, -0x1.44cf32p-1, -0x1.1c73b4p-1, -0x1.e2b5d4p-2,
-    -0x1.87de2ap-2, -0x1.294062p-2, -0x1.8f8b84p-3, -0x1.917a6cp-4};
-
-static LIBC_INLINE int32_t range_reduction(float x, float &y) {
-  float kf = fputil::nearest_integer(x * 32);
-  y = fputil::multiply_add<float>(x, 32.0, -kf);
-
-  return static_cast<int32_t>(kf);
-}
-
 LLVM_LIBC_FUNCTION(float16, sinpif16, (float16 x)) {
   using FPBits = typename fputil::FPBits<float16>;
   FPBits xbits(x);
 
   uint16_t x_u = xbits.uintval();
   uint16_t x_abs = x_u & 0x7fff;
+  float xf = x;
 
   // Range reduction:
   // For |x| > 1/32, we perform range reduction as follows:
@@ -68,12 +39,8 @@ LLVM_LIBC_FUNCTION(float16, sinpif16, (float16 x)) {
   // Once k and y are computed, we then deduce the answer by the sine of sum
   // formula:
   //   sin(x * pi) = sin((k + y) * pi/32)
-  //           = sin(k * pi/32) * cos(y * pi/32) + sin (y * pi/32) * cos (k *
-  //           pi/32)
-  // The values of sin(k * pi/32) and cos (k * pi/32) for k = 0...63 are
-  // precomputed and stored using a vector of 64 single precision floats. sin(y
-  // * pi/32) and cos(y * pi/32) are computed using degree-9 chebyshev
-  // polynomials generated by Sollya.
+  //               = sin(k * pi/32) * cos(y * pi/32) +
+  //                 sin(y * pi/32) * cos(k * pi/32)
 
   // For signed zeros
   if (LIBC_UNLIKELY(x_abs == 0U))
@@ -94,36 +61,8 @@ LLVM_LIBC_FUNCTION(float16, sinpif16, (float16 x)) {
     return FPBits::zero(xbits.sign()).get_val();
   }
 
-  float f32 = x;
-  float y;
-  int32_t k = range_reduction(f32, y);
-
-  float sin_k = SIN_K_PI_OVER_32[k & 63];
-  float cos_k = SIN_K_PI_OVER_32[(k + 16) & 63];
-
-  // Recall;
-  // sin(x * pi/32) = sin((k + y) * pi/32)
-  // 		    = sin(y * pi/32) * cos(k * pi/32) + cos(y * pi/32) * sin(k *
-  // pi/32) Recall, after range reduction, -0.5 <= y <= 0.5. For very small
-  // values of y, calculating sin(y * p/32) can be inaccurate. Generating a
-  // polynomial for sin(y * p/32)/y instead significantly reduces the relative
-  // errors.
-  float ysq = y * y;
-
-  // Degree-6 minimax even polynomial for sin(y*pi/32)/y generated by Sollya
-  // with: > Q = fpminimax(sin(y*pi/32)/y, [|0, 2, 4, 6|], [|SG...|], [0, 0.5]);
-  float sin_y = y * fputil::polyeval(ysq, 0x1.921fb6p-4f, -0x1.4aeabcp-13f,
-                                     0x1.a03354p-21f, -0x1.ad02d2p-20f);
-
-  // Note that cosm1_y = cos(y*pi/32) - 1 = cos_y - 1
-  // Derivation:
-  // sin(x * pi) = sin((k + y) * pi/32)
-  //             = sin_y * cos_k + cos_y * sin_k
-  //             = cos_k * sin_y + sin_k * (1 + cos_y - 1)
-  // Degree-6 minimax even polynomial for cos(y*pi/32) generated by Sollya with:
-  // > P = fpminimax(cos(y*pi/32), [|0, 2, 4, 6|],[|1, SG...|], [0, 0.5]);
-  float cosm1_y = ysq * fputil::polyeval(ysq, -0x1.3bd3ccp-8f, 0x1.03a61ap-18f,
-                                         0x1.a6f7a2p-29f);
+  float sin_k, cos_k, sin_y, cosm1_y;
+  sincospif16_eval(xf, sin_k, cos_k, sin_y, cosm1_y);
 
   if (LIBC_UNLIKELY(sin_y == 0 && sin_k == 0))
     return FPBits::zero(xbits.sign()).get_val();
@@ -133,4 +72,5 @@ LLVM_LIBC_FUNCTION(float16, sinpif16, (float16 x)) {
   return fputil::cast<float16>(fputil::multiply_add(
       sin_y, cos_k, fputil::multiply_add(cosm1_y, sin_k, sin_k)));
 }
+
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt
index 262c717dd27d..b46ef4028915 100644
--- a/libc/test/src/math/CMakeLists.txt
+++ b/libc/test/src/math/CMakeLists.txt
@@ -45,6 +45,17 @@ add_fp_unittest(
 )
 
 
+add_fp_unittest(
+  cospif16_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    cospif16_test.cpp
+  DEPENDS
+    libc.src.math.cospif16
+)
+
 add_fp_unittest(
   daddl_test
   NEED_MPFR
diff --git a/libc/test/src/math/cospif16_test.cpp b/libc/test/src/math/cospif16_test.cpp
new file mode 100644
index 000000000000..6a32498b0570
--- /dev/null
+++ b/libc/test/src/math/cospif16_test.cpp
@@ -0,0 +1,40 @@
+//===-- Exhaustive test for cospif16 --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/cospif16.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+
+using LlvmLibcCospif16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
+
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+
+// Range: [0, Inf]
+static constexpr uint16_t POS_START = 0x0000U;
+static constexpr uint16_t POS_STOP = 0x7c00U;
+
+// Range: [-Inf, 0]
+static constexpr uint16_t NEG_START = 0x8000U;
+static constexpr uint16_t NEG_STOP = 0xfc00U;
+
+TEST_F(LlvmLibcCospif16Test, PositiveRange) {
+  for (uint16_t v = POS_START; v <= POS_STOP; ++v) {
+    float16 x = FPBits(v).get_val();
+    EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Cospi, x,
+                                   LIBC_NAMESPACE::cospif16(x), 0.5);
+  }
+}
+
+TEST_F(LlvmLibcCospif16Test, NegativeRange) {
+  for (uint16_t v = NEG_START; v <= NEG_STOP; ++v) {
+    float16 x = FPBits(v).get_val();
+    EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Cospi, x,
+                                   LIBC_NAMESPACE::cospif16(x), 0.5);
+  }
+}
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index b2d1871541ef..269e92c59006 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -25,6 +25,17 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  cospif16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    cospif16_test.cpp
+  DEPENDS
+    libc.src.errno.errno
+    libc.src.math.cospif16
+)
+
 add_fp_unittest(
   sinf_test
   SUITE
diff --git a/libc/test/src/math/smoke/cospif16_test.cpp b/libc/test/src/math/smoke/cospif16_test.cpp
new file mode 100644
index 000000000000..f6d748339319
--- /dev/null
+++ b/libc/test/src/math/smoke/cospif16_test.cpp
@@ -0,0 +1,44 @@
+//===-- Unittests for cospif16 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/errno/libc_errno.h"
+#include "src/math/cospif16.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+using LlvmLibcCospif16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
+
+TEST_F(LlvmLibcCospif16Test, SpecialNumbers) {
+  LIBC_NAMESPACE::libc_errno = 0;
+
+  EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::cospif16(aNaN));
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cospif16(zero));
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cospif16(neg_zero));
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::cospif16(inf));
+  EXPECT_MATH_ERRNO(EDOM);
+
+  EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::cospif16(neg_inf));
+  EXPECT_MATH_ERRNO(EDOM);
+}
+
+TEST_F(LlvmLibcCospif16Test, Integers) {
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cospif16(-0x420));
+  EXPECT_FP_EQ(1.0f, LIBC_NAMESPACE::cospif16(-0x1.4p+14));
+  EXPECT_FP_EQ(-1.0f, LIBC_NAMESPACE::cospif16(0x421));
+  EXPECT_FP_EQ(-1.0f, LIBC_NAMESPACE::cospif16(0x333));
+  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::cospif16(-0x1.28p4));
+  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::cospif16(-0x1.ffcp9));
+  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::cospif16(0x1.01p7));
+  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::cospif16(0x1.f6cp9));
+}
diff --git a/libc/utils/MPFRWrapper/MPFRUtils.cpp b/libc/utils/MPFRWrapper/MPFRUtils.cpp
index bd4fbe294a62..60e4abadb5e3 100644
--- a/libc/utils/MPFRWrapper/MPFRUtils.cpp
+++ b/libc/utils/MPFRWrapper/MPFRUtils.cpp
@@ -255,19 +255,13 @@ public:
     mpfr_cospi(result.value, value, mpfr_rounding);
     return result;
 #else
-    MPFRNumber value_frac(*this);
-    mpfr_frac(value_frac.value, value, MPFR_RNDN);
-
-    if (mpfr_cmp_si(value_frac.value, 0.0) == 0) {
-      mpz_t integer_part;
-      mpz_init(integer_part);
-      mpfr_get_z(integer_part, value, MPFR_RNDN);
-
-      if (mpz_tstbit(integer_part, 0)) {
-        mpfr_set_si(result.value, -1.0, MPFR_RNDN); // odd
-      } else {
-        mpfr_set_si(result.value, 1.0, MPFR_RNDN); // even
-      }
+    if (mpfr_integer_p(value)) {
+      mpz_t integer;
+      mpz_init(integer);
+      mpfr_get_z(integer, value, mpfr_rounding);
+
+      int d = mpz_tstbit(integer, 0);
+      mpfr_set_si(result.value, d ? -1 : 1, mpfr_rounding);
       return result;
     }
 
-- 
GitLab


From c3260c65e86ac363aa3a39f084db66a8a1d1af7d Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Tue, 29 Oct 2024 10:52:20 +0000
Subject: [PATCH 413/693] [IR] Add `llvm.sincos` intrinsic (#109825)

This adds the `llvm.sincos` intrinsic, legalization, and lowering.

The `llvm.sincos` intrinsic takes a floating-point value and returns
both the sine and cosine (as a struct).

```
declare { float, float }          @llvm.sincos.f32(float  %Val)
declare { double, double }        @llvm.sincos.f64(double %Val)
declare { x86_fp80, x86_fp80 }    @llvm.sincos.f80(x86_fp80  %Val)
declare { fp128, fp128 }          @llvm.sincos.f128(fp128 %Val)
declare { ppc_fp128, ppc_fp128 }  @llvm.sincos.ppcf128(ppc_fp128  %Val)
declare { <4 x float>, <4 x float> } @llvm.sincos.v4f32(<4 x float>  %Val)
```

The lowering is built on top of the existing FSINCOS ISD node, with
additional type legalization to allow for f16, f128, and vector values.
---
 llvm/docs/GlobalISel/GenericOpcode.rst        |   4 +-
 llvm/docs/LangRef.rst                         |  48 ++
 llvm/include/llvm/CodeGen/BasicTTIImpl.h      |   3 +
 .../CodeGen/GlobalISel/MachineIRBuilder.h     |   7 +
 llvm/include/llvm/IR/Intrinsics.td            |   2 +
 llvm/include/llvm/Support/TargetOpcodes.def   |   3 +
 llvm/include/llvm/Target/GenericOpcodes.td    |   7 +
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |   7 +
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp |  21 +
 .../SelectionDAG/LegalizeFloatTypes.cpp       |  83 +++
 llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h |  11 +-
 .../SelectionDAG/LegalizeVectorOps.cpp        |   1 +
 .../SelectionDAG/LegalizeVectorTypes.cpp      |  69 ++-
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |  11 +-
 .../SelectionDAG/SelectionDAGBuilder.cpp      |  16 +-
 llvm/lib/CodeGen/TargetLoweringBase.cpp       |   5 +-
 .../lib/Target/ARM/ARMTargetTransformInfo.cpp |   1 +
 .../AArch64/GlobalISel/irtranslator-sincos.ll | 120 ++++
 .../GlobalISel/legalizer-info-validation.mir  |   3 +
 llvm/test/CodeGen/AArch64/llvm.frexp.ll       | 101 ++++
 llvm/test/CodeGen/AArch64/llvm.sincos-fmf.ll  |  16 +
 llvm/test/CodeGen/AArch64/llvm.sincos.ll      | 553 ++++++++++++++++++
 llvm/test/CodeGen/ARM/llvm.sincos.ll          | 223 +++++++
 23 files changed, 1300 insertions(+), 15 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-sincos.ll
 create mode 100644 llvm/test/CodeGen/AArch64/llvm.frexp.ll
 create mode 100644 llvm/test/CodeGen/AArch64/llvm.sincos-fmf.ll
 create mode 100644 llvm/test/CodeGen/AArch64/llvm.sincos.ll
 create mode 100644 llvm/test/CodeGen/ARM/llvm.sincos.ll

diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst
index 1c4e00b956bc..8920530dc3f1 100644
--- a/llvm/docs/GlobalISel/GenericOpcode.rst
+++ b/llvm/docs/GlobalISel/GenericOpcode.rst
@@ -633,8 +633,8 @@ G_FCEIL, G_FSQRT, G_FFLOOR, G_FRINT, G_FNEARBYINT
 
 These correspond to the standard C functions of the same name.
 
-G_FCOS, G_FSIN, G_FTAN, G_FACOS, G_FASIN, G_FATAN, G_FATAN2, G_FCOSH, G_FSINH, G_FTANH
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+G_FCOS, G_FSIN, G_FSINCOS, G_FTAN, G_FACOS, G_FASIN, G_FATAN, G_FATAN2, G_FCOSH, G_FSINH, G_FTANH
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 These correspond to the standard C trigonometry functions of the same name.
 
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index f9ec33da1b65..177924dca4d1 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -15512,6 +15512,8 @@ Semantics:
 This function returns the first value raised to the second power with an
 unspecified sequence of rounding operations.
 
+.. _t_llvm_sin:
+
 '``llvm.sin.*``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -15549,6 +15551,8 @@ trapping or setting ``errno``.
 When specified with the fast-math-flag 'afn', the result may be approximated
 using a less accurate calculation.
 
+.. _t_llvm_cos:
+
 '``llvm.cos.*``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -15882,6 +15886,50 @@ trapping or setting ``errno``.
 When specified with the fast-math-flag 'afn', the result may be approximated
 using a less accurate calculation.
 
+
+'``llvm.sincos.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+This is an overloaded intrinsic. You can use ``llvm.sincos`` on any
+floating-point or vector of floating-point type. Not all targets support
+all types however.
+
+::
+
+      declare { float, float }          @llvm.sincos.f32(float  %Val)
+      declare { double, double }        @llvm.sincos.f64(double %Val)
+      declare { x86_fp80, x86_fp80 }    @llvm.sincos.f80(x86_fp80  %Val)
+      declare { fp128, fp128 }          @llvm.sincos.f128(fp128 %Val)
+      declare { ppc_fp128, ppc_fp128 }  @llvm.sincos.ppcf128(ppc_fp128  %Val)
+      declare { <4 x float>, <4 x float> } @llvm.sincos.v4f32(<4 x float>  %Val)
+
+Overview:
+"""""""""
+
+The '``llvm.sincos.*``' intrinsics returns the sine and cosine of the operand.
+
+Arguments:
+""""""""""
+
+The argument is a :ref:`floating-point <t_floating>` value or
+:ref:`vector <t_vector>` of floating-point values. Returns two values matching
+the argument type in a struct.
+
+Semantics:
+""""""""""
+
+This intrinsic is equivalent to a calling both :ref:`llvm.sin <t_llvm_sin>`
+and :ref:`llvm.cos <t_llvm_cos>` on the argument.
+
+The first result is the sine of the argument and the second result is the cosine
+of the argument.
+
+When specified with the fast-math-flag 'afn', the result may be approximated
+using a less accurate calculation.
+
 '``llvm.pow.*``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index db3b5cddd7c1..b0316e67654d 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1986,6 +1986,9 @@ public:
     case Intrinsic::cos:
       ISD = ISD::FCOS;
       break;
+    case Intrinsic::sincos:
+      ISD = ISD::FSINCOS;
+      break;
     case Intrinsic::tan:
       ISD = ISD::FTAN;
       break;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 9b993482c8cc..ab3025e4923c 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -2009,6 +2009,13 @@ public:
     return buildInstr(TargetOpcode::G_FFREXP, {Fract, Exp}, {Src}, Flags);
   }
 
+  /// Build and insert \p Sin, \p Cos = G_FSINCOS \p Src
+  MachineInstrBuilder
+  buildFSincos(const DstOp &Sin, const DstOp &Cos, const SrcOp &Src,
+               std::optional<unsigned> Flags = std::nullopt) {
+    return buildInstr(TargetOpcode::G_FSINCOS, {Sin, Cos}, {Src}, Flags);
+  }
+
   /// Build and insert \p Res = G_FCOPYSIGN \p Op0, \p Op1
   MachineInstrBuilder buildFCopysign(const DstOp &Dst, const SrcOp &Src0,
                                      const SrcOp &Src1) {
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 94e53f372127..e91758ed34eb 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1050,6 +1050,8 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn] in {
   def int_nearbyint : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
   def int_round : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
   def int_roundeven    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
+  def int_sincos : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
+                             [llvm_anyfloat_ty]>;
 
   // Truncate a floating point number with a specific rounding mode
   def int_fptrunc_round : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ],
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index 3556a253d875..0c4c6ccd5c56 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -809,6 +809,9 @@ HANDLE_TARGET_OPCODE(G_FCOS)
 /// Floating point sine.
 HANDLE_TARGET_OPCODE(G_FSIN)
 
+/// Floating point combined sine and cosine.
+HANDLE_TARGET_OPCODE(G_FSINCOS)
+
 /// Floating point tangent.
 HANDLE_TARGET_OPCODE(G_FTAN)
 
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 8b8bc9a0e9cf..62bb9789afe5 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -1020,6 +1020,13 @@ def G_FSIN : GenericInstruction {
   let hasSideEffects = false;
 }
 
+// Floating point combined sine and cosine.
+def G_FSINCOS : GenericInstruction {
+  let OutOperandList = (outs type0:$dst1, type0:$dst2);
+  let InOperandList = (ins type0:$src1);
+  let hasSideEffects = false;
+}
+
 // Floating point tangent of a value.
 def G_FTAN : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 563a82644134..5381dce58f9e 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2343,6 +2343,13 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
                            MachineInstr::copyFlagsFromInstruction(CI));
     return true;
   }
+  case Intrinsic::sincos: {
+    ArrayRef<Register> VRegs = getOrCreateVRegs(CI);
+    MIRBuilder.buildFSincos(VRegs[0], VRegs[1],
+                            getOrCreateVReg(*CI.getArgOperand(0)),
+                            MachineInstr::copyFlagsFromInstruction(CI));
+    return true;
+  }
   case Intrinsic::fptosi_sat:
     MIRBuilder.buildFPTOSI_SAT(getOrCreateVReg(CI),
                                getOrCreateVReg(*CI.getArgOperand(0)));
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index e0a03383358b..47a9ae12248c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3714,6 +3714,17 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     }
     break;
   }
+  case ISD::FSINCOS: {
+    if (isSinCosLibcallAvailable(Node, TLI))
+      break;
+    EVT VT = Node->getValueType(0);
+    SDValue Op = Node->getOperand(0);
+    SDNodeFlags Flags = Node->getFlags();
+    Tmp1 = DAG.getNode(ISD::FSIN, dl, VT, Op, Flags);
+    Tmp2 = DAG.getNode(ISD::FCOS, dl, VT, Op, Flags);
+    Results.append({Tmp1, Tmp2});
+    break;
+  }
   case ISD::FMAD:
     llvm_unreachable("Illegal fmad should never be formed");
 
@@ -5586,6 +5597,16 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
     Results.push_back(Tmp2.getValue(1));
     break;
   }
+  case ISD::FSINCOS: {
+    Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
+    Tmp2 = DAG.getNode(ISD::FSINCOS, dl, DAG.getVTList(NVT, NVT), Tmp1,
+                       Node->getFlags());
+    Tmp3 = DAG.getIntPtrConstant(0, dl, /*isTarget=*/true);
+    for (unsigned ResNum = 0; ResNum < Node->getNumValues(); ResNum++)
+      Results.push_back(
+          DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2.getValue(ResNum), Tmp3));
+    break;
+  }
   case ISD::FFLOOR:
   case ISD::FCEIL:
   case ISD::FRINT:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 73c258f0f6f1..fa2731ff7dbd 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -129,6 +129,7 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
     case ISD::FLDEXP:
     case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
     case ISD::FFREXP:        R = SoftenFloatRes_FFREXP(N); break;
+    case ISD::FSINCOS:       R = SoftenFloatRes_FSINCOS(N); break;
     case ISD::STRICT_FREM:
     case ISD::FREM:        R = SoftenFloatRes_FREM(N); break;
     case ISD::STRICT_FRINT:
@@ -774,6 +775,45 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
   return ReturnVal;
 }
 
+SDValue DAGTypeLegalizer::SoftenFloatRes_FSINCOS(SDNode *N) {
+  assert(!N->isStrictFPOpcode() && "strictfp not implemented for fsincos");
+  EVT VT = N->getValueType(0);
+  RTLIB::Libcall LC = RTLIB::getFSINCOS(VT);
+
+  if (!TLI.getLibcallName(LC))
+    return SDValue();
+
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+  SDValue StackSlotSin = DAG.CreateStackTemporary(NVT);
+  SDValue StackSlotCos = DAG.CreateStackTemporary(NVT);
+
+  SDLoc DL(N);
+
+  TargetLowering::MakeLibCallOptions CallOptions;
+  std::array Ops{GetSoftenedFloat(N->getOperand(0)), StackSlotSin,
+                 StackSlotCos};
+  std::array OpsVT{VT, StackSlotSin.getValueType(),
+                   StackSlotCos.getValueType()};
+
+  // TODO: setTypeListBeforeSoften can't properly express multiple return types,
+  // but since both returns have the same type for sincos it should be okay.
+  CallOptions.setTypeListBeforeSoften({OpsVT}, VT, true);
+
+  auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT, Ops, CallOptions, DL,
+                                            /*Chain=*/SDValue());
+
+  auto CreateStackLoad = [&, Chain = Chain](SDValue StackSlot) {
+    int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
+    auto PtrInfo =
+        MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
+    return DAG.getLoad(NVT, DL, Chain, StackSlot, PtrInfo);
+  };
+  SetSoftenedFloat(SDValue(N, 0), CreateStackLoad(StackSlotSin));
+  SetSoftenedFloat(SDValue(N, 1), CreateStackLoad(StackSlotCos));
+
+  return SDValue();
+}
+
 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
   return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
                                                RTLIB::REM_F32,
@@ -2704,6 +2744,10 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
     case ISD::FLDEXP:     R = PromoteFloatRes_ExpOp(N); break;
     case ISD::FFREXP:     R = PromoteFloatRes_FFREXP(N); break;
 
+    case ISD::FSINCOS:
+      R = PromoteFloatRes_FSINCOS(N);
+      break;
+
     case ISD::FP_ROUND:   R = PromoteFloatRes_FP_ROUND(N); break;
     case ISD::STRICT_FP_ROUND:
       R = PromoteFloatRes_STRICT_FP_ROUND(N);
@@ -2899,6 +2943,20 @@ SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
   return Res;
 }
 
+SDValue DAGTypeLegalizer::PromoteFloatRes_FSINCOS(SDNode *N) {
+  EVT VT = N->getValueType(0);
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+  SDValue Op = GetPromotedFloat(N->getOperand(0));
+  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, NVT}, Op);
+
+  for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
+       ++ResNum) {
+    SetPromotedFloat(SDValue(N, ResNum), Res.getValue(ResNum));
+  }
+
+  return SDValue();
+}
+
 // Explicit operation to reduce precision.  Reduce the value to half precision
 // and promote it back to the legal type.
 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
@@ -3148,6 +3206,10 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
 
   case ISD::FFREXP:      R = SoftPromoteHalfRes_FFREXP(N); break;
 
+  case ISD::FSINCOS:
+    R = SoftPromoteHalfRes_FSINCOS(N);
+    break;
+
   case ISD::LOAD:        R = SoftPromoteHalfRes_LOAD(N); break;
   case ISD::ATOMIC_LOAD:
     R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
@@ -3304,6 +3366,27 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
   return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
 }
 
+SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FSINCOS(SDNode *N) {
+  EVT OVT = N->getValueType(0);
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
+  SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
+  SDLoc dl(N);
+
+  // Promote to the larger FP type.
+  Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
+  SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, NVT), Op);
+
+  // Convert back to FP16 as an integer.
+  ISD::NodeType Truncate = GetPromotionOpcode(NVT, OVT);
+  for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
+       ++ResNum) {
+    SDValue Trunc = DAG.getNode(Truncate, dl, MVT::i16, Res.getValue(ResNum));
+    SetSoftPromotedHalf(SDValue(N, ResNum), Trunc);
+  }
+
+  return SDValue();
+}
+
 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
   EVT RVT = N->getValueType(0);
   bool IsStrict = N->isStrictFPOpcode();
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 868da25ca8cb..8d3458aaab9f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -597,6 +597,7 @@ private:
   SDValue SoftenFloatRes_FPOW(SDNode *N);
   SDValue SoftenFloatRes_ExpOp(SDNode *N);
   SDValue SoftenFloatRes_FFREXP(SDNode *N);
+  SDValue SoftenFloatRes_FSINCOS(SDNode *N);
   SDValue SoftenFloatRes_FREEZE(SDNode *N);
   SDValue SoftenFloatRes_FREM(SDNode *N);
   SDValue SoftenFloatRes_FRINT(SDNode *N);
@@ -744,6 +745,7 @@ private:
   SDValue PromoteFloatRes_FMAD(SDNode *N);
   SDValue PromoteFloatRes_ExpOp(SDNode *N);
   SDValue PromoteFloatRes_FFREXP(SDNode *N);
+  SDValue PromoteFloatRes_FSINCOS(SDNode *N);
   SDValue PromoteFloatRes_FP_ROUND(SDNode *N);
   SDValue PromoteFloatRes_STRICT_FP_ROUND(SDNode *N);
   SDValue PromoteFloatRes_LOAD(SDNode *N);
@@ -792,6 +794,7 @@ private:
   SDValue SoftPromoteHalfRes_FMAD(SDNode *N);
   SDValue SoftPromoteHalfRes_ExpOp(SDNode *N);
   SDValue SoftPromoteHalfRes_FFREXP(SDNode *N);
+  SDValue SoftPromoteHalfRes_FSINCOS(SDNode *N);
   SDValue SoftPromoteHalfRes_FP_ROUND(SDNode *N);
   SDValue SoftPromoteHalfRes_LOAD(SDNode *N);
   SDValue SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N);
@@ -863,7 +866,7 @@ private:
   SDValue ScalarizeVecRes_IS_FPCLASS(SDNode *N);
 
   SDValue ScalarizeVecRes_FIX(SDNode *N);
-  SDValue ScalarizeVecRes_FFREXP(SDNode *N, unsigned ResNo);
+  SDValue ScalarizeVecRes_UnaryOpWithTwoResults(SDNode *N, unsigned ResNo);
 
   // Vector Operand Scalarization: <1 x ty> -> ty.
   bool ScalarizeVectorOperand(SDNode *N, unsigned OpNo);
@@ -917,7 +920,8 @@ private:
   void SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo, SDValue &Hi);
-  void SplitVecRes_FFREXP(SDNode *N, unsigned ResNo, SDValue &Lo, SDValue &Hi);
+  void SplitVecRes_UnaryOpWithTwoResults(SDNode *N, unsigned ResNo, SDValue &Lo,
+                                         SDValue &Hi);
   void SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_InregOp(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, SDValue &Hi);
@@ -1068,6 +1072,9 @@ private:
   SDValue WidenVecRes_ExpOp(SDNode *N);
   SDValue WidenVecRes_Unary(SDNode *N);
   SDValue WidenVecRes_InregOp(SDNode *N);
+  SDValue WidenVecRes_UnaryOpWithTwoResults(SDNode *N, unsigned ResNo);
+  void ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,
+                                unsigned WidenResNo);
 
   // Widen Vector Operand.
   bool WidenVectorOperand(SDNode *N, unsigned OpNo);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index a8042fc3e7a6..c80da28b3dc3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -452,6 +452,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   case ISD::UMULO:
   case ISD::FCANONICALIZE:
   case ISD::FFREXP:
+  case ISD::FSINCOS:
   case ISD::SADDSAT:
   case ISD::UADDSAT:
   case ISD::SSUBSAT:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 50e2a923699c..5409ae7d9671 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -130,7 +130,8 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
     R = ScalarizeVecRes_ADDRSPACECAST(N);
     break;
   case ISD::FFREXP:
-    R = ScalarizeVecRes_FFREXP(N, ResNo);
+  case ISD::FSINCOS:
+    R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
     break;
   case ISD::ADD:
   case ISD::AND:
@@ -276,7 +277,9 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
                      Op2, N->getFlags());
 }
 
-SDValue DAGTypeLegalizer::ScalarizeVecRes_FFREXP(SDNode *N, unsigned ResNo) {
+SDValue
+DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithTwoResults(SDNode *N,
+                                                        unsigned ResNo) {
   assert(N->getValueType(0).getVectorNumElements() == 1 &&
          "Unexpected vector type!");
   SDValue Elt = GetScalarizedVector(N->getOperand(0));
@@ -1253,7 +1256,8 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
     SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
     break;
   case ISD::FFREXP:
-    SplitVecRes_FFREXP(N, ResNo, Lo, Hi);
+  case ISD::FSINCOS:
+    SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
     break;
 
   case ISD::ANY_EXTEND:
@@ -2615,8 +2619,10 @@ void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo,
   Hi = DAG.getAddrSpaceCast(dl, HiVT, Hi, SrcAS, DestAS);
 }
 
-void DAGTypeLegalizer::SplitVecRes_FFREXP(SDNode *N, unsigned ResNo,
-                                          SDValue &Lo, SDValue &Hi) {
+void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(SDNode *N,
+                                                         unsigned ResNo,
+                                                         SDValue &Lo,
+                                                         SDValue &Hi) {
   SDLoc dl(N);
   auto [LoVT, HiVT] = DAG.GetSplitDestVTs(N->getValueType(0));
   auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(N->getValueType(1));
@@ -4429,6 +4435,25 @@ SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
 //  Result Vector Widening
 //===----------------------------------------------------------------------===//
 
+void DAGTypeLegalizer::ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,
+                                                unsigned WidenResNo) {
+  unsigned NumResults = N->getNumValues();
+  for (unsigned ResNo = 0; ResNo < NumResults; ResNo++) {
+    if (ResNo == WidenResNo)
+      continue;
+    EVT ResVT = N->getValueType(ResNo);
+    if (getTypeAction(ResVT) == TargetLowering::TypeWidenVector) {
+      SetWidenedVector(SDValue(N, ResNo), SDValue(WidenNode, ResNo));
+    } else {
+      SDLoc DL(N);
+      SDValue ResVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResVT,
+                                   SDValue(WidenNode, ResNo),
+                                   DAG.getVectorIdxConstant(0, DL));
+      ReplaceValueWith(SDValue(N, ResNo), ResVal);
+    }
+  }
+}
+
 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
   LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
 
@@ -4448,6 +4473,8 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
     if (!TLI.isOperationLegalOrCustomOrPromote(N->getOpcode(), WideVecVT) &&
         TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
       Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
+      if (N->getNumValues() > 1)
+        ReplaceOtherWidenResults(N, Res.getNode(), ResNo);
       return true;
     }
     return false;
@@ -4752,6 +4779,12 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::VP_FSHR:
     Res = WidenVecRes_Ternary(N);
     break;
+  case ISD::FFREXP:
+  case ISD::FSINCOS: {
+    if (!unrollExpandedOp())
+      Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
+    break;
+  }
   }
 
   // If Res is null, the sub-method took care of registering the result.
@@ -5500,6 +5533,32 @@ SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
                      WidenVT, WidenLHS, DAG.getValueType(ExtVT));
 }
 
+SDValue DAGTypeLegalizer::WidenVecRes_UnaryOpWithTwoResults(SDNode *N,
+                                                            unsigned ResNo) {
+  EVT VT0 = N->getValueType(0);
+  EVT VT1 = N->getValueType(1);
+
+  assert(VT0.isVector() && VT1.isVector() &&
+         VT0.getVectorElementCount() == VT1.getVectorElementCount() &&
+         "expected both results to be vectors of matching element count");
+
+  LLVMContext &Ctx = *DAG.getContext();
+  SDValue InOp = GetWidenedVector(N->getOperand(0));
+
+  EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(ResNo));
+  ElementCount WidenEC = WidenVT.getVectorElementCount();
+
+  EVT WidenVT0 = EVT::getVectorVT(Ctx, VT0.getVectorElementType(), WidenEC);
+  EVT WidenVT1 = EVT::getVectorVT(Ctx, VT1.getVectorElementType(), WidenEC);
+
+  SDNode *WidenNode =
+      DAG.getNode(N->getOpcode(), SDLoc(N), {WidenVT0, WidenVT1}, InOp)
+          .getNode();
+
+  ReplaceOtherWidenResults(N, WidenNode, ResNo);
+  return SDValue(WidenNode, ResNo);
+}
+
 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
   SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
   return GetWidenedVector(WidenVec);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 1a86b3b51234..8f255cce1fe1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -12533,8 +12533,15 @@ SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
       Scalars1.push_back(EltOp.getValue(1));
     }
 
-    SDValue Vec0 = getBuildVector(VT, dl, Scalars0);
-    SDValue Vec1 = getBuildVector(VT1, dl, Scalars1);
+    for (; i < ResNE; ++i) {
+      Scalars0.push_back(getUNDEF(EltVT));
+      Scalars1.push_back(getUNDEF(EltVT1));
+    }
+
+    EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
+    EVT VecVT1 = EVT::getVectorVT(*getContext(), EltVT1, ResNE);
+    SDValue Vec0 = getBuildVector(VecVT, dl, Scalars0);
+    SDValue Vec1 = getBuildVector(VecVT1, dl, Scalars1);
     return getMergeValues({Vec0, Vec1}, dl);
   }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 845055374307..203e80e36b46 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6936,12 +6936,24 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
                              getValue(I.getArgOperand(0)),
                              getValue(I.getArgOperand(1)), Flags));
     return;
+  case Intrinsic::sincos:
   case Intrinsic::frexp: {
+    unsigned Opcode;
+    switch (Intrinsic) {
+    default:
+      llvm_unreachable("unexpected intrinsic");
+    case Intrinsic::sincos:
+      Opcode = ISD::FSINCOS;
+      break;
+    case Intrinsic::frexp:
+      Opcode = ISD::FFREXP;
+      break;
+    }
     SmallVector<EVT, 2> ValueVTs;
     ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
     SDVTList VTs = DAG.getVTList(ValueVTs);
-    setValue(&I,
-             DAG.getNode(ISD::FFREXP, sdl, VTs, getValue(I.getArgOperand(0))));
+    setValue(
+        &I, DAG.getNode(Opcode, sdl, VTs, getValue(I.getArgOperand(0)), Flags));
     return;
   }
   case Intrinsic::arithmetic_fence: {
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index cab0ed235774..5bcde0e1bbec 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -773,8 +773,9 @@ void TargetLoweringBase::initActions() {
     setOperationAction({ISD::BITREVERSE, ISD::PARITY}, VT, Expand);
 
     // These library functions default to expand.
-    setOperationAction({ISD::FROUND, ISD::FPOWI, ISD::FLDEXP, ISD::FFREXP}, VT,
-                       Expand);
+    setOperationAction(
+        {ISD::FROUND, ISD::FPOWI, ISD::FLDEXP, ISD::FFREXP, ISD::FSINCOS}, VT,
+        Expand);
 
     // These operations default to expand for vector types.
     if (VT.isVector())
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 9f6e5e5ab142..0e29648a7a28 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -2063,6 +2063,7 @@ bool ARMTTIImpl::isLoweredToCall(const Function *F) {
   case Intrinsic::powi:
   case Intrinsic::sin:
   case Intrinsic::cos:
+  case Intrinsic::sincos:
   case Intrinsic::pow:
   case Intrinsic::log:
   case Intrinsic::log10:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-sincos.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-sincos.ll
new file mode 100644
index 000000000000..69cd6ce87b5c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-sincos.ll
@@ -0,0 +1,120 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -O0 -mtriple=aarch64-linux-gnu -global-isel -stop-after=irtranslator %s -o - | FileCheck %s
+
+define { half, half } @test_sincos_f16(half %a) {
+  ; CHECK-LABEL: name: test_sincos_f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $h0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s16) = COPY $h0
+  ; CHECK-NEXT:   [[FSINCOS:%[0-9]+]]:_(s16), [[FSINCOS1:%[0-9]+]]:_ = G_FSINCOS [[COPY]]
+  ; CHECK-NEXT:   $h0 = COPY [[FSINCOS]](s16)
+  ; CHECK-NEXT:   $h1 = COPY [[FSINCOS1]](s16)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $h0, implicit $h1
+  %result = call { half, half } @llvm.sincos.f16(half %a)
+  ret { half, half } %result
+}
+
+define { <2 x half>, <2 x half> } @test_sincos_v2f16(<2 x half> %a) {
+  ; CHECK-LABEL: name: test_sincos_v2f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $d0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $d0
+  ; CHECK-NEXT:   [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+  ; CHECK-NEXT:   [[FSINCOS:%[0-9]+]]:_(<2 x s16>), [[FSINCOS1:%[0-9]+]]:_ = G_FSINCOS [[UV]]
+  ; CHECK-NEXT:   [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[FSINCOS]](<2 x s16>)
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
+  ; CHECK-NEXT:   [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[UV2]](s16), [[UV3]](s16), [[DEF]](s16), [[DEF]](s16)
+  ; CHECK-NEXT:   [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[FSINCOS1]](<2 x s16>)
+  ; CHECK-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[UV4]](s16), [[UV5]](s16), [[DEF]](s16), [[DEF]](s16)
+  ; CHECK-NEXT:   $d0 = COPY [[BUILD_VECTOR]](<4 x s16>)
+  ; CHECK-NEXT:   $d1 = COPY [[BUILD_VECTOR1]](<4 x s16>)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $d0, implicit $d1
+  %result = call { <2 x half>, <2 x half> } @llvm.sincos.v2f16(<2 x half> %a)
+  ret { <2 x half>, <2 x half> } %result
+}
+
+define { float, float } @test_sincos_f32(float %a) {
+  ; CHECK-LABEL: name: test_sincos_f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $s0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+  ; CHECK-NEXT:   [[FSINCOS:%[0-9]+]]:_(s32), [[FSINCOS1:%[0-9]+]]:_ = G_FSINCOS [[COPY]]
+  ; CHECK-NEXT:   $s0 = COPY [[FSINCOS]](s32)
+  ; CHECK-NEXT:   $s1 = COPY [[FSINCOS1]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $s0, implicit $s1
+  %result = call { float, float } @llvm.sincos.f32(float %a)
+  ret { float, float } %result
+}
+
+define { <2 x float>, <2 x float> } @test_sincos_v2f32(<2 x float> %a) {
+  ; CHECK-LABEL: name: test_sincos_v2f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $d0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+  ; CHECK-NEXT:   [[FSINCOS:%[0-9]+]]:_(<2 x s32>), [[FSINCOS1:%[0-9]+]]:_ = G_FSINCOS [[COPY]]
+  ; CHECK-NEXT:   $d0 = COPY [[FSINCOS]](<2 x s32>)
+  ; CHECK-NEXT:   $d1 = COPY [[FSINCOS1]](<2 x s32>)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $d0, implicit $d1
+  %result = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> %a)
+  ret { <2 x float>, <2 x float> } %result
+}
+
+define { double, double } @test_sincos_f64(double %a) {
+  ; CHECK-LABEL: name: test_sincos_f64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $d0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $d0
+  ; CHECK-NEXT:   [[FSINCOS:%[0-9]+]]:_(s64), [[FSINCOS1:%[0-9]+]]:_ = G_FSINCOS [[COPY]]
+  ; CHECK-NEXT:   $d0 = COPY [[FSINCOS]](s64)
+  ; CHECK-NEXT:   $d1 = COPY [[FSINCOS1]](s64)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $d0, implicit $d1
+  %result = call { double, double } @llvm.sincos.f64(double %a)
+  ret { double, double } %result
+}
+
+define { <2 x double>, <2 x double> } @test_sincos_v2f64(<2 x double> %a) {
+  ; CHECK-LABEL: name: test_sincos_v2f64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+  ; CHECK-NEXT:   [[FSINCOS:%[0-9]+]]:_(<2 x s64>), [[FSINCOS1:%[0-9]+]]:_ = G_FSINCOS [[COPY]]
+  ; CHECK-NEXT:   $q0 = COPY [[FSINCOS]](<2 x s64>)
+  ; CHECK-NEXT:   $q1 = COPY [[FSINCOS1]](<2 x s64>)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $q0, implicit $q1
+  %result = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> %a)
+  ret { <2 x double>, <2 x double> } %result
+}
+
+define { fp128, fp128 } @test_sincos_f128(fp128 %a) {
+  ; CHECK-LABEL: name: test_sincos_f128
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s128) = COPY $q0
+  ; CHECK-NEXT:   [[FSINCOS:%[0-9]+]]:_(s128), [[FSINCOS1:%[0-9]+]]:_ = G_FSINCOS [[COPY]]
+  ; CHECK-NEXT:   $q0 = COPY [[FSINCOS]](s128)
+  ; CHECK-NEXT:   $q1 = COPY [[FSINCOS1]](s128)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $q0, implicit $q1
+  %result = call { fp128, fp128 } @llvm.sincos.f16(fp128 %a)
+  ret { fp128, fp128 } %result
+}
+
+define { float, float } @test_sincos_f32_afn(float %a) {
+  ; CHECK-LABEL: name: test_sincos_f32_afn
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $s0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+  ; CHECK-NEXT:   [[FSINCOS:%[0-9]+]]:_(s32), [[FSINCOS1:%[0-9]+]]:_ = afn G_FSINCOS [[COPY]]
+  ; CHECK-NEXT:   $s0 = COPY [[FSINCOS]](s32)
+  ; CHECK-NEXT:   $s1 = COPY [[FSINCOS1]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $s0, implicit $s1
+  %result = call afn { float, float } @llvm.sincos.f32(float %a)
+  ret { float, float } %result
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 0af60a503c5f..6be99d0088f1 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -700,6 +700,9 @@
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: G_FSINCOS (opcode {{[0-9]+}}): 1 type index, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_FTAN (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
diff --git a/llvm/test/CodeGen/AArch64/llvm.frexp.ll b/llvm/test/CodeGen/AArch64/llvm.frexp.ll
new file mode 100644
index 000000000000..e4cb8ed6eaf9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/llvm.frexp.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=aarch64-gnu-linux < %s | FileCheck -check-prefixes=CHECK %s
+
+define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) {
+; CHECK-LABEL: test_frexp_v2f16_v2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    stp x30, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov h1, v0.h[1]
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #36
+; CHECK-NEXT:    add x19, sp, #36
+; CHECK-NEXT:    fcvt s0, h1
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    add x0, sp, #32
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    fmov s0, s1
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    fcvt h2, s0
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    mov h1, v1.h[2]
+; CHECK-NEXT:    fcvt s0, h1
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v2.h[1], v1.h[0]
+; CHECK-NEXT:    str q2, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    fcvt h2, s0
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    mov h1, v1.h[3]
+; CHECK-NEXT:    fcvt s0, h1
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v1.h[2], v2.h[0]
+; CHECK-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    fcvt h2, s0
+; CHECK-NEXT:    ldr s1, [sp, #32]
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ld1 { v1.s }[1], [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    mov v0.h[3], v2.h[0]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+  %result = call { <2 x half>, <2 x i32> } @llvm.frexp.v2f16.v2i32(<2 x half> %a)
+  ret { <2 x half>, <2 x i32> } %result
+}
+
+define { <3 x float>, <3 x i32> } @test_frexp_v3f16_v3i32(<3 x float> %a) {
+; CHECK-LABEL: test_frexp_v3f16_v3i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #80
+; CHECK-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    add x0, sp, #56
+; CHECK-NEXT:    add x19, sp, #56
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    add x0, sp, #60
+; CHECK-NEXT:    add x20, sp, #60
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr s1, [sp, #44]
+; CHECK-NEXT:    ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-NEXT:    ld1 { v1.s }[1], [x19]
+; CHECK-NEXT:    mov v2.s[2], v0.s[0]
+; CHECK-NEXT:    ld1 { v1.s }[2], [x20]
+; CHECK-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    ret
+  %result = call { <3 x float>, <3 x i32> } @llvm.frexp.v3float.v3i32(<3 x float> %a)
+  ret { <3 x float>, <3 x i32> } %result
+}
diff --git a/llvm/test/CodeGen/AArch64/llvm.sincos-fmf.ll b/llvm/test/CodeGen/AArch64/llvm.sincos-fmf.ll
new file mode 100644
index 000000000000..456b7f98974a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/llvm.sincos-fmf.ll
@@ -0,0 +1,16 @@
+; RUN: llc -mtriple=aarch64-gnu-linux -debug-only=isel -o /dev/null < %s 2>&1 | FileCheck %s
+
+; REQUIRES: asserts
+
+define { float, float } @test_sincos_f32_afn(float %a) {
+; CHECK-LABEL: Initial selection DAG: %bb.0 'test_sincos_f32_afn:'
+; CHECK-NEXT:  SelectionDAG has 9 nodes:
+; CHECK-NEXT:    t0: ch,glue = EntryToken
+; CHECK-NEXT:      t2: f32,ch = CopyFromReg t0, Register:f32 %0
+; CHECK-NEXT:    t3: f32,f32 = fsincos afn t2
+; CHECK-NEXT:    t5: ch,glue = CopyToReg t0, Register:f32 $s0, t3
+; CHECK-NEXT:    t7: ch,glue = CopyToReg t5, Register:f32 $s1, t3:1, t5:1
+; CHECK-NEXT:    t8: ch = AArch64ISD::RET_GLUE t7, Register:f32 $s0, Register:f32 $s1, t7:1
+  %result = call afn { float, float } @llvm.sincos.f32(float %a)
+  ret { float, float } %result
+}
diff --git a/llvm/test/CodeGen/AArch64/llvm.sincos.ll b/llvm/test/CodeGen/AArch64/llvm.sincos.ll
new file mode 100644
index 000000000000..c5efc796e7a3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/llvm.sincos.ll
@@ -0,0 +1,553 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=aarch64-gnu-linux < %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -mtriple=aarch64-none-linux < %s | FileCheck -check-prefixes=NO-LIBCALL %s
+
+define { half, half } @test_sincos_f16(half %a) {
+; CHECK-LABEL: test_sincos_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    add x1, sp, #8
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldp s1, s0, [sp, #8]
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    fcvt h1, s1
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; NO-LIBCALL-LABEL: test_sincos_f16:
+; NO-LIBCALL:       // %bb.0:
+; NO-LIBCALL-NEXT:    stp d9, d8, [sp, #-32]! // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; NO-LIBCALL-NEXT:    .cfi_def_cfa_offset 32
+; NO-LIBCALL-NEXT:    .cfi_offset w30, -16
+; NO-LIBCALL-NEXT:    .cfi_offset b8, -24
+; NO-LIBCALL-NEXT:    .cfi_offset b9, -32
+; NO-LIBCALL-NEXT:    fcvt s8, h0
+; NO-LIBCALL-NEXT:    fmov s0, s8
+; NO-LIBCALL-NEXT:    bl sinf
+; NO-LIBCALL-NEXT:    fcvt h9, s0
+; NO-LIBCALL-NEXT:    fmov s0, s8
+; NO-LIBCALL-NEXT:    bl cosf
+; NO-LIBCALL-NEXT:    fmov s1, s0
+; NO-LIBCALL-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; NO-LIBCALL-NEXT:    fmov s0, s9
+; NO-LIBCALL-NEXT:    fcvt h1, s1
+; NO-LIBCALL-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    ret
+  %result = call { half, half } @llvm.sincos.f16(half %a)
+  ret { half, half } %result
+}
+
+define half @test_sincos_f16_only_use_sin(half %a) {
+; CHECK-LABEL: test_sincos_f16_only_use_sin:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    add x1, sp, #8
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldr s0, [sp, #12]
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; NO-LIBCALL-LABEL: test_sincos_f16_only_use_sin:
+; NO-LIBCALL:       // %bb.0:
+; NO-LIBCALL-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; NO-LIBCALL-NEXT:    .cfi_def_cfa_offset 16
+; NO-LIBCALL-NEXT:    .cfi_offset w30, -16
+; NO-LIBCALL-NEXT:    fcvt s0, h0
+; NO-LIBCALL-NEXT:    bl sinf
+; NO-LIBCALL-NEXT:    fcvt h0, s0
+; NO-LIBCALL-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; NO-LIBCALL-NEXT:    ret
+  %result = call { half, half } @llvm.sincos.f16(half %a)
+  %result.0 = extractvalue { half, half } %result, 0
+  ret half %result.0
+}
+
+define half @test_sincos_f16_only_use_cos(half %a) {
+; CHECK-LABEL: test_sincos_f16_only_use_cos:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    add x1, sp, #8
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldr s0, [sp, #8]
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; NO-LIBCALL-LABEL: test_sincos_f16_only_use_cos:
+; NO-LIBCALL:       // %bb.0:
+; NO-LIBCALL-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; NO-LIBCALL-NEXT:    .cfi_def_cfa_offset 16
+; NO-LIBCALL-NEXT:    .cfi_offset w30, -16
+; NO-LIBCALL-NEXT:    fcvt s0, h0
+; NO-LIBCALL-NEXT:    bl cosf
+; NO-LIBCALL-NEXT:    fcvt h0, s0
+; NO-LIBCALL-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; NO-LIBCALL-NEXT:    ret
+  %result = call { half, half } @llvm.sincos.f16(half %a)
+  %result.1 = extractvalue { half, half } %result, 1
+  ret half %result.1
+}
+
+define { <2 x half>, <2 x half> } @test_sincos_v2f16(<2 x half> %a) {
+; CHECK-LABEL: test_sincos_v2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov h1, v0.h[1]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #36
+; CHECK-NEXT:    add x1, sp, #32
+; CHECK-NEXT:    fcvt s0, h1
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #28
+; CHECK-NEXT:    add x1, sp, #24
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    add x1, sp, #40
+; CHECK-NEXT:    mov h0, v0.h[2]
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #60
+; CHECK-NEXT:    add x1, sp, #56
+; CHECK-NEXT:    mov h0, v0.h[3]
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldp s2, s0, [sp, #32]
+; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp s3, s1, [sp, #24]
+; CHECK-NEXT:    fcvt h4, s0
+; CHECK-NEXT:    fcvt h2, s2
+; CHECK-NEXT:    fcvt h0, s1
+; CHECK-NEXT:    fcvt h1, s3
+; CHECK-NEXT:    ldp s5, s3, [sp, #40]
+; CHECK-NEXT:    fcvt h3, s3
+; CHECK-NEXT:    mov v0.h[1], v4.h[0]
+; CHECK-NEXT:    fcvt h4, s5
+; CHECK-NEXT:    mov v1.h[1], v2.h[0]
+; CHECK-NEXT:    ldp s5, s2, [sp, #56]
+; CHECK-NEXT:    mov v0.h[2], v3.h[0]
+; CHECK-NEXT:    fcvt h2, s2
+; CHECK-NEXT:    fcvt h3, s5
+; CHECK-NEXT:    mov v1.h[2], v4.h[0]
+; CHECK-NEXT:    mov v0.h[3], v2.h[0]
+; CHECK-NEXT:    mov v1.h[3], v3.h[0]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+;
+; NO-LIBCALL-LABEL: test_sincos_v2f16:
+; NO-LIBCALL:       // %bb.0:
+; NO-LIBCALL-NEXT:    sub sp, sp, #80
+; NO-LIBCALL-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
+; NO-LIBCALL-NEXT:    .cfi_def_cfa_offset 80
+; NO-LIBCALL-NEXT:    .cfi_offset w30, -16
+; NO-LIBCALL-NEXT:    .cfi_offset b8, -24
+; NO-LIBCALL-NEXT:    .cfi_offset b9, -32
+; NO-LIBCALL-NEXT:    .cfi_offset b10, -40
+; NO-LIBCALL-NEXT:    .cfi_offset b11, -48
+; NO-LIBCALL-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NO-LIBCALL-NEXT:    mov h1, v0.h[1]
+; NO-LIBCALL-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    fcvt s8, h1
+; NO-LIBCALL-NEXT:    fmov s0, s8
+; NO-LIBCALL-NEXT:    bl sinf
+; NO-LIBCALL-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    fcvt h0, s0
+; NO-LIBCALL-NEXT:    fcvt s9, h1
+; NO-LIBCALL-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    fmov s0, s9
+; NO-LIBCALL-NEXT:    bl sinf
+; NO-LIBCALL-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    fcvt h0, s0
+; NO-LIBCALL-NEXT:    mov h1, v1.h[2]
+; NO-LIBCALL-NEXT:    fcvt s10, h1
+; NO-LIBCALL-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    mov v0.h[1], v1.h[0]
+; NO-LIBCALL-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    fmov s0, s10
+; NO-LIBCALL-NEXT:    bl sinf
+; NO-LIBCALL-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    fcvt h0, s0
+; NO-LIBCALL-NEXT:    mov h1, v1.h[3]
+; NO-LIBCALL-NEXT:    fcvt s11, h1
+; NO-LIBCALL-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    mov v1.h[2], v0.h[0]
+; NO-LIBCALL-NEXT:    fmov s0, s11
+; NO-LIBCALL-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    bl sinf
+; NO-LIBCALL-NEXT:    fcvt h0, s0
+; NO-LIBCALL-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    mov v1.h[3], v0.h[0]
+; NO-LIBCALL-NEXT:    fmov s0, s8
+; NO-LIBCALL-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    bl cosf
+; NO-LIBCALL-NEXT:    fcvt h0, s0
+; NO-LIBCALL-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    fmov s0, s9
+; NO-LIBCALL-NEXT:    bl cosf
+; NO-LIBCALL-NEXT:    fcvt h0, s0
+; NO-LIBCALL-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    mov v0.h[1], v1.h[0]
+; NO-LIBCALL-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    fmov s0, s10
+; NO-LIBCALL-NEXT:    bl cosf
+; NO-LIBCALL-NEXT:    fcvt h0, s0
+; NO-LIBCALL-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    mov v1.h[2], v0.h[0]
+; NO-LIBCALL-NEXT:    fmov s0, s11
+; NO-LIBCALL-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    bl cosf
+; NO-LIBCALL-NEXT:    fmov s1, s0
+; NO-LIBCALL-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
+; NO-LIBCALL-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NO-LIBCALL-NEXT:    fcvt h2, s1
+; NO-LIBCALL-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    mov v1.h[3], v2.h[0]
+; NO-LIBCALL-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; NO-LIBCALL-NEXT:    add sp, sp, #80
+; NO-LIBCALL-NEXT:    ret
+  %result = call { <2 x half>, <2 x half> } @llvm.sincos.v2f16(<2 x half> %a)
+  ret { <2 x half>, <2 x half> } %result
+}
+
+define { float, float } @test_sincos_f32(float %a) {
+; CHECK-LABEL: test_sincos_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    add x1, sp, #8
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldp s1, s0, [sp, #8]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; NO-LIBCALL-LABEL: test_sincos_f32:
+; NO-LIBCALL:       // %bb.0:
+; NO-LIBCALL-NEXT:    stp d9, d8, [sp, #-32]! // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; NO-LIBCALL-NEXT:    .cfi_def_cfa_offset 32
+; NO-LIBCALL-NEXT:    .cfi_offset w30, -16
+; NO-LIBCALL-NEXT:    .cfi_offset b8, -24
+; NO-LIBCALL-NEXT:    .cfi_offset b9, -32
+; NO-LIBCALL-NEXT:    fmov s8, s0
+; NO-LIBCALL-NEXT:    bl sinf
+; NO-LIBCALL-NEXT:    fmov s9, s0
+; NO-LIBCALL-NEXT:    fmov s0, s8
+; NO-LIBCALL-NEXT:    bl cosf
+; NO-LIBCALL-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; NO-LIBCALL-NEXT:    fmov s1, s0
+; NO-LIBCALL-NEXT:    fmov s0, s9
+; NO-LIBCALL-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    ret
+  %result = call { float, float } @llvm.sincos.f32(float %a)
+  ret { float, float } %result
+}
+
+define { <3 x float>, <3 x float> } @test_sincos_v3f32(<3 x float> %a) {
+; CHECK-LABEL: test_sincos_v3f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #80
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w21, -24
+; CHECK-NEXT:    .cfi_offset w22, -32
+; CHECK-NEXT:    .cfi_offset w30, -48
+; CHECK-NEXT:    add x0, sp, #20
+; CHECK-NEXT:    add x1, sp, #16
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #28
+; CHECK-NEXT:    add x1, sp, #24
+; CHECK-NEXT:    add x19, sp, #28
+; CHECK-NEXT:    add x20, sp, #24
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    add x1, sp, #40
+; CHECK-NEXT:    add x21, sp, #44
+; CHECK-NEXT:    add x22, sp, #40
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldp s1, s0, [sp, #16]
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    ld1 { v0.s }[1], [x19]
+; CHECK-NEXT:    ld1 { v1.s }[1], [x20]
+; CHECK-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ld1 { v0.s }[2], [x21]
+; CHECK-NEXT:    ld1 { v1.s }[2], [x22]
+; CHECK-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    ret
+;
+; NO-LIBCALL-LABEL: test_sincos_v3f32:
+; NO-LIBCALL:       // %bb.0:
+; NO-LIBCALL-NEXT:    sub sp, sp, #80
+; NO-LIBCALL-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
+; NO-LIBCALL-NEXT:    .cfi_def_cfa_offset 80
+; NO-LIBCALL-NEXT:    .cfi_offset w30, -16
+; NO-LIBCALL-NEXT:    .cfi_offset b8, -24
+; NO-LIBCALL-NEXT:    .cfi_offset b9, -32
+; NO-LIBCALL-NEXT:    mov s8, v0.s[1]
+; NO-LIBCALL-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    fmov s0, s8
+; NO-LIBCALL-NEXT:    bl sinf
+; NO-LIBCALL-NEXT:    // kill: def $s0 killed $s0 def $q0
+; NO-LIBCALL-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; NO-LIBCALL-NEXT:    bl sinf
+; NO-LIBCALL-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    // kill: def $s0 killed $s0 def $q0
+; NO-LIBCALL-NEXT:    mov v0.s[1], v1.s[0]
+; NO-LIBCALL-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    mov s9, v0.s[2]
+; NO-LIBCALL-NEXT:    fmov s0, s9
+; NO-LIBCALL-NEXT:    bl sinf
+; NO-LIBCALL-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    // kill: def $s0 killed $s0 def $q0
+; NO-LIBCALL-NEXT:    mov v1.s[2], v0.s[0]
+; NO-LIBCALL-NEXT:    fmov s0, s8
+; NO-LIBCALL-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    bl cosf
+; NO-LIBCALL-NEXT:    // kill: def $s0 killed $s0 def $q0
+; NO-LIBCALL-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; NO-LIBCALL-NEXT:    bl cosf
+; NO-LIBCALL-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    // kill: def $s0 killed $s0 def $q0
+; NO-LIBCALL-NEXT:    mov v0.s[1], v1.s[0]
+; NO-LIBCALL-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    fmov s0, s9
+; NO-LIBCALL-NEXT:    bl cosf
+; NO-LIBCALL-NEXT:    fmov s2, s0
+; NO-LIBCALL-NEXT:    ldp q0, q1, [sp, #16] // 32-byte Folded Reload
+; NO-LIBCALL-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
+; NO-LIBCALL-NEXT:    mov v1.s[2], v2.s[0]
+; NO-LIBCALL-NEXT:    add sp, sp, #80
+; NO-LIBCALL-NEXT:    ret
+  %result = call { <3 x float>, <3 x float> } @llvm.sincos.v3f32(<3 x float> %a)
+  ret { <3 x float>, <3 x float> } %result
+}
+
+define { <2 x float>, <2 x float> } @test_sincos_v2f32(<2 x float> %a) {
+; CHECK-LABEL: test_sincos_v2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    add x1, sp, #40
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #28
+; CHECK-NEXT:    add x1, sp, #24
+; CHECK-NEXT:    add x19, sp, #28
+; CHECK-NEXT:    add x20, sp, #24
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldp s1, s0, [sp, #40]
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    ld1 { v0.s }[1], [x19]
+; CHECK-NEXT:    ld1 { v1.s }[1], [x20]
+; CHECK-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+;
+; NO-LIBCALL-LABEL: test_sincos_v2f32:
+; NO-LIBCALL:       // %bb.0:
+; NO-LIBCALL-NEXT:    sub sp, sp, #64
+; NO-LIBCALL-NEXT:    str d8, [sp, #48] // 8-byte Folded Spill
+; NO-LIBCALL-NEXT:    str x30, [sp, #56] // 8-byte Folded Spill
+; NO-LIBCALL-NEXT:    .cfi_def_cfa_offset 64
+; NO-LIBCALL-NEXT:    .cfi_offset w30, -8
+; NO-LIBCALL-NEXT:    .cfi_offset b8, -16
+; NO-LIBCALL-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NO-LIBCALL-NEXT:    mov s8, v0.s[1]
+; NO-LIBCALL-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    fmov s0, s8
+; NO-LIBCALL-NEXT:    bl sinf
+; NO-LIBCALL-NEXT:    // kill: def $s0 killed $s0 def $q0
+; NO-LIBCALL-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; NO-LIBCALL-NEXT:    bl sinf
+; NO-LIBCALL-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    // kill: def $s0 killed $s0 def $q0
+; NO-LIBCALL-NEXT:    mov v0.s[1], v1.s[0]
+; NO-LIBCALL-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    fmov s0, s8
+; NO-LIBCALL-NEXT:    bl cosf
+; NO-LIBCALL-NEXT:    // kill: def $s0 killed $s0 def $q0
+; NO-LIBCALL-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; NO-LIBCALL-NEXT:    bl cosf
+; NO-LIBCALL-NEXT:    fmov s1, s0
+; NO-LIBCALL-NEXT:    ldp q2, q0, [sp] // 32-byte Folded Reload
+; NO-LIBCALL-NEXT:    ldr x30, [sp, #56] // 8-byte Folded Reload
+; NO-LIBCALL-NEXT:    ldr d8, [sp, #48] // 8-byte Folded Reload
+; NO-LIBCALL-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NO-LIBCALL-NEXT:    mov v1.s[1], v2.s[0]
+; NO-LIBCALL-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; NO-LIBCALL-NEXT:    add sp, sp, #64
+; NO-LIBCALL-NEXT:    ret
+  %result = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> %a)
+  ret { <2 x float>, <2 x float> } %result
+}
+
+define { double, double } @test_sincos_f64(double %a) {
+; CHECK-LABEL: test_sincos_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #24
+; CHECK-NEXT:    add x1, sp, #8
+; CHECK-NEXT:    bl sincos
+; CHECK-NEXT:    ldr d0, [sp, #24]
+; CHECK-NEXT:    ldr d1, [sp, #8]
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
+;
+; NO-LIBCALL-LABEL: test_sincos_f64:
+; NO-LIBCALL:       // %bb.0:
+; NO-LIBCALL-NEXT:    stp d9, d8, [sp, #-32]! // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; NO-LIBCALL-NEXT:    .cfi_def_cfa_offset 32
+; NO-LIBCALL-NEXT:    .cfi_offset w30, -16
+; NO-LIBCALL-NEXT:    .cfi_offset b8, -24
+; NO-LIBCALL-NEXT:    .cfi_offset b9, -32
+; NO-LIBCALL-NEXT:    fmov d8, d0
+; NO-LIBCALL-NEXT:    bl sin
+; NO-LIBCALL-NEXT:    fmov d9, d0
+; NO-LIBCALL-NEXT:    fmov d0, d8
+; NO-LIBCALL-NEXT:    bl cos
+; NO-LIBCALL-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; NO-LIBCALL-NEXT:    fmov d1, d0
+; NO-LIBCALL-NEXT:    fmov d0, d9
+; NO-LIBCALL-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    ret
+  %result = call { double, double } @llvm.sincos.f64(double %a)
+  ret { double, double } %result
+}
+
+define { <2 x double>, <2 x double> } @test_sincos_v2f64(<2 x double> %a) {
+; CHECK-LABEL: test_sincos_v2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #80
+; CHECK-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    add x0, sp, #56
+; CHECK-NEXT:    add x1, sp, #40
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    bl sincos
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #32
+; CHECK-NEXT:    add x1, sp, #24
+; CHECK-NEXT:    add x19, sp, #32
+; CHECK-NEXT:    add x20, sp, #24
+; CHECK-NEXT:    mov d0, v0.d[1]
+; CHECK-NEXT:    bl sincos
+; CHECK-NEXT:    ldr d0, [sp, #56]
+; CHECK-NEXT:    ldr d1, [sp, #40]
+; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-NEXT:    ld1 { v0.d }[1], [x19]
+; CHECK-NEXT:    ld1 { v1.d }[1], [x20]
+; CHECK-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    ret
+;
+; NO-LIBCALL-LABEL: test_sincos_v2f64:
+; NO-LIBCALL:       // %bb.0:
+; NO-LIBCALL-NEXT:    sub sp, sp, #64
+; NO-LIBCALL-NEXT:    str d8, [sp, #48] // 8-byte Folded Spill
+; NO-LIBCALL-NEXT:    str x30, [sp, #56] // 8-byte Folded Spill
+; NO-LIBCALL-NEXT:    .cfi_def_cfa_offset 64
+; NO-LIBCALL-NEXT:    .cfi_offset w30, -8
+; NO-LIBCALL-NEXT:    .cfi_offset b8, -16
+; NO-LIBCALL-NEXT:    mov d8, v0.d[1]
+; NO-LIBCALL-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    fmov d0, d8
+; NO-LIBCALL-NEXT:    bl sin
+; NO-LIBCALL-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NO-LIBCALL-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NO-LIBCALL-NEXT:    bl sin
+; NO-LIBCALL-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NO-LIBCALL-NEXT:    mov v0.d[1], v1.d[0]
+; NO-LIBCALL-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    fmov d0, d8
+; NO-LIBCALL-NEXT:    bl cos
+; NO-LIBCALL-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NO-LIBCALL-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; NO-LIBCALL-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; NO-LIBCALL-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NO-LIBCALL-NEXT:    bl cos
+; NO-LIBCALL-NEXT:    fmov d1, d0
+; NO-LIBCALL-NEXT:    ldp q2, q0, [sp] // 32-byte Folded Reload
+; NO-LIBCALL-NEXT:    ldr x30, [sp, #56] // 8-byte Folded Reload
+; NO-LIBCALL-NEXT:    ldr d8, [sp, #48] // 8-byte Folded Reload
+; NO-LIBCALL-NEXT:    mov v1.d[1], v2.d[0]
+; NO-LIBCALL-NEXT:    add sp, sp, #64
+; NO-LIBCALL-NEXT:    ret
+  %result = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> %a)
+  ret { <2 x double>, <2 x double> } %result
+}
diff --git a/llvm/test/CodeGen/ARM/llvm.sincos.ll b/llvm/test/CodeGen/ARM/llvm.sincos.ll
new file mode 100644
index 000000000000..9628405df6bc
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/llvm.sincos.ll
@@ -0,0 +1,223 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=thumbv7-gnu-linux < %s | FileCheck -check-prefixes=CHECK %s
+
+define { half, half } @test_sincos_f16(half %a) {
+; CHECK-LABEL: test_sincos_f16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    add r1, sp, #4
+; CHECK-NEXT:    mov r2, sp
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldr r0, [sp, #4]
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    ldr r0, [sp]
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    mov r1, r0
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop {r4, pc}
+  %result = call { half, half } @llvm.sincos.f16(half %a)
+  ret { half, half } %result
+}
+
+define half @test_sincos_f16_only_use_sin(half %a) {
+; CHECK-LABEL: test_sincos_f16_only_use_sin:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    add r1, sp, #4
+; CHECK-NEXT:    mov r2, sp
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldr r0, [sp, #4]
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop {r7, pc}
+  %result = call { half, half } @llvm.sincos.f16(half %a)
+  %result.0 = extractvalue { half, half } %result, 0
+  ret half %result.0
+}
+
+define half @test_sincos_f16_only_use_cos(half %a) {
+; CHECK-LABEL: test_sincos_f16_only_use_cos:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    add r1, sp, #4
+; CHECK-NEXT:    mov r2, sp
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldr r0, [sp]
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop {r7, pc}
+  %result = call { half, half } @llvm.sincos.f16(half %a)
+  %result.1 = extractvalue { half, half } %result, 1
+  ret half %result.1
+}
+
+define { <2 x half>, <2 x half> } @test_sincos_v2f16(<2 x half> %a) {
+; CHECK-LABEL: test_sincos_v2f16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    vpush {d8}
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    add r1, sp, #12
+; CHECK-NEXT:    add r2, sp, #8
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    add r1, sp, #4
+; CHECK-NEXT:    mov r2, sp
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldr r0, [sp, #12]
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    ldr r1, [sp, #4]
+; CHECK-NEXT:    strh.w r0, [sp, #22]
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    strh.w r0, [sp, #20]
+; CHECK-NEXT:    add r0, sp, #20
+; CHECK-NEXT:    vld1.32 {d8[0]}, [r0:32]
+; CHECK-NEXT:    ldr r0, [sp, #8]
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    ldr r1, [sp]
+; CHECK-NEXT:    strh.w r0, [sp, #18]
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    strh.w r0, [sp, #16]
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vmovl.u16 q9, d8
+; CHECK-NEXT:    vld1.32 {d16[0]}, [r0:32]
+; CHECK-NEXT:    vmovl.u16 q8, d16
+; CHECK-NEXT:    vmov.32 r0, d18[0]
+; CHECK-NEXT:    vmov.32 r1, d18[1]
+; CHECK-NEXT:    vmov.32 r2, d16[0]
+; CHECK-NEXT:    vmov.32 r3, d16[1]
+; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    vpop {d8}
+; CHECK-NEXT:    pop {r4, pc}
+  %result = call { <2 x half>, <2 x half> } @llvm.sincos.v2f16(<2 x half> %a)
+  ret { <2 x half>, <2 x half> } %result
+}
+
+define { float, float } @test_sincos_f32(float %a) {
+; CHECK-LABEL: test_sincos_f32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    add r1, sp, #4
+; CHECK-NEXT:    mov r2, sp
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    ldrd r1, r0, [sp], #8
+; CHECK-NEXT:    pop {r7, pc}
+  %result = call { float, float } @llvm.sincos.f32(float %a)
+  ret { float, float } %result
+}
+
+define { <2 x float>, <2 x float> } @test_sincos_v2f32(<2 x float> %a) {
+; CHECK-LABEL: test_sincos_v2f32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vpush {d8}
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    vmov d8, r0, r1
+; CHECK-NEXT:    add r1, sp, #4
+; CHECK-NEXT:    mov r2, sp
+; CHECK-NEXT:    vmov r0, s17
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    add r1, sp, #12
+; CHECK-NEXT:    add r2, sp, #8
+; CHECK-NEXT:    bl sincosf
+; CHECK-NEXT:    vldr s1, [sp, #4]
+; CHECK-NEXT:    vldr s3, [sp]
+; CHECK-NEXT:    vldr s0, [sp, #12]
+; CHECK-NEXT:    vldr s2, [sp, #8]
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    vpop {d8}
+; CHECK-NEXT:    pop {r7, pc}
+  %result = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> %a)
+  ret { <2 x float>, <2 x float> } %result
+}
+
+define { double, double } @test_sincos_f64(double %a) {
+; CHECK-LABEL: test_sincos_f64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    add r2, sp, #8
+; CHECK-NEXT:    mov r3, sp
+; CHECK-NEXT:    bl sincos
+; CHECK-NEXT:    ldrd r0, r1, [sp, #8]
+; CHECK-NEXT:    ldrd r2, r3, [sp], #16
+; CHECK-NEXT:    pop {r7, pc}
+  %result = call { double, double } @llvm.sincos.f64(double %a)
+  ret { double, double } %result
+}
+
+define { <2 x double>, <2 x double> } @test_sincos_v2f64(<2 x double> %a) {
+; CHECK-LABEL: test_sincos_v2f64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    mov r1, r3
+; CHECK-NEXT:    mov r12, r2
+; CHECK-NEXT:    add r2, sp, #24
+; CHECK-NEXT:    add r3, sp, #16
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r12
+; CHECK-NEXT:    bl sincos
+; CHECK-NEXT:    ldrd r0, r1, [sp, #40]
+; CHECK-NEXT:    add r2, sp, #8
+; CHECK-NEXT:    mov r3, sp
+; CHECK-NEXT:    bl sincos
+; CHECK-NEXT:    vldr d19, [sp, #8]
+; CHECK-NEXT:    vldr d18, [sp, #24]
+; CHECK-NEXT:    vldr d17, [sp]
+; CHECK-NEXT:    vldr d16, [sp, #16]
+; CHECK-NEXT:    vst1.64 {d18, d19}, [r4]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r4]
+; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    pop {r4, pc}
+  %result = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> %a)
+  ret { <2 x double>, <2 x double> } %result
+}
+
+define { fp128, fp128 } @test_sincos_f128(fp128 %a) {
+; CHECK-LABEL: test_sincos_f128:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    sub sp, #40
+; CHECK-NEXT:    mov r12, r3
+; CHECK-NEXT:    ldr r3, [sp, #56]
+; CHECK-NEXT:    add.w lr, sp, #8
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    add r0, sp, #24
+; CHECK-NEXT:    strd r0, lr, [sp]
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    mov r1, r2
+; CHECK-NEXT:    mov r2, r12
+; CHECK-NEXT:    bl sincosl
+; CHECK-NEXT:    ldrd r2, r3, [sp, #16]
+; CHECK-NEXT:    ldrd r12, r1, [sp, #8]
+; CHECK-NEXT:    str r3, [r4, #28]
+; CHECK-NEXT:    ldrd r3, r5, [sp, #32]
+; CHECK-NEXT:    ldrd lr, r0, [sp, #24]
+; CHECK-NEXT:    strd r1, r2, [r4, #20]
+; CHECK-NEXT:    add.w r1, r4, #8
+; CHECK-NEXT:    stm.w r1, {r3, r5, r12}
+; CHECK-NEXT:    strd lr, r0, [r4]
+; CHECK-NEXT:    add sp, #40
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+  %result = call { fp128, fp128 } @llvm.sincos.f16(fp128 %a)
+  ret { fp128, fp128 } %result
+}
-- 
GitLab


From 32aa782ea297b3e0ec090cf8fc0055d00c99d24b Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 29 Oct 2024 10:57:36 +0000
Subject: [PATCH 414/693] [PowerPC] copysignl.ll - regenerate to reduce the
 diff in #111269

---
 llvm/test/CodeGen/PowerPC/copysignl.ll | 138 ++++++++++++++++++-------
 1 file changed, 103 insertions(+), 35 deletions(-)

diff --git a/llvm/test/CodeGen/PowerPC/copysignl.ll b/llvm/test/CodeGen/PowerPC/copysignl.ll
index 427826daa2c6..40ed3d803094 100644
--- a/llvm/test/CodeGen/PowerPC/copysignl.ll
+++ b/llvm/test/CodeGen/PowerPC/copysignl.ll
@@ -1,82 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple=powerpc64-unknown-linux-gnu -mattr=-vsx < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple=powerpc64-unknown-linux-gnu -mattr=+vsx < %s | FileCheck %s -check-prefix=CHECK-VSX
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
 define double @foo_d_ll(ppc_fp128 %a, ppc_fp128 %b) #0 {
+; CHECK-LABEL: foo_d_ll:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fcpsgn 1, 3, 1
+; CHECK-NEXT:    blr
+;
+; CHECK-VSX-LABEL: foo_d_ll:
+; CHECK-VSX:       # %bb.0: # %entry
+; CHECK-VSX-NEXT:    xscpsgndp 1, 3, 1
+; CHECK-VSX-NEXT:    blr
 entry:
   %call = tail call ppc_fp128 @copysignl(ppc_fp128 %a, ppc_fp128 %b) #0
   %conv = fptrunc ppc_fp128 %call to double
   ret double %conv
-
-; CHECK-LABEL: @foo_d_ll
-; CHECK: fcpsgn 1, 3, 1
-; CHECK: blr
-; CHECK-VSX-LABEL: @foo_d_ll
-; CHECK-VSX: xscpsgndp 1, 3, 1
-; CHECK-VSX: blr
 }
 
 declare ppc_fp128 @copysignl(ppc_fp128, ppc_fp128) #0
 
 define double @foo_dl(double %a, ppc_fp128 %b) #0 {
+; CHECK-LABEL: foo_dl:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fcpsgn 1, 2, 1
+; CHECK-NEXT:    blr
+;
+; CHECK-VSX-LABEL: foo_dl:
+; CHECK-VSX:       # %bb.0: # %entry
+; CHECK-VSX-NEXT:    xscpsgndp 1, 2, 1
+; CHECK-VSX-NEXT:    blr
 entry:
   %conv = fptrunc ppc_fp128 %b to double
   %call = tail call double @copysign(double %a, double %conv) #0
   ret double %call
-
-; CHECK-LABEL: @foo_dl
-; CHECK: fcpsgn 1, 2, 1
-; CHECK: blr
-; CHECK-VSX-LABEL: @foo_dl
-; CHECK-VSX: xscpsgndp 1, 2, 1
-; CHECK-VSX: blr
 }
 
 declare double @copysign(double, double) #0
 
 define ppc_fp128 @foo_ll(double %a, ppc_fp128 %b) #0 {
+; CHECK-LABEL: foo_ll:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr 0
+; CHECK-NEXT:    stdu 1, -112(1)
+; CHECK-NEXT:    fmr 3, 2
+; CHECK-NEXT:    addis 3, 2, .LCPI2_0@toc@ha
+; CHECK-NEXT:    std 0, 128(1)
+; CHECK-NEXT:    lfs 2, .LCPI2_0@toc@l(3)
+; CHECK-NEXT:    bl copysignl
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi 1, 1, 112
+; CHECK-NEXT:    ld 0, 16(1)
+; CHECK-NEXT:    mtlr 0
+; CHECK-NEXT:    blr
+;
+; CHECK-VSX-LABEL: foo_ll:
+; CHECK-VSX:       # %bb.0: # %entry
+; CHECK-VSX-NEXT:    mflr 0
+; CHECK-VSX-NEXT:    stdu 1, -112(1)
+; CHECK-VSX-NEXT:    fmr 3, 2
+; CHECK-VSX-NEXT:    xxlxor 2, 2, 2
+; CHECK-VSX-NEXT:    std 0, 128(1)
+; CHECK-VSX-NEXT:    bl copysignl
+; CHECK-VSX-NEXT:    nop
+; CHECK-VSX-NEXT:    addi 1, 1, 112
+; CHECK-VSX-NEXT:    ld 0, 16(1)
+; CHECK-VSX-NEXT:    mtlr 0
+; CHECK-VSX-NEXT:    blr
 entry:
   %conv = fpext double %a to ppc_fp128
   %call = tail call ppc_fp128 @copysignl(ppc_fp128 %conv, ppc_fp128 %b) #0
   ret ppc_fp128 %call
-
-; CHECK-LABEL: @foo_ll
-; CHECK: bl copysignl
-; CHECK: blr
-; CHECK-VSX-LABEL: @foo_ll
-; CHECK-VSX: bl copysignl
-; CHECK-VSX: blr
 }
 
 define ppc_fp128 @foo_ld(double %a, double %b) #0 {
+; CHECK-LABEL: foo_ld:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr 0
+; CHECK-NEXT:    stdu 1, -112(1)
+; CHECK-NEXT:    fmr 3, 2
+; CHECK-NEXT:    addis 3, 2, .LCPI3_0@toc@ha
+; CHECK-NEXT:    std 0, 128(1)
+; CHECK-NEXT:    lfs 2, .LCPI3_0@toc@l(3)
+; CHECK-NEXT:    bl copysignl
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi 1, 1, 112
+; CHECK-NEXT:    ld 0, 16(1)
+; CHECK-NEXT:    mtlr 0
+; CHECK-NEXT:    blr
+;
+; CHECK-VSX-LABEL: foo_ld:
+; CHECK-VSX:       # %bb.0: # %entry
+; CHECK-VSX-NEXT:    mflr 0
+; CHECK-VSX-NEXT:    stdu 1, -112(1)
+; CHECK-VSX-NEXT:    fmr 3, 2
+; CHECK-VSX-NEXT:    xxlxor 2, 2, 2
+; CHECK-VSX-NEXT:    std 0, 128(1)
+; CHECK-VSX-NEXT:    bl copysignl
+; CHECK-VSX-NEXT:    nop
+; CHECK-VSX-NEXT:    addi 1, 1, 112
+; CHECK-VSX-NEXT:    ld 0, 16(1)
+; CHECK-VSX-NEXT:    mtlr 0
+; CHECK-VSX-NEXT:    blr
 entry:
   %conv = fpext double %a to ppc_fp128
   %conv1 = fpext double %b to ppc_fp128
   %call = tail call ppc_fp128 @copysignl(ppc_fp128 %conv, ppc_fp128 %conv1) #0
   ret ppc_fp128 %call
-
-; CHECK-LABEL: @foo_ld
-; CHECK: bl copysignl
-; CHECK: blr
-; CHECK-VSX-LABEL: @foo_ld
-; CHECK-VSX: bl copysignl
-; CHECK-VSX: blr
 }
 
 define ppc_fp128 @foo_lf(double %a, float %b) #0 {
+; CHECK-LABEL: foo_lf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr 0
+; CHECK-NEXT:    stdu 1, -112(1)
+; CHECK-NEXT:    fmr 3, 2
+; CHECK-NEXT:    addis 3, 2, .LCPI4_0@toc@ha
+; CHECK-NEXT:    std 0, 128(1)
+; CHECK-NEXT:    lfs 2, .LCPI4_0@toc@l(3)
+; CHECK-NEXT:    bl copysignl
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi 1, 1, 112
+; CHECK-NEXT:    ld 0, 16(1)
+; CHECK-NEXT:    mtlr 0
+; CHECK-NEXT:    blr
+;
+; CHECK-VSX-LABEL: foo_lf:
+; CHECK-VSX:       # %bb.0: # %entry
+; CHECK-VSX-NEXT:    mflr 0
+; CHECK-VSX-NEXT:    stdu 1, -112(1)
+; CHECK-VSX-NEXT:    fmr 3, 2
+; CHECK-VSX-NEXT:    xxlxor 2, 2, 2
+; CHECK-VSX-NEXT:    std 0, 128(1)
+; CHECK-VSX-NEXT:    bl copysignl
+; CHECK-VSX-NEXT:    nop
+; CHECK-VSX-NEXT:    addi 1, 1, 112
+; CHECK-VSX-NEXT:    ld 0, 16(1)
+; CHECK-VSX-NEXT:    mtlr 0
+; CHECK-VSX-NEXT:    blr
 entry:
   %conv = fpext double %a to ppc_fp128
   %conv1 = fpext float %b to ppc_fp128
   %call = tail call ppc_fp128 @copysignl(ppc_fp128 %conv, ppc_fp128 %conv1) #0
   ret ppc_fp128 %call
-
-; CHECK-LABEL: @foo_lf
-; CHECK: bl copysignl
-; CHECK: blr
-; CHECK-VSX-LABEL: @foo_lf
-; CHECK-VSX: bl copysignl
-; CHECK-VSX: blr
 }
 
 attributes #0 = { nounwind readnone }
-- 
GitLab


From f537792f3f4977c8bc887b17ffc25e93833e7d0d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 29 Oct 2024 11:00:35 +0000
Subject: [PATCH 415/693] [X86] Refactor the SSE intrinsics constexpr tests to
 simplify future expansion (#112578)

I'm hoping to make a large proportion of the SSE/AVX intrinsics usable in constant expressions - eventually anything that doesn't touch memory or system settings - making it much easier to utilize SSE/AVX intrinsics in various math libraries etc.

My initial implementation placed the tests at the end of the test file, similar to how smaller files already handle their tests.

However, what I'm finding is that this approach doesn't scale when trying to track coverage of so many intrinsics - many keep getting missed, and it gets messy; so what I'm proposing is to instead keep each intrinsic's generic IR test and its constexpr tests together to make them easier to track together, wrapping the static_assert inside a macro to disable on C and pre-C++11 tests.

I'm open to alternative suggestions before I invest too much time getting this work done :)
---
 clang/test/CodeGen/X86/builtin_test_helpers.h |  25 ++++
 clang/test/CodeGen/X86/sse-builtins.c         | 127 +++++-----------
 clang/test/CodeGen/X86/sse2-builtins.c        | 139 ++++--------------
 clang/test/CodeGen/X86/sse3-builtins.c        |  23 +--
 4 files changed, 95 insertions(+), 219 deletions(-)
 create mode 100644 clang/test/CodeGen/X86/builtin_test_helpers.h

diff --git a/clang/test/CodeGen/X86/builtin_test_helpers.h b/clang/test/CodeGen/X86/builtin_test_helpers.h
new file mode 100644
index 000000000000..043b6ecbc69f
--- /dev/null
+++ b/clang/test/CodeGen/X86/builtin_test_helpers.h
@@ -0,0 +1,25 @@
+/* Helper methods for builtin intrinsic tests */
+
+#include <immintrin.h>
+
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+
+constexpr bool match_m128(__m128 v, float x, float y, float z, float w) {
+  return v[0] == x && v[1] == y && v[2] == z && v[3] == w;
+}
+
+constexpr bool match_m128d(__m128d v, double x, double y) {
+  return v[0] == x && v[1] == y;
+}
+
+constexpr bool match_m128i(__m128i v, unsigned long long x, unsigned long long y) {
+  return v[0] == x && v[1] == y;
+}
+
+#define TEST_CONSTEXPR(...) static_assert(__VA_ARGS__)
+
+#else
+
+#define TEST_CONSTEXPR(...)
+
+#endif
\ No newline at end of file
diff --git a/clang/test/CodeGen/X86/sse-builtins.c b/clang/test/CodeGen/X86/sse-builtins.c
index 391e049a6ae3..f779ab07a266 100644
--- a/clang/test/CodeGen/X86/sse-builtins.c
+++ b/clang/test/CodeGen/X86/sse-builtins.c
@@ -5,6 +5,7 @@
 
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 // NOTE: This should match the tests in llvm/test/CodeGen/X86/sse-intrinsics-fast-isel.ll
 
@@ -13,6 +14,7 @@ __m128 test_mm_add_ps(__m128 A, __m128 B) {
   // CHECK: fadd <4 x float>
   return _mm_add_ps(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_add_ps((__m128){+1.0f, +0.0f, +2.0f, +4.0f}, (__m128){+8.0f, +4.0f, +2.0f, +1.0f}), +9.0f, +4.0f, +4.0f, +5.0f));
 
 __m128 test_mm_add_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_add_ss
@@ -22,12 +24,14 @@ __m128 test_mm_add_ss(__m128 A, __m128 B) {
   // CHECK: insertelement <4 x float> %{{.*}}, float %{{.*}}, i32 0
   return _mm_add_ss(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_add_ss((__m128){+1.0f, +0.0f, +2.0f, +4.0f}, (__m128){+8.0f, +4.0f, +2.0f, +1.0f}), +9.0f, +0.0f, +2.0f, +4.0f));
 
 __m128 test_mm_and_ps(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_and_ps
   // CHECK: and <4 x i32>
   return _mm_and_ps(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_and_ps((__m128){-4.0f, -5.0f, +6.0f, +7.0f}, (__m128){+0.0f, -0.0f, -0.0f, +7.0f}), -0.0f, -0.0f, +0.0f, +7.0f));
 
 __m128 test_mm_andnot_ps(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_andnot_ps
@@ -35,6 +39,7 @@ __m128 test_mm_andnot_ps(__m128 A, __m128 B) {
   // CHECK: and <4 x i32>
   return _mm_andnot_ps(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_andnot_ps((__m128){-4.0f, -5.0f, +6.0f, +7.0f}, (__m128){+0.0f, -0.0f, -0.0f, +7.0f}), +0.0f, +0.0f, +0.0f, +0.0f));
 
 __m128 test_mm_cmp_ps_eq_oq(__m128 a, __m128 b) {
   // CHECK-LABEL: test_mm_cmp_ps_eq_oq
@@ -322,6 +327,15 @@ __m128 test_mm_cvtsi32_ss(__m128 A, int B) {
   // CHECK: insertelement <4 x float> %{{.*}}, float %{{.*}}, i32 0
   return _mm_cvtsi32_ss(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_cvtsi32_ss((__m128){+1.0f, +0.0f, +2.0f, +4.0f}, 42), +42.0f, +0.0f, +2.0f, +4.0f));
+
+__m128 test_mm_cvt_si2ss(__m128 A, int B) {
+  // CHECK-LABEL: test_mm_cvt_si2ss
+  // CHECK: sitofp i32 %{{.*}} to float
+  // CHECK: insertelement <4 x float> %{{.*}}, float %{{.*}}, i32 0
+  return _mm_cvt_si2ss(A, B);
+}
+TEST_CONSTEXPR(match_m128(_mm_cvt_si2ss((__m128){+4.0f, +2.0f, +0.0f, +4.0f}, -99), -99.0f, +2.0f, +0.0f, +4.0f));
 
 #ifdef __x86_64__
 __m128 test_mm_cvtsi64_ss(__m128 A, long long B) {
@@ -330,6 +344,7 @@ __m128 test_mm_cvtsi64_ss(__m128 A, long long B) {
   // CHECK: insertelement <4 x float> %{{.*}}, float %{{.*}}, i32 0
   return _mm_cvtsi64_ss(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_cvtsi64_ss((__m128){+1.0f, +0.0f, +2.0f, +4.0f}, 555), +555.0f, +0.0f, +2.0f, +4.0f));
 #endif
 
 float test_mm_cvtss_f32(__m128 A) {
@@ -337,6 +352,7 @@ float test_mm_cvtss_f32(__m128 A) {
   // CHECK: extractelement <4 x float> %{{.*}}, i32 0
   return _mm_cvtss_f32(A);
 }
+TEST_CONSTEXPR(_mm_cvtss_f32((__m128){+8.0f, +4.0f, +2.0f, +1.0f}) == +8.0f);
 
 int test_mm_cvtss_si32(__m128 A) {
   // CHECK-LABEL: test_mm_cvtss_si32
@@ -377,6 +393,7 @@ __m128 test_mm_div_ps(__m128 A, __m128 B) {
   // CHECK: fdiv <4 x float>
   return _mm_div_ps(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_div_ps((__m128){+1.0f, +0.0f, +2.0f, +4.0f}, (__m128){+8.0f, +4.0f, +2.0f, +1.0f}), +0.125f, +0.0f, +1.0f, +4.0f));
 
 __m128 test_mm_div_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_div_ss
@@ -386,6 +403,7 @@ __m128 test_mm_div_ss(__m128 A, __m128 B) {
   // CHECK: insertelement <4 x float> %{{.*}}, float %{{.*}}, i32 0
   return _mm_div_ss(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_div_ss((__m128){+1.0f, +5.0f, +2.0f, +4.0f}, (__m128){+8.0f, +4.0f, +2.0f, +1.0f}), +0.125f, +5.0f, +2.0f, +4.0f));
 
 unsigned int test_MM_GET_EXCEPTION_MASK(void) {
   // CHECK-LABEL: test_MM_GET_EXCEPTION_MASK
@@ -517,18 +535,21 @@ __m128 test_mm_move_ss(__m128 A, __m128 B) {
   // CHECK: insertelement <4 x float> %{{.*}}, float %{{.*}}, i32 0
   return _mm_move_ss(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_move_ss((__m128){+1.0f, +0.0f, +2.0f, +4.0f}, (__m128){+8.0f, +4.0f, +2.0f, +1.0f}), +8.0f, +0.0f, +2.0f, +4.0f));
 
 __m128 test_mm_movehl_ps(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_movehl_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 6, i32 7, i32 2, i32 3>
   return _mm_movehl_ps(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_movehl_ps((__m128){+1.0f, +0.0f, +2.0f, +4.0f}, (__m128){+8.0f, +4.0f, +2.0f, +1.0f}), +2.0f, +1.0f, +2.0f, +4.0f));
 
 __m128 test_mm_movelh_ps(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_movelh_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   return _mm_movelh_ps(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_movelh_ps((__m128){+1.0f, +0.0f, +2.0f, +4.0f}, (__m128){+8.0f, +4.0f, +2.0f, +1.0f}), +1.0f, +0.0f, +8.0f, +4.0f));
 
 int test_mm_movemask_ps(__m128 A) {
   // CHECK-LABEL: test_mm_movemask_ps
@@ -541,6 +562,7 @@ __m128 test_mm_mul_ps(__m128 A, __m128 B) {
   // CHECK: fmul <4 x float>
   return _mm_mul_ps(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_mul_ps((__m128){+1.0f, +0.0f, +2.0f, +4.0f}, (__m128){+8.0f, +4.0f, +2.0f, +1.0f}), +8.0f, +0.0f, +4.0f, +4.0f));
 
 __m128 test_mm_mul_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_mul_ss
@@ -550,12 +572,14 @@ __m128 test_mm_mul_ss(__m128 A, __m128 B) {
   // CHECK: insertelement <4 x float> %{{.*}}, float %{{.*}}, i32 0
   return _mm_mul_ss(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_mul_ps((__m128){+1.0f, +0.0f, +2.0f, +4.0f}, (__m128){+8.0f, +4.0f, +2.0f, +1.0f}), +8.0f, +0.0f, +4.0f, +4.0f));
 
 __m128 test_mm_or_ps(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_or_ps
   // CHECK: or <4 x i32>
   return _mm_or_ps(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_or_ps((__m128){-4.0f, -5.0f, +6.0f, +7.0f}, (__m128){+0.0f, -0.0f, -0.0f, +7.0f}), -4.0f, -5.0f, -6.0f, +7.0f));
 
 void test_mm_prefetch(char const* p) {
   // CHECK-LABEL: test_mm_prefetch
@@ -628,6 +652,7 @@ __m128 test_mm_set_ps(float A, float B, float C, float D) {
   // CHECK: insertelement <4 x float> {{.*}}, float {{.*}}, i32 3
   return _mm_set_ps(A, B, C, D);
 }
+TEST_CONSTEXPR(match_m128(_mm_set_ps(+0.0f, +1.0f, +2.0f, +3.0f), +3.0f, +2.0f, +1.0f, +.0f));
 
 __m128 test_mm_set_ps1(float A) {
   // CHECK-LABEL: test_mm_set_ps1
@@ -637,6 +662,7 @@ __m128 test_mm_set_ps1(float A) {
   // CHECK: insertelement <4 x float> {{.*}}, float {{.*}}, i32 3
   return _mm_set_ps1(A);
 }
+TEST_CONSTEXPR(match_m128(_mm_set_ps1(-2.0f), -2.0f, -2.0f, -2.0f, -2.0f));
 
 void test_MM_SET_ROUNDING_MODE(unsigned int A) {
   // CHECK-LABEL: test_MM_SET_ROUNDING_MODE
@@ -657,6 +683,7 @@ __m128 test_mm_set_ss(float A) {
   // CHECK: insertelement <4 x float> {{.*}}, float 0.000000e+00, i32 3
   return _mm_set_ss(A);
 }
+TEST_CONSTEXPR(match_m128(_mm_set_ss(1.0f), +1.0f, +0.0f, +0.0f, +0.0f));
 
 __m128 test_mm_set1_ps(float A) {
   // CHECK-LABEL: test_mm_set1_ps
@@ -666,6 +693,7 @@ __m128 test_mm_set1_ps(float A) {
   // CHECK: insertelement <4 x float> {{.*}}, float {{.*}}, i32 3
   return _mm_set1_ps(A);
 }
+TEST_CONSTEXPR(match_m128(_mm_set1_ps(2.0f), +2.0f, +2.0f, +2.0f, +2.0f));
 
 void test_mm_setcsr(unsigned int A) {
   // CHECK-LABEL: test_mm_setcsr
@@ -682,12 +710,14 @@ __m128 test_mm_setr_ps(float A, float B, float C, float D) {
   // CHECK: insertelement <4 x float> {{.*}}, float {{.*}}, i32 3
   return _mm_setr_ps(A, B, C, D);
 }
+TEST_CONSTEXPR(match_m128(_mm_setr_ps(+0.0f, +1.0f, +2.0f, +3.0f), +0.0f, +1.0f, +2.0f, +3.0f));
 
 __m128 test_mm_setzero_ps(void) {
   // CHECK-LABEL: test_mm_setzero_ps
   // CHECK: store <4 x float> zeroinitializer
   return _mm_setzero_ps();
 }
+TEST_CONSTEXPR(match_m128(_mm_setzero_ps(), +0.0f, +0.0f, +0.0f, +0.0f));
 
 void test_mm_sfence(void) {
   // CHECK-LABEL: test_mm_sfence
@@ -787,6 +817,7 @@ __m128 test_mm_sub_ps(__m128 A, __m128 B) {
   // CHECK: fsub <4 x float>
   return _mm_sub_ps(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_sub_ps((__m128){+1.0f, +0.0f, +2.0f, +4.0f}, (__m128){+8.0f, +4.0f, +2.0f, +1.0f}), -7.0f, -4.0f, +0.0f, +3.0f));
 
 __m128 test_mm_sub_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_sub_ss
@@ -796,6 +827,7 @@ __m128 test_mm_sub_ss(__m128 A, __m128 B) {
   // CHECK: insertelement <4 x float> %{{.*}}, float %{{.*}}, i32 0
   return _mm_sub_ss(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_sub_ss((__m128){+1.0f, +0.0f, +2.0f, +4.0f}, (__m128){+8.0f, +4.0f, +2.0f, +1.0f}), -7.0f, +0.0f, +2.0f, +4.0f));
 
 void test_MM_TRANSPOSE4_PS(__m128 *A, __m128 *B, __m128 *C, __m128 *D) {
   // CHECK-LABEL: test_MM_TRANSPOSE4_PS
@@ -857,107 +889,18 @@ __m128 test_mm_unpackhi_ps(__m128 A, __m128 B) {
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
   return _mm_unpackhi_ps(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_unpackhi_ps((__m128){+1.0f, +0.0f, +2.0f, +4.0f}, (__m128){+8.0f, +4.0f, +2.0f, +1.0f}), +2.0f, +2.0f, +4.0f, +1.0f));
 
 __m128 test_mm_unpacklo_ps(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_unpacklo_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
   return _mm_unpacklo_ps(A, B);
 }
+TEST_CONSTEXPR(match_m128(_mm_unpacklo_ps((__m128){+1.0f, +0.0f, +2.0f, +4.0f}, (__m128){+8.0f, +4.0f, +2.0f, +1.0f}), +1.0f, +8.0f, +0.0f, +4.0f));
 
 __m128 test_mm_xor_ps(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_xor_ps
   // CHECK: xor <4 x i32>
   return _mm_xor_ps(A, B);
 }
-
-// Test constexpr handling.
-#if defined(__cplusplus) && (__cplusplus >= 201103L)
-
-void test_constexpr() {
-  constexpr __m128 k1 {+1.0f,+0.0f,+2.0f,+4.0f};
-  constexpr __m128 k2 {+8.0f,+4.0f,+2.0f,+1.0f};
-  constexpr __m128 k3 {-4.0f,-5.0f,+6.0f,+7.0f};
-  constexpr __m128 k4 {+0.0f,-0.0f,-0.0f,+0.0f};
-
-  constexpr __m128 v_mm_set_ss = _mm_set_ss(1.0f);
-  static_assert(v_mm_set_ss[0] == +1.0f && v_mm_set_ss[1] == +0.0f && v_mm_set_ss[2] == +0.0f && v_mm_set_ss[3] == +0.0f);
-
-  constexpr __m128 v_mm_set1_ps = _mm_set1_ps(2.0f);
-  static_assert(v_mm_set1_ps[0] == +2.0f && v_mm_set1_ps[1] == +2.0f && v_mm_set1_ps[2] == +2.0f && v_mm_set1_ps[3] == +2.0f);
-
-  constexpr __m128 v_mm_set_ps1 = _mm_set_ps1(-2.0f);
-  static_assert(v_mm_set_ps1[0] == -2.0f && v_mm_set_ps1[1] == -2.0f && v_mm_set_ps1[2] == -2.0f && v_mm_set_ps1[3] == -2.0f);
-
-  constexpr __m128 v_mm_set_ps = _mm_set_ps(+0.0f, +1.0f, +2.0f, +3.0f);
-  static_assert(v_mm_set_ps[0] == +3.0f && v_mm_set_ps[1] == +2.0f && v_mm_set_ps[2] == +1.0f && v_mm_set_ps[3] == +0.0f);
-
-  constexpr __m128 v_mm_setr_ps = _mm_setr_ps(+0.0f, +1.0f, +2.0f, +3.0f);
-  static_assert(v_mm_setr_ps[0] == +0.0f && v_mm_setr_ps[1] == +1.0f && v_mm_setr_ps[2] == +2.0f && v_mm_setr_ps[3] == +3.0f);
-
-  constexpr __m128 v_mm_setzero_ps = _mm_setzero_ps();
-  static_assert(v_mm_setzero_ps[0] == +0.0f && v_mm_setzero_ps[1] == +0.0f && v_mm_setzero_ps[2] == +0.0f && v_mm_setzero_ps[3] == +0.0f);
-
-  constexpr __m128 v_mm_add_ss = _mm_add_ss(k1, k2);
-  static_assert(v_mm_add_ss[0] == +9.0f && v_mm_add_ss[1] == +0.0f && v_mm_add_ss[2] == +2.0f && v_mm_add_ss[3] == +4.0f);
-
-  constexpr __m128 v_mm_add_ps = _mm_add_ps(k1, k2);
-  static_assert(v_mm_add_ps[0] == +9.0f && v_mm_add_ps[1] == +4.0f && v_mm_add_ps[2] == +4.0f && v_mm_add_ps[3] == +5.0f);
-
-  constexpr __m128 v_mm_sub_ss = _mm_sub_ss(k1, k2);
-  static_assert(v_mm_sub_ss[0] == -7.0f && v_mm_sub_ss[1] == +0.0f && v_mm_sub_ss[2] == +2.0f && v_mm_sub_ss[3] == +4.0f);
-
-  constexpr __m128 v_mm_sub_ps = _mm_sub_ps(k1, k2);
-  static_assert(v_mm_sub_ps[0] == -7.0f && v_mm_sub_ps[1] == -4.0f && v_mm_sub_ps[2] == +0.0f && v_mm_sub_ps[3] == +3.0f);
-
-  constexpr __m128 v_mm_mul_ss = _mm_mul_ss(k1, k2);
-  static_assert(v_mm_mul_ss[0] == +8.0f && v_mm_mul_ss[1] == +0.0f && v_mm_mul_ss[2] == +2.0f && v_mm_mul_ss[3] == +4.0f);
-
-  constexpr __m128 v_mm_mul_ps = _mm_mul_ps(k1, k2);
-  static_assert(v_mm_mul_ps[0] == +8.0f && v_mm_mul_ps[1] == +0.0f && v_mm_mul_ps[2] == +4.0f && v_mm_mul_ps[3] == +4.0f);
-
-  constexpr __m128 v_mm_div_ss = _mm_div_ss(k1, k2);
-  static_assert(v_mm_div_ss[0] == +0.125f && v_mm_div_ss[1] == +0.0f && v_mm_div_ss[2] == +2.0f && v_mm_div_ss[3] == +4.0f);
-
-  constexpr __m128 v_mm_div_ps = _mm_div_ps(k1, k2);
-  static_assert(v_mm_div_ps[0] == +0.125f && v_mm_div_ps[1] == +0.0f && v_mm_div_ps[2] == +1.0f && v_mm_div_ps[3] == +4.0f);
-
-  constexpr __m128 v_mm_and_ps = _mm_and_ps(k3, k4);
-  static_assert(v_mm_and_ps[0] == +0.0f && v_mm_and_ps[1] == +0.0f && v_mm_and_ps[2] == +0.0f && v_mm_and_ps[3] == +0.0f);
-
-  constexpr __m128 v_mm_andnot_ps = _mm_andnot_ps(k3, k4);
-  static_assert(v_mm_andnot_ps[0] == +0.0f && v_mm_andnot_ps[1] == +0.0f && v_mm_andnot_ps[2] == +0.0f && v_mm_andnot_ps[3] == +0.0f);
-
-  constexpr __m128 v_mm_or_ps = _mm_or_ps(k3, k4);
-  static_assert(v_mm_or_ps[0] == -4.0f && v_mm_or_ps[1] == -5.0f && v_mm_or_ps[2] == -6.0f && v_mm_or_ps[3] == +7.0f);
-
-  constexpr __m128 v_mm_xor_ps = _mm_xor_ps(k3, k4);
-  static_assert(v_mm_xor_ps[0] == -4.0f && v_mm_xor_ps[1] == +5.0f && v_mm_xor_ps[2] == -6.0f && v_mm_xor_ps[3] == +7.0f);
-
-  constexpr __m128 v_mm_unpackhi_ps = _mm_unpackhi_ps(k1, k2);
-  static_assert(v_mm_unpackhi_ps[0] == +2.0f && v_mm_unpackhi_ps[1] == +2.0f && v_mm_unpackhi_ps[2] == +4.0f && v_mm_unpackhi_ps[3] == +1.0f);
-
-  constexpr __m128 v_mm_unpacklo_ps = _mm_unpacklo_ps(k1, k2);
-  static_assert(v_mm_unpacklo_ps[0] == +1.0f && v_mm_unpacklo_ps[1] == +8.0f && v_mm_unpacklo_ps[2] == +0.0f && v_mm_unpacklo_ps[3] == +4.0f);
-
-  constexpr __m128 v_mm_move_ss = _mm_move_ss(k1, k2);
-  static_assert(v_mm_move_ss[0] == +8.0f && v_mm_move_ss[1] == +0.0f && v_mm_move_ss[2] == +2.0f && v_mm_move_ss[3] == +4.0f);
-
-  constexpr __m128 v_mm_movehl_ps = _mm_movehl_ps(k1, k2);
-  static_assert(v_mm_movehl_ps[0] == +2.0f && v_mm_movehl_ps[1] == +1.0f && v_mm_movehl_ps[2] == +2.0f && v_mm_movehl_ps[3] == +4.0f);
-
-  constexpr __m128 v_mm_movelh_ps = _mm_movelh_ps(k1, k2);
-  static_assert(v_mm_movelh_ps[0] == +1.0f && v_mm_movelh_ps[1] == +0.0f && v_mm_movelh_ps[2] == +8.0f && v_mm_movelh_ps[3] == +4.0f);
-
-  constexpr __m128 v_mm_cvtsi32_ss = _mm_cvtsi32_ss(k1, 42);
-  static_assert(v_mm_cvtsi32_ss[0] == 42.0f && v_mm_cvtsi32_ss[1] == +0.0f && v_mm_cvtsi32_ss[2] == +2.0f && v_mm_cvtsi32_ss[3] == +4.0f);
-
-  constexpr __m128 v_mm_cvt_si2ss = _mm_cvt_si2ss(k2, -99);
-  static_assert(v_mm_cvt_si2ss[0] == -99.0f && v_mm_cvt_si2ss[1] == +4.0f && v_mm_cvt_si2ss[2] == +2.0f && v_mm_cvt_si2ss[3] == +1.0f);
-
-  constexpr __m128 v_mm_cvtsi64_ss = _mm_cvtsi64_ss(k3, 555);
-  static_assert(v_mm_cvtsi64_ss[0] == 555.0f && v_mm_cvtsi64_ss[1] == -5.0f && v_mm_cvtsi64_ss[2] == +6.0f && v_mm_cvtsi64_ss[3] == +7.0f);
-
-  static_assert(_mm_cvtss_f32(k2) == +8.0f);
-}
-
-#endif
\ No newline at end of file
+TEST_CONSTEXPR(match_m128(_mm_xor_ps((__m128){-4.0f, -5.0f, +6.0f, +7.0f}, (__m128){+0.0f, -0.0f, -0.0f, +7.0f}), -4.0f, +5.0f, -6.0f, +0.0f));
diff --git a/clang/test/CodeGen/X86/sse2-builtins.c b/clang/test/CodeGen/X86/sse2-builtins.c
index 0603ca5f78b6..4287d3d4b5ec 100644
--- a/clang/test/CodeGen/X86/sse2-builtins.c
+++ b/clang/test/CodeGen/X86/sse2-builtins.c
@@ -11,6 +11,7 @@
 
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 // NOTE: This should match the tests in llvm/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll
 
@@ -43,6 +44,7 @@ __m128d test_mm_add_pd(__m128d A, __m128d B) {
   // CHECK: fadd <2 x double>
   return _mm_add_pd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_add_pd((__m128d){+1.0, -3.0}, (__m128d){+5.0, -5.0}), +6.0, -8.0));
 
 __m128d test_mm_add_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_add_sd
@@ -52,6 +54,7 @@ __m128d test_mm_add_sd(__m128d A, __m128d B) {
   // CHECK: insertelement <2 x double> %{{.*}}, double %{{.*}}, i32 0
   return _mm_add_sd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_add_sd((__m128d){+1.0, -3.0}, (__m128d){+5.0, -5.0}), +6.0, -3.0));
 
 __m128i test_mm_adds_epi8(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_adds_epi8
@@ -84,6 +87,7 @@ __m128d test_mm_and_pd(__m128d A, __m128d B) {
   // CHECK: and <2 x i64>
   return _mm_and_pd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_and_pd((__m128d){+1.0, -3.0}, (__m128d){+0.0, -0.0}), +0.0, -0.0));
 
 __m128i test_mm_and_si128(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_and_si128
@@ -97,6 +101,7 @@ __m128d test_mm_andnot_pd(__m128d A, __m128d B) {
   // CHECK: and <2 x i64>
   return _mm_andnot_pd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_andnot_pd((__m128d){+1.0, -3.0}, (__m128d){+0.0, -0.0}), +0.0, -0.0));
 
 __m128i test_mm_andnot_si128(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_andnot_si128
@@ -133,11 +138,13 @@ __m128 test_mm_castpd_ps(__m128d A) {
   // CHECK-LABEL: test_mm_castpd_ps
   return _mm_castpd_ps(A);
 }
+TEST_CONSTEXPR(match_m128(_mm_castpd_ps((__m128d){-1.0, +2.0}), +0.0f, -1.875f, +0.0f, +2.0f));
 
 __m128i test_mm_castpd_si128(__m128d A) {
   // CHECK-LABEL: test_mm_castpd_si128
   return _mm_castpd_si128(A);
 }
+TEST_CONSTEXPR(match_m128i(_mm_castpd_si128((__m128d){-1.0, +2.0}), 0xBFF0000000000000ULL, 0x4000000000000000ULL));
 
 __m128d test_mm_castps_pd(__m128 A) {
   // CHECK-LABEL: test_mm_castps_pd
@@ -499,12 +506,14 @@ __m128d test_mm_cvtepi32_pd(__m128i A) {
   // CHECK: sitofp <2 x i32> %{{.*}} to <2 x double>
   return _mm_cvtepi32_pd(A);
 }
+TEST_CONSTEXPR(match_m128d(_mm_cvtepi32_pd((__m128i)(__v4si){-9, +8, -6, 0}), -9.0, +8.0));
 
 __m128 test_mm_cvtepi32_ps(__m128i A) {
   // CHECK-LABEL: test_mm_cvtepi32_ps
   // CHECK: sitofp <4 x i32> %{{.*}} to <4 x float>
   return _mm_cvtepi32_ps(A);
 }
+TEST_CONSTEXPR(match_m128(_mm_cvtepi32_ps((__m128i)(__v4si){-3, +2, -1, 0}), -3.0f, +2.0f, -1.0f, +0.0f));
 
 __m128i test_mm_cvtpd_epi32(__m128d A) {
   // CHECK-LABEL: test_mm_cvtpd_epi32
@@ -530,12 +539,14 @@ __m128d test_mm_cvtps_pd(__m128 A) {
   // CHECK: fpext <2 x float> %{{.*}} to <2 x double>
   return _mm_cvtps_pd(A);
 }
+TEST_CONSTEXPR(match_m128d(_mm_cvtps_pd((__m128){-1.0f, +2.0f, -3.0f, +4.0f}), -1.0, +2.0));
 
 double test_mm_cvtsd_f64(__m128d A) {
   // CHECK-LABEL: test_mm_cvtsd_f64
   // CHECK: extractelement <2 x double> %{{.*}}, i32 0
   return _mm_cvtsd_f64(A);
 }
+TEST_CONSTEXPR(_mm_cvtsd_f64((__m128d){-4.0, +8.0}) == -4.0);
 
 int test_mm_cvtsd_si32(__m128d A) {
   // CHECK-LABEL: test_mm_cvtsd_si32
@@ -575,6 +586,7 @@ __m128d test_mm_cvtsi32_sd(__m128d A, int B) {
   // CHECK: insertelement <2 x double> %{{.*}}, double %{{.*}}, i32 0
   return _mm_cvtsi32_sd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_cvtsi32_sd((__m128d){-99.0, +42.0}, 55), +55.0, +42.0));
 
 __m128i test_mm_cvtsi32_si128(int A) {
   // CHECK-LABEL: test_mm_cvtsi32_si128
@@ -608,6 +620,7 @@ __m128d test_mm_cvtss_sd(__m128d A, __m128 B) {
   // CHECK: insertelement <2 x double> %{{.*}}, double %{{.*}}, i32 0
   return _mm_cvtss_sd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_cvtss_sd((__m128d){+32.0, +8.0}, (__m128){-1.0f, +2.0f, -3.0f, +4.0f}), -1.0, +8.0));
 
 __m128i test_mm_cvttpd_epi32(__m128d A) {
   // CHECK-LABEL: test_mm_cvttpd_epi32
@@ -640,6 +653,7 @@ __m128d test_mm_div_pd(__m128d A, __m128d B) {
   // CHECK: fdiv <2 x double>
   return _mm_div_pd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_div_pd((__m128d){+2.0, +8.0}, (__m128d){-4.0, +2.0}), -0.5, +4.0));
 
 __m128d test_mm_div_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_div_sd
@@ -649,6 +663,7 @@ __m128d test_mm_div_sd(__m128d A, __m128d B) {
   // CHECK: insertelement <2 x double> %{{.*}}, double %{{.*}}, i32 0
   return _mm_div_sd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_div_sd((__m128d){+2.0, +8.0}, (__m128d){-4.0, +2.0}), -0.5, +8.0));
 
 // Lowering to pextrw requires optimization.
 int test_mm_extract_epi16(__m128i A) {
@@ -873,6 +888,7 @@ __m128d test_mm_move_sd(__m128d A, __m128d B) {
   // CHECK: insertelement <2 x double> %{{.*}}, double %{{.*}}, i32 0
   return _mm_move_sd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_move_sd((__m128d){+2.0, +8.0}, (__m128d){-4.0, +2.0}), -4.0, +8.0));
 
 int test_mm_movemask_epi8(__m128i A) {
   // CHECK-LABEL: test_mm_movemask_epi8
@@ -899,6 +915,7 @@ __m128d test_mm_mul_pd(__m128d A, __m128d B) {
   // CHECK: fmul <2 x double> %{{.*}}, %{{.*}}
   return _mm_mul_pd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_mul_pd((__m128d){+1.0, -3.0}, (__m128d){+5.0, -5.0}), +5.0, +15.0));
 
 __m128d test_mm_mul_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_mul_sd
@@ -908,6 +925,7 @@ __m128d test_mm_mul_sd(__m128d A, __m128d B) {
   // CHECK: insertelement <2 x double> %{{.*}}, double %{{.*}}, i32 0
   return _mm_mul_sd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_mul_sd((__m128d){+1.0, -3.0}, (__m128d){+5.0, -5.0}), +5.0, -3.0));
 
 __m128i test_mm_mulhi_epi16(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_mulhi_epi16
@@ -932,6 +950,7 @@ __m128d test_mm_or_pd(__m128d A, __m128d B) {
   // CHECK: or <2 x i64> %{{.*}}, %{{.*}}
   return _mm_or_pd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_or_pd((__m128d){+1.0, -3.0}, (__m128d){-0.0, +0.0}), -1.0, -3.0));
 
 __m128i test_mm_or_si128(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_or_si128
@@ -1036,6 +1055,7 @@ __m128d test_mm_set_pd(double A, double B) {
   // CHECK: insertelement <2 x double> %{{.*}}, double %{{.*}}, i32 1
   return _mm_set_pd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_set_pd(-9.0, +3.0), +3.0, -9.0));
 
 __m128d test_mm_set_pd1(double A) {
   // CHECK-LABEL: test_mm_set_pd1
@@ -1043,6 +1063,7 @@ __m128d test_mm_set_pd1(double A) {
   // CHECK: insertelement <2 x double> %{{.*}}, double %{{.*}}, i32 1
   return _mm_set_pd1(A);
 }
+TEST_CONSTEXPR(match_m128d(_mm_set_pd1(+5.0), +5.0, +5.0));
 
 __m128d test_mm_set_sd(double A) {
   // CHECK-LABEL: test_mm_set_sd
@@ -1050,6 +1071,7 @@ __m128d test_mm_set_sd(double A) {
   // CHECK: insertelement <2 x double> %{{.*}}, double 0.000000e+00, i32 1
   return _mm_set_sd(A);
 }
+TEST_CONSTEXPR(match_m128d(_mm_set_sd(+1.0), +1.0, +0.0));
 
 __m128i test_mm_set1_epi8(char A) {
   // CHECK-LABEL: test_mm_set1_epi8
@@ -1114,6 +1136,7 @@ __m128d test_mm_set1_pd(double A) {
   // CHECK: insertelement <2 x double> %{{.*}}, double %{{.*}}, i32 1
   return _mm_set1_pd(A);
 }
+TEST_CONSTEXPR(match_m128d(_mm_set1_pd(-42.0), -42.0, -42.0));
 
 __m128i test_mm_setr_epi8(char A, char B, char C, char D,
                           char E, char F, char G, char H,
@@ -1175,18 +1198,21 @@ __m128d test_mm_setr_pd(double A, double B) {
   // CHECK: insertelement <2 x double> %{{.*}}, double %{{.*}}, i32 1
   return _mm_setr_pd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_setr_pd(-9.0, +3.0), -9.0, +3.0));
 
 __m128d test_mm_setzero_pd(void) {
   // CHECK-LABEL: test_mm_setzero_pd
   // CHECK: store <2 x double> zeroinitializer
   return _mm_setzero_pd();
 }
+TEST_CONSTEXPR(match_m128d(_mm_setzero_pd(), +0.0, +0.0));
 
 __m128i test_mm_setzero_si128(void) {
   // CHECK-LABEL: test_mm_setzero_si128
   // CHECK: store <2 x i64> zeroinitializer
   return _mm_setzero_si128();
 }
+TEST_CONSTEXPR(match_m128i(_mm_setzero_si128(), 0, 0));
 
 __m128i test_mm_shuffle_epi32(__m128i A) {
   // CHECK-LABEL: test_mm_shuffle_epi32
@@ -1620,6 +1646,7 @@ __m128d test_mm_sub_pd(__m128d A, __m128d B) {
   // CHECK: fsub <2 x double>
   return _mm_sub_pd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_sub_pd((__m128d){+1.0, -3.0}, (__m128d){+5.0, -5.0}), -4.0, +2.0));
 
 __m128d test_mm_sub_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_sub_sd
@@ -1629,6 +1656,7 @@ __m128d test_mm_sub_sd(__m128d A, __m128d B) {
   // CHECK: insertelement <2 x double> %{{.*}}, double %{{.*}}, i32 0
   return _mm_sub_sd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_sub_sd((__m128d){+1.0, -3.0}, (__m128d){+5.0, -5.0}), -4.0, -3.0));
 
 __m128i test_mm_subs_epi8(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_subs_epi8
@@ -1736,6 +1764,7 @@ __m128d test_mm_unpackhi_pd(__m128d A, __m128d B) {
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> <i32 1, i32 3>
   return _mm_unpackhi_pd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_unpackhi_pd((__m128d){+2.0, +8.0}, (__m128d){-4.0, -2.0}), +8.0, -2.0));
 
 __m128i test_mm_unpacklo_epi8(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_unpacklo_epi8
@@ -1766,123 +1795,17 @@ __m128d test_mm_unpacklo_pd(__m128d A, __m128d B) {
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> <i32 0, i32 2>
   return _mm_unpacklo_pd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_unpacklo_pd((__m128d){+2.0, +8.0}, (__m128d){-4.0, -2.0}), +2.0, -4.0));
 
 __m128d test_mm_xor_pd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_xor_pd
   // CHECK: xor <2 x i64> %{{.*}}, %{{.*}}
   return _mm_xor_pd(A, B);
 }
+TEST_CONSTEXPR(match_m128d(_mm_xor_pd((__m128d){+1.0, -3.0}, (__m128d){+0.0, -0.0}), +1.0, +3.0));
 
 __m128i test_mm_xor_si128(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_xor_si128
   // CHECK: xor <2 x i64> %{{.*}}, %{{.*}}
   return _mm_xor_si128(A, B);
 }
-
-// Test constexpr handling.
-#if defined(__cplusplus) && (__cplusplus >= 201103L)
-
-void test_constexpr() {
-  constexpr __m128d kd1 {+2.0,-1.0};
-  constexpr __m128d kd2 {-4.0,-2.0};
-  constexpr __m128d kd3 {-0.0,+0.0};
-
-  constexpr __m128 kf1 {-1.0f,+2.0f,-3.0f,+4.0f};
-
-  constexpr __m64 km1 {0x00000080FFFFFFF0ULL}; // -16,+128
-  constexpr __m128i ki1 {0x00000010FFFFFFF8ULL, 0x00000001FFFFFFFFULL}; // -8,+16,-1,1
-
-  constexpr __m128d v_mm_set_sd = _mm_set_sd(1.0);
-  static_assert(v_mm_set_sd[0] == +1.0 && v_mm_set_sd[1] == +0.0);
-
-  constexpr __m128d v_mm_set1_pd = _mm_set1_pd(2.0);
-  static_assert(v_mm_set1_pd[0] == +2.0 && v_mm_set1_pd[1] == +2.0);
-
-  constexpr __m128d v_mm_set_pd1 = _mm_set_pd1(-2.0);
-  static_assert(v_mm_set_pd1[0] == -2.0 && v_mm_set_pd1[1] == -2.0);
-
-  constexpr __m128d v_mm_set_pd = _mm_set_pd(+2.0, +3.0);
-  static_assert(v_mm_set_pd[0] == +3.0 && v_mm_set_pd[1] == +2.0);
-
-  constexpr __m128d v_mm_setr_pd = _mm_setr_pd(+2.0, +3.0);
-  static_assert(v_mm_setr_pd[0] == +2.0 && v_mm_setr_pd[1] == +3.0);
-
-  constexpr __m128d v_mm_setzero_pd = _mm_setzero_pd();
-  static_assert(v_mm_setzero_pd[0] == +0.0 && v_mm_setzero_pd[1] == +0.0);
-
-  constexpr __m128i v_mm_setzero_si128 = _mm_setzero_si128();
-  static_assert(v_mm_setzero_si128[0] == 0x0000000000000000ULL && v_mm_setzero_si128[1] == 0x0000000000000000ULL);
-
-  constexpr __m128d v_mm_add_sd = _mm_add_sd(kd1, kd2);
-  static_assert(v_mm_add_sd[0] == -2.0 && v_mm_add_sd[1] == -1.0);
-
-  constexpr __m128d v_mm_add_pd = _mm_add_pd(kd1, kd2);
-  static_assert(v_mm_add_pd[0] == -2.0 && v_mm_add_pd[1] == -3.0);
-
-  constexpr __m128d v_mm_sub_sd = _mm_sub_sd(kd1, kd2);
-  static_assert(v_mm_sub_sd[0] == +6.0 && v_mm_sub_sd[1] == -1.0);
-
-  constexpr __m128d v_mm_sub_pd = _mm_sub_pd(kd1, kd2);
-  static_assert(v_mm_sub_pd[0] == +6.0 && v_mm_sub_pd[1] == +1.0);
-
-  constexpr __m128d v_mm_mul_sd = _mm_mul_sd(kd1, kd2);
-  static_assert(v_mm_mul_sd[0] == -8.0 && v_mm_mul_sd[1] == -1.0);
-
-  constexpr __m128d v_mm_mul_pd = _mm_mul_pd(kd1, kd2);
-  static_assert(v_mm_mul_pd[0] == -8.0 && v_mm_mul_pd[1] == +2.0);
-
-  constexpr __m128d v_mm_div_sd = _mm_div_sd(kd1, kd2);
-  static_assert(v_mm_div_sd[0] == -0.5 && v_mm_div_sd[1] == -1.0);
-
-  constexpr __m128d v_mm_div_pd = _mm_div_pd(kd1, kd2);
-  static_assert(v_mm_div_pd[0] == -0.5 && v_mm_div_pd[1] == +0.5);
-
-  constexpr __m128d v_mm_and_pd = _mm_and_pd(kd1, kd3);
-  static_assert(v_mm_and_pd[0] == +0.0 && v_mm_and_pd[1] == +0.0);
-
-  constexpr __m128d v_mm_andnot_pd = _mm_andnot_pd(kd1, kd3);
-  static_assert(v_mm_andnot_pd[0] == -0.0 && v_mm_andnot_pd[1] == +0.0);
-
-  constexpr __m128d v_mm_or_pd = _mm_or_pd(kd1, kd3);
-  static_assert(v_mm_or_pd[0] == -2.0 && v_mm_or_pd[1] == -1.0);
-
-  constexpr __m128d v_mm_xor_pd = _mm_xor_pd(kd2, kd3);
-  static_assert(v_mm_xor_pd[0] == +4.0 && v_mm_xor_pd[1] == -2.0);
-
-  constexpr __m128d v_mm_cvtps_pd = _mm_cvtps_pd(kf1);
-  static_assert(v_mm_cvtps_pd[0] == -1.0 && v_mm_cvtps_pd[1] == +2.0);
-
-  constexpr __m128d v_mm_cvtepi32_pd = _mm_cvtepi32_pd(ki1);
-  static_assert(v_mm_cvtepi32_pd[0] == -8.0 && v_mm_cvtepi32_pd[1] == +16.0);
-
-  constexpr __m128 v_mm_cvtepi32_ps = _mm_cvtepi32_ps(ki1);
-  static_assert(v_mm_cvtepi32_ps[0] == -8.0f && v_mm_cvtepi32_ps[1] == +16.0f && v_mm_cvtepi32_ps[2] == -1.0f && v_mm_cvtepi32_ps[3] == +1.0f);
-
-  constexpr __m128d v_mm_cvtsi32_sd = _mm_cvtsi32_sd(kd1, 8);
-  static_assert(v_mm_cvtsi32_sd[0] == +8.0 && v_mm_cvtsi32_sd[1] == -1.0);
-
-  constexpr __m128d v_mm_cvtss_sd = _mm_cvtss_sd(kd2, kf1);
-  static_assert(v_mm_cvtss_sd[0] == -1.0 && v_mm_cvtss_sd[1] == -2.0);
-
-  constexpr __m128d v_mm_cvtpi32_pd = _mm_cvtpi32_pd(km1);
-  static_assert(v_mm_cvtpi32_pd[0] == -16.0 && v_mm_cvtpi32_pd[1] == 128.0);
-
-  static_assert(_mm_cvtsd_f64(kd2) == -4.0);
-
-  constexpr __m128d v_mm_move_sd = _mm_move_sd(kd1, kd2);
-  static_assert(v_mm_move_sd[0] == -4.0 && v_mm_move_sd[1] == -1.0);
-
-  constexpr __m128d v_mm_unpackhi_pd = _mm_unpackhi_pd(kd1, kd2);
-  static_assert(v_mm_unpackhi_pd[0] == -1.0f && v_mm_unpackhi_pd[1] == -2.0f);
-
-  constexpr __m128d v_mm_unpacklo_pd = _mm_unpacklo_pd(kd1, kd2);
-  static_assert(v_mm_unpacklo_pd[0] == +2.0f && v_mm_unpacklo_pd[1] == -4.0f);
-
-  constexpr __m128 v_mm_castpd_ps = _mm_castpd_ps(kd3);
-  static_assert(v_mm_castpd_ps[0] == -0.0f && v_mm_castpd_ps[1] == +0.0f && v_mm_castpd_ps[2] == +0.0f && v_mm_castpd_ps[3] == +0.0f);
-
-  constexpr __m128i v_mm_castpd_si128 = _mm_castpd_si128(kd3);
-  static_assert(v_mm_castpd_si128[0] == 0x8000000000000000ULL && v_mm_castpd_si128[1] == 0x0000000000000000ULL);
-}
-
-#endif
diff --git a/clang/test/CodeGen/X86/sse3-builtins.c b/clang/test/CodeGen/X86/sse3-builtins.c
index 18c062f4c14a..d47c19b882cd 100644
--- a/clang/test/CodeGen/X86/sse3-builtins.c
+++ b/clang/test/CodeGen/X86/sse3-builtins.c
@@ -5,6 +5,7 @@
 
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 // NOTE: This should match the tests in llvm/test/CodeGen/X86/sse3-intrinsics-fast-isel.ll
 
@@ -63,34 +64,18 @@ __m128d test_mm_movedup_pd(__m128d A) {
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> zeroinitializer
   return _mm_movedup_pd(A);
 }
+TEST_CONSTEXPR(match_m128d(_mm_movedup_pd((__m128d){+7.0, -7.0}), +7.0, +7.0));
 
 __m128 test_mm_movehdup_ps(__m128 A) {
   // CHECK-LABEL: test_mm_movehdup_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
   return _mm_movehdup_ps(A);
 }
+TEST_CONSTEXPR(match_m128(_mm_movehdup_ps((__m128){+1.0f,-1.0f,+2.0f,+4.0f}), -1.0f, -1.0f, +4.0f, +4.0f));
 
 __m128 test_mm_moveldup_ps(__m128 A) {
   // CHECK-LABEL: test_mm_moveldup_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
   return _mm_moveldup_ps(A);
 }
-
-// Test constexpr handling.
-#if defined(__cplusplus) && (__cplusplus >= 201103L)
-
-void test_constexpr() {
-  constexpr __m128d kd1 {+7.0,-7.0};
-  constexpr __m128 kf1 {+1.0f,-1.0f,+2.0f,+4.0f};
-
-  constexpr __m128d v_mm_movedup_pd = _mm_movedup_pd(kd1);
-  static_assert(v_mm_movedup_pd[0] == +7.0 && v_mm_movedup_pd[1] == +7.0);
-
-  constexpr __m128 v_mm_movehdup_ps = _mm_movehdup_ps(kf1);
-  static_assert(v_mm_movehdup_ps[0] == -1.0f && v_mm_movehdup_ps[1] == -1.0f && v_mm_movehdup_ps[2] == +4.0f && v_mm_movehdup_ps[3] == +4.0f);
-
-  constexpr __m128 v_mm_moveldup_ps = _mm_moveldup_ps(kf1);
-  static_assert(v_mm_moveldup_ps[0] == +1.0f && v_mm_moveldup_ps[1] == +1.0f && v_mm_moveldup_ps[2] == +2.0f && v_mm_moveldup_ps[3] == +2.0f);
-}
-
-#endif
+TEST_CONSTEXPR(match_m128(_mm_moveldup_ps((__m128){+1.0f,-1.0f,+2.0f,+4.0f}), +1.0f, +1.0f, +2.0f, +2.0f));
-- 
GitLab


From 98c8d643539194321f3dba8698e95999165b1024 Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac@arm.com>
Date: Tue, 29 Oct 2024 11:08:36 +0000
Subject: [PATCH 416/693] [AArch64] Add assembly/dissasembly for BFSCALE
 instructions (#113538)

This patch adds assembly/disassembly for following instructions:
   BFSCALE (multiple and single vector)
   BFSCALE (multiple vectors)

As specified in https://developer.arm.com/documentation/ddi0602/2024-09

Co-authored-by: Momchil Velikov
[momchil.velikov@arm.com](mailto:momchil.velikov@arm.com)
---
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |  5 +
 llvm/lib/Target/AArch64/SMEInstrFormats.td    | 12 ++-
 .../MC/AArch64/SME2/bfscale-diagnostics.s     | 87 +++++++++++++++++
 llvm/test/MC/AArch64/SME2/bfscale.s           | 93 +++++++++++++++++++
 4 files changed, 196 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/MC/AArch64/SME2/bfscale-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SME2/bfscale.s

diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index 76e0501a5cc2..b0ce9e1a1b40 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -1000,6 +1000,11 @@ defm FMOPA_MPPZZ_BtoS : sme_outer_product_fp32<0b0, 0b01, ZPR8, "fmopa", null_fr
 
 } //[HasSMEF8F32]
 
+let Predicates = [HasSME2, HasSVEBFSCALE] in {
+  defm BFSCALE : sme2_bfscale_single<"bfscale">;
+  defm BFSCALE : sme2_bfscale_multi<"bfscale">;
+}
+
 let Predicates = [HasSME2p2] in {
   def FTMOPA_M2ZZZI_HtoS  : sme_tmopa_32b<0b11000, ZZ_h_mul_r, ZPR16, "ftmopa">;
   def FTMOPA_M2ZZZI_StoS  : sme_tmopa_32b<0b00000, ZZ_s_mul_r, ZPR32, "ftmopa">;
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index a05c5206320f..62a0c2e83d4a 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -5189,6 +5189,16 @@ class sme2_luti4_vector_vg4_strided<bits<2> sz, bits<2> op, string mnemonic>
   let Inst{1-0}   = Zd{1-0};
 }
 
+multiclass sme2_bfscale_single<string mnemonic> {
+  def _2ZZ : sme2_sve_destructive_vector_vg2_single<0b00, 0b0011000, ZZ_h_mul_r, ZPR4b16, mnemonic>;
+  def _4ZZ : sme2_sve_destructive_vector_vg4_single<0b00, 0b0011000, ZZZZ_h_mul_r, ZPR4b16, mnemonic>;
+}
+
+multiclass sme2_bfscale_multi<string mnemonic> {
+  def _2Z2Z : sme2_sve_destructive_vector_vg2_multi<0b00, 0b0011000, ZZ_h_mul_r, mnemonic>;
+  def _4Z4Z : sme2_sve_destructive_vector_vg4_multi<0b00, 0b0011000, ZZZZ_h_mul_r, mnemonic>;
+}
+
 class sme2_bf16_fp32_quarter_tile_outer_product<bit M, bit N, bit S, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>
     : I<(outs TileOp32:$ZAda),
         (ins TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),
@@ -5334,4 +5344,4 @@ multiclass sme2_multi4_fmul_mm<string mnemonic> {
 multiclass sme2_bfmul_multi<string mnemonic> {
   def _2Z2Z : sme2_multi2_fmul_mm<0b00, mnemonic, ZZ_h_mul_r>;
   def _4Z4Z : sme2_multi4_fmul_mm<0b00, mnemonic, ZZZZ_h_mul_r>;
-}
+}
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SME2/bfscale-diagnostics.s b/llvm/test/MC/AArch64/SME2/bfscale-diagnostics.s
new file mode 100644
index 000000000000..63367eed65b6
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2/bfscale-diagnostics.s
@@ -0,0 +1,87 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2,+sve-bfscale 2>&1 < %s| FileCheck %s
+
+// Multiple and single vector, 2 regs
+
+bfscale  {z0.s-z1.s}, {z0.s-z1.s}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfscale  {z1.h-z2.h}, {z1.h-z2.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+
+bfscale  {z0.h-z2.h}, {z0.h-z2.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfscale  {z0.h-z1.h}, {z0.h-z1.h}, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.h..z15.h
+
+bfscale  {z0.h-z1.h}, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.h..z15.h
+
+bfscale  {z0.h-z1.h}, {z2.h-z3.h}, z8.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register list
+
+// Multiple and single vector, 4 regs
+
+bfscale  {z0.s-z3.s}, {z0.s-z3.s}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfscale  {z1.h-z4.h}, {z1.h-z4.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 4 consecutive SVE vectors, where the first vector is a multiple of 4 and with matching element types
+
+bfscale  {z0.h-z4.h}, {z0.h-z4.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+
+bfscale  {z0.h-z3.h}, {z0.h-z3.h}, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.h..z15.h
+
+bfscale  {z0.h-z3.h}, {z0.h-z3.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.h..z15.h
+
+bfscale  {z0.h-z3.h}, {z4.h-z7.h}, z8.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register list
+
+// Multiple vectors, 2 regs
+
+bfscale  {z0.s-z1.s}, {z0.s-z1.s}, {z2.h-z3.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfscale  {z1.h-z2.h}, {z1.h-z2.h}, {z2.h-z3.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+
+bfscale  {z0.h-z2.h}, {z0.h-z4.h}, {z2.h-z3.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+
+bfscale  {z0.h-z1.h}, {z0.h-z1.h}, {z2.s-z3.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfscale  {z0.h-z1.h}, {z0.h-z1.h}, {z28.h-z30.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfscale  {z0.h-z1.h}, {z0.h-z1.h}, {z29.h-z30.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors, where the first vector is a multiple of 2 and with matching element types
+
+bfscale  {z0.h-z1.h}, {z2.h-z3.h}, {z28.h-z29.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register list
+
+// Multiple vectors, 4 regs
+
+bfscale  {z0.s-z3.s}, {z0.s-z3.s}, {z4.h-z7.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfscale  {z1.h-z4.h}, {z1.h-z4.h}, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 4 consecutive SVE vectors, where the first vector is a multiple of 4 and with matching element types
+
+bfscale  {z0.h-z4.h}, {z0.h-z4.h}, {z4.h-z7.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error:  invalid number of vectors
+
+bfscale  {z0.h-z3.h}, {z0.h-z3.h}, {z4.s-z7.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfscale  {z0.h-z3.h}, {z0.h-z3.h}, {z4.h-z8.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid number of vectors
+
+bfscale  {z0.h-z3.h}, {z0.h-z3.h}, {z5.h-z8.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 4 consecutive SVE vectors, where the first vector is a multiple of 4 and with matching element types
+
+bfscale  {z0.h-z3.h}, {z4.h-z7.h}, {z8.h-z11.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register list
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SME2/bfscale.s b/llvm/test/MC/AArch64/SME2/bfscale.s
new file mode 100644
index 000000000000..719d013b235c
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2/bfscale.s
@@ -0,0 +1,93 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2,+sve-bfscale < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2,+sve-bfscale < %s \
+// RUN:        | llvm-objdump -d --mattr=+sme2,+sve-bfscale - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2,+sve-bfscale < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2,+sve-bfscale < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2,+sve-bfscale -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// Multiple and single vector, 2 regs
+
+bfscale {z0.h-z1.h}, {z0.h-z1.h}, z0.h  // 11000001-00100000-10100001-10000000
+// CHECK-INST: bfscale { z0.h, z1.h }, { z0.h, z1.h }, z0.h
+// CHECK-ENCODING: [0x80,0xa1,0x20,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c120a180 <unknown>
+
+bfscale {z20.h-z21.h}, {z20.h-z21.h}, z5.h  // 11000001-00100101-10100001-10010100
+// CHECK-INST: bfscale { z20.h, z21.h }, { z20.h, z21.h }, z5.h
+// CHECK-ENCODING: [0x94,0xa1,0x25,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c125a194 <unknown>
+
+bfscale {z30.h-z31.h}, {z30.h-z31.h}, z15.h  // 11000001-00101111-10100001-10011110
+// CHECK-INST: bfscale { z30.h, z31.h }, { z30.h, z31.h }, z15.h
+// CHECK-ENCODING: [0x9e,0xa1,0x2f,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c12fa19e <unknown>
+
+// Multiple and single vector, 4 regs
+
+bfscale {z0.h-z3.h}, {z0.h-z3.h}, z0.h  // 11000001-00100000-10101001-10000000
+// CHECK-INST: bfscale { z0.h - z3.h }, { z0.h - z3.h }, z0.h
+// CHECK-ENCODING: [0x80,0xa9,0x20,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c120a980 <unknown>
+
+bfscale {z20.h-z23.h}, {z20.h-z23.h}, z5.h  // 11000001-00100101-10101001-10010100
+// CHECK-INST: bfscale { z20.h - z23.h }, { z20.h - z23.h }, z5.h
+// CHECK-ENCODING: [0x94,0xa9,0x25,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c125a994 <unknown>
+
+bfscale {z28.h-z31.h}, {z28.h-z31.h}, z15.h  // 11000001-00101111-10101001-10011100
+// CHECK-INST: bfscale { z28.h - z31.h }, { z28.h - z31.h }, z15.h
+// CHECK-ENCODING: [0x9c,0xa9,0x2f,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c12fa99c <unknown>
+
+// Multiple vectors, 2 regs
+
+bfscale {z0.h-z1.h}, {z0.h-z1.h}, {z0.h-z1.h}  // 11000001-00100000-10110001-10000000
+// CHECK-INST: bfscale { z0.h, z1.h }, { z0.h, z1.h }, { z0.h, z1.h }
+// CHECK-ENCODING: [0x80,0xb1,0x20,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c120b180 <unknown>
+
+bfscale {z20.h-z21.h}, {z20.h-z21.h}, {z20.h-z21.h}  // 11000001-00110100-10110001-10010100
+// CHECK-INST: bfscale { z20.h, z21.h }, { z20.h, z21.h }, { z20.h, z21.h }
+// CHECK-ENCODING: [0x94,0xb1,0x34,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c134b194 <unknown>
+
+bfscale {z30.h-z31.h}, {z30.h-z31.h}, {z30.h-z31.h}  // 11000001-00111110-10110001-10011110
+// CHECK-INST: bfscale { z30.h, z31.h }, { z30.h, z31.h }, { z30.h, z31.h }
+// CHECK-ENCODING: [0x9e,0xb1,0x3e,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c13eb19e <unknown>
+
+// Multiple vectors, 4 regs
+
+bfscale {z0.h-z3.h}, {z0.h-z3.h}, {z0.h-z3.h}  // 11000001-00100000-10111001-10000000
+// CHECK-INST: bfscale { z0.h - z3.h }, { z0.h - z3.h }, { z0.h - z3.h }
+// CHECK-ENCODING: [0x80,0xb9,0x20,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c120b980 <unknown>
+
+bfscale {z20.h-z23.h}, {z20.h-z23.h}, {z20.h-z23.h}  // 11000001-00110100-10111001-10010100
+// CHECK-INST: bfscale { z20.h - z23.h }, { z20.h - z23.h }, { z20.h - z23.h }
+// CHECK-ENCODING: [0x94,0xb9,0x34,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c134b994 <unknown>
+
+bfscale {z28.h-z31.h}, {z28.h-z31.h}, {z28.h-z31.h}  // 11000001-00111100-10111001-10011100
+// CHECK-INST: bfscale { z28.h - z31.h }, { z28.h - z31.h }, { z28.h - z31.h }
+// CHECK-ENCODING: [0x9c,0xb9,0x3c,0xc1]
+// CHECK-ERROR: instruction requires: sme2 sve-bfscale
+// CHECK-UNKNOWN: c13cb99c <unknown>
-- 
GitLab


From 06664fdc7680f7f9fa9b0a414a8fb8df2f913d48 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye@arm.com>
Date: Tue, 29 Oct 2024 11:41:25 +0000
Subject: [PATCH 417/693] [FuncSpec] Enable SpecializeLiteralConstant by
 default (#113442)

Enable specialization on literal constant arguments by default in
Function Specialization.

---------

Co-authored-by: Alexandros Lamprineas <alexandros.lamprineas@arm.com>
---
 .../Transforms/IPO/FunctionSpecialization.h   |   7 +
 .../Transforms/IPO/FunctionSpecialization.cpp |  20 ++-
 .../Generic/ipsccp-remap-assign-id.ll         |   4 +-
 .../compiler-crash-58759.ll                   |   2 +-
 ...tion-specialization-constant-expression.ll |   2 +-
 .../function-specialization2.ll               |   6 +-
 .../function-specialization4.ll               |   6 +-
 .../get-possible-constants.ll                 |   2 +-
 .../FunctionSpecialization/global-rank.ll     |   2 +-
 .../identical-specializations.ll              |   2 +-
 .../FunctionSpecialization/literal-const.ll   |   2 +-
 .../specialize-multiple-arguments.ll          |   8 +-
 .../track-ptr-return.ll                       | 147 ++++++++++++++++++
 13 files changed, 182 insertions(+), 28 deletions(-)
 create mode 100644 llvm/test/Transforms/FunctionSpecialization/track-ptr-return.ll

diff --git a/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h b/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
index 5920dde9d77d..f20b01c18630 100644
--- a/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
+++ b/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
@@ -64,6 +64,13 @@
 // - Perhaps a post-inlining function specialization pass could be more
 //   aggressive on literal constants.
 //
+// Limitations:
+// ------------
+// - We are unable to consider specializations of functions called from indirect
+//   callsites whose pointer operand has a lattice value that is known to be
+//   constant, either from IPSCCP or previous iterations of FuncSpec. This is
+//   because SCCP has not yet replaced the uses of the known constant.
+//
 // References:
 // -----------
 // 2021 LLVM Dev Mtg “Introducing function specialisation, and can we enable
diff --git a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
index 48971e9a4635..8e6993d35d49 100644
--- a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
@@ -84,14 +84,11 @@ static cl::opt<bool> SpecializeOnAddress(
     "funcspec-on-address", cl::init(false), cl::Hidden, cl::desc(
     "Enable function specialization on the address of global values"));
 
-// Disabled by default as it can significantly increase compilation times.
-//
-// https://llvm-compile-time-tracker.com
-// https://github.com/nikic/llvm-compile-time-tracker
 static cl::opt<bool> SpecializeLiteralConstant(
-    "funcspec-for-literal-constant", cl::init(false), cl::Hidden, cl::desc(
-    "Enable specialization of functions that take a literal constant as an "
-    "argument"));
+    "funcspec-for-literal-constant", cl::init(true), cl::Hidden,
+    cl::desc(
+        "Enable specialization of functions that take a literal constant as an "
+        "argument"));
 
 bool InstCostVisitor::canEliminateSuccessor(BasicBlock *BB, BasicBlock *Succ,
                                          DenseSet<BasicBlock *> &DeadBlocks) {
@@ -682,10 +679,11 @@ bool FunctionSpecializer::run() {
         (RequireMinSize && Metrics.NumInsts < MinFunctionSize))
       continue;
 
-    // TODO: For now only consider recursive functions when running multiple
-    // times. This should change if specialization on literal constants gets
-    // enabled.
-    if (!Inserted && !Metrics.isRecursive && !SpecializeLiteralConstant)
+    // When specialization on literal constants is disabled, only consider
+    // recursive functions when running multiple times to save wasted analysis,
+    // as we will not be able to specialize on any newly found literal constant
+    // return values.
+    if (!SpecializeLiteralConstant && !Inserted && !Metrics.isRecursive)
       continue;
 
     int64_t Sz = *Metrics.NumInsts.getValue();
diff --git a/llvm/test/DebugInfo/Generic/ipsccp-remap-assign-id.ll b/llvm/test/DebugInfo/Generic/ipsccp-remap-assign-id.ll
index 0e8f92cacf66..42560fc3958d 100644
--- a/llvm/test/DebugInfo/Generic/ipsccp-remap-assign-id.ll
+++ b/llvm/test/DebugInfo/Generic/ipsccp-remap-assign-id.ll
@@ -1,5 +1,5 @@
-; RUN: opt -passes=ipsccp %s -S -o - | FileCheck %s
-; RUN: opt --try-experimental-debuginfo-iterators -passes=ipsccp %s -S -o - | FileCheck %s
+; RUN: opt -passes=ipsccp -funcspec-for-literal-constant=false %s -S -o - | FileCheck %s
+; RUN: opt --try-experimental-debuginfo-iterators -passes=ipsccp -funcspec-for-literal-constant=false %s -S -o - | FileCheck %s
 
 ;; Check the dbg.assign DIAssignID operand gets remapped after cloning.
 
diff --git a/llvm/test/Transforms/FunctionSpecialization/compiler-crash-58759.ll b/llvm/test/Transforms/FunctionSpecialization/compiler-crash-58759.ll
index f29cf0d12393..7291d83b8161 100644
--- a/llvm/test/Transforms/FunctionSpecialization/compiler-crash-58759.ll
+++ b/llvm/test/Transforms/FunctionSpecialization/compiler-crash-58759.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S --passes="default<O3>" < %s | FileCheck %s
+; RUN: opt -S --passes="default<O3>" -funcspec-for-literal-constant=false < %s | FileCheck %s
 
 define dso_local i32 @g0(i32 noundef %x) local_unnamed_addr {
 entry:
diff --git a/llvm/test/Transforms/FunctionSpecialization/function-specialization-constant-expression.ll b/llvm/test/Transforms/FunctionSpecialization/function-specialization-constant-expression.ll
index 16a468511631..0c24169d02c2 100644
--- a/llvm/test/Transforms/FunctionSpecialization/function-specialization-constant-expression.ll
+++ b/llvm/test/Transforms/FunctionSpecialization/function-specialization-constant-expression.ll
@@ -4,7 +4,7 @@
 ; Note that this test case shows that function specialization pass would
 ; transform the function even if no specialization happened.
 
-; RUN: opt -passes="ipsccp<func-spec>" -force-specialization -S < %s | FileCheck %s
+; RUN: opt -passes="ipsccp<func-spec>" -force-specialization -funcspec-for-literal-constant=false -S < %s | FileCheck %s
 
 %struct = type { i8, i16, i32, i64, i64}
 @Global = internal constant %struct {i8 0, i16 1, i32 2, i64 3, i64 4}
diff --git a/llvm/test/Transforms/FunctionSpecialization/function-specialization2.ll b/llvm/test/Transforms/FunctionSpecialization/function-specialization2.ll
index ef830a0e9a4a..6f36a394979d 100644
--- a/llvm/test/Transforms/FunctionSpecialization/function-specialization2.ll
+++ b/llvm/test/Transforms/FunctionSpecialization/function-specialization2.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 5
-; RUN: opt -passes="ipsccp<func-spec>,deadargelim" -force-specialization -S < %s | FileCheck %s
-; RUN: opt -passes="ipsccp<func-spec>,deadargelim" -funcspec-max-iters=1 -force-specialization -S < %s | FileCheck %s --check-prefix=ONE-ITER
-; RUN: opt -passes="ipsccp<func-spec>,deadargelim" -funcspec-max-iters=0 -force-specialization -S < %s | FileCheck %s --check-prefix=DISABLED
+; RUN: opt -passes="ipsccp<func-spec>,deadargelim" -funcspec-for-literal-constant=false -force-specialization -S < %s | FileCheck %s
+; RUN: opt -passes="ipsccp<func-spec>,deadargelim" -funcspec-for-literal-constant=false -funcspec-max-iters=1 -force-specialization -S < %s | FileCheck %s --check-prefix=ONE-ITER
+; RUN: opt -passes="ipsccp<func-spec>,deadargelim" -funcspec-for-literal-constant=false -funcspec-max-iters=0 -force-specialization -S < %s | FileCheck %s --check-prefix=DISABLED
 
 
 define internal i32 @func(ptr %0, i32 %1, ptr nocapture %2) {
diff --git a/llvm/test/Transforms/FunctionSpecialization/function-specialization4.ll b/llvm/test/Transforms/FunctionSpecialization/function-specialization4.ll
index 4e5a196d6682..a6a990c34159 100644
--- a/llvm/test/Transforms/FunctionSpecialization/function-specialization4.ll
+++ b/llvm/test/Transforms/FunctionSpecialization/function-specialization4.ll
@@ -1,8 +1,10 @@
 ; RUN: opt -passes="ipsccp<func-spec>" -force-specialization \
-; RUN:   -funcspec-max-clones=2 -S < %s | FileCheck %s
+; RUN:   -funcspec-for-literal-constant=false -funcspec-max-clones=2 \
+; RUN:   -S < %s | FileCheck %s
 
 ; RUN: opt -passes="ipsccp<func-spec>" -force-specialization \
-; RUN:   -funcspec-max-clones=1 -S < %s | FileCheck %s --check-prefix=CONST1
+; RUN:   -funcspec-for-literal-constant=false -funcspec-max-clones=1 \
+; RUN:   -S < %s | FileCheck %s --check-prefix=CONST1
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
diff --git a/llvm/test/Transforms/FunctionSpecialization/get-possible-constants.ll b/llvm/test/Transforms/FunctionSpecialization/get-possible-constants.ll
index dfa1e5a42776..2f42125d8cf9 100644
--- a/llvm/test/Transforms/FunctionSpecialization/get-possible-constants.ll
+++ b/llvm/test/Transforms/FunctionSpecialization/get-possible-constants.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S --passes="ipsccp<func-spec>" < %s | FileCheck %s
+; RUN: opt -S --passes="ipsccp<func-spec>" -funcspec-for-literal-constant=false < %s | FileCheck %s
 define dso_local i32 @p0(i32 noundef %x) {
 entry:
   %add = add nsw i32 %x, 1
diff --git a/llvm/test/Transforms/FunctionSpecialization/global-rank.ll b/llvm/test/Transforms/FunctionSpecialization/global-rank.ll
index 1926e29ddee0..06185332f22e 100644
--- a/llvm/test/Transforms/FunctionSpecialization/global-rank.ll
+++ b/llvm/test/Transforms/FunctionSpecialization/global-rank.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S --passes="ipsccp<func-spec>" -funcspec-max-clones=1 < %s | FileCheck %s
+; RUN: opt -S --passes="ipsccp<func-spec>" -funcspec-for-literal-constant=false -funcspec-max-clones=1 < %s | FileCheck %s
 define internal i32 @f(i32 noundef %x, ptr nocapture noundef readonly %p, ptr nocapture noundef readonly %q) noinline {
 entry:
   %call = tail call i32 %p(i32 noundef %x)
diff --git a/llvm/test/Transforms/FunctionSpecialization/identical-specializations.ll b/llvm/test/Transforms/FunctionSpecialization/identical-specializations.ll
index 930ed6627f7f..97d77971a92d 100644
--- a/llvm/test/Transforms/FunctionSpecialization/identical-specializations.ll
+++ b/llvm/test/Transforms/FunctionSpecialization/identical-specializations.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 5
-; RUN: opt -passes="ipsccp<func-spec>" -force-specialization -S < %s | FileCheck %s
+; RUN: opt -passes="ipsccp<func-spec>" -force-specialization -funcspec-for-literal-constant=false -S < %s | FileCheck %s
 
 define i64 @main(i64 %x, i64 %y, i1 %flag) {
 entry:
diff --git a/llvm/test/Transforms/FunctionSpecialization/literal-const.ll b/llvm/test/Transforms/FunctionSpecialization/literal-const.ll
index 3eae3dc261fb..7d5e506064af 100644
--- a/llvm/test/Transforms/FunctionSpecialization/literal-const.ll
+++ b/llvm/test/Transforms/FunctionSpecialization/literal-const.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -S --passes="ipsccp<func-spec>" \
+; RUN:        -funcspec-for-literal-constant=false \
 ; RUN:        -force-specialization < %s | FileCheck %s -check-prefix CHECK-NOLIT
 ; RUN: opt -S --passes="ipsccp<func-spec>" \
-; RUN:        -funcspec-for-literal-constant \
 ; RUN:        -force-specialization < %s | FileCheck %s -check-prefix CHECK-LIT
 
 define i32 @f0(i32 noundef %x) {
diff --git a/llvm/test/Transforms/FunctionSpecialization/specialize-multiple-arguments.ll b/llvm/test/Transforms/FunctionSpecialization/specialize-multiple-arguments.ll
index a653760abb2c..73291600edb8 100644
--- a/llvm/test/Transforms/FunctionSpecialization/specialize-multiple-arguments.ll
+++ b/llvm/test/Transforms/FunctionSpecialization/specialize-multiple-arguments.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes="ipsccp<func-spec>" -funcspec-max-clones=0 -funcspec-min-function-size=14 -S < %s | FileCheck %s --check-prefix=NONE
-; RUN: opt -passes="ipsccp<func-spec>" -funcspec-max-clones=1 -funcspec-min-function-size=14 -S < %s | FileCheck %s --check-prefix=ONE
-; RUN: opt -passes="ipsccp<func-spec>" -funcspec-max-clones=2 -funcspec-min-function-size=14 -S < %s | FileCheck %s --check-prefix=TWO
-; RUN: opt -passes="ipsccp<func-spec>" -funcspec-max-clones=3 -funcspec-min-function-size=14 -S < %s | FileCheck %s --check-prefix=THREE
+; RUN: opt -passes="ipsccp<func-spec>" -funcspec-for-literal-constant=false -funcspec-max-clones=0 -funcspec-min-function-size=14 -S < %s | FileCheck %s --check-prefix=NONE
+; RUN: opt -passes="ipsccp<func-spec>" -funcspec-for-literal-constant=false -funcspec-max-clones=1 -funcspec-min-function-size=14 -S < %s | FileCheck %s --check-prefix=ONE
+; RUN: opt -passes="ipsccp<func-spec>" -funcspec-for-literal-constant=false -funcspec-max-clones=2 -funcspec-min-function-size=14 -S < %s | FileCheck %s --check-prefix=TWO
+; RUN: opt -passes="ipsccp<func-spec>" -funcspec-for-literal-constant=false -funcspec-max-clones=3 -funcspec-min-function-size=14 -S < %s | FileCheck %s --check-prefix=THREE
 
 ; Make sure that we iterate correctly after sorting the specializations:
 ; FnSpecialization: Specializations for function compute
diff --git a/llvm/test/Transforms/FunctionSpecialization/track-ptr-return.ll b/llvm/test/Transforms/FunctionSpecialization/track-ptr-return.ll
new file mode 100644
index 000000000000..f4ba0e72a1b4
--- /dev/null
+++ b/llvm/test/Transforms/FunctionSpecialization/track-ptr-return.ll
@@ -0,0 +1,147 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 5
+; RUN: opt -passes="ipsccp<func-spec>" -force-specialization \
+; RUN:     -funcspec-max-iters=3 -S < %s | FileCheck %s
+
+; RUN: opt -passes="ipsccp<func-spec>" -force-specialization          \
+; RUN:     -funcspec-for-literal-constant=false -funcspec-max-iters=3 \
+; RUN:     -S < %s | FileCheck %s --check-prefix=NOLIT
+
+@global_true = constant i1 true
+@global_false = constant i1 false
+
+define i64 @main() {
+entry:
+  %op1 = call ptr @select_op(ptr @global_true)
+  %op2 = call ptr @select_op(ptr @global_false)
+
+  %c1 = call i64 @compute(ptr %op1)
+  %c2 = call i64 @compute(ptr %op2)
+  %add = add i64 %c1, %c2
+  ret i64 %add
+}
+
+define ptr @select_op(ptr %flag) {
+  %flag.val = load i1, ptr %flag
+  %op = select i1 %flag.val, ptr @plus, ptr @minus
+  ret ptr %op
+}
+
+define internal i64 @compute(ptr %op) {
+entry:
+  %res = call i64 %op(i64 1)
+  ret i64 %res
+}
+
+define internal i64 @plus(i64 %x) {
+entry:
+  %sum = add i64 %x, 1
+  ret i64 %sum
+}
+
+define internal i64 @minus(i64 %x) {
+entry:
+  %diff = sub i64 %x, 1
+  ret i64 %diff
+}
+; CHECK-LABEL: define i64 @main() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OP1:%.*]] = call ptr @select_op.specialized.1(ptr @global_true)
+; CHECK-NEXT:    [[OP2:%.*]] = call ptr @select_op.specialized.2(ptr @global_false)
+; CHECK-NEXT:    [[C1:%.*]] = call i64 @compute.specialized.3(ptr @plus)
+; CHECK-NEXT:    [[C2:%.*]] = call i64 @compute.specialized.4(ptr @minus)
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[C1]], [[C2]]
+; CHECK-NEXT:    ret i64 [[ADD]]
+;
+;
+; CHECK-LABEL: define ptr @select_op(
+; CHECK-SAME: ptr [[FLAG:%.*]]) {
+; CHECK-NEXT:    [[FLAG_VAL:%.*]] = load i1, ptr [[FLAG]], align 1
+; CHECK-NEXT:    [[OP:%.*]] = select i1 [[FLAG_VAL]], ptr @plus, ptr @minus
+; CHECK-NEXT:    ret ptr [[OP]]
+;
+;
+; CHECK-LABEL: define internal i64 @plus(
+; CHECK-SAME: i64 [[X:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SUM:%.*]] = add i64 [[X]], 1
+; CHECK-NEXT:    ret i64 [[SUM]]
+;
+;
+; CHECK-LABEL: define internal i64 @minus(
+; CHECK-SAME: i64 [[X:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[DIFF:%.*]] = sub i64 [[X]], 1
+; CHECK-NEXT:    ret i64 [[DIFF]]
+;
+;
+; CHECK-LABEL: define internal ptr @select_op.specialized.1(
+; CHECK-SAME: ptr [[FLAG:%.*]]) {
+; CHECK-NEXT:    ret ptr poison
+;
+;
+; CHECK-LABEL: define internal ptr @select_op.specialized.2(
+; CHECK-SAME: ptr [[FLAG:%.*]]) {
+; CHECK-NEXT:    ret ptr poison
+;
+;
+; CHECK-LABEL: define internal i64 @compute.specialized.3(
+; CHECK-SAME: ptr [[OP:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[RES:%.*]] = call i64 @plus(i64 1)
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+;
+; CHECK-LABEL: define internal i64 @compute.specialized.4(
+; CHECK-SAME: ptr [[OP:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[RES:%.*]] = call i64 @minus(i64 1)
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+;
+; NOLIT-LABEL: define i64 @main() {
+; NOLIT-NEXT:  [[ENTRY:.*:]]
+; NOLIT-NEXT:    [[OP1:%.*]] = call ptr @select_op.specialized.1(ptr @global_true)
+; NOLIT-NEXT:    [[OP2:%.*]] = call ptr @select_op.specialized.2(ptr @global_false)
+; NOLIT-NEXT:    [[C1:%.*]] = call i64 @compute(ptr @plus)
+; NOLIT-NEXT:    [[C2:%.*]] = call i64 @compute(ptr @minus)
+; NOLIT-NEXT:    [[ADD:%.*]] = add i64 [[C1]], [[C2]]
+; NOLIT-NEXT:    ret i64 [[ADD]]
+;
+;
+; NOLIT-LABEL: define ptr @select_op(
+; NOLIT-SAME: ptr [[FLAG:%.*]]) {
+; NOLIT-NEXT:    [[FLAG_VAL:%.*]] = load i1, ptr [[FLAG]], align 1
+; NOLIT-NEXT:    [[OP:%.*]] = select i1 [[FLAG_VAL]], ptr @plus, ptr @minus
+; NOLIT-NEXT:    ret ptr [[OP]]
+;
+;
+; NOLIT-LABEL: define internal i64 @compute(
+; NOLIT-SAME: ptr [[OP:%.*]]) {
+; NOLIT-NEXT:  [[ENTRY:.*:]]
+; NOLIT-NEXT:    [[RES:%.*]] = call i64 [[OP]](i64 1)
+; NOLIT-NEXT:    ret i64 [[RES]]
+;
+;
+; NOLIT-LABEL: define internal i64 @plus(
+; NOLIT-SAME: i64 [[X:%.*]]) {
+; NOLIT-NEXT:  [[ENTRY:.*:]]
+; NOLIT-NEXT:    [[SUM:%.*]] = add i64 [[X]], 1
+; NOLIT-NEXT:    ret i64 [[SUM]]
+;
+;
+; NOLIT-LABEL: define internal i64 @minus(
+; NOLIT-SAME: i64 [[X:%.*]]) {
+; NOLIT-NEXT:  [[ENTRY:.*:]]
+; NOLIT-NEXT:    [[DIFF:%.*]] = sub i64 [[X]], 1
+; NOLIT-NEXT:    ret i64 [[DIFF]]
+;
+;
+; NOLIT-LABEL: define internal ptr @select_op.specialized.1(
+; NOLIT-SAME: ptr [[FLAG:%.*]]) {
+; NOLIT-NEXT:    ret ptr poison
+;
+;
+; NOLIT-LABEL: define internal ptr @select_op.specialized.2(
+; NOLIT-SAME: ptr [[FLAG:%.*]]) {
+; NOLIT-NEXT:    ret ptr poison
+;
-- 
GitLab


From 46944d1f950d042695197038ab3f1bf25ace261b Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Tue, 29 Oct 2024 06:43:40 -0500
Subject: [PATCH 418/693] [flang][OpenMP] Extract OMP version hint into helper
 functions, NFC (#113621)

---
 flang/lib/Semantics/check-omp-structure.cpp | 24 +++++++++++++--------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 599cc61a83bf..0c3a0e76df6a 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -38,6 +38,16 @@ namespace Fortran::semantics {
     CheckAllowedClause(llvm::omp::Y); \
   }
 
+std::string ThisVersion(unsigned version) {
+  std::string tv{
+      std::to_string(version / 10) + "." + std::to_string(version % 10)};
+  return "OpenMP v" + tv;
+}
+
+std::string TryVersion(unsigned version) {
+  return "try -fopenmp-version=" + std::to_string(version);
+}
+
 // 'OmpWorkshareBlockChecker' is used to check the validity of the assignment
 // statements and the expressions enclosed in an OpenMP Workshare construct
 class OmpWorkshareBlockChecker {
@@ -200,14 +210,10 @@ bool OmpStructureChecker::CheckAllowedClause(llvmOmpClause clause) {
       auto clauseName{parser::ToUpperCaseLetters(getClauseName(clause).str())};
       auto dirName{parser::ToUpperCaseLetters(getDirectiveName(dir).str())};
 
-      std::string thisVersion{
-          std::to_string(version / 10) + "." + std::to_string(version % 10)};
-      std::string goodVersion{std::to_string(allowedInVersion)};
-
       context_.Say(dirCtx.clauseSource,
-          "%s clause is not allowed on directive %s in OpenMP v%s, "
-          "try -fopenmp-version=%d"_err_en_US,
-          clauseName, dirName, thisVersion, allowedInVersion);
+          "%s clause is not allowed on directive %s in %s, %s"_err_en_US,
+          clauseName, dirName, ThisVersion(version),
+          TryVersion(allowedInVersion));
     }
   }
   return CheckAllowed(clause);
@@ -3378,8 +3384,8 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Lastprivate &x) {
           std::to_string(version / 10) + "." + std::to_string(version % 10)};
       context_.Say(GetContext().clauseSource,
           "LASTPRIVATE clause with CONDITIONAL modifier is not "
-          "allowed in OpenMP v%s, try -fopenmp-version=%d"_err_en_US,
-          thisVersion, allowedInVersion);
+          "allowed in %s, %s"_err_en_US,
+          ThisVersion(version), TryVersion(allowedInVersion));
     }
   }
 }
-- 
GitLab


From 2443549b853908352a3b7b9bd6c07616492814a1 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Tue, 29 Oct 2024 11:44:53 +0000
Subject: [PATCH 419/693] [IR] Remove some uses of StructType::setBody. NFC.
 (#113685)

It is simple to create the struct body up front, now that we have
transitioned to opaque pointers.
---
 llvm/lib/CodeGen/ShadowStackGCLowering.cpp   |  7 +++----
 llvm/lib/Target/X86/X86WinEHState.cpp        |  8 +++----
 llvm/lib/Transforms/Coroutines/CoroEarly.cpp |  4 ++--
 llvm/lib/Transforms/Coroutines/CoroFrame.cpp | 22 +++++++++++---------
 4 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/llvm/lib/CodeGen/ShadowStackGCLowering.cpp b/llvm/lib/CodeGen/ShadowStackGCLowering.cpp
index 232e5e2bb886..f8ab44124b3a 100644
--- a/llvm/lib/CodeGen/ShadowStackGCLowering.cpp
+++ b/llvm/lib/CodeGen/ShadowStackGCLowering.cpp
@@ -242,13 +242,12 @@ bool ShadowStackGCLoweringImpl::doInitialization(Module &M) {
   //   void *Roots[];          // Stack roots (in-place array, so we pretend).
   // };
 
-  StackEntryTy = StructType::create(M.getContext(), "gc_stackentry");
+  PointerType *StackEntryPtrTy = PointerType::getUnqual(M.getContext());
 
   EltTys.clear();
-  EltTys.push_back(PointerType::getUnqual(StackEntryTy));
+  EltTys.push_back(StackEntryPtrTy);
   EltTys.push_back(FrameMapPtrTy);
-  StackEntryTy->setBody(EltTys);
-  PointerType *StackEntryPtrTy = PointerType::getUnqual(StackEntryTy);
+  StackEntryTy = StructType::create(EltTys, "gc_stackentry");
 
   // Get the root chain if it already exists.
   Head = M.getGlobalVariable("llvm_gc_root_chain");
diff --git a/llvm/lib/Target/X86/X86WinEHState.cpp b/llvm/lib/Target/X86/X86WinEHState.cpp
index bc9fd801f94b..ef2127367301 100644
--- a/llvm/lib/Target/X86/X86WinEHState.cpp
+++ b/llvm/lib/Target/X86/X86WinEHState.cpp
@@ -210,13 +210,11 @@ Type *WinEHStatePass::getEHLinkRegistrationType() {
   if (EHLinkRegistrationTy)
     return EHLinkRegistrationTy;
   LLVMContext &Context = TheModule->getContext();
-  EHLinkRegistrationTy = StructType::create(Context, "EHRegistrationNode");
   Type *FieldTys[] = {
-      PointerType::getUnqual(
-          EHLinkRegistrationTy->getContext()), // EHRegistrationNode *Next
-      PointerType::getUnqual(Context) // EXCEPTION_DISPOSITION (*Handler)(...)
+      PointerType::getUnqual(Context), // EHRegistrationNode *Next
+      PointerType::getUnqual(Context)  // EXCEPTION_DISPOSITION (*Handler)(...)
   };
-  EHLinkRegistrationTy->setBody(FieldTys, false);
+  EHLinkRegistrationTy = StructType::create(FieldTys, "EHRegistrationNode");
   return EHLinkRegistrationTy;
 }
 
diff --git a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
index a3674306f3e1..5375448d2d2e 100644
--- a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
@@ -123,11 +123,11 @@ void Lowerer::lowerCoroNoop(IntrinsicInst *II) {
     Module &M = *II->getModule();
 
     // Create a noop.frame struct type.
-    StructType *FrameTy = StructType::create(C, "NoopCoro.Frame");
     auto *FnTy = FunctionType::get(Type::getVoidTy(C), Builder.getPtrTy(0),
                                    /*isVarArg=*/false);
     auto *FnPtrTy = Builder.getPtrTy(0);
-    FrameTy->setBody({FnPtrTy, FnPtrTy});
+    StructType *FrameTy =
+        StructType::create({FnPtrTy, FnPtrTy}, "NoopCoro.Frame");
 
     // Create a Noop function that does nothing.
     Function *NoopFn =
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 021fcc20c1f1..bb6126026d90 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -290,8 +290,8 @@ public:
     return Fields.size() - 1;
   }
 
-  /// Finish the layout and set the body on the given type.
-  void finish(StructType *Ty);
+  /// Finish the layout and create the struct type with the given name.
+  StructType *finish(StringRef Name);
 
   uint64_t getStructSize() const {
     assert(IsFinished && "not yet finished!");
@@ -464,7 +464,7 @@ void FrameTypeBuilder::addFieldForAllocas(const Function &F,
   });
 }
 
-void FrameTypeBuilder::finish(StructType *Ty) {
+StructType *FrameTypeBuilder::finish(StringRef Name) {
   assert(!IsFinished && "already finished!");
 
   // Prepare the optimal-layout field array.
@@ -526,7 +526,7 @@ void FrameTypeBuilder::finish(StructType *Ty) {
     LastOffset = Offset + F.Size;
   }
 
-  Ty->setBody(FieldTypes, Packed);
+  StructType *Ty = StructType::create(Context, FieldTypes, Name, Packed);
 
 #ifndef NDEBUG
   // Check that the IR layout matches the offsets we expect.
@@ -538,6 +538,8 @@ void FrameTypeBuilder::finish(StructType *Ty) {
 #endif
 
   IsFinished = true;
+
+  return Ty;
 }
 
 static void cacheDIVar(FrameDataInfo &FrameData,
@@ -866,11 +868,6 @@ static StructType *buildFrameType(Function &F, coro::Shape &Shape,
                                   bool OptimizeFrame) {
   LLVMContext &C = F.getContext();
   const DataLayout &DL = F.getDataLayout();
-  StructType *FrameTy = [&] {
-    SmallString<32> Name(F.getName());
-    Name.append(".Frame");
-    return StructType::create(C, Name);
-  }();
 
   // We will use this value to cap the alignment of spilled values.
   std::optional<Align> MaxFrameAlignment;
@@ -931,7 +928,12 @@ static StructType *buildFrameType(Function &F, coro::Shape &Shape,
     FrameData.setFieldIndex(S.first, Id);
   }
 
-  B.finish(FrameTy);
+  StructType *FrameTy = [&] {
+    SmallString<32> Name(F.getName());
+    Name.append(".Frame");
+    return B.finish(Name);
+  }();
+
   FrameData.updateLayoutIndex(B);
   Shape.FrameAlign = B.getStructAlign();
   Shape.FrameSize = B.getStructSize();
-- 
GitLab


From ec427df2b9c04cb3323babcf680dad8dcefaf228 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Tue, 29 Oct 2024 11:50:29 +0000
Subject: [PATCH 420/693] [AArch64] Add assembly/disassembly for FMOP4{A,S}
 (non-widening) half-precision instructions (#113343)

The new instructions are described in
https://developer.arm.com/documentation/ddi0602/2024-09/SME-Instructions
---
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |   2 +
 .../AArch64/AsmParser/AArch64AsmParser.cpp    |   3 +
 llvm/lib/Target/AArch64/SMEInstrFormats.td    |  39 +++-
 .../fmop4as-fp16-non-widening-diagnostics.s   | 220 ++++++++++++++++++
 .../SME2p2/fmop4as-fp16-non-widening.s        | 179 ++++++++++++++
 5 files changed, 442 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-non-widening-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-non-widening.s

diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index b0ce9e1a1b40..ae40911cc62a 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -1033,6 +1033,8 @@ let Predicates = [HasSME2p2, HasSMEF8F16], Uses = [FPMR, FPCR] in {
 
 let Predicates = [HasSME2p2, HasSMEF16F16] in {
   def FTMOPA_M2ZZZI_HtoH : sme_tmopa_16b<0b10001, ZZ_h_mul_r, ZPR16, "ftmopa">;
+  defm FMOP4A : sme2_fmop4as_fp16_non_widening<0, "fmop4a">;
+  defm FMOP4S : sme2_fmop4as_fp16_non_widening<1, "fmop4s">;
 } // [HasSME2p2, HasSMEF16F16]
 
 let Predicates = [HasSME2, HasSVEBFSCALE] in {
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index 8e267e658622..5a487be5723c 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -6262,6 +6262,8 @@ bool AArch64AsmParser::showMatchError(SMLoc Loc, unsigned ErrCode,
   case Match_InvalidMatrixTileVectorV128:
     return Error(Loc,
                  "invalid matrix operand, expected za[0-15]h.q or za[0-15]v.q");
+  case Match_InvalidMatrixTile16:
+    return Error(Loc, "invalid matrix operand, expected za[0-1].h");
   case Match_InvalidMatrixTile32:
     return Error(Loc, "invalid matrix operand, expected za[0-3].s");
   case Match_InvalidMatrixTile64:
@@ -6882,6 +6884,7 @@ bool AArch64AsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   case Match_InvalidSVEExactFPImmOperandHalfOne:
   case Match_InvalidSVEExactFPImmOperandHalfTwo:
   case Match_InvalidSVEExactFPImmOperandZeroOne:
+  case Match_InvalidMatrixTile16:
   case Match_InvalidMatrixTile32:
   case Match_InvalidMatrixTile64:
   case Match_InvalidMatrix:
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index 62a0c2e83d4a..330c540ffde4 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -5344,4 +5344,41 @@ multiclass sme2_multi4_fmul_mm<string mnemonic> {
 multiclass sme2_bfmul_multi<string mnemonic> {
   def _2Z2Z : sme2_multi2_fmul_mm<0b00, mnemonic, ZZ_h_mul_r>;
   def _4Z4Z : sme2_multi4_fmul_mm<0b00, mnemonic, ZZZZ_h_mul_r>;
-}
\ No newline at end of file
+}
+
+class sme2_fp16_quarter_tile_outer_product<bit M, bit N, bit S, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>
+    : I<(outs TileOp16:$ZAda),
+        (ins TileOp16:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),
+        mnemonic, "\t$ZAda, $Zn, $Zm",
+        "", []>, Sched<[]> {
+  bit ZAda;
+  bits<3> Zn;
+  bits<3> Zm;
+
+  let Inst{31-21} = 0b10000001000;
+  let Inst{20} = M;
+  let Inst{19-17} = Zm;
+  let Inst{16-10} = 0b0000000;
+  let Inst{9} = N;
+  let Inst{8-6} = Zn;
+  let Inst{5} = 0;
+  let Inst{4} = S;
+  let Inst{3-1} = 0b100;
+  let Inst{0} = ZAda;
+
+  let Constraints = "$ZAda = $_ZAda";
+}
+
+multiclass sme2_fmop4as_fp16_non_widening<bit S, string mnemonic> {
+  // Single vectors
+  def _MZZ_H : sme2_fp16_quarter_tile_outer_product<0, 0, S, mnemonic, ZPR16Mul2_Lo, ZPR16Mul2_Hi>;
+
+  // Multiple and single vectors
+  def _M2ZZ_H : sme2_fp16_quarter_tile_outer_product<0, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZPR16Mul2_Hi>;
+
+  // Single and multiple vectors
+  def _MZ2Z_H : sme2_fp16_quarter_tile_outer_product<1, 0, S, mnemonic, ZPR16Mul2_Lo, ZZ_h_mul_r_Hi>;
+
+  // Multiple vectors
+  def _M2Z2Z_H : sme2_fp16_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi>;
+}
diff --git a/llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-non-widening-diagnostics.s b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-non-widening-diagnostics.s
new file mode 100644
index 000000000000..0272721e0836
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-non-widening-diagnostics.s
@@ -0,0 +1,220 @@
+// RUN: not llvm-mc -triple=aarch64 -mattr=+sme2p2,+sme-f16f16 < %s 2>&1 | FileCheck %s
+
+// FMOP4A
+
+// Single vectors
+
+fmop4a za0.d, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za2.h, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.h, z0.s, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+fmop4a za0.h, z15.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+fmop4a za0.h, z16.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+fmop4a za0.h, z0.h, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4a za0.h, z12.h, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4a za0.h, z12.h, z14.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4a za0.h, z12.h, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Single and multiple vectors
+
+fmop4a za0.d, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za2.h, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.h, z0.s, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+fmop4a za0.h, z1.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+fmop4a za0.h, z16.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+fmop4a za0.h, z0.h, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.h, z0.h, {z17.h-z18.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.h, z0.h, {z12.h-z13.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+// Multiple and single vectors
+
+fmop4a za0.d, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za2.h, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.h, {z0.s-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: mismatched register size suffix
+
+fmop4a za0.h, {z1.h-z2.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.h, {z16.h-z17.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.h, {z0.h-z1.h}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4a za0.h, {z0.h-z1.h}, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4a za0.h, {z0.h-z1.h}, z12.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Multiple vectors
+
+fmop4a za0.d, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za2.h, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.h, {z0.s-z1.s}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.h, {z1.h-z2.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.h, {z18.h-z19.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.h, {z0.h-z1.h}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.h, {z0.h-z1.h}, {z19.h-z20.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.h, {z0.h-z1.h}, {z10.h-z11.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+
+// FMOP4S
+
+// Single vectors
+
+fmop4s za0.d, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4s za2.h, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.h, z0.s, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+fmop4s za0.h, z15.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+fmop4s za0.h, z16.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+fmop4s za0.h, z0.h, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4s za0.h, z12.h, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4s za0.h, z12.h, z14.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4s za0.h, z12.h, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Single and multiple vectors
+
+fmop4s za0.d, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4s za2.h, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.h, z0.s, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+fmop4s za0.h, z1.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+fmop4s za0.h, z16.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+fmop4s za0.h, z0.h, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.h, z0.h, {z17.h-z18.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.h, z0.h, {z12.h-z13.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+// Multiple and single vectors
+
+fmop4s za0.d, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4s za2.h, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.h, {z0.s-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: mismatched register size suffix
+
+fmop4s za0.h, {z1.h-z2.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.h, {z16.h-z17.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.h, {z0.h-z1.h}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4s za0.h, {z0.h-z1.h}, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4s za0.h, {z0.h-z1.h}, z12.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Multiple vectors
+
+fmop4s za0.d, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4s za2.h, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.h, {z0.s-z1.s}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.h, {z1.h-z2.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.h, {z18.h-z19.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.h, {z0.h-z1.h}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.h, {z0.h-z1.h}, {z19.h-z20.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.h, {z0.h-z1.h}, {z10.h-z11.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
diff --git a/llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-non-widening.s b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-non-widening.s
new file mode 100644
index 000000000000..2a94acd35e95
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-non-widening.s
@@ -0,0 +1,179 @@
+
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2,+sme-f16f16 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2,+sme-f16f16 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sme2p2,+sme-f16f16 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2,+sme-f16f16 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2p2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2,+sme-f16f16 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2p2,+sme-f16f16 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+// FMOP4A
+
+// Single vectors
+
+fmop4a  za0.h, z0.h, z16.h  // 10000001-00000000-00000000-00001000
+// CHECK-INST: fmop4a  za0.h, z0.h, z16.h
+// CHECK-ENCODING: [0x08,0x00,0x00,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81000008 <unknown>
+
+fmop4a  za1.h, z12.h, z24.h  // 10000001-00001000-00000001-10001001
+// CHECK-INST: fmop4a  za1.h, z12.h, z24.h
+// CHECK-ENCODING: [0x89,0x01,0x08,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81080189 <unknown>
+
+fmop4a  za1.h, z14.h, z30.h  // 10000001-00001110-00000001-11001001
+// CHECK-INST: fmop4a  za1.h, z14.h, z30.h
+// CHECK-ENCODING: [0xc9,0x01,0x0e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 810e01c9 <unknown>
+
+// Single and multiple vectors
+
+fmop4a  za0.h, z0.h, {z16.h-z17.h}  // 10000001-00010000-00000000-00001000
+// CHECK-INST: fmop4a  za0.h, z0.h, { z16.h, z17.h }
+// CHECK-ENCODING: [0x08,0x00,0x10,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81100008 <unknown>
+
+fmop4a  za1.h, z12.h, {z24.h-z25.h}  // 10000001-00011000-00000001-10001001
+// CHECK-INST: fmop4a  za1.h, z12.h, { z24.h, z25.h }
+// CHECK-ENCODING: [0x89,0x01,0x18,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81180189 <unknown>
+
+fmop4a  za1.h, z14.h, {z30.h-z31.h}  // 10000001-00011110-00000001-11001001
+// CHECK-INST: fmop4a  za1.h, z14.h, { z30.h, z31.h }
+// CHECK-ENCODING: [0xc9,0x01,0x1e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 811e01c9 <unknown>
+
+// Multiple and single vectors
+
+fmop4a  za0.h, {z0.h-z1.h}, z16.h  // 10000001-00000000-00000010-00001000
+// CHECK-INST: fmop4a  za0.h, { z0.h, z1.h }, z16.h
+// CHECK-ENCODING: [0x08,0x02,0x00,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81000208 <unknown>
+
+fmop4a  za1.h, {z12.h-z13.h}, z24.h  // 10000001-00001000-00000011-10001001
+// CHECK-INST: fmop4a  za1.h, { z12.h, z13.h }, z24.h
+// CHECK-ENCODING: [0x89,0x03,0x08,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81080389 <unknown>
+
+fmop4a  za1.h, {z14.h-z15.h}, z30.h  // 10000001-00001110-00000011-11001001
+// CHECK-INST: fmop4a  za1.h, { z14.h, z15.h }, z30.h
+// CHECK-ENCODING: [0xc9,0x03,0x0e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 810e03c9 <unknown>
+
+// Multiple vectors
+
+fmop4a  za0.h, {z0.h-z1.h}, {z16.h-z17.h}  // 10000001-00010000-00000010-00001000
+// CHECK-INST: fmop4a  za0.h, { z0.h, z1.h }, { z16.h, z17.h }
+// CHECK-ENCODING: [0x08,0x02,0x10,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81100208 <unknown>
+
+fmop4a  za1.h, {z12.h-z13.h}, {z24.h-z25.h}  // 10000001-00011000-00000011-10001001
+// CHECK-INST: fmop4a  za1.h, { z12.h, z13.h }, { z24.h, z25.h }
+// CHECK-ENCODING: [0x89,0x03,0x18,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81180389 <unknown>
+
+fmop4a  za1.h, {z14.h-z15.h}, {z30.h-z31.h}  // 10000001-00011110-00000011-11001001
+// CHECK-INST: fmop4a  za1.h, { z14.h, z15.h }, { z30.h, z31.h }
+// CHECK-ENCODING: [0xc9,0x03,0x1e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 811e03c9 <unknown>
+
+// FMOP4S
+
+// Single vectors
+
+fmop4s  za0.h, z0.h, z16.h  // 10000001-00000000-00000000-00011000
+// CHECK-INST: fmop4s  za0.h, z0.h, z16.h
+// CHECK-ENCODING: [0x18,0x00,0x00,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81000018 <unknown>
+
+fmop4s  za1.h, z12.h, z24.h  // 10000001-00001000-00000001-10011001
+// CHECK-INST: fmop4s  za1.h, z12.h, z24.h
+// CHECK-ENCODING: [0x99,0x01,0x08,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81080199 <unknown>
+
+fmop4s  za1.h, z14.h, z30.h  // 10000001-00001110-00000001-11011001
+// CHECK-INST: fmop4s  za1.h, z14.h, z30.h
+// CHECK-ENCODING: [0xd9,0x01,0x0e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 810e01d9 <unknown>
+
+// Single and multiple vectors
+
+fmop4s  za0.h, z0.h, {z16.h-z17.h}  // 10000001-00010000-00000000-00011000
+// CHECK-INST: fmop4s  za0.h, z0.h, { z16.h, z17.h }
+// CHECK-ENCODING: [0x18,0x00,0x10,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81100018 <unknown>
+
+fmop4s  za1.h, z12.h, {z24.h-z25.h}  // 10000001-00011000-00000001-10011001
+// CHECK-INST: fmop4s  za1.h, z12.h, { z24.h, z25.h }
+// CHECK-ENCODING: [0x99,0x01,0x18,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81180199 <unknown>
+
+fmop4s  za1.h, z14.h, {z30.h-z31.h}  // 10000001-00011110-00000001-11011001
+// CHECK-INST: fmop4s  za1.h, z14.h, { z30.h, z31.h }
+// CHECK-ENCODING: [0xd9,0x01,0x1e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 811e01d9 <unknown>
+
+// Multiple and single vectors
+
+fmop4s  za0.h, {z0.h-z1.h}, z16.h  // 10000001-00000000-00000010-00011000
+// CHECK-INST: fmop4s  za0.h, { z0.h, z1.h }, z16.h
+// CHECK-ENCODING: [0x18,0x02,0x00,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81000218 <unknown>
+
+fmop4s  za1.h, {z12.h-z13.h}, z24.h  // 10000001-00001000-00000011-10011001
+// CHECK-INST: fmop4s  za1.h, { z12.h, z13.h }, z24.h
+// CHECK-ENCODING: [0x99,0x03,0x08,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81080399 <unknown>
+
+fmop4s  za1.h, {z14.h-z15.h}, z30.h  // 10000001-00001110-00000011-11011001
+// CHECK-INST: fmop4s  za1.h, { z14.h, z15.h }, z30.h
+// CHECK-ENCODING: [0xd9,0x03,0x0e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 810e03d9 <unknown>
+
+// Multiple vectors
+
+fmop4s  za0.h, {z0.h-z1.h}, {z16.h-z17.h}  // 10000001-00010000-00000010-00011000
+// CHECK-INST: fmop4s  za0.h, { z0.h, z1.h }, { z16.h, z17.h }
+// CHECK-ENCODING: [0x18,0x02,0x10,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81100218 <unknown>
+
+fmop4s  za1.h, {z12.h-z13.h}, {z24.h-z25.h}  // 10000001-00011000-00000011-10011001
+// CHECK-INST: fmop4s  za1.h, { z12.h, z13.h }, { z24.h, z25.h }
+// CHECK-ENCODING: [0x99,0x03,0x18,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 81180399 <unknown>
+
+fmop4s  za1.h, {z14.h-z15.h}, {z30.h-z31.h}  // 10000001-00011110-00000011-11011001
+// CHECK-INST: fmop4s  za1.h, { z14.h, z15.h }, { z30.h, z31.h }
+// CHECK-ENCODING: [0xd9,0x03,0x1e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f16f16
+// CHECK-UNKNOWN: 811e03d9 <unknown>
-- 
GitLab


From e19a5fc6d306a81d181a9597a8b25c444c08d722 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye@arm.com>
Date: Tue, 29 Oct 2024 11:53:12 +0000
Subject: [PATCH 421/693] [FuncSpec] Improve accounting of specialization
 codesize growth (#113448)

Only accumulate the codesize increase of functions that are actually
specialized, rather than for every candidate specialization that we
analyse.

This fixes a subtle bug where prior analysis of candidate
specializations that were deemed unprofitable could prevent subsequent
profitable candidates from being recognised.
---
 .../Transforms/IPO/FunctionSpecialization.h   | 11 +++--
 .../Transforms/IPO/FunctionSpecialization.cpp | 41 +++++++++--------
 .../FunctionSpecialization/maxgrowth.ll       | 44 +++++++++++++++++++
 3 files changed, 73 insertions(+), 23 deletions(-)
 create mode 100644 llvm/test/Transforms/FunctionSpecialization/maxgrowth.ll

diff --git a/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h b/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
index f20b01c18630..e82155a6c729 100644
--- a/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
+++ b/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
@@ -138,13 +138,16 @@ struct Spec {
   // Profitability of the specialization.
   unsigned Score;
 
+  // Number of instructions in the specialization.
+  unsigned CodeSize;
+
   // List of call sites, matching this specialization.
   SmallVector<CallBase *> CallSites;
 
-  Spec(Function *F, const SpecSig &S, unsigned Score)
-      : F(F), Sig(S), Score(Score) {}
-  Spec(Function *F, const SpecSig &&S, unsigned Score)
-      : F(F), Sig(S), Score(Score) {}
+  Spec(Function *F, const SpecSig &S, unsigned Score, unsigned CodeSize)
+      : F(F), Sig(S), Score(Score), CodeSize(CodeSize) {}
+  Spec(Function *F, const SpecSig &&S, unsigned Score, unsigned CodeSize)
+      : F(F), Sig(S), Score(Score), CodeSize(CodeSize) {}
 };
 
 class InstCostVisitor : public InstVisitor<InstCostVisitor, Constant *> {
diff --git a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
index 8e6993d35d49..919d3143a13f 100644
--- a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
@@ -643,6 +643,18 @@ FunctionSpecializer::~FunctionSpecializer() {
   cleanUpSSA();
 }
 
+/// Get the unsigned Value of given Cost object. Assumes the Cost is always
+/// non-negative, which is true for both TCK_CodeSize and TCK_Latency, and
+/// always Valid.
+static unsigned getCostValue(const Cost &C) {
+  int64_t Value = *C.getValue();
+
+  assert(Value >= 0 && "CodeSize and Latency cannot be negative");
+  // It is safe to down cast since we know the arguments cannot be negative and
+  // Cost is of type int64_t.
+  return static_cast<unsigned>(Value);
+}
+
 /// Attempt to specialize functions in the module to enable constant
 /// propagation across function boundaries.
 ///
@@ -757,6 +769,11 @@ bool FunctionSpecializer::run() {
   SmallVector<Function *> Clones;
   for (unsigned I = 0; I < NSpecs; ++I) {
     Spec &S = AllSpecs[BestSpecs[I]];
+
+    // Accumulate the codesize growth for the function, now we are creating the
+    // specialization.
+    FunctionGrowth[S.F] += S.CodeSize;
+
     S.Clone = createSpecialization(S.F, S.Sig);
 
     // Update the known call sites to call the clone.
@@ -835,18 +852,6 @@ static Function *cloneCandidateFunction(Function *F, unsigned NSpecs) {
   return Clone;
 }
 
-/// Get the unsigned Value of given Cost object. Assumes the Cost is always
-/// non-negative, which is true for both TCK_CodeSize and TCK_Latency, and
-/// always Valid.
-static unsigned getCostValue(const Cost &C) {
-  int64_t Value = *C.getValue();
-
-  assert(Value >= 0 && "CodeSize and Latency cannot be negative");
-  // It is safe to down cast since we know the arguments cannot be negative and
-  // Cost is of type int64_t.
-  return static_cast<unsigned>(Value);
-}
-
 bool FunctionSpecializer::findSpecializations(Function *F, unsigned FuncSize,
                                               SmallVectorImpl<Spec> &AllSpecs,
                                               SpecMap &SM) {
@@ -922,16 +927,14 @@ bool FunctionSpecializer::findSpecializations(Function *F, unsigned FuncSize,
       }
       CodeSize += Visitor.getCodeSizeSavingsFromPendingPHIs();
 
+      unsigned CodeSizeSavings = getCostValue(CodeSize);
+      unsigned SpecSize = FuncSize - CodeSizeSavings;
+
       auto IsProfitable = [&]() -> bool {
         // No check required.
         if (ForceSpecialization)
           return true;
 
-        unsigned CodeSizeSavings = getCostValue(CodeSize);
-        // TODO: We should only accumulate codesize increase of specializations
-        // that are actually created.
-        FunctionGrowth[F] += FuncSize - CodeSizeSavings;
-
         LLVM_DEBUG(
             dbgs() << "FnSpecialization: Specialization bonus {Inlining = "
                    << Score << " (" << (Score * 100 / FuncSize) << "%)}\n");
@@ -962,7 +965,7 @@ bool FunctionSpecializer::findSpecializations(Function *F, unsigned FuncSize,
         if (LatencySavings < MinLatencySavings * FuncSize / 100)
           return false;
         // Maximum codesize growth.
-        if (FunctionGrowth[F] / FuncSize > MaxCodeSizeGrowth)
+        if ((FunctionGrowth[F] + SpecSize) / FuncSize > MaxCodeSizeGrowth)
           return false;
 
         Score += std::max(CodeSizeSavings, LatencySavings);
@@ -974,7 +977,7 @@ bool FunctionSpecializer::findSpecializations(Function *F, unsigned FuncSize,
         continue;
 
       // Create a new specialisation entry.
-      auto &Spec = AllSpecs.emplace_back(F, S, Score);
+      auto &Spec = AllSpecs.emplace_back(F, S, Score, SpecSize);
       if (CS.getFunction() != F)
         Spec.CallSites.push_back(&CS);
       const unsigned Index = AllSpecs.size() - 1;
diff --git a/llvm/test/Transforms/FunctionSpecialization/maxgrowth.ll b/llvm/test/Transforms/FunctionSpecialization/maxgrowth.ll
new file mode 100644
index 000000000000..82d1f7ae4a6e
--- /dev/null
+++ b/llvm/test/Transforms/FunctionSpecialization/maxgrowth.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 5
+; RUN: opt -passes="ipsccp<func-spec>" -funcspec-min-function-size=1       \
+; RUN:                                 -funcspec-for-literal-constant=true \
+; RUN:                                 -funcspec-min-codesize-savings=50   \
+; RUN:                                 -funcspec-min-latency-savings=50    \
+; RUN:                                 -funcspec-max-codesize-growth=1     \
+; RUN:                                 -S < %s | FileCheck %s
+
+; Verify that we are able to specialize a function successfully after analysis
+; of other specializations that are found to not be profitable.
+define void @test_specialize_after_failed_analysis(i32 %n) {
+entry:
+  %notspec0 = call i32 @add(i32 0, i32 %n)
+  %notspec1 = call i32 @add(i32 1, i32 %n)
+  %spec = call i32 @add(i32 1, i32 1)
+  ret void
+}
+
+define i32 @add(i32 %x, i32 %y) {
+entry:
+  %res = add i32 %x, %y
+  ret i32 %res
+}
+; CHECK-LABEL: define void @test_specialize_after_failed_analysis(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[NOTSPEC0:%.*]] = call i32 @add(i32 0, i32 [[N]])
+; CHECK-NEXT:    [[NOTSPEC1:%.*]] = call i32 @add(i32 1, i32 [[N]])
+; CHECK-NEXT:    [[SPEC:%.*]] = call i32 @add.specialized.1(i32 1, i32 1)
+; CHECK-NEXT:    ret void
+;
+;
+; CHECK-LABEL: define i32 @add(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+;
+; CHECK-LABEL: define internal i32 @add.specialized.1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    ret i32 poison
+;
-- 
GitLab


From c0cba25cdd06d700bdc15e9ae48c1fcadd0963bd Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Tue, 29 Oct 2024 21:13:54 +0900
Subject: [PATCH 422/693] [mlir][Transforms] Dialect conversion: Hardening
 `replaceOp` (#109540)

This commit adds extra checks/assertions to the
`ConversionPatternRewriterImpl::notifyOpReplaced` to improve its
robustness.

1. Replacing an `unrealized_conversion_cast` op that was created by the
driver is now forbidden and caught early during `replaceOp`. It may work
in some cases, but it is generally dangerous because the conversion
driver keeps track of these ops and performs some extra legalization
steps during the "finalize" phase. (Erasing is them is fine.)
2. `null` replacement values are no longer registered in the
`ConversionValueMapping`. This was an oversight in #106760. There is no
benefit in having `null` values in the `ConversionValueMapping`. (It may
even cause problems.)

This change is in preparation of merging the 1:1 and 1:N dialect
conversion drivers.
---
 .../Transforms/Utils/DialectConversion.cpp    | 31 ++++++++++++++-----
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 9f8a482d6e2d..44cf8331d55a 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -1382,16 +1382,21 @@ void ConversionPatternRewriterImpl::notifyOpReplaced(Operation *op,
   assert(newValues.size() == op->getNumResults());
   assert(!ignoredOps.contains(op) && "operation was already replaced");
 
+  // Check if replaced op is an unresolved materialization, i.e., an
+  // unrealized_conversion_cast op that was created by the conversion driver.
+  bool isUnresolvedMaterialization = false;
+  if (auto castOp = dyn_cast<UnrealizedConversionCastOp>(op))
+    if (unresolvedMaterializations.contains(castOp))
+      isUnresolvedMaterialization = true;
+
   // Create mappings for each of the new result values.
   for (auto [newValue, result] : llvm::zip(newValues, op->getResults())) {
     if (!newValue) {
       // This result was dropped and no replacement value was provided.
-      if (auto castOp = dyn_cast<UnrealizedConversionCastOp>(op)) {
-        if (unresolvedMaterializations.contains(castOp)) {
-          // Do not create another materializations if we are erasing a
-          // materialization.
-          continue;
-        }
+      if (isUnresolvedMaterialization) {
+        // Do not create another materializations if we are erasing a
+        // materialization.
+        continue;
       }
 
       // Materialize a replacement value "out of thin air".
@@ -1400,10 +1405,20 @@ void ConversionPatternRewriterImpl::notifyOpReplaced(Operation *op,
           result.getLoc(), /*inputs=*/ValueRange(),
           /*outputType=*/result.getType(), /*originalType=*/Type(),
           currentTypeConverter);
+    } else {
+      // Make sure that the user does not mess with unresolved materializations
+      // that were inserted by the conversion driver. We keep track of these
+      // ops in internal data structures. Erasing them must be allowed because
+      // this can happen when the user is erasing an entire block (including
+      // its body). But replacing them with another value should be forbidden
+      // to avoid problems with the `mapping`.
+      assert(!isUnresolvedMaterialization &&
+             "attempting to replace an unresolved materialization");
     }
 
-    // Remap, and check for any result type changes.
-    mapping.map(result, newValue);
+    // Remap result to replacement value.
+    if (newValue)
+      mapping.map(result, newValue);
   }
 
   appendRewrite<ReplaceOperationRewrite>(op, currentTypeConverter);
-- 
GitLab


From 8239ea3918828ab9c5ea8be1f4100d464f0bf3c0 Mon Sep 17 00:00:00 2001
From: Abid Qadeer <haqadeer@amd.com>
Date: Tue, 29 Oct 2024 12:22:43 +0000
Subject: [PATCH 423/693] [flang][debug] Support IndexType. (#113921)

---
 flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp |  4 ++++
 flang/test/Transforms/debug-index-type.fir            | 10 ++++++++++
 2 files changed, 14 insertions(+)
 create mode 100644 flang/test/Transforms/debug-index-type.fir

diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index 1ab6c76dae8e..8e516734a908 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -581,6 +581,10 @@ DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
                                   /*genAssociated=*/false);
   } else if (auto vecTy = mlir::dyn_cast_or_null<fir::VectorType>(Ty)) {
     return convertVectorType(vecTy, fileAttr, scope, declOp);
+  } else if (mlir::isa<mlir::IndexType>(Ty)) {
+    return genBasicType(context, mlir::StringAttr::get(context, "integer"),
+                        llvmTypeConverter.getIndexTypeBitwidth(),
+                        llvm::dwarf::DW_ATE_signed);
   } else if (auto boxTy = mlir::dyn_cast_or_null<fir::BoxType>(Ty)) {
     auto elTy = boxTy.getElementType();
     if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(elTy))
diff --git a/flang/test/Transforms/debug-index-type.fir b/flang/test/Transforms/debug-index-type.fir
new file mode 100644
index 000000000000..20bd8471d7cf
--- /dev/null
+++ b/flang/test/Transforms/debug-index-type.fir
@@ -0,0 +1,10 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func private @str(%arg0: index) -> i32 loc(#loc1)
+}
+#loc1 = loc("test.f90":5:1)
+
+// CHECK: #[[INT32_TY:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 32, encoding = DW_ATE_signed>
+// CHECK: #[[INT64_TY:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 64, encoding = DW_ATE_signed>
+// CHECK: #llvm.di_subroutine_type<{{.*}}types = #[[INT32_TY]], #[[INT64_TY]]>
-- 
GitLab


From d48c849ea94efb56d484393816e147afcec28d65 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Tue, 29 Oct 2024 08:00:44 -0500
Subject: [PATCH 424/693] [flang][OpenMP] Parsing support for iterator in
 DEPEND clause (#113622)

Warn about use of iterators OpenMP versions that didn't have them
(support added in 5.0). Emit a TODO error in lowering.
---
 flang/include/flang/Parser/parse-tree.h       |  4 +-
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    | 64 +++++++++++--------
 flang/lib/Lower/OpenMP/Clauses.cpp            | 15 +++--
 flang/lib/Parser/openmp-parsers.cpp           |  3 +-
 flang/lib/Semantics/check-omp-structure.cpp   |  9 +++
 .../test/Lower/OpenMP/Todo/depend-clause.f90  | 10 +++
 flang/test/Semantics/OpenMP/depend05.f90      |  9 +++
 7 files changed, 79 insertions(+), 35 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/Todo/depend-clause.f90
 create mode 100644 flang/test/Semantics/OpenMP/depend05.f90

diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 506a470c5557..174f4c631e9d 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3566,7 +3566,9 @@ struct OmpDependClause {
   WRAPPER_CLASS(Sink, std::list<OmpDependSinkVec>);
   struct InOut {
     TUPLE_CLASS_BOILERPLATE(InOut);
-    std::tuple<OmpTaskDependenceType, OmpObjectList> t;
+    std::tuple<std::optional<OmpIteratorModifier>, OmpTaskDependenceType,
+        OmpObjectList>
+        t;
   };
   std::variant<Source, Sink, InOut> u;
 };
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 8fb0dd4a1ec3..7c254ce67385 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -795,35 +795,43 @@ bool ClauseProcessor::processCopyprivate(
 bool ClauseProcessor::processDepend(mlir::omp::DependClauseOps &result) const {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
-  return findRepeatableClause<omp::clause::Depend>(
-      [&](const omp::clause::Depend &clause, const parser::CharBlock &) {
-        using Depend = omp::clause::Depend;
-        assert(std::holds_alternative<Depend::DepType>(clause.u) &&
-               "Only the form with dependence type is handled at the moment");
-        auto &depType = std::get<Depend::DepType>(clause.u);
-        auto kind = std::get<Depend::TaskDependenceType>(depType.t);
-        auto &objects = std::get<omp::ObjectList>(depType.t);
-
-        mlir::omp::ClauseTaskDependAttr dependTypeOperand =
-            genDependKindAttr(firOpBuilder, kind);
-        result.dependKinds.append(objects.size(), dependTypeOperand);
-
-        for (const omp::Object &object : objects) {
-          assert(object.ref() && "Expecting designator");
-
-          if (evaluate::ExtractSubstring(*object.ref())) {
-            TODO(converter.getCurrentLocation(),
-                 "substring not supported for task depend");
-          } else if (evaluate::IsArrayElement(*object.ref())) {
-            TODO(converter.getCurrentLocation(),
-                 "array sections not supported for task depend");
-          }
+  auto process = [&](const omp::clause::Depend &clause,
+                     const parser::CharBlock &) {
+    using Depend = omp::clause::Depend;
+    if (!std::holds_alternative<Depend::DepType>(clause.u)) {
+      TODO(converter.getCurrentLocation(),
+           "DEPEND clause with SINK or SOURCE is not supported yet");
+    }
+    auto &depType = std::get<Depend::DepType>(clause.u);
+    auto kind = std::get<Depend::TaskDependenceType>(depType.t);
+    auto &objects = std::get<omp::ObjectList>(depType.t);
 
-          semantics::Symbol *sym = object.sym();
-          const mlir::Value variable = converter.getSymbolAddress(*sym);
-          result.dependVars.push_back(variable);
-        }
-      });
+    if (std::get<std::optional<omp::clause::Iterator>>(depType.t)) {
+      TODO(converter.getCurrentLocation(),
+           "Support for iterator modifiers is not implemented yet");
+    }
+    mlir::omp::ClauseTaskDependAttr dependTypeOperand =
+        genDependKindAttr(firOpBuilder, kind);
+    result.dependKinds.append(objects.size(), dependTypeOperand);
+
+    for (const omp::Object &object : objects) {
+      assert(object.ref() && "Expecting designator");
+
+      if (evaluate::ExtractSubstring(*object.ref())) {
+        TODO(converter.getCurrentLocation(),
+             "substring not supported for task depend");
+      } else if (evaluate::IsArrayElement(*object.ref())) {
+        TODO(converter.getCurrentLocation(),
+             "array sections not supported for task depend");
+      }
+
+      semantics::Symbol *sym = object.sym();
+      const mlir::Value variable = converter.getSymbolAddress(*sym);
+      result.dependVars.push_back(variable);
+    }
+  };
+
+  return findRepeatableClause<omp::clause::Depend>(process);
 }
 
 bool ClauseProcessor::processHasDeviceAddr(
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index b1fa52751fbd..9483f643acd5 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -595,11 +595,16 @@ Depend make(const parser::OmpClause::Depend &inp,
           },
           // Depend::DepType
           [&](const wrapped::InOut &s) -> Variant {
-            auto &t0 = std::get<parser::OmpTaskDependenceType>(s.t);
-            auto &t1 = std::get<parser::OmpObjectList>(s.t);
-            return Depend::DepType{{/*TaskDependenceType=*/convert1(t0.v),
-                                    /*Iterator=*/std::nullopt,
-                                    /*LocatorList=*/makeObjects(t1, semaCtx)}};
+            auto &t0 =
+                std::get<std::optional<parser::OmpIteratorModifier>>(s.t);
+            auto &t1 = std::get<parser::OmpTaskDependenceType>(s.t);
+            auto &t2 = std::get<parser::OmpObjectList>(s.t);
+
+            auto &&maybeIter = maybeApply(
+                [&](auto &&s) { return makeIterator(s, semaCtx); }, t0);
+            return Depend::DepType{{/*TaskDependenceType=*/convert1(t1.v),
+                                    /*Iterator=*/std::move(maybeIter),
+                                    /*LocatorList=*/makeObjects(t2, semaCtx)}};
           },
       },
       inp.v.u)};
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 3ca4e93a6c9b..598439cbee87 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -376,7 +376,8 @@ TYPE_CONTEXT_PARSER("Omp Depend clause"_en_US,
         construct<OmpDependClause>(
             construct<OmpDependClause::Source>("SOURCE"_tok)) ||
         construct<OmpDependClause>(construct<OmpDependClause::InOut>(
-            Parser<OmpTaskDependenceType>{}, ":" >> Parser<OmpObjectList>{})))
+            maybe(Parser<OmpIteratorModifier>{} / ","_tok),
+            Parser<OmpTaskDependenceType>{} / ":", Parser<OmpObjectList>{})))
 
 // 2.15.3.7 LINEAR (linear-list: linear-step)
 //          linear-list -> list | modifier(list)
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 0c3a0e76df6a..8f3eb9fefee6 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -3312,6 +3312,15 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Depend &x) {
         }
       }
     }
+    if (std::get<std::optional<parser::OmpIteratorModifier>>(inOut->t)) {
+      unsigned version{context_.langOptions().OpenMPVersion};
+      unsigned allowedInVersion{50};
+      if (version < allowedInVersion) {
+        context_.Say(GetContext().clauseSource,
+            "Iterator modifiers are not supported in %s, %s"_warn_en_US,
+            ThisVersion(version), TryVersion(allowedInVersion));
+      }
+    }
   }
 }
 
diff --git a/flang/test/Lower/OpenMP/Todo/depend-clause.f90 b/flang/test/Lower/OpenMP/Todo/depend-clause.f90
new file mode 100644
index 000000000000..74525888c91d
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/depend-clause.f90
@@ -0,0 +1,10 @@
+!RUN: %not_todo_cmd bbc -emit-hlfir -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
+!RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
+
+!CHECK: Support for iterator modifiers is not implemented yet
+subroutine f00(x)
+  integer :: x(10)
+  !$omp task depend(iterator(i = 1:10), in: x(i))
+  x = 0
+  !$omp end task
+end
diff --git a/flang/test/Semantics/OpenMP/depend05.f90 b/flang/test/Semantics/OpenMP/depend05.f90
new file mode 100644
index 000000000000..53fd82bd08a9
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/depend05.f90
@@ -0,0 +1,9 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=45 -Werror
+
+subroutine f00(x)
+  integer :: x(10)
+!WARNING: Iterator modifiers are not supported in OpenMP v4.5, try -fopenmp-version=50
+  !$omp task depend(iterator(i = 1:10), in: x(i))
+  x = 0
+  !$omp end task
+end
-- 
GitLab


From 3c2d77185e315d4558368ccab92e7a86c74a9a83 Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac@arm.com>
Date: Tue, 29 Oct 2024 13:02:46 +0000
Subject: [PATCH 425/693] [AARCH64] Add assembly/disassembly for FMMLA
 instructions (#113313)

This patch adds assembly/disassembly for the following instructions:
FMMLA (widening, FP16 to FP32)
FMMLA (widening, FP8 to FP16)
FMMLA (widening, FP8 to FP32)

According to [1]

[1]https://developer.arm.com/documentation/ddi0602
---
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 18 +++++-
 llvm/lib/Target/AArch64/SVEInstrFormats.td    | 61 +++++++++----------
 .../SVE/matrix-multiply-fp-diagnostics.s      |  5 --
 .../MC/AArch64/SVE2/directive-arch-negative.s | 20 +++++-
 llvm/test/MC/AArch64/SVE2/directive-arch.s    | 14 ++++-
 .../SVE2/directive-arch_extension-negative.s  | 20 +++++-
 .../AArch64/SVE2/directive-arch_extension.s   | 14 ++++-
 .../MC/AArch64/SVE2/directive-cpu-negative.s  | 20 +++++-
 llvm/test/MC/AArch64/SVE2/directive-cpu.s     | 14 ++++-
 .../AArch64/SVE2/fmmla-f16f32mm-diagnostics.s | 18 ++++++
 llvm/test/MC/AArch64/SVE2/fmmla-f16f32mm.s    | 41 +++++++++++++
 .../AArch64/SVE2/fmmla-f8f16mm-diagnostics.s  | 24 ++++++++
 llvm/test/MC/AArch64/SVE2/fmmla-f8f16mm.s     | 39 ++++++++++++
 .../AArch64/SVE2/fmmla-f8f32mm-diagnostics.s  | 30 +++++++++
 llvm/test/MC/AArch64/SVE2/fmmla-f8f32mm.s     | 39 ++++++++++++
 15 files changed, 331 insertions(+), 46 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/SVE2/fmmla-f16f32mm-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2/fmmla-f16f32mm.s
 create mode 100644 llvm/test/MC/AArch64/SVE2/fmmla-f8f16mm-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2/fmmla-f8f16mm.s
 create mode 100644 llvm/test/MC/AArch64/SVE2/fmmla-f8f32mm-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2/fmmla-f8f32mm.s

diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 2b69903b133f..4f101d0d46b7 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -2427,7 +2427,7 @@ let Predicates = [HasBF16, HasSVEorSME] in {
 } // End HasBF16, HasSVEorSME
 
 let Predicates = [HasBF16, HasSVE] in {
-  defm BFMMLA_ZZZ   : sve_bfloat_matmul<"bfmmla", int_aarch64_sve_bfmmla>;
+  defm BFMMLA_ZZZ   : sve_fp_matrix_mla<0b01, "bfmmla", ZPR32, ZPR16, int_aarch64_sve_bfmmla, nxv4f32, nxv8bf16>;
 } // End HasBF16, HasSVE
 
 let Predicates = [HasBF16, HasSVEorSME] in {
@@ -3449,11 +3449,15 @@ let Predicates = [HasSVEorSME, HasMatMulInt8] in {
 } // End HasSVEorSME, HasMatMulInt8
 
 let Predicates = [HasSVE, HasMatMulFP32] in {
-  defm FMMLA_ZZZ_S : sve_fp_matrix_mla<0, "fmmla", ZPR32, int_aarch64_sve_fmmla, nxv4f32>;
+  defm FMMLA_ZZZ_S : sve_fp_matrix_mla<0b10, "fmmla", ZPR32, ZPR32, int_aarch64_sve_fmmla, nxv4f32, nxv4f32>;
 } // End HasSVE, HasMatMulFP32
 
+let Predicates = [HasSVE_F16F32MM] in {
+  def FMLLA_ZZZ_HtoS : sve_fp_matrix_mla<0b00, "fmmla", ZPR32, ZPR16>;
+} // End HasSVE_F16F32MM
+
 let Predicates = [HasSVE, HasMatMulFP64] in {
-  defm FMMLA_ZZZ_D : sve_fp_matrix_mla<1, "fmmla", ZPR64, int_aarch64_sve_fmmla, nxv2f64>;
+  defm FMMLA_ZZZ_D : sve_fp_matrix_mla<0b11, "fmmla", ZPR64, ZPR64, int_aarch64_sve_fmmla, nxv2f64, nxv2f64>;
   defm LD1RO_B_IMM : sve_mem_ldor_si<0b00, "ld1rob", Z_b, ZPR8,  nxv16i8, nxv16i1, AArch64ld1ro_z>;
   defm LD1RO_H_IMM : sve_mem_ldor_si<0b01, "ld1roh", Z_h, ZPR16, nxv8i16, nxv8i1,  AArch64ld1ro_z>;
   defm LD1RO_W_IMM : sve_mem_ldor_si<0b10, "ld1row", Z_s, ZPR32, nxv4i32, nxv4i1,  AArch64ld1ro_z>;
@@ -4342,6 +4346,14 @@ def FMLALLTB_ZZZ : sve2_fp8_mla<0b010, ZPR32, "fmlalltb">;
 def FMLALLTT_ZZZ : sve2_fp8_mla<0b011, ZPR32, "fmlalltt">;
 } // End HasSSVE_FP8FMA
 
+let Predicates = [HasSVE2, HasF8F32MM] in {
+  def FMMLA_ZZZ_BtoS :  sve2_fp8_mmla<0b0, ZPR32, "fmmla">;
+}
+
+let Predicates = [HasSVE2, HasF8F16MM] in {
+  def FMMLA_ZZZ_BtoH :  sve2_fp8_mmla<0b1, ZPR16, "fmmla">;
+}
+
 let Predicates = [HasSSVE_FP8DOT2] in {
 // FP8 Widening Dot-Product - Indexed Group
 defm FDOT_ZZZI_BtoH : sve2_fp8_dot_indexed_h<"fdot">;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 31312e00b919..d1ceb30f36dc 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -9042,30 +9042,6 @@ multiclass sve_float_dot_indexed<bit bf, bits<2> opc, ZPRRegOp src1_ty,
   def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, InVT, InVT, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>;
 }
 
-class sve_bfloat_matmul<string asm>
-: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm),
-  asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {
-  bits<5> Zm;
-  bits<5> Zda;
-  bits<5> Zn;
-  let Inst{31-21} = 0b01100100011;
-  let Inst{20-16} = Zm;
-  let Inst{15-10} = 0b111001;
-  let Inst{9-5}   = Zn;
-  let Inst{4-0}   = Zda;
-
-  let Constraints = "$Zda = $_Zda";
-  let DestructiveInstType = DestructiveOther;
-  let ElementSize = ElementSizeH;
-  let hasSideEffects = 0;
-  let mayRaiseFPException = 1;
-}
-
-multiclass sve_bfloat_matmul<string asm, SDPatternOperator op> {
-  def NAME : sve_bfloat_matmul<asm>;
-  def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>;
-}
-
 class sve_bfloat_convert<bit N, string asm>
 : I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn),
   asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> {
@@ -9188,14 +9164,14 @@ multiclass sve_int_dot_mixed_indexed<bit U, string asm, SDPatternOperator op> {
 // SVE Floating Point Matrix Multiply Accumulate Group
 //===----------------------------------------------------------------------===//
 
-class sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty>
-: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty:$Zm),
+class sve_fp_matrix_mla<bits<2> opc, string asm, ZPRRegOp zda_ty, ZPRRegOp reg_ty>
+: I<(outs zda_ty:$Zda), (ins zda_ty:$_Zda, reg_ty:$Zn, reg_ty:$Zm),
     asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {
   bits<5> Zda;
   bits<5> Zn;
   bits<5> Zm;
-  let Inst{31-23} = 0b011001001;
-  let Inst{22}    = sz;
+  let Inst{31-24} = 0b01100100;
+  let Inst{23-22} = opc;
   let Inst{21}    = 1;
   let Inst{20-16} = Zm;
   let Inst{15-10} = 0b111001;
@@ -9204,15 +9180,14 @@ class sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty>
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
-  let ElementSize = zprty.ElementSize;
   let hasSideEffects = 0;
   let mayRaiseFPException = 1;
 }
 
-multiclass sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty, SDPatternOperator op, ValueType vt> {
-  def NAME : sve_fp_matrix_mla<sz, asm, zprty>;
+multiclass sve_fp_matrix_mla<bits<2> opc, string asm, ZPRRegOp zda_ty, ZPRRegOp reg_ty, SDPatternOperator op, ValueType zda_vt, ValueType reg_vt> {
+  def NAME : sve_fp_matrix_mla<opc, asm, zda_ty, reg_ty>;
 
-  def : SVE_3_Op_Pat<vt, op , vt, vt, vt, !cast<Instruction>(NAME)>;
+  def : SVE_3_Op_Pat<zda_vt, op , zda_vt, reg_vt, reg_vt, !cast<Instruction>(NAME)>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -10632,6 +10607,28 @@ class sve2_fp8_mla_long_long_by_indexed_elem<bits<2> TT, string mnemonic>
   let Uses = [FPMR, FPCR];
 }
 
+// FP8 Matrix Multiply-accumulate Group
+class sve2_fp8_mmla<bit opc, ZPRRegOp dst_ty, string mnemonic>
+    : I<(outs dst_ty:$Zda),
+      (ins dst_ty:$_Zda, ZPR8:$Zn, ZPR8:$Zm),
+      mnemonic, "\t$Zda, $Zn, $Zm",
+      "", []>, Sched<[]>{
+  bits<5> Zda;
+  bits<5> Zn;
+  bits<5> Zm;
+  let Inst{31-23} = 0b011001000;
+  let Inst{22}    = opc;
+  let Inst{21}    = 0b1;
+  let Inst{20-16} = Zm;
+  let Inst{15-10} = 0b111000;
+  let Inst{9-5}   = Zn;
+  let Inst{4-0}   = Zda;
+  let Constraints = "$Zda = $_Zda";
+  let DestructiveInstType = DestructiveOther;
+  let ElementSize         = dst_ty.ElementSize;
+  let Uses = [FPMR, FPCR];
+}
+
 class sve_fp8_dot_indexed<bits<4> opc, ZPRRegOp dst_ty, Operand iop_ty, string mnemonic>
 : I<(outs dst_ty:$Zda), (ins dst_ty:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, iop_ty:$iop),
     mnemonic, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> {
diff --git a/llvm/test/MC/AArch64/SVE/matrix-multiply-fp-diagnostics.s b/llvm/test/MC/AArch64/SVE/matrix-multiply-fp-diagnostics.s
index 8ae4d4992844..2fe43f7aa844 100644
--- a/llvm/test/MC/AArch64/SVE/matrix-multiply-fp-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE/matrix-multiply-fp-diagnostics.s
@@ -3,11 +3,6 @@
 // --------------------------------------------------------------------------//
 // FMMLA (SVE)
 
-// Invalid element size
-
-fmmla z0.h, z1.h, z2.h
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
-
 // Mis-matched element size
 
 fmmla z0.d, z1.s, z2.s
diff --git a/llvm/test/MC/AArch64/SVE2/directive-arch-negative.s b/llvm/test/MC/AArch64/SVE2/directive-arch-negative.s
index 858aaf9d13ec..966bead071fe 100644
--- a/llvm/test/MC/AArch64/SVE2/directive-arch-negative.s
+++ b/llvm/test/MC/AArch64/SVE2/directive-arch-negative.s
@@ -30,8 +30,26 @@ bgrp z21.s, z10.s, z21.s
 // CHECK: error: instruction requires: sve2-bitperm
 // CHECK-NEXT: bgrp z21.s, z10.s, z21.s
 
+.arch armv9-a+f8f16mm
+.arch armv9-a+nof8f16mm
+fmmla   z23.h, z13.b, z8.b
+// CHECK: error: instruction requires: f8f16mm
+// CHECK-NEXT: fmmla   z23.h, z13.b, z8.b
+
+.arch armv9-a+f8f32mm
+.arch armv9-a+nof8f32mm
+fmmla   z23.s, z13.b, z8.b
+// CHECK: error: instruction requires: f8f32mm
+// CHECK-NEXT: fmmla   z23.s, z13.b, z8.b
+
+.arch armv9-a+sve-f16f32mm
+.arch armv9-a+nosve-f16f32mm
+fmmla   z23.s, z13.h, z8.h
+// CHECK: error: instruction requires: sve-f16f32mm
+// CHECK-NEXT: fmmla   z23.s, z13.h, z8.h
+
 .arch armv9-a+sve-bfscale
 .arch armv9-a+nosve-bfscale
 bfscale z0.h, p0/m, z0.h, z0.h
 // CHECK: error: instruction requires: sve-bfscale
-// CHECK-NEXT: bfscale z0.h, p0/m, z0.h, z0.h
\ No newline at end of file
+// CHECK-NEXT: bfscale z0.h, p0/m, z0.h, z0.h
diff --git a/llvm/test/MC/AArch64/SVE2/directive-arch.s b/llvm/test/MC/AArch64/SVE2/directive-arch.s
index b9710b67f8a1..99f6198a60ab 100644
--- a/llvm/test/MC/AArch64/SVE2/directive-arch.s
+++ b/llvm/test/MC/AArch64/SVE2/directive-arch.s
@@ -20,6 +20,18 @@ rax1 z0.d, z0.d, z0.d
 bgrp z21.s, z10.s, z21.s
 // CHECK: bgrp z21.s, z10.s, z21.s
 
+.arch armv9-a+f8f16mm
+fmmla   z23.h, z13.b, z8.b
+// CHECK: fmmla   z23.h, z13.b, z8.b
+
+.arch armv9-a+f8f32mm
+fmmla   z23.s, z13.b, z8.b
+// CHECK: fmmla   z23.s, z13.b, z8.b
+
+.arch armv9-a+sve-f16f32mm
+fmmla   z23.s, z13.h, z8.h
+// CHECK: fmmla   z23.s, z13.h, z8.h
+
 .arch armv9-a+sve-bfscale
 bfscale z0.h, p0/m, z0.h, z0.h
-// CHECK: bfscale z0.h, p0/m, z0.h, z0.h
\ No newline at end of file
+// CHECK: bfscale z0.h, p0/m, z0.h, z0.h
diff --git a/llvm/test/MC/AArch64/SVE2/directive-arch_extension-negative.s b/llvm/test/MC/AArch64/SVE2/directive-arch_extension-negative.s
index bd625d252626..e967f5aa60bd 100644
--- a/llvm/test/MC/AArch64/SVE2/directive-arch_extension-negative.s
+++ b/llvm/test/MC/AArch64/SVE2/directive-arch_extension-negative.s
@@ -30,8 +30,26 @@ bgrp z21.s, z10.s, z21.s
 // CHECK: error: instruction requires: sve2-bitperm
 // CHECK-NEXT: bgrp z21.s, z10.s, z21.s
 
+.arch_extension f8f16mm
+.arch_extension nof8f16mm
+fmmla   z23.h, z13.b, z8.b
+// CHECK: error: instruction requires: f8f16mm
+// CHECK-NEXT: fmmla   z23.h, z13.b, z8.b
+
+.arch_extension f8f32mm
+.arch_extension nof8f32mm
+fmmla   z23.s, z13.b, z8.b
+// CHECK: error: instruction requires: f8f32mm
+// CHECK-NEXT: fmmla   z23.s, z13.b, z8.b
+
+.arch_extension sve-f16f32mm
+.arch_extension nosve-f16f32mm
+fmmla   z23.s, z13.h, z8.h
+// CHECK: error: instruction requires: sve-f16f32mm
+// CHECK-NEXT: fmmla   z23.s, z13.h, z8.h
+
 .arch_extension sve-bfscale
 .arch_extension nosve-bfscale
 bfscale z0.h, p0/m, z0.h, z0.h
 // CHECK: error: instruction requires: sve-bfscale
-// CHECK-NEXT: bfscale z0.h, p0/m, z0.h, z0.h
\ No newline at end of file
+// CHECK-NEXT: bfscale z0.h, p0/m, z0.h, z0.h
diff --git a/llvm/test/MC/AArch64/SVE2/directive-arch_extension.s b/llvm/test/MC/AArch64/SVE2/directive-arch_extension.s
index 28d803bf7cc8..2fdbb525464d 100644
--- a/llvm/test/MC/AArch64/SVE2/directive-arch_extension.s
+++ b/llvm/test/MC/AArch64/SVE2/directive-arch_extension.s
@@ -20,6 +20,18 @@ rax1 z0.d, z0.d, z0.d
 bgrp z21.s, z10.s, z21.s
 // CHECK: bgrp z21.s, z10.s, z21.s
 
+.arch_extension f8f16mm
+fmmla   z23.h, z13.b, z8.b
+// CHECK: fmmla   z23.h, z13.b, z8.b
+
+.arch_extension f8f32mm
+fmmla   z23.s, z13.b, z8.b
+// CHECK: fmmla   z23.s, z13.b, z8.b
+
+.arch_extension sve-f16f32mm
+fmmla   z23.s, z13.h, z8.h
+// CHECK: fmmla   z23.s, z13.h, z8.h
+
 .arch_extension sve-bfscale
 bfscale z0.h, p0/m, z0.h, z0.h
-// CHECK: bfscale z0.h, p0/m, z0.h, z0.h
\ No newline at end of file
+// CHECK: bfscale z0.h, p0/m, z0.h, z0.h
diff --git a/llvm/test/MC/AArch64/SVE2/directive-cpu-negative.s b/llvm/test/MC/AArch64/SVE2/directive-cpu-negative.s
index 45a04a58eac3..9a8af638b703 100644
--- a/llvm/test/MC/AArch64/SVE2/directive-cpu-negative.s
+++ b/llvm/test/MC/AArch64/SVE2/directive-cpu-negative.s
@@ -30,8 +30,26 @@ bgrp z21.s, z10.s, z21.s
 // CHECK: error: instruction requires: sve2-bitperm
 // CHECK-NEXT: bgrp z21.s, z10.s, z21.s
 
+.cpu generic+sve2+f8f16mm
+.cpu generic+sve2+nof8f16mm
+fmmla   z23.h, z13.b, z8.b
+// CHECK: error: instruction requires: f8f16mm
+// CHECK-NEXT: fmmla   z23.h, z13.b, z8.b
+
+.cpu generic+sve2+f8f32mm
+.cpu generic+sve2+nof8f32mm
+fmmla   z23.s, z13.b, z8.b
+// CHECK: error: instruction requires: f8f32mm
+// CHECK-NEXT: fmmla   z23.s, z13.b, z8.b
+
+.cpu generic+sve-f16f32mm
+.cpu generic+nosve-f16f32mm
+fmmla   z23.s, z13.h, z8.h
+// CHECK: error: instruction requires: sve-f16f32mm
+// CHECK-NEXT: fmmla   z23.s, z13.h, z8.h
+
 .cpu generic+sve-bfscale
 .cpu generic+nosve-bfscale
 bfscale z0.h, p0/m, z0.h, z0.h
 // CHECK: error: instruction requires: sve-bfscale
-// CHECK-NEXT: bfscale z0.h, p0/m, z0.h, z0.h
\ No newline at end of file
+// CHECK-NEXT: bfscale z0.h, p0/m, z0.h, z0.h
diff --git a/llvm/test/MC/AArch64/SVE2/directive-cpu.s b/llvm/test/MC/AArch64/SVE2/directive-cpu.s
index 75d2321bf620..daa5ec510b22 100644
--- a/llvm/test/MC/AArch64/SVE2/directive-cpu.s
+++ b/llvm/test/MC/AArch64/SVE2/directive-cpu.s
@@ -20,6 +20,18 @@ rax1 z0.d, z0.d, z0.d
 bgrp z21.s, z10.s, z21.s
 // CHECK: bgrp z21.s, z10.s, z21.s
 
+.cpu generic+sve2+f8f16mm
+fmmla   z23.h, z13.b, z8.b
+// CHECK: fmmla   z23.h, z13.b, z8.b
+
+.cpu generic+sve2+f8f32mm
+fmmla   z23.s, z13.b, z8.b
+// CHECK: fmmla   z23.s, z13.b, z8.b
+
+.cpu generic+sve-f16f32mm
+fmmla   z23.s, z13.h, z8.h
+// CHECK: fmmla   z23.s, z13.h, z8.h
+
 .cpu generic+sve-bfscale
 bfscale z0.h, p0/m, z0.h, z0.h
-// CHECK: bfscale z0.h, p0/m, z0.h, z0.h
\ No newline at end of file
+// CHECK: bfscale z0.h, p0/m, z0.h, z0.h
diff --git a/llvm/test/MC/AArch64/SVE2/fmmla-f16f32mm-diagnostics.s b/llvm/test/MC/AArch64/SVE2/fmmla-f16f32mm-diagnostics.s
new file mode 100644
index 000000000000..924c123f0ca5
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2/fmmla-f16f32mm-diagnostics.s
@@ -0,0 +1,18 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve-f16f32mm  2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// FMMLA (SVE)
+
+// Invalid element size
+
+fmmla z0.s, z1.b, z2.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: f8f32mm
+fmmla z0.d, z1.h, z2.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+
+// Mis-matched element size
+
+fmmla z0.s, z1.h, z2.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+fmmla z0.s, z1.d, z2.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2/fmmla-f16f32mm.s b/llvm/test/MC/AArch64/SVE2/fmmla-f16f32mm.s
new file mode 100644
index 000000000000..84efcfe8b12e
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2/fmmla-f16f32mm.s
@@ -0,0 +1,41 @@
+
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve,+sve-f16f32mm < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve,+sve-f16f32mm < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve,+sve-f16f32mm - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve,+sve-f16f32mm < %s \
+// RUN:        | llvm-objdump -d --mattr=-sve - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve,+sve-f16f32mm < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve,+sve-f16f32mm -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+movprfx z23, z31
+fmmla   z23.s, z13.h, z8.h  // 01100100-00101000-11100101-10110111
+// CHECK-INST:  movprfx z23, z31
+// CHECK-INST: fmmla   z23.s, z13.h, z8.h
+// CHECK-ENCODING: [0xb7,0xe5,0x28,0x64]
+// CHECK-ERROR: instruction requires: sve-f16f32mm
+// CHECK-UNKNOWN: 6428e5b7 <unknown>
+
+fmmla   z0.s, z0.h, z0.h  // 01100100-00100000-11100100-00000000
+// CHECK-INST: fmmla   z0.s, z0.h, z0.h
+// CHECK-ENCODING: [0x00,0xe4,0x20,0x64]
+// CHECK-ERROR: instruction requires: sve-f16f32mm
+// CHECK-UNKNOWN: 6420e400 <unknown>
+
+fmmla   z23.s, z13.h, z8.h  // 01100100-00101000-11100101-10110111
+// CHECK-INST: fmmla   z23.s, z13.h, z8.h
+// CHECK-ENCODING: [0xb7,0xe5,0x28,0x64]
+// CHECK-ERROR: instruction requires: sve-f16f32mm
+// CHECK-UNKNOWN: 6428e5b7 <unknown>
+
+fmmla   z31.s, z31.h, z31.h  // 01100100-00111111-11100111-11111111
+// CHECK-INST: fmmla   z31.s, z31.h, z31.h
+// CHECK-ENCODING: [0xff,0xe7,0x3f,0x64]
+// CHECK-ERROR: instruction requires: sve-f16f32mm
+// CHECK-UNKNOWN: 643fe7ff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2/fmmla-f8f16mm-diagnostics.s b/llvm/test/MC/AArch64/SVE2/fmmla-f8f16mm-diagnostics.s
new file mode 100644
index 000000000000..59818d2d24a4
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2/fmmla-f8f16mm-diagnostics.s
@@ -0,0 +1,24 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+f8f16mm   2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+fmmla   z21.b, z10.b, z21.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fmmla   z21.b, z10.b, z21.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmmla   z21.d, z10.b, z21.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fmmla   z21.d, z10.b, z21.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmmla   z21.s, z10.h, z21.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sve-f16f32mm
+// CHECK-NEXT: fmmla   z21.s, z10.h, z21.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmmla   z21.s, z10.s, z21.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: f32mm
+// CHECK-NEXT: fmmla   z21.s, z10.s, z21.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2/fmmla-f8f16mm.s b/llvm/test/MC/AArch64/SVE2/fmmla-f8f16mm.s
new file mode 100644
index 000000000000..ff343548993c
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2/fmmla-f8f16mm.s
@@ -0,0 +1,39 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+f8f16mm < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+f8f16mm < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2,+f8f16mm - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+f8f16mm < %s \
+// RUN:        | llvm-objdump -d --mattr=-sve2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+f8f16mm < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2,+f8f16mm -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+movprfx z23, z31
+fmmla   z23.h, z13.b, z8.b  // 01100100-01101000-11100001-10110111
+// CHECK-INST:  movprfx z23, z31
+// CHECK-INST: fmmla   z23.h, z13.b, z8.b
+// CHECK-ENCODING: [0xb7,0xe1,0x68,0x64]
+// CHECK-ERROR: instruction requires: f8f16mm sve2
+// CHECK-UNKNOWN: 6468e1b7 <unknown>
+
+fmmla   z0.h, z0.b, z0.b  // 01100100-01100000-11100000-00000000
+// CHECK-INST: fmmla   z0.h, z0.b, z0.b
+// CHECK-ENCODING: [0x00,0xe0,0x60,0x64]
+// CHECK-ERROR: instruction requires: f8f16mm sve2
+// CHECK-UNKNOWN: 6460e000 <unknown>
+
+fmmla   z21.h, z10.b, z21.b  // 01100100-01110101-11100001-01010101
+// CHECK-INST: fmmla   z21.h, z10.b, z21.b
+// CHECK-ENCODING: [0x55,0xe1,0x75,0x64]
+// CHECK-ERROR: instruction requires: f8f16mm sve2
+// CHECK-UNKNOWN: 6475e155 <unknown>
+
+fmmla   z31.h, z31.b, z31.b  // 01100100-01111111-11100011-11111111
+// CHECK-INST: fmmla   z31.h, z31.b, z31.b
+// CHECK-ENCODING: [0xff,0xe3,0x7f,0x64]
+// CHECK-ERROR: instruction requires: f8f16mm sve2
+// CHECK-UNKNOWN: 647fe3ff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2/fmmla-f8f32mm-diagnostics.s b/llvm/test/MC/AArch64/SVE2/fmmla-f8f32mm-diagnostics.s
new file mode 100644
index 000000000000..0b1eb1b24e26
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2/fmmla-f8f32mm-diagnostics.s
@@ -0,0 +1,30 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+f8f32mm   2>&1 < %s| FileCheck %s
+
+
+// ------------------------------------------------------------------------- //
+// Invalid element width
+
+fmmla   z21.b, z10.b, z21.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fmmla   z21.b, z10.b, z21.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmmla   z21.h, z10.b, z21.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: f8f16mm
+// CHECK-NEXT: fmmla   z21.h, z10.b, z21.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmmla   z21.d, z10.b, z21.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fmmla   z21.d, z10.b, z21.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmmla   z21.s, z10.h, z21.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sve-f16f32mm
+// CHECK-NEXT: fmmla   z21.s, z10.h, z21.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmmla   z21.s, z10.s, z21.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: f32mm
+// CHECK-NEXT: fmmla   z21.s, z10.s, z21.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2/fmmla-f8f32mm.s b/llvm/test/MC/AArch64/SVE2/fmmla-f8f32mm.s
new file mode 100644
index 000000000000..8b59a112dc61
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2/fmmla-f8f32mm.s
@@ -0,0 +1,39 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+f8f32mm < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+f8f32mm < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2,+f8f32mm - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+f8f32mm < %s \
+// RUN:        | llvm-objdump -d --mattr=-sve2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+f8f32mm < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2,+f8f32mm -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+movprfx z23, z31
+fmmla   z23.s, z13.b, z8.b  // 01100100-00101000-11100001-10110111
+// CHECK-INST:  movprfx z23, z31
+// CHECK-INST: fmmla   z23.s, z13.b, z8.b
+// CHECK-ENCODING: [0xb7,0xe1,0x28,0x64]
+// CHECK-ERROR: instruction requires: f8f32mm sve2
+// CHECK-UNKNOWN: 6428e1b7 <unknown>
+
+fmmla   z0.s, z0.b, z0.b  // 01100100-00100000-11100000-00000000
+// CHECK-INST: fmmla   z0.s, z0.b, z0.b
+// CHECK-ENCODING: [0x00,0xe0,0x20,0x64]
+// CHECK-ERROR: instruction requires: f8f32mm sve2
+// CHECK-UNKNOWN: 6420e000 <unknown>
+
+fmmla   z21.s, z10.b, z21.b  // 01100100-00110101-11100001-01010101
+// CHECK-INST: fmmla   z21.s, z10.b, z21.b
+// CHECK-ENCODING: [0x55,0xe1,0x35,0x64]
+// CHECK-ERROR: instruction requires: f8f32mm sve2
+// CHECK-UNKNOWN: 6435e155 <unknown>
+
+fmmla   z31.s, z31.b, z31.b  // 01100100-00111111-11100011-11111111
+// CHECK-INST: fmmla   z31.s, z31.b, z31.b
+// CHECK-ENCODING: [0xff,0xe3,0x3f,0x64]
+// CHECK-ERROR: instruction requires: f8f32mm sve2
+// CHECK-UNKNOWN: 643fe3ff <unknown>
-- 
GitLab


From f7adacf57901ca65977f2af3502f434747cdd183 Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Tue, 29 Oct 2024 09:03:35 -0400
Subject: [PATCH 426/693] Nominate Vlad Serebrennikov for C++ DRs (#114040)

Vlad has been improving our C++ DR conformance testing story for many
months at this point and writing these kinds of test is sometimes non-
trivial, so having a maintainer specific for this is helpful.
---
 clang/Maintainers.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/clang/Maintainers.rst b/clang/Maintainers.rst
index ee5334b02f70..6635727e9eb6 100644
--- a/clang/Maintainers.rst
+++ b/clang/Maintainers.rst
@@ -226,6 +226,12 @@ C++ conformance
 | hubert.reinterpretcast\@gmail.com (email), hubert.reinterpretcast (Phabricator), hubert-reinterpretcast (GitHub)
 
 
+C++ Defect Reports
+~~~~~~~~~~~~~~~~~~
+| Vlad Serebrennikov
+| serebrennikov.vladislav\@gmail.com (email), Endilll (GitHub), Endill (Discord), Endill (Discourse)
+
+
 Objective-C/C++ conformance
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 | John McCall
-- 
GitLab


From 88e23eb2cfadbf92b109b0aec999378f0c2a1062 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 29 Oct 2024 06:08:50 -0700
Subject: [PATCH 427/693] DAG: Fix legalization of vector addrspacecasts
 (#113964)

---
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp |    3 +
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |    8 +
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp |   24 +-
 llvm/test/CodeGen/AMDGPU/addrspacecast.ll     | 1177 +++++++++++++++++
 4 files changed, 1200 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 47a9ae12248c..6ba12cfb8c51 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4375,6 +4375,9 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     Results.push_back(DAG.getNode(ISD::FP_TO_SINT, dl, ResVT, RoundNode));
     break;
   }
+  case ISD::ADDRSPACECAST:
+    Results.push_back(DAG.UnrollVectorOp(Node));
+    break;
   case ISD::GLOBAL_OFFSET_TABLE:
   case ISD::GlobalAddress:
   case ISD::GlobalTLSAddress:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 8f255cce1fe1..5403d787861d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -12590,6 +12590,14 @@ SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
       Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
                                 Operands[0],
                                 getValueType(ExtVT)));
+      break;
+    }
+    case ISD::ADDRSPACECAST: {
+      const auto *ASC = cast<AddrSpaceCastSDNode>(N);
+      Scalars.push_back(getAddrSpaceCast(dl, EltVT, Operands[0],
+                                         ASC->getSrcAddressSpace(),
+                                         ASC->getDestAddressSpace()));
+      break;
     }
     }
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 0f65df0763cc..e4b54c7d72b0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -512,18 +512,18 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
 
   for (MVT VT : VectorIntTypes) {
     // Expand the following operations for the current type by default.
-    setOperationAction({ISD::ADD,        ISD::AND,     ISD::FP_TO_SINT,
-                        ISD::FP_TO_UINT, ISD::MUL,     ISD::MULHU,
-                        ISD::MULHS,      ISD::OR,      ISD::SHL,
-                        ISD::SRA,        ISD::SRL,     ISD::ROTL,
-                        ISD::ROTR,       ISD::SUB,     ISD::SINT_TO_FP,
-                        ISD::UINT_TO_FP, ISD::SDIV,    ISD::UDIV,
-                        ISD::SREM,       ISD::UREM,    ISD::SMUL_LOHI,
-                        ISD::UMUL_LOHI,  ISD::SDIVREM, ISD::UDIVREM,
-                        ISD::SELECT,     ISD::VSELECT, ISD::SELECT_CC,
-                        ISD::XOR,        ISD::BSWAP,   ISD::CTPOP,
-                        ISD::CTTZ,       ISD::CTLZ,    ISD::VECTOR_SHUFFLE,
-                        ISD::SETCC},
+    setOperationAction({ISD::ADD,        ISD::AND,          ISD::FP_TO_SINT,
+                        ISD::FP_TO_UINT, ISD::MUL,          ISD::MULHU,
+                        ISD::MULHS,      ISD::OR,           ISD::SHL,
+                        ISD::SRA,        ISD::SRL,          ISD::ROTL,
+                        ISD::ROTR,       ISD::SUB,          ISD::SINT_TO_FP,
+                        ISD::UINT_TO_FP, ISD::SDIV,         ISD::UDIV,
+                        ISD::SREM,       ISD::UREM,         ISD::SMUL_LOHI,
+                        ISD::UMUL_LOHI,  ISD::SDIVREM,      ISD::UDIVREM,
+                        ISD::SELECT,     ISD::VSELECT,      ISD::SELECT_CC,
+                        ISD::XOR,        ISD::BSWAP,        ISD::CTPOP,
+                        ISD::CTTZ,       ISD::CTLZ,         ISD::VECTOR_SHUFFLE,
+                        ISD::SETCC,      ISD::ADDRSPACECAST},
                        VT, Expand);
   }
 
diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
index 7336543b41cb..236956c1829e 100644
--- a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
+++ b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
@@ -409,6 +409,1183 @@ define amdgpu_kernel void @use_constant32bit_to_flat_addrspacecast_1(ptr addrspa
   ret void
 }
 
+define <2 x ptr addrspace(5)> @addrspacecast_v2p0_to_v2p5(<2 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v2p0_to_v2p5:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; HSA-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; HSA-NEXT:    v_cndmask_b32_e32 v1, -1, v2, vcc
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <2 x ptr> %ptr to <2 x ptr addrspace(5)>
+  ret <2 x ptr addrspace(5)> %cast
+}
+
+define <3 x ptr addrspace(5)> @addrspacecast_v3p0_to_v3p5(<3 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v3p0_to_v3p5:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; HSA-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; HSA-NEXT:    v_cndmask_b32_e32 v1, -1, v2, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; HSA-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <3 x ptr> %ptr to <3 x ptr addrspace(5)>
+  ret <3 x ptr addrspace(5)> %cast
+}
+
+define <4 x ptr addrspace(5)> @addrspacecast_v4p0_to_v4p5(<4 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v4p0_to_v4p5:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; HSA-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; HSA-NEXT:    v_cndmask_b32_e32 v1, -1, v2, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; HSA-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; HSA-NEXT:    v_cndmask_b32_e32 v3, -1, v6, vcc
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <4 x ptr> %ptr to <4 x ptr addrspace(5)>
+  ret <4 x ptr addrspace(5)> %cast
+}
+
+define <8 x ptr addrspace(5)> @addrspacecast_v8p0_to_v8p5(<8 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v8p0_to_v8p5:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; HSA-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; HSA-NEXT:    v_cndmask_b32_e32 v1, -1, v2, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; HSA-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; HSA-NEXT:    v_cndmask_b32_e32 v3, -1, v6, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; HSA-NEXT:    v_cndmask_b32_e32 v4, -1, v8, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; HSA-NEXT:    v_cndmask_b32_e32 v5, -1, v10, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; HSA-NEXT:    v_cndmask_b32_e32 v6, -1, v12, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[14:15]
+; HSA-NEXT:    v_cndmask_b32_e32 v7, -1, v14, vcc
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <8 x ptr> %ptr to <8 x ptr addrspace(5)>
+  ret <8 x ptr addrspace(5)> %cast
+}
+
+define <16 x ptr addrspace(5)> @addrspacecast_v16p0_to_v16p5(<16 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v16p0_to_v16p5:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; HSA-NEXT:    v_cmp_ne_u64_e64 s[4:5], 0, v[24:25]
+; HSA-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; HSA-NEXT:    v_cmp_ne_u64_e64 s[6:7], 0, v[26:27]
+; HSA-NEXT:    v_cndmask_b32_e32 v1, -1, v2, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; HSA-NEXT:    v_cmp_ne_u64_e64 s[8:9], 0, v[28:29]
+; HSA-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; HSA-NEXT:    v_cndmask_b32_e32 v3, -1, v6, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; HSA-NEXT:    v_cndmask_b32_e32 v4, -1, v8, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; HSA-NEXT:    v_cndmask_b32_e32 v5, -1, v10, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; HSA-NEXT:    v_cndmask_b32_e64 v13, -1, v26, s[6:7]
+; HSA-NEXT:    v_cndmask_b32_e32 v6, -1, v12, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[14:15]
+; HSA-NEXT:    v_cndmask_b32_e64 v12, -1, v24, s[4:5]
+; HSA-NEXT:    v_cndmask_b32_e32 v7, -1, v14, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[16:17]
+; HSA-NEXT:    v_cndmask_b32_e64 v14, -1, v28, s[8:9]
+; HSA-NEXT:    v_cndmask_b32_e32 v8, -1, v16, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[18:19]
+; HSA-NEXT:    v_cndmask_b32_e32 v9, -1, v18, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[20:21]
+; HSA-NEXT:    v_cndmask_b32_e32 v10, -1, v20, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[22:23]
+; HSA-NEXT:    v_cndmask_b32_e32 v11, -1, v22, vcc
+; HSA-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[30:31]
+; HSA-NEXT:    v_cndmask_b32_e32 v15, -1, v30, vcc
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <16 x ptr> %ptr to <16 x ptr addrspace(5)>
+  ret <16 x ptr addrspace(5)> %cast
+}
+
+define <2 x ptr> @addrspacecast_v2p5_to_v2p0(<2 x ptr addrspace(5)> %ptr) {
+; CI-LABEL: addrspacecast_v2p5_to_v2p0:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT:    s_load_dword s4, s[6:7], 0x11
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; CI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v3, s4
+; CI-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; CI-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; CI-NEXT:    v_mov_b32_e32 v1, v4
+; CI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: addrspacecast_v2p5_to_v2p0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, v4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <2 x ptr addrspace(5)> %ptr to <2 x ptr>
+  ret <2 x ptr> %cast
+}
+
+define <3 x ptr> @addrspacecast_v3p5_to_v3p0(<3 x ptr addrspace(5)> %ptr) {
+; CI-LABEL: addrspacecast_v3p5_to_v3p0:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT:    s_load_dword s4, s[6:7], 0x11
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; CI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v5, s4
+; CI-NEXT:    v_cndmask_b32_e32 v7, 0, v5, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; CI-NEXT:    v_cndmask_b32_e32 v6, 0, v1, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; CI-NEXT:    v_cndmask_b32_e32 v4, 0, v2, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; CI-NEXT:    v_mov_b32_e32 v1, v7
+; CI-NEXT:    v_mov_b32_e32 v2, v6
+; CI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: addrspacecast_v3p5_to_v3p0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, 0, v5, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, v7
+; GFX9-NEXT:    v_mov_b32_e32 v2, v6
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <3 x ptr addrspace(5)> %ptr to <3 x ptr>
+  ret <3 x ptr> %cast
+}
+
+define <4 x ptr> @addrspacecast_v4p5_to_v4p0(<4 x ptr addrspace(5)> %ptr) {
+; CI-LABEL: addrspacecast_v4p5_to_v4p0:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT:    s_load_dword s4, s[6:7], 0x11
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; CI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v7, s4
+; CI-NEXT:    v_cndmask_b32_e32 v10, 0, v7, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; CI-NEXT:    v_cndmask_b32_e32 v8, 0, v1, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v9, 0, v7, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; CI-NEXT:    v_cndmask_b32_e32 v4, 0, v2, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v5, 0, v7, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v3
+; CI-NEXT:    v_cndmask_b32_e32 v6, 0, v3, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v7, 0, v7, vcc
+; CI-NEXT:    v_mov_b32_e32 v1, v10
+; CI-NEXT:    v_mov_b32_e32 v2, v8
+; CI-NEXT:    v_mov_b32_e32 v3, v9
+; CI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: addrspacecast_v4p5_to_v4p0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v7, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v10, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v9, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, 0, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, 0, v7, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, v10
+; GFX9-NEXT:    v_mov_b32_e32 v2, v8
+; GFX9-NEXT:    v_mov_b32_e32 v3, v9
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <4 x ptr addrspace(5)> %ptr to <4 x ptr>
+  ret <4 x ptr> %cast
+}
+
+define <8 x ptr> @addrspacecast_v8p5_to_v8p0(<8 x ptr addrspace(5)> %ptr) {
+; CI-LABEL: addrspacecast_v8p5_to_v8p0:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT:    s_load_dword s4, s[6:7], 0x11
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; CI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v15, s4
+; CI-NEXT:    v_cndmask_b32_e32 v22, 0, v15, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; CI-NEXT:    v_cndmask_b32_e32 v16, 0, v1, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v17, 0, v15, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; CI-NEXT:    v_cndmask_b32_e32 v18, 0, v2, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v19, 0, v15, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v3
+; CI-NEXT:    v_cndmask_b32_e32 v20, 0, v3, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v21, 0, v15, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v4
+; CI-NEXT:    v_cndmask_b32_e32 v8, 0, v4, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v9, 0, v15, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v5
+; CI-NEXT:    v_cndmask_b32_e32 v10, 0, v5, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v11, 0, v15, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v6
+; CI-NEXT:    v_cndmask_b32_e32 v12, 0, v6, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v13, 0, v15, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v7
+; CI-NEXT:    v_cndmask_b32_e32 v14, 0, v7, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v15, 0, v15, vcc
+; CI-NEXT:    v_mov_b32_e32 v1, v22
+; CI-NEXT:    v_mov_b32_e32 v2, v16
+; CI-NEXT:    v_mov_b32_e32 v3, v17
+; CI-NEXT:    v_mov_b32_e32 v4, v18
+; CI-NEXT:    v_mov_b32_e32 v5, v19
+; CI-NEXT:    v_mov_b32_e32 v6, v20
+; CI-NEXT:    v_mov_b32_e32 v7, v21
+; CI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: addrspacecast_v8p5_to_v8p0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v15, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v22, 0, v15, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v16, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v17, 0, v15, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v18, 0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v19, 0, v15, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v20, 0, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v21, 0, v15, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, 0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v9, 0, v15, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v5
+; GFX9-NEXT:    v_cndmask_b32_e32 v10, 0, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v11, 0, v15, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v6
+; GFX9-NEXT:    v_cndmask_b32_e32 v12, 0, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v13, 0, v15, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v14, 0, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v15, 0, v15, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, v22
+; GFX9-NEXT:    v_mov_b32_e32 v2, v16
+; GFX9-NEXT:    v_mov_b32_e32 v3, v17
+; GFX9-NEXT:    v_mov_b32_e32 v4, v18
+; GFX9-NEXT:    v_mov_b32_e32 v5, v19
+; GFX9-NEXT:    v_mov_b32_e32 v6, v20
+; GFX9-NEXT:    v_mov_b32_e32 v7, v21
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <8 x ptr addrspace(5)> %ptr to <8 x ptr>
+  ret <8 x ptr> %cast
+}
+
+define <16 x ptr> @addrspacecast_v16p5_to_v16p0(<16 x ptr addrspace(5)> %ptr) {
+; CI-LABEL: addrspacecast_v16p5_to_v16p0:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT:    s_load_dword s4, s[6:7], 0x11
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; CI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; CI-NEXT:    v_cmp_ne_u32_e64 s[6:7], -1, v6
+; CI-NEXT:    v_cmp_ne_u32_e64 s[8:9], -1, v7
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v31, s4
+; CI-NEXT:    v_cndmask_b32_e32 v48, 0, v31, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; CI-NEXT:    v_cndmask_b32_e32 v35, 0, v1, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v33, 0, v31, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; CI-NEXT:    v_cndmask_b32_e32 v36, 0, v2, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v49, 0, v31, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v3
+; CI-NEXT:    v_cndmask_b32_e32 v37, 0, v3, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v34, 0, v31, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v4
+; CI-NEXT:    v_cmp_ne_u32_e64 s[4:5], -1, v5
+; CI-NEXT:    v_cndmask_b32_e32 v38, 0, v4, vcc
+; CI-NEXT:    v_cndmask_b32_e64 v50, 0, v5, s[4:5]
+; CI-NEXT:    v_cndmask_b32_e64 v39, 0, v6, s[6:7]
+; CI-NEXT:    v_cndmask_b32_e64 v32, 0, v7, s[8:9]
+; CI-NEXT:    v_cmp_ne_u32_e64 s[10:11], -1, v8
+; CI-NEXT:    v_cmp_ne_u32_e64 s[12:13], -1, v9
+; CI-NEXT:    v_cmp_ne_u32_e64 s[14:15], -1, v10
+; CI-NEXT:    v_cmp_ne_u32_e64 s[16:17], -1, v11
+; CI-NEXT:    v_cmp_ne_u32_e64 s[18:19], -1, v12
+; CI-NEXT:    v_cmp_ne_u32_e64 s[20:21], -1, v13
+; CI-NEXT:    v_cmp_ne_u32_e64 s[22:23], -1, v14
+; CI-NEXT:    v_cmp_ne_u32_e64 s[24:25], -1, v15
+; CI-NEXT:    v_cndmask_b32_e64 v16, 0, v8, s[10:11]
+; CI-NEXT:    v_cndmask_b32_e64 v18, 0, v9, s[12:13]
+; CI-NEXT:    v_cndmask_b32_e64 v20, 0, v10, s[14:15]
+; CI-NEXT:    v_cndmask_b32_e64 v22, 0, v11, s[16:17]
+; CI-NEXT:    v_cndmask_b32_e64 v24, 0, v12, s[18:19]
+; CI-NEXT:    v_cndmask_b32_e64 v26, 0, v13, s[20:21]
+; CI-NEXT:    v_cndmask_b32_e64 v28, 0, v14, s[22:23]
+; CI-NEXT:    v_cndmask_b32_e64 v30, 0, v15, s[24:25]
+; CI-NEXT:    v_cndmask_b32_e32 v9, 0, v31, vcc
+; CI-NEXT:    v_cndmask_b32_e64 v11, 0, v31, s[4:5]
+; CI-NEXT:    v_cndmask_b32_e64 v13, 0, v31, s[6:7]
+; CI-NEXT:    v_cndmask_b32_e64 v15, 0, v31, s[8:9]
+; CI-NEXT:    v_cndmask_b32_e64 v17, 0, v31, s[10:11]
+; CI-NEXT:    v_cndmask_b32_e64 v19, 0, v31, s[12:13]
+; CI-NEXT:    v_cndmask_b32_e64 v21, 0, v31, s[14:15]
+; CI-NEXT:    v_cndmask_b32_e64 v23, 0, v31, s[16:17]
+; CI-NEXT:    v_cndmask_b32_e64 v25, 0, v31, s[18:19]
+; CI-NEXT:    v_cndmask_b32_e64 v27, 0, v31, s[20:21]
+; CI-NEXT:    v_cndmask_b32_e64 v29, 0, v31, s[22:23]
+; CI-NEXT:    v_cndmask_b32_e64 v31, 0, v31, s[24:25]
+; CI-NEXT:    v_mov_b32_e32 v1, v48
+; CI-NEXT:    v_mov_b32_e32 v2, v35
+; CI-NEXT:    v_mov_b32_e32 v3, v33
+; CI-NEXT:    v_mov_b32_e32 v4, v36
+; CI-NEXT:    v_mov_b32_e32 v5, v49
+; CI-NEXT:    v_mov_b32_e32 v6, v37
+; CI-NEXT:    v_mov_b32_e32 v7, v34
+; CI-NEXT:    v_mov_b32_e32 v8, v38
+; CI-NEXT:    v_mov_b32_e32 v10, v50
+; CI-NEXT:    v_mov_b32_e32 v12, v39
+; CI-NEXT:    v_mov_b32_e32 v14, v32
+; CI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: addrspacecast_v16p5_to_v16p0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v31, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v48, 0, v31, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v35, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v33, 0, v31, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v36, 0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v49, 0, v31, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v37, 0, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v34, 0, v31, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v4
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], -1, v5
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[6:7], -1, v6
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[8:9], -1, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v38, 0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v50, 0, v5, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v39, 0, v6, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v32, 0, v7, s[8:9]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[10:11], -1, v8
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[12:13], -1, v9
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[14:15], -1, v10
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[16:17], -1, v11
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[18:19], -1, v12
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[20:21], -1, v13
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[22:23], -1, v14
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[24:25], -1, v15
+; GFX9-NEXT:    v_cndmask_b32_e64 v16, 0, v8, s[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v18, 0, v9, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v20, 0, v10, s[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, v11, s[16:17]
+; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, v12, s[18:19]
+; GFX9-NEXT:    v_cndmask_b32_e64 v26, 0, v13, s[20:21]
+; GFX9-NEXT:    v_cndmask_b32_e64 v28, 0, v14, s[22:23]
+; GFX9-NEXT:    v_cndmask_b32_e64 v30, 0, v15, s[24:25]
+; GFX9-NEXT:    v_cndmask_b32_e32 v9, 0, v31, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v11, 0, v31, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v13, 0, v31, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v15, 0, v31, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v17, 0, v31, s[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v19, 0, v31, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v21, 0, v31, s[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, v31, s[16:17]
+; GFX9-NEXT:    v_cndmask_b32_e64 v25, 0, v31, s[18:19]
+; GFX9-NEXT:    v_cndmask_b32_e64 v27, 0, v31, s[20:21]
+; GFX9-NEXT:    v_cndmask_b32_e64 v29, 0, v31, s[22:23]
+; GFX9-NEXT:    v_cndmask_b32_e64 v31, 0, v31, s[24:25]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v48
+; GFX9-NEXT:    v_mov_b32_e32 v2, v35
+; GFX9-NEXT:    v_mov_b32_e32 v3, v33
+; GFX9-NEXT:    v_mov_b32_e32 v4, v36
+; GFX9-NEXT:    v_mov_b32_e32 v5, v49
+; GFX9-NEXT:    v_mov_b32_e32 v6, v37
+; GFX9-NEXT:    v_mov_b32_e32 v7, v34
+; GFX9-NEXT:    v_mov_b32_e32 v8, v38
+; GFX9-NEXT:    v_mov_b32_e32 v10, v50
+; GFX9-NEXT:    v_mov_b32_e32 v12, v39
+; GFX9-NEXT:    v_mov_b32_e32 v14, v32
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <16 x ptr addrspace(5)> %ptr to <16 x ptr>
+  ret <16 x ptr> %cast
+}
+
+define <2 x ptr addrspace(3)> @addrspacecast_v2p0_to_v2p3(<2 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v2p0_to_v2p3:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; HSA-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; HSA-NEXT:    v_cndmask_b32_e32 v1, -1, v2, vcc
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <2 x ptr> %ptr to <2 x ptr addrspace(3)>
+  ret <2 x ptr addrspace(3)> %cast
+}
+
+define <3 x ptr addrspace(3)> @addrspacecast_v3p0_to_v3p3(<3 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v3p0_to_v3p3:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; HSA-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; HSA-NEXT:    v_cndmask_b32_e32 v1, -1, v2, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; HSA-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <3 x ptr> %ptr to <3 x ptr addrspace(3)>
+  ret <3 x ptr addrspace(3)> %cast
+}
+
+define <4 x ptr addrspace(3)> @addrspacecast_v4p0_to_v4p3(<4 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v4p0_to_v4p3:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; HSA-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; HSA-NEXT:    v_cndmask_b32_e32 v1, -1, v2, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; HSA-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; HSA-NEXT:    v_cndmask_b32_e32 v3, -1, v6, vcc
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <4 x ptr> %ptr to <4 x ptr addrspace(3)>
+  ret <4 x ptr addrspace(3)> %cast
+}
+
+define <8 x ptr addrspace(3)> @addrspacecast_v8p0_to_v8p3(<8 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v8p0_to_v8p3:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; HSA-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; HSA-NEXT:    v_cndmask_b32_e32 v1, -1, v2, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; HSA-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; HSA-NEXT:    v_cndmask_b32_e32 v3, -1, v6, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; HSA-NEXT:    v_cndmask_b32_e32 v4, -1, v8, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; HSA-NEXT:    v_cndmask_b32_e32 v5, -1, v10, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; HSA-NEXT:    v_cndmask_b32_e32 v6, -1, v12, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[14:15]
+; HSA-NEXT:    v_cndmask_b32_e32 v7, -1, v14, vcc
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <8 x ptr> %ptr to <8 x ptr addrspace(3)>
+  ret <8 x ptr addrspace(3)> %cast
+}
+
+define <16 x ptr addrspace(3)> @addrspacecast_v16p0_to_v16p3(<16 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v16p0_to_v16p3:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; HSA-NEXT:    v_cmp_ne_u64_e64 s[4:5], 0, v[24:25]
+; HSA-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; HSA-NEXT:    v_cmp_ne_u64_e64 s[6:7], 0, v[26:27]
+; HSA-NEXT:    v_cndmask_b32_e32 v1, -1, v2, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; HSA-NEXT:    v_cmp_ne_u64_e64 s[8:9], 0, v[28:29]
+; HSA-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; HSA-NEXT:    v_cndmask_b32_e32 v3, -1, v6, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; HSA-NEXT:    v_cndmask_b32_e32 v4, -1, v8, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; HSA-NEXT:    v_cndmask_b32_e32 v5, -1, v10, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; HSA-NEXT:    v_cndmask_b32_e64 v13, -1, v26, s[6:7]
+; HSA-NEXT:    v_cndmask_b32_e32 v6, -1, v12, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[14:15]
+; HSA-NEXT:    v_cndmask_b32_e64 v12, -1, v24, s[4:5]
+; HSA-NEXT:    v_cndmask_b32_e32 v7, -1, v14, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[16:17]
+; HSA-NEXT:    v_cndmask_b32_e64 v14, -1, v28, s[8:9]
+; HSA-NEXT:    v_cndmask_b32_e32 v8, -1, v16, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[18:19]
+; HSA-NEXT:    v_cndmask_b32_e32 v9, -1, v18, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[20:21]
+; HSA-NEXT:    v_cndmask_b32_e32 v10, -1, v20, vcc
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[22:23]
+; HSA-NEXT:    v_cndmask_b32_e32 v11, -1, v22, vcc
+; HSA-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[30:31]
+; HSA-NEXT:    v_cndmask_b32_e32 v15, -1, v30, vcc
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <16 x ptr> %ptr to <16 x ptr addrspace(3)>
+  ret <16 x ptr addrspace(3)> %cast
+}
+
+define <2 x ptr> @addrspacecast_v2p3_to_v2p0(<2 x ptr addrspace(3)> %ptr) {
+; CI-LABEL: addrspacecast_v2p3_to_v2p0:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT:    s_load_dword s4, s[6:7], 0x10
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; CI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v3, s4
+; CI-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; CI-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; CI-NEXT:    v_mov_b32_e32 v1, v4
+; CI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: addrspacecast_v2p3_to_v2p0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b64 s[4:5], src_shared_base
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, v4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <2 x ptr addrspace(3)> %ptr to <2 x ptr>
+  ret <2 x ptr> %cast
+}
+
+define <3 x ptr> @addrspacecast_v3p3_to_v3p0(<3 x ptr addrspace(3)> %ptr) {
+; CI-LABEL: addrspacecast_v3p3_to_v3p0:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT:    s_load_dword s4, s[6:7], 0x10
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; CI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v5, s4
+; CI-NEXT:    v_cndmask_b32_e32 v7, 0, v5, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; CI-NEXT:    v_cndmask_b32_e32 v6, 0, v1, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; CI-NEXT:    v_cndmask_b32_e32 v4, 0, v2, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; CI-NEXT:    v_mov_b32_e32 v1, v7
+; CI-NEXT:    v_mov_b32_e32 v2, v6
+; CI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: addrspacecast_v3p3_to_v3p0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b64 s[4:5], src_shared_base
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, 0, v5, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, v7
+; GFX9-NEXT:    v_mov_b32_e32 v2, v6
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <3 x ptr addrspace(3)> %ptr to <3 x ptr>
+  ret <3 x ptr> %cast
+}
+
+define <4 x ptr> @addrspacecast_v4p3_to_v4p0(<4 x ptr addrspace(3)> %ptr) {
+; CI-LABEL: addrspacecast_v4p3_to_v4p0:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT:    s_load_dword s4, s[6:7], 0x10
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; CI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v7, s4
+; CI-NEXT:    v_cndmask_b32_e32 v10, 0, v7, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; CI-NEXT:    v_cndmask_b32_e32 v8, 0, v1, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v9, 0, v7, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; CI-NEXT:    v_cndmask_b32_e32 v4, 0, v2, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v5, 0, v7, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v3
+; CI-NEXT:    v_cndmask_b32_e32 v6, 0, v3, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v7, 0, v7, vcc
+; CI-NEXT:    v_mov_b32_e32 v1, v10
+; CI-NEXT:    v_mov_b32_e32 v2, v8
+; CI-NEXT:    v_mov_b32_e32 v3, v9
+; CI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: addrspacecast_v4p3_to_v4p0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b64 s[4:5], src_shared_base
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v7, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v10, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v9, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, 0, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, 0, v7, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, v10
+; GFX9-NEXT:    v_mov_b32_e32 v2, v8
+; GFX9-NEXT:    v_mov_b32_e32 v3, v9
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <4 x ptr addrspace(3)> %ptr to <4 x ptr>
+  ret <4 x ptr> %cast
+}
+
+define <8 x ptr> @addrspacecast_v8p3_to_v8p0(<8 x ptr addrspace(3)> %ptr) {
+; CI-LABEL: addrspacecast_v8p3_to_v8p0:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT:    s_load_dword s4, s[6:7], 0x10
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; CI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v15, s4
+; CI-NEXT:    v_cndmask_b32_e32 v22, 0, v15, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; CI-NEXT:    v_cndmask_b32_e32 v16, 0, v1, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v17, 0, v15, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; CI-NEXT:    v_cndmask_b32_e32 v18, 0, v2, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v19, 0, v15, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v3
+; CI-NEXT:    v_cndmask_b32_e32 v20, 0, v3, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v21, 0, v15, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v4
+; CI-NEXT:    v_cndmask_b32_e32 v8, 0, v4, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v9, 0, v15, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v5
+; CI-NEXT:    v_cndmask_b32_e32 v10, 0, v5, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v11, 0, v15, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v6
+; CI-NEXT:    v_cndmask_b32_e32 v12, 0, v6, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v13, 0, v15, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v7
+; CI-NEXT:    v_cndmask_b32_e32 v14, 0, v7, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v15, 0, v15, vcc
+; CI-NEXT:    v_mov_b32_e32 v1, v22
+; CI-NEXT:    v_mov_b32_e32 v2, v16
+; CI-NEXT:    v_mov_b32_e32 v3, v17
+; CI-NEXT:    v_mov_b32_e32 v4, v18
+; CI-NEXT:    v_mov_b32_e32 v5, v19
+; CI-NEXT:    v_mov_b32_e32 v6, v20
+; CI-NEXT:    v_mov_b32_e32 v7, v21
+; CI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: addrspacecast_v8p3_to_v8p0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b64 s[4:5], src_shared_base
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v15, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v22, 0, v15, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v16, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v17, 0, v15, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v18, 0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v19, 0, v15, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v20, 0, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v21, 0, v15, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, 0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v9, 0, v15, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v5
+; GFX9-NEXT:    v_cndmask_b32_e32 v10, 0, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v11, 0, v15, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v6
+; GFX9-NEXT:    v_cndmask_b32_e32 v12, 0, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v13, 0, v15, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v14, 0, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v15, 0, v15, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, v22
+; GFX9-NEXT:    v_mov_b32_e32 v2, v16
+; GFX9-NEXT:    v_mov_b32_e32 v3, v17
+; GFX9-NEXT:    v_mov_b32_e32 v4, v18
+; GFX9-NEXT:    v_mov_b32_e32 v5, v19
+; GFX9-NEXT:    v_mov_b32_e32 v6, v20
+; GFX9-NEXT:    v_mov_b32_e32 v7, v21
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <8 x ptr addrspace(3)> %ptr to <8 x ptr>
+  ret <8 x ptr> %cast
+}
+
+define <16 x ptr> @addrspacecast_v16p3_to_v16p0(<16 x ptr addrspace(3)> %ptr) {
+; CI-LABEL: addrspacecast_v16p3_to_v16p0:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT:    s_load_dword s4, s[6:7], 0x10
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; CI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; CI-NEXT:    v_cmp_ne_u32_e64 s[6:7], -1, v6
+; CI-NEXT:    v_cmp_ne_u32_e64 s[8:9], -1, v7
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v31, s4
+; CI-NEXT:    v_cndmask_b32_e32 v48, 0, v31, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; CI-NEXT:    v_cndmask_b32_e32 v35, 0, v1, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v33, 0, v31, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; CI-NEXT:    v_cndmask_b32_e32 v36, 0, v2, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v49, 0, v31, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v3
+; CI-NEXT:    v_cndmask_b32_e32 v37, 0, v3, vcc
+; CI-NEXT:    v_cndmask_b32_e32 v34, 0, v31, vcc
+; CI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v4
+; CI-NEXT:    v_cmp_ne_u32_e64 s[4:5], -1, v5
+; CI-NEXT:    v_cndmask_b32_e32 v38, 0, v4, vcc
+; CI-NEXT:    v_cndmask_b32_e64 v50, 0, v5, s[4:5]
+; CI-NEXT:    v_cndmask_b32_e64 v39, 0, v6, s[6:7]
+; CI-NEXT:    v_cndmask_b32_e64 v32, 0, v7, s[8:9]
+; CI-NEXT:    v_cmp_ne_u32_e64 s[10:11], -1, v8
+; CI-NEXT:    v_cmp_ne_u32_e64 s[12:13], -1, v9
+; CI-NEXT:    v_cmp_ne_u32_e64 s[14:15], -1, v10
+; CI-NEXT:    v_cmp_ne_u32_e64 s[16:17], -1, v11
+; CI-NEXT:    v_cmp_ne_u32_e64 s[18:19], -1, v12
+; CI-NEXT:    v_cmp_ne_u32_e64 s[20:21], -1, v13
+; CI-NEXT:    v_cmp_ne_u32_e64 s[22:23], -1, v14
+; CI-NEXT:    v_cmp_ne_u32_e64 s[24:25], -1, v15
+; CI-NEXT:    v_cndmask_b32_e64 v16, 0, v8, s[10:11]
+; CI-NEXT:    v_cndmask_b32_e64 v18, 0, v9, s[12:13]
+; CI-NEXT:    v_cndmask_b32_e64 v20, 0, v10, s[14:15]
+; CI-NEXT:    v_cndmask_b32_e64 v22, 0, v11, s[16:17]
+; CI-NEXT:    v_cndmask_b32_e64 v24, 0, v12, s[18:19]
+; CI-NEXT:    v_cndmask_b32_e64 v26, 0, v13, s[20:21]
+; CI-NEXT:    v_cndmask_b32_e64 v28, 0, v14, s[22:23]
+; CI-NEXT:    v_cndmask_b32_e64 v30, 0, v15, s[24:25]
+; CI-NEXT:    v_cndmask_b32_e32 v9, 0, v31, vcc
+; CI-NEXT:    v_cndmask_b32_e64 v11, 0, v31, s[4:5]
+; CI-NEXT:    v_cndmask_b32_e64 v13, 0, v31, s[6:7]
+; CI-NEXT:    v_cndmask_b32_e64 v15, 0, v31, s[8:9]
+; CI-NEXT:    v_cndmask_b32_e64 v17, 0, v31, s[10:11]
+; CI-NEXT:    v_cndmask_b32_e64 v19, 0, v31, s[12:13]
+; CI-NEXT:    v_cndmask_b32_e64 v21, 0, v31, s[14:15]
+; CI-NEXT:    v_cndmask_b32_e64 v23, 0, v31, s[16:17]
+; CI-NEXT:    v_cndmask_b32_e64 v25, 0, v31, s[18:19]
+; CI-NEXT:    v_cndmask_b32_e64 v27, 0, v31, s[20:21]
+; CI-NEXT:    v_cndmask_b32_e64 v29, 0, v31, s[22:23]
+; CI-NEXT:    v_cndmask_b32_e64 v31, 0, v31, s[24:25]
+; CI-NEXT:    v_mov_b32_e32 v1, v48
+; CI-NEXT:    v_mov_b32_e32 v2, v35
+; CI-NEXT:    v_mov_b32_e32 v3, v33
+; CI-NEXT:    v_mov_b32_e32 v4, v36
+; CI-NEXT:    v_mov_b32_e32 v5, v49
+; CI-NEXT:    v_mov_b32_e32 v6, v37
+; CI-NEXT:    v_mov_b32_e32 v7, v34
+; CI-NEXT:    v_mov_b32_e32 v8, v38
+; CI-NEXT:    v_mov_b32_e32 v10, v50
+; CI-NEXT:    v_mov_b32_e32 v12, v39
+; CI-NEXT:    v_mov_b32_e32 v14, v32
+; CI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: addrspacecast_v16p3_to_v16p0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b64 s[4:5], src_shared_base
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v31, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v48, 0, v31, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v35, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v33, 0, v31, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v36, 0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v49, 0, v31, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v37, 0, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v34, 0, v31, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v4
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], -1, v5
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[6:7], -1, v6
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[8:9], -1, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v38, 0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v50, 0, v5, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v39, 0, v6, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v32, 0, v7, s[8:9]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[10:11], -1, v8
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[12:13], -1, v9
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[14:15], -1, v10
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[16:17], -1, v11
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[18:19], -1, v12
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[20:21], -1, v13
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[22:23], -1, v14
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[24:25], -1, v15
+; GFX9-NEXT:    v_cndmask_b32_e64 v16, 0, v8, s[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v18, 0, v9, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v20, 0, v10, s[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, v11, s[16:17]
+; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, v12, s[18:19]
+; GFX9-NEXT:    v_cndmask_b32_e64 v26, 0, v13, s[20:21]
+; GFX9-NEXT:    v_cndmask_b32_e64 v28, 0, v14, s[22:23]
+; GFX9-NEXT:    v_cndmask_b32_e64 v30, 0, v15, s[24:25]
+; GFX9-NEXT:    v_cndmask_b32_e32 v9, 0, v31, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v11, 0, v31, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v13, 0, v31, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v15, 0, v31, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v17, 0, v31, s[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v19, 0, v31, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v21, 0, v31, s[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, v31, s[16:17]
+; GFX9-NEXT:    v_cndmask_b32_e64 v25, 0, v31, s[18:19]
+; GFX9-NEXT:    v_cndmask_b32_e64 v27, 0, v31, s[20:21]
+; GFX9-NEXT:    v_cndmask_b32_e64 v29, 0, v31, s[22:23]
+; GFX9-NEXT:    v_cndmask_b32_e64 v31, 0, v31, s[24:25]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v48
+; GFX9-NEXT:    v_mov_b32_e32 v2, v35
+; GFX9-NEXT:    v_mov_b32_e32 v3, v33
+; GFX9-NEXT:    v_mov_b32_e32 v4, v36
+; GFX9-NEXT:    v_mov_b32_e32 v5, v49
+; GFX9-NEXT:    v_mov_b32_e32 v6, v37
+; GFX9-NEXT:    v_mov_b32_e32 v7, v34
+; GFX9-NEXT:    v_mov_b32_e32 v8, v38
+; GFX9-NEXT:    v_mov_b32_e32 v10, v50
+; GFX9-NEXT:    v_mov_b32_e32 v12, v39
+; GFX9-NEXT:    v_mov_b32_e32 v14, v32
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <16 x ptr addrspace(3)> %ptr to <16 x ptr>
+  ret <16 x ptr> %cast
+}
+
+define <2 x ptr addrspace(1)> @addrspacecast_v2p0_to_v2p1(<2 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v2p0_to_v2p1:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <2 x ptr> %ptr to <2 x ptr addrspace(1)>
+  ret <2 x ptr addrspace(1)> %cast
+}
+
+define <3 x ptr addrspace(1)> @addrspacecast_v3p0_to_v3p1(<3 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v3p0_to_v3p1:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <3 x ptr> %ptr to <3 x ptr addrspace(1)>
+  ret <3 x ptr addrspace(1)> %cast
+}
+
+define <4 x ptr addrspace(1)> @addrspacecast_v4p0_to_v4p1(<4 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v4p0_to_v4p1:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <4 x ptr> %ptr to <4 x ptr addrspace(1)>
+  ret <4 x ptr addrspace(1)> %cast
+}
+
+define <8 x ptr addrspace(1)> @addrspacecast_v8p0_to_v8p1(<8 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v8p0_to_v8p1:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <8 x ptr> %ptr to <8 x ptr addrspace(1)>
+  ret <8 x ptr addrspace(1)> %cast
+}
+
+define <16 x ptr addrspace(1)> @addrspacecast_v16p0_to_v16p1(<16 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v16p0_to_v16p1:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; HSA-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <16 x ptr> %ptr to <16 x ptr addrspace(1)>
+  ret <16 x ptr addrspace(1)> %cast
+}
+
+define <2 x ptr> @addrspacecast_v2p1_to_v2p0(<2 x ptr addrspace(1)> %ptr) {
+; HSA-LABEL: addrspacecast_v2p1_to_v2p0:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <2 x ptr addrspace(1)> %ptr to <2 x ptr>
+  ret <2 x ptr> %cast
+}
+
+define <1 x ptr> @addrspacecast_v1p1_to_v1p0(<1 x ptr addrspace(1)> %ptr) {
+; HSA-LABEL: addrspacecast_v1p1_to_v1p0:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <1 x ptr addrspace(1)> %ptr to <1 x ptr>
+  ret <1 x ptr> %cast
+}
+
+define <4 x ptr> @addrspacecast_v4p1_to_v4p0(<4 x ptr addrspace(1)> %ptr) {
+; HSA-LABEL: addrspacecast_v4p1_to_v4p0:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <4 x ptr addrspace(1)> %ptr to <4 x ptr>
+  ret <4 x ptr> %cast
+}
+
+define <8 x ptr> @addrspacecast_v8p1_to_v8p0(<8 x ptr addrspace(1)> %ptr) {
+; HSA-LABEL: addrspacecast_v8p1_to_v8p0:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <8 x ptr addrspace(1)> %ptr to <8 x ptr>
+  ret <8 x ptr> %cast
+}
+
+define <16 x ptr> @addrspacecast_v16p1_to_v16p0(<16 x ptr addrspace(1)> %ptr) {
+; HSA-LABEL: addrspacecast_v16p1_to_v16p0:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; HSA-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <16 x ptr addrspace(1)> %ptr to <16 x ptr>
+  ret <16 x ptr> %cast
+}
+
+define <2 x ptr addrspace(6)> @addrspacecast_v2p0_to_v2p6(<2 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v2p0_to_v2p6:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_mov_b32_e32 v1, v2
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <2 x ptr> %ptr to <2 x ptr addrspace(6)>
+  ret <2 x ptr addrspace(6)> %cast
+}
+
+define <3 x ptr addrspace(6)> @addrspacecast_v3p0_to_v3p6(<3 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v3p0_to_v3p6:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_mov_b32_e32 v1, v2
+; HSA-NEXT:    v_mov_b32_e32 v2, v4
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <3 x ptr> %ptr to <3 x ptr addrspace(6)>
+  ret <3 x ptr addrspace(6)> %cast
+}
+
+define <4 x ptr addrspace(6)> @addrspacecast_v4p0_to_v4p6(<4 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v4p0_to_v4p6:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_mov_b32_e32 v3, v6
+; HSA-NEXT:    v_mov_b32_e32 v1, v2
+; HSA-NEXT:    v_mov_b32_e32 v2, v4
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <4 x ptr> %ptr to <4 x ptr addrspace(6)>
+  ret <4 x ptr addrspace(6)> %cast
+}
+
+define <8 x ptr addrspace(6)> @addrspacecast_v8p0_to_v8p6(<8 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v8p0_to_v8p6:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_mov_b32_e32 v7, v14
+; HSA-NEXT:    v_mov_b32_e32 v5, v10
+; HSA-NEXT:    v_mov_b32_e32 v3, v6
+; HSA-NEXT:    v_mov_b32_e32 v1, v2
+; HSA-NEXT:    v_mov_b32_e32 v2, v4
+; HSA-NEXT:    v_mov_b32_e32 v4, v8
+; HSA-NEXT:    v_mov_b32_e32 v6, v12
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <8 x ptr> %ptr to <8 x ptr addrspace(6)>
+  ret <8 x ptr addrspace(6)> %cast
+}
+
+define <16 x ptr addrspace(6)> @addrspacecast_v16p0_to_v16p6(<16 x ptr> %ptr) {
+; HSA-LABEL: addrspacecast_v16p0_to_v16p6:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_mov_b32_e32 v15, v30
+; HSA-NEXT:    v_mov_b32_e32 v13, v26
+; HSA-NEXT:    v_mov_b32_e32 v11, v22
+; HSA-NEXT:    v_mov_b32_e32 v9, v18
+; HSA-NEXT:    v_mov_b32_e32 v7, v14
+; HSA-NEXT:    v_mov_b32_e32 v5, v10
+; HSA-NEXT:    v_mov_b32_e32 v3, v6
+; HSA-NEXT:    v_mov_b32_e32 v1, v2
+; HSA-NEXT:    v_mov_b32_e32 v2, v4
+; HSA-NEXT:    v_mov_b32_e32 v4, v8
+; HSA-NEXT:    v_mov_b32_e32 v6, v12
+; HSA-NEXT:    v_mov_b32_e32 v8, v16
+; HSA-NEXT:    v_mov_b32_e32 v10, v20
+; HSA-NEXT:    v_mov_b32_e32 v12, v24
+; HSA-NEXT:    v_mov_b32_e32 v14, v28
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <16 x ptr> %ptr to <16 x ptr addrspace(6)>
+  ret <16 x ptr addrspace(6)> %cast
+}
+
+define <2 x ptr> @addrspacecast_v2p6_to_v2p0(<2 x ptr addrspace(6)> %ptr) {
+; HSA-LABEL: addrspacecast_v2p6_to_v2p0:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_mov_b32_e32 v2, v1
+; HSA-NEXT:    v_mov_b32_e32 v1, 0
+; HSA-NEXT:    v_mov_b32_e32 v3, 0
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <2 x ptr addrspace(6)> %ptr to <2 x ptr>
+  ret <2 x ptr> %cast
+}
+
+define <1 x ptr> @addrspacecast_v1p6_to_v1p0(<1 x ptr addrspace(6)> %ptr) {
+; HSA-LABEL: addrspacecast_v1p6_to_v1p0:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_mov_b32_e32 v1, 0
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <1 x ptr addrspace(6)> %ptr to <1 x ptr>
+  ret <1 x ptr> %cast
+}
+
+define <4 x ptr> @addrspacecast_v4p6_to_v4p0(<4 x ptr addrspace(6)> %ptr) {
+; HSA-LABEL: addrspacecast_v4p6_to_v4p0:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_mov_b32_e32 v6, v3
+; HSA-NEXT:    v_mov_b32_e32 v4, v2
+; HSA-NEXT:    v_mov_b32_e32 v2, v1
+; HSA-NEXT:    v_mov_b32_e32 v1, 0
+; HSA-NEXT:    v_mov_b32_e32 v3, 0
+; HSA-NEXT:    v_mov_b32_e32 v5, 0
+; HSA-NEXT:    v_mov_b32_e32 v7, 0
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <4 x ptr addrspace(6)> %ptr to <4 x ptr>
+  ret <4 x ptr> %cast
+}
+
+define <8 x ptr> @addrspacecast_v8p6_to_v8p0(<8 x ptr addrspace(6)> %ptr) {
+; HSA-LABEL: addrspacecast_v8p6_to_v8p0:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_mov_b32_e32 v14, v7
+; HSA-NEXT:    v_mov_b32_e32 v12, v6
+; HSA-NEXT:    v_mov_b32_e32 v10, v5
+; HSA-NEXT:    v_mov_b32_e32 v8, v4
+; HSA-NEXT:    v_mov_b32_e32 v6, v3
+; HSA-NEXT:    v_mov_b32_e32 v4, v2
+; HSA-NEXT:    v_mov_b32_e32 v2, v1
+; HSA-NEXT:    v_mov_b32_e32 v1, 0
+; HSA-NEXT:    v_mov_b32_e32 v3, 0
+; HSA-NEXT:    v_mov_b32_e32 v5, 0
+; HSA-NEXT:    v_mov_b32_e32 v7, 0
+; HSA-NEXT:    v_mov_b32_e32 v9, 0
+; HSA-NEXT:    v_mov_b32_e32 v11, 0
+; HSA-NEXT:    v_mov_b32_e32 v13, 0
+; HSA-NEXT:    v_mov_b32_e32 v15, 0
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <8 x ptr addrspace(6)> %ptr to <8 x ptr>
+  ret <8 x ptr> %cast
+}
+
+define <16 x ptr> @addrspacecast_v16p6_to_v16p0(<16 x ptr addrspace(6)> %ptr) {
+; HSA-LABEL: addrspacecast_v16p6_to_v16p0:
+; HSA:       ; %bb.0:
+; HSA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; HSA-NEXT:    v_mov_b32_e32 v30, v15
+; HSA-NEXT:    v_mov_b32_e32 v28, v14
+; HSA-NEXT:    v_mov_b32_e32 v26, v13
+; HSA-NEXT:    v_mov_b32_e32 v24, v12
+; HSA-NEXT:    v_mov_b32_e32 v22, v11
+; HSA-NEXT:    v_mov_b32_e32 v20, v10
+; HSA-NEXT:    v_mov_b32_e32 v18, v9
+; HSA-NEXT:    v_mov_b32_e32 v16, v8
+; HSA-NEXT:    v_mov_b32_e32 v14, v7
+; HSA-NEXT:    v_mov_b32_e32 v12, v6
+; HSA-NEXT:    v_mov_b32_e32 v10, v5
+; HSA-NEXT:    v_mov_b32_e32 v8, v4
+; HSA-NEXT:    v_mov_b32_e32 v6, v3
+; HSA-NEXT:    v_mov_b32_e32 v4, v2
+; HSA-NEXT:    v_mov_b32_e32 v2, v1
+; HSA-NEXT:    v_mov_b32_e32 v1, 0
+; HSA-NEXT:    v_mov_b32_e32 v3, 0
+; HSA-NEXT:    v_mov_b32_e32 v5, 0
+; HSA-NEXT:    v_mov_b32_e32 v7, 0
+; HSA-NEXT:    v_mov_b32_e32 v9, 0
+; HSA-NEXT:    v_mov_b32_e32 v11, 0
+; HSA-NEXT:    v_mov_b32_e32 v13, 0
+; HSA-NEXT:    v_mov_b32_e32 v15, 0
+; HSA-NEXT:    v_mov_b32_e32 v17, 0
+; HSA-NEXT:    v_mov_b32_e32 v19, 0
+; HSA-NEXT:    v_mov_b32_e32 v21, 0
+; HSA-NEXT:    v_mov_b32_e32 v23, 0
+; HSA-NEXT:    v_mov_b32_e32 v25, 0
+; HSA-NEXT:    v_mov_b32_e32 v27, 0
+; HSA-NEXT:    v_mov_b32_e32 v29, 0
+; HSA-NEXT:    v_mov_b32_e32 v31, 0
+; HSA-NEXT:    s_setpc_b64 s[30:31]
+  %cast = addrspacecast <16 x ptr addrspace(6)> %ptr to <16 x ptr>
+  ret <16 x ptr> %cast
+}
+
 declare void @llvm.amdgcn.s.barrier() #1
 declare i32 @llvm.amdgcn.workitem.id.x() #2
 
-- 
GitLab


From 183b38eb2261164fdfd6b7deac002edf27a39fe7 Mon Sep 17 00:00:00 2001
From: Fraser Cormack <fraser@codeplay.com>
Date: Mon, 23 Sep 2024 12:54:02 +0100
Subject: [PATCH 428/693] [libclc] Split off library build system into helpers

This splits off several key parts of the build system into utility
methods. This will be used in upcoming patches to help provide
additional sets of target-specific builtin libraries.

Running llvm-diff on the resulting LLVM bytecode binaries, and regular
diff on SPIR-V binaries, shows no differences before and after this
patch.
---
 libclc/CMakeLists.txt                | 172 ++++------------------
 libclc/cmake/modules/AddLibclc.cmake | 205 +++++++++++++++++++++++++++
 2 files changed, 230 insertions(+), 147 deletions(-)

diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt
index 3d7c3591a556..ba04c0bc8618 100644
--- a/libclc/CMakeLists.txt
+++ b/libclc/CMakeLists.txt
@@ -278,49 +278,22 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
     set( DARCH ${ARCH} )
   endif()
 
-  # Enumerate SOURCES* files
-  set( source_list )
-  foreach( l ${dirs} ${DARCH} ${DARCH}-${OS} ${DARCH}-${VENDOR}-${OS} )
-    foreach( s "SOURCES" "SOURCES_${LLVM_MAJOR}.${LLVM_MINOR}" )
-      file( TO_CMAKE_PATH ${l}/lib/${s} file_loc )
-      file( TO_CMAKE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${file_loc} loc )
-      # Prepend the location to give higher priority to
-      # specialized implementation
-      if( EXISTS ${loc} )
-        set( source_list ${file_loc} ${source_list} )
-      endif()
-    endforeach()
-  endforeach()
-
-  # Add the generated convert.cl here to prevent adding the one listed in
-  # SOURCES
-  set( objects )   # A "set" of already-added input files
-  set( rel_files ) # Source directory input files, relative to the root dir
-  set( gen_files ) # Generated binary input files, relative to the binary dir
-  if( NOT ${ARCH} STREQUAL "spirv" AND NOT ${ARCH} STREQUAL "spirv64" )
-    if( NOT ENABLE_RUNTIME_SUBNORMAL AND NOT ${ARCH} STREQUAL "clspv" AND
-        NOT ${ARCH} STREQUAL "clspv64" )
-      list( APPEND gen_files convert.cl )
-      list( APPEND objects convert.cl )
-      list( APPEND rel_files generic/lib/subnormal_use_default.ll )
-    elseif(${ARCH} STREQUAL "clspv" OR ${ARCH} STREQUAL "clspv64")
-      list( APPEND gen_files clspv-convert.cl )
-      list( APPEND objects clspv-convert.cl )
+  set( opencl_lib_files )
+  set( opencl_gen_files )
+
+  if( NOT ARCH STREQUAL spirv AND NOT ARCH STREQUAL spirv64 )
+    if( ARCH STREQUAL clspv OR ARCH STREQUAL clspv64 )
+      list( APPEND opencl_gen_files clspv-convert.cl )
+    elseif ( NOT ENABLE_RUNTIME_SUBNORMAL )
+      list( APPEND opencl_gen_files convert.cl )
+      list( APPEND opencl_lib_files generic/lib/subnormal_use_default.ll )
     endif()
   endif()
 
-  foreach( l ${source_list} )
-    file( READ ${l} file_list )
-    string( REPLACE "\n" ";" file_list ${file_list} )
-    get_filename_component( dir ${l} DIRECTORY )
-    foreach( f ${file_list} )
-      # Only add each file once, so that targets can 'specialize' builtins
-      if( NOT ${f} IN_LIST objects )
-        list( APPEND objects ${f} )
-        list( APPEND rel_files ${dir}/${f} )
-      endif()
-    endforeach()
-  endforeach()
+  libclc_configure_lib_source(
+    opencl_lib_files
+    DIRS ${dirs} ${DARCH} ${DARCH}-${OS} ${DARCH}-${VENDOR}-${OS}
+  )
 
   foreach( d ${${t}_devices} )
     get_libclc_device_info(
@@ -331,11 +304,6 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
       CLANG_TRIPLE clang_triple
     )
 
-    set( mcpu )
-    if( NOT "${cpu}" STREQUAL "" )
-      set( mcpu "-mcpu=${cpu}" )
-    endif()
-
     message( STATUS "  device: ${d} ( ${${d}_aliases} )" )
 
     if ( ARCH STREQUAL spirv OR ARCH STREQUAL spirv64 )
@@ -363,109 +331,19 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
       -Wno-bitwise-conditional-parentheses
     )
 
-    set( bytecode_files "" )
-    foreach( file IN LISTS gen_files rel_files )
-      # We need to take each file and produce an absolute input file, as well
-      # as a unique architecture-specific output file. We deal with a mix of
-      # different input files, which makes this trickier.
-      if( ${file} IN_LIST gen_files )
-        # Generated files are given just as file names, which we must make
-        # absolute to the binary directory.
-        set( input_file ${CMAKE_CURRENT_BINARY_DIR}/${file} )
-        set( output_file "${LIBCLC_ARCH_OBJFILE_DIR}/${file}.bc" )
-      else()
-        # Other files are originally relative to each SOURCE file, which are
-        # then make relative to the libclc root directory. We must normalize
-        # the path (e.g., ironing out any ".."), then make it relative to the
-        # root directory again, and use that relative path component for the
-        # binary path.
-        get_filename_component( abs_path ${file} ABSOLUTE BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR} )
-        file( RELATIVE_PATH root_rel_path ${CMAKE_CURRENT_SOURCE_DIR} ${abs_path} )
-        set( input_file ${CMAKE_CURRENT_SOURCE_DIR}/${file} )
-        set( output_file "${LIBCLC_ARCH_OBJFILE_DIR}/${root_rel_path}.bc" )
-      endif()
-
-      get_filename_component( file_dir ${file} DIRECTORY )
-
-      compile_to_bc(
-        TRIPLE ${clang_triple}
-        INPUT ${input_file}
-        OUTPUT ${output_file}
-        EXTRA_OPTS "${mcpu}" -fno-builtin -nostdlib
-                   "${build_flags}" -I${CMAKE_CURRENT_SOURCE_DIR}/${file_dir}
-        DEPENDENCIES generate_convert.cl clspv-generate_convert.cl
-      )
-      list( APPEND bytecode_files ${output_file} )
-    endforeach()
-
-    set( builtins_comp_lib_tgt builtins.comp.${arch_suffix} )
-    add_custom_target( ${builtins_comp_lib_tgt}
-      DEPENDS ${bytecode_files}
-    )
-    set_target_properties( ${builtins_comp_lib_tgt} PROPERTIES FOLDER "libclc/Device IR/Comp" )
+    if( NOT "${cpu}" STREQUAL "" )
+      list( APPEND build_flags -mcpu=${cpu} )
+    endif()
 
-    set( builtins_link_lib_tgt builtins.link.${arch_suffix} )
-    link_bc(
-      TARGET ${builtins_link_lib_tgt}
-      INPUTS ${bytecode_files}
-      DEPENDENCIES ${builtins_comp_lib_tgt}
+    add_libclc_builtin_set(
+      ARCH ${ARCH}
+      ARCH_SUFFIX ${arch_suffix}
+      TRIPLE ${clang_triple}
+      COMPILE_FLAGS ${build_flags}
+      OPT_FLAGS ${opt_flags}
+      LIB_FILES ${opencl_lib_files}
+      GEN_FILES ${opencl_gen_files}
+      ALIASES ${${d}_aliases}
     )
-
-    set( builtins_link_lib $<TARGET_PROPERTY:${builtins_link_lib_tgt},TARGET_FILE> )
-
-    if( ARCH STREQUAL spirv OR ARCH STREQUAL spirv64 )
-      set( spv_suffix ${arch_suffix}.spv )
-      add_custom_command( OUTPUT ${spv_suffix}
-        COMMAND ${llvm-spirv_exe} ${spvflags} -o ${spv_suffix} ${builtins_link_lib}
-        DEPENDS ${llvm-spirv_target} ${builtins_link_lib} ${builtins_link_lib_tgt}
-      )
-      add_custom_target( "prepare-${spv_suffix}" ALL DEPENDS "${spv_suffix}" )
-      set_target_properties( "prepare-${spv_suffix}" PROPERTIES FOLDER "libclc/Device IR/Prepare" )
-      install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${spv_suffix}
-         DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
-    else()
-      set( builtins_opt_lib_tgt builtins.opt.${arch_suffix} )
-
-      # Add opt target
-      add_custom_command( OUTPUT ${builtins_opt_lib_tgt}.bc
-        COMMAND ${opt_exe} ${opt_flags} -o ${builtins_opt_lib_tgt}.bc
-          ${builtins_link_lib}
-        DEPENDS ${opt_target} ${builtins_link_lib} ${builtins_link_lib_tgt}
-      )
-      add_custom_target( ${builtins_opt_lib_tgt}
-        ALL DEPENDS ${builtins_opt_lib_tgt}.bc
-      )
-      set_target_properties( ${builtins_opt_lib_tgt} PROPERTIES
-        TARGET_FILE ${CMAKE_CURRENT_BINARY_DIR}/${builtins_opt_lib_tgt}.bc
-        FOLDER "libclc/Device IR/Opt"
-      )
-
-      set( builtins_opt_lib $<TARGET_PROPERTY:${builtins_opt_lib_tgt},TARGET_FILE> )
-
-      # Add prepare target
-      set( obj_suffix ${arch_suffix}.bc )
-      add_custom_command( OUTPUT ${obj_suffix}
-        COMMAND ${prepare_builtins_exe} -o ${obj_suffix} ${builtins_opt_lib}
-        DEPENDS ${builtins_opt_lib} ${builtins_opt_lib_tgt} ${prepare_builtins_target} )
-      add_custom_target( prepare-${obj_suffix} ALL DEPENDS ${obj_suffix} )
-      set_target_properties( "prepare-${obj_suffix}" PROPERTIES FOLDER "libclc/Device IR/Prepare" )
-
-      # nvptx-- targets don't include workitem builtins
-      if( NOT clang_triple MATCHES ".*ptx.*--$" )
-        add_test( NAME external-calls-${obj_suffix}
-          COMMAND ./check_external_calls.sh ${CMAKE_CURRENT_BINARY_DIR}/${obj_suffix} ${LLVM_TOOLS_BINARY_DIR}
-          WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} )
-      endif()
-
-      install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${obj_suffix} DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
-      foreach( a ${${d}_aliases} )
-        set( alias_suffix "${a}-${clang_triple}.bc" )
-        add_custom_target( ${alias_suffix} ALL
-          COMMAND ${CMAKE_COMMAND} -E create_symlink ${obj_suffix} ${alias_suffix}
-          DEPENDS prepare-${obj_suffix} )
-        set_target_properties( "${alias_suffix}" PROPERTIES FOLDER "libclc/Device IR/Aliases" )
-        install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${alias_suffix} DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
-      endforeach( a )
-    endif()
   endforeach( d )
 endforeach( t )
diff --git a/libclc/cmake/modules/AddLibclc.cmake b/libclc/cmake/modules/AddLibclc.cmake
index f2032660ba99..147f06bc9a9a 100644
--- a/libclc/cmake/modules/AddLibclc.cmake
+++ b/libclc/cmake/modules/AddLibclc.cmake
@@ -178,3 +178,208 @@ function(get_libclc_device_info)
     set( ${ARG_CLANG_TRIPLE} ${ARG_TRIPLE} PARENT_SCOPE )
   endif()
 endfunction()
+
+# Compiles a list of library source files (provided by LIB_FILES/GEN_FILES) and
+# compiles them to LLVM bytecode (or SPIR-V), links them together and optimizes
+# them.
+#
+# For bytecode libraries, a list of ALIASES may optionally be provided to
+# produce additional symlinks.
+#
+# Arguments:
+#  * ARCH <string>
+#      libclc architecture being built
+#  * ARCH_SUFFIX <string>
+#      libclc architecture/triple suffix
+#  * TRIPLE <string>
+#      Triple used to compile
+#
+# Optional Arguments:
+#  * LIB_FILES <string> ...
+#      List of files that should be built for this library
+#  * GEN_FILES <string> ...
+#      List of generated files (in build dir) that should be built for this library
+#  * COMPILE_FLAGS <string> ...
+#      Compilation options (for clang)
+#  * OPT_FLAGS <string> ...
+#      Optimization options (for opt)
+#  * ALIASES <string> ...
+#      List of aliases
+function(add_libclc_builtin_set)
+  cmake_parse_arguments(ARG
+    ""
+    "ARCH;TRIPLE;ARCH_SUFFIX"
+    "LIB_FILES;GEN_FILES;COMPILE_FLAGS;OPT_FLAGS;ALIASES"
+    ${ARGN}
+  )
+
+  if( NOT ARG_ARCH OR NOT ARG_ARCH_SUFFIX OR NOT ARG_TRIPLE )
+    message( FATAL_ERROR "Must provide ARCH, ARCH_SUFFIX, and TRIPLE" )
+  endif()
+
+  set( bytecode_files "" )
+  foreach( file IN LISTS ARG_GEN_FILES ARG_LIB_FILES )
+    # We need to take each file and produce an absolute input file, as well
+    # as a unique architecture-specific output file. We deal with a mix of
+    # different input files, which makes this trickier.
+    if( ${file} IN_LIST ARG_GEN_FILES )
+      # Generated files are given just as file names, which we must make
+      # absolute to the binary directory.
+      set( input_file ${CMAKE_CURRENT_BINARY_DIR}/${file} )
+      set( output_file "${LIBCLC_ARCH_OBJFILE_DIR}/${file}.bc" )
+    else()
+      # Other files are originally relative to each SOURCE file, which are
+      # then make relative to the libclc root directory. We must normalize
+      # the path (e.g., ironing out any ".."), then make it relative to the
+      # root directory again, and use that relative path component for the
+      # binary path.
+      get_filename_component( abs_path ${file} ABSOLUTE BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR} )
+      file( RELATIVE_PATH root_rel_path ${CMAKE_CURRENT_SOURCE_DIR} ${abs_path} )
+      set( input_file ${CMAKE_CURRENT_SOURCE_DIR}/${file} )
+      set( output_file "${LIBCLC_ARCH_OBJFILE_DIR}/${root_rel_path}.bc" )
+    endif()
+
+    get_filename_component( file_dir ${file} DIRECTORY )
+
+    compile_to_bc(
+      TRIPLE ${ARG_TRIPLE}
+      INPUT ${input_file}
+      OUTPUT ${output_file}
+      EXTRA_OPTS -fno-builtin -nostdlib
+        "${ARG_COMPILE_FLAGS}" -I${CMAKE_CURRENT_SOURCE_DIR}/${file_dir}
+      DEPENDENCIES generate_convert.cl clspv-generate_convert.cl
+    )
+    list( APPEND bytecode_files ${output_file} )
+  endforeach()
+
+  set( builtins_comp_lib_tgt builtins.comp.${ARG_ARCH_SUFFIX} )
+  add_custom_target( ${builtins_comp_lib_tgt}
+    DEPENDS ${bytecode_files}
+  )
+  set_target_properties( ${builtins_comp_lib_tgt} PROPERTIES FOLDER "libclc/Device IR/Comp" )
+
+  set( builtins_link_lib_tgt builtins.link.${ARG_ARCH_SUFFIX} )
+  link_bc(
+    TARGET ${builtins_link_lib_tgt}
+    INPUTS ${bytecode_files}
+    DEPENDENCIES ${builtins_comp_lib_tgt}
+  )
+
+  set( builtins_link_lib $<TARGET_PROPERTY:${builtins_link_lib_tgt},TARGET_FILE> )
+
+  if( ARG_ARCH STREQUAL spirv OR ARG_ARCH STREQUAL spirv64 )
+    set( spv_suffix ${ARG_ARCH_SUFFIX}.spv )
+    add_custom_command( OUTPUT ${spv_suffix}
+      COMMAND ${llvm-spirv_exe} ${spvflags} -o ${spv_suffix} ${builtins_link_lib}
+      DEPENDS ${llvm-spirv_target} ${builtins_link_lib} ${builtins_link_lib_tgt}
+    )
+    add_custom_target( "prepare-${spv_suffix}" ALL DEPENDS "${spv_suffix}" )
+    set_target_properties( "prepare-${spv_suffix}" PROPERTIES FOLDER "libclc/Device IR/Prepare" )
+    install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${spv_suffix}
+       DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
+
+    return()
+  endif()
+
+  set( builtins_opt_lib_tgt builtins.opt.${ARG_ARCH_SUFFIX} )
+
+  # Add opt target
+  add_custom_command( OUTPUT ${builtins_opt_lib_tgt}.bc
+    COMMAND ${opt_exe} ${ARG_OPT_FLAGS} -o ${builtins_opt_lib_tgt}.bc
+      ${builtins_link_lib}
+    DEPENDS ${opt_target} ${builtins_link_lib} ${builtins_link_lib_tgt}
+  )
+  add_custom_target( ${builtins_opt_lib_tgt}
+    ALL DEPENDS ${builtins_opt_lib_tgt}.bc
+  )
+  set_target_properties( ${builtins_opt_lib_tgt} PROPERTIES
+    TARGET_FILE ${CMAKE_CURRENT_BINARY_DIR}/${builtins_opt_lib_tgt}.bc
+    FOLDER "libclc/Device IR/Opt"
+  )
+
+  set( builtins_opt_lib $<TARGET_PROPERTY:${builtins_opt_lib_tgt},TARGET_FILE> )
+
+  # Add prepare target
+  set( obj_suffix ${ARG_ARCH_SUFFIX}.bc )
+  add_custom_command( OUTPUT ${obj_suffix}
+    COMMAND ${prepare_builtins_exe} -o ${obj_suffix} ${builtins_opt_lib}
+    DEPENDS ${builtins_opt_lib} ${builtins_opt_lib_tgt} ${prepare_builtins_target} )
+  add_custom_target( prepare-${obj_suffix} ALL DEPENDS ${obj_suffix} )
+  set_target_properties( "prepare-${obj_suffix}" PROPERTIES FOLDER "libclc/Device IR/Prepare" )
+
+  # nvptx-- targets don't include workitem builtins
+  if( NOT ARG_TRIPLE MATCHES ".*ptx.*--$" )
+    add_test( NAME external-calls-${obj_suffix}
+      COMMAND ./check_external_calls.sh ${CMAKE_CURRENT_BINARY_DIR}/${obj_suffix} ${LLVM_TOOLS_BINARY_DIR}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} )
+  endif()
+
+  install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${obj_suffix} DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
+  foreach( a ${ARG_ALIASES} )
+    set( alias_suffix "${a}-${ARG_TRIPLE}.bc" )
+    add_custom_target( ${alias_suffix} ALL
+      COMMAND ${CMAKE_COMMAND} -E create_symlink ${obj_suffix} ${alias_suffix}
+      DEPENDS prepare-${obj_suffix} )
+    set_target_properties( "${alias_suffix}" PROPERTIES FOLDER "libclc/Device IR/Aliases" )
+    install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${alias_suffix} DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
+  endforeach( a )
+endfunction(add_libclc_builtin_set)
+
+# Produces a list of libclc source files by walking over SOURCES files in a
+# given directory. Outputs the list of files in LIB_FILE_LIST.
+#
+# LIB_FILE_LIST may be pre-populated and is appended to.
+#
+# Arguments:
+# * LIB_ROOT_DIR <string>
+#     Root directory containing target's lib files, relative to libclc root
+#     directory. If not provided, is set to '.'.
+# * DIRS <string> ...
+#     List of directories under LIB_ROOT_DIR to walk over searching for SOURCES
+#     files
+function(libclc_configure_lib_source LIB_FILE_LIST)
+  cmake_parse_arguments(ARG
+    ""
+    "LIB_ROOT_DIR"
+    "DIRS"
+    ${ARGN}
+  )
+
+  if( NOT ARG_LIB_ROOT_DIR )
+    set(ARG_LIB_ROOT_DIR  ".")
+  endif()
+
+  # Enumerate SOURCES* files
+  set( source_list )
+  foreach( l ${ARG_DIRS} )
+    foreach( s "SOURCES" "SOURCES_${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}" )
+      file( TO_CMAKE_PATH ${ARG_LIB_ROOT_DIR}/${l}/lib/${s} file_loc )
+      file( TO_CMAKE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${file_loc} loc )
+      # Prepend the location to give higher priority to
+      # specialized implementation
+      if( EXISTS ${loc} )
+        set( source_list ${file_loc} ${source_list} )
+      endif()
+    endforeach()
+  endforeach()
+
+  ## Add the generated convert files here to prevent adding the ones listed in
+  ## SOURCES
+  set( rel_files ${${LIB_FILE_LIST}} ) # Source directory input files, relative to the root dir
+  set( objects ${${LIB_FILE_LIST}} )   # A "set" of already-added input files
+
+  foreach( l ${source_list} )
+    file( READ ${l} file_list )
+    string( REPLACE "\n" ";" file_list ${file_list} )
+    get_filename_component( dir ${l} DIRECTORY )
+    foreach( f ${file_list} )
+      # Only add each file once, so that targets can 'specialize' builtins
+      if( NOT ${f} IN_LIST objects )
+        list( APPEND objects ${f} )
+        list( APPEND rel_files ${dir}/${f} )
+      endif()
+    endforeach()
+  endforeach()
+
+  set( ${LIB_FILE_LIST} ${rel_files} PARENT_SCOPE )
+endfunction(libclc_configure_lib_source LIB_FILE_LIST)
-- 
GitLab


From b2bdd8bd39e90bfe3c66f6d5600468570a77ede6 Mon Sep 17 00:00:00 2001
From: Fraser Cormack <fraser@codeplay.com>
Date: Tue, 24 Sep 2024 17:34:50 +0100
Subject: [PATCH 429/693] [libclc] Create an internal 'clc' builtins library

Some libclc builtins currently use internal builtins prefixed with
'__clc_' for various reasons, e.g., to avoid naming clashes.

This commit formalizes this concept by starting to isolate the
definitions of these internal clc builtins into a separate
self-contained bytecode library, which is linked into each target's
libclc OpenCL builtins before optimization takes place.

The goal of this step is to allow additional libraries of builtins
that provide entry points (or bindings) that are not written in OpenCL C
but still wish to expose OpenCL-compatible builtins. By moving the
implementations into a separate self-contained library, entry points can
share as much code as possible without going through OpenCL C.

The overall structure of the internal clc library is similar to the
current OpenCL structure, with SOURCES files and targets being able to
override the definitions of builtins as needed. The idea is that the
OpenCL builtins will begin to need fewer target-specific overrides, as
those will slowly move over to the clc builtins instead.

Another advantage of having a separate bytecode library with the CLC
implementations is that we can internalize the symbols when linking it
(separately), whereas currently the CLC symbols make it into the final
builtins library (and perhaps even the final compiled binary).

This patch starts of with 'dot' as it's relatively self-contained, as
opposed to most of the maths builtins which tend to pull in other
builtins.

We can also start to clang-format the builtins as we go, which should
help to modernize the codebase.
---
 libclc/CMakeLists.txt                         | 35 +++++++++-
 libclc/{generic => clc}/include/clc/clcfunc.h |  5 ++
 .../{generic => clc}/include/clc/clctypes.h   | 11 ++-
 libclc/clc/include/clc/geometric/clc_dot.h    |  2 +
 libclc/clc/include/clc/geometric/clc_dot.inc  |  1 +
 libclc/clc/include/clc/internal/clc.h         | 26 +++++++
 libclc/clc/lib/clspv/SOURCES                  |  1 +
 libclc/clc/lib/clspv/dummy.cl                 |  1 +
 libclc/clc/lib/clspv64                        |  1 +
 libclc/clc/lib/generic/SOURCES                |  1 +
 libclc/clc/lib/generic/geometric/clc_dot.cl   | 57 +++++++++++++++
 libclc/clc/lib/spirv/SOURCES                  |  2 +
 libclc/clc/lib/spirv64/SOURCES                |  1 +
 libclc/cmake/modules/AddLibclc.cmake          | 69 +++++++++++++++----
 libclc/generic/lib/geometric/dot.cl           | 27 ++++----
 15 files changed, 209 insertions(+), 31 deletions(-)
 rename libclc/{generic => clc}/include/clc/clcfunc.h (85%)
 rename libclc/{generic => clc}/include/clc/clctypes.h (94%)
 create mode 100644 libclc/clc/include/clc/geometric/clc_dot.h
 create mode 100644 libclc/clc/include/clc/geometric/clc_dot.inc
 create mode 100644 libclc/clc/include/clc/internal/clc.h
 create mode 100644 libclc/clc/lib/clspv/SOURCES
 create mode 100644 libclc/clc/lib/clspv/dummy.cl
 create mode 120000 libclc/clc/lib/clspv64
 create mode 100644 libclc/clc/lib/generic/SOURCES
 create mode 100644 libclc/clc/lib/generic/geometric/clc_dot.cl
 create mode 100644 libclc/clc/lib/spirv/SOURCES
 create mode 100644 libclc/clc/lib/spirv64/SOURCES

diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt
index ba04c0bc8618..16d74e53295c 100644
--- a/libclc/CMakeLists.txt
+++ b/libclc/CMakeLists.txt
@@ -29,7 +29,13 @@ set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS
   ptx-nvidiacl/lib/SOURCES;
   r600/lib/SOURCES;
   spirv/lib/SOURCES;
-  spirv64/lib/SOURCES
+  spirv64/lib/SOURCES;
+  # CLC internal libraries
+  clc/lib/generic/SOURCES;
+  clc/lib/clspv/SOURCES;
+  clc/lib/clspv64/SOURCES;
+  clc/lib/spirv/SOURCES;
+  clc/lib/spirv64/SOURCES;
 )
 
 set( LIBCLC_MIN_LLVM 3.9.0 )
@@ -278,6 +284,14 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
     set( DARCH ${ARCH} )
   endif()
 
+  set( clc_lib_files )
+  libclc_configure_lib_source(
+    clc_lib_files
+    CLC_INTERNAL
+    LIB_ROOT_DIR clc
+    DIRS ${dirs} ${DARCH} ${DARCH}-${OS} ${DARCH}-${VENDOR}-${OS}
+  )
+
   set( opencl_lib_files )
   set( opencl_gen_files )
 
@@ -326,7 +340,8 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
     list( APPEND build_flags
       -D__CLC_INTERNAL
       -D${CLC_TARGET_DEFINE}
-      -I${CMAKE_CURRENT_SOURCE_DIR}/generic/include
+      # All libclc builtin libraries see CLC headers
+      -I${CMAKE_CURRENT_SOURCE_DIR}/clc/include
       # FIXME: Fix libclc to not require disabling this noisy warning
       -Wno-bitwise-conditional-parentheses
     )
@@ -335,6 +350,20 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
       list( APPEND build_flags -mcpu=${cpu} )
     endif()
 
+    add_libclc_builtin_set(
+      CLC_INTERNAL
+      ARCH ${ARCH}
+      ARCH_SUFFIX clc-${arch_suffix}
+      TRIPLE ${clang_triple}
+      COMPILE_FLAGS ${build_flags}
+      OPT_FLAGS ${opt_flags}
+      LIB_FILES ${clc_lib_files}
+    )
+
+    list( APPEND build_flags
+      -I${CMAKE_CURRENT_SOURCE_DIR}/generic/include
+    )
+
     add_libclc_builtin_set(
       ARCH ${ARCH}
       ARCH_SUFFIX ${arch_suffix}
@@ -344,6 +373,8 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
       LIB_FILES ${opencl_lib_files}
       GEN_FILES ${opencl_gen_files}
       ALIASES ${${d}_aliases}
+      # Link in the CLC builtins and internalize their symbols
+      INTERNAL_LINK_DEPENDENCIES $<TARGET_PROPERTY:builtins.link.clc-${arch_suffix},TARGET_FILE>
     )
   endforeach( d )
 endforeach( t )
diff --git a/libclc/generic/include/clc/clcfunc.h b/libclc/clc/include/clc/clcfunc.h
similarity index 85%
rename from libclc/generic/include/clc/clcfunc.h
rename to libclc/clc/include/clc/clcfunc.h
index 086d780b9708..fe3406f64fec 100644
--- a/libclc/generic/include/clc/clcfunc.h
+++ b/libclc/clc/include/clc/clcfunc.h
@@ -1,3 +1,6 @@
+#ifndef __CLC_CLCFUNC_H_
+#define __CLC_CLCFUNC_H_
+
 #define _CLC_OVERLOAD __attribute__((overloadable))
 #define _CLC_DECL
 #define _CLC_INLINE __attribute__((always_inline)) inline
@@ -11,3 +14,5 @@
 #else
 #define _CLC_DEF __attribute__((always_inline))
 #endif
+
+#endif // __CLC_CLCFUNC_H_
diff --git a/libclc/generic/include/clc/clctypes.h b/libclc/clc/include/clc/clctypes.h
similarity index 94%
rename from libclc/generic/include/clc/clctypes.h
rename to libclc/clc/include/clc/clctypes.h
index 76b816d395c2..8ededd967e00 100644
--- a/libclc/generic/include/clc/clctypes.h
+++ b/libclc/clc/include/clc/clctypes.h
@@ -1,3 +1,6 @@
+#ifndef __CLC_CLCTYPES_H_
+#define __CLC_CLCTYPES_H_
+
 /* 6.1.1 Built-in Scalar Data Types */
 
 typedef unsigned char uchar;
@@ -8,12 +11,12 @@ typedef unsigned long ulong;
 typedef __SIZE_TYPE__ size_t;
 typedef __PTRDIFF_TYPE__ ptrdiff_t;
 
-#define __stdint_join3(a,b,c) a ## b ## c
+#define __stdint_join3(a, b, c) a##b##c
 
-#define  __intn_t(n) __stdint_join3(__INT, n, _TYPE__)
+#define __intn_t(n) __stdint_join3(__INT, n, _TYPE__)
 #define __uintn_t(n) __stdint_join3(unsigned __INT, n, _TYPE__)
 
-typedef  __intn_t(__INTPTR_WIDTH__)  intptr_t;
+typedef __intn_t(__INTPTR_WIDTH__) intptr_t;
 typedef __uintn_t(__INTPTR_WIDTH__) uintptr_t;
 
 #undef __uintn_t
@@ -93,3 +96,5 @@ typedef __attribute__((ext_vector_type(4))) half half4;
 typedef __attribute__((ext_vector_type(8))) half half8;
 typedef __attribute__((ext_vector_type(16))) half half16;
 #endif
+
+#endif // __CLC_CLCTYPES_H_
diff --git a/libclc/clc/include/clc/geometric/clc_dot.h b/libclc/clc/include/clc/geometric/clc_dot.h
new file mode 100644
index 000000000000..e0e47ab2093e
--- /dev/null
+++ b/libclc/clc/include/clc/geometric/clc_dot.h
@@ -0,0 +1,2 @@
+#define __CLC_BODY <clc/geometric/clc_dot.inc>
+#include <clc/geometric/floatn.inc>
diff --git a/libclc/clc/include/clc/geometric/clc_dot.inc b/libclc/clc/include/clc/geometric/clc_dot.inc
new file mode 100644
index 000000000000..016b564df362
--- /dev/null
+++ b/libclc/clc/include/clc/geometric/clc_dot.inc
@@ -0,0 +1 @@
+_CLC_OVERLOAD _CLC_DECL __CLC_FLOAT __clc_dot(__CLC_FLOATN p0, __CLC_FLOATN p1);
diff --git a/libclc/clc/include/clc/internal/clc.h b/libclc/clc/include/clc/internal/clc.h
new file mode 100644
index 000000000000..c3bdfd754105
--- /dev/null
+++ b/libclc/clc/include/clc/internal/clc.h
@@ -0,0 +1,26 @@
+#ifndef __CLC_INTERNAL_CLC_H_
+#define __CLC_INTERNAL_CLC_H_
+
+#ifndef cl_clang_storage_class_specifiers
+#error Implementation requires cl_clang_storage_class_specifiers extension!
+#endif
+
+#pragma OPENCL EXTENSION cl_clang_storage_class_specifiers : enable
+
+#ifdef cl_khr_fp64
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+#endif
+
+#ifdef cl_khr_fp16
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#endif
+
+/* Function Attributes */
+#include <clc/clcfunc.h>
+
+/* 6.1 Supported Data Types */
+#include <clc/clctypes.h>
+
+#pragma OPENCL EXTENSION all : disable
+
+#endif // __CLC_INTERNAL_CLC_H_
diff --git a/libclc/clc/lib/clspv/SOURCES b/libclc/clc/lib/clspv/SOURCES
new file mode 100644
index 000000000000..75a3130357c3
--- /dev/null
+++ b/libclc/clc/lib/clspv/SOURCES
@@ -0,0 +1 @@
+dummy.cl
diff --git a/libclc/clc/lib/clspv/dummy.cl b/libclc/clc/lib/clspv/dummy.cl
new file mode 100644
index 000000000000..fab17ac780e3
--- /dev/null
+++ b/libclc/clc/lib/clspv/dummy.cl
@@ -0,0 +1 @@
+// Empty file
diff --git a/libclc/clc/lib/clspv64 b/libclc/clc/lib/clspv64
new file mode 120000
index 000000000000..ea01ba94bc63
--- /dev/null
+++ b/libclc/clc/lib/clspv64
@@ -0,0 +1 @@
+clspv
\ No newline at end of file
diff --git a/libclc/clc/lib/generic/SOURCES b/libclc/clc/lib/generic/SOURCES
new file mode 100644
index 000000000000..fa2e4f50b99c
--- /dev/null
+++ b/libclc/clc/lib/generic/SOURCES
@@ -0,0 +1 @@
+geometric/clc_dot.cl
diff --git a/libclc/clc/lib/generic/geometric/clc_dot.cl b/libclc/clc/lib/generic/geometric/clc_dot.cl
new file mode 100644
index 000000000000..bf0f19b51bc0
--- /dev/null
+++ b/libclc/clc/lib/generic/geometric/clc_dot.cl
@@ -0,0 +1,57 @@
+#include <clc/internal/clc.h>
+
+_CLC_OVERLOAD _CLC_DEF float __clc_dot(float p0, float p1) { return p0 * p1; }
+
+_CLC_OVERLOAD _CLC_DEF float __clc_dot(float2 p0, float2 p1) {
+  return p0.x * p1.x + p0.y * p1.y;
+}
+
+_CLC_OVERLOAD _CLC_DEF float __clc_dot(float3 p0, float3 p1) {
+  return p0.x * p1.x + p0.y * p1.y + p0.z * p1.z;
+}
+
+_CLC_OVERLOAD _CLC_DEF float __clc_dot(float4 p0, float4 p1) {
+  return p0.x * p1.x + p0.y * p1.y + p0.z * p1.z + p0.w * p1.w;
+}
+
+#ifdef cl_khr_fp64
+
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+
+_CLC_OVERLOAD _CLC_DEF double __clc_dot(double p0, double p1) {
+  return p0 * p1;
+}
+
+_CLC_OVERLOAD _CLC_DEF double __clc_dot(double2 p0, double2 p1) {
+  return p0.x * p1.x + p0.y * p1.y;
+}
+
+_CLC_OVERLOAD _CLC_DEF double __clc_dot(double3 p0, double3 p1) {
+  return p0.x * p1.x + p0.y * p1.y + p0.z * p1.z;
+}
+
+_CLC_OVERLOAD _CLC_DEF double __clc_dot(double4 p0, double4 p1) {
+  return p0.x * p1.x + p0.y * p1.y + p0.z * p1.z + p0.w * p1.w;
+}
+
+#endif
+
+#ifdef cl_khr_fp16
+
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+_CLC_OVERLOAD _CLC_DEF half __clc_dot(half p0, half p1) { return p0 * p1; }
+
+_CLC_OVERLOAD _CLC_DEF half __clc_dot(half2 p0, half2 p1) {
+  return p0.x * p1.x + p0.y * p1.y;
+}
+
+_CLC_OVERLOAD _CLC_DEF half __clc_dot(half3 p0, half3 p1) {
+  return p0.x * p1.x + p0.y * p1.y + p0.z * p1.z;
+}
+
+_CLC_OVERLOAD _CLC_DEF half __clc_dot(half4 p0, half4 p1) {
+  return p0.x * p1.x + p0.y * p1.y + p0.z * p1.z + p0.w * p1.w;
+}
+
+#endif
diff --git a/libclc/clc/lib/spirv/SOURCES b/libclc/clc/lib/spirv/SOURCES
new file mode 100644
index 000000000000..d8effd19613c
--- /dev/null
+++ b/libclc/clc/lib/spirv/SOURCES
@@ -0,0 +1,2 @@
+../generic/geometric/clc_dot.cl
+
diff --git a/libclc/clc/lib/spirv64/SOURCES b/libclc/clc/lib/spirv64/SOURCES
new file mode 100644
index 000000000000..9200810ace38
--- /dev/null
+++ b/libclc/clc/lib/spirv64/SOURCES
@@ -0,0 +1 @@
+../generic/geometric/clc_dot.cl
diff --git a/libclc/cmake/modules/AddLibclc.cmake b/libclc/cmake/modules/AddLibclc.cmake
index 147f06bc9a9a..ee7c8500c835 100644
--- a/libclc/cmake/modules/AddLibclc.cmake
+++ b/libclc/cmake/modules/AddLibclc.cmake
@@ -76,6 +76,8 @@ endfunction()
 # Links together one or more bytecode files
 #
 # Arguments:
+# * INTERNALIZE
+#     Set if -internalize flag should be passed when linking
 # * TARGET <string>
 #     Custom target to create
 # * INPUT <string> ...
@@ -84,7 +86,7 @@ endfunction()
 #     List of extra dependencies to inject
 function(link_bc)
   cmake_parse_arguments(ARG
-    ""
+    "INTERNALIZE"
     "TARGET"
     "INPUTS;DEPENDENCIES"
     ${ARGN}
@@ -97,7 +99,7 @@ function(link_bc)
     file( TO_CMAKE_PATH ${LIBCLC_ARCH_OBJFILE_DIR}/${ARG_TARGET}.rsp RSP_FILE )
     # Turn it into a space-separate list of input files
     list( JOIN ARG_INPUTS " " RSP_INPUT )
-    file( WRITE ${RSP_FILE} ${RSP_INPUT} )
+    file( GENERATE OUTPUT ${RSP_FILE} CONTENT ${RSP_INPUT} )
     # Ensure that if this file is removed, we re-run CMake
     set_property( DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS
       ${RSP_FILE}
@@ -107,7 +109,7 @@ function(link_bc)
 
   add_custom_command(
     OUTPUT ${ARG_TARGET}.bc
-    COMMAND ${llvm-link_exe} -o ${ARG_TARGET}.bc ${LINK_INPUT_ARG}
+    COMMAND ${llvm-link_exe} $<$<BOOL:${ARG_INTERNALIZE}>:--internalize> -o ${ARG_TARGET}.bc ${LINK_INPUT_ARG}
     DEPENDS ${llvm-link_target} ${ARG_DEPENDENCIES} ${ARG_INPUTS} ${RSP_FILE}
   )
 
@@ -195,6 +197,9 @@ endfunction()
 #      Triple used to compile
 #
 # Optional Arguments:
+# * CLC_INTERNAL
+#     Pass if compiling the internal CLC builtin libraries, which are not
+#     optimized and do not have aliases created.
 #  * LIB_FILES <string> ...
 #      List of files that should be built for this library
 #  * GEN_FILES <string> ...
@@ -205,11 +210,14 @@ endfunction()
 #      Optimization options (for opt)
 #  * ALIASES <string> ...
 #      List of aliases
+#  * INTERNAL_LINK_DEPENDENCIES <string> ...
+#      A list of extra bytecode files to link into the builtin library. Symbols
+#      from these link dependencies will be internalized during linking.
 function(add_libclc_builtin_set)
   cmake_parse_arguments(ARG
-    ""
+    "CLC_INTERNAL"
     "ARCH;TRIPLE;ARCH_SUFFIX"
-    "LIB_FILES;GEN_FILES;COMPILE_FLAGS;OPT_FLAGS;ALIASES"
+    "LIB_FILES;GEN_FILES;COMPILE_FLAGS;OPT_FLAGS;ALIASES;INTERNAL_LINK_DEPENDENCIES"
     ${ARGN}
   )
 
@@ -258,12 +266,42 @@ function(add_libclc_builtin_set)
   )
   set_target_properties( ${builtins_comp_lib_tgt} PROPERTIES FOLDER "libclc/Device IR/Comp" )
 
+  if( NOT bytecode_files )
+    message(FATAL_ERROR "Cannot create an empty builtins library")
+  endif()
+
   set( builtins_link_lib_tgt builtins.link.${ARG_ARCH_SUFFIX} )
-  link_bc(
-    TARGET ${builtins_link_lib_tgt}
-    INPUTS ${bytecode_files}
-    DEPENDENCIES ${builtins_comp_lib_tgt}
-  )
+
+  if( NOT ARG_INTERNAL_LINK_DEPENDENCIES )
+    link_bc(
+      TARGET ${builtins_link_lib_tgt}
+      INPUTS ${bytecode_files}
+      DEPENDENCIES ${builtins_comp_lib_tgt}
+    )
+  else()
+    # If we have libraries to link while internalizing their symbols, we need
+    # two separate link steps; the --internalize flag applies to all link
+    # inputs but the first.
+    set( builtins_link_lib_tmp_tgt builtins.link.pre-deps.${ARG_ARCH_SUFFIX} )
+    link_bc(
+      TARGET ${builtins_link_lib_tmp_tgt}
+      INPUTS ${bytecode_files}
+      DEPENDENCIES ${builtins_comp_lib_tgt}
+    )
+    link_bc(
+      INTERNALIZE
+      TARGET ${builtins_link_lib_tgt}
+      INPUTS $<TARGET_PROPERTY:${builtins_link_lib_tmp_tgt},TARGET_FILE>
+        ${ARG_INTERNAL_LINK_DEPENDENCIES}
+      DEPENDENCIES ${builtins_link_lib_tmp_tgt}
+    )
+  endif()
+
+  # For the CLC internal builtins, exit here - we only optimize the targets'
+  # entry points once we've linked the CLC buitins into them
+  if( ARG_CLC_INTERNAL )
+    return()
+  endif()
 
   set( builtins_link_lib $<TARGET_PROPERTY:${builtins_link_lib_tgt},TARGET_FILE> )
 
@@ -331,6 +369,9 @@ endfunction(add_libclc_builtin_set)
 # LIB_FILE_LIST may be pre-populated and is appended to.
 #
 # Arguments:
+# * CLC_INTERNAL
+#     Pass if compiling the internal CLC builtin libraries, which have a
+#     different directory structure.
 # * LIB_ROOT_DIR <string>
 #     Root directory containing target's lib files, relative to libclc root
 #     directory. If not provided, is set to '.'.
@@ -339,7 +380,7 @@ endfunction(add_libclc_builtin_set)
 #     files
 function(libclc_configure_lib_source LIB_FILE_LIST)
   cmake_parse_arguments(ARG
-    ""
+    "CLC_INTERNAL"
     "LIB_ROOT_DIR"
     "DIRS"
     ${ARGN}
@@ -353,7 +394,11 @@ function(libclc_configure_lib_source LIB_FILE_LIST)
   set( source_list )
   foreach( l ${ARG_DIRS} )
     foreach( s "SOURCES" "SOURCES_${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}" )
-      file( TO_CMAKE_PATH ${ARG_LIB_ROOT_DIR}/${l}/lib/${s} file_loc )
+      if( ARG_CLC_INTERNAL )
+        file( TO_CMAKE_PATH ${ARG_LIB_ROOT_DIR}/lib/${l}/${s} file_loc )
+      else()
+        file( TO_CMAKE_PATH ${ARG_LIB_ROOT_DIR}/${l}/lib/${s} file_loc )
+      endif()
       file( TO_CMAKE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${file_loc} loc )
       # Prepend the location to give higher priority to
       # specialized implementation
diff --git a/libclc/generic/lib/geometric/dot.cl b/libclc/generic/lib/geometric/dot.cl
index e58bc26f4333..e790d0263656 100644
--- a/libclc/generic/lib/geometric/dot.cl
+++ b/libclc/generic/lib/geometric/dot.cl
@@ -1,19 +1,20 @@
 #include <clc/clc.h>
+#include <clc/geometric/clc_dot.h>
 
 _CLC_OVERLOAD _CLC_DEF float dot(float p0, float p1) {
-  return p0*p1;
+  return __clc_dot(p0, p1);
 }
 
 _CLC_OVERLOAD _CLC_DEF float dot(float2 p0, float2 p1) {
-  return p0.x*p1.x + p0.y*p1.y;
+  return __clc_dot(p0, p1);
 }
 
 _CLC_OVERLOAD _CLC_DEF float dot(float3 p0, float3 p1) {
-  return p0.x*p1.x + p0.y*p1.y + p0.z*p1.z;
+  return __clc_dot(p0, p1);
 }
 
 _CLC_OVERLOAD _CLC_DEF float dot(float4 p0, float4 p1) {
-  return p0.x*p1.x + p0.y*p1.y + p0.z*p1.z + p0.w*p1.w;
+  return __clc_dot(p0, p1);
 }
 
 #ifdef cl_khr_fp64
@@ -21,19 +22,19 @@ _CLC_OVERLOAD _CLC_DEF float dot(float4 p0, float4 p1) {
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
 _CLC_OVERLOAD _CLC_DEF double dot(double p0, double p1) {
-  return p0*p1;
+  return __clc_dot(p0, p1);
 }
 
 _CLC_OVERLOAD _CLC_DEF double dot(double2 p0, double2 p1) {
-  return p0.x*p1.x + p0.y*p1.y;
+  return __clc_dot(p0, p1);
 }
 
 _CLC_OVERLOAD _CLC_DEF double dot(double3 p0, double3 p1) {
-  return p0.x*p1.x + p0.y*p1.y + p0.z*p1.z;
+  return __clc_dot(p0, p1);
 }
 
 _CLC_OVERLOAD _CLC_DEF double dot(double4 p0, double4 p1) {
-  return p0.x*p1.x + p0.y*p1.y + p0.z*p1.z + p0.w*p1.w;
+  return __clc_dot(p0, p1);
 }
 
 #endif
@@ -42,20 +43,18 @@ _CLC_OVERLOAD _CLC_DEF double dot(double4 p0, double4 p1) {
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
-_CLC_OVERLOAD _CLC_DEF half dot(half p0, half p1) {
-  return p0*p1;
-}
+_CLC_OVERLOAD _CLC_DEF half dot(half p0, half p1) { return __clc_dot(p0, p1); }
 
 _CLC_OVERLOAD _CLC_DEF half dot(half2 p0, half2 p1) {
-  return p0.x*p1.x + p0.y*p1.y;
+  return __clc_dot(p0, p1);
 }
 
 _CLC_OVERLOAD _CLC_DEF half dot(half3 p0, half3 p1) {
-  return p0.x*p1.x + p0.y*p1.y + p0.z*p1.z;
+  return __clc_dot(p0, p1);
 }
 
 _CLC_OVERLOAD _CLC_DEF half dot(half4 p0, half4 p1) {
-  return p0.x*p1.x + p0.y*p1.y + p0.z*p1.z + p0.w*p1.w;
+  return __clc_dot(p0, p1);
 }
 
 #endif
-- 
GitLab


From 667deb640870cbdaac941f30d189ef3fe926141c Mon Sep 17 00:00:00 2001
From: Edd Dawson <edd.dawson@sony.com>
Date: Tue, 29 Oct 2024 13:21:48 +0000
Subject: [PATCH 430/693] [PS4/PS5][Driver] Apply clang-format to PS4CPU.cpp
 (NFC) (#114038)

---
 clang/lib/Driver/ToolChains/PS4CPU.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/PS4CPU.cpp b/clang/lib/Driver/ToolChains/PS4CPU.cpp
index 719bba41436a..9daafbe703f6 100644
--- a/clang/lib/Driver/ToolChains/PS4CPU.cpp
+++ b/clang/lib/Driver/ToolChains/PS4CPU.cpp
@@ -430,8 +430,7 @@ toolchains::PS4PS5Base::PS4PS5Base(const Driver &D, const llvm::Triple &Triple,
 }
 
 void toolchains::PS4PS5Base::AddClangSystemIncludeArgs(
-    const ArgList &DriverArgs,
-    ArgStringList &CC1Args) const {
+    const ArgList &DriverArgs, ArgStringList &CC1Args) const {
   const Driver &D = getDriver();
 
   if (DriverArgs.hasArg(options::OPT_nostdinc))
-- 
GitLab


From d732c0b13c55259177f2936516b6087d634078e0 Mon Sep 17 00:00:00 2001
From: neildhickey <nhickey@nvidia.com>
Date: Tue, 29 Oct 2024 13:34:43 +0000
Subject: [PATCH 431/693] [clang][AArch64] Add getHostCPUFeatures to query for
 enabled features in cpu info (#97749)

Add getHostCPUFeatures into the AArch64 Target Parser to query the
cpuinfo for the device in the case where we are compiling with
-mcpu=native.
Add LLVM_CPUINFO environment variable to test mock /proc/cpuinfo
files for -mcpu=native

Co-authored-by: Elvina Yakubova <eyakubova@nvidia.com>
---
 clang/lib/Driver/ToolChains/Arch/AArch64.cpp  |  20 ++-
 clang/test/Driver/Inputs/cpunative/cortex-a57 |   8 +
 clang/test/Driver/Inputs/cpunative/cortex-a72 |   8 +
 clang/test/Driver/Inputs/cpunative/cortex-a76 |   8 +
 .../test/Driver/Inputs/cpunative/neoverse-n1  |   8 +
 .../test/Driver/Inputs/cpunative/neoverse-v2  |   8 +
 clang/test/Driver/aarch64-mcpu-native.c       | 138 ++++++++++++++++++
 llvm/lib/TargetParser/Host.cpp                |  10 +-
 8 files changed, 198 insertions(+), 10 deletions(-)
 create mode 100644 clang/test/Driver/Inputs/cpunative/cortex-a57
 create mode 100644 clang/test/Driver/Inputs/cpunative/cortex-a72
 create mode 100644 clang/test/Driver/Inputs/cpunative/cortex-a76
 create mode 100644 clang/test/Driver/Inputs/cpunative/neoverse-n1
 create mode 100644 clang/test/Driver/Inputs/cpunative/neoverse-v2
 create mode 100644 clang/test/Driver/aarch64-mcpu-native.c

diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
index f083e40df131..1e2ac4e501ba 100644
--- a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
@@ -135,15 +135,21 @@ getAArch64ArchFeaturesFromMarch(const Driver &D, StringRef March,
   return true;
 }
 
-static bool
-getAArch64ArchFeaturesFromMcpu(const Driver &D, StringRef Mcpu,
-                               const ArgList &Args,
-                               llvm::AArch64::ExtensionSet &Extensions) {
+static bool getAArch64ArchFeaturesFromMcpu(
+    const Driver &D, StringRef Mcpu, const ArgList &Args,
+    llvm::AArch64::ExtensionSet &Extensions, std::vector<StringRef> &Features) {
   StringRef CPU;
   std::string McpuLowerCase = Mcpu.lower();
   if (!DecodeAArch64Mcpu(D, McpuLowerCase, CPU, Extensions))
     return false;
 
+  if (Mcpu == "native") {
+    llvm::StringMap<bool> HostFeatures = llvm::sys::getHostCPUFeatures();
+    for (auto &[Feature, Enabled] : HostFeatures) {
+      Features.push_back(Args.MakeArgString((Enabled ? "+" : "-") + Feature));
+    }
+  }
+
   return true;
 }
 
@@ -210,11 +216,11 @@ void aarch64::getAArch64TargetFeatures(const Driver &D,
     success =
         getAArch64ArchFeaturesFromMarch(D, A->getValue(), Args, Extensions);
   else if ((A = Args.getLastArg(options::OPT_mcpu_EQ)))
-    success =
-        getAArch64ArchFeaturesFromMcpu(D, A->getValue(), Args, Extensions);
+    success = getAArch64ArchFeaturesFromMcpu(D, A->getValue(), Args, Extensions,
+                                             Features);
   else if (isCPUDeterminedByTriple(Triple))
     success = getAArch64ArchFeaturesFromMcpu(
-        D, getAArch64TargetCPU(Args, Triple, A), Args, Extensions);
+        D, getAArch64TargetCPU(Args, Triple, A), Args, Extensions, Features);
   else
     // Default to 'A' profile if the architecture is not specified.
     success = getAArch64ArchFeaturesFromMarch(D, "armv8-a", Args, Extensions);
diff --git a/clang/test/Driver/Inputs/cpunative/cortex-a57 b/clang/test/Driver/Inputs/cpunative/cortex-a57
new file mode 100644
index 000000000000..e1903012ab79
--- /dev/null
+++ b/clang/test/Driver/Inputs/cpunative/cortex-a57
@@ -0,0 +1,8 @@
+processor       : 0
+BogoMIPS        : 200.00
+Features        : fp asimd evtstrm crc32 cpuid
+CPU implementer : 0x41
+CPU architecture: 8
+CPU variant     : 0x1
+CPU part        : 0xd07
+CPU revision    : 1
diff --git a/clang/test/Driver/Inputs/cpunative/cortex-a72 b/clang/test/Driver/Inputs/cpunative/cortex-a72
new file mode 100644
index 000000000000..7aed4a6fa732
--- /dev/null
+++ b/clang/test/Driver/Inputs/cpunative/cortex-a72
@@ -0,0 +1,8 @@
+processor       : 0
+BogoMIPS        : 250.00
+Features        : fp asimd evtstrm aes pmull sha1 sha2 crc32 cpuid asimdrdm
+CPU implementer : 0x41
+CPU architecture: 8
+CPU variant     : 0x0
+CPU part        : 0xd08
+CPU revision    : 2
diff --git a/clang/test/Driver/Inputs/cpunative/cortex-a76 b/clang/test/Driver/Inputs/cpunative/cortex-a76
new file mode 100644
index 000000000000..21822cfcec60
--- /dev/null
+++ b/clang/test/Driver/Inputs/cpunative/cortex-a76
@@ -0,0 +1,8 @@
+processor       : 0
+BogoMIPS        : 500.00
+Features        : fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics cpuid asimdrdm ssbs jscvt fcma
+CPU implementer : 0x41
+CPU architecture: 8
+CPU variant     : 0x1
+CPU part        : 0xd0b
+CPU revision    : 2
diff --git a/clang/test/Driver/Inputs/cpunative/neoverse-n1 b/clang/test/Driver/Inputs/cpunative/neoverse-n1
new file mode 100644
index 000000000000..571e8840b09f
--- /dev/null
+++ b/clang/test/Driver/Inputs/cpunative/neoverse-n1
@@ -0,0 +1,8 @@
+processor       : 0
+BogoMIPS        : 50.00
+Features        : fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm lrcpc dcpop asimddp ssbs
+CPU implementer : 0x41
+CPU architecture: 8
+CPU variant     : 0x3
+CPU part        : 0xd0c
+CPU revision    : 1
diff --git a/clang/test/Driver/Inputs/cpunative/neoverse-v2 b/clang/test/Driver/Inputs/cpunative/neoverse-v2
new file mode 100644
index 000000000000..c3c8433415d7
--- /dev/null
+++ b/clang/test/Driver/Inputs/cpunative/neoverse-v2
@@ -0,0 +1,8 @@
+processor       : 0
+BogoMIPS        : 2000.00
+Features        : fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm jscvt fcma lrcpc dcpop sha3 sm3 sm4 asimddp sha512 sve asimdfhm dit uscat ilrcpc flagm ssbs sb paca pacg dcpodp sve2 sveaes svepmull svebitperm svesha3 svesm4 flagm2 frint svei8mm svebf16 i8mm bf16 dgh bti
+CPU implementer : 0x41
+CPU architecture: 8
+CPU variant     : 0x0
+CPU part        : 0xd4f
+CPU revision    : 0
diff --git a/clang/test/Driver/aarch64-mcpu-native.c b/clang/test/Driver/aarch64-mcpu-native.c
new file mode 100644
index 000000000000..f1d0ba76ad79
--- /dev/null
+++ b/clang/test/Driver/aarch64-mcpu-native.c
@@ -0,0 +1,138 @@
+// REQUIRES: aarch64-registered-target
+// RUN: export LLVM_CPUINFO=%S/Inputs/cpunative/neoverse-v2
+// RUN: %clang --target=aarch64 --print-enabled-extensions -mcpu=native | FileCheck --strict-whitespace --check-prefix=CHECK-FEAT-NV2 --implicit-check-not=FEAT_ %s
+
+// CHECK-FEAT-NV2: Extensions enabled for the given AArch64 target
+// CHECK-FEAT-NV2-EMPTY:
+// CHECK-FEAT-NV2:    Architecture Feature(s)                                Description
+// CHECK-FEAT-NV2:    FEAT_AES, FEAT_PMULL                                   Enable AES support
+// CHECK-FEAT-NV2:    FEAT_AMUv1                                             Enable Armv8.4-A Activity Monitors extension
+// CHECK-FEAT-NV2:    FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
+// CHECK-FEAT-NV2:    FEAT_BF16                                              Enable BFloat16 Extension
+// CHECK-FEAT-NV2:    FEAT_BTI                                               Enable Branch Target Identification
+// CHECK-FEAT-NV2:    FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
+// CHECK-FEAT-NV2:    FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
+// CHECK-FEAT-NV2:    FEAT_CSV2_2                                            Enable architectural speculation restriction
+// CHECK-FEAT-NV2:    FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
+// CHECK-FEAT-NV2:    FEAT_DPB                                               Enable Armv8.2-A data Cache Clean to Point of Persistence
+// CHECK-FEAT-NV2:    FEAT_DPB2                                              Enable Armv8.5-A Cache Clean to Point of Deep Persistence
+// CHECK-FEAT-NV2:    FEAT_DotProd                                           Enable dot product support
+// CHECK-FEAT-NV2:    FEAT_ETE                                               Enable Embedded Trace Extension
+// CHECK-FEAT-NV2:    FEAT_FCMA                                              Enable Armv8.3-A Floating-point complex number support
+// CHECK-FEAT-NV2:    FEAT_FHM                                               Enable FP16 FML instructions
+// CHECK-FEAT-NV2:    FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
+// CHECK-FEAT-NV2:    FEAT_FP16                                              Enable half-precision floating-point data processing
+// CHECK-FEAT-NV2:    FEAT_FRINTTS                                           Enable FRInt[32|64][Z|X] instructions that round a floating-point number to an integer (in FP format) forcing it to fit into a 32- or 64-bit int
+// CHECK-FEAT-NV2:    FEAT_FlagM                                             Enable Armv8.4-A Flag Manipulation instructions
+// CHECK-FEAT-NV2:    FEAT_FlagM2                                            Enable alternative NZCV format for floating point comparisons
+// CHECK-FEAT-NV2:    FEAT_I8MM                                              Enable Matrix Multiply Int8 Extension
+// CHECK-FEAT-NV2:    FEAT_JSCVT                                             Enable Armv8.3-A JavaScript FP conversion instructions
+// CHECK-FEAT-NV2:    FEAT_LOR                                               Enable Armv8.1-A Limited Ordering Regions extension
+// CHECK-FEAT-NV2:    FEAT_LRCPC                                             Enable support for RCPC extension
+// CHECK-FEAT-NV2:    FEAT_LRCPC2                                            Enable Armv8.4-A RCPC instructions with Immediate Offsets
+// CHECK-FEAT-NV2:    FEAT_LSE                                               Enable Armv8.1-A Large System Extension (LSE) atomic instructions
+// CHECK-FEAT-NV2:    FEAT_LSE2                                              Enable Armv8.4-A Large System Extension 2 (LSE2) atomicity rules
+// CHECK-FEAT-NV2:    FEAT_MPAM                                              Enable Armv8.4-A Memory system Partitioning and Monitoring extension
+// CHECK-FEAT-NV2:    FEAT_MTE, FEAT_MTE2                                    Enable Memory Tagging Extension
+// CHECK-FEAT-NV2:    FEAT_NV, FEAT_NV2                                      Enable Armv8.4-A Nested Virtualization Enchancement
+// CHECK-FEAT-NV2:    FEAT_PAN                                               Enable Armv8.1-A Privileged Access-Never extension
+// CHECK-FEAT-NV2:    FEAT_PAN2                                              Enable Armv8.2-A PAN s1e1R and s1e1W Variants
+// CHECK-FEAT-NV2:    FEAT_PAuth                                             Enable Armv8.3-A Pointer Authentication extension
+// CHECK-FEAT-NV2:    FEAT_PMUv3                                             Enable Armv8.0-A PMUv3 Performance Monitors extension
+// CHECK-FEAT-NV2:    FEAT_RAS, FEAT_RASv1p1                                 Enable Armv8.0-A Reliability, Availability and Serviceability Extensions
+// CHECK-FEAT-NV2:    FEAT_RDM                                               Enable Armv8.1-A Rounding Double Multiply Add/Subtract instructions
+// CHECK-FEAT-NV2:    FEAT_RNG                                               Enable Random Number generation instructions
+// CHECK-FEAT-NV2:    FEAT_SB                                                Enable Armv8.5-A Speculation Barrier
+// CHECK-FEAT-NV2:    FEAT_SEL2                                              Enable Armv8.4-A Secure Exception Level 2 extension
+// CHECK-FEAT-NV2:    FEAT_SHA1, FEAT_SHA256                                 Enable SHA1 and SHA256 support
+// CHECK-FEAT-NV2:    FEAT_SPE                                               Enable Statistical Profiling extension
+// CHECK-FEAT-NV2:    FEAT_SPECRES                                           Enable Armv8.5-A execution and data prediction invalidation instructions
+// CHECK-FEAT-NV2:    FEAT_SSBS, FEAT_SSBS2                                  Enable Speculative Store Bypass Safe bit
+// CHECK-FEAT-NV2:    FEAT_SVE                                               Enable Scalable Vector Extension (SVE) instructions
+// CHECK-FEAT-NV2:    FEAT_SVE2                                              Enable Scalable Vector Extension 2 (SVE2) instructions
+// CHECK-FEAT-NV2:    FEAT_SVE_BitPerm                                       Enable bit permutation SVE2 instructions
+// CHECK-FEAT-NV2:    FEAT_TLBIOS, FEAT_TLBIRANGE                            Enable Armv8.4-A TLB Range and Maintenance instructions
+// CHECK-FEAT-NV2:    FEAT_TRBE                                              Enable Trace Buffer Extension
+// CHECK-FEAT-NV2:    FEAT_TRF                                               Enable Armv8.4-A Trace extension
+// CHECK-FEAT-NV2:    FEAT_UAO                                               Enable Armv8.2-A UAO PState
+// CHECK-FEAT-NV2:    FEAT_VHE                                               Enable Armv8.1-A Virtual Host extension
+
+// RUN: export LLVM_CPUINFO=%S/Inputs/cpunative/neoverse-n1
+// RUN: %clang --target=aarch64 --print-enabled-extensions -mcpu=native | FileCheck --strict-whitespace --check-prefix=CHECK-FEAT-NN1 --implicit-check-not=FEAT_ %s
+
+// CHECK-FEAT-NN1: Extensions enabled for the given AArch64 target
+// CHECK-FEAT-NN1-EMPTY:
+// CHECK-FEAT-NN1:    Architecture Feature(s)                                Description
+// CHECK-FEAT-NN1:    FEAT_AES, FEAT_PMULL                                   Enable AES support
+// CHECK-FEAT-NN1:    FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
+// CHECK-FEAT-NN1:    FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
+// CHECK-FEAT-NN1:    FEAT_DPB                                               Enable Armv8.2-A data Cache Clean to Point of Persistence
+// CHECK-FEAT-NN1:    FEAT_DotProd                                           Enable dot product support
+// CHECK-FEAT-NN1:    FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
+// CHECK-FEAT-NN1:    FEAT_FP16                                              Enable half-precision floating-point data processing
+// CHECK-FEAT-NN1:    FEAT_LOR                                               Enable Armv8.1-A Limited Ordering Regions extension
+// CHECK-FEAT-NN1:    FEAT_LRCPC                                             Enable support for RCPC extension
+// CHECK-FEAT-NN1:    FEAT_LSE                                               Enable Armv8.1-A Large System Extension (LSE) atomic instructions
+// CHECK-FEAT-NN1:    FEAT_PAN                                               Enable Armv8.1-A Privileged Access-Never extension
+// CHECK-FEAT-NN1:    FEAT_PAN2                                              Enable Armv8.2-A PAN s1e1R and s1e1W Variants
+// CHECK-FEAT-NN1:    FEAT_PMUv3                                             Enable Armv8.0-A PMUv3 Performance Monitors extension
+// CHECK-FEAT-NN1:    FEAT_RAS, FEAT_RASv1p1                                 Enable Armv8.0-A Reliability, Availability and Serviceability Extensions
+// CHECK-FEAT-NN1:    FEAT_RDM                                               Enable Armv8.1-A Rounding Double Multiply Add/Subtract instructions
+// CHECK-FEAT-NN1:    FEAT_SHA1, FEAT_SHA256                                 Enable SHA1 and SHA256 support
+// CHECK-FEAT-NN1:    FEAT_SPE                                               Enable Statistical Profiling extension
+// CHECK-FEAT-NN1:    FEAT_SSBS, FEAT_SSBS2                                  Enable Speculative Store Bypass Safe bit
+// CHECK-FEAT-NN1:    FEAT_UAO                                               Enable Armv8.2-A UAO PState
+// CHECK-FEAT-NN1:    FEAT_VHE                                               Enable Armv8.1-A Virtual Host extension
+
+
+// RUN: export LLVM_CPUINFO=%S/Inputs/cpunative/cortex-a57
+// RUN: %clang --target=aarch64 --print-enabled-extensions -mcpu=native | FileCheck --strict-whitespace --check-prefix=CHECK-FEAT-CA57 --implicit-check-not=FEAT_ %s
+
+// CHECK-FEAT-CA57: Extensions enabled for the given AArch64 target
+// CHECK-FEAT-CA57-EMPTY:
+// CHECK-FEAT-CA57:    Architecture Feature(s)                                Description
+// CHECK-FEAT-CA57:    FEAT_AES, FEAT_PMULL                                   Enable AES support
+// CHECK-FEAT-CA57:    FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
+// CHECK-FEAT-CA57:    FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
+// CHECK-FEAT-CA57:    FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
+// CHECK-FEAT-CA57:    FEAT_PMUv3                                             Enable Armv8.0-A PMUv3 Performance Monitors extension
+// CHECK-FEAT-CA57:    FEAT_SHA1, FEAT_SHA256                                 Enable SHA1 and SHA256 support
+
+// RUN: export LLVM_CPUINFO=%S/Inputs/cpunative/cortex-a72
+// RUN: %clang --target=aarch64 --print-enabled-extensions -mcpu=native | FileCheck --strict-whitespace  --check-prefix=CHECK-FEAT-CA72 --implicit-check-not=FEAT_ %s
+
+// CHECK-FEAT-CA72: Extensions enabled for the given AArch64 target
+// CHECK-EMPTY:
+// CHECK-FEAT-CA72:   Architecture Feature(s)                                Description
+// CHECK-FEAT-CA72:    FEAT_AES, FEAT_PMULL                                   Enable AES support
+// CHECK-FEAT-CA72:    FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
+// CHECK-FEAT-CA72:    FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
+// CHECK-FEAT-CA72:    FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
+// CHECK-FEAT-CA72:    FEAT_PMUv3                                             Enable Armv8.0-A PMUv3 Performance Monitors extension
+// CHECK-FEAT-CA72:    FEAT_SHA1, FEAT_SHA256                                 Enable SHA1 and SHA256 support
+
+// RUN: export LLVM_CPUINFO=%S/Inputs/cpunative/cortex-a76
+// RUN: %clang --target=aarch64 --print-enabled-extensions -mcpu=native | FileCheck --strict-whitespace --check-prefix=CHECK-FEAT-CA76 --implicit-check-not=FEAT_ %s
+
+// CHECK-FEAT-CA76: Extensions enabled for the given AArch64 target
+// CHECK-FEAT-CA76-EMPTY:
+// CHECK-FEAT-CA76:    Architecture Feature(s)                                Description
+// CHECK-FEAT-CA76:    FEAT_AES, FEAT_PMULL                                   Enable AES support
+// CHECK-FEAT-CA76:    FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
+// CHECK-FEAT-CA76:    FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
+// CHECK-FEAT-CA76:    FEAT_DPB                                               Enable Armv8.2-A data Cache Clean to Point of Persistence
+// CHECK-FEAT-CA76:    FEAT_DotProd                                           Enable dot product support
+// CHECK-FEAT-CA76:    FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
+// CHECK-FEAT-CA76:    FEAT_FP16                                              Enable half-precision floating-point data processing
+// CHECK-FEAT-CA76:    FEAT_LOR                                               Enable Armv8.1-A Limited Ordering Regions extension
+// CHECK-FEAT-CA76:    FEAT_LRCPC                                             Enable support for RCPC extension
+// CHECK-FEAT-CA76:    FEAT_LSE                                               Enable Armv8.1-A Large System Extension (LSE) atomic instructions
+// CHECK-FEAT-CA76:    FEAT_PAN                                               Enable Armv8.1-A Privileged Access-Never extension
+// CHECK-FEAT-CA76:    FEAT_PAN2                                              Enable Armv8.2-A PAN s1e1R and s1e1W Variants
+// CHECK-FEAT-CA76:    FEAT_PMUv3                                             Enable Armv8.0-A PMUv3 Performance Monitors extension
+// CHECK-FEAT-CA76:    FEAT_RAS, FEAT_RASv1p1                                 Enable Armv8.0-A Reliability, Availability and Serviceability Extensions
+// CHECK-FEAT-CA76:    FEAT_RDM                                               Enable Armv8.1-A Rounding Double Multiply Add/Subtract instructions
+// CHECK-FEAT-CA76:    FEAT_SHA1, FEAT_SHA256                                 Enable SHA1 and SHA256 support
+// CHECK-FEAT-CA76:    FEAT_SSBS, FEAT_SSBS2                                  Enable Speculative Store Bypass Safe bit
+// CHECK-FEAT-CA76:    FEAT_UAO                                               Enable Armv8.2-A UAO PState
+// CHECK-FEAT-CA76:    FEAT_VHE                                               Enable Armv8.1-A Virtual Host extension
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index 5c4e3a9dc52b..de6c4edebba3 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -68,11 +68,15 @@ using namespace llvm;
 
 static std::unique_ptr<llvm::MemoryBuffer>
     LLVM_ATTRIBUTE_UNUSED getProcCpuinfoContent() {
+  const char *CPUInfoFile = "/proc/cpuinfo";
+  if (const char *CpuinfoIntercept = std::getenv("LLVM_CPUINFO"))
+    CPUInfoFile = CpuinfoIntercept;
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Text =
-      llvm::MemoryBuffer::getFileAsStream("/proc/cpuinfo");
+      llvm::MemoryBuffer::getFileAsStream(CPUInfoFile);
+
   if (std::error_code EC = Text.getError()) {
-    llvm::errs() << "Can't read "
-                 << "/proc/cpuinfo: " << EC.message() << "\n";
+    llvm::errs() << "Can't read " << CPUInfoFile << ": " << EC.message()
+                 << "\n";
     return nullptr;
   }
   return std::move(*Text);
-- 
GitLab


From 4b44639a4320f980b3c9fa3b96e911e0741f179c Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Tue, 29 Oct 2024 09:38:48 -0400
Subject: [PATCH 432/693] Nominate Erich Keane for OpenACC (#114041)

Erich is the driving force behind the OpenACC implementation work that
has recently begun in Clang. Given his expertise on the topic and that
he's already aware of maintainer expectations (he maintains templates
and attributes currently), we should recognize that he's also the one
maintaining OpenACC.
---
 clang/Maintainers.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/clang/Maintainers.rst b/clang/Maintainers.rst
index 6635727e9eb6..54690452681a 100644
--- a/clang/Maintainers.rst
+++ b/clang/Maintainers.rst
@@ -250,6 +250,12 @@ OpenCL conformance
 | anastasia\@compiler-experts.com (email), Anastasia (Phabricator), AnastasiaStulova (GitHub)
 
 
+OpenACC
+~~~~~~~
+| Erich Keane
+| ekeane\@nvidia.com (email), ErichKeane (Phabricator), erichkeane (GitHub)
+
+
 SYCL conformance
 ~~~~~~~~~~~~~~~~
 | Alexey Bader
-- 
GitLab


From ec871cfcdf3a46ca1be9842035e4271524644ed1 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Tue, 29 Oct 2024 15:00:36 +0100
Subject: [PATCH 433/693] [clang][bytecode][NFC] Remove Pointer::elem()
 (#114046)

Unused.
---
 clang/lib/AST/ByteCode/Pointer.h | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
index 72e255dba13f..457fe93b2781 100644
--- a/clang/lib/AST/ByteCode/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -653,15 +653,6 @@ public:
     return *reinterpret_cast<T *>(asBlockPointer().Pointee->rawData() + Offset);
   }
 
-  /// Dereferences a primitive element.
-  template <typename T> T &elem(unsigned I) const {
-    assert(I < getNumElems());
-    assert(isBlockPointer());
-    assert(asBlockPointer().Pointee);
-    return reinterpret_cast<T *>(asBlockPointer().Pointee->data() +
-                                 sizeof(InitMapPtr))[I];
-  }
-
   /// Whether this block can be read from at all. This is only true for
   /// block pointers that point to a valid location inside that block.
   bool isDereferencable() const {
-- 
GitLab


From c370869cd6f66e3c2ab33528e44959279311e499 Mon Sep 17 00:00:00 2001
From: Piotr Fusik <p.fusik@samsung.com>
Date: Tue, 29 Oct 2024 15:01:37 +0100
Subject: [PATCH 434/693] [mlir][NFC] Avoid a warning (#114052)

gcc 14.1 warning: template-id not allowed for destructor in C++20
[-Wtemplate-id-cdtor]
---
 mlir/lib/Pass/PassRegistry.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/lib/Pass/PassRegistry.cpp b/mlir/lib/Pass/PassRegistry.cpp
index fe8427559584..029512fd3ecc 100644
--- a/mlir/lib/Pass/PassRegistry.cpp
+++ b/mlir/lib/Pass/PassRegistry.cpp
@@ -422,7 +422,7 @@ llvm::cl::OptionValue<OpPassManager>::operator=(
   return *this;
 }
 
-llvm::cl::OptionValue<OpPassManager>::~OptionValue<OpPassManager>() = default;
+llvm::cl::OptionValue<OpPassManager>::~OptionValue() = default;
 
 void llvm::cl::OptionValue<OpPassManager>::setValue(
     const OpPassManager &newValue) {
-- 
GitLab


From f490697cb9ad3db101ced7f4844002ffa0c73da8 Mon Sep 17 00:00:00 2001
From: Boaz Brickner <brickner@google.com>
Date: Tue, 29 Oct 2024 15:08:24 +0100
Subject: [PATCH 435/693] [clang] [NFC] Fix a couple of typos: assuments and
 assingment

---
 clang/lib/Sema/CheckExprLifetime.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Sema/CheckExprLifetime.cpp b/clang/lib/Sema/CheckExprLifetime.cpp
index aa0a2e223e70..357082fe3293 100644
--- a/clang/lib/Sema/CheckExprLifetime.cpp
+++ b/clang/lib/Sema/CheckExprLifetime.cpp
@@ -472,7 +472,7 @@ shouldTrackFirstArgumentForConstructor(const CXXConstructExpr *Ctor) {
 }
 
 // Return true if this is an "normal" assignment operator.
-// We assuments that a normal assingment operator always returns *this, that is,
+// We assume that a normal assignment operator always returns *this, that is,
 // an lvalue reference that is the same type as the implicit object parameter
 // (or the LHS for a non-member operator$=).
 static bool isNormalAssignmentOperator(const FunctionDecl *FD) {
-- 
GitLab


From 87b6ec3be6b80f8e35d2eaea468e6bca79e79c2e Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Tue, 29 Oct 2024 15:08:41 +0100
Subject: [PATCH 436/693] [clang][bytecode] Diagnose placement-new construction
 to inactive field (#114047)

We can reuse CheckActive() for this.
---
 clang/lib/AST/ByteCode/Interp.cpp         |  5 +++++
 clang/test/AST/ByteCode/placement-new.cpp | 16 +++++++++++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index 6e45cfb7e8a2..513d4512b45c 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -1451,6 +1451,11 @@ bool CheckNewTypeMismatch(InterpState &S, CodePtr OpPC, const Expr *E,
         << StorageType << AllocType;
     return false;
   }
+
+  // Can't activate fields in a union, unless the direct base is the union.
+  if (Ptr.inUnion() && !Ptr.isActive() && !Ptr.getBase().getRecord()->isUnion())
+    return CheckActive(S, OpPC, Ptr, AK_Construct);
+
   return true;
 }
 
diff --git a/clang/test/AST/ByteCode/placement-new.cpp b/clang/test/AST/ByteCode/placement-new.cpp
index 5673b5cba3f7..56f54ff168f3 100644
--- a/clang/test/AST/ByteCode/placement-new.cpp
+++ b/clang/test/AST/ByteCode/placement-new.cpp
@@ -14,7 +14,9 @@ namespace std {
   template<typename T, typename ...Args>
   constexpr void construct_at(void *p, Args &&...args) {
     new (p) T((Args&&)args...); // both-note {{in call to}} \
-                                // both-note {{placement new would change type of storage from 'int' to 'float'}}
+                                // both-note {{placement new would change type of storage from 'int' to 'float'}} \
+                                // both-note {{construction of subobject of member 'x' of union with active member 'a' is not allowed in a constant expression}}
+
   }
 }
 
@@ -284,6 +286,18 @@ namespace ConstructAt {
   static_assert(bad_construct_at_type()); // both-error {{not an integral constant expression}} \
                                           // both-note {{in call}}
 
+  constexpr bool bad_construct_at_subobject() {
+    struct X { int a, b; };
+    union A {
+      int a;
+      X x;
+    };
+    A a = {1};
+    std::construct_at<int>(&a.x.a, 1); // both-note {{in call}}
+    return true;
+  }
+  static_assert(bad_construct_at_subobject()); // both-error{{not an integral constant expression}} \
+                                               // both-note {{in call}}
 }
 
 namespace UsedToCrash {
-- 
GitLab


From b9376915cf897e79a852497c60f18ddacb1830ae Mon Sep 17 00:00:00 2001
From: Boaz Brickner <brickner@google.com>
Date: Tue, 29 Oct 2024 15:09:23 +0100
Subject: [PATCH 437/693] [clang] [NFC] Fix assingments typo

---
 clang/test/CodeGenOpenCLCXX/local_addrspace_init.clcpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/test/CodeGenOpenCLCXX/local_addrspace_init.clcpp b/clang/test/CodeGenOpenCLCXX/local_addrspace_init.clcpp
index bb9acf09d120..0a6462e42624 100644
--- a/clang/test/CodeGenOpenCLCXX/local_addrspace_init.clcpp
+++ b/clang/test/CodeGenOpenCLCXX/local_addrspace_init.clcpp
@@ -14,7 +14,7 @@ kernel void test() {
   // address space variables. User defined initialization could
   // make sense, but would it mean that all work items need to
   // execute it? Potentially disallowing any initialization would
-  // make things easier and assingments can be used to set specific
+  // make things easier and assignments can be used to set specific
   // values. This rules should make it consistent with OpenCL C.
   //__local C c();
 }
-- 
GitLab


From 340cd4e631d72d02cd79f9aad74d2a354abc977e Mon Sep 17 00:00:00 2001
From: Dmitry Chernenkov <dmitryc@google.com>
Date: Tue, 29 Oct 2024 14:12:49 +0000
Subject: [PATCH 438/693] [Bazel] fix for
 abc49cc19463970d5523d7d3332e4c1f83bc2ef7

---
 .../llvm-project-overlay/libc/BUILD.bazel     | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 774366d48a21..1956233590f7 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -136,6 +136,17 @@ libc_support_library(
     hdrs = ["hdr/fenv_macros.h"],
 )
 
+libc_support_library(
+    name = "hdr_fcntl_macros",
+    hdrs = ["hdr/fcntl_macros.h"],
+    deps = [":hdr_fcntl_overlay"],
+)
+
+libc_support_library(
+    name = "hdr_fcntl_overlay",
+    hdrs = ["hdr/fcntl_overlay.h"],
+)
+
 libc_support_library(
     name = "hdr_signal_macros",
     hdrs = ["hdr/signal_macros.h"],
@@ -201,6 +212,11 @@ libc_support_library(
     hdrs = ["hdr/types/fexcept_t.h"],
 )
 
+libc_support_library(
+    name = "types_mode_t",
+    hdrs = ["hdr/types/mode_t.h"],
+)
+
 libc_support_library(
     name = "types_sigset_t",
     hdrs = ["hdr/types/sigset_t.h"],
@@ -3488,6 +3504,8 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
+        ":types_mode_t",
     ],
 )
 
@@ -3503,6 +3521,8 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
+        ":types_mode_t",
     ],
 )
 
@@ -3514,6 +3534,7 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
     ],
 )
 
@@ -3527,6 +3548,7 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
     ],
 )
 
@@ -3571,6 +3593,7 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
     ],
 )
 
@@ -3710,6 +3733,7 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
     ],
 )
 
@@ -3721,6 +3745,7 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
     ],
 )
 
@@ -3800,6 +3825,7 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
     ],
 )
 
@@ -3811,6 +3837,7 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
     ],
 )
 
@@ -3822,6 +3849,7 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
     ],
 )
 
@@ -3833,6 +3861,7 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
     ],
 )
 
@@ -3844,6 +3873,7 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
     ],
 )
 
@@ -3891,6 +3921,7 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
     ],
 )
 
@@ -3902,6 +3933,7 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
     ],
 )
 
@@ -4161,6 +4193,7 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
         ":hdr_stdio_overlay",
         ":types_FILE",
     ],
@@ -4178,6 +4211,7 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
         ":llvm_libc_macros_fcntl_macros",
     ],
 )
@@ -4192,6 +4226,8 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_fcntl_macros",
+        ":types_mode_t",
     ],
 )
 
-- 
GitLab


From f257e9bdbbb790e4fe0a5e2538c92d7edd85a2e5 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 29 Oct 2024 12:44:32 +0000
Subject: [PATCH 439/693] [clang][x86] Update AVX/AVX512 setzero constexpr
 tests to use the TEST_CONSTEXPR macro

---
 clang/test/CodeGen/X86/avx-builtins.c         | 20 ++--------
 clang/test/CodeGen/X86/avx512f-builtins.c     | 29 +++-----------
 clang/test/CodeGen/X86/builtin_test_helpers.h | 38 +++++++++++++++----
 3 files changed, 41 insertions(+), 46 deletions(-)

diff --git a/clang/test/CodeGen/X86/avx-builtins.c b/clang/test/CodeGen/X86/avx-builtins.c
index 9d6c1897f540..4e56204c8ad4 100644
--- a/clang/test/CodeGen/X86/avx-builtins.c
+++ b/clang/test/CodeGen/X86/avx-builtins.c
@@ -11,6 +11,7 @@
 
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 // NOTE: This should match the tests in llvm/test/CodeGen/X86/avx-intrinsics-fast-isel.ll
 
@@ -1740,18 +1741,21 @@ __m256d test_mm256_setzero_pd(void) {
   // CHECK: store <4 x double> zeroinitializer
   return _mm256_setzero_pd();
 }
+TEST_CONSTEXPR(match_m256d(_mm256_setzero_pd(), +0.0, +0.0, +0.0, +0.0));
 
 __m256 test_mm256_setzero_ps(void) {
   // CHECK-LABEL: test_mm256_setzero_ps
   // CHECK: store <8 x float> zeroinitializer
   return _mm256_setzero_ps();
 }
+TEST_CONSTEXPR(match_m256(_mm256_setzero_ps(), +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f));
 
 __m256i test_mm256_setzero_si256(void) {
   // CHECK-LABEL: test_mm256_setzero_si256
   // CHECK: store <4 x i64> zeroinitializer
   return _mm256_setzero_si256();
 }
+TEST_CONSTEXPR(match_m256i(_mm256_setzero_si256(), 0, 0, 0, 0));
 
 __m256d test_mm256_shuffle_pd(__m256d A, __m256d B) {
   // CHECK-LABEL: test_mm256_shuffle_pd
@@ -2097,19 +2101,3 @@ float test_mm256_cvtss_f32(__m256 __a)
   // CHECK: extractelement <8 x float> %{{.*}}, i32 0
   return _mm256_cvtss_f32(__a);
 }
-
-// Test constexpr handling.
-#if defined(__cplusplus) && (__cplusplus >= 201103L)
-
-void test_constexpr() {
-  constexpr __m256d v_mm256_setzero_pd = _mm256_setzero_pd();
-  static_assert(v_mm256_setzero_pd[0] == +0.0 && v_mm256_setzero_pd[1] == +0.0 && v_mm256_setzero_pd[2] == +0.0 && v_mm256_setzero_pd[3] == +0.0);
-
-  constexpr __m256 v_mm256_setzero_ps = _mm256_setzero_ps();
-  static_assert(v_mm256_setzero_ps[0] == +0.0f && v_mm256_setzero_ps[1] == +0.0f && v_mm256_setzero_ps[2] == +0.0f && v_mm256_setzero_ps[3] == +0.0f && v_mm256_setzero_ps[4] == +0.0f && v_mm256_setzero_ps[5] == +0.0f && v_mm256_setzero_ps[6] == +0.0f && v_mm256_setzero_ps[7] == +0.0f);
-
-  constexpr __m256i v_mm256_setzero_si256 = _mm256_setzero_si256();
-  static_assert(v_mm256_setzero_si256[0] == 0x0000000000000000ULL && v_mm256_setzero_si256[1] == 0x0000000000000000ULL && v_mm256_setzero_si256[2] == 0x0000000000000000ULL && v_mm256_setzero_si256[3] == 0x0000000000000000ULL);
-}
-
-#endif
diff --git a/clang/test/CodeGen/X86/avx512f-builtins.c b/clang/test/CodeGen/X86/avx512f-builtins.c
index 0b4f778a0637..372790a8cd66 100644
--- a/clang/test/CodeGen/X86/avx512f-builtins.c
+++ b/clang/test/CodeGen/X86/avx512f-builtins.c
@@ -4,6 +4,7 @@
 // RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -fms-extensions -fms-compatibility -ffreestanding %s -triple=x86_64-windows-msvc -target-feature +avx512f -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 __m512d test_mm512_sqrt_pd(__m512d a)
 {
@@ -10615,13 +10616,13 @@ __m128 test_mm_maskz_cvtsd_ss(__mmask8 __U, __m128 __A, __m128d __B) {
   return _mm_maskz_cvtsd_ss(__U, __A, __B); 
 }
 
-
 __m512i test_mm512_setzero_epi32(void)
 {
   // CHECK-LABEL: test_mm512_setzero_epi32
   // CHECK: zeroinitializer
   return _mm512_setzero_epi32();
 }
+TEST_CONSTEXPR(match_m512i(_mm512_setzero_epi32(), 0, 0, 0, 0, 0, 0, 0, 0));
 
 __m512 test_mm512_setzero(void)
 {
@@ -10629,6 +10630,7 @@ __m512 test_mm512_setzero(void)
   // CHECK: zeroinitializer
   return _mm512_setzero();
 }
+TEST_CONSTEXPR(match_m512(_mm512_setzero(), +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f));
 
 __m512i test_mm512_setzero_si512(void)
 {
@@ -10636,6 +10638,7 @@ __m512i test_mm512_setzero_si512(void)
   // CHECK: zeroinitializer
   return _mm512_setzero_si512();
 }
+TEST_CONSTEXPR(match_m512i(_mm512_setzero_si512(), 0, 0, 0, 0, 0, 0, 0, 0));
 
 __m512 test_mm512_setzero_ps(void)
 {
@@ -10643,6 +10646,7 @@ __m512 test_mm512_setzero_ps(void)
   // CHECK: zeroinitializer
   return _mm512_setzero_ps();
 }
+TEST_CONSTEXPR(match_m512(_mm512_setzero_ps(), +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f));
 
 __m512d test_mm512_setzero_pd(void)
 {
@@ -10650,6 +10654,7 @@ __m512d test_mm512_setzero_pd(void)
   // CHECK: zeroinitializer
   return _mm512_setzero_pd();
 }
+TEST_CONSTEXPR(match_m512d(_mm512_setzero_pd(), 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0));
 
 __mmask16 test_mm512_int2mask(int __a)
 {
@@ -10880,25 +10885,3 @@ void test_mm512_mask_i32loscatter_epi64(void *__addr, __mmask8 __mask, __m512i _
   // CHECK: @llvm.x86.avx512.mask.scatter.dpq.512
   _mm512_mask_i32loscatter_epi64(__addr, __mask, __index, __v1, 2);
 }
-
-// Test constexpr handling.
-#if defined(__cplusplus) && (__cplusplus >= 201103L)
-
-void test_constexpr() {
-  constexpr __m512 v_mm512_setzero = _mm512_setzero();
-  static_assert(v_mm512_setzero[0] == +0.0f && v_mm512_setzero[1] == +0.0f && v_mm512_setzero[2] == +0.0f && v_mm512_setzero[3] == +0.0f && v_mm512_setzero[4] == +0.0f && v_mm512_setzero[5] == +0.0f && v_mm512_setzero[6] == +0.0f && v_mm512_setzero[7] == +0.0f && v_mm512_setzero[8] == +0.0f && v_mm512_setzero[9] == +0.0f && v_mm512_setzero[10] == +0.0f && v_mm512_setzero[11] == +0.0f && v_mm512_setzero[12] == +0.0f && v_mm512_setzero[13] == +0.0f && v_mm512_setzero[14] == +0.0f && v_mm512_setzero[15] == +0.0f);
-
-  constexpr __m512 v_mm512_setzero_ps = _mm512_setzero_ps();
-  static_assert(v_mm512_setzero_ps[0] == +0.0f && v_mm512_setzero_ps[1] == +0.0f && v_mm512_setzero_ps[2] == +0.0f && v_mm512_setzero_ps[3] == +0.0f && v_mm512_setzero_ps[4] == +0.0f && v_mm512_setzero_ps[5] == +0.0f && v_mm512_setzero_ps[6] == +0.0f && v_mm512_setzero_ps[7] == +0.0f && v_mm512_setzero_ps[8] == +0.0f && v_mm512_setzero_ps[9] == +0.0f && v_mm512_setzero_ps[10] == +0.0f && v_mm512_setzero_ps[11] == +0.0f && v_mm512_setzero_ps[12] == +0.0f && v_mm512_setzero_ps[13] == +0.0f && v_mm512_setzero_ps[14] == +0.0f && v_mm512_setzero_ps[15] == +0.0f);
-
-  constexpr __m512d v_mm512_setzero_pd = _mm512_setzero_pd();
-  static_assert(v_mm512_setzero_pd[0] == +0.0 && v_mm512_setzero_pd[1] == +0.0 && v_mm512_setzero_pd[2] == +0.0 && v_mm512_setzero_pd[3] == +0.0 && v_mm512_setzero_pd[4] == +0.0 && v_mm512_setzero_pd[5] == +0.0 && v_mm512_setzero_pd[6] == +0.0 && v_mm512_setzero_pd[7] == +0.0);
-
-  constexpr __m512i v_mm512_setzero_si512 = _mm512_setzero_si512();
-  static_assert(v_mm512_setzero_si512[0] == 0x0000000000000000ULL && v_mm512_setzero_si512[1] == 0x0000000000000000ULL && v_mm512_setzero_si512[2] == 0x0000000000000000ULL && v_mm512_setzero_si512[3] == 0x0000000000000000ULL && v_mm512_setzero_si512[4] == 0x0000000000000000ULL && v_mm512_setzero_si512[5] == 0x0000000000000000ULL && v_mm512_setzero_si512[6] == 0x0000000000000000ULL && v_mm512_setzero_si512[7] == 0x0000000000000000ULL);
-
-  constexpr __m512i v_mm512_setzero_epi32 = _mm512_setzero_epi32();
-  static_assert(v_mm512_setzero_epi32[0] == 0x0000000000000000ULL && v_mm512_setzero_epi32[1] == 0x0000000000000000ULL && v_mm512_setzero_epi32[2] == 0x0000000000000000ULL && v_mm512_setzero_epi32[3] == 0x0000000000000000ULL && v_mm512_setzero_epi32[4] == 0x0000000000000000ULL && v_mm512_setzero_epi32[5] == 0x0000000000000000ULL && v_mm512_setzero_epi32[6] == 0x0000000000000000ULL && v_mm512_setzero_epi32[7] == 0x0000000000000000ULL);
-}
-
-#endif
diff --git a/clang/test/CodeGen/X86/builtin_test_helpers.h b/clang/test/CodeGen/X86/builtin_test_helpers.h
index 043b6ecbc69f..5e77ff3a7ca4 100644
--- a/clang/test/CodeGen/X86/builtin_test_helpers.h
+++ b/clang/test/CodeGen/X86/builtin_test_helpers.h
@@ -4,16 +4,40 @@
 
 #if defined(__cplusplus) && (__cplusplus >= 201103L)
 
-constexpr bool match_m128(__m128 v, float x, float y, float z, float w) {
-  return v[0] == x && v[1] == y && v[2] == z && v[3] == w;
+constexpr bool match_m128(__m128 v, float a, float b, float c, float d) {
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d;
 }
 
-constexpr bool match_m128d(__m128d v, double x, double y) {
-  return v[0] == x && v[1] == y;
+constexpr bool match_m128d(__m128d v, double a, double b) {
+  return v[0] == a && v[1] == b;
 }
 
-constexpr bool match_m128i(__m128i v, unsigned long long x, unsigned long long y) {
-  return v[0] == x && v[1] == y;
+constexpr bool match_m128i(__m128i v, unsigned long long a, unsigned long long b) {
+  return v[0] == a && v[1] == b;
+}
+
+constexpr bool match_m256(__m256 v, float a, float b, float c, float d, float e, float f, float g, float h) {
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h;
+}
+
+constexpr bool match_m256d(__m256d v, double a, double b, double c, double d) {
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d;
+}
+
+constexpr bool match_m256i(__m256i v, unsigned long long a, unsigned long long b, unsigned long long c, unsigned long long d) {
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d;
+}
+
+constexpr bool match_m512(__m512 v, float a, float b, float c, float d, float e, float f, float g, float h, float i, float j, float k, float l, float m, float n, float o, float p) {
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h && v[8] == i && v[9] == j && v[10] == k && v[11] == l && v[12] == m && v[13] == n && v[14] == o && v[15] == p;
+}
+
+constexpr bool match_m512d(__m512d v, double a, double b, double c, double d, double e, double f, double g, double h) {
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h;
+}
+
+constexpr bool match_m512i(__m512i v, unsigned long long a, unsigned long long b, unsigned long long c, unsigned long long d, unsigned long long e, unsigned long long f, unsigned long long g, unsigned long long h) {
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h;
 }
 
 #define TEST_CONSTEXPR(...) static_assert(__VA_ARGS__)
@@ -22,4 +46,4 @@ constexpr bool match_m128i(__m128i v, unsigned long long x, unsigned long long y
 
 #define TEST_CONSTEXPR(...)
 
-#endif
\ No newline at end of file
+#endif
-- 
GitLab


From e281d96a81bca896692da4a07ca1423ee6dc1f53 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 29 Oct 2024 13:49:50 +0000
Subject: [PATCH 440/693] [clang][x86] Add constexpr support for
 _mm_add_epi32/64 and _mm_sub_epi32/64

---
 clang/lib/Headers/emmintrin.h                 | 16 ++++++++--------
 clang/test/CodeGen/X86/builtin_test_helpers.h |  9 +++++++++
 clang/test/CodeGen/X86/sse2-builtins.c        |  4 ++++
 3 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/clang/lib/Headers/emmintrin.h b/clang/lib/Headers/emmintrin.h
index d6494762169b..778cdf99a129 100644
--- a/clang/lib/Headers/emmintrin.h
+++ b/clang/lib/Headers/emmintrin.h
@@ -2110,8 +2110,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi16(__m128i __a,
 ///    A 128-bit vector of [4 x i32].
 /// \returns A 128-bit vector of [4 x i32] containing the sums of both
 ///    parameters.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi32(__m128i __a,
-                                                           __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_add_epi32(__m128i __a, __m128i __b) {
   return (__m128i)((__v4su)__a + (__v4su)__b);
 }
 
@@ -2147,8 +2147,8 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_add_si64(__m64 __a, __m64 __b) {
 ///    A 128-bit vector of [2 x i64].
 /// \returns A 128-bit vector of [2 x i64] containing the sums of both
 ///    parameters.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi64(__m128i __a,
-                                                           __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_add_epi64(__m128i __a, __m128i __b) {
   return (__m128i)((__v2du)__a + (__v2du)__b);
 }
 
@@ -2539,8 +2539,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi16(__m128i __a,
 ///    A 128-bit integer vector containing the subtrahends.
 /// \returns A 128-bit integer vector containing the differences of the values
 ///    in the operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi32(__m128i __a,
-                                                           __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_sub_epi32(__m128i __a, __m128i __b) {
   return (__m128i)((__v4su)__a - (__v4su)__b);
 }
 
@@ -2573,8 +2573,8 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_sub_si64(__m64 __a, __m64 __b) {
 ///    A 128-bit integer vector containing the subtrahends.
 /// \returns A 128-bit integer vector containing the differences of the values
 ///    in the operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi64(__m128i __a,
-                                                           __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_sub_epi64(__m128i __a, __m128i __b) {
   return (__m128i)((__v2du)__a - (__v2du)__b);
 }
 
diff --git a/clang/test/CodeGen/X86/builtin_test_helpers.h b/clang/test/CodeGen/X86/builtin_test_helpers.h
index 5e77ff3a7ca4..f6547d4cb29c 100644
--- a/clang/test/CodeGen/X86/builtin_test_helpers.h
+++ b/clang/test/CodeGen/X86/builtin_test_helpers.h
@@ -16,6 +16,15 @@ constexpr bool match_m128i(__m128i v, unsigned long long a, unsigned long long b
   return v[0] == a && v[1] == b;
 }
 
+constexpr bool match_v2di(__m128i v, long long a, long long b) {
+  return v[0] == a && v[1] == b;
+}
+
+constexpr bool match_v4si(__m128i _v, int a, int b, int c, int d) {
+  __v4si v = (__v4si)_v;
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d;
+}
+
 constexpr bool match_m256(__m256 v, float a, float b, float c, float d, float e, float f, float g, float h) {
   return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h;
 }
diff --git a/clang/test/CodeGen/X86/sse2-builtins.c b/clang/test/CodeGen/X86/sse2-builtins.c
index 4287d3d4b5ec..82aa7a2d2b49 100644
--- a/clang/test/CodeGen/X86/sse2-builtins.c
+++ b/clang/test/CodeGen/X86/sse2-builtins.c
@@ -32,12 +32,14 @@ __m128i test_mm_add_epi32(__m128i A, __m128i B) {
   // CHECK: add <4 x i32>
   return _mm_add_epi32(A, B);
 }
+TEST_CONSTEXPR(match_v4si(_mm_add_epi32((__m128i)(__v4si){+1, -2, +3, -4}, (__m128i)(__v4si){-10, +8, +6, -4}), -9, +6, +9, -8));
 
 __m128i test_mm_add_epi64(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_add_epi64
   // CHECK: add <2 x i64>
   return _mm_add_epi64(A, B);
 }
+TEST_CONSTEXPR(match_v2di(_mm_add_epi64((__m128i)(__v2di){+5, -3}, (__m128i)(__v2di){-9, +8}), -4, +5));
 
 __m128d test_mm_add_pd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_add_pd
@@ -1634,12 +1636,14 @@ __m128i test_mm_sub_epi32(__m128i A, __m128i B) {
   // CHECK: sub <4 x i32>
   return _mm_sub_epi32(A, B);
 }
+TEST_CONSTEXPR(match_v4si(_mm_sub_epi32((__m128i)(__v4si){+1, -2, +3, -4}, (__m128i)(__v4si){-10, +8, +6, -4}), +11, -10, -3, 0));
 
 __m128i test_mm_sub_epi64(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_sub_epi64
   // CHECK: sub <2 x i64>
   return _mm_sub_epi64(A, B);
 }
+TEST_CONSTEXPR(match_v2di(_mm_sub_epi64((__m128i)(__v2di){+5, -3}, (__m128i)(__v2di){-9, +8}), +14, -11));
 
 __m128d test_mm_sub_pd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_sub_pd
-- 
GitLab


From 872981bd236530b160bf788aafd1cbde7b2bfb30 Mon Sep 17 00:00:00 2001
From: Dmitry Chernenkov <dmitryc@google.com>
Date: Tue, 29 Oct 2024 14:33:04 +0000
Subject: [PATCH 441/693] [Bazel] Fix layering for libc

---
 utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 1956233590f7..d4aeaea6fac8 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -215,6 +215,7 @@ libc_support_library(
 libc_support_library(
     name = "types_mode_t",
     hdrs = ["hdr/types/mode_t.h"],
+    deps = [":hdr_fcntl_overlay"],
 )
 
 libc_support_library(
-- 
GitLab


From b6a84e77b696b0d91b7cbed116d6454b6b1cc62b Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Tue, 29 Oct 2024 14:36:07 +0000
Subject: [PATCH 442/693] [AArch64] Add assembly/disassembly for FMOP4A
 (widening, 4-way) instructions (#113347)

The new instructions are described in
https://developer.arm.com/documentation/ddi0602/2024-09/SME-Instructions
---
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |   6 +
 llvm/lib/Target/AArch64/SMEInstrFormats.td    |  35 +++++
 .../fmop4a-fp8-fp32-widening-diagnostics.s    | 120 ++++++++++++++++++
 .../AArch64/SME2p2/fmop4a-fp8-fp32-widening.s |  93 ++++++++++++++
 4 files changed, 254 insertions(+)
 create mode 100644 llvm/test/MC/AArch64/SME2p2/fmop4a-fp8-fp32-widening-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SME2p2/fmop4a-fp8-fp32-widening.s

diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index ae40911cc62a..b71652942889 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -1041,3 +1041,9 @@ let Predicates = [HasSME2, HasSVEBFSCALE] in {
   defm BFMUL : sme2_bfmul_single<"bfmul">;
   defm BFMUL : sme2_bfmul_multi<"bfmul">;
 } //[HasSME2, HasSVEBFSCALE]
+
+let Uses = [FPMR, FPCR] in {
+let Predicates = [HasSME2p2, HasSMEF8F32] in {
+  defm FMOP4A : sme2_fmop4a_fp8_fp32_4way<"fmop4a">;
+}
+}
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index 330c540ffde4..e7c90b0ed14e 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -5382,3 +5382,38 @@ multiclass sme2_fmop4as_fp16_non_widening<bit S, string mnemonic> {
   // Multiple vectors
   def _M2Z2Z_H : sme2_fp16_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi>;
 }
+
+class sme2_fp8_fp32_quarter_tile_outer_product<bit M, bit N, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>
+    : I<(outs TileOp32:$ZAda),
+        (ins TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),
+        mnemonic, "\t$ZAda, $Zn, $Zm",
+        "", []>, Sched<[]> {
+  bits<2> ZAda;
+  bits<3> Zn;
+  bits<3> Zm;
+
+  let Inst{31-21} = 0b10000000001;
+  let Inst{20} = M;
+  let Inst{19-17} = Zm;
+  let Inst{16-10} = 0b0000000;
+  let Inst{9} = N;
+  let Inst{8-6} = Zn;
+  let Inst{5-2} = 0b0000;
+  let Inst{1-0} = ZAda;
+
+  let Constraints = "$ZAda = $_ZAda";
+}
+
+multiclass sme2_fmop4a_fp8_fp32_4way<string mnemonic> {
+  // Single vectors
+  def _MZZ_BtoS : sme2_fp8_fp32_quarter_tile_outer_product<0, 0, mnemonic, ZPR8Mul2_Lo, ZPR8Mul2_Hi>;
+
+  // Multiple and single vectors
+  def _M2ZZ_BtoS : sme2_fp8_fp32_quarter_tile_outer_product<0, 1, mnemonic, ZZ_b_mul_r_Lo, ZPR8Mul2_Hi>;
+
+  // Single and multiple vectors
+  def _MZ2Z_BtoS : sme2_fp8_fp32_quarter_tile_outer_product<1, 0, mnemonic, ZPR8Mul2_Lo, ZZ_b_mul_r_Hi>;
+
+  // Multiple vectors
+  def _M2Z2Z_BtoS : sme2_fp8_fp32_quarter_tile_outer_product<1, 1, mnemonic, ZZ_b_mul_r_Lo, ZZ_b_mul_r_Hi>;
+}
diff --git a/llvm/test/MC/AArch64/SME2p2/fmop4a-fp8-fp32-widening-diagnostics.s b/llvm/test/MC/AArch64/SME2p2/fmop4a-fp8-fp32-widening-diagnostics.s
new file mode 100644
index 000000000000..9a06192c0b30
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/fmop4a-fp8-fp32-widening-diagnostics.s
@@ -0,0 +1,120 @@
+// RUN: not llvm-mc -triple=aarch64 -mattr=+sme2p2,+sme-f8f32 < %s 2>&1 | FileCheck %s
+
+// Single vectors
+
+fmop4a za0.d, z0.b, z16.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za4.s, z0.b, z16.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, z0.d, z16.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.b..z14.b
+
+fmop4a za0.s, z15.b, z16.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.b..z14.b
+
+fmop4a za0.s, z16.b, z16.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.b..z14.b
+
+fmop4a za0.s, z0.b, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.b..z30.b
+
+fmop4a za0.s, z12.b, z17.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.b..z30.b
+
+fmop4a za0.s, z12.b, z14.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.b..z30.b
+
+fmop4a za0.s, z12.b, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.b..z30.b
+
+// Single and multiple vectors
+
+fmop4a za0.d, z0.b, {z16.b-z17.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za4.s, z0.b, {z16.b-z17.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, z0.d, {z16.b-z17.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.b..z14.b
+
+fmop4a za0.s, z1.b, {z16.b-z17.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.b..z14.b
+
+fmop4a za0.s, z16.b, {z16.b-z17.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.b..z14.b
+
+fmop4a za0.s, z0.b, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, z0.b, {z17.b-z18.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, z0.b, {z16.b-z18.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, z0.b, {z12.b-z13.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+// Multiple and single vectors
+
+fmop4a za0.d, {z0.b-z1.b}, z16.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za4.s, {z0.b-z1.b}, z16.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z0.s-z1.b}, z16.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: mismatched register size suffix
+
+fmop4a za0.s, {z1.b-z2.b}, z16.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, {z0.b-z2.b}, z16.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z16.b-z17.b}, z16.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, {z0.b-z1.b}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.b..z30.b
+
+fmop4a za0.s, {z0.b-z1.b}, z17.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.b..z30.b
+
+fmop4a za0.s, {z0.b-z1.b}, z12.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.b..z30.b
+
+// Multiple vectors
+
+fmop4a za0.d, {z0.b-z1.b}, {z16.b-z17.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za4.s, {z0.b-z1.b}, {z16.b-z17.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z0.s-z1.s}, {z16.b-z17.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z1.b-z2.b}, {z16.b-z17.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, {z0.b-z2.b}, {z16.b-z17.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z18.b-z19.b}, {z16.b-z17.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, {z0.b-z1.b}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z0.b-z1.b}, {z19.b-z20.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, {z0.b-z1.b}, {z18.b-z20.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z0.b-z1.b}, {z10.b-z11.b}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
diff --git a/llvm/test/MC/AArch64/SME2p2/fmop4a-fp8-fp32-widening.s b/llvm/test/MC/AArch64/SME2p2/fmop4a-fp8-fp32-widening.s
new file mode 100644
index 000000000000..9e378bcf3d75
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/fmop4a-fp8-fp32-widening.s
@@ -0,0 +1,93 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2,+sme-f8f32 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2,+sme-f8f32 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sme2p2,+sme-f8f32 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2,+sme-f8f32 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2p2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2,+sme-f8f32 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2p2,+sme-f8f32 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// Single vectors
+
+fmop4a  za0.s, z0.b, z16.b  // 10000000-00100000-00000000-00000000
+// CHECK-INST: fmop4a  za0.s, z0.b, z16.b
+// CHECK-ENCODING: [0x00,0x00,0x20,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f8f32
+// CHECK-UNKNOWN: 80200000 <unknown>
+
+fmop4a  za1.s, z10.b, z20.b  // 10000000-00100100-00000001-01000001
+// CHECK-INST: fmop4a  za1.s, z10.b, z20.b
+// CHECK-ENCODING: [0x41,0x01,0x24,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f8f32
+// CHECK-UNKNOWN: 80240141 <unknown>
+
+fmop4a  za3.s, z14.b, z30.b  // 10000000-00101110-00000001-11000011
+// CHECK-INST: fmop4a  za3.s, z14.b, z30.b
+// CHECK-ENCODING: [0xc3,0x01,0x2e,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f8f32
+// CHECK-UNKNOWN: 802e01c3 <unknown>
+
+// Single and multiple vectors
+
+fmop4a  za0.s, z0.b, {z16.b-z17.b}  // 10000000-00110000-00000000-00000000
+// CHECK-INST: fmop4a  za0.s, z0.b, { z16.b, z17.b }
+// CHECK-ENCODING: [0x00,0x00,0x30,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f8f32
+// CHECK-UNKNOWN: 80300000 <unknown>
+
+fmop4a  za1.s, z10.b, {z20.b-z21.b}  // 10000000-00110100-00000001-01000001
+// CHECK-INST: fmop4a  za1.s, z10.b, { z20.b, z21.b }
+// CHECK-ENCODING: [0x41,0x01,0x34,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f8f32
+// CHECK-UNKNOWN: 80340141 <unknown>
+
+fmop4a  za3.s, z14.b, {z30.b-z31.b}  // 10000000-00111110-00000001-11000011
+// CHECK-INST: fmop4a  za3.s, z14.b, { z30.b, z31.b }
+// CHECK-ENCODING: [0xc3,0x01,0x3e,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f8f32
+// CHECK-UNKNOWN: 803e01c3 <unknown>
+
+// Multiple and single vectors
+
+fmop4a  za0.s, {z0.b-z1.b}, z16.b  // 10000000-00100000-00000010-00000000
+// CHECK-INST: fmop4a  za0.s, { z0.b, z1.b }, z16.b
+// CHECK-ENCODING: [0x00,0x02,0x20,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f8f32
+// CHECK-UNKNOWN: 80200200 <unknown>
+
+fmop4a  za1.s, {z10.b-z11.b}, z20.b  // 10000000-00100100-00000011-01000001
+// CHECK-INST: fmop4a  za1.s, { z10.b, z11.b }, z20.b
+// CHECK-ENCODING: [0x41,0x03,0x24,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f8f32
+// CHECK-UNKNOWN: 80240341 <unknown>
+
+fmop4a  za3.s, {z14.b-z15.b}, z30.b  // 10000000-00101110-00000011-11000011
+// CHECK-INST: fmop4a  za3.s, { z14.b, z15.b }, z30.b
+// CHECK-ENCODING: [0xc3,0x03,0x2e,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f8f32
+// CHECK-UNKNOWN: 802e03c3 <unknown>
+
+// Multiple vectors
+
+fmop4a  za0.s, {z0.b-z1.b}, {z16.b-z17.b}  // 10000000-00110000-00000010-00000000
+// CHECK-INST: fmop4a  za0.s, { z0.b, z1.b }, { z16.b, z17.b }
+// CHECK-ENCODING: [0x00,0x02,0x30,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f8f32
+// CHECK-UNKNOWN: 80300200 <unknown>
+
+fmop4a  za1.s, {z10.b-z11.b}, {z20.b-z21.b}  // 10000000-00110100-00000011-01000001
+// CHECK-INST: fmop4a  za1.s, { z10.b, z11.b }, { z20.b, z21.b }
+// CHECK-ENCODING: [0x41,0x03,0x34,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f8f32
+// CHECK-UNKNOWN: 80340341 <unknown>
+
+fmop4a  za3.s, {z14.b-z15.b}, {z30.b-z31.b}  // 10000000-00111110-00000011-11000011
+// CHECK-INST: fmop4a  za3.s, { z14.b, z15.b }, { z30.b, z31.b }
+// CHECK-ENCODING: [0xc3,0x03,0x3e,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f8f32
+// CHECK-UNKNOWN: 803e03c3 <unknown>
-- 
GitLab


From a388df712700f38ad9a51d49a657a28e739f5eb4 Mon Sep 17 00:00:00 2001
From: Sebastian Kreutzer <SebastianKreutzer@gmx.net>
Date: Tue, 29 Oct 2024 15:40:53 +0100
Subject: [PATCH 443/693] [XRay] Remove reliance on default PIC behavior in DSO
 tests (#113892)

Compiling with `-fxray-shared` requires position-independent code
(introduced in #113548).
Some tests do not explicitly specify this, thus falling back to the
compiler default.
If, for example, Clang is compiled with
`-DCLANG_DEFAULT_PIE_ON_LINUX=OFF`, these checks fail.

This patch addresses this issue in two tests:
- Removing a check in `xray-shared.cpp` that only tests default PIC
behavior
- Adding `-fPIC` explicitly in `clang-xray-shared.cpp`
---
 clang/test/Driver/XRay/xray-shared.cpp                      | 1 -
 compiler-rt/test/xray/TestCases/Posix/clang-xray-shared.cpp | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/clang/test/Driver/XRay/xray-shared.cpp b/clang/test/Driver/XRay/xray-shared.cpp
index 215854e1fc7c..e331fefed1e0 100644
--- a/clang/test/Driver/XRay/xray-shared.cpp
+++ b/clang/test/Driver/XRay/xray-shared.cpp
@@ -1,6 +1,5 @@
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu -fPIC -fxray-instrument -fxray-shared -c %s -o /dev/null 2>&1 | FileCheck %s
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu -fpic -fxray-instrument -fxray-shared -c %s -o /dev/null 2>&1 | FileCheck %s
-// RUN: %clang -### --target=x86_64-unknown-linux-gnu -fxray-instrument -fxray-shared -c %s -o /dev/null 2>&1 | FileCheck %s
 // RUN: not %clang -### --target=x86_64-unknown-linux-gnu -fno-PIC -fxray-instrument -fxray-shared -c %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR-PIC
 // RUN: not %clang -### --target=x86_64-unknown-linux-gnu -fno-pic -fxray-instrument -fxray-shared -c %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR-PIC
 
diff --git a/compiler-rt/test/xray/TestCases/Posix/clang-xray-shared.cpp b/compiler-rt/test/xray/TestCases/Posix/clang-xray-shared.cpp
index 92f3c29e970d..0dd721571de9 100644
--- a/compiler-rt/test/xray/TestCases/Posix/clang-xray-shared.cpp
+++ b/compiler-rt/test/xray/TestCases/Posix/clang-xray-shared.cpp
@@ -1,6 +1,6 @@
 // Test that the DSO-local runtime library has been linked if -fxray-shared is passed.
 //
-// RUN: %clangxx -fxray-instrument -fxray-shared %s -shared -o %t.so
+// RUN: %clangxx -fxray-instrument -fxray-shared -fPIC %s -shared -o %t.so
 // RUN: llvm-nm %t.so | FileCheck %s --check-prefix ENABLED
 
 // RUN: %clangxx -fxray-instrument %s -shared -o %t.so
-- 
GitLab


From 2e612f8d868b3fb88a44964a3d4efd61ee63e06a Mon Sep 17 00:00:00 2001
From: goldsteinn <35538541+goldsteinn@users.noreply.github.com>
Date: Tue, 29 Oct 2024 07:41:59 -0700
Subject: [PATCH 444/693] [MLIR][Arith] Improve accuracy of `inferDivU`
 (#113789)

1) We can always bound the maximum with the numerator.
    - https://alive2.llvm.org/ce/z/PqHvuT
2) Even if denominator min can be zero, we can still bound the minimum
   result with `lhs.umin u/ rhs.umax`.

This is similar to https://github.com/llvm/llvm-project/pull/110169
---
 .../Interfaces/Utils/InferIntRangeCommon.cpp  | 10 +++++++--
 .../Dialect/Arith/int-range-interface.mlir    | 21 +++++++++++++++----
 2 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp b/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp
index ec9ed87723e1..a2acf3e732ad 100644
--- a/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp
+++ b/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp
@@ -298,8 +298,14 @@ static ConstantIntRanges inferDivURange(const ConstantIntRanges &lhs,
     return minMaxBy(udiv, {lhsMin, lhsMax}, {rhsMin, rhsMax},
                     /*isSigned=*/false);
   }
-  // Otherwise, it's possible we might divide by 0.
-  return ConstantIntRanges::maxRange(rhsMin.getBitWidth());
+
+  APInt umin = APInt::getZero(rhsMin.getBitWidth());
+  if (lhsMin.uge(rhsMax) && !rhsMax.isZero())
+    umin = lhsMin.udiv(rhsMax);
+
+  // X u/ Y u<= X.
+  APInt umax = lhsMax;
+  return ConstantIntRanges::fromUnsigned(umin, umax);
 }
 
 ConstantIntRanges
diff --git a/mlir/test/Dialect/Arith/int-range-interface.mlir b/mlir/test/Dialect/Arith/int-range-interface.mlir
index 4b04229e5db5..6d66da2fc1eb 100644
--- a/mlir/test/Dialect/Arith/int-range-interface.mlir
+++ b/mlir/test/Dialect/Arith/int-range-interface.mlir
@@ -178,8 +178,8 @@ func.func @div_bounds_negative(%arg0 : index) -> i1 {
 }
 
 // CHECK-LABEL: func @div_zero_undefined
-// CHECK: %[[ret:.*]] = arith.cmpi ule
-// CHECK: return %[[ret]]
+// CHECK: %[[true:.*]] = arith.constant true
+// CHECK: return %[[true]]
 func.func @div_zero_undefined(%arg0 : index) -> i1 {
     %c0 = arith.constant 0 : index
     %c1 = arith.constant 1 : index
@@ -190,6 +190,19 @@ func.func @div_zero_undefined(%arg0 : index) -> i1 {
     func.return %2 : i1
 }
 
+// CHECK-LABEL: func @div_refine_min
+// CHECK: %[[true:.*]] = arith.constant true
+// CHECK: return %[[true]]
+func.func @div_refine_min(%arg0 : index) -> i1 {
+    %c0 = arith.constant 1 : index
+    %c1 = arith.constant 2 : index
+    %c4 = arith.constant 4 : index
+    %0 = arith.andi %arg0, %c1 : index
+    %1 = arith.divui %c4, %0 : index
+    %2 = arith.cmpi uge, %1, %c0 : index
+    func.return %2 : i1
+}
+
 // CHECK-LABEL: func @ceil_divui
 // CHECK: %[[ret:.*]] = arith.cmpi eq
 // CHECK: return %[[ret]]
@@ -271,13 +284,13 @@ func.func @remui_base(%arg0 : index, %arg1 : index ) -> i1 {
 // CHECK: return %[[true]]
 func.func @remui_base_maybe_zero(%arg0 : index, %arg1 : index ) -> i1 {
     %c4 = arith.constant 4 : index
-    %c5 = arith.constant 5 : index    
+    %c5 = arith.constant 5 : index
 
     %0 = arith.minui %arg1, %c4 : index
     %1 = arith.remui %arg0, %0 : index
     %2 = arith.cmpi ult, %1, %c5 : index
     func.return %2 : i1
-}    
+}
 
 // CHECK-LABEL: func @remsi_base
 // CHECK: %[[ret:.*]] = arith.cmpi sge
-- 
GitLab


From 80a09735ac8bd6e31c824b41f7ee35952e440662 Mon Sep 17 00:00:00 2001
From: Elvina Yakubova <eyakubova@nvidia.com>
Date: Tue, 29 Oct 2024 14:43:01 +0000
Subject: [PATCH 445/693] =?UTF-8?q?Revert=20"[clang][AArch64]=20Add=20getH?=
 =?UTF-8?q?ostCPUFeatures=20to=20query=20for=20enabled=20=E2=80=A6=20(#114?=
 =?UTF-8?q?066)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…features in cpu info (#97749)"

This reverts commit d732c0b13c55259177f2936516b6087d634078e0.

This is breaking buildbots
https://lab.llvm.org/buildbot/#/builders/190/builds/8413,
https://lab.llvm.org/buildbot/#/builders/56/builds/10880 and a few
others.
---
 clang/lib/Driver/ToolChains/Arch/AArch64.cpp  |  20 +--
 clang/test/Driver/Inputs/cpunative/cortex-a57 |   8 -
 clang/test/Driver/Inputs/cpunative/cortex-a72 |   8 -
 clang/test/Driver/Inputs/cpunative/cortex-a76 |   8 -
 .../test/Driver/Inputs/cpunative/neoverse-n1  |   8 -
 .../test/Driver/Inputs/cpunative/neoverse-v2  |   8 -
 clang/test/Driver/aarch64-mcpu-native.c       | 138 ------------------
 llvm/lib/TargetParser/Host.cpp                |  10 +-
 8 files changed, 10 insertions(+), 198 deletions(-)
 delete mode 100644 clang/test/Driver/Inputs/cpunative/cortex-a57
 delete mode 100644 clang/test/Driver/Inputs/cpunative/cortex-a72
 delete mode 100644 clang/test/Driver/Inputs/cpunative/cortex-a76
 delete mode 100644 clang/test/Driver/Inputs/cpunative/neoverse-n1
 delete mode 100644 clang/test/Driver/Inputs/cpunative/neoverse-v2
 delete mode 100644 clang/test/Driver/aarch64-mcpu-native.c

diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
index 1e2ac4e501ba..f083e40df131 100644
--- a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
@@ -135,21 +135,15 @@ getAArch64ArchFeaturesFromMarch(const Driver &D, StringRef March,
   return true;
 }
 
-static bool getAArch64ArchFeaturesFromMcpu(
-    const Driver &D, StringRef Mcpu, const ArgList &Args,
-    llvm::AArch64::ExtensionSet &Extensions, std::vector<StringRef> &Features) {
+static bool
+getAArch64ArchFeaturesFromMcpu(const Driver &D, StringRef Mcpu,
+                               const ArgList &Args,
+                               llvm::AArch64::ExtensionSet &Extensions) {
   StringRef CPU;
   std::string McpuLowerCase = Mcpu.lower();
   if (!DecodeAArch64Mcpu(D, McpuLowerCase, CPU, Extensions))
     return false;
 
-  if (Mcpu == "native") {
-    llvm::StringMap<bool> HostFeatures = llvm::sys::getHostCPUFeatures();
-    for (auto &[Feature, Enabled] : HostFeatures) {
-      Features.push_back(Args.MakeArgString((Enabled ? "+" : "-") + Feature));
-    }
-  }
-
   return true;
 }
 
@@ -216,11 +210,11 @@ void aarch64::getAArch64TargetFeatures(const Driver &D,
     success =
         getAArch64ArchFeaturesFromMarch(D, A->getValue(), Args, Extensions);
   else if ((A = Args.getLastArg(options::OPT_mcpu_EQ)))
-    success = getAArch64ArchFeaturesFromMcpu(D, A->getValue(), Args, Extensions,
-                                             Features);
+    success =
+        getAArch64ArchFeaturesFromMcpu(D, A->getValue(), Args, Extensions);
   else if (isCPUDeterminedByTriple(Triple))
     success = getAArch64ArchFeaturesFromMcpu(
-        D, getAArch64TargetCPU(Args, Triple, A), Args, Extensions, Features);
+        D, getAArch64TargetCPU(Args, Triple, A), Args, Extensions);
   else
     // Default to 'A' profile if the architecture is not specified.
     success = getAArch64ArchFeaturesFromMarch(D, "armv8-a", Args, Extensions);
diff --git a/clang/test/Driver/Inputs/cpunative/cortex-a57 b/clang/test/Driver/Inputs/cpunative/cortex-a57
deleted file mode 100644
index e1903012ab79..000000000000
--- a/clang/test/Driver/Inputs/cpunative/cortex-a57
+++ /dev/null
@@ -1,8 +0,0 @@
-processor       : 0
-BogoMIPS        : 200.00
-Features        : fp asimd evtstrm crc32 cpuid
-CPU implementer : 0x41
-CPU architecture: 8
-CPU variant     : 0x1
-CPU part        : 0xd07
-CPU revision    : 1
diff --git a/clang/test/Driver/Inputs/cpunative/cortex-a72 b/clang/test/Driver/Inputs/cpunative/cortex-a72
deleted file mode 100644
index 7aed4a6fa732..000000000000
--- a/clang/test/Driver/Inputs/cpunative/cortex-a72
+++ /dev/null
@@ -1,8 +0,0 @@
-processor       : 0
-BogoMIPS        : 250.00
-Features        : fp asimd evtstrm aes pmull sha1 sha2 crc32 cpuid asimdrdm
-CPU implementer : 0x41
-CPU architecture: 8
-CPU variant     : 0x0
-CPU part        : 0xd08
-CPU revision    : 2
diff --git a/clang/test/Driver/Inputs/cpunative/cortex-a76 b/clang/test/Driver/Inputs/cpunative/cortex-a76
deleted file mode 100644
index 21822cfcec60..000000000000
--- a/clang/test/Driver/Inputs/cpunative/cortex-a76
+++ /dev/null
@@ -1,8 +0,0 @@
-processor       : 0
-BogoMIPS        : 500.00
-Features        : fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics cpuid asimdrdm ssbs jscvt fcma
-CPU implementer : 0x41
-CPU architecture: 8
-CPU variant     : 0x1
-CPU part        : 0xd0b
-CPU revision    : 2
diff --git a/clang/test/Driver/Inputs/cpunative/neoverse-n1 b/clang/test/Driver/Inputs/cpunative/neoverse-n1
deleted file mode 100644
index 571e8840b09f..000000000000
--- a/clang/test/Driver/Inputs/cpunative/neoverse-n1
+++ /dev/null
@@ -1,8 +0,0 @@
-processor       : 0
-BogoMIPS        : 50.00
-Features        : fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm lrcpc dcpop asimddp ssbs
-CPU implementer : 0x41
-CPU architecture: 8
-CPU variant     : 0x3
-CPU part        : 0xd0c
-CPU revision    : 1
diff --git a/clang/test/Driver/Inputs/cpunative/neoverse-v2 b/clang/test/Driver/Inputs/cpunative/neoverse-v2
deleted file mode 100644
index c3c8433415d7..000000000000
--- a/clang/test/Driver/Inputs/cpunative/neoverse-v2
+++ /dev/null
@@ -1,8 +0,0 @@
-processor       : 0
-BogoMIPS        : 2000.00
-Features        : fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm jscvt fcma lrcpc dcpop sha3 sm3 sm4 asimddp sha512 sve asimdfhm dit uscat ilrcpc flagm ssbs sb paca pacg dcpodp sve2 sveaes svepmull svebitperm svesha3 svesm4 flagm2 frint svei8mm svebf16 i8mm bf16 dgh bti
-CPU implementer : 0x41
-CPU architecture: 8
-CPU variant     : 0x0
-CPU part        : 0xd4f
-CPU revision    : 0
diff --git a/clang/test/Driver/aarch64-mcpu-native.c b/clang/test/Driver/aarch64-mcpu-native.c
deleted file mode 100644
index f1d0ba76ad79..000000000000
--- a/clang/test/Driver/aarch64-mcpu-native.c
+++ /dev/null
@@ -1,138 +0,0 @@
-// REQUIRES: aarch64-registered-target
-// RUN: export LLVM_CPUINFO=%S/Inputs/cpunative/neoverse-v2
-// RUN: %clang --target=aarch64 --print-enabled-extensions -mcpu=native | FileCheck --strict-whitespace --check-prefix=CHECK-FEAT-NV2 --implicit-check-not=FEAT_ %s
-
-// CHECK-FEAT-NV2: Extensions enabled for the given AArch64 target
-// CHECK-FEAT-NV2-EMPTY:
-// CHECK-FEAT-NV2:    Architecture Feature(s)                                Description
-// CHECK-FEAT-NV2:    FEAT_AES, FEAT_PMULL                                   Enable AES support
-// CHECK-FEAT-NV2:    FEAT_AMUv1                                             Enable Armv8.4-A Activity Monitors extension
-// CHECK-FEAT-NV2:    FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
-// CHECK-FEAT-NV2:    FEAT_BF16                                              Enable BFloat16 Extension
-// CHECK-FEAT-NV2:    FEAT_BTI                                               Enable Branch Target Identification
-// CHECK-FEAT-NV2:    FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
-// CHECK-FEAT-NV2:    FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
-// CHECK-FEAT-NV2:    FEAT_CSV2_2                                            Enable architectural speculation restriction
-// CHECK-FEAT-NV2:    FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
-// CHECK-FEAT-NV2:    FEAT_DPB                                               Enable Armv8.2-A data Cache Clean to Point of Persistence
-// CHECK-FEAT-NV2:    FEAT_DPB2                                              Enable Armv8.5-A Cache Clean to Point of Deep Persistence
-// CHECK-FEAT-NV2:    FEAT_DotProd                                           Enable dot product support
-// CHECK-FEAT-NV2:    FEAT_ETE                                               Enable Embedded Trace Extension
-// CHECK-FEAT-NV2:    FEAT_FCMA                                              Enable Armv8.3-A Floating-point complex number support
-// CHECK-FEAT-NV2:    FEAT_FHM                                               Enable FP16 FML instructions
-// CHECK-FEAT-NV2:    FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
-// CHECK-FEAT-NV2:    FEAT_FP16                                              Enable half-precision floating-point data processing
-// CHECK-FEAT-NV2:    FEAT_FRINTTS                                           Enable FRInt[32|64][Z|X] instructions that round a floating-point number to an integer (in FP format) forcing it to fit into a 32- or 64-bit int
-// CHECK-FEAT-NV2:    FEAT_FlagM                                             Enable Armv8.4-A Flag Manipulation instructions
-// CHECK-FEAT-NV2:    FEAT_FlagM2                                            Enable alternative NZCV format for floating point comparisons
-// CHECK-FEAT-NV2:    FEAT_I8MM                                              Enable Matrix Multiply Int8 Extension
-// CHECK-FEAT-NV2:    FEAT_JSCVT                                             Enable Armv8.3-A JavaScript FP conversion instructions
-// CHECK-FEAT-NV2:    FEAT_LOR                                               Enable Armv8.1-A Limited Ordering Regions extension
-// CHECK-FEAT-NV2:    FEAT_LRCPC                                             Enable support for RCPC extension
-// CHECK-FEAT-NV2:    FEAT_LRCPC2                                            Enable Armv8.4-A RCPC instructions with Immediate Offsets
-// CHECK-FEAT-NV2:    FEAT_LSE                                               Enable Armv8.1-A Large System Extension (LSE) atomic instructions
-// CHECK-FEAT-NV2:    FEAT_LSE2                                              Enable Armv8.4-A Large System Extension 2 (LSE2) atomicity rules
-// CHECK-FEAT-NV2:    FEAT_MPAM                                              Enable Armv8.4-A Memory system Partitioning and Monitoring extension
-// CHECK-FEAT-NV2:    FEAT_MTE, FEAT_MTE2                                    Enable Memory Tagging Extension
-// CHECK-FEAT-NV2:    FEAT_NV, FEAT_NV2                                      Enable Armv8.4-A Nested Virtualization Enchancement
-// CHECK-FEAT-NV2:    FEAT_PAN                                               Enable Armv8.1-A Privileged Access-Never extension
-// CHECK-FEAT-NV2:    FEAT_PAN2                                              Enable Armv8.2-A PAN s1e1R and s1e1W Variants
-// CHECK-FEAT-NV2:    FEAT_PAuth                                             Enable Armv8.3-A Pointer Authentication extension
-// CHECK-FEAT-NV2:    FEAT_PMUv3                                             Enable Armv8.0-A PMUv3 Performance Monitors extension
-// CHECK-FEAT-NV2:    FEAT_RAS, FEAT_RASv1p1                                 Enable Armv8.0-A Reliability, Availability and Serviceability Extensions
-// CHECK-FEAT-NV2:    FEAT_RDM                                               Enable Armv8.1-A Rounding Double Multiply Add/Subtract instructions
-// CHECK-FEAT-NV2:    FEAT_RNG                                               Enable Random Number generation instructions
-// CHECK-FEAT-NV2:    FEAT_SB                                                Enable Armv8.5-A Speculation Barrier
-// CHECK-FEAT-NV2:    FEAT_SEL2                                              Enable Armv8.4-A Secure Exception Level 2 extension
-// CHECK-FEAT-NV2:    FEAT_SHA1, FEAT_SHA256                                 Enable SHA1 and SHA256 support
-// CHECK-FEAT-NV2:    FEAT_SPE                                               Enable Statistical Profiling extension
-// CHECK-FEAT-NV2:    FEAT_SPECRES                                           Enable Armv8.5-A execution and data prediction invalidation instructions
-// CHECK-FEAT-NV2:    FEAT_SSBS, FEAT_SSBS2                                  Enable Speculative Store Bypass Safe bit
-// CHECK-FEAT-NV2:    FEAT_SVE                                               Enable Scalable Vector Extension (SVE) instructions
-// CHECK-FEAT-NV2:    FEAT_SVE2                                              Enable Scalable Vector Extension 2 (SVE2) instructions
-// CHECK-FEAT-NV2:    FEAT_SVE_BitPerm                                       Enable bit permutation SVE2 instructions
-// CHECK-FEAT-NV2:    FEAT_TLBIOS, FEAT_TLBIRANGE                            Enable Armv8.4-A TLB Range and Maintenance instructions
-// CHECK-FEAT-NV2:    FEAT_TRBE                                              Enable Trace Buffer Extension
-// CHECK-FEAT-NV2:    FEAT_TRF                                               Enable Armv8.4-A Trace extension
-// CHECK-FEAT-NV2:    FEAT_UAO                                               Enable Armv8.2-A UAO PState
-// CHECK-FEAT-NV2:    FEAT_VHE                                               Enable Armv8.1-A Virtual Host extension
-
-// RUN: export LLVM_CPUINFO=%S/Inputs/cpunative/neoverse-n1
-// RUN: %clang --target=aarch64 --print-enabled-extensions -mcpu=native | FileCheck --strict-whitespace --check-prefix=CHECK-FEAT-NN1 --implicit-check-not=FEAT_ %s
-
-// CHECK-FEAT-NN1: Extensions enabled for the given AArch64 target
-// CHECK-FEAT-NN1-EMPTY:
-// CHECK-FEAT-NN1:    Architecture Feature(s)                                Description
-// CHECK-FEAT-NN1:    FEAT_AES, FEAT_PMULL                                   Enable AES support
-// CHECK-FEAT-NN1:    FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
-// CHECK-FEAT-NN1:    FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
-// CHECK-FEAT-NN1:    FEAT_DPB                                               Enable Armv8.2-A data Cache Clean to Point of Persistence
-// CHECK-FEAT-NN1:    FEAT_DotProd                                           Enable dot product support
-// CHECK-FEAT-NN1:    FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
-// CHECK-FEAT-NN1:    FEAT_FP16                                              Enable half-precision floating-point data processing
-// CHECK-FEAT-NN1:    FEAT_LOR                                               Enable Armv8.1-A Limited Ordering Regions extension
-// CHECK-FEAT-NN1:    FEAT_LRCPC                                             Enable support for RCPC extension
-// CHECK-FEAT-NN1:    FEAT_LSE                                               Enable Armv8.1-A Large System Extension (LSE) atomic instructions
-// CHECK-FEAT-NN1:    FEAT_PAN                                               Enable Armv8.1-A Privileged Access-Never extension
-// CHECK-FEAT-NN1:    FEAT_PAN2                                              Enable Armv8.2-A PAN s1e1R and s1e1W Variants
-// CHECK-FEAT-NN1:    FEAT_PMUv3                                             Enable Armv8.0-A PMUv3 Performance Monitors extension
-// CHECK-FEAT-NN1:    FEAT_RAS, FEAT_RASv1p1                                 Enable Armv8.0-A Reliability, Availability and Serviceability Extensions
-// CHECK-FEAT-NN1:    FEAT_RDM                                               Enable Armv8.1-A Rounding Double Multiply Add/Subtract instructions
-// CHECK-FEAT-NN1:    FEAT_SHA1, FEAT_SHA256                                 Enable SHA1 and SHA256 support
-// CHECK-FEAT-NN1:    FEAT_SPE                                               Enable Statistical Profiling extension
-// CHECK-FEAT-NN1:    FEAT_SSBS, FEAT_SSBS2                                  Enable Speculative Store Bypass Safe bit
-// CHECK-FEAT-NN1:    FEAT_UAO                                               Enable Armv8.2-A UAO PState
-// CHECK-FEAT-NN1:    FEAT_VHE                                               Enable Armv8.1-A Virtual Host extension
-
-
-// RUN: export LLVM_CPUINFO=%S/Inputs/cpunative/cortex-a57
-// RUN: %clang --target=aarch64 --print-enabled-extensions -mcpu=native | FileCheck --strict-whitespace --check-prefix=CHECK-FEAT-CA57 --implicit-check-not=FEAT_ %s
-
-// CHECK-FEAT-CA57: Extensions enabled for the given AArch64 target
-// CHECK-FEAT-CA57-EMPTY:
-// CHECK-FEAT-CA57:    Architecture Feature(s)                                Description
-// CHECK-FEAT-CA57:    FEAT_AES, FEAT_PMULL                                   Enable AES support
-// CHECK-FEAT-CA57:    FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
-// CHECK-FEAT-CA57:    FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
-// CHECK-FEAT-CA57:    FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
-// CHECK-FEAT-CA57:    FEAT_PMUv3                                             Enable Armv8.0-A PMUv3 Performance Monitors extension
-// CHECK-FEAT-CA57:    FEAT_SHA1, FEAT_SHA256                                 Enable SHA1 and SHA256 support
-
-// RUN: export LLVM_CPUINFO=%S/Inputs/cpunative/cortex-a72
-// RUN: %clang --target=aarch64 --print-enabled-extensions -mcpu=native | FileCheck --strict-whitespace  --check-prefix=CHECK-FEAT-CA72 --implicit-check-not=FEAT_ %s
-
-// CHECK-FEAT-CA72: Extensions enabled for the given AArch64 target
-// CHECK-EMPTY:
-// CHECK-FEAT-CA72:   Architecture Feature(s)                                Description
-// CHECK-FEAT-CA72:    FEAT_AES, FEAT_PMULL                                   Enable AES support
-// CHECK-FEAT-CA72:    FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
-// CHECK-FEAT-CA72:    FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
-// CHECK-FEAT-CA72:    FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
-// CHECK-FEAT-CA72:    FEAT_PMUv3                                             Enable Armv8.0-A PMUv3 Performance Monitors extension
-// CHECK-FEAT-CA72:    FEAT_SHA1, FEAT_SHA256                                 Enable SHA1 and SHA256 support
-
-// RUN: export LLVM_CPUINFO=%S/Inputs/cpunative/cortex-a76
-// RUN: %clang --target=aarch64 --print-enabled-extensions -mcpu=native | FileCheck --strict-whitespace --check-prefix=CHECK-FEAT-CA76 --implicit-check-not=FEAT_ %s
-
-// CHECK-FEAT-CA76: Extensions enabled for the given AArch64 target
-// CHECK-FEAT-CA76-EMPTY:
-// CHECK-FEAT-CA76:    Architecture Feature(s)                                Description
-// CHECK-FEAT-CA76:    FEAT_AES, FEAT_PMULL                                   Enable AES support
-// CHECK-FEAT-CA76:    FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
-// CHECK-FEAT-CA76:    FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
-// CHECK-FEAT-CA76:    FEAT_DPB                                               Enable Armv8.2-A data Cache Clean to Point of Persistence
-// CHECK-FEAT-CA76:    FEAT_DotProd                                           Enable dot product support
-// CHECK-FEAT-CA76:    FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
-// CHECK-FEAT-CA76:    FEAT_FP16                                              Enable half-precision floating-point data processing
-// CHECK-FEAT-CA76:    FEAT_LOR                                               Enable Armv8.1-A Limited Ordering Regions extension
-// CHECK-FEAT-CA76:    FEAT_LRCPC                                             Enable support for RCPC extension
-// CHECK-FEAT-CA76:    FEAT_LSE                                               Enable Armv8.1-A Large System Extension (LSE) atomic instructions
-// CHECK-FEAT-CA76:    FEAT_PAN                                               Enable Armv8.1-A Privileged Access-Never extension
-// CHECK-FEAT-CA76:    FEAT_PAN2                                              Enable Armv8.2-A PAN s1e1R and s1e1W Variants
-// CHECK-FEAT-CA76:    FEAT_PMUv3                                             Enable Armv8.0-A PMUv3 Performance Monitors extension
-// CHECK-FEAT-CA76:    FEAT_RAS, FEAT_RASv1p1                                 Enable Armv8.0-A Reliability, Availability and Serviceability Extensions
-// CHECK-FEAT-CA76:    FEAT_RDM                                               Enable Armv8.1-A Rounding Double Multiply Add/Subtract instructions
-// CHECK-FEAT-CA76:    FEAT_SHA1, FEAT_SHA256                                 Enable SHA1 and SHA256 support
-// CHECK-FEAT-CA76:    FEAT_SSBS, FEAT_SSBS2                                  Enable Speculative Store Bypass Safe bit
-// CHECK-FEAT-CA76:    FEAT_UAO                                               Enable Armv8.2-A UAO PState
-// CHECK-FEAT-CA76:    FEAT_VHE                                               Enable Armv8.1-A Virtual Host extension
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index de6c4edebba3..5c4e3a9dc52b 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -68,15 +68,11 @@ using namespace llvm;
 
 static std::unique_ptr<llvm::MemoryBuffer>
     LLVM_ATTRIBUTE_UNUSED getProcCpuinfoContent() {
-  const char *CPUInfoFile = "/proc/cpuinfo";
-  if (const char *CpuinfoIntercept = std::getenv("LLVM_CPUINFO"))
-    CPUInfoFile = CpuinfoIntercept;
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Text =
-      llvm::MemoryBuffer::getFileAsStream(CPUInfoFile);
-
+      llvm::MemoryBuffer::getFileAsStream("/proc/cpuinfo");
   if (std::error_code EC = Text.getError()) {
-    llvm::errs() << "Can't read " << CPUInfoFile << ": " << EC.message()
-                 << "\n";
+    llvm::errs() << "Can't read "
+                 << "/proc/cpuinfo: " << EC.message() << "\n";
     return nullptr;
   }
   return std::move(*Text);
-- 
GitLab


From c9d9dc9c24039d85fdf3036368c9fba3d68722fa Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Tue, 29 Oct 2024 10:48:18 -0400
Subject: [PATCH 446/693] [libc++] Remove _LIBCPP_ENABLE_ASSERTIONS, which had
 been deprecated (#113592)

---
 libcxx/CMakeLists.txt                         |  8 ++---
 libcxx/docs/ReleaseNotes/20.rst               | 11 ++----
 libcxx/include/__config                       | 11 ++----
 ...assertions_enables_extensive_mode.pass.cpp | 35 -------------------
 4 files changed, 7 insertions(+), 58 deletions(-)
 delete mode 100644 libcxx/test/libcxx/assertions/modes/enabling_assertions_enables_extensive_mode.pass.cpp

diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index 574b262018cd..95a7d10f055e 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -45,10 +45,6 @@ include(CMakeDependentOption)
 include(HandleCompilerRT)
 
 # Basic options ---------------------------------------------------------------
-option(LIBCXX_ENABLE_ASSERTIONS
-  "Enable assertions inside the compiled library, and at the same time make it the
-   default when compiling user code. Note that assertions can be enabled or disabled
-   by users in their own code regardless of this option." OFF)
 option(LIBCXX_ENABLE_SHARED "Build libc++ as a shared library." ON)
 option(LIBCXX_ENABLE_STATIC "Build libc++ as a static library." ON)
 option(LIBCXX_ENABLE_FILESYSTEM
@@ -759,9 +755,9 @@ config_define_if_not(LIBCXX_ENABLE_WIDE_CHARACTERS _LIBCPP_HAS_NO_WIDE_CHARACTER
 config_define_if_not(LIBCXX_ENABLE_TIME_ZONE_DATABASE _LIBCPP_HAS_NO_TIME_ZONE_DATABASE)
 config_define_if_not(LIBCXX_ENABLE_VENDOR_AVAILABILITY_ANNOTATIONS _LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS)
 
+# TODO: Remove in LLVM 21. We're leaving an error to make this fail explicitly.
 if (LIBCXX_ENABLE_ASSERTIONS)
-  message(DEPRECATION "LIBCXX_ENABLE_ASSERTIONS is deprecated and will be removed in LLVM 20. Please use LIBCXX_HARDENING_MODE instead.")
-  set(LIBCXX_HARDENING_MODE "extensive")
+  message(FATAL_ERROR "LIBCXX_ENABLE_ASSERTIONS has been removed. Please use LIBCXX_HARDENING_MODE instead.")
 endif()
 if (LIBCXX_HARDENING_MODE STREQUAL "none")
   config_define(2 _LIBCPP_HARDENING_MODE_DEFAULT)
diff --git a/libcxx/docs/ReleaseNotes/20.rst b/libcxx/docs/ReleaseNotes/20.rst
index 84080e7cbafe..38b8df3b2a77 100644
--- a/libcxx/docs/ReleaseNotes/20.rst
+++ b/libcxx/docs/ReleaseNotes/20.rst
@@ -64,8 +64,9 @@ Improvements and New Features
 Deprecations and Removals
 -------------------------
 
-- TODO: The ``LIBCXX_ENABLE_ASSERTIONS`` CMake variable and the ``_LIBCPP_ENABLE_ASSERTIONS`` macro that were used to
-  enable the safe mode will be removed in LLVM 20.
+- The ``LIBCXX_ENABLE_ASSERTIONS`` CMake variable and the ``_LIBCPP_ENABLE_ASSERTIONS`` macro that were used to
+  enable the safe mode have been removed in LLVM 20. Please use :ref:`support for hardening <using-hardening-modes>`
+  instead.
 
 - Support for the C++20 synchronization library (``<barrier>``, ``<latch>``, ``atomic::wait``, etc.) has been
   removed in language modes prior to C++20. If you are using these features prior to C++20, you will need to
@@ -91,12 +92,6 @@ Deprecations and Removals
 Upcoming Deprecations and Removals
 ----------------------------------
 
-LLVM 20
-~~~~~~~
-
-- TODO
-
-
 LLVM 21
 ~~~~~~~
 
diff --git a/libcxx/include/__config b/libcxx/include/__config
index fc09a97274d7..1cf80a46686a 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -39,16 +39,9 @@
 
 // HARDENING {
 
-// This is for backward compatibility -- make enabling `_LIBCPP_ENABLE_ASSERTIONS` (which predates hardening modes)
-// equivalent to setting the extensive mode. This is deprecated and will be removed in LLVM 20.
+// TODO: Remove in LLVM 21. We're making this an error to catch folks who might not have migrated.
 #  ifdef _LIBCPP_ENABLE_ASSERTIONS
-#    warning "_LIBCPP_ENABLE_ASSERTIONS is deprecated, please use _LIBCPP_HARDENING_MODE instead"
-#    if _LIBCPP_ENABLE_ASSERTIONS != 0 && _LIBCPP_ENABLE_ASSERTIONS != 1
-#      error "_LIBCPP_ENABLE_ASSERTIONS must be set to 0 or 1"
-#    endif
-#    if _LIBCPP_ENABLE_ASSERTIONS
-#      define _LIBCPP_HARDENING_MODE _LIBCPP_HARDENING_MODE_EXTENSIVE
-#    endif
+#    error "_LIBCPP_ENABLE_ASSERTIONS has been removed, please use _LIBCPP_HARDENING_MODE instead"
 #  endif
 
 // The library provides the macro `_LIBCPP_HARDENING_MODE` which can be set to one of the following values:
diff --git a/libcxx/test/libcxx/assertions/modes/enabling_assertions_enables_extensive_mode.pass.cpp b/libcxx/test/libcxx/assertions/modes/enabling_assertions_enables_extensive_mode.pass.cpp
deleted file mode 100644
index c496fc32dc93..000000000000
--- a/libcxx/test/libcxx/assertions/modes/enabling_assertions_enables_extensive_mode.pass.cpp
+++ /dev/null
@@ -1,35 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// TODO(hardening): remove in LLVM 20.
-// This test ensures that enabling assertions with the legacy `_LIBCPP_ENABLE_ASSERTIONS` now enables the extensive
-// hardening mode.
-
-// `check_assertion.h` is only available starting from C++11 and requires Unix headers and regex support.
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03, no-localization
-// The ability to set a custom abort message is required to compare the assertion message (which only happens in the
-// debug mode).
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-// HWASAN replaces TRAP with abort or error exit code.
-// XFAIL: hwasan
-// Note that GCC doesn't support `-Wno-macro-redefined`.
-// ADDITIONAL_COMPILE_FLAGS: -U_LIBCPP_HARDENING_MODE -D_LIBCPP_ENABLE_ASSERTIONS=1 -Wno-#warnings -Wno-cpp
-
-#include <cassert>
-#include "check_assertion.h"
-
-int main(int, char**) {
-  static_assert(_LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_EXTENSIVE,
-                "The extensive hardening mode should be implicitly enabled");
-
-  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(true, "Should not fire");
-  TEST_LIBCPP_ASSERT_FAILURE([] { _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(false, "Should fire"); }(), "Should fire");
-
-  return 0;
-}
-- 
GitLab


From e268398fa89c9cc7901ea9b7386fc693023be203 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Tue, 29 Oct 2024 10:50:06 -0400
Subject: [PATCH 447/693] [NFC][AMDGPU] Use `!foreach` to replace explicit list
 of registers (#114005)

---
 llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td | 126 +++++---------------
 1 file changed, 29 insertions(+), 97 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td b/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td
index 21412044d5a0..80969fce3d77 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td
@@ -22,19 +22,13 @@ def CC_SI_Gfx : CallingConv<[
   // 32 is reserved for the stack pointer
   // 33 is reserved for the frame pointer
   // 34 is reserved for the base pointer
-  CCIfInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<[
-    SGPR4, SGPR5, SGPR6, SGPR7,
-    SGPR8, SGPR9, SGPR10, SGPR11, SGPR12, SGPR13, SGPR14, SGPR15,
-    SGPR16, SGPR17, SGPR18, SGPR19, SGPR20, SGPR21, SGPR22, SGPR23,
-    SGPR24, SGPR25, SGPR26, SGPR27, SGPR28, SGPR29
-  ]>>>,
-
-  CCIfNotInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<[
-    VGPR0, VGPR1, VGPR2, VGPR3, VGPR4, VGPR5, VGPR6, VGPR7,
-    VGPR8, VGPR9, VGPR10, VGPR11, VGPR12, VGPR13, VGPR14, VGPR15,
-    VGPR16, VGPR17, VGPR18, VGPR19, VGPR20, VGPR21, VGPR22, VGPR23,
-    VGPR24, VGPR25, VGPR26, VGPR27, VGPR28, VGPR29, VGPR30, VGPR31
-  ]>>>,
+  CCIfInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<
+    !foreach(i, !range(4, 30), !cast<Register>("SGPR"#i))  // SGPR4-29
+  >>>,
+
+  CCIfNotInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<
+    !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i))  // VGPR0-31
+  >>>,
 
   CCIfType<[i32, f32, v2i16, v2f16, i16, f16, i1, bf16, v2bf16], CCAssignToStack<4, 4>>
 ]>;
@@ -43,93 +37,35 @@ def RetCC_SI_Gfx : CallingConv<[
   CCIfType<[i1], CCPromoteToType<i32>>,
   CCIfType<[i1, i16], CCIfExtend<CCPromoteToType<i32>>>,
 
-  CCIfNotInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<[
-    VGPR0, VGPR1, VGPR2, VGPR3, VGPR4, VGPR5, VGPR6, VGPR7,
-    VGPR8, VGPR9, VGPR10, VGPR11, VGPR12, VGPR13, VGPR14, VGPR15,
-    VGPR16, VGPR17, VGPR18, VGPR19, VGPR20, VGPR21, VGPR22, VGPR23,
-    VGPR24, VGPR25, VGPR26, VGPR27, VGPR28, VGPR29, VGPR30, VGPR31,
-    VGPR32, VGPR33, VGPR34, VGPR35, VGPR36, VGPR37, VGPR38, VGPR39,
-    VGPR40, VGPR41, VGPR42, VGPR43, VGPR44, VGPR45, VGPR46, VGPR47,
-    VGPR48, VGPR49, VGPR50, VGPR51, VGPR52, VGPR53, VGPR54, VGPR55,
-    VGPR56, VGPR57, VGPR58, VGPR59, VGPR60, VGPR61, VGPR62, VGPR63,
-    VGPR64, VGPR65, VGPR66, VGPR67, VGPR68, VGPR69, VGPR70, VGPR71,
-    VGPR72, VGPR73, VGPR74, VGPR75, VGPR76, VGPR77, VGPR78, VGPR79,
-    VGPR80, VGPR81, VGPR82, VGPR83, VGPR84, VGPR85, VGPR86, VGPR87,
-    VGPR88, VGPR89, VGPR90, VGPR91, VGPR92, VGPR93, VGPR94, VGPR95,
-    VGPR96, VGPR97, VGPR98, VGPR99, VGPR100, VGPR101, VGPR102, VGPR103,
-    VGPR104, VGPR105, VGPR106, VGPR107, VGPR108, VGPR109, VGPR110, VGPR111,
-    VGPR112, VGPR113, VGPR114, VGPR115, VGPR116, VGPR117, VGPR118, VGPR119,
-    VGPR120, VGPR121, VGPR122, VGPR123, VGPR124, VGPR125, VGPR126, VGPR127,
-    VGPR128, VGPR129, VGPR130, VGPR131, VGPR132, VGPR133, VGPR134, VGPR135
-  ]>>>,
+  CCIfNotInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<
+    !foreach(i, !range(0, 136), !cast<Register>("VGPR"#i))  // VGPR0-135
+  >>>,
 ]>;
 
 def CC_SI_SHADER : CallingConv<[
 
   CCIfType<[i1], CCPromoteToType<i32>>,
-  
-  CCIfInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<[
-    SGPR0, SGPR1, SGPR2, SGPR3, SGPR4, SGPR5, SGPR6, SGPR7,
-    SGPR8, SGPR9, SGPR10, SGPR11, SGPR12, SGPR13, SGPR14, SGPR15,
-    SGPR16, SGPR17, SGPR18, SGPR19, SGPR20, SGPR21, SGPR22, SGPR23,
-    SGPR24, SGPR25, SGPR26, SGPR27, SGPR28, SGPR29, SGPR30, SGPR31,
-    SGPR32, SGPR33, SGPR34, SGPR35, SGPR36, SGPR37, SGPR38, SGPR39,
-    SGPR40, SGPR41, SGPR42, SGPR43
-  ]>>>,
+
+  CCIfInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<
+    !foreach(i, !range(0, 44), !cast<Register>("SGPR"#i))  // SGPR0-43
+  >>>,
 
   // 32*4 + 4 is the minimum for a fetch shader consumer with 32 inputs.
-  CCIfNotInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<[
-    VGPR0, VGPR1, VGPR2, VGPR3, VGPR4, VGPR5, VGPR6, VGPR7,
-    VGPR8, VGPR9, VGPR10, VGPR11, VGPR12, VGPR13, VGPR14, VGPR15,
-    VGPR16, VGPR17, VGPR18, VGPR19, VGPR20, VGPR21, VGPR22, VGPR23,
-    VGPR24, VGPR25, VGPR26, VGPR27, VGPR28, VGPR29, VGPR30, VGPR31,
-    VGPR32, VGPR33, VGPR34, VGPR35, VGPR36, VGPR37, VGPR38, VGPR39,
-    VGPR40, VGPR41, VGPR42, VGPR43, VGPR44, VGPR45, VGPR46, VGPR47,
-    VGPR48, VGPR49, VGPR50, VGPR51, VGPR52, VGPR53, VGPR54, VGPR55,
-    VGPR56, VGPR57, VGPR58, VGPR59, VGPR60, VGPR61, VGPR62, VGPR63,
-    VGPR64, VGPR65, VGPR66, VGPR67, VGPR68, VGPR69, VGPR70, VGPR71,
-    VGPR72, VGPR73, VGPR74, VGPR75, VGPR76, VGPR77, VGPR78, VGPR79,
-    VGPR80, VGPR81, VGPR82, VGPR83, VGPR84, VGPR85, VGPR86, VGPR87,
-    VGPR88, VGPR89, VGPR90, VGPR91, VGPR92, VGPR93, VGPR94, VGPR95,
-    VGPR96, VGPR97, VGPR98, VGPR99, VGPR100, VGPR101, VGPR102, VGPR103,
-    VGPR104, VGPR105, VGPR106, VGPR107, VGPR108, VGPR109, VGPR110, VGPR111,
-    VGPR112, VGPR113, VGPR114, VGPR115, VGPR116, VGPR117, VGPR118, VGPR119,
-    VGPR120, VGPR121, VGPR122, VGPR123, VGPR124, VGPR125, VGPR126, VGPR127,
-    VGPR128, VGPR129, VGPR130, VGPR131, VGPR132, VGPR133, VGPR134, VGPR135
-  ]>>>
+  CCIfNotInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<
+    !foreach(i, !range(0, 136), !cast<Register>("VGPR"#i))  // VGPR0-135
+  >>>
 ]>;
 
 def RetCC_SI_Shader : CallingConv<[
   CCIfType<[i1, i16], CCIfExtend<CCPromoteToType<i32>>>,
-  CCIfType<[i32, i16, v2i16] , CCAssignToReg<[
-    SGPR0, SGPR1, SGPR2, SGPR3, SGPR4, SGPR5, SGPR6, SGPR7,
-    SGPR8, SGPR9, SGPR10, SGPR11, SGPR12, SGPR13, SGPR14, SGPR15,
-    SGPR16, SGPR17, SGPR18, SGPR19, SGPR20, SGPR21, SGPR22, SGPR23,
-    SGPR24, SGPR25, SGPR26, SGPR27, SGPR28, SGPR29, SGPR30, SGPR31,
-    SGPR32, SGPR33, SGPR34, SGPR35, SGPR36, SGPR37, SGPR38, SGPR39,
-    SGPR40, SGPR41, SGPR42, SGPR43
-  ]>>,
+  CCIfType<[i32, i16, v2i16] , CCAssignToReg<
+    !foreach(i, !range(0, 44), !cast<Register>("SGPR"#i))  // SGPR0-43
+  >>,
 
   // 32*4 + 4 is the minimum for a fetch shader with 32 outputs.
-  CCIfType<[f32, f16, v2f16, bf16, v2bf16] , CCAssignToReg<[
-    VGPR0, VGPR1, VGPR2, VGPR3, VGPR4, VGPR5, VGPR6, VGPR7,
-    VGPR8, VGPR9, VGPR10, VGPR11, VGPR12, VGPR13, VGPR14, VGPR15,
-    VGPR16, VGPR17, VGPR18, VGPR19, VGPR20, VGPR21, VGPR22, VGPR23,
-    VGPR24, VGPR25, VGPR26, VGPR27, VGPR28, VGPR29, VGPR30, VGPR31,
-    VGPR32, VGPR33, VGPR34, VGPR35, VGPR36, VGPR37, VGPR38, VGPR39,
-    VGPR40, VGPR41, VGPR42, VGPR43, VGPR44, VGPR45, VGPR46, VGPR47,
-    VGPR48, VGPR49, VGPR50, VGPR51, VGPR52, VGPR53, VGPR54, VGPR55,
-    VGPR56, VGPR57, VGPR58, VGPR59, VGPR60, VGPR61, VGPR62, VGPR63,
-    VGPR64, VGPR65, VGPR66, VGPR67, VGPR68, VGPR69, VGPR70, VGPR71,
-    VGPR72, VGPR73, VGPR74, VGPR75, VGPR76, VGPR77, VGPR78, VGPR79,
-    VGPR80, VGPR81, VGPR82, VGPR83, VGPR84, VGPR85, VGPR86, VGPR87,
-    VGPR88, VGPR89, VGPR90, VGPR91, VGPR92, VGPR93, VGPR94, VGPR95,
-    VGPR96, VGPR97, VGPR98, VGPR99, VGPR100, VGPR101, VGPR102, VGPR103,
-    VGPR104, VGPR105, VGPR106, VGPR107, VGPR108, VGPR109, VGPR110, VGPR111,
-    VGPR112, VGPR113, VGPR114, VGPR115, VGPR116, VGPR117, VGPR118, VGPR119,
-    VGPR120, VGPR121, VGPR122, VGPR123, VGPR124, VGPR125, VGPR126, VGPR127,
-    VGPR128, VGPR129, VGPR130, VGPR131, VGPR132, VGPR133, VGPR134, VGPR135
-  ]>>
+  CCIfType<[f32, f16, v2f16, bf16, v2bf16] , CCAssignToReg<
+    !foreach(i, !range(0, 136), !cast<Register>("VGPR"#i))  // VGPR0-135
+  >>
 ]>;
 
 def CSR_AMDGPU_VGPRs : CalleeSavedRegs<
@@ -194,11 +130,9 @@ def CC_AMDGPU_Func : CallingConv<[
     !foreach(i, !range(0, 30), !cast<Register>("SGPR"#i))  // SGPR0-29
   >>>,
 
-  CCIfType<[i32, f32, i16, f16, v2i16, v2f16, i1, bf16, v2bf16], CCAssignToReg<[
-    VGPR0, VGPR1, VGPR2, VGPR3, VGPR4, VGPR5, VGPR6, VGPR7,
-    VGPR8, VGPR9, VGPR10, VGPR11, VGPR12, VGPR13, VGPR14, VGPR15,
-    VGPR16, VGPR17, VGPR18, VGPR19, VGPR20, VGPR21, VGPR22, VGPR23,
-    VGPR24, VGPR25, VGPR26, VGPR27, VGPR28, VGPR29, VGPR30, VGPR31]>>,
+  CCIfType<[i32, f32, i16, f16, v2i16, v2f16, i1, bf16, v2bf16], CCAssignToReg<
+    !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i))  // VGPR0-31
+  >>,
   CCIfType<[i32, f32, v2i16, v2f16, i16, f16, i1, bf16, v2bf16], CCAssignToStack<4, 4>>
 ]>;
 
@@ -206,11 +140,9 @@ def CC_AMDGPU_Func : CallingConv<[
 def RetCC_AMDGPU_Func : CallingConv<[
   CCIfType<[i1], CCPromoteToType<i32>>,
   CCIfType<[i1, i16], CCIfExtend<CCPromoteToType<i32>>>,
-  CCIfType<[i32, f32, i16, f16, v2i16, v2f16, bf16, v2bf16], CCAssignToReg<[
-    VGPR0, VGPR1, VGPR2, VGPR3, VGPR4, VGPR5, VGPR6, VGPR7,
-    VGPR8, VGPR9, VGPR10, VGPR11, VGPR12, VGPR13, VGPR14, VGPR15,
-    VGPR16, VGPR17, VGPR18, VGPR19, VGPR20, VGPR21, VGPR22, VGPR23,
-    VGPR24, VGPR25, VGPR26, VGPR27, VGPR28, VGPR29, VGPR30, VGPR31]>>,
+  CCIfType<[i32, f32, i16, f16, v2i16, v2f16, bf16, v2bf16], CCAssignToReg<
+    !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i))  // VGPR0-31
+  >>,
 ]>;
 
 def CC_AMDGPU : CallingConv<[
-- 
GitLab


From 75e7ba8c0b7efe75632d328a80391b9086ba8740 Mon Sep 17 00:00:00 2001
From: Sarah Spall <sarahspall@microsoft.com>
Date: Tue, 29 Oct 2024 07:56:05 -0700
Subject: [PATCH 448/693] [HLSL] Re-implement countbits with the correct return
 type (#113189)

Restricts hlsl countbits to always return a uint32.
Implements a lowering from llvm.ctpop which has an overloaded return
type to dxil cbits op which always returns uint32.
Closes #112779
---
 clang/lib/Headers/hlsl/hlsl_intrinsics.h      | 124 +++++++++++-------
 .../test/CodeGenHLSL/builtins/countbits.hlsl  |  62 ++++++---
 .../SemaHLSL/BuiltIns/countbits-errors.hlsl   |  14 +-
 llvm/lib/Target/DirectX/DXIL.td               |   5 +-
 llvm/lib/Target/DirectX/DXILOpLowering.cpp    |  70 ++++++++++
 llvm/test/CodeGen/DirectX/countbits.ll        |  46 ++++++-
 6 files changed, 234 insertions(+), 87 deletions(-)

diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index 8ade4b27f360..d9f3a17ea23d 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -723,66 +723,88 @@ float4 cosh(float4);
 
 #ifdef __HLSL_ENABLE_16_BIT
 _HLSL_AVAILABILITY(shadermodel, 6.2)
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-int16_t countbits(int16_t);
+const inline uint countbits(int16_t x) {
+  return __builtin_elementwise_popcount(x);
+}
 _HLSL_AVAILABILITY(shadermodel, 6.2)
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-int16_t2 countbits(int16_t2);
+const inline uint2 countbits(int16_t2 x) {
+  return __builtin_elementwise_popcount(x);
+}
 _HLSL_AVAILABILITY(shadermodel, 6.2)
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-int16_t3 countbits(int16_t3);
+const inline uint3 countbits(int16_t3 x) {
+  return __builtin_elementwise_popcount(x);
+}
 _HLSL_AVAILABILITY(shadermodel, 6.2)
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-int16_t4 countbits(int16_t4);
+const inline uint4 countbits(int16_t4 x) {
+  return __builtin_elementwise_popcount(x);
+}
 _HLSL_AVAILABILITY(shadermodel, 6.2)
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-uint16_t countbits(uint16_t);
+const inline uint countbits(uint16_t x) {
+  return __builtin_elementwise_popcount(x);
+}
 _HLSL_AVAILABILITY(shadermodel, 6.2)
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-uint16_t2 countbits(uint16_t2);
+const inline uint2 countbits(uint16_t2 x) {
+  return __builtin_elementwise_popcount(x);
+}
 _HLSL_AVAILABILITY(shadermodel, 6.2)
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-uint16_t3 countbits(uint16_t3);
+const inline uint3 countbits(uint16_t3 x) {
+  return __builtin_elementwise_popcount(x);
+}
 _HLSL_AVAILABILITY(shadermodel, 6.2)
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-uint16_t4 countbits(uint16_t4);
+const inline uint4 countbits(uint16_t4 x) {
+  return __builtin_elementwise_popcount(x);
+}
 #endif
 
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-int countbits(int);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-int2 countbits(int2);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-int3 countbits(int3);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-int4 countbits(int4);
-
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-uint countbits(uint);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-uint2 countbits(uint2);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-uint3 countbits(uint3);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-uint4 countbits(uint4);
-
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-int64_t countbits(int64_t);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-int64_t2 countbits(int64_t2);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-int64_t3 countbits(int64_t3);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-int64_t4 countbits(int64_t4);
-
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-uint64_t countbits(uint64_t);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-uint64_t2 countbits(uint64_t2);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-uint64_t3 countbits(uint64_t3);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_popcount)
-uint64_t4 countbits(uint64_t4);
+const inline uint countbits(int x) { return __builtin_elementwise_popcount(x); }
+const inline uint2 countbits(int2 x) {
+  return __builtin_elementwise_popcount(x);
+}
+const inline uint3 countbits(int3 x) {
+  return __builtin_elementwise_popcount(x);
+}
+const inline uint4 countbits(int4 x) {
+  return __builtin_elementwise_popcount(x);
+}
+
+const inline uint countbits(uint x) {
+  return __builtin_elementwise_popcount(x);
+}
+const inline uint2 countbits(uint2 x) {
+  return __builtin_elementwise_popcount(x);
+}
+const inline uint3 countbits(uint3 x) {
+  return __builtin_elementwise_popcount(x);
+}
+const inline uint4 countbits(uint4 x) {
+  return __builtin_elementwise_popcount(x);
+}
+
+const inline uint countbits(int64_t x) {
+  return __builtin_elementwise_popcount(x);
+}
+const inline uint2 countbits(int64_t2 x) {
+  return __builtin_elementwise_popcount(x);
+}
+const inline uint3 countbits(int64_t3 x) {
+  return __builtin_elementwise_popcount(x);
+}
+const inline uint4 countbits(int64_t4 x) {
+  return __builtin_elementwise_popcount(x);
+}
+
+const inline uint countbits(uint64_t x) {
+  return __builtin_elementwise_popcount(x);
+}
+const inline uint2 countbits(uint64_t2 x) {
+  return __builtin_elementwise_popcount(x);
+}
+const inline uint3 countbits(uint64_t3 x) {
+  return __builtin_elementwise_popcount(x);
+}
+const inline uint4 countbits(uint64_t4 x) {
+  return __builtin_elementwise_popcount(x);
+}
 
 //===----------------------------------------------------------------------===//
 // degrees builtins
diff --git a/clang/test/CodeGenHLSL/builtins/countbits.hlsl b/clang/test/CodeGenHLSL/builtins/countbits.hlsl
index 8dfe977bfae6..218d8dcd10f8 100644
--- a/clang/test/CodeGenHLSL/builtins/countbits.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/countbits.hlsl
@@ -4,26 +4,37 @@
 
 #ifdef __HLSL_ENABLE_16_BIT
 // CHECK-LABEL: test_countbits_ushort
-// CHECK: call i16 @llvm.ctpop.i16
-uint16_t test_countbits_ushort(uint16_t p0)
+// CHECK: [[A:%.*]] = call i16 @llvm.ctpop.i16
+// CHECK-NEXT: zext i16 [[A]] to i32
+uint test_countbits_ushort(uint16_t p0)
+{
+	return countbits(p0);
+}
+// CHECK-LABEL: test_countbits_short
+// CHECK: [[A:%.*]] = call i16 @llvm.ctpop.i16
+// CHECK-NEXT: sext i16 [[A]] to i32
+uint test_countbits_short(int16_t p0)
 {
 	return countbits(p0);
 }
 // CHECK-LABEL: test_countbits_ushort2
-// CHECK: call <2 x i16> @llvm.ctpop.v2i16
-uint16_t2 test_countbits_ushort2(uint16_t2 p0)
+// CHECK: [[A:%.*]] = call <2 x i16> @llvm.ctpop.v2i16
+// CHECK-NEXT: zext <2 x i16> [[A]] to <2 x i32>
+uint2 test_countbits_ushort2(uint16_t2 p0)
 {
 	return countbits(p0);
 }
 // CHECK-LABEL: test_countbits_ushort3
-// CHECK: call <3 x i16> @llvm.ctpop.v3i16
-uint16_t3 test_countbits_ushort3(uint16_t3 p0)
+// CHECK: [[A:%.*]] = call <3 x i16> @llvm.ctpop.v3i16
+// CHECK-NEXT: zext <3 x i16> [[A]] to <3 x i32>
+uint3 test_countbits_ushort3(uint16_t3 p0)
 {
 	return countbits(p0);
 }
 // CHECK-LABEL: test_countbits_ushort4
-// CHECK: call <4 x i16> @llvm.ctpop.v4i16
-uint16_t4 test_countbits_ushort4(uint16_t4 p0)
+// CHECK: [[A:%.*]] = call <4 x i16> @llvm.ctpop.v4i16
+// CHECK-NEXT: zext <4 x i16> [[A]] to <4 x i32>
+uint4 test_countbits_ushort4(uint16_t4 p0)
 {
 	return countbits(p0);
 }
@@ -31,7 +42,13 @@ uint16_t4 test_countbits_ushort4(uint16_t4 p0)
 
 // CHECK-LABEL: test_countbits_uint
 // CHECK: call i32 @llvm.ctpop.i32
-int test_countbits_uint(uint p0)
+uint test_countbits_uint(uint p0)
+{
+	return countbits(p0);
+}
+// CHECK-LABEL: test_countbits_int
+// CHECK: call i32 @llvm.ctpop.i32
+uint test_countbits_int(int p0)
 {
 	return countbits(p0);
 }
@@ -55,26 +72,37 @@ uint4 test_countbits_uint4(uint4 p0)
 }
 
 // CHECK-LABEL: test_countbits_long
-// CHECK: call i64 @llvm.ctpop.i64
-uint64_t test_countbits_long(uint64_t p0)
+// CHECK: [[A:%.*]] = call i64 @llvm.ctpop.i64
+// CHECK-NEXT: trunc i64 [[A]] to i32
+uint test_countbits_long(uint64_t p0)
+{
+	return countbits(p0);
+}
+// CHECK-LABEL: test_countbits_slong
+// CHECK: [[A:%.*]] = call i64 @llvm.ctpop.i64
+// CHECK-NEXT: trunc i64 [[A]] to i32
+uint test_countbits_slong(int64_t p0)
 {
 	return countbits(p0);
 }
 // CHECK-LABEL: test_countbits_long2
-// CHECK: call <2 x i64> @llvm.ctpop.v2i64
-uint64_t2 test_countbits_long2(uint64_t2 p0)
+// CHECK: [[A:%.*]] = call <2 x i64> @llvm.ctpop.v2i64
+// CHECK-NEXT: trunc <2 x i64> [[A]] to <2 x i32>
+uint2 test_countbits_long2(uint64_t2 p0)
 {
 	return countbits(p0);
 }
 // CHECK-LABEL: test_countbits_long3
-// CHECK: call <3 x i64> @llvm.ctpop.v3i64
-uint64_t3 test_countbits_long3(uint64_t3 p0)
+// CHECK: [[A:%.*]] = call <3 x i64> @llvm.ctpop.v3i64
+// CHECK-NEXT: trunc <3 x i64> [[A]] to <3 x i32>
+uint3 test_countbits_long3(uint64_t3 p0)
 {
 	return countbits(p0);
 }
 // CHECK-LABEL: test_countbits_long4
-// CHECK: call <4 x i64> @llvm.ctpop.v4i64
-uint64_t4 test_countbits_long4(uint64_t4 p0)
+// CHECK: [[A:%.*]] = call <4 x i64> @llvm.ctpop.v4i64
+// CHECK-NEXT: trunc <4 x i64> [[A]] to <4 x i32>
+uint4 test_countbits_long4(uint64_t4 p0)
 {
 	return countbits(p0);
 }
diff --git a/clang/test/SemaHLSL/BuiltIns/countbits-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/countbits-errors.hlsl
index 8d5f0abb2860..5704165e1a45 100644
--- a/clang/test/SemaHLSL/BuiltIns/countbits-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/countbits-errors.hlsl
@@ -1,6 +1,4 @@
-// RUN: %clang_cc1 -finclude-default-header
-// -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only
-// -disable-llvm-passes -verify -verify-ignore-unexpected
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -verify-ignore-unexpected
 
 
 double test_int_builtin(double p0) {
@@ -9,13 +7,11 @@ double test_int_builtin(double p0) {
 }
 
 double2 test_int_builtin_2(double2 p0) {
-  return __builtin_elementwise_popcount(p0);
-  // expected-error@-1 {{1st argument must be a vector of integers
-  // (was 'double2' (aka 'vector<double, 2>'))}}
+  return countbits(p0);
+  // expected-error@-1 {{call to 'countbits' is ambiguous}}
 }
 
 double test_int_builtin_3(float p0) {
-  return __builtin_elementwise_popcount(p0);
-  // expected-error@-1 {{1st argument must be a vector of integers
-  // (was 'float')}}
+  return countbits(p0);
+  // expected-error@-1 {{call to 'countbits' is ambiguous}}
 }
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 68ae5de06423..1e8dc63ffa25 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -554,11 +554,10 @@ def Rbits :  DXILOp<30, unary> {
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
 
-def CBits :  DXILOp<31, unary> {
+def CountBits :  DXILOp<31, unaryBits> {
   let Doc = "Returns the number of 1 bits in the specified value.";
-  let LLVMIntrinsic = int_ctpop;
   let arguments = [OverloadTy];
-  let result = OverloadTy;
+  let result = Int32Ty;
   let overloads =
       [Overloads<DXIL1_0, [Int16Ty, Int32Ty, Int64Ty]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index f7722d770747..8acc9c1efa08 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -505,6 +505,73 @@ public:
     });
   }
 
+  [[nodiscard]] bool lowerCtpopToCountBits(Function &F) {
+    IRBuilder<> &IRB = OpBuilder.getIRB();
+    Type *Int32Ty = IRB.getInt32Ty();
+
+    return replaceFunction(F, [&](CallInst *CI) -> Error {
+      IRB.SetInsertPoint(CI);
+      SmallVector<Value *> Args;
+      Args.append(CI->arg_begin(), CI->arg_end());
+
+      Type *RetTy = Int32Ty;
+      Type *FRT = F.getReturnType();
+      if (const auto *VT = dyn_cast<VectorType>(FRT))
+        RetTy = VectorType::get(RetTy, VT);
+
+      Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
+          dxil::OpCode::CountBits, Args, CI->getName(), RetTy);
+      if (Error E = OpCall.takeError())
+        return E;
+
+      // If the result type is 32 bits we can do a direct replacement.
+      if (FRT->isIntOrIntVectorTy(32)) {
+        CI->replaceAllUsesWith(*OpCall);
+        CI->eraseFromParent();
+        return Error::success();
+      }
+
+      unsigned CastOp;
+      unsigned CastOp2;
+      if (FRT->isIntOrIntVectorTy(16)) {
+        CastOp = Instruction::ZExt;
+        CastOp2 = Instruction::SExt;
+      } else { // must be 64 bits
+        assert(FRT->isIntOrIntVectorTy(64) &&
+               "Currently only lowering 16, 32, or 64 bit ctpop to CountBits \
+                is supported.");
+        CastOp = Instruction::Trunc;
+        CastOp2 = Instruction::Trunc;
+      }
+
+      // It is correct to replace the ctpop with the dxil op and
+      // remove all casts to i32
+      bool NeedsCast = false;
+      for (User *User : make_early_inc_range(CI->users())) {
+        Instruction *I = dyn_cast<Instruction>(User);
+        if (I && (I->getOpcode() == CastOp || I->getOpcode() == CastOp2) &&
+            I->getType() == RetTy) {
+          I->replaceAllUsesWith(*OpCall);
+          I->eraseFromParent();
+        } else
+          NeedsCast = true;
+      }
+
+      // It is correct to replace a ctpop with the dxil op and
+      // a cast from i32 to the return type of the ctpop
+      // the cast is emitted here if there is a non-cast to i32
+      // instr which uses the ctpop
+      if (NeedsCast) {
+        Value *Cast =
+            IRB.CreateZExtOrTrunc(*OpCall, F.getReturnType(), "ctpop.cast");
+        CI->replaceAllUsesWith(Cast);
+      }
+
+      CI->eraseFromParent();
+      return Error::success();
+    });
+  }
+
   bool lowerIntrinsics() {
     bool Updated = false;
     bool HasErrors = false;
@@ -543,6 +610,9 @@ public:
               return replaceSplitDoubleCallUsages(CI, Op);
             });
         break;
+      case Intrinsic::ctpop:
+        HasErrors |= lowerCtpopToCountBits(F);
+        break;
       }
       Updated = true;
     }
diff --git a/llvm/test/CodeGen/DirectX/countbits.ll b/llvm/test/CodeGen/DirectX/countbits.ll
index c6bc2b679094..f03ab9c5e79c 100644
--- a/llvm/test/CodeGen/DirectX/countbits.ll
+++ b/llvm/test/CodeGen/DirectX/countbits.ll
@@ -4,35 +4,67 @@
 
 define noundef i16 @test_countbits_short(i16 noundef %a) {
 entry:
-; CHECK: call i16 @dx.op.unary.i16(i32 31, i16 %{{.*}})
+; CHECK: [[A:%.*]] = call i32 @dx.op.unaryBits.i16(i32 31, i16 %{{.*}})
+; CHECK-NEXT: [[B:%.*]] = trunc i32 [[A]] to i16
+; CHECK-NEXT ret i16 [[B]]
   %elt.ctpop = call i16 @llvm.ctpop.i16(i16 %a)
   ret i16 %elt.ctpop
 }
 
+define noundef i32 @test_countbits_short2(i16 noundef %a) {
+entry:
+; CHECK: [[A:%.*]] = call i32 @dx.op.unaryBits.i16(i32 31, i16 %{{.*}})
+; CHECK-NEXT: ret i32 [[A]]
+  %elt.ctpop = call i16 @llvm.ctpop.i16(i16 %a)
+  %elt.zext = zext i16 %elt.ctpop to i32
+  ret i32 %elt.zext
+}
+
+define noundef i32 @test_countbits_short3(i16 noundef %a) {
+entry:
+; CHECK: [[A:%.*]] = call i32 @dx.op.unaryBits.i16(i32 31, i16 %{{.*}})
+; CHECK-NEXT: ret i32 [[A]]
+  %elt.ctpop = call i16 @llvm.ctpop.i16(i16 %a)
+  %elt.sext = sext i16 %elt.ctpop to i32
+  ret i32 %elt.sext
+}
+
 define noundef i32 @test_countbits_int(i32 noundef %a) {
 entry:
-; CHECK: call i32 @dx.op.unary.i32(i32 31, i32 %{{.*}})
+; CHECK: [[A:%.*]] = call i32 @dx.op.unaryBits.i32(i32 31, i32 %{{.*}})
+; CHECK-NEXT: ret i32 [[A]]
   %elt.ctpop = call i32 @llvm.ctpop.i32(i32 %a)
   ret i32 %elt.ctpop
 }
 
 define noundef i64 @test_countbits_long(i64 noundef %a) {
 entry:
-; CHECK: call i64 @dx.op.unary.i64(i32 31, i64 %{{.*}})
+; CHECK: [[A:%.*]] = call i32 @dx.op.unaryBits.i64(i32 31, i64 %{{.*}})
+; CHECK-NEXT: [[B:%.*]] = zext i32 [[A]] to i64
+; CHECK-NEXT ret i64 [[B]]
   %elt.ctpop = call i64 @llvm.ctpop.i64(i64 %a)
   ret i64 %elt.ctpop
 }
 
+define noundef i32 @test_countbits_long2(i64 noundef %a) {
+entry:
+; CHECK: [[A:%.*]] = call i32 @dx.op.unaryBits.i64(i32 31, i64 %{{.*}})
+; CHECK-NEXT: ret i32 [[A]]
+  %elt.ctpop = call i64 @llvm.ctpop.i64(i64 %a)
+  %elt.trunc = trunc i64 %elt.ctpop to i32
+  ret i32 %elt.trunc
+}
+
 define noundef <4 x i32> @countbits_vec4_i32(<4 x i32> noundef %a)  {
 entry:
   ; CHECK: [[ee0:%.*]] = extractelement <4 x i32> %a, i64 0
-  ; CHECK: [[ie0:%.*]] = call i32 @dx.op.unary.i32(i32 31, i32 [[ee0]])
+  ; CHECK: [[ie0:%.*]] = call i32 @dx.op.unaryBits.i32(i32 31, i32 [[ee0]])
   ; CHECK: [[ee1:%.*]] = extractelement <4 x i32> %a, i64 1
-  ; CHECK: [[ie1:%.*]] = call i32 @dx.op.unary.i32(i32 31, i32 [[ee1]])
+  ; CHECK: [[ie1:%.*]] = call i32 @dx.op.unaryBits.i32(i32 31, i32 [[ee1]])
   ; CHECK: [[ee2:%.*]] = extractelement <4 x i32> %a, i64 2
-  ; CHECK: [[ie2:%.*]] = call i32 @dx.op.unary.i32(i32 31, i32 [[ee2]])
+  ; CHECK: [[ie2:%.*]] = call i32 @dx.op.unaryBits.i32(i32 31, i32 [[ee2]])
   ; CHECK: [[ee3:%.*]] = extractelement <4 x i32> %a, i64 3
-  ; CHECK: [[ie3:%.*]] = call i32 @dx.op.unary.i32(i32 31, i32 [[ee3]])
+  ; CHECK: [[ie3:%.*]] = call i32 @dx.op.unaryBits.i32(i32 31, i32 [[ee3]])
   ; CHECK: insertelement <4 x i32> poison, i32 [[ie0]], i64 0
   ; CHECK: insertelement <4 x i32> %{{.*}}, i32 [[ie1]], i64 1
   ; CHECK: insertelement <4 x i32> %{{.*}}, i32 [[ie2]], i64 2
-- 
GitLab


From a156362e93eba9513611dc0989d516e9946cae48 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Tue, 29 Oct 2024 14:59:37 +0000
Subject: [PATCH 449/693] [AMDGPU] Fix machine verification failure after
 SIFoldOperandsImpl::tryFoldOMod (#113544)

Fixes #54201
---
 llvm/lib/Target/AMDGPU/SIFoldOperands.cpp    |  3 ++
 llvm/test/CodeGen/AMDGPU/fold-omod-crash.mir | 50 ++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 llvm/test/CodeGen/AMDGPU/fold-omod-crash.mir

diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index c912a580854c..f0c7837e0bb7 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -1793,6 +1793,9 @@ bool SIFoldOperandsImpl::tryFoldOMod(MachineInstr &MI) {
 
   DefOMod->setImm(OMod);
   MRI->replaceRegWith(MI.getOperand(0).getReg(), Def->getOperand(0).getReg());
+  // Kill flags can be wrong if we replaced a def inside a loop with a def
+  // outside the loop.
+  MRI->clearKillFlags(Def->getOperand(0).getReg());
   MI.eraseFromParent();
 
   // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac
diff --git a/llvm/test/CodeGen/AMDGPU/fold-omod-crash.mir b/llvm/test/CodeGen/AMDGPU/fold-omod-crash.mir
new file mode 100644
index 000000000000..8065e2cfc004
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fold-omod-crash.mir
@@ -0,0 +1,50 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=si-fold-operands %s -verify-machineinstrs -o - | FileCheck %s -check-prefix=GFX9
+
+# When V_ADD_F32 is replaced with an output modifier on V_RSQ_F32, check that
+# the kill flag is cleared on the use of %4 in V_MUL_F32.
+---
+name: main
+tracksRegLiveness: true
+machineFunctionInfo:
+  mode:
+    ieee: false
+    fp32-input-denormals: false
+    fp32-output-denormals: false
+body: |
+  ; GFX9-LABEL: name: main
+  ; GFX9: bb.0:
+  ; GFX9-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX9-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX9-NEXT:   [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+  ; GFX9-NEXT:   [[V_RSQ_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_RSQ_F32_e64 0, undef [[DEF]], 0, 1, implicit $mode, implicit $exec
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT: bb.1:
+  ; GFX9-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GFX9-NEXT:   [[V_MUL_F32_e64_:%[0-9]+]]:vgpr_32 = V_MUL_F32_e64 0, killed undef [[DEF2]], 0, [[V_RSQ_F32_e64_]], 0, 0, implicit $mode, implicit $exec
+  ; GFX9-NEXT:   SI_LOOP undef [[DEF1]], %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+  ; GFX9-NEXT:   S_BRANCH %bb.2
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT: bb.2:
+  ; GFX9-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9
+
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:sreg_64 = IMPLICIT_DEF
+    %2:vgpr_32 = nofpexcept V_RSQ_F32_e64 0, undef %0, 0, 0, implicit $mode, implicit $exec
+
+  bb.1:
+    %3:vgpr_32 = IMPLICIT_DEF
+    %4:vgpr_32 = nsz reassoc nofpexcept V_ADD_F32_e64 0, undef %2, 0, undef %2, 0, 0, implicit $mode, implicit $exec
+    %5:vgpr_32 = V_MUL_F32_e64 0, killed undef %3, 0, killed %4, 0, 0, implicit $mode, implicit $exec
+    SI_LOOP undef %1, %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.2:
+    S_ENDPGM 0
+...
-- 
GitLab


From 1e991b1021c1d7694e1a0dfe9e261fb27555f05f Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Tue, 29 Oct 2024 11:06:18 -0400
Subject: [PATCH 450/693] Nominate Corentin Jabot for lambdas (#114043)

Corentin has largely been handling reviews touching lambdas for the past
year or two, so he has significant understanding of the various moving
parts of this fairly substantial C++ feature. Given that work on lambdas
tends to be somewhat specialized, I think it makes sense for it to have
dedicated oversight.
---
 clang/Maintainers.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/clang/Maintainers.rst b/clang/Maintainers.rst
index 54690452681a..39f46457e676 100644
--- a/clang/Maintainers.rst
+++ b/clang/Maintainers.rst
@@ -78,6 +78,12 @@ Templates
 | ekeane\@nvidia.com (email), ErichKeane (Phabricator), erichkeane (GitHub)
 
 
+Lambdas
+~~~~~~~
+| Corentin Jabot
+| corentin.jabot\@gmail.com (email), cor3ntin (Phabricator), cor3ntin (GitHub)
+
+
 Debug information
 ~~~~~~~~~~~~~~~~~
 | Adrian Prantl
-- 
GitLab


From d43e4ce77d0a314139655c9cf7c3b533b5b72440 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Tue, 29 Oct 2024 11:17:11 -0400
Subject: [PATCH 451/693] Revert "[gn] port b1be21394e9c"

b1be21394e9c was reverted in 3ac75ee8ec.

This reverts commit 18f4b7e4a862c11816e62cc72fb2a4ca8fac1987, as well
as follow-ups a69d2a18d207947a25838dd01d2116bee384b75b and
4a6b56960f445d111adc9aef799acad8c6ca41f0.
---
 llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn | 2 --
 1 file changed, 2 deletions(-)

diff --git a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
index 020f3e7d9acd..5fbda794ff17 100644
--- a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
@@ -55,8 +55,6 @@ write_cmake_config("lit_common_configured") {
     "COMPILER_RT_ENABLE_INTERNAL_SYMBOLIZER_PYBOOL=False",
     "COMPILER_RT_HAS_NO_DEFAULT_CONFIG_FLAG_PYBOOL=True",
     "COMPILER_RT_INTERCEPT_LIBDISPATCH_PYBOOL=False",
-    "COMPILER_RT_RESOLVED_EXEC_OUTPUT_DIR=" +
-        rebase_path("$root_build_dir/bin"),
     "COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR=" +
         rebase_path(crt_current_out_dir),
     "COMPILER_RT_RESOLVED_OUTPUT_DIR=" + rebase_path(crt_current_out_dir),
-- 
GitLab


From f906d765baa0a17519b6d3310ba32e1b51b88c6d Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Tue, 29 Oct 2024 15:18:08 +0000
Subject: [PATCH 452/693] [gn build] Port 5ea694816b56

---
 .../unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn
index 44640c6527c9..97df71c6279e 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn
@@ -14,5 +14,6 @@ unittest("SandboxVectorizerTests") {
     "IntervalTest.cpp",
     "LegalityTest.cpp",
     "SchedulerTest.cpp",
+    "VecUtilsTest.cpp",
   ]
 }
-- 
GitLab


From af44976cad04d8470f205f557eaf172ee1eff0df Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Tue, 29 Oct 2024 15:18:09 +0000
Subject: [PATCH 453/693] [gn build] Port 6128ff663076

---
 .../secondary/llvm/unittests/ExecutionEngine/JITLink/BUILD.gn  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/JITLink/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/JITLink/BUILD.gn
index 6f52677cb833..7f74b335e30e 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/JITLink/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/JITLink/BUILD.gn
@@ -17,8 +17,9 @@ unittest("JITLinkTests") {
     "AArch32ErrorTests.cpp",
     "AArch32Tests.cpp",
     "EHFrameSupportTests.cpp",
-    "JITLinkMocks.cpp",
+    "JITLinkTestUtils.cpp",
     "LinkGraphTests.cpp",
+    "MachOLinkGraphTests.cpp",
     "MemoryManagerErrorTests.cpp",
     "StubsTests.cpp",
   ]
-- 
GitLab


From bf6c483e4714841b1511ea3666f05a468bd988fe Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 29 Oct 2024 15:15:38 +0000
Subject: [PATCH 454/693] [clang][x86] Add constexpr support for SSE2
 _mm_set*_epi* intrinsics

---
 clang/lib/Headers/emmintrin.h                 | 46 ++++++++++---------
 clang/test/CodeGen/X86/builtin_test_helpers.h | 10 ++++
 clang/test/CodeGen/X86/sse2-builtins.c        | 14 ++++++
 3 files changed, 49 insertions(+), 21 deletions(-)

diff --git a/clang/lib/Headers/emmintrin.h b/clang/lib/Headers/emmintrin.h
index 778cdf99a129..4f00b7f1a8d9 100644
--- a/clang/lib/Headers/emmintrin.h
+++ b/clang/lib/Headers/emmintrin.h
@@ -3512,8 +3512,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_undefined_si128(void) {
 ///    destination vector of [2 x i64].
 /// \returns An initialized 128-bit vector of [2 x i64] containing the values
 ///    provided in the operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi64x(long long __q1,
-                                                            long long __q0) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_set_epi64x(long long __q1, long long __q0) {
   return __extension__(__m128i)(__v2di){__q0, __q1};
 }
 
@@ -3533,9 +3533,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi64x(long long __q1,
 ///    destination vector of [2 x i64].
 /// \returns An initialized 128-bit vector of [2 x i64] containing the values
 ///    provided in the operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi64(__m64 __q1,
-                                                           __m64 __q0) {
-  return _mm_set_epi64x((long long)__q1, (long long)__q0);
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_set_epi64(__m64 __q1, __m64 __q0) {
+  return _mm_set_epi64x((long long)__q1[0], (long long)__q0[0]);
 }
 
 /// Initializes the 32-bit values in a 128-bit vector of [4 x i32] with
@@ -3560,8 +3560,10 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi64(__m64 __q1,
 ///    vector.
 /// \returns An initialized 128-bit vector of [4 x i32] containing the values
 ///    provided in the operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi32(int __i3, int __i2,
-                                                           int __i1, int __i0) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set_epi32(int __i3,
+                                                                     int __i2,
+                                                                     int __i1,
+                                                                     int __i0) {
   return __extension__(__m128i)(__v4si){__i0, __i1, __i2, __i3};
 }
 
@@ -3599,7 +3601,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi32(int __i3, int __i2,
 ///    vector.
 /// \returns An initialized 128-bit vector of [8 x i16] containing the values
 ///    provided in the operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm_set_epi16(short __w7, short __w6, short __w5, short __w4, short __w3,
               short __w2, short __w1, short __w0) {
   return __extension__(__m128i)(__v8hi){__w0, __w1, __w2, __w3,
@@ -3648,7 +3650,7 @@ _mm_set_epi16(short __w7, short __w6, short __w5, short __w4, short __w3,
 ///    Initializes bits [7:0] of the destination vector.
 /// \returns An initialized 128-bit vector of [16 x i8] containing the values
 ///    provided in the operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm_set_epi8(char __b15, char __b14, char __b13, char __b12, char __b11,
              char __b10, char __b9, char __b8, char __b7, char __b6, char __b5,
              char __b4, char __b3, char __b2, char __b1, char __b0) {
@@ -3670,7 +3672,8 @@ _mm_set_epi8(char __b15, char __b14, char __b13, char __b12, char __b11,
 ///    vector.
 /// \returns An initialized 128-bit integer vector of [2 x i64] with both
 ///    elements containing the value provided in the operand.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi64x(long long __q) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_set1_epi64x(long long __q) {
   return _mm_set_epi64x(__q, __q);
 }
 
@@ -3687,7 +3690,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi64x(long long __q) {
 ///    vector.
 /// \returns An initialized 128-bit vector of [2 x i64] with all elements
 ///    containing the value provided in the operand.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi64(__m64 __q) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_set1_epi64(__m64 __q) {
   return _mm_set_epi64(__q, __q);
 }
 
@@ -3704,7 +3708,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi64(__m64 __q) {
 ///    vector.
 /// \returns An initialized 128-bit vector of [4 x i32] with all elements
 ///    containing the value provided in the operand.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi32(int __i) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set1_epi32(int __i) {
   return _mm_set_epi32(__i, __i, __i, __i);
 }
 
@@ -3721,7 +3725,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi32(int __i) {
 ///    vector.
 /// \returns An initialized 128-bit vector of [8 x i16] with all elements
 ///    containing the value provided in the operand.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi16(short __w) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_set1_epi16(short __w) {
   return _mm_set_epi16(__w, __w, __w, __w, __w, __w, __w, __w);
 }
 
@@ -3738,7 +3743,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi16(short __w) {
 ///    vector.
 /// \returns An initialized 128-bit vector of [16 x i8] with all elements
 ///    containing the value provided in the operand.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi8(char __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set1_epi8(char __b) {
   return _mm_set_epi8(__b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b,
                       __b, __b, __b, __b, __b);
 }
@@ -3757,8 +3762,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi8(char __b) {
 ///    A 64-bit integral value used to initialize the upper 64 bits of the
 ///    result.
 /// \returns An initialized 128-bit integer vector.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setr_epi64(__m64 __q0,
-                                                            __m64 __q1) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_setr_epi64(__m64 __q0, __m64 __q1) {
   return _mm_set_epi64(__q1, __q0);
 }
 
@@ -3779,9 +3784,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setr_epi64(__m64 __q0,
 /// \param __i3
 ///    A 32-bit integral value used to initialize bits [127:96] of the result.
 /// \returns An initialized 128-bit integer vector.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setr_epi32(int __i0, int __i1,
-                                                            int __i2,
-                                                            int __i3) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_setr_epi32(int __i0, int __i1, int __i2, int __i3) {
   return _mm_set_epi32(__i3, __i2, __i1, __i0);
 }
 
@@ -3810,7 +3814,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setr_epi32(int __i0, int __i1,
 /// \param __w7
 ///    A 16-bit integral value used to initialize bits [127:112] of the result.
 /// \returns An initialized 128-bit integer vector.
-static __inline__ __m128i __DEFAULT_FN_ATTRS
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm_setr_epi16(short __w0, short __w1, short __w2, short __w3, short __w4,
                short __w5, short __w6, short __w7) {
   return _mm_set_epi16(__w7, __w6, __w5, __w4, __w3, __w2, __w1, __w0);
@@ -3857,7 +3861,7 @@ _mm_setr_epi16(short __w0, short __w1, short __w2, short __w3, short __w4,
 /// \param __b15
 ///    An 8-bit integral value used to initialize bits [127:120] of the result.
 /// \returns An initialized 128-bit integer vector.
-static __inline__ __m128i __DEFAULT_FN_ATTRS
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm_setr_epi8(char __b0, char __b1, char __b2, char __b3, char __b4, char __b5,
               char __b6, char __b7, char __b8, char __b9, char __b10,
               char __b11, char __b12, char __b13, char __b14, char __b15) {
diff --git a/clang/test/CodeGen/X86/builtin_test_helpers.h b/clang/test/CodeGen/X86/builtin_test_helpers.h
index f6547d4cb29c..01800db33afb 100644
--- a/clang/test/CodeGen/X86/builtin_test_helpers.h
+++ b/clang/test/CodeGen/X86/builtin_test_helpers.h
@@ -25,6 +25,16 @@ constexpr bool match_v4si(__m128i _v, int a, int b, int c, int d) {
   return v[0] == a && v[1] == b && v[2] == c && v[3] == d;
 }
 
+constexpr bool match_v8hi(__m128i _v, short a, short b, short c, short d, short e, short f, short g, short h) {
+  __v8hi v = (__v8hi)_v;
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h;
+}
+
+constexpr bool match_v16qi(__m128i _v, char a, char b, char c, char d, char e, char f, char g, char h, char i, char j, char k, char l, char m, char n, char o, char p) {
+  __v16qi v = (__v16qi)_v;
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h && v[8] == i && v[9] == j && v[10] == k && v[11] == l && v[12] == m && v[13] == n && v[14] == o && v[15] == p;
+}
+
 constexpr bool match_m256(__m256 v, float a, float b, float c, float d, float e, float f, float g, float h) {
   return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h;
 }
diff --git a/clang/test/CodeGen/X86/sse2-builtins.c b/clang/test/CodeGen/X86/sse2-builtins.c
index 82aa7a2d2b49..c4493a491205 100644
--- a/clang/test/CodeGen/X86/sse2-builtins.c
+++ b/clang/test/CodeGen/X86/sse2-builtins.c
@@ -1013,6 +1013,7 @@ __m128i test_mm_set_epi8(char A, char B, char C, char D,
   // CHECK: insertelement <16 x i8> %{{.*}}, i8 %{{.*}}, i32 15
   return _mm_set_epi8(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P);
 }
+TEST_CONSTEXPR(match_v16qi(_mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
 
 __m128i test_mm_set_epi16(short A, short B, short C, short D,
                           short E, short F, short G, short H) {
@@ -1027,6 +1028,7 @@ __m128i test_mm_set_epi16(short A, short B, short C, short D,
   // CHECK: insertelement <8 x i16> %{{.*}}, i16 %{{.*}}, i32 7
   return _mm_set_epi16(A, B, C, D, E, F, G, H);
 }
+TEST_CONSTEXPR(match_v8hi(_mm_set_epi16(0, -1, -2, -3, -4, -5, -6, -7), -7, -6, -5, -4, -3, -2, -1, 0));
 
 __m128i test_mm_set_epi32(int A, int B, int C, int D) {
   // CHECK-LABEL: test_mm_set_epi32
@@ -1036,6 +1038,7 @@ __m128i test_mm_set_epi32(int A, int B, int C, int D) {
   // CHECK: insertelement <4 x i32> %{{.*}}, i32 %{{.*}}, i32 3
   return _mm_set_epi32(A, B, C, D);
 }
+TEST_CONSTEXPR(match_v4si(_mm_set_epi32(1, -3, 5, -7), -7, 5, -3, 1));
 
 __m128i test_mm_set_epi64(__m64 A, __m64 B) {
   // CHECK-LABEL: test_mm_set_epi64
@@ -1043,6 +1046,7 @@ __m128i test_mm_set_epi64(__m64 A, __m64 B) {
   // CHECK: insertelement <2 x i64> %{{.*}}, i64 %{{.*}}, i32 1
   return _mm_set_epi64(A, B);
 }
+TEST_CONSTEXPR(match_v2di(_mm_set_epi64((__m64){-1}, (__m64){42}), 42, -1));
 
 __m128i test_mm_set_epi64x(long long A, long long B) {
   // CHECK-LABEL: test_mm_set_epi64x
@@ -1050,6 +1054,7 @@ __m128i test_mm_set_epi64x(long long A, long long B) {
   // CHECK: insertelement <2 x i64> %{{.*}}, i64 %{{.*}}, i32 1
   return _mm_set_epi64x(A, B);
 }
+TEST_CONSTEXPR(match_v2di(_mm_set_epi64x(100, -1000), -1000, 100));
 
 __m128d test_mm_set_pd(double A, double B) {
   // CHECK-LABEL: test_mm_set_pd
@@ -1095,6 +1100,7 @@ __m128i test_mm_set1_epi8(char A) {
   // CHECK: insertelement <16 x i8> %{{.*}}, i8 %{{.*}}, i32 15
   return _mm_set1_epi8(A);
 }
+TEST_CONSTEXPR(match_v16qi(_mm_set1_epi8(99), 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99));
 
 __m128i test_mm_set1_epi16(short A) {
   // CHECK-LABEL: test_mm_set1_epi16
@@ -1108,6 +1114,7 @@ __m128i test_mm_set1_epi16(short A) {
   // CHECK: insertelement <8 x i16> %{{.*}}, i16 %{{.*}}, i32 7
   return _mm_set1_epi16(A);
 }
+TEST_CONSTEXPR(match_v8hi(_mm_set1_epi16(-128), -128, -128, -128, -128, -128, -128, -128, -128));
 
 __m128i test_mm_set1_epi32(int A) {
   // CHECK-LABEL: test_mm_set1_epi32
@@ -1117,6 +1124,7 @@ __m128i test_mm_set1_epi32(int A) {
   // CHECK: insertelement <4 x i32> %{{.*}}, i32 %{{.*}}, i32 3
   return _mm_set1_epi32(A);
 }
+TEST_CONSTEXPR(match_v4si(_mm_set1_epi32(55), 55, 55, 55, 55));
 
 __m128i test_mm_set1_epi64(__m64 A) {
   // CHECK-LABEL: test_mm_set1_epi64
@@ -1124,6 +1132,7 @@ __m128i test_mm_set1_epi64(__m64 A) {
   // CHECK: insertelement <2 x i64> %{{.*}}, i64 %{{.*}}, i32 1
   return _mm_set1_epi64(A);
 }
+TEST_CONSTEXPR(match_v2di(_mm_set1_epi64((__m64){-65535}), -65535, -65535));
 
 __m128i test_mm_set1_epi64x(long long A) {
   // CHECK-LABEL: test_mm_set1_epi64x
@@ -1131,6 +1140,7 @@ __m128i test_mm_set1_epi64x(long long A) {
   // CHECK: insertelement <2 x i64> %{{.*}}, i64 %{{.*}}, i32 1
   return _mm_set1_epi64x(A);
 }
+TEST_CONSTEXPR(match_v2di(_mm_set1_epi64x(65536), 65536, 65536));
 
 __m128d test_mm_set1_pd(double A) {
   // CHECK-LABEL: test_mm_set1_pd
@@ -1163,6 +1173,7 @@ __m128i test_mm_setr_epi8(char A, char B, char C, char D,
   // CHECK: insertelement <16 x i8> %{{.*}}, i8 %{{.*}}, i32 15
   return _mm_setr_epi8(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P);
 }
+TEST_CONSTEXPR(match_v16qi(_mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15));
 
 __m128i test_mm_setr_epi16(short A, short B, short C, short D,
                            short E, short F, short G, short H) {
@@ -1177,6 +1188,7 @@ __m128i test_mm_setr_epi16(short A, short B, short C, short D,
   // CHECK: insertelement <8 x i16> %{{.*}}, i16 %{{.*}}, i32 7
   return _mm_setr_epi16(A, B, C, D, E, F, G, H);
 }
+TEST_CONSTEXPR(match_v8hi(_mm_setr_epi16(0, -1, -2, -3, -4, -5, -6, -7), 0, -1, -2, -3, -4, -5, -6, -7));
 
 __m128i test_mm_setr_epi32(int A, int B, int C, int D) {
   // CHECK-LABEL: test_mm_setr_epi32
@@ -1186,6 +1198,7 @@ __m128i test_mm_setr_epi32(int A, int B, int C, int D) {
   // CHECK: insertelement <4 x i32> %{{.*}}, i32 %{{.*}}, i32 3
   return _mm_setr_epi32(A, B, C, D);
 }
+TEST_CONSTEXPR(match_v4si(_mm_setr_epi32(1, -3, 5, -7), 1, -3, 5, -7));
 
 __m128i test_mm_setr_epi64(__m64 A, __m64 B) {
   // CHECK-LABEL: test_mm_setr_epi64
@@ -1193,6 +1206,7 @@ __m128i test_mm_setr_epi64(__m64 A, __m64 B) {
   // CHECK: insertelement <2 x i64> %{{.*}}, i64 %{{.*}}, i32 1
   return _mm_setr_epi64(A, B);
 }
+TEST_CONSTEXPR(match_v2di(_mm_setr_epi64((__m64){-1}, (__m64){42}), -1, 42));
 
 __m128d test_mm_setr_pd(double A, double B) {
   // CHECK-LABEL: test_mm_setr_pd
-- 
GitLab


From a9c417c28a25c153aa0fdbe2eb5453a93820a3b1 Mon Sep 17 00:00:00 2001
From: Hugo Trachino <hugo.trachino@huawei.com>
Date: Tue, 29 Oct 2024 15:47:13 +0000
Subject: [PATCH 455/693] [MLIR][SCF] Fix LoopPeelOp documentation (NFC)
 (#113179)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

As an example, I added annotations to the peel_front unit test.

```
func.func @loop_peel_first_iter_op() {
  // CHECK: %[[C0:.+]] = arith.constant 0
  // CHECK: %[[C41:.+]] = arith.constant 41
  // CHECK: %[[C5:.+]] = arith.constant 5
  // CHECK: %[[C5_0:.+]] = arith.constant 5
  // CHECK: scf.for %{{.+}} = %[[C0]] to %[[C5_0]] step %[[C5]]
  // CHECK:   arith.addi
  // CHECK: scf.for %{{.+}} = %[[C5_0]] to %[[C41]] step %[[C5]]
  // CHECK:   arith.addi
  %0 = arith.constant 0 : index
  %1 = arith.constant 41 : index
  %2 = arith.constant 5 : index
  scf.for %i = %0 to %1 step %2 {
    arith.addi %i, %i : index
  }
  return
}

module attributes {transform.with_named_sequence} {
  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
    %0 = transform.structured.match ops{["arith.addi"]} in %arg1 : (!transform.any_op) -> !transform.any_op
    %1 = transform.get_parent_op %0 {op_name = "scf.for"} : (!transform.any_op) -> !transform.op<"scf.for">
    %main_loop, %remainder = transform.loop.peel %1 {peel_front = true} : (!transform.op<"scf.for">) -> (!transform.op<"scf.for">, !transform.op<"scf.for">)
    transform.annotate %main_loop "main_loop" : !transform.op<"scf.for">
    transform.annotate %remainder "remainder" : !transform.op<"scf.for">
    transform.yield
  }
}
```
Gives :
```
  func.func @loop_peel_first_iter_op() {
    %c0 = arith.constant 0 : index
    %c41 = arith.constant 41 : index
    %c5 = arith.constant 5 : index
    %c5_0 = arith.constant 5 : index
    scf.for %arg0 = %c0 to %c5_0 step %c5 {
      %0 = arith.addi %arg0, %arg0 : index
    } {remainder}  // The first iteration loop (second result) has been annotated remainder
    scf.for %arg0 = %c5_0 to %c41 step %c5 {
      %0 = arith.addi %arg0, %arg0 : index
    } {main_loop} // The main loop (first result) has been annotated main_loop
    return
  }
```

---------

Co-authored-by: Andrzej Warzyński <andrzej.warzynski@gmail.com>
---
 .../SCF/TransformOps/SCFTransformOps.td       | 28 ++++++++++---------
 .../SCF/Transforms/LoopSpecialization.cpp     | 11 ++++----
 2 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.td b/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.td
index 20880d94a83c..5dba8c5e57ba 100644
--- a/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.td
+++ b/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.td
@@ -146,7 +146,7 @@ def LoopPeelOp : Op<Transform_Dialect, "loop.peel",
   let summary = "Peels the first or last iteration of the loop";
   let description = [{
      Rewrite the given loop with a main loop and a partial (first or last) loop.
-     When the `peelFront` option is set as true, the first iteration is peeled off.
+     When the `peelFront` option is set to true, the first iteration is peeled off.
      Otherwise, updates the given loop so that its step evenly divides its range and puts
      the remaining iteration into a separate loop or a conditional.
 
@@ -156,18 +156,20 @@ def LoopPeelOp : Op<Transform_Dialect, "loop.peel",
      #### Return modes
 
      This operation ignores non-scf::ForOp ops and drops them in the return.
-
-     When `peelFront` is true, this operation returns two scf::ForOp Ops, the
-     first scf::ForOp corresponds to the first iteration of the loop which can
-     be canonicalized away in the following optimization. The second loop Op
-     contains the remaining iteration, and the new lower bound is the original
-     lower bound plus the number of steps.
-
-     When `peelFront` is not true, this operation returns two scf::ForOp Ops, with the first
-     scf::ForOp satisfying: "the loop trip count is divisible by the step".
-     The second loop Op contains the remaining iteration. Note that even though the
-     Payload IR modification may be performed in-place, this operation consumes
-     the operand handle and produces a new one.
+     The op returns two loops, the peeled loop which has trip count divisible
+     by the step, and the remainder loop.
+
+     When `peelFront` is true, the first result (remainder loop) executes all
+     but the first iteration of the target loop. The second result (peeled
+     loop) corresponds to the first iteration of the loop which can be
+     canonicalized away in the following optimizations.
+
+     When `peelFront` is false, the first result (peeled loop) is the portion
+     of the target loop with the highest upper bound that is divisible by the
+     step. The second result (remainder loop) contains the remaining iterations. 
+     
+     Note that even though the Payload IR modification may be performed
+     in-place, this operation consumes the operand handle and produces a new one.
 
      #### Return Modes
 
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
index a30e349d4913..5104ad4b3a30 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
@@ -206,12 +206,11 @@ LogicalResult mlir::scf::peelForLoopAndSimplifyBounds(RewriterBase &rewriter,
   return success();
 }
 
-/// When the `peelFront` option is set as true, the first iteration of the loop
-/// is peeled off. This function rewrites the original scf::ForOp as two
-/// scf::ForOp Ops, the first scf::ForOp corresponds to the first iteration of
-/// the loop which can be canonicalized away in the following optimization. The
-/// second loop Op contains the remaining iteration, and the new lower bound is
-/// the original lower bound plus the number of steps.
+/// Rewrites the original scf::ForOp as two scf::ForOp Ops, the first
+/// scf::ForOp corresponds to the first iteration of the loop which can be
+/// canonicalized away in the following optimizations. The second loop Op
+/// contains the remaining iterations, with a lower bound updated as the
+/// original lower bound plus the step (i.e. skips the first iteration).
 LogicalResult mlir::scf::peelForLoopFirstIteration(RewriterBase &b, ForOp forOp,
                                                    ForOp &firstIteration) {
   RewriterBase::InsertionGuard guard(b);
-- 
GitLab


From 4df71ab78e9aa729959432bc0f8502760c90235b Mon Sep 17 00:00:00 2001
From: Jorge Gorbe Moya <jgorbe@google.com>
Date: Tue, 29 Oct 2024 09:25:51 -0700
Subject: [PATCH 456/693] [SandboxIR] Add callbacks for instruction
 insert/remove/move ops (#112965)

---
 llvm/include/llvm/SandboxIR/Context.h      | 66 ++++++++++++++-
 llvm/lib/SandboxIR/Context.cpp             | 73 ++++++++++++++--
 llvm/lib/SandboxIR/Instruction.cpp         |  3 +
 llvm/unittests/SandboxIR/SandboxIRTest.cpp | 98 ++++++++++++++++++++++
 4 files changed, 233 insertions(+), 7 deletions(-)

diff --git a/llvm/include/llvm/SandboxIR/Context.h b/llvm/include/llvm/SandboxIR/Context.h
index 1285598a1c02..f2056de87cb9 100644
--- a/llvm/include/llvm/SandboxIR/Context.h
+++ b/llvm/include/llvm/SandboxIR/Context.h
@@ -9,18 +9,39 @@
 #ifndef LLVM_SANDBOXIR_CONTEXT_H
 #define LLVM_SANDBOXIR_CONTEXT_H
 
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/SandboxIR/Tracker.h"
 #include "llvm/SandboxIR/Type.h"
 
+#include <cstdint>
+
 namespace llvm::sandboxir {
 
-class Module;
-class Value;
 class Argument;
+class BBIterator;
 class Constant;
+class Module;
+class Value;
 
 class Context {
+public:
+  // A EraseInstrCallback receives the instruction about to be erased.
+  using EraseInstrCallback = std::function<void(Instruction *)>;
+  // A CreateInstrCallback receives the instruction about to be created.
+  using CreateInstrCallback = std::function<void(Instruction *)>;
+  // A MoveInstrCallback receives the instruction about to be moved, the
+  // destination BB and an iterator pointing to the insertion position.
+  using MoveInstrCallback =
+      std::function<void(Instruction *, const BBIterator &)>;
+
+  /// An ID for a registered callback. Used for deregistration. Using a 64-bit
+  /// integer so we don't have to worry about the unlikely case of overflowing
+  /// a 32-bit counter.
+  using CallbackID = uint64_t;
+
 protected:
   LLVMContext &LLVMCtx;
   friend class Type;        // For LLVMCtx.
@@ -48,6 +69,21 @@ protected:
   /// Type objects.
   DenseMap<llvm::Type *, std::unique_ptr<Type, TypeDeleter>> LLVMTypeToTypeMap;
 
+  /// Callbacks called when an IR instruction is about to get erased. Keys are
+  /// used as IDs for deregistration.
+  MapVector<CallbackID, EraseInstrCallback> EraseInstrCallbacks;
+  /// Callbacks called when an IR instruction is about to get created. Keys are
+  /// used as IDs for deregistration.
+  MapVector<CallbackID, CreateInstrCallback> CreateInstrCallbacks;
+  /// Callbacks called when an IR instruction is about to get moved. Keys are
+  /// used as IDs for deregistration.
+  MapVector<CallbackID, MoveInstrCallback> MoveInstrCallbacks;
+
+  /// A counter used for assigning callback IDs during registration. The same
+  /// counter is used for all kinds of callbacks so we can detect mismatched
+  /// registration/deregistration.
+  CallbackID NextCallbackID = 0;
+
   /// Remove \p V from the maps and returns the unique_ptr.
   std::unique_ptr<Value> detachLLVMValue(llvm::Value *V);
   /// Remove \p SBV from all SandboxIR maps and stop owning it. This effectively
@@ -70,6 +106,10 @@ protected:
   Constant *getOrCreateConstant(llvm::Constant *LLVMC);
   friend class Utils; // For getMemoryBase
 
+  void runEraseInstrCallbacks(Instruction *I);
+  void runCreateInstrCallbacks(Instruction *I);
+  void runMoveInstrCallbacks(Instruction *I, const BBIterator &Where);
+
   // Friends for getOrCreateConstant().
 #define DEF_CONST(ID, CLASS) friend class CLASS;
 #include "llvm/SandboxIR/Values.def"
@@ -198,6 +238,28 @@ public:
 
   /// \Returns the number of values registered with Context.
   size_t getNumValues() const { return LLVMValueToValueMap.size(); }
+
+  /// Register a callback that gets called when a SandboxIR instruction is about
+  /// to be removed from its parent. Note that this will also be called when
+  /// reverting the creation of an instruction.
+  /// \Returns a callback ID for later deregistration.
+  CallbackID registerEraseInstrCallback(EraseInstrCallback CB);
+  void unregisterEraseInstrCallback(CallbackID ID);
+
+  /// Register a callback that gets called right after a SandboxIR instruction
+  /// is created. Note that this will also be called when reverting the removal
+  /// of an instruction.
+  /// \Returns a callback ID for later deregistration.
+  CallbackID registerCreateInstrCallback(CreateInstrCallback CB);
+  void unregisterCreateInstrCallback(CallbackID ID);
+
+  /// Register a callback that gets called when a SandboxIR instruction is about
+  /// to be moved. Note that this will also be called when reverting a move.
+  /// \Returns a callback ID for later deregistration.
+  CallbackID registerMoveInstrCallback(MoveInstrCallback CB);
+  void unregisterMoveInstrCallback(CallbackID ID);
+
+  // TODO: Add callbacks for instructions inserted/removed if needed.
 };
 
 } // namespace llvm::sandboxir
diff --git a/llvm/lib/SandboxIR/Context.cpp b/llvm/lib/SandboxIR/Context.cpp
index 486e935bc35f..5e5cbbbc4515 100644
--- a/llvm/lib/SandboxIR/Context.cpp
+++ b/llvm/lib/SandboxIR/Context.cpp
@@ -35,17 +35,20 @@ Value *Context::registerValue(std::unique_ptr<Value> &&VPtr) {
   assert(VPtr->getSubclassID() != Value::ClassID::User &&
          "Can't register a user!");
 
+  Value *V = VPtr.get();
+  [[maybe_unused]] auto Pair =
+      LLVMValueToValueMap.insert({VPtr->Val, std::move(VPtr)});
+  assert(Pair.second && "Already exists!");
+
   // Track creation of instructions.
   // Please note that we don't allow the creation of detached instructions,
   // meaning that the instructions need to be inserted into a block upon
   // creation. This is why the tracker class combines creation and insertion.
-  if (auto *I = dyn_cast<Instruction>(VPtr.get()))
+  if (auto *I = dyn_cast<Instruction>(V)) {
     getTracker().emplaceIfTracking<CreateAndInsertInst>(I);
+    runCreateInstrCallbacks(I);
+  }
 
-  Value *V = VPtr.get();
-  [[maybe_unused]] auto Pair =
-      LLVMValueToValueMap.insert({VPtr->Val, std::move(VPtr)});
-  assert(Pair.second && "Already exists!");
   return V;
 }
 
@@ -660,4 +663,64 @@ Module *Context::createModule(llvm::Module *LLVMM) {
   return M;
 }
 
+void Context::runEraseInstrCallbacks(Instruction *I) {
+  for (const auto &CBEntry : EraseInstrCallbacks)
+    CBEntry.second(I);
+}
+
+void Context::runCreateInstrCallbacks(Instruction *I) {
+  for (auto &CBEntry : CreateInstrCallbacks)
+    CBEntry.second(I);
+}
+
+void Context::runMoveInstrCallbacks(Instruction *I, const BBIterator &WhereIt) {
+  for (auto &CBEntry : MoveInstrCallbacks)
+    CBEntry.second(I, WhereIt);
+}
+
+// An arbitrary limit, to check for accidental misuse. We expect a small number
+// of callbacks to be registered at a time, but we can increase this number if
+// we discover we needed more.
+static constexpr int MaxRegisteredCallbacks = 16;
+
+Context::CallbackID Context::registerEraseInstrCallback(EraseInstrCallback CB) {
+  assert(EraseInstrCallbacks.size() <= MaxRegisteredCallbacks &&
+         "EraseInstrCallbacks size limit exceeded");
+  CallbackID ID = NextCallbackID++;
+  EraseInstrCallbacks[ID] = CB;
+  return ID;
+}
+void Context::unregisterEraseInstrCallback(CallbackID ID) {
+  [[maybe_unused]] bool Erased = EraseInstrCallbacks.erase(ID);
+  assert(Erased &&
+         "Callback ID not found in EraseInstrCallbacks during deregistration");
+}
+
+Context::CallbackID
+Context::registerCreateInstrCallback(CreateInstrCallback CB) {
+  assert(CreateInstrCallbacks.size() <= MaxRegisteredCallbacks &&
+         "CreateInstrCallbacks size limit exceeded");
+  CallbackID ID = NextCallbackID++;
+  CreateInstrCallbacks[ID] = CB;
+  return ID;
+}
+void Context::unregisterCreateInstrCallback(CallbackID ID) {
+  [[maybe_unused]] bool Erased = CreateInstrCallbacks.erase(ID);
+  assert(Erased &&
+         "Callback ID not found in CreateInstrCallbacks during deregistration");
+}
+
+Context::CallbackID Context::registerMoveInstrCallback(MoveInstrCallback CB) {
+  assert(MoveInstrCallbacks.size() <= MaxRegisteredCallbacks &&
+         "MoveInstrCallbacks size limit exceeded");
+  CallbackID ID = NextCallbackID++;
+  MoveInstrCallbacks[ID] = CB;
+  return ID;
+}
+void Context::unregisterMoveInstrCallback(CallbackID ID) {
+  [[maybe_unused]] bool Erased = MoveInstrCallbacks.erase(ID);
+  assert(Erased &&
+         "Callback ID not found in MoveInstrCallbacks during deregistration");
+}
+
 } // namespace llvm::sandboxir
diff --git a/llvm/lib/SandboxIR/Instruction.cpp b/llvm/lib/SandboxIR/Instruction.cpp
index d80d10370e32..096b827541ee 100644
--- a/llvm/lib/SandboxIR/Instruction.cpp
+++ b/llvm/lib/SandboxIR/Instruction.cpp
@@ -73,6 +73,8 @@ void Instruction::removeFromParent() {
 
 void Instruction::eraseFromParent() {
   assert(users().empty() && "Still connected to users, can't erase!");
+
+  Ctx.runEraseInstrCallbacks(this);
   std::unique_ptr<Value> Detached = Ctx.detach(this);
   auto LLVMInstrs = getLLVMInstrs();
 
@@ -100,6 +102,7 @@ void Instruction::moveBefore(BasicBlock &BB, const BBIterator &WhereIt) {
     // Destination is same as origin, nothing to do.
     return;
 
+  Ctx.runMoveInstrCallbacks(this, WhereIt);
   Ctx.getTracker().emplaceIfTracking<MoveInstr>(this);
 
   auto *LLVMBB = cast<llvm::BasicBlock>(BB.Val);
diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index 97113b303f72..99e14292a91b 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -22,6 +22,7 @@
 #include "llvm/SandboxIR/Value.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gmock/gmock-matchers.h"
+#include "gmock/gmock-more-matchers.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
@@ -5962,3 +5963,100 @@ TEST_F(SandboxIRTest, CheckClassof) {
   EXPECT_NE(&sandboxir::CLASS::classof, &sandboxir::Instruction::classof);
 #include "llvm/SandboxIR/Values.def"
 }
+
+TEST_F(SandboxIRTest, InstructionCallbacks) {
+  parseIR(C, R"IR(
+    define void @foo(ptr %ptr, i8 %val) {
+      ret void
+    }
+  )IR");
+  Function &LLVMF = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto &BB = *F.begin();
+  sandboxir::Argument *Ptr = F.getArg(0);
+  sandboxir::Argument *Val = F.getArg(1);
+  sandboxir::Instruction *Ret = &BB.front();
+
+  SmallVector<sandboxir::Instruction *> Inserted;
+  auto InsertCbId = Ctx.registerCreateInstrCallback(
+      [&Inserted](sandboxir::Instruction *I) { Inserted.push_back(I); });
+
+  SmallVector<sandboxir::Instruction *> Removed;
+  auto RemoveCbId = Ctx.registerEraseInstrCallback(
+      [&Removed](sandboxir::Instruction *I) { Removed.push_back(I); });
+
+  // Keep the moved instruction and the instruction pointed by the Where
+  // iterator so we can check both callback arguments work as expected.
+  SmallVector<std::pair<sandboxir::Instruction *, sandboxir::Instruction *>>
+      Moved;
+  auto MoveCbId = Ctx.registerMoveInstrCallback(
+      [&Moved](sandboxir::Instruction *I, const sandboxir::BBIterator &Where) {
+        // Use a nullptr to signal "move to end" to keep it single. We only
+        // have a basic block in this test case anyway.
+        if (Where == Where.getNodeParent()->end())
+          Moved.push_back(std::make_pair(I, nullptr));
+        else
+          Moved.push_back(std::make_pair(I, &*Where));
+      });
+
+  // Two more insertion callbacks, to check that they're called in registration
+  // order.
+  SmallVector<int> Order;
+  auto CheckOrderInsertCbId1 = Ctx.registerCreateInstrCallback(
+      [&Order](sandboxir::Instruction *I) { Order.push_back(1); });
+
+  auto CheckOrderInsertCbId2 = Ctx.registerCreateInstrCallback(
+      [&Order](sandboxir::Instruction *I) { Order.push_back(2); });
+
+  Ctx.save();
+  auto *NewI = sandboxir::StoreInst::create(Val, Ptr, /*Align=*/std::nullopt,
+                                            Ret->getIterator(), Ctx);
+  EXPECT_THAT(Inserted, testing::ElementsAre(NewI));
+  EXPECT_THAT(Removed, testing::IsEmpty());
+  EXPECT_THAT(Moved, testing::IsEmpty());
+  EXPECT_THAT(Order, testing::ElementsAre(1, 2));
+
+  Ret->moveBefore(NewI);
+  EXPECT_THAT(Inserted, testing::ElementsAre(NewI));
+  EXPECT_THAT(Removed, testing::IsEmpty());
+  EXPECT_THAT(Moved, testing::ElementsAre(std::make_pair(Ret, NewI)));
+
+  Ret->eraseFromParent();
+  EXPECT_THAT(Inserted, testing::ElementsAre(NewI));
+  EXPECT_THAT(Removed, testing::ElementsAre(Ret));
+  EXPECT_THAT(Moved, testing::ElementsAre(std::make_pair(Ret, NewI)));
+
+  NewI->eraseFromParent();
+  EXPECT_THAT(Inserted, testing::ElementsAre(NewI));
+  EXPECT_THAT(Removed, testing::ElementsAre(Ret, NewI));
+  EXPECT_THAT(Moved, testing::ElementsAre(std::make_pair(Ret, NewI)));
+
+  // Check that after revert the callbacks have been called for the inverse
+  // operations of the changes made so far.
+  Ctx.revert();
+  EXPECT_THAT(Inserted, testing::ElementsAre(NewI, NewI, Ret));
+  EXPECT_THAT(Removed, testing::ElementsAre(Ret, NewI, NewI));
+  EXPECT_THAT(Moved, testing::ElementsAre(std::make_pair(Ret, NewI),
+                                          std::make_pair(Ret, nullptr)));
+  EXPECT_THAT(Order, testing::ElementsAre(1, 2, 1, 2, 1, 2));
+
+  // Check that deregistration works. Do an operation of each type after
+  // deregistering callbacks and check.
+  Inserted.clear();
+  Removed.clear();
+  Moved.clear();
+  Ctx.unregisterCreateInstrCallback(InsertCbId);
+  Ctx.unregisterEraseInstrCallback(RemoveCbId);
+  Ctx.unregisterMoveInstrCallback(MoveCbId);
+  Ctx.unregisterCreateInstrCallback(CheckOrderInsertCbId1);
+  Ctx.unregisterCreateInstrCallback(CheckOrderInsertCbId2);
+  auto *NewI2 = sandboxir::StoreInst::create(Val, Ptr, /*Align=*/std::nullopt,
+                                             Ret->getIterator(), Ctx);
+  Ret->moveBefore(NewI2);
+  Ret->eraseFromParent();
+  EXPECT_THAT(Inserted, testing::IsEmpty());
+  EXPECT_THAT(Removed, testing::IsEmpty());
+  EXPECT_THAT(Moved, testing::IsEmpty());
+}
-- 
GitLab


From 318bdd0aeb721c8e9bd67101ac6641e5f9d990f2 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Tue, 29 Oct 2024 09:26:47 -0700
Subject: [PATCH 457/693] [StackSafetyAnalysis] Bail out when calling ifunc

An assertion failure arises when a call instruction calls a GlobalIFunc.
Since we cannot reason about the underlying function, just bail out.

Fix #87923

Pull Request: https://github.com/llvm/llvm-project/pull/113841
---
 llvm/lib/Analysis/StackSafetyAnalysis.cpp       |  2 +-
 llvm/test/Analysis/StackSafetyAnalysis/local.ll | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Analysis/StackSafetyAnalysis.cpp b/llvm/lib/Analysis/StackSafetyAnalysis.cpp
index 27360d0e84cb..5d81658409da 100644
--- a/llvm/lib/Analysis/StackSafetyAnalysis.cpp
+++ b/llvm/lib/Analysis/StackSafetyAnalysis.cpp
@@ -528,7 +528,7 @@ void StackSafetyLocalAnalysis::analyzeAllUses(Value *Ptr,
         // dso_preemptable aliases or aliases with interposable linkage.
         const GlobalValue *Callee =
             dyn_cast<GlobalValue>(CB.getCalledOperand()->stripPointerCasts());
-        if (!Callee) {
+        if (!Callee || isa<GlobalIFunc>(Callee)) {
           US.addRange(I, UnknownRange, /*IsSafe=*/false);
           break;
         }
diff --git a/llvm/test/Analysis/StackSafetyAnalysis/local.ll b/llvm/test/Analysis/StackSafetyAnalysis/local.ll
index 4a833611c789..02d46c8449ba 100644
--- a/llvm/test/Analysis/StackSafetyAnalysis/local.ll
+++ b/llvm/test/Analysis/StackSafetyAnalysis/local.ll
@@ -1120,5 +1120,21 @@ define void @NonPointer(ptr %p) {
   ret void
 }
 
+@ifunc = dso_local ifunc i64 (ptr), ptr @ifunc_resolver
+
+define dso_local void @CallIfunc(ptr noundef %uaddr) local_unnamed_addr {
+; CHECK-LABEL: @CallIfunc
+; CHECK-NEXT:  args uses:
+; CHECK-NEXT:    uaddr[]: full-set
+entry:
+  tail call i64 @ifunc(ptr noundef %uaddr)
+  ret void
+}
+
+define dso_local ptr @ifunc_resolver() {
+entry:
+  ret ptr null
+}
+
 declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
 declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
-- 
GitLab


From 2ab98dfe19ac384f0cfac1a1fafc56b9dd7ad9b7 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Tue, 29 Oct 2024 09:45:23 -0700
Subject: [PATCH 458/693] [lldb] Update link to GreenDragon in the docs

---
 lldb/docs/resources/test.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lldb/docs/resources/test.rst b/lldb/docs/resources/test.rst
index 715d3772fe27..5f1bd0d57383 100644
--- a/lldb/docs/resources/test.rst
+++ b/lldb/docs/resources/test.rst
@@ -418,8 +418,8 @@ An overview of all LLDB builders can be found here:
 `https://lab.llvm.org/buildbot/#/builders?tags=lldb <https://lab.llvm.org/buildbot/#/builders?tags=lldb>`_
 
 Building and testing for macOS uses a different platform called GreenDragon. It
-has a dedicated tab for LLDB: `https://green.lab.llvm.org/green/view/LLDB/
-<https://green.lab.llvm.org/green/view/LLDB/>`_
+has a dedicated tab for LLDB: `https://green.lab.llvm.org/job/llvm.org/view/LLDB/
+<https://green.lab.llvm.org/job/llvm.org/view/LLDB/>`_
 
 
 Running The Tests
-- 
GitLab


From 2a9dd8af5ad9783d8ecba6bf93521de64bab6f81 Mon Sep 17 00:00:00 2001
From: SpencerAbson <Spencer.Abson@arm.com>
Date: Tue, 29 Oct 2024 16:55:19 +0000
Subject: [PATCH 459/693] [AArch64] Add assembly/disassembly for zeroing SVE
 FCVT{X} and BFCVT (#113916)

This patch adds assembly/disassembly support for the following SVE2.2
instructions

    - FCVT (zeroing)
    - FCVTX (zeroing)
    - BFCVT (zeroing)

In accordance with:
https://developer.arm.com/documentation/ddi0602/2024-09/SVE-Instructions
---
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |  7 +++
 llvm/lib/Target/AArch64/SVEInstrFormats.td    |  9 +++
 llvm/test/MC/AArch64/SVE/bfcvt-diagnostics.s  |  2 +-
 llvm/test/MC/AArch64/SVE2/fcvtx-diagnostics.s |  2 +-
 .../MC/AArch64/SVE2p2/bfcvt_z-diagnostics.s   | 60 +++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/bfcvt_z.s         | 33 ++++++++++
 .../MC/AArch64/SVE2p2/fcvt_z-diagnostics.s    | 50 ++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/fcvt_z.s          | 57 ++++++++++++++++++
 .../MC/AArch64/SVE2p2/fcvtx_z-diagnostics.s   | 57 ++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/fcvtx_z.s         | 33 ++++++++++
 10 files changed, 308 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/bfcvt_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/bfcvt_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/fcvt_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/fcvt_z.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/fcvtx_z-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/fcvtx_z.s

diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 4f101d0d46b7..5c5ae898a8ac 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4225,15 +4225,22 @@ defm TBLQ_ZZZ  : sve2p1_tblq<"tblq", int_aarch64_sve_tblq>;
 // SME2.2 or SVE2.2 instructions
 //===----------------------------------------------------------------------===//
 let Predicates = [HasSVE2p2orSME2p2] in {
+  // SVE Floating-point convert precision, zeroing predicate
+  defm FCVT_ZPzZ : sve_fp_z2op_p_zd_b_0<"fcvt">;
+
   // SVE2p2 floating-point convert precision down (placing odd), zeroing predicate
   defm FCVTNT_ZPzZ      : sve_fp_fcvtntz<"fcvtnt">;
   def FCVTXNT_ZPzZ_DtoS : sve_fp_fcvt2z<0b0010, "fcvtxnt", ZPR32, ZPR64>;
+  // Placing even
+  def FCVTX_ZPzZ_DtoS   : sve_fp_z2op_p_zd<0b0001010, "fcvtx", ZPR64, ZPR32>;
 
   // SVE2p2 floating-point convert precision up, zeroing predicate
   defm FCVTLT_ZPzZ      : sve_fp_fcvtltz<"fcvtlt">;
 
   // SVE2p2 floating-point convert single-to-bf (placing odd), zeroing predicate
   def BFCVTNT_ZPzZ      : sve_fp_fcvt2z<0b1010, "bfcvtnt", ZPR16, ZPR32>;
+  // Placing corresponding
+  def BFCVT_ZPzZ_StoH   : sve_fp_z2op_p_zd<0b1001010, "bfcvt", ZPR32, ZPR16>;
 
   // Floating-point convert to integer, zeroing predicate
   defm FCVTZS_ZPzZ : sve_fp_z2op_p_zd_d<0b0, "fcvtzs">;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index d1ceb30f36dc..88a0983aa148 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -3207,6 +3207,15 @@ multiclass sve_fp_z2op_p_zd_d_flogb<string asm> {
   def _D : sve_fp_z2op_p_zd<0b0011011, asm, ZPR64, ZPR64>;
 }
 
+multiclass sve_fp_z2op_p_zd_b_0<string asm> {
+  def _StoH : sve_fp_z2op_p_zd<0b1001000, asm, ZPR32, ZPR16>;
+  def _HtoS : sve_fp_z2op_p_zd<0b1001001, asm, ZPR16, ZPR32>;
+  def _DtoH : sve_fp_z2op_p_zd<0b1101000, asm, ZPR64, ZPR16>;
+  def _HtoD : sve_fp_z2op_p_zd<0b1101001, asm, ZPR16, ZPR64>;
+  def _DtoS : sve_fp_z2op_p_zd<0b1101010, asm, ZPR64, ZPR32>;
+  def _StoD : sve_fp_z2op_p_zd<0b1101011, asm, ZPR32, ZPR64>;
+}
+
 //===----------------------------------------------------------------------===//
 // SVE Integer Arithmetic - Binary Predicated Group
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AArch64/SVE/bfcvt-diagnostics.s b/llvm/test/MC/AArch64/SVE/bfcvt-diagnostics.s
index 013f15f8b6e0..6c55ebe4088f 100644
--- a/llvm/test/MC/AArch64/SVE/bfcvt-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE/bfcvt-diagnostics.s
@@ -11,7 +11,7 @@ bfcvt z0.h, p0/m, z1.h
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 bfcvt z0.h, p0/z, z1.s
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: bfcvt z0.h, p0/z, z1.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE2/fcvtx-diagnostics.s b/llvm/test/MC/AArch64/SVE2/fcvtx-diagnostics.s
index 5f3654960336..36c5d5fe9cbe 100644
--- a/llvm/test/MC/AArch64/SVE2/fcvtx-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE2/fcvtx-diagnostics.s
@@ -29,7 +29,7 @@ fcvtx    z0.d, p0/m, z0.d
 // Invalid predicate operation
 
 fcvtx   z0.s, p0/z, z0.d
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: fcvtx   z0.s, p0/z, z0.d
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE2p2/bfcvt_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/bfcvt_z-diagnostics.s
new file mode 100644
index 000000000000..30be5d19c4aa
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/bfcvt_z-diagnostics.s
@@ -0,0 +1,60 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid operand
+
+bfcvt z0.b, p0/z, z0.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: bfcvt z0.b, p0/z, z0.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid element width
+
+bfcvt z0.h, p0/z, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfcvt z0.h, p0/z, z0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfcvt z0.h, p0/z, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfcvt z0.h, p0/z, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfcvt z0.s, p0/z, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfcvt z0.s, p0/z, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfcvt z0.s, p0/z, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfcvt z0.s, p0/z, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfcvt z0.d, p0/z, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfcvt z0.d, p0/z, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Predicate not in restricted predicate range
+
+bfcvt    z0.h, p8/z, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: bfcvt    z0.h, p8/z, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.s, p0/m, z7.s
+bfcvt z0.h, p7/z, z1.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: bfcvt z0.h, p7/z, z1.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+bfcvt z0.h, p7/z, z1.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: bfcvt z0.h, p7/z, z1.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE2p2/bfcvt_z.s b/llvm/test/MC/AArch64/SVE2p2/bfcvt_z.s
new file mode 100644
index 000000000000..9d63ebf1e830
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/bfcvt_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sve2p2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+bfcvt   z0.h, p0/z, z0.s  // 01100100-10011010-11000000-00000000
+// CHECK-INST: bfcvt   z0.h, p0/z, z0.s
+// CHECK-ENCODING: [0x00,0xc0,0x9a,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 649ac000 <unknown>
+
+bfcvt   z21.h, p5/z, z10.s  // 01100100-10011010-11010101-01010101
+// CHECK-INST: bfcvt   z21.h, p5/z, z10.s
+// CHECK-ENCODING: [0x55,0xd5,0x9a,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 649ad555 <unknown>
+
+bfcvt   z31.h, p7/z, z31.s  // 01100100-10011010-11011111-11111111
+// CHECK-INST: bfcvt   z31.h, p7/z, z31.s
+// CHECK-ENCODING: [0xff,0xdf,0x9a,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 649adfff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/fcvt_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/fcvt_z-diagnostics.s
new file mode 100644
index 000000000000..37f4a0ffbe6a
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/fcvt_z-diagnostics.s
@@ -0,0 +1,50 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid operand
+
+fcvt    z0.b, p0/z, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: fcvt    z0.b, p0/z, z0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid element width
+
+fcvt    z0.h, p0/z, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fcvt    z0.h, p0/z, z0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fcvt    z0.s, p0/z, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fcvt    z0.s, p0/z, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fcvt    z0.d, p0/z, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fcvt    z0.d, p0/z, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Predicate not in restricted predicate range
+
+fcvt    z0.s, p8/z, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: fcvt    z0.s, p8/z, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.s, p0/m, z7.s
+fcvt z0.s, p7/z, z1.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: fcvt z0.s, p7/z, z1.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+fcvt z0.s, p7/z, z1.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: fcvt z0.s, p7/z, z1.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE2p2/fcvt_z.s b/llvm/test/MC/AArch64/SVE2p2/fcvt_z.s
new file mode 100644
index 000000000000..6cd9f1ba5032
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/fcvt_z.s
@@ -0,0 +1,57 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sve2p2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// convert to half
+
+fcvt    z0.h, p0/z, z0.s  // 01100100-10011010-10000000-00000000
+// CHECK-INST: fcvt    z0.h, p0/z, z0.s
+// CHECK-ENCODING: [0x00,0x80,0x9a,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 649a8000 <unknown>
+
+fcvt    z23.h, p3/z, z13.d  // 01100100-11011010-10001101-10110111
+// CHECK-INST: fcvt    z23.h, p3/z, z13.d
+// CHECK-ENCODING: [0xb7,0x8d,0xda,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64da8db7 <unknown>
+
+// convert to single
+
+fcvt    z0.s, p0/z, z0.h  // 01100100-10011010-10100000-00000000
+// CHECK-INST: fcvt    z0.s, p0/z, z0.h
+// CHECK-ENCODING: [0x00,0xa0,0x9a,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 649aa000 <unknown>
+
+fcvt    z31.s, p7/z, z31.d  // 01100100-11011010-11011111-11111111
+// CHECK-INST: fcvt    z31.s, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xdf,0xda,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64dadfff <unknown>
+
+// convert to double
+
+fcvt    z21.d, p5/z, z10.h  // 01100100-11011010-10110101-01010101
+// CHECK-INST: fcvt    z21.d, p5/z, z10.h
+// CHECK-ENCODING: [0x55,0xb5,0xda,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64dab555 <unknown>
+
+fcvt    z31.d, p7/z, z31.s  // 01100100-11011010-11111111-11111111
+// CHECK-INST: fcvt    z31.d, p7/z, z31.s
+// CHECK-ENCODING: [0xff,0xff,0xda,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 64daffff <unknown
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/fcvtx_z-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/fcvtx_z-diagnostics.s
new file mode 100644
index 000000000000..d5876773004f
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/fcvtx_z-diagnostics.s
@@ -0,0 +1,57 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid element width
+
+fcvtx z0.b, p0/z, z0.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fcvtx z0.b, p0/z, z0.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fcvtx z0.h, p0/z, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fcvtx z0.h, p0/z, z0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fcvtx z0.s, p0/z, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fcvtx z0.s, p0/z, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fcvtx z0.d, p0/z, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fcvtx z0.d, p0/z, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fcvtx z0.h, p0/z, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fcvtx z0.h, p0/z, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fcvtx z0.b, p0/z, z0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: fcvtx z0.b, p0/z, z0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Predicate not in restricted predicate range
+
+fcvtx    z0.s, p8/z, z0.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: fcvtx    z0.s, p8/z, z0.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z0.s, p0/m, z7.s
+fcvtx z0.s, p7/z, z1.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: fcvtx z0.s, p7/z, z1.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z7
+fcvtx z0.s, p7/z, z1.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: fcvtx z0.s, p7/z, z1.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE2p2/fcvtx_z.s b/llvm/test/MC/AArch64/SVE2p2/fcvtx_z.s
new file mode 100644
index 000000000000..e5e2155ea5d8
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/fcvtx_z.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sve2p2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+fcvtx   z0.s, p0/z, z0.d  // 01100100-00011010-11000000-00000000
+// CHECK-INST: fcvtx   z0.s, p0/z, z0.d
+// CHECK-ENCODING: [0x00,0xc0,0x1a,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641ac000 <unknown>
+
+fcvtx   z23.s, p3/z, z13.d  // 01100100-00011010-11001101-10110111
+// CHECK-INST: fcvtx   z23.s, p3/z, z13.d
+// CHECK-ENCODING: [0xb7,0xcd,0x1a,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641acdb7 <unknown>
+
+fcvtx   z31.s, p7/z, z31.d  // 01100100-00011010-11011111-11111111
+// CHECK-INST: fcvtx   z31.s, p7/z, z31.d
+// CHECK-ENCODING: [0xff,0xdf,0x1a,0x64]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 641adfff <unknown>
-- 
GitLab


From 39ad84e4d173b43dcd13209dc7c62de7a0476c80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Tue, 29 Oct 2024 16:57:23 +0000
Subject: [PATCH 460/693] [mlir][linalg] Split GenericPadOpVectorizationPattern
 into two patterns (#111349)

At the moment, `GenericPadOpVectorizationPattern` implements two
orthogonal transformations:
  1. Rewrites `tensor::PadOp` into a sequence of `tensor::EmptyOp`,
    `linalg::FillOp` and `tensor::InsertSliceOp`.
  2. Vectorizes (where possible) `tensor::InsertSliceOp` (see
    `tryVectorizeCopy`).

This patch splits `GenericPadOpVectorizationPattern` into two separate
patterns:
  1. `GeneralizePadOpPattern` for the first transformation (note that
    currently `GenericPadOpVectorizationPattern` inherits from
    `GeneralizePadOpPattern`).
  2. `InsertSliceVectorizePattern` to vectorize `tensor::InsertSliceOp`.

With this change, we gain the following:
  * a clear separation between pre-processing and vectorization
    transformations/stages,
  * a path to support masked vectorisation for `tensor.insert_slice`
    (with a dedicated pattern for vectorization, it is much easier to
    specify the input vector sizes used in masking),
  * more opportunities to vectorize `tensor.insert_slice`.

Note for downstream users:
--------------------------

If you were using `populatePadOpVectorizationPatterns`, following this
change you will also have to add
`populateInsertSliceVectorizationPatterns`.

Finer implementation details:
-----------------------------

1.  The majority of changes in this patch are copy & paste + some edits.
  1.1. The only functional change is that the vectorization of
    `tensor.insert_slice` is now broadly available (as opposed to being
    constrained to the pad vectorization pattern:
    `GenericPadOpVectorizationPattern`).
  1.2. Following-on from the above, `@pad_and_insert_slice_dest` is
    updated. As expected, the input `tensor.insert_slice` Op is no
    longer "preserved" and instead gets vectorized successfully.

2. The `linalg.fill` case in `getConstantPadVal` works under the
   assumption that only _scalar_ source values can be used. That's
   consistent with the definition of the Op, but it's not tested at the
   moment. Hence a test case in Linalg/invalid.mlir is added.

3. The behaviour of the two TD vectorization Ops,
   `transform.structured.vectorize_children_and_apply_patterns` and
   `transform.structured.vectorize` is preserved.
---
 .../Dialect/Linalg/Transforms/Transforms.h    |  14 +-
 .../TransformOps/LinalgTransformOps.cpp       |   4 +
 .../Dialect/Linalg/Transforms/Transforms.cpp  |   7 +-
 .../Linalg/Transforms/Vectorization.cpp       | 279 +++++++++++-------
 mlir/test/Dialect/Linalg/invalid.mlir         |   9 +
 .../Linalg/vectorization-pad-patterns.mlir    |  11 +-
 .../Linalg/vectorization-unsupported.mlir     |  29 +-
 .../Linalg/vectorization-with-patterns.mlir   | 115 +++++++-
 8 files changed, 327 insertions(+), 141 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index 70b086641bdc..b5710bd78f00 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -1503,18 +1503,13 @@ using OptimizeCopyFn =
 
 /// Rewrite a tensor::PadOp into a sequence of EmptyOp, FillOp and
 /// InsertSliceOp. For now, only constant padding values are supported.
-/// `OptimizeCopyFn` can be used to customize copying step optimization.
 struct GeneralizePadOpPattern : public OpRewritePattern<tensor::PadOp> {
-  GeneralizePadOpPattern(MLIRContext *context,
-                         OptimizeCopyFn optimizeCopyFn = nullptr,
-                         PatternBenefit benefit = 1)
-      : OpRewritePattern<tensor::PadOp>(context, benefit),
-        optimizeCopyFn(std::move(optimizeCopyFn)) {}
+  GeneralizePadOpPattern(MLIRContext *context, PatternBenefit benefit = 1)
+      : OpRewritePattern<tensor::PadOp>(context, benefit) {}
   LogicalResult matchAndRewrite(tensor::PadOp padOp,
                                 PatternRewriter &rewriter) const override;
 
 protected:
-  OptimizeCopyFn optimizeCopyFn;
   Value createFillOrGenerateOp(RewriterBase &rewriter, tensor::PadOp padOp,
                                Value dest,
                                const SmallVector<Value> &dynSizes) const;
@@ -1663,6 +1658,11 @@ void populateDecomposeConvolutionPatterns(RewritePatternSet &patterns,
 /// \see rewriteInIm2Col for more details.
 void populateConvertConv2DToImg2ColPatterns(RewritePatternSet &patterns);
 
+/// Populates `patterns` with vectorisation patterns for tensor.insert_slice.
+/// TODO: Avoid having a dedicated `populate{}` for one pattern. Instead, either
+/// expand or merge with other `populate{}`.
+void populateInsertSliceVectorizationPatterns(RewritePatternSet &patterns);
+
 /// Populates `patterns` with patterns that vectorize tensor.pad.
 /// These patterns are meant to apply in a complementary fashion. Benefits
 /// are used to encode a certain ordering of pattern application. To avoid
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 3d3f0a93a382..9c0ab4f41b85 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -256,6 +256,7 @@ void transform::ApplyFoldAddIntoDestPatternsOp::populatePatterns(
 void transform::ApplyPadVectorizationPatternsOp::populatePatterns(
     RewritePatternSet &patterns) {
   linalg::populatePadOpVectorizationPatterns(patterns);
+  linalg::populateInsertSliceVectorizationPatterns(patterns);
 }
 
 //===----------------------------------------------------------------------===//
@@ -3482,6 +3483,9 @@ transform::VectorizeChildrenAndApplyPatternsOp::applyToOne(
 
   patterns.add<CopyVectorizationPattern>(ctx);
 
+  // Add misc. vectorization patterns (e.g. for tensor.insert_slice)
+  linalg::populateInsertSliceVectorizationPatterns(patterns);
+
   if (getVectorizePadding())
     linalg::populatePadOpVectorizationPatterns(patterns);
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index 0fe096863d7b..da5233049aaf 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -973,12 +973,7 @@ GeneralizePadOpPattern::matchAndRewrite(tensor::PadOp padOp,
       padOp.getLoc(), staticSizes, resultType.getElementType(), dynSizes);
   Value fill = createFillOrGenerateOp(rewriter, padOp, emptyTensor, dynSizes);
 
-  // Try optimize the copy of source.
-  if (optimizeCopyFn && optimizeCopyFn(rewriter, padOp, fill).succeeded())
-    return success();
-
-  // tensor::PadOps cannot be optimized. Generate a InsertSliceOp instead
-  // for copying the PadOp source.
+  // Generate a InsertSliceOp for copying the PadOp source.
   auto sourceType = padOp.getSourceType();
   // Compute size of source of tensor::PadOp.
   SmallVector<OpFoldResult> srcSizes =
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 0a2457176a1d..090e0b46768d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -2281,115 +2281,6 @@ LogicalResult mlir::linalg::vectorizeCopy(RewriterBase &rewriter,
 //----------------------------------------------------------------------------//
 // Misc. vectorization patterns.
 //----------------------------------------------------------------------------//
-
-/// Helper function that retrieves the value of an IntegerAttr.
-static int64_t getIntFromAttr(Attribute attr) {
-  return cast<IntegerAttr>(attr).getInt();
-}
-
-/// Given an ArrayRef of OpFoldResults, return a vector of Values.
-/// IntegerAttrs are converted to ConstantIndexOps. Other attribute types are
-/// not supported.
-static SmallVector<Value> ofrToIndexValues(RewriterBase &rewriter, Location loc,
-                                           ArrayRef<OpFoldResult> ofrs) {
-  SmallVector<Value> result;
-  for (auto o : ofrs) {
-    if (auto val = llvm::dyn_cast_if_present<Value>(o)) {
-      result.push_back(val);
-    } else {
-      result.push_back(rewriter.create<arith::ConstantIndexOp>(
-          loc, getIntFromAttr(o.template get<Attribute>())));
-    }
-  }
-  return result;
-}
-
-/// Rewrite a tensor::PadOp into a sequence of EmptyOp, FillOp and
-/// InsertSliceOp. For now, only constant padding values are supported.
-/// If there is enough static type information, TransferReadOps and
-/// TransferWriteOps may be generated instead of InsertSliceOps.
-struct GenericPadOpVectorizationPattern : public GeneralizePadOpPattern {
-  GenericPadOpVectorizationPattern(MLIRContext *context,
-                                   PatternBenefit benefit = 1)
-      : GeneralizePadOpPattern(context, tryVectorizeCopy, benefit) {}
-  /// Vectorize the copying of a tensor::PadOp's source. This is possible if
-  /// each dimension size is statically know in the source type or the result
-  /// type (or both).
-  static LogicalResult tryVectorizeCopy(RewriterBase &rewriter,
-                                        tensor::PadOp padOp, Value dest) {
-    auto sourceType = padOp.getSourceType();
-    auto resultType = padOp.getResultType();
-    if (!VectorType::isValidElementType(sourceType.getElementType()))
-      return failure();
-
-    // Copy cannot be vectorized if pad value is non-constant and source shape
-    // is dynamic. In case of a dynamic source shape, padding must be appended
-    // by TransferReadOp, but TransferReadOp supports only constant padding.
-    auto padValue = padOp.getConstantPaddingValue();
-    if (!padValue) {
-      if (!sourceType.hasStaticShape())
-        return failure();
-      // Create dummy padding value.
-      auto elemType = sourceType.getElementType();
-      padValue = rewriter.create<arith::ConstantOp>(
-          padOp.getLoc(), elemType, rewriter.getZeroAttr(elemType));
-    }
-
-    SmallVector<int64_t> vecShape;
-    SmallVector<bool> readInBounds;
-    SmallVector<bool> writeInBounds;
-    for (unsigned i = 0; i < sourceType.getRank(); ++i) {
-      if (!sourceType.isDynamicDim(i)) {
-        vecShape.push_back(sourceType.getDimSize(i));
-        // Source shape is statically known: Neither read nor write are
-        // out-of- bounds.
-        readInBounds.push_back(true);
-        writeInBounds.push_back(true);
-      } else if (!resultType.isDynamicDim(i)) {
-        // Source shape is not statically known, but result shape is.
-        // Vectorize with size of result shape. This may be larger than the
-        // source size.
-        vecShape.push_back(resultType.getDimSize(i));
-        // Read may be out-of-bounds because the result size could be larger
-        // than the source size.
-        readInBounds.push_back(false);
-        // Write is out-of-bounds if low padding > 0.
-        writeInBounds.push_back(
-            getConstantIntValue(padOp.getMixedLowPad()[i]) ==
-            static_cast<int64_t>(0));
-      } else {
-        // Neither source nor result dim of padOp is static. Cannot vectorize
-        // the copy.
-        return failure();
-      }
-    }
-    auto vecType = VectorType::get(vecShape, sourceType.getElementType());
-
-    // Generate TransferReadOp.
-    SmallVector<Value> readIndices(
-        vecType.getRank(),
-        rewriter.create<arith::ConstantIndexOp>(padOp.getLoc(), 0));
-    auto read = rewriter.create<vector::TransferReadOp>(
-        padOp.getLoc(), vecType, padOp.getSource(), readIndices, padValue,
-        ArrayRef<bool>{readInBounds});
-
-    // If `dest` is a FillOp and the TransferWriteOp would overwrite the
-    // entire tensor, write directly to the FillOp's operand.
-    if (llvm::equal(vecShape, resultType.getShape()) &&
-        llvm::all_of(writeInBounds, [](bool b) { return b; }))
-      if (auto fill = dest.getDefiningOp<FillOp>())
-        dest = fill.output();
-
-    // Generate TransferWriteOp.
-    auto writeIndices =
-        ofrToIndexValues(rewriter, padOp.getLoc(), padOp.getMixedLowPad());
-    rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
-        padOp, read, dest, writeIndices, ArrayRef<bool>{writeInBounds});
-
-    return success();
-  }
-};
-
 /// Base pattern for rewriting tensor::PadOps whose result is consumed by a
 /// given operation type OpTy.
 template <typename OpTy>
@@ -2623,6 +2514,163 @@ struct PadOpVectorizationWithTransferWritePattern
   }
 };
 
+/// Returns the effective Pad value for the input op, provided it's a scalar.
+///
+/// Many Ops exhibit pad-like behaviour, but this isn't always explicit. If
+/// this Op performs padding, retrieve the padding value provided that it's
+/// a scalar and static/fixed for all the padded values. Returns an empty value
+/// otherwise.
+static Value getStaticPadVal(Operation *op) {
+  if (!op)
+    return {};
+
+  // 1. vector.broadcast (f32 -> vector <...xf32>) - return the value that's
+  // being broadcast, provided that it's a scalar.
+  if (auto bcast = llvm::dyn_cast<vector::BroadcastOp>(op)) {
+    auto source = bcast.getSource();
+    if (llvm::dyn_cast<VectorType>(source.getType()))
+      return {};
+
+    return source;
+  }
+
+  // 2. linalg.fill - use the scalar input value that used to fill the output
+  // tensor.
+  if (auto fill = llvm::dyn_cast<linalg::FillOp>(op)) {
+    return fill.getInputs()[0];
+  }
+
+  // 3. tensor.generateOp - can't guarantee the value is fixed without
+  // analysing, bail out.
+  if (auto generate = llvm::dyn_cast<tensor::GenerateOp>(op)) {
+    return {};
+  }
+
+  // 4. vector.transfer_write - inspect the input vector that's written from. If
+  // if contains a single value that has been broadcast (e.g. via
+  // vector.broadcast), extract it, fail otherwise.
+  if (auto xferWrite = llvm::dyn_cast<vector::TransferWriteOp>(op))
+    return getStaticPadVal(xferWrite.getVector().getDefiningOp());
+
+  // 5. tensor.insert_slice - inspect the destination tensor. If it's larger
+  // than the input tensor, then, provided it's constant, we'll extract the
+  // value that was used to generate it (via e.g. linalg.fill), fail otherwise.
+  // TODO: Clarify the semantics when the input tensor is larger than the
+  // destination.
+  if (auto slice = llvm::dyn_cast<tensor::InsertSliceOp>(op))
+    return getStaticPadVal(slice.getDest().getDefiningOp());
+
+  return {};
+}
+
+/// Rewrite tensor.insert.slice as a vector.transfer_read +
+/// vector.transfer_write pair. The vector size is inferred from the static
+/// dims in the input and output tensors. If a dim is dynamic in both the input
+/// and output tensors, bails out.
+///
+/// Before:
+///     !t_in_type = tensor<1x2x3xf32>
+///     !t_out_type = tensor<9x8x7x1x2x3xf32>
+///     !v_type = vector<1x2x3xf32>
+///     %inserted_slice = tensor.insert_slice %src into %dest ... : !t_in_type
+///     into !t_out_type
+/// After:
+///     %read = vector.transfer_read %src[...], %pad ... : !t_in_type, !v_type
+///     %write = vector.transfer_write %read, %dest ... : !v_type, !t_out_type
+///
+/// TODO: Support masking
+struct InsertSliceVectorizePattern
+    : public OpRewritePattern<tensor::InsertSliceOp> {
+  using OpRewritePattern<tensor::InsertSliceOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(tensor::InsertSliceOp sliceOp,
+                                PatternRewriter &rewriter) const final {
+    auto sourceType = sliceOp.getSource().getType();
+    if (!VectorType::isValidElementType(sourceType.getElementType()))
+      return failure();
+
+    auto resultType = sliceOp.getResultType();
+
+    // 1. Get the pad value.
+    // TransferReadOp requires a scalar padding value. Note that:
+    //    * for in-bounds access, the value is actually irrelevant.
+    //  There are 2 cases in which xfer.read accesses are known to be in-bounds:
+    //  1. The source shape is static (output vector sizes would be based on
+    //     the source shape and hence all memory accesses would be in-bounds),
+    //  2. Masking is used (output vector sizes would be user-provided, in which
+    //     case it is assumed that all memory accesses are in-bounds). This
+    //     remains a TODO.
+    //
+    // When the value is not known and not needed, use 0. Otherwise, bail out.
+    Value padValue = getStaticPadVal(sliceOp);
+    bool isOutOfBoundsRead = !sourceType.hasStaticShape();
+
+    if (!padValue && isOutOfBoundsRead) {
+      LDBG("Failed to get a pad value for out-of-bounds read access\n");
+      return failure();
+    }
+
+    if (!padValue) {
+      auto elemType = sourceType.getElementType();
+      padValue = rewriter.create<arith::ConstantOp>(
+          sliceOp.getLoc(), elemType, rewriter.getZeroAttr(elemType));
+    }
+
+    // 2. Get the vector shape and in-bounds attributes
+    SmallVector<int64_t> vecShape;
+    SmallVector<bool> readInBounds;
+    SmallVector<bool> writeInBounds;
+    size_t rankDiff = resultType.getRank() - sourceType.getRank();
+    for (unsigned i = 0; i < sourceType.getRank(); ++i) {
+      if (!sourceType.isDynamicDim(i)) {
+        vecShape.push_back(sourceType.getDimSize(i));
+        // Source shape is statically known: Neither read nor write are
+        // out-of-bounds.
+        readInBounds.push_back(true);
+        writeInBounds.push_back(true);
+      } else if (!resultType.isDynamicDim(i)) {
+        // Source shape is not statically known, but result shape is.
+        // Vectorize with size of result shape. This may be larger than the
+        // source size.
+        // FIXME: Using rankDiff implies that the source tensor is inserted at
+        // the end of the destination tensor. However, that's not required.
+        vecShape.push_back(resultType.getDimSize(rankDiff + i));
+        // Read may be out-of-bounds because the result size could be larger
+        // than the source size.
+        readInBounds.push_back(false);
+        // Write will in-bounds provided that the corresponding write idx is 0.
+        // To keep this logic simple, conservatively mark as out-of-bounds.
+        writeInBounds.push_back(false);
+      } else {
+        // Neither source nor result dim of padOp is static. Cannot vectorize
+        // the copy.
+        // TODO: Add support for masking
+        return failure();
+      }
+    }
+    auto vecType = VectorType::get(vecShape, sourceType.getElementType());
+
+    // 3. Generate TransferReadOp.
+    SmallVector<Value> readIndices(
+        vecType.getRank(),
+        rewriter.create<arith::ConstantIndexOp>(sliceOp.getLoc(), 0));
+    auto read = rewriter.create<vector::TransferReadOp>(
+        sliceOp.getLoc(), vecType, sliceOp.getSource(), readIndices, padValue,
+        ArrayRef<bool>{readInBounds});
+
+    // 4. Generate TransferWriteOp.
+    auto writeIndices = getValueOrCreateConstantIndexOp(
+        rewriter, sliceOp.getLoc(), sliceOp.getMixedOffsets());
+
+    // 5. Finalize
+    rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
+        sliceOp, read, sliceOp.getDest(), writeIndices,
+        ArrayRef<bool>{writeInBounds});
+
+    return success();
+  }
+};
+
 /// Rewrite use of tensor::PadOp result in InsertSliceOp. E.g.:
 /// ```
 /// %0 = tensor.pad %src ... : tensor<?x?xf32> to tensor<17x5xf32>
@@ -2699,8 +2747,8 @@ struct PadOpVectorizationWithInsertSlicePattern
     // Generate TransferWriteOp: Write to InsertSliceOp's dest tensor at
     // specified offsets. Write is fully in-bounds because a InsertSliceOp's
     // source must fit into the destination at the specified offsets.
-    auto writeIndices =
-        ofrToIndexValues(rewriter, padOp.getLoc(), insertOp.getMixedOffsets());
+    auto writeIndices = getValueOrCreateConstantIndexOp(
+        rewriter, padOp.getLoc(), insertOp.getMixedOffsets());
     SmallVector<bool> inBounds(vecRank, true);
     rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
         insertOp, read, insertOp.getDest(), writeIndices,
@@ -2710,13 +2758,18 @@ struct PadOpVectorizationWithInsertSlicePattern
   }
 };
 
+void mlir::linalg::populateInsertSliceVectorizationPatterns(
+    RewritePatternSet &patterns) {
+  patterns.add<InsertSliceVectorizePattern>(patterns.getContext());
+}
+
 void mlir::linalg::populatePadOpVectorizationPatterns(
     RewritePatternSet &patterns, PatternBenefit baseBenefit) {
   // TODO: The following pattern implements "decomposition" and
   // optional "vectorization". Seperate "decomposition" into a sepereate
   // pre-processing pattern group.
-  patterns.add<GenericPadOpVectorizationPattern>(patterns.getContext(),
-                                                 baseBenefit);
+  patterns.add<GeneralizePadOpPattern>(patterns.getContext(), baseBenefit);
+
   // Try these specialized patterns first before resorting to the generic one.
   patterns.add<PadOpVectorizationWithTransferReadPattern,
                PadOpVectorizationWithTransferWritePattern,
diff --git a/mlir/test/Dialect/Linalg/invalid.mlir b/mlir/test/Dialect/Linalg/invalid.mlir
index c481a723c562..4b5a66f8fb5b 100644
--- a/mlir/test/Dialect/Linalg/invalid.mlir
+++ b/mlir/test/Dialect/Linalg/invalid.mlir
@@ -352,6 +352,15 @@ func.func @illegal_fill_tensor_with_memref_return
 
 // -----
 
+func.func @illegal_fill_value_type(%arg0 : tensor<2x2xf32>, %arg1 : tensor<2xf32>) -> tensor<2x2xf32>
+{
+  // expected-error @+1 {{expected op with scalar input}}
+  %0 = linalg.fill ins(%arg1 : tensor<2xf32>) outs(%arg0 : tensor<2x2xf32>) -> tensor<2x2xf32>
+  return %0 : tensor<2x2xf32>
+}
+
+// -----
+
 func.func @invalid_static_matmul(%arg0: memref<2x4xf32>, %arg1: memref<3x4xf32>, %arg2: memref<2x4xf32>) {
   // expected-error @+1 {{inferred input/output operand #1 has shape's dimension #0 to be 4, but found 3}}
   linalg.matmul ins(%arg0, %arg1 : memref<2x4xf32>, memref<3x4xf32>)
diff --git a/mlir/test/Dialect/Linalg/vectorization-pad-patterns.mlir b/mlir/test/Dialect/Linalg/vectorization-pad-patterns.mlir
index 2aa4638af3f0..640de85cc5f1 100644
--- a/mlir/test/Dialect/Linalg/vectorization-pad-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization-pad-patterns.mlir
@@ -161,7 +161,8 @@ module attributes {transform.with_named_sequence} {
 
 ///----------------------------------------------------------------------------------------
 /// tensor::PadOp -> tensor::EmptyOp + linalg::FillOp/tensor::GenerateOp + tensor::InsertSliceOp
-/// [Pattern: GenericPadOpVectorizationPattern]
+/// [Pattern: GenericPadOpVectorizationPattern + InsertSliceVectorizePattern]
+/// TODO: Split the test into two, one for each pattern.
 ///----------------------------------------------------------------------------------------
 
 func.func private @make_vector() -> tensor<12x13xf32>
@@ -174,12 +175,14 @@ func.func private @make_vector() -> tensor<12x13xf32>
 //  CHECK-NOT:     tensor.pad
 //  CHECK-DAG:     %[[C0:.*]] = arith.constant 0 : index
 //  CHECK-DAG:     %[[PAD:.*]] = arith.constant 5.000000e+00 : f32
+//  CHECK-DAG:     %[[PAD_READ:.*]] = arith.constant 0.000000e+00 : f32
 //      CHECK:     %[[EMPTY:.*]] = tensor.empty() : tensor<1x12x13xf32>
 //      CHECK:     %[[FILL:.*]] = linalg.fill ins(%[[PAD]] : f32) outs(%[[EMPTY]] : tensor<1x12x13xf32>) -> tensor<1x12x13xf32>
-//      CHECK:     %[[READ:.*]] = vector.transfer_read %[[ARG_0]]{{\[}}%[[C0]], %[[C0]], %[[C0]]], %[[PAD]] {in_bounds = [true, true, true]} : tensor<1x5x6xf32>, vector<1x5x6xf32>
-//      CHECK:     %[[WRITE:.*]] = vector.transfer_write %[[READ]], %[[FILL]]{{\[}}%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x5x6xf32>, tensor<1x12x13xf32>
+//      CHECK:     %[[READ_1:.*]] = vector.transfer_read %[[ARG_0]]{{\[}}%[[C0]], %[[C0]], %[[C0]]], %[[PAD]] {in_bounds = [true, true, true]} : tensor<1x5x6xf32>, vector<1x5x6xf32>
+//      CHECK:     %[[WRITE_1:.*]] = vector.transfer_write %[[READ_1]], %[[FILL]]{{\[}}%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x5x6xf32>, tensor<1x12x13xf32>
 //      CHECK:     %[[VEC:.*]] = call @make_vector() : () -> tensor<12x13xf32>
-//      CHECK:     %[[RES:.*]] = tensor.insert_slice %[[VEC]] into %[[WRITE]][0, 0, 0] [1, 12, 13] [1, 1, 1] : tensor<12x13xf32> into tensor<1x12x13xf32>
+//      CHECK:     %[[READ_2:.*]] = vector.transfer_read %[[VEC]]{{\[}}%[[C0]], %[[C0]]], %[[PAD_READ]] {in_bounds = [true, true]} : tensor<12x13xf32>, vector<12x13xf32>
+//      CHECK:     %[[RES:.*]] = vector.transfer_write %[[READ_2]], %[[WRITE_1]]{{\[}}%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<12x13xf32>, tensor<1x12x13xf32>
 //      CHECK:     return %[[RES]] : tensor<1x12x13xf32>
 
 func.func @pad_and_insert_slice_dest(
diff --git a/mlir/test/Dialect/Linalg/vectorization-unsupported.mlir b/mlir/test/Dialect/Linalg/vectorization-unsupported.mlir
index e9f8e08ca0c6..8fbc74ec345c 100644
--- a/mlir/test/Dialect/Linalg/vectorization-unsupported.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization-unsupported.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -transform-interpreter -split-input-file -verify-diagnostics
+// RUN: mlir-opt %s -transform-interpreter -split-input-file -verify-diagnostics | FileCheck %s
 
 func.func @conv1d_nwc_wcf_dyn_ch_dim(%input: memref<4x6x?xf32>, %filter: memref<1x?x8xf32>, %output: memref<4x2x8xf32>) {
   // expected-error @+1 {{Attempted to vectorize, but failed}}
@@ -253,3 +253,30 @@ module attributes {transform.with_named_sequence} {
     transform.yield
   }
 }
+
+// -----
+
+// With dynamically shaped source, the vectorizer infers the vector size for
+// xfer Ops from the destination tensor and, conservatively, assumes
+// out-of-bounds accesses. Out-of-bounds accesses require a pad value, but
+// that's impossible to recover in this example. Hence no vectorization.
+
+// TODO: Use diagnostics once we can vectorize tensor.insert_slice with
+// transform.structured.vectorize
+
+// CHECK-LABEL: @insert_dynamic_slice_unknown_pad
+// CHECK-NOT: vector
+// CHECK: tensor.insert_slice
+func.func @insert_dynamic_slice_unknown_pad(%arg0: tensor<1x?x3xf32>, %arg1: tensor<9x8x7x1x2x3xf32>, %size: index) -> tensor<9x8x7x1x2x3xf32> {
+  %res = tensor.insert_slice %arg0 into %arg1[0, 0, 0, 0, 0, 0] [1, 1, 1, 1, %size, 3][1, 1, 1, 1, 1, 1] : tensor<1x?x3xf32> into tensor<9x8x7x1x2x3xf32>
+  return %res : tensor<9x8x7x1x2x3xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["tensor.insert_slice"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir b/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
index 189507d97d6d..d2fb3730a2d2 100644
--- a/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
@@ -939,16 +939,20 @@ module attributes {transform.with_named_sequence} {
 
 func.func private @make_vector() -> tensor<12x13xf32>
 
-// CHECK-LABEL: func @pad_and_insert_slice_dest
-//  CHECK-SAME:     %[[ARG0:.*]]: tensor<1x5x6xf32>
-// Check the insert slice is not rewritten if the padded result is used by the destination operand.
-//   CHECK-NOT:   tensor.pad
-//       CHECK:   %[[EMPTY:.*]] = tensor.empty() : tensor<1x12x13xf32>
-//       CHECK:   %[[WRITE_1:.*]] = vector.transfer_write %{{.*}}, %[[EMPTY]]{{.*}} : vector<1x12x13xf32>, tensor<1x12x13xf32>
-//       CHECK:   %[[READ:.*]]  = vector.transfer_read %[[ARG0:.*]]{{.*}} : tensor<1x5x6xf32>, vector<1x5x6xf32>
-//       CHECK:   %[[WRITE_2:.*]] = vector.transfer_write %[[READ]], %[[WRITE_1]]{{.*}} : vector<1x5x6xf32>, tensor<1x12x13xf32>
-//       CHECK:   %[[T1:.*]] = call @make_vector() : () -> tensor<12x13xf32>
-//       CHECK:   tensor.insert_slice %[[T1]] into %[[WRITE_2]]
+// CHECK-LABEL:   func.func @pad_and_insert_slice_dest(
+// CHECK-SAME:      %[[ARG_0:.*]]: tensor<1x5x6xf32>) -> tensor<1x12x13xf32> {
+// CHECK:           %[[C0:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[CST:.*]] = arith.constant dense<5.000000e+00> : vector<1x12x13xf32>
+// CHECK:           %[[C0_IDX:.*]] = arith.constant 0 : index
+// CHECK:           %[[PAD_VAL:.*]] = arith.constant 5.000000e+00 : f32
+// CHECK:           %[[EMPTY:.*]] = tensor.empty() : tensor<1x12x13xf32>
+// CHECK:           %[[WRITE_1:.*]] = vector.transfer_write %[[CST]], %[[EMPTY]]{{\[}}%[[C0_IDX]], %[[C0_IDX]], %[[C0_IDX]]] {in_bounds = [true, true, true]} : vector<1x12x13xf32>, tensor<1x12x13xf32>
+// CHECK:           %[[READ_1:.*]] = vector.transfer_read %[[ARG_0]]{{\[}}%[[C0_IDX]], %[[C0_IDX]], %[[C0_IDX]]], %[[PAD_VAL]] {in_bounds = [true, true, true]} : tensor<1x5x6xf32>, vector<1x5x6xf32>
+// CHECK:           %[[WRITE_2:.*]] = vector.transfer_write %[[READ_1]], %[[WRITE_1]]{{\[}}%[[C0_IDX]], %[[C0_IDX]], %[[C0_IDX]]] {in_bounds = [true, true, true]} : vector<1x5x6xf32>, tensor<1x12x13xf32>
+// CHECK:           %[[MAKE_VEC:.*]] = call @make_vector() : () -> tensor<12x13xf32>
+// CHECK:           %[[READ_2:.*]] = vector.transfer_read %[[MAKE_VEC]]{{\[}}%[[C0_IDX]], %[[C0_IDX]]], %[[C0]] {in_bounds = [true, true]} : tensor<12x13xf32>, vector<12x13xf32>
+// CHECK:           %[[RES:.*]] = vector.transfer_write %[[READ_2]], %[[WRITE_2]]{{\[}}%[[C0_IDX]], %[[C0_IDX]], %[[C0_IDX]]] {in_bounds = [true, true]} : vector<12x13xf32>, tensor<1x12x13xf32>
+// CHECK:           return %[[RES]] : tensor<1x12x13xf32>
 func.func @pad_and_insert_slice_dest(
     %arg0: tensor<1x5x6xf32>) -> tensor<1x12x13xf32> {
   %c5 = arith.constant 5.0 : f32
@@ -1924,3 +1928,94 @@ module attributes {transform.with_named_sequence} {
     transform.yield
   }
 }
+
+// -----
+
+///----------------------------------------------------------------------------------------
+/// tensor.insert_slice
+///----------------------------------------------------------------------------------------
+
+// The pad value for xfer-read is neither needed nor available - use the default (0.0).
+
+// CHECK-LABEL: func @insert_static_slice_default_pad
+// CHECK-SAME:      %[[ARG_0:.*]]: tensor<1x2x3xf32>,
+// CHECK-SAME:      %[[ARG_1:.*]]: tensor<9x8x7x1x2x3xf32>) -> tensor<9x8x7x1x2x3xf32> {
+// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[C0:.*]] = arith.constant 0 : index
+// CHECK:           %[[READ:.*]] = vector.transfer_read %[[ARG_0]]{{\[}}%[[C0]], %[[C0]], %[[C0]]], %[[PAD]] {in_bounds = [true, true, true]} : tensor<1x2x3xf32>, vector<1x2x3xf32>
+// CHECK:           %[[WRITE:.*]] = vector.transfer_write %[[READ]], %[[ARG_1]]{{\[}}%[[C0]], %[[C0]], %[[C0]], %[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x2x3xf32>, tensor<9x8x7x1x2x3xf32>
+// CHECK:           return %[[WRITE]] : tensor<9x8x7x1x2x3xf32>
+func.func @insert_static_slice_default_pad(%arg0: tensor<1x2x3xf32>, %arg1: tensor<9x8x7x1x2x3xf32>) -> tensor<9x8x7x1x2x3xf32> {
+  %res = tensor.insert_slice %arg0 into %arg1[0, 0, 0, 0, 0, 0] [1, 1, 1, 1, 2, 3][1, 1, 1, 1, 1, 1] : tensor<1x2x3xf32> into tensor<9x8x7x1x2x3xf32>
+  return %res : tensor<9x8x7x1x2x3xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["tensor.insert_slice"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_padding } : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// Same as above, but there's a pad value available that should be used instead of the default value.
+
+// CHECK-LABEL:   func.func @insert_static_slice_non_zero_pad
+// CHECK-SAME:      %[[ARG_0:.*]]: tensor<1x2x3xf32>,
+// CHECK-SAME:      %[[PAD:.*]]: f32) -> tensor<9x8x7x1x2x3xf32> {
+// CHECK:           %[[EMPTY:.*]] = tensor.empty() : tensor<9x8x7x1x2x3xf32>
+// CHECK:           %[[BC:.*]] = vector.broadcast %[[PAD]] : f32 to vector<9x8x7x1x2x3xf32>
+// CHECK:           %[[WRITE:.*]] = vector.transfer_write %[[BC]], %[[EMPTY]]{{.*}} {in_bounds = [true, true, true, true, true, true]} : vector<9x8x7x1x2x3xf32>, tensor<9x8x7x1x2x3xf32>
+// CHECK:           %[[READ:.*]] = vector.transfer_read %[[ARG_0]]{{.*}}, %[[PAD]] {in_bounds = [true, true, true]} : tensor<1x2x3xf32>, vector<1x2x3xf32>
+// CHECK:           %[[RES:.*]] = vector.transfer_write %[[READ]], %[[WRITE]]{{.*}} {in_bounds = [true, true, true]} : vector<1x2x3xf32>, tensor<9x8x7x1x2x3xf32>
+// CHECK:           return %[[RES]] : tensor<9x8x7x1x2x3xf32>
+func.func @insert_static_slice_non_zero_pad(%arg0: tensor<1x2x3xf32>, %pad : f32) -> tensor<9x8x7x1x2x3xf32> {
+  %init = tensor.empty() : tensor<9x8x7x1x2x3xf32>
+  %fill = linalg.fill ins(%pad : f32) outs(%init : tensor<9x8x7x1x2x3xf32>) -> tensor<9x8x7x1x2x3xf32>
+  %res = tensor.insert_slice %arg0 into %fill[0, 0, 0, 0, 0, 0] [1, 1, 1, 1, 2, 3][1, 1, 1, 1, 1, 1] : tensor<1x2x3xf32> into tensor<9x8x7x1x2x3xf32>
+  return %res : tensor<9x8x7x1x2x3xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["tensor.insert_slice"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// Same as above, but the source type has is dynamically shaped. This means
+// that the pad value is now required and the vector dim corresponding to the
+// dynamic shape has to be inferred from the shape of the destination tensor.
+
+// CHECK-LABEL:   func.func @insert_dynamic_slice_non_zero_pad(
+// CHECK-SAME:      %[[ARG_0:.*]]: tensor<1x?x3xf32>,
+// CHECK-SAME:      %[[PAD:.*]]: f32,
+// CHECK-SAME:      %[[SIZE:.*]]: index) -> tensor<9x8x7x1x2x3xf32> {
+// CHECK:           %[[EMPTY:.*]] = tensor.empty() : tensor<9x8x7x1x2x3xf32>
+// CHECK:           %[[BC:.*]] = vector.broadcast %[[PAD]] : f32 to vector<9x8x7x1x2x3xf32>
+// CHECK:           %[[WRITE:.*]] = vector.transfer_write %[[BC]], %[[EMPTY]]{{.*}} {in_bounds = [true, true, true, true, true, true]} : vector<9x8x7x1x2x3xf32>, tensor<9x8x7x1x2x3xf32>
+// CHECK:           %[[READ:.*]] = vector.transfer_read %[[ARG_0]]{{.*}}, %[[PAD]] {in_bounds = [true, false, true]} : tensor<1x?x3xf32>, vector<1x2x3xf32>
+// CHECK:           %[[RES:.*]] = vector.transfer_write %[[READ]], %[[WRITE]]{{.*}} {in_bounds = [true, true, true]} : vector<1x2x3xf32>, tensor<9x8x7x1x2x3xf32>
+// CHECK:           return %[[RES]] : tensor<9x8x7x1x2x3xf32>
+func.func @insert_dynamic_slice_non_zero_pad(%arg0: tensor<1x?x3xf32>, %pad : f32, %size: index) -> tensor<9x8x7x1x2x3xf32> {
+  %init = tensor.empty() : tensor<9x8x7x1x2x3xf32>
+  %fill = linalg.fill ins(%pad : f32) outs(%init : tensor<9x8x7x1x2x3xf32>) -> tensor<9x8x7x1x2x3xf32>
+  %res = tensor.insert_slice %arg0 into %fill[0, 0, 0, 0, 0, 0] [1, 1, 1, 1, %size, 3][1, 1, 1, 1, 1, 1] : tensor<1x?x3xf32> into tensor<9x8x7x1x2x3xf32>
+  return %res : tensor<9x8x7x1x2x3xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["tensor.insert_slice"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
-- 
GitLab


From 12a8f504cfe25afab97e288a44e1d5b1925d24cf Mon Sep 17 00:00:00 2001
From: Jorge Gorbe Moya <jgorbe@google.com>
Date: Tue, 29 Oct 2024 09:56:15 -0700
Subject: [PATCH 461/693] [SandboxIR] Use the proper gmock public header in
 unit tests.

This should fix the BuildKite bazel build.
---
 llvm/unittests/SandboxIR/SandboxIRTest.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index 99e14292a91b..874c32c2d439 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -21,8 +21,7 @@
 #include "llvm/SandboxIR/Utils.h"
 #include "llvm/SandboxIR/Value.h"
 #include "llvm/Support/SourceMgr.h"
-#include "gmock/gmock-matchers.h"
-#include "gmock/gmock-more-matchers.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
-- 
GitLab


From 0b700f23335e9206e1e460a477df2103ce3c186d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Tue, 29 Oct 2024 10:01:49 -0700
Subject: [PATCH 462/693] [flang][cuda] Add entry point to launch global
 function with cluster_dims (#113958)

---
 flang/include/flang/Runtime/CUDA/kernel.h |  8 +++++++-
 flang/runtime/CUDA/kernel.cpp             | 25 ++++++++++++++++++++++-
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Runtime/CUDA/kernel.h b/flang/include/flang/Runtime/CUDA/kernel.h
index cf07d874a082..85afda09e347 100644
--- a/flang/include/flang/Runtime/CUDA/kernel.h
+++ b/flang/include/flang/Runtime/CUDA/kernel.h
@@ -15,13 +15,19 @@
 
 extern "C" {
 
-// This function uses intptr_t instead of CUDA's unsigned int to match
+// These functions use intptr_t instead of CUDA's unsigned int to match
 // the type of MLIR's index type. This avoids the need for casts in the
 // generated MLIR code.
+
 void RTDEF(CUFLaunchKernel)(const void *kernelName, intptr_t gridX,
     intptr_t gridY, intptr_t gridZ, intptr_t blockX, intptr_t blockY,
     intptr_t blockZ, int32_t smem, void **params, void **extra);
 
+void RTDEF(CUFLaunchClusterKernel)(const void *kernelName, intptr_t clusterX,
+    intptr_t clusterY, intptr_t clusterZ, intptr_t gridX, intptr_t gridY,
+    intptr_t gridZ, intptr_t blockX, intptr_t blockY, intptr_t blockZ,
+    int32_t smem, void **params, void **extra);
+
 } // extern "C"
 
 #endif // FORTRAN_RUNTIME_CUDA_KERNEL_H_
diff --git a/flang/runtime/CUDA/kernel.cpp b/flang/runtime/CUDA/kernel.cpp
index f81153a1af4b..abb7ebb72e59 100644
--- a/flang/runtime/CUDA/kernel.cpp
+++ b/flang/runtime/CUDA/kernel.cpp
@@ -25,9 +25,32 @@ void RTDEF(CUFLaunchKernel)(const void *kernel, intptr_t gridX, intptr_t gridY,
   blockDim.x = blockX;
   blockDim.y = blockY;
   blockDim.z = blockZ;
-  cudaStream_t stream = 0;
+  cudaStream_t stream = 0; // TODO stream managment
   CUDA_REPORT_IF_ERROR(
       cudaLaunchKernel(kernel, gridDim, blockDim, params, smem, stream));
 }
 
+void RTDEF(CUFLaunchClusterKernel)(const void *kernel, intptr_t clusterX,
+    intptr_t clusterY, intptr_t clusterZ, intptr_t gridX, intptr_t gridY,
+    intptr_t gridZ, intptr_t blockX, intptr_t blockY, intptr_t blockZ,
+    int32_t smem, void **params, void **extra) {
+  cudaLaunchConfig_t config;
+  config.gridDim.x = gridX;
+  config.gridDim.y = gridY;
+  config.gridDim.z = gridZ;
+  config.blockDim.x = blockX;
+  config.blockDim.y = blockY;
+  config.blockDim.z = blockZ;
+  config.dynamicSmemBytes = smem;
+  config.stream = 0; // TODO stream managment
+  cudaLaunchAttribute launchAttr[1];
+  launchAttr[0].id = cudaLaunchAttributeClusterDimension;
+  launchAttr[0].val.clusterDim.x = clusterX;
+  launchAttr[0].val.clusterDim.y = clusterY;
+  launchAttr[0].val.clusterDim.z = clusterZ;
+  config.numAttrs = 1;
+  config.attrs = launchAttr;
+  CUDA_REPORT_IF_ERROR(cudaLaunchKernelExC(&config, kernel, params));
+}
+
 } // extern "C"
-- 
GitLab


From b05fec97d59898a63a3e303122bbc7fc5e29ced8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Tue, 29 Oct 2024 10:02:08 -0700
Subject: [PATCH 463/693] [flang][cuda] Convert gpu.launch_func to
 CUFLaunchClusterKernel when cluster dims are present (#113959)

Kernel launch in CUF are converted to `gpu.launch_func`. When the kernel
has `cluster_dims` specified these get carried over to the
`gpu.launch_func` operation. This patch updates the special conversion
of `gpu.launch_func` when cluster dims are present to the newly added
entry point.
---
 .../Transforms/CUFGPUToLLVMConversion.cpp     | 83 ++++++++++++-------
 flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir | 24 +++++-
 2 files changed, 76 insertions(+), 31 deletions(-)

diff --git a/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp b/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
index 5645ce6e6858..c64f35542a6e 100644
--- a/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
@@ -76,11 +76,6 @@ struct GPULaunchKernelConversion
   mlir::LogicalResult
   matchAndRewrite(mlir::gpu::LaunchFuncOp op, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
-
-    if (op.hasClusterSize()) {
-      return mlir::failure();
-    }
-
     mlir::Location loc = op.getLoc();
     auto *ctx = rewriter.getContext();
     mlir::ModuleOp mod = op->getParentOfType<mlir::ModuleOp>();
@@ -107,37 +102,65 @@ struct GPULaunchKernelConversion
           rewriter.create<LLVM::AddressOfOp>(loc, ptrTy, kernel.getName());
     }
 
-    auto funcOp = mod.lookupSymbol<mlir::LLVM::LLVMFuncOp>(
-        RTNAME_STRING(CUFLaunchKernel));
-
     auto llvmIntPtrType = mlir::IntegerType::get(
         ctx, this->getTypeConverter()->getPointerBitwidth(0));
     auto voidTy = mlir::LLVM::LLVMVoidType::get(ctx);
-    auto funcTy = mlir::LLVM::LLVMFunctionType::get(
-        voidTy,
-        {ptrTy, llvmIntPtrType, llvmIntPtrType, llvmIntPtrType, llvmIntPtrType,
-         llvmIntPtrType, llvmIntPtrType, i32Ty, ptrTy, ptrTy},
-        /*isVarArg=*/false);
-
-    auto cufLaunchKernel = mlir::SymbolRefAttr::get(
-        mod.getContext(), RTNAME_STRING(CUFLaunchKernel));
-    if (!funcOp) {
-      mlir::OpBuilder::InsertionGuard insertGuard(rewriter);
-      rewriter.setInsertionPointToStart(mod.getBody());
-      auto launchKernelFuncOp = rewriter.create<mlir::LLVM::LLVMFuncOp>(
-          loc, RTNAME_STRING(CUFLaunchKernel), funcTy);
-      launchKernelFuncOp.setVisibility(mlir::SymbolTable::Visibility::Private);
-    }
 
     mlir::Value nullPtr = rewriter.create<LLVM::ZeroOp>(loc, ptrTy);
 
-    rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
-        op, funcTy, cufLaunchKernel,
-        mlir::ValueRange{kernelPtr, adaptor.getGridSizeX(),
-                         adaptor.getGridSizeY(), adaptor.getGridSizeZ(),
-                         adaptor.getBlockSizeX(), adaptor.getBlockSizeY(),
-                         adaptor.getBlockSizeZ(), dynamicMemorySize, kernelArgs,
-                         nullPtr});
+    if (op.hasClusterSize()) {
+      auto funcOp = mod.lookupSymbol<mlir::LLVM::LLVMFuncOp>(
+          RTNAME_STRING(CUFLaunchClusterKernel));
+      auto funcTy = mlir::LLVM::LLVMFunctionType::get(
+          voidTy,
+          {ptrTy, llvmIntPtrType, llvmIntPtrType, llvmIntPtrType,
+           llvmIntPtrType, llvmIntPtrType, llvmIntPtrType, llvmIntPtrType,
+           llvmIntPtrType, llvmIntPtrType, i32Ty, ptrTy, ptrTy},
+          /*isVarArg=*/false);
+      auto cufLaunchClusterKernel = mlir::SymbolRefAttr::get(
+          mod.getContext(), RTNAME_STRING(CUFLaunchClusterKernel));
+      if (!funcOp) {
+        mlir::OpBuilder::InsertionGuard insertGuard(rewriter);
+        rewriter.setInsertionPointToStart(mod.getBody());
+        auto launchKernelFuncOp = rewriter.create<mlir::LLVM::LLVMFuncOp>(
+            loc, RTNAME_STRING(CUFLaunchClusterKernel), funcTy);
+        launchKernelFuncOp.setVisibility(
+            mlir::SymbolTable::Visibility::Private);
+      }
+      rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
+          op, funcTy, cufLaunchClusterKernel,
+          mlir::ValueRange{kernelPtr, adaptor.getClusterSizeX(),
+                           adaptor.getClusterSizeY(), adaptor.getClusterSizeZ(),
+                           adaptor.getGridSizeX(), adaptor.getGridSizeY(),
+                           adaptor.getGridSizeZ(), adaptor.getBlockSizeX(),
+                           adaptor.getBlockSizeY(), adaptor.getBlockSizeZ(),
+                           dynamicMemorySize, kernelArgs, nullPtr});
+    } else {
+      auto funcOp = mod.lookupSymbol<mlir::LLVM::LLVMFuncOp>(
+          RTNAME_STRING(CUFLaunchKernel));
+      auto funcTy = mlir::LLVM::LLVMFunctionType::get(
+          voidTy,
+          {ptrTy, llvmIntPtrType, llvmIntPtrType, llvmIntPtrType,
+           llvmIntPtrType, llvmIntPtrType, llvmIntPtrType, i32Ty, ptrTy, ptrTy},
+          /*isVarArg=*/false);
+      auto cufLaunchKernel = mlir::SymbolRefAttr::get(
+          mod.getContext(), RTNAME_STRING(CUFLaunchKernel));
+      if (!funcOp) {
+        mlir::OpBuilder::InsertionGuard insertGuard(rewriter);
+        rewriter.setInsertionPointToStart(mod.getBody());
+        auto launchKernelFuncOp = rewriter.create<mlir::LLVM::LLVMFuncOp>(
+            loc, RTNAME_STRING(CUFLaunchKernel), funcTy);
+        launchKernelFuncOp.setVisibility(
+            mlir::SymbolTable::Visibility::Private);
+      }
+      rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
+          op, funcTy, cufLaunchKernel,
+          mlir::ValueRange{kernelPtr, adaptor.getGridSizeX(),
+                           adaptor.getGridSizeY(), adaptor.getGridSizeZ(),
+                           adaptor.getBlockSizeX(), adaptor.getBlockSizeY(),
+                           adaptor.getBlockSizeZ(), dynamicMemorySize,
+                           kernelArgs, nullPtr});
+    }
 
     return mlir::success();
   }
diff --git a/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir b/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
index f10bd82f978d..7fede7c6c17b 100644
--- a/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
+++ b/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
@@ -1,4 +1,4 @@
-// RUN: fir-opt --cuf-gpu-convert-to-llvm %s | FileCheck %s
+// RUN: fir-opt --split-input-file --cuf-gpu-convert-to-llvm %s | FileCheck %s
 
 module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 20.0.0 (git@github.com:clementval/llvm-project.git ddcfd4d2dc17bf66cee8c3ef6284118684a2b0e6)", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
   llvm.func @_QMmod1Phost_sub() {
@@ -102,3 +102,25 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<i1, dense<8> : ve
 
 // CHECK: %[[KERNEL_PTR:.*]] = llvm.mlir.addressof @_QMmod1Psub1 : !llvm.ptr
 // CHECK: llvm.call @_FortranACUFLaunchKernel(%[[KERNEL_PTR]], {{.*}})
+
+// -----
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 20.0.0 (git@github.com:clementval/llvm-project.git 4116c1370ff76adf1e58eb3c39d0a14721794c70)", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
+  llvm.func @_FortranACUFLaunchClusterKernel(!llvm.ptr, i64, i64, i64, i64, i64, i64, i64, i64, i64, i32, !llvm.ptr, !llvm.ptr) attributes {sym_visibility = "private"}
+  llvm.func @_QMmod1Psub1() attributes {cuf.cluster_dims = #cuf.cluster_dims<x = 2 : i64, y = 2 : i64, z = 1 : i64>} {
+    llvm.return
+  }
+  llvm.func @_QQmain() attributes {fir.bindc_name = "test"} {
+    %0 = llvm.mlir.constant(1 : index) : i64
+    %1 = llvm.mlir.constant(2 : index) : i64
+    %2 = llvm.mlir.constant(0 : i32) : i32
+    %3 = llvm.mlir.constant(10 : index) : i64
+    gpu.launch_func  @cuda_device_mod::@_QMmod1Psub1 clusters in (%1, %1, %0) blocks in (%3, %3, %0) threads in (%3, %3, %0) : i64 dynamic_shared_memory_size %2
+    llvm.return
+  }
+  gpu.binary @cuda_device_mod  [#gpu.object<#nvvm.target, "">]
+}
+
+// CHECK-LABEL: llvm.func @_QQmain()
+// CHECK: %[[KERNEL_PTR:.*]] = llvm.mlir.addressof @_QMmod1Psub1
+// CHECK: llvm.call @_FortranACUFLaunchClusterKernel(%[[KERNEL_PTR]], {{.*}})
-- 
GitLab


From a1f2fb6078bbed8034ce28eafc3518268e25f2ff Mon Sep 17 00:00:00 2001
From: Sergio Afonso <safonsof@amd.com>
Date: Tue, 29 Oct 2024 17:05:12 +0000
Subject: [PATCH 464/693] [MLIR][OpenMP] Prevent composite omp.simd related
 crashes (#113680)

This patch updates the translation of `omp.wsloop` with a nested
`omp.simd` to prevent uses of block arguments defined by the latter from
triggering null pointer dereferences.

This happens because the inner `omp.simd` operation representing
composite `do simd` constructs is currently skipped and not translated,
but this results in block arguments defined by it not being mapped to an
LLVM value. The proposed solution is to map these block arguments to the
LLVM value associated to the corresponding operand, which is defined
above.
---
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 66 ++++++++++++++-
 mlir/test/Target/LLVMIR/openmp-reduction.mlir | 80 +++++++++++++++++++
 2 files changed, 143 insertions(+), 3 deletions(-)

diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index fc2f88b766f1..d20e5e40076b 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -262,6 +262,62 @@ static llvm::omp::ProcBindKind getProcBindKind(omp::ClauseProcBindKind kind) {
   llvm_unreachable("Unknown ClauseProcBindKind kind");
 }
 
+/// Helper function to map block arguments defined by ignored loop wrappers to
+/// LLVM values and prevent any uses of those from triggering null pointer
+/// dereferences.
+///
+/// This must be called after block arguments of parent wrappers have already
+/// been mapped to LLVM IR values.
+static LogicalResult
+convertIgnoredWrapper(omp::LoopWrapperInterface &opInst,
+                      LLVM::ModuleTranslation &moduleTranslation) {
+  // Map block arguments directly to the LLVM value associated to the
+  // corresponding operand. This is semantically equivalent to this wrapper not
+  // being present.
+  auto forwardArgs =
+      [&moduleTranslation](llvm::ArrayRef<BlockArgument> blockArgs,
+                           OperandRange operands) {
+        for (auto [arg, var] : llvm::zip_equal(blockArgs, operands))
+          moduleTranslation.mapValue(arg, moduleTranslation.lookupValue(var));
+      };
+
+  return llvm::TypeSwitch<Operation *, LogicalResult>(opInst)
+      .Case([&](omp::SimdOp op) {
+        auto blockArgIface = cast<omp::BlockArgOpenMPOpInterface>(*op);
+        forwardArgs(blockArgIface.getPrivateBlockArgs(), op.getPrivateVars());
+        forwardArgs(blockArgIface.getReductionBlockArgs(),
+                    op.getReductionVars());
+        return success();
+      })
+      .Default([&](Operation *op) {
+        return op->emitError() << "cannot ignore nested wrapper";
+      });
+}
+
+/// Helper function to call \c convertIgnoredWrapper() for all wrappers of the
+/// given \c loopOp nested inside of \c parentOp. This has the effect of mapping
+/// entry block arguments defined by these operations to outside values.
+///
+/// It must be called after block arguments of \c parentOp have already been
+/// mapped themselves.
+static LogicalResult
+convertIgnoredWrappers(omp::LoopNestOp loopOp,
+                       omp::LoopWrapperInterface parentOp,
+                       LLVM::ModuleTranslation &moduleTranslation) {
+  SmallVector<omp::LoopWrapperInterface> wrappers;
+  loopOp.gatherWrappers(wrappers);
+
+  // Process wrappers nested inside of `parentOp` from outermost to innermost.
+  for (auto it =
+           std::next(std::find(wrappers.rbegin(), wrappers.rend(), parentOp));
+       it != wrappers.rend(); ++it) {
+    if (failed(convertIgnoredWrapper(*it, moduleTranslation)))
+      return failure();
+  }
+
+  return success();
+}
+
 /// Converts an OpenMP 'masked' operation into LLVM IR using OpenMPIRBuilder.
 static LogicalResult
 convertOmpMasked(Operation &opInst, llvm::IRBuilderBase &builder,
@@ -1262,9 +1318,6 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
       !wsloopOp.getPrivateVars().empty() || wsloopOp.getPrivateSyms())
     return opInst.emitError("unhandled clauses for translation to LLVM IR");
 
-  // FIXME: Here any other nested wrappers (e.g. omp.simd) are skipped, so
-  // codegen for composite constructs like 'DO/FOR SIMD' will be the same as for
-  // 'DO/FOR'.
   auto loopOp = cast<omp::LoopNestOp>(wsloopOp.getWrappedLoop());
 
   llvm::ArrayRef<bool> isByRef = getIsByRef(wsloopOp.getReductionByref());
@@ -1302,6 +1355,13 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
           isByRef)))
     return failure();
 
+  // TODO: Replace this with proper composite translation support.
+  // Currently, all nested wrappers are ignored, so 'do/for simd' will be
+  // treated the same as a standalone 'do/for'. This is allowed by the spec,
+  // since it's equivalent to always using a SIMD length of 1.
+  if (failed(convertIgnoredWrappers(loopOp, wsloopOp, moduleTranslation)))
+    return failure();
+
   // Store the mapping between reduction variables and their private copies on
   // ModuleTranslation stack. It can be then recovered when translating
   // omp.reduce operations in a separate call.
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction.mlir b/mlir/test/Target/LLVMIR/openmp-reduction.mlir
index 6d74a925b87b..11c8559044be 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction.mlir
@@ -586,3 +586,83 @@ llvm.func @parallel_nested_workshare_reduction(%ub : i64) {
 // Reduction function.
 // CHECK: define internal void @[[REDFUNC]]
 // CHECK: add i32
+
+// -----
+
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+  %0 = llvm.mlir.constant(0.0 : f32) : f32
+  omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+  %1 = llvm.fadd %arg0, %arg1 : f32
+  omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+  %2 = llvm.load %arg3 : !llvm.ptr -> f32
+  llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+  omp.yield
+}
+
+// CHECK-LABEL: @wsloop_simd_reduction
+llvm.func @wsloop_simd_reduction(%lb : i64, %ub : i64, %step : i64) {
+  %c1 = llvm.mlir.constant(1 : i32) : i32
+  %0 = llvm.alloca %c1 x i32 : (i32) -> !llvm.ptr
+  omp.parallel {
+    omp.wsloop reduction(@add_f32 %0 -> %prv1 : !llvm.ptr) {
+      omp.simd reduction(@add_f32 %prv1 -> %prv2 : !llvm.ptr) {
+        omp.loop_nest (%iv) : i64 = (%lb) to (%ub) step (%step) {
+          %1 = llvm.mlir.constant(2.0 : f32) : f32
+          %2 = llvm.load %prv2 : !llvm.ptr -> f32
+          %3 = llvm.fadd %1, %2 : f32
+          llvm.store %3, %prv2 : f32, !llvm.ptr
+          omp.yield
+        }
+      } {omp.composite}
+    } {omp.composite}
+    omp.terminator
+  }
+  llvm.return
+}
+
+// Same checks as for wsloop reduction, because currently omp.simd is ignored in
+// a composite 'do/for simd' construct.
+// Call to the outlined function.
+// CHECK: call void {{.*}} @__kmpc_fork_call
+// CHECK-SAME: @[[OUTLINED:[A-Za-z_.][A-Za-z0-9_.]*]]
+
+// Outlined function.
+// CHECK: define internal void @[[OUTLINED]]
+
+// Private reduction variable and its initialization.
+// CHECK: %[[PRIVATE:.+]] = alloca float
+// CHECK: store float 0.000000e+00, ptr %[[PRIVATE]]
+
+// Call to the reduction function.
+// CHECK: call i32 @__kmpc_reduce
+// CHECK-SAME: @[[REDFUNC:[A-Za-z_.][A-Za-z0-9_.]*]]
+
+// Atomic reduction.
+// CHECK: %[[PARTIAL:.+]] = load float, ptr %[[PRIVATE]]
+// CHECK: atomicrmw fadd ptr %{{.*}}, float %[[PARTIAL]]
+
+// Non-atomic reduction:
+// CHECK: fadd float
+// CHECK: call void @__kmpc_end_reduce
+// CHECK: br label %[[FINALIZE:.+]]
+
+// CHECK: [[FINALIZE]]:
+// CHECK: call void @__kmpc_barrier
+
+// Update of the private variable using the reduction region
+// (the body block currently comes after all the other blocks).
+// CHECK: %[[PARTIAL:.+]] = load float, ptr %[[PRIVATE]]
+// CHECK: %[[UPDATED:.+]] = fadd float 2.000000e+00, %[[PARTIAL]]
+// CHECK: store float %[[UPDATED]], ptr %[[PRIVATE]]
+
+// Reduction function.
+// CHECK: define internal void @[[REDFUNC]]
+// CHECK: fadd float
-- 
GitLab


From f53889ffcad28bbc0faf671626cc90eb4e7da5a8 Mon Sep 17 00:00:00 2001
From: Jubilee <workingjubilee@gmail.com>
Date: Tue, 29 Oct 2024 10:07:20 -0700
Subject: [PATCH 465/693] [RISCV] Allow crypto features to imply dependents
 (#112659)

This relationship is a logical dependency.

Note Zvbc and Zvknhb. They are explicitly called out in the spec as
requiring 64 bits:
-
https://github.com/riscv/riscv-crypto/blob/56ed7952d13eb5bdff92e2b522404668952f416d/doc/vector/riscv-crypto-spec-vector.adoc
---
 llvm/lib/Target/RISCV/RISCVFeatures.td        | 27 +++++---
 .../TargetParser/RISCVISAInfoTest.cpp         | 68 ++++---------------
 2 files changed, 33 insertions(+), 62 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 608782d7839a..6f43c832fd4d 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -733,7 +733,8 @@ def HasStdExtZfhOrZvfh
 
 def FeatureStdExtZvkb
     : RISCVExtension<"zvkb", 1, 0,
-                     "'Zvkb' (Vector Bit-manipulation used in Cryptography)">,
+                     "'Zvkb' (Vector Bit-manipulation used in Cryptography)",
+                     [FeatureStdExtZve32x]>,
       RISCVExtensionBitmask<0, 52>;
 def HasStdExtZvkb : Predicate<"Subtarget->hasStdExtZvkb()">,
                     AssemblerPredicate<(all_of FeatureStdExtZvkb),
@@ -750,7 +751,8 @@ def HasStdExtZvbb : Predicate<"Subtarget->hasStdExtZvbb()">,
 
 def FeatureStdExtZvbc
     : RISCVExtension<"zvbc", 1, 0,
-                     "'Zvbc' (Vector Carryless Multiplication)">,
+                     "'Zvbc' (Vector Carryless Multiplication)",
+                     [FeatureStdExtZve64x]>,
       RISCVExtensionBitmask<0, 49>;
 def HasStdExtZvbc : Predicate<"Subtarget->hasStdExtZvbc()">,
                     AssemblerPredicate<(all_of FeatureStdExtZvbc),
@@ -758,7 +760,8 @@ def HasStdExtZvbc : Predicate<"Subtarget->hasStdExtZvbc()">,
 
 def FeatureStdExtZvbc32e
     : RISCVExperimentalExtension<"zvbc32e", 0, 7,
-                                 "'Zvbc32e' (Vector Carryless Multiplication with 32-bits elements)">;
+                                 "'Zvbc32e' (Vector Carryless Multiplication with 32-bits elements)",
+                                 [FeatureStdExtZve32x]>;
 
 def HasStdExtZvbcOrZvbc32e : Predicate<"Subtarget->hasStdExtZvbc() || Subtarget->hasStdExtZvbc32e()">,
                              AssemblerPredicate<(any_of FeatureStdExtZvbc, FeatureStdExtZvbc32e),
@@ -766,7 +769,8 @@ def HasStdExtZvbcOrZvbc32e : Predicate<"Subtarget->hasStdExtZvbc() || Subtarget-
 
 def FeatureStdExtZvkg
     : RISCVExtension<"zvkg", 1, 0,
-                     "'Zvkg' (Vector GCM instructions for Cryptography)">,
+                     "'Zvkg' (Vector GCM instructions for Cryptography)",
+                     [FeatureStdExtZve32x]>,
       RISCVExtensionBitmask<0, 53>;
 def HasStdExtZvkg : Predicate<"Subtarget->hasStdExtZvkg()">,
                     AssemblerPredicate<(all_of FeatureStdExtZvkg),
@@ -782,7 +786,8 @@ def HasStdExtZvkgs : Predicate<"Subtarget->hasStdExtZvkgs()">,
 
 def FeatureStdExtZvkned
     : RISCVExtension<"zvkned", 1, 0,
-                     "'Zvkned' (Vector AES Encryption & Decryption (Single Round))">,
+                     "'Zvkned' (Vector AES Encryption & Decryption (Single Round))",
+                     [FeatureStdExtZve32x]>,
       RISCVExtensionBitmask<0, 54>;
 def HasStdExtZvkned : Predicate<"Subtarget->hasStdExtZvkned()">,
                       AssemblerPredicate<(all_of FeatureStdExtZvkned),
@@ -790,7 +795,8 @@ def HasStdExtZvkned : Predicate<"Subtarget->hasStdExtZvkned()">,
 
 def FeatureStdExtZvknha
     : RISCVExtension<"zvknha", 1, 0,
-                     "'Zvknha' (Vector SHA-2 (SHA-256 only))">,
+                     "'Zvknha' (Vector SHA-2 (SHA-256 only))",
+                     [FeatureStdExtZve32x]>,
       RISCVExtensionBitmask<0, 55>;
 def HasStdExtZvknha : Predicate<"Subtarget->hasStdExtZvknha()">,
                       AssemblerPredicate<(all_of FeatureStdExtZvknha),
@@ -798,7 +804,8 @@ def HasStdExtZvknha : Predicate<"Subtarget->hasStdExtZvknha()">,
 
 def FeatureStdExtZvknhb
     : RISCVExtension<"zvknhb", 1, 0,
-                     "'Zvknhb' (Vector SHA-2 (SHA-256 and SHA-512))">,
+                     "'Zvknhb' (Vector SHA-2 (SHA-256 and SHA-512))",
+                     [FeatureStdExtZve64x]>,
       RISCVExtensionBitmask<0, 56>;
 def HasStdExtZvknhb : Predicate<"Subtarget->hasStdExtZvknhb()">,
                       AssemblerPredicate<(all_of FeatureStdExtZvknhb),
@@ -810,7 +817,8 @@ def HasStdExtZvknhaOrZvknhb : Predicate<"Subtarget->hasStdExtZvknha() || Subtarg
 
 def FeatureStdExtZvksed
     : RISCVExtension<"zvksed", 1, 0,
-                     "'Zvksed' (SM4 Block Cipher Instructions)">,
+                     "'Zvksed' (SM4 Block Cipher Instructions)",
+                     [FeatureStdExtZve32x]>,
       RISCVExtensionBitmask<0, 57>;
 def HasStdExtZvksed : Predicate<"Subtarget->hasStdExtZvksed()">,
                       AssemblerPredicate<(all_of FeatureStdExtZvksed),
@@ -818,7 +826,8 @@ def HasStdExtZvksed : Predicate<"Subtarget->hasStdExtZvksed()">,
 
 def FeatureStdExtZvksh
     : RISCVExtension<"zvksh", 1, 0,
-                     "'Zvksh' (SM3 Hash Function Instructions)">,
+                     "'Zvksh' (SM3 Hash Function Instructions)",
+                     [FeatureStdExtZve32x]>,
       RISCVExtensionBitmask<0, 58>;
 def HasStdExtZvksh : Predicate<"Subtarget->hasStdExtZvksh()">,
                      AssemblerPredicate<(all_of FeatureStdExtZvksh),
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index a1d493e12fda..30f80601d96c 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -643,60 +643,22 @@ TEST(ParseArchString, MissingDepency) {
               "'zvl*b' requires 'v' or 'zve*' extension to also be specified");
   }
 
-  for (StringRef Input : {"rv32i_zvbb"}) {
+  // These all have an implication relationship, thus should pass
+  for (StringRef Input : {
+           "rv32i_zvbb",
+           "rv32i_zvbc32e0p7",
+           "rv32i_zvbc",
+           "rv32i_zvkb",
+           "rv32i_zvkg",
+           "rv32i_zvkgs0p7",
+           "rv32i_zvkned",
+           "rv32i_zvknha",
+           "rv32i_zvksed",
+           "rv32i_zvksh",
+           "rv32i_zvknhb",
+       }) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvbb' requires 'v' or 'zve*' extension to also be specified");
-  }
-
-  for (StringRef Input : {"rv32i_zvbc32e0p7"}) {
-    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvbc32e' requires 'v' or 'zve*' extension to also be specified");
-  }
-
-  for (StringRef Input : {"rv32i_zvbc"}) {
-    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvbc' requires 'v' or 'zve64*' extension to also be specified");
-  }
-
-  for (StringRef Input : {"rv32i_zvkb"}) {
-    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvkb' requires 'v' or 'zve*' extension to also be specified");
-  }
-
-  for (StringRef Input : {"rv32i_zvkg"}) {
-    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvkg' requires 'v' or 'zve*' extension to also be specified");
-  }
-
-  for (StringRef Input : {"rv32i_zvkgs0p7"}) {
-    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvkg' requires 'v' or 'zve*' extension to also be specified");
-  }
-
-  for (StringRef Input : {"rv32i_zvkned"}) {
-    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvkned' requires 'v' or 'zve*' extension to also be specified");
-  }
-
-  for (StringRef Input : {"rv32i_zvknha"}) {
-    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvknha' requires 'v' or 'zve*' extension to also be specified");
-  }
-
-  for (StringRef Input : {"rv32i_zvksed"}) {
-    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvksed' requires 'v' or 'zve*' extension to also be specified");
-  }
-
-  for (StringRef Input : {"rv32i_zvksh"}) {
-    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvksh' requires 'v' or 'zve*' extension to also be specified");
-  }
-
-  for (StringRef Input : {"rv32i_zvknhb"}) {
-    EXPECT_EQ(
-        toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-        "'zvknhb' requires 'v' or 'zve64*' extension to also be specified");
+              "");
   }
 
   for (StringRef Input : {"rv32i_zacas1p0"}) {
-- 
GitLab


From b1d0fe095ba93df47b5db20a3bd55f9ff857836e Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 29 Oct 2024 10:09:28 -0700
Subject: [PATCH 466/693] [RISCV] Remove trailing whitespace. NFC

---
 llvm/lib/Target/RISCV/RISCVFeatures.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 6f43c832fd4d..1e4bf1b8830b 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -1040,7 +1040,7 @@ def FeatureStdExtSvpbmt
 
 def FeatureStdExtSvvptc
     : RISCVExtension<"svvptc", 1, 0,
-                     "'svvptc' (Obviating Memory-Management Instructions after Marking PTEs Valid)">;                    
+                     "'svvptc' (Obviating Memory-Management Instructions after Marking PTEs Valid)">;
 
 def FeatureStdExtSha
     : RISCVExtension<"sha", 1, 0,
-- 
GitLab


From f964514490ecf6d57dc9f53ebda913a6fe1e3abd Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Tue, 29 Oct 2024 13:16:20 -0400
Subject: [PATCH 467/693] Nominate Shafik Yaghmour and Vlad Serebrennikov for
 C++ conformance (#114071)

Shafik and Vlad are both members of WG21 and both have familiarity with
reasoning about the C++ standard. They've both volunteered to help
answer conformance related questions, and this is an area where we get
quite a bit of questions so having a larger stable of maintainers is
quite useful.
---
 clang/Maintainers.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/clang/Maintainers.rst b/clang/Maintainers.rst
index 39f46457e676..08dcc584f6c5 100644
--- a/clang/Maintainers.rst
+++ b/clang/Maintainers.rst
@@ -231,6 +231,12 @@ C++ conformance
 | Hubert Tong
 | hubert.reinterpretcast\@gmail.com (email), hubert.reinterpretcast (Phabricator), hubert-reinterpretcast (GitHub)
 
+| Shafik Yaghmour
+| shafik.yaghmour\@intel.com (email), shafik (GitHub), shafik.yaghmour (Discord), shafik (Discourse)
+
+| Vlad Serebrennikov
+| serebrennikov.vladislav\@gmail.com (email), Endilll (GitHub), Endill (Discord), Endill (Discourse)
+
 
 C++ Defect Reports
 ~~~~~~~~~~~~~~~~~~
-- 
GitLab


From 9a5b3a1bbca6790602ec3291da850fc4485cc807 Mon Sep 17 00:00:00 2001
From: Adam Yang <hanbyang@microsoft.com>
Date: Tue, 29 Oct 2024 10:17:35 -0700
Subject: [PATCH 468/693] [DXIL] Add GroupMemoryBarrierWithGroupSync intrinsic
 (#111884)

fixes #112974
partially fixes #70103

### Changes
- Added new tablegen based way of lowering dx intrinsics to DXIL ops.
- Added int_dx_group_memory_barrier_with_group_sync intrinsic in
IntrinsicsDirectX.td
- Added expansion for int_dx_group_memory_barrier_with_group_sync in
DXILIntrinsicExpansion.cpp`
- Added DXIL backend test case

### Related PRs
* [[clang][HLSL] Add GroupMemoryBarrierWithGroupSync intrinsic
#111883](https://github.com/llvm/llvm-project/pull/111883)
* [[SPIRV] Add GroupMemoryBarrierWithGroupSync intrinsic
#111888](https://github.com/llvm/llvm-project/pull/111888)
---
 llvm/include/llvm/IR/IntrinsicsDirectX.td     |   2 +
 llvm/lib/Target/DirectX/DXIL.td               |  54 ++++++++
 llvm/lib/Target/DirectX/DXILOpLowering.cpp    |  45 +++++--
 .../group_memory_barrier_with_group_sync.ll   |   8 ++
 llvm/utils/TableGen/DXILEmitter.cpp           | 122 ++++++++++++++++--
 5 files changed, 209 insertions(+), 22 deletions(-)
 create mode 100644 llvm/test/CodeGen/DirectX/group_memory_barrier_with_group_sync.ll

diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index e30d37f69f78..dada42636899 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -92,4 +92,6 @@ def int_dx_step : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty, L
 def int_dx_splitdouble : DefaultAttrsIntrinsic<[llvm_anyint_ty, LLVMMatchType<0>], 
     [LLVMScalarOrSameVectorWidth<0, llvm_double_ty>], [IntrNoMem]>;
 def int_dx_radians : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
+
+def int_dx_group_memory_barrier_with_group_sync : DefaultAttrsIntrinsic<[], [], []>;
 }
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 1e8dc63ffa25..263ca50011aa 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -294,6 +294,43 @@ class Attributes<Version ver = DXIL1_0, list<DXILAttribute> attrs> {
   list<DXILAttribute> op_attrs = attrs;
 }
 
+class DXILConstant<int value_> {
+  int value = value_;
+}
+
+defset list<DXILConstant> BarrierModes = {
+  def BarrierMode_DeviceMemoryBarrier              : DXILConstant<2>;
+  def BarrierMode_DeviceMemoryBarrierWithGroupSync : DXILConstant<3>;
+  def BarrierMode_GroupMemoryBarrier               : DXILConstant<8>;
+  def BarrierMode_GroupMemoryBarrierWithGroupSync  : DXILConstant<9>;
+  def BarrierMode_AllMemoryBarrier                 : DXILConstant<10>;
+  def BarrierMode_AllMemoryBarrierWithGroupSync    : DXILConstant<11>;
+}
+
+// Intrinsic arg selection
+class Arg {
+  int index = -1;
+  DXILConstant value;
+  bit is_i8 = 0;
+  bit is_i32 = 0;
+}
+class ArgSelect<int index_> : Arg {
+  let index = index_;
+}
+class ArgI32<DXILConstant value_> : Arg {
+  let value = value_;
+  let is_i32 = 1;
+}
+class ArgI8<DXILConstant value_> : Arg {
+  let value = value_;
+  let is_i8 = 1;
+}
+
+class IntrinsicSelect<Intrinsic intrinsic_, list<Arg> args_> {
+  Intrinsic intrinsic = intrinsic_;
+  list<Arg> args = args_;
+}
+
 // Abstraction DXIL Operation
 class DXILOp<int opcode, DXILOpClass opclass> {
   // A short description of the operation
@@ -308,6 +345,9 @@ class DXILOp<int opcode, DXILOpClass opclass> {
   // LLVM Intrinsic DXIL Operation maps to
   Intrinsic LLVMIntrinsic = ?;
 
+  // Non-trivial LLVM Intrinsics DXIL Operation maps to
+  list<IntrinsicSelect> intrinsic_selects = [];
+
   // Result type of the op
   DXILOpParamType result;
 
@@ -829,3 +869,17 @@ def WaveGetLaneIndex : DXILOp<111, waveGetLaneIndex> {
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
+
+def Barrier : DXILOp<80, barrier> {
+  let Doc = "inserts a memory barrier in the shader";
+  let intrinsic_selects = [
+    IntrinsicSelect<
+        int_dx_group_memory_barrier_with_group_sync,
+        [ ArgI32<BarrierMode_GroupMemoryBarrierWithGroupSync> ]>,
+  ];
+
+  let arguments = [Int32Ty];
+  let result = VoidTy;
+  let stages = [Stages<DXIL1_0, [compute, library]>];
+  let attributes = [Attributes<DXIL1_0, []>];
+}
diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index 8acc9c1efa08..b5cf1654181c 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -106,17 +106,43 @@ public:
     return false;
   }
 
-  [[nodiscard]]
-  bool replaceFunctionWithOp(Function &F, dxil::OpCode DXILOp) {
+  struct ArgSelect {
+    enum class Type {
+      Index,
+      I8,
+      I32,
+    };
+    Type Type = Type::Index;
+    int Value = -1;
+  };
+
+  [[nodiscard]] bool replaceFunctionWithOp(Function &F, dxil::OpCode DXILOp,
+                                           ArrayRef<ArgSelect> ArgSelects) {
     bool IsVectorArgExpansion = isVectorArgExpansion(F);
     return replaceFunction(F, [&](CallInst *CI) -> Error {
-      SmallVector<Value *> Args;
       OpBuilder.getIRB().SetInsertPoint(CI);
-      if (IsVectorArgExpansion) {
-        SmallVector<Value *> NewArgs = argVectorFlatten(CI, OpBuilder.getIRB());
-        Args.append(NewArgs.begin(), NewArgs.end());
-      } else
+      SmallVector<Value *> Args;
+      if (ArgSelects.size()) {
+        for (const ArgSelect &A : ArgSelects) {
+          switch (A.Type) {
+          case ArgSelect::Type::Index:
+            Args.push_back(CI->getArgOperand(A.Value));
+            break;
+          case ArgSelect::Type::I8:
+            Args.push_back(OpBuilder.getIRB().getInt8((uint8_t)A.Value));
+            break;
+          case ArgSelect::Type::I32:
+            Args.push_back(OpBuilder.getIRB().getInt32(A.Value));
+            break;
+          default:
+            llvm_unreachable("Invalid type of intrinsic arg select.");
+          }
+        }
+      } else if (IsVectorArgExpansion) {
+        Args = argVectorFlatten(CI, OpBuilder.getIRB());
+      } else {
         Args.append(CI->arg_begin(), CI->arg_end());
+      }
 
       Expected<CallInst *> OpCall =
           OpBuilder.tryCreateOp(DXILOp, Args, CI->getName(), F.getReturnType());
@@ -583,9 +609,10 @@ public:
       switch (ID) {
       default:
         continue;
-#define DXIL_OP_INTRINSIC(OpCode, Intrin)                                      \
+#define DXIL_OP_INTRINSIC(OpCode, Intrin, ...)                                 \
   case Intrin:                                                                 \
-    HasErrors |= replaceFunctionWithOp(F, OpCode);                             \
+    HasErrors |=                                                               \
+        replaceFunctionWithOp(F, OpCode, ArrayRef<ArgSelect>{__VA_ARGS__});    \
     break;
 #include "DXILOperation.inc"
       case Intrinsic::dx_handle_fromBinding:
diff --git a/llvm/test/CodeGen/DirectX/group_memory_barrier_with_group_sync.ll b/llvm/test/CodeGen/DirectX/group_memory_barrier_with_group_sync.ll
new file mode 100644
index 000000000000..baf93d4e177f
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/group_memory_barrier_with_group_sync.ll
@@ -0,0 +1,8 @@
+; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s
+
+define void @test_group_memory_barrier_with_group_sync() {
+entry:
+  ; CHECK: call void @dx.op.barrier(i32 80, i32 9)
+  call void @llvm.dx.group.memory.barrier.with.group.sync()
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp
index 467a6163ae3b..859423324463 100644
--- a/llvm/utils/TableGen/DXILEmitter.cpp
+++ b/llvm/utils/TableGen/DXILEmitter.cpp
@@ -32,6 +32,20 @@ using namespace llvm::dxil;
 
 namespace {
 
+struct DXILArgSelect {
+  enum class Type {
+    Index,
+    I32,
+    I8,
+  };
+  Type Type = Type::Index;
+  int Value = -1;
+};
+struct DXILIntrinsicSelect {
+  StringRef Intrinsic;
+  SmallVector<DXILArgSelect, 4> Args;
+};
+
 struct DXILOperationDesc {
   std::string OpName; // name of DXIL operation
   int OpCode;         // ID of DXIL operation
@@ -42,8 +56,7 @@ struct DXILOperationDesc {
   SmallVector<const Record *> OverloadRecs;
   SmallVector<const Record *> StageRecs;
   SmallVector<const Record *> AttrRecs;
-  StringRef Intrinsic; // The llvm intrinsic map to OpName. Default is "" which
-                       // means no map exists
+  SmallVector<DXILIntrinsicSelect> IntrinsicSelects;
   SmallVector<StringRef, 4>
       ShaderStages; // shader stages to which this applies, empty for all.
   int OverloadParamIndex;             // Index of parameter with overload type.
@@ -71,6 +84,21 @@ static void AscendingSortByVersion(std::vector<const Record *> &Recs) {
   });
 }
 
+/// Take a `int_{intrinsic_name}` and return just the intrinsic_name part if
+/// available. Otherwise return the empty string.
+static StringRef GetIntrinsicName(const RecordVal *RV) {
+  if (RV && RV->getValue()) {
+    if (const DefInit *DI = dyn_cast<DefInit>(RV->getValue())) {
+      auto *IntrinsicDef = DI->getDef();
+      auto DefName = IntrinsicDef->getName();
+      assert(DefName.starts_with("int_") && "invalid intrinsic name");
+      // Remove the int_ from intrinsic name.
+      return DefName.substr(4);
+    }
+  }
+  return "";
+}
+
 /// Construct an object using the DXIL Operation records specified
 /// in DXIL.td. This serves as the single source of reference of
 /// the information extracted from the specified Record R, for
@@ -157,14 +185,63 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
                            OpName);
   }
 
-  const RecordVal *RV = R->getValue("LLVMIntrinsic");
-  if (RV && RV->getValue()) {
-    if (const DefInit *DI = dyn_cast<DefInit>(RV->getValue())) {
-      auto *IntrinsicDef = DI->getDef();
-      auto DefName = IntrinsicDef->getName();
-      assert(DefName.starts_with("int_") && "invalid intrinsic name");
-      // Remove the int_ from intrinsic name.
-      Intrinsic = DefName.substr(4);
+  {
+    DXILIntrinsicSelect IntrSelect;
+    IntrSelect.Intrinsic = GetIntrinsicName(R->getValue("LLVMIntrinsic"));
+    if (IntrSelect.Intrinsic.size())
+      IntrinsicSelects.emplace_back(std::move(IntrSelect));
+  }
+
+  auto IntrinsicSelectRecords = R->getValueAsListOfDefs("intrinsic_selects");
+  if (IntrinsicSelectRecords.size()) {
+    if (IntrinsicSelects.size()) {
+      PrintFatalError(
+          R, Twine("LLVMIntrinsic and intrinsic_selects cannot be both "
+                   "defined for DXIL operation - ") +
+                 OpName);
+    } else {
+      for (const Record *R : IntrinsicSelectRecords) {
+        DXILIntrinsicSelect IntrSelect;
+        IntrSelect.Intrinsic = GetIntrinsicName(R->getValue("intrinsic"));
+        auto Args = R->getValueAsListOfDefs("args");
+        for (const Record *Arg : Args) {
+          bool IsI8 = Arg->getValueAsBit("is_i8");
+          bool IsI32 = Arg->getValueAsBit("is_i32");
+          int Index = Arg->getValueAsInt("index");
+          const Record *ValueRec = Arg->getValueAsOptionalDef("value");
+
+          DXILArgSelect ArgSelect;
+          if (IsI8) {
+            if (!ValueRec) {
+              PrintFatalError(R, Twine("'value' must be defined for i8 "
+                                       "ArgSelect for DXIL operation - ") +
+                                     OpName);
+            }
+            ArgSelect.Type = DXILArgSelect::Type::I8;
+            ArgSelect.Value = ValueRec->getValueAsInt("value");
+          } else if (IsI32) {
+            if (!ValueRec) {
+              PrintFatalError(R, Twine("'value' must be defined for i32 "
+                                       "ArgSelect for DXIL operation - ") +
+                                     OpName);
+            }
+            ArgSelect.Type = DXILArgSelect::Type::I32;
+            ArgSelect.Value = ValueRec->getValueAsInt("value");
+          } else {
+            if (Index < 0) {
+              PrintFatalError(
+                  R, Twine("Index in ArgSelect<index> must be equal to or "
+                           "greater than 0 for DXIL operation - ") +
+                         OpName);
+            }
+            ArgSelect.Type = DXILArgSelect::Type::Index;
+            ArgSelect.Value = Index;
+          }
+
+          IntrSelect.Args.emplace_back(std::move(ArgSelect));
+        }
+        IntrinsicSelects.emplace_back(std::move(IntrSelect));
+      }
     }
   }
 }
@@ -377,10 +454,29 @@ static void emitDXILIntrinsicMap(ArrayRef<DXILOperationDesc> Ops,
   OS << "#ifdef DXIL_OP_INTRINSIC\n";
   OS << "\n";
   for (const auto &Op : Ops) {
-    if (Op.Intrinsic.empty())
+    if (Op.IntrinsicSelects.empty()) {
       continue;
-    OS << "DXIL_OP_INTRINSIC(dxil::OpCode::" << Op.OpName
-       << ", Intrinsic::" << Op.Intrinsic << ")\n";
+    }
+    for (const DXILIntrinsicSelect &MappedIntr : Op.IntrinsicSelects) {
+      OS << "DXIL_OP_INTRINSIC(dxil::OpCode::" << Op.OpName
+         << ", Intrinsic::" << MappedIntr.Intrinsic;
+      for (const DXILArgSelect &ArgSelect : MappedIntr.Args) {
+        OS << ", (ArgSelect { ";
+        switch (ArgSelect.Type) {
+        case DXILArgSelect::Type::Index:
+          OS << "ArgSelect::Type::Index, ";
+          break;
+        case DXILArgSelect::Type::I8:
+          OS << "ArgSelect::Type::I8, ";
+          break;
+        case DXILArgSelect::Type::I32:
+          OS << "ArgSelect::Type::I32, ";
+          break;
+        }
+        OS << ArgSelect.Value << "})";
+      }
+      OS << ")\n";
+    }
   }
   OS << "\n";
   OS << "#undef DXIL_OP_INTRINSIC\n";
-- 
GitLab


From d661aea4c5668fc9b06f4b26d9fb072b1a6d7ff4 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Tue, 29 Oct 2024 10:18:32 -0700
Subject: [PATCH 469/693] [OpenMP] Add support for custom callback in
 AMDGPUStream (#112785)

Summary:
We have the ability to schedule callbacks after certain events complete.
Currently we can register an arbitrary callback in CUDA, but can't in
AMDGPU. I am planning on using this support to move the RPC handling to
a separate thread, then using these callbacks to suspend / resume it
when no kernels are running. This is a preliminary patch to keep this
noise out of that one.
---
 offload/plugins-nextgen/amdgpu/src/rtl.cpp | 69 ++++++++++++++--------
 1 file changed, 44 insertions(+), 25 deletions(-)

diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index f0cc0c2e4d08..bdb33d4f4ab2 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -927,6 +927,8 @@ private:
     AMDGPUSignalManagerTy *SignalManager;
   };
 
+  using AMDGPUStreamCallbackTy = Error(void *Data);
+
   /// The stream is composed of N stream's slots. The struct below represents
   /// the fields of each slot. Each slot has a signal and an optional action
   /// function. When appending an HSA asynchronous operation to the stream, one
@@ -942,65 +944,82 @@ private:
     /// operation as input signal.
     AMDGPUSignalTy *Signal;
 
-    /// The action that must be performed after the operation's completion. Set
+    /// The actions that must be performed after the operation's completion. Set
     /// to nullptr when there is no action to perform.
-    Error (*ActionFunction)(void *);
+    llvm::SmallVector<AMDGPUStreamCallbackTy *> Callbacks;
 
     /// Space for the action's arguments. A pointer to these arguments is passed
     /// to the action function. Notice the space of arguments is limited.
-    union {
+    union ActionArgsTy {
       MemcpyArgsTy MemcpyArgs;
       ReleaseBufferArgsTy ReleaseBufferArgs;
       ReleaseSignalArgsTy ReleaseSignalArgs;
-    } ActionArgs;
+      void *CallbackArgs;
+    };
+
+    llvm::SmallVector<ActionArgsTy> ActionArgs;
 
     /// Create an empty slot.
-    StreamSlotTy() : Signal(nullptr), ActionFunction(nullptr) {}
+    StreamSlotTy() : Signal(nullptr), Callbacks({}), ActionArgs({}) {}
 
     /// Schedule a host memory copy action on the slot.
     Error schedHostMemoryCopy(void *Dst, const void *Src, size_t Size) {
-      ActionFunction = memcpyAction;
-      ActionArgs.MemcpyArgs = MemcpyArgsTy{Dst, Src, Size};
+      Callbacks.emplace_back(memcpyAction);
+      ActionArgs.emplace_back().MemcpyArgs = MemcpyArgsTy{Dst, Src, Size};
       return Plugin::success();
     }
 
     /// Schedule a release buffer action on the slot.
     Error schedReleaseBuffer(void *Buffer, AMDGPUMemoryManagerTy &Manager) {
-      ActionFunction = releaseBufferAction;
-      ActionArgs.ReleaseBufferArgs = ReleaseBufferArgsTy{Buffer, &Manager};
+      Callbacks.emplace_back(releaseBufferAction);
+      ActionArgs.emplace_back().ReleaseBufferArgs =
+          ReleaseBufferArgsTy{Buffer, &Manager};
       return Plugin::success();
     }
 
     /// Schedule a signal release action on the slot.
     Error schedReleaseSignal(AMDGPUSignalTy *SignalToRelease,
                              AMDGPUSignalManagerTy *SignalManager) {
-      ActionFunction = releaseSignalAction;
-      ActionArgs.ReleaseSignalArgs =
+      Callbacks.emplace_back(releaseSignalAction);
+      ActionArgs.emplace_back().ReleaseSignalArgs =
           ReleaseSignalArgsTy{SignalToRelease, SignalManager};
       return Plugin::success();
     }
 
+    /// Register a callback to be called on compleition
+    Error schedCallback(AMDGPUStreamCallbackTy *Func, void *Data) {
+      Callbacks.emplace_back(Func);
+      ActionArgs.emplace_back().CallbackArgs = Data;
+
+      return Plugin::success();
+    }
+
     // Perform the action if needed.
     Error performAction() {
-      if (!ActionFunction)
+      if (Callbacks.empty())
         return Plugin::success();
 
-      // Perform the action.
-      if (ActionFunction == memcpyAction) {
-        if (auto Err = memcpyAction(&ActionArgs))
-          return Err;
-      } else if (ActionFunction == releaseBufferAction) {
-        if (auto Err = releaseBufferAction(&ActionArgs))
-          return Err;
-      } else if (ActionFunction == releaseSignalAction) {
-        if (auto Err = releaseSignalAction(&ActionArgs))
-          return Err;
-      } else {
-        return Plugin::error("Unknown action function!");
+      assert(Callbacks.size() == ActionArgs.size() && "Size mismatch");
+      for (auto [Callback, ActionArg] : llvm::zip(Callbacks, ActionArgs)) {
+        // Perform the action.
+        if (Callback == memcpyAction) {
+          if (auto Err = memcpyAction(&ActionArg))
+            return Err;
+        } else if (Callback == releaseBufferAction) {
+          if (auto Err = releaseBufferAction(&ActionArg))
+            return Err;
+        } else if (Callback == releaseSignalAction) {
+          if (auto Err = releaseSignalAction(&ActionArg))
+            return Err;
+        } else if (Callback) {
+          if (auto Err = Callback(ActionArg.CallbackArgs))
+            return Err;
+        }
       }
 
       // Invalidate the action.
-      ActionFunction = nullptr;
+      Callbacks.clear();
+      ActionArgs.clear();
 
       return Plugin::success();
     }
-- 
GitLab


From 4e1b9d34f922d3b8b04a65f29681cd95dc9ce75f Mon Sep 17 00:00:00 2001
From: Afanasyev Ivan <ivafanas@gmail.com>
Date: Wed, 30 Oct 2024 00:26:15 +0700
Subject: [PATCH 470/693] [mir-strip-debug] Fix debug location info strip for
 bundled instructions (#113676)

Fix bug that `mir-strip-debug` pass does not remove debug location from
bundled instructions.

Problem arises during testing that debug info does not affect
optimization passes output (`llvm-lit` with ` -Dllc="llc
-debugify-and-strip-all-safe"`), when pass operates on MIR with bundled
instructions + memory operands.

Let mir test check looks like:

```
CHECK-NEXT: BUNDLE {
CHECK-NEXT:   $r3 = LD $r1, $r2 :: (load (s64) from %ir.a, !tbaa !2)
CHECK-NEXT: }
```

So as `mir-strip-debug` pass does not process bundled instructions,
running `llc -debugify-and-strip-all-safe` on the test will produce the
following output:

```
BUNDLE {
  $r3 = LD $r1, $r2, debug-location !DILocation(line: 3, column: 1, scope: <0x608cb2b99b10>) :: (load (s64) from %ir.a, !tbaa !2)
}
```

And test will fail, but it shouldn't.

Seems like the root cause is that `mir-strip-debug` pass should remove
debug location from bundled instructions.
---
 llvm/lib/CodeGen/MachineStripDebug.cpp        |  2 +-
 .../CodeGen/Generic/MIRStripDebug/bundles.mir | 63 +++++++++++++++++++
 2 files changed, 64 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/Generic/MIRStripDebug/bundles.mir

diff --git a/llvm/lib/CodeGen/MachineStripDebug.cpp b/llvm/lib/CodeGen/MachineStripDebug.cpp
index 6128248a028e..ea291f64bff4 100644
--- a/llvm/lib/CodeGen/MachineStripDebug.cpp
+++ b/llvm/lib/CodeGen/MachineStripDebug.cpp
@@ -50,7 +50,7 @@ struct StripDebugMachineModule : public ModulePass {
         continue;
       MachineFunction &MF = *MaybeMF;
       for (MachineBasicBlock &MBB : MF) {
-        for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) {
+        for (MachineInstr &MI : llvm::make_early_inc_range(MBB.instrs())) {
           if (MI.isDebugInstr()) {
             // FIXME: We should remove all of them. However, AArch64 emits an
             //        invalid `DBG_VALUE $lr` with only one operand instead of
diff --git a/llvm/test/CodeGen/Generic/MIRStripDebug/bundles.mir b/llvm/test/CodeGen/Generic/MIRStripDebug/bundles.mir
new file mode 100644
index 000000000000..111c886f585c
--- /dev/null
+++ b/llvm/test/CodeGen/Generic/MIRStripDebug/bundles.mir
@@ -0,0 +1,63 @@
+# RUN: llc -run-pass=mir-strip-debug -o - %s | FileCheck %s
+# RUN: llc -run-pass=mir-strip-debug,mir-debugify,mir-strip-debug -o - %s | FileCheck %s
+
+--- |
+  source_filename = "loc-only.ll"
+  
+  define i32 @test(i32 %a, i32 %b) !dbg !6 {
+    %add = add i32 %a, 2, !dbg !12
+    call void @llvm.dbg.value(metadata i32 %add, metadata !9, metadata !DIExpression()), !dbg !12
+    %sub = sub i32 %add, %b, !dbg !13
+    call void @llvm.dbg.value(metadata i32 %sub, metadata !11, metadata !DIExpression()), !dbg !13
+    ret i32 %sub, !dbg !14
+  }
+  
+  declare void @llvm.dbg.value(metadata, metadata, metadata)
+  
+  !llvm.dbg.cu = !{!0}
+  ; CHECK-NOT: !llvm.dbg.cu
+  !llvm.debugify = !{!3, !4}
+  ; CHECK-NOT: !llvm.debugify
+  !llvm.module.flags = !{!5}
+  ; CHECK-NOT: !llvm.module.flags
+
+  ; CHECK-NOT: !DI
+  !0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+  !1 = !DIFile(filename: "<stdin>", directory: "/")
+  !2 = !{}
+  !3 = !{i32 3}
+  !4 = !{i32 2}
+  !5 = !{i32 2, !"Debug Info Version", i32 3}
+  !6 = distinct !DISubprogram(name: "test", linkageName: "test", scope: null, file: !1, line: 1, type: !7, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
+  !7 = !DISubroutineType(types: !2)
+  !8 = !{!9, !11}
+  !9 = !DILocalVariable(name: "1", scope: !6, file: !1, line: 1, type: !10)
+  !10 = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_unsigned)
+  !11 = !DILocalVariable(name: "2", scope: !6, file: !1, line: 2, type: !10)
+  !12 = !DILocation(line: 1, column: 1, scope: !6)
+  !13 = !DILocation(line: 2, column: 1, scope: !6)
+  !14 = !DILocation(line: 3, column: 1, scope: !6)
+
+...
+---
+name:            test
+body:             |
+  bb.1 (%ir-block.0):
+    %0:_(s32) = G_IMPLICIT_DEF
+    %1:_(s32) = G_IMPLICIT_DEF
+    BUNDLE {
+      %2:_(s32) = G_CONSTANT i32 2, debug-location !DILocation(line: 0, scope: !6)
+      %3:_(s32) = G_ADD %0, %1, debug-location !12
+    }
+
+    ; CHECK-LABEL: body:
+    ; CHECK-NOT: debug-location
+    ; CHECK-NOT: !DI
+    ; CHECK-NEXT:    bb
+    ; CHECK-NEXT:      %0:_(s32) = G_IMPLICIT_DEF{{$}}
+    ; CHECK-NEXT:      %1:_(s32) = G_IMPLICIT_DEF{{$}}
+    ; CHECK-NEXT:      BUNDLE {
+    ; CHECK-NEXT:        %2:_(s32) = G_CONSTANT i32 2{{$}}
+    ; CHECK-NEXT:        %3:_(s32) = G_ADD %0, %1{{$}}
+    ; CHECK-NEXT:      }
+...
-- 
GitLab


From b510cdb895b9188e5819c4c85a6dab22a4d14385 Mon Sep 17 00:00:00 2001
From: Steven Wu <stevenwu@apple.com>
Date: Tue, 29 Oct 2024 10:29:39 -0700
Subject: [PATCH 471/693] [ADT] Add TrieRawHashMap (#69528)

Implement TrieRawHashMap can be used to store object with its associated
hash. User needs to supply a strong hashing function to guarantee the
uniqueness of the hash of the objects to be inserted. A hash collision
is not supported and will lead to error or failed to insert.

TrieRawHashMap is thread-safe and lock-free and can be used as
foundation data structure to implement a content addressible storage.
TrieRawHashMap owns the data stored in it and is designed to be:
* Fast to lookup.
* Fast to "insert" if the data has already been inserted.
* Can be used without lock and doesn't require any knowledge of the
participating threads or extra coordination between threads.

It is not currently designed to be used to insert unique new data with
high contention, due to the limitation on the memory allocator.
---
 .../include/llvm/ADT/TrieHashIndexGenerator.h | 122 +++++
 llvm/include/llvm/ADT/TrieRawHashMap.h        | 377 +++++++++++++
 llvm/lib/Support/CMakeLists.txt               |   1 +
 llvm/lib/Support/TrieRawHashMap.cpp           | 515 ++++++++++++++++++
 llvm/unittests/ADT/CMakeLists.txt             |   1 +
 llvm/unittests/ADT/TrieRawHashMapTest.cpp     | 346 ++++++++++++
 6 files changed, 1362 insertions(+)
 create mode 100644 llvm/include/llvm/ADT/TrieHashIndexGenerator.h
 create mode 100644 llvm/include/llvm/ADT/TrieRawHashMap.h
 create mode 100644 llvm/lib/Support/TrieRawHashMap.cpp
 create mode 100644 llvm/unittests/ADT/TrieRawHashMapTest.cpp

diff --git a/llvm/include/llvm/ADT/TrieHashIndexGenerator.h b/llvm/include/llvm/ADT/TrieHashIndexGenerator.h
new file mode 100644
index 000000000000..6f7e53b6b11b
--- /dev/null
+++ b/llvm/include/llvm/ADT/TrieHashIndexGenerator.h
@@ -0,0 +1,122 @@
+//===- TrieHashIndexGenerator.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_TRIEHASHINDEXGENERATOR_H
+#define LLVM_ADT_TRIEHASHINDEXGENERATOR_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include <optional>
+
+namespace llvm {
+
+/// The utility class that helps computing the index of the object inside trie
+/// from its hash. The generator can be configured with the number of bits
+/// used for each level of trie structure with \c NumRootsBits and \c
+/// NumSubtrieBits.
+/// For example, try computing indexes for a 16-bit hash 0x1234 with 8-bit root
+/// and 4-bit sub-trie:
+///
+///   IndexGenerator IndexGen{8, 4, Hash};
+///   size_t index1 = IndexGen.next(); // index 18 in root node.
+///   size_t index2 = IndexGen.next(); // index 3 in sub-trie level 1.
+///   size_t index3 = IndexGen.next(); // index 4 in sub-tire level 2.
+///
+/// This is used by different trie implementation to figure out where to
+/// insert/find the object in the data structure.
+struct TrieHashIndexGenerator {
+  size_t NumRootBits;
+  size_t NumSubtrieBits;
+  ArrayRef<uint8_t> Bytes;
+  std::optional<size_t> StartBit = std::nullopt;
+
+  // Get the number of bits used to generate current index.
+  size_t getNumBits() const {
+    assert(StartBit);
+    size_t TotalNumBits = Bytes.size() * 8;
+    assert(*StartBit <= TotalNumBits);
+    return std::min(*StartBit ? NumSubtrieBits : NumRootBits,
+                    TotalNumBits - *StartBit);
+  }
+
+  // Get the index of the object in the next level of trie.
+  size_t next() {
+    if (!StartBit) {
+      // Compute index for root when StartBit is not set.
+      StartBit = 0;
+      return getIndex(Bytes, *StartBit, NumRootBits);
+    }
+    if (*StartBit < Bytes.size() * 8) {
+      // Compute index for sub-trie.
+      *StartBit += *StartBit ? NumSubtrieBits : NumRootBits;
+      assert((*StartBit - NumRootBits) % NumSubtrieBits == 0);
+      return getIndex(Bytes, *StartBit, NumSubtrieBits);
+    }
+    // All the bits are consumed.
+    return end();
+  }
+
+  // Provide a hint to speed up the index generation by providing the
+  // information of the hash in current level. For example, if the object is
+  // known to have \c Index on a level that already consumes first n \c Bits of
+  // the hash, it can start index generation from this level by calling \c hint
+  // function.
+  size_t hint(unsigned Index, unsigned Bit) {
+    assert(Bit < Bytes.size() * 8);
+    assert(Bit == 0 || (Bit - NumRootBits) % NumSubtrieBits == 0);
+    StartBit = Bit;
+    return Index;
+  }
+
+  // Utility function for looking up the index in the trie for an object that
+  // has colliding hash bits in the front as the hash of the object that is
+  // currently being computed.
+  size_t getCollidingBits(ArrayRef<uint8_t> CollidingBits) const {
+    assert(StartBit);
+    return getIndex(CollidingBits, *StartBit, NumSubtrieBits);
+  }
+
+  size_t end() const { return SIZE_MAX; }
+
+  // Compute the index for the object from its hash, current start bits, and
+  // the number of bits used for current level.
+  static size_t getIndex(ArrayRef<uint8_t> Bytes, size_t StartBit,
+                         size_t NumBits) {
+    assert(StartBit < Bytes.size() * 8);
+    // Drop all the bits before StartBit.
+    Bytes = Bytes.drop_front(StartBit / 8u);
+    StartBit %= 8u;
+    size_t Index = 0;
+    // Compute the index using the bits in range [StartBit, StartBit + NumBits),
+    // note the range can spread across few `uint8_t` in the array.
+    for (uint8_t Byte : Bytes) {
+      size_t ByteStart = 0, ByteEnd = 8;
+      if (StartBit) {
+        ByteStart = StartBit;
+        Byte &= (1u << (8 - StartBit)) - 1u;
+        StartBit = 0;
+      }
+      size_t CurrentNumBits = ByteEnd - ByteStart;
+      if (CurrentNumBits > NumBits) {
+        Byte >>= CurrentNumBits - NumBits;
+        CurrentNumBits = NumBits;
+      }
+      Index <<= CurrentNumBits;
+      Index |= Byte & ((1u << CurrentNumBits) - 1u);
+
+      assert(NumBits >= CurrentNumBits);
+      NumBits -= CurrentNumBits;
+      if (!NumBits)
+        break;
+    }
+    return Index;
+  }
+};
+
+} // namespace llvm
+
+#endif // LLVM_ADT_TRIEHASHINDEXGENERATOR_H
diff --git a/llvm/include/llvm/ADT/TrieRawHashMap.h b/llvm/include/llvm/ADT/TrieRawHashMap.h
new file mode 100644
index 000000000000..5bfe5c9e6a0f
--- /dev/null
+++ b/llvm/include/llvm/ADT/TrieRawHashMap.h
@@ -0,0 +1,377 @@
+//===- TrieRawHashMap.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_TRIERAWHASHMAP_H
+#define LLVM_ADT_TRIERAWHASHMAP_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include <atomic>
+#include <optional>
+
+namespace llvm {
+
+class raw_ostream;
+
+/// TrieRawHashMap - is a lock-free thread-safe trie that is can be used to
+/// store/index data based on a hash value. It can be customized to work with
+/// any hash algorithm or store any data.
+///
+/// Data structure:
+/// Data node stored in the Trie contains both hash and data:
+/// struct {
+///    HashT Hash;
+///    DataT Data;
+/// };
+///
+/// Data is stored/indexed via a prefix tree, where each node in the tree can be
+/// either the root, a sub-trie or a data node. Assuming a 4-bit hash and two
+/// data objects {0001, A} and {0100, B}, it can be stored in a trie
+/// (assuming Root has 2 bits, SubTrie has 1 bit):
+///  +--------+
+///  |Root[00]| -> {0001, A}
+///  |    [01]| -> {0100, B}
+///  |    [10]| (empty)
+///  |    [11]| (empty)
+///  +--------+
+///
+/// Inserting a new object {0010, C} will result in:
+///  +--------+    +----------+
+///  |Root[00]| -> |SubTrie[0]| -> {0001, A}
+///  |        |    |       [1]| -> {0010, C}
+///  |        |    +----------+
+///  |    [01]| -> {0100, B}
+///  |    [10]| (empty)
+///  |    [11]| (empty)
+///  +--------+
+/// Note object A is sunk down to a sub-trie during the insertion. All the
+/// nodes are inserted through compare-exchange to ensure thread-safe and
+/// lock-free.
+///
+/// To find an object in the trie, walk the tree with prefix of the hash until
+/// the data node is found. Then the hash is compared with the hash stored in
+/// the data node to see if the is the same object.
+///
+/// Hash collision is not allowed so it is recommended to use trie with a
+/// "strong" hashing algorithm. A well-distributed hash can also result in
+/// better performance and memory usage.
+///
+/// It currently does not support iteration and deletion.
+
+/// Base class for a lock-free thread-safe hash-mapped trie.
+class ThreadSafeTrieRawHashMapBase {
+public:
+  static constexpr size_t TrieContentBaseSize = 4;
+  static constexpr size_t DefaultNumRootBits = 6;
+  static constexpr size_t DefaultNumSubtrieBits = 4;
+
+private:
+  template <class T> struct AllocValueType {
+    char Base[TrieContentBaseSize];
+    std::aligned_union_t<sizeof(T), T> Content;
+  };
+
+protected:
+  template <class T>
+  static constexpr size_t DefaultContentAllocSize = sizeof(AllocValueType<T>);
+
+  template <class T>
+  static constexpr size_t DefaultContentAllocAlign = alignof(AllocValueType<T>);
+
+  template <class T>
+  static constexpr size_t DefaultContentOffset =
+      offsetof(AllocValueType<T>, Content);
+
+public:
+  static void *operator new(size_t Size) { return ::operator new(Size); }
+  void operator delete(void *Ptr) { ::operator delete(Ptr); }
+
+  LLVM_DUMP_METHOD void dump() const;
+  void print(raw_ostream &OS) const;
+
+protected:
+  /// Result of a lookup. Suitable for an insertion hint. Maybe could be
+  /// expanded into an iterator of sorts, but likely not useful (visiting
+  /// everything in the trie should probably be done some way other than
+  /// through an iterator pattern).
+  class PointerBase {
+  protected:
+    void *get() const { return I == -2u ? P : nullptr; }
+
+  public:
+    PointerBase() noexcept = default;
+
+  private:
+    friend class ThreadSafeTrieRawHashMapBase;
+    explicit PointerBase(void *Content) : P(Content), I(-2u) {}
+    PointerBase(void *P, unsigned I, unsigned B) : P(P), I(I), B(B) {}
+
+    bool isHint() const { return I != -1u && I != -2u; }
+
+    void *P = nullptr;
+    unsigned I = -1u;
+    unsigned B = 0;
+  };
+
+  /// Find the stored content with hash.
+  PointerBase find(ArrayRef<uint8_t> Hash) const;
+
+  /// Insert and return the stored content.
+  PointerBase
+  insert(PointerBase Hint, ArrayRef<uint8_t> Hash,
+         function_ref<const uint8_t *(void *Mem, ArrayRef<uint8_t> Hash)>
+             Constructor);
+
+  ThreadSafeTrieRawHashMapBase() = delete;
+
+  ThreadSafeTrieRawHashMapBase(
+      size_t ContentAllocSize, size_t ContentAllocAlign, size_t ContentOffset,
+      std::optional<size_t> NumRootBits = std::nullopt,
+      std::optional<size_t> NumSubtrieBits = std::nullopt);
+
+  /// Destructor, which asserts if there's anything to do. Subclasses should
+  /// call \a destroyImpl().
+  ///
+  /// \pre \a destroyImpl() was already called.
+  ~ThreadSafeTrieRawHashMapBase();
+  void destroyImpl(function_ref<void(void *ValueMem)> Destructor);
+
+  ThreadSafeTrieRawHashMapBase(ThreadSafeTrieRawHashMapBase &&RHS);
+
+  // Move assignment is not supported as it is not thread-safe.
+  ThreadSafeTrieRawHashMapBase &
+  operator=(ThreadSafeTrieRawHashMapBase &&RHS) = delete;
+
+  // No copy.
+  ThreadSafeTrieRawHashMapBase(const ThreadSafeTrieRawHashMapBase &) = delete;
+  ThreadSafeTrieRawHashMapBase &
+  operator=(const ThreadSafeTrieRawHashMapBase &) = delete;
+
+  // Debug functions. Implementation details and not guaranteed to be
+  // thread-safe.
+  PointerBase getRoot() const;
+  unsigned getStartBit(PointerBase P) const;
+  unsigned getNumBits(PointerBase P) const;
+  unsigned getNumSlotUsed(PointerBase P) const;
+  std::string getTriePrefixAsString(PointerBase P) const;
+  unsigned getNumTries() const;
+  // Visit next trie in the allocation chain.
+  PointerBase getNextTrie(PointerBase P) const;
+
+private:
+  friend class TrieRawHashMapTestHelper;
+  const unsigned short ContentAllocSize;
+  const unsigned short ContentAllocAlign;
+  const unsigned short ContentOffset;
+  unsigned short NumRootBits;
+  unsigned short NumSubtrieBits;
+  class ImplType;
+  // ImplPtr is owned by ThreadSafeTrieRawHashMapBase and needs to be freed in
+  // destroyImpl.
+  std::atomic<ImplType *> ImplPtr;
+  ImplType &getOrCreateImpl();
+  ImplType *getImpl() const;
+};
+
+/// Lock-free thread-safe hash-mapped trie.
+template <class T, size_t NumHashBytes>
+class ThreadSafeTrieRawHashMap : public ThreadSafeTrieRawHashMapBase {
+public:
+  using HashT = std::array<uint8_t, NumHashBytes>;
+
+  class LazyValueConstructor;
+  struct value_type {
+    const HashT Hash;
+    T Data;
+
+    value_type(value_type &&) = default;
+    value_type(const value_type &) = default;
+
+    value_type(ArrayRef<uint8_t> Hash, const T &Data)
+        : Hash(makeHash(Hash)), Data(Data) {}
+    value_type(ArrayRef<uint8_t> Hash, T &&Data)
+        : Hash(makeHash(Hash)), Data(std::move(Data)) {}
+
+  private:
+    friend class LazyValueConstructor;
+
+    struct EmplaceTag {};
+    template <class... ArgsT>
+    value_type(ArrayRef<uint8_t> Hash, EmplaceTag, ArgsT &&...Args)
+        : Hash(makeHash(Hash)), Data(std::forward<ArgsT>(Args)...) {}
+
+    static HashT makeHash(ArrayRef<uint8_t> HashRef) {
+      HashT Hash;
+      std::copy(HashRef.begin(), HashRef.end(), Hash.data());
+      return Hash;
+    }
+  };
+
+  using ThreadSafeTrieRawHashMapBase::operator delete;
+  using HashType = HashT;
+
+  using ThreadSafeTrieRawHashMapBase::dump;
+  using ThreadSafeTrieRawHashMapBase::print;
+
+private:
+  template <class ValueT> class PointerImpl : PointerBase {
+    friend class ThreadSafeTrieRawHashMap;
+
+    ValueT *get() const {
+      return reinterpret_cast<ValueT *>(PointerBase::get());
+    }
+
+  public:
+    ValueT &operator*() const {
+      assert(get());
+      return *get();
+    }
+    ValueT *operator->() const {
+      assert(get());
+      return get();
+    }
+    explicit operator bool() const { return get(); }
+
+    PointerImpl() = default;
+
+  protected:
+    PointerImpl(PointerBase Result) : PointerBase(Result) {}
+  };
+
+public:
+  class pointer;
+  class const_pointer;
+  class pointer : public PointerImpl<value_type> {
+    friend class ThreadSafeTrieRawHashMap;
+    friend class const_pointer;
+
+  public:
+    pointer() = default;
+
+  private:
+    pointer(PointerBase Result) : pointer::PointerImpl(Result) {}
+  };
+
+  class const_pointer : public PointerImpl<const value_type> {
+    friend class ThreadSafeTrieRawHashMap;
+
+  public:
+    const_pointer() = default;
+    const_pointer(const pointer &P) : const_pointer::PointerImpl(P) {}
+
+  private:
+    const_pointer(PointerBase Result) : const_pointer::PointerImpl(Result) {}
+  };
+
+  class LazyValueConstructor {
+  public:
+    value_type &operator()(T &&RHS) {
+      assert(Mem && "Constructor already called, or moved away");
+      return assign(::new (Mem) value_type(Hash, std::move(RHS)));
+    }
+    value_type &operator()(const T &RHS) {
+      assert(Mem && "Constructor already called, or moved away");
+      return assign(::new (Mem) value_type(Hash, RHS));
+    }
+    template <class... ArgsT> value_type &emplace(ArgsT &&...Args) {
+      assert(Mem && "Constructor already called, or moved away");
+      return assign(::new (Mem)
+                        value_type(Hash, typename value_type::EmplaceTag{},
+                                   std::forward<ArgsT>(Args)...));
+    }
+
+    LazyValueConstructor(LazyValueConstructor &&RHS)
+        : Mem(RHS.Mem), Result(RHS.Result), Hash(RHS.Hash) {
+      RHS.Mem = nullptr; // Moved away, cannot call.
+    }
+    ~LazyValueConstructor() { assert(!Mem && "Constructor never called!"); }
+
+  private:
+    value_type &assign(value_type *V) {
+      Mem = nullptr;
+      Result = V;
+      return *V;
+    }
+    friend class ThreadSafeTrieRawHashMap;
+    LazyValueConstructor() = delete;
+    LazyValueConstructor(void *Mem, value_type *&Result, ArrayRef<uint8_t> Hash)
+        : Mem(Mem), Result(Result), Hash(Hash) {
+      assert(Hash.size() == sizeof(HashT) && "Invalid hash");
+      assert(Mem && "Invalid memory for construction");
+    }
+    void *Mem;
+    value_type *&Result;
+    ArrayRef<uint8_t> Hash;
+  };
+
+  /// Insert with a hint. Default-constructed hint will work, but it's
+  /// recommended to start with a lookup to avoid overhead in object creation
+  /// if it already exists.
+  pointer insertLazy(const_pointer Hint, ArrayRef<uint8_t> Hash,
+                     function_ref<void(LazyValueConstructor)> OnConstruct) {
+    return pointer(ThreadSafeTrieRawHashMapBase::insert(
+        Hint, Hash, [&](void *Mem, ArrayRef<uint8_t> Hash) {
+          value_type *Result = nullptr;
+          OnConstruct(LazyValueConstructor(Mem, Result, Hash));
+          return Result->Hash.data();
+        }));
+  }
+
+  pointer insertLazy(ArrayRef<uint8_t> Hash,
+                     function_ref<void(LazyValueConstructor)> OnConstruct) {
+    return insertLazy(const_pointer(), Hash, OnConstruct);
+  }
+
+  pointer insert(const_pointer Hint, value_type &&HashedData) {
+    return insertLazy(Hint, HashedData.Hash, [&](LazyValueConstructor C) {
+      C(std::move(HashedData.Data));
+    });
+  }
+
+  pointer insert(const_pointer Hint, const value_type &HashedData) {
+    return insertLazy(Hint, HashedData.Hash,
+                      [&](LazyValueConstructor C) { C(HashedData.Data); });
+  }
+
+  pointer find(ArrayRef<uint8_t> Hash) {
+    assert(Hash.size() == std::tuple_size<HashT>::value);
+    return ThreadSafeTrieRawHashMapBase::find(Hash);
+  }
+
+  const_pointer find(ArrayRef<uint8_t> Hash) const {
+    assert(Hash.size() == std::tuple_size<HashT>::value);
+    return ThreadSafeTrieRawHashMapBase::find(Hash);
+  }
+
+  ThreadSafeTrieRawHashMap(std::optional<size_t> NumRootBits = std::nullopt,
+                           std::optional<size_t> NumSubtrieBits = std::nullopt)
+      : ThreadSafeTrieRawHashMapBase(DefaultContentAllocSize<value_type>,
+                                     DefaultContentAllocAlign<value_type>,
+                                     DefaultContentOffset<value_type>,
+                                     NumRootBits, NumSubtrieBits) {}
+
+  ~ThreadSafeTrieRawHashMap() {
+    if constexpr (std::is_trivially_destructible<value_type>::value)
+      this->destroyImpl(nullptr);
+    else
+      this->destroyImpl(
+          [](void *P) { static_cast<value_type *>(P)->~value_type(); });
+  }
+
+  // Move constructor okay.
+  ThreadSafeTrieRawHashMap(ThreadSafeTrieRawHashMap &&) = default;
+
+  // No move assignment or any copy.
+  ThreadSafeTrieRawHashMap &operator=(ThreadSafeTrieRawHashMap &&) = delete;
+  ThreadSafeTrieRawHashMap(const ThreadSafeTrieRawHashMap &) = delete;
+  ThreadSafeTrieRawHashMap &
+  operator=(const ThreadSafeTrieRawHashMap &) = delete;
+};
+
+} // namespace llvm
+
+#endif // LLVM_ADT_TRIERAWHASHMAP_H
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index 531bdeaca126..2ecaea4b02bf 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -256,6 +256,7 @@ add_llvm_component_library(LLVMSupport
   TimeProfiler.cpp
   Timer.cpp
   ToolOutputFile.cpp
+  TrieRawHashMap.cpp
   Twine.cpp
   TypeSize.cpp
   Unicode.cpp
diff --git a/llvm/lib/Support/TrieRawHashMap.cpp b/llvm/lib/Support/TrieRawHashMap.cpp
new file mode 100644
index 000000000000..9eeac0bbc5c2
--- /dev/null
+++ b/llvm/lib/Support/TrieRawHashMap.cpp
@@ -0,0 +1,515 @@
+//===- TrieRawHashMap.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/TrieRawHashMap.h"
+#include "llvm/ADT/LazyAtomicPointer.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/TrieHashIndexGenerator.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ThreadSafeAllocator.h"
+#include "llvm/Support/TrailingObjects.h"
+#include "llvm/Support/raw_ostream.h"
+#include <memory>
+
+using namespace llvm;
+
+namespace {
+struct TrieNode {
+  const bool IsSubtrie = false;
+
+  TrieNode(bool IsSubtrie) : IsSubtrie(IsSubtrie) {}
+
+  static void *operator new(size_t Size) { return ::operator new(Size); }
+  void operator delete(void *Ptr) { ::operator delete(Ptr); }
+};
+
+struct TrieContent final : public TrieNode {
+  const uint8_t ContentOffset;
+  const uint8_t HashSize;
+  const uint8_t HashOffset;
+
+  void *getValuePointer() const {
+    auto *Content = reinterpret_cast<const uint8_t *>(this) + ContentOffset;
+    return const_cast<uint8_t *>(Content);
+  }
+
+  ArrayRef<uint8_t> getHash() const {
+    auto *Begin = reinterpret_cast<const uint8_t *>(this) + HashOffset;
+    return ArrayRef(Begin, Begin + HashSize);
+  }
+
+  TrieContent(size_t ContentOffset, size_t HashSize, size_t HashOffset)
+      : TrieNode(/*IsSubtrie=*/false), ContentOffset(ContentOffset),
+        HashSize(HashSize), HashOffset(HashOffset) {}
+
+  static bool classof(const TrieNode *TN) { return !TN->IsSubtrie; }
+};
+
+static_assert(sizeof(TrieContent) ==
+                  ThreadSafeTrieRawHashMapBase::TrieContentBaseSize,
+              "Check header assumption!");
+
+class TrieSubtrie final
+    : public TrieNode,
+      private TrailingObjects<TrieSubtrie, LazyAtomicPointer<TrieNode>> {
+public:
+  using Slot = LazyAtomicPointer<TrieNode>;
+
+  Slot &get(size_t I) { return getTrailingObjects<Slot>()[I]; }
+  TrieNode *load(size_t I) { return get(I).load(); }
+
+  unsigned size() const { return Size; }
+
+  TrieSubtrie *
+  sink(size_t I, TrieContent &Content, size_t NumSubtrieBits, size_t NewI,
+       function_ref<TrieSubtrie *(std::unique_ptr<TrieSubtrie>)> Saver);
+
+  static std::unique_ptr<TrieSubtrie> create(size_t StartBit, size_t NumBits);
+
+  explicit TrieSubtrie(size_t StartBit, size_t NumBits);
+
+  static bool classof(const TrieNode *TN) { return TN->IsSubtrie; }
+
+  static constexpr size_t sizeToAlloc(unsigned NumBits) {
+    assert(NumBits < 20 && "Tries should have fewer than ~1M slots");
+    size_t Count = 1u << NumBits;
+    return totalSizeToAlloc<LazyAtomicPointer<TrieNode>>(Count);
+  }
+
+private:
+  // FIXME: Use a bitset to speed up access:
+  //
+  //     std::array<std::atomic<uint64_t>, NumSlots/64> IsSet;
+  //
+  // This will avoid needing to visit sparsely filled slots in
+  // \a ThreadSafeTrieRawHashMapBase::destroyImpl() when there's a non-trivial
+  // destructor.
+  //
+  // It would also greatly speed up iteration, if we add that some day, and
+  // allow get() to return one level sooner.
+  //
+  // This would be the algorithm for updating IsSet (after updating Slots):
+  //
+  //     std::atomic<uint64_t> &Bits = IsSet[I.High];
+  //     const uint64_t NewBit = 1ULL << I.Low;
+  //     uint64_t Old = 0;
+  //     while (!Bits.compare_exchange_weak(Old, Old | NewBit))
+  //       ;
+
+  // For debugging.
+  unsigned StartBit = 0;
+  unsigned NumBits = 0;
+  unsigned Size = 0;
+  friend class llvm::ThreadSafeTrieRawHashMapBase;
+  friend class TrailingObjects;
+
+public:
+  /// Linked list for ownership of tries. The pointer is owned by TrieSubtrie.
+  std::atomic<TrieSubtrie *> Next;
+};
+} // end namespace
+
+std::unique_ptr<TrieSubtrie> TrieSubtrie::create(size_t StartBit,
+                                                 size_t NumBits) {
+  void *Memory = ::operator new(sizeToAlloc(NumBits));
+  TrieSubtrie *S = ::new (Memory) TrieSubtrie(StartBit, NumBits);
+  return std::unique_ptr<TrieSubtrie>(S);
+}
+
+TrieSubtrie::TrieSubtrie(size_t StartBit, size_t NumBits)
+    : TrieNode(true), StartBit(StartBit), NumBits(NumBits), Size(1u << NumBits),
+      Next(nullptr) {
+  for (unsigned I = 0; I < Size; ++I)
+    new (&get(I)) Slot(nullptr);
+
+  static_assert(
+      std::is_trivially_destructible<LazyAtomicPointer<TrieNode>>::value,
+      "Expected no work in destructor for TrieNode");
+}
+
+// Sink the nodes down sub-trie when the object being inserted collides with
+// the index of existing object in the trie. In this case, a new sub-trie needs
+// to be allocated to hold existing object.
+TrieSubtrie *TrieSubtrie::sink(
+    size_t I, TrieContent &Content, size_t NumSubtrieBits, size_t NewI,
+    function_ref<TrieSubtrie *(std::unique_ptr<TrieSubtrie>)> Saver) {
+  // Create a new sub-trie that points to the existing object with the new
+  // index for the next level.
+  assert(NumSubtrieBits > 0);
+  std::unique_ptr<TrieSubtrie> S = create(StartBit + NumBits, NumSubtrieBits);
+
+  assert(NewI < Size);
+  S->get(NewI).store(&Content);
+
+  // Using compare_exchange to atomically add back the new sub-trie to the trie
+  // in the place of the exsiting object.
+  TrieNode *ExistingNode = &Content;
+  assert(I < Size);
+  if (get(I).compare_exchange_strong(ExistingNode, S.get()))
+    return Saver(std::move(S));
+
+  // Another thread created a subtrie already. Return it and let "S" be
+  // destructed.
+  return cast<TrieSubtrie>(ExistingNode);
+}
+
+class ThreadSafeTrieRawHashMapBase::ImplType final
+    : private TrailingObjects<ThreadSafeTrieRawHashMapBase::ImplType,
+                              TrieSubtrie> {
+public:
+  static std::unique_ptr<ImplType> create(size_t StartBit, size_t NumBits) {
+    size_t Size = sizeof(ImplType) + TrieSubtrie::sizeToAlloc(NumBits);
+    void *Memory = ::operator new(Size);
+    ImplType *Impl = ::new (Memory) ImplType(StartBit, NumBits);
+    return std::unique_ptr<ImplType>(Impl);
+  }
+
+  // Save the Subtrie into the ownship list of the trie structure in a
+  // thread-safe way. The ownership transfer is done by compare_exchange the
+  // pointer value inside the unique_ptr.
+  TrieSubtrie *save(std::unique_ptr<TrieSubtrie> S) {
+    assert(!S->Next && "Expected S to a freshly-constructed leaf");
+
+    TrieSubtrie *CurrentHead = nullptr;
+    // Add ownership of "S" to front of the list, so that Root -> S ->
+    // Root.Next. This works by repeatedly setting S->Next to a candidate value
+    // of Root.Next (initially nullptr), then setting Root.Next to S once the
+    // candidate matches reality.
+    while (!getRoot()->Next.compare_exchange_weak(CurrentHead, S.get()))
+      S->Next.exchange(CurrentHead);
+
+    // Ownership transferred to subtrie successfully. Release the unique_ptr.
+    return S.release();
+  }
+
+  // Get the root which is the trailing object.
+  TrieSubtrie *getRoot() { return getTrailingObjects<TrieSubtrie>(); }
+
+  static void *operator new(size_t Size) { return ::operator new(Size); }
+  void operator delete(void *Ptr) { ::operator delete(Ptr); }
+
+  /// FIXME: This should take a function that allocates and constructs the
+  /// content lazily (taking the hash as a separate parameter), in case of
+  /// collision.
+  ThreadSafeAllocator<BumpPtrAllocator> ContentAlloc;
+
+private:
+  friend class TrailingObjects;
+
+  ImplType(size_t StartBit, size_t NumBits) {
+    ::new (getRoot()) TrieSubtrie(StartBit, NumBits);
+  }
+};
+
+ThreadSafeTrieRawHashMapBase::ImplType &
+ThreadSafeTrieRawHashMapBase::getOrCreateImpl() {
+  if (ImplType *Impl = ImplPtr.load())
+    return *Impl;
+
+  // Create a new ImplType and store it if another thread doesn't do so first.
+  // If another thread wins this one is destroyed locally.
+  std::unique_ptr<ImplType> Impl = ImplType::create(0, NumRootBits);
+  ImplType *ExistingImpl = nullptr;
+
+  // If the ownership transferred succesfully, release unique_ptr and return
+  // the pointer to the new ImplType.
+  if (ImplPtr.compare_exchange_strong(ExistingImpl, Impl.get()))
+    return *Impl.release();
+
+  // Already created, return the existing ImplType.
+  return *ExistingImpl;
+}
+
+ThreadSafeTrieRawHashMapBase::PointerBase
+ThreadSafeTrieRawHashMapBase::find(ArrayRef<uint8_t> Hash) const {
+  assert(!Hash.empty() && "Uninitialized hash");
+
+  ImplType *Impl = ImplPtr.load();
+  if (!Impl)
+    return PointerBase();
+
+  TrieSubtrie *S = Impl->getRoot();
+  TrieHashIndexGenerator IndexGen{NumRootBits, NumSubtrieBits, Hash};
+  size_t Index = IndexGen.next();
+  while (Index != IndexGen.end()) {
+    // Try to set the content.
+    TrieNode *Existing = S->get(Index);
+    if (!Existing)
+      return PointerBase(S, Index, *IndexGen.StartBit);
+
+    // Check for an exact match.
+    if (auto *ExistingContent = dyn_cast<TrieContent>(Existing))
+      return ExistingContent->getHash() == Hash
+                 ? PointerBase(ExistingContent->getValuePointer())
+                 : PointerBase(S, Index, *IndexGen.StartBit);
+
+    Index = IndexGen.next();
+    S = cast<TrieSubtrie>(Existing);
+  }
+  llvm_unreachable("failed to locate the node after consuming all hash bytes");
+}
+
+ThreadSafeTrieRawHashMapBase::PointerBase ThreadSafeTrieRawHashMapBase::insert(
+    PointerBase Hint, ArrayRef<uint8_t> Hash,
+    function_ref<const uint8_t *(void *Mem, ArrayRef<uint8_t> Hash)>
+        Constructor) {
+  assert(!Hash.empty() && "Uninitialized hash");
+
+  ImplType &Impl = getOrCreateImpl();
+  TrieSubtrie *S = Impl.getRoot();
+  TrieHashIndexGenerator IndexGen{NumRootBits, NumSubtrieBits, Hash};
+  size_t Index;
+  if (Hint.isHint()) {
+    S = static_cast<TrieSubtrie *>(Hint.P);
+    Index = IndexGen.hint(Hint.I, Hint.B);
+  } else {
+    Index = IndexGen.next();
+  }
+
+  while (Index != IndexGen.end()) {
+    // Load the node from the slot, allocating and calling the constructor if
+    // the slot is empty.
+    bool Generated = false;
+    TrieNode &Existing = S->get(Index).loadOrGenerate([&]() {
+      Generated = true;
+
+      // Construct the value itself at the tail.
+      uint8_t *Memory = reinterpret_cast<uint8_t *>(
+          Impl.ContentAlloc.Allocate(ContentAllocSize, ContentAllocAlign));
+      const uint8_t *HashStorage = Constructor(Memory + ContentOffset, Hash);
+
+      // Construct the TrieContent header, passing in the offset to the hash.
+      TrieContent *Content = ::new (Memory)
+          TrieContent(ContentOffset, Hash.size(), HashStorage - Memory);
+      assert(Hash == Content->getHash() && "Hash not properly initialized");
+      return Content;
+    });
+    // If we just generated it, return it!
+    if (Generated)
+      return PointerBase(cast<TrieContent>(Existing).getValuePointer());
+
+    if (auto *ST = dyn_cast<TrieSubtrie>(&Existing)) {
+      S = ST;
+      Index = IndexGen.next();
+      continue;
+    }
+
+    // Return the existing content if it's an exact match!
+    auto &ExistingContent = cast<TrieContent>(Existing);
+    if (ExistingContent.getHash() == Hash)
+      return PointerBase(ExistingContent.getValuePointer());
+
+    // Sink the existing content as long as the indexes match.
+    size_t NextIndex = IndexGen.next();
+    while (NextIndex != IndexGen.end()) {
+      size_t NewIndexForExistingContent =
+          IndexGen.getCollidingBits(ExistingContent.getHash());
+      S = S->sink(Index, ExistingContent, IndexGen.getNumBits(),
+                  NewIndexForExistingContent,
+                  [&Impl](std::unique_ptr<TrieSubtrie> S) {
+                    return Impl.save(std::move(S));
+                  });
+      Index = NextIndex;
+
+      // Found the difference.
+      if (NextIndex != NewIndexForExistingContent)
+        break;
+
+      NextIndex = IndexGen.next();
+    }
+  }
+  llvm_unreachable("failed to insert the node after consuming all hash bytes");
+}
+
+ThreadSafeTrieRawHashMapBase::ThreadSafeTrieRawHashMapBase(
+    size_t ContentAllocSize, size_t ContentAllocAlign, size_t ContentOffset,
+    std::optional<size_t> NumRootBits, std::optional<size_t> NumSubtrieBits)
+    : ContentAllocSize(ContentAllocSize), ContentAllocAlign(ContentAllocAlign),
+      ContentOffset(ContentOffset),
+      NumRootBits(NumRootBits ? *NumRootBits : DefaultNumRootBits),
+      NumSubtrieBits(NumSubtrieBits ? *NumSubtrieBits : DefaultNumSubtrieBits),
+      ImplPtr(nullptr) {
+  // Assertion checks for reasonable configuration. The settings below are not
+  // hard limits on most platforms, but a reasonable configuration should fall
+  // within those limits.
+  assert((!NumRootBits || *NumRootBits < 20) &&
+         "Root should have fewer than ~1M slots");
+  assert((!NumSubtrieBits || *NumSubtrieBits < 10) &&
+         "Subtries should have fewer than ~1K slots");
+}
+
+ThreadSafeTrieRawHashMapBase::ThreadSafeTrieRawHashMapBase(
+    ThreadSafeTrieRawHashMapBase &&RHS)
+    : ContentAllocSize(RHS.ContentAllocSize),
+      ContentAllocAlign(RHS.ContentAllocAlign),
+      ContentOffset(RHS.ContentOffset), NumRootBits(RHS.NumRootBits),
+      NumSubtrieBits(RHS.NumSubtrieBits) {
+  // Steal the root from RHS.
+  ImplPtr = RHS.ImplPtr.exchange(nullptr);
+}
+
+ThreadSafeTrieRawHashMapBase::~ThreadSafeTrieRawHashMapBase() {
+  assert(!ImplPtr.load() && "Expected subclass to call destroyImpl()");
+}
+
+void ThreadSafeTrieRawHashMapBase::destroyImpl(
+    function_ref<void(void *)> Destructor) {
+  std::unique_ptr<ImplType> Impl(ImplPtr.exchange(nullptr));
+  if (!Impl)
+    return;
+
+  // Destroy content nodes throughout trie. Avoid destroying any subtries since
+  // we need TrieNode::classof() to find the content nodes.
+  //
+  // FIXME: Once we have bitsets (see FIXME in TrieSubtrie class), use them
+  // facilitate sparse iteration here.
+  if (Destructor)
+    for (TrieSubtrie *Trie = Impl->getRoot(); Trie; Trie = Trie->Next.load())
+      for (unsigned I = 0; I < Trie->size(); ++I)
+        if (auto *Content = dyn_cast_or_null<TrieContent>(Trie->load(I)))
+          Destructor(Content->getValuePointer());
+
+  // Destroy the subtries. Incidentally, this destroys them in the reverse order
+  // of saving.
+  TrieSubtrie *Trie = Impl->getRoot()->Next;
+  while (Trie) {
+    TrieSubtrie *Next = Trie->Next.exchange(nullptr);
+    delete Trie;
+    Trie = Next;
+  }
+}
+
+ThreadSafeTrieRawHashMapBase::PointerBase
+ThreadSafeTrieRawHashMapBase::getRoot() const {
+  ImplType *Impl = ImplPtr.load();
+  if (!Impl)
+    return PointerBase();
+  return PointerBase(Impl->getRoot());
+}
+
+unsigned ThreadSafeTrieRawHashMapBase::getStartBit(
+    ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+  assert(!P.isHint() && "Not a valid trie");
+  if (!P.P)
+    return 0;
+  if (auto *S = dyn_cast<TrieSubtrie>((TrieNode *)P.P))
+    return S->StartBit;
+  return 0;
+}
+
+unsigned ThreadSafeTrieRawHashMapBase::getNumBits(
+    ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+  assert(!P.isHint() && "Not a valid trie");
+  if (!P.P)
+    return 0;
+  if (auto *S = dyn_cast<TrieSubtrie>((TrieNode *)P.P))
+    return S->NumBits;
+  return 0;
+}
+
+unsigned ThreadSafeTrieRawHashMapBase::getNumSlotUsed(
+    ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+  assert(!P.isHint() && "Not a valid trie");
+  if (!P.P)
+    return 0;
+  auto *S = dyn_cast<TrieSubtrie>((TrieNode *)P.P);
+  if (!S)
+    return 0;
+  unsigned Num = 0;
+  for (unsigned I = 0, E = S->size(); I < E; ++I)
+    if (auto *E = S->load(I))
+      ++Num;
+  return Num;
+}
+
+std::string ThreadSafeTrieRawHashMapBase::getTriePrefixAsString(
+    ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+  assert(!P.isHint() && "Not a valid trie");
+  if (!P.P)
+    return "";
+
+  auto *S = dyn_cast<TrieSubtrie>((TrieNode *)P.P);
+  if (!S || !S->IsSubtrie)
+    return "";
+
+  // Find a TrieContent node which has hash stored. Depth search following the
+  // first used slot until a TrieContent node is found.
+  TrieSubtrie *Current = S;
+  TrieContent *Node = nullptr;
+  while (Current) {
+    TrieSubtrie *Next = nullptr;
+    // Find first used slot in the trie.
+    for (unsigned I = 0, E = Current->size(); I < E; ++I) {
+      auto *S = Current->load(I);
+      if (!S)
+        continue;
+
+      if (auto *Content = dyn_cast<TrieContent>(S))
+        Node = Content;
+      else if (auto *Sub = dyn_cast<TrieSubtrie>(S))
+        Next = Sub;
+      break;
+    }
+
+    // Found the node.
+    if (Node)
+      break;
+
+    // Continue to the next level if the node is not found.
+    Current = Next;
+  }
+
+  assert(Node && "malformed trie, cannot find TrieContent on leaf node");
+  // The prefix for the current trie is the first `StartBit` of the content
+  // stored underneath this subtrie.
+  std::string Str;
+  raw_string_ostream SS(Str);
+
+  unsigned StartFullBytes = (S->StartBit + 1) / 8 - 1;
+  SS << toHex(toStringRef(Node->getHash()).take_front(StartFullBytes),
+              /*LowerCase=*/true);
+
+  // For the part of the prefix that doesn't fill a byte, print raw bit values.
+  std::string Bits;
+  for (unsigned I = StartFullBytes * 8, E = S->StartBit; I < E; ++I) {
+    unsigned Index = I / 8;
+    unsigned Offset = 7 - I % 8;
+    Bits.push_back('0' + ((Node->getHash()[Index] >> Offset) & 1));
+  }
+
+  if (!Bits.empty())
+    SS << "[" << Bits << "]";
+
+  return SS.str();
+}
+
+unsigned ThreadSafeTrieRawHashMapBase::getNumTries() const {
+  ImplType *Impl = ImplPtr.load();
+  if (!Impl)
+    return 0;
+  unsigned Num = 0;
+  for (TrieSubtrie *Trie = Impl->getRoot(); Trie; Trie = Trie->Next.load())
+    ++Num;
+  return Num;
+}
+
+ThreadSafeTrieRawHashMapBase::PointerBase
+ThreadSafeTrieRawHashMapBase::getNextTrie(
+    ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+  assert(!P.isHint() && "Not a valid trie");
+  if (!P.P)
+    return PointerBase();
+  auto *S = dyn_cast<TrieSubtrie>((TrieNode *)P.P);
+  if (!S)
+    return PointerBase();
+  if (auto *E = S->Next.load())
+    return PointerBase(E);
+  return PointerBase();
+}
diff --git a/llvm/unittests/ADT/CMakeLists.txt b/llvm/unittests/ADT/CMakeLists.txt
index 745e4d9fb74a..b0077d5b54a3 100644
--- a/llvm/unittests/ADT/CMakeLists.txt
+++ b/llvm/unittests/ADT/CMakeLists.txt
@@ -86,6 +86,7 @@ add_llvm_unittest(ADTTests
   StringSetTest.cpp
   StringSwitchTest.cpp
   TinyPtrVectorTest.cpp
+  TrieRawHashMapTest.cpp
   TwineTest.cpp
   TypeSwitchTest.cpp
   TypeTraitsTest.cpp
diff --git a/llvm/unittests/ADT/TrieRawHashMapTest.cpp b/llvm/unittests/ADT/TrieRawHashMapTest.cpp
new file mode 100644
index 000000000000..c9081f547812
--- /dev/null
+++ b/llvm/unittests/ADT/TrieRawHashMapTest.cpp
@@ -0,0 +1,346 @@
+//===- TrieRawHashMapTest.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/TrieRawHashMap.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/SHA1.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace llvm {
+class TrieRawHashMapTestHelper {
+public:
+  TrieRawHashMapTestHelper() = default;
+
+  void setTrie(ThreadSafeTrieRawHashMapBase *T) { Trie = T; }
+
+  ThreadSafeTrieRawHashMapBase::PointerBase getRoot() const {
+    return Trie->getRoot();
+  }
+  unsigned getStartBit(ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+    return Trie->getStartBit(P);
+  }
+  unsigned getNumBits(ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+    return Trie->getNumBits(P);
+  }
+  unsigned getNumSlotUsed(ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+    return Trie->getNumSlotUsed(P);
+  }
+  unsigned getNumTries() const { return Trie->getNumTries(); }
+  std::string
+  getTriePrefixAsString(ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+    return Trie->getTriePrefixAsString(P);
+  }
+  ThreadSafeTrieRawHashMapBase::PointerBase
+  getNextTrie(ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+    return Trie->getNextTrie(P);
+  }
+
+private:
+  ThreadSafeTrieRawHashMapBase *Trie = nullptr;
+};
+} // namespace llvm
+
+namespace {
+template <typename DataType, size_t HashSize = sizeof(uint64_t)>
+class SimpleTrieHashMapTest : public TrieRawHashMapTestHelper,
+                              public ::testing::Test {
+public:
+  using NumType = DataType;
+  using HashType = std::array<uint8_t, HashSize>;
+  using TrieType = ThreadSafeTrieRawHashMap<DataType, sizeof(HashType)>;
+
+  TrieType &createTrie(size_t RootBits, size_t SubtrieBits) {
+    auto &Ret = Trie.emplace(RootBits, SubtrieBits);
+    TrieRawHashMapTestHelper::setTrie(&Ret);
+    return Ret;
+  }
+
+  void destroyTrie() { Trie.reset(); }
+  ~SimpleTrieHashMapTest() { destroyTrie(); }
+
+  // Use the number itself as hash to test the pathological case.
+  static HashType hash(uint64_t Num) {
+    uint64_t HashN =
+        llvm::support::endian::byte_swap(Num, llvm::endianness::big);
+    HashType Hash;
+    memcpy(&Hash[0], &HashN, sizeof(HashType));
+    return Hash;
+  };
+
+private:
+  std::optional<TrieType> Trie;
+};
+
+using SmallNodeTrieTest = SimpleTrieHashMapTest<uint64_t>;
+
+TEST_F(SmallNodeTrieTest, TrieAllocation) {
+  NumType Numbers[] = {
+      0x0, std::numeric_limits<NumType>::max(),      0x1, 0x2,
+      0x3, std::numeric_limits<NumType>::max() - 1u,
+  };
+
+  unsigned ExpectedTries[] = {
+      1,       // Allocate Root.
+      1,       // Both on the root.
+      64,      // 0 and 1 sinks all the way down.
+      64,      // no new allocation needed.
+      65,      // need a new node between 2 and 3.
+      65 + 63, // 63 new allocation to sink two big numbers all the way.
+  };
+
+  const char *ExpectedPrefix[] = {
+      "", // Root.
+      "", // Root.
+      "00000000000000[0000000]",
+      "00000000000000[0000000]",
+      "00000000000000[0000001]",
+      "ffffffffffffff[1111111]",
+  };
+
+  // Use root and subtrie sizes of 1 so this gets sunk quite deep.
+  auto &Trie = createTrie(/*RootBits=*/1, /*SubtrieBits=*/1);
+
+  for (unsigned I = 0; I < 6; ++I) {
+    // Lookup first to exercise hint code for deep tries.
+    TrieType::pointer Lookup = Trie.find(hash(Numbers[I]));
+    EXPECT_FALSE(Lookup);
+
+    Trie.insert(Lookup, TrieType::value_type(hash(Numbers[I]), Numbers[I]));
+    EXPECT_EQ(getNumTries(), ExpectedTries[I]);
+    EXPECT_EQ(getTriePrefixAsString(getNextTrie(getRoot())), ExpectedPrefix[I]);
+  }
+}
+
+TEST_F(SmallNodeTrieTest, TrieStructure) {
+  NumType Numbers[] = {
+      // Three numbers that will nest deeply to test (1) sinking subtries and
+      // (2) deep, non-trivial hints.
+      std::numeric_limits<NumType>::max(),
+      std::numeric_limits<NumType>::max() - 2u,
+      std::numeric_limits<NumType>::max() - 3u,
+      // One number to stay at the top-level.
+      0x37,
+  };
+
+  // Use root and subtrie sizes of 1 so this gets sunk quite deep.
+  auto &Trie = createTrie(/*RootBits=*/1, /*SubtrieBits=*/1);
+
+  for (NumType N : Numbers) {
+    // Lookup first to exercise hint code for deep tries.
+    TrieType::pointer Lookup = Trie.find(hash(N));
+    EXPECT_FALSE(Lookup);
+
+    Trie.insert(Lookup, TrieType::value_type(hash(N), N));
+  }
+  for (NumType N : Numbers) {
+    TrieType::pointer Lookup = Trie.find(hash(N));
+    EXPECT_TRUE(Lookup);
+    if (!Lookup)
+      continue;
+    EXPECT_EQ(hash(N), Lookup->Hash);
+    EXPECT_EQ(N, Lookup->Data);
+
+    // Confirm a subsequent insertion fails to overwrite by trying to insert a
+    // bad value.
+    auto Result = Trie.insert(Lookup, TrieType::value_type(hash(N), N - 1));
+    EXPECT_EQ(N, Result->Data);
+  }
+
+  // Check the trie so we can confirm the structure is correct. Each subtrie
+  // should have 2 slots. The root's index=0 should have the content for
+  // 0x37 directly, and index=1 should be a linked-list of subtries, finally
+  // ending with content for (max-2) and (max-3).
+  //
+  // Note: This structure is not exhaustive (too expensive to update tests),
+  // but it does test that the dump format is somewhat readable and that the
+  // basic structure is correct.
+  //
+  // Note: This test requires that the trie reads bytes starting from index 0
+  // of the array of uint8_t, and then reads each byte's bits from high to low.
+
+  // Check the Trie.
+  // We should allocated a total of 64 SubTries for 64 bit hash.
+  ASSERT_EQ(getNumTries(), 64u);
+  // Check the root trie. Two slots and both are used.
+  ASSERT_EQ(getNumSlotUsed(getRoot()), 2u);
+  // Check last subtrie.
+  // Last allocated trie is the next node in the allocation chain.
+  auto LastAlloctedSubTrie = getNextTrie(getRoot());
+  ASSERT_EQ(getTriePrefixAsString(LastAlloctedSubTrie),
+            "ffffffffffffff[1111110]");
+  ASSERT_EQ(getStartBit(LastAlloctedSubTrie), 63u);
+  ASSERT_EQ(getNumBits(LastAlloctedSubTrie), 1u);
+  ASSERT_EQ(getNumSlotUsed(LastAlloctedSubTrie), 2u);
+}
+
+TEST_F(SmallNodeTrieTest, TrieStructureSmallFinalSubtrie) {
+  NumType Numbers[] = {
+      // Three numbers that will nest deeply to test (1) sinking subtries and
+      // (2) deep, non-trivial hints.
+      std::numeric_limits<NumType>::max(),
+      std::numeric_limits<NumType>::max() - 2u,
+      std::numeric_limits<NumType>::max() - 3u,
+      // One number to stay at the top-level.
+      0x37,
+  };
+
+  // Use subtrie size of 5 to avoid hitting 64 evenly, making the final subtrie
+  // small.
+  auto &Trie = createTrie(/*RootBits=*/8, /*SubtrieBits=*/5);
+
+  for (NumType N : Numbers) {
+    // Lookup first to exercise hint code for deep tries.
+    TrieType::pointer Lookup = Trie.find(hash(N));
+    EXPECT_FALSE(Lookup);
+
+    Trie.insert(Lookup, TrieType::value_type(hash(N), N));
+  }
+  for (NumType N : Numbers) {
+    TrieType::pointer Lookup = Trie.find(hash(N));
+    ASSERT_TRUE(Lookup);
+    EXPECT_EQ(hash(N), Lookup->Hash);
+    EXPECT_EQ(N, Lookup->Data);
+
+    // Confirm a subsequent insertion fails to overwrite by trying to insert a
+    // bad value.
+    auto Result = Trie.insert(Lookup, TrieType::value_type(hash(N), N - 1));
+    EXPECT_EQ(N, Result->Data);
+  }
+
+  // Check the trie so we can confirm the structure is correct. The root
+  // should have 2^8=256 slots, most subtries should have 2^5=32 slots, and the
+  // deepest subtrie should have 2^1=2 slots (since (64-8)mod(5)=1).
+  // should have 2 slots. The root's index=0 should have the content for
+  // 0x37 directly, and index=1 should be a linked-list of subtries, finally
+  // ending with content for (max-2) and (max-3).
+  //
+  // Note: This structure is not exhaustive (too expensive to update tests),
+  // but it does test that the dump format is somewhat readable and that the
+  // basic structure is correct.
+  //
+  // Note: This test requires that the trie reads bytes starting from index 0
+  // of the array of uint8_t, and then reads each byte's bits from high to low.
+
+  // Check the Trie.
+  // 64 bit hash = 8 + 5 * 11 + 1, so 1 root, 11 8bit subtrie and 1 last level
+  // subtrie, 13 total.
+  ASSERT_EQ(getNumTries(), 13u);
+  // Check the root trie. Two slots and both are used.
+  ASSERT_EQ(getNumSlotUsed(getRoot()), 2u);
+  // Check last subtrie.
+  // Last allocated trie is the next node in the allocation chain.
+  auto LastAlloctedSubTrie = getNextTrie(getRoot());
+  ASSERT_EQ(getTriePrefixAsString(LastAlloctedSubTrie),
+            "ffffffffffffff[1111110]");
+  ASSERT_EQ(getStartBit(LastAlloctedSubTrie), 63u);
+  ASSERT_EQ(getNumBits(LastAlloctedSubTrie), 1u);
+  ASSERT_EQ(getNumSlotUsed(LastAlloctedSubTrie), 2u);
+}
+
+TEST_F(SmallNodeTrieTest, TrieDestructionLoop) {
+  // Test destroying large Trie. Make sure there is no recursion that can
+  // overflow the stack.
+
+  // Limit the tries to 2 slots (1 bit) to generate subtries at a higher rate.
+  auto &Trie = createTrie(/*NumRootBits=*/1, /*NumSubtrieBits=*/1);
+
+  // Fill them up. Pick a MaxN high enough to cause a stack overflow in debug
+  // builds.
+  static constexpr uint64_t MaxN = 100000;
+  for (uint64_t N = 0; N != MaxN; ++N) {
+    HashType Hash = hash(N);
+    Trie.insert(TrieType::pointer(), TrieType::value_type(Hash, NumType{N}));
+  }
+
+  // Destroy tries. If destruction is recursive and MaxN is high enough, these
+  // will both fail.
+  destroyTrie();
+}
+
+struct NumWithDestructorT {
+  uint64_t Num;
+  llvm::function_ref<void()> DestructorCallback;
+  ~NumWithDestructorT() { DestructorCallback(); }
+};
+
+using NodeWithDestructorTrieTest = SimpleTrieHashMapTest<NumWithDestructorT>;
+
+TEST_F(NodeWithDestructorTrieTest, TrieDestructionLoop) {
+  // Test destroying large Trie. Make sure there is no recursion that can
+  // overflow the stack.
+
+  // Limit the tries to 2 slots (1 bit) to generate subtries at a higher rate.
+  auto &Trie = createTrie(/*NumRootBits=*/1, /*NumSubtrieBits=*/1);
+
+  // Fill them up. Pick a MaxN high enough to cause a stack overflow in debug
+  // builds.
+  static constexpr uint64_t MaxN = 100000;
+
+  uint64_t DestructorCalled = 0;
+  auto DtorCallback = [&DestructorCalled]() { ++DestructorCalled; };
+  for (uint64_t N = 0; N != MaxN; ++N) {
+    HashType Hash = hash(N);
+    Trie.insert(TrieType::pointer(),
+                TrieType::value_type(Hash, NumType{N, DtorCallback}));
+  }
+  // Reset the count after all the temporaries get destroyed.
+  DestructorCalled = 0;
+
+  // Destroy tries. If destruction is recursive and MaxN is high enough, these
+  // will both fail.
+  destroyTrie();
+
+  // Count the number of destructor calls during `destroyTrie()`.
+  ASSERT_EQ(DestructorCalled, MaxN);
+}
+
+using NumStrNodeTrieTest = SimpleTrieHashMapTest<std::string>;
+
+TEST_F(NumStrNodeTrieTest, TrieInsertLazy) {
+  for (unsigned RootBits : {2, 3, 6, 10}) {
+    for (unsigned SubtrieBits : {2, 3, 4}) {
+      auto &Trie = createTrie(RootBits, SubtrieBits);
+      for (int I = 0, E = 1000; I != E; ++I) {
+        TrieType::pointer Lookup;
+        HashType H = hash(I);
+        if (I & 1)
+          Lookup = Trie.find(H);
+
+        auto insertNum = [&](uint64_t Num) {
+          std::string S = Twine(I).str();
+          auto Hash = hash(Num);
+          return Trie.insertLazy(
+              Hash, [&](TrieType::LazyValueConstructor C) { C(std::move(S)); });
+        };
+        auto S1 = insertNum(I);
+        // The address of the Data should be the same.
+        EXPECT_EQ(&S1->Data, &insertNum(I)->Data);
+
+        auto insertStr = [&](std::string S) {
+          int Num = std::stoi(S);
+          return insertNum(Num);
+        };
+        std::string S2 = S1->Data;
+        // The address of the Data should be the same.
+        EXPECT_EQ(&S1->Data, &insertStr(S2)->Data);
+      }
+      for (int I = 0, E = 1000; I != E; ++I) {
+        std::string S = Twine(I).str();
+        TrieType::pointer Lookup = Trie.find(hash(I));
+        EXPECT_TRUE(Lookup);
+        if (!Lookup)
+          continue;
+        EXPECT_EQ(S, Lookup->Data);
+      }
+    }
+  }
+}
+} // end anonymous namespace
-- 
GitLab


From 950ee75909d94c582ecac4d3d559c364ed88244f Mon Sep 17 00:00:00 2001
From: Harald van Dijk <harald.vandijk@codeplay.com>
Date: Tue, 29 Oct 2024 17:30:30 +0000
Subject: [PATCH 472/693] [RISC-V] Fix check of minimum vlen. (#114055)

If we have a minimum vlen, we were adjusting StackSize to change the
unit from vscale to bytes, and then calculating the required padding
size for alignment in bytes. However, we then used that padding size as
an offset in vscale units, resulting in misplaced stack objects.

While it would be possible to adjust the object offsets by dividing
AlignmentPadding by ST.getRealMinVLen() / RISCV::RVVBitsPerBlock, we can
simplify the calculation a bit if instead we adjust the alignment to be
in vscale units.

@topperc This fixes a bug I am seeing after #110312, but I am not 100%
certain I am understanding the code correctly, could you please see if
this makes sense to you?
---
 llvm/lib/Target/RISCV/RISCVFrameLowering.cpp  |  24 ++---
 .../CodeGen/RISCV/rvv/allocate-lmul-2-4-8.ll  | 101 ++++++++++++++++++
 2 files changed, 113 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index b49cbab1876d..d70903519ecb 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -1133,23 +1133,23 @@ RISCVFrameLowering::assignRVVStackObjectOffsets(MachineFunction &MF) const {
 
   uint64_t StackSize = Offset;
 
-  // Multiply by vscale.
-  if (ST.getRealMinVLen() >= RISCV::RVVBitsPerBlock)
-    StackSize *= ST.getRealMinVLen() / RISCV::RVVBitsPerBlock;
-
   // Ensure the alignment of the RVV stack. Since we want the most-aligned
   // object right at the bottom (i.e., any padding at the top of the frame),
   // readjust all RVV objects down by the alignment padding.
-  if (auto AlignmentPadding = offsetToAlignment(StackSize, RVVStackAlign)) {
-    StackSize += AlignmentPadding;
-    for (int FI : ObjectsToAllocate)
-      MFI.setObjectOffset(FI, MFI.getObjectOffset(FI) - AlignmentPadding);
+  // Stack size and offsets are multiples of vscale, stack alignment is in
+  // bytes, we can divide stack alignment by minimum vscale to get a maximum
+  // stack alignment multiple of vscale.
+  auto VScale =
+      std::max<uint64_t>(ST.getRealMinVLen() / RISCV::RVVBitsPerBlock, 1);
+  if (auto RVVStackAlignVScale = RVVStackAlign.value() / VScale) {
+    if (auto AlignmentPadding =
+            offsetToAlignment(StackSize, Align(RVVStackAlignVScale))) {
+      StackSize += AlignmentPadding;
+      for (int FI : ObjectsToAllocate)
+        MFI.setObjectOffset(FI, MFI.getObjectOffset(FI) - AlignmentPadding);
+    }
   }
 
-  // Remove vscale.
-  if (ST.getRealMinVLen() >= RISCV::RVVBitsPerBlock)
-    StackSize /= ST.getRealMinVLen() / RISCV::RVVBitsPerBlock;
-
   return std::make_pair(StackSize, RVVStackAlign);
 }
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/allocate-lmul-2-4-8.ll b/llvm/test/CodeGen/RISCV/rvv/allocate-lmul-2-4-8.ll
index 35e269b91190..43be8feece23 100644
--- a/llvm/test/CodeGen/RISCV/rvv/allocate-lmul-2-4-8.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/allocate-lmul-2-4-8.ll
@@ -756,3 +756,104 @@ define void @lmul_8_x9() nounwind {
   %v9 = alloca <vscale x 8 x i64>
   ret void
 }
+
+define void @lmul_16_align() nounwind {
+; NOZBA-LABEL: lmul_16_align:
+; NOZBA:       # %bb.0:
+; NOZBA-NEXT:    addi sp, sp, -144
+; NOZBA-NEXT:    sd ra, 136(sp) # 8-byte Folded Spill
+; NOZBA-NEXT:    sd s0, 128(sp) # 8-byte Folded Spill
+; NOZBA-NEXT:    addi s0, sp, 144
+; NOZBA-NEXT:    csrr a0, vlenb
+; NOZBA-NEXT:    li a1, 24
+; NOZBA-NEXT:    mul a0, a0, a1
+; NOZBA-NEXT:    sub sp, sp, a0
+; NOZBA-NEXT:    andi sp, sp, -128
+; NOZBA-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; NOZBA-NEXT:    vmv.v.i v8, 0
+; NOZBA-NEXT:    csrr a0, vlenb
+; NOZBA-NEXT:    add a0, sp, a0
+; NOZBA-NEXT:    addi a0, a0, 128
+; NOZBA-NEXT:    vs8r.v v8, (a0)
+; NOZBA-NEXT:    csrr a1, vlenb
+; NOZBA-NEXT:    slli a1, a1, 3
+; NOZBA-NEXT:    add a0, a0, a1
+; NOZBA-NEXT:    vs8r.v v8, (a0)
+; NOZBA-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; NOZBA-NEXT:    vmv.v.i v8, 0
+; NOZBA-NEXT:    addi a0, sp, 128
+; NOZBA-NEXT:    vs1r.v v8, (a0)
+; NOZBA-NEXT:    addi sp, s0, -144
+; NOZBA-NEXT:    ld ra, 136(sp) # 8-byte Folded Reload
+; NOZBA-NEXT:    ld s0, 128(sp) # 8-byte Folded Reload
+; NOZBA-NEXT:    addi sp, sp, 144
+; NOZBA-NEXT:    ret
+;
+; ZBA-LABEL: lmul_16_align:
+; ZBA:       # %bb.0:
+; ZBA-NEXT:    addi sp, sp, -144
+; ZBA-NEXT:    sd ra, 136(sp) # 8-byte Folded Spill
+; ZBA-NEXT:    sd s0, 128(sp) # 8-byte Folded Spill
+; ZBA-NEXT:    addi s0, sp, 144
+; ZBA-NEXT:    csrr a0, vlenb
+; ZBA-NEXT:    slli a0, a0, 3
+; ZBA-NEXT:    sh1add a0, a0, a0
+; ZBA-NEXT:    sub sp, sp, a0
+; ZBA-NEXT:    andi sp, sp, -128
+; ZBA-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; ZBA-NEXT:    vmv.v.i v8, 0
+; ZBA-NEXT:    csrr a0, vlenb
+; ZBA-NEXT:    add a0, sp, a0
+; ZBA-NEXT:    addi a0, a0, 128
+; ZBA-NEXT:    vs8r.v v8, (a0)
+; ZBA-NEXT:    csrr a1, vlenb
+; ZBA-NEXT:    sh3add a0, a1, a0
+; ZBA-NEXT:    vs8r.v v8, (a0)
+; ZBA-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; ZBA-NEXT:    vmv.v.i v8, 0
+; ZBA-NEXT:    addi a0, sp, 128
+; ZBA-NEXT:    vs1r.v v8, (a0)
+; ZBA-NEXT:    addi sp, s0, -144
+; ZBA-NEXT:    ld ra, 136(sp) # 8-byte Folded Reload
+; ZBA-NEXT:    ld s0, 128(sp) # 8-byte Folded Reload
+; ZBA-NEXT:    addi sp, sp, 144
+; ZBA-NEXT:    ret
+;
+; NOMUL-LABEL: lmul_16_align:
+; NOMUL:       # %bb.0:
+; NOMUL-NEXT:    addi sp, sp, -144
+; NOMUL-NEXT:    sd ra, 136(sp) # 8-byte Folded Spill
+; NOMUL-NEXT:    sd s0, 128(sp) # 8-byte Folded Spill
+; NOMUL-NEXT:    addi s0, sp, 144
+; NOMUL-NEXT:    csrr a0, vlenb
+; NOMUL-NEXT:    slli a0, a0, 3
+; NOMUL-NEXT:    mv a1, a0
+; NOMUL-NEXT:    slli a0, a0, 1
+; NOMUL-NEXT:    add a0, a0, a1
+; NOMUL-NEXT:    sub sp, sp, a0
+; NOMUL-NEXT:    andi sp, sp, -128
+; NOMUL-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; NOMUL-NEXT:    vmv.v.i v8, 0
+; NOMUL-NEXT:    csrr a0, vlenb
+; NOMUL-NEXT:    add a0, sp, a0
+; NOMUL-NEXT:    addi a0, a0, 128
+; NOMUL-NEXT:    vs8r.v v8, (a0)
+; NOMUL-NEXT:    csrr a1, vlenb
+; NOMUL-NEXT:    slli a1, a1, 3
+; NOMUL-NEXT:    add a0, a0, a1
+; NOMUL-NEXT:    vs8r.v v8, (a0)
+; NOMUL-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; NOMUL-NEXT:    vmv.v.i v8, 0
+; NOMUL-NEXT:    addi a0, sp, 128
+; NOMUL-NEXT:    vs1r.v v8, (a0)
+; NOMUL-NEXT:    addi sp, s0, -144
+; NOMUL-NEXT:    ld ra, 136(sp) # 8-byte Folded Reload
+; NOMUL-NEXT:    ld s0, 128(sp) # 8-byte Folded Reload
+; NOMUL-NEXT:    addi sp, sp, 144
+; NOMUL-NEXT:    ret
+  %v1 = alloca <vscale x 16 x i64>
+  %v2 = alloca <vscale x 1 x i64>
+  store <vscale x 16 x i64> zeroinitializer, ptr %v1
+  store <vscale x 1 x i64> zeroinitializer, ptr %v2
+  ret void
+}
-- 
GitLab


From 4abc35740760b626d3fcabd001593d46c4b595af Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Tue, 29 Oct 2024 13:36:22 -0400
Subject: [PATCH 473/693] Nominate Sirraide for AST visitors and Sema (#114092)

Sirraide has been actively reviewing Sema code for a while now and
definitely has the expertise to help maintain that section of the
compiler. Further, he has been refactoring AST visitors to try to reduce
the compile time overhead associated with them and would be a good
resource for keeping an eye on that part of the code base too.
---
 clang/Maintainers.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/clang/Maintainers.rst b/clang/Maintainers.rst
index 08dcc584f6c5..9d3f6d25f60b 100644
--- a/clang/Maintainers.rst
+++ b/clang/Maintainers.rst
@@ -33,6 +33,12 @@ AST matchers
 | aaron\@aaronballman.com (email), aaron.ballman (Phabricator), AaronBallman (GitHub), AaronBallman (Discourse), aaronballman (Discord), AaronBallman (IRC)
 
 
+AST Visitors
+~~~~~~~~~~~~
+| Sirraide
+| aeternalmail\@gmail.com (email), Sirraide (GitHub), Ætérnal (Discord), Sirraide (Discourse)
+
+
 Clang LLVM IR generation
 ~~~~~~~~~~~~~~~~~~~~~~~~
 | John McCall
@@ -57,6 +63,12 @@ Analysis & CFG
 | sgatev\@google.com (email), sgatev (Phabricator), sgatev (GitHub)
 
 
+Sema
+~~~~
+| Sirraide
+| aeternalmail\@gmail.com (email), Sirraide (GitHub), Ætérnal (Discord), Sirraide (Discourse)
+
+
 Experimental new constant interpreter
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 | Timm Bäder
-- 
GitLab


From 639a7ac648f1e50ccd2556e17d401c04f9cce625 Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Tue, 29 Oct 2024 11:36:55 -0600
Subject: [PATCH 474/693] [Clang][AST] Store injected template arguments in
 TemplateParameterList (#113579)

Currently, we store injected template arguments in
`RedeclarableTemplateDecl::CommonBase`. This approach has a couple
problems:
1. We can only access the injected template arguments of
`RedeclarableTemplateDecl` derived types, but other `Decl` kinds still
make use of the injected arguments (e.g.
`ClassTemplatePartialSpecializationDecl`,
`VarTemplatePartialSpecializationDecl`, and `TemplateTemplateParmDecl`).
2. Accessing the injected template arguments requires the common data
structure to be allocated. This may occur before we determine whether a
previous declaration exists (e.g. when comparing constraints), so if the
template _is_ a redeclaration, we end up discarding the common data
structure.

This patch moves the storage and access of injected template arguments
from `RedeclarableTemplateDecl` to `TemplateParameterList`.
---
 clang/include/clang/AST/ASTContext.h       | 12 +---
 clang/include/clang/AST/DeclTemplate.h     | 44 +++++++--------
 clang/lib/AST/ASTContext.cpp               | 16 ++----
 clang/lib/AST/DeclTemplate.cpp             | 64 +++++-----------------
 clang/lib/Sema/SemaTemplateDeduction.cpp   | 13 ++---
 clang/lib/Sema/SemaTemplateInstantiate.cpp | 14 +++--
 6 files changed, 54 insertions(+), 109 deletions(-)

diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index a4d36f2eacd5..07b4e36f3ef0 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -239,7 +239,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
   mutable llvm::ContextualFoldingSet<DependentTemplateSpecializationType,
                                      ASTContext&>
     DependentTemplateSpecializationTypes;
-  llvm::FoldingSet<PackExpansionType> PackExpansionTypes;
+  mutable llvm::FoldingSet<PackExpansionType> PackExpansionTypes;
   mutable llvm::FoldingSet<ObjCObjectTypeImpl> ObjCObjectTypes;
   mutable llvm::FoldingSet<ObjCObjectPointerType> ObjCObjectPointerTypes;
   mutable llvm::FoldingSet<DependentUnaryTransformType>
@@ -1778,13 +1778,7 @@ public:
       ElaboratedTypeKeyword Keyword, NestedNameSpecifier *NNS,
       const IdentifierInfo *Name, ArrayRef<TemplateArgument> Args) const;
 
-  TemplateArgument getInjectedTemplateArg(NamedDecl *ParamDecl);
-
-  /// Get a template argument list with one argument per template parameter
-  /// in a template parameter list, such as for the injected class name of
-  /// a class template.
-  void getInjectedTemplateArgs(const TemplateParameterList *Params,
-                               SmallVectorImpl<TemplateArgument> &Args);
+  TemplateArgument getInjectedTemplateArg(NamedDecl *ParamDecl) const;
 
   /// Form a pack expansion type with the given pattern.
   /// \param NumExpansions The number of expansions for the pack, if known.
@@ -1795,7 +1789,7 @@ public:
   ///        if this is the canonical type of another pack expansion type.
   QualType getPackExpansionType(QualType Pattern,
                                 std::optional<unsigned> NumExpansions,
-                                bool ExpectPackInType = true);
+                                bool ExpectPackInType = true) const;
 
   QualType getObjCInterfaceType(const ObjCInterfaceDecl *Decl,
                                 ObjCInterfaceDecl *PrevDecl = nullptr) const;
diff --git a/clang/include/clang/AST/DeclTemplate.h b/clang/include/clang/AST/DeclTemplate.h
index 0f0c0bf6e4ef..a572e3380f16 100644
--- a/clang/include/clang/AST/DeclTemplate.h
+++ b/clang/include/clang/AST/DeclTemplate.h
@@ -71,6 +71,9 @@ NamedDecl *getAsNamedDecl(TemplateParameter P);
 class TemplateParameterList final
     : private llvm::TrailingObjects<TemplateParameterList, NamedDecl *,
                                     Expr *> {
+  /// The template argument list of the template parameter list.
+  TemplateArgument *InjectedArgs = nullptr;
+
   /// The location of the 'template' keyword.
   SourceLocation TemplateLoc;
 
@@ -196,6 +199,9 @@ public:
 
   bool hasAssociatedConstraints() const;
 
+  /// Get the template argument list of the template parameter list.
+  ArrayRef<TemplateArgument> getInjectedTemplateArgs(const ASTContext &Context);
+
   SourceLocation getTemplateLoc() const { return TemplateLoc; }
   SourceLocation getLAngleLoc() const { return LAngleLoc; }
   SourceLocation getRAngleLoc() const { return RAngleLoc; }
@@ -793,15 +799,6 @@ protected:
     /// The first value in the array is the number of specializations/partial
     /// specializations that follow.
     GlobalDeclID *LazySpecializations = nullptr;
-
-    /// The set of "injected" template arguments used within this
-    /// template.
-    ///
-    /// This pointer refers to the template arguments (there are as
-    /// many template arguments as template parameters) for the
-    /// template, and is allocated lazily, since most templates do not
-    /// require the use of this information.
-    TemplateArgument *InjectedArgs = nullptr;
   };
 
   /// Pointer to the common data shared by all declarations of this
@@ -927,7 +924,10 @@ public:
   /// Although the C++ standard has no notion of the "injected" template
   /// arguments for a template, the notion is convenient when
   /// we need to perform substitutions inside the definition of a template.
-  ArrayRef<TemplateArgument> getInjectedTemplateArgs();
+  ArrayRef<TemplateArgument>
+  getInjectedTemplateArgs(const ASTContext &Context) const {
+    return getTemplateParameters()->getInjectedTemplateArgs(Context);
+  }
 
   using redecl_range = redeclarable_base::redecl_range;
   using redecl_iterator = redeclarable_base::redecl_iterator;
@@ -2087,10 +2087,6 @@ class ClassTemplatePartialSpecializationDecl
   /// The list of template parameters
   TemplateParameterList *TemplateParams = nullptr;
 
-  /// The set of "injected" template arguments used within this
-  /// partial specialization.
-  TemplateArgument *InjectedArgs = nullptr;
-
   /// The class template partial specialization from which this
   /// class template partial specialization was instantiated.
   ///
@@ -2136,9 +2132,11 @@ public:
     return TemplateParams;
   }
 
-  /// Retrieve the template arguments list of the template parameter list
-  /// of this template.
-  ArrayRef<TemplateArgument> getInjectedTemplateArgs();
+  /// Get the template argument list of the template parameter list.
+  ArrayRef<TemplateArgument>
+  getInjectedTemplateArgs(const ASTContext &Context) const {
+    return getTemplateParameters()->getInjectedTemplateArgs(Context);
+  }
 
   /// \brief All associated constraints of this partial specialization,
   /// including the requires clause and any constraints derived from
@@ -2864,10 +2862,6 @@ class VarTemplatePartialSpecializationDecl
   /// The list of template parameters
   TemplateParameterList *TemplateParams = nullptr;
 
-  /// The set of "injected" template arguments used within this
-  /// partial specialization.
-  TemplateArgument *InjectedArgs = nullptr;
-
   /// The variable template partial specialization from which this
   /// variable template partial specialization was instantiated.
   ///
@@ -2914,9 +2908,11 @@ public:
     return TemplateParams;
   }
 
-  /// Retrieve the template arguments list of the template parameter list
-  /// of this template.
-  ArrayRef<TemplateArgument> getInjectedTemplateArgs();
+  /// Get the template argument list of the template parameter list.
+  ArrayRef<TemplateArgument>
+  getInjectedTemplateArgs(const ASTContext &Context) const {
+    return getTemplateParameters()->getInjectedTemplateArgs(Context);
+  }
 
   /// \brief All associated constraints of this partial specialization,
   /// including the requires clause and any constraints derived from
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 69892bda42b2..1c3f771f417c 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -5634,7 +5634,7 @@ ASTContext::getDependentTemplateSpecializationType(
   return QualType(T, 0);
 }
 
-TemplateArgument ASTContext::getInjectedTemplateArg(NamedDecl *Param) {
+TemplateArgument ASTContext::getInjectedTemplateArg(NamedDecl *Param) const {
   TemplateArgument Arg;
   if (const auto *TTP = dyn_cast<TemplateTypeParmDecl>(Param)) {
     QualType ArgType = getTypeDeclType(TTP);
@@ -5678,23 +5678,15 @@ TemplateArgument ASTContext::getInjectedTemplateArg(NamedDecl *Param) {
   }
 
   if (Param->isTemplateParameterPack())
-    Arg = TemplateArgument::CreatePackCopy(*this, Arg);
+    Arg =
+        TemplateArgument::CreatePackCopy(const_cast<ASTContext &>(*this), Arg);
 
   return Arg;
 }
 
-void
-ASTContext::getInjectedTemplateArgs(const TemplateParameterList *Params,
-                                    SmallVectorImpl<TemplateArgument> &Args) {
-  Args.reserve(Args.size() + Params->size());
-
-  for (NamedDecl *Param : *Params)
-    Args.push_back(getInjectedTemplateArg(Param));
-}
-
 QualType ASTContext::getPackExpansionType(QualType Pattern,
                                           std::optional<unsigned> NumExpansions,
-                                          bool ExpectPackInType) {
+                                          bool ExpectPackInType) const {
   assert((!ExpectPackInType || Pattern->containsUnexpandedParameterPack()) &&
          "Pack expansions must expand one or more parameter packs");
 
diff --git a/clang/lib/AST/DeclTemplate.cpp b/clang/lib/AST/DeclTemplate.cpp
index 4a506b7be456..755ec72f00bf 100644
--- a/clang/lib/AST/DeclTemplate.cpp
+++ b/clang/lib/AST/DeclTemplate.cpp
@@ -51,7 +51,7 @@ DefaultTemplateArgumentContainsUnexpandedPack(const TemplateParam &P) {
          P.getDefaultArgument().getArgument().containsUnexpandedParameterPack();
 }
 
-TemplateParameterList::TemplateParameterList(const ASTContext& C,
+TemplateParameterList::TemplateParameterList(const ASTContext &C,
                                              SourceLocation TemplateLoc,
                                              SourceLocation LAngleLoc,
                                              ArrayRef<NamedDecl *> Params,
@@ -244,6 +244,17 @@ bool TemplateParameterList::hasAssociatedConstraints() const {
   return HasRequiresClause || HasConstrainedParameters;
 }
 
+ArrayRef<TemplateArgument>
+TemplateParameterList::getInjectedTemplateArgs(const ASTContext &Context) {
+  if (!InjectedArgs) {
+    InjectedArgs = new (Context) TemplateArgument[size()];
+    llvm::transform(*this, InjectedArgs, [&](NamedDecl *ND) {
+      return Context.getInjectedTemplateArg(ND);
+    });
+  }
+  return {InjectedArgs, NumParams};
+}
+
 bool TemplateParameterList::shouldIncludeTypeForArgument(
     const PrintingPolicy &Policy, const TemplateParameterList *TPL,
     unsigned Idx) {
@@ -396,22 +407,6 @@ void RedeclarableTemplateDecl::addSpecializationImpl(
                                       SETraits::getDecl(Entry));
 }
 
-ArrayRef<TemplateArgument> RedeclarableTemplateDecl::getInjectedTemplateArgs() {
-  TemplateParameterList *Params = getTemplateParameters();
-  auto *CommonPtr = getCommonPtr();
-  if (!CommonPtr->InjectedArgs) {
-    auto &Context = getASTContext();
-    SmallVector<TemplateArgument, 16> TemplateArgs;
-    Context.getInjectedTemplateArgs(Params, TemplateArgs);
-    CommonPtr->InjectedArgs =
-        new (Context) TemplateArgument[TemplateArgs.size()];
-    std::copy(TemplateArgs.begin(), TemplateArgs.end(),
-              CommonPtr->InjectedArgs);
-  }
-
-  return llvm::ArrayRef(CommonPtr->InjectedArgs, Params->size());
-}
-
 //===----------------------------------------------------------------------===//
 // FunctionTemplateDecl Implementation
 //===----------------------------------------------------------------------===//
@@ -631,13 +626,10 @@ ClassTemplateDecl::getInjectedClassNameSpecialization() {
   //  expansion (14.5.3) whose pattern is the name of the template parameter
   //  pack.
   ASTContext &Context = getASTContext();
-  TemplateParameterList *Params = getTemplateParameters();
-  SmallVector<TemplateArgument, 16> TemplateArgs;
-  Context.getInjectedTemplateArgs(Params, TemplateArgs);
   TemplateName Name = Context.getQualifiedTemplateName(
       /*NNS=*/nullptr, /*TemplateKeyword=*/false, TemplateName(this));
-  CommonPtr->InjectedClassNameType =
-      Context.getTemplateSpecializationType(Name, TemplateArgs);
+  CommonPtr->InjectedClassNameType = Context.getTemplateSpecializationType(
+      Name, getTemplateParameters()->getInjectedTemplateArgs(Context));
   return CommonPtr->InjectedClassNameType;
 }
 
@@ -1184,20 +1176,6 @@ SourceRange ClassTemplatePartialSpecializationDecl::getSourceRange() const {
   return Range;
 }
 
-ArrayRef<TemplateArgument>
-ClassTemplatePartialSpecializationDecl::getInjectedTemplateArgs() {
-  TemplateParameterList *Params = getTemplateParameters();
-  auto *First = cast<ClassTemplatePartialSpecializationDecl>(getFirstDecl());
-  if (!First->InjectedArgs) {
-    auto &Context = getASTContext();
-    SmallVector<TemplateArgument, 16> TemplateArgs;
-    Context.getInjectedTemplateArgs(Params, TemplateArgs);
-    First->InjectedArgs = new (Context) TemplateArgument[TemplateArgs.size()];
-    std::copy(TemplateArgs.begin(), TemplateArgs.end(), First->InjectedArgs);
-  }
-  return llvm::ArrayRef(First->InjectedArgs, Params->size());
-}
-
 //===----------------------------------------------------------------------===//
 // FriendTemplateDecl Implementation
 //===----------------------------------------------------------------------===//
@@ -1548,20 +1526,6 @@ SourceRange VarTemplatePartialSpecializationDecl::getSourceRange() const {
   return Range;
 }
 
-ArrayRef<TemplateArgument>
-VarTemplatePartialSpecializationDecl::getInjectedTemplateArgs() {
-  TemplateParameterList *Params = getTemplateParameters();
-  auto *First = cast<VarTemplatePartialSpecializationDecl>(getFirstDecl());
-  if (!First->InjectedArgs) {
-    auto &Context = getASTContext();
-    SmallVector<TemplateArgument, 16> TemplateArgs;
-    Context.getInjectedTemplateArgs(Params, TemplateArgs);
-    First->InjectedArgs = new (Context) TemplateArgument[TemplateArgs.size()];
-    std::copy(TemplateArgs.begin(), TemplateArgs.end(), First->InjectedArgs);
-  }
-  return llvm::ArrayRef(First->InjectedArgs, Params->size());
-}
-
 static TemplateParameterList *
 createMakeIntegerSeqParameterList(const ASTContext &C, DeclContext *DC) {
   // typename T
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index db1d7fa23713..b45f30fed49a 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -6163,7 +6163,7 @@ struct TemplateArgumentListAreEqual {
             std::enable_if_t<!std::is_same_v<T1, T2>, bool> = true>
   bool operator()(T1 *Spec, T2 *Primary) {
     ArrayRef<TemplateArgument> Args1 = Spec->getTemplateArgs().asArray(),
-                               Args2 = Primary->getInjectedTemplateArgs();
+                               Args2 = Primary->getInjectedTemplateArgs(Ctx);
 
     for (unsigned I = 0, E = Args1.size(); I < E; ++I) {
       // We use profile, instead of structural comparison of the arguments,
@@ -6342,7 +6342,7 @@ bool Sema::isMoreSpecializedThanPrimary(
   VarTemplateDecl *Primary = Spec->getSpecializedTemplate();
   TemplateName Name(Primary);
   QualType PrimaryT = Context.getTemplateSpecializationType(
-      Name, Primary->getInjectedTemplateArgs());
+      Name, Primary->getInjectedTemplateArgs(Context));
   QualType PartialT = Context.getTemplateSpecializationType(
       Name, Spec->getTemplateArgs().asArray());
 
@@ -6372,18 +6372,14 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
   //    - Each function template has a single function parameter whose type is
   //      a specialization of X with template arguments corresponding to the
   //      template parameters from the respective function template
-  SmallVector<TemplateArgument, 8> AArgs;
-  Context.getInjectedTemplateArgs(A, AArgs);
+  SmallVector<TemplateArgument, 8> AArgs(A->getInjectedTemplateArgs(Context));
 
   // Check P's arguments against A's parameter list. This will fill in default
   // template arguments as needed. AArgs are already correct by construction.
   // We can't just use CheckTemplateIdType because that will expand alias
   // templates.
-  SmallVector<TemplateArgument, 4> PArgs;
+  SmallVector<TemplateArgument, 4> PArgs(P->getInjectedTemplateArgs(Context));
   {
-    SFINAETrap Trap(*this);
-
-    Context.getInjectedTemplateArgs(P, PArgs);
     TemplateArgumentListInfo PArgList(P->getLAngleLoc(),
                                       P->getRAngleLoc());
     for (unsigned I = 0, N = P->size(); I != N; ++I) {
@@ -6399,6 +6395,7 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
     }
     PArgs.clear();
 
+    SFINAETrap Trap(*this);
     // C++1z [temp.arg.template]p3:
     //   If the rewrite produces an invalid type, then P is not at least as
     //   specialized as A.
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 6a55861fe5af..dea97bfce532 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -200,7 +200,7 @@ struct TemplateInstantiationArgumentCollecter
     if (Innermost)
       AddInnermostTemplateArguments(FTD);
     else if (ForConstraintInstantiation)
-      AddOuterTemplateArguments(FTD, FTD->getInjectedTemplateArgs(),
+      AddOuterTemplateArguments(FTD, FTD->getInjectedTemplateArgs(S.Context),
                                 /*Final=*/false);
 
     if (FTD->isMemberSpecialization())
@@ -219,7 +219,7 @@ struct TemplateInstantiationArgumentCollecter
     if (Innermost)
       AddInnermostTemplateArguments(VTD);
     else if (ForConstraintInstantiation)
-      AddOuterTemplateArguments(VTD, VTD->getInjectedTemplateArgs(),
+      AddOuterTemplateArguments(VTD, VTD->getInjectedTemplateArgs(S.Context),
                                 /*Final=*/false);
 
     if (VTD->isMemberSpecialization())
@@ -237,7 +237,8 @@ struct TemplateInstantiationArgumentCollecter
     if (Innermost)
       AddInnermostTemplateArguments(VTPSD);
     else if (ForConstraintInstantiation)
-      AddOuterTemplateArguments(VTPSD, VTPSD->getInjectedTemplateArgs(),
+      AddOuterTemplateArguments(VTPSD,
+                                VTPSD->getInjectedTemplateArgs(S.Context),
                                 /*Final=*/false);
 
     if (VTPSD->isMemberSpecialization())
@@ -254,7 +255,7 @@ struct TemplateInstantiationArgumentCollecter
     if (Innermost)
       AddInnermostTemplateArguments(CTD);
     else if (ForConstraintInstantiation)
-      AddOuterTemplateArguments(CTD, CTD->getInjectedTemplateArgs(),
+      AddOuterTemplateArguments(CTD, CTD->getInjectedTemplateArgs(S.Context),
                                 /*Final=*/false);
 
     if (CTD->isMemberSpecialization())
@@ -274,7 +275,8 @@ struct TemplateInstantiationArgumentCollecter
     if (Innermost)
       AddInnermostTemplateArguments(CTPSD);
     else if (ForConstraintInstantiation)
-      AddOuterTemplateArguments(CTPSD, CTPSD->getInjectedTemplateArgs(),
+      AddOuterTemplateArguments(CTPSD,
+                                CTPSD->getInjectedTemplateArgs(S.Context),
                                 /*Final=*/false);
 
     if (CTPSD->isMemberSpecialization())
@@ -290,7 +292,7 @@ struct TemplateInstantiationArgumentCollecter
     if (Innermost)
       AddInnermostTemplateArguments(TATD);
     else if (ForConstraintInstantiation)
-      AddOuterTemplateArguments(TATD, TATD->getInjectedTemplateArgs(),
+      AddOuterTemplateArguments(TATD, TATD->getInjectedTemplateArgs(S.Context),
                                 /*Final=*/false);
 
     return UseNextDecl(TATD);
-- 
GitLab


From 449523fa0f957db0fff1c0cd9ec5f59e858ece0b Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Tue, 29 Oct 2024 13:38:54 -0400
Subject: [PATCH 475/693] Nominate Vassil Vassilev for Modules and Plugins
 (#114058)

Vassil has significant experience helping users with the plugin
interface in Clang, especially around the new efforts to bring plugin
support to Windows. He also is knowledgeable about modules support.
---
 clang/Maintainers.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/clang/Maintainers.rst b/clang/Maintainers.rst
index 9d3f6d25f60b..35c218d8e0e8 100644
--- a/clang/Maintainers.rst
+++ b/clang/Maintainers.rst
@@ -83,6 +83,9 @@ Modules & serialization
 | Michael Spencer
 | bigcheesegs\@gmail.com (email), Bigcheese (Phabricator), Bigcheese (GitHub)
 
+| Vassil Vassilev
+| Vassil.Vassilev\@cern.ch (email), v.g.vassilev (Phabricator), vgvassilev (GitHub)
+
 
 Templates
 ~~~~~~~~~
@@ -190,6 +193,12 @@ Attributes
 | ekeane\@nvidia.com (email), ErichKeane (Phabricator), erichkeane (GitHub)
 
 
+Plugins
+~~~~~~~
+| Vassil Vassilev
+| Vassil.Vassilev\@cern.ch (email), v.g.vassilev (Phabricator), vgvassilev (GitHub)
+
+
 Inline assembly
 ~~~~~~~~~~~~~~~
 | Eric Christopher
-- 
GitLab


From 528e975ac4081c7d84c5664c7ca9a18a916db4c7 Mon Sep 17 00:00:00 2001
From: Brox Chen <guochen2@amd.com>
Date: Tue, 29 Oct 2024 13:48:43 -0400
Subject: [PATCH 476/693] [AMDGPU][test]added unique and sort options for
 update_mc_test_check script (#111769)

add a unique and a sort option to the update_mc_test_check script.

These mc asm/dasm files are usually large in number of lines, and these
lines are mostly similar to each other. These options can be useful when
maintainer is merging or resolving conflicts by making the file
identifical

Also fixed a small issue in asm/dasm such that the auto generated header
line is
1. asm using ";" instead of "//" as comment marker
2. dasm using ";" instead of "#" as comment marker
---
 .../Inputs/amdgpu_asm.s.expected              |  2 +-
 .../Inputs/amdgpu_asm_err.s.expected          |  2 +-
 .../Inputs/amdgpu_asm_sort.s                  |  5 +
 .../Inputs/amdgpu_asm_sort.s.expected         |  8 ++
 .../Inputs/amdgpu_asm_sort_with_comment.s     |  9 ++
 .../amdgpu_asm_sort_with_comment.s.expected   | 13 +++
 .../Inputs/amdgpu_asm_unique.s                | 10 ++
 .../Inputs/amdgpu_asm_unique.s.expected       | 10 ++
 .../Inputs/amdgpu_dasm.txt.expected           |  2 +-
 .../Inputs/amdgpu_dasm_unique.txt             |  5 +
 .../Inputs/amdgpu_dasm_unique.txt.expected    |  5 +
 .../Inputs/amdgpu_multirun_dasm.txt.expected  |  2 +-
 .../update_mc_test_checks/amdgpu-sort.test    |  7 ++
 .../update_mc_test_checks/amdgpu-unique.test  |  7 ++
 llvm/utils/UpdateTestChecks/common.py         |  4 +-
 llvm/utils/update_mc_test_checks.py           | 92 +++++++++++++++++--
 16 files changed, 170 insertions(+), 13 deletions(-)
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort.s
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort.s.expected
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort_with_comment.s
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort_with_comment.s.expected
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_unique.s
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_unique.s.expected
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm_unique.txt
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm_unique.txt.expected
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_mc_test_checks/amdgpu-sort.test
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_mc_test_checks/amdgpu-unique.test

diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm.s.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm.s.expected
index 7336947a3f57..2dc30cd112e4 100644
--- a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm.s.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm.s.expected
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
 // RUN: llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
 
 v_bfrev_b32 v5, v1
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_err.s.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_err.s.expected
index 0a0ad51d15e0..ca287fc2d632 100644
--- a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_err.s.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_err.s.expected
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
 // RUN: not llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
 
 v_bfrev_b32 v5, v299
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort.s b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort.s
new file mode 100644
index 000000000000..ea03c5a6911f
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort.s
@@ -0,0 +1,5 @@
+// RUN: llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
+
+v_bfrev_b32 v5, v1
+
+v_bfrev_b32 v1, v1
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort.s.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort.s.expected
new file mode 100644
index 000000000000..57f72ed406fb
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort.s.expected
@@ -0,0 +1,8 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort
+// RUN: llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
+
+v_bfrev_b32 v1, v1
+// CHECK: v_bfrev_b32_e32 v1, v1                  ; encoding: [0x01,0x71,0x02,0x7e]
+
+v_bfrev_b32 v5, v1
+// CHECK: v_bfrev_b32_e32 v5, v1                  ; encoding: [0x01,0x71,0x0a,0x7e]
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort_with_comment.s b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort_with_comment.s
new file mode 100644
index 000000000000..d60b3bda29ed
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort_with_comment.s
@@ -0,0 +1,9 @@
+// RUN: llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
+
+v_bfrev_b32 v5, v1 //This is comment A
+
+v_bfrev_b32 v1, v1
+// This is comment B
+
+// This is comment C
+v_bfrev_b32 v2, v1
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort_with_comment.s.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort_with_comment.s.expected
new file mode 100644
index 000000000000..692488003271
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort_with_comment.s.expected
@@ -0,0 +1,13 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort
+// RUN: llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
+
+v_bfrev_b32 v1, v1
+// CHECK: v_bfrev_b32_e32 v1, v1                  ; encoding: [0x01,0x71,0x02,0x7e]
+// This is comment B
+
+// This is comment C
+v_bfrev_b32 v2, v1
+// CHECK: v_bfrev_b32_e32 v2, v1                  ; encoding: [0x01,0x71,0x04,0x7e]
+
+v_bfrev_b32 v5, v1 //This is comment A
+// CHECK: v_bfrev_b32_e32 v5, v1                  ; encoding: [0x01,0x71,0x0a,0x7e]
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_unique.s b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_unique.s
new file mode 100644
index 000000000000..63240174cdde
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_unique.s
@@ -0,0 +1,10 @@
+// RUN: llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
+
+//this is commentA
+v_bfrev_b32 v5, v1
+
+v_bfrev_b32 v5, v1
+
+//this is commentB
+
+//this is commentB
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_unique.s.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_unique.s.expected
new file mode 100644
index 000000000000..8203b90040ba
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_unique.s.expected
@@ -0,0 +1,10 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique
+// RUN: llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
+
+//this is commentA
+v_bfrev_b32 v5, v1
+// CHECK: v_bfrev_b32_e32 v5, v1                  ; encoding: [0x01,0x71,0x0a,0x7e]
+
+//this is commentB
+
+//this is commentB
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm.txt.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm.txt.expected
index a6f7abcb1774..b3cbaff6d1c7 100644
--- a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm.txt.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm.txt.expected
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble -show-encoding %s 2>&1 | FileCheck -check-prefixes=CHECK %s
 
 0x00,0x00,0x00,0x7e
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm_unique.txt b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm_unique.txt
new file mode 100644
index 000000000000..3d0d49ddeea4
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm_unique.txt
@@ -0,0 +1,5 @@
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble -show-encoding %s 2>&1 | FileCheck -check-prefixes=CHECK %s
+
+0x00,0x00,0x00,0x7e
+
+0x00,0x00,0x00,0x7e
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm_unique.txt.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm_unique.txt.expected
new file mode 100644
index 000000000000..32bddb20628d
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm_unique.txt.expected
@@ -0,0 +1,5 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble -show-encoding %s 2>&1 | FileCheck -check-prefixes=CHECK %s
+
+0x00,0x00,0x00,0x7e
+# CHECK: v_nop                                   ; encoding: [0x00,0x00,0x00,0x7e]
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_multirun_dasm.txt.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_multirun_dasm.txt.expected
index 03a5ec3c559d..7b6b83280162 100644
--- a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_multirun_dasm.txt.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_multirun_dasm.txt.expected
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
 # RUN: llvm-mc -triple=amdgcn -mcpu=tonga -disassemble -show-encoding %s 2>&1 | FileCheck -check-prefixes=CHECK,CHECKA %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble -show-encoding %s 2>&1 | FileCheck -check-prefixes=CHECK,CHECKB %s
 
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/amdgpu-sort.test b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/amdgpu-sort.test
new file mode 100644
index 000000000000..f8972ffabf09
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/amdgpu-sort.test
@@ -0,0 +1,7 @@
+# REQUIRES: amdgpu-registered-target
+## Check that sort is working
+
+# RUN: cp -f %S/Inputs/amdgpu_asm_sort.s %t.s && %update_mc_test_checks --sort %t.s
+# RUN: diff -u %S/Inputs/amdgpu_asm_sort.s.expected %t.s
+# RUN: cp -f %S/Inputs/amdgpu_asm_sort_with_comment.s %t.s && %update_mc_test_checks --sort %t.s
+# RUN: diff -u %S/Inputs/amdgpu_asm_sort_with_comment.s.expected %t.s
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/amdgpu-unique.test b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/amdgpu-unique.test
new file mode 100644
index 000000000000..8a5d83462cad
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/amdgpu-unique.test
@@ -0,0 +1,7 @@
+# REQUIRES: amdgpu-registered-target
+## Check that unique is working
+
+# RUN: cp -f %S/Inputs/amdgpu_asm_unique.s %t.s && %update_mc_test_checks --unique %t.s
+# RUN: diff -u %S/Inputs/amdgpu_asm_unique.s.expected %t.s
+# RUN: cp -f %S/Inputs/amdgpu_dasm_unique.txt %t.txt && %update_mc_test_checks --unique %t.txt
+# RUN: diff -u %S/Inputs/amdgpu_dasm_unique.txt.expected %t.txt
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index 0fbb73431d2c..cdfa8978566f 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -275,8 +275,10 @@ class TestInfo(object):
         self.run_lines = find_run_lines(test, self.input_lines)
         self.comment_prefix = comment_prefix
         if self.comment_prefix is None:
-            if self.path.endswith(".mir"):
+            if self.path.endswith(".mir") or self.path.endswith(".txt"):
                 self.comment_prefix = "#"
+            elif self.path.endswith(".s"):
+                self.comment_prefix = "//"
             else:
                 self.comment_prefix = ";"
         self.autogenerated_note_prefix = self.comment_prefix + " " + UTC_ADVERT
diff --git a/llvm/utils/update_mc_test_checks.py b/llvm/utils/update_mc_test_checks.py
index f9f8cfdea418..55ed6c82d487 100755
--- a/llvm/utils/update_mc_test_checks.py
+++ b/llvm/utils/update_mc_test_checks.py
@@ -6,6 +6,7 @@ A test update script.  This script is a utility to update LLVM 'llvm-mc' based t
 from __future__ import print_function
 
 import argparse
+import functools
 import os  # Used to advertise this file's name ("autogenerated_note").
 
 from UpdateTestChecks import common
@@ -50,6 +51,10 @@ def isTestLine(input_line, mc_mode):
     return True
 
 
+def isRunLine(l):
+    return common.RUN_LINE_RE.match(l)
+
+
 def hasErr(err):
     return err and ERROR_RE.search(err) is not None
 
@@ -118,6 +123,19 @@ def main():
         default=None,
         help="Set a default -march for when neither triple nor arch are found in a RUN line",
     )
+    parser.add_argument(
+        "--unique",
+        action="store_true",
+        default=False,
+        help="remove duplicated test line if found",
+    )
+    parser.add_argument(
+        "--sort",
+        action="store_true",
+        default=False,
+        help="sort testline in alphabetic order (keep run-lines on top), this option could be dangerous as it"
+        "could change the order of lines that are not expected",
+    )
     parser.add_argument("tests", nargs="+")
     initial_args = common.parse_commandline_args(parser)
 
@@ -130,6 +148,11 @@ def main():
             mc_mode = "asm"
         elif ti.path.endswith(".txt"):
             mc_mode = "dasm"
+
+            if ti.args.sort:
+                print("sorting with dasm(.txt) file is not supported!")
+                return -1
+
         else:
             common.warn("Expected .s and .txt, Skipping file : ", ti.path)
             continue
@@ -196,6 +219,10 @@ def main():
 
         # find all test line from input
         testlines = [l for l in ti.input_lines if isTestLine(l, mc_mode)]
+        # remove duplicated lines to save running time
+        testlines = list(dict.fromkeys(testlines))
+        common.debug("Valid test line found: ", len(testlines))
+
         run_list_size = len(run_list)
         testnum = len(testlines)
 
@@ -233,7 +260,7 @@ def main():
             raw_prefixes.append(prefixes)
 
         output_lines = []
-        generated_prefixes = []
+        generated_prefixes = {}
         used_prefixes = set()
         prefix_set = set([prefix for p in run_list for prefix in p[0]])
         common.debug("Rewriting FileCheck prefixes:", str(prefix_set))
@@ -298,23 +325,72 @@ def main():
                     else:
                         gen_prefix += getStdCheckLine(prefix, o, mc_mode)
 
-            generated_prefixes.append(gen_prefix.rstrip("\n"))
+            generated_prefixes[input_line] = gen_prefix.rstrip("\n")
 
         # write output
-        prefix_id = 0
         for input_info in ti.iterlines(output_lines):
             input_line = input_info.line
-            if isTestLine(input_line, mc_mode):
+            if input_line in testlines:
                 output_lines.append(input_line)
-                output_lines.append(generated_prefixes[prefix_id])
-                prefix_id += 1
+                output_lines.append(generated_prefixes[input_line])
 
             elif should_add_line_to_output(input_line, prefix_set, mc_mode):
                 output_lines.append(input_line)
 
-            elif input_line in ti.run_lines or input_line == "":
-                output_lines.append(input_line)
+        if ti.args.unique or ti.args.sort:
+            # split with double newlines
+            test_units = "\n".join(output_lines).split("\n\n")
+
+            # select the key line for each test unit
+            test_dic = {}
+            for unit in test_units:
+                lines = unit.split("\n")
+                for l in lines:
+                    # if contains multiple lines, use
+                    # the first testline or runline as key
+                    if isTestLine(l, mc_mode):
+                        test_dic[unit] = l
+                        break
+                    if isRunLine(l):
+                        test_dic[unit] = l
+                        break
+
+            # unique
+            if ti.args.unique:
+                new_test_units = []
+                written_lines = set()
+                for unit in test_units:
+                    # if not testline/runline, we just add it
+                    if unit not in test_dic:
+                        new_test_units.append(unit)
+                    else:
+                        if test_dic[unit] in written_lines:
+                            common.debug("Duplicated test skipped: ", unit)
+                            continue
+
+                        written_lines.add(test_dic[unit])
+                        new_test_units.append(unit)
+                test_units = new_test_units
+
+            # sort
+            if ti.args.sort:
+
+                def getkey(l):
+                    # find key of test unit, otherwise use first line
+                    if l in test_dic:
+                        line = test_dic[l]
+                    else:
+                        line = l.split("\n")[0]
+
+                    # runline placed on the top
+                    return (not isRunLine(line), line)
+
+                test_units = sorted(test_units, key=getkey)
+
+            # join back to be output string
+            output_lines = "\n\n".join(test_units).split("\n")
 
+        # output
         if ti.args.gen_unused_prefix_body:
             output_lines.extend(
                 ti.get_checks_for_unused_prefixes(run_list, used_prefixes)
-- 
GitLab


From ba65710908137fe68e7c039f1e2829c3d37480f3 Mon Sep 17 00:00:00 2001
From: Min-Yih Hsu <min.hsu@sifive.com>
Date: Tue, 29 Oct 2024 10:49:35 -0700
Subject: [PATCH 477/693] [RISCV] Avoid redundant SchedRead on _TIED VPseudos
 (#113940)

_TIED and _MASK_TIED pseudos have one less operand compared to other
pseudos, thus we shouldn't attach the same number of SchedRead for these
instructions.

I don't think we have a way to (explicitly) check scheduling classes. So
I only test this patch with existing tests.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoV.td | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
index 4e8619c5ec23..8e0c4826ac00 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
@@ -104,13 +104,28 @@ class SchedCommon<list<SchedWrite> writes, list<SchedRead> reads,
                   string mx = "WorstCase", int sew = 0, bit forceMasked = 0,
                   bit forcePassthruRead = 0> : Sched<[]> {
   defvar isMasked = !ne(!find(NAME, "_MASK"), -1);
+  defvar isTied = !ne(!find(NAME, "_TIED"), -1);
   defvar isMaskedOrForceMasked = !or(forceMasked, isMasked);
+  defvar isTiedMasked = !and(isMaskedOrForceMasked, isTied);
   defvar passthruRead = !if(!or(!eq(mx, "WorstCase"), !eq(sew, 0)),
                             !cast<SchedRead>("ReadVPassthru_" # mx),
                             !cast<SchedRead>("ReadVPassthru_" # mx # "_E" #sew));
-  defvar needsPassthruRead = !or(isMaskedOrForceMasked, forcePassthruRead);
+  // We don't need passthru operand if it's already _TIED without mask.
+  defvar needsForcePassthruRead = !and(forcePassthruRead, !not(isTied));
+  defvar needsPassthruRead = !or(isMaskedOrForceMasked, needsForcePassthruRead);
+  // If this is a _TIED + masked operation, $rs2 (i.e. the first operand) is
+  // merged with the mask.
+  // NOTE: the following if statement is written in such a weird way because
+  // should we want to write something like
+  // `!if(!and(!not(!empty(reads), isTiedMasked), !tail(reads), reads)`
+  // since `!if` doesn't have a proper short-circuit behavior, if the
+  // condition of this `!if` cannot be resolved right away, `!tail(reads)` will
+  // be immediately evaluated anyway even when `reads` is empty, which leads to
+  // an assertion failure.
+  defvar readsWithTiedMask =
+      !if(isTiedMasked, !if(!not(!empty(reads)), !tail(reads), reads), reads);
   defvar readsWithMask =
-      !if(isMaskedOrForceMasked, !listconcat(reads, [ReadVMask]), reads);
+      !if(isMaskedOrForceMasked, !listconcat(readsWithTiedMask, [ReadVMask]), reads);
   defvar allReads =
       !if(needsPassthruRead, !listconcat([passthruRead], readsWithMask), reads);
   let SchedRW = !listconcat(writes, allReads);
-- 
GitLab


From 6f66530fd17a2333939e6b5a46d378ac7379f7ca Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 29 Oct 2024 10:55:34 -0700
Subject: [PATCH 478/693] [mlir] Fix a warning

This patch fixes:

  mlir/lib/Pass/PassRegistry.cpp:425:37: error: ISO C++ requires the
  name after '::~' to be found in the same scope as the name before
  '::~' [-Werror,-Wdtor-name]
---
 mlir/lib/Pass/PassRegistry.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/lib/Pass/PassRegistry.cpp b/mlir/lib/Pass/PassRegistry.cpp
index 029512fd3ecc..fe8427559584 100644
--- a/mlir/lib/Pass/PassRegistry.cpp
+++ b/mlir/lib/Pass/PassRegistry.cpp
@@ -422,7 +422,7 @@ llvm::cl::OptionValue<OpPassManager>::operator=(
   return *this;
 }
 
-llvm::cl::OptionValue<OpPassManager>::~OptionValue() = default;
+llvm::cl::OptionValue<OpPassManager>::~OptionValue<OpPassManager>() = default;
 
 void llvm::cl::OptionValue<OpPassManager>::setValue(
     const OpPassManager &newValue) {
-- 
GitLab


From b0dd368d5741b1ad117848e33148d95406b33241 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Tue, 29 Oct 2024 18:01:23 +0000
Subject: [PATCH 479/693] [gn build] Port b510cdb895b9

---
 llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn   | 1 +
 llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn | 1 +
 2 files changed, 2 insertions(+)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index 64b03b57388c..d152aec19d1b 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -152,6 +152,7 @@ static_library("Support") {
     "TimeProfiler.cpp",
     "Timer.cpp",
     "ToolOutputFile.cpp",
+    "TrieRawHashMap.cpp",
     "Twine.cpp",
     "TypeSize.cpp",
     "Unicode.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
index c27faaaecf30..07ed3b4718af 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
@@ -94,6 +94,7 @@ unittest("ADTTests") {
     "StringSetTest.cpp",
     "StringSwitchTest.cpp",
     "TinyPtrVectorTest.cpp",
+    "TrieRawHashMapTest.cpp",
     "TwineTest.cpp",
     "TypeSwitchTest.cpp",
     "TypeTraitsTest.cpp",
-- 
GitLab


From 6563ed3162d16e7f067dda554e96d0c9d476f207 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Tue, 29 Oct 2024 14:10:25 -0400
Subject: [PATCH 480/693] [libc++][NFC] Remove trailing whitespace in the
 modulemap

---
 libcxx/include/module.modulemap | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index c3561590e06d..c3d080007319 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1229,7 +1229,7 @@ module std [system] {
     header "flat_map"
     export *
   }
-  
+
   module format {
     module buffer                             { header "__format/buffer.h" }
     module concepts                           { header "__format/concepts.h" }
-- 
GitLab


From cdacc9b5c7ec020bad24dbdcbeba96ac1d2713e5 Mon Sep 17 00:00:00 2001
From: Jerry Sun <105613447+jerryyiransun@users.noreply.github.com>
Date: Tue, 29 Oct 2024 14:10:54 -0400
Subject: [PATCH 481/693] [TableGen] [NFC] Refine TableGen code to comply with
 `clang-tidy` checks (#113318)

Code cleanups for TableGen files, changes includes function names,
variable names and unused imports.

---------

Co-authored-by: Matt Arsenault <Matthew.Arsenault@amd.com>
---
 llvm/utils/TableGen/ARMTargetDefEmitter.cpp   |  20 +-
 llvm/utils/TableGen/CallingConvEmitter.cpp    |  48 ++--
 llvm/utils/TableGen/CodeEmitterGen.cpp        | 232 +++++++--------
 llvm/utils/TableGen/CodeGenMapTable.cpp       |  52 ++--
 llvm/utils/TableGen/DAGISelEmitter.cpp        |   8 +-
 llvm/utils/TableGen/DFAPacketizerEmitter.cpp  |  30 +-
 llvm/utils/TableGen/DXILEmitter.cpp           |  42 +--
 llvm/utils/TableGen/DirectiveEmitter.cpp      | 234 +++++++--------
 llvm/utils/TableGen/DisassemblerEmitter.cpp   |   4 +-
 llvm/utils/TableGen/OptionParserEmitter.cpp   |  26 +-
 llvm/utils/TableGen/OptionRSTEmitter.cpp      |   4 +-
 llvm/utils/TableGen/RISCVTargetDefEmitter.cpp |   4 +-
 llvm/utils/TableGen/SubtargetEmitter.cpp      | 266 +++++++++---------
 llvm/utils/TableGen/TableGen.cpp              |  12 +-
 llvm/utils/TableGen/VTEmitter.cpp             |   4 +-
 15 files changed, 493 insertions(+), 493 deletions(-)

diff --git a/llvm/utils/TableGen/ARMTargetDefEmitter.cpp b/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
index 6b8ebf96cdf3..792d04713946 100644
--- a/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
+++ b/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
@@ -25,19 +25,19 @@
 using namespace llvm;
 
 /// Collect the full set of implied features for a SubtargetFeature.
-static void CollectImpliedFeatures(std::set<const Record *> &SeenFeats,
+static void collectImpliedFeatures(std::set<const Record *> &SeenFeats,
                                    const Record *Rec) {
   assert(Rec->isSubClassOf("SubtargetFeature") &&
          "Rec is not a SubtargetFeature");
 
   SeenFeats.insert(Rec);
   for (const Record *Implied : Rec->getValueAsListOfDefs("Implies"))
-    CollectImpliedFeatures(SeenFeats, Implied);
+    collectImpliedFeatures(SeenFeats, Implied);
 }
 
-static void CheckFeatureTree(const Record *Root) {
+static void checkFeatureTree(const Record *Root) {
   std::set<const Record *> SeenFeats;
-  CollectImpliedFeatures(SeenFeats, Root);
+  collectImpliedFeatures(SeenFeats, Root);
 
   // Check that each of the mandatory (implied) features which is an
   // ExtensionWithMArch is also enabled by default.
@@ -53,12 +53,12 @@ static void CheckFeatureTree(const Record *Root) {
   }
 }
 
-static void EmitARMTargetDef(const RecordKeeper &RK, raw_ostream &OS) {
+static void emitARMTargetDef(const RecordKeeper &RK, raw_ostream &OS) {
   OS << "// Autogenerated by ARMTargetDefEmitter.cpp\n\n";
 
   // Look through all SubtargetFeature defs with the given FieldName, and
   // collect the set of all Values that that FieldName is set to.
-  auto gatherSubtargetFeatureFieldValues = [&RK](StringRef FieldName) {
+  auto GatherSubtargetFeatureFieldValues = [&RK](StringRef FieldName) {
     llvm::StringSet<> Set;
     for (const Record *Rec : RK.getAllDerivedDefinitions("SubtargetFeature")) {
       if (Rec->getValueAsString("FieldName") == FieldName) {
@@ -88,7 +88,7 @@ static void EmitARMTargetDef(const RecordKeeper &RK, raw_ostream &OS) {
      << "#define ARM_PROCESSOR_FAMILY(ENUM)\n"
      << "#endif\n\n";
   const StringSet<> ARMProcFamilyVals =
-      gatherSubtargetFeatureFieldValues("ARMProcFamily");
+      GatherSubtargetFeatureFieldValues("ARMProcFamily");
   for (const StringRef &Family : ARMProcFamilyVals.keys())
     OS << "ARM_PROCESSOR_FAMILY(" << Family << ")\n";
   OS << "\n#undef ARM_PROCESSOR_FAMILY\n\n";
@@ -97,7 +97,7 @@ static void EmitARMTargetDef(const RecordKeeper &RK, raw_ostream &OS) {
      << "#define ARM_ARCHITECTURE(ENUM)\n"
      << "#endif\n\n";
   // This should correspond to instances of the Architecture tablegen class.
-  const StringSet<> ARMArchVals = gatherSubtargetFeatureFieldValues("ARMArch");
+  const StringSet<> ARMArchVals = GatherSubtargetFeatureFieldValues("ARMArch");
   for (const StringRef &Arch : ARMArchVals.keys())
     OS << "ARM_ARCHITECTURE(" << Arch << ")\n";
   OS << "\n#undef ARM_ARCHITECTURE\n\n";
@@ -315,7 +315,7 @@ static void EmitARMTargetDef(const RecordKeeper &RK, raw_ostream &OS) {
     auto Profile = Arch->getValueAsString("Profile");
     auto ArchInfo = ArchInfoName(Major, Minor, Profile);
 
-    CheckFeatureTree(Arch);
+    checkFeatureTree(Arch);
 
     OS << "  {\n"
        << "    \"" << Name << "\",\n"
@@ -343,5 +343,5 @@ static void EmitARMTargetDef(const RecordKeeper &RK, raw_ostream &OS) {
 }
 
 static TableGen::Emitter::Opt
-    X("gen-arm-target-def", EmitARMTargetDef,
+    X("gen-arm-target-def", emitARMTargetDef,
       "Generate the ARM or AArch64 Architecture information header.");
diff --git a/llvm/utils/TableGen/CallingConvEmitter.cpp b/llvm/utils/TableGen/CallingConvEmitter.cpp
index c8f263e15d96..de20303a5bfd 100644
--- a/llvm/utils/TableGen/CallingConvEmitter.cpp
+++ b/llvm/utils/TableGen/CallingConvEmitter.cpp
@@ -35,12 +35,12 @@ class CallingConvEmitter {
 public:
   explicit CallingConvEmitter(const RecordKeeper &R) : Records(R) {}
 
-  void run(raw_ostream &o);
+  void run(raw_ostream &O);
 
 private:
-  void EmitCallingConv(const Record *CC, raw_ostream &O);
-  void EmitAction(const Record *Action, indent Indent, raw_ostream &O);
-  void EmitArgRegisterLists(raw_ostream &O);
+  void emitCallingConv(const Record *CC, raw_ostream &O);
+  void emitAction(const Record *Action, indent Indent, raw_ostream &O);
+  void emitArgRegisterLists(raw_ostream &O);
 };
 } // End anonymous namespace
 
@@ -75,16 +75,16 @@ void CallingConvEmitter::run(raw_ostream &O) {
   Records.getTimer().startTimer("Emit full descriptions");
   for (const Record *CC : CCs) {
     if (!CC->getValueAsBit("Custom")) {
-      EmitCallingConv(CC, O);
+      emitCallingConv(CC, O);
     }
   }
 
-  EmitArgRegisterLists(O);
+  emitArgRegisterLists(O);
 
   O << "\n#endif // CC_REGISTER_LIST\n";
 }
 
-void CallingConvEmitter::EmitCallingConv(const Record *CC, raw_ostream &O) {
+void CallingConvEmitter::emitCallingConv(const Record *CC, raw_ostream &O) {
   const ListInit *CCActions = CC->getValueAsListInit("Actions");
   Counter = 0;
 
@@ -107,8 +107,8 @@ void CallingConvEmitter::EmitCallingConv(const Record *CC, raw_ostream &O) {
     << std::string(Pad, ' ') << "MVT LocVT, CCValAssign::LocInfo LocInfo,\n"
     << std::string(Pad, ' ') << "ISD::ArgFlagsTy ArgFlags, CCState &State) {\n";
   // Emit all of the actions, in order.
-  for (unsigned i = 0, e = CCActions->size(); i != e; ++i) {
-    const Record *Action = CCActions->getElementAsRecord(i);
+  for (unsigned I = 0, E = CCActions->size(); I != E; ++I) {
+    const Record *Action = CCActions->getElementAsRecord(I);
     SwiftAction =
         llvm::any_of(Action->getSuperClasses(),
                      [](const std::pair<const Record *, SMRange> &Class) {
@@ -117,23 +117,23 @@ void CallingConvEmitter::EmitCallingConv(const Record *CC, raw_ostream &O) {
                      });
 
     O << "\n";
-    EmitAction(Action, indent(2), O);
+    emitAction(Action, indent(2), O);
   }
 
   O << "\n  return true; // CC didn't match.\n";
   O << "}\n";
 }
 
-void CallingConvEmitter::EmitAction(const Record *Action, indent Indent,
+void CallingConvEmitter::emitAction(const Record *Action, indent Indent,
                                     raw_ostream &O) {
   if (Action->isSubClassOf("CCPredicateAction")) {
     O << Indent << "if (";
 
     if (Action->isSubClassOf("CCIfType")) {
       const ListInit *VTs = Action->getValueAsListInit("VTs");
-      for (unsigned i = 0, e = VTs->size(); i != e; ++i) {
-        const Record *VT = VTs->getElementAsRecord(i);
-        if (i != 0)
+      for (unsigned I = 0, E = VTs->size(); I != E; ++I) {
+        const Record *VT = VTs->getElementAsRecord(I);
+        if (I != 0)
           O << " ||\n    " << Indent;
         O << "LocVT == " << getEnumName(getValueType(VT));
       }
@@ -146,7 +146,7 @@ void CallingConvEmitter::EmitAction(const Record *Action, indent Indent,
     }
 
     O << ") {\n";
-    EmitAction(Action->getValueAsDef("SubAction"), Indent + 2, O);
+    emitAction(Action->getValueAsDef("SubAction"), Indent + 2, O);
     O << Indent << "}\n";
   } else {
     if (Action->isSubClassOf("CCDelegateTo")) {
@@ -171,8 +171,8 @@ void CallingConvEmitter::EmitAction(const Record *Action, indent Indent,
           << "[] = {\n";
         O << Indent << "  ";
         ListSeparator LS;
-        for (unsigned i = 0, e = RegList->size(); i != e; ++i) {
-          std::string Name = getQualifiedName(RegList->getElementAsRecord(i));
+        for (unsigned I = 0, E = RegList->size(); I != E; ++I) {
+          std::string Name = getQualifiedName(RegList->getElementAsRecord(I));
           if (SwiftAction)
             AssignedSwiftRegsMap[CurrentAction].insert(Name);
           else
@@ -230,16 +230,16 @@ void CallingConvEmitter::EmitAction(const Record *Action, indent Indent,
           << "[] = {\n";
         O << Indent << "  ";
         ListSeparator LS;
-        for (unsigned i = 0, e = RegList->size(); i != e; ++i)
-          O << LS << getQualifiedName(RegList->getElementAsRecord(i));
+        for (unsigned I = 0, E = RegList->size(); I != E; ++I)
+          O << LS << getQualifiedName(RegList->getElementAsRecord(I));
         O << "\n" << Indent << "};\n";
 
         O << Indent << "static const MCPhysReg RegList" << ShadowRegListNumber
           << "[] = {\n";
         O << Indent << "  ";
         ListSeparator LSS;
-        for (unsigned i = 0, e = ShadowRegList->size(); i != e; ++i)
-          O << LSS << getQualifiedName(ShadowRegList->getElementAsRecord(i));
+        for (unsigned I = 0, E = ShadowRegList->size(); I != E; ++I)
+          O << LSS << getQualifiedName(ShadowRegList->getElementAsRecord(I));
         O << "\n" << Indent << "};\n";
 
         O << Indent << "if (MCRegister Reg = State.AllocateReg(RegList"
@@ -287,8 +287,8 @@ void CallingConvEmitter::EmitAction(const Record *Action, indent Indent,
         << ShadowRegListNumber << "[] = {\n";
       O << Indent << "  ";
       ListSeparator LS;
-      for (unsigned i = 0, e = ShadowRegList->size(); i != e; ++i)
-        O << LS << getQualifiedName(ShadowRegList->getElementAsRecord(i));
+      for (unsigned I = 0, E = ShadowRegList->size(); I != E; ++I)
+        O << LS << getQualifiedName(ShadowRegList->getElementAsRecord(I));
       O << "\n" << Indent << "};\n";
 
       O << Indent << "int64_t Offset" << ++Counter << " = State.AllocateStack("
@@ -357,7 +357,7 @@ void CallingConvEmitter::EmitAction(const Record *Action, indent Indent,
   }
 }
 
-void CallingConvEmitter::EmitArgRegisterLists(raw_ostream &O) {
+void CallingConvEmitter::emitArgRegisterLists(raw_ostream &O) {
   // Transitively merge all delegated CCs into AssignedRegsMap.
   using EntryTy = std::pair<std::string, std::set<std::string>>;
   bool Redo;
diff --git a/llvm/utils/TableGen/CodeEmitterGen.cpp b/llvm/utils/TableGen/CodeEmitterGen.cpp
index be822c481528..407ee81b7e0b 100644
--- a/llvm/utils/TableGen/CodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/CodeEmitterGen.cpp
@@ -52,10 +52,10 @@ class CodeEmitterGen {
 public:
   CodeEmitterGen(const RecordKeeper &R) : Records(R) {}
 
-  void run(raw_ostream &o);
+  void run(raw_ostream &O);
 
 private:
-  int getVariableBit(const std::string &VarName, const BitsInit *BI, int bit);
+  int getVariableBit(const std::string &VarName, const BitsInit *BI, int Bit);
   std::pair<std::string, std::string>
   getInstructionCases(const Record *R, const CodeGenTarget &Target);
   void addInstructionCasesForEncoding(const Record *R,
@@ -69,10 +69,10 @@ private:
                                const CodeGenTarget &Target);
 
   void emitInstructionBaseValues(
-      raw_ostream &o, ArrayRef<const CodeGenInstruction *> NumberedInstructions,
+      raw_ostream &O, ArrayRef<const CodeGenInstruction *> NumberedInstructions,
       const CodeGenTarget &Target, unsigned HwMode = DefaultMode);
   void
-  emitCaseMap(raw_ostream &o,
+  emitCaseMap(raw_ostream &O,
               const std::map<std::string, std::vector<std::string>> &CaseMap);
   unsigned BitWidth = 0u;
   bool UseAPInt = false;
@@ -81,12 +81,12 @@ private:
 // If the VarBitInit at position 'bit' matches the specified variable then
 // return the variable bit position.  Otherwise return -1.
 int CodeEmitterGen::getVariableBit(const std::string &VarName,
-                                   const BitsInit *BI, int bit) {
-  if (const VarBitInit *VBI = dyn_cast<VarBitInit>(BI->getBit(bit))) {
+                                   const BitsInit *BI, int Bit) {
+  if (const VarBitInit *VBI = dyn_cast<VarBitInit>(BI->getBit(Bit))) {
     if (const VarInit *VI = dyn_cast<VarInit>(VBI->getBitVar()))
       if (VI->getName() == VarName)
         return VBI->getBitNum();
-  } else if (const VarInit *VI = dyn_cast<VarInit>(BI->getBit(bit))) {
+  } else if (const VarInit *VI = dyn_cast<VarInit>(BI->getBit(Bit))) {
     if (VI->getName() == VarName)
       return 0;
   }
@@ -104,19 +104,19 @@ bool CodeEmitterGen::addCodeToMergeInOperand(const Record *R,
   CodeGenInstruction &CGI = Target.getInstruction(R);
 
   // Determine if VarName actually contributes to the Inst encoding.
-  int bit = BI->getNumBits() - 1;
+  int Bit = BI->getNumBits() - 1;
 
   // Scan for a bit that this contributed to.
-  for (; bit >= 0;) {
-    if (getVariableBit(VarName, BI, bit) != -1)
+  for (; Bit >= 0;) {
+    if (getVariableBit(VarName, BI, Bit) != -1)
       break;
 
-    --bit;
+    --Bit;
   }
 
   // If we found no bits, ignore this value, otherwise emit the call to get the
   // operand encoding.
-  if (bit < 0)
+  if (Bit < 0)
     return true;
 
   // If the operand matches by name, reference according to that
@@ -175,97 +175,97 @@ bool CodeEmitterGen::addCodeToMergeInOperand(const Record *R,
   // Precalculate the number of lits this variable contributes to in the
   // operand. If there is a single lit (consecutive range of bits) we can use a
   // destructive sequence on APInt that reduces memory allocations.
-  int numOperandLits = 0;
-  for (int tmpBit = bit; tmpBit >= 0;) {
-    int varBit = getVariableBit(VarName, BI, tmpBit);
+  int NumOperandLits = 0;
+  for (int TmpBit = Bit; TmpBit >= 0;) {
+    int VarBit = getVariableBit(VarName, BI, TmpBit);
 
     // If this bit isn't from a variable, skip it.
-    if (varBit == -1) {
-      --tmpBit;
+    if (VarBit == -1) {
+      --TmpBit;
       continue;
     }
 
     // Figure out the consecutive range of bits covered by this operand, in
     // order to generate better encoding code.
-    int beginVarBit = varBit;
+    int BeginVarBit = VarBit;
     int N = 1;
-    for (--tmpBit; tmpBit >= 0;) {
-      varBit = getVariableBit(VarName, BI, tmpBit);
-      if (varBit == -1 || varBit != (beginVarBit - N))
+    for (--TmpBit; TmpBit >= 0;) {
+      VarBit = getVariableBit(VarName, BI, TmpBit);
+      if (VarBit == -1 || VarBit != (BeginVarBit - N))
         break;
       ++N;
-      --tmpBit;
+      --TmpBit;
     }
-    ++numOperandLits;
+    ++NumOperandLits;
   }
 
   unsigned BitOffset = -1;
-  for (; bit >= 0;) {
-    int varBit = getVariableBit(VarName, BI, bit);
+  for (; Bit >= 0;) {
+    int VarBit = getVariableBit(VarName, BI, Bit);
 
     // If this bit isn't from a variable, skip it.
-    if (varBit == -1) {
-      --bit;
+    if (VarBit == -1) {
+      --Bit;
       continue;
     }
 
     // Figure out the consecutive range of bits covered by this operand, in
     // order to generate better encoding code.
-    int beginInstBit = bit;
-    int beginVarBit = varBit;
+    int BeginInstBit = Bit;
+    int BeginVarBit = VarBit;
     int N = 1;
-    for (--bit; bit >= 0;) {
-      varBit = getVariableBit(VarName, BI, bit);
-      if (varBit == -1 || varBit != (beginVarBit - N))
+    for (--Bit; Bit >= 0;) {
+      VarBit = getVariableBit(VarName, BI, Bit);
+      if (VarBit == -1 || VarBit != (BeginVarBit - N))
         break;
       ++N;
-      --bit;
+      --Bit;
     }
 
-    std::string maskStr;
-    int opShift;
+    std::string MaskStr;
+    int OpShift;
 
-    unsigned loBit = beginVarBit - N + 1;
-    unsigned hiBit = loBit + N;
-    unsigned loInstBit = beginInstBit - N + 1;
-    BitOffset = loInstBit;
+    unsigned LoBit = BeginVarBit - N + 1;
+    unsigned HiBit = LoBit + N;
+    unsigned LoInstBit = BeginInstBit - N + 1;
+    BitOffset = LoInstBit;
     if (UseAPInt) {
-      std::string extractStr;
+      std::string ExtractStr;
       if (N >= 64) {
-        extractStr = "op.extractBits(" + itostr(hiBit - loBit) + ", " +
-                     itostr(loBit) + ")";
-        Case += "      Value.insertBits(" + extractStr + ", " +
-                itostr(loInstBit) + ");\n";
+        ExtractStr = "op.extractBits(" + itostr(HiBit - LoBit) + ", " +
+                     itostr(LoBit) + ")";
+        Case += "      Value.insertBits(" + ExtractStr + ", " +
+                itostr(LoInstBit) + ");\n";
       } else {
-        extractStr = "op.extractBitsAsZExtValue(" + itostr(hiBit - loBit) +
-                     ", " + itostr(loBit) + ")";
-        Case += "      Value.insertBits(" + extractStr + ", " +
-                itostr(loInstBit) + ", " + itostr(hiBit - loBit) + ");\n";
+        ExtractStr = "op.extractBitsAsZExtValue(" + itostr(HiBit - LoBit) +
+                     ", " + itostr(LoBit) + ")";
+        Case += "      Value.insertBits(" + ExtractStr + ", " +
+                itostr(LoInstBit) + ", " + itostr(HiBit - LoBit) + ");\n";
       }
     } else {
-      uint64_t opMask = ~(uint64_t)0 >> (64 - N);
-      opShift = beginVarBit - N + 1;
-      opMask <<= opShift;
-      maskStr = "UINT64_C(" + utostr(opMask) + ")";
-      opShift = beginInstBit - beginVarBit;
-
-      if (numOperandLits == 1) {
-        Case += "      op &= " + maskStr + ";\n";
-        if (opShift > 0) {
-          Case += "      op <<= " + itostr(opShift) + ";\n";
-        } else if (opShift < 0) {
-          Case += "      op >>= " + itostr(-opShift) + ";\n";
+      uint64_t OpMask = ~(uint64_t)0 >> (64 - N);
+      OpShift = BeginVarBit - N + 1;
+      OpMask <<= OpShift;
+      MaskStr = "UINT64_C(" + utostr(OpMask) + ")";
+      OpShift = BeginInstBit - BeginVarBit;
+
+      if (NumOperandLits == 1) {
+        Case += "      op &= " + MaskStr + ";\n";
+        if (OpShift > 0) {
+          Case += "      op <<= " + itostr(OpShift) + ";\n";
+        } else if (OpShift < 0) {
+          Case += "      op >>= " + itostr(-OpShift) + ";\n";
         }
         Case += "      Value |= op;\n";
       } else {
-        if (opShift > 0) {
-          Case += "      Value |= (op & " + maskStr + ") << " +
-                  itostr(opShift) + ";\n";
-        } else if (opShift < 0) {
-          Case += "      Value |= (op & " + maskStr + ") >> " +
-                  itostr(-opShift) + ";\n";
+        if (OpShift > 0) {
+          Case += "      Value |= (op & " + MaskStr + ") << " +
+                  itostr(OpShift) + ";\n";
+        } else if (OpShift < 0) {
+          Case += "      Value |= (op & " + MaskStr + ") >> " +
+                  itostr(-OpShift) + ";\n";
         } else {
-          Case += "      Value |= (op & " + maskStr + ");\n";
+          Case += "      Value |= (op & " + MaskStr + ");\n";
         }
       }
     }
@@ -285,7 +285,7 @@ CodeEmitterGen::getInstructionCases(const Record *R,
                                     const CodeGenTarget &Target) {
   std::string Case, BitOffsetCase;
 
-  auto append = [&](const std::string &S) {
+  auto Append = [&](const std::string &S) {
     Case += S;
     BitOffsetCase += S;
   };
@@ -298,7 +298,7 @@ CodeEmitterGen::getInstructionCases(const Record *R,
       // Invoke the interface to obtain the HwMode ID controlling the
       // EncodingInfo for the current subtarget. This interface will
       // mask off irrelevant HwMode IDs.
-      append("      unsigned HwMode = "
+      Append("      unsigned HwMode = "
              "STI.getHwMode(MCSubtargetInfo::HwMode_EncodingInfo);\n");
       Case += "      switch (HwMode) {\n";
       Case += "      default: llvm_unreachable(\"Unknown hardware mode!\"); "
@@ -328,16 +328,16 @@ CodeEmitterGen::getInstructionCases(const Record *R,
         Case += "      Value = InstBitsByHw[opcode];\n";
       }
 
-      append("      switch (HwMode) {\n");
-      append("      default: llvm_unreachable(\"Unhandled HwMode\");\n");
+      Append("      switch (HwMode) {\n");
+      Append("      default: llvm_unreachable(\"Unhandled HwMode\");\n");
       for (auto &[ModeId, Encoding] : EBM) {
-        append("      case " + itostr(ModeId) + ": {\n");
+        Append("      case " + itostr(ModeId) + ": {\n");
         addInstructionCasesForEncoding(R, Encoding, Target, Case,
                                        BitOffsetCase);
-        append("      break;\n");
-        append("      }\n");
+        Append("      break;\n");
+        Append("      }\n");
       }
-      append("      }\n");
+      Append("      }\n");
       return std::pair(std::move(Case), std::move(BitOffsetCase));
     }
   }
@@ -397,13 +397,13 @@ static void emitInstBits(raw_ostream &OS, const APInt &Bits) {
 }
 
 void CodeEmitterGen::emitInstructionBaseValues(
-    raw_ostream &o, ArrayRef<const CodeGenInstruction *> NumberedInstructions,
+    raw_ostream &O, ArrayRef<const CodeGenInstruction *> NumberedInstructions,
     const CodeGenTarget &Target, unsigned HwMode) {
   const CodeGenHwModes &HWM = Target.getHwModes();
   if (HwMode == DefaultMode)
-    o << "  static const uint64_t InstBits[] = {\n";
+    O << "  static const uint64_t InstBits[] = {\n";
   else
-    o << "  static const uint64_t InstBits_"
+    O << "  static const uint64_t InstBits_"
       << HWM.getModeName(HwMode, /*IncludeDefault=*/true) << "[] = {\n";
 
   for (const CodeGenInstruction *CGI : NumberedInstructions) {
@@ -411,9 +411,9 @@ void CodeEmitterGen::emitInstructionBaseValues(
 
     if (R->getValueAsString("Namespace") == "TargetOpcode" ||
         R->getValueAsBit("isPseudo")) {
-      o << "    ";
-      emitInstBits(o, APInt(BitWidth, 0));
-      o << ",\n";
+      O << "    ";
+      emitInstBits(O, APInt(BitWidth, 0));
+      O << ",\n";
       continue;
     }
 
@@ -427,9 +427,9 @@ void CodeEmitterGen::emitInstructionBaseValues(
           // If the HwMode does not match, then Encoding '0'
           // should be generated.
           APInt Value(BitWidth, 0);
-          o << "    ";
-          emitInstBits(o, Value);
-          o << "," << '\t' << "// " << R->getName() << "\n";
+          O << "    ";
+          emitInstBits(O, Value);
+          O << "," << '\t' << "// " << R->getName() << "\n";
           continue;
         }
       }
@@ -438,37 +438,37 @@ void CodeEmitterGen::emitInstructionBaseValues(
 
     // Start by filling in fixed values.
     APInt Value(BitWidth, 0);
-    for (unsigned i = 0, e = BI->getNumBits(); i != e; ++i) {
-      if (const auto *B = dyn_cast<BitInit>(BI->getBit(i)); B && B->getValue())
-        Value.setBit(i);
+    for (unsigned I = 0, E = BI->getNumBits(); I != E; ++I) {
+      if (const auto *B = dyn_cast<BitInit>(BI->getBit(I)); B && B->getValue())
+        Value.setBit(I);
     }
-    o << "    ";
-    emitInstBits(o, Value);
-    o << "," << '\t' << "// " << R->getName() << "\n";
+    O << "    ";
+    emitInstBits(O, Value);
+    O << "," << '\t' << "// " << R->getName() << "\n";
   }
-  o << "    UINT64_C(0)\n  };\n";
+  O << "    UINT64_C(0)\n  };\n";
 }
 
 void CodeEmitterGen::emitCaseMap(
-    raw_ostream &o,
+    raw_ostream &O,
     const std::map<std::string, std::vector<std::string>> &CaseMap) {
   for (const auto &[Case, InstList] : CaseMap) {
     bool First = true;
     for (const auto &Inst : InstList) {
       if (!First)
-        o << "\n";
-      o << "    case " << Inst << ":";
+        O << "\n";
+      O << "    case " << Inst << ":";
       First = false;
     }
-    o << " {\n";
-    o << Case;
-    o << "      break;\n"
+    O << " {\n";
+    O << Case;
+    O << "      break;\n"
       << "    }\n";
   }
 }
 
-void CodeEmitterGen::run(raw_ostream &o) {
-  emitSourceFileHeader("Machine Code Emitter", o);
+void CodeEmitterGen::run(raw_ostream &O) {
+  emitSourceFileHeader("Machine Code Emitter", O);
 
   CodeGenTarget Target(Records);
 
@@ -479,7 +479,7 @@ void CodeEmitterGen::run(raw_ostream &o) {
       Target.getInstructionsByEnumValue();
 
   if (Target.hasVariableLengthEncodings()) {
-    emitVarLenCodeEmitter(Records, o);
+    emitVarLenCodeEmitter(Records, O);
   } else {
     const CodeGenHwModes &HWM = Target.getHwModes();
     // The set of HwModes used by instruction encodings.
@@ -509,31 +509,31 @@ void CodeEmitterGen::run(raw_ostream &o) {
 
     // Emit function declaration
     if (UseAPInt) {
-      o << "void " << Target.getName()
+      O << "void " << Target.getName()
         << "MCCodeEmitter::getBinaryCodeForInstr(const MCInst &MI,\n"
         << "    SmallVectorImpl<MCFixup> &Fixups,\n"
         << "    APInt &Inst,\n"
         << "    APInt &Scratch,\n"
         << "    const MCSubtargetInfo &STI) const {\n";
     } else {
-      o << "uint64_t " << Target.getName();
-      o << "MCCodeEmitter::getBinaryCodeForInstr(const MCInst &MI,\n"
+      O << "uint64_t " << Target.getName();
+      O << "MCCodeEmitter::getBinaryCodeForInstr(const MCInst &MI,\n"
         << "    SmallVectorImpl<MCFixup> &Fixups,\n"
         << "    const MCSubtargetInfo &STI) const {\n";
     }
 
     // Emit instruction base values
-    emitInstructionBaseValues(o, NumberedInstructions, Target, DefaultMode);
+    emitInstructionBaseValues(O, NumberedInstructions, Target, DefaultMode);
     if (!HwModes.empty()) {
       // Emit table for instrs whose encodings are controlled by HwModes.
       for (unsigned HwMode : HwModes) {
         if (HwMode == DefaultMode)
           continue;
-        emitInstructionBaseValues(o, NumberedInstructions, Target, HwMode);
+        emitInstructionBaseValues(O, NumberedInstructions, Target, HwMode);
       }
 
       // This pointer will be assigned to the HwMode table later.
-      o << "  const uint64_t *InstBitsByHw;\n";
+      O << "  const uint64_t *InstBitsByHw;\n";
     }
 
     // Map to accumulate all the cases.
@@ -557,7 +557,7 @@ void CodeEmitterGen::run(raw_ostream &o) {
     // Emit initial function code
     if (UseAPInt) {
       int NumWords = APInt::getNumWords(BitWidth);
-      o << "  const unsigned opcode = MI.getOpcode();\n"
+      O << "  const unsigned opcode = MI.getOpcode();\n"
         << "  if (Scratch.getBitWidth() != " << BitWidth << ")\n"
         << "    Scratch = Scratch.zext(" << BitWidth << ");\n"
         << "  Inst = APInt(" << BitWidth << ", ArrayRef(InstBits + opcode * "
@@ -566,7 +566,7 @@ void CodeEmitterGen::run(raw_ostream &o) {
         << "  APInt &op = Scratch;\n"
         << "  switch (opcode) {\n";
     } else {
-      o << "  const unsigned opcode = MI.getOpcode();\n"
+      O << "  const unsigned opcode = MI.getOpcode();\n"
         << "  uint64_t Value = InstBits[opcode];\n"
         << "  uint64_t op = 0;\n"
         << "  (void)op;  // suppress warning\n"
@@ -574,30 +574,30 @@ void CodeEmitterGen::run(raw_ostream &o) {
     }
 
     // Emit each case statement
-    emitCaseMap(o, CaseMap);
+    emitCaseMap(O, CaseMap);
 
     // Default case: unhandled opcode
-    o << "  default:\n"
+    O << "  default:\n"
       << "    std::string msg;\n"
       << "    raw_string_ostream Msg(msg);\n"
       << "    Msg << \"Not supported instr: \" << MI;\n"
       << "    report_fatal_error(Msg.str().c_str());\n"
       << "  }\n";
     if (UseAPInt)
-      o << "  Inst = Value;\n";
+      O << "  Inst = Value;\n";
     else
-      o << "  return Value;\n";
-    o << "}\n\n";
+      O << "  return Value;\n";
+    O << "}\n\n";
 
-    o << "#ifdef GET_OPERAND_BIT_OFFSET\n"
+    O << "#ifdef GET_OPERAND_BIT_OFFSET\n"
       << "#undef GET_OPERAND_BIT_OFFSET\n\n"
       << "uint32_t " << Target.getName()
       << "MCCodeEmitter::getOperandBitOffset(const MCInst &MI,\n"
       << "    unsigned OpNum,\n"
       << "    const MCSubtargetInfo &STI) const {\n"
       << "  switch (MI.getOpcode()) {\n";
-    emitCaseMap(o, BitOffsetCaseMap);
-    o << "  }\n"
+    emitCaseMap(O, BitOffsetCaseMap);
+    O << "  }\n"
       << "  std::string msg;\n"
       << "  raw_string_ostream Msg(msg);\n"
       << "  Msg << \"Not supported instr[opcode]: \" << MI << \"[\" << OpNum "
diff --git a/llvm/utils/TableGen/CodeGenMapTable.cpp b/llvm/utils/TableGen/CodeGenMapTable.cpp
index 7876db6f33df..8d22c0013dda 100644
--- a/llvm/utils/TableGen/CodeGenMapTable.cpp
+++ b/llvm/utils/TableGen/CodeGenMapTable.cpp
@@ -258,12 +258,12 @@ bool MapTableEmitter::isKeyColInstr(const Record *CurInstr) {
 
   // Check if the instruction is a KeyCol instruction.
   bool MatchFound = true;
-  for (unsigned j = 0, endCF = ColFields->size(); (j < endCF) && MatchFound;
-       j++) {
+  for (unsigned J = 0, EndCf = ColFields->size(); (J < EndCf) && MatchFound;
+       J++) {
     const RecordVal *ColFieldName =
-        CurInstr->getValue(ColFields->getElement(j));
+        CurInstr->getValue(ColFields->getElement(J));
     std::string CurInstrVal = ColFieldName->getValue()->getAsUnquotedString();
-    std::string KeyColValue = KeyCol->getElement(j)->getAsUnquotedString();
+    std::string KeyColValue = KeyCol->getElement(J)->getAsUnquotedString();
     MatchFound = CurInstrVal == KeyColValue;
   }
   return MatchFound;
@@ -318,12 +318,12 @@ const Record *MapTableEmitter::getInstrForColumn(const Record *KeyInstr,
 
   for (const Record *CurInstr : RelatedInstrVec) {
     bool MatchFound = true;
-    for (unsigned j = 0, endCF = ColFields->size(); (j < endCF) && MatchFound;
-         j++) {
-      const Init *ColFieldJ = ColFields->getElement(j);
+    for (unsigned J = 0, EndCf = ColFields->size(); (J < EndCf) && MatchFound;
+         J++) {
+      const Init *ColFieldJ = ColFields->getElement(J);
       const Init *CurInstrInit = CurInstr->getValue(ColFieldJ)->getValue();
       std::string CurInstrVal = CurInstrInit->getAsUnquotedString();
-      const Init *ColFieldJVallue = CurValueCol->getElement(j);
+      const Init *ColFieldJVallue = CurValueCol->getElement(J);
       MatchFound = CurInstrVal == ColFieldJVallue->getAsUnquotedString();
     }
 
@@ -368,19 +368,19 @@ unsigned MapTableEmitter::emitBinSearchTable(raw_ostream &OS) {
   // Number of columns in the table are NumCol+1 because key instructions are
   // emitted as first column.
   OS << "Table[][" << NumCol + 1 << "] = {\n";
-  for (unsigned i = 0; i < TotalNumInstr; i++) {
-    const Record *CurInstr = NumberedInstructions[i]->TheDef;
+  for (unsigned I = 0; I < TotalNumInstr; I++) {
+    const Record *CurInstr = NumberedInstructions[I]->TheDef;
     ArrayRef<const Record *> ColInstrs = MapTable[CurInstr];
     std::string OutStr;
     unsigned RelExists = 0;
     if (!ColInstrs.empty()) {
-      for (unsigned j = 0; j < NumCol; j++) {
-        if (ColInstrs[j] != nullptr) {
+      for (unsigned J = 0; J < NumCol; J++) {
+        if (ColInstrs[J] != nullptr) {
           RelExists = 1;
           OutStr += ", ";
           OutStr += Namespace;
           OutStr += "::";
-          OutStr += ColInstrs[j]->getName();
+          OutStr += ColInstrs[J]->getName();
         } else {
           OutStr += ", (uint16_t)-1U";
         }
@@ -441,20 +441,20 @@ void MapTableEmitter::emitMapFuncBody(raw_ostream &OS, unsigned TableSize) {
   emitBinSearch(OS, TableSize);
 
   if (ValueCols.size() > 1) {
-    for (unsigned i = 0, e = ValueCols.size(); i < e; i++) {
-      const ListInit *ColumnI = ValueCols[i];
+    for (unsigned I = 0, E = ValueCols.size(); I < E; I++) {
+      const ListInit *ColumnI = ValueCols[I];
       OS << "  if (";
-      for (unsigned j = 0, ColSize = ColumnI->size(); j < ColSize; ++j) {
-        std::string ColName = ColFields->getElement(j)->getAsUnquotedString();
+      for (unsigned J = 0, ColSize = ColumnI->size(); J < ColSize; ++J) {
+        std::string ColName = ColFields->getElement(J)->getAsUnquotedString();
         OS << "in" << ColName;
         OS << " == ";
-        OS << ColName << "_" << ColumnI->getElement(j)->getAsUnquotedString();
-        if (j < ColumnI->size() - 1)
+        OS << ColName << "_" << ColumnI->getElement(J)->getAsUnquotedString();
+        if (J < ColumnI->size() - 1)
           OS << " && ";
       }
       OS << ")\n";
       OS << "    return " << InstrMapDesc.getName();
-      OS << "Table[mid][" << i + 1 << "];\n";
+      OS << "Table[mid][" << I + 1 << "];\n";
     }
     OS << "  return -1;";
   } else
@@ -509,8 +509,8 @@ static void emitEnums(raw_ostream &OS, const RecordKeeper &Records) {
     std::vector<const ListInit *> ValueCols;
     unsigned ListSize = List->size();
 
-    for (unsigned j = 0; j < ListSize; j++) {
-      const auto *ListJ = cast<ListInit>(List->getElement(j));
+    for (unsigned J = 0; J < ListSize; J++) {
+      const auto *ListJ = cast<ListInit>(List->getElement(J));
 
       if (ListJ->size() != ColFields->size())
         PrintFatalError("Record `" + CurMap->getName() +
@@ -520,10 +520,10 @@ static void emitEnums(raw_ostream &OS, const RecordKeeper &Records) {
       ValueCols.push_back(ListJ);
     }
 
-    for (unsigned j = 0, endCF = ColFields->size(); j < endCF; j++) {
-      for (unsigned k = 0; k < ListSize; k++) {
-        std::string ColName = ColFields->getElement(j)->getAsUnquotedString();
-        ColFieldValueMap[ColName].push_back((ValueCols[k])->getElement(j));
+    for (unsigned J = 0, EndCf = ColFields->size(); J < EndCf; J++) {
+      for (unsigned K = 0; K < ListSize; K++) {
+        std::string ColName = ColFields->getElement(J)->getAsUnquotedString();
+        ColFieldValueMap[ColName].push_back((ValueCols[K])->getElement(J));
       }
     }
   }
diff --git a/llvm/utils/TableGen/DAGISelEmitter.cpp b/llvm/utils/TableGen/DAGISelEmitter.cpp
index d3b653b0fba2..3d39ee148373 100644
--- a/llvm/utils/TableGen/DAGISelEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelEmitter.cpp
@@ -55,8 +55,8 @@ static unsigned getResultPatternCost(TreePatternNode &P,
     if (II.usesCustomInserter)
       Cost += 10;
   }
-  for (unsigned i = 0, e = P.getNumChildren(); i != e; ++i)
-    Cost += getResultPatternCost(P.getChild(i), CGP);
+  for (unsigned I = 0, E = P.getNumChildren(); I != E; ++I)
+    Cost += getResultPatternCost(P.getChild(I), CGP);
   return Cost;
 }
 
@@ -72,8 +72,8 @@ static unsigned getResultPatternSize(TreePatternNode &P,
   if (Op->isSubClassOf("Instruction")) {
     Cost += Op->getValueAsInt("CodeSize");
   }
-  for (unsigned i = 0, e = P.getNumChildren(); i != e; ++i)
-    Cost += getResultPatternSize(P.getChild(i), CGP);
+  for (unsigned I = 0, E = P.getNumChildren(); I != E; ++I)
+    Cost += getResultPatternSize(P.getChild(I), CGP);
   return Cost;
 }
 
diff --git a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
index 537bee55978b..a6c0d09f69ba 100644
--- a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
+++ b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
@@ -105,7 +105,7 @@ int DFAPacketizerEmitter::collectAllFuncUnits(
   for (const CodeGenProcModel *Model : ProcModels)
     ProcItinList.insert(Model->ItinsDef);
 
-  int totalFUs = 0;
+  int TotalFUs = 0;
   // Parse functional units for all the itineraries.
   for (const Record *Proc : ProcItinList) {
     std::vector<const Record *> FUs = Proc->getValueAsListOfDefs("FU");
@@ -123,10 +123,10 @@ int DFAPacketizerEmitter::collectAllFuncUnits(
       LLVM_DEBUG(dbgs() << " " << FUs[j]->getName() << ":0x"
                         << Twine::utohexstr(FuncResources));
     }
-    totalFUs += numFUs;
+    TotalFUs += numFUs;
     LLVM_DEBUG(dbgs() << "\n");
   }
-  return totalFUs;
+  return TotalFUs;
 }
 
 int DFAPacketizerEmitter::collectAllComboFuncs(
@@ -136,19 +136,19 @@ int DFAPacketizerEmitter::collectAllComboFuncs(
   LLVM_DEBUG(dbgs() << "collectAllComboFuncs");
   LLVM_DEBUG(dbgs() << " (" << ComboFuncList.size() << " sets)\n");
 
-  int numCombos = 0;
-  for (unsigned i = 0, N = ComboFuncList.size(); i < N; ++i) {
-    const Record *Func = ComboFuncList[i];
+  int NumCombos = 0;
+  for (unsigned I = 0, N = ComboFuncList.size(); I < N; ++I) {
+    const Record *Func = ComboFuncList[I];
     std::vector<const Record *> FUs = Func->getValueAsListOfDefs("CFD");
 
-    LLVM_DEBUG(dbgs() << "    CFD:" << i << " (" << FUs.size() << " combo FUs) "
+    LLVM_DEBUG(dbgs() << "    CFD:" << I << " (" << FUs.size() << " combo FUs) "
                       << Func->getName() << "\n");
 
     // Convert macros to bits for each stage.
-    for (unsigned j = 0, N = FUs.size(); j < N; ++j) {
-      assert((j < DFA_MAX_RESOURCES) &&
+    for (unsigned J = 0, N = FUs.size(); J < N; ++J) {
+      assert((J < DFA_MAX_RESOURCES) &&
              "Exceeded maximum number of DFA resources");
-      const Record *FuncData = FUs[j];
+      const Record *FuncData = FUs[J];
       const Record *ComboFunc = FuncData->getValueAsDef("TheComboFunc");
       const std::vector<const Record *> FuncList =
           FuncData->getValueAsListOfDefs("FuncList");
@@ -165,13 +165,13 @@ int DFAPacketizerEmitter::collectAllComboFuncs(
         ComboResources |= FuncResources;
       }
       ComboBitToBitsMap[ComboBit] = ComboResources;
-      numCombos++;
+      NumCombos++;
       LLVM_DEBUG(dbgs() << "          => combo bits: " << ComboFuncName << ":0x"
                         << Twine::utohexstr(ComboBit) << " = 0x"
                         << Twine::utohexstr(ComboResources) << "\n");
     }
   }
-  return numCombos;
+  return NumCombos;
 }
 
 ResourceVector
@@ -271,7 +271,7 @@ void DFAPacketizerEmitter::emitForItineraries(
 
   // Given a resource state, return all resource states by applying
   // InsnClass.
-  auto applyInsnClass = [&](const ResourceVector &InsnClass,
+  auto ApplyInsnClass = [&](const ResourceVector &InsnClass,
                             NfaStateTy State) -> std::deque<NfaStateTy> {
     std::deque<NfaStateTy> V(1, State);
     // Apply every stage in the class individually.
@@ -304,7 +304,7 @@ void DFAPacketizerEmitter::emitForItineraries(
 
   // Given a resource state, return a quick (conservative) guess as to whether
   // InsnClass can be applied. This is a filter for the more heavyweight
-  // applyInsnClass.
+  // ApplyInsnClass.
   auto canApplyInsnClass = [](const ResourceVector &InsnClass,
                               NfaStateTy State) -> bool {
     for (NfaStateTy Resources : InsnClass) {
@@ -325,7 +325,7 @@ void DFAPacketizerEmitter::emitForItineraries(
       if (!canApplyInsnClass(Resources, State))
         continue;
       unsigned ResourcesID = UniqueResources.idFor(Resources);
-      for (uint64_t NewState : applyInsnClass(Resources, State)) {
+      for (uint64_t NewState : ApplyInsnClass(Resources, State)) {
         if (SeenStates.emplace(NewState).second)
           Worklist.emplace_back(NewState);
         Emitter.addTransition(State, NewState, ResourcesID);
diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp
index 859423324463..8bebe608eece 100644
--- a/llvm/utils/TableGen/DXILEmitter.cpp
+++ b/llvm/utils/TableGen/DXILEmitter.cpp
@@ -61,7 +61,7 @@ struct DXILOperationDesc {
       ShaderStages; // shader stages to which this applies, empty for all.
   int OverloadParamIndex;             // Index of parameter with overload type.
                                       //   -1 : no overload types
-  SmallVector<StringRef, 4> counters; // counters for this inst.
+  SmallVector<StringRef, 4> Counters; // counters for this inst.
   DXILOperationDesc(const Record *);
 };
 } // end anonymous namespace
@@ -69,7 +69,7 @@ struct DXILOperationDesc {
 /// In-place sort TableGen records of class with a field
 ///    Version dxil_version
 /// in the ascending version order.
-static void AscendingSortByVersion(std::vector<const Record *> &Recs) {
+static void ascendingSortByVersion(std::vector<const Record *> &Recs) {
   sort(Recs, [](const Record *RecA, const Record *RecB) {
     unsigned RecAMaj =
         RecA->getValueAsDef("dxil_version")->getValueAsInt("Major");
@@ -125,8 +125,8 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
   // the comment before the definition of class LLVMMatchType in
   // llvm/IR/Intrinsics.td
   OverloadParamIndex = -1; // A sigil meaning none.
-  for (unsigned i = 0; i < ParamTypeRecsSize; i++) {
-    const Record *TR = ParamTypeRecs[i];
+  for (unsigned I = 0; I < ParamTypeRecsSize; I++) {
+    const Record *TR = ParamTypeRecs[I];
     // Track operation parameter indices of any overload types
     if (TR->getValueAsInt("isOverload")) {
       if (OverloadParamIndex != -1) {
@@ -137,7 +137,7 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
       // Keep the earliest parameter index we see, but if it was the return type
       // overwrite it with the first overloaded argument.
       if (OverloadParamIndex <= 0)
-        OverloadParamIndex = i;
+        OverloadParamIndex = I;
     }
     OpTypes.emplace_back(TR);
   }
@@ -146,7 +146,7 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
   std::vector<const Record *> Recs = R->getValueAsListOfDefs("overloads");
 
   // Sort records in ascending order of DXIL version
-  AscendingSortByVersion(Recs);
+  ascendingSortByVersion(Recs);
 
   for (const Record *CR : Recs) {
     OverloadRecs.push_back(CR);
@@ -161,7 +161,7 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
   }
 
   // Sort records in ascending order of DXIL version
-  AscendingSortByVersion(Recs);
+  ascendingSortByVersion(Recs);
 
   for (const Record *CR : Recs) {
     StageRecs.push_back(CR);
@@ -171,7 +171,7 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
   Recs = R->getValueAsListOfDefs("attributes");
 
   // Sort records in ascending order of DXIL version
-  AscendingSortByVersion(Recs);
+  ascendingSortByVersion(Recs);
 
   for (const Record *CR : Recs) {
     AttrRecs.push_back(CR);
@@ -286,7 +286,7 @@ static std::string getOverloadMaskString(ArrayRef<const Record *> Recs) {
   if (Recs.empty()) {
     MaskString.append("{{1, 0}, OverloadKind::UNDEFINED}}");
   } else {
-    for (auto Rec : Recs) {
+    for (const auto *Rec : Recs) {
       unsigned Major =
           Rec->getValueAsDef("dxil_version")->getValueAsInt("Major");
       unsigned Minor =
@@ -332,7 +332,7 @@ static std::string getStageMaskString(ArrayRef<const Record *> Recs) {
                     "operation must be specified");
   }
 
-  for (auto Rec : Recs) {
+  for (const auto *Rec : Recs) {
     unsigned Major = Rec->getValueAsDef("dxil_version")->getValueAsInt("Major");
     unsigned Minor = Rec->getValueAsDef("dxil_version")->getValueAsInt("Minor");
     MaskString.append(Prefix)
@@ -370,7 +370,7 @@ static std::string getAttributeMaskString(ArrayRef<const Record *> Recs) {
   std::string Prefix = "";
   MaskString.append("{");
 
-  for (auto Rec : Recs) {
+  for (const auto *Rec : Recs) {
     unsigned Major = Rec->getValueAsDef("dxil_version")->getValueAsInt("Major");
     unsigned Minor = Rec->getValueAsDef("dxil_version")->getValueAsInt("Minor");
     MaskString.append(Prefix)
@@ -576,21 +576,21 @@ static void emitDXILOperationTableDataStructs(const RecordKeeper &Records,
   size_t ShaderKindCount = ShaderKindRecs.size();
   uint64_t ShaderKindTySz = PowerOf2Ceil(ShaderKindRecs.size() + 1);
   OS << "enum ShaderKind : uint" << ShaderKindTySz << "_t {\n";
-  const std::string allStages("all_stages");
-  const std::string removed("removed");
-  int shiftVal = 1;
-  for (auto R : ShaderKindRecs) {
+  const std::string AllStages("all_stages");
+  const std::string Removed("removed");
+  int ShiftVal = 1;
+  for (const auto *R : ShaderKindRecs) {
     auto Name = R->getName();
-    if (Name.compare(removed) == 0) {
+    if (Name.compare(Removed) == 0) {
       OS << "  " << Name
          << " =  0,  // Pseudo-stage indicating op not supported in any "
             "stage\n";
-    } else if (Name.compare(allStages) == 0) {
+    } else if (Name.compare(AllStages) == 0) {
       OS << "  " << Name << " =  0x"
          << utohexstr(((1 << ShaderKindCount) - 1), false, 0)
          << ", // Pseudo-stage indicating op is supported in all stages\n";
-    } else if (Name.compare(allStages)) {
-      OS << "  " << Name << " = 1 << " << std::to_string(shiftVal++) << ",\n";
+    } else if (Name.compare(AllStages)) {
+      OS << "  " << Name << " = 1 << " << std::to_string(ShiftVal++) << ",\n";
     }
   }
   OS << "}; // enum ShaderKind\n\n";
@@ -599,7 +599,7 @@ static void emitDXILOperationTableDataStructs(const RecordKeeper &Records,
 /// Entry function call that invokes the functionality of this TableGen backend
 /// \param Records TableGen records of DXIL Operations defined in DXIL.td
 /// \param OS output stream
-static void EmitDXILOperation(const RecordKeeper &Records, raw_ostream &OS) {
+static void emitDxilOperation(const RecordKeeper &Records, raw_ostream &OS) {
   OS << "// Generated code, do not edit.\n";
   OS << "\n";
   // Get all DXIL Ops property records
@@ -631,5 +631,5 @@ static void EmitDXILOperation(const RecordKeeper &Records, raw_ostream &OS) {
   OS << "#endif\n\n";
 }
 
-static TableGen::Emitter::Opt X("gen-dxil-operation", EmitDXILOperation,
+static TableGen::Emitter::Opt X("gen-dxil-operation", emitDxilOperation,
                                 "Generate DXIL operation information");
diff --git a/llvm/utils/TableGen/DirectiveEmitter.cpp b/llvm/utils/TableGen/DirectiveEmitter.cpp
index 9dc29d8262fa..fd815f4a31da 100644
--- a/llvm/utils/TableGen/DirectiveEmitter.cpp
+++ b/llvm/utils/TableGen/DirectiveEmitter.cpp
@@ -46,7 +46,7 @@ private:
 
 // Generate enum class. Entries are emitted in the order in which they appear
 // in the `Records` vector.
-static void GenerateEnumClass(ArrayRef<const Record *> Records, raw_ostream &OS,
+static void generateEnumClass(ArrayRef<const Record *> Records, raw_ostream &OS,
                               StringRef Enum, StringRef Prefix,
                               const DirectiveLanguage &DirLang,
                               bool ExportEnums) {
@@ -79,7 +79,7 @@ static void GenerateEnumClass(ArrayRef<const Record *> Records, raw_ostream &OS,
 
 // Generate enums for values that clauses can take.
 // Also generate function declarations for get<Enum>Name(StringRef Str).
-static void GenerateEnumClauseVal(ArrayRef<const Record *> Records,
+static void generateEnumClauseVal(ArrayRef<const Record *> Records,
                                   raw_ostream &OS,
                                   const DirectiveLanguage &DirLang,
                                   std::string &EnumHelperFuncs) {
@@ -121,13 +121,13 @@ static void GenerateEnumClauseVal(ArrayRef<const Record *> Records,
   }
 }
 
-static bool HasDuplicateClauses(ArrayRef<const Record *> Clauses,
+static bool hasDuplicateClauses(ArrayRef<const Record *> Clauses,
                                 const Directive &Directive,
                                 StringSet<> &CrtClauses) {
   bool HasError = false;
   for (const VersionedClause VerClause : Clauses) {
-    const auto insRes = CrtClauses.insert(VerClause.getClause().getName());
-    if (!insRes.second) {
+    const auto InsRes = CrtClauses.insert(VerClause.getClause().getName());
+    if (!InsRes.second) {
       PrintError("Clause " + VerClause.getClause().getRecordName() +
                  " already defined on directive " + Directive.getRecordName());
       HasError = true;
@@ -140,20 +140,20 @@ static bool HasDuplicateClauses(ArrayRef<const Record *> Clauses,
 // three allowed list. Also, since required implies allowed, clauses cannot
 // appear in both the allowedClauses and requiredClauses lists.
 static bool
-HasDuplicateClausesInDirectives(ArrayRef<const Record *> Directives) {
+hasDuplicateClausesInDirectives(ArrayRef<const Record *> Directives) {
   bool HasDuplicate = false;
   for (const Directive Dir : Directives) {
     StringSet<> Clauses;
     // Check for duplicates in the three allowed lists.
-    if (HasDuplicateClauses(Dir.getAllowedClauses(), Dir, Clauses) ||
-        HasDuplicateClauses(Dir.getAllowedOnceClauses(), Dir, Clauses) ||
-        HasDuplicateClauses(Dir.getAllowedExclusiveClauses(), Dir, Clauses)) {
+    if (hasDuplicateClauses(Dir.getAllowedClauses(), Dir, Clauses) ||
+        hasDuplicateClauses(Dir.getAllowedOnceClauses(), Dir, Clauses) ||
+        hasDuplicateClauses(Dir.getAllowedExclusiveClauses(), Dir, Clauses)) {
       HasDuplicate = true;
     }
     // Check for duplicate between allowedClauses and required
     Clauses.clear();
-    if (HasDuplicateClauses(Dir.getAllowedClauses(), Dir, Clauses) ||
-        HasDuplicateClauses(Dir.getRequiredClauses(), Dir, Clauses)) {
+    if (hasDuplicateClauses(Dir.getAllowedClauses(), Dir, Clauses) ||
+        hasDuplicateClauses(Dir.getRequiredClauses(), Dir, Clauses)) {
       HasDuplicate = true;
     }
     if (HasDuplicate)
@@ -173,11 +173,11 @@ bool DirectiveLanguage::HasValidityErrors() const {
     return true;
   }
 
-  return HasDuplicateClausesInDirectives(getDirectives());
+  return hasDuplicateClausesInDirectives(getDirectives());
 }
 
 // Count the maximum number of leaf constituents per construct.
-static size_t GetMaxLeafCount(const DirectiveLanguage &DirLang) {
+static size_t getMaxLeafCount(const DirectiveLanguage &DirLang) {
   size_t MaxCount = 0;
   for (const Directive D : DirLang.getDirectives())
     MaxCount = std::max(MaxCount, D.getLeafConstructs().size());
@@ -186,7 +186,7 @@ static size_t GetMaxLeafCount(const DirectiveLanguage &DirLang) {
 
 // Generate the declaration section for the enumeration in the directive
 // language.
-static void EmitDirectivesDecl(const RecordKeeper &Records, raw_ostream &OS) {
+static void emitDirectivesDecl(const RecordKeeper &Records, raw_ostream &OS) {
   const auto DirLang = DirectiveLanguage(Records);
   if (DirLang.HasValidityErrors())
     return;
@@ -214,29 +214,29 @@ static void EmitDirectivesDecl(const RecordKeeper &Records, raw_ostream &OS) {
     OS << "\nLLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();\n";
 
   // Emit Directive associations
-  std::vector<const Record *> associations;
-  copy_if(DirLang.getAssociations(), std::back_inserter(associations),
+  std::vector<const Record *> Associations;
+  copy_if(DirLang.getAssociations(), std::back_inserter(Associations),
           // Skip the "special" value
           [](const Record *Def) { return Def->getName() != "AS_FromLeaves"; });
-  GenerateEnumClass(associations, OS, "Association",
+  generateEnumClass(Associations, OS, "Association",
                     /*Prefix=*/"", DirLang, /*ExportEnums=*/false);
 
-  GenerateEnumClass(DirLang.getCategories(), OS, "Category", /*Prefix=*/"",
+  generateEnumClass(DirLang.getCategories(), OS, "Category", /*Prefix=*/"",
                     DirLang, /*ExportEnums=*/false);
 
   // Emit Directive enumeration
-  GenerateEnumClass(DirLang.getDirectives(), OS, "Directive",
+  generateEnumClass(DirLang.getDirectives(), OS, "Directive",
                     DirLang.getDirectivePrefix(), DirLang,
                     DirLang.hasMakeEnumAvailableInNamespace());
 
   // Emit Clause enumeration
-  GenerateEnumClass(DirLang.getClauses(), OS, "Clause",
+  generateEnumClass(DirLang.getClauses(), OS, "Clause",
                     DirLang.getClausePrefix(), DirLang,
                     DirLang.hasMakeEnumAvailableInNamespace());
 
   // Emit ClauseVal enumeration
   std::string EnumHelperFuncs;
-  GenerateEnumClauseVal(DirLang.getClauses(), OS, DirLang, EnumHelperFuncs);
+  generateEnumClauseVal(DirLang.getClauses(), OS, DirLang, EnumHelperFuncs);
 
   // Generic function signatures
   OS << "\n";
@@ -259,7 +259,7 @@ static void EmitDirectivesDecl(const RecordKeeper &Records, raw_ostream &OS) {
      << "Clause C, unsigned Version);\n";
   OS << "\n";
   OS << "constexpr std::size_t getMaxLeafCount() { return "
-     << GetMaxLeafCount(DirLang) << "; }\n";
+     << getMaxLeafCount(DirLang) << "; }\n";
   OS << "LLVM_ABI Association getDirectiveAssociation(Directive D);\n";
   OS << "LLVM_ABI Category getDirectiveCategory(Directive D);\n";
   if (EnumHelperFuncs.length() > 0) {
@@ -277,7 +277,7 @@ static void EmitDirectivesDecl(const RecordKeeper &Records, raw_ostream &OS) {
 }
 
 // Generate function implementation for get<Enum>Name(StringRef Str)
-static void GenerateGetName(ArrayRef<const Record *> Records, raw_ostream &OS,
+static void generateGetName(ArrayRef<const Record *> Records, raw_ostream &OS,
                             StringRef Enum, const DirectiveLanguage &DirLang,
                             StringRef Prefix) {
   OS << "\n";
@@ -300,11 +300,11 @@ static void GenerateGetName(ArrayRef<const Record *> Records, raw_ostream &OS,
 }
 
 // Generate function implementation for get<Enum>Kind(StringRef Str)
-static void GenerateGetKind(ArrayRef<const Record *> Records, raw_ostream &OS,
+static void generateGetKind(ArrayRef<const Record *> Records, raw_ostream &OS,
                             StringRef Enum, const DirectiveLanguage &DirLang,
                             StringRef Prefix, bool ImplicitAsUnknown) {
 
-  auto DefaultIt = find_if(
+  const auto *DefaultIt = find_if(
       Records, [](const Record *R) { return R->getValueAsBit("isDefault"); });
 
   if (DefaultIt == Records.end()) {
@@ -334,7 +334,7 @@ static void GenerateGetKind(ArrayRef<const Record *> Records, raw_ostream &OS,
 }
 
 // Generate function implementation for get<ClauseVal>Kind(StringRef Str)
-static void GenerateGetKindClauseVal(const DirectiveLanguage &DirLang,
+static void generateGetKindClauseVal(const DirectiveLanguage &DirLang,
                                      raw_ostream &OS) {
   for (const Clause C : DirLang.getClauses()) {
     const auto &ClauseVals = C.getClauseVals();
@@ -389,7 +389,7 @@ static void GenerateGetKindClauseVal(const DirectiveLanguage &DirLang,
   }
 }
 
-static void GenerateCaseForVersionedClauses(ArrayRef<const Record *> Clauses,
+static void generateCaseForVersionedClauses(ArrayRef<const Record *> Clauses,
                                             raw_ostream &OS,
                                             StringRef DirectiveName,
                                             const DirectiveLanguage &DirLang,
@@ -406,7 +406,7 @@ static void GenerateCaseForVersionedClauses(ArrayRef<const Record *> Clauses,
   }
 }
 
-static std::string GetDirectiveName(const DirectiveLanguage &DirLang,
+static std::string getDirectiveName(const DirectiveLanguage &DirLang,
                                     const Record *Rec) {
   Directive Dir(Rec);
   return (Twine("llvm::") + DirLang.getCppNamespace() +
@@ -414,12 +414,12 @@ static std::string GetDirectiveName(const DirectiveLanguage &DirLang,
       .str();
 }
 
-static std::string GetDirectiveType(const DirectiveLanguage &DirLang) {
+static std::string getDirectiveType(const DirectiveLanguage &DirLang) {
   return (Twine("llvm::") + DirLang.getCppNamespace() + "::Directive").str();
 }
 
 // Generate the isAllowedClauseForDirective function implementation.
-static void GenerateIsAllowedClause(const DirectiveLanguage &DirLang,
+static void generateIsAllowedClause(const DirectiveLanguage &DirLang,
                                     raw_ostream &OS) {
   OS << "\n";
   OS << "bool llvm::" << DirLang.getCppNamespace()
@@ -445,16 +445,16 @@ static void GenerateIsAllowedClause(const DirectiveLanguage &DirLang,
 
       StringSet<> Cases;
 
-      GenerateCaseForVersionedClauses(Dir.getAllowedClauses(), OS,
+      generateCaseForVersionedClauses(Dir.getAllowedClauses(), OS,
                                       Dir.getName(), DirLang, Cases);
 
-      GenerateCaseForVersionedClauses(Dir.getAllowedOnceClauses(), OS,
+      generateCaseForVersionedClauses(Dir.getAllowedOnceClauses(), OS,
                                       Dir.getName(), DirLang, Cases);
 
-      GenerateCaseForVersionedClauses(Dir.getAllowedExclusiveClauses(), OS,
+      generateCaseForVersionedClauses(Dir.getAllowedExclusiveClauses(), OS,
                                       Dir.getName(), DirLang, Cases);
 
-      GenerateCaseForVersionedClauses(Dir.getRequiredClauses(), OS,
+      generateCaseForVersionedClauses(Dir.getRequiredClauses(), OS,
                                       Dir.getName(), DirLang, Cases);
 
       OS << "        default:\n";
@@ -470,7 +470,7 @@ static void GenerateIsAllowedClause(const DirectiveLanguage &DirLang,
   OS << "}\n"; // End of function isAllowedClauseForDirective
 }
 
-static void EmitLeafTable(const DirectiveLanguage &DirLang, raw_ostream &OS,
+static void emitLeafTable(const DirectiveLanguage &DirLang, raw_ostream &OS,
                           StringRef TableName) {
   // The leaf constructs are emitted in a form of a 2D table, where each
   // row corresponds to a directive (and there is a row for each directive).
@@ -498,7 +498,7 @@ static void EmitLeafTable(const DirectiveLanguage &DirLang, raw_ostream &OS,
     DirId.insert(std::make_pair(Rec, Idx));
 
   using LeafList = std::vector<int>;
-  int MaxLeafCount = GetMaxLeafCount(DirLang);
+  int MaxLeafCount = getMaxLeafCount(DirLang);
 
   // The initial leaf table, rows order is same as directive order.
   std::vector<LeafList> LeafTable(Directives.size());
@@ -560,19 +560,19 @@ static void EmitLeafTable(const DirectiveLanguage &DirLang, raw_ostream &OS,
   // type is `int` (by default). The code above uses `int` to store directive
   // ids, so make sure that we catch it when something changes in the
   // underlying type.
-  std::string DirectiveType = GetDirectiveType(DirLang);
+  std::string DirectiveType = getDirectiveType(DirLang);
   OS << "\nstatic_assert(sizeof(" << DirectiveType << ") == sizeof(int));\n";
 
   OS << "[[maybe_unused]] static const " << DirectiveType << ' ' << TableName
      << "[][" << MaxLeafCount + 2 << "] = {\n";
   for (size_t I = 0, E = Directives.size(); I != E; ++I) {
     auto &Leaves = LeafTable[Ordering[I]];
-    OS << "    {" << GetDirectiveName(DirLang, Directives[Leaves[0]]);
+    OS << "    {" << getDirectiveName(DirLang, Directives[Leaves[0]]);
     OS << ", static_cast<" << DirectiveType << ">(" << Leaves[1] << "),";
     for (size_t I = 2, E = Leaves.size(); I != E; ++I) {
       int Idx = Leaves[I];
       if (Idx >= 0)
-        OS << ' ' << GetDirectiveName(DirLang, Directives[Leaves[I]]) << ',';
+        OS << ' ' << getDirectiveName(DirLang, Directives[Leaves[I]]) << ',';
       else
         OS << " static_cast<" << DirectiveType << ">(-1),";
     }
@@ -600,7 +600,7 @@ static void EmitLeafTable(const DirectiveLanguage &DirLang, raw_ostream &OS,
   OS << "\n};\n";
 }
 
-static void GenerateGetDirectiveAssociation(const DirectiveLanguage &DirLang,
+static void generateGetDirectiveAssociation(const DirectiveLanguage &DirLang,
                                             raw_ostream &OS) {
   enum struct Association {
     None = 0, // None should be the smallest value.
@@ -613,10 +613,10 @@ static void GenerateGetDirectiveAssociation(const DirectiveLanguage &DirLang,
     Invalid,
   };
 
-  ArrayRef<const Record *> associations = DirLang.getAssociations();
+  ArrayRef<const Record *> Associations = DirLang.getAssociations();
 
-  auto getAssocValue = [](StringRef name) -> Association {
-    return StringSwitch<Association>(name)
+  auto GetAssocValue = [](StringRef Name) -> Association {
+    return StringSwitch<Association>(Name)
         .Case("AS_Block", Association::Block)
         .Case("AS_Declaration", Association::Declaration)
         .Case("AS_Delimited", Association::Delimited)
@@ -627,24 +627,24 @@ static void GenerateGetDirectiveAssociation(const DirectiveLanguage &DirLang,
         .Default(Association::Invalid);
   };
 
-  auto getAssocName = [&](Association A) -> StringRef {
+  auto GetAssocName = [&](Association A) -> StringRef {
     if (A != Association::Invalid && A != Association::FromLeaves) {
-      auto F = find_if(associations, [&](const Record *R) {
-        return getAssocValue(R->getName()) == A;
+      const auto *F = find_if(Associations, [&](const Record *R) {
+        return GetAssocValue(R->getName()) == A;
       });
-      if (F != associations.end())
+      if (F != Associations.end())
         return (*F)->getValueAsString("name"); // enum name
     }
     llvm_unreachable("Unexpected association value");
   };
 
-  auto errorPrefixFor = [&](Directive D) -> std::string {
+  auto ErrorPrefixFor = [&](Directive D) -> std::string {
     return (Twine("Directive '") + D.getName() + "' in namespace '" +
             DirLang.getCppNamespace() + "' ")
         .str();
   };
 
-  auto reduce = [&](Association A, Association B) -> Association {
+  auto Reduce = [&](Association A, Association B) -> Association {
     if (A > B)
       std::swap(A, B);
 
@@ -663,14 +663,14 @@ static void GenerateGetDirectiveAssociation(const DirectiveLanguage &DirLang,
 
   DenseMap<const Record *, Association> AsMap;
 
-  auto compAssocImpl = [&](const Record *R, auto &&Self) -> Association {
+  auto CompAssocImpl = [&](const Record *R, auto &&Self) -> Association {
     if (auto F = AsMap.find(R); F != AsMap.end())
       return F->second;
 
     Directive D(R);
-    Association AS = getAssocValue(D.getAssociation()->getName());
+    Association AS = GetAssocValue(D.getAssociation()->getName());
     if (AS == Association::Invalid) {
-      PrintFatalError(errorPrefixFor(D) +
+      PrintFatalError(ErrorPrefixFor(D) +
                       "has an unrecognized value for association: '" +
                       D.getAssociation()->getName() + "'");
     }
@@ -679,22 +679,22 @@ static void GenerateGetDirectiveAssociation(const DirectiveLanguage &DirLang,
       return AS;
     }
     // Compute the association from leaf constructs.
-    std::vector<const Record *> leaves = D.getLeafConstructs();
-    if (leaves.empty()) {
+    std::vector<const Record *> Leaves = D.getLeafConstructs();
+    if (Leaves.empty()) {
       errs() << D.getName() << '\n';
-      PrintFatalError(errorPrefixFor(D) +
+      PrintFatalError(ErrorPrefixFor(D) +
                       "requests association to be computed from leaves, "
                       "but it has no leaves");
     }
 
-    Association Result = Self(leaves[0], Self);
-    for (int I = 1, E = leaves.size(); I < E; ++I) {
-      Association A = Self(leaves[I], Self);
-      Association R = reduce(Result, A);
+    Association Result = Self(Leaves[0], Self);
+    for (int I = 1, E = Leaves.size(); I < E; ++I) {
+      Association A = Self(Leaves[I], Self);
+      Association R = Reduce(Result, A);
       if (R == Association::Invalid) {
-        PrintFatalError(errorPrefixFor(D) +
+        PrintFatalError(ErrorPrefixFor(D) +
                         "has leaves with incompatible association values: " +
-                        getAssocName(A) + " and " + getAssocName(R));
+                        GetAssocName(A) + " and " + GetAssocName(R));
       }
       Result = R;
     }
@@ -706,11 +706,11 @@ static void GenerateGetDirectiveAssociation(const DirectiveLanguage &DirLang,
   };
 
   for (const Record *R : DirLang.getDirectives())
-    compAssocImpl(R, compAssocImpl); // Updates AsMap.
+    CompAssocImpl(R, CompAssocImpl); // Updates AsMap.
 
   OS << '\n';
 
-  auto getQualifiedName = [&](StringRef Formatted) -> std::string {
+  auto GetQualifiedName = [&](StringRef Formatted) -> std::string {
     return (Twine("llvm::") + DirLang.getCppNamespace() +
             "::Directive::" + DirLang.getDirectivePrefix() + Formatted)
         .str();
@@ -727,9 +727,9 @@ static void GenerateGetDirectiveAssociation(const DirectiveLanguage &DirLang,
   for (const Record *R : DirLang.getDirectives()) {
     if (auto F = AsMap.find(R); F != AsMap.end()) {
       Directive Dir(R);
-      OS << "  case " << getQualifiedName(Dir.getFormattedName()) << ":\n";
+      OS << "  case " << GetQualifiedName(Dir.getFormattedName()) << ":\n";
       OS << "    return " << AssociationTypeName
-         << "::" << getAssocName(F->second) << ";\n";
+         << "::" << GetAssocName(F->second) << ";\n";
     }
   }
   OS << "  } // switch (Dir)\n";
@@ -737,7 +737,7 @@ static void GenerateGetDirectiveAssociation(const DirectiveLanguage &DirLang,
   OS << "}\n";
 }
 
-static void GenerateGetDirectiveCategory(const DirectiveLanguage &DirLang,
+static void generateGetDirectiveCategory(const DirectiveLanguage &DirLang,
                                          raw_ostream &OS) {
   std::string LangNamespace = "llvm::" + DirLang.getCppNamespace().str();
   std::string CategoryTypeName = LangNamespace + "::Category";
@@ -745,12 +745,12 @@ static void GenerateGetDirectiveCategory(const DirectiveLanguage &DirLang,
 
   OS << '\n';
   OS << CategoryTypeName << ' ' << LangNamespace << "::getDirectiveCategory("
-     << GetDirectiveType(DirLang) << " Dir) {\n";
+     << getDirectiveType(DirLang) << " Dir) {\n";
   OS << "  switch (Dir) {\n";
 
   for (const Record *R : DirLang.getDirectives()) {
     Directive D(R);
-    OS << "  case " << GetDirectiveName(DirLang, R) << ":\n";
+    OS << "  case " << getDirectiveName(DirLang, R) << ":\n";
     OS << "    return " << CategoryNamespace
        << D.getCategory()->getValueAsString("name") << ";\n";
   }
@@ -760,7 +760,7 @@ static void GenerateGetDirectiveCategory(const DirectiveLanguage &DirLang,
 }
 
 // Generate a simple enum set with the give clauses.
-static void GenerateClauseSet(ArrayRef<const Record *> Clauses, raw_ostream &OS,
+static void generateClauseSet(ArrayRef<const Record *> Clauses, raw_ostream &OS,
                               StringRef ClauseSetPrefix, const Directive &Dir,
                               const DirectiveLanguage &DirLang) {
 
@@ -778,7 +778,7 @@ static void GenerateClauseSet(ArrayRef<const Record *> Clauses, raw_ostream &OS,
 }
 
 // Generate an enum set for the 4 kinds of clauses linked to a directive.
-static void GenerateDirectiveClauseSets(const DirectiveLanguage &DirLang,
+static void generateDirectiveClauseSets(const DirectiveLanguage &DirLang,
                                         raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_DIRECTIVE_CLAUSE_SETS", OS);
@@ -796,13 +796,13 @@ static void GenerateDirectiveClauseSets(const DirectiveLanguage &DirLang,
     OS << "\n";
     OS << "  // Sets for " << Dir.getName() << "\n";
 
-    GenerateClauseSet(Dir.getAllowedClauses(), OS, "allowedClauses_", Dir,
+    generateClauseSet(Dir.getAllowedClauses(), OS, "allowedClauses_", Dir,
                       DirLang);
-    GenerateClauseSet(Dir.getAllowedOnceClauses(), OS, "allowedOnceClauses_",
+    generateClauseSet(Dir.getAllowedOnceClauses(), OS, "allowedOnceClauses_",
                       Dir, DirLang);
-    GenerateClauseSet(Dir.getAllowedExclusiveClauses(), OS,
+    generateClauseSet(Dir.getAllowedExclusiveClauses(), OS,
                       "allowedExclusiveClauses_", Dir, DirLang);
-    GenerateClauseSet(Dir.getRequiredClauses(), OS, "requiredClauses_", Dir,
+    generateClauseSet(Dir.getRequiredClauses(), OS, "requiredClauses_", Dir,
                       DirLang);
   }
 
@@ -816,7 +816,7 @@ static void GenerateDirectiveClauseSets(const DirectiveLanguage &DirLang,
 // Generate a map of directive (key) with DirectiveClauses struct as values.
 // The struct holds the 4 sets of enumeration for the 4 kinds of clauses
 // allowances (allowed, allowed once, allowed exclusive and required).
-static void GenerateDirectiveClauseMap(const DirectiveLanguage &DirLang,
+static void generateDirectiveClauseMap(const DirectiveLanguage &DirLang,
                                        raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_DIRECTIVE_CLAUSE_MAP", OS);
@@ -850,7 +850,7 @@ static void GenerateDirectiveClauseMap(const DirectiveLanguage &DirLang,
 // If the clause does not hold a value, an EMPTY_CLASS is used.
 // If the clause class is generic then a WRAPPER_CLASS is used. When the value
 // is optional, the value class is wrapped into a std::optional.
-static void GenerateFlangClauseParserClass(const DirectiveLanguage &DirLang,
+static void generateFlangClauseParserClass(const DirectiveLanguage &DirLang,
                                            raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_CLAUSE_PARSER_CLASSES", OS);
@@ -877,7 +877,7 @@ static void GenerateFlangClauseParserClass(const DirectiveLanguage &DirLang,
 }
 
 // Generate a list of the different clause classes for Flang.
-static void GenerateFlangClauseParserClassList(const DirectiveLanguage &DirLang,
+static void generateFlangClauseParserClassList(const DirectiveLanguage &DirLang,
                                                raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_CLAUSE_PARSER_CLASSES_LIST", OS);
@@ -890,7 +890,7 @@ static void GenerateFlangClauseParserClassList(const DirectiveLanguage &DirLang,
 }
 
 // Generate dump node list for the clauses holding a generic class name.
-static void GenerateFlangClauseDump(const DirectiveLanguage &DirLang,
+static void generateFlangClauseDump(const DirectiveLanguage &DirLang,
                                     raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_DUMP_PARSE_TREE_CLAUSES", OS);
@@ -904,7 +904,7 @@ static void GenerateFlangClauseDump(const DirectiveLanguage &DirLang,
 
 // Generate Unparse functions for clauses classes in the Flang parse-tree
 // If the clause is a non-generic class, no entry is generated.
-static void GenerateFlangClauseUnparse(const DirectiveLanguage &DirLang,
+static void generateFlangClauseUnparse(const DirectiveLanguage &DirLang,
                                        raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_CLAUSE_UNPARSE", OS);
@@ -955,7 +955,7 @@ static void GenerateFlangClauseUnparse(const DirectiveLanguage &DirLang,
 }
 
 // Generate check in the Enter functions for clauses classes.
-static void GenerateFlangClauseCheckPrototypes(const DirectiveLanguage &DirLang,
+static void generateFlangClauseCheckPrototypes(const DirectiveLanguage &DirLang,
                                                raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_CLAUSE_CHECK_ENTER", OS);
@@ -969,7 +969,7 @@ static void GenerateFlangClauseCheckPrototypes(const DirectiveLanguage &DirLang,
 
 // Generate the mapping for clauses between the parser class and the
 // corresponding clause Kind
-static void GenerateFlangClauseParserKindMap(const DirectiveLanguage &DirLang,
+static void generateFlangClauseParserKindMap(const DirectiveLanguage &DirLang,
                                              raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_CLAUSE_PARSER_KIND_MAP", OS);
@@ -996,7 +996,7 @@ static bool compareClauseName(const Record *R1, const Record *R2) {
 }
 
 // Generate the parser for the clauses.
-static void GenerateFlangClausesParser(const DirectiveLanguage &DirLang,
+static void generateFlangClausesParser(const DirectiveLanguage &DirLang,
                                        raw_ostream &OS) {
   std::vector<const Record *> Clauses = DirLang.getClauses();
   // Sort clauses in reverse alphabetical order so with clauses with same
@@ -1004,8 +1004,8 @@ static void GenerateFlangClausesParser(const DirectiveLanguage &DirLang,
   sort(Clauses, compareClauseName);
   IfDefScope Scope("GEN_FLANG_CLAUSES_PARSER", OS);
   OS << "\n";
-  unsigned index = 0;
-  unsigned lastClauseIndex = Clauses.size() - 1;
+  unsigned Index = 0;
+  unsigned LastClauseIndex = Clauses.size() - 1;
   OS << "TYPE_PARSER(\n";
   for (const Clause Clause : Clauses) {
     if (Clause.getAliases().empty()) {
@@ -1013,8 +1013,8 @@ static void GenerateFlangClausesParser(const DirectiveLanguage &DirLang,
     } else {
       OS << "  ("
          << "\"" << Clause.getName() << "\"_tok";
-      for (StringRef alias : Clause.getAliases()) {
-        OS << " || \"" << alias << "\"_tok";
+      for (StringRef Alias : Clause.getAliases()) {
+        OS << " || \"" << Alias << "\"_tok";
       }
       OS << ")";
     }
@@ -1024,10 +1024,10 @@ static void GenerateFlangClausesParser(const DirectiveLanguage &DirLang,
        << "::" << Clause.getFormattedParserClassName() << ">(";
     if (Clause.getFlangClass().empty()) {
       OS << "))";
-      if (index != lastClauseIndex)
+      if (Index != LastClauseIndex)
         OS << " ||";
       OS << "\n";
-      ++index;
+      ++Index;
       continue;
     }
 
@@ -1064,38 +1064,38 @@ static void GenerateFlangClausesParser(const DirectiveLanguage &DirLang,
     if (Clause.isValueOptional()) // close maybe(.
       OS << ")";
     OS << "))";
-    if (index != lastClauseIndex)
+    if (Index != LastClauseIndex)
       OS << " ||";
     OS << "\n";
-    ++index;
+    ++Index;
   }
   OS << ")\n";
 }
 
 // Generate the implementation section for the enumeration in the directive
 // language
-static void EmitDirectivesFlangImpl(const DirectiveLanguage &DirLang,
+static void emitDirectivesFlangImpl(const DirectiveLanguage &DirLang,
                                     raw_ostream &OS) {
-  GenerateDirectiveClauseSets(DirLang, OS);
+  generateDirectiveClauseSets(DirLang, OS);
 
-  GenerateDirectiveClauseMap(DirLang, OS);
+  generateDirectiveClauseMap(DirLang, OS);
 
-  GenerateFlangClauseParserClass(DirLang, OS);
+  generateFlangClauseParserClass(DirLang, OS);
 
-  GenerateFlangClauseParserClassList(DirLang, OS);
+  generateFlangClauseParserClassList(DirLang, OS);
 
-  GenerateFlangClauseDump(DirLang, OS);
+  generateFlangClauseDump(DirLang, OS);
 
-  GenerateFlangClauseUnparse(DirLang, OS);
+  generateFlangClauseUnparse(DirLang, OS);
 
-  GenerateFlangClauseCheckPrototypes(DirLang, OS);
+  generateFlangClauseCheckPrototypes(DirLang, OS);
 
-  GenerateFlangClauseParserKindMap(DirLang, OS);
+  generateFlangClauseParserKindMap(DirLang, OS);
 
-  GenerateFlangClausesParser(DirLang, OS);
+  generateFlangClausesParser(DirLang, OS);
 }
 
-static void GenerateClauseClassMacro(const DirectiveLanguage &DirLang,
+static void generateClauseClassMacro(const DirectiveLanguage &DirLang,
                                      raw_ostream &OS) {
   // Generate macros style information for legacy code in clang
   IfDefScope Scope("GEN_CLANG_CLAUSE_CLASS", OS);
@@ -1163,63 +1163,63 @@ static void GenerateClauseClassMacro(const DirectiveLanguage &DirLang,
 
 // Generate the implemenation for the enumeration in the directive
 // language. This code can be included in library.
-void EmitDirectivesBasicImpl(const DirectiveLanguage &DirLang,
+void emitDirectivesBasicImpl(const DirectiveLanguage &DirLang,
                              raw_ostream &OS) {
   IfDefScope Scope("GEN_DIRECTIVES_IMPL", OS);
 
   OS << "\n#include \"llvm/Support/ErrorHandling.h\"\n";
 
   // getDirectiveKind(StringRef Str)
-  GenerateGetKind(DirLang.getDirectives(), OS, "Directive", DirLang,
+  generateGetKind(DirLang.getDirectives(), OS, "Directive", DirLang,
                   DirLang.getDirectivePrefix(), /*ImplicitAsUnknown=*/false);
 
   // getDirectiveName(Directive Kind)
-  GenerateGetName(DirLang.getDirectives(), OS, "Directive", DirLang,
+  generateGetName(DirLang.getDirectives(), OS, "Directive", DirLang,
                   DirLang.getDirectivePrefix());
 
   // getClauseKind(StringRef Str)
-  GenerateGetKind(DirLang.getClauses(), OS, "Clause", DirLang,
+  generateGetKind(DirLang.getClauses(), OS, "Clause", DirLang,
                   DirLang.getClausePrefix(),
                   /*ImplicitAsUnknown=*/true);
 
   // getClauseName(Clause Kind)
-  GenerateGetName(DirLang.getClauses(), OS, "Clause", DirLang,
+  generateGetName(DirLang.getClauses(), OS, "Clause", DirLang,
                   DirLang.getClausePrefix());
 
   // get<ClauseVal>Kind(StringRef Str)
-  GenerateGetKindClauseVal(DirLang, OS);
+  generateGetKindClauseVal(DirLang, OS);
 
   // isAllowedClauseForDirective(Directive D, Clause C, unsigned Version)
-  GenerateIsAllowedClause(DirLang, OS);
+  generateIsAllowedClause(DirLang, OS);
 
   // getDirectiveAssociation(Directive D)
-  GenerateGetDirectiveAssociation(DirLang, OS);
+  generateGetDirectiveAssociation(DirLang, OS);
 
   // getDirectiveCategory(Directive D)
-  GenerateGetDirectiveCategory(DirLang, OS);
+  generateGetDirectiveCategory(DirLang, OS);
 
   // Leaf table for getLeafConstructs, etc.
-  EmitLeafTable(DirLang, OS, "LeafConstructTable");
+  emitLeafTable(DirLang, OS, "LeafConstructTable");
 }
 
 // Generate the implemenation section for the enumeration in the directive
 // language.
-static void EmitDirectivesImpl(const RecordKeeper &Records, raw_ostream &OS) {
+static void emitDirectivesImpl(const RecordKeeper &Records, raw_ostream &OS) {
   const auto DirLang = DirectiveLanguage(Records);
   if (DirLang.HasValidityErrors())
     return;
 
-  EmitDirectivesFlangImpl(DirLang, OS);
+  emitDirectivesFlangImpl(DirLang, OS);
 
-  GenerateClauseClassMacro(DirLang, OS);
+  generateClauseClassMacro(DirLang, OS);
 
-  EmitDirectivesBasicImpl(DirLang, OS);
+  emitDirectivesBasicImpl(DirLang, OS);
 }
 
 static TableGen::Emitter::Opt
-    X("gen-directive-decl", EmitDirectivesDecl,
+    X("gen-directive-decl", emitDirectivesDecl,
       "Generate directive related declaration code (header file)");
 
 static TableGen::Emitter::Opt
-    Y("gen-directive-impl", EmitDirectivesImpl,
+    Y("gen-directive-impl", emitDirectivesImpl,
       "Generate directive related implementation code");
diff --git a/llvm/utils/TableGen/DisassemblerEmitter.cpp b/llvm/utils/TableGen/DisassemblerEmitter.cpp
index eb15392272a3..70d835e699ff 100644
--- a/llvm/utils/TableGen/DisassemblerEmitter.cpp
+++ b/llvm/utils/TableGen/DisassemblerEmitter.cpp
@@ -95,7 +95,7 @@ using namespace llvm::X86Disassembler;
 /// X86RecognizableInstr.cpp contains the implementation for a single
 ///   instruction.
 
-static void EmitDisassembler(const RecordKeeper &Records, raw_ostream &OS) {
+static void emitDisassembler(const RecordKeeper &Records, raw_ostream &OS) {
   const CodeGenTarget Target(Records);
   emitSourceFileHeader(" * " + Target.getName().str() + " Disassembler", OS);
 
@@ -132,5 +132,5 @@ static void EmitDisassembler(const RecordKeeper &Records, raw_ostream &OS) {
 
 cl::OptionCategory DisassemblerEmitterCat("Options for -gen-disassembler");
 
-static TableGen::Emitter::Opt X("gen-disassembler", EmitDisassembler,
+static TableGen::Emitter::Opt X("gen-disassembler", emitDisassembler,
                                 "Generate disassembler");
diff --git a/llvm/utils/TableGen/OptionParserEmitter.cpp b/llvm/utils/TableGen/OptionParserEmitter.cpp
index cd7a140bb231..86e8378ad5ac 100644
--- a/llvm/utils/TableGen/OptionParserEmitter.cpp
+++ b/llvm/utils/TableGen/OptionParserEmitter.cpp
@@ -26,7 +26,7 @@ static std::string getOptionName(const Record &R) {
   return std::string(R.getValueAsString("EnumName"));
 }
 
-static raw_ostream &write_cstring(raw_ostream &OS, llvm::StringRef Str) {
+static raw_ostream &writeCstring(raw_ostream &OS, llvm::StringRef Str) {
   OS << '"';
   OS.write_escaped(Str);
   OS << '"';
@@ -117,7 +117,7 @@ struct SimpleEnumValueTable {
     OS << "static const SimpleEnumValue " << ValueTableName << "[] = {\n";
     for (unsigned I = 0, E = Values.size(); I != E; ++I) {
       OS << "{";
-      write_cstring(OS, Values[I]);
+      writeCstring(OS, Values[I]);
       OS << ",";
       OS << "static_cast<unsigned>(";
       emitScopedNormalizedValue(OS, NormalizedValues[I]);
@@ -190,7 +190,7 @@ static MarshallingInfo createMarshallingInfo(const Record &R) {
   return Ret;
 }
 
-static void EmitHelpTextsForVariants(
+static void emitHelpTextsForVariants(
     raw_ostream &OS, std::vector<std::pair<std::vector<std::string>, StringRef>>
                          HelpTextsForVariants) {
   // OptTable must be constexpr so it uses std::arrays with these capacities.
@@ -235,7 +235,7 @@ static void EmitHelpTextsForVariants(
     OS << "}}, ";
 
     if (Help.size())
-      write_cstring(OS, Help);
+      writeCstring(OS, Help);
     else
       OS << "nullptr";
     OS << ")";
@@ -249,7 +249,7 @@ static void EmitHelpTextsForVariants(
 /// OptionParserEmitter - This tablegen backend takes an input .td file
 /// describing a list of options and emits a data structure for parsing and
 /// working with those options when given an input command line.
-static void EmitOptionParser(const RecordKeeper &Records, raw_ostream &OS) {
+static void emitOptionParser(const RecordKeeper &Records, raw_ostream &OS) {
   // Get the option groups and options.
   ArrayRef<const Record *> Groups =
       Records.getAllDerivedDefinitions("OptionGroup");
@@ -363,12 +363,12 @@ static void EmitOptionParser(const RecordKeeper &Records, raw_ostream &OS) {
     if (!isa<UnsetInit>(R.getValueInit("HelpText"))) {
       OS << ",\n";
       OS << "       ";
-      write_cstring(OS, R.getValueAsString("HelpText"));
+      writeCstring(OS, R.getValueAsString("HelpText"));
     } else
       OS << ", nullptr";
 
     // Not using Visibility specific text for group help.
-    EmitHelpTextsForVariants(OS, {});
+    emitHelpTextsForVariants(OS, {});
 
     // The option meta-variable name (unused).
     OS << ", nullptr";
@@ -387,7 +387,7 @@ static void EmitOptionParser(const RecordKeeper &Records, raw_ostream &OS) {
     OS << Prefixes[PrefixKeyT(RPrefixes.begin(), RPrefixes.end())] << ", ";
 
     // The option prefixed name.
-    write_cstring(OS, getOptionPrefixedName(R));
+    writeCstring(OS, getOptionPrefixedName(R));
 
     // The option identifier name.
     OS << ", " << getOptionName(R);
@@ -464,7 +464,7 @@ static void EmitOptionParser(const RecordKeeper &Records, raw_ostream &OS) {
     if (!isa<UnsetInit>(R.getValueInit("HelpText"))) {
       OS << ",\n";
       OS << "       ";
-      write_cstring(OS, R.getValueAsString("HelpText"));
+      writeCstring(OS, R.getValueAsString("HelpText"));
     } else
       OS << ", nullptr";
 
@@ -482,19 +482,19 @@ static void EmitOptionParser(const RecordKeeper &Records, raw_ostream &OS) {
       HelpTextsForVariants.push_back(std::make_pair(
           VisibilityNames, VisibilityHelp->getValueAsString("Text")));
     }
-    EmitHelpTextsForVariants(OS, HelpTextsForVariants);
+    emitHelpTextsForVariants(OS, HelpTextsForVariants);
 
     // The option meta-variable name.
     OS << ", ";
     if (!isa<UnsetInit>(R.getValueInit("MetaVarName")))
-      write_cstring(OS, R.getValueAsString("MetaVarName"));
+      writeCstring(OS, R.getValueAsString("MetaVarName"));
     else
       OS << "nullptr";
 
     // The option Values. Used for shell autocompletion.
     OS << ", ";
     if (!isa<UnsetInit>(R.getValueInit("Values")))
-      write_cstring(OS, R.getValueAsString("Values"));
+      writeCstring(OS, R.getValueAsString("Values"));
     else if (!isa<UnsetInit>(R.getValueInit("ValuesCode"))) {
       OS << getOptionName(R) << "_Values";
     } else
@@ -571,5 +571,5 @@ static void EmitOptionParser(const RecordKeeper &Records, raw_ostream &OS) {
   OS << "\n";
 }
 
-static TableGen::Emitter::Opt X("gen-opt-parser-defs", EmitOptionParser,
+static TableGen::Emitter::Opt X("gen-opt-parser-defs", emitOptionParser,
                                 "Generate option definitions");
diff --git a/llvm/utils/TableGen/OptionRSTEmitter.cpp b/llvm/utils/TableGen/OptionRSTEmitter.cpp
index 1b4c4cad4f0a..6eac10e1831f 100644
--- a/llvm/utils/TableGen/OptionRSTEmitter.cpp
+++ b/llvm/utils/TableGen/OptionRSTEmitter.cpp
@@ -16,7 +16,7 @@ using namespace llvm;
 
 /// This tablegen backend takes an input .td file describing a list of options
 /// and emits a RST man page.
-static void EmitOptionRST(const RecordKeeper &Records, raw_ostream &OS) {
+static void emitOptionRst(const RecordKeeper &Records, raw_ostream &OS) {
   llvm::StringMap<std::vector<const Record *>> OptionsByGroup;
 
   // Get the options.
@@ -96,5 +96,5 @@ static void EmitOptionRST(const RecordKeeper &Records, raw_ostream &OS) {
   }
 }
 
-static TableGen::Emitter::Opt X("gen-opt-rst", EmitOptionRST,
+static TableGen::Emitter::Opt X("gen-opt-rst", emitOptionRst,
                                 "Generate option RST");
diff --git a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
index 23496a37d5ea..39211aab6f2d 100644
--- a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
+++ b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
@@ -244,13 +244,13 @@ static void emitRISCVExtensionBitmask(const RecordKeeper &RK, raw_ostream &OS) {
   OS << "#endif\n";
 }
 
-static void EmitRISCVTargetDef(const RecordKeeper &RK, raw_ostream &OS) {
+static void emitRiscvTargetDef(const RecordKeeper &RK, raw_ostream &OS) {
   emitRISCVExtensions(RK, OS);
   emitRISCVProfiles(RK, OS);
   emitRISCVProcs(RK, OS);
   emitRISCVExtensionBitmask(RK, OS);
 }
 
-static TableGen::Emitter::Opt X("gen-riscv-target-def", EmitRISCVTargetDef,
+static TableGen::Emitter::Opt X("gen-riscv-target-def", emitRiscvTargetDef,
                                 "Generate the list of CPUs and extensions for "
                                 "RISC-V");
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index 17b84d06fe85..02c799cb6f14 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -87,65 +87,65 @@ class SubtargetEmitter {
   CodeGenSchedModels &SchedModels;
   std::string Target;
 
-  FeatureMapTy Enumeration(raw_ostream &OS);
-  void EmitSubtargetInfoMacroCalls(raw_ostream &OS);
-  unsigned FeatureKeyValues(raw_ostream &OS, const FeatureMapTy &FeatureMap);
-  unsigned CPUKeyValues(raw_ostream &OS, const FeatureMapTy &FeatureMap);
-  void FormItineraryStageString(const std::string &Names,
+  FeatureMapTy enumeration(raw_ostream &OS);
+  void emitSubtargetInfoMacroCalls(raw_ostream &OS);
+  unsigned featureKeyValues(raw_ostream &OS, const FeatureMapTy &FeatureMap);
+  unsigned cpuKeyValues(raw_ostream &OS, const FeatureMapTy &FeatureMap);
+  void formItineraryStageString(const std::string &Names,
                                 const Record *ItinData, std::string &ItinString,
                                 unsigned &NStages);
-  void FormItineraryOperandCycleString(const Record *ItinData,
+  void formItineraryOperandCycleString(const Record *ItinData,
                                        std::string &ItinString,
                                        unsigned &NOperandCycles);
-  void FormItineraryBypassString(const std::string &Names,
+  void formItineraryBypassString(const std::string &Names,
                                  const Record *ItinData,
                                  std::string &ItinString,
                                  unsigned NOperandCycles);
-  void EmitStageAndOperandCycleData(
+  void emitStageAndOperandCycleData(
       raw_ostream &OS, std::vector<std::vector<InstrItinerary>> &ProcItinLists);
-  void EmitItineraries(raw_ostream &OS,
+  void emitItineraries(raw_ostream &OS,
                        std::vector<std::vector<InstrItinerary>> &ProcItinLists);
-  unsigned EmitRegisterFileTables(const CodeGenProcModel &ProcModel,
+  unsigned emitRegisterFileTables(const CodeGenProcModel &ProcModel,
                                   raw_ostream &OS);
-  void EmitLoadStoreQueueInfo(const CodeGenProcModel &ProcModel,
+  void emitLoadStoreQueueInfo(const CodeGenProcModel &ProcModel,
                               raw_ostream &OS);
-  void EmitExtraProcessorInfo(const CodeGenProcModel &ProcModel,
+  void emitExtraProcessorInfo(const CodeGenProcModel &ProcModel,
                               raw_ostream &OS);
-  void EmitProcessorProp(raw_ostream &OS, const Record *R, StringRef Name,
+  void emitProcessorProp(raw_ostream &OS, const Record *R, StringRef Name,
                          char Separator);
-  void EmitProcessorResourceSubUnits(const CodeGenProcModel &ProcModel,
+  void emitProcessorResourceSubUnits(const CodeGenProcModel &ProcModel,
                                      raw_ostream &OS);
-  void EmitProcessorResources(const CodeGenProcModel &ProcModel,
+  void emitProcessorResources(const CodeGenProcModel &ProcModel,
                               raw_ostream &OS);
-  const Record *FindWriteResources(const CodeGenSchedRW &SchedWrite,
+  const Record *findWriteResources(const CodeGenSchedRW &SchedWrite,
                                    const CodeGenProcModel &ProcModel);
-  const Record *FindReadAdvance(const CodeGenSchedRW &SchedRead,
+  const Record *findReadAdvance(const CodeGenSchedRW &SchedRead,
                                 const CodeGenProcModel &ProcModel);
-  void ExpandProcResources(ConstRecVec &PRVec,
+  void expandProcResources(ConstRecVec &PRVec,
                            std::vector<int64_t> &ReleaseAtCycles,
                            std::vector<int64_t> &AcquireAtCycles,
                            const CodeGenProcModel &ProcModel);
-  void GenSchedClassTables(const CodeGenProcModel &ProcModel,
+  void genSchedClassTables(const CodeGenProcModel &ProcModel,
                            SchedClassTables &SchedTables);
-  void EmitSchedClassTables(SchedClassTables &SchedTables, raw_ostream &OS);
-  void EmitProcessorModels(raw_ostream &OS);
-  void EmitSchedModelHelpers(const std::string &ClassName, raw_ostream &OS);
+  void emitSchedClassTables(SchedClassTables &SchedTables, raw_ostream &OS);
+  void emitProcessorModels(raw_ostream &OS);
+  void emitSchedModelHelpers(const std::string &ClassName, raw_ostream &OS);
   void emitSchedModelHelpersImpl(raw_ostream &OS,
                                  bool OnlyExpandMCInstPredicates = false);
   void emitGenMCSubtargetInfo(raw_ostream &OS);
-  void EmitMCInstrAnalysisPredicateFunctions(raw_ostream &OS);
+  void emitMcInstrAnalysisPredicateFunctions(raw_ostream &OS);
 
-  void EmitSchedModel(raw_ostream &OS);
+  void emitSchedModel(raw_ostream &OS);
   void emitGetMacroFusions(const std::string &ClassName, raw_ostream &OS);
-  void EmitHwModeCheck(const std::string &ClassName, raw_ostream &OS);
-  void ParseFeaturesFunction(raw_ostream &OS);
+  void emitHwModeCheck(const std::string &ClassName, raw_ostream &OS);
+  void parseFeaturesFunction(raw_ostream &OS);
 
 public:
   SubtargetEmitter(const RecordKeeper &R)
       : TGT(R), Records(R), SchedModels(TGT.getSchedModels()),
         Target(TGT.getName()) {}
 
-  void run(raw_ostream &o);
+  void run(raw_ostream &O);
 };
 
 } // end anonymous namespace
@@ -153,7 +153,7 @@ public:
 //
 // Enumeration - Emit the specified class as an enumeration.
 //
-FeatureMapTy SubtargetEmitter::Enumeration(raw_ostream &OS) {
+FeatureMapTy SubtargetEmitter::enumeration(raw_ostream &OS) {
   ArrayRef<const Record *> DefList =
       Records.getAllDerivedDefinitions("SubtargetFeature");
 
@@ -171,15 +171,15 @@ FeatureMapTy SubtargetEmitter::Enumeration(raw_ostream &OS) {
 
   FeatureMapTy FeatureMap;
   // For each record
-  for (unsigned i = 0; i < N; ++i) {
+  for (unsigned I = 0; I < N; ++I) {
     // Next record
-    const Record *Def = DefList[i];
+    const Record *Def = DefList[I];
 
     // Get and emit name
-    OS << "  " << Def->getName() << " = " << i << ",\n";
+    OS << "  " << Def->getName() << " = " << I << ",\n";
 
     // Save the index for this feature.
-    FeatureMap[Def] = i;
+    FeatureMap[Def] = I;
   }
 
   OS << "  "
@@ -201,9 +201,9 @@ static void printFeatureMask(raw_ostream &OS,
   }
 
   OS << "{ { { ";
-  for (unsigned i = 0; i != Mask.size(); ++i) {
+  for (unsigned I = 0; I != Mask.size(); ++I) {
     OS << "0x";
-    OS.write_hex(Mask[i]);
+    OS.write_hex(Mask[I]);
     OS << "ULL, ";
   }
   OS << "} } }";
@@ -211,7 +211,7 @@ static void printFeatureMask(raw_ostream &OS,
 
 /// Emit some information about the SubtargetFeature as calls to a macro so
 /// that they can be used from C++.
-void SubtargetEmitter::EmitSubtargetInfoMacroCalls(raw_ostream &OS) {
+void SubtargetEmitter::emitSubtargetInfoMacroCalls(raw_ostream &OS) {
   OS << "\n#ifdef GET_SUBTARGETINFO_MACRO\n";
 
   std::vector<const Record *> FeatureList =
@@ -252,7 +252,7 @@ void SubtargetEmitter::EmitSubtargetInfoMacroCalls(raw_ostream &OS) {
 // FeatureKeyValues - Emit data of all the subtarget features.  Used by the
 // command line.
 //
-unsigned SubtargetEmitter::FeatureKeyValues(raw_ostream &OS,
+unsigned SubtargetEmitter::featureKeyValues(raw_ostream &OS,
                                             const FeatureMapTy &FeatureMap) {
   std::vector<const Record *> FeatureList =
       Records.getAllDerivedDefinitions("SubtargetFeature");
@@ -301,7 +301,7 @@ unsigned SubtargetEmitter::FeatureKeyValues(raw_ostream &OS,
 // CPUKeyValues - Emit data of all the subtarget processors.  Used by command
 // line.
 //
-unsigned SubtargetEmitter::CPUKeyValues(raw_ostream &OS,
+unsigned SubtargetEmitter::cpuKeyValues(raw_ostream &OS,
                                         const FeatureMapTy &FeatureMap) {
   // Gather and sort processor information
   std::vector<const Record *> ProcessorList =
@@ -349,7 +349,7 @@ unsigned SubtargetEmitter::CPUKeyValues(raw_ostream &OS,
 // data initialization for the specified itinerary.  N is the number
 // of stages.
 //
-void SubtargetEmitter::FormItineraryStageString(const std::string &Name,
+void SubtargetEmitter::formItineraryStageString(const std::string &Name,
                                                 const Record *ItinData,
                                                 std::string &ItinString,
                                                 unsigned &NStages) {
@@ -358,9 +358,9 @@ void SubtargetEmitter::FormItineraryStageString(const std::string &Name,
 
   // For each stage
   unsigned N = NStages = StageList.size();
-  for (unsigned i = 0; i < N;) {
+  for (unsigned I = 0; I < N;) {
     // Next stage
-    const Record *Stage = StageList[i];
+    const Record *Stage = StageList[I];
 
     // Form string as ,{ cycles, u1 | u2 | ... | un, timeinc, kind }
     int Cycles = Stage->getValueAsInt("Cycles");
@@ -370,10 +370,10 @@ void SubtargetEmitter::FormItineraryStageString(const std::string &Name,
     ConstRecVec UnitList = Stage->getValueAsListOfDefs("Units");
 
     // For each unit
-    for (unsigned j = 0, M = UnitList.size(); j < M;) {
+    for (unsigned J = 0, M = UnitList.size(); J < M;) {
       // Add name and bitwise or
-      ItinString += Name + "FU::" + UnitList[j]->getName().str();
-      if (++j < M)
+      ItinString += Name + "FU::" + UnitList[J]->getName().str();
+      if (++J < M)
         ItinString += " | ";
     }
 
@@ -385,7 +385,7 @@ void SubtargetEmitter::FormItineraryStageString(const std::string &Name,
 
     // Close off stage
     ItinString += " }";
-    if (++i < N)
+    if (++I < N)
       ItinString += ", ";
   }
 }
@@ -395,7 +395,7 @@ void SubtargetEmitter::FormItineraryStageString(const std::string &Name,
 // operand cycle initialization for the specified itinerary.  N is the
 // number of operands that has cycles specified.
 //
-void SubtargetEmitter::FormItineraryOperandCycleString(
+void SubtargetEmitter::formItineraryOperandCycleString(
     const Record *ItinData, std::string &ItinString, unsigned &NOperandCycles) {
   // Get operand cycle list
   std::vector<int64_t> OperandCycleList =
@@ -411,19 +411,19 @@ void SubtargetEmitter::FormItineraryOperandCycleString(
   }
 }
 
-void SubtargetEmitter::FormItineraryBypassString(const std::string &Name,
+void SubtargetEmitter::formItineraryBypassString(const std::string &Name,
                                                  const Record *ItinData,
                                                  std::string &ItinString,
                                                  unsigned NOperandCycles) {
   ConstRecVec BypassList = ItinData->getValueAsListOfDefs("Bypasses");
   unsigned N = BypassList.size();
-  unsigned i = 0;
+  unsigned I = 0;
   ListSeparator LS;
-  for (; i < N; ++i) {
+  for (; I < N; ++I) {
     ItinString += LS;
-    ItinString += Name + "Bypass::" + BypassList[i]->getName().str();
+    ItinString += Name + "Bypass::" + BypassList[I]->getName().str();
   }
-  for (; i < NOperandCycles; ++i) {
+  for (; I < NOperandCycles; ++I) {
     ItinString += LS;
     ItinString += " 0";
   }
@@ -434,7 +434,7 @@ void SubtargetEmitter::FormItineraryBypassString(const std::string &Name,
 // cycle tables. Create a list of InstrItinerary objects (ProcItinLists) indexed
 // by CodeGenSchedClass::Index.
 //
-void SubtargetEmitter::EmitStageAndOperandCycleData(
+void SubtargetEmitter::emitStageAndOperandCycleData(
     raw_ostream &OS, std::vector<std::vector<InstrItinerary>> &ProcItinLists) {
   // Multiple processor models may share an itinerary record. Emit it once.
   SmallPtrSet<const Record *, 8> ItinsDefSet;
@@ -453,9 +453,9 @@ void SubtargetEmitter::EmitStageAndOperandCycleData(
     OS << "\n// Functional units for \"" << Name << "\"\n"
        << "namespace " << Name << "FU {\n";
 
-    for (unsigned j = 0, FUN = FUs.size(); j < FUN; ++j)
-      OS << "  const InstrStage::FuncUnits " << FUs[j]->getName()
-         << " = 1ULL << " << j << ";\n";
+    for (unsigned J = 0, FUN = FUs.size(); J < FUN; ++J)
+      OS << "  const InstrStage::FuncUnits " << FUs[J]->getName()
+         << " = 1ULL << " << J << ";\n";
 
     OS << "} // end namespace " << Name << "FU\n";
 
@@ -466,8 +466,8 @@ void SubtargetEmitter::EmitStageAndOperandCycleData(
          << "namespace " << Name << "Bypass {\n";
 
       OS << "  const unsigned NoBypass = 0;\n";
-      for (unsigned j = 0, BPN = BPs.size(); j < BPN; ++j)
-        OS << "  const unsigned " << BPs[j]->getName() << " = 1 << " << j
+      for (unsigned J = 0, BPN = BPs.size(); J < BPN; ++J)
+        OS << "  const unsigned " << BPs[J]->getName() << " = 1 << " << J
            << ";\n";
 
       OS << "} // end namespace " << Name << "Bypass\n";
@@ -518,7 +518,7 @@ void SubtargetEmitter::EmitStageAndOperandCycleData(
       std::string ItinStageString;
       unsigned NStages = 0;
       if (ItinData)
-        FormItineraryStageString(std::string(Name), ItinData, ItinStageString,
+        formItineraryStageString(std::string(Name), ItinData, ItinStageString,
                                  NStages);
 
       // Get string and operand cycle count
@@ -526,10 +526,10 @@ void SubtargetEmitter::EmitStageAndOperandCycleData(
       unsigned NOperandCycles = 0;
       std::string ItinBypassString;
       if (ItinData) {
-        FormItineraryOperandCycleString(ItinData, ItinOperandCycleString,
+        formItineraryOperandCycleString(ItinData, ItinOperandCycleString,
                                         NOperandCycles);
 
-        FormItineraryBypassString(std::string(Name), ItinData, ItinBypassString,
+        formItineraryBypassString(std::string(Name), ItinData, ItinBypassString,
                                   NOperandCycles);
       }
 
@@ -610,7 +610,7 @@ void SubtargetEmitter::EmitStageAndOperandCycleData(
 // Itineraries for each processor. The Itinerary lists are indexed on
 // CodeGenSchedClass::Index.
 //
-void SubtargetEmitter::EmitItineraries(
+void SubtargetEmitter::emitItineraries(
     raw_ostream &OS, std::vector<std::vector<InstrItinerary>> &ProcItinLists) {
   // Multiple processor models may share an itinerary record. Emit it once.
   SmallPtrSet<const Record *, 8> ItinsDefSet;
@@ -642,15 +642,15 @@ void SubtargetEmitter::EmitItineraries(
     OS << ItinsDef->getName() << "[] = {\n";
 
     // For each itinerary class in CodeGenSchedClass::Index order.
-    for (unsigned j = 0, M = ItinList.size(); j < M; ++j) {
-      InstrItinerary &Intinerary = ItinList[j];
+    for (unsigned J = 0, M = ItinList.size(); J < M; ++J) {
+      InstrItinerary &Intinerary = ItinList[J];
 
       // Emit Itinerary in the form of
       // { firstStage, lastStage, firstCycle, lastCycle } // index
       OS << "  { " << Intinerary.NumMicroOps << ", " << Intinerary.FirstStage
          << ", " << Intinerary.LastStage << ", " << Intinerary.FirstOperandCycle
          << ", " << Intinerary.LastOperandCycle << " }"
-         << ", // " << j << " " << SchedModels.getSchedClass(j).Name << "\n";
+         << ", // " << J << " " << SchedModels.getSchedClass(J).Name << "\n";
     }
     // End processor itinerary table
     OS << "  { 0, uint16_t(~0U), uint16_t(~0U), uint16_t(~0U), uint16_t(~0U) }"
@@ -662,7 +662,7 @@ void SubtargetEmitter::EmitItineraries(
 // Emit either the value defined in the TableGen Record, or the default
 // value defined in the C++ header. The Record is null if the processor does not
 // define a model.
-void SubtargetEmitter::EmitProcessorProp(raw_ostream &OS, const Record *R,
+void SubtargetEmitter::emitProcessorProp(raw_ostream &OS, const Record *R,
                                          StringRef Name, char Separator) {
   OS << "  ";
   int V = R ? R->getValueAsInt(Name) : -1;
@@ -673,14 +673,14 @@ void SubtargetEmitter::EmitProcessorProp(raw_ostream &OS, const Record *R,
   OS << '\n';
 }
 
-void SubtargetEmitter::EmitProcessorResourceSubUnits(
+void SubtargetEmitter::emitProcessorResourceSubUnits(
     const CodeGenProcModel &ProcModel, raw_ostream &OS) {
   OS << "\nstatic const unsigned " << ProcModel.ModelName
      << "ProcResourceSubUnits[] = {\n"
      << "  0,  // Invalid\n";
 
-  for (unsigned i = 0, e = ProcModel.ProcResourceDefs.size(); i < e; ++i) {
-    const Record *PRDef = ProcModel.ProcResourceDefs[i];
+  for (unsigned I = 0, E = ProcModel.ProcResourceDefs.size(); I < E; ++I) {
+    const Record *PRDef = ProcModel.ProcResourceDefs[I];
     if (!PRDef->isSubClassOf("ProcResGroup"))
       continue;
     for (const Record *RUDef : PRDef->getValueAsListOfDefs("Resources")) {
@@ -695,7 +695,7 @@ void SubtargetEmitter::EmitProcessorResourceSubUnits(
   OS << "};\n";
 }
 
-static void EmitRetireControlUnitInfo(const CodeGenProcModel &ProcModel,
+static void emitRetireControlUnitInfo(const CodeGenProcModel &ProcModel,
                                       raw_ostream &OS) {
   int64_t ReorderBufferSize = 0, MaxRetirePerCycle = 0;
   if (const Record *RCU = ProcModel.RetireControlUnit) {
@@ -709,7 +709,7 @@ static void EmitRetireControlUnitInfo(const CodeGenProcModel &ProcModel,
   OS << MaxRetirePerCycle << ", // MaxRetirePerCycle\n  ";
 }
 
-static void EmitRegisterFileInfo(const CodeGenProcModel &ProcModel,
+static void emitRegisterFileInfo(const CodeGenProcModel &ProcModel,
                                  unsigned NumRegisterFiles,
                                  unsigned NumCostEntries, raw_ostream &OS) {
   if (NumRegisterFiles)
@@ -726,7 +726,7 @@ static void EmitRegisterFileInfo(const CodeGenProcModel &ProcModel,
 }
 
 unsigned
-SubtargetEmitter::EmitRegisterFileTables(const CodeGenProcModel &ProcModel,
+SubtargetEmitter::emitRegisterFileTables(const CodeGenProcModel &ProcModel,
                                          raw_ostream &OS) {
   if (llvm::all_of(ProcModel.RegisterFiles, [](const CodeGenRegisterFile &RF) {
         return RF.hasDefaultCosts();
@@ -778,7 +778,7 @@ SubtargetEmitter::EmitRegisterFileTables(const CodeGenProcModel &ProcModel,
   return CostTblIndex;
 }
 
-void SubtargetEmitter::EmitLoadStoreQueueInfo(const CodeGenProcModel &ProcModel,
+void SubtargetEmitter::emitLoadStoreQueueInfo(const CodeGenProcModel &ProcModel,
                                               raw_ostream &OS) {
   unsigned QueueID = 0;
   if (ProcModel.LoadQueue) {
@@ -798,33 +798,33 @@ void SubtargetEmitter::EmitLoadStoreQueueInfo(const CodeGenProcModel &ProcModel,
   OS << "  " << QueueID << ", // Resource Descriptor for the Store Queue\n";
 }
 
-void SubtargetEmitter::EmitExtraProcessorInfo(const CodeGenProcModel &ProcModel,
+void SubtargetEmitter::emitExtraProcessorInfo(const CodeGenProcModel &ProcModel,
                                               raw_ostream &OS) {
   // Generate a table of register file descriptors (one entry per each user
   // defined register file), and a table of register costs.
-  unsigned NumCostEntries = EmitRegisterFileTables(ProcModel, OS);
+  unsigned NumCostEntries = emitRegisterFileTables(ProcModel, OS);
 
   // Now generate a table for the extra processor info.
   OS << "\nstatic const llvm::MCExtraProcessorInfo " << ProcModel.ModelName
      << "ExtraInfo = {\n  ";
 
   // Add information related to the retire control unit.
-  EmitRetireControlUnitInfo(ProcModel, OS);
+  emitRetireControlUnitInfo(ProcModel, OS);
 
   // Add information related to the register files (i.e. where to find register
   // file descriptors and register costs).
-  EmitRegisterFileInfo(ProcModel, ProcModel.RegisterFiles.size(),
+  emitRegisterFileInfo(ProcModel, ProcModel.RegisterFiles.size(),
                        NumCostEntries, OS);
 
   // Add information about load/store queues.
-  EmitLoadStoreQueueInfo(ProcModel, OS);
+  emitLoadStoreQueueInfo(ProcModel, OS);
 
   OS << "};\n";
 }
 
-void SubtargetEmitter::EmitProcessorResources(const CodeGenProcModel &ProcModel,
+void SubtargetEmitter::emitProcessorResources(const CodeGenProcModel &ProcModel,
                                               raw_ostream &OS) {
-  EmitProcessorResourceSubUnits(ProcModel, OS);
+  emitProcessorResourceSubUnits(ProcModel, OS);
 
   OS << "\n// {Name, NumUnits, SuperIdx, BufferSize, SubUnitsIdxBegin}\n";
   OS << "static const llvm::MCProcResourceDesc " << ProcModel.ModelName
@@ -833,8 +833,8 @@ void SubtargetEmitter::EmitProcessorResources(const CodeGenProcModel &ProcModel,
      << "  {\"InvalidUnit\", 0, 0, 0, 0},\n";
 
   unsigned SubUnitsOffset = 1;
-  for (unsigned i = 0, e = ProcModel.ProcResourceDefs.size(); i < e; ++i) {
-    const Record *PRDef = ProcModel.ProcResourceDefs[i];
+  for (unsigned I = 0, E = ProcModel.ProcResourceDefs.size(); I < E; ++I) {
+    const Record *PRDef = ProcModel.ProcResourceDefs[I];
 
     const Record *SuperDef = nullptr;
     unsigned SuperIdx = 0;
@@ -866,7 +866,7 @@ void SubtargetEmitter::EmitProcessorResources(const CodeGenProcModel &ProcModel,
     } else {
       OS << "nullptr";
     }
-    OS << "}, // #" << i + 1;
+    OS << "}, // #" << I + 1;
     if (SuperDef)
       OS << ", Super=" << SuperDef->getName();
     OS << "\n";
@@ -877,7 +877,7 @@ void SubtargetEmitter::EmitProcessorResources(const CodeGenProcModel &ProcModel,
 // Find the WriteRes Record that defines processor resources for this
 // SchedWrite.
 const Record *
-SubtargetEmitter::FindWriteResources(const CodeGenSchedRW &SchedWrite,
+SubtargetEmitter::findWriteResources(const CodeGenSchedRW &SchedWrite,
                                      const CodeGenProcModel &ProcModel) {
 
   // Check if the SchedWrite is already subtarget-specific and directly
@@ -938,7 +938,7 @@ SubtargetEmitter::FindWriteResources(const CodeGenSchedRW &SchedWrite,
 /// Find the ReadAdvance record for the given SchedRead on this processor or
 /// return NULL.
 const Record *
-SubtargetEmitter::FindReadAdvance(const CodeGenSchedRW &SchedRead,
+SubtargetEmitter::findReadAdvance(const CodeGenSchedRW &SchedRead,
                                   const CodeGenProcModel &ProcModel) {
   // Check for SchedReads that directly specify a ReadAdvance.
   if (SchedRead.TheDef->isSubClassOf("SchedReadAdvance"))
@@ -997,12 +997,12 @@ SubtargetEmitter::FindReadAdvance(const CodeGenSchedRW &SchedRead,
 
 // Expand an explicit list of processor resources into a full list of implied
 // resource groups and super resources that cover them.
-void SubtargetEmitter::ExpandProcResources(
+void SubtargetEmitter::expandProcResources(
     ConstRecVec &PRVec, std::vector<int64_t> &ReleaseAtCycles,
     std::vector<int64_t> &AcquireAtCycles, const CodeGenProcModel &PM) {
   assert(PRVec.size() == ReleaseAtCycles.size() && "failed precondition");
-  for (unsigned i = 0, e = PRVec.size(); i != e; ++i) {
-    const Record *PRDef = PRVec[i];
+  for (unsigned I = 0, E = PRVec.size(); I != E; ++I) {
+    const Record *PRDef = PRVec[I];
     ConstRecVec SubResources;
     if (PRDef->isSubClassOf("ProcResGroup"))
       SubResources = PRDef->getValueAsListOfDefs("Resources");
@@ -1019,8 +1019,8 @@ void SubtargetEmitter::ExpandProcResources(
         const Record *SuperDef = SchedModels.findProcResUnits(
             SubDef->getValueAsDef("Super"), PM, SubDef->getLoc());
         PRVec.push_back(SuperDef);
-        ReleaseAtCycles.push_back(ReleaseAtCycles[i]);
-        AcquireAtCycles.push_back(AcquireAtCycles[i]);
+        ReleaseAtCycles.push_back(ReleaseAtCycles[I]);
+        AcquireAtCycles.push_back(AcquireAtCycles[I]);
         SubDef = SuperDef;
       }
     }
@@ -1036,8 +1036,8 @@ void SubtargetEmitter::ExpandProcResources(
       }
       if (SubI == SubE) {
         PRVec.push_back(PR);
-        ReleaseAtCycles.push_back(ReleaseAtCycles[i]);
-        AcquireAtCycles.push_back(AcquireAtCycles[i]);
+        ReleaseAtCycles.push_back(ReleaseAtCycles[I]);
+        AcquireAtCycles.push_back(AcquireAtCycles[I]);
       }
     }
   }
@@ -1045,7 +1045,7 @@ void SubtargetEmitter::ExpandProcResources(
 
 // Generate the SchedClass table for this processor and update global
 // tables. Must be called for each processor in order.
-void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
+void SubtargetEmitter::genSchedClassTables(const CodeGenProcModel &ProcModel,
                                            SchedClassTables &SchedTables) {
   std::vector<MCSchedClassDesc> &SCTab =
       SchedTables.ProcSchedClasses.emplace_back();
@@ -1147,7 +1147,7 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
 
       for (unsigned WS : WriteSeq) {
         const Record *WriteRes =
-            FindWriteResources(SchedModels.getSchedWrite(WS), ProcModel);
+            findWriteResources(SchedModels.getSchedWrite(WS), ProcModel);
 
         // Mark the parent class as invalid for unsupported write types.
         if (WriteRes->getValueAsBit("Unsupported")) {
@@ -1209,7 +1209,7 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
 
         assert(AcquireAtCycles.size() == ReleaseAtCycles.size());
 
-        ExpandProcResources(PRVec, ReleaseAtCycles, AcquireAtCycles, ProcModel);
+        expandProcResources(PRVec, ReleaseAtCycles, AcquireAtCycles, ProcModel);
         assert(AcquireAtCycles.size() == ReleaseAtCycles.size());
 
         for (unsigned PRIdx = 0, PREnd = PRVec.size(); PRIdx != PREnd;
@@ -1263,7 +1263,7 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
     for (unsigned UseIdx = 0, EndIdx = Reads.size(); UseIdx != EndIdx;
          ++UseIdx) {
       const Record *ReadAdvance =
-          FindReadAdvance(SchedModels.getSchedRead(Reads[UseIdx]), ProcModel);
+          findReadAdvance(SchedModels.getSchedRead(Reads[UseIdx]), ProcModel);
       if (!ReadAdvance)
         continue;
 
@@ -1323,12 +1323,12 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
         SchedTables.WriteLatencies.begin(), SchedTables.WriteLatencies.end(),
         WriteLatencies.begin(), WriteLatencies.end());
     if (WLPos != SchedTables.WriteLatencies.end()) {
-      unsigned idx = WLPos - SchedTables.WriteLatencies.begin();
-      SCDesc.WriteLatencyIdx = idx;
-      for (unsigned i = 0, e = WriteLatencies.size(); i < e; ++i)
-        if (SchedTables.WriterNames[idx + i].find(WriterNames[i]) ==
+      unsigned Idx = WLPos - SchedTables.WriteLatencies.begin();
+      SCDesc.WriteLatencyIdx = Idx;
+      for (unsigned I = 0, E = WriteLatencies.size(); I < E; ++I)
+        if (SchedTables.WriterNames[Idx + I].find(WriterNames[I]) ==
             std::string::npos) {
-          SchedTables.WriterNames[idx + i] += std::string("_") + WriterNames[i];
+          SchedTables.WriterNames[Idx + I] += std::string("_") + WriterNames[I];
         }
     } else {
       SCDesc.WriteLatencyIdx = SchedTables.WriteLatencies.size();
@@ -1351,7 +1351,7 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
 }
 
 // Emit SchedClass tables for all processors and associated global tables.
-void SubtargetEmitter::EmitSchedClassTables(SchedClassTables &SchedTables,
+void SubtargetEmitter::emitSchedClassTables(SchedClassTables &SchedTables,
                                             raw_ostream &OS) {
   // Emit global WriteProcResTable.
   OS << "\n// {ProcResourceIdx, ReleaseAtCycle, AcquireAtCycle}\n"
@@ -1446,15 +1446,15 @@ void SubtargetEmitter::EmitSchedClassTables(SchedClassTables &SchedTables,
   }
 }
 
-void SubtargetEmitter::EmitProcessorModels(raw_ostream &OS) {
+void SubtargetEmitter::emitProcessorModels(raw_ostream &OS) {
   // For each processor model.
   for (const CodeGenProcModel &PM : SchedModels.procModels()) {
     // Emit extra processor info if available.
     if (PM.hasExtraProcessorInfo())
-      EmitExtraProcessorInfo(PM, OS);
+      emitExtraProcessorInfo(PM, OS);
     // Emit processor resource table.
     if (PM.hasInstrSchedModel())
-      EmitProcessorResources(PM, OS);
+      emitProcessorResources(PM, OS);
     else if (!PM.ProcResourceDefs.empty())
       PrintFatalError(PM.ModelDef->getLoc(),
                       "SchedMachineModel defines "
@@ -1463,12 +1463,12 @@ void SubtargetEmitter::EmitProcessorModels(raw_ostream &OS) {
     // Begin processor itinerary properties
     OS << "\n";
     OS << "static const llvm::MCSchedModel " << PM.ModelName << " = {\n";
-    EmitProcessorProp(OS, PM.ModelDef, "IssueWidth", ',');
-    EmitProcessorProp(OS, PM.ModelDef, "MicroOpBufferSize", ',');
-    EmitProcessorProp(OS, PM.ModelDef, "LoopMicroOpBufferSize", ',');
-    EmitProcessorProp(OS, PM.ModelDef, "LoadLatency", ',');
-    EmitProcessorProp(OS, PM.ModelDef, "HighLatency", ',');
-    EmitProcessorProp(OS, PM.ModelDef, "MispredictPenalty", ',');
+    emitProcessorProp(OS, PM.ModelDef, "IssueWidth", ',');
+    emitProcessorProp(OS, PM.ModelDef, "MicroOpBufferSize", ',');
+    emitProcessorProp(OS, PM.ModelDef, "LoopMicroOpBufferSize", ',');
+    emitProcessorProp(OS, PM.ModelDef, "LoadLatency", ',');
+    emitProcessorProp(OS, PM.ModelDef, "HighLatency", ',');
+    emitProcessorProp(OS, PM.ModelDef, "MispredictPenalty", ',');
 
     bool PostRAScheduler =
         (PM.ModelDef ? PM.ModelDef->getValueAsBit("PostRAScheduler") : false);
@@ -1516,7 +1516,7 @@ void SubtargetEmitter::EmitProcessorModels(raw_ostream &OS) {
 //
 // EmitSchedModel - Emits all scheduling model tables, folding common patterns.
 //
-void SubtargetEmitter::EmitSchedModel(raw_ostream &OS) {
+void SubtargetEmitter::emitSchedModel(raw_ostream &OS) {
   OS << "#ifdef DBGFIELD\n"
      << "#error \"<target>GenSubtargetInfo.inc requires a DBGFIELD macro\"\n"
      << "#endif\n"
@@ -1529,22 +1529,22 @@ void SubtargetEmitter::EmitSchedModel(raw_ostream &OS) {
   if (SchedModels.hasItineraries()) {
     std::vector<std::vector<InstrItinerary>> ProcItinLists;
     // Emit the stage data
-    EmitStageAndOperandCycleData(OS, ProcItinLists);
-    EmitItineraries(OS, ProcItinLists);
+    emitStageAndOperandCycleData(OS, ProcItinLists);
+    emitItineraries(OS, ProcItinLists);
   }
   OS << "\n// ===============================================================\n"
      << "// Data tables for the new per-operand machine model.\n";
 
   SchedClassTables SchedTables;
   for (const CodeGenProcModel &ProcModel : SchedModels.procModels()) {
-    GenSchedClassTables(ProcModel, SchedTables);
+    genSchedClassTables(ProcModel, SchedTables);
   }
-  EmitSchedClassTables(SchedTables, OS);
+  emitSchedClassTables(SchedTables, OS);
 
   OS << "\n#undef DBGFIELD\n";
 
   // Emit the processor machine model
-  EmitProcessorModels(OS);
+  emitProcessorModels(OS);
 }
 
 static void emitPredicateProlog(const RecordKeeper &Records, raw_ostream &OS) {
@@ -1756,7 +1756,7 @@ void SubtargetEmitter::emitSchedModelHelpersImpl(
   emitSchedModelHelperEpilogue(OS, OnlyExpandMCInstPredicates);
 }
 
-void SubtargetEmitter::EmitSchedModelHelpers(const std::string &ClassName,
+void SubtargetEmitter::emitSchedModelHelpers(const std::string &ClassName,
                                              raw_ostream &OS) {
   OS << "unsigned " << ClassName
      << "\n::resolveSchedClass(unsigned SchedClass, const MachineInstr *MI,"
@@ -1786,7 +1786,7 @@ void SubtargetEmitter::EmitSchedModelHelpers(const std::string &ClassName,
     PE.expandSTIPredicate(OS, Fn);
 }
 
-void SubtargetEmitter::EmitHwModeCheck(const std::string &ClassName,
+void SubtargetEmitter::emitHwModeCheck(const std::string &ClassName,
                                        raw_ostream &OS) {
   const CodeGenHwModes &CGH = TGT.getHwModes();
   assert(CGH.getNumModeIds() > 0);
@@ -1825,7 +1825,7 @@ void SubtargetEmitter::EmitHwModeCheck(const std::string &ClassName,
   OS << "  return Modes;\n}\n";
   // End emitting for getHwModeSet().
 
-  auto handlePerMode = [&](std::string ModeType, unsigned ModeInBitSet) {
+  auto HandlePerMode = [&](std::string ModeType, unsigned ModeInBitSet) {
     OS << "  case HwMode_" << ModeType << ":\n"
        << "    Modes &= " << ModeInBitSet << ";\n"
        << "    if (!Modes)\n      return Modes;\n"
@@ -1842,9 +1842,9 @@ void SubtargetEmitter::EmitHwModeCheck(const std::string &ClassName,
   OS << "  if (!Modes)\n    return Modes;\n\n";
   OS << "  switch (type) {\n";
   OS << "  case HwMode_Default:\n    return llvm::countr_zero(Modes) + 1;\n";
-  handlePerMode("ValueType", ValueTypeModes);
-  handlePerMode("RegInfo", RegInfoModes);
-  handlePerMode("EncodingInfo", EncodingInfoModes);
+  HandlePerMode("ValueType", ValueTypeModes);
+  HandlePerMode("RegInfo", RegInfoModes);
+  HandlePerMode("EncodingInfo", EncodingInfoModes);
   OS << "  }\n";
   OS << "  llvm_unreachable(\"unexpected HwModeType\");\n"
      << "  return 0; // should not get here\n}\n";
@@ -1871,7 +1871,7 @@ void SubtargetEmitter::emitGetMacroFusions(const std::string &ClassName,
 
 // Produces a subtarget specific function for parsing
 // the subtarget features string.
-void SubtargetEmitter::ParseFeaturesFunction(raw_ostream &OS) {
+void SubtargetEmitter::parseFeaturesFunction(raw_ostream &OS) {
   ArrayRef<const Record *> Features =
       Records.getAllDerivedDefinitions("SubtargetFeature");
 
@@ -1951,10 +1951,10 @@ void SubtargetEmitter::emitGenMCSubtargetInfo(raw_ostream &OS) {
        << "    return MCSubtargetInfo::isCPUStringValid(CPU);\n"
        << "  }\n";
   OS << "};\n";
-  EmitHwModeCheck(Target + "GenMCSubtargetInfo", OS);
+  emitHwModeCheck(Target + "GenMCSubtargetInfo", OS);
 }
 
-void SubtargetEmitter::EmitMCInstrAnalysisPredicateFunctions(raw_ostream &OS) {
+void SubtargetEmitter::emitMcInstrAnalysisPredicateFunctions(raw_ostream &OS) {
   OS << "\n#ifdef GET_STIPREDICATE_DECLS_FOR_MC_ANALYSIS\n";
   OS << "#undef GET_STIPREDICATE_DECLS_FOR_MC_ANALYSIS\n\n";
 
@@ -1988,18 +1988,18 @@ void SubtargetEmitter::run(raw_ostream &OS) {
   OS << "#undef GET_SUBTARGETINFO_ENUM\n\n";
 
   OS << "namespace llvm {\n";
-  auto FeatureMap = Enumeration(OS);
+  auto FeatureMap = enumeration(OS);
   OS << "} // end namespace llvm\n\n";
   OS << "#endif // GET_SUBTARGETINFO_ENUM\n\n";
 
-  EmitSubtargetInfoMacroCalls(OS);
+  emitSubtargetInfoMacroCalls(OS);
 
   OS << "namespace llvm {\n";
-  unsigned NumFeatures = FeatureKeyValues(OS, FeatureMap);
+  unsigned NumFeatures = featureKeyValues(OS, FeatureMap);
   OS << "\n";
-  EmitSchedModel(OS);
+  emitSchedModel(OS);
   OS << "\n";
-  unsigned NumProcs = CPUKeyValues(OS, FeatureMap);
+  unsigned NumProcs = cpuKeyValues(OS, FeatureMap);
   OS << "\n";
 
   // MCInstrInfo initialization routine.
@@ -2045,7 +2045,7 @@ void SubtargetEmitter::run(raw_ostream &OS) {
   OS << "#include \"llvm/Support/raw_ostream.h\"\n\n";
   if (Target == "AArch64")
     OS << "#include \"llvm/TargetParser/AArch64TargetParser.h\"\n\n";
-  ParseFeaturesFunction(OS);
+  parseFeaturesFunction(OS);
 
   OS << "#endif // GET_SUBTARGETINFO_TARGET_DESC\n\n";
 
@@ -2140,15 +2140,15 @@ void SubtargetEmitter::run(raw_ostream &OS) {
     OS << "nullptr, nullptr, nullptr";
   OS << ") {}\n\n";
 
-  EmitSchedModelHelpers(ClassName, OS);
-  EmitHwModeCheck(ClassName, OS);
+  emitSchedModelHelpers(ClassName, OS);
+  emitHwModeCheck(ClassName, OS);
   emitGetMacroFusions(ClassName, OS);
 
   OS << "} // end namespace llvm\n\n";
 
   OS << "#endif // GET_SUBTARGETINFO_CTOR\n\n";
 
-  EmitMCInstrAnalysisPredicateFunctions(OS);
+  emitMcInstrAnalysisPredicateFunctions(OS);
 }
 
 static TableGen::Emitter::OptClass<SubtargetEmitter>
diff --git a/llvm/utils/TableGen/TableGen.cpp b/llvm/utils/TableGen/TableGen.cpp
index fff4c6b7c27a..bea2a2e735db 100644
--- a/llvm/utils/TableGen/TableGen.cpp
+++ b/llvm/utils/TableGen/TableGen.cpp
@@ -39,17 +39,17 @@ static cl::opt<std::string> Class("class",
                                   cl::value_desc("class name"),
                                   cl::cat(PrintEnumsCat));
 
-static void PrintRecords(const RecordKeeper &Records, raw_ostream &OS) {
+static void printRecords(const RecordKeeper &Records, raw_ostream &OS) {
   OS << Records; // No argument, dump all contents
 }
 
-static void PrintEnums(const RecordKeeper &Records, raw_ostream &OS) {
+static void printEnums(const RecordKeeper &Records, raw_ostream &OS) {
   for (const Record *Rec : Records.getAllDerivedDefinitions(Class))
     OS << Rec->getName() << ", ";
   OS << "\n";
 }
 
-static void PrintSets(const RecordKeeper &Records, raw_ostream &OS) {
+static void printSets(const RecordKeeper &Records, raw_ostream &OS) {
   SetTheory Sets;
   Sets.addFieldExpander("Set", "Elements");
   for (const Record *Rec : Records.getAllDerivedDefinitions("Set")) {
@@ -63,15 +63,15 @@ static void PrintSets(const RecordKeeper &Records, raw_ostream &OS) {
 }
 
 static TableGen::Emitter::Opt X[] = {
-    {"print-records", PrintRecords, "Print all records to stdout (default)",
+    {"print-records", printRecords, "Print all records to stdout (default)",
      true},
     {"print-detailed-records", EmitDetailedRecords,
      "Print full details of all records to stdout"},
     {"null-backend", [](const RecordKeeper &Records, raw_ostream &OS) {},
      "Do nothing after parsing (useful for timing)"},
     {"dump-json", EmitJSON, "Dump all records as machine-readable JSON"},
-    {"print-enums", PrintEnums, "Print enum values for a class"},
-    {"print-sets", PrintSets, "Print expanded sets for testing DAG exprs"},
+    {"print-enums", printEnums, "Print enum values for a class"},
+    {"print-sets", printSets, "Print expanded sets for testing DAG exprs"},
 };
 
 int main(int argc, char **argv) {
diff --git a/llvm/utils/TableGen/VTEmitter.cpp b/llvm/utils/TableGen/VTEmitter.cpp
index 4cbc7abd699d..d02932dd5e7f 100644
--- a/llvm/utils/TableGen/VTEmitter.cpp
+++ b/llvm/utils/TableGen/VTEmitter.cpp
@@ -28,7 +28,7 @@ public:
 
 } // End anonymous namespace.
 
-static void VTtoGetLLVMTyString(raw_ostream &OS, const Record *VT) {
+static void vTtoGetLlvmTyString(raw_ostream &OS, const Record *VT) {
   bool IsVector = VT->getValueAsBit("isVector");
   bool IsRISCVVecTuple = VT->getValueAsBit("isRISCVVecTuple");
 
@@ -207,7 +207,7 @@ void VTEmitter::run(raw_ostream &OS) {
       continue;
 
     OS << "  GET_VT_EVT(" << VT->getValueAsString("LLVMName") << ", ";
-    VTtoGetLLVMTyString(OS, VT);
+    vTtoGetLlvmTyString(OS, VT);
     OS << ")\n";
   }
   OS << "#endif\n\n";
-- 
GitLab


From f22c9ddb36dca84547212e087de3319dcc6bea49 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Tue, 29 Oct 2024 08:17:43 -0700
Subject: [PATCH 482/693] [ORC] Single-symbol convenience method does not need
 to be virtual.

This convenience method just calls the general case which is already virtual.
---
 llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h b/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
index 4004c42d9146..f2ea1f5b64c5 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
@@ -32,8 +32,8 @@ public:
 
   /// Change the redirection destination of given symbol to new destination
   /// symbol.
-  virtual Error redirect(JITDylib &JD, SymbolStringPtr Symbol,
-                         ExecutorSymbolDef NewDest) {
+  Error redirect(JITDylib &JD, SymbolStringPtr Symbol,
+                 ExecutorSymbolDef NewDest) {
     return redirect(JD, {{Symbol, NewDest}});
   }
 
-- 
GitLab


From 9e37cbb469c0ec2fdbf4e3e7b0d9a2938ac30b01 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Tue, 29 Oct 2024 08:36:56 -0700
Subject: [PATCH 483/693] [ORC] Add some missing FIXMEs, move a temporary Error
 into an if condition.

---
 .../Orc/JITLinkRedirectableSymbolManager.cpp          | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.cpp b/llvm/lib/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.cpp
index 4ef217e6c562..81294cad4d7d 100644
--- a/llvm/lib/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.cpp
@@ -52,17 +52,18 @@ void JITLinkRedirectableSymbolManager::emitRedirectableSymbols(
     return;
   }
 
+  // FIXME: return stubs to the pool here too.
   if (auto Err = R->replace(absoluteSymbols(NewSymbolDefs))) {
     ES.reportError(std::move(Err));
     R->failMaterialization();
     return;
   }
 
-  auto Err = R->withResourceKeyDo([&](ResourceKey Key) {
-    TrackedResources[Key].insert(TrackedResources[Key].end(), Symbols.begin(),
-                                 Symbols.end());
-  });
-  if (Err) {
+  // FIXME: return stubs to the pool here too.
+  if (auto Err = R->withResourceKeyDo([&](ResourceKey Key) {
+        TrackedResources[Key].insert(TrackedResources[Key].end(),
+                                     Symbols.begin(), Symbols.end());
+      })) {
     ES.reportError(std::move(Err));
     R->failMaterialization();
     return;
-- 
GitLab


From 8e14c6c172b122203f46a9ad114d51c74535cbb7 Mon Sep 17 00:00:00 2001
From: Kelvin Li <kkwli@users.noreply.github.com>
Date: Tue, 29 Oct 2024 14:20:11 -0400
Subject: [PATCH 484/693] [flang] Support -mabi=vec-extabi and
 -mabi=vec-default on AIX (#113215)

This option is to enable the AIX extended and default vector ABIs.
---
 clang/include/clang/Driver/Options.td        |  4 ++-
 clang/lib/Driver/ToolChains/Flang.cpp        | 31 ++++++++++++++++++++
 clang/lib/Driver/ToolChains/Flang.h          |  7 +++++
 flang/include/flang/Frontend/TargetOptions.h |  3 ++
 flang/lib/Frontend/CompilerInstance.cpp      |  7 +++--
 flang/lib/Frontend/CompilerInvocation.cpp    | 10 +++++++
 flang/test/Driver/mabi.f90                   | 17 +++++++++++
 7 files changed, 76 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/Driver/mabi.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 1ddf488b8bf4..9d595984b63c 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -4678,7 +4678,8 @@ def malign_loops_EQ : Joined<["-"], "malign-loops=">, Group<clang_ignored_m_Grou
 def malign_jumps_EQ : Joined<["-"], "malign-jumps=">, Group<clang_ignored_m_Group>;
 
 let Flags = [TargetSpecific] in {
-def mabi_EQ : Joined<["-"], "mabi=">, Group<m_Group>;
+def mabi_EQ : Joined<["-"], "mabi=">, Group<m_Group>,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>;
 def malign_branch_EQ : CommaJoined<["-"], "malign-branch=">, Group<m_Group>,
   HelpText<"Specify types of branches to align">;
 def malign_branch_boundary_EQ : Joined<["-"], "malign-branch-boundary=">, Group<m_Group>,
@@ -7363,6 +7364,7 @@ def mabi_EQ_ieeelongdouble : Flag<["-"], "mabi=ieeelongdouble">,
   HelpText<"Use IEEE 754 quadruple-precision for long double">,
   MarshallingInfoFlag<LangOpts<"PPCIEEELongDouble">>;
 def mabi_EQ_vec_extabi : Flag<["-"], "mabi=vec-extabi">,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Enable the extended Altivec ABI on AIX. Use volatile and nonvolatile vector registers">,
   MarshallingInfoFlag<LangOpts<"EnableAIXExtendedAltivecABI">>;
 def mfloat_abi : Separate<["-"], "mfloat-abi">,
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index a9d2b7a4dc48..f9d2fdffe3b2 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -203,6 +203,32 @@ void Flang::AddAArch64TargetArgs(const ArgList &Args,
   }
 }
 
+void Flang::AddPPCTargetArgs(const ArgList &Args,
+                             ArgStringList &CmdArgs) const {
+  const Driver &D = getToolChain().getDriver();
+  bool VecExtabi = false;
+
+  if (const Arg *A = Args.getLastArg(options::OPT_mabi_EQ)) {
+    StringRef V = A->getValue();
+    if (V == "vec-extabi")
+      VecExtabi = true;
+    else if (V == "vec-default")
+      VecExtabi = false;
+    else
+      D.Diag(diag::err_drv_unsupported_option_argument)
+          << A->getSpelling() << V;
+  }
+
+  const llvm::Triple &T = getToolChain().getTriple();
+  if (VecExtabi) {
+    if (!T.isOSAIX()) {
+      D.Diag(diag::err_drv_unsupported_opt_for_target)
+          << "-mabi=vec-extabi" << T.str();
+    }
+    CmdArgs.push_back("-mabi=vec-extabi");
+  }
+}
+
 void Flang::AddRISCVTargetArgs(const ArgList &Args,
                                ArgStringList &CmdArgs) const {
   const llvm::Triple &Triple = getToolChain().getTriple();
@@ -383,6 +409,11 @@ void Flang::addTargetOptions(const ArgList &Args,
     getTargetFeatures(D, Triple, Args, CmdArgs, /*ForAs*/ false);
     AddX86_64TargetArgs(Args, CmdArgs);
     break;
+  case llvm::Triple::ppc:
+  case llvm::Triple::ppc64:
+  case llvm::Triple::ppc64le:
+    AddPPCTargetArgs(Args, CmdArgs);
+    break;
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_fveclib)) {
diff --git a/clang/lib/Driver/ToolChains/Flang.h b/clang/lib/Driver/ToolChains/Flang.h
index 9f5e26b86083..4d7d0b8cd9ea 100644
--- a/clang/lib/Driver/ToolChains/Flang.h
+++ b/clang/lib/Driver/ToolChains/Flang.h
@@ -84,6 +84,13 @@ private:
   void AddX86_64TargetArgs(const llvm::opt::ArgList &Args,
                            llvm::opt::ArgStringList &CmdArgs) const;
 
+  /// Add specific options for PPC target.
+  ///
+  /// \param [in] Args The list of input driver arguments
+  /// \param [out] CmdArgs The list of output command arguments
+  void AddPPCTargetArgs(const llvm::opt::ArgList &Args,
+                        llvm::opt::ArgStringList &CmdArgs) const;
+
   /// Extract offload options from the driver arguments and add them to
   /// the command arguments.
   /// \param [in] C The current compilation for the driver invocation
diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 332adcbe6b6a..01c878067b92 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -44,6 +44,9 @@ public:
 
   /// The integer KINDs disabled for this target
   std::vector<int> disabledIntegerKinds;
+
+  /// Extended Altivec ABI on AIX
+  bool EnableAIXExtendedAltivecABI;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/lib/Frontend/CompilerInstance.cpp b/flang/lib/Frontend/CompilerInstance.cpp
index d37430e0e577..35c2ae3c73e6 100644
--- a/flang/lib/Frontend/CompilerInstance.cpp
+++ b/flang/lib/Frontend/CompilerInstance.cpp
@@ -313,7 +313,6 @@ bool CompilerInstance::setUpTargetMachine() {
         << error;
     return false;
   }
-
   // Create `TargetMachine`
   const auto &CGOpts = getInvocation().getCodeGenOpts();
   std::optional<llvm::CodeGenOptLevel> OptLevelOrNone =
@@ -322,9 +321,13 @@ bool CompilerInstance::setUpTargetMachine() {
   llvm::CodeGenOptLevel OptLevel = *OptLevelOrNone;
   std::string featuresStr = getTargetFeatures();
   std::optional<llvm::CodeModel::Model> cm = getCodeModel(CGOpts.CodeModel);
+
+  llvm::TargetOptions tOpts = llvm::TargetOptions();
+  tOpts.EnableAIXExtendedAltivecABI = targetOpts.EnableAIXExtendedAltivecABI;
+
   targetMachine.reset(theTarget->createTargetMachine(
       theTriple, /*CPU=*/targetOpts.cpu,
-      /*Features=*/featuresStr, llvm::TargetOptions(),
+      /*Features=*/featuresStr, /*Options=*/tOpts,
       /*Reloc::Model=*/CGOpts.getRelocationModel(),
       /*CodeModel::Model=*/cm, OptLevel));
   assert(targetMachine && "Failed to create TargetMachine");
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 94d3d1154178..1214a2ea6bf1 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -457,6 +457,16 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
 
   if (args.hasArg(clang::driver::options::OPT_fdisable_integer_16))
     opts.disabledIntegerKinds.push_back(16);
+
+  if (const llvm::opt::Arg *a =
+          args.getLastArg(clang::driver::options::OPT_mabi_EQ)) {
+    llvm::StringRef V = a->getValue();
+    if (V == "vec-extabi") {
+      opts.EnableAIXExtendedAltivecABI = true;
+    } else if (V == "vec-default") {
+      opts.EnableAIXExtendedAltivecABI = false;
+    }
+  }
 }
 // Tweak the frontend configuration based on the frontend action
 static void setUpFrontendBasedOnAction(FrontendOptions &opts) {
diff --git a/flang/test/Driver/mabi.f90 b/flang/test/Driver/mabi.f90
new file mode 100644
index 000000000000..88fd4d2a993f
--- /dev/null
+++ b/flang/test/Driver/mabi.f90
@@ -0,0 +1,17 @@
+! RUN: not %flang -### -c --target=powerpc64le-unknown-linux -mabi=vec-extabi %s 2>&1 | FileCheck --check-prefix=INVALID1 %s
+! RUN: not %flang -### -c --target=x86_64-unknown-linux -mabi=vec-extabi %s 2>&1 | FileCheck --check-prefix=INVALID2 %s
+! RUN: not %flang -### -c --target=powerpc-unknown-aix -mabi=abc %s 2>&1 | FileCheck --check-prefix=INVALID3 %s
+! RUN: %flang -### -c -target powerpc-unknown-aix %s 2>&1 | FileCheck --implicit-check-not=vec-extabi %s
+! RUN: %flang -### -c -target powerpc-unknown-aix -mabi=vec-default %s 2>&1 | FileCheck --implicit-check-not=vec-extabi %s
+! RUN: %flang -### -c -target powerpc-unknown-aix -mabi=vec-extabi %s 2>&1 | FileCheck --check-prefix=EXTABI %s
+
+! REQUIRES: target=powerpc{{.*}}
+
+! INVALID1: error: unsupported option '-mabi=vec-extabi' for target '{{.*}}'
+! INVALID2: error: unsupported option '-mabi=' for target '{{.*}}'
+! INVALID3: error: unsupported argument 'abc' to option '-mabi='
+
+! EXTABI: "-fc1"
+! EXTABI-SAME: "-mabi=vec-extabi"
+
+
-- 
GitLab


From efc6d33be9f4b4d0f0e8d3d5f198f2616b75792b Mon Sep 17 00:00:00 2001
From: Wanyi <kusmour@gmail.com>
Date: Tue, 29 Oct 2024 14:22:51 -0400
Subject: [PATCH 485/693] [lldb] Fix write only file action to truncate the
 file (#112657)

When `FileAction` opens file with write access, it doesn't clear the
file nor append to the end of the file if it already exists. Instead, it
writes from cursor index 0.

For example, by using the settings `target.output-path` and
`target.error-path`, lldb will redirect process stdout/stderr to files.
It then calls this function to write to the files which the above
symptoms appear.

## Test
- Added unit test checking the file flags
- Added 2 api tests checking
  - File content overwritten if the file path already exists
- Stdout and stderr redirection to the same file doesn't change its
behavior
---
 lldb/source/Host/common/FileAction.cpp        |  2 +-
 .../API/commands/settings/TestSettings.py     | 53 +++++++++++++++++++
 .../python_api/process/io/TestProcessIO.py    | 30 +++++++++++
 lldb/unittests/Host/FileActionTest.cpp        | 25 +++++++++
 llvm/docs/ReleaseNotes.md                     |  2 +
 5 files changed, 111 insertions(+), 1 deletion(-)

diff --git a/lldb/source/Host/common/FileAction.cpp b/lldb/source/Host/common/FileAction.cpp
index f980d3224640..e1c3e14a165e 100644
--- a/lldb/source/Host/common/FileAction.cpp
+++ b/lldb/source/Host/common/FileAction.cpp
@@ -41,7 +41,7 @@ bool FileAction::Open(int fd, const FileSpec &file_spec, bool read,
     else if (read)
       m_arg = O_NOCTTY | O_RDONLY;
     else
-      m_arg = O_NOCTTY | O_CREAT | O_WRONLY;
+      m_arg = O_NOCTTY | O_CREAT | O_WRONLY | O_TRUNC;
     m_file_spec = file_spec;
     return true;
   } else {
diff --git a/lldb/test/API/commands/settings/TestSettings.py b/lldb/test/API/commands/settings/TestSettings.py
index 385acceb7a8b..2dd813f6b155 100644
--- a/lldb/test/API/commands/settings/TestSettings.py
+++ b/lldb/test/API/commands/settings/TestSettings.py
@@ -528,6 +528,59 @@ class SettingsCommandTestCase(TestBase):
             output, exe=False, startstr="This message should go to standard out."
         )
 
+    @skipIfDarwinEmbedded  # <rdar://problem/34446098> debugserver on ios etc can't write files
+    def test_same_error_output_path(self):
+        """Test that setting target.error and output-path to the same file path for the launched process works."""
+        self.build()
+
+        exe = self.getBuildArtifact("a.out")
+        self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)
+
+        # Set the error-path and output-path and verify both are set.
+        self.runCmd(
+            "settings set target.error-path '{0}'".format(
+                lldbutil.append_to_process_working_directory(self, "output.txt")
+            )
+        )
+        self.runCmd(
+            "settings set target.output-path '{0}".format(
+                lldbutil.append_to_process_working_directory(self, "output.txt")
+            )
+        )
+        # And add hooks to restore the original settings during tearDown().
+        self.addTearDownHook(lambda: self.runCmd("settings clear target.output-path"))
+        self.addTearDownHook(lambda: self.runCmd("settings clear target.error-path"))
+
+        self.expect(
+            "settings show target.error-path",
+            SETTING_MSG("target.error-path"),
+            substrs=["target.error-path (file)", 'output.txt"'],
+        )
+
+        self.expect(
+            "settings show target.output-path",
+            SETTING_MSG("target.output-path"),
+            substrs=["target.output-path (file)", 'output.txt"'],
+        )
+
+        self.runCmd(
+            "process launch --working-dir '{0}'".format(
+                self.get_process_working_directory()
+            ),
+            RUN_SUCCEEDED,
+        )
+
+        output = lldbutil.read_file_from_process_wd(self, "output.txt")
+        err_message = "This message should go to standard error."
+        out_message = "This message should go to standard out."
+        # Error msg should get flushed by the output msg
+        self.expect(output, exe=False, substrs=[out_message])
+        self.assertNotIn(
+            err_message,
+            output,
+            "Race condition when both stderr/stdout redirects to the same file",
+        )
+
     def test_print_dictionary_setting(self):
         self.runCmd("settings clear target.env-vars")
         self.runCmd('settings set target.env-vars ["MY_VAR"]=some-value')
diff --git a/lldb/test/API/python_api/process/io/TestProcessIO.py b/lldb/test/API/python_api/process/io/TestProcessIO.py
index 5bb91d275831..3b5c7c48c51f 100644
--- a/lldb/test/API/python_api/process/io/TestProcessIO.py
+++ b/lldb/test/API/python_api/process/io/TestProcessIO.py
@@ -95,6 +95,36 @@ class ProcessIOTestCase(TestBase):
         error = self.read_error_file_and_delete()
         self.check_process_output(output, error)
 
+    @skipIfWindows  # stdio manipulation unsupported on Windows
+    @expectedFlakeyLinux(bugnumber="llvm.org/pr26437")
+    @skipIfDarwinEmbedded  # debugserver can't create/write files on the device
+    def test_stdout_stderr_redirection_to_existing_files(self):
+        """Exercise SBLaunchInfo::AddOpenFileAction() for STDOUT and STDERR without redirecting STDIN to output files already exist."""
+        self.setup_test()
+        self.build()
+        self.create_target()
+        self.write_file_with_placeholder(self.output_file)
+        self.write_file_with_placeholder(self.error_file)
+        self.redirect_stdout()
+        self.redirect_stderr()
+        self.run_process(True)
+        output = self.read_output_file_and_delete()
+        error = self.read_error_file_and_delete()
+        self.check_process_output(output, error)
+
+    def write_file_with_placeholder(self, target_file):
+        placeholder = "This content should be overwritten."
+        if lldb.remote_platform:
+            self.runCmd(
+                'platform file write "{target}" -d "{data}"'.format(
+                    target=target_file, data=placeholder
+                )
+            )
+        else:
+            f = open(target_file, "w")
+            f.write(placeholder)
+            f.close()
+
     # target_file - path on local file system or remote file system if running remote
     # local_file - path on local system
     def read_file_and_delete(self, target_file, local_file):
diff --git a/lldb/unittests/Host/FileActionTest.cpp b/lldb/unittests/Host/FileActionTest.cpp
index b208169aac20..3d2c722552c9 100644
--- a/lldb/unittests/Host/FileActionTest.cpp
+++ b/lldb/unittests/Host/FileActionTest.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <fcntl.h>
+
 #include "lldb/Host/FileAction.h"
 #include "gtest/gtest.h"
 
@@ -17,3 +19,26 @@ TEST(FileActionTest, Open) {
   EXPECT_EQ(Action.GetAction(), FileAction::eFileActionOpen);
   EXPECT_EQ(Action.GetFileSpec(), FileSpec("/tmp"));
 }
+
+TEST(FileActionTest, OpenReadWrite) {
+  FileAction Action;
+  Action.Open(48, FileSpec("/tmp_0"), /*read*/ true, /*write*/ true);
+  EXPECT_TRUE(Action.GetActionArgument() & (O_NOCTTY | O_CREAT | O_RDWR));
+  EXPECT_FALSE(Action.GetActionArgument() & O_RDONLY);
+  EXPECT_FALSE(Action.GetActionArgument() & O_WRONLY);
+}
+
+TEST(FileActionTest, OpenReadOnly) {
+  FileAction Action;
+  Action.Open(49, FileSpec("/tmp_1"), /*read*/ true, /*write*/ false);
+  EXPECT_TRUE(Action.GetActionArgument() & (O_NOCTTY | O_RDONLY));
+  EXPECT_FALSE(Action.GetActionArgument() & O_WRONLY);
+}
+
+TEST(FileActionTest, OpenWriteOnly) {
+  FileAction Action;
+  Action.Open(50, FileSpec("/tmp_2"), /*read*/ false, /*write*/ true);
+  EXPECT_TRUE(Action.GetActionArgument() &
+              (O_NOCTTY | O_CREAT | O_WRONLY | O_TRUNC));
+  EXPECT_FALSE(Action.GetActionArgument() & O_RDONLY);
+}
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 92a45d845f1d..d5c650e74eeb 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -301,6 +301,8 @@ Changes to LLDB
 * LLDB can now read the `fpmr` register from AArch64 Linux processes and core
   files.
 
+* Program stdout/stderr redirection will now open the file with O_TRUNC flag, make sure to truncate the file if path already exists.
+  * eg. `settings set target.output-path/target.error-path <path/to/file>`
 
 Changes to BOLT
 ---------------------------------
-- 
GitLab


From b4e1af0096fd05ed4bddf11b48b604d75a7103d0 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Tue, 29 Oct 2024 18:40:06 +0000
Subject: [PATCH 486/693] [lldb-dap] Always pass disableASLR to the DAP
 executable (#113891)

More context can be found in
https://github.com/llvm/llvm-project/pull/110303

For DAP tests running in constrained environments (e.g., Docker
containers), disabling ASLR isn't allowed. So we set `disableASLR=False`
(since https://github.com/llvm/llvm-project/pull/113593).

However, the `dap_server.py` will currently only forward the value
of `disableASLR` to the DAP executable if it's set to `True`. If the
DAP executable wasn't provided a `disableASLR` field it defaults to
`true` too:
https://github.com/llvm/llvm-project/blob/f14743794587db102c6d1b20f9c87a1ac20decfd/lldb/tools/lldb-dap/lldb-dap.cpp#L2103-L2104

This means that passing `disableASLR=False` from the tests is currently
not possible.

This is also true for many of the other boolean arguments of
`request_launch`. But this patch only addresses `disableASLR` for now
since it's blocking a libc++ patch.
---
 .../Python/lldbsuite/test/tools/lldb-dap/dap_server.py         | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
index 63748a71f112..c29992ce9c78 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
@@ -793,8 +793,6 @@ class DebugCommunication(object):
             args_dict["env"] = env
         if stopOnEntry:
             args_dict["stopOnEntry"] = stopOnEntry
-        if disableASLR:
-            args_dict["disableASLR"] = disableASLR
         if disableSTDIO:
             args_dict["disableSTDIO"] = disableSTDIO
         if shellExpandArguments:
@@ -829,6 +827,7 @@ class DebugCommunication(object):
         if customThreadFormat:
             args_dict["customThreadFormat"] = customThreadFormat
 
+        args_dict["disableASLR"] = disableASLR
         args_dict["enableAutoVariableSummaries"] = enableAutoVariableSummaries
         args_dict["enableSyntheticChildDebugging"] = enableSyntheticChildDebugging
         args_dict["displayExtendedBacktrace"] = displayExtendedBacktrace
-- 
GitLab


From b9978f8c7792a8bfdbef8912b3db7617bc5fddff Mon Sep 17 00:00:00 2001
From: Renaud Kauffmann <rkauffmann@nvidia.com>
Date: Tue, 29 Oct 2024 11:48:48 -0700
Subject: [PATCH 487/693] [flang][cuda] Adding variable registration in
 constructor (#113976)

1) Adding variable registration in constructor
2) Applying feedback from PR
https://github.com/llvm/llvm-project/pull/112989
---
 .../Transforms/CUFAddConstructor.cpp          | 72 +++++++++++++++++--
 .../Optimizer/Transforms/CUFOpConversion.cpp  |  2 +-
 flang/test/Fir/CUDA/cuda-constructor-2.f90    | 22 ++++++
 3 files changed, 91 insertions(+), 5 deletions(-)
 create mode 100644 flang/test/Fir/CUDA/cuda-constructor-2.f90

diff --git a/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp b/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp
index 4da06be8ef7d..7cdb2f7ffe27 100644
--- a/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp
@@ -6,15 +6,23 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "flang/Optimizer/Builder/BoxValue.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Builder/Runtime/RTBuilder.h"
+#include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/CodeGen/Target.h"
 #include "flang/Optimizer/Dialect/CUF/CUFOps.h"
 #include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIROpsSupport.h"
+#include "flang/Optimizer/Support/DataLayout.h"
 #include "flang/Optimizer/Transforms/CUFCommon.h"
+#include "flang/Runtime/CUDA/registration.h"
 #include "flang/Runtime/entry-names.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/IR/Value.h"
 #include "mlir/Pass/Pass.h"
 #include "llvm/ADT/SmallVector.h"
 
@@ -23,6 +31,8 @@ namespace fir {
 #include "flang/Optimizer/Transforms/Passes.h.inc"
 } // namespace fir
 
+using namespace Fortran::runtime::cuda;
+
 namespace {
 
 static constexpr llvm::StringRef cudaFortranCtorName{
@@ -34,13 +44,23 @@ struct CUFAddConstructor
   void runOnOperation() override {
     mlir::ModuleOp mod = getOperation();
     mlir::SymbolTable symTab(mod);
-    mlir::OpBuilder builder{mod.getBodyRegion()};
+    mlir::OpBuilder opBuilder{mod.getBodyRegion()};
+    fir::FirOpBuilder builder(opBuilder, mod);
+    fir::KindMapping kindMap{fir::getKindMapping(mod)};
     builder.setInsertionPointToEnd(mod.getBody());
     mlir::Location loc = mod.getLoc();
     auto *ctx = mod.getContext();
     auto voidTy = mlir::LLVM::LLVMVoidType::get(ctx);
+    auto idxTy = builder.getIndexType();
     auto funcTy =
         mlir::LLVM::LLVMFunctionType::get(voidTy, {}, /*isVarArg=*/false);
+    std::optional<mlir::DataLayout> dl =
+        fir::support::getOrSetDataLayout(mod, /*allowDefaultLayout=*/false);
+    if (!dl) {
+      mlir::emitError(mod.getLoc(),
+                      "data layout attribute is required to perform " +
+                          getName() + "pass");
+    }
 
     // Symbol reference to CUFRegisterAllocator.
     builder.setInsertionPointToEnd(mod.getBody());
@@ -58,12 +78,13 @@ struct CUFAddConstructor
     builder.setInsertionPointToStart(func.addEntryBlock(builder));
     builder.create<mlir::LLVM::CallOp>(loc, funcTy, cufRegisterAllocatorRef);
 
-    // Register kernels
     auto gpuMod = symTab.lookup<mlir::gpu::GPUModuleOp>(cudaDeviceModuleName);
     if (gpuMod) {
       auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(ctx);
       auto registeredMod = builder.create<cuf::RegisterModuleOp>(
           loc, llvmPtrTy, mlir::SymbolRefAttr::get(ctx, gpuMod.getName()));
+
+      // Register kernels
       for (auto func : gpuMod.getOps<mlir::gpu::GPUFuncOp>()) {
         if (func.isKernel()) {
           auto kernelName = mlir::SymbolRefAttr::get(
@@ -72,12 +93,55 @@ struct CUFAddConstructor
           builder.create<cuf::RegisterKernelOp>(loc, kernelName, registeredMod);
         }
       }
+
+      // Register variables
+      for (fir::GlobalOp globalOp : mod.getOps<fir::GlobalOp>()) {
+        auto attr = globalOp.getDataAttrAttr();
+        if (!attr)
+          continue;
+
+        mlir::func::FuncOp func;
+        switch (attr.getValue()) {
+        case cuf::DataAttribute::Device:
+        case cuf::DataAttribute::Constant: {
+          func = fir::runtime::getRuntimeFunc<mkRTKey(CUFRegisterVariable)>(
+              loc, builder);
+          auto fTy = func.getFunctionType();
+
+          // Global variable name
+          std::string gblNameStr = globalOp.getSymbol().getValue().str();
+          gblNameStr += '\0';
+          mlir::Value gblName = fir::getBase(
+              fir::factory::createStringLiteral(builder, loc, gblNameStr));
+
+          // Global variable size
+          auto sizeAndAlign = fir::getTypeSizeAndAlignmentOrCrash(
+              loc, globalOp.getType(), *dl, kindMap);
+          auto size =
+              builder.createIntegerConstant(loc, idxTy, sizeAndAlign.first);
+
+          // Global variable address
+          mlir::Value addr = builder.create<fir::AddrOfOp>(
+              loc, globalOp.resultType(), globalOp.getSymbol());
+
+          llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
+              builder, loc, fTy, registeredMod, addr, gblName, size)};
+          builder.create<fir::CallOp>(loc, func, args);
+        } break;
+        case cuf::DataAttribute::Managed:
+          TODO(loc, "registration of managed variables");
+        default:
+          break;
+        }
+        if (!func)
+          continue;
+      }
     }
     builder.create<mlir::LLVM::ReturnOp>(loc, mlir::ValueRange{});
 
     // Create the llvm.global_ctor with the function.
-    // TODO: We might want to have a utility that retrieve it if already created
-    // and adds new functions.
+    // TODO: We might want to have a utility that retrieve it if already
+    // created and adds new functions.
     builder.setInsertionPointToEnd(mod.getBody());
     llvm::SmallVector<mlir::Attribute> funcs;
     funcs.push_back(
diff --git a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
index 9c2b882c7f46..14cc1cb508cf 100644
--- a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
@@ -111,7 +111,7 @@ mlir::Value getDeviceAddress(mlir::PatternRewriter &rewriter,
     switch (attr.getValue()) {
     case cuf::DataAttribute::Device:
     case cuf::DataAttribute::Managed:
-    case cuf::DataAttribute::Pinned:
+    case cuf::DataAttribute::Constant:
       isDevGlobal = true;
       break;
     default:
diff --git a/flang/test/Fir/CUDA/cuda-constructor-2.f90 b/flang/test/Fir/CUDA/cuda-constructor-2.f90
new file mode 100644
index 000000000000..378dabbb7c7e
--- /dev/null
+++ b/flang/test/Fir/CUDA/cuda-constructor-2.f90
@@ -0,0 +1,22 @@
+// RUN: fir-opt --split-input-file --cuf-add-constructor %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 20.0.0 (https://github.com/llvm/llvm-project.git cae351f3453a0a26ec8eb2ddaf773c24a29d929e)", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
+
+  fir.global @_QMmtestsEn(dense<[3, 4, 5, 6, 7]> : tensor<5xi32>) {data_attr = #cuf.cuda<device>} : !fir.array<5xi32>
+
+  gpu.module @cuda_device_mod [#nvvm.target] {
+  }
+}
+
+// CHECK: gpu.module @cuda_device_mod [#nvvm.target] 
+
+// CHECK: llvm.func internal @__cudaFortranConstructor() {
+// CHECK-DAG: %[[MODULE:.*]] = cuf.register_module @cuda_device_mod -> !llvm.ptr
+// CHECK-DAG: %[[VAR_NAME:.*]] = fir.address_of(@_QQ{{.*}}) : !fir.ref<!fir.char<1,12>>
+// CHECK-DAG: %[[VAR_ADDR:.*]] = fir.address_of(@_QMmtestsEn) : !fir.ref<!fir.array<5xi32>>
+// CHECK-DAG: %[[MODULE2:.*]] = fir.convert %[[MODULE]] : (!llvm.ptr) -> !fir.ref<!fir.llvm_ptr<i8>>
+// CHECK-DAG: %[[VAR_ADDR2:.*]] = fir.convert %[[VAR_ADDR]] : (!fir.ref<!fir.array<5xi32>>) -> !fir.ref<i8>
+// CHECK-DAG: %[[VAR_NAME2:.*]] = fir.convert %[[VAR_NAME]] : (!fir.ref<!fir.char<1,12>>) -> !fir.ref<i8>
+// CHECK-DAG: %[[CST:.*]] = arith.constant 20 : index
+// CHECK-DAG %[[CST2:.*]] = fir.convert %[[CST]] : (index) -> i64
+// CHECK fir.call @_FortranACUFRegisterVariable(%[[MODULE2]], %[[VAR_ADDR2]], %[[VAR_NAME2]], %[[CST2]]) : (!fir.ref<!fir.llvm_ptr<i8>>, !fir.ref<i8>, !fir.ref<i8>, i64) -> none
-- 
GitLab


From c79827cd15ad31b77702e63e5050c1a8b0b44825 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 29 Oct 2024 12:05:18 -0700
Subject: [PATCH 488/693] [SandboxIR] Fix a warning

This patch fixes:

  llvm/lib/SandboxIR/Context.cpp:684:22: error: unused variable
  'MaxRegisteredCallbacks' [-Werror,-Wunused-const-variable]
---
 llvm/lib/SandboxIR/Context.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/SandboxIR/Context.cpp b/llvm/lib/SandboxIR/Context.cpp
index 5e5cbbbc4515..b86ed5864c1a 100644
--- a/llvm/lib/SandboxIR/Context.cpp
+++ b/llvm/lib/SandboxIR/Context.cpp
@@ -681,7 +681,7 @@ void Context::runMoveInstrCallbacks(Instruction *I, const BBIterator &WhereIt) {
 // An arbitrary limit, to check for accidental misuse. We expect a small number
 // of callbacks to be registered at a time, but we can increase this number if
 // we discover we needed more.
-static constexpr int MaxRegisteredCallbacks = 16;
+[[maybe_unused]] static constexpr int MaxRegisteredCallbacks = 16;
 
 Context::CallbackID Context::registerEraseInstrCallback(EraseInstrCallback CB) {
   assert(EraseInstrCallbacks.size() <= MaxRegisteredCallbacks &&
-- 
GitLab


From 9cc5a4bf667ffcd2765a6a00a311fb4ec8559b37 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellis.sparky.hoag@gmail.com>
Date: Tue, 29 Oct 2024 12:23:47 -0700
Subject: [PATCH 489/693] Remove llvm::shouldOptForSize() from Utils.h
 (#112630)

Remove `llvm::shouldOptForSize()` from `Utils.h` since we can use
`llvm::shouldOptimizeForSize()` from `SizeOpts.h` instead.

Depends on https://github.com/llvm/llvm-project/pull/112626
---
 .../llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h       | 9 +++++++--
 llvm/include/llvm/CodeGen/GlobalISel/Utils.h             | 4 ----
 llvm/lib/CodeGen/GlobalISel/Utils.cpp                    | 5 -----
 3 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
index 7b42722ca8d4..b4ff4cd178d7 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
@@ -24,6 +24,7 @@
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGenTypes/LowLevelType.h"
 #include "llvm/IR/Function.h"
+#include "llvm/Transforms/Utils/SizeOpts.h"
 #include <bitset>
 #include <cstddef>
 #include <cstdint>
@@ -635,8 +636,12 @@ protected:
 
   bool shouldOptForSize(const MachineFunction *MF) const {
     const auto &F = MF->getFunction();
-    return F.hasOptSize() || F.hasMinSize() ||
-           (PSI && BFI && CurMBB && llvm::shouldOptForSize(*CurMBB, PSI, BFI));
+    if (F.hasOptSize())
+      return true;
+    if (CurMBB)
+      if (auto *BB = CurMBB->getBasicBlock())
+        return llvm::shouldOptimizeForSize(BB, PSI, BFI);
+    return false;
   }
 
 public:
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index 95a8234d3c60..4016247376c4 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -542,10 +542,6 @@ bool isConstFalseVal(const TargetLowering &TLI, int64_t Val, bool IsVector,
 /// TargetBooleanContents.
 int64_t getICmpTrueVal(const TargetLowering &TLI, bool IsVector, bool IsFP);
 
-/// Returns true if the given block should be optimized for size.
-bool shouldOptForSize(const MachineBasicBlock &MBB, ProfileSummaryInfo *PSI,
-                      BlockFrequencyInfo *BFI);
-
 using SmallInstListTy = GISelWorkList<4>;
 void saveUsesAndErase(MachineInstr &MI, MachineRegisterInfo &MRI,
                       LostDebugLocObserver *LocObserver,
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 513a49b4fc2e..dcbbb0871a84 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -1619,11 +1619,6 @@ int64_t llvm::getICmpTrueVal(const TargetLowering &TLI, bool IsVector,
   llvm_unreachable("Invalid boolean contents");
 }
 
-bool llvm::shouldOptForSize(const MachineBasicBlock &MBB,
-                            ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) {
-  return llvm::shouldOptimizeForSize(MBB.getBasicBlock(), PSI, BFI);
-}
-
 void llvm::saveUsesAndErase(MachineInstr &MI, MachineRegisterInfo &MRI,
                             LostDebugLocObserver *LocObserver,
                             SmallInstListTy &DeadInstChain) {
-- 
GitLab


From a18af41c20ac9ca22e3c95da3d71475f9f6c31b5 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Tue, 29 Oct 2024 12:26:33 -0700
Subject: [PATCH 490/693] [LLVM] Change error messages to start with lower case
 (#113748)

Change LLVM Asm and TableGen Lexer/Parser error messages to begin with
lower case.
---
 llvm/lib/AsmParser/LLLexer.cpp                |  24 +++---
 llvm/lib/TableGen/TGLexer.cpp                 |  73 +++++++++---------
 llvm/test/Assembler/invalid-inttype.ll        |   2 +-
 llvm/test/Assembler/invalid-name.ll           | Bin 207 -> 209 bytes
 llvm/test/Assembler/invalid-name2.ll          | Bin 185 -> 187 bytes
 llvm/test/TableGen/64-bit-int.td              |   2 +-
 .../invalid-macro-name-command-line.td        |   6 +-
 llvm/test/TableGen/prep-diag1.td              |   8 +-
 llvm/test/TableGen/prep-diag10.td             |   4 +-
 llvm/test/TableGen/prep-diag11.td             |   4 +-
 llvm/test/TableGen/prep-diag12.td             |   4 +-
 llvm/test/TableGen/prep-diag13.td             |   4 +-
 llvm/test/TableGen/prep-diag14.td             |   4 +-
 llvm/test/TableGen/prep-diag2.td              |   4 +-
 llvm/test/TableGen/prep-diag3.td              |   4 +-
 llvm/test/TableGen/prep-diag4.td              |   2 +-
 llvm/test/TableGen/prep-diag6.td              |   2 +-
 llvm/test/TableGen/prep-diag8.td              |   2 +-
 llvm/test/TableGen/prep-diag9.td              |   4 +-
 llvm/test/TableGen/prep-ifndef-diag-1.td      |   2 +-
 llvm/test/TableGen/prep-ifndef-diag-2.td      |   2 +-
 llvm/test/TableGen/unterminated-c-comment.td  |   2 +-
 llvm/test/TableGen/unterminated-code-block.td |   2 +-
 23 files changed, 81 insertions(+), 80 deletions(-)

diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index 759db6db6077..56abd03d6235 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -60,8 +60,8 @@ uint64_t LLLexer::atoull(const char *Buffer, const char *End) {
     uint64_t OldRes = Result;
     Result *= 10;
     Result += *Buffer-'0';
-    if (Result < OldRes) {  // Uh, oh, overflow detected!!!
-      LexError("constant bigger than 64 bits detected!");
+    if (Result < OldRes) { // overflow detected.
+      LexError("constant bigger than 64 bits detected");
       return 0;
     }
   }
@@ -75,8 +75,8 @@ uint64_t LLLexer::HexIntToVal(const char *Buffer, const char *End) {
     Result *= 16;
     Result += hexDigitValue(*Buffer);
 
-    if (Result < OldRes) {   // Uh, oh, overflow detected!!!
-      LexError("constant bigger than 64 bits detected!");
+    if (Result < OldRes) { // overflow detected.
+      LexError("constant bigger than 64 bits detected");
       return 0;
     }
   }
@@ -99,7 +99,7 @@ void LLLexer::HexToIntPair(const char *Buffer, const char *End,
     Pair[1] += hexDigitValue(*Buffer);
   }
   if (Buffer != End)
-    LexError("constant bigger than 128 bits detected!");
+    LexError("constant bigger than 128 bits detected");
 }
 
 /// FP80HexToIntPair - translate an 80 bit FP80 number (20 hexits) into
@@ -118,7 +118,7 @@ void LLLexer::FP80HexToIntPair(const char *Buffer, const char *End,
     Pair[0] += hexDigitValue(*Buffer);
   }
   if (Buffer != End)
-    LexError("constant bigger than 128 bits detected!");
+    LexError("constant bigger than 128 bits detected");
 }
 
 // UnEscapeLexed - Run through the specified buffer and change \xx codes to the
@@ -292,7 +292,7 @@ lltok::Kind LLLexer::LexDollar() {
         StrVal.assign(TokStart + 2, CurPtr - 1);
         UnEscapeLexed(StrVal);
         if (StringRef(StrVal).contains(0)) {
-          LexError("Null bytes are not allowed in names");
+          LexError("NUL character is not allowed in names");
           return lltok::Error;
         }
         return lltok::ComdatVar;
@@ -354,7 +354,7 @@ lltok::Kind LLLexer::LexUIntID(lltok::Kind Token) {
 
   uint64_t Val = atoull(TokStart + 1, CurPtr);
   if ((unsigned)Val != Val)
-    LexError("invalid value number (too large)!");
+    LexError("invalid value number (too large)");
   UIntVal = unsigned(Val);
   return Token;
 }
@@ -375,7 +375,7 @@ lltok::Kind LLLexer::LexVar(lltok::Kind Var, lltok::Kind VarID) {
         StrVal.assign(TokStart+2, CurPtr-1);
         UnEscapeLexed(StrVal);
         if (StringRef(StrVal).contains(0)) {
-          LexError("Null bytes are not allowed in names");
+          LexError("NUL character is not allowed in names");
           return lltok::Error;
         }
         return Var;
@@ -410,7 +410,7 @@ lltok::Kind LLLexer::LexQuote() {
   if (CurPtr[0] == ':') {
     ++CurPtr;
     if (StringRef(StrVal).contains(0)) {
-      LexError("Null bytes are not allowed in names");
+      LexError("NUL character is not allowed in names");
       kind = lltok::Error;
     } else {
       kind = lltok::LabelStr;
@@ -492,7 +492,7 @@ lltok::Kind LLLexer::LexIdentifier() {
     uint64_t NumBits = atoull(StartChar, CurPtr);
     if (NumBits < IntegerType::MIN_INT_BITS ||
         NumBits > IntegerType::MAX_INT_BITS) {
-      LexError("bitwidth for integer type out of range!");
+      LexError("bitwidth for integer type out of range");
       return lltok::Error;
     }
     TyVal = IntegerType::get(Context, NumBits);
@@ -1122,7 +1122,7 @@ lltok::Kind LLLexer::LexDigitOrNegative() {
     uint64_t Val = atoull(TokStart, CurPtr);
     ++CurPtr; // Skip the colon.
     if ((unsigned)Val != Val)
-      LexError("invalid value number (too large)!");
+      LexError("invalid value number (too large)");
     UIntVal = unsigned(Val);
     return lltok::LabelID;
   }
diff --git a/llvm/lib/TableGen/TGLexer.cpp b/llvm/lib/TableGen/TGLexer.cpp
index 8fe7f69ecf8e..1e93b2c160ba 100644
--- a/llvm/lib/TableGen/TGLexer.cpp
+++ b/llvm/lib/TableGen/TGLexer.cpp
@@ -89,7 +89,7 @@ TGLexer::TGLexer(SourceMgr &SM, ArrayRef<std::string> Macros) : SrcMgr(SM) {
   for (StringRef MacroName : Macros) {
     const char *End = lexMacroName(MacroName);
     if (End != MacroName.end())
-      PrintFatalError("Invalid macro name `" + MacroName +
+      PrintFatalError("invalid macro name `" + MacroName +
                       "` specified on command line");
 
     DefinedMacros.insert(MacroName);
@@ -188,7 +188,7 @@ tgtok::TokKind TGLexer::LexToken(bool FileOrLineStart) {
       return LexIdentifier();
 
     // Unknown character, emit an error.
-    return ReturnError(TokStart, "Unexpected character");
+    return ReturnError(TokStart, "unexpected character");
   case EOF:
     // Lex next token, if we just left an include file.
     // Note that leaving an include file means that the next
@@ -231,7 +231,7 @@ tgtok::TokKind TGLexer::LexToken(bool FileOrLineStart) {
         ++CurPtr; // Eat third dot.
         return tgtok::dotdotdot;
       }
-      return ReturnError(TokStart, "Invalid '..' punctuation");
+      return ReturnError(TokStart, "invalid '..' punctuation");
     }
     return tgtok::dot;
 
@@ -255,7 +255,7 @@ tgtok::TokKind TGLexer::LexToken(bool FileOrLineStart) {
       if (SkipCComment())
         return tgtok::Error;
     } else // Otherwise, this is an error.
-      return ReturnError(TokStart, "Unexpected character");
+      return ReturnError(TokStart, "unexpected character");
     return LexToken(FileOrLineStart);
   case '-': case '+':
   case '0': case '1': case '2': case '3': case '4': case '5': case '6':
@@ -313,10 +313,10 @@ tgtok::TokKind TGLexer::LexString() {
   while (*CurPtr != '"') {
     // If we hit the end of the buffer, report an error.
     if (*CurPtr == 0 && CurPtr == CurBuf.end())
-      return ReturnError(StrStart, "End of file in string literal");
+      return ReturnError(StrStart, "end of file in string literal");
 
     if (*CurPtr == '\n' || *CurPtr == '\r')
-      return ReturnError(StrStart, "End of line in string literal");
+      return ReturnError(StrStart, "end of line in string literal");
 
     if (*CurPtr != '\\') {
       CurStrVal += *CurPtr++;
@@ -346,7 +346,7 @@ tgtok::TokKind TGLexer::LexString() {
     // If we hit the end of the buffer, report an error.
     case '\0':
       if (CurPtr == CurBuf.end())
-        return ReturnError(StrStart, "End of file in string literal");
+        return ReturnError(StrStart, "end of file in string literal");
       [[fallthrough]];
     default:
       return ReturnError(CurPtr, "invalid escape in string literal");
@@ -359,7 +359,7 @@ tgtok::TokKind TGLexer::LexString() {
 
 tgtok::TokKind TGLexer::LexVarName() {
   if (!isValidIDChar(CurPtr[0], /*First=*/true))
-    return ReturnError(TokStart, "Invalid variable name");
+    return ReturnError(TokStart, "invalid variable name");
 
   // Otherwise, we're ok, consume the rest of the characters.
   const char *VarNameStart = CurPtr++;
@@ -433,7 +433,7 @@ bool TGLexer::LexInclude() {
   tgtok::TokKind Tok = LexToken();
   if (Tok == tgtok::Error) return true;
   if (Tok != tgtok::StrVal) {
-    PrintError(getLoc(), "Expected filename after include");
+    PrintError(getLoc(), "expected filename after include");
     return true;
   }
 
@@ -444,7 +444,7 @@ bool TGLexer::LexInclude() {
   CurBuffer = SrcMgr.AddIncludeFile(Filename, SMLoc::getFromPointer(CurPtr),
                                     IncludedFile);
   if (!CurBuffer) {
-    PrintError(getLoc(), "Could not find include file '" + Filename + "'");
+    PrintError(getLoc(), "could not find include file '" + Filename + "'");
     return true;
   }
 
@@ -476,7 +476,7 @@ bool TGLexer::SkipCComment() {
     int CurChar = getNextChar();
     switch (CurChar) {
     case EOF:
-      PrintError(TokStart, "Unterminated comment!");
+      PrintError(TokStart, "unterminated comment");
       return true;
     case '*':
       // End of the comment?
@@ -543,7 +543,7 @@ tgtok::TokKind TGLexer::LexNumber() {
 
   // Requires at least one digit.
   if (CurPtr == NumStart)
-    return ReturnError(TokStart, "Invalid number");
+    return ReturnError(TokStart, "invalid number");
 
   errno = 0;
   if (IsMinus)
@@ -552,9 +552,9 @@ tgtok::TokKind TGLexer::LexNumber() {
     CurIntVal = strtoull(NumStart, nullptr, Base);
 
   if (errno == EINVAL)
-    return ReturnError(TokStart, "Invalid number");
+    return ReturnError(TokStart, "invalid number");
   if (errno == ERANGE)
-    return ReturnError(TokStart, "Number out of range");
+    return ReturnError(TokStart, "number out of range");
 
   return Base == 2 ? tgtok::BinaryIntVal : tgtok::IntVal;
 }
@@ -580,13 +580,13 @@ tgtok::TokKind TGLexer::LexBracket() {
     }
   }
 
-  return ReturnError(CodeStart - 2, "Unterminated code block");
+  return ReturnError(CodeStart - 2, "unterminated code block");
 }
 
 /// LexExclaim - Lex '!' and '![a-zA-Z]+'.
 tgtok::TokKind TGLexer::LexExclaim() {
   if (!isAlpha(*CurPtr))
-    return ReturnError(CurPtr - 1, "Invalid \"!operator\"");
+    return ReturnError(CurPtr - 1, "invalid \"!operator\"");
 
   const char *Start = CurPtr++;
   while (isAlpha(*CurPtr))
@@ -648,7 +648,8 @@ tgtok::TokKind TGLexer::LexExclaim() {
           .Case("repr", tgtok::XRepr)
           .Default(tgtok::Error);
 
-  return Kind != tgtok::Error ? Kind : ReturnError(Start-1, "Unknown operator");
+  return Kind != tgtok::Error ? Kind
+                              : ReturnError(Start - 1, "unknown operator");
 }
 
 bool TGLexer::prepExitInclude(bool IncludeStackMustBeEmpty) {
@@ -662,17 +663,17 @@ bool TGLexer::prepExitInclude(bool IncludeStackMustBeEmpty) {
 
   // Pop the preprocessing controls from the include stack.
   if (PrepIncludeStack.empty()) {
-    PrintFatalError("Preprocessor include stack is empty");
+    PrintFatalError("preprocessor include stack is empty");
   }
 
   PrepIncludeStack.pop_back();
 
   if (IncludeStackMustBeEmpty) {
     if (!PrepIncludeStack.empty())
-      PrintFatalError("Preprocessor include stack is not empty");
+      PrintFatalError("preprocessor include stack is not empty");
   } else {
     if (PrepIncludeStack.empty())
-      PrintFatalError("Preprocessor include stack is empty");
+      PrintFatalError("preprocessor include stack is empty");
   }
 
   return true;
@@ -732,7 +733,7 @@ bool TGLexer::prepEatPreprocessorDirective(tgtok::TokKind Kind) {
       return true;
     }
 
-  PrintFatalError("Unsupported preprocessing token in "
+  PrintFatalError("unsupported preprocessing token in "
                   "prepEatPreprocessorDirective()");
   return false;
 }
@@ -748,7 +749,7 @@ tgtok::TokKind TGLexer::lexPreprocessor(tgtok::TokKind Kind,
     StringRef MacroName = prepLexMacroName();
     StringRef IfTokName = Kind == tgtok::Ifdef ? "#ifdef" : "#ifndef";
     if (MacroName.empty())
-      return ReturnError(TokStart, "Expected macro name after " + IfTokName);
+      return ReturnError(TokStart, "expected macro name after " + IfTokName);
 
     bool MacroIsDefined = DefinedMacros.count(MacroName) != 0;
 
@@ -763,7 +764,7 @@ tgtok::TokKind TGLexer::lexPreprocessor(tgtok::TokKind Kind,
         {tgtok::Ifdef, MacroIsDefined, SMLoc::getFromPointer(TokStart)});
 
     if (!prepSkipDirectiveEnd())
-      return ReturnError(CurPtr, "Only comments are supported after " +
+      return ReturnError(CurPtr, "only comments are supported after " +
                                      IfTokName + " NAME");
 
     // If we were not processing tokens before this #ifdef,
@@ -794,7 +795,7 @@ tgtok::TokKind TGLexer::lexPreprocessor(tgtok::TokKind Kind,
 
     if (IfdefEntry.Kind != tgtok::Ifdef) {
       PrintError(TokStart, "double #else");
-      return ReturnError(IfdefEntry.SrcPos, "Previous #else is here");
+      return ReturnError(IfdefEntry.SrcPos, "previous #else is here");
     }
 
     // Replace the corresponding #ifdef's control with its negation
@@ -804,7 +805,7 @@ tgtok::TokKind TGLexer::lexPreprocessor(tgtok::TokKind Kind,
         {Kind, !IfdefEntry.IsDefined, SMLoc::getFromPointer(TokStart)});
 
     if (!prepSkipDirectiveEnd())
-      return ReturnError(CurPtr, "Only comments are supported after #else");
+      return ReturnError(CurPtr, "only comments are supported after #else");
 
     // If we were processing tokens before this #else,
     // we have to start skipping lines until the matching #endif.
@@ -827,12 +828,12 @@ tgtok::TokKind TGLexer::lexPreprocessor(tgtok::TokKind Kind,
 
     if (IfdefOrElseEntry.Kind != tgtok::Ifdef &&
         IfdefOrElseEntry.Kind != tgtok::Else) {
-      PrintFatalError("Invalid preprocessor control on the stack");
+      PrintFatalError("invalid preprocessor control on the stack");
       return tgtok::Error;
     }
 
     if (!prepSkipDirectiveEnd())
-      return ReturnError(CurPtr, "Only comments are supported after #endif");
+      return ReturnError(CurPtr, "only comments are supported after #endif");
 
     PrepIncludeStack.back()->pop_back();
 
@@ -847,15 +848,15 @@ tgtok::TokKind TGLexer::lexPreprocessor(tgtok::TokKind Kind,
   } else if (Kind == tgtok::Define) {
     StringRef MacroName = prepLexMacroName();
     if (MacroName.empty())
-      return ReturnError(TokStart, "Expected macro name after #define");
+      return ReturnError(TokStart, "expected macro name after #define");
 
     if (!DefinedMacros.insert(MacroName).second)
       PrintWarning(getLoc(),
-                   "Duplicate definition of macro: " + Twine(MacroName));
+                   "duplicate definition of macro: " + Twine(MacroName));
 
     if (!prepSkipDirectiveEnd())
       return ReturnError(CurPtr,
-                         "Only comments are supported after #define NAME");
+                         "only comments are supported after #define NAME");
 
     if (!ReturnNextLiveToken) {
       PrintFatalError("#define must be ignored during the lines skipping");
@@ -865,13 +866,13 @@ tgtok::TokKind TGLexer::lexPreprocessor(tgtok::TokKind Kind,
     return LexToken();
   }
 
-  PrintFatalError("Preprocessing directive is not supported");
+  PrintFatalError("preprocessing directive is not supported");
   return tgtok::Error;
 }
 
 bool TGLexer::prepSkipRegion(bool MustNeverBeFalse) {
   if (!MustNeverBeFalse)
-    PrintFatalError("Invalid recursion.");
+    PrintFatalError("invalid recursion.");
 
   do {
     // Skip all symbols to the line end.
@@ -917,7 +918,7 @@ bool TGLexer::prepSkipRegion(bool MustNeverBeFalse) {
     // due to #else or #endif.
     if (prepIsProcessingEnabled()) {
       if (Kind != tgtok::Else && Kind != tgtok::Endif) {
-        PrintFatalError("Tokens processing was enabled by an unexpected "
+        PrintFatalError("tokens processing was enabled by an unexpected "
                         "preprocessing directive");
         return false;
       }
@@ -1032,7 +1033,7 @@ bool TGLexer::prepSkipDirectiveEnd() {
           return false;
       } else {
         TokStart = CurPtr;
-        PrintError(CurPtr, "Unexpected character");
+        PrintError(CurPtr, "unexpected character");
         return false;
       }
 
@@ -1067,8 +1068,8 @@ void TGLexer::prepReportPreprocessorStackError() {
                     "empty control stack");
 
   auto &PrepControl = PrepIncludeStack.back()->back();
-  PrintError(CurBuf.end(), "Reached EOF without matching #endif");
-  PrintError(PrepControl.SrcPos, "The latest preprocessor control is here");
+  PrintError(CurBuf.end(), "reached EOF without matching #endif");
+  PrintError(PrepControl.SrcPos, "the latest preprocessor control is here");
 
   TokStart = CurPtr;
 }
diff --git a/llvm/test/Assembler/invalid-inttype.ll b/llvm/test/Assembler/invalid-inttype.ll
index c8aa7c66b79e..9e3c31148af2 100644
--- a/llvm/test/Assembler/invalid-inttype.ll
+++ b/llvm/test/Assembler/invalid-inttype.ll
@@ -1,5 +1,5 @@
 ; RUN: not llvm-as --disable-output %s 2>&1 | FileCheck -DFILE=%s %s
 
 ; i8388609 is the smallest integer type that can't be represented in LLVM IR
-; CHECK: [[FILE]]:[[@LINE+1]]:21: error: bitwidth for integer type out of range!
+; CHECK: [[FILE]]:[[@LINE+1]]:21: error: bitwidth for integer type out of range
 @i2 = common global i8388609 0, align 4
diff --git a/llvm/test/Assembler/invalid-name.ll b/llvm/test/Assembler/invalid-name.ll
index 74133e60df54d595c68c50aea282cd90899f334f..52e2bda3adbabde32ffd154cf1cd8256e0f6ab81 100644
GIT binary patch
delta 25
gcmX@lc#&~J3xBAOLUKl8QDSmQYLP-_@x+OC0DD;pkN^Mx

delta 23
ecmcb}c%E@W3vX#ojzUspNouh|Vo~bE$#wv3hY1n@

diff --git a/llvm/test/Assembler/invalid-name2.ll b/llvm/test/Assembler/invalid-name2.ll
index 8a848798a54cafef9c3c151b08f70955e59fa62f..78da4dc3d1b8d04c34b049c222d581088f506e7b 100644
GIT binary patch
delta 25
gcmdnVxSMf83xBAOLUKl8QDSmQYLP-_@x+N50Cyt^P5=M^

delta 23
ecmdnZxRY^03vX#ojzUspNouh|Vo~bE$r=D@sR+;j

diff --git a/llvm/test/TableGen/64-bit-int.td b/llvm/test/TableGen/64-bit-int.td
index 2d2bdb8b560e..d2a2999c14e9 100644
--- a/llvm/test/TableGen/64-bit-int.td
+++ b/llvm/test/TableGen/64-bit-int.td
@@ -16,7 +16,7 @@ def {
 #ifdef OOR3
   bits<64> Val = 0x10000000000000000;
 #endif
-// CHECK-OOR: error: Number out of range
+// CHECK-OOR: error: number out of range
 
   bits<64> BinVal0 = 0x8000000000000000;
   bits<64> HexVal0 = 0b1000000000000000000000000000000000000000000000000000000000000000;
diff --git a/llvm/test/TableGen/invalid-macro-name-command-line.td b/llvm/test/TableGen/invalid-macro-name-command-line.td
index 0d2307997ebe..7d19e8996639 100644
--- a/llvm/test/TableGen/invalid-macro-name-command-line.td
+++ b/llvm/test/TableGen/invalid-macro-name-command-line.td
@@ -3,7 +3,7 @@
 // RUN: not llvm-tblgen %s -D_MAC# 2>&1 | FileCheck %s --check-prefix=CHECK-TEST-3
 // RUN: not llvm-tblgen %s -D 2>&1 | FileCheck %s --check-prefix=CHECK-TEST-4
 
-// CHECK-TEST-1: error: Invalid macro name `MACRO=1` specified on command line
-// CHECK-TEST-2: error: Invalid macro name `0MAC` specified on command line
-// CHECK-TEST-3: error: Invalid macro name `_MAC#` specified on command line
+// CHECK-TEST-1: error: invalid macro name `MACRO=1` specified on command line
+// CHECK-TEST-2: error: invalid macro name `0MAC` specified on command line
+// CHECK-TEST-3: error: invalid macro name `_MAC#` specified on command line
 // CHECK-TEST-4: for the -D option: requires a value!
diff --git a/llvm/test/TableGen/prep-diag1.td b/llvm/test/TableGen/prep-diag1.td
index 41b7d477c694..27f428f4fe95 100644
--- a/llvm/test/TableGen/prep-diag1.td
+++ b/llvm/test/TableGen/prep-diag1.td
@@ -4,22 +4,22 @@
 // RUN: not llvm-tblgen -I %p %s 2>&1 | FileCheck --check-prefixes=DIAG3 %s
 
 #ifdef DIAG1
-// DIAG1: error: Only comments are supported after #define NAME
+// DIAG1: error: only comments are supported after #define NAME
 #define ENABLED1/*
 */class C;
 #endif // DIAG1
 
 #ifdef DIAG4
-// DIAG4: warning: Duplicate definition of macro: ENABLED1
+// DIAG4: warning: duplicate definition of macro: ENABLED1
 #define ENABLED1
 #define ENABLED1
 #endif // DIAG4
 
 #ifdef DIAG2
-// DIAG2: error: Only comments are supported after #ifdef NAME
+// DIAG2: error: only comments are supported after #ifdef NAME
 
 // Invalid #ifdef below should be detected even if DIAG2 is not defined.
-// DIAG3: error: Only comments are supported after #ifdef NAME
+// DIAG3: error: only comments are supported after #ifdef NAME
 #ifdef DIAG2/*
 */class C;
 #endif
diff --git a/llvm/test/TableGen/prep-diag10.td b/llvm/test/TableGen/prep-diag10.td
index eb387a07b066..cfcbab094ad7 100644
--- a/llvm/test/TableGen/prep-diag10.td
+++ b/llvm/test/TableGen/prep-diag10.td
@@ -1,6 +1,6 @@
 // RUN: not llvm-tblgen -I %p %s 2>&1 | FileCheck %s
 
-// CHECK: error: Reached EOF without matching #endif
-// CHECK: error: The latest preprocessor control is here
+// CHECK: error: reached EOF without matching #endif
+// CHECK: error: the latest preprocessor control is here
 #ifdef DISABLED
 #else
diff --git a/llvm/test/TableGen/prep-diag11.td b/llvm/test/TableGen/prep-diag11.td
index 0042bc04f9e1..1fe8a8503076 100644
--- a/llvm/test/TableGen/prep-diag11.td
+++ b/llvm/test/TableGen/prep-diag11.td
@@ -1,7 +1,7 @@
 // RUN: not llvm-tblgen -I %p %s 2>&1 | FileCheck %s
 
-// CHECK: error: Reached EOF without matching #endif
-// CHECK: error: The latest preprocessor control is here
+// CHECK: error: reached EOF without matching #endif
+// CHECK: error: the latest preprocessor control is here
 #ifdef DISABLED
 #else
 #define ENABLED
diff --git a/llvm/test/TableGen/prep-diag12.td b/llvm/test/TableGen/prep-diag12.td
index c26301ee17ac..02ffa672b2fa 100644
--- a/llvm/test/TableGen/prep-diag12.td
+++ b/llvm/test/TableGen/prep-diag12.td
@@ -1,7 +1,7 @@
 // RUN: not llvm-tblgen -I %p %s 2>&1 | FileCheck %s
 
-// CHECK: error: Reached EOF without matching #endif
-// CHECK: error: The latest preprocessor control is here
+// CHECK: error: reached EOF without matching #endif
+// CHECK: error: the latest preprocessor control is here
 #ifdef DISABLED
 #else
 #define ENABLED
diff --git a/llvm/test/TableGen/prep-diag13.td b/llvm/test/TableGen/prep-diag13.td
index aa3fdab4802d..733a46a16181 100644
--- a/llvm/test/TableGen/prep-diag13.td
+++ b/llvm/test/TableGen/prep-diag13.td
@@ -1,7 +1,7 @@
 // RUN: not llvm-tblgen -I %p %s 2>&1 | FileCheck %s
 
-// CHECK: error: Reached EOF without matching #endif
-// CHECK: error: The latest preprocessor control is here
+// CHECK: error: reached EOF without matching #endif
+// CHECK: error: the latest preprocessor control is here
 #ifdef DISABLED
 /*
 #else
diff --git a/llvm/test/TableGen/prep-diag14.td b/llvm/test/TableGen/prep-diag14.td
index cae9bc3b7f5b..a3216ee4f471 100644
--- a/llvm/test/TableGen/prep-diag14.td
+++ b/llvm/test/TableGen/prep-diag14.td
@@ -1,6 +1,6 @@
 // RUN: not llvm-tblgen -I %p %s 2>&1 | FileCheck %s
 
-// CHECK: error: Reached EOF without matching #endif
-// CHECK: error: The latest preprocessor control is here
+// CHECK: error: reached EOF without matching #endif
+// CHECK: error: the latest preprocessor control is here
 #ifdef DISABLED
 // #endif
diff --git a/llvm/test/TableGen/prep-diag2.td b/llvm/test/TableGen/prep-diag2.td
index 741026b9c8a2..e51490600ff6 100644
--- a/llvm/test/TableGen/prep-diag2.td
+++ b/llvm/test/TableGen/prep-diag2.td
@@ -2,10 +2,10 @@
 // RUN: not llvm-tblgen -I %p %s 2>&1 | FileCheck --check-prefixes=DIAG2 %s
 
 #ifdef DIAG1
-// DIAG1: error: Only comments are supported after #else
+// DIAG1: error: only comments are supported after #else
 
 // Invalid #else below should be detected even if DIAG1 is not defined.
-// DIAG2: error: Only comments are supported after #else
+// DIAG2: error: only comments are supported after #else
 #ifdef DIAG2//DIAG2
 #else/*
 */class C;
diff --git a/llvm/test/TableGen/prep-diag3.td b/llvm/test/TableGen/prep-diag3.td
index fbedfa290b99..0b4d40307b40 100644
--- a/llvm/test/TableGen/prep-diag3.td
+++ b/llvm/test/TableGen/prep-diag3.td
@@ -2,10 +2,10 @@
 // RUN: not llvm-tblgen -I %p %s 2>&1 | FileCheck --check-prefixes=DIAG2 %s
 
 #ifdef DIAG1
-// DIAG1: error: Only comments are supported after #endif
+// DIAG1: error: only comments are supported after #endif
 
 // Invalid #else below should be detected even if DIAG1 is not defined.
-// DIAG2: error: Only comments are supported after #endif
+// DIAG2: error: only comments are supported after #endif
 #ifdef DIAG2//DIAG2
 #else/*!DIAG2*/
 #endif/* !DIAG2
diff --git a/llvm/test/TableGen/prep-diag4.td b/llvm/test/TableGen/prep-diag4.td
index 4661ef8667d2..ead116ebde0d 100644
--- a/llvm/test/TableGen/prep-diag4.td
+++ b/llvm/test/TableGen/prep-diag4.td
@@ -1,7 +1,7 @@
 // RUN: not llvm-tblgen -I %p %s 2>&1 | FileCheck %s
 
 // CHECK: error: double #else
-// CHECK: error: Previous #else is here
+// CHECK: error: previous #else is here
 #ifdef DIAG1
 #else
 #else
diff --git a/llvm/test/TableGen/prep-diag6.td b/llvm/test/TableGen/prep-diag6.td
index f4202d115da5..bf1cd3d3490b 100644
--- a/llvm/test/TableGen/prep-diag6.td
+++ b/llvm/test/TableGen/prep-diag6.td
@@ -1,6 +1,6 @@
 // RUN: not llvm-tblgen -I %p %s 2>&1 | FileCheck %s
 
-// CHECK: error: Expected macro name after #ifdef
+// CHECK: error: expected macro name after #ifdef
 #ifdef
 #else
 #else
diff --git a/llvm/test/TableGen/prep-diag8.td b/llvm/test/TableGen/prep-diag8.td
index 7a7bde62c79c..82797d6cf4a6 100644
--- a/llvm/test/TableGen/prep-diag8.td
+++ b/llvm/test/TableGen/prep-diag8.td
@@ -1,5 +1,5 @@
 // RUN: not llvm-tblgen -I %p %s 2>&1 | FileCheck %s
 
-// CHECK: error: Expected macro name after #define
+// CHECK: error: expected macro name after #define
 #define
 #endif
diff --git a/llvm/test/TableGen/prep-diag9.td b/llvm/test/TableGen/prep-diag9.td
index 4ecff575cdc7..6ad208104301 100644
--- a/llvm/test/TableGen/prep-diag9.td
+++ b/llvm/test/TableGen/prep-diag9.td
@@ -1,5 +1,5 @@
 // RUN: not llvm-tblgen -I %p %s 2>&1 | FileCheck %s
 
-// CHECK: error: Reached EOF without matching #endif
-// CHECK: error: The latest preprocessor control is here
+// CHECK: error: reached EOF without matching #endif
+// CHECK: error: the latest preprocessor control is here
 #ifdef DISABLED
diff --git a/llvm/test/TableGen/prep-ifndef-diag-1.td b/llvm/test/TableGen/prep-ifndef-diag-1.td
index 941f2d377a98..4a0d0754ed79 100644
--- a/llvm/test/TableGen/prep-ifndef-diag-1.td
+++ b/llvm/test/TableGen/prep-ifndef-diag-1.td
@@ -1,4 +1,4 @@
 // RUN: not llvm-tblgen %s 2>&1 | FileCheck %s
 
-// CHECK: error: Expected macro name after #ifndef
+// CHECK: error: expected macro name after #ifndef
 #ifndef 1
diff --git a/llvm/test/TableGen/prep-ifndef-diag-2.td b/llvm/test/TableGen/prep-ifndef-diag-2.td
index 7b5f9dfd24b7..c89cbab08e5c 100644
--- a/llvm/test/TableGen/prep-ifndef-diag-2.td
+++ b/llvm/test/TableGen/prep-ifndef-diag-2.td
@@ -1,4 +1,4 @@
 // RUN: not llvm-tblgen %s 2>&1 | FileCheck %s
 
-// CHECK: error: Only comments are supported after #ifndef NAME
+// CHECK: error: only comments are supported after #ifndef NAME
 #ifndef MACRO 42
diff --git a/llvm/test/TableGen/unterminated-c-comment.td b/llvm/test/TableGen/unterminated-c-comment.td
index 0f4cd9d633c6..b5b995342be7 100644
--- a/llvm/test/TableGen/unterminated-c-comment.td
+++ b/llvm/test/TableGen/unterminated-c-comment.td
@@ -1,5 +1,5 @@
 // RUN: not llvm-tblgen -I %p %s 2>&1 | FileCheck %s
 
-// CHECK: error: Unterminated comment!
+// CHECK: error: unterminated comment
 
 include "unterminated-c-comment-include.inc" */
diff --git a/llvm/test/TableGen/unterminated-code-block.td b/llvm/test/TableGen/unterminated-code-block.td
index d6b6f50827a6..5bd4cd7e17d8 100644
--- a/llvm/test/TableGen/unterminated-code-block.td
+++ b/llvm/test/TableGen/unterminated-code-block.td
@@ -1,5 +1,5 @@
 // RUN: not llvm-tblgen -I %p %s 2>&1 | FileCheck %s
 
-// CHECK: error: Unterminated code block
+// CHECK: error: unterminated code block
 
 include "unterminated-code-block-include.inc" }]>;
-- 
GitLab


From 3754fc1e9af38951aa00181c0e8110174d3f94fd Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston@google.com>
Date: Tue, 29 Oct 2024 12:38:56 -0700
Subject: [PATCH 491/693] [hwasan] Flush stderr/stdout in tests (#114083)

The x86_64_lam_qemu buildbots started failing
(https://lab.llvm.org/buildbot/#/builders/139/builds/5462/steps/2/logs/stdio).
Based on the logs, it appears the HWASan check is correct but it did not
match the stderr/stdout output. This patch attempts to fix the issue by
flushing stderr/stdout as appropriate.
---
 compiler-rt/test/hwasan/TestCases/many-threads-uaf.c | 1 +
 compiler-rt/test/hwasan/TestCases/mem-intrinsics.c   | 1 +
 compiler-rt/test/hwasan/TestCases/use-after-free.c   | 1 +
 3 files changed, 3 insertions(+)

diff --git a/compiler-rt/test/hwasan/TestCases/many-threads-uaf.c b/compiler-rt/test/hwasan/TestCases/many-threads-uaf.c
index 8fa07861371d..e02ab5b28ce0 100644
--- a/compiler-rt/test/hwasan/TestCases/many-threads-uaf.c
+++ b/compiler-rt/test/hwasan/TestCases/many-threads-uaf.c
@@ -23,6 +23,7 @@ void *BoringThread(void *arg) {
 void *UAFThread(void *arg) {
   char * volatile x = (char*)malloc(10);
   fprintf(stderr, "ZZZ %p\n", x);
+  fflush(stderr);
   free(x);
   x[5] = 42;
   // CHECK: ERROR: HWAddressSanitizer: tag-mismatch on address
diff --git a/compiler-rt/test/hwasan/TestCases/mem-intrinsics.c b/compiler-rt/test/hwasan/TestCases/mem-intrinsics.c
index 78bef538af11..da1cb6869692 100644
--- a/compiler-rt/test/hwasan/TestCases/mem-intrinsics.c
+++ b/compiler-rt/test/hwasan/TestCases/mem-intrinsics.c
@@ -21,6 +21,7 @@ int main() {
   memcpy(Q, P, 32);
 #endif
   write(STDOUT_FILENO, "recovered\n", 10);
+  fflush(stdout);
   // WRITE: ERROR: HWAddressSanitizer: tag-mismatch on address
   // WRITE: WRITE of size 32 at {{.*}} tags: [[PTR_TAG:..]]/[[MEM_TAG:..]] (ptr/mem)
   // WRITE: Invalid access starting at offset 16
diff --git a/compiler-rt/test/hwasan/TestCases/use-after-free.c b/compiler-rt/test/hwasan/TestCases/use-after-free.c
index 070622f560a2..b4b79875e811 100644
--- a/compiler-rt/test/hwasan/TestCases/use-after-free.c
+++ b/compiler-rt/test/hwasan/TestCases/use-after-free.c
@@ -15,6 +15,7 @@ int main() {
   free(x);
   __hwasan_disable_allocator_tagging();
   fprintf(stderr, ISREAD ? "Going to do a READ\n" : "Going to do a WRITE\n");
+  fflush(stderr);
   // CHECK: Going to do a [[TYPE:[A-Z]*]]
   int r = 0;
   if (ISREAD) r = x[5]; else x[5] = 42;  // should be on the same line.
-- 
GitLab


From 3a1228a543bc85e225809b1f3033fac744f1f122 Mon Sep 17 00:00:00 2001
From: Adam Yang <hanbyang@microsoft.com>
Date: Tue, 29 Oct 2024 12:40:01 -0700
Subject: [PATCH 492/693] [SPIRV] Add GroupMemoryBarrierWithGroupSync intrinsic
 (#111888)

partially fixes #70103

### Changes
* Added int_spv_group_memory_barrier_with_group_sync intrinsic in
IntrinsicsSPIRV.td
* Added lowering for int_spv_group_memory_barrier_with_group_sync in
SPIRVInstructionSelector.cpp
* Added SPIRV backend test case

### Related PRs
* [[clang][HLSL] Add GroupMemoryBarrierWithGroupSync intrinsic
#111883](https://github.com/llvm/llvm-project/pull/111883)
* [[DXIL] Add GroupMemoryBarrierWithGroupSync intrinsic
#111884](https://github.com/llvm/llvm-project/pull/111884)
---
 llvm/include/llvm/IR/IntrinsicsSPIRV.td            |  1 +
 llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp | 11 +++++++++++
 .../group_memory_barrier_with_group_sync.ll        | 14 ++++++++++++++
 3 files changed, 26 insertions(+)
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll

diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 6df2eb156a07..ddb473905374 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -87,6 +87,7 @@ let TargetPrefix = "spv" in {
   def int_spv_wave_readlane : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>, llvm_i32_ty], [IntrConvergent, IntrNoMem]>;
   def int_spv_sign : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_any_ty], [IntrNoMem]>;
   def int_spv_radians : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty], [IntrNoMem]>;
+  def int_spv_group_memory_barrier_with_group_sync : DefaultAttrsIntrinsic<[], [], []>;
 
   // Create resource handle given the binding information. Returns a 
   // type appropriate for the kind of resource given the set id, binding id,
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index d9377fe4b91a..11ed7d660be0 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -2547,6 +2547,17 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
     return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt);
   case Intrinsic::spv_sign:
     return selectSign(ResVReg, ResType, I);
+  case Intrinsic::spv_group_memory_barrier_with_group_sync: {
+    Register MemSemReg =
+        buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I);
+    Register ScopeReg = buildI32Constant(SPIRV::Scope::Workgroup, I);
+    MachineBasicBlock &BB = *I.getParent();
+    return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier))
+        .addUse(ScopeReg)
+        .addUse(ScopeReg)
+        .addUse(MemSemReg)
+        .constrainAllUses(TII, TRI, RBI);
+  } break;
   case Intrinsic::spv_lifetime_start:
   case Intrinsic::spv_lifetime_end: {
     unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll
new file mode 100644
index 000000000000..6955411a0e4e
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll
@@ -0,0 +1,14 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpMemoryModel Logical GLSL450
+
+define void @test_group_memory_barrier_with_group_sync() {
+entry:
+  ; CHECK: %[[#TY:]] = OpTypeInt 32 0
+  ; CHECK-DAG: %[[#MEM_SEM:]] = OpConstant %[[#TY]] 16
+  ; CHECK-DAG: %[[#EXEC_AND_MEM_SCOPE:]] = OpConstant %[[#TY]] 2
+  ; CHECK: OpControlBarrier %[[#EXEC_AND_MEM_SCOPE]] %[[#EXEC_AND_MEM_SCOPE]] %[[#MEM_SEM]]
+  call void @llvm.spv.group.memory.barrier.with.group.sync()
+  ret void
+}
-- 
GitLab


From e205929399d9ee4782b2d8ef1b659f918bdfe7c2 Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston@google.com>
Date: Tue, 29 Oct 2024 12:40:54 -0700
Subject: [PATCH 493/693] [asan] Flush stderr in test (#114084)

This is the ASan equivalent of
https://github.com/llvm/llvm-project/pull/114083.

The x86_64_lam_qemu buildbots started failing

(https://lab.llvm.org/buildbot/#/builders/139/builds/5462/steps/2/logs/stdio).
Based on the logs, it appears the ASan check is correct but it did not
match the stderr/stdout output. This patch attempts to fix the issue by
flushing stderr as appropriate.
---
 compiler-rt/test/asan/TestCases/Posix/ignore_free_hook.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/compiler-rt/test/asan/TestCases/Posix/ignore_free_hook.cpp b/compiler-rt/test/asan/TestCases/Posix/ignore_free_hook.cpp
index 87be90014d56..dfeb8ad5c7b5 100644
--- a/compiler-rt/test/asan/TestCases/Posix/ignore_free_hook.cpp
+++ b/compiler-rt/test/asan/TestCases/Posix/ignore_free_hook.cpp
@@ -26,14 +26,17 @@ bool ignore_free = false;
 
 extern "C" {
 WEAK_ON_APPLE void __sanitizer_free_hook(const volatile void *ptr) {
-  if (ptr == glob_ptr)
+  if (ptr == glob_ptr) {
     fprintf(stderr, "Free Hook\n");
+    fflush(stderr);
+  }
 }
 
 WEAK_ON_APPLE int __sanitizer_ignore_free_hook(const volatile void *ptr) {
   if (ptr != glob_ptr)
     return 0;
   fprintf(stderr, ignore_free ? "Free Ignored\n" : "Free Respected\n");
+  fflush(stderr);
   return ignore_free;
 }
 } // extern "C"
-- 
GitLab


From 8a0cb9ac869334fd6c6bd6aad8408623a7ccd7f6 Mon Sep 17 00:00:00 2001
From: Maryam Moghadas <maryammo@ca.ibm.com>
Date: Tue, 29 Oct 2024 15:43:05 -0400
Subject: [PATCH 494/693] [PowerPC] Add custom lowering for ssubo (#111748)

This patch is to improve the codegen for ssubo node for i32 in 64-bit
mode by custom lowering.
---
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 37 +++++++++++++++++++++
 llvm/lib/Target/PowerPC/PPCISelLowering.h   |  1 +
 llvm/test/CodeGen/PowerPC/saddo-ssubo.ll    | 11 +++---
 3 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index ab31898e262e..d8f3095ed7fb 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -200,6 +200,11 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
 
   setOperationAction(ISD::UADDO, isPPC64 ? MVT::i64 : MVT::i32, Custom);
 
+  // On P10, the default lowering generates better code using the
+  // setbc instruction.
+  if (!Subtarget.hasP10Vector() && isPPC64)
+    setOperationAction(ISD::SSUBO, MVT::i32, Custom);
+
   // Match BITREVERSE to customized fast code sequence in the td file.
   setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
   setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
@@ -12016,6 +12021,36 @@ SDValue PPCTargetLowering::LowerUaddo(SDValue Op, SelectionDAG &DAG) const {
   return Res;
 }
 
+SDValue PPCTargetLowering::LowerSSUBO(SDValue Op, SelectionDAG &DAG) const {
+
+  SDLoc dl(Op);
+
+  SDValue LHS64 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, Op.getOperand(0));
+  SDValue RHS64 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, Op.getOperand(1));
+
+  SDValue Sub = DAG.getNode(ISD::SUB, dl, MVT::i64, LHS64, RHS64);
+
+  SDValue Extsw = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i64, Sub,
+                              DAG.getValueType(MVT::i32));
+
+  SDValue Xor = DAG.getNode(ISD::XOR, dl, MVT::i64, Extsw, Sub);
+
+  SDValue Addic = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(MVT::i64, MVT::Glue),
+                              Xor, DAG.getConstant(-1, dl, MVT::i64));
+
+  SDValue Overflow =
+      DAG.getNode(ISD::SUBE, dl, DAG.getVTList(MVT::i64, MVT::Glue), Xor, Addic,
+                  Addic.getValue(1));
+
+  SDValue OverflowTrunc =
+      DAG.getNode(ISD::TRUNCATE, dl, Op.getNode()->getValueType(1), Overflow);
+  SDValue SubTrunc =
+      (Sub->getValueType(0) != Op.getNode()->getValueType(0))
+          ? DAG.getNode(ISD::TRUNCATE, dl, Op.getNode()->getValueType(0), Sub)
+          : Sub;
+  return DAG.getMergeValues({SubTrunc, OverflowTrunc}, dl);
+}
+
 /// LowerOperation - Provide custom lowering hooks for some operations.
 ///
 SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
@@ -12038,6 +12073,8 @@ SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::SETCC:              return LowerSETCC(Op, DAG);
   case ISD::INIT_TRAMPOLINE:    return LowerINIT_TRAMPOLINE(Op, DAG);
   case ISD::ADJUST_TRAMPOLINE:  return LowerADJUST_TRAMPOLINE(Op, DAG);
+  case ISD::SSUBO:
+    return LowerSSUBO(Op, DAG);
 
   case ISD::INLINEASM:
   case ISD::INLINEASM_BR:       return LowerINLINEASM(Op, DAG);
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 0adbad868459..dde45e4cf6f4 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -1279,6 +1279,7 @@ namespace llvm {
     SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerUaddo(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerSSUBO(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/PowerPC/saddo-ssubo.ll b/llvm/test/CodeGen/PowerPC/saddo-ssubo.ll
index fd5f26ba3574..7147257d27c4 100644
--- a/llvm/test/CodeGen/PowerPC/saddo-ssubo.ll
+++ b/llvm/test/CodeGen/PowerPC/saddo-ssubo.ll
@@ -129,12 +129,11 @@ entry:
 define i1 @test_ssubo_i32(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: test_ssubo_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub 5, 3, 4
-; CHECK-NEXT:    cmpwi 1, 4, 0
-; CHECK-NEXT:    cmpw 5, 3
-; CHECK-NEXT:    li 3, 1
-; CHECK-NEXT:    creqv 20, 5, 0
-; CHECK-NEXT:    isel 3, 0, 3, 20
+; CHECK-NEXT:    sub 3, 3, 4
+; CHECK-NEXT:    extsw 4, 3
+; CHECK-NEXT:    xor 3, 4, 3
+; CHECK-NEXT:    addic 4, 3, -1
+; CHECK-NEXT:    subfe 3, 4, 3
 ; CHECK-NEXT:    blr
 entry:
   %res = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %a, i32 %b) nounwind
-- 
GitLab


From 27ef549af2c2f60d05f38db1ecc7a8ad7294351d Mon Sep 17 00:00:00 2001
From: z1nke <iamczn.cpp@gmail.com>
Date: Wed, 30 Oct 2024 03:48:39 +0800
Subject: [PATCH 495/693] [clang-tidy] Fix crash in
 modernize-use-designated-initializers check (#113688)

Fix #113652.

When calling `Node.isAggregate()` and `Node.isPOD()`, if `Node` is declared but
not defined, it will result in null pointer dereference (and if assertions are
enabled, it will cause an assertion failure).
---
 .../modernize/UseDesignatedInitializersCheck.cpp          | 8 ++++++--
 clang-tools-extra/docs/ReleaseNotes.rst                   | 4 ++++
 .../checkers/modernize/use-designated-initializers.cpp    | 8 ++++++++
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/modernize/UseDesignatedInitializersCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseDesignatedInitializersCheck.cpp
index 2a0cc403b726..3132067f3d5e 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseDesignatedInitializersCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseDesignatedInitializersCheck.cpp
@@ -80,9 +80,13 @@ unsigned getNumberOfDesignated(const InitListExpr *SyntacticInitList) {
   });
 }
 
-AST_MATCHER(CXXRecordDecl, isAggregate) { return Node.isAggregate(); }
+AST_MATCHER(CXXRecordDecl, isAggregate) {
+  return Node.hasDefinition() && Node.isAggregate();
+}
 
-AST_MATCHER(CXXRecordDecl, isPOD) { return Node.isPOD(); }
+AST_MATCHER(CXXRecordDecl, isPOD) {
+  return Node.hasDefinition() && Node.isPOD();
+}
 
 AST_MATCHER(InitListExpr, isFullyDesignated) {
   if (const InitListExpr *SyntacticForm =
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 54118e5f92f4..ccebf74e8a67 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -216,6 +216,10 @@ Changes in existing checks
   a false positive when only an implicit conversion happened inside an
   initializer list.
 
+- Improved :doc:`modernize-use-designated-initializers
+  <clang-tidy/checks/modernize/use-designated-initializers>` check to fix a
+  crash when a class is declared but not defined.
+
 - Improved :doc:`modernize-use-nullptr
   <clang-tidy/checks/modernize/use-nullptr>` check to also recognize
   ``NULL``/``__null`` (but not ``0``) when used with a templated type.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-designated-initializers.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-designated-initializers.cpp
index 9b769ad0be23..048665b2e54a 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-designated-initializers.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-designated-initializers.cpp
@@ -201,3 +201,11 @@ DECLARE_S93;
 // CHECK-MESSAGES-MACROS: :[[@LINE-1]]:1: warning: use designated initializer list to initialize 'S9' [modernize-use-designated-initializers]
 // CHECK-MESSAGES-MACROS: :[[@LINE-4]]:28: note: expanded from macro 'DECLARE_S93'
 // CHECK-MESSAGES-MACROS: :[[@LINE-71]]:1: note: aggregate type is defined here
+
+// Issue #113652.
+struct S14;
+
+struct S15{
+  S15(S14& d):d{d}{}
+  S14& d;
+};
-- 
GitLab


From 5c12434906d85dde4d44036cfb564fd366d9a1a4 Mon Sep 17 00:00:00 2001
From: David Majnemer <david.majnemer@gmail.com>
Date: Tue, 29 Oct 2024 03:46:04 +0000
Subject: [PATCH 496/693] [X86] Emit comments explaining the immediate in
 vfpclass

This makes the assembly a lot more readable at a glance.

As an example:
```
  vfpclasspd $4, %zmm0, %k0 # k0 = isNegativeZero(zmm0)
```
---
 .../X86/MCTargetDesc/X86InstComments.cpp      | 81 +++++++++++++++++++
 .../CodeGen/X86/avx10_2_512bf16-intrinsics.ll | 10 ++-
 .../CodeGen/X86/avx10_2bf16-intrinsics.ll     |  2 +
 .../X86/avx512dq-intrinsics-fast-isel.ll      | 24 +++---
 .../X86/avx512dq-intrinsics-upgrade.ll        |  4 +
 llvm/test/CodeGen/X86/avx512dq-intrinsics.ll  | 12 +++
 .../X86/avx512dqvl-intrinsics-fast-isel.ll    | 12 +--
 .../X86/avx512dqvl-intrinsics-upgrade.ll      |  4 +
 .../test/CodeGen/X86/avx512dqvl-intrinsics.ll |  4 +
 .../X86/stack-folding-fp-avx512fp16.ll        |  4 +
 .../X86/stack-folding-fp-avx512fp16vl.ll      |  2 +
 11 files changed, 137 insertions(+), 22 deletions(-)

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp
index 587f923e789f..49e8bab4c036 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp
@@ -40,6 +40,20 @@ using namespace llvm;
   CASE_MASK_INS_COMMON(Inst, Suffix, src)         \
   CASE_MASKZ_INS_COMMON(Inst, Suffix, src)
 
+#define CASE_FPCLASS_PACKED(Inst, src)    \
+  CASE_AVX_INS_COMMON(Inst, Z, r##src)    \
+  CASE_AVX_INS_COMMON(Inst, Z256, r##src) \
+  CASE_AVX_INS_COMMON(Inst, Z128, r##src) \
+  CASE_MASK_INS_COMMON(Inst, Z, r##src)
+
+#define CASE_FPCLASS_PACKED_MEM(Inst) \
+  CASE_FPCLASS_PACKED(Inst, m)        \
+  CASE_FPCLASS_PACKED(Inst, mb)
+
+#define CASE_FPCLASS_SCALAR(Inst, src)  \
+  CASE_AVX_INS_COMMON(Inst, Z, r##src)  \
+  CASE_MASK_INS_COMMON(Inst, Z, r##src)
+
 #define CASE_PTERNLOG(Inst, src)                                               \
   CASE_AVX512_INS_COMMON(Inst, Z, r##src##i)                                   \
   CASE_AVX512_INS_COMMON(Inst, Z256, r##src##i)                                \
@@ -949,6 +963,70 @@ static bool printPTERNLOGComments(const MCInst *MI, raw_ostream &OS,
   return true;
 }
 
+static bool printFPCLASSComments(const MCInst *MI, raw_ostream &OS,
+                                 const MCInstrInfo &MCII) {
+  unsigned NumOperands = MI->getNumOperands();
+  int SrcIdx;
+  switch (MI->getOpcode()) {
+    CASE_FPCLASS_PACKED(FPCLASSPBF16, r)
+    CASE_FPCLASS_PACKED(FPCLASSPH, r)
+    CASE_FPCLASS_PACKED(FPCLASSPS, r)
+    CASE_FPCLASS_PACKED(FPCLASSPD, r)
+    CASE_FPCLASS_SCALAR(FPCLASSSH, r)
+    CASE_FPCLASS_SCALAR(FPCLASSSS, r)
+    CASE_FPCLASS_SCALAR(FPCLASSSD, r) {
+      SrcIdx = NumOperands - 2;
+      break;
+    }
+    CASE_FPCLASS_PACKED_MEM(FPCLASSPBF16)
+    CASE_FPCLASS_PACKED_MEM(FPCLASSPH)
+    CASE_FPCLASS_PACKED_MEM(FPCLASSPS)
+    CASE_FPCLASS_PACKED_MEM(FPCLASSPD)
+    CASE_FPCLASS_SCALAR(FPCLASSSH, m)
+    CASE_FPCLASS_SCALAR(FPCLASSSS, m)
+    CASE_FPCLASS_SCALAR(FPCLASSSD, m) {
+      SrcIdx = -1;
+      break;
+    }
+  default:
+    return false;
+  }
+  StringRef DestName = getRegName(MI->getOperand(0).getReg());
+  StringRef SrcName =
+      SrcIdx != -1 ? getRegName(MI->getOperand(SrcIdx).getReg()) : "mem";
+
+  OS << DestName;
+  printMasking(OS, MI, MCII);
+  OS << " = ";
+
+  uint8_t Categories = MI->getOperand(NumOperands - 1).getImm();
+  if (Categories == 0) {
+    OS << "false";
+  } else {
+    static constexpr StringLiteral CategoryNames[] = {
+      "QuietNaN",
+      "PositiveZero",
+      "NegativeZero",
+      "PositiveInfinity",
+      "NegativeInfinity",
+      "Subnormal",
+      "Negative",
+      "SignalingNaN",
+    };
+    bool Conjoin = false;
+    for (size_t I = 0, E = std::size(CategoryNames); I != E; ++I) {
+      if (Categories & (1 << I)) {
+        if (Conjoin)
+          OS << " | ";
+        Conjoin = true;
+        OS << "is" << CategoryNames[I] << '(' << SrcName << ')';
+      }
+    }
+  }
+  OS << '\n';
+  return true;
+}
+
 //===----------------------------------------------------------------------===//
 // Top Level Entrypoint
 //===----------------------------------------------------------------------===//
@@ -970,6 +1048,9 @@ bool llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS,
   if (printPTERNLOGComments(MI, OS, MCII))
     return true;
 
+  if (printFPCLASSComments(MI, OS, MCII))
+    return true;
+
   switch (MI->getOpcode()) {
   default:
     // Not an instruction for which we can decode comments.
diff --git a/llvm/test/CodeGen/X86/avx10_2_512bf16-intrinsics.ll b/llvm/test/CodeGen/X86/avx10_2_512bf16-intrinsics.ll
index 7b81d547db08..5f2bcf0556b0 100644
--- a/llvm/test/CodeGen/X86/avx10_2_512bf16-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx10_2_512bf16-intrinsics.ll
@@ -76,13 +76,15 @@ declare <32 x i1> @llvm.x86.avx10.fpclass.nepbf16.512(<32 x bfloat>, i32)
 define i32 @test_int_x86_avx512_fpclass_nepbf16_512(<32 x bfloat> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_fpclass_nepbf16_512:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfpclasspbf16 $2, %zmm0, %k1 # encoding: [0x62,0xf3,0x7f,0x48,0x66,0xc8,0x02]
-; CHECK-NEXT:    vfpclasspbf16 $4, %zmm0, %k0 {%k1} # encoding: [0x62,0xf3,0x7f,0x49,0x66,0xc0,0x04]
+; CHECK-NEXT:    vfpclasspbf16 $6, %zmm0, %k1 # encoding: [0x62,0xf3,0x7f,0x48,0x66,0xc8,0x06]
+; CHECK-NEXT:    # k1 = isPositiveZero(zmm0) | isNegativeZero(zmm0)
+; CHECK-NEXT:    vfpclasspbf16 $0, %zmm0, %k0 {%k1} # encoding: [0x62,0xf3,0x7f,0x49,0x66,0xc0,0x00]
+; CHECK-NEXT:    # k0 {%k1} = false
 ; CHECK-NEXT:    kmovd %k0, %eax # encoding: [0xc5,0xfb,0x93,0xc0]
 ; CHECK-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <32 x i1> @llvm.x86.avx10.fpclass.nepbf16.512(<32 x bfloat> %x0, i32 4)
-  %res1 = call <32 x i1> @llvm.x86.avx10.fpclass.nepbf16.512(<32 x bfloat> %x0, i32 2)
+  %res = call <32 x i1> @llvm.x86.avx10.fpclass.nepbf16.512(<32 x bfloat> %x0, i32 0)
+  %res1 = call <32 x i1> @llvm.x86.avx10.fpclass.nepbf16.512(<32 x bfloat> %x0, i32 6)
   %1 = and <32 x i1> %res1, %res
   %2 = bitcast <32 x i1> %1 to i32
   ret i32 %2
diff --git a/llvm/test/CodeGen/X86/avx10_2bf16-intrinsics.ll b/llvm/test/CodeGen/X86/avx10_2bf16-intrinsics.ll
index 559d866b55cc..59151d4dd960 100644
--- a/llvm/test/CodeGen/X86/avx10_2bf16-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx10_2bf16-intrinsics.ll
@@ -298,6 +298,7 @@ define i8 @test_int_x86_avx512_fpclass_nepbf16_128(<8 x bfloat> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_fpclass_nepbf16_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclasspbf16 $2, %xmm0, %k1 # encoding: [0x62,0xf3,0x7f,0x08,0x66,0xc8,0x02]
+; CHECK-NEXT:    # k1 = isPositiveZero(xmm0)
 ; CHECK-NEXT:    vfpclasspbf16 $4, %xmm0, %k0 {%k1} # encoding: [0x62,0xf3,0x7f,0x09,0x66,0xc0,0x04]
 ; CHECK-NEXT:    kmovd %k0, %eax # encoding: [0xc5,0xfb,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
@@ -313,6 +314,7 @@ define i16 @test_int_x86_avx512_fpclass_nepbf16_256(<16 x bfloat> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_fpclass_nepbf16_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclasspbf16 $2, %ymm0, %k1 # encoding: [0x62,0xf3,0x7f,0x28,0x66,0xc8,0x02]
+; CHECK-NEXT:    # k1 = isPositiveZero(ymm0)
 ; CHECK-NEXT:    vfpclasspbf16 $4, %ymm0, %k0 {%k1} # encoding: [0x62,0xf3,0x7f,0x29,0x66,0xc0,0x04]
 ; CHECK-NEXT:    kmovd %k0, %eax # encoding: [0xc5,0xfb,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
diff --git a/llvm/test/CodeGen/X86/avx512dq-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512dq-intrinsics-fast-isel.ll
index 64063bdf8333..53193597d62f 100644
--- a/llvm/test/CodeGen/X86/avx512dq-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512dq-intrinsics-fast-isel.ll
@@ -7,7 +7,7 @@
 define zeroext i8 @test_mm512_mask_fpclass_pd_mask(i8 zeroext %__U, <8 x double> %__A) {
 ; X86-LABEL: test_mm512_mask_fpclass_pd_mask:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    vfpclasspd $4, %zmm0, %k0
+; X86-NEXT:    vfpclasspd $4, %zmm0, %k0 # k0 = isNegativeZero(zmm0)
 ; X86-NEXT:    kmovw %k0, %eax
 ; X86-NEXT:    andb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
@@ -16,7 +16,7 @@ define zeroext i8 @test_mm512_mask_fpclass_pd_mask(i8 zeroext %__U, <8 x double>
 ;
 ; X64-LABEL: test_mm512_mask_fpclass_pd_mask:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    vfpclasspd $4, %zmm0, %k0
+; X64-NEXT:    vfpclasspd $4, %zmm0, %k0 # k0 = isNegativeZero(zmm0)
 ; X64-NEXT:    kmovw %k0, %eax
 ; X64-NEXT:    andb %dil, %al
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
@@ -35,7 +35,7 @@ declare <8 x i1> @llvm.x86.avx512.fpclass.pd.512(<8 x double>, i32)
 define zeroext i8 @test_mm512_fpclass_pd_mask(<8 x double> %__A) {
 ; CHECK-LABEL: test_mm512_fpclass_pd_mask:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vfpclasspd $4, %zmm0, %k0
+; CHECK-NEXT:    vfpclasspd $4, %zmm0, %k0 # k0 = isNegativeZero(zmm0)
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    vzeroupper
@@ -49,7 +49,7 @@ entry:
 define zeroext i16 @test_mm512_mask_fpclass_ps_mask(i16 zeroext %__U, <16 x float> %__A) {
 ; X86-LABEL: test_mm512_mask_fpclass_ps_mask:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    vfpclassps $4, %zmm0, %k0
+; X86-NEXT:    vfpclassps $4, %zmm0, %k0 # k0 = isNegativeZero(zmm0)
 ; X86-NEXT:    kmovw %k0, %eax
 ; X86-NEXT:    andw {{[0-9]+}}(%esp), %ax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -58,7 +58,7 @@ define zeroext i16 @test_mm512_mask_fpclass_ps_mask(i16 zeroext %__U, <16 x floa
 ;
 ; X64-LABEL: test_mm512_mask_fpclass_ps_mask:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    vfpclassps $4, %zmm0, %k0
+; X64-NEXT:    vfpclassps $4, %zmm0, %k0 # k0 = isNegativeZero(zmm0)
 ; X64-NEXT:    kmovw %k0, %eax
 ; X64-NEXT:    andl %edi, %eax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -77,7 +77,7 @@ declare <16 x i1> @llvm.x86.avx512.fpclass.ps.512(<16 x float>, i32)
 define zeroext i16 @test_mm512_fpclass_ps_mask(<16 x float> %__A) {
 ; CHECK-LABEL: test_mm512_fpclass_ps_mask:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vfpclassps $4, %zmm0, %k0
+; CHECK-NEXT:    vfpclassps $4, %zmm0, %k0 # k0 = isNegativeZero(zmm0)
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    vzeroupper
@@ -91,7 +91,7 @@ entry:
 define zeroext i8 @test_mm_fpclass_sd_mask(<4 x float> %__A) {
 ; CHECK-LABEL: test_mm_fpclass_sd_mask:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vfpclasssd $2, %xmm0, %k0
+; CHECK-NEXT:    vfpclasssd $2, %xmm0, %k0 # k0 = isPositiveZero(xmm0)
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    ret{{[l|q]}}
@@ -107,7 +107,7 @@ define zeroext i8 @test_mm_mask_fpclass_sd_mask(i8 zeroext %__U, <4 x float> %__
 ; X86-LABEL: test_mm_mask_fpclass_sd_mask:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    vfpclasssd $2, %xmm0, %k0 {%k1}
+; X86-NEXT:    vfpclasssd $2, %xmm0, %k0 {%k1} # k0 {%k1} = isPositiveZero(xmm0)
 ; X86-NEXT:    kmovw %k0, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
@@ -115,7 +115,7 @@ define zeroext i8 @test_mm_mask_fpclass_sd_mask(i8 zeroext %__U, <4 x float> %__
 ; X64-LABEL: test_mm_mask_fpclass_sd_mask:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
-; X64-NEXT:    vfpclasssd $2, %xmm0, %k0 {%k1}
+; X64-NEXT:    vfpclasssd $2, %xmm0, %k0 {%k1} # k0 {%k1} = isPositiveZero(xmm0)
 ; X64-NEXT:    kmovw %k0, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
@@ -128,7 +128,7 @@ entry:
 define zeroext i8 @test_mm_fpclass_ss_mask(<4 x float> %__A) {
 ; CHECK-LABEL: test_mm_fpclass_ss_mask:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vfpclassss $2, %xmm0, %k0
+; CHECK-NEXT:    vfpclassss $2, %xmm0, %k0 # k0 = isPositiveZero(xmm0)
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    ret{{[l|q]}}
@@ -143,7 +143,7 @@ define zeroext i8 @test_mm_mask_fpclass_ss_mask(i8 zeroext %__U, <4 x float> %__
 ; X86-LABEL: test_mm_mask_fpclass_ss_mask:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    vfpclassss $2, %xmm0, %k0 {%k1}
+; X86-NEXT:    vfpclassss $2, %xmm0, %k0 {%k1} # k0 {%k1} = isPositiveZero(xmm0)
 ; X86-NEXT:    kmovw %k0, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
@@ -151,7 +151,7 @@ define zeroext i8 @test_mm_mask_fpclass_ss_mask(i8 zeroext %__U, <4 x float> %__
 ; X64-LABEL: test_mm_mask_fpclass_ss_mask:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
-; X64-NEXT:    vfpclassss $2, %xmm0, %k0 {%k1}
+; X64-NEXT:    vfpclassss $2, %xmm0, %k0 {%k1} # k0 {%k1} = isPositiveZero(xmm0)
 ; X64-NEXT:    kmovw %k0, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/avx512dq-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512dq-intrinsics-upgrade.ll
index 23e929aa9d89..8a0428d022b6 100644
--- a/llvm/test/CodeGen/X86/avx512dq-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512dq-intrinsics-upgrade.ll
@@ -654,7 +654,9 @@ define i8 @test_int_x86_avx512_mask_fpclass_pd_512(<8 x double> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_mask_fpclass_pd_512:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclasspd $2, %zmm0, %k1 # encoding: [0x62,0xf3,0xfd,0x48,0x66,0xc8,0x02]
+; CHECK-NEXT:    # k1 = isPositiveZero(zmm0)
 ; CHECK-NEXT:    vfpclasspd $4, %zmm0, %k0 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x66,0xc0,0x04]
+; CHECK-NEXT:    # k0 {%k1} = isNegativeZero(zmm0)
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
@@ -669,7 +671,9 @@ define i16@test_int_x86_avx512_mask_fpclass_ps_512(<16 x float> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_mask_fpclass_ps_512:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclassps $2, %zmm0, %k1 # encoding: [0x62,0xf3,0x7d,0x48,0x66,0xc8,0x02]
+; CHECK-NEXT:    # k1 = isPositiveZero(zmm0)
 ; CHECK-NEXT:    vfpclassps $4, %zmm0, %k0 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x66,0xc0,0x04]
+; CHECK-NEXT:    # k0 {%k1} = isNegativeZero(zmm0)
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
diff --git a/llvm/test/CodeGen/X86/avx512dq-intrinsics.ll b/llvm/test/CodeGen/X86/avx512dq-intrinsics.ll
index 027bca9c8bad..70f60c802a2d 100644
--- a/llvm/test/CodeGen/X86/avx512dq-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512dq-intrinsics.ll
@@ -726,7 +726,9 @@ define i8 @test_int_x86_avx512_fpclass_pd_512(<8 x double> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_fpclass_pd_512:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclasspd $2, %zmm0, %k1 # encoding: [0x62,0xf3,0xfd,0x48,0x66,0xc8,0x02]
+; CHECK-NEXT:    # k1 = isPositiveZero(zmm0)
 ; CHECK-NEXT:    vfpclasspd $4, %zmm0, %k0 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x66,0xc0,0x04]
+; CHECK-NEXT:    # k0 {%k1} = isNegativeZero(zmm0)
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
@@ -743,7 +745,9 @@ define i16@test_int_x86_avx512_fpclass_ps_512(<16 x float> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_fpclass_ps_512:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclassps $2, %zmm0, %k1 # encoding: [0x62,0xf3,0x7d,0x48,0x66,0xc8,0x02]
+; CHECK-NEXT:    # k1 = isPositiveZero(zmm0)
 ; CHECK-NEXT:    vfpclassps $4, %zmm0, %k0 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x66,0xc0,0x04]
+; CHECK-NEXT:    # k0 {%k1} = isNegativeZero(zmm0)
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
@@ -761,7 +765,9 @@ define i8 @test_int_x86_avx512_mask_fpclass_sd(<2 x double> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_mask_fpclass_sd:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclasssd $4, %xmm0, %k1 # encoding: [0x62,0xf3,0xfd,0x08,0x67,0xc8,0x04]
+; CHECK-NEXT:    # k1 = isNegativeZero(xmm0)
 ; CHECK-NEXT:    vfpclasssd $2, %xmm0, %k0 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x67,0xc0,0x02]
+; CHECK-NEXT:    # k0 {%k1} = isPositiveZero(xmm0)
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
@@ -775,6 +781,7 @@ define i8 @test_int_x86_avx512_mask_fpclass_sd_load(ptr %x0ptr) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    vfpclasssd $4, (%eax), %k0 # encoding: [0x62,0xf3,0xfd,0x08,0x67,0x00,0x04]
+; X86-NEXT:    # k0 = isNegativeZero(mem)
 ; X86-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -782,6 +789,7 @@ define i8 @test_int_x86_avx512_mask_fpclass_sd_load(ptr %x0ptr) {
 ; X64-LABEL: test_int_x86_avx512_mask_fpclass_sd_load:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vfpclasssd $4, (%rdi), %k0 # encoding: [0x62,0xf3,0xfd,0x08,0x67,0x07,0x04]
+; X64-NEXT:    # k0 = isNegativeZero(mem)
 ; X64-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq # encoding: [0xc3]
@@ -796,7 +804,9 @@ define i8 @test_int_x86_avx512_mask_fpclass_ss(<4 x float> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_mask_fpclass_ss:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclassss $4, %xmm0, %k1 # encoding: [0x62,0xf3,0x7d,0x08,0x67,0xc8,0x04]
+; CHECK-NEXT:    # k1 = isNegativeZero(xmm0)
 ; CHECK-NEXT:    vfpclassss $2, %xmm0, %k0 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x67,0xc0,0x02]
+; CHECK-NEXT:    # k0 {%k1} = isPositiveZero(xmm0)
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
@@ -810,6 +820,7 @@ define i8 @test_int_x86_avx512_mask_fpclass_ss_load(ptr %x0ptr, i8 %x1) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    vfpclassss $4, (%eax), %k0 # encoding: [0x62,0xf3,0x7d,0x08,0x67,0x00,0x04]
+; X86-NEXT:    # k0 = isNegativeZero(mem)
 ; X86-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -817,6 +828,7 @@ define i8 @test_int_x86_avx512_mask_fpclass_ss_load(ptr %x0ptr, i8 %x1) {
 ; X64-LABEL: test_int_x86_avx512_mask_fpclass_ss_load:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vfpclassss $4, (%rdi), %k0 # encoding: [0x62,0xf3,0x7d,0x08,0x67,0x07,0x04]
+; X64-NEXT:    # k0 = isNegativeZero(mem)
 ; X64-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq # encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/avx512dqvl-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512dqvl-intrinsics-fast-isel.ll
index 703591acef57..a8a38d9c4811 100644
--- a/llvm/test/CodeGen/X86/avx512dqvl-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512dqvl-intrinsics-fast-isel.ll
@@ -235,7 +235,7 @@ declare <2 x i1> @llvm.x86.avx512.fpclass.pd.128(<2 x double>, i32)
 define zeroext i8 @test_mm_fpclass_pd_mask(<2 x double> %__A) {
 ; CHECK-LABEL: test_mm_fpclass_pd_mask:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vfpclasspd $2, %xmm0, %k0
+; CHECK-NEXT:    vfpclasspd $2, %xmm0, %k0 # k0 = isPositiveZero(xmm0)
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    ret{{[l|q]}}
@@ -279,7 +279,7 @@ declare <4 x i1> @llvm.x86.avx512.fpclass.pd.256(<4 x double>, i32)
 define zeroext i8 @test_mm256_fpclass_pd_mask(<4 x double> %__A) {
 ; CHECK-LABEL: test_mm256_fpclass_pd_mask:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vfpclasspd $2, %ymm0, %k0
+; CHECK-NEXT:    vfpclasspd $2, %ymm0, %k0 # k0 = isPositiveZero(ymm0)
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    vzeroupper
@@ -322,7 +322,7 @@ declare <4 x i1> @llvm.x86.avx512.fpclass.ps.128(<4 x float>, i32)
 define zeroext i8 @test_mm_fpclass_ps_mask(<4 x float> %__A) {
 ; CHECK-LABEL: test_mm_fpclass_ps_mask:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vfpclassps $2, %xmm0, %k0
+; CHECK-NEXT:    vfpclassps $2, %xmm0, %k0 # k0 = isPositiveZero(xmm0)
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    ret{{[l|q]}}
@@ -336,7 +336,7 @@ entry:
 define zeroext i8 @test_mm256_mask_fpclass_ps_mask(i8 zeroext %__U, <8 x float> %__A) {
 ; X86-LABEL: test_mm256_mask_fpclass_ps_mask:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    vfpclassps $2, %ymm0, %k0
+; X86-NEXT:    vfpclassps $2, %ymm0, %k0 # k0 = isPositiveZero(ymm0)
 ; X86-NEXT:    kmovw %k0, %eax
 ; X86-NEXT:    andb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
@@ -345,7 +345,7 @@ define zeroext i8 @test_mm256_mask_fpclass_ps_mask(i8 zeroext %__U, <8 x float>
 ;
 ; X64-LABEL: test_mm256_mask_fpclass_ps_mask:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    vfpclassps $2, %ymm0, %k0
+; X64-NEXT:    vfpclassps $2, %ymm0, %k0 # k0 = isPositiveZero(ymm0)
 ; X64-NEXT:    kmovw %k0, %eax
 ; X64-NEXT:    andb %dil, %al
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
@@ -364,7 +364,7 @@ declare <8 x i1> @llvm.x86.avx512.fpclass.ps.256(<8 x float>, i32)
 define zeroext i8 @test_mm256_fpclass_ps_mask(<8 x float> %__A) {
 ; CHECK-LABEL: test_mm256_fpclass_ps_mask:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vfpclassps $2, %ymm0, %k0
+; CHECK-NEXT:    vfpclassps $2, %ymm0, %k0 # k0 = isPositiveZero(ymm0)
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/avx512dqvl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512dqvl-intrinsics-upgrade.ll
index 8d609eb7fdd0..f31dafcd6862 100644
--- a/llvm/test/CodeGen/X86/avx512dqvl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512dqvl-intrinsics-upgrade.ll
@@ -2921,6 +2921,7 @@ define i8 @test_int_x86_avx512_mask_fpclass_ps_128(<4 x float> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_mask_fpclass_ps_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclassps $4, %xmm0, %k1 # encoding: [0x62,0xf3,0x7d,0x08,0x66,0xc8,0x04]
+; CHECK-NEXT:    # k1 = isNegativeZero(xmm0)
 ; CHECK-NEXT:    vfpclassps $2, %xmm0, %k0 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x66,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
@@ -2936,6 +2937,7 @@ define i8 @test_int_x86_avx512_mask_fpclass_ps_256(<8 x float> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_mask_fpclass_ps_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclassps $4, %ymm0, %k1 # encoding: [0x62,0xf3,0x7d,0x28,0x66,0xc8,0x04]
+; CHECK-NEXT:    # k1 = isNegativeZero(ymm0)
 ; CHECK-NEXT:    vfpclassps $2, %ymm0, %k0 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x66,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
@@ -2952,6 +2954,7 @@ define i8 @test_int_x86_avx512_mask_fpclass_pd_128(<2 x double> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_mask_fpclass_pd_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclasspd $2, %xmm0, %k1 # encoding: [0x62,0xf3,0xfd,0x08,0x66,0xc8,0x02]
+; CHECK-NEXT:    # k1 = isPositiveZero(xmm0)
 ; CHECK-NEXT:    vfpclasspd $4, %xmm0, %k0 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x66,0xc0,0x04]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
@@ -2967,6 +2970,7 @@ define i8 @test_int_x86_avx512_mask_fpclass_pd_256(<4 x double> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_mask_fpclass_pd_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclasspd $4, %ymm0, %k1 # encoding: [0x62,0xf3,0xfd,0x28,0x66,0xc8,0x04]
+; CHECK-NEXT:    # k1 = isNegativeZero(ymm0)
 ; CHECK-NEXT:    vfpclasspd $2, %ymm0, %k0 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x66,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
diff --git a/llvm/test/CodeGen/X86/avx512dqvl-intrinsics.ll b/llvm/test/CodeGen/X86/avx512dqvl-intrinsics.ll
index 3b9f96ef452c..ec94b593148d 100644
--- a/llvm/test/CodeGen/X86/avx512dqvl-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512dqvl-intrinsics.ll
@@ -1500,6 +1500,7 @@ define i8 @test_int_x86_avx512_fpclass_ps_128(<4 x float> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_fpclass_ps_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclassps $4, %xmm0, %k1 # encoding: [0x62,0xf3,0x7d,0x08,0x66,0xc8,0x04]
+; CHECK-NEXT:    # k1 = isNegativeZero(xmm0)
 ; CHECK-NEXT:    vfpclassps $2, %xmm0, %k0 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x66,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
@@ -1518,6 +1519,7 @@ define i8 @test_int_x86_avx512_fpclass_ps_256(<8 x float> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_fpclass_ps_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclassps $4, %ymm0, %k1 # encoding: [0x62,0xf3,0x7d,0x28,0x66,0xc8,0x04]
+; CHECK-NEXT:    # k1 = isNegativeZero(ymm0)
 ; CHECK-NEXT:    vfpclassps $2, %ymm0, %k0 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x66,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
@@ -1536,6 +1538,7 @@ define i8 @test_int_x86_avx512_fpclass_pd_128(<2 x double> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_fpclass_pd_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclasspd $2, %xmm0, %k1 # encoding: [0x62,0xf3,0xfd,0x08,0x66,0xc8,0x02]
+; CHECK-NEXT:    # k1 = isPositiveZero(xmm0)
 ; CHECK-NEXT:    vfpclasspd $4, %xmm0, %k0 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x66,0xc0,0x04]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
@@ -1554,6 +1557,7 @@ define i8 @test_int_x86_avx512_fpclass_pd_256(<4 x double> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_fpclass_pd_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vfpclasspd $4, %ymm0, %k1 # encoding: [0x62,0xf3,0xfd,0x28,0x66,0xc8,0x04]
+; CHECK-NEXT:    # k1 = isNegativeZero(ymm0)
 ; CHECK-NEXT:    vfpclasspd $2, %ymm0, %k0 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x66,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
diff --git a/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16.ll b/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16.ll
index ed7109c416e7..9382ba31ab64 100644
--- a/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16.ll
@@ -234,6 +234,7 @@ define i32 @stack_fold_fpclassph(<32 x half> %a0) {
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    vfpclassphz $4, {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 64-byte Folded Reload
+; CHECK-NEXT:    # k0 = isNegativeZero(mem)
 ; CHECK-NEXT:    kmovd %k0, %eax
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
@@ -253,6 +254,7 @@ define i32 @stack_fold_fpclassph_mask(<32 x half> %a0, ptr %p) {
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    kmovd (%rdi), %k1
 ; CHECK-NEXT:    vfpclassphz $4, {{[-0-9]+}}(%r{{[sb]}}p), %k0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # k0 {%k1} = isNegativeZero(mem)
 ; CHECK-NEXT:    kmovd %k0, %eax
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
@@ -272,6 +274,7 @@ define i8 @stack_fold_fpclasssh(<8 x half> %a0) {
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    vfpclasssh $4, {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 16-byte Folded Reload
+; CHECK-NEXT:    # k0 = isNegativeZero(mem)
 ; CHECK-NEXT:    kmovd %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    retq
@@ -290,6 +293,7 @@ define i8 @stack_fold_fpclasssh_mask(<8 x half> %a0, ptr %p) {
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    kmovb (%rdi), %k1
 ; CHECK-NEXT:    vfpclasssh $4, {{[-0-9]+}}(%r{{[sb]}}p), %k0 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    # k0 {%k1} = isNegativeZero(mem)
 ; CHECK-NEXT:    kmovd %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16vl.ll b/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16vl.ll
index e2ed997783f5..3386f4a9b519 100644
--- a/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16vl.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16vl.ll
@@ -110,6 +110,7 @@ define i8 @stack_fold_fpclassph(<8 x half> %a0) {
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    vfpclassphx $4, {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 16-byte Folded Reload
+; CHECK-NEXT:    # k0 = isNegativeZero(mem)
 ; CHECK-NEXT:    kmovd %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    retq
@@ -148,6 +149,7 @@ define i16 @stack_fold_fpclassph_ymm(<16 x half> %a0) {
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    vfpclassphy $4, {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 32-byte Folded Reload
+; CHECK-NEXT:    # k0 = isNegativeZero(mem)
 ; CHECK-NEXT:    kmovd %k0, %eax
 ; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    vzeroupper
-- 
GitLab


From ba8d9ce8d4f0665f29bb4bb43ce16d02acaed751 Mon Sep 17 00:00:00 2001
From: Steven Wu <stevenwu@apple.com>
Date: Tue, 29 Oct 2024 13:00:59 -0700
Subject: [PATCH 497/693] [ADT] Fix unused variable from #69528 (#114114)

Remove unused variable to fix build failures from bot.
---
 llvm/lib/Support/TrieRawHashMap.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Support/TrieRawHashMap.cpp b/llvm/lib/Support/TrieRawHashMap.cpp
index 9eeac0bbc5c2..4741f3d4db04 100644
--- a/llvm/lib/Support/TrieRawHashMap.cpp
+++ b/llvm/lib/Support/TrieRawHashMap.cpp
@@ -424,7 +424,7 @@ unsigned ThreadSafeTrieRawHashMapBase::getNumSlotUsed(
     return 0;
   unsigned Num = 0;
   for (unsigned I = 0, E = S->size(); I < E; ++I)
-    if (auto *E = S->load(I))
+    if (S->load(I))
       ++Num;
   return Num;
 }
-- 
GitLab


From ccd73eeab34b31c7c38e9aca05ca4192fb0913b0 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Tue, 29 Oct 2024 13:06:55 -0700
Subject: [PATCH 498/693] [LinkerWrapper] Remove in-house handling of LTO
 (#113715)

Summary:
This should be the linker's job if the user creates any bitcode files,
then passing `-flto` to the linker for the toolchain should be able to
handle it. Right now this path is only used in the case where someone
does LTO w/ ld.gold targeting a CPU so I think we are safe here as that
will still be forwarded, for bfd it'll be an error as it would on the
host. I think I talked the SYCL team out of using this as well so I
should be good to delete it.
---
 .../ClangLinkerWrapper.cpp                    | 362 +-----------------
 1 file changed, 8 insertions(+), 354 deletions(-)

diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 9fcecaee318a..45274b797236 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -140,9 +140,6 @@ static std::list<SmallString<128>> TempFiles;
 /// Codegen flags for LTO backend.
 static codegen::RegisterCodeGenFlags CodeGenFlags;
 
-/// Global flag to indicate that the LTO pipeline threw an error.
-static std::atomic<bool> LTOError;
-
 using OffloadingImage = OffloadBinary::OffloadingImage;
 
 namespace llvm {
@@ -293,12 +290,10 @@ Expected<std::string> findProgram(StringRef Name, ArrayRef<StringRef> Paths) {
   return *Path;
 }
 
-/// We will defer LTO to the target's linker if we are not doing JIT and it is
-/// supported by the toolchain.
 bool linkerSupportsLTO(const ArgList &Args) {
   llvm::Triple Triple(Args.getLastArgValue(OPT_triple_EQ));
   return Triple.isNVPTX() || Triple.isAMDGPU() ||
-         Args.getLastArgValue(OPT_linker_path_EQ).ends_with("ld.lld");
+         Args.getLastArgValue(OPT_linker_path_EQ).ends_with("lld");
 }
 
 /// Returns the hashed value for a constant string.
@@ -528,13 +523,11 @@ Expected<StringRef> clang(ArrayRef<StringRef> InputFiles, const ArgList &Args) {
   };
 
   // Forward all of the `--offload-opt` and similar options to the device.
-  if (linkerSupportsLTO(Args)) {
-    CmdArgs.push_back("-flto");
-    for (auto &Arg : Args.filtered(OPT_offload_opt_eq_minus, OPT_mllvm))
-      CmdArgs.append(
-          {"-Xlinker",
-           Args.MakeArgString("--plugin-opt=" + StringRef(Arg->getValue()))});
-  }
+  CmdArgs.push_back("-flto");
+  for (auto &Arg : Args.filtered(OPT_offload_opt_eq_minus, OPT_mllvm))
+    CmdArgs.append(
+        {"-Xlinker",
+         Args.MakeArgString("--plugin-opt=" + StringRef(Arg->getValue()))});
 
   if (!Triple.isNVPTX())
     CmdArgs.push_back("-Wl,--no-undefined");
@@ -655,7 +648,6 @@ void diagnosticHandler(const DiagnosticInfo &DI) {
   switch (DI.getSeverity()) {
   case DS_Error:
     WithColor::error(errs(), LinkerExecutable) << ErrStorage << "\n";
-    LTOError = true;
     break;
   case DS_Warning:
     WithColor::warning(errs(), LinkerExecutable) << ErrStorage << "\n";
@@ -669,334 +661,6 @@ void diagnosticHandler(const DiagnosticInfo &DI) {
   }
 }
 
-// Get the list of target features from the input file and unify them such that
-// if there are multiple +xxx or -xxx features we only keep the last one.
-std::vector<std::string> getTargetFeatures(ArrayRef<OffloadFile> InputFiles) {
-  SmallVector<StringRef> Features;
-  for (const OffloadFile &File : InputFiles) {
-    for (auto Arg : llvm::split(File.getBinary()->getString("feature"), ","))
-      Features.emplace_back(Arg);
-  }
-
-  // Only add a feature if it hasn't been seen before starting from the end.
-  std::vector<std::string> UnifiedFeatures;
-  DenseSet<StringRef> UsedFeatures;
-  for (StringRef Feature : llvm::reverse(Features)) {
-    if (UsedFeatures.insert(Feature.drop_front()).second)
-      UnifiedFeatures.push_back(Feature.str());
-  }
-
-  return UnifiedFeatures;
-}
-
-template <typename ModuleHook = function_ref<bool(size_t, const Module &)>>
-std::unique_ptr<lto::LTO> createLTO(
-    const ArgList &Args, const std::vector<std::string> &Features,
-    ModuleHook Hook = [](size_t, const Module &) { return true; }) {
-  const llvm::Triple Triple(Args.getLastArgValue(OPT_triple_EQ));
-  // We need to remove AMD's target-id from the processor if present.
-  StringRef TargetID = Args.getLastArgValue(OPT_arch_EQ);
-  StringRef Arch = clang::getProcessorFromTargetID(Triple, TargetID);
-  lto::Config Conf;
-  lto::ThinBackend Backend;
-  // TODO: Handle index-only thin-LTO
-  Backend =
-      lto::createInProcessThinBackend(llvm::heavyweight_hardware_concurrency());
-
-  Conf.CPU = Arch.str();
-  Conf.Options = codegen::InitTargetOptionsFromCodeGenFlags(Triple);
-
-  Conf.RemarksFilename = RemarksFilename;
-  Conf.RemarksPasses = RemarksPasses;
-  Conf.RemarksWithHotness = RemarksWithHotness;
-  Conf.RemarksHotnessThreshold = RemarksHotnessThreshold;
-  Conf.RemarksFormat = RemarksFormat;
-
-  StringRef OptLevel = Args.getLastArgValue(OPT_opt_level, "O2");
-  Conf.MAttrs = Features;
-  std::optional<CodeGenOptLevel> CGOptLevelOrNone =
-      CodeGenOpt::parseLevel(OptLevel[1]);
-  assert(CGOptLevelOrNone && "Invalid optimization level");
-  Conf.CGOptLevel = *CGOptLevelOrNone;
-  Conf.OptLevel = OptLevel[1] - '0';
-  Conf.DefaultTriple = Triple.getTriple();
-
-  // TODO: Should we complain about combining --opt-level and -passes, as opt
-  // does?  That might be too limiting in clang-linker-wrapper, so for now we
-  // just warn in the help entry for -passes that the default<O?> corresponding
-  // to --opt-level=O? should be included there.  The problem is that
-  // --opt-level produces effects in clang-linker-wrapper beyond what -passes
-  // appears to be able to achieve, so rejecting the combination of --opt-level
-  // and -passes would apparently make it impossible to combine those effects
-  // with a custom pass pipeline.
-  Conf.OptPipeline = PassPipeline;
-  Conf.PassPlugins = PassPlugins;
-
-  LTOError = false;
-  Conf.DiagHandler = diagnosticHandler;
-
-  Conf.PTO.LoopVectorization = Conf.OptLevel > 1;
-  Conf.PTO.SLPVectorization = Conf.OptLevel > 1;
-
-  if (SaveTemps) {
-    std::string TempName = (sys::path::filename(ExecutableName) + "." +
-                            Triple.getTriple() + "." + TargetID)
-                               .str();
-    Conf.PostInternalizeModuleHook = [=](size_t Task, const Module &M) {
-      std::string File =
-          !Task ? TempName + ".postlink.bc"
-                : TempName + "." + std::to_string(Task) + ".postlink.bc";
-      error_code EC;
-      raw_fd_ostream LinkedBitcode(File, EC, sys::fs::OF_None);
-      if (EC)
-        reportError(errorCodeToError(EC));
-      WriteBitcodeToFile(M, LinkedBitcode);
-      return true;
-    };
-    Conf.PreCodeGenModuleHook = [=](size_t Task, const Module &M) {
-      std::string File =
-          !Task ? TempName + ".postopt.bc"
-                : TempName + "." + std::to_string(Task) + ".postopt.bc";
-      error_code EC;
-      raw_fd_ostream LinkedBitcode(File, EC, sys::fs::OF_None);
-      if (EC)
-        reportError(errorCodeToError(EC));
-      WriteBitcodeToFile(M, LinkedBitcode);
-      return true;
-    };
-  }
-  Conf.PostOptModuleHook = Hook;
-  Conf.CGFileType = (Triple.isNVPTX() || SaveTemps)
-                        ? CodeGenFileType::AssemblyFile
-                        : CodeGenFileType::ObjectFile;
-
-  // TODO: Handle remark files
-  Conf.HasWholeProgramVisibility = Args.hasArg(OPT_whole_program);
-
-  return std::make_unique<lto::LTO>(std::move(Conf), Backend);
-}
-
-// Returns true if \p S is valid as a C language identifier and will be given
-// `__start_` and `__stop_` symbols.
-bool isValidCIdentifier(StringRef S) {
-  return !S.empty() && (isAlpha(S[0]) || S[0] == '_') &&
-         llvm::all_of(llvm::drop_begin(S),
-                      [](char C) { return C == '_' || isAlnum(C); });
-}
-
-Error linkBitcodeFiles(SmallVectorImpl<OffloadFile> &InputFiles,
-                       SmallVectorImpl<StringRef> &OutputFiles,
-                       const ArgList &Args) {
-  llvm::TimeTraceScope TimeScope("Link bitcode files");
-  const llvm::Triple Triple(Args.getLastArgValue(OPT_triple_EQ));
-  StringRef Arch = Args.getLastArgValue(OPT_arch_EQ);
-
-  SmallVector<OffloadFile, 4> BitcodeInputFiles;
-  DenseSet<StringRef> StrongResolutions;
-  DenseSet<StringRef> UsedInRegularObj;
-  DenseSet<StringRef> UsedInSharedLib;
-  BumpPtrAllocator Alloc;
-  StringSaver Saver(Alloc);
-
-  // Search for bitcode files in the input and create an LTO input file. If
-  // it is not a bitcode file, scan its symbol table for symbols we need to
-  // save.
-  for (OffloadFile &File : InputFiles) {
-    MemoryBufferRef Buffer = MemoryBufferRef(File.getBinary()->getImage(), "");
-
-    file_magic Type = identify_magic(Buffer.getBuffer());
-    switch (Type) {
-    case file_magic::bitcode: {
-      Expected<IRSymtabFile> IRSymtabOrErr = readIRSymtab(Buffer);
-      if (!IRSymtabOrErr)
-        return IRSymtabOrErr.takeError();
-
-      // Check for any strong resolutions we need to preserve.
-      for (unsigned I = 0; I != IRSymtabOrErr->Mods.size(); ++I) {
-        for (const auto &Sym : IRSymtabOrErr->TheReader.module_symbols(I)) {
-          if (!Sym.isFormatSpecific() && Sym.isGlobal() && !Sym.isWeak() &&
-              !Sym.isUndefined())
-            StrongResolutions.insert(Saver.save(Sym.Name));
-        }
-      }
-      BitcodeInputFiles.emplace_back(std::move(File));
-      continue;
-    }
-    case file_magic::elf_relocatable:
-    case file_magic::elf_shared_object: {
-      Expected<std::unique_ptr<ObjectFile>> ObjFile =
-          ObjectFile::createObjectFile(Buffer);
-      if (!ObjFile)
-        continue;
-
-      for (SymbolRef Sym : (*ObjFile)->symbols()) {
-        Expected<StringRef> Name = Sym.getName();
-        if (!Name)
-          return Name.takeError();
-
-        // Record if we've seen these symbols in any object or shared
-        // libraries.
-        if ((*ObjFile)->isRelocatableObject())
-          UsedInRegularObj.insert(Saver.save(*Name));
-        else
-          UsedInSharedLib.insert(Saver.save(*Name));
-      }
-      continue;
-    }
-    default:
-      continue;
-    }
-  }
-
-  if (BitcodeInputFiles.empty())
-    return Error::success();
-
-  // Remove all the bitcode files that we moved from the original input.
-  llvm::erase_if(InputFiles, [](OffloadFile &F) { return !F.getBinary(); });
-
-  // LTO Module hook to output bitcode without running the backend.
-  SmallVector<StringRef> BitcodeOutput;
-  auto OutputBitcode = [&](size_t, const Module &M) {
-    auto TempFileOrErr = createOutputFile(sys::path::filename(ExecutableName) +
-                                              "-jit-" + Triple.getTriple(),
-                                          "bc");
-    if (!TempFileOrErr)
-      reportError(TempFileOrErr.takeError());
-
-    std::error_code EC;
-    raw_fd_ostream LinkedBitcode(*TempFileOrErr, EC, sys::fs::OF_None);
-    if (EC)
-      reportError(errorCodeToError(EC));
-    WriteBitcodeToFile(M, LinkedBitcode);
-    BitcodeOutput.push_back(*TempFileOrErr);
-    return false;
-  };
-
-  // We assume visibility of the whole program if every input file was
-  // bitcode.
-  auto Features = getTargetFeatures(BitcodeInputFiles);
-  auto LTOBackend = Args.hasArg(OPT_embed_bitcode) ||
-                            Args.hasArg(OPT_builtin_bitcode_EQ) ||
-                            Args.hasArg(OPT_clang_backend)
-                        ? createLTO(Args, Features, OutputBitcode)
-                        : createLTO(Args, Features);
-
-  // We need to resolve the symbols so the LTO backend knows which symbols
-  // need to be kept or can be internalized. This is a simplified symbol
-  // resolution scheme to approximate the full resolution a linker would do.
-  uint64_t Idx = 0;
-  DenseSet<StringRef> PrevailingSymbols;
-  for (auto &BitcodeInput : BitcodeInputFiles) {
-    // Get a semi-unique buffer identifier for Thin-LTO.
-    StringRef Identifier = Saver.save(
-        std::to_string(Idx++) + "." +
-        BitcodeInput.getBinary()->getMemoryBufferRef().getBufferIdentifier());
-    MemoryBufferRef Buffer =
-        MemoryBufferRef(BitcodeInput.getBinary()->getImage(), Identifier);
-    Expected<std::unique_ptr<lto::InputFile>> BitcodeFileOrErr =
-        llvm::lto::InputFile::create(Buffer);
-    if (!BitcodeFileOrErr)
-      return BitcodeFileOrErr.takeError();
-
-    // Save the input file and the buffer associated with its memory.
-    const auto Symbols = (*BitcodeFileOrErr)->symbols();
-    SmallVector<lto::SymbolResolution, 16> Resolutions(Symbols.size());
-    size_t Idx = 0;
-    for (auto &Sym : Symbols) {
-      lto::SymbolResolution &Res = Resolutions[Idx++];
-
-      // We will use this as the prevailing symbol definition in LTO unless
-      // it is undefined or another definition has already been used.
-      Res.Prevailing =
-          !Sym.isUndefined() &&
-          !(Sym.isWeak() && StrongResolutions.contains(Sym.getName())) &&
-          PrevailingSymbols.insert(Saver.save(Sym.getName())).second;
-
-      // We need LTO to preseve the following global symbols:
-      // 1) Symbols used in regular objects.
-      // 2) Sections that will be given a __start/__stop symbol.
-      // 3) Prevailing symbols that are needed visible to external
-      // libraries.
-      Res.VisibleToRegularObj =
-          UsedInRegularObj.contains(Sym.getName()) ||
-          isValidCIdentifier(Sym.getSectionName()) ||
-          (Res.Prevailing &&
-           (Sym.getVisibility() != GlobalValue::HiddenVisibility &&
-            !Sym.canBeOmittedFromSymbolTable()));
-
-      // Identify symbols that must be exported dynamically and can be
-      // referenced by other files.
-      Res.ExportDynamic =
-          Sym.getVisibility() != GlobalValue::HiddenVisibility &&
-          (UsedInSharedLib.contains(Sym.getName()) ||
-           !Sym.canBeOmittedFromSymbolTable());
-
-      // The final definition will reside in this linkage unit if the symbol
-      // is defined and local to the module. This only checks for bitcode
-      // files, full assertion will require complete symbol resolution.
-      Res.FinalDefinitionInLinkageUnit =
-          Sym.getVisibility() != GlobalValue::DefaultVisibility &&
-          (!Sym.isUndefined() && !Sym.isCommon());
-
-      // We do not support linker redefined symbols (e.g. --wrap) for device
-      // image linking, so the symbols will not be changed after LTO.
-      Res.LinkerRedefined = false;
-    }
-
-    // Add the bitcode file with its resolved symbols to the LTO job.
-    if (Error Err = LTOBackend->add(std::move(*BitcodeFileOrErr), Resolutions))
-      return Err;
-  }
-
-  // Run the LTO job to compile the bitcode.
-  size_t MaxTasks = LTOBackend->getMaxTasks();
-  SmallVector<StringRef> Files(MaxTasks);
-  auto AddStream =
-      [&](size_t Task,
-          const Twine &ModuleName) -> std::unique_ptr<CachedFileStream> {
-    int FD = -1;
-    auto &TempFile = Files[Task];
-    StringRef Extension = (Triple.isNVPTX() || SaveTemps) ? "s" : "o";
-    std::string TaskStr = Task ? "." + std::to_string(Task) : "";
-    auto TempFileOrErr =
-        createOutputFile(sys::path::filename(ExecutableName) + "." +
-                             Triple.getTriple() + "." + Arch + TaskStr,
-                         Extension);
-    if (!TempFileOrErr)
-      reportError(TempFileOrErr.takeError());
-    TempFile = *TempFileOrErr;
-    if (std::error_code EC = sys::fs::openFileForWrite(TempFile, FD))
-      reportError(errorCodeToError(EC));
-    return std::make_unique<CachedFileStream>(
-        std::make_unique<llvm::raw_fd_ostream>(FD, true));
-  };
-
-  if (Error Err = LTOBackend->run(AddStream))
-    return Err;
-
-  if (LTOError)
-    return createStringError("Errors encountered inside the LTO pipeline.");
-
-  // If we are embedding bitcode we only need the intermediate output.
-  bool SingleOutput = Files.size() == 1;
-  if (Args.hasArg(OPT_embed_bitcode)) {
-    if (BitcodeOutput.size() != 1 || !SingleOutput)
-      return createStringError("Cannot embed bitcode with multiple files.");
-    OutputFiles.push_back(Args.MakeArgString(BitcodeOutput.front()));
-    return Error::success();
-  }
-
-  // Append the new inputs to the device linker input. If the user requested
-  // an internalizing link we need to pass the bitcode to clang.
-  for (StringRef File :
-       Args.hasArg(OPT_clang_backend) || Args.hasArg(OPT_builtin_bitcode_EQ)
-           ? BitcodeOutput
-           : Files)
-    OutputFiles.push_back(File);
-
-  return Error::success();
-}
-
 Expected<StringRef> writeOffloadFile(const OffloadFile &File) {
   const OffloadBinary &Binary = *File.getBinary();
 
@@ -1327,15 +991,8 @@ Expected<SmallVector<StringRef>> linkAndWrapDeviceFiles(
       if (File.getBinary()->getOffloadKind() != OFK_None)
         ActiveOffloadKinds.insert(File.getBinary()->getOffloadKind());
 
-    // First link and remove all the input files containing bitcode if
-    // the target linker does not support it natively.
+    // Write any remaining device inputs to an output file.
     SmallVector<StringRef> InputFiles;
-    if (!linkerSupportsLTO(LinkerArgs))
-      if (Error Err = linkBitcodeFiles(Input, InputFiles, LinkerArgs))
-        return Err;
-
-    // Write any remaining device inputs to an output file for the
-    // linker.
     for (const OffloadFile &File : Input) {
       auto FileNameOrErr = writeOffloadFile(File);
       if (!FileNameOrErr)
@@ -1344,10 +1001,7 @@ Expected<SmallVector<StringRef>> linkAndWrapDeviceFiles(
     }
 
     // Link the remaining device files using the device linker.
-    auto OutputOrErr =
-        !Args.hasArg(OPT_embed_bitcode) || linkerSupportsLTO(LinkerArgs)
-            ? linkDevice(InputFiles, LinkerArgs)
-            : InputFiles.front();
+    auto OutputOrErr = linkDevice(InputFiles, LinkerArgs);
     if (!OutputOrErr)
       return OutputOrErr.takeError();
 
-- 
GitLab


From 70af40ba74cf62fdaa3ae1d7db972c138655049f Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston@google.com>
Date: Tue, 29 Oct 2024 20:14:14 +0000
Subject: [PATCH 499/693] [hwasan] Fix forward '[hwasan] Flush stderr/stdout in
 tests (#114083)'

3754fc1e9af38951aa00181c0e8110174d3f94fd broke the build because subsequent checks depend on the line numbers

https://lab.llvm.org/buildbot/#/builders/174/builds/7534/steps/6/logs/FAIL__HWAddressSanitizer-x86_64__use-after-free_c
---
 compiler-rt/test/hwasan/TestCases/use-after-free.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/test/hwasan/TestCases/use-after-free.c b/compiler-rt/test/hwasan/TestCases/use-after-free.c
index b4b79875e811..fe4f8b32ea10 100644
--- a/compiler-rt/test/hwasan/TestCases/use-after-free.c
+++ b/compiler-rt/test/hwasan/TestCases/use-after-free.c
@@ -32,11 +32,11 @@ int main() {
   //
   // CHECK: freed by thread {{.*}} here:
   // CHECK: #0 {{.*}} in {{.*}}free{{.*}} {{.*}}hwasan_allocation_functions.cpp
-  // CHECK: #1 {{.*}} in main {{.*}}use-after-free.c:[[@LINE-19]]
+  // CHECK: #1 {{.*}} in main {{.*}}use-after-free.c:[[@LINE-20]]
 
   // CHECK: previously allocated by thread {{.*}} here:
   // CHECK: #0 {{.*}} in {{.*}}malloc{{.*}} {{.*}}hwasan_allocation_functions.cpp
-  // CHECK: #1 {{.*}} in main {{.*}}use-after-free.c:[[@LINE-24]]
+  // CHECK: #1 {{.*}} in main {{.*}}use-after-free.c:[[@LINE-25]]
   // CHECK: Memory tags around the buggy address (one tag corresponds to 16 bytes):
   // CHECK: =>{{.*}}[[MEM_TAG]]
   // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in main
-- 
GitLab


From 8193832fb988e3df1e8e726634783805dca8d9b6 Mon Sep 17 00:00:00 2001
From: Zequan Wu <zequanwu@google.com>
Date: Tue, 29 Oct 2024 13:23:33 -0700
Subject: [PATCH 500/693] [lldb] Search main function with
 lldb::eFunctionNameTypeFull when getting default file and line. (#113980)

This is to work around the fact that
`SymbolFileNativePDB::FindFunctions` only support
`lldb::eFunctionNameTypeFull` and `lldb::eFunctionNameTypeMethod` now.
Since `main`'s full name is the same as base name (`main`), it's okay to
search with `lldb::eFunctionNameTypeFull` when trying to get the default
file and line. With this, `lldb/test/Shell/Driver/TestSingleQuote.test`
passes on Windows with NativePDB plugin.
---
 lldb/source/Core/SourceManager.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/source/Core/SourceManager.cpp b/lldb/source/Core/SourceManager.cpp
index fd5b49946c6a..27a9edeef424 100644
--- a/lldb/source/Core/SourceManager.cpp
+++ b/lldb/source/Core/SourceManager.cpp
@@ -430,7 +430,7 @@ SourceManager::GetDefaultFileAndLine() {
             false; // Force it to be a debug symbol.
         function_options.include_inlines = true;
         executable_ptr->FindFunctions(main_name, CompilerDeclContext(),
-                                      lldb::eFunctionNameTypeBase,
+                                      lldb::eFunctionNameTypeFull,
                                       function_options, sc_list);
         for (const SymbolContext &sc : sc_list) {
           if (sc.function) {
-- 
GitLab


From a78861fc55d18046989ff4d624a037e9181da170 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Tue, 29 Oct 2024 13:34:28 -0700
Subject: [PATCH 501/693] [NvlinkWrapper] Add support for `--undefined`
 (#113934)

Summary:
This flag is pretty canonical in ELF linkers, it allows us to force the
link job to extract a library if it defines a specific symbol. This is
mostly useful for letting us forcibly extract things that don't fit the
normal model (i.e. kernels) from static libraries.
---
 clang/test/Driver/nvlink-wrapper.c                  | 13 ++++++++++---
 .../clang-nvlink-wrapper/ClangNVLinkWrapper.cpp     |  3 +++
 clang/tools/clang-nvlink-wrapper/NVLinkOpts.td      |  7 +++++--
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/clang/test/Driver/nvlink-wrapper.c b/clang/test/Driver/nvlink-wrapper.c
index 2b0993caee42..79f4a6641732 100644
--- a/clang/test/Driver/nvlink-wrapper.c
+++ b/clang/test/Driver/nvlink-wrapper.c
@@ -21,12 +21,13 @@ int bar() {
 }
 #else
 extern int y;
-int __attribute__((visibility("hidden"))) x = 999;
+extern int x;
 int baz() { return y + x; }
 #endif
 
 // Create various inputs to test basic linking and LTO capabilities. Creating a
 // CUDA binary requires access to the `ptxas` executable, so we just use x64.
+// RUN: %clang -cc1 %s -triple nvptx64-nvidia-cuda -emit-llvm-bc -o %t.o
 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -DX -o %t-x.o
 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -DY -o %t-y.o
 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -DZ -o %t-z.o
@@ -36,6 +37,7 @@ int baz() { return y + x; }
 // RUN: llvm-ar rcs %t-y.a %t-y.o
 // RUN: llvm-ar rcs %t-z.a %t-z.o
 // RUN: llvm-ar rcs %t-w.a %t-w.o
+// RUN: llvm-ar rcs %t-u.a %t-u.o
 
 //
 // Check that we forward any unrecognized argument to 'nvlink'.
@@ -49,11 +51,16 @@ int baz() { return y + x; }
 // `libx.a` and `liby.a` because extern weak symbols do not extract and `libz.a`
 // is not used at all.
 //
-// RUN: clang-nvlink-wrapper --dry-run %t-x.a %t-u.o %t-y.a %t-z.a %t-w.a \
+// RUN: clang-nvlink-wrapper --dry-run %t-x.a %t-u.a %t-y.a %t-z.a %t-w.a %t.o \
 // RUN:   -arch sm_52 -o a.out 2>&1 | FileCheck %s --check-prefix=LINK
 // LINK: nvlink{{.*}} -arch sm_52 -o a.out [[INPUT:.+]].cubin {{.*}}-x-{{.*}}.cubin{{.*}}-y-{{.*}}.cubin
 
-// RUN: %clang -cc1 %s -triple nvptx64-nvidia-cuda -emit-llvm-bc -o %t.o
+//
+// Same as above but we use '--undefined' to forcibly extract 'libz.a'
+//
+// RUN: clang-nvlink-wrapper --dry-run %t-x.a %t-u.a %t-y.a %t-z.a %t-w.a %t.o \
+// RUN:   -u z -arch sm_52 -o a.out 2>&1 | FileCheck %s --check-prefix=LINK
+// UNDEFINED: nvlink{{.*}} -arch sm_52 -o a.out [[INPUT:.+]].cubin {{.*}}-x-{{.*}}.cubin{{.*}}-y-{{.*}}.cubin{{.*}}-z-{{.*}}.cubin
 
 //
 // Check that the LTO interface works and properly preserves symbols used in a
diff --git a/clang/tools/clang-nvlink-wrapper/ClangNVLinkWrapper.cpp b/clang/tools/clang-nvlink-wrapper/ClangNVLinkWrapper.cpp
index b9767a7a03d0..bc191afdca73 100644
--- a/clang/tools/clang-nvlink-wrapper/ClangNVLinkWrapper.cpp
+++ b/clang/tools/clang-nvlink-wrapper/ClangNVLinkWrapper.cpp
@@ -250,6 +250,7 @@ struct Symbol {
   };
 
   Symbol() : File(), Flags(None), UsedInRegularObj(false) {}
+  Symbol(Symbol::Flags Flags) : File(), Flags(Flags), UsedInRegularObj(true) {}
 
   Symbol(MemoryBufferRef File, const irsymtab::Reader::SymbolRef Sym)
       : File(File), Flags(0), UsedInRegularObj(false) {
@@ -535,6 +536,8 @@ Expected<SmallVector<StringRef>> getInput(const ArgList &Args) {
 
   bool Extracted = true;
   StringMap<Symbol> SymTab;
+  for (auto &Sym : Args.getAllArgValues(OPT_u))
+    SymTab[Sym] = Symbol(Symbol::Undefined);
   SmallVector<std::unique_ptr<MemoryBuffer>> LinkerInput;
   while (Extracted) {
     Extracted = false;
diff --git a/clang/tools/clang-nvlink-wrapper/NVLinkOpts.td b/clang/tools/clang-nvlink-wrapper/NVLinkOpts.td
index a80c5937b429..6de1a25c14f8 100644
--- a/clang/tools/clang-nvlink-wrapper/NVLinkOpts.td
+++ b/clang/tools/clang-nvlink-wrapper/NVLinkOpts.td
@@ -43,11 +43,11 @@ def plugin : JoinedOrSeparate<["--", "-"], "plugin">,
   Flags<[HelpHidden, WrapperOnlyOption]>;
 
 def arch : Separate<["--", "-"], "arch">,
-  HelpText<"Specify the 'sm_' name of the target architecture.">;
+  HelpText<"Specify the 'sm_' name of the target architecture">;
 def : Joined<["--", "-"], "plugin-opt=mcpu=">,
   Flags<[HelpHidden, WrapperOnlyOption]>, Alias<arch>;
 
-def g : Flag<["-"], "g">, HelpText<"Specify that this was a debug compile.">;
+def g : Flag<["-"], "g">, HelpText<"Specify that this was a debug compile">;
 def debug : Flag<["--"], "debug">, Alias<g>;
 
 def lto_emit_llvm : Flag<["--"], "lto-emit-llvm">, Flags<[WrapperOnlyOption]>,
@@ -55,6 +55,9 @@ def lto_emit_llvm : Flag<["--"], "lto-emit-llvm">, Flags<[WrapperOnlyOption]>,
 def lto_emit_asm : Flag<["--"], "lto-emit-asm">, Flags<[WrapperOnlyOption]>,
   HelpText<"Emit assembly code">;
 
+def u : JoinedOrSeparate<["-"], "u">, HelpText<"Force undefined symbol during linking">;
+def undefined : JoinedOrSeparate<["--"], "undefined">, Alias<u>;
+
 def O : Joined<["--", "-"], "plugin-opt=O">,
   Flags<[WrapperOnlyOption]>, MetaVarName<"<O0, O1, O2, or O3>">,
   HelpText<"Optimization level for LTO">;
-- 
GitLab


From 41baa69a7e2ab3df13334565aa6ccdae1b0113ad Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 29 Oct 2024 13:52:22 -0700
Subject: [PATCH 502/693] [BOLT] Fix warnings (#114116)

This patch fixes:

  bolt/lib/Core/BinaryFunction.cpp:2537:13: error: enumeration value
  'OpNegateRAStateWithPC' not handled in switch [-Werror,-Wswitch]

  bolt/lib/Core/BinaryFunction.cpp:2661:13: error: enumeration value
  'OpNegateRAStateWithPC' not handled in switch [-Werror,-Wswitch]

  bolt/lib/Core/BinaryFunction.cpp:2805:13: error: enumeration value
  'OpNegateRAStateWithPC' not handled in switch [-Werror,-Wswitch]
---
 bolt/lib/Core/BinaryFunction.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index ef3fba37817d..c12217d54947 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -2577,6 +2577,7 @@ private:
     case MCCFIInstruction::OpAdjustCfaOffset:
     case MCCFIInstruction::OpWindowSave:
     case MCCFIInstruction::OpNegateRAState:
+    case MCCFIInstruction::OpNegateRAStateWithPC:
     case MCCFIInstruction::OpLLVMDefAspaceCfa:
     case MCCFIInstruction::OpLabel:
       llvm_unreachable("unsupported CFI opcode");
@@ -2715,6 +2716,7 @@ struct CFISnapshotDiff : public CFISnapshot {
     case MCCFIInstruction::OpAdjustCfaOffset:
     case MCCFIInstruction::OpWindowSave:
     case MCCFIInstruction::OpNegateRAState:
+    case MCCFIInstruction::OpNegateRAStateWithPC:
     case MCCFIInstruction::OpLLVMDefAspaceCfa:
     case MCCFIInstruction::OpLabel:
       llvm_unreachable("unsupported CFI opcode");
@@ -2864,6 +2866,7 @@ BinaryFunction::unwindCFIState(int32_t FromState, int32_t ToState,
     case MCCFIInstruction::OpAdjustCfaOffset:
     case MCCFIInstruction::OpWindowSave:
     case MCCFIInstruction::OpNegateRAState:
+    case MCCFIInstruction::OpNegateRAStateWithPC:
     case MCCFIInstruction::OpLLVMDefAspaceCfa:
     case MCCFIInstruction::OpLabel:
       llvm_unreachable("unsupported CFI opcode");
-- 
GitLab


From 94e7d9c0bfe517507ea08b00fb00c32fb2837a82 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 29 Oct 2024 13:50:41 -0700
Subject: [PATCH 503/693] [RISCV] Remove Zvk* dependency checks from
 RISCVISAInfo::checkDependency.

The Zvk* extensions now imply Zve32x or Zve64x so it shouldn't be
possible to fail these dependency checks.
---
 llvm/lib/TargetParser/RISCVISAInfo.cpp | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index caa5a97747ee..de5b5c39c9ed 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -751,17 +751,6 @@ Error RISCVISAInfo::checkDependency() {
   if (HasZvl && !HasVector)
     return getExtensionRequiresError("zvl*b", "v' or 'zve*");
 
-  if (!HasVector)
-    for (auto Ext :
-         {"zvbb", "zvbc32e", "zvkb", "zvkg", "zvkgs", "zvkned", "zvknha", "zvksed", "zvksh"})
-      if (Exts.count(Ext))
-        return getExtensionRequiresError(Ext, "v' or 'zve*");
-
-  if (!Exts.count("zve64x"))
-    for (auto Ext : {"zvknhb", "zvbc"})
-      if (Exts.count(Ext))
-        return getExtensionRequiresError(Ext, "v' or 'zve64*");
-
   if ((HasZcmt || Exts.count("zcmp")) && HasD && (HasC || Exts.count("zcd")))
     return getError(Twine("'") + (HasZcmt ? "zcmt" : "zcmp") +
                     "' extension is incompatible with '" +
-- 
GitLab


From 83ae171722bea2722afa4efb0558a6d8b8844305 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Tue, 29 Oct 2024 21:02:32 +0000
Subject: [PATCH 504/693] [AArch64] Add ComputeNumSignBits for VASHR. (#113957)

As with a normal ISD::SRA node, they take the number of sign bits of the
incoming value and increase it by the shifted amount.
---
 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp    |  5 +++++
 llvm/test/CodeGen/AArch64/arm64-vshift.ll          | 12 ++++++++++++
 llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp | 13 +++++++++++++
 3 files changed, 30 insertions(+)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 32ba2866ac81..31a720ed7b5c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2536,6 +2536,11 @@ unsigned AArch64TargetLowering::ComputeNumSignBitsForTargetNode(
     case AArch64ISD::FCMLTz:
       // Compares return either 0 or all-ones
       return VTBits;
+    case AArch64ISD::VASHR: {
+      unsigned Tmp =
+          DAG.ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
+      return std::min<uint64_t>(Tmp + Op.getConstantOperandVal(1), VTBits);
+    }
   }
 
   return 1;
diff --git a/llvm/test/CodeGen/AArch64/arm64-vshift.ll b/llvm/test/CodeGen/AArch64/arm64-vshift.ll
index 1dfd977186b0..7af7c235f9ac 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vshift.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vshift.ll
@@ -3560,4 +3560,16 @@ entry:
   ret <4 x i16> %vrshrn_n1
 }
 
+define <8 x i16> @signbits_vashr(<8 x i16> %a)  {
+; CHECK-LABEL: signbits_vashr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sshr.8h v0, v0, #8
+; CHECK-NEXT:    sshr.8h v0, v0, #9
+; CHECK-NEXT:    ret
+  %b = call <8 x i16> @llvm.aarch64.neon.sshl.v8i16(<8 x i16> %a, <8 x i16> <i16 -8, i16 -8, i16 -8, i16 -8, i16 -8, i16 -8, i16 -8, i16 -8>)
+  %c = call <8 x i16> @llvm.aarch64.neon.sshl.v8i16(<8 x i16> %b, <8 x i16> <i16 -9, i16 -9, i16 -9, i16 -9, i16 -9, i16 -9, i16 -9, i16 -9>)
+  %d = ashr <8 x i16> %c, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
+  ret <8 x i16> %d
+}
+
 declare <2 x i64> @llvm.aarch64.neon.addp.v2i64(<2 x i64>, <2 x i64>)
diff --git a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
index 3df72ec8115b..ffedb2c74220 100644
--- a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
+++ b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "../lib/Target/AArch64/AArch64ISelLowering.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/AsmParser/Parser.h"
@@ -167,6 +168,18 @@ TEST_F(AArch64SelectionDAGTest, ComputeNumSignBits_EXTRACT_SUBVECTOR) {
   EXPECT_EQ(DAG->ComputeNumSignBits(Op, DemandedElts), 7u);
 }
 
+TEST_F(AArch64SelectionDAGTest, ComputeNumSignBits_VASHR) {
+  SDLoc Loc;
+  auto VecVT = MVT::v8i8;
+  auto Shift = DAG->getConstant(4, Loc, MVT::i32);
+  auto Vec0 = DAG->getConstant(1, Loc, VecVT);
+  auto Op1 = DAG->getNode(AArch64ISD::VASHR, Loc, VecVT, Vec0, Shift);
+  EXPECT_EQ(DAG->ComputeNumSignBits(Op1), 8u);
+  auto VecA = DAG->getConstant(0xaa, Loc, VecVT);
+  auto Op2 = DAG->getNode(AArch64ISD::VASHR, Loc, VecVT, VecA, Shift);
+  EXPECT_EQ(DAG->ComputeNumSignBits(Op2), 5u);
+}
+
 TEST_F(AArch64SelectionDAGTest, SimplifyDemandedVectorElts_EXTRACT_SUBVECTOR) {
   TargetLowering TL(*TM);
 
-- 
GitLab


From 680901ed8010319843cd81275b845d682f77e27f Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Tue, 29 Oct 2024 21:04:31 +0000
Subject: [PATCH 505/693] [VPlan] Implement VPHeaderPHIRecipe::computeCost.

Fill out computeCost implementations for various header PHI recipes,
matching the legacy cost model for now.
---
 llvm/lib/Transforms/Vectorize/VPlan.h         | 22 +++++++++++++++++++
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 22 +++++++++++++++++++
 2 files changed, 44 insertions(+)

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index a34e34a0d71f..8d6025c89f72 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2050,6 +2050,10 @@ public:
   /// Generate the phi nodes.
   void execute(VPTransformState &State) override = 0;
 
+  /// Return the cost of this header phi recipe.
+  InstructionCost computeCost(ElementCount VF,
+                              VPCostContext &Ctx) const override;
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
@@ -2295,6 +2299,10 @@ struct VPFirstOrderRecurrencePHIRecipe : public VPHeaderPHIRecipe {
 
   void execute(VPTransformState &State) override;
 
+  /// Return the cost of this first-order recurrence phi recipe.
+  InstructionCost computeCost(ElementCount VF,
+                              VPCostContext &Ctx) const override;
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
@@ -3134,6 +3142,13 @@ public:
   /// canonical, i.e.  has the same start and step (of 1) as the canonical IV.
   bool isCanonical(InductionDescriptor::InductionKind Kind, VPValue *Start,
                    VPValue *Step) const;
+
+  /// Return the cost of this VPCanonicalIVPHIRecipe.
+  InstructionCost computeCost(ElementCount VF,
+                              VPCostContext &Ctx) const override {
+    // For now, match the behavior of the legacy cost model.
+    return 0;
+  }
 };
 
 /// A recipe for generating the active lane mask for the vector loop that is
@@ -3196,6 +3211,13 @@ public:
   /// TODO: investigate if it can share the code with VPCanonicalIVPHIRecipe.
   void execute(VPTransformState &State) override;
 
+  /// Return the cost of this VPEVLBasedIVPHIRecipe.
+  InstructionCost computeCost(ElementCount VF,
+                              VPCostContext &Ctx) const override {
+    // For now, match the behavior of the legacy cost model.
+    return 0;
+  }
+
   /// Returns true if the recipe only uses the first lane of operand \p Op.
   bool onlyFirstLaneUsed(const VPValue *Op) const override {
     assert(is_contained(operands(), Op) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b1e6086398c4..de7023167df8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1589,6 +1589,11 @@ void VPWidenCastRecipe::print(raw_ostream &O, const Twine &Indent,
 }
 #endif
 
+InstructionCost VPHeaderPHIRecipe::computeCost(ElementCount VF,
+                                               VPCostContext &Ctx) const {
+  return Ctx.TTI.getCFInstrCost(Instruction::PHI, TTI::TCK_RecipThroughput);
+}
+
 /// This function adds
 /// (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step, ...)
 /// to each vector element of Val. The sequence starts at StartIndex.
@@ -3334,6 +3339,23 @@ void VPFirstOrderRecurrencePHIRecipe::execute(VPTransformState &State) {
   State.set(this, Phi);
 }
 
+InstructionCost
+VPFirstOrderRecurrencePHIRecipe::computeCost(ElementCount VF,
+                                             VPCostContext &Ctx) const {
+  if (VF.isScalable() && VF.getKnownMinValue() == 1)
+    return InstructionCost::getInvalid();
+
+  SmallVector<int> Mask(VF.getKnownMinValue());
+  std::iota(Mask.begin(), Mask.end(), VF.getKnownMinValue() - 1);
+  Type *VectorTy =
+      ToVectorTy(Ctx.Types.inferScalarType(this->getVPSingleValue()), VF);
+
+  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+  return Ctx.TTI.getShuffleCost(TargetTransformInfo::SK_Splice,
+                                cast<VectorType>(VectorTy), Mask, CostKind,
+                                VF.getKnownMinValue() - 1);
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPFirstOrderRecurrencePHIRecipe::print(raw_ostream &O, const Twine &Indent,
                                             VPSlotTracker &SlotTracker) const {
-- 
GitLab


From 4a96081224b6c0f166760eab0c42ef3dfadd5ed1 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 29 Oct 2024 14:12:14 -0700
Subject: [PATCH 506/693] [clang-linker-wrapper] Fix a warning

This patch fixes:

  clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp:642:6:
  error: unused function 'diagnosticHandler'
  [-Werror,-Wunused-function]
---
 .../ClangLinkerWrapper.cpp                    | 22 -------------------
 1 file changed, 22 deletions(-)

diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 45274b797236..561b73c73ad7 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -639,28 +639,6 @@ Expected<StringRef> linkDevice(ArrayRef<StringRef> InputFiles,
   }
 }
 
-void diagnosticHandler(const DiagnosticInfo &DI) {
-  std::string ErrStorage;
-  raw_string_ostream OS(ErrStorage);
-  DiagnosticPrinterRawOStream DP(OS);
-  DI.print(DP);
-
-  switch (DI.getSeverity()) {
-  case DS_Error:
-    WithColor::error(errs(), LinkerExecutable) << ErrStorage << "\n";
-    break;
-  case DS_Warning:
-    WithColor::warning(errs(), LinkerExecutable) << ErrStorage << "\n";
-    break;
-  case DS_Note:
-    WithColor::note(errs(), LinkerExecutable) << ErrStorage << "\n";
-    break;
-  case DS_Remark:
-    WithColor::remark(errs()) << ErrStorage << "\n";
-    break;
-  }
-}
-
 Expected<StringRef> writeOffloadFile(const OffloadFile &File) {
   const OffloadBinary &Binary = *File.getBinary();
 
-- 
GitLab


From a325c5359310316e393e7e446373fca645002ecb Mon Sep 17 00:00:00 2001
From: Zequan Wu <zequanwu@google.com>
Date: Tue, 29 Oct 2024 14:38:02 -0700
Subject: [PATCH 507/693] [lldb] Fix lldb windows build breakage from
 https://github.com/llvm/llvm-project/pull/112657.

LLDB windows build failure: https://lab.llvm.org/buildbot/#/builders/141/builds/3462
---
 lldb/unittests/Host/FileActionTest.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lldb/unittests/Host/FileActionTest.cpp b/lldb/unittests/Host/FileActionTest.cpp
index 3d2c722552c9..56227cd587e5 100644
--- a/lldb/unittests/Host/FileActionTest.cpp
+++ b/lldb/unittests/Host/FileActionTest.cpp
@@ -10,6 +10,9 @@
 
 #include "lldb/Host/FileAction.h"
 #include "gtest/gtest.h"
+#if defined(_WIN32)
+#include "lldb/Host/windows/PosixApi.h"
+#endif
 
 using namespace lldb_private;
 
-- 
GitLab


From 5cfb07a5d067f7729a1578c7272fb314a89c8596 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 29 Oct 2024 14:46:51 -0700
Subject: [PATCH 508/693] Revert "[ADT] Use std::string_view inside StringRef
 (#113775)" (#114133)

This patch reverts commit 89b5d88fb81362b4fb2f833790aa40b7eaa186da.

Some sanitizer failures have been reported, indicating that StringRef
and std::string_view handle data == nulptr differently.  Also, they
support different values for the max size (size_t v.s. ptrdiff_t).

Thanks goes to Jorge Gorbe Moya for reporting these.
---
 llvm/include/llvm/ADT/StringRef.h | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h
index 0dcd4d90086e..5b525c8e56ec 100644
--- a/llvm/include/llvm/ADT/StringRef.h
+++ b/llvm/include/llvm/ADT/StringRef.h
@@ -60,7 +60,11 @@ namespace llvm {
     using const_reverse_iterator = std::reverse_iterator<const_iterator>;
 
   private:
-    std::string_view View;
+    /// The start of the string, in an external buffer.
+    const char *Data = nullptr;
+
+    /// The length of the string.
+    size_t Length = 0;
 
     // Workaround memcmp issue with null pointers (undefined behavior)
     // by providing a specialized version
@@ -82,26 +86,28 @@ namespace llvm {
 
     /// Construct a string ref from a cstring.
     /*implicit*/ constexpr StringRef(const char *Str LLVM_LIFETIME_BOUND)
-        : View(Str, Str ?
+        : Data(Str), Length(Str ?
     // GCC 7 doesn't have constexpr char_traits. Fall back to __builtin_strlen.
 #if defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 8
-                        __builtin_strlen(Str)
+                                __builtin_strlen(Str)
 #else
-                        std::char_traits<char>::length(Str)
+                                std::char_traits<char>::length(Str)
 #endif
-                        : 0) {
+                                : 0) {
     }
 
     /// Construct a string ref from a pointer and length.
     /*implicit*/ constexpr StringRef(const char *data LLVM_LIFETIME_BOUND,
                                      size_t length)
-        : View(data, length) {}
+        : Data(data), Length(length) {}
 
     /// Construct a string ref from an std::string.
-    /*implicit*/ StringRef(const std::string &Str) : View(Str) {}
+    /*implicit*/ StringRef(const std::string &Str)
+        : Data(Str.data()), Length(Str.length()) {}
 
     /// Construct a string ref from an std::string_view.
-    /*implicit*/ constexpr StringRef(std::string_view Str) : View(Str) {}
+    /*implicit*/ constexpr StringRef(std::string_view Str)
+        : Data(Str.data()), Length(Str.size()) {}
 
     /// @}
     /// @name Iterators
@@ -135,13 +141,13 @@ namespace llvm {
 
     /// data - Get a pointer to the start of the string (which may not be null
     /// terminated).
-    [[nodiscard]] constexpr const char *data() const { return View.data(); }
+    [[nodiscard]] constexpr const char *data() const { return Data; }
 
     /// empty - Check if the string is empty.
     [[nodiscard]] constexpr bool empty() const { return size() == 0; }
 
     /// size - Get the string size.
-    [[nodiscard]] constexpr size_t size() const { return View.size(); }
+    [[nodiscard]] constexpr size_t size() const { return Length; }
 
     /// front - Get the first character in the string.
     [[nodiscard]] char front() const {
-- 
GitLab


From 8b55162e195783dd27e1c69fb4d97971ef76725b Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Tue, 29 Oct 2024 23:56:10 +0200
Subject: [PATCH 509/693] [RISCV] Add cost model tests for scalable FP
 reductions. NFC

There are already some in reduce-scalable-fp.ll but this makes it a
bit easier to see the difference alongside their fixed-length
counterparts.
---
 .../Analysis/CostModel/RISCV/reduce-fadd.ll   | 181 ++++++++++++++----
 .../Analysis/CostModel/RISCV/reduce-fmul.ll   | 132 +++++++++++++
 2 files changed, 276 insertions(+), 37 deletions(-)

diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fadd.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fadd.ll
index 588d852d7f26..196e7376677a 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-fadd.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fadd.ll
@@ -13,6 +13,12 @@ define void @reduce_fadd_bfloat() {
 ; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %v32 = call fast bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %V64 = call fast bfloat @llvm.vector.reduce.fadd.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %V128 = call fast bfloat @llvm.vector.reduce.fadd.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast bfloat @llvm.vector.reduce.fadd.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call fast bfloat @llvm.vector.reduce.fadd.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call fast bfloat @llvm.vector.reduce.fadd.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_fadd_bfloat'
@@ -24,6 +30,12 @@ define void @reduce_fadd_bfloat() {
 ; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %v32 = call fast bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
 ; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %V64 = call fast bfloat @llvm.vector.reduce.fadd.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
 ; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %V128 = call fast bfloat @llvm.vector.reduce.fadd.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast bfloat @llvm.vector.reduce.fadd.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call fast bfloat @llvm.vector.reduce.fadd.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call fast bfloat @llvm.vector.reduce.fadd.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call fast bfloat @llvm.vector.reduce.fadd.v1bf16(bfloat 0.0, <1 x bfloat> undef)
@@ -34,6 +46,12 @@ define void @reduce_fadd_bfloat() {
   %v32 = call fast bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0.0, <32 x bfloat> undef)
   %V64 = call fast bfloat @llvm.vector.reduce.fadd.v64bf16(bfloat 0.0, <64 x bfloat> undef)
   %V128 = call fast bfloat @llvm.vector.reduce.fadd.v128bf16(bfloat 0.0, <128 x bfloat> undef)
+  %NXV1 = call fast bfloat @llvm.vector.reduce.fadd.nxv1bf16(bfloat 0.0, <vscale x 1 x bfloat> undef)
+  %NXV2 = call fast bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat 0.0, <vscale x 2 x bfloat> undef)
+  %NXV4 = call fast bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat 0.0, <vscale x 4 x bfloat> undef)
+  %NXV8 = call fast bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat 0.0, <vscale x 8 x bfloat> undef)
+  %NXV16 = call fast bfloat @llvm.vector.reduce.fadd.nxv16bf16(bfloat 0.0, <vscale x 16 x bfloat> undef)
+  %NXV32 = call fast bfloat @llvm.vector.reduce.fadd.nxv32bf16(bfloat 0.0, <vscale x 32 x bfloat> undef)
   ret void
 }
 
@@ -47,6 +65,12 @@ define void @reduce_fadd_half() {
 ; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32 = call fast half @llvm.vector.reduce.fadd.v32f16(half 0xH0000, <32 x half> undef)
 ; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call fast half @llvm.vector.reduce.fadd.v64f16(half 0xH0000, <64 x half> undef)
 ; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128 = call fast half @llvm.vector.reduce.fadd.v128f16(half 0xH0000, <128 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV1 = call fast half @llvm.vector.reduce.fadd.nxv1f16(half 0xH0000, <vscale x 1 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2 = call fast half @llvm.vector.reduce.fadd.nxv2f16(half 0xH0000, <vscale x 2 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NXV4 = call fast half @llvm.vector.reduce.fadd.nxv4f16(half 0xH0000, <vscale x 4 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %NXV8 = call fast half @llvm.vector.reduce.fadd.nxv8f16(half 0xH0000, <vscale x 8 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %NXV16 = call fast half @llvm.vector.reduce.fadd.nxv16f16(half 0xH0000, <vscale x 16 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32 = call fast half @llvm.vector.reduce.fadd.nxv32f16(half 0xH0000, <vscale x 32 x half> undef)
 ; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; FP-REDUCE-ZVFHMIN-LABEL: 'reduce_fadd_half'
@@ -58,6 +82,12 @@ define void @reduce_fadd_half() {
 ; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %v32 = call fast half @llvm.vector.reduce.fadd.v32f16(half 0xH0000, <32 x half> undef)
 ; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %V64 = call fast half @llvm.vector.reduce.fadd.v64f16(half 0xH0000, <64 x half> undef)
 ; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %V128 = call fast half @llvm.vector.reduce.fadd.v128f16(half 0xH0000, <128 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast half @llvm.vector.reduce.fadd.nxv1f16(half 0xH0000, <vscale x 1 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast half @llvm.vector.reduce.fadd.nxv2f16(half 0xH0000, <vscale x 2 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast half @llvm.vector.reduce.fadd.nxv4f16(half 0xH0000, <vscale x 4 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast half @llvm.vector.reduce.fadd.nxv8f16(half 0xH0000, <vscale x 8 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call fast half @llvm.vector.reduce.fadd.nxv16f16(half 0xH0000, <vscale x 16 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call fast half @llvm.vector.reduce.fadd.nxv32f16(half 0xH0000, <vscale x 32 x half> undef)
 ; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_fadd_half'
@@ -69,6 +99,12 @@ define void @reduce_fadd_half() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32 = call fast half @llvm.vector.reduce.fadd.v32f16(half 0xH0000, <32 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call fast half @llvm.vector.reduce.fadd.v64f16(half 0xH0000, <64 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128 = call fast half @llvm.vector.reduce.fadd.v128f16(half 0xH0000, <128 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV1 = call fast half @llvm.vector.reduce.fadd.nxv1f16(half 0xH0000, <vscale x 1 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV2 = call fast half @llvm.vector.reduce.fadd.nxv2f16(half 0xH0000, <vscale x 2 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV4 = call fast half @llvm.vector.reduce.fadd.nxv4f16(half 0xH0000, <vscale x 4 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV8 = call fast half @llvm.vector.reduce.fadd.nxv8f16(half 0xH0000, <vscale x 8 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV16 = call fast half @llvm.vector.reduce.fadd.nxv16f16(half 0xH0000, <vscale x 16 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV32 = call fast half @llvm.vector.reduce.fadd.nxv32f16(half 0xH0000, <vscale x 32 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call fast half @llvm.vector.reduce.fadd.v1f16(half 0.0, <1 x half> undef)
@@ -79,6 +115,12 @@ define void @reduce_fadd_half() {
   %v32 = call fast half @llvm.vector.reduce.fadd.v32f16(half 0.0, <32 x half> undef)
   %V64 = call fast half @llvm.vector.reduce.fadd.v64f16(half 0.0, <64 x half> undef)
   %V128 = call fast half @llvm.vector.reduce.fadd.v128f16(half 0.0, <128 x half> undef)
+  %NXV1 = call fast half @llvm.vector.reduce.fadd.nxv1f16(half 0.0, <vscale x 1 x half> undef)
+  %NXV2 = call fast half @llvm.vector.reduce.fadd.nxv2f16(half 0.0, <vscale x 2 x half> undef)
+  %NXV4 = call fast half @llvm.vector.reduce.fadd.nxv4f16(half 0.0, <vscale x 4 x half> undef)
+  %NXV8 = call fast half @llvm.vector.reduce.fadd.nxv8f16(half 0.0, <vscale x 8 x half> undef)
+  %NXV16 = call fast half @llvm.vector.reduce.fadd.nxv16f16(half 0.0, <vscale x 16 x half> undef)
+  %NXV32 = call fast half @llvm.vector.reduce.fadd.nxv32f16(half 0.0, <vscale x 32 x half> undef)
   ret void
 }
 
@@ -92,6 +134,11 @@ define void @reduce_fadd_float() {
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32 = call fast float @llvm.vector.reduce.fadd.v32f32(float 0.000000e+00, <32 x float> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V64 = call fast float @llvm.vector.reduce.fadd.v64f32(float 0.000000e+00, <64 x float> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V128 = call fast float @llvm.vector.reduce.fadd.v128f32(float 0.000000e+00, <128 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV1 = call fast float @llvm.vector.reduce.fadd.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2 = call fast float @llvm.vector.reduce.fadd.nxv2f32(float 0.000000e+00, <vscale x 2 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NXV4 = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %NXV8 = call fast float @llvm.vector.reduce.fadd.nxv8f32(float 0.000000e+00, <vscale x 8 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %NXV16 = call fast float @llvm.vector.reduce.fadd.nxv16f32(float 0.000000e+00, <vscale x 16 x float> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_fadd_float'
@@ -103,6 +150,11 @@ define void @reduce_fadd_float() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32 = call fast float @llvm.vector.reduce.fadd.v32f32(float 0.000000e+00, <32 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64 = call fast float @llvm.vector.reduce.fadd.v64f32(float 0.000000e+00, <64 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V128 = call fast float @llvm.vector.reduce.fadd.v128f32(float 0.000000e+00, <128 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV1 = call fast float @llvm.vector.reduce.fadd.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV2 = call fast float @llvm.vector.reduce.fadd.nxv2f32(float 0.000000e+00, <vscale x 2 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV4 = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV8 = call fast float @llvm.vector.reduce.fadd.nxv8f32(float 0.000000e+00, <vscale x 8 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV16 = call fast float @llvm.vector.reduce.fadd.nxv16f32(float 0.000000e+00, <vscale x 16 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call fast float @llvm.vector.reduce.fadd.v1f32(float 0.0, <1 x float> undef)
@@ -113,6 +165,11 @@ define void @reduce_fadd_float() {
   %v32 = call fast float @llvm.vector.reduce.fadd.v32f32(float 0.0, <32 x float> undef)
   %V64 = call fast float @llvm.vector.reduce.fadd.v64f32(float 0.0, <64 x float> undef)
   %V128 = call fast float @llvm.vector.reduce.fadd.v128f32(float 0.0, <128 x float> undef)
+  %NXV1 = call fast float @llvm.vector.reduce.fadd.nxv1f32(float 0.0, <vscale x 1 x float> undef)
+  %NXV2 = call fast float @llvm.vector.reduce.fadd.nxv2f32(float 0.0, <vscale x 2 x float> undef)
+  %NXV4 = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.0, <vscale x 4 x float> undef)
+  %NXV8 = call fast float @llvm.vector.reduce.fadd.nxv8f32(float 0.0, <vscale x 8 x float> undef)
+  %NXV16 = call fast float @llvm.vector.reduce.fadd.nxv16f32(float 0.0, <vscale x 16 x float> undef)
   ret void
 }
 
@@ -126,6 +183,10 @@ define void @reduce_fadd_double() {
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v32 = call fast double @llvm.vector.reduce.fadd.v32f64(double 0.000000e+00, <32 x double> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V64 = call fast double @llvm.vector.reduce.fadd.v64f64(double 0.000000e+00, <64 x double> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V128 = call fast double @llvm.vector.reduce.fadd.v128f64(double 0.000000e+00, <128 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV1 = call fast double @llvm.vector.reduce.fadd.nxv1f64(double 0.000000e+00, <vscale x 1 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2 = call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NXV4 = call fast double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %NXV8 = call fast double @llvm.vector.reduce.fadd.nxv8f64(double 0.000000e+00, <vscale x 8 x double> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_fadd_double'
@@ -137,6 +198,10 @@ define void @reduce_fadd_double() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32 = call fast double @llvm.vector.reduce.fadd.v32f64(double 0.000000e+00, <32 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64 = call fast double @llvm.vector.reduce.fadd.v64f64(double 0.000000e+00, <64 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128 = call fast double @llvm.vector.reduce.fadd.v128f64(double 0.000000e+00, <128 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV1 = call fast double @llvm.vector.reduce.fadd.nxv1f64(double 0.000000e+00, <vscale x 1 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV2 = call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV4 = call fast double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV8 = call fast double @llvm.vector.reduce.fadd.nxv8f64(double 0.000000e+00, <vscale x 8 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call fast double @llvm.vector.reduce.fadd.v1f64(double 0.0, <1 x double> undef)
@@ -147,11 +212,15 @@ define void @reduce_fadd_double() {
   %v32 = call fast double @llvm.vector.reduce.fadd.v32f64(double 0.0, <32 x double> undef)
   %V64 = call fast double @llvm.vector.reduce.fadd.v64f64(double 0.0, <64 x double> undef)
   %V128 = call fast double @llvm.vector.reduce.fadd.v128f64(double 0.0, <128 x double> undef)
+  %NXV1 = call fast double @llvm.vector.reduce.fadd.nxv1f64(double 0.0, <vscale x 1 x double> undef)
+  %NXV2 = call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.0, <vscale x 2 x double> undef)
+  %NXV4 = call fast double @llvm.vector.reduce.fadd.nxv4f64(double 0.0, <vscale x 4 x double> undef)
+  %NXV8 = call fast double @llvm.vector.reduce.fadd.nxv8f64(double 0.0, <vscale x 8 x double> undef)
   ret void
 }
 
-define void @reduce_oredered_fadd_bfloat() {
-; FP-REDUCE-LABEL: 'reduce_oredered_fadd_bfloat'
+define void @reduce_ordered_fadd_bfloat() {
+; FP-REDUCE-LABEL: 'reduce_ordered_fadd_bfloat'
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call bfloat @llvm.vector.reduce.fadd.v1bf16(bfloat 0xR0000, <1 x bfloat> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call bfloat @llvm.vector.reduce.fadd.v2bf16(bfloat 0xR0000, <2 x bfloat> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
@@ -160,9 +229,15 @@ define void @reduce_oredered_fadd_bfloat() {
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v32 = call bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V64 = call bfloat @llvm.vector.reduce.fadd.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V128 = call bfloat @llvm.vector.reduce.fadd.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV1 = call bfloat @llvm.vector.reduce.fadd.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %NXV2 = call bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NXV4 = call bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %NXV8 = call bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %NXV16 = call bfloat @llvm.vector.reduce.fadd.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %NXV32 = call bfloat @llvm.vector.reduce.fadd.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SIZE-LABEL: 'reduce_oredered_fadd_bfloat'
+; SIZE-LABEL: 'reduce_ordered_fadd_bfloat'
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call bfloat @llvm.vector.reduce.fadd.v1bf16(bfloat 0xR0000, <1 x bfloat> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call bfloat @llvm.vector.reduce.fadd.v2bf16(bfloat 0xR0000, <2 x bfloat> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
@@ -171,6 +246,12 @@ define void @reduce_oredered_fadd_bfloat() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32 = call bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call bfloat @llvm.vector.reduce.fadd.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128 = call bfloat @llvm.vector.reduce.fadd.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV1 = call bfloat @llvm.vector.reduce.fadd.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV2 = call bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV4 = call bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV8 = call bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV16 = call bfloat @llvm.vector.reduce.fadd.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV32 = call bfloat @llvm.vector.reduce.fadd.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call bfloat @llvm.vector.reduce.fadd.v1bf16(bfloat 0.0, <1 x bfloat> undef)
@@ -181,11 +262,17 @@ define void @reduce_oredered_fadd_bfloat() {
   %v32 = call bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0.0, <32 x bfloat> undef)
   %V64 = call bfloat @llvm.vector.reduce.fadd.v64bf16(bfloat 0.0, <64 x bfloat> undef)
   %V128 = call bfloat @llvm.vector.reduce.fadd.v128bf16(bfloat 0.0, <128 x bfloat> undef)
+  %NXV1 = call bfloat @llvm.vector.reduce.fadd.nxv1bf16(bfloat 0.0, <vscale x 1 x bfloat> undef)
+  %NXV2 = call bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat 0.0, <vscale x 2 x bfloat> undef)
+  %NXV4 = call bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat 0.0, <vscale x 4 x bfloat> undef)
+  %NXV8 = call bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat 0.0, <vscale x 8 x bfloat> undef)
+  %NXV16 = call bfloat @llvm.vector.reduce.fadd.nxv16bf16(bfloat 0.0, <vscale x 16 x bfloat> undef)
+  %NXV32 = call bfloat @llvm.vector.reduce.fadd.nxv32bf16(bfloat 0.0, <vscale x 32 x bfloat> undef)
   ret void
 }
 
-define void @reduce_oredered_fadd_half() {
-; FP-REDUCE-LABEL: 'reduce_oredered_fadd_half'
+define void @reduce_ordered_fadd_half() {
+; FP-REDUCE-LABEL: 'reduce_ordered_fadd_half'
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call half @llvm.vector.reduce.fadd.v1f16(half 0xH0000, <1 x half> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
@@ -194,9 +281,15 @@ define void @reduce_oredered_fadd_half() {
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v32 = call half @llvm.vector.reduce.fadd.v32f16(half 0xH0000, <32 x half> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V64 = call half @llvm.vector.reduce.fadd.v64f16(half 0xH0000, <64 x half> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V128 = call half @llvm.vector.reduce.fadd.v128f16(half 0xH0000, <128 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV1 = call half @llvm.vector.reduce.fadd.nxv1f16(half 0xH0000, <vscale x 1 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %NXV2 = call half @llvm.vector.reduce.fadd.nxv2f16(half 0xH0000, <vscale x 2 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NXV4 = call half @llvm.vector.reduce.fadd.nxv4f16(half 0xH0000, <vscale x 4 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %NXV8 = call half @llvm.vector.reduce.fadd.nxv8f16(half 0xH0000, <vscale x 8 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %NXV16 = call half @llvm.vector.reduce.fadd.nxv16f16(half 0xH0000, <vscale x 16 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %NXV32 = call half @llvm.vector.reduce.fadd.nxv32f16(half 0xH0000, <vscale x 32 x half> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SIZE-LABEL: 'reduce_oredered_fadd_half'
+; SIZE-LABEL: 'reduce_ordered_fadd_half'
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call half @llvm.vector.reduce.fadd.v1f16(half 0xH0000, <1 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
@@ -205,6 +298,12 @@ define void @reduce_oredered_fadd_half() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32 = call half @llvm.vector.reduce.fadd.v32f16(half 0xH0000, <32 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call half @llvm.vector.reduce.fadd.v64f16(half 0xH0000, <64 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128 = call half @llvm.vector.reduce.fadd.v128f16(half 0xH0000, <128 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV1 = call half @llvm.vector.reduce.fadd.nxv1f16(half 0xH0000, <vscale x 1 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV2 = call half @llvm.vector.reduce.fadd.nxv2f16(half 0xH0000, <vscale x 2 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV4 = call half @llvm.vector.reduce.fadd.nxv4f16(half 0xH0000, <vscale x 4 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV8 = call half @llvm.vector.reduce.fadd.nxv8f16(half 0xH0000, <vscale x 8 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV16 = call half @llvm.vector.reduce.fadd.nxv16f16(half 0xH0000, <vscale x 16 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV32 = call half @llvm.vector.reduce.fadd.nxv32f16(half 0xH0000, <vscale x 32 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call half @llvm.vector.reduce.fadd.v1f16(half 0.0, <1 x half> undef)
@@ -215,11 +314,17 @@ define void @reduce_oredered_fadd_half() {
   %v32 = call half @llvm.vector.reduce.fadd.v32f16(half 0.0, <32 x half> undef)
   %V64 = call half @llvm.vector.reduce.fadd.v64f16(half 0.0, <64 x half> undef)
   %V128 = call half @llvm.vector.reduce.fadd.v128f16(half 0.0, <128 x half> undef)
+  %NXV1 = call half @llvm.vector.reduce.fadd.nxv1f16(half 0.0, <vscale x 1 x half> undef)
+  %NXV2 = call half @llvm.vector.reduce.fadd.nxv2f16(half 0.0, <vscale x 2 x half> undef)
+  %NXV4 = call half @llvm.vector.reduce.fadd.nxv4f16(half 0.0, <vscale x 4 x half> undef)
+  %NXV8 = call half @llvm.vector.reduce.fadd.nxv8f16(half 0.0, <vscale x 8 x half> undef)
+  %NXV16 = call half @llvm.vector.reduce.fadd.nxv16f16(half 0.0, <vscale x 16 x half> undef)
+  %NXV32 = call half @llvm.vector.reduce.fadd.nxv32f16(half 0.0, <vscale x 32 x half> undef)
   ret void
 }
 
-define void @reduce_oredered_fadd_float() {
-; FP-REDUCE-LABEL: 'reduce_oredered_fadd_float'
+define void @reduce_ordered_fadd_float() {
+; FP-REDUCE-LABEL: 'reduce_ordered_fadd_float'
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call float @llvm.vector.reduce.fadd.v1f32(float 0.000000e+00, <1 x float> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> undef)
@@ -228,9 +333,14 @@ define void @reduce_oredered_fadd_float() {
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v32 = call float @llvm.vector.reduce.fadd.v32f32(float 0.000000e+00, <32 x float> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V64 = call float @llvm.vector.reduce.fadd.v64f32(float 0.000000e+00, <64 x float> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V128 = call float @llvm.vector.reduce.fadd.v128f32(float 0.000000e+00, <128 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV1 = call float @llvm.vector.reduce.fadd.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %NXV2 = call float @llvm.vector.reduce.fadd.nxv2f32(float 0.000000e+00, <vscale x 2 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NXV4 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %NXV8 = call float @llvm.vector.reduce.fadd.nxv8f32(float 0.000000e+00, <vscale x 8 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %NXV16 = call float @llvm.vector.reduce.fadd.nxv16f32(float 0.000000e+00, <vscale x 16 x float> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SIZE-LABEL: 'reduce_oredered_fadd_float'
+; SIZE-LABEL: 'reduce_ordered_fadd_float'
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call float @llvm.vector.reduce.fadd.v1f32(float 0.000000e+00, <1 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> undef)
@@ -239,6 +349,11 @@ define void @reduce_oredered_fadd_float() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32 = call float @llvm.vector.reduce.fadd.v32f32(float 0.000000e+00, <32 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64 = call float @llvm.vector.reduce.fadd.v64f32(float 0.000000e+00, <64 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V128 = call float @llvm.vector.reduce.fadd.v128f32(float 0.000000e+00, <128 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV1 = call float @llvm.vector.reduce.fadd.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV2 = call float @llvm.vector.reduce.fadd.nxv2f32(float 0.000000e+00, <vscale x 2 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV4 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV8 = call float @llvm.vector.reduce.fadd.nxv8f32(float 0.000000e+00, <vscale x 8 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV16 = call float @llvm.vector.reduce.fadd.nxv16f32(float 0.000000e+00, <vscale x 16 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call float @llvm.vector.reduce.fadd.v1f32(float 0.0, <1 x float> undef)
@@ -249,11 +364,16 @@ define void @reduce_oredered_fadd_float() {
   %v32 = call float @llvm.vector.reduce.fadd.v32f32(float 0.0, <32 x float> undef)
   %V64 = call float @llvm.vector.reduce.fadd.v64f32(float 0.0, <64 x float> undef)
   %V128 = call float @llvm.vector.reduce.fadd.v128f32(float 0.0, <128 x float> undef)
+  %NXV1 = call float @llvm.vector.reduce.fadd.nxv1f32(float 0.0, <vscale x 1 x float> undef)
+  %NXV2 = call float @llvm.vector.reduce.fadd.nxv2f32(float 0.0, <vscale x 2 x float> undef)
+  %NXV4 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.0, <vscale x 4 x float> undef)
+  %NXV8 = call float @llvm.vector.reduce.fadd.nxv8f32(float 0.0, <vscale x 8 x float> undef)
+  %NXV16 = call float @llvm.vector.reduce.fadd.nxv16f32(float 0.0, <vscale x 16 x float> undef)
   ret void
 }
 
-define void @reduce_oredered_fadd_double() {
-; FP-REDUCE-LABEL: 'reduce_oredered_fadd_double'
+define void @reduce_ordered_fadd_double() {
+; FP-REDUCE-LABEL: 'reduce_ordered_fadd_double'
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call double @llvm.vector.reduce.fadd.v1f64(double 0.000000e+00, <1 x double> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> undef)
@@ -262,9 +382,13 @@ define void @reduce_oredered_fadd_double() {
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v32 = call double @llvm.vector.reduce.fadd.v32f64(double 0.000000e+00, <32 x double> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V64 = call double @llvm.vector.reduce.fadd.v64f64(double 0.000000e+00, <64 x double> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V128 = call double @llvm.vector.reduce.fadd.v128f64(double 0.000000e+00, <128 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV1 = call double @llvm.vector.reduce.fadd.nxv1f64(double 0.000000e+00, <vscale x 1 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %NXV2 = call double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NXV4 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %NXV8 = call double @llvm.vector.reduce.fadd.nxv8f64(double 0.000000e+00, <vscale x 8 x double> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SIZE-LABEL: 'reduce_oredered_fadd_double'
+; SIZE-LABEL: 'reduce_ordered_fadd_double'
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call double @llvm.vector.reduce.fadd.v1f64(double 0.000000e+00, <1 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> undef)
@@ -273,6 +397,10 @@ define void @reduce_oredered_fadd_double() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32 = call double @llvm.vector.reduce.fadd.v32f64(double 0.000000e+00, <32 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64 = call double @llvm.vector.reduce.fadd.v64f64(double 0.000000e+00, <64 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128 = call double @llvm.vector.reduce.fadd.v128f64(double 0.000000e+00, <128 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV1 = call double @llvm.vector.reduce.fadd.nxv1f64(double 0.000000e+00, <vscale x 1 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV2 = call double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV4 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV8 = call double @llvm.vector.reduce.fadd.nxv8f64(double 0.000000e+00, <vscale x 8 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call double @llvm.vector.reduce.fadd.v1f64(double 0.0, <1 x double> undef)
@@ -283,30 +411,9 @@ define void @reduce_oredered_fadd_double() {
   %v32 = call double @llvm.vector.reduce.fadd.v32f64(double 0.0, <32 x double> undef)
   %V64 = call double @llvm.vector.reduce.fadd.v64f64(double 0.0, <64 x double> undef)
   %V128 = call double @llvm.vector.reduce.fadd.v128f64(double 0.0, <128 x double> undef)
+  %NXV1 = call double @llvm.vector.reduce.fadd.nxv1f64(double 0.0, <vscale x 1 x double> undef)
+  %NXV2 = call double @llvm.vector.reduce.fadd.nxv2f64(double 0.0, <vscale x 2 x double> undef)
+  %NXV4 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.0, <vscale x 4 x double> undef)
+  %NXV8 = call double @llvm.vector.reduce.fadd.nxv8f64(double 0.0, <vscale x 8 x double> undef)
   ret void
 }
-
-declare half @llvm.vector.reduce.fadd.v1f16(half, <1 x half>)
-declare half @llvm.vector.reduce.fadd.v2f16(half, <2 x half>)
-declare half @llvm.vector.reduce.fadd.v4f16(half, <4 x half>)
-declare half @llvm.vector.reduce.fadd.v8f16(half, <8 x half>)
-declare half @llvm.vector.reduce.fadd.v16f16(half, <16 x half>)
-declare half @llvm.vector.reduce.fadd.v32f16(half, <32 x half>)
-declare half @llvm.vector.reduce.fadd.v64f16(half, <64 x half>)
-declare half @llvm.vector.reduce.fadd.v128f16(half, <128 x half>)
-declare float @llvm.vector.reduce.fadd.v1f32(float, <1 x float>)
-declare float @llvm.vector.reduce.fadd.v2f32(float, <2 x float>)
-declare float @llvm.vector.reduce.fadd.v4f32(float, <4 x float>)
-declare float @llvm.vector.reduce.fadd.v8f32(float, <8 x float>)
-declare float @llvm.vector.reduce.fadd.v16f32(float, <16 x float>)
-declare float @llvm.vector.reduce.fadd.v32f32(float, <32 x float>)
-declare float @llvm.vector.reduce.fadd.v64f32(float, <64 x float>)
-declare float @llvm.vector.reduce.fadd.v128f32(float, <128 x float>)
-declare double @llvm.vector.reduce.fadd.v1f64(double, <1 x double>)
-declare double @llvm.vector.reduce.fadd.v2f64(double, <2 x double>)
-declare double @llvm.vector.reduce.fadd.v4f64(double, <4 x double>)
-declare double @llvm.vector.reduce.fadd.v8f64(double, <8 x double>)
-declare double @llvm.vector.reduce.fadd.v16f64(double, <16 x double>)
-declare double @llvm.vector.reduce.fadd.v32f64(double, <32 x double>)
-declare double @llvm.vector.reduce.fadd.v64f64(double, <64 x double>)
-declare double @llvm.vector.reduce.fadd.v128f64(double, <128 x double>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fmul.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fmul.ll
index 162562c7b893..211bcb1343ee 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-fmul.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fmul.ll
@@ -13,6 +13,12 @@ define void @reduce_fmul_bfloat() {
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 211 for instruction: %v32 = call fast bfloat @llvm.vector.reduce.fmul.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 541 for instruction: %V64 = call fast bfloat @llvm.vector.reduce.fmul.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 573 for instruction: %V128 = call fast bfloat @llvm.vector.reduce.fmul.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast bfloat @llvm.vector.reduce.fmul.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast bfloat @llvm.vector.reduce.fmul.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast bfloat @llvm.vector.reduce.fmul.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast bfloat @llvm.vector.reduce.fmul.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call fast bfloat @llvm.vector.reduce.fmul.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call fast bfloat @llvm.vector.reduce.fmul.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_fmul_bfloat'
@@ -24,6 +30,12 @@ define void @reduce_fmul_bfloat() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %v32 = call fast bfloat @llvm.vector.reduce.fmul.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64 = call fast bfloat @llvm.vector.reduce.fmul.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V128 = call fast bfloat @llvm.vector.reduce.fmul.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast bfloat @llvm.vector.reduce.fmul.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast bfloat @llvm.vector.reduce.fmul.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast bfloat @llvm.vector.reduce.fmul.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast bfloat @llvm.vector.reduce.fmul.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call fast bfloat @llvm.vector.reduce.fmul.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call fast bfloat @llvm.vector.reduce.fmul.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call fast bfloat @llvm.vector.reduce.fmul.v1bf16(bfloat 0.0, <1 x bfloat> undef)
@@ -34,6 +46,12 @@ define void @reduce_fmul_bfloat() {
   %v32 = call fast bfloat @llvm.vector.reduce.fmul.v32bf16(bfloat 0.0, <32 x bfloat> undef)
   %V64 = call fast bfloat @llvm.vector.reduce.fmul.v64bf16(bfloat 0.0, <64 x bfloat> undef)
   %V128 = call fast bfloat @llvm.vector.reduce.fmul.v128bf16(bfloat 0.0, <128 x bfloat> undef)
+  %NXV1 = call fast bfloat @llvm.vector.reduce.fmul.nxv1bf16(bfloat 0.0, <vscale x 1 x bfloat> undef)
+  %NXV2 = call fast bfloat @llvm.vector.reduce.fmul.nxv2bf16(bfloat 0.0, <vscale x 2 x bfloat> undef)
+  %NXV4 = call fast bfloat @llvm.vector.reduce.fmul.nxv4bf16(bfloat 0.0, <vscale x 4 x bfloat> undef)
+  %NXV8 = call fast bfloat @llvm.vector.reduce.fmul.nxv8bf16(bfloat 0.0, <vscale x 8 x bfloat> undef)
+  %NXV16 = call fast bfloat @llvm.vector.reduce.fmul.nxv16bf16(bfloat 0.0, <vscale x 16 x bfloat> undef)
+  %NXV32 = call fast bfloat @llvm.vector.reduce.fmul.nxv32bf16(bfloat 0.0, <vscale x 32 x bfloat> undef)
   ret void
 }
 
@@ -47,6 +65,12 @@ define void @reduce_fmul_half() {
 ; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 151 for instruction: %v32 = call fast half @llvm.vector.reduce.fmul.v32f16(half 0xH0000, <32 x half> undef)
 ; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 541 for instruction: %V64 = call fast half @llvm.vector.reduce.fmul.v64f16(half 0xH0000, <64 x half> undef)
 ; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 573 for instruction: %V128 = call fast half @llvm.vector.reduce.fmul.v128f16(half 0xH0000, <128 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast half @llvm.vector.reduce.fmul.nxv1f16(half 0xH0000, <vscale x 1 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast half @llvm.vector.reduce.fmul.nxv2f16(half 0xH0000, <vscale x 2 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast half @llvm.vector.reduce.fmul.nxv4f16(half 0xH0000, <vscale x 4 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast half @llvm.vector.reduce.fmul.nxv8f16(half 0xH0000, <vscale x 8 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call fast half @llvm.vector.reduce.fmul.nxv16f16(half 0xH0000, <vscale x 16 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call fast half @llvm.vector.reduce.fmul.nxv32f16(half 0xH0000, <vscale x 32 x half> undef)
 ; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; FP-REDUCE-ZVFHMIN-LABEL: 'reduce_fmul_half'
@@ -58,6 +82,12 @@ define void @reduce_fmul_half() {
 ; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 211 for instruction: %v32 = call fast half @llvm.vector.reduce.fmul.v32f16(half 0xH0000, <32 x half> undef)
 ; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 541 for instruction: %V64 = call fast half @llvm.vector.reduce.fmul.v64f16(half 0xH0000, <64 x half> undef)
 ; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 573 for instruction: %V128 = call fast half @llvm.vector.reduce.fmul.v128f16(half 0xH0000, <128 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast half @llvm.vector.reduce.fmul.nxv1f16(half 0xH0000, <vscale x 1 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast half @llvm.vector.reduce.fmul.nxv2f16(half 0xH0000, <vscale x 2 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast half @llvm.vector.reduce.fmul.nxv4f16(half 0xH0000, <vscale x 4 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast half @llvm.vector.reduce.fmul.nxv8f16(half 0xH0000, <vscale x 8 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call fast half @llvm.vector.reduce.fmul.nxv16f16(half 0xH0000, <vscale x 16 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call fast half @llvm.vector.reduce.fmul.nxv32f16(half 0xH0000, <vscale x 32 x half> undef)
 ; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_fmul_half'
@@ -69,6 +99,12 @@ define void @reduce_fmul_half() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %v32 = call fast half @llvm.vector.reduce.fmul.v32f16(half 0xH0000, <32 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64 = call fast half @llvm.vector.reduce.fmul.v64f16(half 0xH0000, <64 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V128 = call fast half @llvm.vector.reduce.fmul.v128f16(half 0xH0000, <128 x half> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast half @llvm.vector.reduce.fmul.nxv1f16(half 0xH0000, <vscale x 1 x half> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast half @llvm.vector.reduce.fmul.nxv2f16(half 0xH0000, <vscale x 2 x half> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast half @llvm.vector.reduce.fmul.nxv4f16(half 0xH0000, <vscale x 4 x half> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast half @llvm.vector.reduce.fmul.nxv8f16(half 0xH0000, <vscale x 8 x half> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call fast half @llvm.vector.reduce.fmul.nxv16f16(half 0xH0000, <vscale x 16 x half> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call fast half @llvm.vector.reduce.fmul.nxv32f16(half 0xH0000, <vscale x 32 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call fast half @llvm.vector.reduce.fmul.v1f16(half 0.0, <1 x half> undef)
@@ -79,6 +115,12 @@ define void @reduce_fmul_half() {
   %v32 = call fast half @llvm.vector.reduce.fmul.v32f16(half 0.0, <32 x half> undef)
   %V64 = call fast half @llvm.vector.reduce.fmul.v64f16(half 0.0, <64 x half> undef)
   %V128 = call fast half @llvm.vector.reduce.fmul.v128f16(half 0.0, <128 x half> undef)
+  %NXV1 = call fast half @llvm.vector.reduce.fmul.nxv1f16(half 0.0, <vscale x 1 x half> undef)
+  %NXV2 = call fast half @llvm.vector.reduce.fmul.nxv2f16(half 0.0, <vscale x 2 x half> undef)
+  %NXV4 = call fast half @llvm.vector.reduce.fmul.nxv4f16(half 0.0, <vscale x 4 x half> undef)
+  %NXV8 = call fast half @llvm.vector.reduce.fmul.nxv8f16(half 0.0, <vscale x 8 x half> undef)
+  %NXV16 = call fast half @llvm.vector.reduce.fmul.nxv16f16(half 0.0, <vscale x 16 x half> undef)
+  %NXV32 = call fast half @llvm.vector.reduce.fmul.nxv32f16(half 0.0, <vscale x 32 x half> undef)
   ret void
 }
 
@@ -92,6 +134,11 @@ define void @reduce_fmul_float() {
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 451 for instruction: %v32 = call fast float @llvm.vector.reduce.fmul.v32f32(float 0.000000e+00, <32 x float> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 483 for instruction: %V64 = call fast float @llvm.vector.reduce.fmul.v64f32(float 0.000000e+00, <64 x float> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 547 for instruction: %V128 = call fast float @llvm.vector.reduce.fmul.v128f32(float 0.000000e+00, <128 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast float @llvm.vector.reduce.fmul.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast float @llvm.vector.reduce.fmul.nxv2f32(float 0.000000e+00, <vscale x 2 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, <vscale x 4 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast float @llvm.vector.reduce.fmul.nxv8f32(float 0.000000e+00, <vscale x 8 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call fast float @llvm.vector.reduce.fmul.nxv16f32(float 0.000000e+00, <vscale x 16 x float> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_fmul_float'
@@ -103,6 +150,11 @@ define void @reduce_fmul_float() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %v32 = call fast float @llvm.vector.reduce.fmul.v32f32(float 0.000000e+00, <32 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V64 = call fast float @llvm.vector.reduce.fmul.v64f32(float 0.000000e+00, <64 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V128 = call fast float @llvm.vector.reduce.fmul.v128f32(float 0.000000e+00, <128 x float> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast float @llvm.vector.reduce.fmul.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast float @llvm.vector.reduce.fmul.nxv2f32(float 0.000000e+00, <vscale x 2 x float> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, <vscale x 4 x float> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast float @llvm.vector.reduce.fmul.nxv8f32(float 0.000000e+00, <vscale x 8 x float> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call fast float @llvm.vector.reduce.fmul.nxv16f32(float 0.000000e+00, <vscale x 16 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call fast float @llvm.vector.reduce.fmul.v1f32(float 0.0, <1 x float> undef)
@@ -113,6 +165,11 @@ define void @reduce_fmul_float() {
   %v32 = call fast float @llvm.vector.reduce.fmul.v32f32(float 0.0, <32 x float> undef)
   %V64 = call fast float @llvm.vector.reduce.fmul.v64f32(float 0.0, <64 x float> undef)
   %V128 = call fast float @llvm.vector.reduce.fmul.v128f32(float 0.0, <128 x float> undef)
+  %NXV1 = call fast float @llvm.vector.reduce.fmul.nxv1f32(float 0.0, <vscale x 1 x float> undef)
+  %NXV2 = call fast float @llvm.vector.reduce.fmul.nxv2f32(float 0.0, <vscale x 2 x float> undef)
+  %NXV4 = call fast float @llvm.vector.reduce.fmul.nxv4f32(float 0.0, <vscale x 4 x float> undef)
+  %NXV8 = call fast float @llvm.vector.reduce.fmul.nxv8f32(float 0.0, <vscale x 8 x float> undef)
+  %NXV16 = call fast float @llvm.vector.reduce.fmul.nxv16f32(float 0.0, <vscale x 16 x float> undef)
   ret void
 }
 
@@ -126,6 +183,10 @@ define void @reduce_fmul_double() {
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 393 for instruction: %v32 = call fast double @llvm.vector.reduce.fmul.v32f64(double 0.000000e+00, <32 x double> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 457 for instruction: %V64 = call fast double @llvm.vector.reduce.fmul.v64f64(double 0.000000e+00, <64 x double> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 585 for instruction: %V128 = call fast double @llvm.vector.reduce.fmul.v128f64(double 0.000000e+00, <128 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast double @llvm.vector.reduce.fmul.nxv1f64(double 0.000000e+00, <vscale x 1 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast double @llvm.vector.reduce.fmul.nxv2f64(double 0.000000e+00, <vscale x 2 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, <vscale x 4 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast double @llvm.vector.reduce.fmul.nxv8f64(double 0.000000e+00, <vscale x 8 x double> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_fmul_double'
@@ -137,6 +198,10 @@ define void @reduce_fmul_double() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v32 = call fast double @llvm.vector.reduce.fmul.v32f64(double 0.000000e+00, <32 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V64 = call fast double @llvm.vector.reduce.fmul.v64f64(double 0.000000e+00, <64 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V128 = call fast double @llvm.vector.reduce.fmul.v128f64(double 0.000000e+00, <128 x double> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast double @llvm.vector.reduce.fmul.nxv1f64(double 0.000000e+00, <vscale x 1 x double> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast double @llvm.vector.reduce.fmul.nxv2f64(double 0.000000e+00, <vscale x 2 x double> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, <vscale x 4 x double> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast double @llvm.vector.reduce.fmul.nxv8f64(double 0.000000e+00, <vscale x 8 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call fast double @llvm.vector.reduce.fmul.v1f64(double 0.0, <1 x double> undef)
@@ -147,6 +212,10 @@ define void @reduce_fmul_double() {
   %v32 = call fast double @llvm.vector.reduce.fmul.v32f64(double 0.0, <32 x double> undef)
   %V64 = call fast double @llvm.vector.reduce.fmul.v64f64(double 0.0, <64 x double> undef)
   %V128 = call fast double @llvm.vector.reduce.fmul.v128f64(double 0.0, <128 x double> undef)
+  %NXV1 = call fast double @llvm.vector.reduce.fmul.nxv1f64(double 0.0, <vscale x 1 x double> undef)
+  %NXV2 = call fast double @llvm.vector.reduce.fmul.nxv2f64(double 0.0, <vscale x 2 x double> undef)
+  %NXV4 = call fast double @llvm.vector.reduce.fmul.nxv4f64(double 0.0, <vscale x 4 x double> undef)
+  %NXV8 = call fast double @llvm.vector.reduce.fmul.nxv8f64(double 0.0, <vscale x 8 x double> undef)
   ret void
 }
 
@@ -160,6 +229,12 @@ define void @reduce_ordered_fmul_bfloat() {
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32 = call bfloat @llvm.vector.reduce.fmul.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %V64 = call bfloat @llvm.vector.reduce.fmul.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 510 for instruction: %V128 = call bfloat @llvm.vector.reduce.fmul.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call bfloat @llvm.vector.reduce.fmul.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call bfloat @llvm.vector.reduce.fmul.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call bfloat @llvm.vector.reduce.fmul.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call bfloat @llvm.vector.reduce.fmul.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call bfloat @llvm.vector.reduce.fmul.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call bfloat @llvm.vector.reduce.fmul.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_ordered_fmul_bfloat'
@@ -171,6 +246,12 @@ define void @reduce_ordered_fmul_bfloat() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %v32 = call bfloat @llvm.vector.reduce.fmul.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 191 for instruction: %V64 = call bfloat @llvm.vector.reduce.fmul.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 382 for instruction: %V128 = call bfloat @llvm.vector.reduce.fmul.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call bfloat @llvm.vector.reduce.fmul.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call bfloat @llvm.vector.reduce.fmul.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call bfloat @llvm.vector.reduce.fmul.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call bfloat @llvm.vector.reduce.fmul.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call bfloat @llvm.vector.reduce.fmul.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call bfloat @llvm.vector.reduce.fmul.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call bfloat @llvm.vector.reduce.fmul.v1bf16(bfloat 0.0, <1 x bfloat> undef)
@@ -181,6 +262,12 @@ define void @reduce_ordered_fmul_bfloat() {
   %v32 = call bfloat @llvm.vector.reduce.fmul.v32bf16(bfloat 0.0, <32 x bfloat> undef)
   %V64 = call bfloat @llvm.vector.reduce.fmul.v64bf16(bfloat 0.0, <64 x bfloat> undef)
   %V128 = call bfloat @llvm.vector.reduce.fmul.v128bf16(bfloat 0.0, <128 x bfloat> undef)
+  %NXV1 = call bfloat @llvm.vector.reduce.fmul.nxv1bf16(bfloat 0.0, <vscale x 1 x bfloat> undef)
+  %NXV2 = call bfloat @llvm.vector.reduce.fmul.nxv2bf16(bfloat 0.0, <vscale x 2 x bfloat> undef)
+  %NXV4 = call bfloat @llvm.vector.reduce.fmul.nxv4bf16(bfloat 0.0, <vscale x 4 x bfloat> undef)
+  %NXV8 = call bfloat @llvm.vector.reduce.fmul.nxv8bf16(bfloat 0.0, <vscale x 8 x bfloat> undef)
+  %NXV16 = call bfloat @llvm.vector.reduce.fmul.nxv16bf16(bfloat 0.0, <vscale x 16 x bfloat> undef)
+  %NXV32 = call bfloat @llvm.vector.reduce.fmul.nxv32bf16(bfloat 0.0, <vscale x 32 x bfloat> undef)
   ret void
 }
 
@@ -194,6 +281,12 @@ define void @reduce_ordered_fmul_half() {
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32 = call half @llvm.vector.reduce.fmul.v32f16(half 0xH0000, <32 x half> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %V64 = call half @llvm.vector.reduce.fmul.v64f16(half 0xH0000, <64 x half> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 510 for instruction: %V128 = call half @llvm.vector.reduce.fmul.v128f16(half 0xH0000, <128 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call half @llvm.vector.reduce.fmul.nxv1f16(half 0xH0000, <vscale x 1 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call half @llvm.vector.reduce.fmul.nxv2f16(half 0xH0000, <vscale x 2 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call half @llvm.vector.reduce.fmul.nxv4f16(half 0xH0000, <vscale x 4 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call half @llvm.vector.reduce.fmul.nxv8f16(half 0xH0000, <vscale x 8 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call half @llvm.vector.reduce.fmul.nxv16f16(half 0xH0000, <vscale x 16 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call half @llvm.vector.reduce.fmul.nxv32f16(half 0xH0000, <vscale x 32 x half> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_ordered_fmul_half'
@@ -205,6 +298,12 @@ define void @reduce_ordered_fmul_half() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %v32 = call half @llvm.vector.reduce.fmul.v32f16(half 0xH0000, <32 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 191 for instruction: %V64 = call half @llvm.vector.reduce.fmul.v64f16(half 0xH0000, <64 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 382 for instruction: %V128 = call half @llvm.vector.reduce.fmul.v128f16(half 0xH0000, <128 x half> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call half @llvm.vector.reduce.fmul.nxv1f16(half 0xH0000, <vscale x 1 x half> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call half @llvm.vector.reduce.fmul.nxv2f16(half 0xH0000, <vscale x 2 x half> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call half @llvm.vector.reduce.fmul.nxv4f16(half 0xH0000, <vscale x 4 x half> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call half @llvm.vector.reduce.fmul.nxv8f16(half 0xH0000, <vscale x 8 x half> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call half @llvm.vector.reduce.fmul.nxv16f16(half 0xH0000, <vscale x 16 x half> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call half @llvm.vector.reduce.fmul.nxv32f16(half 0xH0000, <vscale x 32 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call half @llvm.vector.reduce.fmul.v1f16(half 0.0, <1 x half> undef)
@@ -215,6 +314,12 @@ define void @reduce_ordered_fmul_half() {
   %v32 = call half @llvm.vector.reduce.fmul.v32f16(half 0.0, <32 x half> undef)
   %V64 = call half @llvm.vector.reduce.fmul.v64f16(half 0.0, <64 x half> undef)
   %V128 = call half @llvm.vector.reduce.fmul.v128f16(half 0.0, <128 x half> undef)
+  %NXV1 = call half @llvm.vector.reduce.fmul.nxv1f16(half 0.0, <vscale x 1 x half> undef)
+  %NXV2 = call half @llvm.vector.reduce.fmul.nxv2f16(half 0.0, <vscale x 2 x half> undef)
+  %NXV4 = call half @llvm.vector.reduce.fmul.nxv4f16(half 0.0, <vscale x 4 x half> undef)
+  %NXV8 = call half @llvm.vector.reduce.fmul.nxv8f16(half 0.0, <vscale x 8 x half> undef)
+  %NXV16 = call half @llvm.vector.reduce.fmul.nxv16f16(half 0.0, <vscale x 16 x half> undef)
+  %NXV32 = call half @llvm.vector.reduce.fmul.nxv32f16(half 0.0, <vscale x 32 x half> undef)
   ret void
 }
 
@@ -228,6 +333,11 @@ define void @reduce_ordered_fmul_float() {
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32 = call float @llvm.vector.reduce.fmul.v32f32(float 0.000000e+00, <32 x float> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %V64 = call float @llvm.vector.reduce.fmul.v64f32(float 0.000000e+00, <64 x float> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 508 for instruction: %V128 = call float @llvm.vector.reduce.fmul.v128f32(float 0.000000e+00, <128 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call float @llvm.vector.reduce.fmul.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call float @llvm.vector.reduce.fmul.nxv2f32(float 0.000000e+00, <vscale x 2 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, <vscale x 4 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call float @llvm.vector.reduce.fmul.nxv8f32(float 0.000000e+00, <vscale x 8 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call float @llvm.vector.reduce.fmul.nxv16f32(float 0.000000e+00, <vscale x 16 x float> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_ordered_fmul_float'
@@ -239,6 +349,11 @@ define void @reduce_ordered_fmul_float() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %v32 = call float @llvm.vector.reduce.fmul.v32f32(float 0.000000e+00, <32 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 190 for instruction: %V64 = call float @llvm.vector.reduce.fmul.v64f32(float 0.000000e+00, <64 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 380 for instruction: %V128 = call float @llvm.vector.reduce.fmul.v128f32(float 0.000000e+00, <128 x float> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call float @llvm.vector.reduce.fmul.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call float @llvm.vector.reduce.fmul.nxv2f32(float 0.000000e+00, <vscale x 2 x float> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, <vscale x 4 x float> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call float @llvm.vector.reduce.fmul.nxv8f32(float 0.000000e+00, <vscale x 8 x float> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call float @llvm.vector.reduce.fmul.nxv16f32(float 0.000000e+00, <vscale x 16 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call float @llvm.vector.reduce.fmul.v1f32(float 0.0, <1 x float> undef)
@@ -249,6 +364,11 @@ define void @reduce_ordered_fmul_float() {
   %v32 = call float @llvm.vector.reduce.fmul.v32f32(float 0.0, <32 x float> undef)
   %V64 = call float @llvm.vector.reduce.fmul.v64f32(float 0.0, <64 x float> undef)
   %V128 = call float @llvm.vector.reduce.fmul.v128f32(float 0.0, <128 x float> undef)
+  %NXV1 = call float @llvm.vector.reduce.fmul.nxv1f32(float 0.0, <vscale x 1 x float> undef)
+  %NXV2 = call float @llvm.vector.reduce.fmul.nxv2f32(float 0.0, <vscale x 2 x float> undef)
+  %NXV4 = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.0, <vscale x 4 x float> undef)
+  %NXV8 = call float @llvm.vector.reduce.fmul.nxv8f32(float 0.0, <vscale x 8 x float> undef)
+  %NXV16 = call float @llvm.vector.reduce.fmul.nxv16f32(float 0.0, <vscale x 16 x float> undef)
   ret void
 }
 
@@ -262,6 +382,10 @@ define void @reduce_ordered_fmul_double() {
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %v32 = call double @llvm.vector.reduce.fmul.v32f64(double 0.000000e+00, <32 x double> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 252 for instruction: %V64 = call double @llvm.vector.reduce.fmul.v64f64(double 0.000000e+00, <64 x double> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 504 for instruction: %V128 = call double @llvm.vector.reduce.fmul.v128f64(double 0.000000e+00, <128 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call double @llvm.vector.reduce.fmul.nxv1f64(double 0.000000e+00, <vscale x 1 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call double @llvm.vector.reduce.fmul.nxv2f64(double 0.000000e+00, <vscale x 2 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, <vscale x 4 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call double @llvm.vector.reduce.fmul.nxv8f64(double 0.000000e+00, <vscale x 8 x double> undef)
 ; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_ordered_fmul_double'
@@ -273,6 +397,10 @@ define void @reduce_ordered_fmul_double() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %v32 = call double @llvm.vector.reduce.fmul.v32f64(double 0.000000e+00, <32 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64 = call double @llvm.vector.reduce.fmul.v64f64(double 0.000000e+00, <64 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 376 for instruction: %V128 = call double @llvm.vector.reduce.fmul.v128f64(double 0.000000e+00, <128 x double> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call double @llvm.vector.reduce.fmul.nxv1f64(double 0.000000e+00, <vscale x 1 x double> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call double @llvm.vector.reduce.fmul.nxv2f64(double 0.000000e+00, <vscale x 2 x double> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, <vscale x 4 x double> undef)
+; SIZE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call double @llvm.vector.reduce.fmul.nxv8f64(double 0.000000e+00, <vscale x 8 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V1 = call double @llvm.vector.reduce.fmul.v1f64(double 0.0, <1 x double> undef)
@@ -283,5 +411,9 @@ define void @reduce_ordered_fmul_double() {
   %v32 = call double @llvm.vector.reduce.fmul.v32f64(double 0.0, <32 x double> undef)
   %V64 = call double @llvm.vector.reduce.fmul.v64f64(double 0.0, <64 x double> undef)
   %V128 = call double @llvm.vector.reduce.fmul.v128f64(double 0.0, <128 x double> undef)
+  %NXV1 = call double @llvm.vector.reduce.fmul.nxv1f64(double 0.0, <vscale x 1 x double> undef)
+  %NXV2 = call double @llvm.vector.reduce.fmul.nxv2f64(double 0.0, <vscale x 2 x double> undef)
+  %NXV4 = call double @llvm.vector.reduce.fmul.nxv4f64(double 0.0, <vscale x 4 x double> undef)
+  %NXV8 = call double @llvm.vector.reduce.fmul.nxv8f64(double 0.0, <vscale x 8 x double> undef)
   ret void
 }
-- 
GitLab


From ca998b071eba1c92bf8535964183c7c4c3b258c3 Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Tue, 29 Oct 2024 15:37:03 -0700
Subject: [PATCH 510/693] [SandboxVec][Legality] Check wrap flags (#113975)

---
 .../Vectorize/SandboxVectorizer/Legality.h       |  3 +++
 .../Vectorize/SandboxVectorizer/Legality.cpp     | 15 +++++++++++++++
 .../Vectorize/SandboxVectorizer/LegalityTest.cpp | 16 +++++++++++++++-
 3 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
index 49dcec26dbc5..77ba5cd7f002 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
@@ -32,6 +32,7 @@ enum class ResultReason {
   DiffOpcodes,
   DiffTypes,
   DiffMathFlags,
+  DiffWrapFlags,
 };
 
 #ifndef NDEBUG
@@ -56,6 +57,8 @@ struct ToStr {
       return "DiffTypes";
     case ResultReason::DiffMathFlags:
       return "DiffMathFlags";
+    case ResultReason::DiffWrapFlags:
+      return "DiffWrapFlags";
     }
     llvm_unreachable("Unknown ResultReason enum");
   }
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
index 346d8a90589f..1cc6356300e4 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
@@ -55,6 +55,21 @@ LegalityAnalysis::notVectorizableBasedOnOpcodesAndTypes(
       return ResultReason::DiffMathFlags;
   }
 
+  // TODO: Allow vectorization by using common flags.
+  // For now Pack if they don't have the same wrap flags.
+  bool CanHaveWrapFlags =
+      isa<OverflowingBinaryOperator>(I0) || isa<TruncInst>(I0);
+  if (CanHaveWrapFlags) {
+    bool NUW0 = I0->hasNoUnsignedWrap();
+    bool NSW0 = I0->hasNoSignedWrap();
+    if (any_of(drop_begin(Bndl), [NUW0, NSW0](auto *V) {
+          return cast<Instruction>(V)->hasNoUnsignedWrap() != NUW0 ||
+                 cast<Instruction>(V)->hasNoSignedWrap() != NSW0;
+        })) {
+      return ResultReason::DiffWrapFlags;
+    }
+  }
+
   // TODO: Missing checks
 
   return std::nullopt;
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
index aaa8e96de6d1..50b78f6f48af 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
@@ -29,7 +29,7 @@ struct LegalityTest : public testing::Test {
 
 TEST_F(LegalityTest, Legality) {
   parseIR(C, R"IR(
-define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg, float %farg0, float %farg1) {
+define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg, float %farg0, float %farg1, i64 %v0, i64 %v1) {
   %gep0 = getelementptr float, ptr %ptr, i32 0
   %gep1 = getelementptr float, ptr %ptr, i32 1
   %gep3 = getelementptr float, ptr %ptr, i32 3
@@ -42,6 +42,8 @@ define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg, float
   store i8 %arg, ptr %gep1
   %fadd0 = fadd float %farg0, %farg0
   %fadd1 = fadd fast float %farg1, %farg1
+  %trunc0 = trunc nuw nsw i64 %v0 to i8
+  %trunc1 = trunc nsw i64 %v1 to i8
   ret void
 }
 )IR");
@@ -62,6 +64,8 @@ define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg, float
   auto *StI8 = cast<sandboxir::StoreInst>(&*It++);
   auto *FAdd0 = cast<sandboxir::BinaryOperator>(&*It++);
   auto *FAdd1 = cast<sandboxir::BinaryOperator>(&*It++);
+  auto *Trunc0 = cast<sandboxir::TruncInst>(&*It++);
+  auto *Trunc1 = cast<sandboxir::TruncInst>(&*It++);
 
   sandboxir::LegalityAnalysis Legality;
   const auto &Result = Legality.canVectorize({St0, St1});
@@ -98,6 +102,13 @@ define void @foo(ptr %ptr, <2 x float> %vec2, <3 x float> %vec3, i8 %arg, float
     EXPECT_EQ(cast<sandboxir::Pack>(Result).getReason(),
               sandboxir::ResultReason::DiffMathFlags);
   }
+  {
+    // Check DiffWrapFlags
+    const auto &Result = Legality.canVectorize({Trunc0, Trunc1});
+    EXPECT_TRUE(isa<sandboxir::Pack>(Result));
+    EXPECT_EQ(cast<sandboxir::Pack>(Result).getReason(),
+              sandboxir::ResultReason::DiffWrapFlags);
+  }
 }
 
 #ifndef NDEBUG
@@ -124,5 +135,8 @@ TEST_F(LegalityTest, LegalityResultDump) {
   EXPECT_TRUE(Matches(Legality.createLegalityResult<sandboxir::Pack>(
                           sandboxir::ResultReason::DiffMathFlags),
                       "Pack Reason: DiffMathFlags"));
+  EXPECT_TRUE(Matches(Legality.createLegalityResult<sandboxir::Pack>(
+                          sandboxir::ResultReason::DiffWrapFlags),
+                      "Pack Reason: DiffWrapFlags"));
 }
 #endif // NDEBUG
-- 
GitLab


From 2c5eea0e88a6ef6bf932d90c67aaec2bcc59d340 Mon Sep 17 00:00:00 2001
From: Kunwar Grover <groverkss@gmail.com>
Date: Tue, 29 Oct 2024 22:47:44 +0000
Subject: [PATCH 511/693] [mlir][Vector] Fix vector.insert folder for scalar to
 0-d inserts (#113828)

The current vector.insert folder tries to replace a scalar with a 0-rank
vector. This patch fixes this crash by not folding unless they types of
the result and replacement are same.
---
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp   |  8 ++--
 mlir/test/Dialect/Vector/canonicalize.mlir | 48 +++++++++++++++++-----
 2 files changed, 41 insertions(+), 15 deletions(-)

diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index d71a236f62f4..1853ae04f45d 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -2951,11 +2951,11 @@ void InsertOp::getCanonicalizationPatterns(RewritePatternSet &results,
               InsertOpConstantFolder>(context);
 }
 
-// Eliminates insert operations that produce values identical to their source
-// value. This happens when the source and destination vectors have identical
-// sizes.
 OpFoldResult vector::InsertOp::fold(FoldAdaptor adaptor) {
-  if (getNumIndices() == 0)
+  // Fold "vector.insert %v, %dest [] : vector<2x2xf32> from vector<2x2xf32>" to
+  // %v. Note: Do not fold "vector.insert %v, %dest [] : f32 into vector<f32>"
+  // (type mismatch).
+  if (getNumIndices() == 0 && getSourceType() == getType())
     return getSource();
   return {};
 }
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index 6d6bc199e601..c963460e7259 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -800,6 +800,43 @@ func.func @fold_extract_shapecast_to_shapecast(%arg0 : vector<3x4xf32>) -> vecto
 
 // -----
 
+// CHECK-LABEL: func @extract_no_fold_scalar_to_0d(
+//  CHECK-SAME:     %[[v:.*]]: vector<f32>)
+//       CHECK:   %[[extract:.*]] = vector.extract %[[v]][] : f32 from vector<f32>
+//       CHECK:   return %[[extract]]
+func.func @extract_no_fold_scalar_to_0d(%v: vector<f32>) -> f32 {
+  %0 = vector.extract %v[] : f32 from vector<f32>
+  return %0 : f32
+}
+
+// -----
+
+// CHECK-LABEL: func @insert_fold_same_rank(
+//  CHECK-SAME:     %[[v:.*]]: vector<2x2xf32>)
+//       CHECK:      %[[CST:.+]] = arith.constant
+//  CHECK-SAME:                    : vector<2x2xf32>
+//       CHECK-NOT:  vector.insert
+//       CHECK:   return %[[CST]]
+func.func @insert_fold_same_rank(%v: vector<2x2xf32>) -> vector<2x2xf32> {
+  %cst = arith.constant dense<0.000000e+00> : vector<2x2xf32>
+  %0 = vector.insert %cst, %v [] : vector<2x2xf32> into vector<2x2xf32>
+  return %0 : vector<2x2xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @insert_no_fold_scalar_to_0d(
+//  CHECK-SAME:     %[[v:.*]]: vector<f32>)
+//       CHECK:   %[[extract:.*]] = vector.insert %{{.*}}, %[[v]] [] : f32 into vector<f32>
+//       CHECK:   return %[[extract]]
+func.func @insert_no_fold_scalar_to_0d(%v: vector<f32>) -> vector<f32> {
+  %cst = arith.constant 0.000000e+00 : f32
+  %0 = vector.insert %cst, %v [] : f32 into vector<f32>
+  return %0 : vector<f32>
+}
+
+// -----
+
 // CHECK-LABEL: dont_fold_expand_collapse
 //       CHECK:   %[[A:.*]] = vector.shape_cast %{{.*}} : vector<1x1x64xf32> to vector<1x1x8x8xf32>
 //       CHECK:   %[[B:.*]] = vector.shape_cast %{{.*}} : vector<1x1x8x8xf32> to vector<8x8xf32>
@@ -2606,17 +2643,6 @@ func.func @rank_1_shuffle_to_interleave(%arg0: vector<6xi32>, %arg1: vector<6xi3
 
 // -----
 
-// CHECK-LABEL: func @extract_from_0d_regression(
-//  CHECK-SAME:     %[[v:.*]]: vector<f32>)
-//       CHECK:   %[[extract:.*]] = vector.extract %[[v]][] : f32 from vector<f32>
-//       CHECK:   return %[[extract]]
-func.func @extract_from_0d_regression(%v: vector<f32>) -> f32 {
-  %0 = vector.extract %v[] : f32 from vector<f32>
-  return %0 : f32
-}
-
-// -----
-
 // CHECK-LABEL: func @extract_from_0d_splat_broadcast_regression(
 //  CHECK-SAME:     %[[a:.*]]: f32, %[[b:.*]]: vector<f32>, %[[c:.*]]: vector<2xf32>)
 func.func @extract_from_0d_splat_broadcast_regression(%a: f32, %b: vector<f32>, %c: vector<2xf32>) -> (f32, f32, f32, f32, f32, vector<6x7xf32>, vector<3xf32>) {
-- 
GitLab


From d90a0d1d986e12c4a6ff2eeffe29cedc34e6e2ab Mon Sep 17 00:00:00 2001
From: Alexey Samsonov <vonosmas@gmail.com>
Date: Tue, 29 Oct 2024 16:00:08 -0700
Subject: [PATCH 512/693] Remove spurious includes from sinpif_test.cpp

MPFR functionality is provided by the MPFRWrapper instead, and the direct "mpfr.h" inclusion is not needed - this test doesn't rely on it (similar to its sibling libc/test/src/math/exhaustive/cospif_test.cpp that doesn't have it).
---
 libc/test/src/math/exhaustive/sinpif_test.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/libc/test/src/math/exhaustive/sinpif_test.cpp b/libc/test/src/math/exhaustive/sinpif_test.cpp
index 8bc1d81eb7e3..81abac0b73f2 100644
--- a/libc/test/src/math/exhaustive/sinpif_test.cpp
+++ b/libc/test/src/math/exhaustive/sinpif_test.cpp
@@ -7,10 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "exhaustive_test.h"
-#include "mpfr.h"
 #include "src/math/sinpif.h"
 #include "utils/MPFRWrapper/MPFRUtils.h"
-#include <sys/types.h>
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-- 
GitLab


From f71ea0e72e2419691e3c67bdbbe338d314ee77c0 Mon Sep 17 00:00:00 2001
From: "A. Jiang" <de34@live.cn>
Date: Wed, 30 Oct 2024 07:16:03 +0800
Subject: [PATCH 513/693] [libc++][test] Augment `test_alloc` in
 `deallocate_size.pass.cpp` (#113638)

Making it meet the requirements for allocator since C++11. Fixes
#113609.

This PR doesn't make it meet the C++03 allocator requirements, because
that would make the type too verbose and libc++ has backported many
C++11 features to the C++03 mode.

Drive-by: Removes the `TEST_CONSTEXPR_CXX14` on `allocate`/`dealocate`
which is never in effect (and causes IFNDR-ness before C++23), since
these functions modify the namespace-scoped variable `allocated_`.
---
 .../string.capacity/deallocate_size.pass.cpp  | 21 +++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/libcxx/test/std/strings/basic.string/string.capacity/deallocate_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/deallocate_size.pass.cpp
index 1203b2f3ec18..00f9e2b84678 100644
--- a/libcxx/test/std/strings/basic.string/string.capacity/deallocate_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.capacity/deallocate_size.pass.cpp
@@ -34,15 +34,32 @@ struct test_alloc {
     typedef test_alloc<U, Sz> other;
   };
 
-  TEST_CONSTEXPR_CXX14 pointer allocate(size_type n, const void* = nullptr) {
+  TEST_CONSTEXPR test_alloc() TEST_NOEXCEPT {}
+
+  template <class U>
+  TEST_CONSTEXPR test_alloc(const test_alloc<U, Sz>&) TEST_NOEXCEPT {}
+
+  pointer allocate(size_type n, const void* = nullptr) {
     allocated_ += n;
     return std::allocator<value_type>().allocate(n);
   }
 
-  TEST_CONSTEXPR_CXX14 void deallocate(pointer p, size_type s) {
+  void deallocate(pointer p, size_type s) {
     allocated_ -= s;
     std::allocator<value_type>().deallocate(p, s);
   }
+
+  template <class U>
+  friend TEST_CONSTEXPR bool operator==(const test_alloc&, const test_alloc<U, Sz>&) TEST_NOEXCEPT {
+    return true;
+  }
+
+#if TEST_STD_VER < 20
+  template <class U>
+  friend TEST_CONSTEXPR bool operator!=(const test_alloc&, const test_alloc<U, Sz>&) TEST_NOEXCEPT {
+    return false;
+  }
+#endif
 };
 
 template <class Sz>
-- 
GitLab


From 0f8dbb2fac532e37a9859d52982f0e8994305a11 Mon Sep 17 00:00:00 2001
From: "A. Jiang" <de34@live.cn>
Date: Wed, 30 Oct 2024 07:16:40 +0800
Subject: [PATCH 514/693] [libc++] Constrain additional overloads of `pow` for
 `complex` harder (#110235)

Fixes #109858.

The changes in #81379 broke some 3rd party library code that expected
usability of `std::complex<NonFloatingPoint>`. Although such code isn't
portable per [complex.numbers.general]/2, it might be better to make
these additional overloads not to interfere overload resolution too
much.

---------

Co-authored-by: Louis Dionne <ldionne.2@gmail.com>
---
 libcxx/include/complex                        |  6 +-
 .../complex.number/cmplx.over.pow.pass.cpp    | 84 +++++++++++++++++++
 2 files changed, 87 insertions(+), 3 deletions(-)
 create mode 100644 libcxx/test/libcxx/numerics/complex.number/cmplx.over.pow.pass.cpp

diff --git a/libcxx/include/complex b/libcxx/include/complex
index 4030d96b003d..15e42800fbfa 100644
--- a/libcxx/include/complex
+++ b/libcxx/include/complex
@@ -1097,20 +1097,20 @@ inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> pow(const complex<_Tp>& __x, const com
   return std::exp(__y * std::log(__x));
 }
 
-template <class _Tp, class _Up>
+template <class _Tp, class _Up, __enable_if_t<is_floating_point<_Tp>::value && is_floating_point<_Up>::value, int> = 0>
 inline _LIBCPP_HIDE_FROM_ABI complex<typename __promote<_Tp, _Up>::type>
 pow(const complex<_Tp>& __x, const complex<_Up>& __y) {
   typedef complex<typename __promote<_Tp, _Up>::type> result_type;
   return std::pow(result_type(__x), result_type(__y));
 }
 
-template <class _Tp, class _Up, __enable_if_t<is_arithmetic<_Up>::value, int> = 0>
+template <class _Tp, class _Up, __enable_if_t<is_floating_point<_Tp>::value && is_arithmetic<_Up>::value, int> = 0>
 inline _LIBCPP_HIDE_FROM_ABI complex<typename __promote<_Tp, _Up>::type> pow(const complex<_Tp>& __x, const _Up& __y) {
   typedef complex<typename __promote<_Tp, _Up>::type> result_type;
   return std::pow(result_type(__x), result_type(__y));
 }
 
-template <class _Tp, class _Up, __enable_if_t<is_arithmetic<_Tp>::value, int> = 0>
+template <class _Tp, class _Up, __enable_if_t<is_arithmetic<_Tp>::value && is_floating_point<_Up>::value, int> = 0>
 inline _LIBCPP_HIDE_FROM_ABI complex<typename __promote<_Tp, _Up>::type> pow(const _Tp& __x, const complex<_Up>& __y) {
   typedef complex<typename __promote<_Tp, _Up>::type> result_type;
   return std::pow(result_type(__x), result_type(__y));
diff --git a/libcxx/test/libcxx/numerics/complex.number/cmplx.over.pow.pass.cpp b/libcxx/test/libcxx/numerics/complex.number/cmplx.over.pow.pass.cpp
new file mode 100644
index 000000000000..1c790c283e43
--- /dev/null
+++ b/libcxx/test/libcxx/numerics/complex.number/cmplx.over.pow.pass.cpp
@@ -0,0 +1,84 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <complex>
+
+//  template<class T, class U> complex<__promote<T, U>::type> pow(const complex<T>&, const U&);
+//  template<class T, class U> complex<__promote<T, U>::type> pow(const complex<T>&, const complex<U>&);
+//  template<class T, class U> complex<__promote<T, U>::type> pow(const T&, const complex<U>&);
+
+// Test that these additional overloads are free from catching std::complex<non-floating-point>,
+// which is expected by several 3rd party libraries, see https://github.com/llvm/llvm-project/issues/109858.
+//
+// Note that we reserve the right to break this in the future if we have a reason to, but for the time being,
+// make sure we don't break this property unintentionally.
+#include <cassert>
+#include <cmath>
+#include <complex>
+#include <type_traits>
+
+#include "test_macros.h"
+
+namespace usr {
+struct usr_tag {};
+
+template <class T, class U>
+typename std::enable_if<(std::is_same<T, usr_tag>::value && std::is_floating_point<U>::value) ||
+                            (std::is_floating_point<T>::value && std::is_same<U, usr_tag>::value),
+                        int>::type
+pow(const T&, const std::complex<U>&) {
+  return std::is_same<T, usr_tag>::value ? 0 : 1;
+}
+
+template <class T, class U>
+typename std::enable_if<(std::is_same<T, usr_tag>::value && std::is_floating_point<U>::value) ||
+                            (std::is_floating_point<T>::value && std::is_same<U, usr_tag>::value),
+                        int>::type
+pow(const std::complex<T>&, const U&) {
+  return std::is_same<U, usr_tag>::value ? 2 : 3;
+}
+
+template <class T, class U>
+typename std::enable_if<(std::is_same<T, usr_tag>::value && std::is_floating_point<U>::value) ||
+                            (std::is_floating_point<T>::value && std::is_same<U, usr_tag>::value),
+                        int>::type
+pow(const std::complex<T>&, const std::complex<U>&) {
+  return std::is_same<T, usr_tag>::value ? 4 : 5;
+}
+} // namespace usr
+
+int main(int, char**) {
+  using std::pow;
+  using usr::pow;
+
+  usr::usr_tag tag;
+  const std::complex<usr::usr_tag> ctag;
+
+  assert(pow(tag, std::complex<float>(1.0f)) == 0);
+  assert(pow(std::complex<float>(1.0f), tag) == 2);
+  assert(pow(tag, std::complex<double>(1.0)) == 0);
+  assert(pow(std::complex<double>(1.0), tag) == 2);
+  assert(pow(tag, std::complex<long double>(1.0l)) == 0);
+  assert(pow(std::complex<long double>(1.0l), tag) == 2);
+
+  assert(pow(1.0f, ctag) == 1);
+  assert(pow(ctag, 1.0f) == 3);
+  assert(pow(1.0, ctag) == 1);
+  assert(pow(ctag, 1.0) == 3);
+  assert(pow(1.0l, ctag) == 1);
+  assert(pow(ctag, 1.0l) == 3);
+
+  assert(pow(ctag, std::complex<float>(1.0f)) == 4);
+  assert(pow(std::complex<float>(1.0f), ctag) == 5);
+  assert(pow(ctag, std::complex<double>(1.0)) == 4);
+  assert(pow(std::complex<double>(1.0), ctag) == 5);
+  assert(pow(ctag, std::complex<long double>(1.0l)) == 4);
+  assert(pow(std::complex<long double>(1.0l), ctag) == 5);
+
+  return 0;
+}
-- 
GitLab


From 75b37c3191254d0c418058cb94c3a7922b7ba71e Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Tue, 29 Oct 2024 16:41:01 -0700
Subject: [PATCH 515/693] [DWARF] Fix arity of DW_OP_bra (#114136)

Found by my proof-of-concept DWARF expression evaluator fuzzer.
---
 lldb/unittests/Expression/DWARFExpressionTest.cpp | 3 +++
 llvm/include/llvm/BinaryFormat/Dwarf.def          | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/lldb/unittests/Expression/DWARFExpressionTest.cpp b/lldb/unittests/Expression/DWARFExpressionTest.cpp
index f9e0605fce29..fdc9bfae1876 100644
--- a/lldb/unittests/Expression/DWARFExpressionTest.cpp
+++ b/lldb/unittests/Expression/DWARFExpressionTest.cpp
@@ -181,6 +181,9 @@ TEST(DWARFExpression, DW_OP_bra) {
       }),
       // clang-format on
       llvm::HasValue(0x42));
+
+  EXPECT_THAT_ERROR(Evaluate({DW_OP_bra, 0x01, 0x00}).takeError(),
+                    llvm::Failed());
 }
 
 TEST(DWARFExpression, DW_OP_convert) {
diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.def b/llvm/include/llvm/BinaryFormat/Dwarf.def
index 9336f2a454ae..0cbbbe823c06 100644
--- a/llvm/include/llvm/BinaryFormat/Dwarf.def
+++ b/llvm/include/llvm/BinaryFormat/Dwarf.def
@@ -728,7 +728,7 @@ HANDLE_DW_OP(0x24, shl, 0, 2, 2, DWARF)
 HANDLE_DW_OP(0x25, shr, 0, 2, 2, DWARF)
 HANDLE_DW_OP(0x26, shra, 0, 2, 2, DWARF)
 HANDLE_DW_OP(0x27, xor, 0, 2, 2, DWARF)
-HANDLE_DW_OP(0x28, bra, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x28, bra, 1, 1, 2, DWARF)
 HANDLE_DW_OP(0x29, eq, 0, 2, 2, DWARF)
 HANDLE_DW_OP(0x2a, ge, 0, 2, 2, DWARF)
 HANDLE_DW_OP(0x2b, gt, 0, 2, 2, DWARF)
-- 
GitLab


From 50dd9225f8b33a924970039772faeac03e0a5716 Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston@google.com>
Date: Tue, 29 Oct 2024 23:46:54 +0000
Subject: [PATCH 516/693] Revert "[asan] Flush stderr in test (#114084)"

This reverts commit e205929399d9ee4782b2d8ef1b659f918bdfe7c2.

Reason: did not solve the QEMU bot issues (https://lab.llvm.org/buildbot/#/builders/139/builds/5552/steps/30/logs/stdio) and it shouldn't have been necessary anyway (https://github.com/llvm/llvm-project/pull/114084#issuecomment-2445513320)
---
 compiler-rt/test/asan/TestCases/Posix/ignore_free_hook.cpp | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/compiler-rt/test/asan/TestCases/Posix/ignore_free_hook.cpp b/compiler-rt/test/asan/TestCases/Posix/ignore_free_hook.cpp
index dfeb8ad5c7b5..87be90014d56 100644
--- a/compiler-rt/test/asan/TestCases/Posix/ignore_free_hook.cpp
+++ b/compiler-rt/test/asan/TestCases/Posix/ignore_free_hook.cpp
@@ -26,17 +26,14 @@ bool ignore_free = false;
 
 extern "C" {
 WEAK_ON_APPLE void __sanitizer_free_hook(const volatile void *ptr) {
-  if (ptr == glob_ptr) {
+  if (ptr == glob_ptr)
     fprintf(stderr, "Free Hook\n");
-    fflush(stderr);
-  }
 }
 
 WEAK_ON_APPLE int __sanitizer_ignore_free_hook(const volatile void *ptr) {
   if (ptr != glob_ptr)
     return 0;
   fprintf(stderr, ignore_free ? "Free Ignored\n" : "Free Respected\n");
-  fflush(stderr);
   return ignore_free;
 }
 } // extern "C"
-- 
GitLab


From 0fa2fb3ed0bc726e5dcf8258bf764aacd1c2e6dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Tue, 29 Oct 2024 17:00:41 -0700
Subject: [PATCH 517/693] [flang][cuda] Add conversion pattern for
 cuf.kernel_launch op (#114129)

---
 .../Optimizer/Transforms/CUFOpConversion.cpp  | 70 ++++++++++++++++++-
 flang/test/Fir/CUDA/cuda-launch.fir           | 64 +++++++++++++++++
 2 files changed, 132 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Fir/CUDA/cuda-launch.fir

diff --git a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
index 14cc1cb508cf..fe125db7b406 100644
--- a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
@@ -15,6 +15,7 @@
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Optimizer/Support/DataLayout.h"
+#include "flang/Optimizer/Transforms/CUFCommon.h"
 #include "flang/Runtime/CUDA/allocatable.h"
 #include "flang/Runtime/CUDA/common.h"
 #include "flang/Runtime/CUDA/descriptor.h"
@@ -620,6 +621,69 @@ private:
   const mlir::SymbolTable &symtab;
 };
 
+struct CUFLaunchOpConversion
+    : public mlir::OpRewritePattern<cuf::KernelLaunchOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+
+  CUFLaunchOpConversion(mlir::MLIRContext *context,
+                        const mlir::SymbolTable &symTab)
+      : OpRewritePattern(context), symTab{symTab} {}
+
+  mlir::LogicalResult
+  matchAndRewrite(cuf::KernelLaunchOp op,
+                  mlir::PatternRewriter &rewriter) const override {
+    mlir::Location loc = op.getLoc();
+    auto idxTy = mlir::IndexType::get(op.getContext());
+    auto zero = rewriter.create<mlir::arith::ConstantOp>(
+        loc, rewriter.getIntegerType(32), rewriter.getI32IntegerAttr(0));
+    auto gridSizeX =
+        rewriter.create<mlir::arith::IndexCastOp>(loc, idxTy, op.getGridX());
+    auto gridSizeY =
+        rewriter.create<mlir::arith::IndexCastOp>(loc, idxTy, op.getGridY());
+    auto gridSizeZ =
+        rewriter.create<mlir::arith::IndexCastOp>(loc, idxTy, op.getGridZ());
+    auto blockSizeX =
+        rewriter.create<mlir::arith::IndexCastOp>(loc, idxTy, op.getBlockX());
+    auto blockSizeY =
+        rewriter.create<mlir::arith::IndexCastOp>(loc, idxTy, op.getBlockY());
+    auto blockSizeZ =
+        rewriter.create<mlir::arith::IndexCastOp>(loc, idxTy, op.getBlockZ());
+    auto kernelName = mlir::SymbolRefAttr::get(
+        rewriter.getStringAttr(cudaDeviceModuleName),
+        {mlir::SymbolRefAttr::get(
+            rewriter.getContext(),
+            op.getCallee().getLeafReference().getValue())});
+    mlir::Value clusterDimX, clusterDimY, clusterDimZ;
+    if (auto funcOp = symTab.lookup<mlir::func::FuncOp>(
+            op.getCallee().getLeafReference())) {
+      if (auto clusterDimsAttr = funcOp->getAttrOfType<cuf::ClusterDimsAttr>(
+              cuf::getClusterDimsAttrName())) {
+        clusterDimX = rewriter.create<mlir::arith::ConstantIndexOp>(
+            loc, clusterDimsAttr.getX().getInt());
+        clusterDimY = rewriter.create<mlir::arith::ConstantIndexOp>(
+            loc, clusterDimsAttr.getY().getInt());
+        clusterDimZ = rewriter.create<mlir::arith::ConstantIndexOp>(
+            loc, clusterDimsAttr.getZ().getInt());
+      }
+    }
+    auto gpuLaunchOp = rewriter.create<mlir::gpu::LaunchFuncOp>(
+        loc, kernelName, mlir::gpu::KernelDim3{gridSizeX, gridSizeY, gridSizeZ},
+        mlir::gpu::KernelDim3{blockSizeX, blockSizeY, blockSizeZ}, zero,
+        op.getArgs());
+    if (clusterDimX && clusterDimY && clusterDimZ) {
+      gpuLaunchOp.getClusterSizeXMutable().assign(clusterDimX);
+      gpuLaunchOp.getClusterSizeYMutable().assign(clusterDimY);
+      gpuLaunchOp.getClusterSizeZMutable().assign(clusterDimZ);
+    }
+    rewriter.replaceOp(op, gpuLaunchOp);
+    return mlir::success();
+  }
+
+private:
+  const mlir::SymbolTable &symTab;
+};
+
 class CUFOpConversion : public fir::impl::CUFOpConversionBase<CUFOpConversion> {
 public:
   void runOnOperation() override {
@@ -637,7 +701,8 @@ public:
         fir::support::getOrSetDataLayout(module, /*allowDefaultLayout=*/false);
     fir::LLVMTypeConverter typeConverter(module, /*applyTBAA=*/false,
                                          /*forceUnifiedTBAATree=*/false, *dl);
-    target.addLegalDialect<fir::FIROpsDialect, mlir::arith::ArithDialect>();
+    target.addLegalDialect<fir::FIROpsDialect, mlir::arith::ArithDialect,
+                           mlir::gpu::GPUDialect>();
     cuf::populateCUFToFIRConversionPatterns(typeConverter, *dl, symtab,
                                             patterns);
     if (mlir::failed(mlir::applyPartialConversion(getOperation(), target,
@@ -656,5 +721,6 @@ void cuf::populateCUFToFIRConversionPatterns(
   patterns.insert<CufAllocOpConversion>(patterns.getContext(), &dl, &converter);
   patterns.insert<CufAllocateOpConversion, CufDeallocateOpConversion,
                   CufFreeOpConversion>(patterns.getContext());
-  patterns.insert<CufDataTransferOpConversion>(patterns.getContext(), symtab);
+  patterns.insert<CufDataTransferOpConversion, CUFLaunchOpConversion>(
+      patterns.getContext(), symtab);
 }
diff --git a/flang/test/Fir/CUDA/cuda-launch.fir b/flang/test/Fir/CUDA/cuda-launch.fir
new file mode 100644
index 000000000000..f11bcbdb7fce
--- /dev/null
+++ b/flang/test/Fir/CUDA/cuda-launch.fir
@@ -0,0 +1,64 @@
+// RUN: fir-opt --split-input-file --cuf-convert %s | FileCheck %s
+
+
+module attributes {gpu.container_module, dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>} {
+  gpu.module @cuda_device_mod {
+    gpu.func @_QPsub_device1() kernel {
+      cf.br ^bb1
+    ^bb1:  // pred: ^bb0
+      gpu.return
+    }
+    gpu.func @_QPsub_device2(%arg0: !fir.ref<f32>) kernel {
+      cf.br ^bb1(%arg0 : !fir.ref<f32>)
+    ^bb1(%0: !fir.ref<f32>):  // pred: ^bb0
+      %1 = fir.declare %0 {uniq_name = "_QFsub1Ei"} : (!fir.ref<f32>) -> !fir.ref<f32>
+      %cst = arith.constant 2.000000e+00 : f32
+      fir.store %cst to %1 : !fir.ref<f32>
+      gpu.return
+    }
+  }
+
+  func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+    %0 = fir.alloca f32
+    // CHECK: %[[ALLOCA:.*]] = fir.alloca f32
+    %c1 = arith.constant 1 : index
+    %c11_i32 = arith.constant 11 : i32
+    %c6_i32 = arith.constant 6 : i32
+    %c1_i32 = arith.constant 1 : i32
+    // CHECK: gpu.launch_func  @cuda_device_mod::@_QPsub_device1 blocks in (%{{.*}}, %{{.*}}, %{{.*}}) threads in (%{{.*}}, %{{.*}}, %{{.*}})  dynamic_shared_memory_size %c0{{.*}}
+    cuf.kernel_launch @cuda_device_mod::@_QPsub_device1<<<%c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32>>>()
+
+    // CHECK: gpu.launch_func  @cuda_device_mod::@_QPsub_device2 blocks in (%{{.*}}, %{{.*}}, %{{.*}}) threads in (%{{.*}}, %{{.*}}, %{{.*}})  dynamic_shared_memory_size %c0{{.*}} args(%[[ALLOCA]] : !fir.ref<f32>)
+    cuf.kernel_launch @cuda_device_mod::@_QPsub_device2<<<%c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32>>>(%0) : (!fir.ref<f32>)
+    return
+  }
+
+}
+
+// -----
+
+module attributes {gpu.container_module, dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>} {
+  gpu.module @cuda_device_mod {
+    gpu.func @_QMmod1Psub1(%arg0: !fir.ref<!fir.array<10xi32>>) kernel {
+      gpu.return
+    }
+  }
+
+  func.func @_QMmod1Psub1(%arg0: !fir.ref<!fir.array<10xi32>> {cuf.data_attr = #cuf.cuda<device>, fir.bindc_name = "adev"}) attributes {cuf.cluster_dims = #cuf.cluster_dims<x = 2 : i64, y = 2 : i64, z = 1 : i64>, cuf.proc_attr = #cuf.cuda_proc<global>} {
+    return
+  }
+  func.func @_QMmod1Phost_sub() {
+    %c10 = arith.constant 10 : index
+    %0 = cuf.alloc !fir.array<10xi32> {bindc_name = "adev", data_attr = #cuf.cuda<device>, uniq_name = "_QMmod1Fhost_subEadev"} -> !fir.ref<!fir.array<10xi32>>
+    %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+    %2:2 = hlfir.declare %0(%1) {data_attr = #cuf.cuda<device>, uniq_name = "_QMmod1Fhost_subEadev"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+    %c1_i32 = arith.constant 1 : i32
+    %c10_i32 = arith.constant 10 : i32
+    cuf.kernel_launch @_QMmod1Psub1<<<%c1_i32, %c1_i32, %c1_i32, %c10_i32, %c1_i32, %c1_i32>>>(%2#1) : (!fir.ref<!fir.array<10xi32>>)
+    return
+  }
+}
+
+// CHECK-LABEL: func.func @_QMmod1Phost_sub()
+// CHECK: gpu.launch_func  @cuda_device_mod::@_QMmod1Psub1 clusters in (%c2{{.*}}, %c2{{.*}}, %c1{{.*}})
+
-- 
GitLab


From 255e441613e39a391e9f85d6a605cc9e46dcf273 Mon Sep 17 00:00:00 2001
From: Matthias Braun <matze@braunis.de>
Date: Tue, 29 Oct 2024 17:16:17 -0700
Subject: [PATCH 518/693] X86: Do not return invalid cost for fp16 conversion
 (#114128)

Returning invalid instruction costs when converting from/to fp16 in
`X86TTIImpl::getCastInstrCost` when there is no hardware support
available was triggering asserts. This changes the code to return a
large (arbitrary) number to model the fact that libcalls are used to
implement the conversion.

This also simplifies the code by only reporting costs for the scalar
fp16 conversion; vectorized costs being left to the fallback assuming
scalarization.

This is a follow-up to assertion issues reported for the changes in
#113195
---
 llvm/lib/Target/X86/X86TargetTransformInfo.cpp       | 12 +++++++-----
 .../Transforms/SLPVectorizer/X86/conversion-fp16.ll  | 11 +++--------
 2 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index bae223243b3d..520284d1d7a4 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -3068,6 +3068,13 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
         if (auto KindCost = Entry->Cost[CostKind])
           return *KindCost;
     }
+
+    if ((ISD == ISD::FP_ROUND && SimpleDstTy == MVT::f16) ||
+        (ISD == ISD::FP_EXTEND && SimpleSrcTy == MVT::f16)) {
+      // fp16 conversions not covered by any table entries require a libcall.
+      // Return a large (arbitrary) number to model this.
+      return InstructionCost(64);
+    }
   }
 
   // Fall back to legalized types.
@@ -3174,11 +3181,6 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
                             TTI::CastContextHint::None, CostKind);
   }
 
-  if (ISD == ISD::FP_ROUND && LTDest.second.getScalarType() == MVT::f16) {
-    // Conversion requires a libcall.
-    return InstructionCost::getInvalid();
-  }
-
   // TODO: Allow non-throughput costs that aren't binary.
   auto AdjustCost = [&CostKind](InstructionCost Cost,
                                 InstructionCost N = 1) -> InstructionCost {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/conversion-fp16.ll b/llvm/test/Transforms/SLPVectorizer/X86/conversion-fp16.ll
index bcea147d724f..f23043f0c47f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/conversion-fp16.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/conversion-fp16.ll
@@ -453,14 +453,9 @@ define void @fpround_v16xf32_v16xf16(ptr %s0, ptr %d0) {
 ;
 ; CHECK-F16C-LABEL: define void @fpround_v16xf32_v16xf16(
 ; CHECK-F16C-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
-; CHECK-F16C-NEXT:    [[S8:%.*]] = getelementptr inbounds float, ptr [[S0]], i64 8
-; CHECK-F16C-NEXT:    [[D8:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 8
-; CHECK-F16C-NEXT:    [[TMP1:%.*]] = load <8 x float>, ptr [[S0]], align 4
-; CHECK-F16C-NEXT:    [[TMP2:%.*]] = fptrunc <8 x float> [[TMP1]] to <8 x half>
-; CHECK-F16C-NEXT:    [[TMP3:%.*]] = load <8 x float>, ptr [[S8]], align 4
-; CHECK-F16C-NEXT:    [[TMP4:%.*]] = fptrunc <8 x float> [[TMP3]] to <8 x half>
-; CHECK-F16C-NEXT:    store <8 x half> [[TMP2]], ptr [[D0]], align 2
-; CHECK-F16C-NEXT:    store <8 x half> [[TMP4]], ptr [[D8]], align 2
+; CHECK-F16C-NEXT:    [[TMP1:%.*]] = load <16 x float>, ptr [[S0]], align 4
+; CHECK-F16C-NEXT:    [[TMP2:%.*]] = fptrunc <16 x float> [[TMP1]] to <16 x half>
+; CHECK-F16C-NEXT:    store <16 x half> [[TMP2]], ptr [[D0]], align 2
 ; CHECK-F16C-NEXT:    ret void
 ;
 ; CHECK-AVX512-LABEL: define void @fpround_v16xf32_v16xf16(
-- 
GitLab


From 13a3c4f97cf33279d597148ec48c71337aa16e9a Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 29 Oct 2024 17:46:52 -0700
Subject: [PATCH 519/693] [RISCV] Add OperandType to frmarg and rtzarg.
 (#114142)

Teach RISCVInstrInfo::verifyInstruction to validate them.

This is partially extracted from #89047, but that did not include the
verification.
---
 llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h | 7 ++++++-
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp           | 6 ++++++
 llvm/lib/Target/RISCV/RISCVInstrInfoF.td           | 4 ++++
 llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td         | 2 ++
 4 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index d82f78498418..e18329c3d2dd 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -330,7 +330,12 @@ enum OperandType : unsigned {
   OPERAND_RVKRNUM_1_10,
   OPERAND_RVKRNUM_2_14,
   OPERAND_SPIMM,
-  OPERAND_LAST_RISCV_IMM = OPERAND_SPIMM,
+  // Operand is a 3-bit rounding mode, '111' indicates FRM register.
+  // Represents 'frm' argument passing to floating-point operations.
+  OPERAND_FRMARG,
+  // Operand is a 3-bit rounding mode where only RTZ is valid.
+  OPERAND_RTZARG,
+  OPERAND_LAST_RISCV_IMM = OPERAND_RTZARG,
   // Operand is either a register or uimm5, this is used by V extension pseudo
   // instructions to represent a value that be passed as AVL to either vsetvli
   // or vsetivli.
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index a3963fadf3e4..20e531657eb2 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -2536,6 +2536,12 @@ bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
         case RISCVOp::OPERAND_SPIMM:
           Ok = (Imm & 0xf) == 0;
           break;
+        case RISCVOp::OPERAND_FRMARG:
+          Ok = RISCVFPRndMode::isValidRoundingMode(Imm);
+          break;
+        case RISCVOp::OPERAND_RTZARG:
+          Ok = Imm == RISCVFPRndMode::RTZ;
+          break;
         }
         if (!Ok) {
           ErrInfo = "Invalid immediate";
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
index a134f37c7749..da3f207a2faf 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
@@ -134,6 +134,8 @@ def frmarg : Operand<XLenVT> {
   let ParserMatchClass = FRMArg;
   let PrintMethod = "printFRMArg";
   let DecoderMethod = "decodeFRMArg";
+  let OperandType = "OPERAND_FRMARG";
+  let OperandNamespace = "RISCVOp";
 }
 
 // Variants of the rounding mode operand that default to 'rne'. This is used
@@ -154,6 +156,8 @@ def frmarglegacy : Operand<XLenVT> {
   let ParserMatchClass = FRMArgLegacy;
   let PrintMethod = "printFRMArgLegacy";
   let DecoderMethod = "decodeFRMArg";
+  let OperandType = "OPERAND_FRMARG";
+  let OperandNamespace = "RISCVOp";
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
index f62a7e122112..2bdcfd21270e 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
@@ -48,6 +48,8 @@ def rtzarg : Operand<XLenVT> {
   let ParserMatchClass = RTZArg;
   let PrintMethod = "printFRMArg";
   let DecoderMethod = "decodeRTZArg";
+  let OperandType = "OPERAND_RTZARG";
+  let OperandNamespace = "RISCVOp";
 }
 
 //===----------------------------------------------------------------------===//
-- 
GitLab


From d9268289c3858c4ae877ff3bb90f28c160a977c8 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 29 Oct 2024 18:08:26 -0700
Subject: [PATCH 520/693] [RISCV] Add sha and supm to checks in
 riscv-profiles.c (#114123)

---
 clang/test/Driver/riscv-profiles.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/clang/test/Driver/riscv-profiles.c b/clang/test/Driver/riscv-profiles.c
index 2b4d19422874..67e09d0e69eb 100644
--- a/clang/test/Driver/riscv-profiles.c
+++ b/clang/test/Driver/riscv-profiles.c
@@ -147,6 +147,7 @@
 // RVA23U64: "-target-feature" "+zvbb"
 // RVA23U64: "-target-feature" "+zvfhmin"
 // RVA23U64: "-target-feature" "+zvkt"
+// RVA23U64: "-target-feature" "+supm"
 
 // RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva23s64 \
 // RUN:   | FileCheck -check-prefix=RVA23S64 %s
@@ -186,6 +187,7 @@
 // RVA23S64: "-target-feature" "+zvbb"
 // RVA23S64: "-target-feature" "+zvfhmin"
 // RVA23S64: "-target-feature" "+zvkt"
+// RVA23S64: "-target-feature" "+sha"
 // RVA23S64: "-target-feature" "+shcounterenw"
 // RVA23S64: "-target-feature" "+shgatpa"
 // RVA23S64: "-target-feature" "+shtvala"
@@ -201,6 +203,7 @@
 // RVA23S64: "-target-feature" "+sstvala"
 // RVA23S64: "-target-feature" "+sstvecd"
 // RVA23S64: "-target-feature" "+ssu64xl"
+// RVA23S64: "-target-feature" "+supm"
 // RVA23S64: "-target-feature" "+svade"
 // RVA23S64: "-target-feature" "+svbare"
 // RVA23S64: "-target-feature" "+svinval"
-- 
GitLab


From 8800b739bfe3ddc0bd32c158a016ffd0eee1e352 Mon Sep 17 00:00:00 2001
From: Brandon Wu <brandon.wu@sifive.com>
Date: Wed, 30 Oct 2024 09:27:35 +0800
Subject: [PATCH 521/693] [RISCV] Refactor FP, SP and RA in
 RISCVFrameLowering.cpp. NFC (#113818)

Those registers are too fragmented in terms of usage, some are hard
coded and some are retrieved by calling function. Also some have
comments for alias name, some don't.
---
 llvm/lib/Target/RISCV/RISCVFrameLowering.cpp | 48 +++++++++-----------
 1 file changed, 22 insertions(+), 26 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index d70903519ecb..f5851f371545 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -42,10 +42,19 @@ RISCVFrameLowering::RISCVFrameLowering(const RISCVSubtarget &STI)
           /*TransientStackAlignment=*/getABIStackAlignment(STI.getTargetABI())),
       STI(STI) {}
 
+// The register used to hold the frame pointer.
+static constexpr Register FPReg = RISCV::X8;
+
+// The register used to hold the stack pointer.
+static constexpr Register SPReg = RISCV::X2;
+
+// The register used to hold the return address.
+static constexpr Register RAReg = RISCV::X1;
+
 // Offsets which need to be scale by XLen representing locations of CSRs which
 // are given a fixed location by save/restore libcalls or Zcmp Push/Pop.
 static const std::pair<MCPhysReg, int8_t> FixedCSRFIMap[] = {
-    {/*ra*/ RISCV::X1, -1},   {/*s0*/ RISCV::X8, -2},
+    {/*ra*/ RAReg, -1},       {/*s0*/ FPReg, -2},
     {/*s1*/ RISCV::X9, -3},   {/*s2*/ RISCV::X18, -4},
     {/*s3*/ RISCV::X19, -5},  {/*s4*/ RISCV::X20, -6},
     {/*s5*/ RISCV::X21, -7},  {/*s6*/ RISCV::X22, -8},
@@ -187,6 +196,7 @@ static int getLibCallID(const MachineFunction &MF,
   switch (MaxReg) {
   default:
     llvm_unreachable("Something has gone wrong!");
+    // clang-format off
   case /*s11*/ RISCV::X27: return 12;
   case /*s10*/ RISCV::X26: return 11;
   case /*s9*/  RISCV::X25: return 10;
@@ -198,8 +208,9 @@ static int getLibCallID(const MachineFunction &MF,
   case /*s3*/  RISCV::X19: return 4;
   case /*s2*/  RISCV::X18: return 3;
   case /*s1*/  RISCV::X9:  return 2;
-  case /*s0*/  RISCV::X8:  return 1;
-  case /*ra*/  RISCV::X1:  return 0;
+  case /*s0*/  FPReg:  return 1;
+  case /*ra*/  RAReg:  return 0;
+    // clang-format on
   }
 }
 
@@ -284,9 +295,9 @@ getPushPopEncodingAndNum(const Register MaxReg) {
     return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0_S2, 4);
   case RISCV::X9: /*s1*/
     return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0_S1, 3);
-  case RISCV::X8: /*s0*/
+  case FPReg: /*s0*/
     return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0, 2);
-  case RISCV::X1: /*ra*/
+  case RAReg: /*ra*/
     return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA, 1);
   }
 }
@@ -372,12 +383,6 @@ uint64_t RISCVFrameLowering::getStackSizeWithRVVPadding(
   return alignTo(MFI.getStackSize() + RVFI->getRVVPadding(), getStackAlign());
 }
 
-// Returns the register used to hold the frame pointer.
-static Register getFPReg(const RISCVSubtarget &STI) { return RISCV::X8; }
-
-// Returns the register used to hold the stack pointer.
-static Register getSPReg(const RISCVSubtarget &STI) { return RISCV::X2; }
-
 static SmallVector<CalleeSavedInfo, 8>
 getUnmanagedCSI(const MachineFunction &MF,
                 const std::vector<CalleeSavedInfo> &CSI) {
@@ -415,8 +420,6 @@ void RISCVFrameLowering::adjustStackForRVV(MachineFunction &MF,
                                            MachineInstr::MIFlag Flag) const {
   assert(Amount != 0 && "Did not need to adjust stack pointer for RVV.");
 
-  const Register SPReg = getSPReg(STI);
-
   // Optimize compile time offset case
   StackOffset Offset = StackOffset::getScalable(Amount);
   if (auto VLEN = STI.getRealVLen()) {
@@ -479,7 +482,7 @@ static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
   unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
   Expr.push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
   Expr.push_back(0);
-  if (Reg == RISCV::X2)
+  if (Reg == SPReg)
     Comment << "sp";
   else
     Comment << printReg(Reg, &TRI);
@@ -530,8 +533,6 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   const RISCVInstrInfo *TII = STI.getInstrInfo();
   MachineBasicBlock::iterator MBBI = MBB.begin();
 
-  Register FPReg = getFPReg(STI);
-  Register SPReg = getSPReg(STI);
   Register BPReg = RISCVABI::getBPReg();
 
   // Debug location must be unknown since the first debug location is used
@@ -762,8 +763,6 @@ void RISCVFrameLowering::deallocateStack(MachineFunction &MF,
                                          int64_t CFAOffset) const {
   const RISCVRegisterInfo *RI = STI.getRegisterInfo();
 
-  Register SPReg = getSPReg(STI);
-
   RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(StackSize),
                 MachineInstr::FrameDestroy, getStackAlign());
 }
@@ -773,8 +772,6 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
   const RISCVRegisterInfo *RI = STI.getRegisterInfo();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
-  Register FPReg = getFPReg(STI);
-  Register SPReg = getSPReg(STI);
 
   // All calls are tail calls in GHC calling conv, and functions have no
   // prologue/epilogue.
@@ -922,7 +919,7 @@ RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
   }
 
   if (FI >= MinCSFI && FI <= MaxCSFI) {
-    FrameReg = RISCV::X2;
+    FrameReg = SPReg;
 
     if (FirstSPAdjustAmount)
       Offset += StackOffset::getFixed(FirstSPAdjustAmount);
@@ -969,13 +966,13 @@ RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
     } else {
       // VarSize objects must be empty in this case!
       assert(!MFI.hasVarSizedObjects());
-      FrameReg = RISCV::X2;
+      FrameReg = SPReg;
     }
   } else {
     FrameReg = RI->getFrameRegister(MF);
   }
 
-  if (FrameReg == getFPReg(STI)) {
+  if (FrameReg == FPReg) {
     Offset += StackOffset::getFixed(RVFI->getVarArgsSaveSize());
     // When using FP to access scalable vector objects, we need to minus
     // the frame size.
@@ -1067,8 +1064,8 @@ void RISCVFrameLowering::determineCalleeSaves(MachineFunction &MF,
   // Unconditionally spill RA and FP only if the function uses a frame
   // pointer.
   if (hasFP(MF)) {
-    SavedRegs.set(RISCV::X1);
-    SavedRegs.set(RISCV::X8);
+    SavedRegs.set(RAReg);
+    SavedRegs.set(FPReg);
   }
   // Mark BP as used if function has dedicated base pointer.
   if (hasBP(MF))
@@ -1328,7 +1325,6 @@ bool RISCVFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
 MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr(
     MachineFunction &MF, MachineBasicBlock &MBB,
     MachineBasicBlock::iterator MI) const {
-  Register SPReg = RISCV::X2;
   DebugLoc DL = MI->getDebugLoc();
 
   if (!hasReservedCallFrame(MF)) {
-- 
GitLab


From 63eb40eeb1b7aac57a181f6b5f9170ea94cef738 Mon Sep 17 00:00:00 2001
From: "A. Jiang" <de34@live.cn>
Date: Wed, 30 Oct 2024 09:49:26 +0800
Subject: [PATCH 522/693] [libc++] Deprecate and remove meaningless `<cxxx>`
 headers (#111615)

This PR deprecates `<ccomplex>`, `<cstdbool>`, `<ctgmath>`, and
`<ciso646>` in C++17 and "removes" them in C++20 by special deprecation
warnings.

`<cstdalign>` is previously missing. This PR also tries to add them, and
then deprecates and "removes" `<cstdalign>`.

Papers:
- https://wg21.link/P0063R3
- https://wg21.link/P0619R4

Closes #99985.

---------

Co-authored-by: Louis Dionne <ldionne.2@gmail.com>
---
 libcxx/docs/ReleaseNotes/20.rst               |  6 ++
 libcxx/docs/Status/Cxx20Papers.csv            |  2 +-
 libcxx/include/CMakeLists.txt                 |  1 +
 libcxx/include/ccomplex                       | 12 ++++
 libcxx/include/ciso646                        |  7 +++
 libcxx/include/complex.h                      |  2 +-
 libcxx/include/cstdalign                      | 55 +++++++++++++++++++
 libcxx/include/cstdbool                       | 12 ++++
 libcxx/include/ctgmath                        | 14 ++++-
 libcxx/include/module.modulemap               |  5 ++
 libcxx/include/tgmath.h                       |  3 +-
 .../test/libcxx/clang_modules_include.gen.py  | 13 ++++-
 libcxx/test/libcxx/double_include.gen.py      | 13 ++++-
 libcxx/test/libcxx/header_inclusions.gen.py   | 37 ++++++++-----
 libcxx/test/libcxx/include_as_c.sh.cpp        |  1 +
 libcxx/test/libcxx/libcpp_version.gen.py      | 13 ++++-
 libcxx/test/libcxx/no_assert_include.gen.py   | 18 ++++--
 .../test/libcxx/system_reserved_names.gen.py  |  8 ++-
 .../test/libcxx/transitive_includes/cxx03.csv |  1 -
 .../test/libcxx/transitive_includes/cxx11.csv |  1 -
 .../test/libcxx/transitive_includes/cxx14.csv |  1 -
 .../test/libcxx/transitive_includes/cxx17.csv |  1 -
 .../test/libcxx/transitive_includes/cxx20.csv |  1 -
 .../test/libcxx/transitive_includes/cxx23.csv |  1 -
 .../test/libcxx/transitive_includes/cxx26.csv |  1 -
 .../depr.c.headers/ciso646.compile.pass.cpp   |  4 +-
 .../stdalign_h.compile.pass.cpp               | 33 +++++++++++
 .../depr/depr.cpp.headers/ccomplex.verify.cpp | 25 +++++++++
 .../depr/depr.cpp.headers/ciso646.verify.cpp  | 18 ++++++
 .../depr.cpp.headers/cstdalign.verify.cpp     | 25 +++++++++
 .../depr/depr.cpp.headers/cstdbool.verify.cpp | 25 +++++++++
 .../depr/depr.cpp.headers/ctgmath.verify.cpp  | 25 +++++++++
 .../cstdalign.compile.pass.cpp                | 29 ++++++++++
 .../support.runtime/cstdbool.pass.cpp         |  4 +-
 .../test/std/numerics/c.math/ctgmath.pass.cpp |  4 +-
 .../complex.number/ccmplx/ccomplex.pass.cpp   |  4 +-
 libcxx/utils/libcxx/header_information.py     | 12 +++-
 .../gn/secondary/libcxx/include/BUILD.gn      |  1 +
 38 files changed, 394 insertions(+), 44 deletions(-)
 create mode 100644 libcxx/include/cstdalign
 create mode 100644 libcxx/test/std/depr/depr.c.headers/stdalign_h.compile.pass.cpp
 create mode 100644 libcxx/test/std/depr/depr.cpp.headers/ccomplex.verify.cpp
 create mode 100644 libcxx/test/std/depr/depr.cpp.headers/ciso646.verify.cpp
 create mode 100644 libcxx/test/std/depr/depr.cpp.headers/cstdalign.verify.cpp
 create mode 100644 libcxx/test/std/depr/depr.cpp.headers/cstdbool.verify.cpp
 create mode 100644 libcxx/test/std/depr/depr.cpp.headers/ctgmath.verify.cpp
 create mode 100644 libcxx/test/std/language.support/support.runtime/cstdalign.compile.pass.cpp

diff --git a/libcxx/docs/ReleaseNotes/20.rst b/libcxx/docs/ReleaseNotes/20.rst
index 38b8df3b2a77..bf3aafe6139e 100644
--- a/libcxx/docs/ReleaseNotes/20.rst
+++ b/libcxx/docs/ReleaseNotes/20.rst
@@ -38,6 +38,7 @@ What's New in Libc++ 20.0.0?
 Implemented Papers
 ------------------
 
+- P0619R4: Reviewing Deprecated Facilities of C++17 for C++20 (`Github <https://github.com/llvm/llvm-project/issues/99985>`__)
 - P2747R2: ``constexpr`` placement new (`Github <https://github.com/llvm/llvm-project/issues/105427>`__)
 - P2609R3: Relaxing Ranges Just A Smidge (`Github <https://github.com/llvm/llvm-project/issues/105253>`__)
 - P2985R0: A type trait for detecting virtual base classes (`Github <https://github.com/llvm/llvm-project/issues/105432>`__)
@@ -89,6 +90,11 @@ Deprecations and Removals
   the ``_LIBCPP_VERBOSE_ABORT_NOT_NOEXCEPT`` macro can be defined to make the function non-``noexcept``. That macro
   will be removed in LLVM 21.
 
+- ``<ccomplex>``, ``<cstdalign>`` (previously missing), ``<cstdbool>``, and ``<ctgmath>`` are deprecated since C++17 as
+  specified by the standard. They, together with ``<ciso646>``, are removed in C++20, but libc++ still provides these
+  headers as an extension and only deprecates them. The ``_LIBCPP_DISABLE_DEPRECATION_WARNINGS`` macro can be defined to
+  suppress deprecation for these headers.
+
 Upcoming Deprecations and Removals
 ----------------------------------
 
diff --git a/libcxx/docs/Status/Cxx20Papers.csv b/libcxx/docs/Status/Cxx20Papers.csv
index 9a057be8ad05..5cd77be4d58d 100644
--- a/libcxx/docs/Status/Cxx20Papers.csv
+++ b/libcxx/docs/Status/Cxx20Papers.csv
@@ -34,7 +34,7 @@
 "`P0528R3 <https://wg21.link/P0528R3>`__","The Curious Case of Padding Bits, Featuring Atomic Compare-and-Exchange","2018-06 (Rapperswil)","","",""
 "`P0542R5 <https://wg21.link/P0542R5>`__","Support for contract based programming in C++","2018-06 (Rapperswil)","|Nothing To Do|","n/a","Pulled at the 2019-07 meeting in Cologne"
 "`P0556R3 <https://wg21.link/P0556R3>`__","Integral power-of-2 operations","2018-06 (Rapperswil)","|Complete|","9.0",""
-"`P0619R4 <https://wg21.link/P0619R4>`__","Reviewing Deprecated Facilities of C++17 for C++20","2018-06 (Rapperswil)","|Partial|","","Only sections D.7, D.8, D.9, D.10, D.11, D.12, and D.13 are implemented. Section D.4 remains undone."
+"`P0619R4 <https://wg21.link/P0619R4>`__","Reviewing Deprecated Facilities of C++17 for C++20","2018-06 (Rapperswil)","|Complete|","20.0","Removed headers are still provided as an extension, but with deprecation warnings"
 "`P0646R1 <https://wg21.link/P0646R1>`__","Improving the Return Value of Erase-Like Algorithms","2018-06 (Rapperswil)","|Complete|","10.0",""
 "`P0722R3 <https://wg21.link/P0722R3>`__","Efficient sized delete for variable sized classes","2018-06 (Rapperswil)","|Complete|","9.0",""
 "`P0758R1 <https://wg21.link/P0758R1>`__","Implicit conversion traits and utility functions","2018-06 (Rapperswil)","|Complete|","",""
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index e84a55e25f2f..87eaf64b2450 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -923,6 +923,7 @@ set(files
   coroutine
   csetjmp
   csignal
+  cstdalign
   cstdarg
   cstdbool
   cstddef
diff --git a/libcxx/include/ccomplex b/libcxx/include/ccomplex
index 94d2c8d7d003..d379c9e7f017 100644
--- a/libcxx/include/ccomplex
+++ b/libcxx/include/ccomplex
@@ -23,4 +23,16 @@
 #  pragma GCC system_header
 #endif
 
+#if _LIBCPP_STD_VER >= 20
+
+using __standard_header_ccomplex _LIBCPP_DEPRECATED_("removed in C++20. Include <complex> instead.") = void;
+using __use_standard_header_ccomplex = __standard_header_ccomplex;
+
+#elif _LIBCPP_STD_VER >= 17
+
+using __standard_header_ccomplex _LIBCPP_DEPRECATED_("Include <complex> instead.") = void;
+using __use_standard_header_ccomplex                                               = __standard_header_ccomplex;
+
+#endif
+
 #endif // _LIBCPP_CCOMPLEX
diff --git a/libcxx/include/ciso646 b/libcxx/include/ciso646
index 1d859f08fac5..5fcac79e38a7 100644
--- a/libcxx/include/ciso646
+++ b/libcxx/include/ciso646
@@ -21,4 +21,11 @@
 #  pragma GCC system_header
 #endif
 
+#if _LIBCPP_STD_VER >= 20
+
+using __standard_header_ciso646 _LIBCPP_DEPRECATED_("removed in C++20. Include <version> instead.") = void;
+using __use_standard_header_ciso646 = __standard_header_ciso646;
+
+#endif
+
 #endif // _LIBCPP_CISO646
diff --git a/libcxx/include/complex.h b/libcxx/include/complex.h
index a3da21c843f3..89595ae2068a 100644
--- a/libcxx/include/complex.h
+++ b/libcxx/include/complex.h
@@ -24,7 +24,7 @@
 #endif
 
 #ifdef __cplusplus
-#  include <ccomplex>
+#  include <complex>
 #elif __has_include_next(<complex.h>)
 #  include_next <complex.h>
 #endif
diff --git a/libcxx/include/cstdalign b/libcxx/include/cstdalign
new file mode 100644
index 000000000000..e6a2a3c71774
--- /dev/null
+++ b/libcxx/include/cstdalign
@@ -0,0 +1,55 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_CSTDALIGN
+#define _LIBCPP_CSTDALIGN
+
+/*
+    cstdalign synopsis
+
+Macros:
+
+    __alignas_is_defined
+    __alignof_is_defined
+
+*/
+
+#include <__config>
+
+// <stdalign.h> is not provided by libc++
+#if __has_include(<stdalign.h>)
+#  include <stdalign.h>
+#  ifdef _LIBCPP_STDALIGN_H
+#    error "If libc++ starts defining <stdalign.h>, the __has_include check should move to libc++'s <stdalign.h>"
+#  endif
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#undef __alignas_is_defined
+#define __alignas_is_defined 1
+
+#undef __alignof_is_defined
+#define __alignof_is_defined 1
+
+#if _LIBCPP_STD_VER >= 20
+
+using __standard_header_cstdalign _LIBCPP_DEPRECATED_("removed in C++20.") = void;
+using __use_standard_header_cstdalign                                      = __standard_header_cstdalign;
+
+#elif _LIBCPP_STD_VER >= 17
+
+using __standard_header_cstdalign _LIBCPP_DEPRECATED = void;
+using __use_standard_header_cstdalign                = __standard_header_cstdalign;
+
+#endif
+
+#endif // _LIBCPP_CSTDALIGN
diff --git a/libcxx/include/cstdbool b/libcxx/include/cstdbool
index ef731c021a4a..1d627258e10c 100644
--- a/libcxx/include/cstdbool
+++ b/libcxx/include/cstdbool
@@ -28,4 +28,16 @@ Macros:
 #undef __bool_true_false_are_defined
 #define __bool_true_false_are_defined 1
 
+#if _LIBCPP_STD_VER >= 20
+
+using __standard_header_cstdbool _LIBCPP_DEPRECATED_("removed in C++20.") = void;
+using __use_standard_header_cstdbool                                      = __standard_header_cstdbool;
+
+#elif _LIBCPP_STD_VER >= 17
+
+using __standard_header_cstdbool _LIBCPP_DEPRECATED = void;
+using __use_standard_header_cstdbool                = __standard_header_cstdbool;
+
+#endif
+
 #endif // _LIBCPP_CSTDBOOL
diff --git a/libcxx/include/ctgmath b/libcxx/include/ctgmath
index 6237979be490..7dbe952f021b 100644
--- a/libcxx/include/ctgmath
+++ b/libcxx/include/ctgmath
@@ -18,11 +18,23 @@
 
 */
 
-#include <ccomplex>
 #include <cmath>
+#include <complex>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
+#if _LIBCPP_STD_VER >= 20
+
+using __standard_header_ctgmath _LIBCPP_DEPRECATED_("removed in C++20. Include <cmath> and <complex> instead.") = void;
+using __use_standard_header_ctgmath = __standard_header_ctgmath;
+
+#elif _LIBCPP_STD_VER >= 17
+
+using __standard_header_ctgmath _LIBCPP_DEPRECATED_("Include <cmath> and <complex> instead.") = void;
+using __use_standard_header_ctgmath = __standard_header_ctgmath;
+
+#endif
+
 #endif // _LIBCPP_CTGMATH
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index c3d080007319..af8c3c15eb27 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1109,6 +1109,11 @@ module std [system] {
     export *
   }
 
+  module cstdalign {
+    header "cstdalign"
+    export *
+  }
+
   module cstdarg {
     header "cstdarg"
     export *
diff --git a/libcxx/include/tgmath.h b/libcxx/include/tgmath.h
index e6f0a4ab2611..1c5058cb065a 100644
--- a/libcxx/include/tgmath.h
+++ b/libcxx/include/tgmath.h
@@ -24,7 +24,8 @@
 #endif
 
 #ifdef __cplusplus
-#  include <ctgmath>
+#  include <cmath>
+#  include <complex>
 #else
 #  if __has_include_next(<tgmath.h>)
 #    include_next <tgmath.h>
diff --git a/libcxx/test/libcxx/clang_modules_include.gen.py b/libcxx/test/libcxx/clang_modules_include.gen.py
index bc028f2a0809..b897984f8988 100644
--- a/libcxx/test/libcxx/clang_modules_include.gen.py
+++ b/libcxx/test/libcxx/clang_modules_include.gen.py
@@ -17,10 +17,15 @@
 
 import sys
 sys.path.append(sys.argv[1])
-from libcxx.header_information import lit_header_restrictions, public_headers
+from libcxx.header_information import (
+    lit_header_restrictions,
+    lit_header_undeprecations,
+    public_headers,
+)
 
 for header in public_headers:
-  print(f"""\
+    print(
+        f"""\
 //--- {header}.compile.pass.cpp
 // RUN: %{{cxx}} %s %{{flags}} %{{compile_flags}} -fmodules -fcxx-modules -fmodules-cache-path=%t -fsyntax-only
 
@@ -41,9 +46,11 @@ for header in public_headers:
 // UNSUPPORTED: LIBCXX-PICOLIBC-FIXME
 
 {lit_header_restrictions.get(header, '')}
+{lit_header_undeprecations.get(header, '')}
 
 #include <{header}>
-""")
+"""
+    )
 
 print(
     f"""\
diff --git a/libcxx/test/libcxx/double_include.gen.py b/libcxx/test/libcxx/double_include.gen.py
index afc2947dbece..f58e72f94a35 100644
--- a/libcxx/test/libcxx/double_include.gen.py
+++ b/libcxx/test/libcxx/double_include.gen.py
@@ -15,12 +15,18 @@
 
 import sys
 sys.path.append(sys.argv[1])
-from libcxx.header_information import lit_header_restrictions, public_headers
+from libcxx.header_information import (
+    lit_header_restrictions,
+    lit_header_undeprecations,
+    public_headers,
+)
 
 for header in public_headers:
-  print(f"""\
+    print(
+        f"""\
 //--- {header}.sh.cpp
 {lit_header_restrictions.get(header, '')}
+{lit_header_undeprecations.get(header, '')}
 
 // RUN: %{{cxx}} -c %s -o %t.first.o %{{flags}} %{{compile_flags}}
 // RUN: %{{cxx}} -c %s -o %t.second.o -DWITH_MAIN %{{flags}} %{{compile_flags}}
@@ -32,4 +38,5 @@ for header in public_headers:
 #if defined(WITH_MAIN)
 int main(int, char**) {{ return 0; }}
 #endif
-""")
+"""
+    )
diff --git a/libcxx/test/libcxx/header_inclusions.gen.py b/libcxx/test/libcxx/header_inclusions.gen.py
index e5def1ad4cb7..739caf915c09 100644
--- a/libcxx/test/libcxx/header_inclusions.gen.py
+++ b/libcxx/test/libcxx/header_inclusions.gen.py
@@ -12,32 +12,43 @@
 # RUN: %{python} %s %{libcxx-dir}/utils
 
 import sys
+
 sys.path.append(sys.argv[1])
-from libcxx.header_information import lit_header_restrictions, public_headers, mandatory_inclusions
+from libcxx.header_information import (
+    lit_header_restrictions,
+    lit_header_undeprecations,
+    public_headers,
+    mandatory_inclusions,
+)
 
 for header in public_headers:
-  header_guard = lambda h: f"_LIBCPP_{str(h).upper().replace('.', '_').replace('/', '_')}"
+    header_guard = (
+        lambda h: f"_LIBCPP_{str(h).upper().replace('.', '_').replace('/', '_')}"
+    )
 
-  # <cassert> has no header guards
-  if header == 'cassert':
-    checks = ''
-  else:
-    checks = f'''
+    # <cassert> has no header guards
+    if header == "cassert":
+        checks = ""
+    else:
+        checks = f"""
 #ifndef {header_guard(header)}
 # error <{header}> was expected to define a header guard {header_guard(header)}
 #endif
-'''
-  for includee in mandatory_inclusions.get(header, []):
-    checks += f'''
+"""
+    for includee in mandatory_inclusions.get(header, []):
+        checks += f"""
 #ifndef {header_guard(includee)}
 # error <{header}> was expected to include <{includee}>
 #endif
-'''
+"""
 
-  print(f"""\
+    print(
+        f"""\
 //--- {header}.compile.pass.cpp
 {lit_header_restrictions.get(header, '')}
+{lit_header_undeprecations.get(header, '')}
 
 #include <{header}>
 {checks}
-""")
+"""
+    )
diff --git a/libcxx/test/libcxx/include_as_c.sh.cpp b/libcxx/test/libcxx/include_as_c.sh.cpp
index c9f8dfd9a5a9..204b830462cf 100644
--- a/libcxx/test/libcxx/include_as_c.sh.cpp
+++ b/libcxx/test/libcxx/include_as_c.sh.cpp
@@ -34,6 +34,7 @@
 #endif
 #include <math.h>
 #include <setjmp.h>
+#include <stdalign.h>
 #include <stdatomic.h>
 #include <stdbool.h>
 #include <stddef.h>
diff --git a/libcxx/test/libcxx/libcpp_version.gen.py b/libcxx/test/libcxx/libcpp_version.gen.py
index a9995295e21e..b30623fe2c38 100644
--- a/libcxx/test/libcxx/libcpp_version.gen.py
+++ b/libcxx/test/libcxx/libcpp_version.gen.py
@@ -12,16 +12,23 @@
 
 import sys
 sys.path.append(sys.argv[1])
-from libcxx.header_information import lit_header_restrictions, public_headers
+from libcxx.header_information import (
+    lit_header_restrictions,
+    lit_header_undeprecations,
+    public_headers,
+)
 
 for header in public_headers:
-  print(f"""\
+    print(
+        f"""\
 //--- {header}.compile.pass.cpp
 {lit_header_restrictions.get(header, '')}
+{lit_header_undeprecations.get(header, '')}
 
 #include <{header}>
 
 #ifndef _LIBCPP_VERSION
 # error <{header}> does not seem to define _LIBCPP_VERSION
 #endif
-""")
+"""
+    )
diff --git a/libcxx/test/libcxx/no_assert_include.gen.py b/libcxx/test/libcxx/no_assert_include.gen.py
index 67ab98603ca8..e0dbc3d815f3 100644
--- a/libcxx/test/libcxx/no_assert_include.gen.py
+++ b/libcxx/test/libcxx/no_assert_include.gen.py
@@ -12,20 +12,28 @@
 # RUN: %{python} %s %{libcxx-dir}/utils
 
 import sys
+
 sys.path.append(sys.argv[1])
-from libcxx.header_information import lit_header_restrictions, public_headers
+from libcxx.header_information import (
+    lit_header_restrictions,
+    lit_header_undeprecations,
+    public_headers,
+)
 
 for header in public_headers:
-  if header == 'cassert':
-    continue
+    if header == "cassert":
+        continue
 
-  print(f"""\
+    print(
+        f"""\
 //--- {header}.compile.pass.cpp
 {lit_header_restrictions.get(header, '')}
+{lit_header_undeprecations.get(header, '')}
 
 #include <{header}>
 
 #ifdef assert
 # error "Do not include cassert or assert.h in standard header files"
 #endif
-""")
+"""
+    )
diff --git a/libcxx/test/libcxx/system_reserved_names.gen.py b/libcxx/test/libcxx/system_reserved_names.gen.py
index e29e7a2cdd61..f01126249c88 100644
--- a/libcxx/test/libcxx/system_reserved_names.gen.py
+++ b/libcxx/test/libcxx/system_reserved_names.gen.py
@@ -13,14 +13,20 @@
 # RUN: %{python} %s %{libcxx-dir}/utils
 
 import sys
+
 sys.path.append(sys.argv[1])
-from libcxx.header_information import lit_header_restrictions, public_headers
+from libcxx.header_information import (
+    lit_header_restrictions,
+    lit_header_undeprecations,
+    public_headers,
+)
 
 for header in public_headers:
     print(
         f"""\
 //--- {header}.compile.pass.cpp
 {lit_header_restrictions.get(header, '')}
+{lit_header_undeprecations.get(header, '')}
 
 #define SYSTEM_RESERVED_NAME This name should not be used in libc++
 
diff --git a/libcxx/test/libcxx/transitive_includes/cxx03.csv b/libcxx/test/libcxx/transitive_includes/cxx03.csv
index 2dc84963f089..48c501863cb7 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx03.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx03.csv
@@ -458,7 +458,6 @@ ctgmath array
 ctgmath atomic
 ctgmath bit
 ctgmath bitset
-ctgmath ccomplex
 ctgmath cctype
 ctgmath cerrno
 ctgmath climits
diff --git a/libcxx/test/libcxx/transitive_includes/cxx11.csv b/libcxx/test/libcxx/transitive_includes/cxx11.csv
index 2dc84963f089..48c501863cb7 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx11.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx11.csv
@@ -458,7 +458,6 @@ ctgmath array
 ctgmath atomic
 ctgmath bit
 ctgmath bitset
-ctgmath ccomplex
 ctgmath cctype
 ctgmath cerrno
 ctgmath climits
diff --git a/libcxx/test/libcxx/transitive_includes/cxx14.csv b/libcxx/test/libcxx/transitive_includes/cxx14.csv
index 27e229755735..6191c9012c63 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx14.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx14.csv
@@ -467,7 +467,6 @@ ctgmath array
 ctgmath atomic
 ctgmath bit
 ctgmath bitset
-ctgmath ccomplex
 ctgmath cctype
 ctgmath cerrno
 ctgmath climits
diff --git a/libcxx/test/libcxx/transitive_includes/cxx17.csv b/libcxx/test/libcxx/transitive_includes/cxx17.csv
index b17eb1f2347a..5d46162e3f89 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx17.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx17.csv
@@ -458,7 +458,6 @@ ctgmath array
 ctgmath atomic
 ctgmath bit
 ctgmath bitset
-ctgmath ccomplex
 ctgmath cctype
 ctgmath cerrno
 ctgmath climits
diff --git a/libcxx/test/libcxx/transitive_includes/cxx20.csv b/libcxx/test/libcxx/transitive_includes/cxx20.csv
index 9efec327889c..20fe9878ce3e 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx20.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx20.csv
@@ -446,7 +446,6 @@ ctgmath array
 ctgmath atomic
 ctgmath bit
 ctgmath bitset
-ctgmath ccomplex
 ctgmath cctype
 ctgmath cerrno
 ctgmath climits
diff --git a/libcxx/test/libcxx/transitive_includes/cxx23.csv b/libcxx/test/libcxx/transitive_includes/cxx23.csv
index e17f732663a9..5ee89ec307cc 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx23.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx23.csv
@@ -238,7 +238,6 @@ coroutine limits
 coroutine version
 cstddef version
 ctgmath bitset
-ctgmath ccomplex
 ctgmath cctype
 ctgmath cerrno
 ctgmath climits
diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv
index c56f5cdfad00..ee17223e66be 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx26.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv
@@ -238,7 +238,6 @@ coroutine limits
 coroutine version
 cstddef version
 ctgmath bitset
-ctgmath ccomplex
 ctgmath cctype
 ctgmath cerrno
 ctgmath climits
diff --git a/libcxx/test/std/depr/depr.c.headers/ciso646.compile.pass.cpp b/libcxx/test/std/depr/depr.c.headers/ciso646.compile.pass.cpp
index 4dff57f84f20..764f4d02f44f 100644
--- a/libcxx/test/std/depr/depr.c.headers/ciso646.compile.pass.cpp
+++ b/libcxx/test/std/depr/depr.c.headers/ciso646.compile.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// <ciso646>
+// <ciso646> // removed in C++20, but still provided by libc++ as an extension
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 
 #include <ciso646>
diff --git a/libcxx/test/std/depr/depr.c.headers/stdalign_h.compile.pass.cpp b/libcxx/test/std/depr/depr.c.headers/stdalign_h.compile.pass.cpp
new file mode 100644
index 000000000000..e7290aab2c66
--- /dev/null
+++ b/libcxx/test/std/depr/depr.c.headers/stdalign_h.compile.pass.cpp
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// test <stdalign.h>
+//
+// Even though <stdalign.h> is not provided by libc++,
+// we still test that using it with libc++ on the search path will work.
+
+// TODO: GCC doesn't provide a proper <stdalign.h> for C++ until 15.
+// UNSUPPORTED: gcc
+
+#include <stdalign.h>
+
+#ifndef __alignas_is_defined
+#  error __alignas_is_defined not defined
+#endif
+
+#ifndef __alignof_is_defined
+#  error __alignof_is_defined not defined
+#endif
+
+#ifdef alignas
+#  error alignas should not be defined
+#endif
+
+#ifdef alignof
+#  error alignof should not be defined
+#endif
diff --git a/libcxx/test/std/depr/depr.cpp.headers/ccomplex.verify.cpp b/libcxx/test/std/depr/depr.cpp.headers/ccomplex.verify.cpp
new file mode 100644
index 000000000000..0eaf82ce5cef
--- /dev/null
+++ b/libcxx/test/std/depr/depr.cpp.headers/ccomplex.verify.cpp
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <ccomplex>
+
+// check that <ccomplex> is deprecated in C++17 and removed in C++20
+// When built with modules, <ccomplex> should be omitted.
+
+// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: clang-modules-build
+
+#include "test_macros.h"
+
+#include <ccomplex>
+
+#if TEST_STD_VER >= 20
+// expected-warning@ccomplex:* {{'__standard_header_ccomplex' is deprecated: removed in C++20. Include <complex> instead.}}
+#else
+// expected-warning@ccomplex:* {{'__standard_header_ccomplex' is deprecated: Include <complex> instead.}}
+#endif
diff --git a/libcxx/test/std/depr/depr.cpp.headers/ciso646.verify.cpp b/libcxx/test/std/depr/depr.cpp.headers/ciso646.verify.cpp
new file mode 100644
index 000000000000..04acd1008154
--- /dev/null
+++ b/libcxx/test/std/depr/depr.cpp.headers/ciso646.verify.cpp
@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <ciso646>
+
+// check that <ciso646> is removed in C++20
+// When built with modules, <ciso646> should be omitted.
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: clang-modules-build
+
+#include <ciso646>
+// expected-warning@ciso646:* {{'__standard_header_ciso646' is deprecated: removed in C++20. Include <version> instead.}}
diff --git a/libcxx/test/std/depr/depr.cpp.headers/cstdalign.verify.cpp b/libcxx/test/std/depr/depr.cpp.headers/cstdalign.verify.cpp
new file mode 100644
index 000000000000..dc9f1af55b3f
--- /dev/null
+++ b/libcxx/test/std/depr/depr.cpp.headers/cstdalign.verify.cpp
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <cstdalign>
+
+// check that <cstdalign> is deprecated in C++17 and removed in C++20
+// When built with modules, <cstdalign> should be omitted.
+
+// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: clang-modules-build
+
+#include "test_macros.h"
+
+#include <cstdalign>
+
+#if TEST_STD_VER >= 20
+// expected-warning@cstdalign:* {{'__standard_header_cstdalign' is deprecated: removed in C++20.}}
+#else
+// expected-warning@cstdalign:* {{'__standard_header_cstdalign' is deprecated}}
+#endif
diff --git a/libcxx/test/std/depr/depr.cpp.headers/cstdbool.verify.cpp b/libcxx/test/std/depr/depr.cpp.headers/cstdbool.verify.cpp
new file mode 100644
index 000000000000..eddefe14d35e
--- /dev/null
+++ b/libcxx/test/std/depr/depr.cpp.headers/cstdbool.verify.cpp
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <cstdbool>
+
+// check that <cstdbool> is deprecated in C++17 and removed in C++20
+// When built with modules, <cstdbool> should be omitted.
+
+// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: clang-modules-build
+
+#include "test_macros.h"
+
+#include <cstdbool>
+
+#if TEST_STD_VER >= 20
+// expected-warning@cstdbool:* {{'__standard_header_cstdbool' is deprecated: removed in C++20.}}
+#else
+// expected-warning@cstdbool:* {{'__standard_header_cstdbool' is deprecated}}
+#endif
diff --git a/libcxx/test/std/depr/depr.cpp.headers/ctgmath.verify.cpp b/libcxx/test/std/depr/depr.cpp.headers/ctgmath.verify.cpp
new file mode 100644
index 000000000000..097ab1643d15
--- /dev/null
+++ b/libcxx/test/std/depr/depr.cpp.headers/ctgmath.verify.cpp
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <ctgmath>
+
+// check that <ctgmath> is deprecated in C++17 and removed in C++20
+// When built with modules, <ctgmath> should be omitted.
+
+// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: clang-modules-build
+
+#include "test_macros.h"
+
+#include <ctgmath>
+
+#if TEST_STD_VER >= 20
+// expected-warning@ctgmath:* {{'__standard_header_ctgmath' is deprecated: removed in C++20. Include <cmath> and <complex> instead.}}
+#else
+// expected-warning@ctgmath:* {{'__standard_header_ctgmath' is deprecated: Include <cmath> and <complex> instead.}}
+#endif
diff --git a/libcxx/test/std/language.support/support.runtime/cstdalign.compile.pass.cpp b/libcxx/test/std/language.support/support.runtime/cstdalign.compile.pass.cpp
new file mode 100644
index 000000000000..fbbaf9b2d136
--- /dev/null
+++ b/libcxx/test/std/language.support/support.runtime/cstdalign.compile.pass.cpp
@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// test <cstdalign> // deprecated in C++17, removed in C++20, but still provided by libc++ as an extension
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
+#include <cstdalign>
+
+#ifndef __alignas_is_defined
+#  error __alignas_is_defined not defined
+#endif
+
+#ifndef __alignof_is_defined
+#  error __alignof_is_defined not defined
+#endif
+
+#ifdef alignas
+#  error alignas should not be defined
+#endif
+
+#ifdef alignof
+#  error alignof should not be defined
+#endif
diff --git a/libcxx/test/std/language.support/support.runtime/cstdbool.pass.cpp b/libcxx/test/std/language.support/support.runtime/cstdbool.pass.cpp
index 1d0e9b06a43d..9a35eea507c4 100644
--- a/libcxx/test/std/language.support/support.runtime/cstdbool.pass.cpp
+++ b/libcxx/test/std/language.support/support.runtime/cstdbool.pass.cpp
@@ -6,7 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-// test <cstdbool>
+// test <cstdbool> // deprecated in C++17, removed in C++20, but still provided by libc++ as an extension
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 
 #include <cstdbool>
 
diff --git a/libcxx/test/std/numerics/c.math/ctgmath.pass.cpp b/libcxx/test/std/numerics/c.math/ctgmath.pass.cpp
index 2c8d054fbc52..2e4679980577 100644
--- a/libcxx/test/std/numerics/c.math/ctgmath.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/ctgmath.pass.cpp
@@ -6,7 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-// <ctgmath>
+// <ctgmath> // deprecated in C++17, removed in C++20, but still provided by libc++ as an extension
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 
 #include <ctgmath>
 
diff --git a/libcxx/test/std/numerics/complex.number/ccmplx/ccomplex.pass.cpp b/libcxx/test/std/numerics/complex.number/ccmplx/ccomplex.pass.cpp
index cc3f8cd6a9be..0ed116c64106 100644
--- a/libcxx/test/std/numerics/complex.number/ccmplx/ccomplex.pass.cpp
+++ b/libcxx/test/std/numerics/complex.number/ccmplx/ccomplex.pass.cpp
@@ -6,7 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-// <ccomplex>
+// <ccomplex> // deprecated in C++17, removed in C++20, but still provided by libc++ as an extension
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 
 #include <ccomplex>
 
diff --git a/libcxx/utils/libcxx/header_information.py b/libcxx/utils/libcxx/header_information.py
index 3b12dcb9f56c..cac620e4f1fe 100644
--- a/libcxx/utils/libcxx/header_information.py
+++ b/libcxx/utils/libcxx/header_information.py
@@ -66,6 +66,7 @@ class Header:
             "cmath",
             "csetjmp",
             "csignal",
+            "cstdalign",
             "cstdarg",
             "cstdbool",
             "cstddef",
@@ -92,7 +93,7 @@ class Header:
         experimental headers.
         """
         # These headers have been removed in C++20 so are never part of a module.
-        removed_in_20 = ["ccomplex", "ciso646", "cstdbool", "ctgmath"]
+        removed_in_20 = ["ccomplex", "ciso646", "cstdalign", "cstdbool", "ctgmath"]
         return self.is_public() and not self.is_experimental() and not self.is_C_compatibility() and not self._name in removed_in_20
 
     def is_cxx03_frozen_header(self) -> bool:
@@ -236,6 +237,15 @@ lit_header_restrictions = {
     "wctype.h": "// UNSUPPORTED: no-wide-characters",
 }
 
+# Undeprecate headers that are deprecated in C++17 and removed in C++20.
+lit_header_undeprecations = {
+    "ccomplex": "// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS",
+    "ciso646": "// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS",
+    "cstdalign": "// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS",
+    "cstdbool": "// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS",
+    "ctgmath": "// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS",
+}
+
 # This table was produced manually, by grepping the TeX source of the Standard's
 # library clauses for the string "#include". Each header's synopsis contains
 # explicit "#include" directives for its mandatory inclusions.
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 8121e34dcf6e..776f1d32c5f5 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -995,6 +995,7 @@ if (current_toolchain == default_toolchain) {
       "coroutine",
       "csetjmp",
       "csignal",
+      "cstdalign",
       "cstdarg",
       "cstdbool",
       "cstddef",
-- 
GitLab


From facdae62b7be4fe177c8a130c68aef0305dc6eb3 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Tue, 29 Oct 2024 19:14:54 -0700
Subject: [PATCH 523/693] [MCInstPrinter] Make printRegName non-const

Similar to printInst. printRegName may change states (e.g. #113834).
---
 llvm/include/llvm/MC/MCInstPrinter.h                         | 4 ++--
 llvm/include/llvm/MC/MCParser/MCAsmParser.h                  | 2 +-
 llvm/lib/MC/MCInstPrinter.cpp                                | 5 ++---
 llvm/lib/MC/MCParser/AsmParser.cpp                           | 4 ++--
 llvm/lib/MC/MCParser/MasmParser.cpp                          | 4 ++--
 llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp  | 4 ++--
 llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h    | 4 ++--
 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp    | 2 +-
 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h      | 2 +-
 llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.cpp          | 2 +-
 llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.h            | 2 +-
 llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp          | 4 ++--
 llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.h            | 2 +-
 llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.cpp        | 2 +-
 llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.h          | 2 +-
 llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp  | 2 +-
 llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h    | 2 +-
 llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.cpp      | 2 +-
 llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.h        | 2 +-
 .../Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.cpp   | 2 +-
 .../lib/Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.h | 2 +-
 llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.cpp        | 2 +-
 llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.h          | 2 +-
 llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.cpp    | 2 +-
 llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.h      | 2 +-
 llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp        | 2 +-
 llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.h          | 2 +-
 llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp      | 2 +-
 llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h        | 2 +-
 llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp      | 2 +-
 llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.h        | 2 +-
 llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp      | 2 +-
 llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h        | 2 +-
 llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp      | 2 +-
 llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h        | 2 +-
 .../Target/SystemZ/MCTargetDesc/SystemZGNUInstPrinter.cpp    | 2 +-
 llvm/lib/Target/SystemZ/MCTargetDesc/SystemZGNUInstPrinter.h | 2 +-
 .../Target/SystemZ/MCTargetDesc/SystemZHLASMInstPrinter.cpp  | 2 +-
 .../Target/SystemZ/MCTargetDesc/SystemZHLASMInstPrinter.h    | 2 +-
 .../Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp | 3 +--
 .../Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.h   | 4 ++--
 llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.cpp            | 2 +-
 llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.h              | 2 +-
 .../WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp      | 3 +--
 .../Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h | 2 +-
 llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp       | 2 +-
 llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h         | 2 +-
 llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp     | 2 +-
 llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h       | 2 +-
 llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.cpp      | 2 +-
 llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.h        | 2 +-
 llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp    | 2 +-
 llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.h      | 2 +-
 53 files changed, 61 insertions(+), 64 deletions(-)

diff --git a/llvm/include/llvm/MC/MCInstPrinter.h b/llvm/include/llvm/MC/MCInstPrinter.h
index 60a901e3d0de..0b9c738a7a0a 100644
--- a/llvm/include/llvm/MC/MCInstPrinter.h
+++ b/llvm/include/llvm/MC/MCInstPrinter.h
@@ -144,7 +144,7 @@ public:
   StringRef getOpcodeName(unsigned Opcode) const;
 
   /// Print the assembler register name.
-  virtual void printRegName(raw_ostream &OS, MCRegister Reg) const;
+  virtual void printRegName(raw_ostream &OS, MCRegister Reg);
 
   bool getUseMarkup() const { return UseMarkup; }
   void setUseMarkup(bool Value) { UseMarkup = Value; }
@@ -152,7 +152,7 @@ public:
   bool getUseColor() const { return UseColor; }
   void setUseColor(bool Value) { UseColor = Value; }
 
-  WithMarkup markup(raw_ostream &OS, Markup M) const;
+  WithMarkup markup(raw_ostream &OS, Markup M);
 
   bool getPrintImmHex() const { return PrintImmHex; }
   void setPrintImmHex(bool Value) { PrintImmHex = Value; }
diff --git a/llvm/include/llvm/MC/MCParser/MCAsmParser.h b/llvm/include/llvm/MC/MCParser/MCAsmParser.h
index faa72d5f3144..70fba6977853 100644
--- a/llvm/include/llvm/MC/MCParser/MCAsmParser.h
+++ b/llvm/include/llvm/MC/MCParser/MCAsmParser.h
@@ -206,7 +206,7 @@ public:
       SmallVectorImpl<std::pair<void *, bool>> &OpDecls,
       SmallVectorImpl<std::string> &Constraints,
       SmallVectorImpl<std::string> &Clobbers, const MCInstrInfo *MII,
-      const MCInstPrinter *IP, MCAsmParserSemaCallback &SI) = 0;
+      MCInstPrinter *IP, MCAsmParserSemaCallback &SI) = 0;
 
   /// Emit a note at the location \p L, with the message \p Msg.
   virtual void Note(SMLoc L, const Twine &Msg,
diff --git a/llvm/lib/MC/MCInstPrinter.cpp b/llvm/lib/MC/MCInstPrinter.cpp
index e4faeba04a8f..488e34a6d539 100644
--- a/llvm/lib/MC/MCInstPrinter.cpp
+++ b/llvm/lib/MC/MCInstPrinter.cpp
@@ -43,7 +43,7 @@ StringRef MCInstPrinter::getOpcodeName(unsigned Opcode) const {
   return MII.getName(Opcode);
 }
 
-void MCInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) const {
+void MCInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   llvm_unreachable("Target should implement this");
 }
 
@@ -224,8 +224,7 @@ format_object<uint64_t> MCInstPrinter::formatHex(uint64_t Value) const {
   llvm_unreachable("unsupported print style");
 }
 
-MCInstPrinter::WithMarkup MCInstPrinter::markup(raw_ostream &OS,
-                                                Markup S) const {
+MCInstPrinter::WithMarkup MCInstPrinter::markup(raw_ostream &OS, Markup S) {
   return WithMarkup(OS, S, getUseMarkup(), getUseColor());
 }
 
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 4774e5112af5..ecccb228c8c3 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -264,7 +264,7 @@ public:
                         SmallVectorImpl<std::pair<void *, bool>> &OpDecls,
                         SmallVectorImpl<std::string> &Constraints,
                         SmallVectorImpl<std::string> &Clobbers,
-                        const MCInstrInfo *MII, const MCInstPrinter *IP,
+                        const MCInstrInfo *MII, MCInstPrinter *IP,
                         MCAsmParserSemaCallback &SI) override;
 
   bool parseExpression(const MCExpr *&Res);
@@ -6006,7 +6006,7 @@ bool AsmParser::parseMSInlineAsm(
     SmallVectorImpl<std::pair<void *, bool>> &OpDecls,
     SmallVectorImpl<std::string> &Constraints,
     SmallVectorImpl<std::string> &Clobbers, const MCInstrInfo *MII,
-    const MCInstPrinter *IP, MCAsmParserSemaCallback &SI) {
+    MCInstPrinter *IP, MCAsmParserSemaCallback &SI) {
   SmallVector<void *, 4> InputDecls;
   SmallVector<void *, 4> OutputDecls;
   SmallVector<bool, 4> InputDeclsAddressOf;
diff --git a/llvm/lib/MC/MCParser/MasmParser.cpp b/llvm/lib/MC/MCParser/MasmParser.cpp
index d88fd09a1aa0..a7f37d81f640 100644
--- a/llvm/lib/MC/MCParser/MasmParser.cpp
+++ b/llvm/lib/MC/MCParser/MasmParser.cpp
@@ -539,7 +539,7 @@ public:
                         SmallVectorImpl<std::pair<void *, bool>> &OpDecls,
                         SmallVectorImpl<std::string> &Constraints,
                         SmallVectorImpl<std::string> &Clobbers,
-                        const MCInstrInfo *MII, const MCInstPrinter *IP,
+                        const MCInstrInfo *MII, MCInstPrinter *IP,
                         MCAsmParserSemaCallback &SI) override;
 
   bool parseExpression(const MCExpr *&Res);
@@ -7340,7 +7340,7 @@ bool MasmParser::parseMSInlineAsm(
     SmallVectorImpl<std::pair<void *, bool>> &OpDecls,
     SmallVectorImpl<std::string> &Constraints,
     SmallVectorImpl<std::string> &Clobbers, const MCInstrInfo *MII,
-    const MCInstPrinter *IP, MCAsmParserSemaCallback &SI) {
+    MCInstPrinter *IP, MCAsmParserSemaCallback &SI) {
   SmallVector<void *, 4> InputDecls;
   SmallVector<void *, 4> OutputDecls;
   SmallVector<bool, 4> InputDeclsAddressOf;
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
index 762a7af8c3dd..2ee2ee5a6fa5 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
@@ -59,12 +59,12 @@ bool AArch64InstPrinter::applyTargetSpecificCLOption(StringRef Opt) {
   return false;
 }
 
-void AArch64InstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) const {
+void AArch64InstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   markup(OS, Markup::Register) << getRegisterName(Reg);
 }
 
 void AArch64InstPrinter::printRegName(raw_ostream &OS, MCRegister Reg,
-                                      unsigned AltIdx) const {
+                                      unsigned AltIdx) {
   markup(OS, Markup::Register) << getRegisterName(Reg, AltIdx);
 }
 
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h
index e7b62b320368..9cf2674ae943 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h
@@ -29,8 +29,8 @@ public:
 
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &O) override;
-  void printRegName(raw_ostream &OS, MCRegister Reg) const override;
-  void printRegName(raw_ostream &OS, MCRegister Reg, unsigned AltIdx) const;
+  void printRegName(raw_ostream &OS, MCRegister Reg) override;
+  void printRegName(raw_ostream &OS, MCRegister Reg, unsigned AltIdx);
 
   // Autogenerated by tblgen.
   std::pair<const char *, uint64_t> getMnemonic(const MCInst *MI) override;
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index dd8d93c3f0b7..88caf8196b3c 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -24,7 +24,7 @@
 using namespace llvm;
 using namespace llvm::AMDGPU;
 
-void AMDGPUInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) const {
+void AMDGPUInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   // FIXME: The current implementation of
   // AsmParser::parseRegisterOrRegisterNumber in MC implies we either emit this
   // as an integer or we provide a name which represents a physical register.
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
index a72e0fe6ea76..4729b8a6aa6f 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
@@ -29,7 +29,7 @@ public:
                         const MCSubtargetInfo &STI, raw_ostream &O);
   static const char *getRegisterName(MCRegister Reg);
 
-  void printRegName(raw_ostream &OS, MCRegister Reg) const override;
+  void printRegName(raw_ostream &OS, MCRegister Reg) override;
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &O) override;
   static void printRegOperand(MCRegister Reg, raw_ostream &O,
diff --git a/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.cpp b/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.cpp
index d76c2810c39f..e669b9479369 100644
--- a/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.cpp
+++ b/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.cpp
@@ -93,7 +93,7 @@ static const char *ARCCondCodeToString(ARCCC::CondCode CC) {
   return BadConditionCode(CC);
 }
 
-void ARCInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) const {
+void ARCInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   OS << StringRef(getRegisterName(Reg)).lower();
 }
 
diff --git a/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.h b/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.h
index baf4a6915b70..c4bd73448ca7 100644
--- a/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.h
+++ b/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.h
@@ -30,7 +30,7 @@ public:
   void printInstruction(const MCInst *MI, uint64_t Address, raw_ostream &O);
   static const char *getRegisterName(MCRegister Reg);
 
-  void printRegName(raw_ostream &OS, MCRegister Reg) const override;
+  void printRegName(raw_ostream &OS, MCRegister Reg) override;
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &O) override;
   void printCCOperand(const MCInst *MI, int OpNum, raw_ostream &O);
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp
index e4a2f8c8f2ea..5a6895a4ab84 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp
@@ -50,7 +50,7 @@ static unsigned translateShiftImm(unsigned imm) {
 }
 
 static void printRegImmShift(raw_ostream &O, ARM_AM::ShiftOpc ShOpc,
-                             unsigned ShImm, const ARMInstPrinter &printer) {
+                             unsigned ShImm, ARMInstPrinter &printer) {
   if (ShOpc == ARM_AM::no_shift || (ShOpc == ARM_AM::lsl && !ShImm))
     return;
   O << ", ";
@@ -81,7 +81,7 @@ bool ARMInstPrinter::applyTargetSpecificCLOption(StringRef Opt) {
   return false;
 }
 
-void ARMInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) const {
+void ARMInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   markup(OS, Markup::Register) << getRegisterName(Reg, DefaultAltIdx);
 }
 
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.h
index 494a644cf545..cd1dddc5f331 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.h
@@ -27,7 +27,7 @@ public:
 
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &O) override;
-  void printRegName(raw_ostream &OS, MCRegister Reg) const override;
+  void printRegName(raw_ostream &OS, MCRegister Reg) override;
 
   // Autogenerated by tblgen.
   std::pair<const char *, uint64_t> getMnemonic(const MCInst *MI) override;
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.cpp
index 9af7958112fc..a4b0d8488cf5 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.cpp
@@ -82,7 +82,7 @@ void CSKYInstPrinter::printInst(const MCInst *MI, uint64_t Address,
   printAnnotation(O, Annot);
 }
 
-void CSKYInstPrinter::printRegName(raw_ostream &O, MCRegister Reg) const {
+void CSKYInstPrinter::printRegName(raw_ostream &O, MCRegister Reg) {
   if (PrintBranchImmAsAddress)
     O << getRegisterName(Reg, ABIRegNames ? CSKY::ABIRegAltName
                                           : CSKY::NoRegAltName);
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.h b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.h
index 461d7f6f12b3..16eccfdfb5ce 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.h
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.h
@@ -31,7 +31,7 @@ public:
 
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &O) override;
-  void printRegName(raw_ostream &O, MCRegister Reg) const override;
+  void printRegName(raw_ostream &O, MCRegister Reg) override;
 
   void printOperand(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                     raw_ostream &O, const char *Modifier = nullptr);
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp
index de5c8b86978a..e4e84a80b5d0 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp
@@ -26,7 +26,7 @@ using namespace llvm;
 #define GET_INSTRUCTION_NAME
 #include "HexagonGenAsmWriter.inc"
 
-void HexagonInstPrinter::printRegName(raw_ostream &O, MCRegister Reg) const {
+void HexagonInstPrinter::printRegName(raw_ostream &O, MCRegister Reg) {
   O << getRegisterName(Reg);
 }
 
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h
index 38a9081c93fe..fe37cd91dabc 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h
@@ -30,7 +30,7 @@ public:
 
   void printInst(MCInst const *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &O) override;
-  void printRegName(raw_ostream &O, MCRegister Reg) const override;
+  void printRegName(raw_ostream &O, MCRegister Reg) override;
 
   static char const *getRegisterName(MCRegister Reg);
 
diff --git a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.cpp b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.cpp
index 0265a75fb346..4b5751eaedda 100644
--- a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.cpp
+++ b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.cpp
@@ -31,7 +31,7 @@ using namespace llvm;
 #define PRINT_ALIAS_INSTR
 #include "LanaiGenAsmWriter.inc"
 
-void LanaiInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) const {
+void LanaiInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   OS << StringRef(getRegisterName(Reg)).lower();
 }
 
diff --git a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.h b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.h
index 55a254036fee..851613b27e3d 100644
--- a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.h
+++ b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.h
@@ -49,7 +49,7 @@ public:
                                unsigned OpIdx, unsigned PrintMethodIdx,
                                raw_ostream &O);
   static const char *getRegisterName(MCRegister Reg);
-  void printRegName(raw_ostream &OS, MCRegister Reg) const override;
+  void printRegName(raw_ostream &OS, MCRegister Reg) override;
 
 private:
   bool printAlias(const MCInst *MI, raw_ostream &Ostream);
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.cpp
index cb2521db5217..e3007cfe3d40 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.cpp
@@ -56,7 +56,7 @@ void LoongArchInstPrinter::printInst(const MCInst *MI, uint64_t Address,
   printAnnotation(O, Annot);
 }
 
-void LoongArchInstPrinter::printRegName(raw_ostream &O, MCRegister Reg) const {
+void LoongArchInstPrinter::printRegName(raw_ostream &O, MCRegister Reg) {
   O << '$' << getRegisterName(Reg);
 }
 
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.h
index 4e6092bfcb12..8cda3fdb4510 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.h
@@ -28,7 +28,7 @@ public:
 
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &O) override;
-  void printRegName(raw_ostream &O, MCRegister Reg) const override;
+  void printRegName(raw_ostream &O, MCRegister Reg) override;
   void printAtomicMemOp(const MCInst *MI, unsigned OpNo,
                         const MCSubtargetInfo &STI, raw_ostream &O);
 
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.cpp b/llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.cpp
index 84800fc762cb..68ac15b57508 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.cpp
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.cpp
@@ -41,7 +41,7 @@ using namespace llvm;
 #define PRINT_ALIAS_INSTR
 #include "M68kGenAsmWriter.inc"
 
-void M68kInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) const {
+void M68kInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   OS << "%" << getRegisterName(Reg);
 }
 
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.h b/llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.h
index 096317630458..d6d17ca9568e 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.h
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.h
@@ -34,7 +34,7 @@ public:
   void printInstruction(const MCInst *MI, uint64_t Address, raw_ostream &O);
   static const char *getRegisterName(MCRegister Reg);
 
-  void printRegName(raw_ostream &OS, MCRegister Reg) const override;
+  void printRegName(raw_ostream &OS, MCRegister Reg) override;
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &O) override;
 
diff --git a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.cpp b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.cpp
index 3726c600f4a7..d8a27f34c6fd 100644
--- a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.cpp
+++ b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.cpp
@@ -26,7 +26,7 @@ using namespace llvm;
 #define PRINT_ALIAS_INSTR
 #include "MSP430GenAsmWriter.inc"
 
-void MSP430InstPrinter::printRegName(raw_ostream &O, MCRegister Reg) const {
+void MSP430InstPrinter::printRegName(raw_ostream &O, MCRegister Reg) {
   O << getRegisterName(Reg);
 }
 
diff --git a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.h b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.h
index 40605b92bcb0..413492b8efee 100644
--- a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.h
+++ b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.h
@@ -22,7 +22,7 @@ namespace llvm {
                       const MCRegisterInfo &MRI)
       : MCInstPrinter(MAI, MII, MRI) {}
 
-    void printRegName(raw_ostream &O, MCRegister Reg) const override;
+    void printRegName(raw_ostream &O, MCRegister Reg) override;
 
     void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                    const MCSubtargetInfo &STI, raw_ostream &O) override;
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp
index 1518a539782e..2fd1b344eb68 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp
@@ -72,7 +72,7 @@ const char* Mips::MipsFCCToString(Mips::CondCode CC) {
   llvm_unreachable("Impossible condition code!");
 }
 
-void MipsInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) const {
+void MipsInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   markup(OS, Markup::Register)
       << '$' << StringRef(getRegisterName(Reg)).lower();
 }
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.h b/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.h
index 0652b237509f..8e3b4614a4aa 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.h
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.h
@@ -84,7 +84,7 @@ public:
                         const MCSubtargetInfo &STI, raw_ostream &O);
   static const char *getRegisterName(MCRegister Reg);
 
-  void printRegName(raw_ostream &OS, MCRegister Reg) const override;
+  void printRegName(raw_ostream &OS, MCRegister Reg) override;
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &O) override;
 
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
index 9b5892844632..4211ae5a2eeb 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
@@ -34,7 +34,7 @@ NVPTXInstPrinter::NVPTXInstPrinter(const MCAsmInfo &MAI, const MCInstrInfo &MII,
                                    const MCRegisterInfo &MRI)
     : MCInstPrinter(MAI, MII, MRI) {}
 
-void NVPTXInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) const {
+void NVPTXInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   // Decode the virtual register
   // Must be kept in sync with NVPTXAsmPrinter::encodeVirtualRegister
   unsigned RCId = (Reg.id() >> 28);
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
index e8a4a6dbdd53..63207e8a975a 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
@@ -24,7 +24,7 @@ public:
   NVPTXInstPrinter(const MCAsmInfo &MAI, const MCInstrInfo &MII,
                    const MCRegisterInfo &MRI);
 
-  void printRegName(raw_ostream &OS, MCRegister Reg) const override;
+  void printRegName(raw_ostream &OS, MCRegister Reg) override;
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &OS) override;
 
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
index 9a4291c90408..7511e24f705c 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
@@ -47,7 +47,7 @@ FullRegNamesWithPercent("ppc-reg-with-percent-prefix", cl::Hidden,
 #define PRINT_ALIAS_INSTR
 #include "PPCGenAsmWriter.inc"
 
-void PPCInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) const {
+void PPCInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   const char *RegName = getRegisterName(Reg);
   OS << RegName;
 }
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.h b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.h
index 6ba3eb4c79dc..1b9365fa0496 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.h
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.h
@@ -31,7 +31,7 @@ public:
                  const MCRegisterInfo &MRI, Triple T)
     : MCInstPrinter(MAI, MII, MRI), TT(T) {}
 
-  void printRegName(raw_ostream &OS, MCRegister Reg) const override;
+  void printRegName(raw_ostream &OS, MCRegister Reg) override;
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &O) override;
 
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
index 1f27c934baf0..1445e9da4a62 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
@@ -75,7 +75,7 @@ void RISCVInstPrinter::printInst(const MCInst *MI, uint64_t Address,
   printAnnotation(O, Annot);
 }
 
-void RISCVInstPrinter::printRegName(raw_ostream &O, MCRegister Reg) const {
+void RISCVInstPrinter::printRegName(raw_ostream &O, MCRegister Reg) {
   markup(O, Markup::Register) << getRegisterName(Reg);
 }
 
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
index 77cc7a67e889..c15fd591b9e9 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
@@ -28,7 +28,7 @@ public:
 
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &O) override;
-  void printRegName(raw_ostream &O, MCRegister Reg) const override;
+  void printRegName(raw_ostream &O, MCRegister Reg) override;
 
   void printOperand(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                     raw_ostream &O, const char *Modifier = nullptr);
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp
index 5b407a8b6f54..4bba54463103 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp
@@ -38,7 +38,7 @@ bool SparcInstPrinter::isV9(const MCSubtargetInfo &STI) const {
   return (STI.hasFeature(Sparc::FeatureV9)) != 0;
 }
 
-void SparcInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) const {
+void SparcInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   OS << '%' << getRegisterName(Reg);
 }
 
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h b/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h
index 207a97022805..52321d562118 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h
@@ -24,7 +24,7 @@ public:
                    const MCRegisterInfo &MRI)
       : MCInstPrinter(MAI, MII, MRI) {}
 
-  void printRegName(raw_ostream &OS, MCRegister Reg) const override;
+  void printRegName(raw_ostream &OS, MCRegister Reg) override;
   void printRegName(raw_ostream &OS, MCRegister Reg, unsigned AltIdx) const;
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &O) override;
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZGNUInstPrinter.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZGNUInstPrinter.cpp
index 05113010794e..72b7bd60276a 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZGNUInstPrinter.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZGNUInstPrinter.cpp
@@ -19,7 +19,7 @@ using namespace llvm;
 
 void SystemZGNUInstPrinter::printFormattedRegName(const MCAsmInfo *MAI,
                                                   MCRegister Reg,
-                                                  raw_ostream &O) const {
+                                                  raw_ostream &O) {
   const char *RegName = getRegisterName(Reg);
   markup(O, Markup::Register) << '%' << RegName;
 }
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZGNUInstPrinter.h b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZGNUInstPrinter.h
index 8f62ae0e16c0..7095e325c70b 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZGNUInstPrinter.h
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZGNUInstPrinter.h
@@ -38,7 +38,7 @@ public:
 
 private:
   void printFormattedRegName(const MCAsmInfo *MAI, MCRegister Reg,
-                             raw_ostream &O) const override;
+                             raw_ostream &O) override;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMInstPrinter.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMInstPrinter.cpp
index 9abd408324c0..ef9881932f7c 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMInstPrinter.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMInstPrinter.cpp
@@ -19,7 +19,7 @@ using namespace llvm;
 
 void SystemZHLASMInstPrinter::printFormattedRegName(const MCAsmInfo *MAI,
                                                     MCRegister Reg,
-                                                    raw_ostream &O) const {
+                                                    raw_ostream &O) {
   const char *RegName = getRegisterName(Reg);
   // Skip register prefix so that only register number is left
   assert(isalpha(RegName[0]) && isdigit(RegName[1]));
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMInstPrinter.h b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMInstPrinter.h
index 9a69e012c729..ffccbec36c74 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMInstPrinter.h
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMInstPrinter.h
@@ -37,7 +37,7 @@ public:
 
 private:
   void printFormattedRegName(const MCAsmInfo *MAI, MCRegister Reg,
-                             raw_ostream &O) const override;
+                             raw_ostream &O) override;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp
index 00560ab1f4b1..fe0f38747656 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp
@@ -57,8 +57,7 @@ void SystemZInstPrinterCommon::printOperand(const MCOperand &MO,
     llvm_unreachable("Invalid operand");
 }
 
-void SystemZInstPrinterCommon::printRegName(raw_ostream &O,
-                                            MCRegister Reg) const {
+void SystemZInstPrinterCommon::printRegName(raw_ostream &O, MCRegister Reg) {
   printFormattedRegName(&MAI, Reg, O);
 }
 
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.h b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.h
index 9a972824f7ff..1a11e421691a 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.h
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.h
@@ -36,10 +36,10 @@ public:
   void printOperand(const MCOperand &MO, const MCAsmInfo *MAI, raw_ostream &O);
 
   virtual void printFormattedRegName(const MCAsmInfo *MAI, MCRegister Reg,
-                                     raw_ostream &O) const {}
+                                     raw_ostream &O) {}
 
   // Override MCInstPrinter.
-  void printRegName(raw_ostream &O, MCRegister Reg) const override;
+  void printRegName(raw_ostream &O, MCRegister Reg) override;
 
 protected:
   template <unsigned N>
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.cpp
index 8261b5aa7b4e..47455a9a0274 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.cpp
@@ -27,7 +27,7 @@ using namespace llvm;
 #define PRINT_ALIAS_INSTR
 #include "VEGenAsmWriter.inc"
 
-void VEInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) const {
+void VEInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   // Generic registers have identical register name among register classes.
   unsigned AltIdx = VE::AsmName;
   // Misc registers have each own name, so no use alt-names.
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.h b/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.h
index 65660a49c5e4..d5e0ebd3596c 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.h
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.h
@@ -24,7 +24,7 @@ public:
                 const MCRegisterInfo &MRI)
       : MCInstPrinter(MAI, MII, MRI) {}
 
-  void printRegName(raw_ostream &OS, MCRegister Reg) const override;
+  void printRegName(raw_ostream &OS, MCRegister Reg) override;
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &OS) override;
 
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
index 4c29b59b3302..026f859b15d7 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
@@ -38,8 +38,7 @@ WebAssemblyInstPrinter::WebAssemblyInstPrinter(const MCAsmInfo &MAI,
                                                const MCRegisterInfo &MRI)
     : MCInstPrinter(MAI, MII, MRI) {}
 
-void WebAssemblyInstPrinter::printRegName(raw_ostream &OS,
-                                          MCRegister Reg) const {
+void WebAssemblyInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   assert(Reg.id() != WebAssembly::UnusedReg);
   // Note that there's an implicit local.get/local.set here!
   OS << "$" << Reg.id();
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h
index b499926ab829..e7c5e14973b6 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h
@@ -35,7 +35,7 @@ public:
   WebAssemblyInstPrinter(const MCAsmInfo &MAI, const MCInstrInfo &MII,
                          const MCRegisterInfo &MRI);
 
-  void printRegName(raw_ostream &OS, MCRegister Reg) const override;
+  void printRegName(raw_ostream &OS, MCRegister Reg) override;
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &OS) override;
 
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
index 58b4527af655..c811d621e60e 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
@@ -35,7 +35,7 @@ using namespace llvm;
 #define PRINT_ALIAS_INSTR
 #include "X86GenAsmWriter.inc"
 
-void X86ATTInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) const {
+void X86ATTInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   markup(OS, Markup::Register) << '%' << getRegisterName(Reg);
 }
 
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
index 83040c112b68..7e525e232362 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
@@ -23,7 +23,7 @@ public:
                     const MCRegisterInfo &MRI)
       : X86InstPrinterCommon(MAI, MII, MRI), HasCustomInstComment(false) {}
 
-  void printRegName(raw_ostream &OS, MCRegister Reg) const override;
+  void printRegName(raw_ostream &OS, MCRegister Reg) override;
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &OS) override;
   bool printVecCompareInstr(const MCInst *MI, raw_ostream &OS);
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
index cd8b9aa62573..8e7dae229275 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
@@ -33,7 +33,7 @@ using namespace llvm;
 #define PRINT_ALIAS_INSTR
 #include "X86GenAsmWriter1.inc"
 
-void X86IntelInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) const {
+void X86IntelInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   markup(OS, Markup::Register) << getRegisterName(Reg);
 }
 
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h
index a34c06782f40..988ab9626c3f 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h
@@ -24,7 +24,7 @@ public:
                       const MCRegisterInfo &MRI)
     : X86InstPrinterCommon(MAI, MII, MRI) {}
 
-  void printRegName(raw_ostream &OS, MCRegister Reg) const override;
+  void printRegName(raw_ostream &OS, MCRegister Reg) override;
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &OS) override;
   bool printVecCompareInstr(const MCInst *MI, raw_ostream &OS);
diff --git a/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.cpp b/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.cpp
index eda90d3101ab..707c4a790872 100644
--- a/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.cpp
+++ b/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.cpp
@@ -27,7 +27,7 @@ using namespace llvm;
 
 #include "XCoreGenAsmWriter.inc"
 
-void XCoreInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) const {
+void XCoreInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
   OS << StringRef(getRegisterName(Reg)).lower();
 }
 
diff --git a/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.h b/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.h
index 916ca99968fb..2b47de457322 100644
--- a/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.h
+++ b/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.h
@@ -31,7 +31,7 @@ public:
   void printInstruction(const MCInst *MI, uint64_t Address, raw_ostream &O);
   static const char *getRegisterName(MCRegister Reg);
 
-  void printRegName(raw_ostream &OS, MCRegister Reg) const override;
+  void printRegName(raw_ostream &OS, MCRegister Reg) override;
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &O) override;
 
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp
index fe1dc0e2e483..e04d7bd21121 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp
@@ -74,7 +74,7 @@ void XtensaInstPrinter::printInst(const MCInst *MI, uint64_t Address,
   printAnnotation(O, Annot);
 }
 
-void XtensaInstPrinter::printRegName(raw_ostream &O, MCRegister Reg) const {
+void XtensaInstPrinter::printRegName(raw_ostream &O, MCRegister Reg) {
   O << getRegisterName(Reg);
 }
 
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.h b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.h
index 46a35ae6f4e3..4122b1ff2310 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.h
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.h
@@ -36,7 +36,7 @@ public:
   static void printOperand(const MCOperand &MO, raw_ostream &O);
 
   // Override MCInstPrinter.
-  void printRegName(raw_ostream &O, MCRegister Reg) const override;
+  void printRegName(raw_ostream &O, MCRegister Reg) override;
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &O) override;
 
-- 
GitLab


From 9e8219a78c80442fb0f795f17926595a94a8e7d7 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 29 Oct 2024 19:41:49 -0700
Subject: [PATCH 524/693] IR: Fix verifier missing addrspace mismatch in vector
 GEPs (#114091)

---
 llvm/lib/IR/Verifier.cpp           | 11 +++++------
 llvm/unittests/IR/VerifierTest.cpp | 30 ++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index ee807ca13787..ffcab98db9aa 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4121,8 +4121,9 @@ void Verifier::visitGetElementPtrInst(GetElementPtrInst &GEP) {
       GetElementPtrInst::getIndexedType(GEP.getSourceElementType(), Idxs);
   Check(ElTy, "Invalid indices for GEP pointer type!", &GEP);
 
-  Check(GEP.getType()->isPtrOrPtrVectorTy() &&
-            GEP.getResultElementType() == ElTy,
+  PointerType *PtrTy = dyn_cast<PointerType>(GEP.getType()->getScalarType());
+
+  Check(PtrTy && GEP.getResultElementType() == ElTy,
         "GEP is not of right type for indices!", &GEP, ElTy);
 
   if (auto *GEPVTy = dyn_cast<VectorType>(GEP.getType())) {
@@ -4144,10 +4145,8 @@ void Verifier::visitGetElementPtrInst(GetElementPtrInst &GEP) {
     }
   }
 
-  if (auto *PTy = dyn_cast<PointerType>(GEP.getType())) {
-    Check(GEP.getAddressSpace() == PTy->getAddressSpace(),
-          "GEP address space doesn't match type", &GEP);
-  }
+  Check(GEP.getAddressSpace() == PtrTy->getAddressSpace(),
+        "GEP address space doesn't match type", &GEP);
 
   visitInstruction(GEP);
 }
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index 91cd35a10e9b..462578a34da8 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -385,5 +385,35 @@ TEST(VerifierTest, AtomicRMW) {
       << Error;
 }
 
+TEST(VerifierTest, GetElementPtrInst) {
+  LLVMContext C;
+  Module M("M", C);
+  FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+  BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+  ReturnInst *RI = ReturnInst::Create(C, Entry);
+
+  FixedVectorType *V2P1Ty = FixedVectorType::get(PointerType::get(C, 1), 2);
+  FixedVectorType *V2P2Ty = FixedVectorType::get(PointerType::get(C, 2), 2);
+
+  Instruction *GEPVec = GetElementPtrInst::Create(
+      Type::getInt8Ty(C), ConstantAggregateZero::get(V2P1Ty),
+      {ConstantVector::getSplat(ElementCount::getFixed(2),
+                                ConstantInt::get(Type::getInt64Ty(C), 0))},
+      Entry);
+
+  GEPVec->insertBefore(RI);
+
+  // Break the address space of the source value
+  GEPVec->getOperandUse(0).set(ConstantAggregateZero::get(V2P2Ty));
+
+  std::string Error;
+  raw_string_ostream ErrorOS(Error);
+  EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+  EXPECT_TRUE(
+      StringRef(Error).starts_with("GEP address space doesn't match type"))
+      << Error;
+}
+
 } // end anonymous namespace
 } // end namespace llvm
-- 
GitLab


From c62130f7b35412e7caadf5fd9547f21a736c4543 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 29 Oct 2024 19:42:35 -0700
Subject: [PATCH 525/693] [RISCV] Add OperandType to loadfpimm. (#114150)

This is represented in the MachineInstr and MCInst as a 5-bit unsigned
immediate so we use OPERAND_UIMM5. If someone needs to know for sure its
an FLI constant in the future we can break it out to a new type.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
index 2bdcfd21270e..f13b3e69f84f 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
@@ -35,6 +35,8 @@ def LoadFPImmOperand : AsmOperandClass {
 def loadfpimm : Operand<XLenVT> {
   let ParserMatchClass = LoadFPImmOperand;
   let PrintMethod = "printFPImmOperand";
+  let OperandType = "OPERAND_UIMM5";
+  let OperandNamespace = "RISCVOp";
 }
 
 def RTZArg : AsmOperandClass {
-- 
GitLab


From 2c313259c65317f097d57ab4c6684b25db98f2e4 Mon Sep 17 00:00:00 2001
From: lialan <xunli@amd.com>
Date: Tue, 29 Oct 2024 23:04:48 -0400
Subject: [PATCH 526/693] [MLIR] VectorEmulateNarrowType to support loading of
 unaligned vectors (#113411)

Previously, the pass only supported emulation of loading vector sizes
that are multiples of the emulated data type. This patch expands its
support for emulating sizes that are not multiples of byte sizes. In
such cases, the element values are packed back-to-back to preserve
memory space.

To give a concrete example: if an input has type `memref<3x3xi2>`, it is
actually occupying 3 bytes in memory, with the first 18 bits storing the
values and the last 6 bits as padding. The slice of `vector<3xi2>` at
index `[2, 0]` is stored in memory from bit 12 to bit 18. To properly
load the elements from bit 12 to bit 18 from memory, first load byte 2
and byte 3, and convert it to a vector of `i2` type; then extract bits 4
to 10 (element index 2-5) to form a `vector<3xi2>`.

A limitation of this patch is that the linearized index of the unaligned
vector has to be known at compile time. Extra code needs to be emitted
to handle it if the condition does not hold.

The following ops are updated:
* `vector::LoadOp`
* `vector::TransferReadOp`
* `vector::MaskedLoadOp`
---
 .../mlir/Dialect/MemRef/Utils/MemRefUtils.h   |   8 +-
 mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp |   9 +-
 .../Transforms/VectorEmulateNarrowType.cpp    | 235 ++++++++++++++----
 .../vector-emulate-narrow-type-unaligned.mlir |  67 +++++
 4 files changed, 264 insertions(+), 55 deletions(-)
 create mode 100644 mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir

diff --git a/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h b/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
index ca3326dbbef5..a761a77a407e 100644
--- a/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
+++ b/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
@@ -32,7 +32,8 @@ namespace memref {
 bool isStaticShapeAndContiguousRowMajor(MemRefType type);
 
 /// For a `memref` with `offset`, `sizes` and `strides`, returns the
-/// offset and size to use for the linearized `memref`.
+/// offset, size, and potentially the size padded at the front to use for the
+/// linearized `memref`.
 /// - If the linearization is done for emulating load/stores of
 ///   element type with bitwidth `srcBits` using element type with
 ///   bitwidth `dstBits`, the linearized offset and size are
@@ -42,9 +43,14 @@ bool isStaticShapeAndContiguousRowMajor(MemRefType type);
 ///   index to use in the linearized `memref`. The linearized index
 ///   is also scaled down by `dstBits`/`srcBits`. If `indices` is not provided
 ///   0, is returned for the linearized index.
+/// - If the size of the load/store is smaller than the linearized memref
+/// load/store, the memory region emulated is larger than the actual memory
+/// region needed. `intraDataOffset` returns the element offset of the data
+/// relevant at the beginning.
 struct LinearizedMemRefInfo {
   OpFoldResult linearizedOffset;
   OpFoldResult linearizedSize;
+  OpFoldResult intraDataOffset;
 };
 std::pair<LinearizedMemRefInfo, OpFoldResult> getLinearizedMemRefOffsetAndSize(
     OpBuilder &builder, Location loc, int srcBits, int dstBits,
diff --git a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
index 7321b1906801..6de744a7f752 100644
--- a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
+++ b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
@@ -81,11 +81,10 @@ std::pair<LinearizedMemRefInfo, OpFoldResult> getLinearizedMemRefOffsetAndSize(
 
   // Adjust linearizedIndices and size by the scale factor (dstBits / srcBits).
   int64_t scaler = dstBits / srcBits;
-  addMulMap = addMulMap.floorDiv(scaler);
   mulMap = mulMap.floorDiv(scaler);
 
   OpFoldResult linearizedIndices = affine::makeComposedFoldedAffineApply(
-      builder, loc, addMulMap, offsetValues);
+      builder, loc, addMulMap.floorDiv(scaler), offsetValues);
   OpFoldResult linearizedSize =
       affine::makeComposedFoldedAffineApply(builder, loc, mulMap, sizes);
 
@@ -95,7 +94,11 @@ std::pair<LinearizedMemRefInfo, OpFoldResult> getLinearizedMemRefOffsetAndSize(
   OpFoldResult adjustBaseOffset = affine::makeComposedFoldedAffineApply(
       builder, loc, s0.floorDiv(scaler), {offset});
 
-  return {{adjustBaseOffset, linearizedSize}, linearizedIndices};
+  OpFoldResult intraVectorOffset = affine::makeComposedFoldedAffineApply(
+      builder, loc, addMulMap % scaler, offsetValues);
+
+  return {{adjustBaseOffset, linearizedSize, intraVectorOffset},
+          linearizedIndices};
 }
 
 LinearizedMemRefInfo
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
index 66362d3ca70f..1d6f8a991d9b 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
@@ -13,6 +13,7 @@
 #include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
+#include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
 #include "mlir/IR/BuiltinAttributes.h"
@@ -22,8 +23,10 @@
 #include "mlir/Transforms/DialectConversion.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdint>
+#include <optional>
 
 using namespace mlir;
 
@@ -33,17 +36,22 @@ using namespace mlir;
 #define LDBG(X) LLVM_DEBUG(DBGS() << X << "\n")
 
 /// Returns a compressed mask. The mask value is set only if any mask is present
-/// in the scale range. E.g., if `scale` equals to 2, the following mask:
+/// in the scale range. E.g., if `scale` equals to 2, and `intraDataOffset`
+/// equals to 2, the following mask:
 ///
 ///   %mask = [1, 1, 1, 0, 0, 0]
 ///
-/// will return the following new compressed mask:
+/// will first be padded with number of `intraDataOffset` zeros:
+///   %mask = [0, 0, 1, 1, 1, 0, 0, 0]
 ///
-///   %mask = [1, 1, 0]
+/// then it will return the following new compressed mask:
+///
+///   %mask = [0, 1, 1, 0]
 static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
                                                   Location loc, Value mask,
-                                                  int origElements, int scale) {
-  auto numElements = (origElements + scale - 1) / scale;
+                                                  int origElements, int scale,
+                                                  int intraDataOffset = 0) {
+  auto numElements = (intraDataOffset + origElements + scale - 1) / scale;
 
   Operation *maskOp = mask.getDefiningOp();
   SmallVector<vector::ExtractOp, 2> extractOps;
@@ -67,6 +75,9 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
   shape.back() = numElements;
   auto newMaskType = VectorType::get(shape, rewriter.getI1Type());
   if (createMaskOp) {
+    // TODO: handle the case with non-zero intraDataOffset for CreateMaskOp.
+    if (intraDataOffset != 0)
+      return failure();
     OperandRange maskOperands = createMaskOp.getOperands();
     size_t numMaskOperands = maskOperands.size();
     AffineExpr s0;
@@ -86,11 +97,27 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
     ArrayRef<int64_t> maskDimSizes = constantMaskOp.getMaskDimSizes();
     size_t numMaskOperands = maskDimSizes.size();
     int64_t origIndex = maskDimSizes[numMaskOperands - 1];
-    int64_t maskIndex = (origIndex + scale - 1) / scale;
+    int64_t startIndex = intraDataOffset / scale;
+    int64_t maskIndex = llvm::divideCeil(intraDataOffset + origIndex, scale);
+
+    // TODO: we only want the mask between [startIndex, maskIndex] to be true,
+    // the rest are false.
+    if (intraDataOffset != 0 && maskDimSizes.size() > 1)
+      return failure();
+
     SmallVector<int64_t> newMaskDimSizes(maskDimSizes.drop_back());
     newMaskDimSizes.push_back(maskIndex);
-    newMask = rewriter.create<vector::ConstantMaskOp>(loc, newMaskType,
-                                                      newMaskDimSizes);
+
+    if (intraDataOffset == 0) {
+      newMask = rewriter.create<vector::ConstantMaskOp>(loc, newMaskType,
+                                                        newMaskDimSizes);
+    } else {
+      SmallVector<bool> newMaskValues;
+      for (int64_t i = 0; i < numElements; ++i)
+        newMaskValues.push_back(i >= startIndex && i < maskIndex);
+      auto denseAttr = DenseElementsAttr::get(newMaskType, newMaskValues);
+      newMask = rewriter.create<arith::ConstantOp>(loc, newMaskType, denseAttr);
+    }
   }
 
   while (!extractOps.empty()) {
@@ -102,6 +129,26 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
   return newMask;
 }
 
+static Value extractSubvectorFrom(RewriterBase &rewriter, Location loc,
+                                  VectorType extractType, Value vector,
+                                  int64_t frontOffset, int64_t subvecSize) {
+  auto offsets = rewriter.getI64ArrayAttr({frontOffset});
+  auto sizes = rewriter.getI64ArrayAttr({subvecSize});
+  auto strides = rewriter.getI64ArrayAttr({1});
+  return rewriter
+      .create<vector::ExtractStridedSliceOp>(loc, extractType, vector, offsets,
+                                             sizes, strides)
+      ->getResult(0);
+}
+
+static Value insertSubvectorInto(RewriterBase &rewriter, Location loc,
+                                 Value src, Value dest, int64_t offset) {
+  auto offsets = rewriter.getI64ArrayAttr({offset});
+  auto strides = rewriter.getI64ArrayAttr({1});
+  return rewriter.create<vector::InsertStridedSliceOp>(loc, dest.getType(), src,
+                                                       dest, offsets, strides);
+}
+
 namespace {
 
 //===----------------------------------------------------------------------===//
@@ -201,7 +248,8 @@ struct ConvertVectorMaskedStore final
     auto stridedMetadata =
         rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
     OpFoldResult linearizedIndicesOfr;
-    std::tie(std::ignore, linearizedIndicesOfr) =
+    memref::LinearizedMemRefInfo linearizedInfo;
+    std::tie(linearizedInfo, linearizedIndicesOfr) =
         memref::getLinearizedMemRefOffsetAndSize(
             rewriter, loc, srcBits, dstBits,
             stridedMetadata.getConstifiedMixedOffset(),
@@ -214,19 +262,19 @@ struct ConvertVectorMaskedStore final
     // Load the whole data and use arith.select to handle the corner cases.
     // E.g., given these input values:
     //
-    //   %mask = [1, 1, 1, 0, 0, 0]
-    //   %0[%c0, %c0] contains [0x1, 0x2, 0x3, 0x4, 0x5, 0x6]
-    //   %value_to_store = [0x7, 0x8, 0x9, 0xA, 0xB, 0xC]
+    //   %mask = [0, 1, 1, 1, 1, 1, 0, 0]
+    //   %0[%c0, %c0] contains [0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8]
+    //   %value_to_store = [0x9, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF, 0x0]
     //
     // we'll have
     //
-    //    expected output: [0x7, 0x8, 0x9, 0x4, 0x5, 0x6]
+    //    expected output: [0x1, 0xA, 0xB, 0xC, 0xD, 0xE, 0x7, 0x8]
     //
-    //    %new_mask = [1, 1, 0]
-    //    %maskedload = [0x12, 0x34, 0x0]
-    //    %bitcast = [0x1, 0x2, 0x3, 0x4, 0x0, 0x0]
-    //    %select_using_original_mask = [0x7, 0x8, 0x9, 0x4, 0x0, 0x0]
-    //    %packed_data = [0x78, 0x94, 0x00]
+    //    %new_mask = [1, 1, 1, 0]
+    //    %maskedload = [0x12, 0x34, 0x56, 0x00]
+    //    %bitcast = [0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x0, 0x0]
+    //    %select_using_shifted_mask = [0x1, 0xA, 0xB, 0xC, 0xD, 0xE, 0x0, 0x0]
+    //    %packed_data = [0x1A, 0xBC, 0xDE, 0x00]
     //
     // Using the new mask to store %packed_data results in expected output.
     FailureOr<Operation *> newMask =
@@ -243,8 +291,9 @@ struct ConvertVectorMaskedStore final
         loc, newType, adaptor.getBase(), linearizedIndices,
         newMask.value()->getResult(0), passThru);
 
-    Value valueToStore = rewriter.create<vector::BitCastOp>(
-        loc, op.getValueToStore().getType(), newLoad);
+    auto newBitCastType = VectorType::get(numElements * scale, oldElementType);
+    Value valueToStore =
+        rewriter.create<vector::BitCastOp>(loc, newBitCastType, newLoad);
     valueToStore = rewriter.create<arith::SelectOp>(
         loc, op.getMask(), op.getValueToStore(), valueToStore);
     valueToStore =
@@ -294,19 +343,31 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
     // %1 = vector.load %0[%linear_index] : memref<6xi8>, vector<2xi8>
     // %2 = vector.bitcast %1 : vector<2xi8> to vector<4xi4>
     //
-    // TODO: Currently, only the even number of elements loading is supported.
-    // To deal with the odd number of elements, one has to extract the
-    // subvector at the proper offset after bit-casting.
+    // There are cases where the number of elements to load is not byte-aligned,
+    // for example:
+    //
+    // %1 = vector.load %0[%c1, %c0] : memref<3x3xi2>, vector<3xi2>
+    //
+    // we will have to load extra bytes and extract the exact slice in between.
+    //
+    // %1 = vector.load %0[%c2] : memref<3xi8>, vector<2xi8>
+    // %2 = vector.bitcast %1 : vector<2xi8> to vector<8xi2>
+    // %3 = vector.extract_strided_slice %1 {offsets = [2], sizes = [3], strides
+    // = [1]}
+    //        : vector<8xi2> to vector<3xi2>
+    //
+    // TODO: Currently the extract_strided_slice's attributes must be known at
+    // compile time as they must be constants.
 
     auto origElements = op.getVectorType().getNumElements();
-    if (origElements % scale != 0)
-      return failure();
+    bool isUnalignedEmulation = origElements % scale != 0;
 
     auto stridedMetadata =
         rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
 
     OpFoldResult linearizedIndices;
-    std::tie(std::ignore, linearizedIndices) =
+    memref::LinearizedMemRefInfo linearizedInfo;
+    std::tie(linearizedInfo, linearizedIndices) =
         memref::getLinearizedMemRefOffsetAndSize(
             rewriter, loc, srcBits, dstBits,
             stridedMetadata.getConstifiedMixedOffset(),
@@ -314,15 +375,31 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
             stridedMetadata.getConstifiedMixedStrides(),
             getAsOpFoldResult(adaptor.getIndices()));
 
-    auto numElements = (origElements + scale - 1) / scale;
+    std::optional<int64_t> foldedIntraVectorOffset =
+        isUnalignedEmulation
+            ? getConstantIntValue(linearizedInfo.intraDataOffset)
+            : 0;
+
+    if (!foldedIntraVectorOffset) {
+      // unimplemented case for dynamic intra vector offset
+      return failure();
+    }
+
+    auto numElements =
+        llvm::divideCeil(*foldedIntraVectorOffset + origElements, scale);
     auto newLoad = rewriter.create<vector::LoadOp>(
         loc, VectorType::get(numElements, newElementType), adaptor.getBase(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices));
 
-    auto bitCast =
-        rewriter.create<vector::BitCastOp>(loc, op.getType(), newLoad);
+    Value result = rewriter.create<vector::BitCastOp>(
+        loc, VectorType::get(numElements * scale, oldElementType), newLoad);
 
-    rewriter.replaceOp(op, bitCast->getResult(0));
+    if (isUnalignedEmulation) {
+      result = extractSubvectorFrom(rewriter, loc, op.getType(), result,
+                                    *foldedIntraVectorOffset, origElements);
+    }
+
+    rewriter.replaceOp(op, result);
     return success();
   }
 };
@@ -396,13 +473,13 @@ struct ConvertVectorMaskedLoad final
     // subvector at the proper offset after bit-casting.
     auto origType = op.getVectorType();
     auto origElements = origType.getNumElements();
-    if (origElements % scale != 0)
-      return failure();
+    bool isUnalignedEmulation = origElements % scale != 0;
 
     auto stridedMetadata =
         rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
     OpFoldResult linearizedIndices;
-    std::tie(std::ignore, linearizedIndices) =
+    memref::LinearizedMemRefInfo linearizedInfo;
+    std::tie(linearizedInfo, linearizedIndices) =
         memref::getLinearizedMemRefOffsetAndSize(
             rewriter, loc, srcBits, dstBits,
             stridedMetadata.getConstifiedMixedOffset(),
@@ -410,29 +487,68 @@ struct ConvertVectorMaskedLoad final
             stridedMetadata.getConstifiedMixedStrides(),
             getAsOpFoldResult(adaptor.getIndices()));
 
+    std::optional<int64_t> foldedIntraVectorOffset =
+        isUnalignedEmulation
+            ? getConstantIntValue(linearizedInfo.intraDataOffset)
+            : 0;
+
+    if (!foldedIntraVectorOffset) {
+      // unimplemented case for dynamic intra vector offset
+      return failure();
+    }
+
     FailureOr<Operation *> newMask =
-        getCompressedMaskOp(rewriter, loc, op.getMask(), origElements, scale);
+        getCompressedMaskOp(rewriter, loc, op.getMask(), origElements, scale,
+                            *foldedIntraVectorOffset);
     if (failed(newMask))
       return failure();
 
-    auto numElements = (origElements + scale - 1) / scale;
-    auto newType = VectorType::get(numElements, newElementType);
+    auto numElements =
+        llvm::divideCeil(*foldedIntraVectorOffset + origElements, scale);
+    auto loadType = VectorType::get(numElements, newElementType);
+    auto newBitcastType = VectorType::get(numElements * scale, oldElementType);
+
+    Value passthru = op.getPassThru();
+    if (isUnalignedEmulation) {
+      // create an empty vector of the new type
+      auto emptyVector = rewriter.create<arith::ConstantOp>(
+          loc, newBitcastType, rewriter.getZeroAttr(newBitcastType));
+      passthru = insertSubvectorInto(rewriter, loc, passthru, emptyVector,
+                                     *foldedIntraVectorOffset);
+    }
     auto newPassThru =
-        rewriter.create<vector::BitCastOp>(loc, newType, op.getPassThru());
+        rewriter.create<vector::BitCastOp>(loc, loadType, passthru);
 
     // Generating the new masked load.
     auto newLoad = rewriter.create<vector::MaskedLoadOp>(
-        loc, newType, adaptor.getBase(),
+        loc, loadType, adaptor.getBase(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices),
         newMask.value()->getResult(0), newPassThru);
 
     // Setting the part that originally was not effectively loaded from memory
     // to pass through.
     auto bitCast =
-        rewriter.create<vector::BitCastOp>(loc, op.getType(), newLoad);
-    auto select = rewriter.create<arith::SelectOp>(loc, op.getMask(), bitCast,
-                                                   op.getPassThru());
-    rewriter.replaceOp(op, select->getResult(0));
+        rewriter.create<vector::BitCastOp>(loc, newBitcastType, newLoad);
+
+    Value mask = op.getMask();
+    if (isUnalignedEmulation) {
+      auto newSelectMaskType =
+          VectorType::get(numElements * scale, rewriter.getI1Type());
+      // TODO: can fold if op's mask is constant
+      auto emptyVector = rewriter.create<arith::ConstantOp>(
+          loc, newSelectMaskType, rewriter.getZeroAttr(newSelectMaskType));
+      mask = insertSubvectorInto(rewriter, loc, op.getMask(), emptyVector,
+                                 *foldedIntraVectorOffset);
+    }
+
+    Value result =
+        rewriter.create<arith::SelectOp>(loc, mask, bitCast, passthru);
+
+    if (isUnalignedEmulation) {
+      result = extractSubvectorFrom(rewriter, loc, op.getType(), result,
+                                    *foldedIntraVectorOffset, origElements);
+    }
+    rewriter.replaceOp(op, result);
 
     return success();
   }
@@ -464,8 +580,8 @@ struct ConvertVectorTransferRead final
     int scale = dstBits / srcBits;
 
     auto origElements = op.getVectorType().getNumElements();
-    if (origElements % scale != 0)
-      return failure();
+
+    bool isUnalignedEmulation = origElements % scale != 0;
 
     auto newPadding = rewriter.create<arith::ExtUIOp>(loc, newElementType,
                                                       adaptor.getPadding());
@@ -474,7 +590,8 @@ struct ConvertVectorTransferRead final
         rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getSource());
 
     OpFoldResult linearizedIndices;
-    std::tie(std::ignore, linearizedIndices) =
+    memref::LinearizedMemRefInfo linearizedInfo;
+    std::tie(linearizedInfo, linearizedIndices) =
         memref::getLinearizedMemRefOffsetAndSize(
             rewriter, loc, srcBits, dstBits,
             stridedMetadata.getConstifiedMixedOffset(),
@@ -482,18 +599,34 @@ struct ConvertVectorTransferRead final
             stridedMetadata.getConstifiedMixedStrides(),
             getAsOpFoldResult(adaptor.getIndices()));
 
-    auto numElements = (origElements + scale - 1) / scale;
-    auto newReadType = VectorType::get(numElements, newElementType);
+    std::optional<int64_t> foldedIntraVectorOffset =
+        isUnalignedEmulation
+            ? getConstantIntValue(linearizedInfo.intraDataOffset)
+            : 0;
+
+    if (!foldedIntraVectorOffset) {
+      // unimplemented case for dynamic inra-vector offset
+      return failure();
+    }
+
+    auto numElements =
+        llvm::divideCeil(*foldedIntraVectorOffset + origElements, scale);
 
     auto newRead = rewriter.create<vector::TransferReadOp>(
-        loc, newReadType, adaptor.getSource(),
+        loc, VectorType::get(numElements, newElementType), adaptor.getSource(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices),
         newPadding);
 
-    auto bitCast =
-        rewriter.create<vector::BitCastOp>(loc, op.getType(), newRead);
+    auto bitCast = rewriter.create<vector::BitCastOp>(
+        loc, VectorType::get(numElements * scale, oldElementType), newRead);
+
+    Value result = bitCast->getResult(0);
+    if (isUnalignedEmulation) {
+      result = extractSubvectorFrom(rewriter, loc, op.getType(), result,
+                                    *foldedIntraVectorOffset, origElements);
+    }
+    rewriter.replaceOp(op, result);
 
-    rewriter.replaceOp(op, bitCast->getResult(0));
     return success();
   }
 };
diff --git a/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir b/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir
new file mode 100644
index 000000000000..7ecbad796822
--- /dev/null
+++ b/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir
@@ -0,0 +1,67 @@
+// RUN: mlir-opt --test-emulate-narrow-int="arith-compute-bitwidth=1 memref-load-bitwidth=8" --cse --split-input-file %s | FileCheck %s
+
+func.func @vector_load_i2(%arg1: index, %arg2: index) -> vector<3x3xi2> {
+    %0 = memref.alloc() : memref<3x3xi2>
+    %c0 = arith.constant 0 : index
+    %c2 = arith.constant 2 : index
+    %cst = arith.constant dense<0> : vector<3x3xi2>
+    %1 = vector.load %0[%c2, %c0] : memref<3x3xi2>, vector<3xi2>
+    %2 = vector.insert %1, %cst [0] : vector<3xi2> into vector<3x3xi2>
+    return %2 : vector<3x3xi2>
+}
+
+// CHECK: func @vector_load_i2
+// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<3xi8>
+// CHECK: %[[INDEX:.+]] = arith.constant 1 : index
+// CHECK: %[[VEC:.+]] = vector.load %[[ALLOC]][%[[INDEX]]] : memref<3xi8>, vector<2xi8>
+// CHECK: %[[VEC_I2:.+]] = vector.bitcast %[[VEC]] : vector<2xi8> to vector<8xi2>
+// CHECK: %[[EXCTRACT:.+]] = vector.extract_strided_slice %[[VEC_I2]] {offsets = [2], sizes = [3], strides = [1]} : vector<8xi2> to vector<3xi2>
+
+//-----
+
+func.func @vector_transfer_read_i2() -> vector<3xi2> {
+ %0 = memref.alloc() : memref<3x3xi2>
+ %c0i2 = arith.constant 0 : i2
+ %c0 = arith.constant 0 : index
+ %c2 = arith.constant 2 : index
+ %1 = vector.transfer_read %0[%c2, %c0], %c0i2 {in_bounds = [true]} : memref<3x3xi2>, vector<3xi2>
+ return %1 : vector<3xi2>
+}
+
+// CHECK: func @vector_transfer_read_i2
+// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<3xi8>
+// CHECK: %[[INDEX:.+]] = arith.constant 1 : index
+// CHECK: %[[READ:.+]] = vector.transfer_read %[[ALLOC]][%[[INDEX]]], %0 : memref<3xi8>, vector<2xi8>
+// CHECK: %[[BITCAST:.+]] = vector.bitcast %[[READ]] : vector<2xi8> to vector<8xi2>
+// CHECK: vector.extract_strided_slice %[[BITCAST]] {offsets = [2], sizes = [3], strides = [1]} : vector<8xi2> to vector<3xi2>
+
+//-----
+
+func.func @vector_cst_maskedload_i2(%passthru: vector<5xi2>) -> vector<3x5xi2> {
+    %0 = memref.alloc() : memref<3x5xi2>
+    %cst = arith.constant dense<0> : vector<3x5xi2>
+    %mask = vector.constant_mask [3] : vector<5xi1>
+    %c0 = arith.constant 0 : index
+    %c2 = arith.constant 2 : index
+    %1 = vector.maskedload %0[%c2, %c0], %mask, %passthru :
+      memref<3x5xi2>, vector<5xi1>, vector<5xi2> into vector<5xi2>
+    %2 = vector.insert %1, %cst [0] : vector<5xi2> into vector<3x5xi2>
+    return %2 : vector<3x5xi2>
+}
+
+// CHECK: func @vector_cst_maskedload_i2
+// CHECK: %[[ORIGINMASK:.+]] = vector.constant_mask [3] : vector<5xi1>
+// CHECK: %[[NEWMASK:.+]] = arith.constant dense<true> : vector<2xi1>
+// CHECK: %[[VESSEL:.+]] = arith.constant dense<0> : vector<8xi2>
+// CHECK: %[[INSERT1:.+]] = vector.insert_strided_slice %arg0, %[[VESSEL]]
+// CHECK-SAME: {offsets = [2], strides = [1]} : vector<5xi2> into vector<8xi2>
+// CHECK: %[[BITCAST1:.+]] = vector.bitcast %[[INSERT1]] : vector<8xi2> to vector<2xi8>
+// CHECK: %[[C2:.+]] = arith.constant 2 : index
+// CHECK: %[[MASKEDLOAD:.+]] = vector.maskedload %alloc[%[[C2]]], %[[NEWMASK:.+]], %[[BITCAST1]]
+// CHECK-SAME: : memref<4xi8>, vector<2xi1>, vector<2xi8> into vector<2xi8>
+// CHECK: %[[BITCAST2:.+]] = vector.bitcast %[[MASKEDLOAD]] : vector<2xi8> to vector<8xi2>
+// CHECK: %[[CST2:.+]] = arith.constant dense<false> : vector<8xi1>
+// CHECK: %[[INSERT2:.+]] = vector.insert_strided_slice %[[ORIGINMASK]], %[[CST2]]
+// CHECK-SAME: {offsets = [2], strides = [1]} : vector<5xi1> into vector<8xi1>
+// CHECK: %[[SELECT:.+]] = arith.select %[[INSERT2]], %[[BITCAST2]], %[[INSERT1]] : vector<8xi1>, vector<8xi2>
+// CHECK: vector.extract_strided_slice %[[SELECT]] {offsets = [2], sizes = [5], strides = [1]} : vector<8xi2> to vector<5xi2> 
-- 
GitLab


From 29bff4aad8eb7f54f99e0496b735aee193063b04 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Tue, 29 Oct 2024 20:06:56 -0700
Subject: [PATCH 527/693] [llvm-objdump] Fix coloring with nested WithMarkup

WithMarkup objects may nest, resulting in the `)` in `leaq
(%rdx,%rax), %rbx` to be green instead of the default color,
mismatching the color of `(`.

```
% llvm-mc -triple=x86_64 -mdis <<< '0x48 0x8d 0x1c 0x02'
        .text
        leaq    <mem:(<reg:%rdx>,<reg:%rax>)>, <reg:%rbx>
```

To ensure that `(` and `)` get the same color, maintain a color stack
within MCInstPrinter.

Fix #99661

Pull Request: https://github.com/llvm/llvm-project/pull/113834
---
 llvm/include/llvm/MC/MCInstPrinter.h          | 10 ++++---
 llvm/lib/MC/MCInstPrinter.cpp                 | 26 ++++++++++++-------
 .../llvm-objdump/X86/disassemble-color.s      | 21 +++++++++++++++
 3 files changed, 44 insertions(+), 13 deletions(-)
 create mode 100644 llvm/test/tools/llvm-objdump/X86/disassemble-color.s

diff --git a/llvm/include/llvm/MC/MCInstPrinter.h b/llvm/include/llvm/MC/MCInstPrinter.h
index 0b9c738a7a0a..e825c04a6dba 100644
--- a/llvm/include/llvm/MC/MCInstPrinter.h
+++ b/llvm/include/llvm/MC/MCInstPrinter.h
@@ -9,8 +9,10 @@
 #ifndef LLVM_MC_MCINSTPRINTER_H
 #define LLVM_MC_MCINSTPRINTER_H
 
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
 #include <cstdint>
 
 namespace llvm {
@@ -24,7 +26,6 @@ class MCRegister;
 class MCRegisterInfo;
 class MCSubtargetInfo;
 class StringRef;
-class raw_ostream;
 
 /// Convert `Bytes' to a hex string and output to `OS'
 void dumpBytes(ArrayRef<uint8_t> Bytes, raw_ostream &OS);
@@ -76,6 +77,8 @@ protected:
   /// If true, symbolize branch target and memory reference operands.
   bool SymbolizeOperands = false;
 
+  SmallVector<raw_ostream::Colors, 4> ColorStack{raw_ostream::Colors::RESET};
+
   /// Utility function for printing annotations.
   void printAnnotation(raw_ostream &OS, StringRef Annot);
 
@@ -98,8 +101,8 @@ public:
 
   class WithMarkup {
   public:
-    LLVM_CTOR_NODISCARD WithMarkup(raw_ostream &OS, Markup M, bool EnableMarkup,
-                                   bool EnableColor);
+    LLVM_CTOR_NODISCARD WithMarkup(MCInstPrinter &IP, raw_ostream &OS, Markup M,
+                                   bool EnableMarkup, bool EnableColor);
     ~WithMarkup();
 
     template <typename T> WithMarkup &operator<<(T &O) {
@@ -113,6 +116,7 @@ public:
     }
 
   private:
+    MCInstPrinter &IP;
     raw_ostream &OS;
     bool EnableMarkup;
     bool EnableColor;
diff --git a/llvm/lib/MC/MCInstPrinter.cpp b/llvm/lib/MC/MCInstPrinter.cpp
index 488e34a6d539..069716a3ecf9 100644
--- a/llvm/lib/MC/MCInstPrinter.cpp
+++ b/llvm/lib/MC/MCInstPrinter.cpp
@@ -225,27 +225,31 @@ format_object<uint64_t> MCInstPrinter::formatHex(uint64_t Value) const {
 }
 
 MCInstPrinter::WithMarkup MCInstPrinter::markup(raw_ostream &OS, Markup S) {
-  return WithMarkup(OS, S, getUseMarkup(), getUseColor());
+  return WithMarkup(*this, OS, S, getUseMarkup(), getUseColor());
 }
 
-MCInstPrinter::WithMarkup::WithMarkup(raw_ostream &OS, Markup M,
-                                      bool EnableMarkup, bool EnableColor)
-    : OS(OS), EnableMarkup(EnableMarkup), EnableColor(EnableColor) {
+MCInstPrinter::WithMarkup::WithMarkup(MCInstPrinter &IP, raw_ostream &OS,
+                                      Markup M, bool EnableMarkup,
+                                      bool EnableColor)
+    : IP(IP), OS(OS), EnableMarkup(EnableMarkup), EnableColor(EnableColor) {
   if (EnableColor) {
+    raw_ostream::Colors Color = raw_ostream::Colors::RESET;
     switch (M) {
     case Markup::Immediate:
-      OS.changeColor(raw_ostream::RED);
+      Color = raw_ostream::RED;
       break;
     case Markup::Register:
-      OS.changeColor(raw_ostream::CYAN);
+      Color = raw_ostream::CYAN;
       break;
     case Markup::Target:
-      OS.changeColor(raw_ostream::YELLOW);
+      Color = raw_ostream::YELLOW;
       break;
     case Markup::Memory:
-      OS.changeColor(raw_ostream::GREEN);
+      Color = raw_ostream::GREEN;
       break;
     }
+    IP.ColorStack.push_back(Color);
+    OS.changeColor(Color);
   }
 
   if (EnableMarkup) {
@@ -269,6 +273,8 @@ MCInstPrinter::WithMarkup::WithMarkup(raw_ostream &OS, Markup M,
 MCInstPrinter::WithMarkup::~WithMarkup() {
   if (EnableMarkup)
     OS << '>';
-  if (EnableColor)
-    OS.resetColor();
+  if (!EnableColor)
+    return;
+  IP.ColorStack.pop_back();
+  OS << IP.ColorStack.back();
 }
diff --git a/llvm/test/tools/llvm-objdump/X86/disassemble-color.s b/llvm/test/tools/llvm-objdump/X86/disassemble-color.s
new file mode 100644
index 000000000000..4e1d82562fb5
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/X86/disassemble-color.s
@@ -0,0 +1,21 @@
+# UNSUPPORTED: system-windows
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t
+# RUN: llvm-objdump -d --no-show-raw-insn --disassembler-color=on %t | FileCheck %s --check-prefix=ATT
+# RUN: llvm-objdump -d --no-show-raw-insn --disassembler-color=on -M intel %t | FileCheck %s --check-prefix=INTEL
+
+# ATT:      <.text>:
+# ATT-NEXT:  leaq	(%rdx,%rax,4), %rbx
+# ATT-NEXT:  movq	(,%rax), %rbx
+# ATT-NEXT:  leaq	0x3(%rdx,%rax), %rbx
+# ATT-NEXT:  movq	$0x3, %rax
+
+# INTEL:      <.text>:
+# INTEL-NEXT:  lea	rbx, [rdx + 4*rax]
+# INTEL-NEXT:  mov	rbx, qword ptr [1*rax]
+# INTEL-NEXT:  lea	rbx, [rdx + rax + 0x3]
+# INTEL-NEXT:  mov	rax, 0x3
+
+leaq (%rdx,%rax,4), %rbx
+movq (,%rax), %rbx
+leaq 3(%rdx,%rax), %rbx
+movq $3, %rax
-- 
GitLab


From 3c02fea737d774bbf174c6b763593ad3e7f56221 Mon Sep 17 00:00:00 2001
From: Piotr Fusik <p.fusik@samsung.com>
Date: Wed, 30 Oct 2024 04:07:14 +0100
Subject: [PATCH 528/693] [LV][NFC] Remove stray semicolons (#114057)

---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 778d928252e0..150fc4a42b48 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -581,8 +581,8 @@ protected:
 
   /// Allow subclasses to override and print debug traces before/after vplan
   /// execution, when trace information is requested.
-  virtual void printDebugTracesAtStart(){};
-  virtual void printDebugTracesAtEnd(){};
+  virtual void printDebugTracesAtStart() {}
+  virtual void printDebugTracesAtEnd() {}
 
   /// The original loop.
   Loop *OrigLoop;
@@ -1310,7 +1310,7 @@ public:
       return false;
     case cl::BOU_FALSE:
       return true;
-    };
+    }
     llvm_unreachable("impossible case value");
   }
 
@@ -9065,7 +9065,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
     if (!getDecisionAndClampRange(ApplyIG, Range))
       continue;
     InterleaveGroups.insert(IG);
-  };
+  }
 
   // ---------------------------------------------------------------------------
   // Construct recipes for the instructions in the loop
-- 
GitLab


From ef455e6b16334128c008fc57a4d8ace701934e80 Mon Sep 17 00:00:00 2001
From: Jessica Clarke <jrtc27@jrtc27.com>
Date: Wed, 30 Oct 2024 03:12:23 +0000
Subject: [PATCH 529/693] [TableGen] Replace all lingering uses of getName with
 getEnumName

The former is a wrapper for the latter with two differences: Other is
mapped to "UNKNOWN" (rather than "MVT::Other"), and iPTR(Any) are mapped
to "TLI.getPointerTy()" rather than "MVT::iPTR(Any)".

The only uses are in FastISelMap::printFunctionDefinitions. Most of
these uses are just a form of name mangling to ensure uniqueness, so the
actual string isn't important (and, in the case of MVT::iPTR(Any), were
both to be used, they would clash). Two uses are for a case statement,
which requires the expression to be a constant (of the right type), but
neither UNKNOWN nor TLI.getPointerTy() are constants, so would not work
there. The remaining uses are where an expression is needed, so UNKNOWN
similarly doesn't work, though TLI.getPointerTy() could in this case.
However, neither iPTR nor iPTRAny are supposed to make it this far
through TableGen, and should instead have been replaced with concrete
types, so this case should not be hit. Moreover, for almost all of these
uses, the name is passed to getLegalCName, which will strip an MVT::
prefix but will leave TLI.getPointerTy() unchanged, which is not a valid
C identifier, nor component thereof.

Thus, delete this unnecessary, and mostly-broken, wrapper and just use
the underlying getEnumName. This has been verified to have no effect on
the generated files for any in-tree target, including experimental ones.

Reviewers: arsenm

Reviewed By: arsenm

Pull Request: https://github.com/llvm/llvm-project/pull/113731
---
 llvm/utils/TableGen/Common/CodeGenTarget.cpp | 13 ------------
 llvm/utils/TableGen/Common/CodeGenTarget.h   |  1 -
 llvm/utils/TableGen/FastISelEmitter.cpp      | 21 ++++++++++----------
 3 files changed, 11 insertions(+), 24 deletions(-)

diff --git a/llvm/utils/TableGen/Common/CodeGenTarget.cpp b/llvm/utils/TableGen/Common/CodeGenTarget.cpp
index b358518c4290..4e75db689a0b 100644
--- a/llvm/utils/TableGen/Common/CodeGenTarget.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenTarget.cpp
@@ -47,19 +47,6 @@ MVT::SimpleValueType llvm::getValueType(const Record *Rec) {
   return (MVT::SimpleValueType)Rec->getValueAsInt("Value");
 }
 
-StringRef llvm::getName(MVT::SimpleValueType T) {
-  switch (T) {
-  case MVT::Other:
-    return "UNKNOWN";
-  case MVT::iPTR:
-    return "TLI.getPointerTy()";
-  case MVT::iPTRAny:
-    return "TLI.getPointerTy()";
-  default:
-    return getEnumName(T);
-  }
-}
-
 StringRef llvm::getEnumName(MVT::SimpleValueType T) {
   // clang-format off
   switch (T) {
diff --git a/llvm/utils/TableGen/Common/CodeGenTarget.h b/llvm/utils/TableGen/Common/CodeGenTarget.h
index c7b44f7028eb..8bcb2f677a00 100644
--- a/llvm/utils/TableGen/Common/CodeGenTarget.h
+++ b/llvm/utils/TableGen/Common/CodeGenTarget.h
@@ -46,7 +46,6 @@ class CodeGenSubRegIndex;
 /// record corresponds to.
 MVT::SimpleValueType getValueType(const Record *Rec);
 
-StringRef getName(MVT::SimpleValueType T);
 StringRef getEnumName(MVT::SimpleValueType T);
 
 /// getQualifiedName - Return the name of the specified record, with a
diff --git a/llvm/utils/TableGen/FastISelEmitter.cpp b/llvm/utils/TableGen/FastISelEmitter.cpp
index 17198c85f060..2052222cae5e 100644
--- a/llvm/utils/TableGen/FastISelEmitter.cpp
+++ b/llvm/utils/TableGen/FastISelEmitter.cpp
@@ -718,19 +718,20 @@ void FastISelMap::printFunctionDefinitions(raw_ostream &OS) {
             const PredMap &PM = RI.second;
 
             OS << "unsigned fastEmit_" << getLegalCName(Opcode) << "_"
-               << getLegalCName(std::string(getName(VT))) << "_"
-               << getLegalCName(std::string(getName(RetVT))) << "_";
+               << getLegalCName(std::string(getEnumName(VT))) << "_"
+               << getLegalCName(std::string(getEnumName(RetVT))) << "_";
             Operands.PrintManglingSuffix(OS, ImmediatePredicates);
             OS << "(";
             Operands.PrintParameters(OS);
             OS << ") {\n";
 
-            emitInstructionCode(OS, Operands, PM, std::string(getName(RetVT)));
+            emitInstructionCode(OS, Operands, PM,
+                                std::string(getEnumName(RetVT)));
           }
 
           // Emit one function for the type that demultiplexes on return type.
           OS << "unsigned fastEmit_" << getLegalCName(Opcode) << "_"
-             << getLegalCName(std::string(getName(VT))) << "_";
+             << getLegalCName(std::string(getEnumName(VT))) << "_";
           Operands.PrintManglingSuffix(OS, ImmediatePredicates);
           OS << "(MVT RetVT";
           if (!Operands.empty())
@@ -739,10 +740,10 @@ void FastISelMap::printFunctionDefinitions(raw_ostream &OS) {
           OS << ") {\nswitch (RetVT.SimpleTy) {\n";
           for (const auto &RI : RM) {
             MVT::SimpleValueType RetVT = RI.first;
-            OS << "  case " << getName(RetVT) << ": return fastEmit_"
+            OS << "  case " << getEnumName(RetVT) << ": return fastEmit_"
                << getLegalCName(Opcode) << "_"
-               << getLegalCName(std::string(getName(VT))) << "_"
-               << getLegalCName(std::string(getName(RetVT))) << "_";
+               << getLegalCName(std::string(getEnumName(VT))) << "_"
+               << getLegalCName(std::string(getEnumName(RetVT))) << "_";
             Operands.PrintManglingSuffix(OS, ImmediatePredicates);
             OS << "(";
             Operands.PrintArguments(OS);
@@ -753,7 +754,7 @@ void FastISelMap::printFunctionDefinitions(raw_ostream &OS) {
         } else {
           // Non-variadic return type.
           OS << "unsigned fastEmit_" << getLegalCName(Opcode) << "_"
-             << getLegalCName(std::string(getName(VT))) << "_";
+             << getLegalCName(std::string(getEnumName(VT))) << "_";
           Operands.PrintManglingSuffix(OS, ImmediatePredicates);
           OS << "(MVT RetVT";
           if (!Operands.empty())
@@ -761,7 +762,7 @@ void FastISelMap::printFunctionDefinitions(raw_ostream &OS) {
           Operands.PrintParameters(OS);
           OS << ") {\n";
 
-          OS << "  if (RetVT.SimpleTy != " << getName(RM.begin()->first)
+          OS << "  if (RetVT.SimpleTy != " << getEnumName(RM.begin()->first)
              << ")\n    return 0;\n";
 
           const PredMap &PM = RM.begin()->second;
@@ -781,7 +782,7 @@ void FastISelMap::printFunctionDefinitions(raw_ostream &OS) {
       OS << "  switch (VT.SimpleTy) {\n";
       for (const auto &TI : TM) {
         MVT::SimpleValueType VT = TI.first;
-        std::string TypeName = std::string(getName(VT));
+        std::string TypeName = std::string(getEnumName(VT));
         OS << "  case " << TypeName << ": return fastEmit_"
            << getLegalCName(Opcode) << "_" << getLegalCName(TypeName) << "_";
         Operands.PrintManglingSuffix(OS, ImmediatePredicates);
-- 
GitLab


From e8b7f53fa4dc8a9f74a3d67dfb89eb68fcd78679 Mon Sep 17 00:00:00 2001
From: Jessica Clarke <jrtc27@jrtc27.com>
Date: Wed, 30 Oct 2024 03:19:53 +0000
Subject: [PATCH 530/693] [TableGen] Remove a pointless check for iPTRAny

We've already called EnforceInteger on Types[0], and iPTRAny isn't
regarded as an integer type (note that TableGen special-cases iPTR here
to include that, though), so we cannot possibly still have an iPTRAny by
this point. Delete the check, and let getFixedSizeInBits catch it along
with all the other overloaded types if that ever becomes false. Also
document why we have this check whilst here.

Reviewers: arsenm

Reviewed By: arsenm

Pull Request: https://github.com/llvm/llvm-project/pull/113732
---
 llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
index d2228c902a56..3446bfeb3e7e 100644
--- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
@@ -2461,7 +2461,8 @@ bool TreePatternNode::ApplyTypeConstraints(TreePattern &TP, bool NotRegisters) {
       ValueTypeByHwMode VVT = TP.getInfer().getConcrete(Types[0], false);
       for (auto &P : VVT) {
         MVT::SimpleValueType VT = P.second.SimpleTy;
-        if (VT == MVT::iPTR || VT == MVT::iPTRAny)
+        // Can only check for types of a known size
+        if (VT == MVT::iPTR)
           continue;
         unsigned Size = MVT(VT).getFixedSizeInBits();
         // Make sure that the value is representable for this type.
-- 
GitLab


From 9467645547f99ba8fa8152d514f06e76e0be8585 Mon Sep 17 00:00:00 2001
From: Jessica Clarke <jrtc27@jrtc27.com>
Date: Wed, 30 Oct 2024 03:27:48 +0000
Subject: [PATCH 531/693] [CodeGen] Rename MVT::iPTRAny to MVT::pAny

Whilst in upstream LLVM iPTRAny is only ever an integer, essentially an
alias for iPTR, this is not true in CHERI LLVM, where it gets used to
mean "iPTR or cPTR", i.e. either an integer address or a capability
(with cPTR and cN being the capability equivalents of iPTR and iN).
Moreover, iPTRAny is already not itself regarded as an integer (calling
isInteger() will give false), so the "i" prefix is misleading, and it
stands out as different from all the other xAny that have a single
letter prefix denoting their type.

Thus, rename it to pAny, reflecting that it is an overloaded pointer
type, which could end up being specialised to an integer type, but does
not have to be.

This has been verified to have no effect on the generated files for LLVM
itself or any in-tree target beyond the replacement of the identifier
iPTRAny with pAny in GenVT.inc.

Reviewers: arsenm

Reviewed By: arsenm

Pull Request: https://github.com/llvm/llvm-project/pull/113733
---
 llvm/include/llvm/CodeGen/ValueTypes.h            | 3 ++-
 llvm/include/llvm/CodeGen/ValueTypes.td           | 4 ++--
 llvm/include/llvm/CodeGenTypes/MachineValueType.h | 2 +-
 llvm/include/llvm/IR/Intrinsics.h                 | 2 +-
 llvm/include/llvm/IR/Intrinsics.td                | 6 +++---
 llvm/lib/Target/NVPTX/NVPTXInstrInfo.td           | 2 +-
 llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp | 4 ++--
 mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp     | 2 +-
 8 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/ValueTypes.h b/llvm/include/llvm/CodeGen/ValueTypes.h
index 3db6f33a8093..4de109739227 100644
--- a/llvm/include/llvm/CodeGen/ValueTypes.h
+++ b/llvm/include/llvm/CodeGen/ValueTypes.h
@@ -230,7 +230,8 @@ namespace llvm {
 
     /// Return true if this is an overloaded type for TableGen.
     bool isOverloaded() const {
-      return (V==MVT::iAny || V==MVT::fAny || V==MVT::vAny || V==MVT::iPTRAny);
+      return (V == MVT::iAny || V == MVT::fAny || V == MVT::vAny ||
+              V == MVT::pAny);
     }
 
     /// Return true if the bit size is a multiple of 8.
diff --git a/llvm/include/llvm/CodeGen/ValueTypes.td b/llvm/include/llvm/CodeGen/ValueTypes.td
index 493c0cfcab60..6d6b92958b43 100644
--- a/llvm/include/llvm/CodeGen/ValueTypes.td
+++ b/llvm/include/llvm/CodeGen/ValueTypes.td
@@ -338,9 +338,9 @@ def MetadataVT : ValueType<0, 505> { // Metadata
   let LLVMName = "Metadata";
 }
 
-// Pseudo valuetype mapped to the current pointer size to any address space.
+// Pseudo valuetype to represent "pointer to any address space"
 // Should only be used in TableGen.
-def iPTRAny    : VTAny<506>;
+def pAny       : VTAny<506>;
 
 // Pseudo valuetype to represent "vector of any size"
 // Should only be used in TableGen.
diff --git a/llvm/include/llvm/CodeGenTypes/MachineValueType.h b/llvm/include/llvm/CodeGenTypes/MachineValueType.h
index c9a5098ef162..5c47ad4824a7 100644
--- a/llvm/include/llvm/CodeGenTypes/MachineValueType.h
+++ b/llvm/include/llvm/CodeGenTypes/MachineValueType.h
@@ -320,7 +320,7 @@ namespace llvm {
         llvm_unreachable("Value type is non-standard value, Other.");
       case iPTR:
         llvm_unreachable("Value type size is target-dependent. Ask TLI.");
-      case iPTRAny:
+      case pAny:
       case iAny:
       case fAny:
       case vAny:
diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h
index e893295e3272..89dfff256e0c 100644
--- a/llvm/include/llvm/IR/Intrinsics.h
+++ b/llvm/include/llvm/IR/Intrinsics.h
@@ -92,7 +92,7 @@ namespace Intrinsic {
   /// return the existing declaration.
   ///
   /// The \p Tys parameter is for intrinsics with overloaded types (e.g., those
-  /// using iAny, fAny, vAny, or iPTRAny).  For a declaration of an overloaded
+  /// using iAny, fAny, vAny, or pAny).  For a declaration of an overloaded
   /// intrinsic, Tys must provide exactly one type for each overloaded type in
   /// the intrinsic.
   Function *getOrInsertDeclaration(Module *M, ID id, ArrayRef<Type *> Tys = {});
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index e91758ed34eb..8ed57f818d60 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -388,7 +388,7 @@ class LLVMAnyType<ValueType vt> : LLVMType<vt> {
     !eq(vt, iAny)    : ArgKind.AnyInteger,
     !eq(vt, fAny)    : ArgKind.AnyFloat,
     !eq(vt, vAny)    : ArgKind.AnyVector,
-    !eq(vt, iPTRAny) : ArgKind.AnyPointer,
+    !eq(vt, pAny)    : ArgKind.AnyPointer,
   );
   let Sig = [
     IIT_ARG.Number,
@@ -412,8 +412,8 @@ class LLVMQualPointerType<int addrspace>
     ]);
 }
 
-class LLVMAnyPointerType : LLVMAnyType<iPTRAny> {
-  assert isAny, "iPTRAny should have isOverloaded";
+class LLVMAnyPointerType : LLVMAnyType<pAny> {
+  assert isAny, "pAny should have isOverloaded";
 }
 
 // Match the type of another intrinsic parameter.  Number is an index into the
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 5f6cba397c53..1ca3aefb0b09 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -1922,7 +1922,7 @@ def imem : Operand<iPTR> {
   let PrintMethod = "printOperand";
 }
 
-def imemAny : Operand<iPTRAny> {
+def imemAny : Operand<pAny> {
   let PrintMethod = "printOperand";
 }
 
diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
index 3446bfeb3e7e..f17c62dd1fd9 100644
--- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
@@ -813,8 +813,8 @@ void TypeInfer::expandOverloads(TypeSetByHwMode &VTS) const {
 
 void TypeInfer::expandOverloads(TypeSetByHwMode::SetType &Out,
                                 const TypeSetByHwMode::SetType &Legal) const {
-  if (Out.count(MVT::iPTRAny)) {
-    Out.erase(MVT::iPTRAny);
+  if (Out.count(MVT::pAny)) {
+    Out.erase(MVT::pAny);
     Out.insert(MVT::iPTR);
   } else if (Out.count(MVT::iAny)) {
     Out.erase(MVT::iAny);
diff --git a/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp b/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp
index 411a98a48bfb..525c8d6d3e89 100644
--- a/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp
+++ b/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp
@@ -76,7 +76,7 @@ static IndicesTy getOverloadableTypeIdxs(const Record &record,
     case llvm::MVT::iAny:
     case llvm::MVT::fAny:
     case llvm::MVT::Any:
-    case llvm::MVT::iPTRAny:
+    case llvm::MVT::pAny:
     case llvm::MVT::vAny:
       overloadedOps.set(r.index());
       break;
-- 
GitLab


From 0d94c7b5ceb84b33b50c8e7b1fa66e9996a29373 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Tue, 29 Oct 2024 20:39:17 -0700
Subject: [PATCH 532/693] [flang][cuda][NFC] Make pattern names homogenous
 (#114156)

Dialect name is uppercase. Make all the patterns prefix homogenous.
---
 .../Optimizer/Transforms/CUFOpConversion.cpp  | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
index fe125db7b406..f1f3a95b220d 100644
--- a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
@@ -173,7 +173,7 @@ static mlir::LogicalResult convertOpToCall(OpTy op,
   return mlir::success();
 }
 
-struct CufAllocateOpConversion
+struct CUFAllocateOpConversion
     : public mlir::OpRewritePattern<cuf::AllocateOp> {
   using OpRewritePattern::OpRewritePattern;
 
@@ -216,7 +216,7 @@ struct CufAllocateOpConversion
   }
 };
 
-struct CufDeallocateOpConversion
+struct CUFDeallocateOpConversion
     : public mlir::OpRewritePattern<cuf::DeallocateOp> {
   using OpRewritePattern::OpRewritePattern;
 
@@ -284,10 +284,10 @@ static int computeWidth(mlir::Location loc, mlir::Type type,
   return width;
 }
 
-struct CufAllocOpConversion : public mlir::OpRewritePattern<cuf::AllocOp> {
+struct CUFAllocOpConversion : public mlir::OpRewritePattern<cuf::AllocOp> {
   using OpRewritePattern::OpRewritePattern;
 
-  CufAllocOpConversion(mlir::MLIRContext *context, mlir::DataLayout *dl,
+  CUFAllocOpConversion(mlir::MLIRContext *context, mlir::DataLayout *dl,
                        const fir::LLVMTypeConverter *typeConverter)
       : OpRewritePattern(context), dl{dl}, typeConverter{typeConverter} {}
 
@@ -380,7 +380,7 @@ private:
   const fir::LLVMTypeConverter *typeConverter;
 };
 
-struct CufFreeOpConversion : public mlir::OpRewritePattern<cuf::FreeOp> {
+struct CUFFreeOpConversion : public mlir::OpRewritePattern<cuf::FreeOp> {
   using OpRewritePattern::OpRewritePattern;
 
   mlir::LogicalResult
@@ -429,11 +429,11 @@ struct CufFreeOpConversion : public mlir::OpRewritePattern<cuf::FreeOp> {
   }
 };
 
-struct CufDataTransferOpConversion
+struct CUFDataTransferOpConversion
     : public mlir::OpRewritePattern<cuf::DataTransferOp> {
   using OpRewritePattern::OpRewritePattern;
 
-  CufDataTransferOpConversion(mlir::MLIRContext *context,
+  CUFDataTransferOpConversion(mlir::MLIRContext *context,
                               const mlir::SymbolTable &symtab)
       : OpRewritePattern(context), symtab{symtab} {}
 
@@ -718,9 +718,9 @@ public:
 void cuf::populateCUFToFIRConversionPatterns(
     const fir::LLVMTypeConverter &converter, mlir::DataLayout &dl,
     const mlir::SymbolTable &symtab, mlir::RewritePatternSet &patterns) {
-  patterns.insert<CufAllocOpConversion>(patterns.getContext(), &dl, &converter);
-  patterns.insert<CufAllocateOpConversion, CufDeallocateOpConversion,
-                  CufFreeOpConversion>(patterns.getContext());
-  patterns.insert<CufDataTransferOpConversion, CUFLaunchOpConversion>(
+  patterns.insert<CUFAllocOpConversion>(patterns.getContext(), &dl, &converter);
+  patterns.insert<CUFAllocateOpConversion, CUFDeallocateOpConversion,
+                  CUFFreeOpConversion>(patterns.getContext());
+  patterns.insert<CUFDataTransferOpConversion, CUFLaunchOpConversion>(
       patterns.getContext(), symtab);
 }
-- 
GitLab


From cb04d3378096b83e5e357490ff8b1c479f34c469 Mon Sep 17 00:00:00 2001
From: jeffreytan81 <jeffreytan@meta.com>
Date: Tue, 29 Oct 2024 20:42:54 -0700
Subject: [PATCH 533/693] Improve namespace lookup using .debug_names parent
 chain (#110062)

## Summary
This PR is a continuation of
https://github.com/llvm/llvm-project/pull/108907 by using `.debug_names`
parent chain faster lookup for namespaces.


## Implementation
Similar to https://github.com/llvm/llvm-project/pull/108907. This PR
adds a new API: `GetNamespacesWithParents` in `DWARFIndex` base class.
The API performs the same function as `GetNamespaces()` with additional
filtering using parents `CompilerDeclContext`. A default implementation
is given in `DWARFIndex` class which parses debug info and performs the
matching. In the `DebugNameDWARFIndex` override, parents
`CompilerDeclContext` is cross checked with parent chain in
`.debug_names` for much faster filtering before fallback to base
implementation for final filtering.

## Performance Results
For the same benchmark used in
https://github.com/llvm/llvm-project/pull/108907, this PR improves: 48s
=> 28s

---------

Co-authored-by: jeffreytan81 <jeffreytan@fb.com>
---
 .../Plugins/SymbolFile/DWARF/DWARFIndex.cpp   | 16 +++++
 .../Plugins/SymbolFile/DWARF/DWARFIndex.h     | 11 ++++
 .../SymbolFile/DWARF/DebugNamesDWARFIndex.cpp | 64 ++++++++++++++++---
 .../SymbolFile/DWARF/DebugNamesDWARFIndex.h   |  4 +-
 .../SymbolFile/DWARF/SymbolFileDWARF.cpp      |  2 +-
 5 files changed, 85 insertions(+), 12 deletions(-)

diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.cpp
index dee90804c525..c18edd10b968 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.cpp
@@ -151,3 +151,19 @@ bool DWARFIndex::ProcessTypeDIEMatchQuery(
     return true;
   return callback(die);
 }
+
+void DWARFIndex::GetNamespacesWithParents(
+    ConstString name, const CompilerDeclContext &parent_decl_ctx,
+    llvm::function_ref<bool(DWARFDIE die)> callback) {
+  GetNamespaces(name, [&](DWARFDIE die) {
+    return ProcessNamespaceDieMatchParents(parent_decl_ctx, die, callback);
+  });
+}
+
+bool DWARFIndex::ProcessNamespaceDieMatchParents(
+    const CompilerDeclContext &parent_decl_ctx, DWARFDIE die,
+    llvm::function_ref<bool(DWARFDIE die)> callback) {
+  if (!SymbolFileDWARF::DIEInDeclContext(parent_decl_ctx, die))
+    return true;
+  return callback(die);
+}
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.h
index fea3a4fd6973..ac1f75e91c21 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.h
@@ -71,6 +71,14 @@ public:
   virtual void
   GetTypesWithQuery(TypeQuery &query,
                     llvm::function_ref<bool(DWARFDIE die)> callback);
+  /// Get namespace DIEs whose base name match \param name with \param
+  /// parent_decl_ctx in its decl parent chain.  A base implementation
+  /// is provided. Specializations should override this if they are able to
+  /// provide a faster implementation.
+  virtual void
+  GetNamespacesWithParents(ConstString name,
+                           const CompilerDeclContext &parent_decl_ctx,
+                           llvm::function_ref<bool(DWARFDIE die)> callback);
   virtual void
   GetFunctions(const Module::LookupInfo &lookup_info, SymbolFileDWARF &dwarf,
                const CompilerDeclContext &parent_decl_ctx,
@@ -127,6 +135,9 @@ protected:
   bool
   ProcessTypeDIEMatchQuery(TypeQuery &query, DWARFDIE die,
                            llvm::function_ref<bool(DWARFDIE die)> callback);
+  bool ProcessNamespaceDieMatchParents(
+      const CompilerDeclContext &parent_decl_ctx, DWARFDIE die,
+      llvm::function_ref<bool(DWARFDIE die)> callback);
 };
 } // namespace dwarf
 } // namespace lldb_private::plugin
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp
index c809e5ff7f85..6f2cb455ec00 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp
@@ -368,9 +368,10 @@ void DebugNamesDWARFIndex::GetFullyQualifiedType(
       continue;
     }
 
-    if (SameParentChain(parent_names, *parent_chain) &&
-        !ProcessEntry(entry, callback))
-      return;
+    if (SameParentChain(parent_names, *parent_chain)) {
+      if (!ProcessEntry(entry, callback))
+        return;
+    }
   }
   m_fallback.GetFullyQualifiedType(context, callback);
 }
@@ -554,17 +555,60 @@ void DebugNamesDWARFIndex::GetTypesWithQuery(
       continue;
     }
 
-    if (WithinParentChain(parent_contexts, *parent_chain) &&
-        !ProcessEntry(entry, [&](DWARFDIE die) {
-          // After .debug_names filtering still sending to base class for
-          // further filtering before calling the callback.
-          return ProcessTypeDIEMatchQuery(query, die, callback);
-        }))
-      return;
+    if (WithinParentChain(parent_contexts, *parent_chain)) {
+      if (!ProcessEntry(entry, [&](DWARFDIE die) {
+            // After .debug_names filtering still sending to base class for
+            // further filtering before calling the callback.
+            return ProcessTypeDIEMatchQuery(query, die, callback);
+          }))
+        // If the callback returns false, we're done.
+        return;
+    }
   }
   m_fallback.GetTypesWithQuery(query, callback);
 }
 
+void DebugNamesDWARFIndex::GetNamespacesWithParents(
+    ConstString name, const CompilerDeclContext &parent_decl_ctx,
+    llvm::function_ref<bool(DWARFDIE die)> callback) {
+  std::vector<lldb_private::CompilerContext> parent_contexts =
+      parent_decl_ctx.GetCompilerContext();
+  llvm::SmallVector<CompilerContext> parent_named_contexts;
+  std::copy_if(parent_contexts.rbegin(), parent_contexts.rend(),
+               std::back_inserter(parent_named_contexts),
+               [](const CompilerContext &ctx) { return !ctx.name.IsEmpty(); });
+  for (const DebugNames::Entry &entry :
+       m_debug_names_up->equal_range(name.GetStringRef())) {
+    lldb_private::dwarf::Tag entry_tag = entry.tag();
+    if (entry_tag == DW_TAG_namespace ||
+        entry_tag == DW_TAG_imported_declaration) {
+      std::optional<llvm::SmallVector<Entry, 4>> parent_chain =
+          getParentChain(entry);
+      if (!parent_chain) {
+        // Fallback: use the base class implementation.
+        if (!ProcessEntry(entry, [&](DWARFDIE die) {
+              return ProcessNamespaceDieMatchParents(parent_decl_ctx, die,
+                                                     callback);
+            }))
+          return;
+        continue;
+      }
+
+      if (WithinParentChain(parent_named_contexts, *parent_chain)) {
+        if (!ProcessEntry(entry, [&](DWARFDIE die) {
+              // After .debug_names filtering still sending to base class for
+              // further filtering before calling the callback.
+              return ProcessNamespaceDieMatchParents(parent_decl_ctx, die,
+                                                     callback);
+            }))
+          // If the callback returns false, we're done.
+          return;
+      }
+    }
+  }
+  m_fallback.GetNamespacesWithParents(name, parent_decl_ctx, callback);
+}
+
 void DebugNamesDWARFIndex::GetFunctions(
     const Module::LookupInfo &lookup_info, SymbolFileDWARF &dwarf,
     const CompilerDeclContext &parent_decl_ctx,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.h b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.h
index 074f68a8c559..ab6cde12623f 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.h
@@ -55,7 +55,9 @@ public:
   void
   GetTypesWithQuery(TypeQuery &query,
                     llvm::function_ref<bool(DWARFDIE die)> callback) override;
-
+  void GetNamespacesWithParents(
+      ConstString name, const CompilerDeclContext &parent_decl_ctx,
+      llvm::function_ref<bool(DWARFDIE die)> callback) override;
   void GetFunctions(const Module::LookupInfo &lookup_info,
                     SymbolFileDWARF &dwarf,
                     const CompilerDeclContext &parent_decl_ctx,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index e5b8eee8d08c..f23f8cc3d781 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -2900,7 +2900,7 @@ SymbolFileDWARF::FindNamespace(ConstString name,
   if (!DeclContextMatchesThisSymbolFile(parent_decl_ctx))
     return namespace_decl_ctx;
 
-  m_index->GetNamespaces(name, [&](DWARFDIE die) {
+  m_index->GetNamespacesWithParents(name, parent_decl_ctx, [&](DWARFDIE die) {
     if (!DIEInDeclContext(parent_decl_ctx, die, only_root_namespaces))
       return true; // The containing decl contexts don't match
 
-- 
GitLab


From 3de5dbb1110887d5127e815f3ca247a9d839ee85 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Tue, 29 Oct 2024 23:43:45 -0400
Subject: [PATCH 534/693] [AMDGPU][Attributor] Check the validity of a
 dependent AA before using its value (#114165)

Even though the Attributor framework will invalidate all its dependent
AAs after the current iteration, a dependent AA can still use the worst
state of a depending AA if it doesn't check the state of the depending
AA in current iteration.
---
 llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
index 687a7339da37..6a69b9d2bfc7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
@@ -358,7 +358,7 @@ struct AAUniformWorkGroupSizeFunction : public AAUniformWorkGroupSize {
 
       const auto *CallerInfo = A.getAAFor<AAUniformWorkGroupSize>(
           *this, IRPosition::function(*Caller), DepClassTy::REQUIRED);
-      if (!CallerInfo)
+      if (!CallerInfo || !CallerInfo->isValidState())
         return false;
 
       Change = Change | clampStateAndIndicateChange(this->getState(),
@@ -449,7 +449,8 @@ struct AAAMDAttributesFunction : public AAAMDAttributes {
     // Check for Intrinsics and propagate attributes.
     const AACallEdges *AAEdges = A.getAAFor<AACallEdges>(
         *this, this->getIRPosition(), DepClassTy::REQUIRED);
-    if (!AAEdges || AAEdges->hasNonAsmUnknownCallee())
+    if (!AAEdges || !AAEdges->isValidState() ||
+        AAEdges->hasNonAsmUnknownCallee())
       return indicatePessimisticFixpoint();
 
     bool IsNonEntryFunc = !AMDGPU::isEntryFunctionCC(F->getCallingConv());
@@ -465,7 +466,7 @@ struct AAAMDAttributesFunction : public AAAMDAttributes {
       if (IID == Intrinsic::not_intrinsic) {
         const AAAMDAttributes *AAAMD = A.getAAFor<AAAMDAttributes>(
             *this, IRPosition::function(*Callee), DepClassTy::REQUIRED);
-        if (!AAAMD)
+        if (!AAAMD || !AAAMD->isValidState())
           return indicatePessimisticFixpoint();
         *this &= *AAAMD;
         continue;
@@ -660,7 +661,7 @@ private:
 
       const auto *PointerInfoAA = A.getAAFor<AAPointerInfo>(
           *this, IRPosition::callsite_returned(Call), DepClassTy::REQUIRED);
-      if (!PointerInfoAA)
+      if (!PointerInfoAA || !PointerInfoAA->getState().isValidState())
         return false;
 
       return PointerInfoAA->forallInterferingAccesses(
@@ -717,7 +718,7 @@ struct AAAMDSizeRangeAttribute
 
       const auto *CallerInfo = A.getAAFor<AttributeImpl>(
           *this, IRPosition::function(*Caller), DepClassTy::REQUIRED);
-      if (!CallerInfo)
+      if (!CallerInfo || !CallerInfo->isValidState())
         return false;
 
       Change |=
@@ -835,7 +836,8 @@ struct AAAMDWavesPerEU : public AAAMDSizeRangeAttribute {
     auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
 
     if (const auto *AssumedGroupSize = A.getAAFor<AAAMDFlatWorkGroupSize>(
-            *this, IRPosition::function(*F), DepClassTy::REQUIRED)) {
+            *this, IRPosition::function(*F), DepClassTy::REQUIRED);
+        AssumedGroupSize->isValidState()) {
 
       unsigned Min, Max;
       std::tie(Min, Max) = InfoCache.getWavesPerEU(
@@ -864,7 +866,8 @@ struct AAAMDWavesPerEU : public AAAMDSizeRangeAttribute {
           *this, IRPosition::function(*Caller), DepClassTy::REQUIRED);
       const auto *AssumedGroupSize = A.getAAFor<AAAMDFlatWorkGroupSize>(
           *this, IRPosition::function(*Func), DepClassTy::REQUIRED);
-      if (!CallerInfo || !AssumedGroupSize)
+      if (!CallerInfo || !AssumedGroupSize || !CallerInfo->isValidState() ||
+          !AssumedGroupSize->isValidState())
         return false;
 
       unsigned Min, Max;
@@ -982,7 +985,8 @@ struct AAAMDGPUNoAGPR
       // TODO: Handle callsite attributes
       const auto *CalleeInfo = A.getAAFor<AAAMDGPUNoAGPR>(
           *this, IRPosition::function(*Callee), DepClassTy::REQUIRED);
-      return CalleeInfo && CalleeInfo->getAssumed();
+      return CalleeInfo && CalleeInfo->isValidState() &&
+             CalleeInfo->getAssumed();
     };
 
     bool UsedAssumedInformation = false;
-- 
GitLab


From cc60c46e39b0fffadc83a905b37d98aff426ac17 Mon Sep 17 00:00:00 2001
From: Lei Wang <wlei@fb.com>
Date: Tue, 29 Oct 2024 21:06:43 -0700
Subject: [PATCH 535/693] specify clang --target to fix breakage on AIX
 (#114127)

`-fprofile-sample-use` is not supported on AIX, which caused a CI
failure.
---
 clang/test/CodeGen/pgo-cold-function-coverage.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/test/CodeGen/pgo-cold-function-coverage.c b/clang/test/CodeGen/pgo-cold-function-coverage.c
index fd1e1e7e14cd..3003cdc3e15e 100644
--- a/clang/test/CodeGen/pgo-cold-function-coverage.c
+++ b/clang/test/CodeGen/pgo-cold-function-coverage.c
@@ -1,7 +1,7 @@
 // Test -fprofile-generate-cold-function-coverage 
 
 // RUN: rm -rf %t && split-file %s %t
-// RUN: %clang -O2 -fprofile-generate-cold-function-coverage=/xxx/yyy/ -fprofile-sample-accurate -fprofile-sample-use=%t/pgo-cold-func.prof  -S -emit-llvm -o - %t/pgo-cold-func.c | FileCheck %s
+// RUN: %clang --target=x86_64 -O2 -fprofile-generate-cold-function-coverage=/xxx/yyy/ -fprofile-sample-accurate -fprofile-sample-use=%t/pgo-cold-func.prof  -S -emit-llvm -o - %t/pgo-cold-func.c | FileCheck %s
 
 // CHECK: @__llvm_profile_filename = {{.*}} c"/xxx/yyy/default_%m.profraw\00"
 
-- 
GitLab


From 8420dbf2b98edcaf966281912e7a2a4f7a2d6572 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen@sifive.com>
Date: Wed, 30 Oct 2024 12:22:28 +0800
Subject: [PATCH 536/693] [VPlan] Refine the constructor of
 VPWidenIntrinsicRecipe. nfc (#113890)

Infers member MayReadFromMemory, MayWriteToMemory, and
MayHaveSideEffects based on intrinsic attributes.

---------

Co-authored-by: Florian Hahn <flo@fhahn.com>
---
 llvm/lib/Transforms/Vectorize/VPlan.h           | 17 +++++++++++------
 .../Transforms/Vectorize/VPlanTransforms.cpp    |  2 +-
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 8d6025c89f72..0e0c64f6df9c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1688,13 +1688,18 @@ public:
 
   VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID,
                          ArrayRef<VPValue *> CallArguments, Type *Ty,
-                         bool MayReadFromMemory, bool MayWriteToMemory,
-                         bool MayHaveSideEffects, DebugLoc DL = {})
+                         DebugLoc DL = {})
       : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments),
-        VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
-        MayReadFromMemory(MayReadFromMemory),
-        MayWriteToMemory(MayWriteToMemory),
-        MayHaveSideEffects(MayHaveSideEffects) {}
+        VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) {
+    LLVMContext &Ctx = Ty->getContext();
+    AttributeList Attrs = Intrinsic::getAttributes(Ctx, VectorIntrinsicID);
+    MemoryEffects ME = Attrs.getMemoryEffects();
+    MayReadFromMemory = ME.onlyWritesMemory();
+    MayWriteToMemory = ME.onlyReadsMemory();
+    MayHaveSideEffects = MayWriteToMemory ||
+                         !Attrs.hasFnAttr(Attribute::NoUnwind) ||
+                         !Attrs.hasFnAttr(Attribute::WillReturn);
+  }
 
   ~VPWidenIntrinsicRecipe() override = default;
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 03c4110761ac..355781f95505 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1489,7 +1489,7 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
                 Ops.push_back(&EVL);
                 return new VPWidenIntrinsicRecipe(Intrinsic::vp_select, Ops,
                                                   TypeInfo.inferScalarType(Sel),
-                                                  false, false, false);
+                                                  Sel->getDebugLoc());
               })
 
               .Default([&](VPRecipeBase *R) { return nullptr; });
-- 
GitLab


From f672cc1ee1a4315f83f08cdca7dd2ccf099ff09c Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 29 Oct 2024 21:23:31 -0700
Subject: [PATCH 537/693] [RISCV] Add OperandType for condition code arguments
 used by select and SFB pseudos. (#114163)

---
 .../Target/RISCV/MCTargetDesc/RISCVBaseInfo.h |  4 +-
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp      |  3 +
 llvm/lib/Target/RISCV/RISCVInstrInfo.td       |  7 ++-
 llvm/lib/Target/RISCV/RISCVInstrInfoSFB.td    | 58 +++++++++----------
 llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td    |  2 +-
 5 files changed, 42 insertions(+), 32 deletions(-)

diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index e18329c3d2dd..d3899425ff84 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -335,7 +335,9 @@ enum OperandType : unsigned {
   OPERAND_FRMARG,
   // Operand is a 3-bit rounding mode where only RTZ is valid.
   OPERAND_RTZARG,
-  OPERAND_LAST_RISCV_IMM = OPERAND_RTZARG,
+  // Condition code used by select and short forward branch pseudos.
+  OPERAND_COND_CODE,
+  OPERAND_LAST_RISCV_IMM = OPERAND_COND_CODE,
   // Operand is either a register or uimm5, this is used by V extension pseudo
   // instructions to represent a value that be passed as AVL to either vsetvli
   // or vsetivli.
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 20e531657eb2..0cfe4eb06348 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -2542,6 +2542,9 @@ bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
         case RISCVOp::OPERAND_RTZARG:
           Ok = Imm == RISCVFPRndMode::RTZ;
           break;
+        case RISCVOp::OPERAND_COND_CODE:
+          Ok = Imm >= 0 && Imm < RISCVCC::COND_INVALID;
+          break;
         }
         if (!Ok) {
           ErrInfo = "Invalid immediate";
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 86cc638fd04a..a86736823558 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -387,6 +387,11 @@ def csr_sysreg : RISCVOp, TImmLeaf<XLenVT, "return isUInt<12>(Imm);"> {
 // A parameterized register class alternative to i32imm/i64imm from Target.td.
 def ixlenimm : Operand<XLenVT>;
 
+// Condition code used by select and short forward branch pseudos.
+def cond_code : RISCVOp {
+  let OperandType = "OPERAND_COND_CODE";
+}
+
 def ixlenimm_li : Operand<XLenVT> {
   let ParserMatchClass = ImmXLenAsmOperand<"", "LI">;
 }
@@ -1450,7 +1455,7 @@ def riscv_selectcc_frag : PatFrag<(ops node:$lhs, node:$rhs, node:$cc,
 multiclass SelectCC_GPR_rrirr<DAGOperand valty, ValueType vt> {
   let usesCustomInserter = 1 in
   def _Using_CC_GPR : Pseudo<(outs valty:$dst),
-                             (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                             (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                               valty:$truev, valty:$falsev),
                              [(set valty:$dst,
                                (riscv_selectcc_frag:$cc (XLenVT GPR:$lhs), GPR:$rhs, cond,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoSFB.td b/llvm/lib/Target/RISCV/RISCVInstrInfoSFB.td
index f25dc7302608..16cc0e5a61f0 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoSFB.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoSFB.td
@@ -15,7 +15,7 @@ let Predicates = [HasShortForwardBranchOpt], isSelect = 1,
 // This instruction moves $truev to $dst when the condition is true. It will
 // be expanded to control flow in RISCVExpandPseudoInsts.
 def PseudoCCMOVGPR : Pseudo<(outs GPR:$dst),
-                            (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                            (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                              GPR:$falsev, GPR:$truev),
                             [(set GPR:$dst,
                               (riscv_selectcc_frag:$cc (XLenVT GPR:$lhs),
@@ -34,7 +34,7 @@ let Predicates = [HasConditionalMoveFusion, NoShortForwardBranchOpt],
 // be expanded to control flow in RISCVExpandPseudoInsts.
 // We use GPRNoX0 because c.mv cannot encode X0.
 def PseudoCCMOVGPRNoX0 : Pseudo<(outs GPRNoX0:$dst),
-                                (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                                (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                                  GPRNoX0:$falsev, GPRNoX0:$truev),
                                 [(set GPRNoX0:$dst,
                                   (riscv_selectcc_frag:$cc (XLenVT GPR:$lhs),
@@ -51,143 +51,143 @@ def PseudoCCMOVGPRNoX0 : Pseudo<(outs GPRNoX0:$dst),
 let Predicates = [HasShortForwardBranchOpt], hasSideEffects = 0,
     mayLoad = 0, mayStore = 0, Size = 8, Constraints = "$dst = $falsev" in {
 def PseudoCCADD : Pseudo<(outs GPR:$dst),
-                         (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                         (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                           GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                   Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp,
                          ReadSFBALU, ReadSFBALU, ReadSFBALU]>;
 def PseudoCCSUB : Pseudo<(outs GPR:$dst),
-                         (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                         (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                           GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                   Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp,
                          ReadSFBALU, ReadSFBALU, ReadSFBALU]>;
 def PseudoCCSLL : Pseudo<(outs GPR:$dst),
-                         (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                         (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                           GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                   Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                          ReadSFBALU, ReadSFBALU]>;
 def PseudoCCSRL : Pseudo<(outs GPR:$dst),
-                         (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                         (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                           GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                   Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                          ReadSFBALU, ReadSFBALU]>;
 def PseudoCCSRA : Pseudo<(outs GPR:$dst),
-                         (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                         (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                           GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                   Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                          ReadSFBALU, ReadSFBALU]>;
 def PseudoCCAND : Pseudo<(outs GPR:$dst),
-                         (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                         (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                           GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                   Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp,
                          ReadSFBALU, ReadSFBALU, ReadSFBALU]>;
 def PseudoCCOR  : Pseudo<(outs GPR:$dst),
-                         (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                         (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                           GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                   Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp,
                          ReadSFBALU, ReadSFBALU, ReadSFBALU]>;
 def PseudoCCXOR : Pseudo<(outs GPR:$dst),
-                         (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                         (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                           GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                   Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp,
                          ReadSFBALU, ReadSFBALU, ReadSFBALU]>;
 
 def PseudoCCADDI : Pseudo<(outs GPR:$dst),
-                          (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                          (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                            GPR:$falsev, GPR:$rs1, simm12:$rs2), []>,
                    Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                           ReadSFBALU]>;
 def PseudoCCSLLI : Pseudo<(outs GPR:$dst),
-                          (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                          (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                            GPR:$falsev, GPR:$rs1, simm12:$rs2), []>,
                    Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                           ReadSFBALU]>;
 def PseudoCCSRLI : Pseudo<(outs GPR:$dst),
-                          (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                          (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                            GPR:$falsev, GPR:$rs1, simm12:$rs2), []>,
                    Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                           ReadSFBALU]>;
 def PseudoCCSRAI : Pseudo<(outs GPR:$dst),
-                          (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                          (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                            GPR:$falsev, GPR:$rs1, simm12:$rs2), []>,
                    Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                           ReadSFBALU]>;
 def PseudoCCANDI : Pseudo<(outs GPR:$dst),
-                          (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                          (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                            GPR:$falsev, GPR:$rs1, simm12:$rs2), []>,
                    Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                           ReadSFBALU]>;
 def PseudoCCORI  : Pseudo<(outs GPR:$dst),
-                          (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                          (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                            GPR:$falsev, GPR:$rs1, simm12:$rs2), []>,
                    Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                           ReadSFBALU]>;
 def PseudoCCXORI : Pseudo<(outs GPR:$dst),
-                          (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                          (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                            GPR:$falsev, GPR:$rs1, simm12:$rs2), []>,
                    Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                           ReadSFBALU]>;
 
 // RV64I instructions
 def PseudoCCADDW : Pseudo<(outs GPR:$dst),
-                          (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                          (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                            GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                    Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp,
                           ReadSFBALU, ReadSFBALU, ReadSFBALU]>;
 def PseudoCCSUBW : Pseudo<(outs GPR:$dst),
-                          (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                          (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                            GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                    Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp,
                           ReadSFBALU, ReadSFBALU, ReadSFBALU]>;
 def PseudoCCSLLW : Pseudo<(outs GPR:$dst),
-                          (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                          (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                            GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                    Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                           ReadSFBALU, ReadSFBALU]>;
 def PseudoCCSRLW : Pseudo<(outs GPR:$dst),
-                          (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                          (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                            GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                    Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                           ReadSFBALU, ReadSFBALU]>;
 def PseudoCCSRAW : Pseudo<(outs GPR:$dst),
-                          (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                          (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                            GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                    Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                           ReadSFBALU, ReadSFBALU]>;
 
 def PseudoCCADDIW : Pseudo<(outs GPR:$dst),
-                           (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                           (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                             GPR:$falsev, GPR:$rs1, simm12:$rs2), []>,
                     Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                            ReadSFBALU]>;
 def PseudoCCSLLIW : Pseudo<(outs GPR:$dst),
-                           (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                           (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                             GPR:$falsev, GPR:$rs1, simm12:$rs2), []>,
                     Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                            ReadSFBALU]>;
 def PseudoCCSRLIW : Pseudo<(outs GPR:$dst),
-                           (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                           (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                             GPR:$falsev, GPR:$rs1, simm12:$rs2), []>,
                     Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                            ReadSFBALU]>;
 def PseudoCCSRAIW : Pseudo<(outs GPR:$dst),
-                           (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                           (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                             GPR:$falsev, GPR:$rs1, simm12:$rs2), []>,
                     Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                            ReadSFBALU]>;
 
 // Zbb/Zbkb instructions
 def PseudoCCANDN : Pseudo<(outs GPR:$dst),
-                          (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                          (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                            GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                    Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp,
                           ReadSFBALU, ReadSFBALU, ReadSFBALU]>;
 def PseudoCCORN : Pseudo<(outs GPR:$dst),
-                         (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                         (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                           GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                   Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp,
                          ReadSFBALU, ReadSFBALU, ReadSFBALU]>;
 def PseudoCCXNOR : Pseudo<(outs GPR:$dst),
-                          (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                          (ins GPR:$lhs, GPR:$rhs, cond_code:$cc,
                            GPR:$falsev, GPR:$rs1, GPR:$rs2), []>,
                    Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp,
                           ReadSFBALU, ReadSFBALU, ReadSFBALU]>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
index b54baa16d928..4478e2461110 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
@@ -818,7 +818,7 @@ let Predicates = [HasVendorXCVbi, IsRV32], AddedComplexity = 2 in {
 
   let usesCustomInserter = 1 in
   def Select_GPR_Using_CC_Imm : Pseudo<(outs GPR:$dst),
-                             (ins GPR:$lhs, simm5:$imm5, ixlenimm:$cc,
+                             (ins GPR:$lhs, simm5:$imm5, cond_code:$cc,
                               GPR:$truev, GPR:$falsev), []>;
 
 
-- 
GitLab


From 922a0d3dfe2db7a2ef50e8cef4537fa94a7b95bb Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Wed, 30 Oct 2024 00:42:44 -0400
Subject: [PATCH 538/693] [NFC][AMDGPU][Attributor] Exit earlier if entry CC
 (#114177)

Avoid calling TTI or other stuff unnecessarily
---
 llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
index 6a69b9d2bfc7..04d3e482359a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
@@ -767,14 +767,17 @@ struct AAAMDFlatWorkGroupSize : public AAAMDSizeRangeAttribute {
 
   void initialize(Attributor &A) override {
     Function *F = getAssociatedFunction();
+
+    if (AMDGPU::isEntryFunctionCC(F->getCallingConv())) {
+      indicatePessimisticFixpoint();
+      return;
+    }
+
     auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
     unsigned MinGroupSize, MaxGroupSize;
     std::tie(MinGroupSize, MaxGroupSize) = InfoCache.getFlatWorkGroupSizes(*F);
     intersectKnown(
         ConstantRange(APInt(32, MinGroupSize), APInt(32, MaxGroupSize + 1)));
-
-    if (AMDGPU::isEntryFunctionCC(F->getCallingConv()))
-      indicatePessimisticFixpoint();
   }
 
   ChangeStatus updateImpl(Attributor &A) override {
@@ -833,6 +836,12 @@ struct AAAMDWavesPerEU : public AAAMDSizeRangeAttribute {
 
   void initialize(Attributor &A) override {
     Function *F = getAssociatedFunction();
+
+    if (AMDGPU::isEntryFunctionCC(F->getCallingConv())) {
+      indicatePessimisticFixpoint();
+      return;
+    }
+
     auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
 
     if (const auto *AssumedGroupSize = A.getAAFor<AAAMDFlatWorkGroupSize>(
@@ -847,9 +856,6 @@ struct AAAMDWavesPerEU : public AAAMDSizeRangeAttribute {
       ConstantRange Range(APInt(32, Min), APInt(32, Max + 1));
       intersectKnown(Range);
     }
-
-    if (AMDGPU::isEntryFunctionCC(F->getCallingConv()))
-      indicatePessimisticFixpoint();
   }
 
   ChangeStatus updateImpl(Attributor &A) override {
-- 
GitLab


From 9a7519fdb39f21a807189e1ed06826b43db929e1 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Wed, 30 Oct 2024 00:53:43 -0400
Subject: [PATCH 539/693] Revert "[NFC][AMDGPU][Attributor] Exit earlier if
 entry CC (#114177)"

This reverts commit 922a0d3dfe2db7a2ef50e8cef4537fa94a7b95bb.
---
 llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
index 04d3e482359a..6a69b9d2bfc7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
@@ -767,17 +767,14 @@ struct AAAMDFlatWorkGroupSize : public AAAMDSizeRangeAttribute {
 
   void initialize(Attributor &A) override {
     Function *F = getAssociatedFunction();
-
-    if (AMDGPU::isEntryFunctionCC(F->getCallingConv())) {
-      indicatePessimisticFixpoint();
-      return;
-    }
-
     auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
     unsigned MinGroupSize, MaxGroupSize;
     std::tie(MinGroupSize, MaxGroupSize) = InfoCache.getFlatWorkGroupSizes(*F);
     intersectKnown(
         ConstantRange(APInt(32, MinGroupSize), APInt(32, MaxGroupSize + 1)));
+
+    if (AMDGPU::isEntryFunctionCC(F->getCallingConv()))
+      indicatePessimisticFixpoint();
   }
 
   ChangeStatus updateImpl(Attributor &A) override {
@@ -836,12 +833,6 @@ struct AAAMDWavesPerEU : public AAAMDSizeRangeAttribute {
 
   void initialize(Attributor &A) override {
     Function *F = getAssociatedFunction();
-
-    if (AMDGPU::isEntryFunctionCC(F->getCallingConv())) {
-      indicatePessimisticFixpoint();
-      return;
-    }
-
     auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
 
     if (const auto *AssumedGroupSize = A.getAAFor<AAAMDFlatWorkGroupSize>(
@@ -856,6 +847,9 @@ struct AAAMDWavesPerEU : public AAAMDSizeRangeAttribute {
       ConstantRange Range(APInt(32, Min), APInt(32, Max + 1));
       intersectKnown(Range);
     }
+
+    if (AMDGPU::isEntryFunctionCC(F->getCallingConv()))
+      indicatePessimisticFixpoint();
   }
 
   ChangeStatus updateImpl(Attributor &A) override {
-- 
GitLab


From bb3915149a7c9b1660db9caebfc96343352e8454 Mon Sep 17 00:00:00 2001
From: Teresa Johnson <tejohnson@google.com>
Date: Tue, 29 Oct 2024 22:10:33 -0700
Subject: [PATCH 540/693] [MemProf] Support for random hotness when writing
 profile (#113998)

Add support for generating random hotness in the memprof profile writer,
to be used for testing. The random seed is printed to stderr, and an
additional option enables providing a specific seed in order to
reproduce a particular random profile.
---
 .../llvm/ProfileData/InstrProfWriter.h        | 10 ++++-
 llvm/include/llvm/ProfileData/MemProf.h       |  9 ++++
 llvm/lib/ProfileData/InstrProfWriter.cpp      | 42 +++++++++++++++++--
 llvm/test/Transforms/PGOProfile/memprof.ll    | 19 +++++++++
 llvm/tools/llvm-profdata/llvm-profdata.cpp    | 12 +++++-
 5 files changed, 86 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h
index b8b6c684717b..559549b0a22c 100644
--- a/llvm/include/llvm/ProfileData/InstrProfWriter.h
+++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h
@@ -78,12 +78,20 @@ private:
   // Whether to serialize the full schema.
   bool MemProfFullSchema;
 
+  // Whether to generated random memprof hotness for testing.
+  bool MemprofGenerateRandomHotness;
+
 public:
+  // For memprof testing, random hotness can be assigned to the contexts if
+  // MemprofGenerateRandomHotness is enabled. The random seed can be either
+  // provided by MemprofGenerateRandomHotnessSeed, or if that is 0, one will be
+  // generated in the writer using the current time.
   InstrProfWriter(
       bool Sparse = false, uint64_t TemporalProfTraceReservoirSize = 0,
       uint64_t MaxTemporalProfTraceLength = 0, bool WritePrevVersion = false,
       memprof::IndexedVersion MemProfVersionRequested = memprof::Version0,
-      bool MemProfFullSchema = false);
+      bool MemProfFullSchema = false, bool MemprofGenerateRandomHotness = false,
+      unsigned MemprofGenerateRandomHotnessSeed = 0);
   ~InstrProfWriter();
 
   StringMap<ProfilingData> &getProfileData() { return FunctionData; }
diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h
index f8121d357325..da2cc8073700 100644
--- a/llvm/include/llvm/ProfileData/MemProf.h
+++ b/llvm/include/llvm/ProfileData/MemProf.h
@@ -147,6 +147,15 @@ struct PortableMemInfoBlock {
     return Name;                                                               \
   }
 #include "llvm/ProfileData/MIBEntryDef.inc"
+#undef MIBEntryDef
+
+  // Define setters for each type which can be called by the writer.
+#define MIBEntryDef(NameTag, Name, Type)                                       \
+  void set##Name(Type NewVal) {                                                \
+    assert(Schema[llvm::to_underlying(Meta::Name)]);                           \
+    Name = NewVal;                                                             \
+  }
+#include "llvm/ProfileData/MIBEntryDef.inc"
 #undef MIBEntryDef
 
   void clear() { *this = PortableMemInfoBlock(); }
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 1a3721bf1035..f09241681b92 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/ProfileData/MemProf.h"
 #include "llvm/ProfileData/ProfileCommon.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/EndianStream.h"
@@ -184,13 +185,25 @@ public:
 InstrProfWriter::InstrProfWriter(
     bool Sparse, uint64_t TemporalProfTraceReservoirSize,
     uint64_t MaxTemporalProfTraceLength, bool WritePrevVersion,
-    memprof::IndexedVersion MemProfVersionRequested, bool MemProfFullSchema)
+    memprof::IndexedVersion MemProfVersionRequested, bool MemProfFullSchema,
+    bool MemprofGenerateRandomHotness,
+    unsigned MemprofGenerateRandomHotnessSeed)
     : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),
       TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),
       InfoObj(new InstrProfRecordWriterTrait()),
       WritePrevVersion(WritePrevVersion),
       MemProfVersionRequested(MemProfVersionRequested),
-      MemProfFullSchema(MemProfFullSchema) {}
+      MemProfFullSchema(MemProfFullSchema),
+      MemprofGenerateRandomHotness(MemprofGenerateRandomHotness) {
+  // Set up the random number seed if requested.
+  if (MemprofGenerateRandomHotness) {
+    unsigned seed = MemprofGenerateRandomHotnessSeed
+                        ? MemprofGenerateRandomHotnessSeed
+                        : std::time(nullptr);
+    errs() << "random hotness seed = " << seed << "\n";
+    std::srand(seed);
+  }
+}
 
 InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
 
@@ -273,13 +286,34 @@ void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash,
 
 void InstrProfWriter::addMemProfRecord(
     const Function::GUID Id, const memprof::IndexedMemProfRecord &Record) {
-  auto [Iter, Inserted] = MemProfData.Records.insert({Id, Record});
+  auto NewRecord = Record;
+  // Provoke random hotness values if requested. We specify the lifetime access
+  // density and lifetime length that will result in a cold or not cold hotness.
+  // See the logic in getAllocType() in Analysis/MemoryProfileInfo.cpp.
+  if (MemprofGenerateRandomHotness) {
+    for (auto &Alloc : NewRecord.AllocSites) {
+      // To get a not cold context, set the lifetime access density to the
+      // maximum value and the lifetime to 0.
+      uint64_t NewTLAD = std::numeric_limits<uint64_t>::max();
+      uint64_t NewTL = 0;
+      bool IsCold = std::rand() % 2;
+      if (IsCold) {
+        // To get a cold context, set the lifetime access density to 0 and the
+        // lifetime to the maximum value.
+        NewTLAD = 0;
+        NewTL = std::numeric_limits<uint64_t>::max();
+      }
+      Alloc.Info.setTotalLifetimeAccessDensity(NewTLAD);
+      Alloc.Info.setTotalLifetime(NewTL);
+    }
+  }
+  auto [Iter, Inserted] = MemProfData.Records.insert({Id, NewRecord});
   // If we inserted a new record then we are done.
   if (Inserted) {
     return;
   }
   memprof::IndexedMemProfRecord &Existing = Iter->second;
-  Existing.merge(Record);
+  Existing.merge(NewRecord);
 }
 
 bool InstrProfWriter::addMemProfFrame(const memprof::FrameId Id,
diff --git a/llvm/test/Transforms/PGOProfile/memprof.ll b/llvm/test/Transforms/PGOProfile/memprof.ll
index e1457ca7251e..205eeb887898 100644
--- a/llvm/test/Transforms/PGOProfile/memprof.ll
+++ b/llvm/test/Transforms/PGOProfile/memprof.ll
@@ -66,6 +66,18 @@
 ;; Check that the total sizes are reported if requested.
 ; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdata>' -pgo-warn-missing-function -S -memprof-report-hinted-sizes 2>&1 | FileCheck %s --check-prefixes=TOTALSIZES
 
+;; Make sure we emit a random hotness seed if requested.
+; RUN: llvm-profdata merge -memprof-random-hotness %S/Inputs/memprof.memprofraw --profiled-binary %S/Inputs/memprof.exe -o %t.memprofdatarand 2>&1 | FileCheck %s --check-prefix=RAND
+; RAND: random hotness seed =
+;; Can't check the exact values, but make sure applying the random profile
+;; succeeds with the same stats
+; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdatarand>' -pgo-warn-missing-function -S -stats 2>&1 | FileCheck %s --check-prefixes=ALL,MEMPROFONLY,MEMPROFSTATS
+
+;; Make sure we use a specific random hotness seed if requested.
+; RUN: llvm-profdata merge -memprof-random-hotness -memprof-random-hotness-seed=1730170724 %S/Inputs/memprof.memprofraw --profiled-binary %S/Inputs/memprof.exe -o %t.memprofdatarand2 2>&1 | FileCheck %s --check-prefix=RAND2
+; RAND2: random hotness seed = 1730170724
+; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdatarand2>' -pgo-warn-missing-function -S -stats 2>&1 | FileCheck %s --check-prefixes=MEMPROFRAND2,ALL,MEMPROFONLY,MEMPROFSTATS
+
 ; MEMPROFMATCHINFO: MemProf notcold context with id 1093248920606587996 has total profiled size 10 is matched
 ; MEMPROFMATCHINFO: MemProf notcold context with id 5725971306423925017 has total profiled size 10 is matched
 ; MEMPROFMATCHINFO: MemProf notcold context with id 6792096022461663180 has total profiled size 10 is matched
@@ -372,6 +384,13 @@ for.end:                                          ; preds = %for.cond
 ; MEMPROFNOCOLINFO: ![[C10]] = !{i64 -4535090212904553409}
 ; MEMPROFNOCOLINFO: ![[C11]] = !{i64 3577763375057267810}
 
+;; For the specific random seed, this is the expected order of hotness
+; MEMPROFRAND2: !"cold"
+; MEMPROFRAND2: !"cold"
+; MEMPROFRAND2: !"cold"
+; MEMPROFRAND2: !"hot"
+; MEMPROFRAND2: !"hot"
+
 ; MEMPROFSTATS:  8 memprof - Number of alloc contexts in memory profile.
 ; MEMPROFSTATS: 10 memprof - Number of callsites in memory profile.
 ; MEMPROFSTATS:  6 memprof - Number of functions having valid memory profile.
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 59f0f1f1fae8..f7023aa966ad 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -342,6 +342,15 @@ cl::opt<bool> MemProfFullSchema(
     "memprof-full-schema", cl::Hidden, cl::sub(MergeSubcommand),
     cl::desc("Use the full schema for serialization"), cl::init(false));
 
+static cl::opt<bool>
+    MemprofGenerateRandomHotness("memprof-random-hotness", cl::init(false),
+                                 cl::Hidden, cl::sub(MergeSubcommand),
+                                 cl::desc("Generate random hotness values"));
+static cl::opt<unsigned> MemprofGenerateRandomHotnessSeed(
+    "memprof-random-hotness-seed", cl::init(0), cl::Hidden,
+    cl::sub(MergeSubcommand),
+    cl::desc("Random hotness seed to use (0 to generate new seed)"));
+
 // Options specific to overlap subcommand.
 cl::opt<std::string> BaseFilename(cl::Positional, cl::Required,
                                   cl::desc("<base profile file>"),
@@ -641,7 +650,8 @@ struct WriterContext {
                 SmallSet<instrprof_error, 4> &WriterErrorCodes,
                 uint64_t ReservoirSize = 0, uint64_t MaxTraceLength = 0)
       : Writer(IsSparse, ReservoirSize, MaxTraceLength, DoWritePrevVersion,
-               MemProfVersionRequested, MemProfFullSchema),
+               MemProfVersionRequested, MemProfFullSchema,
+               MemprofGenerateRandomHotness, MemprofGenerateRandomHotnessSeed),
         ErrLock(ErrLock), WriterErrorCodes(WriterErrorCodes) {}
 };
 
-- 
GitLab


From 6d9fc1b84619ca22f3e70d581c87940bcfbf3a93 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 29 Oct 2024 22:14:24 -0700
Subject: [PATCH 541/693] AMDGPU: Fix producing invalid IR on vector typed
 getelementptr (#114113)

This did not consider the IR change to allow a scalar base with a vector
offset part. Reject any users that are not explicitly handled.

In this situation we could handle the vector GEP, but that is a larger
change. This just avoids the IR verifier error by rejecting it.
---
 .../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 18 ++++++--
 .../promote-alloca-invalid-vector-gep.ll      | 44 +++++++++++++++++++
 2 files changed, 58 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/promote-alloca-invalid-vector-gep.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index f8744d6a483c..7dd7388376f4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -1159,7 +1159,6 @@ bool AMDGPUPromoteAllocaImpl::collectUsesWithPtrTypes(
     if (LoadInst *LI = dyn_cast<LoadInst>(UseInst)) {
       if (LI->isVolatile())
         return false;
-
       continue;
     }
 
@@ -1170,12 +1169,19 @@ bool AMDGPUPromoteAllocaImpl::collectUsesWithPtrTypes(
       // Reject if the stored value is not the pointer operand.
       if (SI->getPointerOperand() != Val)
         return false;
-    } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(UseInst)) {
+      continue;
+    }
+
+    if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(UseInst)) {
       if (RMW->isVolatile())
         return false;
-    } else if (AtomicCmpXchgInst *CAS = dyn_cast<AtomicCmpXchgInst>(UseInst)) {
+      continue;
+    }
+
+    if (AtomicCmpXchgInst *CAS = dyn_cast<AtomicCmpXchgInst>(UseInst)) {
       if (CAS->isVolatile())
         return false;
+      continue;
     }
 
     // Only promote a select if we know that the other select operand
@@ -1186,6 +1192,7 @@ bool AMDGPUPromoteAllocaImpl::collectUsesWithPtrTypes(
 
       // May need to rewrite constant operands.
       WorkList.push_back(ICmp);
+      continue;
     }
 
     // TODO: If we know the address is only observed through flat pointers, we
@@ -1198,8 +1205,9 @@ bool AMDGPUPromoteAllocaImpl::collectUsesWithPtrTypes(
     if (isa<InsertValueInst>(User) || isa<InsertElementInst>(User))
       return false;
 
+    // TODO: Handle vectors of pointers.
     if (!User->getType()->isPointerTy())
-      continue;
+      return false;
 
     if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(UseInst)) {
       // Be conservative if an address could be computed outside the bounds of
@@ -1504,6 +1512,8 @@ bool AMDGPUPromoteAllocaImpl::tryPromoteAllocaToLDS(AllocaInst &I,
 
       PointerType *NewTy = PointerType::get(Context, AMDGPUAS::LOCAL_ADDRESS);
 
+      assert(isa<PointerType>(V->getType()));
+
       // FIXME: It doesn't really make sense to try to do this for all
       // instructions.
       V->mutateType(NewTy);
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-invalid-vector-gep.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-invalid-vector-gep.ll
new file mode 100644
index 000000000000..b0d578e421e2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-invalid-vector-gep.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-promote-alloca < %s | FileCheck %s
+
+; Check that invalid IR is not produced on a vector typed
+; getelementptr with a scalar alloca pointer base.
+
+define amdgpu_kernel void @scalar_alloca_ptr_with_vector_gep_offset() {
+; CHECK-LABEL: define amdgpu_kernel void @scalar_alloca_ptr_with_vector_gep_offset() {
+; CHECK-NEXT:  [[BB:.*:]]
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[ALLOCA]], <4 x i64> <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[EXTRACTELEMENT:%.*]] = extractelement <4 x ptr addrspace(5)> [[GETELEMENTPTR]], i64 0
+; CHECK-NEXT:    store i32 0, ptr addrspace(5) [[EXTRACTELEMENT]], align 4
+; CHECK-NEXT:    ret void
+;
+bb:
+  %alloca = alloca i32, align 4, addrspace(5)
+  %getelementptr = getelementptr inbounds i8, ptr addrspace(5) %alloca, <4 x i64> <i64 0, i64 1, i64 2, i64 3>
+  %extractelement = extractelement <4 x ptr addrspace(5)> %getelementptr, i64 0
+  store i32 0, ptr addrspace(5) %extractelement
+  ret void
+}
+
+define amdgpu_kernel void @scalar_alloca_ptr_with_vector_gep_offset_select(i1 %cond) {
+; CHECK-LABEL: define amdgpu_kernel void @scalar_alloca_ptr_with_vector_gep_offset_select(
+; CHECK-SAME: i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[BB:.*:]]
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT:    [[GETELEMENTPTR0:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[ALLOCA]], <4 x i64> <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[GETELEMENTPTR1:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[ALLOCA]], <4 x i64> <i64 3, i64 2, i64 1, i64 0>
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], <4 x ptr addrspace(5)> [[GETELEMENTPTR0]], <4 x ptr addrspace(5)> [[GETELEMENTPTR1]]
+; CHECK-NEXT:    [[EXTRACTELEMENT:%.*]] = extractelement <4 x ptr addrspace(5)> [[SELECT]], i64 1
+; CHECK-NEXT:    store i32 0, ptr addrspace(5) [[EXTRACTELEMENT]], align 4
+; CHECK-NEXT:    ret void
+;
+bb:
+  %alloca = alloca i32, align 4, addrspace(5)
+  %getelementptr0 = getelementptr inbounds i8, ptr addrspace(5) %alloca, <4 x i64> <i64 0, i64 1, i64 2, i64 3>
+  %getelementptr1 = getelementptr inbounds i8, ptr addrspace(5) %alloca, <4 x i64> <i64 3, i64 2, i64 1, i64 0>
+  %select = select i1 %cond, <4 x ptr addrspace(5)> %getelementptr0, <4 x ptr addrspace(5)> %getelementptr1
+  %extractelement = extractelement <4 x ptr addrspace(5)> %select, i64 1
+  store i32 0, ptr addrspace(5) %extractelement
+  ret void
+}
-- 
GitLab


From 62ff85f0799560b42754ef77b5f64ca2c7feeff7 Mon Sep 17 00:00:00 2001
From: Santhosh Kumar Ellendula <quic_sellendu@quicinc.com>
Date: Wed, 30 Oct 2024 10:50:59 +0530
Subject: [PATCH 542/693] [lldb-dap] Fix for missing
 'raw_string_ostream::flush' removal in ProgressEvent.cpp; addressing #108745
 (#114087)

I hope it was missed unintentionally, pushing the same for the review.
Ref: https://github.com/llvm/llvm-project/pull/108745

---------

Co-authored-by: Santhosh Kumar Ellendula <sellendu@hu-sellendu-hyd.qualcomm.com>
Co-authored-by: Santhosh Kumar Ellendula <sellendu@hu-sellendu-lv.qualcomm.com>
---
 lldb/tools/lldb-dap/ProgressEvent.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lldb/tools/lldb-dap/ProgressEvent.cpp b/lldb/tools/lldb-dap/ProgressEvent.cpp
index 8a660b50af12..0dcc2ee81001 100644
--- a/lldb/tools/lldb-dap/ProgressEvent.cpp
+++ b/lldb/tools/lldb-dap/ProgressEvent.cpp
@@ -110,7 +110,6 @@ json::Value ProgressEvent::ToJSON() const {
   std::string progress_id_str;
   llvm::raw_string_ostream progress_id_strm(progress_id_str);
   progress_id_strm << m_progress_id;
-  progress_id_strm.flush();
   body.try_emplace("progressId", progress_id_str);
 
   if (m_event_type == progressStart) {
-- 
GitLab


From e7262c15d3a2aef7cf4065e654181ab86eed24cc Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 29 Oct 2024 22:34:47 -0700
Subject: [PATCH 543/693] [RISCV] Add OperandType for sew and vecpolicy
 operands. (#114168)

---
 .../Target/RISCV/MCTargetDesc/RISCVBaseInfo.h |   6 +-
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp      |   6 +
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    | 146 +++++++++---------
 llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td    |  12 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td    |   4 +-
 5 files changed, 96 insertions(+), 78 deletions(-)

diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index d3899425ff84..b3a6cd40ea03 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -337,7 +337,11 @@ enum OperandType : unsigned {
   OPERAND_RTZARG,
   // Condition code used by select and short forward branch pseudos.
   OPERAND_COND_CODE,
-  OPERAND_LAST_RISCV_IMM = OPERAND_COND_CODE,
+  // Vector policy operand.
+  OPERAND_VEC_POLICY,
+  // Vector SEW operand.
+  OPERAND_SEW,
+  OPERAND_LAST_RISCV_IMM = OPERAND_SEW,
   // Operand is either a register or uimm5, this is used by V extension pseudo
   // instructions to represent a value that be passed as AVL to either vsetvli
   // or vsetivli.
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 0cfe4eb06348..d5b086861d71 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -2545,6 +2545,12 @@ bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
         case RISCVOp::OPERAND_COND_CODE:
           Ok = Imm >= 0 && Imm < RISCVCC::COND_INVALID;
           break;
+        case RISCVOp::OPERAND_VEC_POLICY:
+          Ok = (Imm & (RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC)) == Imm;
+          break;
+        case RISCVOp::OPERAND_SEW:
+          Ok = Imm == 0 || (Imm >= 3 && Imm <= 6);
+          break;
         }
         if (!Ok) {
           ErrInfo = "Invalid immediate";
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index af4f653f57af..6ffdae1d7df2 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -84,6 +84,14 @@ def AVL : RegisterOperand<GPRNoX0> {
   let OperandType = "OPERAND_AVL";
 }
 
+def vec_policy : RISCVOp {
+  let OperandType = "OPERAND_VEC_POLICY";
+}
+
+def sew : RISCVOp {
+  let OperandType = "OPERAND_SEW";
+}
+
 // X0 has special meaning for vsetvl/vsetvli.
 //  rd | rs1 |   AVL value | Effect on vl
 //--------------------------------------------------------------
@@ -764,8 +772,8 @@ class GetVTypePredicates<VTypeInfo vti> {
 class VPseudoUSLoadNoMask<VReg RetClass,
                           int EEW> :
       Pseudo<(outs RetClass:$rd),
-             (ins RetClass:$dest, GPRMem:$rs1, AVL:$vl, ixlenimm:$sew,
-                  ixlenimm:$policy), []>,
+             (ins RetClass:$dest, GPRMem:$rs1, AVL:$vl, sew:$sew,
+                  vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLE</*Masked*/0, /*Strided*/0, /*FF*/0, !logtwo(EEW), VLMul> {
   let mayLoad = 1;
@@ -782,7 +790,7 @@ class VPseudoUSLoadMask<VReg RetClass,
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   GPRMem:$rs1,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLE</*Masked*/1, /*Strided*/0, /*FF*/0, !logtwo(EEW), VLMul> {
   let mayLoad = 1;
@@ -799,7 +807,7 @@ class VPseudoUSLoadFFNoMask<VReg RetClass,
                             int EEW> :
       Pseudo<(outs RetClass:$rd, GPR:$vl),
              (ins RetClass:$dest, GPRMem:$rs1, AVL:$avl,
-                  ixlenimm:$sew, ixlenimm:$policy), []>,
+                  sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLE</*Masked*/0, /*Strided*/0, /*FF*/1, !logtwo(EEW), VLMul> {
   let mayLoad = 1;
@@ -816,7 +824,7 @@ class VPseudoUSLoadFFMask<VReg RetClass,
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd, GPR:$vl),
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   GPRMem:$rs1,
-                  VMaskOp:$vm, AVL:$avl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  VMaskOp:$vm, AVL:$avl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLE</*Masked*/1, /*Strided*/0, /*FF*/1, !logtwo(EEW), VLMul> {
   let mayLoad = 1;
@@ -833,7 +841,7 @@ class VPseudoSLoadNoMask<VReg RetClass,
                          int EEW> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$dest, GPRMem:$rs1, GPR:$rs2, AVL:$vl,
-                  ixlenimm:$sew, ixlenimm:$policy), []>,
+                  sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLE</*Masked*/0, /*Strided*/1, /*FF*/0, !logtwo(EEW), VLMul> {
   let mayLoad = 1;
@@ -850,7 +858,7 @@ class VPseudoSLoadMask<VReg RetClass,
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   GPRMem:$rs1, GPR:$rs2,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLE</*Masked*/1, /*Strided*/1, /*FF*/0, !logtwo(EEW), VLMul> {
   let mayLoad = 1;
@@ -872,7 +880,7 @@ class VPseudoILoadNoMask<VReg RetClass,
                          int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$dest, GPRMem:$rs1, IdxClass:$rs2, AVL:$vl,
-                  ixlenimm:$sew, ixlenimm:$policy), []>,
+                  sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLX</*Masked*/0, Ordered, !logtwo(EEW), VLMul, LMUL> {
   let mayLoad = 1;
@@ -895,7 +903,7 @@ class VPseudoILoadMask<VReg RetClass,
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   GPRMem:$rs1, IdxClass:$rs2,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLX</*Masked*/1, Ordered, !logtwo(EEW), VLMul, LMUL> {
   let mayLoad = 1;
@@ -912,7 +920,7 @@ class VPseudoILoadMask<VReg RetClass,
 class VPseudoUSStoreNoMask<VReg StClass,
                            int EEW> :
       Pseudo<(outs),
-             (ins StClass:$rd, GPRMem:$rs1, AVL:$vl, ixlenimm:$sew), []>,
+             (ins StClass:$rd, GPRMem:$rs1, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo,
       RISCVVSE</*Masked*/0, /*Strided*/0, !logtwo(EEW), VLMul> {
   let mayLoad = 0;
@@ -926,7 +934,7 @@ class VPseudoUSStoreMask<VReg StClass,
                          int EEW> :
       Pseudo<(outs),
              (ins StClass:$rd, GPRMem:$rs1,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo,
       RISCVVSE</*Masked*/1, /*Strided*/0, !logtwo(EEW), VLMul> {
   let mayLoad = 0;
@@ -940,7 +948,7 @@ class VPseudoSStoreNoMask<VReg StClass,
                           int EEW> :
       Pseudo<(outs),
              (ins StClass:$rd, GPRMem:$rs1, GPR:$rs2,
-                  AVL:$vl, ixlenimm:$sew), []>,
+                  AVL:$vl, sew:$sew), []>,
       RISCVVPseudo,
       RISCVVSE</*Masked*/0, /*Strided*/1, !logtwo(EEW), VLMul> {
   let mayLoad = 0;
@@ -954,7 +962,7 @@ class VPseudoSStoreMask<VReg StClass,
                         int EEW> :
       Pseudo<(outs),
              (ins StClass:$rd, GPRMem:$rs1, GPR:$rs2,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo,
       RISCVVSE</*Masked*/1, /*Strided*/1, !logtwo(EEW), VLMul> {
   let mayLoad = 0;
@@ -967,7 +975,7 @@ class VPseudoSStoreMask<VReg StClass,
 class VPseudoNullaryNoMask<VReg RegClass> :
       Pseudo<(outs RegClass:$rd),
              (ins RegClass:$passthru,
-                  AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -981,7 +989,7 @@ class VPseudoNullaryNoMask<VReg RegClass> :
 class VPseudoNullaryMask<VReg RegClass> :
       Pseudo<(outs GetVRegNoV0<RegClass>.R:$rd),
              (ins GetVRegNoV0<RegClass>.R:$passthru,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -996,7 +1004,7 @@ class VPseudoNullaryMask<VReg RegClass> :
 // Nullary for pseudo instructions. They are expanded in
 // RISCVExpandPseudoInsts pass.
 class VPseudoNullaryPseudoM<string BaseInst> :
-      Pseudo<(outs VR:$rd), (ins AVL:$vl, ixlenimm:$sew), []>,
+      Pseudo<(outs VR:$rd), (ins AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1016,7 +1024,7 @@ class VPseudoUnaryNoMask<DAGOperand RetClass,
                          int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru, OpClass:$rs2,
-                  AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1033,7 +1041,7 @@ class VPseudoUnaryNoMaskNoPolicy<DAGOperand RetClass,
                                  string Constraint = "",
                                  int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
-             (ins OpClass:$rs2, AVL:$vl, ixlenimm:$sew), []>,
+             (ins OpClass:$rs2, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1050,7 +1058,7 @@ class VPseudoUnaryNoMaskRoundingMode<DAGOperand RetClass,
                                      int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru, OpClass:$rs2, ixlenimm:$rm,
-                  AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1070,7 +1078,7 @@ class VPseudoUnaryMask<VReg RetClass,
                        int TargetConstraintType = 1> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru, OpClass:$rs2,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1090,7 +1098,7 @@ class VPseudoUnaryMaskRoundingMode<VReg RetClass,
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru, OpClass:$rs2,
                   VMaskOp:$vm, ixlenimm:$rm,
-                  AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1110,7 +1118,7 @@ class VPseudoUnaryMask_NoExcept<VReg RetClass,
                                 string Constraint = ""> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru, OpClass:$rs2,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []> {
+                  VMaskOp:$vm, AVL:$vl, sew:$sew, vec_policy:$policy), []> {
   let mayLoad = 0;
   let mayStore = 0;
   let hasSideEffects = 0;
@@ -1128,7 +1136,7 @@ class VPseudoUnaryNoMask_FRM<VReg RetClass,
                              int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru, OpClass:$rs2, ixlenimm:$frm,
-                  AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1148,7 +1156,7 @@ class VPseudoUnaryMask_FRM<VReg RetClass,
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru, OpClass:$rs2,
                   VMaskOp:$vm, ixlenimm:$frm,
-                  AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1164,7 +1172,7 @@ class VPseudoUnaryMask_FRM<VReg RetClass,
 
 class VPseudoUnaryNoMaskGPROut :
       Pseudo<(outs GPR:$rd),
-             (ins VR:$rs2, AVL:$vl, ixlenimm:$sew), []>,
+             (ins VR:$rs2, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1175,7 +1183,7 @@ class VPseudoUnaryNoMaskGPROut :
 
 class VPseudoUnaryMaskGPROut :
       Pseudo<(outs GPR:$rd),
-             (ins VR:$rs1, VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>,
+             (ins VR:$rs1, VMaskOp:$vm, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1189,7 +1197,7 @@ class VPseudoUnaryAnyMask<VReg RetClass,
                           VReg Op1Class> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru, Op1Class:$rs2,
-                  VR:$vm, AVL:$vl, ixlenimm:$sew), []>,
+                  VR:$vm, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1205,7 +1213,7 @@ class VPseudoBinaryNoMask<VReg RetClass,
                           string Constraint,
                           int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
-             (ins Op1Class:$rs2, Op2Class:$rs1, AVL:$vl, ixlenimm:$sew), []>,
+             (ins Op1Class:$rs2, Op2Class:$rs1, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1223,7 +1231,7 @@ class VPseudoBinaryNoMaskPolicy<VReg RetClass,
                                 int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru, Op1Class:$rs2, Op2Class:$rs1, AVL:$vl,
-                  ixlenimm:$sew, ixlenimm:$policy), []>,
+                  sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1243,7 +1251,7 @@ class VPseudoBinaryNoMaskRoundingMode<VReg RetClass,
                                       int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru, Op1Class:$rs2, Op2Class:$rs1, ixlenimm:$rm,
-                  AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1266,7 +1274,7 @@ class VPseudoBinaryMaskPolicyRoundingMode<VReg RetClass,
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   Op1Class:$rs2, Op2Class:$rs1,
                   VMaskOp:$vm, ixlenimm:$rm, AVL:$vl,
-                  ixlenimm:$sew, ixlenimm:$policy), []>,
+                  sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1288,8 +1296,8 @@ class VPseudoTiedBinaryNoMask<VReg RetClass,
                               string Constraint,
                               int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
-             (ins RetClass:$rs2, Op2Class:$rs1, AVL:$vl, ixlenimm:$sew,
-                  ixlenimm:$policy), []>,
+             (ins RetClass:$rs2, Op2Class:$rs1, AVL:$vl, sew:$sew,
+                  vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1310,8 +1318,8 @@ class VPseudoTiedBinaryNoMaskRoundingMode<VReg RetClass,
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$rs2, Op2Class:$rs1,
                   ixlenimm:$rm,
-                  AVL:$vl, ixlenimm:$sew,
-                  ixlenimm:$policy), []>,
+                  AVL:$vl, sew:$sew,
+                  vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1331,7 +1339,7 @@ class VPseudoIStoreNoMask<VReg StClass, VReg IdxClass, int EEW, bits<3> LMUL,
                           bit Ordered>:
       Pseudo<(outs),
              (ins StClass:$rd, GPRMem:$rs1, IdxClass:$rs2, AVL:$vl,
-                  ixlenimm:$sew),[]>,
+                  sew:$sew),[]>,
       RISCVVPseudo,
       RISCVVSX</*Masked*/0, Ordered, !logtwo(EEW), VLMul, LMUL> {
   let mayLoad = 0;
@@ -1345,7 +1353,7 @@ class VPseudoIStoreMask<VReg StClass, VReg IdxClass, int EEW, bits<3> LMUL,
                         bit Ordered>:
       Pseudo<(outs),
              (ins StClass:$rd, GPRMem:$rs1, IdxClass:$rs2,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew),[]>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew),[]>,
       RISCVVPseudo,
       RISCVVSX</*Masked*/1, Ordered, !logtwo(EEW), VLMul, LMUL> {
   let mayLoad = 0;
@@ -1363,7 +1371,7 @@ class VPseudoBinaryMaskPolicy<VReg RetClass,
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   Op1Class:$rs2, Op2Class:$rs1,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1382,7 +1390,7 @@ class VPseudoTernaryMaskPolicy<VReg RetClass,
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   Op1Class:$rs2, Op2Class:$rs1,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1401,7 +1409,7 @@ class VPseudoTernaryMaskPolicyRoundingMode<VReg RetClass,
                   Op1Class:$rs2, Op2Class:$rs1,
                   VMaskOp:$vm,
                   ixlenimm:$rm,
-                  AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1423,7 +1431,7 @@ class VPseudoBinaryMOutMask<VReg RetClass,
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru,
                   Op1Class:$rs2, Op2Class:$rs1,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1445,7 +1453,7 @@ class VPseudoTiedBinaryMask<VReg RetClass,
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   Op2Class:$rs1,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1468,7 +1476,7 @@ class VPseudoTiedBinaryMaskRoundingMode<VReg RetClass,
                   Op2Class:$rs1,
                   VMaskOp:$vm,
                   ixlenimm:$rm,
-                  AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1494,9 +1502,9 @@ class VPseudoBinaryCarry<VReg RetClass,
       Pseudo<(outs RetClass:$rd),
              !if(CarryIn,
                 (ins Op1Class:$rs2, Op2Class:$rs1,
-                     VMV0:$carry, AVL:$vl, ixlenimm:$sew),
+                     VMV0:$carry, AVL:$vl, sew:$sew),
                 (ins Op1Class:$rs2, Op2Class:$rs1,
-                     AVL:$vl, ixlenimm:$sew)), []>,
+                     AVL:$vl, sew:$sew)), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1515,7 +1523,7 @@ class VPseudoTiedBinaryCarryIn<VReg RetClass,
                                int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru, Op1Class:$rs2, Op2Class:$rs1,
-                  VMV0:$carry, AVL:$vl, ixlenimm:$sew), []>,
+                  VMV0:$carry, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1534,7 +1542,7 @@ class VPseudoTernaryNoMask<VReg RetClass,
                            string Constraint> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$rs3, Op1Class:$rs1, Op2Class:$rs2,
-                  AVL:$vl, ixlenimm:$sew), []>,
+                  AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1551,7 +1559,7 @@ class VPseudoTernaryNoMaskWithPolicy<VReg RetClass,
                                      int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$rs3, Op1Class:$rs1, Op2Class:$rs2,
-                  AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1570,7 +1578,7 @@ class VPseudoTernaryNoMaskWithPolicyRoundingMode<VReg RetClass,
                                                  int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$rs3, Op1Class:$rs1, Op2Class:$rs2,
-                  ixlenimm:$rm, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  ixlenimm:$rm, AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -1589,7 +1597,7 @@ class VPseudoUSSegLoadNoMask<VReg RetClass,
                              bits<4> NF> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$dest, GPRMem:$rs1, AVL:$vl,
-                  ixlenimm:$sew, ixlenimm:$policy), []>,
+                  sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLSEG<NF, /*Masked*/0, /*Strided*/0, /*FF*/0, !logtwo(EEW), VLMul> {
   let mayLoad = 1;
@@ -1606,7 +1614,7 @@ class VPseudoUSSegLoadMask<VReg RetClass,
                            bits<4> NF> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru, GPRMem:$rs1,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLSEG<NF, /*Masked*/1, /*Strided*/0, /*FF*/0, !logtwo(EEW), VLMul> {
   let mayLoad = 1;
@@ -1624,7 +1632,7 @@ class VPseudoUSSegLoadFFNoMask<VReg RetClass,
                                bits<4> NF> :
       Pseudo<(outs RetClass:$rd, GPR:$vl),
              (ins RetClass:$dest, GPRMem:$rs1, AVL:$avl,
-                  ixlenimm:$sew, ixlenimm:$policy), []>,
+                  sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLSEG<NF, /*Masked*/0, /*Strided*/0, /*FF*/1, !logtwo(EEW), VLMul> {
   let mayLoad = 1;
@@ -1641,7 +1649,7 @@ class VPseudoUSSegLoadFFMask<VReg RetClass,
                              bits<4> NF> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd, GPR:$vl),
              (ins GetVRegNoV0<RetClass>.R:$passthru, GPRMem:$rs1,
-                  VMaskOp:$vm, AVL:$avl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                  VMaskOp:$vm, AVL:$avl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLSEG<NF, /*Masked*/1, /*Strided*/0, /*FF*/1, !logtwo(EEW), VLMul> {
   let mayLoad = 1;
@@ -1659,7 +1667,7 @@ class VPseudoSSegLoadNoMask<VReg RetClass,
                             bits<4> NF> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru, GPRMem:$rs1, GPR:$offset, AVL:$vl,
-             ixlenimm:$sew, ixlenimm:$policy), []>,
+                 sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLSEG<NF, /*Masked*/0, /*Strided*/1, /*FF*/0, !logtwo(EEW), VLMul> {
   let mayLoad = 1;
@@ -1676,8 +1684,8 @@ class VPseudoSSegLoadMask<VReg RetClass,
                           bits<4> NF> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru, GPRMem:$rs1,
-                  GPR:$offset, VMaskOp:$vm, AVL:$vl, ixlenimm:$sew,
-                  ixlenimm:$policy), []>,
+                  GPR:$offset, VMaskOp:$vm, AVL:$vl, sew:$sew,
+                  vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLSEG<NF, /*Masked*/1, /*Strided*/1, /*FF*/0, !logtwo(EEW), VLMul> {
   let mayLoad = 1;
@@ -1698,7 +1706,7 @@ class VPseudoISegLoadNoMask<VReg RetClass,
                             bit Ordered> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru, GPRMem:$rs1, IdxClass:$offset, AVL:$vl,
-                  ixlenimm:$sew, ixlenimm:$policy), []>,
+                  sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLXSEG<NF, /*Masked*/0, Ordered, !logtwo(EEW), VLMul, LMUL> {
   let mayLoad = 1;
@@ -1720,8 +1728,8 @@ class VPseudoISegLoadMask<VReg RetClass,
                           bit Ordered> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru, GPRMem:$rs1,
-                  IdxClass:$offset, VMaskOp:$vm, AVL:$vl, ixlenimm:$sew,
-                  ixlenimm:$policy), []>,
+                  IdxClass:$offset, VMaskOp:$vm, AVL:$vl, sew:$sew,
+                  vec_policy:$policy), []>,
       RISCVVPseudo,
       RISCVVLXSEG<NF, /*Masked*/1, Ordered, !logtwo(EEW), VLMul, LMUL> {
   let mayLoad = 1;
@@ -1740,7 +1748,7 @@ class VPseudoUSSegStoreNoMask<VReg ValClass,
                               int EEW,
                               bits<4> NF> :
       Pseudo<(outs),
-             (ins ValClass:$rd, GPRMem:$rs1, AVL:$vl, ixlenimm:$sew), []>,
+             (ins ValClass:$rd, GPRMem:$rs1, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo,
       RISCVVSSEG<NF, /*Masked*/0, /*Strided*/0, !logtwo(EEW), VLMul> {
   let mayLoad = 0;
@@ -1755,7 +1763,7 @@ class VPseudoUSSegStoreMask<VReg ValClass,
                             bits<4> NF> :
       Pseudo<(outs),
              (ins ValClass:$rd, GPRMem:$rs1,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo,
       RISCVVSSEG<NF, /*Masked*/1, /*Strided*/0, !logtwo(EEW), VLMul> {
   let mayLoad = 0;
@@ -1770,7 +1778,7 @@ class VPseudoSSegStoreNoMask<VReg ValClass,
                              bits<4> NF> :
       Pseudo<(outs),
              (ins ValClass:$rd, GPRMem:$rs1, GPR:$offset,
-                  AVL:$vl, ixlenimm:$sew), []>,
+                  AVL:$vl, sew:$sew), []>,
       RISCVVPseudo,
       RISCVVSSEG<NF, /*Masked*/0, /*Strided*/1, !logtwo(EEW), VLMul> {
   let mayLoad = 0;
@@ -1785,7 +1793,7 @@ class VPseudoSSegStoreMask<VReg ValClass,
                            bits<4> NF> :
       Pseudo<(outs),
              (ins ValClass:$rd, GPRMem:$rs1, GPR: $offset,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo,
       RISCVVSSEG<NF, /*Masked*/1, /*Strided*/1, !logtwo(EEW), VLMul> {
   let mayLoad = 0;
@@ -1803,7 +1811,7 @@ class VPseudoISegStoreNoMask<VReg ValClass,
                              bit Ordered> :
       Pseudo<(outs),
              (ins ValClass:$rd, GPRMem:$rs1, IdxClass: $index,
-                  AVL:$vl, ixlenimm:$sew), []>,
+                  AVL:$vl, sew:$sew), []>,
       RISCVVPseudo,
       RISCVVSXSEG<NF, /*Masked*/0, Ordered, !logtwo(EEW), VLMul, LMUL> {
   let mayLoad = 0;
@@ -1821,7 +1829,7 @@ class VPseudoISegStoreMask<VReg ValClass,
                            bit Ordered> :
       Pseudo<(outs),
              (ins ValClass:$rd, GPRMem:$rs1, IdxClass: $index,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>,
+                  VMaskOp:$vm, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo,
       RISCVVSXSEG<NF, /*Masked*/1, Ordered, !logtwo(EEW), VLMul, LMUL> {
   let mayLoad = 0;
@@ -6762,13 +6770,13 @@ let Predicates = [HasVInstructions] in {
 let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in {
   let HasSEWOp = 1, BaseInstr = VMV_X_S in
   def PseudoVMV_X_S:
-    Pseudo<(outs GPR:$rd), (ins VR:$rs2, ixlenimm:$sew), []>,
+    Pseudo<(outs GPR:$rd), (ins VR:$rs2, sew:$sew), []>,
     Sched<[WriteVMovXS, ReadVMovXS]>,
     RISCVVPseudo;
   let HasVLOp = 1, HasSEWOp = 1, BaseInstr = VMV_S_X, isReMaterializable = 1,
       Constraints = "$rd = $rs1" in
   def PseudoVMV_S_X: Pseudo<(outs VR:$rd),
-                            (ins VR:$rs1, GPR:$rs2, AVL:$vl, ixlenimm:$sew),
+                            (ins VR:$rs1, GPR:$rs2, AVL:$vl, sew:$sew),
                             []>,
     Sched<[WriteVMovSX, ReadVMovSX_V, ReadVMovSX_X]>,
     RISCVVPseudo;
@@ -6785,14 +6793,14 @@ let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in {
     let HasSEWOp = 1, BaseInstr = VFMV_F_S in
     def "PseudoVFMV_" # f.FX # "_S" :
       Pseudo<(outs f.fprclass:$rd),
-             (ins VR:$rs2, ixlenimm:$sew), []>,
+             (ins VR:$rs2, sew:$sew), []>,
       Sched<[WriteVMovFS, ReadVMovFS]>,
       RISCVVPseudo;
     let HasVLOp = 1, HasSEWOp = 1, BaseInstr = VFMV_S_F, isReMaterializable = 1,
         Constraints = "$rd = $rs1" in
     def "PseudoVFMV_S_" # f.FX :
       Pseudo<(outs VR:$rd),
-             (ins VR:$rs1, f.fprclass:$rs2, AVL:$vl, ixlenimm:$sew),
+             (ins VR:$rs1, f.fprclass:$rs2, AVL:$vl, sew:$sew),
              []>,
       Sched<[WriteVMovSF, ReadVMovSF_V, ReadVMovSF_F]>,
       RISCVVPseudo;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
index 5068d0be0fb4..81467ada0044 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
@@ -230,7 +230,7 @@ let Predicates = [HasVendorXSfvfnrclipxfqf], DecoderNamespace = "XSfvfnrclipxfqf
 class VPseudoVC_X<Operand OpClass, DAGOperand RS1Class> :
       Pseudo<(outs),
              (ins OpClass:$op1, payload5:$rs2, payload5:$rd, RS1Class:$r1,
-                  AVL:$vl, ixlenimm:$sew), []>,
+                  AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -243,7 +243,7 @@ class VPseudoVC_X<Operand OpClass, DAGOperand RS1Class> :
 class VPseudoVC_XV<Operand OpClass, VReg RS2Class, DAGOperand RS1Class> :
       Pseudo<(outs),
              (ins OpClass:$op1, payload5:$rd, RS2Class:$rs2, RS1Class:$r1,
-                  AVL:$vl, ixlenimm:$sew), []>,
+                  AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -257,7 +257,7 @@ class VPseudoVC_XVV<Operand OpClass, VReg RDClass, VReg RS2Class,
                     DAGOperand RS1Class> :
       Pseudo<(outs),
              (ins OpClass:$op1, RDClass:$rd, RS2Class:$rs2, RS1Class:$r1,
-                  AVL:$vl, ixlenimm:$sew), []>,
+                  AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -270,7 +270,7 @@ class VPseudoVC_XVV<Operand OpClass, VReg RDClass, VReg RS2Class,
 class VPseudoVC_V_X<Operand OpClass, VReg RDClass, DAGOperand RS1Class> :
       Pseudo<(outs RDClass:$rd),
              (ins OpClass:$op1, payload5:$rs2, RS1Class:$r1,
-                  AVL:$vl, ixlenimm:$sew), []>,
+                  AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -284,7 +284,7 @@ class VPseudoVC_V_XV<Operand OpClass, VReg RDClass, VReg RS2Class,
                      DAGOperand RS1Class> :
       Pseudo<(outs RDClass:$rd),
              (ins OpClass:$op1, RS2Class:$rs2, RS1Class:$r1,
-                  AVL:$vl, ixlenimm:$sew), []>,
+                  AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -298,7 +298,7 @@ class VPseudoVC_V_XVV<Operand OpClass, VReg RDClass, VReg RS2Class,
                       DAGOperand RS1Class> :
       Pseudo<(outs RDClass:$rd),
              (ins OpClass:$op1, RDClass:$rs3, RS2Class:$rs2, RS1Class:$r1,
-                  AVL:$vl, ixlenimm:$sew), []>,
+                  AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index 7ec13e4eaafa..782651fd6d01 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -231,7 +231,7 @@ class ZvkMxSet<string vd_lmul> {
 
 class VPseudoBinaryNoMask_Zvk<DAGOperand RetClass, VReg OpClass> :
       Pseudo<(outs RetClass:$rd_wb),
-        (ins RetClass:$rd, OpClass:$rs2, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+        (ins RetClass:$rd, OpClass:$rs2, AVL:$vl, sew:$sew, vec_policy:$policy), []>,
         RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
@@ -248,7 +248,7 @@ class VPseudoTernaryNoMask_Zvk<VReg RetClass,
                                DAGOperand Op2Class> :
         Pseudo<(outs RetClass:$rd_wb),
                (ins RetClass:$rd, Op1Class:$rs2, Op2Class:$rs1,
-                    AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+                    AVL:$vl, sew:$sew, vec_policy:$policy), []>,
         RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
-- 
GitLab


From f1467b3f73e2849fd8349ff215cf01987fa51a9d Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Wed, 30 Oct 2024 13:59:56 +0800
Subject: [PATCH 544/693] [SDAG][NFC] Convert `SDNodeFlags` into an enumeration
 (#114167)

This patch converts `SDNodeFlags` into an enumeration as we did for
`FastMathFlags`. It simplifies the implementation and improves
compile-time. This patch is NFC since it doesn't break SDNodeFlags API.
---
 llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 159 ++++++++----------
 1 file changed, 73 insertions(+), 86 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index bda0120a2df4..26488413fe58 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -378,36 +378,48 @@ template<> struct simplify_type<SDUse> {
 /// the backend.
 struct SDNodeFlags {
 private:
-  bool NoUnsignedWrap : 1;
-  bool NoSignedWrap : 1;
-  bool Exact : 1;
-  bool Disjoint : 1;
-  bool NonNeg : 1;
-  bool NoNaNs : 1;
-  bool NoInfs : 1;
-  bool NoSignedZeros : 1;
-  bool AllowReciprocal : 1;
-  bool AllowContract : 1;
-  bool ApproximateFuncs : 1;
-  bool AllowReassociation : 1;
-
-  // We assume instructions do not raise floating-point exceptions by default,
-  // and only those marked explicitly may do so.  We could choose to represent
-  // this via a positive "FPExcept" flags like on the MI level, but having a
-  // negative "NoFPExcept" flag here makes the flag intersection logic more
-  // straightforward.
-  bool NoFPExcept : 1;
-  // Instructions with attached 'unpredictable' metadata on IR level.
-  bool Unpredictable : 1;
+  friend class SDNode;
+
+  unsigned Flags = 0;
+
+  template <unsigned Flag> void setFlag(bool B) {
+    Flags = (Flags & ~Flag) | (B ? Flag : 0);
+  }
 
 public:
+  enum : unsigned {
+    None = 0,
+    NoUnsignedWrap = 1 << 0,
+    NoSignedWrap = 1 << 1,
+    Exact = 1 << 2,
+    Disjoint = 1 << 3,
+    NonNeg = 1 << 4,
+    NoNaNs = 1 << 5,
+    NoInfs = 1 << 6,
+    NoSignedZeros = 1 << 7,
+    AllowReciprocal = 1 << 8,
+    AllowContract = 1 << 9,
+    ApproximateFuncs = 1 << 10,
+    AllowReassociation = 1 << 11,
+
+    // We assume instructions do not raise floating-point exceptions by default,
+    // and only those marked explicitly may do so.  We could choose to represent
+    // this via a positive "FPExcept" flags like on the MI level, but having a
+    // negative "NoFPExcept" flag here makes the flag intersection logic more
+    // straightforward.
+    NoFPExcept = 1 << 12,
+    // Instructions with attached 'unpredictable' metadata on IR level.
+    Unpredictable = 1 << 13,
+
+    // NOTE: Please update LargestValue in LLVM_DECLARE_ENUM_AS_BITMASK below
+    // the class definition when adding new flags.
+
+    PoisonGeneratingFlags = NoUnsignedWrap | NoSignedWrap | Exact | Disjoint |
+                            NonNeg | NoNaNs | NoInfs,
+  };
+
   /// Default constructor turns off all optimization flags.
-  SDNodeFlags()
-      : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false),
-        Disjoint(false), NonNeg(false), NoNaNs(false), NoInfs(false),
-        NoSignedZeros(false), AllowReciprocal(false), AllowContract(false),
-        ApproximateFuncs(false), AllowReassociation(false), NoFPExcept(false),
-        Unpredictable(false) {}
+  SDNodeFlags() : Flags(0) {}
 
   /// Propagate the fast-math-flags from an IR FPMathOperator.
   void copyFMF(const FPMathOperator &FPMO) {
@@ -421,71 +433,49 @@ public:
   }
 
   // These are mutators for each flag.
-  void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
-  void setNoSignedWrap(bool b) { NoSignedWrap = b; }
-  void setExact(bool b) { Exact = b; }
-  void setDisjoint(bool b) { Disjoint = b; }
-  void setNonNeg(bool b) { NonNeg = b; }
-  void setNoNaNs(bool b) { NoNaNs = b; }
-  void setNoInfs(bool b) { NoInfs = b; }
-  void setNoSignedZeros(bool b) { NoSignedZeros = b; }
-  void setAllowReciprocal(bool b) { AllowReciprocal = b; }
-  void setAllowContract(bool b) { AllowContract = b; }
-  void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
-  void setAllowReassociation(bool b) { AllowReassociation = b; }
-  void setNoFPExcept(bool b) { NoFPExcept = b; }
-  void setUnpredictable(bool b) { Unpredictable = b; }
+  void setNoUnsignedWrap(bool b) { setFlag<NoUnsignedWrap>(b); }
+  void setNoSignedWrap(bool b) { setFlag<NoSignedWrap>(b); }
+  void setExact(bool b) { setFlag<Exact>(b); }
+  void setDisjoint(bool b) { setFlag<Disjoint>(b); }
+  void setNonNeg(bool b) { setFlag<NonNeg>(b); }
+  void setNoNaNs(bool b) { setFlag<NoNaNs>(b); }
+  void setNoInfs(bool b) { setFlag<NoInfs>(b); }
+  void setNoSignedZeros(bool b) { setFlag<NoSignedZeros>(b); }
+  void setAllowReciprocal(bool b) { setFlag<AllowReciprocal>(b); }
+  void setAllowContract(bool b) { setFlag<AllowContract>(b); }
+  void setApproximateFuncs(bool b) { setFlag<ApproximateFuncs>(b); }
+  void setAllowReassociation(bool b) { setFlag<AllowReassociation>(b); }
+  void setNoFPExcept(bool b) { setFlag<NoFPExcept>(b); }
+  void setUnpredictable(bool b) { setFlag<Unpredictable>(b); }
 
   // These are accessors for each flag.
-  bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
-  bool hasNoSignedWrap() const { return NoSignedWrap; }
-  bool hasExact() const { return Exact; }
-  bool hasDisjoint() const { return Disjoint; }
-  bool hasNonNeg() const { return NonNeg; }
-  bool hasNoNaNs() const { return NoNaNs; }
-  bool hasNoInfs() const { return NoInfs; }
-  bool hasNoSignedZeros() const { return NoSignedZeros; }
-  bool hasAllowReciprocal() const { return AllowReciprocal; }
-  bool hasAllowContract() const { return AllowContract; }
-  bool hasApproximateFuncs() const { return ApproximateFuncs; }
-  bool hasAllowReassociation() const { return AllowReassociation; }
-  bool hasNoFPExcept() const { return NoFPExcept; }
-  bool hasUnpredictable() const { return Unpredictable; }
+  bool hasNoUnsignedWrap() const { return Flags & NoUnsignedWrap; }
+  bool hasNoSignedWrap() const { return Flags & NoSignedWrap; }
+  bool hasExact() const { return Flags & Exact; }
+  bool hasDisjoint() const { return Flags & Disjoint; }
+  bool hasNonNeg() const { return Flags & NonNeg; }
+  bool hasNoNaNs() const { return Flags & NoNaNs; }
+  bool hasNoInfs() const { return Flags & NoInfs; }
+  bool hasNoSignedZeros() const { return Flags & NoSignedZeros; }
+  bool hasAllowReciprocal() const { return Flags & AllowReciprocal; }
+  bool hasAllowContract() const { return Flags & AllowContract; }
+  bool hasApproximateFuncs() const { return Flags & ApproximateFuncs; }
+  bool hasAllowReassociation() const { return Flags & AllowReassociation; }
+  bool hasNoFPExcept() const { return Flags & NoFPExcept; }
+  bool hasUnpredictable() const { return Flags & Unpredictable; }
 
   bool operator==(const SDNodeFlags &Other) const {
-    return NoUnsignedWrap == Other.NoUnsignedWrap &&
-           NoSignedWrap == Other.NoSignedWrap && Exact == Other.Exact &&
-           Disjoint == Other.Disjoint && NonNeg == Other.NonNeg &&
-           NoNaNs == Other.NoNaNs && NoInfs == Other.NoInfs &&
-           NoSignedZeros == Other.NoSignedZeros &&
-           AllowReciprocal == Other.AllowReciprocal &&
-           AllowContract == Other.AllowContract &&
-           ApproximateFuncs == Other.ApproximateFuncs &&
-           AllowReassociation == Other.AllowReassociation &&
-           NoFPExcept == Other.NoFPExcept &&
-           Unpredictable == Other.Unpredictable;
+    return Flags == Other.Flags;
   }
 
   /// Clear any flags in this flag set that aren't also set in Flags. All
   /// flags will be cleared if Flags are undefined.
-  void intersectWith(const SDNodeFlags Flags) {
-    NoUnsignedWrap &= Flags.NoUnsignedWrap;
-    NoSignedWrap &= Flags.NoSignedWrap;
-    Exact &= Flags.Exact;
-    Disjoint &= Flags.Disjoint;
-    NonNeg &= Flags.NonNeg;
-    NoNaNs &= Flags.NoNaNs;
-    NoInfs &= Flags.NoInfs;
-    NoSignedZeros &= Flags.NoSignedZeros;
-    AllowReciprocal &= Flags.AllowReciprocal;
-    AllowContract &= Flags.AllowContract;
-    ApproximateFuncs &= Flags.ApproximateFuncs;
-    AllowReassociation &= Flags.AllowReassociation;
-    NoFPExcept &= Flags.NoFPExcept;
-    Unpredictable &= Flags.Unpredictable;
-  }
+  void intersectWith(const SDNodeFlags Flags) { this->Flags &= Flags.Flags; }
 };
 
+LLVM_DECLARE_ENUM_AS_BITMASK(decltype(SDNodeFlags::None),
+                             SDNodeFlags::Unpredictable);
+
 /// Represents one node in the SelectionDAG.
 ///
 class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
@@ -1029,10 +1019,7 @@ public:
   void intersectFlagsWith(const SDNodeFlags Flags);
 
   bool hasPoisonGeneratingFlags() const {
-    SDNodeFlags Flags = getFlags();
-    return Flags.hasNoUnsignedWrap() || Flags.hasNoSignedWrap() ||
-           Flags.hasExact() || Flags.hasDisjoint() || Flags.hasNonNeg() ||
-           Flags.hasNoNaNs() || Flags.hasNoInfs();
+    return Flags.Flags & SDNodeFlags::PoisonGeneratingFlags;
   }
 
   void setCFIType(uint32_t Type) { CFIType = Type; }
-- 
GitLab


From df0d249b6511289f1e8c1389f4fd33d7b4c083fa Mon Sep 17 00:00:00 2001
From: donald chen <chenxunyu1993@gmail.com>
Date: Wed, 30 Oct 2024 14:01:49 +0800
Subject: [PATCH 545/693] [mlir] [linalg] fix side effect of linalg op
 (#114045)

Linalg op need to take into account memory side effects happening inside
the region when determining their own side effects.

This patch fixed issue
https://github.com/llvm/llvm-project/issues/112881
---
 .../Dialect/Linalg/IR/LinalgStructuredOps.td    |  1 +
 mlir/test/Dialect/Linalg/canonicalize.mlir      | 17 +++++++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
index bfc609bd7081..c2fee8ea55c9 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
@@ -30,6 +30,7 @@ class LinalgStructuredBase_Op<string mnemonic, list<Trait> props>
        SingleBlockImplicitTerminator<"YieldOp">,
        DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
        DeclareOpInterfaceMethods<ConditionallySpeculatable>,
+       RecursiveMemoryEffects,
        DestinationStyleOpInterface,
        LinalgStructuredInterface,
        ReifyRankedShapedTypeOpInterface], props)> {
diff --git a/mlir/test/Dialect/Linalg/canonicalize.mlir b/mlir/test/Dialect/Linalg/canonicalize.mlir
index 4bc2ed140da9..5de007b390c5 100644
--- a/mlir/test/Dialect/Linalg/canonicalize.mlir
+++ b/mlir/test/Dialect/Linalg/canonicalize.mlir
@@ -1232,3 +1232,20 @@ func.func @transpose_buffer(%input: memref<?xf32>,
 //  CHECK-SAME:            %[[VAL_1:.*]]: memref<?xf32>) {
 //       CHECK:     linalg.transpose ins(%[[VAL_0]] : memref<?xf32>)
 //  CHECK-SAME:       outs(%[[VAL_1]] : memref<?xf32>) permutation = [0]
+
+// -----
+
+// This test checks linalg op has a recursive memory effect. Otherwise
+// linalg.map without a user would be DCEd.
+func.func @recursive_effect(%arg : tensor<1xf32>) {
+  %init = arith.constant dense<0.0> : tensor<1xf32>
+  %mapped = linalg.map ins(%arg:tensor<1xf32>) outs(%init :tensor<1xf32>)
+            (%in : f32) {
+              vector.print %in : f32
+              linalg.yield %in : f32
+            }
+  func.return
+}
+
+// CHECK-LABEL: @recursive_effect
+//       CHECK: linalg.map
-- 
GitLab


From b47e2316bf083cd2e0e5ac2ef1e9c913f839a51b Mon Sep 17 00:00:00 2001
From: Ryosuke Niwa <rniwa@webkit.org>
Date: Tue, 29 Oct 2024 23:13:23 -0700
Subject: [PATCH 546/693] [alpha.webkit.UncountedLocalVarsChecker] Warn the use
 of a raw pointer/reference when the guardian variable gets mutated. (#113859)

This checker has a notion of a guardian variable which is a variable and
keeps the object pointed to by a raw pointer / reference in an inner
scope alive long enough to "guard" it from use-after-free. But such a
guardian variable fails to flawed to keep the object alive if it ever
gets mutated within the scope of a raw pointer / reference.

This PR fixes this bug by introducing a new AST visitor class,
GuardianVisitor, which traverses the compound statements of a guarded
variable (raw pointer / reference) and looks for any operator=, move
constructor, or calls to "swap", "leakRef", or "releaseNonNull"
functions.
---
 .../WebKit/UncountedLocalVarsChecker.cpp      | 72 +++++++++++++++--
 .../Analysis/Checkers/WebKit/mock-types.h     | 34 +++++++-
 .../Checkers/WebKit/uncounted-local-vars.cpp  | 77 +++++++++++++++++++
 3 files changed, 177 insertions(+), 6 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
index 5cdf047738ab..76a4599cc8d7 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
@@ -48,6 +48,65 @@ bool isRefcountedStringsHack(const VarDecl *V) {
   return false;
 }
 
+struct GuardianVisitor : public RecursiveASTVisitor<GuardianVisitor> {
+  using Base = RecursiveASTVisitor<GuardianVisitor>;
+
+  const VarDecl *Guardian{nullptr};
+
+public:
+  explicit GuardianVisitor(const VarDecl *Guardian) : Guardian(Guardian) {
+    assert(Guardian);
+  }
+
+  bool VisitBinaryOperator(const BinaryOperator *BO) {
+    if (BO->isAssignmentOp()) {
+      if (auto *VarRef = dyn_cast<DeclRefExpr>(BO->getLHS())) {
+        if (VarRef->getDecl() == Guardian)
+          return false;
+      }
+    }
+    return true;
+  }
+
+  bool VisitCXXConstructExpr(const CXXConstructExpr *CE) {
+    if (auto *Ctor = CE->getConstructor()) {
+      if (Ctor->isMoveConstructor() && CE->getNumArgs() == 1) {
+        auto *Arg = CE->getArg(0)->IgnoreParenCasts();
+        if (auto *VarRef = dyn_cast<DeclRefExpr>(Arg)) {
+          if (VarRef->getDecl() == Guardian)
+            return false;
+        }
+      }
+    }
+    return true;
+  }
+
+  bool VisitCXXMemberCallExpr(const CXXMemberCallExpr *MCE) {
+    auto MethodName = safeGetName(MCE->getMethodDecl());
+    if (MethodName == "swap" || MethodName == "leakRef" ||
+        MethodName == "releaseNonNull") {
+      auto *ThisArg = MCE->getImplicitObjectArgument()->IgnoreParenCasts();
+      if (auto *VarRef = dyn_cast<DeclRefExpr>(ThisArg)) {
+        if (VarRef->getDecl() == Guardian)
+          return false;
+      }
+    }
+    return true;
+  }
+
+  bool VisitCXXOperatorCallExpr(const CXXOperatorCallExpr *OCE) {
+    if (OCE->isAssignmentOp()) {
+      assert(OCE->getNumArgs() == 2);
+      auto *ThisArg = OCE->getArg(0)->IgnoreParenCasts();
+      if (auto *VarRef = dyn_cast<DeclRefExpr>(ThisArg)) {
+        if (VarRef->getDecl() == Guardian)
+          return false;
+      }
+    }
+    return true;
+  }
+};
+
 bool isGuardedScopeEmbeddedInGuardianScope(const VarDecl *Guarded,
                                            const VarDecl *MaybeGuardian) {
   assert(Guarded);
@@ -81,7 +140,7 @@ bool isGuardedScopeEmbeddedInGuardianScope(const VarDecl *Guarded,
 
   // We need to skip the first CompoundStmt to avoid situation when guardian is
   // defined in the same scope as guarded variable.
-  bool HaveSkippedFirstCompoundStmt = false;
+  const CompoundStmt *FirstCompondStmt = nullptr;
   for (DynTypedNodeList guardedVarAncestors = ctx.getParents(*Guarded);
        !guardedVarAncestors.empty();
        guardedVarAncestors = ctx.getParents(
@@ -90,12 +149,15 @@ bool isGuardedScopeEmbeddedInGuardianScope(const VarDecl *Guarded,
   ) {
     for (auto &guardedVarAncestor : guardedVarAncestors) {
       if (auto *CStmtAncestor = guardedVarAncestor.get<CompoundStmt>()) {
-        if (!HaveSkippedFirstCompoundStmt) {
-          HaveSkippedFirstCompoundStmt = true;
+        if (!FirstCompondStmt) {
+          FirstCompondStmt = CStmtAncestor;
           continue;
         }
-        if (CStmtAncestor == guardiansClosestCompStmtAncestor)
-          return true;
+        if (CStmtAncestor == guardiansClosestCompStmtAncestor) {
+          GuardianVisitor guardianVisitor(MaybeGuardian);
+          auto *GuardedScope = const_cast<CompoundStmt *>(FirstCompondStmt);
+          return guardianVisitor.TraverseCompoundStmt(GuardedScope);
+        }
       }
     }
   }
diff --git a/clang/test/Analysis/Checkers/WebKit/mock-types.h b/clang/test/Analysis/Checkers/WebKit/mock-types.h
index 8d8a90f0afae..82c79c97a83d 100644
--- a/clang/test/Analysis/Checkers/WebKit/mock-types.h
+++ b/clang/test/Analysis/Checkers/WebKit/mock-types.h
@@ -49,7 +49,23 @@ template <typename T, typename PtrTraits = RawPtrTraits<T>, typename RefDerefTra
   Ref() : t{} {};
   Ref(T &t) : t(&RefDerefTraits::ref(t)) { }
   Ref(const Ref& o) : t(RefDerefTraits::refIfNotNull(PtrTraits::unwrap(o.t))) { }
+  Ref(Ref&& o) : t(o.leakRef()) { }
   ~Ref() { RefDerefTraits::derefIfNotNull(PtrTraits::exchange(t, nullptr)); }
+  Ref& operator=(T &t) {
+    Ref o(t);
+    swap(o);
+    return *this;
+  }
+  Ref& operator=(Ref &&o) {
+    Ref m(o);
+    swap(m);
+    return *this;
+  }
+  void swap(Ref& o) {
+    typename PtrTraits::StorageType tmp = t;
+    t = o.t;
+    o.t = tmp;
+  }
   T &get() { return *PtrTraits::unwrap(t); }
   T *ptr() { return PtrTraits::unwrap(t); }
   T *operator->() { return PtrTraits::unwrap(t); }
@@ -74,11 +90,27 @@ template <typename T> struct RefPtr {
     if (t)
       t->deref();
   }
+  Ref<T> releaseNonNull() {
+    Ref<T> tmp(*t);
+    if (t)
+      t->deref();
+    t = nullptr;
+    return tmp;
+  }
+  void swap(RefPtr& o) {
+    T* tmp = t;
+    t = o.t;
+    o.t = tmp;
+  }
   T *get() { return t; }
   T *operator->() { return t; }
   const T *operator->() const { return t; }
   T &operator*() { return *t; }
-  RefPtr &operator=(T *) { return *this; }
+  RefPtr &operator=(T *t) {
+    RefPtr o(t);
+    swap(o);
+    return *this;
+  }
   operator bool() const { return t; }
 };
 
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
index 1c0df42cdda6..d7fb689557a6 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
@@ -83,6 +83,83 @@ void foo7(RefCountable* obj) {
   bar.obj->method();
 }
 
+void foo8(RefCountable* obj) {
+  RefPtr<RefCountable> foo;
+  {
+    RefCountable *bar = foo.get();
+    // expected-warning@-1{{Local variable 'bar' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+    foo = nullptr;
+    bar->method();
+  }
+  RefPtr<RefCountable> baz;
+  {
+    RefCountable *bar = baz.get();
+    // expected-warning@-1{{Local variable 'bar' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+    baz = obj;
+    bar->method();
+  }
+  foo = nullptr;
+  {
+    RefCountable *bar = foo.get();
+    // No warning. It's okay to mutate RefPtr in an outer scope.
+    bar->method();
+  }
+  foo = obj;
+  {
+    RefCountable *bar = foo.get();
+    // expected-warning@-1{{Local variable 'bar' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+    foo.releaseNonNull();
+    bar->method();
+  }
+  {
+    RefCountable *bar = foo.get();
+    // expected-warning@-1{{Local variable 'bar' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+    foo = obj ? obj : nullptr;
+    bar->method();
+  }
+  {
+    RefCountable *bar = foo->trivial() ? foo.get() : nullptr;
+    // expected-warning@-1{{Local variable 'bar' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+    foo = nullptr;
+    bar->method();
+  }
+}
+
+void foo9(RefCountable& o) {
+  Ref<RefCountable> guardian(o);
+  {
+    RefCountable &bar = guardian.get();
+    // expected-warning@-1{{Local variable 'bar' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+    guardian = o; // We don't detect that we're setting it to the same value.
+    bar.method();
+  }
+  {
+    RefCountable *bar = guardian.ptr();
+    // expected-warning@-1{{Local variable 'bar' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+    Ref<RefCountable> other(*bar); // We don't detect other has the same value as guardian.
+    guardian.swap(other);
+    bar->method();
+  }
+  {
+    RefCountable *bar = guardian.ptr();
+    // expected-warning@-1{{Local variable 'bar' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+    Ref<RefCountable> other(static_cast<Ref<RefCountable>&&>(guardian));
+    bar->method();
+  }
+  {
+    RefCountable *bar = guardian.ptr();
+    // expected-warning@-1{{Local variable 'bar' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+    guardian.leakRef();
+    bar->method();
+  }
+  {
+    RefCountable *bar = guardian.ptr();
+    // expected-warning@-1{{Local variable 'bar' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+    guardian = o.trivial() ? o : *bar;
+    bar->method();
+  }
+}
+
 } // namespace guardian_scopes
 
 namespace auto_keyword {
-- 
GitLab


From 44d0e9522a80e1301e96c4751b7572ae0c9cb4dd Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke@amd.com>
Date: Wed, 30 Oct 2024 11:48:40 +0530
Subject: [PATCH 547/693] [CodeGen][NewPM] Port TailDuplicate pass to NPM
 (#113293)

---
 llvm/include/llvm/CodeGen/Passes.h            |  4 +-
 llvm/include/llvm/CodeGen/TailDuplication.h   | 47 +++++++++++++
 llvm/include/llvm/InitializePasses.h          |  4 +-
 llvm/include/llvm/Passes/CodeGenPassBuilder.h |  1 +
 .../llvm/Passes/MachinePassRegistry.def       |  4 +-
 llvm/lib/CodeGen/CodeGen.cpp                  |  4 +-
 llvm/lib/CodeGen/TailDuplication.cpp          | 69 ++++++++++++++-----
 llvm/lib/CodeGen/TargetPassConfig.cpp         |  8 +--
 llvm/lib/Passes/PassBuilder.cpp               |  1 +
 llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp  |  4 +-
 .../CodeGen/AArch64/jump-table-duplicate.mir  |  1 +
 .../AMDGPU/early-tailduplicator-nophis.mir    |  1 +
 .../early-tailduplicator-terminator.mir       |  1 +
 .../stop-tail-duplicate-cfg-intrinsic.mir     |  1 +
 14 files changed, 120 insertions(+), 30 deletions(-)
 create mode 100644 llvm/include/llvm/CodeGen/TailDuplication.h

diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index e12c1f076f13..d1c71fc95818 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -261,11 +261,11 @@ namespace llvm {
 
   /// TailDuplicate - Duplicate blocks with unconditional branches
   /// into tails of their predecessors.
-  extern char &TailDuplicateID;
+  extern char &TailDuplicateLegacyID;
 
   /// Duplicate blocks with unconditional branches into tails of their
   /// predecessors. Variant that works before register allocation.
-  extern char &EarlyTailDuplicateID;
+  extern char &EarlyTailDuplicateLegacyID;
 
   /// MachineTraceMetrics - This pass computes critical path and CPU resource
   /// usage in an ensemble of traces.
diff --git a/llvm/include/llvm/CodeGen/TailDuplication.h b/llvm/include/llvm/CodeGen/TailDuplication.h
new file mode 100644
index 000000000000..687a592ccf2f
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/TailDuplication.h
@@ -0,0 +1,47 @@
+//===- llvm/CodeGen/TailDuplication.h ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_TAILDUPLICATIONPASS_H
+#define LLVM_CODEGEN_TAILDUPLICATIONPASS_H
+
+#include "llvm/CodeGen/MBFIWrapper.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+
+template <typename DerivedT, bool PreRegAlloc>
+class TailDuplicatePassBase : public PassInfoMixin<DerivedT> {
+private:
+  std::unique_ptr<MBFIWrapper> MBFIW;
+
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+};
+
+class EarlyTailDuplicatePass
+    : public TailDuplicatePassBase<EarlyTailDuplicatePass, true> {
+public:
+  MachineFunctionProperties getClearedProperties() const {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::NoPHIs);
+  }
+};
+
+class TailDuplicatePass
+    : public TailDuplicatePassBase<TailDuplicatePass, false> {};
+
+} // namespace llvm
+
+extern template class llvm::TailDuplicatePassBase<llvm::EarlyTailDuplicatePass,
+                                                  true>;
+extern template class llvm::TailDuplicatePassBase<llvm::TailDuplicatePass,
+                                                  false>;
+
+#endif // LLVM_CODEGEN_TAILDUPLICATIONPASS_H
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 26f5d63553c5..54c070401ec8 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -101,7 +101,7 @@ void initializeEarlyCSEMemSSALegacyPassPass(PassRegistry &);
 void initializeEarlyIfConverterLegacyPass(PassRegistry &);
 void initializeEarlyIfPredicatorPass(PassRegistry &);
 void initializeEarlyMachineLICMPass(PassRegistry &);
-void initializeEarlyTailDuplicatePass(PassRegistry &);
+void initializeEarlyTailDuplicateLegacyPass(PassRegistry &);
 void initializeEdgeBundlesPass(PassRegistry &);
 void initializeEHContGuardCatchretPass(PassRegistry &);
 void initializeExpandLargeFpConvertLegacyPassPass(PassRegistry &);
@@ -300,7 +300,7 @@ void initializeStraightLineStrengthReduceLegacyPassPass(PassRegistry &);
 void initializeStripDebugMachineModulePass(PassRegistry &);
 void initializeStructurizeCFGLegacyPassPass(PassRegistry &);
 void initializeTailCallElimPass(PassRegistry &);
-void initializeTailDuplicatePass(PassRegistry &);
+void initializeTailDuplicateLegacyPass(PassRegistry &);
 void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &);
 void initializeTargetPassConfigPass(PassRegistry &);
 void initializeTargetTransformInfoWrapperPassPass(PassRegistry &);
diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
index ad80c661147d..9e95625fd1d8 100644
--- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
@@ -60,6 +60,7 @@
 #include "llvm/CodeGen/SjLjEHPrepare.h"
 #include "llvm/CodeGen/StackColoring.h"
 #include "llvm/CodeGen/StackProtector.h"
+#include "llvm/CodeGen/TailDuplication.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TwoAddressInstructionPass.h"
 #include "llvm/CodeGen/UnreachableBlockElim.h"
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 4f32a917738c..9d12a120ff7a 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -133,6 +133,7 @@ MACHINE_FUNCTION_ANALYSIS("virtregmap", VirtRegMapAnalysis())
 MACHINE_FUNCTION_PASS("dead-mi-elimination", DeadMachineInstructionElimPass())
 MACHINE_FUNCTION_PASS("early-ifcvt", EarlyIfConverterPass())
 MACHINE_FUNCTION_PASS("early-machinelicm", EarlyMachineLICMPass())
+MACHINE_FUNCTION_PASS("early-tailduplication", EarlyTailDuplicatePass())
 MACHINE_FUNCTION_PASS("finalize-isel", FinalizeISelPass())
 MACHINE_FUNCTION_PASS("localstackalloc", LocalStackSlotAllocationPass())
 MACHINE_FUNCTION_PASS("machine-cse", MachineCSEPass())
@@ -157,6 +158,7 @@ MACHINE_FUNCTION_PASS("print<virtregmap>", VirtRegMapPrinterPass(dbgs()))
 MACHINE_FUNCTION_PASS("require-all-machine-function-properties",
                       RequireAllMachineFunctionPropertiesPass())
 MACHINE_FUNCTION_PASS("stack-coloring", StackColoringPass())
+MACHINE_FUNCTION_PASS("tailduplication", TailDuplicatePass())
 MACHINE_FUNCTION_PASS("trigger-verifier-error", TriggerVerifierErrorPass())
 MACHINE_FUNCTION_PASS("two-address-instruction", TwoAddressInstructionPass())
 MACHINE_FUNCTION_PASS("verify", MachineVerifierPass())
@@ -210,7 +212,6 @@ DUMMY_MACHINE_FUNCTION_PASS("cfi-fixup", CFIFixupPass)
 DUMMY_MACHINE_FUNCTION_PASS("cfi-instr-inserter", CFIInstrInserterPass)
 DUMMY_MACHINE_FUNCTION_PASS("detect-dead-lanes", DetectDeadLanesPass)
 DUMMY_MACHINE_FUNCTION_PASS("dot-machine-cfg", MachineCFGPrinter)
-DUMMY_MACHINE_FUNCTION_PASS("early-tailduplication", EarlyTailDuplicatePass)
 DUMMY_MACHINE_FUNCTION_PASS("fentry-insert", FEntryInserterPass)
 DUMMY_MACHINE_FUNCTION_PASS("fixup-statepoint-caller-saved", FixupStatepointCallerSavedPass)
 DUMMY_MACHINE_FUNCTION_PASS("fs-profile-loader", MIRProfileLoaderNewPass)
@@ -262,7 +263,6 @@ DUMMY_MACHINE_FUNCTION_PASS("simple-register-coalescing", RegisterCoalescerPass)
 DUMMY_MACHINE_FUNCTION_PASS("stack-frame-layout", StackFrameLayoutAnalysisPass)
 DUMMY_MACHINE_FUNCTION_PASS("stack-slot-coloring", StackSlotColoringPass)
 DUMMY_MACHINE_FUNCTION_PASS("stackmap-liveness", StackMapLivenessPass)
-DUMMY_MACHINE_FUNCTION_PASS("tailduplication", TailDuplicatePass)
 DUMMY_MACHINE_FUNCTION_PASS("unpack-mi-bundles", UnpackMachineBundlesPass)
 DUMMY_MACHINE_FUNCTION_PASS("virtregrewriter", VirtRegRewriterPass)
 DUMMY_MACHINE_FUNCTION_PASS("xray-instrumentation", XRayInstrumentationPass)
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index cf5c35fe81b4..39fba1d0b527 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -38,7 +38,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeEarlyIfConverterLegacyPass(Registry);
   initializeEarlyIfPredicatorPass(Registry);
   initializeEarlyMachineLICMPass(Registry);
-  initializeEarlyTailDuplicatePass(Registry);
+  initializeEarlyTailDuplicateLegacyPass(Registry);
   initializeExpandLargeDivRemLegacyPassPass(Registry);
   initializeExpandLargeFpConvertLegacyPassPass(Registry);
   initializeExpandMemCmpLegacyPassPass(Registry);
@@ -131,7 +131,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeStackProtectorPass(Registry);
   initializeStackSlotColoringPass(Registry);
   initializeStripDebugMachineModulePass(Registry);
-  initializeTailDuplicatePass(Registry);
+  initializeTailDuplicateLegacyPass(Registry);
   initializeTargetPassConfigPass(Registry);
   initializeTwoAddressInstructionLegacyPassPass(Registry);
   initializeTypePromotionLegacyPass(Registry);
diff --git a/llvm/lib/CodeGen/TailDuplication.cpp b/llvm/lib/CodeGen/TailDuplication.cpp
index 25f20d9c899b..b698ca675b65 100644
--- a/llvm/lib/CodeGen/TailDuplication.cpp
+++ b/llvm/lib/CodeGen/TailDuplication.cpp
@@ -12,13 +12,16 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/CodeGen/TailDuplication.h"
 #include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/CodeGen/LazyMachineBlockFrequencyInfo.h"
 #include "llvm/CodeGen/MBFIWrapper.h"
 #include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/CodeGen/TailDuplicator.h"
+#include "llvm/IR/Analysis.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/PassRegistry.h"
@@ -29,13 +32,13 @@ using namespace llvm;
 
 namespace {
 
-class TailDuplicateBase : public MachineFunctionPass {
+class TailDuplicateBaseLegacy : public MachineFunctionPass {
   TailDuplicator Duplicator;
   std::unique_ptr<MBFIWrapper> MBFIW;
   bool PreRegAlloc;
 public:
-  TailDuplicateBase(char &PassID, bool PreRegAlloc)
-    : MachineFunctionPass(PassID), PreRegAlloc(PreRegAlloc) {}
+  TailDuplicateBaseLegacy(char &PassID, bool PreRegAlloc)
+      : MachineFunctionPass(PassID), PreRegAlloc(PreRegAlloc) {}
 
   bool runOnMachineFunction(MachineFunction &MF) override;
 
@@ -47,19 +50,19 @@ public:
   }
 };
 
-class TailDuplicate : public TailDuplicateBase {
+class TailDuplicateLegacy : public TailDuplicateBaseLegacy {
 public:
   static char ID;
-  TailDuplicate() : TailDuplicateBase(ID, false) {
-    initializeTailDuplicatePass(*PassRegistry::getPassRegistry());
+  TailDuplicateLegacy() : TailDuplicateBaseLegacy(ID, false) {
+    initializeTailDuplicateLegacyPass(*PassRegistry::getPassRegistry());
   }
 };
 
-class EarlyTailDuplicate : public TailDuplicateBase {
+class EarlyTailDuplicateLegacy : public TailDuplicateBaseLegacy {
 public:
   static char ID;
-  EarlyTailDuplicate() : TailDuplicateBase(ID, true) {
-    initializeEarlyTailDuplicatePass(*PassRegistry::getPassRegistry());
+  EarlyTailDuplicateLegacy() : TailDuplicateBaseLegacy(ID, true) {
+    initializeEarlyTailDuplicateLegacyPass(*PassRegistry::getPassRegistry());
   }
 
   MachineFunctionProperties getClearedProperties() const override {
@@ -70,17 +73,18 @@ public:
 
 } // end anonymous namespace
 
-char TailDuplicate::ID;
-char EarlyTailDuplicate::ID;
+char TailDuplicateLegacy::ID;
+char EarlyTailDuplicateLegacy::ID;
 
-char &llvm::TailDuplicateID = TailDuplicate::ID;
-char &llvm::EarlyTailDuplicateID = EarlyTailDuplicate::ID;
+char &llvm::TailDuplicateLegacyID = TailDuplicateLegacy::ID;
+char &llvm::EarlyTailDuplicateLegacyID = EarlyTailDuplicateLegacy::ID;
 
-INITIALIZE_PASS(TailDuplicate, DEBUG_TYPE, "Tail Duplication", false, false)
-INITIALIZE_PASS(EarlyTailDuplicate, "early-tailduplication",
+INITIALIZE_PASS(TailDuplicateLegacy, DEBUG_TYPE, "Tail Duplication", false,
+                false)
+INITIALIZE_PASS(EarlyTailDuplicateLegacy, "early-tailduplication",
                 "Early Tail Duplication", false, false)
 
-bool TailDuplicateBase::runOnMachineFunction(MachineFunction &MF) {
+bool TailDuplicateBaseLegacy::runOnMachineFunction(MachineFunction &MF) {
   if (skipFunction(MF.getFunction()))
     return false;
 
@@ -100,3 +104,36 @@ bool TailDuplicateBase::runOnMachineFunction(MachineFunction &MF) {
 
   return MadeChange;
 }
+
+template <typename DerivedT, bool PreRegAlloc>
+PreservedAnalyses TailDuplicatePassBase<DerivedT, PreRegAlloc>::run(
+    MachineFunction &MF, MachineFunctionAnalysisManager &MFAM) {
+  MFPropsModifier _(static_cast<DerivedT &>(*this), MF);
+
+  if (MF.getFunction().hasOptNone())
+    return PreservedAnalyses::all();
+
+  auto *MBPI = &MFAM.getResult<MachineBranchProbabilityAnalysis>(MF);
+  auto *PSI = MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(MF)
+                  .getCachedResult<ProfileSummaryAnalysis>(
+                      *MF.getFunction().getParent());
+  auto *MBFI = (PSI && PSI->hasProfileSummary()
+                    ? &MFAM.getResult<MachineBlockFrequencyAnalysis>(MF)
+                    : nullptr);
+  if (MBFI)
+    MBFIW = std::make_unique<MBFIWrapper>(*MBFI);
+
+  TailDuplicator Duplicator;
+  Duplicator.initMF(MF, PreRegAlloc, MBPI, MBFI ? MBFIW.get() : nullptr, PSI,
+                    /*LayoutMode=*/false);
+  bool MadeChange = false;
+  while (Duplicator.tailDuplicateBlocks())
+    MadeChange = true;
+
+  if (!MadeChange)
+    return PreservedAnalyses::all();
+  return getMachineFunctionPassPreservedAnalyses();
+}
+
+template class llvm::TailDuplicatePassBase<TailDuplicatePass, false>;
+template class llvm::TailDuplicatePassBase<EarlyTailDuplicatePass, true>;
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 12225c9946e9..aff74104006e 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -290,10 +290,10 @@ static IdentifyingPassPtr overridePass(AnalysisID StandardID,
   if (StandardID == &BranchFolderPassID)
     return applyDisable(TargetID, DisableBranchFold);
 
-  if (StandardID == &TailDuplicateID)
+  if (StandardID == &TailDuplicateLegacyID)
     return applyDisable(TargetID, DisableTailDuplicate);
 
-  if (StandardID == &EarlyTailDuplicateID)
+  if (StandardID == &EarlyTailDuplicateLegacyID)
     return applyDisable(TargetID, DisableEarlyTailDup);
 
   if (StandardID == &MachineBlockPlacementID)
@@ -1279,7 +1279,7 @@ void TargetPassConfig::addMachinePasses() {
 /// Add passes that optimize machine instructions in SSA form.
 void TargetPassConfig::addMachineSSAOptimization() {
   // Pre-ra tail duplication.
-  addPass(&EarlyTailDuplicateID);
+  addPass(&EarlyTailDuplicateLegacyID);
 
   // Optimize PHIs before DCE: removing dead PHI cycles may make more
   // instructions dead.
@@ -1507,7 +1507,7 @@ void TargetPassConfig::addMachineLateOptimization() {
   // performance for targets that require Structured Control Flow.
   // In addition it can also make CFG irreducible. Thus we disable it.
   if (!TM->requiresStructuredCFG())
-    addPass(&TailDuplicateID);
+    addPass(&TailDuplicateLegacyID);
 
   // Copy propagation.
   addPass(&MachineCopyPropagationID);
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index d1f75dfb5350..a879918005ca 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -125,6 +125,7 @@
 #include "llvm/CodeGen/SlotIndexes.h"
 #include "llvm/CodeGen/StackColoring.h"
 #include "llvm/CodeGen/StackProtector.h"
+#include "llvm/CodeGen/TailDuplication.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TwoAddressInstructionPass.h"
 #include "llvm/CodeGen/TypePromotion.h"
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
index 1d6f39b29053..a5a147da8da1 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
@@ -332,7 +332,7 @@ void NVPTXPassConfig::addIRPasses() {
   disablePass(&PrologEpilogCodeInserterID);
   disablePass(&MachineLateInstrsCleanupID);
   disablePass(&MachineCopyPropagationID);
-  disablePass(&TailDuplicateID);
+  disablePass(&TailDuplicateLegacyID);
   disablePass(&StackMapLivenessID);
   disablePass(&PostRAMachineSinkingID);
   disablePass(&PostRASchedulerID);
@@ -461,7 +461,7 @@ void NVPTXPassConfig::addOptimizedRegAlloc() {
 
 void NVPTXPassConfig::addMachineSSAOptimization() {
   // Pre-ra tail duplication.
-  if (addPass(&EarlyTailDuplicateID))
+  if (addPass(&EarlyTailDuplicateLegacyID))
     printAndVerify("After Pre-RegAlloc TailDuplicate");
 
   // Optimize PHIs before DCE: removing dead PHI cycles may make more
diff --git a/llvm/test/CodeGen/AArch64/jump-table-duplicate.mir b/llvm/test/CodeGen/AArch64/jump-table-duplicate.mir
index 0963ecbb1231..a2532a854923 100644
--- a/llvm/test/CodeGen/AArch64/jump-table-duplicate.mir
+++ b/llvm/test/CodeGen/AArch64/jump-table-duplicate.mir
@@ -1,4 +1,5 @@
 # RUN: llc -run-pass=tailduplication -tail-dup-size=4 %s -o - | FileCheck %s
+# RUN: llc -passes=tailduplication -tail-dup-size=4 %s -o - | FileCheck %s
 
 # JumpTableDest32 uses an `adr` to a temporary label (itself). If duplicated we
 # cannot guarantee reachability for any uses after the first.
diff --git a/llvm/test/CodeGen/AMDGPU/early-tailduplicator-nophis.mir b/llvm/test/CodeGen/AMDGPU/early-tailduplicator-nophis.mir
index 2cb84c7ef463..072cc3a60a60 100644
--- a/llvm/test/CodeGen/AMDGPU/early-tailduplicator-nophis.mir
+++ b/llvm/test/CodeGen/AMDGPU/early-tailduplicator-nophis.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=early-tailduplication -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=early-tailduplication -o - %s | FileCheck %s
 
  # There are no phis in this testcase. Early tail duplication introduces them,
  # so the NoPHIs property needs to be cleared to avoid verifier errors
diff --git a/llvm/test/CodeGen/AMDGPU/early-tailduplicator-terminator.mir b/llvm/test/CodeGen/AMDGPU/early-tailduplicator-terminator.mir
index 41c6906b3c85..8132fa4df89e 100644
--- a/llvm/test/CodeGen/AMDGPU/early-tailduplicator-terminator.mir
+++ b/llvm/test/CodeGen/AMDGPU/early-tailduplicator-terminator.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=early-tailduplication -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=early-tailduplication -o - %s | FileCheck %s
 
 # Early tail duplication should not merge bb.6 into bb.5, adding a
 # non-terminator (S_SLEEP) after the terminator S_MOV_B32_term.
diff --git a/llvm/test/CodeGen/AMDGPU/stop-tail-duplicate-cfg-intrinsic.mir b/llvm/test/CodeGen/AMDGPU/stop-tail-duplicate-cfg-intrinsic.mir
index c23c8900096f..be1a8aceb8c9 100644
--- a/llvm/test/CodeGen/AMDGPU/stop-tail-duplicate-cfg-intrinsic.mir
+++ b/llvm/test/CodeGen/AMDGPU/stop-tail-duplicate-cfg-intrinsic.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-amd-amdhsa -run-pass=early-tailduplication -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -passes=early-tailduplication -o - %s | FileCheck %s
 
 ---
 name:            stop_duplicate_cfg_intrinsic
-- 
GitLab


From cad09404cc804dd35d2f3b742d1d6efb6d5a9449 Mon Sep 17 00:00:00 2001
From: Congcong Cai <congcongcai0907@163.com>
Date: Wed, 30 Oct 2024 14:34:19 +0800
Subject: [PATCH 548/693] [sema] enhance error handling for compound stmt body
 in `StmtExpr` (#113760)

Mark the whole StmtExpr invalid when the last statement in compound
statement is invalid.
Because the last statement need to do copy initialization, it causes
subsequent errors to simply ignore last invalid statement.

Fixed: #113468
---
 clang/docs/ReleaseNotes.rst     |  1 +
 clang/lib/Parse/ParseStmt.cpp   |  9 +++++++++
 clang/test/SemaCXX/gh113468.cpp | 12 ++++++++++++
 3 files changed, 22 insertions(+)
 create mode 100644 clang/test/SemaCXX/gh113468.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index a39ffc8366dd..1837707b8cae 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -470,6 +470,7 @@ Bug Fixes in This Version
 - The warning emitted for an unsupported register variable type now points to
   the unsupported type instead of the ``register`` keyword (#GH109776).
 - Fixed a crash when emit ctor for global variant with flexible array init  (#GH113187).
+- Fixed a crash when GNU statement expression contains invalid statement (#GH113468).
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index 7d727efb2287..6470e55e521a 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -1243,6 +1243,7 @@ StmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
       ParsedStmtContext::Compound |
       (isStmtExpr ? ParsedStmtContext::InStmtExpr : ParsedStmtContext());
 
+  bool LastIsError = false;
   while (!tryParseMisplacedModuleImport() && Tok.isNot(tok::r_brace) &&
          Tok.isNot(tok::eof)) {
     if (Tok.is(tok::annot_pragma_unused)) {
@@ -1299,7 +1300,15 @@ StmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
 
     if (R.isUsable())
       Stmts.push_back(R.get());
+    LastIsError = R.isInvalid();
   }
+  // StmtExpr needs to do copy initialization for last statement.
+  // If last statement is invalid, the last statement in `Stmts` will be
+  // incorrect. Then the whole compound statement should also be marked as
+  // invalid to prevent subsequent errors.
+  if (isStmtExpr && LastIsError && !Stmts.empty())
+    return StmtError();
+
   // Warn the user that using option `-ffp-eval-method=source` on a
   // 32-bit target and feature `sse` disabled, or using
   // `pragma clang fp eval_method=source` and feature `sse` disabled, is not
diff --git a/clang/test/SemaCXX/gh113468.cpp b/clang/test/SemaCXX/gh113468.cpp
new file mode 100644
index 000000000000..94551986b0ef
--- /dev/null
+++ b/clang/test/SemaCXX/gh113468.cpp
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify %s
+
+constexpr int expr() {
+  if (({
+        int f;
+        f = 0;
+        if (f)
+          break; // expected-error {{'break' statement not in loop or switch statement}}
+      }))
+    return 2;
+  return 1;
+}
-- 
GitLab


From 5df84a75351d0e9c3e20d50ac1047c937e3b8e88 Mon Sep 17 00:00:00 2001
From: Congcong Cai <congcongcai0907@163.com>
Date: Wed, 30 Oct 2024 14:37:04 +0800
Subject: [PATCH 549/693] [NFC] clean space in clang release note (#114188)

---
 clang/docs/ReleaseNotes.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 1837707b8cae..6085352dfafe 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -140,7 +140,7 @@ C++ Specific Potentially Breaking Changes
     unsigned operator""_udl_name(unsigned long long);
 
 - Clang will now produce an error diagnostic when [[clang::lifetimebound]] is
-  applied on a parameter of a function that returns void. This was previously 
+  applied on a parameter of a function that returns void. This was previously
   ignored and had no effect. (#GH107556)
 
   .. code-block:: c++
@@ -469,7 +469,7 @@ Bug Fixes in This Version
 - Fixed a crash using ``__array_rank`` on 64-bit targets. (#GH113044).
 - The warning emitted for an unsupported register variable type now points to
   the unsupported type instead of the ``register`` keyword (#GH109776).
-- Fixed a crash when emit ctor for global variant with flexible array init  (#GH113187).
+- Fixed a crash when emit ctor for global variant with flexible array init (#GH113187).
 - Fixed a crash when GNU statement expression contains invalid statement (#GH113468).
 
 Bug Fixes to Compiler Builtins
-- 
GitLab


From dc56a86b96d77a93f761995d50f7b2f112856311 Mon Sep 17 00:00:00 2001
From: serge-sans-paille <sguelton@mozilla.com>
Date: Wed, 30 Oct 2024 07:32:05 +0000
Subject: [PATCH 550/693] [clang] Fix 71315698c9 in presence of incomplete
 types (#114095)

Incomplete types are not considered trivially copyable by clang but we
don't want to warn about invalid argument for memcpy / memset in that
case because we cannot prove they are not Trivially Copyable.
---
 clang/lib/Sema/SemaChecking.cpp         | 11 ++++++++---
 clang/test/SemaCXX/constexpr-string.cpp |  2 --
 clang/test/SemaCXX/warn-memaccess.cpp   | 25 +++++++++++++++++++++----
 3 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 3308b898a5b6..dae271c1ff50 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -8900,7 +8900,12 @@ void Sema::CheckMemaccessArguments(const CallExpr *Call,
           << Call->getCallee()->getSourceRange());
     else if (const auto *RT = PointeeTy->getAs<RecordType>()) {
 
-      bool IsTriviallyCopyableCXXRecord =
+      // FIXME: Do not consider incomplete types even though they may be
+      // completed later. GCC does not diagnose such code, but we may want to
+      // consider diagnosing it in the future, perhaps under a different, but
+      // related, diagnostic group.
+      bool MayBeTriviallyCopyableCXXRecord =
+          RT->isIncompleteType() ||
           RT->desugar().isTriviallyCopyableType(Context);
 
       if ((BId == Builtin::BImemset || BId == Builtin::BIbzero) &&
@@ -8910,7 +8915,7 @@ void Sema::CheckMemaccessArguments(const CallExpr *Call,
                                 << ArgIdx << FnName << PointeeTy << 0);
         SearchNonTrivialToInitializeField::diag(PointeeTy, Dest, *this);
       } else if ((BId == Builtin::BImemset || BId == Builtin::BIbzero) &&
-                 !IsTriviallyCopyableCXXRecord && ArgIdx == 0) {
+                 !MayBeTriviallyCopyableCXXRecord && ArgIdx == 0) {
         // FIXME: Limiting this warning to dest argument until we decide
         // whether it's valid for source argument too.
         DiagRuntimeBehavior(Dest->getExprLoc(), Dest,
@@ -8923,7 +8928,7 @@ void Sema::CheckMemaccessArguments(const CallExpr *Call,
                                 << ArgIdx << FnName << PointeeTy << 1);
         SearchNonTrivialToCopyField::diag(PointeeTy, Dest, *this);
       } else if ((BId == Builtin::BImemcpy || BId == Builtin::BImemmove) &&
-                 !IsTriviallyCopyableCXXRecord && ArgIdx == 0) {
+                 !MayBeTriviallyCopyableCXXRecord && ArgIdx == 0) {
         // FIXME: Limiting this warning to dest argument until we decide
         // whether it's valid for source argument too.
         DiagRuntimeBehavior(Dest->getExprLoc(), Dest,
diff --git a/clang/test/SemaCXX/constexpr-string.cpp b/clang/test/SemaCXX/constexpr-string.cpp
index 5448365489a5..c456740ef755 100644
--- a/clang/test/SemaCXX/constexpr-string.cpp
+++ b/clang/test/SemaCXX/constexpr-string.cpp
@@ -670,8 +670,6 @@ namespace MemcpyEtc {
   constexpr bool test_address_of_incomplete_struct_type() { // expected-error {{never produces a constant}}
     struct Incomplete;
     extern Incomplete x, y;
-    // expected-warning@+2 {{first argument in call to '__builtin_memcpy' is a pointer to non-trivially copyable type 'Incomplete'}}
-    // expected-note@+1 {{explicitly cast the pointer to silence this warning}}
     __builtin_memcpy(&x, &x, 4);
     // expected-note@-1 2{{cannot constant evaluate 'memcpy' between objects of incomplete type 'Incomplete'}}
     return true;
diff --git a/clang/test/SemaCXX/warn-memaccess.cpp b/clang/test/SemaCXX/warn-memaccess.cpp
index b4b7f6a6905b..070b44891a91 100644
--- a/clang/test/SemaCXX/warn-memaccess.cpp
+++ b/clang/test/SemaCXX/warn-memaccess.cpp
@@ -7,12 +7,17 @@ extern "C" void *memcpy(void *s1, const void *s2, unsigned n);
 
 class TriviallyCopyable {};
 class NonTriviallyCopyable { NonTriviallyCopyable(const NonTriviallyCopyable&);};
+struct Incomplete;
 
 void test_bzero(TriviallyCopyable* tc,
-                 NonTriviallyCopyable *ntc) {
+                NonTriviallyCopyable *ntc,
+                Incomplete* i) {
   // OK
   bzero(tc, sizeof(*tc));
 
+  // OK
+  bzero(i, 10);
+
   // expected-warning@+2{{first argument in call to 'bzero' is a pointer to non-trivially copyable type 'NonTriviallyCopyable'}}
   // expected-note@+1{{explicitly cast the pointer to silence this warning}}
   bzero(ntc, sizeof(*ntc));
@@ -22,10 +27,14 @@ void test_bzero(TriviallyCopyable* tc,
 }
 
 void test_memset(TriviallyCopyable* tc,
-                 NonTriviallyCopyable *ntc) {
+                 NonTriviallyCopyable *ntc,
+                 Incomplete* i) {
   // OK
   memset(tc, 0, sizeof(*tc));
 
+  // OK
+  memset(i, 0, 10);
+
   // expected-warning@+2{{first argument in call to 'memset' is a pointer to non-trivially copyable type 'NonTriviallyCopyable'}}
   // expected-note@+1{{explicitly cast the pointer to silence this warning}}
   memset(ntc, 0, sizeof(*ntc));
@@ -36,10 +45,14 @@ void test_memset(TriviallyCopyable* tc,
 
 
 void test_memcpy(TriviallyCopyable* tc0, TriviallyCopyable* tc1,
-                 NonTriviallyCopyable *ntc0, NonTriviallyCopyable *ntc1) {
+                 NonTriviallyCopyable *ntc0, NonTriviallyCopyable *ntc1,
+                 Incomplete *i0, Incomplete *i1) {
   // OK
   memcpy(tc0, tc1, sizeof(*tc0));
 
+  // OK
+  memcpy(i0, i1, 10);
+
   // expected-warning@+2{{first argument in call to 'memcpy' is a pointer to non-trivially copyable type 'NonTriviallyCopyable'}}
   // expected-note@+1{{explicitly cast the pointer to silence this warning}}
   memcpy(ntc0, ntc1, sizeof(*ntc0));
@@ -52,10 +65,14 @@ void test_memcpy(TriviallyCopyable* tc0, TriviallyCopyable* tc1,
 }
 
 void test_memmove(TriviallyCopyable* tc0, TriviallyCopyable* tc1,
-                 NonTriviallyCopyable *ntc0, NonTriviallyCopyable *ntc1) {
+                  NonTriviallyCopyable *ntc0, NonTriviallyCopyable *ntc1,
+                  Incomplete *i0, Incomplete *i1) {
   // OK
   memmove(tc0, tc1, sizeof(*tc0));
 
+  // OK
+  memmove(i0, i1, 10);
+
   // expected-warning@+2{{first argument in call to 'memmove' is a pointer to non-trivially copyable type 'NonTriviallyCopyable'}}
   // expected-note@+1{{explicitly cast the pointer to silence this warning}}
   memmove(ntc0, ntc1, sizeof(*ntc0));
-- 
GitLab


From 362273d1435c0cc104418f88b0140d0388e9ee22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nagy-Egri=20M=C3=A1t=C3=A9=20Ferenc?=
 <beiktatas+github@outlook.hu>
Date: Wed, 30 Oct 2024 08:51:11 +0100
Subject: [PATCH 551/693] [clang-format] Fix path expansion inside
 git-clang-format.bat (#114078)

The trampoline script used on Windows (due to the absence of shebang
support) doesn't properly expand the path to the Python script, as it
leaves out the drive letter.

Functionally equivalent reproducer in action
```
PS C:\Users\mate> gc (gcm git-clang-formatish.bat).Source
@ECHO OFF
echo "%~pn0" %*
PS C:\Users\mate> git-clang-formatish
"\Users\mate\git-clang-formatish"
```

Adding `d` to the variable modifiers [as per the
docs](https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/for)
the drive letter is added. Even in the magical cases when it works.

(I couldn't reproduce, but I suspect it's only tested from some
bash/cygwin variant, where the path becomes `/c/Program Files/...`, but
the drive letter is needed. Without it, I also observed cases when used
via `git clang-format` (without the inital dash) it tries to infer the
drive letter based on the current working directory. In that case it
fails to find `D:\Program Files\LLVM\bin\clang-format.exe`, which
naturally fails, because `Program Files` is on `C:`)
---
 clang/tools/clang-format/git-clang-format.bat | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/tools/clang-format/git-clang-format.bat b/clang/tools/clang-format/git-clang-format.bat
index 19c82d8a0413..a40276e63c58 100644
--- a/clang/tools/clang-format/git-clang-format.bat
+++ b/clang/tools/clang-format/git-clang-format.bat
@@ -1 +1 @@
-py -3 "%~pn0" %*
+py -3 "%~dpn0" %*
-- 
GitLab


From 9c8dab018dee3143c28a7d7f5fdb32385da36101 Mon Sep 17 00:00:00 2001
From: Boaz Brickner <brickner@google.com>
Date: Wed, 30 Oct 2024 08:59:49 +0100
Subject: [PATCH 552/693] [clang] Update the lifetimebound example with
 up-to-date expected warning and change the sample code to be a fully working
 example (#113437)

Tested the code: https://godbolt.org/z/n5xcq65YM
Tested the generated documentation:
![BruDQ2UkTXHA9PE](https://github.com/user-attachments/assets/cf527d1a-ef3b-41f2-84c2-4ca38af16d2d)
---
 clang/include/clang/Basic/AttrDocs.td | 28 +++++++++++++++++++--------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 7a130c434e73..fbbfc4acdf39 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -3702,20 +3702,32 @@ user-declared functions. For example:
 
 .. code-block:: c++
 
+    #include <map>
+    #include <string>
+
+    using namespace std::literals;
+
     // Returns m[key] if key is present, or default_value if not.
     template<typename T, typename U>
     const U &get_or_default(const std::map<T, U> &m [[clang::lifetimebound]],
                             const T &key, /* note, not lifetimebound */
-                            const U &default_value [[clang::lifetimebound]]);
+                            const U &default_value [[clang::lifetimebound]]) {
+      if (auto iter = m.find(key); iter != m.end()) return iter->second;
+      else return default_value;
+    }
 
-    std::map<std::string, std::string> m;
-    // warning: temporary "bar"s that might be bound to local reference 'val'
-    // will be destroyed at the end of the full-expression
-    const std::string &val = get_or_default(m, "foo"s, "bar"s);
+    int main() {
+      std::map<std::string, std::string> m;
+      // warning: temporary bound to local reference 'val1' will be destroyed
+      // at the end of the full-expression
+      const std::string &val1 = get_or_default(m, "foo"s, "bar"s);
 
-    // No warning in this case.
-    std::string def_val = "bar"s;
-    const std::string &val = get_or_default(m, "foo"s, def_val);
+      // No warning in this case.
+      std::string def_val = "bar"s;
+      const std::string &val2 = get_or_default(m, "foo"s, def_val);
+
+      return 0;
+    }
 
 The attribute can be applied to the implicit ``this`` parameter of a member
 function by writing the attribute after the function type:
-- 
GitLab


From f3584222682bd64daa89cbfe41c071c6bfc2347a Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Wed, 30 Oct 2024 08:10:35 +0000
Subject: [PATCH 553/693] [Attributor] Add nofpclass test for phi+select
 recurrences. NFC

---
 .../Attributor/nofpclass-phiselect.ll         | 176 ++++++++++++++++++
 1 file changed, 176 insertions(+)
 create mode 100644 llvm/test/Transforms/Attributor/nofpclass-phiselect.ll

diff --git a/llvm/test/Transforms/Attributor/nofpclass-phiselect.ll b/llvm/test/Transforms/Attributor/nofpclass-phiselect.ll
new file mode 100644
index 000000000000..6635280bc436
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nofpclass-phiselect.ll
@@ -0,0 +1,176 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -S < %s | FileCheck %s
+
+define float @phi_select(i1 %c, float nofpclass(inf) %base, float nofpclass(inf) %arg) {
+; CHECK-LABEL: define float @phi_select
+; CHECK-SAME: (i1 [[C:%.*]], float nofpclass(inf) [[BASE:%.*]], float nofpclass(inf) [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI:%.*]] = phi float [ [[BASE]], [[ENTRY:%.*]] ], [ [[SELECT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SELECT]] = select i1 [[C]], float [[PHI]], float [[ARG]]
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret float [[SELECT]]
+;
+entry:
+  br label %loop
+
+loop:
+  %phi = phi float [ %base, %entry ], [ %select, %loop ]
+  %select = select i1 %c, float %phi, float %arg
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret float %select
+}
+
+define float @phi_select_onlybase(i1 %c, float nofpclass(inf) %base, float %arg) {
+; CHECK-LABEL: define float @phi_select_onlybase
+; CHECK-SAME: (i1 [[C:%.*]], float nofpclass(inf) [[BASE:%.*]], float [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI:%.*]] = phi float [ [[BASE]], [[ENTRY:%.*]] ], [ [[SELECT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SELECT]] = select i1 [[C]], float [[PHI]], float [[ARG]]
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret float [[SELECT]]
+;
+entry:
+  br label %loop
+
+loop:
+  %phi = phi float [ %base, %entry ], [ %select, %loop ]
+  %select = select i1 %c, float %phi, float %arg
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret float %select
+}
+
+define float @phi_select_onlyarg(i1 %c, float %base, float nofpclass(inf) %arg) {
+; CHECK-LABEL: define float @phi_select_onlyarg
+; CHECK-SAME: (i1 [[C:%.*]], float [[BASE:%.*]], float nofpclass(inf) [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI:%.*]] = phi float [ [[BASE]], [[ENTRY:%.*]] ], [ [[SELECT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SELECT]] = select i1 [[C]], float [[PHI]], float [[ARG]]
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret float [[SELECT]]
+;
+entry:
+  br label %loop
+
+loop:
+  %phi = phi float [ %base, %entry ], [ %select, %loop ]
+  %select = select i1 %c, float %phi, float %arg
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret float %select
+}
+
+define float @phi_phi(i1 %c, float nofpclass(inf) %base, float nofpclass(inf) %arg) {
+; CHECK-LABEL: define float @phi_phi
+; CHECK-SAME: (i1 [[C:%.*]], float nofpclass(inf) [[BASE:%.*]], float nofpclass(inf) [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI:%.*]] = phi float [ [[BASE]], [[ENTRY:%.*]] ], [ [[PHI2:%.*]], [[EXIT1:%.*]] ]
+; CHECK-NEXT:    br label [[INNER:%.*]]
+; CHECK:       inner:
+; CHECK-NEXT:    [[PHI2]] = phi float [ [[PHI]], [[LOOP]] ], [ [[ARG]], [[INNER]] ]
+; CHECK-NEXT:    br i1 [[C]], label [[INNER]], label [[EXIT1]]
+; CHECK:       exit1:
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret float [[PHI2]]
+;
+entry:
+  br label %loop
+
+loop:
+  %phi = phi float [ %base, %entry ], [ %phi2, %exit1 ]
+  br label %inner
+
+inner:
+  %phi2 = phi float [ %phi, %loop ], [ %arg, %inner ]
+  br i1 %c, label %inner, label %exit1
+
+exit1:
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret float %phi2
+}
+
+define float @phi_phi_onlybase(i1 %c, float nofpclass(inf) %base, float %arg) {
+; CHECK-LABEL: define float @phi_phi_onlybase
+; CHECK-SAME: (i1 [[C:%.*]], float nofpclass(inf) [[BASE:%.*]], float [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI:%.*]] = phi float [ [[BASE]], [[ENTRY:%.*]] ], [ [[PHI2:%.*]], [[EXIT1:%.*]] ]
+; CHECK-NEXT:    br label [[INNER:%.*]]
+; CHECK:       inner:
+; CHECK-NEXT:    [[PHI2]] = phi float [ [[PHI]], [[LOOP]] ], [ [[ARG]], [[INNER]] ]
+; CHECK-NEXT:    br i1 [[C]], label [[INNER]], label [[EXIT1]]
+; CHECK:       exit1:
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret float [[PHI2]]
+;
+entry:
+  br label %loop
+
+loop:
+  %phi = phi float [ %base, %entry ], [ %phi2, %exit1 ]
+  br label %inner
+
+inner:
+  %phi2 = phi float [ %phi, %loop ], [ %arg, %inner ]
+  br i1 %c, label %inner, label %exit1
+
+exit1:
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret float %phi2
+}
+
+define float @phi_phi_onlyarg(i1 %c, float %base, float nofpclass(inf) %arg) {
+; CHECK-LABEL: define float @phi_phi_onlyarg
+; CHECK-SAME: (i1 [[C:%.*]], float [[BASE:%.*]], float nofpclass(inf) [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI:%.*]] = phi float [ [[BASE]], [[ENTRY:%.*]] ], [ [[PHI2:%.*]], [[EXIT1:%.*]] ]
+; CHECK-NEXT:    br label [[INNER:%.*]]
+; CHECK:       inner:
+; CHECK-NEXT:    [[PHI2]] = phi float [ [[PHI]], [[LOOP]] ], [ [[ARG]], [[INNER]] ]
+; CHECK-NEXT:    br i1 [[C]], label [[INNER]], label [[EXIT1]]
+; CHECK:       exit1:
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret float [[PHI2]]
+;
+entry:
+  br label %loop
+
+loop:
+  %phi = phi float [ %base, %entry ], [ %phi2, %exit1 ]
+  br label %inner
+
+inner:
+  %phi2 = phi float [ %phi, %loop ], [ %arg, %inner ]
+  br i1 %c, label %inner, label %exit1
+
+exit1:
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret float %phi2
+}
-- 
GitLab


From e61a7dc256bd530a0b9551e2732e5b5b77e2cd1e Mon Sep 17 00:00:00 2001
From: Mahesh-Attarde <145317060+mahesh-attarde@users.noreply.github.com>
Date: Wed, 30 Oct 2024 01:17:25 -0700
Subject: [PATCH 554/693] [X86][AVX512] Use comx for compare (#113567)

We added AVX10.2 COMEF ISA in LLVM, This does not optimize correctly in
scenario mentioned below.
Summary
Input
```
define i1 @oeq(float %x, float %y) {
    %1 = fcmp oeq float %x, %y
    ret i1 %1
}define i1 @une(float %x, float %y) {
    %1 = fcmp une float %x, %y
    ret i1 %1
}define i1 @ogt(float %x, float %y) {
    %1 = fcmp ogt float %x, %y
    ret i1 %1
}
// Prior AVX10.2, default code generation

oeq:                                    # @oeq
        cmpeqss xmm0, xmm1
        movd    eax, xmm0
        and     eax, 1
        ret
une:                                    # @une
        cmpneqss        xmm0, xmm1
        movd    eax, xmm0
        and     eax, 1
        ret
ogt:                                    # @ogt
        ucomiss xmm0, xmm1
        seta    al
        ret
```

This patch will remove `cmpeqss` and `cmpneqss`. For complete transform
check unit test.

Continuing on what PR https://github.com/llvm/llvm-project/pull/113098
added

Earlier Legalization and combine expanded `setcc oeq:ch` node into `and`
and `setcc eq` , `setcc o`. From suggestions in community
new internal transform
```
Optimized type-legalized selection DAG: %bb.0 'hoeq:'
SelectionDAG has 11 nodes:
  t0: ch,glue = EntryToken
      t2: f16,ch = CopyFromReg t0, Register:f16 %0
      t4: f16,ch = CopyFromReg t0, Register:f16 %1
    t14: i8 = setcc t2, t4, setoeq:ch
  t10: ch,glue = CopyToReg t0, Register:i8 $al, t14
  t11: ch = X86ISD::RET_GLUE t10, TargetConstant:i32<0>, Register:i8 $al, t10:1

Optimized legalized selection DAG: %bb.0 'hoeq:'
SelectionDAG has 12 nodes:
  t0: ch,glue = EntryToken
        t2: f16,ch = CopyFromReg t0, Register:f16 %0
        t4: f16,ch = CopyFromReg t0, Register:f16 %1
      t15: i32 = X86ISD::UCOMX t2, t4
    t17: i8 = X86ISD::SETCC TargetConstant:i8<4>, t15
  t10: ch,glue = CopyToReg t0, Register:i8 $al, t17
  t11: ch = X86ISD::RET_GLUE t10, TargetConstant:i32<0>, Register:i8 $al, t10:1
```
Earlier transform is mentioned here
https://github.com/llvm/llvm-project/pull/113098#discussion_r1810307663

---------

Co-authored-by: mattarde <mattarde@intel.com>
---
 llvm/lib/Target/X86/X86ISelLowering.cpp |  11 ++
 llvm/lib/Target/X86/X86InstrAVX10.td    |  27 +++
 llvm/test/CodeGen/X86/avx10_2-cmp.ll    | 237 ++++++++++++++++++++++++
 llvm/test/TableGen/x86-fold-tables.inc  |   3 +
 4 files changed, 278 insertions(+)
 create mode 100644 llvm/test/CodeGen/X86/avx10_2-cmp.ll

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 1c790f3813b7..34bc5d76c15c 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2440,6 +2440,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::FMA, MVT::v32bf16, Legal);
       setOperationAction(ISD::SETCC, MVT::v32bf16, Custom);
     }
+    for (auto VT : {MVT::f16, MVT::f32, MVT::f64}) {
+      setCondCodeAction(ISD::SETOEQ, VT, Custom);
+      setCondCodeAction(ISD::SETUNE, VT, Custom);
+    }
   }
 
   if (!Subtarget.useSoftFloat() && Subtarget.hasVLX()) {
@@ -24072,6 +24076,13 @@ SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
     return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res;
   }
 
+  if (Subtarget.hasAVX10_2()) {
+    if (CC == ISD::SETOEQ || CC == ISD::SETUNE) {
+      auto NewCC = (CC == ISD::SETOEQ) ? X86::COND_E : (X86::COND_NE);
+      return getSETCC(NewCC, DAG.getNode(X86ISD::UCOMX, dl, MVT::i32, Op0, Op1),
+                      dl, DAG);
+    }
+  }
   // Handle floating point.
   X86::CondCode CondCode = TranslateX86CC(CC, dl, /*IsFP*/ true, Op0, Op1, DAG);
   if (CondCode == X86::COND_INVALID)
diff --git a/llvm/lib/Target/X86/X86InstrAVX10.td b/llvm/lib/Target/X86/X86InstrAVX10.td
index 4d64eb776e09..0301c07dfb54 100644
--- a/llvm/lib/Target/X86/X86InstrAVX10.td
+++ b/llvm/lib/Target/X86/X86InstrAVX10.td
@@ -1541,6 +1541,24 @@ defm VFNMSUB132NEPBF16 : avx10_fma3p_132_bf16<0x9E, "vfnmsub132nepbf16", X86any_
 //-------------------------------------------------
 // AVX10  COMEF instructions
 //-------------------------------------------------
+multiclass avx10_com_ef<bits<8> Opc, RegisterClass RC, ValueType VT,
+                        SDPatternOperator OpNode, string OpcodeStr,
+                        X86MemOperand x86memop, PatFrag ld_frag,
+                        Domain d, X86FoldableSchedWrite sched = WriteFComX>{
+  let ExeDomain = d, mayRaiseFPException = 1, isCodeGenOnly = 1 in {
+    def rr : AVX512<Opc, MRMSrcReg, (outs), (ins RC:$src1, RC:$src2),
+                    !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"),
+                    [(set EFLAGS, (OpNode (VT RC:$src1), RC:$src2))]>,
+                    EVEX, EVEX_V128, Sched<[sched]>, SIMD_EXC;
+    let mayLoad = 1 in {
+      def rm : AVX512<Opc, MRMSrcMem, (outs), (ins RC:$src1, x86memop:$src2),
+                      !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"),
+                      [(set EFLAGS, (OpNode (VT RC:$src1), (ld_frag addr:$src2)))]>,
+                      EVEX, EVEX_V128, Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;
+    }
+  }
+}
+
 multiclass avx10_com_ef_int<bits<8> Opc, X86VectorVTInfo _, SDNode OpNode,
                              string OpcodeStr,
                              Domain d,
@@ -1564,6 +1582,15 @@ multiclass avx10_com_ef_int<bits<8> Opc, X86VectorVTInfo _, SDNode OpNode,
 }
 
 let Defs = [EFLAGS], Uses = [MXCSR], Predicates = [HasAVX10_2] in {
+  defm VUCOMXSDZ  :  avx10_com_ef<0x2e, FR64X, f64, X86ucomi512,
+                                  "vucomxsd", f64mem, loadf64, SSEPackedDouble>,
+                                  TB, XS, VEX_LIG, REX_W, EVEX_CD8<64, CD8VT1>;
+  defm VUCOMXSHZ  :  avx10_com_ef<0x2e, FR16X, f16, X86ucomi512,
+                                  "vucomxsh", f16mem, loadf16, SSEPackedSingle>,
+                                  T_MAP5, XD, EVEX_CD8<16, CD8VT1>;
+  defm VUCOMXSSZ  :  avx10_com_ef<0x2e, FR32X, f32, X86ucomi512,
+                                  "vucomxss", f32mem, loadf32, SSEPackedSingle>,
+                                  TB, XD, VEX_LIG, EVEX_CD8<32, CD8VT1>;
   defm VCOMXSDZ   :  avx10_com_ef_int<0x2f, v2f64x_info, X86comi512,
                                       "vcomxsd", SSEPackedDouble>,
                                       TB, XS, VEX_LIG, REX_W, EVEX_CD8<64, CD8VT1>;
diff --git a/llvm/test/CodeGen/X86/avx10_2-cmp.ll b/llvm/test/CodeGen/X86/avx10_2-cmp.ll
new file mode 100644
index 000000000000..de0bec7ea269
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx10_2-cmp.ll
@@ -0,0 +1,237 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx10.2-256 | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=+avx10.2-256 | FileCheck %s --check-prefix=X86
+
+define i1 @hoeq(half %x, half %y) {
+; X64-LABEL: hoeq:
+; X64:       # %bb.0:
+; X64-NEXT:    vucomxsh %xmm1, %xmm0
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+;
+; X86-LABEL: hoeq:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT:    vucomxsh {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+    %1 = fcmp oeq half %x, %y
+    ret i1 %1
+}
+
+define i1 @hune(half %x, half %y) {
+; X64-LABEL: hune:
+; X64:       # %bb.0:
+; X64-NEXT:    vucomxsh %xmm1, %xmm0
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+;
+; X86-LABEL: hune:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT:    vucomxsh {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+    %1 = fcmp une half %x, %y
+    ret i1 %1
+}
+
+define i1 @hoeq_mem(ptr %xp, ptr %yp) {
+; X64-LABEL: hoeq_mem:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X64-NEXT:    vucomxsh (%rsi), %xmm0
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+;
+; X86-LABEL: hoeq_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT:    vucomxsh (%eax), %xmm0
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+    %x = load half, ptr %xp
+    %y = load half, ptr %yp
+    %1 = fcmp oeq half %x, %y
+    ret i1 %1
+}
+
+define i1 @hune_mem(ptr %xp, ptr %yp) {
+; X64-LABEL: hune_mem:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X64-NEXT:    vucomxsh (%rsi), %xmm0
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+;
+; X86-LABEL: hune_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT:    vucomxsh (%eax), %xmm0
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+    %x = load half, ptr %xp
+    %y = load half, ptr %yp
+    %1 = fcmp une half %x, %y
+    ret i1 %1
+}
+
+define i1 @foeq(float %x, float %y) {
+; X64-LABEL: foeq:
+; X64:       # %bb.0:
+; X64-NEXT:    vucomxss %xmm1, %xmm0
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+;
+; X86-LABEL: foeq:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vucomxss {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+    %1 = fcmp oeq float %x, %y
+    ret i1 %1
+}
+
+define i1 @fune(float %x, float %y) {
+; X64-LABEL: fune:
+; X64:       # %bb.0:
+; X64-NEXT:    vucomxss %xmm1, %xmm0
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+;
+; X86-LABEL: fune:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vucomxss {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+    %1 = fcmp une float %x, %y
+    ret i1 %1
+}
+
+define i1 @foeq_mem(ptr %xp, ptr %yp) {
+; X64-LABEL: foeq_mem:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    vucomxss (%rsi), %xmm0
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+;
+; X86-LABEL: foeq_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vucomxss (%eax), %xmm0
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+    %x = load float, ptr %xp
+    %y = load float, ptr %yp
+    %1 = fcmp oeq float %x, %y
+    ret i1 %1
+}
+
+define i1 @fune_mem(ptr %xp, ptr %yp) {
+; X64-LABEL: fune_mem:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    vucomxss (%rsi), %xmm0
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+;
+; X86-LABEL: fune_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vucomxss (%eax), %xmm0
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+    %x = load float, ptr %xp
+    %y = load float, ptr %yp
+    %1 = fcmp une float %x, %y
+    ret i1 %1
+}
+
+define i1 @doeq(double %x, double %y) {
+; X64-LABEL: doeq:
+; X64:       # %bb.0:
+; X64-NEXT:    vucomxsd %xmm1, %xmm0
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+;
+; X86-LABEL: doeq:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    vucomxsd {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+    %1 = fcmp oeq double %x, %y
+    ret i1 %1
+}
+
+define i1 @dune(double %x, double %y) {
+; X64-LABEL: dune:
+; X64:       # %bb.0:
+; X64-NEXT:    vucomxsd %xmm1, %xmm0
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+;
+; X86-LABEL: dune:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    vucomxsd {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+    %1 = fcmp une double %x, %y
+    ret i1 %1
+}
+
+define i1 @doeq_mem(ptr %xp, ptr %yp) {
+; X64-LABEL: doeq_mem:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    vucomxsd (%rsi), %xmm0
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+;
+; X86-LABEL: doeq_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    vucomxsd (%eax), %xmm0
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+    %x = load double, ptr %xp
+    %y = load double, ptr %yp
+    %1 = fcmp oeq double %x, %y
+    ret i1 %1
+}
+
+define i1 @dune_mem(ptr %xp, ptr %yp) {
+; X64-LABEL: dune_mem:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    vucomxsd (%rsi), %xmm0
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+;
+; X86-LABEL: dune_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    vucomxsd (%eax), %xmm0
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+    %x = load double, ptr %xp
+    %y = load double, ptr %yp
+    %1 = fcmp une double %x, %y
+    ret i1 %1
+}
diff --git a/llvm/test/TableGen/x86-fold-tables.inc b/llvm/test/TableGen/x86-fold-tables.inc
index 43c206fa0af6..21f3c8593a71 100644
--- a/llvm/test/TableGen/x86-fold-tables.inc
+++ b/llvm/test/TableGen/x86-fold-tables.inc
@@ -1959,8 +1959,11 @@ static const X86FoldTableEntry Table1[] = {
   {X86::VUCOMISSZrr_Int, X86::VUCOMISSZrm_Int, TB_NO_REVERSE},
   {X86::VUCOMISSrr, X86::VUCOMISSrm, 0},
   {X86::VUCOMISSrr_Int, X86::VUCOMISSrm_Int, TB_NO_REVERSE},
+  {X86::VUCOMXSDZrr, X86::VUCOMXSDZrm, 0},
   {X86::VUCOMXSDZrr_Int, X86::VUCOMXSDZrm_Int, TB_NO_REVERSE},
+  {X86::VUCOMXSHZrr, X86::VUCOMXSHZrm, 0},
   {X86::VUCOMXSHZrr_Int, X86::VUCOMXSHZrm_Int, TB_NO_REVERSE},
+  {X86::VUCOMXSSZrr, X86::VUCOMXSSZrm, 0},
   {X86::VUCOMXSSZrr_Int, X86::VUCOMXSSZrm_Int, TB_NO_REVERSE},
   {X86::XOR16ri8_ND, X86::XOR16mi8_ND, 0},
   {X86::XOR16ri8_NF_ND, X86::XOR16mi8_NF_ND, 0},
-- 
GitLab


From 259eaa6878ead1e2e7ef572a874dc3d885c1899b Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Wed, 30 Oct 2024 17:27:04 +0800
Subject: [PATCH 555/693] [C++20] [Modules] Fix the duplicated static
 initializer problem (#114193)

Reproducer:

```
//--- a.cppm
export module a;
int func();
static int a = func();

//--- a.cpp
import a;
```

The `func()` should only execute once. However, before this patch we
will somehow import `static int a` from a.cppm incorrectly and
initialize that again.

This is super bad and can introduce serious runtime behaviors.

And also surprisingly, it looks like the root cause of the problem is
simply some oversight choosing APIs.
---
 clang/lib/CodeGen/CodeGenModule.cpp        |  4 ++--
 clang/test/Modules/static-initializer.cppm | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/Modules/static-initializer.cppm

diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 2bcca5e85bdf..ba376f9ecfac 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -7146,8 +7146,8 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) {
     // For C++ standard modules we are done - we will call the module
     // initializer for imported modules, and that will likewise call those for
     // any imports it has.
-    if (CXX20ModuleInits && Import->getImportedOwningModule() &&
-        !Import->getImportedOwningModule()->isModuleMapModule())
+    if (CXX20ModuleInits && Import->getImportedModule() &&
+        Import->getImportedModule()->isNamedModule())
       break;
 
     // For clang C++ module map modules the initializers for sub-modules are
diff --git a/clang/test/Modules/static-initializer.cppm b/clang/test/Modules/static-initializer.cppm
new file mode 100644
index 000000000000..10d4854ee67f
--- /dev/null
+++ b/clang/test/Modules/static-initializer.cppm
@@ -0,0 +1,18 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 %t/a.cppm -emit-module-interface -o %t/a.pcm
+// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 %t/a.cpp -fmodule-file=a=%t/a.pcm -emit-llvm -o - | FileCheck %t/a.cpp
+
+//--- a.cppm
+export module a;
+int func();
+static int a = func();
+
+//--- a.cpp
+import a;
+
+// CHECK-NOT: internal global
+// CHECK-NOT: __cxx_global_var_init
+
-- 
GitLab


From e8b95a02bff8498c888ed5e85d0197ec82b95cd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mirko=20Brku=C5=A1anin?= <Mirko.Brkusanin@amd.com>
Date: Wed, 30 Oct 2024 10:45:33 +0100
Subject: [PATCH 556/693] [AMDGPU][MC][NFC] Add more VIMAGE encoding tests
 (#114054)

These are primarily meant to test disassembler and that no more than
one variant per instruction is in DisassemblerTables as that can cause
confusion when decoding v0 (vgpr0) whose value when encoded is 0.
---
 llvm/test/MC/AMDGPU/gfx12_asm_vimage.s        | 24 +++++++++++++++++++
 .../Disassembler/AMDGPU/gfx12_dasm_vimage.txt | 24 +++++++++++++++++++
 2 files changed, 48 insertions(+)

diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vimage.s b/llvm/test/MC/AMDGPU/gfx12_asm_vimage.s
index 196d75db4260..8bf9b92e8d1d 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vimage.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vimage.s
@@ -158,6 +158,12 @@ image_load v[0:2], [v4, v5], s[8:15] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY th:TH_LO
 image_load v[4:7], [v1, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_2D
 // GFX12: encoding: [0x01,0x00,0xc0,0xd3,0x04,0x08,0x00,0x00,0x01,0x00,0x00,0x00]
 
+image_load v[1:4], [v2, v1, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_3D
+// GFX12: encoding: [0x02,0x00,0xc0,0xd3,0x01,0x08,0x00,0x00,0x02,0x01,0x00,0x00]
+
+image_load v[1:4], [v3, v2, v1, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY
+// GFX12: encoding: [0x07,0x00,0xc0,0xd3,0x01,0x08,0x00,0x00,0x03,0x02,0x01,0x00]
+
 image_load_mip v[252:255], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D
 // GFX12: encoding: [0x00,0x40,0xc0,0xd3,0xfc,0x00,0x00,0x00,0x00,0x01,0x00,0x00]
 
@@ -408,6 +414,12 @@ image_store v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_STORE_BYPASS scope
 image_store v[1:4], [v2, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_2D
 // GFX12: encoding: [0x01,0x80,0xc1,0xd3,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00]
 
+image_store v[1:4], [v2, v1, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_3D
+// GFX12: encoding: [0x02,0x80,0xc1,0xd3,0x01,0x08,0x00,0x00,0x02,0x01,0x00,0x00]
+
+image_store v[1:4], [v3, v2, v1, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY
+// GFX12: encoding: [0x07,0x80,0xc1,0xd3,0x01,0x08,0x00,0x00,0x03,0x02,0x01,0x00]
+
 image_store_mip v[252:255], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D
 // GFX12: encoding: [0x00,0xc0,0xc1,0xd3,0xfc,0x00,0x00,0x00,0x00,0x01,0x00,0x00]
 
@@ -568,6 +580,12 @@ image_atomic_swap v[254:255], [v4, v5], s[96:103] dmask:0x3 dim:SQ_RSRC_IMG_2D_M
 image_atomic_swap v1, [v2, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 // GFX12: encoding: [0x01,0x80,0x42,0xd0,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00]
 
+image_atomic_swap v1, [v2, v1, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_3D
+// GFX12: encoding: [0x02,0x80,0x42,0xd0,0x01,0x08,0x00,0x00,0x02,0x01,0x00,0x00]
+
+image_atomic_swap v1, [v3, v2, v1, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY
+// GFX12: encoding: [0x07,0x80,0x42,0xd0,0x01,0x08,0x00,0x00,0x03,0x02,0x01,0x00]
+
 image_atomic_cmpswap v[0:1], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D
 // GFX12: encoding: [0x00,0xc0,0xc2,0xd0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
 
@@ -625,6 +643,12 @@ image_atomic_add_uint v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_N
 image_atomic_add_uint v1, [v2, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 // GFX12: encoding: [0x01,0x00,0x43,0xd0,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00]
 
+image_atomic_add_uint v1, [v2, v1, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_3D
+// GFX12: encoding: [0x02,0x00,0x43,0xd0,0x01,0x08,0x00,0x00,0x02,0x01,0x00,0x00]
+
+image_atomic_add_uint v1, [v3, v2, v1, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY
+// GFX12: encoding: [0x07,0x00,0x43,0xd0,0x01,0x08,0x00,0x00,0x03,0x02,0x01,0x00]
+
 image_atomic_sub_uint v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
 // GFX12: encoding: [0x00,0x40,0x43,0xd0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage.txt
index 08e9bef8cf67..233c2e1b9d08 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage.txt
@@ -160,6 +160,12 @@
 # GFX12: image_load v[4:7], [v1, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x01,0x00,0xc0,0xd3,0x04,0x08,0x00,0x00,0x01,0x00,0x00,0x00]
 0x01,0x00,0xc0,0xd3,0x04,0x08,0x00,0x00,0x01,0x00,0x00,0x00
 
+# GFX12: image_load v[1:4], [v2, v1, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_3D ; encoding: [0x02,0x00,0xc0,0xd3,0x01,0x08,0x00,0x00,0x02,0x01,0x00,0x00]
+0x02,0x00,0xc0,0xd3,0x01,0x08,0x00,0x00,0x02,0x01,0x00,0x00
+
+# GFX12: image_load v[1:4], [v3, v2, v1, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY ; encoding: [0x07,0x00,0xc0,0xd3,0x01,0x08,0x00,0x00,0x03,0x02,0x01,0x00]
+0x07,0x00,0xc0,0xd3,0x01,0x08,0x00,0x00,0x03,0x02,0x01,0x00
+
 # GFX12: image_load_mip v[252:255], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x40,0xc0,0xd3,0xfc,0x00,0x00,0x00,0x00,0x01,0x00,0x00]
 0x00,0x40,0xc0,0xd3,0xfc,0x00,0x00,0x00,0x00,0x01,0x00,0x00
 
@@ -409,6 +415,12 @@
 # GFX12: image_store v[1:4], [v2, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x01,0x80,0xc1,0xd3,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00]
 0x01,0x80,0xc1,0xd3,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00
 
+# GFX12: image_store v[1:4], [v2, v1, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_3D ; encoding: [0x02,0x80,0xc1,0xd3,0x01,0x08,0x00,0x00,0x02,0x01,0x00,0x00]
+0x02,0x80,0xc1,0xd3,0x01,0x08,0x00,0x00,0x02,0x01,0x00,0x00
+
+# GFX12: image_store v[1:4], [v3, v2, v1, v0], s[4:11] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY ; encoding: [0x07,0x80,0xc1,0xd3,0x01,0x08,0x00,0x00,0x03,0x02,0x01,0x00]
+0x07,0x80,0xc1,0xd3,0x01,0x08,0x00,0x00,0x03,0x02,0x01,0x00
+
 # GFX12: image_store_mip v[252:255], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0xc0,0xc1,0xd3,0xfc,0x00,0x00,0x00,0x00,0x01,0x00,0x00]
 0x00,0xc0,0xc1,0xd3,0xfc,0x00,0x00,0x00,0x00,0x01,0x00,0x00
 
@@ -568,6 +580,12 @@
 # GFX12: image_atomic_swap v1, [v2, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D ; encoding: [0x01,0x80,0x42,0xd0,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00]
 0x01,0x80,0x42,0xd0,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00
 
+# GFX12: image_atomic_swap v1, [v2, v1, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_3D ; encoding: [0x02,0x80,0x42,0xd0,0x01,0x08,0x00,0x00,0x02,0x01,0x00,0x00]
+0x02,0x80,0x42,0xd0,0x01,0x08,0x00,0x00,0x02,0x01,0x00,0x00
+
+# GFX12: image_atomic_swap v1, [v3, v2, v1, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY ; encoding: [0x07,0x80,0x42,0xd0,0x01,0x08,0x00,0x00,0x03,0x02,0x01,0x00]
+0x07,0x80,0x42,0xd0,0x01,0x08,0x00,0x00,0x03,0x02,0x01,0x00
+
 # GFX12: image_atomic_cmpswap v[0:1], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0xc0,0xc2,0xd0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
 0x00,0xc0,0xc2,0xd0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
 
@@ -625,6 +643,12 @@
 # GFX12: image_atomic_add_uint v1, [v2, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D ; encoding: [0x01,0x00,0x43,0xd0,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00]
 0x01,0x00,0x43,0xd0,0x01,0x08,0x00,0x00,0x02,0x00,0x00,0x00
 
+# GFX12: image_atomic_add_uint v1, [v2, v1, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_3D ; encoding: [0x02,0x00,0x43,0xd0,0x01,0x08,0x00,0x00,0x02,0x01,0x00,0x00]
+0x02,0x00,0x43,0xd0,0x01,0x08,0x00,0x00,0x02,0x01,0x00,0x00
+
+# GFX12: image_atomic_add_uint v1, [v3, v2, v1, v0], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY ; encoding: [0x07,0x00,0x43,0xd0,0x01,0x08,0x00,0x00,0x03,0x02,0x01,0x00]
+0x07,0x00,0x43,0xd0,0x01,0x08,0x00,0x00,0x03,0x02,0x01,0x00
+
 # GFX12: image_atomic_sub_uint v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x40,0x43,0xd0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
 0x00,0x40,0x43,0xd0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
 
-- 
GitLab


From 652988b65805b23f228db34adfff068cffd517cc Mon Sep 17 00:00:00 2001
From: Abid Qadeer <haqadeer@amd.com>
Date: Wed, 30 Oct 2024 09:52:56 +0000
Subject: [PATCH 557/693] [flang][debug] Support TupleType. (#113917)

Handling is similar to RecordType with following differences:

1. No check for cyclic references
2. No extra processing for lower bounds of array members.
3. No line information as TupleType is a lowering artefact and does not
really represent an entity in the code.
---
 .../Transforms/DebugTypeGenerator.cpp         | 61 ++++++++++++++++---
 .../Optimizer/Transforms/DebugTypeGenerator.h |  6 ++
 flang/test/Transforms/debug-tuple-type.fir    | 15 +++++
 3 files changed, 73 insertions(+), 9 deletions(-)
 create mode 100644 flang/test/Transforms/debug-tuple-type.fir

diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index 8e516734a908..a070c87137fa 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -271,6 +271,19 @@ static bool canCacheThisType(mlir::LLVM::DICompositeTypeAttr comTy) {
   return true;
 }
 
+std::pair<std::uint64_t, unsigned short>
+DebugTypeGenerator::getFieldSizeAndAlign(mlir::Type fieldTy) {
+  mlir::Type llvmTy;
+  if (auto boxTy = mlir::dyn_cast_or_null<fir::BaseBoxType>(fieldTy))
+    llvmTy = llvmTypeConverter.convertBoxTypeAsStruct(boxTy, getBoxRank(boxTy));
+  else
+    llvmTy = llvmTypeConverter.convertType(fieldTy);
+
+  uint64_t byteSize = dataLayout->getTypeSize(llvmTy);
+  unsigned short byteAlign = dataLayout->getTypeABIAlignment(llvmTy);
+  return std::pair{byteSize, byteAlign};
+}
+
 mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
     fir::RecordType Ty, mlir::LLVM::DIFileAttr fileAttr,
     mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp) {
@@ -303,15 +316,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
   mlir::IntegerType intTy = mlir::IntegerType::get(context, 64);
   std::uint64_t offset = 0;
   for (auto [fieldName, fieldTy] : Ty.getTypeList()) {
-    mlir::Type llvmTy;
-    if (auto boxTy = mlir::dyn_cast_or_null<fir::BaseBoxType>(fieldTy))
-      llvmTy =
-          llvmTypeConverter.convertBoxTypeAsStruct(boxTy, getBoxRank(boxTy));
-    else
-      llvmTy = llvmTypeConverter.convertType(fieldTy);
-
-    uint64_t byteSize = dataLayout->getTypeSize(llvmTy);
-    unsigned short byteAlign = dataLayout->getTypeABIAlignment(llvmTy);
+    auto [byteSize, byteAlign] = getFieldSizeAndAlign(fieldTy);
     std::optional<llvm::ArrayRef<int64_t>> lowerBounds =
         fir::getComponentLowerBoundsIfNonDefault(Ty, fieldName, module,
                                                  symbolTable);
@@ -368,6 +373,42 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
   return finalAttr;
 }
 
+mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
+    mlir::TupleType Ty, mlir::LLVM::DIFileAttr fileAttr,
+    mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp) {
+  // Check if this type has already been converted.
+  auto iter = typeCache.find(Ty);
+  if (iter != typeCache.end())
+    return iter->second;
+
+  llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
+  mlir::MLIRContext *context = module.getContext();
+
+  std::uint64_t offset = 0;
+  for (auto fieldTy : Ty.getTypes()) {
+    auto [byteSize, byteAlign] = getFieldSizeAndAlign(fieldTy);
+    mlir::LLVM::DITypeAttr elemTy =
+        convertType(fieldTy, fileAttr, scope, /*declOp=*/nullptr);
+    offset = llvm::alignTo(offset, byteAlign);
+    mlir::LLVM::DIDerivedTypeAttr tyAttr = mlir::LLVM::DIDerivedTypeAttr::get(
+        context, llvm::dwarf::DW_TAG_member, mlir::StringAttr::get(context, ""),
+        elemTy, byteSize * 8, byteAlign * 8, offset * 8,
+        /*optional<address space>=*/std::nullopt,
+        /*extra data=*/nullptr);
+    elements.push_back(tyAttr);
+    offset += llvm::alignTo(byteSize, byteAlign);
+  }
+
+  auto typeAttr = mlir::LLVM::DICompositeTypeAttr::get(
+      context, llvm::dwarf::DW_TAG_structure_type,
+      mlir::StringAttr::get(context, ""), fileAttr, /*line=*/0, scope,
+      /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, offset * 8,
+      /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
+      /*allocated=*/nullptr, /*associated=*/nullptr);
+  typeCache[Ty] = typeAttr;
+  return typeAttr;
+}
+
 mlir::LLVM::DITypeAttr DebugTypeGenerator::convertSequenceType(
     fir::SequenceType seqTy, mlir::LLVM::DIFileAttr fileAttr,
     mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp) {
@@ -574,6 +615,8 @@ DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
                                 /*hasDescriptor=*/false);
   } else if (auto recTy = mlir::dyn_cast_or_null<fir::RecordType>(Ty)) {
     return convertRecordType(recTy, fileAttr, scope, declOp);
+  } else if (auto tupleTy = mlir::dyn_cast_if_present<mlir::TupleType>(Ty)) {
+    return convertTupleType(tupleTy, fileAttr, scope, declOp);
   } else if (auto refTy = mlir::dyn_cast_if_present<fir::ReferenceType>(Ty)) {
     auto elTy = refTy.getEleTy();
     return convertPointerLikeType(elTy, fileAttr, scope, declOp,
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
index eeefb6c463d9..c1fce4bdae5c 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
@@ -39,6 +39,10 @@ private:
                                            mlir::LLVM::DIFileAttr fileAttr,
                                            mlir::LLVM::DIScopeAttr scope,
                                            fir::cg::XDeclareOp declOp);
+  mlir::LLVM::DITypeAttr convertTupleType(mlir::TupleType Ty,
+                                          mlir::LLVM::DIFileAttr fileAttr,
+                                          mlir::LLVM::DIScopeAttr scope,
+                                          fir::cg::XDeclareOp declOp);
   mlir::LLVM::DITypeAttr convertSequenceType(fir::SequenceType seqTy,
                                              mlir::LLVM::DIFileAttr fileAttr,
                                              mlir::LLVM::DIScopeAttr scope,
@@ -73,6 +77,8 @@ private:
                              mlir::LLVM::DIFileAttr fileAttr,
                              mlir::LLVM::DIScopeAttr scope,
                              fir::cg::XDeclareOp declOp);
+  std::pair<std::uint64_t, unsigned short>
+  getFieldSizeAndAlign(mlir::Type fieldTy);
 
   mlir::ModuleOp module;
   mlir::SymbolTable *symbolTable;
diff --git a/flang/test/Transforms/debug-tuple-type.fir b/flang/test/Transforms/debug-tuple-type.fir
new file mode 100644
index 000000000000..c9b0d16c06e1
--- /dev/null
+++ b/flang/test/Transforms/debug-tuple-type.fir
@@ -0,0 +1,15 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func private @fn1(!fir.ref<tuple<f64, f64>>)
+  func.func private @_FortranAioOutputDerivedType(!fir.ref<tuple<>>)
+}
+
+// CHECK: #[[F64:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real", sizeInBits = 64, encoding = DW_ATE_float>
+// CHECK: #[[CU:.*]] = #llvm.di_compile_unit<{{.*}}>
+// CHECK: #[[DTY1:.*]] = #llvm.di_derived_type<tag = DW_TAG_member, name = "", baseType = #[[F64]], sizeInBits = 64, alignInBits = {{.*}}>
+// CHECK: #[[DTY2:.*]] = #llvm.di_derived_type<tag = DW_TAG_member, name = "", baseType = #[[F64]], sizeInBits = 64, alignInBits = {{.*}}, offsetInBits = {{.*}}>
+// CHECK: #[[COM_TY1:.*]] = #llvm.di_composite_type<tag = DW_TAG_structure_type, name = "", file = #{{.*}}, scope = #[[CU]]{{.*}}elements = #[[DTY1]], #[[DTY2]]>
+// CHECK: #[[COM_TY2:.*]] = #llvm.di_composite_type<tag = DW_TAG_structure_type, name = "", file = #{{.*}}, scope = #[[CU]]>
+// CHECK: #llvm.di_subroutine_type<callingConvention = DW_CC_normal, types = #di_null_type, #[[COM_TY1]]>
+// CHECK: #llvm.di_subroutine_type<callingConvention = DW_CC_normal, types = #di_null_type, #[[COM_TY2]]>
-- 
GitLab


From 03948882d3bac33cf71a47df1c7ee0f87aad9fc2 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 30 Oct 2024 10:12:57 +0000
Subject: [PATCH 558/693] Fix MSVC "32-bit shift implicitly converted to 64
 bits" warning. NFC

NumBits should be less than 20 so using an unsigned instead of size_t should be OK
---
 llvm/lib/Support/TrieRawHashMap.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Support/TrieRawHashMap.cpp b/llvm/lib/Support/TrieRawHashMap.cpp
index 4741f3d4db04..11d79a62d011 100644
--- a/llvm/lib/Support/TrieRawHashMap.cpp
+++ b/llvm/lib/Support/TrieRawHashMap.cpp
@@ -79,7 +79,7 @@ public:
 
   static constexpr size_t sizeToAlloc(unsigned NumBits) {
     assert(NumBits < 20 && "Tries should have fewer than ~1M slots");
-    size_t Count = 1u << NumBits;
+    unsigned Count = 1u << NumBits;
     return totalSizeToAlloc<LazyAtomicPointer<TrieNode>>(Count);
   }
 
-- 
GitLab


From f7b5f0c805c899b59bcc37279a0a05dca35d3a25 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 30 Oct 2024 10:46:12 +0000
Subject: [PATCH 559/693] [DAG] Fold (and X, (rot (not Y), Z)) -> (and X, (not
 (rot Y, Z)))

On ANDNOT capable targets we can always do this profitably, without ANDNOT we only attempt this if we don't introduce an additional NOT

Followup to #112547
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  11 +-
 llvm/test/CodeGen/X86/andnot-patterns.ll      | 463 ++++++++++++------
 2 files changed, 314 insertions(+), 160 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index b800204d9175..ceaf5d664131 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -7355,7 +7355,7 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
 
   // Fold (and X, (bswap (not Y))) -> (and X, (not (bswap Y)))
   // Fold (and X, (bitreverse (not Y))) -> (and X, (not (bitreverse Y)))
-  SDValue X, Y, NotY;
+  SDValue X, Y, Z, NotY;
   for (unsigned Opc : {ISD::BSWAP, ISD::BITREVERSE})
     if (sd_match(N,
                  m_And(m_Value(X), m_OneUse(m_UnaryOp(Opc, m_Value(NotY))))) &&
@@ -7364,6 +7364,15 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
       return DAG.getNode(ISD::AND, DL, VT, X,
                          DAG.getNOT(DL, DAG.getNode(Opc, DL, VT, Y), VT));
 
+  // Fold (and X, (rot (not Y), Z)) -> (and X, (not (rot Y, Z)))
+  for (unsigned Opc : {ISD::ROTL, ISD::ROTR})
+    if (sd_match(N, m_And(m_Value(X),
+                          m_OneUse(m_BinOp(Opc, m_Value(NotY), m_Value(Z))))) &&
+        sd_match(NotY, m_Not(m_Value(Y))) &&
+        (TLI.hasAndNot(SDValue(N, 0)) || NotY->hasOneUse()))
+      return DAG.getNode(ISD::AND, DL, VT, X,
+                         DAG.getNOT(DL, DAG.getNode(Opc, DL, VT, Y, Z), VT));
+
   // Masking the negated extension of a boolean is just the zero-extended
   // boolean:
   // and (sub 0, zext(bool X)), 1 --> zext(bool X)
diff --git a/llvm/test/CodeGen/X86/andnot-patterns.ll b/llvm/test/CodeGen/X86/andnot-patterns.ll
index 1df29f0b12d1..fc573fbd4fc9 100644
--- a/llvm/test/CodeGen/X86/andnot-patterns.ll
+++ b/llvm/test/CodeGen/X86/andnot-patterns.ll
@@ -14,41 +14,73 @@ declare void @use_i32(i32)
 ;
 
 define i64 @andnot_rotl_i64(i64 %a0, i64 %a1, i64 %a2) nounwind {
-; X86-LABEL: andnot_rotl_i64:
-; X86:       # %bb.0:
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    notl %esi
-; X86-NEXT:    notl %edx
-; X86-NEXT:    testb $32, %cl
-; X86-NEXT:    jne .LBB0_1
-; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movl %edx, %eax
-; X86-NEXT:    jmp .LBB0_3
-; X86-NEXT:  .LBB0_1:
-; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    movl %edx, %esi
-; X86-NEXT:  .LBB0_3:
-; X86-NEXT:    movl %esi, %edx
-; X86-NEXT:    shldl %cl, %eax, %edx
-; X86-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X86-NEXT:    shldl %cl, %esi, %eax
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    popl %esi
-; X86-NEXT:    retl
+; X86-NOBMI-LABEL: andnot_rotl_i64:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    jne .LBB0_1
+; X86-NOBMI-NEXT:  # %bb.2:
+; X86-NOBMI-NEXT:    movl %eax, %edx
+; X86-NOBMI-NEXT:    jmp .LBB0_3
+; X86-NOBMI-NEXT:  .LBB0_1:
+; X86-NOBMI-NEXT:    movl %esi, %edx
+; X86-NOBMI-NEXT:    movl %eax, %esi
+; X86-NOBMI-NEXT:  .LBB0_3:
+; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:    shldl %cl, %edx, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-NOBMI-NEXT:    shldl %cl, %esi, %edx
+; X86-NOBMI-NEXT:    notl %edx
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    retl
 ;
-; X64-LABEL: andnot_rotl_i64:
-; X64:       # %bb.0:
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    notq %rax
-; X64-NEXT:    # kill: def $cl killed $cl killed $rcx
-; X64-NEXT:    rolq %cl, %rax
-; X64-NEXT:    andq %rdi, %rax
-; X64-NEXT:    retq
+; X86-BMI-LABEL: andnot_rotl_i64:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    pushl %esi
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI-NEXT:    testb $32, %cl
+; X86-BMI-NEXT:    jne .LBB0_1
+; X86-BMI-NEXT:  # %bb.2:
+; X86-BMI-NEXT:    movl %eax, %esi
+; X86-BMI-NEXT:    jmp .LBB0_3
+; X86-BMI-NEXT:  .LBB0_1:
+; X86-BMI-NEXT:    movl %edx, %esi
+; X86-BMI-NEXT:    movl %eax, %edx
+; X86-BMI-NEXT:  .LBB0_3:
+; X86-BMI-NEXT:    movl %edx, %eax
+; X86-BMI-NEXT:    shldl %cl, %esi, %eax
+; X86-BMI-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
+; X86-BMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-BMI-NEXT:    shldl %cl, %edx, %esi
+; X86-BMI-NEXT:    andnl {{[0-9]+}}(%esp), %esi, %edx
+; X86-BMI-NEXT:    popl %esi
+; X86-BMI-NEXT:    retl
+;
+; X64-NOBMI-LABEL: andnot_rotl_i64:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movq %rdx, %rcx
+; X64-NOBMI-NEXT:    movq %rsi, %rax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    rolq %cl, %rax
+; X64-NOBMI-NEXT:    notq %rax
+; X64-NOBMI-NEXT:    andq %rdi, %rax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_rotl_i64:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    movq %rdx, %rcx
+; X64-BMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-BMI-NEXT:    rolq %cl, %rsi
+; X64-BMI-NEXT:    andnq %rdi, %rsi, %rax
+; X64-BMI-NEXT:    retq
   %not = xor i64 %a1, -1
   %rot = tail call i64 @llvm.fshl.i64(i64 %not, i64 %not, i64 %a2)
   %and = and i64 %rot, %a0
@@ -56,24 +88,40 @@ define i64 @andnot_rotl_i64(i64 %a0, i64 %a1, i64 %a2) nounwind {
 }
 
 define i32 @andnot_rotl_i32(i32 %a0, i32 %a1, i32 %a2) nounwind {
-; X86-LABEL: andnot_rotl_i32:
-; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    notl %eax
-; X86-NEXT:    roll %cl, %eax
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    retl
+; X86-NOBMI-LABEL: andnot_rotl_i32:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    roll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    retl
 ;
-; X64-LABEL: andnot_rotl_i32:
-; X64:       # %bb.0:
-; X64-NEXT:    movl %edx, %ecx
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    notl %eax
-; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NEXT:    roll %cl, %eax
-; X64-NEXT:    andl %edi, %eax
-; X64-NEXT:    retq
+; X86-BMI-LABEL: andnot_rotl_i32:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    roll %cl, %eax
+; X86-BMI-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
+; X86-BMI-NEXT:    retl
+;
+; X64-NOBMI-LABEL: andnot_rotl_i32:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    movl %esi, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    roll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_rotl_i32:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    movl %edx, %ecx
+; X64-BMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-BMI-NEXT:    roll %cl, %esi
+; X64-BMI-NEXT:    andnl %edi, %esi, %eax
+; X64-BMI-NEXT:    retq
   %not = xor i32 %a1, -1
   %rot = tail call i32 @llvm.fshl.i32(i32 %not, i32 %not, i32 %a2)
   %and = and i32 %rot, %a0
@@ -84,23 +132,32 @@ define i16 @andnot_rotl_i16(i16 %a0, i16 %a1, i16 %a2) nounwind {
 ; X86-LABEL: andnot_rotl_i16:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    notl %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    rolw %cl, %ax
+; X86-NEXT:    notl %eax
 ; X86-NEXT:    andw {{[0-9]+}}(%esp), %ax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: andnot_rotl_i16:
-; X64:       # %bb.0:
-; X64-NEXT:    movl %edx, %ecx
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    notl %eax
-; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NEXT:    rolw %cl, %ax
-; X64-NEXT:    andl %edi, %eax
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    retq
+; X64-NOBMI-LABEL: andnot_rotl_i16:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    movl %esi, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    rolw %cl, %ax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_rotl_i16:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    movl %edx, %ecx
+; X64-BMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-BMI-NEXT:    rolw %cl, %si
+; X64-BMI-NEXT:    andnl %edi, %esi, %eax
+; X64-BMI-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-BMI-NEXT:    retq
   %not = xor i16 %a1, -1
   %rot = tail call i16 @llvm.fshl.i16(i16 %not, i16 %not, i16 %a2)
   %and = and i16 %rot, %a0
@@ -112,8 +169,8 @@ define i8 @andnot_rotl_i8(i8 %a0, i8 %a1, i8 %a2) nounwind {
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    notb %al
 ; X86-NEXT:    rolb %cl, %al
+; X86-NEXT:    notb %al
 ; X86-NEXT:    andb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    retl
 ;
@@ -121,9 +178,9 @@ define i8 @andnot_rotl_i8(i8 %a0, i8 %a1, i8 %a2) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edx, %ecx
 ; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    notb %al
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    rolb %cl, %al
+; X64-NEXT:    notb %al
 ; X64-NEXT:    andb %dil, %al
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
@@ -198,41 +255,73 @@ define i64 @andnot_rotl_i64_multiuse_rot(i64 %a0, i64 %a1, i64 %a2) nounwind {
 ;
 
 define i64 @andnot_rotr_i64(i64 %a0, i64 %a1, i64 %a2) nounwind {
-; X86-LABEL: andnot_rotr_i64:
-; X86:       # %bb.0:
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    notl %esi
-; X86-NEXT:    notl %edx
-; X86-NEXT:    testb $32, %cl
-; X86-NEXT:    je .LBB5_1
-; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movl %edx, %eax
-; X86-NEXT:    jmp .LBB5_3
-; X86-NEXT:  .LBB5_1:
-; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    movl %edx, %esi
-; X86-NEXT:  .LBB5_3:
-; X86-NEXT:    movl %esi, %edx
-; X86-NEXT:    shrdl %cl, %eax, %edx
-; X86-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X86-NEXT:    shrdl %cl, %esi, %eax
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    popl %esi
-; X86-NEXT:    retl
+; X86-NOBMI-LABEL: andnot_rotr_i64:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    je .LBB5_1
+; X86-NOBMI-NEXT:  # %bb.2:
+; X86-NOBMI-NEXT:    movl %eax, %edx
+; X86-NOBMI-NEXT:    jmp .LBB5_3
+; X86-NOBMI-NEXT:  .LBB5_1:
+; X86-NOBMI-NEXT:    movl %esi, %edx
+; X86-NOBMI-NEXT:    movl %eax, %esi
+; X86-NOBMI-NEXT:  .LBB5_3:
+; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:    shrdl %cl, %edx, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-NOBMI-NEXT:    shrdl %cl, %esi, %edx
+; X86-NOBMI-NEXT:    notl %edx
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    retl
 ;
-; X64-LABEL: andnot_rotr_i64:
-; X64:       # %bb.0:
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    notq %rax
-; X64-NEXT:    # kill: def $cl killed $cl killed $rcx
-; X64-NEXT:    rorq %cl, %rax
-; X64-NEXT:    andq %rdi, %rax
-; X64-NEXT:    retq
+; X86-BMI-LABEL: andnot_rotr_i64:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    pushl %esi
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI-NEXT:    testb $32, %cl
+; X86-BMI-NEXT:    je .LBB5_1
+; X86-BMI-NEXT:  # %bb.2:
+; X86-BMI-NEXT:    movl %eax, %esi
+; X86-BMI-NEXT:    jmp .LBB5_3
+; X86-BMI-NEXT:  .LBB5_1:
+; X86-BMI-NEXT:    movl %edx, %esi
+; X86-BMI-NEXT:    movl %eax, %edx
+; X86-BMI-NEXT:  .LBB5_3:
+; X86-BMI-NEXT:    movl %edx, %eax
+; X86-BMI-NEXT:    shrdl %cl, %esi, %eax
+; X86-BMI-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
+; X86-BMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-BMI-NEXT:    shrdl %cl, %edx, %esi
+; X86-BMI-NEXT:    andnl {{[0-9]+}}(%esp), %esi, %edx
+; X86-BMI-NEXT:    popl %esi
+; X86-BMI-NEXT:    retl
+;
+; X64-NOBMI-LABEL: andnot_rotr_i64:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movq %rdx, %rcx
+; X64-NOBMI-NEXT:    movq %rsi, %rax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    rorq %cl, %rax
+; X64-NOBMI-NEXT:    notq %rax
+; X64-NOBMI-NEXT:    andq %rdi, %rax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_rotr_i64:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    movq %rdx, %rcx
+; X64-BMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-BMI-NEXT:    rorq %cl, %rsi
+; X64-BMI-NEXT:    andnq %rdi, %rsi, %rax
+; X64-BMI-NEXT:    retq
   %not = xor i64 %a1, -1
   %rot = tail call i64 @llvm.fshr.i64(i64 %not, i64 %not, i64 %a2)
   %and = and i64 %rot, %a0
@@ -240,24 +329,40 @@ define i64 @andnot_rotr_i64(i64 %a0, i64 %a1, i64 %a2) nounwind {
 }
 
 define i32 @andnot_rotr_i32(i32 %a0, i32 %a1, i32 %a2) nounwind {
-; X86-LABEL: andnot_rotr_i32:
-; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    notl %eax
-; X86-NEXT:    rorl %cl, %eax
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    retl
+; X86-NOBMI-LABEL: andnot_rotr_i32:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    rorl %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    retl
 ;
-; X64-LABEL: andnot_rotr_i32:
-; X64:       # %bb.0:
-; X64-NEXT:    movl %edx, %ecx
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    notl %eax
-; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NEXT:    rorl %cl, %eax
-; X64-NEXT:    andl %edi, %eax
-; X64-NEXT:    retq
+; X86-BMI-LABEL: andnot_rotr_i32:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    rorl %cl, %eax
+; X86-BMI-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
+; X86-BMI-NEXT:    retl
+;
+; X64-NOBMI-LABEL: andnot_rotr_i32:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    movl %esi, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    rorl %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_rotr_i32:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    movl %edx, %ecx
+; X64-BMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-BMI-NEXT:    rorl %cl, %esi
+; X64-BMI-NEXT:    andnl %edi, %esi, %eax
+; X64-BMI-NEXT:    retq
   %not = xor i32 %a1, -1
   %rot = tail call i32 @llvm.fshr.i32(i32 %not, i32 %not, i32 %a2)
   %and = and i32 %rot, %a0
@@ -268,23 +373,32 @@ define i16 @andnot_rotr_i16(i16 %a0, i16 %a1, i16 %a2) nounwind {
 ; X86-LABEL: andnot_rotr_i16:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    notl %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    rorw %cl, %ax
+; X86-NEXT:    notl %eax
 ; X86-NEXT:    andw {{[0-9]+}}(%esp), %ax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: andnot_rotr_i16:
-; X64:       # %bb.0:
-; X64-NEXT:    movl %edx, %ecx
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    notl %eax
-; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NEXT:    rorw %cl, %ax
-; X64-NEXT:    andl %edi, %eax
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    retq
+; X64-NOBMI-LABEL: andnot_rotr_i16:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    movl %esi, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    rorw %cl, %ax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_rotr_i16:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    movl %edx, %ecx
+; X64-BMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-BMI-NEXT:    rorw %cl, %si
+; X64-BMI-NEXT:    andnl %edi, %esi, %eax
+; X64-BMI-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-BMI-NEXT:    retq
   %not = xor i16 %a1, -1
   %rot = tail call i16 @llvm.fshr.i16(i16 %not, i16 %not, i16 %a2)
   %and = and i16 %rot, %a0
@@ -296,8 +410,8 @@ define i8 @andnot_rotr_i8(i8 %a0, i8 %a1, i8 %a2) nounwind {
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    notb %al
 ; X86-NEXT:    rorb %cl, %al
+; X86-NEXT:    notb %al
 ; X86-NEXT:    andb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    retl
 ;
@@ -305,9 +419,9 @@ define i8 @andnot_rotr_i8(i8 %a0, i8 %a1, i8 %a2) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edx, %ecx
 ; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    notb %al
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    rorb %cl, %al
+; X64-NEXT:    notb %al
 ; X64-NEXT:    andb %dil, %al
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
@@ -318,36 +432,67 @@ define i8 @andnot_rotr_i8(i8 %a0, i8 %a1, i8 %a2) nounwind {
 }
 
 define i32 @andnot_rotr_i32_multiuse_not(i32 %a0, i32 %a1, i32 %a2) nounwind {
-; X86-LABEL: andnot_rotr_i32_multiuse_not:
-; X86:       # %bb.0:
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    notl %eax
-; X86-NEXT:    movl %eax, %esi
-; X86-NEXT:    rorl %cl, %esi
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll use_i32@PLT
-; X86-NEXT:    addl $4, %esp
-; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    popl %esi
-; X86-NEXT:    retl
+; X86-NOBMI-LABEL: andnot_rotr_i32_multiuse_not:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    movl %eax, %esi
+; X86-NOBMI-NEXT:    rorl %cl, %esi
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    pushl %eax
+; X86-NOBMI-NEXT:    calll use_i32@PLT
+; X86-NOBMI-NEXT:    addl $4, %esp
+; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    retl
 ;
-; X64-LABEL: andnot_rotr_i32_multiuse_not:
-; X64:       # %bb.0:
-; X64-NEXT:    pushq %rbx
-; X64-NEXT:    movl %edx, %ecx
-; X64-NEXT:    notl %esi
-; X64-NEXT:    movl %esi, %ebx
-; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NEXT:    rorl %cl, %ebx
-; X64-NEXT:    andl %edi, %ebx
-; X64-NEXT:    movl %esi, %edi
-; X64-NEXT:    callq use_i32@PLT
-; X64-NEXT:    movl %ebx, %eax
-; X64-NEXT:    popq %rbx
-; X64-NEXT:    retq
+; X86-BMI-LABEL: andnot_rotr_i32_multiuse_not:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    pushl %esi
+; X86-BMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    movl %eax, %edx
+; X86-BMI-NEXT:    notl %edx
+; X86-BMI-NEXT:    rorl %cl, %eax
+; X86-BMI-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %esi
+; X86-BMI-NEXT:    pushl %edx
+; X86-BMI-NEXT:    calll use_i32@PLT
+; X86-BMI-NEXT:    addl $4, %esp
+; X86-BMI-NEXT:    movl %esi, %eax
+; X86-BMI-NEXT:    popl %esi
+; X86-BMI-NEXT:    retl
+;
+; X64-NOBMI-LABEL: andnot_rotr_i32_multiuse_not:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    pushq %rbx
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    notl %esi
+; X64-NOBMI-NEXT:    movl %esi, %ebx
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    rorl %cl, %ebx
+; X64-NOBMI-NEXT:    andl %edi, %ebx
+; X64-NOBMI-NEXT:    movl %esi, %edi
+; X64-NOBMI-NEXT:    callq use_i32@PLT
+; X64-NOBMI-NEXT:    movl %ebx, %eax
+; X64-NOBMI-NEXT:    popq %rbx
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_rotr_i32_multiuse_not:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    pushq %rbx
+; X64-BMI-NEXT:    movl %edx, %ecx
+; X64-BMI-NEXT:    movl %esi, %eax
+; X64-BMI-NEXT:    notl %eax
+; X64-BMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-BMI-NEXT:    rorl %cl, %esi
+; X64-BMI-NEXT:    andnl %edi, %esi, %ebx
+; X64-BMI-NEXT:    movl %eax, %edi
+; X64-BMI-NEXT:    callq use_i32@PLT
+; X64-BMI-NEXT:    movl %ebx, %eax
+; X64-BMI-NEXT:    popq %rbx
+; X64-BMI-NEXT:    retq
   %not = xor i32 %a1, -1
   %rot = tail call i32 @llvm.fshr.i32(i32 %not, i32 %not, i32 %a2)
   %and = and i32 %rot, %a0
-- 
GitLab


From 0fb76bae6b2abfe5e0a34557f365a586be989364 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Wed, 30 Oct 2024 11:51:55 +0100
Subject: [PATCH 560/693] Reapply "[libc++] Simplify the implementation of
 std::sort a bit (#104902)" (#114023)

This reverts commit ef44e4659878f2. The patch was originally reverted
because it was
deemed to introduce a performance regression for small inputs, however
it also fixed
a previous performance regression for larger inputs. So overall, this
patch is desirable.
---
 libcxx/include/__algorithm/comp.h             |   3 +
 libcxx/include/__algorithm/ranges_minmax.h    |   2 +-
 libcxx/include/__algorithm/sort.h             | 285 ++++++++----------
 libcxx/include/__functional/operations.h      |  12 +
 .../include/__functional/ranges_operations.h  |   6 +
 libcxx/include/__type_traits/desugars_to.h    |   6 +
 .../__type_traits/is_trivially_copyable.h     |   4 +-
 libcxx/src/algorithm.cpp                      |   3 +-
 8 files changed, 150 insertions(+), 171 deletions(-)

diff --git a/libcxx/include/__algorithm/comp.h b/libcxx/include/__algorithm/comp.h
index 1f38f5d2d99b..ab3c59841882 100644
--- a/libcxx/include/__algorithm/comp.h
+++ b/libcxx/include/__algorithm/comp.h
@@ -42,6 +42,9 @@ struct __less<void, void> {
   }
 };
 
+template <class _Tp>
+inline const bool __desugars_to_v<__less_tag, __less<>, _Tp, _Tp> = true;
+
 template <class _Tp>
 inline const bool __desugars_to_v<__totally_ordered_less_tag, __less<>, _Tp, _Tp> = is_integral<_Tp>::value;
 
diff --git a/libcxx/include/__algorithm/ranges_minmax.h b/libcxx/include/__algorithm/ranges_minmax.h
index 4f2b2bf26382..5f2e5cb2a1ee 100644
--- a/libcxx/include/__algorithm/ranges_minmax.h
+++ b/libcxx/include/__algorithm/ranges_minmax.h
@@ -89,7 +89,7 @@ struct __minmax {
     // vectorize the code.
     if constexpr (contiguous_range<_Range> && is_integral_v<_ValueT> &&
                   __is_cheap_to_copy<_ValueT> & __is_identity<_Proj>::value &&
-                  __desugars_to_v<__totally_ordered_less_tag, _Comp, _ValueT, _ValueT>) {
+                  __desugars_to_v<__less_tag, _Comp, _ValueT, _ValueT>) {
       minmax_result<_ValueT> __result = {__r[0], __r[0]};
       for (auto __e : __r) {
         if (__e < __result.min)
diff --git a/libcxx/include/__algorithm/sort.h b/libcxx/include/__algorithm/sort.h
index 0b2137dee2f7..39868b8b6a30 100644
--- a/libcxx/include/__algorithm/sort.h
+++ b/libcxx/include/__algorithm/sort.h
@@ -27,11 +27,13 @@
 #include <__functional/ranges_operations.h>
 #include <__iterator/iterator_traits.h>
 #include <__type_traits/conditional.h>
+#include <__type_traits/desugars_to.h>
 #include <__type_traits/disjunction.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_arithmetic.h>
 #include <__type_traits/is_constant_evaluated.h>
 #include <__type_traits/is_same.h>
+#include <__type_traits/is_trivially_copyable.h>
 #include <__type_traits/remove_cvref.h>
 #include <__utility/move.h>
 #include <__utility/pair.h>
@@ -47,110 +49,11 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-// stable, 2-3 compares, 0-2 swaps
-
-template <class _AlgPolicy, class _Compare, class _ForwardIterator>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 unsigned
-__sort3(_ForwardIterator __x, _ForwardIterator __y, _ForwardIterator __z, _Compare __c) {
-  using _Ops = _IterOps<_AlgPolicy>;
-
-  unsigned __r = 0;
-  if (!__c(*__y, *__x)) // if x <= y
-  {
-    if (!__c(*__z, *__y))      // if y <= z
-      return __r;              // x <= y && y <= z
-                               // x <= y && y > z
-    _Ops::iter_swap(__y, __z); // x <= z && y < z
-    __r = 1;
-    if (__c(*__y, *__x)) // if x > y
-    {
-      _Ops::iter_swap(__x, __y); // x < y && y <= z
-      __r = 2;
-    }
-    return __r; // x <= y && y < z
-  }
-  if (__c(*__z, *__y)) // x > y, if y > z
-  {
-    _Ops::iter_swap(__x, __z); // x < y && y < z
-    __r = 1;
-    return __r;
-  }
-  _Ops::iter_swap(__x, __y); // x > y && y <= z
-  __r = 1;                   // x < y && x <= z
-  if (__c(*__z, *__y))       // if y > z
-  {
-    _Ops::iter_swap(__y, __z); // x <= y && y < z
-    __r = 2;
-  }
-  return __r;
-} // x <= y && y <= z
-
-// stable, 3-6 compares, 0-5 swaps
-
-template <class _AlgPolicy, class _Compare, class _ForwardIterator>
-_LIBCPP_HIDE_FROM_ABI void
-__sort4(_ForwardIterator __x1, _ForwardIterator __x2, _ForwardIterator __x3, _ForwardIterator __x4, _Compare __c) {
-  using _Ops = _IterOps<_AlgPolicy>;
-  std::__sort3<_AlgPolicy, _Compare>(__x1, __x2, __x3, __c);
-  if (__c(*__x4, *__x3)) {
-    _Ops::iter_swap(__x3, __x4);
-    if (__c(*__x3, *__x2)) {
-      _Ops::iter_swap(__x2, __x3);
-      if (__c(*__x2, *__x1)) {
-        _Ops::iter_swap(__x1, __x2);
-      }
-    }
-  }
-}
-
-// stable, 4-10 compares, 0-9 swaps
-
-template <class _AlgPolicy, class _Comp, class _ForwardIterator>
-_LIBCPP_HIDE_FROM_ABI void
-__sort5(_ForwardIterator __x1,
-        _ForwardIterator __x2,
-        _ForwardIterator __x3,
-        _ForwardIterator __x4,
-        _ForwardIterator __x5,
-        _Comp __comp) {
-  using _Ops = _IterOps<_AlgPolicy>;
-
-  std::__sort4<_AlgPolicy, _Comp>(__x1, __x2, __x3, __x4, __comp);
-  if (__comp(*__x5, *__x4)) {
-    _Ops::iter_swap(__x4, __x5);
-    if (__comp(*__x4, *__x3)) {
-      _Ops::iter_swap(__x3, __x4);
-      if (__comp(*__x3, *__x2)) {
-        _Ops::iter_swap(__x2, __x3);
-        if (__comp(*__x2, *__x1)) {
-          _Ops::iter_swap(__x1, __x2);
-        }
-      }
-    }
-  }
-}
-
-// The comparator being simple is a prerequisite for using the branchless optimization.
-template <class _Tp>
-struct __is_simple_comparator : false_type {};
-template <>
-struct __is_simple_comparator<__less<>&> : true_type {};
-template <class _Tp>
-struct __is_simple_comparator<less<_Tp>&> : true_type {};
-template <class _Tp>
-struct __is_simple_comparator<greater<_Tp>&> : true_type {};
-#if _LIBCPP_STD_VER >= 20
-template <>
-struct __is_simple_comparator<ranges::less&> : true_type {};
-template <>
-struct __is_simple_comparator<ranges::greater&> : true_type {};
-#endif
-
 template <class _Compare, class _Iter, class _Tp = typename iterator_traits<_Iter>::value_type>
-using __use_branchless_sort =
-    integral_constant<bool,
-                      __libcpp_is_contiguous_iterator<_Iter>::value && sizeof(_Tp) <= sizeof(void*) &&
-                          is_arithmetic<_Tp>::value && __is_simple_comparator<_Compare>::value>;
+inline const bool __use_branchless_sort =
+    __libcpp_is_contiguous_iterator<_Iter>::value && __is_cheap_to_copy<_Tp> && is_arithmetic<_Tp>::value &&
+    (__desugars_to_v<__less_tag, __remove_cvref_t<_Compare>, _Tp, _Tp> ||
+     __desugars_to_v<__greater_tag, __remove_cvref_t<_Compare>, _Tp, _Tp>);
 
 namespace __detail {
 
@@ -161,59 +64,88 @@ enum { __block_size = sizeof(uint64_t) * 8 };
 
 // Ensures that __c(*__x, *__y) is true by swapping *__x and *__y if necessary.
 template <class _Compare, class _RandomAccessIterator>
-inline _LIBCPP_HIDE_FROM_ABI void __cond_swap(_RandomAccessIterator __x, _RandomAccessIterator __y, _Compare __c) {
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+__cond_swap(_RandomAccessIterator __x, _RandomAccessIterator __y, _Compare __c) {
   // Note: this function behaves correctly even with proxy iterators (because it relies on `value_type`).
   using value_type = typename iterator_traits<_RandomAccessIterator>::value_type;
   bool __r         = __c(*__x, *__y);
   value_type __tmp = __r ? *__x : *__y;
   *__y             = __r ? *__y : *__x;
   *__x             = __tmp;
+  return !__r;
 }
 
 // Ensures that *__x, *__y and *__z are ordered according to the comparator __c,
 // under the assumption that *__y and *__z are already ordered.
 template <class _Compare, class _RandomAccessIterator>
-inline _LIBCPP_HIDE_FROM_ABI void
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
 __partially_sorted_swap(_RandomAccessIterator __x, _RandomAccessIterator __y, _RandomAccessIterator __z, _Compare __c) {
   // Note: this function behaves correctly even with proxy iterators (because it relies on `value_type`).
   using value_type = typename iterator_traits<_RandomAccessIterator>::value_type;
-  bool __r         = __c(*__z, *__x);
-  value_type __tmp = __r ? *__z : *__x;
-  *__z             = __r ? *__x : *__z;
-  __r              = __c(__tmp, *__y);
-  *__x             = __r ? *__x : *__y;
-  *__y             = __r ? *__y : __tmp;
+  bool __r1        = __c(*__z, *__x);
+  value_type __tmp = __r1 ? *__z : *__x;
+  *__z             = __r1 ? *__x : *__z;
+  bool __r2        = __c(__tmp, *__y);
+  *__x             = __r2 ? *__x : *__y;
+  *__y             = __r2 ? *__y : __tmp;
+  return !__r1 || !__r2;
 }
 
+// stable, 2-3 compares, 0-2 swaps
+
 template <class,
           class _Compare,
           class _RandomAccessIterator,
-          __enable_if_t<__use_branchless_sort<_Compare, _RandomAccessIterator>::value, int> = 0>
-inline _LIBCPP_HIDE_FROM_ABI void __sort3_maybe_branchless(
-    _RandomAccessIterator __x1, _RandomAccessIterator __x2, _RandomAccessIterator __x3, _Compare __c) {
-  std::__cond_swap<_Compare>(__x2, __x3, __c);
-  std::__partially_sorted_swap<_Compare>(__x1, __x2, __x3, __c);
+          __enable_if_t<__use_branchless_sort<_Compare, _RandomAccessIterator>, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+__sort3(_RandomAccessIterator __x1, _RandomAccessIterator __x2, _RandomAccessIterator __x3, _Compare __c) {
+  bool __swapped1 = std::__cond_swap<_Compare>(__x2, __x3, __c);
+  bool __swapped2 = std::__partially_sorted_swap<_Compare>(__x1, __x2, __x3, __c);
+  return __swapped1 || __swapped2;
 }
 
 template <class _AlgPolicy,
           class _Compare,
           class _RandomAccessIterator,
-          __enable_if_t<!__use_branchless_sort<_Compare, _RandomAccessIterator>::value, int> = 0>
-inline _LIBCPP_HIDE_FROM_ABI void __sort3_maybe_branchless(
-    _RandomAccessIterator __x1, _RandomAccessIterator __x2, _RandomAccessIterator __x3, _Compare __c) {
-  std::__sort3<_AlgPolicy, _Compare>(__x1, __x2, __x3, __c);
-}
+          __enable_if_t<!__use_branchless_sort<_Compare, _RandomAccessIterator>, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
+__sort3(_RandomAccessIterator __x, _RandomAccessIterator __y, _RandomAccessIterator __z, _Compare __c) {
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  if (!__c(*__y, *__x)) // if x <= y
+  {
+    if (!__c(*__z, *__y))        // if y <= z
+      return false;              // x <= y && y <= z
+                                 // x <= y && y > z
+    _Ops::iter_swap(__y, __z);   // x <= z && y < z
+    if (__c(*__y, *__x))         // if x > y
+      _Ops::iter_swap(__x, __y); // x < y && y <= z
+    return true;                 // x <= y && y < z
+  }
+  if (__c(*__z, *__y)) // x > y, if y > z
+  {
+    _Ops::iter_swap(__x, __z); // x < y && y < z
+    return true;
+  }
+  _Ops::iter_swap(__x, __y); // x > y && y <= z
+  // x < y && x <= z
+  if (__c(*__z, *__y))         // if y > z
+    _Ops::iter_swap(__y, __z); // x <= y && y < z
+  return true;
+} // x <= y && y <= z
+
+// stable, 3-6 compares, 0-5 swaps
 
 template <class,
           class _Compare,
           class _RandomAccessIterator,
-          __enable_if_t<__use_branchless_sort<_Compare, _RandomAccessIterator>::value, int> = 0>
-inline _LIBCPP_HIDE_FROM_ABI void __sort4_maybe_branchless(
-    _RandomAccessIterator __x1,
-    _RandomAccessIterator __x2,
-    _RandomAccessIterator __x3,
-    _RandomAccessIterator __x4,
-    _Compare __c) {
+          __enable_if_t<__use_branchless_sort<_Compare, _RandomAccessIterator>, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI void
+__sort4(_RandomAccessIterator __x1,
+        _RandomAccessIterator __x2,
+        _RandomAccessIterator __x3,
+        _RandomAccessIterator __x4,
+        _Compare __c) {
   std::__cond_swap<_Compare>(__x1, __x3, __c);
   std::__cond_swap<_Compare>(__x2, __x4, __c);
   std::__cond_swap<_Compare>(__x1, __x2, __c);
@@ -224,27 +156,39 @@ inline _LIBCPP_HIDE_FROM_ABI void __sort4_maybe_branchless(
 template <class _AlgPolicy,
           class _Compare,
           class _RandomAccessIterator,
-          __enable_if_t<!__use_branchless_sort<_Compare, _RandomAccessIterator>::value, int> = 0>
-inline _LIBCPP_HIDE_FROM_ABI void __sort4_maybe_branchless(
-    _RandomAccessIterator __x1,
-    _RandomAccessIterator __x2,
-    _RandomAccessIterator __x3,
-    _RandomAccessIterator __x4,
-    _Compare __c) {
-  std::__sort4<_AlgPolicy, _Compare>(__x1, __x2, __x3, __x4, __c);
+          __enable_if_t<!__use_branchless_sort<_Compare, _RandomAccessIterator>, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI void
+__sort4(_RandomAccessIterator __x1,
+        _RandomAccessIterator __x2,
+        _RandomAccessIterator __x3,
+        _RandomAccessIterator __x4,
+        _Compare __c) {
+  using _Ops = _IterOps<_AlgPolicy>;
+  std::__sort3<_AlgPolicy, _Compare>(__x1, __x2, __x3, __c);
+  if (__c(*__x4, *__x3)) {
+    _Ops::iter_swap(__x3, __x4);
+    if (__c(*__x3, *__x2)) {
+      _Ops::iter_swap(__x2, __x3);
+      if (__c(*__x2, *__x1)) {
+        _Ops::iter_swap(__x1, __x2);
+      }
+    }
+  }
 }
 
+// stable, 4-10 compares, 0-9 swaps
+
 template <class _AlgPolicy,
           class _Compare,
           class _RandomAccessIterator,
-          __enable_if_t<__use_branchless_sort<_Compare, _RandomAccessIterator>::value, int> = 0>
-inline _LIBCPP_HIDE_FROM_ABI void __sort5_maybe_branchless(
-    _RandomAccessIterator __x1,
-    _RandomAccessIterator __x2,
-    _RandomAccessIterator __x3,
-    _RandomAccessIterator __x4,
-    _RandomAccessIterator __x5,
-    _Compare __c) {
+          __enable_if_t<__use_branchless_sort<_Compare, _RandomAccessIterator>, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI void
+__sort5(_RandomAccessIterator __x1,
+        _RandomAccessIterator __x2,
+        _RandomAccessIterator __x3,
+        _RandomAccessIterator __x4,
+        _RandomAccessIterator __x5,
+        _Compare __c) {
   std::__cond_swap<_Compare>(__x1, __x2, __c);
   std::__cond_swap<_Compare>(__x4, __x5, __c);
   std::__partially_sorted_swap<_Compare>(__x3, __x4, __x5, __c);
@@ -256,16 +200,29 @@ inline _LIBCPP_HIDE_FROM_ABI void __sort5_maybe_branchless(
 template <class _AlgPolicy,
           class _Compare,
           class _RandomAccessIterator,
-          __enable_if_t<!__use_branchless_sort<_Compare, _RandomAccessIterator>::value, int> = 0>
-inline _LIBCPP_HIDE_FROM_ABI void __sort5_maybe_branchless(
-    _RandomAccessIterator __x1,
-    _RandomAccessIterator __x2,
-    _RandomAccessIterator __x3,
-    _RandomAccessIterator __x4,
-    _RandomAccessIterator __x5,
-    _Compare __c) {
-  std::__sort5<_AlgPolicy, _Compare, _RandomAccessIterator>(
-      std::move(__x1), std::move(__x2), std::move(__x3), std::move(__x4), std::move(__x5), __c);
+          __enable_if_t<!__use_branchless_sort<_Compare, _RandomAccessIterator>, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI void
+__sort5(_RandomAccessIterator __x1,
+        _RandomAccessIterator __x2,
+        _RandomAccessIterator __x3,
+        _RandomAccessIterator __x4,
+        _RandomAccessIterator __x5,
+        _Compare __comp) {
+  using _Ops = _IterOps<_AlgPolicy>;
+
+  std::__sort4<_AlgPolicy, _Compare>(__x1, __x2, __x3, __x4, __comp);
+  if (__comp(*__x5, *__x4)) {
+    _Ops::iter_swap(__x4, __x5);
+    if (__comp(*__x4, *__x3)) {
+      _Ops::iter_swap(__x3, __x4);
+      if (__comp(*__x3, *__x2)) {
+        _Ops::iter_swap(__x2, __x3);
+        if (__comp(*__x2, *__x1)) {
+          _Ops::iter_swap(__x1, __x2);
+        }
+      }
+    }
+  }
 }
 
 // Assumes size > 0
@@ -355,14 +312,14 @@ __insertion_sort_incomplete(_RandomAccessIterator __first, _RandomAccessIterator
       _Ops::iter_swap(__first, __last);
     return true;
   case 3:
-    std::__sort3_maybe_branchless<_AlgPolicy, _Comp>(__first, __first + difference_type(1), --__last, __comp);
+    std::__sort3<_AlgPolicy, _Comp>(__first, __first + difference_type(1), --__last, __comp);
     return true;
   case 4:
-    std::__sort4_maybe_branchless<_AlgPolicy, _Comp>(
+    std::__sort4<_AlgPolicy, _Comp>(
         __first, __first + difference_type(1), __first + difference_type(2), --__last, __comp);
     return true;
   case 5:
-    std::__sort5_maybe_branchless<_AlgPolicy, _Comp>(
+    std::__sort5<_AlgPolicy, _Comp>(
         __first,
         __first + difference_type(1),
         __first + difference_type(2),
@@ -373,7 +330,7 @@ __insertion_sort_incomplete(_RandomAccessIterator __first, _RandomAccessIterator
   }
   typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type;
   _RandomAccessIterator __j = __first + difference_type(2);
-  std::__sort3_maybe_branchless<_AlgPolicy, _Comp>(__first, __first + difference_type(1), __j, __comp);
+  std::__sort3<_AlgPolicy, _Comp>(__first, __first + difference_type(1), __j, __comp);
   const unsigned __limit = 8;
   unsigned __count       = 0;
   for (_RandomAccessIterator __i = __j + difference_type(1); __i != __last; ++__i) {
@@ -780,14 +737,14 @@ void __introsort(_RandomAccessIterator __first,
         _Ops::iter_swap(__first, __last);
       return;
     case 3:
-      std::__sort3_maybe_branchless<_AlgPolicy, _Compare>(__first, __first + difference_type(1), --__last, __comp);
+      std::__sort3<_AlgPolicy, _Compare>(__first, __first + difference_type(1), --__last, __comp);
       return;
     case 4:
-      std::__sort4_maybe_branchless<_AlgPolicy, _Compare>(
+      std::__sort4<_AlgPolicy, _Compare>(
           __first, __first + difference_type(1), __first + difference_type(2), --__last, __comp);
       return;
     case 5:
-      std::__sort5_maybe_branchless<_AlgPolicy, _Compare>(
+      std::__sort5<_AlgPolicy, _Compare>(
           __first,
           __first + difference_type(1),
           __first + difference_type(2),
@@ -928,10 +885,8 @@ __sort_dispatch(_RandomAccessIterator __first, _RandomAccessIterator __last, _Co
   // Only use bitset partitioning for arithmetic types.  We should also check
   // that the default comparator is in use so that we are sure that there are no
   // branches in the comparator.
-  std::__introsort<_AlgPolicy,
-                   _Comp&,
-                   _RandomAccessIterator,
-                   __use_branchless_sort<_Comp, _RandomAccessIterator>::value>(__first, __last, __comp, __depth_limit);
+  std::__introsort<_AlgPolicy, _Comp&, _RandomAccessIterator, __use_branchless_sort<_Comp, _RandomAccessIterator> >(
+      __first, __last, __comp, __depth_limit);
 }
 
 template <class _Type, class... _Options>
diff --git a/libcxx/include/__functional/operations.h b/libcxx/include/__functional/operations.h
index 6022bd679ed3..67d9da289aea 100644
--- a/libcxx/include/__functional/operations.h
+++ b/libcxx/include/__functional/operations.h
@@ -362,6 +362,9 @@ struct _LIBCPP_TEMPLATE_VIS less : __binary_function<_Tp, _Tp, bool> {
 };
 _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(less);
 
+template <class _Tp>
+inline const bool __desugars_to_v<__less_tag, less<_Tp>, _Tp, _Tp> = true;
+
 template <class _Tp>
 inline const bool __desugars_to_v<__totally_ordered_less_tag, less<_Tp>, _Tp, _Tp> = is_integral<_Tp>::value;
 
@@ -377,6 +380,9 @@ struct _LIBCPP_TEMPLATE_VIS less<void> {
   typedef void is_transparent;
 };
 
+template <class _Tp, class _Up>
+inline const bool __desugars_to_v<__less_tag, less<>, _Tp, _Up> = true;
+
 template <class _Tp>
 inline const bool __desugars_to_v<__totally_ordered_less_tag, less<>, _Tp, _Tp> = is_integral<_Tp>::value;
 #endif
@@ -446,6 +452,9 @@ struct _LIBCPP_TEMPLATE_VIS greater : __binary_function<_Tp, _Tp, bool> {
 };
 _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(greater);
 
+template <class _Tp>
+inline const bool __desugars_to_v<__greater_tag, greater<_Tp>, _Tp, _Tp> = true;
+
 #if _LIBCPP_STD_VER >= 14
 template <>
 struct _LIBCPP_TEMPLATE_VIS greater<void> {
@@ -457,6 +466,9 @@ struct _LIBCPP_TEMPLATE_VIS greater<void> {
   }
   typedef void is_transparent;
 };
+
+template <class _Tp, class _Up>
+inline const bool __desugars_to_v<__greater_tag, greater<>, _Tp, _Up> = true;
 #endif
 
 // Logical operations
diff --git a/libcxx/include/__functional/ranges_operations.h b/libcxx/include/__functional/ranges_operations.h
index f023d765a6c8..df95843e7c9a 100644
--- a/libcxx/include/__functional/ranges_operations.h
+++ b/libcxx/include/__functional/ranges_operations.h
@@ -102,6 +102,12 @@ inline const bool __desugars_to_v<__equal_tag, ranges::equal_to, _Tp, _Up> = tru
 template <class _Tp, class _Up>
 inline const bool __desugars_to_v<__totally_ordered_less_tag, ranges::less, _Tp, _Up> = true;
 
+template <class _Tp, class _Up>
+inline const bool __desugars_to_v<__less_tag, ranges::less, _Tp, _Up> = true;
+
+template <class _Tp, class _Up>
+inline const bool __desugars_to_v<__greater_tag, ranges::greater, _Tp, _Up> = true;
+
 #endif // _LIBCPP_STD_VER >= 20
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/desugars_to.h b/libcxx/include/__type_traits/desugars_to.h
index b0ce7c414e5d..452c70bfbad6 100644
--- a/libcxx/include/__type_traits/desugars_to.h
+++ b/libcxx/include/__type_traits/desugars_to.h
@@ -25,6 +25,12 @@ struct __equal_tag {};
 // syntactically, the operation is equivalent to calling `a + b`
 struct __plus_tag {};
 
+// syntactically, the operation is equivalent to calling `a < b`
+struct __less_tag {};
+
+// syntactically, the operation is equivalent to calling `a > b`
+struct __greater_tag {};
+
 // syntactically, the operation is equivalent to calling `a < b`, and these expressions
 // have to be true for any `a` and `b`:
 // - `(a < b) == (b > a)`
diff --git a/libcxx/include/__type_traits/is_trivially_copyable.h b/libcxx/include/__type_traits/is_trivially_copyable.h
index e92af126ee94..8eb3ba7581af 100644
--- a/libcxx/include/__type_traits/is_trivially_copyable.h
+++ b/libcxx/include/__type_traits/is_trivially_copyable.h
@@ -27,10 +27,8 @@ template <class _Tp>
 inline constexpr bool is_trivially_copyable_v = __is_trivially_copyable(_Tp);
 #endif
 
-#if _LIBCPP_STD_VER >= 20
 template <class _Tp>
-inline constexpr bool __is_cheap_to_copy = is_trivially_copyable_v<_Tp> && sizeof(_Tp) <= sizeof(std::intmax_t);
-#endif
+inline const bool __is_cheap_to_copy = __is_trivially_copyable(_Tp) && sizeof(_Tp) <= sizeof(std::intmax_t);
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/src/algorithm.cpp b/libcxx/src/algorithm.cpp
index af9d60a8e271..a7c39b5e5183 100644
--- a/libcxx/src/algorithm.cpp
+++ b/libcxx/src/algorithm.cpp
@@ -21,8 +21,7 @@ void __sort(RandomAccessIterator first, RandomAccessIterator last, Comp comp) {
   std::__introsort<_ClassicAlgPolicy,
                    ranges::less,
                    RandomAccessIterator,
-                   __use_branchless_sort<ranges::less, RandomAccessIterator>::value>(
-      first, last, ranges::less{}, depth_limit);
+                   __use_branchless_sort<ranges::less, RandomAccessIterator>>(first, last, ranges::less{}, depth_limit);
 }
 
 // clang-format off
-- 
GitLab


From f447cf15b2fcf40e519633d4cd211bb4211bfc08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alex=20R=C3=B8nne=20Petersen?= <alex@alexrp.com>
Date: Wed, 30 Oct 2024 11:55:57 +0100
Subject: [PATCH 561/693] [CSKY] Fix some typos in CPU feature descriptions
 (NFC) (#105774)

In Zig, we have a tool that updates our CPU model/feature data from
LLVM's. Noticed these typos when running it for LLVM 19.

Note: I don't have commit access.
---
 llvm/lib/Target/CSKY/CSKY.td | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/Target/CSKY/CSKY.td b/llvm/lib/Target/CSKY/CSKY.td
index 9809caa8bd8f..f88daeed8d42 100644
--- a/llvm/lib/Target/CSKY/CSKY.td
+++ b/llvm/lib/Target/CSKY/CSKY.td
@@ -97,28 +97,28 @@ def iHasFLOAT7E60 : Predicate<"Subtarget->hasFLOAT7E60()">,
              "Support CSKY float7e60 instructions">;
 
 def FeatureHWDiv : SubtargetFeature<"hwdiv", "HasHardwareDivide", "true",
-                                    "Enable divide instrutions">;
+                                    "Enable divide instructions">;
 def HasHWDiv : Predicate<"Subtarget->hasHardwareDivide()">,
                AssemblerPredicate<(all_of FeatureHWDiv),
-               "Enable divide instrutions">;
+               "Enable divide instructions">;
 
 def FeatureSTM : SubtargetFeature<"multiple_stld", "HasSTM", "true",
-                                  "Enable multiple load/store instrutions">;
+                                  "Enable multiple load/store instructions">;
 def HasSTM : Predicate<"Subtarget->hasSTM()">,
              AssemblerPredicate<(all_of FeatureSTM),
-             "Enable multiple load/store instrutions">;
+             "Enable multiple load/store instructions">;
 
 def FeaturePushPop : SubtargetFeature<"pushpop", "HasPushPop", "true",
-                                      "Enable push/pop instrutions">;
+                                      "Enable push/pop instructions">;
 def HasPushPop : Predicate<"Subtarget->hasPushPop()">,
                  AssemblerPredicate<(all_of FeaturePushPop),
-                 "Enable push/pop instrutions">;
+                 "Enable push/pop instructions">;
 
 def FeatureDSP
-    : SubtargetFeature<"edsp", "HasDSP", "true", "Enable DSP instrutions">;
+    : SubtargetFeature<"edsp", "HasDSP", "true", "Enable DSP instructions">;
 def HasDSP : Predicate<"Subtarget->hasDSP()">,
              AssemblerPredicate<(all_of FeatureDSP),
-             "Enable DSP instrutions">;
+             "Enable DSP instructions">;
 
 def HasDSP1E2
     : SubtargetFeature<"dsp1e2", "HasDSP1E2", "true", "Support CSKY dsp1e2 instructions">;
@@ -133,16 +133,16 @@ def iHasDSPE60 : Predicate<"Subtarget->hasDSPE60()">,
              "Support CSKY dspe60 instructions">;
 
 def FeatureDSPV2 : SubtargetFeature<"dspv2", "HasDSPV2", "true",
-                                    "Enable DSP V2.0 instrutions">;
+                                    "Enable DSP V2.0 instructions">;
 def HasDSPV2 : Predicate<"Subtarget->hasDSPV2()">,
                AssemblerPredicate<(all_of FeatureDSPV2),
-               "Enable DSP V2.0 instrutions">;
+               "Enable DSP V2.0 instructions">;
 
 def FeatureDSP_Silan : SubtargetFeature<"dsp_silan", "HasDSP_Silan", "true",
-                                    "Enable DSP Silan instrutions">;
+                                    "Enable DSP Silan instructions">;
 def HasDSP_Silan : Predicate<"Subtarget->hasDSP_Silan()">,
                AssemblerPredicate<(all_of FeatureDSP_Silan),
-               "Enable DSP Silan instrutions">;
+               "Enable DSP Silan instructions">;
 
 // Atomic Support
 def FeatureBTST16 : SubtargetFeature<"btst16", "HasBTST16", "true",
@@ -232,11 +232,11 @@ def FeatureSoftTP : SubtargetFeature<"soft-tp", "ReadTPHard", "false",
                                      "Disable TLS Pointer register">;
 
 def FeatureIstack : SubtargetFeature<"istack", "EnableInterruptAttribute",
-                                     "true", "Enable interrput attribute">;
+                                     "true", "Enable interrupt attribute">;
 def EnableInterruptAttribute
     : Predicate<"Subtarget->enableInterruptAttribute()">,
       AssemblerPredicate<(all_of FeatureIstack),
-      "Enable interrput attribute">;
+      "Enable interrupt attribute">;
 
 def FeatureConstPool : SubtargetFeature<"constpool", "DumpConstPool", "true",
                                         "Dump the constant pool by compiler">;
-- 
GitLab


From 092a819e94da3fc0cac6982e99861546237fcb04 Mon Sep 17 00:00:00 2001
From: Kiran Chandramohan <kiran.chandramohan@arm.com>
Date: Wed, 30 Oct 2024 10:58:26 +0000
Subject: [PATCH 562/693] [Flang][OpenMP] Add frontend support for directives
 involving master (#113893)

Issue deprecation warning for these directives.
Lowering currently supports parallel master, for all other combined or
composite directives involving master, issue TODO errors.

Note: The first commit changes the formatting and generalizes the
deprecation message emission for reuse in the second commit. I can pull
it out into a separate commit if required.
---
 .../flang/Semantics/openmp-directive-sets.h   |  1 +
 flang/lib/Parser/openmp-parsers.cpp           |  8 ++
 flang/lib/Parser/unparse.cpp                  | 15 ++++
 flang/lib/Semantics/resolve-directives.cpp    | 53 +++++++++++---
 flang/test/Lower/OpenMP/master_taskloop.f90   | 14 ++++
 .../Lower/OpenMP/master_taskloop_simd.f90     | 14 ++++
 .../OpenMP/parallel-master-taskloop-simd.f90  | 14 ++++
 .../Lower/OpenMP/parallel-master-taskloop.f90 | 14 ++++
 flang/test/Lower/OpenMP/parallel-master.f90   | 16 ++++
 flang/test/Parser/OpenMP/master-unparse.f90   | 73 +++++++++++++++++++
 .../Semantics/OpenMP/clause-validity01.f90    |  4 +-
 flang/test/Semantics/OpenMP/deprecation.f90   | 59 +++++++++++++++
 flang/test/Semantics/OpenMP/flush02.f90       |  2 +-
 .../test/Semantics/OpenMP/nested-barrier.f90  |  4 +-
 flang/test/Semantics/OpenMP/nested-master.f90 | 24 +++---
 flang/test/Semantics/OpenMP/nested-teams.f90  |  2 +-
 flang/test/Semantics/OpenMP/ordered-simd.f90  |  8 +-
 17 files changed, 293 insertions(+), 32 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/master_taskloop.f90
 create mode 100644 flang/test/Lower/OpenMP/master_taskloop_simd.f90
 create mode 100644 flang/test/Lower/OpenMP/parallel-master-taskloop-simd.f90
 create mode 100644 flang/test/Lower/OpenMP/parallel-master-taskloop.f90
 create mode 100644 flang/test/Lower/OpenMP/parallel-master.f90
 create mode 100644 flang/test/Parser/OpenMP/master-unparse.f90
 create mode 100644 flang/test/Semantics/OpenMP/deprecation.f90

diff --git a/flang/include/flang/Semantics/openmp-directive-sets.h b/flang/include/flang/Semantics/openmp-directive-sets.h
index 50d6d5b59ef7..55ef1e0ca61b 100644
--- a/flang/include/flang/Semantics/openmp-directive-sets.h
+++ b/flang/include/flang/Semantics/openmp-directive-sets.h
@@ -210,6 +210,7 @@ static const OmpDirectiveSet blockConstructSet{
     Directive::OMPD_ordered,
     Directive::OMPD_parallel,
     Directive::OMPD_parallel_masked,
+    Directive::OMPD_parallel_master,
     Directive::OMPD_parallel_workshare,
     Directive::OMPD_scope,
     Directive::OMPD_single,
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 598439cbee87..5276e1ec1dca 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -583,12 +583,19 @@ TYPE_PARSER(sourced(construct<OmpLoopDirective>(first(
     "MASKED TASKLOOP SIMD" >>
         pure(llvm::omp::Directive::OMPD_masked_taskloop_simd),
     "MASKED TASKLOOP" >> pure(llvm::omp::Directive::OMPD_masked_taskloop),
+    "MASTER TASKLOOP SIMD" >>
+        pure(llvm::omp::Directive::OMPD_master_taskloop_simd),
+    "MASTER TASKLOOP" >> pure(llvm::omp::Directive::OMPD_master_taskloop),
     "PARALLEL DO SIMD" >> pure(llvm::omp::Directive::OMPD_parallel_do_simd),
     "PARALLEL DO" >> pure(llvm::omp::Directive::OMPD_parallel_do),
     "PARALLEL MASKED TASKLOOP SIMD" >>
         pure(llvm::omp::Directive::OMPD_parallel_masked_taskloop_simd),
     "PARALLEL MASKED TASKLOOP" >>
         pure(llvm::omp::Directive::OMPD_parallel_masked_taskloop),
+    "PARALLEL MASTER TASKLOOP SIMD" >>
+        pure(llvm::omp::Directive::OMPD_parallel_master_taskloop_simd),
+    "PARALLEL MASTER TASKLOOP" >>
+        pure(llvm::omp::Directive::OMPD_parallel_master_taskloop),
     "SIMD" >> pure(llvm::omp::Directive::OMPD_simd),
     "TARGET LOOP" >> pure(llvm::omp::Directive::OMPD_target_loop),
     "TARGET PARALLEL DO SIMD" >>
@@ -706,6 +713,7 @@ TYPE_PARSER(construct<OmpBlockDirective>(first(
     "MASTER" >> pure(llvm::omp::Directive::OMPD_master),
     "ORDERED" >> pure(llvm::omp::Directive::OMPD_ordered),
     "PARALLEL MASKED" >> pure(llvm::omp::Directive::OMPD_parallel_masked),
+    "PARALLEL MASTER" >> pure(llvm::omp::Directive::OMPD_parallel_master),
     "PARALLEL WORKSHARE" >> pure(llvm::omp::Directive::OMPD_parallel_workshare),
     "PARALLEL" >> pure(llvm::omp::Directive::OMPD_parallel),
     "SCOPE" >> pure(llvm::omp::Directive::OMPD_scope),
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 39fcb61609e3..e80ab0da1360 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2274,6 +2274,12 @@ public:
     case llvm::omp::Directive::OMPD_masked_taskloop:
       Word("MASKED TASKLOOP");
       break;
+    case llvm::omp::Directive::OMPD_master_taskloop_simd:
+      Word("MASTER TASKLOOP SIMD");
+      break;
+    case llvm::omp::Directive::OMPD_master_taskloop:
+      Word("MASTER TASKLOOP");
+      break;
     case llvm::omp::Directive::OMPD_parallel_do:
       Word("PARALLEL DO ");
       break;
@@ -2286,6 +2292,12 @@ public:
     case llvm::omp::Directive::OMPD_parallel_masked_taskloop:
       Word("PARALLEL MASKED TASKLOOP");
       break;
+    case llvm::omp::Directive::OMPD_parallel_master_taskloop_simd:
+      Word("PARALLEL MASTER TASKLOOP SIMD");
+      break;
+    case llvm::omp::Directive::OMPD_parallel_master_taskloop:
+      Word("PARALLEL MASTER TASKLOOP");
+      break;
     case llvm::omp::Directive::OMPD_simd:
       Word("SIMD ");
       break;
@@ -2390,6 +2402,9 @@ public:
     case llvm::omp::Directive::OMPD_parallel_masked:
       Word("PARALLEL MASKED");
       break;
+    case llvm::omp::Directive::OMPD_parallel_master:
+      Word("PARALLEL MASTER");
+      break;
     case llvm::omp::Directive::OMPD_parallel_workshare:
       Word("PARALLEL WORKSHARE ");
       break;
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 014b7987a658..5e3ad5f3b477 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -1531,6 +1531,7 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_masked:
   case llvm::omp::Directive::OMPD_parallel_masked:
   case llvm::omp::Directive::OMPD_master:
+  case llvm::omp::Directive::OMPD_parallel_master:
   case llvm::omp::Directive::OMPD_ordered:
   case llvm::omp::Directive::OMPD_parallel:
   case llvm::omp::Directive::OMPD_scope:
@@ -1550,7 +1551,8 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
     // TODO others
     break;
   }
-  if (beginDir.v == llvm::omp::Directive::OMPD_master)
+  if (beginDir.v == llvm::omp::Directive::OMPD_master ||
+      beginDir.v == llvm::omp::Directive::OMPD_parallel_master)
     IssueNonConformanceWarning(beginDir.v, beginDir.source);
   ClearDataSharingAttributeObjects();
   ClearPrivateDataSharingAttributeObjects();
@@ -1563,7 +1565,9 @@ void OmpAttributeVisitor::Post(const parser::OpenMPBlockConstruct &x) {
   const auto &beginDir{std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
   switch (beginDir.v) {
   case llvm::omp::Directive::OMPD_masked:
+  case llvm::omp::Directive::OMPD_master:
   case llvm::omp::Directive::OMPD_parallel_masked:
+  case llvm::omp::Directive::OMPD_parallel_master:
   case llvm::omp::Directive::OMPD_parallel:
   case llvm::omp::Directive::OMPD_scope:
   case llvm::omp::Directive::OMPD_single:
@@ -1634,10 +1638,14 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPLoopConstruct &x) {
   case llvm::omp::Directive::OMPD_loop:
   case llvm::omp::Directive::OMPD_masked_taskloop_simd:
   case llvm::omp::Directive::OMPD_masked_taskloop:
+  case llvm::omp::Directive::OMPD_master_taskloop_simd:
+  case llvm::omp::Directive::OMPD_master_taskloop:
   case llvm::omp::Directive::OMPD_parallel_do:
   case llvm::omp::Directive::OMPD_parallel_do_simd:
   case llvm::omp::Directive::OMPD_parallel_masked_taskloop_simd:
   case llvm::omp::Directive::OMPD_parallel_masked_taskloop:
+  case llvm::omp::Directive::OMPD_parallel_master_taskloop_simd:
+  case llvm::omp::Directive::OMPD_parallel_master_taskloop:
   case llvm::omp::Directive::OMPD_simd:
   case llvm::omp::Directive::OMPD_target_loop:
   case llvm::omp::Directive::OMPD_target_parallel_do:
@@ -1662,7 +1670,11 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPLoopConstruct &x) {
   default:
     break;
   }
-  if (beginDir.v == llvm::omp::Directive::OMPD_target_loop)
+  if (beginDir.v == llvm::omp::OMPD_master_taskloop ||
+      beginDir.v == llvm::omp::OMPD_master_taskloop_simd ||
+      beginDir.v == llvm::omp::OMPD_parallel_master_taskloop ||
+      beginDir.v == llvm::omp::OMPD_parallel_master_taskloop_simd ||
+      beginDir.v == llvm::omp::Directive::OMPD_target_loop)
     IssueNonConformanceWarning(beginDir.v, beginDir.source);
   ClearDataSharingAttributeObjects();
   SetContextAssociatedLoopLevel(GetAssociatedLoopLevelFromClauses(clauseList));
@@ -2891,18 +2903,39 @@ void OmpAttributeVisitor::AddOmpRequiresToScope(Scope &scope,
 
 void OmpAttributeVisitor::IssueNonConformanceWarning(
     llvm::omp::Directive D, parser::CharBlock source) {
-  std::string warnStr = "";
-  std::string dirName = llvm::omp::getOpenMPDirectiveName(D).str();
+  std::string warnStr;
+  llvm::raw_string_ostream warnStrOS(warnStr);
+  warnStrOS << "OpenMP directive "
+            << parser::ToUpperCaseLetters(
+                   llvm::omp::getOpenMPDirectiveName(D).str())
+            << " has been deprecated";
+
+  auto setAlternativeStr = [&warnStrOS](llvm::StringRef alt) {
+    warnStrOS << ", please use " << alt << " instead.";
+  };
   switch (D) {
   case llvm::omp::OMPD_master:
-    warnStr = "OpenMP directive '" + dirName +
-        "' has been deprecated, please use 'masked' instead.";
+    setAlternativeStr("MASKED");
+    break;
+  case llvm::omp::OMPD_master_taskloop:
+    setAlternativeStr("MASKED TASKLOOP");
+    break;
+  case llvm::omp::OMPD_master_taskloop_simd:
+    setAlternativeStr("MASKED TASKLOOP SIMD");
+    break;
+  case llvm::omp::OMPD_parallel_master:
+    setAlternativeStr("PARALLEL MASKED");
+    break;
+  case llvm::omp::OMPD_parallel_master_taskloop:
+    setAlternativeStr("PARALLEL MASKED TASKLOOP");
+    break;
+  case llvm::omp::OMPD_parallel_master_taskloop_simd:
+    setAlternativeStr("PARALLEL_MASKED TASKLOOP SIMD");
     break;
   case llvm::omp::OMPD_target_loop:
-  default:
-    warnStr = "OpenMP directive '" + dirName + "' has been deprecated.";
+  default:;
   }
-  context_.Warn(
-      common::UsageWarning::OpenMPUsage, source, "%s"_warn_en_US, warnStr);
+  context_.Warn(common::UsageWarning::OpenMPUsage, source, "%s"_warn_en_US,
+      warnStrOS.str());
 }
 } // namespace Fortran::semantics
diff --git a/flang/test/Lower/OpenMP/master_taskloop.f90 b/flang/test/Lower/OpenMP/master_taskloop.f90
new file mode 100644
index 000000000000..26f664b2662d
--- /dev/null
+++ b/flang/test/Lower/OpenMP/master_taskloop.f90
@@ -0,0 +1,14 @@
+! This test checks lowering of OpenMP master taskloop Directive.
+
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+subroutine test_master_taskloop
+  integer :: i, j = 1
+  !CHECK: not yet implemented: Taskloop construct
+  !$omp master taskloop
+  do i=1,10
+   j = j + 1
+  end do
+  !$omp end master taskloop 
+end subroutine
diff --git a/flang/test/Lower/OpenMP/master_taskloop_simd.f90 b/flang/test/Lower/OpenMP/master_taskloop_simd.f90
new file mode 100644
index 000000000000..e928afd65244
--- /dev/null
+++ b/flang/test/Lower/OpenMP/master_taskloop_simd.f90
@@ -0,0 +1,14 @@
+! This test checks lowering of OpenMP master taskloop simd Directive.
+
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+subroutine test_master_taskloop_simd()
+  integer :: i, j = 1
+  !CHECK: not yet implemented: Composite TASKLOOP SIMD
+  !$omp master taskloop simd 
+  do i=1,10
+   j = j + 1
+  end do
+  !$omp end master taskloop simd
+end subroutine
diff --git a/flang/test/Lower/OpenMP/parallel-master-taskloop-simd.f90 b/flang/test/Lower/OpenMP/parallel-master-taskloop-simd.f90
new file mode 100644
index 000000000000..086ed01d16d3
--- /dev/null
+++ b/flang/test/Lower/OpenMP/parallel-master-taskloop-simd.f90
@@ -0,0 +1,14 @@
+! This test checks lowering of OpenMP parallel master taskloop simd Directive.
+
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+subroutine test_parallel_master_taskloop_simd
+  integer :: i, j = 1
+  !CHECK: not yet implemented: Composite TASKLOOP SIMD
+  !$omp parallel master taskloop simd 
+  do i=1,10
+   j = j + 1
+  end do
+  !$omp end parallel master taskloop simd
+end subroutine
diff --git a/flang/test/Lower/OpenMP/parallel-master-taskloop.f90 b/flang/test/Lower/OpenMP/parallel-master-taskloop.f90
new file mode 100644
index 000000000000..17ceb9496c8d
--- /dev/null
+++ b/flang/test/Lower/OpenMP/parallel-master-taskloop.f90
@@ -0,0 +1,14 @@
+! This test checks lowering of OpenMP parallel master taskloop Directive.
+
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+subroutine test_parallel_master_taskloop
+  integer :: i, j = 1
+  !CHECK: not yet implemented: Taskloop construct
+  !$omp parallel master taskloop
+  do i=1,10
+   j = j + 1
+  end do
+  !$omp end parallel master taskloop 
+end subroutine
diff --git a/flang/test/Lower/OpenMP/parallel-master.f90 b/flang/test/Lower/OpenMP/parallel-master.f90
new file mode 100644
index 000000000000..8f3ee31b3285
--- /dev/null
+++ b/flang/test/Lower/OpenMP/parallel-master.f90
@@ -0,0 +1,16 @@
+! This test checks lowering of the parallel master combined construct.
+
+! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -fopenmp -emit-hlfir %s -o - | FileCheck %s
+
+! CHECK-LABEL: func @_QPparallel_master
+subroutine parallel_master(x)
+  integer :: x
+  !CHECK: omp.parallel {
+  !CHECK: omp.master {
+  !$omp parallel master
+  x = 1
+  !$omp end parallel master
+  !CHECK: }
+  !CHECK: }
+end subroutine parallel_master
diff --git a/flang/test/Parser/OpenMP/master-unparse.f90 b/flang/test/Parser/OpenMP/master-unparse.f90
new file mode 100644
index 000000000000..30c293a521b5
--- /dev/null
+++ b/flang/test/Parser/OpenMP/master-unparse.f90
@@ -0,0 +1,73 @@
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp %s | FileCheck --ignore-case %s
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+! Check for parsing of master directive
+
+
+subroutine test_master()
+  integer :: c = 1
+  !PARSE-TREE: OmpBeginBlockDirective
+  !PARSE-TREE-NEXT: OmpBlockDirective -> llvm::omp::Directive = master
+  !CHECK: !$omp master
+  !$omp master 
+  c = c + 1
+  !$omp end master
+end subroutine
+
+subroutine test_master_taskloop_simd()
+  integer :: i, j = 1
+  !PARSE-TREE: OmpBeginLoopDirective
+  !PARSE-TREE-NEXT: OmpLoopDirective -> llvm::omp::Directive = master taskloop simd
+  !CHECK: !$omp master taskloop simd
+  !$omp master taskloop simd 
+  do i=1,10
+   j = j + 1
+  end do
+  !$omp end master taskloop simd
+end subroutine
+
+subroutine test_master_taskloop
+  integer :: i, j = 1
+  !PARSE-TREE: OmpBeginLoopDirective
+  !PARSE-TREE-NEXT: OmpLoopDirective -> llvm::omp::Directive = master taskloop
+  !CHECK: !$omp master taskloop
+  !$omp master taskloop
+  do i=1,10
+   j = j + 1
+  end do
+  !$omp end master taskloop 
+end subroutine
+
+subroutine test_parallel_master
+  integer :: c = 2
+  !PARSE-TREE: OmpBeginBlockDirective
+  !PARSE-TREE-NEXT: OmpBlockDirective -> llvm::omp::Directive = parallel master
+  !CHECK: !$omp parallel master
+  !$omp parallel master
+  c = c + 2
+  !$omp end parallel master
+end subroutine
+
+subroutine test_parallel_master_taskloop_simd
+  integer :: i, j = 1
+  !PARSE-TREE: OmpBeginLoopDirective
+  !PARSE-TREE-NEXT: OmpLoopDirective -> llvm::omp::Directive = parallel master taskloop simd
+  !CHECK: !$omp parallel master taskloop simd
+  !$omp parallel master taskloop simd 
+  do i=1,10
+   j = j + 1
+  end do
+  !$omp end parallel master taskloop simd
+end subroutine
+
+subroutine test_parallel_master_taskloop
+  integer :: i, j = 1
+  !PARSE-TREE: OmpBeginLoopDirective
+  !PARSE-TREE-NEXT: OmpLoopDirective -> llvm::omp::Directive = parallel master taskloop
+  !CHECK: !$omp parallel master taskloop
+  !$omp parallel master taskloop
+  do i=1,10
+   j = j + 1
+  end do
+  !$omp end parallel master taskloop 
+end subroutine
diff --git a/flang/test/Semantics/OpenMP/clause-validity01.f90 b/flang/test/Semantics/OpenMP/clause-validity01.f90
index 1a7a57b124e9..124f1a02d99f 100644
--- a/flang/test/Semantics/OpenMP/clause-validity01.f90
+++ b/flang/test/Semantics/OpenMP/clause-validity01.f90
@@ -476,14 +476,14 @@ use omp_lib
 ! 2.13.1 master
 
   !$omp parallel
-  !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
   !$omp master
   a=3.14
   !$omp end master
   !$omp end parallel
 
   !$omp parallel
-  !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
   !ERROR: NUM_THREADS clause is not allowed on the MASTER directive
   !$omp master num_threads(4)
   a=3.14
diff --git a/flang/test/Semantics/OpenMP/deprecation.f90 b/flang/test/Semantics/OpenMP/deprecation.f90
new file mode 100644
index 000000000000..e04f43026bbc
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/deprecation.f90
@@ -0,0 +1,59 @@
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp -Werror
+
+! Check for deprecation of master directive and its combined/composite variants
+
+subroutine test_master()
+  integer :: c = 1
+!WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
+  !$omp master 
+  c = c + 1
+  !$omp end master
+end subroutine
+
+subroutine test_parallel_master
+  integer :: c = 2
+!WARNING: OpenMP directive PARALLEL MASTER has been deprecated, please use PARALLEL MASKED instead.
+  !$omp parallel master
+  c = c + 2
+  !$omp end parallel master
+end subroutine
+
+subroutine test_master_taskloop_simd()
+  integer :: i, j = 1
+!WARNING: OpenMP directive MASTER TASKLOOP SIMD has been deprecated, please use MASKED TASKLOOP SIMD instead.
+  !$omp master taskloop simd 
+  do i=1,10
+   j = j + 1
+  end do
+  !$omp end master taskloop simd
+end subroutine
+
+subroutine test_master_taskloop
+  integer :: i, j = 1
+!WARNING: OpenMP directive MASTER TASKLOOP has been deprecated, please use MASKED TASKLOOP instead.
+  !$omp master taskloop
+  do i=1,10
+   j = j + 1
+  end do
+  !$omp end master taskloop 
+end subroutine
+
+subroutine test_parallel_master_taskloop_simd
+  integer :: i, j = 1
+!WARNING: OpenMP directive PARALLEL MASTER TASKLOOP SIMD has been deprecated, please use PARALLEL_MASKED TASKLOOP SIMD instead.
+  !$omp parallel master taskloop simd 
+  do i=1,10
+   j = j + 1
+  end do
+  !$omp end parallel master taskloop simd
+end subroutine
+
+subroutine test_parallel_master_taskloop
+  integer :: i, j = 1
+!WARNING: OpenMP directive PARALLEL MASTER TASKLOOP has been deprecated, please use PARALLEL MASKED TASKLOOP instead.
+  !$omp parallel master taskloop
+  do i=1,10
+   j = j + 1
+  end do
+  !$omp end parallel master taskloop 
+end subroutine
diff --git a/flang/test/Semantics/OpenMP/flush02.f90 b/flang/test/Semantics/OpenMP/flush02.f90
index f06719f302fd..ed0cf6602d57 100644
--- a/flang/test/Semantics/OpenMP/flush02.f90
+++ b/flang/test/Semantics/OpenMP/flush02.f90
@@ -80,7 +80,7 @@ use omp_lib
 
   !$omp parallel num_threads(4)
     array = (/1, 2, 3, 4, 5, 6, 7, 8, 9, 10/)
-    !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !$omp master
       !$omp flush (array)
     !$omp end master
diff --git a/flang/test/Semantics/OpenMP/nested-barrier.f90 b/flang/test/Semantics/OpenMP/nested-barrier.f90
index aae283229e33..7c635d8e23cc 100644
--- a/flang/test/Semantics/OpenMP/nested-barrier.f90
+++ b/flang/test/Semantics/OpenMP/nested-barrier.f90
@@ -75,7 +75,7 @@ program omp_nest_barrier
   end do
   !$omp end critical
 
-  !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
   !$omp master
   do i = 1, 10
     k = k + 1
@@ -108,7 +108,7 @@ program omp_nest_barrier
   end do
   !$omp end ordered
 
-  !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
   !$omp master
   do i = 1, 10
     !ERROR: `DISTRIBUTE` region has to be strictly nested inside `TEAMS` region.
diff --git a/flang/test/Semantics/OpenMP/nested-master.f90 b/flang/test/Semantics/OpenMP/nested-master.f90
index 069de67cafae..b21ca5d14159 100644
--- a/flang/test/Semantics/OpenMP/nested-master.f90
+++ b/flang/test/Semantics/OpenMP/nested-master.f90
@@ -9,7 +9,7 @@ program omp_nest_master
   !$omp do
   do i = 1, 10
     k = k + 1
-    !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$omp master
     j = j -1
@@ -17,7 +17,7 @@ program omp_nest_master
   end do
 
   !$omp sections 
-  !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
   !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$omp master
     do i = 1, 10
@@ -27,7 +27,7 @@ program omp_nest_master
   !$omp end sections
 
   !$omp single 
-  !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
   !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$omp master
     do i = 1, 10
@@ -41,7 +41,7 @@ program omp_nest_master
   !$omp task
   do i = 1, 10
     k = k + 1
-    !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$omp master
     j = j -1
@@ -52,7 +52,7 @@ program omp_nest_master
   !$omp taskloop
   do i = 1, 10
     k = k + 1
-    !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$omp master
     j = j -1
@@ -63,7 +63,7 @@ program omp_nest_master
   !$omp target parallel do simd
   do i = 1, 10
     k = k + 1
-    !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !ERROR: The only OpenMP constructs that can be encountered during execution of a 'SIMD' region are the `ATOMIC` construct, the `LOOP` construct, the `SIMD` construct and the `ORDERED` construct with the `SIMD` clause.
     !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$omp master
@@ -75,7 +75,7 @@ program omp_nest_master
   !$omp critical
   do i = 1, 10
     k = k + 1
-    !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !$omp master
     j = j -1
     !$omp end master
@@ -85,7 +85,7 @@ program omp_nest_master
   !$omp ordered
   do i = 1, 10
     k = k + 1
-    !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !$omp master
     j = j -1
     !$omp end master
@@ -99,7 +99,7 @@ program omp_nest_master
     !$omp distribute
     do k =1, 10
       print *, "hello"
-      !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+      !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
       !$omp master
       j = j -1
       !$omp end master
@@ -116,7 +116,7 @@ program omp_nest_master
     !$omp distribute
     do k =1, 10
       print *, "hello"
-      !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+      !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
       !$omp master
       j = j -1
       !$omp end master
@@ -133,7 +133,7 @@ program omp_nest_master
     !$omp distribute
     do k =1, 10
       print *, "hello"
-      !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+      !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
       !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
       !$omp master
       j = j -1
@@ -151,7 +151,7 @@ program omp_nest_master
     !$omp distribute
     do k =1, 10
       print *, "hello"
-      !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+      !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
       !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
       !$omp master
       j = j -1
diff --git a/flang/test/Semantics/OpenMP/nested-teams.f90 b/flang/test/Semantics/OpenMP/nested-teams.f90
index f3b96b0ab439..06eea12aba55 100644
--- a/flang/test/Semantics/OpenMP/nested-teams.f90
+++ b/flang/test/Semantics/OpenMP/nested-teams.f90
@@ -42,7 +42,7 @@ program main
   !$omp end teams
   end do
 
-  !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
   !$omp master
   !ERROR: TEAMS region can only be strictly nested within the implicit parallel region or TARGET region
   !$omp teams
diff --git a/flang/test/Semantics/OpenMP/ordered-simd.f90 b/flang/test/Semantics/OpenMP/ordered-simd.f90
index ed52b7594910..716dc42c28bb 100644
--- a/flang/test/Semantics/OpenMP/ordered-simd.f90
+++ b/flang/test/Semantics/OpenMP/ordered-simd.f90
@@ -95,7 +95,7 @@ SUBROUTINE ORDERED_BAD(N)
 
   !$OMP CRITICAL  
     C =  C - A * B
-    !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !$OMP MASTER
     DO I = 1,N
       !ERROR: `ORDERED` region may not be closely nested inside of `CRITICAL`, `ORDERED`, explicit `TASK` or `TASKLOOP` region.
@@ -108,7 +108,7 @@ SUBROUTINE ORDERED_BAD(N)
 
   !$OMP ORDERED  
     C =  C - A * B
-    !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !$OMP MASTER
     DO I = 1,N
       !ERROR: `ORDERED` region may not be closely nested inside of `CRITICAL`, `ORDERED`, explicit `TASK` or `TASKLOOP` region.
@@ -121,7 +121,7 @@ SUBROUTINE ORDERED_BAD(N)
 
   !$OMP TASK  
     C =  C - A * B
-    !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$OMP MASTER
     DO I = 1,N
@@ -136,7 +136,7 @@ SUBROUTINE ORDERED_BAD(N)
   !$OMP TASKLOOP
   DO J= 1,N  
     C =  C - A * B
-    !WARNING: OpenMP directive 'master' has been deprecated, please use 'masked' instead.
+    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$OMP MASTER
     DO I = 1,N
-- 
GitLab


From 2de1fc82861edbc484b7a1b82a37aa29d4b982de Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 30 Oct 2024 11:10:35 +0000
Subject: [PATCH 563/693] [PhaseOrdering][X86] Add additional test coverage for
 #49736

I've kept the old PR50392 tag since this is such an old issue....
---
 .../Transforms/PhaseOrdering/X86/pr50392.ll   | 64 +++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 llvm/test/Transforms/PhaseOrdering/X86/pr50392.ll

diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr50392.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr50392.ll
new file mode 100644
index 000000000000..4a024cc4c030
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pr50392.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64    -O3                   -S < %s  | FileCheck %s --check-prefixes=SSE
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v2 -O3                   -S < %s  | FileCheck %s --check-prefixes=SSE
+; RUN: opt -mtriple=x86_64-- -mcpu=btver2    -O3                   -S < %s  | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v3 -O3                   -S < %s  | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64    -passes="default<O3>" -S < %s  | FileCheck %s --check-prefixes=SSE
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v2 -passes="default<O3>" -S < %s  | FileCheck %s --check-prefixes=SSE
+; RUN: opt -mtriple=x86_64-- -mcpu=btver2    -passes="default<O3>" -S < %s  | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v3 -passes="default<O3>" -S < %s  | FileCheck %s --check-prefixes=AVX,AVX2
+
+define <4 x double> @PR50392(<4 x double> %a, <4 x double> %b) {
+; SSE-LABEL: @PR50392(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
+; SSE-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
+; SSE-NEXT:    [[VECEXT10:%.*]] = extractelement <4 x double> [[B]], i64 2
+; SSE-NEXT:    [[VECEXT11:%.*]] = extractelement <4 x double> [[B]], i64 3
+; SSE-NEXT:    [[ADD12:%.*]] = fadd double [[VECEXT10]], [[VECEXT11]]
+; SSE-NEXT:    [[SHUFFLE:%.*]] = insertelement <4 x double> [[TMP4]], double [[ADD12]], i64 3
+; SSE-NEXT:    ret <4 x double> [[SHUFFLE]]
+;
+; AVX1-LABEL: @PR50392(
+; AVX1-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
+; AVX1-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
+; AVX1-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
+; AVX1-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
+; AVX1-NEXT:    [[VECEXT10:%.*]] = extractelement <4 x double> [[B]], i64 2
+; AVX1-NEXT:    [[VECEXT11:%.*]] = extractelement <4 x double> [[B]], i64 3
+; AVX1-NEXT:    [[ADD12:%.*]] = fadd double [[VECEXT10]], [[VECEXT11]]
+; AVX1-NEXT:    [[SHUFFLE:%.*]] = insertelement <4 x double> [[TMP4]], double [[ADD12]], i64 3
+; AVX1-NEXT:    ret <4 x double> [[SHUFFLE]]
+;
+; AVX2-LABEL: @PR50392(
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
+; AVX2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
+; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
+; AVX2-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x double> [[B]], <4 x double> poison, <4 x i32> <i32 poison, i32 poison, i32 3, i32 poison>
+; AVX2-NEXT:    [[TMP5:%.*]] = fadd <4 x double> [[B]], [[SHIFT]]
+; AVX2-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> [[TMP5]], <4 x i32> <i32 0, i32 poison, i32 2, i32 6>
+; AVX2-NEXT:    ret <4 x double> [[SHUFFLE]]
+;
+  %vecext = extractelement <4 x double> %a, i32 0
+  %vecext1 = extractelement <4 x double> %a, i32 1
+  %add = fadd double %vecext, %vecext1
+  %vecinit = insertelement <4 x double> poison, double %add, i32 0
+  %vecext2 = extractelement <4 x double> %a, i32 2
+  %vecext3 = extractelement <4 x double> %a, i32 3
+  %add4 = fadd double %vecext2, %vecext3
+  %vecinit5 = insertelement <4 x double> %vecinit, double %add4, i32 1
+  %vecext6 = extractelement <4 x double> %b, i32 0
+  %vecext7 = extractelement <4 x double> %b, i32 1
+  %add8 = fadd double %vecext6, %vecext7
+  %vecinit9 = insertelement <4 x double> %vecinit5, double %add8, i32 2
+  %vecext10 = extractelement <4 x double> %b, i32 2
+  %vecext11 = extractelement <4 x double> %b, i32 3
+  %add12 = fadd double %vecext10, %vecext11
+  %vecinit13 = insertelement <4 x double> %vecinit9, double %add12, i32 3
+  %shuffle = shufflevector <4 x double> %vecinit13, <4 x double> %a, <4 x i32> <i32 0, i32 poison, i32 2, i32 3>
+  ret <4 x double> %shuffle
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX: {{.*}}
-- 
GitLab


From fcfd64304fce91747b8b03ce84919c4415a941d6 Mon Sep 17 00:00:00 2001
From: Enna1 <xumingjie.enna1@bytedance.com>
Date: Wed, 30 Oct 2024 19:25:08 +0800
Subject: [PATCH 564/693] [lld][ELF] Fix typo in help text for
 plugin-opt=opt-remarks-with-hotness (NFC) (#114016)

---
 lld/ELF/Options.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index c80c4017d351..ebe772042642 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -720,7 +720,7 @@ def: J<"plugin-opt=opt-remarks-format=">,
   HelpText<"Alias for --opt-remarks-format">;
 def: F<"plugin-opt=opt-remarks-with-hotness">,
   Alias<opt_remarks_with_hotness>,
-  HelpText<"Alias for --opt-remarks-with_hotness">;
+  HelpText<"Alias for --opt-remarks-with-hotness">;
 def: J<"plugin-opt=opt-remarks-hotness-threshold=">,
   Alias<opt_remarks_hotness_threshold>,
   HelpText<"Alias for --opt-remarks-hotness-threshold">;
-- 
GitLab


From 5dac2db5a8dab1feccc176cfb6cc4080fa5656e4 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas@arm.com>
Date: Wed, 30 Oct 2024 11:53:50 +0000
Subject: [PATCH 565/693] [FMV][AArch64] Remove features which can be expressed
 as a combination of others. (#113580)

Removes sve-bf16, sve-ebf16, and sve-i8mm since they are obsolete. One
could write target_version("sve+bf16") instead of sve-bf16 for instance.

Approved in ACLE as https://github.com/ARM-software/acle/pull/353
---
 clang/lib/Basic/Targets/AArch64.cpp           |  2 --
 .../CodeGen/aarch64-cpu-supports-target.c     |  4 ++--
 clang/test/CodeGen/aarch64-fmv-dependencies.c | 11 ----------
 clang/test/CodeGen/attr-target-version.c      | 20 +++++++++----------
 clang/test/Sema/attr-target-clones-aarch64.c  |  4 ++--
 clang/test/SemaCXX/attr-target-version.cpp    |  4 ++--
 .../builtins/cpu_model/AArch64CPUFeatures.inc |  6 +++---
 .../builtins/cpu_model/aarch64/fmv/mrs.inc    |  6 ------
 .../llvm/TargetParser/AArch64CPUFeatures.inc  |  6 +++---
 llvm/lib/Target/AArch64/AArch64FMV.td         |  3 ---
 10 files changed, 22 insertions(+), 44 deletions(-)

diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index a0f94d5d3154..3d8de0294d4b 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -765,8 +765,6 @@ bool AArch64TargetInfo::hasFeature(StringRef Feature) const {
       .Case("i8mm", HasMatMul)
       .Case("bf16", HasBFloat16)
       .Case("sve", FPU & SveMode)
-      .Case("sve-bf16", FPU & SveMode && HasBFloat16)
-      .Case("sve-i8mm", FPU & SveMode && HasMatMul)
       .Case("sve-b16b16", HasSVEB16B16)
       .Case("f32mm", FPU & SveMode && HasMatmulFP32)
       .Case("f64mm", FPU & SveMode && HasMatmulFP64)
diff --git a/clang/test/CodeGen/aarch64-cpu-supports-target.c b/clang/test/CodeGen/aarch64-cpu-supports-target.c
index 5186cab92a92..e3a75e9a1fc7 100644
--- a/clang/test/CodeGen/aarch64-cpu-supports-target.c
+++ b/clang/test/CodeGen/aarch64-cpu-supports-target.c
@@ -9,9 +9,9 @@ int check_all_feature() {
     return 3;
   else if (__builtin_cpu_supports("fcma+rcpc+rcpc2+rcpc3+frintts+dgh"))
     return 4;
-  else if (__builtin_cpu_supports("i8mm+bf16+ebf16+rpres+sve+sve-bf16"))
+  else if (__builtin_cpu_supports("i8mm+bf16+ebf16+rpres+sve"))
     return 5;
-  else if (__builtin_cpu_supports("sve-ebf16+sve-i8mm+f32mm+f64mm"))
+  else if (__builtin_cpu_supports("sve+ebf16+i8mm+f32mm+f64mm"))
     return 6;
   else if (__builtin_cpu_supports("sve2+sve2-aes+sve2-pmull128"))
     return 7;
diff --git a/clang/test/CodeGen/aarch64-fmv-dependencies.c b/clang/test/CodeGen/aarch64-fmv-dependencies.c
index 6d230007f91f..db6be423b99f 100644
--- a/clang/test/CodeGen/aarch64-fmv-dependencies.c
+++ b/clang/test/CodeGen/aarch64-fmv-dependencies.c
@@ -135,15 +135,6 @@ __attribute__((target_version("ssbs"))) int fmv(void) { return 0; }
 // CHECK: define dso_local i32 @fmv._Msve() #[[sve:[0-9]+]] {
 __attribute__((target_version("sve"))) int fmv(void) { return 0; }
 
-// CHECK: define dso_local i32 @fmv._Msve-bf16() #[[sve_bf16_ebf16:[0-9]+]] {
-__attribute__((target_version("sve-bf16"))) int fmv(void) { return 0; }
-
-// CHECK: define dso_local i32 @fmv._Msve-ebf16() #[[sve_bf16_ebf16:[0-9]+]] {
-__attribute__((target_version("sve-ebf16"))) int fmv(void) { return 0; }
-
-// CHECK: define dso_local i32 @fmv._Msve-i8mm() #[[sve_i8mm:[0-9]+]] {
-__attribute__((target_version("sve-i8mm"))) int fmv(void) { return 0; }
-
 // CHECK: define dso_local i32 @fmv._Msve2() #[[sve2:[0-9]+]] {
 __attribute__((target_version("sve2"))) int fmv(void) { return 0; }
 
@@ -209,8 +200,6 @@ int caller() {
 // CHECK: attributes #[[sme2]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+neon,+outline-atomics,+sme,+sme2,+v8a"
 // CHECK: attributes #[[ssbs]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+ssbs,+v8a"
 // CHECK: attributes #[[sve]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+v8a"
-// CHECK: attributes #[[sve_bf16_ebf16]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+v8a"
-// CHECK: attributes #[[sve_i8mm]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+i8mm,+neon,+outline-atomics,+sve,+v8a"
 // CHECK: attributes #[[sve2]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve2,+v8a"
 // CHECK: attributes #[[sve2_aes]] = { {{.*}} "target-features"="+aes,+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve2,+sve2-aes,+v8a"
 // CHECK: attributes #[[sve2_bitperm]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve2,+sve2-bitperm,+v8a"
diff --git a/clang/test/CodeGen/attr-target-version.c b/clang/test/CodeGen/attr-target-version.c
index dc0cc429abff..cd09e05b25e4 100644
--- a/clang/test/CodeGen/attr-target-version.c
+++ b/clang/test/CodeGen/attr-target-version.c
@@ -27,11 +27,11 @@ int foo() {
 inline int __attribute__((target_version("sha2+aes+f64mm"))) fmv_inline(void) { return 1; }
 inline int __attribute__((target_version("fp16+fcma+rdma+sme+ fp16 "))) fmv_inline(void) { return 2; }
 inline int __attribute__((target_version("sha3+i8mm+f32mm"))) fmv_inline(void) { return 12; }
-inline int __attribute__((target_version("dit+sve-ebf16"))) fmv_inline(void) { return 8; }
+inline int __attribute__((target_version("dit+ebf16"))) fmv_inline(void) { return 8; }
 inline int __attribute__((target_version("dpb+rcpc2 "))) fmv_inline(void) { return 6; }
 inline int __attribute__((target_version(" dpb2 + jscvt"))) fmv_inline(void) { return 7; }
 inline int __attribute__((target_version("rcpc+frintts"))) fmv_inline(void) { return 3; }
-inline int __attribute__((target_version("sve+sve-bf16"))) fmv_inline(void) { return 4; }
+inline int __attribute__((target_version("sve+bf16"))) fmv_inline(void) { return 4; }
 inline int __attribute__((target_version("sve2-aes+sve2-sha3"))) fmv_inline(void) { return 5; }
 inline int __attribute__((target_version("sve2+sve2-aes+sve2-bitperm"))) fmv_inline(void) { return 9; }
 inline int __attribute__((target_version("sve2-sm4+memtag"))) fmv_inline(void) { return 10; }
@@ -680,7 +680,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MditMsve-ebf16
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MditMebf16
 // CHECK-SAME: () #[[ATTR28:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 8
@@ -708,7 +708,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MsveMsve-bf16
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mbf16Msve
 // CHECK-SAME: () #[[ATTR32:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 4
@@ -837,20 +837,20 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 // CHECK-NEXT:    ret ptr @fmv_inline._Msve2-aesMsve2-sha3
 // CHECK:       resolver_else12:
 // CHECK-NEXT:    [[TMP28:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[TMP28]], 4295098368
-// CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[TMP29]], 4295098368
+// CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[TMP28]], 1207959552
+// CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[TMP29]], 1207959552
 // CHECK-NEXT:    [[TMP31:%.*]] = and i1 true, [[TMP30]]
 // CHECK-NEXT:    br i1 [[TMP31]], label [[RESOLVER_RETURN13:%.*]], label [[RESOLVER_ELSE14:%.*]]
 // CHECK:       resolver_return13:
-// CHECK-NEXT:    ret ptr @fmv_inline._MditMsve-ebf16
+// CHECK-NEXT:    ret ptr @fmv_inline._Mbf16Msve
 // CHECK:       resolver_else14:
 // CHECK-NEXT:    [[TMP32:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP33:%.*]] = and i64 [[TMP32]], 3221225472
-// CHECK-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 3221225472
+// CHECK-NEXT:    [[TMP33:%.*]] = and i64 [[TMP32]], 268566528
+// CHECK-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 268566528
 // CHECK-NEXT:    [[TMP35:%.*]] = and i1 true, [[TMP34]]
 // CHECK-NEXT:    br i1 [[TMP35]], label [[RESOLVER_RETURN15:%.*]], label [[RESOLVER_ELSE16:%.*]]
 // CHECK:       resolver_return15:
-// CHECK-NEXT:    ret ptr @fmv_inline._MsveMsve-bf16
+// CHECK-NEXT:    ret ptr @fmv_inline._MditMebf16
 // CHECK:       resolver_else16:
 // CHECK-NEXT:    [[TMP36:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
 // CHECK-NEXT:    [[TMP37:%.*]] = and i64 [[TMP36]], 20971520
diff --git a/clang/test/Sema/attr-target-clones-aarch64.c b/clang/test/Sema/attr-target-clones-aarch64.c
index a723c5965c5b..e101fefd2b67 100644
--- a/clang/test/Sema/attr-target-clones-aarch64.c
+++ b/clang/test/Sema/attr-target-clones-aarch64.c
@@ -7,7 +7,7 @@ void __attribute__((target_clones("default+sha3"))) warn1(void);
 
 // expected-error@+2 {{'target_clones' and 'target_version' attributes are not compatible}}
 // expected-note@+1 {{conflicting attribute is here}}
-void __attribute__((target_version("sve-bf16"), target_clones("sme+memtag"))) not_compat(void);
+void __attribute__((target_version("sve"), target_clones("sme+memtag"))) not_compat(void);
 
 int redecl(void);
 int __attribute__((target_clones("frintts", "simd+fp", "default"))) redecl(void) { return 1; }
@@ -78,4 +78,4 @@ int useage(void) {
 // expected-error@+1 {{function declaration cannot become a multiversioned function after first usage}}
 int __attribute__((target_clones("sve2-sha3+ssbs", "sm4"))) mv_after_use(void) { return 1; }
 // expected-error@+1 {{'main' cannot be a multiversioned function}}
-int __attribute__((target_clones("sve-i8mm"))) main() { return 1; }
+int __attribute__((target_clones("i8mm"))) main() { return 1; }
diff --git a/clang/test/SemaCXX/attr-target-version.cpp b/clang/test/SemaCXX/attr-target-version.cpp
index 2c85f9735a87..c0a645713b21 100644
--- a/clang/test/SemaCXX/attr-target-version.cpp
+++ b/clang/test/SemaCXX/attr-target-version.cpp
@@ -49,7 +49,7 @@ double __attribute__((target_version("rcpc"))) diff_type1(void);
 
 auto __attribute__((target_version("rcpc2"))) diff_type2(void) -> int { return 1; }
 //expected-error@+1 {{multiversioned function declaration has a different return type}}
-auto __attribute__((target_version("sve-bf16"))) diff_type2(void) -> long { return (long)1; }
+auto __attribute__((target_version("bf16"))) diff_type2(void) -> long { return (long)1; }
 
 int __attribute__((target_version("fp16fml"))) diff_type3(void) noexcept(false) { return 1; }
 //expected-error@+2 {{exception specification in declaration does not match previous declaration}}
@@ -75,7 +75,7 @@ auto __attribute__((target_version("dpb2"))) ret3(void) -> int { return 1; }
 class Cls {
   __attribute__((target_version("rng"))) Cls();
   // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support constructors}}
-  __attribute__((target_version("sve-i8mm"))) ~Cls();
+  __attribute__((target_version("i8mm"))) ~Cls();
   // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support destructors}}
 
   Cls &__attribute__((target_version("f32mm"))) operator=(const Cls &) = default;
diff --git a/compiler-rt/lib/builtins/cpu_model/AArch64CPUFeatures.inc b/compiler-rt/lib/builtins/cpu_model/AArch64CPUFeatures.inc
index 902fa8f79ab8..e454524c9cb6 100644
--- a/compiler-rt/lib/builtins/cpu_model/AArch64CPUFeatures.inc
+++ b/compiler-rt/lib/builtins/cpu_model/AArch64CPUFeatures.inc
@@ -53,9 +53,9 @@ enum CPUFeatures {
   FEAT_EBF16,
   FEAT_RPRES,
   FEAT_SVE,
-  FEAT_SVE_BF16,
-  FEAT_SVE_EBF16,
-  FEAT_SVE_I8MM,
+  RESERVED_FEAT_SVE_BF16,  // previously used and now ABI legacy
+  RESERVED_FEAT_SVE_EBF16, // previously used and now ABI legacy
+  RESERVED_FEAT_SVE_I8MM,  // previously used and now ABI legacy
   FEAT_SVE_F32MM,
   FEAT_SVE_F64MM,
   FEAT_SVE2,
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc
index 0c76a4fe9b9f..4e25feb2e90c 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc
@@ -65,14 +65,10 @@ static void __init_cpu_features_constructor(unsigned long hwcap,
     setCPUFeature(FEAT_I8MM);
   if (hwcap2 & HWCAP2_EBF16)
     setCPUFeature(FEAT_EBF16);
-  if (hwcap2 & HWCAP2_SVE_EBF16)
-    setCPUFeature(FEAT_SVE_EBF16);
   if (hwcap2 & HWCAP2_DGH)
     setCPUFeature(FEAT_DGH);
   if (hwcap2 & HWCAP2_FRINT)
     setCPUFeature(FEAT_FRINTTS);
-  if (hwcap2 & HWCAP2_SVEI8MM)
-    setCPUFeature(FEAT_SVE_I8MM);
   if (hwcap2 & HWCAP2_SVEF32MM)
     setCPUFeature(FEAT_SVE_F32MM);
   if (hwcap2 & HWCAP2_SVEF64MM)
@@ -119,8 +115,6 @@ static void __init_cpu_features_constructor(unsigned long hwcap,
     setCPUFeature(FEAT_RCPC3);
   if (hwcap2 & HWCAP2_BF16)
     setCPUFeature(FEAT_BF16);
-  if (hwcap2 & HWCAP2_SVEBF16)
-    setCPUFeature(FEAT_SVE_BF16);
   if (hwcap & HWCAP_SVE)
     setCPUFeature(FEAT_SVE);
   if (hwcap2 & HWCAP2_SVE2)
diff --git a/llvm/include/llvm/TargetParser/AArch64CPUFeatures.inc b/llvm/include/llvm/TargetParser/AArch64CPUFeatures.inc
index 902fa8f79ab8..e454524c9cb6 100644
--- a/llvm/include/llvm/TargetParser/AArch64CPUFeatures.inc
+++ b/llvm/include/llvm/TargetParser/AArch64CPUFeatures.inc
@@ -53,9 +53,9 @@ enum CPUFeatures {
   FEAT_EBF16,
   FEAT_RPRES,
   FEAT_SVE,
-  FEAT_SVE_BF16,
-  FEAT_SVE_EBF16,
-  FEAT_SVE_I8MM,
+  RESERVED_FEAT_SVE_BF16,  // previously used and now ABI legacy
+  RESERVED_FEAT_SVE_EBF16, // previously used and now ABI legacy
+  RESERVED_FEAT_SVE_I8MM,  // previously used and now ABI legacy
   FEAT_SVE_F32MM,
   FEAT_SVE_F64MM,
   FEAT_SVE2,
diff --git a/llvm/lib/Target/AArch64/AArch64FMV.td b/llvm/lib/Target/AArch64/AArch64FMV.td
index 7146b041fe5d..12d841445b80 100644
--- a/llvm/lib/Target/AArch64/AArch64FMV.td
+++ b/llvm/lib/Target/AArch64/AArch64FMV.td
@@ -81,9 +81,6 @@ def : FMVExtension<"sme-i16i64", "FEAT_SME_I64", "+sme,+sme-i16i64,+bf16", 570>;
 def : FMVExtension<"sme2", "FEAT_SME2", "+sme2,+sme,+bf16", 580>;
 def : FMVExtension<"ssbs", "FEAT_SSBS2", "+ssbs", 490>;
 def : FMVExtension<"sve", "FEAT_SVE", "+sve,+fullfp16,+fp-armv8,+neon", 310>;
-def : FMVExtension<"sve-bf16", "FEAT_SVE_BF16", "+sve,+bf16,+fullfp16,+fp-armv8,+neon", 320>;
-def : FMVExtension<"sve-ebf16", "FEAT_SVE_EBF16", "+sve,+bf16,+fullfp16,+fp-armv8,+neon", 330>;
-def : FMVExtension<"sve-i8mm", "FEAT_SVE_I8MM", "+sve,+i8mm,+fullfp16,+fp-armv8,+neon", 340>;
 def : FMVExtension<"sve2", "FEAT_SVE2", "+sve2,+sve,+fullfp16,+fp-armv8,+neon", 370>;
 def : FMVExtension<"sve2-aes", "FEAT_SVE_PMULL128", "+sve2,+sve,+aes,+sve2-aes,+fullfp16,+fp-armv8,+neon", 380>;
 def : FMVExtension<"sve2-bitperm", "FEAT_SVE_BITPERM", "+sve2,+sve,+sve2-bitperm,+fullfp16,+fp-armv8,+neon", 400>;
-- 
GitLab


From bc999ee57af61a75511f73b9544051984490344d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 30 Oct 2024 11:54:48 +0000
Subject: [PATCH 566/693] [PhaseOrdering][X86] Add test coverage for #94546

---
 .../Transforms/PhaseOrdering/X86/pr94546.ll   | 47 +++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 llvm/test/Transforms/PhaseOrdering/X86/pr94546.ll

diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr94546.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr94546.ll
new file mode 100644
index 000000000000..1d4cee45b668
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pr94546.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64    -O3                   -S < %s  | FileCheck %s --check-prefixes=SSE
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v2 -O3                   -S < %s  | FileCheck %s --check-prefixes=SSE
+; RUN: opt -mtriple=x86_64-- -mcpu=btver2    -O3                   -S < %s  | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v3 -O3                   -S < %s  | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64    -passes="default<O3>" -S < %s  | FileCheck %s --check-prefixes=SSE
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v2 -passes="default<O3>" -S < %s  | FileCheck %s --check-prefixes=SSE
+; RUN: opt -mtriple=x86_64-- -mcpu=btver2    -passes="default<O3>" -S < %s  | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v3 -passes="default<O3>" -S < %s  | FileCheck %s --check-prefixes=AVX,AVX2
+
+define <4 x double> @PR94546(<4 x double> %a, <4 x double> %b) {
+; SSE-LABEL: @PR94546(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 6>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 7>
+; SSE-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 1>
+; SSE-NEXT:    ret <4 x double> [[TMP4]]
+;
+; AVX-LABEL: @PR94546(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 6>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 7>
+; AVX-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 1>
+; AVX-NEXT:    ret <4 x double> [[TMP4]]
+;
+  %vecext = extractelement <4 x double> %a, i32 0
+  %vecext1 = extractelement <4 x double> %a, i32 1
+  %add = fadd double %vecext, %vecext1
+  %vecinit = insertelement <4 x double> poison, double %add, i32 0
+  %vecext2 = extractelement <4 x double> %a, i32 2
+  %vecext3 = extractelement <4 x double> %a, i32 3
+  %add4 = fadd double %vecext2, %vecext3
+  %vecinit5 = insertelement <4 x double> %vecinit, double %add4, i32 1
+  %vecext6 = extractelement <4 x double> %b, i32 0
+  %vecext7 = extractelement <4 x double> %b, i32 1
+  %add8 = fadd double %vecext6, %vecext7
+  %vecinit9 = insertelement <4 x double> %vecinit5, double %add8, i32 2
+  %vecext10 = extractelement <4 x double> %b, i32 2
+  %vecext11 = extractelement <4 x double> %b, i32 3
+  %add12 = fadd double %vecext10, %vecext11
+  %vecinit13 = insertelement <4 x double> %vecinit9, double %add12, i32 3
+  %shuffle = shufflevector <4 x double> %vecinit13, <4 x double> %a, <4 x i32> <i32 0, i32 poison, i32 poison, i32 3>
+  ret <4 x double> %shuffle
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX1: {{.*}}
+; AVX2: {{.*}}
-- 
GitLab


From cea9dd833cf800aeb005286b2667483cc5a8d688 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Wed, 30 Oct 2024 11:58:59 +0000
Subject: [PATCH 567/693] [CodeGen] Change MachineInstr::isConstantValuePHI to
 return Register. NFC. (#112901)

---
 llvm/include/llvm/CodeGen/MachineInstr.h | 4 ++--
 llvm/lib/CodeGen/MachineInstr.cpp        | 9 +++------
 llvm/lib/CodeGen/MachineSSAUpdater.cpp   | 2 +-
 3 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index 76a7b8662bae..360517324746 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -1764,8 +1764,8 @@ public:
   bool isDereferenceableInvariantLoad() const;
 
   /// If the specified instruction is a PHI that always merges together the
-  /// same virtual register, return the register, otherwise return 0.
-  unsigned isConstantValuePHI() const;
+  /// same virtual register, return the register, otherwise return Register().
+  Register isConstantValuePHI() const;
 
   /// Return true if this instruction has side effects that are not modeled
   /// by mayLoad / mayStore, etc.
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 0d78c2cafbaf..c1bd0bb5b716 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1535,19 +1535,16 @@ bool MachineInstr::isDereferenceableInvariantLoad() const {
   return true;
 }
 
-/// isConstantValuePHI - If the specified instruction is a PHI that always
-/// merges together the same virtual register, return the register, otherwise
-/// return 0.
-unsigned MachineInstr::isConstantValuePHI() const {
+Register MachineInstr::isConstantValuePHI() const {
   if (!isPHI())
-    return 0;
+    return {};
   assert(getNumOperands() >= 3 &&
          "It's illegal to have a PHI without source operands");
 
   Register Reg = getOperand(1).getReg();
   for (unsigned i = 3, e = getNumOperands(); i < e; i += 2)
     if (getOperand(i).getReg() != Reg)
-      return 0;
+      return {};
   return Reg;
 }
 
diff --git a/llvm/lib/CodeGen/MachineSSAUpdater.cpp b/llvm/lib/CodeGen/MachineSSAUpdater.cpp
index c7a673b12d8c..f0a136751bbf 100644
--- a/llvm/lib/CodeGen/MachineSSAUpdater.cpp
+++ b/llvm/lib/CodeGen/MachineSSAUpdater.cpp
@@ -201,7 +201,7 @@ Register MachineSSAUpdater::GetValueInMiddleOfBlock(MachineBasicBlock *BB,
 
   // See if the PHI node can be merged to a single value.  This can happen in
   // loop cases when we get a PHI of itself and one other value.
-  if (unsigned ConstVal = InsertedPHI->isConstantValuePHI()) {
+  if (Register ConstVal = InsertedPHI->isConstantValuePHI()) {
     InsertedPHI->eraseFromParent();
     return ConstVal;
   }
-- 
GitLab


From 55e4e3ff653356a9079906e209099684723caa4c Mon Sep 17 00:00:00 2001
From: Sergio Afonso <safonsof@amd.com>
Date: Wed, 30 Oct 2024 12:07:47 +0000
Subject: [PATCH 568/693] [Flang][OpenMP] Access full list of entry block syms
 and vars (NFC) (#113681)

This patch adds methods to `EntryBlockArgs` to access the full list of
entry block argument-related symbols and variables, in their standard
order. This helps centralizing this logic in as few places as possible
to avoid future inconsistencies.
---
 flang/lib/Lower/OpenMP/OpenMP.cpp | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 01a40d6e2204..876feca9b6f5 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -76,6 +76,18 @@ struct EntryBlockArgs {
            reduction.isValid() && taskReduction.isValid() &&
            useDeviceAddr.isValid() && useDevicePtr.isValid();
   }
+
+  auto getSyms() const {
+    return llvm::concat<const semantics::Symbol *const>(
+        inReduction.syms, map.syms, priv.syms, reduction.syms,
+        taskReduction.syms, useDeviceAddr.syms, useDevicePtr.syms);
+  }
+
+  auto getVars() const {
+    return llvm::concat<const mlir::Value>(
+        inReduction.vars, map.vars, priv.vars, reduction.vars,
+        taskReduction.vars, useDeviceAddr.vars, useDevicePtr.vars);
+  }
 };
 } // namespace
 
@@ -1506,8 +1518,7 @@ genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
     genEntryBlock(converter, args, op->getRegion(0));
     bindEntryBlockArgs(
         converter, llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op), args);
-    return llvm::to_vector(llvm::concat<const semantics::Symbol *const>(
-        args.priv.syms, args.reduction.syms));
+    return llvm::to_vector(args.getSyms());
   };
 
   assert((!enableDelayedPrivatization || dsp) &&
@@ -1581,11 +1592,11 @@ genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
   mlir::Operation *terminator =
       lower::genOpenMPTerminator(builder, sectionsOp, loc);
 
-  auto reductionCallback = [&](mlir::Operation *op) {
+  auto genRegionEntryCB = [&](mlir::Operation *op) {
     genEntryBlock(converter, args, op->getRegion(0));
     bindEntryBlockArgs(
         converter, llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op), args);
-    return reductionSyms;
+    return llvm::to_vector(args.getSyms());
   };
 
   // Generate nested SECTION constructs.
@@ -1611,7 +1622,7 @@ genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
         OpWithBodyGenInfo(converter, symTable, semaCtx, loc, nestedEval,
                           llvm::omp::Directive::OMPD_section)
             .setClauses(&sectionQueue.begin()->clauses)
-            .setGenRegionEntryCb(reductionCallback),
+            .setGenRegionEntryCb(genRegionEntryCB),
         sectionQueue, sectionQueue.begin());
   }
 
-- 
GitLab


From 85f3d5ca4994ff70a72f6ad81948bf4721e15ef1 Mon Sep 17 00:00:00 2001
From: SpencerAbson <Spencer.Abson@arm.com>
Date: Wed, 30 Oct 2024 12:11:34 +0000
Subject: [PATCH 569/693] [AArch64] Add assembly/disassembly for SVE COMPACT
 (b/h) and EXPAND (#114053)

This patch adds assembly/disassembly support for the following SVE2.2
instructions

      - COMPACT (byte, halfword)
      - EXPAND

- Allow selection of `COMPACT` (word/halfword) in streaming mode if the
target has FEAT_SME2p2 (see [COMPACT ](
https://developer.arm.com/documentation/ddi0602/2024-09/SVE-Instructions/COMPACT--Copy-active-vector-elements-to-lower-numbered-elements-))
- Rename predicates guarding instructions that are illegal in streaming
SVE mode without FEAT_SME2p2
- In accordance with
https://developer.arm.com/documentation/ddi0602/2024-09/SVE-Instructions
Co-authored-by: Marian Lukac marian.lukac@arm.com
---
 llvm/lib/Target/AArch64/AArch64.td            |   3 +-
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |   7 +-
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |  17 ++-
 llvm/lib/Target/AArch64/SVEInstrFormats.td    |  43 ++++++-
 .../test/MC/AArch64/SVE/compact-diagnostics.s |   4 +-
 llvm/test/MC/AArch64/SVE/compact.s            |   4 +-
 .../MC/AArch64/SVE2p2/compact-diagnostics.s   |  65 ++++++++++
 llvm/test/MC/AArch64/SVE2p2/compact.s         |  33 +++++
 .../MC/AArch64/SVE2p2/expand-diagnostics.s    | 120 ++++++++++++++++++
 llvm/test/MC/AArch64/SVE2p2/expand.s          |  39 ++++++
 10 files changed, 320 insertions(+), 15 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/compact-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/compact.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/expand-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/expand.s

diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index 9bb508b783c3..6854cccaafa1 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -73,7 +73,8 @@ def SVEUnsupported : AArch64Unsupported {
                       SVE2Unsupported.F);
 }
 
-let F = [HasSME2p2, HasSVE2p2orSME2p2] in
+let F = [HasSME2p2, HasSVE2p2orSME2p2, HasNonStreamingSVEorSME2p2,
+         HasNonStreamingSVE2p2orSME2p2] in
 def SME2p2Unsupported : AArch64Unsupported;
 
 def SME2p1Unsupported : AArch64Unsupported {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 6194de2d56b6..457e918728ae 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -244,7 +244,7 @@ def HasSVEorSME
     : Predicate<"Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME),
                 "sve or sme">;
-def HasSVEorSME2p2
+def HasNonStreamingSVEorSME2p2
     : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE()) ||"
                 "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME2p2),
@@ -281,6 +281,11 @@ def HasSMEF16F16orSMEF8F16
     : Predicate<"Subtarget->isStreaming() && (Subtarget->hasSMEF16F16() || Subtarget->hasSMEF8F16())">,
                 AssemblerPredicateWithAll<(any_of FeatureSMEF16F16, FeatureSMEF8F16),
                 "sme-f16f16 or sme-f8f16">;
+def HasNonStreamingSVE2p2orSME2p2
+    : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE2p2()) ||"
+                "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())">,
+                AssemblerPredicateWithAll<(any_of FeatureSVE2p2, FeatureSME2p2),
+                "sme2p2 or sve2p2">;
 
 // A subset of NEON instructions are legal in Streaming SVE execution mode,
 // so don't need the additional check for 'isNeonAvailable'.
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 5c5ae898a8ac..c9ee2d0059a9 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -928,9 +928,10 @@ let Predicates = [HasSVEorSME] in {
   defm SPLICE_ZPZ : sve_int_perm_splice<"splice", AArch64splice>;
 } // End HasSVEorSME
 
-let Predicates = [HasSVE] in {
-  defm COMPACT_ZPZ : sve_int_perm_compact<"compact", int_aarch64_sve_compact>;
-} // End HasSVE
+// COMPACT - word and doubleword
+let Predicates = [HasNonStreamingSVEorSME2p2] in {
+  defm COMPACT_ZPZ : sve_int_perm_compact_sd<"compact", int_aarch64_sve_compact>;
+}
 
 let Predicates = [HasSVEorSME] in {
   defm INSR_ZR : sve_int_perm_insrs<"insr", AArch64insr>;
@@ -4305,6 +4306,16 @@ let Predicates = [HasSVE2p2orSME2p2] in {
 
 } // End HasSME2p2orSVE2p2
 
+//===----------------------------------------------------------------------===//
+// SME2.2 or SVE2.2 instructions - Legal in streaming mode iff target has SME2p2
+//===----------------------------------------------------------------------===//
+let Predicates = [HasNonStreamingSVE2p2orSME2p2] in {
+  // SVE2 EXPAND
+  defm EXPAND_ZPZ : sve2_int_perm_expand<"expand">;
+  // SVE COMPACT - byte and halfword
+  defm COMPACT_ZPZ : sve_int_perm_compact_bh<"compact">;
+}
+
 //===----------------------------------------------------------------------===//
 // SVE2 FP8 instructions
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 88a0983aa148..3637a63684a0 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -7315,6 +7315,32 @@ multiclass sve2_int_perm_splice_cons<string asm> {
   def _D : sve2_int_perm_splice_cons<0b11, asm, ZPR64, ZZ_d>;
 }
 
+class sve2_int_perm_expand<bits<2> sz, string asm,
+                           ZPRRegOp zprty>
+: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn),
+  asm, "\t$Zd, $Pg, $Zn",
+  "",
+  []>, Sched<[]> {
+  bits<3> Pg;
+  bits<5> Zn;
+  bits<5> Zd;
+  let Inst{31-24} = 0b00000101;
+  let Inst{23-22} = sz;
+  let Inst{21-13} = 0b110001100;
+  let Inst{12-10} = Pg;
+  let Inst{9-5}   = Zn;
+  let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
+}
+
+multiclass sve2_int_perm_expand<string asm> {
+  def _B : sve2_int_perm_expand<0b00, asm, ZPR8>;
+  def _H : sve2_int_perm_expand<0b01, asm, ZPR16>;
+  def _S : sve2_int_perm_expand<0b10, asm, ZPR32>;
+  def _D : sve2_int_perm_expand<0b11, asm, ZPR64>;
+}
+
 class sve_int_perm_rev<bits<2> sz8_64, bits<2> opc, string asm,
                        ZPRRegOp zprty>
 : I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn),
@@ -7476,7 +7502,7 @@ multiclass sve_int_perm_cpy_v<string asm, SDPatternOperator op> {
             (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>;
 }
 
-class sve_int_perm_compact<bit sz, string asm, ZPRRegOp zprty>
+class sve_int_perm_compact<bits<2> sz, string asm, ZPRRegOp zprty>
 : I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn),
   asm, "\t$Zd, $Pg, $Zn",
   "",
@@ -7484,8 +7510,8 @@ class sve_int_perm_compact<bit sz, string asm, ZPRRegOp zprty>
   bits<3> Pg;
   bits<5> Zd;
   bits<5> Zn;
-  let Inst{31-23} = 0b000001011;
-  let Inst{22}    = sz;
+  let Inst{31-24} = 0b00000101;
+  let Inst{23-22} = sz;
   let Inst{21-13} = 0b100001100;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
@@ -7494,9 +7520,9 @@ class sve_int_perm_compact<bit sz, string asm, ZPRRegOp zprty>
   let hasSideEffects = 0;
 }
 
-multiclass sve_int_perm_compact<string asm, SDPatternOperator op> {
-  def _S : sve_int_perm_compact<0b0, asm, ZPR32>;
-  def _D : sve_int_perm_compact<0b1, asm, ZPR64>;
+multiclass sve_int_perm_compact_sd<string asm, SDPatternOperator op> {
+  def _S : sve_int_perm_compact<0b10, asm, ZPR32>;
+  def _D : sve_int_perm_compact<0b11, asm, ZPR64>;
 
   def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;
   def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
@@ -7504,6 +7530,11 @@ multiclass sve_int_perm_compact<string asm, SDPatternOperator op> {
   def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
 }
 
+multiclass sve_int_perm_compact_bh<string asm> {
+  def _B : sve_int_perm_compact<0b00, asm, ZPR8>;
+  def _H : sve_int_perm_compact<0b01, asm, ZPR16>;
+}
+
 //===----------------------------------------------------------------------===//
 // SVE Memory - Contiguous Load Group
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AArch64/SVE/compact-diagnostics.s b/llvm/test/MC/AArch64/SVE/compact-diagnostics.s
index a3d86267d917..b8ff8cc46201 100644
--- a/llvm/test/MC/AArch64/SVE/compact-diagnostics.s
+++ b/llvm/test/MC/AArch64/SVE/compact-diagnostics.s
@@ -28,12 +28,12 @@ compact z31.s, p7, z31.d
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 compact z31.b, p7, z31.b
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: compact z31.b, p7, z31.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 compact z31.h, p7, z31.h
-// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: sme2p2 or sve2p2
 // CHECK-NEXT: compact z31.h, p7, z31.h
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
diff --git a/llvm/test/MC/AArch64/SVE/compact.s b/llvm/test/MC/AArch64/SVE/compact.s
index ff815980781d..a9b47dea246b 100644
--- a/llvm/test/MC/AArch64/SVE/compact.s
+++ b/llvm/test/MC/AArch64/SVE/compact.s
@@ -12,11 +12,11 @@
 compact z31.s, p7, z31.s
 // CHECK-INST: compact z31.s, p7, z31.s
 // CHECK-ENCODING: [0xff,0x9f,0xa1,0x05]
-// CHECK-ERROR: instruction requires: sve
+// CHECK-ERROR: instruction requires: sve or sme2p2
 // CHECK-UNKNOWN: 05a19fff <unknown>
 
 compact z31.d, p7, z31.d
 // CHECK-INST: compact z31.d, p7, z31.d
 // CHECK-ENCODING: [0xff,0x9f,0xe1,0x05]
-// CHECK-ERROR: instruction requires: sve
+// CHECK-ERROR: instruction requires: sve or sme2p2
 // CHECK-UNKNOWN: 05e19fff <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2p2/compact-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/compact-diagnostics.s
new file mode 100644
index 000000000000..acf00e7f7a60
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/compact-diagnostics.s
@@ -0,0 +1,65 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2  2>&1 < %s| FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid element widths
+
+compact z31.h, p7, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: compact z31.h, p7, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+compact z31.b, p7, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: compact z31.b, p7, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid predicate operation
+
+compact z23.b, p7/m, z13.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: compact z23.b, p7/m, z13.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+compact z23.b, p7.b, z13.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: compact z23.b, p7.b, z13.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+compact z23.h, p7/z, z13.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: compact z23.h, p7/z, z13.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+compact z23.h, p7.h, z13.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: compact z23.h, p7.h, z13.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Predicate not in restricted predicate range
+
+compact z23.b, p8, z13.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: compact z23.b, p8, z13.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+compact z23.h, p8, z13.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: compact z23.h, p8, z13.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z31.b, p7/z, z6.b
+compact z31.b, p7, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: compact z31.b, p7, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z31, z6
+compact z31.h, p7, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: compact z31.h, p7, z31.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE2p2/compact.s b/llvm/test/MC/AArch64/SVE2p2/compact.s
new file mode 100644
index 000000000000..0170b3832bea
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/compact.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sve2p2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+compact z0.b, p0, z0.b  // 00000101-00100001-10000000-00000000
+// CHECK-INST: compact z0.b, p0, z0.b
+// CHECK-ENCODING: [0x00,0x80,0x21,0x05]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 05218000 <unknown>
+
+compact z21.b, p5, z10.b  // 00000101-00100001-10010101-01010101
+// CHECK-INST: compact z21.b, p5, z10.b
+// CHECK-ENCODING: [0x55,0x95,0x21,0x05]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 05219555 <unknown>
+
+compact z31.h, p7, z31.h  // 00000101-01100001-10011111-11111111
+// CHECK-INST: compact z31.h, p7, z31.h
+// CHECK-ENCODING: [0xff,0x9f,0x61,0x05]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 05619fff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/expand-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/expand-diagnostics.s
new file mode 100644
index 000000000000..b9a95f399a16
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/expand-diagnostics.s
@@ -0,0 +1,120 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2  2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid element widths.
+
+expand  z23.b, p3, z13.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: expand  z23.b, p3, z13.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+expand  z23.h, p3, z13.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: expand  z23.h, p3, z13.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+expand  z23.s, p3, z13.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: expand  z23.s, p3, z13.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+expand  z23.d, p3, z13.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: expand  z23.d, p3, z13.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+expand  z23.q, p3, z13.q
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: expand  z23.q, p3, z13.q
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid predicate operation
+
+expand  z23.b, p3/z, z13.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: expand  z23.b, p3/z, z13.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+expand  z23.b, p3.b, z13.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: expand  z23.b, p3.b, z13.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+expand  z23.h, p3/m, z13.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: expand  z23.h, p3/m, z13.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+expand  z23.h, p3.h, z13.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: expand  z23.h, p3.h, z13.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+expand  z23.s, p3/z, z13.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: expand  z23.s, p3/z, z13.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+expand  z23.s, p3.s, z13.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: expand  z23.s, p3.s, z13.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+expand  z23.d, p3/m, z13.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: expand  z23.d, p3/m, z13.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+expand  z23.d, p3.d, z13.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: expand  z23.d, p3.d, z13.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Predicate not in restricted predicate range
+
+expand  z23.b, p8, z13.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: expand  z23.b, p8, z13.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+expand  z23.b, p3.b, z13.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: expand  z23.b, p3.b, z13.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+expand  z23.h, p8, z13.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: expand  z23.h, p8, z13.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+expand  z23.h, p3.h, z13.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: expand  z23.h, p3.h, z13.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}
+
+expand  z23.s, p8, z13.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: expand  z23.s, p8, z13.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+expand  z23.d, p8, z13.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: expand  z23.d, p8, z13.d
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Negative tests for instructions that are incompatible with movprfx
+
+movprfx z31, z6
+expand  z31.b, p7, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: expand  z31.b, p7, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z31.b, p0/z, z6.b
+expand  z31.b, p0, z31.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a movprfx, suggest replacing movprfx with mov
+// CHECK-NEXT: expand  z31.b, p0, z31.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE2p2/expand.s b/llvm/test/MC/AArch64/SVE2p2/expand.s
new file mode 100644
index 000000000000..7523978380fb
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/expand.s
@@ -0,0 +1,39 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sve2p2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+expand  z0.b, p0, z0.b  // 00000101-00110001-10000000-00000000
+// CHECK-INST: expand  z0.b, p0, z0.b
+// CHECK-ENCODING: [0x00,0x80,0x31,0x05]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 05318000 <unknown>
+
+expand  z21.h, p5, z10.h  // 00000101-01110001-10010101-01010101
+// CHECK-INST: expand  z21.h, p5, z10.h
+// CHECK-ENCODING: [0x55,0x95,0x71,0x05]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 05719555 <unknown>
+
+expand  z23.s, p3, z13.s  // 00000101-10110001-10001101-10110111
+// CHECK-INST: expand  z23.s, p3, z13.s
+// CHECK-ENCODING: [0xb7,0x8d,0xb1,0x05]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 05b18db7 <unknown>
+
+expand  z31.d, p7, z31.d  // 00000101-11110001-10011111-11111111
+// CHECK-INST: expand  z31.d, p7, z31.d
+// CHECK-ENCODING: [0xff,0x9f,0xf1,0x05]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 05f19fff <unknown>
\ No newline at end of file
-- 
GitLab


From 15f63ec19cde170f4cdddc5513a0f0be9515569a Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac@arm.com>
Date: Wed, 30 Oct 2024 12:32:32 +0000
Subject: [PATCH 570/693] [AARCH64] Add assembly/disassmbly for FIRST,LASTP
 instr. (#114049)

This patch adds assembly/disassembly and tests for new FIRSTP
and LASTP instructions introduced in
https://developer.arm.com/documentation/ddi0602/2024-09

---------

Co-authored-by: SpencerAbson <Spencer.Abson@arm.com>
---
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |  5 +-
 llvm/lib/Target/AArch64/SVEInstrFormats.td    | 14 ++-
 .../MC/AArch64/SVE2p2/firstp-diagnostics.s    | 32 +++++++
 llvm/test/MC/AArch64/SVE2p2/firstp.s          | 87 +++++++++++++++++++
 .../MC/AArch64/SVE2p2/lastp-diagnostics.s     | 32 +++++++
 llvm/test/MC/AArch64/SVE2p2/lastp.s           | 87 +++++++++++++++++++
 6 files changed, 252 insertions(+), 5 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/firstp-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/firstp.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/lastp-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SVE2p2/lastp.s

diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index c9ee2d0059a9..2564ddc5f2e5 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -2129,7 +2129,7 @@ let Predicates = [HasSVEorSME] in {
   defm CNTH_XPiI : sve_int_count<0b010, "cnth", int_aarch64_sve_cnth>;
   defm CNTW_XPiI : sve_int_count<0b100, "cntw", int_aarch64_sve_cntw>;
   defm CNTD_XPiI : sve_int_count<0b110, "cntd", int_aarch64_sve_cntd>;
-  defm CNTP_XPP : sve_int_pcount_pred<0b0000, "cntp", int_aarch64_sve_cntp>;
+  defm CNTP_XPP : sve_int_pcount_pred<0b000, "cntp", int_aarch64_sve_cntp>;
 
   def : Pat<(i64 (AArch64CttzElts nxv16i1:$Op1)),
             (CNTP_XPP_B (BRKB_PPzP (PTRUE_B 31), PPR:$Op1),
@@ -4304,6 +4304,9 @@ let Predicates = [HasSVE2p2orSME2p2] in {
   def SXTW_ZPzZ_D : sve_int_un_pred_arit_z<0b11, 0b1000, "sxtw", ZPR64>;
   def UXTW_ZPzZ_D : sve_int_un_pred_arit_z<0b11, 0b1010, "uxtw", ZPR64>;
 
+  // SVE predicate count
+  defm FIRSTP_XPP : sve_int_pcount_pred_tmp<0b001, "firstp">;
+  defm LASTP_XPP  : sve_int_pcount_pred_tmp<0b010, "lastp">;
 } // End HasSME2p2orSVE2p2
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 3637a63684a0..9fa184c54570 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -1046,7 +1046,7 @@ multiclass sve_int_count_v<bits<5> opc, string asm,
                   (!cast<Instruction>(NAME # "_D") ZPR64:$Zdn, PPRAny:$Pm), 0>;
 }
 
-class sve_int_pcount_pred<bits<2> sz8_64, bits<4> opc, string asm,
+class sve_int_pcount_pred<bits<2> sz8_64, bits<3> opc, string asm,
                           PPRRegOp pprty>
 : I<(outs GPR64:$Rd), (ins PPRAny:$Pg, pprty:$Pn),
   asm, "\t$Rd, $Pg, $Pn",
@@ -1058,17 +1058,17 @@ class sve_int_pcount_pred<bits<2> sz8_64, bits<4> opc, string asm,
   let Inst{31-24} = 0b00100101;
   let Inst{23-22} = sz8_64;
   let Inst{21-19} = 0b100;
-  let Inst{18-16} = opc{3-1};
+  let Inst{18-16} = opc{2-0};
   let Inst{15-14} = 0b10;
   let Inst{13-10} = Pg;
-  let Inst{9}     = opc{0};
+  let Inst{9}     = 0b0;
   let Inst{8-5}   = Pn;
   let Inst{4-0}   = Rd;
 
   let hasSideEffects = 0;
 }
 
-multiclass sve_int_pcount_pred<bits<4> opc, string asm,
+multiclass sve_int_pcount_pred<bits<3> opc, string asm,
                                SDPatternOperator int_op> {
   def _B : sve_int_pcount_pred<0b00, opc, asm, PPR8>;
   def _H : sve_int_pcount_pred<0b01, opc, asm, PPR16>;
@@ -1081,6 +1081,12 @@ multiclass sve_int_pcount_pred<bits<4> opc, string asm,
   def : SVE_2_Op_Pat<i64, int_op, nxv2i1,  nxv2i1,  !cast<Instruction>(NAME # _D)>;
 }
 
+multiclass sve_int_pcount_pred_tmp<bits<3> opc, string asm> {
+  def _B : sve_int_pcount_pred<0b00, opc, asm, PPR8>;
+  def _H : sve_int_pcount_pred<0b01, opc, asm, PPR16>;
+  def _S : sve_int_pcount_pred<0b10, opc, asm, PPR32>;
+  def _D : sve_int_pcount_pred<0b11, opc, asm, PPR64>;
+}
 //===----------------------------------------------------------------------===//
 // SVE Element Count Group
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AArch64/SVE2p2/firstp-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/firstp-diagnostics.s
new file mode 100644
index 000000000000..4309fd49ecf7
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/firstp-diagnostics.s
@@ -0,0 +1,32 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate operand
+
+firstp  x0, p15, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register.
+// CHECK-NEXT: firstp  x0, p15, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+firstp  x0, p15.b, p0.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register.
+// CHECK-NEXT: firstp  x0, p15.b, p0.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+firstp  x0, p15.q, p0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register.
+// CHECK-NEXT: firstp  x0, p15.q, p0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid register types
+
+firstp  sp, p15, p0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: firstp  sp, p15, p0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+firstp  w0, p15, p0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: firstp  w0, p15, p0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE2p2/firstp.s b/llvm/test/MC/AArch64/SVE2p2/firstp.s
new file mode 100644
index 000000000000..629bee5576fc
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/firstp.s
@@ -0,0 +1,87 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+firstp  x0, p0, p0.b  // 00100101-00100001-10000000-00000000
+// CHECK-INST: firstp  x0, p0, p0.b
+// CHECK-ENCODING: [0x00,0x80,0x21,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25218000 <unknown>
+
+firstp  x23, p11, p13.b  // 00100101-00100001-10101101-10110111
+// CHECK-INST: firstp  x23, p11, p13.b
+// CHECK-ENCODING: [0xb7,0xad,0x21,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 2521adb7 <unknown>
+
+firstp  xzr, p15, p15.b  // 00100101-00100001-10111101-11111111
+// CHECK-INST: firstp  xzr, p15, p15.b
+// CHECK-ENCODING: [0xff,0xbd,0x21,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 2521bdff <unknown>
+
+firstp  x0, p0, p0.h  // 00100101-01100001-10000000-00000000
+// CHECK-INST: firstp  x0, p0, p0.h
+// CHECK-ENCODING: [0x00,0x80,0x61,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25618000 <unknown>
+
+firstp  x23, p11, p13.h  // 00100101-01100001-10101101-10110111
+// CHECK-INST: firstp  x23, p11, p13.h
+// CHECK-ENCODING: [0xb7,0xad,0x61,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 2561adb7 <unknown>
+
+firstp  xzr, p15, p15.h  // 00100101-01100001-10111101-11111111
+// CHECK-INST: firstp  xzr, p15, p15.h
+// CHECK-ENCODING: [0xff,0xbd,0x61,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 2561bdff <unknown>
+
+firstp  x0, p0, p0.s  // 00100101-10100001-10000000-00000000
+// CHECK-INST: firstp  x0, p0, p0.s
+// CHECK-ENCODING: [0x00,0x80,0xa1,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25a18000 <unknown>
+
+firstp  x23, p11, p13.s  // 00100101-10100001-10101101-10110111
+// CHECK-INST: firstp  x23, p11, p13.s
+// CHECK-ENCODING: [0xb7,0xad,0xa1,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25a1adb7 <unknown>
+
+firstp  xzr, p15, p15.s  // 00100101-10100001-10111101-11111111
+// CHECK-INST: firstp  xzr, p15, p15.s
+// CHECK-ENCODING: [0xff,0xbd,0xa1,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25a1bdff <unknown>
+
+firstp  x0, p0, p0.d  // 00100101-11100001-10000000-00000000
+// CHECK-INST: firstp  x0, p0, p0.d
+// CHECK-ENCODING: [0x00,0x80,0xe1,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25e18000 <unknown>
+
+firstp  x23, p11, p13.d  // 00100101-11100001-10101101-10110111
+// CHECK-INST: firstp  x23, p11, p13.d
+// CHECK-ENCODING: [0xb7,0xad,0xe1,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25e1adb7 <unknown>
+
+firstp  xzr, p15, p15.d  // 00100101-11100001-10111101-11111111
+// CHECK-INST: firstp  xzr, p15, p15.d
+// CHECK-ENCODING: [0xff,0xbd,0xe1,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25e1bdff <unknown>
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/lastp-diagnostics.s b/llvm/test/MC/AArch64/SVE2p2/lastp-diagnostics.s
new file mode 100644
index 000000000000..e277bdbc6aa8
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/lastp-diagnostics.s
@@ -0,0 +1,32 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate operand
+
+lastp  x0, p15, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register.
+// CHECK-NEXT: lastp  x0, p15, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+lastp  x0, p15.b, p0.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register.
+// CHECK-NEXT: lastp  x0, p15.b, p0.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+lastp  x0, p15.q, p0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register.
+// CHECK-NEXT: lastp  x0, p15.q, p0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Invalid register types
+
+lastp  sp, p15, p0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: lastp  sp, p15, p0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+lastp  w0, p15, p0.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: lastp  w0, p15, p0.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/SVE2p2/lastp.s b/llvm/test/MC/AArch64/SVE2p2/lastp.s
new file mode 100644
index 000000000000..1ffa0a7d1fcc
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE2p2/lastp.s
@@ -0,0 +1,87 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+lastp   x0, p0, p0.b  // 00100101-00100010-10000000-00000000
+// CHECK-INST: lastp   x0, p0, p0.b
+// CHECK-ENCODING: [0x00,0x80,0x22,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25228000 <unknown>
+
+lastp   x23, p11, p13.b  // 00100101-00100010-10101101-10110111
+// CHECK-INST: lastp   x23, p11, p13.b
+// CHECK-ENCODING: [0xb7,0xad,0x22,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 2522adb7 <unknown>
+
+lastp   xzr, p15, p15.b  // 00100101-00100010-10111101-11111111
+// CHECK-INST: lastp   xzr, p15, p15.b
+// CHECK-ENCODING: [0xff,0xbd,0x22,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 2522bdff <unknown>
+
+lastp   x0, p0, p0.h  // 00100101-01100010-10000000-00000000
+// CHECK-INST: lastp   x0, p0, p0.h
+// CHECK-ENCODING: [0x00,0x80,0x62,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25628000 <unknown>
+
+lastp   x23, p11, p13.h  // 00100101-01100010-10101101-10110111
+// CHECK-INST: lastp   x23, p11, p13.h
+// CHECK-ENCODING: [0xb7,0xad,0x62,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 2562adb7 <unknown>
+
+lastp   xzr, p15, p15.h  // 00100101-01100010-10111101-11111111
+// CHECK-INST: lastp   xzr, p15, p15.h
+// CHECK-ENCODING: [0xff,0xbd,0x62,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 2562bdff <unknown>
+
+lastp   x0, p0, p0.s  // 00100101-10100010-10000000-00000000
+// CHECK-INST: lastp   x0, p0, p0.s
+// CHECK-ENCODING: [0x00,0x80,0xa2,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25a28000 <unknown>
+
+lastp   x23, p11, p13.s  // 00100101-10100010-10101101-10110111
+// CHECK-INST: lastp   x23, p11, p13.s
+// CHECK-ENCODING: [0xb7,0xad,0xa2,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25a2adb7 <unknown>
+
+lastp   xzr, p15, p15.s  // 00100101-10100010-10111101-11111111
+// CHECK-INST: lastp   xzr, p15, p15.s
+// CHECK-ENCODING: [0xff,0xbd,0xa2,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25a2bdff <unknown>
+
+lastp   x0, p0, p0.d  // 00100101-11100010-10000000-00000000
+// CHECK-INST: lastp   x0, p0, p0.d
+// CHECK-ENCODING: [0x00,0x80,0xe2,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25e28000 <unknown>
+
+lastp   x23, p11, p13.d  // 00100101-11100010-10101101-10110111
+// CHECK-INST: lastp   x23, p11, p13.d
+// CHECK-ENCODING: [0xb7,0xad,0xe2,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25e2adb7 <unknown>
+
+lastp   xzr, p15, p15.d  // 00100101-11100010-10111101-11111111
+// CHECK-INST: lastp   xzr, p15, p15.d
+// CHECK-ENCODING: [0xff,0xbd,0xe2,0x25]
+// CHECK-ERROR: instruction requires: sme2p2 or sve2p2
+// CHECK-UNKNOWN: 25e2bdff <unknown>
\ No newline at end of file
-- 
GitLab


From ea050ab1a99547294e195064bd90ca9822d292cf Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Wed, 30 Oct 2024 21:36:39 +0900
Subject: [PATCH 571/693] [mlir][Transforms][NFC] Dialect conversion: Reformat
 materialization error message (#114176)

This commit changes the format of the materialization error message.

Previously: `failed to legalize unresolved materialization from ('f64')
to 'f32' that remained live after conversion`
Now: `failed to legalize unresolved materialization from ('f64') to
('f32') that remained live after conversion`

This commit is in preparation of merging the 1:1 and 1:N dialect
conversions. At that point, target materializations may create more than
one SSA value. I am sending this change as a separate PR to keep the
main PR smaller.
---
 mlir/lib/Transforms/Utils/DialectConversion.cpp        | 10 +++++-----
 .../Bufferization/Transforms/finalizing-bufferize.mlir |  2 +-
 .../Transforms/test-legalize-erased-op-with-uses.mlir  |  2 +-
 .../test/Transforms/test-legalize-type-conversion.mlir | 10 +++++-----
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 44cf8331d55a..0a62628b9ad2 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -2457,11 +2457,11 @@ legalizeUnresolvedMaterialization(RewriterBase &rewriter,
     }
   }
 
-  InFlightDiagnostic diag = op->emitError()
-                            << "failed to legalize unresolved materialization "
-                               "from ("
-                            << inputOperands.getTypes() << ") to " << outputType
-                            << " that remained live after conversion";
+  InFlightDiagnostic diag =
+      op->emitError() << "failed to legalize unresolved materialization "
+                         "from ("
+                      << inputOperands.getTypes() << ") to (" << outputType
+                      << ") that remained live after conversion";
   diag.attachNote(op->getUsers().begin()->getLoc())
       << "see existing live user here: " << *op->getUsers().begin();
   return failure();
diff --git a/mlir/test/Dialect/Bufferization/Transforms/finalizing-bufferize.mlir b/mlir/test/Dialect/Bufferization/Transforms/finalizing-bufferize.mlir
index ab18ce05e355..bae94c1be4da 100644
--- a/mlir/test/Dialect/Bufferization/Transforms/finalizing-bufferize.mlir
+++ b/mlir/test/Dialect/Bufferization/Transforms/finalizing-bufferize.mlir
@@ -78,7 +78,7 @@ func.func @static_layout_to_no_layout_cast(%m: memref<?xf32, strided<[1], offset
 // memref.cast.
 func.func @no_layout_to_dyn_layout_cast(%m: memref<?xf32>) -> memref<?xf32, strided<[1], offset: ?>> {
   %0 = bufferization.to_tensor %m : memref<?xf32>
-  // expected-error @+1 {{failed to legalize unresolved materialization from ('memref<?xf32>') to 'memref<?xf32, strided<[1], offset: ?>>' that remained live after conversion}}
+  // expected-error @+1 {{failed to legalize unresolved materialization from ('memref<?xf32>') to ('memref<?xf32, strided<[1], offset: ?>>') that remained live after conversion}}
   %1 = bufferization.to_memref %0 : memref<?xf32, strided<[1], offset: ?>>
   // expected-note @below{{see existing live user here}}
   return %1 : memref<?xf32, strided<[1], offset: ?>>
diff --git a/mlir/test/Transforms/test-legalize-erased-op-with-uses.mlir b/mlir/test/Transforms/test-legalize-erased-op-with-uses.mlir
index 6e8f0162e505..031442b0ee2d 100644
--- a/mlir/test/Transforms/test-legalize-erased-op-with-uses.mlir
+++ b/mlir/test/Transforms/test-legalize-erased-op-with-uses.mlir
@@ -3,7 +3,7 @@
 // Test that an error is emitted when an operation is marked as "erased", but
 // has users that live across the conversion.
 func.func @remove_all_ops(%arg0: i32) -> i32 {
-  // expected-error@below {{failed to legalize unresolved materialization from () to 'i32' that remained live after conversion}}
+  // expected-error@below {{failed to legalize unresolved materialization from () to ('i32') that remained live after conversion}}
   %0 = "test.illegal_op_a"() : () -> i32
   // expected-note@below {{see existing live user here}}
   return %0 : i32
diff --git a/mlir/test/Transforms/test-legalize-type-conversion.mlir b/mlir/test/Transforms/test-legalize-type-conversion.mlir
index f130adff42f8..db8bd0f6378d 100644
--- a/mlir/test/Transforms/test-legalize-type-conversion.mlir
+++ b/mlir/test/Transforms/test-legalize-type-conversion.mlir
@@ -2,7 +2,7 @@
 
 
 func.func @test_invalid_arg_materialization(
-  // expected-error@below {{failed to legalize unresolved materialization from () to 'i16' that remained live after conversion}}
+  // expected-error@below {{failed to legalize unresolved materialization from () to ('i16') that remained live after conversion}}
   %arg0: i16) {
   // expected-note@below{{see existing live user here}}
   "foo.return"(%arg0) : (i16) -> ()
@@ -21,7 +21,7 @@ func.func @test_valid_arg_materialization(%arg0: i64) {
 // -----
 
 func.func @test_invalid_result_materialization() {
-  // expected-error@below {{failed to legalize unresolved materialization from ('f64') to 'f16' that remained live after conversion}}
+  // expected-error@below {{failed to legalize unresolved materialization from ('f64') to ('f16') that remained live after conversion}}
   %result = "test.type_producer"() : () -> f16
   // expected-note@below{{see existing live user here}}
   "foo.return"(%result) : (f16) -> ()
@@ -30,7 +30,7 @@ func.func @test_invalid_result_materialization() {
 // -----
 
 func.func @test_invalid_result_materialization() {
-  // expected-error@below {{failed to legalize unresolved materialization from ('f64') to 'f16' that remained live after conversion}}
+  // expected-error@below {{failed to legalize unresolved materialization from ('f64') to ('f16') that remained live after conversion}}
   %result = "test.type_producer"() : () -> f16
   // expected-note@below{{see existing live user here}}
   "foo.return"(%result) : (f16) -> ()
@@ -50,7 +50,7 @@ func.func @test_transitive_use_materialization() {
 // -----
 
 func.func @test_transitive_use_invalid_materialization() {
-  // expected-error@below {{failed to legalize unresolved materialization from ('f64') to 'f16' that remained live after conversion}}
+  // expected-error@below {{failed to legalize unresolved materialization from ('f64') to ('f16') that remained live after conversion}}
   %result = "test.another_type_producer"() : () -> f16
   // expected-note@below{{see existing live user here}}
   "foo.return"(%result) : (f16) -> ()
@@ -102,7 +102,7 @@ func.func @test_block_argument_not_converted() {
 // Make sure argument type changes aren't implicitly forwarded.
 func.func @test_signature_conversion_no_converter() {
   "test.signature_conversion_no_converter"() ({
-  // expected-error@below {{failed to legalize unresolved materialization from ('f64') to 'f32' that remained live after conversion}}
+  // expected-error@below {{failed to legalize unresolved materialization from ('f64') to ('f32') that remained live after conversion}}
   ^bb0(%arg0: f32):
     "test.type_consumer"(%arg0) : (f32) -> ()
     // expected-note@below{{see existing live user here}}
-- 
GitLab


From 217700baf760ea3959d79c2090e7930144b698a1 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Wed, 30 Oct 2024 21:49:10 +0900
Subject: [PATCH 572/693] [mlir][bufferization] Support bufferization of
 external functions (#113999)

This commit adds support for bufferizing external functions that have no
body. Such functions were previously rejected by One-Shot Bufferize if
they returned a tensor value.

This commit is in preparation of removing the deprecated
`func-bufferize` pass. That pass can bufferize external functions.

Also update a few comments.
---
 .../IR/BufferizableOpInterface.h              | 11 ++--
 .../FuncBufferizableOpInterfaceImpl.cpp       | 56 ++++++++++---------
 .../one-shot-module-bufferize-invalid.mlir    | 17 ------
 .../Transforms/one-shot-module-bufferize.mlir | 15 +++++
 4 files changed, 51 insertions(+), 48 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
index aceb9d059b95..4866e31b19d5 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
@@ -60,7 +60,8 @@ struct AliasingValue {
   bool isDefinite;
 };
 
-template <typename T> class AliasList {
+template <typename T>
+class AliasList {
 public:
   /// Create an empty list of aliases.
   AliasList() = default;
@@ -259,7 +260,7 @@ struct BufferizationOptions {
   /// Initializer function for analysis state.
   using AnalysisStateInitFn = std::function<void(AnalysisState &)>;
   /// Tensor -> MemRef type converter.
-  /// Parameters: Value, memory space, func op, bufferization options
+  /// Parameters: tensor type, memory space, func op, bufferization options
   using FunctionArgTypeConverterFn =
       std::function<BaseMemRefType(TensorType, Attribute memorySpace,
                                    func::FuncOp, const BufferizationOptions &)>;
@@ -344,9 +345,9 @@ struct BufferizationOptions {
   void setFunctionBoundaryTypeConversion(LayoutMapOption layoutMapOption);
 
   /// Type converter from tensors to memrefs. This type converter is used to
-  /// determine bufferized function argument types. By default, a type
-  /// converter that returns a memref type with a fully dynamic layout map is
-  /// used.
+  /// determine bufferized function argument and result types. By default, a
+  /// type converter that returns a memref type with a fully dynamic layout map
+  /// is used.
   ///
   /// If `bufferizeFunctionBoundaries` is not set, this function isn't used.
   FunctionArgTypeConverterFn functionArgTypeConverterFn = nullptr;
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
index 9fbe574ec392..6e91d3b89a7c 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
@@ -82,7 +82,8 @@ getBufferizedFunctionArgType(FuncOp funcOp, int64_t index,
 
 /// Return the FuncOp called by `callOp`.
 static FuncOp getCalledFunction(CallOpInterface callOp) {
-  SymbolRefAttr sym = llvm::dyn_cast_if_present<SymbolRefAttr>(callOp.getCallableForCallee());
+  SymbolRefAttr sym =
+      llvm::dyn_cast_if_present<SymbolRefAttr>(callOp.getCallableForCallee());
   if (!sym)
     return nullptr;
   return dyn_cast_or_null<FuncOp>(
@@ -392,11 +393,11 @@ struct FuncOpInterface
     auto funcOp = cast<FuncOp>(op);
     FunctionType funcType = funcOp.getFunctionType();
 
-    // Construct the bufferized function type.
+    // Compute the argument types.
     SmallVector<Type> argTypes;
     for (const auto &it : llvm::enumerate(funcType.getInputs())) {
       Type argType = it.value();
-      if (dyn_cast<TensorType>(argType)) {
+      if (isa<TensorType>(argType)) {
         argTypes.push_back(
             getBufferizedFunctionArgType(funcOp, it.index(), options));
         continue;
@@ -404,24 +405,33 @@ struct FuncOpInterface
       argTypes.push_back(argType);
     }
 
-    // Bodiless functions are assumed opaque and we cannot know the
-    // bufferization contract they want to enforce. As a consequence, only
-    // support functions that don't return any tensors atm.
-    if (funcOp.isExternal()) {
-      SmallVector<Type> retTypes;
-      for (Type resultType : funcType.getResults()) {
-        if (isa<TensorType>(resultType))
-          return funcOp->emitError() << "cannot bufferize bodiless function "
-                                     << "that returns a tensor";
+    // Compute the result types.
+    SmallVector<Type> retTypes;
+    for (Type resultType : funcType.getResults()) {
+      if (auto tensorType = dyn_cast<TensorType>(resultType)) {
+        BaseMemRefType resultType = options.functionArgTypeConverterFn(
+            tensorType, *options.defaultMemorySpaceFn(tensorType), funcOp,
+            options);
         retTypes.push_back(resultType);
+        continue;
       }
-      funcOp.setType(FunctionType::get(op->getContext(), argTypes, retTypes));
+      retTypes.push_back(resultType);
+    }
+
+    // Compute the new function type.
+    auto newFuncType = FunctionType::get(op->getContext(), argTypes, retTypes);
+
+    // If the function has no body, set the new function type and we are done.
+    if (funcOp.isExternal()) {
+      funcOp.setType(newFuncType);
       return success();
     }
 
     // TODO: Support functions with multiple returns.
     func::ReturnOp returnOp = getAssumedUniqueReturnOp(funcOp);
     assert(returnOp && "expected func with single return op");
+    assert(returnOp->getNumOperands() == retTypes.size() &&
+           "incorrect number of return values");
     Location loc = returnOp.getLoc();
 
     // 1. Bufferize every block.
@@ -430,10 +440,10 @@ struct FuncOpInterface
                                                         options)))
         return failure();
 
-    // 2. For each result, keep track of which inplace argument it reuses.
+    // 2. Bufferize all operands of the return op.
     SmallVector<Value> returnValues;
-    for (OpOperand &returnOperand : returnOp->getOpOperands()) {
-      Value returnVal = returnOperand.get();
+    for (auto [returnVal, bufferizedType] :
+         llvm::zip_equal(returnOp->getOperands(), retTypes)) {
       auto tensorType = dyn_cast<TensorType>(returnVal.getType());
       rewriter.setInsertionPoint(returnOp);
 
@@ -443,23 +453,17 @@ struct FuncOpInterface
         continue;
       }
 
-      // Note: If `inferFunctionResultLayout = true`, cast are later folded
+      // Note: If `inferFunctionResultLayout = true`, casts are later folded
       // away.
-      BaseMemRefType resultType = options.functionArgTypeConverterFn(
-          tensorType, *options.defaultMemorySpaceFn(tensorType), funcOp,
-          options);
       Value toMemrefOp = rewriter.create<bufferization::ToMemrefOp>(
-          loc, resultType, returnVal);
+          loc, bufferizedType, returnVal);
       returnValues.push_back(toMemrefOp);
     }
 
-    // 3. Rewrite the terminator without the in-place bufferizable values.
     returnOp.getOperandsMutable().assign(returnValues);
 
-    // 4. Rewrite the FuncOp type to buffer form.
-    funcOp.setType(FunctionType::get(op->getContext(), argTypes,
-                                     ValueRange(returnValues).getTypes()));
-
+    // 3. Set the new function type.
+    funcOp.setType(newFuncType);
     return success();
   }
 
diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-invalid.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-invalid.mlir
index ee0f71f668dc..2829eafb7c1c 100644
--- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-invalid.mlir
+++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-invalid.mlir
@@ -1,11 +1,5 @@
 // RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="bufferize-function-boundaries=1" -split-input-file -verify-diagnostics
 
-// expected-error @+2 {{cannot bufferize bodiless function that returns a tensor}}
-// expected-error @+1 {{failed to bufferize op}}
-func.func private @foo() -> tensor<?xf32>
-
-// -----
-
 // expected-error @+1 {{cannot bufferize a FuncOp with tensors and without a unique ReturnOp}}
 func.func @swappy(%cond1 : i1, %cond2 : i1, %t1 : tensor<f32>, %t2 : tensor<f32>)
     -> (tensor<f32>, tensor<f32>)
@@ -123,17 +117,6 @@ func.func @to_tensor_op_unsupported(%m: memref<?xf32>, %idx: index) -> (f32) {
 
 // -----
 
-// expected-error @+2 {{failed to bufferize op}}
-// expected-error @+1 {{cannot bufferize bodiless function that returns a tensor}}
-func.func private @foo(%t : tensor<?xf32>) -> (f32, tensor<?xf32>, f32)
-
-func.func @call_to_unknown_tensor_returning_func(%t : tensor<?xf32>) {
-  call @foo(%t) : (tensor<?xf32>) -> (f32, tensor<?xf32>, f32)
-  return
-}
-
-// -----
-
 func.func @yield_alloc_dominance_test_2(%cst : f32, %idx : index,
                                         %idx2 : index) -> f32 {
   %1 = bufferization.alloc_tensor(%idx) : tensor<?xf32>
diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir
index 0d5224514e3a..d31b43477beb 100644
--- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir
+++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir
@@ -42,6 +42,21 @@ func.func private @external_func_with_return_val(tensor<4xi32>) -> f32
 
 // -----
 
+// Bufferization of bodiless function that returns a tensor.
+
+// CHECK: func.func private @foo(memref<?xf32, strided<[?], offset: ?>>) -> (f32, memref<?xf32, strided<[?], offset: ?>>, f32)
+func.func private @foo(%t : tensor<?xf32>) -> (f32, tensor<?xf32>, f32)
+
+// CHECK: func.func @call_to_unknown_tensor_returning_func(
+// CHECK-SAME: %[[arg0:.*]]: memref<?xf32, strided<[?], offset: ?>>) {
+func.func @call_to_unknown_tensor_returning_func(%t : tensor<?xf32>) {
+  // CHECK: call @foo(%[[arg0]]) : (memref<?xf32, strided<[?], offset: ?>>) -> (f32, memref<?xf32, strided<[?], offset: ?>>, f32)
+  call @foo(%t) : (tensor<?xf32>) -> (f32, tensor<?xf32>, f32)
+  return
+}
+
+// -----
+
 // A function that returns a non-equivalent tensor with layout map.
 
 // CHECK-LABEL: func @return_extract_slice(%{{.*}}) -> memref<2x?xf32, strided<[10, 1], offset: ?>>
-- 
GitLab


From c4e135ec04a2bef5d5a5a69dfbb069a15dbf2f5e Mon Sep 17 00:00:00 2001
From: Jonas Hahnfeld <jonas.hahnfeld@cern.ch>
Date: Wed, 30 Oct 2024 13:56:27 +0100
Subject: [PATCH 573/693] [ORC] Fix transfer to unknown ResourceTrackers
 (#114063)

When transferring resources, the destination tracker key may not be in
the internal map, invalidating iterators and value references. The added
test creates such situation and would fail before with "Finalized
allocation was not deallocated."

For good measure, fix the same pattern in RTDyldObjectLinkingLayer
which is harder to test because it "only" results in memory managers
being deleted in the wrong order.
---
 .../Orc/ObjectLinkingLayer.cpp                |  9 +++---
 .../Orc/RTDyldObjectLinkingLayer.cpp          |  9 +++---
 .../Orc/ObjectLinkingLayerTest.cpp            | 30 +++++++++++++++++++
 3 files changed, 38 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
index 25ab154a01d6..86c08cbdee5f 100644
--- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
@@ -701,16 +701,15 @@ Error ObjectLinkingLayer::handleRemoveResources(JITDylib &JD, ResourceKey K) {
 void ObjectLinkingLayer::handleTransferResources(JITDylib &JD,
                                                  ResourceKey DstKey,
                                                  ResourceKey SrcKey) {
-  auto I = Allocs.find(SrcKey);
-  if (I != Allocs.end()) {
-    auto &SrcAllocs = I->second;
+  if (Allocs.contains(SrcKey)) {
+    // DstKey may not be in the DenseMap yet, so the following line may resize
+    // the container and invalidate iterators and value references.
     auto &DstAllocs = Allocs[DstKey];
+    auto &SrcAllocs = Allocs[SrcKey];
     DstAllocs.reserve(DstAllocs.size() + SrcAllocs.size());
     for (auto &Alloc : SrcAllocs)
       DstAllocs.push_back(std::move(Alloc));
 
-    // Erase SrcKey entry using value rather than iterator I: I may have been
-    // invalidated when we looked up DstKey.
     Allocs.erase(SrcKey);
   }
 
diff --git a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
index bc3433d01155..a73b2310d193 100644
--- a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
@@ -430,16 +430,15 @@ Error RTDyldObjectLinkingLayer::handleRemoveResources(JITDylib &JD,
 void RTDyldObjectLinkingLayer::handleTransferResources(JITDylib &JD,
                                                        ResourceKey DstKey,
                                                        ResourceKey SrcKey) {
-  auto I = MemMgrs.find(SrcKey);
-  if (I != MemMgrs.end()) {
-    auto &SrcMemMgrs = I->second;
+  if (MemMgrs.contains(SrcKey)) {
+    // DstKey may not be in the DenseMap yet, so the following line may resize
+    // the container and invalidate iterators and value references.
     auto &DstMemMgrs = MemMgrs[DstKey];
+    auto &SrcMemMgrs = MemMgrs[SrcKey];
     DstMemMgrs.reserve(DstMemMgrs.size() + SrcMemMgrs.size());
     for (auto &MemMgr : SrcMemMgrs)
       DstMemMgrs.push_back(std::move(MemMgr));
 
-    // Erase SrcKey entry using value rather than iterator I: I may have been
-    // invalidated when we looked up DstKey.
     MemMgrs.erase(SrcKey);
   }
 }
diff --git a/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp
index 63cf3a397cb3..bc996711f7ec 100644
--- a/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp
@@ -65,6 +65,36 @@ TEST_F(ObjectLinkingLayerTest, AddLinkGraph) {
   EXPECT_THAT_EXPECTED(ES.lookup(&JD, "_X"), Succeeded());
 }
 
+TEST_F(ObjectLinkingLayerTest, ResourceTracker) {
+  // This test transfers allocations to previously unknown ResourceTrackers,
+  // while increasing the number of trackers in the ObjectLinkingLayer, which
+  // may invalidate some iterators internally.
+  std::vector<ResourceTrackerSP> Trackers;
+  for (unsigned I = 0; I < 64; I++) {
+    auto G = std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"),
+                                         8, llvm::endianness::little,
+                                         x86_64::getEdgeKindName);
+
+    auto &Sec1 = G->createSection("__data", MemProt::Read | MemProt::Write);
+    auto &B1 = G->createContentBlock(Sec1, BlockContent,
+                                     orc::ExecutorAddr(0x1000), 8, 0);
+    llvm::SmallString<0> SymbolName;
+    SymbolName += "_X";
+    SymbolName += std::to_string(I);
+    G->addDefinedSymbol(B1, 4, SymbolName, 4, Linkage::Strong, Scope::Default,
+                        false, false);
+
+    auto RT1 = JD.createResourceTracker();
+    EXPECT_THAT_ERROR(ObjLinkingLayer.add(RT1, std::move(G)), Succeeded());
+    EXPECT_THAT_EXPECTED(ES.lookup(&JD, SymbolName), Succeeded());
+
+    auto RT2 = JD.createResourceTracker();
+    RT1->transferTo(*RT2);
+
+    Trackers.push_back(RT2);
+  }
+}
+
 TEST_F(ObjectLinkingLayerTest, ClaimLateDefinedWeakSymbols) {
   // Check that claiming weak symbols works as expected.
   //
-- 
GitLab


From 6f973fd4ab18ff58689e83383190ed4767c2a7dd Mon Sep 17 00:00:00 2001
From: Brox Chen <guochen2@amd.com>
Date: Wed, 30 Oct 2024 09:02:40 -0400
Subject: [PATCH 574/693] [AMDGPU][test] fix the error case in
 update_mc_test_check script (#112731)

update_mc_test_check script handle the "error case testline" wrong in
three cases:

1. when user select "--llvm-mc-binary" with a path, the script does not
add "not" on top of the "--llvm-mc-binary" and thus getting non-zero
exit code and failed.
2. When "not" is presented in runline while not all testlines are
expected to fail, the script need to check if the "not" is needed when
it execute llvm-mc line by line. Otherwise the script will fail on
testline which is passing.
3. When there are multiple runlines, the error checkline need to use
correct line offset for "[[LINE-X]]"

This patch solve these three issues
---
 .../Inputs/amdgpu_asm_err.s                   |  5 +-
 .../Inputs/amdgpu_asm_err.s.expected          | 10 +++-
 llvm/utils/UpdateTestChecks/common.py         |  1 +
 llvm/utils/update_mc_test_checks.py           | 53 +++++++++++++++----
 4 files changed, 56 insertions(+), 13 deletions(-)

diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_err.s b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_err.s
index 489bd1801d86..76f8e7880d83 100644
--- a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_err.s
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_err.s
@@ -1,3 +1,6 @@
-// RUN: not llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
+// RUN: not llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECKA %s
+// RUN: not llvm-mc -triple=amdgcn %s 2>&1 | FileCheck --check-prefixes=CHECKB %s
 
 v_bfrev_b32 v5, v299
+
+v_bfrev_b32 v5, v1
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_err.s.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_err.s.expected
index ca287fc2d632..fffe299f3d16 100644
--- a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_err.s.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_err.s.expected
@@ -1,5 +1,11 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
-// RUN: not llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
+// RUN: not llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECKA %s
+// RUN: not llvm-mc -triple=amdgcn %s 2>&1 | FileCheck --check-prefixes=CHECKB %s
 
 v_bfrev_b32 v5, v299
-// CHECK: :[[@LINE-1]]:17: error: register index is out of range
+// CHECKA: :[[@LINE-1]]:17: error: register index is out of range
+// CHECKB: :[[@LINE-2]]:17: error: register index is out of range
+
+v_bfrev_b32 v5, v1
+// CHECKA: v_bfrev_b32_e32 v5, v1                  ; encoding: [0x01,0x71,0x0a,0x7e]
+// CHECKB: v_bfrev_b32_e32 v5, v1
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index cdfa8978566f..b108a21dbc52 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -2470,6 +2470,7 @@ def get_autogennote_suffix(parser, args):
             "verbose",
             "force_update",
             "reset_variable_names",
+            "llvm_mc_binary",
         ):
             continue
         value = getattr(args, action.dest)
diff --git a/llvm/utils/update_mc_test_checks.py b/llvm/utils/update_mc_test_checks.py
index 55ed6c82d487..c8a40b37088a 100755
--- a/llvm/utils/update_mc_test_checks.py
+++ b/llvm/utils/update_mc_test_checks.py
@@ -16,7 +16,6 @@ import re
 
 mc_LIKE_TOOLS = [
     "llvm-mc",
-    "not llvm-mc",
 ]
 ERROR_RE = re.compile(r":\d+: (warning|error): .*")
 ERROR_CHECK_RE = re.compile(r"# COM: .*")
@@ -24,7 +23,7 @@ OUTPUT_SKIPPED_RE = re.compile(r"(.text)")
 COMMENT = {"asm": "//", "dasm": "#"}
 
 
-def invoke_tool(exe, cmd_args, testline, verbose=False):
+def invoke_tool(exe, check_rc, cmd_args, testline, verbose=False):
     if isinstance(cmd_args, list):
         args = [applySubstitutions(a, substitutions) for a in cmd_args]
     else:
@@ -33,7 +32,15 @@ def invoke_tool(exe, cmd_args, testline, verbose=False):
     cmd = 'echo "' + testline + '" | ' + exe + " " + args
     if verbose:
         print("Command: ", cmd)
-    out = subprocess.check_output(cmd, shell=True)
+
+    out = subprocess.run(
+        cmd,
+        shell=True,
+        check=check_rc,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+    ).stdout
+
     # Fix line endings to unix CR style.
     return out.decode().replace("\r\n", "\n")
 
@@ -102,8 +109,16 @@ def getStdCheckLine(prefix, output, mc_mode):
     return o
 
 
-def getErrCheckLine(prefix, output, mc_mode):
-    return COMMENT[mc_mode] + " " + prefix + ": " + ":[[@LINE-1]]" + output + "\n"
+def getErrCheckLine(prefix, output, mc_mode, line_offset=1):
+    return (
+        COMMENT[mc_mode]
+        + " "
+        + prefix
+        + ": "
+        + ":[[@LINE-{}]]".format(line_offset)
+        + output
+        + "\n"
+    )
 
 
 def main():
@@ -174,11 +189,19 @@ def main():
             assert len(commands) >= 2
             mc_cmd = " | ".join(commands[:-1])
             filecheck_cmd = commands[-1]
-            mc_tool = mc_cmd.split(" ")[0]
 
             # special handling for negating exit status
-            if mc_tool == "not":
-                mc_tool = mc_tool + " " + mc_cmd.split(" ")[1]
+            # if not is used in runline, disable rc check, since
+            # the command might or might not
+            # return non-zero code on a single line run
+            check_rc = True
+            mc_cmd_args = mc_cmd.strip().split()
+            if mc_cmd_args[0] == "not":
+                check_rc = False
+                mc_tool = mc_cmd_args[1]
+                mc_cmd = mc_cmd[len(mc_cmd_args[0]) :].strip()
+            else:
+                mc_tool = mc_cmd_args[0]
 
             triple_in_cmd = None
             m = common.TRIPLE_ARG_RE.search(mc_cmd)
@@ -211,6 +234,7 @@ def main():
                 (
                     check_prefixes,
                     mc_tool,
+                    check_rc,
                     mc_cmd_args,
                     triple_in_cmd,
                     march_in_cmd,
@@ -231,6 +255,7 @@ def main():
         for (
             prefixes,
             mc_tool,
+            check_rc,
             mc_args,
             triple_in_cmd,
             march_in_cmd,
@@ -249,6 +274,7 @@ def main():
                 # get output for each testline
                 out = invoke_tool(
                     ti.args.llvm_mc_binary or mc_tool,
+                    check_rc,
                     mc_args,
                     line,
                     verbose=ti.args.verbose,
@@ -305,6 +331,9 @@ def main():
             # each run_id can only be used once
             gen_prefix = ""
             used_runid = set()
+
+            # line number diff between generated prefix and testline
+            line_offset = 1
             for prefix, tup in p_dict_sorted.items():
                 o, run_ids = tup
 
@@ -321,9 +350,13 @@ def main():
                     used_prefixes.add(prefix)
 
                     if hasErr(o):
-                        gen_prefix += getErrCheckLine(prefix, o, mc_mode)
+                        newline = getErrCheckLine(prefix, o, mc_mode, line_offset)
                     else:
-                        gen_prefix += getStdCheckLine(prefix, o, mc_mode)
+                        newline = getStdCheckLine(prefix, o, mc_mode)
+
+                    if newline:
+                        gen_prefix += newline
+                        line_offset += 1
 
             generated_prefixes[input_line] = gen_prefix.rstrip("\n")
 
-- 
GitLab


From 84b7bcfcac02ca32c2211655627c352dd99ce296 Mon Sep 17 00:00:00 2001
From: Petar Avramovic <Petar.Avramovic@amd.com>
Date: Wed, 30 Oct 2024 14:15:42 +0100
Subject: [PATCH 575/693] GlobalISel/MachineIRBuilder: Construct DstOp with
 VRegAttrs (#113581)

Allow construction of DstOp with VRegAttrs.
Also allow construction with register class or bank and LLT.
Intended to be used in lowering code for reg-bank-select where
new registers need to have both register bank and LLT.
Add support for new type of DstOp in CSEMIRBuilder.
---
 .../include/llvm/CodeGen/GlobalISel/CSEInfo.h |  3 +
 .../CodeGen/GlobalISel/MachineIRBuilder.h     | 25 +++++--
 .../llvm/CodeGen/MachineRegisterInfo.h        |  2 +-
 llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp       | 26 ++++---
 llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp | 10 ++-
 llvm/unittests/Target/AMDGPU/CMakeLists.txt   |  2 +
 llvm/unittests/Target/AMDGPU/CSETest.cpp      | 74 +++++++++++++++++++
 7 files changed, 121 insertions(+), 21 deletions(-)
 create mode 100644 llvm/unittests/Target/AMDGPU/CSETest.cpp

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h
index 816e94362f02..8ce6eaa69c4a 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h
@@ -17,6 +17,7 @@
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 #include "llvm/CodeGen/GlobalISel/GISelWorkList.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/CodeGen.h"
 
@@ -177,6 +178,8 @@ public:
   const GISelInstProfileBuilder &addNodeIDOpcode(unsigned Opc) const;
   const GISelInstProfileBuilder &addNodeIDRegType(const LLT Ty) const;
   const GISelInstProfileBuilder &addNodeIDRegType(const Register) const;
+  const GISelInstProfileBuilder &
+      addNodeIDRegType(MachineRegisterInfo::VRegAttrs) const;
 
   const GISelInstProfileBuilder &
   addNodeIDRegType(const TargetRegisterClass *RC) const;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index ab3025e4923c..c41e74ec7ebd 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -72,15 +72,20 @@ class DstOp {
     LLT LLTTy;
     Register Reg;
     const TargetRegisterClass *RC;
+    MachineRegisterInfo::VRegAttrs Attrs;
   };
 
 public:
-  enum class DstType { Ty_LLT, Ty_Reg, Ty_RC };
+  enum class DstType { Ty_LLT, Ty_Reg, Ty_RC, Ty_VRegAttrs };
   DstOp(unsigned R) : Reg(R), Ty(DstType::Ty_Reg) {}
   DstOp(Register R) : Reg(R), Ty(DstType::Ty_Reg) {}
   DstOp(const MachineOperand &Op) : Reg(Op.getReg()), Ty(DstType::Ty_Reg) {}
   DstOp(const LLT T) : LLTTy(T), Ty(DstType::Ty_LLT) {}
   DstOp(const TargetRegisterClass *TRC) : RC(TRC), Ty(DstType::Ty_RC) {}
+  DstOp(MachineRegisterInfo::VRegAttrs Attrs)
+      : Attrs(Attrs), Ty(DstType::Ty_VRegAttrs) {}
+  DstOp(RegClassOrRegBank RCOrRB, LLT Ty)
+      : Attrs({RCOrRB, Ty}), Ty(DstType::Ty_VRegAttrs) {}
 
   void addDefToMIB(MachineRegisterInfo &MRI, MachineInstrBuilder &MIB) const {
     switch (Ty) {
@@ -93,6 +98,9 @@ public:
     case DstType::Ty_RC:
       MIB.addDef(MRI.createVirtualRegister(RC));
       break;
+    case DstType::Ty_VRegAttrs:
+      MIB.addDef(MRI.createVirtualRegister(Attrs));
+      break;
     }
   }
 
@@ -104,6 +112,8 @@ public:
       return LLTTy;
     case DstType::Ty_Reg:
       return MRI.getType(Reg);
+    case DstType::Ty_VRegAttrs:
+      return Attrs.Ty;
     }
     llvm_unreachable("Unrecognised DstOp::DstType enum");
   }
@@ -114,12 +124,13 @@ public:
   }
 
   const TargetRegisterClass *getRegClass() const {
-    switch (Ty) {
-    case DstType::Ty_RC:
-      return RC;
-    default:
-      llvm_unreachable("Not a RC Operand");
-    }
+    assert(Ty == DstType::Ty_RC && "Not a RC Operand");
+    return RC;
+  }
+
+  MachineRegisterInfo::VRegAttrs getVRegAttrs() const {
+    assert(Ty == DstType::Ty_VRegAttrs && "Not a VRegAttrs Operand");
+    return Attrs;
   }
 
   DstType getDstOpKind() const { return Ty; }
diff --git a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
index 7a2c23c13a3c..5dc51aaed81c 100644
--- a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
@@ -754,7 +754,7 @@ public:
   /// Returns register class or bank and low level type of \p Reg. Always safe
   /// to use. Special values are returned when \p Reg does not have some of the
   /// attributes.
-  VRegAttrs getVRegAttrs(Register Reg) {
+  VRegAttrs getVRegAttrs(Register Reg) const {
     return {getRegClassOrRegBank(Reg), getType(Reg)};
   }
 
diff --git a/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp b/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp
index ca4d0986b442..cfb4ae85aa4f 100644
--- a/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp
@@ -356,6 +356,20 @@ GISelInstProfileBuilder::addNodeIDRegType(const RegisterBank *RB) const {
   return *this;
 }
 
+const GISelInstProfileBuilder &GISelInstProfileBuilder::addNodeIDRegType(
+    MachineRegisterInfo::VRegAttrs Attrs) const {
+  addNodeIDRegType(Attrs.Ty);
+
+  const RegClassOrRegBank &RCOrRB = Attrs.RCOrRB;
+  if (RCOrRB) {
+    if (const auto *RB = dyn_cast_if_present<const RegisterBank *>(RCOrRB))
+      addNodeIDRegType(RB);
+    else
+      addNodeIDRegType(cast<const TargetRegisterClass *>(RCOrRB));
+  }
+  return *this;
+}
+
 const GISelInstProfileBuilder &
 GISelInstProfileBuilder::addNodeIDImmediate(int64_t Imm) const {
   ID.AddInteger(Imm);
@@ -389,17 +403,7 @@ GISelInstProfileBuilder::addNodeIDFlag(unsigned Flag) const {
 
 const GISelInstProfileBuilder &
 GISelInstProfileBuilder::addNodeIDReg(Register Reg) const {
-  LLT Ty = MRI.getType(Reg);
-  if (Ty.isValid())
-    addNodeIDRegType(Ty);
-
-  if (const RegClassOrRegBank &RCOrRB = MRI.getRegClassOrRegBank(Reg)) {
-    if (const auto *RB = dyn_cast_if_present<const RegisterBank *>(RCOrRB))
-      addNodeIDRegType(RB);
-    else if (const auto *RC =
-                 dyn_cast_if_present<const TargetRegisterClass *>(RCOrRB))
-      addNodeIDRegType(RC);
-  }
+  addNodeIDRegType(MRI.getVRegAttrs(Reg));
   return *this;
 }
 
diff --git a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
index 547529bbe699..bf8e847011d7 100644
--- a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
@@ -73,18 +73,24 @@ bool CSEMIRBuilder::canPerformCSEForOpc(unsigned Opc) const {
 void CSEMIRBuilder::profileDstOp(const DstOp &Op,
                                  GISelInstProfileBuilder &B) const {
   switch (Op.getDstOpKind()) {
-  case DstOp::DstType::Ty_RC:
+  case DstOp::DstType::Ty_RC: {
     B.addNodeIDRegType(Op.getRegClass());
     break;
+  }
   case DstOp::DstType::Ty_Reg: {
     // Regs can have LLT&(RB|RC). If those exist, profile them as well.
     B.addNodeIDReg(Op.getReg());
     break;
   }
-  default:
+  case DstOp::DstType::Ty_LLT: {
     B.addNodeIDRegType(Op.getLLTTy(*getMRI()));
     break;
   }
+  case DstOp::DstType::Ty_VRegAttrs: {
+    B.addNodeIDRegType(Op.getVRegAttrs());
+    break;
+  }
+  }
 }
 
 void CSEMIRBuilder::profileSrcOp(const SrcOp &Op,
diff --git a/llvm/unittests/Target/AMDGPU/CMakeLists.txt b/llvm/unittests/Target/AMDGPU/CMakeLists.txt
index e0efb967b594..ca8f48bc393e 100644
--- a/llvm/unittests/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/unittests/Target/AMDGPU/CMakeLists.txt
@@ -11,6 +11,7 @@ set(LLVM_LINK_COMPONENTS
   CodeGen
   CodeGenTypes
   Core
+  GlobalISel
   MC
   Support
   TargetParser
@@ -18,6 +19,7 @@ set(LLVM_LINK_COMPONENTS
 
 add_llvm_target_unittest(AMDGPUTests
   AMDGPUUnitTests.cpp
+  CSETest.cpp
   DwarfRegMappings.cpp
   ExecMayBeModifiedBeforeAnyUse.cpp
   PALMetadata.cpp
diff --git a/llvm/unittests/Target/AMDGPU/CSETest.cpp b/llvm/unittests/Target/AMDGPU/CSETest.cpp
new file mode 100644
index 000000000000..3de5b8859964
--- /dev/null
+++ b/llvm/unittests/Target/AMDGPU/CSETest.cpp
@@ -0,0 +1,74 @@
+//===- llvm/unittests/Target/AMDGPU/CSETest.cpp ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUTargetMachine.h"
+#include "AMDGPUUnitTests.h"
+#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
+#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+TEST(AMDGPU, TestCSEForRegisterClassOrBankAndLLT) {
+  auto TM = createAMDGPUTargetMachine("amdgcn-amd-", "gfx1100", "");
+  if (!TM)
+    GTEST_SKIP();
+
+  GCNSubtarget ST(TM->getTargetTriple(), std::string(TM->getTargetCPU()),
+                  std::string(TM->getTargetFeatureString()), *TM);
+
+  LLVMContext Ctx;
+  Module Mod("Module", Ctx);
+  Mod.setDataLayout(TM->createDataLayout());
+
+  auto *Type = FunctionType::get(Type::getVoidTy(Ctx), false);
+  auto *F = Function::Create(Type, GlobalValue::ExternalLinkage, "Test", &Mod);
+
+  MachineModuleInfo MMI(TM.get());
+  auto MF =
+      std::make_unique<MachineFunction>(*F, *TM, ST, MMI.getContext(), 42);
+  auto *BB = MF->CreateMachineBasicBlock();
+  MF->push_back(BB);
+
+  MachineIRBuilder B(*MF);
+  B.setMBB(*BB);
+
+  LLT S32{LLT::scalar(32)};
+  Register R0 = B.buildCopy(S32, Register(AMDGPU::SGPR0)).getReg(0);
+  Register R1 = B.buildCopy(S32, Register(AMDGPU::SGPR1)).getReg(0);
+
+  GISelCSEInfo CSEInfo;
+  CSEInfo.setCSEConfig(std::make_unique<CSEConfigFull>());
+  CSEInfo.analyze(*MF);
+  B.setCSEInfo(&CSEInfo);
+  CSEMIRBuilder CSEB(B.getState());
+  CSEB.setInsertPt(B.getMBB(), B.getInsertPt());
+
+  const RegisterBankInfo &RBI = *MF->getSubtarget().getRegBankInfo();
+
+  const TargetRegisterClass *SgprRC = &AMDGPU::SReg_32RegClass;
+  const RegisterBank *SgprRB = &RBI.getRegBank(AMDGPU::SGPRRegBankID);
+  MachineRegisterInfo::VRegAttrs SgprRCS32 = {SgprRC, S32};
+  MachineRegisterInfo::VRegAttrs SgprRBS32 = {SgprRB, S32};
+
+  auto Add = CSEB.buildAdd(S32, R0, R1);
+  auto AddRC = CSEB.buildInstr(AMDGPU::G_ADD, {SgprRCS32}, {R0, R1});
+  auto AddRB = CSEB.buildInstr(AMDGPU::G_ADD, {{SgprRB, S32}}, {R0, R1});
+
+  EXPECT_NE(Add, AddRC);
+  EXPECT_NE(Add, AddRB);
+  EXPECT_NE(AddRC, AddRB);
+
+  auto Add_CSE = CSEB.buildAdd(S32, R0, R1);
+  auto AddRC_CSE = CSEB.buildInstr(AMDGPU::G_ADD, {{SgprRC, S32}}, {R0, R1});
+  auto AddRB_CSE = CSEB.buildInstr(AMDGPU::G_ADD, {SgprRBS32}, {R0, R1});
+
+  EXPECT_EQ(Add, Add_CSE);
+  EXPECT_EQ(AddRC, AddRC_CSE);
+  EXPECT_EQ(AddRB, AddRB_CSE);
+}
-- 
GitLab


From 602f43686c45017e3140789f8d574d2c344b4d71 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Wed, 30 Oct 2024 13:17:31 +0000
Subject: [PATCH 576/693] [AArch64] Add patterns for constructive splice.
 (#113912)

SVE2 adds the constructive splice instruction, which takes a tuple.
Even though the register allocator must ensure that the tuple uses
consecutive registers for the tuple, it's likely to be more efficient
than using the destructive splice instruction when the first operand
is reused.
---
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |   2 +-
 llvm/lib/Target/AArch64/SVEInstrFormats.td    |  24 +-
 .../sve-streaming-mode-fixed-length-concat.ll |  84 +-
 ...e-streaming-mode-fixed-length-fcopysign.ll |  32 +-
 ...sve-streaming-mode-fixed-length-int-div.ll | 670 ++-----------
 ...sve-streaming-mode-fixed-length-int-rem.ll | 182 ++--
 .../sve-streaming-mode-fixed-length-ptest.ll  | 160 +--
 ...treaming-mode-fixed-length-trunc-stores.ll |  10 +-
 .../sve-streaming-mode-fixed-length-trunc.ll  | 940 +++++++++---------
 9 files changed, 837 insertions(+), 1267 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 2564ddc5f2e5..d6662d15617f 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -3851,7 +3851,7 @@ let Predicates = [HasSVE2] in {
 
 let Predicates = [HasSVE2orSME] in {
   // SVE2 vector splice (constructive)
-  defm SPLICE_ZPZZ : sve2_int_perm_splice_cons<"splice">;
+  defm SPLICE_ZPZZ : sve2_int_perm_splice_cons<"splice", AArch64splice>;
 } // End HasSVE2orSME
 
 let Predicates = [HasSVE2] in {
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 9fa184c54570..552d5b9b23a7 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -7314,11 +7314,33 @@ class sve2_int_perm_splice_cons<bits<2> sz8_64, string asm,
   let hasSideEffects = 0;
 }
 
-multiclass sve2_int_perm_splice_cons<string asm> {
+multiclass sve2_int_perm_splice_cons<string asm, SDPatternOperator op> {
   def _B : sve2_int_perm_splice_cons<0b00, asm, ZPR8,  ZZ_b>;
   def _H : sve2_int_perm_splice_cons<0b01, asm, ZPR16, ZZ_h>;
   def _S : sve2_int_perm_splice_cons<0b10, asm, ZPR32, ZZ_s>;
   def _D : sve2_int_perm_splice_cons<0b11, asm, ZPR64, ZZ_d>;
+
+  let AddedComplexity = 2 in {
+  foreach VT = [nxv16i8] in
+    def : Pat<(VT (op nxv16i1:$pred, VT:$zn1, VT:$zn2)),
+              (!cast<Instruction>(NAME # _B)
+               nxv16i1:$pred, (REG_SEQUENCE ZPR2, VT:$zn1, zsub0, VT:$zn2, zsub1))>;
+
+  foreach VT = [nxv8i16, nxv8f16, nxv8bf16] in
+    def : Pat<(VT (op nxv8i1:$pred, VT:$zn1, VT:$zn2)),
+              (!cast<Instruction>(NAME # _H)
+               nxv8i1:$pred, (REG_SEQUENCE ZPR2, VT:$zn1, zsub0, VT:$zn2, zsub1))>;
+
+  foreach VT = [nxv4i32, nxv4f16, nxv4f32, nxv4bf16] in
+    def : Pat<(VT (op nxv4i1:$pred, VT:$zn1, VT:$zn2)),
+              (!cast<Instruction>(NAME # _S)
+               nxv4i1:$pred, (REG_SEQUENCE ZPR2, VT:$zn1, zsub0, VT:$zn2, zsub1))>;
+
+  foreach VT = [nxv2i64, nxv2f16, nxv2f32, nxv2f64, nxv2bf16] in
+    def : Pat<(VT (op nxv2i1:$pred, VT:$zn1, VT:$zn2)),
+              (!cast<Instruction>(NAME # _D)
+               nxv2i1:$pred, (REG_SEQUENCE ZPR2, VT:$zn1, zsub0, VT:$zn2, zsub1))>;
+  }
 }
 
 class sve2_int_perm_expand<bits<2> sz, string asm,
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
index c1810c678ea5..6e2ecfca9e96 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mattr=+sve -force-streaming-compatible  < %s | FileCheck %s
-; RUN: llc -mattr=+sme -force-streaming  < %s | FileCheck %s
+; RUN: llc -mattr=+sve2 -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
+; RUN: llc -mattr=+sme -force-streaming  < %s | FileCheck %s --check-prefixes=CHECK,SME
 ; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -61,10 +61,10 @@ define <8 x i8> @concat_v8i8(<4 x i8> %op1, <4 x i8> %op2)  {
 define <16 x i8> @concat_v16i8(<8 x i8> %op1, <8 x i8> %op2)  {
 ; CHECK-LABEL: concat_v16i8:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
-; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    splice z0.b, p0, { z0.b, z1.b }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
@@ -172,10 +172,10 @@ define <4 x i16> @concat_v4i16(<2 x i16> %op1, <2 x i16> %op2)  {
 define <8 x i16> @concat_v8i16(<4 x i16> %op1, <4 x i16> %op2)  {
 ; CHECK-LABEL: concat_v8i16:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    splice z0.h, p0, { z0.h, z1.h }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
@@ -270,10 +270,10 @@ define <2 x i32> @concat_v2i32(<1 x i32> %op1, <1 x i32> %op2)  {
 define <4 x i32> @concat_v4i32(<2 x i32> %op1, <2 x i32> %op2)  {
 ; CHECK-LABEL: concat_v4i32:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    ptrue p0.s, vl2
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
-; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    splice z0.s, p0, { z0.s, z1.s }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
@@ -340,10 +340,10 @@ define void @concat_v16i32(ptr %a, ptr %b, ptr %c) {
 define <2 x i64> @concat_v2i64(<1 x i64> %op1, <1 x i64> %op2)  {
 ; CHECK-LABEL: concat_v2i64:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    ptrue p0.d, vl1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
-; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    splice z0.d, p0, { z0.d, z1.d }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
@@ -406,17 +406,33 @@ define void @concat_v8i64(ptr %a, ptr %b, ptr %c) {
 ;
 
 define <4 x half> @concat_v4f16(<2 x half> %op1, <2 x half> %op2)  {
-; CHECK-LABEL: concat_v4f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT:    mov z2.h, z1.h[1]
-; CHECK-NEXT:    mov z3.h, z0.h[1]
-; CHECK-NEXT:    zip1 z1.h, z1.h, z2.h
-; CHECK-NEXT:    zip1 z0.h, z0.h, z3.h
-; CHECK-NEXT:    zip1 z0.s, z0.s, z1.s
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
-; CHECK-NEXT:    ret
+; SVE2-LABEL: concat_v4f16:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    cnth x8
+; SVE2-NEXT:    adrp x9, .LCPI15_0
+; SVE2-NEXT:    adrp x10, .LCPI15_1
+; SVE2-NEXT:    mov z2.h, w8
+; SVE2-NEXT:    ldr q3, [x9, :lo12:.LCPI15_0]
+; SVE2-NEXT:    ldr q4, [x10, :lo12:.LCPI15_1]
+; SVE2-NEXT:    ptrue p0.h, vl8
+; SVE2-NEXT:    // kill: def $d1 killed $d1 killed $z0_z1 def $z0_z1
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0_z1 def $z0_z1
+; SVE2-NEXT:    mad z2.h, p0/m, z3.h, z4.h
+; SVE2-NEXT:    tbl z0.h, { z0.h, z1.h }, z2.h
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE2-NEXT:    ret
+;
+; SME-LABEL: concat_v4f16:
+; SME:       // %bb.0:
+; SME-NEXT:    // kill: def $d1 killed $d1 def $z1
+; SME-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SME-NEXT:    mov z2.h, z1.h[1]
+; SME-NEXT:    mov z3.h, z0.h[1]
+; SME-NEXT:    zip1 z1.h, z1.h, z2.h
+; SME-NEXT:    zip1 z0.h, z0.h, z3.h
+; SME-NEXT:    zip1 z0.s, z0.s, z1.s
+; SME-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SME-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: concat_v4f16:
 ; NONEON-NOSVE:       // %bb.0:
@@ -436,10 +452,10 @@ define <4 x half> @concat_v4f16(<2 x half> %op1, <2 x half> %op2)  {
 define <8 x half> @concat_v8f16(<4 x half> %op1, <4 x half> %op2)  {
 ; CHECK-LABEL: concat_v8f16:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    splice z0.h, p0, { z0.h, z1.h }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
@@ -534,10 +550,10 @@ define <2 x float> @concat_v2f32(<1 x float> %op1, <1 x float> %op2)  {
 define <4 x float> @concat_v4f32(<2 x float> %op1, <2 x float> %op2)  {
 ; CHECK-LABEL: concat_v4f32:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    ptrue p0.s, vl2
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
-; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    splice z0.s, p0, { z0.s, z1.s }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
@@ -604,10 +620,10 @@ define void @concat_v16f32(ptr %a, ptr %b, ptr %c) {
 define <2 x double> @concat_v2f64(<1 x double> %op1, <1 x double> %op2)  {
 ; CHECK-LABEL: concat_v2f64:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $z0_z1 def $z0_z1
 ; CHECK-NEXT:    ptrue p0.d, vl1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
-; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    splice z0.d, p0, { z0.d, z1.d }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
index f1771a753826..2282e74af5d0 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mattr=+sve -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=CHECK,SVE
-; RUN: llc -mattr=+sve2 -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
-; RUN: llc -mattr=+sme -force-streaming  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
+; RUN: llc -mattr=+sve -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=SVE
+; RUN: llc -mattr=+sve2 -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=SVE2
+; RUN: llc -mattr=+sme -force-streaming  < %s | FileCheck %s --check-prefixes=SVE2
 ; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
@@ -842,16 +842,16 @@ define void @test_copysign_v4f32_v4f64(ptr %ap, ptr %bp) {
 ;
 ; SVE2-LABEL: test_copysign_v4f32_v4f64:
 ; SVE2:       // %bb.0:
-; SVE2-NEXT:    ldp q0, q1, [x1]
+; SVE2-NEXT:    ldp q1, q0, [x1]
 ; SVE2-NEXT:    ptrue p0.d
-; SVE2-NEXT:    ldr q2, [x0]
-; SVE2-NEXT:    fcvt z1.s, p0/m, z1.d
 ; SVE2-NEXT:    fcvt z0.s, p0/m, z0.d
+; SVE2-NEXT:    fcvt z1.s, p0/m, z1.d
 ; SVE2-NEXT:    ptrue p0.s, vl2
-; SVE2-NEXT:    uzp1 z1.s, z1.s, z1.s
-; SVE2-NEXT:    uzp1 z0.s, z0.s, z0.s
-; SVE2-NEXT:    splice z0.s, p0, z0.s, z1.s
+; SVE2-NEXT:    uzp1 z3.s, z0.s, z0.s
+; SVE2-NEXT:    uzp1 z2.s, z1.s, z1.s
 ; SVE2-NEXT:    mov z1.s, #0x7fffffff
+; SVE2-NEXT:    splice z0.s, p0, { z2.s, z3.s }
+; SVE2-NEXT:    ldr q2, [x0]
 ; SVE2-NEXT:    bsl z2.d, z2.d, z0.d, z1.d
 ; SVE2-NEXT:    str q2, [x0]
 ; SVE2-NEXT:    ret
@@ -1237,16 +1237,16 @@ define void @test_copysign_v8f16_v8f32(ptr %ap, ptr %bp) {
 ;
 ; SVE2-LABEL: test_copysign_v8f16_v8f32:
 ; SVE2:       // %bb.0:
-; SVE2-NEXT:    ldp q0, q1, [x1]
+; SVE2-NEXT:    ldp q1, q0, [x1]
 ; SVE2-NEXT:    ptrue p0.s
-; SVE2-NEXT:    ldr q2, [x0]
-; SVE2-NEXT:    fcvt z1.h, p0/m, z1.s
 ; SVE2-NEXT:    fcvt z0.h, p0/m, z0.s
+; SVE2-NEXT:    fcvt z1.h, p0/m, z1.s
 ; SVE2-NEXT:    ptrue p0.h, vl4
-; SVE2-NEXT:    uzp1 z1.h, z1.h, z1.h
-; SVE2-NEXT:    uzp1 z0.h, z0.h, z0.h
-; SVE2-NEXT:    splice z0.h, p0, z0.h, z1.h
+; SVE2-NEXT:    uzp1 z3.h, z0.h, z0.h
+; SVE2-NEXT:    uzp1 z2.h, z1.h, z1.h
 ; SVE2-NEXT:    mov z1.h, #32767 // =0x7fff
+; SVE2-NEXT:    splice z0.h, p0, { z2.h, z3.h }
+; SVE2-NEXT:    ldr q2, [x0]
 ; SVE2-NEXT:    bsl z2.d, z2.d, z0.d, z1.d
 ; SVE2-NEXT:    str q2, [x0]
 ; SVE2-NEXT:    ret
@@ -1349,5 +1349,3 @@ declare <8 x float> @llvm.copysign.v8f32(<8 x float> %a, <8 x float> %b) #0
 
 declare <2 x double> @llvm.copysign.v2f64(<2 x double> %a, <2 x double> %b) #0
 declare <4 x double> @llvm.copysign.v4f64(<4 x double> %a, <4 x double> %b) #0
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll
index 516772b8ca66..1fdcd4f82687 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mattr=+sve -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=CHECK,SVE
-; RUN: llc -mattr=+sve2 -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
-; RUN: llc -mattr=+sme -force-streaming  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
+; RUN: llc -mattr=+sve2 -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: llc -mattr=+sme -force-streaming  < %s | FileCheck %s --check-prefixes=CHECK
 ; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -26,19 +25,6 @@ define <4 x i8> @sdiv_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: sdiv_v4i8:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    shl v0.4h, v0.4h, #8
-; NEON-NOSVE-NEXT:    shl v1.4h, v1.4h, #8
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    sshr v0.4h, v0.4h, #8
-; NEON-NOSVE-NEXT:    sshr v1.4h, v1.4h, #8
-; NEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NEON-NOSVE-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    xtn v0.4h, v0.4s
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: sdiv_v4i8:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #32
@@ -85,27 +71,12 @@ define <8 x i8> @sdiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    uzp1 z1.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    splice z1.h, p0, z1.h, z0.h
-; CHECK-NEXT:    uzp1 z0.b, z1.b, z1.b
+; CHECK-NEXT:    uzp1 z2.h, z0.h, z0.h
+; CHECK-NEXT:    splice z0.h, p0, { z1.h, z2.h }
+; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: sdiv_v8i8:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    sshll v1.8h, v1.8b, #0
-; NEON-NOSVE-NEXT:    sshll v0.8h, v0.8b, #0
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    sshll2 v2.4s, v1.8h, #0
-; NEON-NOSVE-NEXT:    sshll2 v3.4s, v0.8h, #0
-; NEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NEON-NOSVE-NEXT:    sdivr z2.s, p0/m, z2.s, z3.s
-; NEON-NOSVE-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; NEON-NOSVE-NEXT:    xtn v0.8b, v0.8h
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: sdiv_v8i8:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #32
@@ -177,45 +148,21 @@ define <16 x i8> @sdiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
 ; CHECK-NEXT:    sunpklo z1.s, z1.h
 ; CHECK-NEXT:    sdivr z3.s, p0/m, z3.s, z5.s
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
+; CHECK-NEXT:    uzp1 z5.h, z2.h, z2.h
 ; CHECK-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
-; CHECK-NEXT:    uzp1 z1.h, z4.h, z4.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    splice z1.h, p0, z1.h, z2.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    splice z3.h, p0, z3.h, z0.h
-; CHECK-NEXT:    uzp1 z0.b, z1.b, z1.b
+; CHECK-NEXT:    uzp1 z1.h, z3.h, z3.h
+; CHECK-NEXT:    uzp1 z2.h, z0.h, z0.h
+; CHECK-NEXT:    splice z0.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    splice z1.h, p0, { z1.h, z2.h }
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z1.b, z3.b, z3.b
-; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
+; CHECK-NEXT:    uzp1 z2.b, z0.b, z0.b
+; CHECK-NEXT:    uzp1 z3.b, z1.b, z1.b
+; CHECK-NEXT:    splice z0.b, p0, { z2.b, z3.b }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: sdiv_v16i8:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    sshll2 v2.8h, v1.16b, #0
-; NEON-NOSVE-NEXT:    sshll2 v3.8h, v0.16b, #0
-; NEON-NOSVE-NEXT:    sshll v1.8h, v1.8b, #0
-; NEON-NOSVE-NEXT:    sshll v0.8h, v0.8b, #0
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    sshll2 v4.4s, v2.8h, #0
-; NEON-NOSVE-NEXT:    sshll2 v5.4s, v3.8h, #0
-; NEON-NOSVE-NEXT:    sshll v2.4s, v2.4h, #0
-; NEON-NOSVE-NEXT:    sshll v3.4s, v3.4h, #0
-; NEON-NOSVE-NEXT:    sdivr z4.s, p0/m, z4.s, z5.s
-; NEON-NOSVE-NEXT:    sshll2 v5.4s, v0.8h, #0
-; NEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NEON-NOSVE-NEXT:    sdivr z2.s, p0/m, z2.s, z3.s
-; NEON-NOSVE-NEXT:    sshll2 v3.4s, v1.8h, #0
-; NEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NEON-NOSVE-NEXT:    sdivr z3.s, p0/m, z3.s, z5.s
-; NEON-NOSVE-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    uzp1 v1.8h, v2.8h, v4.8h
-; NEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
-; NEON-NOSVE-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: sdiv_v16i8:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
@@ -319,7 +266,6 @@ define void @sdiv_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sunpklo z4.h, z2.b
 ; CHECK-NEXT:    sunpklo z2.s, z3.h
 ; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    sunpklo z5.s, z4.h
 ; CHECK-NEXT:    ext z4.b, z4.b, z4.b, #8
 ; CHECK-NEXT:    sunpklo z3.s, z3.h
@@ -328,7 +274,6 @@ define void @sdiv_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q5, [x0]
 ; CHECK-NEXT:    sunpklo z16.h, z5.b
 ; CHECK-NEXT:    ext z5.b, z5.b, z5.b, #8
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
 ; CHECK-NEXT:    sunpklo z5.h, z5.b
 ; CHECK-NEXT:    sunpklo z18.s, z16.h
 ; CHECK-NEXT:    ext z16.b, z16.b, z16.b, #8
@@ -337,81 +282,36 @@ define void @sdiv_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sunpklo z18.s, z5.h
 ; CHECK-NEXT:    ext z5.b, z5.b, z5.b, #8
 ; CHECK-NEXT:    sunpklo z5.s, z5.h
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
 ; CHECK-NEXT:    sdivr z7.s, p0/m, z7.s, z16.s
 ; CHECK-NEXT:    sunpklo z16.s, z6.h
 ; CHECK-NEXT:    ext z6.b, z6.b, z6.b, #8
 ; CHECK-NEXT:    sunpklo z6.s, z6.h
+; CHECK-NEXT:    uzp1 z20.h, z17.h, z17.h
 ; CHECK-NEXT:    sdivr z16.s, p0/m, z16.s, z18.s
+; CHECK-NEXT:    uzp1 z18.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z19.h, z1.h, z1.h
+; CHECK-NEXT:    uzp1 z21.h, z7.h, z7.h
 ; CHECK-NEXT:    sdiv z5.s, p0/m, z5.s, z6.s
-; CHECK-NEXT:    uzp1 z6.h, z7.h, z7.h
-; CHECK-NEXT:    uzp1 z7.h, z16.h, z16.h
+; CHECK-NEXT:    uzp1 z0.h, z16.h, z16.h
 ; CHECK-NEXT:    sdivr z3.s, p0/m, z3.s, z4.s
-; CHECK-NEXT:    uzp1 z4.h, z17.h, z17.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
-; CHECK-NEXT:    uzp1 z5.h, z5.h, z5.h
-; CHECK-NEXT:    splice z4.h, p0, z4.h, z6.h
-; CHECK-NEXT:    splice z7.h, p0, z7.h, z5.h
-; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
-; CHECK-NEXT:    uzp1 z1.b, z4.b, z4.b
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z3.h
-; CHECK-NEXT:    uzp1 z3.b, z7.b, z7.b
+; CHECK-NEXT:    uzp1 z1.h, z5.h, z5.h
+; CHECK-NEXT:    uzp1 z4.h, z2.h, z2.h
+; CHECK-NEXT:    splice z2.h, p0, { z20.h, z21.h }
+; CHECK-NEXT:    splice z0.h, p0, { z0.h, z1.h }
+; CHECK-NEXT:    uzp1 z5.h, z3.h, z3.h
+; CHECK-NEXT:    splice z3.h, p0, { z18.h, z19.h }
+; CHECK-NEXT:    splice z1.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    uzp1 z4.b, z2.b, z2.b
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    splice z1.b, p0, z1.b, z3.b
-; CHECK-NEXT:    uzp1 z2.b, z2.b, z2.b
-; CHECK-NEXT:    splice z0.b, p0, z0.b, z2.b
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    uzp1 z2.b, z3.b, z3.b
+; CHECK-NEXT:    uzp1 z5.b, z0.b, z0.b
+; CHECK-NEXT:    uzp1 z3.b, z1.b, z1.b
+; CHECK-NEXT:    splice z0.b, p0, { z4.b, z5.b }
+; CHECK-NEXT:    splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: sdiv_v32i8:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ldp q6, q3, [x1]
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    ldr q2, [x0, #16]
-; NEON-NOSVE-NEXT:    sshll2 v1.8h, v3.16b, #0
-; NEON-NOSVE-NEXT:    sshll2 v4.8h, v2.16b, #0
-; NEON-NOSVE-NEXT:    sshll v3.8h, v3.8b, #0
-; NEON-NOSVE-NEXT:    sshll v2.8h, v2.8b, #0
-; NEON-NOSVE-NEXT:    sshll2 v7.8h, v6.16b, #0
-; NEON-NOSVE-NEXT:    sshll v6.8h, v6.8b, #0
-; NEON-NOSVE-NEXT:    sshll2 v0.4s, v1.8h, #0
-; NEON-NOSVE-NEXT:    sshll2 v5.4s, v4.8h, #0
-; NEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NEON-NOSVE-NEXT:    sshll v4.4s, v4.4h, #0
-; NEON-NOSVE-NEXT:    sshll2 v17.4s, v7.8h, #0
-; NEON-NOSVE-NEXT:    sshll v7.4s, v7.4h, #0
-; NEON-NOSVE-NEXT:    sdivr z0.s, p0/m, z0.s, z5.s
-; NEON-NOSVE-NEXT:    sshll2 v5.4s, v2.8h, #0
-; NEON-NOSVE-NEXT:    sshll v2.4s, v2.4h, #0
-; NEON-NOSVE-NEXT:    sdivr z1.s, p0/m, z1.s, z4.s
-; NEON-NOSVE-NEXT:    sshll2 v4.4s, v3.8h, #0
-; NEON-NOSVE-NEXT:    sshll v3.4s, v3.4h, #0
-; NEON-NOSVE-NEXT:    sdivr z4.s, p0/m, z4.s, z5.s
-; NEON-NOSVE-NEXT:    ldr q5, [x0]
-; NEON-NOSVE-NEXT:    sshll2 v16.8h, v5.16b, #0
-; NEON-NOSVE-NEXT:    sshll v5.8h, v5.8b, #0
-; NEON-NOSVE-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; NEON-NOSVE-NEXT:    sshll2 v18.4s, v16.8h, #0
-; NEON-NOSVE-NEXT:    sshll v16.4s, v16.4h, #0
-; NEON-NOSVE-NEXT:    sdivr z17.s, p0/m, z17.s, z18.s
-; NEON-NOSVE-NEXT:    sshll2 v18.4s, v5.8h, #0
-; NEON-NOSVE-NEXT:    sshll v5.4s, v5.4h, #0
-; NEON-NOSVE-NEXT:    sdivr z7.s, p0/m, z7.s, z16.s
-; NEON-NOSVE-NEXT:    sshll2 v16.4s, v6.8h, #0
-; NEON-NOSVE-NEXT:    sshll v6.4s, v6.4h, #0
-; NEON-NOSVE-NEXT:    sdivr z16.s, p0/m, z16.s, z18.s
-; NEON-NOSVE-NEXT:    sdiv z5.s, p0/m, z5.s, z6.s
-; NEON-NOSVE-NEXT:    sdiv z2.s, p0/m, z2.s, z3.s
-; NEON-NOSVE-NEXT:    uzp1 v3.8h, v7.8h, v17.8h
-; NEON-NOSVE-NEXT:    uzp1 v5.8h, v5.8h, v16.8h
-; NEON-NOSVE-NEXT:    uzp1 v1.8h, v2.8h, v4.8h
-; NEON-NOSVE-NEXT:    uzp1 v2.16b, v5.16b, v3.16b
-; NEON-NOSVE-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
-; NEON-NOSVE-NEXT:    stp q2, q0, [x0]
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: sdiv_v32i8:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #96
@@ -571,17 +471,6 @@ define <2 x i16> @sdiv_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: sdiv_v2i16:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    shl v1.2s, v1.2s, #16
-; NEON-NOSVE-NEXT:    shl v0.2s, v0.2s, #16
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl2
-; NEON-NOSVE-NEXT:    sshr v1.2s, v1.2s, #16
-; NEON-NOSVE-NEXT:    sshr v0.2s, v0.2s, #16
-; NEON-NOSVE-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: sdiv_v2i16:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #32
@@ -614,15 +503,6 @@ define <4 x i16> @sdiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: sdiv_v4i16:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    xtn v0.4h, v0.4s
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: sdiv_v4i16:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #32
@@ -664,26 +544,14 @@ define <8 x i16> @sdiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    sunpklo z1.s, z1.h
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
 ; CHECK-NEXT:    sdivr z2.s, p0/m, z2.s, z3.s
-; CHECK-NEXT:    sdivr z1.s, p0/m, z1.s, z0.s
+; CHECK-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z0.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
+; CHECK-NEXT:    uzp1 z1.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z2.h, z0.h, z0.h
+; CHECK-NEXT:    splice z0.h, p0, { z1.h, z2.h }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: sdiv_v8i16:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    sshll2 v2.4s, v1.8h, #0
-; NEON-NOSVE-NEXT:    sshll2 v3.4s, v0.8h, #0
-; NEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    sdivr z2.s, p0/m, z2.s, z3.s
-; NEON-NOSVE-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: sdiv_v8i16:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
@@ -748,41 +616,18 @@ define void @sdiv_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
 ; CHECK-NEXT:    sunpklo z3.s, z3.h
 ; CHECK-NEXT:    sdivr z5.s, p0/m, z5.s, z6.s
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
 ; CHECK-NEXT:    sdiv z3.s, p0/m, z3.s, z4.s
+; CHECK-NEXT:    uzp1 z4.h, z5.h, z5.h
 ; CHECK-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
-; CHECK-NEXT:    uzp1 z1.h, z5.h, z5.h
+; CHECK-NEXT:    uzp1 z1.h, z2.h, z2.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    splice z1.h, p0, z1.h, z3.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z0.h
-; CHECK-NEXT:    stp q1, q2, [x0]
+; CHECK-NEXT:    uzp1 z5.h, z3.h, z3.h
+; CHECK-NEXT:    uzp1 z2.h, z0.h, z0.h
+; CHECK-NEXT:    splice z0.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    splice z1.h, p0, { z1.h, z2.h }
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: sdiv_v16i16:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ldp q4, q1, [x1]
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NEON-NOSVE-NEXT:    sshll2 v2.4s, v1.8h, #0
-; NEON-NOSVE-NEXT:    sshll2 v3.4s, v0.8h, #0
-; NEON-NOSVE-NEXT:    sshll2 v5.4s, v4.8h, #0
-; NEON-NOSVE-NEXT:    sshll v4.4s, v4.4h, #0
-; NEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NEON-NOSVE-NEXT:    sdivr z2.s, p0/m, z2.s, z3.s
-; NEON-NOSVE-NEXT:    ldr q3, [x0]
-; NEON-NOSVE-NEXT:    sshll2 v6.4s, v3.8h, #0
-; NEON-NOSVE-NEXT:    sshll v3.4s, v3.4h, #0
-; NEON-NOSVE-NEXT:    sdivr z5.s, p0/m, z5.s, z6.s
-; NEON-NOSVE-NEXT:    sdiv z3.s, p0/m, z3.s, z4.s
-; NEON-NOSVE-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    uzp1 v1.8h, v3.8h, v5.8h
-; NEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; NEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: sdiv_v16i16:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #96
@@ -876,15 +721,6 @@ define <2 x i32> @sdiv_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: sdiv_v2i32:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl2
-; NEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $z0
-; NEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $z1
-; NEON-NOSVE-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: sdiv_v2i32:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #32
@@ -913,15 +749,6 @@ define <4 x i32> @sdiv_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: sdiv_v4i32:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    // kill: def $q0 killed $q0 def $z0
-; NEON-NOSVE-NEXT:    // kill: def $q1 killed $q1 def $z1
-; NEON-NOSVE-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: sdiv_v4i32:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
@@ -957,17 +784,6 @@ define void @sdiv_v8i32(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: sdiv_v8i32:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NEON-NOSVE-NEXT:    sdivr z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    movprfx z1, z2
-; NEON-NOSVE-NEXT:    sdiv z1.s, p0/m, z1.s, z3.s
-; NEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: sdiv_v8i32:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #96
@@ -1021,15 +837,6 @@ define <1 x i64> @sdiv_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: sdiv_v1i64:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ptrue p0.d, vl1
-; NEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $z0
-; NEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $z1
-; NEON-NOSVE-NEXT:    sdiv z0.d, p0/m, z0.d, z1.d
-; NEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: sdiv_v1i64:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #16
@@ -1055,15 +862,6 @@ define <2 x i64> @sdiv_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: sdiv_v2i64:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ptrue p0.d, vl2
-; NEON-NOSVE-NEXT:    // kill: def $q0 killed $q0 def $z0
-; NEON-NOSVE-NEXT:    // kill: def $q1 killed $q1 def $z1
-; NEON-NOSVE-NEXT:    sdiv z0.d, p0/m, z0.d, z1.d
-; NEON-NOSVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: sdiv_v2i64:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
@@ -1093,17 +891,6 @@ define void @sdiv_v4i64(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: sdiv_v4i64:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NEON-NOSVE-NEXT:    ptrue p0.d, vl2
-; NEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NEON-NOSVE-NEXT:    sdivr z0.d, p0/m, z0.d, z1.d
-; NEON-NOSVE-NEXT:    movprfx z1, z2
-; NEON-NOSVE-NEXT:    sdiv z1.d, p0/m, z1.d, z3.d
-; NEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: sdiv_v4i64:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #96
@@ -1135,9 +922,7 @@ define void @sdiv_v4i64(ptr %a, ptr %b)  {
   ret void
 }
 
-;
 ; UDIV
-;
 
 define <4 x i8> @udiv_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; CHECK-LABEL: udiv_v4i8:
@@ -1154,17 +939,6 @@ define <4 x i8> @udiv_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: udiv_v4i8:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    bic v0.4h, #255, lsl #8
-; NEON-NOSVE-NEXT:    bic v1.4h, #255, lsl #8
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NEON-NOSVE-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    xtn v0.4h, v0.4s
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: udiv_v4i8:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #32
@@ -1211,27 +985,12 @@ define <8 x i8> @udiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    uzp1 z1.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    splice z1.h, p0, z1.h, z0.h
-; CHECK-NEXT:    uzp1 z0.b, z1.b, z1.b
+; CHECK-NEXT:    uzp1 z2.h, z0.h, z0.h
+; CHECK-NEXT:    splice z0.h, p0, { z1.h, z2.h }
+; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: udiv_v8i8:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ushll v1.8h, v1.8b, #0
-; NEON-NOSVE-NEXT:    ushll v0.8h, v0.8b, #0
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    ushll2 v2.4s, v1.8h, #0
-; NEON-NOSVE-NEXT:    ushll2 v3.4s, v0.8h, #0
-; NEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NEON-NOSVE-NEXT:    udivr z2.s, p0/m, z2.s, z3.s
-; NEON-NOSVE-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; NEON-NOSVE-NEXT:    xtn v0.8b, v0.8h
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: udiv_v8i8:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #32
@@ -1303,45 +1062,21 @@ define <16 x i8> @udiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z1.s, z1.h
 ; CHECK-NEXT:    udivr z3.s, p0/m, z3.s, z5.s
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
+; CHECK-NEXT:    uzp1 z5.h, z2.h, z2.h
 ; CHECK-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
-; CHECK-NEXT:    uzp1 z1.h, z4.h, z4.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    splice z1.h, p0, z1.h, z2.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    splice z3.h, p0, z3.h, z0.h
-; CHECK-NEXT:    uzp1 z0.b, z1.b, z1.b
+; CHECK-NEXT:    uzp1 z1.h, z3.h, z3.h
+; CHECK-NEXT:    uzp1 z2.h, z0.h, z0.h
+; CHECK-NEXT:    splice z0.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    splice z1.h, p0, { z1.h, z2.h }
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z1.b, z3.b, z3.b
-; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
+; CHECK-NEXT:    uzp1 z2.b, z0.b, z0.b
+; CHECK-NEXT:    uzp1 z3.b, z1.b, z1.b
+; CHECK-NEXT:    splice z0.b, p0, { z2.b, z3.b }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: udiv_v16i8:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ushll2 v2.8h, v1.16b, #0
-; NEON-NOSVE-NEXT:    ushll2 v3.8h, v0.16b, #0
-; NEON-NOSVE-NEXT:    ushll v1.8h, v1.8b, #0
-; NEON-NOSVE-NEXT:    ushll v0.8h, v0.8b, #0
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    ushll2 v4.4s, v2.8h, #0
-; NEON-NOSVE-NEXT:    ushll2 v5.4s, v3.8h, #0
-; NEON-NOSVE-NEXT:    ushll v2.4s, v2.4h, #0
-; NEON-NOSVE-NEXT:    ushll v3.4s, v3.4h, #0
-; NEON-NOSVE-NEXT:    udivr z4.s, p0/m, z4.s, z5.s
-; NEON-NOSVE-NEXT:    ushll2 v5.4s, v0.8h, #0
-; NEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NEON-NOSVE-NEXT:    udivr z2.s, p0/m, z2.s, z3.s
-; NEON-NOSVE-NEXT:    ushll2 v3.4s, v1.8h, #0
-; NEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NEON-NOSVE-NEXT:    udivr z3.s, p0/m, z3.s, z5.s
-; NEON-NOSVE-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    uzp1 v1.8h, v2.8h, v4.8h
-; NEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
-; NEON-NOSVE-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: udiv_v16i8:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
@@ -1445,7 +1180,6 @@ define void @udiv_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    uunpklo z4.h, z2.b
 ; CHECK-NEXT:    uunpklo z2.s, z3.h
 ; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    uunpklo z5.s, z4.h
 ; CHECK-NEXT:    ext z4.b, z4.b, z4.b, #8
 ; CHECK-NEXT:    uunpklo z3.s, z3.h
@@ -1454,7 +1188,6 @@ define void @udiv_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q5, [x0]
 ; CHECK-NEXT:    uunpklo z16.h, z5.b
 ; CHECK-NEXT:    ext z5.b, z5.b, z5.b, #8
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
 ; CHECK-NEXT:    uunpklo z5.h, z5.b
 ; CHECK-NEXT:    uunpklo z18.s, z16.h
 ; CHECK-NEXT:    ext z16.b, z16.b, z16.b, #8
@@ -1463,81 +1196,36 @@ define void @udiv_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    uunpklo z18.s, z5.h
 ; CHECK-NEXT:    ext z5.b, z5.b, z5.b, #8
 ; CHECK-NEXT:    uunpklo z5.s, z5.h
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
 ; CHECK-NEXT:    udivr z7.s, p0/m, z7.s, z16.s
 ; CHECK-NEXT:    uunpklo z16.s, z6.h
 ; CHECK-NEXT:    ext z6.b, z6.b, z6.b, #8
 ; CHECK-NEXT:    uunpklo z6.s, z6.h
+; CHECK-NEXT:    uzp1 z20.h, z17.h, z17.h
 ; CHECK-NEXT:    udivr z16.s, p0/m, z16.s, z18.s
+; CHECK-NEXT:    uzp1 z18.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z19.h, z1.h, z1.h
+; CHECK-NEXT:    uzp1 z21.h, z7.h, z7.h
 ; CHECK-NEXT:    udiv z5.s, p0/m, z5.s, z6.s
-; CHECK-NEXT:    uzp1 z6.h, z7.h, z7.h
-; CHECK-NEXT:    uzp1 z7.h, z16.h, z16.h
+; CHECK-NEXT:    uzp1 z0.h, z16.h, z16.h
 ; CHECK-NEXT:    udivr z3.s, p0/m, z3.s, z4.s
-; CHECK-NEXT:    uzp1 z4.h, z17.h, z17.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
-; CHECK-NEXT:    uzp1 z5.h, z5.h, z5.h
-; CHECK-NEXT:    splice z4.h, p0, z4.h, z6.h
-; CHECK-NEXT:    splice z7.h, p0, z7.h, z5.h
-; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
-; CHECK-NEXT:    uzp1 z1.b, z4.b, z4.b
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z3.h
-; CHECK-NEXT:    uzp1 z3.b, z7.b, z7.b
+; CHECK-NEXT:    uzp1 z1.h, z5.h, z5.h
+; CHECK-NEXT:    uzp1 z4.h, z2.h, z2.h
+; CHECK-NEXT:    splice z2.h, p0, { z20.h, z21.h }
+; CHECK-NEXT:    splice z0.h, p0, { z0.h, z1.h }
+; CHECK-NEXT:    uzp1 z5.h, z3.h, z3.h
+; CHECK-NEXT:    splice z3.h, p0, { z18.h, z19.h }
+; CHECK-NEXT:    splice z1.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    uzp1 z4.b, z2.b, z2.b
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    splice z1.b, p0, z1.b, z3.b
-; CHECK-NEXT:    uzp1 z2.b, z2.b, z2.b
-; CHECK-NEXT:    splice z0.b, p0, z0.b, z2.b
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    uzp1 z2.b, z3.b, z3.b
+; CHECK-NEXT:    uzp1 z5.b, z0.b, z0.b
+; CHECK-NEXT:    uzp1 z3.b, z1.b, z1.b
+; CHECK-NEXT:    splice z0.b, p0, { z4.b, z5.b }
+; CHECK-NEXT:    splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: udiv_v32i8:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ldp q6, q3, [x1]
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    ldr q2, [x0, #16]
-; NEON-NOSVE-NEXT:    ushll2 v1.8h, v3.16b, #0
-; NEON-NOSVE-NEXT:    ushll2 v4.8h, v2.16b, #0
-; NEON-NOSVE-NEXT:    ushll v3.8h, v3.8b, #0
-; NEON-NOSVE-NEXT:    ushll v2.8h, v2.8b, #0
-; NEON-NOSVE-NEXT:    ushll2 v7.8h, v6.16b, #0
-; NEON-NOSVE-NEXT:    ushll v6.8h, v6.8b, #0
-; NEON-NOSVE-NEXT:    ushll2 v0.4s, v1.8h, #0
-; NEON-NOSVE-NEXT:    ushll2 v5.4s, v4.8h, #0
-; NEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NEON-NOSVE-NEXT:    ushll v4.4s, v4.4h, #0
-; NEON-NOSVE-NEXT:    ushll2 v17.4s, v7.8h, #0
-; NEON-NOSVE-NEXT:    ushll v7.4s, v7.4h, #0
-; NEON-NOSVE-NEXT:    udivr z0.s, p0/m, z0.s, z5.s
-; NEON-NOSVE-NEXT:    ushll2 v5.4s, v2.8h, #0
-; NEON-NOSVE-NEXT:    ushll v2.4s, v2.4h, #0
-; NEON-NOSVE-NEXT:    udivr z1.s, p0/m, z1.s, z4.s
-; NEON-NOSVE-NEXT:    ushll2 v4.4s, v3.8h, #0
-; NEON-NOSVE-NEXT:    ushll v3.4s, v3.4h, #0
-; NEON-NOSVE-NEXT:    udivr z4.s, p0/m, z4.s, z5.s
-; NEON-NOSVE-NEXT:    ldr q5, [x0]
-; NEON-NOSVE-NEXT:    ushll2 v16.8h, v5.16b, #0
-; NEON-NOSVE-NEXT:    ushll v5.8h, v5.8b, #0
-; NEON-NOSVE-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; NEON-NOSVE-NEXT:    ushll2 v18.4s, v16.8h, #0
-; NEON-NOSVE-NEXT:    ushll v16.4s, v16.4h, #0
-; NEON-NOSVE-NEXT:    udivr z17.s, p0/m, z17.s, z18.s
-; NEON-NOSVE-NEXT:    ushll2 v18.4s, v5.8h, #0
-; NEON-NOSVE-NEXT:    ushll v5.4s, v5.4h, #0
-; NEON-NOSVE-NEXT:    udivr z7.s, p0/m, z7.s, z16.s
-; NEON-NOSVE-NEXT:    ushll2 v16.4s, v6.8h, #0
-; NEON-NOSVE-NEXT:    ushll v6.4s, v6.4h, #0
-; NEON-NOSVE-NEXT:    udivr z16.s, p0/m, z16.s, z18.s
-; NEON-NOSVE-NEXT:    udiv z5.s, p0/m, z5.s, z6.s
-; NEON-NOSVE-NEXT:    udiv z2.s, p0/m, z2.s, z3.s
-; NEON-NOSVE-NEXT:    uzp1 v3.8h, v7.8h, v17.8h
-; NEON-NOSVE-NEXT:    uzp1 v5.8h, v5.8h, v16.8h
-; NEON-NOSVE-NEXT:    uzp1 v1.8h, v2.8h, v4.8h
-; NEON-NOSVE-NEXT:    uzp1 v2.16b, v5.16b, v3.16b
-; NEON-NOSVE-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
-; NEON-NOSVE-NEXT:    stp q2, q0, [x0]
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: udiv_v32i8:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #96
@@ -1697,16 +1385,6 @@ define <2 x i16> @udiv_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: udiv_v2i16:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    movi d2, #0x00ffff0000ffff
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl2
-; NEON-NOSVE-NEXT:    and v1.8b, v1.8b, v2.8b
-; NEON-NOSVE-NEXT:    and v0.8b, v0.8b, v2.8b
-; NEON-NOSVE-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: udiv_v2i16:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #32
@@ -1739,15 +1417,6 @@ define <4 x i16> @udiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: udiv_v4i16:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    xtn v0.4h, v0.4s
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: udiv_v4i16:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #32
@@ -1789,26 +1458,14 @@ define <8 x i16> @udiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    uunpklo z1.s, z1.h
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
 ; CHECK-NEXT:    udivr z2.s, p0/m, z2.s, z3.s
-; CHECK-NEXT:    udivr z1.s, p0/m, z1.s, z0.s
+; CHECK-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z0.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
+; CHECK-NEXT:    uzp1 z1.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z2.h, z0.h, z0.h
+; CHECK-NEXT:    splice z0.h, p0, { z1.h, z2.h }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: udiv_v8i16:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ushll2 v2.4s, v1.8h, #0
-; NEON-NOSVE-NEXT:    ushll2 v3.4s, v0.8h, #0
-; NEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    udivr z2.s, p0/m, z2.s, z3.s
-; NEON-NOSVE-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: udiv_v8i16:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
@@ -1873,41 +1530,18 @@ define void @udiv_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
 ; CHECK-NEXT:    uunpklo z3.s, z3.h
 ; CHECK-NEXT:    udivr z5.s, p0/m, z5.s, z6.s
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
 ; CHECK-NEXT:    udiv z3.s, p0/m, z3.s, z4.s
+; CHECK-NEXT:    uzp1 z4.h, z5.h, z5.h
 ; CHECK-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
-; CHECK-NEXT:    uzp1 z1.h, z5.h, z5.h
+; CHECK-NEXT:    uzp1 z1.h, z2.h, z2.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    splice z1.h, p0, z1.h, z3.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z0.h
-; CHECK-NEXT:    stp q1, q2, [x0]
+; CHECK-NEXT:    uzp1 z5.h, z3.h, z3.h
+; CHECK-NEXT:    uzp1 z2.h, z0.h, z0.h
+; CHECK-NEXT:    splice z0.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    splice z1.h, p0, { z1.h, z2.h }
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: udiv_v16i16:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ldp q4, q1, [x1]
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NEON-NOSVE-NEXT:    ushll2 v2.4s, v1.8h, #0
-; NEON-NOSVE-NEXT:    ushll2 v3.4s, v0.8h, #0
-; NEON-NOSVE-NEXT:    ushll2 v5.4s, v4.8h, #0
-; NEON-NOSVE-NEXT:    ushll v4.4s, v4.4h, #0
-; NEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NEON-NOSVE-NEXT:    udivr z2.s, p0/m, z2.s, z3.s
-; NEON-NOSVE-NEXT:    ldr q3, [x0]
-; NEON-NOSVE-NEXT:    ushll2 v6.4s, v3.8h, #0
-; NEON-NOSVE-NEXT:    ushll v3.4s, v3.4h, #0
-; NEON-NOSVE-NEXT:    udivr z5.s, p0/m, z5.s, z6.s
-; NEON-NOSVE-NEXT:    udiv z3.s, p0/m, z3.s, z4.s
-; NEON-NOSVE-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    uzp1 v1.8h, v3.8h, v5.8h
-; NEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; NEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: udiv_v16i16:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #96
@@ -2001,15 +1635,6 @@ define <2 x i32> @udiv_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: udiv_v2i32:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl2
-; NEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $z0
-; NEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $z1
-; NEON-NOSVE-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: udiv_v2i32:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #32
@@ -2038,15 +1663,6 @@ define <4 x i32> @udiv_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: udiv_v4i32:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    // kill: def $q0 killed $q0 def $z0
-; NEON-NOSVE-NEXT:    // kill: def $q1 killed $q1 def $z1
-; NEON-NOSVE-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: udiv_v4i32:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
@@ -2082,17 +1698,6 @@ define void @udiv_v8i32(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: udiv_v8i32:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NEON-NOSVE-NEXT:    ptrue p0.s, vl4
-; NEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NEON-NOSVE-NEXT:    udivr z0.s, p0/m, z0.s, z1.s
-; NEON-NOSVE-NEXT:    movprfx z1, z2
-; NEON-NOSVE-NEXT:    udiv z1.s, p0/m, z1.s, z3.s
-; NEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: udiv_v8i32:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #96
@@ -2146,15 +1751,6 @@ define <1 x i64> @udiv_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: udiv_v1i64:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ptrue p0.d, vl1
-; NEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $z0
-; NEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $z1
-; NEON-NOSVE-NEXT:    udiv z0.d, p0/m, z0.d, z1.d
-; NEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: udiv_v1i64:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #16
@@ -2180,15 +1776,6 @@ define <2 x i64> @udiv_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: udiv_v2i64:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ptrue p0.d, vl2
-; NEON-NOSVE-NEXT:    // kill: def $q0 killed $q0 def $z0
-; NEON-NOSVE-NEXT:    // kill: def $q1 killed $q1 def $z1
-; NEON-NOSVE-NEXT:    udiv z0.d, p0/m, z0.d, z1.d
-; NEON-NOSVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: udiv_v2i64:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
@@ -2218,17 +1805,6 @@ define void @udiv_v4i64(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
-; NEON-NOSVE-LABEL: udiv_v4i64:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NEON-NOSVE-NEXT:    ptrue p0.d, vl2
-; NEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NEON-NOSVE-NEXT:    udivr z0.d, p0/m, z0.d, z1.d
-; NEON-NOSVE-NEXT:    movprfx z1, z2
-; NEON-NOSVE-NEXT:    udiv z1.d, p0/m, z1.d, z3.d
-; NEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NEON-NOSVE-NEXT:    ret
-;
 ; NONEON-NOSVE-LABEL: udiv_v4i64:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    sub sp, sp, #96
@@ -2261,64 +1837,22 @@ define void @udiv_v4i64(ptr %a, ptr %b)  {
 }
 
 define void @udiv_constantsplat_v8i32(ptr %a)  {
-; SVE-LABEL: udiv_constantsplat_v8i32:
-; SVE:       // %bb.0:
-; SVE-NEXT:    mov w8, #8969 // =0x2309
-; SVE-NEXT:    ldp q1, q2, [x0]
-; SVE-NEXT:    movk w8, #22765, lsl #16
-; SVE-NEXT:    ptrue p0.s, vl4
-; SVE-NEXT:    mov z0.s, w8
-; SVE-NEXT:    movprfx z3, z1
-; SVE-NEXT:    umulh z3.s, p0/m, z3.s, z0.s
-; SVE-NEXT:    umulh z0.s, p0/m, z0.s, z2.s
-; SVE-NEXT:    sub z1.s, z1.s, z3.s
-; SVE-NEXT:    sub z2.s, z2.s, z0.s
-; SVE-NEXT:    lsr z1.s, z1.s, #1
-; SVE-NEXT:    lsr z2.s, z2.s, #1
-; SVE-NEXT:    add z1.s, z1.s, z3.s
-; SVE-NEXT:    add z0.s, z2.s, z0.s
-; SVE-NEXT:    lsr z1.s, z1.s, #6
-; SVE-NEXT:    lsr z0.s, z0.s, #6
-; SVE-NEXT:    stp q1, q0, [x0]
-; SVE-NEXT:    ret
-;
-; SVE2-LABEL: udiv_constantsplat_v8i32:
-; SVE2:       // %bb.0:
-; SVE2-NEXT:    mov w8, #8969 // =0x2309
-; SVE2-NEXT:    ldp q1, q2, [x0]
-; SVE2-NEXT:    movk w8, #22765, lsl #16
-; SVE2-NEXT:    mov z0.s, w8
-; SVE2-NEXT:    umulh z3.s, z1.s, z0.s
-; SVE2-NEXT:    umulh z0.s, z2.s, z0.s
-; SVE2-NEXT:    sub z1.s, z1.s, z3.s
-; SVE2-NEXT:    sub z2.s, z2.s, z0.s
-; SVE2-NEXT:    usra z3.s, z1.s, #1
-; SVE2-NEXT:    usra z0.s, z2.s, #1
-; SVE2-NEXT:    lsr z1.s, z3.s, #6
-; SVE2-NEXT:    lsr z0.s, z0.s, #6
-; SVE2-NEXT:    stp q1, q0, [x0]
-; SVE2-NEXT:    ret
-;
-; NEON-NOSVE-LABEL: udiv_constantsplat_v8i32:
-; NEON-NOSVE:       // %bb.0:
-; NEON-NOSVE-NEXT:    mov w8, #8969 // =0x2309
-; NEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NEON-NOSVE-NEXT:    movk w8, #22765, lsl #16
-; NEON-NOSVE-NEXT:    dup v0.4s, w8
-; NEON-NOSVE-NEXT:    umull2 v3.2d, v1.4s, v0.4s
-; NEON-NOSVE-NEXT:    umull v4.2d, v1.2s, v0.2s
-; NEON-NOSVE-NEXT:    umull2 v5.2d, v2.4s, v0.4s
-; NEON-NOSVE-NEXT:    umull v0.2d, v2.2s, v0.2s
-; NEON-NOSVE-NEXT:    uzp2 v3.4s, v4.4s, v3.4s
-; NEON-NOSVE-NEXT:    uzp2 v0.4s, v0.4s, v5.4s
-; NEON-NOSVE-NEXT:    sub v1.4s, v1.4s, v3.4s
-; NEON-NOSVE-NEXT:    sub v2.4s, v2.4s, v0.4s
-; NEON-NOSVE-NEXT:    usra v3.4s, v1.4s, #1
-; NEON-NOSVE-NEXT:    usra v0.4s, v2.4s, #1
-; NEON-NOSVE-NEXT:    ushr v1.4s, v3.4s, #6
-; NEON-NOSVE-NEXT:    ushr v0.4s, v0.4s, #6
-; NEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NEON-NOSVE-NEXT:    ret
+; CHECK-LABEL: udiv_constantsplat_v8i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #8969 // =0x2309
+; CHECK-NEXT:    ldp q1, q2, [x0]
+; CHECK-NEXT:    movk w8, #22765, lsl #16
+; CHECK-NEXT:    mov z0.s, w8
+; CHECK-NEXT:    umulh z3.s, z1.s, z0.s
+; CHECK-NEXT:    umulh z0.s, z2.s, z0.s
+; CHECK-NEXT:    sub z1.s, z1.s, z3.s
+; CHECK-NEXT:    sub z2.s, z2.s, z0.s
+; CHECK-NEXT:    usra z3.s, z1.s, #1
+; CHECK-NEXT:    usra z0.s, z2.s, #1
+; CHECK-NEXT:    lsr z1.s, z3.s, #6
+; CHECK-NEXT:    lsr z0.s, z0.s, #6
+; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: udiv_constantsplat_v8i32:
 ; NONEON-NOSVE:       // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll
index b4641172f8b0..9497ec88e57b 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mattr=+sve -force-streaming-compatible  < %s | FileCheck %s
+; RUN: llc -mattr=+sve2 -force-streaming-compatible  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming  < %s | FileCheck %s
 ; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
@@ -76,10 +76,10 @@ define <8 x i8> @srem_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    sdivr z2.s, p0/m, z2.s, z3.s
 ; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    uzp1 z3.h, z4.h, z4.h
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
-; CHECK-NEXT:    splice z3.h, p0, z3.h, z2.h
+; CHECK-NEXT:    uzp1 z4.h, z2.h, z2.h
+; CHECK-NEXT:    splice z2.h, p0, { z3.h, z4.h }
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z2.b, z3.b, z3.b
+; CHECK-NEXT:    uzp1 z2.b, z2.b, z2.b
 ; CHECK-NEXT:    mls z0.b, p0/m, z2.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
@@ -160,23 +160,23 @@ define <16 x i8> @srem_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    mov z3.d, z1.d
 ; CHECK-NEXT:    sunpklo z5.s, z5.h
 ; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
-; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
 ; CHECK-NEXT:    sunpklo z3.h, z3.b
 ; CHECK-NEXT:    sunpklo z6.s, z3.h
 ; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
 ; CHECK-NEXT:    sunpklo z3.s, z3.h
 ; CHECK-NEXT:    sdivr z6.s, p0/m, z6.s, z7.s
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
 ; CHECK-NEXT:    sdivr z3.s, p0/m, z3.s, z5.s
+; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    splice z4.h, p0, z4.h, z2.h
-; CHECK-NEXT:    uzp1 z5.h, z6.h, z6.h
-; CHECK-NEXT:    uzp1 z2.b, z4.b, z4.b
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    splice z5.h, p0, z5.h, z3.h
+; CHECK-NEXT:    uzp1 z5.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
+; CHECK-NEXT:    splice z2.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    uzp1 z4.b, z2.b, z2.b
+; CHECK-NEXT:    uzp1 z7.h, z3.h, z3.h
+; CHECK-NEXT:    splice z3.h, p0, { z6.h, z7.h }
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z3.b, z5.b, z5.b
-; CHECK-NEXT:    splice z2.b, p0, z2.b, z3.b
+; CHECK-NEXT:    uzp1 z5.b, z3.b, z3.b
+; CHECK-NEXT:    splice z2.b, p0, { z4.b, z5.b }
 ; CHECK-NEXT:    ptrue p0.b, vl16
 ; CHECK-NEXT:    mls z0.b, p0/m, z2.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
@@ -300,14 +300,12 @@ define void @srem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sunpklo z4.s, z16.h
 ; CHECK-NEXT:    ext z7.b, z7.b, z7.b, #8
 ; CHECK-NEXT:    ext z16.b, z16.b, z16.b, #8
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
 ; CHECK-NEXT:    sunpklo z7.s, z7.h
 ; CHECK-NEXT:    movprfx z6, z4
 ; CHECK-NEXT:    sdiv z6.s, p0/m, z6.s, z3.s
 ; CHECK-NEXT:    ldr q3, [x0]
 ; CHECK-NEXT:    ldr q4, [x1]
 ; CHECK-NEXT:    sunpklo z16.s, z16.h
-; CHECK-NEXT:    uzp1 z5.h, z5.h, z5.h
 ; CHECK-NEXT:    sunpklo z17.h, z4.b
 ; CHECK-NEXT:    sunpklo z18.h, z3.b
 ; CHECK-NEXT:    sdivr z7.s, p0/m, z7.s, z16.s
@@ -317,11 +315,9 @@ define void @srem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ext z18.b, z18.b, z18.b, #8
 ; CHECK-NEXT:    sunpklo z17.s, z17.h
 ; CHECK-NEXT:    sunpklo z18.s, z18.h
-; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
 ; CHECK-NEXT:    sdivr z19.s, p0/m, z19.s, z20.s
 ; CHECK-NEXT:    mov z20.d, z3.d
 ; CHECK-NEXT:    ext z20.b, z20.b, z3.b, #8
-; CHECK-NEXT:    uzp1 z7.h, z7.h, z7.h
 ; CHECK-NEXT:    sunpklo z20.h, z20.b
 ; CHECK-NEXT:    sunpklo z22.s, z20.h
 ; CHECK-NEXT:    ext z20.b, z20.b, z20.b, #8
@@ -329,32 +325,36 @@ define void @srem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z18.d, z4.d
 ; CHECK-NEXT:    sunpklo z20.s, z20.h
 ; CHECK-NEXT:    ext z18.b, z18.b, z4.b, #8
-; CHECK-NEXT:    uzp1 z16.h, z19.h, z19.h
 ; CHECK-NEXT:    sunpklo z18.h, z18.b
 ; CHECK-NEXT:    sunpklo z21.s, z18.h
 ; CHECK-NEXT:    ext z18.b, z18.b, z18.b, #8
 ; CHECK-NEXT:    sunpklo z18.s, z18.h
 ; CHECK-NEXT:    sdivr z21.s, p0/m, z21.s, z22.s
-; CHECK-NEXT:    uzp1 z17.h, z17.h, z17.h
+; CHECK-NEXT:    uzp1 z22.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z23.h, z5.h, z5.h
+; CHECK-NEXT:    uzp1 z5.h, z6.h, z6.h
+; CHECK-NEXT:    uzp1 z6.h, z7.h, z7.h
 ; CHECK-NEXT:    sdivr z18.s, p0/m, z18.s, z20.s
+; CHECK-NEXT:    uzp1 z19.h, z19.h, z19.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    splice z16.h, p0, z16.h, z17.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z5.h
-; CHECK-NEXT:    splice z6.h, p0, z6.h, z7.h
-; CHECK-NEXT:    uzp1 z19.h, z21.h, z21.h
-; CHECK-NEXT:    uzp1 z5.b, z16.b, z16.b
-; CHECK-NEXT:    uzp1 z2.b, z2.b, z2.b
-; CHECK-NEXT:    uzp1 z6.b, z6.b, z6.b
-; CHECK-NEXT:    uzp1 z18.h, z18.h, z18.h
-; CHECK-NEXT:    splice z19.h, p0, z19.h, z18.h
+; CHECK-NEXT:    uzp1 z20.h, z17.h, z17.h
+; CHECK-NEXT:    splice z7.h, p0, { z22.h, z23.h }
+; CHECK-NEXT:    splice z5.h, p0, { z5.h, z6.h }
+; CHECK-NEXT:    uzp1 z16.h, z21.h, z21.h
+; CHECK-NEXT:    splice z2.h, p0, { z19.h, z20.h }
+; CHECK-NEXT:    uzp1 z6.b, z7.b, z7.b
+; CHECK-NEXT:    uzp1 z7.b, z5.b, z5.b
+; CHECK-NEXT:    uzp1 z17.h, z18.h, z18.h
+; CHECK-NEXT:    splice z16.h, p0, { z16.h, z17.h }
+; CHECK-NEXT:    uzp1 z17.b, z2.b, z2.b
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    splice z2.b, p0, z2.b, z6.b
-; CHECK-NEXT:    uzp1 z7.b, z19.b, z19.b
-; CHECK-NEXT:    splice z5.b, p0, z5.b, z7.b
+; CHECK-NEXT:    splice z5.b, p0, { z6.b, z7.b }
+; CHECK-NEXT:    uzp1 z18.b, z16.b, z16.b
+; CHECK-NEXT:    splice z2.b, p0, { z17.b, z18.b }
 ; CHECK-NEXT:    ptrue p0.b, vl16
-; CHECK-NEXT:    mls z0.b, p0/m, z2.b, z1.b
-; CHECK-NEXT:    mls z3.b, p0/m, z5.b, z4.b
-; CHECK-NEXT:    stp q3, q0, [x0]
+; CHECK-NEXT:    mls z0.b, p0/m, z5.b, z1.b
+; CHECK-NEXT:    msb z2.b, p0/m, z4.b, z3.b
+; CHECK-NEXT:    stp q2, q0, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: srem_v32i8:
@@ -600,9 +600,9 @@ define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    sunpklo z3.s, z3.h
 ; CHECK-NEXT:    sdivr z3.s, p0/m, z3.s, z4.s
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z3.h
+; CHECK-NEXT:    uzp1 z4.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z5.h, z3.h, z3.h
+; CHECK-NEXT:    splice z2.h, p0, { z4.h, z5.h }
 ; CHECK-NEXT:    ptrue p0.h, vl8
 ; CHECK-NEXT:    mls z0.h, p0/m, z2.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
@@ -680,23 +680,23 @@ define void @srem_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sdivr z5.s, p0/m, z5.s, z6.s
 ; CHECK-NEXT:    mov z6.d, z4.d
 ; CHECK-NEXT:    ext z6.b, z6.b, z4.b, #8
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
 ; CHECK-NEXT:    sunpklo z6.s, z6.h
 ; CHECK-NEXT:    sdivr z6.s, p0/m, z6.s, z7.s
 ; CHECK-NEXT:    mov z7.d, z1.d
 ; CHECK-NEXT:    ext z7.b, z7.b, z1.b, #8
-; CHECK-NEXT:    uzp1 z5.h, z5.h, z5.h
 ; CHECK-NEXT:    sunpklo z7.s, z7.h
 ; CHECK-NEXT:    sdivr z7.s, p0/m, z7.s, z16.s
+; CHECK-NEXT:    uzp1 z16.h, z5.h, z5.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
-; CHECK-NEXT:    splice z5.h, p0, z5.h, z6.h
-; CHECK-NEXT:    uzp1 z7.h, z7.h, z7.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z7.h
+; CHECK-NEXT:    uzp1 z17.h, z6.h, z6.h
+; CHECK-NEXT:    uzp1 z5.h, z2.h, z2.h
+; CHECK-NEXT:    splice z2.h, p0, { z16.h, z17.h }
+; CHECK-NEXT:    uzp1 z6.h, z7.h, z7.h
+; CHECK-NEXT:    splice z5.h, p0, { z5.h, z6.h }
 ; CHECK-NEXT:    ptrue p0.h, vl8
-; CHECK-NEXT:    mls z3.h, p0/m, z5.h, z4.h
-; CHECK-NEXT:    mls z0.h, p0/m, z2.h, z1.h
-; CHECK-NEXT:    stp q3, q0, [x0]
+; CHECK-NEXT:    msb z2.h, p0/m, z4.h, z3.h
+; CHECK-NEXT:    mls z0.h, p0/m, z5.h, z1.h
+; CHECK-NEXT:    stp q2, q0, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: srem_v16i16:
@@ -1126,10 +1126,10 @@ define <8 x i8> @urem_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    udivr z2.s, p0/m, z2.s, z3.s
 ; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    uzp1 z3.h, z4.h, z4.h
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
-; CHECK-NEXT:    splice z3.h, p0, z3.h, z2.h
+; CHECK-NEXT:    uzp1 z4.h, z2.h, z2.h
+; CHECK-NEXT:    splice z2.h, p0, { z3.h, z4.h }
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z2.b, z3.b, z3.b
+; CHECK-NEXT:    uzp1 z2.b, z2.b, z2.b
 ; CHECK-NEXT:    mls z0.b, p0/m, z2.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
@@ -1210,23 +1210,23 @@ define <16 x i8> @urem_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    mov z3.d, z1.d
 ; CHECK-NEXT:    uunpklo z5.s, z5.h
 ; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
-; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
 ; CHECK-NEXT:    uunpklo z3.h, z3.b
 ; CHECK-NEXT:    uunpklo z6.s, z3.h
 ; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
 ; CHECK-NEXT:    uunpklo z3.s, z3.h
 ; CHECK-NEXT:    udivr z6.s, p0/m, z6.s, z7.s
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
 ; CHECK-NEXT:    udivr z3.s, p0/m, z3.s, z5.s
+; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    splice z4.h, p0, z4.h, z2.h
-; CHECK-NEXT:    uzp1 z5.h, z6.h, z6.h
-; CHECK-NEXT:    uzp1 z2.b, z4.b, z4.b
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    splice z5.h, p0, z5.h, z3.h
+; CHECK-NEXT:    uzp1 z5.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
+; CHECK-NEXT:    splice z2.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    uzp1 z4.b, z2.b, z2.b
+; CHECK-NEXT:    uzp1 z7.h, z3.h, z3.h
+; CHECK-NEXT:    splice z3.h, p0, { z6.h, z7.h }
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z3.b, z5.b, z5.b
-; CHECK-NEXT:    splice z2.b, p0, z2.b, z3.b
+; CHECK-NEXT:    uzp1 z5.b, z3.b, z3.b
+; CHECK-NEXT:    splice z2.b, p0, { z4.b, z5.b }
 ; CHECK-NEXT:    ptrue p0.b, vl16
 ; CHECK-NEXT:    mls z0.b, p0/m, z2.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
@@ -1350,14 +1350,12 @@ define void @urem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    uunpklo z4.s, z16.h
 ; CHECK-NEXT:    ext z7.b, z7.b, z7.b, #8
 ; CHECK-NEXT:    ext z16.b, z16.b, z16.b, #8
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
 ; CHECK-NEXT:    uunpklo z7.s, z7.h
 ; CHECK-NEXT:    movprfx z6, z4
 ; CHECK-NEXT:    udiv z6.s, p0/m, z6.s, z3.s
 ; CHECK-NEXT:    ldr q3, [x0]
 ; CHECK-NEXT:    ldr q4, [x1]
 ; CHECK-NEXT:    uunpklo z16.s, z16.h
-; CHECK-NEXT:    uzp1 z5.h, z5.h, z5.h
 ; CHECK-NEXT:    uunpklo z17.h, z4.b
 ; CHECK-NEXT:    uunpklo z18.h, z3.b
 ; CHECK-NEXT:    udivr z7.s, p0/m, z7.s, z16.s
@@ -1367,11 +1365,9 @@ define void @urem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ext z18.b, z18.b, z18.b, #8
 ; CHECK-NEXT:    uunpklo z17.s, z17.h
 ; CHECK-NEXT:    uunpklo z18.s, z18.h
-; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
 ; CHECK-NEXT:    udivr z19.s, p0/m, z19.s, z20.s
 ; CHECK-NEXT:    mov z20.d, z3.d
 ; CHECK-NEXT:    ext z20.b, z20.b, z3.b, #8
-; CHECK-NEXT:    uzp1 z7.h, z7.h, z7.h
 ; CHECK-NEXT:    uunpklo z20.h, z20.b
 ; CHECK-NEXT:    uunpklo z22.s, z20.h
 ; CHECK-NEXT:    ext z20.b, z20.b, z20.b, #8
@@ -1379,32 +1375,36 @@ define void @urem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z18.d, z4.d
 ; CHECK-NEXT:    uunpklo z20.s, z20.h
 ; CHECK-NEXT:    ext z18.b, z18.b, z4.b, #8
-; CHECK-NEXT:    uzp1 z16.h, z19.h, z19.h
 ; CHECK-NEXT:    uunpklo z18.h, z18.b
 ; CHECK-NEXT:    uunpklo z21.s, z18.h
 ; CHECK-NEXT:    ext z18.b, z18.b, z18.b, #8
 ; CHECK-NEXT:    uunpklo z18.s, z18.h
 ; CHECK-NEXT:    udivr z21.s, p0/m, z21.s, z22.s
-; CHECK-NEXT:    uzp1 z17.h, z17.h, z17.h
+; CHECK-NEXT:    uzp1 z22.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z23.h, z5.h, z5.h
+; CHECK-NEXT:    uzp1 z5.h, z6.h, z6.h
+; CHECK-NEXT:    uzp1 z6.h, z7.h, z7.h
 ; CHECK-NEXT:    udivr z18.s, p0/m, z18.s, z20.s
+; CHECK-NEXT:    uzp1 z19.h, z19.h, z19.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    splice z16.h, p0, z16.h, z17.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z5.h
-; CHECK-NEXT:    splice z6.h, p0, z6.h, z7.h
-; CHECK-NEXT:    uzp1 z19.h, z21.h, z21.h
-; CHECK-NEXT:    uzp1 z5.b, z16.b, z16.b
-; CHECK-NEXT:    uzp1 z2.b, z2.b, z2.b
-; CHECK-NEXT:    uzp1 z6.b, z6.b, z6.b
-; CHECK-NEXT:    uzp1 z18.h, z18.h, z18.h
-; CHECK-NEXT:    splice z19.h, p0, z19.h, z18.h
+; CHECK-NEXT:    uzp1 z20.h, z17.h, z17.h
+; CHECK-NEXT:    splice z7.h, p0, { z22.h, z23.h }
+; CHECK-NEXT:    splice z5.h, p0, { z5.h, z6.h }
+; CHECK-NEXT:    uzp1 z16.h, z21.h, z21.h
+; CHECK-NEXT:    splice z2.h, p0, { z19.h, z20.h }
+; CHECK-NEXT:    uzp1 z6.b, z7.b, z7.b
+; CHECK-NEXT:    uzp1 z7.b, z5.b, z5.b
+; CHECK-NEXT:    uzp1 z17.h, z18.h, z18.h
+; CHECK-NEXT:    splice z16.h, p0, { z16.h, z17.h }
+; CHECK-NEXT:    uzp1 z17.b, z2.b, z2.b
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    splice z2.b, p0, z2.b, z6.b
-; CHECK-NEXT:    uzp1 z7.b, z19.b, z19.b
-; CHECK-NEXT:    splice z5.b, p0, z5.b, z7.b
+; CHECK-NEXT:    splice z5.b, p0, { z6.b, z7.b }
+; CHECK-NEXT:    uzp1 z18.b, z16.b, z16.b
+; CHECK-NEXT:    splice z2.b, p0, { z17.b, z18.b }
 ; CHECK-NEXT:    ptrue p0.b, vl16
-; CHECK-NEXT:    mls z0.b, p0/m, z2.b, z1.b
-; CHECK-NEXT:    mls z3.b, p0/m, z5.b, z4.b
-; CHECK-NEXT:    stp q3, q0, [x0]
+; CHECK-NEXT:    mls z0.b, p0/m, z5.b, z1.b
+; CHECK-NEXT:    msb z2.b, p0/m, z4.b, z3.b
+; CHECK-NEXT:    stp q2, q0, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: urem_v32i8:
@@ -1650,9 +1650,9 @@ define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    uunpklo z3.s, z3.h
 ; CHECK-NEXT:    udivr z3.s, p0/m, z3.s, z4.s
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z3.h
+; CHECK-NEXT:    uzp1 z4.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z5.h, z3.h, z3.h
+; CHECK-NEXT:    splice z2.h, p0, { z4.h, z5.h }
 ; CHECK-NEXT:    ptrue p0.h, vl8
 ; CHECK-NEXT:    mls z0.h, p0/m, z2.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
@@ -1730,23 +1730,23 @@ define void @urem_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    udivr z5.s, p0/m, z5.s, z6.s
 ; CHECK-NEXT:    mov z6.d, z4.d
 ; CHECK-NEXT:    ext z6.b, z6.b, z4.b, #8
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
 ; CHECK-NEXT:    uunpklo z6.s, z6.h
 ; CHECK-NEXT:    udivr z6.s, p0/m, z6.s, z7.s
 ; CHECK-NEXT:    mov z7.d, z1.d
 ; CHECK-NEXT:    ext z7.b, z7.b, z1.b, #8
-; CHECK-NEXT:    uzp1 z5.h, z5.h, z5.h
 ; CHECK-NEXT:    uunpklo z7.s, z7.h
 ; CHECK-NEXT:    udivr z7.s, p0/m, z7.s, z16.s
+; CHECK-NEXT:    uzp1 z16.h, z5.h, z5.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
-; CHECK-NEXT:    splice z5.h, p0, z5.h, z6.h
-; CHECK-NEXT:    uzp1 z7.h, z7.h, z7.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z7.h
+; CHECK-NEXT:    uzp1 z17.h, z6.h, z6.h
+; CHECK-NEXT:    uzp1 z5.h, z2.h, z2.h
+; CHECK-NEXT:    splice z2.h, p0, { z16.h, z17.h }
+; CHECK-NEXT:    uzp1 z6.h, z7.h, z7.h
+; CHECK-NEXT:    splice z5.h, p0, { z5.h, z6.h }
 ; CHECK-NEXT:    ptrue p0.h, vl8
-; CHECK-NEXT:    mls z3.h, p0/m, z5.h, z4.h
-; CHECK-NEXT:    mls z0.h, p0/m, z2.h, z1.h
-; CHECK-NEXT:    stp q3, q0, [x0]
+; CHECK-NEXT:    msb z2.h, p0/m, z4.h, z3.h
+; CHECK-NEXT:    mls z0.h, p0/m, z5.h, z1.h
+; CHECK-NEXT:    stp q2, q0, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: urem_v16i16:
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll
index 5235423c00d9..e07036f2a1ac 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mattr=+sve -force-streaming-compatible < %s | FileCheck %s
+; RUN: llc -mattr=+sve2 -force-streaming-compatible < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s
 ; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
@@ -11,28 +11,28 @@ define i1 @ptest_v16i1(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0, #32]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    ldp q2, q3, [x0]
+; CHECK-NEXT:    ldp q3, q2, [x0]
 ; CHECK-NEXT:    fcmne p1.s, p0/z, z0.s, #0.0
 ; CHECK-NEXT:    fcmne p2.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    fcmne p3.s, p0/z, z3.s, #0.0
-; CHECK-NEXT:    fcmne p0.s, p0/z, z2.s, #0.0
+; CHECK-NEXT:    fcmne p3.s, p0/z, z2.s, #0.0
+; CHECK-NEXT:    fcmne p0.s, p0/z, z3.s, #0.0
 ; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z1.s, p2/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z2.s, p3/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z3.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    splice z1.h, p0, z1.h, z0.h
-; CHECK-NEXT:    splice z3.h, p0, z3.h, z2.h
+; CHECK-NEXT:    uzp1 z5.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z4.h, z1.h, z1.h
+; CHECK-NEXT:    uzp1 z1.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z0.h, z3.h, z3.h
+; CHECK-NEXT:    splice z2.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    splice z0.h, p0, { z0.h, z1.h }
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z0.b, z1.b, z1.b
-; CHECK-NEXT:    uzp1 z1.b, z3.b, z3.b
-; CHECK-NEXT:    splice z1.b, p0, z1.b, z0.b
+; CHECK-NEXT:    uzp1 z2.b, z2.b, z2.b
+; CHECK-NEXT:    uzp1 z1.b, z0.b, z0.b
+; CHECK-NEXT:    splice z0.b, p0, { z1.b, z2.b }
 ; CHECK-NEXT:    ptrue p0.b, vl16
-; CHECK-NEXT:    umaxv b0, p0, z1.b
+; CHECK-NEXT:    umaxv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
@@ -120,49 +120,49 @@ define i1 @ptest_v16i1(ptr %a, ptr %b) {
 define i1 @ptest_or_v16i1(ptr %a, ptr %b) {
 ; CHECK-LABEL: ptest_or_v16i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q1, q0, [x0, #32]
+; CHECK-NEXT:    ldp q0, q1, [x0, #32]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    ldp q2, q3, [x0]
-; CHECK-NEXT:    ldp q4, q5, [x1, #32]
-; CHECK-NEXT:    fcmne p1.s, p0/z, z0.s, #0.0
-; CHECK-NEXT:    fcmne p2.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    ldp q0, q1, [x1]
+; CHECK-NEXT:    ldp q2, q3, [x1, #32]
+; CHECK-NEXT:    ldp q4, q5, [x0]
+; CHECK-NEXT:    fcmne p1.s, p0/z, z1.s, #0.0
+; CHECK-NEXT:    ldp q1, q6, [x1]
 ; CHECK-NEXT:    fcmne p3.s, p0/z, z3.s, #0.0
-; CHECK-NEXT:    fcmne p4.s, p0/z, z2.s, #0.0
-; CHECK-NEXT:    fcmne p5.s, p0/z, z5.s, #0.0
-; CHECK-NEXT:    fcmne p6.s, p0/z, z4.s, #0.0
-; CHECK-NEXT:    fcmne p7.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    fcmne p2.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    fcmne p5.s, p0/z, z2.s, #0.0
+; CHECK-NEXT:    fcmne p4.s, p0/z, z5.s, #0.0
+; CHECK-NEXT:    fcmne p7.s, p0/z, z4.s, #0.0
+; CHECK-NEXT:    fcmne p6.s, p0/z, z6.s, #0.0
+; CHECK-NEXT:    fcmne p0.s, p0/z, z1.s, #0.0
 ; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z1.s, p2/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z2.s, p3/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z3.s, p4/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    mov z1.s, p2/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z4.s, p5/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z5.s, p6/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
+; CHECK-NEXT:    mov z3.s, p4/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z6.s, p7/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    mov z5.s, p6/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z7.s, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
-; CHECK-NEXT:    uzp1 z5.h, z5.h, z5.h
+; CHECK-NEXT:    uzp1 z17.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z19.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z16.h, z1.h, z1.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
-; CHECK-NEXT:    uzp1 z7.h, z7.h, z7.h
-; CHECK-NEXT:    splice z1.h, p0, z1.h, z0.h
-; CHECK-NEXT:    splice z3.h, p0, z3.h, z2.h
-; CHECK-NEXT:    splice z5.h, p0, z5.h, z4.h
-; CHECK-NEXT:    splice z7.h, p0, z7.h, z6.h
+; CHECK-NEXT:    uzp1 z1.h, z3.h, z3.h
+; CHECK-NEXT:    uzp1 z18.h, z4.h, z4.h
+; CHECK-NEXT:    uzp1 z3.h, z5.h, z5.h
+; CHECK-NEXT:    uzp1 z0.h, z6.h, z6.h
+; CHECK-NEXT:    uzp1 z2.h, z7.h, z7.h
+; CHECK-NEXT:    splice z4.h, p0, { z16.h, z17.h }
+; CHECK-NEXT:    splice z5.h, p0, { z18.h, z19.h }
+; CHECK-NEXT:    splice z0.h, p0, { z0.h, z1.h }
+; CHECK-NEXT:    splice z1.h, p0, { z2.h, z3.h }
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z0.b, z1.b, z1.b
-; CHECK-NEXT:    uzp1 z1.b, z3.b, z3.b
-; CHECK-NEXT:    uzp1 z2.b, z5.b, z5.b
-; CHECK-NEXT:    uzp1 z3.b, z7.b, z7.b
-; CHECK-NEXT:    splice z1.b, p0, z1.b, z0.b
-; CHECK-NEXT:    splice z3.b, p0, z3.b, z2.b
+; CHECK-NEXT:    uzp1 z3.b, z4.b, z4.b
+; CHECK-NEXT:    uzp1 z5.b, z5.b, z5.b
+; CHECK-NEXT:    uzp1 z2.b, z0.b, z0.b
+; CHECK-NEXT:    uzp1 z4.b, z1.b, z1.b
+; CHECK-NEXT:    splice z0.b, p0, { z2.b, z3.b }
+; CHECK-NEXT:    splice z1.b, p0, { z4.b, z5.b }
 ; CHECK-NEXT:    ptrue p0.b, vl16
-; CHECK-NEXT:    orr z0.d, z1.d, z3.d
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
 ; CHECK-NEXT:    umaxv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
@@ -329,49 +329,49 @@ declare i1 @llvm.vector.reduce.or.i1.v16i1(<16 x i1>)
 define i1 @ptest_and_v16i1(ptr %a, ptr %b) {
 ; CHECK-LABEL: ptest_and_v16i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q1, q0, [x0, #32]
+; CHECK-NEXT:    ldp q0, q1, [x0, #32]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    ldp q2, q3, [x0]
-; CHECK-NEXT:    ldp q4, q5, [x1, #32]
-; CHECK-NEXT:    fcmne p1.s, p0/z, z0.s, #0.0
-; CHECK-NEXT:    fcmne p2.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    ldp q0, q1, [x1]
+; CHECK-NEXT:    ldp q2, q3, [x1, #32]
+; CHECK-NEXT:    ldp q4, q5, [x0]
+; CHECK-NEXT:    fcmne p1.s, p0/z, z1.s, #0.0
+; CHECK-NEXT:    ldp q1, q6, [x1]
 ; CHECK-NEXT:    fcmne p3.s, p0/z, z3.s, #0.0
-; CHECK-NEXT:    fcmne p4.s, p0/z, z2.s, #0.0
-; CHECK-NEXT:    fcmne p5.s, p0/z, z5.s, #0.0
-; CHECK-NEXT:    fcmne p6.s, p0/z, z4.s, #0.0
-; CHECK-NEXT:    fcmne p7.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    fcmne p2.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    fcmne p5.s, p0/z, z2.s, #0.0
+; CHECK-NEXT:    fcmne p4.s, p0/z, z5.s, #0.0
+; CHECK-NEXT:    fcmne p7.s, p0/z, z4.s, #0.0
+; CHECK-NEXT:    fcmne p6.s, p0/z, z6.s, #0.0
+; CHECK-NEXT:    fcmne p0.s, p0/z, z1.s, #0.0
 ; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z1.s, p2/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z2.s, p3/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z3.s, p4/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    mov z1.s, p2/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z4.s, p5/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z5.s, p6/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
+; CHECK-NEXT:    mov z3.s, p4/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z6.s, p7/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    mov z5.s, p6/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z7.s, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
-; CHECK-NEXT:    uzp1 z5.h, z5.h, z5.h
+; CHECK-NEXT:    uzp1 z17.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z19.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z16.h, z1.h, z1.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
-; CHECK-NEXT:    uzp1 z7.h, z7.h, z7.h
-; CHECK-NEXT:    splice z1.h, p0, z1.h, z0.h
-; CHECK-NEXT:    splice z3.h, p0, z3.h, z2.h
-; CHECK-NEXT:    splice z5.h, p0, z5.h, z4.h
-; CHECK-NEXT:    splice z7.h, p0, z7.h, z6.h
+; CHECK-NEXT:    uzp1 z1.h, z3.h, z3.h
+; CHECK-NEXT:    uzp1 z18.h, z4.h, z4.h
+; CHECK-NEXT:    uzp1 z3.h, z5.h, z5.h
+; CHECK-NEXT:    uzp1 z0.h, z6.h, z6.h
+; CHECK-NEXT:    uzp1 z2.h, z7.h, z7.h
+; CHECK-NEXT:    splice z4.h, p0, { z16.h, z17.h }
+; CHECK-NEXT:    splice z5.h, p0, { z18.h, z19.h }
+; CHECK-NEXT:    splice z0.h, p0, { z0.h, z1.h }
+; CHECK-NEXT:    splice z1.h, p0, { z2.h, z3.h }
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z0.b, z1.b, z1.b
-; CHECK-NEXT:    uzp1 z1.b, z3.b, z3.b
-; CHECK-NEXT:    uzp1 z2.b, z5.b, z5.b
-; CHECK-NEXT:    uzp1 z3.b, z7.b, z7.b
-; CHECK-NEXT:    splice z1.b, p0, z1.b, z0.b
-; CHECK-NEXT:    splice z3.b, p0, z3.b, z2.b
+; CHECK-NEXT:    uzp1 z3.b, z4.b, z4.b
+; CHECK-NEXT:    uzp1 z5.b, z5.b, z5.b
+; CHECK-NEXT:    uzp1 z2.b, z0.b, z0.b
+; CHECK-NEXT:    uzp1 z4.b, z1.b, z1.b
+; CHECK-NEXT:    splice z0.b, p0, { z2.b, z3.b }
+; CHECK-NEXT:    splice z1.b, p0, { z4.b, z5.b }
 ; CHECK-NEXT:    ptrue p0.b, vl16
-; CHECK-NEXT:    and z0.d, z1.d, z3.d
+; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    uminv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll
index c0aa162b19b7..13fcd94ea8a2 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mattr=+sve -force-streaming-compatible < %s | FileCheck %s
+; RUN: llc -mattr=+sve2 -force-streaming-compatible < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s
 ; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
@@ -129,11 +129,11 @@ define void @store_trunc_v2i64i8(ptr %ap, ptr %dest) {
 define void @store_trunc_v2i256i64(ptr %ap, ptr %dest) {
 ; CHECK-LABEL: store_trunc_v2i256i64:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0, #32]
 ; CHECK-NEXT:    ptrue p0.d, vl1
-; CHECK-NEXT:    ldr d0, [x0, #32]
-; CHECK-NEXT:    ldr d1, [x0]
-; CHECK-NEXT:    splice z1.d, p0, z1.d, z0.d
-; CHECK-NEXT:    str q1, [x1]
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    splice z0.d, p0, { z0.d, z1.d }
+; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: store_trunc_v2i256i64:
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll
index 77aaeeadcfc2..9d241f6f927e 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mattr=+sve -force-streaming-compatible < %s | FileCheck %s
+; RUN: llc -mattr=+sve2 -force-streaming-compatible < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s
 ; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
@@ -13,11 +13,11 @@ target triple = "aarch64-unknown-linux-gnu"
 define <16 x i8> @trunc_v16i16_v16i8(ptr %in) nounwind {
 ; CHECK-LABEL: trunc_v16i16_v16i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z1.b, z1.b, z1.b
-; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
-; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
+; CHECK-NEXT:    uzp1 z3.b, z0.b, z0.b
+; CHECK-NEXT:    uzp1 z2.b, z1.b, z1.b
+; CHECK-NEXT:    splice z0.b, p0, { z2.b, z3.b }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
@@ -69,18 +69,18 @@ define <16 x i8> @trunc_v16i16_v16i8(ptr %in) nounwind {
 define void @trunc_v32i16_v32i8(ptr %in, ptr %out) nounwind {
 ; CHECK-LABEL: trunc_v32i16_v32i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0, #32]
+; CHECK-NEXT:    ldp q1, q0, [x0, #32]
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    ldp q2, q3, [x0]
-; CHECK-NEXT:    uzp1 z1.b, z1.b, z1.b
-; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
-; CHECK-NEXT:    uzp1 z3.b, z3.b, z3.b
-; CHECK-NEXT:    uzp1 z2.b, z2.b, z2.b
-; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
-; CHECK-NEXT:    splice z2.b, p0, z2.b, z3.b
-; CHECK-NEXT:    add z0.b, z0.b, z0.b
+; CHECK-NEXT:    ldp q3, q2, [x0]
+; CHECK-NEXT:    uzp1 z5.b, z0.b, z0.b
+; CHECK-NEXT:    uzp1 z4.b, z1.b, z1.b
+; CHECK-NEXT:    uzp1 z1.b, z2.b, z2.b
+; CHECK-NEXT:    uzp1 z0.b, z3.b, z3.b
+; CHECK-NEXT:    splice z2.b, p0, { z4.b, z5.b }
+; CHECK-NEXT:    splice z0.b, p0, { z0.b, z1.b }
 ; CHECK-NEXT:    add z1.b, z2.b, z2.b
-; CHECK-NEXT:    stp q1, q0, [x1]
+; CHECK-NEXT:    add z0.b, z0.b, z0.b
+; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: trunc_v32i16_v32i8:
@@ -216,27 +216,27 @@ define void @trunc_v32i16_v32i8(ptr %in, ptr %out) nounwind {
 define void @trunc_v64i16_v64i8(ptr %in, ptr %out) nounwind {
 ; CHECK-LABEL: trunc_v64i16_v64i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0, #64]
+; CHECK-NEXT:    ldp q1, q0, [x0, #64]
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    ldp q2, q3, [x0]
-; CHECK-NEXT:    ldp q4, q5, [x0, #96]
-; CHECK-NEXT:    ldp q6, q7, [x0, #32]
-; CHECK-NEXT:    uzp1 z1.b, z1.b, z1.b
-; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
-; CHECK-NEXT:    uzp1 z3.b, z3.b, z3.b
-; CHECK-NEXT:    uzp1 z2.b, z2.b, z2.b
-; CHECK-NEXT:    uzp1 z5.b, z5.b, z5.b
-; CHECK-NEXT:    uzp1 z4.b, z4.b, z4.b
-; CHECK-NEXT:    uzp1 z7.b, z7.b, z7.b
-; CHECK-NEXT:    uzp1 z6.b, z6.b, z6.b
-; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
-; CHECK-NEXT:    splice z2.b, p0, z2.b, z3.b
-; CHECK-NEXT:    splice z4.b, p0, z4.b, z5.b
-; CHECK-NEXT:    splice z6.b, p0, z6.b, z7.b
+; CHECK-NEXT:    ldp q2, q3, [x0, #96]
+; CHECK-NEXT:    ldp q4, q5, [x0]
+; CHECK-NEXT:    uzp1 z7.b, z0.b, z0.b
+; CHECK-NEXT:    uzp1 z6.b, z1.b, z1.b
+; CHECK-NEXT:    ldp q1, q0, [x0, #32]
+; CHECK-NEXT:    uzp1 z17.b, z3.b, z3.b
+; CHECK-NEXT:    uzp1 z16.b, z2.b, z2.b
+; CHECK-NEXT:    uzp1 z3.b, z5.b, z5.b
+; CHECK-NEXT:    uzp1 z2.b, z4.b, z4.b
+; CHECK-NEXT:    uzp1 z5.b, z0.b, z0.b
+; CHECK-NEXT:    splice z0.b, p0, { z6.b, z7.b }
+; CHECK-NEXT:    uzp1 z4.b, z1.b, z1.b
+; CHECK-NEXT:    splice z1.b, p0, { z16.b, z17.b }
+; CHECK-NEXT:    splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT:    splice z3.b, p0, { z4.b, z5.b }
 ; CHECK-NEXT:    add z0.b, z0.b, z0.b
+; CHECK-NEXT:    add z1.b, z1.b, z1.b
 ; CHECK-NEXT:    add z2.b, z2.b, z2.b
-; CHECK-NEXT:    add z1.b, z4.b, z4.b
-; CHECK-NEXT:    add z3.b, z6.b, z6.b
+; CHECK-NEXT:    add z3.b, z3.b, z3.b
 ; CHECK-NEXT:    stp q0, q1, [x1, #32]
 ; CHECK-NEXT:    stp q2, q3, [x1]
 ; CHECK-NEXT:    ret
@@ -527,49 +527,49 @@ define void @trunc_v128i16_v128i8(ptr %in, ptr %out) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q2, q3, [x0, #192]
 ; CHECK-NEXT:    ptrue p0.b, vl8
+; CHECK-NEXT:    ldp q4, q5, [x0]
 ; CHECK-NEXT:    ldp q6, q7, [x0, #64]
-; CHECK-NEXT:    ldp q16, q17, [x0, #224]
-; CHECK-NEXT:    uzp1 z3.b, z3.b, z3.b
-; CHECK-NEXT:    uzp1 z2.b, z2.b, z2.b
-; CHECK-NEXT:    ldp q20, q21, [x0, #160]
-; CHECK-NEXT:    uzp1 z7.b, z7.b, z7.b
+; CHECK-NEXT:    uzp1 z17.b, z3.b, z3.b
+; CHECK-NEXT:    ldp q3, q18, [x0, #224]
+; CHECK-NEXT:    uzp1 z16.b, z2.b, z2.b
+; CHECK-NEXT:    ldp q2, q19, [x0, #128]
 ; CHECK-NEXT:    ldp q0, q1, [x0, #32]
-; CHECK-NEXT:    uzp1 z17.b, z17.b, z17.b
-; CHECK-NEXT:    ldp q4, q5, [x0, #96]
-; CHECK-NEXT:    uzp1 z16.b, z16.b, z16.b
-; CHECK-NEXT:    ldp q18, q19, [x0, #128]
-; CHECK-NEXT:    splice z2.b, p0, z2.b, z3.b
-; CHECK-NEXT:    uzp1 z3.b, z21.b, z21.b
-; CHECK-NEXT:    uzp1 z20.b, z20.b, z20.b
-; CHECK-NEXT:    uzp1 z6.b, z6.b, z6.b
-; CHECK-NEXT:    ldp q21, q22, [x0]
-; CHECK-NEXT:    splice z16.b, p0, z16.b, z17.b
+; CHECK-NEXT:    uzp1 z21.b, z18.b, z18.b
+; CHECK-NEXT:    ldp q18, q22, [x0, #160]
+; CHECK-NEXT:    uzp1 z20.b, z3.b, z3.b
+; CHECK-NEXT:    uzp1 z24.b, z19.b, z19.b
+; CHECK-NEXT:    ldp q3, q19, [x0, #96]
+; CHECK-NEXT:    uzp1 z23.b, z2.b, z2.b
+; CHECK-NEXT:    uzp1 z26.b, z22.b, z22.b
+; CHECK-NEXT:    splice z2.b, p0, { z16.b, z17.b }
+; CHECK-NEXT:    uzp1 z17.b, z7.b, z7.b
+; CHECK-NEXT:    uzp1 z25.b, z18.b, z18.b
+; CHECK-NEXT:    splice z7.b, p0, { z20.b, z21.b }
+; CHECK-NEXT:    uzp1 z21.b, z5.b, z5.b
 ; CHECK-NEXT:    uzp1 z19.b, z19.b, z19.b
-; CHECK-NEXT:    uzp1 z18.b, z18.b, z18.b
-; CHECK-NEXT:    uzp1 z4.b, z4.b, z4.b
-; CHECK-NEXT:    splice z20.b, p0, z20.b, z3.b
-; CHECK-NEXT:    uzp1 z3.b, z5.b, z5.b
-; CHECK-NEXT:    splice z6.b, p0, z6.b, z7.b
-; CHECK-NEXT:    uzp1 z5.b, z22.b, z22.b
-; CHECK-NEXT:    uzp1 z7.b, z21.b, z21.b
-; CHECK-NEXT:    uzp1 z1.b, z1.b, z1.b
-; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
-; CHECK-NEXT:    splice z18.b, p0, z18.b, z19.b
-; CHECK-NEXT:    add z2.b, z2.b, z2.b
-; CHECK-NEXT:    splice z4.b, p0, z4.b, z3.b
-; CHECK-NEXT:    add z3.b, z16.b, z16.b
-; CHECK-NEXT:    splice z7.b, p0, z7.b, z5.b
-; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
-; CHECK-NEXT:    add z1.b, z20.b, z20.b
-; CHECK-NEXT:    add z5.b, z18.b, z18.b
-; CHECK-NEXT:    stp q2, q3, [x1, #96]
-; CHECK-NEXT:    add z2.b, z6.b, z6.b
+; CHECK-NEXT:    uzp1 z20.b, z4.b, z4.b
+; CHECK-NEXT:    uzp1 z5.b, z1.b, z1.b
+; CHECK-NEXT:    uzp1 z16.b, z6.b, z6.b
+; CHECK-NEXT:    splice z6.b, p0, { z23.b, z24.b }
+; CHECK-NEXT:    uzp1 z18.b, z3.b, z3.b
+; CHECK-NEXT:    splice z3.b, p0, { z25.b, z26.b }
+; CHECK-NEXT:    uzp1 z4.b, z0.b, z0.b
+; CHECK-NEXT:    add z0.b, z2.b, z2.b
+; CHECK-NEXT:    add z7.b, z7.b, z7.b
+; CHECK-NEXT:    splice z1.b, p0, { z16.b, z17.b }
+; CHECK-NEXT:    splice z2.b, p0, { z18.b, z19.b }
+; CHECK-NEXT:    splice z16.b, p0, { z20.b, z21.b }
+; CHECK-NEXT:    splice z4.b, p0, { z4.b, z5.b }
+; CHECK-NEXT:    add z6.b, z6.b, z6.b
+; CHECK-NEXT:    add z3.b, z3.b, z3.b
+; CHECK-NEXT:    stp q0, q7, [x1, #96]
+; CHECK-NEXT:    add z0.b, z1.b, z1.b
+; CHECK-NEXT:    add z1.b, z2.b, z2.b
+; CHECK-NEXT:    add z2.b, z16.b, z16.b
+; CHECK-NEXT:    stp q6, q3, [x1, #64]
 ; CHECK-NEXT:    add z3.b, z4.b, z4.b
-; CHECK-NEXT:    add z4.b, z7.b, z7.b
-; CHECK-NEXT:    add z0.b, z0.b, z0.b
-; CHECK-NEXT:    stp q5, q1, [x1, #64]
-; CHECK-NEXT:    stp q2, q3, [x1, #32]
-; CHECK-NEXT:    stp q4, q0, [x1]
+; CHECK-NEXT:    stp q0, q1, [x1, #32]
+; CHECK-NEXT:    stp q2, q3, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: trunc_v128i16_v128i8:
@@ -1181,11 +1181,11 @@ define void @trunc_v128i16_v128i8(ptr %in, ptr %out) nounwind {
 define <8 x i8> @trunc_v8i32_v8i8(ptr %in) nounwind {
 ; CHECK-LABEL: trunc_v8i32_v8i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
+; CHECK-NEXT:    uzp1 z3.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z2.h, z1.h, z1.h
+; CHECK-NEXT:    splice z0.h, p0, { z2.h, z3.h }
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
@@ -1219,17 +1219,17 @@ define <16 x i8> @trunc_v16i32_v16i8(ptr %in) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0, #32]
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    ldp q2, q3, [x0]
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
-; CHECK-NEXT:    splice z1.h, p0, z1.h, z0.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z3.h
+; CHECK-NEXT:    ldp q3, q2, [x0]
+; CHECK-NEXT:    uzp1 z5.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z4.h, z1.h, z1.h
+; CHECK-NEXT:    uzp1 z1.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z0.h, z3.h, z3.h
+; CHECK-NEXT:    splice z2.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    splice z0.h, p0, { z0.h, z1.h }
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z1.b, z1.b, z1.b
-; CHECK-NEXT:    uzp1 z0.b, z2.b, z2.b
-; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
+; CHECK-NEXT:    uzp1 z2.b, z2.b, z2.b
+; CHECK-NEXT:    uzp1 z1.b, z0.b, z0.b
+; CHECK-NEXT:    splice z0.b, p0, { z1.b, z2.b }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
@@ -1277,32 +1277,32 @@ define <16 x i8> @trunc_v16i32_v16i8(ptr %in) nounwind {
 define void @trunc_v32i32_v32i8(ptr %in, ptr %out) nounwind {
 ; CHECK-LABEL: trunc_v32i32_v32i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0, #32]
+; CHECK-NEXT:    ldp q0, q1, [x0, #96]
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    ldp q2, q3, [x0, #96]
+; CHECK-NEXT:    ldp q2, q3, [x0, #32]
 ; CHECK-NEXT:    ldp q4, q5, [x0, #64]
 ; CHECK-NEXT:    ldp q6, q7, [x0]
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    uzp1 z5.h, z5.h, z5.h
-; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
-; CHECK-NEXT:    uzp1 z7.h, z7.h, z7.h
-; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z3.h
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
-; CHECK-NEXT:    splice z4.h, p0, z4.h, z5.h
-; CHECK-NEXT:    splice z6.h, p0, z6.h, z7.h
+; CHECK-NEXT:    uzp1 z17.h, z1.h, z1.h
+; CHECK-NEXT:    uzp1 z16.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z1.h, z3.h, z3.h
+; CHECK-NEXT:    uzp1 z19.h, z5.h, z5.h
+; CHECK-NEXT:    uzp1 z0.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z3.h, z7.h, z7.h
+; CHECK-NEXT:    uzp1 z18.h, z4.h, z4.h
+; CHECK-NEXT:    uzp1 z2.h, z6.h, z6.h
+; CHECK-NEXT:    splice z4.h, p0, { z16.h, z17.h }
+; CHECK-NEXT:    splice z0.h, p0, { z0.h, z1.h }
+; CHECK-NEXT:    splice z5.h, p0, { z18.h, z19.h }
+; CHECK-NEXT:    splice z1.h, p0, { z2.h, z3.h }
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z1.b, z2.b, z2.b
-; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
-; CHECK-NEXT:    uzp1 z2.b, z4.b, z4.b
-; CHECK-NEXT:    uzp1 z3.b, z6.b, z6.b
-; CHECK-NEXT:    splice z2.b, p0, z2.b, z1.b
-; CHECK-NEXT:    splice z3.b, p0, z3.b, z0.b
-; CHECK-NEXT:    add z0.b, z2.b, z2.b
-; CHECK-NEXT:    add z1.b, z3.b, z3.b
+; CHECK-NEXT:    uzp1 z3.b, z4.b, z4.b
+; CHECK-NEXT:    uzp1 z7.b, z0.b, z0.b
+; CHECK-NEXT:    uzp1 z2.b, z5.b, z5.b
+; CHECK-NEXT:    uzp1 z6.b, z1.b, z1.b
+; CHECK-NEXT:    splice z0.b, p0, { z2.b, z3.b }
+; CHECK-NEXT:    splice z1.b, p0, { z6.b, z7.b }
+; CHECK-NEXT:    add z0.b, z0.b, z0.b
+; CHECK-NEXT:    add z1.b, z1.b, z1.b
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
 ;
@@ -1429,56 +1429,56 @@ define void @trunc_v64i32_v64i8(ptr %in, ptr %out) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q2, q3, [x0, #160]
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    ldp q4, q5, [x0, #128]
+; CHECK-NEXT:    ldp q4, q5, [x0, #96]
+; CHECK-NEXT:    ldp q6, q7, [x0]
+; CHECK-NEXT:    uzp1 z17.h, z3.h, z3.h
+; CHECK-NEXT:    ldp q3, q18, [x0, #128]
+; CHECK-NEXT:    uzp1 z16.h, z2.h, z2.h
+; CHECK-NEXT:    ldp q2, q19, [x0, #192]
 ; CHECK-NEXT:    ldp q0, q1, [x0, #64]
-; CHECK-NEXT:    ldp q6, q7, [x0, #96]
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z5.h, z5.h, z5.h
-; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
-; CHECK-NEXT:    ldp q16, q17, [x0]
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
-; CHECK-NEXT:    ldp q18, q19, [x0, #192]
-; CHECK-NEXT:    uzp1 z7.h, z7.h, z7.h
-; CHECK-NEXT:    ldp q20, q21, [x0, #224]
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z3.h
-; CHECK-NEXT:    ldp q22, q23, [x0, #32]
-; CHECK-NEXT:    splice z4.h, p0, z4.h, z5.h
-; CHECK-NEXT:    uzp1 z19.h, z19.h, z19.h
-; CHECK-NEXT:    uzp1 z18.h, z18.h, z18.h
-; CHECK-NEXT:    uzp1 z17.h, z17.h, z17.h
-; CHECK-NEXT:    uzp1 z3.h, z21.h, z21.h
-; CHECK-NEXT:    uzp1 z5.h, z20.h, z20.h
-; CHECK-NEXT:    uzp1 z16.h, z16.h, z16.h
-; CHECK-NEXT:    uzp1 z20.h, z23.h, z23.h
-; CHECK-NEXT:    uzp1 z21.h, z22.h, z22.h
-; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    splice z18.h, p0, z18.h, z19.h
-; CHECK-NEXT:    splice z5.h, p0, z5.h, z3.h
-; CHECK-NEXT:    splice z16.h, p0, z16.h, z17.h
-; CHECK-NEXT:    splice z21.h, p0, z21.h, z20.h
-; CHECK-NEXT:    splice z6.h, p0, z6.h, z7.h
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
-; CHECK-NEXT:    uzp1 z1.b, z2.b, z2.b
-; CHECK-NEXT:    uzp1 z2.b, z4.b, z4.b
+; CHECK-NEXT:    uzp1 z21.h, z18.h, z18.h
+; CHECK-NEXT:    ldp q18, q22, [x0, #224]
+; CHECK-NEXT:    uzp1 z20.h, z3.h, z3.h
+; CHECK-NEXT:    ldp q3, q23, [x0, #32]
+; CHECK-NEXT:    splice z16.h, p0, { z16.h, z17.h }
+; CHECK-NEXT:    uzp1 z27.h, z19.h, z19.h
+; CHECK-NEXT:    uzp1 z25.h, z22.h, z22.h
+; CHECK-NEXT:    uzp1 z26.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z24.h, z18.h, z18.h
+; CHECK-NEXT:    uzp1 z18.h, z23.h, z23.h
+; CHECK-NEXT:    uzp1 z23.h, z5.h, z5.h
+; CHECK-NEXT:    uzp1 z17.h, z3.h, z3.h
+; CHECK-NEXT:    uzp1 z3.h, z7.h, z7.h
+; CHECK-NEXT:    uzp1 z22.h, z4.h, z4.h
+; CHECK-NEXT:    uzp1 z2.h, z6.h, z6.h
+; CHECK-NEXT:    uzp1 z5.h, z1.h, z1.h
+; CHECK-NEXT:    splice z1.h, p0, { z20.h, z21.h }
+; CHECK-NEXT:    splice z6.h, p0, { z24.h, z25.h }
+; CHECK-NEXT:    uzp1 z4.h, z0.h, z0.h
+; CHECK-NEXT:    splice z0.h, p0, { z26.h, z27.h }
+; CHECK-NEXT:    splice z7.h, p0, { z17.h, z18.h }
+; CHECK-NEXT:    uzp1 z17.b, z16.b, z16.b
+; CHECK-NEXT:    splice z2.h, p0, { z2.h, z3.h }
+; CHECK-NEXT:    splice z3.h, p0, { z22.h, z23.h }
+; CHECK-NEXT:    splice z4.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    uzp1 z16.b, z1.b, z1.b
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z4.b, z18.b, z18.b
-; CHECK-NEXT:    uzp1 z3.b, z5.b, z5.b
-; CHECK-NEXT:    uzp1 z7.b, z16.b, z16.b
-; CHECK-NEXT:    uzp1 z5.b, z21.b, z21.b
-; CHECK-NEXT:    splice z2.b, p0, z2.b, z1.b
-; CHECK-NEXT:    uzp1 z1.b, z6.b, z6.b
-; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
-; CHECK-NEXT:    splice z4.b, p0, z4.b, z3.b
-; CHECK-NEXT:    splice z7.b, p0, z7.b, z5.b
-; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
-; CHECK-NEXT:    add z1.b, z2.b, z2.b
-; CHECK-NEXT:    add z2.b, z4.b, z4.b
-; CHECK-NEXT:    add z3.b, z7.b, z7.b
+; CHECK-NEXT:    uzp1 z6.b, z6.b, z6.b
+; CHECK-NEXT:    uzp1 z5.b, z0.b, z0.b
+; CHECK-NEXT:    uzp1 z1.b, z7.b, z7.b
+; CHECK-NEXT:    uzp1 z0.b, z2.b, z2.b
+; CHECK-NEXT:    uzp1 z3.b, z3.b, z3.b
+; CHECK-NEXT:    splice z7.b, p0, { z16.b, z17.b }
+; CHECK-NEXT:    uzp1 z2.b, z4.b, z4.b
+; CHECK-NEXT:    splice z4.b, p0, { z5.b, z6.b }
+; CHECK-NEXT:    splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT:    splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT:    add z2.b, z7.b, z7.b
+; CHECK-NEXT:    add z3.b, z4.b, z4.b
 ; CHECK-NEXT:    add z0.b, z0.b, z0.b
-; CHECK-NEXT:    stp q1, q2, [x1, #32]
-; CHECK-NEXT:    stp q3, q0, [x1]
+; CHECK-NEXT:    add z1.b, z1.b, z1.b
+; CHECK-NEXT:    stp q2, q3, [x1, #32]
+; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: trunc_v64i32_v64i8:
@@ -1765,11 +1765,11 @@ define void @trunc_v64i32_v64i8(ptr %in, ptr %out) nounwind {
 define <8 x i16> @trunc_v8i32_v8i16(ptr %in) nounwind {
 ; CHECK-LABEL: trunc_v8i32_v8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
+; CHECK-NEXT:    uzp1 z3.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z2.h, z1.h, z1.h
+; CHECK-NEXT:    splice z0.h, p0, { z2.h, z3.h }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
@@ -1801,18 +1801,18 @@ define <8 x i16> @trunc_v8i32_v8i16(ptr %in) nounwind {
 define void @trunc_v16i32_v16i16(ptr %in, ptr %out) nounwind {
 ; CHECK-LABEL: trunc_v16i32_v16i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0, #32]
+; CHECK-NEXT:    ldp q1, q0, [x0, #32]
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    ldp q2, q3, [x0]
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z3.h
-; CHECK-NEXT:    add z0.h, z0.h, z0.h
+; CHECK-NEXT:    ldp q3, q2, [x0]
+; CHECK-NEXT:    uzp1 z5.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z4.h, z1.h, z1.h
+; CHECK-NEXT:    uzp1 z1.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z0.h, z3.h, z3.h
+; CHECK-NEXT:    splice z2.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    splice z0.h, p0, { z0.h, z1.h }
 ; CHECK-NEXT:    add z1.h, z2.h, z2.h
-; CHECK-NEXT:    stp q1, q0, [x1]
+; CHECK-NEXT:    add z0.h, z0.h, z0.h
+; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: trunc_v16i32_v16i16:
@@ -1877,27 +1877,27 @@ define void @trunc_v16i32_v16i16(ptr %in, ptr %out) nounwind {
 define void @trunc_v32i32_v32i16(ptr %in, ptr %out) nounwind {
 ; CHECK-LABEL: trunc_v32i32_v32i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0, #64]
+; CHECK-NEXT:    ldp q1, q0, [x0, #64]
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    ldp q2, q3, [x0]
-; CHECK-NEXT:    ldp q4, q5, [x0, #96]
-; CHECK-NEXT:    ldp q6, q7, [x0, #32]
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z5.h, z5.h, z5.h
-; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
-; CHECK-NEXT:    uzp1 z7.h, z7.h, z7.h
-; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z3.h
-; CHECK-NEXT:    splice z4.h, p0, z4.h, z5.h
-; CHECK-NEXT:    splice z6.h, p0, z6.h, z7.h
+; CHECK-NEXT:    ldp q2, q3, [x0, #96]
+; CHECK-NEXT:    ldp q4, q5, [x0]
+; CHECK-NEXT:    uzp1 z7.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z6.h, z1.h, z1.h
+; CHECK-NEXT:    ldp q1, q0, [x0, #32]
+; CHECK-NEXT:    uzp1 z17.h, z3.h, z3.h
+; CHECK-NEXT:    uzp1 z16.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z3.h, z5.h, z5.h
+; CHECK-NEXT:    uzp1 z2.h, z4.h, z4.h
+; CHECK-NEXT:    uzp1 z5.h, z0.h, z0.h
+; CHECK-NEXT:    splice z0.h, p0, { z6.h, z7.h }
+; CHECK-NEXT:    uzp1 z4.h, z1.h, z1.h
+; CHECK-NEXT:    splice z1.h, p0, { z16.h, z17.h }
+; CHECK-NEXT:    splice z2.h, p0, { z2.h, z3.h }
+; CHECK-NEXT:    splice z3.h, p0, { z4.h, z5.h }
 ; CHECK-NEXT:    add z0.h, z0.h, z0.h
+; CHECK-NEXT:    add z1.h, z1.h, z1.h
 ; CHECK-NEXT:    add z2.h, z2.h, z2.h
-; CHECK-NEXT:    add z1.h, z4.h, z4.h
-; CHECK-NEXT:    add z3.h, z6.h, z6.h
+; CHECK-NEXT:    add z3.h, z3.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x1, #32]
 ; CHECK-NEXT:    stp q2, q3, [x1]
 ; CHECK-NEXT:    ret
@@ -2027,49 +2027,49 @@ define void @trunc_v64i32_v64i16(ptr %in, ptr %out) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q2, q3, [x0, #192]
 ; CHECK-NEXT:    ptrue p0.h, vl4
+; CHECK-NEXT:    ldp q4, q5, [x0]
 ; CHECK-NEXT:    ldp q6, q7, [x0, #64]
-; CHECK-NEXT:    ldp q16, q17, [x0, #224]
-; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
-; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
-; CHECK-NEXT:    ldp q20, q21, [x0, #160]
-; CHECK-NEXT:    uzp1 z7.h, z7.h, z7.h
+; CHECK-NEXT:    uzp1 z17.h, z3.h, z3.h
+; CHECK-NEXT:    ldp q3, q18, [x0, #224]
+; CHECK-NEXT:    uzp1 z16.h, z2.h, z2.h
+; CHECK-NEXT:    ldp q2, q19, [x0, #128]
 ; CHECK-NEXT:    ldp q0, q1, [x0, #32]
-; CHECK-NEXT:    uzp1 z17.h, z17.h, z17.h
-; CHECK-NEXT:    ldp q4, q5, [x0, #96]
-; CHECK-NEXT:    uzp1 z16.h, z16.h, z16.h
-; CHECK-NEXT:    ldp q18, q19, [x0, #128]
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z3.h
-; CHECK-NEXT:    uzp1 z3.h, z21.h, z21.h
-; CHECK-NEXT:    uzp1 z20.h, z20.h, z20.h
-; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
-; CHECK-NEXT:    ldp q21, q22, [x0]
-; CHECK-NEXT:    splice z16.h, p0, z16.h, z17.h
+; CHECK-NEXT:    uzp1 z21.h, z18.h, z18.h
+; CHECK-NEXT:    ldp q18, q22, [x0, #160]
+; CHECK-NEXT:    uzp1 z20.h, z3.h, z3.h
+; CHECK-NEXT:    uzp1 z24.h, z19.h, z19.h
+; CHECK-NEXT:    ldp q3, q19, [x0, #96]
+; CHECK-NEXT:    uzp1 z23.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z26.h, z22.h, z22.h
+; CHECK-NEXT:    splice z2.h, p0, { z16.h, z17.h }
+; CHECK-NEXT:    uzp1 z17.h, z7.h, z7.h
+; CHECK-NEXT:    uzp1 z25.h, z18.h, z18.h
+; CHECK-NEXT:    splice z7.h, p0, { z20.h, z21.h }
+; CHECK-NEXT:    uzp1 z21.h, z5.h, z5.h
 ; CHECK-NEXT:    uzp1 z19.h, z19.h, z19.h
-; CHECK-NEXT:    uzp1 z18.h, z18.h, z18.h
-; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
-; CHECK-NEXT:    splice z20.h, p0, z20.h, z3.h
-; CHECK-NEXT:    uzp1 z3.h, z5.h, z5.h
-; CHECK-NEXT:    splice z6.h, p0, z6.h, z7.h
-; CHECK-NEXT:    uzp1 z5.h, z22.h, z22.h
-; CHECK-NEXT:    uzp1 z7.h, z21.h, z21.h
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    splice z18.h, p0, z18.h, z19.h
-; CHECK-NEXT:    add z2.h, z2.h, z2.h
-; CHECK-NEXT:    splice z4.h, p0, z4.h, z3.h
-; CHECK-NEXT:    add z3.h, z16.h, z16.h
-; CHECK-NEXT:    splice z7.h, p0, z7.h, z5.h
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
-; CHECK-NEXT:    add z1.h, z20.h, z20.h
-; CHECK-NEXT:    add z5.h, z18.h, z18.h
-; CHECK-NEXT:    stp q2, q3, [x1, #96]
-; CHECK-NEXT:    add z2.h, z6.h, z6.h
+; CHECK-NEXT:    uzp1 z20.h, z4.h, z4.h
+; CHECK-NEXT:    uzp1 z5.h, z1.h, z1.h
+; CHECK-NEXT:    uzp1 z16.h, z6.h, z6.h
+; CHECK-NEXT:    splice z6.h, p0, { z23.h, z24.h }
+; CHECK-NEXT:    uzp1 z18.h, z3.h, z3.h
+; CHECK-NEXT:    splice z3.h, p0, { z25.h, z26.h }
+; CHECK-NEXT:    uzp1 z4.h, z0.h, z0.h
+; CHECK-NEXT:    add z0.h, z2.h, z2.h
+; CHECK-NEXT:    add z7.h, z7.h, z7.h
+; CHECK-NEXT:    splice z1.h, p0, { z16.h, z17.h }
+; CHECK-NEXT:    splice z2.h, p0, { z18.h, z19.h }
+; CHECK-NEXT:    splice z16.h, p0, { z20.h, z21.h }
+; CHECK-NEXT:    splice z4.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    add z6.h, z6.h, z6.h
+; CHECK-NEXT:    add z3.h, z3.h, z3.h
+; CHECK-NEXT:    stp q0, q7, [x1, #96]
+; CHECK-NEXT:    add z0.h, z1.h, z1.h
+; CHECK-NEXT:    add z1.h, z2.h, z2.h
+; CHECK-NEXT:    add z2.h, z16.h, z16.h
+; CHECK-NEXT:    stp q6, q3, [x1, #64]
 ; CHECK-NEXT:    add z3.h, z4.h, z4.h
-; CHECK-NEXT:    add z4.h, z7.h, z7.h
-; CHECK-NEXT:    add z0.h, z0.h, z0.h
-; CHECK-NEXT:    stp q5, q1, [x1, #64]
-; CHECK-NEXT:    stp q2, q3, [x1, #32]
-; CHECK-NEXT:    stp q4, q0, [x1]
+; CHECK-NEXT:    stp q0, q1, [x1, #32]
+; CHECK-NEXT:    stp q2, q3, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: trunc_v64i32_v64i16:
@@ -2360,11 +2360,11 @@ define void @trunc_v64i32_v64i16(ptr %in, ptr %out) nounwind {
 define <4 x i8> @trunc_v4i64_v4i8(ptr %in) nounwind {
 ; CHECK-LABEL: trunc_v4i64_v4i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.s, vl2
-; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
-; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
+; CHECK-NEXT:    uzp1 z3.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z2.s, z1.s, z1.s
+; CHECK-NEXT:    splice z0.s, p0, { z2.s, z3.s }
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
@@ -2392,18 +2392,18 @@ define <8 x i8> @trunc_v8i64_v8i8(ptr %in) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0, #32]
 ; CHECK-NEXT:    ptrue p0.s, vl2
-; CHECK-NEXT:    ldp q2, q3, [x0]
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
-; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
-; CHECK-NEXT:    uzp1 z3.s, z3.s, z3.s
-; CHECK-NEXT:    uzp1 z2.s, z2.s, z2.s
-; CHECK-NEXT:    splice z1.s, p0, z1.s, z0.s
-; CHECK-NEXT:    splice z2.s, p0, z2.s, z3.s
+; CHECK-NEXT:    ldp q3, q2, [x0]
+; CHECK-NEXT:    uzp1 z5.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z4.s, z1.s, z1.s
+; CHECK-NEXT:    uzp1 z1.s, z2.s, z2.s
+; CHECK-NEXT:    uzp1 z0.s, z3.s, z3.s
+; CHECK-NEXT:    splice z2.s, p0, { z4.s, z5.s }
+; CHECK-NEXT:    splice z0.s, p0, { z0.s, z1.s }
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z0.h, z1.h, z1.h
-; CHECK-NEXT:    uzp1 z1.h, z2.h, z2.h
-; CHECK-NEXT:    splice z1.h, p0, z1.h, z0.h
-; CHECK-NEXT:    uzp1 z0.b, z1.b, z1.b
+; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z1.h, z0.h, z0.h
+; CHECK-NEXT:    splice z0.h, p0, { z1.h, z2.h }
+; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
 ;
@@ -2439,34 +2439,34 @@ define <8 x i8> @trunc_v8i64_v8i8(ptr %in) nounwind {
 define <16 x i8> @trunc_v16i64_v16i8(ptr %in) nounwind {
 ; CHECK-LABEL: trunc_v16i64_v16i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0, #32]
+; CHECK-NEXT:    ldp q0, q1, [x0, #96]
 ; CHECK-NEXT:    ptrue p0.s, vl2
-; CHECK-NEXT:    ldp q2, q3, [x0, #96]
+; CHECK-NEXT:    ldp q2, q3, [x0, #32]
 ; CHECK-NEXT:    ldp q4, q5, [x0, #64]
 ; CHECK-NEXT:    ldp q6, q7, [x0]
-; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
-; CHECK-NEXT:    uzp1 z3.s, z3.s, z3.s
-; CHECK-NEXT:    uzp1 z2.s, z2.s, z2.s
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
-; CHECK-NEXT:    uzp1 z5.s, z5.s, z5.s
-; CHECK-NEXT:    uzp1 z4.s, z4.s, z4.s
-; CHECK-NEXT:    uzp1 z7.s, z7.s, z7.s
-; CHECK-NEXT:    uzp1 z6.s, z6.s, z6.s
-; CHECK-NEXT:    splice z2.s, p0, z2.s, z3.s
-; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
-; CHECK-NEXT:    splice z4.s, p0, z4.s, z5.s
-; CHECK-NEXT:    splice z6.s, p0, z6.s, z7.s
+; CHECK-NEXT:    uzp1 z17.s, z1.s, z1.s
+; CHECK-NEXT:    uzp1 z16.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z19.s, z3.s, z3.s
+; CHECK-NEXT:    uzp1 z1.s, z5.s, z5.s
+; CHECK-NEXT:    uzp1 z18.s, z2.s, z2.s
+; CHECK-NEXT:    uzp1 z0.s, z4.s, z4.s
+; CHECK-NEXT:    uzp1 z3.s, z7.s, z7.s
+; CHECK-NEXT:    uzp1 z2.s, z6.s, z6.s
+; CHECK-NEXT:    splice z4.s, p0, { z16.s, z17.s }
+; CHECK-NEXT:    splice z0.s, p0, { z0.s, z1.s }
+; CHECK-NEXT:    splice z1.s, p0, { z18.s, z19.s }
+; CHECK-NEXT:    splice z2.s, p0, { z2.s, z3.s }
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z1.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    uzp1 z2.h, z4.h, z4.h
-; CHECK-NEXT:    uzp1 z3.h, z6.h, z6.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z1.h
-; CHECK-NEXT:    splice z3.h, p0, z3.h, z0.h
+; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
+; CHECK-NEXT:    uzp1 z3.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
+; CHECK-NEXT:    uzp1 z0.h, z2.h, z2.h
+; CHECK-NEXT:    splice z2.h, p0, { z3.h, z4.h }
+; CHECK-NEXT:    splice z0.h, p0, { z0.h, z1.h }
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z1.b, z2.b, z2.b
-; CHECK-NEXT:    uzp1 z0.b, z3.b, z3.b
-; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
+; CHECK-NEXT:    uzp1 z2.b, z2.b, z2.b
+; CHECK-NEXT:    uzp1 z1.b, z0.b, z0.b
+; CHECK-NEXT:    splice z0.b, p0, { z1.b, z2.b }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
@@ -2523,62 +2523,62 @@ define <16 x i8> @trunc_v16i64_v16i8(ptr %in) nounwind {
 define void @trunc_v32i64_v32i8(ptr %in, ptr %out) nounwind {
 ; CHECK-LABEL: trunc_v32i64_v32i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q5, q6, [x0, #224]
 ; CHECK-NEXT:    ptrue p0.s, vl2
-; CHECK-NEXT:    ldp q2, q3, [x0, #224]
-; CHECK-NEXT:    ldp q4, q5, [x0, #32]
-; CHECK-NEXT:    ldp q6, q7, [x0, #64]
-; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
-; CHECK-NEXT:    ldp q16, q17, [x0, #192]
-; CHECK-NEXT:    uzp1 z3.s, z3.s, z3.s
-; CHECK-NEXT:    ldp q18, q19, [x0, #128]
-; CHECK-NEXT:    uzp1 z2.s, z2.s, z2.s
-; CHECK-NEXT:    ldp q20, q21, [x0, #160]
-; CHECK-NEXT:    uzp1 z7.s, z7.s, z7.s
-; CHECK-NEXT:    ldp q22, q23, [x0, #96]
-; CHECK-NEXT:    uzp1 z17.s, z17.s, z17.s
-; CHECK-NEXT:    uzp1 z16.s, z16.s, z16.s
-; CHECK-NEXT:    uzp1 z19.s, z19.s, z19.s
-; CHECK-NEXT:    uzp1 z18.s, z18.s, z18.s
-; CHECK-NEXT:    uzp1 z21.s, z21.s, z21.s
-; CHECK-NEXT:    uzp1 z20.s, z20.s, z20.s
-; CHECK-NEXT:    uzp1 z6.s, z6.s, z6.s
-; CHECK-NEXT:    uzp1 z23.s, z23.s, z23.s
-; CHECK-NEXT:    uzp1 z22.s, z22.s, z22.s
-; CHECK-NEXT:    uzp1 z5.s, z5.s, z5.s
-; CHECK-NEXT:    uzp1 z4.s, z4.s, z4.s
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
-; CHECK-NEXT:    splice z2.s, p0, z2.s, z3.s
-; CHECK-NEXT:    splice z16.s, p0, z16.s, z17.s
-; CHECK-NEXT:    splice z20.s, p0, z20.s, z21.s
-; CHECK-NEXT:    splice z18.s, p0, z18.s, z19.s
-; CHECK-NEXT:    splice z22.s, p0, z22.s, z23.s
-; CHECK-NEXT:    splice z6.s, p0, z6.s, z7.s
-; CHECK-NEXT:    splice z4.s, p0, z4.s, z5.s
-; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
+; CHECK-NEXT:    ldp q2, q3, [x0, #32]
+; CHECK-NEXT:    ldp q4, q7, [x0, #64]
+; CHECK-NEXT:    uzp1 z17.s, z6.s, z6.s
+; CHECK-NEXT:    ldp q6, q18, [x0, #192]
+; CHECK-NEXT:    uzp1 z16.s, z5.s, z5.s
+; CHECK-NEXT:    ldp q5, q19, [x0, #128]
+; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    uzp1 z21.s, z18.s, z18.s
+; CHECK-NEXT:    ldp q18, q22, [x0, #160]
+; CHECK-NEXT:    uzp1 z20.s, z6.s, z6.s
+; CHECK-NEXT:    ldp q6, q23, [x0, #96]
+; CHECK-NEXT:    splice z16.s, p0, { z16.s, z17.s }
+; CHECK-NEXT:    uzp1 z27.s, z19.s, z19.s
+; CHECK-NEXT:    uzp1 z25.s, z22.s, z22.s
+; CHECK-NEXT:    uzp1 z26.s, z5.s, z5.s
+; CHECK-NEXT:    uzp1 z24.s, z18.s, z18.s
+; CHECK-NEXT:    uzp1 z18.s, z23.s, z23.s
+; CHECK-NEXT:    uzp1 z23.s, z3.s, z3.s
+; CHECK-NEXT:    uzp1 z17.s, z6.s, z6.s
+; CHECK-NEXT:    uzp1 z6.s, z7.s, z7.s
+; CHECK-NEXT:    uzp1 z22.s, z2.s, z2.s
+; CHECK-NEXT:    uzp1 z5.s, z4.s, z4.s
+; CHECK-NEXT:    uzp1 z2.s, z1.s, z1.s
+; CHECK-NEXT:    splice z3.s, p0, { z20.s, z21.s }
+; CHECK-NEXT:    uzp1 z1.s, z0.s, z0.s
+; CHECK-NEXT:    splice z0.s, p0, { z24.s, z25.s }
+; CHECK-NEXT:    splice z7.s, p0, { z26.s, z27.s }
+; CHECK-NEXT:    splice z4.s, p0, { z17.s, z18.s }
+; CHECK-NEXT:    uzp1 z17.h, z16.h, z16.h
+; CHECK-NEXT:    splice z5.s, p0, { z5.s, z6.s }
+; CHECK-NEXT:    splice z6.s, p0, { z22.s, z23.s }
+; CHECK-NEXT:    splice z1.s, p0, { z1.s, z2.s }
+; CHECK-NEXT:    uzp1 z16.h, z3.h, z3.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z1.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z2.h, z16.h, z16.h
-; CHECK-NEXT:    uzp1 z3.h, z20.h, z20.h
-; CHECK-NEXT:    uzp1 z5.h, z18.h, z18.h
-; CHECK-NEXT:    uzp1 z7.h, z22.h, z22.h
-; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
-; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z1.h
-; CHECK-NEXT:    splice z5.h, p0, z5.h, z3.h
-; CHECK-NEXT:    splice z6.h, p0, z6.h, z7.h
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z4.h
+; CHECK-NEXT:    uzp1 z3.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z19.h, z4.h, z4.h
+; CHECK-NEXT:    uzp1 z2.h, z7.h, z7.h
+; CHECK-NEXT:    uzp1 z18.h, z5.h, z5.h
+; CHECK-NEXT:    uzp1 z5.h, z6.h, z6.h
+; CHECK-NEXT:    splice z0.h, p0, { z16.h, z17.h }
+; CHECK-NEXT:    uzp1 z4.h, z1.h, z1.h
+; CHECK-NEXT:    splice z1.h, p0, { z2.h, z3.h }
+; CHECK-NEXT:    splice z2.h, p0, { z18.h, z19.h }
+; CHECK-NEXT:    splice z3.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    uzp1 z5.b, z0.b, z0.b
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z1.b, z2.b, z2.b
-; CHECK-NEXT:    uzp1 z2.b, z5.b, z5.b
-; CHECK-NEXT:    uzp1 z3.b, z6.b, z6.b
-; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
-; CHECK-NEXT:    splice z2.b, p0, z2.b, z1.b
-; CHECK-NEXT:    splice z0.b, p0, z0.b, z3.b
-; CHECK-NEXT:    add z1.b, z2.b, z2.b
+; CHECK-NEXT:    uzp1 z4.b, z1.b, z1.b
+; CHECK-NEXT:    uzp1 z7.b, z2.b, z2.b
+; CHECK-NEXT:    uzp1 z6.b, z3.b, z3.b
+; CHECK-NEXT:    splice z0.b, p0, { z4.b, z5.b }
+; CHECK-NEXT:    splice z1.b, p0, { z6.b, z7.b }
 ; CHECK-NEXT:    add z0.b, z0.b, z0.b
-; CHECK-NEXT:    stp q0, q1, [x1]
+; CHECK-NEXT:    add z1.b, z1.b, z1.b
+; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: trunc_v32i64_v32i8:
@@ -2731,11 +2731,11 @@ define void @trunc_v32i64_v32i8(ptr %in, ptr %out) nounwind {
 define <4 x i16> @trunc_v4i64_v4i16(ptr %in) nounwind {
 ; CHECK-LABEL: trunc_v4i64_v4i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.s, vl2
-; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
-; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
+; CHECK-NEXT:    uzp1 z3.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z2.s, z1.s, z1.s
+; CHECK-NEXT:    splice z0.s, p0, { z2.s, z3.s }
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
@@ -2763,17 +2763,17 @@ define <8 x i16> @trunc_v8i64_v8i16(ptr %in) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0, #32]
 ; CHECK-NEXT:    ptrue p0.s, vl2
-; CHECK-NEXT:    ldp q2, q3, [x0]
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
-; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
-; CHECK-NEXT:    uzp1 z3.s, z3.s, z3.s
-; CHECK-NEXT:    uzp1 z2.s, z2.s, z2.s
-; CHECK-NEXT:    splice z1.s, p0, z1.s, z0.s
-; CHECK-NEXT:    splice z2.s, p0, z2.s, z3.s
+; CHECK-NEXT:    ldp q3, q2, [x0]
+; CHECK-NEXT:    uzp1 z5.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z4.s, z1.s, z1.s
+; CHECK-NEXT:    uzp1 z1.s, z2.s, z2.s
+; CHECK-NEXT:    uzp1 z0.s, z3.s, z3.s
+; CHECK-NEXT:    splice z2.s, p0, { z4.s, z5.s }
+; CHECK-NEXT:    splice z0.s, p0, { z0.s, z1.s }
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
-; CHECK-NEXT:    uzp1 z0.h, z2.h, z2.h
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
+; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z1.h, z0.h, z0.h
+; CHECK-NEXT:    splice z0.h, p0, { z1.h, z2.h }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
@@ -2810,32 +2810,32 @@ define <8 x i16> @trunc_v8i64_v8i16(ptr %in) nounwind {
 define void @trunc_v16i64_v16i16(ptr %in, ptr %out) nounwind {
 ; CHECK-LABEL: trunc_v16i64_v16i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0, #32]
+; CHECK-NEXT:    ldp q0, q1, [x0, #96]
 ; CHECK-NEXT:    ptrue p0.s, vl2
-; CHECK-NEXT:    ldp q2, q3, [x0, #96]
+; CHECK-NEXT:    ldp q2, q3, [x0, #32]
 ; CHECK-NEXT:    ldp q4, q5, [x0, #64]
 ; CHECK-NEXT:    ldp q6, q7, [x0]
-; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
-; CHECK-NEXT:    uzp1 z3.s, z3.s, z3.s
-; CHECK-NEXT:    uzp1 z2.s, z2.s, z2.s
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
-; CHECK-NEXT:    uzp1 z5.s, z5.s, z5.s
-; CHECK-NEXT:    uzp1 z4.s, z4.s, z4.s
-; CHECK-NEXT:    uzp1 z7.s, z7.s, z7.s
-; CHECK-NEXT:    uzp1 z6.s, z6.s, z6.s
-; CHECK-NEXT:    splice z2.s, p0, z2.s, z3.s
-; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
-; CHECK-NEXT:    splice z4.s, p0, z4.s, z5.s
-; CHECK-NEXT:    splice z6.s, p0, z6.s, z7.s
+; CHECK-NEXT:    uzp1 z17.s, z1.s, z1.s
+; CHECK-NEXT:    uzp1 z16.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z1.s, z3.s, z3.s
+; CHECK-NEXT:    uzp1 z19.s, z5.s, z5.s
+; CHECK-NEXT:    uzp1 z0.s, z2.s, z2.s
+; CHECK-NEXT:    uzp1 z3.s, z7.s, z7.s
+; CHECK-NEXT:    uzp1 z18.s, z4.s, z4.s
+; CHECK-NEXT:    uzp1 z2.s, z6.s, z6.s
+; CHECK-NEXT:    splice z4.s, p0, { z16.s, z17.s }
+; CHECK-NEXT:    splice z0.s, p0, { z0.s, z1.s }
+; CHECK-NEXT:    splice z5.s, p0, { z18.s, z19.s }
+; CHECK-NEXT:    splice z1.s, p0, { z2.s, z3.s }
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z1.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    uzp1 z2.h, z4.h, z4.h
-; CHECK-NEXT:    uzp1 z3.h, z6.h, z6.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z1.h
-; CHECK-NEXT:    splice z3.h, p0, z3.h, z0.h
-; CHECK-NEXT:    add z0.h, z2.h, z2.h
-; CHECK-NEXT:    add z1.h, z3.h, z3.h
+; CHECK-NEXT:    uzp1 z3.h, z4.h, z4.h
+; CHECK-NEXT:    uzp1 z7.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z2.h, z5.h, z5.h
+; CHECK-NEXT:    uzp1 z6.h, z1.h, z1.h
+; CHECK-NEXT:    splice z0.h, p0, { z2.h, z3.h }
+; CHECK-NEXT:    splice z1.h, p0, { z6.h, z7.h }
+; CHECK-NEXT:    add z0.h, z0.h, z0.h
+; CHECK-NEXT:    add z1.h, z1.h, z1.h
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
 ;
@@ -2915,56 +2915,56 @@ define void @trunc_v32i64_v32i16(ptr %in, ptr %out) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q2, q3, [x0, #160]
 ; CHECK-NEXT:    ptrue p0.s, vl2
-; CHECK-NEXT:    ldp q4, q5, [x0, #128]
+; CHECK-NEXT:    ldp q4, q5, [x0, #96]
+; CHECK-NEXT:    ldp q6, q7, [x0]
+; CHECK-NEXT:    uzp1 z17.s, z3.s, z3.s
+; CHECK-NEXT:    ldp q3, q18, [x0, #128]
+; CHECK-NEXT:    uzp1 z16.s, z2.s, z2.s
+; CHECK-NEXT:    ldp q2, q19, [x0, #192]
 ; CHECK-NEXT:    ldp q0, q1, [x0, #64]
-; CHECK-NEXT:    ldp q6, q7, [x0, #96]
-; CHECK-NEXT:    uzp1 z3.s, z3.s, z3.s
-; CHECK-NEXT:    uzp1 z2.s, z2.s, z2.s
-; CHECK-NEXT:    uzp1 z5.s, z5.s, z5.s
-; CHECK-NEXT:    uzp1 z4.s, z4.s, z4.s
-; CHECK-NEXT:    ldp q16, q17, [x0]
-; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
-; CHECK-NEXT:    ldp q18, q19, [x0, #192]
-; CHECK-NEXT:    uzp1 z7.s, z7.s, z7.s
-; CHECK-NEXT:    ldp q20, q21, [x0, #224]
-; CHECK-NEXT:    splice z2.s, p0, z2.s, z3.s
-; CHECK-NEXT:    ldp q22, q23, [x0, #32]
-; CHECK-NEXT:    splice z4.s, p0, z4.s, z5.s
-; CHECK-NEXT:    uzp1 z19.s, z19.s, z19.s
-; CHECK-NEXT:    uzp1 z18.s, z18.s, z18.s
-; CHECK-NEXT:    uzp1 z17.s, z17.s, z17.s
-; CHECK-NEXT:    uzp1 z3.s, z21.s, z21.s
-; CHECK-NEXT:    uzp1 z5.s, z20.s, z20.s
-; CHECK-NEXT:    uzp1 z16.s, z16.s, z16.s
-; CHECK-NEXT:    uzp1 z20.s, z23.s, z23.s
-; CHECK-NEXT:    uzp1 z21.s, z22.s, z22.s
-; CHECK-NEXT:    uzp1 z6.s, z6.s, z6.s
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
-; CHECK-NEXT:    splice z18.s, p0, z18.s, z19.s
-; CHECK-NEXT:    splice z5.s, p0, z5.s, z3.s
-; CHECK-NEXT:    splice z16.s, p0, z16.s, z17.s
-; CHECK-NEXT:    splice z21.s, p0, z21.s, z20.s
-; CHECK-NEXT:    splice z6.s, p0, z6.s, z7.s
-; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
-; CHECK-NEXT:    uzp1 z1.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z2.h, z4.h, z4.h
+; CHECK-NEXT:    uzp1 z21.s, z18.s, z18.s
+; CHECK-NEXT:    ldp q18, q22, [x0, #224]
+; CHECK-NEXT:    uzp1 z20.s, z3.s, z3.s
+; CHECK-NEXT:    ldp q3, q23, [x0, #32]
+; CHECK-NEXT:    splice z16.s, p0, { z16.s, z17.s }
+; CHECK-NEXT:    uzp1 z27.s, z19.s, z19.s
+; CHECK-NEXT:    uzp1 z25.s, z22.s, z22.s
+; CHECK-NEXT:    uzp1 z26.s, z2.s, z2.s
+; CHECK-NEXT:    uzp1 z24.s, z18.s, z18.s
+; CHECK-NEXT:    uzp1 z18.s, z23.s, z23.s
+; CHECK-NEXT:    uzp1 z23.s, z5.s, z5.s
+; CHECK-NEXT:    uzp1 z17.s, z3.s, z3.s
+; CHECK-NEXT:    uzp1 z3.s, z7.s, z7.s
+; CHECK-NEXT:    uzp1 z22.s, z4.s, z4.s
+; CHECK-NEXT:    uzp1 z2.s, z6.s, z6.s
+; CHECK-NEXT:    uzp1 z5.s, z1.s, z1.s
+; CHECK-NEXT:    splice z1.s, p0, { z20.s, z21.s }
+; CHECK-NEXT:    splice z6.s, p0, { z24.s, z25.s }
+; CHECK-NEXT:    uzp1 z4.s, z0.s, z0.s
+; CHECK-NEXT:    splice z0.s, p0, { z26.s, z27.s }
+; CHECK-NEXT:    splice z7.s, p0, { z17.s, z18.s }
+; CHECK-NEXT:    uzp1 z17.h, z16.h, z16.h
+; CHECK-NEXT:    splice z2.s, p0, { z2.s, z3.s }
+; CHECK-NEXT:    splice z3.s, p0, { z22.s, z23.s }
+; CHECK-NEXT:    splice z4.s, p0, { z4.s, z5.s }
+; CHECK-NEXT:    uzp1 z16.h, z1.h, z1.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z4.h, z18.h, z18.h
-; CHECK-NEXT:    uzp1 z3.h, z5.h, z5.h
-; CHECK-NEXT:    uzp1 z7.h, z16.h, z16.h
-; CHECK-NEXT:    uzp1 z5.h, z21.h, z21.h
-; CHECK-NEXT:    splice z2.h, p0, z2.h, z1.h
-; CHECK-NEXT:    uzp1 z1.h, z6.h, z6.h
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT:    splice z4.h, p0, z4.h, z3.h
-; CHECK-NEXT:    splice z7.h, p0, z7.h, z5.h
-; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
-; CHECK-NEXT:    add z1.h, z2.h, z2.h
-; CHECK-NEXT:    add z2.h, z4.h, z4.h
-; CHECK-NEXT:    add z3.h, z7.h, z7.h
+; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
+; CHECK-NEXT:    uzp1 z5.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z1.h, z7.h, z7.h
+; CHECK-NEXT:    uzp1 z0.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z3.h, z3.h, z3.h
+; CHECK-NEXT:    splice z7.h, p0, { z16.h, z17.h }
+; CHECK-NEXT:    uzp1 z2.h, z4.h, z4.h
+; CHECK-NEXT:    splice z4.h, p0, { z5.h, z6.h }
+; CHECK-NEXT:    splice z0.h, p0, { z0.h, z1.h }
+; CHECK-NEXT:    splice z1.h, p0, { z2.h, z3.h }
+; CHECK-NEXT:    add z2.h, z7.h, z7.h
+; CHECK-NEXT:    add z3.h, z4.h, z4.h
 ; CHECK-NEXT:    add z0.h, z0.h, z0.h
-; CHECK-NEXT:    stp q1, q2, [x1, #32]
-; CHECK-NEXT:    stp q3, q0, [x1]
+; CHECK-NEXT:    add z1.h, z1.h, z1.h
+; CHECK-NEXT:    stp q2, q3, [x1, #32]
+; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: trunc_v32i64_v32i16:
@@ -3118,11 +3118,11 @@ define void @trunc_v32i64_v32i16(ptr %in, ptr %out) nounwind {
 define <4 x i32> @trunc_v4i64_v4i32(ptr %in) nounwind {
 ; CHECK-LABEL: trunc_v4i64_v4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.s, vl2
-; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
-; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
+; CHECK-NEXT:    uzp1 z3.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z2.s, z1.s, z1.s
+; CHECK-NEXT:    splice z0.s, p0, { z2.s, z3.s }
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
 ;
@@ -3146,18 +3146,18 @@ define <4 x i32> @trunc_v4i64_v4i32(ptr %in) nounwind {
 define void @trunc_v8i64_v8i32(ptr %in, ptr %out) nounwind {
 ; CHECK-LABEL: trunc_v8i64_v8i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0, #32]
+; CHECK-NEXT:    ldp q1, q0, [x0, #32]
 ; CHECK-NEXT:    ptrue p0.s, vl2
-; CHECK-NEXT:    ldp q2, q3, [x0]
-; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
-; CHECK-NEXT:    uzp1 z3.s, z3.s, z3.s
-; CHECK-NEXT:    uzp1 z2.s, z2.s, z2.s
-; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
-; CHECK-NEXT:    splice z2.s, p0, z2.s, z3.s
-; CHECK-NEXT:    add z0.s, z0.s, z0.s
+; CHECK-NEXT:    ldp q3, q2, [x0]
+; CHECK-NEXT:    uzp1 z5.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z4.s, z1.s, z1.s
+; CHECK-NEXT:    uzp1 z1.s, z2.s, z2.s
+; CHECK-NEXT:    uzp1 z0.s, z3.s, z3.s
+; CHECK-NEXT:    splice z2.s, p0, { z4.s, z5.s }
+; CHECK-NEXT:    splice z0.s, p0, { z0.s, z1.s }
 ; CHECK-NEXT:    add z1.s, z2.s, z2.s
-; CHECK-NEXT:    stp q1, q0, [x1]
+; CHECK-NEXT:    add z0.s, z0.s, z0.s
+; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: trunc_v8i64_v8i32:
@@ -3202,27 +3202,27 @@ define void @trunc_v8i64_v8i32(ptr %in, ptr %out) nounwind {
 define void @trunc_v16i64_v16i32(ptr %in, ptr %out) nounwind {
 ; CHECK-LABEL: trunc_v16i64_v16i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0, #64]
+; CHECK-NEXT:    ldp q1, q0, [x0, #64]
 ; CHECK-NEXT:    ptrue p0.s, vl2
-; CHECK-NEXT:    ldp q2, q3, [x0]
-; CHECK-NEXT:    ldp q4, q5, [x0, #96]
-; CHECK-NEXT:    ldp q6, q7, [x0, #32]
-; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
-; CHECK-NEXT:    uzp1 z3.s, z3.s, z3.s
-; CHECK-NEXT:    uzp1 z2.s, z2.s, z2.s
-; CHECK-NEXT:    uzp1 z5.s, z5.s, z5.s
-; CHECK-NEXT:    uzp1 z4.s, z4.s, z4.s
-; CHECK-NEXT:    uzp1 z7.s, z7.s, z7.s
-; CHECK-NEXT:    uzp1 z6.s, z6.s, z6.s
-; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
-; CHECK-NEXT:    splice z2.s, p0, z2.s, z3.s
-; CHECK-NEXT:    splice z4.s, p0, z4.s, z5.s
-; CHECK-NEXT:    splice z6.s, p0, z6.s, z7.s
+; CHECK-NEXT:    ldp q2, q3, [x0, #96]
+; CHECK-NEXT:    ldp q4, q5, [x0]
+; CHECK-NEXT:    uzp1 z7.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z6.s, z1.s, z1.s
+; CHECK-NEXT:    ldp q1, q0, [x0, #32]
+; CHECK-NEXT:    uzp1 z17.s, z3.s, z3.s
+; CHECK-NEXT:    uzp1 z16.s, z2.s, z2.s
+; CHECK-NEXT:    uzp1 z3.s, z5.s, z5.s
+; CHECK-NEXT:    uzp1 z2.s, z4.s, z4.s
+; CHECK-NEXT:    uzp1 z5.s, z0.s, z0.s
+; CHECK-NEXT:    splice z0.s, p0, { z6.s, z7.s }
+; CHECK-NEXT:    uzp1 z4.s, z1.s, z1.s
+; CHECK-NEXT:    splice z1.s, p0, { z16.s, z17.s }
+; CHECK-NEXT:    splice z2.s, p0, { z2.s, z3.s }
+; CHECK-NEXT:    splice z3.s, p0, { z4.s, z5.s }
 ; CHECK-NEXT:    add z0.s, z0.s, z0.s
+; CHECK-NEXT:    add z1.s, z1.s, z1.s
 ; CHECK-NEXT:    add z2.s, z2.s, z2.s
-; CHECK-NEXT:    add z1.s, z4.s, z4.s
-; CHECK-NEXT:    add z3.s, z6.s, z6.s
+; CHECK-NEXT:    add z3.s, z3.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x1, #32]
 ; CHECK-NEXT:    stp q2, q3, [x1]
 ; CHECK-NEXT:    ret
@@ -3297,49 +3297,49 @@ define void @trunc_v32i64_v32i32(ptr %in, ptr %out) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q2, q3, [x0, #192]
 ; CHECK-NEXT:    ptrue p0.s, vl2
+; CHECK-NEXT:    ldp q4, q5, [x0]
 ; CHECK-NEXT:    ldp q6, q7, [x0, #64]
-; CHECK-NEXT:    ldp q16, q17, [x0, #224]
-; CHECK-NEXT:    uzp1 z3.s, z3.s, z3.s
-; CHECK-NEXT:    uzp1 z2.s, z2.s, z2.s
-; CHECK-NEXT:    ldp q20, q21, [x0, #160]
-; CHECK-NEXT:    uzp1 z7.s, z7.s, z7.s
+; CHECK-NEXT:    uzp1 z17.s, z3.s, z3.s
+; CHECK-NEXT:    ldp q3, q18, [x0, #224]
+; CHECK-NEXT:    uzp1 z16.s, z2.s, z2.s
+; CHECK-NEXT:    ldp q2, q19, [x0, #128]
 ; CHECK-NEXT:    ldp q0, q1, [x0, #32]
-; CHECK-NEXT:    uzp1 z17.s, z17.s, z17.s
-; CHECK-NEXT:    ldp q4, q5, [x0, #96]
-; CHECK-NEXT:    uzp1 z16.s, z16.s, z16.s
-; CHECK-NEXT:    ldp q18, q19, [x0, #128]
-; CHECK-NEXT:    splice z2.s, p0, z2.s, z3.s
-; CHECK-NEXT:    uzp1 z3.s, z21.s, z21.s
-; CHECK-NEXT:    uzp1 z20.s, z20.s, z20.s
-; CHECK-NEXT:    uzp1 z6.s, z6.s, z6.s
-; CHECK-NEXT:    ldp q21, q22, [x0]
-; CHECK-NEXT:    splice z16.s, p0, z16.s, z17.s
+; CHECK-NEXT:    uzp1 z21.s, z18.s, z18.s
+; CHECK-NEXT:    ldp q18, q22, [x0, #160]
+; CHECK-NEXT:    uzp1 z20.s, z3.s, z3.s
+; CHECK-NEXT:    uzp1 z24.s, z19.s, z19.s
+; CHECK-NEXT:    ldp q3, q19, [x0, #96]
+; CHECK-NEXT:    uzp1 z23.s, z2.s, z2.s
+; CHECK-NEXT:    uzp1 z26.s, z22.s, z22.s
+; CHECK-NEXT:    splice z2.s, p0, { z16.s, z17.s }
+; CHECK-NEXT:    uzp1 z17.s, z7.s, z7.s
+; CHECK-NEXT:    uzp1 z25.s, z18.s, z18.s
+; CHECK-NEXT:    splice z7.s, p0, { z20.s, z21.s }
+; CHECK-NEXT:    uzp1 z21.s, z5.s, z5.s
 ; CHECK-NEXT:    uzp1 z19.s, z19.s, z19.s
-; CHECK-NEXT:    uzp1 z18.s, z18.s, z18.s
-; CHECK-NEXT:    uzp1 z4.s, z4.s, z4.s
-; CHECK-NEXT:    splice z20.s, p0, z20.s, z3.s
-; CHECK-NEXT:    uzp1 z3.s, z5.s, z5.s
-; CHECK-NEXT:    splice z6.s, p0, z6.s, z7.s
-; CHECK-NEXT:    uzp1 z5.s, z22.s, z22.s
-; CHECK-NEXT:    uzp1 z7.s, z21.s, z21.s
-; CHECK-NEXT:    uzp1 z1.s, z1.s, z1.s
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
-; CHECK-NEXT:    splice z18.s, p0, z18.s, z19.s
-; CHECK-NEXT:    add z2.s, z2.s, z2.s
-; CHECK-NEXT:    splice z4.s, p0, z4.s, z3.s
-; CHECK-NEXT:    add z3.s, z16.s, z16.s
-; CHECK-NEXT:    splice z7.s, p0, z7.s, z5.s
-; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
-; CHECK-NEXT:    add z1.s, z20.s, z20.s
-; CHECK-NEXT:    add z5.s, z18.s, z18.s
-; CHECK-NEXT:    stp q2, q3, [x1, #96]
-; CHECK-NEXT:    add z2.s, z6.s, z6.s
+; CHECK-NEXT:    uzp1 z20.s, z4.s, z4.s
+; CHECK-NEXT:    uzp1 z5.s, z1.s, z1.s
+; CHECK-NEXT:    uzp1 z16.s, z6.s, z6.s
+; CHECK-NEXT:    splice z6.s, p0, { z23.s, z24.s }
+; CHECK-NEXT:    uzp1 z18.s, z3.s, z3.s
+; CHECK-NEXT:    splice z3.s, p0, { z25.s, z26.s }
+; CHECK-NEXT:    uzp1 z4.s, z0.s, z0.s
+; CHECK-NEXT:    add z0.s, z2.s, z2.s
+; CHECK-NEXT:    add z7.s, z7.s, z7.s
+; CHECK-NEXT:    splice z1.s, p0, { z16.s, z17.s }
+; CHECK-NEXT:    splice z2.s, p0, { z18.s, z19.s }
+; CHECK-NEXT:    splice z16.s, p0, { z20.s, z21.s }
+; CHECK-NEXT:    splice z4.s, p0, { z4.s, z5.s }
+; CHECK-NEXT:    add z6.s, z6.s, z6.s
+; CHECK-NEXT:    add z3.s, z3.s, z3.s
+; CHECK-NEXT:    stp q0, q7, [x1, #96]
+; CHECK-NEXT:    add z0.s, z1.s, z1.s
+; CHECK-NEXT:    add z1.s, z2.s, z2.s
+; CHECK-NEXT:    add z2.s, z16.s, z16.s
+; CHECK-NEXT:    stp q6, q3, [x1, #64]
 ; CHECK-NEXT:    add z3.s, z4.s, z4.s
-; CHECK-NEXT:    add z4.s, z7.s, z7.s
-; CHECK-NEXT:    add z0.s, z0.s, z0.s
-; CHECK-NEXT:    stp q5, q1, [x1, #64]
-; CHECK-NEXT:    stp q2, q3, [x1, #32]
-; CHECK-NEXT:    stp q4, q0, [x1]
+; CHECK-NEXT:    stp q0, q1, [x1, #32]
+; CHECK-NEXT:    stp q2, q3, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: trunc_v32i64_v32i32:
-- 
GitLab


From 508263824f4ef0c70f37523810e5f7d56bcfa653 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Wed, 30 Oct 2024 14:23:35 +0100
Subject: [PATCH 577/693] [Clang] Start moving X86Builtins.def to
 X86Builtins.td (#106005)

This starts moving `X86Builtins.def` to be a tablegen file. It's quite
large, so I think it'd be good to move things in multiple steps to avoid
a bunch of merge conflicts due to the amount of time this takes to
complete.
---
 clang/include/clang/Basic/BuiltinsBase.td     |   4 +
 clang/include/clang/Basic/BuiltinsX86.def     | 126 ----------------
 clang/include/clang/Basic/BuiltinsX86.td      | 137 ++++++++++++++++++
 clang/include/clang/Basic/CMakeLists.txt      |   4 +
 clang/include/clang/Basic/TargetBuiltins.h    |   2 +
 clang/lib/Basic/Targets/X86.cpp               |   8 +
 clang/utils/TableGen/ClangBuiltinsEmitter.cpp |   8 +-
 7 files changed, 160 insertions(+), 129 deletions(-)
 create mode 100644 clang/include/clang/Basic/BuiltinsX86.td

diff --git a/clang/include/clang/Basic/BuiltinsBase.td b/clang/include/clang/Basic/BuiltinsBase.td
index 58dee22fc0a4..cff182f3f282 100644
--- a/clang/include/clang/Basic/BuiltinsBase.td
+++ b/clang/include/clang/Basic/BuiltinsBase.td
@@ -60,6 +60,10 @@ def ConstIgnoringExceptions : Attribute<"g">;
 // This function requires a specific header or an explicit declaration.
 def RequireDeclaration : Attribute<"h">;
 
+// FIXME: Why is this not simply the min_vector_width attribute?
+// Vector has to be at least N bits wide.
+class RequiredVectorWidth<int N> : IndexedAttribute<"V", N>;
+
 class PrintfFormat<int I> : IndexedAttribute<"p", I>;
 class VPrintfFormat<int I> : IndexedAttribute<"P", I>;
 class ScanfFormat<int I> : IndexedAttribute<"s", I>;
diff --git a/clang/include/clang/Basic/BuiltinsX86.def b/clang/include/clang/Basic/BuiltinsX86.def
index 4486eb73a11f..c93ea27f164e 100644
--- a/clang/include/clang/Basic/BuiltinsX86.def
+++ b/clang/include/clang/Basic/BuiltinsX86.def
@@ -26,17 +26,6 @@
 #  define TARGET_HEADER_BUILTIN(ID, TYPE, ATTRS, HEADER, LANG, FEATURE) BUILTIN(ID, TYPE, ATTRS)
 #endif
 
-// Undefined Values
-//
-TARGET_BUILTIN(__builtin_ia32_undef128, "V2d", "ncV:128:", "")
-TARGET_BUILTIN(__builtin_ia32_undef256, "V4d", "ncV:256:", "")
-TARGET_BUILTIN(__builtin_ia32_undef512, "V8d", "ncV:512:", "")
-
-// FLAGS
-//
-TARGET_BUILTIN(__builtin_ia32_readeflags_u32, "Ui", "n", "")
-TARGET_BUILTIN(__builtin_ia32_writeeflags_u32, "vUi", "n", "")
-
 // MMX
 //
 // All MMX instructions will be generated via builtins. Any MMX vector
@@ -46,113 +35,8 @@ TARGET_BUILTIN(__builtin_ia32_writeeflags_u32, "vUi", "n", "")
 // argument and our prior approach of using a #define to the current built-in
 // doesn't work in the presence of re-declaration of _mm_prefetch for windows.
 TARGET_BUILTIN(_mm_prefetch, "vcC*i", "nc", "mmx")
-TARGET_BUILTIN(__builtin_ia32_emms, "v", "n", "mmx")
-TARGET_BUILTIN(__builtin_ia32_vec_ext_v4hi, "sV4sIi", "ncV:64:", "sse")
-TARGET_BUILTIN(__builtin_ia32_vec_set_v4hi, "V4sV4ssIi", "ncV:64:", "sse")
 
 // SSE intrinsics.
-TARGET_BUILTIN(__builtin_ia32_comieq, "iV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_comilt, "iV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_comile, "iV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_comigt, "iV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_comige, "iV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_comineq, "iV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_ucomieq, "iV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_ucomilt, "iV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_ucomile, "iV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_ucomigt, "iV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_ucomige, "iV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_ucomineq, "iV4fV4f", "ncV:128:", "sse")
-
-TARGET_BUILTIN(__builtin_ia32_comisdeq, "iV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_comisdlt, "iV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_comisdle, "iV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_comisdgt, "iV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_comisdge, "iV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_comisdneq, "iV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_ucomisdeq, "iV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_ucomisdlt, "iV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_ucomisdle, "iV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_ucomisdgt, "iV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_ucomisdge, "iV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_ucomisdneq, "iV2dV2d", "ncV:128:", "sse2")
-
-TARGET_BUILTIN(__builtin_ia32_cmpeqps, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpltps, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpleps, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpunordps, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpneqps, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpnltps, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpnleps, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpordps, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpeqss, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpltss, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpless, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpunordss, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpneqss, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpnltss, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpnless, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpordss, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_minps, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_maxps, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_minss, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_maxss, "V4fV4fV4f", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpps, "V4fV4fV4fIc", "ncV:128:", "sse")
-TARGET_BUILTIN(__builtin_ia32_cmpss, "V4fV4fV4fIc", "ncV:128:", "sse")
-
-TARGET_BUILTIN(__builtin_ia32_cmpeqpd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmpltpd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmplepd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmpunordpd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmpneqpd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmpnltpd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmpnlepd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmpordpd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmpeqsd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmpltsd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmplesd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmpunordsd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmpneqsd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmpnltsd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmpnlesd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmpordsd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmpsd, "V2dV2dV2dIc", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_cmppd, "V2dV2dV2dIc", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_minpd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_maxpd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_minsd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_maxsd, "V2dV2dV2d", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_pmulhw128, "V8sV8sV8s", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_pavgb128, "V16cV16cV16c", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_pavgw128, "V8sV8sV8s", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_packsswb128, "V16cV8sV8s", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_packssdw128, "V8sV4iV4i", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_packuswb128, "V16cV8sV8s", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_pmulhuw128, "V8sV8sV8s", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_vec_ext_v2di, "OiV2OiIi", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_vec_ext_v4si, "iV4iIi", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_vec_ext_v4sf, "fV4fIi", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_vec_ext_v8hi, "sV8sIi", "ncV:128:", "sse2")
-TARGET_BUILTIN(__builtin_ia32_vec_set_v8hi, "V8sV8ssIi", "ncV:128:", "sse2")
-
-TARGET_BUILTIN(__builtin_ia32_addsubps, "V4fV4fV4f", "ncV:128:", "sse3")
-TARGET_BUILTIN(__builtin_ia32_addsubpd, "V2dV2dV2d", "ncV:128:", "sse3")
-TARGET_BUILTIN(__builtin_ia32_haddps, "V4fV4fV4f", "ncV:128:", "sse3")
-TARGET_BUILTIN(__builtin_ia32_haddpd, "V2dV2dV2d", "ncV:128:", "sse3")
-TARGET_BUILTIN(__builtin_ia32_hsubps, "V4fV4fV4f", "ncV:128:", "sse3")
-TARGET_BUILTIN(__builtin_ia32_hsubpd, "V2dV2dV2d", "ncV:128:", "sse3")
-TARGET_BUILTIN(__builtin_ia32_phaddw128, "V8sV8sV8s", "ncV:128:", "ssse3")
-TARGET_BUILTIN(__builtin_ia32_phaddd128, "V4iV4iV4i", "ncV:128:", "ssse3")
-TARGET_BUILTIN(__builtin_ia32_phaddsw128, "V8sV8sV8s", "ncV:128:", "ssse3")
-TARGET_BUILTIN(__builtin_ia32_phsubw128, "V8sV8sV8s", "ncV:128:", "ssse3")
-TARGET_BUILTIN(__builtin_ia32_phsubd128, "V4iV4iV4i", "ncV:128:", "ssse3")
-TARGET_BUILTIN(__builtin_ia32_phsubsw128, "V8sV8sV8s", "ncV:128:", "ssse3")
-TARGET_BUILTIN(__builtin_ia32_pmaddubsw128, "V8sV16cV16c", "ncV:128:", "ssse3")
-TARGET_BUILTIN(__builtin_ia32_pmulhrsw128, "V8sV8sV8s", "ncV:128:", "ssse3")
-TARGET_BUILTIN(__builtin_ia32_pshufb128, "V16cV16cV16c", "ncV:128:", "ssse3")
-TARGET_BUILTIN(__builtin_ia32_psignb128, "V16cV16cV16c", "ncV:128:", "ssse3")
-TARGET_BUILTIN(__builtin_ia32_psignw128, "V8sV8sV8s", "ncV:128:", "ssse3")
-TARGET_BUILTIN(__builtin_ia32_psignd128, "V4iV4iV4i", "ncV:128:", "ssse3")
 
 TARGET_BUILTIN(__builtin_ia32_ldmxcsr, "vUi", "n", "sse")
 TARGET_HEADER_BUILTIN(_mm_setcsr, "vUi", "nh",XMMINTRIN_H, ALL_LANGUAGES, "sse")
@@ -316,16 +200,6 @@ TARGET_BUILTIN(__builtin_ia32_pclmulqdq256, "V4OiV4OiV4OiIc", "ncV:256:", "vpclm
 TARGET_BUILTIN(__builtin_ia32_pclmulqdq512, "V8OiV8OiV8OiIc", "ncV:512:", "avx512f,evex512,vpclmulqdq")
 
 // AVX
-TARGET_BUILTIN(__builtin_ia32_addsubpd256, "V4dV4dV4d", "ncV:256:", "avx")
-TARGET_BUILTIN(__builtin_ia32_addsubps256, "V8fV8fV8f", "ncV:256:", "avx")
-TARGET_BUILTIN(__builtin_ia32_haddpd256, "V4dV4dV4d", "ncV:256:", "avx")
-TARGET_BUILTIN(__builtin_ia32_hsubps256, "V8fV8fV8f", "ncV:256:", "avx")
-TARGET_BUILTIN(__builtin_ia32_hsubpd256, "V4dV4dV4d", "ncV:256:", "avx")
-TARGET_BUILTIN(__builtin_ia32_haddps256, "V8fV8fV8f", "ncV:256:", "avx")
-TARGET_BUILTIN(__builtin_ia32_maxpd256, "V4dV4dV4d", "ncV:256:", "avx")
-TARGET_BUILTIN(__builtin_ia32_maxps256, "V8fV8fV8f", "ncV:256:", "avx")
-TARGET_BUILTIN(__builtin_ia32_minpd256, "V4dV4dV4d", "ncV:256:", "avx")
-TARGET_BUILTIN(__builtin_ia32_minps256, "V8fV8fV8f", "ncV:256:", "avx")
 TARGET_BUILTIN(__builtin_ia32_vpermilvarpd, "V2dV2dV2Oi", "ncV:256:", "avx")
 TARGET_BUILTIN(__builtin_ia32_vpermilvarps, "V4fV4fV4i", "ncV:256:", "avx")
 TARGET_BUILTIN(__builtin_ia32_vpermilvarpd256, "V4dV4dV4Oi", "ncV:256:", "avx")
diff --git a/clang/include/clang/Basic/BuiltinsX86.td b/clang/include/clang/Basic/BuiltinsX86.td
new file mode 100644
index 000000000000..cf8d2771310e
--- /dev/null
+++ b/clang/include/clang/Basic/BuiltinsX86.td
@@ -0,0 +1,137 @@
+//===--- BuiltinsX86.td - X86 Builtin function database ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the X86-specific builtin function database.
+//
+//===----------------------------------------------------------------------===//
+
+include "clang/Basic/BuiltinsBase.td"
+
+class X86Builtin<string prototype> : TargetBuiltin {
+  let Spellings = ["__builtin_ia32_" # NAME];
+  let Prototype = prototype;
+}
+
+// Undefined Values
+def undef128 : X86Builtin<"_Vector<2, double>()"> {
+  let Attributes = [Const, NoThrow, RequiredVectorWidth<128>];
+}
+
+def undef256 : X86Builtin<"_Vector<4, double>()"> {
+  let Attributes = [Const, NoThrow, RequiredVectorWidth<256>];
+}
+
+def undef512 : X86Builtin<"_Vector<8, double>()"> {
+  let Attributes = [Const, NoThrow, RequiredVectorWidth<512>];
+}
+
+// FLAGS
+def readeflags_u32 : X86Builtin<"unsigned int()"> {
+  let Attributes = [NoThrow];
+}
+
+def writeeflags_u32 : X86Builtin<"void(unsigned int)"> {
+  let Attributes = [NoThrow];
+}
+
+// MMX
+//
+// All MMX instructions will be generated via builtins. Any MMX vector
+// types (<1 x i64>, <2 x i32>, etc.) that aren't used by these builtins will be
+// expanded by the back-end.
+
+def emms : X86Builtin<"void()"> {
+  let Attributes = [NoThrow];
+  let Features = "mmx";
+}
+
+let Attributes = [NoThrow, Const, RequiredVectorWidth<64>], Features = "sse" in {
+  def vec_ext_v4hi : X86Builtin<"short(_Vector<4, short>, _Constant int)">;
+  def vec_set_v4hi : X86Builtin<"_Vector<4, short>(_Vector<4, short>, short, _Constant int)">;
+}
+
+// SSE intrinsics
+let Attributes = [Const, NoThrow, RequiredVectorWidth<128>] in {
+  foreach Cmp = ["eq", "lt", "le", "gt", "ge", "neq"] in {
+    let Features = "sse" in {
+      def comi#Cmp : X86Builtin<"int(_Vector<4, float>, _Vector<4, float>)">;
+      def ucomi#Cmp : X86Builtin<"int(_Vector<4, float>, _Vector<4, float>)">;
+    }
+    let Features = "sse2" in {
+      def comisd#Cmp : X86Builtin<"int(_Vector<2, double>, _Vector<2, double>)">;
+      def ucomisd#Cmp : X86Builtin<"int(_Vector<2, double>, _Vector<2, double>)">;
+    }
+  }
+
+  foreach Cmp = ["cmpeq", "cmplt", "cmple", "cmpunord", "cmpneq", "cmpnlt",
+                 "cmpnle", "cmpord", "min", "max"] in {
+    let Features = "sse" in {
+      def Cmp#ps : X86Builtin<"_Vector<4, float>(_Vector<4, float>, _Vector<4, float>)">;
+      def Cmp#ss : X86Builtin<"_Vector<4, float>(_Vector<4, float>, _Vector<4, float>)">;
+    }
+    let Features = "sse2" in {
+      def Cmp#pd : X86Builtin<"_Vector<2, double>(_Vector<2, double>, _Vector<2, double>)">;
+      def Cmp#sd : X86Builtin<"_Vector<2, double>(_Vector<2, double>, _Vector<2, double>)">;
+    }
+  }
+
+  let Features = "sse" in {
+    def cmpps : X86Builtin<"_Vector<4, float>(_Vector<4, float>, _Vector<4, float>, _Constant char)">;
+    def cmpss : X86Builtin<"_Vector<4, float>(_Vector<4, float>, _Vector<4, float>, _Constant char)">;
+  }
+
+  let Features = "sse2" in {
+    def cmppd : X86Builtin<"_Vector<2, double>(_Vector<2, double>, _Vector<2, double>, _Constant char)">;
+    def cmpsd : X86Builtin<"_Vector<2, double>(_Vector<2, double>, _Vector<2, double>, _Constant char)">;
+  }
+
+  let Features = "sse2" in {
+    def pmulhw128 : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>)">;
+    def pavgb128 : X86Builtin<"_Vector<16, char>(_Vector<16, char>, _Vector<16, char>)">;
+    def pavgw128 : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>)">;
+    def packsswb128 : X86Builtin<"_Vector<16, char>(_Vector<8, short>, _Vector<8, short>)">;
+    def packssdw128 : X86Builtin<"_Vector<8, short>(_Vector<4, int>, _Vector<4, int>)">;
+    def packuswb128 : X86Builtin<"_Vector<16, char>(_Vector<8, short>, _Vector<8, short>)">;
+    def pmulhuw128 : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>)">;
+    def vec_ext_v2di : X86Builtin<"long long int(_Vector<2, long long int>, _Constant int)">;
+    def vec_ext_v4si : X86Builtin<"int(_Vector<4, int>, _Constant int)">;
+    def vec_ext_v4sf : X86Builtin<"float(_Vector<4, float>, _Constant int)">;
+    def vec_ext_v8hi : X86Builtin<"short(_Vector<8, short>, _Constant int)">;
+    def vec_set_v8hi : X86Builtin<"_Vector<8, short>(_Vector<8, short>, short, _Constant int)">;
+  }
+
+  let Features = "sse3" in {
+    foreach Op = ["addsub", "hadd", "hsub"] in {
+      def Op#ps : X86Builtin<"_Vector<4, float>(_Vector<4, float>, _Vector<4, float>)">;
+      def Op#pd : X86Builtin<"_Vector<2, double>(_Vector<2, double>, _Vector<2, double>)">;
+    }
+  }
+
+  let Features = "ssse3" in {
+    foreach Op = ["phadd", "phsub"] in {
+      def Op#w128 : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>)">;
+      def Op#sw128 : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>)">;
+      def Op#d128 : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>)">;
+    }
+
+    def pmaddubsw128 : X86Builtin<"_Vector<8, short>(_Vector<16, char>, _Vector<16, char>)">;
+    def pmulhrsw128 : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>)">;
+    def pshufb128 : X86Builtin<"_Vector<16, char>(_Vector<16, char>, _Vector<16, char>)">;
+    def psignb128 : X86Builtin<"_Vector<16, char>(_Vector<16, char>, _Vector<16, char>)">;
+    def psignw128 : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>)">;
+    def psignd128 : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>)">;
+  }
+}
+
+// AVX
+let Attributes = [Const, NoThrow, RequiredVectorWidth<256>], Features = "avx" in {
+  foreach Op = ["addsub", "hadd", "hsub", "max", "min"] in {
+    def Op#pd256 : X86Builtin<"_Vector<4, double>(_Vector<4, double>, _Vector<4, double>)">;
+    def Op#ps256 : X86Builtin<"_Vector<8, float>(_Vector<8, float>, _Vector<8, float>)">;
+  }
+}
diff --git a/clang/include/clang/Basic/CMakeLists.txt b/clang/include/clang/Basic/CMakeLists.txt
index f069f4fc118f..76ac3367e23a 100644
--- a/clang/include/clang/Basic/CMakeLists.txt
+++ b/clang/include/clang/Basic/CMakeLists.txt
@@ -60,6 +60,10 @@ clang_tablegen(BuiltinsRISCV.inc -gen-clang-builtins
   SOURCE BuiltinsRISCV.td
   TARGET ClangBuiltinsRISCV)
 
+clang_tablegen(BuiltinsX86.inc -gen-clang-builtins
+  SOURCE BuiltinsX86.td
+  TARGET ClangBuiltinsX86)
+
 # ARM NEON and MVE
 clang_tablegen(arm_neon.inc -gen-arm-neon-sema
   SOURCE arm_neon.td
diff --git a/clang/include/clang/Basic/TargetBuiltins.h b/clang/include/clang/Basic/TargetBuiltins.h
index d0f41b17c154..89ebf5758a5b 100644
--- a/clang/include/clang/Basic/TargetBuiltins.h
+++ b/clang/include/clang/Basic/TargetBuiltins.h
@@ -125,6 +125,8 @@ namespace clang {
     LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1,
 #define BUILTIN(ID, TYPE, ATTRS) BI##ID,
 #include "clang/Basic/BuiltinsX86.def"
+#define BUILTIN(ID, TYPE, ATTRS) BI##ID,
+#include "clang/Basic/BuiltinsX86.inc"
     FirstX86_64Builtin,
     LastX86CommonBuiltin = FirstX86_64Builtin - 1,
 #define BUILTIN(ID, TYPE, ATTRS) BI##ID,
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 700c2f9a5dbd..82d29ea9fea5 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -32,6 +32,14 @@ static constexpr Builtin::Info BuiltinInfoX86[] = {
   {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::HEADER, LANGS},
 #include "clang/Basic/BuiltinsX86.def"
 
+#define BUILTIN(ID, TYPE, ATTRS)                                               \
+  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
+  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+#define TARGET_HEADER_BUILTIN(ID, TYPE, ATTRS, HEADER, LANGS, FEATURE)         \
+  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::HEADER, LANGS},
+#include "clang/Basic/BuiltinsX86.inc"
+
 #define BUILTIN(ID, TYPE, ATTRS)                                               \
   {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
 #define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
diff --git a/clang/utils/TableGen/ClangBuiltinsEmitter.cpp b/clang/utils/TableGen/ClangBuiltinsEmitter.cpp
index 20231ee1502c..57a1fc06b26b 100644
--- a/clang/utils/TableGen/ClangBuiltinsEmitter.cpp
+++ b/clang/utils/TableGen/ClangBuiltinsEmitter.cpp
@@ -64,7 +64,8 @@ private:
       // detecting the comma of the template class as a separator for
       // the parameters of the prototype. Note: the assumption is that
       // we cannot have nested _ExtVector.
-      if (Current.starts_with("_ExtVector<")) {
+      if (Current.starts_with("_ExtVector<") ||
+          Current.starts_with("_Vector<")) {
         const size_t EndTemplate = Current.find('>', 0);
         ParseType(Current.substr(0, EndTemplate + 1));
         // Move the prototype beyond _ExtVector<...>
@@ -123,7 +124,8 @@ private:
       if (Substitution.empty())
         PrintFatalError(Loc, "Not a template");
       ParseType(Substitution);
-    } else if (T.consume_front("_ExtVector")) {
+    } else if (auto IsExt = T.consume_front("_ExtVector");
+               IsExt || T.consume_front("_Vector")) {
       // Clang extended vector types are mangled as follows:
       //
       // '_ExtVector<' <lanes> ',' <scalar type> '>'
@@ -135,7 +137,7 @@ private:
       unsigned long long Lanes;
       if (consumeUnsignedInteger(T, 10, Lanes))
         PrintFatalError(Loc, "Expected number of lanes after '_ExtVector<'");
-      Type += "E" + std::to_string(Lanes);
+      Type += (IsExt ? "E" : "V") + std::to_string(Lanes);
       if (!T.consume_front(","))
         PrintFatalError(Loc,
                         "Expected ',' after number of lanes in '_ExtVector<'");
-- 
GitLab


From c478aab684be007ac14e51565c0d4ae39293d208 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Wed, 30 Oct 2024 08:36:08 -0500
Subject: [PATCH 578/693] [flang][OpenMP] Parser support for DEPOBJ plus
 DEPEND, DESTROY, UPDATE (#114074)

Parse the DEPOBJ construct and the associated clauses, perform basic
semantic checks.
---
 flang/include/flang/Parser/dump-parse-tree.h  |   3 +
 flang/include/flang/Parser/parse-tree.h       |  57 +++++--
 flang/include/flang/Semantics/symbol.h        |   2 +-
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    |   2 +
 flang/lib/Lower/OpenMP/Clauses.cpp            |  49 ++++--
 flang/lib/Lower/OpenMP/Clauses.h              |   1 +
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  15 ++
 flang/lib/Parser/openmp-parsers.cpp           |  18 ++-
 flang/lib/Parser/parse-tree.cpp               |  17 ++
 flang/lib/Parser/unparse.cpp                  |  12 +-
 flang/lib/Semantics/check-omp-structure.cpp   | 152 ++++++++++++++++--
 flang/lib/Semantics/check-omp-structure.h     |   2 +
 flang/lib/Semantics/resolve-directives.cpp    |   8 +
 flang/lib/Semantics/resolve-names.cpp         |   7 +
 .../Lower/OpenMP/Todo/depobj-construct.f90    |   9 ++
 flang/test/Parser/OpenMP/depobj-construct.f90 |  64 ++++++++
 .../Semantics/OpenMP/depobj-construct-v50.f90 |  28 ++++
 .../Semantics/OpenMP/depobj-construct-v51.f90 |  13 ++
 .../Semantics/OpenMP/depobj-construct-v52.f90 |  15 ++
 llvm/include/llvm/Frontend/OpenMP/ClauseT.h   |   3 +-
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |   3 +
 21 files changed, 429 insertions(+), 51 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/Todo/depobj-construct.f90
 create mode 100644 flang/test/Parser/OpenMP/depobj-construct.f90
 create mode 100644 flang/test/Semantics/OpenMP/depobj-construct-v50.f90
 create mode 100644 flang/test/Semantics/OpenMP/depobj-construct-v51.f90
 create mode 100644 flang/test/Semantics/OpenMP/depobj-construct-v52.f90

diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 31ad1b7c6ce5..67f7e1aac40e 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -517,6 +517,7 @@ public:
   NODE_ENUM(OmpTaskDependenceType, Type)
   NODE(parser, OmpDependSinkVec)
   NODE(parser, OmpDependSinkVecLength)
+  NODE(parser, OmpDestroyClause)
   NODE(parser, OmpEndAllocators)
   NODE(parser, OmpEndAtomic)
   NODE(parser, OmpEndBlockDirective)
@@ -571,6 +572,7 @@ public:
   NODE_ENUM(OmpDeviceClause, DeviceModifier)
   NODE(parser, OmpDeviceTypeClause)
   NODE_ENUM(OmpDeviceTypeClause, Type)
+  NODE(parser, OmpUpdateClause)
   NODE(parser, OmpScheduleModifier)
   NODE(OmpScheduleModifier, Modifier1)
   NODE(OmpScheduleModifier, Modifier2)
@@ -609,6 +611,7 @@ public:
   NODE(parser, OmpAtomicClauseList)
   NODE(parser, OmpAtomicDefaultMemOrderClause)
   NODE_ENUM(common, OmpAtomicDefaultMemOrderType)
+  NODE(parser, OpenMPDepobjConstruct)
   NODE(parser, OpenMPFlushConstruct)
   NODE(parser, OpenMPLoopConstruct)
   NODE(parser, OpenMPExecutableAllocate)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 174f4c631e9d..13c335351220 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3447,7 +3447,7 @@ WRAPPER_CLASS(OmpObjectList, std::list<OmpObject>);
 //    MUTEXINOUTSET | DEPOBJ |  // since 5.0
 //    INOUTSET                  // since 5.2
 struct OmpTaskDependenceType {
-  ENUM_CLASS(Type, In, Out, Inout, Source, Sink)
+  ENUM_CLASS(Type, In, Out, Inout, Source, Sink, Depobj)
   WRAPPER_CLASS_BOILERPLATE(OmpTaskDependenceType, Type);
 };
 
@@ -3527,19 +3527,6 @@ struct OmpDefaultmapClause {
   std::tuple<ImplicitBehavior, std::optional<VariableCategory>> t;
 };
 
-// device([ device-modifier :] scalar-integer-expression)
-struct OmpDeviceClause {
-  TUPLE_CLASS_BOILERPLATE(OmpDeviceClause);
-  ENUM_CLASS(DeviceModifier, Ancestor, Device_Num)
-  std::tuple<std::optional<DeviceModifier>, ScalarIntExpr> t;
-};
-
-// device_type(any | host | nohost)
-struct OmpDeviceTypeClause {
-  ENUM_CLASS(Type, Any, Host, Nohost)
-  WRAPPER_CLASS_BOILERPLATE(OmpDeviceTypeClause, Type);
-};
-
 // 2.13.9 depend-vec-length -> +/- non-negative-constant
 struct OmpDependSinkVecLength {
   TUPLE_CLASS_BOILERPLATE(OmpDependSinkVecLength);
@@ -3561,6 +3548,8 @@ struct OmpDependSinkVec {
 //
 // depend-modifier -> iterator-modifier              // since 5.0
 struct OmpDependClause {
+  OmpTaskDependenceType::Type GetDepType() const;
+
   UNION_CLASS_BOILERPLATE(OmpDependClause);
   EMPTY_CLASS(Source);
   WRAPPER_CLASS(Sink, std::list<OmpDependSinkVec>);
@@ -3573,6 +3562,26 @@ struct OmpDependClause {
   std::variant<Source, Sink, InOut> u;
 };
 
+// Ref: [5.0:254-255], [5.1:287-288], [5.2:73]
+//
+// destroy-clause ->
+//    DESTROY |             // since 5.0, until 5.2
+//    DESTROY(variable)     // since 5.2
+WRAPPER_CLASS(OmpDestroyClause, OmpObject);
+
+// device([ device-modifier :] scalar-integer-expression)
+struct OmpDeviceClause {
+  TUPLE_CLASS_BOILERPLATE(OmpDeviceClause);
+  ENUM_CLASS(DeviceModifier, Ancestor, Device_Num)
+  std::tuple<std::optional<DeviceModifier>, ScalarIntExpr> t;
+};
+
+// device_type(any | host | nohost)
+struct OmpDeviceTypeClause {
+  ENUM_CLASS(Type, Any, Host, Nohost)
+  WRAPPER_CLASS_BOILERPLATE(OmpDeviceTypeClause, Type);
+};
+
 // OMP 5.2 12.6.1 grainsize-clause -> grainsize ([prescriptiveness :] value)
 struct OmpGrainsizeClause {
   TUPLE_CLASS_BOILERPLATE(OmpGrainsizeClause);
@@ -3716,6 +3725,11 @@ struct OmpNumTasksClause {
   std::tuple<std::optional<Prescriptiveness>, ScalarIntExpr> t;
 };
 
+// Ref: [5.0:254-255], [5.1:287-288], [5.2:321-322]
+//
+// update-clause -> UPDATE(task-dependence-type)    // since 5.0
+WRAPPER_CLASS(OmpUpdateClause, OmpTaskDependenceType);
+
 // OpenMP Clauses
 struct OmpClause {
   UNION_CLASS_BOILERPLATE(OmpClause);
@@ -4023,6 +4037,18 @@ struct OpenMPCancelConstruct {
   std::tuple<Verbatim, OmpCancelType, std::optional<If>> t;
 };
 
+// Ref: [5.0:254-255], [5.1:287-288], [5.2:322-323]
+//
+// depobj-construct -> DEPOBJ(depend-object) depobj-clause  // since 5.0
+// depobj-clause -> depend-clause |                         // until 5.2
+//                  destroy-clause |
+//                  update-clause
+struct OpenMPDepobjConstruct {
+  TUPLE_CLASS_BOILERPLATE(OpenMPDepobjConstruct);
+  CharBlock source;
+  std::tuple<Verbatim, OmpObject, OmpClause> t;
+};
+
 // 2.17.8 flush -> FLUSH [memory-order-clause] [(variable-name-list)]
 struct OpenMPFlushConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPFlushConstruct);
@@ -4047,7 +4073,8 @@ struct OpenMPStandaloneConstruct {
   UNION_CLASS_BOILERPLATE(OpenMPStandaloneConstruct);
   CharBlock source;
   std::variant<OpenMPSimpleStandaloneConstruct, OpenMPFlushConstruct,
-      OpenMPCancelConstruct, OpenMPCancellationPointConstruct>
+      OpenMPCancelConstruct, OpenMPCancellationPointConstruct,
+      OpenMPDepobjConstruct>
       u;
 };
 
diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h
index 0767d8ea84bc..b9512f33eaac 100644
--- a/flang/include/flang/Semantics/symbol.h
+++ b/flang/include/flang/Semantics/symbol.h
@@ -755,7 +755,7 @@ public:
       OmpDeclarativeAllocateDirective, OmpExecutableAllocateDirective,
       OmpDeclareSimd, OmpDeclareTarget, OmpThreadprivate, OmpDeclareReduction,
       OmpFlushed, OmpCriticalLock, OmpIfSpecified, OmpNone, OmpPreDetermined,
-      OmpImplicit);
+      OmpImplicit, OmpDependObject);
   using Flags = common::EnumSet<Flag, Flag_enumSize>;
 
   const Scope &owner() const { return *owner_; }
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 7c254ce67385..8eb1fdb47091 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -137,6 +137,8 @@ genDependKindAttr(fir::FirOpBuilder &firOpBuilder,
   case omp::clause::Depend::TaskDependenceType::Mutexinoutset:
   case omp::clause::Depend::TaskDependenceType::Inoutset:
   case omp::clause::Depend::TaskDependenceType::Depobj:
+  case omp::clause::Depend::TaskDependenceType::Sink:
+  case omp::clause::Depend::TaskDependenceType::Source:
     llvm_unreachable("unhandled parser task dependence type");
     break;
   }
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 9483f643acd5..45b89de023a4 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -338,6 +338,27 @@ ReductionOperator makeReductionOperator(const parser::OmpReductionOperator &inp,
       inp.u);
 }
 
+clause::TaskDependenceType
+makeDepType(const parser::OmpTaskDependenceType &inp) {
+  switch (inp.v) {
+  case parser::OmpTaskDependenceType::Type::Depobj:
+    return clause::TaskDependenceType::Depobj;
+  case parser::OmpTaskDependenceType::Type::In:
+    return clause::TaskDependenceType::In;
+  case parser::OmpTaskDependenceType::Type::Inout:
+    return clause::TaskDependenceType::Inout;
+  // Inoutset        // missing-in-parser
+  // Mutexinoutset   // missing-in-parser
+  case parser::OmpTaskDependenceType::Type::Out:
+    return clause::TaskDependenceType::Out;
+  case parser::OmpTaskDependenceType::Type::Sink:
+    return clause::TaskDependenceType::Sink;
+  case parser::OmpTaskDependenceType::Type::Source:
+    return clause::TaskDependenceType::Source;
+  }
+  llvm_unreachable("Unexpected dependence type");
+}
+
 // --------------------------------------------------------------------
 // Actual clauses. Each T (where tomp::T exists in ClauseT) has its "make".
 
@@ -554,18 +575,6 @@ Depend make(const parser::OmpClause::Depend &inp,
   // Iteration is the equivalent of parser::OmpDependSinkVec
   using Iteration = Doacross::Vector::value_type; // LoopIterationT
 
-  CLAUSET_ENUM_CONVERT( //
-      convert1, parser::OmpTaskDependenceType::Type, Depend::TaskDependenceType,
-      // clang-format off
-      MS(In,     In)
-      MS(Out,    Out)
-      MS(Inout,  Inout)
-      // MS(, Mutexinoutset)   // missing-in-parser
-      // MS(, Inputset)        // missing-in-parser
-      // MS(, Depobj)          // missing-in-parser
-      // clang-format on
-  );
-
   return Depend{Fortran::common::visit( //
       common::visitors{
           // Doacross
@@ -602,7 +611,7 @@ Depend make(const parser::OmpClause::Depend &inp,
 
             auto &&maybeIter = maybeApply(
                 [&](auto &&s) { return makeIterator(s, semaCtx); }, t0);
-            return Depend::DepType{{/*TaskDependenceType=*/convert1(t1.v),
+            return Depend::DepType{{/*TaskDependenceType=*/makeDepType(t1),
                                     /*Iterator=*/std::move(maybeIter),
                                     /*LocatorList=*/makeObjects(t2, semaCtx)}};
           },
@@ -614,8 +623,14 @@ Depend make(const parser::OmpClause::Depend &inp,
 
 Destroy make(const parser::OmpClause::Destroy &inp,
              semantics::SemanticsContext &semaCtx) {
-  // inp -> empty
-  llvm_unreachable("Empty: destroy");
+  // inp.v -> std::optional<OmpDestroyClause>
+  auto &&maybeObject = maybeApply(
+      [&](const parser::OmpDestroyClause &c) {
+        return makeObject(c.v, semaCtx);
+      },
+      inp.v);
+
+  return Destroy{/*DestroyVar=*/std::move(maybeObject)};
 }
 
 Detach make(const parser::OmpClause::Detach &inp,
@@ -1279,8 +1294,8 @@ Uniform make(const parser::OmpClause::Uniform &inp,
 
 Update make(const parser::OmpClause::Update &inp,
             semantics::SemanticsContext &semaCtx) {
-  // inp -> empty
-  return Update{/*TaskDependenceType=*/std::nullopt};
+  // inp.v -> parser::OmpUpdateClause
+  return Update{/*TaskDependenceType=*/makeDepType(inp.v.v)};
 }
 
 Use make(const parser::OmpClause::Use &inp,
diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h
index 1e911a204685..51180ebfe574 100644
--- a/flang/lib/Lower/OpenMP/Clauses.h
+++ b/flang/lib/Lower/OpenMP/Clauses.h
@@ -152,6 +152,7 @@ using IteratorSpecifier = tomp::type::IteratorSpecifierT<TypeTy, IdTy, ExprTy>;
 using DefinedOperator = tomp::type::DefinedOperatorT<IdTy, ExprTy>;
 using ProcedureDesignator = tomp::type::ProcedureDesignatorT<IdTy, ExprTy>;
 using ReductionOperator = tomp::type::ReductionIdentifierT<IdTy, ExprTy>;
+using TaskDependenceType = tomp::type::TaskDependenceType;
 
 // "Requires" clauses are handled early on, and the aggregated information
 // is stored in the Symbol details of modules, programs, and subprograms.
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 876feca9b6f5..84985b880b1e 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2710,6 +2710,21 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   TODO(converter.getCurrentLocation(), "OpenMPCancelConstruct");
 }
 
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPDepobjConstruct &construct) {
+  // These values will be ignored until the construct itself is implemented,
+  // but run them anyway for the sake of testing (via a Todo test).
+  auto &ompObj = std::get<parser::OmpObject>(construct.t);
+  const Object &depObj = makeObject(ompObj, semaCtx);
+  Clause clause = makeClause(std::get<parser::OmpClause>(construct.t), semaCtx);
+  (void)depObj;
+  (void)clause;
+
+  TODO(converter.getCurrentLocation(), "OpenMPDepobjConstruct");
+}
+
 static void
 genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
        semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 5276e1ec1dca..6fde70fc5c38 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -366,9 +366,12 @@ TYPE_PARSER(
     construct<OmpDependSinkVec>(name, maybe(Parser<OmpDependSinkVecLength>{})))
 
 TYPE_PARSER(construct<OmpTaskDependenceType>(
+    "DEPOBJ" >> pure(OmpTaskDependenceType::Type::Depobj) ||
     "IN"_id >> pure(OmpTaskDependenceType::Type::In) ||
     "INOUT" >> pure(OmpTaskDependenceType::Type::Inout) ||
-    "OUT" >> pure(OmpTaskDependenceType::Type::Out)))
+    "OUT" >> pure(OmpTaskDependenceType::Type::Out) ||
+    "SINK" >> pure(OmpTaskDependenceType::Type::Sink) ||
+    "SOURCE" >> pure(OmpTaskDependenceType::Type::Source)))
 
 TYPE_CONTEXT_PARSER("Omp Depend clause"_en_US,
     construct<OmpDependClause>(construct<OmpDependClause::Sink>(
@@ -454,6 +457,9 @@ TYPE_PARSER(
                         parenthesized(Parser<OmpDefaultmapClause>{}))) ||
     "DEPEND" >> construct<OmpClause>(construct<OmpClause::Depend>(
                     parenthesized(Parser<OmpDependClause>{}))) ||
+    "DESTROY" >>
+        construct<OmpClause>(construct<OmpClause::Destroy>(maybe(parenthesized(
+            construct<OmpDestroyClause>(Parser<OmpObject>{}))))) ||
     "DEVICE" >> construct<OmpClause>(construct<OmpClause::Device>(
                     parenthesized(Parser<OmpDeviceClause>{}))) ||
     "DEVICE_TYPE" >> construct<OmpClause>(construct<OmpClause::DeviceType>(
@@ -560,7 +566,9 @@ TYPE_PARSER(
         construct<OmpClause>(construct<OmpClause::UnifiedSharedMemory>()) ||
     "UNIFORM" >> construct<OmpClause>(construct<OmpClause::Uniform>(
                      parenthesized(nonemptyList(name)))) ||
-    "UNTIED" >> construct<OmpClause>(construct<OmpClause::Untied>()))
+    "UNTIED" >> construct<OmpClause>(construct<OmpClause::Untied>()) ||
+    "UPDATE" >> construct<OmpClause>(construct<OmpClause::Update>(
+                    parenthesized(Parser<OmpTaskDependenceType>{}))))
 
 // [Clause, [Clause], ...]
 TYPE_PARSER(sourced(construct<OmpClauseList>(
@@ -680,6 +688,9 @@ TYPE_PARSER(sourced(construct<OmpAtomicClause>(
 TYPE_PARSER(sourced(construct<OmpAtomicClauseList>(
     many(maybe(","_tok) >> sourced(Parser<OmpAtomicClause>{})))))
 
+TYPE_PARSER(sourced(construct<OpenMPDepobjConstruct>(verbatim("DEPOBJ"_tok),
+    parenthesized(Parser<OmpObject>{}), sourced(Parser<OmpClause>{}))))
+
 TYPE_PARSER(sourced(construct<OpenMPFlushConstruct>(verbatim("FLUSH"_tok),
     many(maybe(","_tok) >> sourced(Parser<OmpMemoryOrderClause>{})),
     maybe(parenthesized(Parser<OmpObjectList>{})))))
@@ -704,7 +715,8 @@ TYPE_PARSER(
         construct<OpenMPStandaloneConstruct>(Parser<OpenMPFlushConstruct>{}) ||
         construct<OpenMPStandaloneConstruct>(Parser<OpenMPCancelConstruct>{}) ||
         construct<OpenMPStandaloneConstruct>(
-            Parser<OpenMPCancellationPointConstruct>{})) /
+            Parser<OpenMPCancellationPointConstruct>{}) ||
+        construct<OpenMPStandaloneConstruct>(Parser<OpenMPDepobjConstruct>{})) /
     endOfLine)
 
 // Directives enclosing structured-block
diff --git a/flang/lib/Parser/parse-tree.cpp b/flang/lib/Parser/parse-tree.cpp
index 948ad04a091a..60aef1666e9b 100644
--- a/flang/lib/Parser/parse-tree.cpp
+++ b/flang/lib/Parser/parse-tree.cpp
@@ -252,6 +252,23 @@ CharBlock Variable::GetSource() const {
 llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const Name &x) {
   return os << x.ToString();
 }
+
+OmpTaskDependenceType::Type OmpDependClause::GetDepType() const {
+  return common::visit(
+      common::visitors{
+          [&](const parser::OmpDependClause::Source &) {
+            return parser::OmpTaskDependenceType::Type::Source;
+          },
+          [&](const parser::OmpDependClause::Sink &) {
+            return parser::OmpTaskDependenceType::Type::Sink;
+          },
+          [&](const parser::OmpDependClause::InOut &y) {
+            return std::get<parser::OmpTaskDependenceType>(y.t).v;
+          },
+      },
+      u);
+}
+
 } // namespace Fortran::parser
 
 template <typename C> static llvm::omp::Clause getClauseIdForClass(C &&) {
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index e80ab0da1360..3b0824f80161 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2215,11 +2215,9 @@ public:
     Walk(std::get<std::optional<OmpDependSinkVecLength>>(x.t));
   }
   void Unparse(const OmpDependClause::InOut &x) {
-    Put("(");
     Walk(std::get<OmpTaskDependenceType>(x.t));
     Put(":");
     Walk(std::get<OmpObjectList>(x.t));
-    Put(")");
   }
   bool Pre(const OmpDependClause &x) {
     return common::visit(
@@ -2721,6 +2719,16 @@ public:
                   },
         x.u);
   }
+  void Unparse(const OpenMPDepobjConstruct &x) {
+    BeginOpenMP();
+    Word("!$OMP DEPOBJ");
+    Put("(");
+    Walk(std::get<OmpObject>(x.t));
+    Put(") ");
+    Walk(std::get<OmpClause>(x.t));
+    Put("\n");
+    EndOpenMP();
+  }
   void Unparse(const OpenMPFlushConstruct &x) {
     BeginOpenMP();
     Word("!$OMP FLUSH ");
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 8f3eb9fefee6..c813100b4b16 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -1261,6 +1261,39 @@ void OmpStructureChecker::Leave(const parser::OpenMPDeclareSimdConstruct &) {
   dirContext_.pop_back();
 }
 
+void OmpStructureChecker::Enter(const parser::OpenMPDepobjConstruct &x) {
+  const auto &dir{std::get<parser::Verbatim>(x.t)};
+  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_depobj);
+
+  // [5.2:73:27-28]
+  // If the destroy clause appears on a depobj construct, destroy-var must
+  // refer to the same depend object as the depobj argument of the construct.
+  auto &clause{std::get<parser::OmpClause>(x.t)};
+  if (clause.Id() == llvm::omp::Clause::OMPC_destroy) {
+    auto getSymbol{[&](const parser::OmpObject &obj) {
+      return common::visit(
+          [&](auto &&s) { return GetLastName(s).symbol; }, obj.u);
+    }};
+
+    auto &wrapper{std::get<parser::OmpClause::Destroy>(clause.u)};
+    if (const std::optional<parser::OmpDestroyClause> &destroy{wrapper.v}) {
+      const Symbol *constrSym{getSymbol(std::get<parser::OmpObject>(x.t))};
+      const Symbol *clauseSym{getSymbol(destroy->v)};
+      assert(constrSym && "Unresolved depobj construct symbol");
+      assert(clauseSym && "Unresolved destroy symbol on depobj construct");
+      if (constrSym != clauseSym) {
+        context_.Say(x.source,
+            "The DESTROY clause must refer to the same object as the "
+            "DEPOBJ construct"_err_en_US);
+      }
+    }
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPDepobjConstruct &x) {
+  dirContext_.pop_back();
+}
+
 void OmpStructureChecker::Enter(const parser::OpenMPRequiresConstruct &x) {
   const auto &dir{std::get<parser::Verbatim>(x.t)};
   PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_requires);
@@ -2476,7 +2509,6 @@ CHECK_SIMPLE_CLAUSE(Capture, OMPC_capture)
 CHECK_SIMPLE_CLAUSE(Contains, OMPC_contains)
 CHECK_SIMPLE_CLAUSE(Default, OMPC_default)
 CHECK_SIMPLE_CLAUSE(Depobj, OMPC_depobj)
-CHECK_SIMPLE_CLAUSE(Destroy, OMPC_destroy)
 CHECK_SIMPLE_CLAUSE(Detach, OMPC_detach)
 CHECK_SIMPLE_CLAUSE(DeviceType, OMPC_device_type)
 CHECK_SIMPLE_CLAUSE(DistSchedule, OMPC_dist_schedule)
@@ -2519,7 +2551,6 @@ CHECK_SIMPLE_CLAUSE(Uniform, OMPC_uniform)
 CHECK_SIMPLE_CLAUSE(Unknown, OMPC_unknown)
 CHECK_SIMPLE_CLAUSE(Untied, OMPC_untied)
 CHECK_SIMPLE_CLAUSE(UsesAllocators, OMPC_uses_allocators)
-CHECK_SIMPLE_CLAUSE(Update, OMPC_update)
 CHECK_SIMPLE_CLAUSE(Write, OMPC_write)
 CHECK_SIMPLE_CLAUSE(Init, OMPC_init)
 CHECK_SIMPLE_CLAUSE(Use, OMPC_use)
@@ -2555,6 +2586,22 @@ CHECK_REQ_CONSTANT_SCALAR_INT_CLAUSE(Simdlen, OMPC_simdlen)
 
 // Restrictions specific to each clause are implemented apart from the
 // generalized restrictions.
+
+void OmpStructureChecker::Enter(const parser::OmpClause::Destroy &x) {
+  CheckAllowedClause(llvm::omp::Clause::OMPC_destroy);
+
+  llvm::omp::Directive dir{GetContext().directive};
+  unsigned version{context_.langOptions().OpenMPVersion};
+  if (dir == llvm::omp::Directive::OMPD_depobj) {
+    if (version < 52) {
+      context_.Say(GetContext().clauseSource,
+          "The object parameter in DESTROY clause in DEPOPJ construct "
+          "was introduced in %s"_port_en_US,
+          ThisVersion(52));
+    }
+  }
+}
+
 void OmpStructureChecker::Enter(const parser::OmpClause::Reduction &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_reduction);
   if (CheckReductionOperators(x)) {
@@ -3285,16 +3332,63 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Device &x) {
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Depend &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_depend);
-  if ((std::holds_alternative<parser::OmpDependClause::Source>(x.v.u) ||
-          std::holds_alternative<parser::OmpDependClause::Sink>(x.v.u)) &&
-      GetContext().directive != llvm::omp::OMPD_ordered) {
-    context_.Say(GetContext().clauseSource,
-        "DEPEND(SOURCE) or DEPEND(SINK : vec) can be used only with the ordered"
-        " directive. Used here in the %s construct."_err_en_US,
-        parser::ToUpperCaseLetters(getDirectiveName(GetContext().directive)));
+  llvm::omp::Directive directive{GetContext().directive};
+  unsigned version{context_.langOptions().OpenMPVersion};
+
+  using DepType = parser::OmpTaskDependenceType::Type;
+  DepType depType = x.v.GetDepType();
+
+  if (version >= 52) {
+    switch (depType) {
+    case DepType::Sink:
+    case DepType::Source:
+      context_.Say(GetContext().clauseSource,
+          "The %s task-dependence-type is deprecated in %s"_warn_en_US,
+          parser::ToUpperCaseLetters(
+              parser::OmpTaskDependenceType::EnumToString(depType)),
+          ThisVersion(version));
+      break;
+    default:
+      break;
+    }
+  }
+
+  if (directive == llvm::omp::OMPD_depobj) {
+    // [5.0:255:11], [5.1:288:3]
+    // A depend clause on a depobj construct must not have source, sink [or
+    // depobj](5.0) as dependence-type.
+    if (version >= 50) {
+      bool invalidDep{depType == DepType::Source || depType == DepType::Sink};
+      if (version == 50) {
+        invalidDep = invalidDep || depType == DepType::Depobj;
+      }
+      if (invalidDep) {
+        context_.Say(GetContext().clauseSource,
+            "A DEPEND clause on a DEPOBJ construct must not have SOURCE%s "
+            "as dependence-type"_err_en_US,
+            version == 50 ? ", SINK or DEPOBJ" : " or SINK");
+      }
+    }
+  } else if (directive != llvm::omp::OMPD_ordered) {
+    if (depType == DepType::Source || depType == DepType::Sink) {
+      context_.Say(GetContext().clauseSource,
+          "DEPEND(SOURCE) or DEPEND(SINK : vec) can be used only with the "
+          "ordered directive. Used here in the %s construct."_err_en_US,
+          parser::ToUpperCaseLetters(getDirectiveName(directive)));
+    }
   }
   if (const auto *inOut{std::get_if<parser::OmpDependClause::InOut>(&x.v.u)}) {
-    for (const auto &object : std::get<parser::OmpObjectList>(inOut->t).v) {
+    auto &objList{std::get<parser::OmpObjectList>(inOut->t)};
+    if (directive == llvm::omp::OMPD_depobj) {
+      // [5.0:255:13], [5.1:288:6], [5.2:322:26]
+      // A depend clause on a depobj construct must only specify one locator.
+      if (objList.v.size() != 1) {
+        context_.Say(GetContext().clauseSource,
+            "A DEPEND clause on a DEPOBJ construct must only specify "
+            "one locator"_err_en_US);
+      }
+    }
+    for (const auto &object : objList.v) {
       if (const auto *name{std::get_if<parser::Name>(&object.u)}) {
         context_.Say(GetContext().clauseSource,
             "Common block name ('%s') cannot appear in a DEPEND "
@@ -3313,12 +3407,18 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Depend &x) {
       }
     }
     if (std::get<std::optional<parser::OmpIteratorModifier>>(inOut->t)) {
-      unsigned version{context_.langOptions().OpenMPVersion};
       unsigned allowedInVersion{50};
       if (version < allowedInVersion) {
         context_.Say(GetContext().clauseSource,
             "Iterator modifiers are not supported in %s, %s"_warn_en_US,
             ThisVersion(version), TryVersion(allowedInVersion));
+      } else {
+        if (directive == llvm::omp::OMPD_depobj) {
+          context_.Say(GetContext().clauseSource,
+              "An iterator-modifier may specify multiple locators, "
+              "a DEPEND clause on a DEPOBJ construct must only specify "
+              "one locator"_warn_en_US);
+        }
       }
     }
   }
@@ -3433,6 +3533,34 @@ void OmpStructureChecker::CheckStructureElement(
   return;
 }
 
+void OmpStructureChecker::Enter(const parser::OmpClause::Update &x) {
+  CheckAllowedClause(llvm::omp::Clause::OMPC_update);
+  llvm::omp::Directive directive{GetContext().directive};
+  unsigned version{context_.langOptions().OpenMPVersion};
+
+  // [5.1:288:4-5]
+  // An update clause on a depobj construct must not have source, sink or depobj
+  // as dependence-type.
+  // [5.2:322:3]
+  // task-dependence-type must not be depobj.
+  if (directive == llvm::omp::OMPD_depobj) {
+    if (version >= 51) {
+      // Update -> OmpUpdateClause -> OmpTaskDependenceType -> Type
+      switch (x.v.v.v) {
+      case parser::OmpTaskDependenceType::Type::Source:
+      case parser::OmpTaskDependenceType::Type::Sink:
+      case parser::OmpTaskDependenceType::Type::Depobj:
+        context_.Say(GetContext().clauseSource,
+            "An UPDATE clause on a DEPOBJ construct must not have SOURCE, "
+            "SINK or DEPOBJ as dependence-type"_err_en_US);
+        break;
+      default:
+        break;
+      }
+    }
+  }
+}
+
 void OmpStructureChecker::Enter(const parser::OmpClause::UseDevicePtr &x) {
   CheckStructureElement(x.v, llvm::omp::Clause::OMPC_use_device_ptr);
   CheckAllowedClause(llvm::omp::Clause::OMPC_use_device_ptr);
@@ -3616,7 +3744,7 @@ void OmpStructureChecker::CheckDependList(const parser::DataRef &d) {
             context_.Say(GetContext().clauseSource,
                 "Coarrays are not supported in DEPEND clause"_err_en_US);
           },
-          [&](const parser::Name &) { return; },
+          [&](const parser::Name &) {},
       },
       d.u);
 }
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index 237569bc40c4..d5fd558cea23 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -92,6 +92,8 @@ public:
   void Leave(const parser::OpenMPDeclarativeAllocate &);
   void Enter(const parser::OpenMPDeclareTargetConstruct &);
   void Leave(const parser::OpenMPDeclareTargetConstruct &);
+  void Enter(const parser::OpenMPDepobjConstruct &);
+  void Leave(const parser::OpenMPDepobjConstruct &);
   void Enter(const parser::OmpDeclareTargetWithList &);
   void Enter(const parser::OmpDeclareTargetWithClause &);
   void Leave(const parser::OmpDeclareTargetWithClause &);
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 5e3ad5f3b477..359dac911b8c 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -383,6 +383,14 @@ public:
   }
   void Post(const parser::OpenMPDeclareSimdConstruct &) { PopContext(); }
 
+  bool Pre(const parser::OpenMPDepobjConstruct &x) {
+    PushContext(x.source, llvm::omp::Directive::OMPD_depobj);
+    auto &object{std::get<parser::OmpObject>(x.t)};
+    ResolveOmpObject(object, Symbol::Flag::OmpDependObject);
+    return true;
+  }
+  void Post(const parser::OpenMPDepobjConstruct &) { PopContext(); }
+
   bool Pre(const parser::OpenMPRequiresConstruct &x) {
     using Flags = WithOmpDeclarative::RequiresFlags;
     using Requires = WithOmpDeclarative::RequiresFlag;
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index add4e4befd3a..e0a8246ebc75 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -1538,6 +1538,13 @@ public:
   void Post(const parser::OpenMPDeclarativeConstruct &) {
     messageHandler().set_currStmtSource(std::nullopt);
   }
+  bool Pre(const parser::OpenMPDepobjConstruct &x) {
+    AddOmpSourceRange(x.source);
+    return true;
+  }
+  void Post(const parser::OpenMPDepobjConstruct &x) {
+    messageHandler().set_currStmtSource(std::nullopt);
+  }
   bool Pre(const parser::OpenMPAtomicConstruct &x) {
     return common::visit(common::visitors{[&](const auto &u) -> bool {
       AddOmpSourceRange(u.source);
diff --git a/flang/test/Lower/OpenMP/Todo/depobj-construct.f90 b/flang/test/Lower/OpenMP/Todo/depobj-construct.f90
new file mode 100644
index 000000000000..2b3c4d92c4a4
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/depobj-construct.f90
@@ -0,0 +1,9 @@
+!RUN: %not_todo_cmd bbc -emit-hlfir -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
+!RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
+
+!CHECK: not yet implemented: OpenMPDepobjConstruct
+subroutine f00()
+  integer :: obj
+  integer :: x
+  !$omp depobj(obj) depend(in: x)
+end
diff --git a/flang/test/Parser/OpenMP/depobj-construct.f90 b/flang/test/Parser/OpenMP/depobj-construct.f90
new file mode 100644
index 000000000000..7c474071bc1e
--- /dev/null
+++ b/flang/test/Parser/OpenMP/depobj-construct.f90
@@ -0,0 +1,64 @@
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=52 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=52 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f00
+  integer :: x, y
+  !$omp depobj(x) depend(in: y)
+end
+
+!UNPARSE: SUBROUTINE f00
+!UNPARSE:  INTEGER x, y
+!UNPARSE: !$OMP DEPOBJ(x) DEPEND(IN:y)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPDepobjConstruct
+!PARSE-TREE: | Verbatim
+!PARSE-TREE: | OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | OmpClause -> Depend -> OmpDependClause -> InOut
+!PARSE-TREE: | | OmpTaskDependenceType -> Type = In
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'y'
+
+subroutine f01
+  integer :: x
+  !$omp depobj(x) update(out)
+end
+
+!UNPARSE: SUBROUTINE f01
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP DEPOBJ(x) UPDATE(OUT)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPDepobjConstruct
+!PARSE-TREE: | Verbatim
+!PARSE-TREE: | OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | OmpClause -> Update -> OmpUpdateClause -> OmpTaskDependenceType -> Type = Out
+
+subroutine f02
+  integer :: x
+  !$omp depobj(x) destroy(x)
+end
+
+!UNPARSE: SUBROUTINE f02
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP DEPOBJ(x) DESTROY(x)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPDepobjConstruct
+!PARSE-TREE: | Verbatim
+!PARSE-TREE: | OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | OmpClause -> Destroy -> OmpDestroyClause -> OmpObject -> Designator -> DataRef -> Name = 'x'
+
+subroutine f03
+  integer :: x
+  !$omp depobj(x) destroy
+end
+
+!UNPARSE: SUBROUTINE f03
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP DEPOBJ(x) DESTROY
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPDepobjConstruct
+!PARSE-TREE: | Verbatim
+!PARSE-TREE: | OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | OmpClause -> Destroy ->
diff --git a/flang/test/Semantics/OpenMP/depobj-construct-v50.f90 b/flang/test/Semantics/OpenMP/depobj-construct-v50.f90
new file mode 100644
index 000000000000..e7fa24d521b6
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/depobj-construct-v50.f90
@@ -0,0 +1,28 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=50
+
+subroutine f00
+  integer :: obj
+!ERROR: A DEPEND clause on a DEPOBJ construct must not have SOURCE, SINK or DEPOBJ as dependence-type
+  !$omp depobj(obj) depend(source)
+end
+
+subroutine f01
+  integer :: obj
+  integer :: x, y
+!ERROR: A DEPEND clause on a DEPOBJ construct must only specify one locator
+  !$omp depobj(obj) depend(in: x, y)
+end
+
+subroutine f02
+  integer :: obj
+  integer :: x(10)
+!WARNING: An iterator-modifier may specify multiple locators, a DEPEND clause on a DEPOBJ construct must only specify one locator
+  !$omp depobj(obj) depend(iterator(i = 1:10), in: x(i))
+end
+
+subroutine f03
+  integer :: obj, jbo
+!ERROR: The DESTROY clause must refer to the same object as the DEPOBJ construct
+!PORTABILITY: The object parameter in DESTROY clause in DEPOPJ construct was introduced in OpenMP v5.2
+  !$omp depobj(obj) destroy(jbo)
+end
diff --git a/flang/test/Semantics/OpenMP/depobj-construct-v51.f90 b/flang/test/Semantics/OpenMP/depobj-construct-v51.f90
new file mode 100644
index 000000000000..fa0c025a1101
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/depobj-construct-v51.f90
@@ -0,0 +1,13 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=51
+
+subroutine f04
+  integer :: obj
+!ERROR: An UPDATE clause on a DEPOBJ construct must not have SOURCE, SINK or DEPOBJ as dependence-type
+  !$omp depobj(obj) update(source)
+end
+
+subroutine f05
+  integer :: obj
+!ERROR: An UPDATE clause on a DEPOBJ construct must not have SOURCE, SINK or DEPOBJ as dependence-type
+  !$omp depobj(obj) update(depobj)
+end
diff --git a/flang/test/Semantics/OpenMP/depobj-construct-v52.f90 b/flang/test/Semantics/OpenMP/depobj-construct-v52.f90
new file mode 100644
index 000000000000..f2e66485c6c8
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/depobj-construct-v52.f90
@@ -0,0 +1,15 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
+
+subroutine f00
+  integer :: obj
+!WARNING: The SOURCE task-dependence-type is deprecated in OpenMP v5.2
+!ERROR: A DEPEND clause on a DEPOBJ construct must not have SOURCE or SINK as dependence-type
+  !$omp depobj(obj) depend(source)
+end
+
+subroutine f03
+  integer :: obj, jbo
+!Note: no portability message
+!ERROR: The DESTROY clause must refer to the same object as the DEPOBJ construct
+  !$omp depobj(obj) destroy(jbo)
+end
diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
index 2a890905dc63..8ff15b51f1ab 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -239,7 +239,8 @@ struct MapperT {
 ENUM(MemoryOrder, AcqRel, Acquire, Relaxed, Release, SeqCst);
 ENUM(MotionExpectation, Present);
 // V5.2: [15.9.1] `task-dependence-type` modifier
-ENUM(TaskDependenceType, In, Out, Inout, Mutexinoutset, Inoutset, Depobj);
+ENUM(TaskDependenceType, Depobj, In, Inout, Inoutset, Mutexinoutset, Out, Sink,
+     Source);
 
 template <typename I, typename E> //
 struct LoopIterationT {
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 70179bab4757..97496d4aae5a 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -130,6 +130,8 @@ def OMPC_Depobj : Clause<"depobj"> {
 }
 def OMPC_Destroy : Clause<"destroy"> {
   let clangClass = "OMPDestroyClause";
+  let flangClass = "OmpDestroyClause";
+  let isValueOptional = true;
 }
 def OMPC_Detach : Clause<"detach"> {
   let clangClass = "OMPDetachClause";
@@ -481,6 +483,7 @@ def OMPC_Untied : Clause<"untied"> {
 }
 def OMPC_Update : Clause<"update"> {
   let clangClass = "OMPUpdateClause";
+  let flangClass = "OmpUpdateClause";
 }
 def OMPC_Use : Clause<"use"> {
   let clangClass = "OMPUseClause";
-- 
GitLab


From 80c8ecd56586bbdb03ffdd93c94ba78306affd93 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 30 Oct 2024 13:32:45 +0000
Subject: [PATCH 579/693] [VectorCombine] Add baseline "shuffle (binop
 (shuffle, shuffle)), undef" tests for #114101

---
 .../VectorCombine/X86/permute-of-binops.ll    | 146 ++++++++++++++++++
 1 file changed, 146 insertions(+)
 create mode 100644 llvm/test/Transforms/VectorCombine/X86/permute-of-binops.ll

diff --git a/llvm/test/Transforms/VectorCombine/X86/permute-of-binops.ll b/llvm/test/Transforms/VectorCombine/X86/permute-of-binops.ll
new file mode 100644
index 000000000000..e94868c7b9e5
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/permute-of-binops.ll
@@ -0,0 +1,146 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=sse2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=avx2 | FileCheck %s --check-prefixes=CHECK,AVX
+
+; Fold "shuffle (binop (shuffle, shuffle)), undef" --> "binop (shuffle), (shuffle)"
+
+declare void @use_v4f64(<4 x double>)
+
+define <4 x double> @fadd_v4f64(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: define <4 x double> @fadd_v4f64(
+; CHECK-SAME: <4 x double> [[A:%.*]], <4 x double> [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B]], <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+; CHECK-NEXT:    [[POST:%.*]] = fadd <4 x double> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[POST1:%.*]] = shufflevector <4 x double> [[POST]], <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:    ret <4 x double> [[POST1]]
+;
+  %a1 = shufflevector <4 x double> %a, <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %b1 = shufflevector <4 x double> %b, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+  %op = fadd <4 x double> %a1, %b1
+  %post = shufflevector <4 x double> %op, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+  ret <4 x double> %post
+}
+
+define <4 x double> @fadd_v4f64_poison_idx(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: define <4 x double> @fadd_v4f64_poison_idx(
+; CHECK-SAME: <4 x double> [[A:%.*]], <4 x double> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B]], <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+; CHECK-NEXT:    [[POST:%.*]] = fadd <4 x double> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[POST1:%.*]] = shufflevector <4 x double> [[POST]], <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 4>
+; CHECK-NEXT:    ret <4 x double> [[POST1]]
+;
+  %a1 = shufflevector <4 x double> %a, <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %b1 = shufflevector <4 x double> %b, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+  %op = fadd <4 x double> %a1, %b1
+  %post = shufflevector <4 x double> %op, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 4>
+  ret <4 x double> %post
+}
+
+define <4 x double> @fadd_mixed_types(<4 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: define <4 x double> @fadd_mixed_types(
+; CHECK-SAME: <4 x double> [[A:%.*]], <2 x double> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[B]], <2 x double> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+; CHECK-NEXT:    [[POST:%.*]] = fadd <4 x double> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[POST1:%.*]] = shufflevector <4 x double> [[POST]], <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:    ret <4 x double> [[POST1]]
+;
+  %a1 = shufflevector <4 x double> %a, <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %b1 = shufflevector <2 x double> %b, <2 x double> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+  %op = fadd <4 x double> %a1, %b1
+  %post = shufflevector <4 x double> %op, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+  ret <4 x double> %post
+}
+
+; Negative test - multiple use of fadd
+define <4 x double> @fadd_v4f64_multiuse_op(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: define <4 x double> @fadd_v4f64_multiuse_op(
+; CHECK-SAME: <4 x double> [[A:%.*]], <4 x double> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[A1:%.*]] = shufflevector <4 x double> [[A]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[B1:%.*]] = shufflevector <4 x double> [[B]], <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+; CHECK-NEXT:    [[OP:%.*]] = fadd <4 x double> [[A1]], [[B1]]
+; CHECK-NEXT:    [[POST:%.*]] = shufflevector <4 x double> [[OP]], <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:    call void @use_v4f64(<4 x double> [[OP]])
+; CHECK-NEXT:    ret <4 x double> [[POST]]
+;
+  %a1 = shufflevector <4 x double> %a, <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %b1 = shufflevector <4 x double> %b, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+  %op = fadd <4 x double> %a1, %b1
+  %post = shufflevector <4 x double> %op, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+  call void @use_v4f64(<4 x double> %op)
+  ret <4 x double> %post
+}
+
+; Negative test - multiple use of inner shuffle
+define <4 x double> @fadd_v4f64_multiuse_shuffle(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: define <4 x double> @fadd_v4f64_multiuse_shuffle(
+; CHECK-SAME: <4 x double> [[A:%.*]], <4 x double> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[A1:%.*]] = shufflevector <4 x double> [[A]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[B1:%.*]] = shufflevector <4 x double> [[B]], <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+; CHECK-NEXT:    [[OP:%.*]] = fadd <4 x double> [[A1]], [[B1]]
+; CHECK-NEXT:    [[POST:%.*]] = shufflevector <4 x double> [[OP]], <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:    call void @use_v4f64(<4 x double> [[A1]])
+; CHECK-NEXT:    ret <4 x double> [[POST]]
+;
+  %a1 = shufflevector <4 x double> %a, <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %b1 = shufflevector <4 x double> %b, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+  %op = fadd <4 x double> %a1, %b1
+  %post = shufflevector <4 x double> %op, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+  call void @use_v4f64(<4 x double> %a1)
+  ret <4 x double> %post
+}
+
+define <4 x i32> @sdiv_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: define <4 x i32> @sdiv_v4i32(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+; CHECK-NEXT:    [[POST:%.*]] = sdiv <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[POST1:%.*]] = shufflevector <4 x i32> [[POST]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[POST1]]
+;
+  %a1 = shufflevector <4 x i32> %a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %b1 = shufflevector <4 x i32> %b, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+  %op = sdiv <4 x i32> %a1, %b1
+  %post = shufflevector <4 x i32> %op, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 0>
+  ret <4 x i32> %post
+}
+
+; Negative test - don't introduce poison element into div/rem instruction
+define <4 x i32> @sdiv_v4i32_poison(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: define <4 x i32> @sdiv_v4i32_poison(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[A1:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[B1:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+; CHECK-NEXT:    [[OP:%.*]] = sdiv <4 x i32> [[A1]], [[B1]]
+; CHECK-NEXT:    [[POST:%.*]] = shufflevector <4 x i32> [[OP]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 poison>
+; CHECK-NEXT:    ret <4 x i32> [[POST]]
+;
+  %a1 = shufflevector <4 x i32> %a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %b1 = shufflevector <4 x i32> %b, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+  %op = sdiv <4 x i32> %a1, %b1
+  %post = shufflevector <4 x i32> %op, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 poison>
+  ret <4 x i32> %post
+}
+
+; Negative test - don't introduce poison element into div/rem instruction
+define <4 x i32> @sdiv_v4i32_poison_idx(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: define <4 x i32> @sdiv_v4i32_poison_idx(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[A1:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[B1:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+; CHECK-NEXT:    [[OP:%.*]] = sdiv <4 x i32> [[A1]], [[B1]]
+; CHECK-NEXT:    [[POST:%.*]] = shufflevector <4 x i32> [[OP]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 4>
+; CHECK-NEXT:    ret <4 x i32> [[POST]]
+;
+  %a1 = shufflevector <4 x i32> %a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %b1 = shufflevector <4 x i32> %b, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+  %op = sdiv <4 x i32> %a1, %b1
+  %post = shufflevector <4 x i32> %op, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 4>
+  ret <4 x i32> %post
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX: {{.*}}
+; SSE: {{.*}}
-- 
GitLab


From 7f498a865fd946bb92e592b31b41509073306ab3 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood@arm.com>
Date: Wed, 30 Oct 2024 13:50:02 +0000
Subject: [PATCH 580/693] [CostModel][LoopVectorize] Move some loop vectoriser
 tests (#113702)

Many tests that were in test/Analysis/CostModel were actually
loop vectoriser tests. I've moved them as follows:

Analysis/CostModel/X86 -> Transforms/LoopVectorize/X86/CostModel
Analysis/CostModel/AArch64/arith-fp-frem.ll ->
  Transforms/LoopVectorize/AArch64/arith-fp-frem-costs.ll
---
 .../LoopVectorize/AArch64/arith-fp-frem-costs.ll}                 | 0
 .../LoopVectorize/X86/CostModel}/gather-i16-with-i8-index.ll      | 0
 .../LoopVectorize/X86/CostModel}/gather-i32-with-i8-index.ll      | 0
 .../LoopVectorize/X86/CostModel}/gather-i64-with-i8-index.ll      | 0
 .../LoopVectorize/X86/CostModel}/gather-i8-with-i8-index.ll       | 0
 .../X86/CostModel}/handle-iptr-with-data-layout-to-not-assert.ll  | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-f32-stride-2.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-f32-stride-3.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-f32-stride-4.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-f32-stride-5.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-f32-stride-6.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-f32-stride-7.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-f32-stride-8.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-f64-stride-2.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-f64-stride-3.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-f64-stride-4.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-f64-stride-5.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-f64-stride-6.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-f64-stride-7.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-f64-stride-8.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-half.ll         | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i16-stride-2.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i16-stride-3.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i16-stride-4.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i16-stride-5.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i16-stride-6.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i16-stride-7.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i16-stride-8.ll | 0
 .../X86/CostModel}/interleaved-load-i32-stride-2-indices-0u.ll    | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-2.ll | 0
 .../X86/CostModel}/interleaved-load-i32-stride-3-indices-01u.ll   | 0
 .../X86/CostModel}/interleaved-load-i32-stride-3-indices-0uu.ll   | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-3.ll | 0
 .../X86/CostModel}/interleaved-load-i32-stride-4-indices-012u.ll  | 0
 .../X86/CostModel}/interleaved-load-i32-stride-4-indices-01uu.ll  | 0
 .../X86/CostModel}/interleaved-load-i32-stride-4-indices-0uuu.ll  | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-4.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-5.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-6.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-7.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-8.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i64-stride-2.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i64-stride-3.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i64-stride-4.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i64-stride-5.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i64-stride-6.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i64-stride-7.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i64-stride-8.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i8-stride-2.ll  | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i8-stride-3.ll  | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i8-stride-4.ll  | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i8-stride-5.ll  | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i8-stride-6.ll  | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i8-stride-7.ll  | 0
 .../LoopVectorize/X86/CostModel}/interleaved-load-i8-stride-8.ll  | 0
 .../X86/CostModel}/interleaved-store-f32-stride-2.ll              | 0
 .../X86/CostModel}/interleaved-store-f32-stride-3.ll              | 0
 .../X86/CostModel}/interleaved-store-f32-stride-4.ll              | 0
 .../X86/CostModel}/interleaved-store-f32-stride-5.ll              | 0
 .../X86/CostModel}/interleaved-store-f32-stride-6.ll              | 0
 .../X86/CostModel}/interleaved-store-f32-stride-7.ll              | 0
 .../X86/CostModel}/interleaved-store-f32-stride-8.ll              | 0
 .../X86/CostModel}/interleaved-store-f64-stride-2.ll              | 0
 .../X86/CostModel}/interleaved-store-f64-stride-3.ll              | 0
 .../X86/CostModel}/interleaved-store-f64-stride-4.ll              | 0
 .../X86/CostModel}/interleaved-store-f64-stride-5.ll              | 0
 .../X86/CostModel}/interleaved-store-f64-stride-6.ll              | 0
 .../X86/CostModel}/interleaved-store-f64-stride-7.ll              | 0
 .../X86/CostModel}/interleaved-store-f64-stride-8.ll              | 0
 .../X86/CostModel}/interleaved-store-i16-stride-2.ll              | 0
 .../X86/CostModel}/interleaved-store-i16-stride-3.ll              | 0
 .../X86/CostModel}/interleaved-store-i16-stride-4.ll              | 0
 .../X86/CostModel}/interleaved-store-i16-stride-5.ll              | 0
 .../X86/CostModel}/interleaved-store-i16-stride-6.ll              | 0
 .../X86/CostModel}/interleaved-store-i16-stride-7.ll              | 0
 .../X86/CostModel}/interleaved-store-i16-stride-8.ll              | 0
 .../X86/CostModel}/interleaved-store-i32-stride-2.ll              | 0
 .../X86/CostModel}/interleaved-store-i32-stride-3.ll              | 0
 .../X86/CostModel}/interleaved-store-i32-stride-4.ll              | 0
 .../X86/CostModel}/interleaved-store-i32-stride-5.ll              | 0
 .../X86/CostModel}/interleaved-store-i32-stride-6.ll              | 0
 .../X86/CostModel}/interleaved-store-i32-stride-7.ll              | 0
 .../X86/CostModel}/interleaved-store-i32-stride-8.ll              | 0
 .../X86/CostModel}/interleaved-store-i64-stride-2.ll              | 0
 .../X86/CostModel}/interleaved-store-i64-stride-3.ll              | 0
 .../X86/CostModel}/interleaved-store-i64-stride-4.ll              | 0
 .../X86/CostModel}/interleaved-store-i64-stride-5.ll              | 0
 .../X86/CostModel}/interleaved-store-i64-stride-6.ll              | 0
 .../X86/CostModel}/interleaved-store-i64-stride-7.ll              | 0
 .../X86/CostModel}/interleaved-store-i64-stride-8.ll              | 0
 .../LoopVectorize/X86/CostModel}/interleaved-store-i8-stride-2.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-store-i8-stride-3.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-store-i8-stride-4.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-store-i8-stride-5.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-store-i8-stride-6.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-store-i8-stride-7.ll | 0
 .../LoopVectorize/X86/CostModel}/interleaved-store-i8-stride-8.ll | 0
 .../X86/CostModel}/masked-gather-i32-with-i8-index.ll             | 0
 .../X86/CostModel}/masked-gather-i64-with-i8-index.ll             | 0
 .../LoopVectorize/X86/CostModel}/masked-interleaved-load-i16.ll   | 0
 .../LoopVectorize/X86/CostModel}/masked-interleaved-store-i16.ll  | 0
 .../LoopVectorize/X86/CostModel}/masked-load-i16.ll               | 0
 .../LoopVectorize/X86/CostModel}/masked-load-i32.ll               | 0
 .../LoopVectorize/X86/CostModel}/masked-load-i64.ll               | 0
 .../LoopVectorize/X86/CostModel}/masked-load-i8.ll                | 0
 .../X86/CostModel}/masked-scatter-i32-with-i8-index.ll            | 0
 .../X86/CostModel}/masked-scatter-i64-with-i8-index.ll            | 0
 .../LoopVectorize/X86/CostModel}/masked-store-i16.ll              | 0
 .../LoopVectorize/X86/CostModel}/masked-store-i32.ll              | 0
 .../LoopVectorize/X86/CostModel}/masked-store-i64.ll              | 0
 .../LoopVectorize/X86/CostModel}/masked-store-i8.ll               | 0
 .../LoopVectorize/X86/CostModel}/scatter-i16-with-i8-index.ll     | 0
 .../LoopVectorize/X86/CostModel}/scatter-i32-with-i8-index.ll     | 0
 .../LoopVectorize/X86/CostModel}/scatter-i64-with-i8-index.ll     | 0
 .../LoopVectorize/X86/CostModel}/scatter-i8-with-i8-index.ll      | 0
 .../LoopVectorize/X86/CostModel}/strided-load-i16.ll              | 0
 .../LoopVectorize/X86/CostModel}/strided-load-i32.ll              | 0
 .../LoopVectorize/X86/CostModel}/strided-load-i64.ll              | 0
 .../LoopVectorize/X86/CostModel}/strided-load-i8.ll               | 0
 119 files changed, 0 insertions(+), 0 deletions(-)
 rename llvm/test/{Analysis/CostModel/AArch64/arith-fp-frem.ll => Transforms/LoopVectorize/AArch64/arith-fp-frem-costs.ll} (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/gather-i16-with-i8-index.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/gather-i32-with-i8-index.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/gather-i64-with-i8-index.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/gather-i8-with-i8-index.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/handle-iptr-with-data-layout-to-not-assert.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-f32-stride-2.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-f32-stride-3.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-f32-stride-4.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-f32-stride-5.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-f32-stride-6.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-f32-stride-7.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-f32-stride-8.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-f64-stride-2.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-f64-stride-3.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-f64-stride-4.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-f64-stride-5.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-f64-stride-6.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-f64-stride-7.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-f64-stride-8.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-half.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i16-stride-2.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i16-stride-3.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i16-stride-4.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i16-stride-5.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i16-stride-6.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i16-stride-7.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i16-stride-8.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-2-indices-0u.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-2.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-3-indices-01u.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-3-indices-0uu.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-3.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-4-indices-012u.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-4-indices-01uu.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-4-indices-0uuu.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-4.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-5.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-6.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-7.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i32-stride-8.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i64-stride-2.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i64-stride-3.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i64-stride-4.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i64-stride-5.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i64-stride-6.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i64-stride-7.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i64-stride-8.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i8-stride-2.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i8-stride-3.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i8-stride-4.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i8-stride-5.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i8-stride-6.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i8-stride-7.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-load-i8-stride-8.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-f32-stride-2.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-f32-stride-3.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-f32-stride-4.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-f32-stride-5.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-f32-stride-6.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-f32-stride-7.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-f32-stride-8.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-f64-stride-2.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-f64-stride-3.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-f64-stride-4.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-f64-stride-5.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-f64-stride-6.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-f64-stride-7.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-f64-stride-8.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i16-stride-2.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i16-stride-3.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i16-stride-4.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i16-stride-5.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i16-stride-6.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i16-stride-7.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i16-stride-8.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i32-stride-2.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i32-stride-3.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i32-stride-4.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i32-stride-5.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i32-stride-6.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i32-stride-7.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i32-stride-8.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i64-stride-2.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i64-stride-3.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i64-stride-4.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i64-stride-5.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i64-stride-6.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i64-stride-7.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i64-stride-8.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i8-stride-2.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i8-stride-3.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i8-stride-4.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i8-stride-5.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i8-stride-6.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i8-stride-7.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/interleaved-store-i8-stride-8.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/masked-gather-i32-with-i8-index.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/masked-gather-i64-with-i8-index.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/masked-interleaved-load-i16.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/masked-interleaved-store-i16.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/masked-load-i16.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/masked-load-i32.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/masked-load-i64.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/masked-load-i8.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/masked-scatter-i32-with-i8-index.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/masked-scatter-i64-with-i8-index.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/masked-store-i16.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/masked-store-i32.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/masked-store-i64.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/masked-store-i8.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/scatter-i16-with-i8-index.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/scatter-i32-with-i8-index.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/scatter-i64-with-i8-index.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/scatter-i8-with-i8-index.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/strided-load-i16.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/strided-load-i32.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/strided-load-i64.ll (100%)
 rename llvm/test/{Analysis/CostModel/X86 => Transforms/LoopVectorize/X86/CostModel}/strided-load-i8.ll (100%)

diff --git a/llvm/test/Analysis/CostModel/AArch64/arith-fp-frem.ll b/llvm/test/Transforms/LoopVectorize/AArch64/arith-fp-frem-costs.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/AArch64/arith-fp-frem.ll
rename to llvm/test/Transforms/LoopVectorize/AArch64/arith-fp-frem-costs.ll
diff --git a/llvm/test/Analysis/CostModel/X86/gather-i16-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i16-with-i8-index.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/gather-i16-with-i8-index.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i16-with-i8-index.ll
diff --git a/llvm/test/Analysis/CostModel/X86/gather-i32-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i32-with-i8-index.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/gather-i32-with-i8-index.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i32-with-i8-index.ll
diff --git a/llvm/test/Analysis/CostModel/X86/gather-i64-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i64-with-i8-index.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/gather-i64-with-i8-index.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i64-with-i8-index.ll
diff --git a/llvm/test/Analysis/CostModel/X86/gather-i8-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i8-with-i8-index.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/gather-i8-with-i8-index.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i8-with-i8-index.ll
diff --git a/llvm/test/Analysis/CostModel/X86/handle-iptr-with-data-layout-to-not-assert.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/handle-iptr-with-data-layout-to-not-assert.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/handle-iptr-with-data-layout-to-not-assert.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/handle-iptr-with-data-layout-to-not-assert.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-f32-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-2.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-f32-stride-2.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-2.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-f32-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-3.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-f32-stride-3.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-3.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-f32-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-4.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-f32-stride-4.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-4.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-f32-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-5.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-f32-stride-5.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-5.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-f32-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-6.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-f32-stride-6.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-6.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-f32-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-7.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-f32-stride-7.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-7.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-f32-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-8.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-f32-stride-8.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-8.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-2.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-2.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-2.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-3.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-3.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-3.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-4.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-4.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-4.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-5.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-5.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-5.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-6.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-6.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-6.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-7.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-7.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-7.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-8.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-8.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-8.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-half.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-half.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-half.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-half.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-2.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-2.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-2.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-3.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-3.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-3.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-4.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-4.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-4.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-5.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-5.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-5.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-6.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-6.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-6.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-7.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-7.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-7.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-8.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-8.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-8.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-2-indices-0u.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-2-indices-0u.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-2-indices-0u.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-2-indices-0u.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-2.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-2.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-2.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-3-indices-01u.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-01u.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-3-indices-01u.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-01u.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-3-indices-0uu.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-0uu.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-3-indices-0uu.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-0uu.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-3.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-4-indices-012u.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-012u.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-4-indices-012u.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-012u.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-4-indices-01uu.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-01uu.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-4-indices-01uu.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-01uu.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-4-indices-0uuu.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-0uuu.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-4-indices-0uuu.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4-indices-0uuu.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-4.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-4.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-5.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-5.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-5.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-6.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-6.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-6.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-7.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-7.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-7.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-8.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i32-stride-8.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-8.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-2.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-2.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-2.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-3.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-3.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-3.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-4.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-4.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-4.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-5.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-5.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-5.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-6.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-6.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-6.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-7.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-7.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-7.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-8.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-8.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-8.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i8-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-2.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i8-stride-2.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-2.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i8-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-3.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i8-stride-3.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-3.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i8-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-4.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i8-stride-4.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-4.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i8-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-5.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i8-stride-5.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-5.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i8-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-6.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i8-stride-6.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-6.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i8-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-7.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i8-stride-7.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-7.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i8-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-8.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-load-i8-stride-8.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-8.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-2.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-2.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-2.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-3.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-3.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-3.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-4.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-4.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-4.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-5.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-5.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-5.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-6.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-6.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-6.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-7.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-7.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-7.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-8.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-8.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-8.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-2.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-2.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-2.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-3.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-3.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-3.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-4.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-4.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-4.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-5.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-5.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-5.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-6.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-6.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-6.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-7.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-7.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-7.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-8.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-8.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-8.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-2.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-2.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-2.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-3.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-3.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-3.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-4.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-4.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-4.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-5.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-5.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-5.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-6.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-6.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-6.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-7.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-7.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-7.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-8.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-8.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-8.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-2.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-2.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-2.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-3.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-3.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-3.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-4.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-4.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-4.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-5.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-5.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-5.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-6.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-6.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-6.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-7.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-7.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-7.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-8.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-8.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-8.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-2.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-2.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-2.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-3.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-3.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-3.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-4.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-4.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-4.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-5.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-5.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-5.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-6.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-6.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-6.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-7.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-7.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-7.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-8.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-8.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-8.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-2.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-2.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-2.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-2.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-3.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-3.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-3.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-3.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-4.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-4.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-4.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-4.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-5.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-5.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-5.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-5.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-6.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-6.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-6.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-6.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-7.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-7.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-7.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-7.ll
diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-8.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-8.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-8.ll
diff --git a/llvm/test/Analysis/CostModel/X86/masked-gather-i32-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i32-with-i8-index.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/masked-gather-i32-with-i8-index.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i32-with-i8-index.ll
diff --git a/llvm/test/Analysis/CostModel/X86/masked-gather-i64-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i64-with-i8-index.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/masked-gather-i64-with-i8-index.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i64-with-i8-index.ll
diff --git a/llvm/test/Analysis/CostModel/X86/masked-interleaved-load-i16.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-load-i16.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/masked-interleaved-load-i16.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-load-i16.ll
diff --git a/llvm/test/Analysis/CostModel/X86/masked-interleaved-store-i16.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-store-i16.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/masked-interleaved-store-i16.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-store-i16.ll
diff --git a/llvm/test/Analysis/CostModel/X86/masked-load-i16.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i16.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/masked-load-i16.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i16.ll
diff --git a/llvm/test/Analysis/CostModel/X86/masked-load-i32.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i32.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/masked-load-i32.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i32.ll
diff --git a/llvm/test/Analysis/CostModel/X86/masked-load-i64.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i64.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/masked-load-i64.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i64.ll
diff --git a/llvm/test/Analysis/CostModel/X86/masked-load-i8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i8.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/masked-load-i8.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i8.ll
diff --git a/llvm/test/Analysis/CostModel/X86/masked-scatter-i32-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i32-with-i8-index.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/masked-scatter-i32-with-i8-index.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i32-with-i8-index.ll
diff --git a/llvm/test/Analysis/CostModel/X86/masked-scatter-i64-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i64-with-i8-index.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/masked-scatter-i64-with-i8-index.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i64-with-i8-index.ll
diff --git a/llvm/test/Analysis/CostModel/X86/masked-store-i16.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i16.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/masked-store-i16.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i16.ll
diff --git a/llvm/test/Analysis/CostModel/X86/masked-store-i32.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i32.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/masked-store-i32.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i32.ll
diff --git a/llvm/test/Analysis/CostModel/X86/masked-store-i64.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i64.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/masked-store-i64.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i64.ll
diff --git a/llvm/test/Analysis/CostModel/X86/masked-store-i8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i8.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/masked-store-i8.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i8.ll
diff --git a/llvm/test/Analysis/CostModel/X86/scatter-i16-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i16-with-i8-index.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/scatter-i16-with-i8-index.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i16-with-i8-index.ll
diff --git a/llvm/test/Analysis/CostModel/X86/scatter-i32-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i32-with-i8-index.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/scatter-i32-with-i8-index.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i32-with-i8-index.ll
diff --git a/llvm/test/Analysis/CostModel/X86/scatter-i64-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i64-with-i8-index.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/scatter-i64-with-i8-index.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i64-with-i8-index.ll
diff --git a/llvm/test/Analysis/CostModel/X86/scatter-i8-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i8-with-i8-index.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/scatter-i8-with-i8-index.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i8-with-i8-index.ll
diff --git a/llvm/test/Analysis/CostModel/X86/strided-load-i16.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i16.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/strided-load-i16.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i16.ll
diff --git a/llvm/test/Analysis/CostModel/X86/strided-load-i32.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i32.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/strided-load-i32.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i32.ll
diff --git a/llvm/test/Analysis/CostModel/X86/strided-load-i64.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i64.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/strided-load-i64.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i64.ll
diff --git a/llvm/test/Analysis/CostModel/X86/strided-load-i8.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i8.ll
similarity index 100%
rename from llvm/test/Analysis/CostModel/X86/strided-load-i8.ll
rename to llvm/test/Transforms/LoopVectorize/X86/CostModel/strided-load-i8.ll
-- 
GitLab


From 8d406d882d49204cf76ed6406610c702db6e3e8e Mon Sep 17 00:00:00 2001
From: vdonaldson <37090318+vdonaldson@users.noreply.github.com>
Date: Wed, 30 Oct 2024 09:56:42 -0400
Subject: [PATCH 581/693] [flang] IEEE_REAL (#113948)

IEEE_REAL converts an integer or real argument to a real of a given
kind.
---
 .../flang/Optimizer/Builder/IntrinsicCall.h   |   1 +
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 235 +++++++++++++++++-
 flang/test/Lower/Intrinsics/ieee_real.f90     | 217 ++++++++++++++++
 3 files changed, 451 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Lower/Intrinsics/ieee_real.f90

diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index 868a8b4e2874..f5fb272b4cc3 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -289,6 +289,7 @@ struct IntrinsicLibrary {
   template <mlir::arith::CmpFPredicate pred>
   mlir::Value genIeeeQuietCompare(mlir::Type resultType,
                                   llvm::ArrayRef<mlir::Value>);
+  mlir::Value genIeeeReal(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genIeeeRint(mlir::Type, llvm::ArrayRef<mlir::Value>);
   template <bool isFlag>
   void genIeeeSetFlagOrHaltingMode(llvm::ArrayRef<fir::ExtendedValue>);
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index 462193a850c4..7c7c8ee53911 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -97,7 +97,6 @@ static bool isStaticallyPresent(const fir::ExtendedValue &exv) {
 
 /// IEEE module procedure names not yet implemented for genModuleProcTODO.
 static constexpr char ieee_get_underflow_mode[] = "ieee_get_underflow_mode";
-static constexpr char ieee_real[] = "ieee_real";
 static constexpr char ieee_rem[] = "ieee_rem";
 static constexpr char ieee_set_underflow_mode[] = "ieee_set_underflow_mode";
 
@@ -362,7 +361,7 @@ static constexpr IntrinsicHandler handlers[]{
     {"ieee_quiet_le", &I::genIeeeQuietCompare<mlir::arith::CmpFPredicate::OLE>},
     {"ieee_quiet_lt", &I::genIeeeQuietCompare<mlir::arith::CmpFPredicate::OLT>},
     {"ieee_quiet_ne", &I::genIeeeQuietCompare<mlir::arith::CmpFPredicate::UNE>},
-    {"ieee_real", &I::genModuleProcTODO<ieee_real>},
+    {"ieee_real", &I::genIeeeReal},
     {"ieee_rem", &I::genModuleProcTODO<ieee_rem>},
     {"ieee_rint", &I::genIeeeRint},
     {"ieee_round_eq", &I::genIeeeTypeCompare<mlir::arith::CmpIPredicate::eq>},
@@ -4799,6 +4798,238 @@ IntrinsicLibrary::genIeeeQuietCompare(mlir::Type resultType,
   return builder.create<fir::ConvertOp>(loc, resultType, res);
 }
 
+// IEEE_REAL
+mlir::Value IntrinsicLibrary::genIeeeReal(mlir::Type resultType,
+                                          llvm::ArrayRef<mlir::Value> args) {
+  // Convert integer or real argument A to a real of a specified kind.
+  // Round according to the current rounding mode.
+  // Signal IEEE_INVALID if A is an sNaN, and return a qNaN.
+  // Signal IEEE_UNDERFLOW for an inexact subnormal or zero result.
+  // Signal IEEE_OVERFLOW if A is finite and the result is infinite.
+  // Signal IEEE_INEXACT for an inexact result.
+  //
+  // if (type(a) == resultType) {
+  //   // Conversion to the same type is a nop except for sNaN processing.
+  //   result = a
+  // } else {
+  //   result = r = real(a, kind(result))
+  //   // Conversion to a larger type is exact.
+  //   if (c_sizeof(a) >= c_sizeof(r)) {
+  //     b = (a is integer) ? int(r, kind(a)) : real(r, kind(a))
+  //     if (a == b || isNaN(a)) {
+  //       // a is {-0, +0, -inf, +inf, NaN} or exact; result is r
+  //     } else {
+  //       // odd(r) is true if the low bit of significand(r) is 1
+  //       // rounding mode ieee_other is an alias for mode ieee_nearest
+  //       if (a < b) {
+  //         if (mode == ieee_nearest && odd(r)) result = ieee_next_down(r)
+  //         if (mode == ieee_other   && odd(r)) result = ieee_next_down(r)
+  //         if (mode == ieee_to_zero && a > 0)  result = ieee_next_down(r)
+  //         if (mode == ieee_away    && a < 0)  result = ieee_next_down(r)
+  //         if (mode == ieee_down)              result = ieee_next_down(r)
+  //       } else { // a > b
+  //         if (mode == ieee_nearest && odd(r)) result = ieee_next_up(r)
+  //         if (mode == ieee_other   && odd(r)) result = ieee_next_up(r)
+  //         if (mode == ieee_to_zero && a < 0)  result = ieee_next_up(r)
+  //         if (mode == ieee_away    && a > 0)  result = ieee_next_up(r)
+  //         if (mode == ieee_up)                result = ieee_next_up(r)
+  //       }
+  //     }
+  //   }
+  // }
+
+  assert(args.size() == 2);
+  mlir::Type i1Ty = builder.getI1Type();
+  mlir::Type f32Ty = mlir::FloatType::getF32(builder.getContext());
+  mlir::Value a = args[0];
+  mlir::Type aType = a.getType();
+
+  // If the argument is an sNaN, raise an invalid exception and return a qNaN.
+  // Otherwise return the argument.
+  auto processSnan = [&](mlir::Value x) {
+    fir::IfOp ifOp = builder.create<fir::IfOp>(loc, resultType,
+                                               genIsFPClass(i1Ty, x, snanTest),
+                                               /*withElseRegion=*/true);
+    builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+    genRaiseExcept(_FORTRAN_RUNTIME_IEEE_INVALID);
+    builder.create<fir::ResultOp>(loc, genQNan(resultType));
+    builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+    builder.create<fir::ResultOp>(loc, x);
+    builder.setInsertionPointAfter(ifOp);
+    return ifOp.getResult(0);
+  };
+
+  // Conversion is a nop, except that A may be an sNaN.
+  if (resultType == aType)
+    return processSnan(a);
+
+  // Can't directly convert between kind=2 and kind=3.
+  mlir::Value r, r1;
+  if ((aType.isBF16() && resultType.isF16()) ||
+      (aType.isF16() && resultType.isBF16())) {
+    a = builder.createConvert(loc, f32Ty, a);
+    aType = f32Ty;
+  }
+  r = builder.create<fir::ConvertOp>(loc, resultType, a);
+
+  mlir::IntegerType aIntType = mlir::dyn_cast<mlir::IntegerType>(aType);
+  mlir::FloatType aFloatType = mlir::dyn_cast<mlir::FloatType>(aType);
+  mlir::FloatType resultFloatType = mlir::dyn_cast<mlir::FloatType>(resultType);
+
+  // Conversion from a smaller type to a larger type is exact.
+  if ((aIntType ? aIntType.getWidth() : aFloatType.getWidth()) <
+      resultFloatType.getWidth())
+    return aIntType ? r : processSnan(r);
+
+  // A possibly inexact conversion result may need to be rounded up or down.
+  mlir::Value b = builder.create<fir::ConvertOp>(loc, aType, r);
+  mlir::Value aEqB;
+  if (aIntType)
+    aEqB = builder.create<mlir::arith::CmpIOp>(
+        loc, mlir::arith::CmpIPredicate::eq, a, b);
+  else
+    aEqB = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::UEQ, a, b);
+
+  // [a == b] a is a NaN or r is exact (a may be -0, +0, -inf, +inf) -- return r
+  fir::IfOp ifOp1 = builder.create<fir::IfOp>(loc, resultType, aEqB,
+                                              /*withElseRegion=*/true);
+  builder.setInsertionPointToStart(&ifOp1.getThenRegion().front());
+  builder.create<fir::ResultOp>(loc, aIntType ? r : processSnan(r));
+
+  // Code common to (a < b) and (a > b) branches.
+  builder.setInsertionPointToStart(&ifOp1.getElseRegion().front());
+  mlir::func::FuncOp getRound = fir::factory::getLlvmGetRounding(builder);
+  mlir::Value mode = builder.create<fir::CallOp>(loc, getRound).getResult(0);
+  mlir::Value aIsNegative, aIsPositive;
+  if (aIntType) {
+    mlir::Value zero = builder.createIntegerConstant(loc, aIntType, 0);
+    aIsNegative = builder.create<mlir::arith::CmpIOp>(
+        loc, mlir::arith::CmpIPredicate::slt, a, zero);
+    aIsPositive = builder.create<mlir::arith::CmpIOp>(
+        loc, mlir::arith::CmpIPredicate::sgt, a, zero);
+  } else {
+    mlir::Value zero = builder.createRealZeroConstant(loc, aFloatType);
+    aIsNegative = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::OLT, a, zero);
+    aIsPositive = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::OGT, a, zero);
+  }
+  mlir::Type resultIntType = builder.getIntegerType(resultFloatType.getWidth());
+  mlir::Value resultCast =
+      builder.create<mlir::arith::BitcastOp>(loc, resultIntType, r);
+  mlir::Value one = builder.createIntegerConstant(loc, resultIntType, 1);
+  mlir::Value rIsOdd = builder.create<fir::ConvertOp>(
+      loc, i1Ty, builder.create<mlir::arith::AndIOp>(loc, resultCast, one));
+  // Check for a rounding mode match.
+  auto match = [&](int m) {
+    return builder.create<mlir::arith::CmpIOp>(
+        loc, mlir::arith::CmpIPredicate::eq, mode,
+        builder.createIntegerConstant(loc, mode.getType(), m));
+  };
+  mlir::Value roundToNearestBit = builder.create<mlir::arith::OrIOp>(
+      loc,
+      // IEEE_OTHER is an alias for IEEE_NEAREST.
+      match(_FORTRAN_RUNTIME_IEEE_NEAREST), match(_FORTRAN_RUNTIME_IEEE_OTHER));
+  mlir::Value roundToNearest =
+      builder.create<mlir::arith::AndIOp>(loc, roundToNearestBit, rIsOdd);
+  mlir::Value roundToZeroBit = match(_FORTRAN_RUNTIME_IEEE_TO_ZERO);
+  mlir::Value roundAwayBit = match(_FORTRAN_RUNTIME_IEEE_AWAY);
+  mlir::Value roundToZero, roundAway, mustAdjust;
+  fir::IfOp adjustIfOp;
+  mlir::Value aLtB;
+  if (aIntType)
+    aLtB = builder.create<mlir::arith::CmpIOp>(
+        loc, mlir::arith::CmpIPredicate::slt, a, b);
+  else
+    aLtB = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::OLT, a, b);
+  mlir::Value upResult =
+      builder.create<mlir::arith::AddIOp>(loc, resultCast, one);
+  mlir::Value downResult =
+      builder.create<mlir::arith::SubIOp>(loc, resultCast, one);
+
+  // (a < b): r is inexact -- return r or ieee_next_down(r)
+  fir::IfOp ifOp2 = builder.create<fir::IfOp>(loc, resultType, aLtB,
+                                              /*withElseRegion=*/true);
+  builder.setInsertionPointToStart(&ifOp2.getThenRegion().front());
+  roundToZero =
+      builder.create<mlir::arith::AndIOp>(loc, roundToZeroBit, aIsPositive);
+  roundAway =
+      builder.create<mlir::arith::AndIOp>(loc, roundAwayBit, aIsNegative);
+  mlir::Value roundDown = match(_FORTRAN_RUNTIME_IEEE_DOWN);
+  mustAdjust =
+      builder.create<mlir::arith::OrIOp>(loc, roundToNearest, roundToZero);
+  mustAdjust = builder.create<mlir::arith::OrIOp>(loc, mustAdjust, roundAway);
+  mustAdjust = builder.create<mlir::arith::OrIOp>(loc, mustAdjust, roundDown);
+  adjustIfOp = builder.create<fir::IfOp>(loc, resultType, mustAdjust,
+                                         /*withElseRegion=*/true);
+  builder.setInsertionPointToStart(&adjustIfOp.getThenRegion().front());
+  if (resultType.isF80())
+    r1 = fir::runtime::genNearest(builder, loc, r,
+                                  builder.createBool(loc, false));
+  else
+    r1 = builder.create<mlir::arith::BitcastOp>(
+        loc, resultType,
+        builder.create<mlir::arith::SelectOp>(loc, aIsNegative, upResult,
+                                              downResult));
+  builder.create<fir::ResultOp>(loc, r1);
+  builder.setInsertionPointToStart(&adjustIfOp.getElseRegion().front());
+  builder.create<fir::ResultOp>(loc, r);
+  builder.setInsertionPointAfter(adjustIfOp);
+  builder.create<fir::ResultOp>(loc, adjustIfOp.getResult(0));
+
+  // (a > b): r is inexact -- return r or ieee_next_up(r)
+  builder.setInsertionPointToStart(&ifOp2.getElseRegion().front());
+  roundToZero =
+      builder.create<mlir::arith::AndIOp>(loc, roundToZeroBit, aIsNegative);
+  roundAway =
+      builder.create<mlir::arith::AndIOp>(loc, roundAwayBit, aIsPositive);
+  mlir::Value roundUp = match(_FORTRAN_RUNTIME_IEEE_UP);
+  mustAdjust =
+      builder.create<mlir::arith::OrIOp>(loc, roundToNearest, roundToZero);
+  mustAdjust = builder.create<mlir::arith::OrIOp>(loc, mustAdjust, roundAway);
+  mustAdjust = builder.create<mlir::arith::OrIOp>(loc, mustAdjust, roundUp);
+  adjustIfOp = builder.create<fir::IfOp>(loc, resultType, mustAdjust,
+                                         /*withElseRegion=*/true);
+  builder.setInsertionPointToStart(&adjustIfOp.getThenRegion().front());
+  if (resultType.isF80())
+    r1 = fir::runtime::genNearest(builder, loc, r,
+                                  builder.createBool(loc, true));
+  else
+    r1 = builder.create<mlir::arith::BitcastOp>(
+        loc, resultType,
+        builder.create<mlir::arith::SelectOp>(loc, aIsPositive, upResult,
+                                              downResult));
+  builder.create<fir::ResultOp>(loc, r1);
+  builder.setInsertionPointToStart(&adjustIfOp.getElseRegion().front());
+  builder.create<fir::ResultOp>(loc, r);
+  builder.setInsertionPointAfter(adjustIfOp);
+  builder.create<fir::ResultOp>(loc, adjustIfOp.getResult(0));
+
+  // Generate exceptions for (a < b) and (a > b) branches.
+  builder.setInsertionPointAfter(ifOp2);
+  r = ifOp2.getResult(0);
+  fir::IfOp exceptIfOp1 = builder.create<fir::IfOp>(
+      loc, genIsFPClass(i1Ty, r, infiniteTest), /*withElseRegion=*/true);
+  builder.setInsertionPointToStart(&exceptIfOp1.getThenRegion().front());
+  genRaiseExcept(_FORTRAN_RUNTIME_IEEE_OVERFLOW |
+                 _FORTRAN_RUNTIME_IEEE_INEXACT);
+  builder.setInsertionPointToStart(&exceptIfOp1.getElseRegion().front());
+  fir::IfOp exceptIfOp2 = builder.create<fir::IfOp>(
+      loc, genIsFPClass(i1Ty, r, subnormalTest | zeroTest),
+      /*withElseRegion=*/true);
+  builder.setInsertionPointToStart(&exceptIfOp2.getThenRegion().front());
+  genRaiseExcept(_FORTRAN_RUNTIME_IEEE_UNDERFLOW |
+                 _FORTRAN_RUNTIME_IEEE_INEXACT);
+  builder.setInsertionPointToStart(&exceptIfOp2.getElseRegion().front());
+  genRaiseExcept(_FORTRAN_RUNTIME_IEEE_INEXACT);
+  builder.setInsertionPointAfter(exceptIfOp1);
+  builder.create<fir::ResultOp>(loc, ifOp2.getResult(0));
+  builder.setInsertionPointAfter(ifOp1);
+  return ifOp1.getResult(0);
+}
+
 // IEEE_RINT
 mlir::Value IntrinsicLibrary::genIeeeRint(mlir::Type resultType,
                                           llvm::ArrayRef<mlir::Value> args) {
diff --git a/flang/test/Lower/Intrinsics/ieee_real.f90 b/flang/test/Lower/Intrinsics/ieee_real.f90
new file mode 100644
index 000000000000..20b7441e6e3a
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/ieee_real.f90
@@ -0,0 +1,217 @@
+! RUN: bbc -emit-hlfir -o - %s | FileCheck %s
+
+! CHECK-LABEL: c.func @_QQmain
+program p
+  use ieee_arithmetic, only: ieee_real
+
+  ! CHECK:     %[[V_0:[0-9]+]] = fir.alloca i16 {bindc_name = "j2", uniq_name = "_QFEj2"}
+  ! CHECK:     %[[V_1:[0-9]+]]:2 = hlfir.declare %[[V_0]] {uniq_name = "_QFEj2"} : (!fir.ref<i16>) -> (!fir.ref<i16>, !fir.ref<i16>)
+  ! CHECK:     %[[V_2:[0-9]+]] = fir.alloca i64 {bindc_name = "j8", uniq_name = "_QFEj8"}
+  ! CHECK:     %[[V_3:[0-9]+]]:2 = hlfir.declare %[[V_2]] {uniq_name = "_QFEj8"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
+  ! CHECK:     %[[V_4:[0-9]+]] = fir.alloca f16 {bindc_name = "x2", uniq_name = "_QFEx2"}
+  ! CHECK:     %[[V_5:[0-9]+]]:2 = hlfir.declare %[[V_4]] {uniq_name = "_QFEx2"} : (!fir.ref<f16>) -> (!fir.ref<f16>, !fir.ref<f16>)
+  ! CHECK:     %[[V_6:[0-9]+]] = fir.alloca f32 {bindc_name = "x4", uniq_name = "_QFEx4"}
+  ! CHECK:     %[[V_7:[0-9]+]]:2 = hlfir.declare %[[V_6]] {uniq_name = "_QFEx4"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  ! CHECK:     %[[V_8:[0-9]+]] = fir.alloca f64 {bindc_name = "x8", uniq_name = "_QFEx8"}
+  ! CHECK:     %[[V_9:[0-9]+]]:2 = hlfir.declare %[[V_8]] {uniq_name = "_QFEx8"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
+  integer(2) :: j2
+  integer(8) :: j8
+  real(2) ::  x2
+  real(4) ::  x4
+  real(8) ::  x8
+
+  ! CHECK:     hlfir.assign %c-32768{{.*}} to %[[V_1]]#0 : i16, !fir.ref<i16>
+  j2 = -huge(j2) - 1
+
+  ! CHECK:     %[[V_10:[0-9]+]] = fir.load %[[V_1]]#0 : !fir.ref<i16>
+  ! CHECK:     %[[V_11:[0-9]+]] = fir.convert %[[V_10]] : (i16) -> f32
+  ! CHECK:     hlfir.assign %[[V_11]] to %[[V_7]]#0 : f32, !fir.ref<f32>
+  x4 = ieee_real(j2,4) ! exact
+! print*, j2, ' -> ', x4
+
+  ! CHECK:     hlfir.assign %c33{{.*}} to %[[V_3]]#0 : i64, !fir.ref<i64>
+  j8 = 33
+
+  ! CHECK:     %[[V_12:[0-9]+]] = fir.load %[[V_3]]#0 : !fir.ref<i64>
+  ! CHECK:     %[[V_13:[0-9]+]] = fir.convert %[[V_12]] : (i64) -> f32
+  ! CHECK:     %[[V_14:[0-9]+]] = fir.convert %[[V_13]] : (f32) -> i64
+  ! CHECK:     %[[V_15:[0-9]+]] = arith.cmpi eq, %[[V_12]], %[[V_14]] : i64
+  ! CHECK:     %[[V_16:[0-9]+]] = fir.if %[[V_15]] -> (f32) {
+  ! CHECK:       fir.result %[[V_13]] : f32
+  ! CHECK:     } else {
+  ! CHECK:       %[[V_27:[0-9]+]] = fir.call @llvm.get.rounding() fastmath<contract> : () -> i32
+  ! CHECK-DAG:   %[[V_28:[0-9]+]] = arith.cmpi slt, %[[V_12]], %c0{{.*}} : i64
+  ! CHECK-DAG:   %[[V_29:[0-9]+]] = arith.cmpi sgt, %[[V_12]], %c0{{.*}} : i64
+  ! CHECK-DAG:   %[[V_30:[0-9]+]] = arith.bitcast %[[V_13]] : f32 to i32
+  ! CHECK-DAG:   %[[V_31:[0-9]+]] = arith.andi %[[V_30]], %c1{{.*}} : i32
+  ! CHECK-DAG:   %[[V_32:[0-9]+]] = fir.convert %[[V_31]] : (i32) -> i1
+  ! CHECK-DAG:   %[[V_33:[0-9]+]] = arith.cmpi eq, %[[V_27]], %c5{{.*}} : i32
+  ! CHECK-DAG:   %[[V_34:[0-9]+]] = arith.cmpi eq, %[[V_27]], %c1{{.*}} : i32
+  ! CHECK-DAG:   %[[V_35:[0-9]+]] = arith.ori %[[V_34]], %[[V_33]] : i1
+  ! CHECK-DAG:   %[[V_36:[0-9]+]] = arith.andi %[[V_35]], %[[V_32]] : i1
+  ! CHECK-DAG:   %[[V_37:[0-9]+]] = arith.cmpi eq, %[[V_27]], %c0{{.*}} : i32
+  ! CHECK-DAG:   %[[V_38:[0-9]+]] = arith.cmpi eq, %[[V_27]], %c4{{.*}} : i32
+  ! CHECK-DAG:   %[[V_39:[0-9]+]] = arith.cmpi slt, %[[V_12]], %[[V_14]] : i64
+  ! CHECK-DAG:   %[[V_40:[0-9]+]] = arith.addi %[[V_30]], %c1{{.*}} : i32
+  ! CHECK-DAG:   %[[V_41:[0-9]+]] = arith.subi %[[V_30]], %c1{{.*}} : i32
+  ! CHECK:       %[[V_42:[0-9]+]] = fir.if %[[V_39]] -> (f32) {
+  ! CHECK-DAG:     %[[V_44:[0-9]+]] = arith.andi %[[V_37]], %[[V_29]] : i1
+  ! CHECK-DAG:     %[[V_45:[0-9]+]] = arith.andi %[[V_38]], %[[V_28]] : i1
+  ! CHECK-DAG:     %[[V_46:[0-9]+]] = arith.cmpi eq, %[[V_27]], %c3{{.*}} : i32
+  ! CHECK-DAG:     %[[V_47:[0-9]+]] = arith.ori %[[V_36]], %[[V_44]] : i1
+  ! CHECK-DAG:     %[[V_48:[0-9]+]] = arith.ori %[[V_47]], %[[V_45]] : i1
+  ! CHECK-DAG:     %[[V_49:[0-9]+]] = arith.ori %[[V_48]], %[[V_46]] : i1
+  ! CHECK:         %[[V_50:[0-9]+]] = fir.if %[[V_49]] -> (f32) {
+  ! CHECK:           %[[V_51:[0-9]+]] = arith.select %[[V_28]], %[[V_40]], %[[V_41]] : i32
+  ! CHECK:           %[[V_52:[0-9]+]] = arith.bitcast %[[V_51]] : i32 to f32
+  ! CHECK:           fir.result %[[V_52]] : f32
+  ! CHECK:         } else {
+  ! CHECK:           fir.result %[[V_13]] : f32
+  ! CHECK:         }
+  ! CHECK:         fir.result %[[V_50]] : f32
+  ! CHECK:       } else {
+  ! CHECK-DAG:     %[[V_44:[0-9]+]] = arith.andi %[[V_37]], %[[V_28]] : i1
+  ! CHECK-DAG:     %[[V_45:[0-9]+]] = arith.andi %[[V_38]], %[[V_29]] : i1
+  ! CHECK-DAG:     %[[V_46:[0-9]+]] = arith.cmpi eq, %[[V_27]], %c2{{.*}} : i32
+  ! CHECK-DAG:     %[[V_47:[0-9]+]] = arith.ori %[[V_36]], %[[V_44]] : i1
+  ! CHECK-DAG:     %[[V_48:[0-9]+]] = arith.ori %[[V_47]], %[[V_45]] : i1
+  ! CHECK-DAG:     %[[V_49:[0-9]+]] = arith.ori %[[V_48]], %[[V_46]] : i1
+  ! CHECK:         %[[V_50:[0-9]+]] = fir.if %[[V_49]] -> (f32) {
+  ! CHECK:           %[[V_51:[0-9]+]] = arith.select %[[V_29]], %[[V_40]], %[[V_41]] : i32
+  ! CHECK:           %[[V_52:[0-9]+]] = arith.bitcast %[[V_51]] : i32 to f32
+  ! CHECK:           fir.result %[[V_52]] : f32
+  ! CHECK:         } else {
+  ! CHECK:           fir.result %[[V_13]] : f32
+  ! CHECK:         }
+  ! CHECK:         fir.result %[[V_50]] : f32
+  ! CHECK:       }
+  ! CHECK:       %[[V_43:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_42]]) <{bit = 516 : i32}> : (f32) -> i1
+  ! CHECK:       fir.if %[[V_43]] {
+  ! CHECK:         %[[V_44:[0-9]+]] = fir.call @_FortranAMapException(%c40{{.*}}) fastmath<contract> : (i32) -> i32
+  ! CHECK:         %[[V_45:[0-9]+]] = fir.call @feraiseexcept(%[[V_44]]) fastmath<contract> : (i32) -> i32
+  ! CHECK:       } else {
+  ! CHECK:         %[[V_44:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_42]]) <{bit = 240 : i32}> : (f32) -> i1
+  ! CHECK:         fir.if %[[V_44]] {
+  ! CHECK:           %[[V_45:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
+  ! CHECK:           %[[V_46:[0-9]+]] = fir.call @feraiseexcept(%[[V_45]]) fastmath<contract> : (i32) -> i32
+  ! CHECK:         } else {
+  ! CHECK:           %[[V_45:[0-9]+]] = fir.call @_FortranAMapException(%c32{{.*}}) fastmath<contract> : (i32) -> i32
+  ! CHECK:           %[[V_46:[0-9]+]] = fir.call @feraiseexcept(%[[V_45]]) fastmath<contract> : (i32) -> i32
+  ! CHECK:         }
+  ! CHECK:       }
+  ! CHECK:       fir.result %[[V_42]] : f32
+  ! CHECK:     }
+  ! CHECK:     hlfir.assign %[[V_16]] to %[[V_7]]#0 : f32, !fir.ref<f32>
+  x4 = ieee_real(j8,4)
+! print*, j8, ' -> ', x4
+
+  ! CHECK:     hlfir.assign %cst{{[_0-9]*}} to %[[V_5]]#0 : f16, !fir.ref<f16>
+  x2 = 3.33
+
+  ! CHECK:     %[[V_17:[0-9]+]] = fir.load %[[V_5]]#0 : !fir.ref<f16>
+  ! CHECK:     %[[V_18:[0-9]+]] = fir.convert %[[V_17]] : (f16) -> f32
+  ! CHECK:     %[[V_19:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_18]]) <{bit = 1 : i32}> : (f32) -> i1
+  ! CHECK:     %[[V_20:[0-9]+]] = fir.if %[[V_19]] -> (f32) {
+  ! CHECK:       %[[V_27:[0-9]+]] = fir.call @_FortranAMapException(%c1{{.*}}) fastmath<contract> : (i32) -> i32
+  ! CHECK:       %[[V_28:[0-9]+]] = fir.call @feraiseexcept(%[[V_27]]) fastmath<contract> : (i32) -> i32
+  ! CHECK:       %[[V_29:[0-9]+]] = fir.address_of(@_FortranAIeeeValueTable_4) : !fir.ref<!fir.array<12xi32>>
+  ! CHECK:       %[[V_30:[0-9]+]] = fir.coordinate_of %[[V_29]], %c2{{.*}} : (!fir.ref<!fir.array<12xi32>>, i8) -> !fir.ref<i32>
+  ! CHECK:       %[[V_31:[0-9]+]] = fir.load %[[V_30]] : !fir.ref<i32>
+  ! CHECK:       %[[V_32:[0-9]+]] = arith.bitcast %[[V_31]] : i32 to f32
+  ! CHECK:       fir.result %[[V_32]] : f32
+  ! CHECK:     } else {
+  ! CHECK:       fir.result %[[V_18]] : f32
+  ! CHECK:     }
+  ! CHECK:     %[[V_21:[0-9]+]] = fir.convert %[[V_20]] : (f32) -> f16
+  ! CHECK:     hlfir.assign %[[V_21]] to %[[V_5]]#0 : f16, !fir.ref<f16>
+  x2 = ieee_real(x2,4) ! exact
+! print*, x2, ' -> ', x2
+
+  ! CHECK:     hlfir.assign %cst{{[_0-9]*}} to %[[V_9]]#0 : f64, !fir.ref<f64>
+  x8 = -0.
+
+  ! CHECK:     %[[V_22:[0-9]+]] = fir.load %[[V_9]]#0 : !fir.ref<f64>
+  ! CHECK:     %[[V_23:[0-9]+]] = fir.convert %[[V_22]] : (f64) -> f32
+  ! CHECK:     %[[V_24:[0-9]+]] = fir.convert %[[V_23]] : (f32) -> f64
+  ! CHECK:     %[[V_25:[0-9]+]] = arith.cmpf ueq, %[[V_22]], %[[V_24]] fastmath<contract> : f64
+  ! CHECK:     %[[V_26:[0-9]+]] = fir.if %[[V_25]] -> (f32) {
+  ! CHECK:       %[[V_27:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_23]]) <{bit = 1 : i32}> : (f32) -> i1
+  ! CHECK:       %[[V_28:[0-9]+]] = fir.if %[[V_27]] -> (f32) {
+  ! CHECK:         %[[V_29:[0-9]+]] = fir.call @_FortranAMapException(%c1{{.*}}) fastmath<contract> : (i32) -> i32
+  ! CHECK:         %[[V_30:[0-9]+]] = fir.call @feraiseexcept(%[[V_29]]) fastmath<contract> : (i32) -> i32
+  ! CHECK:         %[[V_31:[0-9]+]] = fir.address_of(@_FortranAIeeeValueTable_4) : !fir.ref<!fir.array<12xi32>>
+  ! CHECK:         %[[V_32:[0-9]+]] = fir.coordinate_of %[[V_31]], %c2{{.*}} : (!fir.ref<!fir.array<12xi32>>, i8) -> !fir.ref<i32>
+  ! CHECK:         %[[V_33:[0-9]+]] = fir.load %[[V_32]] : !fir.ref<i32>
+  ! CHECK:         %[[V_34:[0-9]+]] = arith.bitcast %[[V_33]] : i32 to f32
+  ! CHECK:         fir.result %[[V_34]] : f32
+  ! CHECK:       } else {
+  ! CHECK:         fir.result %[[V_23]] : f32
+  ! CHECK:       }
+  ! CHECK:       fir.result %[[V_28]] : f32
+  ! CHECK:     } else {
+  ! CHECK-DAG:   %[[V_27:[0-9]+]] = fir.call @llvm.get.rounding() fastmath<contract> : () -> i32
+  ! CHECK-DAG:   %[[V_28:[0-9]+]] = arith.cmpf olt, %[[V_22]], %cst{{[_0-9]*}} fastmath<contract> : f64
+  ! CHECK-DAG:   %[[V_29:[0-9]+]] = arith.cmpf ogt, %[[V_22]], %cst{{[_0-9]*}} fastmath<contract> : f64
+  ! CHECK-DAG:   %[[V_30:[0-9]+]] = arith.bitcast %[[V_23]] : f32 to i32
+  ! CHECK-DAG:   %[[V_31:[0-9]+]] = arith.andi %[[V_30]], %c1{{.*}} : i32
+  ! CHECK-DAG:   %[[V_32:[0-9]+]] = fir.convert %[[V_31]] : (i32) -> i1
+  ! CHECK-DAG:   %[[V_33:[0-9]+]] = arith.cmpi eq, %[[V_27]], %c5{{.*}} : i32
+  ! CHECK-DAG:   %[[V_34:[0-9]+]] = arith.cmpi eq, %[[V_27]], %c1{{.*}} : i32
+  ! CHECK-DAG:   %[[V_35:[0-9]+]] = arith.ori %[[V_34]], %[[V_33]] : i1
+  ! CHECK-DAG:   %[[V_36:[0-9]+]] = arith.andi %[[V_35]], %[[V_32]] : i1
+  ! CHECK-DAG:   %[[V_37:[0-9]+]] = arith.cmpi eq, %[[V_27]], %c0{{.*}} : i32
+  ! CHECK-DAG:   %[[V_38:[0-9]+]] = arith.cmpi eq, %[[V_27]], %c4{{.*}} : i32
+  ! CHECK-DAG:   %[[V_39:[0-9]+]] = arith.cmpf olt, %[[V_22]], %[[V_24]] fastmath<contract> : f64
+  ! CHECK-DAG:   %[[V_40:[0-9]+]] = arith.addi %[[V_30]], %c1{{.*}} : i32
+  ! CHECK-DAG:   %[[V_41:[0-9]+]] = arith.subi %[[V_30]], %c1{{.*}} : i32
+  ! CHECK:       %[[V_42:[0-9]+]] = fir.if %[[V_39]] -> (f32) {
+  ! CHECK-DAG:     %[[V_44:[0-9]+]] = arith.andi %[[V_37]], %[[V_29]] : i1
+  ! CHECK-DAG:     %[[V_45:[0-9]+]] = arith.andi %[[V_38]], %[[V_28]] : i1
+  ! CHECK-DAG:     %[[V_46:[0-9]+]] = arith.cmpi eq, %[[V_27]], %c3{{.*}} : i32
+  ! CHECK-DAG:     %[[V_47:[0-9]+]] = arith.ori %[[V_36]], %[[V_44]] : i1
+  ! CHECK-DAG:     %[[V_48:[0-9]+]] = arith.ori %[[V_47]], %[[V_45]] : i1
+  ! CHECK-DAG:     %[[V_49:[0-9]+]] = arith.ori %[[V_48]], %[[V_46]] : i1
+  ! CHECK:         %[[V_50:[0-9]+]] = fir.if %[[V_49]] -> (f32) {
+  ! CHECK:           %[[V_51:[0-9]+]] = arith.select %[[V_28]], %[[V_40]], %[[V_41]] : i32
+  ! CHECK:           %[[V_52:[0-9]+]] = arith.bitcast %[[V_51]] : i32 to f32
+  ! CHECK:           fir.result %[[V_52]] : f32
+  ! CHECK:         } else {
+  ! CHECK:           fir.result %[[V_23]] : f32
+  ! CHECK:         }
+  ! CHECK:         fir.result %[[V_50]] : f32
+  ! CHECK:       } else {
+  ! CHECK-DAG:     %[[V_44:[0-9]+]] = arith.andi %[[V_37]], %[[V_28]] : i1
+  ! CHECK-DAG:     %[[V_45:[0-9]+]] = arith.andi %[[V_38]], %[[V_29]] : i1
+  ! CHECK-DAG:     %[[V_46:[0-9]+]] = arith.cmpi eq, %[[V_27]], %c2{{.*}} : i32
+  ! CHECK-DAG:     %[[V_47:[0-9]+]] = arith.ori %[[V_36]], %[[V_44]] : i1
+  ! CHECK-DAG:     %[[V_48:[0-9]+]] = arith.ori %[[V_47]], %[[V_45]] : i1
+  ! CHECK-DAG:     %[[V_49:[0-9]+]] = arith.ori %[[V_48]], %[[V_46]] : i1
+  ! CHECK:         %[[V_50:[0-9]+]] = fir.if %[[V_49]] -> (f32) {
+  ! CHECK:           %[[V_51:[0-9]+]] = arith.select %[[V_29]], %[[V_40]], %[[V_41]] : i32
+  ! CHECK:           %[[V_52:[0-9]+]] = arith.bitcast %[[V_51]] : i32 to f32
+  ! CHECK:           fir.result %[[V_52]] : f32
+  ! CHECK:         } else {
+  ! CHECK:           fir.result %[[V_23]] : f32
+  ! CHECK:         }
+  ! CHECK:         fir.result %[[V_50]] : f32
+  ! CHECK:       }
+  ! CHECK:       %[[V_43:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_42]]) <{bit = 516 : i32}> : (f32) -> i1
+  ! CHECK:       fir.if %[[V_43]] {
+  ! CHECK:         %[[V_44:[0-9]+]] = fir.call @_FortranAMapException(%c40{{.*}}) fastmath<contract> : (i32) -> i32
+  ! CHECK:         %[[V_45:[0-9]+]] = fir.call @feraiseexcept(%[[V_44]]) fastmath<contract> : (i32) -> i32
+  ! CHECK:       } else {
+  ! CHECK:         %[[V_44:[0-9]+]] = "llvm.intr.is.fpclass"(%[[V_42]]) <{bit = 240 : i32}> : (f32) -> i1
+  ! CHECK:         fir.if %[[V_44]] {
+  ! CHECK:           %[[V_45:[0-9]+]] = fir.call @_FortranAMapException(%c48{{.*}}) fastmath<contract> : (i32) -> i32
+  ! CHECK:           %[[V_46:[0-9]+]] = fir.call @feraiseexcept(%[[V_45]]) fastmath<contract> : (i32) -> i32
+  ! CHECK:         } else {
+  ! CHECK:           %[[V_45:[0-9]+]] = fir.call @_FortranAMapException(%c32{{.*}}) fastmath<contract> : (i32) -> i32
+  ! CHECK:           %[[V_46:[0-9]+]] = fir.call @feraiseexcept(%[[V_45]]) fastmath<contract> : (i32) -> i32
+  ! CHECK:         }
+  ! CHECK:       }
+  ! CHECK:       fir.result %[[V_42]] : f32
+  ! CHECK:     }
+  ! CHECK:     hlfir.assign %[[V_26]] to %[[V_7]]#0 : f32, !fir.ref<f32>
+  x4 = ieee_real(x8,4)
+! print*, x8, ' -> ', x4
+end
-- 
GitLab


From cba70550ccf55c6ad3daa621bb8caf3c4ca6cbd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= <brioche@google.com>
Date: Wed, 30 Oct 2024 14:57:32 +0100
Subject: [PATCH 582/693] [SPIR-V] Fix BB ordering & register lifetime
 (#111026)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The "topological" sorting was behaving incorrectly in some cases:
the exit of a loop could have a lower rank than a node in the loop.
This causes issues when structurizing some patterns, and also codegen
issues as we could generate BBs in the incorrect order in regard to the
SPIR-V spec.

Fixing this ordering alone broke other parts of the structurizer, which
by luck worked. Had to fix those.

Added more test cases, especially to test basic patterns.

I also needed to tweak/disable some tests for 2 reasons:
 - SPIR-V now required reg2mem/mem2reg to run. Meaning dead stores
   are optimized away. Some tests require tweaks to avoid having the
   whole function removed.
 - Mem2Reg will generate variable & load/stores. This generates
   G_BITCAST in several cases. And there is currently something wrong
   we do with G_BITCAST which causes MIR verifier to complain.
   Until this is resolved, I disabled -verify-machineinstrs flag on
   those tests.

---------

Signed-off-by: Nathan Gauër <brioche@google.com>
---
 .../SPIRV/SPIRVMergeRegionExitTargets.cpp     |  22 ++-
 llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp   | 131 ++++++-------
 llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp  |  13 +-
 llvm/lib/Target/SPIRV/SPIRVUtils.cpp          | 109 ++++++++---
 llvm/lib/Target/SPIRV/SPIRVUtils.h            |   8 +-
 llvm/test/CodeGen/SPIRV/HlslBufferLoad.ll     |  11 +-
 llvm/test/CodeGen/SPIRV/OpVariable_order.ll   |   4 +-
 llvm/test/CodeGen/SPIRV/ShaderBufferImage.ll  |   3 +-
 llvm/test/CodeGen/SPIRV/ShaderImage.ll        |   3 +-
 llvm/test/CodeGen/SPIRV/basic_float_types.ll  |   1 -
 llvm/test/CodeGen/SPIRV/basic_int_types.ll    |   4 +-
 .../CodeGen/SPIRV/basic_int_types_spirvdis.ll |   2 +-
 .../hlsl-intrinsics/SV_DispatchThreadID.ll    |   3 +-
 .../SPIRV/hlsl-intrinsics/WaveGetLaneIndex.ll |   3 +-
 .../test/CodeGen/SPIRV/hlsl-intrinsics/abs.ll |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/acos.ll     |   3 +-
 .../test/CodeGen/SPIRV/hlsl-intrinsics/all.ll |   5 +-
 .../test/CodeGen/SPIRV/hlsl-intrinsics/any.ll |   5 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/asin.ll     |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/atan.ll     |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/atan2.ll    |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/ceil.ll     |   3 +-
 .../test/CodeGen/SPIRV/hlsl-intrinsics/cos.ll |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/cosh.ll     |   3 +-
 .../SPIRV/hlsl-intrinsics/countbits.ll        |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/degrees.ll  |   3 +-
 .../test/CodeGen/SPIRV/hlsl-intrinsics/exp.ll |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/exp2.ll     |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/floor.ll    |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/fmad.ll     |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/fmax.ll     |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/fmin.ll     |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/frac.ll     |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/imad.ll     |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/lerp.ll     |   3 +-
 .../test/CodeGen/SPIRV/hlsl-intrinsics/log.ll |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/log10.ll    |  32 +--
 .../CodeGen/SPIRV/hlsl-intrinsics/log2.ll     |   3 +-
 .../SPIRV/hlsl-intrinsics/normalize.ll        |   1 +
 .../test/CodeGen/SPIRV/hlsl-intrinsics/pow.ll |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/radians.ll  |   3 +-
 .../test/CodeGen/SPIRV/hlsl-intrinsics/rcp.ll |   3 +-
 .../SPIRV/hlsl-intrinsics/reversebits.ll      |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/round.ll    |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/rsqrt.ll    |   3 +-
 .../test/CodeGen/SPIRV/hlsl-intrinsics/sin.ll |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/sinh.ll     |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/smax.ll     |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/smin.ll     |   3 +-
 .../SPIRV/hlsl-intrinsics/splitdouble.ll      |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/sqrt.ll     |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/step.ll     |   1 +
 .../test/CodeGen/SPIRV/hlsl-intrinsics/tan.ll |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/tanh.ll     |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/trunc.ll    |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/umax.ll     |   3 +-
 .../CodeGen/SPIRV/hlsl-intrinsics/umin.ll     |   3 +-
 llvm/test/CodeGen/SPIRV/literals.ll           |   4 +-
 .../CodeGen/SPIRV/structurizer/basic-if.ll    |  52 +++++
 .../SPIRV/structurizer/basic-imbalanced-if.ll |  47 +++++
 .../CodeGen/SPIRV/structurizer/basic-loop.ll  |  59 ++++++
 .../CodeGen/SPIRV/structurizer/basic-phi.ll   |  58 ++++++
 .../CodeGen/SPIRV/structurizer/cf.cond-op.ll  | 185 ++++++++----------
 .../CodeGen/SPIRV/structurizer/cf.do.break.ll |   1 -
 .../SPIRV/structurizer/cf.do.continue.ll      |   1 -
 .../SPIRV/structurizer/cf.do.nested.ll        |   1 -
 .../SPIRV/structurizer/cf.for.break.ll        |   1 -
 .../SPIRV/structurizer/cf.for.continue.ll     |   1 -
 .../SPIRV/structurizer/cf.for.nested.ll       |   1 -
 .../cf.for.short-circuited-cond.ll            |   5 +-
 .../SPIRV/structurizer/cf.if.const-cond.ll    |   1 -
 .../CodeGen/SPIRV/structurizer/cf.if.for.ll   |   1 -
 .../SPIRV/structurizer/cf.if.nested.ll        |   1 -
 .../CodeGen/SPIRV/structurizer/cf.if.plain.ll |   1 -
 .../SPIRV/structurizer/cf.logical-and.ll      |   1 -
 .../SPIRV/structurizer/cf.logical-or.ll       |   1 -
 .../SPIRV/structurizer/cf.return.early.ll     |   1 -
 .../SPIRV/structurizer/cf.switch.ifstmt.ll    |   1 -
 .../structurizer/cf.switch.ifstmt.simple.ll   |   1 -
 .../structurizer/cf.switch.ifstmt.simple2.ll  |   1 -
 .../SPIRV/structurizer/cf.while.break.ll      |  56 +++---
 .../SPIRV/structurizer/condition-linear.ll    | 142 +++++++-------
 .../CodeGen/SPIRV/structurizer/do-continue.ll | 156 ++++++++-------
 .../CodeGen/SPIRV/structurizer/do-nested.ll   | 122 ++++++------
 .../CodeGen/SPIRV/structurizer/do-plain.ll    | 124 ++++++------
 .../CodeGen/SPIRV/structurizer/logical-or.ll  | 115 +++++------
 .../SPIRV/structurizer/loop-continue-split.ll | 104 ++++++++++
 .../SPIRV/structurizer/merge-exit-break.ll    |  38 ++--
 .../merge-exit-convergence-in-break.ll        |  18 +-
 .../structurizer/merge-exit-multiple-break.ll |  35 ++--
 .../CodeGen/SPIRV/structurizer/phi-exit.ll    |  45 +++++
 .../SPIRV/structurizer/return-early.ll        |  20 +-
 92 files changed, 1143 insertions(+), 733 deletions(-)
 create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/basic-if.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/basic-imbalanced-if.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/basic-loop.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/basic-phi.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/loop-continue-split.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/phi-exit.ll

diff --git a/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp b/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp
index 9930d067173d..c22492ec43b0 100644
--- a/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp
@@ -130,6 +130,13 @@ public:
     assert(false && "Unhandled terminator type.");
   }
 
+  AllocaInst *CreateVariable(Function &F, Type *Type,
+                             BasicBlock::iterator Position) {
+    const DataLayout &DL = F.getDataLayout();
+    return new AllocaInst(Type, DL.getAllocaAddrSpace(), nullptr, "reg",
+                          Position);
+  }
+
   // Run the pass on the given convergence region, ignoring the sub-regions.
   // Returns true if the CFG changed, false otherwise.
   bool runOnConvergenceRegionNoRecurse(LoopInfo &LI,
@@ -152,6 +159,9 @@ public:
     auto NewExitTarget = BasicBlock::Create(F->getContext(), "new.exit", F);
     IRBuilder<> Builder(NewExitTarget);
 
+    AllocaInst *Variable = CreateVariable(*F, Builder.getInt32Ty(),
+                                          F->begin()->getFirstInsertionPt());
+
     // CodeGen output needs to be stable. Using the set as-is would order
     // the targets differently depending on the allocation pattern.
     // Sorting per basic-block ordering in the function.
@@ -176,18 +186,16 @@ public:
     std::vector<std::pair<BasicBlock *, Value *>> ExitToVariable;
     for (auto Exit : SortedExits) {
       llvm::Value *Value = createExitVariable(Exit, TargetToValue);
+      IRBuilder<> B2(Exit);
+      B2.SetInsertPoint(Exit->getFirstInsertionPt());
+      B2.CreateStore(Value, Variable);
       ExitToVariable.emplace_back(std::make_pair(Exit, Value));
     }
 
-    // Gather the correct value depending on the exit we came from.
-    llvm::PHINode *node =
-        Builder.CreatePHI(Builder.getInt32Ty(), ExitToVariable.size());
-    for (auto [BB, Value] : ExitToVariable) {
-      node->addIncoming(Value, BB);
-    }
+    llvm::Value *Load = Builder.CreateLoad(Builder.getInt32Ty(), Variable);
 
     // Creating the switch to jump to the correct exit target.
-    llvm::SwitchInst *Sw = Builder.CreateSwitch(node, SortedExitTargets[0],
+    llvm::SwitchInst *Sw = Builder.CreateSwitch(Load, SortedExitTargets[0],
                                                 SortedExitTargets.size() - 1);
     for (size_t i = 1; i < SortedExitTargets.size(); i++) {
       BasicBlock *BB = SortedExitTargets[i];
diff --git a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
index 211a060ee103..13e05b679275 100644
--- a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
@@ -87,7 +87,7 @@ BasicBlock *getExitFor(const ConvergenceRegion *CR) {
 // Returns the merge block designated by I if I is a merge instruction, nullptr
 // otherwise.
 BasicBlock *getDesignatedMergeBlock(Instruction *I) {
-  IntrinsicInst *II = dyn_cast<IntrinsicInst>(I);
+  IntrinsicInst *II = dyn_cast_or_null<IntrinsicInst>(I);
   if (II == nullptr)
     return nullptr;
 
@@ -102,7 +102,7 @@ BasicBlock *getDesignatedMergeBlock(Instruction *I) {
 // Returns the continue block designated by I if I is an OpLoopMerge, nullptr
 // otherwise.
 BasicBlock *getDesignatedContinueBlock(Instruction *I) {
-  IntrinsicInst *II = dyn_cast<IntrinsicInst>(I);
+  IntrinsicInst *II = dyn_cast_or_null<IntrinsicInst>(I);
   if (II == nullptr)
     return nullptr;
 
@@ -284,18 +284,6 @@ void replaceBranchTargets(BasicBlock *BB, BasicBlock *OldTarget,
   assert(false && "Unhandled terminator type.");
 }
 
-// Replaces basic bloc operands |OldSrc| or OpPhi instructions in |BB| by
-// |NewSrc|. This function does not simplify the OpPhi instruction once
-// transformed.
-void replacePhiTargets(BasicBlock *BB, BasicBlock *OldSrc, BasicBlock *NewSrc) {
-  for (PHINode &Phi : BB->phis()) {
-    int index = Phi.getBasicBlockIndex(OldSrc);
-    if (index == -1)
-      continue;
-    Phi.setIncomingBlock(index, NewSrc);
-  }
-}
-
 } // anonymous namespace
 
 // Given a reducible CFG, produces a structurized CFG in the SPIR-V sense,
@@ -423,7 +411,7 @@ class SPIRVStructurizer : public FunctionPass {
     }
 
     // Splits the given edges by recreating proxy nodes so that the destination
-    // OpPhi instruction can still be viable.
+    // has unique incoming edges from this region.
     //
     // clang-format off
     //
@@ -436,66 +424,58 @@ class SPIRVStructurizer : public FunctionPass {
     // A -> D -> C
     // B -> D -> C
     //
-    // But if C had a phi node, adding such proxy-block breaks it. In such case, we must add 1 new block per
-    // exit, and patchup the phi node:
+    // This is fine (assuming C has no PHI nodes), but requires handling the merge instruction here.
+    // By adding a proxy node, we create a regular divergent shape which can easily be regularized later on.
     // A -> D -> D1 -> C
     // B -> D -> D2 -> C
     //
-    // A, B, D belongs to the construct. D is the exit. D1 and D2 are empty, just used as
-    // source operands for C's phi node.
+    // A, B, D belongs to the construct. D is the exit. D1 and D2 are empty.
     //
     // clang-format on
     std::vector<Edge>
     createAliasBlocksForComplexEdges(std::vector<Edge> Edges) {
-      std::unordered_map<BasicBlock *, BasicBlock *> Seen;
+      std::unordered_set<BasicBlock *> Seen;
       std::vector<Edge> Output;
       Output.reserve(Edges.size());
 
       for (auto &[Src, Dst] : Edges) {
-        auto [iterator, inserted] = Seen.insert({Src, Dst});
-        if (inserted) {
-          Output.emplace_back(Src, Dst);
-          continue;
+        auto [Iterator, Inserted] = Seen.insert(Src);
+        if (!Inserted) {
+          // Src already a source node. Cannot have 2 edges from A to B.
+          // Creating alias source block.
+          BasicBlock *NewSrc = BasicBlock::Create(
+              F.getContext(), Src->getName() + ".new.src", &F);
+          replaceBranchTargets(Src, Dst, NewSrc);
+          IRBuilder<> Builder(NewSrc);
+          Builder.CreateBr(Dst);
+          Src = NewSrc;
         }
 
-        // The exact same edge was already seen. Ignoring.
-        if (iterator->second == Dst)
-          continue;
-
-        // The same Src block branches to 2 distinct blocks. This will be an
-        // issue for the generated OpPhi. Creating alias block.
-        BasicBlock *NewSrc =
-            BasicBlock::Create(F.getContext(), "new.exit.src", &F);
-        replaceBranchTargets(Src, Dst, NewSrc);
-        replacePhiTargets(Dst, Src, NewSrc);
-
-        IRBuilder<> Builder(NewSrc);
-        Builder.CreateBr(Dst);
-
-        Seen.emplace(NewSrc, Dst);
-        Output.emplace_back(NewSrc, Dst);
+        Output.emplace_back(Src, Dst);
       }
 
       return Output;
     }
 
+    AllocaInst *CreateVariable(Function &F, Type *Type,
+                               BasicBlock::iterator Position) {
+      const DataLayout &DL = F.getDataLayout();
+      return new AllocaInst(Type, DL.getAllocaAddrSpace(), nullptr, "reg",
+                            Position);
+    }
+
     // Given a construct defined by |Header|, and a list of exiting edges
     // |Edges|, creates a new single exit node, fixing up those edges.
     BasicBlock *createSingleExitNode(BasicBlock *Header,
                                      std::vector<Edge> &Edges) {
-      auto NewExit = BasicBlock::Create(F.getContext(), "new.exit", &F);
-      IRBuilder<> ExitBuilder(NewExit);
 
-      std::vector<BasicBlock *> Dsts;
-      std::unordered_map<BasicBlock *, ConstantInt *> DstToIndex;
-
-      // Given 2 edges: Src1 -> Dst, Src2 -> Dst:
-      // If Dst has an PHI node, and Src1 and Src2 are both operands, both Src1
-      // and Src2 cannot be hidden by NewExit. Create 2 new nodes: Alias1,
-      // Alias2 to which NewExit will branch before going to Dst. Then, patchup
-      // Dst PHI node to look for Alias1 and Alias2.
       std::vector<Edge> FixedEdges = createAliasBlocksForComplexEdges(Edges);
 
+      std::vector<BasicBlock *> Dsts;
+      std::unordered_map<BasicBlock *, ConstantInt *> DstToIndex;
+      auto NewExit = BasicBlock::Create(F.getContext(),
+                                        Header->getName() + ".new.exit", &F);
+      IRBuilder<> ExitBuilder(NewExit);
       for (auto &[Src, Dst] : FixedEdges) {
         if (DstToIndex.count(Dst) != 0)
           continue;
@@ -506,33 +486,34 @@ class SPIRVStructurizer : public FunctionPass {
       if (Dsts.size() == 1) {
         for (auto &[Src, Dst] : FixedEdges) {
           replaceBranchTargets(Src, Dst, NewExit);
-          replacePhiTargets(Dst, Src, NewExit);
         }
         ExitBuilder.CreateBr(Dsts[0]);
         return NewExit;
       }
 
-      PHINode *PhiNode =
-          ExitBuilder.CreatePHI(ExitBuilder.getInt32Ty(), FixedEdges.size());
-
+      AllocaInst *Variable = CreateVariable(F, ExitBuilder.getInt32Ty(),
+                                            F.begin()->getFirstInsertionPt());
       for (auto &[Src, Dst] : FixedEdges) {
-        PhiNode->addIncoming(DstToIndex[Dst], Src);
+        IRBuilder<> B2(Src);
+        B2.SetInsertPoint(Src->getFirstInsertionPt());
+        B2.CreateStore(DstToIndex[Dst], Variable);
         replaceBranchTargets(Src, Dst, NewExit);
-        replacePhiTargets(Dst, Src, NewExit);
       }
 
+      llvm::Value *Load =
+          ExitBuilder.CreateLoad(ExitBuilder.getInt32Ty(), Variable);
+
       // If we can avoid an OpSwitch, generate an OpBranch. Reason is some
       // OpBranch are allowed to exist without a new OpSelectionMerge if one of
       // the branch is the parent's merge node, while OpSwitches are not.
       if (Dsts.size() == 2) {
-        Value *Condition = ExitBuilder.CreateCmp(CmpInst::ICMP_EQ,
-                                                 DstToIndex[Dsts[0]], PhiNode);
+        Value *Condition =
+            ExitBuilder.CreateCmp(CmpInst::ICMP_EQ, DstToIndex[Dsts[0]], Load);
         ExitBuilder.CreateCondBr(Condition, Dsts[0], Dsts[1]);
         return NewExit;
       }
 
-      SwitchInst *Sw =
-          ExitBuilder.CreateSwitch(PhiNode, Dsts[0], Dsts.size() - 1);
+      SwitchInst *Sw = ExitBuilder.CreateSwitch(Load, Dsts[0], Dsts.size() - 1);
       for (auto It = Dsts.begin() + 1; It != Dsts.end(); ++It) {
         Sw->addCase(DstToIndex[*It], *It);
       }
@@ -576,7 +557,7 @@ class SPIRVStructurizer : public FunctionPass {
 
   // Creates a new basic block in F with a single OpUnreachable instruction.
   BasicBlock *CreateUnreachable(Function &F) {
-    BasicBlock *BB = BasicBlock::Create(F.getContext(), "new.exit", &F);
+    BasicBlock *BB = BasicBlock::Create(F.getContext(), "unreachable", &F);
     IRBuilder<> Builder(BB);
     Builder.CreateUnreachable();
     return BB;
@@ -1027,17 +1008,8 @@ class SPIRVStructurizer : public FunctionPass {
     return Modified;
   }
 
-  bool IsRequiredForPhiNode(BasicBlock *BB) {
-    for (BasicBlock *Successor : successors(BB)) {
-      for (PHINode &Phi : Successor->phis()) {
-        if (Phi.getBasicBlockIndex(BB) != -1)
-          return true;
-      }
-    }
-
-    return false;
-  }
-
+  // Removes blocks not contributing to any structured CFG. This assumes there
+  // is no PHI nodes.
   bool removeUselessBlocks(Function &F) {
     std::vector<BasicBlock *> ToRemove;
 
@@ -1054,9 +1026,6 @@ class SPIRVStructurizer : public FunctionPass {
       if (MergeBlocks.count(&BB) != 0 || ContinueBlocks.count(&BB) != 0)
         continue;
 
-      if (IsRequiredForPhiNode(&BB))
-        continue;
-
       if (BB.getUniqueSuccessor() == nullptr)
         continue;
 
@@ -1127,6 +1096,18 @@ class SPIRVStructurizer : public FunctionPass {
         continue;
 
       Modified = true;
+
+      if (Merge == nullptr) {
+        Merge = *successors(Header).begin();
+        IRBuilder<> Builder(Header);
+        Builder.SetInsertPoint(Header->getTerminator());
+
+        auto MergeAddress = BlockAddress::get(Merge->getParent(), Merge);
+        SmallVector<Value *, 1> Args = {MergeAddress};
+        Builder.CreateIntrinsic(Intrinsic::spv_selection_merge, {}, {Args});
+        continue;
+      }
+
       Instruction *SplitInstruction = Merge->getTerminator();
       if (isMergeInstruction(SplitInstruction->getPrevNode()))
         SplitInstruction = SplitInstruction->getPrevNode();
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
index e5384b2eb2c2..34854f31b3e3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
@@ -29,6 +29,7 @@
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Pass.h"
 #include "llvm/Target/TargetOptions.h"
+#include "llvm/Transforms/Scalar/Reg2Mem.h"
 #include "llvm/Transforms/Utils.h"
 #include <optional>
 
@@ -169,13 +170,21 @@ void SPIRVPassConfig::addIRPasses() {
     //  - loops have a single back-edge.
     addPass(createLoopSimplifyPass());
 
-    // 2. Merge the convergence region exit nodes into one. After this step,
+    // 2. Removes registers whose lifetime spans across basic blocks. Also
+    // removes phi nodes. This will greatly simplify the next steps.
+    addPass(createRegToMemWrapperPass());
+
+    // 3. Merge the convergence region exit nodes into one. After this step,
     // regions are single-entry, single-exit. This will help determine the
     // correct merge block.
     addPass(createSPIRVMergeRegionExitTargetsPass());
 
-    // 3. Structurize.
+    // 4. Structurize.
     addPass(createSPIRVStructurizerPass());
+
+    // 5. Reduce the amount of variables required by pushing some operations
+    // back to virtual registers.
+    addPass(createPromoteMemoryToRegisterPass());
   }
 
   addPass(createSPIRVRegularizerPass());
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index dff33b16b9cf..f9b361e163c9 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -460,53 +460,98 @@ PartialOrderingVisitor::getReachableFrom(BasicBlock *Start) {
   return Output;
 }
 
-size_t PartialOrderingVisitor::visit(BasicBlock *BB, size_t Rank) {
-  if (Visited.count(BB) != 0)
-    return Rank;
+bool PartialOrderingVisitor::CanBeVisited(BasicBlock *BB) const {
+  for (BasicBlock *P : predecessors(BB)) {
+    // Ignore back-edges.
+    if (DT.dominates(BB, P))
+      continue;
 
-  Loop *L = LI.getLoopFor(BB);
-  const bool isLoopHeader = LI.isLoopHeader(BB);
+    // One of the predecessor hasn't been visited. Not ready yet.
+    if (BlockToOrder.count(P) == 0)
+      return false;
 
-  if (BlockToOrder.count(BB) == 0) {
-    OrderInfo Info = {Rank, Visited.size()};
-    BlockToOrder.emplace(BB, Info);
-  } else {
-    BlockToOrder[BB].Rank = std::max(BlockToOrder[BB].Rank, Rank);
+    // If the block is a loop exit, the loop must be finished before
+    // we can continue.
+    Loop *L = LI.getLoopFor(P);
+    if (L == nullptr || L->contains(BB))
+      continue;
+
+    // SPIR-V requires a single back-edge. And the backend first
+    // step transforms loops into the simplified format. If we have
+    // more than 1 back-edge, something is wrong.
+    assert(L->getNumBackEdges() <= 1);
+
+    // If the loop has no latch, loop's rank won't matter, so we can
+    // proceed.
+    BasicBlock *Latch = L->getLoopLatch();
+    assert(Latch);
+    if (Latch == nullptr)
+      continue;
+
+    // The latch is not ready yet, let's wait.
+    if (BlockToOrder.count(Latch) == 0)
+      return false;
   }
 
-  for (BasicBlock *Predecessor : predecessors(BB)) {
-    if (isLoopHeader && L->contains(Predecessor)) {
+  return true;
+}
+
+size_t PartialOrderingVisitor::GetNodeRank(BasicBlock *BB) const {
+  size_t result = 0;
+
+  for (BasicBlock *P : predecessors(BB)) {
+    // Ignore back-edges.
+    if (DT.dominates(BB, P))
       continue;
-    }
 
-    if (BlockToOrder.count(Predecessor) == 0) {
-      return Rank;
+    auto Iterator = BlockToOrder.end();
+    Loop *L = LI.getLoopFor(P);
+    BasicBlock *Latch = L ? L->getLoopLatch() : nullptr;
+
+    // If the predecessor is either outside a loop, or part of
+    // the same loop, simply take its rank + 1.
+    if (L == nullptr || L->contains(BB) || Latch == nullptr) {
+      Iterator = BlockToOrder.find(P);
+    } else {
+      // Otherwise, take the loop's rank (highest rank in the loop) as base.
+      // Since loops have a single latch, highest rank is easy to find.
+      // If the loop has no latch, then it doesn't matter.
+      Iterator = BlockToOrder.find(Latch);
     }
+
+    assert(Iterator != BlockToOrder.end());
+    result = std::max(result, Iterator->second.Rank + 1);
   }
 
-  Visited.insert(BB);
+  return result;
+}
+
+size_t PartialOrderingVisitor::visit(BasicBlock *BB, size_t Unused) {
+  ToVisit.push(BB);
+  Queued.insert(BB);
 
-  SmallVector<BasicBlock *, 2> OtherSuccessors;
-  SmallVector<BasicBlock *, 2> LoopSuccessors;
+  while (ToVisit.size() != 0) {
+    BasicBlock *BB = ToVisit.front();
+    ToVisit.pop();
 
-  for (BasicBlock *Successor : successors(BB)) {
-    // Ignoring back-edges.
-    if (DT.dominates(Successor, BB))
+    if (!CanBeVisited(BB)) {
+      ToVisit.push(BB);
       continue;
+    }
 
-    if (isLoopHeader && L->contains(Successor)) {
-      LoopSuccessors.push_back(Successor);
-    } else
-      OtherSuccessors.push_back(Successor);
-  }
+    size_t Rank = GetNodeRank(BB);
+    OrderInfo Info = {Rank, BlockToOrder.size()};
+    BlockToOrder.emplace(BB, Info);
 
-  for (BasicBlock *BB : LoopSuccessors)
-    Rank = std::max(Rank, visit(BB, Rank + 1));
+    for (BasicBlock *S : successors(BB)) {
+      if (Queued.count(S) != 0)
+        continue;
+      ToVisit.push(S);
+      Queued.insert(S);
+    }
+  }
 
-  size_t OutputRank = Rank;
-  for (BasicBlock *Item : OtherSuccessors)
-    OutputRank = std::max(OutputRank, visit(Item, Rank + 1));
-  return OutputRank;
+  return 0;
 }
 
 PartialOrderingVisitor::PartialOrderingVisitor(Function &F) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index 83e717e6ea58..11fd3a5c61dc 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -18,6 +18,7 @@
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/TypedPointerType.h"
+#include <queue>
 #include <string>
 #include <unordered_set>
 
@@ -62,7 +63,9 @@ class SPIRVSubtarget;
 class PartialOrderingVisitor {
   DomTreeBuilder::BBDomTree DT;
   LoopInfo LI;
-  std::unordered_set<BasicBlock *> Visited = {};
+
+  std::unordered_set<BasicBlock *> Queued = {};
+  std::queue<BasicBlock *> ToVisit = {};
 
   struct OrderInfo {
     size_t Rank;
@@ -80,6 +83,9 @@ class PartialOrderingVisitor {
   // Visits |BB| with the current rank being |Rank|.
   size_t visit(BasicBlock *BB, size_t Rank);
 
+  size_t GetNodeRank(BasicBlock *BB) const;
+  bool CanBeVisited(BasicBlock *BB) const;
+
 public:
   // Build the visitor to operate on the function F.
   PartialOrderingVisitor(Function &F);
diff --git a/llvm/test/CodeGen/SPIRV/HlslBufferLoad.ll b/llvm/test/CodeGen/SPIRV/HlslBufferLoad.ll
index fe960f0d6f2f..66d5f0f4b05f 100644
--- a/llvm/test/CodeGen/SPIRV/HlslBufferLoad.ll
+++ b/llvm/test/CodeGen/SPIRV/HlslBufferLoad.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-library %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-vulkan-library %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-library %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: OpDecorate [[IntBufferVar:%[0-9]+]] DescriptorSet 16
@@ -18,13 +19,13 @@
 ; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
 ; CHECK-NEXT: OpLabel
 define void @RWBufferLoad() #0 {
-; CHECK-NEXT: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeInt]] [[IntBufferVar]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeInt]] [[IntBufferVar]]
   %buffer0 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 24)
       @llvm.spv.handle.fromBinding.tspirv.Image_f32_5_2_0_0_2_24(
           i32 16, i32 7, i32 1, i32 0, i1 false)
 
 ; Make sure we use the same variable with multiple loads.
-; CHECK-NEXT: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeInt]] [[IntBufferVar]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeInt]] [[IntBufferVar]]
   %buffer1 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 24)
       @llvm.spv.handle.fromBinding.tspirv.Image_f32_5_2_0_0_2_24(
           i32 16, i32 7, i32 1, i32 0, i1 false)
@@ -36,7 +37,7 @@ define void @RWBufferLoad() #0 {
 define void @UseDifferentGlobalVar() #0 {
 ; Make sure we use a different variable from the first function. They have
 ; different types.
-; CHECK-NEXT: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeFloat]] [[FloatBufferVar]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeFloat]] [[FloatBufferVar]]
   %buffer0 = call target("spirv.Image", float, 5, 2, 0, 0, 2, 3)
       @llvm.spv.handle.fromBinding.tspirv.Image_f32_5_2_0_0_2_3(
           i32 16, i32 7, i32 1, i32 0, i1 false)
@@ -48,7 +49,7 @@ define void @UseDifferentGlobalVar() #0 {
 define void @ReuseGlobalVarFromFirstFunction() #0 {
 ; Make sure we use the same variable as the first function. They should be the
 ; same in case one function calls the other.
-; CHECK-NEXT: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeInt]] [[IntBufferVar]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeInt]] [[IntBufferVar]]
   %buffer1 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 24)
       @llvm.spv.handle.fromBinding.tspirv.Image_f32_5_2_0_0_2_24(
           i32 16, i32 7, i32 1, i32 0, i1 false)
diff --git a/llvm/test/CodeGen/SPIRV/OpVariable_order.ll b/llvm/test/CodeGen/SPIRV/OpVariable_order.ll
index 6057bf38d4c4..c68250697c4a 100644
--- a/llvm/test/CodeGen/SPIRV/OpVariable_order.ll
+++ b/llvm/test/CodeGen/SPIRV/OpVariable_order.ll
@@ -1,7 +1,7 @@
 ; All OpVariable instructions in a function must be the first instructions in the first block
 
-; RUN: llc -O0 -mtriple=spirv-unknown-linux %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-linux %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv32-unknown-linux %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-linux %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-SPIRV: OpFunction
 ; CHECK-SPIRV-NEXT: OpLabel
diff --git a/llvm/test/CodeGen/SPIRV/ShaderBufferImage.ll b/llvm/test/CodeGen/SPIRV/ShaderBufferImage.ll
index 3c002e1849b8..1f203043e6a1 100644
--- a/llvm/test/CodeGen/SPIRV/ShaderBufferImage.ll
+++ b/llvm/test/CodeGen/SPIRV/ShaderBufferImage.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-library %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-vulkan-library %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-library %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-NOT: OpCapability ImageBasic
diff --git a/llvm/test/CodeGen/SPIRV/ShaderImage.ll b/llvm/test/CodeGen/SPIRV/ShaderImage.ll
index 6ac58ce42f95..9cd5fb338080 100644
--- a/llvm/test/CodeGen/SPIRV/ShaderImage.ll
+++ b/llvm/test/CodeGen/SPIRV/ShaderImage.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-library %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-vulkan-library %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-library %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: [[Float:%[0-9]+]] = OpTypeFloat 32
diff --git a/llvm/test/CodeGen/SPIRV/basic_float_types.ll b/llvm/test/CodeGen/SPIRV/basic_float_types.ll
index 1c7a8a851f59..dfee1ace2205 100644
--- a/llvm/test/CodeGen/SPIRV/basic_float_types.ll
+++ b/llvm/test/CodeGen/SPIRV/basic_float_types.ll
@@ -1,4 +1,3 @@
-; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
 ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
diff --git a/llvm/test/CodeGen/SPIRV/basic_int_types.ll b/llvm/test/CodeGen/SPIRV/basic_int_types.ll
index bb664568ed84..e85e0919d179 100644
--- a/llvm/test/CodeGen/SPIRV/basic_int_types.ll
+++ b/llvm/test/CodeGen/SPIRV/basic_int_types.ll
@@ -1,6 +1,4 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
 ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
diff --git a/llvm/test/CodeGen/SPIRV/basic_int_types_spirvdis.ll b/llvm/test/CodeGen/SPIRV/basic_int_types_spirvdis.ll
index 3778d8979291..0d7b2b99f64e 100644
--- a/llvm/test/CodeGen/SPIRV/basic_int_types_spirvdis.ll
+++ b/llvm/test/CodeGen/SPIRV/basic_int_types_spirvdis.ll
@@ -1,5 +1,5 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
 ; REQUIRES: spirv-tools
-; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - --filetype=obj | spirv-dis | FileCheck %s
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - --filetype=obj | spirv-dis | FileCheck %s
 ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - --filetype=obj | spirv-dis | FileCheck %s
 
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_DispatchThreadID.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_DispatchThreadID.ll
index c84b1c4b06c1..2d8692adf12a 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_DispatchThreadID.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_DispatchThreadID.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-vulkan-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; This file generated from the following command:
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveGetLaneIndex.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveGetLaneIndex.ll
index 89a8575fa159..d0a56854c32f 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveGetLaneIndex.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveGetLaneIndex.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-vulkan-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; This file generated from the following command:
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/abs.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/abs.ll
index 8f1092c2206e..c3e894afd710 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/abs.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/abs.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/acos.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/acos.ll
index 7c9450267cbe..1936f6d27207 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/acos.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/acos.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all.ll
index 7c40eed8465a..1edd69e2b0d5 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all.ll
@@ -1,5 +1,6 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-HLSL
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-OCL
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-HLSL
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-OCL
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 ; Make sure spirv operation function calls for all are generated.
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/any.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/any.ll
index 54f5b7774b57..dc6e9dc20330 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/any.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/any.ll
@@ -1,5 +1,6 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-HLSL
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-OCL
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-HLSL
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-OCL
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 ; Make sure spirv operation function calls for any are generated.
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/asin.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/asin.ll
index 4d57c6fce77f..be338f22bf12 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/asin.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/asin.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan.ll
index 65e198d0e71a..5d352eb80af2 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan2.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan2.ll
index bdbfc133efa2..aba6f7583b68 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan2.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan2.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/ceil.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/ceil.ll
index 93677aadffa5..2c36459bdac9 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/ceil.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/ceil.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cos.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cos.ll
index e9e9642354f5..937a545cc563 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cos.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cos.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cosh.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cosh.ll
index 1560f9b9bd76..2d7a4caada7d 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cosh.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cosh.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/countbits.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/countbits.ll
index 57ec0bda2e18..d47ec3ec27aa 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/countbits.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/countbits.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpMemoryModel Logical GLSL450
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/degrees.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/degrees.ll
index 533bcca6f621..691536200128 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/degrees.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/degrees.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp.ll
index c1734a264ea0..43bb8e217a67 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp2.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp2.ll
index 4753b7bd9fe5..ae6c33cb0c7e 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp2.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp2.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/floor.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/floor.ll
index ea19fa94ea32..1ecaafc22e6f 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/floor.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/floor.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmad.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmad.ll
index b1ca34dc504c..add94601bd16 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmad.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmad.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmax.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmax.ll
index ca0fcfe8d646..b202025f5dc8 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmax.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmax.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmin.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmin.ll
index adc563bcea5c..77e2ed1748e6 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmin.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmin.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/frac.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/frac.ll
index 4c088b6b3810..41c18b693574 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/frac.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/frac.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/imad.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/imad.ll
index 1be8eb7e6516..a161147c8b96 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/imad.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/imad.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#int_16:]] = OpTypeInt 16 0
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/lerp.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/lerp.ll
index aa7ad8c74d33..94272a84bd63 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/lerp.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/lerp.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; Make sure SPIRV operation function calls for lerp are generated as FMix
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log.ll
index f85b20324da5..d5dd92042537 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log10.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log10.ll
index 32d63a0c0f1d..a829422d84eb 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log10.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log10.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: %[[#extinst:]] = OpExtInstImport "GLSL.std.450"
@@ -6,35 +7,22 @@
 ; CHECK: %[[#float:]] = OpTypeFloat 32
 ; CHECK: %[[#v4float:]] = OpTypeVector %[[#float]] 4
 ; CHECK: %[[#float_0_30103001:]] = OpConstant %[[#float]] 0.30103000998497009
-; CHECK: %[[#_ptr_Function_v4float:]] = OpTypePointer Function %[[#v4float]]
-; CHECK: %[[#_ptr_Function_float:]] = OpTypePointer Function %[[#float]]
 
-define void @main() {
+define void @main(float %f, <4 x float> %f4) {
 entry:
-; CHECK: %[[#f:]] = OpVariable %[[#_ptr_Function_float]] Function
-; CHECK: %[[#logf:]] = OpVariable %[[#_ptr_Function_float]] Function
-; CHECK: %[[#f4:]] = OpVariable %[[#_ptr_Function_v4float]] Function
-; CHECK: %[[#logf4:]] = OpVariable %[[#_ptr_Function_v4float]] Function
-  %f = alloca float, align 4
+; CHECK-DAG: %[[#f:]] = OpFunctionParameter %[[#float]]
+; CHECK-DAG: %[[#f4:]] = OpFunctionParameter %[[#v4float]]
   %logf = alloca float, align 4
-  %f4 = alloca <4 x float>, align 16
   %logf4 = alloca <4 x float>, align 16
 
-; CHECK: %[[#load:]] = OpLoad %[[#float]] %[[#f]] Aligned 4
-; CHECK: %[[#log2:]] = OpExtInst %[[#float]] %[[#extinst]] Log2 %[[#load]]
+
+; CHECK: %[[#log2:]] = OpExtInst %[[#float]] %[[#extinst]] Log2 %[[#f]]
 ; CHECK: %[[#res:]] = OpFMul %[[#float]] %[[#log2]] %[[#float_0_30103001]]
-; CHECK: OpStore %[[#logf]] %[[#res]] Aligned 4
-  %0 = load float, ptr %f, align 4
-  %elt.log10 = call float @llvm.log10.f32(float %0)
-  store float %elt.log10, ptr %logf, align 4
+  %elt.log10 = call float @llvm.log10.f32(float %f)
 
-; CHECK: %[[#load:]] = OpLoad %[[#v4float]] %[[#f4]] Aligned 16
-; CHECK: %[[#log2:]] = OpExtInst %[[#v4float]] %[[#extinst]] Log2 %[[#load]]
+; CHECK: %[[#log2:]] = OpExtInst %[[#v4float]] %[[#extinst]] Log2 %[[#f4]]
 ; CHECK: %[[#res:]] = OpVectorTimesScalar %[[#v4float]] %[[#log2]] %[[#float_0_30103001]]
-; CHECK: OpStore %[[#logf4]] %[[#res]] Aligned 16
-  %1 = load <4 x float>, ptr %f4, align 16
-  %elt.log101 = call <4 x float> @llvm.log10.v4f32(<4 x float> %1)
-  store <4 x float> %elt.log101, ptr %logf4, align 16
+  %elt.log101 = call <4 x float> @llvm.log10.v4f32(<4 x float> %f4)
 
   ret void
 }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log2.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log2.ll
index add7f77897f7..c71ca125c172 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log2.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log2.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/normalize.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/normalize.ll
index fa73b9c2a4d3..ddf89221be2a 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/normalize.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/normalize.ll
@@ -1,3 +1,4 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
 ; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/pow.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/pow.ll
index 3ac98853b92f..38c51ca47d86 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/pow.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/pow.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/radians.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/radians.ll
index 1fe8ab30ed95..7aad4df76e31 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/radians.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/radians.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rcp.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rcp.ll
index 6f91162a378c..9c8c14c2a722 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rcp.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rcp.ll
@@ -1,4 +1,5 @@
- ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+ ; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 
 ; CHECK-DAG: %[[#float_64:]] = OpTypeFloat 64
 ; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/reversebits.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/reversebits.ll
index a23b15ab075d..ce8175fdceb2 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/reversebits.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/reversebits.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpMemoryModel Logical GLSL450
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/round.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/round.ll
index 1c7e78261ffe..0c88c55cbd39 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/round.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/round.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rsqrt.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rsqrt.ll
index 91023a1e401e..33d3edc080fd 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rsqrt.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rsqrt.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sin.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sin.ll
index a6ae70a48e5d..7474b7599451 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sin.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sin.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sinh.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sinh.ll
index 3b8bdbed0041..6a31b7021877 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sinh.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sinh.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smax.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smax.ll
index 901e4764e15f..cbf0b243ab2b 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smax.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smax.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smin.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smin.ll
index c39c39f0455f..960de853f3af 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smin.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smin.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/splitdouble.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/splitdouble.ll
index d18b16b843c3..a05a31c18a75 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/splitdouble.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/splitdouble.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; Make sure lowering is correctly generating spirv code.
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sqrt.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sqrt.ll
index bb1f0346047e..55d8a286a0e7 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sqrt.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sqrt.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/step.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/step.ll
index bb50d8c790f8..eac0b8589555 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/step.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/step.ll
@@ -1,3 +1,4 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
 ; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tan.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tan.ll
index b4a6e1574f73..6e2f0698b7b6 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tan.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tan.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tanh.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tanh.ll
index 94fc3f0ec7ab..1dfdf83fee31 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tanh.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tanh.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/trunc.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/trunc.ll
index 2a308028a9b4..bae614ee5967 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/trunc.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/trunc.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umax.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umax.ll
index 01606a387327..e2b14b089bc1 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umax.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umax.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umin.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umin.ll
index 34185ad7143e..708b76a93e66 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umin.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umin.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
diff --git a/llvm/test/CodeGen/SPIRV/literals.ll b/llvm/test/CodeGen/SPIRV/literals.ll
index 4109bb6de561..86a366976a6e 100644
--- a/llvm/test/CodeGen/SPIRV/literals.ll
+++ b/llvm/test/CodeGen/SPIRV/literals.ll
@@ -1,12 +1,10 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
 ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-
 ; CHECK: %[[#F32:]] = OpTypeFloat 32
 ; CHECK: %[[#F64:]] = OpTypeFloat 64
 
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/basic-if.ll b/llvm/test/CodeGen/SPIRV/structurizer/basic-if.ll
new file mode 100644
index 000000000000..5585e4a07590
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/basic-if.ll
@@ -0,0 +1,52 @@
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+; Function Attrs: convergent noinline norecurse nounwind optnone
+define spir_func noundef i32 @_Z7processv() #0 {
+
+; CHECK: %[[#entry:]] = OpLabel
+; CHECK:                OpSelectionMerge %[[#merge:]] None
+; CHECK:                OpBranchConditional %[[#]] %[[#left:]] %[[#right:]]
+entry:
+  %0 = call token @llvm.experimental.convergence.entry()
+  %1 = alloca i32, align 4
+  br i1 true, label %left, label %right
+
+; CHECK: %[[#left]] = OpLabel
+; CHECK:              OpBranch %[[#merge]]
+left:
+  store i32 0, ptr %1
+  br label %end
+
+; CHECK: %[[#right]] = OpLabel
+; CHECK:               OpBranch %[[#merge]]
+right:
+  store i32 0, ptr %1
+  br label %end
+
+; CHECK: %[[#merge]] = OpLabel
+; CHECK:               OpReturnValue %[[#]]
+end:
+  ret i32 0
+}
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.entry() #1
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.loop() #1
+
+
+attributes #0 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { convergent norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #3 = { convergent }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/basic-imbalanced-if.ll b/llvm/test/CodeGen/SPIRV/structurizer/basic-imbalanced-if.ll
new file mode 100644
index 000000000000..810b5785e4b1
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/basic-imbalanced-if.ll
@@ -0,0 +1,47 @@
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+; Function Attrs: convergent noinline norecurse nounwind optnone
+define spir_func noundef i32 @_Z7processv() #0 {
+
+; CHECK: %[[#entry:]] = OpLabel
+; CHECK:                OpSelectionMerge %[[#merge:]] None
+; CHECK:                OpBranchConditional %[[#]] %[[#left:]] %[[#merge]]
+entry:
+  %0 = call token @llvm.experimental.convergence.entry()
+  %1 = alloca i32, align 4
+  br i1 true, label %left, label %end
+
+; CHECK: %[[#left]] = OpLabel
+; CHECK:              OpBranch %[[#merge]]
+left:
+  store i32 0, ptr %1
+  br label %end
+
+; CHECK: %[[#merge]] = OpLabel
+; CHECK:               OpReturnValue %[[#]]
+end:
+  ret i32 0
+}
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.entry() #1
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.loop() #1
+
+
+attributes #0 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { convergent norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #3 = { convergent }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/basic-loop.ll b/llvm/test/CodeGen/SPIRV/structurizer/basic-loop.ll
new file mode 100644
index 000000000000..ded9c335c5a2
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/basic-loop.ll
@@ -0,0 +1,59 @@
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+; Function Attrs: convergent noinline norecurse nounwind optnone
+define spir_func noundef i32 @_Z7processv() #0 {
+
+; CHECK: %[[#entry:]] = OpLabel
+; CHECK:                OpBranch %[[#header:]]
+entry:
+  %0 = call token @llvm.experimental.convergence.entry()
+  %1 = alloca i32, align 4
+  br label %header
+
+; CHECK: %[[#header]] = OpLabel
+; CHECK:                OpLoopMerge %[[#merge:]] %[[#continue:]] None
+; CHECK:                OpBranchConditional %[[#]] %[[#body:]] %[[#merge]]
+header:
+  %2 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+  br i1 true, label %body, label %merge
+
+; CHECK: %[[#body]] = OpLabel
+; CHECK:              OpBranch %[[#continue]]
+body:
+  store i32 0, ptr %1
+  br label %continue
+
+continue:
+  br label %header
+; CHECK: %[[#continue]] = OpLabel
+; CHECK:                  OpBranch %[[#header]]
+
+; CHECK: %[[#merge]] = OpLabel
+; CHECK:               OpReturnValue %[[#]]
+merge:
+  ret i32 0
+}
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.entry() #1
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.loop() #1
+
+
+attributes #0 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { convergent norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #3 = { convergent }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
+
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/basic-phi.ll b/llvm/test/CodeGen/SPIRV/structurizer/basic-phi.ll
new file mode 100644
index 000000000000..a43d25e9b06d
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/basic-phi.ll
@@ -0,0 +1,58 @@
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+; Function Attrs: convergent noinline norecurse nounwind optnone
+define spir_func noundef i32 @_Z7processv() #0 {
+
+; CHECK-DAG:    %[[#int_0:]] = OpConstant %[[#]] 0
+; CHECK-DAG:    %[[#int_1:]] = OpConstant %[[#]] 1
+
+; CHECK: %[[#entry:]] = OpLabel
+; CHECK:   %[[#var:]] = OpVariable %[[#]] Function
+; CHECK:                OpSelectionMerge %[[#merge:]] None
+; CHECK:                OpBranchConditional %[[#]] %[[#left:]] %[[#right:]]
+entry:
+  %0 = call token @llvm.experimental.convergence.entry()
+  br i1 true, label %left, label %right
+
+; CHECK:      %[[#left]] = OpLabel
+; CHECK-NEXT:              OpStore %[[#var]] %[[#int_0]]
+; CHECK-NEXT:              OpBranch %[[#merge]]
+left:
+  br label %end
+
+; CHECK:      %[[#right]] = OpLabel
+; CHECK-NEXT:               OpStore %[[#var]] %[[#int_1]]
+; CHECK-NEXT:               OpBranch %[[#merge]]
+right:
+  br label %end
+
+; CHECK: %[[#merge]] = OpLabel
+; CHECK:  %[[#tmp:]] = OpLoad %[[#]] %[[#var]]
+; CHECK:               OpReturnValue %[[#tmp]]
+end:
+  %1 = phi i32 [ 0, %left ], [ 1, %right ]
+  ret i32 %1
+}
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.entry() #1
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.loop() #1
+
+
+attributes #0 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { convergent norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #3 = { convergent }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll
index 4934b17c8c00..86033608deb6 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll
@@ -8,16 +8,17 @@ target triple = "spirv-unknown-vulkan1.3-compute"
 ; CHECK-DAG: OpName %[[#fn:]] "_Z2fnv"
 ; CHECK-DAG: OpName %[[#fn1:]] "_Z3fn1v"
 ; CHECK-DAG: OpName %[[#fn2:]] "_Z3fn2v"
-; CHECK-DAG: OpName %[[#val:]] "val"
-; CHECK-DAG: OpName %[[#a:]] "a"
-; CHECK-DAG: OpName %[[#b:]] "b"
-; CHECK-DAG: OpName %[[#c:]] "c"
+
+; CHECK-DAG: OpName %[[#r2m_a:]] ".reg2mem3"
+; CHECK-DAG: OpName %[[#r2m_b:]] ".reg2mem1"
+; CHECK-DAG: OpName %[[#r2m_c:]] ".reg2mem"
 
 ; CHECK-DAG: %[[#int_ty:]] = OpTypeInt 32 0
-; CHECK-DAG: %[[#bool_ty:]] = OpTypeBool
-; CHECK-DAG: %[[#int_pfty:]] = OpTypePointer Function %[[#int_ty]]
 
-; CHECK-DAG: %[[#int_0:]] = OpConstant %[[#int_ty]] 0
+; CHECK-DAG: %[[#int_0:]] = OpConstant %[[#]] 0
+; CHECK-DAG: %[[#int_1:]] = OpConstant %[[#]] 1
+; CHECK-DAG: %[[#true:]] = OpConstantTrue
+; CHECK-DAG: %[[#false:]] = OpConstantFalse
 
 declare token @llvm.experimental.convergence.entry() #1
 
@@ -44,100 +45,86 @@ entry:
 
 ; CHECK: %[[#process]] = OpFunction %[[#int_ty]]
 define spir_func noundef i32 @_Z7processv() #0 {
+
+; CHECK:         %[[#entry:]] = OpLabel
+; CHECK-DAG:      %[[#r2m_a]] = OpVariable %[[#]] Function
+; CHECK:                        OpSelectionMerge %[[#a_merge:]]
+; CHECK:                        OpBranchConditional %[[#]] %[[#a_true:]] %[[#a_false:]]
 entry:
-  ; CHECK:     %[[#entry:]] = OpLabel
   %0 = call token @llvm.experimental.convergence.entry()
-  %a = alloca i32, align 4
-  %b = alloca i32, align 4
-  %c = alloca i32, align 4
-  %val = alloca i32, align 4
-  store i32 0, ptr %a, align 4
-  store i32 1, ptr %b, align 4
-  store i32 2, ptr %c, align 4
-  store i32 0, ptr %val, align 4
-  ; CHECK-DAG:      %[[#a]] = OpVariable %[[#int_pfty]] Function
-  ; CHECK-DAG:      %[[#b]] = OpVariable %[[#int_pfty]] Function
-  ; CHECK-DAG:      %[[#c]] = OpVariable %[[#int_pfty]] Function
-  ; CHECK-DAG:    %[[#val]] = OpVariable %[[#int_pfty]] Function
-  %1 = load i32, ptr %a, align 4
-  %tobool = icmp ne i32 %1, 0
-  br i1 %tobool, label %cond.true, label %cond.false
-  ; CHECK:        %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#a]]
-  ; CHECK:       %[[#cond:]] = OpINotEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
-  ; CHECK:                     OpSelectionMerge %[[#cond_end:]]
-  ; CHECK:                     OpBranchConditional %[[#cond]] %[[#cond_true:]] %[[#cond_false:]]
-
-cond.true:                                        ; preds = %entry
-  %2 = load i32, ptr %b, align 4
-  br label %cond.end
-  ; CHECK: %[[#cond_true]] = OpLabel
-  ; CHECK:                   OpBranch %[[#cond_end]]
-
-cond.false:                                       ; preds = %entry
-  %3 = load i32, ptr %c, align 4
-  br label %cond.end
-  ; CHECK: %[[#cond_false]] = OpLabel
-  ; CHECK:    %[[#load_c:]] = OpLoad %[[#]] %[[#c]]
-  ; CHECK:                    OpBranch %[[#cond_end]]
-
-cond.end:                                         ; preds = %cond.false, %cond.true
-  %cond = phi i32 [ %2, %cond.true ], [ %3, %cond.false ]
-  %tobool1 = icmp ne i32 %cond, 0
-  br i1 %tobool1, label %if.then, label %if.end
-  ; CHECK: %[[#cond_end]] = OpLabel
-  ; CHECK:     %[[#tmp:]] = OpPhi %[[#int_ty]] %[[#load_b:]] %[[#cond_true]] %[[#load_c]] %[[#cond_false]]
-  ; CHECK:                  OpSelectionMerge %[[#if_end:]]
-  ; CHECK:                  OpBranchConditional %[[#]] %[[#if_then:]] %[[#if_end]]
-
-if.then:                                          ; preds = %cond.end
-  %4 = load i32, ptr %val, align 4
-  %inc = add nsw i32 %4, 1
-  store i32 %inc, ptr %val, align 4
-  br label %if.end
-  ; CHECK: %[[#if_then]] = OpLabel
-  ; CHECK:                 OpBranch %[[#if_end]]
-
-if.end:                                           ; preds = %if.then, %cond.end
-  %call2 = call spir_func noundef i32 @_Z2fnv() #4 [ "convergencectrl"(token %0) ]
-  %tobool3 = icmp ne i32 %call2, 0
-  br i1 %tobool3, label %cond.true4, label %cond.false6
-  ; CHECK: %[[#if_end]] = OpLabel
-  ; CHECK:                OpSelectionMerge %[[#cond_end8:]]
-  ; CHECK:                OpBranchConditional %[[#]] %[[#cond_true4:]] %[[#cond_false6:]]
-
-cond.true4:                                       ; preds = %if.end
-  %call5 = call spir_func noundef i32 @_Z3fn1v() #4 [ "convergencectrl"(token %0) ]
-  br label %cond.end8
-  ; CHECK: %[[#cond_true4]] = OpLabel
-  ; CHECK:                   OpBranch %[[#cond_end8]]
-
-cond.false6:                                      ; preds = %if.end
-  %call7 = call spir_func noundef i32 @_Z3fn2v() #4 [ "convergencectrl"(token %0) ]
-  br label %cond.end8
-  ; CHECK: %[[#cond_false6]] = OpLabel
-  ; CHECK:                     OpBranch %[[#cond_end8]]
-
-cond.end8:                                        ; preds = %cond.false6, %cond.true4
-  %cond9 = phi i32 [ %call5, %cond.true4 ], [ %call7, %cond.false6 ]
-  %tobool10 = icmp ne i32 %cond9, 0
-  br i1 %tobool10, label %if.then11, label %if.end13
-  ; CHECK: %[[#cond_end8]] = OpLabel
-  ; CHECK:                   OpSelectionMerge %[[#if_end13:]]
-  ; CHECK:                   OpBranchConditional %[[#]] %[[#if_then11:]] %[[#if_end13]]
-
-if.then11:                                        ; preds = %cond.end8
-  %5 = load i32, ptr %val, align 4
-  %inc12 = add nsw i32 %5, 1
-  store i32 %inc12, ptr %val, align 4
-  br label %if.end13
-  ; CHECK: %[[#if_then11]] = OpLabel
-  ; CHECK:                   OpBranch %[[#if_end13]]
-
-if.end13:                                         ; preds = %if.then11, %cond.end8
-  %6 = load i32, ptr %val, align 4
-  ret i32 %6
-  ; CHECK: %[[#if_end13]] = OpLabel
-  ; CHECK:                  OpReturnValue
+  %var = alloca i32
+  br i1 true, label %a_true, label %a_false
+
+; CHECK: %[[#a_true]] = OpLabel
+; CHECK:                OpStore %[[#r2m_a]] %[[#true]]
+; CHECK:                OpBranch %[[#a_merge]]
+a_true:
+  br label %a_merge
+
+; CHECK: %[[#a_false]] = OpLabel
+; CHECK:                 OpStore %[[#r2m_a]] %[[#false]]
+; CHECK:                 OpBranch %[[#a_merge]]
+a_false:
+  br label %a_merge
+
+; CHECK: %[[#a_merge]] = OpLabel
+; CHECK:    %[[#tmp:]] = OpLoad %[[#]] %[[#r2m_a]]
+; CHECK:                 OpSelectionMerge %[[#b_merge:]]
+; CHECK:                 OpBranchConditional %[[#]] %[[#b_true:]] %[[#b_merge]]
+a_merge:
+  %1 = phi i1 [ true, %a_true ], [ false, %a_false ]
+  br i1 %1, label %b_true, label %b_merge
+
+; CHECK: %[[#b_true]] = OpLabel
+; CHECK:                OpBranch %[[#b_merge]]
+b_true:
+  store i32 0, ptr %var ; Prevents whole branch optimization.
+  br label %b_merge
+
+; CHECK: %[[#b_merge]] = OpLabel
+; CHECK:                 OpFunctionCall
+; CHECK:                 OpSelectionMerge %[[#c_merge:]]
+; CHECK:                 OpBranchConditional %[[#]] %[[#c_true:]] %[[#c_false:]]
+b_merge:
+  %f1 = call spir_func noundef i32 @_Z2fnv() #4 [ "convergencectrl"(token %0) ]
+  br i1 true, label %c_true, label %c_false
+
+; CHECK: %[[#c_true]] = OpLabel
+; CHECK:       %[[#]] = OpFunctionCall
+; CHECK:                OpStore %[[#r2m_b]] %[[#]]
+; CHECK:                OpBranch %[[#c_merge]]
+c_true:
+  %f2 = call spir_func noundef i32 @_Z3fn1v() #4 [ "convergencectrl"(token %0) ]
+  br label %c_merge
+
+; CHECK: %[[#c_false]] = OpLabel
+; CHECK:        %[[#]] = OpFunctionCall
+; CHECK:                 OpStore %[[#r2m_b]] %[[#]]
+; CHECK:                 OpBranch %[[#c_merge]]
+c_false:
+  %f3 = call spir_func noundef i32 @_Z3fn2v() #4 [ "convergencectrl"(token %0) ]
+  br label %c_merge
+
+; CHECK: %[[#c_merge]] = OpLabel
+; CHECK:    %[[#tmp:]] = OpLoad %[[#]] %[[#r2m_b]]
+; CHECK:                 OpStore %[[#r2m_c]] %[[#tmp:]]
+; CHECK:                 OpSelectionMerge %[[#d_merge:]]
+; CHECK:                 OpBranchConditional %[[#]] %[[#d_true:]] %[[#d_merge]]
+c_merge:
+  %5 = phi i32 [ %f2, %c_true ], [ %f3, %c_false ]
+  br i1 true, label %d_true, label %d_merge
+
+; CHECK: %[[#d_true]] = OpLabel
+; CHECK:                OpBranch %[[#d_merge]]
+d_true:
+  store i32 0, ptr %var ; Prevents whole branch optimization.
+  br label %d_merge
+
+; CHECK: %[[#d_merge]] = OpLabel
+; CHECK:    %[[#tmp:]] = OpLoad %[[#]] %[[#r2m_c]]
+; CHECK:                 OpReturnValue %[[#tmp]]
+d_merge:
+  ret i32 %5
 }
 
 ; Function Attrs: convergent noinline norecurse nounwind optnone
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.do.break.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.do.break.ll
index 3fc440dc445e..8e05bf1ebdaa 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.do.break.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.do.break.ll
@@ -1,5 +1,4 @@
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=2
 
 ; int foo() { return true; }
 ;
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.do.continue.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.do.continue.ll
index 051f0685a404..36b61745fa55 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.do.continue.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.do.continue.ll
@@ -1,4 +1,3 @@
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=10
 ; RUN: %if spirv-tools %{ llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
 ;
 ; Source HLSL:
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.do.nested.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.do.nested.ll
index a28e1c7b942d..d8e17c2291a7 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.do.nested.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.do.nested.ll
@@ -1,5 +1,4 @@
 ; RUN: %if spirv-tools %{ llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=142
 ;
 ; Source HLSL:
 ;
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.break.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.break.ll
index f2e60f916c79..9d8cab44c0cb 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.break.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.break.ll
@@ -1,5 +1,4 @@
 ; RUN: %if spirv-tools %{ llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=4
 ;
 ; Source HLSL:
 ;
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.continue.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.continue.ll
index 31a3433cae4c..12e6473c15c7 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.continue.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.continue.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=19
 
 ;
 ; int process() {
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.nested.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.nested.ll
index 1619a519273b..ddd4c5222301 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.nested.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.nested.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=2563170
 
 ;
 ; int process() {
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.short-circuited-cond.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.short-circuited-cond.ll
index 1b5e868317fb..07c20ebadd15 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.short-circuited-cond.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.short-circuited-cond.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=9
 
 ;
 ; int process() {
@@ -139,10 +138,10 @@
 ; CHECK:                  OpBranchConditional %[[#]] %[[#bb130:]] %[[#bb125:]]
 ; CHECK:   %[[#bb130:]] = OpLabel
 ; CHECK:                  OpBranch %[[#bb126:]]
-; CHECK:   %[[#bb125:]] = OpLabel
-; CHECK:                  OpReturnValue %[[#]]
 ; CHECK:   %[[#bb126:]] = OpLabel
 ; CHECK:                  OpBranch %[[#bb124:]]
+; CHECK:   %[[#bb125:]] = OpLabel
+; CHECK:                  OpReturnValue %[[#]]
 ; CHECK:                  OpFunctionEnd
 ; CHECK: %[[#func_83:]] = OpFunction %[[#void:]] DontInline %[[#]]
 ; CHECK:   %[[#bb131:]] = OpLabel
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.const-cond.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.const-cond.ll
index f3a9109b06ee..df406917fdff 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.const-cond.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.const-cond.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=3
 
 ;
 ; int process() {
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.for.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.for.ll
index 42c885070453..93effc141fc8 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.for.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.for.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=6
 
 ;
 ; int process() {
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.nested.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.nested.ll
index 1fea1ebd888f..a69475a59db6 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.nested.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.nested.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=3
 
 
 ;
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.plain.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.plain.ll
index c3b0caa4e269..8fa8c2c14878 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.plain.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.plain.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=2
 
 ;
 ; int process() {
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.logical-and.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.logical-and.ll
index a5f00071ca27..9d35fb3c82b0 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.logical-and.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.logical-and.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=1
 
 ;
 ; int fn() { return true; }
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.logical-or.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.logical-or.ll
index 73db1c897711..0a986661e50d 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.logical-or.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.logical-or.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=3
 
 ;
 ; int fn() { return true; }
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.ll
index 62d18cdf538c..dfaca85be228 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=0
 
 ;
 ; int process() {
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.ll
index d2447fe45624..8e2a0506d286 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=308
 
 ;
 ; int foo() { return 200; }
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple.ll
index 74c5a2edf7c2..125e3f751315 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=5
 
 ;
 ; int process() {
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple2.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple2.ll
index bfe3b45779af..cf50b982b23d 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple2.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple2.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=5
 
 ;
 ; int foo() { return 200; }
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.while.break.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.break.ll
index 8f3981a24496..769be32c9fc4 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.while.break.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.break.ll
@@ -41,47 +41,45 @@
 ; }
 
 ; CHECK: %[[#func_16:]] = OpFunction %[[#uint:]] DontInline %[[#]]
-; CHECK:    %[[#bb37:]] = OpLabel
+; CHECK:    %[[#bb44:]] = OpLabel
 ; CHECK:                  OpReturnValue %[[#]]
 ; CHECK:                  OpFunctionEnd
-; CHECK: %[[#func_17:]] = OpFunction %[[#void:]] DontInline %[[#]]
-; CHECK:    %[[#bb38:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb39:]]
-; CHECK:    %[[#bb39:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb40:]] %[[#bb41:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb42:]] %[[#bb40:]]
-; CHECK:    %[[#bb42:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb40:]] %[[#bb43:]]
-; CHECK:    %[[#bb43:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb40:]] %[[#bb41:]]
-; CHECK:    %[[#bb40:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb44:]] None
-; CHECK:                  OpSwitch %[[#]] %[[#bb44:]] 1 %[[#bb44:]] 2 %[[#bb44:]]
-; CHECK:    %[[#bb41:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb39:]]
-; CHECK:    %[[#bb44:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb45:]]
+; CHECK: %[[#func_19:]] = OpFunction %[[#void:]] DontInline %[[#]]
 ; CHECK:    %[[#bb45:]] = OpLabel
 ; CHECK:                  OpBranch %[[#bb46:]]
 ; CHECK:    %[[#bb46:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb47:]]
-; CHECK:    %[[#bb47:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb48:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb49:]] %[[#bb48:]]
+; CHECK:                  OpLoopMerge %[[#bb47:]] %[[#bb48:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb49:]] %[[#bb47:]]
 ; CHECK:    %[[#bb49:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb48:]]
-; CHECK:    %[[#bb48:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb50:]]
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb47:]] %[[#bb50:]]
 ; CHECK:    %[[#bb50:]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb47:]] %[[#bb48:]]
+; CHECK:    %[[#bb48:]] = OpLabel
+; CHECK:                  OpBranch %[[#bb46:]]
+; CHECK:    %[[#bb47:]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb51:]] None
+; CHECK:                  OpSwitch %[[#]] %[[#bb51:]] 1 %[[#bb51:]] 2 %[[#bb51:]]
+; CHECK:    %[[#bb51:]] = OpLabel
+; CHECK:                  OpBranch %[[#bb52:]]
+; CHECK:    %[[#bb52:]] = OpLabel
+; CHECK:                  OpBranch %[[#bb53:]]
+; CHECK:    %[[#bb53:]] = OpLabel
+; CHECK:                  OpBranch %[[#bb54:]]
+; CHECK:    %[[#bb54:]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb55:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb56:]] %[[#bb55:]]
+; CHECK:    %[[#bb56:]] = OpLabel
+; CHECK:                  OpBranch %[[#bb55:]]
+; CHECK:    %[[#bb55:]] = OpLabel
+; CHECK:                  OpBranch %[[#bb57:]]
+; CHECK:    %[[#bb57:]] = OpLabel
 ; CHECK:                  OpReturn
 ; CHECK:                  OpFunctionEnd
-; CHECK: %[[#func_35:]] = OpFunction %[[#void:]] None %[[#]]
-; CHECK:    %[[#bb51:]] = OpLabel
+; CHECK: %[[#func_40:]] = OpFunction %[[#void:]] None %[[#]]
+; CHECK:    %[[#bb58:]] = OpLabel
 ; CHECK:                  OpReturn
 ; CHECK:                  OpFunctionEnd
 
-
-
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 target triple = "spirv-unknown-vulkan1.3-compute"
 
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll b/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll
index faab2553ae6f..71f3ce9263da 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll
@@ -1,5 +1,5 @@
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 target triple = "spirv-unknown-vulkan-compute"
@@ -25,90 +25,92 @@ entry:
   ret i32 1
 }
 
+
+; CHECK-DAG:             OpName %[[#reg_0:]] "cond.reg2mem"
+; CHECK-DAG:             OpName %[[#reg_1:]] "cond9.reg2mem"
+
 define internal spir_func void @main() #0 {
-; CHECK:    %[[#cond:]] = OpINotEqual %[[#bool_ty:]] %[[#a:]] %[[#b:]]
-; CHECK:                  OpSelectionMerge %[[#cond_end:]] None
-; CHECK:                  OpBranchConditional %[[#cond]] %[[#cond_true:]] %[[#cond_false:]]
+; CHECK:                  OpSelectionMerge %[[#cond1_merge:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#cond1_true:]] %[[#cond1_false:]]
 entry:
   %0 = call token @llvm.experimental.convergence.entry()
   %a = alloca i32, align 4
   %b = alloca i32, align 4
-  %c = alloca i32, align 4
-  %val = alloca i32, align 4
-  store i32 0, ptr %val, align 4
-  %1 = load i32, ptr %a, align 4
-  %tobool = icmp ne i32 %1, 0
-  br i1 %tobool, label %cond.true, label %cond.false
-
-; CHECK:  %[[#cond_true]] = OpLabel
-; CHECK:                    OpBranch %[[#cond_end]]
-cond.true:
-  %2 = load i32, ptr %b, align 4
-  br label %cond.end
-
-; CHECK:  %[[#cond_false]] = OpLabel
-; CHECK:                     OpBranch %[[#cond_end]]
-cond.false:
-  %3 = load i32, ptr %c, align 4
-  br label %cond.end
-
-; CHECK:  %[[#cond_end]] = OpLabel
-; CHECK:     %[[#tmp:]]  = OpPhi %[[#int_ty:]] %[[#load_cond_true:]] %[[#cond_true]] %[[#load_cond_false:]] %[[#cond_false:]]
-; CHECK:     %[[#cond:]] = OpINotEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0:]]
-; CHECK:                   OpSelectionMerge %[[#if_end:]] None
-; CHECK:                   OpBranchConditional %[[#cond]] %[[#if_then:]] %[[#if_end]]
-cond.end:
-  %cond = phi i32 [ %2, %cond.true ], [ %3, %cond.false ]
+  br i1 true, label %cond1_true, label %cond1_false
+
+; CHECK:  %[[#cond1_true]] = OpLabel
+; CHECK:                     OpStore %[[#reg_0]] %[[#]]
+; CHECK:                     OpBranch %[[#cond1_merge]]
+cond1_true:
+  %2 = load i32, ptr %a, align 4
+  br label %cond1_merge
+
+; CHECK:  %[[#cond1_false]] = OpLabel
+; CHECK:                      OpStore %[[#reg_0]] %[[#]]
+; CHECK:                      OpBranch %[[#cond1_merge]]
+cond1_false:
+  %3 = load i32, ptr %b, align 4
+  br label %cond1_merge
+
+; CHECK: %[[#cond1_merge]] = OpLabel
+; CHECK:        %[[#tmp:]] = OpLoad %[[#]] %[[#reg_0]]
+; CHECK:       %[[#cond:]] = OpINotEqual %[[#]] %[[#tmp]] %[[#]]
+; CHECK:                     OpSelectionMerge %[[#cond2_merge:]] None
+; CHECK:                     OpBranchConditional %[[#cond]] %[[#cond2_true:]] %[[#cond2_merge]]
+cond1_merge:
+  %cond = phi i32 [ %2, %cond1_true ], [ %3, %cond1_false ]
   %tobool1 = icmp ne i32 %cond, 0
-  br i1 %tobool1, label %if.then, label %if.end
-
-; CHECK:  %[[#if_then]] = OpLabel
-; CHECK:                  OpBranch %[[#if_end]]
-if.then:
-  %4 = load i32, ptr %val, align 4
-  %inc = add nsw i32 %4, 1
-  store i32 %inc, ptr %val, align 4
-  br label %if.end
-
-; CHECK:    %[[#if_end]] = OpLabel
-; CHECK:                   OpSelectionMerge %[[#cond_end8:]] None
-; CHECK:                   OpBranchConditional %[[#tmp:]] %[[#cond4_true:]] %[[#cond_false6:]]
-if.end:
+  br i1 %tobool1, label %cond2_true, label %cond2_merge
+
+; CHECK:  %[[#cond2_true]] = OpLabel
+; CHECK:                     OpBranch %[[#cond2_merge]]
+cond2_true:
+  store i32 0, ptr %a
+  br label %cond2_merge
+
+; CHECK:    %[[#cond2_merge]] = OpLabel
+; CHECK:                        OpFunctionCall
+; CHECK:                        OpSelectionMerge %[[#cond3_merge:]] None
+; CHECK:                        OpBranchConditional %[[#]] %[[#cond3_true:]] %[[#cond3_false:]]
+cond2_merge:
   %call2 = call spir_func noundef i32 @fn() #4 [ "convergencectrl"(token %0) ]
-  %tobool3 = icmp ne i32 %call2, 0
-  br i1 %tobool3, label %cond.true4, label %cond.false6
+  br i1 true, label %cond3_true, label %cond3_false
 
-; CHECK:  %[[#cond4_true]] = OpLabel
-; CHECK:                     OpBranch %[[#cond_end8]]
-cond.true4:
+; CHECK:  %[[#cond3_true]] = OpLabel
+; CHECK:                     OpFunctionCall
+; CHECK:                     OpStore %[[#reg_1]] %[[#]]
+; CHECK:                     OpBranch %[[#cond3_merge]]
+cond3_true:
   %call5 = call spir_func noundef i32 @fn1() #4 [ "convergencectrl"(token %0) ]
-  br label %cond.end8
+  br label %cond3_merge
 
-; CHECK:  %[[#cond_false6]] = OpLabel
-; CHECK:                      OpBranch %[[#cond_end8]]
-cond.false6:
+; CHECK:  %[[#cond3_false]] = OpLabel
+; CHECK:                      OpFunctionCall
+; CHECK:                      OpStore %[[#reg_1]] %[[#]]
+; CHECK:                      OpBranch %[[#cond3_merge]]
+cond3_false:
   %call7 = call spir_func noundef i32 @fn2() #4 [ "convergencectrl"(token %0) ]
-  br label %cond.end8
-
-; CHECK:  %[[#cond_end8]] = OpLabel
-; CHECK:                      OpSelectionMerge %[[#if_end13:]] None
-; CHECK:                      OpBranchConditional %[[#tmp:]] %[[#if_then11:]] %[[#if_end13]]
-cond.end8:
-  %cond9 = phi i32 [ %call5, %cond.true4 ], [ %call7, %cond.false6 ]
+  br label %cond3_merge
+
+; CHECK:  %[[#cond3_merge]] = OpLabel
+; CHECK:         %[[#tmp:]] = OpLoad %[[#]] %[[#reg_1]]
+; CHECK:       %[[#cond:]] = OpINotEqual %[[#]] %[[#tmp]] %[[#]]
+; CHECK:                      OpSelectionMerge %[[#cond4_merge:]] None
+; CHECK:                      OpBranchConditional %[[#cond]] %[[#cond4_true:]] %[[#cond4_merge]]
+cond3_merge:
+  %cond9 = phi i32 [ %call5, %cond3_true ], [ %call7, %cond3_false ]
   %tobool10 = icmp ne i32 %cond9, 0
-  br i1 %tobool10, label %if.then11, label %if.end13
+  br i1 %tobool10, label %cond4_true, label %cond4_merge
 
-; CHECK:  %[[#if_then11]] = OpLabel
-; CHECK:                    OpBranch %[[#if_end13]]
-if.then11:
-  %5 = load i32, ptr %val, align 4
-  %inc12 = add nsw i32 %5, 1
-  store i32 %inc12, ptr %val, align 4
-  br label %if.end13
+; CHECK:  %[[#cond4_true]] = OpLabel
+; CHECK:                     OpBranch %[[#cond4_merge]]
+cond4_true:
+  store i32 0, ptr %a
+  br label %cond4_merge
 
-; CHECK:  %[[#if_end13]] = OpLabel
+; CHECK:  %[[#cond4_merge]] = OpLabel
 ; CHECK:                  OpReturn
-if.end13:
+cond4_merge:
   ret void
 }
 
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll b/llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll
index d547ad8eded9..05071d03164d 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll
@@ -1,5 +1,5 @@
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 target triple = "spirv-unknown-vulkan1.3-compute"
@@ -12,100 +12,98 @@ entry:
 
 define internal spir_func void @main() #2 {
 ; CHECK: %[[#entry:]] = OpLabel
-; CHECK:                OpBranch %[[#do_body:]]
+; CHECK:                OpBranch %[[#do_header:]]
 entry:
   %0 = call token @llvm.experimental.convergence.entry()
-  %val = alloca i32, align 4
-  %i = alloca i32, align 4
-  store i32 0, ptr %val, align 4
-  store i32 0, ptr %i, align 4
-  br label %do.body
+  %var = alloca i32, align 4
+  br label %do_header
 
+; Here a the loop header had to be split in two:
+; - 1 header for the loop
+; - 1 header for the condition.
+; In SPIR-V, a loop header cannot directly 
 ; CHECK: %[[#do_header:]] = OpLabel
-; CHECK:                    OpLoopMerge %[[#do_end:]] %[[#do_cond:]] None
-; CHECK:                    OpBranch %[[#do_body:]]
+; CHECK:                    OpLoopMerge %[[#do_merge:]] %[[#do_latch:]] None
+; CHECK:                    OpBranch %[[#new_header:]]
 
-; CHECK: %[[#do_body]] = OpLabel
-; CHECK:                 OpSelectionMerge %[[#if_then:]] None
-; CHECK:                 OpBranchConditional %[[#cond:]] %[[#if_then]] %[[#if_end:]]
-do.body:
+; CHECK: %[[#new_header]] = OpLabel
+; CHECK:                    OpSelectionMerge %[[#if_merge:]] None
+; CHECK:                    OpBranchConditional %[[#]] %[[#if_then:]] %[[#if_end:]]
+do_header:
   %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
-  %2 = load i32, ptr %i, align 4
-  %inc = add nsw i32 %2, 1
-  store i32 %inc, ptr %i, align 4
-  %3 = load i32, ptr %i, align 4
-  %cmp = icmp sgt i32 %3, 5
-  br i1 %cmp, label %if.then, label %if.end
+  store i32 0, ptr %var
+  br i1 true, label %if.then, label %if.end
+
+; CHECK: %[[#if_then]] = OpLabel
+; CHECK:                 OpBranch %[[#if_merge]]
+if.then:
+  store i32 0, ptr %var
+  br label %do_latch
 
 ; CHECK: %[[#if_end]] = OpLabel
-; CHECK:                OpBranch %[[#if_then]]
+; CHECK:                OpBranch %[[#if_merge]]
 if.end:
-  %4 = load i32, ptr %i, align 4
-  store i32 %4, ptr %val, align 4
-  br label %do.cond
+  store i32 0, ptr %var
+  br label %do_latch
 
-; CHECK: %[[#if_then]] = OpLabel
-; CHECK:                  OpBranch %[[#do_cond]]
-if.then:
-  br label %do.cond
+; CHECK: %[[#if_merge]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#do_latch]] %[[#do_merge]]
 
-; CHECK: %[[#do_cond]] = OpLabel
-; CHECK:                 OpBranchConditional %[[#cond:]] %[[#do_header]] %[[#do_end]]
-do.cond:
-  %5 = load i32, ptr %i, align 4
-  %cmp1 = icmp slt i32 %5, 10
-  br i1 %cmp1, label %do.body, label %do.end
+; CHECK: %[[#do_latch]] = OpLabel
+; CHECK:                  OpBranch %[[#do_header]]
+do_latch:
+  store i32 0, ptr %var
+  br i1 true, label %do_header, label %do.end
 
-; CHECK: %[[#do_end]] = OpLabel
-; CHECK:                OpBranch %[[#do_body2:]]
+; CHECK: %[[#do_merge]] = OpLabel
+; CHECK:                  OpBranch %[[#do2_header:]]
 do.end:
-  br label %do.body2
+  store i32 0, ptr %var
+  br label %do2_header
 
-; CHECK: %[[#do_body2]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#do_end11:]] %[[#do_cond9:]] None
-; CHECK:                  OpBranch %[[#do_body4:]]
-do.body2:
+; CHECK: %[[#do2_header]] = OpLabel
+; CHECK:                    OpLoopMerge %[[#do2_merge:]] %[[#do2_continue:]] None
+; CHECK:                    OpBranch %[[#do3_header:]]
+do2_header:
   %6 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
-  %7 = load i32, ptr %i, align 4
-  %inc3 = add nsw i32 %7, 1
-  store i32 %inc3, ptr %i, align 4
-  br label %do.body4
-
-; CHECK: %[[#do_body4]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#do_end8:]] %[[#do_cond6:]] None
-; CHECK:                  OpBranch %[[#do_cond6]]
-do.body4:
+  store i32 0, ptr %var
+  br label %do3_header
+
+; CHECK: %[[#do3_header]] = OpLabel
+; CHECK:                  OpLoopMerge %[[#do3_merge:]] %[[#do3_continue:]] None
+; CHECK:                  OpBranch %[[#do3_body:]]
+do3_header:
   %8 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %6) ]
-  %9 = load i32, ptr %val, align 4
-  %inc5 = add nsw i32 %9, 1
-  store i32 %inc5, ptr %val, align 4
-  br label %do.cond6
-
-; CHECK: %[[#do_cond6]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#cond:]] %[[#do_body4]] %[[#do_end8]]
-do.cond6:
-  %10 = load i32, ptr %i, align 4
-  %cmp7 = icmp slt i32 %10, 10
-  br i1 %cmp7, label %do.body4, label %do.end8
-
-; CHECK: %[[#do_end8]] = OpLabel
-; CHECK:                 OpBranch %[[#do_cond9]]
-do.end8:
-  %11 = load i32, ptr %i, align 4
-  %dec = add nsw i32 %11, -1
-  store i32 %dec, ptr %i, align 4
-  br label %do.cond9
-
-; CHECK: %[[#do_cond9]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#cond:]] %[[#do_body2]] %[[#do_end11]]
-do.cond9:
-  %12 = load i32, ptr %val, align 4
-  %cmp10 = icmp slt i32 %12, 10
-  br i1 %cmp10, label %do.body2, label %do.end11
-
-; CHECK: %[[#do_end11]] = OpLabel
-; CHECK:                  OpReturn
-do.end11:
+  store i32 0, ptr %var
+  br label %do3_continue
+
+; CHECK: %[[#do3_body]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#do3_continue]] %[[#do3_merge]]
+
+; CHECK: %[[#do3_continue]] = OpLabel
+; CHECK:                      OpBranch %[[#do3_header]]
+do3_continue:
+  store i32 0, ptr %var
+  br i1 true, label %do3_header, label %do3_merge
+
+; CHECK: %[[#do3_merge]] = OpLabel
+; CHECK:                   OpBranch %[[#do2_new_latch:]]
+do3_merge:
+  store i32 0, ptr %var
+  br label %do2_continue
+
+; CHECK: %[[#do2_new_latch]] = OpLabel
+; CHECK:                       OpBranchConditional %[[#]] %[[#do2_continue]] %[[#do2_merge]]
+
+; CHECK: %[[#do2_continue]] = OpLabel
+; CHECK:                      OpBranch %[[#do2_header]]
+do2_continue:
+  store i32 0, ptr %var
+  br i1 true, label %do2_header, label %do2_merge
+
+; CHECK: %[[#do2_merge]] = OpLabel
+; CHECK:                   OpReturn
+do2_merge:
   ret void
 }
 
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll b/llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll
index a16eed5cdfb4..bef95f5f63bf 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll
@@ -1,5 +1,5 @@
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 target triple = "spirv-unknown-vulkan1.3-compute"
@@ -8,84 +8,72 @@ define internal spir_func void @main() #0 {
 ; CHECK:    %[[#entry:]] = OpLabel
 entry:
   %0 = call token @llvm.experimental.convergence.entry()
-  %val = alloca i32, align 4
-  %i = alloca i32, align 4
-  %j = alloca i32, align 4
-  %k = alloca i32, align 4
-  store i32 0, ptr %val, align 4
-  store i32 0, ptr %i, align 4
-  store i32 0, ptr %j, align 4
-  store i32 0, ptr %k, align 4
-  br label %do.body
+  %var = alloca i32, align 4
+  br label %do1_header
 
-; CHECK:    %[[#do_1_header:]] = OpLabel
-; CHECK:                         OpLoopMerge %[[#end:]] %[[#do_1_latch:]] None
-; CHECK:                         OpBranch %[[#do_2_header:]]
-do.body:
+; CHECK:    %[[#do1_header:]] = OpLabel
+; CHECK:                        OpLoopMerge %[[#do1_merge:]] %[[#do1_continue:]] None
+; CHECK:                        OpBranch %[[#do2_header:]]
+do1_header:
   %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
-  %2 = load i32, ptr %val, align 4
-  %3 = load i32, ptr %i, align 4
-  %add = add nsw i32 %2, %3
-  store i32 %add, ptr %val, align 4
-  br label %do.body1
+  store i32 0, ptr %var
+  br label %do2_header
 
-; CHECK:    %[[#do_2_header]] = OpLabel
-; CHECK:                        OpLoopMerge %[[#do_2_end:]] %[[#do_2_latch:]] None
-; CHECK:                        OpBranch %[[#do_2_body:]]
-do.body1:
+; CHECK:    %[[#do2_header:]] = OpLabel
+; CHECK:                        OpLoopMerge %[[#do2_merge:]] %[[#do2_continue:]] None
+; CHECK:                        OpBranch %[[#do3_header:]]
+do2_header:
   %4 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %1) ]
-  br label %do.body2
+  store i32 0, ptr %var
+  br label %do3_header
 
-; CHECK:    %[[#do_2_body]] = OpLabel
-; CHECK:                      OpLoopMerge %[[#do_3_end:]] %[[#do_3_header:]] None
-; CHECK:                      OpBranch %[[#do_3_header]]
-do.body2:
+; CHECK:    %[[#do3_header:]] = OpLabel
+; CHECK:                        OpLoopMerge %[[#do3_merge:]] %[[#do3_continue:]] None
+; CHECK:                        OpBranch %[[#do3_cond:]]
+do3_header:
   %5 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %4) ]
-  %6 = load i32, ptr %k, align 4
-  %inc = add nsw i32 %6, 1
-  store i32 %inc, ptr %k, align 4
-  br label %do.cond
+  store i32 0, ptr %var
+  br label %do3_continue
 
-; CHECK:    %[[#do_3_header]] = OpLabel
-; CHECK:                        OpBranchConditional %[[#cond:]] %[[#do_2_body]] %[[#do_3_end]]
-do.cond:
-  %7 = load i32, ptr %k, align 4
-  %cmp = icmp slt i32 %7, 30
-  br i1 %cmp, label %do.body2, label %do.end
+; CHECK:        %[[#do3_cond]] = OpLabel
+; CHECK:                         OpBranchConditional %[[#]] %[[#do3_continue]] %[[#do3_merge]]
+; CHECK:    %[[#do3_continue]] = OpLabel
+; CHECK:                         OpBranch %[[#do3_header]]
+do3_continue:
+  store i32 0, ptr %var
+  br i1 true, label %do3_header, label %do3_merge
 
-; CHECK:    %[[#do_3_end]] = OpLabel
-; CHECK:                     OpBranch %[[#do_2_latch]]
-do.end:
-  %8 = load i32, ptr %j, align 4
-  %inc3 = add nsw i32 %8, 1
-  store i32 %inc3, ptr %j, align 4
-  br label %do.cond4
+; CHECK:    %[[#do3_merge]] = OpLabel
+; CHECK:                      OpBranch %[[#do2_cond:]]
+do3_merge:
+  store i32 0, ptr %var
+  br label %do2_continue
 
-; CHECK:    %[[#do_2_latch]] = OpLabel
-; CHECK:                     OpBranchConditional %[[#cond:]] %[[#do_2_header]] %[[#do_2_end]]
-do.cond4:
-  %9 = load i32, ptr %j, align 4
-  %cmp5 = icmp slt i32 %9, 20
-  br i1 %cmp5, label %do.body1, label %do.end6
+; CHECK:        %[[#do2_cond]] = OpLabel
+; CHECK:                         OpBranchConditional %[[#]] %[[#do2_continue]] %[[#do2_merge]]
+; CHECK:    %[[#do2_continue]] = OpLabel
+; CHECK:                         OpBranch %[[#do2_header]]
+do2_continue:
+  store i32 0, ptr %var
+  br i1 true, label %do2_header, label %do2_merge
 
-; CHECK:    %[[#do_2_end]] = OpLabel
-; CHECK:                     OpBranch %[[#do_1_latch]]
-do.end6:
-  %10 = load i32, ptr %i, align 4
-  %inc7 = add nsw i32 %10, 1
-  store i32 %inc7, ptr %i, align 4
-  br label %do.cond8
+; CHECK:    %[[#do2_merge]] = OpLabel
+; CHECK:                      OpBranch %[[#do1_cond:]]
+do2_merge:
+  store i32 0, ptr %var
+  br label %do1_continue
 
-; CHECK:    %[[#do_1_latch]] = OpLabel
-; CHECK:                       OpBranchConditional %[[#cond:]] %[[#do_1_header]] %[[#end]]
-do.cond8:
-  %11 = load i32, ptr %i, align 4
-  %cmp9 = icmp slt i32 %11, 10
-  br i1 %cmp9, label %do.body, label %do.end10
+; CHECK:        %[[#do1_cond]] = OpLabel
+; CHECK:                         OpBranchConditional %[[#]] %[[#do1_continue]] %[[#do1_merge]]
+; CHECK:    %[[#do1_continue]] = OpLabel
+; CHECK:                         OpBranch %[[#do1_header]]
+do1_continue:
+  store i32 0, ptr %var
+  br i1 true, label %do1_header, label %do1_merge
 
-; CHECK:    %[[#end]] = OpLabel
-; CHECK:                OpReturn
-do.end10:
+; CHECK:    %[[#do1_merge]] = OpLabel
+; CHECK:                      OpReturn
+do1_merge:
   ret void
 }
 
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll b/llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll
index 6d4a0e591cf5..9f84fc317021 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll
@@ -15,71 +15,75 @@ entry:
 
 define internal spir_func void @main() #2 {
 ; CHECK: %[[#entry:]] = OpLabel
-; CHECK:                OpBranch %[[#do_body:]]
+; CHECK:                OpBranch %[[#do1_header:]]
 entry:
   %0 = call token @llvm.experimental.convergence.entry()
-  %val = alloca i32, align 4
-  %i = alloca i32, align 4
-  store i32 0, ptr %val, align 4
-  store i32 0, ptr %i, align 4
-  br label %do.body
-
-; CHECK: %[[#do_body]] = OpLabel
-; CHECK:                 OpLoopMerge %[[#do_end:]] %[[#do_cond:]] None
-; CHECK:                 OpBranch %[[#do_cond]]
-do.body:
+  %var = alloca i32, align 4
+  br label %do1_header
+
+; CHECK:    %[[#do1_header]] = OpLabel
+; CHECK:                       OpLoopMerge %[[#do1_merge:]] %[[#do1_continue:]] None
+; CHECK:                       OpBranch %[[#do1_cond:]]
+do1_header:
   %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
-  %2 = load i32, ptr %i, align 4
-  store i32 %2, ptr %val, align 4
-  br label %do.cond
-
-; CHECK: %[[#do_cond]] = OpLabel
-; CHECK:                 OpBranchConditional %[[#cond:]] %[[#do_body]] %[[#do_end]]
-do.cond:
-  %3 = load i32, ptr %i, align 4
-  %cmp = icmp slt i32 %3, 10
-  br i1 %cmp, label %do.body, label %do.end
-
-; CHECK: %[[#do_end]] = OpLabel
-; CHECK:                OpBranch %[[#do_body1:]]
-do.end:
-  br label %do.body1
-
-; CHECK: %[[#do_body1]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#do_end3:]] %[[#do_cond2:]] None
-; CHECK:                  OpBranch %[[#do_cond2]]
-do.body1:
+  store i32 0, ptr %var
+  br label %do1_continue
+
+; CHECK:        %[[#do1_cond]] = OpLabel
+; CHECK:                         OpBranchConditional %[[#]] %[[#do1_continue]] %[[#do1_merge]]
+; CHECK:    %[[#do1_continue]] = OpLabel
+; CHECK:                         OpBranch %[[#do1_header]]
+do1_continue:
+  store i32 0, ptr %var
+  br i1 true, label %do1_header, label %do1_merge
+
+; CHECK: %[[#do1_merge]] = OpLabel
+; CHECK:                   OpBranch %[[#do2_header:]]
+do1_merge:
+  store i32 0, ptr %var
+  br label %do2_header
+
+; CHECK:    %[[#do2_header]] = OpLabel
+; CHECK:                       OpLoopMerge %[[#do2_merge:]] %[[#do2_continue:]] None
+; CHECK:                       OpBranch %[[#do2_cond:]]
+do2_header:
   %4 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
-  store i32 0, ptr %val, align 4
-  br label %do.cond2
-
-; CHECK: %[[#do_cond2]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#cond:]] %[[#do_body1]] %[[#do_end3]]
-do.cond2:
-  br i1 true, label %do.body1, label %do.end3
-
-; CHECK: %[[#do_end3]] = OpLabel
-; CHECK:                 OpBranch %[[#do_body4:]]
-do.end3:
-  br label %do.body4
-
-; CHECK: %[[#do_body4]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#do_end7:]] %[[#do_cond5:]] None
-; CHECK:                  OpBranch %[[#do_cond5]]
-do.body4:
+  store i32 0, ptr %var
+  br label %do2_continue
+
+; CHECK:        %[[#do2_cond]] = OpLabel
+; CHECK:                         OpBranchConditional %[[#]] %[[#do2_continue]] %[[#do2_merge]]
+; CHECK:    %[[#do2_continue]] = OpLabel
+; CHECK:                         OpBranch %[[#do2_header]]
+do2_continue:
+  store i32 0, ptr %var
+  br i1 true, label %do2_header, label %do2_merge
+
+; CHECK: %[[#do2_merge]] = OpLabel
+; CHECK:                   OpBranch %[[#do3_header:]]
+do2_merge:
+  store i32 0, ptr %var
+  br label %do3_header
+
+; CHECK:    %[[#do3_header]] = OpLabel
+; CHECK:                       OpLoopMerge %[[#do3_merge:]] %[[#do3_continue:]] None
+; CHECK:                       OpBranch %[[#do3_cond:]]
+do3_header:
   %5 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
-  br label %do.cond5
-
-; CHECK: %[[#do_cond5]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#cond:]] %[[#do_body4]] %[[#do_end7]]
-do.cond5:
-  %6 = load i32, ptr %val, align 4
-  %cmp6 = icmp slt i32 %6, 20
-  br i1 %cmp6, label %do.body4, label %do.end7
-
-; CHECK: %[[#do_end7]] = OpLabel
-; CHECK:                 OpReturn
-do.end7:
+  store i32 0, ptr %var
+  br label %do3_continue
+
+; CHECK:        %[[#do3_cond]] = OpLabel
+; CHECK:                         OpBranchConditional %[[#]] %[[#do3_continue]] %[[#do3_merge]]
+; CHECK:    %[[#do3_continue]] = OpLabel
+; CHECK:                         OpBranch %[[#do3_header]]
+do3_continue:
+  store i32 0, ptr %var
+  br i1 true, label %do3_header, label %do3_merge
+
+; CHECK: %[[#do3_merge]] = OpLabel
+; CHECK:                   OpReturn
+do3_merge:
   ret void
 }
 
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll b/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll
index 26b12a1e14f0..235f15b06447 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll
@@ -1,84 +1,77 @@
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - --asm-verbose=0 | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - --asm-verbose=0 | FileCheck %s --match-full-lines
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 target triple = "spirv-unknown-vulkan1.3-compute"
 
-; CHECK-DAG:  OpName %[[#fn:]] "fn"
-; CHECK-DAG:  OpName %[[#main:]] "main"
-; CHECK-DAG:  OpName %[[#var_a:]] "a"
-; CHECK-DAG:  OpName %[[#var_b:]] "b"
+define internal spir_func void @main() #3 {
+; CHECK-DAG:   OpName %[[#switch_0:]] "reg1"
+; CHECK-DAG:   OpName %[[#switch_1:]] "reg"
 
-; CHECK-DAG:  %[[#bool:]] = OpTypeBool
-; CHECK-DAG:  %[[#true:]] = OpConstantTrue %[[#bool]]
+; CHECK-DAG:   %[[#int_0:]] = OpConstant %[[#]] 0
+; CHECK-DAG:   %[[#int_1:]] = OpConstant %[[#]] 1
 
-; CHECK:  %[[#fn]] = OpFunction %[[#param:]] DontInline %[[#ftype:]]
-define spir_func noundef i32 @fn() #0 {
-entry:
-  %0 = call token @llvm.experimental.convergence.entry()
-  ret i32 1
-}
+; CHECK:       %[[#entry:]] = OpLabel
+; CHECK-DAG: %[[#switch_0]] = OpVariable %[[#]] Function
+; CHECK-DAG: %[[#switch_1]] = OpVariable %[[#]] Function
+; CHECK:                      OpSelectionMerge %[[#merge:]] None
+; CHECK:                      OpBranchConditional %[[#]] %[[#new_header:]] %[[#unreachable:]]
 
-; CHECK: %[[#main]] = OpFunction %[[#param:]] DontInline %[[#ftype:]]
+; CHECK:     %[[#new_header]] = OpLabel
+; CHECK:                        OpSelectionMerge %[[#new_merge:]] None
+; CHECK:                        OpBranchConditional %[[#]] %[[#taint_true_merge:]] %[[#br_false:]]
 
-define internal spir_func void @main() #3 {
+; CHECK:       %[[#unreachable]] = OpLabel
+; CHECK-NEXT:                      OpUnreachable
 
-; CHECK:     %[[#entry:]] = OpLabel
-; CHECK-DAG:  %[[#var_a]] = OpVariable %[[#type:]] Function
-; CHECK-DAG:  %[[#var_b]] = OpVariable %[[#type:]] Function
-; CHECK:       %[[#tmp:]] = OpLoad %[[#type:]] %[[#var_a]] Aligned 4
-; CHECK:      %[[#cond:]] = OpINotEqual %[[#bool]] %[[#tmp]] %[[#const:]]
-; CHECK:                    OpSelectionMerge %[[#if_end:]] None
-; CHECK:                    OpBranchConditional %[[#true]] %[[#cond1:]] %[[#dead:]]
+; CHECK: %[[#taint_true_merge]] = OpLabel
+; CHECK:                          OpStore %[[#switch_0]] %[[#int_1]]
+; CHECK:                          OpBranch %[[#new_merge]]
 
-; CHECK:      %[[#cond1]] = OpLabel
-; CHECK:                    OpSelectionMerge %[[#new_exit:]] None
-; CHECK:                    OpBranchConditional %[[#cond]] %[[#new_exit]] %[[#lor_lhs_false:]]
+; CHECK:      %[[#br_false]] = OpLabel
+; CHECK-DAG:                   OpStore %[[#switch_1]] %[[#int_0]]
+; CHECK:                       OpSelectionMerge %[[#taint_merge:]] None
+; CHECK:                       OpBranchConditional %[[#]] %[[#taint_merge]] %[[#taint_false:]]
 
-; CHECK:       %[[#dead]] = OpLabel
-; CHECK-NEXT:               OpUnreachable
+; CHECK:      %[[#taint_false]] = OpLabel
+; CHECK:                          OpStore %[[#switch_1]] %[[#int_1]]
+; CHECK:                          OpBranch %[[#taint_merge]]
 
-; CHECK:  %[[#lor_lhs_false]] = OpLabel
-; CHECK:           %[[#tmp:]] = OpLoad %[[#type:]] %[[#var_b]] Aligned 4
-; CHECK:          %[[#cond:]] = OpINotEqual %[[#bool]] %[[#tmp]] %[[#value:]]
-; CHECK:                        OpBranchConditional %[[#cond]] %[[#new_exit]] %[[#alias_exit:]]
+; CHECK:      %[[#taint_merge]] = OpLabel
+; CHECK:                          OpStore %[[#switch_0]] %[[#int_0]]
+; CHECK:             %[[#tmp:]] = OpLoad %[[#]] %[[#switch_1]]
+; CHECK:            %[[#cond:]] = OpIEqual %[[#]] %[[#int_0]] %[[#tmp]]
+; CHECK:                          OpBranchConditional %[[#cond]] %[[#taint_false_true:]] %[[#new_merge]]
 
-; CHECK: %[[#alias_exit]] = OpLabel
-; CHECK:                    OpBranch %[[#new_exit]]
+; CHECK: %[[#taint_false_true]] = OpLabel
+; CHECK:                          OpStore %[[#switch_0]] %[[#int_1]]
+; CHECK:                          OpBranch %[[#new_merge]]
 
-; CHECK:   %[[#new_exit]] = OpLabel
-; CHECK:       %[[#tmp:]] = OpPhi %[[#type:]] %[[#A:]] %[[#cond1]] %[[#A:]] %[[#lor_lhs_false]] %[[#B:]] %[[#alias_exit]]
-; CHECK:      %[[#cond:]] = OpIEqual %[[#bool]] %[[#A]] %[[#tmp]]
-; CHECK:                    OpBranchConditional %[[#cond]] %[[#if_then:]] %[[#if_end]]
+; CHECK:      %[[#new_merge]] = OpLabel
+; CHECK:             %[[#tmp:]] = OpLoad %[[#]] %[[#switch_0]]
+; CHECK:            %[[#cond:]] = OpIEqual %[[#]] %[[#int_0]] %[[#tmp]]
+; CHECK:                          OpBranchConditional %[[#cond]] %[[#merge]] %[[#br_true:]]
 
-; CHECK:    %[[#if_then]] = OpLabel
-; CHECK:                    OpBranch %[[#if_end]]
+; CHECK:    %[[#br_true]] = OpLabel
+; CHECK:                    OpBranch %[[#merge]]
 
-; CHECK:     %[[#if_end]] = OpLabel
-; CHECK:                    OpReturn
+; CHECK:     %[[#merge]] = OpLabel
+; CHECK:                   OpReturn
 
 entry:
   %0 = call token @llvm.experimental.convergence.entry()
-  %a = alloca i32, align 4
-  %b = alloca i32, align 4
-  %val = alloca i32, align 4
-  store i32 0, ptr %val, align 4
-  %1 = load i32, ptr %a, align 4
-  %tobool = icmp ne i32 %1, 0
-  br i1 %tobool, label %if.then, label %lor.lhs.false
-
-lor.lhs.false:
-  %2 = load i32, ptr %b, align 4
-  %tobool1 = icmp ne i32 %2, 0
-  br i1 %tobool1, label %if.then, label %if.end
-
-if.then:
-  %8 = load i32, ptr %val, align 4
-  %inc = add nsw i32 %8, 1
-  store i32 %inc, ptr %val, align 4
-  br label %if.end
-
-if.end:
+  %var = alloca i32, align 4
+  br i1 true, label %br_true, label %br_false
+
+br_false:
+  store i32 0, ptr %var, align 4
+  br i1 true, label %br_true, label %merge
+
+br_true:
+  store i32 0, ptr %var, align 4
+  br label %merge
+
+merge:
   ret void
 }
 
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/loop-continue-split.ll b/llvm/test/CodeGen/SPIRV/structurizer/loop-continue-split.ll
new file mode 100644
index 000000000000..5a5ea002e094
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/loop-continue-split.ll
@@ -0,0 +1,104 @@
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
+
+; The goal of this test is to voluntarily create 2 overlapping convergence
+; structures: the loop, and the inner condition.
+; Here, the condition header also branches to 2 internal nodes, which are not
+; directly a merge/exits.
+; This will require a proper header-split.
+; In addition, splitting the header makes the continue the merge of the inner
+; condition, so we need to properly split the continue block to create a
+; valid inner merge, in the correct order.
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+; CHECK-DAG:    OpName %[[#switch_0:]] "reg1"
+; CHECK-DAG:    OpName %[[#variable:]] "var"
+
+; CHECK-DAG:    %[[#int_0:]] = OpConstant %[[#]] 0
+; CHECK-DAG:    %[[#int_1:]] = OpConstant %[[#]] 1
+; CHECK-DAG:    %[[#int_2:]] = OpConstant %[[#]] 2
+; CHECK-DAG:    %[[#int_3:]] = OpConstant %[[#]] 3
+; CHECK-DAG:    %[[#int_4:]] = OpConstant %[[#]] 4
+
+define internal spir_func void @main() #1 {
+; CHECK:      %[[#entry:]] = OpLabel
+; CHECK:    %[[#switch_0]] = OpVariable %[[#]] Function
+; CHECK:    %[[#variable]] = OpVariable %[[#]] Function
+; CHECK:                     OpBranch %[[#header:]]
+entry:
+  %0 = call token @llvm.experimental.convergence.entry()
+  %var = alloca i32, align 4
+  br label %header
+
+; CHECK: %[[#header]] = OpLabel
+; CHECK:                OpLoopMerge %[[#merge:]] %[[#continue:]] None
+; CHECK:                OpBranch %[[#split_header:]]
+
+; CHECK: %[[#split_header]] = OpLabel
+; CHECK:                      OpSelectionMerge %[[#inner_merge:]] None
+; CHECK:                      OpBranchConditional %[[#]] %[[#left:]] %[[#right:]]
+header:
+  %2 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+  br i1 true, label %left, label %right
+
+; CHECK:     %[[#left]] = OpLabel
+; CHECK-DAG:              OpStore %[[#switch_0]] %[[#int_0]]
+; CHECK-DAG:              OpStore %[[#variable]] %[[#int_1]]
+; CHECK:                  OpBranchConditional %[[#]] %[[#inner_merge]] %[[#left_next:]]
+left:
+  store i32 1, ptr %var
+  br i1 true, label %merge, label %left_next
+
+; CHECK:     %[[#right]] = OpLabel
+; CHECK-DAG:               OpStore %[[#switch_0]] %[[#int_0]]
+; CHECK-DAG:               OpStore %[[#variable]] %[[#int_2]]
+; CHECK:                   OpBranchConditional %[[#]] %[[#inner_merge]] %[[#right_next:]]
+right:
+  store i32 2, ptr %var
+  br i1 true, label %merge, label %right_next
+
+; CHECK:     %[[#left_next]] = OpLabel
+; CHECK-DAG:                   OpStore %[[#switch_0]] %[[#int_1]]
+; CHECK-DAG:                   OpStore %[[#variable]] %[[#int_3]]
+; CHECK:                       OpBranch %[[#inner_merge]]
+left_next:
+  store i32 3, ptr %var
+  br label %continue
+
+; CHECK:     %[[#right_next]] = OpLabel
+; CHECK-DAG:                    OpStore %[[#switch_0]] %[[#int_1]]
+; CHECK-DAG:                    OpStore %[[#variable]] %[[#int_4]]
+; CHECK:                        OpBranch %[[#inner_merge]]
+right_next:
+  store i32 4, ptr %var
+  br label %continue
+
+; CHECK: %[[#inner_merge]] = OpLabel
+; CHECK:        %[[#tmp:]] = OpLoad %[[#]] %[[#switch_0]]
+; CHECK:       %[[#cond:]] = OpIEqual %[[#]] %[[#int_0]] %[[#tmp]]
+; CHECK:                     OpBranchConditional %[[#cond]] %[[#merge]] %[[#continue]]
+
+; CHECK: %[[#continue]] = OpLabel
+; CHECK:                  OpBranch %[[#header]]
+continue:
+  br label %header
+
+; CHECK: %[[#merge]] = OpLabel
+; CHECK:               OpReturn
+merge:
+  ret void
+}
+
+
+declare token @llvm.experimental.convergence.entry() #0
+declare token @llvm.experimental.convergence.loop() #0
+
+attributes #0 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #1 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll
index a9a0397718e1..b421ae7990c6 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll
@@ -1,23 +1,23 @@
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 target triple = "spirv-unknown-vulkan-compute"
 
 define internal spir_func void @main() #0 {
 
-; CHECK:                      OpDecorate %[[#builtin:]] BuiltIn SubgroupLocalInvocationId
-; CHECK-DAG:  %[[#int_ty:]] = OpTypeInt 32 0
-; CHECK-DAG: %[[#int_fpty:]] = OpTypePointer Function %[[#int_ty]]
+; CHECK-DAG:                   OpName %[[#idx:]] "idx"
+; CHECK-DAG:                   OpDecorate %[[#builtin:]] BuiltIn SubgroupLocalInvocationId
+; CHECK-DAG:   %[[#int_ty:]] = OpTypeInt 32 0
 ; CHECK-DAG: %[[#int_ipty:]] = OpTypePointer Input %[[#int_ty]]
-; CHECK-DAG: %[[#bool_ty:]] = OpTypeBool
-; CHECK-DAG:   %[[#int_0:]] = OpConstant %[[#int_ty]] 0
-; CHECK-DAG:  %[[#int_10:]] = OpConstant %[[#int_ty]] 10
-; CHECK-DAG:  %[[#builtin]] = OpVariable %[[#int_ipty]] Input
+; CHECK-DAG:  %[[#bool_ty:]] = OpTypeBool
+; CHECK-DAG:    %[[#int_0:]] = OpConstant %[[#int_ty]] 0
+; CHECK-DAG:   %[[#int_10:]] = OpConstant %[[#int_ty]] 10
+; CHECK-DAG:   %[[#builtin]] = OpVariable %[[#int_ipty]] Input
 
 ; CHECK:   %[[#entry:]] = OpLabel
-; CHECK:     %[[#idx:]] = OpVariable %[[#int_fpty]] Function
-; CHECK:                  OpStore %[[#idx]] %[[#int_0]] Aligned 4
+; CHECK:      %[[#idx]] = OpVariable %[[#]] Function
+; ACHECK:                 OpStore %[[#idx]] %[[#int_0]] Aligned 4
 ; CHECK:                  OpBranch %[[#while_cond:]]
 entry:
   %0 = call token @llvm.experimental.convergence.entry()
@@ -37,12 +37,12 @@ while.cond:
   %cmp = icmp ne i32 %2, 10
   br i1 %cmp, label %while.body, label %while.end
 
-; CHECK:   %[[#while_body]] = OpLabel
-; CHECK-NEXT:    %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
-; CHECK-NEXT:                 OpStore %[[#idx]] %[[#tmp]] Aligned 4
-; CHECK-NEXT:    %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
-; CHECK-NEXT:   %[[#cmp1:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
-; CHECK:                      OpBranchConditional %[[#cmp1]] %[[#new_end]] %[[#if_end]]
+; CHECK: %[[#while_body]] = OpLabel
+; CHECK:       %[[#tmp:]] = OpLoad %[[#]] %[[#builtin]] Aligned 1
+; CHECK:                    OpStore %[[#idx]] %[[#tmp]] Aligned 4
+; CHECK:       %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
+; CHECK:      %[[#cmp1:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
+; CHECK:                    OpBranchConditional %[[#cmp1]] %[[#new_end]] %[[#if_end]]
 while.body:
   %3 = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %1) ]
   store i32 %3, ptr %idx, align 4
@@ -50,14 +50,14 @@ while.body:
   %cmp1 = icmp eq i32 %4, 0
   br i1 %cmp1, label %if.then, label %if.end
 
+; CHECK:   %[[#if_end]] = OpLabel
+; CHECK:                  OpBranch %[[#while_cond]]
+
 ; CHECK:   %[[#new_end]] = OpLabel
 ; CHECK:                   OpBranch %[[#while_end:]]
-
 if.then:
   br label %while.end
 
-; CHECK:   %[[#if_end]] = OpLabel
-; CHECK:                  OpBranch %[[#while_cond]]
 if.end:
   br label %while.cond
 
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll
index 3db7545b8178..ac330a96444b 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll
@@ -1,21 +1,20 @@
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 target triple = "spirv-unknown-vulkan-compute"
 
 define internal spir_func void @main() #0 {
 
-; CHECK:                      OpDecorate %[[#builtin:]] BuiltIn SubgroupLocalInvocationId
-
+; CHECK-DAG:                   OpName %[[#idx:]] "idx"
+; CHECK-DAG:                      OpDecorate %[[#builtin:]] BuiltIn SubgroupLocalInvocationId
 ; CHECK-DAG:  %[[#int_ty:]] = OpTypeInt 32 0
-; CHECK-DAG: %[[#pint_ty:]] = OpTypePointer Function %[[#int_ty]]
 ; CHECK-DAG: %[[#bool_ty:]] = OpTypeBool
 ; CHECK-DAG:   %[[#int_0:]] = OpConstant %[[#int_ty]] 0
 ; CHECK-DAG:  %[[#int_10:]] = OpConstant %[[#int_ty]] 10
 
 ; CHECK:   %[[#entry:]] = OpLabel
-; CHECK:     %[[#idx:]] = OpVariable %[[#pint_ty]] Function
+; CHECK:      %[[#idx]] = OpVariable %[[#]] Function
 ; CHECK:                  OpStore %[[#idx]] %[[#int_0]] Aligned 4
 ; CHECK:                  OpBranch %[[#while_cond:]]
 entry:
@@ -48,17 +47,18 @@ while.body:
   %cmp1 = icmp eq i32 %4, 0
   br i1 %cmp1, label %if.then, label %if.end
 
+; CHECK: %[[#if_end]] = OpLabel
+; CHECK:                OpBranch %[[#while_cond]]
+
 ; CHECK:      %[[#if_then]] = OpLabel
-; CHECK-NEXT:    %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
-; CHECK-NEXT:                 OpStore %[[#idx]] %[[#tmp]] Aligned 4
+; CHECK:         %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
+; CHECK:                      OpStore %[[#idx]] %[[#tmp]] Aligned 4
 ; CHECK:                      OpBranch %[[#new_end]]
 if.then:
   %5 = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %1) ]
   store i32 %5, ptr %idx, align 4
   br label %while.end
 
-; CHECK: %[[#if_end]] = OpLabel
-; CHECK:                OpBranch %[[#while_cond]]
 if.end:
   br label %while.cond
 
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll
index d25b30df45ae..784bd38a6fba 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll
@@ -1,14 +1,15 @@
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 target triple = "spirv-unknown-vulkan-compute"
 
 define internal spir_func void @main() #0 {
 
-; CHECK:                      OpDecorate %[[#builtin:]] BuiltIn SubgroupLocalInvocationId
+; CHECK-DAG:                  OpName %[[#idx:]] "idx"
+; CHECK-DAG:                  OpName %[[#reg_0:]] "reg"
+; CHECK-DAG:                  OpDecorate %[[#builtin:]] BuiltIn SubgroupLocalInvocationId
 ; CHECK-DAG:  %[[#int_ty:]] = OpTypeInt 32 0
-; CHECK-DAG: %[[#pint_ty:]] = OpTypePointer Function %[[#int_ty]]
 ; CHECK-DAG: %[[#bool_ty:]] = OpTypeBool
 ; CHECK-DAG:   %[[#int_0:]] = OpConstant %[[#int_ty]] 0
 ; CHECK-DAG:   %[[#int_1:]] = OpConstant %[[#int_ty]] 1
@@ -16,7 +17,7 @@ define internal spir_func void @main() #0 {
 ; CHECK-DAG:  %[[#int_10:]] = OpConstant %[[#int_ty]] 10
 
 ; CHECK:   %[[#entry:]] = OpLabel
-; CHECK:     %[[#idx:]] = OpVariable %[[#pint_ty]] Function
+; CHECK:      %[[#idx]] = OpVariable %[[#]] Function
 ; CHECK:                  OpStore %[[#idx]] %[[#int_0]] Aligned 4
 ; CHECK:                  OpBranch %[[#while_cond:]]
 entry:
@@ -26,6 +27,7 @@ entry:
   br label %while.cond
 
 ; CHECK:   %[[#while_cond]] = OpLabel
+; CHECK:                      OpStore %[[#reg_0]] %[[#]] Aligned 4
 ; CHECK:         %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
 ; CHECK:         %[[#cmp:]] = OpINotEqual %[[#bool_ty]] %[[#tmp]] %[[#int_10]]
 ; CHECK:                      OpLoopMerge %[[#new_end:]] %[[#if_end2:]] None
@@ -37,10 +39,11 @@ while.cond:
   br i1 %cmp, label %while.body, label %while.end
 
 ; CHECK:   %[[#while_body]] = OpLabel
-; CHECK-NEXT:    %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
-; CHECK-NEXT:                 OpStore %[[#idx]] %[[#tmp]] Aligned 4
-; CHECK-NEXT:    %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
-; CHECK-NEXT:   %[[#cmp1:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
+; CHECK:                      OpStore %[[#reg_0]] %[[#]] Aligned 4
+; CHECK:         %[[#tmp:]] = OpLoad %[[#]] %[[#builtin]] Aligned 1
+; CHECK:                      OpStore %[[#idx]] %[[#tmp]] Aligned 4
+; CHECK:         %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
+; CHECK:        %[[#cmp1:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
 ; CHECK:                      OpBranchConditional %[[#cmp1]] %[[#new_end]] %[[#if_end:]]
 while.body:
   %3 = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %1) ]
@@ -50,10 +53,11 @@ while.body:
   br i1 %cmp1, label %if.then, label %if.end
 
 ; CHECK:               %[[#if_end]] = OpLabel
-; CHECK-NEXT:            %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
-; CHECK-NEXT:                         OpStore %[[#idx]] %[[#tmp]] Aligned 4
-; CHECK-NEXT:            %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
-; CHECK-NEXT:           %[[#cmp2:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
+; CHECK:                              OpStore %[[#reg_0]] %[[#]] Aligned 4
+; CHECK:                 %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
+; CHECK:                              OpStore %[[#idx]] %[[#tmp]] Aligned 4
+; CHECK:                 %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
+; CHECK:                %[[#cmp2:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
 ; CHECK:                              OpBranchConditional %[[#cmp2]] %[[#new_end]] %[[#if_end2]]
 if.end:
   %5 = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %1) ]
@@ -62,14 +66,15 @@ if.end:
   %cmp2 = icmp eq i32 %6, 0
   br i1 %cmp2, label %if.then2, label %if.end2
 
+; CHECK:   %[[#if_end2]] = OpLabel
+; CHECK:                   OpBranch %[[#while_cond]]
+
 ; TODO: this OpSwitch is useless. Improve the "remove useless branches" step of the structurizer to
 ;       cleanup those.
 ; CHECK:   %[[#new_end]] = OpLabel
-; CHECK:    %[[#route:]] = OpPhi %[[#int_ty]] %[[#int_0]] %[[#while_cond]] %[[#int_1]] %[[#while_body]] %[[#int_2]] %[[#if_end]]
+; CHECK:    %[[#route:]] = OpLoad %[[#]] %[[#reg_0]] Aligned 4
 ; CHECK:                   OpSwitch %[[#route]] %[[#while_end:]] 1 %[[#while_end:]] 2 %[[#while_end:]]
 
-; CHECK:   %[[#if_end2]] = OpLabel
-; CHECK:                   OpBranch %[[#while_cond]]
 if.end2:
   br label %while.cond
 
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/phi-exit.ll b/llvm/test/CodeGen/SPIRV/structurizer/phi-exit.ll
new file mode 100644
index 000000000000..541b23a6495c
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/phi-exit.ll
@@ -0,0 +1,45 @@
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+define internal spir_func void @main() #1 {
+; CHECK: %[[#entry:]] = OpLabel
+; CHECK:                OpBranch %[[#do_body:]]
+entry:
+  %0 = call token @llvm.experimental.convergence.entry()
+  %a = alloca i32, align 4
+  br label %loop_body
+
+loop_body:
+  br i1 true, label %left, label %right
+
+left:
+  br i1 true, label %loop_exit, label %loop_continue
+
+right:
+  br i1 true, label %loop_exit, label %loop_continue
+
+loop_continue:
+  br label %loop_body
+
+loop_exit:
+  %r = phi i32 [ 0, %left ], [ 1, %right ]
+  store i32 %r, ptr %a, align 4
+  ret void
+
+}
+
+
+declare token @llvm.experimental.convergence.entry() #0
+declare token @llvm.experimental.convergence.loop() #0
+
+attributes #0 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #1 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll b/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll
index 6f60538153df..e27aca878428 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll
@@ -1,6 +1,9 @@
-; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
 
+; CHECK-DAG:         OpName %[[#reg_0:]] "reg2"
+; CHECK-DAG:         OpName %[[#reg_1:]] "reg1"
+
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 target triple = "spirv-unknown-vulkan1.3-compute"
 
@@ -37,36 +40,40 @@ while.body:
   ]
 
 ; CHECK: %[[#case_1]] = OpLabel
+; CHECK:                OpStore %[[#reg_0]] %[[#]]
 ; CHECK:                OpBranch %[[#switch_end]]
 sw.bb:
   store i32 1, ptr %a, align 4
   br label %while.end
 
 ; CHECK: %[[#case_2]] = OpLabel
+; CHECK:                OpStore %[[#reg_0]] %[[#]]
 ; CHECK:                OpBranch %[[#switch_end]]
 sw.bb1:
   store i32 3, ptr %a, align 4
   br label %while.end
 
 ; CHECK: %[[#case_5]] = OpLabel
+; CHECK:                OpStore %[[#reg_0]] %[[#]]
 ; CHECK:                OpBranch %[[#switch_end]]
 sw.bb2:
   store i32 5, ptr %a, align 4
   br label %while.end
 
 ; CHECK: %[[#switch_end]] = OpLabel
-; CHECK:       %[[#phi:]] = OpPhi %[[#type:]] %[[#A:]] %[[#while_body]] %[[#B:]] %[[#case_5]] %[[#B:]] %[[#case_2]] %[[#B:]] %[[#case_1]]
-; CHECK:       %[[#tmp:]] = OpIEqual %[[#type:]] %[[#A]] %[[#phi]]
+; CHECK:       %[[#val:]] = OpLoad %[[#]] %[[#reg_0]]
+; CHECK:       %[[#tmp:]] = OpIEqual %[[#type:]] %[[#]] %[[#val]]
 ; CHECK:                    OpBranchConditional %[[#tmp]] %[[#sw_default:]] %[[#while_end]]
 
 ; CHECK: %[[#sw_default]] = OpLabel
-; CHECK:                    OpStore %[[#A:]] %[[#B:]] Aligned 4
+; CHECK:                    OpStore %[[#]] %[[#B:]] Aligned 4
 ; CHECK:                    OpBranch %[[#for_cond:]]
 sw.default:
   store i32 0, ptr %i, align 4
   br label %for.cond
 
 ; CHECK: %[[#for_cond]] = OpLabel
+; CHECK:                  OpStore %[[#reg_1]] %[[#]]
 ; CHECK:                  OpSelectionMerge %[[#for_merge:]] None
 ; CHECK-NEXT:             OpBranchConditional %[[#cond:]] %[[#for_merge]] %[[#for_end:]]
 for.cond:
@@ -76,13 +83,14 @@ for.cond:
   br i1 %cmp, label %for.body, label %for.end
 
 ; CHECK: %[[#for_end]] = OpLabel
+; CHECK:                 OpStore %[[#reg_1]] %[[#]]
 ; CHECK:                 OpBranch %[[#for_merge]]
 for.end:
   br label %while.end
 
 ; CHECK: %[[#for_merge]] = OpLabel
-; CHECK:      %[[#phi:]] = OpPhi %[[#type:]] %[[#A:]] %[[#for_cond]] %[[#B:]] %[[#for_end]]
-; CHECK:      %[[#tmp:]] = OpIEqual %[[#type:]] %[[#A]] %[[#phi]]
+; CHECK:      %[[#val:]] = OpLoad %[[#]] %[[#reg_1]]
+; CHECK:      %[[#tmp:]] = OpIEqual %[[#type:]] %[[#]] %[[#val]]
 ; CHECK:                   OpBranchConditional %[[#tmp]] %[[#for_body:]] %[[#while_end]]
 
 ; CHECK: %[[#for_body]] = OpLabel
-- 
GitLab


From e6e04633c678901b012adcf53e33592b08ff5baf Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Wed, 30 Oct 2024 07:02:07 -0700
Subject: [PATCH 583/693] [NFC] Delete unused
 clang-formatted-file.txt/ClangFormattedStatus.rst files (#109220)

Looks like these files are generated by the
`generate_formatted_state.py` script as a "status report" of state of
clang-format compliance of files in the LLVM git repo. As such, they do
not belong to the repo itself, so deleting them.

Please see:
https://discourse.llvm.org/t/clang-docs-tools-clang-formatted-files-txt/82803
---
 clang/docs/ClangFormattedStatus.rst        | 8536 -------------------
 clang/docs/tools/clang-formatted-files.txt | 8827 --------------------
 2 files changed, 17363 deletions(-)
 delete mode 100644 clang/docs/ClangFormattedStatus.rst
 delete mode 100644 clang/docs/tools/clang-formatted-files.txt

diff --git a/clang/docs/ClangFormattedStatus.rst b/clang/docs/ClangFormattedStatus.rst
deleted file mode 100644
index 2475a5d4b277..000000000000
--- a/clang/docs/ClangFormattedStatus.rst
+++ /dev/null
@@ -1,8536 +0,0 @@
-.. raw:: html
-
-      <style type="text/css">
-        .total { font-weight: bold; }
-        .none { background-color: #FFFF99; height: 20px; display: inline-block; width: 120px; text-align: center; border-radius: 5px; color: #000000; font-family="Verdana,Geneva,DejaVu Sans,sans-serif" }
-        .part { background-color: #FFCC99; height: 20px; display: inline-block; width: 120px; text-align: center; border-radius: 5px; color: #000000; font-family="Verdana,Geneva,DejaVu Sans,sans-serif" }
-        .good { background-color: #2CCCFF; height: 20px; display: inline-block; width: 120px; text-align: center; border-radius: 5px; color: #000000; font-family="Verdana,Geneva,DejaVu Sans,sans-serif" }
-      </style>
-
-.. role:: none
-.. role:: part
-.. role:: good
-.. role:: total
-
-======================
-Clang Formatted Status
-======================
-
-:doc:`ClangFormattedStatus` describes the state of LLVM source
-tree in terms of conformance to :doc:`ClangFormat` as of: March 06, 2022 17:32:26 (`830ba4cebe79 <https://github.com/llvm/llvm-project/commit/830ba4cebe79>`_).
-
-
-.. list-table:: LLVM Clang-Format Status
-   :widths: 50 25 25 25 25
-   :header-rows: 1
-
-   * - Directory
-     - Total Files
-     - Formatted Files
-     - Unformatted Files
-     - % Complete
-   * - bolt/include/bolt/Core
-     - `15`
-     - `10`
-     - `5`
-     - :part:`66%`
-   * - bolt/include/bolt/Passes
-     - `47`
-     - `47`
-     - `0`
-     - :good:`100%`
-   * - bolt/include/bolt/Profile
-     - `8`
-     - `8`
-     - `0`
-     - :good:`100%`
-   * - bolt/include/bolt/Rewrite
-     - `5`
-     - `4`
-     - `1`
-     - :part:`80%`
-   * - bolt/include/bolt/RuntimeLibs
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - bolt/include/bolt/Utils
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - bolt/lib/Core
-     - `14`
-     - `5`
-     - `9`
-     - :part:`35%`
-   * - bolt/lib/Passes
-     - `45`
-     - `21`
-     - `24`
-     - :part:`46%`
-   * - bolt/lib/Profile
-     - `7`
-     - `3`
-     - `4`
-     - :part:`42%`
-   * - bolt/lib/Rewrite
-     - `6`
-     - `0`
-     - `6`
-     - :none:`0%`
-   * - bolt/lib/RuntimeLibs
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - bolt/lib/Target/AArch64
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - bolt/lib/Target/X86
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - bolt/lib/Utils
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - bolt/runtime
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - bolt/tools/driver
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - bolt/tools/heatmap
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - bolt/tools/llvm-bolt-fuzzer
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - bolt/tools/merge-fdata
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - bolt/unittests/Core
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/bindings/python/tests/cindex/INPUTS
-     - `5`
-     - `3`
-     - `2`
-     - :part:`60%`
-   * - clang/docs/analyzer/checkers
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - clang/examples/AnnotateFunctions
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/examples/Attribute
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/examples/CallSuperAttribute
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/examples/PluginsOrder
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/examples/PrintFunctionNames
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/include/clang/Analysis
-     - `16`
-     - `4`
-     - `12`
-     - :part:`25%`
-   * - clang/include/clang/Analysis/Analyses
-     - `15`
-     - `3`
-     - `12`
-     - :part:`20%`
-   * - clang/include/clang/Analysis/DomainSpecific
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - clang/include/clang/Analysis/FlowSensitive
-     - `16`
-     - `15`
-     - `1`
-     - :part:`93%`
-   * - clang/include/clang/Analysis/Support
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/include/clang/APINotes
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - clang/include/clang/ARCMigrate
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - clang/include/clang/AST
-     - `114`
-     - `20`
-     - `94`
-     - :part:`17%`
-   * - clang/include/clang/ASTMatchers
-     - `5`
-     - `1`
-     - `4`
-     - :part:`20%`
-   * - clang/include/clang/ASTMatchers/Dynamic
-     - `4`
-     - `1`
-     - `3`
-     - :part:`25%`
-   * - clang/include/clang/Basic
-     - `82`
-     - `32`
-     - `50`
-     - :part:`39%`
-   * - clang/include/clang/CodeGen
-     - `9`
-     - `0`
-     - `9`
-     - :none:`0%`
-   * - clang/include/clang/CrossTU
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - clang/include/clang/DirectoryWatcher
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/include/clang/Driver
-     - `17`
-     - `4`
-     - `13`
-     - :part:`23%`
-   * - clang/include/clang/Edit
-     - `5`
-     - `1`
-     - `4`
-     - :part:`20%`
-   * - clang/include/clang/Format
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/include/clang/Frontend
-     - `28`
-     - `7`
-     - `21`
-     - :part:`25%`
-   * - clang/include/clang/FrontendTool
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/include/clang/Index
-     - `7`
-     - `2`
-     - `5`
-     - :part:`28%`
-   * - clang/include/clang/IndexSerialization
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/include/clang/Interpreter
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - clang/include/clang/Lex
-     - `29`
-     - `6`
-     - `23`
-     - :part:`20%`
-   * - clang/include/clang/Parse
-     - `5`
-     - `2`
-     - `3`
-     - :part:`40%`
-   * - clang/include/clang/Rewrite/Core
-     - `6`
-     - `0`
-     - `6`
-     - :none:`0%`
-   * - clang/include/clang/Rewrite/Frontend
-     - `4`
-     - `0`
-     - `4`
-     - :none:`0%`
-   * - clang/include/clang/Sema
-     - `32`
-     - `3`
-     - `29`
-     - :part:`9%`
-   * - clang/include/clang/Serialization
-     - `14`
-     - `3`
-     - `11`
-     - :part:`21%`
-   * - clang/include/clang/StaticAnalyzer/Checkers
-     - `4`
-     - `1`
-     - `3`
-     - :part:`25%`
-   * - clang/include/clang/StaticAnalyzer/Core
-     - `5`
-     - `1`
-     - `4`
-     - :part:`20%`
-   * - clang/include/clang/StaticAnalyzer/Core/BugReporter
-     - `4`
-     - `1`
-     - `3`
-     - :part:`25%`
-   * - clang/include/clang/StaticAnalyzer/Core/PathSensitive
-     - `37`
-     - `10`
-     - `27`
-     - :part:`27%`
-   * - clang/include/clang/StaticAnalyzer/Frontend
-     - `5`
-     - `2`
-     - `3`
-     - :part:`40%`
-   * - clang/include/clang/Testing
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - clang/include/clang/Tooling
-     - `17`
-     - `10`
-     - `7`
-     - :part:`58%`
-   * - clang/include/clang/Tooling/ASTDiff
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - clang/include/clang/Tooling/Core
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - clang/include/clang/Tooling/DependencyScanning
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - clang/include/clang/Tooling/Inclusions
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - clang/include/clang/Tooling/Refactoring
-     - `15`
-     - `12`
-     - `3`
-     - :part:`80%`
-   * - clang/include/clang/Tooling/Refactoring/Extract
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - clang/include/clang/Tooling/Refactoring/Rename
-     - `6`
-     - `5`
-     - `1`
-     - :part:`83%`
-   * - clang/include/clang/Tooling/Syntax
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - clang/include/clang/Tooling/Syntax/Pseudo
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - clang/include/clang/Tooling/Transformer
-     - `8`
-     - `6`
-     - `2`
-     - :part:`75%`
-   * - clang/include/clang-c
-     - `10`
-     - `3`
-     - `7`
-     - :part:`30%`
-   * - clang/INPUTS
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - clang/lib/Analysis
-     - `28`
-     - `3`
-     - `25`
-     - :part:`10%`
-   * - clang/lib/Analysis/FlowSensitive
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - clang/lib/Analysis/plugins/CheckerDependencyHandling
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/lib/Analysis/plugins/CheckerOptionHandling
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/lib/Analysis/plugins/SampleAnalyzer
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/lib/APINotes
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - clang/lib/ARCMigrate
-     - `22`
-     - `0`
-     - `22`
-     - :none:`0%`
-   * - clang/lib/AST
-     - `81`
-     - `2`
-     - `79`
-     - :part:`2%`
-   * - clang/lib/AST/ByteCode
-     - `44`
-     - `18`
-     - `26`
-     - :part:`40%`
-   * - clang/lib/ASTMatchers
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - clang/lib/ASTMatchers/Dynamic
-     - `6`
-     - `1`
-     - `5`
-     - :part:`16%`
-   * - clang/lib/Basic
-     - `39`
-     - `13`
-     - `26`
-     - :part:`33%`
-   * - clang/lib/Basic/Targets
-     - `50`
-     - `25`
-     - `25`
-     - :part:`50%`
-   * - clang/lib/CodeGen
-     - `87`
-     - `9`
-     - `78`
-     - :part:`10%`
-   * - clang/lib/CrossTU
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/lib/DirectoryWatcher
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - clang/lib/DirectoryWatcher/default
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/lib/DirectoryWatcher/linux
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/lib/DirectoryWatcher/mac
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/lib/DirectoryWatcher/windows
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/lib/Driver
-     - `14`
-     - `2`
-     - `12`
-     - :part:`14%`
-   * - clang/lib/Driver/ToolChains
-     - `94`
-     - `41`
-     - `53`
-     - :part:`43%`
-   * - clang/lib/Driver/ToolChains/Arch
-     - `20`
-     - `7`
-     - `13`
-     - :part:`35%`
-   * - clang/lib/Edit
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - clang/lib/Format
-     - `35`
-     - `35`
-     - `0`
-     - :good:`100%`
-   * - clang/lib/Frontend
-     - `32`
-     - `4`
-     - `28`
-     - :part:`12%`
-   * - clang/lib/Frontend/Rewrite
-     - `8`
-     - `0`
-     - `8`
-     - :none:`0%`
-   * - clang/lib/FrontendTool
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/lib/Headers
-     - `146`
-     - `14`
-     - `132`
-     - :part:`9%`
-   * - clang/lib/Headers/openmp_wrappers
-     - `5`
-     - `4`
-     - `1`
-     - :part:`80%`
-   * - clang/lib/Headers/ppc_wrappers
-     - `7`
-     - `2`
-     - `5`
-     - :part:`28%`
-   * - clang/lib/Index
-     - `11`
-     - `2`
-     - `9`
-     - :part:`18%`
-   * - clang/lib/IndexSerialization
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/lib/Interpreter
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - clang/lib/Lex
-     - `24`
-     - `1`
-     - `23`
-     - :part:`4%`
-   * - clang/lib/Parse
-     - `15`
-     - `1`
-     - `14`
-     - :part:`6%`
-   * - clang/lib/Rewrite
-     - `5`
-     - `0`
-     - `5`
-     - :none:`0%`
-   * - clang/lib/Sema
-     - `55`
-     - `4`
-     - `51`
-     - :part:`7%`
-   * - clang/lib/Serialization
-     - `17`
-     - `2`
-     - `15`
-     - :part:`11%`
-   * - clang/lib/StaticAnalyzer/Checkers
-     - `122`
-     - `19`
-     - `103`
-     - :part:`15%`
-   * - clang/lib/StaticAnalyzer/Checkers/cert
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - clang/lib/StaticAnalyzer/Checkers/MPI-Checker
-     - `6`
-     - `0`
-     - `6`
-     - :none:`0%`
-   * - clang/lib/StaticAnalyzer/Checkers/RetainCountChecker
-     - `4`
-     - `0`
-     - `4`
-     - :none:`0%`
-   * - clang/lib/StaticAnalyzer/Checkers/UninitializedObject
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - clang/lib/StaticAnalyzer/Checkers/WebKit
-     - `10`
-     - `8`
-     - `2`
-     - :part:`80%`
-   * - clang/lib/StaticAnalyzer/Core
-     - `47`
-     - `10`
-     - `37`
-     - :part:`21%`
-   * - clang/lib/StaticAnalyzer/Frontend
-     - `8`
-     - `3`
-     - `5`
-     - :part:`37%`
-   * - clang/lib/Testing
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/lib/Tooling
-     - `16`
-     - `7`
-     - `9`
-     - :part:`43%`
-   * - clang/lib/Tooling/ASTDiff
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/lib/Tooling/Core
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - clang/lib/Tooling/DependencyScanning
-     - `5`
-     - `4`
-     - `1`
-     - :part:`80%`
-   * - clang/lib/Tooling/DumpTool
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - clang/lib/Tooling/Inclusions
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - clang/lib/Tooling/Refactoring
-     - `5`
-     - `3`
-     - `2`
-     - :part:`60%`
-   * - clang/lib/Tooling/Refactoring/Extract
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - clang/lib/Tooling/Refactoring/Rename
-     - `5`
-     - `2`
-     - `3`
-     - :part:`40%`
-   * - clang/lib/Tooling/Syntax
-     - `7`
-     - `6`
-     - `1`
-     - :part:`85%`
-   * - clang/lib/Tooling/Syntax/Pseudo
-     - `8`
-     - `8`
-     - `0`
-     - :good:`100%`
-   * - clang/lib/Tooling/Transformer
-     - `7`
-     - `4`
-     - `3`
-     - :part:`57%`
-   * - clang/tools/amdgpu-arch
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/tools/apinotes-test
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/tools/arcmt-test
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/tools/c-index-test
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/tools/clang-check
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/tools/clang-diff
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/tools/clang-extdef-mapping
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/tools/clang-format
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/tools/clang-format/fuzzer
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/tools/clang-fuzzer
-     - `6`
-     - `4`
-     - `2`
-     - :part:`66%`
-   * - clang/tools/clang-fuzzer/fuzzer-initialize
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - clang/tools/clang-fuzzer/handle-cxx
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - clang/tools/clang-fuzzer/handle-llvm
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - clang/tools/clang-fuzzer/proto-to-cxx
-     - `5`
-     - `0`
-     - `5`
-     - :none:`0%`
-   * - clang/tools/clang-fuzzer/proto-to-llvm
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - clang/tools/clang-import-test
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/tools/clang-linker-wrapper
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - clang/tools/clang-nvlink-wrapper
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/tools/clang-offload-bundler
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/tools/clang-offload-wrapper
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/tools/clang-refactor
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - clang/tools/clang-repl
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/tools/clang-scan-deps
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/tools/clang-shlib
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/tools/diagtool
-     - `9`
-     - `0`
-     - `9`
-     - :none:`0%`
-   * - clang/tools/driver
-     - `4`
-     - `1`
-     - `3`
-     - :part:`25%`
-   * - clang/tools/libclang
-     - `35`
-     - `5`
-     - `30`
-     - :part:`14%`
-   * - clang/tools/scan-build-py/tests/functional/src/include
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/unittests/Analysis
-     - `6`
-     - `2`
-     - `4`
-     - :part:`33%`
-   * - clang/unittests/Analysis/FlowSensitive
-     - `14`
-     - `13`
-     - `1`
-     - :part:`92%`
-   * - clang/unittests/AST
-     - `30`
-     - `8`
-     - `22`
-     - :part:`26%`
-   * - clang/unittests/ASTMatchers
-     - `6`
-     - `3`
-     - `3`
-     - :part:`50%`
-   * - clang/unittests/ASTMatchers/Dynamic
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - clang/unittests/Basic
-     - `8`
-     - `4`
-     - `4`
-     - :part:`50%`
-   * - clang/unittests/CodeGen
-     - `6`
-     - `1`
-     - `5`
-     - :part:`16%`
-   * - clang/unittests/CrossTU
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/unittests/DirectoryWatcher
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/unittests/Driver
-     - `5`
-     - `1`
-     - `4`
-     - :part:`20%`
-   * - clang/unittests/Format
-     - `24`
-     - `24`
-     - `0`
-     - :good:`100%`
-   * - clang/unittests/Frontend
-     - `11`
-     - `7`
-     - `4`
-     - :part:`63%`
-   * - clang/unittests/Index
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/unittests/Interpreter
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - clang/unittests/Interpreter/ExceptionTests
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/unittests/Introspection
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/unittests/Lex
-     - `8`
-     - `4`
-     - `4`
-     - :part:`50%`
-   * - clang/unittests/libclang
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - clang/unittests/libclang/CrashTests
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang/unittests/Rename
-     - `6`
-     - `0`
-     - `6`
-     - :none:`0%`
-   * - clang/unittests/Rewrite
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - clang/unittests/Sema
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - clang/unittests/Serialization
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - clang/unittests/StaticAnalyzer
-     - `16`
-     - `7`
-     - `9`
-     - :part:`43%`
-   * - clang/unittests/Tooling
-     - `30`
-     - `10`
-     - `20`
-     - :part:`33%`
-   * - clang/unittests/Tooling/RecursiveASTVisitorTests
-     - `30`
-     - `12`
-     - `18`
-     - :part:`40%`
-   * - clang/unittests/Tooling/Syntax
-     - `7`
-     - `3`
-     - `4`
-     - :part:`42%`
-   * - clang/unittests/Tooling/Syntax/Pseudo
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - clang/utils/perf-training/cxx
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang/utils/TableGen
-     - `22`
-     - `3`
-     - `19`
-     - :part:`13%`
-   * - clang-tools-extra/clang-apply-replacements/include/clang-apply-replacements/Tooling
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clang-apply-replacements/lib/Tooling
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clang-apply-replacements/tool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clang-change-namespace
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - clang-tools-extra/clang-change-namespace/tool
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang-tools-extra/clang-doc
-     - `17`
-     - `16`
-     - `1`
-     - :part:`94%`
-   * - clang-tools-extra/clang-doc/tool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clang-include-fixer
-     - `13`
-     - `8`
-     - `5`
-     - :part:`61%`
-   * - clang-tools-extra/clang-include-fixer/find-all-symbols
-     - `17`
-     - `13`
-     - `4`
-     - :part:`76%`
-   * - clang-tools-extra/clang-include-fixer/find-all-symbols/tool
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang-tools-extra/clang-include-fixer/plugin
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clang-include-fixer/tool
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang-tools-extra/clang-move
-     - `4`
-     - `1`
-     - `3`
-     - :part:`25%`
-   * - clang-tools-extra/clang-move/tool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clang-query
-     - `5`
-     - `4`
-     - `1`
-     - :part:`80%`
-   * - clang-tools-extra/clang-query/tool
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang-tools-extra/clang-reorder-fields
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - clang-tools-extra/clang-reorder-fields/tool
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang-tools-extra/clang-tidy
-     - `20`
-     - `14`
-     - `6`
-     - :part:`70%`
-   * - clang-tools-extra/clang-tidy/abseil
-     - `42`
-     - `31`
-     - `11`
-     - :part:`73%`
-   * - clang-tools-extra/clang-tidy/altera
-     - `11`
-     - `9`
-     - `2`
-     - :part:`81%`
-   * - clang-tools-extra/clang-tidy/android
-     - `33`
-     - `23`
-     - `10`
-     - :part:`69%`
-   * - clang-tools-extra/clang-tidy/boost
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clang-tidy/bugprone
-     - `125`
-     - `106`
-     - `19`
-     - :part:`84%`
-   * - clang-tools-extra/clang-tidy/cert
-     - `29`
-     - `28`
-     - `1`
-     - :part:`96%`
-   * - clang-tools-extra/clang-tidy/concurrency
-     - `5`
-     - `4`
-     - `1`
-     - :part:`80%`
-   * - clang-tools-extra/clang-tidy/cppcoreguidelines
-     - `45`
-     - `42`
-     - `3`
-     - :part:`93%`
-   * - clang-tools-extra/clang-tidy/darwin
-     - `5`
-     - `2`
-     - `3`
-     - :part:`40%`
-   * - clang-tools-extra/clang-tidy/fuchsia
-     - `15`
-     - `10`
-     - `5`
-     - :part:`66%`
-   * - clang-tools-extra/clang-tidy/google
-     - `33`
-     - `22`
-     - `11`
-     - :part:`66%`
-   * - clang-tools-extra/clang-tidy/hicpp
-     - `9`
-     - `7`
-     - `2`
-     - :part:`77%`
-   * - clang-tools-extra/clang-tidy/linuxkernel
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - clang-tools-extra/clang-tidy/llvm
-     - `11`
-     - `10`
-     - `1`
-     - :part:`90%`
-   * - clang-tools-extra/clang-tidy/llvmlibc
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clang-tidy/misc
-     - `33`
-     - `30`
-     - `3`
-     - :part:`90%`
-   * - clang-tools-extra/clang-tidy/modernize
-     - `67`
-     - `48`
-     - `19`
-     - :part:`71%`
-   * - clang-tools-extra/clang-tidy/mpi
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clang-tidy/objc
-     - `17`
-     - `12`
-     - `5`
-     - :part:`70%`
-   * - clang-tools-extra/clang-tidy/openmp
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clang-tidy/performance
-     - `31`
-     - `24`
-     - `7`
-     - :part:`77%`
-   * - clang-tools-extra/clang-tidy/plugin
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clang-tidy/portability
-     - `5`
-     - `3`
-     - `2`
-     - :part:`60%`
-   * - clang-tools-extra/clang-tidy/readability
-     - `88`
-     - `76`
-     - `12`
-     - :part:`86%`
-   * - clang-tools-extra/clang-tidy/tool
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - clang-tools-extra/clang-tidy/utils
-     - `35`
-     - `31`
-     - `4`
-     - :part:`88%`
-   * - clang-tools-extra/clang-tidy/zircon
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd
-     - `97`
-     - `81`
-     - `16`
-     - :part:`83%`
-   * - clang-tools-extra/clangd/benchmarks
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/benchmarks/CompletionModel
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang-tools-extra/clangd/fuzzer
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/index
-     - `39`
-     - `36`
-     - `3`
-     - :part:`92%`
-   * - clang-tools-extra/clangd/index/dex
-     - `9`
-     - `7`
-     - `2`
-     - :part:`77%`
-   * - clang-tools-extra/clangd/index/dex/dexp
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/index/remote
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/index/remote/marshalling
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/index/remote/monitor
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/index/remote/server
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/index/remote/unimplemented
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/indexer
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/refactor
-     - `6`
-     - `5`
-     - `1`
-     - :part:`83%`
-   * - clang-tools-extra/clangd/refactor/tweaks
-     - `14`
-     - `10`
-     - `4`
-     - :part:`71%`
-   * - clang-tools-extra/clangd/support
-     - `25`
-     - `24`
-     - `1`
-     - :part:`96%`
-   * - clang-tools-extra/clangd/tool
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/unittests
-     - `79`
-     - `66`
-     - `13`
-     - :part:`83%`
-   * - clang-tools-extra/clangd/unittests/decision_forest_model
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/unittests/remote
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/unittests/support
-     - `11`
-     - `11`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/unittests/tweaks
-     - `20`
-     - `19`
-     - `1`
-     - :part:`95%`
-   * - clang-tools-extra/clangd/unittests/xpc
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/xpc
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/xpc/framework
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/clangd/xpc/test-client
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/modularize
-     - `9`
-     - `1`
-     - `8`
-     - :part:`11%`
-   * - clang-tools-extra/pp-trace
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - clang-tools-extra/tool-template
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/unittests/clang-apply-replacements
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/unittests/clang-change-namespace
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang-tools-extra/unittests/clang-doc
-     - `9`
-     - `9`
-     - `0`
-     - :good:`100%`
-   * - clang-tools-extra/unittests/clang-include-fixer
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - clang-tools-extra/unittests/clang-include-fixer/find-all-symbols
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang-tools-extra/unittests/clang-move
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - clang-tools-extra/unittests/clang-query
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - clang-tools-extra/unittests/clang-tidy
-     - `16`
-     - `9`
-     - `7`
-     - :part:`56%`
-   * - clang-tools-extra/unittests/include/common
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - compiler-rt/include/fuzzer
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - compiler-rt/include/sanitizer
-     - `15`
-     - `3`
-     - `12`
-     - :part:`20%`
-   * - compiler-rt/include/xray
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - compiler-rt/lib/asan
-     - `57`
-     - `5`
-     - `52`
-     - :part:`8%`
-   * - compiler-rt/lib/asan/tests
-     - `17`
-     - `1`
-     - `16`
-     - :part:`5%`
-   * - compiler-rt/lib/BlocksRuntime
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - compiler-rt/lib/builtins
-     - `11`
-     - `9`
-     - `2`
-     - :part:`81%`
-   * - compiler-rt/lib/builtins/arm
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - compiler-rt/lib/builtins/ppc
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - compiler-rt/lib/cfi
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - compiler-rt/lib/dfsan
-     - `14`
-     - `9`
-     - `5`
-     - :part:`64%`
-   * - compiler-rt/lib/fuzzer
-     - `47`
-     - `9`
-     - `38`
-     - :part:`19%`
-   * - compiler-rt/lib/fuzzer/afl
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - compiler-rt/lib/fuzzer/dataflow
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - compiler-rt/lib/fuzzer/tests
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - compiler-rt/lib/gwp_asan
-     - `12`
-     - `12`
-     - `0`
-     - :good:`100%`
-   * - compiler-rt/lib/gwp_asan/optional
-     - `10`
-     - `10`
-     - `0`
-     - :good:`100%`
-   * - compiler-rt/lib/gwp_asan/platform_specific
-     - `13`
-     - `13`
-     - `0`
-     - :good:`100%`
-   * - compiler-rt/lib/gwp_asan/tests
-     - `15`
-     - `14`
-     - `1`
-     - :part:`93%`
-   * - compiler-rt/lib/gwp_asan/tests/platform_specific
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - compiler-rt/lib/hwasan
-     - `30`
-     - `9`
-     - `21`
-     - :part:`30%`
-   * - compiler-rt/lib/interception
-     - `8`
-     - `1`
-     - `7`
-     - :part:`12%`
-   * - compiler-rt/lib/interception/tests
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - compiler-rt/lib/lsan
-     - `20`
-     - `4`
-     - `16`
-     - :part:`20%`
-   * - compiler-rt/lib/memprof
-     - `31`
-     - `29`
-     - `2`
-     - :part:`93%`
-   * - compiler-rt/lib/memprof/tests
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - compiler-rt/lib/msan
-     - `18`
-     - `4`
-     - `14`
-     - :part:`22%`
-   * - compiler-rt/lib/msan/tests
-     - `4`
-     - `0`
-     - `4`
-     - :none:`0%`
-   * - compiler-rt/lib/orc
-     - `21`
-     - `16`
-     - `5`
-     - :part:`76%`
-   * - compiler-rt/lib/orc/unittests
-     - `10`
-     - `9`
-     - `1`
-     - :part:`90%`
-   * - compiler-rt/lib/profile
-     - `6`
-     - `0`
-     - `6`
-     - :none:`0%`
-   * - compiler-rt/lib/safestack
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - compiler-rt/lib/sanitizer_common
-     - `167`
-     - `29`
-     - `138`
-     - :part:`17%`
-   * - compiler-rt/lib/sanitizer_common/symbolizer
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - compiler-rt/lib/sanitizer_common/tests
-     - `46`
-     - `12`
-     - `34`
-     - :part:`26%`
-   * - compiler-rt/lib/scudo
-     - `20`
-     - `0`
-     - `20`
-     - :none:`0%`
-   * - compiler-rt/lib/scudo/standalone
-     - `49`
-     - `48`
-     - `1`
-     - :part:`97%`
-   * - compiler-rt/lib/scudo/standalone/benchmarks
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - compiler-rt/lib/scudo/standalone/fuzz
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - compiler-rt/lib/scudo/standalone/include/scudo
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - compiler-rt/lib/scudo/standalone/tests
-     - `25`
-     - `24`
-     - `1`
-     - :part:`96%`
-   * - compiler-rt/lib/scudo/standalone/tools
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - compiler-rt/lib/stats
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - compiler-rt/lib/tsan/benchmarks
-     - `6`
-     - `0`
-     - `6`
-     - :none:`0%`
-   * - compiler-rt/lib/tsan/dd
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - compiler-rt/lib/tsan/go
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - compiler-rt/lib/tsan/rtl
-     - `59`
-     - `14`
-     - `45`
-     - :part:`23%`
-   * - compiler-rt/lib/tsan/rtl-old
-     - `61`
-     - `13`
-     - `48`
-     - :part:`21%`
-   * - compiler-rt/lib/tsan/tests/rtl
-     - `10`
-     - `0`
-     - `10`
-     - :none:`0%`
-   * - compiler-rt/lib/tsan/tests/unit
-     - `11`
-     - `3`
-     - `8`
-     - :part:`27%`
-   * - compiler-rt/lib/ubsan
-     - `27`
-     - `7`
-     - `20`
-     - :part:`25%`
-   * - compiler-rt/lib/ubsan_minimal
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - compiler-rt/lib/xray
-     - `40`
-     - `27`
-     - `13`
-     - :part:`67%`
-   * - compiler-rt/lib/xray/tests/unit
-     - `10`
-     - `8`
-     - `2`
-     - :part:`80%`
-   * - compiler-rt/tools/gwp_asan
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - cross-project-tests/debuginfo-tests/clang_llvm_roundtrip
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - cross-project-tests/debuginfo-tests/dexter/feature_tests/commands/penalty
-     - `10`
-     - `0`
-     - `10`
-     - :none:`0%`
-   * - cross-project-tests/debuginfo-tests/dexter/feature_tests/commands/perfect
-     - `7`
-     - `0`
-     - `7`
-     - :none:`0%`
-   * - cross-project-tests/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_address
-     - `7`
-     - `0`
-     - `7`
-     - :none:`0%`
-   * - cross-project-tests/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - cross-project-tests/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - cross-project-tests/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/source
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - cross-project-tests/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/source
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - cross-project-tests/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_finish_test
-     - `8`
-     - `0`
-     - `8`
-     - :none:`0%`
-   * - cross-project-tests/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind
-     - `5`
-     - `0`
-     - `5`
-     - :none:`0%`
-   * - cross-project-tests/debuginfo-tests/dexter/feature_tests/commands/perfect/limit_steps
-     - `8`
-     - `2`
-     - `6`
-     - :part:`25%`
-   * - cross-project-tests/debuginfo-tests/dexter/feature_tests/subtools
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - cross-project-tests/debuginfo-tests/dexter/feature_tests/subtools/clang-opt-bisect
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - cross-project-tests/debuginfo-tests/dexter-tests
-     - `15`
-     - `3`
-     - `12`
-     - :part:`20%`
-   * - cross-project-tests/debuginfo-tests/llgdb-tests
-     - `8`
-     - `0`
-     - `8`
-     - :none:`0%`
-   * - cross-project-tests/debuginfo-tests/llvm-prettyprinters/gdb
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - flang/examples
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - flang/examples/FlangOmpReport
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - flang/examples/PrintFlangFunctionNames
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - flang/include/flang
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - flang/include/flang/Common
-     - `21`
-     - `21`
-     - `0`
-     - :good:`100%`
-   * - flang/include/flang/Decimal
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - flang/include/flang/Evaluate
-     - `23`
-     - `23`
-     - `0`
-     - :good:`100%`
-   * - flang/include/flang/Frontend
-     - `11`
-     - `10`
-     - `1`
-     - :part:`90%`
-   * - flang/include/flang/FrontendTool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - flang/include/flang/Lower
-     - `25`
-     - `24`
-     - `1`
-     - :part:`96%`
-   * - flang/include/flang/Lower/Support
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - flang/include/flang/Optimizer/Builder
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - flang/include/flang/Optimizer/Builder/Runtime
-     - `10`
-     - `10`
-     - `0`
-     - :good:`100%`
-   * - flang/include/flang/Optimizer/CodeGen
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - flang/include/flang/Optimizer/Dialect
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - flang/include/flang/Optimizer/Support
-     - `8`
-     - `8`
-     - `0`
-     - :good:`100%`
-   * - flang/include/flang/Optimizer/Transforms
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - flang/include/flang/Parser
-     - `17`
-     - `16`
-     - `1`
-     - :part:`94%`
-   * - flang/include/flang/Runtime
-     - `28`
-     - `27`
-     - `1`
-     - :part:`96%`
-   * - flang/include/flang/Semantics
-     - `9`
-     - `8`
-     - `1`
-     - :part:`88%`
-   * - flang/lib/Common
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - flang/lib/Decimal
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - flang/lib/Evaluate
-     - `33`
-     - `31`
-     - `2`
-     - :part:`93%`
-   * - flang/lib/Frontend
-     - `8`
-     - `6`
-     - `2`
-     - :part:`75%`
-   * - flang/lib/FrontendTool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - flang/lib/Lower
-     - `20`
-     - `20`
-     - `0`
-     - :good:`100%`
-   * - flang/lib/Optimizer/Builder
-     - `6`
-     - `6`
-     - `0`
-     - :good:`100%`
-   * - flang/lib/Optimizer/Builder/Runtime
-     - `9`
-     - `9`
-     - `0`
-     - :good:`100%`
-   * - flang/lib/Optimizer/CodeGen
-     - `10`
-     - `10`
-     - `0`
-     - :good:`100%`
-   * - flang/lib/Optimizer/Dialect
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - flang/lib/Optimizer/Support
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - flang/lib/Optimizer/Transforms
-     - `10`
-     - `10`
-     - `0`
-     - :good:`100%`
-   * - flang/lib/Parser
-     - `35`
-     - `35`
-     - `0`
-     - :good:`100%`
-   * - flang/lib/Semantics
-     - `78`
-     - `69`
-     - `9`
-     - :part:`88%`
-   * - flang/module
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - flang/runtime
-     - `74`
-     - `72`
-     - `2`
-     - :part:`97%`
-   * - flang/tools/bbc
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - flang/tools/f18
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - flang/tools/f18-parse-demo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - flang/tools/fir-opt
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - flang/tools/flang-driver
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - flang/tools/tco
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - flang/unittests/Common
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - flang/unittests/Decimal
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - flang/unittests/Evaluate
-     - `15`
-     - `15`
-     - `0`
-     - :good:`100%`
-   * - flang/unittests/Frontend
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - flang/unittests/Optimizer
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - flang/unittests/Optimizer/Builder
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - flang/unittests/Optimizer/Builder/Runtime
-     - `10`
-     - `10`
-     - `0`
-     - :good:`100%`
-   * - flang/unittests/Runtime
-     - `22`
-     - `22`
-     - `0`
-     - :good:`100%`
-   * - libc/AOR_v20.02/math
-     - `4`
-     - `1`
-     - `3`
-     - :part:`25%`
-   * - libc/AOR_v20.02/math/include
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libc/AOR_v20.02/networking
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libc/AOR_v20.02/networking/include
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libc/AOR_v20.02/string
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libc/AOR_v20.02/string/include
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libc/benchmarks
-     - `15`
-     - `14`
-     - `1`
-     - :part:`93%`
-   * - libc/benchmarks/automemcpy/include/automemcpy
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - libc/benchmarks/automemcpy/lib
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - libc/benchmarks/automemcpy/unittests
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - libc/config/linux
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - libc/fuzzing/math
-     - `6`
-     - `6`
-     - `0`
-     - :good:`100%`
-   * - libc/fuzzing/stdlib
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - libc/fuzzing/string
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - libc/include
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - libc/include/llvm-libc-macros
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - libc/include/llvm-libc-macros/linux
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - libc/include/llvm-libc-types
-     - `28`
-     - `28`
-     - `0`
-     - :good:`100%`
-   * - libc/loader/linux/aarch64
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - libc/loader/linux/x86_64
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - libc/src/assert
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - libc/src/ctype
-     - `32`
-     - `32`
-     - `0`
-     - :good:`100%`
-   * - libc/src/errno
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - libc/src/fcntl
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - libc/src/fcntl/linux
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - libc/src/fenv
-     - `28`
-     - `28`
-     - `0`
-     - :good:`100%`
-   * - libc/src/inttypes
-     - `6`
-     - `6`
-     - `0`
-     - :good:`100%`
-   * - libc/src/math
-     - `91`
-     - `91`
-     - `0`
-     - :good:`100%`
-   * - libc/src/math/aarch64
-     - `10`
-     - `10`
-     - `0`
-     - :good:`100%`
-   * - libc/src/math/generic
-     - `94`
-     - `94`
-     - `0`
-     - :good:`100%`
-   * - libc/src/math/x86_64
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - libc/src/signal
-     - `8`
-     - `8`
-     - `0`
-     - :good:`100%`
-   * - libc/src/signal/linux
-     - `10`
-     - `10`
-     - `0`
-     - :good:`100%`
-   * - libc/src/stdio
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - libc/src/stdlib
-     - `46`
-     - `46`
-     - `0`
-     - :good:`100%`
-   * - libc/src/stdlib/linux
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - libc/src/string
-     - `61`
-     - `61`
-     - `0`
-     - :good:`100%`
-   * - libc/src/string/memory_utils
-     - `8`
-     - `7`
-     - `1`
-     - :part:`87%`
-   * - libc/src/sys/mman
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - libc/src/sys/mman/linux
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - libc/src/sys/stat
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - libc/src/sys/stat/linux
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - libc/src/threads
-     - `16`
-     - `16`
-     - `0`
-     - :good:`100%`
-   * - libc/src/threads/linux
-     - `11`
-     - `7`
-     - `4`
-     - :part:`63%`
-   * - libc/src/time
-     - `12`
-     - `12`
-     - `0`
-     - :good:`100%`
-   * - libc/src/unistd
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - libc/src/unistd/linux
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - libc/src/__support
-     - `10`
-     - `10`
-     - `0`
-     - :good:`100%`
-   * - libc/src/__support/CPP
-     - `11`
-     - `10`
-     - `1`
-     - :part:`90%`
-   * - libc/src/__support/File
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - libc/src/__support/FPUtil
-     - `15`
-     - `14`
-     - `1`
-     - :part:`93%`
-   * - libc/src/__support/FPUtil/aarch64
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - libc/src/__support/FPUtil/generic
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - libc/src/__support/FPUtil/x86_64
-     - `6`
-     - `5`
-     - `1`
-     - :part:`83%`
-   * - libc/src/__support/OSUtil
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - libc/src/__support/OSUtil/linux
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - libc/src/__support/OSUtil/linux/aarch64
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - libc/src/__support/OSUtil/linux/x86_64
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - libc/src/__support/threads
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - libc/src/__support/threads/linux
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - libc/utils/HdrGen
-     - `9`
-     - `9`
-     - `0`
-     - :good:`100%`
-   * - libc/utils/HdrGen/PrototypeTestGen
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - libc/utils/LibcTableGenUtil
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - libc/utils/MPFRWrapper
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - libc/utils/testutils
-     - `10`
-     - `9`
-     - `1`
-     - :part:`90%`
-   * - libc/utils/tools/WrapperGen
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - libc/utils/UnitTest
-     - `12`
-     - `11`
-     - `1`
-     - :part:`91%`
-   * - libclc/generic/include
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - libclc/generic/include/clc
-     - `6`
-     - `2`
-     - `4`
-     - :part:`33%`
-   * - libclc/generic/include/clc/async
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - libclc/generic/include/clc/atomic
-     - `11`
-     - `7`
-     - `4`
-     - :part:`63%`
-   * - libclc/generic/include/clc/cl_khr_global_int32_base_atomics
-     - `6`
-     - `5`
-     - `1`
-     - :part:`83%`
-   * - libclc/generic/include/clc/cl_khr_global_int32_extended_atomics
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - libclc/generic/include/clc/cl_khr_int64_base_atomics
-     - `6`
-     - `3`
-     - `3`
-     - :part:`50%`
-   * - libclc/generic/include/clc/cl_khr_int64_extended_atomics
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - libclc/generic/include/clc/cl_khr_local_int32_base_atomics
-     - `6`
-     - `5`
-     - `1`
-     - :part:`83%`
-   * - libclc/generic/include/clc/cl_khr_local_int32_extended_atomics
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - libclc/generic/include/clc/common
-     - `6`
-     - `6`
-     - `0`
-     - :good:`100%`
-   * - libclc/generic/include/clc/explicit_fence
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - libclc/generic/include/clc/float
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libclc/generic/include/clc/geometric
-     - `8`
-     - `8`
-     - `0`
-     - :good:`100%`
-   * - libclc/generic/include/clc/image
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - libclc/generic/include/clc/integer
-     - `16`
-     - `13`
-     - `3`
-     - :part:`81%`
-   * - libclc/generic/include/clc/math
-     - `95`
-     - `92`
-     - `3`
-     - :part:`96%`
-   * - libclc/generic/include/clc/misc
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - libclc/generic/include/clc/relational
-     - `18`
-     - `12`
-     - `6`
-     - :part:`66%`
-   * - libclc/generic/include/clc/shared
-     - `5`
-     - `3`
-     - `2`
-     - :part:`60%`
-   * - libclc/generic/include/clc/synchronization
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - libclc/generic/include/clc/workitem
-     - `8`
-     - `8`
-     - `0`
-     - :good:`100%`
-   * - libclc/generic/include/integer
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - libclc/generic/include/math
-     - `15`
-     - `15`
-     - `0`
-     - :good:`100%`
-   * - libclc/generic/lib
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libclc/generic/lib/math
-     - `8`
-     - `1`
-     - `7`
-     - :part:`12%`
-   * - libclc/generic/lib/relational
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libclc/utils
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libcxx/benchmarks
-     - `28`
-     - `10`
-     - `18`
-     - :part:`35%`
-   * - libcxx/include
-     - `22`
-     - `0`
-     - `22`
-     - :none:`0%`
-   * - libcxx/include/__algorithm
-     - `102`
-     - `15`
-     - `87`
-     - :part:`14%`
-   * - libcxx/include/__bit
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - libcxx/include/__charconv
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - libcxx/include/__chrono
-     - `8`
-     - `0`
-     - `8`
-     - :none:`0%`
-   * - libcxx/include/__compare
-     - `13`
-     - `1`
-     - `12`
-     - :part:`7%`
-   * - libcxx/include/__concepts
-     - `22`
-     - `0`
-     - `22`
-     - :none:`0%`
-   * - libcxx/include/__coroutine
-     - `4`
-     - `0`
-     - `4`
-     - :none:`0%`
-   * - libcxx/include/__filesystem
-     - `16`
-     - `3`
-     - `13`
-     - :part:`18%`
-   * - libcxx/include/__format
-     - `17`
-     - `2`
-     - `15`
-     - :part:`11%`
-   * - libcxx/include/__functional
-     - `27`
-     - `0`
-     - `27`
-     - :none:`0%`
-   * - libcxx/include/__ios
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libcxx/include/__iterator
-     - `36`
-     - `0`
-     - `36`
-     - :none:`0%`
-   * - libcxx/include/__memory
-     - `19`
-     - `1`
-     - `18`
-     - :part:`5%`
-   * - libcxx/include/__numeric
-     - `13`
-     - `4`
-     - `9`
-     - :part:`30%`
-   * - libcxx/include/__random
-     - `37`
-     - `2`
-     - `35`
-     - :part:`5%`
-   * - libcxx/include/__ranges
-     - `29`
-     - `2`
-     - `27`
-     - :part:`6%`
-   * - libcxx/include/__support/android
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libcxx/include/__support/fuchsia
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libcxx/include/__support/ibm
-     - `6`
-     - `2`
-     - `4`
-     - :part:`33%`
-   * - libcxx/include/__support/musl
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libcxx/include/__support/newlib
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libcxx/include/__support/openbsd
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - libcxx/include/__support/solaris
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - libcxx/include/__support/win32
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - libcxx/include/__support/xlocale
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - libcxx/include/__thread
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - libcxx/include/__utility
-     - `17`
-     - `5`
-     - `12`
-     - :part:`29%`
-   * - libcxx/include/__variant
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libcxx/src
-     - `42`
-     - `6`
-     - `36`
-     - :part:`14%`
-   * - libcxx/src/experimental
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - libcxx/src/filesystem
-     - `5`
-     - `0`
-     - `5`
-     - :none:`0%`
-   * - libcxx/src/include
-     - `6`
-     - `1`
-     - `5`
-     - :part:`16%`
-   * - libcxx/src/include/ryu
-     - `9`
-     - `8`
-     - `1`
-     - :part:`88%`
-   * - libcxx/src/ryu
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - libcxx/src/support/ibm
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - libcxx/src/support/solaris
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libcxx/src/support/win32
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - libcxxabi/fuzz
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libcxxabi/include
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - libcxxabi/src
-     - `25`
-     - `1`
-     - `24`
-     - :part:`4%`
-   * - libcxxabi/src/demangle
-     - `4`
-     - `2`
-     - `2`
-     - :part:`50%`
-   * - libunwind/include
-     - `5`
-     - `0`
-     - `5`
-     - :none:`0%`
-   * - libunwind/include/mach-o
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - libunwind/src
-     - `10`
-     - `1`
-     - `9`
-     - :part:`10%`
-   * - lld/COFF
-     - `37`
-     - `13`
-     - `24`
-     - :part:`35%`
-   * - lld/Common
-     - `11`
-     - `9`
-     - `2`
-     - :part:`81%`
-   * - lld/ELF
-     - `48`
-     - `25`
-     - `23`
-     - :part:`52%`
-   * - lld/ELF/Arch
-     - `14`
-     - `4`
-     - `10`
-     - :part:`28%`
-   * - lld/include/lld/Common
-     - `14`
-     - `8`
-     - `6`
-     - :part:`57%`
-   * - lld/include/lld/Core
-     - `20`
-     - `4`
-     - `16`
-     - :part:`20%`
-   * - lld/MachO
-     - `45`
-     - `43`
-     - `2`
-     - :part:`95%`
-   * - lld/MachO/Arch
-     - `6`
-     - `6`
-     - `0`
-     - :good:`100%`
-   * - lld/MinGW
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lld/tools/lld
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lld/wasm
-     - `29`
-     - `15`
-     - `14`
-     - :part:`51%`
-   * - lldb/bindings/python
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/examples/darwin/heap_find/heap
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/examples/functions
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/examples/interposing/darwin/fd_interposing
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/examples/lookup
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/examples/plugins/commands
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/examples/synthetic/bitfield
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/include/lldb
-     - `12`
-     - `6`
-     - `6`
-     - :part:`50%`
-   * - lldb/include/lldb/API
-     - `70`
-     - `60`
-     - `10`
-     - :part:`85%`
-   * - lldb/include/lldb/Breakpoint
-     - `25`
-     - `9`
-     - `16`
-     - :part:`36%`
-   * - lldb/include/lldb/Core
-     - `61`
-     - `31`
-     - `30`
-     - :part:`50%`
-   * - lldb/include/lldb/DataFormatters
-     - `18`
-     - `10`
-     - `8`
-     - :part:`55%`
-   * - lldb/include/lldb/Expression
-     - `17`
-     - `7`
-     - `10`
-     - :part:`41%`
-   * - lldb/include/lldb/Host
-     - `39`
-     - `20`
-     - `19`
-     - :part:`51%`
-   * - lldb/include/lldb/Host/android
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/include/lldb/Host/common
-     - `8`
-     - `2`
-     - `6`
-     - :part:`25%`
-   * - lldb/include/lldb/Host/freebsd
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/include/lldb/Host/linux
-     - `6`
-     - `4`
-     - `2`
-     - :part:`66%`
-   * - lldb/include/lldb/Host/macosx
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/include/lldb/Host/netbsd
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/include/lldb/Host/openbsd
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/include/lldb/Host/posix
-     - `9`
-     - `7`
-     - `2`
-     - :part:`77%`
-   * - lldb/include/lldb/Host/windows
-     - `10`
-     - `4`
-     - `6`
-     - :part:`40%`
-   * - lldb/include/lldb/Initialization
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - lldb/include/lldb/Interpreter
-     - `49`
-     - `36`
-     - `13`
-     - :part:`73%`
-   * - lldb/include/lldb/Symbol
-     - `35`
-     - `14`
-     - `21`
-     - :part:`40%`
-   * - lldb/include/lldb/Target
-     - `78`
-     - `51`
-     - `27`
-     - :part:`65%`
-   * - lldb/include/lldb/Utility
-     - `63`
-     - `41`
-     - `22`
-     - :part:`65%`
-   * - lldb/include/lldb/Version
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/API
-     - `73`
-     - `36`
-     - `37`
-     - :part:`49%`
-   * - lldb/source/Breakpoint
-     - `24`
-     - `6`
-     - `18`
-     - :part:`25%`
-   * - lldb/source/Commands
-     - `70`
-     - `57`
-     - `13`
-     - :part:`81%`
-   * - lldb/source/Core
-     - `49`
-     - `26`
-     - `23`
-     - :part:`53%`
-   * - lldb/source/DataFormatters
-     - `16`
-     - `3`
-     - `13`
-     - :part:`18%`
-   * - lldb/source/Expression
-     - `13`
-     - `5`
-     - `8`
-     - :part:`38%`
-   * - lldb/source/Host/android
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Host/common
-     - `31`
-     - `16`
-     - `15`
-     - :part:`51%`
-   * - lldb/source/Host/freebsd
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Host/linux
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Host/macosx/cfcpp
-     - `14`
-     - `12`
-     - `2`
-     - :part:`85%`
-   * - lldb/source/Host/macosx/objcxx
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Host/netbsd
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Host/openbsd
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Host/posix
-     - `9`
-     - `6`
-     - `3`
-     - :part:`66%`
-   * - lldb/source/Host/windows
-     - `11`
-     - `7`
-     - `4`
-     - :part:`63%`
-   * - lldb/source/Initialization
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Interpreter
-     - `44`
-     - `24`
-     - `20`
-     - :part:`54%`
-   * - lldb/source/Plugins/ABI/AArch64
-     - `6`
-     - `3`
-     - `3`
-     - :part:`50%`
-   * - lldb/source/Plugins/ABI/ARC
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/ABI/ARM
-     - `6`
-     - `2`
-     - `4`
-     - :part:`33%`
-   * - lldb/source/Plugins/ABI/Hexagon
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/ABI/Mips
-     - `6`
-     - `2`
-     - `4`
-     - :part:`33%`
-   * - lldb/source/Plugins/ABI/PowerPC
-     - `6`
-     - `3`
-     - `3`
-     - :part:`50%`
-   * - lldb/source/Plugins/ABI/SystemZ
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/ABI/X86
-     - `13`
-     - `4`
-     - `9`
-     - :part:`30%`
-   * - lldb/source/Plugins/Architecture/AArch64
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/Architecture/Arm
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/Architecture/Mips
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/Architecture/PPC64
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/Disassembler/LLVMC
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/DynamicLoader/Darwin-Kernel
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/DynamicLoader/Hexagon-DYLD
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - lldb/source/Plugins/DynamicLoader/MacOSX-DYLD
-     - `6`
-     - `3`
-     - `3`
-     - :part:`50%`
-   * - lldb/source/Plugins/DynamicLoader/POSIX-DYLD
-     - `4`
-     - `2`
-     - `2`
-     - :part:`50%`
-   * - lldb/source/Plugins/DynamicLoader/Static
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/DynamicLoader/wasm-DYLD
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/DynamicLoader/Windows-DYLD
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/ExpressionParser/Clang
-     - `51`
-     - `25`
-     - `26`
-     - :part:`49%`
-   * - lldb/source/Plugins/Instruction/ARM
-     - `4`
-     - `2`
-     - `2`
-     - :part:`50%`
-   * - lldb/source/Plugins/Instruction/ARM64
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/Instruction/MIPS
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/Instruction/MIPS64
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/Instruction/PPC64
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/InstrumentationRuntime/ASan
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/InstrumentationRuntime/MainThreadChecker
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/InstrumentationRuntime/TSan
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/InstrumentationRuntime/UBSan
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/JITLoader/GDB
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/Language/ClangCommon
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/Language/CPlusPlus
-     - `30`
-     - `19`
-     - `11`
-     - :part:`63%`
-   * - lldb/source/Plugins/Language/ObjC
-     - `21`
-     - `14`
-     - `7`
-     - :part:`66%`
-   * - lldb/source/Plugins/Language/ObjCPlusPlus
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/LanguageRuntime/CPlusPlus
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/LanguageRuntime/ObjC
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime
-     - `16`
-     - `5`
-     - `11`
-     - :part:`31%`
-   * - lldb/source/Plugins/LanguageRuntime/RenderScript/RenderScriptRuntime
-     - `8`
-     - `3`
-     - `5`
-     - :part:`37%`
-   * - lldb/source/Plugins/MemoryHistory/asan
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/ObjectContainer/BSD-Archive
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/ObjectContainer/Universal-Mach-O
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/ObjectFile/Breakpad
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - lldb/source/Plugins/ObjectFile/ELF
-     - `4`
-     - `1`
-     - `3`
-     - :part:`25%`
-   * - lldb/source/Plugins/ObjectFile/JIT
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/ObjectFile/Mach-O
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/ObjectFile/Minidump
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/ObjectFile/PDB
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/ObjectFile/PECOFF
-     - `6`
-     - `3`
-     - `3`
-     - :part:`50%`
-   * - lldb/source/Plugins/ObjectFile/wasm
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/OperatingSystem/Python
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/Platform/Android
-     - `6`
-     - `3`
-     - `3`
-     - :part:`50%`
-   * - lldb/source/Plugins/Platform/FreeBSD
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/Platform/gdb-server
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/Platform/Linux
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/Platform/MacOSX
-     - `20`
-     - `11`
-     - `9`
-     - :part:`55%`
-   * - lldb/source/Plugins/Platform/MacOSX/objcxx
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/Platform/NetBSD
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/Platform/OpenBSD
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/Platform/POSIX
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/Platform/QemuUser
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/Platform/Windows
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/Process/elf-core
-     - `20`
-     - `18`
-     - `2`
-     - :part:`90%`
-   * - lldb/source/Plugins/Process/FreeBSD
-     - `16`
-     - `12`
-     - `4`
-     - :part:`75%`
-   * - lldb/source/Plugins/Process/FreeBSDKernel
-     - `10`
-     - `8`
-     - `2`
-     - :part:`80%`
-   * - lldb/source/Plugins/Process/gdb-remote
-     - `26`
-     - `15`
-     - `11`
-     - :part:`57%`
-   * - lldb/source/Plugins/Process/Linux
-     - `21`
-     - `11`
-     - `10`
-     - :part:`52%`
-   * - lldb/source/Plugins/Process/mach-core
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - lldb/source/Plugins/Process/MacOSX-Kernel
-     - `16`
-     - `13`
-     - `3`
-     - :part:`81%`
-   * - lldb/source/Plugins/Process/minidump
-     - `17`
-     - `10`
-     - `7`
-     - :part:`58%`
-   * - lldb/source/Plugins/Process/NetBSD
-     - `8`
-     - `4`
-     - `4`
-     - :part:`50%`
-   * - lldb/source/Plugins/Process/POSIX
-     - `8`
-     - `7`
-     - `1`
-     - :part:`87%`
-   * - lldb/source/Plugins/Process/scripted
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/Process/Utility
-     - `132`
-     - `97`
-     - `35`
-     - :part:`73%`
-   * - lldb/source/Plugins/Process/Windows/Common
-     - `34`
-     - `22`
-     - `12`
-     - :part:`64%`
-   * - lldb/source/Plugins/Process/Windows/Common/arm
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/Process/Windows/Common/arm64
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/Process/Windows/Common/x64
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/Process/Windows/Common/x86
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/REPL/Clang
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/ScriptInterpreter/Lua
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/ScriptInterpreter/None
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/ScriptInterpreter/Python
-     - `16`
-     - `12`
-     - `4`
-     - :part:`75%`
-   * - lldb/source/Plugins/StructuredData/DarwinLog
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/SymbolFile/Breakpad
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/SymbolFile/DWARF
-     - `65`
-     - `39`
-     - `26`
-     - :part:`60%`
-   * - lldb/source/Plugins/SymbolFile/NativePDB
-     - `20`
-     - `10`
-     - `10`
-     - :part:`50%`
-   * - lldb/source/Plugins/SymbolFile/PDB
-     - `6`
-     - `4`
-     - `2`
-     - :part:`66%`
-   * - lldb/source/Plugins/SymbolFile/Symtab
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/SymbolVendor/ELF
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/SymbolVendor/MacOSX
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/SymbolVendor/wasm
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/SystemRuntime/MacOSX
-     - `10`
-     - `1`
-     - `9`
-     - :part:`10%`
-   * - lldb/source/Plugins/Trace/common
-     - `8`
-     - `7`
-     - `1`
-     - :part:`87%`
-   * - lldb/source/Plugins/Trace/intel-pt
-     - `18`
-     - `17`
-     - `1`
-     - :part:`94%`
-   * - lldb/source/Plugins/TraceExporter/common
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/source/Plugins/TraceExporter/ctf
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - lldb/source/Plugins/TypeSystem/Clang
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/source/Plugins/UnwindAssembly/InstEmulation
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/source/Plugins/UnwindAssembly/x86
-     - `4`
-     - `2`
-     - `2`
-     - :part:`50%`
-   * - lldb/source/Symbol
-     - `31`
-     - `18`
-     - `13`
-     - :part:`58%`
-   * - lldb/source/Target
-     - `69`
-     - `34`
-     - `35`
-     - :part:`49%`
-   * - lldb/source/Utility
-     - `58`
-     - `46`
-     - `12`
-     - :part:`79%`
-   * - lldb/source/Version
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/tools/argdumper
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/tools/darwin-debug
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/tools/debugserver/source
-     - `51`
-     - `40`
-     - `11`
-     - :part:`78%`
-   * - lldb/tools/debugserver/source/MacOSX
-     - `24`
-     - `16`
-     - `8`
-     - :part:`66%`
-   * - lldb/tools/debugserver/source/MacOSX/arm
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/tools/debugserver/source/MacOSX/arm64
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/tools/debugserver/source/MacOSX/i386
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - lldb/tools/debugserver/source/MacOSX/x86_64
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - lldb/tools/driver
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - lldb/tools/intel-features
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/tools/intel-features/intel-mpx
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - lldb/tools/lldb-instr
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/tools/lldb-server
-     - `9`
-     - `4`
-     - `5`
-     - :part:`44%`
-   * - lldb/tools/lldb-test
-     - `5`
-     - `2`
-     - `3`
-     - :part:`40%`
-   * - lldb/tools/lldb-vscode
-     - `27`
-     - `24`
-     - `3`
-     - :part:`88%`
-   * - lldb/unittests
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/API
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/Breakpoint
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/Core
-     - `10`
-     - `9`
-     - `1`
-     - :part:`90%`
-   * - lldb/unittests/DataFormatter
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/debugserver
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - lldb/unittests/Disassembler
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/unittests/Editline
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/Expression
-     - `5`
-     - `3`
-     - `2`
-     - :part:`60%`
-   * - lldb/unittests/Host
-     - `16`
-     - `11`
-     - `5`
-     - :part:`68%`
-   * - lldb/unittests/Host/linux
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/Host/posix
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/unittests/Instruction
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/unittests/Interpreter
-     - `6`
-     - `2`
-     - `4`
-     - :part:`33%`
-   * - lldb/unittests/Language/CLanguages
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/Language/CPlusPlus
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/unittests/Language/Highlighting
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/ObjectFile/Breakpad
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/ObjectFile/ELF
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/unittests/ObjectFile/MachO
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/unittests/ObjectFile/PECOFF
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/unittests/Platform
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - lldb/unittests/Platform/Android
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/unittests/Process
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/Process/gdb-remote
-     - `8`
-     - `6`
-     - `2`
-     - :part:`75%`
-   * - lldb/unittests/Process/Linux
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/unittests/Process/minidump
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/unittests/Process/minidump/Inputs
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/Process/POSIX
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/Process/Utility
-     - `6`
-     - `4`
-     - `2`
-     - :part:`66%`
-   * - lldb/unittests/ScriptInterpreter/Lua
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/ScriptInterpreter/Python
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - lldb/unittests/Signals
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/Symbol
-     - `11`
-     - `7`
-     - `4`
-     - :part:`63%`
-   * - lldb/unittests/SymbolFile/DWARF
-     - `6`
-     - `4`
-     - `2`
-     - :part:`66%`
-   * - lldb/unittests/SymbolFile/DWARF/Inputs
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/SymbolFile/NativePDB
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/SymbolFile/PDB
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/unittests/SymbolFile/PDB/Inputs
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/Target
-     - `10`
-     - `6`
-     - `4`
-     - :part:`60%`
-   * - lldb/unittests/TestingSupport
-     - `5`
-     - `4`
-     - `1`
-     - :part:`80%`
-   * - lldb/unittests/TestingSupport/Host
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/TestingSupport/Symbol
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/Thread
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/tools/lldb-server/inferior
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - lldb/unittests/tools/lldb-server/tests
-     - `7`
-     - `0`
-     - `7`
-     - :none:`0%`
-   * - lldb/unittests/UnwindAssembly/ARM64
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/unittests/UnwindAssembly/PPC64
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - lldb/unittests/UnwindAssembly/x86
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/unittests/Utility
-     - `45`
-     - `32`
-     - `13`
-     - :part:`71%`
-   * - lldb/utils/lit-cpuid
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - lldb/utils/TableGen
-     - `6`
-     - `6`
-     - `0`
-     - :good:`100%`
-   * - llvm/benchmarks
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/bindings/go/llvm
-     - `6`
-     - `3`
-     - `3`
-     - :part:`50%`
-   * - llvm/bindings/ocaml/llvm
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/cmake
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/examples/BrainF
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - llvm/examples/Bye
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/examples/ExceptionDemo
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/Fibonacci
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/HowToUseJIT
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/HowToUseLLJIT
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/examples/IRTransforms
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - llvm/examples/Kaleidoscope/Chapter2
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/examples/Kaleidoscope/Chapter3
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/Kaleidoscope/Chapter4
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/Kaleidoscope/Chapter5
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/Kaleidoscope/Chapter6
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/Kaleidoscope/Chapter7
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/Kaleidoscope/Chapter8
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/Kaleidoscope/Chapter9
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/Kaleidoscope/include
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/examples/Kaleidoscope/MCJIT/cached
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - llvm/examples/Kaleidoscope/MCJIT/complete
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/Kaleidoscope/MCJIT/initial
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/Kaleidoscope/MCJIT/lazy
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - llvm/examples/ModuleMaker
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/OrcV2Examples
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/examples/OrcV2Examples/LLJITDumpObjects
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/examples/OrcV2Examples/LLJITWithCustomObjectLinkingLayer
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/examples/OrcV2Examples/LLJITWithExecutorProcessControl
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/examples/OrcV2Examples/LLJITWithInitializers
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/examples/OrcV2Examples/LLJITWithLazyReexports
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/examples/OrcV2Examples/LLJITWithObjectCache
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/examples/OrcV2Examples/LLJITWithObjectLinkingLayerPlugin
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/OrcV2Examples/LLJITWithOptimizingIRTransform
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - llvm/examples/OrcV2Examples/LLJITWithThinLTOSummaries
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/ParallelJIT
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/examples/SpeculativeJIT
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/include/llvm
-     - `8`
-     - `2`
-     - `6`
-     - :part:`25%`
-   * - llvm/include/llvm/ADT
-     - `93`
-     - `25`
-     - `68`
-     - :part:`26%`
-   * - llvm/include/llvm/Analysis
-     - `130`
-     - `52`
-     - `78`
-     - :part:`40%`
-   * - llvm/include/llvm/Analysis/Utils
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - llvm/include/llvm/AsmParser
-     - `5`
-     - `2`
-     - `3`
-     - :part:`40%`
-   * - llvm/include/llvm/BinaryFormat
-     - `15`
-     - `8`
-     - `7`
-     - :part:`53%`
-   * - llvm/include/llvm/Bitcode
-     - `7`
-     - `2`
-     - `5`
-     - :part:`28%`
-   * - llvm/include/llvm/Bitstream
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - llvm/include/llvm/CodeGen
-     - `158`
-     - `51`
-     - `107`
-     - :part:`32%`
-   * - llvm/include/llvm/CodeGen/GlobalISel
-     - `27`
-     - `8`
-     - `19`
-     - :part:`29%`
-   * - llvm/include/llvm/CodeGen/MIRParser
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/include/llvm/CodeGen/PBQP
-     - `5`
-     - `1`
-     - `4`
-     - :part:`20%`
-   * - llvm/include/llvm/DebugInfo
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/DebugInfo/CodeView
-     - `57`
-     - `40`
-     - `17`
-     - :part:`70%`
-   * - llvm/include/llvm/DebugInfo/DWARF
-     - `32`
-     - `14`
-     - `18`
-     - :part:`43%`
-   * - llvm/include/llvm/DebugInfo/GSYM
-     - `14`
-     - `4`
-     - `10`
-     - :part:`28%`
-   * - llvm/include/llvm/DebugInfo/MSF
-     - `5`
-     - `4`
-     - `1`
-     - :part:`80%`
-   * - llvm/include/llvm/DebugInfo/PDB
-     - `50`
-     - `30`
-     - `20`
-     - :part:`60%`
-   * - llvm/include/llvm/DebugInfo/PDB/DIA
-     - `20`
-     - `9`
-     - `11`
-     - :part:`45%`
-   * - llvm/include/llvm/DebugInfo/PDB/Native
-     - `54`
-     - `35`
-     - `19`
-     - :part:`64%`
-   * - llvm/include/llvm/DebugInfo/Symbolize
-     - `5`
-     - `3`
-     - `2`
-     - :part:`60%`
-   * - llvm/include/llvm/Debuginfod
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/Demangle
-     - `7`
-     - `3`
-     - `4`
-     - :part:`42%`
-   * - llvm/include/llvm/DWARFLinker
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/DWP
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/ExecutionEngine
-     - `12`
-     - `2`
-     - `10`
-     - :part:`16%`
-   * - llvm/include/llvm/ExecutionEngine/JITLink
-     - `16`
-     - `14`
-     - `2`
-     - :part:`87%`
-   * - llvm/include/llvm/ExecutionEngine/Orc
-     - `38`
-     - `29`
-     - `9`
-     - :part:`76%`
-   * - llvm/include/llvm/ExecutionEngine/Orc/Shared
-     - `8`
-     - `4`
-     - `4`
-     - :part:`50%`
-   * - llvm/include/llvm/ExecutionEngine/Orc/TargetProcess
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/FileCheck
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/Frontend/OpenMP
-     - `5`
-     - `4`
-     - `1`
-     - :part:`80%`
-   * - llvm/include/llvm/FuzzMutate
-     - `6`
-     - `0`
-     - `6`
-     - :none:`0%`
-   * - llvm/include/llvm/InterfaceStub
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/IR
-     - `93`
-     - `28`
-     - `65`
-     - :part:`30%`
-   * - llvm/include/llvm/IRReader
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/include/llvm/LineEditor
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/include/llvm/Linker
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - llvm/include/llvm/LTO
-     - `4`
-     - `1`
-     - `3`
-     - :part:`25%`
-   * - llvm/include/llvm/LTO/legacy
-     - `4`
-     - `0`
-     - `4`
-     - :none:`0%`
-   * - llvm/include/llvm/MC
-     - `74`
-     - `24`
-     - `50`
-     - :part:`32%`
-   * - llvm/include/llvm/MC/MCDisassembler
-     - `4`
-     - `1`
-     - `3`
-     - :part:`25%`
-   * - llvm/include/llvm/MC/MCParser
-     - `8`
-     - `3`
-     - `5`
-     - :part:`37%`
-   * - llvm/include/llvm/MCA
-     - `10`
-     - `10`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/MCA/HardwareUnits
-     - `6`
-     - `4`
-     - `2`
-     - :part:`66%`
-   * - llvm/include/llvm/MCA/Stages
-     - `8`
-     - `8`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/ObjCopy
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - llvm/include/llvm/ObjCopy/COFF
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/ObjCopy/ELF
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/ObjCopy/MachO
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/ObjCopy/wasm
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/ObjCopy/XCOFF
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/Object
-     - `31`
-     - `12`
-     - `19`
-     - :part:`38%`
-   * - llvm/include/llvm/ObjectYAML
-     - `16`
-     - `12`
-     - `4`
-     - :part:`75%`
-   * - llvm/include/llvm/Option
-     - `5`
-     - `1`
-     - `4`
-     - :part:`20%`
-   * - llvm/include/llvm/Passes
-     - `4`
-     - `2`
-     - `2`
-     - :part:`50%`
-   * - llvm/include/llvm/ProfileData
-     - `11`
-     - `5`
-     - `6`
-     - :part:`45%`
-   * - llvm/include/llvm/ProfileData/Coverage
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - llvm/include/llvm/Remarks
-     - `12`
-     - `11`
-     - `1`
-     - :part:`91%`
-   * - llvm/include/llvm/Support
-     - `186`
-     - `68`
-     - `118`
-     - :part:`36%`
-   * - llvm/include/llvm/Support/FileSystem
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/Support/Solaris/sys
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/Support/Windows
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/include/llvm/TableGen
-     - `9`
-     - `3`
-     - `6`
-     - :part:`33%`
-   * - llvm/include/llvm/Target
-     - `6`
-     - `2`
-     - `4`
-     - :part:`33%`
-   * - llvm/include/llvm/Testing/Support
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - llvm/include/llvm/TextAPI
-     - `9`
-     - `9`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/ToolDrivers/llvm-dlltool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/ToolDrivers/llvm-lib
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/include/llvm/Transforms
-     - `8`
-     - `2`
-     - `6`
-     - :part:`25%`
-   * - llvm/include/llvm/Transforms/AggressiveInstCombine
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/include/llvm/Transforms/Coroutines
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/Transforms/InstCombine
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/include/llvm/Transforms/Instrumentation
-     - `17`
-     - `10`
-     - `7`
-     - :part:`58%`
-   * - llvm/include/llvm/Transforms/IPO
-     - `38`
-     - `28`
-     - `10`
-     - :part:`73%`
-   * - llvm/include/llvm/Transforms/Scalar
-     - `75`
-     - `47`
-     - `28`
-     - :part:`62%`
-   * - llvm/include/llvm/Transforms/Utils
-     - `74`
-     - `44`
-     - `30`
-     - :part:`59%`
-   * - llvm/include/llvm/Transforms/Vectorize
-     - `5`
-     - `1`
-     - `4`
-     - :part:`20%`
-   * - llvm/include/llvm/WindowsDriver
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/include/llvm/WindowsManifest
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/include/llvm/WindowsResource
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - llvm/include/llvm/XRay
-     - `17`
-     - `13`
-     - `4`
-     - :part:`76%`
-   * - llvm/include/llvm-c
-     - `27`
-     - `12`
-     - `15`
-     - :part:`44%`
-   * - llvm/include/llvm-c/Transforms
-     - `9`
-     - `3`
-     - `6`
-     - :part:`33%`
-   * - llvm/lib/Analysis
-     - `119`
-     - `40`
-     - `79`
-     - :part:`33%`
-   * - llvm/lib/AsmParser
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - llvm/lib/BinaryFormat
-     - `13`
-     - `10`
-     - `3`
-     - :part:`76%`
-   * - llvm/lib/Bitcode/Reader
-     - `7`
-     - `2`
-     - `5`
-     - :part:`28%`
-   * - llvm/lib/Bitcode/Writer
-     - `5`
-     - `0`
-     - `5`
-     - :none:`0%`
-   * - llvm/lib/Bitstream/Reader
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/CodeGen
-     - `220`
-     - `60`
-     - `160`
-     - :part:`27%`
-   * - llvm/lib/CodeGen/AsmPrinter
-     - `45`
-     - `18`
-     - `27`
-     - :part:`40%`
-   * - llvm/lib/CodeGen/GlobalISel
-     - `24`
-     - `9`
-     - `15`
-     - :part:`37%`
-   * - llvm/lib/CodeGen/LiveDebugValues
-     - `5`
-     - `1`
-     - `4`
-     - :part:`20%`
-   * - llvm/lib/CodeGen/MIRParser
-     - `4`
-     - `1`
-     - `3`
-     - :part:`25%`
-   * - llvm/lib/CodeGen/SelectionDAG
-     - `31`
-     - `2`
-     - `29`
-     - :part:`6%`
-   * - llvm/lib/DebugInfo/CodeView
-     - `40`
-     - `23`
-     - `17`
-     - :part:`57%`
-   * - llvm/lib/DebugInfo/DWARF
-     - `28`
-     - `9`
-     - `19`
-     - :part:`32%`
-   * - llvm/lib/DebugInfo/GSYM
-     - `11`
-     - `2`
-     - `9`
-     - :part:`18%`
-   * - llvm/lib/DebugInfo/MSF
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - llvm/lib/DebugInfo/PDB
-     - `40`
-     - `35`
-     - `5`
-     - :part:`87%`
-   * - llvm/lib/DebugInfo/PDB/DIA
-     - `18`
-     - `15`
-     - `3`
-     - :part:`83%`
-   * - llvm/lib/DebugInfo/PDB/Native
-     - `50`
-     - `37`
-     - `13`
-     - :part:`74%`
-   * - llvm/lib/DebugInfo/Symbolize
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - llvm/lib/Debuginfod
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Demangle
-     - `6`
-     - `4`
-     - `2`
-     - :part:`66%`
-   * - llvm/lib/DWARFLinker
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - llvm/lib/DWP
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/ExecutionEngine
-     - `5`
-     - `1`
-     - `4`
-     - :part:`20%`
-   * - llvm/lib/ExecutionEngine/IntelJITEvents
-     - `5`
-     - `0`
-     - `5`
-     - :none:`0%`
-   * - llvm/lib/ExecutionEngine/Interpreter
-     - `4`
-     - `0`
-     - `4`
-     - :none:`0%`
-   * - llvm/lib/ExecutionEngine/JITLink
-     - `23`
-     - `15`
-     - `8`
-     - :part:`65%`
-   * - llvm/lib/ExecutionEngine/MCJIT
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - llvm/lib/ExecutionEngine/OProfileJIT
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - llvm/lib/ExecutionEngine/Orc
-     - `37`
-     - `22`
-     - `15`
-     - :part:`59%`
-   * - llvm/lib/ExecutionEngine/Orc/Shared
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/ExecutionEngine/Orc/TargetProcess
-     - `8`
-     - `7`
-     - `1`
-     - :part:`87%`
-   * - llvm/lib/ExecutionEngine/PerfJITEvents
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/ExecutionEngine/RuntimeDyld
-     - `12`
-     - `1`
-     - `11`
-     - :part:`8%`
-   * - llvm/lib/ExecutionEngine/RuntimeDyld/Targets
-     - `10`
-     - `1`
-     - `9`
-     - :part:`10%`
-   * - llvm/lib/Extensions
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/FileCheck
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/lib/Frontend/OpenACC
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Frontend/OpenMP
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/FuzzMutate
-     - `5`
-     - `2`
-     - `3`
-     - :part:`40%`
-   * - llvm/lib/InterfaceStub
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/IR
-     - `69`
-     - `20`
-     - `49`
-     - :part:`28%`
-   * - llvm/lib/IRReader
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/LineEditor
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Linker
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - llvm/lib/LTO
-     - `7`
-     - `1`
-     - `6`
-     - :part:`14%`
-   * - llvm/lib/MC
-     - `65`
-     - `21`
-     - `44`
-     - :part:`32%`
-   * - llvm/lib/MC/MCDisassembler
-     - `6`
-     - `3`
-     - `3`
-     - :part:`50%`
-   * - llvm/lib/MC/MCParser
-     - `14`
-     - `3`
-     - `11`
-     - :part:`21%`
-   * - llvm/lib/MCA
-     - `9`
-     - `8`
-     - `1`
-     - :part:`88%`
-   * - llvm/lib/MCA/HardwareUnits
-     - `6`
-     - `4`
-     - `2`
-     - :part:`66%`
-   * - llvm/lib/MCA/Stages
-     - `8`
-     - `7`
-     - `1`
-     - :part:`87%`
-   * - llvm/lib/ObjCopy
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - llvm/lib/ObjCopy/COFF
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/ObjCopy/ELF
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/ObjCopy/MachO
-     - `9`
-     - `9`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/ObjCopy/wasm
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/ObjCopy/XCOFF
-     - `6`
-     - `3`
-     - `3`
-     - :part:`50%`
-   * - llvm/lib/Object
-     - `31`
-     - `16`
-     - `15`
-     - :part:`51%`
-   * - llvm/lib/ObjectYAML
-     - `23`
-     - `9`
-     - `14`
-     - :part:`39%`
-   * - llvm/lib/Option
-     - `4`
-     - `0`
-     - `4`
-     - :none:`0%`
-   * - llvm/lib/Passes
-     - `6`
-     - `3`
-     - `3`
-     - :part:`50%`
-   * - llvm/lib/ProfileData
-     - `11`
-     - `4`
-     - `7`
-     - :part:`36%`
-   * - llvm/lib/ProfileData/Coverage
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - llvm/lib/Remarks
-     - `13`
-     - `10`
-     - `3`
-     - :part:`76%`
-   * - llvm/lib/Support
-     - `144`
-     - `61`
-     - `83`
-     - :part:`42%`
-   * - llvm/lib/Support/Unix
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/TableGen
-     - `15`
-     - `3`
-     - `12`
-     - :part:`20%`
-   * - llvm/lib/Target
-     - `5`
-     - `1`
-     - `4`
-     - :part:`20%`
-   * - llvm/lib/Target/AArch64
-     - `60`
-     - `7`
-     - `53`
-     - :part:`11%`
-   * - llvm/lib/Target/AArch64/AsmParser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/AArch64/Disassembler
-     - `4`
-     - `1`
-     - `3`
-     - :part:`25%`
-   * - llvm/lib/Target/AArch64/GISel
-     - `14`
-     - `3`
-     - `11`
-     - :part:`21%`
-   * - llvm/lib/Target/AArch64/MCTargetDesc
-     - `21`
-     - `6`
-     - `15`
-     - :part:`28%`
-   * - llvm/lib/Target/AArch64/TargetInfo
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/lib/Target/AArch64/Utils
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - llvm/lib/Target/AMDGPU
-     - `169`
-     - `38`
-     - `131`
-     - :part:`22%`
-   * - llvm/lib/Target/AMDGPU/AsmParser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/AMDGPU/Disassembler
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - llvm/lib/Target/AMDGPU/MCA
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/AMDGPU/MCTargetDesc
-     - `21`
-     - `5`
-     - `16`
-     - :part:`23%`
-   * - llvm/lib/Target/AMDGPU/TargetInfo
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/lib/Target/AMDGPU/Utils
-     - `11`
-     - `4`
-     - `7`
-     - :part:`36%`
-   * - llvm/lib/Target/ARC
-     - `24`
-     - `19`
-     - `5`
-     - :part:`79%`
-   * - llvm/lib/Target/ARC/Disassembler
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/ARC/MCTargetDesc
-     - `7`
-     - `6`
-     - `1`
-     - :part:`85%`
-   * - llvm/lib/Target/ARC/TargetInfo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/ARM
-     - `76`
-     - `10`
-     - `66`
-     - :part:`13%`
-   * - llvm/lib/Target/ARM/AsmParser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/ARM/Disassembler
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/ARM/MCTargetDesc
-     - `26`
-     - `2`
-     - `24`
-     - :part:`7%`
-   * - llvm/lib/Target/ARM/TargetInfo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/ARM/Utils
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - llvm/lib/Target/AVR
-     - `24`
-     - `23`
-     - `1`
-     - :part:`95%`
-   * - llvm/lib/Target/AVR/AsmParser
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/AVR/Disassembler
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/AVR/MCTargetDesc
-     - `20`
-     - `18`
-     - `2`
-     - :part:`90%`
-   * - llvm/lib/Target/AVR/TargetInfo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/BPF
-     - `32`
-     - `9`
-     - `23`
-     - :part:`28%`
-   * - llvm/lib/Target/BPF/AsmParser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/BPF/Disassembler
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/BPF/MCTargetDesc
-     - `8`
-     - `1`
-     - `7`
-     - :part:`12%`
-   * - llvm/lib/Target/BPF/TargetInfo
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/lib/Target/CSKY
-     - `23`
-     - `23`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/CSKY/AsmParser
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/CSKY/Disassembler
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/CSKY/MCTargetDesc
-     - `15`
-     - `14`
-     - `1`
-     - :part:`93%`
-   * - llvm/lib/Target/CSKY/TargetInfo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/Hexagon
-     - `80`
-     - `6`
-     - `74`
-     - :part:`7%`
-   * - llvm/lib/Target/Hexagon/AsmParser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/Hexagon/Disassembler
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/Hexagon/MCTargetDesc
-     - `26`
-     - `6`
-     - `20`
-     - :part:`23%`
-   * - llvm/lib/Target/Hexagon/TargetInfo
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/lib/Target/Lanai
-     - `28`
-     - `20`
-     - `8`
-     - :part:`71%`
-   * - llvm/lib/Target/Lanai/AsmParser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/Lanai/Disassembler
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/Lanai/MCTargetDesc
-     - `13`
-     - `12`
-     - `1`
-     - :part:`92%`
-   * - llvm/lib/Target/Lanai/TargetInfo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/LoongArch
-     - `19`
-     - `19`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/LoongArch/MCTargetDesc
-     - `12`
-     - `12`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/LoongArch/TargetInfo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/M68k
-     - `26`
-     - `25`
-     - `1`
-     - :part:`96%`
-   * - llvm/lib/Target/M68k/AsmParser
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/M68k/Disassembler
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/M68k/GISel
-     - `7`
-     - `6`
-     - `1`
-     - :part:`85%`
-   * - llvm/lib/Target/M68k/MCTargetDesc
-     - `12`
-     - `11`
-     - `1`
-     - :part:`91%`
-   * - llvm/lib/Target/M68k/TargetInfo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/Mips
-     - `70`
-     - `12`
-     - `58`
-     - :part:`17%`
-   * - llvm/lib/Target/Mips/AsmParser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/Mips/Disassembler
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/Mips/MCTargetDesc
-     - `25`
-     - `6`
-     - `19`
-     - :part:`24%`
-   * - llvm/lib/Target/Mips/TargetInfo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/MSP430
-     - `20`
-     - `0`
-     - `20`
-     - :none:`0%`
-   * - llvm/lib/Target/MSP430/AsmParser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/MSP430/Disassembler
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/MSP430/MCTargetDesc
-     - `11`
-     - `3`
-     - `8`
-     - :part:`27%`
-   * - llvm/lib/Target/MSP430/TargetInfo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/NVPTX
-     - `44`
-     - `10`
-     - `34`
-     - :part:`22%`
-   * - llvm/lib/Target/NVPTX/MCTargetDesc
-     - `9`
-     - `6`
-     - `3`
-     - :part:`66%`
-   * - llvm/lib/Target/NVPTX/TargetInfo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/PowerPC
-     - `54`
-     - `5`
-     - `49`
-     - :part:`9%`
-   * - llvm/lib/Target/PowerPC/AsmParser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/PowerPC/Disassembler
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/PowerPC/GISel
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/PowerPC/MCTargetDesc
-     - `20`
-     - `5`
-     - `15`
-     - :part:`25%`
-   * - llvm/lib/Target/PowerPC/TargetInfo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/RISCV
-     - `36`
-     - `17`
-     - `19`
-     - :part:`47%`
-   * - llvm/lib/Target/RISCV/AsmParser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/RISCV/Disassembler
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/RISCV/MCTargetDesc
-     - `23`
-     - `13`
-     - `10`
-     - :part:`56%`
-   * - llvm/lib/Target/RISCV/TargetInfo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/Sparc
-     - `23`
-     - `3`
-     - `20`
-     - :part:`13%`
-   * - llvm/lib/Target/Sparc/AsmParser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/Sparc/Disassembler
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/Sparc/MCTargetDesc
-     - `14`
-     - `4`
-     - `10`
-     - :part:`28%`
-   * - llvm/lib/Target/Sparc/TargetInfo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/SystemZ
-     - `41`
-     - `6`
-     - `35`
-     - :part:`14%`
-   * - llvm/lib/Target/SystemZ/AsmParser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/SystemZ/Disassembler
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/SystemZ/MCTargetDesc
-     - `10`
-     - `4`
-     - `6`
-     - :part:`40%`
-   * - llvm/lib/Target/SystemZ/TargetInfo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/VE
-     - `24`
-     - `19`
-     - `5`
-     - :part:`79%`
-   * - llvm/lib/Target/VE/AsmParser
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/VE/Disassembler
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/VE/MCTargetDesc
-     - `14`
-     - `14`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/VE/TargetInfo
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/lib/Target/WebAssembly
-     - `61`
-     - `44`
-     - `17`
-     - :part:`72%`
-   * - llvm/lib/Target/WebAssembly/AsmParser
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - llvm/lib/Target/WebAssembly/Disassembler
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/WebAssembly/MCTargetDesc
-     - `12`
-     - `8`
-     - `4`
-     - :part:`66%`
-   * - llvm/lib/Target/WebAssembly/TargetInfo
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/WebAssembly/Utils
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/X86
-     - `82`
-     - `19`
-     - `63`
-     - :part:`23%`
-   * - llvm/lib/Target/X86/AsmParser
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - llvm/lib/Target/X86/Disassembler
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - llvm/lib/Target/X86/MCA
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Target/X86/MCTargetDesc
-     - `25`
-     - `5`
-     - `20`
-     - :part:`20%`
-   * - llvm/lib/Target/X86/TargetInfo
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/lib/Target/XCore
-     - `27`
-     - `2`
-     - `25`
-     - :part:`7%`
-   * - llvm/lib/Target/XCore/Disassembler
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Target/XCore/MCTargetDesc
-     - `6`
-     - `3`
-     - `3`
-     - :part:`50%`
-   * - llvm/lib/Target/XCore/TargetInfo
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/lib/Testing/Support
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/TextAPI
-     - `11`
-     - `9`
-     - `2`
-     - :part:`81%`
-   * - llvm/lib/ToolDrivers/llvm-dlltool
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/ToolDrivers/llvm-lib
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Transforms/AggressiveInstCombine
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - llvm/lib/Transforms/CFGuard
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/Transforms/Coroutines
-     - `8`
-     - `0`
-     - `8`
-     - :none:`0%`
-   * - llvm/lib/Transforms/Hello
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/lib/Transforms/InstCombine
-     - `16`
-     - `1`
-     - `15`
-     - :part:`6%`
-   * - llvm/lib/Transforms/Instrumentation
-     - `21`
-     - `7`
-     - `14`
-     - :part:`33%`
-   * - llvm/lib/Transforms/IPO
-     - `44`
-     - `9`
-     - `35`
-     - :part:`20%`
-   * - llvm/lib/Transforms/ObjCARC
-     - `15`
-     - `4`
-     - `11`
-     - :part:`26%`
-   * - llvm/lib/Transforms/Scalar
-     - `79`
-     - `16`
-     - `63`
-     - :part:`20%`
-   * - llvm/lib/Transforms/Utils
-     - `78`
-     - `19`
-     - `59`
-     - :part:`24%`
-   * - llvm/lib/Transforms/Vectorize
-     - `22`
-     - `13`
-     - `9`
-     - :part:`59%`
-   * - llvm/lib/WindowsDriver
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/WindowsManifest
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/lib/XRay
-     - `14`
-     - `11`
-     - `3`
-     - :part:`78%`
-   * - llvm/tools/bugpoint
-     - `12`
-     - `1`
-     - `11`
-     - :part:`8%`
-   * - llvm/tools/bugpoint-passes
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/dsymutil
-     - `18`
-     - `16`
-     - `2`
-     - :part:`88%`
-   * - llvm/tools/gold
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llc
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/lli
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - llvm/tools/lli/ChildTarget
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-ar
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-as
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-as-fuzzer
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-bcanalyzer
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-c-test
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - llvm/tools/llvm-cat
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-cfi-verify
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-cfi-verify/lib
-     - `4`
-     - `1`
-     - `3`
-     - :part:`25%`
-   * - llvm/tools/llvm-config
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-cov
-     - `23`
-     - `12`
-     - `11`
-     - :part:`52%`
-   * - llvm/tools/llvm-cvtres
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-cxxdump
-     - `4`
-     - `1`
-     - `3`
-     - :part:`25%`
-   * - llvm/tools/llvm-cxxfilt
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-cxxmap
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-debuginfod-find
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-diff
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-diff/lib
-     - `6`
-     - `0`
-     - `6`
-     - :none:`0%`
-   * - llvm/tools/llvm-dis
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-dis-fuzzer
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-dlang-demangle-fuzzer
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-dwarfdump
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - llvm/tools/llvm-dwarfdump/fuzzer
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-dwp
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-exegesis
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-exegesis/lib
-     - `44`
-     - `33`
-     - `11`
-     - :part:`75%`
-   * - llvm/tools/llvm-exegesis/lib/AArch64
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-exegesis/lib/Mips
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-exegesis/lib/PowerPC
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-exegesis/lib/X86
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - llvm/tools/llvm-extract
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-gsymutil
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-ifs
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - llvm/tools/llvm-isel-fuzzer
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/tools/llvm-itanium-demangle-fuzzer
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/tools/llvm-jitlink
-     - `4`
-     - `2`
-     - `2`
-     - :part:`50%`
-   * - llvm/tools/llvm-jitlink/llvm-jitlink-executor
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-jitlistener
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-libtool-darwin
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-link
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-lipo
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-lto
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-lto2
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-mc
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - llvm/tools/llvm-mc-assemble-fuzzer
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-mc-disassemble-fuzzer
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-mca
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-mca/Views
-     - `20`
-     - `19`
-     - `1`
-     - :part:`95%`
-   * - llvm/tools/llvm-microsoft-demangle-fuzzer
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-ml
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - llvm/tools/llvm-modextract
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-mt
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-nm
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-objcopy
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - llvm/tools/llvm-objdump
-     - `15`
-     - `10`
-     - `5`
-     - :part:`66%`
-   * - llvm/tools/llvm-opt-fuzzer
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - llvm/tools/llvm-opt-report
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-pdbutil
-     - `47`
-     - `15`
-     - `32`
-     - :part:`31%`
-   * - llvm/tools/llvm-profdata
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-profgen
-     - `11`
-     - `6`
-     - `5`
-     - :part:`54%`
-   * - llvm/tools/llvm-rc
-     - `12`
-     - `6`
-     - `6`
-     - :part:`50%`
-   * - llvm/tools/llvm-readobj
-     - `19`
-     - `3`
-     - `16`
-     - :part:`15%`
-   * - llvm/tools/llvm-reduce
-     - `7`
-     - `6`
-     - `1`
-     - :part:`85%`
-   * - llvm/tools/llvm-reduce/deltas
-     - `40`
-     - `39`
-     - `1`
-     - :part:`97%`
-   * - llvm/tools/llvm-rtdyld
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-rust-demangle-fuzzer
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-shlib
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-sim
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-size
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-special-case-list-fuzzer
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-split
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-stress
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-strings
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-symbolizer
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-tapi-diff
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-tli-checker
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/llvm-undname
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-xray
-     - `19`
-     - `15`
-     - `4`
-     - :part:`78%`
-   * - llvm/tools/llvm-yaml-numeric-parser-fuzzer
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/llvm-yaml-parser-fuzzer
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/lto
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/tools/obj2yaml
-     - `10`
-     - `5`
-     - `5`
-     - :part:`50%`
-   * - llvm/tools/opt
-     - `10`
-     - `3`
-     - `7`
-     - :part:`30%`
-   * - llvm/tools/remarks-shlib
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/sancov
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/sanstats
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/split-file
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/verify-uselistorder
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/tools/vfabi-demangle-fuzzer
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/tools/yaml2obj
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/ADT
-     - `77`
-     - `29`
-     - `48`
-     - :part:`37%`
-   * - llvm/unittests/Analysis
-     - `38`
-     - `13`
-     - `25`
-     - :part:`34%`
-   * - llvm/unittests/AsmParser
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/BinaryFormat
-     - `6`
-     - `5`
-     - `1`
-     - :part:`83%`
-   * - llvm/unittests/Bitcode
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/unittests/Bitstream
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/unittests/CodeGen
-     - `20`
-     - `10`
-     - `10`
-     - :part:`50%`
-   * - llvm/unittests/CodeGen/GlobalISel
-     - `13`
-     - `2`
-     - `11`
-     - :part:`15%`
-   * - llvm/unittests/DebugInfo/CodeView
-     - `4`
-     - `2`
-     - `2`
-     - :part:`50%`
-   * - llvm/unittests/DebugInfo/DWARF
-     - `17`
-     - `13`
-     - `4`
-     - :part:`76%`
-   * - llvm/unittests/DebugInfo/GSYM
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/unittests/DebugInfo/MSF
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - llvm/unittests/DebugInfo/PDB
-     - `5`
-     - `3`
-     - `2`
-     - :part:`60%`
-   * - llvm/unittests/DebugInfo/PDB/Inputs
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/Debuginfod
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/Demangle
-     - `7`
-     - `5`
-     - `2`
-     - :part:`71%`
-   * - llvm/unittests/ExecutionEngine
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/unittests/ExecutionEngine/JITLink
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/ExecutionEngine/MCJIT
-     - `7`
-     - `0`
-     - `7`
-     - :none:`0%`
-   * - llvm/unittests/ExecutionEngine/Orc
-     - `21`
-     - `14`
-     - `7`
-     - :part:`66%`
-   * - llvm/unittests/FileCheck
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/unittests/Frontend
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - llvm/unittests/FuzzMutate
-     - `4`
-     - `0`
-     - `4`
-     - :none:`0%`
-   * - llvm/unittests/InterfaceStub
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/IR
-     - `36`
-     - `6`
-     - `30`
-     - :part:`16%`
-   * - llvm/unittests/LineEditor
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/unittests/Linker
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/unittests/MC
-     - `7`
-     - `4`
-     - `3`
-     - :part:`57%`
-   * - llvm/unittests/MC/AMDGPU
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/MC/SystemZ
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/MI
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/unittests/MIR
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/unittests/ObjCopy
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/Object
-     - `9`
-     - `6`
-     - `3`
-     - :part:`66%`
-   * - llvm/unittests/ObjectYAML
-     - `5`
-     - `3`
-     - `2`
-     - :part:`60%`
-   * - llvm/unittests/Option
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/unittests/Passes
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/ProfileData
-     - `5`
-     - `2`
-     - `3`
-     - :part:`40%`
-   * - llvm/unittests/Remarks
-     - `8`
-     - `5`
-     - `3`
-     - :part:`62%`
-   * - llvm/unittests/Support
-     - `100`
-     - `35`
-     - `65`
-     - :part:`35%`
-   * - llvm/unittests/Support/CommandLineInit
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/Support/DynamicLibrary
-     - `4`
-     - `0`
-     - `4`
-     - :none:`0%`
-   * - llvm/unittests/TableGen
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - llvm/unittests/Target/AArch64
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - llvm/unittests/Target/AMDGPU
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/Target/ARM
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/unittests/Target/PowerPC
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/Target/WebAssembly
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/unittests/Target/X86
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/unittests/Testing/Support
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/TextAPI
-     - `5`
-     - `3`
-     - `2`
-     - :part:`60%`
-   * - llvm/unittests/tools/llvm-cfi-verify
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - llvm/unittests/tools/llvm-exegesis
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - llvm/unittests/tools/llvm-exegesis/AArch64
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/tools/llvm-exegesis/ARM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/tools/llvm-exegesis/Common
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/tools/llvm-exegesis/Mips
-     - `5`
-     - `3`
-     - `2`
-     - :part:`60%`
-   * - llvm/unittests/tools/llvm-exegesis/PowerPC
-     - `4`
-     - `1`
-     - `3`
-     - :part:`25%`
-   * - llvm/unittests/tools/llvm-exegesis/X86
-     - `9`
-     - `6`
-     - `3`
-     - :part:`66%`
-   * - llvm/unittests/tools/llvm-profgen
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/unittests/Transforms/IPO
-     - `4`
-     - `2`
-     - `2`
-     - :part:`50%`
-   * - llvm/unittests/Transforms/Scalar
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
-   * - llvm/unittests/Transforms/Utils
-     - `19`
-     - `8`
-     - `11`
-     - :part:`42%`
-   * - llvm/unittests/Transforms/Vectorize
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - llvm/unittests/XRay
-     - `8`
-     - `7`
-     - `1`
-     - :part:`87%`
-   * - llvm/utils/FileCheck
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/utils/fpcmp
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/utils/KillTheDoctor
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/utils/not
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/PerfectShuffle
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/utils/TableGen
-     - `78`
-     - `13`
-     - `65`
-     - :part:`16%`
-   * - llvm/utils/TableGen/GlobalISel
-     - `17`
-     - `10`
-     - `7`
-     - :part:`58%`
-   * - llvm/utils/unittest/googlemock/include/gmock
-     - `12`
-     - `0`
-     - `12`
-     - :none:`0%`
-   * - llvm/utils/unittest/googlemock/include/gmock/internal
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - llvm/utils/unittest/googlemock/include/gmock/internal/custom
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - llvm/utils/unittest/googletest/include/gtest
-     - `11`
-     - `0`
-     - `11`
-     - :none:`0%`
-   * - llvm/utils/unittest/googletest/include/gtest/internal
-     - `8`
-     - `0`
-     - `8`
-     - :none:`0%`
-   * - llvm/utils/unittest/googletest/include/gtest/internal/custom
-     - `4`
-     - `0`
-     - `4`
-     - :none:`0%`
-   * - llvm/utils/unittest/googletest/src
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/utils/unittest/UnitTestMain
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - llvm/utils/yaml-bench
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - mlir/examples/standalone/include/Standalone
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/standalone/include/Standalone-c
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/standalone/lib/CAPI
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/standalone/lib/Standalone
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/standalone/python
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/standalone/standalone-opt
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/standalone/standalone-translate
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch1
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch1/include/toy
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch1/parser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - mlir/examples/toy/Ch2
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch2/include/toy
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch2/mlir
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch2/parser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - mlir/examples/toy/Ch3
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch3/include/toy
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch3/mlir
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch3/parser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - mlir/examples/toy/Ch4
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch4/include/toy
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch4/mlir
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch4/parser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - mlir/examples/toy/Ch5
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch5/include/toy
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch5/mlir
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch5/parser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - mlir/examples/toy/Ch6
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch6/include/toy
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch6/mlir
-     - `6`
-     - `6`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch6/parser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - mlir/examples/toy/Ch7
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch7/include/toy
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch7/mlir
-     - `6`
-     - `6`
-     - `0`
-     - :good:`100%`
-   * - mlir/examples/toy/Ch7/parser
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - mlir/include/mlir
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Analysis
-     - `7`
-     - `5`
-     - `2`
-     - :part:`71%`
-   * - mlir/include/mlir/Analysis/AliasAnalysis
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Analysis/Presburger
-     - `9`
-     - `9`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Bindings/Python
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - mlir/include/mlir/CAPI
-     - `12`
-     - `12`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/AffineToStandard
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/ArithmeticToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/ArithmeticToSPIRV
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/ArmNeon2dToIntr
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/AsyncToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/BufferizationToMemRef
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/ComplexToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/ComplexToStandard
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/ControlFlowToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/ControlFlowToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/FuncToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/GPUCommon
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/GPUToNVVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/GPUToROCDL
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/GPUToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/GPUToVulkan
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - mlir/include/mlir/Conversion/LinalgToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/LinalgToStandard
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/LLVMCommon
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/MathToLibm
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/MathToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/MathToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/MemRefToLLVM
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/MemRefToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/OpenACCToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/OpenACCToSCF
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/OpenMPToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/PDLToPDLInterp
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/ReconcileUnrealizedCasts
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/SCFToControlFlow
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/SCFToGPU
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/SCFToOpenMP
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/SCFToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/ShapeToStandard
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/SPIRVToLLVM
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/StandardToLLVM
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/TensorToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/TosaToLinalg
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/TosaToSCF
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/TosaToStandard
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/VectorToGPU
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/VectorToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/VectorToSCF
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Conversion/VectorToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Affine
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Affine/Analysis
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Affine/IR
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/AMX
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Arithmetic/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Arithmetic/Transforms
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Arithmetic/Utils
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/ArmNeon
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/ArmSVE
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Async
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Async/IR
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Bufferization/IR
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Bufferization/Transforms
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Complex/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/ControlFlow/IR
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/DLTI
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/EmitC/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Func/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Func/Transforms
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/GPU
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Linalg
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Linalg/Analysis
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Linalg/ComprehensiveBufferize
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Linalg/IR
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Linalg/Transforms
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Linalg/Utils
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/LLVMIR
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/LLVMIR/Transforms
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Math/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Math/Transforms
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/MemRef/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/MemRef/Transforms
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/MemRef/Utils
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/OpenACC
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/OpenMP
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/PDL/IR
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/PDLInterp/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Quant
-     - `6`
-     - `6`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/SCF
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/SCF/Utils
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Shape/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Shape/Transforms
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/SparseTensor/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/SparseTensor/Pipelines
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/SparseTensor/Transforms
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/SparseTensor/Utils
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/SPIRV/IR
-     - `9`
-     - `9`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/SPIRV/Linking
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/SPIRV/Transforms
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/SPIRV/Utils
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Tensor/IR
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Tensor/Transforms
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Tensor/Utils
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Tosa/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Tosa/Transforms
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Tosa/Utils
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Utils
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Vector/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Vector/Transforms
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/Vector/Utils
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Dialect/X86Vector
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/ExecutionEngine
-     - `8`
-     - `7`
-     - `1`
-     - :part:`87%`
-   * - mlir/include/mlir/Interfaces
-     - `14`
-     - `13`
-     - `1`
-     - :part:`92%`
-   * - mlir/include/mlir/IR
-     - `49`
-     - `29`
-     - `20`
-     - :part:`59%`
-   * - mlir/include/mlir/Parser
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Pass
-     - `6`
-     - `0`
-     - `6`
-     - :none:`0%`
-   * - mlir/include/mlir/Reducer
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Rewrite
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Support
-     - `15`
-     - `9`
-     - `6`
-     - :part:`60%`
-   * - mlir/include/mlir/TableGen
-     - `21`
-     - `19`
-     - `2`
-     - :part:`90%`
-   * - mlir/include/mlir/Target/Cpp
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Target/LLVMIR
-     - `6`
-     - `6`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Target/LLVMIR/Dialect
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Target/LLVMIR/Dialect/AMX
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Target/LLVMIR/Dialect/ArmNeon
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Target/LLVMIR/Dialect/ArmSVE
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Target/LLVMIR/Dialect/LLVMIR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Target/LLVMIR/Dialect/NVVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Target/LLVMIR/Dialect/OpenACC
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Target/LLVMIR/Dialect/OpenMP
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Target/LLVMIR/Dialect/ROCDL
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Target/LLVMIR/Dialect/X86Vector
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Target/SPIRV
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Tools/mlir-lsp-server
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Tools/mlir-reduce
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Tools/PDLL/AST
-     - `4`
-     - `2`
-     - `2`
-     - :part:`50%`
-   * - mlir/include/mlir/Tools/PDLL/CodeGen
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Tools/PDLL/ODS
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Tools/PDLL/Parser
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir/Transforms
-     - `9`
-     - `7`
-     - `2`
-     - :part:`77%`
-   * - mlir/include/mlir-c
-     - `15`
-     - `15`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir-c/Bindings/Python
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/include/mlir-c/Dialect
-     - `11`
-     - `11`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Analysis
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Analysis/AliasAnalysis
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Analysis/Presburger
-     - `8`
-     - `8`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Bindings/Python
-     - `23`
-     - `23`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Bindings/Python/Conversions
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Bindings/Python/Transforms
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/CAPI/Conversion
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/CAPI/Debug
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/CAPI/Dialect
-     - `15`
-     - `15`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/CAPI/ExecutionEngine
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/CAPI/Interfaces
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/CAPI/IR
-     - `10`
-     - `10`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/CAPI/Registration
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/CAPI/Transforms
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/AffineToStandard
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/ArithmeticToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/ArithmeticToSPIRV
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/ArmNeon2dToIntr
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/AsyncToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/BufferizationToMemRef
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - mlir/lib/Conversion/ComplexToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/ComplexToStandard
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/ControlFlowToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/ControlFlowToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/FuncToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/GPUCommon
-     - `5`
-     - `4`
-     - `1`
-     - :part:`80%`
-   * - mlir/lib/Conversion/GPUToNVVM
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/GPUToROCDL
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/GPUToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/GPUToVulkan
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/LinalgToSPIRV
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - mlir/lib/Conversion/LinalgToStandard
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - mlir/lib/Conversion/LLVMCommon
-     - `8`
-     - `8`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/MathToLibm
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/MathToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/MathToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/MemRefToLLVM
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/MemRefToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/OpenACCToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/OpenACCToSCF
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/OpenMPToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/PDLToPDLInterp
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/ReconcileUnrealizedCasts
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/SCFToControlFlow
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/SCFToGPU
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/SCFToOpenMP
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/SCFToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/ShapeToStandard
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/SPIRVCommon
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/SPIRVToLLVM
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/StandardToLLVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/TensorToSPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/TosaToLinalg
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/TosaToSCF
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/TosaToStandard
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/VectorToGPU
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - mlir/lib/Conversion/VectorToLLVM
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/VectorToSCF
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Conversion/VectorToSPIRV
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - mlir/lib/Dialect
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Affine/Analysis
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Affine/IR
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - mlir/lib/Dialect/Affine/Transforms
-     - `14`
-     - `14`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Affine/Utils
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/AMX/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/AMX/Transforms
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Arithmetic/IR
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - mlir/lib/Dialect/Arithmetic/Transforms
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - mlir/lib/Dialect/Arithmetic/Utils
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/ArmNeon/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/ArmSVE/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/ArmSVE/Transforms
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Async/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Async/Transforms
-     - `6`
-     - `6`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Bufferization/IR
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Bufferization/Transforms
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Complex/IR
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/ControlFlow/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/DLTI
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/EmitC/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Func/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Func/Transforms
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/GPU/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/GPU/Transforms
-     - `9`
-     - `7`
-     - `2`
-     - :part:`77%`
-   * - mlir/lib/Dialect/Linalg/Analysis
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Linalg/ComprehensiveBufferize
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Linalg/IR
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Linalg/Transforms
-     - `25`
-     - `25`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Linalg/Utils
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/LLVMIR/IR
-     - `7`
-     - `5`
-     - `2`
-     - :part:`71%`
-   * - mlir/lib/Dialect/LLVMIR/Transforms
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Math/IR
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Math/Transforms
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/MemRef/IR
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/MemRef/Transforms
-     - `7`
-     - `6`
-     - `1`
-     - :part:`85%`
-   * - mlir/lib/Dialect/MemRef/Utils
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/OpenACC/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/OpenMP/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/PDL/IR
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/PDLInterp/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Quant/IR
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Quant/Transforms
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Quant/Utils
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/SCF
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/SCF/Transforms
-     - `12`
-     - `11`
-     - `1`
-     - :part:`91%`
-   * - mlir/lib/Dialect/SCF/Utils
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Shape/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Shape/Transforms
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/SparseTensor/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/SparseTensor/Pipelines
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/SparseTensor/Transforms
-     - `5`
-     - `4`
-     - `1`
-     - :part:`80%`
-   * - mlir/lib/Dialect/SparseTensor/Utils
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/SPIRV/IR
-     - `8`
-     - `6`
-     - `2`
-     - :part:`75%`
-   * - mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/SPIRV/Transforms
-     - `7`
-     - `6`
-     - `1`
-     - :part:`85%`
-   * - mlir/lib/Dialect/SPIRV/Utils
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Tensor/IR
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Tensor/Transforms
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Tensor/Utils
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Tosa/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Tosa/Transforms
-     - `6`
-     - `6`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Tosa/Utils
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Utils
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Vector/IR
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - mlir/lib/Dialect/Vector/Transforms
-     - `11`
-     - `11`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/Vector/Utils
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/X86Vector/IR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Dialect/X86Vector/Transforms
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/ExecutionEngine
-     - `9`
-     - `9`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Interfaces
-     - `12`
-     - `12`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/IR
-     - `38`
-     - `31`
-     - `7`
-     - :part:`81%`
-   * - mlir/lib/Parser
-     - `14`
-     - `10`
-     - `4`
-     - :part:`71%`
-   * - mlir/lib/Pass
-     - `8`
-     - `6`
-     - `2`
-     - :part:`75%`
-   * - mlir/lib/Reducer
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Rewrite
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - mlir/lib/Support
-     - `8`
-     - `8`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/TableGen
-     - `18`
-     - `18`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Target/Cpp
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Target/LLVMIR
-     - `7`
-     - `6`
-     - `1`
-     - :part:`85%`
-   * - mlir/lib/Target/LLVMIR/Dialect/AMX
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Target/LLVMIR/Dialect/ArmNeon
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Target/LLVMIR/Dialect/ArmSVE
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Target/LLVMIR/Dialect/LLVMIR
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Target/LLVMIR/Dialect/NVVM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Target/LLVMIR/Dialect/OpenACC
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - mlir/lib/Target/LLVMIR/Dialect/OpenMP
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Target/LLVMIR/Dialect/ROCDL
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Target/LLVMIR/Dialect/X86Vector
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Target/SPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Target/SPIRV/Deserialization
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - mlir/lib/Target/SPIRV/Serialization
-     - `4`
-     - `3`
-     - `1`
-     - :part:`75%`
-   * - mlir/lib/Tools/mlir-lsp-server
-     - `5`
-     - `4`
-     - `1`
-     - :part:`80%`
-   * - mlir/lib/Tools/mlir-lsp-server/lsp
-     - `6`
-     - `4`
-     - `2`
-     - :part:`66%`
-   * - mlir/lib/Tools/mlir-reduce
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Tools/PDLL/AST
-     - `6`
-     - `5`
-     - `1`
-     - :part:`83%`
-   * - mlir/lib/Tools/PDLL/CodeGen
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - mlir/lib/Tools/PDLL/ODS
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Tools/PDLL/Parser
-     - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
-   * - mlir/lib/Transforms
-     - `13`
-     - `11`
-     - `2`
-     - :part:`84%`
-   * - mlir/lib/Transforms/Utils
-     - `6`
-     - `6`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Translation
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/tools/mlir-cpu-runner
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/tools/mlir-linalg-ods-gen
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/tools/mlir-lsp-server
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/tools/mlir-opt
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/tools/mlir-pdll
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/tools/mlir-reduce
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/tools/mlir-shlib
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/tools/mlir-spirv-cpu-runner
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/tools/mlir-tblgen
-     - `29`
-     - `28`
-     - `1`
-     - :part:`96%`
-   * - mlir/tools/mlir-translate
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/tools/mlir-vulkan-runner
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - mlir/unittests/Analysis/Presburger
-     - `8`
-     - `8`
-     - `0`
-     - :good:`100%`
-   * - mlir/unittests/Conversion/PDLToPDLInterp
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/unittests/Dialect
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/unittests/Dialect/Affine/Analysis
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/unittests/Dialect/Quant
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/unittests/Dialect/SparseTensor
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/unittests/Dialect/SPIRV
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - mlir/unittests/Dialect/Utils
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/unittests/ExecutionEngine
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/unittests/Interfaces
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/unittests/IR
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - mlir/unittests/Pass
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - mlir/unittests/Rewrite
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - mlir/unittests/Support
-     - `5`
-     - `4`
-     - `1`
-     - :part:`80%`
-   * - mlir/unittests/TableGen
-     - `5`
-     - `3`
-     - `2`
-     - :part:`60%`
-   * - mlir/unittests/Transforms
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - openmp/libompd/src
-     - `9`
-     - `9`
-     - `0`
-     - :good:`100%`
-   * - openmp/libomptarget/DeviceRTL/include
-     - `8`
-     - `8`
-     - `0`
-     - :good:`100%`
-   * - openmp/libomptarget/DeviceRTL/src
-     - `12`
-     - `9`
-     - `3`
-     - :part:`75%`
-   * - openmp/libomptarget/include
-     - `9`
-     - `8`
-     - `1`
-     - :part:`88%`
-   * - openmp/libomptarget/plugins/amdgpu/dynamic_hsa
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - openmp/libomptarget/plugins/amdgpu/impl
-     - `13`
-     - `10`
-     - `3`
-     - :part:`76%`
-   * - openmp/libomptarget/plugins/amdgpu/src
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - openmp/libomptarget/plugins/common/elf_common
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - openmp/libomptarget/plugins/common/MemoryManager
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - openmp/libomptarget/plugins/cuda/dynamic_cuda
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - openmp/libomptarget/plugins/cuda/src
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - openmp/libomptarget/plugins/generic-elf-64bit/src
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - openmp/libomptarget/plugins/remote/include
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - openmp/libomptarget/plugins/remote/lib
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - openmp/libomptarget/plugins/remote/server
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - openmp/libomptarget/plugins/remote/src
-     - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
-   * - openmp/libomptarget/plugins/ve/src
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - openmp/libomptarget/src
-     - `7`
-     - `6`
-     - `1`
-     - :part:`85%`
-   * - openmp/libomptarget/tools/deviceinfo
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - openmp/runtime/doc/doxygen
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - openmp/runtime/src
-     - `75`
-     - `65`
-     - `10`
-     - :part:`86%`
-   * - openmp/runtime/src/thirdparty/ittnotify
-     - `6`
-     - `5`
-     - `1`
-     - :part:`83%`
-   * - openmp/runtime/src/thirdparty/ittnotify/legacy
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - openmp/tools/archer
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - openmp/tools/archer/tests/ompt
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - openmp/tools/multiplex
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - openmp/tools/multiplex/tests
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - openmp/tools/multiplex/tests/custom_data_storage
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - openmp/tools/multiplex/tests/print
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - polly/include/polly
-     - `25`
-     - `25`
-     - `0`
-     - :good:`100%`
-   * - polly/include/polly/CodeGen
-     - `14`
-     - `14`
-     - `0`
-     - :good:`100%`
-   * - polly/include/polly/Support
-     - `12`
-     - `12`
-     - `0`
-     - :good:`100%`
-   * - polly/lib/Analysis
-     - `9`
-     - `9`
-     - `0`
-     - :good:`100%`
-   * - polly/lib/CodeGen
-     - `15`
-     - `15`
-     - `0`
-     - :good:`100%`
-   * - polly/lib/Exchange
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - polly/lib/External/isl
-     - `68`
-     - `1`
-     - `67`
-     - :part:`1%`
-   * - polly/lib/External/isl/imath
-     - `6`
-     - `1`
-     - `5`
-     - :part:`16%`
-   * - polly/lib/External/isl/imath_wrap
-     - `4`
-     - `0`
-     - `4`
-     - :none:`0%`
-   * - polly/lib/External/isl/include/isl
-     - `59`
-     - `9`
-     - `50`
-     - :part:`15%`
-   * - polly/lib/External/isl/interface
-     - `8`
-     - `1`
-     - `7`
-     - :part:`12%`
-   * - polly/lib/External/pet/include
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - polly/lib/External/ppcg
-     - `17`
-     - `0`
-     - `17`
-     - :none:`0%`
-   * - polly/lib/Plugin
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - polly/lib/Support
-     - `11`
-     - `11`
-     - `0`
-     - :good:`100%`
-   * - polly/lib/Transform
-     - `15`
-     - `15`
-     - `0`
-     - :good:`100%`
-   * - polly/tools/GPURuntime
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - polly/unittests/DeLICM
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - polly/unittests/Flatten
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - polly/unittests/Isl
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - polly/unittests/ScheduleOptimizer
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - polly/unittests/ScopPassManager
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - polly/unittests/Support
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - pstl/include/pstl/internal
-     - `23`
-     - `16`
-     - `7`
-     - :part:`69%`
-   * - pstl/include/pstl/internal/omp
-     - `11`
-     - `8`
-     - `3`
-     - :part:`72%`
-   * - third-party/benchmark/cmake
-     - `5`
-     - `1`
-     - `4`
-     - :part:`20%`
-   * - third-party/benchmark/include/benchmark
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
-   * - third-party/benchmark/src
-     - `21`
-     - `21`
-     - `0`
-     - :good:`100%`
-   * - utils/bazel/llvm-project-overlay/clang/include/clang/Config
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - utils/bazel/llvm-project-overlay/llvm/include/llvm/Config
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
-   * - Total
-     - :total:`16432`
-     - :total:`8857`
-     - :total:`7575`
-     - :total:`53%`
diff --git a/clang/docs/tools/clang-formatted-files.txt b/clang/docs/tools/clang-formatted-files.txt
deleted file mode 100644
index 67ff085144f4..000000000000
--- a/clang/docs/tools/clang-formatted-files.txt
+++ /dev/null
@@ -1,8827 +0,0 @@
-bolt/include/bolt/Core/BinaryData.h
-bolt/include/bolt/Core/BinaryEmitter.h
-bolt/include/bolt/Core/BinaryLoop.h
-bolt/include/bolt/Core/BinarySection.h
-bolt/include/bolt/Core/DebugData.h
-bolt/include/bolt/Core/Exceptions.h
-bolt/include/bolt/Core/JumpTable.h
-bolt/include/bolt/Core/MCPlus.h
-bolt/include/bolt/Core/MCPlusBuilder.h
-bolt/include/bolt/Core/ParallelUtilities.h
-bolt/include/bolt/Passes/ADRRelaxationPass.h
-bolt/include/bolt/Passes/Aligner.h
-bolt/include/bolt/Passes/AllocCombiner.h
-bolt/include/bolt/Passes/AsmDump.h
-bolt/include/bolt/Passes/BinaryFunctionCallGraph.h
-bolt/include/bolt/Passes/BinaryPasses.h
-bolt/include/bolt/Passes/CacheMetrics.h
-bolt/include/bolt/Passes/CallGraph.h
-bolt/include/bolt/Passes/CallGraphWalker.h
-bolt/include/bolt/Passes/DataflowAnalysis.h
-bolt/include/bolt/Passes/DataflowInfoManager.h
-bolt/include/bolt/Passes/DominatorAnalysis.h
-bolt/include/bolt/Passes/FrameAnalysis.h
-bolt/include/bolt/Passes/FrameOptimizer.h
-bolt/include/bolt/Passes/HFSort.h
-bolt/include/bolt/Passes/IdenticalCodeFolding.h
-bolt/include/bolt/Passes/IndirectCallPromotion.h
-bolt/include/bolt/Passes/Inliner.h
-bolt/include/bolt/Passes/Instrumentation.h
-bolt/include/bolt/Passes/InstrumentationSummary.h
-bolt/include/bolt/Passes/JTFootprintReduction.h
-bolt/include/bolt/Passes/LivenessAnalysis.h
-bolt/include/bolt/Passes/LongJmp.h
-bolt/include/bolt/Passes/LoopInversionPass.h
-bolt/include/bolt/Passes/MCF.h
-bolt/include/bolt/Passes/PatchEntries.h
-bolt/include/bolt/Passes/PLTCall.h
-bolt/include/bolt/Passes/ReachingDefOrUse.h
-bolt/include/bolt/Passes/ReachingInsns.h
-bolt/include/bolt/Passes/RegAnalysis.h
-bolt/include/bolt/Passes/RegReAssign.h
-bolt/include/bolt/Passes/ReorderAlgorithm.h
-bolt/include/bolt/Passes/ReorderData.h
-bolt/include/bolt/Passes/ReorderFunctions.h
-bolt/include/bolt/Passes/ReorderUtils.h
-bolt/include/bolt/Passes/RetpolineInsertion.h
-bolt/include/bolt/Passes/ShrinkWrapping.h
-bolt/include/bolt/Passes/SplitFunctions.h
-bolt/include/bolt/Passes/StackAllocationAnalysis.h
-bolt/include/bolt/Passes/StackAvailableExpressions.h
-bolt/include/bolt/Passes/StackPointerTracking.h
-bolt/include/bolt/Passes/StackReachingUses.h
-bolt/include/bolt/Passes/StokeInfo.h
-bolt/include/bolt/Passes/TailDuplication.h
-bolt/include/bolt/Passes/ThreeWayBranch.h
-bolt/include/bolt/Passes/ValidateInternalCalls.h
-bolt/include/bolt/Passes/VeneerElimination.h
-bolt/include/bolt/Profile/BoltAddressTranslation.h
-bolt/include/bolt/Profile/DataAggregator.h
-bolt/include/bolt/Profile/DataReader.h
-bolt/include/bolt/Profile/Heatmap.h
-bolt/include/bolt/Profile/ProfileReaderBase.h
-bolt/include/bolt/Profile/ProfileYAMLMapping.h
-bolt/include/bolt/Profile/YAMLProfileReader.h
-bolt/include/bolt/Profile/YAMLProfileWriter.h
-bolt/include/bolt/Rewrite/BinaryPassManager.h
-bolt/include/bolt/Rewrite/DWARFRewriter.h
-bolt/include/bolt/Rewrite/ExecutableFileMemoryManager.h
-bolt/include/bolt/Rewrite/MachORewriteInstance.h
-bolt/include/bolt/RuntimeLibs/HugifyRuntimeLibrary.h
-bolt/include/bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h
-bolt/include/bolt/RuntimeLibs/RuntimeLibrary.h
-bolt/include/bolt/Utils/CommandLineOpts.h
-bolt/include/bolt/Utils/NameResolver.h
-bolt/include/bolt/Utils/NameShortener.h
-bolt/include/bolt/Utils/Utils.h
-bolt/lib/Core/BinaryBasicBlock.cpp
-bolt/lib/Core/BinarySection.cpp
-bolt/lib/Core/DebugData.cpp
-bolt/lib/Core/JumpTable.cpp
-bolt/lib/Core/MCPlusBuilder.cpp
-bolt/lib/Passes/ADRRelaxationPass.cpp
-bolt/lib/Passes/AllocCombiner.cpp
-bolt/lib/Passes/AsmDump.cpp
-bolt/lib/Passes/BinaryFunctionCallGraph.cpp
-bolt/lib/Passes/CacheMetrics.cpp
-bolt/lib/Passes/CallGraphWalker.cpp
-bolt/lib/Passes/DataflowAnalysis.cpp
-bolt/lib/Passes/DataflowInfoManager.cpp
-bolt/lib/Passes/HFSort.cpp
-bolt/lib/Passes/IndirectCallPromotion.cpp
-bolt/lib/Passes/Instrumentation.cpp
-bolt/lib/Passes/JTFootprintReduction.cpp
-bolt/lib/Passes/LivenessAnalysis.cpp
-bolt/lib/Passes/LoopInversionPass.cpp
-bolt/lib/Passes/PettisAndHansen.cpp
-bolt/lib/Passes/StackAllocationAnalysis.cpp
-bolt/lib/Passes/StackPointerTracking.cpp
-bolt/lib/Passes/StackReachingUses.cpp
-bolt/lib/Passes/TailDuplication.cpp
-bolt/lib/Passes/ThreeWayBranch.cpp
-bolt/lib/Passes/ValidateInternalCalls.cpp
-bolt/lib/Profile/BoltAddressTranslation.cpp
-bolt/lib/Profile/Heatmap.cpp
-bolt/lib/Profile/ProfileReaderBase.cpp
-bolt/lib/RuntimeLibs/HugifyRuntimeLibrary.cpp
-bolt/lib/RuntimeLibs/InstrumentationRuntimeLibrary.cpp
-bolt/lib/RuntimeLibs/RuntimeLibrary.cpp
-bolt/lib/Utils/Utils.cpp
-bolt/tools/heatmap/heatmap.cpp
-bolt/tools/llvm-bolt-fuzzer/llvm-bolt-fuzzer.cpp
-bolt/unittests/Core/MCPlusBuilder.cpp
-clang/bindings/python/tests/cindex/INPUTS/header1.h
-clang/bindings/python/tests/cindex/INPUTS/header2.h
-clang/bindings/python/tests/cindex/INPUTS/header3.h
-clang/examples/Attribute/Attribute.cpp
-clang/examples/CallSuperAttribute/CallSuperAttrInfo.cpp
-clang/examples/PluginsOrder/PluginsOrder.cpp
-clang/include/clang/Analysis/BodyFarm.h
-clang/include/clang/Analysis/IssueHash.h
-clang/include/clang/Analysis/MacroExpansionContext.h
-clang/include/clang/Analysis/Analyses/CalledOnceCheck.h
-clang/include/clang/Analysis/Analyses/CFGReachabilityAnalysis.h
-clang/include/clang/Analysis/Analyses/ExprMutationAnalyzer.h
-clang/include/clang/Analysis/FlowSensitive/AdornedCFG.h
-clang/include/clang/Analysis/FlowSensitive/ASTOps.h
-clang/include/clang/Analysis/FlowSensitive/CNFFormula.h
-clang/include/clang/Analysis/FlowSensitive/DataflowAnalysis.h
-clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h
-clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
-clang/include/clang/Analysis/FlowSensitive/DataflowLattice.h
-clang/include/clang/Analysis/FlowSensitive/DataflowWorklist.h
-clang/include/clang/Analysis/FlowSensitive/DebugSupport.h
-clang/include/clang/Analysis/FlowSensitive/MapLattice.h
-clang/include/clang/Analysis/FlowSensitive/MatchSwitch.h
-clang/include/clang/Analysis/FlowSensitive/NoopAnalysis.h
-clang/include/clang/Analysis/FlowSensitive/NoopLattice.h
-clang/include/clang/Analysis/FlowSensitive/Solver.h
-clang/include/clang/Analysis/FlowSensitive/StorageLocation.h
-clang/include/clang/Analysis/FlowSensitive/Transfer.h
-clang/include/clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h
-clang/include/clang/Analysis/FlowSensitive/Value.h
-clang/include/clang/Analysis/FlowSensitive/WatchedLiteralsSolver.h
-clang/include/clang/APINotes/APINotesYAMLCompiler.h
-clang/include/clang/APINotes/Types.h
-clang/include/clang/AST/AST.h
-clang/include/clang/AST/ASTContextAllocate.h
-clang/include/clang/AST/ASTDumper.h
-clang/include/clang/AST/ASTFwd.h
-clang/include/clang/AST/ASTImporterLookupTable.h
-clang/include/clang/AST/ASTImporterSharedState.h
-clang/include/clang/AST/AttrVisitor.h
-clang/include/clang/AST/Availability.h
-clang/include/clang/AST/ComputeDependence.h
-clang/include/clang/AST/CurrentSourceLocExprScope.h
-clang/include/clang/AST/DataCollection.h
-clang/include/clang/AST/ExprOpenMP.h
-clang/include/clang/AST/LexicallyOrderedRecursiveASTVisitor.h
-clang/include/clang/AST/LocInfoType.h
-clang/include/clang/AST/MangleNumberingContext.h
-clang/include/clang/AST/OptionalDiagnostic.h
-clang/include/clang/AST/OSLog.h
-clang/include/clang/AST/QualTypeNames.h
-clang/include/clang/AST/RecordLayout.h
-clang/include/clang/AST/TemplateArgumentVisitor.h
-clang/include/clang/ASTMatchers/ASTMatchersMacros.h
-clang/include/clang/ASTMatchers/Dynamic/Registry.h
-clang/include/clang/Basic/AddressSpaces.h
-clang/include/clang/Basic/AlignedAllocation.h
-clang/include/clang/Basic/AttributeCommonInfo.h
-clang/include/clang/Basic/Attributes.h
-clang/include/clang/Basic/AttrSubjectMatchRules.h
-clang/include/clang/Basic/CLWarnings.h
-clang/include/clang/Basic/CommentOptions.h
-clang/include/clang/Basic/Cuda.h
-clang/include/clang/Basic/DarwinSDKInfo.h
-clang/include/clang/Basic/DiagnosticAnalysis.h
-clang/include/clang/Basic/DiagnosticAST.h
-clang/include/clang/Basic/DiagnosticComment.h
-clang/include/clang/Basic/DiagnosticCrossTU.h
-clang/include/clang/Basic/DiagnosticDriver.h
-clang/include/clang/Basic/DiagnosticError.h
-clang/include/clang/Basic/DiagnosticFrontend.h
-clang/include/clang/Basic/DiagnosticLex.h
-clang/include/clang/Basic/DiagnosticParse.h
-clang/include/clang/Basic/DiagnosticRefactoring.h
-clang/include/clang/Basic/DiagnosticSema.h
-clang/include/clang/Basic/DiagnosticSerialization.h
-clang/include/clang/Basic/ExpressionTraits.h
-clang/include/clang/Basic/FileSystemOptions.h
-clang/include/clang/Basic/NoSanitizeList.h
-clang/include/clang/Basic/ProfileList.h
-clang/include/clang/Basic/SanitizerSpecialCaseList.h
-clang/include/clang/Basic/SyncScope.h
-clang/include/clang/Basic/TargetID.h
-clang/include/clang/Basic/Thunk.h
-clang/include/clang/Basic/TypeTraits.h
-clang/include/clang/Basic/XRayInstr.h
-clang/include/clang/Basic/XRayLists.h
-clang/include/clang/CrossTU/CrossTUDiagnostic.h
-clang/include/clang/DirectoryWatcher/DirectoryWatcher.h
-clang/include/clang/Driver/Distro.h
-clang/include/clang/Driver/DriverDiagnostic.h
-clang/include/clang/Driver/OptionUtils.h
-clang/include/clang/Driver/XRayArgs.h
-clang/include/clang/Edit/EditsReceiver.h
-clang/include/clang/Format/Format.h
-clang/include/clang/Frontend/FrontendDiagnostic.h
-clang/include/clang/Frontend/FrontendPluginRegistry.h
-clang/include/clang/Frontend/LogDiagnosticPrinter.h
-clang/include/clang/Frontend/PCHContainerOperations.h
-clang/include/clang/Frontend/PrecompiledPreamble.h
-clang/include/clang/Frontend/TextDiagnosticBuffer.h
-clang/include/clang/Frontend/TextDiagnosticPrinter.h
-clang/include/clang/Index/DeclOccurrence.h
-clang/include/clang/Index/IndexingOptions.h
-clang/include/clang/IndexSerialization/SerializablePathCollection.h
-clang/include/clang/Interpreter/Interpreter.h
-clang/include/clang/Interpreter/PartialTranslationUnit.h
-clang/include/clang/Lex/DependencyDirectivesSourceMinimizer.h
-clang/include/clang/Lex/HeaderMap.h
-clang/include/clang/Lex/HeaderMapTypes.h
-clang/include/clang/Lex/HeaderSearchOptions.h
-clang/include/clang/Lex/LexDiagnostic.h
-clang/include/clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h
-clang/include/clang/Parse/LoopHint.h
-clang/include/clang/Parse/ParseDiagnostic.h
-clang/include/clang/Sema/CleanupInfo.h
-clang/include/clang/Sema/SemaDiagnostic.h
-clang/include/clang/Sema/TemplateInstCallback.h
-clang/include/clang/Serialization/ASTBitCodes.h
-clang/include/clang/Serialization/InMemoryModuleCache.h
-clang/include/clang/Serialization/SerializationDiagnostic.h
-clang/include/clang/StaticAnalyzer/Core/CheckerRegistryData.h
-clang/include/clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h
-clang/include/clang/StaticAnalyzer/Core/PathSensitive/DynamicCastInfo.h
-clang/include/clang/StaticAnalyzer/Core/PathSensitive/DynamicExtent.h
-clang/include/clang/StaticAnalyzer/Core/PathSensitive/DynamicType.h
-clang/include/clang/StaticAnalyzer/Core/PathSensitive/DynamicTypeInfo.h
-clang/include/clang/StaticAnalyzer/Core/PathSensitive/LoopWidening.h
-clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h
-clang/include/clang/StaticAnalyzer/Core/PathSensitive/SimpleConstraintManager.h
-clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConstraintManager.h
-clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h
-clang/include/clang/StaticAnalyzer/Core/PathSensitive/StoreRef.h
-clang/include/clang/StaticAnalyzer/Frontend/AnalyzerHelpFlags.h
-clang/include/clang/StaticAnalyzer/Frontend/FrontendActions.h
-clang/include/clang/Testing/CommandLineArgs.h
-clang/include/clang/Testing/TestClangConfig.h
-clang/include/clang/Tooling/AllTUsExecution.h
-clang/include/clang/Tooling/ArgumentsAdjusters.h
-clang/include/clang/Tooling/CompilationDatabasePluginRegistry.h
-clang/include/clang/Tooling/DiagnosticsYaml.h
-clang/include/clang/Tooling/Execution.h
-clang/include/clang/Tooling/JSONCompilationDatabase.h
-clang/include/clang/Tooling/Refactoring.h
-clang/include/clang/Tooling/StandaloneExecution.h
-clang/include/clang/Tooling/ToolExecutorPluginRegistry.h
-clang/include/clang/Tooling/ASTDiff/ASTDiff.h
-clang/include/clang/Tooling/ASTDiff/ASTDiffInternal.h
-clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
-clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
-clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
-clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
-clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
-clang/include/clang/Tooling/Inclusions/HeaderIncludes.h
-clang/include/clang/Tooling/Inclusions/IncludeStyle.h
-clang/include/clang/Tooling/Inclusions/StandardLibrary.h
-clang/include/clang/Tooling/Refactoring/ASTSelection.h
-clang/include/clang/Tooling/Refactoring/AtomicChange.h
-clang/include/clang/Tooling/Refactoring/Lookup.h
-clang/include/clang/Tooling/Refactoring/RecursiveSymbolVisitor.h
-clang/include/clang/Tooling/Refactoring/RefactoringAction.h
-clang/include/clang/Tooling/Refactoring/RefactoringActionRule.h
-clang/include/clang/Tooling/Refactoring/RefactoringActionRuleRequirements.h
-clang/include/clang/Tooling/Refactoring/RefactoringDiagnostic.h
-clang/include/clang/Tooling/Refactoring/RefactoringOption.h
-clang/include/clang/Tooling/Refactoring/RefactoringOptions.h
-clang/include/clang/Tooling/Refactoring/RefactoringOptionVisitor.h
-clang/include/clang/Tooling/Refactoring/RefactoringRuleContext.h
-clang/include/clang/Tooling/Refactoring/Extract/Extract.h
-clang/include/clang/Tooling/Refactoring/Extract/SourceExtraction.h
-clang/include/clang/Tooling/Refactoring/Rename/SymbolName.h
-clang/include/clang/Tooling/Refactoring/Rename/SymbolOccurrences.h
-clang/include/clang/Tooling/Refactoring/Rename/USRFinder.h
-clang/include/clang/Tooling/Refactoring/Rename/USRFindingAction.h
-clang/include/clang/Tooling/Refactoring/Rename/USRLocFinder.h
-clang/include/clang/Tooling/Syntax/BuildTree.h
-clang/include/clang/Tooling/Syntax/Mutations.h
-clang/include/clang/Tooling/Syntax/Nodes.h
-clang/include/clang/Tooling/Syntax/Tokens.h
-clang/include/clang/Tooling/Syntax/Tree.h
-clang/include/clang/Tooling/Syntax/Pseudo/Grammar.h
-clang/include/clang/Tooling/Syntax/Pseudo/LRGraph.h
-clang/include/clang/Tooling/Syntax/Pseudo/LRTable.h
-clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h
-clang/include/clang/Tooling/Syntax/Pseudo/Token.h
-clang/include/clang/Tooling/Transformer/MatchConsumer.h
-clang/include/clang/Tooling/Transformer/Parsing.h
-clang/include/clang/Tooling/Transformer/RangeSelector.h
-clang/include/clang/Tooling/Transformer/SourceCode.h
-clang/include/clang/Tooling/Transformer/SourceCodeBuilders.h
-clang/include/clang/Tooling/Transformer/Transformer.h
-clang/include/clang-c/ExternC.h
-clang/include/clang-c/FatalErrorHandler.h
-clang/include/clang-c/Index.h
-clang/lib/Analysis/CalledOnceCheck.cpp
-clang/lib/Analysis/CloneDetection.cpp
-clang/lib/Analysis/CodeInjector.cpp
-clang/lib/Analysis/FlowSensitive/AdornedCFG.cpp
-clang/lib/Analysis/FlowSensitive/ASTOps.cpp
-clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
-clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
-clang/lib/Analysis/FlowSensitive/DebugSupport.cpp
-clang/lib/Analysis/FlowSensitive/Transfer.cpp
-clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
-clang/lib/Analysis/FlowSensitive/WatchedLiteralsSolver.cpp
-clang/lib/Analysis/plugins/CheckerDependencyHandling/CheckerDependencyHandling.cpp
-clang/lib/Analysis/plugins/SampleAnalyzer/MainCallChecker.cpp
-clang/lib/APINotes/APINotesFormat.h
-clang/lib/APINotes/APINotesTypes.cpp
-clang/lib/APINotes/APINotesYAMLCompiler.cpp
-clang/lib/AST/DataCollection.cpp
-clang/lib/AST/Linkage.h
-clang/lib/AST/ByteCode/ByteCodeGenError.cpp
-clang/lib/AST/ByteCode/ByteCodeGenError.h
-clang/lib/AST/ByteCode/Context.cpp
-clang/lib/AST/ByteCode/Context.h
-clang/lib/AST/ByteCode/Descriptor.cpp
-clang/lib/AST/ByteCode/Disasm.cpp
-clang/lib/AST/ByteCode/EvalEmitter.h
-clang/lib/AST/ByteCode/Frame.cpp
-clang/lib/AST/ByteCode/Frame.h
-clang/lib/AST/ByteCode/InterpState.h
-clang/lib/AST/ByteCode/Opcode.h
-clang/lib/AST/ByteCode/Pointer.cpp
-clang/lib/AST/ByteCode/PrimType.cpp
-clang/lib/AST/ByteCode/Record.h
-clang/lib/AST/ByteCode/Source.cpp
-clang/lib/AST/ByteCode/Source.h
-clang/lib/AST/ByteCode/State.cpp
-clang/lib/AST/ByteCode/State.h
-clang/lib/ASTMatchers/GtestMatchers.cpp
-clang/lib/ASTMatchers/Dynamic/Marshallers.cpp
-clang/lib/Basic/Attributes.cpp
-clang/lib/Basic/DarwinSDKInfo.cpp
-clang/lib/Basic/DiagnosticOptions.cpp
-clang/lib/Basic/ExpressionTraits.cpp
-clang/lib/Basic/FileEntry.cpp
-clang/lib/Basic/NoSanitizeList.cpp
-clang/lib/Basic/OpenCLOptions.cpp
-clang/lib/Basic/SanitizerSpecialCaseList.cpp
-clang/lib/Basic/TargetID.cpp
-clang/lib/Basic/Targets.h
-clang/lib/Basic/TypeTraits.cpp
-clang/lib/Basic/XRayInstr.cpp
-clang/lib/Basic/XRayLists.cpp
-clang/lib/Basic/Targets/ARC.cpp
-clang/lib/Basic/Targets/ARC.h
-clang/lib/Basic/Targets/AVR.cpp
-clang/lib/Basic/Targets/BPF.cpp
-clang/lib/Basic/Targets/BPF.h
-clang/lib/Basic/Targets/Hexagon.h
-clang/lib/Basic/Targets/Lanai.h
-clang/lib/Basic/Targets/M68k.h
-clang/lib/Basic/Targets/MSP430.h
-clang/lib/Basic/Targets/NVPTX.cpp
-clang/lib/Basic/Targets/OSTargets.cpp
-clang/lib/Basic/Targets/PNaCl.cpp
-clang/lib/Basic/Targets/PNaCl.h
-clang/lib/Basic/Targets/RISCV.h
-clang/lib/Basic/Targets/Sparc.h
-clang/lib/Basic/Targets/SPIR.cpp
-clang/lib/Basic/Targets/SystemZ.h
-clang/lib/Basic/Targets/TCE.cpp
-clang/lib/Basic/Targets/TCE.h
-clang/lib/Basic/Targets/VE.cpp
-clang/lib/Basic/Targets/VE.h
-clang/lib/Basic/Targets/WebAssembly.cpp
-clang/lib/Basic/Targets/WebAssembly.h
-clang/lib/Basic/Targets/XCore.cpp
-clang/lib/CodeGen/ABIInfoImpl.cpp
-clang/lib/CodeGen/ABIInfoImpl.h
-clang/lib/CodeGen/CGCUDARuntime.cpp
-clang/lib/CodeGen/CGLoopInfo.cpp
-clang/lib/CodeGen/CGLoopInfo.h
-clang/lib/CodeGen/CGStmtOpenMP.cpp
-clang/lib/CodeGen/MacroPPCallbacks.cpp
-clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp
-clang/lib/CodeGen/PatternInit.cpp
-clang/lib/CodeGen/PatternInit.h
-clang/lib/CodeGen/VarBypassDetector.cpp
-clang/lib/DirectoryWatcher/DirectoryScanner.cpp
-clang/lib/DirectoryWatcher/DirectoryScanner.h
-clang/lib/Driver/Distro.cpp
-clang/lib/Driver/XRayArgs.cpp
-clang/lib/Driver/ToolChains/AIX.cpp
-clang/lib/Driver/ToolChains/AIX.h
-clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp
-clang/lib/Driver/ToolChains/AMDGPUOpenMP.h
-clang/lib/Driver/ToolChains/AVR.cpp
-clang/lib/Driver/ToolChains/AVR.h
-clang/lib/Driver/ToolChains/CommonArgs.h
-clang/lib/Driver/ToolChains/CrossWindows.h
-clang/lib/Driver/ToolChains/DragonFly.h
-clang/lib/Driver/ToolChains/FreeBSD.cpp
-clang/lib/Driver/ToolChains/FreeBSD.h
-clang/lib/Driver/ToolChains/HIPAMD.h
-clang/lib/Driver/ToolChains/HIPSPV.cpp
-clang/lib/Driver/ToolChains/HIPSPV.h
-clang/lib/Driver/ToolChains/HIPUtility.cpp
-clang/lib/Driver/ToolChains/HIPUtility.h
-clang/lib/Driver/ToolChains/Hurd.cpp
-clang/lib/Driver/ToolChains/Hurd.h
-clang/lib/Driver/ToolChains/InterfaceStubs.cpp
-clang/lib/Driver/ToolChains/InterfaceStubs.h
-clang/lib/Driver/ToolChains/MipsLinux.cpp
-clang/lib/Driver/ToolChains/MSP430.h
-clang/lib/Driver/ToolChains/PPCFreeBSD.cpp
-clang/lib/Driver/ToolChains/PPCFreeBSD.h
-clang/lib/Driver/ToolChains/PPCLinux.h
-clang/lib/Driver/ToolChains/ROCm.h
-clang/lib/Driver/ToolChains/Solaris.cpp
-clang/lib/Driver/ToolChains/Solaris.h
-clang/lib/Driver/ToolChains/SPIRV.cpp
-clang/lib/Driver/ToolChains/SPIRV.h
-clang/lib/Driver/ToolChains/TCE.h
-clang/lib/Driver/ToolChains/VEToolchain.cpp
-clang/lib/Driver/ToolChains/VEToolchain.h
-clang/lib/Driver/ToolChains/WebAssembly.h
-clang/lib/Driver/ToolChains/XCore.cpp
-clang/lib/Driver/ToolChains/ZOS.cpp
-clang/lib/Driver/ToolChains/ZOS.h
-clang/lib/Driver/ToolChains/Arch/ARM.h
-clang/lib/Driver/ToolChains/Arch/M68k.cpp
-clang/lib/Driver/ToolChains/Arch/M68k.h
-clang/lib/Driver/ToolChains/Arch/RISCV.h
-clang/lib/Driver/ToolChains/Arch/VE.cpp
-clang/lib/Driver/ToolChains/Arch/VE.h
-clang/lib/Driver/ToolChains/Arch/X86.cpp
-clang/lib/Format/AffectedRangeManager.cpp
-clang/lib/Format/AffectedRangeManager.h
-clang/lib/Format/BreakableToken.cpp
-clang/lib/Format/BreakableToken.h
-clang/lib/Format/ContinuationIndenter.cpp
-clang/lib/Format/ContinuationIndenter.h
-clang/lib/Format/DefinitionBlockSeparator.cpp
-clang/lib/Format/DefinitionBlockSeparator.h
-clang/lib/Format/Encoding.h
-clang/lib/Format/Format.cpp
-clang/lib/Format/FormatInternal.h
-clang/lib/Format/FormatToken.cpp
-clang/lib/Format/FormatToken.h
-clang/lib/Format/FormatTokenLexer.cpp
-clang/lib/Format/FormatTokenLexer.h
-clang/lib/Format/MacroExpander.cpp
-clang/lib/Format/Macros.h
-clang/lib/Format/NamespaceEndCommentsFixer.cpp
-clang/lib/Format/NamespaceEndCommentsFixer.h
-clang/lib/Format/QualifierAlignmentFixer.cpp
-clang/lib/Format/QualifierAlignmentFixer.h
-clang/lib/Format/SortJavaScriptImports.cpp
-clang/lib/Format/SortJavaScriptImports.h
-clang/lib/Format/TokenAnalyzer.cpp
-clang/lib/Format/TokenAnalyzer.h
-clang/lib/Format/TokenAnnotator.cpp
-clang/lib/Format/TokenAnnotator.h
-clang/lib/Format/UnwrappedLineFormatter.cpp
-clang/lib/Format/UnwrappedLineFormatter.h
-clang/lib/Format/UnwrappedLineParser.cpp
-clang/lib/Format/UnwrappedLineParser.h
-clang/lib/Format/UsingDeclarationsSorter.cpp
-clang/lib/Format/UsingDeclarationsSorter.h
-clang/lib/Format/WhitespaceManager.cpp
-clang/lib/Format/WhitespaceManager.h
-clang/lib/Frontend/ExtractAPIConsumer.cpp
-clang/lib/Frontend/FrontendOptions.cpp
-clang/lib/Frontend/InterfaceStubFunctionsConsumer.cpp
-clang/lib/Frontend/SerializedDiagnosticReader.cpp
-clang/lib/Headers/amxintrin.h
-clang/lib/Headers/arm_neon_sve_bridge.h
-clang/lib/Headers/avx512fp16intrin.h
-clang/lib/Headers/avx512vlfp16intrin.h
-clang/lib/Headers/builtins.h
-clang/lib/Headers/inttypes.h
-clang/lib/Headers/nmmintrin.h
-clang/lib/Headers/s390intrin.h
-clang/lib/Headers/stdalign.h
-clang/lib/Headers/wmmintrin.h
-clang/lib/Headers/xtestintrin.h
-clang/lib/Headers/__clang_cuda_texture_intrinsics.h
-clang/lib/Headers/__clang_hip_libdevice_declares.h
-clang/lib/Headers/__stddef_max_align_t.h
-clang/lib/Headers/openmp_wrappers/complex.h
-clang/lib/Headers/openmp_wrappers/complex_cmath.h
-clang/lib/Headers/openmp_wrappers/math.h
-clang/lib/Headers/openmp_wrappers/time.h
-clang/lib/Headers/ppc_wrappers/mmintrin.h
-clang/lib/Headers/ppc_wrappers/smmintrin.h
-clang/lib/Index/FileIndexRecord.cpp
-clang/lib/Index/FileIndexRecord.h
-clang/lib/IndexSerialization/SerializablePathCollection.cpp
-clang/lib/Interpreter/IncrementalExecutor.cpp
-clang/lib/Interpreter/IncrementalExecutor.h
-clang/lib/Interpreter/IncrementalParser.cpp
-clang/lib/Interpreter/IncrementalParser.h
-clang/lib/Interpreter/Interpreter.cpp
-clang/lib/Lex/PreprocessorLexer.cpp
-clang/lib/Parse/ParseOpenMP.cpp
-clang/lib/Sema/CodeCompleteConsumer.cpp
-clang/lib/Sema/CoroutineStmtBuilder.h
-clang/lib/Sema/SemaSYCL.cpp
-clang/lib/Sema/UsedDeclVisitor.h
-clang/lib/Serialization/InMemoryModuleCache.cpp
-clang/lib/Serialization/ModuleFileExtension.cpp
-clang/lib/StaticAnalyzer/Checkers/AllocationState.h
-clang/lib/StaticAnalyzer/Checkers/CheckPlacementNew.cpp
-clang/lib/StaticAnalyzer/Checkers/ErrnoModeling.cpp
-clang/lib/StaticAnalyzer/Checkers/ErrnoModeling.h
-clang/lib/StaticAnalyzer/Checkers/ErrnoTesterChecker.cpp
-clang/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp
-clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp
-clang/lib/StaticAnalyzer/Checkers/InterCheckerAPI.h
-clang/lib/StaticAnalyzer/Checkers/Move.h
-clang/lib/StaticAnalyzer/Checkers/ReturnValueChecker.cpp
-clang/lib/StaticAnalyzer/Checkers/SmartPtr.h
-clang/lib/StaticAnalyzer/Checkers/SmartPtrChecker.cpp
-clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
-clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
-clang/lib/StaticAnalyzer/Checkers/StringChecker.cpp
-clang/lib/StaticAnalyzer/Checkers/Taint.cpp
-clang/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp
-clang/lib/StaticAnalyzer/Checkers/Yaml.h
-clang/lib/StaticAnalyzer/Checkers/cert/InvalidPtrChecker.cpp
-clang/lib/StaticAnalyzer/Checkers/cert/PutenvWithAutoChecker.cpp
-clang/lib/StaticAnalyzer/Checkers/UninitializedObject/UninitializedPointee.cpp
-clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
-clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.h
-clang/lib/StaticAnalyzer/Checkers/WebKit/DiagOutputUtils.h
-clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
-clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
-clang/lib/StaticAnalyzer/Checkers/WebKit/RefCntblBaseVirtualDtorChecker.cpp
-clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLambdaCapturesChecker.cpp
-clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
-clang/lib/StaticAnalyzer/Core/CallDescription.cpp
-clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp
-clang/lib/StaticAnalyzer/Core/CheckerRegistryData.cpp
-clang/lib/StaticAnalyzer/Core/CommonBugCategories.cpp
-clang/lib/StaticAnalyzer/Core/ConstraintManager.cpp
-clang/lib/StaticAnalyzer/Core/DynamicExtent.cpp
-clang/lib/StaticAnalyzer/Core/DynamicType.cpp
-clang/lib/StaticAnalyzer/Core/FunctionSummary.cpp
-clang/lib/StaticAnalyzer/Core/SimpleConstraintManager.cpp
-clang/lib/StaticAnalyzer/Core/SMTConstraintManager.cpp
-clang/lib/StaticAnalyzer/Frontend/CreateCheckerManager.cpp
-clang/lib/StaticAnalyzer/Frontend/FrontendActions.cpp
-clang/lib/StaticAnalyzer/Frontend/ModelConsumer.cpp
-clang/lib/Testing/CommandLineArgs.cpp
-clang/lib/Tooling/ArgumentsAdjusters.cpp
-clang/lib/Tooling/Execution.cpp
-clang/lib/Tooling/ExpandResponseFilesCompilationDatabase.cpp
-clang/lib/Tooling/FixIt.cpp
-clang/lib/Tooling/GuessTargetAndModeCompilationDatabase.cpp
-clang/lib/Tooling/StandaloneExecution.cpp
-clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
-clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
-clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
-clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
-clang/lib/Tooling/Inclusions/HeaderIncludes.cpp
-clang/lib/Tooling/Inclusions/IncludeStyle.cpp
-clang/lib/Tooling/Inclusions/StandardLibrary.cpp
-clang/lib/Tooling/Refactoring/ASTSelection.cpp
-clang/lib/Tooling/Refactoring/Lookup.cpp
-clang/lib/Tooling/Refactoring/RefactoringActions.cpp
-clang/lib/Tooling/Refactoring/Extract/Extract.cpp
-clang/lib/Tooling/Refactoring/Rename/SymbolOccurrences.cpp
-clang/lib/Tooling/Refactoring/Rename/USRFinder.cpp
-clang/lib/Tooling/Syntax/BuildTree.cpp
-clang/lib/Tooling/Syntax/ComputeReplacements.cpp
-clang/lib/Tooling/Syntax/Mutations.cpp
-clang/lib/Tooling/Syntax/Nodes.cpp
-clang/lib/Tooling/Syntax/Synthesis.cpp
-clang/lib/Tooling/Syntax/Tree.cpp
-clang/lib/Tooling/Syntax/Pseudo/Grammar.cpp
-clang/lib/Tooling/Syntax/Pseudo/GrammarBNF.cpp
-clang/lib/Tooling/Syntax/Pseudo/Lex.cpp
-clang/lib/Tooling/Syntax/Pseudo/LRGraph.cpp
-clang/lib/Tooling/Syntax/Pseudo/LRTable.cpp
-clang/lib/Tooling/Syntax/Pseudo/LRTableBuild.cpp
-clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp
-clang/lib/Tooling/Syntax/Pseudo/Token.cpp
-clang/lib/Tooling/Transformer/Parsing.cpp
-clang/lib/Tooling/Transformer/SourceCodeBuilders.cpp
-clang/lib/Tooling/Transformer/Stencil.cpp
-clang/lib/Tooling/Transformer/Transformer.cpp
-clang/tools/amdgpu-arch/AMDGPUArch.cpp
-clang/tools/apinotes-test/APINotesTest.cpp
-clang/tools/clang-format/ClangFormat.cpp
-clang/tools/clang-fuzzer/ClangFuzzer.cpp
-clang/tools/clang-fuzzer/DummyClangFuzzer.cpp
-clang/tools/clang-fuzzer/ExampleClangLLVMProtoFuzzer.cpp
-clang/tools/clang-fuzzer/ExampleClangLoopProtoFuzzer.cpp
-clang/tools/clang-fuzzer/handle-llvm/handle_llvm.h
-clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
-clang/tools/clang-linker-wrapper/OffloadWrapper.cpp
-clang/tools/clang-refactor/ClangRefactor.cpp
-clang/tools/clang-refactor/TestSupport.cpp
-clang/tools/clang-refactor/TestSupport.h
-clang/tools/clang-refactor/ToolRefactoringResultConsumer.h
-clang/tools/clang-repl/ClangRepl.cpp
-clang/tools/clang-scan-deps/ClangScanDeps.cpp
-clang/tools/clang-shlib/clang-shlib.cpp
-clang/tools/driver/cc1gen_reproducer_main.cpp
-clang/tools/libclang/CIndex.cpp
-clang/tools/libclang/CIndexUSRs.cpp
-clang/tools/libclang/CursorVisitor.h
-clang/tools/libclang/CXCursor.cpp
-clang/tools/libclang/CXCursor.h
-clang/tools/scan-build-py/tests/functional/src/include/clean-one.h
-clang/unittests/Analysis/CFGBuildResult.h
-clang/unittests/Analysis/MacroExpansionContextTest.cpp
-clang/unittests/Analysis/FlowSensitive/ASTOpsTest.cpp
-clang/unittests/Analysis/FlowSensitive/CNFFormula.cpp
-clang/unittests/Analysis/FlowSensitive/DataflowAnalysisContextTest.cpp
-clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp
-clang/unittests/Analysis/FlowSensitive/MapLatticeTest.cpp
-clang/unittests/Analysis/FlowSensitive/MatchSwitchTest.cpp
-clang/unittests/Analysis/FlowSensitive/MultiVarConstantPropagationTest.cpp
-clang/unittests/Analysis/FlowSensitive/SingleVarConstantPropagationTest.cpp
-clang/unittests/Analysis/FlowSensitive/SolverTest.h
-clang/unittests/Analysis/FlowSensitive/TestingSupport.cpp
-clang/unittests/Analysis/FlowSensitive/TestingSupport.h
-clang/unittests/Analysis/FlowSensitive/TestingSupportTest.cpp
-clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp
-clang/unittests/Analysis/FlowSensitive/WatchedLiteralsSolver.cpp
-clang/unittests/Analysis/FlowSensitive/WatchedLiteralsSolverTest.cpp
-clang/unittests/AST/ASTImporterFixtures.cpp
-clang/unittests/AST/ASTImporterFixtures.h
-clang/unittests/AST/ASTImporterObjCTest.cpp
-clang/unittests/AST/ASTPrint.h
-clang/unittests/AST/AttrTest.cpp
-clang/unittests/AST/RecursiveASTVisitorTest.cpp
-clang/unittests/AST/SizelessTypesTest.cpp
-clang/unittests/AST/TypePrinterTest.cpp
-clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
-clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
-clang/unittests/ASTMatchers/ASTMatchersTest.h
-clang/unittests/Basic/DarwinSDKInfoTest.cpp
-clang/unittests/Basic/FileEntryTest.cpp
-clang/unittests/Basic/LineOffsetMappingTest.cpp
-clang/unittests/Basic/SanitizersTest.cpp
-clang/unittests/CodeGen/CheckTargetFeaturesTest.cpp
-clang/unittests/CrossTU/CrossTranslationUnitTest.cpp
-clang/unittests/Driver/SanitizerArgsTest.cpp
-clang/unittests/Format/CleanupTest.cpp
-clang/unittests/Format/DefinitionBlockSeparatorTest.cpp
-clang/unittests/Format/FormatTest.cpp
-clang/unittests/Format/FormatTestComments.cpp
-clang/unittests/Format/FormatTestCSharp.cpp
-clang/unittests/Format/FormatTestJava.cpp
-clang/unittests/Format/FormatTestJS.cpp
-clang/unittests/Format/FormatTestJson.cpp
-clang/unittests/Format/FormatTestObjC.cpp
-clang/unittests/Format/FormatTestProto.cpp
-clang/unittests/Format/FormatTestRawStrings.cpp
-clang/unittests/Format/FormatTestSelective.cpp
-clang/unittests/Format/FormatTestTableGen.cpp
-clang/unittests/Format/FormatTestTextProto.cpp
-clang/unittests/Format/FormatTestUtils.h
-clang/unittests/Format/MacroExpanderTest.cpp
-clang/unittests/Format/NamespaceEndCommentsFixerTest.cpp
-clang/unittests/Format/QualifierFixerTest.cpp
-clang/unittests/Format/SortImportsTestJava.cpp
-clang/unittests/Format/SortImportsTestJS.cpp
-clang/unittests/Format/SortIncludesTest.cpp
-clang/unittests/Format/TestLexer.h
-clang/unittests/Format/TokenAnnotatorTest.cpp
-clang/unittests/Format/UsingDeclarationsSorterTest.cpp
-clang/unittests/Frontend/ASTUnitTest.cpp
-clang/unittests/Frontend/CompilerInstanceTest.cpp
-clang/unittests/Frontend/FixedPointString.cpp
-clang/unittests/Frontend/OutputStreamTest.cpp
-clang/unittests/Frontend/ParsedSourceLocationTest.cpp
-clang/unittests/Frontend/TextDiagnosticTest.cpp
-clang/unittests/Frontend/UtilsTest.cpp
-clang/unittests/Index/IndexTests.cpp
-clang/unittests/Interpreter/IncrementalProcessingTest.cpp
-clang/unittests/Interpreter/InterpreterTest.cpp
-clang/unittests/Lex/HeaderMapTest.cpp
-clang/unittests/Lex/HeaderMapTestUtils.h
-clang/unittests/Lex/HeaderSearchTest.cpp
-clang/unittests/Lex/PPMemoryAllocationsTest.cpp
-clang/unittests/libclang/CrashTests/LibclangCrashTest.cpp
-clang/unittests/Rewrite/RewriterTest.cpp
-clang/unittests/Sema/CodeCompleteTest.cpp
-clang/unittests/Sema/GslOwnerPointerInference.cpp
-clang/unittests/Serialization/InMemoryModuleCacheTest.cpp
-clang/unittests/Serialization/ModuleCacheTest.cpp
-clang/unittests/StaticAnalyzer/BugReportInterestingnessTest.cpp
-clang/unittests/StaticAnalyzer/CallEventTest.cpp
-clang/unittests/StaticAnalyzer/CheckerRegistration.h
-clang/unittests/StaticAnalyzer/ConflictingEvalCallsTest.cpp
-clang/unittests/StaticAnalyzer/StoreTest.cpp
-clang/unittests/StaticAnalyzer/SValTest.cpp
-clang/unittests/StaticAnalyzer/SymbolReaperTest.cpp
-clang/unittests/Tooling/CastExprTest.cpp
-clang/unittests/Tooling/DependencyScannerTest.cpp
-clang/unittests/Tooling/ExecutionTest.cpp
-clang/unittests/Tooling/LookupTest.cpp
-clang/unittests/Tooling/RecursiveASTVisitorTestPostOrderVisitor.cpp
-clang/unittests/Tooling/RefactoringActionRulesTest.cpp
-clang/unittests/Tooling/ReplacementTest.h
-clang/unittests/Tooling/SourceCodeBuildersTest.cpp
-clang/unittests/Tooling/StandardLibraryTest.cpp
-clang/unittests/Tooling/StencilTest.cpp
-clang/unittests/Tooling/RecursiveASTVisitorTests/CallbacksCallExpr.cpp
-clang/unittests/Tooling/RecursiveASTVisitorTests/CallbacksLeaf.cpp
-clang/unittests/Tooling/RecursiveASTVisitorTests/Concept.cpp
-clang/unittests/Tooling/RecursiveASTVisitorTests/CXXMethodDecl.cpp
-clang/unittests/Tooling/RecursiveASTVisitorTests/ImplicitCtorInitializer.cpp
-clang/unittests/Tooling/RecursiveASTVisitorTests/InitListExprPostOrder.cpp
-clang/unittests/Tooling/RecursiveASTVisitorTests/InitListExprPreOrder.cpp
-clang/unittests/Tooling/RecursiveASTVisitorTests/IntegerLiteral.cpp
-clang/unittests/Tooling/RecursiveASTVisitorTests/MemberPointerTypeLoc.cpp
-clang/unittests/Tooling/RecursiveASTVisitorTests/NestedNameSpecifiers.cpp
-clang/unittests/Tooling/RecursiveASTVisitorTests/ParenExpr.cpp
-clang/unittests/Tooling/RecursiveASTVisitorTests/TraversalScope.cpp
-clang/unittests/Tooling/Syntax/TokensTest.cpp
-clang/unittests/Tooling/Syntax/TreeTestBase.cpp
-clang/unittests/Tooling/Syntax/TreeTestBase.h
-clang/unittests/Tooling/Syntax/Pseudo/GrammarTest.cpp
-clang/unittests/Tooling/Syntax/Pseudo/LRTableTest.cpp
-clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp
-clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
-clang/utils/TableGen/ClangDataCollectorsEmitter.cpp
-clang/utils/TableGen/ClangSyntaxEmitter.cpp
-clang/utils/TableGen/TableGenBackends.h
-clang-tools-extra/clang-apply-replacements/include/clang-apply-replacements/Tooling/ApplyReplacements.h
-clang-tools-extra/clang-apply-replacements/lib/Tooling/ApplyReplacements.cpp
-clang-tools-extra/clang-apply-replacements/tool/ClangApplyReplacementsMain.cpp
-clang-tools-extra/clang-doc/BitcodeReader.cpp
-clang-tools-extra/clang-doc/BitcodeReader.h
-clang-tools-extra/clang-doc/BitcodeWriter.cpp
-clang-tools-extra/clang-doc/BitcodeWriter.h
-clang-tools-extra/clang-doc/ClangDoc.cpp
-clang-tools-extra/clang-doc/ClangDoc.h
-clang-tools-extra/clang-doc/Generators.cpp
-clang-tools-extra/clang-doc/Generators.h
-clang-tools-extra/clang-doc/Mapper.cpp
-clang-tools-extra/clang-doc/Mapper.h
-clang-tools-extra/clang-doc/MDGenerator.cpp
-clang-tools-extra/clang-doc/Representation.cpp
-clang-tools-extra/clang-doc/Representation.h
-clang-tools-extra/clang-doc/Serialize.cpp
-clang-tools-extra/clang-doc/Serialize.h
-clang-tools-extra/clang-doc/YAMLGenerator.cpp
-clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
-clang-tools-extra/clang-include-fixer/FuzzySymbolIndex.cpp
-clang-tools-extra/clang-include-fixer/IncludeFixer.h
-clang-tools-extra/clang-include-fixer/IncludeFixerContext.h
-clang-tools-extra/clang-include-fixer/InMemorySymbolIndex.cpp
-clang-tools-extra/clang-include-fixer/InMemorySymbolIndex.h
-clang-tools-extra/clang-include-fixer/SymbolIndex.h
-clang-tools-extra/clang-include-fixer/YamlSymbolIndex.cpp
-clang-tools-extra/clang-include-fixer/YamlSymbolIndex.h
-clang-tools-extra/clang-include-fixer/find-all-symbols/FindAllMacros.cpp
-clang-tools-extra/clang-include-fixer/find-all-symbols/FindAllMacros.h
-clang-tools-extra/clang-include-fixer/find-all-symbols/FindAllSymbols.h
-clang-tools-extra/clang-include-fixer/find-all-symbols/FindAllSymbolsAction.h
-clang-tools-extra/clang-include-fixer/find-all-symbols/HeaderMapCollector.cpp
-clang-tools-extra/clang-include-fixer/find-all-symbols/HeaderMapCollector.h
-clang-tools-extra/clang-include-fixer/find-all-symbols/PathConfig.cpp
-clang-tools-extra/clang-include-fixer/find-all-symbols/PathConfig.h
-clang-tools-extra/clang-include-fixer/find-all-symbols/PragmaCommentHandler.cpp
-clang-tools-extra/clang-include-fixer/find-all-symbols/PragmaCommentHandler.h
-clang-tools-extra/clang-include-fixer/find-all-symbols/STLPostfixHeaderMap.cpp
-clang-tools-extra/clang-include-fixer/find-all-symbols/STLPostfixHeaderMap.h
-clang-tools-extra/clang-include-fixer/find-all-symbols/SymbolReporter.h
-clang-tools-extra/clang-include-fixer/plugin/IncludeFixerPlugin.cpp
-clang-tools-extra/clang-move/HelperDeclRefGraph.h
-clang-tools-extra/clang-move/tool/ClangMove.cpp
-clang-tools-extra/clang-query/Query.cpp
-clang-tools-extra/clang-query/Query.h
-clang-tools-extra/clang-query/QueryParser.h
-clang-tools-extra/clang-query/QuerySession.h
-clang-tools-extra/clang-reorder-fields/ReorderFieldsAction.h
-clang-tools-extra/clang-tidy/ClangTidy.h
-clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
-clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
-clang-tools-extra/clang-tidy/ClangTidyForceLinker.h
-clang-tools-extra/clang-tidy/ClangTidyModule.cpp
-clang-tools-extra/clang-tidy/ClangTidyModule.h
-clang-tools-extra/clang-tidy/ClangTidyModuleRegistry.h
-clang-tools-extra/clang-tidy/ClangTidyOptions.h
-clang-tools-extra/clang-tidy/ClangTidyProfiling.cpp
-clang-tools-extra/clang-tidy/ClangTidyProfiling.h
-clang-tools-extra/clang-tidy/GlobList.cpp
-clang-tools-extra/clang-tidy/GlobList.h
-clang-tools-extra/clang-tidy/NoLintDirectiveHandler.cpp
-clang-tools-extra/clang-tidy/NoLintDirectiveHandler.h
-clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h
-clang-tools-extra/clang-tidy/abseil/CleanupCtadCheck.cpp
-clang-tools-extra/clang-tidy/abseil/CleanupCtadCheck.h
-clang-tools-extra/clang-tidy/abseil/DurationAdditionCheck.cpp
-clang-tools-extra/clang-tidy/abseil/DurationAdditionCheck.h
-clang-tools-extra/clang-tidy/abseil/DurationComparisonCheck.cpp
-clang-tools-extra/clang-tidy/abseil/DurationComparisonCheck.h
-clang-tools-extra/clang-tidy/abseil/DurationConversionCastCheck.cpp
-clang-tools-extra/clang-tidy/abseil/DurationConversionCastCheck.h
-clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.cpp
-clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.cpp
-clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.h
-clang-tools-extra/clang-tidy/abseil/DurationFactoryScaleCheck.cpp
-clang-tools-extra/clang-tidy/abseil/DurationFactoryScaleCheck.h
-clang-tools-extra/clang-tidy/abseil/DurationRewriter.cpp
-clang-tools-extra/clang-tidy/abseil/DurationRewriter.h
-clang-tools-extra/clang-tidy/abseil/DurationSubtractionCheck.cpp
-clang-tools-extra/clang-tidy/abseil/DurationSubtractionCheck.h
-clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.cpp
-clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.h
-clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp
-clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.h
-clang-tools-extra/clang-tidy/abseil/NoNamespaceCheck.h
-clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.cpp
-clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.h
-clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp
-clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.h
-clang-tools-extra/clang-tidy/abseil/TimeComparisonCheck.cpp
-clang-tools-extra/clang-tidy/abseil/TimeComparisonCheck.h
-clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp
-clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp
-clang-tools-extra/clang-tidy/altera/AlteraTidyModule.cpp
-clang-tools-extra/clang-tidy/altera/IdDependentBackwardBranchCheck.cpp
-clang-tools-extra/clang-tidy/altera/IdDependentBackwardBranchCheck.h
-clang-tools-extra/clang-tidy/altera/KernelNameRestrictionCheck.cpp
-clang-tools-extra/clang-tidy/altera/KernelNameRestrictionCheck.h
-clang-tools-extra/clang-tidy/altera/SingleWorkItemBarrierCheck.cpp
-clang-tools-extra/clang-tidy/altera/SingleWorkItemBarrierCheck.h
-clang-tools-extra/clang-tidy/altera/UnrollLoopsCheck.cpp
-clang-tools-extra/clang-tidy/altera/UnrollLoopsCheck.h
-clang-tools-extra/clang-tidy/android/CloexecAccept4Check.cpp
-clang-tools-extra/clang-tidy/android/CloexecAccept4Check.h
-clang-tools-extra/clang-tidy/android/CloexecAcceptCheck.h
-clang-tools-extra/clang-tidy/android/CloexecCheck.h
-clang-tools-extra/clang-tidy/android/CloexecCreatCheck.h
-clang-tools-extra/clang-tidy/android/CloexecDupCheck.cpp
-clang-tools-extra/clang-tidy/android/CloexecDupCheck.h
-clang-tools-extra/clang-tidy/android/CloexecEpollCreate1Check.cpp
-clang-tools-extra/clang-tidy/android/CloexecEpollCreate1Check.h
-clang-tools-extra/clang-tidy/android/CloexecEpollCreateCheck.cpp
-clang-tools-extra/clang-tidy/android/CloexecEpollCreateCheck.h
-clang-tools-extra/clang-tidy/android/CloexecFopenCheck.h
-clang-tools-extra/clang-tidy/android/CloexecInotifyInit1Check.cpp
-clang-tools-extra/clang-tidy/android/CloexecInotifyInit1Check.h
-clang-tools-extra/clang-tidy/android/CloexecInotifyInitCheck.cpp
-clang-tools-extra/clang-tidy/android/CloexecInotifyInitCheck.h
-clang-tools-extra/clang-tidy/android/CloexecMemfdCreateCheck.cpp
-clang-tools-extra/clang-tidy/android/CloexecMemfdCreateCheck.h
-clang-tools-extra/clang-tidy/android/CloexecOpenCheck.h
-clang-tools-extra/clang-tidy/android/CloexecPipe2Check.h
-clang-tools-extra/clang-tidy/android/CloexecPipeCheck.h
-clang-tools-extra/clang-tidy/android/CloexecSocketCheck.h
-clang-tools-extra/clang-tidy/android/ComparisonInTempFailureRetryCheck.h
-clang-tools-extra/clang-tidy/boost/BoostTidyModule.cpp
-clang-tools-extra/clang-tidy/boost/UseToStringCheck.cpp
-clang-tools-extra/clang-tidy/boost/UseToStringCheck.h
-clang-tools-extra/clang-tidy/bugprone/ArgumentCommentCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/ArgumentCommentCheck.h
-clang-tools-extra/clang-tidy/bugprone/AssertSideEffectCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/AssertSideEffectCheck.h
-clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.h
-clang-tools-extra/clang-tidy/bugprone/BoolPointerImplicitConversionCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/BoolPointerImplicitConversionCheck.h
-clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.h
-clang-tools-extra/clang-tidy/bugprone/CopyConstructorInitCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/CopyConstructorInitCheck.h
-clang-tools-extra/clang-tidy/bugprone/DanglingHandleCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/DanglingHandleCheck.h
-clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.h
-clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.h
-clang-tools-extra/clang-tidy/bugprone/ExceptionEscapeCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/ExceptionEscapeCheck.h
-clang-tools-extra/clang-tidy/bugprone/FoldInitTypeCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/FoldInitTypeCheck.h
-clang-tools-extra/clang-tidy/bugprone/ForwardDeclarationNamespaceCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/ForwardDeclarationNamespaceCheck.h
-clang-tools-extra/clang-tidy/bugprone/ForwardingReferenceOverloadCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/ForwardingReferenceOverloadCheck.h
-clang-tools-extra/clang-tidy/bugprone/ImplicitWideningOfMultiplicationResultCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/ImplicitWideningOfMultiplicationResultCheck.h
-clang-tools-extra/clang-tidy/bugprone/InaccurateEraseCheck.h
-clang-tools-extra/clang-tidy/bugprone/IncorrectRoundingsCheck.h
-clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.h
-clang-tools-extra/clang-tidy/bugprone/IntegerDivisionCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/IntegerDivisionCheck.h
-clang-tools-extra/clang-tidy/bugprone/LambdaFunctionNameCheck.h
-clang-tools-extra/clang-tidy/bugprone/MacroParenthesesCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/MacroParenthesesCheck.h
-clang-tools-extra/clang-tidy/bugprone/MacroRepeatedSideEffectsCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/MacroRepeatedSideEffectsCheck.h
-clang-tools-extra/clang-tidy/bugprone/MisplacedPointerArithmeticInAllocCheck.h
-clang-tools-extra/clang-tidy/bugprone/MisplacedWideningCastCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/MisplacedWideningCastCheck.h
-clang-tools-extra/clang-tidy/bugprone/MoveForwardingReferenceCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/MoveForwardingReferenceCheck.h
-clang-tools-extra/clang-tidy/bugprone/MultipleStatementMacroCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/MultipleStatementMacroCheck.h
-clang-tools-extra/clang-tidy/bugprone/NoEscapeCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/NoEscapeCheck.h
-clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.h
-clang-tools-extra/clang-tidy/bugprone/ParentVirtualCallCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/ParentVirtualCallCheck.h
-clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/RedundantBranchConditionCheck.h
-clang-tools-extra/clang-tidy/bugprone/ReservedIdentifierCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/ReservedIdentifierCheck.h
-clang-tools-extra/clang-tidy/bugprone/SharedPtrArrayMismatchCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/SharedPtrArrayMismatchCheck.h
-clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.h
-clang-tools-extra/clang-tidy/bugprone/SignedCharMisuseCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/SignedCharMisuseCheck.h
-clang-tools-extra/clang-tidy/bugprone/SizeofContainerCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/SizeofContainerCheck.h
-clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.h
-clang-tools-extra/clang-tidy/bugprone/SmartPtrArrayMismatchCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/SmartPtrArrayMismatchCheck.h
-clang-tools-extra/clang-tidy/bugprone/SpuriouslyWakeUpFunctionsCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/StringConstructorCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/StringConstructorCheck.h
-clang-tools-extra/clang-tidy/bugprone/StringIntegerAssignmentCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/StringIntegerAssignmentCheck.h
-clang-tools-extra/clang-tidy/bugprone/StringLiteralWithEmbeddedNulCheck.h
-clang-tools-extra/clang-tidy/bugprone/StringviewNullptrCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/StringviewNullptrCheck.h
-clang-tools-extra/clang-tidy/bugprone/SuspiciousEnumUsageCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/SuspiciousIncludeCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/SuspiciousIncludeCheck.h
-clang-tools-extra/clang-tidy/bugprone/SuspiciousMemoryComparisonCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/SuspiciousMemoryComparisonCheck.h
-clang-tools-extra/clang-tidy/bugprone/SuspiciousMemsetUsageCheck.h
-clang-tools-extra/clang-tidy/bugprone/SuspiciousMissingCommaCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/SuspiciousMissingCommaCheck.h
-clang-tools-extra/clang-tidy/bugprone/SuspiciousSemicolonCheck.h
-clang-tools-extra/clang-tidy/bugprone/SuspiciousStringCompareCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/SuspiciousStringCompareCheck.h
-clang-tools-extra/clang-tidy/bugprone/SwappedArgumentsCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/SwappedArgumentsCheck.h
-clang-tools-extra/clang-tidy/bugprone/TerminatingContinueCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/TerminatingContinueCheck.h
-clang-tools-extra/clang-tidy/bugprone/ThrowKeywordMissingCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/ThrowKeywordMissingCheck.h
-clang-tools-extra/clang-tidy/bugprone/TooSmallLoopVariableCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/TooSmallLoopVariableCheck.h
-clang-tools-extra/clang-tidy/bugprone/UndefinedMemoryManipulationCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/UndefinedMemoryManipulationCheck.h
-clang-tools-extra/clang-tidy/bugprone/UndelegatedConstructorCheck.h
-clang-tools-extra/clang-tidy/bugprone/UnhandledExceptionAtNewCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/UnhandledExceptionAtNewCheck.h
-clang-tools-extra/clang-tidy/bugprone/UnhandledSelfAssignmentCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/UnhandledSelfAssignmentCheck.h
-clang-tools-extra/clang-tidy/bugprone/UnusedRaiiCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/UnusedRaiiCheck.h
-clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.h
-clang-tools-extra/clang-tidy/bugprone/UseAfterMoveCheck.h
-clang-tools-extra/clang-tidy/bugprone/VirtualNearMissCheck.cpp
-clang-tools-extra/clang-tidy/bugprone/VirtualNearMissCheck.h
-clang-tools-extra/clang-tidy/cert/CommandProcessorCheck.cpp
-clang-tools-extra/clang-tidy/cert/CommandProcessorCheck.h
-clang-tools-extra/clang-tidy/cert/DefaultOperatorNewAlignmentCheck.cpp
-clang-tools-extra/clang-tidy/cert/DefaultOperatorNewAlignmentCheck.h
-clang-tools-extra/clang-tidy/cert/DontModifyStdNamespaceCheck.cpp
-clang-tools-extra/clang-tidy/cert/DontModifyStdNamespaceCheck.h
-clang-tools-extra/clang-tidy/cert/FloatLoopCounter.cpp
-clang-tools-extra/clang-tidy/cert/FloatLoopCounter.h
-clang-tools-extra/clang-tidy/cert/LimitedRandomnessCheck.cpp
-clang-tools-extra/clang-tidy/cert/LimitedRandomnessCheck.h
-clang-tools-extra/clang-tidy/cert/MutatingCopyCheck.cpp
-clang-tools-extra/clang-tidy/cert/MutatingCopyCheck.h
-clang-tools-extra/clang-tidy/cert/NonTrivialTypesLibcMemoryCallsCheck.cpp
-clang-tools-extra/clang-tidy/cert/NonTrivialTypesLibcMemoryCallsCheck.h
-clang-tools-extra/clang-tidy/cert/ProperlySeededRandomGeneratorCheck.cpp
-clang-tools-extra/clang-tidy/cert/ProperlySeededRandomGeneratorCheck.h
-clang-tools-extra/clang-tidy/cert/SetLongJmpCheck.cpp
-clang-tools-extra/clang-tidy/cert/SetLongJmpCheck.h
-clang-tools-extra/clang-tidy/cert/StaticObjectExceptionCheck.cpp
-clang-tools-extra/clang-tidy/cert/StaticObjectExceptionCheck.h
-clang-tools-extra/clang-tidy/cert/StrToNumCheck.cpp
-clang-tools-extra/clang-tidy/cert/StrToNumCheck.h
-clang-tools-extra/clang-tidy/cert/ThrownExceptionTypeCheck.cpp
-clang-tools-extra/clang-tidy/cert/ThrownExceptionTypeCheck.h
-clang-tools-extra/clang-tidy/cert/VariadicFunctionDefCheck.cpp
-clang-tools-extra/clang-tidy/cert/VariadicFunctionDefCheck.h
-clang-tools-extra/clang-tidy/concurrency/MtUnsafeCheck.cpp
-clang-tools-extra/clang-tidy/concurrency/MtUnsafeCheck.h
-clang-tools-extra/clang-tidy/concurrency/ThreadCanceltypeAsynchronousCheck.cpp
-clang-tools-extra/clang-tidy/concurrency/ThreadCanceltypeAsynchronousCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/AvoidGotoCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/AvoidGotoCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/AvoidNonConstGlobalVariablesCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/AvoidNonConstGlobalVariablesCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/InterfacesGlobalInitCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/InterfacesGlobalInitCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/MacroUsageCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/MacroUsageCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/NarrowingConversionsCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/NarrowingConversionsCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/NoMallocCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/NoMallocCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/OwningMemoryCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/OwningMemoryCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/PreferMemberInitializerCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/PreferMemberInitializerCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsArrayToPointerDecayCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsArrayToPointerDecayCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeConstCastCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeConstCastCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeCstyleCastCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeCstyleCastCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeReinterpretCastCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeReinterpretCastCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeStaticCastDowncastCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeStaticCastDowncastCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeUnionAccessCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeUnionAccessCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeVarargCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeVarargCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.h
-clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp
-clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.h
-clang-tools-extra/clang-tidy/darwin/DispatchOnceNonstaticCheck.cpp
-clang-tools-extra/clang-tidy/darwin/DispatchOnceNonstaticCheck.h
-clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsCallsCheck.cpp
-clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsCallsCheck.h
-clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsDeclarationsCheck.cpp
-clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsDeclarationsCheck.h
-clang-tools-extra/clang-tidy/fuchsia/FuchsiaTidyModule.cpp
-clang-tools-extra/clang-tidy/fuchsia/MultipleInheritanceCheck.h
-clang-tools-extra/clang-tidy/fuchsia/OverloadedOperatorCheck.cpp
-clang-tools-extra/clang-tidy/fuchsia/OverloadedOperatorCheck.h
-clang-tools-extra/clang-tidy/fuchsia/StaticallyConstructedObjectsCheck.cpp
-clang-tools-extra/clang-tidy/fuchsia/TrailingReturnCheck.cpp
-clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.h
-clang-tools-extra/clang-tidy/google/AvoidNSObjectNewCheck.cpp
-clang-tools-extra/clang-tidy/google/AvoidNSObjectNewCheck.h
-clang-tools-extra/clang-tidy/google/AvoidUnderscoreInGoogletestNameCheck.cpp
-clang-tools-extra/clang-tidy/google/AvoidUnderscoreInGoogletestNameCheck.h
-clang-tools-extra/clang-tidy/google/DefaultArgumentsCheck.cpp
-clang-tools-extra/clang-tidy/google/DefaultArgumentsCheck.h
-clang-tools-extra/clang-tidy/google/ExplicitConstructorCheck.h
-clang-tools-extra/clang-tidy/google/ExplicitMakePairCheck.cpp
-clang-tools-extra/clang-tidy/google/ExplicitMakePairCheck.h
-clang-tools-extra/clang-tidy/google/FunctionNamingCheck.cpp
-clang-tools-extra/clang-tidy/google/FunctionNamingCheck.h
-clang-tools-extra/clang-tidy/google/GlobalNamesInHeadersCheck.h
-clang-tools-extra/clang-tidy/google/IntegerTypesCheck.h
-clang-tools-extra/clang-tidy/google/OverloadedUnaryAndCheck.cpp
-clang-tools-extra/clang-tidy/google/OverloadedUnaryAndCheck.h
-clang-tools-extra/clang-tidy/google/TodoCommentCheck.cpp
-clang-tools-extra/clang-tidy/google/TodoCommentCheck.h
-clang-tools-extra/clang-tidy/google/UnnamedNamespaceInHeaderCheck.h
-clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp
-clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.h
-clang-tools-extra/clang-tidy/google/UsingNamespaceDirectiveCheck.h
-clang-tools-extra/clang-tidy/hicpp/ExceptionBaseclassCheck.cpp
-clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp
-clang-tools-extra/clang-tidy/hicpp/MultiwayPathsCoveredCheck.h
-clang-tools-extra/clang-tidy/hicpp/NoAssemblerCheck.cpp
-clang-tools-extra/clang-tidy/hicpp/NoAssemblerCheck.h
-clang-tools-extra/clang-tidy/hicpp/SignedBitwiseCheck.cpp
-clang-tools-extra/clang-tidy/hicpp/SignedBitwiseCheck.h
-clang-tools-extra/clang-tidy/linuxkernel/LinuxKernelTidyModule.cpp
-clang-tools-extra/clang-tidy/linuxkernel/MustCheckErrsCheck.h
-clang-tools-extra/clang-tidy/llvm/HeaderGuardCheck.cpp
-clang-tools-extra/clang-tidy/llvm/HeaderGuardCheck.h
-clang-tools-extra/clang-tidy/llvm/IncludeOrderCheck.cpp
-clang-tools-extra/clang-tidy/llvm/IncludeOrderCheck.h
-clang-tools-extra/clang-tidy/llvm/LLVMTidyModule.cpp
-clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.cpp
-clang-tools-extra/clang-tidy/llvm/PreferRegisterOverUnsignedCheck.cpp
-clang-tools-extra/clang-tidy/llvm/PreferRegisterOverUnsignedCheck.h
-clang-tools-extra/clang-tidy/llvm/TwineLocalCheck.cpp
-clang-tools-extra/clang-tidy/llvm/TwineLocalCheck.h
-clang-tools-extra/clang-tidy/llvmlibc/CalleeNamespaceCheck.cpp
-clang-tools-extra/clang-tidy/llvmlibc/CalleeNamespaceCheck.h
-clang-tools-extra/clang-tidy/llvmlibc/ImplementationInNamespaceCheck.cpp
-clang-tools-extra/clang-tidy/llvmlibc/ImplementationInNamespaceCheck.h
-clang-tools-extra/clang-tidy/llvmlibc/LLVMLibcTidyModule.cpp
-clang-tools-extra/clang-tidy/llvmlibc/RestrictSystemLibcHeadersCheck.cpp
-clang-tools-extra/clang-tidy/llvmlibc/RestrictSystemLibcHeadersCheck.h
-clang-tools-extra/clang-tidy/misc/DefinitionsInHeadersCheck.cpp
-clang-tools-extra/clang-tidy/misc/DefinitionsInHeadersCheck.h
-clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp
-clang-tools-extra/clang-tidy/misc/MisleadingBidirectional.cpp
-clang-tools-extra/clang-tidy/misc/MisleadingBidirectional.h
-clang-tools-extra/clang-tidy/misc/MisleadingIdentifier.cpp
-clang-tools-extra/clang-tidy/misc/MisleadingIdentifier.h
-clang-tools-extra/clang-tidy/misc/MisplacedConstCheck.cpp
-clang-tools-extra/clang-tidy/misc/MisplacedConstCheck.h
-clang-tools-extra/clang-tidy/misc/NewDeleteOverloadsCheck.cpp
-clang-tools-extra/clang-tidy/misc/NewDeleteOverloadsCheck.h
-clang-tools-extra/clang-tidy/misc/NonCopyableObjects.h
-clang-tools-extra/clang-tidy/misc/NonPrivateMemberVariablesInClassesCheck.cpp
-clang-tools-extra/clang-tidy/misc/NonPrivateMemberVariablesInClassesCheck.h
-clang-tools-extra/clang-tidy/misc/NoRecursionCheck.cpp
-clang-tools-extra/clang-tidy/misc/NoRecursionCheck.h
-clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.h
-clang-tools-extra/clang-tidy/misc/StaticAssertCheck.cpp
-clang-tools-extra/clang-tidy/misc/StaticAssertCheck.h
-clang-tools-extra/clang-tidy/misc/ThrowByValueCatchByReferenceCheck.cpp
-clang-tools-extra/clang-tidy/misc/UnconventionalAssignOperatorCheck.cpp
-clang-tools-extra/clang-tidy/misc/UnconventionalAssignOperatorCheck.h
-clang-tools-extra/clang-tidy/misc/UniqueptrResetReleaseCheck.cpp
-clang-tools-extra/clang-tidy/misc/UniqueptrResetReleaseCheck.h
-clang-tools-extra/clang-tidy/misc/UnusedAliasDeclsCheck.cpp
-clang-tools-extra/clang-tidy/misc/UnusedAliasDeclsCheck.h
-clang-tools-extra/clang-tidy/misc/UnusedParametersCheck.cpp
-clang-tools-extra/clang-tidy/misc/UnusedParametersCheck.h
-clang-tools-extra/clang-tidy/misc/UnusedUsingDeclsCheck.cpp
-clang-tools-extra/clang-tidy/misc/UnusedUsingDeclsCheck.h
-clang-tools-extra/clang-tidy/modernize/AvoidBindCheck.cpp
-clang-tools-extra/clang-tidy/modernize/AvoidBindCheck.h
-clang-tools-extra/clang-tidy/modernize/AvoidCArraysCheck.cpp
-clang-tools-extra/clang-tidy/modernize/AvoidCArraysCheck.h
-clang-tools-extra/clang-tidy/modernize/ConcatNestedNamespacesCheck.cpp
-clang-tools-extra/clang-tidy/modernize/ConcatNestedNamespacesCheck.h
-clang-tools-extra/clang-tidy/modernize/DeprecatedHeadersCheck.h
-clang-tools-extra/clang-tidy/modernize/DeprecatedIosBaseAliasesCheck.cpp
-clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.h
-clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.h
-clang-tools-extra/clang-tidy/modernize/MakeSharedCheck.cpp
-clang-tools-extra/clang-tidy/modernize/MakeSharedCheck.h
-clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.h
-clang-tools-extra/clang-tidy/modernize/MakeUniqueCheck.cpp
-clang-tools-extra/clang-tidy/modernize/MakeUniqueCheck.h
-clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp
-clang-tools-extra/clang-tidy/modernize/PassByValueCheck.h
-clang-tools-extra/clang-tidy/modernize/RawStringLiteralCheck.cpp
-clang-tools-extra/clang-tidy/modernize/RawStringLiteralCheck.h
-clang-tools-extra/clang-tidy/modernize/RedundantVoidArgCheck.cpp
-clang-tools-extra/clang-tidy/modernize/RedundantVoidArgCheck.h
-clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.h
-clang-tools-extra/clang-tidy/modernize/ReplaceDisallowCopyAndAssignMacroCheck.cpp
-clang-tools-extra/clang-tidy/modernize/ReplaceDisallowCopyAndAssignMacroCheck.h
-clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp
-clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.h
-clang-tools-extra/clang-tidy/modernize/ReturnBracedInitListCheck.cpp
-clang-tools-extra/clang-tidy/modernize/ReturnBracedInitListCheck.h
-clang-tools-extra/clang-tidy/modernize/ShrinkToFitCheck.cpp
-clang-tools-extra/clang-tidy/modernize/ShrinkToFitCheck.h
-clang-tools-extra/clang-tidy/modernize/UnaryStaticAssertCheck.cpp
-clang-tools-extra/clang-tidy/modernize/UnaryStaticAssertCheck.h
-clang-tools-extra/clang-tidy/modernize/UseAutoCheck.h
-clang-tools-extra/clang-tidy/modernize/UseBoolLiteralsCheck.cpp
-clang-tools-extra/clang-tidy/modernize/UseBoolLiteralsCheck.h
-clang-tools-extra/clang-tidy/modernize/UseDefaultMemberInitCheck.h
-clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.h
-clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp
-clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.cpp
-clang-tools-extra/clang-tidy/modernize/UseNodiscardCheck.h
-clang-tools-extra/clang-tidy/modernize/UseNoexceptCheck.h
-clang-tools-extra/clang-tidy/modernize/UseNullptrCheck.h
-clang-tools-extra/clang-tidy/modernize/UseOverrideCheck.cpp
-clang-tools-extra/clang-tidy/modernize/UseOverrideCheck.h
-clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.h
-clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.cpp
-clang-tools-extra/clang-tidy/modernize/UseUsingCheck.cpp
-clang-tools-extra/clang-tidy/modernize/UseUsingCheck.h
-clang-tools-extra/clang-tidy/mpi/BufferDerefCheck.cpp
-clang-tools-extra/clang-tidy/mpi/BufferDerefCheck.h
-clang-tools-extra/clang-tidy/mpi/MPITidyModule.cpp
-clang-tools-extra/clang-tidy/mpi/TypeMismatchCheck.cpp
-clang-tools-extra/clang-tidy/mpi/TypeMismatchCheck.h
-clang-tools-extra/clang-tidy/objc/AssertEquals.cpp
-clang-tools-extra/clang-tidy/objc/AssertEquals.h
-clang-tools-extra/clang-tidy/objc/DeallocInCategoryCheck.cpp
-clang-tools-extra/clang-tidy/objc/DeallocInCategoryCheck.h
-clang-tools-extra/clang-tidy/objc/ForbiddenSubclassingCheck.h
-clang-tools-extra/clang-tidy/objc/MissingHashCheck.cpp
-clang-tools-extra/clang-tidy/objc/MissingHashCheck.h
-clang-tools-extra/clang-tidy/objc/NSInvocationArgumentLifetimeCheck.cpp
-clang-tools-extra/clang-tidy/objc/NSInvocationArgumentLifetimeCheck.h
-clang-tools-extra/clang-tidy/objc/PropertyDeclarationCheck.h
-clang-tools-extra/clang-tidy/objc/SuperSelfCheck.cpp
-clang-tools-extra/clang-tidy/objc/SuperSelfCheck.h
-clang-tools-extra/clang-tidy/openmp/ExceptionEscapeCheck.cpp
-clang-tools-extra/clang-tidy/openmp/ExceptionEscapeCheck.h
-clang-tools-extra/clang-tidy/openmp/OpenMPTidyModule.cpp
-clang-tools-extra/clang-tidy/openmp/UseDefaultNoneCheck.cpp
-clang-tools-extra/clang-tidy/openmp/UseDefaultNoneCheck.h
-clang-tools-extra/clang-tidy/performance/FasterStringFindCheck.cpp
-clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.cpp
-clang-tools-extra/clang-tidy/performance/InefficientAlgorithmCheck.cpp
-clang-tools-extra/clang-tidy/performance/InefficientAlgorithmCheck.h
-clang-tools-extra/clang-tidy/performance/InefficientStringConcatenationCheck.cpp
-clang-tools-extra/clang-tidy/performance/InefficientStringConcatenationCheck.h
-clang-tools-extra/clang-tidy/performance/MoveConstArgCheck.cpp
-clang-tools-extra/clang-tidy/performance/MoveConstArgCheck.h
-clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.cpp
-clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.h
-clang-tools-extra/clang-tidy/performance/NoAutomaticMoveCheck.cpp
-clang-tools-extra/clang-tidy/performance/NoAutomaticMoveCheck.h
-clang-tools-extra/clang-tidy/performance/NoexceptMoveConstructorCheck.cpp
-clang-tools-extra/clang-tidy/performance/NoexceptMoveConstructorCheck.h
-clang-tools-extra/clang-tidy/performance/NoIntToPtrCheck.cpp
-clang-tools-extra/clang-tidy/performance/NoIntToPtrCheck.h
-clang-tools-extra/clang-tidy/performance/PerformanceTidyModule.cpp
-clang-tools-extra/clang-tidy/performance/TriviallyDestructibleCheck.cpp
-clang-tools-extra/clang-tidy/performance/TriviallyDestructibleCheck.h
-clang-tools-extra/clang-tidy/performance/TypePromotionInMathFnCheck.cpp
-clang-tools-extra/clang-tidy/performance/TypePromotionInMathFnCheck.h
-clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp
-clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp
-clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.h
-clang-tools-extra/clang-tidy/plugin/ClangTidyPlugin.cpp
-clang-tools-extra/clang-tidy/portability/PortabilityTidyModule.cpp
-clang-tools-extra/clang-tidy/portability/RestrictSystemIncludesCheck.cpp
-clang-tools-extra/clang-tidy/portability/SIMDIntrinsicsCheck.cpp
-clang-tools-extra/clang-tidy/readability/AvoidConstParamsInDecls.h
-clang-tools-extra/clang-tidy/readability/BracesAroundStatementsCheck.cpp
-clang-tools-extra/clang-tidy/readability/BracesAroundStatementsCheck.h
-clang-tools-extra/clang-tidy/readability/ConstReturnTypeCheck.cpp
-clang-tools-extra/clang-tidy/readability/ContainerContainsCheck.cpp
-clang-tools-extra/clang-tidy/readability/ContainerContainsCheck.h
-clang-tools-extra/clang-tidy/readability/ContainerDataPointerCheck.cpp
-clang-tools-extra/clang-tidy/readability/ContainerDataPointerCheck.h
-clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.h
-clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp
-clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.h
-clang-tools-extra/clang-tidy/readability/DeleteNullPointerCheck.cpp
-clang-tools-extra/clang-tidy/readability/DeleteNullPointerCheck.h
-clang-tools-extra/clang-tidy/readability/DuplicateIncludeCheck.cpp
-clang-tools-extra/clang-tidy/readability/DuplicateIncludeCheck.h
-clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.h
-clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp
-clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.h
-clang-tools-extra/clang-tidy/readability/FunctionSizeCheck.cpp
-clang-tools-extra/clang-tidy/readability/FunctionSizeCheck.h
-clang-tools-extra/clang-tidy/readability/IdentifierLengthCheck.cpp
-clang-tools-extra/clang-tidy/readability/IdentifierLengthCheck.h
-clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
-clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.h
-clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
-clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.h
-clang-tools-extra/clang-tidy/readability/InconsistentDeclarationParameterNameCheck.h
-clang-tools-extra/clang-tidy/readability/IsolateDeclarationCheck.cpp
-clang-tools-extra/clang-tidy/readability/IsolateDeclarationCheck.h
-clang-tools-extra/clang-tidy/readability/MagicNumbersCheck.cpp
-clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp
-clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.h
-clang-tools-extra/clang-tidy/readability/MisleadingIndentationCheck.cpp
-clang-tools-extra/clang-tidy/readability/MisleadingIndentationCheck.h
-clang-tools-extra/clang-tidy/readability/MisplacedArrayIndexCheck.cpp
-clang-tools-extra/clang-tidy/readability/MisplacedArrayIndexCheck.h
-clang-tools-extra/clang-tidy/readability/NamedParameterCheck.cpp
-clang-tools-extra/clang-tidy/readability/NamedParameterCheck.h
-clang-tools-extra/clang-tidy/readability/NamespaceCommentCheck.h
-clang-tools-extra/clang-tidy/readability/NonConstParameterCheck.cpp
-clang-tools-extra/clang-tidy/readability/NonConstParameterCheck.h
-clang-tools-extra/clang-tidy/readability/QualifiedAutoCheck.h
-clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp
-clang-tools-extra/clang-tidy/readability/RedundantAccessSpecifiersCheck.cpp
-clang-tools-extra/clang-tidy/readability/RedundantAccessSpecifiersCheck.h
-clang-tools-extra/clang-tidy/readability/RedundantControlFlowCheck.cpp
-clang-tools-extra/clang-tidy/readability/RedundantControlFlowCheck.h
-clang-tools-extra/clang-tidy/readability/RedundantDeclarationCheck.cpp
-clang-tools-extra/clang-tidy/readability/RedundantDeclarationCheck.h
-clang-tools-extra/clang-tidy/readability/RedundantMemberInitCheck.cpp
-clang-tools-extra/clang-tidy/readability/RedundantMemberInitCheck.h
-clang-tools-extra/clang-tidy/readability/RedundantPreprocessorCheck.cpp
-clang-tools-extra/clang-tidy/readability/RedundantPreprocessorCheck.h
-clang-tools-extra/clang-tidy/readability/RedundantSmartptrGetCheck.cpp
-clang-tools-extra/clang-tidy/readability/RedundantSmartptrGetCheck.h
-clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.h
-clang-tools-extra/clang-tidy/readability/RedundantStringInitCheck.cpp
-clang-tools-extra/clang-tidy/readability/RedundantStringInitCheck.h
-clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.cpp
-clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.h
-clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprMatchers.h
-clang-tools-extra/clang-tidy/readability/SimplifySubscriptExprCheck.cpp
-clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp
-clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.h
-clang-tools-extra/clang-tidy/readability/StaticDefinitionInAnonymousNamespaceCheck.cpp
-clang-tools-extra/clang-tidy/readability/StaticDefinitionInAnonymousNamespaceCheck.h
-clang-tools-extra/clang-tidy/readability/StringCompareCheck.cpp
-clang-tools-extra/clang-tidy/readability/StringCompareCheck.h
-clang-tools-extra/clang-tidy/readability/SuspiciousCallArgumentCheck.cpp
-clang-tools-extra/clang-tidy/readability/SuspiciousCallArgumentCheck.h
-clang-tools-extra/clang-tidy/readability/UniqueptrDeleteReleaseCheck.cpp
-clang-tools-extra/clang-tidy/readability/UniqueptrDeleteReleaseCheck.h
-clang-tools-extra/clang-tidy/readability/UppercaseLiteralSuffixCheck.cpp
-clang-tools-extra/clang-tidy/readability/UppercaseLiteralSuffixCheck.h
-clang-tools-extra/clang-tidy/readability/UseAnyOfAllOfCheck.cpp
-clang-tools-extra/clang-tidy/readability/UseAnyOfAllOfCheck.h
-clang-tools-extra/clang-tidy/tool/ClangTidyMain.h
-clang-tools-extra/clang-tidy/tool/ClangTidyToolMain.cpp
-clang-tools-extra/clang-tidy/utils/Aliasing.cpp
-clang-tools-extra/clang-tidy/utils/Aliasing.h
-clang-tools-extra/clang-tidy/utils/ASTUtils.cpp
-clang-tools-extra/clang-tidy/utils/ASTUtils.h
-clang-tools-extra/clang-tidy/utils/DeclRefExprUtils.cpp
-clang-tools-extra/clang-tidy/utils/DeclRefExprUtils.h
-clang-tools-extra/clang-tidy/utils/ExceptionAnalyzer.h
-clang-tools-extra/clang-tidy/utils/ExprSequence.cpp
-clang-tools-extra/clang-tidy/utils/ExprSequence.h
-clang-tools-extra/clang-tidy/utils/FileExtensionsUtils.cpp
-clang-tools-extra/clang-tidy/utils/FileExtensionsUtils.h
-clang-tools-extra/clang-tidy/utils/FixItHintUtils.cpp
-clang-tools-extra/clang-tidy/utils/FixItHintUtils.h
-clang-tools-extra/clang-tidy/utils/HeaderGuard.cpp
-clang-tools-extra/clang-tidy/utils/HeaderGuard.h
-clang-tools-extra/clang-tidy/utils/IncludeInserter.cpp
-clang-tools-extra/clang-tidy/utils/IncludeInserter.h
-clang-tools-extra/clang-tidy/utils/IncludeSorter.h
-clang-tools-extra/clang-tidy/utils/LexerUtils.h
-clang-tools-extra/clang-tidy/utils/Matchers.h
-clang-tools-extra/clang-tidy/utils/NamespaceAliaser.cpp
-clang-tools-extra/clang-tidy/utils/NamespaceAliaser.h
-clang-tools-extra/clang-tidy/utils/OptionsUtils.cpp
-clang-tools-extra/clang-tidy/utils/OptionsUtils.h
-clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
-clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.h
-clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.cpp
-clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.h
-clang-tools-extra/clang-tidy/utils/TypeTraits.h
-clang-tools-extra/clang-tidy/utils/UsingInserter.cpp
-clang-tools-extra/clang-tidy/utils/UsingInserter.h
-clang-tools-extra/clang-tidy/zircon/TemporaryObjectsCheck.cpp
-clang-tools-extra/clang-tidy/zircon/TemporaryObjectsCheck.h
-clang-tools-extra/clang-tidy/zircon/ZirconTidyModule.cpp
-clang-tools-extra/clangd/AST.cpp
-clang-tools-extra/clangd/AST.h
-clang-tools-extra/clangd/ASTSignals.cpp
-clang-tools-extra/clangd/ASTSignals.h
-clang-tools-extra/clangd/ClangdLSPServer.cpp
-clang-tools-extra/clangd/ClangdLSPServer.h
-clang-tools-extra/clangd/ClangdServer.h
-clang-tools-extra/clangd/CodeComplete.cpp
-clang-tools-extra/clangd/CodeComplete.h
-clang-tools-extra/clangd/CodeCompletionStrings.h
-clang-tools-extra/clangd/CollectMacros.cpp
-clang-tools-extra/clangd/CollectMacros.h
-clang-tools-extra/clangd/CompileCommands.cpp
-clang-tools-extra/clangd/CompileCommands.h
-clang-tools-extra/clangd/Compiler.cpp
-clang-tools-extra/clangd/Compiler.h
-clang-tools-extra/clangd/Config.cpp
-clang-tools-extra/clangd/Config.h
-clang-tools-extra/clangd/ConfigCompile.cpp
-clang-tools-extra/clangd/ConfigFragment.h
-clang-tools-extra/clangd/ConfigProvider.cpp
-clang-tools-extra/clangd/ConfigProvider.h
-clang-tools-extra/clangd/Diagnostics.cpp
-clang-tools-extra/clangd/Diagnostics.h
-clang-tools-extra/clangd/DraftStore.cpp
-clang-tools-extra/clangd/DraftStore.h
-clang-tools-extra/clangd/DumpAST.cpp
-clang-tools-extra/clangd/DumpAST.h
-clang-tools-extra/clangd/ExpectedTypes.cpp
-clang-tools-extra/clangd/ExpectedTypes.h
-clang-tools-extra/clangd/Feature.cpp
-clang-tools-extra/clangd/Feature.h
-clang-tools-extra/clangd/FeatureModule.cpp
-clang-tools-extra/clangd/FeatureModule.h
-clang-tools-extra/clangd/FileDistance.cpp
-clang-tools-extra/clangd/FileDistance.h
-clang-tools-extra/clangd/FindSymbols.cpp
-clang-tools-extra/clangd/FindSymbols.h
-clang-tools-extra/clangd/FindTarget.cpp
-clang-tools-extra/clangd/FindTarget.h
-clang-tools-extra/clangd/FS.h
-clang-tools-extra/clangd/FuzzyMatch.cpp
-clang-tools-extra/clangd/FuzzyMatch.h
-clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
-clang-tools-extra/clangd/GlobalCompilationDatabase.h
-clang-tools-extra/clangd/Headers.cpp
-clang-tools-extra/clangd/Headers.h
-clang-tools-extra/clangd/HeaderSourceSwitch.cpp
-clang-tools-extra/clangd/HeaderSourceSwitch.h
-clang-tools-extra/clangd/HeuristicResolver.cpp
-clang-tools-extra/clangd/HeuristicResolver.h
-clang-tools-extra/clangd/Hover.cpp
-clang-tools-extra/clangd/Hover.h
-clang-tools-extra/clangd/IncludeCleaner.cpp
-clang-tools-extra/clangd/IncludeCleaner.h
-clang-tools-extra/clangd/IncludeFixer.cpp
-clang-tools-extra/clangd/InlayHints.h
-clang-tools-extra/clangd/LSPBinder.h
-clang-tools-extra/clangd/ParsedAST.cpp
-clang-tools-extra/clangd/ParsedAST.h
-clang-tools-extra/clangd/PathMapping.h
-clang-tools-extra/clangd/Preamble.cpp
-clang-tools-extra/clangd/Preamble.h
-clang-tools-extra/clangd/Protocol.cpp
-clang-tools-extra/clangd/Protocol.h
-clang-tools-extra/clangd/Quality.cpp
-clang-tools-extra/clangd/RIFF.cpp
-clang-tools-extra/clangd/RIFF.h
-clang-tools-extra/clangd/Selection.h
-clang-tools-extra/clangd/SemanticHighlighting.h
-clang-tools-extra/clangd/SemanticSelection.cpp
-clang-tools-extra/clangd/SemanticSelection.h
-clang-tools-extra/clangd/SourceCode.cpp
-clang-tools-extra/clangd/SourceCode.h
-clang-tools-extra/clangd/TidyProvider.cpp
-clang-tools-extra/clangd/TidyProvider.h
-clang-tools-extra/clangd/Transport.h
-clang-tools-extra/clangd/TUScheduler.cpp
-clang-tools-extra/clangd/TUScheduler.h
-clang-tools-extra/clangd/URI.h
-clang-tools-extra/clangd/XRefs.h
-clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp
-clang-tools-extra/clangd/fuzzer/clangd-fuzzer.cpp
-clang-tools-extra/clangd/fuzzer/FuzzerClangdMain.cpp
-clang-tools-extra/clangd/index/Background.cpp
-clang-tools-extra/clangd/index/Background.h
-clang-tools-extra/clangd/index/BackgroundIndexLoader.cpp
-clang-tools-extra/clangd/index/BackgroundIndexLoader.h
-clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp
-clang-tools-extra/clangd/index/BackgroundQueue.cpp
-clang-tools-extra/clangd/index/BackgroundRebuild.cpp
-clang-tools-extra/clangd/index/BackgroundRebuild.h
-clang-tools-extra/clangd/index/CanonicalIncludes.cpp
-clang-tools-extra/clangd/index/CanonicalIncludes.h
-clang-tools-extra/clangd/index/FileIndex.cpp
-clang-tools-extra/clangd/index/FileIndex.h
-clang-tools-extra/clangd/index/Index.cpp
-clang-tools-extra/clangd/index/Index.h
-clang-tools-extra/clangd/index/IndexAction.cpp
-clang-tools-extra/clangd/index/IndexAction.h
-clang-tools-extra/clangd/index/MemIndex.h
-clang-tools-extra/clangd/index/Merge.cpp
-clang-tools-extra/clangd/index/Merge.h
-clang-tools-extra/clangd/index/ProjectAware.cpp
-clang-tools-extra/clangd/index/ProjectAware.h
-clang-tools-extra/clangd/index/Ref.cpp
-clang-tools-extra/clangd/index/Ref.h
-clang-tools-extra/clangd/index/Relation.cpp
-clang-tools-extra/clangd/index/Relation.h
-clang-tools-extra/clangd/index/Serialization.cpp
-clang-tools-extra/clangd/index/Serialization.h
-clang-tools-extra/clangd/index/Symbol.cpp
-clang-tools-extra/clangd/index/Symbol.h
-clang-tools-extra/clangd/index/SymbolCollector.cpp
-clang-tools-extra/clangd/index/SymbolID.cpp
-clang-tools-extra/clangd/index/SymbolLocation.cpp
-clang-tools-extra/clangd/index/SymbolLocation.h
-clang-tools-extra/clangd/index/SymbolOrigin.cpp
-clang-tools-extra/clangd/index/SymbolOrigin.h
-clang-tools-extra/clangd/index/YAMLSerialization.cpp
-clang-tools-extra/clangd/index/dex/Iterator.cpp
-clang-tools-extra/clangd/index/dex/Iterator.h
-clang-tools-extra/clangd/index/dex/PostingList.cpp
-clang-tools-extra/clangd/index/dex/PostingList.h
-clang-tools-extra/clangd/index/dex/Token.h
-clang-tools-extra/clangd/index/dex/Trigram.cpp
-clang-tools-extra/clangd/index/dex/Trigram.h
-clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
-clang-tools-extra/clangd/index/remote/Client.cpp
-clang-tools-extra/clangd/index/remote/Client.h
-clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
-clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
-clang-tools-extra/clangd/index/remote/monitor/Monitor.cpp
-clang-tools-extra/clangd/index/remote/server/Server.cpp
-clang-tools-extra/clangd/index/remote/unimplemented/UnimplementedClient.cpp
-clang-tools-extra/clangd/indexer/IndexerMain.cpp
-clang-tools-extra/clangd/refactor/InsertionPoint.cpp
-clang-tools-extra/clangd/refactor/InsertionPoint.h
-clang-tools-extra/clangd/refactor/Rename.h
-clang-tools-extra/clangd/refactor/Tweak.cpp
-clang-tools-extra/clangd/refactor/Tweak.h
-clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp
-clang-tools-extra/clangd/refactor/tweaks/AnnotateHighlightings.cpp
-clang-tools-extra/clangd/refactor/tweaks/DefineInline.cpp
-clang-tools-extra/clangd/refactor/tweaks/DefineOutline.cpp
-clang-tools-extra/clangd/refactor/tweaks/DumpAST.cpp
-clang-tools-extra/clangd/refactor/tweaks/ExpandMacro.cpp
-clang-tools-extra/clangd/refactor/tweaks/ExtractFunction.cpp
-clang-tools-extra/clangd/refactor/tweaks/ObjCLocalizeStringLiteral.cpp
-clang-tools-extra/clangd/refactor/tweaks/RemoveUsingNamespace.cpp
-clang-tools-extra/clangd/refactor/tweaks/SwapIfBranches.cpp
-clang-tools-extra/clangd/support/Cancellation.cpp
-clang-tools-extra/clangd/support/Cancellation.h
-clang-tools-extra/clangd/support/Context.cpp
-clang-tools-extra/clangd/support/Context.h
-clang-tools-extra/clangd/support/FileCache.cpp
-clang-tools-extra/clangd/support/FileCache.h
-clang-tools-extra/clangd/support/Function.h
-clang-tools-extra/clangd/support/Logger.cpp
-clang-tools-extra/clangd/support/Markup.cpp
-clang-tools-extra/clangd/support/Markup.h
-clang-tools-extra/clangd/support/MemoryTree.cpp
-clang-tools-extra/clangd/support/MemoryTree.h
-clang-tools-extra/clangd/support/Path.cpp
-clang-tools-extra/clangd/support/Path.h
-clang-tools-extra/clangd/support/Shutdown.cpp
-clang-tools-extra/clangd/support/Shutdown.h
-clang-tools-extra/clangd/support/ThreadCrashReporter.cpp
-clang-tools-extra/clangd/support/ThreadCrashReporter.h
-clang-tools-extra/clangd/support/Threading.cpp
-clang-tools-extra/clangd/support/Threading.h
-clang-tools-extra/clangd/support/ThreadsafeFS.cpp
-clang-tools-extra/clangd/support/ThreadsafeFS.h
-clang-tools-extra/clangd/support/Trace.cpp
-clang-tools-extra/clangd/support/Trace.h
-clang-tools-extra/clangd/tool/Check.cpp
-clang-tools-extra/clangd/tool/ClangdMain.cpp
-clang-tools-extra/clangd/unittests/Annotations.cpp
-clang-tools-extra/clangd/unittests/Annotations.h
-clang-tools-extra/clangd/unittests/ASTSignalsTests.cpp
-clang-tools-extra/clangd/unittests/ASTTests.cpp
-clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
-clang-tools-extra/clangd/unittests/CallHierarchyTests.cpp
-clang-tools-extra/clangd/unittests/CanonicalIncludesTests.cpp
-clang-tools-extra/clangd/unittests/ClangdLSPServerTests.cpp
-clang-tools-extra/clangd/unittests/ClangdTests.cpp
-clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
-clang-tools-extra/clangd/unittests/CodeCompletionStringsTests.cpp
-clang-tools-extra/clangd/unittests/CollectMacrosTests.cpp
-clang-tools-extra/clangd/unittests/CompilerTests.cpp
-clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp
-clang-tools-extra/clangd/unittests/ConfigProviderTests.cpp
-clang-tools-extra/clangd/unittests/ConfigTesting.h
-clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp
-clang-tools-extra/clangd/unittests/DecisionForestTests.cpp
-clang-tools-extra/clangd/unittests/DexTests.cpp
-clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
-clang-tools-extra/clangd/unittests/DraftStoreTests.cpp
-clang-tools-extra/clangd/unittests/DumpASTTests.cpp
-clang-tools-extra/clangd/unittests/ExpectedTypeTest.cpp
-clang-tools-extra/clangd/unittests/FeatureModulesTests.cpp
-clang-tools-extra/clangd/unittests/FileDistanceTests.cpp
-clang-tools-extra/clangd/unittests/FileIndexTests.cpp
-clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp
-clang-tools-extra/clangd/unittests/FindTargetTests.cpp
-clang-tools-extra/clangd/unittests/FSTests.cpp
-clang-tools-extra/clangd/unittests/FuzzyMatchTests.cpp
-clang-tools-extra/clangd/unittests/GlobalCompilationDatabaseTests.cpp
-clang-tools-extra/clangd/unittests/HeadersTests.cpp
-clang-tools-extra/clangd/unittests/HoverTests.cpp
-clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp
-clang-tools-extra/clangd/unittests/IndexActionTests.cpp
-clang-tools-extra/clangd/unittests/InlayHintTests.cpp
-clang-tools-extra/clangd/unittests/InsertionPointTests.cpp
-clang-tools-extra/clangd/unittests/LoggerTests.cpp
-clang-tools-extra/clangd/unittests/LSPBinderTests.cpp
-clang-tools-extra/clangd/unittests/LSPClient.cpp
-clang-tools-extra/clangd/unittests/ModulesTests.cpp
-clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
-clang-tools-extra/clangd/unittests/PreambleTests.cpp
-clang-tools-extra/clangd/unittests/PrintASTTests.cpp
-clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
-clang-tools-extra/clangd/unittests/QualityTests.cpp
-clang-tools-extra/clangd/unittests/RIFFTests.cpp
-clang-tools-extra/clangd/unittests/SelectionTests.cpp
-clang-tools-extra/clangd/unittests/SemanticSelectionTests.cpp
-clang-tools-extra/clangd/unittests/SerializationTests.cpp
-clang-tools-extra/clangd/unittests/SourceCodeTests.cpp
-clang-tools-extra/clangd/unittests/SymbolInfoTests.cpp
-clang-tools-extra/clangd/unittests/SyncAPI.cpp
-clang-tools-extra/clangd/unittests/SyncAPI.h
-clang-tools-extra/clangd/unittests/TestFS.cpp
-clang-tools-extra/clangd/unittests/TestFS.h
-clang-tools-extra/clangd/unittests/TestIndex.cpp
-clang-tools-extra/clangd/unittests/TestIndex.h
-clang-tools-extra/clangd/unittests/TestTU.cpp
-clang-tools-extra/clangd/unittests/TestTU.h
-clang-tools-extra/clangd/unittests/TestWorkspace.cpp
-clang-tools-extra/clangd/unittests/TestWorkspace.h
-clang-tools-extra/clangd/unittests/ThreadCrashReporterTests.cpp
-clang-tools-extra/clangd/unittests/TidyProviderTests.cpp
-clang-tools-extra/clangd/unittests/TypeHierarchyTests.cpp
-clang-tools-extra/clangd/unittests/URITests.cpp
-clang-tools-extra/clangd/unittests/decision_forest_model/CategoricalFeature.h
-clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
-clang-tools-extra/clangd/unittests/support/CancellationTests.cpp
-clang-tools-extra/clangd/unittests/support/ContextTests.cpp
-clang-tools-extra/clangd/unittests/support/FileCacheTests.cpp
-clang-tools-extra/clangd/unittests/support/FunctionTests.cpp
-clang-tools-extra/clangd/unittests/support/MarkupTests.cpp
-clang-tools-extra/clangd/unittests/support/MemoryTreeTests.cpp
-clang-tools-extra/clangd/unittests/support/PathTests.cpp
-clang-tools-extra/clangd/unittests/support/TestTracer.cpp
-clang-tools-extra/clangd/unittests/support/TestTracer.h
-clang-tools-extra/clangd/unittests/support/ThreadingTests.cpp
-clang-tools-extra/clangd/unittests/support/TraceTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/AddUsingTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/AnnotateHighlightingsTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/DefineOutlineTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/DumpASTTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/DumpRecordLayoutTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/DumpSymbolTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/ExpandDeducedTypeTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/ExpandMacroTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/ExtractFunctionTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/ExtractVariableTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/ObjCLocalizeStringLiteralTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/PopulateSwitchTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/RawStringLiteralTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/RemoveUsingNamespaceTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/ShowSelectionTreeTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/SwapIfBranchesTests.cpp
-clang-tools-extra/clangd/unittests/tweaks/TweakTesting.cpp
-clang-tools-extra/clangd/unittests/tweaks/TweakTesting.h
-clang-tools-extra/clangd/unittests/tweaks/TweakTests.cpp
-clang-tools-extra/clangd/unittests/xpc/ConversionTests.cpp
-clang-tools-extra/clangd/xpc/Conversion.cpp
-clang-tools-extra/clangd/xpc/Conversion.h
-clang-tools-extra/clangd/xpc/XPCTransport.cpp
-clang-tools-extra/clangd/xpc/framework/ClangdXPC.cpp
-clang-tools-extra/clangd/xpc/test-client/ClangdXPCTestClient.cpp
-clang-tools-extra/modularize/Modularize.h
-clang-tools-extra/pp-trace/PPTrace.cpp
-clang-tools-extra/tool-template/ToolTemplate.cpp
-clang-tools-extra/unittests/clang-apply-replacements/ApplyReplacementsTest.cpp
-clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp
-clang-tools-extra/unittests/clang-doc/ClangDocTest.cpp
-clang-tools-extra/unittests/clang-doc/ClangDocTest.h
-clang-tools-extra/unittests/clang-doc/GeneratorTest.cpp
-clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
-clang-tools-extra/unittests/clang-doc/MDGeneratorTest.cpp
-clang-tools-extra/unittests/clang-doc/MergeTest.cpp
-clang-tools-extra/unittests/clang-doc/SerializeTest.cpp
-clang-tools-extra/unittests/clang-doc/YAMLGeneratorTest.cpp
-clang-tools-extra/unittests/clang-tidy/AddConstTest.cpp
-clang-tools-extra/unittests/clang-tidy/ClangTidyDiagnosticConsumerTest.cpp
-clang-tools-extra/unittests/clang-tidy/ClangTidyTest.h
-clang-tools-extra/unittests/clang-tidy/DeclRefExprUtilsTest.cpp
-clang-tools-extra/unittests/clang-tidy/GlobListTest.cpp
-clang-tools-extra/unittests/clang-tidy/OptionsProviderTest.cpp
-clang-tools-extra/unittests/clang-tidy/OverlappingReplacementsTest.cpp
-clang-tools-extra/unittests/clang-tidy/ReadabilityModuleTest.cpp
-clang-tools-extra/unittests/clang-tidy/TransformerClangTidyCheckTest.cpp
-compiler-rt/include/sanitizer/linux_syscall_hooks.h
-compiler-rt/include/sanitizer/memprof_interface.h
-compiler-rt/include/sanitizer/netbsd_syscall_hooks.h
-compiler-rt/include/xray/xray_interface.h
-compiler-rt/include/xray/xray_log_interface.h
-compiler-rt/lib/asan/asan_activation.h
-compiler-rt/lib/asan/asan_lock.h
-compiler-rt/lib/asan/asan_mapping.h
-compiler-rt/lib/asan/asan_mapping_sparc64.h
-compiler-rt/lib/asan/asan_rtl_static.cpp
-compiler-rt/lib/asan/tests/asan_globals_test.cpp
-compiler-rt/lib/builtins/fp_extend.h
-compiler-rt/lib/builtins/fp_lib.h
-compiler-rt/lib/builtins/fp_mode.h
-compiler-rt/lib/builtins/fp_trunc.h
-compiler-rt/lib/builtins/int_endianness.h
-compiler-rt/lib/builtins/int_math.h
-compiler-rt/lib/builtins/int_types.h
-compiler-rt/lib/builtins/int_util.h
-compiler-rt/lib/builtins/unwind-ehabi-helpers.h
-compiler-rt/lib/builtins/ppc/DD.h
-compiler-rt/lib/dfsan/dfsan_allocator.cpp
-compiler-rt/lib/dfsan/dfsan_allocator.h
-compiler-rt/lib/dfsan/dfsan_chained_origin_depot.cpp
-compiler-rt/lib/dfsan/dfsan_chained_origin_depot.h
-compiler-rt/lib/dfsan/dfsan_flags.h
-compiler-rt/lib/dfsan/dfsan_interceptors.cpp
-compiler-rt/lib/dfsan/dfsan_origin.h
-compiler-rt/lib/dfsan/dfsan_platform.h
-compiler-rt/lib/dfsan/dfsan_thread.h
-compiler-rt/lib/fuzzer/FuzzerCommand.h
-compiler-rt/lib/fuzzer/FuzzerExtFunctions.h
-compiler-rt/lib/fuzzer/FuzzerExtFunctionsDlsym.cpp
-compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeak.cpp
-compiler-rt/lib/fuzzer/FuzzerExtraCountersDarwin.cpp
-compiler-rt/lib/fuzzer/FuzzerExtraCountersWindows.cpp
-compiler-rt/lib/fuzzer/FuzzerFork.h
-compiler-rt/lib/fuzzer/FuzzerInterceptors.cpp
-compiler-rt/lib/fuzzer/FuzzerPlatform.h
-compiler-rt/lib/fuzzer/tests/FuzzedDataProviderUnittest.cpp
-compiler-rt/lib/gwp_asan/common.cpp
-compiler-rt/lib/gwp_asan/common.h
-compiler-rt/lib/gwp_asan/crash_handler.cpp
-compiler-rt/lib/gwp_asan/crash_handler.h
-compiler-rt/lib/gwp_asan/definitions.h
-compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp
-compiler-rt/lib/gwp_asan/guarded_pool_allocator.h
-compiler-rt/lib/gwp_asan/mutex.h
-compiler-rt/lib/gwp_asan/options.h
-compiler-rt/lib/gwp_asan/stack_trace_compressor.cpp
-compiler-rt/lib/gwp_asan/stack_trace_compressor.h
-compiler-rt/lib/gwp_asan/utilities.h
-compiler-rt/lib/gwp_asan/optional/backtrace.h
-compiler-rt/lib/gwp_asan/optional/backtrace_fuchsia.cpp
-compiler-rt/lib/gwp_asan/optional/backtrace_linux_libc.cpp
-compiler-rt/lib/gwp_asan/optional/backtrace_sanitizer_common.cpp
-compiler-rt/lib/gwp_asan/optional/options_parser.cpp
-compiler-rt/lib/gwp_asan/optional/options_parser.h
-compiler-rt/lib/gwp_asan/optional/printf.h
-compiler-rt/lib/gwp_asan/optional/segv_handler.h
-compiler-rt/lib/gwp_asan/optional/segv_handler_fuchsia.cpp
-compiler-rt/lib/gwp_asan/optional/segv_handler_posix.cpp
-compiler-rt/lib/gwp_asan/platform_specific/common_fuchsia.cpp
-compiler-rt/lib/gwp_asan/platform_specific/common_posix.cpp
-compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_fuchsia.cpp
-compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_fuchsia.h
-compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp
-compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.h
-compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_tls.h
-compiler-rt/lib/gwp_asan/platform_specific/mutex_fuchsia.cpp
-compiler-rt/lib/gwp_asan/platform_specific/mutex_fuchsia.h
-compiler-rt/lib/gwp_asan/platform_specific/mutex_posix.cpp
-compiler-rt/lib/gwp_asan/platform_specific/mutex_posix.h
-compiler-rt/lib/gwp_asan/platform_specific/utilities_fuchsia.cpp
-compiler-rt/lib/gwp_asan/platform_specific/utilities_posix.cpp
-compiler-rt/lib/gwp_asan/tests/backtrace.cpp
-compiler-rt/lib/gwp_asan/tests/basic.cpp
-compiler-rt/lib/gwp_asan/tests/compression.cpp
-compiler-rt/lib/gwp_asan/tests/crash_handler_api.cpp
-compiler-rt/lib/gwp_asan/tests/driver.cpp
-compiler-rt/lib/gwp_asan/tests/enable_disable.cpp
-compiler-rt/lib/gwp_asan/tests/harness.cpp
-compiler-rt/lib/gwp_asan/tests/harness.h
-compiler-rt/lib/gwp_asan/tests/iterate.cpp
-compiler-rt/lib/gwp_asan/tests/late_init.cpp
-compiler-rt/lib/gwp_asan/tests/mutex_test.cpp
-compiler-rt/lib/gwp_asan/tests/options.cpp
-compiler-rt/lib/gwp_asan/tests/slot_reuse.cpp
-compiler-rt/lib/gwp_asan/tests/thread_contention.cpp
-compiler-rt/lib/gwp_asan/tests/platform_specific/printf_sanitizer_common.cpp
-compiler-rt/lib/hwasan/hwasan_checks.h
-compiler-rt/lib/hwasan/hwasan_dynamic_shadow.h
-compiler-rt/lib/hwasan/hwasan_flags.h
-compiler-rt/lib/hwasan/hwasan_globals.cpp
-compiler-rt/lib/hwasan/hwasan_globals.h
-compiler-rt/lib/hwasan/hwasan_linux.cpp
-compiler-rt/lib/hwasan/hwasan_poisoning.cpp
-compiler-rt/lib/hwasan/hwasan_poisoning.h
-compiler-rt/lib/hwasan/hwasan_preinit.cpp
-compiler-rt/lib/interception/interception_mac.cpp
-compiler-rt/lib/interception/tests/interception_test_main.cpp
-compiler-rt/lib/lsan/lsan.h
-compiler-rt/lib/lsan/lsan_common.cpp
-compiler-rt/lib/lsan/lsan_thread.cpp
-compiler-rt/lib/lsan/lsan_thread.h
-compiler-rt/lib/memprof/memprof_allocator.cpp
-compiler-rt/lib/memprof/memprof_allocator.h
-compiler-rt/lib/memprof/memprof_descriptions.cpp
-compiler-rt/lib/memprof/memprof_descriptions.h
-compiler-rt/lib/memprof/memprof_flags.cpp
-compiler-rt/lib/memprof/memprof_flags.h
-compiler-rt/lib/memprof/memprof_init_version.h
-compiler-rt/lib/memprof/memprof_interceptors.cpp
-compiler-rt/lib/memprof/memprof_interceptors.h
-compiler-rt/lib/memprof/memprof_interceptors_memintrinsics.cpp
-compiler-rt/lib/memprof/memprof_interceptors_memintrinsics.h
-compiler-rt/lib/memprof/memprof_interface_internal.h
-compiler-rt/lib/memprof/memprof_internal.h
-compiler-rt/lib/memprof/memprof_linux.cpp
-compiler-rt/lib/memprof/memprof_malloc_linux.cpp
-compiler-rt/lib/memprof/memprof_mibmap.cpp
-compiler-rt/lib/memprof/memprof_mibmap.h
-compiler-rt/lib/memprof/memprof_posix.cpp
-compiler-rt/lib/memprof/memprof_preinit.cpp
-compiler-rt/lib/memprof/memprof_rawprofile.cpp
-compiler-rt/lib/memprof/memprof_rawprofile.h
-compiler-rt/lib/memprof/memprof_rtl.cpp
-compiler-rt/lib/memprof/memprof_shadow_setup.cpp
-compiler-rt/lib/memprof/memprof_stack.cpp
-compiler-rt/lib/memprof/memprof_stack.h
-compiler-rt/lib/memprof/memprof_stats.cpp
-compiler-rt/lib/memprof/memprof_stats.h
-compiler-rt/lib/memprof/memprof_thread.cpp
-compiler-rt/lib/memprof/memprof_thread.h
-compiler-rt/lib/memprof/tests/driver.cpp
-compiler-rt/lib/memprof/tests/rawprofile.cpp
-compiler-rt/lib/msan/msan_chained_origin_depot.h
-compiler-rt/lib/msan/msan_flags.h
-compiler-rt/lib/msan/msan_poisoning.h
-compiler-rt/lib/msan/msan_report.h
-compiler-rt/lib/orc/adt.h
-compiler-rt/lib/orc/debug.h
-compiler-rt/lib/orc/elfnix_platform.cpp
-compiler-rt/lib/orc/elfnix_platform.h
-compiler-rt/lib/orc/endianness.h
-compiler-rt/lib/orc/error.h
-compiler-rt/lib/orc/executor_address.h
-compiler-rt/lib/orc/extensible_rtti.cpp
-compiler-rt/lib/orc/extensible_rtti.h
-compiler-rt/lib/orc/log_error_to_stderr.cpp
-compiler-rt/lib/orc/macho_ehframe_registration.cpp
-compiler-rt/lib/orc/macho_platform.cpp
-compiler-rt/lib/orc/macho_platform.h
-compiler-rt/lib/orc/run_program_wrapper.cpp
-compiler-rt/lib/orc/simple_packed_serialization.h
-compiler-rt/lib/orc/wrapper_function_utils.h
-compiler-rt/lib/orc/unittests/adt_test.cpp
-compiler-rt/lib/orc/unittests/c_api_test.cpp
-compiler-rt/lib/orc/unittests/endian_test.cpp
-compiler-rt/lib/orc/unittests/error_test.cpp
-compiler-rt/lib/orc/unittests/executor_address_test.cpp
-compiler-rt/lib/orc/unittests/extensible_rtti_test.cpp
-compiler-rt/lib/orc/unittests/orc_unit_test_main.cpp
-compiler-rt/lib/orc/unittests/simple_packed_serialization_test.cpp
-compiler-rt/lib/orc/unittests/wrapper_function_utils_test.cpp
-compiler-rt/lib/safestack/safestack_util.h
-compiler-rt/lib/sanitizer_common/sancov_flags.h
-compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h
-compiler-rt/lib/sanitizer_common/sanitizer_allocator_report.h
-compiler-rt/lib/sanitizer_common/sanitizer_chained_origin_depot.cpp
-compiler-rt/lib/sanitizer_common/sanitizer_chained_origin_depot.h
-compiler-rt/lib/sanitizer_common/sanitizer_dense_map.h
-compiler-rt/lib/sanitizer_common/sanitizer_dense_map_info.h
-compiler-rt/lib/sanitizer_common/sanitizer_errno.h
-compiler-rt/lib/sanitizer_common/sanitizer_errno_codes.h
-compiler-rt/lib/sanitizer_common/sanitizer_flat_map.h
-compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
-compiler-rt/lib/sanitizer_common/sanitizer_leb128.h
-compiler-rt/lib/sanitizer_common/sanitizer_local_address_space_view.h
-compiler-rt/lib/sanitizer_common/sanitizer_lzw.h
-compiler-rt/lib/sanitizer_common/sanitizer_placement_new.h
-compiler-rt/lib/sanitizer_common/sanitizer_platform.h
-compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_openbsd.cpp
-compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_openbsd.h
-compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h
-compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp
-compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h
-compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_fuchsia.h
-compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_win.cpp
-compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup_constants.h
-compiler-rt/lib/sanitizer_common/sanitizer_thread_safety.h
-compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h
-compiler-rt/lib/sanitizer_common/sanitizer_type_traits.cpp
-compiler-rt/lib/sanitizer_common/sanitizer_type_traits.h
-compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp
-compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_wrappers.cpp
-compiler-rt/lib/sanitizer_common/tests/sanitizer_addrhashmap_test.cpp
-compiler-rt/lib/sanitizer_common/tests/sanitizer_chained_origin_depot_test.cpp
-compiler-rt/lib/sanitizer_common/tests/sanitizer_dense_map_test.cpp
-compiler-rt/lib/sanitizer_common/tests/sanitizer_flat_map_test.cpp
-compiler-rt/lib/sanitizer_common/tests/sanitizer_hash_test.cpp
-compiler-rt/lib/sanitizer_common/tests/sanitizer_leb128_test.cpp
-compiler-rt/lib/sanitizer_common/tests/sanitizer_lzw_test.cpp
-compiler-rt/lib/sanitizer_common/tests/sanitizer_stackdepot_test.cpp
-compiler-rt/lib/sanitizer_common/tests/sanitizer_stack_store_test.cpp
-compiler-rt/lib/sanitizer_common/tests/sanitizer_stoptheworld_test.cpp
-compiler-rt/lib/sanitizer_common/tests/sanitizer_test_main.cpp
-compiler-rt/lib/sanitizer_common/tests/sanitizer_type_traits_test.cpp
-compiler-rt/lib/scudo/standalone/allocator_config.h
-compiler-rt/lib/scudo/standalone/atomic_helpers.h
-compiler-rt/lib/scudo/standalone/bytemap.h
-compiler-rt/lib/scudo/standalone/checksum.cpp
-compiler-rt/lib/scudo/standalone/checksum.h
-compiler-rt/lib/scudo/standalone/chunk.h
-compiler-rt/lib/scudo/standalone/combined.h
-compiler-rt/lib/scudo/standalone/common.cpp
-compiler-rt/lib/scudo/standalone/common.h
-compiler-rt/lib/scudo/standalone/crc32_hw.cpp
-compiler-rt/lib/scudo/standalone/flags.cpp
-compiler-rt/lib/scudo/standalone/flags.h
-compiler-rt/lib/scudo/standalone/flags_parser.cpp
-compiler-rt/lib/scudo/standalone/flags_parser.h
-compiler-rt/lib/scudo/standalone/fuchsia.cpp
-compiler-rt/lib/scudo/standalone/fuchsia.h
-compiler-rt/lib/scudo/standalone/internal_defs.h
-compiler-rt/lib/scudo/standalone/linux.cpp
-compiler-rt/lib/scudo/standalone/linux.h
-compiler-rt/lib/scudo/standalone/list.h
-compiler-rt/lib/scudo/standalone/local_cache.h
-compiler-rt/lib/scudo/standalone/memtag.h
-compiler-rt/lib/scudo/standalone/mutex.h
-compiler-rt/lib/scudo/standalone/options.h
-compiler-rt/lib/scudo/standalone/platform.h
-compiler-rt/lib/scudo/standalone/primary32.h
-compiler-rt/lib/scudo/standalone/primary64.h
-compiler-rt/lib/scudo/standalone/quarantine.h
-compiler-rt/lib/scudo/standalone/release.cpp
-compiler-rt/lib/scudo/standalone/release.h
-compiler-rt/lib/scudo/standalone/report.cpp
-compiler-rt/lib/scudo/standalone/report.h
-compiler-rt/lib/scudo/standalone/secondary.h
-compiler-rt/lib/scudo/standalone/size_class_map.h
-compiler-rt/lib/scudo/standalone/stack_depot.h
-compiler-rt/lib/scudo/standalone/stats.h
-compiler-rt/lib/scudo/standalone/string_utils.cpp
-compiler-rt/lib/scudo/standalone/string_utils.h
-compiler-rt/lib/scudo/standalone/trusty.cpp
-compiler-rt/lib/scudo/standalone/trusty.h
-compiler-rt/lib/scudo/standalone/tsd.h
-compiler-rt/lib/scudo/standalone/tsd_exclusive.h
-compiler-rt/lib/scudo/standalone/tsd_shared.h
-compiler-rt/lib/scudo/standalone/vector.h
-compiler-rt/lib/scudo/standalone/wrappers_c.cpp
-compiler-rt/lib/scudo/standalone/wrappers_c.h
-compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp
-compiler-rt/lib/scudo/standalone/wrappers_c_checks.h
-compiler-rt/lib/scudo/standalone/benchmarks/malloc_benchmark.cpp
-compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp
-compiler-rt/lib/scudo/standalone/include/scudo/interface.h
-compiler-rt/lib/scudo/standalone/tests/atomic_test.cpp
-compiler-rt/lib/scudo/standalone/tests/bytemap_test.cpp
-compiler-rt/lib/scudo/standalone/tests/checksum_test.cpp
-compiler-rt/lib/scudo/standalone/tests/chunk_test.cpp
-compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
-compiler-rt/lib/scudo/standalone/tests/common_test.cpp
-compiler-rt/lib/scudo/standalone/tests/flags_test.cpp
-compiler-rt/lib/scudo/standalone/tests/list_test.cpp
-compiler-rt/lib/scudo/standalone/tests/map_test.cpp
-compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp
-compiler-rt/lib/scudo/standalone/tests/mutex_test.cpp
-compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
-compiler-rt/lib/scudo/standalone/tests/quarantine_test.cpp
-compiler-rt/lib/scudo/standalone/tests/release_test.cpp
-compiler-rt/lib/scudo/standalone/tests/report_test.cpp
-compiler-rt/lib/scudo/standalone/tests/scudo_unit_test.h
-compiler-rt/lib/scudo/standalone/tests/scudo_unit_test_main.cpp
-compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
-compiler-rt/lib/scudo/standalone/tests/size_class_map_test.cpp
-compiler-rt/lib/scudo/standalone/tests/stats_test.cpp
-compiler-rt/lib/scudo/standalone/tests/strings_test.cpp
-compiler-rt/lib/scudo/standalone/tests/vector_test.cpp
-compiler-rt/lib/scudo/standalone/tests/wrappers_cpp_test.cpp
-compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
-compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp
-compiler-rt/lib/tsan/rtl/tsan_fd.h
-compiler-rt/lib/tsan/rtl/tsan_ignoreset.h
-compiler-rt/lib/tsan/rtl/tsan_ilist.h
-compiler-rt/lib/tsan/rtl/tsan_interface_ann.h
-compiler-rt/lib/tsan/rtl/tsan_mman.h
-compiler-rt/lib/tsan/rtl/tsan_mutexset.h
-compiler-rt/lib/tsan/rtl/tsan_ppc_regs.h
-compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp
-compiler-rt/lib/tsan/rtl/tsan_shadow.h
-compiler-rt/lib/tsan/rtl/tsan_stack_trace.h
-compiler-rt/lib/tsan/rtl/tsan_suppressions.h
-compiler-rt/lib/tsan/rtl/tsan_symbolize.h
-compiler-rt/lib/tsan/rtl/tsan_trace.h
-compiler-rt/lib/tsan/rtl/tsan_vector_clock.h
-compiler-rt/lib/tsan/rtl-old/tsan_fd.h
-compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.h
-compiler-rt/lib/tsan/rtl-old/tsan_ilist.h
-compiler-rt/lib/tsan/rtl-old/tsan_interface_ann.h
-compiler-rt/lib/tsan/rtl-old/tsan_mman.h
-compiler-rt/lib/tsan/rtl-old/tsan_mutexset.h
-compiler-rt/lib/tsan/rtl-old/tsan_ppc_regs.h
-compiler-rt/lib/tsan/rtl-old/tsan_rtl_access.cpp
-compiler-rt/lib/tsan/rtl-old/tsan_shadow.h
-compiler-rt/lib/tsan/rtl-old/tsan_stack_trace.h
-compiler-rt/lib/tsan/rtl-old/tsan_suppressions.h
-compiler-rt/lib/tsan/rtl-old/tsan_symbolize.h
-compiler-rt/lib/tsan/rtl-old/tsan_vector_clock.h
-compiler-rt/lib/tsan/tests/unit/tsan_ilist_test.cpp
-compiler-rt/lib/tsan/tests/unit/tsan_trace_test.cpp
-compiler-rt/lib/tsan/tests/unit/tsan_vector_clock_test.cpp
-compiler-rt/lib/ubsan/ubsan_init_standalone_preinit.cpp
-compiler-rt/lib/ubsan/ubsan_monitor.cpp
-compiler-rt/lib/ubsan/ubsan_monitor.h
-compiler-rt/lib/ubsan/ubsan_signals_standalone.h
-compiler-rt/lib/ubsan/ubsan_win_dll_thunk.cpp
-compiler-rt/lib/ubsan/ubsan_win_dynamic_runtime_thunk.cpp
-compiler-rt/lib/ubsan/ubsan_win_weak_interception.cpp
-compiler-rt/lib/xray/xray_AArch64.cpp
-compiler-rt/lib/xray/xray_arm.cpp
-compiler-rt/lib/xray/xray_basic_flags.cpp
-compiler-rt/lib/xray/xray_basic_flags.h
-compiler-rt/lib/xray/xray_basic_logging.h
-compiler-rt/lib/xray/xray_buffer_queue.cpp
-compiler-rt/lib/xray/xray_buffer_queue.h
-compiler-rt/lib/xray/xray_fdr_controller.h
-compiler-rt/lib/xray/xray_fdr_flags.cpp
-compiler-rt/lib/xray/xray_fdr_flags.h
-compiler-rt/lib/xray/xray_fdr_logging.h
-compiler-rt/lib/xray/xray_fdr_log_records.h
-compiler-rt/lib/xray/xray_flags.cpp
-compiler-rt/lib/xray/xray_flags.h
-compiler-rt/lib/xray/xray_interface_internal.h
-compiler-rt/lib/xray/xray_log_interface.cpp
-compiler-rt/lib/xray/xray_mips.cpp
-compiler-rt/lib/xray/xray_mips64.cpp
-compiler-rt/lib/xray/xray_powerpc64.cpp
-compiler-rt/lib/xray/xray_profile_collector.cpp
-compiler-rt/lib/xray/xray_profile_collector.h
-compiler-rt/lib/xray/xray_profiling.cpp
-compiler-rt/lib/xray/xray_profiling_flags.cpp
-compiler-rt/lib/xray/xray_profiling_flags.h
-compiler-rt/lib/xray/xray_recursion_guard.h
-compiler-rt/lib/xray/xray_trampoline_powerpc64.cpp
-compiler-rt/lib/xray/xray_tsc.h
-compiler-rt/lib/xray/tests/unit/allocator_test.cpp
-compiler-rt/lib/xray/tests/unit/buffer_queue_test.cpp
-compiler-rt/lib/xray/tests/unit/fdr_log_writer_test.cpp
-compiler-rt/lib/xray/tests/unit/function_call_trie_test.cpp
-compiler-rt/lib/xray/tests/unit/profile_collector_test.cpp
-compiler-rt/lib/xray/tests/unit/segmented_array_test.cpp
-compiler-rt/lib/xray/tests/unit/test_helpers.h
-compiler-rt/lib/xray/tests/unit/xray_unit_test_main.cpp
-compiler-rt/tools/gwp_asan/options_parser_fuzzer.cpp
-compiler-rt/tools/gwp_asan/stack_trace_compressor_fuzzer.cpp
-cross-project-tests/debuginfo-tests/clang_llvm_roundtrip/simplified_template_names_noncanonical_type_units.cpp
-cross-project-tests/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cpp
-cross-project-tests/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/test.cpp
-cross-project-tests/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/source/test.cpp
-cross-project-tests/debuginfo-tests/dexter/feature_tests/commands/perfect/limit_steps/hit_count.cpp
-cross-project-tests/debuginfo-tests/dexter/feature_tests/commands/perfect/limit_steps/limit_steps_line_mismatch.cpp
-cross-project-tests/debuginfo-tests/dexter-tests/global-constant.cpp
-cross-project-tests/debuginfo-tests/dexter-tests/nrvo.cpp
-cross-project-tests/debuginfo-tests/dexter-tests/realigned-frame.cpp
-cross-project-tests/debuginfo-tests/llvm-prettyprinters/gdb/llvm-support.cpp
-flang/examples/external-hello.cpp
-flang/examples/FlangOmpReport/FlangOmpReport.cpp
-flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
-flang/examples/FlangOmpReport/FlangOmpReportVisitor.h
-flang/examples/PrintFlangFunctionNames/PrintFlangFunctionNames.cpp
-flang/include/flang/ISO_Fortran_binding.h
-flang/include/flang/Common/bit-population-count.h
-flang/include/flang/Common/constexpr-bitset.h
-flang/include/flang/Common/default-kinds.h
-flang/include/flang/Common/enum-set.h
-flang/include/flang/Common/fast-int-set.h
-flang/include/flang/Common/format.h
-flang/include/flang/Common/Fortran-features.h
-flang/include/flang/Common/Fortran.h
-flang/include/flang/Common/idioms.h
-flang/include/flang/Common/indirection.h
-flang/include/flang/Common/interval.h
-flang/include/flang/Common/leading-zero-bit-count.h
-flang/include/flang/Common/long-double.h
-flang/include/flang/Common/real.h
-flang/include/flang/Common/reference-counted.h
-flang/include/flang/Common/reference.h
-flang/include/flang/Common/restorer.h
-flang/include/flang/Common/static-multimap-view.h
-flang/include/flang/Common/template.h
-flang/include/flang/Common/uint128.h
-flang/include/flang/Common/unwrap.h
-flang/include/flang/Decimal/binary-floating-point.h
-flang/include/flang/Decimal/decimal.h
-flang/include/flang/Evaluate/call.h
-flang/include/flang/Evaluate/characteristics.h
-flang/include/flang/Evaluate/check-expression.h
-flang/include/flang/Evaluate/common.h
-flang/include/flang/Evaluate/complex.h
-flang/include/flang/Evaluate/constant.h
-flang/include/flang/Evaluate/expression.h
-flang/include/flang/Evaluate/fold-designator.h
-flang/include/flang/Evaluate/fold.h
-flang/include/flang/Evaluate/formatting.h
-flang/include/flang/Evaluate/initial-image.h
-flang/include/flang/Evaluate/integer.h
-flang/include/flang/Evaluate/intrinsics-library.h
-flang/include/flang/Evaluate/intrinsics.h
-flang/include/flang/Evaluate/logical.h
-flang/include/flang/Evaluate/real.h
-flang/include/flang/Evaluate/rounding-bits.h
-flang/include/flang/Evaluate/shape.h
-flang/include/flang/Evaluate/static-data.h
-flang/include/flang/Evaluate/tools.h
-flang/include/flang/Evaluate/traverse.h
-flang/include/flang/Evaluate/type.h
-flang/include/flang/Evaluate/variable.h
-flang/include/flang/Frontend/CompilerInstance.h
-flang/include/flang/Frontend/FrontendAction.h
-flang/include/flang/Frontend/FrontendActions.h
-flang/include/flang/Frontend/FrontendOptions.h
-flang/include/flang/Frontend/FrontendPluginRegistry.h
-flang/include/flang/Frontend/PreprocessorOptions.h
-flang/include/flang/Frontend/TargetOptions.h
-flang/include/flang/Frontend/TextDiagnostic.h
-flang/include/flang/Frontend/TextDiagnosticBuffer.h
-flang/include/flang/Frontend/TextDiagnosticPrinter.h
-flang/include/flang/FrontendTool/Utils.h
-flang/include/flang/Lower/AbstractConverter.h
-flang/include/flang/Lower/Allocatable.h
-flang/include/flang/Lower/BoxAnalyzer.h
-flang/include/flang/Lower/Bridge.h
-flang/include/flang/Lower/CallInterface.h
-flang/include/flang/Lower/Coarray.h
-flang/include/flang/Lower/ComponentPath.h
-flang/include/flang/Lower/ConvertExpr.h
-flang/include/flang/Lower/ConvertType.h
-flang/include/flang/Lower/ConvertVariable.h
-flang/include/flang/Lower/DumpEvaluateExpr.h
-flang/include/flang/Lower/HostAssociations.h
-flang/include/flang/Lower/IntervalSet.h
-flang/include/flang/Lower/IntrinsicCall.h
-flang/include/flang/Lower/IO.h
-flang/include/flang/Lower/IterationSpace.h
-flang/include/flang/Lower/Mangler.h
-flang/include/flang/Lower/OpenACC.h
-flang/include/flang/Lower/OpenMP.h
-flang/include/flang/Lower/PFTBuilder.h
-flang/include/flang/Lower/PFTDefs.h
-flang/include/flang/Lower/Runtime.h
-flang/include/flang/Lower/StatementContext.h
-flang/include/flang/Lower/Todo.h
-flang/include/flang/Lower/Support/Utils.h
-flang/include/flang/Lower/Support/Verifier.h
-flang/include/flang/Optimizer/Builder/BoxValue.h
-flang/include/flang/Optimizer/Builder/Character.h
-flang/include/flang/Optimizer/Builder/Complex.h
-flang/include/flang/Optimizer/Builder/DoLoopHelper.h
-flang/include/flang/Optimizer/Builder/Factory.h
-flang/include/flang/Optimizer/Builder/FIRBuilder.h
-flang/include/flang/Optimizer/Builder/MutableBox.h
-flang/include/flang/Optimizer/Builder/Runtime/Assign.h
-flang/include/flang/Optimizer/Builder/Runtime/Character.h
-flang/include/flang/Optimizer/Builder/Runtime/Command.h
-flang/include/flang/Optimizer/Builder/Runtime/Derived.h
-flang/include/flang/Optimizer/Builder/Runtime/Numeric.h
-flang/include/flang/Optimizer/Builder/Runtime/Ragged.h
-flang/include/flang/Optimizer/Builder/Runtime/Reduction.h
-flang/include/flang/Optimizer/Builder/Runtime/RTBuilder.h
-flang/include/flang/Optimizer/Builder/Runtime/Stop.h
-flang/include/flang/Optimizer/Builder/Runtime/Transformational.h
-flang/include/flang/Optimizer/CodeGen/CodeGen.h
-flang/include/flang/Optimizer/Dialect/FIRAttr.h
-flang/include/flang/Optimizer/Dialect/FIRDialect.h
-flang/include/flang/Optimizer/Dialect/FIROps.h
-flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
-flang/include/flang/Optimizer/Dialect/FIRType.h
-flang/include/flang/Optimizer/Support/FatalError.h
-flang/include/flang/Optimizer/Support/FIRContext.h
-flang/include/flang/Optimizer/Support/InitFIR.h
-flang/include/flang/Optimizer/Support/InternalNames.h
-flang/include/flang/Optimizer/Support/KindMapping.h
-flang/include/flang/Optimizer/Support/Matcher.h
-flang/include/flang/Optimizer/Support/TypeCode.h
-flang/include/flang/Optimizer/Support/Utils.h
-flang/include/flang/Optimizer/Transforms/Passes.h
-flang/include/flang/Parser/char-block.h
-flang/include/flang/Parser/char-buffer.h
-flang/include/flang/Parser/char-set.h
-flang/include/flang/Parser/characters.h
-flang/include/flang/Parser/dump-parse-tree.h
-flang/include/flang/Parser/format-specification.h
-flang/include/flang/Parser/instrumented-parser.h
-flang/include/flang/Parser/message.h
-flang/include/flang/Parser/parse-state.h
-flang/include/flang/Parser/parse-tree-visitor.h
-flang/include/flang/Parser/parsing.h
-flang/include/flang/Parser/preprocessor.h
-flang/include/flang/Parser/provenance.h
-flang/include/flang/Parser/source.h
-flang/include/flang/Parser/token-sequence.h
-flang/include/flang/Parser/tools.h
-flang/include/flang/Parser/unparse.h
-flang/include/flang/Parser/user-state.h
-flang/include/flang/Runtime/allocatable.h
-flang/include/flang/Runtime/assign.h
-flang/include/flang/Runtime/c-or-cpp.h
-flang/include/flang/Runtime/character.h
-flang/include/flang/Runtime/command.h
-flang/include/flang/Runtime/cpp-type.h
-flang/include/flang/Runtime/derived-api.h
-flang/include/flang/Runtime/descriptor.h
-flang/include/flang/Runtime/entry-names.h
-flang/include/flang/Runtime/extensions.h
-flang/include/flang/Runtime/inquiry.h
-flang/include/flang/Runtime/io-api.h
-flang/include/flang/Runtime/iostat.h
-flang/include/flang/Runtime/main.h
-flang/include/flang/Runtime/matmul.h
-flang/include/flang/Runtime/memory.h
-flang/include/flang/Runtime/misc-intrinsic.h
-flang/include/flang/Runtime/numeric.h
-flang/include/flang/Runtime/pointer.h
-flang/include/flang/Runtime/ragged.h
-flang/include/flang/Runtime/random.h
-flang/include/flang/Runtime/reduction.h
-flang/include/flang/Runtime/stop.h
-flang/include/flang/Runtime/support.h
-flang/include/flang/Runtime/time-intrinsic.h
-flang/include/flang/Runtime/transformational.h
-flang/include/flang/Runtime/type-code.h
-flang/include/flang/Semantics/attr.h
-flang/include/flang/Semantics/expression.h
-flang/include/flang/Semantics/openmp-directive-sets.h
-flang/include/flang/Semantics/runtime-type-info.h
-flang/include/flang/Semantics/scope.h
-flang/include/flang/Semantics/semantics.h
-flang/include/flang/Semantics/symbol.h
-flang/include/flang/Semantics/tools.h
-flang/include/flang/Semantics/type.h
-flang/include/flang/Semantics/unparse-with-symbols.h
-flang/lib/Common/default-kinds.cpp
-flang/lib/Common/Fortran-features.cpp
-flang/lib/Common/Fortran.cpp
-flang/lib/Common/idioms.cpp
-flang/lib/Decimal/big-radix-floating-point.h
-flang/lib/Decimal/binary-to-decimal.cpp
-flang/lib/Decimal/decimal-to-binary.cpp
-flang/lib/Evaluate/call.cpp
-flang/lib/Evaluate/character.h
-flang/lib/Evaluate/check-expression.cpp
-flang/lib/Evaluate/common.cpp
-flang/lib/Evaluate/complex.cpp
-flang/lib/Evaluate/constant.cpp
-flang/lib/Evaluate/expression.cpp
-flang/lib/Evaluate/fold-character.cpp
-flang/lib/Evaluate/fold-complex.cpp
-flang/lib/Evaluate/fold-designator.cpp
-flang/lib/Evaluate/fold-implementation.h
-flang/lib/Evaluate/fold-logical.cpp
-flang/lib/Evaluate/fold-real.cpp
-flang/lib/Evaluate/fold-reduction.cpp
-flang/lib/Evaluate/fold-reduction.h
-flang/lib/Evaluate/fold.cpp
-flang/lib/Evaluate/formatting.cpp
-flang/lib/Evaluate/host.cpp
-flang/lib/Evaluate/host.h
-flang/lib/Evaluate/initial-image.cpp
-flang/lib/Evaluate/int-power.h
-flang/lib/Evaluate/integer.cpp
-flang/lib/Evaluate/intrinsics-library.cpp
-flang/lib/Evaluate/intrinsics.cpp
-flang/lib/Evaluate/logical.cpp
-flang/lib/Evaluate/real.cpp
-flang/lib/Evaluate/shape.cpp
-flang/lib/Evaluate/static-data.cpp
-flang/lib/Evaluate/tools.cpp
-flang/lib/Evaluate/type.cpp
-flang/lib/Evaluate/variable.cpp
-flang/lib/Frontend/CompilerInstance.cpp
-flang/lib/Frontend/FrontendAction.cpp
-flang/lib/Frontend/FrontendOptions.cpp
-flang/lib/Frontend/TextDiagnostic.cpp
-flang/lib/Frontend/TextDiagnosticBuffer.cpp
-flang/lib/Frontend/TextDiagnosticPrinter.cpp
-flang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
-flang/lib/Lower/Allocatable.cpp
-flang/lib/Lower/Bridge.cpp
-flang/lib/Lower/CallInterface.cpp
-flang/lib/Lower/Coarray.cpp
-flang/lib/Lower/ComponentPath.cpp
-flang/lib/Lower/ConvertExpr.cpp
-flang/lib/Lower/ConvertType.cpp
-flang/lib/Lower/ConvertVariable.cpp
-flang/lib/Lower/DumpEvaluateExpr.cpp
-flang/lib/Lower/IntervalSet.h
-flang/lib/Lower/IntrinsicCall.cpp
-flang/lib/Lower/IO.cpp
-flang/lib/Lower/IterationSpace.cpp
-flang/lib/Lower/Mangler.cpp
-flang/lib/Lower/OpenACC.cpp
-flang/lib/Lower/OpenMP.cpp
-flang/lib/Lower/PFTBuilder.cpp
-flang/lib/Lower/RTBuilder.h
-flang/lib/Lower/Runtime.cpp
-flang/lib/Lower/SymbolMap.cpp
-flang/lib/Optimizer/Builder/BoxValue.cpp
-flang/lib/Optimizer/Builder/Character.cpp
-flang/lib/Optimizer/Builder/Complex.cpp
-flang/lib/Optimizer/Builder/DoLoopHelper.cpp
-flang/lib/Optimizer/Builder/FIRBuilder.cpp
-flang/lib/Optimizer/Builder/MutableBox.cpp
-flang/lib/Optimizer/Builder/Runtime/Assign.cpp
-flang/lib/Optimizer/Builder/Runtime/Character.cpp
-flang/lib/Optimizer/Builder/Runtime/Command.cpp
-flang/lib/Optimizer/Builder/Runtime/Derived.cpp
-flang/lib/Optimizer/Builder/Runtime/Numeric.cpp
-flang/lib/Optimizer/Builder/Runtime/Ragged.cpp
-flang/lib/Optimizer/Builder/Runtime/Reduction.cpp
-flang/lib/Optimizer/Builder/Runtime/Stop.cpp
-flang/lib/Optimizer/Builder/Runtime/Transformational.cpp
-flang/lib/Optimizer/CodeGen/CGOps.cpp
-flang/lib/Optimizer/CodeGen/CGOps.h
-flang/lib/Optimizer/CodeGen/CodeGen.cpp
-flang/lib/Optimizer/CodeGen/DescriptorModel.h
-flang/lib/Optimizer/CodeGen/PassDetail.h
-flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
-flang/lib/Optimizer/CodeGen/Target.cpp
-flang/lib/Optimizer/CodeGen/Target.h
-flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
-flang/lib/Optimizer/CodeGen/TypeConverter.h
-flang/lib/Optimizer/Dialect/FIRAttr.cpp
-flang/lib/Optimizer/Dialect/FIRDialect.cpp
-flang/lib/Optimizer/Dialect/FIROps.cpp
-flang/lib/Optimizer/Dialect/FIRType.cpp
-flang/lib/Optimizer/Dialect/Inliner.cpp
-flang/lib/Optimizer/Support/FIRContext.cpp
-flang/lib/Optimizer/Support/InitFIR.cpp
-flang/lib/Optimizer/Support/InternalNames.cpp
-flang/lib/Optimizer/Support/KindMapping.cpp
-flang/lib/Optimizer/Transforms/AbstractResult.cpp
-flang/lib/Optimizer/Transforms/AffineDemotion.cpp
-flang/lib/Optimizer/Transforms/AffinePromotion.cpp
-flang/lib/Optimizer/Transforms/ArrayValueCopy.cpp
-flang/lib/Optimizer/Transforms/CharacterConversion.cpp
-flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
-flang/lib/Optimizer/Transforms/MemoryAllocation.cpp
-flang/lib/Optimizer/Transforms/MemRefDataFlowOpt.cpp
-flang/lib/Optimizer/Transforms/PassDetail.h
-flang/lib/Optimizer/Transforms/RewriteLoop.cpp
-flang/lib/Optimizer/Transforms/StackArrays.cpp
-flang/lib/Parser/basic-parsers.h
-flang/lib/Parser/char-block.cpp
-flang/lib/Parser/char-buffer.cpp
-flang/lib/Parser/char-set.cpp
-flang/lib/Parser/characters.cpp
-flang/lib/Parser/debug-parser.cpp
-flang/lib/Parser/debug-parser.h
-flang/lib/Parser/executable-parsers.cpp
-flang/lib/Parser/expr-parsers.cpp
-flang/lib/Parser/expr-parsers.h
-flang/lib/Parser/Fortran-parsers.cpp
-flang/lib/Parser/instrumented-parser.cpp
-flang/lib/Parser/io-parsers.cpp
-flang/lib/Parser/message.cpp
-flang/lib/Parser/misc-parsers.h
-flang/lib/Parser/openacc-parsers.cpp
-flang/lib/Parser/openmp-parsers.cpp
-flang/lib/Parser/parse-tree.cpp
-flang/lib/Parser/parsing.cpp
-flang/lib/Parser/preprocessor.cpp
-flang/lib/Parser/prescan.cpp
-flang/lib/Parser/prescan.h
-flang/lib/Parser/program-parsers.cpp
-flang/lib/Parser/provenance.cpp
-flang/lib/Parser/source.cpp
-flang/lib/Parser/stmt-parser.h
-flang/lib/Parser/token-parsers.h
-flang/lib/Parser/token-sequence.cpp
-flang/lib/Parser/tools.cpp
-flang/lib/Parser/type-parser-implementation.h
-flang/lib/Parser/type-parsers.h
-flang/lib/Parser/unparse.cpp
-flang/lib/Parser/user-state.cpp
-flang/lib/Semantics/assignment.cpp
-flang/lib/Semantics/assignment.h
-flang/lib/Semantics/attr.cpp
-flang/lib/Semantics/canonicalize-acc.cpp
-flang/lib/Semantics/canonicalize-acc.h
-flang/lib/Semantics/canonicalize-do.cpp
-flang/lib/Semantics/canonicalize-do.h
-flang/lib/Semantics/canonicalize-omp.cpp
-flang/lib/Semantics/canonicalize-omp.h
-flang/lib/Semantics/check-acc-structure.cpp
-flang/lib/Semantics/check-allocate.cpp
-flang/lib/Semantics/check-allocate.h
-flang/lib/Semantics/check-arithmeticif.cpp
-flang/lib/Semantics/check-arithmeticif.h
-flang/lib/Semantics/check-call.h
-flang/lib/Semantics/check-case.cpp
-flang/lib/Semantics/check-case.h
-flang/lib/Semantics/check-coarray.cpp
-flang/lib/Semantics/check-coarray.h
-flang/lib/Semantics/check-data.cpp
-flang/lib/Semantics/check-data.h
-flang/lib/Semantics/check-deallocate.cpp
-flang/lib/Semantics/check-deallocate.h
-flang/lib/Semantics/check-declarations.h
-flang/lib/Semantics/check-directive-structure.h
-flang/lib/Semantics/check-do-forall.cpp
-flang/lib/Semantics/check-do-forall.h
-flang/lib/Semantics/check-if-stmt.cpp
-flang/lib/Semantics/check-if-stmt.h
-flang/lib/Semantics/check-io.cpp
-flang/lib/Semantics/check-io.h
-flang/lib/Semantics/check-namelist.cpp
-flang/lib/Semantics/check-namelist.h
-flang/lib/Semantics/check-nullify.cpp
-flang/lib/Semantics/check-nullify.h
-flang/lib/Semantics/check-omp-structure.cpp
-flang/lib/Semantics/check-omp-structure.h
-flang/lib/Semantics/check-purity.cpp
-flang/lib/Semantics/check-purity.h
-flang/lib/Semantics/check-return.cpp
-flang/lib/Semantics/check-return.h
-flang/lib/Semantics/check-select-rank.cpp
-flang/lib/Semantics/check-select-rank.h
-flang/lib/Semantics/check-select-type.cpp
-flang/lib/Semantics/check-select-type.h
-flang/lib/Semantics/check-stop.cpp
-flang/lib/Semantics/check-stop.h
-flang/lib/Semantics/compute-offsets.cpp
-flang/lib/Semantics/compute-offsets.h
-flang/lib/Semantics/data-to-inits.cpp
-flang/lib/Semantics/mod-file.h
-flang/lib/Semantics/pointer-assignment.cpp
-flang/lib/Semantics/pointer-assignment.h
-flang/lib/Semantics/program-tree.cpp
-flang/lib/Semantics/program-tree.h
-flang/lib/Semantics/resolve-directives.cpp
-flang/lib/Semantics/resolve-directives.h
-flang/lib/Semantics/resolve-labels.cpp
-flang/lib/Semantics/resolve-labels.h
-flang/lib/Semantics/resolve-names-utils.cpp
-flang/lib/Semantics/resolve-names-utils.h
-flang/lib/Semantics/resolve-names.h
-flang/lib/Semantics/rewrite-parse-tree.cpp
-flang/lib/Semantics/rewrite-parse-tree.h
-flang/lib/Semantics/runtime-type-info.cpp
-flang/lib/Semantics/scope.cpp
-flang/lib/Semantics/semantics.cpp
-flang/lib/Semantics/tools.cpp
-flang/lib/Semantics/unparse-with-symbols.cpp
-flang/module/omp_lib.h
-flang/runtime/allocatable.cpp
-flang/runtime/assign.cpp
-flang/runtime/buffer.cpp
-flang/runtime/buffer.h
-flang/runtime/character.cpp
-flang/runtime/command.cpp
-flang/runtime/complex-reduction.h
-flang/runtime/connection.cpp
-flang/runtime/connection.h
-flang/runtime/copy.cpp
-flang/runtime/copy.h
-flang/runtime/derived-api.cpp
-flang/runtime/derived.h
-flang/runtime/descriptor-io.cpp
-flang/runtime/descriptor-io.h
-flang/runtime/descriptor.cpp
-flang/runtime/dot-product.cpp
-flang/runtime/edit-input.cpp
-flang/runtime/edit-input.h
-flang/runtime/edit-output.cpp
-flang/runtime/edit-output.h
-flang/runtime/environment.cpp
-flang/runtime/environment.h
-flang/runtime/extensions.cpp
-flang/runtime/extrema.cpp
-flang/runtime/file.cpp
-flang/runtime/file.h
-flang/runtime/findloc.cpp
-flang/runtime/format-implementation.h
-flang/runtime/format.cpp
-flang/runtime/format.h
-flang/runtime/inquiry.cpp
-flang/runtime/internal-unit.cpp
-flang/runtime/internal-unit.h
-flang/runtime/io-api.cpp
-flang/runtime/io-error.cpp
-flang/runtime/io-error.h
-flang/runtime/io-stmt.cpp
-flang/runtime/io-stmt.h
-flang/runtime/iostat.cpp
-flang/runtime/ISO_Fortran_binding.cpp
-flang/runtime/lock.h
-flang/runtime/main.cpp
-flang/runtime/matmul.cpp
-flang/runtime/memory.cpp
-flang/runtime/misc-intrinsic.cpp
-flang/runtime/namelist.cpp
-flang/runtime/namelist.h
-flang/runtime/numeric.cpp
-flang/runtime/pointer.cpp
-flang/runtime/product.cpp
-flang/runtime/ragged.cpp
-flang/runtime/random.cpp
-flang/runtime/reduction-templates.h
-flang/runtime/reduction.cpp
-flang/runtime/stat.cpp
-flang/runtime/stat.h
-flang/runtime/stop.cpp
-flang/runtime/sum.cpp
-flang/runtime/support.cpp
-flang/runtime/terminator.cpp
-flang/runtime/terminator.h
-flang/runtime/time-intrinsic.cpp
-flang/runtime/tools.cpp
-flang/runtime/tools.h
-flang/runtime/transformational.cpp
-flang/runtime/type-code.cpp
-flang/runtime/type-info.cpp
-flang/runtime/type-info.h
-flang/runtime/unit-map.cpp
-flang/runtime/unit-map.h
-flang/runtime/unit.h
-flang/tools/bbc/bbc.cpp
-flang/tools/f18/dump.cpp
-flang/tools/f18-parse-demo/f18-parse-demo.cpp
-flang/tools/f18-parse-demo/stub-evaluate.cpp
-flang/tools/fir-opt/fir-opt.cpp
-flang/tools/flang-driver/driver.cpp
-flang/tools/flang-driver/fc1_main.cpp
-flang/tools/tco/tco.cpp
-flang/unittests/Common/FastIntSetTest.cpp
-flang/unittests/Decimal/quick-sanity-test.cpp
-flang/unittests/Decimal/thorough-test.cpp
-flang/unittests/Evaluate/bit-population-count.cpp
-flang/unittests/Evaluate/expression.cpp
-flang/unittests/Evaluate/folding.cpp
-flang/unittests/Evaluate/fp-testing.cpp
-flang/unittests/Evaluate/fp-testing.h
-flang/unittests/Evaluate/integer.cpp
-flang/unittests/Evaluate/intrinsics.cpp
-flang/unittests/Evaluate/ISO-Fortran-binding.cpp
-flang/unittests/Evaluate/leading-zero-bit-count.cpp
-flang/unittests/Evaluate/logical.cpp
-flang/unittests/Evaluate/real.cpp
-flang/unittests/Evaluate/reshape.cpp
-flang/unittests/Evaluate/testing.cpp
-flang/unittests/Evaluate/testing.h
-flang/unittests/Evaluate/uint128.cpp
-flang/unittests/Frontend/CompilerInstanceTest.cpp
-flang/unittests/Frontend/FrontendActionTest.cpp
-flang/unittests/Optimizer/InternalNamesTest.cpp
-flang/unittests/Optimizer/KindMappingTest.cpp
-flang/unittests/Optimizer/RTBuilder.cpp
-flang/unittests/Optimizer/Builder/CharacterTest.cpp
-flang/unittests/Optimizer/Builder/ComplexTest.cpp
-flang/unittests/Optimizer/Builder/DoLoopHelperTest.cpp
-flang/unittests/Optimizer/Builder/FIRBuilderTest.cpp
-flang/unittests/Optimizer/Builder/Runtime/AssignTest.cpp
-flang/unittests/Optimizer/Builder/Runtime/CharacterTest.cpp
-flang/unittests/Optimizer/Builder/Runtime/CommandTest.cpp
-flang/unittests/Optimizer/Builder/Runtime/DerivedTest.cpp
-flang/unittests/Optimizer/Builder/Runtime/NumericTest.cpp
-flang/unittests/Optimizer/Builder/Runtime/RaggedTest.cpp
-flang/unittests/Optimizer/Builder/Runtime/ReductionTest.cpp
-flang/unittests/Optimizer/Builder/Runtime/RuntimeCallTestBase.h
-flang/unittests/Optimizer/Builder/Runtime/StopTest.cpp
-flang/unittests/Optimizer/Builder/Runtime/TransformationalTest.cpp
-flang/unittests/Runtime/BufferTest.cpp
-flang/unittests/Runtime/CharacterTest.cpp
-flang/unittests/Runtime/CommandTest.cpp
-flang/unittests/Runtime/CrashHandlerFixture.cpp
-flang/unittests/Runtime/CrashHandlerFixture.h
-flang/unittests/Runtime/ExternalIOTest.cpp
-flang/unittests/Runtime/Format.cpp
-flang/unittests/Runtime/Inquiry.cpp
-flang/unittests/Runtime/ListInputTest.cpp
-flang/unittests/Runtime/Matmul.cpp
-flang/unittests/Runtime/MiscIntrinsic.cpp
-flang/unittests/Runtime/Namelist.cpp
-flang/unittests/Runtime/Numeric.cpp
-flang/unittests/Runtime/NumericalFormatTest.cpp
-flang/unittests/Runtime/Ragged.cpp
-flang/unittests/Runtime/Random.cpp
-flang/unittests/Runtime/Reduction.cpp
-flang/unittests/Runtime/RuntimeCrashTest.cpp
-flang/unittests/Runtime/Stop.cpp
-flang/unittests/Runtime/Time.cpp
-flang/unittests/Runtime/tools.h
-flang/unittests/Runtime/Transformational.cpp
-libc/AOR_v20.02/math/v_exp.h
-libc/benchmarks/JSON.cpp
-libc/benchmarks/JSON.h
-libc/benchmarks/LibcBenchmark.cpp
-libc/benchmarks/LibcBenchmark.h
-libc/benchmarks/LibcBenchmarkTest.cpp
-libc/benchmarks/LibcDefaultImplementations.cpp
-libc/benchmarks/LibcFunctionPrototypes.h
-libc/benchmarks/LibcMemoryBenchmark.cpp
-libc/benchmarks/LibcMemoryBenchmark.h
-libc/benchmarks/LibcMemoryBenchmarkMain.cpp
-libc/benchmarks/LibcMemoryBenchmarkTest.cpp
-libc/benchmarks/LibcMemoryGoogleBenchmarkMain.cpp
-libc/benchmarks/MemorySizeDistributions.cpp
-libc/benchmarks/MemorySizeDistributions.h
-libc/benchmarks/automemcpy/include/automemcpy/CodeGen.h
-libc/benchmarks/automemcpy/include/automemcpy/FunctionDescriptor.h
-libc/benchmarks/automemcpy/include/automemcpy/RandomFunctionGenerator.h
-libc/benchmarks/automemcpy/include/automemcpy/ResultAnalyzer.h
-libc/benchmarks/automemcpy/lib/CodeGen.cpp
-libc/benchmarks/automemcpy/lib/CodeGenMain.cpp
-libc/benchmarks/automemcpy/lib/RandomFunctionGenerator.cpp
-libc/benchmarks/automemcpy/lib/ResultAnalyzer.cpp
-libc/benchmarks/automemcpy/lib/ResultAnalyzerMain.cpp
-libc/benchmarks/automemcpy/unittests/CodeGenTest.cpp
-libc/benchmarks/automemcpy/unittests/ResultAnalyzerTest.cpp
-libc/config/linux/app.h
-libc/fuzzing/math/Compare.h
-libc/fuzzing/math/math_differential_fuzz.cpp
-libc/fuzzing/math/nextafter_differential_fuzz.cpp
-libc/fuzzing/math/RemQuoDiff.h
-libc/fuzzing/math/SingleInputSingleOutputDiff.h
-libc/fuzzing/math/TwoInputSingleOutputDiff.h
-libc/fuzzing/stdlib/atof_differential_fuzz.cpp
-libc/fuzzing/stdlib/qsort_fuzz.cpp
-libc/fuzzing/stdlib/StringParserOutputDiff.h
-libc/fuzzing/string/strcmp_fuzz.cpp
-libc/fuzzing/string/strstr_fuzz.cpp
-libc/include/__llvm-libc-common.h
-libc/include/llvm-libc-macros/fcntl-macros.h
-libc/include/llvm-libc-macros/stdio-macros.h
-libc/include/llvm-libc-macros/linux/fcntl-macros.h
-libc/include/llvm-libc-types/cnd_t.h
-libc/include/llvm-libc-types/div_t.h
-libc/include/llvm-libc-types/double_t.h
-libc/include/llvm-libc-types/fenv_t.h
-libc/include/llvm-libc-types/fexcept_t.h
-libc/include/llvm-libc-types/FILE.h
-libc/include/llvm-libc-types/float_t.h
-libc/include/llvm-libc-types/imaxdiv_t.h
-libc/include/llvm-libc-types/jmp_buf.h
-libc/include/llvm-libc-types/ldiv_t.h
-libc/include/llvm-libc-types/lldiv_t.h
-libc/include/llvm-libc-types/mode_t.h
-libc/include/llvm-libc-types/mtx_t.h
-libc/include/llvm-libc-types/off_t.h
-libc/include/llvm-libc-types/once_flag.h
-libc/include/llvm-libc-types/size_t.h
-libc/include/llvm-libc-types/ssize_t.h
-libc/include/llvm-libc-types/struct_sigaction.h
-libc/include/llvm-libc-types/struct_tm.h
-libc/include/llvm-libc-types/thrd_start_t.h
-libc/include/llvm-libc-types/thrd_t.h
-libc/include/llvm-libc-types/time_t.h
-libc/include/llvm-libc-types/__atexithandler_t.h
-libc/include/llvm-libc-types/__bsearchcompare_t.h
-libc/include/llvm-libc-types/__call_once_func_t.h
-libc/include/llvm-libc-types/__futex_word.h
-libc/include/llvm-libc-types/__mutex_type.h
-libc/include/llvm-libc-types/__qsortcompare_t.h
-libc/include/llvm-libc-types/__sighandler_t.h
-libc/loader/linux/aarch64/start.cpp
-libc/loader/linux/x86_64/start.cpp
-libc/src/assert/__assert_fail.h
-libc/src/ctype/isalnum.cpp
-libc/src/ctype/isalnum.h
-libc/src/ctype/isalpha.cpp
-libc/src/ctype/isalpha.h
-libc/src/ctype/isascii.cpp
-libc/src/ctype/isascii.h
-libc/src/ctype/isblank.cpp
-libc/src/ctype/isblank.h
-libc/src/ctype/iscntrl.cpp
-libc/src/ctype/iscntrl.h
-libc/src/ctype/isdigit.cpp
-libc/src/ctype/isdigit.h
-libc/src/ctype/isgraph.cpp
-libc/src/ctype/isgraph.h
-libc/src/ctype/islower.cpp
-libc/src/ctype/islower.h
-libc/src/ctype/isprint.cpp
-libc/src/ctype/isprint.h
-libc/src/ctype/ispunct.cpp
-libc/src/ctype/ispunct.h
-libc/src/ctype/isspace.cpp
-libc/src/ctype/isspace.h
-libc/src/ctype/isupper.cpp
-libc/src/ctype/isupper.h
-libc/src/ctype/isxdigit.cpp
-libc/src/ctype/isxdigit.h
-libc/src/ctype/toascii.cpp
-libc/src/ctype/toascii.h
-libc/src/ctype/tolower.cpp
-libc/src/ctype/tolower.h
-libc/src/ctype/toupper.cpp
-libc/src/ctype/toupper.h
-libc/src/errno/dummy_errno.cpp
-libc/src/errno/dummy_errno.h
-libc/src/errno/errno.cpp
-libc/src/errno/llvmlibc_errno.h
-libc/src/fcntl/creat.h
-libc/src/fcntl/open.h
-libc/src/fcntl/openat.h
-libc/src/fcntl/linux/creat.cpp
-libc/src/fcntl/linux/open.cpp
-libc/src/fcntl/linux/openat.cpp
-libc/src/fenv/feclearexcept.cpp
-libc/src/fenv/feclearexcept.h
-libc/src/fenv/fedisableexcept.cpp
-libc/src/fenv/fedisableexcept.h
-libc/src/fenv/feenableexcept.cpp
-libc/src/fenv/feenableexcept.h
-libc/src/fenv/fegetenv.cpp
-libc/src/fenv/fegetenv.h
-libc/src/fenv/fegetexcept.cpp
-libc/src/fenv/fegetexcept.h
-libc/src/fenv/fegetexceptflag.cpp
-libc/src/fenv/fegetexceptflag.h
-libc/src/fenv/fegetround.cpp
-libc/src/fenv/fegetround.h
-libc/src/fenv/feholdexcept.cpp
-libc/src/fenv/feholdexcept.h
-libc/src/fenv/feraiseexcept.cpp
-libc/src/fenv/feraiseexcept.h
-libc/src/fenv/fesetenv.cpp
-libc/src/fenv/fesetenv.h
-libc/src/fenv/fesetexceptflag.cpp
-libc/src/fenv/fesetexceptflag.h
-libc/src/fenv/fesetround.cpp
-libc/src/fenv/fesetround.h
-libc/src/fenv/fetestexcept.cpp
-libc/src/fenv/fetestexcept.h
-libc/src/fenv/feupdateenv.cpp
-libc/src/fenv/feupdateenv.h
-libc/src/inttypes/imaxdiv.cpp
-libc/src/inttypes/imaxdiv.h
-libc/src/inttypes/strtoimax.cpp
-libc/src/inttypes/strtoimax.h
-libc/src/inttypes/strtoumax.cpp
-libc/src/inttypes/strtoumax.h
-libc/src/math/ceil.h
-libc/src/math/ceilf.h
-libc/src/math/ceill.h
-libc/src/math/copysign.h
-libc/src/math/copysignf.h
-libc/src/math/copysignl.h
-libc/src/math/cos.h
-libc/src/math/cosf.h
-libc/src/math/exp2f.h
-libc/src/math/expf.h
-libc/src/math/expm1f.h
-libc/src/math/fabs.h
-libc/src/math/fabsf.h
-libc/src/math/fabsl.h
-libc/src/math/fdim.h
-libc/src/math/fdimf.h
-libc/src/math/fdiml.h
-libc/src/math/floor.h
-libc/src/math/floorf.h
-libc/src/math/floorl.h
-libc/src/math/fma.cpp
-libc/src/math/fma.h
-libc/src/math/fmaf.cpp
-libc/src/math/fmaf.h
-libc/src/math/fmax.h
-libc/src/math/fmaxf.h
-libc/src/math/fmaxl.h
-libc/src/math/fmin.h
-libc/src/math/fminf.h
-libc/src/math/fminl.h
-libc/src/math/frexp.h
-libc/src/math/frexpf.h
-libc/src/math/frexpl.h
-libc/src/math/hypot.h
-libc/src/math/hypotf.h
-libc/src/math/ilogb.h
-libc/src/math/ilogbf.h
-libc/src/math/ilogbl.h
-libc/src/math/ldexp.h
-libc/src/math/ldexpf.h
-libc/src/math/ldexpl.h
-libc/src/math/llrint.h
-libc/src/math/llrintf.h
-libc/src/math/llrintl.h
-libc/src/math/llround.h
-libc/src/math/llroundf.h
-libc/src/math/llroundl.h
-libc/src/math/log10f.h
-libc/src/math/log1pf.h
-libc/src/math/log2f.h
-libc/src/math/logb.h
-libc/src/math/logbf.h
-libc/src/math/logbl.h
-libc/src/math/logf.h
-libc/src/math/lrint.h
-libc/src/math/lrintf.h
-libc/src/math/lrintl.h
-libc/src/math/lround.h
-libc/src/math/lroundf.h
-libc/src/math/lroundl.h
-libc/src/math/modf.h
-libc/src/math/modff.h
-libc/src/math/modfl.h
-libc/src/math/nearbyint.h
-libc/src/math/nearbyintf.h
-libc/src/math/nearbyintl.h
-libc/src/math/nextafter.h
-libc/src/math/nextafterf.h
-libc/src/math/nextafterl.h
-libc/src/math/remainder.h
-libc/src/math/remainderf.h
-libc/src/math/remainderl.h
-libc/src/math/remquo.h
-libc/src/math/remquof.h
-libc/src/math/remquol.h
-libc/src/math/rint.h
-libc/src/math/rintf.h
-libc/src/math/rintl.h
-libc/src/math/round.h
-libc/src/math/roundf.h
-libc/src/math/roundl.h
-libc/src/math/sin.h
-libc/src/math/sincosf.h
-libc/src/math/sinf.h
-libc/src/math/sqrt.h
-libc/src/math/sqrtf.h
-libc/src/math/sqrtl.h
-libc/src/math/tan.h
-libc/src/math/trunc.h
-libc/src/math/truncf.h
-libc/src/math/truncl.h
-libc/src/math/aarch64/ceil.cpp
-libc/src/math/aarch64/ceilf.cpp
-libc/src/math/aarch64/floor.cpp
-libc/src/math/aarch64/floorf.cpp
-libc/src/math/aarch64/round.cpp
-libc/src/math/aarch64/roundf.cpp
-libc/src/math/aarch64/sqrt.cpp
-libc/src/math/aarch64/sqrtf.cpp
-libc/src/math/aarch64/trunc.cpp
-libc/src/math/aarch64/truncf.cpp
-libc/src/math/generic/ceil.cpp
-libc/src/math/generic/ceilf.cpp
-libc/src/math/generic/ceill.cpp
-libc/src/math/generic/common_constants.cpp
-libc/src/math/generic/common_constants.h
-libc/src/math/generic/copysign.cpp
-libc/src/math/generic/copysignf.cpp
-libc/src/math/generic/copysignl.cpp
-libc/src/math/generic/cosf.cpp
-libc/src/math/generic/dp_trig.cpp
-libc/src/math/generic/dp_trig.h
-libc/src/math/generic/exp2f.cpp
-libc/src/math/generic/expf.cpp
-libc/src/math/generic/expm1f.cpp
-libc/src/math/generic/exp_utils.cpp
-libc/src/math/generic/exp_utils.h
-libc/src/math/generic/fabs.cpp
-libc/src/math/generic/fabsf.cpp
-libc/src/math/generic/fabsl.cpp
-libc/src/math/generic/fdim.cpp
-libc/src/math/generic/fdimf.cpp
-libc/src/math/generic/fdiml.cpp
-libc/src/math/generic/floor.cpp
-libc/src/math/generic/floorf.cpp
-libc/src/math/generic/floorl.cpp
-libc/src/math/generic/fmax.cpp
-libc/src/math/generic/fmaxf.cpp
-libc/src/math/generic/fmaxl.cpp
-libc/src/math/generic/fmin.cpp
-libc/src/math/generic/fminf.cpp
-libc/src/math/generic/fminl.cpp
-libc/src/math/generic/frexp.cpp
-libc/src/math/generic/frexpf.cpp
-libc/src/math/generic/frexpl.cpp
-libc/src/math/generic/hypot.cpp
-libc/src/math/generic/hypotf.cpp
-libc/src/math/generic/ilogb.cpp
-libc/src/math/generic/ilogbf.cpp
-libc/src/math/generic/ilogbl.cpp
-libc/src/math/generic/ldexp.cpp
-libc/src/math/generic/ldexpf.cpp
-libc/src/math/generic/ldexpl.cpp
-libc/src/math/generic/llrint.cpp
-libc/src/math/generic/llrintf.cpp
-libc/src/math/generic/llrintl.cpp
-libc/src/math/generic/llround.cpp
-libc/src/math/generic/llroundf.cpp
-libc/src/math/generic/llroundl.cpp
-libc/src/math/generic/log10f.cpp
-libc/src/math/generic/log1pf.cpp
-libc/src/math/generic/log2f.cpp
-libc/src/math/generic/logb.cpp
-libc/src/math/generic/logbf.cpp
-libc/src/math/generic/logbl.cpp
-libc/src/math/generic/logf.cpp
-libc/src/math/generic/lrint.cpp
-libc/src/math/generic/lrintf.cpp
-libc/src/math/generic/lrintl.cpp
-libc/src/math/generic/lround.cpp
-libc/src/math/generic/lroundf.cpp
-libc/src/math/generic/lroundl.cpp
-libc/src/math/generic/math_utils.cpp
-libc/src/math/generic/math_utils.h
-libc/src/math/generic/modf.cpp
-libc/src/math/generic/modff.cpp
-libc/src/math/generic/modfl.cpp
-libc/src/math/generic/nearbyint.cpp
-libc/src/math/generic/nearbyintf.cpp
-libc/src/math/generic/nearbyintl.cpp
-libc/src/math/generic/nextafter.cpp
-libc/src/math/generic/nextafterf.cpp
-libc/src/math/generic/nextafterl.cpp
-libc/src/math/generic/remainder.cpp
-libc/src/math/generic/remainderf.cpp
-libc/src/math/generic/remainderl.cpp
-libc/src/math/generic/remquo.cpp
-libc/src/math/generic/remquof.cpp
-libc/src/math/generic/remquol.cpp
-libc/src/math/generic/rint.cpp
-libc/src/math/generic/rintf.cpp
-libc/src/math/generic/rintl.cpp
-libc/src/math/generic/round.cpp
-libc/src/math/generic/roundf.cpp
-libc/src/math/generic/roundl.cpp
-libc/src/math/generic/sincosf.cpp
-libc/src/math/generic/sincosf_data.cpp
-libc/src/math/generic/sincosf_utils.h
-libc/src/math/generic/sinf.cpp
-libc/src/math/generic/sqrt.cpp
-libc/src/math/generic/sqrtf.cpp
-libc/src/math/generic/sqrtl.cpp
-libc/src/math/generic/trunc.cpp
-libc/src/math/generic/truncf.cpp
-libc/src/math/generic/truncl.cpp
-libc/src/math/x86_64/cos.cpp
-libc/src/math/x86_64/sin.cpp
-libc/src/math/x86_64/tan.cpp
-libc/src/signal/raise.h
-libc/src/signal/sigaction.h
-libc/src/signal/sigaddset.h
-libc/src/signal/sigdelset.h
-libc/src/signal/sigemptyset.h
-libc/src/signal/sigfillset.h
-libc/src/signal/signal.h
-libc/src/signal/sigprocmask.h
-libc/src/signal/linux/raise.cpp
-libc/src/signal/linux/sigaction.cpp
-libc/src/signal/linux/sigaddset.cpp
-libc/src/signal/linux/sigdelset.cpp
-libc/src/signal/linux/sigemptyset.cpp
-libc/src/signal/linux/sigfillset.cpp
-libc/src/signal/linux/signal.cpp
-libc/src/signal/linux/signal.h
-libc/src/signal/linux/sigprocmask.cpp
-libc/src/signal/linux/__restore.cpp
-libc/src/stdio/FILE.h
-libc/src/stdio/fwrite.cpp
-libc/src/stdio/fwrite.h
-libc/src/stdlib/abort.h
-libc/src/stdlib/abs.cpp
-libc/src/stdlib/abs.h
-libc/src/stdlib/atexit.cpp
-libc/src/stdlib/atexit.h
-libc/src/stdlib/atof.cpp
-libc/src/stdlib/atof.h
-libc/src/stdlib/atoi.cpp
-libc/src/stdlib/atoi.h
-libc/src/stdlib/atol.cpp
-libc/src/stdlib/atol.h
-libc/src/stdlib/atoll.cpp
-libc/src/stdlib/atoll.h
-libc/src/stdlib/bsearch.cpp
-libc/src/stdlib/bsearch.h
-libc/src/stdlib/div.cpp
-libc/src/stdlib/div.h
-libc/src/stdlib/exit.cpp
-libc/src/stdlib/exit.h
-libc/src/stdlib/getenv.cpp
-libc/src/stdlib/getenv.h
-libc/src/stdlib/labs.cpp
-libc/src/stdlib/labs.h
-libc/src/stdlib/ldiv.cpp
-libc/src/stdlib/ldiv.h
-libc/src/stdlib/llabs.cpp
-libc/src/stdlib/llabs.h
-libc/src/stdlib/lldiv.cpp
-libc/src/stdlib/lldiv.h
-libc/src/stdlib/qsort.cpp
-libc/src/stdlib/qsort.h
-libc/src/stdlib/strtod.cpp
-libc/src/stdlib/strtod.h
-libc/src/stdlib/strtof.cpp
-libc/src/stdlib/strtof.h
-libc/src/stdlib/strtol.cpp
-libc/src/stdlib/strtol.h
-libc/src/stdlib/strtold.cpp
-libc/src/stdlib/strtold.h
-libc/src/stdlib/strtoll.cpp
-libc/src/stdlib/strtoll.h
-libc/src/stdlib/strtoul.cpp
-libc/src/stdlib/strtoul.h
-libc/src/stdlib/strtoull.cpp
-libc/src/stdlib/strtoull.h
-libc/src/stdlib/_Exit.h
-libc/src/stdlib/linux/abort.cpp
-libc/src/stdlib/linux/_Exit.cpp
-libc/src/string/bcmp.cpp
-libc/src/string/bcmp.h
-libc/src/string/bzero.cpp
-libc/src/string/bzero.h
-libc/src/string/memccpy.cpp
-libc/src/string/memccpy.h
-libc/src/string/memchr.cpp
-libc/src/string/memchr.h
-libc/src/string/memcmp.cpp
-libc/src/string/memcmp.h
-libc/src/string/memcpy.cpp
-libc/src/string/memcpy.h
-libc/src/string/memmove.cpp
-libc/src/string/memmove.h
-libc/src/string/mempcpy.cpp
-libc/src/string/mempcpy.h
-libc/src/string/memrchr.cpp
-libc/src/string/memrchr.h
-libc/src/string/memset.cpp
-libc/src/string/memset.h
-libc/src/string/stpcpy.cpp
-libc/src/string/stpcpy.h
-libc/src/string/stpncpy.cpp
-libc/src/string/stpncpy.h
-libc/src/string/strcat.cpp
-libc/src/string/strcat.h
-libc/src/string/strchr.cpp
-libc/src/string/strchr.h
-libc/src/string/strcmp.cpp
-libc/src/string/strcmp.h
-libc/src/string/strcpy.cpp
-libc/src/string/strcpy.h
-libc/src/string/strcspn.cpp
-libc/src/string/strcspn.h
-libc/src/string/strdup.cpp
-libc/src/string/strdup.h
-libc/src/string/string_utils.h
-libc/src/string/strlen.cpp
-libc/src/string/strlen.h
-libc/src/string/strncat.cpp
-libc/src/string/strncat.h
-libc/src/string/strncmp.cpp
-libc/src/string/strncmp.h
-libc/src/string/strncpy.cpp
-libc/src/string/strncpy.h
-libc/src/string/strndup.cpp
-libc/src/string/strndup.h
-libc/src/string/strnlen.cpp
-libc/src/string/strnlen.h
-libc/src/string/strpbrk.cpp
-libc/src/string/strpbrk.h
-libc/src/string/strrchr.cpp
-libc/src/string/strrchr.h
-libc/src/string/strspn.cpp
-libc/src/string/strspn.h
-libc/src/string/strstr.cpp
-libc/src/string/strstr.h
-libc/src/string/strtok.cpp
-libc/src/string/strtok.h
-libc/src/string/strtok_r.cpp
-libc/src/string/strtok_r.h
-libc/src/string/memory_utils/bcmp_implementations.h
-libc/src/string/memory_utils/elements_aarch64.h
-libc/src/string/memory_utils/elements_x86.h
-libc/src/string/memory_utils/memcmp_implementations.h
-libc/src/string/memory_utils/memcpy_implementations.h
-libc/src/string/memory_utils/memset_implementations.h
-libc/src/string/memory_utils/utils.h
-libc/src/sys/mman/mmap.h
-libc/src/sys/mman/munmap.h
-libc/src/sys/mman/linux/mmap.cpp
-libc/src/sys/stat/mkdir.h
-libc/src/sys/stat/mkdirat.h
-libc/src/sys/stat/linux/mkdir.cpp
-libc/src/sys/stat/linux/mkdirat.cpp
-libc/src/threads/call_once.h
-libc/src/threads/cnd_broadcast.h
-libc/src/threads/cnd_destroy.h
-libc/src/threads/cnd_init.h
-libc/src/threads/cnd_signal.h
-libc/src/threads/cnd_wait.h
-libc/src/threads/mtx_destroy.cpp
-libc/src/threads/mtx_destroy.h
-libc/src/threads/mtx_init.cpp
-libc/src/threads/mtx_init.h
-libc/src/threads/mtx_lock.cpp
-libc/src/threads/mtx_lock.h
-libc/src/threads/mtx_unlock.cpp
-libc/src/threads/mtx_unlock.h
-libc/src/threads/thrd_create.h
-libc/src/threads/thrd_join.h
-libc/src/threads/linux/call_once.cpp
-libc/src/threads/linux/CndVar.h
-libc/src/threads/linux/cnd_wait.cpp
-libc/src/threads/linux/Futex.h
-libc/src/threads/linux/thrd_create.cpp
-libc/src/threads/linux/thrd_join.cpp
-libc/src/threads/linux/Thread.h
-libc/src/time/asctime.cpp
-libc/src/time/asctime.h
-libc/src/time/asctime_r.cpp
-libc/src/time/asctime_r.h
-libc/src/time/gmtime.cpp
-libc/src/time/gmtime.h
-libc/src/time/gmtime_r.cpp
-libc/src/time/gmtime_r.h
-libc/src/time/mktime.cpp
-libc/src/time/mktime.h
-libc/src/time/time_utils.cpp
-libc/src/time/time_utils.h
-libc/src/unistd/close.h
-libc/src/unistd/fsync.h
-libc/src/unistd/read.h
-libc/src/unistd/rmdir.h
-libc/src/unistd/unlink.h
-libc/src/unistd/unlinkat.h
-libc/src/unistd/write.h
-libc/src/unistd/linux/close.cpp
-libc/src/unistd/linux/fsync.cpp
-libc/src/unistd/linux/read.cpp
-libc/src/unistd/linux/rmdir.cpp
-libc/src/unistd/linux/unlink.cpp
-libc/src/unistd/linux/unlinkat.cpp
-libc/src/unistd/linux/write.cpp
-libc/src/__support/architectures.h
-libc/src/__support/common.h
-libc/src/__support/ctype_utils.h
-libc/src/__support/detailed_powers_of_ten.h
-libc/src/__support/endian.h
-libc/src/__support/high_precision_decimal.h
-libc/src/__support/integer_operations.h
-libc/src/__support/sanitizer.h
-libc/src/__support/str_to_float.h
-libc/src/__support/str_to_integer.h
-libc/src/__support/CPP/Array.h
-libc/src/__support/CPP/ArrayRef.h
-libc/src/__support/CPP/atomic.h
-libc/src/__support/CPP/Bit.h
-libc/src/__support/CPP/Bitset.h
-libc/src/__support/CPP/Functional.h
-libc/src/__support/CPP/Limits.h
-libc/src/__support/CPP/Utility.h
-libc/src/__support/CPP/vector.h
-libc/src/__support/File/file.cpp
-libc/src/__support/File/file.h
-libc/src/__support/FPUtil/BasicOperations.h
-libc/src/__support/FPUtil/DivisionAndRemainderOperations.h
-libc/src/__support/FPUtil/FEnvImpl.h
-libc/src/__support/FPUtil/FloatProperties.h
-libc/src/__support/FPUtil/FMA.h
-libc/src/__support/FPUtil/FPBits.h
-libc/src/__support/FPUtil/ManipulationFunctions.h
-libc/src/__support/FPUtil/NearestIntegerOperations.h
-libc/src/__support/FPUtil/NormalFloat.h
-libc/src/__support/FPUtil/PlatformDefs.h
-libc/src/__support/FPUtil/PolyEval.h
-libc/src/__support/FPUtil/sqrt.h
-libc/src/__support/FPUtil/UInt.h
-libc/src/__support/FPUtil/XFloat.h
-libc/src/__support/FPUtil/aarch64/FEnvImpl.h
-libc/src/__support/FPUtil/aarch64/FMA.h
-libc/src/__support/FPUtil/aarch64/sqrt.h
-libc/src/__support/FPUtil/generic/FMA.h
-libc/src/__support/FPUtil/generic/sqrt.h
-libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h
-libc/src/__support/FPUtil/x86_64/FEnvImpl.h
-libc/src/__support/FPUtil/x86_64/FMA.h
-libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h
-libc/src/__support/FPUtil/x86_64/PolyEval.h
-libc/src/__support/FPUtil/x86_64/sqrt.h
-libc/src/__support/OSUtil/io.h
-libc/src/__support/OSUtil/quick_exit.h
-libc/src/__support/OSUtil/syscall.h
-libc/src/__support/OSUtil/linux/io.h
-libc/src/__support/OSUtil/linux/syscall.h
-libc/src/__support/OSUtil/linux/aarch64/syscall.h
-libc/src/__support/OSUtil/linux/x86_64/syscall.h
-libc/src/__support/threads/mutex.h
-libc/src/__support/threads/linux/mutex.h
-libc/utils/HdrGen/Command.cpp
-libc/utils/HdrGen/Command.h
-libc/utils/HdrGen/Generator.cpp
-libc/utils/HdrGen/Generator.h
-libc/utils/HdrGen/IncludeFileCommand.cpp
-libc/utils/HdrGen/IncludeFileCommand.h
-libc/utils/HdrGen/Main.cpp
-libc/utils/HdrGen/PublicAPICommand.cpp
-libc/utils/HdrGen/PublicAPICommand.h
-libc/utils/HdrGen/PrototypeTestGen/PrototypeTestGen.cpp
-libc/utils/LibcTableGenUtil/APIIndexer.cpp
-libc/utils/LibcTableGenUtil/APIIndexer.h
-libc/utils/MPFRWrapper/check_mpfr.cpp
-libc/utils/MPFRWrapper/MPFRUtils.cpp
-libc/utils/MPFRWrapper/MPFRUtils.h
-libc/utils/testutils/ExecuteFunction.h
-libc/utils/testutils/ExecuteFunctionUnix.cpp
-libc/utils/testutils/FDReader.h
-libc/utils/testutils/FDReaderUnix.cpp
-libc/utils/testutils/RandUtils.cpp
-libc/utils/testutils/RandUtils.h
-libc/utils/testutils/StreamWrapper.h
-libc/utils/testutils/Timer.cpp
-libc/utils/testutils/Timer.h
-libc/utils/tools/WrapperGen/Main.cpp
-libc/utils/UnitTest/FPExceptMatcher.cpp
-libc/utils/UnitTest/FPExceptMatcher.h
-libc/utils/UnitTest/FPMatcher.cpp
-libc/utils/UnitTest/FPMatcher.h
-libc/utils/UnitTest/FuchsiaTest.h
-libc/utils/UnitTest/LibcTest.cpp
-libc/utils/UnitTest/LibcTestMain.cpp
-libc/utils/UnitTest/MemoryMatcher.cpp
-libc/utils/UnitTest/MemoryMatcher.h
-libc/utils/UnitTest/PlatformDefs.h
-libc/utils/UnitTest/Test.h
-libclc/generic/include/config.h
-libclc/generic/include/clc/as_type.h
-libclc/generic/include/clc/clcfunc.h
-libclc/generic/include/clc/async/async_work_group_copy.h
-libclc/generic/include/clc/async/async_work_group_strided_copy.h
-libclc/generic/include/clc/async/prefetch.h
-libclc/generic/include/clc/async/wait_group_events.h
-libclc/generic/include/clc/atomic/atomic_add.h
-libclc/generic/include/clc/atomic/atomic_and.h
-libclc/generic/include/clc/atomic/atomic_max.h
-libclc/generic/include/clc/atomic/atomic_min.h
-libclc/generic/include/clc/atomic/atomic_or.h
-libclc/generic/include/clc/atomic/atomic_sub.h
-libclc/generic/include/clc/atomic/atomic_xor.h
-libclc/generic/include/clc/cl_khr_global_int32_base_atomics/atom_add.h
-libclc/generic/include/clc/cl_khr_global_int32_base_atomics/atom_dec.h
-libclc/generic/include/clc/cl_khr_global_int32_base_atomics/atom_inc.h
-libclc/generic/include/clc/cl_khr_global_int32_base_atomics/atom_sub.h
-libclc/generic/include/clc/cl_khr_global_int32_base_atomics/atom_xchg.h
-libclc/generic/include/clc/cl_khr_global_int32_extended_atomics/atom_and.h
-libclc/generic/include/clc/cl_khr_global_int32_extended_atomics/atom_max.h
-libclc/generic/include/clc/cl_khr_global_int32_extended_atomics/atom_min.h
-libclc/generic/include/clc/cl_khr_global_int32_extended_atomics/atom_or.h
-libclc/generic/include/clc/cl_khr_global_int32_extended_atomics/atom_xor.h
-libclc/generic/include/clc/cl_khr_int64_base_atomics/atom_add.h
-libclc/generic/include/clc/cl_khr_int64_base_atomics/atom_sub.h
-libclc/generic/include/clc/cl_khr_int64_base_atomics/atom_xchg.h
-libclc/generic/include/clc/cl_khr_int64_extended_atomics/atom_and.h
-libclc/generic/include/clc/cl_khr_int64_extended_atomics/atom_max.h
-libclc/generic/include/clc/cl_khr_int64_extended_atomics/atom_min.h
-libclc/generic/include/clc/cl_khr_int64_extended_atomics/atom_or.h
-libclc/generic/include/clc/cl_khr_int64_extended_atomics/atom_xor.h
-libclc/generic/include/clc/cl_khr_local_int32_base_atomics/atom_add.h
-libclc/generic/include/clc/cl_khr_local_int32_base_atomics/atom_dec.h
-libclc/generic/include/clc/cl_khr_local_int32_base_atomics/atom_inc.h
-libclc/generic/include/clc/cl_khr_local_int32_base_atomics/atom_sub.h
-libclc/generic/include/clc/cl_khr_local_int32_base_atomics/atom_xchg.h
-libclc/generic/include/clc/cl_khr_local_int32_extended_atomics/atom_and.h
-libclc/generic/include/clc/cl_khr_local_int32_extended_atomics/atom_max.h
-libclc/generic/include/clc/cl_khr_local_int32_extended_atomics/atom_min.h
-libclc/generic/include/clc/cl_khr_local_int32_extended_atomics/atom_or.h
-libclc/generic/include/clc/cl_khr_local_int32_extended_atomics/atom_xor.h
-libclc/generic/include/clc/common/degrees.h
-libclc/generic/include/clc/common/mix.h
-libclc/generic/include/clc/common/radians.h
-libclc/generic/include/clc/common/sign.h
-libclc/generic/include/clc/common/smoothstep.h
-libclc/generic/include/clc/common/step.h
-libclc/generic/include/clc/explicit_fence/explicit_memory_fence.h
-libclc/generic/include/clc/geometric/cross.h
-libclc/generic/include/clc/geometric/distance.h
-libclc/generic/include/clc/geometric/dot.h
-libclc/generic/include/clc/geometric/fast_distance.h
-libclc/generic/include/clc/geometric/fast_length.h
-libclc/generic/include/clc/geometric/fast_normalize.h
-libclc/generic/include/clc/geometric/length.h
-libclc/generic/include/clc/geometric/normalize.h
-libclc/generic/include/clc/integer/abs.h
-libclc/generic/include/clc/integer/abs_diff.h
-libclc/generic/include/clc/integer/add_sat.h
-libclc/generic/include/clc/integer/clz.h
-libclc/generic/include/clc/integer/hadd.h
-libclc/generic/include/clc/integer/mad24.h
-libclc/generic/include/clc/integer/mad_sat.h
-libclc/generic/include/clc/integer/mul24.h
-libclc/generic/include/clc/integer/mul_hi.h
-libclc/generic/include/clc/integer/popcount.h
-libclc/generic/include/clc/integer/rhadd.h
-libclc/generic/include/clc/integer/rotate.h
-libclc/generic/include/clc/integer/sub_sat.h
-libclc/generic/include/clc/math/acos.h
-libclc/generic/include/clc/math/acosh.h
-libclc/generic/include/clc/math/acospi.h
-libclc/generic/include/clc/math/asin.h
-libclc/generic/include/clc/math/asinh.h
-libclc/generic/include/clc/math/asinpi.h
-libclc/generic/include/clc/math/atan.h
-libclc/generic/include/clc/math/atan2.h
-libclc/generic/include/clc/math/atan2pi.h
-libclc/generic/include/clc/math/atanh.h
-libclc/generic/include/clc/math/atanpi.h
-libclc/generic/include/clc/math/cbrt.h
-libclc/generic/include/clc/math/ceil.h
-libclc/generic/include/clc/math/copysign.h
-libclc/generic/include/clc/math/cos.h
-libclc/generic/include/clc/math/cosh.h
-libclc/generic/include/clc/math/cospi.h
-libclc/generic/include/clc/math/erf.h
-libclc/generic/include/clc/math/erfc.h
-libclc/generic/include/clc/math/exp.h
-libclc/generic/include/clc/math/exp10.h
-libclc/generic/include/clc/math/exp2.h
-libclc/generic/include/clc/math/expm1.h
-libclc/generic/include/clc/math/fabs.h
-libclc/generic/include/clc/math/fdim.h
-libclc/generic/include/clc/math/floor.h
-libclc/generic/include/clc/math/fma.h
-libclc/generic/include/clc/math/fmod.h
-libclc/generic/include/clc/math/fract.h
-libclc/generic/include/clc/math/frexp.h
-libclc/generic/include/clc/math/half_cos.h
-libclc/generic/include/clc/math/half_divide.h
-libclc/generic/include/clc/math/half_exp.h
-libclc/generic/include/clc/math/half_exp10.h
-libclc/generic/include/clc/math/half_exp2.h
-libclc/generic/include/clc/math/half_log.h
-libclc/generic/include/clc/math/half_log10.h
-libclc/generic/include/clc/math/half_log2.h
-libclc/generic/include/clc/math/half_powr.h
-libclc/generic/include/clc/math/half_recip.h
-libclc/generic/include/clc/math/half_rsqrt.h
-libclc/generic/include/clc/math/half_sin.h
-libclc/generic/include/clc/math/half_sqrt.h
-libclc/generic/include/clc/math/half_tan.h
-libclc/generic/include/clc/math/hypot.h
-libclc/generic/include/clc/math/ilogb.h
-libclc/generic/include/clc/math/ldexp.h
-libclc/generic/include/clc/math/lgamma.h
-libclc/generic/include/clc/math/lgamma_r.h
-libclc/generic/include/clc/math/log.h
-libclc/generic/include/clc/math/log10.h
-libclc/generic/include/clc/math/log1p.h
-libclc/generic/include/clc/math/log2.h
-libclc/generic/include/clc/math/logb.h
-libclc/generic/include/clc/math/mad.h
-libclc/generic/include/clc/math/maxmag.h
-libclc/generic/include/clc/math/minmag.h
-libclc/generic/include/clc/math/modf.h
-libclc/generic/include/clc/math/native_cos.h
-libclc/generic/include/clc/math/native_divide.h
-libclc/generic/include/clc/math/native_exp.h
-libclc/generic/include/clc/math/native_exp10.h
-libclc/generic/include/clc/math/native_exp2.h
-libclc/generic/include/clc/math/native_log.h
-libclc/generic/include/clc/math/native_log10.h
-libclc/generic/include/clc/math/native_log2.h
-libclc/generic/include/clc/math/native_powr.h
-libclc/generic/include/clc/math/native_recip.h
-libclc/generic/include/clc/math/native_rsqrt.h
-libclc/generic/include/clc/math/native_sin.h
-libclc/generic/include/clc/math/native_sqrt.h
-libclc/generic/include/clc/math/native_tan.h
-libclc/generic/include/clc/math/nextafter.h
-libclc/generic/include/clc/math/pow.h
-libclc/generic/include/clc/math/pown.h
-libclc/generic/include/clc/math/powr.h
-libclc/generic/include/clc/math/remainder.h
-libclc/generic/include/clc/math/remquo.h
-libclc/generic/include/clc/math/rint.h
-libclc/generic/include/clc/math/rootn.h
-libclc/generic/include/clc/math/round.h
-libclc/generic/include/clc/math/rsqrt.h
-libclc/generic/include/clc/math/sin.h
-libclc/generic/include/clc/math/sincos.h
-libclc/generic/include/clc/math/sinh.h
-libclc/generic/include/clc/math/sinpi.h
-libclc/generic/include/clc/math/sqrt.h
-libclc/generic/include/clc/math/tan.h
-libclc/generic/include/clc/math/tanh.h
-libclc/generic/include/clc/math/tanpi.h
-libclc/generic/include/clc/math/tgamma.h
-libclc/generic/include/clc/math/trunc.h
-libclc/generic/include/clc/relational/bitselect.h
-libclc/generic/include/clc/relational/isfinite.h
-libclc/generic/include/clc/relational/isgreater.h
-libclc/generic/include/clc/relational/isgreaterequal.h
-libclc/generic/include/clc/relational/isless.h
-libclc/generic/include/clc/relational/islessequal.h
-libclc/generic/include/clc/relational/islessgreater.h
-libclc/generic/include/clc/relational/isnormal.h
-libclc/generic/include/clc/relational/isnotequal.h
-libclc/generic/include/clc/relational/isordered.h
-libclc/generic/include/clc/relational/isunordered.h
-libclc/generic/include/clc/relational/signbit.h
-libclc/generic/include/clc/shared/clamp.h
-libclc/generic/include/clc/shared/max.h
-libclc/generic/include/clc/shared/min.h
-libclc/generic/include/clc/synchronization/barrier.h
-libclc/generic/include/clc/synchronization/cl_mem_fence_flags.h
-libclc/generic/include/clc/workitem/get_global_id.h
-libclc/generic/include/clc/workitem/get_global_offset.h
-libclc/generic/include/clc/workitem/get_global_size.h
-libclc/generic/include/clc/workitem/get_group_id.h
-libclc/generic/include/clc/workitem/get_local_id.h
-libclc/generic/include/clc/workitem/get_local_size.h
-libclc/generic/include/clc/workitem/get_num_groups.h
-libclc/generic/include/clc/workitem/get_work_dim.h
-libclc/generic/include/integer/popcount.h
-libclc/generic/include/math/clc_exp10.h
-libclc/generic/include/math/clc_fma.h
-libclc/generic/include/math/clc_fmod.h
-libclc/generic/include/math/clc_hypot.h
-libclc/generic/include/math/clc_ldexp.h
-libclc/generic/include/math/clc_nextafter.h
-libclc/generic/include/math/clc_pow.h
-libclc/generic/include/math/clc_pown.h
-libclc/generic/include/math/clc_powr.h
-libclc/generic/include/math/clc_remainder.h
-libclc/generic/include/math/clc_remquo.h
-libclc/generic/include/math/clc_rootn.h
-libclc/generic/include/math/clc_sqrt.h
-libclc/generic/include/math/clc_tan.h
-libclc/generic/include/math/clc_tanpi.h
-libclc/generic/lib/math/ep_log.h
-libcxx/benchmarks/format.bench.cpp
-libcxx/benchmarks/formatted_size.bench.cpp
-libcxx/benchmarks/formatter_float.bench.cpp
-libcxx/benchmarks/format_to.bench.cpp
-libcxx/benchmarks/format_to_n.bench.cpp
-libcxx/benchmarks/to_chars.bench.cpp
-libcxx/benchmarks/util_smartptr.bench.cpp
-libcxx/benchmarks/variant_visit_1.bench.cpp
-libcxx/benchmarks/variant_visit_2.bench.cpp
-libcxx/benchmarks/variant_visit_3.bench.cpp
-libcxx/include/__algorithm/adjacent_find.h
-libcxx/include/__algorithm/all_of.h
-libcxx/include/__algorithm/any_of.h
-libcxx/include/__algorithm/count.h
-libcxx/include/__algorithm/count_if.h
-libcxx/include/__algorithm/find.h
-libcxx/include/__algorithm/find_first_of.h
-libcxx/include/__algorithm/find_if.h
-libcxx/include/__algorithm/find_if_not.h
-libcxx/include/__algorithm/for_each.h
-libcxx/include/__algorithm/for_each_n.h
-libcxx/include/__algorithm/iter_swap.h
-libcxx/include/__algorithm/mismatch.h
-libcxx/include/__algorithm/none_of.h
-libcxx/include/__algorithm/swap_ranges.h
-libcxx/include/__compare/is_eq.h
-libcxx/include/__filesystem/file_time_type.h
-libcxx/include/__filesystem/file_type.h
-libcxx/include/__filesystem/space_info.h
-libcxx/include/__format/formatter_floating_point.h
-libcxx/include/__format/formatter_pointer.h
-libcxx/include/__memory/voidify.h
-libcxx/include/__numeric/exclusive_scan.h
-libcxx/include/__numeric/inclusive_scan.h
-libcxx/include/__numeric/reduce.h
-libcxx/include/__numeric/transform_reduce.h
-libcxx/include/__random/default_random_engine.h
-libcxx/include/__random/knuth_b.h
-libcxx/include/__ranges/dangling.h
-libcxx/include/__ranges/enable_borrowed_range.h
-libcxx/include/__support/ibm/gettod_zos.h
-libcxx/include/__support/ibm/nanosleep.h
-libcxx/include/__support/openbsd/xlocale.h
-libcxx/include/__support/solaris/floatingpoint.h
-libcxx/include/__support/solaris/wchar.h
-libcxx/include/__utility/auto_cast.h
-libcxx/include/__utility/declval.h
-libcxx/include/__utility/forward.h
-libcxx/include/__utility/move.h
-libcxx/include/__utility/swap.h
-libcxx/src/chrono_system_time_init.h
-libcxx/src/format.cpp
-libcxx/src/ios.instantiations.cpp
-libcxx/src/iostream_init.h
-libcxx/src/legacy_pointer_safety.cpp
-libcxx/src/utility.cpp
-libcxx/src/experimental/memory_resource_init_helper.h
-libcxx/src/include/to_chars_floating_point.h
-libcxx/src/include/ryu/common.h
-libcxx/src/include/ryu/d2fixed.h
-libcxx/src/include/ryu/d2fixed_full_table.h
-libcxx/src/include/ryu/d2s.h
-libcxx/src/include/ryu/d2s_full_table.h
-libcxx/src/include/ryu/d2s_intrinsics.h
-libcxx/src/include/ryu/digit_table.h
-libcxx/src/include/ryu/f2s.h
-libcxx/src/ryu/d2fixed.cpp
-libcxx/src/ryu/d2s.cpp
-libcxx/src/ryu/f2s.cpp
-libcxxabi/src/cxa_guard_impl.h
-libcxxabi/src/demangle/Utility.h
-libunwind/src/cet_unwind.h
-lld/COFF/CallGraphSort.cpp
-lld/COFF/CallGraphSort.h
-lld/COFF/COFFLinkerContext.cpp
-lld/COFF/COFFLinkerContext.h
-lld/COFF/DebugTypes.cpp
-lld/COFF/DLL.h
-lld/COFF/ICF.h
-lld/COFF/MarkLive.h
-lld/COFF/MinGW.h
-lld/COFF/SymbolTable.cpp
-lld/COFF/SymbolTable.h
-lld/COFF/TypeMerger.h
-lld/COFF/Writer.h
-lld/Common/Args.cpp
-lld/Common/CommonLinkerContext.cpp
-lld/Common/DWARF.cpp
-lld/Common/Memory.cpp
-lld/Common/Reproduce.cpp
-lld/Common/Strings.cpp
-lld/Common/TargetOptionsCommandFlags.cpp
-lld/Common/Timer.cpp
-lld/Common/Version.cpp
-lld/ELF/AArch64ErrataFix.h
-lld/ELF/ARMErrataFix.h
-lld/ELF/CallGraphSort.cpp
-lld/ELF/CallGraphSort.h
-lld/ELF/Driver.h
-lld/ELF/DWARF.cpp
-lld/ELF/DWARF.h
-lld/ELF/EhFrame.h
-lld/ELF/ICF.h
-lld/ELF/LinkerScript.cpp
-lld/ELF/LTO.h
-lld/ELF/MapFile.h
-lld/ELF/MarkLive.cpp
-lld/ELF/MarkLive.h
-lld/ELF/OutputSections.h
-lld/ELF/Relocations.h
-lld/ELF/ScriptLexer.cpp
-lld/ELF/ScriptLexer.h
-lld/ELF/ScriptParser.h
-lld/ELF/Symbols.cpp
-lld/ELF/Symbols.h
-lld/ELF/SymbolTable.cpp
-lld/ELF/SymbolTable.h
-lld/ELF/Target.cpp
-lld/ELF/Writer.h
-lld/ELF/Arch/AVR.cpp
-lld/ELF/Arch/MipsArchTree.cpp
-lld/ELF/Arch/MSP430.cpp
-lld/ELF/Arch/SPARCV9.cpp
-lld/include/lld/Common/Args.h
-lld/include/lld/Common/Arrays.h
-lld/include/lld/Common/CommonLinkerContext.h
-lld/include/lld/Common/Driver.h
-lld/include/lld/Common/DWARF.h
-lld/include/lld/Common/Filesystem.h
-lld/include/lld/Common/Strings.h
-lld/include/lld/Common/Timer.h
-lld/include/lld/Core/Pass.h
-lld/include/lld/Core/SharedLibraryAtom.h
-lld/include/lld/Core/UndefinedAtom.h
-lld/include/lld/Core/Writer.h
-lld/MachO/ConcatOutputSection.cpp
-lld/MachO/ConcatOutputSection.h
-lld/MachO/Config.h
-lld/MachO/Driver.cpp
-lld/MachO/Driver.h
-lld/MachO/DriverUtils.cpp
-lld/MachO/Dwarf.cpp
-lld/MachO/Dwarf.h
-lld/MachO/ExportTrie.cpp
-lld/MachO/ExportTrie.h
-lld/MachO/ICF.cpp
-lld/MachO/ICF.h
-lld/MachO/InputFiles.cpp
-lld/MachO/InputFiles.h
-lld/MachO/InputSection.cpp
-lld/MachO/InputSection.h
-lld/MachO/LTO.cpp
-lld/MachO/LTO.h
-lld/MachO/MachOStructs.h
-lld/MachO/MapFile.cpp
-lld/MachO/MapFile.h
-lld/MachO/MarkLive.cpp
-lld/MachO/MarkLive.h
-lld/MachO/ObjC.cpp
-lld/MachO/ObjC.h
-lld/MachO/OutputSection.h
-lld/MachO/OutputSegment.cpp
-lld/MachO/OutputSegment.h
-lld/MachO/Relocations.cpp
-lld/MachO/Relocations.h
-lld/MachO/SectionPriorities.cpp
-lld/MachO/SectionPriorities.h
-lld/MachO/Symbols.cpp
-lld/MachO/Symbols.h
-lld/MachO/SymbolTable.cpp
-lld/MachO/SymbolTable.h
-lld/MachO/SyntheticSections.h
-lld/MachO/Target.cpp
-lld/MachO/Target.h
-lld/MachO/UnwindInfoSection.cpp
-lld/MachO/UnwindInfoSection.h
-lld/MachO/Writer.cpp
-lld/MachO/Writer.h
-lld/MachO/Arch/ARM.cpp
-lld/MachO/Arch/ARM64.cpp
-lld/MachO/Arch/ARM64Common.cpp
-lld/MachO/Arch/ARM64Common.h
-lld/MachO/Arch/ARM64_32.cpp
-lld/MachO/Arch/X86_64.cpp
-lld/MinGW/Driver.cpp
-lld/tools/lld/lld.cpp
-lld/wasm/Config.h
-lld/wasm/InputChunks.h
-lld/wasm/InputElement.h
-lld/wasm/InputFiles.h
-lld/wasm/MapFile.cpp
-lld/wasm/MapFile.h
-lld/wasm/MarkLive.cpp
-lld/wasm/MarkLive.h
-lld/wasm/OutputSections.cpp
-lld/wasm/OutputSections.h
-lld/wasm/OutputSegment.cpp
-lld/wasm/OutputSegment.h
-lld/wasm/Relocations.h
-lld/wasm/Writer.h
-lld/wasm/WriterUtils.h
-lldb/bindings/python/python-typemaps.h
-lldb/examples/darwin/heap_find/heap/heap_find.cpp
-lldb/examples/plugins/commands/fooplugin.cpp
-lldb/examples/synthetic/bitfield/program.cpp
-lldb/include/lldb/lldb-defines.h
-lldb/include/lldb/lldb-forward.h
-lldb/include/lldb/lldb-private.h
-lldb/include/lldb/lldb-public.h
-lldb/include/lldb/lldb-versioning.h
-lldb/include/lldb/API/LLDB.h
-lldb/include/lldb/API/SBAddress.h
-lldb/include/lldb/API/SBAttachInfo.h
-lldb/include/lldb/API/SBBlock.h
-lldb/include/lldb/API/SBBroadcaster.h
-lldb/include/lldb/API/SBCommandInterpreter.h
-lldb/include/lldb/API/SBCommandInterpreterRunOptions.h
-lldb/include/lldb/API/SBCommandReturnObject.h
-lldb/include/lldb/API/SBCommunication.h
-lldb/include/lldb/API/SBCompileUnit.h
-lldb/include/lldb/API/SBData.h
-lldb/include/lldb/API/SBDebugger.h
-lldb/include/lldb/API/SBDeclaration.h
-lldb/include/lldb/API/SBEnvironment.h
-lldb/include/lldb/API/SBError.h
-lldb/include/lldb/API/SBEvent.h
-lldb/include/lldb/API/SBExecutionContext.h
-lldb/include/lldb/API/SBExpressionOptions.h
-lldb/include/lldb/API/SBFile.h
-lldb/include/lldb/API/SBFileSpec.h
-lldb/include/lldb/API/SBFileSpecList.h
-lldb/include/lldb/API/SBFunction.h
-lldb/include/lldb/API/SBHostOS.h
-lldb/include/lldb/API/SBInstruction.h
-lldb/include/lldb/API/SBLanguageRuntime.h
-lldb/include/lldb/API/SBLaunchInfo.h
-lldb/include/lldb/API/SBLineEntry.h
-lldb/include/lldb/API/SBListener.h
-lldb/include/lldb/API/SBMemoryRegionInfo.h
-lldb/include/lldb/API/SBMemoryRegionInfoList.h
-lldb/include/lldb/API/SBModule.h
-lldb/include/lldb/API/SBModuleSpec.h
-lldb/include/lldb/API/SBPlatform.h
-lldb/include/lldb/API/SBProcess.h
-lldb/include/lldb/API/SBProcessInfo.h
-lldb/include/lldb/API/SBQueue.h
-lldb/include/lldb/API/SBQueueItem.h
-lldb/include/lldb/API/SBReproducer.h
-lldb/include/lldb/API/SBSourceManager.h
-lldb/include/lldb/API/SBStream.h
-lldb/include/lldb/API/SBStringList.h
-lldb/include/lldb/API/SBSymbol.h
-lldb/include/lldb/API/SBSymbolContext.h
-lldb/include/lldb/API/SBSymbolContextList.h
-lldb/include/lldb/API/SBThread.h
-lldb/include/lldb/API/SBThreadCollection.h
-lldb/include/lldb/API/SBTrace.h
-lldb/include/lldb/API/SBType.h
-lldb/include/lldb/API/SBTypeCategory.h
-lldb/include/lldb/API/SBTypeEnumMember.h
-lldb/include/lldb/API/SBTypeFilter.h
-lldb/include/lldb/API/SBTypeFormat.h
-lldb/include/lldb/API/SBTypeNameSpecifier.h
-lldb/include/lldb/API/SBTypeSummary.h
-lldb/include/lldb/API/SBTypeSynthetic.h
-lldb/include/lldb/API/SBUnixSignals.h
-lldb/include/lldb/API/SBValue.h
-lldb/include/lldb/API/SBValueList.h
-lldb/include/lldb/API/SBVariablesOptions.h
-lldb/include/lldb/API/SBWatchpoint.h
-lldb/include/lldb/Breakpoint/BreakpointID.h
-lldb/include/lldb/Breakpoint/BreakpointList.h
-lldb/include/lldb/Breakpoint/BreakpointLocationList.h
-lldb/include/lldb/Breakpoint/BreakpointPrecondition.h
-lldb/include/lldb/Breakpoint/BreakpointResolverFileLine.h
-lldb/include/lldb/Breakpoint/BreakpointSiteList.h
-lldb/include/lldb/Breakpoint/Stoppoint.h
-lldb/include/lldb/Breakpoint/WatchpointList.h
-lldb/include/lldb/Breakpoint/WatchpointOptions.h
-lldb/include/lldb/Core/AddressResolver.h
-lldb/include/lldb/Core/AddressResolverFileLine.h
-lldb/include/lldb/Core/DataFileCache.h
-lldb/include/lldb/Core/Debugger.h
-lldb/include/lldb/Core/Declaration.h
-lldb/include/lldb/Core/DumpRegisterValue.h
-lldb/include/lldb/Core/EmulateInstruction.h
-lldb/include/lldb/Core/Highlighter.h
-lldb/include/lldb/Core/IOHandlerCursesGUI.h
-lldb/include/lldb/Core/LoadedModuleInfoList.h
-lldb/include/lldb/Core/MappedHash.h
-lldb/include/lldb/Core/ModuleChild.h
-lldb/include/lldb/Core/Opcode.h
-lldb/include/lldb/Core/PluginInterface.h
-lldb/include/lldb/Core/PluginManager.h
-lldb/include/lldb/Core/Progress.h
-lldb/include/lldb/Core/RichManglingContext.h
-lldb/include/lldb/Core/Section.h
-lldb/include/lldb/Core/SourceLocationSpec.h
-lldb/include/lldb/Core/SourceManager.h
-lldb/include/lldb/Core/StreamAsynchronousIO.h
-lldb/include/lldb/Core/StreamBuffer.h
-lldb/include/lldb/Core/StreamFile.h
-lldb/include/lldb/Core/ThreadSafeValue.h
-lldb/include/lldb/Core/UniqueCStringMap.h
-lldb/include/lldb/Core/ValueObjectConstResultCast.h
-lldb/include/lldb/Core/ValueObjectDynamicValue.h
-lldb/include/lldb/Core/ValueObjectMemory.h
-lldb/include/lldb/Core/ValueObjectRegister.h
-lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
-lldb/include/lldb/Core/ValueObjectUpdater.h
-lldb/include/lldb/DataFormatters/CXXFunctionPointer.h
-lldb/include/lldb/DataFormatters/DumpValueObjectOptions.h
-lldb/include/lldb/DataFormatters/FormattersContainer.h
-lldb/include/lldb/DataFormatters/FormattersHelpers.h
-lldb/include/lldb/DataFormatters/TypeCategory.h
-lldb/include/lldb/DataFormatters/TypeCategoryMap.h
-lldb/include/lldb/DataFormatters/TypeSummary.h
-lldb/include/lldb/DataFormatters/ValueObjectPrinter.h
-lldb/include/lldb/DataFormatters/VectorIterator.h
-lldb/include/lldb/DataFormatters/VectorType.h
-lldb/include/lldb/Expression/DWARFExpression.h
-lldb/include/lldb/Expression/DynamicCheckerFunctions.h
-lldb/include/lldb/Expression/ExpressionSourceCode.h
-lldb/include/lldb/Expression/ExpressionTypeSystemHelper.h
-lldb/include/lldb/Expression/Materializer.h
-lldb/include/lldb/Expression/REPL.h
-lldb/include/lldb/Expression/UtilityFunction.h
-lldb/include/lldb/Host/ConnectionFileDescriptor.h
-lldb/include/lldb/Host/File.h
-lldb/include/lldb/Host/FileAction.h
-lldb/include/lldb/Host/FileSystem.h
-lldb/include/lldb/Host/Host.h
-lldb/include/lldb/Host/HostGetOpt.h
-lldb/include/lldb/Host/HostInfo.h
-lldb/include/lldb/Host/HostNativeProcess.h
-lldb/include/lldb/Host/HostNativeThread.h
-lldb/include/lldb/Host/LockFile.h
-lldb/include/lldb/Host/MainLoop.h
-lldb/include/lldb/Host/MainLoopBase.h
-lldb/include/lldb/Host/Pipe.h
-lldb/include/lldb/Host/ProcessRunLock.h
-lldb/include/lldb/Host/PseudoTerminal.h
-lldb/include/lldb/Host/SafeMachO.h
-lldb/include/lldb/Host/Socket.h
-lldb/include/lldb/Host/Terminal.h
-lldb/include/lldb/Host/Time.h
-lldb/include/lldb/Host/XML.h
-lldb/include/lldb/Host/android/HostInfoAndroid.h
-lldb/include/lldb/Host/common/GetOptInc.h
-lldb/include/lldb/Host/common/NativeRegisterContext.h
-lldb/include/lldb/Host/linux/Host.h
-lldb/include/lldb/Host/linux/Ptrace.h
-lldb/include/lldb/Host/linux/Support.h
-lldb/include/lldb/Host/linux/Uio.h
-lldb/include/lldb/Host/posix/ConnectionFileDescriptorPosix.h
-lldb/include/lldb/Host/posix/Fcntl.h
-lldb/include/lldb/Host/posix/HostProcessPosix.h
-lldb/include/lldb/Host/posix/HostThreadPosix.h
-lldb/include/lldb/Host/posix/LockFilePosix.h
-lldb/include/lldb/Host/posix/PipePosix.h
-lldb/include/lldb/Host/posix/ProcessLauncherPosixFork.h
-lldb/include/lldb/Host/windows/LockFileWindows.h
-lldb/include/lldb/Host/windows/PipeWindows.h
-lldb/include/lldb/Host/windows/PosixApi.h
-lldb/include/lldb/Host/windows/windows.h
-lldb/include/lldb/Initialization/SystemInitializerCommon.h
-lldb/include/lldb/Interpreter/CommandAlias.h
-lldb/include/lldb/Interpreter/CommandCompletions.h
-lldb/include/lldb/Interpreter/CommandHistory.h
-lldb/include/lldb/Interpreter/CommandOptionValidators.h
-lldb/include/lldb/Interpreter/OptionArgParser.h
-lldb/include/lldb/Interpreter/OptionGroupArchitecture.h
-lldb/include/lldb/Interpreter/OptionGroupBoolean.h
-lldb/include/lldb/Interpreter/OptionGroupFile.h
-lldb/include/lldb/Interpreter/OptionGroupFormat.h
-lldb/include/lldb/Interpreter/OptionGroupOutputFile.h
-lldb/include/lldb/Interpreter/OptionGroupPlatform.h
-lldb/include/lldb/Interpreter/OptionGroupString.h
-lldb/include/lldb/Interpreter/OptionGroupUInt64.h
-lldb/include/lldb/Interpreter/OptionGroupUUID.h
-lldb/include/lldb/Interpreter/OptionGroupValueObjectDisplay.h
-lldb/include/lldb/Interpreter/OptionGroupWatchpoint.h
-lldb/include/lldb/Interpreter/Options.h
-lldb/include/lldb/Interpreter/OptionValueArch.h
-lldb/include/lldb/Interpreter/OptionValueArgs.h
-lldb/include/lldb/Interpreter/OptionValueArray.h
-lldb/include/lldb/Interpreter/OptionValueBoolean.h
-lldb/include/lldb/Interpreter/OptionValueChar.h
-lldb/include/lldb/Interpreter/OptionValueEnumeration.h
-lldb/include/lldb/Interpreter/OptionValueFileSpec.h
-lldb/include/lldb/Interpreter/OptionValueFileSpecList.h
-lldb/include/lldb/Interpreter/OptionValueFormatEntity.h
-lldb/include/lldb/Interpreter/OptionValueLanguage.h
-lldb/include/lldb/Interpreter/OptionValuePathMappings.h
-lldb/include/lldb/Interpreter/OptionValueRegex.h
-lldb/include/lldb/Interpreter/OptionValues.h
-lldb/include/lldb/Interpreter/OptionValueSInt64.h
-lldb/include/lldb/Interpreter/OptionValueString.h
-lldb/include/lldb/Interpreter/OptionValueUInt64.h
-lldb/include/lldb/Interpreter/Property.h
-lldb/include/lldb/Interpreter/ScriptedInterface.h
-lldb/include/lldb/Interpreter/ScriptedProcessInterface.h
-lldb/include/lldb/Symbol/ArmUnwindInfo.h
-lldb/include/lldb/Symbol/CompilerDecl.h
-lldb/include/lldb/Symbol/CompilerDeclContext.h
-lldb/include/lldb/Symbol/DeclVendor.h
-lldb/include/lldb/Symbol/Function.h
-lldb/include/lldb/Symbol/LocateSymbolFile.h
-lldb/include/lldb/Symbol/SourceModule.h
-lldb/include/lldb/Symbol/SymbolContextScope.h
-lldb/include/lldb/Symbol/SymbolVendor.h
-lldb/include/lldb/Symbol/Type.h
-lldb/include/lldb/Symbol/TypeList.h
-lldb/include/lldb/Symbol/TypeMap.h
-lldb/include/lldb/Symbol/UnwindTable.h
-lldb/include/lldb/Symbol/Variable.h
-lldb/include/lldb/Target/AppleArm64ExceptionClass.h
-lldb/include/lldb/Target/AssertFrameRecognizer.h
-lldb/include/lldb/Target/DynamicRegisterInfo.h
-lldb/include/lldb/Target/ExecutionContextScope.h
-lldb/include/lldb/Target/InstrumentationRuntime.h
-lldb/include/lldb/Target/InstrumentationRuntimeStopInfo.h
-lldb/include/lldb/Target/JITLoader.h
-lldb/include/lldb/Target/JITLoaderList.h
-lldb/include/lldb/Target/MemoryTagManager.h
-lldb/include/lldb/Target/MemoryTagMap.h
-lldb/include/lldb/Target/ModuleCache.h
-lldb/include/lldb/Target/OperatingSystem.h
-lldb/include/lldb/Target/PostMortemProcess.h
-lldb/include/lldb/Target/ProcessTrace.h
-lldb/include/lldb/Target/Queue.h
-lldb/include/lldb/Target/QueueItem.h
-lldb/include/lldb/Target/QueueList.h
-lldb/include/lldb/Target/RegisterCheckpoint.h
-lldb/include/lldb/Target/RegisterContext.h
-lldb/include/lldb/Target/RegisterNumber.h
-lldb/include/lldb/Target/Runtime.h
-lldb/include/lldb/Target/SectionLoadHistory.h
-lldb/include/lldb/Target/StackFrame.h
-lldb/include/lldb/Target/StackFrameList.h
-lldb/include/lldb/Target/StackID.h
-lldb/include/lldb/Target/Statistics.h
-lldb/include/lldb/Target/StopInfo.h
-lldb/include/lldb/Target/SystemRuntime.h
-lldb/include/lldb/Target/TargetList.h
-lldb/include/lldb/Target/ThreadCollection.h
-lldb/include/lldb/Target/ThreadList.h
-lldb/include/lldb/Target/ThreadPlan.h
-lldb/include/lldb/Target/ThreadPlanBase.h
-lldb/include/lldb/Target/ThreadPlanCallFunction.h
-lldb/include/lldb/Target/ThreadPlanCallFunctionUsingABI.h
-lldb/include/lldb/Target/ThreadPlanRunToAddress.h
-lldb/include/lldb/Target/ThreadPlanShouldStopHere.h
-lldb/include/lldb/Target/ThreadPlanStepInstruction.h
-lldb/include/lldb/Target/ThreadPlanStepOut.h
-lldb/include/lldb/Target/ThreadPlanStepOverBreakpoint.h
-lldb/include/lldb/Target/ThreadPlanStepOverRange.h
-lldb/include/lldb/Target/ThreadPlanStepThrough.h
-lldb/include/lldb/Target/ThreadPlanStepUntil.h
-lldb/include/lldb/Target/ThreadPlanTracer.h
-lldb/include/lldb/Target/Trace.h
-lldb/include/lldb/Target/TraceCursor.h
-lldb/include/lldb/Target/TraceExporter.h
-lldb/include/lldb/Target/TraceInstructionDumper.h
-lldb/include/lldb/Target/Unwind.h
-lldb/include/lldb/Target/UnwindAssembly.h
-lldb/include/lldb/Target/UnwindLLDB.h
-lldb/include/lldb/Utility/ArchSpec.h
-lldb/include/lldb/Utility/Args.h
-lldb/include/lldb/Utility/Baton.h
-lldb/include/lldb/Utility/Broadcaster.h
-lldb/include/lldb/Utility/CompletionRequest.h
-lldb/include/lldb/Utility/ConstString.h
-lldb/include/lldb/Utility/DataBuffer.h
-lldb/include/lldb/Utility/DataBufferHeap.h
-lldb/include/lldb/Utility/DataEncoder.h
-lldb/include/lldb/Utility/FileSpec.h
-lldb/include/lldb/Utility/Flags.h
-lldb/include/lldb/Utility/GDBRemote.h
-lldb/include/lldb/Utility/Instrumentation.h
-lldb/include/lldb/Utility/IOObject.h
-lldb/include/lldb/Utility/LLDBAssert.h
-lldb/include/lldb/Utility/LLDBLog.h
-lldb/include/lldb/Utility/Predicate.h
-lldb/include/lldb/Utility/ProcessInfo.h
-lldb/include/lldb/Utility/RangeMap.h
-lldb/include/lldb/Utility/RegisterValue.h
-lldb/include/lldb/Utility/RegularExpression.h
-lldb/include/lldb/Utility/Reproducer.h
-lldb/include/lldb/Utility/ReproducerProvider.h
-lldb/include/lldb/Utility/SelectHelper.h
-lldb/include/lldb/Utility/SharedCluster.h
-lldb/include/lldb/Utility/State.h
-lldb/include/lldb/Utility/StreamCallback.h
-lldb/include/lldb/Utility/StreamString.h
-lldb/include/lldb/Utility/StreamTee.h
-lldb/include/lldb/Utility/StringExtractor.h
-lldb/include/lldb/Utility/StringExtractorGDBRemote.h
-lldb/include/lldb/Utility/StringLexer.h
-lldb/include/lldb/Utility/StructuredData.h
-lldb/include/lldb/Utility/TraceGDBRemotePackets.h
-lldb/include/lldb/Utility/TraceIntelPTGDBRemotePackets.h
-lldb/include/lldb/Utility/UnimplementedError.h
-lldb/include/lldb/Utility/UriParser.h
-lldb/include/lldb/Utility/UserID.h
-lldb/include/lldb/Utility/UserIDResolver.h
-lldb/include/lldb/Utility/VASPrintf.h
-lldb/include/lldb/Utility/VMRange.h
-lldb/include/lldb/Version/Version.h
-lldb/source/API/SBAddress.cpp
-lldb/source/API/SBAttachInfo.cpp
-lldb/source/API/SBBroadcaster.cpp
-lldb/source/API/SBCommandInterpreterRunOptions.cpp
-lldb/source/API/SBCommunication.cpp
-lldb/source/API/SBCompileUnit.cpp
-lldb/source/API/SBDebugger.cpp
-lldb/source/API/SBEnvironment.cpp
-lldb/source/API/SBFile.cpp
-lldb/source/API/SBFileSpec.cpp
-lldb/source/API/SBFileSpecList.cpp
-lldb/source/API/SBFunction.cpp
-lldb/source/API/SBHostOS.cpp
-lldb/source/API/SBLanguageRuntime.cpp
-lldb/source/API/SBLaunchInfo.cpp
-lldb/source/API/SBLineEntry.cpp
-lldb/source/API/SBListener.cpp
-lldb/source/API/SBModule.cpp
-lldb/source/API/SBModuleSpec.cpp
-lldb/source/API/SBProcessInfo.cpp
-lldb/source/API/SBQueueItem.cpp
-lldb/source/API/SBSection.cpp
-lldb/source/API/SBStream.cpp
-lldb/source/API/SBStringList.cpp
-lldb/source/API/SBSymbol.cpp
-lldb/source/API/SBSymbolContext.cpp
-lldb/source/API/SBThreadPlan.cpp
-lldb/source/API/SBTrace.cpp
-lldb/source/API/SBTypeFilter.cpp
-lldb/source/API/SBTypeFormat.cpp
-lldb/source/API/SBUnixSignals.cpp
-lldb/source/API/SBValueList.cpp
-lldb/source/API/SBWatchpoint.cpp
-lldb/source/API/SystemInitializerFull.cpp
-lldb/source/API/SystemInitializerFull.h
-lldb/source/API/Utils.h
-lldb/source/Breakpoint/BreakpointList.cpp
-lldb/source/Breakpoint/BreakpointPrecondition.cpp
-lldb/source/Breakpoint/BreakpointResolverAddress.cpp
-lldb/source/Breakpoint/BreakpointSiteList.cpp
-lldb/source/Breakpoint/StoppointCallbackContext.cpp
-lldb/source/Breakpoint/WatchpointList.cpp
-lldb/source/Commands/CommandObjectApropos.cpp
-lldb/source/Commands/CommandObjectApropos.h
-lldb/source/Commands/CommandObjectBreakpoint.h
-lldb/source/Commands/CommandObjectBreakpointCommand.cpp
-lldb/source/Commands/CommandObjectBreakpointCommand.h
-lldb/source/Commands/CommandObjectCommands.h
-lldb/source/Commands/CommandObjectDisassemble.cpp
-lldb/source/Commands/CommandObjectDisassemble.h
-lldb/source/Commands/CommandObjectExpression.h
-lldb/source/Commands/CommandObjectFrame.cpp
-lldb/source/Commands/CommandObjectFrame.h
-lldb/source/Commands/CommandObjectGUI.cpp
-lldb/source/Commands/CommandObjectGUI.h
-lldb/source/Commands/CommandObjectHelp.cpp
-lldb/source/Commands/CommandObjectHelp.h
-lldb/source/Commands/CommandObjectLanguage.h
-lldb/source/Commands/CommandObjectLog.cpp
-lldb/source/Commands/CommandObjectLog.h
-lldb/source/Commands/CommandObjectMemory.cpp
-lldb/source/Commands/CommandObjectMemory.h
-lldb/source/Commands/CommandObjectMemoryTag.cpp
-lldb/source/Commands/CommandObjectMemoryTag.h
-lldb/source/Commands/CommandObjectPlatform.h
-lldb/source/Commands/CommandObjectPlugin.cpp
-lldb/source/Commands/CommandObjectPlugin.h
-lldb/source/Commands/CommandObjectProcess.h
-lldb/source/Commands/CommandObjectQuit.cpp
-lldb/source/Commands/CommandObjectQuit.h
-lldb/source/Commands/CommandObjectRegexCommand.cpp
-lldb/source/Commands/CommandObjectRegexCommand.h
-lldb/source/Commands/CommandObjectRegister.cpp
-lldb/source/Commands/CommandObjectRegister.h
-lldb/source/Commands/CommandObjectReproducer.cpp
-lldb/source/Commands/CommandObjectReproducer.h
-lldb/source/Commands/CommandObjectScript.cpp
-lldb/source/Commands/CommandObjectScript.h
-lldb/source/Commands/CommandObjectSession.cpp
-lldb/source/Commands/CommandObjectSession.h
-lldb/source/Commands/CommandObjectSettings.h
-lldb/source/Commands/CommandObjectSource.h
-lldb/source/Commands/CommandObjectStats.cpp
-lldb/source/Commands/CommandObjectStats.h
-lldb/source/Commands/CommandObjectTarget.h
-lldb/source/Commands/CommandObjectThread.h
-lldb/source/Commands/CommandObjectThreadUtil.cpp
-lldb/source/Commands/CommandObjectThreadUtil.h
-lldb/source/Commands/CommandObjectTrace.cpp
-lldb/source/Commands/CommandObjectTrace.h
-lldb/source/Commands/CommandObjectType.cpp
-lldb/source/Commands/CommandObjectType.h
-lldb/source/Commands/CommandObjectVersion.cpp
-lldb/source/Commands/CommandObjectVersion.h
-lldb/source/Commands/CommandObjectWatchpoint.cpp
-lldb/source/Commands/CommandObjectWatchpoint.h
-lldb/source/Commands/CommandObjectWatchpointCommand.h
-lldb/source/Commands/CommandOptionsProcessLaunch.cpp
-lldb/source/Commands/CommandOptionsProcessLaunch.h
-lldb/source/Core/AddressRange.cpp
-lldb/source/Core/AddressResolver.cpp
-lldb/source/Core/AddressResolverFileLine.cpp
-lldb/source/Core/Communication.cpp
-lldb/source/Core/Declaration.cpp
-lldb/source/Core/DumpDataExtractor.cpp
-lldb/source/Core/DumpRegisterValue.cpp
-lldb/source/Core/EmulateInstruction.cpp
-lldb/source/Core/FileLineResolver.cpp
-lldb/source/Core/FileSpecList.cpp
-lldb/source/Core/FormatEntity.cpp
-lldb/source/Core/Highlighter.cpp
-lldb/source/Core/IOHandler.cpp
-lldb/source/Core/IOHandlerCursesGUI.cpp
-lldb/source/Core/ModuleChild.cpp
-lldb/source/Core/Opcode.cpp
-lldb/source/Core/Progress.cpp
-lldb/source/Core/RichManglingContext.cpp
-lldb/source/Core/SourceLocationSpec.cpp
-lldb/source/Core/StreamAsynchronousIO.cpp
-lldb/source/Core/StreamFile.cpp
-lldb/source/Core/ValueObjectConstResultCast.cpp
-lldb/source/Core/ValueObjectDynamicValue.cpp
-lldb/source/Core/ValueObjectList.cpp
-lldb/source/Core/ValueObjectMemory.cpp
-lldb/source/Core/ValueObjectUpdater.cpp
-lldb/source/DataFormatters/CXXFunctionPointer.cpp
-lldb/source/DataFormatters/DumpValueObjectOptions.cpp
-lldb/source/DataFormatters/ValueObjectPrinter.cpp
-lldb/source/Expression/DiagnosticManager.cpp
-lldb/source/Expression/Expression.cpp
-lldb/source/Expression/ExpressionVariable.cpp
-lldb/source/Expression/REPL.cpp
-lldb/source/Expression/UtilityFunction.cpp
-lldb/source/Host/android/HostInfoAndroid.cpp
-lldb/source/Host/android/LibcGlue.cpp
-lldb/source/Host/common/FileAction.cpp
-lldb/source/Host/common/FileCache.cpp
-lldb/source/Host/common/FileSystem.cpp
-lldb/source/Host/common/HostNativeThreadBase.cpp
-lldb/source/Host/common/HostProcess.cpp
-lldb/source/Host/common/HostThread.cpp
-lldb/source/Host/common/LockFileBase.cpp
-lldb/source/Host/common/LZMA.cpp
-lldb/source/Host/common/MonitoringProcessLauncher.cpp
-lldb/source/Host/common/NativeRegisterContext.cpp
-lldb/source/Host/common/NativeThreadProtocol.cpp
-lldb/source/Host/common/NativeWatchpointList.cpp
-lldb/source/Host/common/OptionParser.cpp
-lldb/source/Host/common/PipeBase.cpp
-lldb/source/Host/common/PseudoTerminal.cpp
-lldb/source/Host/common/ThreadLauncher.cpp
-lldb/source/Host/freebsd/Host.cpp
-lldb/source/Host/freebsd/HostInfoFreeBSD.cpp
-lldb/source/Host/linux/AbstractSocket.cpp
-lldb/source/Host/linux/Host.cpp
-lldb/source/Host/linux/HostInfoLinux.cpp
-lldb/source/Host/linux/LibcGlue.cpp
-lldb/source/Host/linux/Support.cpp
-lldb/source/Host/macosx/cfcpp/CFCBundle.cpp
-lldb/source/Host/macosx/cfcpp/CFCBundle.h
-lldb/source/Host/macosx/cfcpp/CFCData.cpp
-lldb/source/Host/macosx/cfcpp/CFCData.h
-lldb/source/Host/macosx/cfcpp/CFCMutableArray.cpp
-lldb/source/Host/macosx/cfcpp/CFCMutableArray.h
-lldb/source/Host/macosx/cfcpp/CFCMutableDictionary.h
-lldb/source/Host/macosx/cfcpp/CFCMutableSet.h
-lldb/source/Host/macosx/cfcpp/CFCReleaser.h
-lldb/source/Host/macosx/cfcpp/CFCString.cpp
-lldb/source/Host/macosx/cfcpp/CFCString.h
-lldb/source/Host/macosx/cfcpp/CoreFoundationCPP.h
-lldb/source/Host/macosx/objcxx/PosixSpawnResponsible.h
-lldb/source/Host/openbsd/HostInfoOpenBSD.cpp
-lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
-lldb/source/Host/posix/FileSystemPosix.cpp
-lldb/source/Host/posix/HostInfoPosix.cpp
-lldb/source/Host/posix/HostThreadPosix.cpp
-lldb/source/Host/posix/LockFilePosix.cpp
-lldb/source/Host/posix/PipePosix.cpp
-lldb/source/Host/windows/FileSystem.cpp
-lldb/source/Host/windows/Host.cpp
-lldb/source/Host/windows/HostInfoWindows.cpp
-lldb/source/Host/windows/HostThreadWindows.cpp
-lldb/source/Host/windows/LockFileWindows.cpp
-lldb/source/Host/windows/ProcessLauncherWindows.cpp
-lldb/source/Host/windows/ProcessRunLock.cpp
-lldb/source/Initialization/SystemInitializer.cpp
-lldb/source/Initialization/SystemInitializerCommon.cpp
-lldb/source/Initialization/SystemLifetimeManager.cpp
-lldb/source/Interpreter/CommandAlias.cpp
-lldb/source/Interpreter/CommandHistory.cpp
-lldb/source/Interpreter/CommandOptionValidators.cpp
-lldb/source/Interpreter/CommandReturnObject.cpp
-lldb/source/Interpreter/OptionGroupBoolean.cpp
-lldb/source/Interpreter/OptionGroupFile.cpp
-lldb/source/Interpreter/OptionGroupString.cpp
-lldb/source/Interpreter/OptionGroupUInt64.cpp
-lldb/source/Interpreter/OptionValue.cpp
-lldb/source/Interpreter/OptionValueArch.cpp
-lldb/source/Interpreter/OptionValueArgs.cpp
-lldb/source/Interpreter/OptionValueBoolean.cpp
-lldb/source/Interpreter/OptionValueChar.cpp
-lldb/source/Interpreter/OptionValueFileColonLine.cpp
-lldb/source/Interpreter/OptionValueFileSpecList.cpp
-lldb/source/Interpreter/OptionValueFormat.cpp
-lldb/source/Interpreter/OptionValueFormatEntity.cpp
-lldb/source/Interpreter/OptionValuePathMappings.cpp
-lldb/source/Interpreter/OptionValueRegex.cpp
-lldb/source/Interpreter/OptionValueSInt64.cpp
-lldb/source/Interpreter/OptionValueString.cpp
-lldb/source/Interpreter/OptionValueUInt64.cpp
-lldb/source/Interpreter/OptionValueUUID.cpp
-lldb/source/Interpreter/ScriptInterpreter.cpp
-lldb/source/Plugins/ABI/AArch64/ABIAArch64.cpp
-lldb/source/Plugins/ABI/AArch64/ABIAArch64.h
-lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp
-lldb/source/Plugins/ABI/ARM/ABIARM.cpp
-lldb/source/Plugins/ABI/ARM/ABIARM.h
-lldb/source/Plugins/ABI/Mips/ABIMips.cpp
-lldb/source/Plugins/ABI/Mips/ABIMips.h
-lldb/source/Plugins/ABI/PowerPC/ABIPowerPC.cpp
-lldb/source/Plugins/ABI/PowerPC/ABIPowerPC.h
-lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp
-lldb/source/Plugins/ABI/X86/ABIX86.h
-lldb/source/Plugins/ABI/X86/ABIX86_64.h
-lldb/source/Plugins/ABI/X86/ABIX86_i386.cpp
-lldb/source/Plugins/ABI/X86/ABIX86_i386.h
-lldb/source/Plugins/Architecture/AArch64/ArchitectureAArch64.cpp
-lldb/source/Plugins/Architecture/AArch64/ArchitectureAArch64.h
-lldb/source/Plugins/Architecture/Arm/ArchitectureArm.h
-lldb/source/Plugins/Architecture/PPC64/ArchitecturePPC64.cpp
-lldb/source/Plugins/Architecture/PPC64/ArchitecturePPC64.h
-lldb/source/Plugins/Disassembler/LLVMC/DisassemblerLLVMC.h
-lldb/source/Plugins/DynamicLoader/Hexagon-DYLD/DynamicLoaderHexagonDYLD.cpp
-lldb/source/Plugins/DynamicLoader/Hexagon-DYLD/DynamicLoaderHexagonDYLD.h
-lldb/source/Plugins/DynamicLoader/Hexagon-DYLD/HexagonDYLDRendezvous.h
-lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.h
-lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOS.h
-lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.h
-lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DYLDRendezvous.h
-lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.h
-lldb/source/Plugins/DynamicLoader/Static/DynamicLoaderStatic.h
-lldb/source/Plugins/DynamicLoader/wasm-DYLD/DynamicLoaderWasmDYLD.cpp
-lldb/source/Plugins/DynamicLoader/wasm-DYLD/DynamicLoaderWasmDYLD.h
-lldb/source/Plugins/DynamicLoader/Windows-DYLD/DynamicLoaderWindowsDYLD.h
-lldb/source/Plugins/ExpressionParser/Clang/ASTResultSynthesizer.cpp
-lldb/source/Plugins/ExpressionParser/Clang/ASTStructExtractor.cpp
-lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.cpp
-lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.h
-lldb/source/Plugins/ExpressionParser/Clang/ClangASTMetadata.cpp
-lldb/source/Plugins/ExpressionParser/Clang/ClangASTMetadata.h
-lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.h
-lldb/source/Plugins/ExpressionParser/Clang/ClangDeclVendor.cpp
-lldb/source/Plugins/ExpressionParser/Clang/ClangDeclVendor.h
-lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionHelper.h
-lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionVariable.cpp
-lldb/source/Plugins/ExpressionParser/Clang/ClangExternalASTSourceCallbacks.cpp
-lldb/source/Plugins/ExpressionParser/Clang/ClangExternalASTSourceCallbacks.h
-lldb/source/Plugins/ExpressionParser/Clang/ClangFunctionCaller.cpp
-lldb/source/Plugins/ExpressionParser/Clang/ClangHost.h
-lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.h
-lldb/source/Plugins/ExpressionParser/Clang/ClangPersistentVariables.cpp
-lldb/source/Plugins/ExpressionParser/Clang/ClangUtil.cpp
-lldb/source/Plugins/ExpressionParser/Clang/ClangUtilityFunction.h
-lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.cpp
-lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.h
-lldb/source/Plugins/ExpressionParser/Clang/CxxModuleHandler.h
-lldb/source/Plugins/ExpressionParser/Clang/IRDynamicChecks.cpp
-lldb/source/Plugins/ExpressionParser/Clang/ModuleDependencyCollector.h
-lldb/source/Plugins/ExpressionParser/Clang/NameSearchContext.h
-lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.h
-lldb/source/Plugins/Instruction/ARM/EmulationStateARM.h
-lldb/source/Plugins/Instruction/MIPS64/EmulateInstructionMIPS64.h
-lldb/source/Plugins/Instruction/PPC64/EmulateInstructionPPC64.cpp
-lldb/source/Plugins/Instruction/PPC64/EmulateInstructionPPC64.h
-lldb/source/Plugins/InstrumentationRuntime/ASan/InstrumentationRuntimeASan.cpp
-lldb/source/Plugins/InstrumentationRuntime/ASan/InstrumentationRuntimeASan.h
-lldb/source/Plugins/InstrumentationRuntime/MainThreadChecker/InstrumentationRuntimeMainThreadChecker.cpp
-lldb/source/Plugins/InstrumentationRuntime/MainThreadChecker/InstrumentationRuntimeMainThreadChecker.h
-lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
-lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.h
-lldb/source/Plugins/InstrumentationRuntime/UBSan/InstrumentationRuntimeUBSan.cpp
-lldb/source/Plugins/InstrumentationRuntime/UBSan/InstrumentationRuntimeUBSan.h
-lldb/source/Plugins/JITLoader/GDB/JITLoaderGDB.h
-lldb/source/Plugins/Language/ClangCommon/ClangHighlighter.cpp
-lldb/source/Plugins/Language/ClangCommon/ClangHighlighter.h
-lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
-lldb/source/Plugins/Language/CPlusPlus/BlockPointer.h
-lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
-lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
-lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.h
-lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.h
-lldb/source/Plugins/Language/CPlusPlus/CxxStringTypes.h
-lldb/source/Plugins/Language/CPlusPlus/Generic.h
-lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
-lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
-lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
-lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
-lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.h
-lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
-lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
-lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
-lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
-lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.h
-lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h
-lldb/source/Plugins/Language/CPlusPlus/MSVCUndecoratedNameParser.cpp
-lldb/source/Plugins/Language/CPlusPlus/MSVCUndecoratedNameParser.h
-lldb/source/Plugins/Language/ObjC/CF.cpp
-lldb/source/Plugins/Language/ObjC/CF.h
-lldb/source/Plugins/Language/ObjC/CFBasicHash.h
-lldb/source/Plugins/Language/ObjC/Cocoa.h
-lldb/source/Plugins/Language/ObjC/CoreMedia.cpp
-lldb/source/Plugins/Language/ObjC/CoreMedia.h
-lldb/source/Plugins/Language/ObjC/NSDictionary.h
-lldb/source/Plugins/Language/ObjC/NSError.cpp
-lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
-lldb/source/Plugins/Language/ObjC/NSSet.h
-lldb/source/Plugins/Language/ObjC/NSString.cpp
-lldb/source/Plugins/Language/ObjC/NSString.h
-lldb/source/Plugins/Language/ObjC/ObjCConstants.h
-lldb/source/Plugins/Language/ObjC/ObjCLanguage.h
-lldb/source/Plugins/Language/ObjCPlusPlus/ObjCPlusPlusLanguage.cpp
-lldb/source/Plugins/Language/ObjCPlusPlus/ObjCPlusPlusLanguage.h
-lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCClassDescriptorV2.h
-lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCDeclVendor.h
-lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h
-lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTypeEncodingParser.cpp
-lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTypeEncodingParser.h
-lldb/source/Plugins/LanguageRuntime/RenderScript/RenderScriptRuntime/RenderScriptExpressionOpts.cpp
-lldb/source/Plugins/LanguageRuntime/RenderScript/RenderScriptRuntime/RenderScriptExpressionOpts.h
-lldb/source/Plugins/LanguageRuntime/RenderScript/RenderScriptRuntime/RenderScriptScriptGroup.h
-lldb/source/Plugins/MemoryHistory/asan/MemoryHistoryASan.cpp
-lldb/source/Plugins/MemoryHistory/asan/MemoryHistoryASan.h
-lldb/source/Plugins/ObjectContainer/Universal-Mach-O/ObjectContainerUniversalMachO.cpp
-lldb/source/Plugins/ObjectContainer/Universal-Mach-O/ObjectContainerUniversalMachO.h
-lldb/source/Plugins/ObjectFile/Breakpad/BreakpadRecords.h
-lldb/source/Plugins/ObjectFile/Breakpad/ObjectFileBreakpad.cpp
-lldb/source/Plugins/ObjectFile/Breakpad/ObjectFileBreakpad.h
-lldb/source/Plugins/ObjectFile/ELF/ELFHeader.cpp
-lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
-lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.h
-lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.cpp
-lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.h
-lldb/source/Plugins/ObjectFile/PDB/ObjectFilePDB.cpp
-lldb/source/Plugins/ObjectFile/PDB/ObjectFilePDB.h
-lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h
-lldb/source/Plugins/ObjectFile/PECOFF/PECallFrameInfo.h
-lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.h
-lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp
-lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.h
-lldb/source/Plugins/OperatingSystem/Python/OperatingSystemPython.cpp
-lldb/source/Plugins/OperatingSystem/Python/OperatingSystemPython.h
-lldb/source/Plugins/Platform/Android/AdbClient.h
-lldb/source/Plugins/Platform/Android/PlatformAndroidRemoteGDBServer.cpp
-lldb/source/Plugins/Platform/Android/PlatformAndroidRemoteGDBServer.h
-lldb/source/Plugins/Platform/FreeBSD/PlatformFreeBSD.h
-lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.h
-lldb/source/Plugins/Platform/Linux/PlatformLinux.h
-lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.h
-lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.h
-lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.h
-lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.cpp
-lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.h
-lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleTV.h
-lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleWatch.cpp
-lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleWatch.h
-lldb/source/Plugins/Platform/MacOSX/PlatformRemoteiOS.h
-lldb/source/Plugins/Platform/MacOSX/PlatformRemoteMacOSX.cpp
-lldb/source/Plugins/Platform/MacOSX/PlatformRemoteMacOSX.h
-lldb/source/Plugins/Platform/MacOSX/objcxx/PlatformiOSSimulatorCoreSimulatorSupport.h
-lldb/source/Plugins/Platform/NetBSD/PlatformNetBSD.h
-lldb/source/Plugins/Platform/OpenBSD/PlatformOpenBSD.h
-lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.cpp
-lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.h
-lldb/source/Plugins/Platform/Windows/PlatformWindows.h
-lldb/source/Plugins/Process/elf-core/ProcessElfCore.h
-lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm.cpp
-lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm.h
-lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp
-lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h
-lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_mips64.h
-lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_powerpc.cpp
-lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_powerpc.h
-lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_ppc64le.cpp
-lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_ppc64le.h
-lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_s390x.cpp
-lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_s390x.h
-lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_x86_64.cpp
-lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_x86_64.h
-lldb/source/Plugins/Process/elf-core/RegisterUtilities.cpp
-lldb/source/Plugins/Process/elf-core/RegisterUtilities.h
-lldb/source/Plugins/Process/elf-core/ThreadElfCore.cpp
-lldb/source/Plugins/Process/elf-core/ThreadElfCore.h
-lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD.cpp
-lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD.h
-lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm.cpp
-lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm.h
-lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm64.cpp
-lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_mips64.cpp
-lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_mips64.h
-lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_powerpc.cpp
-lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_powerpc.h
-lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_x86_64.cpp
-lldb/source/Plugins/Process/FreeBSD/NativeThreadFreeBSD.cpp
-lldb/source/Plugins/Process/FreeBSD/NativeThreadFreeBSD.h
-lldb/source/Plugins/Process/FreeBSDKernel/ProcessFreeBSDKernel.cpp
-lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_arm64.cpp
-lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_arm64.h
-lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_i386.h
-lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_x86_64.cpp
-lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_x86_64.h
-lldb/source/Plugins/Process/FreeBSDKernel/ThreadFreeBSDKernel.cpp
-lldb/source/Plugins/Process/FreeBSDKernel/ThreadFreeBSDKernel.h
-lldb/source/Plugins/Process/gdb-remote/GDBRemoteClientBase.h
-lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.h
-lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationHistory.h
-lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.cpp
-lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.h
-lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.h
-lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.h
-lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
-lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h
-lldb/source/Plugins/Process/gdb-remote/GDBRemoteRegisterContext.h
-lldb/source/Plugins/Process/gdb-remote/GDBRemoteRegisterFallback.cpp
-lldb/source/Plugins/Process/gdb-remote/GDBRemoteRegisterFallback.h
-lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemoteLog.h
-lldb/source/Plugins/Process/gdb-remote/ThreadGDBRemote.cpp
-lldb/source/Plugins/Process/gdb-remote/ThreadGDBRemote.h
-lldb/source/Plugins/Process/Linux/IntelPTManager.cpp
-lldb/source/Plugins/Process/Linux/IntelPTManager.h
-lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux.cpp
-lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm.h
-lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h
-lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_ppc64le.h
-lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_s390x.h
-lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_x86_64.h
-lldb/source/Plugins/Process/Linux/NativeThreadLinux.h
-lldb/source/Plugins/Process/Linux/Procfs.h
-lldb/source/Plugins/Process/Linux/SingleStepCheck.h
-lldb/source/Plugins/Process/mach-core/ProcessMachCore.h
-lldb/source/Plugins/Process/mach-core/ThreadMachCore.cpp
-lldb/source/Plugins/Process/mach-core/ThreadMachCore.h
-lldb/source/Plugins/Process/MacOSX-Kernel/CommunicationKDP.h
-lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.h
-lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDPLog.cpp
-lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_arm.cpp
-lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_arm.h
-lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_arm64.cpp
-lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_arm64.h
-lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.cpp
-lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.h
-lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_x86_64.cpp
-lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_x86_64.h
-lldb/source/Plugins/Process/MacOSX-Kernel/ThreadKDP.cpp
-lldb/source/Plugins/Process/MacOSX-Kernel/ThreadKDP.h
-lldb/source/Plugins/Process/minidump/MinidumpParser.h
-lldb/source/Plugins/Process/minidump/MinidumpTypes.cpp
-lldb/source/Plugins/Process/minidump/NtStructures.h
-lldb/source/Plugins/Process/minidump/RegisterContextMinidump_ARM.h
-lldb/source/Plugins/Process/minidump/RegisterContextMinidump_ARM64.h
-lldb/source/Plugins/Process/minidump/RegisterContextMinidump_x86_32.cpp
-lldb/source/Plugins/Process/minidump/RegisterContextMinidump_x86_32.h
-lldb/source/Plugins/Process/minidump/RegisterContextMinidump_x86_64.cpp
-lldb/source/Plugins/Process/minidump/RegisterContextMinidump_x86_64.h
-lldb/source/Plugins/Process/minidump/ThreadMinidump.cpp
-lldb/source/Plugins/Process/NetBSD/NativeProcessNetBSD.cpp
-lldb/source/Plugins/Process/NetBSD/NativeRegisterContextNetBSD.cpp
-lldb/source/Plugins/Process/NetBSD/NativeRegisterContextNetBSD.h
-lldb/source/Plugins/Process/NetBSD/NativeRegisterContextNetBSD_x86_64.cpp
-lldb/source/Plugins/Process/POSIX/CrashReason.h
-lldb/source/Plugins/Process/POSIX/NativeProcessELF.cpp
-lldb/source/Plugins/Process/POSIX/NativeProcessELF.h
-lldb/source/Plugins/Process/POSIX/ProcessMessage.cpp
-lldb/source/Plugins/Process/POSIX/ProcessMessage.h
-lldb/source/Plugins/Process/POSIX/ProcessPOSIXLog.cpp
-lldb/source/Plugins/Process/POSIX/ProcessPOSIXLog.h
-lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp
-lldb/source/Plugins/Process/scripted/ScriptedProcess.h
-lldb/source/Plugins/Process/scripted/ScriptedThread.cpp
-lldb/source/Plugins/Process/scripted/ScriptedThread.h
-lldb/source/Plugins/Process/Utility/ARMDefines.h
-lldb/source/Plugins/Process/Utility/ARMUtils.h
-lldb/source/Plugins/Process/Utility/AuxVector.cpp
-lldb/source/Plugins/Process/Utility/AuxVector.h
-lldb/source/Plugins/Process/Utility/FreeBSDSignals.cpp
-lldb/source/Plugins/Process/Utility/FreeBSDSignals.h
-lldb/source/Plugins/Process/Utility/GDBRemoteSignals.cpp
-lldb/source/Plugins/Process/Utility/GDBRemoteSignals.h
-lldb/source/Plugins/Process/Utility/HistoryThread.cpp
-lldb/source/Plugins/Process/Utility/HistoryThread.h
-lldb/source/Plugins/Process/Utility/HistoryUnwind.cpp
-lldb/source/Plugins/Process/Utility/HistoryUnwind.h
-lldb/source/Plugins/Process/Utility/InferiorCallPOSIX.cpp
-lldb/source/Plugins/Process/Utility/InferiorCallPOSIX.h
-lldb/source/Plugins/Process/Utility/InstructionUtils.h
-lldb/source/Plugins/Process/Utility/LinuxProcMaps.cpp
-lldb/source/Plugins/Process/Utility/LinuxProcMaps.h
-lldb/source/Plugins/Process/Utility/LinuxSignals.cpp
-lldb/source/Plugins/Process/Utility/LinuxSignals.h
-lldb/source/Plugins/Process/Utility/lldb-mips-freebsd-register-enums.h
-lldb/source/Plugins/Process/Utility/lldb-ppc64-register-enums.h
-lldb/source/Plugins/Process/Utility/lldb-ppc64le-register-enums.h
-lldb/source/Plugins/Process/Utility/MemoryTagManagerAArch64MTE.cpp
-lldb/source/Plugins/Process/Utility/MemoryTagManagerAArch64MTE.h
-lldb/source/Plugins/Process/Utility/MipsLinuxSignals.cpp
-lldb/source/Plugins/Process/Utility/MipsLinuxSignals.h
-lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.cpp
-lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.h
-lldb/source/Plugins/Process/Utility/NativeRegisterContextDBReg_arm64.cpp
-lldb/source/Plugins/Process/Utility/NativeRegisterContextDBReg_arm64.h
-lldb/source/Plugins/Process/Utility/NativeRegisterContextDBReg_x86.cpp
-lldb/source/Plugins/Process/Utility/NativeRegisterContextDBReg_x86.h
-lldb/source/Plugins/Process/Utility/NativeRegisterContextRegisterInfo.cpp
-lldb/source/Plugins/Process/Utility/NetBSDSignals.cpp
-lldb/source/Plugins/Process/Utility/NetBSDSignals.h
-lldb/source/Plugins/Process/Utility/RegisterContextDarwinConstants.h
-lldb/source/Plugins/Process/Utility/RegisterContextDarwin_arm.h
-lldb/source/Plugins/Process/Utility/RegisterContextDarwin_arm64.h
-lldb/source/Plugins/Process/Utility/RegisterContextDarwin_i386.h
-lldb/source/Plugins/Process/Utility/RegisterContextDarwin_x86_64.h
-lldb/source/Plugins/Process/Utility/RegisterContextDummy.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextDummy.h
-lldb/source/Plugins/Process/Utility/RegisterContextFreeBSD_i386.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextFreeBSD_i386.h
-lldb/source/Plugins/Process/Utility/RegisterContextFreeBSD_mips64.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextFreeBSD_mips64.h
-lldb/source/Plugins/Process/Utility/RegisterContextFreeBSD_powerpc.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextFreeBSD_powerpc.h
-lldb/source/Plugins/Process/Utility/RegisterContextFreeBSD_x86_64.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextFreeBSD_x86_64.h
-lldb/source/Plugins/Process/Utility/RegisterContextHistory.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextHistory.h
-lldb/source/Plugins/Process/Utility/RegisterContextLinux_i386.h
-lldb/source/Plugins/Process/Utility/RegisterContextLinux_s390x.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextLinux_s390x.h
-lldb/source/Plugins/Process/Utility/RegisterContextLinux_x86_64.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextLinux_x86_64.h
-lldb/source/Plugins/Process/Utility/RegisterContextMach_arm.h
-lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.h
-lldb/source/Plugins/Process/Utility/RegisterContextMach_x86_64.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextMach_x86_64.h
-lldb/source/Plugins/Process/Utility/RegisterContextMemory.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextMemory.h
-lldb/source/Plugins/Process/Utility/RegisterContextNetBSD_i386.h
-lldb/source/Plugins/Process/Utility/RegisterContextNetBSD_x86_64.h
-lldb/source/Plugins/Process/Utility/RegisterContextOpenBSD_i386.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextOpenBSD_i386.h
-lldb/source/Plugins/Process/Utility/RegisterContextOpenBSD_x86_64.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextOpenBSD_x86_64.h
-lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm.h
-lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.h
-lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_powerpc.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_powerpc.h
-lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_ppc64le.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_s390x.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_s390x.h
-lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_x86.h
-lldb/source/Plugins/Process/Utility/RegisterContextThreadMemory.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextThreadMemory.h
-lldb/source/Plugins/Process/Utility/RegisterContextWindows_i386.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextWindows_i386.h
-lldb/source/Plugins/Process/Utility/RegisterContextWindows_x86_64.cpp
-lldb/source/Plugins/Process/Utility/RegisterContextWindows_x86_64.h
-lldb/source/Plugins/Process/Utility/RegisterContext_mips.h
-lldb/source/Plugins/Process/Utility/RegisterContext_powerpc.h
-lldb/source/Plugins/Process/Utility/RegisterContext_s390x.h
-lldb/source/Plugins/Process/Utility/RegisterContext_x86.cpp
-lldb/source/Plugins/Process/Utility/RegisterInfoAndSetInterface.h
-lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
-lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_ppc64le.h
-lldb/source/Plugins/Process/Utility/StopInfoMachException.cpp
-lldb/source/Plugins/Process/Utility/StopInfoMachException.h
-lldb/source/Plugins/Process/Utility/ThreadMemory.cpp
-lldb/source/Plugins/Process/Utility/ThreadMemory.h
-lldb/source/Plugins/Process/Windows/Common/DebuggerThread.cpp
-lldb/source/Plugins/Process/Windows/Common/LocalDebugDelegate.cpp
-lldb/source/Plugins/Process/Windows/Common/NativeProcessWindows.h
-lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows.cpp
-lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm.cpp
-lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm.h
-lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm64.cpp
-lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_arm64.h
-lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_i386.cpp
-lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_i386.h
-lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_WoW64.cpp
-lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_WoW64.h
-lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_x86_64.cpp
-lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_x86_64.h
-lldb/source/Plugins/Process/Windows/Common/NativeThreadWindows.cpp
-lldb/source/Plugins/Process/Windows/Common/NativeThreadWindows.h
-lldb/source/Plugins/Process/Windows/Common/NtStructures.h
-lldb/source/Plugins/Process/Windows/Common/ProcessDebugger.cpp
-lldb/source/Plugins/Process/Windows/Common/ProcessDebugger.h
-lldb/source/Plugins/Process/Windows/Common/RegisterContextWindows.h
-lldb/source/Plugins/Process/Windows/Common/TargetThreadWindows.cpp
-lldb/source/Plugins/Process/Windows/Common/TargetThreadWindows.h
-lldb/source/Plugins/Process/Windows/Common/arm/RegisterContextWindows_arm.h
-lldb/source/Plugins/Process/Windows/Common/arm64/RegisterContextWindows_arm64.h
-lldb/source/Plugins/REPL/Clang/ClangREPL.h
-lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp
-lldb/source/Plugins/ScriptInterpreter/Lua/Lua.h
-lldb/source/Plugins/ScriptInterpreter/Lua/ScriptInterpreterLua.cpp
-lldb/source/Plugins/ScriptInterpreter/Lua/ScriptInterpreterLua.h
-lldb/source/Plugins/ScriptInterpreter/Lua/SWIGLuaBridge.h
-lldb/source/Plugins/ScriptInterpreter/None/ScriptInterpreterNone.cpp
-lldb/source/Plugins/ScriptInterpreter/None/ScriptInterpreterNone.h
-lldb/source/Plugins/ScriptInterpreter/Python/lldb-python.h
-lldb/source/Plugins/ScriptInterpreter/Python/PythonReadline.cpp
-lldb/source/Plugins/ScriptInterpreter/Python/PythonReadline.h
-lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.cpp
-lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.h
-lldb/source/Plugins/ScriptInterpreter/Python/ScriptedPythonInterface.cpp
-lldb/source/Plugins/ScriptInterpreter/Python/ScriptedPythonInterface.h
-lldb/source/Plugins/ScriptInterpreter/Python/ScriptedThreadPythonInterface.cpp
-lldb/source/Plugins/ScriptInterpreter/Python/ScriptedThreadPythonInterface.h
-lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h
-lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.cpp
-lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h
-lldb/source/Plugins/SymbolFile/DWARF/AppleDWARFIndex.cpp
-lldb/source/Plugins/SymbolFile/DWARF/AppleDWARFIndex.h
-lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.h
-lldb/source/Plugins/SymbolFile/DWARF/DIERef.cpp
-lldb/source/Plugins/SymbolFile/DWARF/DWARFAbbreviationDeclaration.cpp
-lldb/source/Plugins/SymbolFile/DWARF/DWARFAbbreviationDeclaration.h
-lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
-lldb/source/Plugins/SymbolFile/DWARF/DWARFAttribute.h
-lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.h
-lldb/source/Plugins/SymbolFile/DWARF/DWARFCompileUnit.cpp
-lldb/source/Plugins/SymbolFile/DWARF/DWARFCompileUnit.h
-lldb/source/Plugins/SymbolFile/DWARF/DWARFContext.cpp
-lldb/source/Plugins/SymbolFile/DWARF/DWARFContext.h
-lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.cpp
-lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugAbbrev.h
-lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugAranges.cpp
-lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugAranges.h
-lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.cpp
-lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.h
-lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugMacro.cpp
-lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugMacro.h
-lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugRanges.h
-lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.cpp
-lldb/source/Plugins/SymbolFile/DWARF/DWARFDefines.cpp
-lldb/source/Plugins/SymbolFile/DWARF/DWARFDefines.h
-lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h
-lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.cpp
-lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.h
-lldb/source/Plugins/SymbolFile/DWARF/DWARFTypeUnit.cpp
-lldb/source/Plugins/SymbolFile/DWARF/DWARFTypeUnit.h
-lldb/source/Plugins/SymbolFile/DWARF/HashedNameToDIE.cpp
-lldb/source/Plugins/SymbolFile/DWARF/HashedNameToDIE.h
-lldb/source/Plugins/SymbolFile/DWARF/LogChannelDWARF.h
-lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.h
-lldb/source/Plugins/SymbolFile/DWARF/NameToDIE.cpp
-lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
-lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
-lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp
-lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
-lldb/source/Plugins/SymbolFile/NativePDB/CodeViewRegisterMapping.cpp
-lldb/source/Plugins/SymbolFile/NativePDB/CodeViewRegisterMapping.h
-lldb/source/Plugins/SymbolFile/NativePDB/CompileUnitIndex.h
-lldb/source/Plugins/SymbolFile/NativePDB/DWARFLocationExpression.h
-lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.h
-lldb/source/Plugins/SymbolFile/NativePDB/PdbIndex.cpp
-lldb/source/Plugins/SymbolFile/NativePDB/PdbIndex.h
-lldb/source/Plugins/SymbolFile/NativePDB/PdbSymUid.cpp
-lldb/source/Plugins/SymbolFile/NativePDB/PdbSymUid.h
-lldb/source/Plugins/SymbolFile/NativePDB/PdbUtil.cpp
-lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
-lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.h
-lldb/source/Plugins/SymbolFile/PDB/PDBLocationToDWARFExpression.cpp
-lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h
-lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.cpp
-lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.h
-lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp
-lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.h
-lldb/source/Plugins/SymbolVendor/MacOSX/SymbolVendorMacOSX.cpp
-lldb/source/Plugins/SymbolVendor/MacOSX/SymbolVendorMacOSX.h
-lldb/source/Plugins/SymbolVendor/wasm/SymbolVendorWasm.cpp
-lldb/source/Plugins/SymbolVendor/wasm/SymbolVendorWasm.h
-lldb/source/Plugins/SystemRuntime/MacOSX/SystemRuntimeMacOSX.h
-lldb/source/Plugins/Trace/common/ThreadPostMortemTrace.cpp
-lldb/source/Plugins/Trace/common/ThreadPostMortemTrace.h
-lldb/source/Plugins/Trace/common/TraceJSONStructs.cpp
-lldb/source/Plugins/Trace/common/TraceJSONStructs.h
-lldb/source/Plugins/Trace/common/TraceSessionFileParser.cpp
-lldb/source/Plugins/Trace/common/TraceSessionSaver.cpp
-lldb/source/Plugins/Trace/common/TraceSessionSaver.h
-lldb/source/Plugins/Trace/intel-pt/CommandObjectTraceStartIntelPT.cpp
-lldb/source/Plugins/Trace/intel-pt/CommandObjectTraceStartIntelPT.h
-lldb/source/Plugins/Trace/intel-pt/DecodedThread.cpp
-lldb/source/Plugins/Trace/intel-pt/DecodedThread.h
-lldb/source/Plugins/Trace/intel-pt/forward-declarations.h
-lldb/source/Plugins/Trace/intel-pt/IntelPTDecoder.cpp
-lldb/source/Plugins/Trace/intel-pt/IntelPTDecoder.h
-lldb/source/Plugins/Trace/intel-pt/TraceCursorIntelPT.cpp
-lldb/source/Plugins/Trace/intel-pt/TraceCursorIntelPT.h
-lldb/source/Plugins/Trace/intel-pt/TraceIntelPT.cpp
-lldb/source/Plugins/Trace/intel-pt/TraceIntelPT.h
-lldb/source/Plugins/Trace/intel-pt/TraceIntelPTConstants.h
-lldb/source/Plugins/Trace/intel-pt/TraceIntelPTJSONStructs.cpp
-lldb/source/Plugins/Trace/intel-pt/TraceIntelPTJSONStructs.h
-lldb/source/Plugins/Trace/intel-pt/TraceIntelPTSessionFileParser.cpp
-lldb/source/Plugins/Trace/intel-pt/TraceIntelPTSessionSaver.cpp
-lldb/source/Plugins/Trace/intel-pt/TraceIntelPTSessionSaver.h
-lldb/source/Plugins/TraceExporter/common/TraceHTR.cpp
-lldb/source/Plugins/TraceExporter/common/TraceHTR.h
-lldb/source/Plugins/TraceExporter/ctf/CommandObjectThreadTraceExportCTF.cpp
-lldb/source/Plugins/TraceExporter/ctf/CommandObjectThreadTraceExportCTF.h
-lldb/source/Plugins/TraceExporter/ctf/TraceExporterCTF.cpp
-lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.h
-lldb/source/Plugins/UnwindAssembly/x86/UnwindAssembly-x86.cpp
-lldb/source/Plugins/UnwindAssembly/x86/UnwindAssembly-x86.h
-lldb/source/Symbol/ArmUnwindInfo.cpp
-lldb/source/Symbol/Block.cpp
-lldb/source/Symbol/CompilerDecl.cpp
-lldb/source/Symbol/CompilerDeclContext.cpp
-lldb/source/Symbol/DebugMacros.cpp
-lldb/source/Symbol/DeclVendor.cpp
-lldb/source/Symbol/LineEntry.cpp
-lldb/source/Symbol/LocateSymbolFile.cpp
-lldb/source/Symbol/PostfixExpression.cpp
-lldb/source/Symbol/SymbolContext.cpp
-lldb/source/Symbol/SymbolFile.cpp
-lldb/source/Symbol/SymbolVendor.cpp
-lldb/source/Symbol/TypeList.cpp
-lldb/source/Symbol/TypeMap.cpp
-lldb/source/Symbol/TypeSystem.cpp
-lldb/source/Symbol/UnwindTable.cpp
-lldb/source/Symbol/Variable.cpp
-lldb/source/Symbol/VariableList.cpp
-lldb/source/Target/AssertFrameRecognizer.cpp
-lldb/source/Target/InstrumentationRuntime.cpp
-lldb/source/Target/InstrumentationRuntimeStopInfo.cpp
-lldb/source/Target/JITLoader.cpp
-lldb/source/Target/Language.cpp
-lldb/source/Target/MemoryHistory.cpp
-lldb/source/Target/MemoryRegionInfo.cpp
-lldb/source/Target/MemoryTagMap.cpp
-lldb/source/Target/ModuleCache.cpp
-lldb/source/Target/OperatingSystem.cpp
-lldb/source/Target/ProcessTrace.cpp
-lldb/source/Target/Queue.cpp
-lldb/source/Target/RegisterContext.cpp
-lldb/source/Target/RegisterNumber.cpp
-lldb/source/Target/SectionLoadHistory.cpp
-lldb/source/Target/SectionLoadList.cpp
-lldb/source/Target/StackID.cpp
-lldb/source/Target/SystemRuntime.cpp
-lldb/source/Target/ThreadCollection.cpp
-lldb/source/Target/ThreadPlanCallFunctionUsingABI.cpp
-lldb/source/Target/ThreadPlanCallOnFunctionExit.cpp
-lldb/source/Target/ThreadPlanCallUserExpression.cpp
-lldb/source/Target/ThreadPlanRunToAddress.cpp
-lldb/source/Target/ThreadPlanShouldStopHere.cpp
-lldb/source/Target/ThreadPlanStepInRange.cpp
-lldb/source/Target/ThreadPlanStepThrough.cpp
-lldb/source/Target/ThreadPlanStepUntil.cpp
-lldb/source/Target/ThreadSpec.cpp
-lldb/source/Target/Trace.cpp
-lldb/source/Target/TraceCursor.cpp
-lldb/source/Target/TraceExporter.cpp
-lldb/source/Target/TraceInstructionDumper.cpp
-lldb/source/Target/UnwindAssembly.cpp
-lldb/source/Target/UnwindLLDB.cpp
-lldb/source/Utility/Args.cpp
-lldb/source/Utility/ARM64_DWARF_Registers.h
-lldb/source/Utility/ARM64_ehframe_Registers.h
-lldb/source/Utility/ARM_DWARF_Registers.h
-lldb/source/Utility/ARM_ehframe_Registers.h
-lldb/source/Utility/Baton.cpp
-lldb/source/Utility/Broadcaster.cpp
-lldb/source/Utility/Connection.cpp
-lldb/source/Utility/DataBufferLLVM.cpp
-lldb/source/Utility/DataEncoder.cpp
-lldb/source/Utility/DataExtractor.cpp
-lldb/source/Utility/Environment.cpp
-lldb/source/Utility/GDBRemote.cpp
-lldb/source/Utility/Instrumentation.cpp
-lldb/source/Utility/IOObject.cpp
-lldb/source/Utility/Listener.cpp
-lldb/source/Utility/LLDBAssert.cpp
-lldb/source/Utility/LLDBLog.cpp
-lldb/source/Utility/NameMatches.cpp
-lldb/source/Utility/PPC64LE_DWARF_Registers.h
-lldb/source/Utility/PPC64_DWARF_Registers.h
-lldb/source/Utility/RegularExpression.cpp
-lldb/source/Utility/Reproducer.cpp
-lldb/source/Utility/ReproducerProvider.cpp
-lldb/source/Utility/State.cpp
-lldb/source/Utility/Status.cpp
-lldb/source/Utility/Stream.cpp
-lldb/source/Utility/StreamCallback.cpp
-lldb/source/Utility/StreamString.cpp
-lldb/source/Utility/StringExtractor.cpp
-lldb/source/Utility/StringExtractorGDBRemote.cpp
-lldb/source/Utility/StringLexer.cpp
-lldb/source/Utility/StringList.cpp
-lldb/source/Utility/StructuredData.cpp
-lldb/source/Utility/TildeExpressionResolver.cpp
-lldb/source/Utility/Timer.cpp
-lldb/source/Utility/TraceGDBRemotePackets.cpp
-lldb/source/Utility/TraceIntelPTGDBRemotePackets.cpp
-lldb/source/Utility/UnimplementedError.cpp
-lldb/source/Utility/UriParser.cpp
-lldb/source/Utility/UserID.cpp
-lldb/source/Utility/UserIDResolver.cpp
-lldb/source/Utility/UuidCompatibility.h
-lldb/source/Utility/VASprintf.cpp
-lldb/source/Utility/VMRange.cpp
-lldb/source/Utility/XcodeSDK.cpp
-lldb/source/Version/Version.cpp
-lldb/tools/argdumper/argdumper.cpp
-lldb/tools/darwin-debug/darwin-debug.cpp
-lldb/tools/debugserver/source/ARM_DWARF_Registers.h
-lldb/tools/debugserver/source/ARM_ehframe_Registers.h
-lldb/tools/debugserver/source/DNB.h
-lldb/tools/debugserver/source/DNBArch.cpp
-lldb/tools/debugserver/source/DNBArch.h
-lldb/tools/debugserver/source/DNBBreakpoint.cpp
-lldb/tools/debugserver/source/DNBBreakpoint.h
-lldb/tools/debugserver/source/DNBDataRef.cpp
-lldb/tools/debugserver/source/DNBDataRef.h
-lldb/tools/debugserver/source/DNBDefs.h
-lldb/tools/debugserver/source/DNBError.cpp
-lldb/tools/debugserver/source/DNBError.h
-lldb/tools/debugserver/source/DNBLog.cpp
-lldb/tools/debugserver/source/DNBLog.h
-lldb/tools/debugserver/source/DNBRegisterInfo.cpp
-lldb/tools/debugserver/source/DNBRegisterInfo.h
-lldb/tools/debugserver/source/DNBRuntimeAction.h
-lldb/tools/debugserver/source/DNBThreadResumeActions.cpp
-lldb/tools/debugserver/source/DNBThreadResumeActions.h
-lldb/tools/debugserver/source/DNBTimer.h
-lldb/tools/debugserver/source/JSON.cpp
-lldb/tools/debugserver/source/JSON.h
-lldb/tools/debugserver/source/JSONGenerator.h
-lldb/tools/debugserver/source/libdebugserver.h
-lldb/tools/debugserver/source/PseudoTerminal.cpp
-lldb/tools/debugserver/source/PseudoTerminal.h
-lldb/tools/debugserver/source/PThreadCondition.h
-lldb/tools/debugserver/source/PThreadEvent.cpp
-lldb/tools/debugserver/source/PThreadEvent.h
-lldb/tools/debugserver/source/PThreadMutex.h
-lldb/tools/debugserver/source/RNBDefs.h
-lldb/tools/debugserver/source/RNBServices.h
-lldb/tools/debugserver/source/RNBSocket.h
-lldb/tools/debugserver/source/StdStringExtractor.cpp
-lldb/tools/debugserver/source/StringConvert.cpp
-lldb/tools/debugserver/source/StringConvert.h
-lldb/tools/debugserver/source/SysSignal.cpp
-lldb/tools/debugserver/source/SysSignal.h
-lldb/tools/debugserver/source/TTYState.cpp
-lldb/tools/debugserver/source/TTYState.h
-lldb/tools/debugserver/source/MacOSX/CFBundle.cpp
-lldb/tools/debugserver/source/MacOSX/CFBundle.h
-lldb/tools/debugserver/source/MacOSX/CFString.cpp
-lldb/tools/debugserver/source/MacOSX/CFString.h
-lldb/tools/debugserver/source/MacOSX/CFUtils.h
-lldb/tools/debugserver/source/MacOSX/Genealogy.cpp
-lldb/tools/debugserver/source/MacOSX/Genealogy.h
-lldb/tools/debugserver/source/MacOSX/GenealogySPI.h
-lldb/tools/debugserver/source/MacOSX/MachException.h
-lldb/tools/debugserver/source/MacOSX/MachThread.cpp
-lldb/tools/debugserver/source/MacOSX/MachThread.h
-lldb/tools/debugserver/source/MacOSX/MachThreadList.h
-lldb/tools/debugserver/source/MacOSX/MachVMMemory.h
-lldb/tools/debugserver/source/MacOSX/MachVMRegion.cpp
-lldb/tools/debugserver/source/MacOSX/MachVMRegion.h
-lldb/tools/debugserver/source/MacOSX/OsLogger.h
-lldb/tools/debugserver/source/MacOSX/arm/DNBArchImpl.h
-lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.h
-lldb/tools/driver/Driver.cpp
-lldb/tools/driver/Driver.h
-lldb/tools/driver/Platform.cpp
-lldb/tools/driver/Platform.h
-lldb/tools/intel-features/cli-wrapper.cpp
-lldb/tools/intel-features/intel-mpx/cli-wrapper-mpxtable.h
-lldb/tools/lldb-instr/Instrument.cpp
-lldb/tools/lldb-server/Acceptor.h
-lldb/tools/lldb-server/LLDBServerUtilities.cpp
-lldb/tools/lldb-server/SystemInitializerLLGS.cpp
-lldb/tools/lldb-server/SystemInitializerLLGS.h
-lldb/tools/lldb-test/SystemInitializerTest.cpp
-lldb/tools/lldb-test/SystemInitializerTest.h
-lldb/tools/lldb-vscode/BreakpointBase.cpp
-lldb/tools/lldb-vscode/BreakpointBase.h
-lldb/tools/lldb-vscode/ExceptionBreakpoint.cpp
-lldb/tools/lldb-vscode/ExceptionBreakpoint.h
-lldb/tools/lldb-vscode/FifoFiles.cpp
-lldb/tools/lldb-vscode/FifoFiles.h
-lldb/tools/lldb-vscode/FunctionBreakpoint.cpp
-lldb/tools/lldb-vscode/FunctionBreakpoint.h
-lldb/tools/lldb-vscode/IOStream.cpp
-lldb/tools/lldb-vscode/IOStream.h
-lldb/tools/lldb-vscode/JSONUtils.cpp
-lldb/tools/lldb-vscode/JSONUtils.h
-lldb/tools/lldb-vscode/lldb-vscode.cpp
-lldb/tools/lldb-vscode/LLDBUtils.cpp
-lldb/tools/lldb-vscode/LLDBUtils.h
-lldb/tools/lldb-vscode/OutputRedirector.cpp
-lldb/tools/lldb-vscode/OutputRedirector.h
-lldb/tools/lldb-vscode/RunInTerminal.cpp
-lldb/tools/lldb-vscode/RunInTerminal.h
-lldb/tools/lldb-vscode/SourceBreakpoint.cpp
-lldb/tools/lldb-vscode/SourceBreakpoint.h
-lldb/tools/lldb-vscode/SourceReference.h
-lldb/tools/lldb-vscode/VSCode.h
-lldb/tools/lldb-vscode/VSCodeForward.h
-lldb/unittests/gtest_common.h
-lldb/unittests/API/SBCommandInterpreterTest.cpp
-lldb/unittests/API/SBStructuredDataTest.cpp
-lldb/unittests/Breakpoint/BreakpointIDTest.cpp
-lldb/unittests/Core/CommunicationTest.cpp
-lldb/unittests/Core/DumpDataExtractorTest.cpp
-lldb/unittests/Core/FormatEntityTest.cpp
-lldb/unittests/Core/MangledTest.cpp
-lldb/unittests/Core/ModuleSpecTest.cpp
-lldb/unittests/Core/RichManglingContextTest.cpp
-lldb/unittests/Core/SourceLocationSpecTest.cpp
-lldb/unittests/Core/SourceManagerTest.cpp
-lldb/unittests/Core/StreamCallbackTest.cpp
-lldb/unittests/DataFormatter/FormatManagerTests.cpp
-lldb/unittests/DataFormatter/FormattersContainerTest.cpp
-lldb/unittests/DataFormatter/StringPrinterTests.cpp
-lldb/unittests/debugserver/JSONTest.cpp
-lldb/unittests/debugserver/RNBSocketTest.cpp
-lldb/unittests/Editline/EditlineTest.cpp
-lldb/unittests/Expression/ClangExpressionDeclMapTest.cpp
-lldb/unittests/Expression/CppModuleConfigurationTest.cpp
-lldb/unittests/Expression/DiagnosticManagerTest.cpp
-lldb/unittests/Host/ConnectionFileDescriptorTest.cpp
-lldb/unittests/Host/FileActionTest.cpp
-lldb/unittests/Host/FileSystemTest.cpp
-lldb/unittests/Host/FileTest.cpp
-lldb/unittests/Host/HostTest.cpp
-lldb/unittests/Host/NativeProcessProtocolTest.cpp
-lldb/unittests/Host/PipeTest.cpp
-lldb/unittests/Host/ProcessLaunchInfoTest.cpp
-lldb/unittests/Host/SocketAddressTest.cpp
-lldb/unittests/Host/SocketTestUtilities.h
-lldb/unittests/Host/ThreadLauncherTest.cpp
-lldb/unittests/Host/linux/HostTest.cpp
-lldb/unittests/Host/linux/SupportTest.cpp
-lldb/unittests/Interpreter/TestOptionValueFileColonLine.cpp
-lldb/unittests/Interpreter/TestRegexCommand.cpp
-lldb/unittests/Language/CLanguages/CLanguagesTest.cpp
-lldb/unittests/Language/Highlighting/HighlighterTest.cpp
-lldb/unittests/ObjectFile/Breakpad/BreakpadRecordsTest.cpp
-lldb/unittests/Platform/PlatformDarwinTest.cpp
-lldb/unittests/Platform/PlatformSiginfoTest.cpp
-lldb/unittests/Process/ProcessEventDataTest.cpp
-lldb/unittests/Process/gdb-remote/GDBRemoteCommunicationServerLLGSTest.cpp
-lldb/unittests/Process/gdb-remote/GDBRemoteCommunicationServerTest.cpp
-lldb/unittests/Process/gdb-remote/GDBRemoteCommunicationTest.cpp
-lldb/unittests/Process/gdb-remote/GDBRemoteTestUtils.cpp
-lldb/unittests/Process/gdb-remote/GDBRemoteTestUtils.h
-lldb/unittests/Process/gdb-remote/PortMapTest.cpp
-lldb/unittests/Process/minidump/Inputs/linux-x86_64.cpp
-lldb/unittests/Process/POSIX/NativeProcessELFTest.cpp
-lldb/unittests/Process/Utility/LinuxProcMapsTest.cpp
-lldb/unittests/Process/Utility/MemoryTagManagerAArch64MTETest.cpp
-lldb/unittests/Process/Utility/RegisterContextFreeBSDTest.cpp
-lldb/unittests/Process/Utility/RegisterContextTest.cpp
-lldb/unittests/ScriptInterpreter/Lua/LuaTests.cpp
-lldb/unittests/ScriptInterpreter/Lua/ScriptInterpreterTests.cpp
-lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
-lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.h
-lldb/unittests/Signals/UnixSignalsTest.cpp
-lldb/unittests/Symbol/LocateSymbolFileTest.cpp
-lldb/unittests/Symbol/MangledTest.cpp
-lldb/unittests/Symbol/SymbolTest.cpp
-lldb/unittests/Symbol/SymtabTest.cpp
-lldb/unittests/Symbol/TestClangASTImporter.cpp
-lldb/unittests/Symbol/TestDWARFCallFrameInfo.cpp
-lldb/unittests/Symbol/TestTypeSystem.cpp
-lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
-lldb/unittests/SymbolFile/DWARF/DWARFDIETest.cpp
-lldb/unittests/SymbolFile/DWARF/DWARFIndexCachingTest.cpp
-lldb/unittests/SymbolFile/DWARF/DWARFUnitTest.cpp
-lldb/unittests/SymbolFile/DWARF/Inputs/test-dwarf.cpp
-lldb/unittests/SymbolFile/NativePDB/PdbFPOProgramToDWARFExpressionTests.cpp
-lldb/unittests/SymbolFile/PDB/Inputs/test-pdb-alt.cpp
-lldb/unittests/SymbolFile/PDB/Inputs/test-pdb-nested.h
-lldb/unittests/SymbolFile/PDB/Inputs/test-pdb-types.cpp
-lldb/unittests/SymbolFile/PDB/Inputs/test-pdb.cpp
-lldb/unittests/SymbolFile/PDB/Inputs/test-pdb.h
-lldb/unittests/Target/DynamicRegisterInfoTest.cpp
-lldb/unittests/Target/ExecutionContextTest.cpp
-lldb/unittests/Target/FindFileTest.cpp
-lldb/unittests/Target/MemoryTagMapTest.cpp
-lldb/unittests/Target/RemoteAwarePlatformTest.cpp
-lldb/unittests/Target/StackFrameRecognizerTest.cpp
-lldb/unittests/TestingSupport/MockTildeExpressionResolver.cpp
-lldb/unittests/TestingSupport/MockTildeExpressionResolver.h
-lldb/unittests/TestingSupport/SubsystemRAII.h
-lldb/unittests/TestingSupport/TestUtilities.cpp
-lldb/unittests/TestingSupport/Host/NativeProcessTestUtils.h
-lldb/unittests/TestingSupport/Symbol/ClangTestUtils.h
-lldb/unittests/TestingSupport/Symbol/YAMLModuleTester.cpp
-lldb/unittests/TestingSupport/Symbol/YAMLModuleTester.h
-lldb/unittests/Thread/ThreadTest.cpp
-lldb/unittests/UnwindAssembly/PPC64/TestPPC64InstEmulation.cpp
-lldb/unittests/Utility/AnsiTerminalTest.cpp
-lldb/unittests/Utility/ArgsTest.cpp
-lldb/unittests/Utility/BroadcasterTest.cpp
-lldb/unittests/Utility/CompletionRequestTest.cpp
-lldb/unittests/Utility/ConstStringTest.cpp
-lldb/unittests/Utility/DataExtractorTest.cpp
-lldb/unittests/Utility/EnvironmentTest.cpp
-lldb/unittests/Utility/EventTest.cpp
-lldb/unittests/Utility/ListenerTest.cpp
-lldb/unittests/Utility/NameMatchesTest.cpp
-lldb/unittests/Utility/OptionsWithRawTest.cpp
-lldb/unittests/Utility/PredicateTest.cpp
-lldb/unittests/Utility/ProcessInfoTest.cpp
-lldb/unittests/Utility/RangeMapTest.cpp
-lldb/unittests/Utility/RangeTest.cpp
-lldb/unittests/Utility/RegisterValueTest.cpp
-lldb/unittests/Utility/RegularExpressionTest.cpp
-lldb/unittests/Utility/ReproducerTest.cpp
-lldb/unittests/Utility/SharedClusterTest.cpp
-lldb/unittests/Utility/StatusTest.cpp
-lldb/unittests/Utility/StringExtractorGDBRemoteTest.cpp
-lldb/unittests/Utility/StringExtractorTest.cpp
-lldb/unittests/Utility/StringLexerTest.cpp
-lldb/unittests/Utility/StringListTest.cpp
-lldb/unittests/Utility/StructuredDataTest.cpp
-lldb/unittests/Utility/SubsystemRAIITest.cpp
-lldb/unittests/Utility/TildeExpressionResolverTest.cpp
-lldb/unittests/Utility/TimeoutTest.cpp
-lldb/unittests/Utility/UriParserTest.cpp
-lldb/unittests/Utility/UserIDResolverTest.cpp
-lldb/unittests/Utility/UUIDTest.cpp
-lldb/unittests/Utility/VMRangeTest.cpp
-lldb/utils/TableGen/LLDBOptionDefEmitter.cpp
-lldb/utils/TableGen/LLDBPropertyDefEmitter.cpp
-lldb/utils/TableGen/LLDBTableGen.cpp
-lldb/utils/TableGen/LLDBTableGenBackends.h
-lldb/utils/TableGen/LLDBTableGenUtils.cpp
-lldb/utils/TableGen/LLDBTableGenUtils.h
-llvm/bindings/ocaml/llvm/llvm_ocaml.h
-llvm/cmake/dummy.cpp
-llvm/cmake/unwind.h
-llvm/examples/Bye/Bye.cpp
-llvm/examples/HowToUseLLJIT/HowToUseLLJIT.cpp
-llvm/examples/IRTransforms/InitializePasses.cpp
-llvm/examples/IRTransforms/InitializePasses.h
-llvm/examples/IRTransforms/SimplifyCFG.cpp
-llvm/examples/IRTransforms/SimplifyCFG.h
-llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h
-llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h
-llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
-llvm/examples/Kaleidoscope/Chapter2/toy.cpp
-llvm/examples/Kaleidoscope/include/KaleidoscopeJIT.h
-llvm/examples/OrcV2Examples/ExampleModules.h
-llvm/examples/OrcV2Examples/LLJITDumpObjects/LLJITDumpObjects.cpp
-llvm/examples/OrcV2Examples/LLJITWithCustomObjectLinkingLayer/LLJITWithCustomObjectLinkingLayer.cpp
-llvm/examples/OrcV2Examples/LLJITWithExecutorProcessControl/LLJITWithExecutorProcessControl.cpp
-llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener/LLJITWithGDBRegistrationListener.cpp
-llvm/examples/OrcV2Examples/LLJITWithInitializers/LLJITWithInitializers.cpp
-llvm/examples/OrcV2Examples/LLJITWithLazyReexports/LLJITWithLazyReexports.cpp
-llvm/examples/OrcV2Examples/LLJITWithObjectCache/LLJITWithObjectCache.cpp
-llvm/examples/OrcV2Examples/LLJITWithOptimizingIRTransform/LLJITWithOptimizingIRTransform.cpp
-llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/RemoteJITUtils.h
-llvm/include/llvm/PassRegistry.h
-llvm/include/llvm/PassSupport.h
-llvm/include/llvm/ADT/APInt.h
-llvm/include/llvm/ADT/Bitfields.h
-llvm/include/llvm/ADT/BitmaskEnum.h
-llvm/include/llvm/ADT/BreadthFirstIterator.h
-llvm/include/llvm/ADT/CachedHashString.h
-llvm/include/llvm/ADT/CombinationGenerator.h
-llvm/include/llvm/ADT/DAGDeltaAlgorithm.h
-llvm/include/llvm/ADT/EnumeratedArray.h
-llvm/include/llvm/ADT/EpochTracker.h
-llvm/include/llvm/ADT/GenericCycleInfo.h
-llvm/include/llvm/ADT/GenericSSAContext.h
-llvm/include/llvm/ADT/ilist_base.h
-llvm/include/llvm/ADT/ilist_iterator.h
-llvm/include/llvm/ADT/ilist_node.h
-llvm/include/llvm/ADT/ilist_node_base.h
-llvm/include/llvm/ADT/IntrusiveRefCntPtr.h
-llvm/include/llvm/ADT/PointerEmbeddedInt.h
-llvm/include/llvm/ADT/ScopeExit.h
-llvm/include/llvm/ADT/Sequence.h
-llvm/include/llvm/ADT/simple_ilist.h
-llvm/include/llvm/ADT/Statistic.h
-llvm/include/llvm/ADT/STLArrayExtras.h
-llvm/include/llvm/ADT/STLForwardCompat.h
-llvm/include/llvm/ADT/StringSet.h
-llvm/include/llvm/ADT/TypeSwitch.h
-llvm/include/llvm/Analysis/BlockFrequencyInfo.h
-llvm/include/llvm/Analysis/ConstraintSystem.h
-llvm/include/llvm/Analysis/CostModel.h
-llvm/include/llvm/Analysis/CycleAnalysis.h
-llvm/include/llvm/Analysis/DDGPrinter.h
-llvm/include/llvm/Analysis/Delinearization.h
-llvm/include/llvm/Analysis/DependenceGraphBuilder.h
-llvm/include/llvm/Analysis/DivergenceAnalysis.h
-llvm/include/llvm/Analysis/DomTreeUpdater.h
-llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
-llvm/include/llvm/Analysis/IndirectCallPromotionAnalysis.h
-llvm/include/llvm/Analysis/IndirectCallVisitor.h
-llvm/include/llvm/Analysis/InlineAdvisor.h
-llvm/include/llvm/Analysis/InlineCost.h
-llvm/include/llvm/Analysis/InlineModelFeatureMaps.h
-llvm/include/llvm/Analysis/InlineOrder.h
-llvm/include/llvm/Analysis/InlineSizeEstimatorAnalysis.h
-llvm/include/llvm/Analysis/InstCount.h
-llvm/include/llvm/Analysis/InstructionSimplify.h
-llvm/include/llvm/Analysis/InstSimplifyFolder.h
-llvm/include/llvm/Analysis/IteratedDominanceFrontier.h
-llvm/include/llvm/Analysis/Lint.h
-llvm/include/llvm/Analysis/LoopCacheAnalysis.h
-llvm/include/llvm/Analysis/LoopNestAnalysis.h
-llvm/include/llvm/Analysis/MemDerefPrinter.h
-llvm/include/llvm/Analysis/MLInlineAdvisor.h
-llvm/include/llvm/Analysis/MLModelRunner.h
-llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h
-llvm/include/llvm/Analysis/ModuleDebugInfoPrinter.h
-llvm/include/llvm/Analysis/ModuleSummaryAnalysis.h
-llvm/include/llvm/Analysis/NoInferenceModelRunner.h
-llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h
-llvm/include/llvm/Analysis/ObjCARCInstKind.h
-llvm/include/llvm/Analysis/ObjCARCUtil.h
-llvm/include/llvm/Analysis/OverflowInstAnalysis.h
-llvm/include/llvm/Analysis/PhiValues.h
-llvm/include/llvm/Analysis/ReplayInlineAdvisor.h
-llvm/include/llvm/Analysis/ScalarEvolutionDivision.h
-llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
-llvm/include/llvm/Analysis/ScalarEvolutionNormalization.h
-llvm/include/llvm/Analysis/ScopedNoAliasAA.h
-llvm/include/llvm/Analysis/StackLifetime.h
-llvm/include/llvm/Analysis/StackSafetyAnalysis.h
-llvm/include/llvm/Analysis/SyncDependenceAnalysis.h
-llvm/include/llvm/Analysis/SyntheticCountsUtils.h
-llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h
-llvm/include/llvm/Analysis/TypeMetadataUtils.h
-llvm/include/llvm/Analysis/ValueLatticeUtils.h
-llvm/include/llvm/Analysis/Utils/TFUtils.h
-llvm/include/llvm/AsmParser/LLToken.h
-llvm/include/llvm/AsmParser/SlotMapping.h
-llvm/include/llvm/BinaryFormat/COFF.h
-llvm/include/llvm/BinaryFormat/Magic.h
-llvm/include/llvm/BinaryFormat/Minidump.h
-llvm/include/llvm/BinaryFormat/MsgPackDocument.h
-llvm/include/llvm/BinaryFormat/MsgPackReader.h
-llvm/include/llvm/BinaryFormat/MsgPackWriter.h
-llvm/include/llvm/BinaryFormat/Swift.h
-llvm/include/llvm/BinaryFormat/WasmTraits.h
-llvm/include/llvm/Bitcode/BitcodeAnalyzer.h
-llvm/include/llvm/Bitcode/BitcodeCommon.h
-llvm/include/llvm/CodeGen/AsmPrinter.h
-llvm/include/llvm/CodeGen/AsmPrinterHandler.h
-llvm/include/llvm/CodeGen/BasicBlockSectionUtils.h
-llvm/include/llvm/CodeGen/CodeGenCommonISel.h
-llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
-llvm/include/llvm/CodeGen/CommandFlags.h
-llvm/include/llvm/CodeGen/CSEConfigBase.h
-llvm/include/llvm/CodeGen/DebugHandlerBase.h
-llvm/include/llvm/CodeGen/DwarfStringPoolEntry.h
-llvm/include/llvm/CodeGen/ExecutionDomainFix.h
-llvm/include/llvm/CodeGen/ExpandVectorPredication.h
-llvm/include/llvm/CodeGen/GCMetadataPrinter.h
-llvm/include/llvm/CodeGen/IndirectThunks.h
-llvm/include/llvm/CodeGen/ISDOpcodes.h
-llvm/include/llvm/CodeGen/LiveIntervalCalc.h
-llvm/include/llvm/CodeGen/LiveRangeCalc.h
-llvm/include/llvm/CodeGen/LiveRegMatrix.h
-llvm/include/llvm/CodeGen/LiveStacks.h
-llvm/include/llvm/CodeGen/LoopTraversal.h
-llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
-llvm/include/llvm/CodeGen/MachineCombinerPattern.h
-llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
-llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h
-llvm/include/llvm/CodeGen/MachineLoopUtils.h
-llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h
-llvm/include/llvm/CodeGen/MachineModuleSlotTracker.h
-llvm/include/llvm/CodeGen/MachinePassManager.h
-llvm/include/llvm/CodeGen/MachineRegionInfo.h
-llvm/include/llvm/CodeGen/MachineSSAContext.h
-llvm/include/llvm/CodeGen/MachineStableHash.h
-llvm/include/llvm/CodeGen/MIRFormatter.h
-llvm/include/llvm/CodeGen/MIRFSDiscriminator.h
-llvm/include/llvm/CodeGen/MIRSampleProfile.h
-llvm/include/llvm/CodeGen/MultiHazardRecognizer.h
-llvm/include/llvm/CodeGen/NonRelocatableStringpool.h
-llvm/include/llvm/CodeGen/ParallelCG.h
-llvm/include/llvm/CodeGen/PBQPRAConstraint.h
-llvm/include/llvm/CodeGen/PreISelIntrinsicLowering.h
-llvm/include/llvm/CodeGen/RegisterBank.h
-llvm/include/llvm/CodeGen/RegisterBankInfo.h
-llvm/include/llvm/CodeGen/RegisterClassInfo.h
-llvm/include/llvm/CodeGen/ReplaceWithVeclib.h
-llvm/include/llvm/CodeGen/ScheduleDAGMutation.h
-llvm/include/llvm/CodeGen/Spiller.h
-llvm/include/llvm/CodeGen/StableHashing.h
-llvm/include/llvm/CodeGen/TargetOpcodes.h
-llvm/include/llvm/CodeGen/TileShapeInfo.h
-llvm/include/llvm/CodeGen/UnreachableBlockElim.h
-llvm/include/llvm/CodeGen/VLIWMachineScheduler.h
-llvm/include/llvm/CodeGen/WasmEHFuncInfo.h
-llvm/include/llvm/CodeGen/WinEHFuncInfo.h
-llvm/include/llvm/CodeGen/GlobalISel/Combiner.h
-llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
-llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h
-llvm/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h
-llvm/include/llvm/CodeGen/GlobalISel/InlineAsmLowering.h
-llvm/include/llvm/CodeGen/GlobalISel/InstructionSelect.h
-llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h
-llvm/include/llvm/CodeGen/GlobalISel/LostDebugLocObserver.h
-llvm/include/llvm/CodeGen/MIRParser/MIRParser.h
-llvm/include/llvm/CodeGen/PBQP/CostAllocator.h
-llvm/include/llvm/DebugInfo/DIContext.h
-llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h
-llvm/include/llvm/DebugInfo/CodeView/CodeViewError.h
-llvm/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h
-llvm/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h
-llvm/include/llvm/DebugInfo/CodeView/CVRecord.h
-llvm/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h
-llvm/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h
-llvm/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h
-llvm/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h
-llvm/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h
-llvm/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h
-llvm/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h
-llvm/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h
-llvm/include/llvm/DebugInfo/CodeView/DebugSubsection.h
-llvm/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h
-llvm/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h
-llvm/include/llvm/DebugInfo/CodeView/DebugSymbolRVASubsection.h
-llvm/include/llvm/DebugInfo/CodeView/EnumTables.h
-llvm/include/llvm/DebugInfo/CodeView/Formatters.h
-llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h
-llvm/include/llvm/DebugInfo/CodeView/GUID.h
-llvm/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h
-llvm/include/llvm/DebugInfo/CodeView/Line.h
-llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h
-llvm/include/llvm/DebugInfo/CodeView/RecordName.h
-llvm/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h
-llvm/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h
-llvm/include/llvm/DebugInfo/CodeView/SymbolDumpDelegate.h
-llvm/include/llvm/DebugInfo/CodeView/SymbolDumper.h
-llvm/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h
-llvm/include/llvm/DebugInfo/CodeView/SymbolSerializer.h
-llvm/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbackPipeline.h
-llvm/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbacks.h
-llvm/include/llvm/DebugInfo/CodeView/SymbolVisitorDelegate.h
-llvm/include/llvm/DebugInfo/CodeView/TypeDeserializer.h
-llvm/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h
-llvm/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h
-llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h
-llvm/include/llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h
-llvm/include/llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h
-llvm/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h
-llvm/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h
-llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h
-llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h
-llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h
-llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h
-llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h
-llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h
-llvm/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h
-llvm/include/llvm/DebugInfo/DWARF/DWARFLocationExpression.h
-llvm/include/llvm/DebugInfo/DWARF/DWARFRelocMap.h
-llvm/include/llvm/DebugInfo/DWARF/DWARFSection.h
-llvm/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h
-llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h
-llvm/include/llvm/DebugInfo/GSYM/FileEntry.h
-llvm/include/llvm/DebugInfo/GSYM/Header.h
-llvm/include/llvm/DebugInfo/GSYM/LookupResult.h
-llvm/include/llvm/DebugInfo/GSYM/StringTable.h
-llvm/include/llvm/DebugInfo/MSF/IMSFFile.h
-llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h
-llvm/include/llvm/DebugInfo/MSF/MSFCommon.h
-llvm/include/llvm/DebugInfo/MSF/MSFError.h
-llvm/include/llvm/DebugInfo/PDB/GenericError.h
-llvm/include/llvm/DebugInfo/PDB/IPDBDataStream.h
-llvm/include/llvm/DebugInfo/PDB/IPDBFrameData.h
-llvm/include/llvm/DebugInfo/PDB/IPDBInjectedSource.h
-llvm/include/llvm/DebugInfo/PDB/PDB.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolData.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolExe.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h
-llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h
-llvm/include/llvm/DebugInfo/PDB/UDTLayout.h
-llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumFrameData.h
-llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumInjectedSources.h
-llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumSectionContribs.h
-llvm/include/llvm/DebugInfo/PDB/DIA/DIAError.h
-llvm/include/llvm/DebugInfo/PDB/DIA/DIAFrameData.h
-llvm/include/llvm/DebugInfo/PDB/DIA/DIAInjectedSource.h
-llvm/include/llvm/DebugInfo/PDB/DIA/DIASectionContrib.h
-llvm/include/llvm/DebugInfo/PDB/DIA/DIASupport.h
-llvm/include/llvm/DebugInfo/PDB/DIA/DIAUtils.h
-llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h
-llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h
-llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h
-llvm/include/llvm/DebugInfo/PDB/Native/Hash.h
-llvm/include/llvm/DebugInfo/PDB/Native/ISectionContribVisitor.h
-llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h
-llvm/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumInjectedSources.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumLineNumbers.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumSymbols.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeFunctionSymbol.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeInlineSiteSymbol.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeLineNumber.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativePublicSymbol.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeSourceFile.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeArray.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeBuiltin.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h
-llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h
-llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h
-llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h
-llvm/include/llvm/DebugInfo/PDB/Native/RawConstants.h
-llvm/include/llvm/DebugInfo/PDB/Native/RawError.h
-llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h
-llvm/include/llvm/DebugInfo/PDB/Native/TpiHashing.h
-llvm/include/llvm/DebugInfo/Symbolize/DIFetcher.h
-llvm/include/llvm/DebugInfo/Symbolize/DIPrinter.h
-llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h
-llvm/include/llvm/Debuginfod/Debuginfod.h
-llvm/include/llvm/Debuginfod/DIFetcher.h
-llvm/include/llvm/Debuginfod/HTTPClient.h
-llvm/include/llvm/Demangle/Demangle.h
-llvm/include/llvm/Demangle/StringViewExtras.h
-llvm/include/llvm/Demangle/Utility.h
-llvm/include/llvm/DWARFLinker/DWARFLinker.h
-llvm/include/llvm/DWARFLinker/DWARFLinkerCompileUnit.h
-llvm/include/llvm/DWARFLinker/DWARFLinkerDeclContext.h
-llvm/include/llvm/DWARFLinker/DWARFStreamer.h
-llvm/include/llvm/DWP/DWP.h
-llvm/include/llvm/DWP/DWPError.h
-llvm/include/llvm/DWP/DWPStringPool.h
-llvm/include/llvm/ExecutionEngine/GenericValue.h
-llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h
-llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
-llvm/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h
-llvm/include/llvm/ExecutionEngine/JITLink/ELF.h
-llvm/include/llvm/ExecutionEngine/JITLink/ELF_aarch64.h
-llvm/include/llvm/ExecutionEngine/JITLink/ELF_riscv.h
-llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h
-llvm/include/llvm/ExecutionEngine/JITLink/JITLinkDylib.h
-llvm/include/llvm/ExecutionEngine/JITLink/MachO.h
-llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h
-llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h
-llvm/include/llvm/ExecutionEngine/JITLink/MemoryFlags.h
-llvm/include/llvm/ExecutionEngine/JITLink/riscv.h
-llvm/include/llvm/ExecutionEngine/JITLink/TableManager.h
-llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h
-llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h
-llvm/include/llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h
-llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h
-llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h
-llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
-llvm/include/llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h
-llvm/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h
-llvm/include/llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h
-llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h
-llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h
-llvm/include/llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h
-llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h
-llvm/include/llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h
-llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h
-llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h
-llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h
-llvm/include/llvm/ExecutionEngine/Orc/Layer.h
-llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
-llvm/include/llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h
-llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
-llvm/include/llvm/ExecutionEngine/Orc/Mangling.h
-llvm/include/llvm/ExecutionEngine/Orc/ObjectFileInterface.h
-llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h
-llvm/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h
-llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h
-llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h
-llvm/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h
-llvm/include/llvm/ExecutionEngine/Orc/Speculation.h
-llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
-llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h
-llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
-llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h
-llvm/include/llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h
-llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h
-llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h
-llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h
-llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h
-llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h
-llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.h
-llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h
-llvm/include/llvm/FileCheck/FileCheck.h
-llvm/include/llvm/Frontend/OpenMP/OMPAssume.h
-llvm/include/llvm/Frontend/OpenMP/OMPConstants.h
-llvm/include/llvm/Frontend/OpenMP/OMPContext.h
-llvm/include/llvm/Frontend/OpenMP/OMPGridValues.h
-llvm/include/llvm/InterfaceStub/ELFObjHandler.h
-llvm/include/llvm/InterfaceStub/IFSHandler.h
-llvm/include/llvm/InterfaceStub/IFSStub.h
-llvm/include/llvm/IR/Assumptions.h
-llvm/include/llvm/IR/BuiltinGCs.h
-llvm/include/llvm/IR/Comdat.h
-llvm/include/llvm/IR/Constants.h
-llvm/include/llvm/IR/DebugInfo.h
-llvm/include/llvm/IR/DebugInfoMetadata.h
-llvm/include/llvm/IR/DiagnosticHandler.h
-llvm/include/llvm/IR/DiagnosticPrinter.h
-llvm/include/llvm/IR/EHPersonalities.h
-llvm/include/llvm/IR/GlobalIFunc.h
-llvm/include/llvm/IR/GlobalObject.h
-llvm/include/llvm/IR/GVMaterializer.h
-llvm/include/llvm/IR/IRPrintingPasses.h
-llvm/include/llvm/IR/LLVMRemarkStreamer.h
-llvm/include/llvm/IR/MatrixBuilder.h
-llvm/include/llvm/IR/ModuleSlotTracker.h
-llvm/include/llvm/IR/OptBisect.h
-llvm/include/llvm/IR/PassInstrumentation.h
-llvm/include/llvm/IR/PassManagerImpl.h
-llvm/include/llvm/IR/PassTimingInfo.h
-llvm/include/llvm/IR/PredIteratorCache.h
-llvm/include/llvm/IR/PrintPasses.h
-llvm/include/llvm/IR/ProfileSummary.h
-llvm/include/llvm/IR/PseudoProbe.h
-llvm/include/llvm/IR/ReplaceConstant.h
-llvm/include/llvm/IR/SSAContext.h
-llvm/include/llvm/IR/StructuralHash.h
-llvm/include/llvm/IR/TrackingMDRef.h
-llvm/include/llvm/IR/UseListOrder.h
-llvm/include/llvm/MC/MCAsmInfoCOFF.h
-llvm/include/llvm/MC/MCAsmInfoDarwin.h
-llvm/include/llvm/MC/MCAsmInfoELF.h
-llvm/include/llvm/MC/MCAsmInfoGOFF.h
-llvm/include/llvm/MC/MCAsmInfoWasm.h
-llvm/include/llvm/MC/MCAsmInfoXCOFF.h
-llvm/include/llvm/MC/MCCodeView.h
-llvm/include/llvm/MC/MCContext.h
-llvm/include/llvm/MC/MCFixedLenDisassembler.h
-llvm/include/llvm/MC/MCLabel.h
-llvm/include/llvm/MC/MCObjectWriter.h
-llvm/include/llvm/MC/MCPseudoProbe.h
-llvm/include/llvm/MC/MCSectionCOFF.h
-llvm/include/llvm/MC/MCSectionGOFF.h
-llvm/include/llvm/MC/MCSectionWasm.h
-llvm/include/llvm/MC/MCSectionXCOFF.h
-llvm/include/llvm/MC/MCSymbolGOFF.h
-llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h
-llvm/include/llvm/MC/MCWasmObjectWriter.h
-llvm/include/llvm/MC/MCWasmStreamer.h
-llvm/include/llvm/MC/MCWinCOFFStreamer.h
-llvm/include/llvm/MC/MCXCOFFObjectWriter.h
-llvm/include/llvm/MC/MCXCOFFStreamer.h
-llvm/include/llvm/MC/MCDisassembler/MCRelocationInfo.h
-llvm/include/llvm/MC/MCParser/AsmCond.h
-llvm/include/llvm/MC/MCParser/AsmLexer.h
-llvm/include/llvm/MC/MCParser/MCAsmParserUtils.h
-llvm/include/llvm/MCA/CodeEmitter.h
-llvm/include/llvm/MCA/Context.h
-llvm/include/llvm/MCA/CustomBehaviour.h
-llvm/include/llvm/MCA/HWEventListener.h
-llvm/include/llvm/MCA/InstrBuilder.h
-llvm/include/llvm/MCA/Instruction.h
-llvm/include/llvm/MCA/Pipeline.h
-llvm/include/llvm/MCA/SourceMgr.h
-llvm/include/llvm/MCA/Support.h
-llvm/include/llvm/MCA/View.h
-llvm/include/llvm/MCA/HardwareUnits/HardwareUnit.h
-llvm/include/llvm/MCA/HardwareUnits/RegisterFile.h
-llvm/include/llvm/MCA/HardwareUnits/ResourceManager.h
-llvm/include/llvm/MCA/HardwareUnits/RetireControlUnit.h
-llvm/include/llvm/MCA/Stages/DispatchStage.h
-llvm/include/llvm/MCA/Stages/EntryStage.h
-llvm/include/llvm/MCA/Stages/ExecuteStage.h
-llvm/include/llvm/MCA/Stages/InOrderIssueStage.h
-llvm/include/llvm/MCA/Stages/InstructionTables.h
-llvm/include/llvm/MCA/Stages/MicroOpQueueStage.h
-llvm/include/llvm/MCA/Stages/RetireStage.h
-llvm/include/llvm/MCA/Stages/Stage.h
-llvm/include/llvm/ObjCopy/CommonConfig.h
-llvm/include/llvm/ObjCopy/MultiFormatConfig.h
-llvm/include/llvm/ObjCopy/ObjCopy.h
-llvm/include/llvm/ObjCopy/COFF/COFFConfig.h
-llvm/include/llvm/ObjCopy/COFF/COFFObjcopy.h
-llvm/include/llvm/ObjCopy/ELF/ELFConfig.h
-llvm/include/llvm/ObjCopy/ELF/ELFObjcopy.h
-llvm/include/llvm/ObjCopy/MachO/MachOConfig.h
-llvm/include/llvm/ObjCopy/MachO/MachOObjcopy.h
-llvm/include/llvm/ObjCopy/wasm/WasmConfig.h
-llvm/include/llvm/ObjCopy/wasm/WasmObjcopy.h
-llvm/include/llvm/ObjCopy/XCOFF/XCOFFConfig.h
-llvm/include/llvm/ObjCopy/XCOFF/XCOFFObjcopy.h
-llvm/include/llvm/Object/Archive.h
-llvm/include/llvm/Object/COFFModuleDefinition.h
-llvm/include/llvm/Object/Decompressor.h
-llvm/include/llvm/Object/FaultMapParser.h
-llvm/include/llvm/Object/MachOUniversalWriter.h
-llvm/include/llvm/Object/Minidump.h
-llvm/include/llvm/Object/ModuleSymbolTable.h
-llvm/include/llvm/Object/RelocationResolver.h
-llvm/include/llvm/Object/TapiFile.h
-llvm/include/llvm/Object/TapiUniversal.h
-llvm/include/llvm/Object/WindowsResource.h
-llvm/include/llvm/Object/XCOFFObjectFile.h
-llvm/include/llvm/ObjectYAML/CodeViewYAMLDebugSections.h
-llvm/include/llvm/ObjectYAML/CodeViewYAMLSymbols.h
-llvm/include/llvm/ObjectYAML/CodeViewYAMLTypeHashing.h
-llvm/include/llvm/ObjectYAML/CodeViewYAMLTypes.h
-llvm/include/llvm/ObjectYAML/DWARFEmitter.h
-llvm/include/llvm/ObjectYAML/DWARFYAML.h
-llvm/include/llvm/ObjectYAML/MachOYAML.h
-llvm/include/llvm/ObjectYAML/MinidumpYAML.h
-llvm/include/llvm/ObjectYAML/ObjectYAML.h
-llvm/include/llvm/ObjectYAML/WasmYAML.h
-llvm/include/llvm/ObjectYAML/YAML.h
-llvm/include/llvm/ObjectYAML/yaml2obj.h
-llvm/include/llvm/Option/OptSpecifier.h
-llvm/include/llvm/Passes/OptimizationLevel.h
-llvm/include/llvm/Passes/StandardInstrumentations.h
-llvm/include/llvm/ProfileData/GCOV.h
-llvm/include/llvm/ProfileData/InstrProfCorrelator.h
-llvm/include/llvm/ProfileData/InstrProfWriter.h
-llvm/include/llvm/ProfileData/ItaniumManglingCanonicalizer.h
-llvm/include/llvm/ProfileData/ProfileCommon.h
-llvm/include/llvm/ProfileData/RawMemProfReader.h
-llvm/include/llvm/ProfileData/SymbolRemappingReader.h
-llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
-llvm/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
-llvm/include/llvm/Remarks/BitstreamRemarkContainer.h
-llvm/include/llvm/Remarks/BitstreamRemarkParser.h
-llvm/include/llvm/Remarks/BitstreamRemarkSerializer.h
-llvm/include/llvm/Remarks/HotnessThresholdParser.h
-llvm/include/llvm/Remarks/RemarkFormat.h
-llvm/include/llvm/Remarks/RemarkLinker.h
-llvm/include/llvm/Remarks/RemarkParser.h
-llvm/include/llvm/Remarks/RemarkSerializer.h
-llvm/include/llvm/Remarks/RemarkStreamer.h
-llvm/include/llvm/Remarks/RemarkStringTable.h
-llvm/include/llvm/Remarks/YAMLRemarkSerializer.h
-llvm/include/llvm/Support/Alignment.h
-llvm/include/llvm/Support/AlignOf.h
-llvm/include/llvm/Support/AllocatorBase.h
-llvm/include/llvm/Support/AutoConvert.h
-llvm/include/llvm/Support/Base64.h
-llvm/include/llvm/Support/BCD.h
-llvm/include/llvm/Support/BinaryByteStream.h
-llvm/include/llvm/Support/BinaryItemStream.h
-llvm/include/llvm/Support/BinaryStream.h
-llvm/include/llvm/Support/BinaryStreamError.h
-llvm/include/llvm/Support/BinaryStreamReader.h
-llvm/include/llvm/Support/BinaryStreamRef.h
-llvm/include/llvm/Support/BinaryStreamWriter.h
-llvm/include/llvm/Support/BuryPointer.h
-llvm/include/llvm/Support/CachePruning.h
-llvm/include/llvm/Support/Caching.h
-llvm/include/llvm/Support/CFGDiff.h
-llvm/include/llvm/Support/CFGUpdate.h
-llvm/include/llvm/Support/CodeGenCoverage.h
-llvm/include/llvm/Support/CRC.h
-llvm/include/llvm/Support/CSKYAttributeParser.h
-llvm/include/llvm/Support/CSKYAttributes.h
-llvm/include/llvm/TargetParser/CSKYTargetParser.h
-llvm/include/llvm/Support/DataTypes.h
-llvm/include/llvm/Support/DebugCounter.h
-llvm/include/llvm/Support/Discriminator.h
-llvm/include/llvm/Support/DivisionByConstantInfo.h
-llvm/include/llvm/Support/DJB.h
-llvm/include/llvm/Support/ELFAttributeParser.h
-llvm/include/llvm/Support/ELFAttributes.h
-llvm/include/llvm/Support/ExitCodes.h
-llvm/include/llvm/Support/FileCollector.h
-llvm/include/llvm/Support/FileOutputBuffer.h
-llvm/include/llvm/Support/GenericIteratedDominanceFrontier.h
-llvm/include/llvm/Support/HashBuilder.h
-llvm/include/llvm/Support/InitLLVM.h
-llvm/include/llvm/Support/InstructionCost.h
-llvm/include/llvm/Support/MD5.h
-llvm/include/llvm/Support/MemAlloc.h
-llvm/include/llvm/Support/MemoryBufferRef.h
-llvm/include/llvm/Support/MSP430AttributeParser.h
-llvm/include/llvm/Support/MSP430Attributes.h
-llvm/include/llvm/Support/MSVCErrorWorkarounds.h
-llvm/include/llvm/Support/Parallel.h
-llvm/include/llvm/Support/PGOOptions.h
-llvm/include/llvm/Support/PointerLikeTypeTraits.h
-llvm/include/llvm/Support/RISCVAttributeParser.h
-llvm/include/llvm/Support/RISCVAttributes.h
-llvm/include/llvm/TargetParser/RISCVISAInfo.h
-llvm/include/llvm/Support/RWMutex.h
-llvm/include/llvm/Support/ScopedPrinter.h
-llvm/include/llvm/Support/SHA256.h
-llvm/include/llvm/Support/Signposts.h
-llvm/include/llvm/Support/SmallVectorMemoryBuffer.h
-llvm/include/llvm/Support/SMLoc.h
-llvm/include/llvm/Support/SMTAPI.h
-llvm/include/llvm/Support/SourceMgr.h
-llvm/include/llvm/Support/SuffixTree.h
-llvm/include/llvm/Support/SystemUtils.h
-llvm/include/llvm/TargetParser/TargetParser.h
-llvm/include/llvm/Support/TrailingObjects.h
-llvm/include/llvm/Support/Unicode.h
-llvm/include/llvm/Support/UnicodeCharRanges.h
-llvm/include/llvm/Support/VersionTuple.h
-llvm/include/llvm/Support/WindowsError.h
-llvm/include/llvm/Support/WithColor.h
-llvm/include/llvm/Support/FileSystem/UniqueID.h
-llvm/include/llvm/Support/Solaris/sys/regset.h
-llvm/include/llvm/TableGen/DirectiveEmitter.h
-llvm/include/llvm/TableGen/Parser.h
-llvm/include/llvm/TableGen/StringToOffsetTable.h
-llvm/include/llvm/Target/CGPassBuilderOption.h
-llvm/include/llvm/Target/CodeGenCWrappers.h
-llvm/include/llvm/Testing/Annotations/Annotations.h
-llvm/include/llvm/Testing/Support/SupportHelpers.h
-llvm/include/llvm/TextAPI/Architecture.h
-llvm/include/llvm/TextAPI/ArchitectureSet.h
-llvm/include/llvm/TextAPI/InterfaceFile.h
-llvm/include/llvm/TextAPI/PackedVersion.h
-llvm/include/llvm/TextAPI/Platform.h
-llvm/include/llvm/TextAPI/Symbol.h
-llvm/include/llvm/TextAPI/Target.h
-llvm/include/llvm/TextAPI/TextAPIReader.h
-llvm/include/llvm/TextAPI/TextAPIWriter.h
-llvm/include/llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h
-llvm/include/llvm/Transforms/CFGuard.h
-llvm/include/llvm/Transforms/Utils.h
-llvm/include/llvm/Transforms/Coroutines/CoroCleanup.h
-llvm/include/llvm/Transforms/Coroutines/CoroEarly.h
-llvm/include/llvm/Transforms/Coroutines/CoroElide.h
-llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
-llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
-llvm/include/llvm/Transforms/Instrumentation/AddressSanitizer.h
-llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h
-llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerOptions.h
-llvm/include/llvm/Transforms/Instrumentation/CGProfile.h
-llvm/include/llvm/Transforms/Instrumentation/DataFlowSanitizer.h
-llvm/include/llvm/Transforms/Instrumentation/HWAddressSanitizer.h
-llvm/include/llvm/Transforms/Instrumentation/InstrOrderFile.h
-llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
-llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h
-llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h
-llvm/include/llvm/Transforms/IPO/Annotation2Metadata.h
-llvm/include/llvm/Transforms/IPO/ArgumentPromotion.h
-llvm/include/llvm/Transforms/IPO/Attributor.h
-llvm/include/llvm/Transforms/IPO/BlockExtractor.h
-llvm/include/llvm/Transforms/IPO/CalledValuePropagation.h
-llvm/include/llvm/Transforms/IPO/ConstantMerge.h
-llvm/include/llvm/Transforms/IPO/DeadArgumentElimination.h
-llvm/include/llvm/Transforms/IPO/ElimAvailExtern.h
-llvm/include/llvm/Transforms/IPO/FunctionAttrs.h
-llvm/include/llvm/Transforms/IPO/FunctionImport.h
-llvm/include/llvm/Transforms/IPO/GlobalOpt.h
-llvm/include/llvm/Transforms/IPO/GlobalSplit.h
-llvm/include/llvm/Transforms/IPO/Inliner.h
-llvm/include/llvm/Transforms/IPO/Internalize.h
-llvm/include/llvm/Transforms/IPO/LoopExtractor.h
-llvm/include/llvm/Transforms/IPO/MergeFunctions.h
-llvm/include/llvm/Transforms/IPO/ModuleInliner.h
-llvm/include/llvm/Transforms/IPO/OpenMPOpt.h
-llvm/include/llvm/Transforms/IPO/PartialInlining.h
-llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h
-llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
-llvm/include/llvm/Transforms/IPO/SampleProfile.h
-llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h
-llvm/include/llvm/Transforms/IPO/SCCP.h
-llvm/include/llvm/Transforms/IPO/StripSymbols.h
-llvm/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h
-llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h
-llvm/include/llvm/Transforms/Scalar/ADCE.h
-llvm/include/llvm/Transforms/Scalar/AnnotationRemarks.h
-llvm/include/llvm/Transforms/Scalar/CallSiteSplitting.h
-llvm/include/llvm/Transforms/Scalar/ConstraintElimination.h
-llvm/include/llvm/Transforms/Scalar/CorrelatedValuePropagation.h
-llvm/include/llvm/Transforms/Scalar/DeadStoreElimination.h
-llvm/include/llvm/Transforms/Scalar/DFAJumpThreading.h
-llvm/include/llvm/Transforms/Scalar/EarlyCSE.h
-llvm/include/llvm/Transforms/Scalar/FlattenCFG.h
-llvm/include/llvm/Transforms/Scalar/GVNExpression.h
-llvm/include/llvm/Transforms/Scalar/InductiveRangeCheckElimination.h
-llvm/include/llvm/Transforms/Scalar/IndVarSimplify.h
-llvm/include/llvm/Transforms/Scalar/InferAddressSpaces.h
-llvm/include/llvm/Transforms/Scalar/InstSimplifyPass.h
-llvm/include/llvm/Transforms/Scalar/JumpThreading.h
-llvm/include/llvm/Transforms/Scalar/LICM.h
-llvm/include/llvm/Transforms/Scalar/LoopBoundSplit.h
-llvm/include/llvm/Transforms/Scalar/LoopDataPrefetch.h
-llvm/include/llvm/Transforms/Scalar/LoopDeletion.h
-llvm/include/llvm/Transforms/Scalar/LoopDistribute.h
-llvm/include/llvm/Transforms/Scalar/LoopFlatten.h
-llvm/include/llvm/Transforms/Scalar/LoopFuse.h
-llvm/include/llvm/Transforms/Scalar/LoopIdiomRecognize.h
-llvm/include/llvm/Transforms/Scalar/LoopInstSimplify.h
-llvm/include/llvm/Transforms/Scalar/LoopInterchange.h
-llvm/include/llvm/Transforms/Scalar/LoopLoadElimination.h
-llvm/include/llvm/Transforms/Scalar/LoopPredication.h
-llvm/include/llvm/Transforms/Scalar/LoopReroll.h
-llvm/include/llvm/Transforms/Scalar/LoopSimplifyCFG.h
-llvm/include/llvm/Transforms/Scalar/LoopStrengthReduce.h
-llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h
-llvm/include/llvm/Transforms/Scalar/LoopUnrollPass.h
-llvm/include/llvm/Transforms/Scalar/LoopVersioningLICM.h
-llvm/include/llvm/Transforms/Scalar/LowerMatrixIntrinsics.h
-llvm/include/llvm/Transforms/Scalar/MakeGuardsExplicit.h
-llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h
-llvm/include/llvm/Transforms/Scalar/NaryReassociate.h
-llvm/include/llvm/Transforms/Scalar/Reassociate.h
-llvm/include/llvm/Transforms/Scalar/Reg2Mem.h
-llvm/include/llvm/Transforms/Scalar/RewriteStatepointsForGC.h
-llvm/include/llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h
-llvm/include/llvm/Transforms/Scalar/SCCP.h
-llvm/include/llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h
-llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h
-llvm/include/llvm/Transforms/Scalar/StraightLineStrengthReduce.h
-llvm/include/llvm/Transforms/Scalar/StructurizeCFG.h
-llvm/include/llvm/Transforms/Scalar/WarnMissedTransforms.h
-llvm/include/llvm/Transforms/Utils/AddDiscriminators.h
-llvm/include/llvm/Transforms/Utils/AMDGPUEmitPrintf.h
-llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h
-llvm/include/llvm/Transforms/Utils/BreakCriticalEdges.h
-llvm/include/llvm/Transforms/Utils/BypassSlowDivision.h
-llvm/include/llvm/Transforms/Utils/CallGraphUpdater.h
-llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h
-llvm/include/llvm/Transforms/Utils/CanonicalizeAliases.h
-llvm/include/llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h
-llvm/include/llvm/Transforms/Utils/CodeLayout.h
-llvm/include/llvm/Transforms/Utils/CodeMoverUtils.h
-llvm/include/llvm/Transforms/Utils/Debugify.h
-llvm/include/llvm/Transforms/Utils/EntryExitInstrumenter.h
-llvm/include/llvm/Transforms/Utils/FixIrreducible.h
-llvm/include/llvm/Transforms/Utils/GlobalStatus.h
-llvm/include/llvm/Transforms/Utils/HelloWorld.h
-llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h
-llvm/include/llvm/Transforms/Utils/InstructionNamer.h
-llvm/include/llvm/Transforms/Utils/InstructionWorklist.h
-llvm/include/llvm/Transforms/Utils/LCSSA.h
-llvm/include/llvm/Transforms/Utils/LibCallsShrinkWrap.h
-llvm/include/llvm/Transforms/Utils/LoopPeel.h
-llvm/include/llvm/Transforms/Utils/LoopRotationUtils.h
-llvm/include/llvm/Transforms/Utils/LoopSimplify.h
-llvm/include/llvm/Transforms/Utils/LowerSwitch.h
-llvm/include/llvm/Transforms/Utils/MatrixUtils.h
-llvm/include/llvm/Transforms/Utils/Mem2Reg.h
-llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h
-llvm/include/llvm/Transforms/Utils/MetaRenamer.h
-llvm/include/llvm/Transforms/Utils/NameAnonGlobals.h
-llvm/include/llvm/Transforms/Utils/RelLookupTableConverter.h
-llvm/include/llvm/Transforms/Utils/SampleProfileInference.h
-llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h
-llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h
-llvm/include/llvm/Transforms/Utils/SCCPSolver.h
-llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h
-llvm/include/llvm/Transforms/Utils/SplitModule.h
-llvm/include/llvm/Transforms/Utils/SSAUpdaterBulk.h
-llvm/include/llvm/Transforms/Utils/StripGCRelocates.h
-llvm/include/llvm/Transforms/Utils/StripNonLineTableDebugInfo.h
-llvm/include/llvm/Transforms/Utils/SymbolRewriter.h
-llvm/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h
-llvm/include/llvm/Transforms/Utils/UnifyLoopExits.h
-llvm/include/llvm/Transforms/Utils/ValueMapper.h
-llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
-llvm/include/llvm/WindowsDriver/MSVCSetupApi.h
-llvm/include/llvm/WindowsManifest/WindowsManifestMerger.h
-llvm/include/llvm/WindowsResource/ResourceScriptToken.h
-llvm/include/llvm/XRay/BlockIndexer.h
-llvm/include/llvm/XRay/BlockPrinter.h
-llvm/include/llvm/XRay/BlockVerifier.h
-llvm/include/llvm/XRay/FDRRecordConsumer.h
-llvm/include/llvm/XRay/FDRRecordProducer.h
-llvm/include/llvm/XRay/FDRRecords.h
-llvm/include/llvm/XRay/FDRTraceExpander.h
-llvm/include/llvm/XRay/FileHeaderReader.h
-llvm/include/llvm/XRay/InstrumentationMap.h
-llvm/include/llvm/XRay/Profile.h
-llvm/include/llvm/XRay/RecordPrinter.h
-llvm/include/llvm/XRay/Trace.h
-llvm/include/llvm/XRay/YAMLXRayRecord.h
-llvm/include/llvm-c/BitReader.h
-llvm/include/llvm-c/BitWriter.h
-llvm/include/llvm-c/Comdat.h
-llvm/include/llvm-c/Error.h
-llvm/include/llvm-c/ErrorHandling.h
-llvm/include/llvm-c/ExternC.h
-llvm/include/llvm-c/IRReader.h
-llvm/include/llvm-c/LLJIT.h
-llvm/include/llvm-c/OrcEE.h
-llvm/include/llvm-c/Remarks.h
-llvm/include/llvm-c/Types.h
-llvm/include/llvm-c/Transforms/PassBuilder.h
-llvm/lib/Analysis/CodeMetrics.cpp
-llvm/lib/Analysis/CycleAnalysis.cpp
-llvm/lib/Analysis/DDGPrinter.cpp
-llvm/lib/Analysis/Delinearization.cpp
-llvm/lib/Analysis/DependenceGraphBuilder.cpp
-llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
-llvm/lib/Analysis/DivergenceAnalysis.cpp
-llvm/lib/Analysis/DomTreeUpdater.cpp
-llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp
-llvm/lib/Analysis/ImportedFunctionsInliningStatistics.cpp
-llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp
-llvm/lib/Analysis/InlineAdvisor.cpp
-llvm/lib/Analysis/InlineCost.cpp
-llvm/lib/Analysis/InstCount.cpp
-llvm/lib/Analysis/LazyBlockFrequencyInfo.cpp
-llvm/lib/Analysis/LazyBranchProbabilityInfo.cpp
-llvm/lib/Analysis/LoopNestAnalysis.cpp
-llvm/lib/Analysis/LoopUnrollAnalyzer.cpp
-llvm/lib/Analysis/MLInlineAdvisor.cpp
-llvm/lib/Analysis/ModelUnderTrainingRunner.cpp
-llvm/lib/Analysis/NoInferenceModelRunner.cpp
-llvm/lib/Analysis/ObjCARCAliasAnalysis.cpp
-llvm/lib/Analysis/ObjCARCAnalysisUtils.cpp
-llvm/lib/Analysis/ObjCARCInstKind.cpp
-llvm/lib/Analysis/OptimizationRemarkEmitter.cpp
-llvm/lib/Analysis/OverflowInstAnalysis.cpp
-llvm/lib/Analysis/ReplayInlineAdvisor.cpp
-llvm/lib/Analysis/ScalarEvolutionDivision.cpp
-llvm/lib/Analysis/ScalarEvolutionNormalization.cpp
-llvm/lib/Analysis/ScopedNoAliasAA.cpp
-llvm/lib/Analysis/SyntheticCountsUtils.cpp
-llvm/lib/Analysis/TargetTransformInfo.cpp
-llvm/lib/Analysis/TFUtils.cpp
-llvm/lib/Analysis/TypeMetadataUtils.cpp
-llvm/lib/Analysis/ValueLattice.cpp
-llvm/lib/Analysis/ValueLatticeUtils.cpp
-llvm/lib/Analysis/VFABIDemangling.cpp
-llvm/lib/AsmParser/Parser.cpp
-llvm/lib/BinaryFormat/COFF.cpp
-llvm/lib/BinaryFormat/ELF.cpp
-llvm/lib/BinaryFormat/MachO.cpp
-llvm/lib/BinaryFormat/Magic.cpp
-llvm/lib/BinaryFormat/Minidump.cpp
-llvm/lib/BinaryFormat/MsgPackDocument.cpp
-llvm/lib/BinaryFormat/MsgPackReader.cpp
-llvm/lib/BinaryFormat/MsgPackWriter.cpp
-llvm/lib/BinaryFormat/Wasm.cpp
-llvm/lib/BinaryFormat/XCOFF.cpp
-llvm/lib/Bitcode/Reader/MetadataLoader.cpp
-llvm/lib/Bitcode/Reader/ValueList.cpp
-llvm/lib/CodeGen/AllocationOrder.cpp
-llvm/lib/CodeGen/AllocationOrder.h
-llvm/lib/CodeGen/CFGuardLongjmp.cpp
-llvm/lib/CodeGen/CodeGen.cpp
-llvm/lib/CodeGen/CodeGenPassBuilder.cpp
-llvm/lib/CodeGen/DwarfEHPrepare.cpp
-llvm/lib/CodeGen/EHContGuardCatchret.cpp
-llvm/lib/CodeGen/ExecutionDomainFix.cpp
-llvm/lib/CodeGen/ExpandVectorPredication.cpp
-llvm/lib/CodeGen/FaultMaps.cpp
-llvm/lib/CodeGen/FixupStatepointCallerSaved.cpp
-llvm/lib/CodeGen/GCMetadataPrinter.cpp
-llvm/lib/CodeGen/IndirectBrExpandPass.cpp
-llvm/lib/CodeGen/JMCInstrumenter.cpp
-llvm/lib/CodeGen/LiveDebugVariables.h
-llvm/lib/CodeGen/LiveIntervalCalc.cpp
-llvm/lib/CodeGen/LiveRangeShrink.cpp
-llvm/lib/CodeGen/LiveRegUnits.cpp
-llvm/lib/CodeGen/LoopTraversal.cpp
-llvm/lib/CodeGen/LowLevelType.cpp
-llvm/lib/CodeGen/MachineBranchProbabilityInfo.cpp
-llvm/lib/CodeGen/MachineCheckDebugify.cpp
-llvm/lib/CodeGen/MachineCycleAnalysis.cpp
-llvm/lib/CodeGen/MachineDebugify.cpp
-llvm/lib/CodeGen/MachineFunctionPass.cpp
-llvm/lib/CodeGen/MachineFunctionSplitter.cpp
-llvm/lib/CodeGen/MachineModuleInfoImpls.cpp
-llvm/lib/CodeGen/MachineModuleSlotTracker.cpp
-llvm/lib/CodeGen/MachineOutliner.cpp
-llvm/lib/CodeGen/MachinePassManager.cpp
-llvm/lib/CodeGen/MachineSSAContext.cpp
-llvm/lib/CodeGen/MachineStableHash.cpp
-llvm/lib/CodeGen/MachineStripDebug.cpp
-llvm/lib/CodeGen/MIRFSDiscriminator.cpp
-llvm/lib/CodeGen/MIRNamerPass.cpp
-llvm/lib/CodeGen/MIRPrintingPass.cpp
-llvm/lib/CodeGen/MIRSampleProfile.cpp
-llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
-llvm/lib/CodeGen/MIRYamlMapping.cpp
-llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
-llvm/lib/CodeGen/MultiHazardRecognizer.cpp
-llvm/lib/CodeGen/NonRelocatableStringpool.cpp
-llvm/lib/CodeGen/ParallelCG.cpp
-llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
-llvm/lib/CodeGen/PseudoProbeInserter.cpp
-llvm/lib/CodeGen/RegAllocBase.cpp
-llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp
-llvm/lib/CodeGen/RegAllocEvictionAdvisor.h
-llvm/lib/CodeGen/RegAllocGreedy.h
-llvm/lib/CodeGen/RegAllocScore.cpp
-llvm/lib/CodeGen/RegAllocScore.h
-llvm/lib/CodeGen/RemoveRedundantDebugValues.cpp
-llvm/lib/CodeGen/ReplaceWithVeclib.cpp
-llvm/lib/CodeGen/SafeStackLayout.cpp
-llvm/lib/CodeGen/SafeStackLayout.h
-llvm/lib/CodeGen/SpillPlacement.h
-llvm/lib/CodeGen/TargetOptionsImpl.cpp
-llvm/lib/CodeGen/VLIWMachineScheduler.cpp
-llvm/lib/CodeGen/WasmEHPrepare.cpp
-llvm/lib/CodeGen/XRayInstrumentation.cpp
-llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp
-llvm/lib/CodeGen/AsmPrinter/AddressPool.cpp
-llvm/lib/CodeGen/AsmPrinter/AIXException.cpp
-llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp
-llvm/lib/CodeGen/AsmPrinter/DebugLocStream.cpp
-llvm/lib/CodeGen/AsmPrinter/DwarfException.h
-llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
-llvm/lib/CodeGen/AsmPrinter/DwarfFile.cpp
-llvm/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp
-llvm/lib/CodeGen/AsmPrinter/DwarfStringPool.h
-llvm/lib/CodeGen/AsmPrinter/EHStreamer.h
-llvm/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
-llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp
-llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.h
-llvm/lib/CodeGen/AsmPrinter/WasmException.cpp
-llvm/lib/CodeGen/AsmPrinter/WasmException.h
-llvm/lib/CodeGen/AsmPrinter/WinCFGuard.cpp
-llvm/lib/CodeGen/AsmPrinter/WinCFGuard.h
-llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp
-llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
-llvm/lib/CodeGen/GlobalISel/GlobalISel.cpp
-llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
-llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
-llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp
-llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp
-llvm/lib/CodeGen/GlobalISel/Localizer.cpp
-llvm/lib/CodeGen/GlobalISel/LostDebugLocObserver.cpp
-llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp
-llvm/lib/CodeGen/MIRParser/MILexer.h
-llvm/lib/CodeGen/SelectionDAG/SelectionDAGTargetInfo.cpp
-llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h
-llvm/lib/DebugInfo/CodeView/CodeViewError.cpp
-llvm/lib/DebugInfo/CodeView/CodeViewRecordIO.cpp
-llvm/lib/DebugInfo/CodeView/CVSymbolVisitor.cpp
-llvm/lib/DebugInfo/CodeView/DebugCrossExSubsection.cpp
-llvm/lib/DebugInfo/CodeView/DebugFrameDataSubsection.cpp
-llvm/lib/DebugInfo/CodeView/DebugLinesSubsection.cpp
-llvm/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp
-llvm/lib/DebugInfo/CodeView/DebugSubsection.cpp
-llvm/lib/DebugInfo/CodeView/DebugSubsectionRecord.cpp
-llvm/lib/DebugInfo/CodeView/DebugSubsectionVisitor.cpp
-llvm/lib/DebugInfo/CodeView/DebugSymbolRVASubsection.cpp
-llvm/lib/DebugInfo/CodeView/DebugSymbolsSubsection.cpp
-llvm/lib/DebugInfo/CodeView/GlobalTypeTableBuilder.cpp
-llvm/lib/DebugInfo/CodeView/Line.cpp
-llvm/lib/DebugInfo/CodeView/MergingTypeTableBuilder.cpp
-llvm/lib/DebugInfo/CodeView/RecordSerialization.cpp
-llvm/lib/DebugInfo/CodeView/SimpleTypeSerializer.cpp
-llvm/lib/DebugInfo/CodeView/StringsAndChecksums.cpp
-llvm/lib/DebugInfo/CodeView/SymbolRecordHelpers.cpp
-llvm/lib/DebugInfo/CodeView/SymbolSerializer.cpp
-llvm/lib/DebugInfo/CodeView/TypeHashing.cpp
-llvm/lib/DebugInfo/CodeView/TypeIndex.cpp
-llvm/lib/DebugInfo/CodeView/TypeTableCollection.cpp
-llvm/lib/DebugInfo/DWARF/DWARFAddressRange.cpp
-llvm/lib/DebugInfo/DWARF/DWARFCompileUnit.cpp
-llvm/lib/DebugInfo/DWARF/DWARFDataExtractor.cpp
-llvm/lib/DebugInfo/DWARF/DWARFDebugInfoEntry.cpp
-llvm/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp
-llvm/lib/DebugInfo/DWARF/DWARFDebugRnglists.cpp
-llvm/lib/DebugInfo/DWARF/DWARFGdbIndex.cpp
-llvm/lib/DebugInfo/DWARF/DWARFLocationExpression.cpp
-llvm/lib/DebugInfo/DWARF/DWARFTypeUnit.cpp
-llvm/lib/DebugInfo/GSYM/GsymCreator.cpp
-llvm/lib/DebugInfo/GSYM/LookupResult.cpp
-llvm/lib/DebugInfo/MSF/MSFBuilder.cpp
-llvm/lib/DebugInfo/MSF/MSFCommon.cpp
-llvm/lib/DebugInfo/MSF/MSFError.cpp
-llvm/lib/DebugInfo/PDB/GenericError.cpp
-llvm/lib/DebugInfo/PDB/IPDBSourceFile.cpp
-llvm/lib/DebugInfo/PDB/PDB.cpp
-llvm/lib/DebugInfo/PDB/PDBInterfaceAnchors.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbol.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolAnnotation.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolBlock.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolCompiland.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolCompilandDetails.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolCompilandEnv.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolCustom.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolData.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolExe.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolFuncDebugEnd.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolFuncDebugStart.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolLabel.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolPublicSymbol.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolThunk.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolTypeArray.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolTypeBaseClass.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolTypeBuiltin.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolTypeCustom.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolTypeDimension.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolTypeEnum.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolTypeFriend.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolTypeFunctionArg.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolTypeManaged.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolTypePointer.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolTypeTypedef.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolTypeUDT.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolTypeVTable.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolTypeVTableShape.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolUnknown.cpp
-llvm/lib/DebugInfo/PDB/PDBSymbolUsingNamespace.cpp
-llvm/lib/DebugInfo/PDB/PDBSymDumper.cpp
-llvm/lib/DebugInfo/PDB/DIA/DIADataStream.cpp
-llvm/lib/DebugInfo/PDB/DIA/DIAEnumDebugStreams.cpp
-llvm/lib/DebugInfo/PDB/DIA/DIAEnumFrameData.cpp
-llvm/lib/DebugInfo/PDB/DIA/DIAEnumInjectedSources.cpp
-llvm/lib/DebugInfo/PDB/DIA/DIAEnumLineNumbers.cpp
-llvm/lib/DebugInfo/PDB/DIA/DIAEnumSectionContribs.cpp
-llvm/lib/DebugInfo/PDB/DIA/DIAEnumSourceFiles.cpp
-llvm/lib/DebugInfo/PDB/DIA/DIAEnumTables.cpp
-llvm/lib/DebugInfo/PDB/DIA/DIAError.cpp
-llvm/lib/DebugInfo/PDB/DIA/DIAFrameData.cpp
-llvm/lib/DebugInfo/PDB/DIA/DIAInjectedSource.cpp
-llvm/lib/DebugInfo/PDB/DIA/DIALineNumber.cpp
-llvm/lib/DebugInfo/PDB/DIA/DIARawSymbol.cpp
-llvm/lib/DebugInfo/PDB/DIA/DIASourceFile.cpp
-llvm/lib/DebugInfo/PDB/DIA/DIATable.cpp
-llvm/lib/DebugInfo/PDB/Native/DbiModuleDescriptor.cpp
-llvm/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp
-llvm/lib/DebugInfo/PDB/Native/GlobalsStream.cpp
-llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
-llvm/lib/DebugInfo/PDB/Native/Hash.cpp
-llvm/lib/DebugInfo/PDB/Native/HashTable.cpp
-llvm/lib/DebugInfo/PDB/Native/InfoStream.cpp
-llvm/lib/DebugInfo/PDB/Native/ModuleDebugStream.cpp
-llvm/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeCompilandSymbol.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeEnumGlobals.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeEnumLineNumbers.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeEnumSymbols.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeEnumTypes.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeExeSymbol.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeFunctionSymbol.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeInlineSiteSymbol.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeLineNumber.cpp
-llvm/lib/DebugInfo/PDB/Native/NativePublicSymbol.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeSourceFile.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeSymbolEnumerator.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeTypeArray.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeTypeBuiltin.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeTypeEnum.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeTypePointer.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeTypeTypedef.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeTypeUDT.cpp
-llvm/lib/DebugInfo/PDB/Native/NativeTypeVTShape.cpp
-llvm/lib/DebugInfo/PDB/Native/PDBFile.cpp
-llvm/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp
-llvm/lib/DebugInfo/PDB/Native/PDBStringTable.cpp
-llvm/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp
-llvm/lib/DebugInfo/PDB/Native/PublicsStream.cpp
-llvm/lib/DebugInfo/PDB/Native/RawError.cpp
-llvm/lib/DebugInfo/PDB/Native/SymbolStream.cpp
-llvm/lib/DebugInfo/PDB/Native/TpiHashing.cpp
-llvm/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp
-llvm/lib/DebugInfo/Symbolize/DIFetcher.cpp
-llvm/lib/DebugInfo/Symbolize/DIPrinter.cpp
-llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
-llvm/lib/Debuginfod/Debuginfod.cpp
-llvm/lib/Debuginfod/DIFetcher.cpp
-llvm/lib/Debuginfod/HTTPClient.cpp
-llvm/lib/Demangle/Demangle.cpp
-llvm/lib/Demangle/DLangDemangle.cpp
-llvm/lib/Demangle/MicrosoftDemangleNodes.cpp
-llvm/lib/Demangle/RustDemangle.cpp
-llvm/lib/DWARFLinker/DWARFLinkerCompileUnit.cpp
-llvm/lib/DWARFLinker/DWARFLinkerDeclContext.cpp
-llvm/lib/DWARFLinker/DWARFStreamer.cpp
-llvm/lib/DWP/DWP.cpp
-llvm/lib/DWP/DWPError.cpp
-llvm/lib/ExecutionEngine/SectionMemoryManager.cpp
-llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
-llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h
-llvm/lib/ExecutionEngine/JITLink/ELF.cpp
-llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.cpp
-llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
-llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
-llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
-llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
-llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
-llvm/lib/ExecutionEngine/JITLink/MachO.cpp
-llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
-llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
-llvm/lib/ExecutionEngine/JITLink/MemoryFlags.cpp
-llvm/lib/ExecutionEngine/JITLink/riscv.cpp
-llvm/lib/ExecutionEngine/JITLink/x86_64.cpp
-llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp
-llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp
-llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp
-llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp
-llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
-llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp
-llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp
-llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp
-llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp
-llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp
-llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp
-llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp
-llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp
-llvm/lib/ExecutionEngine/Orc/Layer.cpp
-llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
-llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp
-llvm/lib/ExecutionEngine/Orc/Mangling.cpp
-llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp
-llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp
-llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp
-llvm/lib/ExecutionEngine/Orc/Speculation.cpp
-llvm/lib/ExecutionEngine/Orc/TaskDispatch.cpp
-llvm/lib/ExecutionEngine/Orc/Shared/AllocationActions.cpp
-llvm/lib/ExecutionEngine/Orc/Shared/OrcError.cpp
-llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
-llvm/lib/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.cpp
-llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp
-llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp
-llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.h
-llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp
-llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.cpp
-llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp
-llvm/lib/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.cpp
-llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h
-llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h
-llvm/lib/FileCheck/FileCheckImpl.h
-llvm/lib/Frontend/OpenACC/ACC.cpp
-llvm/lib/Frontend/OpenMP/OMP.cpp
-llvm/lib/Frontend/OpenMP/OMPContext.cpp
-llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
-llvm/lib/FuzzMutate/OpDescriptor.cpp
-llvm/lib/FuzzMutate/RandomIRBuilder.cpp
-llvm/lib/InterfaceStub/ELFObjHandler.cpp
-llvm/lib/InterfaceStub/IFSHandler.cpp
-llvm/lib/InterfaceStub/IFSStub.cpp
-llvm/lib/IR/Assumptions.cpp
-llvm/lib/IR/Comdat.cpp
-llvm/lib/IR/DebugInfoMetadata.cpp
-llvm/lib/IR/DebugLoc.cpp
-llvm/lib/IR/DIBuilder.cpp
-llvm/lib/IR/FPEnv.cpp
-llvm/lib/IR/GCStrategy.cpp
-llvm/lib/IR/GVMaterializer.cpp
-llvm/lib/IR/LLVMContextImpl.h
-llvm/lib/IR/MetadataImpl.h
-llvm/lib/IR/OptBisect.cpp
-llvm/lib/IR/PassInstrumentation.cpp
-llvm/lib/IR/PassManager.cpp
-llvm/lib/IR/PrintPasses.cpp
-llvm/lib/IR/PseudoProbe.cpp
-llvm/lib/IR/ReplaceConstant.cpp
-llvm/lib/IR/SSAContext.cpp
-llvm/lib/IR/Statepoint.cpp
-llvm/lib/IR/StructuralHash.cpp
-llvm/lib/IR/ValueSymbolTable.cpp
-llvm/lib/MC/MCAsmInfoCOFF.cpp
-llvm/lib/MC/MCAsmInfoELF.cpp
-llvm/lib/MC/MCAsmInfoGOFF.cpp
-llvm/lib/MC/MCAsmInfoWasm.cpp
-llvm/lib/MC/MCAsmInfoXCOFF.cpp
-llvm/lib/MC/MCAsmMacro.cpp
-llvm/lib/MC/MCCodeEmitter.cpp
-llvm/lib/MC/MCInstrAnalysis.cpp
-llvm/lib/MC/MCInstrDesc.cpp
-llvm/lib/MC/MCInstrInfo.cpp
-llvm/lib/MC/MCLinkerOptimizationHint.cpp
-llvm/lib/MC/MCMachObjectTargetWriter.cpp
-llvm/lib/MC/MCObjectWriter.cpp
-llvm/lib/MC/MCPseudoProbe.cpp
-llvm/lib/MC/MCSectionWasm.cpp
-llvm/lib/MC/MCSymbolXCOFF.cpp
-llvm/lib/MC/MCWasmObjectTargetWriter.cpp
-llvm/lib/MC/MCWasmStreamer.cpp
-llvm/lib/MC/MCXCOFFObjectTargetWriter.cpp
-llvm/lib/MC/MCXCOFFStreamer.cpp
-llvm/lib/MC/StringTableBuilder.cpp
-llvm/lib/MC/MCDisassembler/Disassembler.h
-llvm/lib/MC/MCDisassembler/MCRelocationInfo.cpp
-llvm/lib/MC/MCDisassembler/MCSymbolizer.cpp
-llvm/lib/MC/MCParser/GOFFAsmParser.cpp
-llvm/lib/MC/MCParser/MCAsmParserExtension.cpp
-llvm/lib/MC/MCParser/XCOFFAsmParser.cpp
-llvm/lib/MCA/CodeEmitter.cpp
-llvm/lib/MCA/Context.cpp
-llvm/lib/MCA/CustomBehaviour.cpp
-llvm/lib/MCA/HWEventListener.cpp
-llvm/lib/MCA/InstrBuilder.cpp
-llvm/lib/MCA/Instruction.cpp
-llvm/lib/MCA/Pipeline.cpp
-llvm/lib/MCA/View.cpp
-llvm/lib/MCA/HardwareUnits/HardwareUnit.cpp
-llvm/lib/MCA/HardwareUnits/RegisterFile.cpp
-llvm/lib/MCA/HardwareUnits/ResourceManager.cpp
-llvm/lib/MCA/HardwareUnits/Scheduler.cpp
-llvm/lib/MCA/Stages/DispatchStage.cpp
-llvm/lib/MCA/Stages/EntryStage.cpp
-llvm/lib/MCA/Stages/ExecuteStage.cpp
-llvm/lib/MCA/Stages/InOrderIssueStage.cpp
-llvm/lib/MCA/Stages/MicroOpQueueStage.cpp
-llvm/lib/MCA/Stages/RetireStage.cpp
-llvm/lib/MCA/Stages/Stage.cpp
-llvm/lib/ObjCopy/Archive.cpp
-llvm/lib/ObjCopy/Archive.h
-llvm/lib/ObjCopy/ConfigManager.cpp
-llvm/lib/ObjCopy/COFF/COFFObjcopy.cpp
-llvm/lib/ObjCopy/COFF/COFFObject.cpp
-llvm/lib/ObjCopy/COFF/COFFObject.h
-llvm/lib/ObjCopy/COFF/COFFReader.cpp
-llvm/lib/ObjCopy/COFF/COFFReader.h
-llvm/lib/ObjCopy/COFF/COFFWriter.cpp
-llvm/lib/ObjCopy/COFF/COFFWriter.h
-llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
-llvm/lib/ObjCopy/ELF/ELFObject.cpp
-llvm/lib/ObjCopy/ELF/ELFObject.h
-llvm/lib/ObjCopy/MachO/MachOLayoutBuilder.cpp
-llvm/lib/ObjCopy/MachO/MachOLayoutBuilder.h
-llvm/lib/ObjCopy/MachO/MachOObjcopy.cpp
-llvm/lib/ObjCopy/MachO/MachOObject.cpp
-llvm/lib/ObjCopy/MachO/MachOObject.h
-llvm/lib/ObjCopy/MachO/MachOReader.cpp
-llvm/lib/ObjCopy/MachO/MachOReader.h
-llvm/lib/ObjCopy/MachO/MachOWriter.cpp
-llvm/lib/ObjCopy/MachO/MachOWriter.h
-llvm/lib/ObjCopy/wasm/WasmObjcopy.cpp
-llvm/lib/ObjCopy/wasm/WasmObject.cpp
-llvm/lib/ObjCopy/wasm/WasmObject.h
-llvm/lib/ObjCopy/wasm/WasmReader.cpp
-llvm/lib/ObjCopy/wasm/WasmReader.h
-llvm/lib/ObjCopy/wasm/WasmWriter.cpp
-llvm/lib/ObjCopy/wasm/WasmWriter.h
-llvm/lib/ObjCopy/XCOFF/XCOFFObject.h
-llvm/lib/ObjCopy/XCOFF/XCOFFReader.cpp
-llvm/lib/ObjCopy/XCOFF/XCOFFReader.h
-llvm/lib/Object/Archive.cpp
-llvm/lib/Object/Binary.cpp
-llvm/lib/Object/Decompressor.cpp
-llvm/lib/Object/FaultMapParser.cpp
-llvm/lib/Object/IRObjectFile.cpp
-llvm/lib/Object/IRSymtab.cpp
-llvm/lib/Object/MachOUniversalWriter.cpp
-llvm/lib/Object/Minidump.cpp
-llvm/lib/Object/ModuleSymbolTable.cpp
-llvm/lib/Object/ObjectFile.cpp
-llvm/lib/Object/RecordStreamer.cpp
-llvm/lib/Object/SymbolicFile.cpp
-llvm/lib/Object/SymbolSize.cpp
-llvm/lib/Object/TapiFile.cpp
-llvm/lib/Object/TapiUniversal.cpp
-llvm/lib/Object/WindowsMachineFlag.cpp
-llvm/lib/ObjectYAML/ArchiveEmitter.cpp
-llvm/lib/ObjectYAML/CodeViewYAMLDebugSections.cpp
-llvm/lib/ObjectYAML/CodeViewYAMLSymbols.cpp
-llvm/lib/ObjectYAML/CodeViewYAMLTypeHashing.cpp
-llvm/lib/ObjectYAML/DWARFEmitter.cpp
-llvm/lib/ObjectYAML/MachOEmitter.cpp
-llvm/lib/ObjectYAML/ObjectYAML.cpp
-llvm/lib/ObjectYAML/WasmYAML.cpp
-llvm/lib/ObjectYAML/yaml2obj.cpp
-llvm/lib/Passes/OptimizationLevel.cpp
-llvm/lib/Passes/PassBuilderBindings.cpp
-llvm/lib/Passes/PassPlugin.cpp
-llvm/lib/ProfileData/GCOV.cpp
-llvm/lib/ProfileData/InstrProfCorrelator.cpp
-llvm/lib/ProfileData/MemProf.cpp
-llvm/lib/ProfileData/SampleProfWriter.cpp
-llvm/lib/Remarks/BitstreamRemarkParser.h
-llvm/lib/Remarks/BitstreamRemarkSerializer.cpp
-llvm/lib/Remarks/Remark.cpp
-llvm/lib/Remarks/RemarkLinker.cpp
-llvm/lib/Remarks/RemarkParser.cpp
-llvm/lib/Remarks/RemarkSerializer.cpp
-llvm/lib/Remarks/RemarkStreamer.cpp
-llvm/lib/Remarks/RemarkStringTable.cpp
-llvm/lib/Remarks/YAMLRemarkParser.h
-llvm/lib/Remarks/YAMLRemarkSerializer.cpp
-llvm/lib/Support/ABIBreak.cpp
-llvm/lib/Support/ARMBuildAttrs.cpp
-llvm/lib/Support/AutoConvert.cpp
-llvm/lib/Support/BinaryStreamError.cpp
-llvm/lib/Support/BinaryStreamReader.cpp
-llvm/lib/Support/BinaryStreamRef.cpp
-llvm/lib/Support/BinaryStreamWriter.cpp
-llvm/lib/Support/BlockFrequency.cpp
-llvm/lib/Support/BranchProbability.cpp
-llvm/lib/Support/BuryPointer.cpp
-llvm/lib/Support/Caching.cpp
-llvm/lib/Support/CodeGenCoverage.cpp
-llvm/lib/Support/COM.cpp
-llvm/lib/Support/Compression.cpp
-llvm/lib/Support/CRC.cpp
-llvm/lib/Support/CSKYAttributeParser.cpp
-llvm/lib/Support/CSKYAttributes.cpp
-llvm/lib/Support/CSKYTargetParser.cpp
-llvm/lib/Support/DebugOptions.h
-llvm/lib/Support/DivisionByConstantInfo.cpp
-llvm/lib/Support/DJB.cpp
-llvm/lib/Support/ELFAttributeParser.cpp
-llvm/lib/Support/ELFAttributes.cpp
-llvm/lib/Support/ExtensibleRTTI.cpp
-llvm/lib/Support/FormattedStream.cpp
-llvm/lib/Support/GlobPattern.cpp
-llvm/lib/Support/Hashing.cpp
-llvm/lib/Support/InitLLVM.cpp
-llvm/lib/Support/InstructionCost.cpp
-llvm/lib/Support/IntEqClasses.cpp
-llvm/lib/Support/LineIterator.cpp
-llvm/lib/Support/LowLevelType.cpp
-llvm/lib/Support/MemAlloc.cpp
-llvm/lib/Support/Memory.cpp
-llvm/lib/Support/MemoryBufferRef.cpp
-llvm/lib/Support/MSP430AttributeParser.cpp
-llvm/lib/Support/MSP430Attributes.cpp
-llvm/lib/Support/Optional.cpp
-llvm/lib/Support/Parallel.cpp
-llvm/lib/Support/Program.cpp
-llvm/lib/Support/RISCVAttributeParser.cpp
-llvm/lib/Support/RISCVAttributes.cpp
-llvm/lib/Support/ScopedPrinter.cpp
-llvm/lib/Support/SHA1.cpp
-llvm/lib/Support/SHA256.cpp
-llvm/lib/Support/Signposts.cpp
-llvm/lib/Support/SourceMgr.cpp
-llvm/lib/Support/StringExtras.cpp
-llvm/lib/Support/StringMap.cpp
-llvm/lib/Support/StringSaver.cpp
-llvm/lib/Support/SuffixTree.cpp
-llvm/lib/Support/SystemUtils.cpp
-llvm/lib/Support/TarWriter.cpp
-llvm/lib/Support/ThreadPool.cpp
-llvm/lib/Support/TimeProfiler.cpp
-llvm/lib/Support/ToolOutputFile.cpp
-llvm/lib/Support/TypeSize.cpp
-llvm/lib/Support/UnicodeCaseFold.cpp
-llvm/lib/Support/VersionTuple.cpp
-llvm/lib/Support/Watchdog.cpp
-llvm/lib/Support/WithColor.cpp
-llvm/lib/TableGen/Parser.cpp
-llvm/lib/TableGen/RecordContext.h
-llvm/lib/TableGen/TableGenBackendSkeleton.cpp
-llvm/lib/Target/TargetIntrinsicInfo.cpp
-llvm/lib/Target/AArch64/AArch64CompressJumpTables.cpp
-llvm/lib/Target/AArch64/AArch64LowerHomogeneousPrologEpilog.cpp
-llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
-llvm/lib/Target/AArch64/AArch64PBQPRegAlloc.h
-llvm/lib/Target/AArch64/AArch64SpeculationHardening.cpp
-llvm/lib/Target/AArch64/AArch64StackTagging.cpp
-llvm/lib/Target/AArch64/AArch64TargetObjectFile.h
-llvm/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.h
-llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.cpp
-llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.h
-llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp
-llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
-llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.h
-llvm/lib/Target/AArch64/MCTargetDesc/AArch64FixupKinds.h
-llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
-llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
-llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
-llvm/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.h
-llvm/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp
-llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
-llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
-llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.h
-llvm/lib/Target/AMDGPU/AMDGPUCtorDtorLowering.cpp
-llvm/lib/Target/AMDGPU/AMDGPUExportClustering.cpp
-llvm/lib/Target/AMDGPU/AMDGPUExportClustering.h
-llvm/lib/Target/AMDGPU/AMDGPUFixFunctionBitcasts.cpp
-llvm/lib/Target/AMDGPU/AMDGPUFrameLowering.cpp
-llvm/lib/Target/AMDGPU/AMDGPUFrameLowering.h
-llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
-llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
-llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
-llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
-llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
-llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.h
-llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.cpp
-llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.h
-llvm/lib/Target/AMDGPU/AMDGPUPTNote.h
-llvm/lib/Target/AMDGPU/AMDGPUReplaceLDSUseWithPointer.cpp
-llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.h
-llvm/lib/Target/AMDGPU/AMDGPUTargetObjectFile.cpp
-llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
-llvm/lib/Target/AMDGPU/R600.h
-llvm/lib/Target/AMDGPU/R600ISelDAGToDAG.cpp
-llvm/lib/Target/AMDGPU/R600MCInstLower.cpp
-llvm/lib/Target/AMDGPU/R600RegisterInfo.h
-llvm/lib/Target/AMDGPU/R600Subtarget.cpp
-llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
-llvm/lib/Target/AMDGPU/R600TargetMachine.h
-llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
-llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
-llvm/lib/Target/AMDGPU/SIFixVGPRCopies.cpp
-llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
-llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp
-llvm/lib/Target/AMDGPU/SIModeRegister.cpp
-llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp
-llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
-llvm/lib/Target/AMDGPU/MCA/AMDGPUCustomBehaviour.cpp
-llvm/lib/Target/AMDGPU/MCA/AMDGPUCustomBehaviour.h
-llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFStreamer.h
-llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.h
-llvm/lib/Target/AMDGPU/MCTargetDesc/R600InstPrinter.cpp
-llvm/lib/Target/AMDGPU/MCTargetDesc/R600InstPrinter.h
-llvm/lib/Target/AMDGPU/MCTargetDesc/R600MCTargetDesc.cpp
-llvm/lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp
-llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp
-llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h
-llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.h
-llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.h
-llvm/lib/Target/ARC/ARC.h
-llvm/lib/Target/ARC/ARCAsmPrinter.cpp
-llvm/lib/Target/ARC/ARCBranchFinalize.cpp
-llvm/lib/Target/ARC/ARCExpandPseudos.cpp
-llvm/lib/Target/ARC/ARCFrameLowering.h
-llvm/lib/Target/ARC/ARCISelDAGToDAG.cpp
-llvm/lib/Target/ARC/ARCISelLowering.cpp
-llvm/lib/Target/ARC/ARCISelLowering.h
-llvm/lib/Target/ARC/ARCMachineFunctionInfo.cpp
-llvm/lib/Target/ARC/ARCMachineFunctionInfo.h
-llvm/lib/Target/ARC/ARCMCInstLower.cpp
-llvm/lib/Target/ARC/ARCMCInstLower.h
-llvm/lib/Target/ARC/ARCRegisterInfo.cpp
-llvm/lib/Target/ARC/ARCRegisterInfo.h
-llvm/lib/Target/ARC/ARCSubtarget.cpp
-llvm/lib/Target/ARC/ARCSubtarget.h
-llvm/lib/Target/ARC/ARCTargetMachine.h
-llvm/lib/Target/ARC/ARCTargetStreamer.h
-llvm/lib/Target/ARC/ARCTargetTransformInfo.h
-llvm/lib/Target/ARC/MCTargetDesc/ARCInfo.h
-llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.h
-llvm/lib/Target/ARC/MCTargetDesc/ARCMCAsmInfo.cpp
-llvm/lib/Target/ARC/MCTargetDesc/ARCMCAsmInfo.h
-llvm/lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.cpp
-llvm/lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.h
-llvm/lib/Target/ARC/TargetInfo/ARCTargetInfo.cpp
-llvm/lib/Target/ARC/TargetInfo/ARCTargetInfo.h
-llvm/lib/Target/ARM/ARMBlockPlacement.cpp
-llvm/lib/Target/ARM/ARMBranchTargets.cpp
-llvm/lib/Target/ARM/ARMCallingConv.h
-llvm/lib/Target/ARM/ARMHazardRecognizer.h
-llvm/lib/Target/ARM/ARMInstrInfo.cpp
-llvm/lib/Target/ARM/ARMMachineFunctionInfo.cpp
-llvm/lib/Target/ARM/ARMTargetMachine.h
-llvm/lib/Target/ARM/ARMTargetObjectFile.h
-llvm/lib/Target/ARM/MVETailPredUtils.h
-llvm/lib/Target/ARM/MVEVPTBlockPass.cpp
-llvm/lib/Target/ARM/MCTargetDesc/ARMFixupKinds.h
-llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
-llvm/lib/Target/ARM/TargetInfo/ARMTargetInfo.cpp
-llvm/lib/Target/ARM/TargetInfo/ARMTargetInfo.h
-llvm/lib/Target/AVR/AVR.h
-llvm/lib/Target/AVR/AVRAsmPrinter.cpp
-llvm/lib/Target/AVR/AVRExpandPseudoInsts.cpp
-llvm/lib/Target/AVR/AVRFrameLowering.h
-llvm/lib/Target/AVR/AVRInstrInfo.cpp
-llvm/lib/Target/AVR/AVRInstrInfo.h
-llvm/lib/Target/AVR/AVRISelDAGToDAG.cpp
-llvm/lib/Target/AVR/AVRISelLowering.cpp
-llvm/lib/Target/AVR/AVRISelLowering.h
-llvm/lib/Target/AVR/AVRMachineFunctionInfo.h
-llvm/lib/Target/AVR/AVRMCInstLower.cpp
-llvm/lib/Target/AVR/AVRMCInstLower.h
-llvm/lib/Target/AVR/AVRRegisterInfo.cpp
-llvm/lib/Target/AVR/AVRRegisterInfo.h
-llvm/lib/Target/AVR/AVRSelectionDAGInfo.h
-llvm/lib/Target/AVR/AVRShiftExpand.cpp
-llvm/lib/Target/AVR/AVRSubtarget.cpp
-llvm/lib/Target/AVR/AVRSubtarget.h
-llvm/lib/Target/AVR/AVRTargetMachine.cpp
-llvm/lib/Target/AVR/AVRTargetMachine.h
-llvm/lib/Target/AVR/AVRTargetObjectFile.cpp
-llvm/lib/Target/AVR/AVRTargetObjectFile.h
-llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
-llvm/lib/Target/AVR/Disassembler/AVRDisassembler.cpp
-llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
-llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
-llvm/lib/Target/AVR/MCTargetDesc/AVRELFObjectWriter.cpp
-llvm/lib/Target/AVR/MCTargetDesc/AVRELFStreamer.cpp
-llvm/lib/Target/AVR/MCTargetDesc/AVRELFStreamer.h
-llvm/lib/Target/AVR/MCTargetDesc/AVRFixupKinds.h
-llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
-llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.h
-llvm/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.cpp
-llvm/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.h
-llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.h
-llvm/lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.cpp
-llvm/lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.h
-llvm/lib/Target/AVR/MCTargetDesc/AVRMCExpr.cpp
-llvm/lib/Target/AVR/MCTargetDesc/AVRMCExpr.h
-llvm/lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.cpp
-llvm/lib/Target/AVR/MCTargetDesc/AVRTargetStreamer.cpp
-llvm/lib/Target/AVR/MCTargetDesc/AVRTargetStreamer.h
-llvm/lib/Target/AVR/TargetInfo/AVRTargetInfo.cpp
-llvm/lib/Target/AVR/TargetInfo/AVRTargetInfo.h
-llvm/lib/Target/BPF/BPFAdjustOpt.cpp
-llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
-llvm/lib/Target/BPF/BPFCORE.h
-llvm/lib/Target/BPF/BPFFrameLowering.cpp
-llvm/lib/Target/BPF/BPFIRPeephole.cpp
-llvm/lib/Target/BPF/BPFMCInstLower.cpp
-llvm/lib/Target/BPF/BPFPreserveDIType.cpp
-llvm/lib/Target/BPF/BPFSelectionDAGInfo.cpp
-llvm/lib/Target/BPF/BPFSubtarget.cpp
-llvm/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp
-llvm/lib/Target/BPF/TargetInfo/BPFTargetInfo.h
-llvm/lib/Target/CSKY/CSKY.h
-llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp
-llvm/lib/Target/CSKY/CSKYAsmPrinter.h
-llvm/lib/Target/CSKY/CSKYCallingConv.h
-llvm/lib/Target/CSKY/CSKYConstantIslandPass.cpp
-llvm/lib/Target/CSKY/CSKYConstantPoolValue.cpp
-llvm/lib/Target/CSKY/CSKYConstantPoolValue.h
-llvm/lib/Target/CSKY/CSKYFrameLowering.cpp
-llvm/lib/Target/CSKY/CSKYFrameLowering.h
-llvm/lib/Target/CSKY/CSKYInstrInfo.cpp
-llvm/lib/Target/CSKY/CSKYInstrInfo.h
-llvm/lib/Target/CSKY/CSKYISelDAGToDAG.cpp
-llvm/lib/Target/CSKY/CSKYISelLowering.cpp
-llvm/lib/Target/CSKY/CSKYISelLowering.h
-llvm/lib/Target/CSKY/CSKYMachineFunctionInfo.h
-llvm/lib/Target/CSKY/CSKYMCInstLower.cpp
-llvm/lib/Target/CSKY/CSKYMCInstLower.h
-llvm/lib/Target/CSKY/CSKYRegisterInfo.cpp
-llvm/lib/Target/CSKY/CSKYRegisterInfo.h
-llvm/lib/Target/CSKY/CSKYSubtarget.cpp
-llvm/lib/Target/CSKY/CSKYSubtarget.h
-llvm/lib/Target/CSKY/CSKYTargetMachine.cpp
-llvm/lib/Target/CSKY/CSKYTargetMachine.h
-llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
-llvm/lib/Target/CSKY/Disassembler/CSKYDisassembler.cpp
-llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
-llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h
-llvm/lib/Target/CSKY/MCTargetDesc/CSKYBaseInfo.h
-llvm/lib/Target/CSKY/MCTargetDesc/CSKYELFObjectWriter.cpp
-llvm/lib/Target/CSKY/MCTargetDesc/CSKYFixupKinds.h
-llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.cpp
-llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.h
-llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.cpp
-llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.h
-llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp
-llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.h
-llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.cpp
-llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.h
-llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCTargetDesc.cpp
-llvm/lib/Target/CSKY/TargetInfo/CSKYTargetInfo.cpp
-llvm/lib/Target/CSKY/TargetInfo/CSKYTargetInfo.h
-llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.h
-llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp
-llvm/lib/Target/Hexagon/HexagonMachineScheduler.h
-llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
-llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
-llvm/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.h
-llvm/lib/Target/Hexagon/MCTargetDesc/HexagonFixupKinds.h
-llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.h
-llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h
-llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.h
-llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.h
-llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCShuffler.h
-llvm/lib/Target/Hexagon/TargetInfo/HexagonTargetInfo.h
-llvm/lib/Target/Lanai/Lanai.h
-llvm/lib/Target/Lanai/LanaiAluCode.h
-llvm/lib/Target/Lanai/LanaiAsmPrinter.cpp
-llvm/lib/Target/Lanai/LanaiCondCode.h
-llvm/lib/Target/Lanai/LanaiDelaySlotFiller.cpp
-llvm/lib/Target/Lanai/LanaiFrameLowering.cpp
-llvm/lib/Target/Lanai/LanaiFrameLowering.h
-llvm/lib/Target/Lanai/LanaiISelDAGToDAG.cpp
-llvm/lib/Target/Lanai/LanaiISelLowering.h
-llvm/lib/Target/Lanai/LanaiMachineFunctionInfo.cpp
-llvm/lib/Target/Lanai/LanaiMachineFunctionInfo.h
-llvm/lib/Target/Lanai/LanaiMCInstLower.h
-llvm/lib/Target/Lanai/LanaiRegisterInfo.cpp
-llvm/lib/Target/Lanai/LanaiRegisterInfo.h
-llvm/lib/Target/Lanai/LanaiSelectionDAGInfo.cpp
-llvm/lib/Target/Lanai/LanaiSelectionDAGInfo.h
-llvm/lib/Target/Lanai/LanaiSubtarget.cpp
-llvm/lib/Target/Lanai/LanaiSubtarget.h
-llvm/lib/Target/Lanai/LanaiTargetMachine.cpp
-llvm/lib/Target/Lanai/LanaiTargetObjectFile.h
-llvm/lib/Target/Lanai/Disassembler/LanaiDisassembler.cpp
-llvm/lib/Target/Lanai/Disassembler/LanaiDisassembler.h
-llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
-llvm/lib/Target/Lanai/MCTargetDesc/LanaiBaseInfo.h
-llvm/lib/Target/Lanai/MCTargetDesc/LanaiELFObjectWriter.cpp
-llvm/lib/Target/Lanai/MCTargetDesc/LanaiFixupKinds.h
-llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.h
-llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.cpp
-llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.h
-llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
-llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.cpp
-llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.h
-llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCTargetDesc.cpp
-llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCTargetDesc.h
-llvm/lib/Target/Lanai/TargetInfo/LanaiTargetInfo.cpp
-llvm/lib/Target/Lanai/TargetInfo/LanaiTargetInfo.h
-llvm/lib/Target/LoongArch/LoongArch.h
-llvm/lib/Target/LoongArch/LoongArchAsmPrinter.cpp
-llvm/lib/Target/LoongArch/LoongArchAsmPrinter.h
-llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp
-llvm/lib/Target/LoongArch/LoongArchFrameLowering.h
-llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp
-llvm/lib/Target/LoongArch/LoongArchInstrInfo.h
-llvm/lib/Target/LoongArch/LoongArchISelDAGToDAG.cpp
-llvm/lib/Target/LoongArch/LoongArchISelDAGToDAG.h
-llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
-llvm/lib/Target/LoongArch/LoongArchISelLowering.h
-llvm/lib/Target/LoongArch/LoongArchMachineFunctionInfo.h
-llvm/lib/Target/LoongArch/LoongArchMCInstLower.cpp
-llvm/lib/Target/LoongArch/LoongArchRegisterInfo.cpp
-llvm/lib/Target/LoongArch/LoongArchRegisterInfo.h
-llvm/lib/Target/LoongArch/LoongArchSubtarget.cpp
-llvm/lib/Target/LoongArch/LoongArchSubtarget.h
-llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp
-llvm/lib/Target/LoongArch/LoongArchTargetMachine.h
-llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
-llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
-llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.cpp
-llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.h
-llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
-llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.cpp
-llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.h
-llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.cpp
-llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.h
-llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
-llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCTargetDesc.cpp
-llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCTargetDesc.h
-llvm/lib/Target/LoongArch/TargetInfo/LoongArchTargetInfo.cpp
-llvm/lib/Target/LoongArch/TargetInfo/LoongArchTargetInfo.h
-llvm/lib/Target/M68k/M68k.h
-llvm/lib/Target/M68k/M68kAsmPrinter.cpp
-llvm/lib/Target/M68k/M68kAsmPrinter.h
-llvm/lib/Target/M68k/M68kCallingConv.h
-llvm/lib/Target/M68k/M68kCollapseMOVEMPass.cpp
-llvm/lib/Target/M68k/M68kExpandPseudo.cpp
-llvm/lib/Target/M68k/M68kFrameLowering.cpp
-llvm/lib/Target/M68k/M68kFrameLowering.h
-llvm/lib/Target/M68k/M68kInstrBuilder.h
-llvm/lib/Target/M68k/M68kInstrInfo.cpp
-llvm/lib/Target/M68k/M68kInstrInfo.h
-llvm/lib/Target/M68k/M68kISelDAGToDAG.cpp
-llvm/lib/Target/M68k/M68kISelLowering.h
-llvm/lib/Target/M68k/M68kMachineFunction.cpp
-llvm/lib/Target/M68k/M68kMachineFunction.h
-llvm/lib/Target/M68k/M68kMCInstLower.cpp
-llvm/lib/Target/M68k/M68kMCInstLower.h
-llvm/lib/Target/M68k/M68kRegisterInfo.cpp
-llvm/lib/Target/M68k/M68kRegisterInfo.h
-llvm/lib/Target/M68k/M68kSubtarget.cpp
-llvm/lib/Target/M68k/M68kSubtarget.h
-llvm/lib/Target/M68k/M68kTargetMachine.cpp
-llvm/lib/Target/M68k/M68kTargetMachine.h
-llvm/lib/Target/M68k/M68kTargetObjectFile.cpp
-llvm/lib/Target/M68k/M68kTargetObjectFile.h
-llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp
-llvm/lib/Target/M68k/Disassembler/M68kDisassembler.cpp
-llvm/lib/Target/M68k/GISel/M68kCallLowering.h
-llvm/lib/Target/M68k/GISel/M68kInstructionSelector.cpp
-llvm/lib/Target/M68k/GISel/M68kLegalizerInfo.cpp
-llvm/lib/Target/M68k/GISel/M68kLegalizerInfo.h
-llvm/lib/Target/M68k/GISel/M68kRegisterBankInfo.cpp
-llvm/lib/Target/M68k/GISel/M68kRegisterBankInfo.h
-llvm/lib/Target/M68k/MCTargetDesc/M68kAsmBackend.cpp
-llvm/lib/Target/M68k/MCTargetDesc/M68kBaseInfo.h
-llvm/lib/Target/M68k/MCTargetDesc/M68kELFObjectWriter.cpp
-llvm/lib/Target/M68k/MCTargetDesc/M68kFixupKinds.h
-llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.cpp
-llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.h
-llvm/lib/Target/M68k/MCTargetDesc/M68kMCAsmInfo.cpp
-llvm/lib/Target/M68k/MCTargetDesc/M68kMCAsmInfo.h
-llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp
-llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.h
-llvm/lib/Target/M68k/MCTargetDesc/M68kMCTargetDesc.cpp
-llvm/lib/Target/M68k/TargetInfo/M68kTargetInfo.cpp
-llvm/lib/Target/M68k/TargetInfo/M68kTargetInfo.h
-llvm/lib/Target/Mips/Mips16RegisterInfo.h
-llvm/lib/Target/Mips/MipsCallLowering.h
-llvm/lib/Target/Mips/MipsLegalizerInfo.h
-llvm/lib/Target/Mips/MipsMCInstLower.h
-llvm/lib/Target/Mips/MipsMulMulBugPass.cpp
-llvm/lib/Target/Mips/MipsOptionRecord.h
-llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
-llvm/lib/Target/Mips/MipsRegisterBankInfo.h
-llvm/lib/Target/Mips/MipsSEFrameLowering.h
-llvm/lib/Target/Mips/MipsSERegisterInfo.h
-llvm/lib/Target/Mips/MipsTargetMachine.cpp
-llvm/lib/Target/Mips/MipsTargetMachine.h
-llvm/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.cpp
-llvm/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.h
-llvm/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp
-llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.h
-llvm/lib/Target/Mips/MCTargetDesc/MipsMCExpr.h
-llvm/lib/Target/Mips/MCTargetDesc/MipsOptionRecord.cpp
-llvm/lib/Target/Mips/TargetInfo/MipsTargetInfo.cpp
-llvm/lib/Target/Mips/TargetInfo/MipsTargetInfo.h
-llvm/lib/Target/MSP430/MCTargetDesc/MSP430FixupKinds.h
-llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.h
-llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp
-llvm/lib/Target/MSP430/TargetInfo/MSP430TargetInfo.cpp
-llvm/lib/Target/MSP430/TargetInfo/MSP430TargetInfo.h
-llvm/lib/Target/NVPTX/ManagedStringPool.h
-llvm/lib/Target/NVPTX/NVPTXAllocaHoisting.cpp
-llvm/lib/Target/NVPTX/NVPTXAllocaHoisting.h
-llvm/lib/Target/NVPTX/NVPTXAtomicLower.cpp
-llvm/lib/Target/NVPTX/NVPTXAtomicLower.h
-llvm/lib/Target/NVPTX/NVPTXFrameLowering.cpp
-llvm/lib/Target/NVPTX/NVPTXGenericToNVVM.cpp
-llvm/lib/Target/NVPTX/NVPTXISelLowering.h
-llvm/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp
-llvm/lib/Target/NVPTX/NVPTXTargetObjectFile.h
-llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h
-llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.cpp
-llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.h
-llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp
-llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.h
-llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXTargetStreamer.h
-llvm/lib/Target/NVPTX/TargetInfo/NVPTXTargetInfo.cpp
-llvm/lib/Target/NVPTX/TargetInfo/NVPTXTargetInfo.h
-llvm/lib/Target/PowerPC/PPCExpandAtomicPseudoInsts.cpp
-llvm/lib/Target/PowerPC/PPCGenScalarMASSEntries.cpp
-llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp
-llvm/lib/Target/PowerPC/PPCTargetMachine.h
-llvm/lib/Target/PowerPC/PPCTargetStreamer.h
-llvm/lib/Target/PowerPC/GISel/PPCCallLowering.cpp
-llvm/lib/Target/PowerPC/GISel/PPCCallLowering.h
-llvm/lib/Target/PowerPC/GISel/PPCInstructionSelector.cpp
-llvm/lib/Target/PowerPC/GISel/PPCLegalizerInfo.cpp
-llvm/lib/Target/PowerPC/GISel/PPCLegalizerInfo.h
-llvm/lib/Target/PowerPC/GISel/PPCRegisterBankInfo.cpp
-llvm/lib/Target/PowerPC/GISel/PPCRegisterBankInfo.h
-llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.h
-llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
-llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp
-llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFStreamer.cpp
-llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFStreamer.h
-llvm/lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.cpp
-llvm/lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.h
-llvm/lib/Target/RISCV/RISCVCallLowering.cpp
-llvm/lib/Target/RISCV/RISCVCallLowering.h
-llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
-llvm/lib/Target/RISCV/RISCVInstructionSelector.cpp
-llvm/lib/Target/RISCV/RISCVLegalizerInfo.cpp
-llvm/lib/Target/RISCV/RISCVLegalizerInfo.h
-llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
-llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
-llvm/lib/Target/RISCV/RISCVRedundantCopyElimination.cpp
-llvm/lib/Target/RISCV/RISCVRegisterBankInfo.cpp
-llvm/lib/Target/RISCV/RISCVRegisterBankInfo.h
-llvm/lib/Target/RISCV/RISCVSExtWRemoval.cpp
-llvm/lib/Target/RISCV/RISCVTargetMachine.h
-llvm/lib/Target/RISCV/RISCVTargetObjectFile.cpp
-llvm/lib/Target/RISCV/RISCVTargetObjectFile.h
-llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
-llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
-llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
-llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.cpp
-llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
-llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
-llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
-llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h
-llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp
-llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.h
-llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
-llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h
-llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCObjectFileInfo.cpp
-llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCObjectFileInfo.h
-llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp
-llvm/lib/Target/RISCV/TargetInfo/RISCVTargetInfo.cpp
-llvm/lib/Target/RISCV/TargetInfo/RISCVTargetInfo.h
-llvm/lib/Target/Sparc/LeonPasses.h
-llvm/lib/Target/Sparc/SparcTargetObjectFile.cpp
-llvm/lib/Target/Sparc/SparcTargetObjectFile.h
-llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h
-llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
-llvm/lib/Target/Sparc/MCTargetDesc/SparcTargetStreamer.cpp
-llvm/lib/Target/Sparc/MCTargetDesc/SparcTargetStreamer.h
-llvm/lib/Target/Sparc/TargetInfo/SparcTargetInfo.cpp
-llvm/lib/Target/Sparc/TargetInfo/SparcTargetInfo.h
-llvm/lib/Target/SystemZ/SystemZAsmPrinter.h
-llvm/lib/Target/SystemZ/SystemZInstrBuilder.h
-llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
-llvm/lib/Target/SystemZ/SystemZSubtarget.cpp
-llvm/lib/Target/SystemZ/SystemZTargetStreamer.h
-llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
-llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
-llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.h
-llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCFixups.h
-llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.h
-llvm/lib/Target/SystemZ/TargetInfo/SystemZTargetInfo.cpp
-llvm/lib/Target/SystemZ/TargetInfo/SystemZTargetInfo.h
-llvm/lib/Target/VE/LVLGen.cpp
-llvm/lib/Target/VE/VEAsmPrinter.cpp
-llvm/lib/Target/VE/VECustomDAG.cpp
-llvm/lib/Target/VE/VECustomDAG.h
-llvm/lib/Target/VE/VEFrameLowering.h
-llvm/lib/Target/VE/VEInstrBuilder.h
-llvm/lib/Target/VE/VEInstrInfo.h
-llvm/lib/Target/VE/VEISelDAGToDAG.cpp
-llvm/lib/Target/VE/VEMachineFunctionInfo.cpp
-llvm/lib/Target/VE/VEMachineFunctionInfo.h
-llvm/lib/Target/VE/VEMCInstLower.cpp
-llvm/lib/Target/VE/VERegisterInfo.cpp
-llvm/lib/Target/VE/VERegisterInfo.h
-llvm/lib/Target/VE/VESubtarget.cpp
-llvm/lib/Target/VE/VESubtarget.h
-llvm/lib/Target/VE/VETargetMachine.cpp
-llvm/lib/Target/VE/VETargetMachine.h
-llvm/lib/Target/VE/VETargetTransformInfo.h
-llvm/lib/Target/VE/VVPISelLowering.cpp
-llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
-llvm/lib/Target/VE/Disassembler/VEDisassembler.cpp
-llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
-llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp
-llvm/lib/Target/VE/MCTargetDesc/VEFixupKinds.h
-llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.cpp
-llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.h
-llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.cpp
-llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.h
-llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
-llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp
-llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.h
-llvm/lib/Target/VE/MCTargetDesc/VEMCTargetDesc.cpp
-llvm/lib/Target/VE/MCTargetDesc/VEMCTargetDesc.h
-llvm/lib/Target/VE/MCTargetDesc/VETargetStreamer.cpp
-llvm/lib/Target/VE/MCTargetDesc/VETargetStreamer.h
-llvm/lib/Target/VE/TargetInfo/VETargetInfo.h
-llvm/lib/Target/WebAssembly/WebAssembly.h
-llvm/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyArgumentMove.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h
-llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyDebugFixup.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.h
-llvm/lib/Target/WebAssembly/WebAssemblyExceptionInfo.h
-llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyFixIrreducibleControlFlow.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyFrameLowering.h
-llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.h
-llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
-llvm/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyLowerBrUnless.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyLowerRefTypesIntPtrConv.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h
-llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.h
-llvm/lib/Target/WebAssembly/WebAssemblyMemIntrinsicResults.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyNullifyDebugValueLists.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyOptimizeLiveIntervals.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyOptimizeReturned.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyPeephole.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyPrepareForLiveIntervals.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyRegColoring.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyRegisterInfo.h
-llvm/lib/Target/WebAssembly/WebAssemblyRegNumbering.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyReplacePhysRegs.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h
-llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h
-llvm/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
-llvm/lib/Target/WebAssembly/WebAssemblySortRegion.h
-llvm/lib/Target/WebAssembly/WebAssemblySubtarget.cpp
-llvm/lib/Target/WebAssembly/WebAssemblySubtarget.h
-llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.h
-llvm/lib/Target/WebAssembly/WebAssemblyTargetObjectFile.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyTargetObjectFile.h
-llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
-llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
-llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
-llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
-llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyFixupKinds.h
-llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
-llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h
-llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp
-llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.h
-llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
-llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
-llvm/lib/Target/WebAssembly/TargetInfo/WebAssemblyTargetInfo.cpp
-llvm/lib/Target/WebAssembly/TargetInfo/WebAssemblyTargetInfo.h
-llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.cpp
-llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.h
-llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.cpp
-llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.h
-llvm/lib/Target/X86/X86CallLowering.cpp
-llvm/lib/Target/X86/X86CallLowering.h
-llvm/lib/Target/X86/X86FastTileConfig.cpp
-llvm/lib/Target/X86/X86InsertPrefetch.cpp
-llvm/lib/Target/X86/X86InsertWait.cpp
-llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
-llvm/lib/Target/X86/X86InterleavedAccess.cpp
-llvm/lib/Target/X86/X86LegalizerInfo.h
-llvm/lib/Target/X86/X86LoadValueInjectionRetHardening.cpp
-llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp
-llvm/lib/Target/X86/X86LowerAMXType.cpp
-llvm/lib/Target/X86/X86LowerTileCopy.cpp
-llvm/lib/Target/X86/X86PreTileConfig.cpp
-llvm/lib/Target/X86/X86RegisterBankInfo.h
-llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp
-llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp
-llvm/lib/Target/X86/X86TargetMachine.h
-llvm/lib/Target/X86/X86TileConfig.cpp
-llvm/lib/Target/X86/MCA/X86CustomBehaviour.cpp
-llvm/lib/Target/X86/MCA/X86CustomBehaviour.h
-llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
-llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
-llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h
-llvm/lib/Target/X86/MCTargetDesc/X86MCExpr.h
-llvm/lib/Target/X86/MCTargetDesc/X86TargetStreamer.h
-llvm/lib/Target/X86/TargetInfo/X86TargetInfo.cpp
-llvm/lib/Target/XCore/XCoreTargetMachine.h
-llvm/lib/Target/XCore/XCoreTargetTransformInfo.h
-llvm/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.h
-llvm/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp
-llvm/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.h
-llvm/lib/Target/XCore/TargetInfo/XCoreTargetInfo.cpp
-llvm/lib/Testing/Annotations/Annotations.cpp
-llvm/lib/Testing/Support/Error.cpp
-llvm/lib/Testing/Support/SupportHelpers.cpp
-llvm/lib/TextAPI/ArchitectureSet.cpp
-llvm/lib/TextAPI/InterfaceFile.cpp
-llvm/lib/TextAPI/PackedVersion.cpp
-llvm/lib/TextAPI/Platform.cpp
-llvm/lib/TextAPI/Symbol.cpp
-llvm/lib/TextAPI/Target.cpp
-llvm/lib/TextAPI/TextAPIContext.h
-llvm/lib/TextAPI/TextStub.cpp
-llvm/lib/TextAPI/TextStubCommon.cpp
-llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombineInternal.h
-llvm/lib/Transforms/CFGuard/CFGuard.cpp
-llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
-llvm/lib/Transforms/Instrumentation/CFGMST.h
-llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
-llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
-llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
-llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
-llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
-llvm/lib/Transforms/Instrumentation/ValueProfileCollector.h
-llvm/lib/Transforms/IPO/Annotation2Metadata.cpp
-llvm/lib/Transforms/IPO/Attributor.cpp
-llvm/lib/Transforms/IPO/ElimAvailExtern.cpp
-llvm/lib/Transforms/IPO/ModuleInliner.cpp
-llvm/lib/Transforms/IPO/OpenMPOpt.cpp
-llvm/lib/Transforms/IPO/SampleContextTracker.cpp
-llvm/lib/Transforms/IPO/SampleProfileProbe.cpp
-llvm/lib/Transforms/IPO/StripDeadPrototypes.cpp
-llvm/lib/Transforms/ObjCARC/BlotMapVector.h
-llvm/lib/Transforms/ObjCARC/ObjCARCExpand.cpp
-llvm/lib/Transforms/ObjCARC/ProvenanceAnalysis.h
-llvm/lib/Transforms/ObjCARC/PtrState.h
-llvm/lib/Transforms/Scalar/AnnotationRemarks.cpp
-llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
-llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp
-llvm/lib/Transforms/Scalar/DivRemPairs.cpp
-llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp
-llvm/lib/Transforms/Scalar/GVNHoist.cpp
-llvm/lib/Transforms/Scalar/InstSimplifyPass.cpp
-llvm/lib/Transforms/Scalar/IVUsersPrinter.cpp
-llvm/lib/Transforms/Scalar/LoopAccessAnalysisPrinter.cpp
-llvm/lib/Transforms/Scalar/LoopBoundSplit.cpp
-llvm/lib/Transforms/Scalar/LoopInstSimplify.cpp
-llvm/lib/Transforms/Scalar/LoopPassManager.cpp
-llvm/lib/Transforms/Scalar/LoopRotation.cpp
-llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
-llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp
-llvm/lib/Transforms/Scalar/WarnMissedTransforms.cpp
-llvm/lib/Transforms/Utils/CallGraphUpdater.cpp
-llvm/lib/Transforms/Utils/CanonicalizeAliases.cpp
-llvm/lib/Transforms/Utils/CanonicalizeFreezeInLoops.cpp
-llvm/lib/Transforms/Utils/CodeLayout.cpp
-llvm/lib/Transforms/Utils/CodeMoverUtils.cpp
-llvm/lib/Transforms/Utils/EscapeEnumerator.cpp
-llvm/lib/Transforms/Utils/FunctionComparator.cpp
-llvm/lib/Transforms/Utils/FunctionImportUtils.cpp
-llvm/lib/Transforms/Utils/GlobalStatus.cpp
-llvm/lib/Transforms/Utils/HelloWorld.cpp
-llvm/lib/Transforms/Utils/InjectTLIMappings.cpp
-llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp
-llvm/lib/Transforms/Utils/MatrixUtils.cpp
-llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp
-llvm/lib/Transforms/Utils/SampleProfileInference.cpp
-llvm/lib/Transforms/Utils/SampleProfileLoaderBaseUtil.cpp
-llvm/lib/Transforms/Utils/SCCPSolver.cpp
-llvm/lib/Transforms/Utils/SSAUpdaterBulk.cpp
-llvm/lib/Transforms/Utils/Utils.cpp
-llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
-llvm/lib/Transforms/Vectorize/Vectorize.cpp
-llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h
-llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
-llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
-llvm/lib/Transforms/Vectorize/VPlanLoopInfo.h
-llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
-llvm/lib/Transforms/Vectorize/VPlanPredicator.h
-llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
-llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
-llvm/lib/Transforms/Vectorize/VPlanTransforms.h
-llvm/lib/Transforms/Vectorize/VPlanValue.h
-llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
-llvm/lib/WindowsDriver/MSVCPaths.cpp
-llvm/lib/WindowsManifest/WindowsManifestMerger.cpp
-llvm/lib/XRay/BlockIndexer.cpp
-llvm/lib/XRay/BlockVerifier.cpp
-llvm/lib/XRay/FDRRecordProducer.cpp
-llvm/lib/XRay/FDRRecords.cpp
-llvm/lib/XRay/FDRTraceExpander.cpp
-llvm/lib/XRay/FileHeaderReader.cpp
-llvm/lib/XRay/InstrumentationMap.cpp
-llvm/lib/XRay/LogBuilderConsumer.cpp
-llvm/lib/XRay/Profile.cpp
-llvm/lib/XRay/RecordPrinter.cpp
-llvm/lib/XRay/Trace.cpp
-llvm/tools/bugpoint/Miscompilation.cpp
-llvm/tools/dsymutil/BinaryHolder.cpp
-llvm/tools/dsymutil/BinaryHolder.h
-llvm/tools/dsymutil/CFBundle.cpp
-llvm/tools/dsymutil/CFBundle.h
-llvm/tools/dsymutil/DebugMap.cpp
-llvm/tools/dsymutil/DebugMap.h
-llvm/tools/dsymutil/dsymutil.cpp
-llvm/tools/dsymutil/dsymutil.h
-llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
-llvm/tools/dsymutil/LinkUtils.h
-llvm/tools/dsymutil/MachODebugMapParser.cpp
-llvm/tools/dsymutil/MachOUtils.h
-llvm/tools/dsymutil/Reproducer.cpp
-llvm/tools/dsymutil/Reproducer.h
-llvm/tools/dsymutil/SymbolMap.cpp
-llvm/tools/dsymutil/SymbolMap.h
-llvm/tools/lli/ExecutionUtils.cpp
-llvm/tools/lli/ExecutionUtils.h
-llvm/tools/lli/ForwardingMemoryManager.h
-llvm/tools/lli/ChildTarget/ChildTarget.cpp
-llvm/tools/llvm-as-fuzzer/llvm-as-fuzzer.cpp
-llvm/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp
-llvm/tools/llvm-cfi-verify/lib/GraphBuilder.h
-llvm/tools/llvm-cov/CoverageExporter.h
-llvm/tools/llvm-cov/CoverageExporterJson.h
-llvm/tools/llvm-cov/CoverageExporterLcov.cpp
-llvm/tools/llvm-cov/CoverageExporterLcov.h
-llvm/tools/llvm-cov/CoverageFilters.cpp
-llvm/tools/llvm-cov/CoverageSummaryInfo.cpp
-llvm/tools/llvm-cov/CoverageSummaryInfo.h
-llvm/tools/llvm-cov/llvm-cov.cpp
-llvm/tools/llvm-cov/RenderingSupport.h
-llvm/tools/llvm-cov/SourceCoverageViewHTML.h
-llvm/tools/llvm-cov/SourceCoverageViewText.h
-llvm/tools/llvm-cov/TestingSupport.cpp
-llvm/tools/llvm-cxxdump/Error.cpp
-llvm/tools/llvm-cxxfilt/llvm-cxxfilt.cpp
-llvm/tools/llvm-debuginfod-find/llvm-debuginfod-find.cpp
-llvm/tools/llvm-dis-fuzzer/llvm-dis-fuzzer.cpp
-llvm/tools/llvm-dlang-demangle-fuzzer/DummyDemanglerFuzzer.cpp
-llvm/tools/llvm-dlang-demangle-fuzzer/llvm-dlang-demangle-fuzzer.cpp
-llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
-llvm/tools/llvm-dwarfdump/llvm-dwarfdump.h
-llvm/tools/llvm-dwarfdump/SectionSizes.cpp
-llvm/tools/llvm-exegesis/lib/Analysis.h
-llvm/tools/llvm-exegesis/lib/Assembler.h
-llvm/tools/llvm-exegesis/lib/BenchmarkCode.h
-llvm/tools/llvm-exegesis/lib/BenchmarkResult.h
-llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
-llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h
-llvm/tools/llvm-exegesis/lib/Clustering.h
-llvm/tools/llvm-exegesis/lib/CodeTemplate.h
-llvm/tools/llvm-exegesis/lib/Error.cpp
-llvm/tools/llvm-exegesis/lib/Error.h
-llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp
-llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h
-llvm/tools/llvm-exegesis/lib/MCInstrDescView.h
-llvm/tools/llvm-exegesis/lib/ParallelSnippetGenerator.cpp
-llvm/tools/llvm-exegesis/lib/ParallelSnippetGenerator.h
-llvm/tools/llvm-exegesis/lib/PerfHelper.h
-llvm/tools/llvm-exegesis/lib/RegisterAliasing.cpp
-llvm/tools/llvm-exegesis/lib/RegisterAliasing.h
-llvm/tools/llvm-exegesis/lib/RegisterValue.cpp
-llvm/tools/llvm-exegesis/lib/RegisterValue.h
-llvm/tools/llvm-exegesis/lib/SchedClassResolution.cpp
-llvm/tools/llvm-exegesis/lib/SchedClassResolution.h
-llvm/tools/llvm-exegesis/lib/SerialSnippetGenerator.h
-llvm/tools/llvm-exegesis/lib/SnippetFile.cpp
-llvm/tools/llvm-exegesis/lib/SnippetFile.h
-llvm/tools/llvm-exegesis/lib/SnippetGenerator.cpp
-llvm/tools/llvm-exegesis/lib/SnippetGenerator.h
-llvm/tools/llvm-exegesis/lib/SnippetRepetitor.cpp
-llvm/tools/llvm-exegesis/lib/SnippetRepetitor.h
-llvm/tools/llvm-exegesis/lib/Target.h
-llvm/tools/llvm-exegesis/lib/TargetSelect.h
-llvm/tools/llvm-exegesis/lib/UopsBenchmarkRunner.cpp
-llvm/tools/llvm-exegesis/lib/UopsBenchmarkRunner.h
-llvm/tools/llvm-exegesis/lib/AArch64/Target.cpp
-llvm/tools/llvm-exegesis/lib/PowerPC/Target.cpp
-llvm/tools/llvm-exegesis/lib/X86/X86Counter.cpp
-llvm/tools/llvm-exegesis/lib/X86/X86Counter.h
-llvm/tools/llvm-gsymutil/llvm-gsymutil.cpp
-llvm/tools/llvm-ifs/ErrorCollector.cpp
-llvm/tools/llvm-ifs/ErrorCollector.h
-llvm/tools/llvm-isel-fuzzer/DummyISelFuzzer.cpp
-llvm/tools/llvm-itanium-demangle-fuzzer/DummyDemanglerFuzzer.cpp
-llvm/tools/llvm-jitlink/llvm-jitlink-macho.cpp
-llvm/tools/llvm-jitlink/llvm-jitlink.h
-llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp
-llvm/tools/llvm-libtool-darwin/llvm-libtool-darwin.cpp
-llvm/tools/llvm-link/llvm-link.cpp
-llvm/tools/llvm-mc/Disassembler.h
-llvm/tools/llvm-mca/CodeRegion.cpp
-llvm/tools/llvm-mca/CodeRegion.h
-llvm/tools/llvm-mca/CodeRegionGenerator.cpp
-llvm/tools/llvm-mca/CodeRegionGenerator.h
-llvm/tools/llvm-mca/llvm-mca.cpp
-llvm/tools/llvm-mca/PipelinePrinter.cpp
-llvm/tools/llvm-mca/PipelinePrinter.h
-llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
-llvm/tools/llvm-mca/Views/DispatchStatistics.cpp
-llvm/tools/llvm-mca/Views/DispatchStatistics.h
-llvm/tools/llvm-mca/Views/InstructionInfoView.cpp
-llvm/tools/llvm-mca/Views/InstructionInfoView.h
-llvm/tools/llvm-mca/Views/InstructionView.cpp
-llvm/tools/llvm-mca/Views/InstructionView.h
-llvm/tools/llvm-mca/Views/RegisterFileStatistics.cpp
-llvm/tools/llvm-mca/Views/RegisterFileStatistics.h
-llvm/tools/llvm-mca/Views/ResourcePressureView.cpp
-llvm/tools/llvm-mca/Views/ResourcePressureView.h
-llvm/tools/llvm-mca/Views/RetireControlUnitStatistics.cpp
-llvm/tools/llvm-mca/Views/RetireControlUnitStatistics.h
-llvm/tools/llvm-mca/Views/SchedulerStatistics.cpp
-llvm/tools/llvm-mca/Views/SchedulerStatistics.h
-llvm/tools/llvm-mca/Views/SummaryView.cpp
-llvm/tools/llvm-mca/Views/SummaryView.h
-llvm/tools/llvm-mca/Views/TimelineView.cpp
-llvm/tools/llvm-mca/Views/TimelineView.h
-llvm/tools/llvm-microsoft-demangle-fuzzer/DummyDemanglerFuzzer.cpp
-llvm/tools/llvm-microsoft-demangle-fuzzer/llvm-microsoft-demangle-fuzzer.cpp
-llvm/tools/llvm-ml/Disassembler.h
-llvm/tools/llvm-modextract/llvm-modextract.cpp
-llvm/tools/llvm-objcopy/llvm-objcopy.cpp
-llvm/tools/llvm-objcopy/ObjcopyOptions.h
-llvm/tools/llvm-objdump/COFFDump.h
-llvm/tools/llvm-objdump/ELFDump.h
-llvm/tools/llvm-objdump/MachODump.h
-llvm/tools/llvm-objdump/ObjdumpOptID.h
-llvm/tools/llvm-objdump/SourcePrinter.cpp
-llvm/tools/llvm-objdump/SourcePrinter.h
-llvm/tools/llvm-objdump/WasmDump.cpp
-llvm/tools/llvm-objdump/WasmDump.h
-llvm/tools/llvm-objdump/XCOFFDump.cpp
-llvm/tools/llvm-objdump/XCOFFDump.h
-llvm/tools/llvm-pdbutil/BytesOutputStyle.cpp
-llvm/tools/llvm-pdbutil/BytesOutputStyle.h
-llvm/tools/llvm-pdbutil/DumpOutputStyle.h
-llvm/tools/llvm-pdbutil/ExplainOutputStyle.h
-llvm/tools/llvm-pdbutil/FormatUtil.cpp
-llvm/tools/llvm-pdbutil/InputFile.h
-llvm/tools/llvm-pdbutil/MinimalSymbolDumper.h
-llvm/tools/llvm-pdbutil/MinimalTypeDumper.cpp
-llvm/tools/llvm-pdbutil/MinimalTypeDumper.h
-llvm/tools/llvm-pdbutil/PrettyBuiltinDumper.cpp
-llvm/tools/llvm-pdbutil/PrettyEnumDumper.cpp
-llvm/tools/llvm-pdbutil/PrettyExternalSymbolDumper.cpp
-llvm/tools/llvm-pdbutil/PrettyTypeDumper.cpp
-llvm/tools/llvm-pdbutil/TypeReferenceTracker.h
-llvm/tools/llvm-pdbutil/YAMLOutputStyle.h
-llvm/tools/llvm-profgen/CallContext.h
-llvm/tools/llvm-profgen/CSPreInliner.cpp
-llvm/tools/llvm-profgen/CSPreInliner.h
-llvm/tools/llvm-profgen/llvm-profgen.cpp
-llvm/tools/llvm-profgen/PerfReader.cpp
-llvm/tools/llvm-profgen/PerfReader.h
-llvm/tools/llvm-rc/ResourceScriptCppFilter.cpp
-llvm/tools/llvm-rc/ResourceScriptCppFilter.h
-llvm/tools/llvm-rc/ResourceScriptParser.h
-llvm/tools/llvm-rc/ResourceScriptStmt.cpp
-llvm/tools/llvm-rc/ResourceScriptToken.h
-llvm/tools/llvm-rc/ResourceVisitor.h
-llvm/tools/llvm-readobj/ObjDumper.cpp
-llvm/tools/llvm-readobj/WindowsResourceDumper.cpp
-llvm/tools/llvm-readobj/WindowsResourceDumper.h
-llvm/tools/llvm-reduce/DeltaManager.cpp
-llvm/tools/llvm-reduce/DeltaManager.h
-llvm/tools/llvm-reduce/ReducerWorkItem.cpp
-llvm/tools/llvm-reduce/ReducerWorkItem.h
-llvm/tools/llvm-reduce/TestRunner.cpp
-llvm/tools/llvm-reduce/TestRunner.h
-llvm/tools/llvm-reduce/deltas/Delta.cpp
-llvm/tools/llvm-reduce/deltas/Delta.h
-llvm/tools/llvm-reduce/deltas/ReduceAliases.cpp
-llvm/tools/llvm-reduce/deltas/ReduceAliases.h
-llvm/tools/llvm-reduce/deltas/ReduceArguments.h
-llvm/tools/llvm-reduce/deltas/ReduceAttributes.cpp
-llvm/tools/llvm-reduce/deltas/ReduceAttributes.h
-llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.cpp
-llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.h
-llvm/tools/llvm-reduce/deltas/ReduceFunctionBodies.cpp
-llvm/tools/llvm-reduce/deltas/ReduceFunctionBodies.h
-llvm/tools/llvm-reduce/deltas/ReduceFunctions.cpp
-llvm/tools/llvm-reduce/deltas/ReduceFunctions.h
-llvm/tools/llvm-reduce/deltas/ReduceGlobalObjects.cpp
-llvm/tools/llvm-reduce/deltas/ReduceGlobalObjects.h
-llvm/tools/llvm-reduce/deltas/ReduceGlobalValues.cpp
-llvm/tools/llvm-reduce/deltas/ReduceGlobalValues.h
-llvm/tools/llvm-reduce/deltas/ReduceGlobalVarInitializers.cpp
-llvm/tools/llvm-reduce/deltas/ReduceGlobalVarInitializers.h
-llvm/tools/llvm-reduce/deltas/ReduceGlobalVars.cpp
-llvm/tools/llvm-reduce/deltas/ReduceGlobalVars.h
-llvm/tools/llvm-reduce/deltas/ReduceInstructions.cpp
-llvm/tools/llvm-reduce/deltas/ReduceInstructions.h
-llvm/tools/llvm-reduce/deltas/ReduceInstructionsMIR.cpp
-llvm/tools/llvm-reduce/deltas/ReduceInstructionsMIR.h
-llvm/tools/llvm-reduce/deltas/ReduceMetadata.cpp
-llvm/tools/llvm-reduce/deltas/ReduceMetadata.h
-llvm/tools/llvm-reduce/deltas/ReduceModuleData.cpp
-llvm/tools/llvm-reduce/deltas/ReduceModuleData.h
-llvm/tools/llvm-reduce/deltas/ReduceOperandBundles.cpp
-llvm/tools/llvm-reduce/deltas/ReduceOperandBundles.h
-llvm/tools/llvm-reduce/deltas/ReduceOperands.cpp
-llvm/tools/llvm-reduce/deltas/ReduceOperands.h
-llvm/tools/llvm-reduce/deltas/ReduceOperandsSkip.cpp
-llvm/tools/llvm-reduce/deltas/ReduceOperandsSkip.h
-llvm/tools/llvm-reduce/deltas/ReduceOperandsToArgs.cpp
-llvm/tools/llvm-reduce/deltas/ReduceOperandsToArgs.h
-llvm/tools/llvm-reduce/deltas/ReduceSpecialGlobals.cpp
-llvm/tools/llvm-reduce/deltas/ReduceSpecialGlobals.h
-llvm/tools/llvm-rust-demangle-fuzzer/DummyDemanglerFuzzer.cpp
-llvm/tools/llvm-rust-demangle-fuzzer/llvm-rust-demangle-fuzzer.cpp
-llvm/tools/llvm-shlib/libllvm.cpp
-llvm/tools/llvm-special-case-list-fuzzer/DummySpecialCaseListFuzzer.cpp
-llvm/tools/llvm-special-case-list-fuzzer/special-case-list-fuzzer.cpp
-llvm/tools/llvm-strings/llvm-strings.cpp
-llvm/tools/llvm-tapi-diff/DiffEngine.cpp
-llvm/tools/llvm-tapi-diff/DiffEngine.h
-llvm/tools/llvm-tapi-diff/llvm-tapi-diff.cpp
-llvm/tools/llvm-undname/llvm-undname.cpp
-llvm/tools/llvm-xray/func-id-helper.cpp
-llvm/tools/llvm-xray/func-id-helper.h
-llvm/tools/llvm-xray/llvm-xray.cpp
-llvm/tools/llvm-xray/trie-node.h
-llvm/tools/llvm-xray/xray-account.h
-llvm/tools/llvm-xray/xray-color-helper.cpp
-llvm/tools/llvm-xray/xray-color-helper.h
-llvm/tools/llvm-xray/xray-converter.cpp
-llvm/tools/llvm-xray/xray-converter.h
-llvm/tools/llvm-xray/xray-fdr-dump.cpp
-llvm/tools/llvm-xray/xray-graph-diff.cpp
-llvm/tools/llvm-xray/xray-graph.h
-llvm/tools/llvm-xray/xray-registry.cpp
-llvm/tools/llvm-xray/xray-registry.h
-llvm/tools/llvm-xray/xray-stacks.cpp
-llvm/tools/llvm-yaml-numeric-parser-fuzzer/DummyYAMLNumericParserFuzzer.cpp
-llvm/tools/llvm-yaml-numeric-parser-fuzzer/yaml-numeric-parser-fuzzer.cpp
-llvm/tools/llvm-yaml-parser-fuzzer/DummyYAMLParserFuzzer.cpp
-llvm/tools/llvm-yaml-parser-fuzzer/yaml-parser-fuzzer.cpp
-llvm/tools/lto/LTODisassembler.cpp
-llvm/tools/obj2yaml/archive2yaml.cpp
-llvm/tools/obj2yaml/dwarf2yaml.cpp
-llvm/tools/obj2yaml/minidump2yaml.cpp
-llvm/tools/obj2yaml/obj2yaml.cpp
-llvm/tools/obj2yaml/xcoff2yaml.cpp
-llvm/tools/opt/NewPMDriver.cpp
-llvm/tools/opt/PassPrinters.cpp
-llvm/tools/opt/PassPrinters.h
-llvm/tools/sanstats/sanstats.cpp
-llvm/tools/vfabi-demangle-fuzzer/vfabi-demangler-fuzzer.cpp
-llvm/tools/yaml2obj/yaml2obj.cpp
-llvm/unittests/ADT/AnyTest.cpp
-llvm/unittests/ADT/APSIntTest.cpp
-llvm/unittests/ADT/BitFieldsTest.cpp
-llvm/unittests/ADT/BreadthFirstIteratorTest.cpp
-llvm/unittests/ADT/BumpPtrListTest.cpp
-llvm/unittests/ADT/CombinationGeneratorTest.cpp
-llvm/unittests/ADT/DirectedGraphTest.cpp
-llvm/unittests/ADT/EnumeratedArrayTest.cpp
-llvm/unittests/ADT/FallibleIteratorTest.cpp
-llvm/unittests/ADT/FunctionRefTest.cpp
-llvm/unittests/ADT/IListBaseTest.cpp
-llvm/unittests/ADT/IListNodeBaseTest.cpp
-llvm/unittests/ADT/IListNodeTest.cpp
-llvm/unittests/ADT/ImmutableListTest.cpp
-llvm/unittests/ADT/IntEqClassesTest.cpp
-llvm/unittests/ADT/PointerEmbeddedIntTest.cpp
-llvm/unittests/ADT/ScopeExitTest.cpp
-llvm/unittests/ADT/SequenceTest.cpp
-llvm/unittests/ADT/SetVectorTest.cpp
-llvm/unittests/ADT/SmallSetTest.cpp
-llvm/unittests/ADT/SparseMultiSetTest.cpp
-llvm/unittests/ADT/SparseSetTest.cpp
-llvm/unittests/ADT/StatisticTest.cpp
-llvm/unittests/ADT/STLForwardCompatTest.cpp
-llvm/unittests/ADT/StringExtrasTest.cpp
-llvm/unittests/ADT/StringSetTest.cpp
-llvm/unittests/ADT/StringSwitchTest.cpp
-llvm/unittests/ADT/TypeSwitchTest.cpp
-llvm/unittests/ADT/TypeTraitsTest.cpp
-llvm/unittests/Analysis/BasicAliasAnalysisTest.cpp
-llvm/unittests/Analysis/BlockFrequencyInfoTest.cpp
-llvm/unittests/Analysis/BranchProbabilityInfoTest.cpp
-llvm/unittests/Analysis/ConstraintSystemTest.cpp
-llvm/unittests/Analysis/DDGTest.cpp
-llvm/unittests/Analysis/DomTreeUpdaterTest.cpp
-llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp
-llvm/unittests/Analysis/GlobalsModRefTest.cpp
-llvm/unittests/Analysis/InlineCostTest.cpp
-llvm/unittests/Analysis/MLModelRunnerTest.cpp
-llvm/unittests/Analysis/SparsePropagation.cpp
-llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
-llvm/unittests/Analysis/ValueLatticeTest.cpp
-llvm/unittests/AsmParser/AsmParserTest.cpp
-llvm/unittests/BinaryFormat/DwarfTest.cpp
-llvm/unittests/BinaryFormat/MsgPackDocumentTest.cpp
-llvm/unittests/BinaryFormat/MsgPackReaderTest.cpp
-llvm/unittests/BinaryFormat/MsgPackWriterTest.cpp
-llvm/unittests/BinaryFormat/TestFileMagic.cpp
-llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
-llvm/unittests/Bitstream/BitstreamWriterTest.cpp
-llvm/unittests/CodeGen/AllocationOrderTest.cpp
-llvm/unittests/CodeGen/AMDGPUMetadataTest.cpp
-llvm/unittests/CodeGen/AsmPrinterDwarfTest.cpp
-llvm/unittests/CodeGen/DIETest.cpp
-llvm/unittests/CodeGen/LexicalScopesTest.cpp
-llvm/unittests/CodeGen/MachineInstrBundleIteratorTest.cpp
-llvm/unittests/CodeGen/RegAllocScoreTest.cpp
-llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
-llvm/unittests/CodeGen/TestAsmPrinter.cpp
-llvm/unittests/CodeGen/TestAsmPrinter.h
-llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
-llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
-llvm/unittests/DebugInfo/CodeView/GUIDFormatTest.cpp
-llvm/unittests/DebugInfo/CodeView/TypeHashingTest.cpp
-llvm/unittests/DebugInfo/DWARF/DWARFAcceleratorTableTest.cpp
-llvm/unittests/DebugInfo/DWARF/DWARFDataExtractorTest.cpp
-llvm/unittests/DebugInfo/DWARF/DWARFDebugArangeSetTest.cpp
-llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp
-llvm/unittests/DebugInfo/DWARF/DWARFDieManualExtractTest.cpp
-llvm/unittests/DebugInfo/DWARF/DWARFDieTest.cpp
-llvm/unittests/DebugInfo/DWARF/DWARFExpressionCompactPrinterTest.cpp
-llvm/unittests/DebugInfo/DWARF/DWARFExpressionCopyBytesTest.cpp
-llvm/unittests/DebugInfo/DWARF/DwarfGenerator.h
-llvm/unittests/DebugInfo/DWARF/DWARFListTableTest.cpp
-llvm/unittests/DebugInfo/DWARF/DWARFLocationExpressionTest.cpp
-llvm/unittests/DebugInfo/DWARF/DwarfUtils.cpp
-llvm/unittests/DebugInfo/DWARF/DwarfUtils.h
-llvm/unittests/DebugInfo/MSF/MSFBuilderTest.cpp
-llvm/unittests/DebugInfo/MSF/MSFCommonTest.cpp
-llvm/unittests/DebugInfo/PDB/NativeSessionTest.cpp
-llvm/unittests/DebugInfo/PDB/NativeSymbolReuseTest.cpp
-llvm/unittests/DebugInfo/PDB/StringTableBuilderTest.cpp
-llvm/unittests/DebugInfo/PDB/Inputs/SimpleTest.cpp
-llvm/unittests/Debuginfod/DebuginfodTests.cpp
-llvm/unittests/Debuginfod/HTTPClientTests.cpp
-llvm/unittests/Demangle/DemangleTest.cpp
-llvm/unittests/Demangle/DLangDemangleTest.cpp
-llvm/unittests/Demangle/OutputBufferTest.cpp
-llvm/unittests/Demangle/RustDemangleTest.cpp
-llvm/unittests/Demangle/StringViewTest.cpp
-llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
-llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp
-llvm/unittests/ExecutionEngine/Orc/EPCGenericMemoryAccessTest.cpp
-llvm/unittests/ExecutionEngine/Orc/ExecutionSessionWrapperFunctionCallsTest.cpp
-llvm/unittests/ExecutionEngine/Orc/ExecutorAddressTest.cpp
-llvm/unittests/ExecutionEngine/Orc/JITTargetMachineBuilderTest.cpp
-llvm/unittests/ExecutionEngine/Orc/LazyCallThroughAndReexportsTest.cpp
-llvm/unittests/ExecutionEngine/Orc/LookupAndRecordAddrsTest.cpp
-llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp
-llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp
-llvm/unittests/ExecutionEngine/Orc/ResourceTrackerTest.cpp
-llvm/unittests/ExecutionEngine/Orc/SimpleExecutorMemoryManagerTest.cpp
-llvm/unittests/ExecutionEngine/Orc/SimplePackedSerializationTest.cpp
-llvm/unittests/ExecutionEngine/Orc/TaskDispatchTest.cpp
-llvm/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp
-llvm/unittests/Frontend/OpenACCTest.cpp
-llvm/unittests/Frontend/OpenMPContextTest.cpp
-llvm/unittests/Frontend/OpenMPParsingTest.cpp
-llvm/unittests/InterfaceStub/ELFYAMLTest.cpp
-llvm/unittests/IR/DemandedBitsTest.cpp
-llvm/unittests/IR/ManglerTest.cpp
-llvm/unittests/IR/ModuleTest.cpp
-llvm/unittests/IR/TimePassesTest.cpp
-llvm/unittests/IR/UseTest.cpp
-llvm/unittests/IR/VectorTypesTest.cpp
-llvm/unittests/MC/Disassembler.cpp
-llvm/unittests/MC/DwarfLineTableHeaders.cpp
-llvm/unittests/MC/MCInstPrinter.cpp
-llvm/unittests/MC/TargetRegistry.cpp
-llvm/unittests/MC/AMDGPU/DwarfRegMappings.cpp
-llvm/unittests/MC/SystemZ/SystemZAsmLexerTest.cpp
-llvm/unittests/ObjCopy/ObjCopyTest.cpp
-llvm/unittests/Object/ArchiveTest.cpp
-llvm/unittests/Object/ELFObjectFileTest.cpp
-llvm/unittests/Object/ELFTest.cpp
-llvm/unittests/Object/ELFTypesTest.cpp
-llvm/unittests/Object/MinidumpTest.cpp
-llvm/unittests/Object/ObjectFileTest.cpp
-llvm/unittests/Object/SymbolSizeTest.cpp
-llvm/unittests/ObjectYAML/DWARFYAMLTest.cpp
-llvm/unittests/ObjectYAML/ELFYAMLTest.cpp
-llvm/unittests/ObjectYAML/MinidumpYAMLTest.cpp
-llvm/unittests/Option/OptionMarshallingTest.cpp
-llvm/unittests/Passes/DoublerPlugin.cpp
-llvm/unittests/Passes/PassBuilderBindingsTest.cpp
-llvm/unittests/Passes/PluginsTest.cpp
-llvm/unittests/Passes/TestPlugin.cpp
-llvm/unittests/Passes/TestPlugin.h
-llvm/unittests/ProfileData/InstrProfDataTest.cpp
-llvm/unittests/ProfileData/MemProfTest.cpp
-llvm/unittests/ProfileData/SymbolRemappingReaderTest.cpp
-llvm/unittests/Remarks/BitstreamRemarksFormatTest.cpp
-llvm/unittests/Remarks/BitstreamRemarksParsingTest.cpp
-llvm/unittests/Remarks/RemarksLinkingTest.cpp
-llvm/unittests/Remarks/RemarksStrTabParsingTest.cpp
-llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp
-llvm/unittests/Support/AnnotationsTest.cpp
-llvm/unittests/Support/ArrayRecyclerTest.cpp
-llvm/unittests/Support/Base64Test.cpp
-llvm/unittests/Support/buffer_ostream_test.cpp
-llvm/unittests/Support/Chrono.cpp
-llvm/unittests/Support/CSKYAttributeParserTest.cpp
-llvm/unittests/Support/CSKYTargetParserTest.cpp
-llvm/unittests/Support/DebugCounterTest.cpp
-llvm/unittests/Support/DJBTest.cpp
-llvm/unittests/Support/ELFAttributeParserTest.cpp
-llvm/unittests/Support/ExtensibleRTTITest.cpp
-llvm/unittests/Support/FileOutputBufferTest.cpp
-llvm/unittests/Support/FSUniqueIDTest.cpp
-llvm/unittests/Support/HashBuilderTest.cpp
-llvm/unittests/Support/IndexedAccessorTest.cpp
-llvm/unittests/Support/InstructionCostTest.cpp
-llvm/unittests/Support/KnownBitsTest.h
-llvm/unittests/Support/MatchersTest.cpp
-llvm/unittests/Support/MemoryBufferRefTest.cpp
-llvm/unittests/Support/ParallelTest.cpp
-llvm/unittests/Support/raw_fd_stream_test.cpp
-llvm/unittests/Support/raw_sha1_ostream_test.cpp
-llvm/unittests/Support/RISCVAttributeParserTest.cpp
-llvm/unittests/Support/ScaledNumberTest.cpp
-llvm/unittests/Support/ScopedPrinterTest.cpp
-llvm/unittests/Support/SHA256.cpp
-llvm/unittests/Support/SuffixTreeTest.cpp
-llvm/unittests/Support/TarWriterTest.cpp
-llvm/unittests/Support/ToolOutputFileTest.cpp
-llvm/unittests/Support/TypeTraitsTest.cpp
-llvm/unittests/Support/UnicodeTest.cpp
-llvm/unittests/Support/VersionTupleTest.cpp
-llvm/unittests/Support/WithColorTest.cpp
-llvm/unittests/Support/xxhashTest.cpp
-llvm/unittests/Support/CommandLineInit/CommandLineInitTest.cpp
-llvm/unittests/TableGen/ParserEntryPointTest.cpp
-llvm/unittests/Target/AArch64/MatrixRegisterAliasing.cpp
-llvm/unittests/Target/AMDGPU/DwarfRegMappings.cpp
-llvm/unittests/Target/AMDGPU/ExecMayBeModifiedBeforeAnyUse.cpp
-llvm/unittests/Target/ARM/InstSizes.cpp
-llvm/unittests/Target/PowerPC/AIXRelocModelTest.cpp
-llvm/unittests/Testing/Support/TempPathTest.cpp
-llvm/unittests/TextAPI/TextStubHelpers.h
-llvm/unittests/TextAPI/TextStubV1Tests.cpp
-llvm/unittests/TextAPI/TextStubV2Tests.cpp
-llvm/unittests/tools/llvm-cfi-verify/GraphBuilder.cpp
-llvm/unittests/tools/llvm-exegesis/BenchmarkRunnerTest.cpp
-llvm/unittests/tools/llvm-exegesis/PerfHelperTest.cpp
-llvm/unittests/tools/llvm-exegesis/RegisterValueTest.cpp
-llvm/unittests/tools/llvm-exegesis/AArch64/TargetTest.cpp
-llvm/unittests/tools/llvm-exegesis/ARM/AssemblerTest.cpp
-llvm/unittests/tools/llvm-exegesis/Common/AssemblerUtils.h
-llvm/unittests/tools/llvm-exegesis/Mips/RegisterAliasingTest.cpp
-llvm/unittests/tools/llvm-exegesis/Mips/TargetTest.cpp
-llvm/unittests/tools/llvm-exegesis/Mips/TestBase.h
-llvm/unittests/tools/llvm-exegesis/PowerPC/TestBase.h
-llvm/unittests/tools/llvm-exegesis/X86/AssemblerTest.cpp
-llvm/unittests/tools/llvm-exegesis/X86/BenchmarkResultTest.cpp
-llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp
-llvm/unittests/tools/llvm-exegesis/X86/SnippetRepetitorTest.cpp
-llvm/unittests/tools/llvm-exegesis/X86/TargetTest.cpp
-llvm/unittests/tools/llvm-exegesis/X86/TestBase.h
-llvm/unittests/Transforms/IPO/AttributorTest.cpp
-llvm/unittests/Transforms/IPO/AttributorTestBase.h
-llvm/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp
-llvm/unittests/Transforms/Utils/CallPromotionUtilsTest.cpp
-llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp
-llvm/unittests/Transforms/Utils/LoopUtilsTest.cpp
-llvm/unittests/Transforms/Utils/ModuleUtilsTest.cpp
-llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp
-llvm/unittests/Transforms/Utils/SSAUpdaterBulkTest.cpp
-llvm/unittests/Transforms/Utils/VFABIUtils.cpp
-llvm/unittests/Transforms/Vectorize/VPlanDominatorTreeTest.cpp
-llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
-llvm/unittests/Transforms/Vectorize/VPlanLoopInfoTest.cpp
-llvm/unittests/Transforms/Vectorize/VPlanPredicatorTest.cpp
-llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
-llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
-llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
-llvm/unittests/XRay/FDRBlockIndexerTest.cpp
-llvm/unittests/XRay/FDRBlockVerifierTest.cpp
-llvm/unittests/XRay/FDRProducerConsumerTest.cpp
-llvm/unittests/XRay/FDRRecordPrinterTest.cpp
-llvm/unittests/XRay/FDRRecordsTest.cpp
-llvm/unittests/XRay/FDRTraceWriterTest.cpp
-llvm/unittests/XRay/ProfileTest.cpp
-llvm/utils/not/not.cpp
-llvm/Utils/TableGen/CodeGenInstAlias.cpp
-llvm/Utils/TableGen/CodeGenInstAlias.h
-llvm/utils/TableGen/CodeBeadsGen.cpp
-llvm/utils/TableGen/CompressInstEmitter.cpp
-llvm/utils/TableGen/DFAEmitter.h
-llvm/utils/TableGen/DirectiveEmitter.cpp
-llvm/utils/TableGen/ExegesisEmitter.cpp
-llvm/utils/TableGen/OptEmitter.cpp
-llvm/utils/TableGen/OptEmitter.h
-llvm/utils/TableGen/OptRSTEmitter.cpp
-llvm/utils/TableGen/PredicateExpander.h
-llvm/utils/TableGen/SDNodeProperties.cpp
-llvm/utils/TableGen/VarLenCodeEmitterGen.cpp
-llvm/utils/TableGen/VarLenCodeEmitterGen.h
-llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.h
-llvm/utils/TableGen/GlobalISel/CodeExpander.cpp
-llvm/utils/TableGen/GlobalISel/CodeExpander.h
-llvm/utils/TableGen/GlobalISel/CodeExpansions.h
-llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.cpp
-llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.cpp
-llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.h
-llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.cpp
-llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h
-llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.cpp
-llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.h
-mlir/examples/standalone/include/Standalone/StandaloneDialect.h
-mlir/examples/standalone/include/Standalone/StandaloneOps.h
-mlir/examples/standalone/include/Standalone-c/Dialects.h
-mlir/examples/standalone/lib/CAPI/Dialects.cpp
-mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
-mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp
-mlir/examples/standalone/python/StandaloneExtension.cpp
-mlir/examples/standalone/standalone-opt/standalone-opt.cpp
-mlir/examples/standalone/standalone-translate/standalone-translate.cpp
-mlir/examples/toy/Ch1/toyc.cpp
-mlir/examples/toy/Ch1/include/toy/AST.h
-mlir/examples/toy/Ch1/include/toy/Lexer.h
-mlir/examples/toy/Ch1/include/toy/Parser.h
-mlir/examples/toy/Ch2/toyc.cpp
-mlir/examples/toy/Ch2/include/toy/AST.h
-mlir/examples/toy/Ch2/include/toy/Dialect.h
-mlir/examples/toy/Ch2/include/toy/Lexer.h
-mlir/examples/toy/Ch2/include/toy/MLIRGen.h
-mlir/examples/toy/Ch2/include/toy/Parser.h
-mlir/examples/toy/Ch2/mlir/Dialect.cpp
-mlir/examples/toy/Ch2/mlir/MLIRGen.cpp
-mlir/examples/toy/Ch3/toyc.cpp
-mlir/examples/toy/Ch3/include/toy/AST.h
-mlir/examples/toy/Ch3/include/toy/Dialect.h
-mlir/examples/toy/Ch3/include/toy/Lexer.h
-mlir/examples/toy/Ch3/include/toy/MLIRGen.h
-mlir/examples/toy/Ch3/include/toy/Parser.h
-mlir/examples/toy/Ch3/mlir/Dialect.cpp
-mlir/examples/toy/Ch3/mlir/MLIRGen.cpp
-mlir/examples/toy/Ch3/mlir/ToyCombine.cpp
-mlir/examples/toy/Ch4/toyc.cpp
-mlir/examples/toy/Ch4/include/toy/AST.h
-mlir/examples/toy/Ch4/include/toy/Dialect.h
-mlir/examples/toy/Ch4/include/toy/Lexer.h
-mlir/examples/toy/Ch4/include/toy/MLIRGen.h
-mlir/examples/toy/Ch4/include/toy/Parser.h
-mlir/examples/toy/Ch4/include/toy/Passes.h
-mlir/examples/toy/Ch4/include/toy/ShapeInferenceInterface.h
-mlir/examples/toy/Ch4/mlir/Dialect.cpp
-mlir/examples/toy/Ch4/mlir/MLIRGen.cpp
-mlir/examples/toy/Ch4/mlir/ShapeInferencePass.cpp
-mlir/examples/toy/Ch4/mlir/ToyCombine.cpp
-mlir/examples/toy/Ch5/toyc.cpp
-mlir/examples/toy/Ch5/include/toy/AST.h
-mlir/examples/toy/Ch5/include/toy/Dialect.h
-mlir/examples/toy/Ch5/include/toy/Lexer.h
-mlir/examples/toy/Ch5/include/toy/MLIRGen.h
-mlir/examples/toy/Ch5/include/toy/Parser.h
-mlir/examples/toy/Ch5/include/toy/Passes.h
-mlir/examples/toy/Ch5/include/toy/ShapeInferenceInterface.h
-mlir/examples/toy/Ch5/mlir/Dialect.cpp
-mlir/examples/toy/Ch5/mlir/LowerToAffineLoops.cpp
-mlir/examples/toy/Ch5/mlir/MLIRGen.cpp
-mlir/examples/toy/Ch5/mlir/ShapeInferencePass.cpp
-mlir/examples/toy/Ch5/mlir/ToyCombine.cpp
-mlir/examples/toy/Ch6/toyc.cpp
-mlir/examples/toy/Ch6/include/toy/AST.h
-mlir/examples/toy/Ch6/include/toy/Dialect.h
-mlir/examples/toy/Ch6/include/toy/Lexer.h
-mlir/examples/toy/Ch6/include/toy/MLIRGen.h
-mlir/examples/toy/Ch6/include/toy/Parser.h
-mlir/examples/toy/Ch6/include/toy/Passes.h
-mlir/examples/toy/Ch6/include/toy/ShapeInferenceInterface.h
-mlir/examples/toy/Ch6/mlir/Dialect.cpp
-mlir/examples/toy/Ch6/mlir/LowerToAffineLoops.cpp
-mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
-mlir/examples/toy/Ch6/mlir/MLIRGen.cpp
-mlir/examples/toy/Ch6/mlir/ShapeInferencePass.cpp
-mlir/examples/toy/Ch6/mlir/ToyCombine.cpp
-mlir/examples/toy/Ch7/toyc.cpp
-mlir/examples/toy/Ch7/include/toy/AST.h
-mlir/examples/toy/Ch7/include/toy/Dialect.h
-mlir/examples/toy/Ch7/include/toy/Lexer.h
-mlir/examples/toy/Ch7/include/toy/MLIRGen.h
-mlir/examples/toy/Ch7/include/toy/Parser.h
-mlir/examples/toy/Ch7/include/toy/Passes.h
-mlir/examples/toy/Ch7/include/toy/ShapeInferenceInterface.h
-mlir/examples/toy/Ch7/mlir/Dialect.cpp
-mlir/examples/toy/Ch7/mlir/LowerToAffineLoops.cpp
-mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
-mlir/examples/toy/Ch7/mlir/MLIRGen.cpp
-mlir/examples/toy/Ch7/mlir/ShapeInferencePass.cpp
-mlir/examples/toy/Ch7/mlir/ToyCombine.cpp
-mlir/include/mlir/InitAllDialects.h
-mlir/include/mlir/InitAllPasses.h
-mlir/include/mlir/InitAllTranslations.h
-mlir/include/mlir/Parser/Parser.h
-mlir/include/mlir/Translation.h
-mlir/include/mlir/Analysis/BufferViewFlowAnalysis.h
-mlir/include/mlir/Analysis/DataFlowAnalysis.h
-mlir/include/mlir/Analysis/DataLayoutAnalysis.h
-mlir/include/mlir/Analysis/Liveness.h
-mlir/include/mlir/Analysis/SliceAnalysis.h
-mlir/include/mlir/Analysis/AliasAnalysis/LocalAliasAnalysis.h
-mlir/include/mlir/Analysis/Presburger/Fraction.h
-mlir/include/mlir/Analysis/Presburger/IntegerRelation.h
-mlir/include/mlir/Analysis/Presburger/LinearTransform.h
-mlir/include/mlir/Analysis/Presburger/Matrix.h
-mlir/include/mlir/Analysis/Presburger/PresburgerSet.h
-mlir/include/mlir/Analysis/Presburger/PresburgerSpace.h
-mlir/include/mlir/Analysis/Presburger/PWMAFunction.h
-mlir/include/mlir/Analysis/Presburger/Simplex.h
-mlir/include/mlir/Analysis/Presburger/Utils.h
-mlir/include/mlir/CAPI/AffineExpr.h
-mlir/include/mlir/CAPI/AffineMap.h
-mlir/include/mlir/CAPI/Diagnostics.h
-mlir/include/mlir/CAPI/ExecutionEngine.h
-mlir/include/mlir/CAPI/IntegerSet.h
-mlir/include/mlir/CAPI/Interfaces.h
-mlir/include/mlir/CAPI/IR.h
-mlir/include/mlir/CAPI/Pass.h
-mlir/include/mlir/CAPI/Registration.h
-mlir/include/mlir/CAPI/Support.h
-mlir/include/mlir/CAPI/Utils.h
-mlir/include/mlir/CAPI/Wrap.h
-mlir/include/mlir/Conversion/Passes.h
-mlir/include/mlir/Conversion/AffineToStandard/AffineToStandard.h
-mlir/include/mlir/Conversion/ArithmeticToLLVM/ArithmeticToLLVM.h
-mlir/include/mlir/Conversion/ArithmeticToSPIRV/ArithmeticToSPIRV.h
-mlir/include/mlir/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.h
-mlir/include/mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h
-mlir/include/mlir/Conversion/BufferizationToMemRef/BufferizationToMemRef.h
-mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h
-mlir/include/mlir/Conversion/ComplexToStandard/ComplexToStandard.h
-mlir/include/mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h
-mlir/include/mlir/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRV.h
-mlir/include/mlir/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRVPass.h
-mlir/include/mlir/Conversion/FuncToSPIRV/FuncToSPIRV.h
-mlir/include/mlir/Conversion/FuncToSPIRV/FuncToSPIRVPass.h
-mlir/include/mlir/Conversion/GPUCommon/GPUCommonPass.h
-mlir/include/mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h
-mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h
-mlir/include/mlir/Conversion/GPUToROCDL/Runtimes.h
-mlir/include/mlir/Conversion/GPUToSPIRV/GPUToSPIRV.h
-mlir/include/mlir/Conversion/GPUToSPIRV/GPUToSPIRVPass.h
-mlir/include/mlir/Conversion/LinalgToSPIRV/LinalgToSPIRV.h
-mlir/include/mlir/Conversion/LinalgToSPIRV/LinalgToSPIRVPass.h
-mlir/include/mlir/Conversion/LinalgToStandard/LinalgToStandard.h
-mlir/include/mlir/Conversion/LLVMCommon/ConversionTarget.h
-mlir/include/mlir/Conversion/LLVMCommon/LoweringOptions.h
-mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h
-mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
-mlir/include/mlir/Conversion/LLVMCommon/StructBuilder.h
-mlir/include/mlir/Conversion/LLVMCommon/TypeConverter.h
-mlir/include/mlir/Conversion/LLVMCommon/VectorPattern.h
-mlir/include/mlir/Conversion/MathToLibm/MathToLibm.h
-mlir/include/mlir/Conversion/MathToLLVM/MathToLLVM.h
-mlir/include/mlir/Conversion/MathToSPIRV/MathToSPIRV.h
-mlir/include/mlir/Conversion/MathToSPIRV/MathToSPIRVPass.h
-mlir/include/mlir/Conversion/MemRefToLLVM/AllocLikeConversion.h
-mlir/include/mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h
-mlir/include/mlir/Conversion/MemRefToSPIRV/MemRefToSPIRV.h
-mlir/include/mlir/Conversion/MemRefToSPIRV/MemRefToSPIRVPass.h
-mlir/include/mlir/Conversion/OpenACCToLLVM/ConvertOpenACCToLLVM.h
-mlir/include/mlir/Conversion/OpenACCToSCF/ConvertOpenACCToSCF.h
-mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h
-mlir/include/mlir/Conversion/PDLToPDLInterp/PDLToPDLInterp.h
-mlir/include/mlir/Conversion/ReconcileUnrealizedCasts/ReconcileUnrealizedCasts.h
-mlir/include/mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h
-mlir/include/mlir/Conversion/SCFToGPU/SCFToGPU.h
-mlir/include/mlir/Conversion/SCFToGPU/SCFToGPUPass.h
-mlir/include/mlir/Conversion/SCFToOpenMP/SCFToOpenMP.h
-mlir/include/mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h
-mlir/include/mlir/Conversion/SCFToSPIRV/SCFToSPIRVPass.h
-mlir/include/mlir/Conversion/ShapeToStandard/ShapeToStandard.h
-mlir/include/mlir/Conversion/SPIRVToLLVM/SPIRVToLLVM.h
-mlir/include/mlir/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.h
-mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
-mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
-mlir/include/mlir/Conversion/TensorToSPIRV/TensorToSPIRV.h
-mlir/include/mlir/Conversion/TensorToSPIRV/TensorToSPIRVPass.h
-mlir/include/mlir/Conversion/TosaToLinalg/TosaToLinalg.h
-mlir/include/mlir/Conversion/TosaToSCF/TosaToSCF.h
-mlir/include/mlir/Conversion/TosaToStandard/TosaToStandard.h
-mlir/include/mlir/Conversion/VectorToGPU/VectorToGPU.h
-mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
-mlir/include/mlir/Conversion/VectorToSCF/VectorToSCF.h
-mlir/include/mlir/Conversion/VectorToSPIRV/VectorToSPIRV.h
-mlir/include/mlir/Conversion/VectorToSPIRV/VectorToSPIRVPass.h
-mlir/include/mlir/Dialect/CommonFolders.h
-mlir/include/mlir/Dialect/Traits.h
-mlir/include/mlir/Dialect/Affine/LoopFusionUtils.h
-mlir/include/mlir/Dialect/Affine/LoopUtils.h
-mlir/include/mlir/Dialect/Affine/Passes.h
-mlir/include/mlir/Dialect/Affine/Utils.h
-mlir/include/mlir/Dialect/Affine/Analysis/AffineAnalysis.h
-mlir/include/mlir/Dialect/Affine/Analysis/AffineStructures.h
-mlir/include/mlir/Dialect/Affine/Analysis/LoopAnalysis.h
-mlir/include/mlir/Dialect/Affine/Analysis/NestedMatcher.h
-mlir/include/mlir/Dialect/Affine/Analysis/Utils.h
-mlir/include/mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.h
-mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
-mlir/include/mlir/Dialect/Affine/IR/AffineValueMap.h
-mlir/include/mlir/Dialect/AMX/AMXDialect.h
-mlir/include/mlir/Dialect/AMX/Transforms.h
-mlir/include/mlir/Dialect/Arithmetic/IR/Arithmetic.h
-mlir/include/mlir/Dialect/Arithmetic/Transforms/BufferizableOpInterfaceImpl.h
-mlir/include/mlir/Dialect/Arithmetic/Transforms/Passes.h
-mlir/include/mlir/Dialect/Arithmetic/Utils/Utils.h
-mlir/include/mlir/Dialect/ArmNeon/ArmNeonDialect.h
-mlir/include/mlir/Dialect/ArmSVE/ArmSVEDialect.h
-mlir/include/mlir/Dialect/ArmSVE/Transforms.h
-mlir/include/mlir/Dialect/Async/Passes.h
-mlir/include/mlir/Dialect/Async/Transforms.h
-mlir/include/mlir/Dialect/Async/IR/Async.h
-mlir/include/mlir/Dialect/Async/IR/AsyncTypes.h
-mlir/include/mlir/Dialect/Bufferization/IR/AllocationOpInterface.h
-mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
-mlir/include/mlir/Dialect/Bufferization/IR/Bufferization.h
-mlir/include/mlir/Dialect/Bufferization/Transforms/Bufferize.h
-mlir/include/mlir/Dialect/Bufferization/Transforms/BufferUtils.h
-mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h
-mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.h
-mlir/include/mlir/Dialect/Complex/IR/Complex.h
-mlir/include/mlir/Dialect/ControlFlow/IR/ControlFlow.h
-mlir/include/mlir/Dialect/ControlFlow/IR/ControlFlowOps.h
-mlir/include/mlir/Dialect/DLTI/DLTI.h
-mlir/include/mlir/Dialect/DLTI/Traits.h
-mlir/include/mlir/Dialect/EmitC/IR/EmitC.h
-mlir/include/mlir/Dialect/Func/IR/FuncOps.h
-mlir/include/mlir/Dialect/Func/Transforms/DecomposeCallGraphTypes.h
-mlir/include/mlir/Dialect/Func/Transforms/FuncConversions.h
-mlir/include/mlir/Dialect/Func/Transforms/Passes.h
-mlir/include/mlir/Dialect/GPU/GPUDialect.h
-mlir/include/mlir/Dialect/GPU/MemoryPromotion.h
-mlir/include/mlir/Dialect/GPU/ParallelLoopMapper.h
-mlir/include/mlir/Dialect/GPU/Passes.h
-mlir/include/mlir/Dialect/GPU/Utils.h
-mlir/include/mlir/Dialect/Linalg/Passes.h
-mlir/include/mlir/Dialect/Linalg/Analysis/DependenceAnalysis.h
-mlir/include/mlir/Dialect/Linalg/ComprehensiveBufferize/AffineInterfaceImpl.h
-mlir/include/mlir/Dialect/Linalg/ComprehensiveBufferize/ModuleBufferization.h
-mlir/include/mlir/Dialect/Linalg/IR/Linalg.h
-mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.h
-mlir/include/mlir/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.h
-mlir/include/mlir/Dialect/Linalg/Transforms/CodegenStrategy.h
-mlir/include/mlir/Dialect/Linalg/Transforms/Hoisting.h
-mlir/include/mlir/Dialect/Linalg/Transforms/HoistPadding.h
-mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
-mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
-mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h
-mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
-mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h
-mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h
-mlir/include/mlir/Dialect/LLVMIR/ROCDLDialect.h
-mlir/include/mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h
-mlir/include/mlir/Dialect/LLVMIR/Transforms/Passes.h
-mlir/include/mlir/Dialect/Math/IR/Math.h
-mlir/include/mlir/Dialect/Math/Transforms/Approximation.h
-mlir/include/mlir/Dialect/Math/Transforms/Passes.h
-mlir/include/mlir/Dialect/MemRef/IR/MemRef.h
-mlir/include/mlir/Dialect/MemRef/Transforms/ComposeSubView.h
-mlir/include/mlir/Dialect/MemRef/Transforms/Passes.h
-mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
-mlir/include/mlir/Dialect/OpenACC/OpenACC.h
-mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
-mlir/include/mlir/Dialect/PDL/IR/PDL.h
-mlir/include/mlir/Dialect/PDL/IR/PDLOps.h
-mlir/include/mlir/Dialect/PDL/IR/PDLTypes.h
-mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterp.h
-mlir/include/mlir/Dialect/Quant/FakeQuantSupport.h
-mlir/include/mlir/Dialect/Quant/Passes.h
-mlir/include/mlir/Dialect/Quant/QuantizeUtils.h
-mlir/include/mlir/Dialect/Quant/QuantOps.h
-mlir/include/mlir/Dialect/Quant/QuantTypes.h
-mlir/include/mlir/Dialect/Quant/UniformSupport.h
-mlir/include/mlir/Dialect/SCF/BufferizableOpInterfaceImpl.h
-mlir/include/mlir/Dialect/SCF/Passes.h
-mlir/include/mlir/Dialect/SCF/SCF.h
-mlir/include/mlir/Dialect/SCF/Transforms.h
-mlir/include/mlir/Dialect/SCF/Utils/AffineCanonicalizationUtils.h
-mlir/include/mlir/Dialect/SCF/Utils/Utils.h
-mlir/include/mlir/Dialect/Shape/IR/Shape.h
-mlir/include/mlir/Dialect/Shape/Transforms/Passes.h
-mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
-mlir/include/mlir/Dialect/SparseTensor/Pipelines/Passes.h
-mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h
-mlir/include/mlir/Dialect/SparseTensor/Utils/Merger.h
-mlir/include/mlir/Dialect/SPIRV/IR/ParserUtils.h
-mlir/include/mlir/Dialect/SPIRV/IR/SPIRVAttributes.h
-mlir/include/mlir/Dialect/SPIRV/IR/SPIRVDialect.h
-mlir/include/mlir/Dialect/SPIRV/IR/SPIRVEnums.h
-mlir/include/mlir/Dialect/SPIRV/IR/SPIRVGLSLCanonicalization.h
-mlir/include/mlir/Dialect/SPIRV/IR/SPIRVOps.h
-mlir/include/mlir/Dialect/SPIRV/IR/SPIRVOpTraits.h
-mlir/include/mlir/Dialect/SPIRV/IR/SPIRVTypes.h
-mlir/include/mlir/Dialect/SPIRV/IR/TargetAndABI.h
-mlir/include/mlir/Dialect/SPIRV/Linking/ModuleCombiner.h
-mlir/include/mlir/Dialect/SPIRV/Transforms/Passes.h
-mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h
-mlir/include/mlir/Dialect/SPIRV/Utils/LayoutUtils.h
-mlir/include/mlir/Dialect/Tensor/IR/Tensor.h
-mlir/include/mlir/Dialect/Tensor/IR/TensorInferTypeOpInterfaceImpl.h
-mlir/include/mlir/Dialect/Tensor/IR/TensorTilingInterfaceImpl.h
-mlir/include/mlir/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.h
-mlir/include/mlir/Dialect/Tensor/Transforms/Passes.h
-mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h
-mlir/include/mlir/Dialect/Tensor/Utils/Utils.h
-mlir/include/mlir/Dialect/Tosa/IR/TosaOps.h
-mlir/include/mlir/Dialect/Tosa/Transforms/PassDetail.h
-mlir/include/mlir/Dialect/Tosa/Transforms/Passes.h
-mlir/include/mlir/Dialect/Tosa/Utils/CoversionUtils.h
-mlir/include/mlir/Dialect/Tosa/Utils/QuantUtils.h
-mlir/include/mlir/Dialect/Tosa/Utils/ShapeUtils.h
-mlir/include/mlir/Dialect/Utils/IndexingUtils.h
-mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
-mlir/include/mlir/Dialect/Utils/StaticValueUtils.h
-mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h
-mlir/include/mlir/Dialect/Vector/IR/VectorOps.h
-mlir/include/mlir/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.h
-mlir/include/mlir/Dialect/Vector/Transforms/Passes.h
-mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
-mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h
-mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h
-mlir/include/mlir/Dialect/X86Vector/Transforms.h
-mlir/include/mlir/Dialect/X86Vector/X86VectorDialect.h
-mlir/include/mlir/ExecutionEngine/AsyncRuntime.h
-mlir/include/mlir/ExecutionEngine/CRunnerUtils.h
-mlir/include/mlir/ExecutionEngine/JitRunner.h
-mlir/include/mlir/ExecutionEngine/MemRefUtils.h
-mlir/include/mlir/ExecutionEngine/OptUtils.h
-mlir/include/mlir/ExecutionEngine/RunnerUtils.h
-mlir/include/mlir/ExecutionEngine/SparseTensorUtils.h
-mlir/include/mlir/Interfaces/CallInterfaces.h
-mlir/include/mlir/Interfaces/CastInterfaces.h
-mlir/include/mlir/Interfaces/ControlFlowInterfaces.h
-mlir/include/mlir/Interfaces/CopyOpInterface.h
-mlir/include/mlir/Interfaces/DataLayoutInterfaces.h
-mlir/include/mlir/Interfaces/DecodeAttributesInterfaces.h
-mlir/include/mlir/Interfaces/DerivedAttributeOpInterface.h
-mlir/include/mlir/Interfaces/FoldInterfaces.h
-mlir/include/mlir/Interfaces/InferTypeOpInterface.h
-mlir/include/mlir/Interfaces/LoopLikeInterface.h
-mlir/include/mlir/Interfaces/TilingInterface.h
-mlir/include/mlir/Interfaces/VectorInterfaces.h
-mlir/include/mlir/Interfaces/ViewLikeInterface.h
-mlir/include/mlir/IR/AffineExpr.h
-mlir/include/mlir/IR/AffineMap.h
-mlir/include/mlir/IR/AsmState.h
-mlir/include/mlir/IR/AttributeSupport.h
-mlir/include/mlir/IR/Block.h
-mlir/include/mlir/IR/Builders.h
-mlir/include/mlir/IR/BuiltinDialect.h
-mlir/include/mlir/IR/BuiltinOps.h
-mlir/include/mlir/IR/BuiltinTypes.h
-mlir/include/mlir/IR/DialectImplementation.h
-mlir/include/mlir/IR/Dominance.h
-mlir/include/mlir/IR/FunctionImplementation.h
-mlir/include/mlir/IR/FunctionInterfaces.h
-mlir/include/mlir/IR/ImplicitLocOpBuilder.h
-mlir/include/mlir/IR/Matchers.h
-mlir/include/mlir/IR/MLIRContext.h
-mlir/include/mlir/IR/OpDefinition.h
-mlir/include/mlir/IR/OwningOpRef.h
-mlir/include/mlir/IR/Region.h
-mlir/include/mlir/IR/RegionKindInterface.h
-mlir/include/mlir/IR/SubElementInterfaces.h
-mlir/include/mlir/IR/SymbolTable.h
-mlir/include/mlir/IR/TensorEncoding.h
-mlir/include/mlir/IR/Threading.h
-mlir/include/mlir/IR/TypeRange.h
-mlir/include/mlir/IR/TypeUtilities.h
-mlir/include/mlir/IR/Value.h
-mlir/include/mlir/IR/Verifier.h
-mlir/include/mlir/IR/Visitors.h
-mlir/include/mlir/Parser/AsmParserState.h
-mlir/include/mlir/Reducer/PassDetail.h
-mlir/include/mlir/Reducer/Passes.h
-mlir/include/mlir/Reducer/ReductionNode.h
-mlir/include/mlir/Reducer/ReductionPatternInterface.h
-mlir/include/mlir/Reducer/Tester.h
-mlir/include/mlir/Rewrite/FrozenRewritePatternSet.h
-mlir/include/mlir/Rewrite/PatternApplicator.h
-mlir/include/mlir/Support/DebugCounter.h
-mlir/include/mlir/Support/DebugStringHelper.h
-mlir/include/mlir/Support/FileUtilities.h
-mlir/include/mlir/Support/IndentedOstream.h
-mlir/include/mlir/Support/MathExtras.h
-mlir/include/mlir/Support/MlirOptMain.h
-mlir/include/mlir/Support/ThreadLocalCache.h
-mlir/include/mlir/Support/Timing.h
-mlir/include/mlir/Support/ToolUtilities.h
-mlir/include/mlir/TableGen/Argument.h
-mlir/include/mlir/TableGen/Attribute.h
-mlir/include/mlir/TableGen/AttrOrTypeDef.h
-mlir/include/mlir/TableGen/Builder.h
-mlir/include/mlir/TableGen/Class.h
-mlir/include/mlir/TableGen/Constraint.h
-mlir/include/mlir/TableGen/Dialect.h
-mlir/include/mlir/TableGen/GenInfo.h
-mlir/include/mlir/TableGen/GenNameParser.h
-mlir/include/mlir/TableGen/Interfaces.h
-mlir/include/mlir/TableGen/Operator.h
-mlir/include/mlir/TableGen/Pass.h
-mlir/include/mlir/TableGen/Pattern.h
-mlir/include/mlir/TableGen/Predicate.h
-mlir/include/mlir/TableGen/Region.h
-mlir/include/mlir/TableGen/SideEffects.h
-mlir/include/mlir/TableGen/Successor.h
-mlir/include/mlir/TableGen/Trait.h
-mlir/include/mlir/TableGen/Type.h
-mlir/include/mlir/Target/Cpp/CppEmitter.h
-mlir/include/mlir/Target/LLVMIR/Export.h
-mlir/include/mlir/Target/LLVMIR/Import.h
-mlir/include/mlir/Target/LLVMIR/LLVMTranslationInterface.h
-mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
-mlir/include/mlir/Target/LLVMIR/TypeFromLLVM.h
-mlir/include/mlir/Target/LLVMIR/TypeToLLVM.h
-mlir/include/mlir/Target/LLVMIR/Dialect/All.h
-mlir/include/mlir/Target/LLVMIR/Dialect/AMX/AMXToLLVMIRTranslation.h
-mlir/include/mlir/Target/LLVMIR/Dialect/ArmNeon/ArmNeonToLLVMIRTranslation.h
-mlir/include/mlir/Target/LLVMIR/Dialect/ArmSVE/ArmSVEToLLVMIRTranslation.h
-mlir/include/mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h
-mlir/include/mlir/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.h
-mlir/include/mlir/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.h
-mlir/include/mlir/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.h
-mlir/include/mlir/Target/LLVMIR/Dialect/ROCDL/ROCDLToLLVMIRTranslation.h
-mlir/include/mlir/Target/LLVMIR/Dialect/X86Vector/X86VectorToLLVMIRTranslation.h
-mlir/include/mlir/Target/SPIRV/Deserialization.h
-mlir/include/mlir/Target/SPIRV/Serialization.h
-mlir/include/mlir/Target/SPIRV/SPIRVBinaryUtils.h
-mlir/include/mlir/Tools/mlir-lsp-server/MlirLspServerMain.h
-mlir/include/mlir/Tools/mlir-reduce/MlirReduceMain.h
-mlir/include/mlir/Tools/PDLL/AST/Context.h
-mlir/include/mlir/Tools/PDLL/AST/Diagnostic.h
-mlir/include/mlir/Tools/PDLL/CodeGen/CPPGen.h
-mlir/include/mlir/Tools/PDLL/CodeGen/MLIRGen.h
-mlir/include/mlir/Tools/PDLL/ODS/Constraint.h
-mlir/include/mlir/Tools/PDLL/ODS/Context.h
-mlir/include/mlir/Tools/PDLL/ODS/Dialect.h
-mlir/include/mlir/Tools/PDLL/ODS/Operation.h
-mlir/include/mlir/Tools/PDLL/Parser/Parser.h
-mlir/include/mlir/Transforms/ControlFlowSinkUtils.h
-mlir/include/mlir/Transforms/DialectConversion.h
-mlir/include/mlir/Transforms/GreedyPatternRewriteDriver.h
-mlir/include/mlir/Transforms/InliningUtils.h
-mlir/include/mlir/Transforms/LocationSnapshot.h
-mlir/include/mlir/Transforms/Passes.h
-mlir/include/mlir/Transforms/RegionUtils.h
-mlir/include/mlir-c/AffineExpr.h
-mlir/include/mlir-c/AffineMap.h
-mlir/include/mlir-c/BuiltinAttributes.h
-mlir/include/mlir-c/BuiltinTypes.h
-mlir/include/mlir-c/Conversion.h
-mlir/include/mlir-c/Debug.h
-mlir/include/mlir-c/Diagnostics.h
-mlir/include/mlir-c/ExecutionEngine.h
-mlir/include/mlir-c/IntegerSet.h
-mlir/include/mlir-c/Interfaces.h
-mlir/include/mlir-c/IR.h
-mlir/include/mlir-c/Pass.h
-mlir/include/mlir-c/Registration.h
-mlir/include/mlir-c/Support.h
-mlir/include/mlir-c/Transforms.h
-mlir/include/mlir-c/Bindings/Python/Interop.h
-mlir/include/mlir-c/Dialect/Async.h
-mlir/include/mlir-c/Dialect/Func.h
-mlir/include/mlir-c/Dialect/GPU.h
-mlir/include/mlir-c/Dialect/Linalg.h
-mlir/include/mlir-c/Dialect/LLVM.h
-mlir/include/mlir-c/Dialect/PDL.h
-mlir/include/mlir-c/Dialect/Quant.h
-mlir/include/mlir-c/Dialect/SCF.h
-mlir/include/mlir-c/Dialect/Shape.h
-mlir/include/mlir-c/Dialect/SparseTensor.h
-mlir/include/mlir-c/Dialect/Tensor.h
-mlir/lib/Analysis/AliasAnalysis.cpp
-mlir/lib/Analysis/BufferViewFlowAnalysis.cpp
-mlir/lib/Analysis/CallGraph.cpp
-mlir/lib/Analysis/DataFlowAnalysis.cpp
-mlir/lib/Analysis/DataLayoutAnalysis.cpp
-mlir/lib/Analysis/Liveness.cpp
-mlir/lib/Analysis/SliceAnalysis.cpp
-mlir/lib/Analysis/AliasAnalysis/LocalAliasAnalysis.cpp
-mlir/lib/Analysis/Presburger/IntegerRelation.cpp
-mlir/lib/Analysis/Presburger/LinearTransform.cpp
-mlir/lib/Analysis/Presburger/Matrix.cpp
-mlir/lib/Analysis/Presburger/PresburgerSet.cpp
-mlir/lib/Analysis/Presburger/PresburgerSpace.cpp
-mlir/lib/Analysis/Presburger/PWMAFunction.cpp
-mlir/lib/Analysis/Presburger/Simplex.cpp
-mlir/lib/Analysis/Presburger/Utils.cpp
-mlir/lib/Bindings/Python/AllPassesRegistration.cpp
-mlir/lib/Bindings/Python/AsyncPasses.cpp
-mlir/lib/Bindings/Python/DialectLinalg.cpp
-mlir/lib/Bindings/Python/DialectPDL.cpp
-mlir/lib/Bindings/Python/DialectQuant.cpp
-mlir/lib/Bindings/Python/DialectSparseTensor.cpp
-mlir/lib/Bindings/Python/ExecutionEngineModule.cpp
-mlir/lib/Bindings/Python/Globals.h
-mlir/lib/Bindings/Python/GPUPasses.cpp
-mlir/lib/Bindings/Python/IRAffine.cpp
-mlir/lib/Bindings/Python/IRAttributes.cpp
-mlir/lib/Bindings/Python/IRCore.cpp
-mlir/lib/Bindings/Python/IRInterfaces.cpp
-mlir/lib/Bindings/Python/IRModule.cpp
-mlir/lib/Bindings/Python/IRModule.h
-mlir/lib/Bindings/Python/IRTypes.cpp
-mlir/lib/Bindings/Python/LinalgPasses.cpp
-mlir/lib/Bindings/Python/MainModule.cpp
-mlir/lib/Bindings/Python/Pass.cpp
-mlir/lib/Bindings/Python/Pass.h
-mlir/lib/Bindings/Python/PybindUtils.cpp
-mlir/lib/Bindings/Python/PybindUtils.h
-mlir/lib/Bindings/Python/SparseTensorPasses.cpp
-mlir/lib/Bindings/Python/Conversions/Conversions.cpp
-mlir/lib/Bindings/Python/Transforms/Transforms.cpp
-mlir/lib/CAPI/Conversion/Passes.cpp
-mlir/lib/CAPI/Debug/Debug.cpp
-mlir/lib/CAPI/Dialect/Async.cpp
-mlir/lib/CAPI/Dialect/AsyncPasses.cpp
-mlir/lib/CAPI/Dialect/Func.cpp
-mlir/lib/CAPI/Dialect/GPU.cpp
-mlir/lib/CAPI/Dialect/GPUPasses.cpp
-mlir/lib/CAPI/Dialect/Linalg.cpp
-mlir/lib/CAPI/Dialect/LinalgPasses.cpp
-mlir/lib/CAPI/Dialect/LLVM.cpp
-mlir/lib/CAPI/Dialect/PDL.cpp
-mlir/lib/CAPI/Dialect/Quant.cpp
-mlir/lib/CAPI/Dialect/SCF.cpp
-mlir/lib/CAPI/Dialect/Shape.cpp
-mlir/lib/CAPI/Dialect/SparseTensor.cpp
-mlir/lib/CAPI/Dialect/SparseTensorPasses.cpp
-mlir/lib/CAPI/Dialect/Tensor.cpp
-mlir/lib/CAPI/ExecutionEngine/ExecutionEngine.cpp
-mlir/lib/CAPI/Interfaces/Interfaces.cpp
-mlir/lib/CAPI/IR/AffineExpr.cpp
-mlir/lib/CAPI/IR/AffineMap.cpp
-mlir/lib/CAPI/IR/BuiltinAttributes.cpp
-mlir/lib/CAPI/IR/BuiltinTypes.cpp
-mlir/lib/CAPI/IR/Diagnostics.cpp
-mlir/lib/CAPI/IR/DialectHandle.cpp
-mlir/lib/CAPI/IR/IntegerSet.cpp
-mlir/lib/CAPI/IR/IR.cpp
-mlir/lib/CAPI/IR/Pass.cpp
-mlir/lib/CAPI/IR/Support.cpp
-mlir/lib/CAPI/Registration/Registration.cpp
-mlir/lib/CAPI/Transforms/Passes.cpp
-mlir/lib/Conversion/PassDetail.h
-mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
-mlir/lib/Conversion/ArithmeticToLLVM/ArithmeticToLLVM.cpp
-mlir/lib/Conversion/ArithmeticToSPIRV/ArithmeticToSPIRV.cpp
-mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
-mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
-mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
-mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
-mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
-mlir/lib/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRV.cpp
-mlir/lib/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRVPass.cpp
-mlir/lib/Conversion/FuncToSPIRV/FuncToSPIRV.cpp
-mlir/lib/Conversion/FuncToSPIRV/FuncToSPIRVPass.cpp
-mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
-mlir/lib/Conversion/GPUCommon/GPUOpsLowering.h
-mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
-mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
-mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
-mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
-mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
-mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
-mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
-mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp
-mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp
-mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRVPass.cpp
-mlir/lib/Conversion/LLVMCommon/ConversionTarget.cpp
-mlir/lib/Conversion/LLVMCommon/LoweringOptions.cpp
-mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
-mlir/lib/Conversion/LLVMCommon/MemRefDescriptor.h
-mlir/lib/Conversion/LLVMCommon/Pattern.cpp
-mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp
-mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
-mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
-mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
-mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
-mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
-mlir/lib/Conversion/MathToSPIRV/MathToSPIRVPass.cpp
-mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp
-mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
-mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
-mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRVPass.cpp
-mlir/lib/Conversion/OpenACCToLLVM/OpenACCToLLVM.cpp
-mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
-mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
-mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
-mlir/lib/Conversion/PDLToPDLInterp/Predicate.cpp
-mlir/lib/Conversion/PDLToPDLInterp/Predicate.h
-mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
-mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.h
-mlir/lib/Conversion/PDLToPDLInterp/RootOrdering.cpp
-mlir/lib/Conversion/PDLToPDLInterp/RootOrdering.h
-mlir/lib/Conversion/ReconcileUnrealizedCasts/ReconcileUnrealizedCasts.cpp
-mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
-mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
-mlir/lib/Conversion/SCFToGPU/SCFToGPUPass.cpp
-mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
-mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
-mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp
-mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
-mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
-mlir/lib/Conversion/SPIRVCommon/Pattern.h
-mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
-mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
-mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.cpp
-mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
-mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
-mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRVPass.cpp
-mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
-mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
-mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamedPass.cpp
-mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
-mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
-mlir/lib/Conversion/TosaToSCF/TosaToSCFPass.cpp
-mlir/lib/Conversion/TosaToStandard/TosaToStandard.cpp
-mlir/lib/Conversion/TosaToStandard/TosaToStandardPass.cpp
-mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
-mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
-mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
-mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp
-mlir/lib/Dialect/Traits.cpp
-mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp
-mlir/lib/Dialect/Affine/Analysis/AffineStructures.cpp
-mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
-mlir/lib/Dialect/Affine/Analysis/NestedMatcher.cpp
-mlir/lib/Dialect/Affine/Analysis/Utils.cpp
-mlir/lib/Dialect/Affine/IR/AffineMemoryOpInterfaces.cpp
-mlir/lib/Dialect/Affine/IR/AffineValueMap.cpp
-mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
-mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp
-mlir/lib/Dialect/Affine/Transforms/AffineLoopNormalize.cpp
-mlir/lib/Dialect/Affine/Transforms/AffineParallelize.cpp
-mlir/lib/Dialect/Affine/Transforms/AffineScalarReplacement.cpp
-mlir/lib/Dialect/Affine/Transforms/LoopCoalescing.cpp
-mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
-mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
-mlir/lib/Dialect/Affine/Transforms/LoopUnroll.cpp
-mlir/lib/Dialect/Affine/Transforms/LoopUnrollAndJam.cpp
-mlir/lib/Dialect/Affine/Transforms/PassDetail.h
-mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
-mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
-mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
-mlir/lib/Dialect/Affine/Utils/LoopFusionUtils.cpp
-mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
-mlir/lib/Dialect/Affine/Utils/Utils.cpp
-mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
-mlir/lib/Dialect/AMX/Transforms/LegalizeForLLVMExport.cpp
-mlir/lib/Dialect/Arithmetic/IR/ArithmeticDialect.cpp
-mlir/lib/Dialect/Arithmetic/Transforms/BufferizableOpInterfaceImpl.cpp
-mlir/lib/Dialect/Arithmetic/Transforms/Bufferize.cpp
-mlir/lib/Dialect/Arithmetic/Transforms/PassDetail.h
-mlir/lib/Dialect/Arithmetic/Utils/Utils.cpp
-mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
-mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp
-mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
-mlir/lib/Dialect/Async/IR/Async.cpp
-mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
-mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
-mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCountingOpt.cpp
-mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
-mlir/lib/Dialect/Async/Transforms/PassDetail.cpp
-mlir/lib/Dialect/Async/Transforms/PassDetail.h
-mlir/lib/Dialect/Bufferization/IR/AllocationOpInterface.cpp
-mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
-mlir/lib/Dialect/Bufferization/IR/BufferizationDialect.cpp
-mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
-mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocation.cpp
-mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
-mlir/lib/Dialect/Bufferization/Transforms/BufferOptimizations.cpp
-mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
-mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
-mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
-mlir/lib/Dialect/Bufferization/Transforms/PassDetail.h
-mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
-mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
-mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
-mlir/lib/Dialect/DLTI/DLTI.cpp
-mlir/lib/Dialect/DLTI/Traits.cpp
-mlir/lib/Dialect/EmitC/IR/EmitC.cpp
-mlir/lib/Dialect/Func/IR/FuncOps.cpp
-mlir/lib/Dialect/Func/Transforms/DecomposeCallGraphTypes.cpp
-mlir/lib/Dialect/Func/Transforms/FuncBufferize.cpp
-mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
-mlir/lib/Dialect/Func/Transforms/PassDetail.h
-mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
-mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
-mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
-mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
-mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
-mlir/lib/Dialect/GPU/Transforms/ParallelLoopMapper.cpp
-mlir/lib/Dialect/GPU/Transforms/PassDetail.h
-mlir/lib/Dialect/GPU/Transforms/SerializeToBlob.cpp
-mlir/lib/Dialect/Linalg/Analysis/DependenceAnalysis.cpp
-mlir/lib/Dialect/Linalg/ComprehensiveBufferize/AffineInterfaceImpl.cpp
-mlir/lib/Dialect/Linalg/ComprehensiveBufferize/ModuleBufferization.cpp
-mlir/lib/Dialect/Linalg/IR/LinalgDialect.cpp
-mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
-mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
-mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
-mlir/lib/Dialect/Linalg/Transforms/Bufferize.cpp
-mlir/lib/Dialect/Linalg/Transforms/CodegenStrategy.cpp
-mlir/lib/Dialect/Linalg/Transforms/ComprehensiveBufferizePass.cpp
-mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
-mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
-mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
-mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
-mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
-mlir/lib/Dialect/Linalg/Transforms/FusionOnTensors.cpp
-mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
-mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
-mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
-mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
-mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
-mlir/lib/Dialect/Linalg/Transforms/LinalgStrategyPasses.cpp
-mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
-mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
-mlir/lib/Dialect/Linalg/Transforms/PadOpInterchange.cpp
-mlir/lib/Dialect/Linalg/Transforms/PassDetail.h
-mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
-mlir/lib/Dialect/Linalg/Transforms/SparseTensorRewriting.cpp
-mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
-mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
-mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
-mlir/lib/Dialect/Linalg/Utils/Utils.cpp
-mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
-mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
-mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
-mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
-mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h
-mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
-mlir/lib/Dialect/LLVMIR/Transforms/PassDetail.h
-mlir/lib/Dialect/Math/IR/MathDialect.cpp
-mlir/lib/Dialect/Math/IR/MathOps.cpp
-mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
-mlir/lib/Dialect/Math/Transforms/ExpandTanh.cpp
-mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
-mlir/lib/Dialect/MemRef/IR/MemRefDialect.cpp
-mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
-mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
-mlir/lib/Dialect/MemRef/Transforms/FoldSubViewOps.cpp
-mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
-mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
-mlir/lib/Dialect/MemRef/Transforms/PassDetail.h
-mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
-mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
-mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
-mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
-mlir/lib/Dialect/PDL/IR/PDL.cpp
-mlir/lib/Dialect/PDL/IR/PDLTypes.cpp
-mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
-mlir/lib/Dialect/Quant/IR/QuantOps.cpp
-mlir/lib/Dialect/Quant/IR/QuantTypes.cpp
-mlir/lib/Dialect/Quant/IR/TypeDetail.h
-mlir/lib/Dialect/Quant/IR/TypeParser.cpp
-mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp
-mlir/lib/Dialect/Quant/Transforms/ConvertSimQuant.cpp
-mlir/lib/Dialect/Quant/Transforms/PassDetail.h
-mlir/lib/Dialect/Quant/Utils/FakeQuantSupport.cpp
-mlir/lib/Dialect/Quant/Utils/QuantizeUtils.cpp
-mlir/lib/Dialect/Quant/Utils/UniformSupport.cpp
-mlir/lib/Dialect/SCF/SCF.cpp
-mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
-mlir/lib/Dialect/SCF/Transforms/Bufferize.cpp
-mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
-mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
-mlir/lib/Dialect/SCF/Transforms/LoopRangeFolding.cpp
-mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
-mlir/lib/Dialect/SCF/Transforms/ParallelLoopCollapsing.cpp
-mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
-mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
-mlir/lib/Dialect/SCF/Transforms/PassDetail.h
-mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
-mlir/lib/Dialect/SCF/Utils/AffineCanonicalizationUtils.cpp
-mlir/lib/Dialect/SCF/Utils/Utils.cpp
-mlir/lib/Dialect/Shape/IR/Shape.cpp
-mlir/lib/Dialect/Shape/Transforms/Bufferize.cpp
-mlir/lib/Dialect/Shape/Transforms/PassDetail.h
-mlir/lib/Dialect/Shape/Transforms/RemoveShapeConstraints.cpp
-mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
-mlir/lib/Dialect/Shape/Transforms/StructuralTypeConversions.cpp
-mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
-mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp
-mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.cpp
-mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.h
-mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
-mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorPasses.cpp
-mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
-mlir/lib/Dialect/SPIRV/IR/SPIRVAttributes.cpp
-mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
-mlir/lib/Dialect/SPIRV/IR/SPIRVEnums.cpp
-mlir/lib/Dialect/SPIRV/IR/SPIRVGLSLCanonicalization.cpp
-mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
-mlir/lib/Dialect/SPIRV/IR/TargetAndABI.cpp
-mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
-mlir/lib/Dialect/SPIRV/Transforms/DecorateCompositeTypeLayoutPass.cpp
-mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
-mlir/lib/Dialect/SPIRV/Transforms/PassDetail.h
-mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
-mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
-mlir/lib/Dialect/SPIRV/Transforms/UpdateVCEPass.cpp
-mlir/lib/Dialect/SPIRV/Utils/LayoutUtils.cpp
-mlir/lib/Dialect/Tensor/IR/TensorDialect.cpp
-mlir/lib/Dialect/Tensor/IR/TensorInferTypeOpInterfaceImpl.cpp
-mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
-mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
-mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
-mlir/lib/Dialect/Tensor/Transforms/Bufferize.cpp
-mlir/lib/Dialect/Tensor/Transforms/PassDetail.h
-mlir/lib/Dialect/Tensor/Utils/Utils.cpp
-mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
-mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeConv2D.cpp
-mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
-mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
-mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
-mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
-mlir/lib/Dialect/Tosa/Transforms/TosaOptionalDecompositions.cpp
-mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
-mlir/lib/Dialect/Tosa/Utils/QuantUtils.cpp
-mlir/lib/Dialect/Utils/IndexingUtils.cpp
-mlir/lib/Dialect/Utils/ReshapeOpsUtils.cpp
-mlir/lib/Dialect/Utils/StaticValueUtils.cpp
-mlir/lib/Dialect/Utils/StructuredOpsUtils.cpp
-mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
-mlir/lib/Dialect/Vector/Transforms/Bufferize.cpp
-mlir/lib/Dialect/Vector/Transforms/PassDetail.h
-mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
-mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
-mlir/lib/Dialect/Vector/Transforms/VectorMultiDimReductionTransforms.cpp
-mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
-mlir/lib/Dialect/Vector/Transforms/VectorTransferPermutationMapRewritePatterns.cpp
-mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
-mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
-mlir/lib/Dialect/Vector/Transforms/VectorUnrollDistribute.cpp
-mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
-mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
-mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
-mlir/lib/Dialect/X86Vector/Transforms/LegalizeForLLVMExport.cpp
-mlir/lib/ExecutionEngine/AsyncRuntime.cpp
-mlir/lib/ExecutionEngine/CRunnerUtils.cpp
-mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp
-mlir/lib/ExecutionEngine/ExecutionEngine.cpp
-mlir/lib/ExecutionEngine/JitRunner.cpp
-mlir/lib/ExecutionEngine/OptUtils.cpp
-mlir/lib/ExecutionEngine/RocmRuntimeWrappers.cpp
-mlir/lib/ExecutionEngine/RunnerUtils.cpp
-mlir/lib/ExecutionEngine/SparseTensorUtils.cpp
-mlir/lib/Interfaces/CallInterfaces.cpp
-mlir/lib/Interfaces/CastInterfaces.cpp
-mlir/lib/Interfaces/ControlFlowInterfaces.cpp
-mlir/lib/Interfaces/CopyOpInterface.cpp
-mlir/lib/Interfaces/DataLayoutInterfaces.cpp
-mlir/lib/Interfaces/DerivedAttributeOpInterface.cpp
-mlir/lib/Interfaces/InferTypeOpInterface.cpp
-mlir/lib/Interfaces/LoopLikeInterface.cpp
-mlir/lib/Interfaces/SideEffectInterfaces.cpp
-mlir/lib/Interfaces/TilingInterface.cpp
-mlir/lib/Interfaces/VectorInterfaces.cpp
-mlir/lib/Interfaces/ViewLikeInterface.cpp
-mlir/lib/IR/AffineExprDetail.h
-mlir/lib/IR/AffineMap.cpp
-mlir/lib/IR/AffineMapDetail.h
-mlir/lib/IR/AttributeDetail.h
-mlir/lib/IR/Attributes.cpp
-mlir/lib/IR/Builders.cpp
-mlir/lib/IR/BuiltinAttributeInterfaces.cpp
-mlir/lib/IR/BuiltinAttributes.cpp
-mlir/lib/IR/BuiltinDialect.cpp
-mlir/lib/IR/BuiltinTypeInterfaces.cpp
-mlir/lib/IR/BuiltinTypes.cpp
-mlir/lib/IR/Dialect.cpp
-mlir/lib/IR/Dominance.cpp
-mlir/lib/IR/FunctionImplementation.cpp
-mlir/lib/IR/IntegerSet.cpp
-mlir/lib/IR/IntegerSetDetail.h
-mlir/lib/IR/Location.cpp
-mlir/lib/IR/MLIRContext.cpp
-mlir/lib/IR/Operation.cpp
-mlir/lib/IR/PatternMatch.cpp
-mlir/lib/IR/Region.cpp
-mlir/lib/IR/RegionKindInterface.cpp
-mlir/lib/IR/SubElementInterfaces.cpp
-mlir/lib/IR/SymbolTable.cpp
-mlir/lib/IR/TensorEncoding.cpp
-mlir/lib/IR/TypeDetail.h
-mlir/lib/IR/TypeRange.cpp
-mlir/lib/IR/Types.cpp
-mlir/lib/IR/TypeUtilities.cpp
-mlir/lib/IR/Verifier.cpp
-mlir/lib/IR/Visitors.cpp
-mlir/lib/Parser/AffineParser.cpp
-mlir/lib/Parser/Lexer.cpp
-mlir/lib/Parser/Lexer.h
-mlir/lib/Parser/LocationParser.cpp
-mlir/lib/Parser/Parser.cpp
-mlir/lib/Parser/Parser.h
-mlir/lib/Parser/ParserState.h
-mlir/lib/Parser/Token.cpp
-mlir/lib/Parser/Token.h
-mlir/lib/Parser/TypeParser.cpp
-mlir/lib/Pass/Pass.cpp
-mlir/lib/Pass/PassCrashRecovery.cpp
-mlir/lib/Pass/PassDetail.h
-mlir/lib/Pass/PassManagerOptions.cpp
-mlir/lib/Pass/PassStatistics.cpp
-mlir/lib/Pass/PassTiming.cpp
-mlir/lib/Reducer/OptReductionPass.cpp
-mlir/lib/Reducer/ReductionNode.cpp
-mlir/lib/Reducer/ReductionTreePass.cpp
-mlir/lib/Reducer/Tester.cpp
-mlir/lib/Rewrite/ByteCode.cpp
-mlir/lib/Rewrite/ByteCode.h
-mlir/lib/Rewrite/FrozenRewritePatternSet.cpp
-mlir/lib/Support/DebugCounter.cpp
-mlir/lib/Support/FileUtilities.cpp
-mlir/lib/Support/IndentedOstream.cpp
-mlir/lib/Support/InterfaceSupport.cpp
-mlir/lib/Support/MlirOptMain.cpp
-mlir/lib/Support/StorageUniquer.cpp
-mlir/lib/Support/Timing.cpp
-mlir/lib/Support/ToolUtilities.cpp
-mlir/lib/TableGen/Argument.cpp
-mlir/lib/TableGen/Attribute.cpp
-mlir/lib/TableGen/AttrOrTypeDef.cpp
-mlir/lib/TableGen/Builder.cpp
-mlir/lib/TableGen/Class.cpp
-mlir/lib/TableGen/Constraint.cpp
-mlir/lib/TableGen/Dialect.cpp
-mlir/lib/TableGen/Format.cpp
-mlir/lib/TableGen/Interfaces.cpp
-mlir/lib/TableGen/Operator.cpp
-mlir/lib/TableGen/Pass.cpp
-mlir/lib/TableGen/Pattern.cpp
-mlir/lib/TableGen/Predicate.cpp
-mlir/lib/TableGen/Region.cpp
-mlir/lib/TableGen/SideEffects.cpp
-mlir/lib/TableGen/Successor.cpp
-mlir/lib/TableGen/Trait.cpp
-mlir/lib/TableGen/Type.cpp
-mlir/lib/Target/Cpp/TranslateRegistration.cpp
-mlir/lib/Target/Cpp/TranslateToCpp.cpp
-mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
-mlir/lib/Target/LLVMIR/ConvertToLLVMIR.cpp
-mlir/lib/Target/LLVMIR/DebugTranslation.cpp
-mlir/lib/Target/LLVMIR/DebugTranslation.h
-mlir/lib/Target/LLVMIR/TypeFromLLVM.cpp
-mlir/lib/Target/LLVMIR/TypeToLLVM.cpp
-mlir/lib/Target/LLVMIR/Dialect/AMX/AMXToLLVMIRTranslation.cpp
-mlir/lib/Target/LLVMIR/Dialect/ArmNeon/ArmNeonToLLVMIRTranslation.cpp
-mlir/lib/Target/LLVMIR/Dialect/ArmSVE/ArmSVEToLLVMIRTranslation.cpp
-mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
-mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
-mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
-mlir/lib/Target/LLVMIR/Dialect/ROCDL/ROCDLToLLVMIRTranslation.cpp
-mlir/lib/Target/LLVMIR/Dialect/X86Vector/X86VectorToLLVMIRTranslation.cpp
-mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
-mlir/lib/Target/SPIRV/TranslateRegistration.cpp
-mlir/lib/Target/SPIRV/Deserialization/Deserialization.cpp
-mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
-mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
-mlir/lib/Target/SPIRV/Serialization/Serialization.cpp
-mlir/lib/Target/SPIRV/Serialization/SerializeOps.cpp
-mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
-mlir/lib/Tools/mlir-lsp-server/LSPServer.cpp
-mlir/lib/Tools/mlir-lsp-server/LSPServer.h
-mlir/lib/Tools/mlir-lsp-server/MlirLspServerMain.cpp
-mlir/lib/Tools/mlir-lsp-server/MLIRServer.h
-mlir/lib/Tools/mlir-lsp-server/lsp/Logging.cpp
-mlir/lib/Tools/mlir-lsp-server/lsp/Protocol.cpp
-mlir/lib/Tools/mlir-lsp-server/lsp/Transport.cpp
-mlir/lib/Tools/mlir-lsp-server/lsp/Transport.h
-mlir/lib/Tools/mlir-reduce/MlirReduceMain.cpp
-mlir/lib/Tools/PDLL/AST/Context.cpp
-mlir/lib/Tools/PDLL/AST/Diagnostic.cpp
-mlir/lib/Tools/PDLL/AST/NodePrinter.cpp
-mlir/lib/Tools/PDLL/AST/TypeDetail.h
-mlir/lib/Tools/PDLL/AST/Types.cpp
-mlir/lib/Tools/PDLL/CodeGen/CPPGen.cpp
-mlir/lib/Tools/PDLL/ODS/Context.cpp
-mlir/lib/Tools/PDLL/ODS/Dialect.cpp
-mlir/lib/Tools/PDLL/ODS/Operation.cpp
-mlir/lib/Tools/PDLL/Parser/Parser.cpp
-mlir/lib/Transforms/Canonicalizer.cpp
-mlir/lib/Transforms/ControlFlowSink.cpp
-mlir/lib/Transforms/CSE.cpp
-mlir/lib/Transforms/Inliner.cpp
-mlir/lib/Transforms/LocationSnapshot.cpp
-mlir/lib/Transforms/LoopInvariantCodeMotion.cpp
-mlir/lib/Transforms/PassDetail.h
-mlir/lib/Transforms/SCCP.cpp
-mlir/lib/Transforms/StripDebugInfo.cpp
-mlir/lib/Transforms/SymbolDCE.cpp
-mlir/lib/Transforms/SymbolPrivatize.cpp
-mlir/lib/Transforms/Utils/ControlFlowSinkUtils.cpp
-mlir/lib/Transforms/Utils/DialectConversion.cpp
-mlir/lib/Transforms/Utils/FoldUtils.cpp
-mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
-mlir/lib/Transforms/Utils/InliningUtils.cpp
-mlir/lib/Transforms/Utils/RegionUtils.cpp
-mlir/lib/Translation/Translation.cpp
-mlir/tools/mlir-cpu-runner/mlir-cpu-runner.cpp
-mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp
-mlir/tools/mlir-lsp-server/mlir-lsp-server.cpp
-mlir/tools/mlir-opt/mlir-opt.cpp
-mlir/tools/mlir-pdll/mlir-pdll.cpp
-mlir/tools/mlir-reduce/mlir-reduce.cpp
-mlir/tools/mlir-shlib/mlir-shlib.cpp
-mlir/tools/mlir-spirv-cpu-runner/mlir-spirv-cpu-runner.cpp
-mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
-mlir/tools/mlir-tblgen/AttrOrTypeFormatGen.cpp
-mlir/tools/mlir-tblgen/AttrOrTypeFormatGen.h
-mlir/tools/mlir-tblgen/CodeGenHelpers.cpp
-mlir/tools/mlir-tblgen/DialectGen.cpp
-mlir/tools/mlir-tblgen/DirectiveCommonGen.cpp
-mlir/tools/mlir-tblgen/DocGenUtilities.h
-mlir/tools/mlir-tblgen/EnumsGen.cpp
-mlir/tools/mlir-tblgen/FormatGen.cpp
-mlir/tools/mlir-tblgen/FormatGen.h
-mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
-mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp
-mlir/tools/mlir-tblgen/mlir-tblgen.cpp
-mlir/tools/mlir-tblgen/OpClass.cpp
-mlir/tools/mlir-tblgen/OpClass.h
-mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
-mlir/tools/mlir-tblgen/OpDocGen.cpp
-mlir/tools/mlir-tblgen/OpFormatGen.h
-mlir/tools/mlir-tblgen/OpGenHelpers.cpp
-mlir/tools/mlir-tblgen/OpGenHelpers.h
-mlir/tools/mlir-tblgen/OpInterfacesGen.cpp
-mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
-mlir/tools/mlir-tblgen/PassCAPIGen.cpp
-mlir/tools/mlir-tblgen/PassDocGen.cpp
-mlir/tools/mlir-tblgen/PassGen.cpp
-mlir/tools/mlir-tblgen/RewriterGen.cpp
-mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
-mlir/tools/mlir-tblgen/StructsGen.cpp
-mlir/tools/mlir-translate/mlir-translate.cpp
-mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp
-mlir/tools/mlir-vulkan-runner/vulkan-runtime-wrappers.cpp
-mlir/tools/mlir-vulkan-runner/VulkanRuntime.cpp
-mlir/tools/mlir-vulkan-runner/VulkanRuntime.h
-mlir/unittests/Analysis/Presburger/IntegerPolyhedronTest.cpp
-mlir/unittests/Analysis/Presburger/LinearTransformTest.cpp
-mlir/unittests/Analysis/Presburger/MatrixTest.cpp
-mlir/unittests/Analysis/Presburger/PresburgerSetTest.cpp
-mlir/unittests/Analysis/Presburger/PresburgerSpaceTest.cpp
-mlir/unittests/Analysis/Presburger/PWMAFunctionTest.cpp
-mlir/unittests/Analysis/Presburger/SimplexTest.cpp
-mlir/unittests/Analysis/Presburger/Utils.h
-mlir/unittests/Conversion/PDLToPDLInterp/RootOrderingTest.cpp
-mlir/unittests/Dialect/BroadcastShapeTest.cpp
-mlir/unittests/Dialect/Affine/Analysis/AffineStructuresParser.cpp
-mlir/unittests/Dialect/Affine/Analysis/AffineStructuresParser.h
-mlir/unittests/Dialect/Affine/Analysis/AffineStructuresParserTest.cpp
-mlir/unittests/Dialect/Quant/QuantizationUtilsTest.cpp
-mlir/unittests/Dialect/SparseTensor/MergerTest.cpp
-mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp
-mlir/unittests/Dialect/SPIRV/SerializationTest.cpp
-mlir/unittests/Dialect/Utils/StructuredOpsUtilsTest.cpp
-mlir/unittests/ExecutionEngine/Invoke.cpp
-mlir/unittests/Interfaces/ControlFlowInterfacesTest.cpp
-mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp
-mlir/unittests/Interfaces/InferTypeOpInterfaceTest.cpp
-mlir/unittests/IR/AttributeTest.cpp
-mlir/unittests/IR/DialectTest.cpp
-mlir/unittests/IR/InterfaceAttachmentTest.cpp
-mlir/unittests/IR/OperationSupportTest.cpp
-mlir/unittests/IR/PatternMatchTest.cpp
-mlir/unittests/IR/ShapedTypeTest.cpp
-mlir/unittests/IR/SubElementInterfaceTest.cpp
-mlir/unittests/Pass/AnalysisManagerTest.cpp
-mlir/unittests/Pass/PassManagerTest.cpp
-mlir/unittests/Pass/PassPipelineParserTest.cpp
-mlir/unittests/Rewrite/PatternBenefit.cpp
-mlir/unittests/Support/DebugCounterTest.cpp
-mlir/unittests/Support/IndentedOstreamTest.cpp
-mlir/unittests/Support/MathExtrasTest.cpp
-mlir/unittests/Support/StorageUniquerTest.cpp
-mlir/unittests/TableGen/EnumsGenTest.cpp
-mlir/unittests/TableGen/FormatTest.cpp
-mlir/unittests/TableGen/PassGenTest.cpp
-mlir/unittests/Transforms/Canonicalizer.cpp
-mlir/unittests/Transforms/DialectConversion.cpp
-openmp/libompd/src/Debug.h
-openmp/libompd/src/omp-debug.cpp
-openmp/libompd/src/omp-debug.h
-openmp/libompd/src/omp-icv.cpp
-openmp/libompd/src/omp-state.cpp
-openmp/libompd/src/ompd-private.h
-openmp/libompd/src/ompd-types.h
-openmp/libompd/src/TargetValue.cpp
-openmp/libompd/src/TargetValue.h
-openmp/libomptarget/DeviceRTL/include/Configuration.h
-openmp/libomptarget/DeviceRTL/include/Debug.h
-openmp/libomptarget/DeviceRTL/include/Interface.h
-openmp/libomptarget/DeviceRTL/include/Mapping.h
-openmp/libomptarget/DeviceRTL/include/State.h
-openmp/libomptarget/DeviceRTL/include/Synchronization.h
-openmp/libomptarget/DeviceRTL/include/Types.h
-openmp/libomptarget/DeviceRTL/include/Utils.h
-openmp/libomptarget/DeviceRTL/src/Configuration.cpp
-openmp/libomptarget/DeviceRTL/src/Kernel.cpp
-openmp/libomptarget/DeviceRTL/src/Misc.cpp
-openmp/libomptarget/DeviceRTL/src/Parallelism.cpp
-openmp/libomptarget/DeviceRTL/src/Reduction.cpp
-openmp/libomptarget/DeviceRTL/src/State.cpp
-openmp/libomptarget/DeviceRTL/src/Synchronization.cpp
-openmp/libomptarget/DeviceRTL/src/Tasking.cpp
-openmp/libomptarget/DeviceRTL/src/Utils.cpp
-openmp/libomptarget/include/Debug.h
-openmp/libomptarget/include/device.h
-openmp/libomptarget/include/DeviceEnvironment.h
-openmp/libomptarget/include/interop.h
-openmp/libomptarget/include/omptarget.h
-openmp/libomptarget/include/omptargetplugin.h
-openmp/libomptarget/include/rtl.h
-openmp/libomptarget/include/SourceInfo.h
-openmp/libomptarget/plugins/amdgpu/dynamic_hsa/hsa.cpp
-openmp/libomptarget/plugins/amdgpu/dynamic_hsa/hsa.h
-openmp/libomptarget/plugins/amdgpu/impl/get_elf_mach_gfx_name.cpp
-openmp/libomptarget/plugins/amdgpu/impl/get_elf_mach_gfx_name.h
-openmp/libomptarget/plugins/amdgpu/impl/hsa_api.h
-openmp/libomptarget/plugins/amdgpu/impl/impl.cpp
-openmp/libomptarget/plugins/amdgpu/impl/impl_runtime.h
-openmp/libomptarget/plugins/amdgpu/impl/internal.h
-openmp/libomptarget/plugins/amdgpu/impl/interop_hsa.cpp
-openmp/libomptarget/plugins/amdgpu/impl/msgpack.cpp
-openmp/libomptarget/plugins/amdgpu/impl/msgpack.h
-openmp/libomptarget/plugins/amdgpu/impl/rt.h
-openmp/libomptarget/plugins/amdgpu/src/print_tracing.h
-openmp/libomptarget/plugins/common/elf_common/elf_common.cpp
-openmp/libomptarget/plugins/common/elf_common/elf_common.h
-openmp/libomptarget/plugins/common/MemoryManager/MemoryManager.h
-openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.cpp
-openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.h
-openmp/libomptarget/plugins/generic-elf-64bit/src/rtl.cpp
-openmp/libomptarget/plugins/remote/include/Utils.h
-openmp/libomptarget/plugins/remote/server/OffloadingServer.cpp
-openmp/libomptarget/plugins/remote/server/Server.cpp
-openmp/libomptarget/plugins/remote/server/Server.h
-openmp/libomptarget/plugins/remote/src/Client.cpp
-openmp/libomptarget/plugins/remote/src/Client.h
-openmp/libomptarget/plugins/ve/src/rtl.cpp
-openmp/libomptarget/src/api.cpp
-openmp/libomptarget/src/interface.cpp
-openmp/libomptarget/src/interop.cpp
-openmp/libomptarget/src/omptarget.cpp
-openmp/libomptarget/src/private.h
-openmp/libomptarget/src/rtl.cpp
-openmp/libomptarget/tools/deviceinfo/llvm-omp-device-info.cpp
-openmp/runtime/doc/doxygen/libomp_interface.h
-openmp/runtime/src/extractExternal.cpp
-openmp/runtime/src/kmp.h
-openmp/runtime/src/kmp_affinity.h
-openmp/runtime/src/kmp_alloc.cpp
-openmp/runtime/src/kmp_atomic.cpp
-openmp/runtime/src/kmp_atomic.h
-openmp/runtime/src/kmp_barrier.h
-openmp/runtime/src/kmp_cancel.cpp
-openmp/runtime/src/kmp_debug.cpp
-openmp/runtime/src/kmp_debug.h
-openmp/runtime/src/kmp_debugger.cpp
-openmp/runtime/src/kmp_debugger.h
-openmp/runtime/src/kmp_dispatch.cpp
-openmp/runtime/src/kmp_dispatch.h
-openmp/runtime/src/kmp_dispatch_hier.h
-openmp/runtime/src/kmp_environment.cpp
-openmp/runtime/src/kmp_environment.h
-openmp/runtime/src/kmp_error.cpp
-openmp/runtime/src/kmp_error.h
-openmp/runtime/src/kmp_ftn_cdecl.cpp
-openmp/runtime/src/kmp_ftn_extra.cpp
-openmp/runtime/src/kmp_ftn_os.h
-openmp/runtime/src/kmp_ftn_stdcall.cpp
-openmp/runtime/src/kmp_global.cpp
-openmp/runtime/src/kmp_i18n.cpp
-openmp/runtime/src/kmp_i18n.h
-openmp/runtime/src/kmp_import.cpp
-openmp/runtime/src/kmp_io.cpp
-openmp/runtime/src/kmp_io.h
-openmp/runtime/src/kmp_itt.cpp
-openmp/runtime/src/kmp_itt.h
-openmp/runtime/src/kmp_lock.cpp
-openmp/runtime/src/kmp_omp.h
-openmp/runtime/src/kmp_platform.h
-openmp/runtime/src/kmp_safe_c_api.h
-openmp/runtime/src/kmp_sched.cpp
-openmp/runtime/src/kmp_settings.cpp
-openmp/runtime/src/kmp_settings.h
-openmp/runtime/src/kmp_stats.h
-openmp/runtime/src/kmp_stats_timing.cpp
-openmp/runtime/src/kmp_stats_timing.h
-openmp/runtime/src/kmp_str.cpp
-openmp/runtime/src/kmp_str.h
-openmp/runtime/src/kmp_stub.cpp
-openmp/runtime/src/kmp_stub.h
-openmp/runtime/src/kmp_taskdeps.cpp
-openmp/runtime/src/kmp_taskdeps.h
-openmp/runtime/src/kmp_threadprivate.cpp
-openmp/runtime/src/kmp_utility.cpp
-openmp/runtime/src/kmp_version.cpp
-openmp/runtime/src/kmp_version.h
-openmp/runtime/src/kmp_wait_release.cpp
-openmp/runtime/src/kmp_wait_release.h
-openmp/runtime/src/kmp_wrapper_getpid.h
-openmp/runtime/src/kmp_wrapper_malloc.h
-openmp/runtime/src/ompd-specific.cpp
-openmp/runtime/src/ompd-specific.h
-openmp/runtime/src/ompt-event-specific.h
-openmp/runtime/src/ompt-general.cpp
-openmp/runtime/src/ompt-internal.h
-openmp/runtime/src/ompt-specific.cpp
-openmp/runtime/src/ompt-specific.h
-openmp/runtime/src/z_Linux_util.cpp
-openmp/runtime/src/z_Windows_NT-586_util.cpp
-openmp/runtime/src/z_Windows_NT_util.cpp
-openmp/runtime/src/thirdparty/ittnotify/ittnotify.h
-openmp/runtime/src/thirdparty/ittnotify/ittnotify_config.h
-openmp/runtime/src/thirdparty/ittnotify/ittnotify_static.cpp
-openmp/runtime/src/thirdparty/ittnotify/ittnotify_static.h
-openmp/runtime/src/thirdparty/ittnotify/ittnotify_types.h
-openmp/runtime/src/thirdparty/ittnotify/legacy/ittnotify.h
-openmp/tools/archer/ompt-tsan.cpp
-openmp/tools/archer/tests/ompt/ompt-signal.h
-openmp/tools/multiplex/ompt-multiplex.h
-openmp/tools/multiplex/tests/ompt-signal.h
-openmp/tools/multiplex/tests/custom_data_storage/second-tool.h
-openmp/tools/multiplex/tests/print/first-tool.h
-openmp/tools/multiplex/tests/print/second-tool.h
-polly/include/polly/Canonicalization.h
-polly/include/polly/CodePreparation.h
-polly/include/polly/DeadCodeElimination.h
-polly/include/polly/DeLICM.h
-polly/include/polly/DependenceInfo.h
-polly/include/polly/FlattenAlgo.h
-polly/include/polly/FlattenSchedule.h
-polly/include/polly/ForwardOpTree.h
-polly/include/polly/JSONExporter.h
-polly/include/polly/LinkAllPasses.h
-polly/include/polly/ManualOptimizer.h
-polly/include/polly/MatmulOptimizer.h
-polly/include/polly/Options.h
-polly/include/polly/PolyhedralInfo.h
-polly/include/polly/PruneUnprofitable.h
-polly/include/polly/RegisterPasses.h
-polly/include/polly/ScheduleOptimizer.h
-polly/include/polly/ScheduleTreeTransform.h
-polly/include/polly/ScopBuilder.h
-polly/include/polly/ScopDetection.h
-polly/include/polly/ScopDetectionDiagnostic.h
-polly/include/polly/ScopInfo.h
-polly/include/polly/ScopPass.h
-polly/include/polly/Simplify.h
-polly/include/polly/ZoneAlgo.h
-polly/include/polly/CodeGen/BlockGenerators.h
-polly/include/polly/CodeGen/CodegenCleanup.h
-polly/include/polly/CodeGen/CodeGeneration.h
-polly/include/polly/CodeGen/IRBuilder.h
-polly/include/polly/CodeGen/IslAst.h
-polly/include/polly/CodeGen/IslExprBuilder.h
-polly/include/polly/CodeGen/IslNodeBuilder.h
-polly/include/polly/CodeGen/LoopGenerators.h
-polly/include/polly/CodeGen/LoopGeneratorsGOMP.h
-polly/include/polly/CodeGen/LoopGeneratorsKMP.h
-polly/include/polly/CodeGen/PerfMonitor.h
-polly/include/polly/CodeGen/PPCGCodeGeneration.h
-polly/include/polly/CodeGen/RuntimeDebugBuilder.h
-polly/include/polly/CodeGen/Utils.h
-polly/include/polly/Support/DumpFunctionPass.h
-polly/include/polly/Support/DumpModulePass.h
-polly/include/polly/Support/GICHelper.h
-polly/include/polly/Support/ISLOperators.h
-polly/include/polly/Support/ISLOStream.h
-polly/include/polly/Support/ISLTools.h
-polly/include/polly/Support/LinkGPURuntime.h
-polly/include/polly/Support/SCEVAffinator.h
-polly/include/polly/Support/SCEVValidator.h
-polly/include/polly/Support/ScopHelper.h
-polly/include/polly/Support/ScopLocation.h
-polly/include/polly/Support/VirtualInstruction.h
-polly/lib/Analysis/DependenceInfo.cpp
-polly/lib/Analysis/PolyhedralInfo.cpp
-polly/lib/Analysis/PruneUnprofitable.cpp
-polly/lib/Analysis/ScopBuilder.cpp
-polly/lib/Analysis/ScopDetection.cpp
-polly/lib/Analysis/ScopDetectionDiagnostic.cpp
-polly/lib/Analysis/ScopGraphPrinter.cpp
-polly/lib/Analysis/ScopInfo.cpp
-polly/lib/Analysis/ScopPass.cpp
-polly/lib/CodeGen/BlockGenerators.cpp
-polly/lib/CodeGen/CodegenCleanup.cpp
-polly/lib/CodeGen/CodeGeneration.cpp
-polly/lib/CodeGen/IRBuilder.cpp
-polly/lib/CodeGen/IslAst.cpp
-polly/lib/CodeGen/IslExprBuilder.cpp
-polly/lib/CodeGen/IslNodeBuilder.cpp
-polly/lib/CodeGen/LoopGenerators.cpp
-polly/lib/CodeGen/LoopGeneratorsGOMP.cpp
-polly/lib/CodeGen/LoopGeneratorsKMP.cpp
-polly/lib/CodeGen/ManagedMemoryRewrite.cpp
-polly/lib/CodeGen/PerfMonitor.cpp
-polly/lib/CodeGen/PPCGCodeGeneration.cpp
-polly/lib/CodeGen/RuntimeDebugBuilder.cpp
-polly/lib/CodeGen/Utils.cpp
-polly/lib/Exchange/JSONExporter.cpp
-polly/lib/External/isl/isl_local_private.h
-polly/lib/External/isl/imath/iprime.h
-polly/lib/External/isl/include/isl/id_type.h
-polly/lib/External/isl/include/isl/isl-noexceptions.h
-polly/lib/External/isl/include/isl/map_type.h
-polly/lib/External/isl/include/isl/printer_type.h
-polly/lib/External/isl/include/isl/set_type.h
-polly/lib/External/isl/include/isl/union_map_type.h
-polly/lib/External/isl/include/isl/union_set_type.h
-polly/lib/External/isl/include/isl/val_type.h
-polly/lib/External/isl/include/isl/version.h
-polly/lib/External/isl/interface/extract_interface.h
-polly/lib/Plugin/Polly.cpp
-polly/lib/Support/DumpFunctionPass.cpp
-polly/lib/Support/DumpModulePass.cpp
-polly/lib/Support/GICHelper.cpp
-polly/lib/Support/ISLTools.cpp
-polly/lib/Support/Mainpage.h
-polly/lib/Support/RegisterPasses.cpp
-polly/lib/Support/SCEVAffinator.cpp
-polly/lib/Support/SCEVValidator.cpp
-polly/lib/Support/ScopHelper.cpp
-polly/lib/Support/ScopLocation.cpp
-polly/lib/Support/VirtualInstruction.cpp
-polly/lib/Transform/Canonicalization.cpp
-polly/lib/Transform/CodePreparation.cpp
-polly/lib/Transform/DeadCodeElimination.cpp
-polly/lib/Transform/DeLICM.cpp
-polly/lib/Transform/FlattenAlgo.cpp
-polly/lib/Transform/FlattenSchedule.cpp
-polly/lib/Transform/ForwardOpTree.cpp
-polly/lib/Transform/ManualOptimizer.cpp
-polly/lib/Transform/MatmulOptimizer.cpp
-polly/lib/Transform/MaximalStaticExpansion.cpp
-polly/lib/Transform/ScheduleOptimizer.cpp
-polly/lib/Transform/ScheduleTreeTransform.cpp
-polly/lib/Transform/ScopInliner.cpp
-polly/lib/Transform/Simplify.cpp
-polly/lib/Transform/ZoneAlgo.cpp
-polly/tools/GPURuntime/GPUJIT.h
-polly/unittests/DeLICM/DeLICMTest.cpp
-polly/unittests/Flatten/FlattenTest.cpp
-polly/unittests/Isl/IslTest.cpp
-polly/unittests/ScheduleOptimizer/ScheduleTreeTransformTest.cpp
-polly/unittests/ScopPassManager/PassManagerTest.cpp
-polly/unittests/Support/ISLTools.cpp
-pstl/include/pstl/internal/algorithm_fwd.h
-pstl/include/pstl/internal/execution_defs.h
-pstl/include/pstl/internal/execution_impl.h
-pstl/include/pstl/internal/glue_algorithm_defs.h
-pstl/include/pstl/internal/glue_algorithm_impl.h
-pstl/include/pstl/internal/glue_execution_defs.h
-pstl/include/pstl/internal/glue_memory_defs.h
-pstl/include/pstl/internal/glue_memory_impl.h
-pstl/include/pstl/internal/glue_numeric_defs.h
-pstl/include/pstl/internal/glue_numeric_impl.h
-pstl/include/pstl/internal/numeric_fwd.h
-pstl/include/pstl/internal/parallel_backend.h
-pstl/include/pstl/internal/parallel_backend_omp.h
-pstl/include/pstl/internal/parallel_backend_serial.h
-pstl/include/pstl/internal/parallel_backend_utils.h
-pstl/include/pstl/internal/parallel_impl.h
-pstl/include/pstl/internal/omp/parallel_for.h
-pstl/include/pstl/internal/omp/parallel_for_each.h
-pstl/include/pstl/internal/omp/parallel_invoke.h
-pstl/include/pstl/internal/omp/parallel_reduce.h
-pstl/include/pstl/internal/omp/parallel_scan.h
-pstl/include/pstl/internal/omp/parallel_stable_partial_sort.h
-pstl/include/pstl/internal/omp/parallel_transform_scan.h
-pstl/include/pstl/internal/omp/util.h
-third-party/benchmark/cmake/thread_safety_attributes.cpp
-third-party/benchmark/src/arraysize.h
-third-party/benchmark/src/benchmark_api_internal.h
-third-party/benchmark/src/benchmark_register.h
-third-party/benchmark/src/benchmark_runner.h
-third-party/benchmark/src/check.h
-third-party/benchmark/src/colorprint.h
-third-party/benchmark/src/commandlineflags.h
-third-party/benchmark/src/complexity.h
-third-party/benchmark/src/counter.h
-third-party/benchmark/src/cycleclock.h
-third-party/benchmark/src/internal_macros.h
-third-party/benchmark/src/log.h
-third-party/benchmark/src/mutex.h
-third-party/benchmark/src/perf_counters.h
-third-party/benchmark/src/re.h
-third-party/benchmark/src/sleep.h
-third-party/benchmark/src/statistics.h
-third-party/benchmark/src/string_util.h
-third-party/benchmark/src/thread_manager.h
-third-party/benchmark/src/thread_timer.h
-third-party/benchmark/src/timers.h
-utils/bazel/llvm-project-overlay/clang/include/clang/Config/config.h
-utils/bazel/llvm-project-overlay/llvm/include/llvm/Config/config.h
-- 
GitLab


From 68daf7d27ecc085fe7347552736197db6453f71c Mon Sep 17 00:00:00 2001
From: c8ef <c8ef@outlook.com>
Date: Wed, 30 Oct 2024 22:04:14 +0800
Subject: [PATCH 584/693] [Tooling/Inclusion] Add binary search related
 `std::ranges` symbols to the mapping. (#113796)

Fixes #94459.

This patch adds binary search related `std::ranges` symbols to the mapping.
---
 clang/lib/Tooling/Inclusions/Stdlib/StdSpecialSymbolMap.inc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/clang/lib/Tooling/Inclusions/Stdlib/StdSpecialSymbolMap.inc b/clang/lib/Tooling/Inclusions/Stdlib/StdSpecialSymbolMap.inc
index 0d351d688a32..4d466013eeac 100644
--- a/clang/lib/Tooling/Inclusions/Stdlib/StdSpecialSymbolMap.inc
+++ b/clang/lib/Tooling/Inclusions/Stdlib/StdSpecialSymbolMap.inc
@@ -367,6 +367,11 @@ SYMBOL(any_cast, std::, <any>)
 SYMBOL(div, std::, <cstdlib>)
 SYMBOL(abort, std::, <cstdlib>)
 
+SYMBOL(binary_search, std::ranges::, <algorithm>)
+SYMBOL(equal_range, std::ranges::, <algorithm>)
+SYMBOL(lower_bound, std::ranges::, <algorithm>)
+SYMBOL(upper_bound, std::ranges::, <algorithm>)
+
 // These are C symbols that are not under std namespace.
 SYMBOL(localtime_r, None, <ctime>)
 SYMBOL(localtime_r, None, <time.h>)
-- 
GitLab


From 6af275b72ecd35e3918744b0ef4a750912ce3de5 Mon Sep 17 00:00:00 2001
From: Asher Mancinelli <ashermancinelli@gmail.com>
Date: Wed, 30 Oct 2024 07:07:49 -0700
Subject: [PATCH 585/693] [mlir][doc] Fix nitpicks in documentation (#114157)

A couple of these are probably up to preference, but the
grammar/capitalization changes are probably more critical for
readability.
---
 .../Rationale/SideEffectsAndSpeculation.md     | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/mlir/docs/Rationale/SideEffectsAndSpeculation.md b/mlir/docs/Rationale/SideEffectsAndSpeculation.md
index 8b08b757531b..4d9021a356df 100644
--- a/mlir/docs/Rationale/SideEffectsAndSpeculation.md
+++ b/mlir/docs/Rationale/SideEffectsAndSpeculation.md
@@ -79,9 +79,9 @@ When adding a new op, ask:
 
 1. Does it read from or write to the heap or stack? It should probably implement
    `MemoryEffectsOpInterface`.
-1. Does these side effects ordered? It should probably set the stage of
-   side effects to make analysis more accurate.
-1. Does These side effects act on every single value of resource? It probably
+1. Are these side effects ordered? The op should probably set the stage of
+   side effects to make analyses more accurate.
+1. Do these side effects act on every single value of a resource? It probably
    should set the FullEffect on effect.
 1. Does it have side effects that must be preserved, like a volatile store or a
    syscall? It should probably implement `MemoryEffectsOpInterface` and model
@@ -106,9 +106,9 @@ add side effect correctly.
 
 ### SIMD compute operation
 
-If we have a SIMD backend dialect with a "simd.abs" operation, which reads all
+Consider a SIMD backend dialect with a "simd.abs" operation which reads all
 values from the source memref, calculates their absolute values, and writes them
-to the target memref.
+to the target memref:
 
 ```mlir
   func.func @abs(%source : memref<10xf32>, %target : memref<10xf32>) {
@@ -139,10 +139,10 @@ A typical approach is as follows:
   }
 ```
 
-In the above example, we attach the side effect [MemReadAt<0, FullEffect>] to
+In the above example, we attach the side effect `[MemReadAt<0, FullEffect>]` to
 the source, indicating that the abs operation reads each individual value from
 the source during stage 0. Likewise, we attach the side effect
-[MemWriteAt<1, FullEffect>] to the target, indicating that the abs operation
+`[MemWriteAt<1, FullEffect>]` to the target, indicating that the abs operation
 writes to each individual value within the target during stage 1 (after reading
 from the source).
 
@@ -174,7 +174,7 @@ A typical approach is as follows:
   }
 ```
 
-In the above example, we attach the side effect [MemReadAt<0, PartialEffect>] to
+In the above example, we attach the side effect `[MemReadAt<0, PartialEffect>]` to
 the source, indicating that the load operation reads parts of values from the
 memref during stage 0. Since side effects typically occur at stage 0 and are
-partial by default, we can abbreviate it as "[MemRead]".
+partial by default, we can abbreviate it as `[MemRead]`.
-- 
GitLab


From d693e7c7f7bd50abaa625603a7f9e452b92b2adc Mon Sep 17 00:00:00 2001
From: Dmitry Chernenkov <dmitryc@google.com>
Date: Wed, 30 Oct 2024 14:10:45 +0000
Subject: [PATCH 586/693] [Bazel] fix clang for
 508263824f4ef0c70f37523810e5f7d56bcfa653

---
 .../bazel/llvm-project-overlay/clang/BUILD.bazel  | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index db928deb1706..5e756e2ed962 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -364,6 +364,20 @@ gentbl(
     ],
 )
 
+gentbl(
+    name="basic_builtins_x86_gen",
+    tbl_outs = [(
+        "-gen-clang-builtins",
+        "include/clang/Basic/BuiltinsX86.inc",
+    )],
+    tblgen = ":clang-tblgen",
+    td_file = "include/clang/Basic/BuiltinsX86.td",
+    td_srcs = [
+        "include/clang/Basic/BuiltinsX86.td",
+        "include/clang/Basic/BuiltinsBase.td",
+    ],
+)
+
 gentbl(
     name = "basic_builtins_gen",
     tbl_outs = [(
@@ -701,6 +715,7 @@ cc_library(
         ":basic_builtins_bpf_gen",
         ":basic_builtins_gen",
         ":basic_builtins_riscv_gen",
+        ":basic_builtins_x86_gen",
         ":basic_internal_headers",
         ":basic_riscv_sifive_vector_builtins_gen",
         ":basic_riscv_vector_builtin_cg_gen",
-- 
GitLab


From 4ba623f24479879fb7100988f6ad5d9a62c19842 Mon Sep 17 00:00:00 2001
From: Balazs Benics <benicsbalazs@gmail.com>
Date: Wed, 30 Oct 2024 15:19:37 +0100
Subject: [PATCH 587/693] [clang][Index][USR][NFC] Allow customizing langopts
 for USR generation (#109574)

This helps to produce USRs for custom LangOpts - that differ from the
one associated with the given Decl. This can unlock usecases in tooling
opportunities that we have downstream.

This is NFC because existing calls will still result in the right
overload, thus using the LangOpts associated with the ASTContext of the
Decls and Types.
---
 clang/include/clang/Index/USRGeneration.h |  8 +++-
 clang/lib/Index/USRGeneration.cpp         | 48 ++++++++++++++---------
 2 files changed, 36 insertions(+), 20 deletions(-)

diff --git a/clang/include/clang/Index/USRGeneration.h b/clang/include/clang/Index/USRGeneration.h
index f89fc5cf4930..61d267f3545a 100644
--- a/clang/include/clang/Index/USRGeneration.h
+++ b/clang/include/clang/Index/USRGeneration.h
@@ -15,6 +15,7 @@
 namespace clang {
 class ASTContext;
 class Decl;
+class LangOptions;
 class MacroDefinitionRecord;
 class Module;
 class SourceLocation;
@@ -30,6 +31,8 @@ static inline StringRef getUSRSpacePrefix() {
 /// Generate a USR for a Decl, including the USR prefix.
 /// \returns true if the results should be ignored, false otherwise.
 bool generateUSRForDecl(const Decl *D, SmallVectorImpl<char> &Buf);
+bool generateUSRForDecl(const Decl *D, SmallVectorImpl<char> &Buf,
+                        const LangOptions &LangOpts);
 
 /// Generate a USR fragment for an Objective-C class.
 void generateUSRForObjCClass(StringRef Cls, raw_ostream &OS,
@@ -75,7 +78,10 @@ bool generateUSRForMacro(StringRef MacroName, SourceLocation Loc,
 /// Generates a USR for a type.
 ///
 /// \return true on error, false on success.
-bool generateUSRForType(QualType T, ASTContext &Ctx, SmallVectorImpl<char> &Buf);
+bool generateUSRForType(QualType T, ASTContext &Ctx,
+                        SmallVectorImpl<char> &Buf);
+bool generateUSRForType(QualType T, ASTContext &Ctx, SmallVectorImpl<char> &Buf,
+                        const LangOptions &LangOpts);
 
 /// Generate a USR for a module, including the USR prefix.
 /// \returns true on error, false on success.
diff --git a/clang/lib/Index/USRGeneration.cpp b/clang/lib/Index/USRGeneration.cpp
index 35d0aefaf69a..493123459a5a 100644
--- a/clang/lib/Index/USRGeneration.cpp
+++ b/clang/lib/Index/USRGeneration.cpp
@@ -62,20 +62,17 @@ namespace {
 class USRGenerator : public ConstDeclVisitor<USRGenerator> {
   SmallVectorImpl<char> &Buf;
   llvm::raw_svector_ostream Out;
-  bool IgnoreResults;
   ASTContext *Context;
-  bool generatedLoc;
+  const LangOptions &LangOpts;
+  bool IgnoreResults = false;
+  bool generatedLoc = false;
 
   llvm::DenseMap<const Type *, unsigned> TypeSubstitutions;
 
 public:
-  explicit USRGenerator(ASTContext *Ctx, SmallVectorImpl<char> &Buf)
-  : Buf(Buf),
-    Out(Buf),
-    IgnoreResults(false),
-    Context(Ctx),
-    generatedLoc(false)
-  {
+  USRGenerator(ASTContext *Ctx, SmallVectorImpl<char> &Buf,
+               const LangOptions &LangOpts)
+      : Buf(Buf), Out(Buf), Context(Ctx), LangOpts(LangOpts) {
     // Add the USR space prefix.
     Out << getUSRSpacePrefix();
   }
@@ -246,14 +243,13 @@ void USRGenerator::VisitFunctionDecl(const FunctionDecl *D) {
   } else
     Out << "@F@";
 
-  PrintingPolicy Policy(Context->getLangOpts());
+  PrintingPolicy Policy(LangOpts);
   // Forward references can have different template argument names. Suppress the
   // template argument names in constructors to make their USR more stable.
   Policy.SuppressTemplateArgsInCXXConstructors = true;
   D->getDeclName().print(Out, Policy);
 
-  ASTContext &Ctx = *Context;
-  if ((!Ctx.getLangOpts().CPlusPlus || D->isExternC()) &&
+  if ((!LangOpts.CPlusPlus || D->isExternC()) &&
       !D->hasAttr<OverloadableAttr>())
     return;
 
@@ -657,9 +653,10 @@ bool USRGenerator::GenLoc(const Decl *D, bool IncludeOffset) {
   return IgnoreResults;
 }
 
-static void printQualifier(llvm::raw_ostream &Out, ASTContext &Ctx, NestedNameSpecifier *NNS) {
+static void printQualifier(llvm::raw_ostream &Out, const LangOptions &LangOpts,
+                           NestedNameSpecifier *NNS) {
   // FIXME: Encode the qualifier, don't just print it.
-  PrintingPolicy PO(Ctx.getLangOpts());
+  PrintingPolicy PO(LangOpts);
   PO.SuppressTagKeyword = true;
   PO.SuppressUnwrittenScope = true;
   PO.ConstantArraySizeAsWritten = false;
@@ -948,7 +945,7 @@ void USRGenerator::VisitType(QualType T) {
     }
     if (const DependentNameType *DNT = T->getAs<DependentNameType>()) {
       Out << '^';
-      printQualifier(Out, Ctx, DNT->getQualifier());
+      printQualifier(Out, LangOpts, DNT->getQualifier());
       Out << ':' << DNT->getIdentifier()->getName();
       return;
     }
@@ -1090,7 +1087,7 @@ void USRGenerator::VisitUnresolvedUsingValueDecl(const UnresolvedUsingValueDecl
     return;
   VisitDeclContext(D->getDeclContext());
   Out << "@UUV@";
-  printQualifier(Out, D->getASTContext(), D->getQualifier());
+  printQualifier(Out, LangOpts, D->getQualifier());
   EmitDeclName(D);
 }
 
@@ -1099,7 +1096,7 @@ void USRGenerator::VisitUnresolvedUsingTypenameDecl(const UnresolvedUsingTypenam
     return;
   VisitDeclContext(D->getDeclContext());
   Out << "@UUT@";
-  printQualifier(Out, D->getASTContext(), D->getQualifier());
+  printQualifier(Out, LangOpts, D->getQualifier());
   Out << D->getName(); // Simple name.
 }
 
@@ -1190,6 +1187,13 @@ bool clang::index::generateUSRForDecl(const Decl *D,
                                       SmallVectorImpl<char> &Buf) {
   if (!D)
     return true;
+  return generateUSRForDecl(D, Buf, D->getASTContext().getLangOpts());
+}
+
+bool clang::index::generateUSRForDecl(const Decl *D, SmallVectorImpl<char> &Buf,
+                                      const LangOptions &LangOpts) {
+  if (!D)
+    return true;
   // We don't ignore decls with invalid source locations. Implicit decls, like
   // C++'s operator new function, can have invalid locations but it is fine to
   // create USRs that can identify them.
@@ -1203,7 +1207,7 @@ bool clang::index::generateUSRForDecl(const Decl *D,
       return false;
     }
   }
-  USRGenerator UG(&D->getASTContext(), Buf);
+  USRGenerator UG(&D->getASTContext(), Buf, LangOpts);
   UG.Visit(D);
   return UG.ignoreResults();
 }
@@ -1240,11 +1244,17 @@ bool clang::index::generateUSRForMacro(StringRef MacroName, SourceLocation Loc,
 
 bool clang::index::generateUSRForType(QualType T, ASTContext &Ctx,
                                       SmallVectorImpl<char> &Buf) {
+  return generateUSRForType(T, Ctx, Buf, Ctx.getLangOpts());
+}
+
+bool clang::index::generateUSRForType(QualType T, ASTContext &Ctx,
+                                      SmallVectorImpl<char> &Buf,
+                                      const LangOptions &LangOpts) {
   if (T.isNull())
     return true;
   T = T.getCanonicalType();
 
-  USRGenerator UG(&Ctx, Buf);
+  USRGenerator UG(&Ctx, Buf, LangOpts);
   UG.VisitType(T);
   return UG.ignoreResults();
 }
-- 
GitLab


From 95c2d798148f12565dd4c9ddc753d196e47f230f Mon Sep 17 00:00:00 2001
From: Simon Camphausen <simon.camphausen@iml.fraunhofer.de>
Date: Wed, 30 Oct 2024 15:27:23 +0100
Subject: [PATCH 588/693] [mlir][EmitC] memref-to-emitc: insert
 conversion_casts (#114204)

Add materializations to the conversion pass, such that types of
non-converted operands are legalized.
---
 .../MemRefToEmitC/MemRefToEmitCPass.cpp       | 13 +++++++
 .../MemRefToEmitC/memref-to-emitc.mlir        | 35 +++++++++++--------
 2 files changed, 34 insertions(+), 14 deletions(-)

diff --git a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp
index 11bfde890bce..7f433254e95a 100644
--- a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp
+++ b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp
@@ -40,6 +40,19 @@ struct ConvertMemRefToEmitCPass
 
     populateMemRefToEmitCTypeConversion(converter);
 
+    auto materializeAsUnrealizedCast = [](OpBuilder &builder, Type resultType,
+                                          ValueRange inputs,
+                                          Location loc) -> Value {
+      if (inputs.size() != 1)
+        return Value();
+
+      return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+          .getResult(0);
+    };
+
+    converter.addSourceMaterialization(materializeAsUnrealizedCast);
+    converter.addTargetMaterialization(materializeAsUnrealizedCast);
+
     RewritePatternSet patterns(&getContext());
     populateMemRefToEmitCConversionPatterns(patterns, converter);
 
diff --git a/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir
index f4722da08cc4..f5ef821cc9c0 100644
--- a/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir
+++ b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir
@@ -1,28 +1,35 @@
 // RUN: mlir-opt -convert-memref-to-emitc %s -split-input-file | FileCheck %s
 
-// CHECK-LABEL: memref_store
-// CHECK-SAME:  %[[v:.*]]: f32, %[[i:.*]]: index, %[[j:.*]]: index
-func.func @memref_store(%v : f32, %i: index, %j: index) {
-  // CHECK-NEXT: %[[ALLOCA:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> !emitc.array<4x8xf32>
-  %0 = memref.alloca() : memref<4x8xf32>
+// CHECK-LABEL: alloca()
+func.func @alloca() {
+  // CHECK-NEXT: %[[ALLOCA:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> !emitc.array<2xf32>
+  %0 = memref.alloca() : memref<2xf32>
+  return
+}
 
-  // CHECK-NEXT: %[[SUBSCRIPT:.*]] = emitc.subscript %[[ALLOCA]][%[[i]], %[[j]]] : (!emitc.array<4x8xf32>, index, index) -> !emitc.lvalue<f32>
+// -----
+
+// CHECK-LABEL: memref_store
+// CHECK-SAME:  %[[buff:.*]]: memref<4x8xf32>, %[[v:.*]]: f32, %[[i:.*]]: index, %[[j:.*]]: index
+func.func @memref_store(%buff : memref<4x8xf32>, %v : f32, %i: index, %j: index) {
+  // CHECK-NEXT: %[[BUFFER:.*]] = builtin.unrealized_conversion_cast %[[buff]] : memref<4x8xf32> to !emitc.array<4x8xf32>
+  
+  // CHECK-NEXT: %[[SUBSCRIPT:.*]] = emitc.subscript %[[BUFFER]][%[[i]], %[[j]]] : (!emitc.array<4x8xf32>, index, index) -> !emitc.lvalue<f32>
   // CHECK-NEXT: emitc.assign %[[v]] : f32 to %[[SUBSCRIPT]] : <f32>
-  memref.store %v, %0[%i, %j] : memref<4x8xf32>
+  memref.store %v, %buff[%i, %j] : memref<4x8xf32>
   return
 }
 
 // -----
 
 // CHECK-LABEL: memref_load
-// CHECK-SAME:  %[[i:.*]]: index, %[[j:.*]]: index
-func.func @memref_load(%i: index, %j: index) -> f32 {
-  // CHECK-NEXT: %[[ALLOCA:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> !emitc.array<4x8xf32>
-  %0 = memref.alloca() : memref<4x8xf32>
-
-  // CHECK-NEXT: %[[SUBSCRIPT:.*]] = emitc.subscript %[[ALLOCA]][%[[i]], %[[j]]] : (!emitc.array<4x8xf32>, index, index) -> !emitc.lvalue<f32>
+// CHECK-SAME:  %[[buff:.*]]: memref<4x8xf32>, %[[i:.*]]: index, %[[j:.*]]: index
+func.func @memref_load(%buff : memref<4x8xf32>, %i: index, %j: index) -> f32 {
+  // CHECK-NEXT: %[[BUFFER:.*]] = builtin.unrealized_conversion_cast %[[buff]] : memref<4x8xf32> to !emitc.array<4x8xf32>
+  
+  // CHECK-NEXT: %[[SUBSCRIPT:.*]] = emitc.subscript %[[BUFFER]][%[[i]], %[[j]]] : (!emitc.array<4x8xf32>, index, index) -> !emitc.lvalue<f32>
   // CHECK-NEXT: %[[LOAD:.*]] = emitc.load %[[SUBSCRIPT]] : <f32>
-  %1 = memref.load %0[%i, %j] : memref<4x8xf32>
+  %1 = memref.load %buff[%i, %j] : memref<4x8xf32>
   // CHECK-NEXT: return %[[LOAD]] : f32
   return %1 : f32
 }
-- 
GitLab


From dda20ea73d958584e6b162b34dd421582c52ddbb Mon Sep 17 00:00:00 2001
From: David Truby <david.truby@arm.com>
Date: Wed, 30 Oct 2024 15:05:18 +0000
Subject: [PATCH 589/693] [flang] Add fir-lsp-server (#114059)

This patch adds a fir-lsp-server tool for editor support for editing fir
files, using the existing MLIR lsp server support.

See https://mlir.llvm.org/docs/Tools/MLIRLSP/ for more information.
---
 flang/tools/CMakeLists.txt                    |  1 +
 flang/tools/fir-lsp-server/CMakeLists.txt     | 17 +++++++++++++++++
 flang/tools/fir-lsp-server/fir-lsp-server.cpp |  9 +++++++++
 3 files changed, 27 insertions(+)
 create mode 100644 flang/tools/fir-lsp-server/CMakeLists.txt
 create mode 100644 flang/tools/fir-lsp-server/fir-lsp-server.cpp

diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 337545ae0d4d..1d2d2c608faf 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -12,3 +12,4 @@ add_subdirectory(flang-driver)
 add_subdirectory(tco)
 add_subdirectory(f18-parse-demo)
 add_subdirectory(fir-opt)
+add_subdirectory(fir-lsp-server)
diff --git a/flang/tools/fir-lsp-server/CMakeLists.txt b/flang/tools/fir-lsp-server/CMakeLists.txt
new file mode 100644
index 000000000000..ff0ced6693b9
--- /dev/null
+++ b/flang/tools/fir-lsp-server/CMakeLists.txt
@@ -0,0 +1,17 @@
+set(LLVM_LINK_COMPONENTS
+  Core
+  Support
+  AsmParser
+  )
+
+add_flang_tool(fir-lsp-server fir-lsp-server.cpp)
+
+get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
+get_property(extension_libs GLOBAL PROPERTY MLIR_EXTENSION_LIBS)
+target_link_libraries(fir-lsp-server PRIVATE
+  CUFDialect
+  FIRDialect
+  HLFIRDialect
+  MLIRLspServerLib
+  ${dialect_libs}
+  ${extension_libs})
diff --git a/flang/tools/fir-lsp-server/fir-lsp-server.cpp b/flang/tools/fir-lsp-server/fir-lsp-server.cpp
new file mode 100644
index 000000000000..8b724e292b5a
--- /dev/null
+++ b/flang/tools/fir-lsp-server/fir-lsp-server.cpp
@@ -0,0 +1,9 @@
+#include "mlir/Tools/mlir-lsp-server/MlirLspServerMain.h"
+#include "flang/Optimizer/Support/InitFIR.h"
+
+int main(int argc, char **argv) {
+  mlir::DialectRegistry registry;
+  fir::support::registerNonCodegenDialects(registry);
+  fir::support::addFIRExtensions(registry);
+  return mlir::failed(mlir::MlirLspServerMain(argc, argv, registry));
+}
-- 
GitLab


From 4015e18d6713cdceb0640e77b2d5aa3b256d5ddb Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Wed, 30 Oct 2024 15:14:14 +0000
Subject: [PATCH 590/693] [AArch64] Add assembly/disassembly for BFMOP4{A,S}
 (non-widening) instructions (#113342)

The new instructions are described in
https://developer.arm.com/documentation/ddi0602/2024-09/SME-Instructions
---
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |   5 +
 llvm/lib/Target/AArch64/SMEInstrFormats.td    |  37 +++
 .../bfmop4as-non-widening-diagnostics.s       | 220 ++++++++++++++++++
 .../MC/AArch64/SME2p2/bfmop4as-non-widening.s | 178 ++++++++++++++
 4 files changed, 440 insertions(+)
 create mode 100644 llvm/test/MC/AArch64/SME2p2/bfmop4as-non-widening-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SME2p2/bfmop4as-non-widening.s

diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index b71652942889..e78cd7146df2 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -1047,3 +1047,8 @@ let Predicates = [HasSME2p2, HasSMEF8F32] in {
   defm FMOP4A : sme2_fmop4a_fp8_fp32_4way<"fmop4a">;
 }
 }
+
+let Predicates = [HasSME2p2, HasSMEB16B16] in {
+  defm BFMOP4A : sme2_bfmop4as_non_widening<0, "bfmop4a">;
+  defm BFMOP4S : sme2_bfmop4as_non_widening<1, "bfmop4s">;
+}
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index e7c90b0ed14e..b31bea712a76 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -5417,3 +5417,40 @@ multiclass sme2_fmop4a_fp8_fp32_4way<string mnemonic> {
   // Multiple vectors
   def _M2Z2Z_BtoS : sme2_fp8_fp32_quarter_tile_outer_product<1, 1, mnemonic, ZZ_b_mul_r_Lo, ZZ_b_mul_r_Hi>;
 }
+
+class sme2_bf16_fp16_quarter_tile_outer_product<bit M, bit N, bit S, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>
+    : I<(outs TileOp16:$ZAda),
+        (ins TileOp16:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),
+        mnemonic, "\t$ZAda, $Zn, $Zm",
+        "", []>, Sched<[]> {
+  bit ZAda;
+  bits<3> Zn;
+  bits<3> Zm;
+
+  let Inst{31-21} = 0b10000001001;
+  let Inst{20} = M;
+  let Inst{19-17} = Zm;
+  let Inst{16-10} = 0b0000000;
+  let Inst{9} = N;
+  let Inst{8-6} = Zn;
+  let Inst{5} = 0;
+  let Inst{4} = S;
+  let Inst{3-1} = 0b100;
+  let Inst{0} = ZAda;
+
+  let Constraints = "$ZAda = $_ZAda";
+}
+
+multiclass sme2_bfmop4as_non_widening<bit S, string mnemonic> {
+  // Single vectors
+  def _MZZ_H : sme2_bf16_fp16_quarter_tile_outer_product<0, 0, S, mnemonic, ZPR16Mul2_Lo, ZPR16Mul2_Hi>;
+
+  // Multiple and single vectors
+  def _M2ZZ_H : sme2_bf16_fp16_quarter_tile_outer_product<0, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZPR16Mul2_Hi>;
+
+  // Single and multiple vectors
+  def _MZ2Z_H : sme2_bf16_fp16_quarter_tile_outer_product<1, 0, S, mnemonic, ZPR16Mul2_Lo, ZZ_h_mul_r_Hi>;
+
+  // Multiple vectors
+  def _M2Z2Z_H : sme2_bf16_fp16_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi>;
+}
diff --git a/llvm/test/MC/AArch64/SME2p2/bfmop4as-non-widening-diagnostics.s b/llvm/test/MC/AArch64/SME2p2/bfmop4as-non-widening-diagnostics.s
new file mode 100644
index 000000000000..231d4cd9967a
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/bfmop4as-non-widening-diagnostics.s
@@ -0,0 +1,220 @@
+// RUN: not llvm-mc -triple=aarch64 -mattr=+sme2p2,+sme-b16b16 < %s 2>&1 | FileCheck %s
+
+// BFMOP4A
+
+// Single vectors
+
+bfmop4a za0.d, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4a za4.h, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4a za0.h, z0.s, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4a za0.h, z15.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4a za0.h, z16.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4a za0.h, z0.h, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4a za0.h, z12.h, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4a za0.h, z12.h, z14.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4a za0.h, z12.h, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Single and multiple vectors
+
+bfmop4a za0.d, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4a za4.h, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4a za0.h, z0.s, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4a za0.h, z1.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4a za0.h, z16.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4a za0.h, z0.h, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4a za0.h, z0.h, {z17.h-z18.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4a za0.h, z0.h, {z12.h-z13.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+// Multiple and single vectors
+
+bfmop4a za0.d, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4a za4.h, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4a za0.h, {z0.s-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: mismatched register size suffix
+
+bfmop4a za0.h, {z1.h-z2.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4a za0.h, {z16.h-z17.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4a za0.h, {z0.h-z1.h}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4a za0.h, {z0.h-z1.h}, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4a za0.h, {z0.h-z1.h}, z12.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Multiple vectors
+
+bfmop4a za0.d, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4a za4.h, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4a za0.h, {z0.s-z1.s}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4a za0.h, {z1.h-z2.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4a za0.h, {z18.h-z19.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4a za0.h, {z0.h-z1.h}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4a za0.h, {z0.h-z1.h}, {z19.h-z20.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4a za0.h, {z0.h-z1.h}, {z10.h-z11.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+
+// BFMOP4S
+
+// Single vectors
+
+bfmop4s za0.d, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4s za4.h, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4s za0.h, z0.s, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4s za0.h, z15.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4s za0.h, z16.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4s za0.h, z0.h, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4s za0.h, z12.h, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4s za0.h, z12.h, z14.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4s za0.h, z12.h, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Single and multiple vectors
+
+bfmop4s za0.d, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4s za4.h, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4s za0.h, z0.s, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4s za0.h, z1.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4s za0.h, z16.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.h..z14.h
+
+bfmop4s za0.h, z0.h, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4s za0.h, z0.h, {z17.h-z18.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4s za0.h, z0.h, {z12.h-z13.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+// Multiple and single vectors
+
+bfmop4s za0.d, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4s za4.h, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4s za0.h, {z0.s-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: mismatched register size suffix
+
+bfmop4s za0.h, {z1.h-z2.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4s za0.h, {z16.h-z17.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4s za0.h, {z0.h-z1.h}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4s za0.h, {z0.h-z1.h}, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+bfmop4s za0.h, {z0.h-z1.h}, z12.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Multiple vectors
+
+bfmop4s za0.d, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand, expected za[0-3].s
+
+bfmop4s za4.h, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4s za0.h, {z0.s-z1.s}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4s za0.h, {z1.h-z2.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4s za0.h, {z18.h-z19.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4s za0.h, {z0.h-z1.h}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+bfmop4s za0.h, {z0.h-z1.h}, {z19.h-z20.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+bfmop4s za0.h, {z0.h-z1.h}, {z10.h-z11.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
diff --git a/llvm/test/MC/AArch64/SME2p2/bfmop4as-non-widening.s b/llvm/test/MC/AArch64/SME2p2/bfmop4as-non-widening.s
new file mode 100644
index 000000000000..b98bb99def05
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/bfmop4as-non-widening.s
@@ -0,0 +1,178 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2,+sme-b16b16 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2,+sme-b16b16 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sme2p2,+sme-b16b16 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2,+sme-b16b16 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2p2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2,+sme-b16b16 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2p2,+sme-b16b16 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+// BFMOP4A
+
+// Single vectors
+
+bfmop4a za0.h, z0.h, z16.h  // 10000001-00100000-00000000-00001000
+// CHECK-INST: bfmop4a za0.h, z0.h, z16.h
+// CHECK-ENCODING: [0x08,0x00,0x20,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81200008 <unknown>
+
+bfmop4a za1.h, z12.h, z24.h  // 10000001-00101000-00000001-10001001
+// CHECK-INST: bfmop4a za1.h, z12.h, z24.h
+// CHECK-ENCODING: [0x89,0x01,0x28,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81280189 <unknown>
+
+bfmop4a za1.h, z14.h, z30.h  // 10000001-00101110-00000001-11001001
+// CHECK-INST: bfmop4a za1.h, z14.h, z30.h
+// CHECK-ENCODING: [0xc9,0x01,0x2e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 812e01c9 <unknown>
+
+// Single and multiple vectors
+
+bfmop4a za0.h, z0.h, {z16.h-z17.h}  // 10000001-00110000-00000000-00001000
+// CHECK-INST: bfmop4a za0.h, z0.h, { z16.h, z17.h }
+// CHECK-ENCODING: [0x08,0x00,0x30,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81300008 <unknown>
+
+bfmop4a za1.h, z12.h, {z24.h-z25.h}  // 10000001-00111000-00000001-10001001
+// CHECK-INST: bfmop4a za1.h, z12.h, { z24.h, z25.h }
+// CHECK-ENCODING: [0x89,0x01,0x38,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81380189 <unknown>
+
+bfmop4a za1.h, z14.h, {z30.h-z31.h}  // 10000001-00111110-00000001-11001001
+// CHECK-INST: bfmop4a za1.h, z14.h, { z30.h, z31.h }
+// CHECK-ENCODING: [0xc9,0x01,0x3e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 813e01c9 <unknown>
+
+// Multiple and single vectors
+
+bfmop4a za0.h, {z0.h-z1.h}, z16.h  // 10000001-00100000-00000010-00001000
+// CHECK-INST: bfmop4a za0.h, { z0.h, z1.h }, z16.h
+// CHECK-ENCODING: [0x08,0x02,0x20,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81200208 <unknown>
+
+bfmop4a za1.h, {z12.h-z13.h}, z24.h  // 10000001-00101000-00000011-10001001
+// CHECK-INST: bfmop4a za1.h, { z12.h, z13.h }, z24.h
+// CHECK-ENCODING: [0x89,0x03,0x28,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81280389 <unknown>
+
+bfmop4a za1.h, {z14.h-z15.h}, z30.h  // 10000001-00101110-00000011-11001001
+// CHECK-INST: bfmop4a za1.h, { z14.h, z15.h }, z30.h
+// CHECK-ENCODING: [0xc9,0x03,0x2e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 812e03c9 <unknown>
+
+// Multiple vectors
+
+bfmop4a za0.h, {z0.h-z1.h}, {z16.h-z17.h}  // 10000001-00110000-00000010-00001000
+// CHECK-INST: bfmop4a za0.h, { z0.h, z1.h }, { z16.h, z17.h }
+// CHECK-ENCODING: [0x08,0x02,0x30,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81300208 <unknown>
+
+bfmop4a za1.h, {z12.h-z13.h}, {z24.h-z25.h}  // 10000001-00111000-00000011-10001001
+// CHECK-INST: bfmop4a za1.h, { z12.h, z13.h }, { z24.h, z25.h }
+// CHECK-ENCODING: [0x89,0x03,0x38,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81380389 <unknown>
+
+bfmop4a za1.h, {z14.h-z15.h}, {z30.h-z31.h}  // 10000001-00111110-00000011-11001001
+// CHECK-INST: bfmop4a za1.h, { z14.h, z15.h }, { z30.h, z31.h }
+// CHECK-ENCODING: [0xc9,0x03,0x3e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 813e03c9 <unknown>
+
+
+// BFMOP4S
+
+// Single vectors
+
+bfmop4s za0.h, z0.h, z16.h  // 10000001-00100000-00000000-00011000
+// CHECK-INST: bfmop4s za0.h, z0.h, z16.h
+// CHECK-ENCODING: [0x18,0x00,0x20,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81200018 <unknown>
+
+bfmop4s za1.h, z12.h, z24.h  // 10000001-00101000-00000001-10011001
+// CHECK-INST: bfmop4s za1.h, z12.h, z24.h
+// CHECK-ENCODING: [0x99,0x01,0x28,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81280199 <unknown>
+
+bfmop4s za1.h, z14.h, z30.h  // 10000001-00101110-00000001-11011001
+// CHECK-INST: bfmop4s za1.h, z14.h, z30.h
+// CHECK-ENCODING: [0xd9,0x01,0x2e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 812e01d9 <unknown>
+
+// Single and multiple vectors
+
+bfmop4s za0.h, z0.h, {z16.h-z17.h}  // 10000001-00110000-00000000-00011000
+// CHECK-INST: bfmop4s za0.h, z0.h, { z16.h, z17.h }
+// CHECK-ENCODING: [0x18,0x00,0x30,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81300018 <unknown>
+
+bfmop4s za1.h, z12.h, {z24.h-z25.h}  // 10000001-00111000-00000001-10011001
+// CHECK-INST: bfmop4s za1.h, z12.h, { z24.h, z25.h }
+// CHECK-ENCODING: [0x99,0x01,0x38,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81380199 <unknown>
+
+bfmop4s za1.h, z14.h, {z30.h-z31.h}  // 10000001-00111110-00000001-11011001
+// CHECK-INST: bfmop4s za1.h, z14.h, { z30.h, z31.h }
+// CHECK-ENCODING: [0xd9,0x01,0x3e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 813e01d9 <unknown>
+
+// Multiple and single vectors
+
+bfmop4s za0.h, {z0.h-z1.h}, z16.h  // 10000001-00100000-00000010-00011000
+// CHECK-INST: bfmop4s za0.h, { z0.h, z1.h }, z16.h
+// CHECK-ENCODING: [0x18,0x02,0x20,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81200218 <unknown>
+
+bfmop4s za1.h, {z12.h-z13.h}, z24.h  // 10000001-00101000-00000011-10011001
+// CHECK-INST: bfmop4s za1.h, { z12.h, z13.h }, z24.h
+// CHECK-ENCODING: [0x99,0x03,0x28,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81280399 <unknown>
+
+bfmop4s za1.h, {z14.h-z15.h}, z30.h  // 10000001-00101110-00000011-11011001
+// CHECK-INST: bfmop4s za1.h, { z14.h, z15.h }, z30.h
+// CHECK-ENCODING: [0xd9,0x03,0x2e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 812e03d9 <unknown>
+
+// Multiple vectors
+
+bfmop4s za0.h, {z0.h-z1.h}, {z16.h-z17.h}  // 10000001-00110000-00000010-00011000
+// CHECK-INST: bfmop4s za0.h, { z0.h, z1.h }, { z16.h, z17.h }
+// CHECK-ENCODING: [0x18,0x02,0x30,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81300218 <unknown>
+
+bfmop4s za1.h, {z12.h-z13.h}, {z24.h-z25.h}  // 10000001-00111000-00000011-10011001
+// CHECK-INST: bfmop4s za1.h, { z12.h, z13.h }, { z24.h, z25.h }
+// CHECK-ENCODING: [0x99,0x03,0x38,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 81380399 <unknown>
+
+bfmop4s za1.h, {z14.h-z15.h}, {z30.h-z31.h}  // 10000001-00111110-00000011-11011001
+// CHECK-INST: bfmop4s za1.h, { z14.h, z15.h }, { z30.h, z31.h }
+// CHECK-ENCODING: [0xd9,0x03,0x3e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2 sme-b16b16
+// CHECK-UNKNOWN: 813e03d9 <unknown>
-- 
GitLab


From f405c683ba929fcd0bcaa435ca2fbe4bb221d04b Mon Sep 17 00:00:00 2001
From: Steven Perron <stevenperron@google.com>
Date: Wed, 30 Oct 2024 11:19:23 -0400
Subject: [PATCH 591/693] [OPT] Search whole BB for convergence token.
 (#112728)

The spec for llvm.experimental.convergence.entry says that is must be in
the entry block for a function, and must preceed any other convergent
operation. It does not have to be the first instruction in the entry
block.

Inlining assumes that the call to llvm.experimental.convergence.entry
will be the first instruction after any phi instructions. This commit
modifies inlining to search the entire block for the call.
---
 llvm/lib/Transforms/Utils/InlineFunction.cpp  | 38 ++++++++++---------
 .../Transforms/Inline/convergence-inline.ll   | 24 ++++++++++++
 2 files changed, 45 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 4ad426285ce2..a27cb4dd219c 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -181,9 +181,21 @@ namespace {
       }
     }
   };
-
 } // end anonymous namespace
 
+static IntrinsicInst *getConvergenceEntry(BasicBlock &BB) {
+  auto *I = BB.getFirstNonPHI();
+  while (I) {
+    if (auto *IntrinsicCall = dyn_cast<ConvergenceControlInst>(I)) {
+      if (IntrinsicCall->isEntry()) {
+        return IntrinsicCall;
+      }
+    }
+    I = I->getNextNode();
+  }
+  return nullptr;
+}
+
 /// Get or create a target for the branch from ResumeInsts.
 BasicBlock *LandingPadInliningInfo::getInnerResumeDest() {
   if (InnerResumeDest) return InnerResumeDest;
@@ -2496,15 +2508,10 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
   // fully implements convergence control tokens, there is no mixing of
   // controlled and uncontrolled convergent operations in the whole program.
   if (CB.isConvergent()) {
-    auto *I = CalledFunc->getEntryBlock().getFirstNonPHI();
-    if (auto *IntrinsicCall = dyn_cast<IntrinsicInst>(I)) {
-      if (IntrinsicCall->getIntrinsicID() ==
-          Intrinsic::experimental_convergence_entry) {
-        if (!ConvergenceControlToken) {
-          return InlineResult::failure(
-              "convergent call needs convergencectrl operand");
-        }
-      }
+    if (!ConvergenceControlToken &&
+        getConvergenceEntry(CalledFunc->getEntryBlock())) {
+      return InlineResult::failure(
+          "convergent call needs convergencectrl operand");
     }
   }
 
@@ -2795,13 +2802,10 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
   }
 
   if (ConvergenceControlToken) {
-    auto *I = FirstNewBlock->getFirstNonPHI();
-    if (auto *IntrinsicCall = dyn_cast<IntrinsicInst>(I)) {
-      if (IntrinsicCall->getIntrinsicID() ==
-          Intrinsic::experimental_convergence_entry) {
-        IntrinsicCall->replaceAllUsesWith(ConvergenceControlToken);
-        IntrinsicCall->eraseFromParent();
-      }
+    IntrinsicInst *IntrinsicCall = getConvergenceEntry(*FirstNewBlock);
+    if (IntrinsicCall) {
+      IntrinsicCall->replaceAllUsesWith(ConvergenceControlToken);
+      IntrinsicCall->eraseFromParent();
     }
   }
 
diff --git a/llvm/test/Transforms/Inline/convergence-inline.ll b/llvm/test/Transforms/Inline/convergence-inline.ll
index 8c67e6a59b7d..4996a2376be6 100644
--- a/llvm/test/Transforms/Inline/convergence-inline.ll
+++ b/llvm/test/Transforms/Inline/convergence-inline.ll
@@ -185,6 +185,30 @@ define void @test_two_calls() convergent {
   ret void
 }
 
+define i32 @token_not_first(i32 %x) convergent alwaysinline {
+; CHECK-LABEL: @token_not_first(
+; CHECK-NEXT:    {{%.*}} = alloca ptr, align 8
+; CHECK-NEXT:    [[TOKEN:%.*]] = call token @llvm.experimental.convergence.entry()
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @g(i32 [[X:%.*]]) [ "convergencectrl"(token [[TOKEN]]) ]
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %p = alloca ptr, align 8
+  %token = call token @llvm.experimental.convergence.entry()
+  %y = call i32 @g(i32 %x) [ "convergencectrl"(token %token) ]
+  ret i32 %y
+}
+
+define void @test_token_not_first() convergent {
+; CHECK-LABEL: @test_token_not_first(
+; CHECK-NEXT:    [[TOKEN:%.*]] = call token @llvm.experimental.convergence.entry()
+; CHECK-NEXT:    {{%.*}} = call i32 @g(i32 23) [ "convergencectrl"(token [[TOKEN]]) ]
+; CHECK-NEXT:    ret void
+;
+  %token = call token @llvm.experimental.convergence.entry()
+  %x = call i32 @token_not_first(i32 23) [ "convergencectrl"(token %token) ]
+  ret void
+}
+
 declare void @f(i32) convergent
 declare i32 @g(i32) convergent
 
-- 
GitLab


From e989e31a47375a7d556269eead538dc65edcef2b Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Wed, 30 Oct 2024 15:21:18 +0000
Subject: [PATCH 592/693] [RISCV] Mark f16/bf16 lrint and llrint cost as
 invalid (#113924)

We currently can't lower scalable vector lrint and llrint nodes for bf16
and f16, even with zvfh, and will crash.

Mark the cost as invalid for now to prevent the vectorizers from
emitting them.

Note that we can actually lower fixed-length vectors fine by scalarizing
them, but we were still undercosting these too so I've also included
them. I presume there's an opportunity to improve the codegen later on.
---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp |  9 ++-
 llvm/test/Analysis/CostModel/RISCV/fround.ll  | 72 +++++++++----------
 2 files changed, 43 insertions(+), 38 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 395baa5f1aab..988cb194cd60 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -948,12 +948,17 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                     TTI::TargetCostKind CostKind) {
   auto *RetTy = ICA.getReturnType();
   switch (ICA.getID()) {
+  case Intrinsic::lrint:
+  case Intrinsic::llrint:
+    // We can't currently lower half or bfloat vector lrint/llrint.
+    if (auto *VecTy = dyn_cast<VectorType>(ICA.getArgTypes()[0]);
+        VecTy && VecTy->getElementType()->is16bitFPTy())
+      return InstructionCost::getInvalid();
+    [[fallthrough]];
   case Intrinsic::ceil:
   case Intrinsic::floor:
   case Intrinsic::trunc:
   case Intrinsic::rint:
-  case Intrinsic::lrint:
-  case Intrinsic::llrint:
   case Intrinsic::round:
   case Intrinsic::roundeven: {
     // These all use the same code.
diff --git a/llvm/test/Analysis/CostModel/RISCV/fround.ll b/llvm/test/Analysis/CostModel/RISCV/fround.ll
index c6826760a45b..b09649835508 100644
--- a/llvm/test/Analysis/CostModel/RISCV/fround.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/fround.ll
@@ -425,15 +425,15 @@ define void @rint_fp16() {
 define void @lrint() {
 ; CHECK-LABEL: 'lrint'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call i64 @llvm.lrint.i64.bf16(bfloat undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x i64> @llvm.lrint.v2i64.v2bf16(<2 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x i64> @llvm.lrint.v4i64.v4bf16(<4 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x i64> @llvm.lrint.v8i64.v8bf16(<8 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x i64> @llvm.lrint.v16i64.v16bf16(<16 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x i64> @llvm.lrint.nxv1i64.nxv1bf16(<vscale x 1 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %2 = call <2 x i64> @llvm.lrint.v2i64.v2bf16(<2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %3 = call <4 x i64> @llvm.lrint.v4i64.v4bf16(<4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %4 = call <8 x i64> @llvm.lrint.v8i64.v8bf16(<8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %5 = call <16 x i64> @llvm.lrint.v16i64.v16bf16(<16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %6 = call <vscale x 1 x i64> @llvm.lrint.nxv1i64.nxv1bf16(<vscale x 1 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %7 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %8 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %9 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %10 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call i64 @llvm.lrint.i64.f32(float undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float> undef)
@@ -490,15 +490,15 @@ define void @lrint() {
 define void @lrint_fp16() {
 ; CHECK-LABEL: 'lrint_fp16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call i64 @llvm.lrint.i64.f16(half undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x i64> @llvm.lrint.v2i64.v2f16(<2 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x i64> @llvm.lrint.v4i64.v4f16(<4 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x i64> @llvm.lrint.v8i64.v8f16(<8 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x i64> @llvm.lrint.v16i64.v16f16(<16 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x i64> @llvm.lrint.nxv1i64.nxv1f16(<vscale x 1 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2f16(<vscale x 2 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4f16(<vscale x 4 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8f16(<vscale x 8 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16f16(<vscale x 16 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %2 = call <2 x i64> @llvm.lrint.v2i64.v2f16(<2 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %3 = call <4 x i64> @llvm.lrint.v4i64.v4f16(<4 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %4 = call <8 x i64> @llvm.lrint.v8i64.v8f16(<8 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %5 = call <16 x i64> @llvm.lrint.v16i64.v16f16(<16 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %6 = call <vscale x 1 x i64> @llvm.lrint.nxv1i64.nxv1f16(<vscale x 1 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %7 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2f16(<vscale x 2 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %8 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4f16(<vscale x 4 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %9 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8f16(<vscale x 8 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %10 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16f16(<vscale x 16 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call i64 @llvm.lrint.f16(half undef)
@@ -517,15 +517,15 @@ define void @lrint_fp16() {
 define void @llrint() {
 ; CHECK-LABEL: 'llrint'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call i64 @llvm.llrint.i64.bf16(bfloat undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x i64> @llvm.llrint.v2i64.v2bf16(<2 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x i64> @llvm.llrint.v4i64.v4bf16(<4 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x i64> @llvm.llrint.v8i64.v8bf16(<8 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x i64> @llvm.llrint.v16i64.v16bf16(<16 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1bf16(<vscale x 1 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %2 = call <2 x i64> @llvm.llrint.v2i64.v2bf16(<2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %3 = call <4 x i64> @llvm.llrint.v4i64.v4bf16(<4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %4 = call <8 x i64> @llvm.llrint.v8i64.v8bf16(<8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %5 = call <16 x i64> @llvm.llrint.v16i64.v16bf16(<16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %6 = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1bf16(<vscale x 1 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %7 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %8 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %9 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %10 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call i64 @llvm.llrint.i64.f32(float undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> undef)
@@ -582,15 +582,15 @@ define void @llrint() {
 define void @llrint_fp16() {
 ; CHECK-LABEL: 'llrint_fp16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call i64 @llvm.llrint.i64.f16(half undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x i64> @llvm.llrint.v2i64.v2f16(<2 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x i64> @llvm.llrint.v4i64.v4f16(<4 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x i64> @llvm.llrint.v8i64.v8f16(<8 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x i64> @llvm.llrint.v16i64.v16f16(<16 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f16(<vscale x 1 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f16(<vscale x 2 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f16(<vscale x 4 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f16(<vscale x 8 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f16(<vscale x 16 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %2 = call <2 x i64> @llvm.llrint.v2i64.v2f16(<2 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %3 = call <4 x i64> @llvm.llrint.v4i64.v4f16(<4 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %4 = call <8 x i64> @llvm.llrint.v8i64.v8f16(<8 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %5 = call <16 x i64> @llvm.llrint.v16i64.v16f16(<16 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %6 = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f16(<vscale x 1 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %7 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f16(<vscale x 2 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %8 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f16(<vscale x 4 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %9 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f16(<vscale x 8 x half> undef)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %10 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f16(<vscale x 16 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call i64 @llvm.llrint.f16(half undef)
-- 
GitLab


From 7d1e283bd3b4440aea9ac375ca51e2ee6b0e86f5 Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Wed, 30 Oct 2024 09:24:10 -0600
Subject: [PATCH 593/693] [Clang][Sema] Ignore previous partial specializations
 of member templates explicitly specialized for an implicitly instantiated
 class template specialization (#113464)

Consider the following:
```
template<typename T>
struct A {
  template<typename U>
  struct B {
    static constexpr int x = 0; // #1
  };

  template<typename U>
  struct B<U*> {
    static constexpr int x = 1; // #2
  };
};

template<>
template<typename U>
struct A<long>::B {
  static constexpr int x = 2; // #3
};

static_assert(A<short>::B<int>::y == 0); // uses #1
static_assert(A<short>::B<int*>::y == 1); // uses #2

static_assert(A<long>::B<int>::y == 2); // uses #3
static_assert(A<long>::B<int*>::y == 2); // uses #3
```

According to [temp.spec.partial.member] p2:
> If the primary member template is explicitly specialized for a given
(implicit) specialization of the enclosing class template, the partial
specializations of the member template are ignored for this
specialization of the enclosing class template.
If a partial specialization of the member template is explicitly
specialized for a given (implicit) specialization of the enclosing class
template, the primary member template and its other partial
specializations are still considered for this specialization of the
enclosing class template.

The example above fails to compile because we currently don't implement
[temp.spec.partial.member] p2. This patch implements the wording, fixing #51051.
---
 clang/docs/ReleaseNotes.rst                   |  2 +
 clang/lib/Sema/SemaTemplate.cpp               | 16 +++-
 clang/lib/Sema/SemaTemplateInstantiate.cpp    | 38 +++++++--
 .../temp.spec.partial.member/p2.cpp           | 85 +++++++++++++++++++
 4 files changed, 133 insertions(+), 8 deletions(-)
 create mode 100644 clang/test/CXX/temp/temp.decls/temp.spec.partial/temp.spec.partial.member/p2.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 6085352dfafe..1a179e63f902 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -574,6 +574,8 @@ Bug Fixes to C++ Support
   (#GH95854).
 - Fixed an assertion failure when evaluating an invalid expression in an array initializer. (#GH112140)
 - Fixed an assertion failure in range calculations for conditional throw expressions. (#GH111854)
+- Clang now correctly ignores previous partial specializations of member templates explicitly specialized for
+  an implicitly instantiated class template specialization. (#GH51051)
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index fcf05798d9c7..4503e60cff8c 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -4381,8 +4381,20 @@ Sema::CheckVarTemplateId(VarTemplateDecl *Template, SourceLocation TemplateLoc,
   SmallVector<VarTemplatePartialSpecializationDecl *, 4> PartialSpecs;
   Template->getPartialSpecializations(PartialSpecs);
 
-  for (unsigned I = 0, N = PartialSpecs.size(); I != N; ++I) {
-    VarTemplatePartialSpecializationDecl *Partial = PartialSpecs[I];
+  for (VarTemplatePartialSpecializationDecl *Partial : PartialSpecs) {
+    // C++ [temp.spec.partial.member]p2:
+    //   If the primary member template is explicitly specialized for a given
+    //   (implicit) specialization of the enclosing class template, the partial
+    //   specializations of the member template are ignored for this
+    //   specialization of the enclosing class template. If a partial
+    //   specialization of the member template is explicitly specialized for a
+    //   given (implicit) specialization of the enclosing class template, the
+    //   primary member template and its other partial specializations are still
+    //   considered for this specialization of the enclosing class template.
+    if (Template->getMostRecentDecl()->isMemberSpecialization() &&
+        !Partial->getMostRecentDecl()->isMemberSpecialization())
+      continue;
+
     TemplateDeductionInfo Info(FailedCandidates.getLocation());
 
     if (TemplateDeductionResult Result =
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index dea97bfce532..b63063813f1b 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -3978,11 +3978,24 @@ bool Sema::usesPartialOrExplicitSpecialization(
     return true;
 
   SmallVector<ClassTemplatePartialSpecializationDecl *, 4> PartialSpecs;
-  ClassTemplateSpec->getSpecializedTemplate()
-                   ->getPartialSpecializations(PartialSpecs);
-  for (unsigned I = 0, N = PartialSpecs.size(); I != N; ++I) {
+  ClassTemplateDecl *CTD = ClassTemplateSpec->getSpecializedTemplate();
+  CTD->getPartialSpecializations(PartialSpecs);
+  for (ClassTemplatePartialSpecializationDecl *CTPSD : PartialSpecs) {
+    // C++ [temp.spec.partial.member]p2:
+    //   If the primary member template is explicitly specialized for a given
+    //   (implicit) specialization of the enclosing class template, the partial
+    //   specializations of the member template are ignored for this
+    //   specialization of the enclosing class template. If a partial
+    //   specialization of the member template is explicitly specialized for a
+    //   given (implicit) specialization of the enclosing class template, the
+    //   primary member template and its other partial specializations are still
+    //   considered for this specialization of the enclosing class template.
+    if (CTD->getMostRecentDecl()->isMemberSpecialization() &&
+        !CTPSD->getMostRecentDecl()->isMemberSpecialization())
+      continue;
+
     TemplateDeductionInfo Info(Loc);
-    if (DeduceTemplateArguments(PartialSpecs[I],
+    if (DeduceTemplateArguments(CTPSD,
                                 ClassTemplateSpec->getTemplateArgs().asArray(),
                                 Info) == TemplateDeductionResult::Success)
       return true;
@@ -4025,8 +4038,21 @@ getPatternForClassTemplateSpecialization(
     SmallVector<ClassTemplatePartialSpecializationDecl *, 4> PartialSpecs;
     Template->getPartialSpecializations(PartialSpecs);
     TemplateSpecCandidateSet FailedCandidates(PointOfInstantiation);
-    for (unsigned I = 0, N = PartialSpecs.size(); I != N; ++I) {
-      ClassTemplatePartialSpecializationDecl *Partial = PartialSpecs[I];
+    for (ClassTemplatePartialSpecializationDecl *Partial : PartialSpecs) {
+      // C++ [temp.spec.partial.member]p2:
+      //   If the primary member template is explicitly specialized for a given
+      //   (implicit) specialization of the enclosing class template, the
+      //   partial specializations of the member template are ignored for this
+      //   specialization of the enclosing class template. If a partial
+      //   specialization of the member template is explicitly specialized for a
+      //   given (implicit) specialization of the enclosing class template, the
+      //   primary member template and its other partial specializations are
+      //   still considered for this specialization of the enclosing class
+      //   template.
+      if (Template->getMostRecentDecl()->isMemberSpecialization() &&
+          !Partial->getMostRecentDecl()->isMemberSpecialization())
+        continue;
+
       TemplateDeductionInfo Info(FailedCandidates.getLocation());
       if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
               Partial, ClassTemplateSpec->getTemplateArgs().asArray(), Info);
diff --git a/clang/test/CXX/temp/temp.decls/temp.spec.partial/temp.spec.partial.member/p2.cpp b/clang/test/CXX/temp/temp.decls/temp.spec.partial/temp.spec.partial.member/p2.cpp
new file mode 100644
index 000000000000..7969b7efe597
--- /dev/null
+++ b/clang/test/CXX/temp/temp.decls/temp.spec.partial/temp.spec.partial.member/p2.cpp
@@ -0,0 +1,85 @@
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+template<typename T>
+struct A {
+  template<typename U>
+  struct B {
+    static constexpr int y = 0;
+  };
+
+  template<typename U>
+  struct B<U*> {
+    static constexpr int y = 1;
+  };
+
+  template<typename U>
+  static constexpr int x = 0;
+
+  template<typename U>
+  static constexpr int x<U*> = 1;
+};
+
+template<typename T>
+template<typename U>
+struct A<T>::B<U[]> {
+  static constexpr int y = 2;
+};
+
+template<typename T>
+template<typename U>
+constexpr int A<T>::x<U[]> = 2;
+
+static_assert(A<short>::B<int>::y == 0);
+static_assert(A<short>::B<int*>::y == 1);
+static_assert(A<short>::B<int[]>::y == 2);
+static_assert(A<short>::x<int> == 0);
+static_assert(A<short>::x<int*> == 1);
+static_assert(A<short>::x<int[]> == 2);
+
+template<>
+template<typename U>
+struct A<int>::B {
+  static constexpr int y = 3;
+};
+
+template<>
+template<typename U>
+struct A<int>::B<U&> {
+  static constexpr int y = 4;
+};
+
+template<>
+template<typename U>
+struct A<long>::B<U&> {
+  static constexpr int y = 5;
+};
+
+template<>
+template<typename U>
+constexpr int A<int>::x = 3;
+
+template<>
+template<typename U>
+constexpr int A<int>::x<U&> = 4;
+
+template<>
+template<typename U>
+constexpr int A<long>::x<U&> = 5;
+
+static_assert(A<int>::B<int>::y == 3);
+static_assert(A<int>::B<int*>::y == 3);
+static_assert(A<int>::B<int[]>::y == 3);
+static_assert(A<int>::B<int&>::y == 4);
+static_assert(A<int>::x<int> == 3);
+static_assert(A<int>::x<int*> == 3);
+static_assert(A<int>::x<int[]> == 3);
+static_assert(A<int>::x<int&> == 4);
+static_assert(A<long>::B<int>::y == 0);
+static_assert(A<long>::B<int*>::y == 1);
+static_assert(A<long>::B<int[]>::y == 2);
+static_assert(A<long>::B<int&>::y == 5);
+static_assert(A<long>::x<int> == 0);
+static_assert(A<long>::x<int*> == 1);
+static_assert(A<long>::x<int[]> == 2);
+static_assert(A<long>::x<int&> == 5);
-- 
GitLab


From 475e736bb5eeea8ec70aca51d1a3d98179c69530 Mon Sep 17 00:00:00 2001
From: Teresa Johnson <tejohnson@google.com>
Date: Wed, 30 Oct 2024 08:28:22 -0700
Subject: [PATCH 594/693] [MemProf] Include <ctime> to avoid MSVC failure
 (#114246)

My change in bb3915149a7c9b1660db9caebfc96343352e8454 added a call to
std::time which worked generally as there must be some transitive
include of <ctime>. However, I saw one MSVC bot failure:

InstrProfWriter.cpp(202): error C2039: 'time': is not a member of 'std'

from https://lab.llvm.org/buildbot/#/builders/63/builds/2325.

Presumably explictly including <ctime> should fix this.
---
 llvm/lib/ProfileData/InstrProfWriter.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index f09241681b92..0ab9f942a085 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -29,6 +29,7 @@
 #include "llvm/Support/OnDiskHashTable.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdint>
+#include <ctime>
 #include <memory>
 #include <string>
 #include <tuple>
-- 
GitLab


From 72b115301d1c0d56f40f5030bb8d16f422ac211b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Wed, 30 Oct 2024 16:34:01 +0100
Subject: [PATCH 595/693] [GlobalISel] Import samesign flag (#113090)

Credits: https://github.com/llvm/llvm-project/pull/111419
---
 .../CodeGen/GlobalISel/GenericMachineInstrs.h |  2 +-
 .../CodeGen/GlobalISel/MachineIRBuilder.h     |  3 +-
 llvm/include/llvm/CodeGen/MachineInstr.h      |  1 +
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |  9 +--
 .../CodeGen/GlobalISel/MachineIRBuilder.cpp   |  5 +-
 llvm/lib/CodeGen/MIRParser/MILexer.cpp        |  1 +
 llvm/lib/CodeGen/MIRParser/MILexer.h          |  1 +
 llvm/lib/CodeGen/MIRParser/MIParser.cpp       |  5 +-
 llvm/lib/CodeGen/MIRPrinter.cpp               |  2 +
 llvm/lib/CodeGen/MachineInstr.cpp             |  7 ++
 .../GlobalISel/irtranslator-samesign.ll       | 69 +++++++++++++++++++
 llvm/test/CodeGen/MIR/icmp-flags.mir          | 50 ++++++++++++++
 12 files changed, 144 insertions(+), 11 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-samesign.ll
 create mode 100644 llvm/test/CodeGen/MIR/icmp-flags.mir

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index b6309a9ea0ec..cd7ebcf54c9e 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -28,7 +28,7 @@ namespace llvm {
 class GenericMachineInstr : public MachineInstr {
   constexpr static unsigned PoisonFlags = NoUWrap | NoSWrap | NoUSWrap |
                                           IsExact | Disjoint | NonNeg |
-                                          FmNoNans | FmNoInfs;
+                                          FmNoNans | FmNoInfs | SameSign;
 
 public:
   GenericMachineInstr() = delete;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index c41e74ec7ebd..a38dd34a1709 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1266,7 +1266,8 @@ public:
   ///
   /// \return a MachineInstrBuilder for the newly created instruction.
   MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res,
-                                const SrcOp &Op0, const SrcOp &Op1);
+                                const SrcOp &Op0, const SrcOp &Op1,
+                                std::optional<unsigned> Flags = std::nullopt);
 
   /// Build and insert a \p Res = G_FCMP \p Pred\p Op0, \p Op1
   ///
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index 360517324746..ead6bbe1d5f6 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -119,6 +119,7 @@ public:
     Disjoint = 1 << 19,      // Each bit is zero in at least one of the inputs.
     NoUSWrap = 1 << 20,      // Instruction supports geps
                              // no unsigned signed wrap.
+    SameSign = 1 << 21       // Both operands have the same sign.
   };
 
 private:
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 5381dce58f9e..a87754389cc8 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -340,20 +340,17 @@ bool IRTranslator::translateCompare(const User &U,
   Register Op1 = getOrCreateVReg(*U.getOperand(1));
   Register Res = getOrCreateVReg(U);
   CmpInst::Predicate Pred = CI->getPredicate();
+  uint32_t Flags = MachineInstr::copyFlagsFromInstruction(*CI);
   if (CmpInst::isIntPredicate(Pred))
-    MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
+    MIRBuilder.buildICmp(Pred, Res, Op0, Op1, Flags);
   else if (Pred == CmpInst::FCMP_FALSE)
     MIRBuilder.buildCopy(
         Res, getOrCreateVReg(*Constant::getNullValue(U.getType())));
   else if (Pred == CmpInst::FCMP_TRUE)
     MIRBuilder.buildCopy(
         Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
-  else {
-    uint32_t Flags = 0;
-    if (CI)
-      Flags = MachineInstr::copyFlagsFromInstruction(*CI);
+  else
     MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
-  }
 
   return true;
 }
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 59f2fc633f5d..15b916424784 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -898,8 +898,9 @@ MachineIRBuilder::buildFPTrunc(const DstOp &Res, const SrcOp &Op,
 MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred,
                                                 const DstOp &Res,
                                                 const SrcOp &Op0,
-                                                const SrcOp &Op1) {
-  return buildInstr(TargetOpcode::G_ICMP, Res, {Pred, Op0, Op1});
+                                                const SrcOp &Op1,
+                                                std::optional<unsigned> Flags) {
+  return buildInstr(TargetOpcode::G_ICMP, Res, {Pred, Op0, Op1}, Flags);
 }
 
 MachineInstrBuilder MachineIRBuilder::buildFCmp(CmpInst::Predicate Pred,
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.cpp b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
index 5a3806ce5733..1c450b05f49e 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
@@ -216,6 +216,7 @@ static MIToken::TokenKind getIdentifierKind(StringRef Identifier) {
       .Case("exact", MIToken::kw_exact)
       .Case("nneg", MIToken::kw_nneg)
       .Case("disjoint", MIToken::kw_disjoint)
+      .Case("samesign", MIToken::kw_samesign)
       .Case("nofpexcept", MIToken::kw_nofpexcept)
       .Case("unpredictable", MIToken::kw_unpredictable)
       .Case("debug-location", MIToken::kw_debug_location)
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.h b/llvm/lib/CodeGen/MIRParser/MILexer.h
index 3931da3eaae1..d7cd06759cfb 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.h
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.h
@@ -77,6 +77,7 @@ struct MIToken {
     kw_unpredictable,
     kw_nneg,
     kw_disjoint,
+    kw_samesign,
     kw_debug_location,
     kw_debug_instr_number,
     kw_dbg_instr_ref,
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index 45847b5830da..059814c70f82 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -1476,7 +1476,8 @@ bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) {
          Token.is(MIToken::kw_noconvergent) ||
          Token.is(MIToken::kw_unpredictable) ||
          Token.is(MIToken::kw_nneg) ||
-         Token.is(MIToken::kw_disjoint)) {
+         Token.is(MIToken::kw_disjoint) ||
+         Token.is(MIToken::kw_samesign)) {
     // clang-format on
     // Mine frame and fast math flags
     if (Token.is(MIToken::kw_frame_setup))
@@ -1513,6 +1514,8 @@ bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) {
       Flags |= MachineInstr::NonNeg;
     if (Token.is(MIToken::kw_disjoint))
       Flags |= MachineInstr::Disjoint;
+    if (Token.is(MIToken::kw_samesign))
+      Flags |= MachineInstr::SameSign;
 
     lex();
   }
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index a015cd3c2a55..658bbe0e577e 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -837,6 +837,8 @@ void MIPrinter::print(const MachineInstr &MI) {
     OS << "disjoint ";
   if (MI.getFlag(MachineInstr::NoUSWrap))
     OS << "nusw ";
+  if (MI.getFlag(MachineInstr::SameSign))
+    OS << "samesign ";
 
   OS << TII->getName(MI.getOpcode());
   if (I < E)
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index c1bd0bb5b716..941861da5c56 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -596,6 +596,11 @@ uint32_t MachineInstr::copyFlagsFromInstruction(const Instruction &I) {
       MIFlags |= MachineInstr::MIFlag::Disjoint;
   }
 
+  // Copy the samesign flag.
+  if (const ICmpInst *ICmp = dyn_cast<ICmpInst>(&I))
+    if (ICmp->hasSameSign())
+      MIFlags |= MachineInstr::MIFlag::SameSign;
+
   // Copy the exact flag.
   if (const PossiblyExactOperator *PE = dyn_cast<PossiblyExactOperator>(&I))
     if (PE->isExact())
@@ -1770,6 +1775,8 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST,
     OS << "nneg ";
   if (getFlag(MachineInstr::Disjoint))
     OS << "disjoint ";
+  if (getFlag(MachineInstr::SameSign))
+    OS << "samesign ";
 
   // Print the opcode name.
   if (TII)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-samesign.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-samesign.ll
new file mode 100644
index 000000000000..0173f92c9822
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-samesign.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -global-isel -mtriple=aarch64-linux-gnu -O0 -stop-after=irtranslator < %s | FileCheck %s
+
+
+define <2 x i1> @call_icmp_samesign_vector(<2 x i32> %a, <2 x i32> %b) {
+  ; CHECK-LABEL: name: call_icmp_samesign_vector
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $d0, $d1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+  ; CHECK-NEXT:   %2:_(<2 x s1>) = samesign G_ICMP intpred(ult), [[COPY]](<2 x s32>), [[COPY1]]
+  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(<2 x s32>) = G_ANYEXT %2(<2 x s1>)
+  ; CHECK-NEXT:   $d0 = COPY [[ANYEXT]](<2 x s32>)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $d0
+entry:
+  %result = icmp samesign ult <2 x i32> %a, %b
+  ret <2 x i1> %result
+}
+
+define <2 x i1> @call_icmp_vector(<2 x i32> %a, <2 x i32> %b) {
+  ; CHECK-LABEL: name: call_icmp_vector
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $d0, $d1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:_(<2 x s1>) = G_ICMP intpred(ult), [[COPY]](<2 x s32>), [[COPY1]]
+  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(<2 x s32>) = G_ANYEXT [[ICMP]](<2 x s1>)
+  ; CHECK-NEXT:   $d0 = COPY [[ANYEXT]](<2 x s32>)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $d0
+entry:
+  %result = icmp ult <2 x i32> %a, %b
+  ret <2 x i1> %result
+}
+
+define i1 @call_icmp(i32 %a) {
+  ; CHECK-LABEL: name: call_icmp
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $w0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY]](s32), [[C]]
+  ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:_(s8) = G_ZEXT [[ICMP]](s1)
+  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ZEXT]](s8)
+  ; CHECK-NEXT:   $w0 = COPY [[ANYEXT]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
+entry:
+  %result = icmp ult i32 %a, 3
+  ret i1 %result
+}
+
+define i1 @call_icmp_samesign(i32 %a) {
+  ; CHECK-LABEL: name: call_icmp_samesign
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $w0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; CHECK-NEXT:   %2:_(s1) = samesign G_ICMP intpred(ult), [[COPY]](s32), [[C]]
+  ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:_(s8) = G_ZEXT %2(s1)
+  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ZEXT]](s8)
+  ; CHECK-NEXT:   $w0 = COPY [[ANYEXT]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
+entry:
+  %result = icmp samesign ult i32 %a, 3
+  ret i1 %result
+}
diff --git a/llvm/test/CodeGen/MIR/icmp-flags.mir b/llvm/test/CodeGen/MIR/icmp-flags.mir
new file mode 100644
index 000000000000..3c03a7aaa9bc
--- /dev/null
+++ b/llvm/test/CodeGen/MIR/icmp-flags.mir
@@ -0,0 +1,50 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -run-pass=none -verify-machineinstrs %s -o - | FileCheck %s
+
+
+---
+name:            icmp_samesign
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+
+    ; CHECK-LABEL: name: icmp_samesign
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %x:_(s32) = COPY $w0
+    ; CHECK-NEXT: %y:_(s32) = COPY $w1
+    ; CHECK-NEXT: %cmp:_(s1) = samesign G_ICMP intpred(eq), %y(s32), %y
+    ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %cmp(s1)
+    ; CHECK-NEXT: $w0 = COPY %zext(s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %x:_(s32) = COPY $w0
+    %y:_(s32) = COPY $w1
+    %cmp:_(s1) = samesign G_ICMP intpred(eq), %y:_(s32), %y:_
+    %zext:_(s32) = G_ZEXT %cmp:_(s1)
+    $w0 = COPY %zext
+    RET_ReallyLR implicit $w0
+
+
+...
+---
+name:            icmp_differentsign
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+
+    ; CHECK-LABEL: name: icmp_differentsign
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %x:_(s32) = COPY $w0
+    ; CHECK-NEXT: %y:_(s32) = COPY $w1
+    ; CHECK-NEXT: %cmp:_(s1) = G_ICMP intpred(eq), %y(s32), %y
+    ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %cmp(s1)
+    ; CHECK-NEXT: $w0 = COPY %zext(s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %x:_(s32) = COPY $w0
+    %y:_(s32) = COPY $w1
+    %cmp:_(s1) = G_ICMP intpred(eq), %y:_(s32), %y:_
+    %zext:_(s32) = G_ZEXT %cmp:_(s1)
+    $w0 = COPY %zext
+    RET_ReallyLR implicit $w0
+---
-- 
GitLab


From 4c03d373f043700e3c8feeea8855125c718de31b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= <brioche@google.com>
Date: Wed, 30 Oct 2024 16:40:36 +0100
Subject: [PATCH 596/693] [SPIR-V] Fix broken test due to G_BITCAST (#114242)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

G_BITCAST emission in the SPIR-V backend is not accepted by the
verifier. DIsabling verifier for impacted tests until
https://github.com/llvm/llvm-project/pull/114216 is merged.

Signed-off-by: Nathan Gauër <brioche@google.com>
---
 .../hlsl-intrinsics/group_memory_barrier_with_group_sync.ll    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll
index 6955411a0e4e..e314361fe418 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpMemoryModel Logical GLSL450
-- 
GitLab


From 45f420e34476d2963e13b2f916be1e5a73ec95ae Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Wed, 30 Oct 2024 08:41:30 -0700
Subject: [PATCH 597/693] [lldb] Use Py_InitializeFromConfig with Python >= 3.8
 (NFC) (#114112)

This fixes the deprecation warning for Py_SetPythonHome, which was
deprecated in Python 3.11. With this patch, when building against Python
3.8 or later, we now use Py_InitializeFromConfig instead.

Fixes #113475
---
 .../Python/ScriptInterpreterPython.cpp        | 68 +++++++++++--------
 1 file changed, 40 insertions(+), 28 deletions(-)

diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index 7cc38da6a6a9..6158083a9828 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -92,7 +92,38 @@ namespace {
 struct InitializePythonRAII {
 public:
   InitializePythonRAII() {
-    InitializePythonHome();
+#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 8) || (PY_MAJOR_VERSION > 3)
+    PyConfig config;
+    PyConfig_InitPythonConfig(&config);
+#endif
+
+#if LLDB_EMBED_PYTHON_HOME
+    typedef wchar_t *str_type;
+    static str_type g_python_home = []() -> str_type {
+      const char *lldb_python_home = LLDB_PYTHON_HOME;
+      const char *absolute_python_home = nullptr;
+      llvm::SmallString<64> path;
+      if (llvm::sys::path::is_absolute(lldb_python_home)) {
+        absolute_python_home = lldb_python_home;
+      } else {
+        FileSpec spec = HostInfo::GetShlibDir();
+        if (!spec)
+          return nullptr;
+        spec.GetPath(path);
+        llvm::sys::path::append(path, lldb_python_home);
+        absolute_python_home = path.c_str();
+      }
+      size_t size = 0;
+      return Py_DecodeLocale(absolute_python_home, &size);
+    }();
+    if (g_python_home != nullptr) {
+#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 8) || (PY_MAJOR_VERSION > 3)
+      PyConfig_SetBytesString(&config, &config.home, g_python_home);
+#else
+      Py_SetPythonHome(g_python_home);
+#endif
+    }
+#endif
 
     // The table of built-in modules can only be extended before Python is
     // initialized.
@@ -117,15 +148,22 @@ public:
       PyImport_AppendInittab("_lldb", LLDBSwigPyInit);
     }
 
+#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 8) || (PY_MAJOR_VERSION > 3)
+    config.install_signal_handlers = 0;
+    Py_InitializeFromConfig(&config);
+    PyConfig_Clear(&config);
+    InitializeThreadsPrivate();
+#else
 // Python < 3.2 and Python >= 3.2 reversed the ordering requirements for
 // calling `Py_Initialize` and `PyEval_InitThreads`.  < 3.2 requires that you
 // call `PyEval_InitThreads` first, and >= 3.2 requires that you call it last.
-#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 2) || (PY_MAJOR_VERSION > 3)
+#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 2)
     Py_InitializeEx(0);
     InitializeThreadsPrivate();
 #else
     InitializeThreadsPrivate();
     Py_InitializeEx(0);
+#endif
 #endif
   }
 
@@ -142,32 +180,6 @@ public:
   }
 
 private:
-  void InitializePythonHome() {
-#if LLDB_EMBED_PYTHON_HOME
-    typedef wchar_t *str_type;
-    static str_type g_python_home = []() -> str_type {
-      const char *lldb_python_home = LLDB_PYTHON_HOME;
-      const char *absolute_python_home = nullptr;
-      llvm::SmallString<64> path;
-      if (llvm::sys::path::is_absolute(lldb_python_home)) {
-        absolute_python_home = lldb_python_home;
-      } else {
-        FileSpec spec = HostInfo::GetShlibDir();
-        if (!spec)
-          return nullptr;
-        spec.GetPath(path);
-        llvm::sys::path::append(path, lldb_python_home);
-        absolute_python_home = path.c_str();
-      }
-      size_t size = 0;
-      return Py_DecodeLocale(absolute_python_home, &size);
-    }();
-    if (g_python_home != nullptr) {
-      Py_SetPythonHome(g_python_home);
-    }
-#endif
-  }
-
   void InitializeThreadsPrivate() {
 // Since Python 3.7 `Py_Initialize` calls `PyEval_InitThreads` inside itself,
 // so there is no way to determine whether the embedded interpreter
-- 
GitLab


From eac2c182c6f852fc187af9952250a43d6fb17b28 Mon Sep 17 00:00:00 2001
From: Adrian Prantl <aprantl@apple.com>
Date: Wed, 30 Oct 2024 08:59:08 -0700
Subject: [PATCH 598/693] Remove a flaky and unnecessary check (#114251)

The order in which the libraries appear is not always stable and even if
it were, this test is not the right place to check for this.
---
 .../libcxx/initializerlist/TestInitializerList.py               | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/TestInitializerList.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/TestInitializerList.py
index 0919eb3c5dd8..93d5392830b5 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/TestInitializerList.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/TestInitializerList.py
@@ -40,5 +40,3 @@ class InitializerListTestCase(TestBase):
             "frame variable ils",
             substrs=['[4] = "surprise it is a long string!! yay!!"'],
         )
-
-        self.expect("image list", substrs=self.getLibcPlusPlusLibs())
-- 
GitLab


From 4b028773b2c977eb8494a39e4b3fb2f114d1e2b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Wed, 30 Oct 2024 17:03:17 +0100
Subject: [PATCH 599/693] Revert "[GlobalISel] Import samesign flag" (#114256)

Reverts llvm/llvm-project#113090
---
 .../CodeGen/GlobalISel/GenericMachineInstrs.h |  2 +-
 .../CodeGen/GlobalISel/MachineIRBuilder.h     |  3 +-
 llvm/include/llvm/CodeGen/MachineInstr.h      |  1 -
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |  9 ++-
 .../CodeGen/GlobalISel/MachineIRBuilder.cpp   |  5 +-
 llvm/lib/CodeGen/MIRParser/MILexer.cpp        |  1 -
 llvm/lib/CodeGen/MIRParser/MILexer.h          |  1 -
 llvm/lib/CodeGen/MIRParser/MIParser.cpp       |  5 +-
 llvm/lib/CodeGen/MIRPrinter.cpp               |  2 -
 llvm/lib/CodeGen/MachineInstr.cpp             |  7 --
 .../GlobalISel/irtranslator-samesign.ll       | 69 -------------------
 llvm/test/CodeGen/MIR/icmp-flags.mir          | 50 --------------
 12 files changed, 11 insertions(+), 144 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-samesign.ll
 delete mode 100644 llvm/test/CodeGen/MIR/icmp-flags.mir

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index cd7ebcf54c9e..b6309a9ea0ec 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -28,7 +28,7 @@ namespace llvm {
 class GenericMachineInstr : public MachineInstr {
   constexpr static unsigned PoisonFlags = NoUWrap | NoSWrap | NoUSWrap |
                                           IsExact | Disjoint | NonNeg |
-                                          FmNoNans | FmNoInfs | SameSign;
+                                          FmNoNans | FmNoInfs;
 
 public:
   GenericMachineInstr() = delete;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index a38dd34a1709..c41e74ec7ebd 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1266,8 +1266,7 @@ public:
   ///
   /// \return a MachineInstrBuilder for the newly created instruction.
   MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res,
-                                const SrcOp &Op0, const SrcOp &Op1,
-                                std::optional<unsigned> Flags = std::nullopt);
+                                const SrcOp &Op0, const SrcOp &Op1);
 
   /// Build and insert a \p Res = G_FCMP \p Pred\p Op0, \p Op1
   ///
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index ead6bbe1d5f6..360517324746 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -119,7 +119,6 @@ public:
     Disjoint = 1 << 19,      // Each bit is zero in at least one of the inputs.
     NoUSWrap = 1 << 20,      // Instruction supports geps
                              // no unsigned signed wrap.
-    SameSign = 1 << 21       // Both operands have the same sign.
   };
 
 private:
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index a87754389cc8..5381dce58f9e 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -340,17 +340,20 @@ bool IRTranslator::translateCompare(const User &U,
   Register Op1 = getOrCreateVReg(*U.getOperand(1));
   Register Res = getOrCreateVReg(U);
   CmpInst::Predicate Pred = CI->getPredicate();
-  uint32_t Flags = MachineInstr::copyFlagsFromInstruction(*CI);
   if (CmpInst::isIntPredicate(Pred))
-    MIRBuilder.buildICmp(Pred, Res, Op0, Op1, Flags);
+    MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
   else if (Pred == CmpInst::FCMP_FALSE)
     MIRBuilder.buildCopy(
         Res, getOrCreateVReg(*Constant::getNullValue(U.getType())));
   else if (Pred == CmpInst::FCMP_TRUE)
     MIRBuilder.buildCopy(
         Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
-  else
+  else {
+    uint32_t Flags = 0;
+    if (CI)
+      Flags = MachineInstr::copyFlagsFromInstruction(*CI);
     MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
+  }
 
   return true;
 }
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 15b916424784..59f2fc633f5d 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -898,9 +898,8 @@ MachineIRBuilder::buildFPTrunc(const DstOp &Res, const SrcOp &Op,
 MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred,
                                                 const DstOp &Res,
                                                 const SrcOp &Op0,
-                                                const SrcOp &Op1,
-                                                std::optional<unsigned> Flags) {
-  return buildInstr(TargetOpcode::G_ICMP, Res, {Pred, Op0, Op1}, Flags);
+                                                const SrcOp &Op1) {
+  return buildInstr(TargetOpcode::G_ICMP, Res, {Pred, Op0, Op1});
 }
 
 MachineInstrBuilder MachineIRBuilder::buildFCmp(CmpInst::Predicate Pred,
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.cpp b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
index 1c450b05f49e..5a3806ce5733 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
@@ -216,7 +216,6 @@ static MIToken::TokenKind getIdentifierKind(StringRef Identifier) {
       .Case("exact", MIToken::kw_exact)
       .Case("nneg", MIToken::kw_nneg)
       .Case("disjoint", MIToken::kw_disjoint)
-      .Case("samesign", MIToken::kw_samesign)
       .Case("nofpexcept", MIToken::kw_nofpexcept)
       .Case("unpredictable", MIToken::kw_unpredictable)
       .Case("debug-location", MIToken::kw_debug_location)
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.h b/llvm/lib/CodeGen/MIRParser/MILexer.h
index d7cd06759cfb..3931da3eaae1 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.h
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.h
@@ -77,7 +77,6 @@ struct MIToken {
     kw_unpredictable,
     kw_nneg,
     kw_disjoint,
-    kw_samesign,
     kw_debug_location,
     kw_debug_instr_number,
     kw_dbg_instr_ref,
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index 059814c70f82..45847b5830da 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -1476,8 +1476,7 @@ bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) {
          Token.is(MIToken::kw_noconvergent) ||
          Token.is(MIToken::kw_unpredictable) ||
          Token.is(MIToken::kw_nneg) ||
-         Token.is(MIToken::kw_disjoint) ||
-         Token.is(MIToken::kw_samesign)) {
+         Token.is(MIToken::kw_disjoint)) {
     // clang-format on
     // Mine frame and fast math flags
     if (Token.is(MIToken::kw_frame_setup))
@@ -1514,8 +1513,6 @@ bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) {
       Flags |= MachineInstr::NonNeg;
     if (Token.is(MIToken::kw_disjoint))
       Flags |= MachineInstr::Disjoint;
-    if (Token.is(MIToken::kw_samesign))
-      Flags |= MachineInstr::SameSign;
 
     lex();
   }
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index 658bbe0e577e..a015cd3c2a55 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -837,8 +837,6 @@ void MIPrinter::print(const MachineInstr &MI) {
     OS << "disjoint ";
   if (MI.getFlag(MachineInstr::NoUSWrap))
     OS << "nusw ";
-  if (MI.getFlag(MachineInstr::SameSign))
-    OS << "samesign ";
 
   OS << TII->getName(MI.getOpcode());
   if (I < E)
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 941861da5c56..c1bd0bb5b716 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -596,11 +596,6 @@ uint32_t MachineInstr::copyFlagsFromInstruction(const Instruction &I) {
       MIFlags |= MachineInstr::MIFlag::Disjoint;
   }
 
-  // Copy the samesign flag.
-  if (const ICmpInst *ICmp = dyn_cast<ICmpInst>(&I))
-    if (ICmp->hasSameSign())
-      MIFlags |= MachineInstr::MIFlag::SameSign;
-
   // Copy the exact flag.
   if (const PossiblyExactOperator *PE = dyn_cast<PossiblyExactOperator>(&I))
     if (PE->isExact())
@@ -1775,8 +1770,6 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST,
     OS << "nneg ";
   if (getFlag(MachineInstr::Disjoint))
     OS << "disjoint ";
-  if (getFlag(MachineInstr::SameSign))
-    OS << "samesign ";
 
   // Print the opcode name.
   if (TII)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-samesign.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-samesign.ll
deleted file mode 100644
index 0173f92c9822..000000000000
--- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-samesign.ll
+++ /dev/null
@@ -1,69 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -global-isel -mtriple=aarch64-linux-gnu -O0 -stop-after=irtranslator < %s | FileCheck %s
-
-
-define <2 x i1> @call_icmp_samesign_vector(<2 x i32> %a, <2 x i32> %b) {
-  ; CHECK-LABEL: name: call_icmp_samesign_vector
-  ; CHECK: bb.1.entry:
-  ; CHECK-NEXT:   liveins: $d0, $d1
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
-  ; CHECK-NEXT:   %2:_(<2 x s1>) = samesign G_ICMP intpred(ult), [[COPY]](<2 x s32>), [[COPY1]]
-  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(<2 x s32>) = G_ANYEXT %2(<2 x s1>)
-  ; CHECK-NEXT:   $d0 = COPY [[ANYEXT]](<2 x s32>)
-  ; CHECK-NEXT:   RET_ReallyLR implicit $d0
-entry:
-  %result = icmp samesign ult <2 x i32> %a, %b
-  ret <2 x i1> %result
-}
-
-define <2 x i1> @call_icmp_vector(<2 x i32> %a, <2 x i32> %b) {
-  ; CHECK-LABEL: name: call_icmp_vector
-  ; CHECK: bb.1.entry:
-  ; CHECK-NEXT:   liveins: $d0, $d1
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
-  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:_(<2 x s1>) = G_ICMP intpred(ult), [[COPY]](<2 x s32>), [[COPY1]]
-  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(<2 x s32>) = G_ANYEXT [[ICMP]](<2 x s1>)
-  ; CHECK-NEXT:   $d0 = COPY [[ANYEXT]](<2 x s32>)
-  ; CHECK-NEXT:   RET_ReallyLR implicit $d0
-entry:
-  %result = icmp ult <2 x i32> %a, %b
-  ret <2 x i1> %result
-}
-
-define i1 @call_icmp(i32 %a) {
-  ; CHECK-LABEL: name: call_icmp
-  ; CHECK: bb.1.entry:
-  ; CHECK-NEXT:   liveins: $w0
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
-  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY]](s32), [[C]]
-  ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:_(s8) = G_ZEXT [[ICMP]](s1)
-  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ZEXT]](s8)
-  ; CHECK-NEXT:   $w0 = COPY [[ANYEXT]](s32)
-  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
-entry:
-  %result = icmp ult i32 %a, 3
-  ret i1 %result
-}
-
-define i1 @call_icmp_samesign(i32 %a) {
-  ; CHECK-LABEL: name: call_icmp_samesign
-  ; CHECK: bb.1.entry:
-  ; CHECK-NEXT:   liveins: $w0
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
-  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-  ; CHECK-NEXT:   %2:_(s1) = samesign G_ICMP intpred(ult), [[COPY]](s32), [[C]]
-  ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:_(s8) = G_ZEXT %2(s1)
-  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ZEXT]](s8)
-  ; CHECK-NEXT:   $w0 = COPY [[ANYEXT]](s32)
-  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
-entry:
-  %result = icmp samesign ult i32 %a, 3
-  ret i1 %result
-}
diff --git a/llvm/test/CodeGen/MIR/icmp-flags.mir b/llvm/test/CodeGen/MIR/icmp-flags.mir
deleted file mode 100644
index 3c03a7aaa9bc..000000000000
--- a/llvm/test/CodeGen/MIR/icmp-flags.mir
+++ /dev/null
@@ -1,50 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple aarch64 -run-pass=none -verify-machineinstrs %s -o - | FileCheck %s
-
-
----
-name:            icmp_samesign
-body:             |
-  bb.0:
-    liveins: $w0, $w1
-
-    ; CHECK-LABEL: name: icmp_samesign
-    ; CHECK: liveins: $w0, $w1
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: %x:_(s32) = COPY $w0
-    ; CHECK-NEXT: %y:_(s32) = COPY $w1
-    ; CHECK-NEXT: %cmp:_(s1) = samesign G_ICMP intpred(eq), %y(s32), %y
-    ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %cmp(s1)
-    ; CHECK-NEXT: $w0 = COPY %zext(s32)
-    ; CHECK-NEXT: RET_ReallyLR implicit $w0
-    %x:_(s32) = COPY $w0
-    %y:_(s32) = COPY $w1
-    %cmp:_(s1) = samesign G_ICMP intpred(eq), %y:_(s32), %y:_
-    %zext:_(s32) = G_ZEXT %cmp:_(s1)
-    $w0 = COPY %zext
-    RET_ReallyLR implicit $w0
-
-
-...
----
-name:            icmp_differentsign
-body:             |
-  bb.0:
-    liveins: $w0, $w1
-
-    ; CHECK-LABEL: name: icmp_differentsign
-    ; CHECK: liveins: $w0, $w1
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: %x:_(s32) = COPY $w0
-    ; CHECK-NEXT: %y:_(s32) = COPY $w1
-    ; CHECK-NEXT: %cmp:_(s1) = G_ICMP intpred(eq), %y(s32), %y
-    ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %cmp(s1)
-    ; CHECK-NEXT: $w0 = COPY %zext(s32)
-    ; CHECK-NEXT: RET_ReallyLR implicit $w0
-    %x:_(s32) = COPY $w0
-    %y:_(s32) = COPY $w1
-    %cmp:_(s1) = G_ICMP intpred(eq), %y:_(s32), %y:_
-    %zext:_(s32) = G_ZEXT %cmp:_(s1)
-    $w0 = COPY %zext
-    RET_ReallyLR implicit $w0
----
-- 
GitLab


From 8ee5e19c879ee2d467aa0f1eb8f1d8ed34321496 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Wed, 30 Oct 2024 16:12:37 +0000
Subject: [PATCH 600/693] [AMDGPU] Fix @llvm.amdgcn.cs.chain with SGPR args not
 provably uniform (#114232)

The correct behaviour is to insert a readfirstlane. SelectionDAG was
already doing this in some cases, but not in the general case for chain
calls. GlobalISel was already doing this for return values but not for
arguments.
---
 llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp |   7 -
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  12 +-
 .../irtranslator-amdgcn-cs-chain.ll           |  36 ++-
 .../GlobalISel/irtranslator-call-non-fixed.ll |   9 +-
 .../AMDGPU/GlobalISel/irtranslator-call.ll    |  96 ++++---
 .../test/CodeGen/AMDGPU/amdgpu-cs-chain-cc.ll |  73 ++++++
 .../isel-amdgcn-cs-chain-intrinsic-w32.ll     | 236 ++++++++++++------
 .../isel-amdgcn-cs-chain-intrinsic-w64.ll     | 236 ++++++++++++------
 8 files changed, 497 insertions(+), 208 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index 351e9f25e29c..ab62e530a18d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -230,13 +230,6 @@ struct AMDGPUOutgoingArgHandler : public AMDGPUOutgoingValueHandler {
     return AddrReg.getReg(0);
   }
 
-  void assignValueToReg(Register ValVReg, Register PhysReg,
-                        const CCValAssign &VA) override {
-    MIB.addUse(PhysReg, RegState::Implicit);
-    Register ExtReg = extendRegisterMin32(*this, ValVReg, VA);
-    MIRBuilder.buildCopy(PhysReg, ExtReg);
-  }
-
   void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
                             const MachinePointerInfo &MPO,
                             const CCValAssign &VA) override {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 52ca38aca5c7..059b415b75ff 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -3855,10 +3855,14 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
 
   unsigned ArgIdx = 0;
   for (auto [Reg, Val] : RegsToPass) {
-    if (ArgIdx++ >= NumSpecialInputs && !Val->isDivergent() &&
-        TRI->isSGPRPhysReg(Reg)) {
-      // Speculatively insert a readfirstlane in case this is a uniform value in
-      // a VGPR.
+    if (ArgIdx++ >= NumSpecialInputs &&
+        (IsChainCallConv || !Val->isDivergent()) && TRI->isSGPRPhysReg(Reg)) {
+      // For chain calls, the inreg arguments are required to be
+      // uniform. Speculatively Insert a readfirstlane in case we cannot prove
+      // they are uniform.
+      //
+      // For other calls, if an inreg arguments is known to be uniform,
+      // speculatively insert a readfirstlane in case it is in a VGPR.
       //
       // FIXME: We need to execute this in a waterfall loop if it is a divergent
       // value, so let that continue to produce invalid code.
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-cs-chain.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-cs-chain.ll
index 3438cbdd476d..4b0ff1b2eb47 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-cs-chain.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-cs-chain.ll
@@ -24,9 +24,12 @@ define amdgpu_cs_chain void @chain_call(<3 x i32> inreg %sgpr, { i32, ptr addrsp
   ; GFX11-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX11-NEXT:   [[GV1:%[0-9]+]]:ccr_sgpr_64(p0) = G_GLOBAL_VALUE @callee
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x s32>)
-  ; GFX11-NEXT:   $sgpr0 = COPY [[UV]](s32)
-  ; GFX11-NEXT:   $sgpr1 = COPY [[UV1]](s32)
-  ; GFX11-NEXT:   $sgpr2 = COPY [[UV2]](s32)
+  ; GFX11-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV]](s32)
+  ; GFX11-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; GFX11-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
+  ; GFX11-NEXT:   $sgpr1 = COPY [[INTRINSIC_CONVERGENT1]](s32)
+  ; GFX11-NEXT:   [[INTRINSIC_CONVERGENT2:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](s32)
+  ; GFX11-NEXT:   $sgpr2 = COPY [[INTRINSIC_CONVERGENT2]](s32)
   ; GFX11-NEXT:   $vgpr8 = COPY [[COPY3]](s32)
   ; GFX11-NEXT:   $vgpr9 = COPY [[COPY4]](p5)
   ; GFX11-NEXT:   $vgpr10 = COPY [[COPY5]](s32)
@@ -50,9 +53,12 @@ define amdgpu_cs_chain void @chain_call(<3 x i32> inreg %sgpr, { i32, ptr addrsp
   ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[GV1:%[0-9]+]]:ccr_sgpr_64(p0) = G_GLOBAL_VALUE @callee
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x s32>)
-  ; GFX10-NEXT:   $sgpr0 = COPY [[UV]](s32)
-  ; GFX10-NEXT:   $sgpr1 = COPY [[UV1]](s32)
-  ; GFX10-NEXT:   $sgpr2 = COPY [[UV2]](s32)
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV]](s32)
+  ; GFX10-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
+  ; GFX10-NEXT:   $sgpr1 = COPY [[INTRINSIC_CONVERGENT1]](s32)
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT2:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](s32)
+  ; GFX10-NEXT:   $sgpr2 = COPY [[INTRINSIC_CONVERGENT2]](s32)
   ; GFX10-NEXT:   $vgpr8 = COPY [[COPY3]](s32)
   ; GFX10-NEXT:   $vgpr9 = COPY [[COPY4]](p5)
   ; GFX10-NEXT:   $vgpr10 = COPY [[COPY5]](s32)
@@ -82,9 +88,12 @@ define amdgpu_cs_chain void @chain_preserve_call(<3 x i32> inreg %sgpr, { i32, p
   ; GFX11-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX11-NEXT:   [[GV1:%[0-9]+]]:ccr_sgpr_64(p0) = G_GLOBAL_VALUE @callee_preserve
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x s32>)
-  ; GFX11-NEXT:   $sgpr0 = COPY [[UV]](s32)
-  ; GFX11-NEXT:   $sgpr1 = COPY [[UV1]](s32)
-  ; GFX11-NEXT:   $sgpr2 = COPY [[UV2]](s32)
+  ; GFX11-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV]](s32)
+  ; GFX11-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; GFX11-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
+  ; GFX11-NEXT:   $sgpr1 = COPY [[INTRINSIC_CONVERGENT1]](s32)
+  ; GFX11-NEXT:   [[INTRINSIC_CONVERGENT2:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](s32)
+  ; GFX11-NEXT:   $sgpr2 = COPY [[INTRINSIC_CONVERGENT2]](s32)
   ; GFX11-NEXT:   $vgpr8 = COPY [[COPY3]](s32)
   ; GFX11-NEXT:   $vgpr9 = COPY [[COPY4]](p5)
   ; GFX11-NEXT:   $vgpr10 = COPY [[COPY5]](s32)
@@ -108,9 +117,12 @@ define amdgpu_cs_chain void @chain_preserve_call(<3 x i32> inreg %sgpr, { i32, p
   ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[GV1:%[0-9]+]]:ccr_sgpr_64(p0) = G_GLOBAL_VALUE @callee_preserve
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x s32>)
-  ; GFX10-NEXT:   $sgpr0 = COPY [[UV]](s32)
-  ; GFX10-NEXT:   $sgpr1 = COPY [[UV1]](s32)
-  ; GFX10-NEXT:   $sgpr2 = COPY [[UV2]](s32)
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV]](s32)
+  ; GFX10-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
+  ; GFX10-NEXT:   $sgpr1 = COPY [[INTRINSIC_CONVERGENT1]](s32)
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT2:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](s32)
+  ; GFX10-NEXT:   $sgpr2 = COPY [[INTRINSIC_CONVERGENT2]](s32)
   ; GFX10-NEXT:   $vgpr8 = COPY [[COPY3]](s32)
   ; GFX10-NEXT:   $vgpr9 = COPY [[COPY4]](p5)
   ; GFX10-NEXT:   $vgpr10 = COPY [[COPY5]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll
index 5effd24a7520..adad38de380d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll
@@ -50,7 +50,8 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_i32_imm_inreg(i32 inreg
   ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
   ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $scc
   ; CHECK-NEXT:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_i32_inreg
-  ; CHECK-NEXT:   $sgpr4 = COPY [[C]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[C]](s32)
+  ; CHECK-NEXT:   $sgpr4 = COPY [[INTRINSIC_CONVERGENT]](s32)
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY1]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr0_sgpr1_sgpr2_sgpr3
@@ -99,8 +100,10 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_struct_i8_i32_inreg() #
   ; CHECK-NEXT:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_struct_i8_i32_inreg
   ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[LOAD1]](s8)
   ; CHECK-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT]](s16)
-  ; CHECK-NEXT:   $sgpr4 = COPY [[ANYEXT1]](s32)
-  ; CHECK-NEXT:   $sgpr5 = COPY [[LOAD2]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[ANYEXT1]](s32)
+  ; CHECK-NEXT:   $sgpr4 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[LOAD2]](s32)
+  ; CHECK-NEXT:   $sgpr5 = COPY [[INTRINSIC_CONVERGENT1]](s32)
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_struct_i8_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr5, implicit $sgpr0_sgpr1_sgpr2_sgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
index c3694158e7b9..96c3575e3190 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
@@ -942,7 +942,8 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_i32_imm_inreg(i32 inreg
   ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
   ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $scc
   ; CHECK-NEXT:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_i32_inreg
-  ; CHECK-NEXT:   $sgpr4 = COPY [[C]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[C]](s32)
+  ; CHECK-NEXT:   $sgpr4 = COPY [[INTRINSIC_CONVERGENT]](s32)
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY1]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr0_sgpr1_sgpr2_sgpr3
@@ -3984,8 +3985,10 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_struct_i8_i32_inreg() #
   ; CHECK-NEXT:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_struct_i8_i32_inreg
   ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[LOAD1]](s8)
   ; CHECK-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT]](s16)
-  ; CHECK-NEXT:   $sgpr4 = COPY [[ANYEXT1]](s32)
-  ; CHECK-NEXT:   $sgpr5 = COPY [[LOAD2]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[ANYEXT1]](s32)
+  ; CHECK-NEXT:   $sgpr4 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[LOAD2]](s32)
+  ; CHECK-NEXT:   $sgpr5 = COPY [[INTRINSIC_CONVERGENT1]](s32)
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_struct_i8_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr5, implicit $sgpr0_sgpr1_sgpr2_sgpr3
@@ -5309,7 +5312,8 @@ define void @test_call_external_void_func_i16_inreg(i16 inreg %arg) #0 {
   ; CHECK-NEXT:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
   ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[ANYEXT]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
   ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY19]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
@@ -5354,7 +5358,8 @@ define void @test_call_external_void_func_i32_inreg(i32 inreg %arg) #0 {
   ; CHECK-NEXT:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY2]]
   ; CHECK-NEXT:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[COPY9]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY9]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
   ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY19]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
@@ -5402,8 +5407,10 @@ define void @test_call_external_void_func_i64_inreg(i64 inreg %arg) #0 {
   ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
   ; CHECK-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](s64)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[UV]](s32)
-  ; CHECK-NEXT:   $sgpr1 = COPY [[UV1]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
+  ; CHECK-NEXT:   $sgpr1 = COPY [[INTRINSIC_CONVERGENT1]](s32)
   ; CHECK-NEXT:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY11]](p4)
@@ -5451,8 +5458,10 @@ define void @test_call_external_void_func_v2i32_inreg(<2 x i32> inreg %arg) #0 {
   ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
   ; CHECK-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s32>)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[UV]](s32)
-  ; CHECK-NEXT:   $sgpr1 = COPY [[UV1]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
+  ; CHECK-NEXT:   $sgpr1 = COPY [[INTRINSIC_CONVERGENT1]](s32)
   ; CHECK-NEXT:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY11]](p4)
@@ -5499,7 +5508,8 @@ define void @test_call_external_void_func_f16_inreg(half inreg %arg) #0 {
   ; CHECK-NEXT:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
   ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[ANYEXT]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
   ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY19]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
@@ -5546,7 +5556,8 @@ define void @test_call_external_void_func_bf16_inreg(bfloat inreg %arg) #0 {
   ; CHECK-NEXT:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
   ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[ANYEXT]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
   ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY19]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
@@ -5591,7 +5602,8 @@ define void @test_call_external_void_func_f32_inreg(float inreg %arg) #0 {
   ; CHECK-NEXT:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY2]]
   ; CHECK-NEXT:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[COPY9]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY9]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
   ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY19]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
@@ -5639,8 +5651,10 @@ define void @test_call_external_void_func_f64_inreg(double inreg %arg) #0 {
   ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
   ; CHECK-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](s64)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[UV]](s32)
-  ; CHECK-NEXT:   $sgpr1 = COPY [[UV1]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
+  ; CHECK-NEXT:   $sgpr1 = COPY [[INTRINSIC_CONVERGENT1]](s32)
   ; CHECK-NEXT:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY11]](p4)
@@ -5685,7 +5699,9 @@ define void @test_call_external_void_func_v2f16_inreg(<2 x half> inreg %arg) #0
   ; CHECK-NEXT:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY2]]
   ; CHECK-NEXT:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[COPY9]](<2 x s16>)
+  ; CHECK-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[BITCAST]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
   ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY19]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
@@ -5738,8 +5754,12 @@ define void @test_call_external_void_func_v3f16_inreg(<3 x half> inreg %arg) #0
   ; CHECK-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; CHECK-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[UV4]](s16), [[UV5]](s16), [[UV6]](s16), [[DEF]](s16)
   ; CHECK-NEXT:   [[UV7:%[0-9]+]]:_(<2 x s16>), [[UV8:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s16>)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[UV7]](<2 x s16>)
-  ; CHECK-NEXT:   $sgpr1 = COPY [[UV8]](<2 x s16>)
+  ; CHECK-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[BITCAST]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; CHECK-NEXT:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[BITCAST1]](s32)
+  ; CHECK-NEXT:   $sgpr1 = COPY [[INTRINSIC_CONVERGENT1]](s32)
   ; CHECK-NEXT:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY11]](p4)
@@ -5787,8 +5807,12 @@ define void @test_call_external_void_func_v4f16_inreg(<4 x half> inreg %arg) #0
   ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
   ; CHECK-NEXT:   [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x s16>)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[UV]](<2 x s16>)
-  ; CHECK-NEXT:   $sgpr1 = COPY [[UV1]](<2 x s16>)
+  ; CHECK-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[BITCAST]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; CHECK-NEXT:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[BITCAST1]](s32)
+  ; CHECK-NEXT:   $sgpr1 = COPY [[INTRINSIC_CONVERGENT1]](s32)
   ; CHECK-NEXT:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY11]](p4)
@@ -5836,8 +5860,10 @@ define void @test_call_external_void_func_p0_inreg(ptr inreg %arg) #0 {
   ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
   ; CHECK-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p0)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[UV]](s32)
-  ; CHECK-NEXT:   $sgpr1 = COPY [[UV1]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
+  ; CHECK-NEXT:   $sgpr1 = COPY [[INTRINSIC_CONVERGENT1]](s32)
   ; CHECK-NEXT:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY11]](p4)
@@ -5885,8 +5911,10 @@ define void @test_call_external_void_func_p1_inreg(ptr addrspace(1) inreg %arg)
   ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
   ; CHECK-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p1)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[UV]](s32)
-  ; CHECK-NEXT:   $sgpr1 = COPY [[UV1]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
+  ; CHECK-NEXT:   $sgpr1 = COPY [[INTRINSIC_CONVERGENT1]](s32)
   ; CHECK-NEXT:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY11]](p4)
@@ -5931,7 +5959,9 @@ define void @test_call_external_void_func_p3_inreg(ptr addrspace(3) inreg %arg)
   ; CHECK-NEXT:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY2]]
   ; CHECK-NEXT:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[COPY9]](p3)
+  ; CHECK-NEXT:   [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY9]](p3)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[PTRTOINT]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
   ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY19]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
@@ -5983,10 +6013,14 @@ define void @test_call_external_void_func_v2p1_inreg(<2 x ptr addrspace(1)> inre
   ; CHECK-NEXT:   [[COPY20:%[0-9]+]]:_(s32) = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY21:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
   ; CHECK-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x p1>)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[UV]](s32)
-  ; CHECK-NEXT:   $sgpr1 = COPY [[UV1]](s32)
-  ; CHECK-NEXT:   $sgpr2 = COPY [[UV2]](s32)
-  ; CHECK-NEXT:   $sgpr3 = COPY [[UV3]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
+  ; CHECK-NEXT:   $sgpr1 = COPY [[INTRINSIC_CONVERGENT1]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT2:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](s32)
+  ; CHECK-NEXT:   $sgpr2 = COPY [[INTRINSIC_CONVERGENT2]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT3:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](s32)
+  ; CHECK-NEXT:   $sgpr3 = COPY [[INTRINSIC_CONVERGENT3]](s32)
   ; CHECK-NEXT:   [[COPY22:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY22]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY13]](p4)
@@ -6034,8 +6068,10 @@ define void @test_call_external_void_func_v2p5_inreg(<2 x ptr addrspace(5)> inre
   ; CHECK-NEXT:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
   ; CHECK-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x p5>)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[UV]](s32)
-  ; CHECK-NEXT:   $sgpr1 = COPY [[UV1]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV]](s32)
+  ; CHECK-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
+  ; CHECK-NEXT:   $sgpr1 = COPY [[INTRINSIC_CONVERGENT1]](s32)
   ; CHECK-NEXT:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
   ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY11]](p4)
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-cc.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-cc.ll
index 06f66e05d674..8ca3e8255b63 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-cc.ll
@@ -501,6 +501,79 @@ define amdgpu_cs void @cs_to_chain(<3 x i32> inreg %a, <3 x i32> %b) {
   unreachable
 }
 
+; Chain call with SGPR arguments that we cannot prove are uniform.
+define amdgpu_cs void @cs_to_chain_nonuniform(<3 x i32> %a, <3 x i32> %b) {
+; GISEL-GFX11-LABEL: cs_to_chain_nonuniform:
+; GISEL-GFX11:       ; %bb.0:
+; GISEL-GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GISEL-GFX11-NEXT:    v_readfirstlane_b32 s1, v1
+; GISEL-GFX11-NEXT:    v_readfirstlane_b32 s2, v2
+; GISEL-GFX11-NEXT:    v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
+; GISEL-GFX11-NEXT:    v_mov_b32_e32 v10, v5
+; GISEL-GFX11-NEXT:    s_mov_b32 s4, chain_callee@abs32@lo
+; GISEL-GFX11-NEXT:    s_mov_b32 s5, chain_callee@abs32@hi
+; GISEL-GFX11-NEXT:    s_mov_b32 exec_lo, -1
+; GISEL-GFX11-NEXT:    s_setpc_b64 s[4:5]
+;
+; GISEL-GFX10-LABEL: cs_to_chain_nonuniform:
+; GISEL-GFX10:       ; %bb.0:
+; GISEL-GFX10-NEXT:    s_getpc_b64 s[100:101]
+; GISEL-GFX10-NEXT:    s_mov_b32 s100, s0
+; GISEL-GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GISEL-GFX10-NEXT:    s_load_dwordx4 s[100:103], s[100:101], 0x10
+; GISEL-GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GISEL-GFX10-NEXT:    v_mov_b32_e32 v8, v3
+; GISEL-GFX10-NEXT:    v_mov_b32_e32 v9, v4
+; GISEL-GFX10-NEXT:    v_mov_b32_e32 v10, v5
+; GISEL-GFX10-NEXT:    s_mov_b32 s4, chain_callee@abs32@lo
+; GISEL-GFX10-NEXT:    s_mov_b32 s5, chain_callee@abs32@hi
+; GISEL-GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-GFX10-NEXT:    s_bitset0_b32 s103, 21
+; GISEL-GFX10-NEXT:    s_add_u32 s100, s100, s0
+; GISEL-GFX10-NEXT:    s_addc_u32 s101, s101, 0
+; GISEL-GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GISEL-GFX10-NEXT:    s_mov_b64 s[48:49], s[100:101]
+; GISEL-GFX10-NEXT:    s_mov_b64 s[50:51], s[102:103]
+; GISEL-GFX10-NEXT:    s_mov_b32 exec_lo, -1
+; GISEL-GFX10-NEXT:    s_setpc_b64 s[4:5]
+;
+; DAGISEL-GFX11-LABEL: cs_to_chain_nonuniform:
+; DAGISEL-GFX11:       ; %bb.0:
+; DAGISEL-GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; DAGISEL-GFX11-NEXT:    v_readfirstlane_b32 s1, v1
+; DAGISEL-GFX11-NEXT:    v_readfirstlane_b32 s2, v2
+; DAGISEL-GFX11-NEXT:    v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
+; DAGISEL-GFX11-NEXT:    v_mov_b32_e32 v10, v5
+; DAGISEL-GFX11-NEXT:    s_mov_b32 s5, chain_callee@abs32@hi
+; DAGISEL-GFX11-NEXT:    s_mov_b32 s4, chain_callee@abs32@lo
+; DAGISEL-GFX11-NEXT:    s_mov_b32 exec_lo, -1
+; DAGISEL-GFX11-NEXT:    s_setpc_b64 s[4:5]
+;
+; DAGISEL-GFX10-LABEL: cs_to_chain_nonuniform:
+; DAGISEL-GFX10:       ; %bb.0:
+; DAGISEL-GFX10-NEXT:    s_getpc_b64 s[100:101]
+; DAGISEL-GFX10-NEXT:    s_mov_b32 s100, s0
+; DAGISEL-GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; DAGISEL-GFX10-NEXT:    s_load_dwordx4 s[100:103], s[100:101], 0x10
+; DAGISEL-GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; DAGISEL-GFX10-NEXT:    v_mov_b32_e32 v8, v3
+; DAGISEL-GFX10-NEXT:    v_mov_b32_e32 v9, v4
+; DAGISEL-GFX10-NEXT:    v_mov_b32_e32 v10, v5
+; DAGISEL-GFX10-NEXT:    s_mov_b32 s5, chain_callee@abs32@hi
+; DAGISEL-GFX10-NEXT:    s_mov_b32 s4, chain_callee@abs32@lo
+; DAGISEL-GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; DAGISEL-GFX10-NEXT:    s_bitset0_b32 s103, 21
+; DAGISEL-GFX10-NEXT:    s_add_u32 s100, s100, s0
+; DAGISEL-GFX10-NEXT:    s_addc_u32 s101, s101, 0
+; DAGISEL-GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; DAGISEL-GFX10-NEXT:    s_mov_b64 s[48:49], s[100:101]
+; DAGISEL-GFX10-NEXT:    s_mov_b64 s[50:51], s[102:103]
+; DAGISEL-GFX10-NEXT:    s_mov_b32 exec_lo, -1
+; DAGISEL-GFX10-NEXT:    s_setpc_b64 s[4:5]
+  call void(ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
+  unreachable
+}
+
 define amdgpu_cs_chain void @chain_to_chain(<3 x i32> inreg %a, <3 x i32> %b) {
 ; GISEL-GFX11-LABEL: chain_to_chain:
 ; GISEL-GFX11:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll
index 469d0453b9df..75616d276754 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll
@@ -20,9 +20,15 @@ define amdgpu_cs_chain void @chain_to_chain(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; GISEL-GFX11-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY5]]
@@ -30,8 +36,8 @@ define amdgpu_cs_chain void @chain_to_chain(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; GISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY7]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; GISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY10]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   ;
   ; GISEL-GFX10-LABEL: name: chain_to_chain
   ; GISEL-GFX10: bb.1 (%ir-block.0):
@@ -44,20 +50,26 @@ define amdgpu_cs_chain void @chain_to_chain(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; GISEL-GFX10-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY5]]
   ; GISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY6]]
-  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
+  ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; GISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY8]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY11]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: chain_to_chain
   ; DAGISEL-GFX11: bb.0 (%ir-block.0):
@@ -136,9 +148,15 @@ define amdgpu_cs void @cs_to_chain(<3 x i32> inreg %sgpr, { i32, ptr addrspace(5
   ; GISEL-GFX11-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GISEL-GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY5]]
@@ -146,8 +164,8 @@ define amdgpu_cs void @cs_to_chain(<3 x i32> inreg %sgpr, { i32, ptr addrspace(5
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; GISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY7]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; GISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY10]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   ;
   ; GISEL-GFX10-LABEL: name: cs_to_chain
   ; GISEL-GFX10: bb.1 (%ir-block.0):
@@ -160,20 +178,26 @@ define amdgpu_cs void @cs_to_chain(<3 x i32> inreg %sgpr, { i32, ptr addrspace(5
   ; GISEL-GFX10-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY5]]
   ; GISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY6]]
-  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
+  ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; GISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY8]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY11]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: cs_to_chain
   ; DAGISEL-GFX11: bb.0 (%ir-block.0):
@@ -252,9 +276,15 @@ define amdgpu_cs_chain void @chain_to_chain_preserve(<3 x i32> inreg %sgpr, { i3
   ; GISEL-GFX11-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY5]]
@@ -262,8 +292,8 @@ define amdgpu_cs_chain void @chain_to_chain_preserve(<3 x i32> inreg %sgpr, { i3
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; GISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY7]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; GISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY10]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   ;
   ; GISEL-GFX10-LABEL: name: chain_to_chain_preserve
   ; GISEL-GFX10: bb.1 (%ir-block.0):
@@ -276,20 +306,26 @@ define amdgpu_cs_chain void @chain_to_chain_preserve(<3 x i32> inreg %sgpr, { i3
   ; GISEL-GFX10-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY5]]
   ; GISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY6]]
-  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
+  ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; GISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY8]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY11]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: chain_to_chain_preserve
   ; DAGISEL-GFX11: bb.0 (%ir-block.0):
@@ -368,9 +404,15 @@ define amdgpu_cs void @cs_to_chain_preserve(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; GISEL-GFX11-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GISEL-GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY5]]
@@ -378,8 +420,8 @@ define amdgpu_cs void @cs_to_chain_preserve(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; GISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY7]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; GISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY10]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   ;
   ; GISEL-GFX10-LABEL: name: cs_to_chain_preserve
   ; GISEL-GFX10: bb.1 (%ir-block.0):
@@ -392,20 +434,26 @@ define amdgpu_cs void @cs_to_chain_preserve(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; GISEL-GFX10-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY5]]
   ; GISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY6]]
-  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
+  ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; GISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY8]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY11]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: cs_to_chain_preserve
   ; DAGISEL-GFX11: bb.0 (%ir-block.0):
@@ -487,9 +535,15 @@ define amdgpu_cs_chain void @indirect(ptr inreg %callee, <3 x i32> inreg %sgpr,
   ; GISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY2]]
-  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY3]]
-  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; GISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX11-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY5]]
   ; GISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY6]]
   ; GISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY7]]
@@ -510,15 +564,21 @@ define amdgpu_cs_chain void @indirect(ptr inreg %callee, <3 x i32> inreg %sgpr,
   ; GISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY2]]
-  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY3]]
-  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY5]]
   ; GISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY6]]
   ; GISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY7]]
   ; GISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY8]]
-  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY9]]
+  ; GISEL-GFX10-NEXT:   [[COPY12:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY12]]
   ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 [[REG_SEQUENCE]], 0, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: indirect
@@ -613,9 +673,15 @@ define amdgpu_cs_chain void @non_imm_exec(i32 inreg %exec, <3 x i32> inreg %sgpr
   ; GISEL-GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY1]]
-  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY2]]
-  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY3]]
+  ; GISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY4]]
   ; GISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY5]]
   ; GISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY6]]
@@ -623,8 +689,8 @@ define amdgpu_cs_chain void @non_imm_exec(i32 inreg %exec, <3 x i32> inreg %sgpr
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; GISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY8]], @callee, 0, [[COPY]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; GISEL-GFX11-NEXT:   [[COPY11:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY11]], @callee, 0, [[COPY]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   ;
   ; GISEL-GFX10-LABEL: name: non_imm_exec
   ; GISEL-GFX10: bb.1 (%ir-block.0):
@@ -638,20 +704,26 @@ define amdgpu_cs_chain void @non_imm_exec(i32 inreg %exec, <3 x i32> inreg %sgpr
   ; GISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY1]]
-  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY2]]
-  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY3]]
+  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY4]]
   ; GISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY5]]
   ; GISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY6]]
   ; GISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY7]]
-  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY8]]
+  ; GISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY11]]
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; GISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY9]], @callee, 0, [[COPY]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   [[COPY12:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY12]], @callee, 0, [[COPY]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: non_imm_exec
   ; DAGISEL-GFX11: bb.0 (%ir-block.0):
@@ -734,9 +806,15 @@ define amdgpu_cs_chain void @indirect_with_non_imm_exec(ptr inreg %callee, i32 i
   ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY3]]
-  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY4]]
-  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY5]]
+  ; GISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX11-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX11-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY6]]
   ; GISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY7]]
   ; GISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY8]]
@@ -758,15 +836,21 @@ define amdgpu_cs_chain void @indirect_with_non_imm_exec(ptr inreg %callee, i32 i
   ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY3]]
-  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY4]]
-  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY5]]
+  ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX10-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY6]]
   ; GISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY7]]
   ; GISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY8]]
   ; GISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY9]]
-  ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
+  ; GISEL-GFX10-NEXT:   [[COPY13:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY13]]
   ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 [[REG_SEQUENCE]], 0, 0, [[COPY2]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: indirect_with_non_imm_exec
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll
index 51c28a02b7f8..6deac9f55f32 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll
@@ -20,9 +20,15 @@ define amdgpu_cs_chain void @chain_to_chain(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; GISEL-GFX11-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY5]]
@@ -30,8 +36,8 @@ define amdgpu_cs_chain void @chain_to_chain(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; GISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY7]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; GISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY10]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   ;
   ; GISEL-GFX10-LABEL: name: chain_to_chain
   ; GISEL-GFX10: bb.1 (%ir-block.0):
@@ -44,20 +50,26 @@ define amdgpu_cs_chain void @chain_to_chain(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; GISEL-GFX10-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY5]]
   ; GISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY6]]
-  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
+  ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; GISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY8]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY11]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: chain_to_chain
   ; DAGISEL-GFX11: bb.0 (%ir-block.0):
@@ -136,9 +148,15 @@ define amdgpu_cs void @cs_to_chain(<3 x i32> inreg %sgpr, { i32, ptr addrspace(5
   ; GISEL-GFX11-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GISEL-GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY5]]
@@ -146,8 +164,8 @@ define amdgpu_cs void @cs_to_chain(<3 x i32> inreg %sgpr, { i32, ptr addrspace(5
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; GISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY7]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; GISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY10]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   ;
   ; GISEL-GFX10-LABEL: name: cs_to_chain
   ; GISEL-GFX10: bb.1 (%ir-block.0):
@@ -160,20 +178,26 @@ define amdgpu_cs void @cs_to_chain(<3 x i32> inreg %sgpr, { i32, ptr addrspace(5
   ; GISEL-GFX10-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY5]]
   ; GISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY6]]
-  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
+  ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; GISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY8]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY11]], @callee, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: cs_to_chain
   ; DAGISEL-GFX11: bb.0 (%ir-block.0):
@@ -252,9 +276,15 @@ define amdgpu_cs_chain void @chain_to_chain_preserve(<3 x i32> inreg %sgpr, { i3
   ; GISEL-GFX11-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY5]]
@@ -262,8 +292,8 @@ define amdgpu_cs_chain void @chain_to_chain_preserve(<3 x i32> inreg %sgpr, { i3
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; GISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY7]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; GISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY10]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   ;
   ; GISEL-GFX10-LABEL: name: chain_to_chain_preserve
   ; GISEL-GFX10: bb.1 (%ir-block.0):
@@ -276,20 +306,26 @@ define amdgpu_cs_chain void @chain_to_chain_preserve(<3 x i32> inreg %sgpr, { i3
   ; GISEL-GFX10-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY5]]
   ; GISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY6]]
-  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
+  ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; GISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY8]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY11]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: chain_to_chain_preserve
   ; DAGISEL-GFX11: bb.0 (%ir-block.0):
@@ -368,9 +404,15 @@ define amdgpu_cs void @cs_to_chain_preserve(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; GISEL-GFX11-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GISEL-GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY5]]
@@ -378,8 +420,8 @@ define amdgpu_cs void @cs_to_chain_preserve(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; GISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY7]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; GISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY10]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   ;
   ; GISEL-GFX10-LABEL: name: cs_to_chain_preserve
   ; GISEL-GFX10: bb.1 (%ir-block.0):
@@ -392,20 +434,26 @@ define amdgpu_cs void @cs_to_chain_preserve(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; GISEL-GFX10-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY]]
-  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY1]]
-  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; GISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY4]]
   ; GISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY5]]
   ; GISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY6]]
-  ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
+  ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; GISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
-  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY8]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY11]], @callee_preserve, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: cs_to_chain_preserve
   ; DAGISEL-GFX11: bb.0 (%ir-block.0):
@@ -487,9 +535,15 @@ define amdgpu_cs_chain void @indirect(ptr inreg %callee, <3 x i32> inreg %sgpr,
   ; GISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY2]]
-  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY3]]
-  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; GISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX11-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY5]]
   ; GISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY6]]
   ; GISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY7]]
@@ -510,15 +564,21 @@ define amdgpu_cs_chain void @indirect(ptr inreg %callee, <3 x i32> inreg %sgpr,
   ; GISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY2]]
-  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY3]]
-  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY5]]
   ; GISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY6]]
   ; GISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY7]]
   ; GISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY8]]
-  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY9]]
+  ; GISEL-GFX10-NEXT:   [[COPY12:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY12]]
   ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 [[REG_SEQUENCE]], 0, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: indirect
@@ -615,9 +675,15 @@ define amdgpu_cs_chain void @non_imm_exec(i64 inreg %exec, <3 x i32> inreg %sgpr
   ; GISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY2]]
-  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY3]]
-  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; GISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX11-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY5]]
   ; GISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY6]]
   ; GISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY7]]
@@ -625,8 +691,8 @@ define amdgpu_cs_chain void @non_imm_exec(i64 inreg %exec, <3 x i32> inreg %sgpr
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; GISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; GISEL-GFX11-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE1]]
-  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY9]], @callee, 0, [[REG_SEQUENCE]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; GISEL-GFX11-NEXT:   [[COPY12:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE1]]
+  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY12]], @callee, 0, [[REG_SEQUENCE]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   ;
   ; GISEL-GFX10-LABEL: name: non_imm_exec
   ; GISEL-GFX10: bb.1 (%ir-block.0):
@@ -642,20 +708,26 @@ define amdgpu_cs_chain void @non_imm_exec(i64 inreg %exec, <3 x i32> inreg %sgpr
   ; GISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY2]]
-  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY3]]
-  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY5]]
   ; GISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY6]]
   ; GISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY7]]
   ; GISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY8]]
-  ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY9]]
+  ; GISEL-GFX10-NEXT:   [[COPY12:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY12]]
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; GISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; GISEL-GFX10-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
-  ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE1]]
-  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY10]], @callee, 0, [[REG_SEQUENCE]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   [[COPY13:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE1]]
+  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY13]], @callee, 0, [[REG_SEQUENCE]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: non_imm_exec
   ; DAGISEL-GFX11: bb.0 (%ir-block.0):
@@ -744,9 +816,15 @@ define amdgpu_cs_chain void @indirect_with_non_imm_exec(ptr inreg %callee, i64 i
   ; GISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY4]]
-  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY6]]
+  ; GISEL-GFX11-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX11-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX11-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY7]]
   ; GISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY8]]
   ; GISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY9]]
@@ -770,15 +848,21 @@ define amdgpu_cs_chain void @indirect_with_non_imm_exec(ptr inreg %callee, i64 i
   ; GISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr9
   ; GISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr10
   ; GISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY $vgpr11
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY4]]
-  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY6]]
+  ; GISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GISEL-GFX10-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GISEL-GFX10-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY7]]
   ; GISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY8]]
   ; GISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY9]]
   ; GISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY10]]
-  ; GISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY11]]
+  ; GISEL-GFX10-NEXT:   [[COPY14:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY14]]
   ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 [[REG_SEQUENCE]], 0, 0, [[REG_SEQUENCE1]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: indirect_with_non_imm_exec
-- 
GitLab


From 1ddea4fc13eb12ddb4e71f7675a496de6d517ec4 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Wed, 30 Oct 2024 15:50:09 +0000
Subject: [PATCH 601/693] [AArch64] NFC: Refactoring of the SubRegIndexes in
 AArch64RegisterInfo.td

This is just moving some of the definitions around to all have them in
the same place. This is preparation for a follow-up patch that redefines
the SubRegIndexes to require less bits, and to define the top bits
of registers.
---
 .../lib/Target/AArch64/AArch64RegisterInfo.td | 42 +++++++++----------
 1 file changed, 20 insertions(+), 22 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
index 8516ab2c7dd7..4117d74d10c1 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
@@ -19,18 +19,24 @@ class AArch64Reg<bits<16> enc, string n, list<Register> subregs = [],
 }
 
 let Namespace = "AArch64" in {
+  // SubRegIndexes for GPR registers
   def sub_32 : SubRegIndex<32>;
+  def sube64 : SubRegIndex<64>;
+  def subo64 : SubRegIndex<64>;
+  def sube32 : SubRegIndex<32>;
+  def subo32 : SubRegIndex<32>;
 
+  // SubRegIndexes for FPR/Vector registers
   def bsub : SubRegIndex<8>;
   def hsub : SubRegIndex<16>;
   def ssub : SubRegIndex<32>;
   def dsub : SubRegIndex<64>;
-  def sube32 : SubRegIndex<32>;
-  def subo32 : SubRegIndex<32>;
-  def sube64 : SubRegIndex<64>;
-  def subo64 : SubRegIndex<64>;
-  // SVE
-  def zsub    : SubRegIndex<128>;
+  def zsub : SubRegIndex<128>;
+  // Note: Code depends on these having consecutive numbers
+  def zsub0 : SubRegIndex<128, -1>;
+  def zsub1 : SubRegIndex<128, -1>;
+  def zsub2 : SubRegIndex<128, -1>;
+  def zsub3 : SubRegIndex<128, -1>;
   // Note: Code depends on these having consecutive numbers
   def dsub0 : SubRegIndex<64>;
   def dsub1 : SubRegIndex<64>;
@@ -41,7 +47,8 @@ let Namespace = "AArch64" in {
   def qsub1 : SubRegIndex<128>;
   def qsub2 : SubRegIndex<128>;
   def qsub3 : SubRegIndex<128>;
-  // Note: Code depends on these having consecutive numbers
+
+  // SubRegIndexes for SME Matrix tiles
   def zasubb  : SubRegIndex<2048>; // (16 x 16)/1 bytes  = 2048 bits
   def zasubh0 : SubRegIndex<1024>; // (16 x 16)/2 bytes  = 1024 bits
   def zasubh1 : SubRegIndex<1024>; // (16 x 16)/2 bytes  = 1024 bits
@@ -52,7 +59,11 @@ let Namespace = "AArch64" in {
   def zasubq0 : SubRegIndex<128>;  // (16 x 16)/16 bytes = 128 bits
   def zasubq1 : SubRegIndex<128>;  // (16 x 16)/16 bytes = 128 bits
 
-  def psub : SubRegIndex<16>;
+  // SubRegIndexes for SVE Predicates
+  def psub  : SubRegIndex<16>;
+  // Note: Code depends on these having consecutive numbers
+  def psub0 : SubRegIndex<16, -1>;
+  def psub1 : SubRegIndex<16, -1>;
 }
 
 let Namespace = "AArch64" in {
@@ -1026,11 +1037,6 @@ def PNR16_p8to15  : PNRP8to15RegOp<"h", PNRAsmOp16_p8to15, 16, PNR_p8to15>;
 def PNR32_p8to15  : PNRP8to15RegOp<"s", PNRAsmOp32_p8to15, 32, PNR_p8to15>;
 def PNR64_p8to15  : PNRP8to15RegOp<"d", PNRAsmOp64_p8to15, 64, PNR_p8to15>;
 
-let Namespace = "AArch64" in {
-  def psub0 : SubRegIndex<16, -1>;
-  def psub1 : SubRegIndex<16, -1>;
-}
-
 class PPRorPNRClass : RegisterClass<
                                   "AArch64",
                                   [ nxv16i1, nxv8i1, nxv4i1, nxv2i1, nxv1i1, aarch64svcount ], 16,
@@ -1123,8 +1129,7 @@ let EncoderMethod = "EncodeRegMul_MinMax<2, 0, 14>",
 }  // end let EncoderMethod/DecoderMethod
 
 
-//******************************************************************************
-
+//===----------------------------------------------------------------------===//
 // SVE vector register classes
 class ZPRClass<int firstreg, int lastreg, int step = 1> : RegisterClass<"AArch64",
                                             [nxv16i8, nxv8i16, nxv4i32, nxv2i64,
@@ -1245,13 +1250,6 @@ def FPR32asZPR  : FPRasZPROperand<32>;
 def FPR64asZPR  : FPRasZPROperand<64>;
 def FPR128asZPR : FPRasZPROperand<128>;
 
-let Namespace = "AArch64" in {
-  def zsub0 : SubRegIndex<128, -1>;
-  def zsub1 : SubRegIndex<128, -1>;
-  def zsub2 : SubRegIndex<128, -1>;
-  def zsub3 : SubRegIndex<128, -1>;
-}
-
 // Pairs, triples, and quads of SVE vector registers.
 def ZSeqPairs   : RegisterTuples<[zsub0, zsub1], [(rotl ZPR, 0), (rotl ZPR, 1)]>;
 def ZSeqTriples : RegisterTuples<[zsub0, zsub1, zsub2], [(rotl ZPR, 0), (rotl ZPR, 1), (rotl ZPR, 2)]>;
-- 
GitLab


From 6bf4476ffb6bab661d59dee361ab845b2f68d9b1 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Wed, 30 Oct 2024 16:18:29 +0000
Subject: [PATCH 602/693] [AMDGPU] Fix @llvm.amdgcn.cs.chain with callee not
 provably uniform (#114200)

The correct behavior is to insert a readfirstlane. This worked except
for an inappropriate assertion in SITargetLowering::LowerCall.
---
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  3 -
 .../isel-amdgcn-cs-chain-intrinsic-w32.ll     | 84 ++++++++++++++++++-
 .../isel-amdgcn-cs-chain-intrinsic-w64.ll     | 83 +++++++++++++++++-
 3 files changed, 165 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 059b415b75ff..bddb6e822b81 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -3897,9 +3897,6 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
     Ops.push_back(DAG.getTargetGlobalAddress(GV, DL, MVT::i64));
   } else {
     if (IsTailCall) {
-      assert(!Callee->isDivergent() &&
-             "cannot tail call a divergent call target");
-
       // isEligibleForTailCallOptimization considered whether the call target is
       // divergent, but we may still end up with a uniform value in a VGPR.
       // Insert a readfirstlane just in case.
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll
index 75616d276754..c202476d85ba 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll
@@ -6,7 +6,6 @@
 
 declare amdgpu_cs_chain void @callee(<3 x i32> inreg, { i32, ptr addrspace(5), i32, i32 })
 declare amdgpu_cs_chain_preserve void @callee_preserve(<3 x i32> inreg, { i32, ptr addrspace(5), i32, i32 })
-declare void @llvm.amdgcn.cs.chain(ptr, i32, <3 x i32>, { i32, ptr addrspace(5), i32, i32 }, i32, ...) noreturn
 
 define amdgpu_cs_chain void @chain_to_chain(<3 x i32> inreg %sgpr, { i32, ptr addrspace(5), i32, i32 } %vgpr) {
   ; GISEL-GFX11-LABEL: name: chain_to_chain
@@ -660,6 +659,89 @@ define amdgpu_cs_chain void @indirect(ptr inreg %callee, <3 x i32> inreg %sgpr,
   unreachable
 }
 
+; Indirect with callee that we cannot prove is uniform.
+define amdgpu_cs_chain void @nonuniform_callee(ptr %callee, i32 inreg %sgpr, i32 %vgpr) {
+  ; GISEL-GFX11-LABEL: name: nonuniform_callee
+  ; GISEL-GFX11: bb.1 (%ir-block.0):
+  ; GISEL-GFX11-NEXT:   liveins: $sgpr0, $vgpr8, $vgpr9, $vgpr10
+  ; GISEL-GFX11-NEXT: {{  $}}
+  ; GISEL-GFX11-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; GISEL-GFX11-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; GISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GISEL-GFX11-NEXT:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX11-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GISEL-GFX11-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY3]]
+  ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY4]]
+  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY2]], 0, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $vgpr8
+  ;
+  ; GISEL-GFX10-LABEL: name: nonuniform_callee
+  ; GISEL-GFX10: bb.1 (%ir-block.0):
+  ; GISEL-GFX10-NEXT:   liveins: $sgpr0, $vgpr8, $vgpr9, $vgpr10
+  ; GISEL-GFX10-NEXT: {{  $}}
+  ; GISEL-GFX10-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; GISEL-GFX10-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; GISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GISEL-GFX10-NEXT:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX10-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GISEL-GFX10-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY3]]
+  ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY4]]
+  ; GISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY5]]
+  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY2]], 0, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $vgpr8, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+  ;
+  ; DAGISEL-GFX11-LABEL: name: nonuniform_callee
+  ; DAGISEL-GFX11: bb.0 (%ir-block.0):
+  ; DAGISEL-GFX11-NEXT:   liveins: $vgpr8, $vgpr9, $sgpr0, $vgpr10
+  ; DAGISEL-GFX11-NEXT: {{  $}}
+  ; DAGISEL-GFX11-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+  ; DAGISEL-GFX11-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; DAGISEL-GFX11-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; DAGISEL-GFX11-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; DAGISEL-GFX11-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; DAGISEL-GFX11-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY4]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY5]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_1]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_2]]
+  ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY]]
+  ; DAGISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 killed [[REG_SEQUENCE1]], 0, 0, killed [[S_MOV_B32_]], amdgpu_allvgprs, implicit $sgpr0, implicit $vgpr8
+  ;
+  ; DAGISEL-GFX10-LABEL: name: nonuniform_callee
+  ; DAGISEL-GFX10: bb.0 (%ir-block.0):
+  ; DAGISEL-GFX10-NEXT:   liveins: $vgpr8, $vgpr9, $sgpr0, $vgpr10
+  ; DAGISEL-GFX10-NEXT: {{  $}}
+  ; DAGISEL-GFX10-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+  ; DAGISEL-GFX10-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; DAGISEL-GFX10-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; DAGISEL-GFX10-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; DAGISEL-GFX10-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; DAGISEL-GFX10-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY4]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY5]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_1]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_]], %subreg.sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_2]]
+  ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY]]
+  ; DAGISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 killed [[REG_SEQUENCE1]], 0, 0, killed [[S_MOV_B32_]], amdgpu_allvgprs, implicit $sgpr48_sgpr49_sgpr50_sgpr51, implicit $sgpr0, implicit $vgpr8
+  call void(ptr, i32, i32, i32, i32, ...) @llvm.amdgcn.cs.chain(ptr %callee, i32 -1, i32 inreg %sgpr, i32 %vgpr, i32 0)
+  unreachable
+}
+
 define amdgpu_cs_chain void @non_imm_exec(i32 inreg %exec, <3 x i32> inreg %sgpr, { i32, ptr addrspace(5), i32, i32 } %vgpr) {
   ; GISEL-GFX11-LABEL: name: non_imm_exec
   ; GISEL-GFX11: bb.1 (%ir-block.0):
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll
index 6deac9f55f32..a456f549174c 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll
@@ -6,7 +6,6 @@
 
 declare amdgpu_cs_chain void @callee(<3 x i32> inreg, { i32, ptr addrspace(5), i32, i32 })
 declare amdgpu_cs_chain_preserve void @callee_preserve(<3 x i32> inreg, { i32, ptr addrspace(5), i32, i32 })
-declare void @llvm.amdgcn.cs.chain(ptr, i64, <3 x i32>, { i32, ptr addrspace(5), i32, i32 }, i32, ...) noreturn
 
 define amdgpu_cs_chain void @chain_to_chain(<3 x i32> inreg %sgpr, { i32, ptr addrspace(5), i32, i32 } %vgpr) {
   ; GISEL-GFX11-LABEL: name: chain_to_chain
@@ -660,6 +659,88 @@ define amdgpu_cs_chain void @indirect(ptr inreg %callee, <3 x i32> inreg %sgpr,
   unreachable
 }
 
+; Indirect with callee that we cannot prove is uniform.
+define amdgpu_cs_chain void @nonuniform_callee(ptr %callee, i32 inreg %sgpr, i32 %vgpr) {
+  ; GISEL-GFX11-LABEL: name: nonuniform_callee
+  ; GISEL-GFX11: bb.1 (%ir-block.0):
+  ; GISEL-GFX11-NEXT:   liveins: $sgpr0, $vgpr8, $vgpr9, $vgpr10
+  ; GISEL-GFX11-NEXT: {{  $}}
+  ; GISEL-GFX11-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; GISEL-GFX11-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; GISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GISEL-GFX11-NEXT:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX11-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GISEL-GFX11-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY3]]
+  ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY4]]
+  ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY2]], 0, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $vgpr8
+  ;
+  ; GISEL-GFX10-LABEL: name: nonuniform_callee
+  ; GISEL-GFX10: bb.1 (%ir-block.0):
+  ; GISEL-GFX10-NEXT:   liveins: $sgpr0, $vgpr8, $vgpr9, $vgpr10
+  ; GISEL-GFX10-NEXT: {{  $}}
+  ; GISEL-GFX10-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; GISEL-GFX10-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; GISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GISEL-GFX10-NEXT:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
+  ; GISEL-GFX10-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GISEL-GFX10-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY3]]
+  ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY4]]
+  ; GISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY5]]
+  ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY2]], 0, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $vgpr8, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+  ;
+  ; DAGISEL-GFX11-LABEL: name: nonuniform_callee
+  ; DAGISEL-GFX11: bb.0 (%ir-block.0):
+  ; DAGISEL-GFX11-NEXT:   liveins: $vgpr8, $vgpr9, $sgpr0, $vgpr10
+  ; DAGISEL-GFX11-NEXT: {{  $}}
+  ; DAGISEL-GFX11-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+  ; DAGISEL-GFX11-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; DAGISEL-GFX11-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; DAGISEL-GFX11-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; DAGISEL-GFX11-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; DAGISEL-GFX11-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY4]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY5]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_1]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_2]]
+  ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY]]
+  ; DAGISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 killed [[REG_SEQUENCE1]], 0, 0, killed [[S_MOV_B64_]], amdgpu_allvgprs, implicit $sgpr0, implicit $vgpr8
+  ;
+  ; DAGISEL-GFX10-LABEL: name: nonuniform_callee
+  ; DAGISEL-GFX10: bb.0 (%ir-block.0):
+  ; DAGISEL-GFX10-NEXT:   liveins: $vgpr8, $vgpr9, $sgpr0, $vgpr10
+  ; DAGISEL-GFX10-NEXT: {{  $}}
+  ; DAGISEL-GFX10-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+  ; DAGISEL-GFX10-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; DAGISEL-GFX10-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; DAGISEL-GFX10-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; DAGISEL-GFX10-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; DAGISEL-GFX10-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY4]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY5]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_1]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_]], %subreg.sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; DAGISEL-GFX10-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_2]]
+  ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY]]
+  ; DAGISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 killed [[REG_SEQUENCE1]], 0, 0, killed [[S_MOV_B64_]], amdgpu_allvgprs, implicit $sgpr48_sgpr49_sgpr50_sgpr51, implicit $sgpr0, implicit $vgpr8
+  call void(ptr, i64, i32, i32, i32, ...) @llvm.amdgcn.cs.chain(ptr %callee, i64 -1, i32 inreg %sgpr, i32 %vgpr, i32 0)
+  unreachable
+}
 define amdgpu_cs_chain void @non_imm_exec(i64 inreg %exec, <3 x i32> inreg %sgpr, { i32, ptr addrspace(5), i32, i32 } %vgpr) {
   ; GISEL-GFX11-LABEL: name: non_imm_exec
   ; GISEL-GFX11: bb.1 (%ir-block.0):
-- 
GitLab


From a575e6e5ca1eb7b2ae4b906f9bf3be2ba20a80a0 Mon Sep 17 00:00:00 2001
From: jimingham <jingham@apple.com>
Date: Wed, 30 Oct 2024 09:25:47 -0700
Subject: [PATCH 603/693] Fix a couple of tests that were incorrectly using
 configuration.dwarf_version (#114161)

The tests were using the variable directly to get the dwarf version used
for the test. That's only the overridden value, and won't be set if
we're using the compiler default. I also put a comment by the variable
to make sure people don't make the same mistake in the future.
---
 lldb/packages/Python/lldbsuite/test/configuration.py    | 4 ++++
 lldb/test/API/lang/cpp/namespace/TestNamespaceLookup.py | 7 +++++--
 lldb/test/API/python_api/type/TestTypeList.py           | 4 ++--
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/lldb/packages/Python/lldbsuite/test/configuration.py b/lldb/packages/Python/lldbsuite/test/configuration.py
index 1bacd74a968c..bcc179346836 100644
--- a/lldb/packages/Python/lldbsuite/test/configuration.py
+++ b/lldb/packages/Python/lldbsuite/test/configuration.py
@@ -46,6 +46,10 @@ sdkroot = None
 make_path = None
 
 # The overriden dwarf verison.
+# Don't use this to test the current compiler's
+# DWARF version, as this won't be set if the
+# version isn't overridden.
+# Use lldbplatformutils.getDwarfVersion() instead.
 dwarf_version = 0
 
 # Any overridden settings.
diff --git a/lldb/test/API/lang/cpp/namespace/TestNamespaceLookup.py b/lldb/test/API/lang/cpp/namespace/TestNamespaceLookup.py
index b5e8115160d2..41141164769e 100644
--- a/lldb/test/API/lang/cpp/namespace/TestNamespaceLookup.py
+++ b/lldb/test/API/lang/cpp/namespace/TestNamespaceLookup.py
@@ -8,7 +8,7 @@ import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-
+from lldbsuite.test import lldbplatformutil
 
 class NamespaceLookupTestCase(TestBase):
     def setUp(self):
@@ -167,7 +167,10 @@ class NamespaceLookupTestCase(TestBase):
         self.runToBkpt("continue")
         # FIXME: In DWARF 5 with dsyms, the ordering of functions is slightly
         # different, which also hits the same issues mentioned previously.
-        if configuration.dwarf_version <= 4 or self.getDebugInfo() == "dwarf":
+        if (
+            int(lldbplatformutil.getDwarfVersion()) <= 4
+            or self.getDebugInfo() == "dwarf"
+        ):
             self.expect_expr("func()", result_type="int", result_value="2")
 
         # Continue to BP_ns_scope at ns scope
diff --git a/lldb/test/API/python_api/type/TestTypeList.py b/lldb/test/API/python_api/type/TestTypeList.py
index bc4d00c17c55..09879276b44a 100644
--- a/lldb/test/API/python_api/type/TestTypeList.py
+++ b/lldb/test/API/python_api/type/TestTypeList.py
@@ -6,7 +6,7 @@ import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-
+from lldbsuite.test import lldbplatformutil
 
 class TypeAndTypeListTestCase(TestBase):
     def setUp(self):
@@ -248,7 +248,7 @@ class TypeAndTypeListTestCase(TestBase):
         self.assertEqual(myint_arr_element_type, myint_type)
 
         # Test enum methods. Requires DW_AT_enum_class which was added in Dwarf 4.
-        if configuration.dwarf_version >= 4:
+        if int(lldbplatformutil.getDwarfVersion()) >= 4:
             enum_type = target.FindFirstType("EnumType")
             self.assertTrue(enum_type)
             self.DebugSBType(enum_type)
-- 
GitLab


From 9cd30b1ef311edb0aa0527bead52e2fc490160ef Mon Sep 17 00:00:00 2001
From: jimingham <jingham@apple.com>
Date: Wed, 30 Oct 2024 09:26:37 -0700
Subject: [PATCH 604/693] Fix the sort function for languages to have "strict
 weak ordering". (#114160)

If you build libstdc++ with "debug" strictness, the test
TestTypeLookup.py will assert. That's because we're calling llvm::sort
(which redirects to std::sort) with a function that doesn't obey strict
weak ordering.

The error was that when the two languages were equal, we're sometimes
returning `true` but strict weak ordering requires that always be false.

This patch just makes the function behave properly.
---
 lldb/source/Commands/CommandObjectType.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lldb/source/Commands/CommandObjectType.cpp b/lldb/source/Commands/CommandObjectType.cpp
index f9786529bcdb..e4c6e374446e 100644
--- a/lldb/source/Commands/CommandObjectType.cpp
+++ b/lldb/source/Commands/CommandObjectType.cpp
@@ -2649,6 +2649,8 @@ public:
                 return false;
               LanguageType lt1 = lang1->GetLanguageType();
               LanguageType lt2 = lang2->GetLanguageType();
+              if (lt1 == lt2)
+                return false;
               if (lt1 == guessed_language)
                 return true; // make the selected frame's language come first
               if (lt2 == guessed_language)
-- 
GitLab


From 7dbbd2b251412b7b0809aabe672f3f57f0805dbb Mon Sep 17 00:00:00 2001
From: jimingham <jingham@apple.com>
Date: Wed, 30 Oct 2024 09:28:38 -0700
Subject: [PATCH 605/693] Fix call site breakpoint patch (#114158)

This fixes the two test suite failures that I missed in the PR:

https://github.com/llvm/llvm-project/pull/112939

One was a poorly written test case - it assumed that on connect to a
gdb-remote with a running process, lldb MUST have fetched all the frame
0 registers. In fact, there's no need for it to do so (as the CallSite
patch showed...) and if we don't need to we shouldn't. So I fixed the
test to only expect a `g` packet AFTER calling read_registers.

The other was a place where some code had used 0 when it meant
LLDB_INVALID_LINE_NUMBER, which I had fixed but missed one place where
it was still compared to 0.
---
 .../lldb/Breakpoint/BreakpointLocation.h      |  36 ++++
 lldb/include/lldb/Breakpoint/BreakpointSite.h |   5 +
 lldb/include/lldb/Core/Declaration.h          |   6 +-
 lldb/include/lldb/Target/StopInfo.h           |  12 ++
 .../lldb/Target/ThreadPlanStepInRange.h       |   4 +-
 lldb/source/Breakpoint/BreakpointLocation.cpp |  63 ++++++-
 lldb/source/Breakpoint/BreakpointResolver.cpp |  15 ++
 lldb/source/Breakpoint/BreakpointSite.cpp     |  17 ++
 lldb/source/Core/Declaration.cpp              |   5 +-
 lldb/source/Symbol/Block.cpp                  |   2 +-
 lldb/source/Symbol/CompileUnit.cpp            | 113 +++++++++++-
 lldb/source/Target/StackFrameList.cpp         | 171 ++++++------------
 lldb/source/Target/StopInfo.cpp               |  55 ++++++
 lldb/source/Target/Thread.cpp                 |   8 +
 lldb/source/Target/ThreadPlanStepInRange.cpp  |  24 ++-
 .../source/Target/ThreadPlanStepOverRange.cpp |   2 +-
 .../gdb_remote_client/TestGDBRemoteClient.py  |  35 +++-
 .../inline-stepping/TestInlineStepping.py     |  63 +++++++
 .../inline-stepping/calling.cpp               |  25 +++
 19 files changed, 525 insertions(+), 136 deletions(-)

diff --git a/lldb/include/lldb/Breakpoint/BreakpointLocation.h b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
index cca00335bc3c..3592291bb2d0 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointLocation.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
@@ -11,10 +11,12 @@
 
 #include <memory>
 #include <mutex>
+#include <optional>
 
 #include "lldb/Breakpoint/BreakpointOptions.h"
 #include "lldb/Breakpoint/StoppointHitCounter.h"
 #include "lldb/Core/Address.h"
+#include "lldb/Symbol/LineEntry.h"
 #include "lldb/Utility/UserID.h"
 #include "lldb/lldb-private.h"
 
@@ -282,6 +284,25 @@ public:
   /// Returns the breakpoint location ID.
   lldb::break_id_t GetID() const { return m_loc_id; }
 
+  /// Set the line entry that should be shown to users for this location.
+  /// It is up to the caller to verify that this is a valid entry to show.
+  /// The current use of this is to distinguish among line entries from a
+  /// virtual inlined call stack that all share the same address.
+  /// The line entry must have the same start address as the address for this
+  /// location.
+  bool SetPreferredLineEntry(const LineEntry &line_entry) {
+    if (m_address == line_entry.range.GetBaseAddress()) {
+      m_preferred_line_entry = line_entry;
+      return true;
+    }
+    assert(0 && "Tried to set a preferred line entry with a different address");
+    return false;
+  }
+
+  const std::optional<LineEntry> GetPreferredLineEntry() {
+    return m_preferred_line_entry;
+  }
+
 protected:
   friend class BreakpointSite;
   friend class BreakpointLocationList;
@@ -306,6 +327,16 @@ protected:
   /// If it returns false we should continue, otherwise stop.
   bool IgnoreCountShouldStop();
 
+  /// If this location knows that the virtual stack frame it represents is
+  /// not frame 0, return the suggested stack frame instead.  This will happen
+  /// when the location's address contains a "virtual inlined call stack" and
+  /// the breakpoint was set on a file & line that are not at the bottom of that
+  /// stack.  For now we key off the "preferred line entry" - looking for that
+  /// in the blocks that start with the stop PC.
+  /// This version of the API doesn't take an "inlined" parameter because it
+  /// only changes frames in the inline stack.
+  std::optional<uint32_t> GetSuggestedStackFrameIndex();
+
 private:
   void SwapLocation(lldb::BreakpointLocationSP swap_from);
 
@@ -369,6 +400,11 @@ private:
   lldb::break_id_t m_loc_id; ///< Breakpoint location ID.
   StoppointHitCounter m_hit_counter; ///< Number of times this breakpoint
                                      /// location has been hit.
+  /// If this exists, use it to print the stop description rather than the
+  /// LineEntry m_address resolves to directly.  Use this for instance when the
+  /// location was given somewhere in the virtual inlined call stack since the
+  /// Address always resolves to the lowest entry in the stack.
+  std::optional<LineEntry> m_preferred_line_entry;
 
   void SetShouldResolveIndirectFunctions(bool do_resolve) {
     m_should_resolve_indirect_functions = do_resolve;
diff --git a/lldb/include/lldb/Breakpoint/BreakpointSite.h b/lldb/include/lldb/Breakpoint/BreakpointSite.h
index 17b76d51c1ae..7b3f7be23639 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointSite.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointSite.h
@@ -170,6 +170,11 @@ public:
   /// \see lldb::DescriptionLevel
   void GetDescription(Stream *s, lldb::DescriptionLevel level);
 
+  // This runs through all the breakpoint locations owning this site and returns
+  // the greatest of their suggested stack frame indexes.  This only handles
+  // inlined stack changes.
+  std::optional<uint32_t> GetSuggestedStackFrameIndex();
+
   /// Tell whether a breakpoint has a location at this site.
   ///
   /// \param[in] bp_id
diff --git a/lldb/include/lldb/Core/Declaration.h b/lldb/include/lldb/Core/Declaration.h
index 4a0e9047b546..c864b88c6b32 100644
--- a/lldb/include/lldb/Core/Declaration.h
+++ b/lldb/include/lldb/Core/Declaration.h
@@ -84,10 +84,14 @@ public:
   /// \param[in] declaration
   ///     The const Declaration object to compare with.
   ///
+  /// \param[in] full
+  ///     Same meaning as Full in FileSpec::Equal.  True means an empty
+  ///     directory is not equal to a specified one, false means it is equal.
+  ///
   /// \return
   ///     Returns \b true if \b declaration is at the same file and
   ///     line, \b false otherwise.
-  bool FileAndLineEqual(const Declaration &declaration) const;
+  bool FileAndLineEqual(const Declaration &declaration, bool full) const;
 
   /// Dump a description of this object to a Stream.
   ///
diff --git a/lldb/include/lldb/Target/StopInfo.h b/lldb/include/lldb/Target/StopInfo.h
index fae90364deaf..45beac129e86 100644
--- a/lldb/include/lldb/Target/StopInfo.h
+++ b/lldb/include/lldb/Target/StopInfo.h
@@ -77,6 +77,18 @@ public:
       m_description.clear();
   }
 
+  /// This gives the StopInfo a chance to suggest a stack frame to select.
+  /// Passing true for inlined_stack will request changes to the inlined
+  /// call stack.  Passing false will request changes to the real stack
+  /// frame.  The inlined stack gets adjusted before we call into the thread
+  /// plans so they can reason based on the correct values.  The real stack
+  /// adjustment is handled after the frame recognizers get a chance to adjust
+  /// the frame.
+  virtual std::optional<uint32_t>
+  GetSuggestedStackFrameIndex(bool inlined_stack) {
+    return {};
+  }
+
   virtual bool IsValidForOperatingSystemThread(Thread &thread) { return true; }
 
   /// A Continue operation can result in a false stop event
diff --git a/lldb/include/lldb/Target/ThreadPlanStepInRange.h b/lldb/include/lldb/Target/ThreadPlanStepInRange.h
index f9ef87942a7c..9da8370ef1c9 100644
--- a/lldb/include/lldb/Target/ThreadPlanStepInRange.h
+++ b/lldb/include/lldb/Target/ThreadPlanStepInRange.h
@@ -80,8 +80,8 @@ private:
   bool m_step_past_prologue; // FIXME: For now hard-coded to true, we could put
                              // a switch in for this if there's
                              // demand for that.
-  bool m_virtual_step; // true if we've just done a "virtual step", i.e. just
-                       // moved the inline stack depth.
+  LazyBool m_virtual_step;   // true if we've just done a "virtual step", i.e.
+                             // just moved the inline stack depth.
   ConstString m_step_into_target;
   ThreadPlanStepInRange(const ThreadPlanStepInRange &) = delete;
   const ThreadPlanStepInRange &
diff --git a/lldb/source/Breakpoint/BreakpointLocation.cpp b/lldb/source/Breakpoint/BreakpointLocation.cpp
index ad9057c8141e..c7ea50407ae1 100644
--- a/lldb/source/Breakpoint/BreakpointLocation.cpp
+++ b/lldb/source/Breakpoint/BreakpointLocation.cpp
@@ -508,8 +508,20 @@ void BreakpointLocation::GetDescription(Stream *s,
         s->PutCString("re-exported target = ");
       else
         s->PutCString("where = ");
+
+      // If there's a preferred line entry for printing, use that.
+      bool show_function_info = true;
+      if (auto preferred = GetPreferredLineEntry()) {
+        sc.line_entry = *preferred;
+        // FIXME: We're going to get the function name wrong when the preferred
+        // line entry is not the lowest one.  For now, just leave the function
+        // out in this case, but we really should also figure out how to easily
+        // fake the function name here.
+        show_function_info = false;
+      }
       sc.DumpStopContext(s, m_owner.GetTarget().GetProcessSP().get(), m_address,
-                         false, true, false, true, true, true);
+                         false, true, false, show_function_info,
+                         show_function_info, show_function_info);
     } else {
       if (sc.module_sp) {
         s->EOL();
@@ -537,7 +549,10 @@ void BreakpointLocation::GetDescription(Stream *s,
         if (sc.line_entry.line > 0) {
           s->EOL();
           s->Indent("location = ");
-          sc.line_entry.DumpStopContext(s, true);
+          if (auto preferred = GetPreferredLineEntry())
+            preferred->DumpStopContext(s, true);
+          else
+            sc.line_entry.DumpStopContext(s, true);
         }
 
       } else {
@@ -656,6 +671,50 @@ void BreakpointLocation::SendBreakpointLocationChangedEvent(
   }
 }
 
+std::optional<uint32_t> BreakpointLocation::GetSuggestedStackFrameIndex() {
+  auto preferred_opt = GetPreferredLineEntry();
+  if (!preferred_opt)
+    return {};
+  LineEntry preferred = *preferred_opt;
+  SymbolContext sc;
+  if (!m_address.CalculateSymbolContext(&sc))
+    return {};
+  // Don't return anything special if frame 0 is the preferred line entry.
+  // We not really telling the stack frame list to do anything special in that
+  // case.
+  if (!LineEntry::Compare(sc.line_entry, preferred))
+    return {};
+
+  if (!sc.block)
+    return {};
+
+  // Blocks have their line info in Declaration form, so make one here:
+  Declaration preferred_decl(preferred.GetFile(), preferred.line,
+                             preferred.column);
+
+  uint32_t depth = 0;
+  Block *inlined_block = sc.block->GetContainingInlinedBlock();
+  while (inlined_block) {
+    // If we've moved to a block that this isn't the start of, that's not
+    // our inlining info or call site, so we can stop here.
+    Address start_address;
+    if (!inlined_block->GetStartAddress(start_address) ||
+        start_address != m_address)
+      return {};
+
+    const InlineFunctionInfo *info = inlined_block->GetInlinedFunctionInfo();
+    if (info) {
+      if (preferred_decl == info->GetDeclaration())
+        return depth;
+      if (preferred_decl == info->GetCallSite())
+        return depth + 1;
+    }
+    inlined_block = inlined_block->GetInlinedParent();
+    depth++;
+  }
+  return {};
+}
+
 void BreakpointLocation::SwapLocation(BreakpointLocationSP swap_from) {
   m_address = swap_from->m_address;
   m_should_resolve_indirect_functions =
diff --git a/lldb/source/Breakpoint/BreakpointResolver.cpp b/lldb/source/Breakpoint/BreakpointResolver.cpp
index 8307689c7640..9643602d78c7 100644
--- a/lldb/source/Breakpoint/BreakpointResolver.cpp
+++ b/lldb/source/Breakpoint/BreakpointResolver.cpp
@@ -340,6 +340,21 @@ void BreakpointResolver::AddLocation(SearchFilter &filter,
   }
 
   BreakpointLocationSP bp_loc_sp(AddLocation(line_start));
+  // If the address that we resolved the location to returns a different
+  // LineEntry from the one in the incoming SC, we're probably dealing with an
+  // inlined call site, so set that as the preferred LineEntry:
+  LineEntry resolved_entry;
+  if (!skipped_prologue && bp_loc_sp &&
+      line_start.CalculateSymbolContextLineEntry(resolved_entry) &&
+      LineEntry::Compare(resolved_entry, sc.line_entry)) {
+    // FIXME: The function name will also be wrong here.  Do we need to record
+    // that as well, or can we figure that out again when we report this
+    // breakpoint location.
+    if (!bp_loc_sp->SetPreferredLineEntry(sc.line_entry)) {
+      LLDB_LOG(log, "Tried to add a preferred line entry that didn't have the "
+                    "same address as this location's address.");
+    }
+  }
   if (log && bp_loc_sp && !GetBreakpoint()->IsInternal()) {
     StreamString s;
     bp_loc_sp->GetDescription(&s, lldb::eDescriptionLevelVerbose);
diff --git a/lldb/source/Breakpoint/BreakpointSite.cpp b/lldb/source/Breakpoint/BreakpointSite.cpp
index 3ca93f908e30..9700a57d3346 100644
--- a/lldb/source/Breakpoint/BreakpointSite.cpp
+++ b/lldb/source/Breakpoint/BreakpointSite.cpp
@@ -87,6 +87,23 @@ void BreakpointSite::GetDescription(Stream *s, lldb::DescriptionLevel level) {
   m_constituents.GetDescription(s, level);
 }
 
+std::optional<uint32_t> BreakpointSite::GetSuggestedStackFrameIndex() {
+
+  std::optional<uint32_t> result;
+  std::lock_guard<std::recursive_mutex> guard(m_constituents_mutex);
+  for (BreakpointLocationSP loc_sp : m_constituents.BreakpointLocations()) {
+    std::optional<uint32_t> loc_frame_index =
+        loc_sp->GetSuggestedStackFrameIndex();
+    if (loc_frame_index) {
+      if (result)
+        result = std::max(*loc_frame_index, *result);
+      else
+        result = loc_frame_index;
+    }
+  }
+  return result;
+}
+
 bool BreakpointSite::IsInternal() const { return m_constituents.IsInternal(); }
 
 uint8_t *BreakpointSite::GetTrapOpcodeBytes() { return &m_trap_opcode[0]; }
diff --git a/lldb/source/Core/Declaration.cpp b/lldb/source/Core/Declaration.cpp
index 579a3999d14e..a485c4b9ba48 100644
--- a/lldb/source/Core/Declaration.cpp
+++ b/lldb/source/Core/Declaration.cpp
@@ -70,8 +70,9 @@ int Declaration::Compare(const Declaration &a, const Declaration &b) {
   return 0;
 }
 
-bool Declaration::FileAndLineEqual(const Declaration &declaration) const {
-  int file_compare = FileSpec::Compare(this->m_file, declaration.m_file, true);
+bool Declaration::FileAndLineEqual(const Declaration &declaration,
+                                   bool full) const {
+  int file_compare = FileSpec::Compare(this->m_file, declaration.m_file, full);
   return file_compare == 0 && this->m_line == declaration.m_line;
 }
 
diff --git a/lldb/source/Symbol/Block.cpp b/lldb/source/Symbol/Block.cpp
index f7d9c0d2d330..5c7772a6db78 100644
--- a/lldb/source/Symbol/Block.cpp
+++ b/lldb/source/Symbol/Block.cpp
@@ -230,7 +230,7 @@ Block *Block::GetContainingInlinedBlockWithCallSite(
     const auto *function_info = inlined_block->GetInlinedFunctionInfo();
 
     if (function_info &&
-        function_info->GetCallSite().FileAndLineEqual(find_call_site))
+        function_info->GetCallSite().FileAndLineEqual(find_call_site, true))
       return inlined_block;
     inlined_block = inlined_block->GetInlinedParent();
   }
diff --git a/lldb/source/Symbol/CompileUnit.cpp b/lldb/source/Symbol/CompileUnit.cpp
index db8f8ce6bcbc..73389b2e8479 100644
--- a/lldb/source/Symbol/CompileUnit.cpp
+++ b/lldb/source/Symbol/CompileUnit.cpp
@@ -251,7 +251,10 @@ void CompileUnit::ResolveSymbolContext(
     SymbolContextItem resolve_scope, SymbolContextList &sc_list,
     RealpathPrefixes *realpath_prefixes) {
   const FileSpec file_spec = src_location_spec.GetFileSpec();
-  const uint32_t line = src_location_spec.GetLine().value_or(0);
+  const uint32_t line =
+      src_location_spec.GetLine().value_or(LLDB_INVALID_LINE_NUMBER);
+  const uint32_t column_num =
+      src_location_spec.GetColumn().value_or(LLDB_INVALID_COLUMN_NUMBER);
   const bool check_inlines = src_location_spec.GetCheckInlines();
 
   // First find all of the file indexes that match our "file_spec". If
@@ -268,7 +271,7 @@ void CompileUnit::ResolveSymbolContext(
   SymbolContext sc(GetModule());
   sc.comp_unit = this;
 
-  if (line == 0) {
+  if (line == LLDB_INVALID_LINE_NUMBER) {
     if (file_spec_matches_cu_file_spec && !check_inlines) {
       // only append the context if we aren't looking for inline call sites by
       // file and line and if the file spec matches that of the compile unit
@@ -312,6 +315,112 @@ void CompileUnit::ResolveSymbolContext(
         0, file_indexes, src_location_spec, &line_entry);
   }
 
+  // If we didn't manage to find a breakpoint that matched the line number
+  // requested, that might be because it is only an inline call site, and
+  // doesn't have a line entry in the line table.  Scan for that here.
+  //
+  // We are making the assumption that if there was an inlined function it will
+  // contribute at least 1 non-call-site entry to the line table.  That's handy
+  // because we don't move line breakpoints over function boundaries, so if we
+  // found a hit, and there were also a call site entry, it would have to be in
+  // the function containing the PC of the line table match.  That way we can
+  // limit the call site search to that function.
+  // We will miss functions that ONLY exist as a call site entry.
+
+  if (line_entry.IsValid() &&
+      (line_entry.line != line || line_entry.column != column_num) &&
+      resolve_scope & eSymbolContextLineEntry && check_inlines) {
+    // We don't move lines over function boundaries, so the address in the
+    // line entry will be the in function that contained the line that might
+    // be a CallSite, and we can just iterate over that function to find any
+    // inline records, and dig up their call sites.
+    Address start_addr = line_entry.range.GetBaseAddress();
+    Function *function = start_addr.CalculateSymbolContextFunction();
+
+    Declaration sought_decl(file_spec, line, column_num);
+    // We use this recursive function to descend the block structure looking
+    // for a block that has this Declaration as in it's CallSite info.
+    // This function recursively scans the sibling blocks of the incoming
+    // block parameter.
+    std::function<void(Block &)> examine_block =
+        [&sought_decl, &sc_list, &src_location_spec, resolve_scope,
+         &examine_block](Block &block) -> void {
+      // Iterate over the sibling child blocks of the incoming block.
+      Block *sibling_block = block.GetFirstChild();
+      while (sibling_block) {
+        // We only have to descend through the regular blocks, looking for
+        // immediate inlines, since those are the only ones that will have this
+        // callsite.
+        const InlineFunctionInfo *inline_info =
+            sibling_block->GetInlinedFunctionInfo();
+        if (inline_info) {
+          // If this is the call-site we are looking for, record that:
+          // We need to be careful because the call site from the debug info
+          // will generally have a column, but the user might not have specified
+          // it.
+          Declaration found_decl = inline_info->GetCallSite();
+          uint32_t sought_column = sought_decl.GetColumn();
+          if (found_decl.FileAndLineEqual(sought_decl, false) &&
+              (sought_column == LLDB_INVALID_COLUMN_NUMBER ||
+               sought_column == found_decl.GetColumn())) {
+            // If we found a call site, it belongs not in this inlined block,
+            // but in the parent block that inlined it.
+            Address parent_start_addr;
+            if (sibling_block->GetParent()->GetStartAddress(
+                    parent_start_addr)) {
+              SymbolContext sc;
+              parent_start_addr.CalculateSymbolContext(&sc, resolve_scope);
+              // Now swap out the line entry for the one we found.
+              LineEntry call_site_line = sc.line_entry;
+              call_site_line.line = found_decl.GetLine();
+              call_site_line.column = found_decl.GetColumn();
+              bool matches_spec = true;
+              // If the user asked for an exact match, we need to make sure the
+              // call site we found actually matches the location.
+              if (src_location_spec.GetExactMatch()) {
+                matches_spec = false;
+                if ((src_location_spec.GetFileSpec() ==
+                     sc.line_entry.GetFile()) &&
+                    (src_location_spec.GetLine() &&
+                     *src_location_spec.GetLine() == call_site_line.line) &&
+                    (src_location_spec.GetColumn() &&
+                     *src_location_spec.GetColumn() == call_site_line.column))
+                  matches_spec = true;
+              }
+              if (matches_spec &&
+                  sibling_block->GetRangeAtIndex(0, call_site_line.range)) {
+                SymbolContext call_site_sc(sc.target_sp, sc.module_sp,
+                                           sc.comp_unit, sc.function, sc.block,
+                                           &call_site_line, sc.symbol);
+                sc_list.Append(call_site_sc);
+              }
+            }
+          }
+        }
+
+        // Descend into the child blocks:
+        examine_block(*sibling_block);
+        // Now go to the next sibling:
+        sibling_block = sibling_block->GetSibling();
+      }
+    };
+
+    if (function) {
+      // We don't need to examine the function block, it can't be inlined.
+      Block &func_block = function->GetBlock(true);
+      examine_block(func_block);
+    }
+    // If we found entries here, we are done.  We only get here because we
+    // didn't find an exact line entry for this line & column, but if we found
+    // an exact match from the call site info that's strictly better than
+    // continuing to look for matches further on in the file.
+    // FIXME: Should I also do this for "call site line exists between the
+    // given line number and the later line we found in the line table"?  That's
+    // a closer approximation to our general sliding algorithm.
+    if (sc_list.GetSize())
+      return;
+  }
+
   // If "exact == true", then "found_line" will be the same as "line". If
   // "exact == false", the "found_line" will be the closest line entry
   // with a line number greater than "line" and we will use this for our
diff --git a/lldb/source/Target/StackFrameList.cpp b/lldb/source/Target/StackFrameList.cpp
index 3849ec5ed178..94a381edd5e2 100644
--- a/lldb/source/Target/StackFrameList.cpp
+++ b/lldb/source/Target/StackFrameList.cpp
@@ -85,121 +85,32 @@ void StackFrameList::ResetCurrentInlinedDepth() {
     return;
 
   std::lock_guard<std::recursive_mutex> guard(m_mutex);
-  
-  GetFramesUpTo(0, DoNotAllowInterruption);
-  if (m_frames.empty())
-    return;
-  if (!m_frames[0]->IsInlined()) {
-    m_current_inlined_depth = UINT32_MAX;
-    m_current_inlined_pc = LLDB_INVALID_ADDRESS;
-    Log *log = GetLog(LLDBLog::Step);
-    if (log && log->GetVerbose())
-      LLDB_LOGF(
-          log,
-          "ResetCurrentInlinedDepth: Invalidating current inlined depth.\n");
-    return;
-  }
 
-  // We only need to do something special about inlined blocks when we are
-  // at the beginning of an inlined function:
-  // FIXME: We probably also have to do something special if the PC is at
-  // the END of an inlined function, which coincides with the end of either
-  // its containing function or another inlined function.
-
-  Block *block_ptr = m_frames[0]->GetFrameBlock();
-  if (!block_ptr)
-    return;
+  m_current_inlined_pc = LLDB_INVALID_ADDRESS;
+  m_current_inlined_depth = UINT32_MAX;
 
-  Address pc_as_address;
-  lldb::addr_t curr_pc = m_thread.GetRegisterContext()->GetPC();
-  pc_as_address.SetLoadAddress(curr_pc, &(m_thread.GetProcess()->GetTarget()));
-  AddressRange containing_range;
-  if (!block_ptr->GetRangeContainingAddress(pc_as_address, containing_range) ||
-      pc_as_address != containing_range.GetBaseAddress())
-    return;
-
-  // If we got here because of a breakpoint hit, then set the inlined depth
-  // depending on where the breakpoint was set. If we got here because of a
-  // crash, then set the inlined depth to the deepest most block.  Otherwise,
-  // we stopped here naturally as the result of a step, so set ourselves in the
-  // containing frame of the whole set of nested inlines, so the user can then
-  // "virtually" step into the frames one by one, or next over the whole mess.
-  // Note: We don't have to handle being somewhere in the middle of the stack
-  // here, since ResetCurrentInlinedDepth doesn't get called if there is a
-  // valid inlined depth set.
   StopInfoSP stop_info_sp = m_thread.GetStopInfo();
   if (!stop_info_sp)
     return;
-  switch (stop_info_sp->GetStopReason()) {
-  case eStopReasonWatchpoint:
-  case eStopReasonException:
-  case eStopReasonExec:
-  case eStopReasonFork:
-  case eStopReasonVFork:
-  case eStopReasonVForkDone:
-  case eStopReasonSignal:
-    // In all these cases we want to stop in the deepest frame.
-    m_current_inlined_pc = curr_pc;
-    m_current_inlined_depth = 0;
-    break;
-  case eStopReasonBreakpoint: {
-    // FIXME: Figure out what this break point is doing, and set the inline
-    // depth appropriately.  Be careful to take into account breakpoints that
-    // implement step over prologue, since that should do the default
-    // calculation. For now, if the breakpoints corresponding to this hit are
-    // all internal, I set the stop location to the top of the inlined stack,
-    // since that will make things like stepping over prologues work right.
-    // But if there are any non-internal breakpoints I do to the bottom of the
-    // stack, since that was the old behavior.
-    uint32_t bp_site_id = stop_info_sp->GetValue();
-    BreakpointSiteSP bp_site_sp(
-        m_thread.GetProcess()->GetBreakpointSiteList().FindByID(bp_site_id));
-    bool all_internal = true;
-    if (bp_site_sp) {
-      uint32_t num_owners = bp_site_sp->GetNumberOfConstituents();
-      for (uint32_t i = 0; i < num_owners; i++) {
-        Breakpoint &bp_ref =
-            bp_site_sp->GetConstituentAtIndex(i)->GetBreakpoint();
-        if (!bp_ref.IsInternal()) {
-          all_internal = false;
-        }
-      }
-    }
-    if (!all_internal) {
-      m_current_inlined_pc = curr_pc;
-      m_current_inlined_depth = 0;
-      break;
-    }
-  }
-    [[fallthrough]];
-  default: {
-    // Otherwise, we should set ourselves at the container of the inlining, so
-    // that the user can descend into them. So first we check whether we have
-    // more than one inlined block sharing this PC:
-    int num_inlined_functions = 0;
-
-    for (Block *container_ptr = block_ptr->GetInlinedParent();
-         container_ptr != nullptr;
-         container_ptr = container_ptr->GetInlinedParent()) {
-      if (!container_ptr->GetRangeContainingAddress(pc_as_address,
-                                                    containing_range))
-        break;
-      if (pc_as_address != containing_range.GetBaseAddress())
-        break;
 
-      num_inlined_functions++;
-    }
-    m_current_inlined_pc = curr_pc;
-    m_current_inlined_depth = num_inlined_functions + 1;
-    Log *log = GetLog(LLDBLog::Step);
+  bool inlined = true;
+  auto inline_depth = stop_info_sp->GetSuggestedStackFrameIndex(inlined);
+  // We're only adjusting the inlined stack here.
+  Log *log = GetLog(LLDBLog::Step);
+  if (inline_depth) {
+    m_current_inlined_depth = *inline_depth;
+    m_current_inlined_pc = m_thread.GetRegisterContext()->GetPC();
+
     if (log && log->GetVerbose())
       LLDB_LOGF(log,
                 "ResetCurrentInlinedDepth: setting inlined "
                 "depth: %d 0x%" PRIx64 ".\n",
-                m_current_inlined_depth, curr_pc);
-
-    break;
-  }
+                m_current_inlined_depth, m_current_inlined_pc);
+  } else {
+    if (log && log->GetVerbose())
+      LLDB_LOGF(
+          log,
+          "ResetCurrentInlinedDepth: Invalidating current inlined depth.\n");
   }
 }
 
@@ -816,19 +727,48 @@ void StackFrameList::SelectMostRelevantFrame() {
 
   RecognizedStackFrameSP recognized_frame_sp = frame_sp->GetRecognizedFrame();
 
-  if (!recognized_frame_sp) {
-    LLDB_LOG(log, "Frame #0 not recognized");
-    return;
+  if (recognized_frame_sp) {
+    if (StackFrameSP most_relevant_frame_sp =
+            recognized_frame_sp->GetMostRelevantFrame()) {
+      LLDB_LOG(log, "Found most relevant frame at index {0}",
+               most_relevant_frame_sp->GetFrameIndex());
+      SetSelectedFrame(most_relevant_frame_sp.get());
+      return;
+    }
   }
+  LLDB_LOG(log, "Frame #0 not recognized");
 
-  if (StackFrameSP most_relevant_frame_sp =
-          recognized_frame_sp->GetMostRelevantFrame()) {
-    LLDB_LOG(log, "Found most relevant frame at index {0}",
-             most_relevant_frame_sp->GetFrameIndex());
-    SetSelectedFrame(most_relevant_frame_sp.get());
-  } else {
-    LLDB_LOG(log, "No relevant frame!");
+  // If this thread has a non-trivial StopInof, then let it suggest
+  // a most relevant frame:
+  StopInfoSP stop_info_sp = m_thread.GetStopInfo();
+  uint32_t stack_idx = 0;
+  bool found_relevant = false;
+  if (stop_info_sp) {
+    // Here we're only asking the stop info if it wants to adjust the real stack
+    // index.  We have to ask about the m_inlined_stack_depth in
+    // Thread::ShouldStop since the plans need to reason with that info.
+    bool inlined = false;
+    std::optional<uint32_t> stack_opt =
+        stop_info_sp->GetSuggestedStackFrameIndex(inlined);
+    if (stack_opt) {
+      stack_idx = *stack_opt;
+      found_relevant = true;
+    }
   }
+
+  frame_sp = GetFrameAtIndex(stack_idx);
+  if (!frame_sp)
+    LLDB_LOG(log, "Stop info suggested relevant frame {0} but it didn't exist",
+             stack_idx);
+  else if (found_relevant)
+    LLDB_LOG(log, "Setting selected frame from stop info to {0}", stack_idx);
+  // Note, we don't have to worry about "inlined" frames here, because we've
+  // already calculated the inlined frame in Thread::ShouldStop, and
+  // SetSelectedFrame will take care of that adjustment for us.
+  SetSelectedFrame(frame_sp.get());
+
+  if (!found_relevant)
+    LLDB_LOG(log, "No relevant frame!");
 }
 
 uint32_t StackFrameList::GetSelectedFrameIndex(
@@ -841,6 +781,7 @@ uint32_t StackFrameList::GetSelectedFrameIndex(
     // isn't set, then don't force a selection here, just return 0.
     if (!select_most_relevant)
       return 0;
+    // If the inlined stack frame is set, then use that:
     m_selected_frame_idx = 0;
   }
   return *m_selected_frame_idx;
diff --git a/lldb/source/Target/StopInfo.cpp b/lldb/source/Target/StopInfo.cpp
index 60aa65ed38c7..f6387d47504e 100644
--- a/lldb/source/Target/StopInfo.cpp
+++ b/lldb/source/Target/StopInfo.cpp
@@ -15,6 +15,7 @@
 #include "lldb/Breakpoint/WatchpointResource.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Expression/UserExpression.h"
+#include "lldb/Symbol/Block.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/StopInfo.h"
 #include "lldb/Target/Target.h"
@@ -246,6 +247,22 @@ public:
     return m_description.c_str();
   }
 
+  std::optional<uint32_t>
+  GetSuggestedStackFrameIndex(bool inlined_stack) override {
+    if (!inlined_stack)
+      return {};
+
+    ThreadSP thread_sp(m_thread_wp.lock());
+    if (!thread_sp)
+      return {};
+    BreakpointSiteSP bp_site_sp(
+        thread_sp->GetProcess()->GetBreakpointSiteList().FindByID(m_value));
+    if (!bp_site_sp)
+      return {};
+
+    return bp_site_sp->GetSuggestedStackFrameIndex();
+  }
+
 protected:
   bool ShouldStop(Event *event_ptr) override {
     // This just reports the work done by PerformAction or the synchronous
@@ -1164,6 +1181,44 @@ public:
     else
       return m_description.c_str();
   }
+
+  std::optional<uint32_t>
+  GetSuggestedStackFrameIndex(bool inlined_stack) override {
+    // Trace only knows how to adjust inlined stacks:
+    if (!inlined_stack)
+      return {};
+
+    ThreadSP thread_sp = GetThread();
+    StackFrameSP frame_0_sp = thread_sp->GetStackFrameAtIndex(0);
+    if (!frame_0_sp)
+      return {};
+    if (!frame_0_sp->IsInlined())
+      return {};
+    Block *block_ptr = frame_0_sp->GetFrameBlock();
+    if (!block_ptr)
+      return {};
+    Address pc_address = frame_0_sp->GetFrameCodeAddress();
+    AddressRange containing_range;
+    if (!block_ptr->GetRangeContainingAddress(pc_address, containing_range) ||
+        pc_address != containing_range.GetBaseAddress())
+      return {};
+
+    int num_inlined_functions = 0;
+
+    for (Block *container_ptr = block_ptr->GetInlinedParent();
+         container_ptr != nullptr;
+         container_ptr = container_ptr->GetInlinedParent()) {
+      if (!container_ptr->GetRangeContainingAddress(pc_address,
+                                                    containing_range))
+        break;
+      if (pc_address != containing_range.GetBaseAddress())
+        break;
+
+      num_inlined_functions++;
+    }
+    inlined_stack = true;
+    return num_inlined_functions + 1;
+  }
 };
 
 // StopInfoException
diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp
index 8373cdc36268..735295e6f259 100644
--- a/lldb/source/Target/Thread.cpp
+++ b/lldb/source/Target/Thread.cpp
@@ -619,6 +619,14 @@ void Thread::WillStop() {
 
 void Thread::SetupForResume() {
   if (GetResumeState() != eStateSuspended) {
+    // First check whether this thread is going to "actually" resume at all.
+    // For instance, if we're stepping from one level to the next of an
+    // virtual inlined call stack, we just change the inlined call stack index
+    // without actually running this thread.  In that case, for this thread we
+    // shouldn't push a step over breakpoint plan or do that work.
+    if (GetCurrentPlan()->IsVirtualStep())
+      return;
+
     // If we're at a breakpoint push the step-over breakpoint plan.  Do this
     // before telling the current plan it will resume, since we might change
     // what the current plan is.
diff --git a/lldb/source/Target/ThreadPlanStepInRange.cpp b/lldb/source/Target/ThreadPlanStepInRange.cpp
index 567dcc26d0d3..325a70619908 100644
--- a/lldb/source/Target/ThreadPlanStepInRange.cpp
+++ b/lldb/source/Target/ThreadPlanStepInRange.cpp
@@ -41,7 +41,7 @@ ThreadPlanStepInRange::ThreadPlanStepInRange(
                           "Step Range stepping in", thread, range, addr_context,
                           stop_others),
       ThreadPlanShouldStopHere(this), m_step_past_prologue(true),
-      m_virtual_step(false), m_step_into_target(step_into_target) {
+      m_virtual_step(eLazyBoolCalculate), m_step_into_target(step_into_target) {
   SetCallbacks();
   SetFlagsToDefault();
   SetupAvoidNoDebug(step_in_avoids_code_without_debug_info,
@@ -149,7 +149,7 @@ bool ThreadPlanStepInRange::ShouldStop(Event *event_ptr) {
       m_sub_plan_sp.reset();
   }
 
-  if (m_virtual_step) {
+  if (m_virtual_step == eLazyBoolYes) {
     // If we've just completed a virtual step, all we need to do is check for a
     // ShouldStopHere plan, and otherwise we're done.
     // FIXME - This can be both a step in and a step out.  Probably should
@@ -431,7 +431,7 @@ bool ThreadPlanStepInRange::DoPlanExplainsStop(Event *event_ptr) {
 
   bool return_value = false;
 
-  if (m_virtual_step) {
+  if (m_virtual_step == eLazyBoolYes) {
     return_value = true;
   } else {
     StopInfoSP stop_info_sp = GetPrivateStopInfo();
@@ -460,10 +460,13 @@ bool ThreadPlanStepInRange::DoPlanExplainsStop(Event *event_ptr) {
 
 bool ThreadPlanStepInRange::DoWillResume(lldb::StateType resume_state,
                                          bool current_plan) {
-  m_virtual_step = false;
+  m_virtual_step = eLazyBoolCalculate;
   if (resume_state == eStateStepping && current_plan) {
     Thread &thread = GetThread();
     // See if we are about to step over a virtual inlined call.
+    // But if we already know we're virtual stepping, don't decrement the
+    // inlined depth again...
+
     bool step_without_resume = thread.DecrementCurrentInlinedDepth();
     if (step_without_resume) {
       Log *log = GetLog(LLDBLog::Step);
@@ -476,11 +479,20 @@ bool ThreadPlanStepInRange::DoWillResume(lldb::StateType resume_state,
       // FIXME: Maybe it would be better to create a InlineStep stop reason, but
       // then
       // the whole rest of the world would have to handle that stop reason.
-      m_virtual_step = true;
+      m_virtual_step = eLazyBoolYes;
     }
     return !step_without_resume;
   }
   return true;
 }
 
-bool ThreadPlanStepInRange::IsVirtualStep() { return m_virtual_step; }
+bool ThreadPlanStepInRange::IsVirtualStep() {
+  if (m_virtual_step == eLazyBoolCalculate) {
+    Thread &thread = GetThread();
+    if (thread.GetCurrentInlinedDepth() == UINT32_MAX)
+      m_virtual_step = eLazyBoolNo;
+    else
+      m_virtual_step = eLazyBoolYes;
+  }
+  return m_virtual_step == eLazyBoolYes;
+}
diff --git a/lldb/source/Target/ThreadPlanStepOverRange.cpp b/lldb/source/Target/ThreadPlanStepOverRange.cpp
index ef5b4b5c434d..643ee827c865 100644
--- a/lldb/source/Target/ThreadPlanStepOverRange.cpp
+++ b/lldb/source/Target/ThreadPlanStepOverRange.cpp
@@ -402,7 +402,7 @@ bool ThreadPlanStepOverRange::DoWillResume(lldb::StateType resume_state,
       if (in_inlined_stack) {
         Log *log = GetLog(LLDBLog::Step);
         LLDB_LOGF(log,
-                  "ThreadPlanStepInRange::DoWillResume: adjusting range to "
+                  "ThreadPlanStepOverRange::DoWillResume: adjusting range to "
                   "the frame at inlined depth %d.",
                   thread.GetCurrentInlinedDepth());
         StackFrameSP stack_sp = thread.GetStackFrameAtIndex(0);
diff --git a/lldb/test/API/functionalities/gdb_remote_client/TestGDBRemoteClient.py b/lldb/test/API/functionalities/gdb_remote_client/TestGDBRemoteClient.py
index 5eb3fc3cada9..08ac9290ee85 100644
--- a/lldb/test/API/functionalities/gdb_remote_client/TestGDBRemoteClient.py
+++ b/lldb/test/API/functionalities/gdb_remote_client/TestGDBRemoteClient.py
@@ -132,12 +132,39 @@ class TestGDBRemoteClient(GDBRemoteTestBase):
         target = self.createTarget("a.yaml")
         process = self.connect(target)
 
-        self.assertEqual(1, self.server.responder.packetLog.count("g"))
-        self.server.responder.packetLog = []
+        # We want to make sure that the process is using the g packet, but it's
+        # not required the "connect" should read all registers.  However, it might
+        # have...  So we need to wait till we explicitly 'read_registers' to do
+        # test.
+        # Also, even with the use-g-packet-for-reading lldb will sometimes send p0
+        # early on to see if the packet is supported.  So we can't say that there
+        # will be NO p packets.
+        # But there certainly should be no p packets after the g packet.
+
         self.read_registers(process)
-        # Reading registers should not cause any 'p' packets to be exchanged.
+        print(f"\nPACKET LOG:\n{self.server.responder.packetLog}\n")
+        g_pos = 0
+        try:
+            g_pos = self.server.responder.packetLog.index("g")
+        except err:
+            self.fail("'g' packet not found after fetching registers")
+
+        try:
+            second_g = self.server.responder.packetLog.index("g", g_pos)
+            self.fail("Found more than one 'g' packet")
+        except:
+            pass
+
+        # Make sure there aren't any `p` packets after the `g` packet:
         self.assertEqual(
-            0, len([p for p in self.server.responder.packetLog if p.startswith("p")])
+            0,
+            len(
+                [
+                    p
+                    for p in self.server.responder.packetLog[g_pos:]
+                    if p.startswith("p")
+                ]
+            ),
         )
 
     def test_read_registers_using_p_packets(self):
diff --git a/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py b/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
index 752c3a9cbd28..f52e0f0fd5bc 100644
--- a/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
+++ b/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
@@ -32,6 +32,12 @@ class TestInlineStepping(TestBase):
         self.build()
         self.step_in_template()
 
+    @add_test_categories(["pyapi"])
+    def test_virtual_inline_stepping(self):
+        """Test stepping through a virtual inlined call stack"""
+        self.build()
+        self.virtual_inline_stepping()
+
     def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)
@@ -357,3 +363,60 @@ class TestInlineStepping(TestBase):
 
         step_sequence = [["// In max_value specialized", "into"]]
         self.run_step_sequence(step_sequence)
+
+    def run_to_call_site_and_step(self, source_regex, func_name, start_pos):
+        main_spec = lldb.SBFileSpec("calling.cpp")
+        # Set the breakpoint by file and line, not sourced regex because
+        # we want to make sure we can set breakpoints on call sites:
+        call_site_line_num = line_number(self.main_source, source_regex)
+        target, process, thread, bkpt = lldbutil.run_to_line_breakpoint(
+            self, main_spec, call_site_line_num
+        )
+
+        # Make sure that the location is at the call site (run_to_line_breakpoint already asserted
+        # that there's one location.):
+        bkpt_loc = bkpt.location[0]
+        strm = lldb.SBStream()
+        result = bkpt_loc.GetDescription(strm, lldb.eDescriptionLevelFull)
+
+        self.assertTrue(result, "Got a location description")
+        desc = strm.GetData()
+        self.assertIn(f"calling.cpp:{call_site_line_num}", desc, "Right line listed")
+        # We don't get the function name right yet - so we omit it in printing.
+        # Turn on this test when that is working.
+        # self.assertIn(func_name, desc, "Right function listed")
+
+        pc = thread.frame[0].pc
+        for i in range(start_pos, 3):
+            thread.StepInto()
+            frame_0 = thread.frame[0]
+
+            trivial_line_num = line_number(
+                self.main_source, f"In caller_trivial_inline_{i}."
+            )
+            self.assertEqual(
+                frame_0.line_entry.line,
+                trivial_line_num,
+                f"Stepped into the caller_trivial_inline_{i}",
+            )
+            if pc != frame_0.pc:
+                # If we get here, we stepped to the expected line number, but
+                # the compiler on this system has decided to insert an instruction
+                # between the call site of an inlined function with no arguments,
+                # returning void, and its immediate call to another void inlined function
+                # with no arguments.  We aren't going to be testing virtual inline
+                # stepping for this function...
+                break
+
+        process.Kill()
+        target.Clear()
+
+    def virtual_inline_stepping(self):
+        """Use the Python API's to step through a virtual inlined stack"""
+        self.run_to_call_site_and_step("At caller_trivial_inline_1", "main", 1)
+        self.run_to_call_site_and_step(
+            "In caller_trivial_inline_1", "caller_trivial_inline_1", 2
+        )
+        self.run_to_call_site_and_step(
+            "In caller_trivial_inline_2", "caller_trivial_inline_2", 3
+        )
diff --git a/lldb/test/API/functionalities/inline-stepping/calling.cpp b/lldb/test/API/functionalities/inline-stepping/calling.cpp
index 49179ce7c978..d7ee56b3c079 100644
--- a/lldb/test/API/functionalities/inline-stepping/calling.cpp
+++ b/lldb/test/API/functionalities/inline-stepping/calling.cpp
@@ -13,6 +13,12 @@ int called_by_inline_ref (int &value);
 inline void inline_trivial_1 () __attribute__((always_inline));
 inline void inline_trivial_2 () __attribute__((always_inline));
 
+// These three should share the same initial pc so we can test
+// virtual inline stepping.
+inline void caller_trivial_inline_1() __attribute__((always_inline));
+inline void caller_trivial_inline_2() __attribute__((always_inline));
+inline void caller_trivial_inline_3() __attribute__((always_inline));
+
 void caller_trivial_1 ();
 void caller_trivial_2 ();
 
@@ -79,6 +85,23 @@ caller_trivial_2 ()
     inline_value += 1;  // At increment in caller_trivial_2.
 }
 
+// When you call caller_trivial_inline_1, the inlined call-site
+// should share a PC with all three of the following inlined
+// functions, so we can exercise "virtual inline stepping".
+void caller_trivial_inline_1() {
+  caller_trivial_inline_2(); // In caller_trivial_inline_1.
+  inline_value += 1;
+}
+
+void caller_trivial_inline_2() {
+  caller_trivial_inline_3(); // In caller_trivial_inline_2.
+  inline_value += 1;
+}
+
+void caller_trivial_inline_3() {
+  inline_value += 1; // In caller_trivial_inline_3.
+}
+
 void
 called_by_inline_trivial ()
 {
@@ -132,5 +155,7 @@ main (int argc, char **argv)
     max_value(123, 456);                                // Call max_value template
     max_value(std::string("abc"), std::string("0022")); // Call max_value specialized
 
+    caller_trivial_inline_1(); // At caller_trivial_inline_1.
+
     return 0;            // About to return from main.
 }
-- 
GitLab


From 49277253f016268e4a10109f1db2e53c60d35881 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Wed, 30 Oct 2024 09:31:32 -0700
Subject: [PATCH 606/693] [lldb] Use LLVM's helper for Unicode conversion (NFC)
 (#112582)

The codecvt header has been deprecated in C++17. Use LLVM's unicode
helpers to convert between UTF-8 and UTF-16.
---
 lldb/include/lldb/Host/Editline.h    | 25 ---------------
 lldb/source/Host/common/Editline.cpp | 48 +++++++++++++++-------------
 2 files changed, 26 insertions(+), 47 deletions(-)

diff --git a/lldb/include/lldb/Host/Editline.h b/lldb/include/lldb/Host/Editline.h
index a02f90891599..57e2c831e349 100644
--- a/lldb/include/lldb/Host/Editline.h
+++ b/lldb/include/lldb/Host/Editline.h
@@ -30,9 +30,6 @@
 
 #include "lldb/Host/Config.h"
 
-#if LLDB_EDITLINE_USE_WCHAR
-#include <codecvt>
-#endif
 #include <locale>
 #include <sstream>
 #include <vector>
@@ -57,23 +54,6 @@
 
 #include "llvm/ADT/FunctionExtras.h"
 
-#if defined(__clang__) && defined(__has_warning)
-#if __has_warning("-Wdeprecated-declarations")
-#define LLDB_DEPRECATED_WARNING_DISABLE                                        \
-  _Pragma("clang diagnostic push")                                             \
-      _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"")
-#define LLDB_DEPRECATED_WARNING_RESTORE _Pragma("clang diagnostic pop")
-#endif
-#elif defined(__GNUC__) && __GNUC__ > 6
-#define LLDB_DEPRECATED_WARNING_DISABLE                                        \
-  _Pragma("GCC diagnostic push")                                               \
-      _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"")
-#define LLDB_DEPRECATED_WARNING_RESTORE _Pragma("GCC diagnostic pop")
-#else
-#define LLDB_DEPRECATED_WARNING_DISABLE
-#define LLDB_DEPRECATED_WARNING_RESTORE
-#endif
-
 namespace lldb_private {
 namespace line_editor {
 
@@ -383,11 +363,6 @@ private:
   void SetEditLinePromptCallback(EditlinePromptCallbackType callbackFn);
   void SetGetCharacterFunction(EditlineGetCharCallbackType callbackFn);
 
-#if LLDB_EDITLINE_USE_WCHAR
-  LLDB_DEPRECATED_WARNING_DISABLE
-  std::wstring_convert<std::codecvt_utf8<wchar_t>> m_utf8conv;
-  LLDB_DEPRECATED_WARNING_RESTORE
-#endif
   ::EditLine *m_editline = nullptr;
   EditlineHistorySP m_history_sp;
   bool m_in_history = false;
diff --git a/lldb/source/Host/common/Editline.cpp b/lldb/source/Host/common/Editline.cpp
index 60117cb5f0e6..f95f854c5f22 100644
--- a/lldb/source/Host/common/Editline.cpp
+++ b/lldb/source/Host/common/Editline.cpp
@@ -10,9 +10,8 @@
 #include <iomanip>
 #include <optional>
 
-#include "lldb/Host/Editline.h"
-
 #include "lldb/Host/ConnectionFileDescriptor.h"
+#include "lldb/Host/Editline.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/Host.h"
 #include "lldb/Utility/CompletionRequest.h"
@@ -23,6 +22,7 @@
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/StringList.h"
 #include "lldb/Utility/Timeout.h"
+#include "llvm/Support/ConvertUTF.h"
 
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Locale.h"
@@ -444,7 +444,9 @@ StringList Editline::GetInputAsStringList(int line_count) {
     if (line_count == 0)
       break;
 #if LLDB_EDITLINE_USE_WCHAR
-    lines.AppendString(m_utf8conv.to_bytes(line));
+    std::string buffer;
+    llvm::convertWideToUTF8(line, buffer);
+    lines.AppendString(buffer);
 #else
     lines.AppendString(line);
 #endif
@@ -636,7 +638,9 @@ unsigned char Editline::BreakLineCommand(int ch) {
     if (m_fix_indentation_callback) {
       StringList lines = GetInputAsStringList(m_current_line_index + 1);
 #if LLDB_EDITLINE_USE_WCHAR
-      lines.AppendString(m_utf8conv.to_bytes(new_line_fragment));
+      std::string buffer;
+      llvm::convertWideToUTF8(new_line_fragment, buffer);
+      lines.AppendString(buffer);
 #else
       lines.AppendString(new_line_fragment);
 #endif
@@ -684,8 +688,9 @@ unsigned char Editline::EndOrAddLineCommand(int ch) {
       m_input_lines.clear();
       for (unsigned index = 0; index < lines.GetSize(); index++) {
 #if LLDB_EDITLINE_USE_WCHAR
-        m_input_lines.insert(m_input_lines.end(),
-                             m_utf8conv.from_bytes(lines[index]));
+        std::wstring wbuffer;
+        llvm::ConvertUTF8toWide(lines[index], wbuffer);
+        m_input_lines.insert(m_input_lines.end(), wbuffer);
 #else
         m_input_lines.insert(m_input_lines.end(), lines[index]);
 #endif
@@ -869,7 +874,9 @@ unsigned char Editline::FixIndentationCommand(int ch) {
     currentLine = currentLine.erase(0, -indent_correction);
   }
 #if LLDB_EDITLINE_USE_WCHAR
-  m_input_lines[m_current_line_index] = m_utf8conv.from_bytes(currentLine);
+  std::wstring wbuffer;
+  llvm::ConvertUTF8toWide(currentLine, wbuffer);
+  m_input_lines[m_current_line_index] = wbuffer;
 #else
   m_input_lines[m_current_line_index] = currentLine;
 #endif
@@ -1502,7 +1509,7 @@ bool Editline::GetLine(std::string &line, bool &interrupted) {
     } else {
       m_history_sp->Enter(input);
 #if LLDB_EDITLINE_USE_WCHAR
-      line = m_utf8conv.to_bytes(SplitLines(input)[0]);
+      llvm::convertWideToUTF8(SplitLines(input)[0], line);
 #else
       line = SplitLines(input)[0];
 #endif
@@ -1574,25 +1581,22 @@ bool Editline::CompleteCharacter(char ch, EditLineGetCharType &out) {
   out = (unsigned char)ch;
   return true;
 #else
-  LLDB_DEPRECATED_WARNING_DISABLE
-  std::codecvt_utf8<wchar_t> cvt;
-  LLDB_DEPRECATED_WARNING_RESTORE
   llvm::SmallString<4> input;
   for (;;) {
-    const char *from_next;
-    wchar_t *to_next;
-    std::mbstate_t state = std::mbstate_t();
     input.push_back(ch);
-    switch (cvt.in(state, input.begin(), input.end(), from_next, &out, &out + 1,
-                   to_next)) {
-    case std::codecvt_base::ok:
+    auto *cur_ptr = reinterpret_cast<const llvm::UTF8 *>(input.begin());
+    auto *end_ptr = reinterpret_cast<const llvm::UTF8 *>(input.end());
+    llvm::UTF32 code_point = 0;
+    llvm::ConversionResult cr = llvm::convertUTF8Sequence(
+        &cur_ptr, end_ptr, &code_point, llvm::lenientConversion);
+    switch (cr) {
+    case llvm::conversionOK:
+      out = code_point;
       return out != (EditLineGetCharType)WEOF;
-
-    case std::codecvt_base::error:
-    case std::codecvt_base::noconv:
+    case llvm::targetExhausted:
+    case llvm::sourceIllegal:
       return false;
-
-    case std::codecvt_base::partial:
+    case llvm::sourceExhausted:
       lldb::ConnectionStatus status;
       size_t read_count = m_input_connection.Read(
           &ch, 1, std::chrono::seconds(0), status, nullptr);
-- 
GitLab


From 0c9a02355abc3b037be53c072fc46a13bb5aa2c1 Mon Sep 17 00:00:00 2001
From: Asher Mancinelli <ashermancinelli@gmail.com>
Date: Wed, 30 Oct 2024 09:50:27 -0700
Subject: [PATCH 607/693] [flang][fir] always use memcpy for fir.box  (#113949)

@jeanPerier explained the importance of converting box loads and stores
into `memcpy`s instead of aggregate loads and stores, and I'll do my
best to explain it here.

* [(godbolt link) Example comparing opt transformations on memcpys vs
aggregate load/stores](https://godbolt.org/z/be7xM83cG)
* LLVM can more effectively reason about memcpys compared to aggregate
load/stores.
* This came up when others were discussing array descriptors for
assumed-rank arrays passed to `bind(c)` subroutines, with the
implication that the array descriptors are known to have lower bounds of
1 and that they are not pointer/allocatable types.
* [(godbolt link) Clang also uses memcpys so we should probably follow
them, assuming the clang developers are generatign what they know Opt
will handle more effectively.](https://godbolt.org/z/YT4x7387W)
* This currently may not help much without the `nocapture` attribute
being propagated to function calls, but [it looks like someone may do
this soon (discourse
link)](https://discourse.llvm.org/t/applying-the-nocapture-attribute-to-reference-passed-arguments-in-fortran-subroutines/81401/23)
or I can do this in a follow-up patch.

Note on test `flang/test/Fir/embox-char.fir`: it looks like the original
test was auto-generated. I wasn't too sure which parts were especially
important to test, so I regenerated the test. If we want the updated
version to look more like the old version, I'll make those changes.
---
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |  59 ++---
 flang/test/Fir/box.fir                        |  19 +-
 .../Fir/convert-to-llvm-openmp-and-fir.fir    |   4 +-
 flang/test/Fir/convert-to-llvm.fir            |  28 +-
 flang/test/Fir/embox-char.fir                 | 239 +++++++++---------
 flang/test/Fir/polymorphic.fir                |  12 +-
 flang/test/Fir/tbaa.fir                       |   4 +-
 .../Integration/OpenMP/private-global.f90     |   5 +-
 ...privatization-allocatable-firstprivate.f90 |   3 +-
 .../Lower/OpenMP/parallel-reduction-mixed.f90 |   2 +-
 flang/test/Lower/allocatable-polymorphic.f90  |  18 +-
 11 files changed, 188 insertions(+), 205 deletions(-)

diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index e6eeb0d5db4a..4c8c56e0f21c 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -2949,9 +2949,10 @@ struct LoadOpConversion : public fir::FIROpConversion<fir::LoadOp> {
   llvm::LogicalResult
   matchAndRewrite(fir::LoadOp load, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
+
     mlir::Type llvmLoadTy = convertObjectType(load.getType());
     if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(load.getType())) {
-      // fir.box is a special case because it is considered as an ssa values in
+      // fir.box is a special case because it is considered an ssa value in
       // fir, but it is lowered as a pointer to a descriptor. So
       // fir.ref<fir.box> and fir.box end up being the same llvm types and
       // loading a fir.ref<fir.box> is implemented as taking a snapshot of the
@@ -2960,30 +2961,17 @@ struct LoadOpConversion : public fir::FIROpConversion<fir::LoadOp> {
       mlir::Location loc = load.getLoc();
       auto newBoxStorage =
           genAllocaAndAddrCastWithType(loc, llvmLoadTy, defaultAlign, rewriter);
-      // TODO: always generate llvm.memcpy, LLVM is better at optimizing it than
-      // aggregate loads + stores.
-      if (boxTy.isAssumedRank()) {
-
-        TypePair boxTypePair{boxTy, llvmLoadTy};
-        mlir::Value boxSize =
-            computeBoxSize(loc, boxTypePair, inputBoxStorage, rewriter);
-        auto memcpy = rewriter.create<mlir::LLVM::MemcpyOp>(
-            loc, newBoxStorage, inputBoxStorage, boxSize, /*isVolatile=*/false);
-        if (std::optional<mlir::ArrayAttr> optionalTag = load.getTbaa())
-          memcpy.setTBAATags(*optionalTag);
-        else
-          attachTBAATag(memcpy, boxTy, boxTy, nullptr);
-      } else {
-        auto boxValue = rewriter.create<mlir::LLVM::LoadOp>(loc, llvmLoadTy,
-                                                            inputBoxStorage);
-        if (std::optional<mlir::ArrayAttr> optionalTag = load.getTbaa())
-          boxValue.setTBAATags(*optionalTag);
-        else
-          attachTBAATag(boxValue, boxTy, boxTy, nullptr);
-        auto storeOp =
-            rewriter.create<mlir::LLVM::StoreOp>(loc, boxValue, newBoxStorage);
-        attachTBAATag(storeOp, boxTy, boxTy, nullptr);
-      }
+
+      TypePair boxTypePair{boxTy, llvmLoadTy};
+      mlir::Value boxSize =
+          computeBoxSize(loc, boxTypePair, inputBoxStorage, rewriter);
+      auto memcpy = rewriter.create<mlir::LLVM::MemcpyOp>(
+          loc, newBoxStorage, inputBoxStorage, boxSize, /*isVolatile=*/false);
+
+      if (std::optional<mlir::ArrayAttr> optionalTag = load.getTbaa())
+        memcpy.setTBAATags(*optionalTag);
+      else
+        attachTBAATag(memcpy, boxTy, boxTy, nullptr);
       rewriter.replaceOp(load, newBoxStorage);
     } else {
       auto loadOp = rewriter.create<mlir::LLVM::LoadOp>(
@@ -3227,20 +3215,13 @@ struct StoreOpConversion : public fir::FIROpConversion<fir::StoreOp> {
     mlir::LLVM::AliasAnalysisOpInterface newOp;
     if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(storeTy)) {
       mlir::Type llvmBoxTy = lowerTy().convertBoxTypeAsStruct(boxTy);
-      // fir.box value is actually in memory, load it first before storing it,
-      // or do a memcopy for assumed-rank descriptors.
-      if (boxTy.isAssumedRank()) {
-        TypePair boxTypePair{boxTy, llvmBoxTy};
-        mlir::Value boxSize =
-            computeBoxSize(loc, boxTypePair, llvmValue, rewriter);
-        newOp = rewriter.create<mlir::LLVM::MemcpyOp>(
-            loc, llvmMemref, llvmValue, boxSize, /*isVolatile=*/false);
-      } else {
-        auto val =
-            rewriter.create<mlir::LLVM::LoadOp>(loc, llvmBoxTy, llvmValue);
-        attachTBAATag(val, boxTy, boxTy, nullptr);
-        newOp = rewriter.create<mlir::LLVM::StoreOp>(loc, val, llvmMemref);
-      }
+      // Always use memcpy because LLVM is not as effective at optimizing
+      // aggregate loads/stores as it is optimizing memcpy.
+      TypePair boxTypePair{boxTy, llvmBoxTy};
+      mlir::Value boxSize =
+          computeBoxSize(loc, boxTypePair, llvmValue, rewriter);
+      newOp = rewriter.create<mlir::LLVM::MemcpyOp>(
+          loc, llvmMemref, llvmValue, boxSize, /*isVolatile=*/false);
     } else {
       newOp = rewriter.create<mlir::LLVM::StoreOp>(loc, llvmValue, llvmMemref);
     }
diff --git a/flang/test/Fir/box.fir b/flang/test/Fir/box.fir
index 81a4d8bc13bf..fd9fa1f2b3aa 100644
--- a/flang/test/Fir/box.fir
+++ b/flang/test/Fir/box.fir
@@ -56,12 +56,14 @@ func.func @fa(%a : !fir.ref<!fir.array<100xf32>>) {
 // CHECK-LABEL: define void @b1(
 // CHECK-SAME: ptr %[[res:.*]], ptr %[[arg0:.*]], i64 %[[arg1:.*]])
 func.func @b1(%arg0 : !fir.ref<!fir.char<1,?>>, %arg1 : index) -> !fir.box<!fir.char<1,?>> {
+  // CHECK: %[[alloca:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8 }
   // CHECK: %[[size:.*]] = mul i64 ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64), %[[arg1]]
   // CHECK: insertvalue {{.*}} undef, i64 %[[size]], 1
   // CHECK: insertvalue {{.*}} i32 20240719, 2
   // CHECK: insertvalue {{.*}} ptr %[[arg0]], 0
   %x = fir.embox %arg0 typeparams %arg1 : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
-  // CHECK: store {{.*}}, ptr %[[res]]
+  // CHECK: store {{.*}}, ptr %[[alloca]]
+  // CHECK: call void @llvm.memcpy.p0.p0.i32(ptr %[[res]], ptr %[[alloca]], i32 24, i1 false)
   return %x : !fir.box<!fir.char<1,?>>
 }
 
@@ -71,11 +73,13 @@ func.func @b1(%arg0 : !fir.ref<!fir.char<1,?>>, %arg1 : index) -> !fir.box<!fir.
 // CHECK-SAME: ptr %[[arg0:.*]], i64 %[[arg1:.*]])
 func.func @b2(%arg0 : !fir.ref<!fir.array<?x!fir.char<1,5>>>, %arg1 : index) -> !fir.box<!fir.array<?x!fir.char<1,5>>> {
   %1 = fir.shape %arg1 : (index) -> !fir.shape<1>
+  // CHECK: %[[alloca:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }
   // CHECK: insertvalue {{.*}} { ptr undef, i64 ptrtoint (ptr getelementptr ([5 x i8], ptr null, i32 1) to i64), i32 20240719, i8 1, i8 40, i8 0, i8 0, {{.*}} }, i64 %[[arg1]], 7, 0, 1
   // CHECK: insertvalue {{.*}} %{{.*}}, i64 ptrtoint (ptr getelementptr ([5 x i8], ptr null, i32 1) to i64), 7, 0, 2
   // CHECK: insertvalue {{.*}} ptr %[[arg0]], 0
   %2 = fir.embox %arg0(%1) : (!fir.ref<!fir.array<?x!fir.char<1,5>>>, !fir.shape<1>) -> !fir.box<!fir.array<?x!fir.char<1,5>>>
-  // CHECK: store {{.*}}, ptr %[[res]]
+  // CHECK: store {{.*}}, ptr %[[alloca]]
+  // CHECK: call void @llvm.memcpy.p0.p0.i32(ptr %[[res]], ptr %[[alloca]], i32 48, i1 false)
   return %2 : !fir.box<!fir.array<?x!fir.char<1,5>>>
 }
 
@@ -84,6 +88,7 @@ func.func @b2(%arg0 : !fir.ref<!fir.array<?x!fir.char<1,5>>>, %arg1 : index) ->
 // CHECK-SAME: ptr %[[res:.*]], ptr %[[arg0:.*]], i64 %[[arg1:.*]], i64 %[[arg2:.*]])
 func.func @b3(%arg0 : !fir.ref<!fir.array<?x!fir.char<1,?>>>, %arg1 : index, %arg2 : index) -> !fir.box<!fir.array<?x!fir.char<1,?>>> {
   %1 = fir.shape %arg2 : (index) -> !fir.shape<1>
+  // CHECK: %[[alloca:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }
   // CHECK: %[[size:.*]] = mul i64 ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64), %[[arg1]]
   // CHECK: insertvalue {{.*}} i64 %[[size]], 1
   // CHECK: insertvalue {{.*}} i32 20240719, 2
@@ -91,7 +96,8 @@ func.func @b3(%arg0 : !fir.ref<!fir.array<?x!fir.char<1,?>>>, %arg1 : index, %ar
   // CHECK: insertvalue {{.*}} i64 %[[size]], 7, 0, 2
   // CHECK: insertvalue {{.*}} ptr %[[arg0]], 0
   %2 = fir.embox %arg0(%1) typeparams %arg1 : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, index) -> !fir.box<!fir.array<?x!fir.char<1,?>>>
-  // CHECK: store {{.*}}, ptr %[[res]]
+  // CHECK: store {{.*}}, ptr %[[alloca]]
+  // CHECK: call void @llvm.memcpy.p0.p0.i32(ptr %[[res]], ptr %[[alloca]], i32 48, i1 false)
   return %2 : !fir.box<!fir.array<?x!fir.char<1,?>>>
 }
 
@@ -101,6 +107,7 @@ func.func @b3(%arg0 : !fir.ref<!fir.array<?x!fir.char<1,?>>>, %arg1 : index, %ar
 func.func @b4(%arg0 : !fir.ref<!fir.array<7x!fir.char<1,?>>>, %arg1 : index) -> !fir.box<!fir.array<7x!fir.char<1,?>>> {
   %c_7 = arith.constant 7 : index
   %1 = fir.shape %c_7 : (index) -> !fir.shape<1>
+  // CHECK: %[[alloca:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }
   // CHECK:   %[[size:.*]] = mul i64 ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64), %[[arg1]]
   // CHECK: insertvalue {{.*}} i64 %[[size]], 1
   // CHECK: insertvalue {{.*}} i32 20240719, 2
@@ -108,7 +115,8 @@ func.func @b4(%arg0 : !fir.ref<!fir.array<7x!fir.char<1,?>>>, %arg1 : index) ->
   // CHECK: insertvalue {{.*}} i64 %[[size]], 7, 0, 2
   // CHECK: insertvalue {{.*}} ptr %[[arg0]], 0
   %x = fir.embox %arg0(%1) typeparams %arg1 : (!fir.ref<!fir.array<7x!fir.char<1,?>>>, !fir.shape<1>, index) -> !fir.box<!fir.array<7x!fir.char<1,?>>>
-  // CHECK: store {{.*}}, ptr %[[res]]
+  // CHECK: store {{.*}}, ptr %[[alloca]]
+  // CHECK: call void @llvm.memcpy.p0.p0.i32(ptr %[[res]], ptr %[[alloca]], i32 48, i1 false)
   return %x : !fir.box<!fir.array<7x!fir.char<1,?>>>
 }
 
@@ -117,8 +125,7 @@ func.func @b4(%arg0 : !fir.ref<!fir.array<7x!fir.char<1,?>>>, %arg1 : index) ->
 // CHECK-SAME: ptr %[[arg0:.*]], ptr %[[arg1:.*]])
 func.func @b5(%arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, %arg1 : !fir.box<!fir.heap<!fir.array<?x?xf32>>>) {
   fir.store %arg1 to %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
-  // CHECK: %[[boxLoad:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, [2 x [3 x i64]] }, ptr %[[arg1]]
-  // CHECK: store { ptr, i64, i32, i8, i8, i8, i8, [2 x [3 x i64]] } %[[boxLoad]], ptr %[[arg0]]
+  // CHECK: call void @llvm.memcpy.p0.p0.i32(ptr %0, ptr %1, i32 72, i1 false)
   return
 }
 
diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index 335877e7c9a8..168526518865 100644
--- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -799,8 +799,8 @@ func.func @_QPs(%arg0: !fir.ref<complex<f32>> {fir.bindc_name = "x"}) {
 //CHECK:  omp.parallel   {
 //CHECK:    %[[CONST_1:.*]] = llvm.mlir.constant(1 : i32) : i32
 //CHECK:    %[[ALLOCA_1:.*]] = llvm.alloca %[[CONST_1:.*]] x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
-//CHECK:    %[[LOAD:.*]] = llvm.load %[[ALLOCA]] : !llvm.ptr -> !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>
-//CHECK:    llvm.store %[[LOAD]], %[[ALLOCA_1]] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>, !llvm.ptr
+//CHECK:    %[[SIZE:.*]] = llvm.mlir.constant(24 : i32) : i32
+//CHECK:    "llvm.intr.memcpy"(%[[ALLOCA_1]], %[[ALLOCA]], %[[SIZE]]) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i32) -> ()
 //CHECK:    %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA_1]][0, 0] : (!llvm.ptr) -> !llvm.ptr
 //CHECK:    %[[LOAD_2:.*]] = llvm.load %[[GEP]] : !llvm.ptr -> !llvm.ptr
 //CHECK:    omp.terminator
diff --git a/flang/test/Fir/convert-to-llvm.fir b/flang/test/Fir/convert-to-llvm.fir
index 1182a0a10f21..fa391fa6cc7a 100644
--- a/flang/test/Fir/convert-to-llvm.fir
+++ b/flang/test/Fir/convert-to-llvm.fir
@@ -862,8 +862,8 @@ func.func @test_store_box(%array : !fir.ref<!fir.box<!fir.array<?x?xf32>>>, %box
 // CHECK-LABEL:  llvm.func @test_store_box
 // CHECK-SAME:  (%[[arg0:.*]]: !llvm.ptr,
 // CHECK-SAME:  %[[arg1:.*]]: !llvm.ptr) {
-// CHECK-NEXT:  %[[box_to_store:.*]] = llvm.load %arg1 : !llvm.ptr -> !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<2 x array<3 x i{{.*}}>>)>
-// CHECK-NEXT:  llvm.store %[[box_to_store]], %[[arg0]] : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<2 x array<3 x i{{.*}}>>)>, !llvm.ptr
+// CHECK-NEXT:  %[[size:.*]] = llvm.mlir.constant(72 : i32) : i32
+// CHECK-NEXT:  "llvm.intr.memcpy"(%[[arg0]], %[[arg1]], %[[size]]) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i32) -> ()
 // CHECK-NEXT:  llvm.return
 // CHECK-NEXT:  }
 
@@ -875,15 +875,17 @@ func.func @store_unlimited_polymorphic_box(%arg0 : !fir.class<none>, %arg1 : !fi
   fir.store %arg3 to %arg3r : !fir.ref<!fir.box<!fir.array<?xnone>>>
   return
 }
-// CHECK-LABEL:   llvm.func @store_unlimited_polymorphic_box(
-// CHECK:  %[[VAL_8:.*]] = llvm.load %{{.*}} : !llvm.ptr -> !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, ptr, array<1 x i{{.*}}>)>
-// CHECK:  llvm.store %[[VAL_8]], %{{.*}} : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, ptr, array<1 x i{{.*}}>)>, !llvm.ptr
-// CHECK:  %[[VAL_9:.*]] = llvm.load %{{.*}} : !llvm.ptr -> !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i{{.*}}>>, ptr, array<1 x i{{.*}}>)>
-// CHECK:  llvm.store %[[VAL_9]], %{{.*}} : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i{{.*}}>>, ptr, array<1 x i{{.*}}>)>, !llvm.ptr
-// CHECK:  %[[VAL_10:.*]] = llvm.load %{{.*}} : !llvm.ptr -> !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, ptr, array<1 x i{{.*}}>)>
-// CHECK:  llvm.store %[[VAL_10]], %{{.*}} : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, ptr, array<1 x i{{.*}}>)>, !llvm.ptr
-// CHECK:  %[[VAL_11:.*]] = llvm.load %{{.*}}: !llvm.ptr
-// CHECK:  llvm.store %[[VAL_11]], %{{.*}} : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i{{.*}}>>, ptr, array<1 x i{{.*}}>)>, !llvm.ptr
+// CHECK:   llvm.func @store_unlimited_polymorphic_box(%[[VAL_0:.*]]: !llvm.ptr, %[[VAL_1:.*]]: !llvm.ptr, %[[VAL_2:.*]]: !llvm.ptr, %[[VAL_3:.*]]: !llvm.ptr, %[[VAL_4:.*]]: !llvm.ptr, %[[VAL_5:.*]]: !llvm.ptr, %[[VAL_6:.*]]: !llvm.ptr, %[[VAL_7:.*]]: !llvm.ptr) {
+// CHECK:     %[[VAL_8:.*]] = llvm.mlir.constant(40 : i32) : i32
+// CHECK:     "llvm.intr.memcpy"(%[[VAL_4]], %[[VAL_0]], %[[VAL_8]]) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i32) -> ()
+// CHECK:     %[[VAL_9:.*]] = llvm.mlir.constant(64 : i32) : i32
+// CHECK:     "llvm.intr.memcpy"(%[[VAL_5]], %[[VAL_1]], %[[VAL_9]]) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i32) -> ()
+// CHECK:     %[[VAL_10:.*]] = llvm.mlir.constant(40 : i32) : i32
+// CHECK:     "llvm.intr.memcpy"(%[[VAL_6]], %[[VAL_2]], %[[VAL_10]]) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i32) -> ()
+// CHECK:     %[[VAL_11:.*]] = llvm.mlir.constant(64 : i32) : i32
+// CHECK:     "llvm.intr.memcpy"(%[[VAL_7]], %[[VAL_3]], %[[VAL_11]]) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i32) -> ()
+// CHECK:     llvm.return
+// CHECK:   }
 
 
 // -----
@@ -935,8 +937,8 @@ func.func @test_load_box(%addr : !fir.ref<!fir.box<!fir.array<10xf32>>>) {
 // GENERIC-NEXT:  %[[box_copy:.*]] = llvm.alloca %[[c1]] x !llvm.struct<([[DESC_TYPE:.*]])>
 // AMDGPU-NEXT:   %[[alloca_box_copy:.*]] = llvm.alloca %[[c1]] x !llvm.struct<([[DESC_TYPE:.*]])>{{.*}} : (i32) -> !llvm.ptr<5>
 // AMDGPU-NEXT:   %[[box_copy:.*]] = llvm.addrspacecast %[[alloca_box_copy]] : !llvm.ptr<5> to !llvm.ptr
-// CHECK-NEXT:    %[[box_val:.*]] = llvm.load %[[arg0]] : !llvm.ptr -> !llvm.struct<([[DESC_TYPE]])>
-// CHECK-NEXT:    llvm.store %[[box_val]], %[[box_copy]] : !llvm.struct<([[DESC_TYPE]])>, !llvm.ptr
+// CHECK-NEXT:    %[[size:.*]] = llvm.mlir.constant(48 : i32) : i32
+// CHECK-NEXT:    "llvm.intr.memcpy"(%[[box_copy]], %[[arg0]], %[[size]]) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i32) -> ()
 // CHECK-NEXT:    llvm.call @takes_box(%[[box_copy]]) : (!llvm.ptr) -> ()
 // CHECK-NEXT:    llvm.return
 // CHECK-NEXT:  }
diff --git a/flang/test/Fir/embox-char.fir b/flang/test/Fir/embox-char.fir
index bf8344dbb60f..efb069f96520 100644
--- a/flang/test/Fir/embox-char.fir
+++ b/flang/test/Fir/embox-char.fir
@@ -1,3 +1,10 @@
+// NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
+
+// The script is designed to make adding checks to
+// a test case fast, it is *not* designed to be authoritative
+// about what constitutes a good test! The CHECK should be
+// minimized and named to reflect the test intent.
+
 // Test that the offset of the first element of the slice
 // is computed in elements of the type used for the GEP
 // computing the base of the slice.
@@ -10,42 +17,40 @@
 //   print *, x(2,:)
 // end subroutine
 
-// CHECK-LABEL:   llvm.func @test_char4(
-// CHECK-SAME:        %[[VAL_0:.*]]: !llvm.ptr,
-// CHECK-SAME:        %[[VAL_1_SLICE_LB0:.*]]: i64, %[[VAL_2_SLICE_EX0:.*]]: i64, %[[VAL_3_SLICE_ST0:.*]]: i64, %[[VAL_4_SLICE_LB1:.*]]: i64, %[[VAL_5_SLICE_EX1:.*]]: i64, %[[VAL_6_SLICE_ST1:.*]]: i64) {
+// CHECK:   llvm.func @test_char4(%[[VAL_0:.*]]: !llvm.ptr, %[[VAL_1:.*]]: i64, %[[VAL_2:.*]]: i64, %[[VAL_3:.*]]: i64, %[[VAL_4:.*]]: i64, %[[VAL_5:.*]]: i64, %[[VAL_6:.*]]: i64) {
 // CHECK:           %[[VAL_7:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:           %[[VAL_8:.*]] = llvm.alloca %[[VAL_7]] x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
 // CHECK:           %[[VAL_9:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:           %[[VAL_10:.*]] = llvm.alloca %[[VAL_9]] x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
 // CHECK:           %[[VAL_11:.*]] = llvm.mlir.constant(0 : index) : i64
 // CHECK:           %[[VAL_12:.*]] = llvm.mlir.constant(1 : index) : i64
-// CHECK:           %[[VAL_13_WIDTH:.*]] = llvm.mlir.constant(4 : index) : i64
-// CHECK:           %[[VAL_14:.*]] = llvm.load %[[VAL_0]] : !llvm.ptr -> !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           llvm.store %[[VAL_14]], %[[VAL_10]] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>, !llvm.ptr
+// CHECK:           %[[VAL_13:.*]] = llvm.mlir.constant(4 : index) : i64
+// CHECK:           %[[VAL_14:.*]] = llvm.mlir.constant(72 : i32) : i32
+// CHECK:           "llvm.intr.memcpy"(%[[VAL_10]], %[[VAL_0]], %[[VAL_14]]) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i32) -> ()
 // CHECK:           %[[VAL_15:.*]] = llvm.getelementptr %[[VAL_10]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_16_BYTESIZE:.*]] = llvm.load %[[VAL_15]] : !llvm.ptr -> i64
+// CHECK:           %[[VAL_16:.*]] = llvm.load %[[VAL_15]] : !llvm.ptr -> i64
 // CHECK:           %[[VAL_17:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_12]], 0] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_18_LB1:.*]] = llvm.load %[[VAL_17]] : !llvm.ptr -> i64
+// CHECK:           %[[VAL_18:.*]] = llvm.load %[[VAL_17]] : !llvm.ptr -> i64
 // CHECK:           %[[VAL_19:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_12]], 1] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_20_EX1:.*]] = llvm.load %[[VAL_19]] : !llvm.ptr -> i64
+// CHECK:           %[[VAL_20:.*]] = llvm.load %[[VAL_19]] : !llvm.ptr -> i64
 // CHECK:           %[[VAL_21:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_12]], 2] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_22_ST1:.*]] = llvm.load %[[VAL_21]] : !llvm.ptr -> i64
+// CHECK:           %[[VAL_22:.*]] = llvm.load %[[VAL_21]] : !llvm.ptr -> i64
 // CHECK:           %[[VAL_23:.*]] = llvm.getelementptr %[[VAL_10]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_24_BASEPTR:.*]] = llvm.load %[[VAL_23]] : !llvm.ptr -> !llvm.ptr
+// CHECK:           %[[VAL_24:.*]] = llvm.load %[[VAL_23]] : !llvm.ptr -> !llvm.ptr
 // CHECK:           %[[VAL_25:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_11]], 0] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_26_LB0:.*]] = llvm.load %[[VAL_25]] : !llvm.ptr -> i64
+// CHECK:           %[[VAL_26:.*]] = llvm.load %[[VAL_25]] : !llvm.ptr -> i64
 // CHECK:           %[[VAL_27:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_11]], 1] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_28_EX0:.*]] = llvm.load %[[VAL_27]] : !llvm.ptr -> i64
+// CHECK:           %[[VAL_28:.*]] = llvm.load %[[VAL_27]] : !llvm.ptr -> i64
 // CHECK:           %[[VAL_29:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_11]], 2] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_30_ST0:.*]] = llvm.load %[[VAL_29]] : !llvm.ptr -> i64
-// CHECK:           %[[VAL_31_LEN:.*]] = llvm.sdiv %[[VAL_16_BYTESIZE]], %[[VAL_13_WIDTH]]  : i64
+// CHECK:           %[[VAL_30:.*]] = llvm.load %[[VAL_29]] : !llvm.ptr -> i64
+// CHECK:           %[[VAL_31:.*]] = llvm.sdiv %[[VAL_16]], %[[VAL_13]]  : i64
 // CHECK:           %[[VAL_32:.*]] = llvm.mlir.constant(44 : i32) : i32
 // CHECK:           %[[VAL_33:.*]] = llvm.mlir.zero : !llvm.ptr
 // CHECK:           %[[VAL_34:.*]] = llvm.getelementptr %[[VAL_33]][1] : (!llvm.ptr) -> !llvm.ptr, i32
 // CHECK:           %[[VAL_35:.*]] = llvm.ptrtoint %[[VAL_34]] : !llvm.ptr to i64
-// CHECK:           %[[VAL_36_BYTESIZE:.*]] = llvm.mul %[[VAL_35]], %[[VAL_31_LEN]]  : i64
+// CHECK:           %[[VAL_36:.*]] = llvm.mul %[[VAL_35]], %[[VAL_31]] : i64
 // CHECK:           %[[VAL_37:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_38:.*]] = llvm.insertvalue %[[VAL_36_BYTESIZE]], %[[VAL_37]][1] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_38:.*]] = llvm.insertvalue %[[VAL_36]], %[[VAL_37]][1] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
 // CHECK:           %[[VAL_39:.*]] = llvm.mlir.constant(20240719 : i32) : i32
 // CHECK:           %[[VAL_40:.*]] = llvm.insertvalue %[[VAL_39]], %[[VAL_38]][2] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
 // CHECK:           %[[VAL_41:.*]] = llvm.mlir.constant(2 : i32) : i32
@@ -59,39 +64,39 @@
 // CHECK:           %[[VAL_49:.*]] = llvm.mlir.constant(0 : i32) : i32
 // CHECK:           %[[VAL_50:.*]] = llvm.trunc %[[VAL_49]] : i32 to i8
 // CHECK:           %[[VAL_51:.*]] = llvm.insertvalue %[[VAL_50]], %[[VAL_48]][6] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_52_c0:.*]] = llvm.mlir.constant(0 : i64) : i64
+// CHECK:           %[[VAL_52:.*]] = llvm.mlir.constant(0 : i64) : i64
 // CHECK:           %[[VAL_53:.*]] = llvm.mlir.constant(1 : i64) : i64
-// CHECK:           %[[VAL_54:.*]] = llvm.sub %[[VAL_1_SLICE_LB0]], %[[VAL_26_LB0]]  : i64
-// CHECK:           %[[VAL_55:.*]] = llvm.mul %[[VAL_54]], %[[VAL_31_LEN]]  : i64
-// CHECK:           %[[VAL_56_SLICE_OFF0:.*]] = llvm.add %[[VAL_55]], %[[VAL_52_c0]]  : i64
-// CHECK:           %[[VAL_57:.*]] = llvm.sub %[[VAL_2_SLICE_EX0]], %[[VAL_1_SLICE_LB0]]  : i64
-// CHECK:           %[[VAL_58:.*]] = llvm.add %[[VAL_57]], %[[VAL_3_SLICE_ST0]]  : i64
-// CHECK:           %[[VAL_59:.*]] = llvm.sdiv %[[VAL_58]], %[[VAL_3_SLICE_ST0]]  : i64
-// CHECK:           %[[VAL_60:.*]] = llvm.icmp "sgt" %[[VAL_59]], %[[VAL_52_c0]] : i64
-// CHECK:           %[[VAL_61:.*]] = llvm.select %[[VAL_60]], %[[VAL_59]], %[[VAL_52_c0]] : i1, i64
+// CHECK:           %[[VAL_54:.*]] = llvm.sub %[[VAL_1]], %[[VAL_26]] : i64
+// CHECK:           %[[VAL_55:.*]] = llvm.mul %[[VAL_54]], %[[VAL_31]] : i64
+// CHECK:           %[[VAL_56:.*]] = llvm.add %[[VAL_55]], %[[VAL_52]] : i64
+// CHECK:           %[[VAL_57:.*]] = llvm.sub %[[VAL_2]], %[[VAL_1]] : i64
+// CHECK:           %[[VAL_58:.*]] = llvm.add %[[VAL_57]], %[[VAL_3]] : i64
+// CHECK:           %[[VAL_59:.*]] = llvm.sdiv %[[VAL_58]], %[[VAL_3]]  : i64
+// CHECK:           %[[VAL_60:.*]] = llvm.icmp "sgt" %[[VAL_59]], %[[VAL_52]] : i64
+// CHECK:           %[[VAL_61:.*]] = llvm.select %[[VAL_60]], %[[VAL_59]], %[[VAL_52]] : i1, i64
 // CHECK:           %[[VAL_62:.*]] = llvm.insertvalue %[[VAL_53]], %[[VAL_51]][7, 0, 0] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
 // CHECK:           %[[VAL_63:.*]] = llvm.insertvalue %[[VAL_61]], %[[VAL_62]][7, 0, 1] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_64:.*]] = llvm.mul %[[VAL_36_BYTESIZE]], %[[VAL_3_SLICE_ST0]]  : i64
+// CHECK:           %[[VAL_64:.*]] = llvm.mul %[[VAL_36]], %[[VAL_3]] : i64
 // CHECK:           %[[VAL_65:.*]] = llvm.insertvalue %[[VAL_64]], %[[VAL_63]][7, 0, 2] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_66:.*]] = llvm.mul %[[VAL_36_BYTESIZE]], %[[VAL_28_EX0]]  : i64
-// CHECK:           %[[VAL_67:.*]] = llvm.mul %[[VAL_31_LEN]], %[[VAL_28_EX0]]  : i64
-// CHECK:           %[[VAL_68:.*]] = llvm.sub %[[VAL_4_SLICE_LB1]], %[[VAL_18_LB1]]  : i64
-// CHECK:           %[[VAL_69_SLICE_OFF1:.*]] = llvm.mul %[[VAL_68]], %[[VAL_67]]  : i64
-// CHECK:           %[[VAL_70_OFFSET:.*]] = llvm.add %[[VAL_69_SLICE_OFF1]], %[[VAL_56_SLICE_OFF0]]  : i64
-// CHECK:           %[[VAL_71:.*]] = llvm.sub %[[VAL_5_SLICE_EX1]], %[[VAL_4_SLICE_LB1]]  : i64
-// CHECK:           %[[VAL_72:.*]] = llvm.add %[[VAL_71]], %[[VAL_6_SLICE_ST1]]  : i64
-// CHECK:           %[[VAL_73:.*]] = llvm.sdiv %[[VAL_72]], %[[VAL_6_SLICE_ST1]]  : i64
-// CHECK:           %[[VAL_74:.*]] = llvm.icmp "sgt" %[[VAL_73]], %[[VAL_52_c0]] : i64
-// CHECK:           %[[VAL_75:.*]] = llvm.select %[[VAL_74]], %[[VAL_73]], %[[VAL_52_c0]] : i1, i64
+// CHECK:           %[[VAL_66:.*]] = llvm.mul %[[VAL_36]], %[[VAL_28]] : i64
+// CHECK:           %[[VAL_67:.*]] = llvm.mul %[[VAL_31]], %[[VAL_28]] : i64
+// CHECK:           %[[VAL_68:.*]] = llvm.sub %[[VAL_4]], %[[VAL_18]] : i64
+// CHECK:           %[[VAL_69:.*]] = llvm.mul %[[VAL_68]], %[[VAL_67]] : i64
+// CHECK:           %[[VAL_70:.*]] = llvm.add %[[VAL_69]], %[[VAL_56]] : i64
+// CHECK:           %[[VAL_71:.*]] = llvm.sub %[[VAL_5]], %[[VAL_4]] : i64
+// CHECK:           %[[VAL_72:.*]] = llvm.add %[[VAL_71]], %[[VAL_6]] : i64
+// CHECK:           %[[VAL_73:.*]] = llvm.sdiv %[[VAL_72]], %[[VAL_6]]  : i64
+// CHECK:           %[[VAL_74:.*]] = llvm.icmp "sgt" %[[VAL_73]], %[[VAL_52]] : i64
+// CHECK:           %[[VAL_75:.*]] = llvm.select %[[VAL_74]], %[[VAL_73]], %[[VAL_52]] : i1, i64
 // CHECK:           %[[VAL_76:.*]] = llvm.insertvalue %[[VAL_53]], %[[VAL_65]][7, 1, 0] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
 // CHECK:           %[[VAL_77:.*]] = llvm.insertvalue %[[VAL_75]], %[[VAL_76]][7, 1, 1] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_78:.*]] = llvm.mul %[[VAL_66]], %[[VAL_6_SLICE_ST1]]  : i64
+// CHECK:           %[[VAL_78:.*]] = llvm.mul %[[VAL_66]], %[[VAL_6]] : i64
 // CHECK:           %[[VAL_79:.*]] = llvm.insertvalue %[[VAL_78]], %[[VAL_77]][7, 1, 2] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_80:.*]] = llvm.mul %[[VAL_66]], %[[VAL_20_EX1]]  : i64
-// CHECK:           %[[VAL_81:.*]] = llvm.mul %[[VAL_67]], %[[VAL_20_EX1]]  : i64
-// CHECK:           %[[VAL_82:.*]] = llvm.getelementptr %[[VAL_24_BASEPTR]]{{\[}}%[[VAL_70_OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32
-// CHECK:           %[[VAL_84:.*]] = llvm.insertvalue %[[VAL_82]], %[[VAL_79]][0] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           llvm.store %[[VAL_84]], %[[VAL_8]] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>, !llvm.ptr
+// CHECK:           %[[VAL_80:.*]] = llvm.mul %[[VAL_66]], %[[VAL_20]] : i64
+// CHECK:           %[[VAL_81:.*]] = llvm.mul %[[VAL_67]], %[[VAL_20]] : i64
+// CHECK:           %[[VAL_82:.*]] = llvm.getelementptr %[[VAL_24]]{{\[}}%[[VAL_70]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+// CHECK:           %[[VAL_83:.*]] = llvm.insertvalue %[[VAL_82]], %[[VAL_79]][0] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           llvm.store %[[VAL_83]], %[[VAL_8]] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>, !llvm.ptr
 // CHECK:           llvm.return
 // CHECK:         }
 func.func @test_char4(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x!fir.char<4,?>>>>>, %arg1 : index, %arg2 : index, %arg3 : index, %arg4 : index, %arg5 : index, %arg6 : index) {
@@ -108,86 +113,84 @@ func.func @test_char4(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x!fir.cha
   return
 }
 
-// CHECK-LABEL:   llvm.func @test_char1(
-// CHECK-SAME:        %[[VAL_0:.*]]: !llvm.ptr,
-// CHECK-SAME:        %[[VAL_1_SLICE_LB0:.*]]: i64, %[[VAL_2_SLICE_EX0:.*]]: i64, %[[VAL_3_SLICE_ST0:.*]]: i64, %[[VAL_4_SLICE_LB1:.*]]: i64, %[[VAL_5_SLICE_EX1:.*]]: i64, %[[VAL_6_SLICE_ST1:.*]]: i64) {
+// CHECK:   llvm.func @test_char1(%[[VAL_0:.*]]: !llvm.ptr, %[[VAL_1:.*]]: i64, %[[VAL_2:.*]]: i64, %[[VAL_3:.*]]: i64, %[[VAL_4:.*]]: i64, %[[VAL_5:.*]]: i64, %[[VAL_6:.*]]: i64) {
 // CHECK:           %[[VAL_7:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:           %[[VAL_8:.*]] = llvm.alloca %[[VAL_7]] x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
 // CHECK:           %[[VAL_9:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:           %[[VAL_10:.*]] = llvm.alloca %[[VAL_9]] x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
 // CHECK:           %[[VAL_11:.*]] = llvm.mlir.constant(0 : index) : i64
-// CHECK:           %[[VAL_12_c1:.*]] = llvm.mlir.constant(1 : index) : i64
-// CHECK:           %[[VAL_14:.*]] = llvm.load %[[VAL_0]] : !llvm.ptr -> !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           llvm.store %[[VAL_14]], %[[VAL_10]] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>, !llvm.ptr
-// CHECK:           %[[VAL_15:.*]] = llvm.getelementptr %[[VAL_10]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_16_BYTESIZE:.*]] = llvm.load %[[VAL_15]] : !llvm.ptr -> i64
-// CHECK:           %[[VAL_17:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_12]], 0] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_18_LB1:.*]] = llvm.load %[[VAL_17]] : !llvm.ptr -> i64
-// CHECK:           %[[VAL_19:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_12]], 1] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_20_EX1:.*]] = llvm.load %[[VAL_19]] : !llvm.ptr -> i64
-// CHECK:           %[[VAL_21:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_12]], 2] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_22_ST1:.*]] = llvm.load %[[VAL_21]] : !llvm.ptr -> i64
-// CHECK:           %[[VAL_23:.*]] = llvm.getelementptr %[[VAL_10]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_24_BASEPTR:.*]] = llvm.load %[[VAL_23]] : !llvm.ptr -> !llvm.ptr
-// CHECK:           %[[VAL_25:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_11]], 0] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_26_LB0:.*]] = llvm.load %[[VAL_25]] : !llvm.ptr -> i64
-// CHECK:           %[[VAL_27:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_11]], 1] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_28_EX0:.*]] = llvm.load %[[VAL_27]] : !llvm.ptr -> i64
-// CHECK:           %[[VAL_29:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_11]], 2] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_30_ST0:.*]] = llvm.load %[[VAL_29]] : !llvm.ptr -> i64
-// CHECK:           %[[VAL_32:.*]] = llvm.mlir.constant(40 : i32) : i32
-// CHECK:           %[[VAL_33:.*]] = llvm.mlir.zero : !llvm.ptr
-// CHECK:           %[[VAL_34:.*]] = llvm.getelementptr %[[VAL_33]][1] : (!llvm.ptr) -> !llvm.ptr, i8
-// CHECK:           %[[VAL_35:.*]] = llvm.ptrtoint %[[VAL_34]] : !llvm.ptr to i64
-// CHECK:           %[[VAL_36_BYTESIZE:.*]] = llvm.mul %[[VAL_35]], %[[VAL_16_BYTESIZE]]  : i64
-// CHECK:           %[[VAL_37:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_38:.*]] = llvm.insertvalue %[[VAL_36_BYTESIZE]], %[[VAL_37]][1] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_39:.*]] = llvm.mlir.constant(20240719 : i32) : i32
-// CHECK:           %[[VAL_40:.*]] = llvm.insertvalue %[[VAL_39]], %[[VAL_38]][2] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_41:.*]] = llvm.mlir.constant(2 : i32) : i32
-// CHECK:           %[[VAL_42:.*]] = llvm.trunc %[[VAL_41]] : i32 to i8
-// CHECK:           %[[VAL_43:.*]] = llvm.insertvalue %[[VAL_42]], %[[VAL_40]][3] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_44:.*]] = llvm.trunc %[[VAL_32]] : i32 to i8
-// CHECK:           %[[VAL_45:.*]] = llvm.insertvalue %[[VAL_44]], %[[VAL_43]][4] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_46:.*]] = llvm.mlir.constant(0 : i32) : i32
-// CHECK:           %[[VAL_47:.*]] = llvm.trunc %[[VAL_46]] : i32 to i8
-// CHECK:           %[[VAL_48:.*]] = llvm.insertvalue %[[VAL_47]], %[[VAL_45]][5] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_49:.*]] = llvm.mlir.constant(0 : i32) : i32
-// CHECK:           %[[VAL_50:.*]] = llvm.trunc %[[VAL_49]] : i32 to i8
-// CHECK:           %[[VAL_51:.*]] = llvm.insertvalue %[[VAL_50]], %[[VAL_48]][6] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_52_c0:.*]] = llvm.mlir.constant(0 : i64) : i64
-// CHECK:           %[[VAL_53:.*]] = llvm.mlir.constant(1 : i64) : i64
-// CHECK:           %[[VAL_54:.*]] = llvm.sub %[[VAL_1_SLICE_LB0]], %[[VAL_26_LB0]]  : i64
-// CHECK:           %[[VAL_55:.*]] = llvm.mul %[[VAL_54]], %[[VAL_16_BYTESIZE]]  : i64
-// CHECK:           %[[VAL_56_SLICE_OFF0:.*]] = llvm.add %[[VAL_55]], %[[VAL_52_c0]]  : i64
-// CHECK:           %[[VAL_57:.*]] = llvm.sub %[[VAL_2_SLICE_EX0]], %[[VAL_1_SLICE_LB0]]  : i64
-// CHECK:           %[[VAL_58:.*]] = llvm.add %[[VAL_57]], %[[VAL_3_SLICE_ST0]]  : i64
-// CHECK:           %[[VAL_59:.*]] = llvm.sdiv %[[VAL_58]], %[[VAL_3_SLICE_ST0]]  : i64
-// CHECK:           %[[VAL_60:.*]] = llvm.icmp "sgt" %[[VAL_59]], %[[VAL_52_c0]] : i64
-// CHECK:           %[[VAL_61:.*]] = llvm.select %[[VAL_60]], %[[VAL_59]], %[[VAL_52_c0]] : i1, i64
-// CHECK:           %[[VAL_62:.*]] = llvm.insertvalue %[[VAL_53]], %[[VAL_51]][7, 0, 0] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_63:.*]] = llvm.insertvalue %[[VAL_61]], %[[VAL_62]][7, 0, 1] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_64:.*]] = llvm.mul %[[VAL_36_BYTESIZE]], %[[VAL_3_SLICE_ST0]]  : i64
-// CHECK:           %[[VAL_65:.*]] = llvm.insertvalue %[[VAL_64]], %[[VAL_63]][7, 0, 2] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_66:.*]] = llvm.mul %[[VAL_36_BYTESIZE]], %[[VAL_28_EX0]]  : i64
-// CHECK:           %[[VAL_67:.*]] = llvm.mul %[[VAL_16_BYTESIZE]], %[[VAL_28_EX0]]  : i64
-// CHECK:           %[[VAL_68:.*]] = llvm.sub %[[VAL_4_SLICE_LB1]], %[[VAL_18_LB1]]  : i64
-// CHECK:           %[[VAL_69_SLICE_OFF1:.*]] = llvm.mul %[[VAL_68]], %[[VAL_67]]  : i64
-// CHECK:           %[[VAL_70_OFFSET:.*]] = llvm.add %[[VAL_69_SLICE_OFF1]], %[[VAL_56_SLICE_OFF0]]  : i64
-// CHECK:           %[[VAL_71:.*]] = llvm.sub %[[VAL_5_SLICE_EX1]], %[[VAL_4_SLICE_LB1]]  : i64
-// CHECK:           %[[VAL_72:.*]] = llvm.add %[[VAL_71]], %[[VAL_6_SLICE_ST1]]  : i64
-// CHECK:           %[[VAL_73:.*]] = llvm.sdiv %[[VAL_72]], %[[VAL_6_SLICE_ST1]]  : i64
-// CHECK:           %[[VAL_74:.*]] = llvm.icmp "sgt" %[[VAL_73]], %[[VAL_52_c0]] : i64
-// CHECK:           %[[VAL_75:.*]] = llvm.select %[[VAL_74]], %[[VAL_73]], %[[VAL_52_c0]] : i1, i64
-// CHECK:           %[[VAL_76:.*]] = llvm.insertvalue %[[VAL_53]], %[[VAL_65]][7, 1, 0] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_77:.*]] = llvm.insertvalue %[[VAL_75]], %[[VAL_76]][7, 1, 1] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_78:.*]] = llvm.mul %[[VAL_66]], %[[VAL_6_SLICE_ST1]]  : i64
-// CHECK:           %[[VAL_79:.*]] = llvm.insertvalue %[[VAL_78]], %[[VAL_77]][7, 1, 2] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           %[[VAL_80:.*]] = llvm.mul %[[VAL_66]], %[[VAL_20_EX1]]  : i64
-// CHECK:           %[[VAL_81:.*]] = llvm.mul %[[VAL_67]], %[[VAL_20_EX1]]  : i64
-// CHECK:           %[[VAL_82:.*]] = llvm.getelementptr %[[VAL_24_BASEPTR]]{{\[}}%[[VAL_70_OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8
-// CHECK:           %[[VAL_84:.*]] = llvm.insertvalue %[[VAL_82]], %[[VAL_79]][0] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
-// CHECK:           llvm.store %[[VAL_84]], %[[VAL_8]] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>, !llvm.ptr
+// CHECK:           %[[VAL_12:.*]] = llvm.mlir.constant(1 : index) : i64
+// CHECK:           %[[VAL_13:.*]] = llvm.mlir.constant(72 : i32) : i32
+// CHECK:           "llvm.intr.memcpy"(%[[VAL_10]], %[[VAL_0]], %[[VAL_13]]) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i32) -> ()
+// CHECK:           %[[VAL_14:.*]] = llvm.getelementptr %[[VAL_10]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_15:.*]] = llvm.load %[[VAL_14]] : !llvm.ptr -> i64
+// CHECK:           %[[VAL_16:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_12]], 0] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_17:.*]] = llvm.load %[[VAL_16]] : !llvm.ptr -> i64
+// CHECK:           %[[VAL_18:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_12]], 1] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_19:.*]] = llvm.load %[[VAL_18]] : !llvm.ptr -> i64
+// CHECK:           %[[VAL_20:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_12]], 2] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_21:.*]] = llvm.load %[[VAL_20]] : !llvm.ptr -> i64
+// CHECK:           %[[VAL_22:.*]] = llvm.getelementptr %[[VAL_10]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_23:.*]] = llvm.load %[[VAL_22]] : !llvm.ptr -> !llvm.ptr
+// CHECK:           %[[VAL_24:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_11]], 0] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_25:.*]] = llvm.load %[[VAL_24]] : !llvm.ptr -> i64
+// CHECK:           %[[VAL_26:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_11]], 1] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_27:.*]] = llvm.load %[[VAL_26]] : !llvm.ptr -> i64
+// CHECK:           %[[VAL_28:.*]] = llvm.getelementptr %[[VAL_10]][0, 7, %[[VAL_11]], 2] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_29:.*]] = llvm.load %[[VAL_28]] : !llvm.ptr -> i64
+// CHECK:           %[[VAL_30:.*]] = llvm.mlir.constant(40 : i32) : i32
+// CHECK:           %[[VAL_31:.*]] = llvm.mlir.zero : !llvm.ptr
+// CHECK:           %[[VAL_32:.*]] = llvm.getelementptr %[[VAL_31]][1] : (!llvm.ptr) -> !llvm.ptr, i8
+// CHECK:           %[[VAL_33:.*]] = llvm.ptrtoint %[[VAL_32]] : !llvm.ptr to i64
+// CHECK:           %[[VAL_34:.*]] = llvm.mul %[[VAL_33]], %[[VAL_15]] : i64
+// CHECK:           %[[VAL_35:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_36:.*]] = llvm.insertvalue %[[VAL_34]], %[[VAL_35]][1] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_37:.*]] = llvm.mlir.constant(20240719 : i32) : i32
+// CHECK:           %[[VAL_38:.*]] = llvm.insertvalue %[[VAL_37]], %[[VAL_36]][2] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_39:.*]] = llvm.mlir.constant(2 : i32) : i32
+// CHECK:           %[[VAL_40:.*]] = llvm.trunc %[[VAL_39]] : i32 to i8
+// CHECK:           %[[VAL_41:.*]] = llvm.insertvalue %[[VAL_40]], %[[VAL_38]][3] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_42:.*]] = llvm.trunc %[[VAL_30]] : i32 to i8
+// CHECK:           %[[VAL_43:.*]] = llvm.insertvalue %[[VAL_42]], %[[VAL_41]][4] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_44:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:           %[[VAL_45:.*]] = llvm.trunc %[[VAL_44]] : i32 to i8
+// CHECK:           %[[VAL_46:.*]] = llvm.insertvalue %[[VAL_45]], %[[VAL_43]][5] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_47:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:           %[[VAL_48:.*]] = llvm.trunc %[[VAL_47]] : i32 to i8
+// CHECK:           %[[VAL_49:.*]] = llvm.insertvalue %[[VAL_48]], %[[VAL_46]][6] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_50:.*]] = llvm.mlir.constant(0 : i64) : i64
+// CHECK:           %[[VAL_51:.*]] = llvm.mlir.constant(1 : i64) : i64
+// CHECK:           %[[VAL_52:.*]] = llvm.sub %[[VAL_1]], %[[VAL_25]] : i64
+// CHECK:           %[[VAL_53:.*]] = llvm.mul %[[VAL_52]], %[[VAL_15]] : i64
+// CHECK:           %[[VAL_54:.*]] = llvm.add %[[VAL_53]], %[[VAL_50]] : i64
+// CHECK:           %[[VAL_55:.*]] = llvm.sub %[[VAL_2]], %[[VAL_1]] : i64
+// CHECK:           %[[VAL_56:.*]] = llvm.add %[[VAL_55]], %[[VAL_3]] : i64
+// CHECK:           %[[VAL_57:.*]] = llvm.sdiv %[[VAL_56]], %[[VAL_3]]  : i64
+// CHECK:           %[[VAL_58:.*]] = llvm.icmp "sgt" %[[VAL_57]], %[[VAL_50]] : i64
+// CHECK:           %[[VAL_59:.*]] = llvm.select %[[VAL_58]], %[[VAL_57]], %[[VAL_50]] : i1, i64
+// CHECK:           %[[VAL_60:.*]] = llvm.insertvalue %[[VAL_51]], %[[VAL_49]][7, 0, 0] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_61:.*]] = llvm.insertvalue %[[VAL_59]], %[[VAL_60]][7, 0, 1] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_62:.*]] = llvm.mul %[[VAL_34]], %[[VAL_3]] : i64
+// CHECK:           %[[VAL_63:.*]] = llvm.insertvalue %[[VAL_62]], %[[VAL_61]][7, 0, 2] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_64:.*]] = llvm.mul %[[VAL_34]], %[[VAL_27]] : i64
+// CHECK:           %[[VAL_65:.*]] = llvm.mul %[[VAL_15]], %[[VAL_27]] : i64
+// CHECK:           %[[VAL_66:.*]] = llvm.sub %[[VAL_4]], %[[VAL_17]] : i64
+// CHECK:           %[[VAL_67:.*]] = llvm.mul %[[VAL_66]], %[[VAL_65]] : i64
+// CHECK:           %[[VAL_68:.*]] = llvm.add %[[VAL_67]], %[[VAL_54]] : i64
+// CHECK:           %[[VAL_69:.*]] = llvm.sub %[[VAL_5]], %[[VAL_4]] : i64
+// CHECK:           %[[VAL_70:.*]] = llvm.add %[[VAL_69]], %[[VAL_6]] : i64
+// CHECK:           %[[VAL_71:.*]] = llvm.sdiv %[[VAL_70]], %[[VAL_6]]  : i64
+// CHECK:           %[[VAL_72:.*]] = llvm.icmp "sgt" %[[VAL_71]], %[[VAL_50]] : i64
+// CHECK:           %[[VAL_73:.*]] = llvm.select %[[VAL_72]], %[[VAL_71]], %[[VAL_50]] : i1, i64
+// CHECK:           %[[VAL_74:.*]] = llvm.insertvalue %[[VAL_51]], %[[VAL_63]][7, 1, 0] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_75:.*]] = llvm.insertvalue %[[VAL_73]], %[[VAL_74]][7, 1, 1] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_76:.*]] = llvm.mul %[[VAL_64]], %[[VAL_6]] : i64
+// CHECK:           %[[VAL_77:.*]] = llvm.insertvalue %[[VAL_76]], %[[VAL_75]][7, 1, 2] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           %[[VAL_78:.*]] = llvm.mul %[[VAL_64]], %[[VAL_19]] : i64
+// CHECK:           %[[VAL_79:.*]] = llvm.mul %[[VAL_65]], %[[VAL_19]] : i64
+// CHECK:           %[[VAL_80:.*]] = llvm.getelementptr %[[VAL_23]]{{\[}}%[[VAL_68]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8
+// CHECK:           %[[VAL_81:.*]] = llvm.insertvalue %[[VAL_80]], %[[VAL_77]][0] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>
+// CHECK:           llvm.store %[[VAL_81]], %[[VAL_8]] : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<2 x array<3 x i64>>)>, !llvm.ptr
 // CHECK:           llvm.return
 // CHECK:         }
 func.func @test_char1(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x!fir.char<1,?>>>>>, %arg1 : index, %arg2 : index, %arg3 : index, %arg4 : index, %arg5 : index, %arg6 : index) {
diff --git a/flang/test/Fir/polymorphic.fir b/flang/test/Fir/polymorphic.fir
index 40204314e8df..78e5b8dcf84c 100644
--- a/flang/test/Fir/polymorphic.fir
+++ b/flang/test/Fir/polymorphic.fir
@@ -14,8 +14,7 @@ func.func @_QMpolymorphic_testPtest_allocate_unlimited_polymorphic_non_derived()
 // CHECK:   %[[MEM:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }
 // CHECK:   %[[DESC:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, i64 1
 // CHECK:   store { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] } { ptr null, i64 0, i32 20240719, i8 0, i8 -1, i8 1, i8 1, ptr null, [1 x i64] zeroinitializer }, ptr %[[MEM]]
-// CHECK:   %[[LOADED:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[MEM]], align 8
-// CHECK:   store { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] } %[[LOADED]], ptr %[[DESC]]
+// CHECK:   call void @llvm.memcpy.p0.p0.i32(ptr %[[DESC]], ptr %[[MEM]], i32 40, i1 false)
 // CHECK:   ret void
 // CHECK: }
 
@@ -66,8 +65,7 @@ func.func @_QMpolymorphic_testPtest_embox() {
 // CHECK-LABEL: @_QMpolymorphic_testPtest_embox()
 // CHECK: %[[ALLOCA_DESC:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }
 // CHECK: store { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] } { ptr @_QFEy, i64 ptrtoint (ptr getelementptr (i32, ptr null, i32 1) to i64), i32 20240719, i8 1, i8 9, {{.*}}, ptr %[[ALLOCA_DESC]]
-// CHECK: %[[LOADED_DESC:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %[[ALLOCA_DESC]], align 8
-// CHECK: store { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] } %[[LOADED_DESC]], ptr @_QFEx, align 8
+// CHECK: call void @llvm.memcpy.p0.p0.i32(ptr @_QFEx, ptr %[[ALLOCA_DESC]], i32 64, i1 false)
 
 // Test emboxing of an array element from an unlimited polymorphic array.
 
@@ -158,8 +156,7 @@ func.func @_QQmain() {
 // CHECK: %[[CLASS_NONE:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }
 // CHECK: %[[DESC:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, i64 1
 // CHECK: store { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] } { ptr @_QMmod1Ea, i64 ptrtoint (ptr getelementptr (%_QMmod1TtK2, ptr null, i32 1) to i64), i32 20240719, i8 0, i8 42, i8 1, i8 1, ptr @_QMmod1EXdtXtX2, [1 x i64] zeroinitializer }, ptr %[[CLASS_NONE]], align 8
-// CHECK: %[[LOAD:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[CLASS_NONE]]
-// CHECK: store { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] } %[[LOAD]], ptr %[[DESC]]
+// CHECK: call void @llvm.memcpy.p0.p0.i32(ptr %[[DESC]], ptr %[[CLASS_NONE]], i32 40, i1 false)
 // CHECK: call void @_QMmod1Psub1(ptr %[[DESC]])
 
 fir.global @_QMmod2Ep : !fir.class<!fir.ptr<none>> {
@@ -180,8 +177,7 @@ func.func private @_FortranAPointerAssociate(!fir.ref<!fir.box<none>>, !fir.box<
 // CHECK-LABEL: define void @_QMmod2Pinitp(
 // CHECK-SAME: ptr %[[ARG0:.*]]){{.*}}{
 // CHECK: %[[ALLOCA_CLASS_NONE:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }
-// CHECK: %[[LOAD:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[ARG0]]
-// CHECK: store { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] } %[[LOAD]], ptr %[[ALLOCA_CLASS_NONE]]
+// CHECK: call void @llvm.memcpy.p0.p0.i32(ptr %[[ALLOCA_CLASS_NONE]], ptr %[[ARG0]], i32 40, i1 false)
 // CHECK: %{{.*}} = call {} @_FortranAPointerAssociate(ptr @_QMmod2Ep, ptr %[[ALLOCA_CLASS_NONE]])
 // CHECK: ret void
 
diff --git a/flang/test/Fir/tbaa.fir b/flang/test/Fir/tbaa.fir
index 809ab3a922a0..401ebbc8c49f 100644
--- a/flang/test/Fir/tbaa.fir
+++ b/flang/test/Fir/tbaa.fir
@@ -137,8 +137,8 @@ module {
 // CHECK:           %[[VAL_7:.*]] = llvm.mlir.addressof @_QFEx : !llvm.ptr
 // CHECK:           %[[VAL_8:.*]] = llvm.mlir.addressof @_QQclX2E2F64756D6D792E66393000 : !llvm.ptr
 // CHECK:           %[[VAL_10:.*]] = llvm.call @_FortranAioBeginExternalListOutput(%[[VAL_6]], %[[VAL_8]], %[[VAL_5]]) {fastmathFlags = #llvm.fastmath<contract>} : (i32, !llvm.ptr, i32) -> !llvm.ptr
-// CHECK:           %[[VAL_11:.*]] = llvm.load %[[VAL_7]] {tbaa = [#[[$BOXT]]]} : !llvm.ptr -> !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>
-// CHECK:           llvm.store %[[VAL_11]], %[[VAL_3]] {tbaa = [#[[$BOXT]]]} : !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>, !llvm.ptr
+// CHECK:           %[[VAL_11:.*]] = llvm.mlir.constant(64 : i32) : i32
+// CHECK:           "llvm.intr.memcpy"(%[[VAL_3]], %[[VAL_7]], %[[VAL_11]]) <{isVolatile = false, tbaa = [#[[$BOXT]]]}>
 // CHECK:           %[[VAL_12:.*]] = llvm.getelementptr %[[VAL_3]][0, 7, %[[VAL_4]], 0] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>
 // CHECK:           %[[VAL_13:.*]] = llvm.load %[[VAL_12]] {tbaa = [#[[$BOXT]]]} : !llvm.ptr -> i64
 // CHECK:           %[[VAL_14:.*]] = llvm.getelementptr %[[VAL_3]][0, 7, %[[VAL_4]], 1] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>
diff --git a/flang/test/Integration/OpenMP/private-global.f90 b/flang/test/Integration/OpenMP/private-global.f90
index 62d0a3faf0c5..63ac6fbe05ee 100644
--- a/flang/test/Integration/OpenMP/private-global.f90
+++ b/flang/test/Integration/OpenMP/private-global.f90
@@ -31,8 +31,9 @@ End Program
 ! CHECK:         %[[TABLE_BOX_ADDR2:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, i64 1, align 8
 ! CHECK:         %[[TABLE_BOX_VAL:.*]] = insertvalue { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] } { ptr undef, i64 ptrtoint (ptr getelementptr (i32, ptr null, i32 1) to i64), i32 20240719, i8 1, i8 9, i8 0, i8 0, [1 x [3 x i64]] {{\[\[}}3 x i64] [i64 1, i64 10, i64 ptrtoint (ptr getelementptr (i32, ptr null, i32 1) to i64)]] }, ptr %[[PRIV_TABLE]], 0
 ! CHECK:         store { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] } %[[TABLE_BOX_VAL]], ptr %[[TABLE_BOX_ADDR]], align 8
-! CHECK:         %[[TABLE_BOX_VAL2:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[TABLE_BOX_ADDR]], align 8
-! CHECK:         store { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] } %[[TABLE_BOX_VAL2]], ptr %[[TABLE_BOX_ADDR2]], align 8
+! CHECK :         %[[TABLE_BOX_VAL2:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[TABLE_BOX_ADDR]], align 8
+! CHECK :         store { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] } %[[TABLE_BOX_VAL2]], ptr %[[TABLE_BOX_ADDR2]], align 8
+! CHECK:         call void @llvm.memcpy.p0.p0.i32(ptr %[[TABLE_BOX_ADDR2]], ptr %[[TABLE_BOX_ADDR]], i32 48, i1 false)
 ! CHECK:         %[[VAL_26:.*]] = call {} @_FortranAAssign(ptr %[[TABLE_BOX_ADDR2]], ptr %[[BOXED_FIFTY]], ptr @{{.*}}, i32 9)
 ! ...
 ! check that we use the private copy of table for table/=50
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-allocatable-firstprivate.f90 b/flang/test/Lower/OpenMP/delayed-privatization-allocatable-firstprivate.f90
index 9c97c689dad7..b3a668018df1 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-allocatable-firstprivate.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-allocatable-firstprivate.f90
@@ -57,5 +57,4 @@ end program compilation_to_obj
 ! LLVM: @[[GLOB_VAR:[^[:space:]]+]]t = internal global
 
 ! LLVM: define internal void @_QQmain..omp_par
-! LLVM:      %[[GLOB_VAL:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr @[[GLOB_VAR]]t, align 8
-! LLVM-NEXT: store { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] } %[[GLOB_VAL]], ptr %{{.*}}, align 8
+! LLVM: call void @llvm.memcpy.p0.p0.i32(ptr %{{.+}}, ptr @[[GLOB_VAR]]t, i32 48, i1 false)
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-mixed.f90 b/flang/test/Lower/OpenMP/parallel-reduction-mixed.f90
index 262075ec9b25..8e6f55abd567 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-mixed.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-mixed.f90
@@ -17,7 +17,7 @@ subroutine proc
 end subroutine proc
 
 !CHECK-LABEL: define void @proc_()
-!CHECK: call void
+!CHECK: call void (ptr, i32, ptr, ...)
 !CHECK-SAME: @__kmpc_fork_call(ptr {{.*}}, i32 1, ptr @[[OMP_PAR:.*]], {{.*}})
 
 !CHECK: define internal void @[[OMP_PAR]](ptr {{.*}} %[[TID_ADDR:.*]], ptr noalias 
diff --git a/flang/test/Lower/allocatable-polymorphic.f90 b/flang/test/Lower/allocatable-polymorphic.f90
index e23e38ffb4b0..4d70e1ea4c73 100644
--- a/flang/test/Lower/allocatable-polymorphic.f90
+++ b/flang/test/Lower/allocatable-polymorphic.f90
@@ -603,10 +603,9 @@ end
 ! LLVM: %{{.*}} = call {} @_FortranAAllocatableInitDerivedForAllocate(ptr %{{.*}}, ptr @_QMpolyEXdtXp2, i32 1, i32 0)
 ! LLVM: %{{.*}} = call {} @_FortranAAllocatableSetBounds(ptr %{{.*}}, i32 0, i64 1, i64 20)
 ! LLVM: %{{.*}} = call i32 @_FortranAAllocatableAllocate(ptr %{{.*}}, i1 false, ptr null, ptr @_QQclX{{.*}}, i32 {{.*}})
-! LLVM-COUNT-2:  call void %{{.*}}()
+! LLVM-COUNT-2:  call void %{{[0-9]*}}()
 
-! LLVM: %[[C1_LOAD:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}
-! LLVM: store { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] } %[[C1_LOAD]], ptr %{{.*}}
+! LLVM: call void @llvm.memcpy.p0.p0.i32
 ! LLVM: %[[GEP_TDESC_C1:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 7
 ! LLVM: %[[TDESC_C1:.*]] = load ptr, ptr %[[GEP_TDESC_C1]]
 ! LLVM: %[[ELEM_SIZE_GEP:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 1
@@ -620,8 +619,7 @@ end
 ! LLVM: store { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] } %{{.*}}, ptr %[[TMP:.*]]
 ! LLVM: call void %{{.*}}(ptr %{{.*}}) 
 
-! LLVM: %[[LOAD_C2:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}
-! LLVM: store { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] } %[[LOAD_C2]], ptr %{{.*}}
+! LLVM: call void @llvm.memcpy.p0.p0.i32
 ! LLVM: %[[GEP_TDESC_C2:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 7
 ! LLVM: %[[TDESC_C2:.*]] = load ptr, ptr %[[GEP_TDESC_C2]]
 ! LLVM: %[[ELEM_SIZE_GEP:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 1
@@ -635,9 +633,7 @@ end
 ! LLVM: store { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] } %{{.*}}, ptr %{{.*}}
 ! LLVM: call void %{{.*}}(ptr %{{.*}})
 
-! LLVM: %[[C3_LOAD:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %{{.*}}
-! LLVM: store { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] } %[[C3_LOAD]], ptr %{{.*}}
-
+! LLVM: call void @llvm.memcpy.p0.p0.i32
 ! LLVM: %[[GEP_TDESC_C3:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 8
 ! LLVM: %[[TDESC_C3:.*]] = load ptr, ptr %[[GEP_TDESC_C3]]
 ! LLVM: %[[ELE_SIZE_GEP:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 1
@@ -658,8 +654,7 @@ end
 ! LLVM: store { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] } %[[BOX7]], ptr %{{.*}}
 ! LLVM: call void %{{.*}}(ptr %{{.*}})
 
-! LLVM: %[[C4_LOAD:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %{{.*}}
-! LLVM: store { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] } %[[C4_LOAD]], ptr %{{.*}}
+! LLVM: call void @llvm.memcpy.p0.p0.i32
 ! LLVM: %[[GEP_TDESC_C4:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 8
 ! LLVM: %[[TDESC_C4:.*]] = load ptr, ptr %[[GEP_TDESC_C4]]
 ! LLVM: %[[ELE_SIZE_GEP:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 1
@@ -686,8 +681,7 @@ end
 
 ! LLVM-LABEL: define void @_QMpolyPtest_deallocate()
 ! LLVM: store { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] } { ptr null, i64 ptrtoint (ptr getelementptr (%_QMpolyTp1, ptr null, i32 1) to i64), i32 20240719, i8 0, i8 42, i8 2, i8 1, ptr @_QMpolyEXdtXp1, [1 x i64] zeroinitializer }, ptr %[[ALLOCA1:[0-9]*]]
-! LLVM: %[[LOAD:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[ALLOCA1]]
-! LLVM: store { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] } %[[LOAD]], ptr %[[ALLOCA2:[0-9]*]]
+! LLVM: call void @llvm.memcpy.p0.p0.i32(ptr %[[ALLOCA2:[0-9]+]], ptr %[[ALLOCA1]], i32 40, i1 false)
 ! LLVM: %{{.*}} = call {} @_FortranAAllocatableInitDerivedForAllocate(ptr %[[ALLOCA2]], ptr @_QMpolyEXdtXp1, i32 0, i32 0)
 ! LLVM: %{{.*}} = call i32 @_FortranAAllocatableAllocate(ptr %[[ALLOCA2]], i1 false, ptr null, ptr @_QQclX{{.*}}, i32 {{.*}})
 ! LLVM: %{{.*}} = call i32 @_FortranAAllocatableDeallocatePolymorphic(ptr %[[ALLOCA2]], ptr {{.*}}, i1 false, ptr null, ptr @_QQclX{{.*}}, i32 {{.*}})
-- 
GitLab


From 0227b73b513154a2bde90ddf1e167b6257765d05 Mon Sep 17 00:00:00 2001
From: Brox Chen <guochen2@amd.com>
Date: Wed, 30 Oct 2024 12:50:40 -0400
Subject: [PATCH 608/693] [AMDGPU][True16][test] update VOP2 asm/dasm file with
 true16/fake16 (#113101)

This is a non-functional change

update GFX11/GFX12 VOP2 asm/dasm test for true16/fake16:

1. duplicate files to be true16/fake16 by adding
"-mattr=+real-true16/-mattr=-real-true16" while true16 test file will be
updated to true16 format when the true16 instructions are supported
2. sort "*t16_err.s" and "*t16_promote.s" tests to alphabetic order.
This is for the upcoming true16 mc changes, and mainly trying to help
repo maintainer to resolve conflicts in the tests quickly. A script is
proposed to help for the sorting
https://github.com/llvm/llvm-project/pull/111769. Since these two files
are t16 only, it should not create conflicts in downstream branches
3. add -filetype=null to seperate stdout and stderr to avoid disordered
output from llvm-mc
---
 llvm/test/MC/AMDGPU/gfx11_asm_vop2-fake16.s   | 2554 ++++++++++++++++
 llvm/test/MC/AMDGPU/gfx11_asm_vop2.s          |    8 +-
 .../MC/AMDGPU/gfx11_asm_vop2_dpp16-fake16.s   | 2114 ++++++++++++++
 llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp16.s    |    8 +-
 .../MC/AMDGPU/gfx11_asm_vop2_dpp8-fake16.s    |  451 +++
 llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp8.s     |    8 +-
 .../MC/AMDGPU/gfx11_asm_vop2_err-fake16.s     |   13 +
 llvm/test/MC/AMDGPU/gfx11_asm_vop2_err.s      |    2 +-
 llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_err.s  |  309 +-
 .../MC/AMDGPU/gfx11_asm_vop2_t16_promote.s    |  261 +-
 llvm/test/MC/AMDGPU/gfx12_asm_vop2-fake16.s   | 2560 +++++++++++++++++
 llvm/test/MC/AMDGPU/gfx12_asm_vop2.s          |    8 +-
 .../MC/AMDGPU/gfx12_asm_vop2_aliases-fake16.s |   19 +
 llvm/test/MC/AMDGPU/gfx12_asm_vop2_aliases.s  |    2 +-
 .../MC/AMDGPU/gfx12_asm_vop2_dpp16-fake16.s   | 2006 +++++++++++++
 llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s    |    8 +-
 .../MC/AMDGPU/gfx12_asm_vop2_dpp8-fake16.s    |  433 +++
 llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s     |    8 +-
 llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_err.s  |  295 +-
 .../MC/AMDGPU/gfx12_asm_vop2_t16_promote.s    |  247 +-
 .../Disassembler/AMDGPU/gfx11_dasm_vop2.txt   | 1480 +++++-----
 .../AMDGPU/gfx11_dasm_vop2_dpp16.txt          | 1231 ++++----
 .../AMDGPU/gfx11_dasm_vop2_dpp8.txt           |  181 +-
 .../Disassembler/AMDGPU/gfx12_dasm_vop2.txt   | 1584 +++++-----
 .../AMDGPU/gfx12_dasm_vop2_dpp16.txt          | 1197 ++++----
 .../AMDGPU/gfx12_dasm_vop2_dpp8.txt           |  177 +-
 26 files changed, 13826 insertions(+), 3338 deletions(-)
 create mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vop2-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp16-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp8-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vop2_err-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx12_asm_vop2-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx12_asm_vop2_aliases-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8-fake16.s

diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop2-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop2-fake16.s
new file mode 100644
index 000000000000..96dd57208943
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop2-fake16.s
@@ -0,0 +1,2554 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_add_co_ci_u32_e32 v5, vcc_lo, v1, v2, vcc_lo
+// W32: encoding: [0x01,0x05,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v255, v2, vcc_lo
+// W32: encoding: [0xff,0x05,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, s1, v2, vcc_lo
+// W32: encoding: [0x01,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, s105, v2, vcc_lo
+// W32: encoding: [0x69,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, vcc_lo, v2, vcc_lo
+// W32: encoding: [0x6a,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, vcc_hi, v2, vcc_lo
+// W32: encoding: [0x6b,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, ttmp15, v2, vcc_lo
+// W32: encoding: [0x7b,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, m0, v2, vcc_lo
+// W32: encoding: [0x7d,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, exec_lo, v2, vcc_lo
+// W32: encoding: [0x7e,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, exec_hi, v2, vcc_lo
+// W32: encoding: [0x7f,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, null, v2, vcc_lo
+// W32: encoding: [0x7c,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, -1, v2, vcc_lo
+// W32: encoding: [0xc1,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, 0.5, v2, vcc_lo
+// W32: encoding: [0xf0,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, src_scc, v2, vcc_lo
+// W32: encoding: [0xfd,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v255, vcc_lo, 0xaf123456, v255, vcc_lo
+// W32: encoding: [0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc
+// W64: encoding: [0x01,0x05,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v255, v2, vcc
+// W64: encoding: [0xff,0x05,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, s1, v2, vcc
+// W64: encoding: [0x01,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, s105, v2, vcc
+// W64: encoding: [0x69,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, vcc_lo, v2, vcc
+// W64: encoding: [0x6a,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, vcc_hi, v2, vcc
+// W64: encoding: [0x6b,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, ttmp15, v2, vcc
+// W64: encoding: [0x7b,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, m0, v2, vcc
+// W64: encoding: [0x7d,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, exec_lo, v2, vcc
+// W64: encoding: [0x7e,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, exec_hi, v2, vcc
+// W64: encoding: [0x7f,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, null, v2, vcc
+// W64: encoding: [0x7c,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, -1, v2, vcc
+// W64: encoding: [0xc1,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, 0.5, v2, vcc
+// W64: encoding: [0xf0,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, src_scc, v2, vcc
+// W64: encoding: [0xfd,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v255, vcc, 0xaf123456, v255, vcc
+// W64: encoding: [0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_f16 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x64]
+
+v_add_f16 v5, v127, v2
+// GFX11: encoding: [0x7f,0x05,0x0a,0x64]
+
+v_add_f16 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x64]
+
+v_add_f16 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x64]
+
+v_add_f16 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x64]
+
+v_add_f16 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x64]
+
+v_add_f16 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x64]
+
+v_add_f16 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x64]
+
+v_add_f16 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x64]
+
+v_add_f16 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x64]
+
+v_add_f16 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x64]
+
+v_add_f16 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x64]
+
+v_add_f16 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x64]
+
+v_add_f16 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x64]
+
+v_add_f16 v127, 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00]
+
+v_add_f32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x06]
+
+v_add_f32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x06]
+
+v_add_f32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x06]
+
+v_add_f32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x06]
+
+v_add_f32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x06]
+
+v_add_f32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x06]
+
+v_add_f32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x06]
+
+v_add_f32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x06]
+
+v_add_f32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x06]
+
+v_add_f32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x06]
+
+v_add_f32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x06]
+
+v_add_f32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x06]
+
+v_add_f32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x06]
+
+v_add_f32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x06]
+
+v_add_f32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x07,0x56,0x34,0x12,0xaf]
+
+v_add_nc_u32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x4a]
+
+v_add_nc_u32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x4a]
+
+v_add_nc_u32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x4b,0x56,0x34,0x12,0xaf]
+
+v_and_b32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x36]
+
+v_and_b32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x36]
+
+v_and_b32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x36]
+
+v_and_b32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x36]
+
+v_and_b32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x36]
+
+v_and_b32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x36]
+
+v_and_b32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x36]
+
+v_and_b32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x36]
+
+v_and_b32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x36]
+
+v_and_b32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x36]
+
+v_and_b32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x36]
+
+v_and_b32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x36]
+
+v_and_b32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x36]
+
+v_and_b32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x36]
+
+v_and_b32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x37,0x56,0x34,0x12,0xaf]
+
+v_ashrrev_i32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x34]
+
+v_ashrrev_i32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x34]
+
+v_ashrrev_i32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x35,0x56,0x34,0x12,0xaf]
+
+v_cndmask_b32 v5, v1, v2, vcc_lo
+// W32: encoding: [0x01,0x05,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v255, v2, vcc_lo
+// W32: encoding: [0xff,0x05,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, s1, v2, vcc_lo
+// W32: encoding: [0x01,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, s105, v2, vcc_lo
+// W32: encoding: [0x69,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, vcc_lo, v2, vcc_lo
+// W32: encoding: [0x6a,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, vcc_hi, v2, vcc_lo
+// W32: encoding: [0x6b,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, ttmp15, v2, vcc_lo
+// W32: encoding: [0x7b,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, m0, v2, vcc_lo
+// W32: encoding: [0x7d,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, exec_lo, v2, vcc_lo
+// W32: encoding: [0x7e,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, exec_hi, v2, vcc_lo
+// W32: encoding: [0x7f,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, null, v2, vcc_lo
+// W32: encoding: [0x7c,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, -1, v2, vcc_lo
+// W32: encoding: [0xc1,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, 0.5, v2, vcc_lo
+// W32: encoding: [0xf0,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, src_scc, v2, vcc_lo
+// W32: encoding: [0xfd,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v255, 0xaf123456, v255, vcc_lo
+// W32: encoding: [0xff,0xfe,0xff,0x03,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc
+// W64: encoding: [0x01,0x05,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v255, v2, vcc
+// W64: encoding: [0xff,0x05,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, s1, v2, vcc
+// W64: encoding: [0x01,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, s105, v2, vcc
+// W64: encoding: [0x69,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, vcc_lo, v2, vcc
+// W64: encoding: [0x6a,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, vcc_hi, v2, vcc
+// W64: encoding: [0x6b,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, ttmp15, v2, vcc
+// W64: encoding: [0x7b,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, m0, v2, vcc
+// W64: encoding: [0x7d,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, exec_lo, v2, vcc
+// W64: encoding: [0x7e,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, exec_hi, v2, vcc
+// W64: encoding: [0x7f,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, null, v2, vcc
+// W64: encoding: [0x7c,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, -1, v2, vcc
+// W64: encoding: [0xc1,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, 0.5, v2, vcc
+// W64: encoding: [0xf0,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, src_scc, v2, vcc
+// W64: encoding: [0xfd,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v255, 0xaf123456, v255, vcc
+// W64: encoding: [0xff,0xfe,0xff,0x03,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x5f,0x56,0x34,0x12,0xaf]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x5f,0x56,0x34,0x12,0xaf]
+
+v_dot2acc_f32_f16 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x04]
+
+v_dot2acc_f32_f16 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x04]
+
+v_dot2acc_f32_f16 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x04]
+
+v_dot2acc_f32_f16 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x04]
+
+v_dot2acc_f32_f16 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x04]
+
+v_dot2acc_f32_f16 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x04]
+
+v_dot2acc_f32_f16 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x04]
+
+v_dot2acc_f32_f16 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x04]
+
+v_dot2acc_f32_f16 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x04]
+
+v_dot2acc_f32_f16 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x04]
+
+v_dot2acc_f32_f16 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x04]
+
+v_dot2acc_f32_f16 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x04]
+
+v_dot2acc_f32_f16 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x04]
+
+v_dot2acc_f32_f16 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x04]
+
+v_dot2acc_f32_f16 v255, 0xfe0b, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x05,0x0b,0xfe,0x00,0x00]
+
+v_dot2c_f32_f16 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x04]
+
+v_dot2c_f32_f16 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x04]
+
+v_dot2c_f32_f16 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x04]
+
+v_dot2c_f32_f16 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x04]
+
+v_dot2c_f32_f16 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x04]
+
+v_dot2c_f32_f16 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x04]
+
+v_dot2c_f32_f16 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x04]
+
+v_dot2c_f32_f16 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x04]
+
+v_dot2c_f32_f16 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x04]
+
+v_dot2c_f32_f16 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x04]
+
+v_dot2c_f32_f16 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x04]
+
+v_dot2c_f32_f16 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x04]
+
+v_dot2c_f32_f16 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x04]
+
+v_dot2c_f32_f16 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x04]
+
+v_dot2c_f32_f16 v255, 0xfe0b, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x05,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, v1, v2, 0xfe0b
+// GFX11: encoding: [0x01,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, v127, v2, 0xfe0b
+// GFX11: encoding: [0x7f,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, s1, v2, 0xfe0b
+// GFX11: encoding: [0x01,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, s105, v2, 0xfe0b
+// GFX11: encoding: [0x69,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, vcc_lo, v2, 0xfe0b
+// GFX11: encoding: [0x6a,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, vcc_hi, v2, 0xfe0b
+// GFX11: encoding: [0x6b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, ttmp15, v2, 0xfe0b
+// GFX11: encoding: [0x7b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, m0, v2, 0xfe0b
+// GFX11: encoding: [0x7d,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, exec_lo, v2, 0xfe0b
+// GFX11: encoding: [0x7e,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, exec_hi, v2, 0xfe0b
+// GFX11: encoding: [0x7f,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, null, v2, 0xfe0b
+// GFX11: encoding: [0x7c,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, -1, v2, 0xfe0b
+// GFX11: encoding: [0xc1,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, 0.5, v2, 0xfe0b
+// GFX11: encoding: [0xf0,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, src_scc, v2, 0xfe0b
+// GFX11: encoding: [0xfd,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v127, 0xfe0b, v127, 0xfe0b
+// GFX11: encoding: [0xff,0xfe,0xfe,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f32 v5, v1, v2, 0xaf123456
+// GFX11: encoding: [0x01,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, v255, v2, 0xaf123456
+// GFX11: encoding: [0xff,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, s1, v2, 0xaf123456
+// GFX11: encoding: [0x01,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, s105, v2, 0xaf123456
+// GFX11: encoding: [0x69,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, vcc_lo, v2, 0xaf123456
+// GFX11: encoding: [0x6a,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, vcc_hi, v2, 0xaf123456
+// GFX11: encoding: [0x6b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, ttmp15, v2, 0xaf123456
+// GFX11: encoding: [0x7b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, m0, v2, 0xaf123456
+// GFX11: encoding: [0x7d,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, exec_lo, v2, 0xaf123456
+// GFX11: encoding: [0x7e,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, exec_hi, v2, 0xaf123456
+// GFX11: encoding: [0x7f,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, null, v2, 0xaf123456
+// GFX11: encoding: [0x7c,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, -1, v2, 0xaf123456
+// GFX11: encoding: [0xc1,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, 0.5, v2, 0xaf123456
+// GFX11: encoding: [0xf0,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, src_scc, v2, 0xaf123456
+// GFX11: encoding: [0xfd,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v255, 0xaf123456, v255, 0xaf123456
+// GFX11: encoding: [0xff,0xfe,0xff,0x5b,0x56,0x34,0x12,0xaf]
+
+v_fmac_dx9_zero_f32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x0c]
+
+v_fmac_dx9_zero_f32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x0c]
+
+v_fmac_dx9_zero_f32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x0c]
+
+v_fmac_dx9_zero_f32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x0c]
+
+v_fmac_dx9_zero_f32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x0c]
+
+v_fmac_dx9_zero_f32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x0c]
+
+v_fmac_dx9_zero_f32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x0c]
+
+v_fmac_dx9_zero_f32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x0c]
+
+v_fmac_dx9_zero_f32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x0c]
+
+v_fmac_dx9_zero_f32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x0c]
+
+v_fmac_dx9_zero_f32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x0c]
+
+v_fmac_dx9_zero_f32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x0c]
+
+v_fmac_dx9_zero_f32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x0c]
+
+v_fmac_dx9_zero_f32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x0c]
+
+v_fmac_dx9_zero_f32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x0d,0x56,0x34,0x12,0xaf]
+
+v_fmac_f16 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x6c]
+
+v_fmac_f16 v5, v127, v2
+// GFX11: encoding: [0x7f,0x05,0x0a,0x6c]
+
+v_fmac_f16 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x6c]
+
+v_fmac_f16 v127, 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0xfe,0x6c,0x0b,0xfe,0x00,0x00]
+
+v_fmac_f32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x56]
+
+v_fmac_f32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x56]
+
+v_fmac_f32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x56]
+
+v_fmac_f32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x57,0x56,0x34,0x12,0xaf]
+
+v_fmac_legacy_f32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x0c]
+
+v_fmac_legacy_f32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x0c]
+
+v_fmac_legacy_f32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x0c]
+
+v_fmac_legacy_f32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x0c]
+
+v_fmac_legacy_f32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x0c]
+
+v_fmac_legacy_f32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x0c]
+
+v_fmac_legacy_f32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x0c]
+
+v_fmac_legacy_f32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x0c]
+
+v_fmac_legacy_f32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x0c]
+
+v_fmac_legacy_f32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x0c]
+
+v_fmac_legacy_f32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x0c]
+
+v_fmac_legacy_f32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x0c]
+
+v_fmac_legacy_f32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x0c]
+
+v_fmac_legacy_f32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x0c]
+
+v_fmac_legacy_f32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x0d,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f16 v5, v1, 0xfe0b, v3
+// GFX11: encoding: [0x01,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, v127, 0xfe0b, v3
+// GFX11: encoding: [0x7f,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, s1, 0xfe0b, v3
+// GFX11: encoding: [0x01,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, s105, 0xfe0b, v3
+// GFX11: encoding: [0x69,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, vcc_lo, 0xfe0b, v3
+// GFX11: encoding: [0x6a,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, vcc_hi, 0xfe0b, v3
+// GFX11: encoding: [0x6b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, ttmp15, 0xfe0b, v3
+// GFX11: encoding: [0x7b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, m0, 0xfe0b, v3
+// GFX11: encoding: [0x7d,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, exec_lo, 0xfe0b, v3
+// GFX11: encoding: [0x7e,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, exec_hi, 0xfe0b, v3
+// GFX11: encoding: [0x7f,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, null, 0xfe0b, v3
+// GFX11: encoding: [0x7c,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, -1, 0xfe0b, v3
+// GFX11: encoding: [0xc1,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, 0.5, 0xfe0b, v3
+// GFX11: encoding: [0xf0,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, src_scc, 0xfe0b, v3
+// GFX11: encoding: [0xfd,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v127, 0xfe0b, 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0xfe,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f32 v5, v1, 0xaf123456, v3
+// GFX11: encoding: [0x01,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, v255, 0xaf123456, v3
+// GFX11: encoding: [0xff,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, s1, 0xaf123456, v3
+// GFX11: encoding: [0x01,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, s105, 0xaf123456, v3
+// GFX11: encoding: [0x69,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, vcc_lo, 0xaf123456, v3
+// GFX11: encoding: [0x6a,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, vcc_hi, 0xaf123456, v3
+// GFX11: encoding: [0x6b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, ttmp15, 0xaf123456, v3
+// GFX11: encoding: [0x7b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, m0, 0xaf123456, v3
+// GFX11: encoding: [0x7d,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, exec_lo, 0xaf123456, v3
+// GFX11: encoding: [0x7e,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, exec_hi, 0xaf123456, v3
+// GFX11: encoding: [0x7f,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, null, 0xaf123456, v3
+// GFX11: encoding: [0x7c,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, -1, 0xaf123456, v3
+// GFX11: encoding: [0xc1,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, 0.5, 0xaf123456, v3
+// GFX11: encoding: [0xf0,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, src_scc, 0xaf123456, v3
+// GFX11: encoding: [0xfd,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v255, 0xaf123456, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x59,0x56,0x34,0x12,0xaf]
+
+v_ldexp_f16 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x76]
+
+v_ldexp_f16 v5, v127, v2
+// GFX11: encoding: [0x7f,0x05,0x0a,0x76]
+
+v_ldexp_f16 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x76]
+
+v_ldexp_f16 v127, 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0xfe,0x76,0x0b,0xfe,0x00,0x00]
+
+v_lshlrev_b32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x30]
+
+v_lshlrev_b32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x30]
+
+v_lshlrev_b32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x31,0x56,0x34,0x12,0xaf]
+
+v_lshrrev_b32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x32]
+
+v_lshrrev_b32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x32]
+
+v_lshrrev_b32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x33,0x56,0x34,0x12,0xaf]
+
+v_max_f16 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x72]
+
+v_max_f16 v5, v127, v2
+// GFX11: encoding: [0x7f,0x05,0x0a,0x72]
+
+v_max_f16 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x72]
+
+v_max_f16 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x72]
+
+v_max_f16 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x72]
+
+v_max_f16 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x72]
+
+v_max_f16 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x72]
+
+v_max_f16 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x72]
+
+v_max_f16 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x72]
+
+v_max_f16 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x72]
+
+v_max_f16 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x72]
+
+v_max_f16 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x72]
+
+v_max_f16 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x72]
+
+v_max_f16 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x72]
+
+v_max_f16 v127, 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0xfe,0x72,0x0b,0xfe,0x00,0x00]
+
+v_max_f32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x20]
+
+v_max_f32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x20]
+
+v_max_f32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x20]
+
+v_max_f32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x20]
+
+v_max_f32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x20]
+
+v_max_f32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x20]
+
+v_max_f32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x20]
+
+v_max_f32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x20]
+
+v_max_f32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x20]
+
+v_max_f32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x20]
+
+v_max_f32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x20]
+
+v_max_f32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x20]
+
+v_max_f32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x20]
+
+v_max_f32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x20]
+
+v_max_f32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x21,0x56,0x34,0x12,0xaf]
+
+v_max_i32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x24]
+
+v_max_i32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x24]
+
+v_max_i32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x24]
+
+v_max_i32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x24]
+
+v_max_i32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x24]
+
+v_max_i32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x24]
+
+v_max_i32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x24]
+
+v_max_i32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x24]
+
+v_max_i32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x24]
+
+v_max_i32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x24]
+
+v_max_i32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x24]
+
+v_max_i32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x24]
+
+v_max_i32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x24]
+
+v_max_i32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x24]
+
+v_max_i32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x25,0x56,0x34,0x12,0xaf]
+
+v_max_u32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x28]
+
+v_max_u32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x28]
+
+v_max_u32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x28]
+
+v_max_u32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x28]
+
+v_max_u32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x28]
+
+v_max_u32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x28]
+
+v_max_u32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x28]
+
+v_max_u32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x28]
+
+v_max_u32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x28]
+
+v_max_u32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x28]
+
+v_max_u32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x28]
+
+v_max_u32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x28]
+
+v_max_u32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x28]
+
+v_max_u32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x28]
+
+v_max_u32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x29,0x56,0x34,0x12,0xaf]
+
+v_min_f16 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x74]
+
+v_min_f16 v5, v127, v2
+// GFX11: encoding: [0x7f,0x05,0x0a,0x74]
+
+v_min_f16 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x74]
+
+v_min_f16 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x74]
+
+v_min_f16 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x74]
+
+v_min_f16 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x74]
+
+v_min_f16 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x74]
+
+v_min_f16 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x74]
+
+v_min_f16 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x74]
+
+v_min_f16 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x74]
+
+v_min_f16 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x74]
+
+v_min_f16 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x74]
+
+v_min_f16 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x74]
+
+v_min_f16 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x74]
+
+v_min_f16 v127, 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0xfe,0x74,0x0b,0xfe,0x00,0x00]
+
+v_min_f32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x1e]
+
+v_min_f32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x1e]
+
+v_min_f32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x1e]
+
+v_min_f32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x1e]
+
+v_min_f32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x1e]
+
+v_min_f32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x1e]
+
+v_min_f32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x1e]
+
+v_min_f32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x1e]
+
+v_min_f32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x1e]
+
+v_min_f32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x1e]
+
+v_min_f32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x1e]
+
+v_min_f32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x1e]
+
+v_min_f32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x1e]
+
+v_min_f32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x1e]
+
+v_min_f32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x1f,0x56,0x34,0x12,0xaf]
+
+v_min_i32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x22]
+
+v_min_i32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x22]
+
+v_min_i32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x22]
+
+v_min_i32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x22]
+
+v_min_i32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x22]
+
+v_min_i32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x22]
+
+v_min_i32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x22]
+
+v_min_i32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x22]
+
+v_min_i32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x22]
+
+v_min_i32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x22]
+
+v_min_i32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x22]
+
+v_min_i32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x22]
+
+v_min_i32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x22]
+
+v_min_i32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x22]
+
+v_min_i32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x23,0x56,0x34,0x12,0xaf]
+
+v_min_u32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x26]
+
+v_min_u32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x26]
+
+v_min_u32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x26]
+
+v_min_u32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x26]
+
+v_min_u32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x26]
+
+v_min_u32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x26]
+
+v_min_u32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x26]
+
+v_min_u32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x26]
+
+v_min_u32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x26]
+
+v_min_u32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x26]
+
+v_min_u32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x26]
+
+v_min_u32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x26]
+
+v_min_u32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x26]
+
+v_min_u32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x26]
+
+v_min_u32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x27,0x56,0x34,0x12,0xaf]
+
+v_mul_dx9_zero_f32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x0f,0x56,0x34,0x12,0xaf]
+
+v_mul_f16 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x6a]
+
+v_mul_f16 v5, v127, v2
+// GFX11: encoding: [0x7f,0x05,0x0a,0x6a]
+
+v_mul_f16 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x6a]
+
+v_mul_f16 v127, 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0xfe,0x6a,0x0b,0xfe,0x00,0x00]
+
+v_mul_f32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x10]
+
+v_mul_f32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x10]
+
+v_mul_f32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x10]
+
+v_mul_f32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x10]
+
+v_mul_f32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x10]
+
+v_mul_f32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x10]
+
+v_mul_f32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x10]
+
+v_mul_f32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x10]
+
+v_mul_f32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x10]
+
+v_mul_f32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x10]
+
+v_mul_f32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x10]
+
+v_mul_f32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x10]
+
+v_mul_f32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x10]
+
+v_mul_f32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x10]
+
+v_mul_f32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x11,0x56,0x34,0x12,0xaf]
+
+v_mul_hi_i32_i24 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x15,0x56,0x34,0x12,0xaf]
+
+v_mul_hi_u32_u24 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x19,0x56,0x34,0x12,0xaf]
+
+v_mul_i32_i24 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x12]
+
+v_mul_i32_i24 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x12]
+
+v_mul_i32_i24 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x13,0x56,0x34,0x12,0xaf]
+
+v_mul_legacy_f32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x0f,0x56,0x34,0x12,0xaf]
+
+v_mul_u32_u24 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x16]
+
+v_mul_u32_u24 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x16]
+
+v_mul_u32_u24 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x17,0x56,0x34,0x12,0xaf]
+
+v_or_b32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x38]
+
+v_or_b32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x38]
+
+v_or_b32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x38]
+
+v_or_b32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x38]
+
+v_or_b32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x38]
+
+v_or_b32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x38]
+
+v_or_b32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x38]
+
+v_or_b32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x38]
+
+v_or_b32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x38]
+
+v_or_b32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x38]
+
+v_or_b32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x38]
+
+v_or_b32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x38]
+
+v_or_b32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x38]
+
+v_or_b32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x38]
+
+v_or_b32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x39,0x56,0x34,0x12,0xaf]
+
+v_pk_fmac_f16 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x78]
+
+v_pk_fmac_f16 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x78]
+
+v_pk_fmac_f16 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v255, 0xfe0b, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x79,0x0b,0xfe,0x00,0x00]
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo
+// W32: encoding: [0x01,0x05,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v255, v2, vcc_lo
+// W32: encoding: [0xff,0x05,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, s1, v2, vcc_lo
+// W32: encoding: [0x01,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, s105, v2, vcc_lo
+// W32: encoding: [0x69,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, vcc_lo, v2, vcc_lo
+// W32: encoding: [0x6a,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, vcc_hi, v2, vcc_lo
+// W32: encoding: [0x6b,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, ttmp15, v2, vcc_lo
+// W32: encoding: [0x7b,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, m0, v2, vcc_lo
+// W32: encoding: [0x7d,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, exec_lo, v2, vcc_lo
+// W32: encoding: [0x7e,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, exec_hi, v2, vcc_lo
+// W32: encoding: [0x7f,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, null, v2, vcc_lo
+// W32: encoding: [0x7c,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, -1, v2, vcc_lo
+// W32: encoding: [0xc1,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, 0.5, v2, vcc_lo
+// W32: encoding: [0xf0,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, src_scc, v2, vcc_lo
+// W32: encoding: [0xfd,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v255, vcc_lo, 0xaf123456, v255, vcc_lo
+// W32: encoding: [0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc
+// W64: encoding: [0x01,0x05,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v255, v2, vcc
+// W64: encoding: [0xff,0x05,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, s1, v2, vcc
+// W64: encoding: [0x01,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, s105, v2, vcc
+// W64: encoding: [0x69,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, vcc_lo, v2, vcc
+// W64: encoding: [0x6a,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, vcc_hi, v2, vcc
+// W64: encoding: [0x6b,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, ttmp15, v2, vcc
+// W64: encoding: [0x7b,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, m0, v2, vcc
+// W64: encoding: [0x7d,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, exec_lo, v2, vcc
+// W64: encoding: [0x7e,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, exec_hi, v2, vcc
+// W64: encoding: [0x7f,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, null, v2, vcc
+// W64: encoding: [0x7c,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, -1, v2, vcc
+// W64: encoding: [0xc1,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, 0.5, v2, vcc
+// W64: encoding: [0xf0,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, src_scc, v2, vcc
+// W64: encoding: [0xfd,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v255, vcc, 0xaf123456, v255, vcc
+// W64: encoding: [0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_f16 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x66]
+
+v_sub_f16 v5, v127, v2
+// GFX11: encoding: [0x7f,0x05,0x0a,0x66]
+
+v_sub_f16 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x66]
+
+v_sub_f16 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x66]
+
+v_sub_f16 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x66]
+
+v_sub_f16 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x66]
+
+v_sub_f16 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x66]
+
+v_sub_f16 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x66]
+
+v_sub_f16 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x66]
+
+v_sub_f16 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x66]
+
+v_sub_f16 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x66]
+
+v_sub_f16 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x66]
+
+v_sub_f16 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x66]
+
+v_sub_f16 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x66]
+
+v_sub_f16 v127, 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00]
+
+v_sub_f32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x08]
+
+v_sub_f32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x08]
+
+v_sub_f32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x08]
+
+v_sub_f32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x08]
+
+v_sub_f32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x08]
+
+v_sub_f32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x08]
+
+v_sub_f32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x08]
+
+v_sub_f32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x08]
+
+v_sub_f32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x08]
+
+v_sub_f32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x08]
+
+v_sub_f32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x08]
+
+v_sub_f32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x08]
+
+v_sub_f32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x08]
+
+v_sub_f32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x08]
+
+v_sub_f32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x09,0x56,0x34,0x12,0xaf]
+
+v_sub_nc_u32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x4c]
+
+v_sub_nc_u32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x4c]
+
+v_sub_nc_u32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x4d,0x56,0x34,0x12,0xaf]
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo
+// W32: encoding: [0x01,0x05,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v255, v2, vcc_lo
+// W32: encoding: [0xff,0x05,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, s1, v2, vcc_lo
+// W32: encoding: [0x01,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, s105, v2, vcc_lo
+// W32: encoding: [0x69,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, vcc_lo, v2, vcc_lo
+// W32: encoding: [0x6a,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, vcc_hi, v2, vcc_lo
+// W32: encoding: [0x6b,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, ttmp15, v2, vcc_lo
+// W32: encoding: [0x7b,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, m0, v2, vcc_lo
+// W32: encoding: [0x7d,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, exec_lo, v2, vcc_lo
+// W32: encoding: [0x7e,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, exec_hi, v2, vcc_lo
+// W32: encoding: [0x7f,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, null, v2, vcc_lo
+// W32: encoding: [0x7c,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, -1, v2, vcc_lo
+// W32: encoding: [0xc1,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, 0.5, v2, vcc_lo
+// W32: encoding: [0xf0,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, src_scc, v2, vcc_lo
+// W32: encoding: [0xfd,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v255, vcc_lo, 0xaf123456, v255, vcc_lo
+// W32: encoding: [0xff,0xfe,0xff,0x45,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc
+// W64: encoding: [0x01,0x05,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v255, v2, vcc
+// W64: encoding: [0xff,0x05,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, s1, v2, vcc
+// W64: encoding: [0x01,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, s105, v2, vcc
+// W64: encoding: [0x69,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, vcc_lo, v2, vcc
+// W64: encoding: [0x6a,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, vcc_hi, v2, vcc
+// W64: encoding: [0x6b,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, ttmp15, v2, vcc
+// W64: encoding: [0x7b,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, m0, v2, vcc
+// W64: encoding: [0x7d,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, exec_lo, v2, vcc
+// W64: encoding: [0x7e,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, exec_hi, v2, vcc
+// W64: encoding: [0x7f,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, null, v2, vcc
+// W64: encoding: [0x7c,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, -1, v2, vcc
+// W64: encoding: [0xc1,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, 0.5, v2, vcc
+// W64: encoding: [0xf0,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, src_scc, v2, vcc
+// W64: encoding: [0xfd,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v255, vcc, 0xaf123456, v255, vcc
+// W64: encoding: [0xff,0xfe,0xff,0x45,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_f16 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x68]
+
+v_subrev_f16 v5, v127, v2
+// GFX11: encoding: [0x7f,0x05,0x0a,0x68]
+
+v_subrev_f16 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x68]
+
+v_subrev_f16 v127, 0xfe0b, v127
+// GFX11: encoding: [0xff,0xfe,0xfe,0x68,0x0b,0xfe,0x00,0x00]
+
+v_subrev_f32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x0a]
+
+v_subrev_f32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x0a]
+
+v_subrev_f32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x0a]
+
+v_subrev_f32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x0b,0x56,0x34,0x12,0xaf]
+
+v_subrev_nc_u32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x4f,0x56,0x34,0x12,0xaf]
+
+v_xnor_b32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x3c]
+
+v_xnor_b32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x3c]
+
+v_xnor_b32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x3c]
+
+v_xnor_b32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x3d,0x56,0x34,0x12,0xaf]
+
+v_xor_b32 v5, v1, v2
+// GFX11: encoding: [0x01,0x05,0x0a,0x3a]
+
+v_xor_b32 v5, v255, v2
+// GFX11: encoding: [0xff,0x05,0x0a,0x3a]
+
+v_xor_b32 v5, s1, v2
+// GFX11: encoding: [0x01,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, s105, v2
+// GFX11: encoding: [0x69,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, vcc_lo, v2
+// GFX11: encoding: [0x6a,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, vcc_hi, v2
+// GFX11: encoding: [0x6b,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, ttmp15, v2
+// GFX11: encoding: [0x7b,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, m0, v2
+// GFX11: encoding: [0x7d,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, exec_lo, v2
+// GFX11: encoding: [0x7e,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, exec_hi, v2
+// GFX11: encoding: [0x7f,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, null, v2
+// GFX11: encoding: [0x7c,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, -1, v2
+// GFX11: encoding: [0xc1,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, 0.5, v2
+// GFX11: encoding: [0xf0,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, src_scc, v2
+// GFX11: encoding: [0xfd,0x04,0x0a,0x3a]
+
+v_xor_b32 v255, 0xaf123456, v255
+// GFX11: encoding: [0xff,0xfe,0xff,0x3b,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop2.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop2.s
index fb300b2e9497..2a4b3ea20170 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop2.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop2.s
@@ -1,7 +1,7 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_add_co_ci_u32_e32 v5, vcc_lo, v1, v2, vcc_lo
 // W32: encoding: [0x01,0x05,0x0a,0x40]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp16-fake16.s
new file mode 100644
index 000000000000..6b9092f501e5
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp16-fake16.s
@@ -0,0 +1,2114 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_half_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shl:1
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shl:15
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shr:1
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shr:15
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_ror:1
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_ror:15
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v255, vcc_lo, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0xff,0x41,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_half_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_shl:1
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_shl:15
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_shr:1
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_shr:15
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_ror:1
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_ror:15
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0xff,0x41,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
+
+v_add_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x64,0x01,0xe4,0x00,0xff]
+
+v_add_f16 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x40,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x41,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x01,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x0f,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x11,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1f,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x21,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x2f,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01]
+
+v_add_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x60,0x09,0x13]
+
+v_add_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xf5,0x30]
+
+v_add_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1b,0x00,0xff]
+
+v_add_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x06,0x01,0xe4,0x00,0xff]
+
+v_add_f32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x40,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x41,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x01,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x0f,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x11,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1f,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x21,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x2f,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x50,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x5f,0x01,0x01]
+
+v_add_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x60,0x09,0x13]
+
+v_add_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x07,0xff,0x6f,0xf5,0x30]
+
+v_add_nc_u32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x1b,0x00,0xff]
+
+v_add_nc_u32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0xe4,0x00,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x40,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x41,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x01,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x0f,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x11,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x1f,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x21,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x2f,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x50,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x5f,0x01,0x01]
+
+v_add_nc_u32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x60,0x09,0x13]
+
+v_add_nc_u32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x4b,0xff,0x6f,0x05,0x30]
+
+v_and_b32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x1b,0x00,0xff]
+
+v_and_b32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x36,0x01,0xe4,0x00,0xff]
+
+v_and_b32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x40,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x41,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x01,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x0f,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x11,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x1f,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x21,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x2f,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x50,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x5f,0x01,0x01]
+
+v_and_b32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x60,0x09,0x13]
+
+v_and_b32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x37,0xff,0x6f,0x05,0x30]
+
+v_ashrrev_i32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x1b,0x00,0xff]
+
+v_ashrrev_i32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x34,0x01,0xe4,0x00,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x40,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x41,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x01,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x0f,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x11,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x1f,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x21,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x2f,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x50,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x5f,0x01,0x01]
+
+v_ashrrev_i32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x60,0x09,0x13]
+
+v_ashrrev_i32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x35,0xff,0x6f,0x05,0x30]
+
+v_cndmask_b32 v5, v1, v2, vcc_lo quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_half_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_shl:1
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_shl:15
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_shr:1
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_shr:15
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_ror:1
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_ror:15
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, -v1, |v2|, vcc_lo quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 bound_ctrl:0
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x90,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, |v1|, -v2, vcc_lo quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 bound_ctrl:0
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x60,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, -|v1|, -|v2|, vcc_lo quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 bound_ctrl:0
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0xf0,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v255, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0xff,0x03,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_half_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_shl:1
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_shl:15
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_shr:1
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_shr:15
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_ror:1
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_ror:15
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v255, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0xff,0x03,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32_dpp v5, -v1, |v2|, vcc quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 bound_ctrl:0
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x90,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32_dpp v5, |v1|, -v2, vcc quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 bound_ctrl:0
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x60,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32_dpp v5, -|v1|, -|v2|, vcc quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 bound_ctrl:0
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0xf0,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1b,0x00,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0xe4,0x00,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x40,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x41,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x01,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x0f,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x11,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1f,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x21,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x2f,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x50,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x5f,0x01,0x01]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x60,0x09,0x13]
+
+v_cvt_pk_rtz_f16_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x5f,0xff,0x6f,0xf5,0x30]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1b,0x00,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0xe4,0x00,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x40,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x41,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x01,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x0f,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x11,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1f,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x21,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x2f,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x50,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x5f,0x01,0x01]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x60,0x09,0x13]
+
+v_cvt_pkrtz_f16_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x5f,0xff,0x6f,0xf5,0x30]
+
+v_dot2acc_f32_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x1b,0x00,0xff]
+
+v_dot2acc_f32_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0xe4,0x00,0xff]
+
+v_dot2acc_f32_f16 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x40,0x01,0xff]
+
+v_dot2acc_f32_f16 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x41,0x01,0xff]
+
+v_dot2acc_f32_f16 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x01,0x01,0xff]
+
+v_dot2acc_f32_f16 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x0f,0x01,0xff]
+
+v_dot2acc_f32_f16 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x11,0x01,0xff]
+
+v_dot2acc_f32_f16 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x1f,0x01,0xff]
+
+v_dot2acc_f32_f16 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x21,0x01,0xff]
+
+v_dot2acc_f32_f16 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x2f,0x01,0xff]
+
+v_dot2acc_f32_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x50,0x01,0xff]
+
+v_dot2acc_f32_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x5f,0x01,0x01]
+
+v_dot2acc_f32_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x60,0x09,0x13]
+
+v_dot2acc_f32_f16 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x05,0xff,0x6f,0xf5,0x30]
+
+v_dot2c_f32_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x1b,0x00,0xff]
+
+v_dot2c_f32_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0xe4,0x00,0xff]
+
+v_dot2c_f32_f16 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x40,0x01,0xff]
+
+v_dot2c_f32_f16 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x41,0x01,0xff]
+
+v_dot2c_f32_f16 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x01,0x01,0xff]
+
+v_dot2c_f32_f16 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x0f,0x01,0xff]
+
+v_dot2c_f32_f16 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x11,0x01,0xff]
+
+v_dot2c_f32_f16 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x1f,0x01,0xff]
+
+v_dot2c_f32_f16 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x21,0x01,0xff]
+
+v_dot2c_f32_f16 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x2f,0x01,0xff]
+
+v_dot2c_f32_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x50,0x01,0xff]
+
+v_dot2c_f32_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x5f,0x01,0x01]
+
+v_dot2c_f32_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x04,0x01,0x60,0x09,0x13]
+
+v_dot2c_f32_f16 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x05,0xff,0x6f,0xf5,0x30]
+
+v_fmac_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1b,0x00,0xff]
+
+v_fmac_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0xe4,0x00,0xff]
+
+v_fmac_f16 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x40,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x41,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x01,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x0f,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x11,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1f,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x21,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x2f,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x50,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x5f,0x01,0x01]
+
+v_fmac_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x60,0x09,0x13]
+
+v_fmac_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xfe,0x6c,0x7f,0x6f,0xf5,0x30]
+
+v_fmac_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x1b,0x00,0xff]
+
+v_fmac_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x56,0x01,0xe4,0x00,0xff]
+
+v_fmac_f32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x40,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x41,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x01,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x0f,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x11,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x1f,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x21,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x2f,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x50,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x5f,0x01,0x01]
+
+v_fmac_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x60,0x09,0x13]
+
+v_fmac_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x57,0xff,0x6f,0xf5,0x30]
+
+v_ldexp_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1b,0x00,0xff]
+
+v_ldexp_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x76,0x01,0xe4,0x00,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x40,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x41,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x01,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x0f,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x11,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1f,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x21,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x2f,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x50,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x5f,0x01,0x01]
+
+v_ldexp_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x60,0x09,0x13]
+
+v_ldexp_f16 v127, -|v127|, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xfe,0x76,0x7f,0x6f,0x35,0x30]
+
+v_lshlrev_b32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x1b,0x00,0xff]
+
+v_lshlrev_b32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x30,0x01,0xe4,0x00,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x40,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x41,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x01,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x0f,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x11,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x1f,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x21,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x2f,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x50,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x5f,0x01,0x01]
+
+v_lshlrev_b32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x60,0x09,0x13]
+
+v_lshlrev_b32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x31,0xff,0x6f,0x05,0x30]
+
+v_lshrrev_b32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x1b,0x00,0xff]
+
+v_lshrrev_b32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x32,0x01,0xe4,0x00,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x40,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x41,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x01,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x0f,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x11,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x1f,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x21,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x2f,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x50,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x5f,0x01,0x01]
+
+v_lshrrev_b32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x60,0x09,0x13]
+
+v_lshrrev_b32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x33,0xff,0x6f,0x05,0x30]
+
+v_max_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x72,0x01,0x1b,0x00,0xff]
+
+v_max_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x72,0x01,0xe4,0x00,0xff]
+
+v_max_f16 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x72,0x01,0x40,0x01,0xff]
+
+v_max_f16 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x72,0x01,0x41,0x01,0xff]
+
+v_max_f16 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x72,0x01,0x01,0x01,0xff]
+
+v_max_f16 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x72,0x01,0x0f,0x01,0xff]
+
+v_max_f16 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x72,0x01,0x11,0x01,0xff]
+
+v_max_f16 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x72,0x01,0x1f,0x01,0xff]
+
+v_max_f16 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x72,0x01,0x21,0x01,0xff]
+
+v_max_f16 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x72,0x01,0x2f,0x01,0xff]
+
+v_max_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x72,0x01,0x50,0x01,0xff]
+
+v_max_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x72,0x01,0x5f,0x01,0x01]
+
+v_max_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x72,0x01,0x60,0x09,0x13]
+
+v_max_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xfe,0x72,0x7f,0x6f,0xf5,0x30]
+
+v_max_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x20,0x01,0x1b,0x00,0xff]
+
+v_max_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x20,0x01,0xe4,0x00,0xff]
+
+v_max_f32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x20,0x01,0x40,0x01,0xff]
+
+v_max_f32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x20,0x01,0x41,0x01,0xff]
+
+v_max_f32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x20,0x01,0x01,0x01,0xff]
+
+v_max_f32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x20,0x01,0x0f,0x01,0xff]
+
+v_max_f32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x20,0x01,0x11,0x01,0xff]
+
+v_max_f32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x20,0x01,0x1f,0x01,0xff]
+
+v_max_f32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x20,0x01,0x21,0x01,0xff]
+
+v_max_f32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x20,0x01,0x2f,0x01,0xff]
+
+v_max_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x20,0x01,0x50,0x01,0xff]
+
+v_max_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x20,0x01,0x5f,0x01,0x01]
+
+v_max_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x20,0x01,0x60,0x09,0x13]
+
+v_max_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x21,0xff,0x6f,0xf5,0x30]
+
+v_max_i32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x1b,0x00,0xff]
+
+v_max_i32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x24,0x01,0xe4,0x00,0xff]
+
+v_max_i32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x40,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x41,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x01,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x0f,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x11,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x1f,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x21,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x2f,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x50,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x5f,0x01,0x01]
+
+v_max_i32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x60,0x09,0x13]
+
+v_max_i32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x25,0xff,0x6f,0x05,0x30]
+
+v_max_u32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x1b,0x00,0xff]
+
+v_max_u32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x28,0x01,0xe4,0x00,0xff]
+
+v_max_u32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x40,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x41,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x01,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x0f,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x11,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x1f,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x21,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x2f,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x50,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x5f,0x01,0x01]
+
+v_max_u32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x60,0x09,0x13]
+
+v_max_u32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x29,0xff,0x6f,0x05,0x30]
+
+v_min_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x74,0x01,0x1b,0x00,0xff]
+
+v_min_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x74,0x01,0xe4,0x00,0xff]
+
+v_min_f16 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x74,0x01,0x40,0x01,0xff]
+
+v_min_f16 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x74,0x01,0x41,0x01,0xff]
+
+v_min_f16 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x74,0x01,0x01,0x01,0xff]
+
+v_min_f16 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x74,0x01,0x0f,0x01,0xff]
+
+v_min_f16 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x74,0x01,0x11,0x01,0xff]
+
+v_min_f16 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x74,0x01,0x1f,0x01,0xff]
+
+v_min_f16 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x74,0x01,0x21,0x01,0xff]
+
+v_min_f16 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x74,0x01,0x2f,0x01,0xff]
+
+v_min_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x74,0x01,0x50,0x01,0xff]
+
+v_min_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x74,0x01,0x5f,0x01,0x01]
+
+v_min_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x74,0x01,0x60,0x09,0x13]
+
+v_min_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xfe,0x74,0x7f,0x6f,0xf5,0x30]
+
+v_min_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x1b,0x00,0xff]
+
+v_min_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x1e,0x01,0xe4,0x00,0xff]
+
+v_min_f32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x40,0x01,0xff]
+
+v_min_f32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x41,0x01,0xff]
+
+v_min_f32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x01,0x01,0xff]
+
+v_min_f32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x0f,0x01,0xff]
+
+v_min_f32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x11,0x01,0xff]
+
+v_min_f32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x1f,0x01,0xff]
+
+v_min_f32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x21,0x01,0xff]
+
+v_min_f32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x2f,0x01,0xff]
+
+v_min_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x50,0x01,0xff]
+
+v_min_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x5f,0x01,0x01]
+
+v_min_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x60,0x09,0x13]
+
+v_min_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x1f,0xff,0x6f,0xf5,0x30]
+
+v_min_i32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x1b,0x00,0xff]
+
+v_min_i32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x22,0x01,0xe4,0x00,0xff]
+
+v_min_i32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x40,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x41,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x01,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x0f,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x11,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x1f,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x21,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x2f,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x50,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x5f,0x01,0x01]
+
+v_min_i32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x60,0x09,0x13]
+
+v_min_i32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x23,0xff,0x6f,0x05,0x30]
+
+v_min_u32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x1b,0x00,0xff]
+
+v_min_u32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x26,0x01,0xe4,0x00,0xff]
+
+v_min_u32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x40,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x41,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x01,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x0f,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x11,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x1f,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x21,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x2f,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x50,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x5f,0x01,0x01]
+
+v_min_u32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x60,0x09,0x13]
+
+v_min_u32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x27,0xff,0x6f,0x05,0x30]
+
+v_mul_dx9_zero_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1b,0x00,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0xe4,0x00,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x40,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x41,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x01,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x0f,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x11,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1f,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x21,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x2f,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x50,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x5f,0x01,0x01]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x60,0x09,0x13]
+
+v_mul_dx9_zero_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x0f,0xff,0x6f,0xf5,0x30]
+
+v_mul_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1b,0x00,0xff]
+
+v_mul_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0xe4,0x00,0xff]
+
+v_mul_f16 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x40,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x41,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x01,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x0f,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x11,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1f,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x21,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x2f,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x50,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x5f,0x01,0x01]
+
+v_mul_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x60,0x09,0x13]
+
+v_mul_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xfe,0x6a,0x7f,0x6f,0xf5,0x30]
+
+v_mul_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x1b,0x00,0xff]
+
+v_mul_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x10,0x01,0xe4,0x00,0xff]
+
+v_mul_f32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x40,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x41,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x01,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x0f,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x11,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x1f,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x21,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x2f,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x50,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x5f,0x01,0x01]
+
+v_mul_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x60,0x09,0x13]
+
+v_mul_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x11,0xff,0x6f,0xf5,0x30]
+
+v_mul_hi_i32_i24 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x1b,0x00,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x14,0x01,0xe4,0x00,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x40,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x41,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x01,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x0f,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x11,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x1f,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x21,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x2f,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x50,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x5f,0x01,0x01]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x60,0x09,0x13]
+
+v_mul_hi_i32_i24 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x15,0xff,0x6f,0x05,0x30]
+
+v_mul_hi_u32_u24 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x1b,0x00,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x18,0x01,0xe4,0x00,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x40,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x41,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x01,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x0f,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x11,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x1f,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x21,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x2f,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x50,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x5f,0x01,0x01]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x60,0x09,0x13]
+
+v_mul_hi_u32_u24 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x19,0xff,0x6f,0x05,0x30]
+
+v_mul_i32_i24 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x1b,0x00,0xff]
+
+v_mul_i32_i24 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x12,0x01,0xe4,0x00,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x40,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x41,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x01,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x0f,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x11,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x1f,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x21,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x2f,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x50,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x5f,0x01,0x01]
+
+v_mul_i32_i24 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x60,0x09,0x13]
+
+v_mul_i32_i24 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x13,0xff,0x6f,0x05,0x30]
+
+v_mul_legacy_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1b,0x00,0xff]
+
+v_mul_legacy_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0xe4,0x00,0xff]
+
+v_mul_legacy_f32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x40,0x01,0xff]
+
+v_mul_legacy_f32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x41,0x01,0xff]
+
+v_mul_legacy_f32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x01,0x01,0xff]
+
+v_mul_legacy_f32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x0f,0x01,0xff]
+
+v_mul_legacy_f32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x11,0x01,0xff]
+
+v_mul_legacy_f32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1f,0x01,0xff]
+
+v_mul_legacy_f32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x21,0x01,0xff]
+
+v_mul_legacy_f32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x2f,0x01,0xff]
+
+v_mul_legacy_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x50,0x01,0xff]
+
+v_mul_legacy_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x5f,0x01,0x01]
+
+v_mul_legacy_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x60,0x09,0x13]
+
+v_mul_legacy_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x0f,0xff,0x6f,0xf5,0x30]
+
+v_mul_u32_u24 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x1b,0x00,0xff]
+
+v_mul_u32_u24 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x16,0x01,0xe4,0x00,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x40,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x41,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x01,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x0f,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x11,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x1f,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x21,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x2f,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x50,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x5f,0x01,0x01]
+
+v_mul_u32_u24 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x60,0x09,0x13]
+
+v_mul_u32_u24 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x17,0xff,0x6f,0x05,0x30]
+
+v_or_b32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x1b,0x00,0xff]
+
+v_or_b32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x38,0x01,0xe4,0x00,0xff]
+
+v_or_b32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x40,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x41,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x01,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x0f,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x11,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x1f,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x21,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x2f,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x50,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x5f,0x01,0x01]
+
+v_or_b32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x60,0x09,0x13]
+
+v_or_b32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x39,0xff,0x6f,0x05,0x30]
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_half_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shl:1
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shl:15
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shr:1
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shr:15
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_ror:1
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_ror:15
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v255, vcc_lo, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0xff,0x43,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_half_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_shl:1
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_shl:15
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_shr:1
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_shr:15
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_ror:1
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_ror:15
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0xff,0x43,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1b,0x00,0xff]
+
+v_sub_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x66,0x01,0xe4,0x00,0xff]
+
+v_sub_f16 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x40,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x41,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x01,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x0f,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x11,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1f,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x21,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x2f,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01]
+
+v_sub_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x60,0x09,0x13]
+
+v_sub_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xf5,0x30]
+
+v_sub_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1b,0x00,0xff]
+
+v_sub_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x08,0x01,0xe4,0x00,0xff]
+
+v_sub_f32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x40,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x41,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x01,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x0f,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x11,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1f,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x21,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x2f,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x50,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x5f,0x01,0x01]
+
+v_sub_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x60,0x09,0x13]
+
+v_sub_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x09,0xff,0x6f,0xf5,0x30]
+
+v_sub_nc_u32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x1b,0x00,0xff]
+
+v_sub_nc_u32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0xe4,0x00,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x40,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x41,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x01,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x0f,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x11,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x1f,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x21,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x2f,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x50,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x5f,0x01,0x01]
+
+v_sub_nc_u32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x60,0x09,0x13]
+
+v_sub_nc_u32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x4d,0xff,0x6f,0x05,0x30]
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_half_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shl:1
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shl:15
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shr:1
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shr:15
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_ror:1
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_ror:15
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v255, vcc_lo, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0xff,0x45,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_half_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_shl:1
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_shl:15
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_shr:1
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_shr:15
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_ror:1
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_ror:15
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0xff,0x45,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1b,0x00,0xff]
+
+v_subrev_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x68,0x01,0xe4,0x00,0xff]
+
+v_subrev_f16 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x40,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x41,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x01,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x0f,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x11,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1f,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x21,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x2f,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x50,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x5f,0x01,0x01]
+
+v_subrev_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x60,0x09,0x13]
+
+v_subrev_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xfe,0x68,0x7f,0x6f,0xf5,0x30]
+
+v_subrev_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x1b,0x00,0xff]
+
+v_subrev_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0xe4,0x00,0xff]
+
+v_subrev_f32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x40,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x41,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x01,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x0f,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x11,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x1f,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x21,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x2f,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x50,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x5f,0x01,0x01]
+
+v_subrev_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x60,0x09,0x13]
+
+v_subrev_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x0b,0xff,0x6f,0xf5,0x30]
+
+v_subrev_nc_u32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x1b,0x00,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0xe4,0x00,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x40,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x41,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x01,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x0f,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x11,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x1f,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x21,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x2f,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x50,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x5f,0x01,0x01]
+
+v_subrev_nc_u32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x60,0x09,0x13]
+
+v_subrev_nc_u32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x4f,0xff,0x6f,0x05,0x30]
+
+v_xnor_b32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x1b,0x00,0xff]
+
+v_xnor_b32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0xe4,0x00,0xff]
+
+v_xnor_b32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x40,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x41,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x01,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x0f,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x11,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x1f,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x21,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x2f,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x50,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x5f,0x01,0x01]
+
+v_xnor_b32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x60,0x09,0x13]
+
+v_xnor_b32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x3d,0xff,0x6f,0x05,0x30]
+
+v_xor_b32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x1b,0x00,0xff]
+
+v_xor_b32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0xe4,0x00,0xff]
+
+v_xor_b32 v5, v1, v2 row_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x40,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_half_mirror
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x41,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_shl:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x01,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_shl:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x0f,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_shr:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x11,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_shr:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x1f,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_ror:1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x21,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_ror:15
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x2f,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x50,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x5f,0x01,0x01]
+
+v_xor_b32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x60,0x09,0x13]
+
+v_xor_b32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: encoding: [0xfa,0xfe,0xff,0x3b,0xff,0x6f,0x05,0x30]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp16.s
index 62c0deaecd96..3eff00bb96e4 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp16.s
@@ -1,7 +1,7 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0]
 // W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp8-fake16.s
new file mode 100644
index 000000000000..a4fea037a4de
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp8-fake16.s
@@ -0,0 +1,451 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x0a,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v255, vcc_lo, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0xff,0x41,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x0a,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0xff,0x41,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
+
+v_add_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
+
+v_add_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
+
+v_add_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x06,0x01,0x77,0x39,0x05]
+
+v_add_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x06,0x01,0x77,0x39,0x05]
+
+v_add_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x07,0xff,0x00,0x00,0x00]
+
+v_add_nc_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x4a,0x01,0x77,0x39,0x05]
+
+v_add_nc_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x4a,0x01,0x77,0x39,0x05]
+
+v_add_nc_u32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x4b,0xff,0x00,0x00,0x00]
+
+v_and_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x36,0x01,0x77,0x39,0x05]
+
+v_and_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x36,0x01,0x77,0x39,0x05]
+
+v_and_b32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x37,0xff,0x00,0x00,0x00]
+
+v_ashrrev_i32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x34,0x01,0x77,0x39,0x05]
+
+v_ashrrev_i32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x34,0x01,0x77,0x39,0x05]
+
+v_ashrrev_i32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x35,0xff,0x00,0x00,0x00]
+
+v_cndmask_b32 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x0a,0x02,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x0a,0x02,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v255, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0xff,0x03,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x0a,0x02,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x0a,0x02,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v255, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0xff,0x03,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x5e,0x01,0x77,0x39,0x05]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x5e,0x01,0x77,0x39,0x05]
+
+v_cvt_pk_rtz_f16_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x5f,0xff,0x00,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x5e,0x01,0x77,0x39,0x05]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x5e,0x01,0x77,0x39,0x05]
+
+v_cvt_pkrtz_f16_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x5f,0xff,0x00,0x00,0x00]
+
+v_dot2acc_f32_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x04,0x01,0x77,0x39,0x05]
+
+v_dot2acc_f32_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x04,0x01,0x77,0x39,0x05]
+
+v_dot2acc_f32_f16 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x05,0xff,0x00,0x00,0x00]
+
+v_dot2c_f32_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x04,0x01,0x77,0x39,0x05]
+
+v_dot2c_f32_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x04,0x01,0x77,0x39,0x05]
+
+v_dot2c_f32_f16 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x05,0xff,0x00,0x00,0x00]
+
+v_fmac_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05]
+
+v_fmac_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05]
+
+v_fmac_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xfe,0x6c,0x7f,0x00,0x00,0x00]
+
+v_fmac_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x56,0x01,0x77,0x39,0x05]
+
+v_fmac_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x56,0x01,0x77,0x39,0x05]
+
+v_fmac_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x57,0xff,0x00,0x00,0x00]
+
+v_ldexp_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x76,0x01,0x77,0x39,0x05]
+
+v_ldexp_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x76,0x01,0x77,0x39,0x05]
+
+v_ldexp_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xfe,0x76,0x7f,0x00,0x00,0x00]
+
+v_lshlrev_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x30,0x01,0x77,0x39,0x05]
+
+v_lshlrev_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x30,0x01,0x77,0x39,0x05]
+
+v_lshlrev_b32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x31,0xff,0x00,0x00,0x00]
+
+v_lshrrev_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x32,0x01,0x77,0x39,0x05]
+
+v_lshrrev_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x32,0x01,0x77,0x39,0x05]
+
+v_lshrrev_b32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x33,0xff,0x00,0x00,0x00]
+
+v_max_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x72,0x01,0x77,0x39,0x05]
+
+v_max_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x72,0x01,0x77,0x39,0x05]
+
+v_max_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xfe,0x72,0x7f,0x00,0x00,0x00]
+
+v_max_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x20,0x01,0x77,0x39,0x05]
+
+v_max_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x20,0x01,0x77,0x39,0x05]
+
+v_max_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x21,0xff,0x00,0x00,0x00]
+
+v_max_i32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x24,0x01,0x77,0x39,0x05]
+
+v_max_i32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x24,0x01,0x77,0x39,0x05]
+
+v_max_i32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x25,0xff,0x00,0x00,0x00]
+
+v_max_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x28,0x01,0x77,0x39,0x05]
+
+v_max_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x28,0x01,0x77,0x39,0x05]
+
+v_max_u32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x29,0xff,0x00,0x00,0x00]
+
+v_min_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x74,0x01,0x77,0x39,0x05]
+
+v_min_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x74,0x01,0x77,0x39,0x05]
+
+v_min_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xfe,0x74,0x7f,0x00,0x00,0x00]
+
+v_min_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x1e,0x01,0x77,0x39,0x05]
+
+v_min_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x1e,0x01,0x77,0x39,0x05]
+
+v_min_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x1f,0xff,0x00,0x00,0x00]
+
+v_min_i32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x22,0x01,0x77,0x39,0x05]
+
+v_min_i32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x22,0x01,0x77,0x39,0x05]
+
+v_min_i32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x23,0xff,0x00,0x00,0x00]
+
+v_min_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x26,0x01,0x77,0x39,0x05]
+
+v_min_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x26,0x01,0x77,0x39,0x05]
+
+v_min_u32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x27,0xff,0x00,0x00,0x00]
+
+v_mul_dx9_zero_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x0e,0x01,0x77,0x39,0x05]
+
+v_mul_dx9_zero_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x0e,0x01,0x77,0x39,0x05]
+
+v_mul_dx9_zero_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x0f,0xff,0x00,0x00,0x00]
+
+v_mul_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x6a,0x01,0x77,0x39,0x05]
+
+v_mul_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x6a,0x01,0x77,0x39,0x05]
+
+v_mul_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xfe,0x6a,0x7f,0x00,0x00,0x00]
+
+v_mul_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x10,0x01,0x77,0x39,0x05]
+
+v_mul_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x10,0x01,0x77,0x39,0x05]
+
+v_mul_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x11,0xff,0x00,0x00,0x00]
+
+v_mul_hi_i32_i24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x14,0x01,0x77,0x39,0x05]
+
+v_mul_hi_i32_i24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x14,0x01,0x77,0x39,0x05]
+
+v_mul_hi_i32_i24 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x15,0xff,0x00,0x00,0x00]
+
+v_mul_hi_u32_u24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x18,0x01,0x77,0x39,0x05]
+
+v_mul_hi_u32_u24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x18,0x01,0x77,0x39,0x05]
+
+v_mul_hi_u32_u24 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x19,0xff,0x00,0x00,0x00]
+
+v_mul_i32_i24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x12,0x01,0x77,0x39,0x05]
+
+v_mul_i32_i24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x12,0x01,0x77,0x39,0x05]
+
+v_mul_i32_i24 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x13,0xff,0x00,0x00,0x00]
+
+v_mul_legacy_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x0e,0x01,0x77,0x39,0x05]
+
+v_mul_legacy_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x0e,0x01,0x77,0x39,0x05]
+
+v_mul_legacy_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x0f,0xff,0x00,0x00,0x00]
+
+v_mul_u32_u24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x16,0x01,0x77,0x39,0x05]
+
+v_mul_u32_u24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x16,0x01,0x77,0x39,0x05]
+
+v_mul_u32_u24 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x17,0xff,0x00,0x00,0x00]
+
+v_or_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x38,0x01,0x77,0x39,0x05]
+
+v_or_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x38,0x01,0x77,0x39,0x05]
+
+v_or_b32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x39,0xff,0x00,0x00,0x00]
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x0a,0x42,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x0a,0x42,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v255, vcc_lo, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0xff,0x43,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x0a,0x42,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x0a,0x42,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0xff,0x43,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
+
+v_sub_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
+
+v_sub_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
+
+v_sub_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x08,0x01,0x77,0x39,0x05]
+
+v_sub_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x08,0x01,0x77,0x39,0x05]
+
+v_sub_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x09,0xff,0x00,0x00,0x00]
+
+v_sub_nc_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x4c,0x01,0x77,0x39,0x05]
+
+v_sub_nc_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x4c,0x01,0x77,0x39,0x05]
+
+v_sub_nc_u32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x4d,0xff,0x00,0x00,0x00]
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x0a,0x44,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x0a,0x44,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v255, vcc_lo, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0xff,0x45,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x0a,0x44,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x0a,0x44,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0xff,0x45,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x68,0x01,0x77,0x39,0x05]
+
+v_subrev_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x68,0x01,0x77,0x39,0x05]
+
+v_subrev_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xfe,0x68,0x7f,0x00,0x00,0x00]
+
+v_subrev_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x0a,0x01,0x77,0x39,0x05]
+
+v_subrev_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x0a,0x01,0x77,0x39,0x05]
+
+v_subrev_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x0b,0xff,0x00,0x00,0x00]
+
+v_subrev_nc_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x4e,0x01,0x77,0x39,0x05]
+
+v_subrev_nc_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x4e,0x01,0x77,0x39,0x05]
+
+v_subrev_nc_u32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x4f,0xff,0x00,0x00,0x00]
+
+v_xnor_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x3c,0x01,0x77,0x39,0x05]
+
+v_xnor_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x3c,0x01,0x77,0x39,0x05]
+
+v_xnor_b32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x3d,0xff,0x00,0x00,0x00]
+
+v_xor_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: encoding: [0xe9,0x04,0x0a,0x3a,0x01,0x77,0x39,0x05]
+
+v_xor_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: encoding: [0xea,0x04,0x0a,0x3a,0x01,0x77,0x39,0x05]
+
+v_xor_b32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: encoding: [0xe9,0xfe,0xff,0x3b,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp8.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp8.s
index d235fcdeb526..0f19cf002852 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp8.s
@@ -1,7 +1,7 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
 // W32: encoding: [0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_err-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_err-fake16.s
new file mode 100644
index 000000000000..2d52828d1e28
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_err-fake16.s
@@ -0,0 +1,13 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=error: %s
+
+v_fmaak_f32 v0, 0xff32, v0, 0
+// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: only one unique literal operand is allowed
+
+v_fmaak_f16 v0, 0xff32, v0, 0
+// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: only one unique literal operand is allowed
+
+v_fmamk_f32 v0, 0xff32, 1, v0
+// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: only one unique literal operand is allowed
+
+v_fmamk_f16 v0, 0xff32, 1, v0
+// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: only one unique literal operand is allowed
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_err.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_err.s
index 164a49dcdd47..dedbcb55d797 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_err.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_err.s
@@ -1,4 +1,4 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=error: %s
 
 v_fmaak_f32 v0, 0xff32, v0, 0
 // GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: only one unique literal operand is allowed
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_err.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_err.s
index 76b1c38fad43..dd619f3077f7 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_err.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_err.s
@@ -1,237 +1,238 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort --version 5
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=error: %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=error: %s
 
-v_add_f16_e32 v255, v1, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmaak_f16_e32 v255, v1, v2, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmac_f16_e32 v255, v1, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmamk_f16_e32 v255, v1, 0xfe0b, v3
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_ldexp_f16_e32 v255.l, v1.l, v2.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_add_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_max_f16_e32 v255, v1, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_min_f16_e32 v255, v1, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_add_f16_e32 v255, v1, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_mul_f16_e32 v255, v1, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_add_f16_e32 v5, v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16_e32 v255, v1, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_add_f16_e32 v5, v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_subrev_f16_e32 v255, v1, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmaak_f16_e32 v255, v1, v2, 0xfe0b
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16_e32 v5, v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmaak_f16_e32 v5, v1, v255, 0xfe0b
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_fmaak_f16_e32 v5, v255, v2, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_fmac_f16_e32 v5, v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmamk_f16_e32 v5, v255, 0xfe0b, v3
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmac_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_ldexp_f16_e32 v5.l, v255.l, v2.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_fmac_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_max_f16_e32 v5, v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmac_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_min_f16_e32 v5, v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmac_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_mul_f16_e32 v5, v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmac_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16_e32 v5, v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmac_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_subrev_f16_e32 v5, v255, v2
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmac_f16_e32 v255, v1, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16_e32 v5, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmac_f16_e32 v5, v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmaak_f16_e32 v5, v1, v255, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmac_f16_e32 v5, v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmac_f16_e32 v5, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmamk_f16_e32 v255, v1, 0xfe0b, v3
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_fmamk_f16_e32 v5, v1, 0xfe0b, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_ldexp_f16_e32 v5.l, v1.l, v255.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_fmamk_f16_e32 v5, v255, 0xfe0b, v3
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_max_f16_e32 v5, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_ldexp_f16_dpp v255.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:17: error: invalid operand for instruction
 
-v_min_f16_e32 v5, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_ldexp_f16_dpp v255.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:17: error: invalid operand for instruction
 
-v_mul_f16_e32 v5, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_ldexp_f16_dpp v5.l, v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_sub_f16_e32 v5, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_ldexp_f16_dpp v5.l, v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_subrev_f16_e32 v5, v1, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_ldexp_f16_dpp v5.l, v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_add_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_ldexp_f16_dpp v5.l, v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_fmac_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_ldexp_f16_e32 v255.l, v1.l, v2.l
+// GFX11: :[[@LINE-1]]:17: error: invalid operand for instruction
 
-v_ldexp_f16_dpp v255.l, v1.l, v2.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_ldexp_f16_e32 v5.l, v1.l, v255.l
+// GFX11: :[[@LINE-1]]:29: error: invalid operand for instruction
 
-v_max_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_ldexp_f16_e32 v5.l, v255.l, v2.l
+// GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_min_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_max_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_mul_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_max_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_max_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_subrev_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_max_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_max_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmac_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_max_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_ldexp_f16_dpp v5.l, v255.l, v2.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_max_f16_e32 v255, v1, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_max_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_max_f16_e32 v5, v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_min_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_max_f16_e32 v5, v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_mul_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_min_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_min_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_subrev_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_min_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_min_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmac_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_min_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_ldexp_f16_dpp v5.l, v1.l, v255.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_min_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_max_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_min_f16_e32 v255, v1, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_min_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_min_f16_e32 v5, v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_mul_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_min_f16_e32 v5, v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_mul_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_subrev_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_mul_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_mul_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmac_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_mul_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_ldexp_f16_dpp v255.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_mul_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_max_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_mul_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_min_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_mul_f16_e32 v255, v1, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_mul_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_mul_f16_e32 v5, v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_mul_f16_e32 v5, v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_sub_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_subrev_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmac_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_ldexp_f16_dpp v5.l, v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sub_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_max_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_min_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_sub_f16_e32 v255, v1, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_mul_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_sub_f16_e32 v5, v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_sub_f16_e32 v5, v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_subrev_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_subrev_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_subrev_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmac_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_subrev_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_ldexp_f16_dpp v5.l, v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
 
-v_max_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_subrev_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_min_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_subrev_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_mul_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_subrev_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_subrev_f16_e32 v255, v1, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_subrev_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_subrev_f16_e32 v5, v1, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
+v_subrev_f16_e32 v5, v255, v2
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_promote.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_promote.s
index a5b5f32e9762..a6dcce40fd0e 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_promote.s
@@ -1,201 +1,202 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort --version 5
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=_e32 %s
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=_e32 %s
 
 v_add_f16 v255, v1, v2
-// GFX11: v_add_f16_e64
+// GFX11: v_add_f16_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x32,0xd5,0x01,0x05,0x02,0x00]
 
-v_fmac_f16 v255, v1, v2
-// GFX11: v_fmac_f16_e64
-
-v_ldexp_f16 v255, v1, v2
-// GFX11: v_ldexp_f16_e64
-
-v_max_f16 v255, v1, v2
-// GFX11: v_max_f16_e64
+v_add_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_min_f16 v255, v1, v2
-// GFX11: v_min_f16_e64
+v_add_f16 v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_mul_f16 v255, v1, v2
-// GFX11: v_mul_f16_e64
+v_add_f16 v5, v1, v255
+// GFX11: v_add_f16_e64 v5, v1, v255              ; encoding: [0x05,0x00,0x32,0xd5,0x01,0xff,0x03,0x00]
 
-v_sub_f16 v255, v1, v2
-// GFX11: v_sub_f16_e64
+v_add_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_subrev_f16 v255, v1, v2
-// GFX11: v_subrev_f16_e64
+v_add_f16 v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_add_f16 v5, v255, v2
-// GFX11: v_add_f16_e64
-
-v_fmac_f16 v5, v255, v2
-// GFX11: v_fmac_f16_e64
+// GFX11: v_add_f16_e64 v5, v255, v2              ; encoding: [0x05,0x00,0x32,0xd5,0xff,0x05,0x02,0x00]
 
-v_ldexp_f16 v5, v255, v2
-// GFX11: v_ldexp_f16_e64
+v_add_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_max_f16 v5, v255, v2
-// GFX11: v_max_f16_e64
+v_add_f16 v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_min_f16 v5, v255, v2
-// GFX11: v_min_f16_e64
+v_fmac_f16 v255, v1, v2
+// GFX11: v_fmac_f16_e64 v255, v1, v2             ; encoding: [0xff,0x00,0x36,0xd5,0x01,0x05,0x02,0x00]
 
-v_mul_f16 v5, v255, v2
-// GFX11: v_mul_f16_e64
+v_fmac_f16 v5, v1, v255
+// GFX11: v_fmac_f16_e64 v5, v1, v255             ; encoding: [0x05,0x00,0x36,0xd5,0x01,0xff,0x03,0x00]
 
-v_sub_f16 v5, v255, v2
-// GFX11: v_sub_f16_e64
+v_fmac_f16 v5, v255, v2
+// GFX11: v_fmac_f16_e64 v5, v255, v2             ; encoding: [0x05,0x00,0x36,0xd5,0xff,0x05,0x02,0x00]
 
-v_subrev_f16 v5, v255, v2
-// GFX11: v_subrev_f16_e64
+v_ldexp_f16 v255, v1, v2
+// GFX11: v_ldexp_f16_e64 v255, v1, v2            ; encoding: [0xff,0x00,0x3b,0xd5,0x01,0x05,0x02,0x00]
 
-v_add_f16 v5, v1, v255
-// GFX11: v_add_f16_e64
+v_ldexp_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_ldexp_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x3b,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_fmac_f16 v5, v1, v255
-// GFX11: v_fmac_f16_e64
+v_ldexp_f16 v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_ldexp_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 v_ldexp_f16 v5, v1, v255
-// GFX11: v_ldexp_f16_e64
+// GFX11: v_ldexp_f16_e64 v5, v1, v255            ; encoding: [0x05,0x00,0x3b,0xd5,0x01,0xff,0x03,0x00]
 
-v_max_f16 v5, v1, v255
-// GFX11: v_max_f16_e64
+v_ldexp_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3b,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_min_f16 v5, v1, v255
-// GFX11: v_min_f16_e64
+v_ldexp_f16 v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_mul_f16 v5, v1, v255
-// GFX11: v_mul_f16_e64
+v_ldexp_f16 v5, v255, v2
+// GFX11: v_ldexp_f16_e64 v5, v255, v2            ; encoding: [0x05,0x00,0x3b,0xd5,0xff,0x05,0x02,0x00]
 
-v_sub_f16 v5, v1, v255
-// GFX11: v_sub_f16_e64
+v_ldexp_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_ldexp_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3b,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_subrev_f16 v5, v1, v255
-// GFX11: v_subrev_f16_e64
+v_ldexp_f16 v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_ldexp_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_add_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_add_f16_e64
+v_max_f16 v255, v1, v2
+// GFX11: v_max_f16_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x39,0xd5,0x01,0x05,0x02,0x00]
 
-v_ldexp_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_ldexp_f16_e64
+v_max_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_max_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x39,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 v_max_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_max_f16_e64
-
-v_min_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_min_f16_e64
+// GFX11: v_max_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x39,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_mul_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_mul_f16_e64
+v_max_f16 v5, v1, v255
+// GFX11: v_max_f16_e64 v5, v1, v255              ; encoding: [0x05,0x00,0x39,0xd5,0x01,0xff,0x03,0x00]
 
-v_sub_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_sub_f16_e64
+v_max_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_max_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_subrev_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_subrev_f16_e64
+v_max_f16 v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_max_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_add_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_add_f16_e64
+v_max_f16 v5, v255, v2
+// GFX11: v_max_f16_e64 v5, v255, v2              ; encoding: [0x05,0x00,0x39,0xd5,0xff,0x05,0x02,0x00]
 
-v_ldexp_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_ldexp_f16_e64
+v_max_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_max_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
 v_max_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_max_f16_e64
+// GFX11: v_max_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_min_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_min_f16_e64
+v_min_f16 v255, v1, v2
+// GFX11: v_min_f16_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x3a,0xd5,0x01,0x05,0x02,0x00]
 
-v_mul_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_mul_f16_e64
+v_min_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_min_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x3a,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_sub_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_sub_f16_e64
+v_min_f16 v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_min_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x3a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_subrev_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_subrev_f16_e64
+v_min_f16 v5, v1, v255
+// GFX11: v_min_f16_e64 v5, v1, v255              ; encoding: [0x05,0x00,0x3a,0xd5,0x01,0xff,0x03,0x00]
 
-v_add_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_add_f16_e64
+v_min_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_min_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_ldexp_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_ldexp_f16_e64
+v_min_f16 v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_min_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_max_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_max_f16_e64
+v_min_f16 v5, v255, v2
+// GFX11: v_min_f16_e64 v5, v255, v2              ; encoding: [0x05,0x00,0x3a,0xd5,0xff,0x05,0x02,0x00]
 
-v_min_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_min_f16_e64
+v_min_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_min_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_mul_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_mul_f16_e64
+v_min_f16 v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_min_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_sub_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_sub_f16_e64
+v_mul_f16 v255, v1, v2
+// GFX11: v_mul_f16_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x35,0xd5,0x01,0x05,0x02,0x00]
 
-v_subrev_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_subrev_f16_e64
+v_mul_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mul_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x35,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_add_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_add_f16_e64
+v_mul_f16 v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_mul_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_ldexp_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_ldexp_f16_e64
+v_mul_f16 v5, v1, v255
+// GFX11: v_mul_f16_e64 v5, v1, v255              ; encoding: [0x05,0x00,0x35,0xd5,0x01,0xff,0x03,0x00]
 
-v_max_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_max_f16_e64
+v_mul_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mul_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x35,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_min_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_min_f16_e64
+v_mul_f16 v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_mul_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_mul_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mul_f16_e64
+v_mul_f16 v5, v255, v2
+// GFX11: v_mul_f16_e64 v5, v255, v2              ; encoding: [0x05,0x00,0x35,0xd5,0xff,0x05,0x02,0x00]
 
-v_sub_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sub_f16_e64
+v_mul_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mul_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x35,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_subrev_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_subrev_f16_e64
+v_mul_f16 v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_mul_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_add_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_add_f16_e64
+v_sub_f16 v255, v1, v2
+// GFX11: v_sub_f16_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x33,0xd5,0x01,0x05,0x02,0x00]
 
-v_ldexp_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_ldexp_f16_e64
+v_sub_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_max_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_max_f16_e64
+v_sub_f16 v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_min_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_min_f16_e64
+v_sub_f16 v5, v1, v255
+// GFX11: v_sub_f16_e64 v5, v1, v255              ; encoding: [0x05,0x00,0x33,0xd5,0x01,0xff,0x03,0x00]
 
-v_mul_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mul_f16_e64
+v_sub_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_sub_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sub_f16_e64
+v_sub_f16 v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_subrev_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_subrev_f16_e64
+v_sub_f16 v5, v255, v2
+// GFX11: v_sub_f16_e64 v5, v255, v2              ; encoding: [0x05,0x00,0x33,0xd5,0xff,0x05,0x02,0x00]
 
-v_add_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_add_f16_e64
+v_sub_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_ldexp_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_ldexp_f16_e64
+v_sub_f16 v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_max_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_max_f16_e64
+v_subrev_f16 v255, v1, v2
+// GFX11: v_subrev_f16_e64 v255, v1, v2           ; encoding: [0xff,0x00,0x34,0xd5,0x01,0x05,0x02,0x00]
 
-v_min_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_min_f16_e64
+v_subrev_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_subrev_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x34,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_mul_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_mul_f16_e64
+v_subrev_f16 v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_subrev_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_sub_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sub_f16_e64
+v_subrev_f16 v5, v1, v255
+// GFX11: v_subrev_f16_e64 v5, v1, v255           ; encoding: [0x05,0x00,0x34,0xd5,0x01,0xff,0x03,0x00]
 
 v_subrev_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_subrev_f16_e64
+// GFX11: v_subrev_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x34,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
+
+v_subrev_f16 v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX11: v_subrev_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_subrev_f16 v5, v255, v2
+// GFX11: v_subrev_f16_e64 v5, v255, v2           ; encoding: [0x05,0x00,0x34,0xd5,0xff,0x05,0x02,0x00]
+
+v_subrev_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_subrev_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x34,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_subrev_f16 v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX11: v_subrev_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2-fake16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2-fake16.s
new file mode 100644
index 000000000000..4c37502e1b24
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2-fake16.s
@@ -0,0 +1,2560 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_add_co_ci_u32_e32 v5, vcc_lo, v1, v2, vcc_lo
+// W32: encoding: [0x01,0x05,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v255, v2, vcc_lo
+// W32: encoding: [0xff,0x05,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, s1, v2, vcc_lo
+// W32: encoding: [0x01,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, s105, v2, vcc_lo
+// W32: encoding: [0x69,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, vcc_lo, v2, vcc_lo
+// W32: encoding: [0x6a,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, vcc_hi, v2, vcc_lo
+// W32: encoding: [0x6b,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, ttmp15, v2, vcc_lo
+// W32: encoding: [0x7b,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, m0, v2, vcc_lo
+// W32: encoding: [0x7d,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, exec_lo, v2, vcc_lo
+// W32: encoding: [0x7e,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, exec_hi, v2, vcc_lo
+// W32: encoding: [0x7f,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, null, v2, vcc_lo
+// W32: encoding: [0x7c,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, -1, v2, vcc_lo
+// W32: encoding: [0xc1,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, 0.5, v2, vcc_lo
+// W32: encoding: [0xf0,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, src_scc, v2, vcc_lo
+// W32: encoding: [0xfd,0x04,0x0a,0x40]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v255, vcc_lo, 0xaf123456, v255, vcc_lo
+// W32: encoding: [0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc
+// W64: encoding: [0x01,0x05,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v255, v2, vcc
+// W64: encoding: [0xff,0x05,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, s1, v2, vcc
+// W64: encoding: [0x01,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, s105, v2, vcc
+// W64: encoding: [0x69,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, vcc_lo, v2, vcc
+// W64: encoding: [0x6a,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, vcc_hi, v2, vcc
+// W64: encoding: [0x6b,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, ttmp15, v2, vcc
+// W64: encoding: [0x7b,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, m0, v2, vcc
+// W64: encoding: [0x7d,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, exec_lo, v2, vcc
+// W64: encoding: [0x7e,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, exec_hi, v2, vcc
+// W64: encoding: [0x7f,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, null, v2, vcc
+// W64: encoding: [0x7c,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, -1, v2, vcc
+// W64: encoding: [0xc1,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, 0.5, v2, vcc
+// W64: encoding: [0xf0,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, src_scc, v2, vcc
+// W64: encoding: [0xfd,0x04,0x0a,0x40]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v255, vcc, 0xaf123456, v255, vcc
+// W64: encoding: [0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_f16 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x64]
+
+v_add_f16 v5, v127, v2
+// GFX12: encoding: [0x7f,0x05,0x0a,0x64]
+
+v_add_f16 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x64]
+
+v_add_f16 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x64]
+
+v_add_f16 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x64]
+
+v_add_f16 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x64]
+
+v_add_f16 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x64]
+
+v_add_f16 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x64]
+
+v_add_f16 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x64]
+
+v_add_f16 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x64]
+
+v_add_f16 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x64]
+
+v_add_f16 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x64]
+
+v_add_f16 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x64]
+
+v_add_f16 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x64]
+
+v_add_f16 v127, 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00]
+
+v_add_f32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x06]
+
+v_add_f32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x06]
+
+v_add_f32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x06]
+
+v_add_f32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x06]
+
+v_add_f32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x06]
+
+v_add_f32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x06]
+
+v_add_f32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x06]
+
+v_add_f32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x06]
+
+v_add_f32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x06]
+
+v_add_f32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x06]
+
+v_add_f32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x06]
+
+v_add_f32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x06]
+
+v_add_f32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x06]
+
+v_add_f32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x06]
+
+v_add_f32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x07,0x56,0x34,0x12,0xaf]
+
+v_add_f64 v[5:6], v[1:2], v[3:4]
+// GFX12: encoding: [0x01,0x07,0x0a,0x04]
+
+v_add_f64 v[5:6], v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x0a,0x04]
+
+v_add_f64 v[5:6], s[0:1], v[2:3]
+// GFX12: encoding: [0x00,0x04,0x0a,0x04]
+
+v_add_f64 v[5:6], s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x0a,0x04]
+
+v_add_f64 v[5:6], vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x0a,0x04]
+
+v_add_f64 v[5:6], ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x0a,0x04]
+
+v_add_f64 v[5:6], exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x0a,0x04]
+
+v_add_f64 v[5:6], null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x0a,0x04]
+
+v_add_f64 v[5:6], -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x0a,0x04]
+
+v_add_f64 v[5:6], 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x0a,0x04]
+
+v_add_f64 v[5:6], src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x0a,0x04]
+
+v_add_f64 v[254:255], 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xfd,0x05,0x56,0x34,0x12,0xaf]
+
+v_add_nc_u32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x4a]
+
+v_add_nc_u32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x4a]
+
+v_add_nc_u32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x4a]
+
+v_add_nc_u32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x4b,0x56,0x34,0x12,0xaf]
+
+v_and_b32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x36]
+
+v_and_b32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x36]
+
+v_and_b32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x36]
+
+v_and_b32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x36]
+
+v_and_b32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x36]
+
+v_and_b32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x36]
+
+v_and_b32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x36]
+
+v_and_b32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x36]
+
+v_and_b32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x36]
+
+v_and_b32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x36]
+
+v_and_b32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x36]
+
+v_and_b32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x36]
+
+v_and_b32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x36]
+
+v_and_b32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x36]
+
+v_and_b32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x37,0x56,0x34,0x12,0xaf]
+
+v_ashrrev_i32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x34]
+
+v_ashrrev_i32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x34]
+
+v_ashrrev_i32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x34]
+
+v_ashrrev_i32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x35,0x56,0x34,0x12,0xaf]
+
+v_cndmask_b32 v5, v1, v2, vcc_lo
+// W32: encoding: [0x01,0x05,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v255, v2, vcc_lo
+// W32: encoding: [0xff,0x05,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, s1, v2, vcc_lo
+// W32: encoding: [0x01,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, s105, v2, vcc_lo
+// W32: encoding: [0x69,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, vcc_lo, v2, vcc_lo
+// W32: encoding: [0x6a,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, vcc_hi, v2, vcc_lo
+// W32: encoding: [0x6b,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, ttmp15, v2, vcc_lo
+// W32: encoding: [0x7b,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, m0, v2, vcc_lo
+// W32: encoding: [0x7d,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, exec_lo, v2, vcc_lo
+// W32: encoding: [0x7e,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, exec_hi, v2, vcc_lo
+// W32: encoding: [0x7f,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, null, v2, vcc_lo
+// W32: encoding: [0x7c,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, -1, v2, vcc_lo
+// W32: encoding: [0xc1,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, 0.5, v2, vcc_lo
+// W32: encoding: [0xf0,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, src_scc, v2, vcc_lo
+// W32: encoding: [0xfd,0x04,0x0a,0x02]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v255, 0xaf123456, v255, vcc_lo
+// W32: encoding: [0xff,0xfe,0xff,0x03,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc
+// W64: encoding: [0x01,0x05,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v255, v2, vcc
+// W64: encoding: [0xff,0x05,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, s1, v2, vcc
+// W64: encoding: [0x01,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, s105, v2, vcc
+// W64: encoding: [0x69,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, vcc_lo, v2, vcc
+// W64: encoding: [0x6a,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, vcc_hi, v2, vcc
+// W64: encoding: [0x6b,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, ttmp15, v2, vcc
+// W64: encoding: [0x7b,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, m0, v2, vcc
+// W64: encoding: [0x7d,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, exec_lo, v2, vcc
+// W64: encoding: [0x7e,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, exec_hi, v2, vcc
+// W64: encoding: [0x7f,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, null, v2, vcc
+// W64: encoding: [0x7c,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, -1, v2, vcc
+// W64: encoding: [0xc1,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, 0.5, v2, vcc
+// W64: encoding: [0xf0,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, src_scc, v2, vcc
+// W64: encoding: [0xfd,0x04,0x0a,0x02]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v255, 0xaf123456, v255, vcc
+// W64: encoding: [0xff,0xfe,0xff,0x03,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x5e]
+
+v_cvt_pk_rtz_f16_f32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x5f,0x56,0x34,0x12,0xaf]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x5e]
+
+v_cvt_pkrtz_f16_f32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x5f,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f16 v5, v1, v2, 0xfe0b
+// GFX12: encoding: [0x01,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, v127, v2, 0xfe0b
+// GFX12: encoding: [0x7f,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, s1, v2, 0xfe0b
+// GFX12: encoding: [0x01,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, s105, v2, 0xfe0b
+// GFX12: encoding: [0x69,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, vcc_lo, v2, 0xfe0b
+// GFX12: encoding: [0x6a,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, vcc_hi, v2, 0xfe0b
+// GFX12: encoding: [0x6b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, ttmp15, v2, 0xfe0b
+// GFX12: encoding: [0x7b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, m0, v2, 0xfe0b
+// GFX12: encoding: [0x7d,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, exec_lo, v2, 0xfe0b
+// GFX12: encoding: [0x7e,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, exec_hi, v2, 0xfe0b
+// GFX12: encoding: [0x7f,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, null, v2, 0xfe0b
+// GFX12: encoding: [0x7c,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, -1, v2, 0xfe0b
+// GFX12: encoding: [0xc1,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, 0.5, v2, 0xfe0b
+// GFX12: encoding: [0xf0,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5, src_scc, v2, 0xfe0b
+// GFX12: encoding: [0xfd,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v127, 0xfe0b, v127, 0xfe0b
+// GFX12: encoding: [0xff,0xfe,0xfe,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f32 v5, v1, v2, 0xaf123456
+// GFX12: encoding: [0x01,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, v255, v2, 0xaf123456
+// GFX12: encoding: [0xff,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, s1, v2, 0xaf123456
+// GFX12: encoding: [0x01,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, s105, v2, 0xaf123456
+// GFX12: encoding: [0x69,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, vcc_lo, v2, 0xaf123456
+// GFX12: encoding: [0x6a,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, vcc_hi, v2, 0xaf123456
+// GFX12: encoding: [0x6b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, ttmp15, v2, 0xaf123456
+// GFX12: encoding: [0x7b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, m0, v2, 0xaf123456
+// GFX12: encoding: [0x7d,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, exec_lo, v2, 0xaf123456
+// GFX12: encoding: [0x7e,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, exec_hi, v2, 0xaf123456
+// GFX12: encoding: [0x7f,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, null, v2, 0xaf123456
+// GFX12: encoding: [0x7c,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, -1, v2, 0xaf123456
+// GFX12: encoding: [0xc1,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, 0.5, v2, 0xaf123456
+// GFX12: encoding: [0xf0,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v5, src_scc, v2, 0xaf123456
+// GFX12: encoding: [0xfd,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
+
+v_fmaak_f32 v255, 0xaf123456, v255, 0xaf123456
+// GFX12: encoding: [0xff,0xfe,0xff,0x5b,0x56,0x34,0x12,0xaf]
+
+v_fmac_f16 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x6c]
+
+v_fmac_f16 v5, v127, v2
+// GFX12: encoding: [0x7f,0x05,0x0a,0x6c]
+
+v_fmac_f16 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x6c]
+
+v_fmac_f16 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x6c]
+
+v_fmac_f16 v127, 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0xfe,0x6c,0x0b,0xfe,0x00,0x00]
+
+v_fmac_f32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x56]
+
+v_fmac_f32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x56]
+
+v_fmac_f32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x56]
+
+v_fmac_f32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x56]
+
+v_fmac_f32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x57,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f16 v5, v1, 0xfe0b, v3
+// GFX12: encoding: [0x01,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, v127, 0xfe0b, v3
+// GFX12: encoding: [0x7f,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, s1, 0xfe0b, v3
+// GFX12: encoding: [0x01,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, s105, 0xfe0b, v3
+// GFX12: encoding: [0x69,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, vcc_lo, 0xfe0b, v3
+// GFX12: encoding: [0x6a,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, vcc_hi, 0xfe0b, v3
+// GFX12: encoding: [0x6b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, ttmp15, 0xfe0b, v3
+// GFX12: encoding: [0x7b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, m0, 0xfe0b, v3
+// GFX12: encoding: [0x7d,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, exec_lo, 0xfe0b, v3
+// GFX12: encoding: [0x7e,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, exec_hi, 0xfe0b, v3
+// GFX12: encoding: [0x7f,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, null, 0xfe0b, v3
+// GFX12: encoding: [0x7c,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, -1, 0xfe0b, v3
+// GFX12: encoding: [0xc1,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, 0.5, 0xfe0b, v3
+// GFX12: encoding: [0xf0,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5, src_scc, 0xfe0b, v3
+// GFX12: encoding: [0xfd,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v127, 0xfe0b, 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0xfe,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f32 v5, v1, 0xaf123456, v3
+// GFX12: encoding: [0x01,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, v255, 0xaf123456, v3
+// GFX12: encoding: [0xff,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, s1, 0xaf123456, v3
+// GFX12: encoding: [0x01,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, s105, 0xaf123456, v3
+// GFX12: encoding: [0x69,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, vcc_lo, 0xaf123456, v3
+// GFX12: encoding: [0x6a,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, vcc_hi, 0xaf123456, v3
+// GFX12: encoding: [0x6b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, ttmp15, 0xaf123456, v3
+// GFX12: encoding: [0x7b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, m0, 0xaf123456, v3
+// GFX12: encoding: [0x7d,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, exec_lo, 0xaf123456, v3
+// GFX12: encoding: [0x7e,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, exec_hi, 0xaf123456, v3
+// GFX12: encoding: [0x7f,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, null, 0xaf123456, v3
+// GFX12: encoding: [0x7c,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, -1, 0xaf123456, v3
+// GFX12: encoding: [0xc1,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, 0.5, 0xaf123456, v3
+// GFX12: encoding: [0xf0,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v5, src_scc, 0xaf123456, v3
+// GFX12: encoding: [0xfd,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
+
+v_fmamk_f32 v255, 0xaf123456, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x59,0x56,0x34,0x12,0xaf]
+
+v_ldexp_f16 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x76]
+
+v_ldexp_f16 v5, v127, v2
+// GFX12: encoding: [0x7f,0x05,0x0a,0x76]
+
+v_ldexp_f16 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x76]
+
+v_ldexp_f16 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x76]
+
+v_ldexp_f16 v127, 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0xfe,0x76,0x0b,0xfe,0x00,0x00]
+
+v_lshlrev_b32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x30]
+
+v_lshlrev_b32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x30]
+
+v_lshlrev_b32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x30]
+
+v_lshlrev_b32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x31,0x56,0x34,0x12,0xaf]
+
+v_lshlrev_b64 v[5:6], v1, v[3:4]
+// GFX12: encoding: [0x01,0x07,0x0a,0x3e]
+
+v_lshlrev_b64 v[5:6], v255, v[2:3]
+// GFX12: encoding: [0xff,0x05,0x0a,0x3e]
+
+v_lshlrev_b64 v[5:6], s1, v[2:3]
+// GFX12: encoding: [0x01,0x04,0x0a,0x3e]
+
+v_lshlrev_b64 v[5:6], s105, v[2:3]
+// GFX12: encoding: [0x69,0x04,0x0a,0x3e]
+
+v_lshlrev_b64 v[5:6], vcc_lo, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x0a,0x3e]
+
+v_lshlrev_b64 v[5:6], vcc_hi, v[2:3]
+// GFX12: encoding: [0x6b,0x04,0x0a,0x3e]
+
+v_lshlrev_b64 v[5:6], ttmp15, v[2:3]
+// GFX12: encoding: [0x7b,0x04,0x0a,0x3e]
+
+v_lshlrev_b64 v[5:6], exec_lo, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x0a,0x3e]
+
+v_lshlrev_b64 v[5:6], exec_hi, v[2:3]
+// GFX12: encoding: [0x7f,0x04,0x0a,0x3e]
+
+v_lshlrev_b64 v[5:6], null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x0a,0x3e]
+
+v_lshlrev_b64 v[5:6], -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x0a,0x3e]
+
+v_lshlrev_b64 v[5:6], 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x0a,0x3e]
+
+v_lshlrev_b64 v[5:6], src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x0a,0x3e]
+
+v_lshlrev_b64 v[254:255], 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xfd,0x3f,0x56,0x34,0x12,0xaf]
+
+v_lshrrev_b32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x32]
+
+v_lshrrev_b32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x32]
+
+v_lshrrev_b32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x32]
+
+v_lshrrev_b32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x33,0x56,0x34,0x12,0xaf]
+
+v_max_num_f16 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x62]
+
+v_max_num_f16 v5, v127, v2
+// GFX12: encoding: [0x7f,0x05,0x0a,0x62]
+
+v_max_num_f16 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x62]
+
+v_max_num_f16 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x62]
+
+v_max_num_f16 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x62]
+
+v_max_num_f16 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x62]
+
+v_max_num_f16 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x62]
+
+v_max_num_f16 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x62]
+
+v_max_num_f16 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x62]
+
+v_max_num_f16 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x62]
+
+v_max_num_f16 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x62]
+
+v_max_num_f16 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x62]
+
+v_max_num_f16 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x62]
+
+v_max_num_f16 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x62]
+
+v_max_num_f16 v127, 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0xfe,0x62,0x0b,0xfe,0x00,0x00]
+
+v_max_num_f32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x2c]
+
+v_max_num_f32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x2c]
+
+v_max_num_f32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x2c]
+
+v_max_num_f32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x2c]
+
+v_max_num_f32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x2c]
+
+v_max_num_f32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x2c]
+
+v_max_num_f32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x2c]
+
+v_max_num_f32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x2c]
+
+v_max_num_f32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x2c]
+
+v_max_num_f32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x2c]
+
+v_max_num_f32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x2c]
+
+v_max_num_f32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x2c]
+
+v_max_num_f32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x2c]
+
+v_max_num_f32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x2c]
+
+v_max_num_f32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x2d,0x56,0x34,0x12,0xaf]
+
+v_max_num_f64 v[5:6], v[1:2], v[3:4]
+// GFX12: encoding: [0x01,0x07,0x0a,0x1c]
+
+v_max_num_f64 v[5:6], v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x0a,0x1c]
+
+v_max_num_f64 v[5:6], s[0:1], v[2:3]
+// GFX12: encoding: [0x00,0x04,0x0a,0x1c]
+
+v_max_num_f64 v[5:6], s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x0a,0x1c]
+
+v_max_num_f64 v[5:6], vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x0a,0x1c]
+
+v_max_num_f64 v[5:6], ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x0a,0x1c]
+
+v_max_num_f64 v[5:6], exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x0a,0x1c]
+
+v_max_num_f64 v[5:6], null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x0a,0x1c]
+
+v_max_num_f64 v[5:6], -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x0a,0x1c]
+
+v_max_num_f64 v[5:6], 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x0a,0x1c]
+
+v_max_num_f64 v[5:6], src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x0a,0x1c]
+
+v_max_num_f64 v[254:255], 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xfd,0x1d,0x56,0x34,0x12,0xaf]
+
+v_max_i32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x24]
+
+v_max_i32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x24]
+
+v_max_i32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x24]
+
+v_max_i32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x24]
+
+v_max_i32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x24]
+
+v_max_i32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x24]
+
+v_max_i32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x24]
+
+v_max_i32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x24]
+
+v_max_i32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x24]
+
+v_max_i32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x24]
+
+v_max_i32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x24]
+
+v_max_i32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x24]
+
+v_max_i32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x24]
+
+v_max_i32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x24]
+
+v_max_i32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x25,0x56,0x34,0x12,0xaf]
+
+v_max_u32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x28]
+
+v_max_u32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x28]
+
+v_max_u32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x28]
+
+v_max_u32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x28]
+
+v_max_u32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x28]
+
+v_max_u32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x28]
+
+v_max_u32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x28]
+
+v_max_u32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x28]
+
+v_max_u32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x28]
+
+v_max_u32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x28]
+
+v_max_u32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x28]
+
+v_max_u32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x28]
+
+v_max_u32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x28]
+
+v_max_u32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x28]
+
+v_max_u32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x29,0x56,0x34,0x12,0xaf]
+
+v_min_num_f16 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x60]
+
+v_min_num_f16 v5, v127, v2
+// GFX12: encoding: [0x7f,0x05,0x0a,0x60]
+
+v_min_num_f16 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x60]
+
+v_min_num_f16 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x60]
+
+v_min_num_f16 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x60]
+
+v_min_num_f16 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x60]
+
+v_min_num_f16 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x60]
+
+v_min_num_f16 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x60]
+
+v_min_num_f16 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x60]
+
+v_min_num_f16 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x60]
+
+v_min_num_f16 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x60]
+
+v_min_num_f16 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x60]
+
+v_min_num_f16 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x60]
+
+v_min_num_f16 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x60]
+
+v_min_num_f16 v127, 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0xfe,0x60,0x0b,0xfe,0x00,0x00]
+
+v_min_num_f32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x2a]
+
+v_min_num_f32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x2a]
+
+v_min_num_f32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x2a]
+
+v_min_num_f32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x2a]
+
+v_min_num_f32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x2a]
+
+v_min_num_f32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x2a]
+
+v_min_num_f32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x2a]
+
+v_min_num_f32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x2a]
+
+v_min_num_f32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x2a]
+
+v_min_num_f32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x2a]
+
+v_min_num_f32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x2a]
+
+v_min_num_f32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x2a]
+
+v_min_num_f32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x2a]
+
+v_min_num_f32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x2a]
+
+v_min_num_f32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x2b,0x56,0x34,0x12,0xaf]
+
+v_min_num_f64 v[5:6], v[1:2], v[3:4]
+// GFX12: encoding: [0x01,0x07,0x0a,0x1a]
+
+v_min_num_f64 v[5:6], v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x0a,0x1a]
+
+v_min_num_f64 v[5:6], s[0:1], v[2:3]
+// GFX12: encoding: [0x00,0x04,0x0a,0x1a]
+
+v_min_num_f64 v[5:6], s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x0a,0x1a]
+
+v_min_num_f64 v[5:6], vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x0a,0x1a]
+
+v_min_num_f64 v[5:6], ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x0a,0x1a]
+
+v_min_num_f64 v[5:6], exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x0a,0x1a]
+
+v_min_num_f64 v[5:6], null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x0a,0x1a]
+
+v_min_num_f64 v[5:6], -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x0a,0x1a]
+
+v_min_num_f64 v[5:6], 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x0a,0x1a]
+
+v_min_num_f64 v[5:6], src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x0a,0x1a]
+
+v_min_num_f64 v[254:255], 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xfd,0x1b,0x56,0x34,0x12,0xaf]
+
+v_min_i32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x22]
+
+v_min_i32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x22]
+
+v_min_i32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x22]
+
+v_min_i32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x22]
+
+v_min_i32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x22]
+
+v_min_i32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x22]
+
+v_min_i32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x22]
+
+v_min_i32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x22]
+
+v_min_i32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x22]
+
+v_min_i32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x22]
+
+v_min_i32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x22]
+
+v_min_i32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x22]
+
+v_min_i32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x22]
+
+v_min_i32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x22]
+
+v_min_i32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x23,0x56,0x34,0x12,0xaf]
+
+v_min_u32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x26]
+
+v_min_u32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x26]
+
+v_min_u32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x26]
+
+v_min_u32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x26]
+
+v_min_u32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x26]
+
+v_min_u32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x26]
+
+v_min_u32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x26]
+
+v_min_u32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x26]
+
+v_min_u32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x26]
+
+v_min_u32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x26]
+
+v_min_u32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x26]
+
+v_min_u32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x26]
+
+v_min_u32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x26]
+
+v_min_u32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x26]
+
+v_min_u32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x27,0x56,0x34,0x12,0xaf]
+
+v_mul_dx9_zero_f32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x0e]
+
+v_mul_dx9_zero_f32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x0f,0x56,0x34,0x12,0xaf]
+
+v_mul_f16 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x6a]
+
+v_mul_f16 v5, v127, v2
+// GFX12: encoding: [0x7f,0x05,0x0a,0x6a]
+
+v_mul_f16 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x6a]
+
+v_mul_f16 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x6a]
+
+v_mul_f16 v127, 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0xfe,0x6a,0x0b,0xfe,0x00,0x00]
+
+v_mul_f32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x10]
+
+v_mul_f32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x10]
+
+v_mul_f32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x10]
+
+v_mul_f32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x10]
+
+v_mul_f32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x10]
+
+v_mul_f32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x10]
+
+v_mul_f32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x10]
+
+v_mul_f32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x10]
+
+v_mul_f32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x10]
+
+v_mul_f32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x10]
+
+v_mul_f32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x10]
+
+v_mul_f32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x10]
+
+v_mul_f32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x10]
+
+v_mul_f32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x10]
+
+v_mul_f32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x11,0x56,0x34,0x12,0xaf]
+
+v_mul_f64 v[5:6], v[1:2], v[3:4]
+// GFX12: encoding: [0x01,0x07,0x0a,0x0c]
+
+v_mul_f64 v[5:6], v[254:255], v[2:3]
+// GFX12: encoding: [0xfe,0x05,0x0a,0x0c]
+
+v_mul_f64 v[5:6], s[0:1], v[2:3]
+// GFX12: encoding: [0x00,0x04,0x0a,0x0c]
+
+v_mul_f64 v[5:6], s[104:105], v[2:3]
+// GFX12: encoding: [0x68,0x04,0x0a,0x0c]
+
+v_mul_f64 v[5:6], vcc, v[2:3]
+// GFX12: encoding: [0x6a,0x04,0x0a,0x0c]
+
+v_mul_f64 v[5:6], ttmp[14:15], v[2:3]
+// GFX12: encoding: [0x7a,0x04,0x0a,0x0c]
+
+v_mul_f64 v[5:6], exec, v[2:3]
+// GFX12: encoding: [0x7e,0x04,0x0a,0x0c]
+
+v_mul_f64 v[5:6], null, v[2:3]
+// GFX12: encoding: [0x7c,0x04,0x0a,0x0c]
+
+v_mul_f64 v[5:6], -1, v[2:3]
+// GFX12: encoding: [0xc1,0x04,0x0a,0x0c]
+
+v_mul_f64 v[5:6], 0.5, v[2:3]
+// GFX12: encoding: [0xf0,0x04,0x0a,0x0c]
+
+v_mul_f64 v[5:6], src_scc, v[2:3]
+// GFX12: encoding: [0xfd,0x04,0x0a,0x0c]
+
+v_mul_f64 v[254:255], 0xaf123456, v[254:255]
+// GFX12: encoding: [0xff,0xfc,0xfd,0x0d,0x56,0x34,0x12,0xaf]
+
+v_mul_hi_i32_i24 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x14]
+
+v_mul_hi_i32_i24 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x15,0x56,0x34,0x12,0xaf]
+
+v_mul_hi_u32_u24 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x18]
+
+v_mul_hi_u32_u24 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x19,0x56,0x34,0x12,0xaf]
+
+v_mul_i32_i24 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x12]
+
+v_mul_i32_i24 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x12]
+
+v_mul_i32_i24 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x12]
+
+v_mul_i32_i24 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x13,0x56,0x34,0x12,0xaf]
+
+v_mul_legacy_f32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x0e]
+
+v_mul_legacy_f32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x0f,0x56,0x34,0x12,0xaf]
+
+v_mul_u32_u24 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x16]
+
+v_mul_u32_u24 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x16]
+
+v_mul_u32_u24 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x16]
+
+v_mul_u32_u24 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x17,0x56,0x34,0x12,0xaf]
+
+v_or_b32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x38]
+
+v_or_b32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x38]
+
+v_or_b32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x38]
+
+v_or_b32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x38]
+
+v_or_b32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x38]
+
+v_or_b32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x38]
+
+v_or_b32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x38]
+
+v_or_b32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x38]
+
+v_or_b32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x38]
+
+v_or_b32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x38]
+
+v_or_b32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x38]
+
+v_or_b32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x38]
+
+v_or_b32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x38]
+
+v_or_b32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x38]
+
+v_or_b32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x39,0x56,0x34,0x12,0xaf]
+
+v_pk_fmac_f16 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x78]
+
+v_pk_fmac_f16 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x78]
+
+v_pk_fmac_f16 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x78]
+
+v_pk_fmac_f16 v255, 0xfe0b, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x79,0x0b,0xfe,0x00,0x00]
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo
+// W32: encoding: [0x01,0x05,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v255, v2, vcc_lo
+// W32: encoding: [0xff,0x05,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, s1, v2, vcc_lo
+// W32: encoding: [0x01,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, s105, v2, vcc_lo
+// W32: encoding: [0x69,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, vcc_lo, v2, vcc_lo
+// W32: encoding: [0x6a,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, vcc_hi, v2, vcc_lo
+// W32: encoding: [0x6b,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, ttmp15, v2, vcc_lo
+// W32: encoding: [0x7b,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, m0, v2, vcc_lo
+// W32: encoding: [0x7d,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, exec_lo, v2, vcc_lo
+// W32: encoding: [0x7e,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, exec_hi, v2, vcc_lo
+// W32: encoding: [0x7f,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, null, v2, vcc_lo
+// W32: encoding: [0x7c,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, -1, v2, vcc_lo
+// W32: encoding: [0xc1,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, 0.5, v2, vcc_lo
+// W32: encoding: [0xf0,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, src_scc, v2, vcc_lo
+// W32: encoding: [0xfd,0x04,0x0a,0x42]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v255, vcc_lo, 0xaf123456, v255, vcc_lo
+// W32: encoding: [0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc
+// W64: encoding: [0x01,0x05,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v255, v2, vcc
+// W64: encoding: [0xff,0x05,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, s1, v2, vcc
+// W64: encoding: [0x01,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, s105, v2, vcc
+// W64: encoding: [0x69,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, vcc_lo, v2, vcc
+// W64: encoding: [0x6a,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, vcc_hi, v2, vcc
+// W64: encoding: [0x6b,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, ttmp15, v2, vcc
+// W64: encoding: [0x7b,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, m0, v2, vcc
+// W64: encoding: [0x7d,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, exec_lo, v2, vcc
+// W64: encoding: [0x7e,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, exec_hi, v2, vcc
+// W64: encoding: [0x7f,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, null, v2, vcc
+// W64: encoding: [0x7c,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, -1, v2, vcc
+// W64: encoding: [0xc1,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, 0.5, v2, vcc
+// W64: encoding: [0xf0,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, src_scc, v2, vcc
+// W64: encoding: [0xfd,0x04,0x0a,0x42]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v255, vcc, 0xaf123456, v255, vcc
+// W64: encoding: [0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_f16 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x66]
+
+v_sub_f16 v5, v127, v2
+// GFX12: encoding: [0x7f,0x05,0x0a,0x66]
+
+v_sub_f16 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x66]
+
+v_sub_f16 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x66]
+
+v_sub_f16 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x66]
+
+v_sub_f16 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x66]
+
+v_sub_f16 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x66]
+
+v_sub_f16 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x66]
+
+v_sub_f16 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x66]
+
+v_sub_f16 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x66]
+
+v_sub_f16 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x66]
+
+v_sub_f16 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x66]
+
+v_sub_f16 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x66]
+
+v_sub_f16 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x66]
+
+v_sub_f16 v127, 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00]
+
+v_sub_f32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x08]
+
+v_sub_f32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x08]
+
+v_sub_f32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x08]
+
+v_sub_f32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x08]
+
+v_sub_f32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x08]
+
+v_sub_f32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x08]
+
+v_sub_f32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x08]
+
+v_sub_f32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x08]
+
+v_sub_f32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x08]
+
+v_sub_f32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x08]
+
+v_sub_f32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x08]
+
+v_sub_f32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x08]
+
+v_sub_f32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x08]
+
+v_sub_f32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x08]
+
+v_sub_f32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x09,0x56,0x34,0x12,0xaf]
+
+v_sub_nc_u32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x4c]
+
+v_sub_nc_u32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x4c]
+
+v_sub_nc_u32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x4c]
+
+v_sub_nc_u32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x4d,0x56,0x34,0x12,0xaf]
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo
+// W32: encoding: [0x01,0x05,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v255, v2, vcc_lo
+// W32: encoding: [0xff,0x05,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, s1, v2, vcc_lo
+// W32: encoding: [0x01,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, s105, v2, vcc_lo
+// W32: encoding: [0x69,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, vcc_lo, v2, vcc_lo
+// W32: encoding: [0x6a,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, vcc_hi, v2, vcc_lo
+// W32: encoding: [0x6b,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, ttmp15, v2, vcc_lo
+// W32: encoding: [0x7b,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, m0, v2, vcc_lo
+// W32: encoding: [0x7d,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, exec_lo, v2, vcc_lo
+// W32: encoding: [0x7e,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, exec_hi, v2, vcc_lo
+// W32: encoding: [0x7f,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, null, v2, vcc_lo
+// W32: encoding: [0x7c,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, -1, v2, vcc_lo
+// W32: encoding: [0xc1,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, 0.5, v2, vcc_lo
+// W32: encoding: [0xf0,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, src_scc, v2, vcc_lo
+// W32: encoding: [0xfd,0x04,0x0a,0x44]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v255, vcc_lo, 0xaf123456, v255, vcc_lo
+// W32: encoding: [0xff,0xfe,0xff,0x45,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc
+// W64: encoding: [0x01,0x05,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v255, v2, vcc
+// W64: encoding: [0xff,0x05,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, s1, v2, vcc
+// W64: encoding: [0x01,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, s105, v2, vcc
+// W64: encoding: [0x69,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, vcc_lo, v2, vcc
+// W64: encoding: [0x6a,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, vcc_hi, v2, vcc
+// W64: encoding: [0x6b,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, ttmp15, v2, vcc
+// W64: encoding: [0x7b,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, m0, v2, vcc
+// W64: encoding: [0x7d,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, exec_lo, v2, vcc
+// W64: encoding: [0x7e,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, exec_hi, v2, vcc
+// W64: encoding: [0x7f,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, null, v2, vcc
+// W64: encoding: [0x7c,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, -1, v2, vcc
+// W64: encoding: [0xc1,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, 0.5, v2, vcc
+// W64: encoding: [0xf0,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, src_scc, v2, vcc
+// W64: encoding: [0xfd,0x04,0x0a,0x44]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v255, vcc, 0xaf123456, v255, vcc
+// W64: encoding: [0xff,0xfe,0xff,0x45,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_f16 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x68]
+
+v_subrev_f16 v5, v127, v2
+// GFX12: encoding: [0x7f,0x05,0x0a,0x68]
+
+v_subrev_f16 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x68]
+
+v_subrev_f16 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x68]
+
+v_subrev_f16 v127, 0xfe0b, v127
+// GFX12: encoding: [0xff,0xfe,0xfe,0x68,0x0b,0xfe,0x00,0x00]
+
+v_subrev_f32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x0a]
+
+v_subrev_f32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x0a]
+
+v_subrev_f32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x0a]
+
+v_subrev_f32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x0a]
+
+v_subrev_f32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x0b,0x56,0x34,0x12,0xaf]
+
+v_subrev_nc_u32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x4e]
+
+v_subrev_nc_u32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x4f,0x56,0x34,0x12,0xaf]
+
+v_xnor_b32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x3c]
+
+v_xnor_b32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x3c]
+
+v_xnor_b32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x3c]
+
+v_xnor_b32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x3c]
+
+v_xnor_b32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x3d,0x56,0x34,0x12,0xaf]
+
+v_xor_b32 v5, v1, v2
+// GFX12: encoding: [0x01,0x05,0x0a,0x3a]
+
+v_xor_b32 v5, v255, v2
+// GFX12: encoding: [0xff,0x05,0x0a,0x3a]
+
+v_xor_b32 v5, s1, v2
+// GFX12: encoding: [0x01,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, s105, v2
+// GFX12: encoding: [0x69,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, vcc_lo, v2
+// GFX12: encoding: [0x6a,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, vcc_hi, v2
+// GFX12: encoding: [0x6b,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, ttmp15, v2
+// GFX12: encoding: [0x7b,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, m0, v2
+// GFX12: encoding: [0x7d,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, exec_lo, v2
+// GFX12: encoding: [0x7e,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, exec_hi, v2
+// GFX12: encoding: [0x7f,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, null, v2
+// GFX12: encoding: [0x7c,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, -1, v2
+// GFX12: encoding: [0xc1,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, 0.5, v2
+// GFX12: encoding: [0xf0,0x04,0x0a,0x3a]
+
+v_xor_b32 v5, src_scc, v2
+// GFX12: encoding: [0xfd,0x04,0x0a,0x3a]
+
+v_xor_b32 v255, 0xaf123456, v255
+// GFX12: encoding: [0xff,0xfe,0xff,0x3b,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2.s
index 08d4be088131..5593ea77d942 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2.s
@@ -1,7 +1,7 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_add_co_ci_u32_e32 v5, vcc_lo, v1, v2, vcc_lo
 // W32: encoding: [0x01,0x05,0x0a,0x40]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_aliases-fake16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_aliases-fake16.s
new file mode 100644
index 000000000000..ebab0859b348
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_aliases-fake16.s
@@ -0,0 +1,19 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+
+v_min_f32 v5, v1, v2
+// GFX12: v_min_num_f32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x2a]
+
+v_max_f32 v5, v1, v2
+// GFX12: v_max_num_f32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x2c]
+
+v_min_f16 v5, v1, v2
+// GFX12: v_min_num_f16_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x60]
+
+v_max_f16 v5, v1, v2
+// GFX12: v_max_num_f16_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x62]
+
+v_max_f64 v[5:6], v[1:2], v[2:3]
+// GFX12: v_max_num_f64_e32 v[5:6], v[1:2], v[2:3] ; encoding: [0x01,0x05,0x0a,0x1c]
+
+v_min_f64 v[5:6], v[1:2], v[2:3]
+// GFX12: v_min_num_f64_e32 v[5:6], v[1:2], v[2:3] ; encoding: [0x01,0x05,0x0a,0x1a]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_aliases.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_aliases.s
index 3918dd48cfc0..b7e51cf27064 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_aliases.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_aliases.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
 
 v_min_f32 v5, v1, v2
 // GFX12: v_min_num_f32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x2a]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16-fake16.s
new file mode 100644
index 000000000000..53373d1f4697
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16-fake16.s
@@ -0,0 +1,2006 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_half_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shl:1
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shl:15
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shr:1
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shr:15
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_ror:1
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_ror:15
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v255, vcc_lo, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0xff,0x41,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_half_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_shl:1
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_shl:15
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_shr:1
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_shr:15
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_ror:1
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_ror:15
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0xff,0x41,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
+
+v_add_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x64,0x01,0xe4,0x00,0xff]
+
+v_add_f16 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x40,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x41,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x01,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x0f,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x11,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1f,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x21,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x2f,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff]
+
+v_add_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01]
+
+v_add_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x64,0x01,0x60,0x09,0x13]
+
+v_add_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xf5,0x30]
+
+v_add_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1b,0x00,0xff]
+
+v_add_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x06,0x01,0xe4,0x00,0xff]
+
+v_add_f32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x40,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x41,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x01,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x0f,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x11,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1f,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x21,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x2f,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x50,0x01,0xff]
+
+v_add_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x5f,0x01,0x01]
+
+v_add_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x06,0x01,0x60,0x09,0x13]
+
+v_add_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x07,0xff,0x6f,0xf5,0x30]
+
+v_add_nc_u32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x1b,0x00,0xff]
+
+v_add_nc_u32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0xe4,0x00,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x40,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x41,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x01,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x0f,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x11,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x1f,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x21,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x2f,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x50,0x01,0xff]
+
+v_add_nc_u32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x5f,0x01,0x01]
+
+v_add_nc_u32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x60,0x09,0x13]
+
+v_add_nc_u32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x4b,0xff,0x6f,0x05,0x30]
+
+v_and_b32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x1b,0x00,0xff]
+
+v_and_b32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x36,0x01,0xe4,0x00,0xff]
+
+v_and_b32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x40,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x41,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x01,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x0f,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x11,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x1f,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x21,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x2f,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x50,0x01,0xff]
+
+v_and_b32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x5f,0x01,0x01]
+
+v_and_b32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x36,0x01,0x60,0x09,0x13]
+
+v_and_b32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x37,0xff,0x6f,0x05,0x30]
+
+v_ashrrev_i32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x1b,0x00,0xff]
+
+v_ashrrev_i32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x34,0x01,0xe4,0x00,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x40,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x41,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x01,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x0f,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x11,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x1f,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x21,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x2f,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x50,0x01,0xff]
+
+v_ashrrev_i32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x5f,0x01,0x01]
+
+v_ashrrev_i32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x34,0x01,0x60,0x09,0x13]
+
+v_ashrrev_i32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x35,0xff,0x6f,0x05,0x30]
+
+v_cndmask_b32 v5, v1, v2, vcc_lo quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_half_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_shl:1
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_shl:15
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_shr:1
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_shr:15
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_ror:1
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_ror:15
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v255, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0xff,0x03,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_half_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_shl:1
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_shl:15
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_shr:1
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_shr:15
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_ror:1
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_ror:15
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x0a,0x02,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v255, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0xff,0x03,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1b,0x00,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0xe4,0x00,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x40,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x41,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x01,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x0f,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x11,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1f,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x21,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x2f,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x50,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x5f,0x01,0x01]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x60,0x09,0x13]
+
+v_cvt_pk_rtz_f16_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x5f,0xff,0x6f,0xf5,0x30]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1b,0x00,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0xe4,0x00,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x40,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x41,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x01,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x0f,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x11,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1f,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x21,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x2f,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x50,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x5f,0x01,0x01]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x60,0x09,0x13]
+
+v_cvt_pkrtz_f16_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x5f,0xff,0x6f,0xf5,0x30]
+
+v_fmac_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1b,0x00,0xff]
+
+v_fmac_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0xe4,0x00,0xff]
+
+v_fmac_f16 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x40,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x41,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x01,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x0f,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x11,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1f,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x21,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x2f,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x50,0x01,0xff]
+
+v_fmac_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x5f,0x01,0x01]
+
+v_fmac_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x60,0x09,0x13]
+
+v_fmac_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xfe,0x6c,0x7f,0x6f,0xf5,0x30]
+
+v_fmac_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x1b,0x00,0xff]
+
+v_fmac_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x56,0x01,0xe4,0x00,0xff]
+
+v_fmac_f32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x40,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x41,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x01,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x0f,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x11,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x1f,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x21,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x2f,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x50,0x01,0xff]
+
+v_fmac_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x5f,0x01,0x01]
+
+v_fmac_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x56,0x01,0x60,0x09,0x13]
+
+v_fmac_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x57,0xff,0x6f,0xf5,0x30]
+
+v_ldexp_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1b,0x00,0xff]
+
+v_ldexp_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x76,0x01,0xe4,0x00,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x40,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x41,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x01,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x0f,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x11,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1f,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x21,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x2f,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x50,0x01,0xff]
+
+v_ldexp_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x5f,0x01,0x01]
+
+v_ldexp_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x76,0x01,0x60,0x09,0x13]
+
+v_ldexp_f16 v127, -|v127|, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xfe,0x76,0x7f,0x6f,0x35,0x30]
+
+v_lshlrev_b32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x1b,0x00,0xff]
+
+v_lshlrev_b32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x30,0x01,0xe4,0x00,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x40,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x41,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x01,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x0f,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x11,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x1f,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x21,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x2f,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x50,0x01,0xff]
+
+v_lshlrev_b32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x5f,0x01,0x01]
+
+v_lshlrev_b32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x30,0x01,0x60,0x09,0x13]
+
+v_lshlrev_b32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x31,0xff,0x6f,0x05,0x30]
+
+v_lshrrev_b32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x1b,0x00,0xff]
+
+v_lshrrev_b32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x32,0x01,0xe4,0x00,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x40,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x41,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x01,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x0f,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x11,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x1f,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x21,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x2f,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x50,0x01,0xff]
+
+v_lshrrev_b32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x5f,0x01,0x01]
+
+v_lshrrev_b32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x32,0x01,0x60,0x09,0x13]
+
+v_lshrrev_b32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x33,0xff,0x6f,0x05,0x30]
+
+v_max_num_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x62,0x01,0x1b,0x00,0xff]
+
+v_max_num_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x62,0x01,0xe4,0x00,0xff]
+
+v_max_num_f16 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x62,0x01,0x40,0x01,0xff]
+
+v_max_num_f16 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x62,0x01,0x41,0x01,0xff]
+
+v_max_num_f16 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x62,0x01,0x01,0x01,0xff]
+
+v_max_num_f16 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x62,0x01,0x0f,0x01,0xff]
+
+v_max_num_f16 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x62,0x01,0x11,0x01,0xff]
+
+v_max_num_f16 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x62,0x01,0x1f,0x01,0xff]
+
+v_max_num_f16 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x62,0x01,0x21,0x01,0xff]
+
+v_max_num_f16 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x62,0x01,0x2f,0x01,0xff]
+
+v_max_num_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x62,0x01,0x50,0x01,0xff]
+
+v_max_num_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x62,0x01,0x5f,0x01,0x01]
+
+v_max_num_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x62,0x01,0x60,0x09,0x13]
+
+v_max_num_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xfe,0x62,0x7f,0x6f,0xf5,0x30]
+
+v_max_num_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x1b,0x00,0xff]
+
+v_max_num_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2c,0x01,0xe4,0x00,0xff]
+
+v_max_num_f32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x40,0x01,0xff]
+
+v_max_num_f32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x41,0x01,0xff]
+
+v_max_num_f32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x01,0x01,0xff]
+
+v_max_num_f32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x0f,0x01,0xff]
+
+v_max_num_f32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x11,0x01,0xff]
+
+v_max_num_f32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x1f,0x01,0xff]
+
+v_max_num_f32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x21,0x01,0xff]
+
+v_max_num_f32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x2f,0x01,0xff]
+
+v_max_num_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x50,0x01,0xff]
+
+v_max_num_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x5f,0x01,0x01]
+
+v_max_num_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x60,0x09,0x13]
+
+v_max_num_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x2d,0xff,0x6f,0xf5,0x30]
+
+v_max_i32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x1b,0x00,0xff]
+
+v_max_i32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x24,0x01,0xe4,0x00,0xff]
+
+v_max_i32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x40,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x41,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x01,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x0f,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x11,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x1f,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x21,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x2f,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x50,0x01,0xff]
+
+v_max_i32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x5f,0x01,0x01]
+
+v_max_i32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x24,0x01,0x60,0x09,0x13]
+
+v_max_i32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x25,0xff,0x6f,0x05,0x30]
+
+v_max_u32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x1b,0x00,0xff]
+
+v_max_u32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x28,0x01,0xe4,0x00,0xff]
+
+v_max_u32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x40,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x41,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x01,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x0f,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x11,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x1f,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x21,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x2f,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x50,0x01,0xff]
+
+v_max_u32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x5f,0x01,0x01]
+
+v_max_u32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x28,0x01,0x60,0x09,0x13]
+
+v_max_u32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x29,0xff,0x6f,0x05,0x30]
+
+v_min_num_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x60,0x01,0x1b,0x00,0xff]
+
+v_min_num_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x60,0x01,0xe4,0x00,0xff]
+
+v_min_num_f16 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x60,0x01,0x40,0x01,0xff]
+
+v_min_num_f16 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x60,0x01,0x41,0x01,0xff]
+
+v_min_num_f16 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x60,0x01,0x01,0x01,0xff]
+
+v_min_num_f16 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x60,0x01,0x0f,0x01,0xff]
+
+v_min_num_f16 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x60,0x01,0x11,0x01,0xff]
+
+v_min_num_f16 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x60,0x01,0x1f,0x01,0xff]
+
+v_min_num_f16 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x60,0x01,0x21,0x01,0xff]
+
+v_min_num_f16 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x60,0x01,0x2f,0x01,0xff]
+
+v_min_num_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x60,0x01,0x50,0x01,0xff]
+
+v_min_num_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x60,0x01,0x5f,0x01,0x01]
+
+v_min_num_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x60,0x01,0x60,0x09,0x13]
+
+v_min_num_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xfe,0x60,0x7f,0x6f,0xf5,0x30]
+
+v_min_num_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x1b,0x00,0xff]
+
+v_min_num_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2a,0x01,0xe4,0x00,0xff]
+
+v_min_num_f32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x40,0x01,0xff]
+
+v_min_num_f32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x41,0x01,0xff]
+
+v_min_num_f32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x01,0x01,0xff]
+
+v_min_num_f32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x0f,0x01,0xff]
+
+v_min_num_f32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x11,0x01,0xff]
+
+v_min_num_f32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x1f,0x01,0xff]
+
+v_min_num_f32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x21,0x01,0xff]
+
+v_min_num_f32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x2f,0x01,0xff]
+
+v_min_num_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x50,0x01,0xff]
+
+v_min_num_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x5f,0x01,0x01]
+
+v_min_num_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x60,0x09,0x13]
+
+v_min_num_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x2b,0xff,0x6f,0xf5,0x30]
+
+v_min_i32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x1b,0x00,0xff]
+
+v_min_i32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x22,0x01,0xe4,0x00,0xff]
+
+v_min_i32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x40,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x41,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x01,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x0f,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x11,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x1f,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x21,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x2f,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x50,0x01,0xff]
+
+v_min_i32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x5f,0x01,0x01]
+
+v_min_i32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x22,0x01,0x60,0x09,0x13]
+
+v_min_i32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x23,0xff,0x6f,0x05,0x30]
+
+v_min_u32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x1b,0x00,0xff]
+
+v_min_u32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x26,0x01,0xe4,0x00,0xff]
+
+v_min_u32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x40,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x41,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x01,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x0f,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x11,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x1f,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x21,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x2f,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x50,0x01,0xff]
+
+v_min_u32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x5f,0x01,0x01]
+
+v_min_u32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x26,0x01,0x60,0x09,0x13]
+
+v_min_u32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x27,0xff,0x6f,0x05,0x30]
+
+v_mul_dx9_zero_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1b,0x00,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0xe4,0x00,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x40,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x41,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x01,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x0f,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x11,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1f,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x21,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x2f,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x50,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x5f,0x01,0x01]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x60,0x09,0x13]
+
+v_mul_dx9_zero_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x0f,0xff,0x6f,0xf5,0x30]
+
+v_mul_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1b,0x00,0xff]
+
+v_mul_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0xe4,0x00,0xff]
+
+v_mul_f16 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x40,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x41,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x01,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x0f,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x11,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1f,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x21,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x2f,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x50,0x01,0xff]
+
+v_mul_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x5f,0x01,0x01]
+
+v_mul_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x60,0x09,0x13]
+
+v_mul_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xfe,0x6a,0x7f,0x6f,0xf5,0x30]
+
+v_mul_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x1b,0x00,0xff]
+
+v_mul_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x10,0x01,0xe4,0x00,0xff]
+
+v_mul_f32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x40,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x41,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x01,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x0f,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x11,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x1f,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x21,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x2f,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x50,0x01,0xff]
+
+v_mul_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x5f,0x01,0x01]
+
+v_mul_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x10,0x01,0x60,0x09,0x13]
+
+v_mul_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x11,0xff,0x6f,0xf5,0x30]
+
+v_mul_hi_i32_i24 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x1b,0x00,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x14,0x01,0xe4,0x00,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x40,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x41,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x01,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x0f,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x11,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x1f,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x21,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x2f,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x50,0x01,0xff]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x5f,0x01,0x01]
+
+v_mul_hi_i32_i24 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x14,0x01,0x60,0x09,0x13]
+
+v_mul_hi_i32_i24 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x15,0xff,0x6f,0x05,0x30]
+
+v_mul_hi_u32_u24 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x1b,0x00,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x18,0x01,0xe4,0x00,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x40,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x41,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x01,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x0f,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x11,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x1f,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x21,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x2f,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x50,0x01,0xff]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x5f,0x01,0x01]
+
+v_mul_hi_u32_u24 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x18,0x01,0x60,0x09,0x13]
+
+v_mul_hi_u32_u24 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x19,0xff,0x6f,0x05,0x30]
+
+v_mul_i32_i24 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x1b,0x00,0xff]
+
+v_mul_i32_i24 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x12,0x01,0xe4,0x00,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x40,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x41,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x01,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x0f,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x11,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x1f,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x21,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x2f,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x50,0x01,0xff]
+
+v_mul_i32_i24 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x5f,0x01,0x01]
+
+v_mul_i32_i24 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x12,0x01,0x60,0x09,0x13]
+
+v_mul_i32_i24 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x13,0xff,0x6f,0x05,0x30]
+
+v_mul_dx9_zero_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1b,0x00,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0xe4,0x00,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x40,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x41,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x01,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x0f,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x11,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1f,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x21,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x2f,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x50,0x01,0xff]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x5f,0x01,0x01]
+
+v_mul_dx9_zero_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x60,0x09,0x13]
+
+v_mul_dx9_zero_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x0f,0xff,0x6f,0xf5,0x30]
+
+v_mul_u32_u24 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x1b,0x00,0xff]
+
+v_mul_u32_u24 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x16,0x01,0xe4,0x00,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x40,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x41,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x01,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x0f,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x11,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x1f,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x21,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x2f,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x50,0x01,0xff]
+
+v_mul_u32_u24 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x5f,0x01,0x01]
+
+v_mul_u32_u24 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x16,0x01,0x60,0x09,0x13]
+
+v_mul_u32_u24 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x17,0xff,0x6f,0x05,0x30]
+
+v_or_b32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x1b,0x00,0xff]
+
+v_or_b32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x38,0x01,0xe4,0x00,0xff]
+
+v_or_b32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x40,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x41,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x01,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x0f,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x11,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x1f,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x21,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x2f,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x50,0x01,0xff]
+
+v_or_b32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x5f,0x01,0x01]
+
+v_or_b32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x38,0x01,0x60,0x09,0x13]
+
+v_or_b32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x39,0xff,0x6f,0x05,0x30]
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_half_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shl:1
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shl:15
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shr:1
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shr:15
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_ror:1
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_ror:15
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v255, vcc_lo, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0xff,0x43,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_half_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_shl:1
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_shl:15
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_shr:1
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_shr:15
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_ror:1
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_ror:15
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x0a,0x42,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0xff,0x43,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1b,0x00,0xff]
+
+v_sub_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x66,0x01,0xe4,0x00,0xff]
+
+v_sub_f16 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x40,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x41,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x01,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x0f,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x11,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1f,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x21,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x2f,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff]
+
+v_sub_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01]
+
+v_sub_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x66,0x01,0x60,0x09,0x13]
+
+v_sub_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xf5,0x30]
+
+v_sub_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1b,0x00,0xff]
+
+v_sub_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x08,0x01,0xe4,0x00,0xff]
+
+v_sub_f32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x40,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x41,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x01,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x0f,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x11,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1f,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x21,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x2f,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x50,0x01,0xff]
+
+v_sub_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x5f,0x01,0x01]
+
+v_sub_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x08,0x01,0x60,0x09,0x13]
+
+v_sub_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x09,0xff,0x6f,0xf5,0x30]
+
+v_sub_nc_u32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x1b,0x00,0xff]
+
+v_sub_nc_u32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0xe4,0x00,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x40,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x41,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x01,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x0f,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x11,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x1f,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x21,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x2f,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x50,0x01,0xff]
+
+v_sub_nc_u32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x5f,0x01,0x01]
+
+v_sub_nc_u32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x60,0x09,0x13]
+
+v_sub_nc_u32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x4d,0xff,0x6f,0x05,0x30]
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0]
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo quad_perm:[0,1,2,3]
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_half_mirror
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shl:1
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shl:15
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shr:1
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_shr:15
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_ror:1
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_ror:15
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v255, vcc_lo, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W32: encoding: [0xfa,0xfe,0xff,0x45,0xff,0x6f,0x05,0x30]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc quad_perm:[3,2,1,0]
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc quad_perm:[0,1,2,3]
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_half_mirror
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_shl:1
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_shl:15
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_shr:1
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_shr:15
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_ror:1
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_ror:15
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: encoding: [0xfa,0x04,0x0a,0x44,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// W64: encoding: [0xfa,0xfe,0xff,0x45,0xff,0x6f,0x05,0x30]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1b,0x00,0xff]
+
+v_subrev_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x68,0x01,0xe4,0x00,0xff]
+
+v_subrev_f16 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x40,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x41,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x01,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x0f,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x11,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1f,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x21,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x2f,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x50,0x01,0xff]
+
+v_subrev_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x5f,0x01,0x01]
+
+v_subrev_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x68,0x01,0x60,0x09,0x13]
+
+v_subrev_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xfe,0x68,0x7f,0x6f,0xf5,0x30]
+
+v_subrev_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x1b,0x00,0xff]
+
+v_subrev_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0xe4,0x00,0xff]
+
+v_subrev_f32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x40,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x41,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x01,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x0f,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x11,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x1f,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x21,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x2f,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x50,0x01,0xff]
+
+v_subrev_f32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x5f,0x01,0x01]
+
+v_subrev_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x60,0x09,0x13]
+
+v_subrev_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x0b,0xff,0x6f,0xf5,0x30]
+
+v_subrev_nc_u32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x1b,0x00,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0xe4,0x00,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x40,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x41,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x01,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x0f,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x11,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x1f,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x21,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x2f,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x50,0x01,0xff]
+
+v_subrev_nc_u32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x5f,0x01,0x01]
+
+v_subrev_nc_u32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x60,0x09,0x13]
+
+v_subrev_nc_u32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x4f,0xff,0x6f,0x05,0x30]
+
+v_xnor_b32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x1b,0x00,0xff]
+
+v_xnor_b32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0xe4,0x00,0xff]
+
+v_xnor_b32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x40,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x41,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x01,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x0f,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x11,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x1f,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x21,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x2f,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x50,0x01,0xff]
+
+v_xnor_b32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x5f,0x01,0x01]
+
+v_xnor_b32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x60,0x09,0x13]
+
+v_xnor_b32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x3d,0xff,0x6f,0x05,0x30]
+
+v_xor_b32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x1b,0x00,0xff]
+
+v_xor_b32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0xe4,0x00,0xff]
+
+v_xor_b32 v5, v1, v2 row_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x40,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_half_mirror
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x41,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_shl:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x01,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_shl:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x0f,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_shr:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x11,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_shr:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x1f,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_ror:1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x21,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_ror:15
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x2f,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x50,0x01,0xff]
+
+v_xor_b32 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x5f,0x01,0x01]
+
+v_xor_b32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x60,0x09,0x13]
+
+v_xor_b32 v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: encoding: [0xfa,0xfe,0xff,0x3b,0xff,0x6f,0x05,0x30]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s
index 63ffdbe821af..a0f93f459f91 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s
@@ -1,7 +1,7 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0]
 // W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8-fake16.s
new file mode 100644
index 000000000000..a7a035f4a9ef
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8-fake16.s
@@ -0,0 +1,433 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x0a,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v255, vcc_lo, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0xff,0x41,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x0a,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_co_ci_u32 v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0xff,0x41,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_add_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
+
+v_add_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
+
+v_add_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
+
+v_add_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x06,0x01,0x77,0x39,0x05]
+
+v_add_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x06,0x01,0x77,0x39,0x05]
+
+v_add_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x07,0xff,0x00,0x00,0x00]
+
+v_add_nc_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x4a,0x01,0x77,0x39,0x05]
+
+v_add_nc_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x4a,0x01,0x77,0x39,0x05]
+
+v_add_nc_u32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x4b,0xff,0x00,0x00,0x00]
+
+v_and_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x36,0x01,0x77,0x39,0x05]
+
+v_and_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x36,0x01,0x77,0x39,0x05]
+
+v_and_b32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x37,0xff,0x00,0x00,0x00]
+
+v_ashrrev_i32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x34,0x01,0x77,0x39,0x05]
+
+v_ashrrev_i32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x34,0x01,0x77,0x39,0x05]
+
+v_ashrrev_i32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x35,0xff,0x00,0x00,0x00]
+
+v_cndmask_b32 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x0a,0x02,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x0a,0x02,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v255, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0xff,0x03,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x0a,0x02,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v5, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x0a,0x02,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cndmask_b32 v255, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0xff,0x03,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x5e,0x01,0x77,0x39,0x05]
+
+v_cvt_pk_rtz_f16_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x5e,0x01,0x77,0x39,0x05]
+
+v_cvt_pk_rtz_f16_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x5f,0xff,0x00,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x5e,0x01,0x77,0x39,0x05]
+
+v_cvt_pkrtz_f16_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x5e,0x01,0x77,0x39,0x05]
+
+v_cvt_pkrtz_f16_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x5f,0xff,0x00,0x00,0x00]
+
+v_fmac_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05]
+
+v_fmac_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05]
+
+v_fmac_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xfe,0x6c,0x7f,0x00,0x00,0x00]
+
+v_fmac_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x56,0x01,0x77,0x39,0x05]
+
+v_fmac_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x56,0x01,0x77,0x39,0x05]
+
+v_fmac_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x57,0xff,0x00,0x00,0x00]
+
+v_ldexp_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x76,0x01,0x77,0x39,0x05]
+
+v_ldexp_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x76,0x01,0x77,0x39,0x05]
+
+v_ldexp_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xfe,0x76,0x7f,0x00,0x00,0x00]
+
+v_lshlrev_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x30,0x01,0x77,0x39,0x05]
+
+v_lshlrev_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x30,0x01,0x77,0x39,0x05]
+
+v_lshlrev_b32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x31,0xff,0x00,0x00,0x00]
+
+v_lshrrev_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x32,0x01,0x77,0x39,0x05]
+
+v_lshrrev_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x32,0x01,0x77,0x39,0x05]
+
+v_lshrrev_b32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x33,0xff,0x00,0x00,0x00]
+
+v_max_num_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x62,0x01,0x77,0x39,0x05]
+
+v_max_num_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x62,0x01,0x77,0x39,0x05]
+
+v_max_num_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xfe,0x62,0x7f,0x00,0x00,0x00]
+
+v_max_num_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x2c,0x01,0x77,0x39,0x05]
+
+v_max_num_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x2c,0x01,0x77,0x39,0x05]
+
+v_max_num_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x2d,0xff,0x00,0x00,0x00]
+
+v_max_i32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x24,0x01,0x77,0x39,0x05]
+
+v_max_i32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x24,0x01,0x77,0x39,0x05]
+
+v_max_i32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x25,0xff,0x00,0x00,0x00]
+
+v_max_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x28,0x01,0x77,0x39,0x05]
+
+v_max_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x28,0x01,0x77,0x39,0x05]
+
+v_max_u32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x29,0xff,0x00,0x00,0x00]
+
+v_min_num_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x60,0x01,0x77,0x39,0x05]
+
+v_min_num_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x60,0x01,0x77,0x39,0x05]
+
+v_min_num_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xfe,0x60,0x7f,0x00,0x00,0x00]
+
+v_min_num_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x2a,0x01,0x77,0x39,0x05]
+
+v_min_num_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x2a,0x01,0x77,0x39,0x05]
+
+v_min_num_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x2b,0xff,0x00,0x00,0x00]
+
+v_min_i32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x22,0x01,0x77,0x39,0x05]
+
+v_min_i32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x22,0x01,0x77,0x39,0x05]
+
+v_min_i32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x23,0xff,0x00,0x00,0x00]
+
+v_min_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x26,0x01,0x77,0x39,0x05]
+
+v_min_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x26,0x01,0x77,0x39,0x05]
+
+v_min_u32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x27,0xff,0x00,0x00,0x00]
+
+v_mul_dx9_zero_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x0e,0x01,0x77,0x39,0x05]
+
+v_mul_dx9_zero_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x0e,0x01,0x77,0x39,0x05]
+
+v_mul_dx9_zero_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x0f,0xff,0x00,0x00,0x00]
+
+v_mul_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x6a,0x01,0x77,0x39,0x05]
+
+v_mul_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x6a,0x01,0x77,0x39,0x05]
+
+v_mul_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xfe,0x6a,0x7f,0x00,0x00,0x00]
+
+v_mul_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x10,0x01,0x77,0x39,0x05]
+
+v_mul_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x10,0x01,0x77,0x39,0x05]
+
+v_mul_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x11,0xff,0x00,0x00,0x00]
+
+v_mul_hi_i32_i24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x14,0x01,0x77,0x39,0x05]
+
+v_mul_hi_i32_i24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x14,0x01,0x77,0x39,0x05]
+
+v_mul_hi_i32_i24 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x15,0xff,0x00,0x00,0x00]
+
+v_mul_hi_u32_u24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x18,0x01,0x77,0x39,0x05]
+
+v_mul_hi_u32_u24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x18,0x01,0x77,0x39,0x05]
+
+v_mul_hi_u32_u24 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x19,0xff,0x00,0x00,0x00]
+
+v_mul_i32_i24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x12,0x01,0x77,0x39,0x05]
+
+v_mul_i32_i24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x12,0x01,0x77,0x39,0x05]
+
+v_mul_i32_i24 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x13,0xff,0x00,0x00,0x00]
+
+v_mul_dx9_zero_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x0e,0x01,0x77,0x39,0x05]
+
+v_mul_dx9_zero_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x0e,0x01,0x77,0x39,0x05]
+
+v_mul_dx9_zero_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x0f,0xff,0x00,0x00,0x00]
+
+v_mul_u32_u24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x16,0x01,0x77,0x39,0x05]
+
+v_mul_u32_u24 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x16,0x01,0x77,0x39,0x05]
+
+v_mul_u32_u24 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x17,0xff,0x00,0x00,0x00]
+
+v_or_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x38,0x01,0x77,0x39,0x05]
+
+v_or_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x38,0x01,0x77,0x39,0x05]
+
+v_or_b32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x39,0xff,0x00,0x00,0x00]
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x0a,0x42,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x0a,0x42,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v255, vcc_lo, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0xff,0x43,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x0a,0x42,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x0a,0x42,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_co_ci_u32 v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0xff,0x43,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_sub_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
+
+v_sub_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
+
+v_sub_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
+
+v_sub_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x08,0x01,0x77,0x39,0x05]
+
+v_sub_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x08,0x01,0x77,0x39,0x05]
+
+v_sub_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x09,0xff,0x00,0x00,0x00]
+
+v_sub_nc_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x4c,0x01,0x77,0x39,0x05]
+
+v_sub_nc_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x4c,0x01,0x77,0x39,0x05]
+
+v_sub_nc_u32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x4d,0xff,0x00,0x00,0x00]
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: encoding: [0xe9,0x04,0x0a,0x44,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: encoding: [0xea,0x04,0x0a,0x44,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v255, vcc_lo, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W32: encoding: [0xe9,0xfe,0xff,0x45,0xff,0x00,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: encoding: [0xe9,0x04,0x0a,0x44,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: encoding: [0xea,0x04,0x0a,0x44,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_co_ci_u32 v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:0
+// W64: encoding: [0xe9,0xfe,0xff,0x45,0xff,0x00,0x00,0x00]
+// W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+v_subrev_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x68,0x01,0x77,0x39,0x05]
+
+v_subrev_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x68,0x01,0x77,0x39,0x05]
+
+v_subrev_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xfe,0x68,0x7f,0x00,0x00,0x00]
+
+v_subrev_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x0a,0x01,0x77,0x39,0x05]
+
+v_subrev_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x0a,0x01,0x77,0x39,0x05]
+
+v_subrev_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x0b,0xff,0x00,0x00,0x00]
+
+v_subrev_nc_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x4e,0x01,0x77,0x39,0x05]
+
+v_subrev_nc_u32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x4e,0x01,0x77,0x39,0x05]
+
+v_subrev_nc_u32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x4f,0xff,0x00,0x00,0x00]
+
+v_xnor_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x3c,0x01,0x77,0x39,0x05]
+
+v_xnor_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x3c,0x01,0x77,0x39,0x05]
+
+v_xnor_b32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x3d,0xff,0x00,0x00,0x00]
+
+v_xor_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: encoding: [0xe9,0x04,0x0a,0x3a,0x01,0x77,0x39,0x05]
+
+v_xor_b32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: encoding: [0xea,0x04,0x0a,0x3a,0x01,0x77,0x39,0x05]
+
+v_xor_b32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: encoding: [0xe9,0xfe,0xff,0x3b,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s
index 54baafb5366f..81fcb323e271 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s
@@ -1,7 +1,7 @@
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
 // W32: encoding: [0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_err.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_err.s
index 045d698bd504..b339bc1960f3 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_err.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_err.s
@@ -1,226 +1,227 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort --version 5
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s
 
-v_add_f16_e32 v255, v1, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmaak_f16_e32 v255, v1, v2, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmac_f16_e32 v255, v1, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmamk_f16_e32 v255, v1, 0xfe0b, v3
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_ldexp_f16_e32 v255, v1, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_max_num_f16_e32 v255, v1, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_add_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_min_num_f16_e32 v255, v1, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_add_f16_e32 v255, v1, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_mul_f16_e32 v255, v1, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_add_f16_e32 v5, v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16_e32 v255, v1, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_add_f16_e32 v5, v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_subrev_f16_e32 v255, v1, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmaak_f16_e32 v255, v1, v2, 0xfe0b
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16_e32 v5, v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmaak_f16_e32 v5, v1, v255, 0xfe0b
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_fmaak_f16_e32 v5, v255, v2, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmac_f16_e32 v5, v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_fmamk_f16_e32 v5, v255, 0xfe0b, v3
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmac_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_ldexp_f16_e32 v5, v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmac_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_max_num_f16_e32 v5, v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_fmac_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_min_num_f16_e32 v5, v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_fmac_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_mul_f16_e32 v5, v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmac_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16_e32 v5, v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmac_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_subrev_f16_e32 v5, v255, v2
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmac_f16_e32 v255, v1, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16_e32 v5, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmac_f16_e32 v5, v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmaak_f16_e32 v5, v1, v255, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmac_f16_e32 v5, v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmac_f16_e32 v5, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fmamk_f16_e32 v255, v1, 0xfe0b, v3
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_fmamk_f16_e32 v5, v1, 0xfe0b, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_max_num_f16_e32 v5, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_fmamk_f16_e32 v5, v255, 0xfe0b, v3
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_min_num_f16_e32 v5, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_ldexp_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_mul_f16_e32 v5, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_ldexp_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16_e32 v5, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_ldexp_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_subrev_f16_e32 v5, v1, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_ldexp_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_ldexp_f16_e32 v255, v1, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmac_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_ldexp_f16_e32 v5, v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_ldexp_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_max_num_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_max_num_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_min_num_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_max_num_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_mul_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_max_num_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_sub_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_max_num_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_subrev_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_max_num_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_add_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_max_num_f16_e32 v255, v1, v2
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_fmac_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_max_num_f16_e32 v5, v1, v255
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_ldexp_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_max_num_f16_e32 v5, v255, v2
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_max_num_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_min_num_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_min_num_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_min_num_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_mul_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_min_num_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_sub_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_min_num_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_subrev_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_min_num_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_add_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_min_num_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_fmac_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_min_num_f16_e32 v255, v1, v2
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_max_num_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_min_num_f16_e32 v5, v1, v255
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_min_num_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_min_num_f16_e32 v5, v255, v2
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_mul_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_mul_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_mul_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_subrev_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_mul_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_mul_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmac_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_mul_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_ldexp_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_mul_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_max_num_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_mul_f16_e32 v255, v1, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_min_num_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_mul_f16_e32 v5, v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_mul_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_mul_f16_e32 v5, v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_sub_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_subrev_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmac_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_ldexp_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_max_num_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sub_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_min_num_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sub_f16_e32 v255, v1, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_mul_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_sub_f16_e32 v5, v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_sub_f16_e32 v5, v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_subrev_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_subrev_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_subrev_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fmac_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_subrev_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_max_num_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_subrev_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_min_num_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_subrev_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_mul_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_subrev_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_subrev_f16_e32 v255, v1, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_subrev_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_subrev_f16_e32 v5, v1, v255
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_subrev_f16_e32 v5, v255, v2
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_promote.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_promote.s
index 13939842f730..e9e91fa70773 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_promote.s
@@ -1,190 +1,191 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort --version 5
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=_e32 %s
 
 v_add_f16 v255, v1, v2
-// GFX12: v_add_f16_e64
+// GFX12: v_add_f16_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x32,0xd5,0x01,0x05,0x02,0x00]
 
-v_fmac_f16 v255, v1, v2
-// GFX12: v_fmac_f16_e64
+v_add_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_ldexp_f16 v255, v1, v2
-// GFX12: v_ldexp_f16_e64
+v_add_f16 v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_max_num_f16 v255, v1, v2
-// GFX12: v_max_num_f16_e64
+v_add_f16 v5, v1, v255
+// GFX12: v_add_f16_e64 v5, v1, v255              ; encoding: [0x05,0x00,0x32,0xd5,0x01,0xff,0x03,0x00]
 
-v_min_num_f16 v255, v1, v2
-// GFX12: v_min_num_f16_e64
+v_add_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_mul_f16 v255, v1, v2
-// GFX12: v_mul_f16_e64
+v_add_f16 v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_sub_f16 v255, v1, v2
-// GFX12: v_sub_f16_e64
+v_add_f16 v5, v255, v2
+// GFX12: v_add_f16_e64 v5, v255, v2              ; encoding: [0x05,0x00,0x32,0xd5,0xff,0x05,0x02,0x00]
 
-v_subrev_f16 v255, v1, v2
-// GFX12: v_subrev_f16_e64
+v_add_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_add_f16 v5, v255, v2
-// GFX12: v_add_f16_e64
+v_add_f16 v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_fmac_f16 v255, v1, v2
+// GFX12: v_fmac_f16_e64 v255, v1, v2             ; encoding: [0xff,0x00,0x36,0xd5,0x01,0x05,0x02,0x00]
+
+v_fmac_f16 v5, v1, v255
+// GFX12: v_fmac_f16_e64 v5, v1, v255             ; encoding: [0x05,0x00,0x36,0xd5,0x01,0xff,0x03,0x00]
 
 v_fmac_f16 v5, v255, v2
-// GFX12: v_fmac_f16_e64
+// GFX12: v_fmac_f16_e64 v5, v255, v2             ; encoding: [0x05,0x00,0x36,0xd5,0xff,0x05,0x02,0x00]
 
-v_ldexp_f16 v5, v255, v2
-// GFX12: v_ldexp_f16_e64
+v_ldexp_f16 v255, v1, v2
+// GFX12: v_ldexp_f16_e64 v255, v1, v2            ; encoding: [0xff,0x00,0x3b,0xd5,0x01,0x05,0x02,0x00]
 
-v_max_num_f16 v5, v255, v2
-// GFX12: v_max_num_f16_e64
+v_ldexp_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_ldexp_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x3b,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_min_num_f16 v5, v255, v2
-// GFX12: v_min_num_f16_e64
+v_ldexp_f16 v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_ldexp_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_mul_f16 v5, v255, v2
-// GFX12: v_mul_f16_e64
+v_ldexp_f16 v5, v255, v2
+// GFX12: v_ldexp_f16_e64 v5, v255, v2            ; encoding: [0x05,0x00,0x3b,0xd5,0xff,0x05,0x02,0x00]
 
-v_sub_f16 v5, v255, v2
-// GFX12: v_sub_f16_e64
+v_ldexp_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_ldexp_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3b,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_subrev_f16 v5, v255, v2
-// GFX12: v_subrev_f16_e64
+v_ldexp_f16 v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_ldexp_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_add_f16 v5, v1, v255
-// GFX12: v_add_f16_e64
+v_max_num_f16 v255, v1, v2
+// GFX12: v_max_num_f16_e64 v255, v1, v2          ; encoding: [0xff,0x00,0x31,0xd5,0x01,0x05,0x02,0x00]
 
-v_fmac_f16 v5, v1, v255
-// GFX12: v_fmac_f16_e64
+v_max_num_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_num_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x31,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_max_num_f16 v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_max_num_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x31,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
 v_max_num_f16 v5, v1, v255
-// GFX12: v_max_num_f16_e64
+// GFX12: v_max_num_f16_e64 v5, v1, v255          ; encoding: [0x05,0x00,0x31,0xd5,0x01,0xff,0x03,0x00]
 
-v_min_num_f16 v5, v1, v255
-// GFX12: v_min_num_f16_e64
+v_max_num_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_num_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_mul_f16 v5, v1, v255
-// GFX12: v_mul_f16_e64
+v_max_num_f16 v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_max_num_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_sub_f16 v5, v1, v255
-// GFX12: v_sub_f16_e64
+v_max_num_f16 v5, v255, v2
+// GFX12: v_max_num_f16_e64 v5, v255, v2          ; encoding: [0x05,0x00,0x31,0xd5,0xff,0x05,0x02,0x00]
 
-v_subrev_f16 v5, v1, v255
-// GFX12: v_subrev_f16_e64
+v_max_num_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_num_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_add_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_add_f16_e64
+v_max_num_f16 v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_max_num_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_ldexp_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_ldexp_f16_e64
+v_min_num_f16 v255, v1, v2
+// GFX12: v_min_num_f16_e64 v255, v1, v2          ; encoding: [0xff,0x00,0x30,0xd5,0x01,0x05,0x02,0x00]
 
-v_max_num_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_max_num_f16_e64
+v_min_num_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_num_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x30,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 v_min_num_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_min_num_f16_e64
+// GFX12: v_min_num_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x30,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_mul_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_mul_f16_e64
-
-v_sub_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_sub_f16_e64
+v_min_num_f16 v5, v1, v255
+// GFX12: v_min_num_f16_e64 v5, v1, v255          ; encoding: [0x05,0x00,0x30,0xd5,0x01,0xff,0x03,0x00]
 
-v_subrev_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_subrev_f16_e64
+v_min_num_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_num_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_add_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_add_f16_e64
+v_min_num_f16 v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_min_num_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_ldexp_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_ldexp_f16_e64
+v_min_num_f16 v5, v255, v2
+// GFX12: v_min_num_f16_e64 v5, v255, v2          ; encoding: [0x05,0x00,0x30,0xd5,0xff,0x05,0x02,0x00]
 
-v_max_num_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_max_num_f16_e64
+v_min_num_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_num_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
 v_min_num_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_min_num_f16_e64
+// GFX12: v_min_num_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_mul_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_mul_f16_e64
-
-v_sub_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_sub_f16_e64
+v_mul_f16 v255, v1, v2
+// GFX12: v_mul_f16_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x35,0xd5,0x01,0x05,0x02,0x00]
 
-v_subrev_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_subrev_f16_e64
+v_mul_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x35,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_add_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_add_f16_e64
+v_mul_f16 v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_mul_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_max_num_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_max_num_f16_e64
+v_mul_f16 v5, v1, v255
+// GFX12: v_mul_f16_e64 v5, v1, v255              ; encoding: [0x05,0x00,0x35,0xd5,0x01,0xff,0x03,0x00]
 
-v_min_num_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_min_num_f16_e64
+v_mul_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x35,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
 v_mul_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_mul_f16_e64
-
-v_sub_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_sub_f16_e64
+// GFX12: v_mul_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_subrev_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_subrev_f16_e64
+v_mul_f16 v5, v255, v2
+// GFX12: v_mul_f16_e64 v5, v255, v2              ; encoding: [0x05,0x00,0x35,0xd5,0xff,0x05,0x02,0x00]
 
-v_add_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_add_f16_e64
+v_mul_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x35,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_ldexp_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_ldexp_f16_e64
+v_mul_f16 v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_mul_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_max_num_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_max_num_f16_e64
+v_sub_f16 v255, v1, v2
+// GFX12: v_sub_f16_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x33,0xd5,0x01,0x05,0x02,0x00]
 
-v_min_num_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_min_num_f16_e64
+v_sub_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_mul_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mul_f16_e64
+v_sub_f16 v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_sub_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sub_f16_e64
+v_sub_f16 v5, v1, v255
+// GFX12: v_sub_f16_e64 v5, v1, v255              ; encoding: [0x05,0x00,0x33,0xd5,0x01,0xff,0x03,0x00]
 
-v_subrev_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_subrev_f16_e64
+v_sub_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_add_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_add_f16_e64
+v_sub_f16 v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_ldexp_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_ldexp_f16_e64
+v_sub_f16 v5, v255, v2
+// GFX12: v_sub_f16_e64 v5, v255, v2              ; encoding: [0x05,0x00,0x33,0xd5,0xff,0x05,0x02,0x00]
 
-v_max_num_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_max_num_f16_e64
+v_sub_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_min_num_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_min_num_f16_e64
+v_sub_f16 v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_mul_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mul_f16_e64
+v_subrev_f16 v255, v1, v2
+// GFX12: v_subrev_f16_e64 v255, v1, v2           ; encoding: [0xff,0x00,0x34,0xd5,0x01,0x05,0x02,0x00]
 
-v_sub_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sub_f16_e64
+v_subrev_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_subrev_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x34,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_subrev_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_subrev_f16_e64
+v_subrev_f16 v255, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_subrev_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_add_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_add_f16_e64
+v_subrev_f16 v5, v1, v255
+// GFX12: v_subrev_f16_e64 v5, v1, v255           ; encoding: [0x05,0x00,0x34,0xd5,0x01,0xff,0x03,0x00]
 
-v_max_num_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_max_num_f16_e64
+v_subrev_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_subrev_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x34,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_min_num_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_min_num_f16_e64
+v_subrev_f16 v5, v1, v255 quad_perm:[3,2,1,0]
+// GFX12: v_subrev_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_mul_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_mul_f16_e64
+v_subrev_f16 v5, v255, v2
+// GFX12: v_subrev_f16_e64 v5, v255, v2           ; encoding: [0x05,0x00,0x34,0xd5,0xff,0x05,0x02,0x00]
 
-v_sub_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sub_f16_e64
+v_subrev_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_subrev_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x34,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_subrev_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_subrev_f16_e64
+v_subrev_f16 v5, v255, v2 quad_perm:[3,2,1,0]
+// GFX12: v_subrev_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2.txt
index 26ffd3a4e383..4f638cd8ff54 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2.txt
@@ -1,2334 +1,2324 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,W32,GFX11-FAKE16 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,W32,GFX11-REAL16 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,W64 %s
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -disassemble -show-encoding %s | FileCheck -strict-whitespace -check-prefixes=GFX11,W32,GFX11-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble -show-encoding %s | FileCheck -strict-whitespace -check-prefixes=GFX11,W32,GFX11-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding %s | FileCheck -strict-whitespace -check-prefixes=GFX11,W64,GFX11-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding %s | FileCheck -strict-whitespace -check-prefixes=GFX11,W64,GFX11-FAKE16 %s
 
+0x01,0x05,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, v1, v2, vcc_lo ; encoding: [0x01,0x05,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, v1, v2, vcc ; encoding: [0x01,0x05,0x0a,0x40]
-0x01,0x05,0x0a,0x40
 
+0xff,0x05,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, v255, v2, vcc_lo ; encoding: [0xff,0x05,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, v255, v2, vcc ; encoding: [0xff,0x05,0x0a,0x40]
-0xff,0x05,0x0a,0x40
 
+0x01,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, s1, v2, vcc_lo ; encoding: [0x01,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, s1, v2, vcc ; encoding: [0x01,0x04,0x0a,0x40]
-0x01,0x04,0x0a,0x40
 
+0x69,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, s105, v2, vcc_lo ; encoding: [0x69,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, s105, v2, vcc ; encoding: [0x69,0x04,0x0a,0x40]
-0x69,0x04,0x0a,0x40
 
+0x6a,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, vcc_lo, v2, vcc_lo ; encoding: [0x6a,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, vcc_lo, v2, vcc ; encoding: [0x6a,0x04,0x0a,0x40]
-0x6a,0x04,0x0a,0x40
 
+0x6b,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, vcc_hi, v2, vcc_lo ; encoding: [0x6b,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, vcc_hi, v2, vcc ; encoding: [0x6b,0x04,0x0a,0x40]
-0x6b,0x04,0x0a,0x40
 
+0x7b,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, ttmp15, v2, vcc_lo ; encoding: [0x7b,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, ttmp15, v2, vcc ; encoding: [0x7b,0x04,0x0a,0x40]
-0x7b,0x04,0x0a,0x40
 
+0x7d,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, m0, v2, vcc_lo ; encoding: [0x7d,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, m0, v2, vcc ; encoding: [0x7d,0x04,0x0a,0x40]
-0x7d,0x04,0x0a,0x40
 
+0x7e,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, exec_lo, v2, vcc_lo ; encoding: [0x7e,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, exec_lo, v2, vcc ; encoding: [0x7e,0x04,0x0a,0x40]
-0x7e,0x04,0x0a,0x40
 
+0x7f,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, exec_hi, v2, vcc_lo ; encoding: [0x7f,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, exec_hi, v2, vcc ; encoding: [0x7f,0x04,0x0a,0x40]
-0x7f,0x04,0x0a,0x40
 
+0x7c,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, null, v2, vcc_lo ; encoding: [0x7c,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, null, v2, vcc ; encoding: [0x7c,0x04,0x0a,0x40]
-0x7c,0x04,0x0a,0x40
 
+0xc1,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, -1, v2, vcc_lo ; encoding: [0xc1,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, -1, v2, vcc ; encoding: [0xc1,0x04,0x0a,0x40]
-0xc1,0x04,0x0a,0x40
 
+0xf0,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, 0.5, v2, vcc_lo ; encoding: [0xf0,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, 0.5, v2, vcc ; encoding: [0xf0,0x04,0x0a,0x40]
-0xf0,0x04,0x0a,0x40
 
+0xfd,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, src_scc, v2, vcc_lo ; encoding: [0xfd,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, src_scc, v2, vcc ; encoding: [0xfd,0x04,0x0a,0x40]
-0xfd,0x04,0x0a,0x40
 
+0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf
 # W32: v_add_co_ci_u32_e32 v255, vcc_lo, 0xaf123456, v255, vcc_lo ; encoding: [0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf]
 # W64: v_add_co_ci_u32_e32 v255, vcc, 0xaf123456, v255, vcc ; encoding: [0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf]
-0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf
 
+0x01,0x05,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, v1.l, v2.l          ; encoding: [0x01,0x05,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x64]
-0x01,0x05,0x0a,0x64
 
+0x81,0x05,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, v1.h, v2.l          ; encoding: [0x81,0x05,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x64]
-0x81,0x05,0x0a,0x64
 
+0x7f,0x05,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, v127.l, v2.l        ; encoding: [0x7f,0x05,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x64]
-0x7f,0x05,0x0a,0x64
 
+0xff,0x05,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, v127.h, v2.l        ; encoding: [0xff,0x05,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x64]
-0xff,0x05,0x0a,0x64
 
+0x01,0x04,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, s1, v2.l            ; encoding: [0x01,0x04,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x64]
-0x01,0x04,0x0a,0x64
 
+0x69,0x04,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, s105, v2.l          ; encoding: [0x69,0x04,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x64]
-0x69,0x04,0x0a,0x64
 
+0x6a,0x04,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, vcc_lo, v2.l        ; encoding: [0x6a,0x04,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x64]
-0x6a,0x04,0x0a,0x64
 
+0x6b,0x04,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, vcc_hi, v2.l        ; encoding: [0x6b,0x04,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x64]
-0x6b,0x04,0x0a,0x64
 
+0x7b,0x04,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, ttmp15, v2.l        ; encoding: [0x7b,0x04,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x64]
-0x7b,0x04,0x0a,0x64
 
+0x7d,0x04,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, m0, v2.l            ; encoding: [0x7d,0x04,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x64]
-0x7d,0x04,0x0a,0x64
 
+0x7e,0x04,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, exec_lo, v2.l       ; encoding: [0x7e,0x04,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x64]
-0x7e,0x04,0x0a,0x64
 
+0x7f,0x04,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, exec_hi, v2.l       ; encoding: [0x7f,0x04,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x64]
-0x7f,0x04,0x0a,0x64
 
+0x7c,0x04,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, null, v2.l          ; encoding: [0x7c,0x04,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x64]
-0x7c,0x04,0x0a,0x64
 
+0xc1,0x04,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, -1, v2.l            ; encoding: [0xc1,0x04,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x64]
-0xc1,0x04,0x0a,0x64
 
+0xf0,0x04,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, 0.5, v2.l           ; encoding: [0xf0,0x04,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x64]
-0xf0,0x04,0x0a,0x64
 
+0xfd,0x04,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, src_scc, v2.l       ; encoding: [0xfd,0x04,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x64]
-0xfd,0x04,0x0a,0x64
 
-# GFX11-REAL16: v_add_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x65]
-# COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xfd,0x04,0x0b,0x65
 0xfd,0x04,0x0b,0x65
+# GFX11-REAL16: v_add_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x65]
 
+0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00
 # GFX11-REAL16: v_add_f16_e32 v127.l, 0xfe0b, v127.l    ; encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00]
 # GFX11-FAKE16: v_add_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00]
-0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00
 
-# GFX11-REAL16: v_add_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x65,0x0b,0xfe,0x00,0x00]
-# COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xff,0xfe,0xff,0x65,0x0b,0xfe,0x00,0x00
 0xff,0xfe,0xff,0x65,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_add_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x65,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_add_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x06]
 0x01,0x05,0x0a,0x06
+# GFX11: v_add_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x06]
 
-# GFX11: v_add_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x06]
 0xff,0x05,0x0a,0x06
+# GFX11: v_add_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x06]
 
-# GFX11: v_add_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x06]
 0x01,0x04,0x0a,0x06
+# GFX11: v_add_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x06]
 
-# GFX11: v_add_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x06]
 0x69,0x04,0x0a,0x06
+# GFX11: v_add_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x06]
 
-# GFX11: v_add_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x06]
 0x6a,0x04,0x0a,0x06
+# GFX11: v_add_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x06]
 
-# GFX11: v_add_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x06]
 0x6b,0x04,0x0a,0x06
+# GFX11: v_add_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x06]
 
-# GFX11: v_add_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x06]
 0x7b,0x04,0x0a,0x06
+# GFX11: v_add_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x06]
 
-# GFX11: v_add_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x06]
 0x7d,0x04,0x0a,0x06
+# GFX11: v_add_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x06]
 
-# GFX11: v_add_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x06]
 0x7e,0x04,0x0a,0x06
+# GFX11: v_add_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x06]
 
-# GFX11: v_add_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x06]
 0x7f,0x04,0x0a,0x06
+# GFX11: v_add_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x06]
 
-# GFX11: v_add_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x06]
 0x7c,0x04,0x0a,0x06
+# GFX11: v_add_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x06]
 
-# GFX11: v_add_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x06]
 0xc1,0x04,0x0a,0x06
+# GFX11: v_add_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x06]
 
-# GFX11: v_add_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x06]
 0xf0,0x04,0x0a,0x06
+# GFX11: v_add_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x06]
 
-# GFX11: v_add_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x06]
 0xfd,0x04,0x0a,0x06
+# GFX11: v_add_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x06]
 
-# GFX11: v_add_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x07,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_add_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_add_nc_u32_e32 v5, v1, v2             ; encoding: [0x01,0x05,0x0a,0x4a]
 0x01,0x05,0x0a,0x4a
+# GFX11: v_add_nc_u32_e32 v5, v1, v2             ; encoding: [0x01,0x05,0x0a,0x4a]
 
-# GFX11: v_add_nc_u32_e32 v5, v255, v2           ; encoding: [0xff,0x05,0x0a,0x4a]
 0xff,0x05,0x0a,0x4a
+# GFX11: v_add_nc_u32_e32 v5, v255, v2           ; encoding: [0xff,0x05,0x0a,0x4a]
 
-# GFX11: v_add_nc_u32_e32 v5, s1, v2             ; encoding: [0x01,0x04,0x0a,0x4a]
 0x01,0x04,0x0a,0x4a
+# GFX11: v_add_nc_u32_e32 v5, s1, v2             ; encoding: [0x01,0x04,0x0a,0x4a]
 
-# GFX11: v_add_nc_u32_e32 v5, s105, v2           ; encoding: [0x69,0x04,0x0a,0x4a]
 0x69,0x04,0x0a,0x4a
+# GFX11: v_add_nc_u32_e32 v5, s105, v2           ; encoding: [0x69,0x04,0x0a,0x4a]
 
-# GFX11: v_add_nc_u32_e32 v5, vcc_lo, v2         ; encoding: [0x6a,0x04,0x0a,0x4a]
 0x6a,0x04,0x0a,0x4a
+# GFX11: v_add_nc_u32_e32 v5, vcc_lo, v2         ; encoding: [0x6a,0x04,0x0a,0x4a]
 
-# GFX11: v_add_nc_u32_e32 v5, vcc_hi, v2         ; encoding: [0x6b,0x04,0x0a,0x4a]
 0x6b,0x04,0x0a,0x4a
+# GFX11: v_add_nc_u32_e32 v5, vcc_hi, v2         ; encoding: [0x6b,0x04,0x0a,0x4a]
 
-# GFX11: v_add_nc_u32_e32 v5, ttmp15, v2         ; encoding: [0x7b,0x04,0x0a,0x4a]
 0x7b,0x04,0x0a,0x4a
+# GFX11: v_add_nc_u32_e32 v5, ttmp15, v2         ; encoding: [0x7b,0x04,0x0a,0x4a]
 
-# GFX11: v_add_nc_u32_e32 v5, m0, v2             ; encoding: [0x7d,0x04,0x0a,0x4a]
 0x7d,0x04,0x0a,0x4a
+# GFX11: v_add_nc_u32_e32 v5, m0, v2             ; encoding: [0x7d,0x04,0x0a,0x4a]
 
-# GFX11: v_add_nc_u32_e32 v5, exec_lo, v2        ; encoding: [0x7e,0x04,0x0a,0x4a]
 0x7e,0x04,0x0a,0x4a
+# GFX11: v_add_nc_u32_e32 v5, exec_lo, v2        ; encoding: [0x7e,0x04,0x0a,0x4a]
 
-# GFX11: v_add_nc_u32_e32 v5, exec_hi, v2        ; encoding: [0x7f,0x04,0x0a,0x4a]
 0x7f,0x04,0x0a,0x4a
+# GFX11: v_add_nc_u32_e32 v5, exec_hi, v2        ; encoding: [0x7f,0x04,0x0a,0x4a]
 
-# GFX11: v_add_nc_u32_e32 v5, null, v2           ; encoding: [0x7c,0x04,0x0a,0x4a]
 0x7c,0x04,0x0a,0x4a
+# GFX11: v_add_nc_u32_e32 v5, null, v2           ; encoding: [0x7c,0x04,0x0a,0x4a]
 
-# GFX11: v_add_nc_u32_e32 v5, -1, v2             ; encoding: [0xc1,0x04,0x0a,0x4a]
 0xc1,0x04,0x0a,0x4a
+# GFX11: v_add_nc_u32_e32 v5, -1, v2             ; encoding: [0xc1,0x04,0x0a,0x4a]
 
-# GFX11: v_add_nc_u32_e32 v5, 0.5, v2            ; encoding: [0xf0,0x04,0x0a,0x4a]
 0xf0,0x04,0x0a,0x4a
+# GFX11: v_add_nc_u32_e32 v5, 0.5, v2            ; encoding: [0xf0,0x04,0x0a,0x4a]
 
-# GFX11: v_add_nc_u32_e32 v5, src_scc, v2        ; encoding: [0xfd,0x04,0x0a,0x4a]
 0xfd,0x04,0x0a,0x4a
+# GFX11: v_add_nc_u32_e32 v5, src_scc, v2        ; encoding: [0xfd,0x04,0x0a,0x4a]
 
-# GFX11: v_add_nc_u32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x4b,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x4b,0x56,0x34,0x12,0xaf
+# GFX11: v_add_nc_u32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x4b,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_and_b32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x36]
 0x01,0x05,0x0a,0x36
+# GFX11: v_and_b32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x36]
 
-# GFX11: v_and_b32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x36]
 0xff,0x05,0x0a,0x36
+# GFX11: v_and_b32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x36]
 
-# GFX11: v_and_b32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x36]
 0x01,0x04,0x0a,0x36
+# GFX11: v_and_b32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x36]
 
-# GFX11: v_and_b32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x36]
 0x69,0x04,0x0a,0x36
+# GFX11: v_and_b32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x36]
 
-# GFX11: v_and_b32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x36]
 0x6a,0x04,0x0a,0x36
+# GFX11: v_and_b32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x36]
 
-# GFX11: v_and_b32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x36]
 0x6b,0x04,0x0a,0x36
+# GFX11: v_and_b32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x36]
 
-# GFX11: v_and_b32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x36]
 0x7b,0x04,0x0a,0x36
+# GFX11: v_and_b32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x36]
 
-# GFX11: v_and_b32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x36]
 0x7d,0x04,0x0a,0x36
+# GFX11: v_and_b32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x36]
 
-# GFX11: v_and_b32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x36]
 0x7e,0x04,0x0a,0x36
+# GFX11: v_and_b32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x36]
 
-# GFX11: v_and_b32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x36]
 0x7f,0x04,0x0a,0x36
+# GFX11: v_and_b32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x36]
 
-# GFX11: v_and_b32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x36]
 0x7c,0x04,0x0a,0x36
+# GFX11: v_and_b32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x36]
 
-# GFX11: v_and_b32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x36]
 0xc1,0x04,0x0a,0x36
+# GFX11: v_and_b32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x36]
 
-# GFX11: v_and_b32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x36]
 0xf0,0x04,0x0a,0x36
+# GFX11: v_and_b32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x36]
 
-# GFX11: v_and_b32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x36]
 0xfd,0x04,0x0a,0x36
+# GFX11: v_and_b32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x36]
 
-# GFX11: v_and_b32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x37,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x37,0x56,0x34,0x12,0xaf
+# GFX11: v_and_b32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x37,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_ashrrev_i32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x34]
 0x01,0x05,0x0a,0x34
+# GFX11: v_ashrrev_i32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x34]
 
-# GFX11: v_ashrrev_i32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x34]
 0xff,0x05,0x0a,0x34
+# GFX11: v_ashrrev_i32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x34]
 
-# GFX11: v_ashrrev_i32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x34]
 0x01,0x04,0x0a,0x34
+# GFX11: v_ashrrev_i32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x34]
 
-# GFX11: v_ashrrev_i32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x34]
 0x69,0x04,0x0a,0x34
+# GFX11: v_ashrrev_i32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x34]
 
-# GFX11: v_ashrrev_i32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x34]
 0x6a,0x04,0x0a,0x34
+# GFX11: v_ashrrev_i32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x34]
 
-# GFX11: v_ashrrev_i32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x34]
 0x6b,0x04,0x0a,0x34
+# GFX11: v_ashrrev_i32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x34]
 
-# GFX11: v_ashrrev_i32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x34]
 0x7b,0x04,0x0a,0x34
+# GFX11: v_ashrrev_i32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x34]
 
-# GFX11: v_ashrrev_i32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x34]
 0x7d,0x04,0x0a,0x34
+# GFX11: v_ashrrev_i32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x34]
 
-# GFX11: v_ashrrev_i32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x34]
 0x7e,0x04,0x0a,0x34
+# GFX11: v_ashrrev_i32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x34]
 
-# GFX11: v_ashrrev_i32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x34]
 0x7f,0x04,0x0a,0x34
+# GFX11: v_ashrrev_i32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x34]
 
-# GFX11: v_ashrrev_i32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x34]
 0x7c,0x04,0x0a,0x34
+# GFX11: v_ashrrev_i32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x34]
 
-# GFX11: v_ashrrev_i32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x34]
 0xc1,0x04,0x0a,0x34
+# GFX11: v_ashrrev_i32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x34]
 
-# GFX11: v_ashrrev_i32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x34]
 0xf0,0x04,0x0a,0x34
+# GFX11: v_ashrrev_i32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x34]
 
-# GFX11: v_ashrrev_i32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x34]
 0xfd,0x04,0x0a,0x34
+# GFX11: v_ashrrev_i32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x34]
 
-# GFX11: v_ashrrev_i32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x35,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x35,0x56,0x34,0x12,0xaf
+# GFX11: v_ashrrev_i32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x35,0x56,0x34,0x12,0xaf]
 
+0x01,0x05,0x0a,0x02
 # W32: v_cndmask_b32_e32 v5, v1, v2, vcc_lo    ; encoding: [0x01,0x05,0x0a,0x02]
 # W64: v_cndmask_b32_e32 v5, v1, v2, vcc       ; encoding: [0x01,0x05,0x0a,0x02]
-0x01,0x05,0x0a,0x02
 
+0xff,0x05,0x0a,0x02
 # W32: v_cndmask_b32_e32 v5, v255, v2, vcc_lo  ; encoding: [0xff,0x05,0x0a,0x02]
 # W64: v_cndmask_b32_e32 v5, v255, v2, vcc     ; encoding: [0xff,0x05,0x0a,0x02]
-0xff,0x05,0x0a,0x02
 
+0x01,0x04,0x0a,0x02
 # W32: v_cndmask_b32_e32 v5, s1, v2, vcc_lo    ; encoding: [0x01,0x04,0x0a,0x02]
 # W64: v_cndmask_b32_e32 v5, s1, v2, vcc       ; encoding: [0x01,0x04,0x0a,0x02]
-0x01,0x04,0x0a,0x02
 
+0x69,0x04,0x0a,0x02
 # W32: v_cndmask_b32_e32 v5, s105, v2, vcc_lo  ; encoding: [0x69,0x04,0x0a,0x02]
 # W64: v_cndmask_b32_e32 v5, s105, v2, vcc     ; encoding: [0x69,0x04,0x0a,0x02]
-0x69,0x04,0x0a,0x02
 
+0x6a,0x04,0x0a,0x02
 # W32: v_cndmask_b32_e32 v5, vcc_lo, v2, vcc_lo ; encoding: [0x6a,0x04,0x0a,0x02]
 # W64: v_cndmask_b32_e32 v5, vcc_lo, v2, vcc   ; encoding: [0x6a,0x04,0x0a,0x02]
-0x6a,0x04,0x0a,0x02
 
+0x6b,0x04,0x0a,0x02
 # W32: v_cndmask_b32_e32 v5, vcc_hi, v2, vcc_lo ; encoding: [0x6b,0x04,0x0a,0x02]
 # W64: v_cndmask_b32_e32 v5, vcc_hi, v2, vcc   ; encoding: [0x6b,0x04,0x0a,0x02]
-0x6b,0x04,0x0a,0x02
 
+0x7b,0x04,0x0a,0x02
 # W32: v_cndmask_b32_e32 v5, ttmp15, v2, vcc_lo ; encoding: [0x7b,0x04,0x0a,0x02]
 # W64: v_cndmask_b32_e32 v5, ttmp15, v2, vcc   ; encoding: [0x7b,0x04,0x0a,0x02]
-0x7b,0x04,0x0a,0x02
 
+0x7d,0x04,0x0a,0x02
 # W32: v_cndmask_b32_e32 v5, m0, v2, vcc_lo    ; encoding: [0x7d,0x04,0x0a,0x02]
 # W64: v_cndmask_b32_e32 v5, m0, v2, vcc       ; encoding: [0x7d,0x04,0x0a,0x02]
-0x7d,0x04,0x0a,0x02
 
+0x7e,0x04,0x0a,0x02
 # W32: v_cndmask_b32_e32 v5, exec_lo, v2, vcc_lo ; encoding: [0x7e,0x04,0x0a,0x02]
 # W64: v_cndmask_b32_e32 v5, exec_lo, v2, vcc  ; encoding: [0x7e,0x04,0x0a,0x02]
-0x7e,0x04,0x0a,0x02
 
+0x7f,0x04,0x0a,0x02
 # W32: v_cndmask_b32_e32 v5, exec_hi, v2, vcc_lo ; encoding: [0x7f,0x04,0x0a,0x02]
 # W64: v_cndmask_b32_e32 v5, exec_hi, v2, vcc  ; encoding: [0x7f,0x04,0x0a,0x02]
-0x7f,0x04,0x0a,0x02
 
+0x7c,0x04,0x0a,0x02
 # W32: v_cndmask_b32_e32 v5, null, v2, vcc_lo  ; encoding: [0x7c,0x04,0x0a,0x02]
 # W64: v_cndmask_b32_e32 v5, null, v2, vcc     ; encoding: [0x7c,0x04,0x0a,0x02]
-0x7c,0x04,0x0a,0x02
 
+0xc1,0x04,0x0a,0x02
 # W32: v_cndmask_b32_e32 v5, -1, v2, vcc_lo    ; encoding: [0xc1,0x04,0x0a,0x02]
 # W64: v_cndmask_b32_e32 v5, -1, v2, vcc       ; encoding: [0xc1,0x04,0x0a,0x02]
-0xc1,0x04,0x0a,0x02
 
+0xf0,0x04,0x0a,0x02
 # W32: v_cndmask_b32_e32 v5, 0.5, v2, vcc_lo   ; encoding: [0xf0,0x04,0x0a,0x02]
 # W64: v_cndmask_b32_e32 v5, 0.5, v2, vcc      ; encoding: [0xf0,0x04,0x0a,0x02]
-0xf0,0x04,0x0a,0x02
 
+0xfd,0x04,0x0a,0x02
 # W32: v_cndmask_b32_e32 v5, src_scc, v2, vcc_lo ; encoding: [0xfd,0x04,0x0a,0x02]
 # W64: v_cndmask_b32_e32 v5, src_scc, v2, vcc  ; encoding: [0xfd,0x04,0x0a,0x02]
-0xfd,0x04,0x0a,0x02
 
+0xff,0xfe,0xff,0x03,0x56,0x34,0x12,0xaf
 # W32: v_cndmask_b32_e32 v255, 0xaf123456, v255, vcc_lo ; encoding: [0xff,0xfe,0xff,0x03,0x56,0x34,0x12,0xaf]
 # W64: v_cndmask_b32_e32 v255, 0xaf123456, v255, vcc ; encoding: [0xff,0xfe,0xff,0x03,0x56,0x34,0x12,0xaf]
-0xff,0xfe,0xff,0x03,0x56,0x34,0x12,0xaf
 
-# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, v1, v2     ; encoding: [0x01,0x05,0x0a,0x5e]
 0x01,0x05,0x0a,0x5e
+# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, v1, v2     ; encoding: [0x01,0x05,0x0a,0x5e]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, v255, v2   ; encoding: [0xff,0x05,0x0a,0x5e]
 0xff,0x05,0x0a,0x5e
+# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, v255, v2   ; encoding: [0xff,0x05,0x0a,0x5e]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, s1, v2     ; encoding: [0x01,0x04,0x0a,0x5e]
 0x01,0x04,0x0a,0x5e
+# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, s1, v2     ; encoding: [0x01,0x04,0x0a,0x5e]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, s105, v2   ; encoding: [0x69,0x04,0x0a,0x5e]
 0x69,0x04,0x0a,0x5e
+# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, s105, v2   ; encoding: [0x69,0x04,0x0a,0x5e]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x5e]
 0x6a,0x04,0x0a,0x5e
+# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x5e]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x5e]
 0x6b,0x04,0x0a,0x5e
+# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x5e]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x5e]
 0x7b,0x04,0x0a,0x5e
+# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x5e]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, m0, v2     ; encoding: [0x7d,0x04,0x0a,0x5e]
 0x7d,0x04,0x0a,0x5e
+# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, m0, v2     ; encoding: [0x7d,0x04,0x0a,0x5e]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x5e]
 0x7e,0x04,0x0a,0x5e
+# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x5e]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x5e]
 0x7f,0x04,0x0a,0x5e
+# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x5e]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, null, v2   ; encoding: [0x7c,0x04,0x0a,0x5e]
 0x7c,0x04,0x0a,0x5e
+# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, null, v2   ; encoding: [0x7c,0x04,0x0a,0x5e]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, -1, v2     ; encoding: [0xc1,0x04,0x0a,0x5e]
 0xc1,0x04,0x0a,0x5e
+# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, -1, v2     ; encoding: [0xc1,0x04,0x0a,0x5e]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, 0.5, v2    ; encoding: [0xf0,0x04,0x0a,0x5e]
 0xf0,0x04,0x0a,0x5e
+# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, 0.5, v2    ; encoding: [0xf0,0x04,0x0a,0x5e]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x5e]
 0xfd,0x04,0x0a,0x5e
+# GFX11: v_cvt_pk_rtz_f16_f32_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x5e]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x5f,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x5f,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_rtz_f16_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x5f,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_dot2acc_f32_f16 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x04]
 0x01,0x05,0x0a,0x04
+# GFX11: v_dot2acc_f32_f16 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x04]
 
-# GFX11: v_dot2acc_f32_f16 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x04]
 0xff,0x05,0x0a,0x04
+# GFX11: v_dot2acc_f32_f16 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x04]
 
-# GFX11: v_dot2acc_f32_f16 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x04]
 0x01,0x04,0x0a,0x04
+# GFX11: v_dot2acc_f32_f16 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x04]
 
-# GFX11: v_dot2acc_f32_f16 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x04]
 0x69,0x04,0x0a,0x04
+# GFX11: v_dot2acc_f32_f16 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x04]
 
-# GFX11: v_dot2acc_f32_f16 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x04]
 0x6a,0x04,0x0a,0x04
+# GFX11: v_dot2acc_f32_f16 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x04]
 
-# GFX11: v_dot2acc_f32_f16 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x04]
 0x6b,0x04,0x0a,0x04
+# GFX11: v_dot2acc_f32_f16 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x04]
 
-# GFX11: v_dot2acc_f32_f16 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x04]
 0x7b,0x04,0x0a,0x04
+# GFX11: v_dot2acc_f32_f16 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x04]
 
-# GFX11: v_dot2acc_f32_f16 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x04]
 0x7d,0x04,0x0a,0x04
+# GFX11: v_dot2acc_f32_f16 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x04]
 
-# GFX11: v_dot2acc_f32_f16 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x04]
 0x7e,0x04,0x0a,0x04
+# GFX11: v_dot2acc_f32_f16 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x04]
 
-# GFX11: v_dot2acc_f32_f16 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x04]
 0x7f,0x04,0x0a,0x04
+# GFX11: v_dot2acc_f32_f16 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x04]
 
-# GFX11: v_dot2acc_f32_f16 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x04]
 0x7c,0x04,0x0a,0x04
+# GFX11: v_dot2acc_f32_f16 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x04]
 
-# GFX11: v_dot2acc_f32_f16 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x04]
 0xc1,0x04,0x0a,0x04
+# GFX11: v_dot2acc_f32_f16 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x04]
 
-# GFX11: v_dot2acc_f32_f16 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x04]
 0xf0,0x04,0x0a,0x04
+# GFX11: v_dot2acc_f32_f16 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x04]
 
-# GFX11: v_dot2acc_f32_f16 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x04]
 0xfd,0x04,0x0a,0x04
+# GFX11: v_dot2acc_f32_f16 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x04]
 
-# GFX11: v_dot2acc_f32_f16 v255, 0xfe0b, v255    ; encoding: [0xff,0xfe,0xff,0x05,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xff,0x05,0x0b,0xfe,0x00,0x00
+# GFX11: v_dot2acc_f32_f16 v255, 0xfe0b, v255    ; encoding: [0xff,0xfe,0xff,0x05,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f16 v5, v1, v2, 0xfe0b          ; encoding: [0x01,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x01,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmaak_f16 v5, v1, v2, 0xfe0b          ; encoding: [0x01,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f16 v5, v127, v2, 0xfe0b        ; encoding: [0x7f,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x7f,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmaak_f16 v5, v127, v2, 0xfe0b        ; encoding: [0x7f,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f16 v5, s1, v2, 0xfe0b          ; encoding: [0x01,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x01,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmaak_f16 v5, s1, v2, 0xfe0b          ; encoding: [0x01,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f16 v5, s105, v2, 0xfe0b        ; encoding: [0x69,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x69,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmaak_f16 v5, s105, v2, 0xfe0b        ; encoding: [0x69,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f16 v5, vcc_lo, v2, 0xfe0b      ; encoding: [0x6a,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x6a,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmaak_f16 v5, vcc_lo, v2, 0xfe0b      ; encoding: [0x6a,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f16 v5, vcc_hi, v2, 0xfe0b      ; encoding: [0x6b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x6b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmaak_f16 v5, vcc_hi, v2, 0xfe0b      ; encoding: [0x6b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f16 v5, ttmp15, v2, 0xfe0b      ; encoding: [0x7b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x7b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmaak_f16 v5, ttmp15, v2, 0xfe0b      ; encoding: [0x7b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f16 v5, m0, v2, 0xfe0b          ; encoding: [0x7d,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x7d,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmaak_f16 v5, m0, v2, 0xfe0b          ; encoding: [0x7d,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f16 v5, exec_lo, v2, 0xfe0b     ; encoding: [0x7e,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x7e,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmaak_f16 v5, exec_lo, v2, 0xfe0b     ; encoding: [0x7e,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f16 v5, exec_hi, v2, 0xfe0b     ; encoding: [0x7f,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x7f,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmaak_f16 v5, exec_hi, v2, 0xfe0b     ; encoding: [0x7f,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f16 v5, null, v2, 0xfe0b        ; encoding: [0x7c,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x7c,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmaak_f16 v5, null, v2, 0xfe0b        ; encoding: [0x7c,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f16 v5, -1, v2, 0xfe0b          ; encoding: [0xc1,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0xc1,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmaak_f16 v5, -1, v2, 0xfe0b          ; encoding: [0xc1,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f16 v5, 0.5, v2, 0xfe0b         ; encoding: [0xf0,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0xf0,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmaak_f16 v5, 0.5, v2, 0xfe0b         ; encoding: [0xf0,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f16 v5, src_scc, v2, 0xfe0b     ; encoding: [0xfd,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0xfd,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmaak_f16 v5, src_scc, v2, 0xfe0b     ; encoding: [0xfd,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f16 v127, 0xfe0b, v127, 0xfe0b  ; encoding: [0xff,0xfe,0xfe,0x70,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xfe,0x70,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmaak_f16 v127, 0xfe0b, v127, 0xfe0b  ; encoding: [0xff,0xfe,0xfe,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmaak_f32 v5, v1, v2, 0xaf123456      ; encoding: [0x01,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x01,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX11: v_fmaak_f32 v5, v1, v2, 0xaf123456      ; encoding: [0x01,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmaak_f32 v5, v255, v2, 0xaf123456    ; encoding: [0xff,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0xff,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX11: v_fmaak_f32 v5, v255, v2, 0xaf123456    ; encoding: [0xff,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmaak_f32 v5, s1, v2, 0xaf123456      ; encoding: [0x01,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x01,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX11: v_fmaak_f32 v5, s1, v2, 0xaf123456      ; encoding: [0x01,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmaak_f32 v5, s105, v2, 0xaf123456    ; encoding: [0x69,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x69,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX11: v_fmaak_f32 v5, s105, v2, 0xaf123456    ; encoding: [0x69,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmaak_f32 v5, vcc_lo, v2, 0xaf123456  ; encoding: [0x6a,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x6a,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX11: v_fmaak_f32 v5, vcc_lo, v2, 0xaf123456  ; encoding: [0x6a,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmaak_f32 v5, vcc_hi, v2, 0xaf123456  ; encoding: [0x6b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x6b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX11: v_fmaak_f32 v5, vcc_hi, v2, 0xaf123456  ; encoding: [0x6b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmaak_f32 v5, ttmp15, v2, 0xaf123456  ; encoding: [0x7b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x7b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX11: v_fmaak_f32 v5, ttmp15, v2, 0xaf123456  ; encoding: [0x7b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmaak_f32 v5, m0, v2, 0xaf123456      ; encoding: [0x7d,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x7d,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX11: v_fmaak_f32 v5, m0, v2, 0xaf123456      ; encoding: [0x7d,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmaak_f32 v5, exec_lo, v2, 0xaf123456 ; encoding: [0x7e,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x7e,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX11: v_fmaak_f32 v5, exec_lo, v2, 0xaf123456 ; encoding: [0x7e,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmaak_f32 v5, exec_hi, v2, 0xaf123456 ; encoding: [0x7f,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x7f,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX11: v_fmaak_f32 v5, exec_hi, v2, 0xaf123456 ; encoding: [0x7f,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmaak_f32 v5, null, v2, 0xaf123456    ; encoding: [0x7c,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x7c,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX11: v_fmaak_f32 v5, null, v2, 0xaf123456    ; encoding: [0x7c,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmaak_f32 v5, -1, v2, 0xaf123456      ; encoding: [0xc1,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0xc1,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX11: v_fmaak_f32 v5, -1, v2, 0xaf123456      ; encoding: [0xc1,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmaak_f32 v5, 0.5, v2, 0xaf123456     ; encoding: [0xf0,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0xf0,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX11: v_fmaak_f32 v5, 0.5, v2, 0xaf123456     ; encoding: [0xf0,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmaak_f32 v5, src_scc, v2, 0xaf123456 ; encoding: [0xfd,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0xfd,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX11: v_fmaak_f32 v5, src_scc, v2, 0xaf123456 ; encoding: [0xfd,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmaak_f32 v255, 0xaf123456, v255, 0xaf123456 ; encoding: [0xff,0xfe,0xff,0x5b,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x5b,0x56,0x34,0x12,0xaf
+# GFX11: v_fmaak_f32 v255, 0xaf123456, v255, 0xaf123456 ; encoding: [0xff,0xfe,0xff,0x5b,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmac_dx9_zero_f32_e32 v5, v1, v2      ; encoding: [0x01,0x05,0x0a,0x0c]
 0x01,0x05,0x0a,0x0c
+# GFX11: v_fmac_dx9_zero_f32_e32 v5, v1, v2      ; encoding: [0x01,0x05,0x0a,0x0c]
 
-# GFX11: v_fmac_dx9_zero_f32_e32 v5, v255, v2    ; encoding: [0xff,0x05,0x0a,0x0c]
 0xff,0x05,0x0a,0x0c
+# GFX11: v_fmac_dx9_zero_f32_e32 v5, v255, v2    ; encoding: [0xff,0x05,0x0a,0x0c]
 
-# GFX11: v_fmac_dx9_zero_f32_e32 v5, s1, v2      ; encoding: [0x01,0x04,0x0a,0x0c]
 0x01,0x04,0x0a,0x0c
+# GFX11: v_fmac_dx9_zero_f32_e32 v5, s1, v2      ; encoding: [0x01,0x04,0x0a,0x0c]
 
-# GFX11: v_fmac_dx9_zero_f32_e32 v5, s105, v2    ; encoding: [0x69,0x04,0x0a,0x0c]
 0x69,0x04,0x0a,0x0c
+# GFX11: v_fmac_dx9_zero_f32_e32 v5, s105, v2    ; encoding: [0x69,0x04,0x0a,0x0c]
 
-# GFX11: v_fmac_dx9_zero_f32_e32 v5, vcc_lo, v2  ; encoding: [0x6a,0x04,0x0a,0x0c]
 0x6a,0x04,0x0a,0x0c
+# GFX11: v_fmac_dx9_zero_f32_e32 v5, vcc_lo, v2  ; encoding: [0x6a,0x04,0x0a,0x0c]
 
-# GFX11: v_fmac_dx9_zero_f32_e32 v5, vcc_hi, v2  ; encoding: [0x6b,0x04,0x0a,0x0c]
 0x6b,0x04,0x0a,0x0c
+# GFX11: v_fmac_dx9_zero_f32_e32 v5, vcc_hi, v2  ; encoding: [0x6b,0x04,0x0a,0x0c]
 
-# GFX11: v_fmac_dx9_zero_f32_e32 v5, ttmp15, v2  ; encoding: [0x7b,0x04,0x0a,0x0c]
 0x7b,0x04,0x0a,0x0c
+# GFX11: v_fmac_dx9_zero_f32_e32 v5, ttmp15, v2  ; encoding: [0x7b,0x04,0x0a,0x0c]
 
-# GFX11: v_fmac_dx9_zero_f32_e32 v5, m0, v2      ; encoding: [0x7d,0x04,0x0a,0x0c]
 0x7d,0x04,0x0a,0x0c
+# GFX11: v_fmac_dx9_zero_f32_e32 v5, m0, v2      ; encoding: [0x7d,0x04,0x0a,0x0c]
 
-# GFX11: v_fmac_dx9_zero_f32_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x0c]
 0x7e,0x04,0x0a,0x0c
+# GFX11: v_fmac_dx9_zero_f32_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x0c]
 
-# GFX11: v_fmac_dx9_zero_f32_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x0c]
 0x7f,0x04,0x0a,0x0c
+# GFX11: v_fmac_dx9_zero_f32_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x0c]
 
-# GFX11: v_fmac_dx9_zero_f32_e32 v5, null, v2    ; encoding: [0x7c,0x04,0x0a,0x0c]
 0x7c,0x04,0x0a,0x0c
+# GFX11: v_fmac_dx9_zero_f32_e32 v5, null, v2    ; encoding: [0x7c,0x04,0x0a,0x0c]
 
-# GFX11: v_fmac_dx9_zero_f32_e32 v5, -1, v2      ; encoding: [0xc1,0x04,0x0a,0x0c]
 0xc1,0x04,0x0a,0x0c
+# GFX11: v_fmac_dx9_zero_f32_e32 v5, -1, v2      ; encoding: [0xc1,0x04,0x0a,0x0c]
 
-# GFX11: v_fmac_dx9_zero_f32_e32 v5, 0.5, v2     ; encoding: [0xf0,0x04,0x0a,0x0c]
 0xf0,0x04,0x0a,0x0c
+# GFX11: v_fmac_dx9_zero_f32_e32 v5, 0.5, v2     ; encoding: [0xf0,0x04,0x0a,0x0c]
 
-# GFX11: v_fmac_dx9_zero_f32_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x0c]
 0xfd,0x04,0x0a,0x0c
+# GFX11: v_fmac_dx9_zero_f32_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x0c]
 
-# GFX11: v_fmac_dx9_zero_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x0d,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x0d,0x56,0x34,0x12,0xaf
+# GFX11: v_fmac_dx9_zero_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x0d,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmac_f16_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x6c]
 0x01,0x05,0x0a,0x6c
+# GFX11: v_fmac_f16_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x6c]
 
-# GFX11: v_fmac_f16_e32 v5, v127, v2             ; encoding: [0x7f,0x05,0x0a,0x6c]
 0x7f,0x05,0x0a,0x6c
+# GFX11: v_fmac_f16_e32 v5, v127, v2             ; encoding: [0x7f,0x05,0x0a,0x6c]
 
-# GFX11: v_fmac_f16_e32 v5, s1, v2               ; encoding: [0x01,0x04,0x0a,0x6c]
 0x01,0x04,0x0a,0x6c
+# GFX11: v_fmac_f16_e32 v5, s1, v2               ; encoding: [0x01,0x04,0x0a,0x6c]
 
-# GFX11: v_fmac_f16_e32 v5, s105, v2             ; encoding: [0x69,0x04,0x0a,0x6c]
 0x69,0x04,0x0a,0x6c
+# GFX11: v_fmac_f16_e32 v5, s105, v2             ; encoding: [0x69,0x04,0x0a,0x6c]
 
-# GFX11: v_fmac_f16_e32 v5, vcc_lo, v2           ; encoding: [0x6a,0x04,0x0a,0x6c]
 0x6a,0x04,0x0a,0x6c
+# GFX11: v_fmac_f16_e32 v5, vcc_lo, v2           ; encoding: [0x6a,0x04,0x0a,0x6c]
 
-# GFX11: v_fmac_f16_e32 v5, vcc_hi, v2           ; encoding: [0x6b,0x04,0x0a,0x6c]
 0x6b,0x04,0x0a,0x6c
+# GFX11: v_fmac_f16_e32 v5, vcc_hi, v2           ; encoding: [0x6b,0x04,0x0a,0x6c]
 
-# GFX11: v_fmac_f16_e32 v5, ttmp15, v2           ; encoding: [0x7b,0x04,0x0a,0x6c]
 0x7b,0x04,0x0a,0x6c
+# GFX11: v_fmac_f16_e32 v5, ttmp15, v2           ; encoding: [0x7b,0x04,0x0a,0x6c]
 
-# GFX11: v_fmac_f16_e32 v5, m0, v2               ; encoding: [0x7d,0x04,0x0a,0x6c]
 0x7d,0x04,0x0a,0x6c
+# GFX11: v_fmac_f16_e32 v5, m0, v2               ; encoding: [0x7d,0x04,0x0a,0x6c]
 
-# GFX11: v_fmac_f16_e32 v5, exec_lo, v2          ; encoding: [0x7e,0x04,0x0a,0x6c]
 0x7e,0x04,0x0a,0x6c
+# GFX11: v_fmac_f16_e32 v5, exec_lo, v2          ; encoding: [0x7e,0x04,0x0a,0x6c]
 
-# GFX11: v_fmac_f16_e32 v5, exec_hi, v2          ; encoding: [0x7f,0x04,0x0a,0x6c]
 0x7f,0x04,0x0a,0x6c
+# GFX11: v_fmac_f16_e32 v5, exec_hi, v2          ; encoding: [0x7f,0x04,0x0a,0x6c]
 
-# GFX11: v_fmac_f16_e32 v5, null, v2             ; encoding: [0x7c,0x04,0x0a,0x6c]
 0x7c,0x04,0x0a,0x6c
+# GFX11: v_fmac_f16_e32 v5, null, v2             ; encoding: [0x7c,0x04,0x0a,0x6c]
 
-# GFX11: v_fmac_f16_e32 v5, -1, v2               ; encoding: [0xc1,0x04,0x0a,0x6c]
 0xc1,0x04,0x0a,0x6c
+# GFX11: v_fmac_f16_e32 v5, -1, v2               ; encoding: [0xc1,0x04,0x0a,0x6c]
 
-# GFX11: v_fmac_f16_e32 v5, 0.5, v2              ; encoding: [0xf0,0x04,0x0a,0x6c]
 0xf0,0x04,0x0a,0x6c
+# GFX11: v_fmac_f16_e32 v5, 0.5, v2              ; encoding: [0xf0,0x04,0x0a,0x6c]
 
-# GFX11: v_fmac_f16_e32 v5, src_scc, v2          ; encoding: [0xfd,0x04,0x0a,0x6c]
 0xfd,0x04,0x0a,0x6c
+# GFX11: v_fmac_f16_e32 v5, src_scc, v2          ; encoding: [0xfd,0x04,0x0a,0x6c]
 
-# GFX11: v_fmac_f16_e32 v127, 0xfe0b, v127       ; encoding: [0xff,0xfe,0xfe,0x6c,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xfe,0x6c,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmac_f16_e32 v127, 0xfe0b, v127       ; encoding: [0xff,0xfe,0xfe,0x6c,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmac_f32_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x56]
 0x01,0x05,0x0a,0x56
+# GFX11: v_fmac_f32_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x56]
 
-# GFX11: v_fmac_f32_e32 v5, v255, v2             ; encoding: [0xff,0x05,0x0a,0x56]
 0xff,0x05,0x0a,0x56
+# GFX11: v_fmac_f32_e32 v5, v255, v2             ; encoding: [0xff,0x05,0x0a,0x56]
 
-# GFX11: v_fmac_f32_e32 v5, s1, v2               ; encoding: [0x01,0x04,0x0a,0x56]
 0x01,0x04,0x0a,0x56
+# GFX11: v_fmac_f32_e32 v5, s1, v2               ; encoding: [0x01,0x04,0x0a,0x56]
 
-# GFX11: v_fmac_f32_e32 v5, s105, v2             ; encoding: [0x69,0x04,0x0a,0x56]
 0x69,0x04,0x0a,0x56
+# GFX11: v_fmac_f32_e32 v5, s105, v2             ; encoding: [0x69,0x04,0x0a,0x56]
 
-# GFX11: v_fmac_f32_e32 v5, vcc_lo, v2           ; encoding: [0x6a,0x04,0x0a,0x56]
 0x6a,0x04,0x0a,0x56
+# GFX11: v_fmac_f32_e32 v5, vcc_lo, v2           ; encoding: [0x6a,0x04,0x0a,0x56]
 
-# GFX11: v_fmac_f32_e32 v5, vcc_hi, v2           ; encoding: [0x6b,0x04,0x0a,0x56]
 0x6b,0x04,0x0a,0x56
+# GFX11: v_fmac_f32_e32 v5, vcc_hi, v2           ; encoding: [0x6b,0x04,0x0a,0x56]
 
-# GFX11: v_fmac_f32_e32 v5, ttmp15, v2           ; encoding: [0x7b,0x04,0x0a,0x56]
 0x7b,0x04,0x0a,0x56
+# GFX11: v_fmac_f32_e32 v5, ttmp15, v2           ; encoding: [0x7b,0x04,0x0a,0x56]
 
-# GFX11: v_fmac_f32_e32 v5, m0, v2               ; encoding: [0x7d,0x04,0x0a,0x56]
 0x7d,0x04,0x0a,0x56
+# GFX11: v_fmac_f32_e32 v5, m0, v2               ; encoding: [0x7d,0x04,0x0a,0x56]
 
-# GFX11: v_fmac_f32_e32 v5, exec_lo, v2          ; encoding: [0x7e,0x04,0x0a,0x56]
 0x7e,0x04,0x0a,0x56
+# GFX11: v_fmac_f32_e32 v5, exec_lo, v2          ; encoding: [0x7e,0x04,0x0a,0x56]
 
-# GFX11: v_fmac_f32_e32 v5, exec_hi, v2          ; encoding: [0x7f,0x04,0x0a,0x56]
 0x7f,0x04,0x0a,0x56
+# GFX11: v_fmac_f32_e32 v5, exec_hi, v2          ; encoding: [0x7f,0x04,0x0a,0x56]
 
-# GFX11: v_fmac_f32_e32 v5, null, v2             ; encoding: [0x7c,0x04,0x0a,0x56]
 0x7c,0x04,0x0a,0x56
+# GFX11: v_fmac_f32_e32 v5, null, v2             ; encoding: [0x7c,0x04,0x0a,0x56]
 
-# GFX11: v_fmac_f32_e32 v5, -1, v2               ; encoding: [0xc1,0x04,0x0a,0x56]
 0xc1,0x04,0x0a,0x56
+# GFX11: v_fmac_f32_e32 v5, -1, v2               ; encoding: [0xc1,0x04,0x0a,0x56]
 
-# GFX11: v_fmac_f32_e32 v5, 0.5, v2              ; encoding: [0xf0,0x04,0x0a,0x56]
 0xf0,0x04,0x0a,0x56
+# GFX11: v_fmac_f32_e32 v5, 0.5, v2              ; encoding: [0xf0,0x04,0x0a,0x56]
 
-# GFX11: v_fmac_f32_e32 v5, src_scc, v2          ; encoding: [0xfd,0x04,0x0a,0x56]
 0xfd,0x04,0x0a,0x56
+# GFX11: v_fmac_f32_e32 v5, src_scc, v2          ; encoding: [0xfd,0x04,0x0a,0x56]
 
-# GFX11: v_fmac_f32_e32 v255, 0xaf123456, v255   ; encoding: [0xff,0xfe,0xff,0x57,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x57,0x56,0x34,0x12,0xaf
+# GFX11: v_fmac_f32_e32 v255, 0xaf123456, v255   ; encoding: [0xff,0xfe,0xff,0x57,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmamk_f16 v5, v1, 0xfe0b, v3          ; encoding: [0x01,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x01,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmamk_f16 v5, v1, 0xfe0b, v3          ; encoding: [0x01,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmamk_f16 v5, v127, 0xfe0b, v3        ; encoding: [0x7f,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x7f,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmamk_f16 v5, v127, 0xfe0b, v3        ; encoding: [0x7f,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmamk_f16 v5, s1, 0xfe0b, v3          ; encoding: [0x01,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x01,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmamk_f16 v5, s1, 0xfe0b, v3          ; encoding: [0x01,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmamk_f16 v5, s105, 0xfe0b, v3        ; encoding: [0x69,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x69,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmamk_f16 v5, s105, 0xfe0b, v3        ; encoding: [0x69,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmamk_f16 v5, vcc_lo, 0xfe0b, v3      ; encoding: [0x6a,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x6a,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmamk_f16 v5, vcc_lo, 0xfe0b, v3      ; encoding: [0x6a,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmamk_f16 v5, vcc_hi, 0xfe0b, v3      ; encoding: [0x6b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x6b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmamk_f16 v5, vcc_hi, 0xfe0b, v3      ; encoding: [0x6b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmamk_f16 v5, ttmp15, 0xfe0b, v3      ; encoding: [0x7b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x7b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmamk_f16 v5, ttmp15, 0xfe0b, v3      ; encoding: [0x7b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmamk_f16 v5, m0, 0xfe0b, v3          ; encoding: [0x7d,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x7d,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmamk_f16 v5, m0, 0xfe0b, v3          ; encoding: [0x7d,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmamk_f16 v5, exec_lo, 0xfe0b, v3     ; encoding: [0x7e,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x7e,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmamk_f16 v5, exec_lo, 0xfe0b, v3     ; encoding: [0x7e,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmamk_f16 v5, exec_hi, 0xfe0b, v3     ; encoding: [0x7f,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x7f,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmamk_f16 v5, exec_hi, 0xfe0b, v3     ; encoding: [0x7f,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmamk_f16 v5, null, 0xfe0b, v3        ; encoding: [0x7c,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x7c,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmamk_f16 v5, null, 0xfe0b, v3        ; encoding: [0x7c,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmamk_f16 v5, -1, 0xfe0b, v3          ; encoding: [0xc1,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0xc1,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmamk_f16 v5, -1, 0xfe0b, v3          ; encoding: [0xc1,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmamk_f16 v5, 0.5, 0xfe0b, v3         ; encoding: [0xf0,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0xf0,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmamk_f16 v5, 0.5, 0xfe0b, v3         ; encoding: [0xf0,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmamk_f16 v5, src_scc, 0xfe0b, v3     ; encoding: [0xfd,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0xfd,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmamk_f16 v5, src_scc, 0xfe0b, v3     ; encoding: [0xfd,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmamk_f16 v127, 0xfe0b, 0xfe0b, v127  ; encoding: [0xff,0xfe,0xfe,0x6e,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xfe,0x6e,0x0b,0xfe,0x00,0x00
+# GFX11: v_fmamk_f16 v127, 0xfe0b, 0xfe0b, v127  ; encoding: [0xff,0xfe,0xfe,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fmamk_f32 v5, v1, 0xaf123456, v3      ; encoding: [0x01,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x01,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_fmamk_f32 v5, v1, 0xaf123456, v3      ; encoding: [0x01,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmamk_f32 v5, v255, 0xaf123456, v3    ; encoding: [0xff,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0xff,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_fmamk_f32 v5, v255, 0xaf123456, v3    ; encoding: [0xff,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmamk_f32 v5, s1, 0xaf123456, v3      ; encoding: [0x01,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x01,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_fmamk_f32 v5, s1, 0xaf123456, v3      ; encoding: [0x01,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmamk_f32 v5, s105, 0xaf123456, v3    ; encoding: [0x69,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x69,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_fmamk_f32 v5, s105, 0xaf123456, v3    ; encoding: [0x69,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmamk_f32 v5, vcc_lo, 0xaf123456, v3  ; encoding: [0x6a,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x6a,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_fmamk_f32 v5, vcc_lo, 0xaf123456, v3  ; encoding: [0x6a,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmamk_f32 v5, vcc_hi, 0xaf123456, v3  ; encoding: [0x6b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x6b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_fmamk_f32 v5, vcc_hi, 0xaf123456, v3  ; encoding: [0x6b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmamk_f32 v5, ttmp15, 0xaf123456, v3  ; encoding: [0x7b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x7b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_fmamk_f32 v5, ttmp15, 0xaf123456, v3  ; encoding: [0x7b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmamk_f32 v5, m0, 0xaf123456, v3      ; encoding: [0x7d,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x7d,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_fmamk_f32 v5, m0, 0xaf123456, v3      ; encoding: [0x7d,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmamk_f32 v5, exec_lo, 0xaf123456, v3 ; encoding: [0x7e,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x7e,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_fmamk_f32 v5, exec_lo, 0xaf123456, v3 ; encoding: [0x7e,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmamk_f32 v5, exec_hi, 0xaf123456, v3 ; encoding: [0x7f,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x7f,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_fmamk_f32 v5, exec_hi, 0xaf123456, v3 ; encoding: [0x7f,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmamk_f32 v5, null, 0xaf123456, v3    ; encoding: [0x7c,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x7c,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_fmamk_f32 v5, null, 0xaf123456, v3    ; encoding: [0x7c,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmamk_f32 v5, -1, 0xaf123456, v3      ; encoding: [0xc1,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0xc1,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_fmamk_f32 v5, -1, 0xaf123456, v3      ; encoding: [0xc1,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmamk_f32 v5, 0.5, 0xaf123456, v3     ; encoding: [0xf0,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0xf0,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_fmamk_f32 v5, 0.5, 0xaf123456, v3     ; encoding: [0xf0,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmamk_f32 v5, src_scc, 0xaf123456, v3 ; encoding: [0xfd,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0xfd,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_fmamk_f32 v5, src_scc, 0xaf123456, v3 ; encoding: [0xfd,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fmamk_f32 v255, 0xaf123456, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x59,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x59,0x56,0x34,0x12,0xaf
+# GFX11: v_fmamk_f32 v255, 0xaf123456, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x59,0x56,0x34,0x12,0xaf]
 
-# GFX11-FAKE16: v_ldexp_f16_e32 v5, v1, v2              ; encoding: [0x01,0x05,0x0a,0x76]
-# GFX11-REAL16: v_ldexp_f16_e32 v5.l, v1.l, v2.l        ; encoding: [0x01,0x05,0x0a,0x76]
 0x01,0x05,0x0a,0x76
+# GFX11-REAL16: v_ldexp_f16_e32 v5.l, v1.l, v2.l        ; encoding: [0x01,0x05,0x0a,0x76]
+# GFX11-FAKE16: v_ldexp_f16_e32 v5, v1, v2              ; encoding: [0x01,0x05,0x0a,0x76]
 
-# GFX11-FAKE16: v_ldexp_f16_e32 v5, v127, v2            ; encoding: [0x7f,0x05,0x0a,0x76]
-# GFX11-REAL16:	v_ldexp_f16_e32 v5.l, v127.l, v2.l      ; encoding: [0x7f,0x05,0x0a,0x76]
 0x7f,0x05,0x0a,0x76
+# GFX11-REAL16: v_ldexp_f16_e32 v5.l, v127.l, v2.l      ; encoding: [0x7f,0x05,0x0a,0x76]
+# GFX11-FAKE16: v_ldexp_f16_e32 v5, v127, v2            ; encoding: [0x7f,0x05,0x0a,0x76]
 
-# GFX11-FAKE16: v_ldexp_f16_e32 v5, s1, v2              ; encoding: [0x01,0x04,0x0a,0x76]
-# GFX11-REAL16:	v_ldexp_f16_e32 v5.l, s1, v2.l          ; encoding: [0x01,0x04,0x0a,0x76]
 0x01,0x04,0x0a,0x76
+# GFX11-REAL16: v_ldexp_f16_e32 v5.l, s1, v2.l          ; encoding: [0x01,0x04,0x0a,0x76]
+# GFX11-FAKE16: v_ldexp_f16_e32 v5, s1, v2              ; encoding: [0x01,0x04,0x0a,0x76]
 
-# GFX11-FAKE16: v_ldexp_f16_e32 v5, s105, v2            ; encoding: [0x69,0x04,0x0a,0x76]
-# GFX11-REAL16:	v_ldexp_f16_e32 v5.l, s105, v2.l        ; encoding: [0x69,0x04,0x0a,0x76]
 0x69,0x04,0x0a,0x76
+# GFX11-REAL16: v_ldexp_f16_e32 v5.l, s105, v2.l        ; encoding: [0x69,0x04,0x0a,0x76]
+# GFX11-FAKE16: v_ldexp_f16_e32 v5, s105, v2            ; encoding: [0x69,0x04,0x0a,0x76]
 
-# GFX11-FAKE16: v_ldexp_f16_e32 v5, vcc_lo, v2          ; encoding: [0x6a,0x04,0x0a,0x76]
-# GFX11-REAL16:	v_ldexp_f16_e32 v5.l, vcc_lo, v2.l      ; encoding: [0x6a,0x04,0x0a,0x76]
 0x6a,0x04,0x0a,0x76
+# GFX11-REAL16: v_ldexp_f16_e32 v5.l, vcc_lo, v2.l      ; encoding: [0x6a,0x04,0x0a,0x76]
+# GFX11-FAKE16: v_ldexp_f16_e32 v5, vcc_lo, v2          ; encoding: [0x6a,0x04,0x0a,0x76]
 
-# GFX11-FAKE16: v_ldexp_f16_e32 v5, vcc_hi, v2          ; encoding: [0x6b,0x04,0x0a,0x76]
-# GFX11-REAL16:	v_ldexp_f16_e32 v5.l, vcc_hi, v2.l      ; encoding: [0x6b,0x04,0x0a,0x76]
 0x6b,0x04,0x0a,0x76
+# GFX11-REAL16: v_ldexp_f16_e32 v5.l, vcc_hi, v2.l      ; encoding: [0x6b,0x04,0x0a,0x76]
+# GFX11-FAKE16: v_ldexp_f16_e32 v5, vcc_hi, v2          ; encoding: [0x6b,0x04,0x0a,0x76]
 
-# GFX11-FAKE16: v_ldexp_f16_e32 v5, ttmp15, v2          ; encoding: [0x7b,0x04,0x0a,0x76]
-# GFX11-REAL16:	v_ldexp_f16_e32 v5.l, ttmp15, v2.l      ; encoding: [0x7b,0x04,0x0a,0x76]
 0x7b,0x04,0x0a,0x76
+# GFX11-REAL16: v_ldexp_f16_e32 v5.l, ttmp15, v2.l      ; encoding: [0x7b,0x04,0x0a,0x76]
+# GFX11-FAKE16: v_ldexp_f16_e32 v5, ttmp15, v2          ; encoding: [0x7b,0x04,0x0a,0x76]
 
-# GFX11-FAKE16: v_ldexp_f16_e32 v5, m0, v2              ; encoding: [0x7d,0x04,0x0a,0x76]
-# GFX11-REAL16:	v_ldexp_f16_e32 v5.l, m0, v2.l          ; encoding: [0x7d,0x04,0x0a,0x76]
 0x7d,0x04,0x0a,0x76
+# GFX11-REAL16: v_ldexp_f16_e32 v5.l, m0, v2.l          ; encoding: [0x7d,0x04,0x0a,0x76]
+# GFX11-FAKE16: v_ldexp_f16_e32 v5, m0, v2              ; encoding: [0x7d,0x04,0x0a,0x76]
 
-# GFX11-FAKE16: v_ldexp_f16_e32 v5, exec_lo, v2         ; encoding: [0x7e,0x04,0x0a,0x76]
-# GFX11-REAL16:	v_ldexp_f16_e32 v5.l, exec_lo, v2.l     ; encoding: [0x7e,0x04,0x0a,0x76]
 0x7e,0x04,0x0a,0x76
+# GFX11-REAL16: v_ldexp_f16_e32 v5.l, exec_lo, v2.l     ; encoding: [0x7e,0x04,0x0a,0x76]
+# GFX11-FAKE16: v_ldexp_f16_e32 v5, exec_lo, v2         ; encoding: [0x7e,0x04,0x0a,0x76]
 
-# GFX11-FAKE16: v_ldexp_f16_e32 v5, exec_hi, v2         ; encoding: [0x7f,0x04,0x0a,0x76]
-# GFX11-REAL16:	v_ldexp_f16_e32 v5.l, exec_hi, v2.l     ; encoding: [0x7f,0x04,0x0a,0x76]
 0x7f,0x04,0x0a,0x76
+# GFX11-REAL16: v_ldexp_f16_e32 v5.l, exec_hi, v2.l     ; encoding: [0x7f,0x04,0x0a,0x76]
+# GFX11-FAKE16: v_ldexp_f16_e32 v5, exec_hi, v2         ; encoding: [0x7f,0x04,0x0a,0x76]
 
-# GFX11-FAKE16: v_ldexp_f16_e32 v5, null, v2            ; encoding: [0x7c,0x04,0x0a,0x76]
-# GFX11-REAL16:	v_ldexp_f16_e32 v5.l, null, v2.l        ; encoding: [0x7c,0x04,0x0a,0x76]
 0x7c,0x04,0x0a,0x76
+# GFX11-REAL16: v_ldexp_f16_e32 v5.l, null, v2.l        ; encoding: [0x7c,0x04,0x0a,0x76]
+# GFX11-FAKE16: v_ldexp_f16_e32 v5, null, v2            ; encoding: [0x7c,0x04,0x0a,0x76]
 
-# GFX11-FAKE16: v_ldexp_f16_e32 v5, -1, v2              ; encoding: [0xc1,0x04,0x0a,0x76]
-# GFX11-REAL16:	v_ldexp_f16_e32 v5.l, -1, v2.l          ; encoding: [0xc1,0x04,0x0a,0x76]
 0xc1,0x04,0x0a,0x76
+# GFX11-REAL16: v_ldexp_f16_e32 v5.l, -1, v2.l          ; encoding: [0xc1,0x04,0x0a,0x76]
+# GFX11-FAKE16: v_ldexp_f16_e32 v5, -1, v2              ; encoding: [0xc1,0x04,0x0a,0x76]
 
-# GFX11-FAKE16: v_ldexp_f16_e32 v5, 0.5, v2             ; encoding: [0xf0,0x04,0x0a,0x76]
-# GFX11-REAL16:	v_ldexp_f16_e32 v5.l, 0.5, v2.l         ; encoding: [0xf0,0x04,0x0a,0x76]
 0xf0,0x04,0x0a,0x76
+# GFX11-REAL16: v_ldexp_f16_e32 v5.l, 0.5, v2.l         ; encoding: [0xf0,0x04,0x0a,0x76]
+# GFX11-FAKE16: v_ldexp_f16_e32 v5, 0.5, v2             ; encoding: [0xf0,0x04,0x0a,0x76]
 
-# GFX11-FAKE16: v_ldexp_f16_e32 v5, src_scc, v2         ; encoding: [0xfd,0x04,0x0a,0x76]
-# GFX11-REAL16:	v_ldexp_f16_e32 v5.l, src_scc, v2.l     ; encoding: [0xfd,0x04,0x0a,0x76]
 0xfd,0x04,0x0a,0x76
+# GFX11-REAL16: v_ldexp_f16_e32 v5.l, src_scc, v2.l     ; encoding: [0xfd,0x04,0x0a,0x76]
+# GFX11-FAKE16: v_ldexp_f16_e32 v5, src_scc, v2         ; encoding: [0xfd,0x04,0x0a,0x76]
 
-# GFX11-FAKE16: v_ldexp_f16_e32 v127, 0xfe0b, v127      ; encoding: [0xff,0xfe,0xfe,0x76,0x0b,0xfe,0x00,0x00]
-# GFX11-REAL16:	v_ldexp_f16_e32 v127.l, 0xfe0b, v127.l  ; encoding: [0xff,0xfe,0xfe,0x76,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xfe,0x76,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_ldexp_f16_e32 v127.l, 0xfe0b, v127.l  ; encoding: [0xff,0xfe,0xfe,0x76,0x0b,0xfe,0x00,0x00]
+# GFX11-FAKE16: v_ldexp_f16_e32 v127, 0xfe0b, v127      ; encoding: [0xff,0xfe,0xfe,0x76,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_lshlrev_b32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x30]
 0x01,0x05,0x0a,0x30
+# GFX11: v_lshlrev_b32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x30]
 
-# GFX11: v_lshlrev_b32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x30]
 0xff,0x05,0x0a,0x30
+# GFX11: v_lshlrev_b32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x30]
 
-# GFX11: v_lshlrev_b32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x30]
 0x01,0x04,0x0a,0x30
+# GFX11: v_lshlrev_b32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x30]
 
-# GFX11: v_lshlrev_b32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x30]
 0x69,0x04,0x0a,0x30
+# GFX11: v_lshlrev_b32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x30]
 
-# GFX11: v_lshlrev_b32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x30]
 0x6a,0x04,0x0a,0x30
+# GFX11: v_lshlrev_b32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x30]
 
-# GFX11: v_lshlrev_b32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x30]
 0x6b,0x04,0x0a,0x30
+# GFX11: v_lshlrev_b32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x30]
 
-# GFX11: v_lshlrev_b32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x30]
 0x7b,0x04,0x0a,0x30
+# GFX11: v_lshlrev_b32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x30]
 
-# GFX11: v_lshlrev_b32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x30]
 0x7d,0x04,0x0a,0x30
+# GFX11: v_lshlrev_b32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x30]
 
-# GFX11: v_lshlrev_b32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x30]
 0x7e,0x04,0x0a,0x30
+# GFX11: v_lshlrev_b32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x30]
 
-# GFX11: v_lshlrev_b32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x30]
 0x7f,0x04,0x0a,0x30
+# GFX11: v_lshlrev_b32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x30]
 
-# GFX11: v_lshlrev_b32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x30]
 0x7c,0x04,0x0a,0x30
+# GFX11: v_lshlrev_b32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x30]
 
-# GFX11: v_lshlrev_b32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x30]
 0xc1,0x04,0x0a,0x30
+# GFX11: v_lshlrev_b32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x30]
 
-# GFX11: v_lshlrev_b32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x30]
 0xf0,0x04,0x0a,0x30
+# GFX11: v_lshlrev_b32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x30]
 
-# GFX11: v_lshlrev_b32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x30]
 0xfd,0x04,0x0a,0x30
+# GFX11: v_lshlrev_b32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x30]
 
-# GFX11: v_lshlrev_b32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x31,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x31,0x56,0x34,0x12,0xaf
+# GFX11: v_lshlrev_b32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x31,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_lshrrev_b32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x32]
 0x01,0x05,0x0a,0x32
+# GFX11: v_lshrrev_b32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x32]
 
-# GFX11: v_lshrrev_b32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x32]
 0xff,0x05,0x0a,0x32
+# GFX11: v_lshrrev_b32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x32]
 
-# GFX11: v_lshrrev_b32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x32]
 0x01,0x04,0x0a,0x32
+# GFX11: v_lshrrev_b32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x32]
 
-# GFX11: v_lshrrev_b32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x32]
 0x69,0x04,0x0a,0x32
+# GFX11: v_lshrrev_b32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x32]
 
-# GFX11: v_lshrrev_b32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x32]
 0x6a,0x04,0x0a,0x32
+# GFX11: v_lshrrev_b32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x32]
 
-# GFX11: v_lshrrev_b32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x32]
 0x6b,0x04,0x0a,0x32
+# GFX11: v_lshrrev_b32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x32]
 
-# GFX11: v_lshrrev_b32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x32]
 0x7b,0x04,0x0a,0x32
+# GFX11: v_lshrrev_b32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x32]
 
-# GFX11: v_lshrrev_b32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x32]
 0x7d,0x04,0x0a,0x32
+# GFX11: v_lshrrev_b32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x32]
 
-# GFX11: v_lshrrev_b32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x32]
 0x7e,0x04,0x0a,0x32
+# GFX11: v_lshrrev_b32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x32]
 
-# GFX11: v_lshrrev_b32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x32]
 0x7f,0x04,0x0a,0x32
+# GFX11: v_lshrrev_b32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x32]
 
-# GFX11: v_lshrrev_b32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x32]
 0x7c,0x04,0x0a,0x32
+# GFX11: v_lshrrev_b32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x32]
 
-# GFX11: v_lshrrev_b32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x32]
 0xc1,0x04,0x0a,0x32
+# GFX11: v_lshrrev_b32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x32]
 
-# GFX11: v_lshrrev_b32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x32]
 0xf0,0x04,0x0a,0x32
+# GFX11: v_lshrrev_b32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x32]
 
-# GFX11: v_lshrrev_b32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x32]
 0xfd,0x04,0x0a,0x32
+# GFX11: v_lshrrev_b32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x32]
 
-# GFX11: v_lshrrev_b32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x33,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x33,0x56,0x34,0x12,0xaf
+# GFX11: v_lshrrev_b32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x33,0x56,0x34,0x12,0xaf]
 
+0x01,0x05,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, v1.l, v2.l          ; encoding: [0x01,0x05,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x72]
-0x01,0x05,0x0a,0x72
 
+0x81,0x05,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, v1.h, v2.l          ; encoding: [0x81,0x05,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x72]
-0x81,0x05,0x0a,0x72
 
+0x7f,0x05,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, v127.l, v2.l        ; encoding: [0x7f,0x05,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x72]
-0x7f,0x05,0x0a,0x72
 
+0xff,0x05,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, v127.h, v2.l        ; encoding: [0xff,0x05,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x72]
-0xff,0x05,0x0a,0x72
 
+0x01,0x04,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, s1, v2.l            ; encoding: [0x01,0x04,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x72]
-0x01,0x04,0x0a,0x72
 
+0x69,0x04,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, s105, v2.l          ; encoding: [0x69,0x04,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x72]
-0x69,0x04,0x0a,0x72
 
+0x6a,0x04,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, vcc_lo, v2.l        ; encoding: [0x6a,0x04,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x72]
-0x6a,0x04,0x0a,0x72
 
+0x6b,0x04,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, vcc_hi, v2.l        ; encoding: [0x6b,0x04,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x72]
-0x6b,0x04,0x0a,0x72
 
+0x7b,0x04,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, ttmp15, v2.l        ; encoding: [0x7b,0x04,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x72]
-0x7b,0x04,0x0a,0x72
 
+0x7d,0x04,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, m0, v2.l            ; encoding: [0x7d,0x04,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x72]
-0x7d,0x04,0x0a,0x72
 
+0x7e,0x04,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, exec_lo, v2.l       ; encoding: [0x7e,0x04,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x72]
-0x7e,0x04,0x0a,0x72
 
+0x7f,0x04,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, exec_hi, v2.l       ; encoding: [0x7f,0x04,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x72]
-0x7f,0x04,0x0a,0x72
 
+0x7c,0x04,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, null, v2.l          ; encoding: [0x7c,0x04,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x72]
-0x7c,0x04,0x0a,0x72
 
+0xc1,0x04,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, -1, v2.l            ; encoding: [0xc1,0x04,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x72]
-0xc1,0x04,0x0a,0x72
 
+0xf0,0x04,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, 0.5, v2.l           ; encoding: [0xf0,0x04,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x72]
-0xf0,0x04,0x0a,0x72
 
+0xfd,0x04,0x0a,0x72
 # GFX11-REAL16: v_max_f16_e32 v5.l, src_scc, v2.l       ; encoding: [0xfd,0x04,0x0a,0x72]
 # GFX11-FAKE16: v_max_f16_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x72]
-0xfd,0x04,0x0a,0x72
 
-# GFX11-REAL16: v_max_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x73]
-# COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xfd,0x04,0x0b,0x73
 0xfd,0x04,0x0b,0x73
+# GFX11-REAL16: v_max_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x73]
 
+0xff,0xfe,0xfe,0x72,0x0b,0xfe,0x00,0x00
 # GFX11-REAL16: v_max_f16_e32 v127.l, 0xfe0b, v127.l    ; encoding: [0xff,0xfe,0xfe,0x72,0x0b,0xfe,0x00,0x00]
 # GFX11-FAKE16: v_max_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x72,0x0b,0xfe,0x00,0x00]
-0xff,0xfe,0xfe,0x72,0x0b,0xfe,0x00,0x00
 
-# GFX11-REAL16: v_max_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x73,0x0b,0xfe,0x00,0x00]
-# COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xff,0xfe,0xff,0x73,0x0b,0xfe,0x00,0x00
 0xff,0xfe,0xff,0x73,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_max_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x73,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_max_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x20]
 0x01,0x05,0x0a,0x20
+# GFX11: v_max_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x20]
 
-# GFX11: v_max_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x20]
 0xff,0x05,0x0a,0x20
+# GFX11: v_max_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x20]
 
-# GFX11: v_max_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x20]
 0x01,0x04,0x0a,0x20
+# GFX11: v_max_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x20]
 
-# GFX11: v_max_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x20]
 0x69,0x04,0x0a,0x20
+# GFX11: v_max_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x20]
 
-# GFX11: v_max_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x20]
 0x6a,0x04,0x0a,0x20
+# GFX11: v_max_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x20]
 
-# GFX11: v_max_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x20]
 0x6b,0x04,0x0a,0x20
+# GFX11: v_max_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x20]
 
-# GFX11: v_max_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x20]
 0x7b,0x04,0x0a,0x20
+# GFX11: v_max_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x20]
 
-# GFX11: v_max_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x20]
 0x7d,0x04,0x0a,0x20
+# GFX11: v_max_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x20]
 
-# GFX11: v_max_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x20]
 0x7e,0x04,0x0a,0x20
+# GFX11: v_max_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x20]
 
-# GFX11: v_max_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x20]
 0x7f,0x04,0x0a,0x20
+# GFX11: v_max_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x20]
 
-# GFX11: v_max_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x20]
 0x7c,0x04,0x0a,0x20
+# GFX11: v_max_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x20]
 
-# GFX11: v_max_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x20]
 0xc1,0x04,0x0a,0x20
+# GFX11: v_max_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x20]
 
-# GFX11: v_max_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x20]
 0xf0,0x04,0x0a,0x20
+# GFX11: v_max_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x20]
 
-# GFX11: v_max_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x20]
 0xfd,0x04,0x0a,0x20
+# GFX11: v_max_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x20]
 
-# GFX11: v_max_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x21,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x21,0x56,0x34,0x12,0xaf
+# GFX11: v_max_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x21,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_max_i32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x24]
 0x01,0x05,0x0a,0x24
+# GFX11: v_max_i32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x24]
 
-# GFX11: v_max_i32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x24]
 0xff,0x05,0x0a,0x24
+# GFX11: v_max_i32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x24]
 
-# GFX11: v_max_i32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x24]
 0x01,0x04,0x0a,0x24
+# GFX11: v_max_i32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x24]
 
-# GFX11: v_max_i32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x24]
 0x69,0x04,0x0a,0x24
+# GFX11: v_max_i32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x24]
 
-# GFX11: v_max_i32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x24]
 0x6a,0x04,0x0a,0x24
+# GFX11: v_max_i32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x24]
 
-# GFX11: v_max_i32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x24]
 0x6b,0x04,0x0a,0x24
+# GFX11: v_max_i32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x24]
 
-# GFX11: v_max_i32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x24]
 0x7b,0x04,0x0a,0x24
+# GFX11: v_max_i32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x24]
 
-# GFX11: v_max_i32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x24]
 0x7d,0x04,0x0a,0x24
+# GFX11: v_max_i32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x24]
 
-# GFX11: v_max_i32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x24]
 0x7e,0x04,0x0a,0x24
+# GFX11: v_max_i32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x24]
 
-# GFX11: v_max_i32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x24]
 0x7f,0x04,0x0a,0x24
+# GFX11: v_max_i32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x24]
 
-# GFX11: v_max_i32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x24]
 0x7c,0x04,0x0a,0x24
+# GFX11: v_max_i32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x24]
 
-# GFX11: v_max_i32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x24]
 0xc1,0x04,0x0a,0x24
+# GFX11: v_max_i32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x24]
 
-# GFX11: v_max_i32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x24]
 0xf0,0x04,0x0a,0x24
+# GFX11: v_max_i32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x24]
 
-# GFX11: v_max_i32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x24]
 0xfd,0x04,0x0a,0x24
+# GFX11: v_max_i32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x24]
 
-# GFX11: v_max_i32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x25,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x25,0x56,0x34,0x12,0xaf
+# GFX11: v_max_i32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x25,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_max_u32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x28]
 0x01,0x05,0x0a,0x28
+# GFX11: v_max_u32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x28]
 
-# GFX11: v_max_u32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x28]
 0xff,0x05,0x0a,0x28
+# GFX11: v_max_u32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x28]
 
-# GFX11: v_max_u32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x28]
 0x01,0x04,0x0a,0x28
+# GFX11: v_max_u32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x28]
 
-# GFX11: v_max_u32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x28]
 0x69,0x04,0x0a,0x28
+# GFX11: v_max_u32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x28]
 
-# GFX11: v_max_u32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x28]
 0x6a,0x04,0x0a,0x28
+# GFX11: v_max_u32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x28]
 
-# GFX11: v_max_u32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x28]
 0x6b,0x04,0x0a,0x28
+# GFX11: v_max_u32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x28]
 
-# GFX11: v_max_u32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x28]
 0x7b,0x04,0x0a,0x28
+# GFX11: v_max_u32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x28]
 
-# GFX11: v_max_u32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x28]
 0x7d,0x04,0x0a,0x28
+# GFX11: v_max_u32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x28]
 
-# GFX11: v_max_u32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x28]
 0x7e,0x04,0x0a,0x28
+# GFX11: v_max_u32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x28]
 
-# GFX11: v_max_u32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x28]
 0x7f,0x04,0x0a,0x28
+# GFX11: v_max_u32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x28]
 
-# GFX11: v_max_u32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x28]
 0x7c,0x04,0x0a,0x28
+# GFX11: v_max_u32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x28]
 
-# GFX11: v_max_u32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x28]
 0xc1,0x04,0x0a,0x28
+# GFX11: v_max_u32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x28]
 
-# GFX11: v_max_u32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x28]
 0xf0,0x04,0x0a,0x28
+# GFX11: v_max_u32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x28]
 
-# GFX11: v_max_u32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x28]
 0xfd,0x04,0x0a,0x28
+# GFX11: v_max_u32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x28]
 
-# GFX11: v_max_u32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x29,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x29,0x56,0x34,0x12,0xaf
+# GFX11: v_max_u32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x29,0x56,0x34,0x12,0xaf]
 
+0x01,0x05,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, v1.l, v2.l          ; encoding: [0x01,0x05,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x74]
-0x01,0x05,0x0a,0x74
 
+0x81,0x05,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, v1.h, v2.l          ; encoding: [0x81,0x05,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x74]
-0x81,0x05,0x0a,0x74
 
+0x7f,0x05,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, v127.l, v2.l        ; encoding: [0x7f,0x05,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x74]
-0x7f,0x05,0x0a,0x74
 
+0xff,0x05,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, v127.h, v2.l        ; encoding: [0xff,0x05,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x74]
-0xff,0x05,0x0a,0x74
 
+0x01,0x04,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, s1, v2.l            ; encoding: [0x01,0x04,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x74]
-0x01,0x04,0x0a,0x74
 
+0x69,0x04,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, s105, v2.l          ; encoding: [0x69,0x04,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x74]
-0x69,0x04,0x0a,0x74
 
+0x6a,0x04,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, vcc_lo, v2.l        ; encoding: [0x6a,0x04,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x74]
-0x6a,0x04,0x0a,0x74
 
+0x6b,0x04,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, vcc_hi, v2.l        ; encoding: [0x6b,0x04,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x74]
-0x6b,0x04,0x0a,0x74
 
+0x7b,0x04,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, ttmp15, v2.l        ; encoding: [0x7b,0x04,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x74]
-0x7b,0x04,0x0a,0x74
 
+0x7d,0x04,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, m0, v2.l            ; encoding: [0x7d,0x04,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x74]
-0x7d,0x04,0x0a,0x74
 
+0x7e,0x04,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, exec_lo, v2.l       ; encoding: [0x7e,0x04,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x74]
-0x7e,0x04,0x0a,0x74
 
+0x7f,0x04,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, exec_hi, v2.l       ; encoding: [0x7f,0x04,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x74]
-0x7f,0x04,0x0a,0x74
 
+0x7c,0x04,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, null, v2.l          ; encoding: [0x7c,0x04,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x74]
-0x7c,0x04,0x0a,0x74
 
+0xc1,0x04,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, -1, v2.l            ; encoding: [0xc1,0x04,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x74]
-0xc1,0x04,0x0a,0x74
 
+0xf0,0x04,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, 0.5, v2.l           ; encoding: [0xf0,0x04,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x74]
-0xf0,0x04,0x0a,0x74
 
+0xfd,0x04,0x0a,0x74
 # GFX11-REAL16: v_min_f16_e32 v5.l, src_scc, v2.l       ; encoding: [0xfd,0x04,0x0a,0x74]
 # GFX11-FAKE16: v_min_f16_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x74]
-0xfd,0x04,0x0a,0x74
 
-# GFX11-REAL16: v_min_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x75]
-# COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xfd,0x04,0x0b,0x75
 0xfd,0x04,0x0b,0x75
+# GFX11-REAL16: v_min_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x75]
 
+0xff,0xfe,0xfe,0x74,0x0b,0xfe,0x00,0x00
 # GFX11-REAL16: v_min_f16_e32 v127.l, 0xfe0b, v127.l    ; encoding: [0xff,0xfe,0xfe,0x74,0x0b,0xfe,0x00,0x00]
 # GFX11-FAKE16: v_min_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x74,0x0b,0xfe,0x00,0x00]
-0xff,0xfe,0xfe,0x74,0x0b,0xfe,0x00,0x00
 
-# GFX11-REAL16: v_min_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x75,0x0b,0xfe,0x00,0x00]
-# COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xff,0xfe,0xff,0x75,0x0b,0xfe,0x00,0x00
 0xff,0xfe,0xff,0x75,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_min_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x75,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_min_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x1e]
 0x01,0x05,0x0a,0x1e
+# GFX11: v_min_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x1e]
 
-# GFX11: v_min_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x1e]
 0xff,0x05,0x0a,0x1e
+# GFX11: v_min_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x1e]
 
-# GFX11: v_min_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x1e]
 0x01,0x04,0x0a,0x1e
+# GFX11: v_min_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x1e]
 
-# GFX11: v_min_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x1e]
 0x69,0x04,0x0a,0x1e
+# GFX11: v_min_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x1e]
 
-# GFX11: v_min_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x1e]
 0x6a,0x04,0x0a,0x1e
+# GFX11: v_min_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x1e]
 
-# GFX11: v_min_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x1e]
 0x6b,0x04,0x0a,0x1e
+# GFX11: v_min_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x1e]
 
-# GFX11: v_min_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x1e]
 0x7b,0x04,0x0a,0x1e
+# GFX11: v_min_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x1e]
 
-# GFX11: v_min_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x1e]
 0x7d,0x04,0x0a,0x1e
+# GFX11: v_min_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x1e]
 
-# GFX11: v_min_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x1e]
 0x7e,0x04,0x0a,0x1e
+# GFX11: v_min_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x1e]
 
-# GFX11: v_min_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x1e]
 0x7f,0x04,0x0a,0x1e
+# GFX11: v_min_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x1e]
 
-# GFX11: v_min_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x1e]
 0x7c,0x04,0x0a,0x1e
+# GFX11: v_min_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x1e]
 
-# GFX11: v_min_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x1e]
 0xc1,0x04,0x0a,0x1e
+# GFX11: v_min_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x1e]
 
-# GFX11: v_min_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x1e]
 0xf0,0x04,0x0a,0x1e
+# GFX11: v_min_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x1e]
 
-# GFX11: v_min_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x1e]
 0xfd,0x04,0x0a,0x1e
+# GFX11: v_min_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x1e]
 
-# GFX11: v_min_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x1f,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x1f,0x56,0x34,0x12,0xaf
+# GFX11: v_min_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x1f,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_min_i32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x22]
 0x01,0x05,0x0a,0x22
+# GFX11: v_min_i32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x22]
 
-# GFX11: v_min_i32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x22]
 0xff,0x05,0x0a,0x22
+# GFX11: v_min_i32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x22]
 
-# GFX11: v_min_i32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x22]
 0x01,0x04,0x0a,0x22
+# GFX11: v_min_i32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x22]
 
-# GFX11: v_min_i32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x22]
 0x69,0x04,0x0a,0x22
+# GFX11: v_min_i32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x22]
 
-# GFX11: v_min_i32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x22]
 0x6a,0x04,0x0a,0x22
+# GFX11: v_min_i32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x22]
 
-# GFX11: v_min_i32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x22]
 0x6b,0x04,0x0a,0x22
+# GFX11: v_min_i32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x22]
 
-# GFX11: v_min_i32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x22]
 0x7b,0x04,0x0a,0x22
+# GFX11: v_min_i32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x22]
 
-# GFX11: v_min_i32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x22]
 0x7d,0x04,0x0a,0x22
+# GFX11: v_min_i32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x22]
 
-# GFX11: v_min_i32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x22]
 0x7e,0x04,0x0a,0x22
+# GFX11: v_min_i32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x22]
 
-# GFX11: v_min_i32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x22]
 0x7f,0x04,0x0a,0x22
+# GFX11: v_min_i32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x22]
 
-# GFX11: v_min_i32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x22]
 0x7c,0x04,0x0a,0x22
+# GFX11: v_min_i32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x22]
 
-# GFX11: v_min_i32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x22]
 0xc1,0x04,0x0a,0x22
+# GFX11: v_min_i32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x22]
 
-# GFX11: v_min_i32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x22]
 0xf0,0x04,0x0a,0x22
+# GFX11: v_min_i32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x22]
 
-# GFX11: v_min_i32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x22]
 0xfd,0x04,0x0a,0x22
+# GFX11: v_min_i32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x22]
 
-# GFX11: v_min_i32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x23,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x23,0x56,0x34,0x12,0xaf
+# GFX11: v_min_i32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x23,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_min_u32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x26]
 0x01,0x05,0x0a,0x26
+# GFX11: v_min_u32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x26]
 
-# GFX11: v_min_u32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x26]
 0xff,0x05,0x0a,0x26
+# GFX11: v_min_u32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x26]
 
-# GFX11: v_min_u32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x26]
 0x01,0x04,0x0a,0x26
+# GFX11: v_min_u32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x26]
 
-# GFX11: v_min_u32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x26]
 0x69,0x04,0x0a,0x26
+# GFX11: v_min_u32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x26]
 
-# GFX11: v_min_u32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x26]
 0x6a,0x04,0x0a,0x26
+# GFX11: v_min_u32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x26]
 
-# GFX11: v_min_u32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x26]
 0x6b,0x04,0x0a,0x26
+# GFX11: v_min_u32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x26]
 
-# GFX11: v_min_u32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x26]
 0x7b,0x04,0x0a,0x26
+# GFX11: v_min_u32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x26]
 
-# GFX11: v_min_u32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x26]
 0x7d,0x04,0x0a,0x26
+# GFX11: v_min_u32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x26]
 
-# GFX11: v_min_u32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x26]
 0x7e,0x04,0x0a,0x26
+# GFX11: v_min_u32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x26]
 
-# GFX11: v_min_u32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x26]
 0x7f,0x04,0x0a,0x26
+# GFX11: v_min_u32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x26]
 
-# GFX11: v_min_u32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x26]
 0x7c,0x04,0x0a,0x26
+# GFX11: v_min_u32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x26]
 
-# GFX11: v_min_u32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x26]
 0xc1,0x04,0x0a,0x26
+# GFX11: v_min_u32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x26]
 
-# GFX11: v_min_u32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x26]
 0xf0,0x04,0x0a,0x26
+# GFX11: v_min_u32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x26]
 
-# GFX11: v_min_u32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x26]
 0xfd,0x04,0x0a,0x26
+# GFX11: v_min_u32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x26]
 
-# GFX11: v_min_u32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x27,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x27,0x56,0x34,0x12,0xaf
+# GFX11: v_min_u32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x27,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mul_dx9_zero_f32_e32 v5, v1, v2       ; encoding: [0x01,0x05,0x0a,0x0e]
 0x01,0x05,0x0a,0x0e
+# GFX11: v_mul_dx9_zero_f32_e32 v5, v1, v2       ; encoding: [0x01,0x05,0x0a,0x0e]
 
-# GFX11: v_mul_dx9_zero_f32_e32 v5, v255, v2     ; encoding: [0xff,0x05,0x0a,0x0e]
 0xff,0x05,0x0a,0x0e
+# GFX11: v_mul_dx9_zero_f32_e32 v5, v255, v2     ; encoding: [0xff,0x05,0x0a,0x0e]
 
-# GFX11: v_mul_dx9_zero_f32_e32 v5, s1, v2       ; encoding: [0x01,0x04,0x0a,0x0e]
 0x01,0x04,0x0a,0x0e
+# GFX11: v_mul_dx9_zero_f32_e32 v5, s1, v2       ; encoding: [0x01,0x04,0x0a,0x0e]
 
-# GFX11: v_mul_dx9_zero_f32_e32 v5, s105, v2     ; encoding: [0x69,0x04,0x0a,0x0e]
 0x69,0x04,0x0a,0x0e
+# GFX11: v_mul_dx9_zero_f32_e32 v5, s105, v2     ; encoding: [0x69,0x04,0x0a,0x0e]
 
-# GFX11: v_mul_dx9_zero_f32_e32 v5, vcc_lo, v2   ; encoding: [0x6a,0x04,0x0a,0x0e]
 0x6a,0x04,0x0a,0x0e
+# GFX11: v_mul_dx9_zero_f32_e32 v5, vcc_lo, v2   ; encoding: [0x6a,0x04,0x0a,0x0e]
 
-# GFX11: v_mul_dx9_zero_f32_e32 v5, vcc_hi, v2   ; encoding: [0x6b,0x04,0x0a,0x0e]
 0x6b,0x04,0x0a,0x0e
+# GFX11: v_mul_dx9_zero_f32_e32 v5, vcc_hi, v2   ; encoding: [0x6b,0x04,0x0a,0x0e]
 
-# GFX11: v_mul_dx9_zero_f32_e32 v5, ttmp15, v2   ; encoding: [0x7b,0x04,0x0a,0x0e]
 0x7b,0x04,0x0a,0x0e
+# GFX11: v_mul_dx9_zero_f32_e32 v5, ttmp15, v2   ; encoding: [0x7b,0x04,0x0a,0x0e]
 
-# GFX11: v_mul_dx9_zero_f32_e32 v5, m0, v2       ; encoding: [0x7d,0x04,0x0a,0x0e]
 0x7d,0x04,0x0a,0x0e
+# GFX11: v_mul_dx9_zero_f32_e32 v5, m0, v2       ; encoding: [0x7d,0x04,0x0a,0x0e]
 
-# GFX11: v_mul_dx9_zero_f32_e32 v5, exec_lo, v2  ; encoding: [0x7e,0x04,0x0a,0x0e]
 0x7e,0x04,0x0a,0x0e
+# GFX11: v_mul_dx9_zero_f32_e32 v5, exec_lo, v2  ; encoding: [0x7e,0x04,0x0a,0x0e]
 
-# GFX11: v_mul_dx9_zero_f32_e32 v5, exec_hi, v2  ; encoding: [0x7f,0x04,0x0a,0x0e]
 0x7f,0x04,0x0a,0x0e
+# GFX11: v_mul_dx9_zero_f32_e32 v5, exec_hi, v2  ; encoding: [0x7f,0x04,0x0a,0x0e]
 
-# GFX11: v_mul_dx9_zero_f32_e32 v5, null, v2     ; encoding: [0x7c,0x04,0x0a,0x0e]
 0x7c,0x04,0x0a,0x0e
+# GFX11: v_mul_dx9_zero_f32_e32 v5, null, v2     ; encoding: [0x7c,0x04,0x0a,0x0e]
 
-# GFX11: v_mul_dx9_zero_f32_e32 v5, -1, v2       ; encoding: [0xc1,0x04,0x0a,0x0e]
 0xc1,0x04,0x0a,0x0e
+# GFX11: v_mul_dx9_zero_f32_e32 v5, -1, v2       ; encoding: [0xc1,0x04,0x0a,0x0e]
 
-# GFX11: v_mul_dx9_zero_f32_e32 v5, 0.5, v2      ; encoding: [0xf0,0x04,0x0a,0x0e]
 0xf0,0x04,0x0a,0x0e
+# GFX11: v_mul_dx9_zero_f32_e32 v5, 0.5, v2      ; encoding: [0xf0,0x04,0x0a,0x0e]
 
-# GFX11: v_mul_dx9_zero_f32_e32 v5, src_scc, v2  ; encoding: [0xfd,0x04,0x0a,0x0e]
 0xfd,0x04,0x0a,0x0e
+# GFX11: v_mul_dx9_zero_f32_e32 v5, src_scc, v2  ; encoding: [0xfd,0x04,0x0a,0x0e]
 
-# GFX11: v_mul_dx9_zero_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x0f,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x0f,0x56,0x34,0x12,0xaf
+# GFX11: v_mul_dx9_zero_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x0f,0x56,0x34,0x12,0xaf]
 
+0x01,0x05,0x0a,0x6a
 # GFX11-REAL16: v_mul_f16_e32 v5.l, v1.l, v2.l          ; encoding: [0x01,0x05,0x0a,0x6a]
 # GFX11-FAKE16: v_mul_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x6a]
-0x01,0x05,0x0a,0x6a
 
-# GFX11-REAL16: v_mul_f16_e32 v5.l, v1.h, v2.l          ; encoding: [0x81,0x05,0x0a,0x6a]
-# GFX11-FAKE16: v_mul_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x6a
 0x81,0x05,0x0a,0x6a
+# GFX11-REAL16: v_mul_f16_e32 v5.l, v1.h, v2.l          ; encoding: [0x81,0x05,0x0a,0x6a]
+# GFX11-FAKE16: v_mul_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x6a]
 
+0x7f,0x05,0x0a,0x6a
 # GFX11-REAL16: v_mul_f16_e32 v5.l, v127.l, v2.l        ; encoding: [0x7f,0x05,0x0a,0x6a]
 # GFX11-FAKE16: v_mul_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x6a]
-0x7f,0x05,0x0a,0x6a
 
+0xff,0x05,0x0a,0x6a
 # GFX11-REAL16: v_mul_f16_e32 v5.l, v127.h, v2.l        ; encoding: [0xff,0x05,0x0a,0x6a]
 # GFX11-FAKE16: v_mul_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x6a]
-0xff,0x05,0x0a,0x6a
 
+0x01,0x04,0x0a,0x6a
 # GFX11-REAL16: v_mul_f16_e32 v5.l, s1, v2.l            ; encoding: [0x01,0x04,0x0a,0x6a]
 # GFX11-FAKE16: v_mul_f16_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x6a]
-0x01,0x04,0x0a,0x6a
 
+0x69,0x04,0x0a,0x6a
 # GFX11-REAL16: v_mul_f16_e32 v5.l, s105, v2.l          ; encoding: [0x69,0x04,0x0a,0x6a]
 # GFX11-FAKE16: v_mul_f16_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x6a]
-0x69,0x04,0x0a,0x6a
 
+0x6a,0x04,0x0a,0x6a
 # GFX11-REAL16: v_mul_f16_e32 v5.l, vcc_lo, v2.l        ; encoding: [0x6a,0x04,0x0a,0x6a]
 # GFX11-FAKE16: v_mul_f16_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x6a]
-0x6a,0x04,0x0a,0x6a
 
+0x6b,0x04,0x0a,0x6a
 # GFX11-REAL16: v_mul_f16_e32 v5.l, vcc_hi, v2.l        ; encoding: [0x6b,0x04,0x0a,0x6a]
 # GFX11-FAKE16: v_mul_f16_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x6a]
-0x6b,0x04,0x0a,0x6a
 
+0x7b,0x04,0x0a,0x6a
 # GFX11-REAL16: v_mul_f16_e32 v5.l, ttmp15, v2.l        ; encoding: [0x7b,0x04,0x0a,0x6a]
 # GFX11-FAKE16: v_mul_f16_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x6a]
-0x7b,0x04,0x0a,0x6a
 
+0x7d,0x04,0x0a,0x6a
 # GFX11-REAL16: v_mul_f16_e32 v5.l, m0, v2.l            ; encoding: [0x7d,0x04,0x0a,0x6a]
 # GFX11-FAKE16: v_mul_f16_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x6a]
-0x7d,0x04,0x0a,0x6a
 
+0x7e,0x04,0x0a,0x6a
 # GFX11-REAL16: v_mul_f16_e32 v5.l, exec_lo, v2.l       ; encoding: [0x7e,0x04,0x0a,0x6a]
 # GFX11-FAKE16: v_mul_f16_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x6a]
-0x7e,0x04,0x0a,0x6a
 
+0x7f,0x04,0x0a,0x6a
 # GFX11-REAL16: v_mul_f16_e32 v5.l, exec_hi, v2.l       ; encoding: [0x7f,0x04,0x0a,0x6a]
 # GFX11-FAKE16: v_mul_f16_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x6a]
-0x7f,0x04,0x0a,0x6a
 
+0x7c,0x04,0x0a,0x6a
 # GFX11-REAL16: v_mul_f16_e32 v5.l, null, v2.l          ; encoding: [0x7c,0x04,0x0a,0x6a]
 # GFX11-FAKE16: v_mul_f16_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x6a]
-0x7c,0x04,0x0a,0x6a
 
+0xc1,0x04,0x0a,0x6a
 # GFX11-REAL16: v_mul_f16_e32 v5.l, -1, v2.l            ; encoding: [0xc1,0x04,0x0a,0x6a]
 # GFX11-FAKE16: v_mul_f16_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x6a]
-0xc1,0x04,0x0a,0x6a
 
+0xf0,0x04,0x0a,0x6a
 # GFX11-REAL16: v_mul_f16_e32 v5.l, 0.5, v2.l           ; encoding: [0xf0,0x04,0x0a,0x6a]
 # GFX11-FAKE16: v_mul_f16_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x6a]
-0xf0,0x04,0x0a,0x6a
 
+0xfd,0x04,0x0a,0x6a
 # GFX11-REAL16: v_mul_f16_e32 v5.l, src_scc, v2.l       ; encoding: [0xfd,0x04,0x0a,0x6a]
 # GFX11-FAKE16: v_mul_f16_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x6a]
-0xfd,0x04,0x0a,0x6a
 
-# GFX11-REAL16: v_mul_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x6b]
-# COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xfd,0x04,0x0b,0x6b
 0xfd,0x04,0x0b,0x6b
+# GFX11-REAL16: v_mul_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x6b]
 
+0xff,0xfe,0xfe,0x6a,0x0b,0xfe,0x00,0x00
 # GFX11-REAL16: v_mul_f16_e32 v127.l, 0xfe0b, v127.l    ; encoding: [0xff,0xfe,0xfe,0x6a,0x0b,0xfe,0x00,0x00]
 # GFX11-FAKE16: v_mul_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x6a,0x0b,0xfe,0x00,0x00]
-0xff,0xfe,0xfe,0x6a,0x0b,0xfe,0x00,0x00
 
-# GFX11-REAL16: v_mul_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x6b,0x0b,0xfe,0x00,0x00]
-# COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xff,0xfe,0xff,0x6b,0x0b,0xfe,0x00,0x00
 0xff,0xfe,0xff,0x6b,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_mul_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x6b,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_mul_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x10]
 0x01,0x05,0x0a,0x10
+# GFX11: v_mul_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x10]
 
-# GFX11: v_mul_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x10]
 0xff,0x05,0x0a,0x10
+# GFX11: v_mul_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x10]
 
-# GFX11: v_mul_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x10]
 0x01,0x04,0x0a,0x10
+# GFX11: v_mul_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x10]
 
-# GFX11: v_mul_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x10]
 0x69,0x04,0x0a,0x10
+# GFX11: v_mul_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x10]
 
-# GFX11: v_mul_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x10]
 0x6a,0x04,0x0a,0x10
+# GFX11: v_mul_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x10]
 
-# GFX11: v_mul_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x10]
 0x6b,0x04,0x0a,0x10
+# GFX11: v_mul_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x10]
 
-# GFX11: v_mul_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x10]
 0x7b,0x04,0x0a,0x10
+# GFX11: v_mul_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x10]
 
-# GFX11: v_mul_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x10]
 0x7d,0x04,0x0a,0x10
+# GFX11: v_mul_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x10]
 
-# GFX11: v_mul_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x10]
 0x7e,0x04,0x0a,0x10
+# GFX11: v_mul_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x10]
 
-# GFX11: v_mul_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x10]
 0x7f,0x04,0x0a,0x10
+# GFX11: v_mul_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x10]
 
-# GFX11: v_mul_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x10]
 0x7c,0x04,0x0a,0x10
+# GFX11: v_mul_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x10]
 
-# GFX11: v_mul_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x10]
 0xc1,0x04,0x0a,0x10
+# GFX11: v_mul_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x10]
 
-# GFX11: v_mul_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x10]
 0xf0,0x04,0x0a,0x10
+# GFX11: v_mul_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x10]
 
-# GFX11: v_mul_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x10]
 0xfd,0x04,0x0a,0x10
+# GFX11: v_mul_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x10]
 
-# GFX11: v_mul_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x11,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x11,0x56,0x34,0x12,0xaf
+# GFX11: v_mul_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x11,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mul_hi_i32_i24_e32 v5, v1, v2         ; encoding: [0x01,0x05,0x0a,0x14]
 0x01,0x05,0x0a,0x14
+# GFX11: v_mul_hi_i32_i24_e32 v5, v1, v2         ; encoding: [0x01,0x05,0x0a,0x14]
 
-# GFX11: v_mul_hi_i32_i24_e32 v5, v255, v2       ; encoding: [0xff,0x05,0x0a,0x14]
 0xff,0x05,0x0a,0x14
+# GFX11: v_mul_hi_i32_i24_e32 v5, v255, v2       ; encoding: [0xff,0x05,0x0a,0x14]
 
-# GFX11: v_mul_hi_i32_i24_e32 v5, s1, v2         ; encoding: [0x01,0x04,0x0a,0x14]
 0x01,0x04,0x0a,0x14
+# GFX11: v_mul_hi_i32_i24_e32 v5, s1, v2         ; encoding: [0x01,0x04,0x0a,0x14]
 
-# GFX11: v_mul_hi_i32_i24_e32 v5, s105, v2       ; encoding: [0x69,0x04,0x0a,0x14]
 0x69,0x04,0x0a,0x14
+# GFX11: v_mul_hi_i32_i24_e32 v5, s105, v2       ; encoding: [0x69,0x04,0x0a,0x14]
 
-# GFX11: v_mul_hi_i32_i24_e32 v5, vcc_lo, v2     ; encoding: [0x6a,0x04,0x0a,0x14]
 0x6a,0x04,0x0a,0x14
+# GFX11: v_mul_hi_i32_i24_e32 v5, vcc_lo, v2     ; encoding: [0x6a,0x04,0x0a,0x14]
 
-# GFX11: v_mul_hi_i32_i24_e32 v5, vcc_hi, v2     ; encoding: [0x6b,0x04,0x0a,0x14]
 0x6b,0x04,0x0a,0x14
+# GFX11: v_mul_hi_i32_i24_e32 v5, vcc_hi, v2     ; encoding: [0x6b,0x04,0x0a,0x14]
 
-# GFX11: v_mul_hi_i32_i24_e32 v5, ttmp15, v2     ; encoding: [0x7b,0x04,0x0a,0x14]
 0x7b,0x04,0x0a,0x14
+# GFX11: v_mul_hi_i32_i24_e32 v5, ttmp15, v2     ; encoding: [0x7b,0x04,0x0a,0x14]
 
-# GFX11: v_mul_hi_i32_i24_e32 v5, m0, v2         ; encoding: [0x7d,0x04,0x0a,0x14]
 0x7d,0x04,0x0a,0x14
+# GFX11: v_mul_hi_i32_i24_e32 v5, m0, v2         ; encoding: [0x7d,0x04,0x0a,0x14]
 
-# GFX11: v_mul_hi_i32_i24_e32 v5, exec_lo, v2    ; encoding: [0x7e,0x04,0x0a,0x14]
 0x7e,0x04,0x0a,0x14
+# GFX11: v_mul_hi_i32_i24_e32 v5, exec_lo, v2    ; encoding: [0x7e,0x04,0x0a,0x14]
 
-# GFX11: v_mul_hi_i32_i24_e32 v5, exec_hi, v2    ; encoding: [0x7f,0x04,0x0a,0x14]
 0x7f,0x04,0x0a,0x14
+# GFX11: v_mul_hi_i32_i24_e32 v5, exec_hi, v2    ; encoding: [0x7f,0x04,0x0a,0x14]
 
-# GFX11: v_mul_hi_i32_i24_e32 v5, null, v2       ; encoding: [0x7c,0x04,0x0a,0x14]
 0x7c,0x04,0x0a,0x14
+# GFX11: v_mul_hi_i32_i24_e32 v5, null, v2       ; encoding: [0x7c,0x04,0x0a,0x14]
 
-# GFX11: v_mul_hi_i32_i24_e32 v5, -1, v2         ; encoding: [0xc1,0x04,0x0a,0x14]
 0xc1,0x04,0x0a,0x14
+# GFX11: v_mul_hi_i32_i24_e32 v5, -1, v2         ; encoding: [0xc1,0x04,0x0a,0x14]
 
-# GFX11: v_mul_hi_i32_i24_e32 v5, 0.5, v2        ; encoding: [0xf0,0x04,0x0a,0x14]
 0xf0,0x04,0x0a,0x14
+# GFX11: v_mul_hi_i32_i24_e32 v5, 0.5, v2        ; encoding: [0xf0,0x04,0x0a,0x14]
 
-# GFX11: v_mul_hi_i32_i24_e32 v5, src_scc, v2    ; encoding: [0xfd,0x04,0x0a,0x14]
 0xfd,0x04,0x0a,0x14
+# GFX11: v_mul_hi_i32_i24_e32 v5, src_scc, v2    ; encoding: [0xfd,0x04,0x0a,0x14]
 
-# GFX11: v_mul_hi_i32_i24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x15,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x15,0x56,0x34,0x12,0xaf
+# GFX11: v_mul_hi_i32_i24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x15,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mul_hi_u32_u24_e32 v5, v1, v2         ; encoding: [0x01,0x05,0x0a,0x18]
 0x01,0x05,0x0a,0x18
+# GFX11: v_mul_hi_u32_u24_e32 v5, v1, v2         ; encoding: [0x01,0x05,0x0a,0x18]
 
-# GFX11: v_mul_hi_u32_u24_e32 v5, v255, v2       ; encoding: [0xff,0x05,0x0a,0x18]
 0xff,0x05,0x0a,0x18
+# GFX11: v_mul_hi_u32_u24_e32 v5, v255, v2       ; encoding: [0xff,0x05,0x0a,0x18]
 
-# GFX11: v_mul_hi_u32_u24_e32 v5, s1, v2         ; encoding: [0x01,0x04,0x0a,0x18]
 0x01,0x04,0x0a,0x18
+# GFX11: v_mul_hi_u32_u24_e32 v5, s1, v2         ; encoding: [0x01,0x04,0x0a,0x18]
 
-# GFX11: v_mul_hi_u32_u24_e32 v5, s105, v2       ; encoding: [0x69,0x04,0x0a,0x18]
 0x69,0x04,0x0a,0x18
+# GFX11: v_mul_hi_u32_u24_e32 v5, s105, v2       ; encoding: [0x69,0x04,0x0a,0x18]
 
-# GFX11: v_mul_hi_u32_u24_e32 v5, vcc_lo, v2     ; encoding: [0x6a,0x04,0x0a,0x18]
 0x6a,0x04,0x0a,0x18
+# GFX11: v_mul_hi_u32_u24_e32 v5, vcc_lo, v2     ; encoding: [0x6a,0x04,0x0a,0x18]
 
-# GFX11: v_mul_hi_u32_u24_e32 v5, vcc_hi, v2     ; encoding: [0x6b,0x04,0x0a,0x18]
 0x6b,0x04,0x0a,0x18
+# GFX11: v_mul_hi_u32_u24_e32 v5, vcc_hi, v2     ; encoding: [0x6b,0x04,0x0a,0x18]
 
-# GFX11: v_mul_hi_u32_u24_e32 v5, ttmp15, v2     ; encoding: [0x7b,0x04,0x0a,0x18]
 0x7b,0x04,0x0a,0x18
+# GFX11: v_mul_hi_u32_u24_e32 v5, ttmp15, v2     ; encoding: [0x7b,0x04,0x0a,0x18]
 
-# GFX11: v_mul_hi_u32_u24_e32 v5, m0, v2         ; encoding: [0x7d,0x04,0x0a,0x18]
 0x7d,0x04,0x0a,0x18
+# GFX11: v_mul_hi_u32_u24_e32 v5, m0, v2         ; encoding: [0x7d,0x04,0x0a,0x18]
 
-# GFX11: v_mul_hi_u32_u24_e32 v5, exec_lo, v2    ; encoding: [0x7e,0x04,0x0a,0x18]
 0x7e,0x04,0x0a,0x18
+# GFX11: v_mul_hi_u32_u24_e32 v5, exec_lo, v2    ; encoding: [0x7e,0x04,0x0a,0x18]
 
-# GFX11: v_mul_hi_u32_u24_e32 v5, exec_hi, v2    ; encoding: [0x7f,0x04,0x0a,0x18]
 0x7f,0x04,0x0a,0x18
+# GFX11: v_mul_hi_u32_u24_e32 v5, exec_hi, v2    ; encoding: [0x7f,0x04,0x0a,0x18]
 
-# GFX11: v_mul_hi_u32_u24_e32 v5, null, v2       ; encoding: [0x7c,0x04,0x0a,0x18]
 0x7c,0x04,0x0a,0x18
+# GFX11: v_mul_hi_u32_u24_e32 v5, null, v2       ; encoding: [0x7c,0x04,0x0a,0x18]
 
-# GFX11: v_mul_hi_u32_u24_e32 v5, -1, v2         ; encoding: [0xc1,0x04,0x0a,0x18]
 0xc1,0x04,0x0a,0x18
+# GFX11: v_mul_hi_u32_u24_e32 v5, -1, v2         ; encoding: [0xc1,0x04,0x0a,0x18]
 
-# GFX11: v_mul_hi_u32_u24_e32 v5, 0.5, v2        ; encoding: [0xf0,0x04,0x0a,0x18]
 0xf0,0x04,0x0a,0x18
+# GFX11: v_mul_hi_u32_u24_e32 v5, 0.5, v2        ; encoding: [0xf0,0x04,0x0a,0x18]
 
-# GFX11: v_mul_hi_u32_u24_e32 v5, src_scc, v2    ; encoding: [0xfd,0x04,0x0a,0x18]
 0xfd,0x04,0x0a,0x18
+# GFX11: v_mul_hi_u32_u24_e32 v5, src_scc, v2    ; encoding: [0xfd,0x04,0x0a,0x18]
 
-# GFX11: v_mul_hi_u32_u24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x19,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x19,0x56,0x34,0x12,0xaf
+# GFX11: v_mul_hi_u32_u24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x19,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mul_i32_i24_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x12]
 0x01,0x05,0x0a,0x12
+# GFX11: v_mul_i32_i24_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x12]
 
-# GFX11: v_mul_i32_i24_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x12]
 0xff,0x05,0x0a,0x12
+# GFX11: v_mul_i32_i24_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x12]
 
-# GFX11: v_mul_i32_i24_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x12]
 0x01,0x04,0x0a,0x12
+# GFX11: v_mul_i32_i24_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x12]
 
-# GFX11: v_mul_i32_i24_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x12]
 0x69,0x04,0x0a,0x12
+# GFX11: v_mul_i32_i24_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x12]
 
-# GFX11: v_mul_i32_i24_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x12]
 0x6a,0x04,0x0a,0x12
+# GFX11: v_mul_i32_i24_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x12]
 
-# GFX11: v_mul_i32_i24_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x12]
 0x6b,0x04,0x0a,0x12
+# GFX11: v_mul_i32_i24_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x12]
 
-# GFX11: v_mul_i32_i24_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x12]
 0x7b,0x04,0x0a,0x12
+# GFX11: v_mul_i32_i24_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x12]
 
-# GFX11: v_mul_i32_i24_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x12]
 0x7d,0x04,0x0a,0x12
+# GFX11: v_mul_i32_i24_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x12]
 
-# GFX11: v_mul_i32_i24_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x12]
 0x7e,0x04,0x0a,0x12
+# GFX11: v_mul_i32_i24_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x12]
 
-# GFX11: v_mul_i32_i24_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x12]
 0x7f,0x04,0x0a,0x12
+# GFX11: v_mul_i32_i24_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x12]
 
-# GFX11: v_mul_i32_i24_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x12]
 0x7c,0x04,0x0a,0x12
+# GFX11: v_mul_i32_i24_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x12]
 
-# GFX11: v_mul_i32_i24_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x12]
 0xc1,0x04,0x0a,0x12
+# GFX11: v_mul_i32_i24_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x12]
 
-# GFX11: v_mul_i32_i24_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x12]
 0xf0,0x04,0x0a,0x12
+# GFX11: v_mul_i32_i24_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x12]
 
-# GFX11: v_mul_i32_i24_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x12]
 0xfd,0x04,0x0a,0x12
+# GFX11: v_mul_i32_i24_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x12]
 
-# GFX11: v_mul_i32_i24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x13,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x13,0x56,0x34,0x12,0xaf
+# GFX11: v_mul_i32_i24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x13,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mul_u32_u24_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x16]
 0x01,0x05,0x0a,0x16
+# GFX11: v_mul_u32_u24_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x16]
 
-# GFX11: v_mul_u32_u24_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x16]
 0xff,0x05,0x0a,0x16
+# GFX11: v_mul_u32_u24_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x16]
 
-# GFX11: v_mul_u32_u24_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x16]
 0x01,0x04,0x0a,0x16
+# GFX11: v_mul_u32_u24_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x16]
 
-# GFX11: v_mul_u32_u24_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x16]
 0x69,0x04,0x0a,0x16
+# GFX11: v_mul_u32_u24_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x16]
 
-# GFX11: v_mul_u32_u24_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x16]
 0x6a,0x04,0x0a,0x16
+# GFX11: v_mul_u32_u24_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x16]
 
-# GFX11: v_mul_u32_u24_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x16]
 0x6b,0x04,0x0a,0x16
+# GFX11: v_mul_u32_u24_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x16]
 
-# GFX11: v_mul_u32_u24_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x16]
 0x7b,0x04,0x0a,0x16
+# GFX11: v_mul_u32_u24_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x16]
 
-# GFX11: v_mul_u32_u24_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x16]
 0x7d,0x04,0x0a,0x16
+# GFX11: v_mul_u32_u24_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x16]
 
-# GFX11: v_mul_u32_u24_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x16]
 0x7e,0x04,0x0a,0x16
+# GFX11: v_mul_u32_u24_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x16]
 
-# GFX11: v_mul_u32_u24_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x16]
 0x7f,0x04,0x0a,0x16
+# GFX11: v_mul_u32_u24_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x16]
 
-# GFX11: v_mul_u32_u24_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x16]
 0x7c,0x04,0x0a,0x16
+# GFX11: v_mul_u32_u24_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x16]
 
-# GFX11: v_mul_u32_u24_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x16]
 0xc1,0x04,0x0a,0x16
+# GFX11: v_mul_u32_u24_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x16]
 
-# GFX11: v_mul_u32_u24_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x16]
 0xf0,0x04,0x0a,0x16
+# GFX11: v_mul_u32_u24_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x16]
 
-# GFX11: v_mul_u32_u24_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x16]
 0xfd,0x04,0x0a,0x16
+# GFX11: v_mul_u32_u24_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x16]
 
-# GFX11: v_mul_u32_u24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x17,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x17,0x56,0x34,0x12,0xaf
+# GFX11: v_mul_u32_u24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x17,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_or_b32_e32 v5, v1, v2                 ; encoding: [0x01,0x05,0x0a,0x38]
 0x01,0x05,0x0a,0x38
+# GFX11: v_or_b32_e32 v5, v1, v2                 ; encoding: [0x01,0x05,0x0a,0x38]
 
-# GFX11: v_or_b32_e32 v5, v255, v2               ; encoding: [0xff,0x05,0x0a,0x38]
 0xff,0x05,0x0a,0x38
+# GFX11: v_or_b32_e32 v5, v255, v2               ; encoding: [0xff,0x05,0x0a,0x38]
 
-# GFX11: v_or_b32_e32 v5, s1, v2                 ; encoding: [0x01,0x04,0x0a,0x38]
 0x01,0x04,0x0a,0x38
+# GFX11: v_or_b32_e32 v5, s1, v2                 ; encoding: [0x01,0x04,0x0a,0x38]
 
-# GFX11: v_or_b32_e32 v5, s105, v2               ; encoding: [0x69,0x04,0x0a,0x38]
 0x69,0x04,0x0a,0x38
+# GFX11: v_or_b32_e32 v5, s105, v2               ; encoding: [0x69,0x04,0x0a,0x38]
 
-# GFX11: v_or_b32_e32 v5, vcc_lo, v2             ; encoding: [0x6a,0x04,0x0a,0x38]
 0x6a,0x04,0x0a,0x38
+# GFX11: v_or_b32_e32 v5, vcc_lo, v2             ; encoding: [0x6a,0x04,0x0a,0x38]
 
-# GFX11: v_or_b32_e32 v5, vcc_hi, v2             ; encoding: [0x6b,0x04,0x0a,0x38]
 0x6b,0x04,0x0a,0x38
+# GFX11: v_or_b32_e32 v5, vcc_hi, v2             ; encoding: [0x6b,0x04,0x0a,0x38]
 
-# GFX11: v_or_b32_e32 v5, ttmp15, v2             ; encoding: [0x7b,0x04,0x0a,0x38]
 0x7b,0x04,0x0a,0x38
+# GFX11: v_or_b32_e32 v5, ttmp15, v2             ; encoding: [0x7b,0x04,0x0a,0x38]
 
-# GFX11: v_or_b32_e32 v5, m0, v2                 ; encoding: [0x7d,0x04,0x0a,0x38]
 0x7d,0x04,0x0a,0x38
+# GFX11: v_or_b32_e32 v5, m0, v2                 ; encoding: [0x7d,0x04,0x0a,0x38]
 
-# GFX11: v_or_b32_e32 v5, exec_lo, v2            ; encoding: [0x7e,0x04,0x0a,0x38]
 0x7e,0x04,0x0a,0x38
+# GFX11: v_or_b32_e32 v5, exec_lo, v2            ; encoding: [0x7e,0x04,0x0a,0x38]
 
-# GFX11: v_or_b32_e32 v5, exec_hi, v2            ; encoding: [0x7f,0x04,0x0a,0x38]
 0x7f,0x04,0x0a,0x38
+# GFX11: v_or_b32_e32 v5, exec_hi, v2            ; encoding: [0x7f,0x04,0x0a,0x38]
 
-# GFX11: v_or_b32_e32 v5, null, v2               ; encoding: [0x7c,0x04,0x0a,0x38]
 0x7c,0x04,0x0a,0x38
+# GFX11: v_or_b32_e32 v5, null, v2               ; encoding: [0x7c,0x04,0x0a,0x38]
 
-# GFX11: v_or_b32_e32 v5, -1, v2                 ; encoding: [0xc1,0x04,0x0a,0x38]
 0xc1,0x04,0x0a,0x38
+# GFX11: v_or_b32_e32 v5, -1, v2                 ; encoding: [0xc1,0x04,0x0a,0x38]
 
-# GFX11: v_or_b32_e32 v5, 0.5, v2                ; encoding: [0xf0,0x04,0x0a,0x38]
 0xf0,0x04,0x0a,0x38
+# GFX11: v_or_b32_e32 v5, 0.5, v2                ; encoding: [0xf0,0x04,0x0a,0x38]
 
-# GFX11: v_or_b32_e32 v5, src_scc, v2            ; encoding: [0xfd,0x04,0x0a,0x38]
 0xfd,0x04,0x0a,0x38
+# GFX11: v_or_b32_e32 v5, src_scc, v2            ; encoding: [0xfd,0x04,0x0a,0x38]
 
-# GFX11: v_or_b32_e32 v255, 0xaf123456, v255     ; encoding: [0xff,0xfe,0xff,0x39,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x39,0x56,0x34,0x12,0xaf
+# GFX11: v_or_b32_e32 v255, 0xaf123456, v255     ; encoding: [0xff,0xfe,0xff,0x39,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_pk_fmac_f16 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x78]
 0x01,0x05,0x0a,0x78
+# GFX11: v_pk_fmac_f16 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x78]
 
-# GFX11: v_pk_fmac_f16 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x78]
 0xff,0x05,0x0a,0x78
+# GFX11: v_pk_fmac_f16 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x78]
 
-# GFX11: v_pk_fmac_f16 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x78]
 0x01,0x04,0x0a,0x78
+# GFX11: v_pk_fmac_f16 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x78]
 
-# GFX11: v_pk_fmac_f16 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x78]
 0x69,0x04,0x0a,0x78
+# GFX11: v_pk_fmac_f16 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x78]
 
-# GFX11: v_pk_fmac_f16 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x78]
 0x6a,0x04,0x0a,0x78
+# GFX11: v_pk_fmac_f16 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x78]
 
-# GFX11: v_pk_fmac_f16 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x78]
 0x6b,0x04,0x0a,0x78
+# GFX11: v_pk_fmac_f16 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x78]
 
-# GFX11: v_pk_fmac_f16 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x78]
 0x7b,0x04,0x0a,0x78
+# GFX11: v_pk_fmac_f16 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x78]
 
-# GFX11: v_pk_fmac_f16 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x78]
 0x7d,0x04,0x0a,0x78
+# GFX11: v_pk_fmac_f16 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x78]
 
-# GFX11: v_pk_fmac_f16 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x78]
 0x7e,0x04,0x0a,0x78
+# GFX11: v_pk_fmac_f16 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x78]
 
-# GFX11: v_pk_fmac_f16 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x78]
 0x7f,0x04,0x0a,0x78
+# GFX11: v_pk_fmac_f16 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x78]
 
-# GFX11: v_pk_fmac_f16 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x78]
 0x7c,0x04,0x0a,0x78
+# GFX11: v_pk_fmac_f16 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x78]
 
-# GFX11: v_pk_fmac_f16 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x78]
 0xc1,0x04,0x0a,0x78
+# GFX11: v_pk_fmac_f16 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x78]
 
-# GFX11: v_pk_fmac_f16 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x78]
 0xf0,0x04,0x0a,0x78
+# GFX11: v_pk_fmac_f16 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x78]
 
-# GFX11: v_pk_fmac_f16 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x78]
 0xfd,0x04,0x0a,0x78
+# GFX11: v_pk_fmac_f16 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x78]
 
-# GFX11: v_pk_fmac_f16 v255, 0xfe0b, v255        ; encoding: [0xff,0xfe,0xff,0x79,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xff,0x79,0x0b,0xfe,0x00,0x00
+# GFX11: v_pk_fmac_f16 v255, 0xfe0b, v255        ; encoding: [0xff,0xfe,0xff,0x79,0x0b,0xfe,0x00,0x00]
 
+0x01,0x05,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, v1, v2, vcc_lo ; encoding: [0x01,0x05,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, v1, v2, vcc ; encoding: [0x01,0x05,0x0a,0x42]
-0x01,0x05,0x0a,0x42
 
+0xff,0x05,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, v255, v2, vcc_lo ; encoding: [0xff,0x05,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, v255, v2, vcc ; encoding: [0xff,0x05,0x0a,0x42]
-0xff,0x05,0x0a,0x42
 
+0x01,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, s1, v2, vcc_lo ; encoding: [0x01,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, s1, v2, vcc ; encoding: [0x01,0x04,0x0a,0x42]
-0x01,0x04,0x0a,0x42
 
+0x69,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, s105, v2, vcc_lo ; encoding: [0x69,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, s105, v2, vcc ; encoding: [0x69,0x04,0x0a,0x42]
-0x69,0x04,0x0a,0x42
 
+0x6a,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, vcc_lo, v2, vcc_lo ; encoding: [0x6a,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, vcc_lo, v2, vcc ; encoding: [0x6a,0x04,0x0a,0x42]
-0x6a,0x04,0x0a,0x42
 
+0x6b,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, vcc_hi, v2, vcc_lo ; encoding: [0x6b,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, vcc_hi, v2, vcc ; encoding: [0x6b,0x04,0x0a,0x42]
-0x6b,0x04,0x0a,0x42
 
+0x7b,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, ttmp15, v2, vcc_lo ; encoding: [0x7b,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, ttmp15, v2, vcc ; encoding: [0x7b,0x04,0x0a,0x42]
-0x7b,0x04,0x0a,0x42
 
+0x7d,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, m0, v2, vcc_lo ; encoding: [0x7d,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, m0, v2, vcc ; encoding: [0x7d,0x04,0x0a,0x42]
-0x7d,0x04,0x0a,0x42
 
+0x7e,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, exec_lo, v2, vcc_lo ; encoding: [0x7e,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, exec_lo, v2, vcc ; encoding: [0x7e,0x04,0x0a,0x42]
-0x7e,0x04,0x0a,0x42
 
+0x7f,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, exec_hi, v2, vcc_lo ; encoding: [0x7f,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, exec_hi, v2, vcc ; encoding: [0x7f,0x04,0x0a,0x42]
-0x7f,0x04,0x0a,0x42
 
+0x7c,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, null, v2, vcc_lo ; encoding: [0x7c,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, null, v2, vcc ; encoding: [0x7c,0x04,0x0a,0x42]
-0x7c,0x04,0x0a,0x42
 
+0xc1,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, -1, v2, vcc_lo ; encoding: [0xc1,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, -1, v2, vcc ; encoding: [0xc1,0x04,0x0a,0x42]
-0xc1,0x04,0x0a,0x42
 
+0xf0,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, 0.5, v2, vcc_lo ; encoding: [0xf0,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, 0.5, v2, vcc ; encoding: [0xf0,0x04,0x0a,0x42]
-0xf0,0x04,0x0a,0x42
 
+0xfd,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, src_scc, v2, vcc_lo ; encoding: [0xfd,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, src_scc, v2, vcc ; encoding: [0xfd,0x04,0x0a,0x42]
-0xfd,0x04,0x0a,0x42
 
+0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf
 # W32: v_sub_co_ci_u32_e32 v255, vcc_lo, 0xaf123456, v255, vcc_lo ; encoding: [0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf]
 # W64: v_sub_co_ci_u32_e32 v255, vcc, 0xaf123456, v255, vcc ; encoding: [0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf]
-0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf
 
+0x01,0x05,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, v1.l, v2.l          ; encoding: [0x01,0x05,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x66]
-0x01,0x05,0x0a,0x66
 
+0x81,0x05,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, v1.h, v2.l          ; encoding: [0x81,0x05,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x66]
-0x81,0x05,0x0a,0x66
 
+0x7f,0x05,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, v127.l, v2.l        ; encoding: [0x7f,0x05,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x66]
-0x7f,0x05,0x0a,0x66
 
+0xff,0x05,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, v127.h, v2.l        ; encoding: [0xff,0x05,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x66]
-0xff,0x05,0x0a,0x66
 
+0x01,0x04,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, s1, v2.l            ; encoding: [0x01,0x04,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x66]
-0x01,0x04,0x0a,0x66
 
+0x69,0x04,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, s105, v2.l          ; encoding: [0x69,0x04,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x66]
-0x69,0x04,0x0a,0x66
 
+0x6a,0x04,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, vcc_lo, v2.l        ; encoding: [0x6a,0x04,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x66]
-0x6a,0x04,0x0a,0x66
 
+0x6b,0x04,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, vcc_hi, v2.l        ; encoding: [0x6b,0x04,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x66]
-0x6b,0x04,0x0a,0x66
 
+0x7b,0x04,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, ttmp15, v2.l        ; encoding: [0x7b,0x04,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x66]
-0x7b,0x04,0x0a,0x66
 
+0x7d,0x04,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, m0, v2.l            ; encoding: [0x7d,0x04,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x66]
-0x7d,0x04,0x0a,0x66
 
+0x7e,0x04,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, exec_lo, v2.l       ; encoding: [0x7e,0x04,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x66]
-0x7e,0x04,0x0a,0x66
 
+0x7f,0x04,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, exec_hi, v2.l       ; encoding: [0x7f,0x04,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x66]
-0x7f,0x04,0x0a,0x66
 
+0x7c,0x04,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, null, v2.l          ; encoding: [0x7c,0x04,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x66]
-0x7c,0x04,0x0a,0x66
 
+0xc1,0x04,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, -1, v2.l            ; encoding: [0xc1,0x04,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x66]
-0xc1,0x04,0x0a,0x66
 
+0xf0,0x04,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, 0.5, v2.l           ; encoding: [0xf0,0x04,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x66]
-0xf0,0x04,0x0a,0x66
 
+0xfd,0x04,0x0a,0x66
 # GFX11-REAL16: v_sub_f16_e32 v5.l, src_scc, v2.l       ; encoding: [0xfd,0x04,0x0a,0x66]
 # GFX11-FAKE16: v_sub_f16_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x66]
-0xfd,0x04,0x0a,0x66
 
-# GFX11-REAL16: v_sub_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x67]
-# COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xfd,0x04,0x0b,0x67
 0xfd,0x04,0x0b,0x67
+# GFX11-REAL16: v_sub_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x67]
 
+0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00
 # GFX11-REAL16: v_sub_f16_e32 v127.l, 0xfe0b, v127.l    ; encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00]
 # GFX11-FAKE16: v_sub_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00]
-0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00
 
-# GFX11-REAL16: v_sub_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x67,0x0b,0xfe,0x00,0x00]
-# COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xff,0xfe,0xff,0x67,0x0b,0xfe,0x00,0x00
 0xff,0xfe,0xff,0x67,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_sub_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x67,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_sub_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x08]
 0x01,0x05,0x0a,0x08
+# GFX11: v_sub_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x08]
 
-# GFX11: v_sub_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x08]
 0xff,0x05,0x0a,0x08
+# GFX11: v_sub_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x08]
 
-# GFX11: v_sub_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x08]
 0x01,0x04,0x0a,0x08
+# GFX11: v_sub_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x08]
 
-# GFX11: v_sub_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x08]
 0x69,0x04,0x0a,0x08
+# GFX11: v_sub_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x08]
 
-# GFX11: v_sub_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x08]
 0x6a,0x04,0x0a,0x08
+# GFX11: v_sub_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x08]
 
-# GFX11: v_sub_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x08]
 0x6b,0x04,0x0a,0x08
+# GFX11: v_sub_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x08]
 
-# GFX11: v_sub_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x08]
 0x7b,0x04,0x0a,0x08
+# GFX11: v_sub_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x08]
 
-# GFX11: v_sub_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x08]
 0x7d,0x04,0x0a,0x08
+# GFX11: v_sub_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x08]
 
-# GFX11: v_sub_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x08]
 0x7e,0x04,0x0a,0x08
+# GFX11: v_sub_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x08]
 
-# GFX11: v_sub_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x08]
 0x7f,0x04,0x0a,0x08
+# GFX11: v_sub_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x08]
 
-# GFX11: v_sub_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x08]
 0x7c,0x04,0x0a,0x08
+# GFX11: v_sub_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x08]
 
-# GFX11: v_sub_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x08]
 0xc1,0x04,0x0a,0x08
+# GFX11: v_sub_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x08]
 
-# GFX11: v_sub_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x08]
 0xf0,0x04,0x0a,0x08
+# GFX11: v_sub_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x08]
 
-# GFX11: v_sub_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x08]
 0xfd,0x04,0x0a,0x08
+# GFX11: v_sub_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x08]
 
-# GFX11: v_sub_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x09,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x09,0x56,0x34,0x12,0xaf
+# GFX11: v_sub_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x09,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_sub_nc_u32_e32 v5, v1, v2             ; encoding: [0x01,0x05,0x0a,0x4c]
 0x01,0x05,0x0a,0x4c
+# GFX11: v_sub_nc_u32_e32 v5, v1, v2             ; encoding: [0x01,0x05,0x0a,0x4c]
 
-# GFX11: v_sub_nc_u32_e32 v5, v255, v2           ; encoding: [0xff,0x05,0x0a,0x4c]
 0xff,0x05,0x0a,0x4c
+# GFX11: v_sub_nc_u32_e32 v5, v255, v2           ; encoding: [0xff,0x05,0x0a,0x4c]
 
-# GFX11: v_sub_nc_u32_e32 v5, s1, v2             ; encoding: [0x01,0x04,0x0a,0x4c]
 0x01,0x04,0x0a,0x4c
+# GFX11: v_sub_nc_u32_e32 v5, s1, v2             ; encoding: [0x01,0x04,0x0a,0x4c]
 
-# GFX11: v_sub_nc_u32_e32 v5, s105, v2           ; encoding: [0x69,0x04,0x0a,0x4c]
 0x69,0x04,0x0a,0x4c
+# GFX11: v_sub_nc_u32_e32 v5, s105, v2           ; encoding: [0x69,0x04,0x0a,0x4c]
 
-# GFX11: v_sub_nc_u32_e32 v5, vcc_lo, v2         ; encoding: [0x6a,0x04,0x0a,0x4c]
 0x6a,0x04,0x0a,0x4c
+# GFX11: v_sub_nc_u32_e32 v5, vcc_lo, v2         ; encoding: [0x6a,0x04,0x0a,0x4c]
 
-# GFX11: v_sub_nc_u32_e32 v5, vcc_hi, v2         ; encoding: [0x6b,0x04,0x0a,0x4c]
 0x6b,0x04,0x0a,0x4c
+# GFX11: v_sub_nc_u32_e32 v5, vcc_hi, v2         ; encoding: [0x6b,0x04,0x0a,0x4c]
 
-# GFX11: v_sub_nc_u32_e32 v5, ttmp15, v2         ; encoding: [0x7b,0x04,0x0a,0x4c]
 0x7b,0x04,0x0a,0x4c
+# GFX11: v_sub_nc_u32_e32 v5, ttmp15, v2         ; encoding: [0x7b,0x04,0x0a,0x4c]
 
-# GFX11: v_sub_nc_u32_e32 v5, m0, v2             ; encoding: [0x7d,0x04,0x0a,0x4c]
 0x7d,0x04,0x0a,0x4c
+# GFX11: v_sub_nc_u32_e32 v5, m0, v2             ; encoding: [0x7d,0x04,0x0a,0x4c]
 
-# GFX11: v_sub_nc_u32_e32 v5, exec_lo, v2        ; encoding: [0x7e,0x04,0x0a,0x4c]
 0x7e,0x04,0x0a,0x4c
+# GFX11: v_sub_nc_u32_e32 v5, exec_lo, v2        ; encoding: [0x7e,0x04,0x0a,0x4c]
 
-# GFX11: v_sub_nc_u32_e32 v5, exec_hi, v2        ; encoding: [0x7f,0x04,0x0a,0x4c]
 0x7f,0x04,0x0a,0x4c
+# GFX11: v_sub_nc_u32_e32 v5, exec_hi, v2        ; encoding: [0x7f,0x04,0x0a,0x4c]
 
-# GFX11: v_sub_nc_u32_e32 v5, null, v2           ; encoding: [0x7c,0x04,0x0a,0x4c]
 0x7c,0x04,0x0a,0x4c
+# GFX11: v_sub_nc_u32_e32 v5, null, v2           ; encoding: [0x7c,0x04,0x0a,0x4c]
 
-# GFX11: v_sub_nc_u32_e32 v5, -1, v2             ; encoding: [0xc1,0x04,0x0a,0x4c]
 0xc1,0x04,0x0a,0x4c
+# GFX11: v_sub_nc_u32_e32 v5, -1, v2             ; encoding: [0xc1,0x04,0x0a,0x4c]
 
-# GFX11: v_sub_nc_u32_e32 v5, 0.5, v2            ; encoding: [0xf0,0x04,0x0a,0x4c]
 0xf0,0x04,0x0a,0x4c
+# GFX11: v_sub_nc_u32_e32 v5, 0.5, v2            ; encoding: [0xf0,0x04,0x0a,0x4c]
 
-# GFX11: v_sub_nc_u32_e32 v5, src_scc, v2        ; encoding: [0xfd,0x04,0x0a,0x4c]
 0xfd,0x04,0x0a,0x4c
+# GFX11: v_sub_nc_u32_e32 v5, src_scc, v2        ; encoding: [0xfd,0x04,0x0a,0x4c]
 
-# GFX11: v_sub_nc_u32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x4d,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x4d,0x56,0x34,0x12,0xaf
+# GFX11: v_sub_nc_u32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x4d,0x56,0x34,0x12,0xaf]
 
+0x01,0x05,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, v1, v2, vcc_lo ; encoding: [0x01,0x05,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, v1, v2, vcc ; encoding: [0x01,0x05,0x0a,0x44]
-0x01,0x05,0x0a,0x44
 
+0xff,0x05,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, v255, v2, vcc_lo ; encoding: [0xff,0x05,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, v255, v2, vcc ; encoding: [0xff,0x05,0x0a,0x44]
-0xff,0x05,0x0a,0x44
 
+0x01,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, s1, v2, vcc_lo ; encoding: [0x01,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, s1, v2, vcc ; encoding: [0x01,0x04,0x0a,0x44]
-0x01,0x04,0x0a,0x44
 
+0x69,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, s105, v2, vcc_lo ; encoding: [0x69,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, s105, v2, vcc ; encoding: [0x69,0x04,0x0a,0x44]
-0x69,0x04,0x0a,0x44
 
+0x6a,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, vcc_lo, v2, vcc_lo ; encoding: [0x6a,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, vcc_lo, v2, vcc ; encoding: [0x6a,0x04,0x0a,0x44]
-0x6a,0x04,0x0a,0x44
 
+0x6b,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, vcc_hi, v2, vcc_lo ; encoding: [0x6b,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, vcc_hi, v2, vcc ; encoding: [0x6b,0x04,0x0a,0x44]
-0x6b,0x04,0x0a,0x44
 
+0x7b,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, ttmp15, v2, vcc_lo ; encoding: [0x7b,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, ttmp15, v2, vcc ; encoding: [0x7b,0x04,0x0a,0x44]
-0x7b,0x04,0x0a,0x44
 
+0x7d,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, m0, v2, vcc_lo ; encoding: [0x7d,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, m0, v2, vcc ; encoding: [0x7d,0x04,0x0a,0x44]
-0x7d,0x04,0x0a,0x44
 
+0x7e,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, exec_lo, v2, vcc_lo ; encoding: [0x7e,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, exec_lo, v2, vcc ; encoding: [0x7e,0x04,0x0a,0x44]
-0x7e,0x04,0x0a,0x44
 
+0x7f,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, exec_hi, v2, vcc_lo ; encoding: [0x7f,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, exec_hi, v2, vcc ; encoding: [0x7f,0x04,0x0a,0x44]
-0x7f,0x04,0x0a,0x44
 
+0x7c,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, null, v2, vcc_lo ; encoding: [0x7c,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, null, v2, vcc ; encoding: [0x7c,0x04,0x0a,0x44]
-0x7c,0x04,0x0a,0x44
 
+0xc1,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, -1, v2, vcc_lo ; encoding: [0xc1,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, -1, v2, vcc ; encoding: [0xc1,0x04,0x0a,0x44]
-0xc1,0x04,0x0a,0x44
 
+0xf0,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, 0.5, v2, vcc_lo ; encoding: [0xf0,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, 0.5, v2, vcc ; encoding: [0xf0,0x04,0x0a,0x44]
-0xf0,0x04,0x0a,0x44
 
+0xfd,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, src_scc, v2, vcc_lo ; encoding: [0xfd,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, src_scc, v2, vcc ; encoding: [0xfd,0x04,0x0a,0x44]
-0xfd,0x04,0x0a,0x44
 
+0xff,0xfe,0xff,0x45,0x56,0x34,0x12,0xaf
 # W32: v_subrev_co_ci_u32_e32 v255, vcc_lo, 0xaf123456, v255, vcc_lo ; encoding: [0xff,0xfe,0xff,0x45,0x56,0x34,0x12,0xaf]
 # W64: v_subrev_co_ci_u32_e32 v255, vcc, 0xaf123456, v255, vcc ; encoding: [0xff,0xfe,0xff,0x45,0x56,0x34,0x12,0xaf]
-0xff,0xfe,0xff,0x45,0x56,0x34,0x12,0xaf
 
+0x01,0x05,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, v1.l, v2.l       ; encoding: [0x01,0x05,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, v1, v2             ; encoding: [0x01,0x05,0x0a,0x68]
-0x01,0x05,0x0a,0x68
 
+0x81,0x05,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, v1.h, v2.l       ; encoding: [0x81,0x05,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x68]
-0x81,0x05,0x0a,0x68
 
+0x7f,0x05,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, v127.l, v2.l     ; encoding: [0x7f,0x05,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, v127, v2           ; encoding: [0x7f,0x05,0x0a,0x68]
-0x7f,0x05,0x0a,0x68
 
+0xff,0x05,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, v127.h, v2.l     ; encoding: [0xff,0x05,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x68]
-0xff,0x05,0x0a,0x68
 
+0x01,0x04,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, s1, v2.l         ; encoding: [0x01,0x04,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, s1, v2             ; encoding: [0x01,0x04,0x0a,0x68]
-0x01,0x04,0x0a,0x68
 
+0x69,0x04,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, s105, v2.l       ; encoding: [0x69,0x04,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, s105, v2           ; encoding: [0x69,0x04,0x0a,0x68]
-0x69,0x04,0x0a,0x68
 
+0x6a,0x04,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, vcc_lo, v2.l     ; encoding: [0x6a,0x04,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, vcc_lo, v2         ; encoding: [0x6a,0x04,0x0a,0x68]
-0x6a,0x04,0x0a,0x68
 
+0x6b,0x04,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, vcc_hi, v2.l     ; encoding: [0x6b,0x04,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, vcc_hi, v2         ; encoding: [0x6b,0x04,0x0a,0x68]
-0x6b,0x04,0x0a,0x68
 
+0x7b,0x04,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, ttmp15, v2.l     ; encoding: [0x7b,0x04,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, ttmp15, v2         ; encoding: [0x7b,0x04,0x0a,0x68]
-0x7b,0x04,0x0a,0x68
 
+0x7d,0x04,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, m0, v2.l         ; encoding: [0x7d,0x04,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, m0, v2             ; encoding: [0x7d,0x04,0x0a,0x68]
-0x7d,0x04,0x0a,0x68
 
+0x7e,0x04,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, exec_lo, v2.l    ; encoding: [0x7e,0x04,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, exec_lo, v2        ; encoding: [0x7e,0x04,0x0a,0x68]
-0x7e,0x04,0x0a,0x68
 
+0x7f,0x04,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, exec_hi, v2.l    ; encoding: [0x7f,0x04,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, exec_hi, v2        ; encoding: [0x7f,0x04,0x0a,0x68]
-0x7f,0x04,0x0a,0x68
 
+0x7c,0x04,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, null, v2.l       ; encoding: [0x7c,0x04,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, null, v2           ; encoding: [0x7c,0x04,0x0a,0x68]
-0x7c,0x04,0x0a,0x68
 
+0xc1,0x04,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, -1, v2.l         ; encoding: [0xc1,0x04,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, -1, v2             ; encoding: [0xc1,0x04,0x0a,0x68]
-0xc1,0x04,0x0a,0x68
 
+0xf0,0x04,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, 0.5, v2.l        ; encoding: [0xf0,0x04,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, 0.5, v2            ; encoding: [0xf0,0x04,0x0a,0x68]
-0xf0,0x04,0x0a,0x68
 
+0xfd,0x04,0x0a,0x68
 # GFX11-REAL16: v_subrev_f16_e32 v5.l, src_scc, v2.l    ; encoding: [0xfd,0x04,0x0a,0x68]
 # GFX11-FAKE16: v_subrev_f16_e32 v5, src_scc, v2        ; encoding: [0xfd,0x04,0x0a,0x68]
-0xfd,0x04,0x0a,0x68
 
-# GFX11-REAL16: v_subrev_f16_e32 v5.h, src_scc, v2.h    ; encoding: [0xfd,0x04,0x0b,0x69]
-# COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xfd,0x04,0x0b,0x69
 0xfd,0x04,0x0b,0x69
+# GFX11-REAL16: v_subrev_f16_e32 v5.h, src_scc, v2.h    ; encoding: [0xfd,0x04,0x0b,0x69]
 
+0xff,0xfe,0xfe,0x68,0x0b,0xfe,0x00,0x00
 # GFX11-REAL16: v_subrev_f16_e32 v127.l, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x68,0x0b,0xfe,0x00,0x00]
 # GFX11-FAKE16: v_subrev_f16_e32 v127, 0xfe0b, v127     ; encoding: [0xff,0xfe,0xfe,0x68,0x0b,0xfe,0x00,0x00]
-0xff,0xfe,0xfe,0x68,0x0b,0xfe,0x00,0x00
 
-# GFX11-REAL16: v_subrev_f16_e32 v127.h, 0xfe0b, v127.h ; encoding: [0xff,0xfe,0xff,0x69,0x0b,0xfe,0x00,0x00]
-# COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xff,0xfe,0xff,0x69,0x0b,0xfe,0x00,0x00
 0xff,0xfe,0xff,0x69,0x0b,0xfe,0x00,0x00
+# GFX11-REAL16: v_subrev_f16_e32 v127.h, 0xfe0b, v127.h ; encoding: [0xff,0xfe,0xff,0x69,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_subrev_f32_e32 v5, v1, v2             ; encoding: [0x01,0x05,0x0a,0x0a]
 0x01,0x05,0x0a,0x0a
+# GFX11: v_subrev_f32_e32 v5, v1, v2             ; encoding: [0x01,0x05,0x0a,0x0a]
 
-# GFX11: v_subrev_f32_e32 v5, v255, v2           ; encoding: [0xff,0x05,0x0a,0x0a]
 0xff,0x05,0x0a,0x0a
+# GFX11: v_subrev_f32_e32 v5, v255, v2           ; encoding: [0xff,0x05,0x0a,0x0a]
 
-# GFX11: v_subrev_f32_e32 v5, s1, v2             ; encoding: [0x01,0x04,0x0a,0x0a]
 0x01,0x04,0x0a,0x0a
+# GFX11: v_subrev_f32_e32 v5, s1, v2             ; encoding: [0x01,0x04,0x0a,0x0a]
 
-# GFX11: v_subrev_f32_e32 v5, s105, v2           ; encoding: [0x69,0x04,0x0a,0x0a]
 0x69,0x04,0x0a,0x0a
+# GFX11: v_subrev_f32_e32 v5, s105, v2           ; encoding: [0x69,0x04,0x0a,0x0a]
 
-# GFX11: v_subrev_f32_e32 v5, vcc_lo, v2         ; encoding: [0x6a,0x04,0x0a,0x0a]
 0x6a,0x04,0x0a,0x0a
+# GFX11: v_subrev_f32_e32 v5, vcc_lo, v2         ; encoding: [0x6a,0x04,0x0a,0x0a]
 
-# GFX11: v_subrev_f32_e32 v5, vcc_hi, v2         ; encoding: [0x6b,0x04,0x0a,0x0a]
 0x6b,0x04,0x0a,0x0a
+# GFX11: v_subrev_f32_e32 v5, vcc_hi, v2         ; encoding: [0x6b,0x04,0x0a,0x0a]
 
-# GFX11: v_subrev_f32_e32 v5, ttmp15, v2         ; encoding: [0x7b,0x04,0x0a,0x0a]
 0x7b,0x04,0x0a,0x0a
+# GFX11: v_subrev_f32_e32 v5, ttmp15, v2         ; encoding: [0x7b,0x04,0x0a,0x0a]
 
-# GFX11: v_subrev_f32_e32 v5, m0, v2             ; encoding: [0x7d,0x04,0x0a,0x0a]
 0x7d,0x04,0x0a,0x0a
+# GFX11: v_subrev_f32_e32 v5, m0, v2             ; encoding: [0x7d,0x04,0x0a,0x0a]
 
-# GFX11: v_subrev_f32_e32 v5, exec_lo, v2        ; encoding: [0x7e,0x04,0x0a,0x0a]
 0x7e,0x04,0x0a,0x0a
+# GFX11: v_subrev_f32_e32 v5, exec_lo, v2        ; encoding: [0x7e,0x04,0x0a,0x0a]
 
-# GFX11: v_subrev_f32_e32 v5, exec_hi, v2        ; encoding: [0x7f,0x04,0x0a,0x0a]
 0x7f,0x04,0x0a,0x0a
+# GFX11: v_subrev_f32_e32 v5, exec_hi, v2        ; encoding: [0x7f,0x04,0x0a,0x0a]
 
-# GFX11: v_subrev_f32_e32 v5, null, v2           ; encoding: [0x7c,0x04,0x0a,0x0a]
 0x7c,0x04,0x0a,0x0a
+# GFX11: v_subrev_f32_e32 v5, null, v2           ; encoding: [0x7c,0x04,0x0a,0x0a]
 
-# GFX11: v_subrev_f32_e32 v5, -1, v2             ; encoding: [0xc1,0x04,0x0a,0x0a]
 0xc1,0x04,0x0a,0x0a
+# GFX11: v_subrev_f32_e32 v5, -1, v2             ; encoding: [0xc1,0x04,0x0a,0x0a]
 
-# GFX11: v_subrev_f32_e32 v5, 0.5, v2            ; encoding: [0xf0,0x04,0x0a,0x0a]
 0xf0,0x04,0x0a,0x0a
+# GFX11: v_subrev_f32_e32 v5, 0.5, v2            ; encoding: [0xf0,0x04,0x0a,0x0a]
 
-# GFX11: v_subrev_f32_e32 v5, src_scc, v2        ; encoding: [0xfd,0x04,0x0a,0x0a]
 0xfd,0x04,0x0a,0x0a
+# GFX11: v_subrev_f32_e32 v5, src_scc, v2        ; encoding: [0xfd,0x04,0x0a,0x0a]
 
-# GFX11: v_subrev_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x0b,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x0b,0x56,0x34,0x12,0xaf
+# GFX11: v_subrev_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x0b,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_subrev_nc_u32_e32 v5, v1, v2          ; encoding: [0x01,0x05,0x0a,0x4e]
 0x01,0x05,0x0a,0x4e
+# GFX11: v_subrev_nc_u32_e32 v5, v1, v2          ; encoding: [0x01,0x05,0x0a,0x4e]
 
-# GFX11: v_subrev_nc_u32_e32 v5, v255, v2        ; encoding: [0xff,0x05,0x0a,0x4e]
 0xff,0x05,0x0a,0x4e
+# GFX11: v_subrev_nc_u32_e32 v5, v255, v2        ; encoding: [0xff,0x05,0x0a,0x4e]
 
-# GFX11: v_subrev_nc_u32_e32 v5, s1, v2          ; encoding: [0x01,0x04,0x0a,0x4e]
 0x01,0x04,0x0a,0x4e
+# GFX11: v_subrev_nc_u32_e32 v5, s1, v2          ; encoding: [0x01,0x04,0x0a,0x4e]
 
-# GFX11: v_subrev_nc_u32_e32 v5, s105, v2        ; encoding: [0x69,0x04,0x0a,0x4e]
 0x69,0x04,0x0a,0x4e
+# GFX11: v_subrev_nc_u32_e32 v5, s105, v2        ; encoding: [0x69,0x04,0x0a,0x4e]
 
-# GFX11: v_subrev_nc_u32_e32 v5, vcc_lo, v2      ; encoding: [0x6a,0x04,0x0a,0x4e]
 0x6a,0x04,0x0a,0x4e
+# GFX11: v_subrev_nc_u32_e32 v5, vcc_lo, v2      ; encoding: [0x6a,0x04,0x0a,0x4e]
 
-# GFX11: v_subrev_nc_u32_e32 v5, vcc_hi, v2      ; encoding: [0x6b,0x04,0x0a,0x4e]
 0x6b,0x04,0x0a,0x4e
+# GFX11: v_subrev_nc_u32_e32 v5, vcc_hi, v2      ; encoding: [0x6b,0x04,0x0a,0x4e]
 
-# GFX11: v_subrev_nc_u32_e32 v5, ttmp15, v2      ; encoding: [0x7b,0x04,0x0a,0x4e]
 0x7b,0x04,0x0a,0x4e
+# GFX11: v_subrev_nc_u32_e32 v5, ttmp15, v2      ; encoding: [0x7b,0x04,0x0a,0x4e]
 
-# GFX11: v_subrev_nc_u32_e32 v5, m0, v2          ; encoding: [0x7d,0x04,0x0a,0x4e]
 0x7d,0x04,0x0a,0x4e
+# GFX11: v_subrev_nc_u32_e32 v5, m0, v2          ; encoding: [0x7d,0x04,0x0a,0x4e]
 
-# GFX11: v_subrev_nc_u32_e32 v5, exec_lo, v2     ; encoding: [0x7e,0x04,0x0a,0x4e]
 0x7e,0x04,0x0a,0x4e
+# GFX11: v_subrev_nc_u32_e32 v5, exec_lo, v2     ; encoding: [0x7e,0x04,0x0a,0x4e]
 
-# GFX11: v_subrev_nc_u32_e32 v5, exec_hi, v2     ; encoding: [0x7f,0x04,0x0a,0x4e]
 0x7f,0x04,0x0a,0x4e
+# GFX11: v_subrev_nc_u32_e32 v5, exec_hi, v2     ; encoding: [0x7f,0x04,0x0a,0x4e]
 
-# GFX11: v_subrev_nc_u32_e32 v5, null, v2        ; encoding: [0x7c,0x04,0x0a,0x4e]
 0x7c,0x04,0x0a,0x4e
+# GFX11: v_subrev_nc_u32_e32 v5, null, v2        ; encoding: [0x7c,0x04,0x0a,0x4e]
 
-# GFX11: v_subrev_nc_u32_e32 v5, -1, v2          ; encoding: [0xc1,0x04,0x0a,0x4e]
 0xc1,0x04,0x0a,0x4e
+# GFX11: v_subrev_nc_u32_e32 v5, -1, v2          ; encoding: [0xc1,0x04,0x0a,0x4e]
 
-# GFX11: v_subrev_nc_u32_e32 v5, 0.5, v2         ; encoding: [0xf0,0x04,0x0a,0x4e]
 0xf0,0x04,0x0a,0x4e
+# GFX11: v_subrev_nc_u32_e32 v5, 0.5, v2         ; encoding: [0xf0,0x04,0x0a,0x4e]
 
-# GFX11: v_subrev_nc_u32_e32 v5, src_scc, v2     ; encoding: [0xfd,0x04,0x0a,0x4e]
 0xfd,0x04,0x0a,0x4e
+# GFX11: v_subrev_nc_u32_e32 v5, src_scc, v2     ; encoding: [0xfd,0x04,0x0a,0x4e]
 
-# GFX11: v_subrev_nc_u32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x4f,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x4f,0x56,0x34,0x12,0xaf
+# GFX11: v_subrev_nc_u32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x4f,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_xnor_b32_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x3c]
 0x01,0x05,0x0a,0x3c
+# GFX11: v_xnor_b32_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x3c]
 
-# GFX11: v_xnor_b32_e32 v5, v255, v2             ; encoding: [0xff,0x05,0x0a,0x3c]
 0xff,0x05,0x0a,0x3c
+# GFX11: v_xnor_b32_e32 v5, v255, v2             ; encoding: [0xff,0x05,0x0a,0x3c]
 
-# GFX11: v_xnor_b32_e32 v5, s1, v2               ; encoding: [0x01,0x04,0x0a,0x3c]
 0x01,0x04,0x0a,0x3c
+# GFX11: v_xnor_b32_e32 v5, s1, v2               ; encoding: [0x01,0x04,0x0a,0x3c]
 
-# GFX11: v_xnor_b32_e32 v5, s105, v2             ; encoding: [0x69,0x04,0x0a,0x3c]
 0x69,0x04,0x0a,0x3c
+# GFX11: v_xnor_b32_e32 v5, s105, v2             ; encoding: [0x69,0x04,0x0a,0x3c]
 
-# GFX11: v_xnor_b32_e32 v5, vcc_lo, v2           ; encoding: [0x6a,0x04,0x0a,0x3c]
 0x6a,0x04,0x0a,0x3c
+# GFX11: v_xnor_b32_e32 v5, vcc_lo, v2           ; encoding: [0x6a,0x04,0x0a,0x3c]
 
-# GFX11: v_xnor_b32_e32 v5, vcc_hi, v2           ; encoding: [0x6b,0x04,0x0a,0x3c]
 0x6b,0x04,0x0a,0x3c
+# GFX11: v_xnor_b32_e32 v5, vcc_hi, v2           ; encoding: [0x6b,0x04,0x0a,0x3c]
 
-# GFX11: v_xnor_b32_e32 v5, ttmp15, v2           ; encoding: [0x7b,0x04,0x0a,0x3c]
 0x7b,0x04,0x0a,0x3c
+# GFX11: v_xnor_b32_e32 v5, ttmp15, v2           ; encoding: [0x7b,0x04,0x0a,0x3c]
 
-# GFX11: v_xnor_b32_e32 v5, m0, v2               ; encoding: [0x7d,0x04,0x0a,0x3c]
 0x7d,0x04,0x0a,0x3c
+# GFX11: v_xnor_b32_e32 v5, m0, v2               ; encoding: [0x7d,0x04,0x0a,0x3c]
 
-# GFX11: v_xnor_b32_e32 v5, exec_lo, v2          ; encoding: [0x7e,0x04,0x0a,0x3c]
 0x7e,0x04,0x0a,0x3c
+# GFX11: v_xnor_b32_e32 v5, exec_lo, v2          ; encoding: [0x7e,0x04,0x0a,0x3c]
 
-# GFX11: v_xnor_b32_e32 v5, exec_hi, v2          ; encoding: [0x7f,0x04,0x0a,0x3c]
 0x7f,0x04,0x0a,0x3c
+# GFX11: v_xnor_b32_e32 v5, exec_hi, v2          ; encoding: [0x7f,0x04,0x0a,0x3c]
 
-# GFX11: v_xnor_b32_e32 v5, null, v2             ; encoding: [0x7c,0x04,0x0a,0x3c]
 0x7c,0x04,0x0a,0x3c
+# GFX11: v_xnor_b32_e32 v5, null, v2             ; encoding: [0x7c,0x04,0x0a,0x3c]
 
-# GFX11: v_xnor_b32_e32 v5, -1, v2               ; encoding: [0xc1,0x04,0x0a,0x3c]
 0xc1,0x04,0x0a,0x3c
+# GFX11: v_xnor_b32_e32 v5, -1, v2               ; encoding: [0xc1,0x04,0x0a,0x3c]
 
-# GFX11: v_xnor_b32_e32 v5, 0.5, v2              ; encoding: [0xf0,0x04,0x0a,0x3c]
 0xf0,0x04,0x0a,0x3c
+# GFX11: v_xnor_b32_e32 v5, 0.5, v2              ; encoding: [0xf0,0x04,0x0a,0x3c]
 
-# GFX11: v_xnor_b32_e32 v5, src_scc, v2          ; encoding: [0xfd,0x04,0x0a,0x3c]
 0xfd,0x04,0x0a,0x3c
+# GFX11: v_xnor_b32_e32 v5, src_scc, v2          ; encoding: [0xfd,0x04,0x0a,0x3c]
 
-# GFX11: v_xnor_b32_e32 v255, 0xaf123456, v255   ; encoding: [0xff,0xfe,0xff,0x3d,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x3d,0x56,0x34,0x12,0xaf
+# GFX11: v_xnor_b32_e32 v255, 0xaf123456, v255   ; encoding: [0xff,0xfe,0xff,0x3d,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_xor_b32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x3a]
 0x01,0x05,0x0a,0x3a
+# GFX11: v_xor_b32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x3a]
 
-# GFX11: v_xor_b32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x3a]
 0xff,0x05,0x0a,0x3a
+# GFX11: v_xor_b32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x3a]
 
-# GFX11: v_xor_b32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x3a]
 0x01,0x04,0x0a,0x3a
+# GFX11: v_xor_b32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x3a]
 
-# GFX11: v_xor_b32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x3a]
 0x69,0x04,0x0a,0x3a
+# GFX11: v_xor_b32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x3a]
 
-# GFX11: v_xor_b32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x3a]
 0x6a,0x04,0x0a,0x3a
+# GFX11: v_xor_b32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x3a]
 
-# GFX11: v_xor_b32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x3a]
 0x6b,0x04,0x0a,0x3a
+# GFX11: v_xor_b32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x3a]
 
-# GFX11: v_xor_b32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x3a]
 0x7b,0x04,0x0a,0x3a
+# GFX11: v_xor_b32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x3a]
 
-# GFX11: v_xor_b32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x3a]
 0x7d,0x04,0x0a,0x3a
+# GFX11: v_xor_b32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x3a]
 
-# GFX11: v_xor_b32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x3a]
 0x7e,0x04,0x0a,0x3a
+# GFX11: v_xor_b32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x3a]
 
-# GFX11: v_xor_b32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x3a]
 0x7f,0x04,0x0a,0x3a
+# GFX11: v_xor_b32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x3a]
 
-# GFX11: v_xor_b32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x3a]
 0x7c,0x04,0x0a,0x3a
+# GFX11: v_xor_b32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x3a]
 
-# GFX11: v_xor_b32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x3a]
 0xc1,0x04,0x0a,0x3a
+# GFX11: v_xor_b32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x3a]
 
-# GFX11: v_xor_b32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x3a]
 0xf0,0x04,0x0a,0x3a
+# GFX11: v_xor_b32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x3a]
 
-# GFX11: v_xor_b32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x3a]
 0xfd,0x04,0x0a,0x3a
+# GFX11: v_xor_b32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x3a]
 
-# GFX11: v_xor_b32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x3b,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x3b,0x56,0x34,0x12,0xaf
+# GFX11: v_xor_b32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x3b,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp16.txt
index eebf0cc13cee..a8a40f883cc4 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp16.txt
@@ -1,1750 +1,1851 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64 %s
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,GFX11-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64,GFX11-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,GFX11-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64,GFX11-FAKE16 %s
 
+0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0xe4,0x00,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0xe4,0x00,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0xe4,0x00,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0xe4,0x00,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x40,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x40,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x40,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x40,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x41,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x41,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x41,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x41,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x01,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x01,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x01,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x01,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x0f,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x0f,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x0f,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x0f,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x11,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x11,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x11,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x11,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x1f,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1f,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1f,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x1f,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x21,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x21,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x21,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x21,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x2f,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x2f,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x2f,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x2f,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x50,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x50,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x50,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x50,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x5f,0x01,0x01
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x5f,0x01,0x01]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x5f,0x01,0x01]
-0xfa,0x04,0x0a,0x40,0x01,0x5f,0x01,0x01
 
+0xfa,0x04,0x0a,0x40,0x01,0x60,0x01,0x13
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x60,0x01,0x13]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x60,0x01,0x13]
-0xfa,0x04,0x0a,0x40,0x01,0x60,0x01,0x13
 
+0xfa,0xfe,0xff,0x41,0xff,0x6f,0x0d,0x30
 # W32: v_add_co_ci_u32_dpp v255, vcc_lo, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x41,0xff,0x6f,0x0d,0x30]
 # W64: v_add_co_ci_u32_dpp v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x41,0xff,0x6f,0x0d,0x30]
-0xfa,0xfe,0xff,0x41,0xff,0x6f,0x0d,0x30
 
-# GFX11: v_add_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff
+# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_add_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0xe4,0x00,0xff
+# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_add_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x40,0x01,0xff
+# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x40,0x01,0xff]
 
-# GFX11: v_add_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x41,0x01,0xff
+# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x41,0x01,0xff]
 
-# GFX11: v_add_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x01,0x01,0xff
+# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x01,0x01,0xff]
 
-# GFX11: v_add_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x0f,0x01,0xff
+# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_add_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x11,0x01,0xff
+# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x11,0x01,0xff]
 
-# GFX11: v_add_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x1f,0x01,0xff
+# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_add_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x21,0x01,0xff
+# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x21,0x01,0xff]
 
-# GFX11: v_add_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x2f,0x01,0xff
+# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_add_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff
+# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff]
 
-# GFX11: v_add_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_add_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x64,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x60,0x01,0x13]
 
-# GFX11: v_add_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xfd,0x30]
 0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xfd,0x30
+# GFX11-REAL16: v_add_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_add_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xfd,0x30]
 
-# GFX11: v_add_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x1b,0x00,0xff
+# GFX11: v_add_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_add_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0xe4,0x00,0xff
+# GFX11: v_add_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_add_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x40,0x01,0xff
+# GFX11: v_add_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x40,0x01,0xff]
 
-# GFX11: v_add_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x41,0x01,0xff
+# GFX11: v_add_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x41,0x01,0xff]
 
-# GFX11: v_add_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x01,0x01,0xff
+# GFX11: v_add_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x01,0x01,0xff]
 
-# GFX11: v_add_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x0f,0x01,0xff
+# GFX11: v_add_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_add_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x11,0x01,0xff
+# GFX11: v_add_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x11,0x01,0xff]
 
-# GFX11: v_add_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x1f,0x01,0xff
+# GFX11: v_add_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_add_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x21,0x01,0xff
+# GFX11: v_add_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x21,0x01,0xff]
 
-# GFX11: v_add_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x2f,0x01,0xff
+# GFX11: v_add_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_add_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x50,0x01,0xff
+# GFX11: v_add_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x50,0x01,0xff]
 
-# GFX11: v_add_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x06,0x01,0x5f,0x01,0x01
+# GFX11: v_add_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_add_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x06,0x01,0x60,0x01,0x13
+# GFX11: v_add_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x60,0x01,0x13]
 
-# GFX11: v_add_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x07,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x07,0xff,0x6f,0xfd,0x30
+# GFX11: v_add_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x07,0xff,0x6f,0xfd,0x30]
 
-# GFX11: v_add_nc_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x1b,0x00,0xff
+# GFX11: v_add_nc_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_add_nc_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0xe4,0x00,0xff
+# GFX11: v_add_nc_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x40,0x01,0xff
+# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x40,0x01,0xff]
 
-# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x41,0x01,0xff
+# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x41,0x01,0xff]
 
-# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x01,0x01,0xff
+# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x01,0x01,0xff]
 
-# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x0f,0x01,0xff
+# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x11,0x01,0xff
+# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x11,0x01,0xff]
 
-# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x1f,0x01,0xff
+# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x21,0x01,0xff
+# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x21,0x01,0xff]
 
-# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x2f,0x01,0xff
+# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x50,0x01,0xff
+# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x50,0x01,0xff]
 
-# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x4a,0x01,0x5f,0x01,0x01
+# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x4a,0x01,0x60,0x01,0x13
+# GFX11: v_add_nc_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x60,0x01,0x13]
 
-# GFX11: v_add_nc_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x4b,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x4b,0xff,0x6f,0x0d,0x30
+# GFX11: v_add_nc_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x4b,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_and_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x1b,0x00,0xff
+# GFX11: v_and_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_and_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0xe4,0x00,0xff
+# GFX11: v_and_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_and_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x40,0x01,0xff
+# GFX11: v_and_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x40,0x01,0xff]
 
-# GFX11: v_and_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x41,0x01,0xff
+# GFX11: v_and_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x41,0x01,0xff]
 
-# GFX11: v_and_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x01,0x01,0xff
+# GFX11: v_and_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x01,0x01,0xff]
 
-# GFX11: v_and_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x0f,0x01,0xff
+# GFX11: v_and_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_and_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x11,0x01,0xff
+# GFX11: v_and_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x11,0x01,0xff]
 
-# GFX11: v_and_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x1f,0x01,0xff
+# GFX11: v_and_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_and_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x21,0x01,0xff
+# GFX11: v_and_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x21,0x01,0xff]
 
-# GFX11: v_and_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x2f,0x01,0xff
+# GFX11: v_and_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_and_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x50,0x01,0xff
+# GFX11: v_and_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x50,0x01,0xff]
 
-# GFX11: v_and_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x36,0x01,0x5f,0x01,0x01
+# GFX11: v_and_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_and_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x36,0x01,0x60,0x01,0x13
+# GFX11: v_and_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x60,0x01,0x13]
 
-# GFX11: v_and_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x37,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x37,0xff,0x6f,0x0d,0x30
+# GFX11: v_and_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x37,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_ashrrev_i32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x1b,0x00,0xff
+# GFX11: v_ashrrev_i32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_ashrrev_i32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0xe4,0x00,0xff
+# GFX11: v_ashrrev_i32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x40,0x01,0xff
+# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x40,0x01,0xff]
 
-# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x41,0x01,0xff
+# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x41,0x01,0xff]
 
-# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x01,0x01,0xff
+# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x01,0x01,0xff]
 
-# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x0f,0x01,0xff
+# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x11,0x01,0xff
+# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x11,0x01,0xff]
 
-# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x1f,0x01,0xff
+# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x21,0x01,0xff
+# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x21,0x01,0xff]
 
-# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x2f,0x01,0xff
+# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x50,0x01,0xff
+# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x50,0x01,0xff]
 
-# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x34,0x01,0x5f,0x01,0x01
+# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x34,0x01,0x60,0x01,0x13
+# GFX11: v_ashrrev_i32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x60,0x01,0x13]
 
-# GFX11: v_ashrrev_i32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x35,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x35,0xff,0x6f,0x0d,0x30
+# GFX11: v_ashrrev_i32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x35,0xff,0x6f,0x0d,0x30]
 
+0xfa,0x04,0x0a,0x02,0x01,0x1b,0x00,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1b,0x00,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1b,0x00,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x1b,0x00,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0xe4,0x00,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x00,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x00,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0xe4,0x00,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x40,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x40,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x40,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x40,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x41,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x41,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x41,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x41,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x01,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x01,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x01,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x01,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x0f,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x0f,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x0f,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x0f,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x11,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x11,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x11,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x11,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x1f,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1f,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1f,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x1f,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x21,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x21,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x21,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x21,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x2f,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x2f,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x2f,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x2f,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x50,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x50,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x50,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x50,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x5f,0x01,0x01
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x5f,0x01,0x01]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x5f,0x01,0x01]
-0xfa,0x04,0x0a,0x02,0x01,0x5f,0x01,0x01
 
+0xfa,0x04,0x0a,0x02,0x01,0x60,0x01,0x13
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x60,0x01,0x13]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x60,0x01,0x13]
-0xfa,0x04,0x0a,0x02,0x01,0x60,0x01,0x13
 
+0xfa,0xfe,0xff,0x03,0xff,0x6f,0x0d,0x30
 # W32: v_cndmask_b32_dpp v255, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x03,0xff,0x6f,0x0d,0x30]
 # W64: v_cndmask_b32_dpp v255, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x03,0xff,0x6f,0x0d,0x30]
-0xfa,0xfe,0xff,0x03,0xff,0x6f,0x0d,0x30
 
+0xfa,0x04,0x0a,0x02,0x01,0xe4,0x90,0xff
 # W32: v_cndmask_b32_dpp v5, -v1, |v2|, vcc_lo quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x90,0xff]
 # W64: v_cndmask_b32_dpp v5, -v1, |v2|, vcc quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x90,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0xe4,0x90,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0xe4,0x60,0xff
 # W32: v_cndmask_b32_dpp v5, |v1|, -v2, vcc_lo quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x60,0xff]
 # W64: v_cndmask_b32_dpp v5, |v1|, -v2, vcc quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x60,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0xe4,0x60,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0xe4,0xf0,0xff
 # W32: v_cndmask_b32_dpp v5, -|v1|, -|v2|, vcc_lo quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0xf0,0xff]
 # W64: v_cndmask_b32_dpp v5, -|v1|, -|v2|, vcc quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0xf0,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0xe4,0xf0,0xff
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x1b,0x00,0xff
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0xe4,0x00,0xff
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x40,0x01,0xff
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x40,0x01,0xff]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x41,0x01,0xff
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x41,0x01,0xff]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x01,0x01,0xff
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x01,0x01,0xff]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x0f,0x01,0xff
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x11,0x01,0xff
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x11,0x01,0xff]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x1f,0x01,0xff
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x21,0x01,0xff
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x21,0x01,0xff]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x2f,0x01,0xff
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x50,0x01,0xff
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x50,0x01,0xff]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x5e,0x01,0x5f,0x01,0x01
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x5e,0x01,0x60,0x01,0x13
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x5f,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x5f,0xff,0x6f,0xfd,0x30
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x5f,0xff,0x6f,0xfd,0x30]
 
-# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x04,0x01,0x41,0x01,0xff
+# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x41,0x01,0xff]
 
-# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x04,0x01,0x01,0x01,0xff
+# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x01,0x01,0xff]
 
-# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x04,0x01,0x0f,0x01,0xff
+# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x04,0x01,0x11,0x01,0xff
+# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x11,0x01,0xff]
 
-# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x04,0x01,0x1f,0x01,0xff
+# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x04,0x01,0x21,0x01,0xff
+# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x21,0x01,0xff]
 
-# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x04,0x01,0x2f,0x01,0xff
+# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x04,0x01,0x50,0x01,0xff
+# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x50,0x01,0xff]
 
-# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x04,0x01,0x5f,0x01,0x01
+# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x04,0x01,0x60,0x01,0x13
+# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x04,0x01,0x60,0x01,0x13]
 
-# GFX11: v_dot2acc_f32_f16_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x05,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x05,0xff,0x6f,0xfd,0x30
+# GFX11: v_dot2acc_f32_f16_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x05,0xff,0x6f,0xfd,0x30]
 
-# GFX11: v_fmac_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x1b,0x00,0xff
+# GFX11: v_fmac_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_fmac_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0xe4,0x00,0xff
+# GFX11: v_fmac_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_fmac_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x40,0x01,0xff
+# GFX11: v_fmac_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x40,0x01,0xff]
 
-# GFX11: v_fmac_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x41,0x01,0xff
+# GFX11: v_fmac_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x41,0x01,0xff]
 
-# GFX11: v_fmac_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x01,0x01,0xff
+# GFX11: v_fmac_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x01,0x01,0xff]
 
-# GFX11: v_fmac_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x0f,0x01,0xff
+# GFX11: v_fmac_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_fmac_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x11,0x01,0xff
+# GFX11: v_fmac_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x11,0x01,0xff]
 
-# GFX11: v_fmac_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x1f,0x01,0xff
+# GFX11: v_fmac_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_fmac_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x21,0x01,0xff
+# GFX11: v_fmac_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x21,0x01,0xff]
 
-# GFX11: v_fmac_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x2f,0x01,0xff
+# GFX11: v_fmac_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_fmac_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x50,0x01,0xff
+# GFX11: v_fmac_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x50,0x01,0xff]
 
-# GFX11: v_fmac_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x6c,0x01,0x5f,0x01,0x01
+# GFX11: v_fmac_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_fmac_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x6c,0x01,0x60,0x01,0x13
+# GFX11: v_fmac_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x60,0x01,0x13]
 
-# GFX11: v_fmac_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x6c,0x7f,0x6f,0xfd,0x30]
 0xfa,0xfe,0xfe,0x6c,0x7f,0x6f,0xfd,0x30
+# GFX11: v_fmac_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x6c,0x7f,0x6f,0xfd,0x30]
 
-# GFX11: v_fmac_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x1b,0x00,0xff
+# GFX11: v_fmac_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_fmac_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0xe4,0x00,0xff
+# GFX11: v_fmac_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_fmac_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x40,0x01,0xff
+# GFX11: v_fmac_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x40,0x01,0xff]
 
-# GFX11: v_fmac_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x41,0x01,0xff
+# GFX11: v_fmac_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x41,0x01,0xff]
 
-# GFX11: v_fmac_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x01,0x01,0xff
+# GFX11: v_fmac_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x01,0x01,0xff]
 
-# GFX11: v_fmac_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x0f,0x01,0xff
+# GFX11: v_fmac_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_fmac_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x11,0x01,0xff
+# GFX11: v_fmac_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x11,0x01,0xff]
 
-# GFX11: v_fmac_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x1f,0x01,0xff
+# GFX11: v_fmac_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_fmac_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x21,0x01,0xff
+# GFX11: v_fmac_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x21,0x01,0xff]
 
-# GFX11: v_fmac_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x2f,0x01,0xff
+# GFX11: v_fmac_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_fmac_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x50,0x01,0xff
+# GFX11: v_fmac_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x50,0x01,0xff]
 
-# GFX11: v_fmac_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x56,0x01,0x5f,0x01,0x01
+# GFX11: v_fmac_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_fmac_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x56,0x01,0x60,0x01,0x13
+# GFX11: v_fmac_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x60,0x01,0x13]
 
-# GFX11: v_fmac_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x57,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x57,0xff,0x6f,0xfd,0x30
+# GFX11: v_fmac_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x57,0xff,0x6f,0xfd,0x30]
 
-# GFX11: v_ldexp_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x1b,0x00,0xff
+# GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_ldexp_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0xe4,0x00,0xff
+# GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_ldexp_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x40,0x01,0xff
+# GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x40,0x01,0xff]
 
-# GFX11: v_ldexp_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x41,0x01,0xff
+# GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x41,0x01,0xff]
 
-# GFX11: v_ldexp_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x01,0x01,0xff
+# GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x01,0x01,0xff]
 
-# GFX11: v_ldexp_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x0f,0x01,0xff
+# GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_ldexp_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x11,0x01,0xff
+# GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x11,0x01,0xff]
 
-# GFX11: v_ldexp_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x1f,0x01,0xff
+# GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1f,0x01,0xff
 
-# GFX11: v_ldexp_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x21,0x01,0xff
+# GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x21,0x01,0xff]
 
-# GFX11: v_ldexp_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x2f,0x01,0xff
+# GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_ldexp_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x50,0x01,0xff
+# GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x50,0x01,0xff]
 
-# GFX11: v_ldexp_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x76,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_ldexp_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x76,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x60,0x01,0x13]
 
-# GFX11: v_ldexp_f16_dpp v127, -|v127|, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x76,0x7f,0x6f,0x3d,0x30]
 0xfa,0xfe,0xfe,0x76,0x7f,0x6f,0x3d,0x30
+# GFX11-REAL16: v_ldexp_f16_dpp v127.l, -|v127.l|, v127.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x76,0x7f,0x6f,0x3d,0x30]
+# GFX11-FAKE16: v_ldexp_f16_dpp v127, -|v127|, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x76,0x7f,0x6f,0x3d,0x30]
 
-# GFX11: v_lshlrev_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x1b,0x00,0xff
+# GFX11: v_lshlrev_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_lshlrev_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0xe4,0x00,0xff
+# GFX11: v_lshlrev_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x40,0x01,0xff
+# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x40,0x01,0xff]
 
-# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x41,0x01,0xff
+# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x41,0x01,0xff]
 
-# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x01,0x01,0xff
+# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x01,0x01,0xff]
 
-# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x0f,0x01,0xff
+# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x11,0x01,0xff
+# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x11,0x01,0xff]
 
-# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x1f,0x01,0xff
+# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x21,0x01,0xff
+# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x21,0x01,0xff]
 
-# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x2f,0x01,0xff
+# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x50,0x01,0xff
+# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x50,0x01,0xff]
 
-# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x30,0x01,0x5f,0x01,0x01
+# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x30,0x01,0x60,0x01,0x13
+# GFX11: v_lshlrev_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x60,0x01,0x13]
 
-# GFX11: v_lshlrev_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x31,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x31,0xff,0x6f,0x0d,0x30
+# GFX11: v_lshlrev_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x31,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_lshrrev_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x1b,0x00,0xff
+# GFX11: v_lshrrev_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_lshrrev_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0xe4,0x00,0xff
+# GFX11: v_lshrrev_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x40,0x01,0xff
+# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x40,0x01,0xff]
 
-# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x41,0x01,0xff
+# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x41,0x01,0xff]
 
-# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x01,0x01,0xff
+# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x01,0x01,0xff]
 
-# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x0f,0x01,0xff
+# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x11,0x01,0xff
+# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x11,0x01,0xff]
 
-# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x1f,0x01,0xff
+# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x21,0x01,0xff
+# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x21,0x01,0xff]
 
-# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x2f,0x01,0xff
+# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x50,0x01,0xff
+# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x50,0x01,0xff]
 
-# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x32,0x01,0x5f,0x01,0x01
+# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x32,0x01,0x60,0x01,0x13
+# GFX11: v_lshrrev_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x60,0x01,0x13]
 
-# GFX11: v_lshrrev_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x33,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x33,0xff,0x6f,0x0d,0x30
+# GFX11: v_lshrrev_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x33,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_max_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x72,0x01,0x1b,0x00,0xff
+# GFX11-REAL16: v_max_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_max_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_max_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x72,0x01,0xe4,0x00,0xff
+# GFX11-REAL16: v_max_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_max_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_max_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x72,0x01,0x40,0x01,0xff
+# GFX11-REAL16: v_max_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_max_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x40,0x01,0xff]
 
-# GFX11: v_max_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x72,0x01,0x41,0x01,0xff
+# GFX11-REAL16: v_max_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_max_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x41,0x01,0xff]
 
-# GFX11: v_max_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x72,0x01,0x01,0x01,0xff
+# GFX11-REAL16: v_max_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_max_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x01,0x01,0xff]
 
-# GFX11: v_max_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x72,0x01,0x0f,0x01,0xff
+# GFX11-REAL16: v_max_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_max_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_max_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x72,0x01,0x11,0x01,0xff
+# GFX11-REAL16: v_max_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_max_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x11,0x01,0xff]
 
-# GFX11: v_max_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x72,0x01,0x1f,0x01,0xff
+# GFX11-REAL16: v_max_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_max_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_max_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x72,0x01,0x21,0x01,0xff
+# GFX11-REAL16: v_max_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_max_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x21,0x01,0xff]
 
-# GFX11: v_max_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x72,0x01,0x2f,0x01,0xff
+# GFX11-REAL16: v_max_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_max_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_max_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x72,0x01,0x50,0x01,0xff
+# GFX11-REAL16: v_max_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_max_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x50,0x01,0xff]
 
-# GFX11: v_max_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x72,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_max_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_max_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_max_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x72,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_max_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_max_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x72,0x01,0x60,0x01,0x13]
 
-# GFX11: v_max_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x72,0x7f,0x6f,0xfd,0x30]
 0xfa,0xfe,0xfe,0x72,0x7f,0x6f,0xfd,0x30
+# GFX11-REAL16: v_max_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x72,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_max_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x72,0x7f,0x6f,0xfd,0x30]
 
-# GFX11: v_max_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x20,0x01,0x1b,0x00,0xff
+# GFX11: v_max_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_max_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x20,0x01,0xe4,0x00,0xff
+# GFX11: v_max_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_max_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x20,0x01,0x40,0x01,0xff
+# GFX11: v_max_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x40,0x01,0xff]
 
-# GFX11: v_max_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x20,0x01,0x41,0x01,0xff
+# GFX11: v_max_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x41,0x01,0xff]
 
-# GFX11: v_max_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x20,0x01,0x01,0x01,0xff
+# GFX11: v_max_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x01,0x01,0xff]
 
-# GFX11: v_max_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x20,0x01,0x0f,0x01,0xff
+# GFX11: v_max_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_max_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x20,0x01,0x11,0x01,0xff
+# GFX11: v_max_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x11,0x01,0xff]
 
-# GFX11: v_max_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x20,0x01,0x1f,0x01,0xff
+# GFX11: v_max_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_max_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x20,0x01,0x21,0x01,0xff
+# GFX11: v_max_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x21,0x01,0xff]
 
-# GFX11: v_max_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x20,0x01,0x2f,0x01,0xff
+# GFX11: v_max_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_max_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x20,0x01,0x50,0x01,0xff
+# GFX11: v_max_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x50,0x01,0xff]
 
-# GFX11: v_max_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x20,0x01,0x5f,0x01,0x01
+# GFX11: v_max_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_max_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x20,0x01,0x60,0x01,0x13
+# GFX11: v_max_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x20,0x01,0x60,0x01,0x13]
 
-# GFX11: v_max_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x21,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x21,0xff,0x6f,0xfd,0x30
+# GFX11: v_max_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x21,0xff,0x6f,0xfd,0x30]
 
-# GFX11: v_max_i32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x1b,0x00,0xff
+# GFX11: v_max_i32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_max_i32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0xe4,0x00,0xff
+# GFX11: v_max_i32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_max_i32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x40,0x01,0xff
+# GFX11: v_max_i32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x40,0x01,0xff]
 
-# GFX11: v_max_i32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x41,0x01,0xff
+# GFX11: v_max_i32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x41,0x01,0xff]
 
-# GFX11: v_max_i32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x01,0x01,0xff
+# GFX11: v_max_i32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x01,0x01,0xff]
 
-# GFX11: v_max_i32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x0f,0x01,0xff
+# GFX11: v_max_i32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_max_i32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x11,0x01,0xff
+# GFX11: v_max_i32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x11,0x01,0xff]
 
-# GFX11: v_max_i32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x1f,0x01,0xff
+# GFX11: v_max_i32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_max_i32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x21,0x01,0xff
+# GFX11: v_max_i32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x21,0x01,0xff]
 
-# GFX11: v_max_i32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x2f,0x01,0xff
+# GFX11: v_max_i32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_max_i32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x50,0x01,0xff
+# GFX11: v_max_i32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x50,0x01,0xff]
 
-# GFX11: v_max_i32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x24,0x01,0x5f,0x01,0x01
+# GFX11: v_max_i32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_max_i32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x24,0x01,0x60,0x01,0x13
+# GFX11: v_max_i32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x60,0x01,0x13]
 
-# GFX11: v_max_i32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x25,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x25,0xff,0x6f,0x0d,0x30
+# GFX11: v_max_i32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x25,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_max_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x1b,0x00,0xff
+# GFX11: v_max_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_max_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0xe4,0x00,0xff
+# GFX11: v_max_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_max_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x40,0x01,0xff
+# GFX11: v_max_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x40,0x01,0xff]
 
-# GFX11: v_max_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x41,0x01,0xff
+# GFX11: v_max_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x41,0x01,0xff]
 
-# GFX11: v_max_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x01,0x01,0xff
+# GFX11: v_max_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x01,0x01,0xff]
 
-# GFX11: v_max_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x0f,0x01,0xff
+# GFX11: v_max_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_max_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x11,0x01,0xff
+# GFX11: v_max_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x11,0x01,0xff]
 
-# GFX11: v_max_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x1f,0x01,0xff
+# GFX11: v_max_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_max_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x21,0x01,0xff
+# GFX11: v_max_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x21,0x01,0xff]
 
-# GFX11: v_max_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x2f,0x01,0xff
+# GFX11: v_max_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_max_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x50,0x01,0xff
+# GFX11: v_max_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x50,0x01,0xff]
 
-# GFX11: v_max_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x28,0x01,0x5f,0x01,0x01
+# GFX11: v_max_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_max_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x28,0x01,0x60,0x01,0x13
+# GFX11: v_max_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x60,0x01,0x13]
 
-# GFX11: v_max_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x29,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x29,0xff,0x6f,0x0d,0x30
+# GFX11: v_max_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x29,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_min_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x74,0x01,0x1b,0x00,0xff
+# GFX11-REAL16: v_min_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_min_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_min_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x74,0x01,0xe4,0x00,0xff
+# GFX11-REAL16: v_min_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_min_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_min_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x74,0x01,0x40,0x01,0xff
+# GFX11-REAL16: v_min_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_min_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x40,0x01,0xff]
 
-# GFX11: v_min_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x74,0x01,0x41,0x01,0xff
+# GFX11-REAL16: v_min_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_min_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x41,0x01,0xff]
 
-# GFX11: v_min_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x74,0x01,0x01,0x01,0xff
+# GFX11-REAL16: v_min_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_min_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x01,0x01,0xff]
 
-# GFX11: v_min_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x74,0x01,0x0f,0x01,0xff
+# GFX11-REAL16: v_min_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_min_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_min_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x74,0x01,0x11,0x01,0xff
+# GFX11-REAL16: v_min_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_min_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x11,0x01,0xff]
 
-# GFX11: v_min_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x74,0x01,0x1f,0x01,0xff
+# GFX11-REAL16: v_min_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_min_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_min_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x74,0x01,0x21,0x01,0xff
+# GFX11-REAL16: v_min_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_min_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x21,0x01,0xff]
 
-# GFX11: v_min_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x74,0x01,0x2f,0x01,0xff
+# GFX11-REAL16: v_min_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_min_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_min_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x74,0x01,0x50,0x01,0xff
+# GFX11-REAL16: v_min_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_min_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x50,0x01,0xff]
 
-# GFX11: v_min_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x74,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_min_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_min_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_min_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x74,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_min_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_min_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x74,0x01,0x60,0x01,0x13]
 
-# GFX11: v_min_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x74,0x7f,0x6f,0xfd,0x30]
 0xfa,0xfe,0xfe,0x74,0x7f,0x6f,0xfd,0x30
+# GFX11-REAL16: v_min_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x74,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_min_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x74,0x7f,0x6f,0xfd,0x30]
 
-# GFX11: v_min_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x1e,0x01,0x1b,0x00,0xff
+# GFX11: v_min_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_min_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x1e,0x01,0xe4,0x00,0xff
+# GFX11: v_min_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_min_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x1e,0x01,0x40,0x01,0xff
+# GFX11: v_min_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x40,0x01,0xff]
 
-# GFX11: v_min_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x1e,0x01,0x41,0x01,0xff
+# GFX11: v_min_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x41,0x01,0xff]
 
-# GFX11: v_min_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x1e,0x01,0x01,0x01,0xff
+# GFX11: v_min_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x01,0x01,0xff]
 
-# GFX11: v_min_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x1e,0x01,0x0f,0x01,0xff
+# GFX11: v_min_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_min_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x1e,0x01,0x11,0x01,0xff
+# GFX11: v_min_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x11,0x01,0xff]
 
-# GFX11: v_min_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x1e,0x01,0x1f,0x01,0xff
+# GFX11: v_min_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_min_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x1e,0x01,0x21,0x01,0xff
+# GFX11: v_min_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x21,0x01,0xff]
 
-# GFX11: v_min_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x1e,0x01,0x2f,0x01,0xff
+# GFX11: v_min_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_min_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x1e,0x01,0x50,0x01,0xff
+# GFX11: v_min_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x50,0x01,0xff]
 
-# GFX11: v_min_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x1e,0x01,0x5f,0x01,0x01
+# GFX11: v_min_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_min_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x1e,0x01,0x60,0x01,0x13
+# GFX11: v_min_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x1e,0x01,0x60,0x01,0x13]
 
-# GFX11: v_min_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x1f,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x1f,0xff,0x6f,0xfd,0x30
+# GFX11: v_min_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x1f,0xff,0x6f,0xfd,0x30]
 
-# GFX11: v_min_i32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x1b,0x00,0xff
+# GFX11: v_min_i32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_min_i32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0xe4,0x00,0xff
+# GFX11: v_min_i32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_min_i32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x40,0x01,0xff
+# GFX11: v_min_i32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x40,0x01,0xff]
 
-# GFX11: v_min_i32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x41,0x01,0xff
+# GFX11: v_min_i32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x41,0x01,0xff]
 
-# GFX11: v_min_i32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x01,0x01,0xff
+# GFX11: v_min_i32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x01,0x01,0xff]
 
-# GFX11: v_min_i32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x0f,0x01,0xff
+# GFX11: v_min_i32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_min_i32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x11,0x01,0xff
+# GFX11: v_min_i32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x11,0x01,0xff]
 
-# GFX11: v_min_i32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x1f,0x01,0xff
+# GFX11: v_min_i32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_min_i32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x21,0x01,0xff
+# GFX11: v_min_i32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x21,0x01,0xff]
 
-# GFX11: v_min_i32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x2f,0x01,0xff
+# GFX11: v_min_i32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_min_i32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x50,0x01,0xff
+# GFX11: v_min_i32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x50,0x01,0xff]
 
-# GFX11: v_min_i32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x22,0x01,0x5f,0x01,0x01
+# GFX11: v_min_i32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_min_i32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x22,0x01,0x60,0x01,0x13
+# GFX11: v_min_i32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x60,0x01,0x13]
 
-# GFX11: v_min_i32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x23,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x23,0xff,0x6f,0x0d,0x30
+# GFX11: v_min_i32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x23,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_min_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x1b,0x00,0xff
+# GFX11: v_min_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_min_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0xe4,0x00,0xff
+# GFX11: v_min_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_min_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x40,0x01,0xff
+# GFX11: v_min_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x40,0x01,0xff]
 
-# GFX11: v_min_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x41,0x01,0xff
+# GFX11: v_min_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x41,0x01,0xff]
 
-# GFX11: v_min_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x01,0x01,0xff
+# GFX11: v_min_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x01,0x01,0xff]
 
-# GFX11: v_min_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x0f,0x01,0xff
+# GFX11: v_min_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_min_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x11,0x01,0xff
+# GFX11: v_min_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x11,0x01,0xff]
 
-# GFX11: v_min_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x1f,0x01,0xff
+# GFX11: v_min_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_min_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x21,0x01,0xff
+# GFX11: v_min_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x21,0x01,0xff]
 
-# GFX11: v_min_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x2f,0x01,0xff
+# GFX11: v_min_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_min_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x50,0x01,0xff
+# GFX11: v_min_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x50,0x01,0xff]
 
-# GFX11: v_min_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x26,0x01,0x5f,0x01,0x01
+# GFX11: v_min_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_min_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x26,0x01,0x60,0x01,0x13
+# GFX11: v_min_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x60,0x01,0x13]
 
-# GFX11: v_min_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x27,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x27,0xff,0x6f,0x0d,0x30
+# GFX11: v_min_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x27,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x1b,0x00,0xff
+# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0xe4,0x00,0xff
+# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x40,0x01,0xff
+# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x41,0x01,0xff
+# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x41,0x01,0xff]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x01,0x01,0xff
+# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x0f,0x01,0xff
+# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x11,0x01,0xff
+# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x1f,0x01,0xff
+# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x21,0x01,0xff
+# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x2f,0x01,0xff
+# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x50,0x01,0xff
+# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x0e,0x01,0x5f,0x01,0x01
+# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x0e,0x01,0x60,0x01,0x13
+# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x0f,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x0f,0xff,0x6f,0xfd,0x30
+# GFX11: v_mul_dx9_zero_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x0f,0xff,0x6f,0xfd,0x30]
 
-# GFX11: v_mul_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x1b,0x00,0xff
+# GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mul_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0xe4,0x00,0xff
+# GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mul_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x40,0x01,0xff
+# GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mul_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x41,0x01,0xff
+# GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x41,0x01,0xff
 
-# GFX11: v_mul_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x01,0x01,0xff
+# GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mul_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x0f,0x01,0xff
+# GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mul_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x11,0x01,0xff
+# GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mul_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x1f,0x01,0xff
+# GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mul_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x21,0x01,0xff
+# GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mul_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x2f,0x01,0xff
+# GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mul_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x50,0x01,0xff
+# GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mul_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x6a,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mul_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x6a,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mul_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x6a,0x7f,0x6f,0xfd,0x30]
 0xfa,0xfe,0xfe,0x6a,0x7f,0x6f,0xfd,0x30
+# GFX11-REAL16: v_mul_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x6a,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_mul_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x6a,0x7f,0x6f,0xfd,0x30]
 
-# GFX11: v_mul_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x1b,0x00,0xff
+# GFX11: v_mul_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mul_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0xe4,0x00,0xff
+# GFX11: v_mul_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mul_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x40,0x01,0xff
+# GFX11: v_mul_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mul_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x41,0x01,0xff
+# GFX11: v_mul_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x41,0x01,0xff]
 
-# GFX11: v_mul_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x01,0x01,0xff
+# GFX11: v_mul_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mul_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x0f,0x01,0xff
+# GFX11: v_mul_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mul_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x11,0x01,0xff
+# GFX11: v_mul_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mul_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x1f,0x01,0xff
+# GFX11: v_mul_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mul_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x21,0x01,0xff
+# GFX11: v_mul_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mul_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x2f,0x01,0xff
+# GFX11: v_mul_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mul_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x50,0x01,0xff
+# GFX11: v_mul_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mul_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x10,0x01,0x5f,0x01,0x01
+# GFX11: v_mul_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mul_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x10,0x01,0x60,0x01,0x13
+# GFX11: v_mul_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mul_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x11,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x11,0xff,0x6f,0xfd,0x30
+# GFX11: v_mul_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x11,0xff,0x6f,0xfd,0x30]
 
-# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x1b,0x00,0xff
+# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0xe4,0x00,0xff
+# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x40,0x01,0xff
+# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x41,0x01,0xff
+# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x41,0x01,0xff]
 
-# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x01,0x01,0xff
+# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x0f,0x01,0xff
+# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x11,0x01,0xff
+# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x1f,0x01,0xff
+# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x21,0x01,0xff
+# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x2f,0x01,0xff
+# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x50,0x01,0xff
+# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x14,0x01,0x5f,0x01,0x01
+# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x14,0x01,0x60,0x01,0x13
+# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mul_hi_i32_i24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x15,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x15,0xff,0x6f,0x0d,0x30
+# GFX11: v_mul_hi_i32_i24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x15,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x1b,0x00,0xff
+# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0xe4,0x00,0xff
+# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x40,0x01,0xff
+# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x41,0x01,0xff
+# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x41,0x01,0xff]
 
-# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x01,0x01,0xff
+# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x0f,0x01,0xff
+# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x11,0x01,0xff
+# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x1f,0x01,0xff
+# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x21,0x01,0xff
+# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x2f,0x01,0xff
+# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x50,0x01,0xff
+# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x18,0x01,0x5f,0x01,0x01
+# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x18,0x01,0x60,0x01,0x13
+# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mul_hi_u32_u24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x19,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x19,0xff,0x6f,0x0d,0x30
+# GFX11: v_mul_hi_u32_u24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x19,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_mul_i32_i24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x1b,0x00,0xff
+# GFX11: v_mul_i32_i24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mul_i32_i24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0xe4,0x00,0xff
+# GFX11: v_mul_i32_i24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x40,0x01,0xff
+# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x41,0x01,0xff
+# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x41,0x01,0xff]
 
-# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x01,0x01,0xff
+# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x0f,0x01,0xff
+# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x11,0x01,0xff
+# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x1f,0x01,0xff
+# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x21,0x01,0xff
+# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x2f,0x01,0xff
+# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x50,0x01,0xff
+# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x12,0x01,0x5f,0x01,0x01
+# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x12,0x01,0x60,0x01,0x13
+# GFX11: v_mul_i32_i24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mul_i32_i24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x13,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x13,0xff,0x6f,0x0d,0x30
+# GFX11: v_mul_i32_i24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x13,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_mul_u32_u24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x1b,0x00,0xff
+# GFX11: v_mul_u32_u24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mul_u32_u24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0xe4,0x00,0xff
+# GFX11: v_mul_u32_u24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x40,0x01,0xff
+# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x41,0x01,0xff
+# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x41,0x01,0xff]
 
-# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x01,0x01,0xff
+# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x0f,0x01,0xff
+# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x11,0x01,0xff
+# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x1f,0x01,0xff
+# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x21,0x01,0xff
+# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x2f,0x01,0xff
+# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x50,0x01,0xff
+# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x16,0x01,0x5f,0x01,0x01
+# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x16,0x01,0x60,0x01,0x13
+# GFX11: v_mul_u32_u24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mul_u32_u24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x17,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x17,0xff,0x6f,0x0d,0x30
+# GFX11: v_mul_u32_u24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x17,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_or_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x1b,0x00,0xff
+# GFX11: v_or_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_or_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0xe4,0x00,0xff
+# GFX11: v_or_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_or_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x40,0x01,0xff
+# GFX11: v_or_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x40,0x01,0xff]
 
-# GFX11: v_or_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x41,0x01,0xff
+# GFX11: v_or_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x41,0x01,0xff]
 
-# GFX11: v_or_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x01,0x01,0xff
+# GFX11: v_or_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x01,0x01,0xff]
 
-# GFX11: v_or_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x0f,0x01,0xff
+# GFX11: v_or_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_or_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x11,0x01,0xff
+# GFX11: v_or_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x11,0x01,0xff]
 
-# GFX11: v_or_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x1f,0x01,0xff
+# GFX11: v_or_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_or_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x21,0x01,0xff
+# GFX11: v_or_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x21,0x01,0xff]
 
-# GFX11: v_or_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x2f,0x01,0xff
+# GFX11: v_or_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_or_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x50,0x01,0xff
+# GFX11: v_or_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x50,0x01,0xff]
 
-# GFX11: v_or_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x38,0x01,0x5f,0x01,0x01
+# GFX11: v_or_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_or_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x38,0x01,0x60,0x01,0x13
+# GFX11: v_or_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x60,0x01,0x13]
 
-# GFX11: v_or_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x39,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x39,0xff,0x6f,0x0d,0x30
+# GFX11: v_or_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x39,0xff,0x6f,0x0d,0x30]
 
+0xfa,0x04,0x0a,0x42,0x01,0x1b,0x00,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1b,0x00,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1b,0x00,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x1b,0x00,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0xe4,0x00,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0xe4,0x00,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0xe4,0x00,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0xe4,0x00,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x40,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x40,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x40,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x40,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x41,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x41,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x41,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x41,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x01,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x01,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x01,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x01,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x0f,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x0f,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x0f,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x0f,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x11,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x11,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x11,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x11,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x1f,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1f,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1f,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x1f,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x21,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x21,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x21,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x21,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x2f,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x2f,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x2f,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x2f,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x50,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x50,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x50,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x50,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x5f,0x01,0x01
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x5f,0x01,0x01]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x5f,0x01,0x01]
-0xfa,0x04,0x0a,0x42,0x01,0x5f,0x01,0x01
 
+0xfa,0x04,0x0a,0x42,0x01,0x60,0x01,0x13
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x60,0x01,0x13]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x60,0x01,0x13]
-0xfa,0x04,0x0a,0x42,0x01,0x60,0x01,0x13
 
+0xfa,0xfe,0xff,0x43,0xff,0x6f,0x0d,0x30
 # W32: v_sub_co_ci_u32_dpp v255, vcc_lo, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x43,0xff,0x6f,0x0d,0x30]
 # W64: v_sub_co_ci_u32_dpp v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x43,0xff,0x6f,0x0d,0x30]
-0xfa,0xfe,0xff,0x43,0xff,0x6f,0x0d,0x30
 
-# GFX11: v_sub_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x1b,0x00,0xff
+# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_sub_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0xe4,0x00,0xff
+# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_sub_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x40,0x01,0xff
+# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x40,0x01,0xff]
 
-# GFX11: v_sub_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x41,0x01,0xff
+# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x41,0x01,0xff]
 
-# GFX11: v_sub_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x01,0x01,0xff
+# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x01,0x01,0xff]
 
-# GFX11: v_sub_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x0f,0x01,0xff
+# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_sub_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x11,0x01,0xff
+# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x11,0x01,0xff]
 
-# GFX11: v_sub_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x1f,0x01,0xff
+# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_sub_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x21,0x01,0xff
+# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x21,0x01,0xff]
 
-# GFX11: v_sub_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x2f,0x01,0xff
+# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_sub_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff
+# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff]
 
-# GFX11: v_sub_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_sub_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x66,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x60,0x01,0x13]
 
-# GFX11: v_sub_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xfd,0x30]
 0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xfd,0x30
+# GFX11-REAL16: v_sub_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_sub_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xfd,0x30]
 
-# GFX11: v_sub_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x1b,0x00,0xff
+# GFX11: v_sub_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_sub_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0xe4,0x00,0xff
+# GFX11: v_sub_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_sub_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x40,0x01,0xff
+# GFX11: v_sub_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x40,0x01,0xff]
 
-# GFX11: v_sub_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x41,0x01,0xff
+# GFX11: v_sub_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x41,0x01,0xff]
 
-# GFX11: v_sub_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x01,0x01,0xff
+# GFX11: v_sub_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x01,0x01,0xff]
 
-# GFX11: v_sub_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x0f,0x01,0xff
+# GFX11: v_sub_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_sub_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x11,0x01,0xff
+# GFX11: v_sub_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x11,0x01,0xff]
 
-# GFX11: v_sub_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x1f,0x01,0xff
+# GFX11: v_sub_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_sub_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x21,0x01,0xff
+# GFX11: v_sub_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x21,0x01,0xff]
 
-# GFX11: v_sub_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x2f,0x01,0xff
+# GFX11: v_sub_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_sub_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x50,0x01,0xff
+# GFX11: v_sub_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x50,0x01,0xff]
 
-# GFX11: v_sub_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x08,0x01,0x5f,0x01,0x01
+# GFX11: v_sub_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_sub_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x08,0x01,0x60,0x01,0x13
+# GFX11: v_sub_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x60,0x01,0x13]
 
-# GFX11: v_sub_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x09,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x09,0xff,0x6f,0xfd,0x30
+# GFX11: v_sub_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x09,0xff,0x6f,0xfd,0x30]
 
-# GFX11: v_sub_nc_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x1b,0x00,0xff
+# GFX11: v_sub_nc_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_sub_nc_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0xe4,0x00,0xff
+# GFX11: v_sub_nc_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x40,0x01,0xff
+# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x40,0x01,0xff]
 
-# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x41,0x01,0xff
+# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x41,0x01,0xff]
 
-# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x01,0x01,0xff
+# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x01,0x01,0xff]
 
-# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x0f,0x01,0xff
+# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x11,0x01,0xff
+# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x11,0x01,0xff]
 
-# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x1f,0x01,0xff
+# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x21,0x01,0xff
+# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x21,0x01,0xff]
 
-# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x2f,0x01,0xff
+# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x50,0x01,0xff
+# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x50,0x01,0xff]
 
-# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x4c,0x01,0x5f,0x01,0x01
+# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x4c,0x01,0x60,0x01,0x13
+# GFX11: v_sub_nc_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x60,0x01,0x13]
 
-# GFX11: v_sub_nc_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x4d,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x4d,0xff,0x6f,0x0d,0x30
+# GFX11: v_sub_nc_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x4d,0xff,0x6f,0x0d,0x30]
 
+0xfa,0x04,0x0a,0x44,0x01,0x1b,0x00,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1b,0x00,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1b,0x00,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x1b,0x00,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0xe4,0x00,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0xe4,0x00,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0xe4,0x00,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0xe4,0x00,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x40,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x40,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x40,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x40,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x41,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x41,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x41,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x41,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x01,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x01,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x01,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x01,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x0f,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x0f,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x0f,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x0f,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x11,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x11,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x11,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x11,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x1f,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1f,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1f,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x1f,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x21,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x21,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x21,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x21,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x2f,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x2f,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x2f,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x2f,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x50,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x50,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x50,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x50,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x5f,0x01,0x01
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x5f,0x01,0x01]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x5f,0x01,0x01]
-0xfa,0x04,0x0a,0x44,0x01,0x5f,0x01,0x01
 
+0xfa,0x04,0x0a,0x44,0x01,0x60,0x01,0x13
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x60,0x01,0x13]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x60,0x01,0x13]
-0xfa,0x04,0x0a,0x44,0x01,0x60,0x01,0x13
 
+0xfa,0xfe,0xff,0x45,0xff,0x6f,0x0d,0x30
 # W32: v_subrev_co_ci_u32_dpp v255, vcc_lo, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x45,0xff,0x6f,0x0d,0x30]
 # W64: v_subrev_co_ci_u32_dpp v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x45,0xff,0x6f,0x0d,0x30]
-0xfa,0xfe,0xff,0x45,0xff,0x6f,0x0d,0x30
 
-# GFX11: v_subrev_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x1b,0x00,0xff
+# GFX11-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1b,0x00,0xff]
+# GFX11-FAKE16: v_subrev_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_subrev_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0xe4,0x00,0xff
+# GFX11-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0xe4,0x00,0xff]
+# GFX11-FAKE16: v_subrev_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_subrev_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x40,0x01,0xff
+# GFX11-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x40,0x01,0xff]
+# GFX11-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x40,0x01,0xff]
 
-# GFX11: v_subrev_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x41,0x01,0xff
+# GFX11-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x41,0x01,0xff]
+# GFX11-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x41,0x01,0xff]
 
-# GFX11: v_subrev_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x01,0x01,0xff
+# GFX11-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x01,0x01,0xff]
+# GFX11-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x01,0x01,0xff]
 
-# GFX11: v_subrev_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x0f,0x01,0xff
+# GFX11-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x0f,0x01,0xff]
+# GFX11-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_subrev_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x11,0x01,0xff
+# GFX11-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x11,0x01,0xff]
+# GFX11-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x11,0x01,0xff]
 
-# GFX11: v_subrev_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x1f,0x01,0xff
+# GFX11-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1f,0x01,0xff]
+# GFX11-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_subrev_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x21,0x01,0xff
+# GFX11-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x21,0x01,0xff]
+# GFX11-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x21,0x01,0xff]
 
-# GFX11: v_subrev_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x2f,0x01,0xff
+# GFX11-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x2f,0x01,0xff]
+# GFX11-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_subrev_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x50,0x01,0xff
+# GFX11-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x50,0x01,0xff]
+# GFX11-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x50,0x01,0xff]
 
-# GFX11: v_subrev_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x68,0x01,0x5f,0x01,0x01
+# GFX11-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_subrev_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x68,0x01,0x60,0x01,0x13
+# GFX11-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x60,0x01,0x13]
+# GFX11-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x60,0x01,0x13]
 
-# GFX11: v_subrev_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x68,0x7f,0x6f,0xfd,0x30]
 0xfa,0xfe,0xfe,0x68,0x7f,0x6f,0xfd,0x30
+# GFX11-REAL16: v_subrev_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x68,0x7f,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_subrev_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x68,0x7f,0x6f,0xfd,0x30]
 
-# GFX11: v_subrev_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x1b,0x00,0xff
+# GFX11: v_subrev_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_subrev_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0xe4,0x00,0xff
+# GFX11: v_subrev_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_subrev_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x40,0x01,0xff
+# GFX11: v_subrev_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x40,0x01,0xff]
 
-# GFX11: v_subrev_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x41,0x01,0xff
+# GFX11: v_subrev_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x41,0x01,0xff]
 
-# GFX11: v_subrev_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x01,0x01,0xff
+# GFX11: v_subrev_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x01,0x01,0xff]
 
-# GFX11: v_subrev_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x0f,0x01,0xff
+# GFX11: v_subrev_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_subrev_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x11,0x01,0xff
+# GFX11: v_subrev_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x11,0x01,0xff]
 
-# GFX11: v_subrev_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x1f,0x01,0xff
+# GFX11: v_subrev_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_subrev_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x21,0x01,0xff
+# GFX11: v_subrev_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x21,0x01,0xff]
 
-# GFX11: v_subrev_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x2f,0x01,0xff
+# GFX11: v_subrev_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_subrev_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x50,0x01,0xff
+# GFX11: v_subrev_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x50,0x01,0xff]
 
-# GFX11: v_subrev_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x0a,0x01,0x5f,0x01,0x01
+# GFX11: v_subrev_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_subrev_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x0a,0x01,0x60,0x01,0x13
+# GFX11: v_subrev_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x60,0x01,0x13]
 
-# GFX11: v_subrev_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x0b,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x0b,0xff,0x6f,0xfd,0x30
+# GFX11: v_subrev_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x0b,0xff,0x6f,0xfd,0x30]
 
-# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x1b,0x00,0xff
+# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0xe4,0x00,0xff
+# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x40,0x01,0xff
+# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x40,0x01,0xff]
 
-# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x41,0x01,0xff
+# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x41,0x01,0xff]
 
-# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x01,0x01,0xff
+# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x01,0x01,0xff]
 
-# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x0f,0x01,0xff
+# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x11,0x01,0xff
+# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x11,0x01,0xff]
 
-# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x1f,0x01,0xff
+# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x21,0x01,0xff
+# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x21,0x01,0xff]
 
-# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x2f,0x01,0xff
+# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x50,0x01,0xff
+# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x50,0x01,0xff]
 
-# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x4e,0x01,0x5f,0x01,0x01
+# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x4e,0x01,0x60,0x01,0x13
+# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x60,0x01,0x13]
 
-# GFX11: v_subrev_nc_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x4f,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x4f,0xff,0x6f,0x0d,0x30
+# GFX11: v_subrev_nc_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x4f,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_xnor_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x1b,0x00,0xff
+# GFX11: v_xnor_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_xnor_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0xe4,0x00,0xff
+# GFX11: v_xnor_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_xnor_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x40,0x01,0xff
+# GFX11: v_xnor_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x40,0x01,0xff]
 
-# GFX11: v_xnor_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x41,0x01,0xff
+# GFX11: v_xnor_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x41,0x01,0xff]
 
-# GFX11: v_xnor_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x01,0x01,0xff
+# GFX11: v_xnor_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x01,0x01,0xff]
 
-# GFX11: v_xnor_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x0f,0x01,0xff
+# GFX11: v_xnor_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_xnor_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x11,0x01,0xff
+# GFX11: v_xnor_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x11,0x01,0xff]
 
-# GFX11: v_xnor_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x1f,0x01,0xff
+# GFX11: v_xnor_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_xnor_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x21,0x01,0xff
+# GFX11: v_xnor_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x21,0x01,0xff]
 
-# GFX11: v_xnor_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x2f,0x01,0xff
+# GFX11: v_xnor_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_xnor_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x50,0x01,0xff
+# GFX11: v_xnor_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x50,0x01,0xff]
 
-# GFX11: v_xnor_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x3c,0x01,0x5f,0x01,0x01
+# GFX11: v_xnor_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_xnor_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x3c,0x01,0x60,0x01,0x13
+# GFX11: v_xnor_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x60,0x01,0x13]
 
-# GFX11: v_xnor_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x3d,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x3d,0xff,0x6f,0x0d,0x30
+# GFX11: v_xnor_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x3d,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_xor_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x1b,0x00,0xff
+# GFX11: v_xor_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_xor_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0xe4,0x00,0xff
+# GFX11: v_xor_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_xor_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x40,0x01,0xff
+# GFX11: v_xor_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x40,0x01,0xff]
 
-# GFX11: v_xor_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x41,0x01,0xff
+# GFX11: v_xor_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x41,0x01,0xff]
 
-# GFX11: v_xor_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x01,0x01,0xff
+# GFX11: v_xor_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x01,0x01,0xff]
 
-# GFX11: v_xor_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x0f,0x01,0xff
+# GFX11: v_xor_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_xor_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x11,0x01,0xff
+# GFX11: v_xor_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x11,0x01,0xff]
 
-# GFX11: v_xor_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x1f,0x01,0xff
+# GFX11: v_xor_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_xor_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x21,0x01,0xff
+# GFX11: v_xor_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x21,0x01,0xff]
 
-# GFX11: v_xor_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x2f,0x01,0xff
+# GFX11: v_xor_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_xor_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x50,0x01,0xff
+# GFX11: v_xor_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x50,0x01,0xff]
 
-# GFX11: v_xor_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x3a,0x01,0x5f,0x01,0x01
+# GFX11: v_xor_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_xor_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x3a,0x01,0x60,0x01,0x13
+# GFX11: v_xor_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x60,0x01,0x13]
 
-# GFX11: v_xor_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x3b,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x3b,0xff,0x6f,0x0d,0x30
+# GFX11: v_xor_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x3b,0xff,0x6f,0x0d,0x30]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp8.txt
index 5f1d4d4b33cb..a1d2c34f09f2 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp8.txt
@@ -1,250 +1,267 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64 %s
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,GFX11-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64,GFX11-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,GFX11-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64,GFX11-FAKE16 %s
 
+0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05]
-0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05
 
+0xea,0xfe,0xff,0x41,0xff,0x00,0x00,0x00
 # W32: v_add_co_ci_u32_dpp v255, vcc_lo, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x41,0xff,0x00,0x00,0x00]
 # W64: v_add_co_ci_u32_dpp v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x41,0xff,0x00,0x00,0x00]
-0xea,0xfe,0xff,0x41,0xff,0x00,0x00,0x00
 
-# GFX11: v_add_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00
+# GFX11-REAL16: v_add_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_add_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
 
-# GFX11: v_add_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x06,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x06,0x01,0x77,0x39,0x05
+# GFX11: v_add_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x06,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x07,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x07,0xff,0x00,0x00,0x00
+# GFX11: v_add_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x07,0xff,0x00,0x00,0x00]
 
-# GFX11: v_add_nc_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x4a,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x4a,0x01,0x77,0x39,0x05
+# GFX11: v_add_nc_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x4a,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add_nc_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x4b,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x4b,0xff,0x00,0x00,0x00
+# GFX11: v_add_nc_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x4b,0xff,0x00,0x00,0x00]
 
-# GFX11: v_and_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x36,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x36,0x01,0x77,0x39,0x05
+# GFX11: v_and_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x36,0x01,0x77,0x39,0x05]
 
-# GFX11: v_and_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x37,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x37,0xff,0x00,0x00,0x00
+# GFX11: v_and_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x37,0xff,0x00,0x00,0x00]
 
-# GFX11: v_ashrrev_i32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x34,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x34,0x01,0x77,0x39,0x05
+# GFX11: v_ashrrev_i32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x34,0x01,0x77,0x39,0x05]
 
-# GFX11: v_ashrrev_i32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x35,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x35,0xff,0x00,0x00,0x00
+# GFX11: v_ashrrev_i32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x35,0xff,0x00,0x00,0x00]
 
+0xe9,0x04,0x0a,0x02,0x01,0x77,0x39,0x05
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x02,0x01,0x77,0x39,0x05]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x02,0x01,0x77,0x39,0x05]
-0xe9,0x04,0x0a,0x02,0x01,0x77,0x39,0x05
 
+0xea,0xfe,0xff,0x03,0xff,0x00,0x00,0x00
 # W32: v_cndmask_b32_dpp v255, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x03,0xff,0x00,0x00,0x00]
 # W64: v_cndmask_b32_dpp v255, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x03,0xff,0x00,0x00,0x00]
-0xea,0xfe,0xff,0x03,0xff,0x00,0x00,0x00
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x5e,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x5e,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x5e,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_rtz_f16_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x5f,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x5f,0xff,0x00,0x00,0x00
+# GFX11: v_cvt_pk_rtz_f16_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x5f,0xff,0x00,0x00,0x00]
 
-# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x04,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_dot2acc_f32_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_dot2acc_f32_f16_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x05,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x05,0xff,0x00,0x00,0x00
+# GFX11: v_dot2acc_f32_f16_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x05,0xff,0x00,0x00,0x00]
 
-# GFX11: v_fmac_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05
+# GFX11: v_fmac_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fmac_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x6c,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x6c,0x7f,0x00,0x00,0x00
+# GFX11: v_fmac_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x6c,0x7f,0x00,0x00,0x00]
 
-# GFX11: v_fmac_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x56,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x56,0x01,0x77,0x39,0x05
+# GFX11: v_fmac_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x56,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fmac_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x57,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x57,0xff,0x00,0x00,0x00
+# GFX11: v_fmac_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x57,0xff,0x00,0x00,0x00]
 
-# GFX11: v_ldexp_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x76,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x76,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x76,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x76,0x01,0x77,0x39,0x05]
 
-# GFX11: v_ldexp_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x76,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x76,0x7f,0x00,0x00,0x00
+# GFX11-REAL16: v_ldexp_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x76,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_ldexp_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x76,0x7f,0x00,0x00,0x00]
 
-# GFX11: v_lshlrev_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x30,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x30,0x01,0x77,0x39,0x05
+# GFX11: v_lshlrev_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x30,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshlrev_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x31,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x31,0xff,0x00,0x00,0x00
+# GFX11: v_lshlrev_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x31,0xff,0x00,0x00,0x00]
 
-# GFX11: v_lshrrev_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x32,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x32,0x01,0x77,0x39,0x05
+# GFX11: v_lshrrev_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x32,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshrrev_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x33,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x33,0xff,0x00,0x00,0x00
+# GFX11: v_lshrrev_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x33,0xff,0x00,0x00,0x00]
 
-# GFX11: v_max_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x72,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x72,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_max_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x72,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_max_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x72,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x72,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x72,0x7f,0x00,0x00,0x00
+# GFX11-REAL16: v_max_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x72,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_max_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x72,0x7f,0x00,0x00,0x00]
 
-# GFX11: v_max_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x20,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x20,0x01,0x77,0x39,0x05
+# GFX11: v_max_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x20,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x21,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x21,0xff,0x00,0x00,0x00
+# GFX11: v_max_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x21,0xff,0x00,0x00,0x00]
 
-# GFX11: v_max_i32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x24,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x24,0x01,0x77,0x39,0x05
+# GFX11: v_max_i32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x24,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max_i32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x25,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x25,0xff,0x00,0x00,0x00
+# GFX11: v_max_i32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x25,0xff,0x00,0x00,0x00]
 
-# GFX11: v_max_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x28,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x28,0x01,0x77,0x39,0x05
+# GFX11: v_max_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x28,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x29,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x29,0xff,0x00,0x00,0x00
+# GFX11: v_max_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x29,0xff,0x00,0x00,0x00]
 
-# GFX11: v_min_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x74,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x74,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_min_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x74,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_min_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x74,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x74,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x74,0x7f,0x00,0x00,0x00
+# GFX11-REAL16: v_min_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x74,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_min_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x74,0x7f,0x00,0x00,0x00]
 
-# GFX11: v_min_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x1e,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x1e,0x01,0x77,0x39,0x05
+# GFX11: v_min_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x1e,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x1f,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x1f,0xff,0x00,0x00,0x00
+# GFX11: v_min_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x1f,0xff,0x00,0x00,0x00]
 
-# GFX11: v_min_i32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x22,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x22,0x01,0x77,0x39,0x05
+# GFX11: v_min_i32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x22,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min_i32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x23,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x23,0xff,0x00,0x00,0x00
+# GFX11: v_min_i32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x23,0xff,0x00,0x00,0x00]
 
-# GFX11: v_min_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x26,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x26,0x01,0x77,0x39,0x05
+# GFX11: v_min_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x26,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x27,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x27,0xff,0x00,0x00,0x00
+# GFX11: v_min_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x27,0xff,0x00,0x00,0x00]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x0e,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x0e,0x01,0x77,0x39,0x05
+# GFX11: v_mul_dx9_zero_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x0e,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mul_dx9_zero_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x0f,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x0f,0xff,0x00,0x00,0x00
+# GFX11: v_mul_dx9_zero_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x0f,0xff,0x00,0x00,0x00]
 
-# GFX11: v_mul_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x6a,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x6a,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x6a,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x6a,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mul_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x6a,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x6a,0x7f,0x00,0x00,0x00
+# GFX11-REAL16: v_mul_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x6a,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_mul_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x6a,0x7f,0x00,0x00,0x00]
 
-# GFX11: v_mul_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x10,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x10,0x01,0x77,0x39,0x05
+# GFX11: v_mul_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x10,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mul_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x11,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x11,0xff,0x00,0x00,0x00
+# GFX11: v_mul_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x11,0xff,0x00,0x00,0x00]
 
-# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x14,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x14,0x01,0x77,0x39,0x05
+# GFX11: v_mul_hi_i32_i24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x14,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mul_hi_i32_i24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x15,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x15,0xff,0x00,0x00,0x00
+# GFX11: v_mul_hi_i32_i24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x15,0xff,0x00,0x00,0x00]
 
-# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x18,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x18,0x01,0x77,0x39,0x05
+# GFX11: v_mul_hi_u32_u24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x18,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mul_hi_u32_u24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x19,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x19,0xff,0x00,0x00,0x00
+# GFX11: v_mul_hi_u32_u24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x19,0xff,0x00,0x00,0x00]
 
-# GFX11: v_mul_i32_i24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x12,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x12,0x01,0x77,0x39,0x05
+# GFX11: v_mul_i32_i24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x12,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mul_i32_i24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x13,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x13,0xff,0x00,0x00,0x00
+# GFX11: v_mul_i32_i24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x13,0xff,0x00,0x00,0x00]
 
-# GFX11: v_mul_u32_u24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x16,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x16,0x01,0x77,0x39,0x05
+# GFX11: v_mul_u32_u24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x16,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mul_u32_u24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x17,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x17,0xff,0x00,0x00,0x00
+# GFX11: v_mul_u32_u24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x17,0xff,0x00,0x00,0x00]
 
-# GFX11: v_or_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x38,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x38,0x01,0x77,0x39,0x05
+# GFX11: v_or_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x38,0x01,0x77,0x39,0x05]
 
-# GFX11: v_or_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x39,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x39,0xff,0x00,0x00,0x00
+# GFX11: v_or_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x39,0xff,0x00,0x00,0x00]
 
+0xe9,0x04,0x0a,0x42,0x01,0x77,0x39,0x05
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x42,0x01,0x77,0x39,0x05]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x42,0x01,0x77,0x39,0x05]
-0xe9,0x04,0x0a,0x42,0x01,0x77,0x39,0x05
 
+0xea,0xfe,0xff,0x43,0xff,0x00,0x00,0x00
 # W32: v_sub_co_ci_u32_dpp v255, vcc_lo, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x43,0xff,0x00,0x00,0x00]
 # W64: v_sub_co_ci_u32_dpp v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x43,0xff,0x00,0x00,0x00]
-0xea,0xfe,0xff,0x43,0xff,0x00,0x00,0x00
 
-# GFX11: v_sub_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sub_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00
+# GFX11-REAL16: v_sub_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_sub_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
 
-# GFX11: v_sub_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x08,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x08,0x01,0x77,0x39,0x05
+# GFX11: v_sub_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x08,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sub_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x09,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x09,0xff,0x00,0x00,0x00
+# GFX11: v_sub_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x09,0xff,0x00,0x00,0x00]
 
-# GFX11: v_sub_nc_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x4c,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x4c,0x01,0x77,0x39,0x05
+# GFX11: v_sub_nc_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x4c,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sub_nc_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x4d,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x4d,0xff,0x00,0x00,0x00
+# GFX11: v_sub_nc_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x4d,0xff,0x00,0x00,0x00]
 
+0xe9,0x04,0x0a,0x44,0x01,0x77,0x39,0x05
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x44,0x01,0x77,0x39,0x05]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x44,0x01,0x77,0x39,0x05]
-0xe9,0x04,0x0a,0x44,0x01,0x77,0x39,0x05
 
+0xea,0xfe,0xff,0x45,0xff,0x00,0x00,0x00
 # W32: v_subrev_co_ci_u32_dpp v255, vcc_lo, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x45,0xff,0x00,0x00,0x00]
 # W64: v_subrev_co_ci_u32_dpp v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x45,0xff,0x00,0x00,0x00]
-0xea,0xfe,0xff,0x45,0xff,0x00,0x00,0x00
 
-# GFX11: v_subrev_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x68,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x68,0x01,0x77,0x39,0x05
+# GFX11-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x68,0x01,0x77,0x39,0x05]
+# GFX11-FAKE16: v_subrev_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x68,0x01,0x77,0x39,0x05]
 
-# GFX11: v_subrev_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x68,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x68,0x7f,0x00,0x00,0x00
+# GFX11-REAL16: v_subrev_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x68,0x7f,0x00,0x00,0x00]
+# GFX11-FAKE16: v_subrev_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x68,0x7f,0x00,0x00,0x00]
 
-# GFX11: v_subrev_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x0a,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x0a,0x01,0x77,0x39,0x05
+# GFX11: v_subrev_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x0a,0x01,0x77,0x39,0x05]
 
-# GFX11: v_subrev_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x0b,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x0b,0xff,0x00,0x00,0x00
+# GFX11: v_subrev_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x0b,0xff,0x00,0x00,0x00]
 
-# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x4e,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x4e,0x01,0x77,0x39,0x05
+# GFX11: v_subrev_nc_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x4e,0x01,0x77,0x39,0x05]
 
-# GFX11: v_subrev_nc_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x4f,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x4f,0xff,0x00,0x00,0x00
+# GFX11: v_subrev_nc_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x4f,0xff,0x00,0x00,0x00]
 
-# GFX11: v_xnor_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x3c,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x3c,0x01,0x77,0x39,0x05
+# GFX11: v_xnor_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x3c,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xnor_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x3d,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x3d,0xff,0x00,0x00,0x00
+# GFX11: v_xnor_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x3d,0xff,0x00,0x00,0x00]
 
-# GFX11: v_xor_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x3a,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x3a,0x01,0x77,0x39,0x05
+# GFX11: v_xor_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x3a,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xor_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x3b,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x3b,0xff,0x00,0x00,0x00
+# GFX11: v_xor_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x3b,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2.txt
index 673db0664fc6..1276d898160b 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2.txt
@@ -1,2228 +1,2336 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,GFX12-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,GFX12-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,GFX12-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,GFX12-FAKE16 %s
 
+0x01,0x05,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, v1, v2, vcc_lo ; encoding: [0x01,0x05,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, v1, v2, vcc ; encoding: [0x01,0x05,0x0a,0x40]
-0x01,0x05,0x0a,0x40
 
+0xff,0x05,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, v255, v2, vcc_lo ; encoding: [0xff,0x05,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, v255, v2, vcc ; encoding: [0xff,0x05,0x0a,0x40]
-0xff,0x05,0x0a,0x40
 
+0x01,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, s1, v2, vcc_lo ; encoding: [0x01,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, s1, v2, vcc ; encoding: [0x01,0x04,0x0a,0x40]
-0x01,0x04,0x0a,0x40
 
+0x69,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, s105, v2, vcc_lo ; encoding: [0x69,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, s105, v2, vcc ; encoding: [0x69,0x04,0x0a,0x40]
-0x69,0x04,0x0a,0x40
 
+0x6a,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, vcc_lo, v2, vcc_lo ; encoding: [0x6a,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, vcc_lo, v2, vcc ; encoding: [0x6a,0x04,0x0a,0x40]
-0x6a,0x04,0x0a,0x40
 
+0x6b,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, vcc_hi, v2, vcc_lo ; encoding: [0x6b,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, vcc_hi, v2, vcc ; encoding: [0x6b,0x04,0x0a,0x40]
-0x6b,0x04,0x0a,0x40
 
+0x7b,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, ttmp15, v2, vcc_lo ; encoding: [0x7b,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, ttmp15, v2, vcc ; encoding: [0x7b,0x04,0x0a,0x40]
-0x7b,0x04,0x0a,0x40
 
+0x7d,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, m0, v2, vcc_lo ; encoding: [0x7d,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, m0, v2, vcc ; encoding: [0x7d,0x04,0x0a,0x40]
-0x7d,0x04,0x0a,0x40
 
+0x7e,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, exec_lo, v2, vcc_lo ; encoding: [0x7e,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, exec_lo, v2, vcc ; encoding: [0x7e,0x04,0x0a,0x40]
-0x7e,0x04,0x0a,0x40
 
+0x7f,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, exec_hi, v2, vcc_lo ; encoding: [0x7f,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, exec_hi, v2, vcc ; encoding: [0x7f,0x04,0x0a,0x40]
-0x7f,0x04,0x0a,0x40
 
+0x7c,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, null, v2, vcc_lo ; encoding: [0x7c,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, null, v2, vcc ; encoding: [0x7c,0x04,0x0a,0x40]
-0x7c,0x04,0x0a,0x40
 
+0xc1,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, -1, v2, vcc_lo ; encoding: [0xc1,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, -1, v2, vcc ; encoding: [0xc1,0x04,0x0a,0x40]
-0xc1,0x04,0x0a,0x40
 
+0xf0,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, 0.5, v2, vcc_lo ; encoding: [0xf0,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, 0.5, v2, vcc ; encoding: [0xf0,0x04,0x0a,0x40]
-0xf0,0x04,0x0a,0x40
 
+0xfd,0x04,0x0a,0x40
 # W32: v_add_co_ci_u32_e32 v5, vcc_lo, src_scc, v2, vcc_lo ; encoding: [0xfd,0x04,0x0a,0x40]
 # W64: v_add_co_ci_u32_e32 v5, vcc, src_scc, v2, vcc ; encoding: [0xfd,0x04,0x0a,0x40]
-0xfd,0x04,0x0a,0x40
 
+0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf
 # W32: v_add_co_ci_u32_e32 v255, vcc_lo, 0xaf123456, v255, vcc_lo ; encoding: [0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf]
 # W64: v_add_co_ci_u32_e32 v255, vcc, 0xaf123456, v255, vcc ; encoding: [0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf]
-0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf
 
-# GFX12: v_add_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x64]
 0x01,0x05,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, v1.l, v2.l          ; encoding: [0x01,0x05,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x64]
 
-# GFX12: v_add_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x64]
 0x7f,0x05,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, v127.l, v2.l        ; encoding: [0x7f,0x05,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x64]
 
-# GFX12: v_add_f16_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x64]
 0x01,0x04,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, s1, v2.l            ; encoding: [0x01,0x04,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x64]
 
-# GFX12: v_add_f16_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x64]
 0x69,0x04,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, s105, v2.l          ; encoding: [0x69,0x04,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x64]
 
-# GFX12: v_add_f16_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x64]
 0x6a,0x04,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, vcc_lo, v2.l        ; encoding: [0x6a,0x04,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x64]
 
-# GFX12: v_add_f16_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x64]
 0x6b,0x04,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, vcc_hi, v2.l        ; encoding: [0x6b,0x04,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x64]
 
-# GFX12: v_add_f16_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x64]
 0x7b,0x04,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, ttmp15, v2.l        ; encoding: [0x7b,0x04,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x64]
 
-# GFX12: v_add_f16_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x64]
 0x7d,0x04,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, m0, v2.l            ; encoding: [0x7d,0x04,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x64]
 
-# GFX12: v_add_f16_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x64]
 0x7e,0x04,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, exec_lo, v2.l       ; encoding: [0x7e,0x04,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x64]
 
-# GFX12: v_add_f16_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x64]
 0x7f,0x04,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, exec_hi, v2.l       ; encoding: [0x7f,0x04,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x64]
 
-# GFX12: v_add_f16_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x64]
 0x7c,0x04,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, null, v2.l          ; encoding: [0x7c,0x04,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x64]
 
-# GFX12: v_add_f16_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x64]
 0xc1,0x04,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, -1, v2.l            ; encoding: [0xc1,0x04,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x64]
 
-# GFX12: v_add_f16_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x64]
 0xf0,0x04,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, 0.5, v2.l           ; encoding: [0xf0,0x04,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x64]
 
-# GFX12: v_add_f16_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x64]
 0xfd,0x04,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, src_scc, v2.l       ; encoding: [0xfd,0x04,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x64]
 
-# GFX12: v_add_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_add_f16_e32 v127.l, 0xfe0b, v127.l    ; encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_add_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_add_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x06]
 0x01,0x05,0x0a,0x06
+# GFX12: v_add_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x06]
 
-# GFX12: v_add_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x06]
 0xff,0x05,0x0a,0x06
+# GFX12: v_add_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x06]
 
-# GFX12: v_add_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x06]
 0x01,0x04,0x0a,0x06
+# GFX12: v_add_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x06]
 
-# GFX12: v_add_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x06]
 0x69,0x04,0x0a,0x06
+# GFX12: v_add_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x06]
 
-# GFX12: v_add_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x06]
 0x6a,0x04,0x0a,0x06
+# GFX12: v_add_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x06]
 
-# GFX12: v_add_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x06]
 0x6b,0x04,0x0a,0x06
+# GFX12: v_add_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x06]
 
-# GFX12: v_add_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x06]
 0x7b,0x04,0x0a,0x06
+# GFX12: v_add_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x06]
 
-# GFX12: v_add_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x06]
 0x7d,0x04,0x0a,0x06
+# GFX12: v_add_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x06]
 
-# GFX12: v_add_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x06]
 0x7e,0x04,0x0a,0x06
+# GFX12: v_add_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x06]
 
-# GFX12: v_add_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x06]
 0x7f,0x04,0x0a,0x06
+# GFX12: v_add_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x06]
 
-# GFX12: v_add_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x06]
 0x7c,0x04,0x0a,0x06
+# GFX12: v_add_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x06]
 
-# GFX12: v_add_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x06]
 0xc1,0x04,0x0a,0x06
+# GFX12: v_add_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x06]
 
-# GFX12: v_add_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x06]
 0xf0,0x04,0x0a,0x06
+# GFX12: v_add_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x06]
 
-# GFX12: v_add_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x06]
 0xfd,0x04,0x0a,0x06
+# GFX12: v_add_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x06]
 
-# GFX12: v_add_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x07,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_add_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_add_f64_e32 v[5:6], v[1:2], v[3:4]    ; encoding: [0x01,0x07,0x0a,0x04]
 0x01,0x07,0x0a,0x04
+# GFX12: v_add_f64_e32 v[5:6], v[1:2], v[3:4]    ; encoding: [0x01,0x07,0x0a,0x04]
 
-# GFX12: v_add_f64_e32 v[5:6], v[254:255], v[2:3] ; encoding: [0xfe,0x05,0x0a,0x04]
 0xfe,0x05,0x0a,0x04
+# GFX12: v_add_f64_e32 v[5:6], v[254:255], v[2:3] ; encoding: [0xfe,0x05,0x0a,0x04]
 
-# GFX12: v_add_f64_e32 v[5:6], s[0:1], v[2:3]    ; encoding: [0x00,0x04,0x0a,0x04]
 0x00,0x04,0x0a,0x04
+# GFX12: v_add_f64_e32 v[5:6], s[0:1], v[2:3]    ; encoding: [0x00,0x04,0x0a,0x04]
 
-# GFX12: v_add_f64_e32 v[5:6], s[104:105], v[2:3] ; encoding: [0x68,0x04,0x0a,0x04]
 0x68,0x04,0x0a,0x04
+# GFX12: v_add_f64_e32 v[5:6], s[104:105], v[2:3] ; encoding: [0x68,0x04,0x0a,0x04]
 
-# GFX12: v_add_f64_e32 v[5:6], vcc, v[2:3]       ; encoding: [0x6a,0x04,0x0a,0x04]
 0x6a,0x04,0x0a,0x04
+# GFX12: v_add_f64_e32 v[5:6], vcc, v[2:3]       ; encoding: [0x6a,0x04,0x0a,0x04]
 
-# GFX12: v_add_f64_e32 v[5:6], ttmp[14:15], v[2:3] ; encoding: [0x7a,0x04,0x0a,0x04]
 0x7a,0x04,0x0a,0x04
+# GFX12: v_add_f64_e32 v[5:6], ttmp[14:15], v[2:3] ; encoding: [0x7a,0x04,0x0a,0x04]
 
-# GFX12: v_add_f64_e32 v[5:6], exec, v[2:3]      ; encoding: [0x7e,0x04,0x0a,0x04]
 0x7e,0x04,0x0a,0x04
+# GFX12: v_add_f64_e32 v[5:6], exec, v[2:3]      ; encoding: [0x7e,0x04,0x0a,0x04]
 
-# GFX12: v_add_f64_e32 v[5:6], null, v[2:3]      ; encoding: [0x7c,0x04,0x0a,0x04]
 0x7c,0x04,0x0a,0x04
+# GFX12: v_add_f64_e32 v[5:6], null, v[2:3]      ; encoding: [0x7c,0x04,0x0a,0x04]
 
-# GFX12: v_add_f64_e32 v[5:6], -1, v[2:3]        ; encoding: [0xc1,0x04,0x0a,0x04]
 0xc1,0x04,0x0a,0x04
+# GFX12: v_add_f64_e32 v[5:6], -1, v[2:3]        ; encoding: [0xc1,0x04,0x0a,0x04]
 
-# GFX12: v_add_f64_e32 v[5:6], 0.5, v[2:3]       ; encoding: [0xf0,0x04,0x0a,0x04]
 0xf0,0x04,0x0a,0x04
+# GFX12: v_add_f64_e32 v[5:6], 0.5, v[2:3]       ; encoding: [0xf0,0x04,0x0a,0x04]
 
-# GFX12: v_add_f64_e32 v[5:6], src_scc, v[2:3]   ; encoding: [0xfd,0x04,0x0a,0x04]
 0xfd,0x04,0x0a,0x04
+# GFX12: v_add_f64_e32 v[5:6], src_scc, v[2:3]   ; encoding: [0xfd,0x04,0x0a,0x04]
 
-# GFX12: v_add_f64_e32 v[254:255], 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xfd,0x05,0x56,0x34,0x12,0xaf]
 0xff,0xfc,0xfd,0x05,0x56,0x34,0x12,0xaf
+# GFX12: v_add_f64_e32 v[254:255], 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xfd,0x05,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_add_nc_u32_e32 v5, v1, v2             ; encoding: [0x01,0x05,0x0a,0x4a]
 0x01,0x05,0x0a,0x4a
+# GFX12: v_add_nc_u32_e32 v5, v1, v2             ; encoding: [0x01,0x05,0x0a,0x4a]
 
-# GFX12: v_add_nc_u32_e32 v5, v255, v2           ; encoding: [0xff,0x05,0x0a,0x4a]
 0xff,0x05,0x0a,0x4a
+# GFX12: v_add_nc_u32_e32 v5, v255, v2           ; encoding: [0xff,0x05,0x0a,0x4a]
 
-# GFX12: v_add_nc_u32_e32 v5, s1, v2             ; encoding: [0x01,0x04,0x0a,0x4a]
 0x01,0x04,0x0a,0x4a
+# GFX12: v_add_nc_u32_e32 v5, s1, v2             ; encoding: [0x01,0x04,0x0a,0x4a]
 
-# GFX12: v_add_nc_u32_e32 v5, s105, v2           ; encoding: [0x69,0x04,0x0a,0x4a]
 0x69,0x04,0x0a,0x4a
+# GFX12: v_add_nc_u32_e32 v5, s105, v2           ; encoding: [0x69,0x04,0x0a,0x4a]
 
-# GFX12: v_add_nc_u32_e32 v5, vcc_lo, v2         ; encoding: [0x6a,0x04,0x0a,0x4a]
 0x6a,0x04,0x0a,0x4a
+# GFX12: v_add_nc_u32_e32 v5, vcc_lo, v2         ; encoding: [0x6a,0x04,0x0a,0x4a]
 
-# GFX12: v_add_nc_u32_e32 v5, vcc_hi, v2         ; encoding: [0x6b,0x04,0x0a,0x4a]
 0x6b,0x04,0x0a,0x4a
+# GFX12: v_add_nc_u32_e32 v5, vcc_hi, v2         ; encoding: [0x6b,0x04,0x0a,0x4a]
 
-# GFX12: v_add_nc_u32_e32 v5, ttmp15, v2         ; encoding: [0x7b,0x04,0x0a,0x4a]
 0x7b,0x04,0x0a,0x4a
+# GFX12: v_add_nc_u32_e32 v5, ttmp15, v2         ; encoding: [0x7b,0x04,0x0a,0x4a]
 
-# GFX12: v_add_nc_u32_e32 v5, m0, v2             ; encoding: [0x7d,0x04,0x0a,0x4a]
 0x7d,0x04,0x0a,0x4a
+# GFX12: v_add_nc_u32_e32 v5, m0, v2             ; encoding: [0x7d,0x04,0x0a,0x4a]
 
-# GFX12: v_add_nc_u32_e32 v5, exec_lo, v2        ; encoding: [0x7e,0x04,0x0a,0x4a]
 0x7e,0x04,0x0a,0x4a
+# GFX12: v_add_nc_u32_e32 v5, exec_lo, v2        ; encoding: [0x7e,0x04,0x0a,0x4a]
 
-# GFX12: v_add_nc_u32_e32 v5, exec_hi, v2        ; encoding: [0x7f,0x04,0x0a,0x4a]
 0x7f,0x04,0x0a,0x4a
+# GFX12: v_add_nc_u32_e32 v5, exec_hi, v2        ; encoding: [0x7f,0x04,0x0a,0x4a]
 
-# GFX12: v_add_nc_u32_e32 v5, null, v2           ; encoding: [0x7c,0x04,0x0a,0x4a]
 0x7c,0x04,0x0a,0x4a
+# GFX12: v_add_nc_u32_e32 v5, null, v2           ; encoding: [0x7c,0x04,0x0a,0x4a]
 
-# GFX12: v_add_nc_u32_e32 v5, -1, v2             ; encoding: [0xc1,0x04,0x0a,0x4a]
 0xc1,0x04,0x0a,0x4a
+# GFX12: v_add_nc_u32_e32 v5, -1, v2             ; encoding: [0xc1,0x04,0x0a,0x4a]
 
-# GFX12: v_add_nc_u32_e32 v5, 0.5, v2            ; encoding: [0xf0,0x04,0x0a,0x4a]
 0xf0,0x04,0x0a,0x4a
+# GFX12: v_add_nc_u32_e32 v5, 0.5, v2            ; encoding: [0xf0,0x04,0x0a,0x4a]
 
-# GFX12: v_add_nc_u32_e32 v5, src_scc, v2        ; encoding: [0xfd,0x04,0x0a,0x4a]
 0xfd,0x04,0x0a,0x4a
+# GFX12: v_add_nc_u32_e32 v5, src_scc, v2        ; encoding: [0xfd,0x04,0x0a,0x4a]
 
-# GFX12: v_add_nc_u32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x4b,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x4b,0x56,0x34,0x12,0xaf
+# GFX12: v_add_nc_u32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x4b,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_and_b32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x36]
 0x01,0x05,0x0a,0x36
+# GFX12: v_and_b32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x36]
 
-# GFX12: v_and_b32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x36]
 0xff,0x05,0x0a,0x36
+# GFX12: v_and_b32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x36]
 
-# GFX12: v_and_b32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x36]
 0x01,0x04,0x0a,0x36
+# GFX12: v_and_b32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x36]
 
-# GFX12: v_and_b32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x36]
 0x69,0x04,0x0a,0x36
+# GFX12: v_and_b32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x36]
 
-# GFX12: v_and_b32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x36]
 0x6a,0x04,0x0a,0x36
+# GFX12: v_and_b32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x36]
 
-# GFX12: v_and_b32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x36]
 0x6b,0x04,0x0a,0x36
+# GFX12: v_and_b32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x36]
 
-# GFX12: v_and_b32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x36]
 0x7b,0x04,0x0a,0x36
+# GFX12: v_and_b32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x36]
 
-# GFX12: v_and_b32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x36]
 0x7d,0x04,0x0a,0x36
+# GFX12: v_and_b32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x36]
 
-# GFX12: v_and_b32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x36]
 0x7e,0x04,0x0a,0x36
+# GFX12: v_and_b32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x36]
 
-# GFX12: v_and_b32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x36]
 0x7f,0x04,0x0a,0x36
+# GFX12: v_and_b32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x36]
 
-# GFX12: v_and_b32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x36]
 0x7c,0x04,0x0a,0x36
+# GFX12: v_and_b32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x36]
 
-# GFX12: v_and_b32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x36]
 0xc1,0x04,0x0a,0x36
+# GFX12: v_and_b32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x36]
 
-# GFX12: v_and_b32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x36]
 0xf0,0x04,0x0a,0x36
+# GFX12: v_and_b32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x36]
 
-# GFX12: v_and_b32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x36]
 0xfd,0x04,0x0a,0x36
+# GFX12: v_and_b32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x36]
 
-# GFX12: v_and_b32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x37,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x37,0x56,0x34,0x12,0xaf
+# GFX12: v_and_b32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x37,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_ashrrev_i32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x34]
 0x01,0x05,0x0a,0x34
+# GFX12: v_ashrrev_i32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x34]
 
-# GFX12: v_ashrrev_i32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x34]
 0xff,0x05,0x0a,0x34
+# GFX12: v_ashrrev_i32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x34]
 
-# GFX12: v_ashrrev_i32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x34]
 0x01,0x04,0x0a,0x34
+# GFX12: v_ashrrev_i32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x34]
 
-# GFX12: v_ashrrev_i32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x34]
 0x69,0x04,0x0a,0x34
+# GFX12: v_ashrrev_i32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x34]
 
-# GFX12: v_ashrrev_i32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x34]
 0x6a,0x04,0x0a,0x34
+# GFX12: v_ashrrev_i32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x34]
 
-# GFX12: v_ashrrev_i32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x34]
 0x6b,0x04,0x0a,0x34
+# GFX12: v_ashrrev_i32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x34]
 
-# GFX12: v_ashrrev_i32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x34]
 0x7b,0x04,0x0a,0x34
+# GFX12: v_ashrrev_i32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x34]
 
-# GFX12: v_ashrrev_i32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x34]
 0x7d,0x04,0x0a,0x34
+# GFX12: v_ashrrev_i32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x34]
 
-# GFX12: v_ashrrev_i32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x34]
 0x7e,0x04,0x0a,0x34
+# GFX12: v_ashrrev_i32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x34]
 
-# GFX12: v_ashrrev_i32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x34]
 0x7f,0x04,0x0a,0x34
+# GFX12: v_ashrrev_i32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x34]
 
-# GFX12: v_ashrrev_i32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x34]
 0x7c,0x04,0x0a,0x34
+# GFX12: v_ashrrev_i32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x34]
 
-# GFX12: v_ashrrev_i32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x34]
 0xc1,0x04,0x0a,0x34
+# GFX12: v_ashrrev_i32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x34]
 
-# GFX12: v_ashrrev_i32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x34]
 0xf0,0x04,0x0a,0x34
+# GFX12: v_ashrrev_i32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x34]
 
-# GFX12: v_ashrrev_i32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x34]
 0xfd,0x04,0x0a,0x34
+# GFX12: v_ashrrev_i32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x34]
 
-# GFX12: v_ashrrev_i32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x35,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x35,0x56,0x34,0x12,0xaf
+# GFX12: v_ashrrev_i32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x35,0x56,0x34,0x12,0xaf]
 
-# W32: v_cndmask_b32_e32 v5, v1, v2, vcc_lo      ; encoding: [0x01,0x05,0x0a,0x02]
-# W64: v_cndmask_b32_e32 v5, v1, v2, vcc         ; encoding: [0x01,0x05,0x0a,0x02]
 0x01,0x05,0x0a,0x02
+# W32: v_cndmask_b32_e32 v5, v1, v2, vcc_lo    ; encoding: [0x01,0x05,0x0a,0x02]
+# W64: v_cndmask_b32_e32 v5, v1, v2, vcc       ; encoding: [0x01,0x05,0x0a,0x02]
 
-# W32: v_cndmask_b32_e32 v5, v255, v2, vcc_lo    ; encoding: [0xff,0x05,0x0a,0x02]
-# W64: v_cndmask_b32_e32 v5, v255, v2, vcc       ; encoding: [0xff,0x05,0x0a,0x02]
 0xff,0x05,0x0a,0x02
+# W32: v_cndmask_b32_e32 v5, v255, v2, vcc_lo  ; encoding: [0xff,0x05,0x0a,0x02]
+# W64: v_cndmask_b32_e32 v5, v255, v2, vcc     ; encoding: [0xff,0x05,0x0a,0x02]
 
-# W32: v_cndmask_b32_e32 v5, s1, v2, vcc_lo      ; encoding: [0x01,0x04,0x0a,0x02]
-# W64: v_cndmask_b32_e32 v5, s1, v2, vcc         ; encoding: [0x01,0x04,0x0a,0x02]
 0x01,0x04,0x0a,0x02
+# W32: v_cndmask_b32_e32 v5, s1, v2, vcc_lo    ; encoding: [0x01,0x04,0x0a,0x02]
+# W64: v_cndmask_b32_e32 v5, s1, v2, vcc       ; encoding: [0x01,0x04,0x0a,0x02]
 
-# W32: v_cndmask_b32_e32 v5, s105, v2, vcc_lo    ; encoding: [0x69,0x04,0x0a,0x02]
-# W64: v_cndmask_b32_e32 v5, s105, v2, vcc       ; encoding: [0x69,0x04,0x0a,0x02]
 0x69,0x04,0x0a,0x02
+# W32: v_cndmask_b32_e32 v5, s105, v2, vcc_lo  ; encoding: [0x69,0x04,0x0a,0x02]
+# W64: v_cndmask_b32_e32 v5, s105, v2, vcc     ; encoding: [0x69,0x04,0x0a,0x02]
 
-# W32: v_cndmask_b32_e32 v5, vcc_lo, v2, vcc_lo  ; encoding: [0x6a,0x04,0x0a,0x02]
-# W64: v_cndmask_b32_e32 v5, vcc_lo, v2, vcc     ; encoding: [0x6a,0x04,0x0a,0x02]
 0x6a,0x04,0x0a,0x02
+# W32: v_cndmask_b32_e32 v5, vcc_lo, v2, vcc_lo ; encoding: [0x6a,0x04,0x0a,0x02]
+# W64: v_cndmask_b32_e32 v5, vcc_lo, v2, vcc   ; encoding: [0x6a,0x04,0x0a,0x02]
 
-# W32: v_cndmask_b32_e32 v5, vcc_hi, v2, vcc_lo  ; encoding: [0x6b,0x04,0x0a,0x02]
-# W64: v_cndmask_b32_e32 v5, vcc_hi, v2, vcc     ; encoding: [0x6b,0x04,0x0a,0x02]
 0x6b,0x04,0x0a,0x02
+# W32: v_cndmask_b32_e32 v5, vcc_hi, v2, vcc_lo ; encoding: [0x6b,0x04,0x0a,0x02]
+# W64: v_cndmask_b32_e32 v5, vcc_hi, v2, vcc   ; encoding: [0x6b,0x04,0x0a,0x02]
 
-# W32: v_cndmask_b32_e32 v5, ttmp15, v2, vcc_lo  ; encoding: [0x7b,0x04,0x0a,0x02]
-# W64: v_cndmask_b32_e32 v5, ttmp15, v2, vcc     ; encoding: [0x7b,0x04,0x0a,0x02]
 0x7b,0x04,0x0a,0x02
+# W32: v_cndmask_b32_e32 v5, ttmp15, v2, vcc_lo ; encoding: [0x7b,0x04,0x0a,0x02]
+# W64: v_cndmask_b32_e32 v5, ttmp15, v2, vcc   ; encoding: [0x7b,0x04,0x0a,0x02]
 
-# W32: v_cndmask_b32_e32 v5, m0, v2, vcc_lo      ; encoding: [0x7d,0x04,0x0a,0x02]
-# W64: v_cndmask_b32_e32 v5, m0, v2, vcc         ; encoding: [0x7d,0x04,0x0a,0x02]
 0x7d,0x04,0x0a,0x02
+# W32: v_cndmask_b32_e32 v5, m0, v2, vcc_lo    ; encoding: [0x7d,0x04,0x0a,0x02]
+# W64: v_cndmask_b32_e32 v5, m0, v2, vcc       ; encoding: [0x7d,0x04,0x0a,0x02]
 
-# W32: v_cndmask_b32_e32 v5, exec_lo, v2, vcc_lo ; encoding: [0x7e,0x04,0x0a,0x02]
-# W64: v_cndmask_b32_e32 v5, exec_lo, v2, vcc    ; encoding: [0x7e,0x04,0x0a,0x02]
 0x7e,0x04,0x0a,0x02
+# W32: v_cndmask_b32_e32 v5, exec_lo, v2, vcc_lo ; encoding: [0x7e,0x04,0x0a,0x02]
+# W64: v_cndmask_b32_e32 v5, exec_lo, v2, vcc  ; encoding: [0x7e,0x04,0x0a,0x02]
 
-# W32: v_cndmask_b32_e32 v5, exec_hi, v2, vcc_lo ; encoding: [0x7f,0x04,0x0a,0x02]
-# W64: v_cndmask_b32_e32 v5, exec_hi, v2, vcc    ; encoding: [0x7f,0x04,0x0a,0x02]
 0x7f,0x04,0x0a,0x02
+# W32: v_cndmask_b32_e32 v5, exec_hi, v2, vcc_lo ; encoding: [0x7f,0x04,0x0a,0x02]
+# W64: v_cndmask_b32_e32 v5, exec_hi, v2, vcc  ; encoding: [0x7f,0x04,0x0a,0x02]
 
-# W32: v_cndmask_b32_e32 v5, null, v2, vcc_lo    ; encoding: [0x7c,0x04,0x0a,0x02]
-# W64: v_cndmask_b32_e32 v5, null, v2, vcc       ; encoding: [0x7c,0x04,0x0a,0x02]
 0x7c,0x04,0x0a,0x02
+# W32: v_cndmask_b32_e32 v5, null, v2, vcc_lo  ; encoding: [0x7c,0x04,0x0a,0x02]
+# W64: v_cndmask_b32_e32 v5, null, v2, vcc     ; encoding: [0x7c,0x04,0x0a,0x02]
 
-# W32: v_cndmask_b32_e32 v5, -1, v2, vcc_lo      ; encoding: [0xc1,0x04,0x0a,0x02]
-# W64: v_cndmask_b32_e32 v5, -1, v2, vcc         ; encoding: [0xc1,0x04,0x0a,0x02]
 0xc1,0x04,0x0a,0x02
+# W32: v_cndmask_b32_e32 v5, -1, v2, vcc_lo    ; encoding: [0xc1,0x04,0x0a,0x02]
+# W64: v_cndmask_b32_e32 v5, -1, v2, vcc       ; encoding: [0xc1,0x04,0x0a,0x02]
 
-# W32: v_cndmask_b32_e32 v5, 0.5, v2, vcc_lo     ; encoding: [0xf0,0x04,0x0a,0x02]
-# W64: v_cndmask_b32_e32 v5, 0.5, v2, vcc        ; encoding: [0xf0,0x04,0x0a,0x02]
 0xf0,0x04,0x0a,0x02
+# W32: v_cndmask_b32_e32 v5, 0.5, v2, vcc_lo   ; encoding: [0xf0,0x04,0x0a,0x02]
+# W64: v_cndmask_b32_e32 v5, 0.5, v2, vcc      ; encoding: [0xf0,0x04,0x0a,0x02]
 
-# W32: v_cndmask_b32_e32 v5, src_scc, v2, vcc_lo ; encoding: [0xfd,0x04,0x0a,0x02]
-# W64: v_cndmask_b32_e32 v5, src_scc, v2, vcc    ; encoding: [0xfd,0x04,0x0a,0x02]
 0xfd,0x04,0x0a,0x02
+# W32: v_cndmask_b32_e32 v5, src_scc, v2, vcc_lo ; encoding: [0xfd,0x04,0x0a,0x02]
+# W64: v_cndmask_b32_e32 v5, src_scc, v2, vcc  ; encoding: [0xfd,0x04,0x0a,0x02]
 
+0xff,0xfe,0xff,0x03,0x56,0x34,0x12,0xaf
 # W32: v_cndmask_b32_e32 v255, 0xaf123456, v255, vcc_lo ; encoding: [0xff,0xfe,0xff,0x03,0x56,0x34,0x12,0xaf]
 # W64: v_cndmask_b32_e32 v255, 0xaf123456, v255, vcc ; encoding: [0xff,0xfe,0xff,0x03,0x56,0x34,0x12,0xaf]
-0xff,0xfe,0xff,0x03,0x56,0x34,0x12,0xaf
 
-# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, v1, v2     ; encoding: [0x01,0x05,0x0a,0x5e]
 0x01,0x05,0x0a,0x5e
+# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, v1, v2     ; encoding: [0x01,0x05,0x0a,0x5e]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, v255, v2   ; encoding: [0xff,0x05,0x0a,0x5e]
 0xff,0x05,0x0a,0x5e
+# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, v255, v2   ; encoding: [0xff,0x05,0x0a,0x5e]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, s1, v2     ; encoding: [0x01,0x04,0x0a,0x5e]
 0x01,0x04,0x0a,0x5e
+# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, s1, v2     ; encoding: [0x01,0x04,0x0a,0x5e]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, s105, v2   ; encoding: [0x69,0x04,0x0a,0x5e]
 0x69,0x04,0x0a,0x5e
+# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, s105, v2   ; encoding: [0x69,0x04,0x0a,0x5e]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x5e]
 0x6a,0x04,0x0a,0x5e
+# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x5e]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x5e]
 0x6b,0x04,0x0a,0x5e
+# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x5e]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x5e]
 0x7b,0x04,0x0a,0x5e
+# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x5e]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, m0, v2     ; encoding: [0x7d,0x04,0x0a,0x5e]
 0x7d,0x04,0x0a,0x5e
+# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, m0, v2     ; encoding: [0x7d,0x04,0x0a,0x5e]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x5e]
 0x7e,0x04,0x0a,0x5e
+# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x5e]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x5e]
 0x7f,0x04,0x0a,0x5e
+# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x5e]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, null, v2   ; encoding: [0x7c,0x04,0x0a,0x5e]
 0x7c,0x04,0x0a,0x5e
+# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, null, v2   ; encoding: [0x7c,0x04,0x0a,0x5e]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, -1, v2     ; encoding: [0xc1,0x04,0x0a,0x5e]
 0xc1,0x04,0x0a,0x5e
+# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, -1, v2     ; encoding: [0xc1,0x04,0x0a,0x5e]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, 0.5, v2    ; encoding: [0xf0,0x04,0x0a,0x5e]
 0xf0,0x04,0x0a,0x5e
+# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, 0.5, v2    ; encoding: [0xf0,0x04,0x0a,0x5e]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x5e]
 0xfd,0x04,0x0a,0x5e
+# GFX12: v_cvt_pk_rtz_f16_f32_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x5e]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x5f,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x5f,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_rtz_f16_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x5f,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmaak_f16 v5, v1, v2, 0xfe0b          ; encoding: [0x01,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x01,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmaak_f16 v5, v1, v2, 0xfe0b          ; encoding: [0x01,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmaak_f16 v5, v127, v2, 0xfe0b        ; encoding: [0x7f,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x7f,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmaak_f16 v5, v127, v2, 0xfe0b        ; encoding: [0x7f,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmaak_f16 v5, s1, v2, 0xfe0b          ; encoding: [0x01,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x01,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmaak_f16 v5, s1, v2, 0xfe0b          ; encoding: [0x01,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmaak_f16 v5, s105, v2, 0xfe0b        ; encoding: [0x69,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x69,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmaak_f16 v5, s105, v2, 0xfe0b        ; encoding: [0x69,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmaak_f16 v5, vcc_lo, v2, 0xfe0b      ; encoding: [0x6a,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x6a,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmaak_f16 v5, vcc_lo, v2, 0xfe0b      ; encoding: [0x6a,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmaak_f16 v5, vcc_hi, v2, 0xfe0b      ; encoding: [0x6b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x6b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmaak_f16 v5, vcc_hi, v2, 0xfe0b      ; encoding: [0x6b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmaak_f16 v5, ttmp15, v2, 0xfe0b      ; encoding: [0x7b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x7b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmaak_f16 v5, ttmp15, v2, 0xfe0b      ; encoding: [0x7b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmaak_f16 v5, m0, v2, 0xfe0b          ; encoding: [0x7d,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x7d,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmaak_f16 v5, m0, v2, 0xfe0b          ; encoding: [0x7d,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmaak_f16 v5, exec_lo, v2, 0xfe0b     ; encoding: [0x7e,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x7e,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmaak_f16 v5, exec_lo, v2, 0xfe0b     ; encoding: [0x7e,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmaak_f16 v5, exec_hi, v2, 0xfe0b     ; encoding: [0x7f,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x7f,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmaak_f16 v5, exec_hi, v2, 0xfe0b     ; encoding: [0x7f,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmaak_f16 v5, null, v2, 0xfe0b        ; encoding: [0x7c,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0x7c,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmaak_f16 v5, null, v2, 0xfe0b        ; encoding: [0x7c,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmaak_f16 v5, -1, v2, 0xfe0b          ; encoding: [0xc1,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0xc1,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmaak_f16 v5, -1, v2, 0xfe0b          ; encoding: [0xc1,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmaak_f16 v5, 0.5, v2, 0xfe0b         ; encoding: [0xf0,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0xf0,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmaak_f16 v5, 0.5, v2, 0xfe0b         ; encoding: [0xf0,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmaak_f16 v5, src_scc, v2, 0xfe0b     ; encoding: [0xfd,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 0xfd,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmaak_f16 v5, src_scc, v2, 0xfe0b     ; encoding: [0xfd,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmaak_f16 v127, 0xfe0b, v127, 0xfe0b  ; encoding: [0xff,0xfe,0xfe,0x70,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xfe,0x70,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmaak_f16 v127, 0xfe0b, v127, 0xfe0b  ; encoding: [0xff,0xfe,0xfe,0x70,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmaak_f32 v5, v1, v2, 0xaf123456      ; encoding: [0x01,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x01,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX12: v_fmaak_f32 v5, v1, v2, 0xaf123456      ; encoding: [0x01,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmaak_f32 v5, v255, v2, 0xaf123456    ; encoding: [0xff,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0xff,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX12: v_fmaak_f32 v5, v255, v2, 0xaf123456    ; encoding: [0xff,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmaak_f32 v5, s1, v2, 0xaf123456      ; encoding: [0x01,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x01,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX12: v_fmaak_f32 v5, s1, v2, 0xaf123456      ; encoding: [0x01,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmaak_f32 v5, s105, v2, 0xaf123456    ; encoding: [0x69,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x69,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX12: v_fmaak_f32 v5, s105, v2, 0xaf123456    ; encoding: [0x69,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmaak_f32 v5, vcc_lo, v2, 0xaf123456  ; encoding: [0x6a,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x6a,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX12: v_fmaak_f32 v5, vcc_lo, v2, 0xaf123456  ; encoding: [0x6a,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmaak_f32 v5, vcc_hi, v2, 0xaf123456  ; encoding: [0x6b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x6b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX12: v_fmaak_f32 v5, vcc_hi, v2, 0xaf123456  ; encoding: [0x6b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmaak_f32 v5, ttmp15, v2, 0xaf123456  ; encoding: [0x7b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x7b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX12: v_fmaak_f32 v5, ttmp15, v2, 0xaf123456  ; encoding: [0x7b,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmaak_f32 v5, m0, v2, 0xaf123456      ; encoding: [0x7d,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x7d,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX12: v_fmaak_f32 v5, m0, v2, 0xaf123456      ; encoding: [0x7d,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmaak_f32 v5, exec_lo, v2, 0xaf123456 ; encoding: [0x7e,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x7e,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX12: v_fmaak_f32 v5, exec_lo, v2, 0xaf123456 ; encoding: [0x7e,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmaak_f32 v5, exec_hi, v2, 0xaf123456 ; encoding: [0x7f,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x7f,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX12: v_fmaak_f32 v5, exec_hi, v2, 0xaf123456 ; encoding: [0x7f,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmaak_f32 v5, null, v2, 0xaf123456    ; encoding: [0x7c,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0x7c,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX12: v_fmaak_f32 v5, null, v2, 0xaf123456    ; encoding: [0x7c,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmaak_f32 v5, -1, v2, 0xaf123456      ; encoding: [0xc1,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0xc1,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX12: v_fmaak_f32 v5, -1, v2, 0xaf123456      ; encoding: [0xc1,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmaak_f32 v5, 0.5, v2, 0xaf123456     ; encoding: [0xf0,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0xf0,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX12: v_fmaak_f32 v5, 0.5, v2, 0xaf123456     ; encoding: [0xf0,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmaak_f32 v5, src_scc, v2, 0xaf123456 ; encoding: [0xfd,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 0xfd,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf
+# GFX12: v_fmaak_f32 v5, src_scc, v2, 0xaf123456 ; encoding: [0xfd,0x04,0x0a,0x5a,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmaak_f32 v255, 0xaf123456, v255, 0xaf123456 ; encoding: [0xff,0xfe,0xff,0x5b,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x5b,0x56,0x34,0x12,0xaf
+# GFX12: v_fmaak_f32 v255, 0xaf123456, v255, 0xaf123456 ; encoding: [0xff,0xfe,0xff,0x5b,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmac_f16_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x6c]
 0x01,0x05,0x0a,0x6c
+# GFX12: v_fmac_f16_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x6c]
 
-# GFX12: v_fmac_f16_e32 v5, v127, v2             ; encoding: [0x7f,0x05,0x0a,0x6c]
 0x7f,0x05,0x0a,0x6c
+# GFX12: v_fmac_f16_e32 v5, v127, v2             ; encoding: [0x7f,0x05,0x0a,0x6c]
 
-# GFX12: v_fmac_f16_e32 v5, s1, v2               ; encoding: [0x01,0x04,0x0a,0x6c]
 0x01,0x04,0x0a,0x6c
+# GFX12: v_fmac_f16_e32 v5, s1, v2               ; encoding: [0x01,0x04,0x0a,0x6c]
 
-# GFX12: v_fmac_f16_e32 v5, s105, v2             ; encoding: [0x69,0x04,0x0a,0x6c]
 0x69,0x04,0x0a,0x6c
+# GFX12: v_fmac_f16_e32 v5, s105, v2             ; encoding: [0x69,0x04,0x0a,0x6c]
 
-# GFX12: v_fmac_f16_e32 v5, vcc_lo, v2           ; encoding: [0x6a,0x04,0x0a,0x6c]
 0x6a,0x04,0x0a,0x6c
+# GFX12: v_fmac_f16_e32 v5, vcc_lo, v2           ; encoding: [0x6a,0x04,0x0a,0x6c]
 
-# GFX12: v_fmac_f16_e32 v5, vcc_hi, v2           ; encoding: [0x6b,0x04,0x0a,0x6c]
 0x6b,0x04,0x0a,0x6c
+# GFX12: v_fmac_f16_e32 v5, vcc_hi, v2           ; encoding: [0x6b,0x04,0x0a,0x6c]
 
-# GFX12: v_fmac_f16_e32 v5, ttmp15, v2           ; encoding: [0x7b,0x04,0x0a,0x6c]
 0x7b,0x04,0x0a,0x6c
+# GFX12: v_fmac_f16_e32 v5, ttmp15, v2           ; encoding: [0x7b,0x04,0x0a,0x6c]
 
-# GFX12: v_fmac_f16_e32 v5, m0, v2               ; encoding: [0x7d,0x04,0x0a,0x6c]
 0x7d,0x04,0x0a,0x6c
+# GFX12: v_fmac_f16_e32 v5, m0, v2               ; encoding: [0x7d,0x04,0x0a,0x6c]
 
-# GFX12: v_fmac_f16_e32 v5, exec_lo, v2          ; encoding: [0x7e,0x04,0x0a,0x6c]
 0x7e,0x04,0x0a,0x6c
+# GFX12: v_fmac_f16_e32 v5, exec_lo, v2          ; encoding: [0x7e,0x04,0x0a,0x6c]
 
-# GFX12: v_fmac_f16_e32 v5, exec_hi, v2          ; encoding: [0x7f,0x04,0x0a,0x6c]
 0x7f,0x04,0x0a,0x6c
+# GFX12: v_fmac_f16_e32 v5, exec_hi, v2          ; encoding: [0x7f,0x04,0x0a,0x6c]
 
-# GFX12: v_fmac_f16_e32 v5, null, v2             ; encoding: [0x7c,0x04,0x0a,0x6c]
 0x7c,0x04,0x0a,0x6c
+# GFX12: v_fmac_f16_e32 v5, null, v2             ; encoding: [0x7c,0x04,0x0a,0x6c]
 
-# GFX12: v_fmac_f16_e32 v5, -1, v2               ; encoding: [0xc1,0x04,0x0a,0x6c]
 0xc1,0x04,0x0a,0x6c
+# GFX12: v_fmac_f16_e32 v5, -1, v2               ; encoding: [0xc1,0x04,0x0a,0x6c]
 
-# GFX12: v_fmac_f16_e32 v5, 0.5, v2              ; encoding: [0xf0,0x04,0x0a,0x6c]
 0xf0,0x04,0x0a,0x6c
+# GFX12: v_fmac_f16_e32 v5, 0.5, v2              ; encoding: [0xf0,0x04,0x0a,0x6c]
 
-# GFX12: v_fmac_f16_e32 v5, src_scc, v2          ; encoding: [0xfd,0x04,0x0a,0x6c]
 0xfd,0x04,0x0a,0x6c
+# GFX12: v_fmac_f16_e32 v5, src_scc, v2          ; encoding: [0xfd,0x04,0x0a,0x6c]
 
-# GFX12: v_fmac_f16_e32 v127, 0xfe0b, v127       ; encoding: [0xff,0xfe,0xfe,0x6c,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xfe,0x6c,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmac_f16_e32 v127, 0xfe0b, v127       ; encoding: [0xff,0xfe,0xfe,0x6c,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmac_f32_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x56]
 0x01,0x05,0x0a,0x56
+# GFX12: v_fmac_f32_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x56]
 
-# GFX12: v_fmac_f32_e32 v5, v255, v2             ; encoding: [0xff,0x05,0x0a,0x56]
 0xff,0x05,0x0a,0x56
+# GFX12: v_fmac_f32_e32 v5, v255, v2             ; encoding: [0xff,0x05,0x0a,0x56]
 
-# GFX12: v_fmac_f32_e32 v5, s1, v2               ; encoding: [0x01,0x04,0x0a,0x56]
 0x01,0x04,0x0a,0x56
+# GFX12: v_fmac_f32_e32 v5, s1, v2               ; encoding: [0x01,0x04,0x0a,0x56]
 
-# GFX12: v_fmac_f32_e32 v5, s105, v2             ; encoding: [0x69,0x04,0x0a,0x56]
 0x69,0x04,0x0a,0x56
+# GFX12: v_fmac_f32_e32 v5, s105, v2             ; encoding: [0x69,0x04,0x0a,0x56]
 
-# GFX12: v_fmac_f32_e32 v5, vcc_lo, v2           ; encoding: [0x6a,0x04,0x0a,0x56]
 0x6a,0x04,0x0a,0x56
+# GFX12: v_fmac_f32_e32 v5, vcc_lo, v2           ; encoding: [0x6a,0x04,0x0a,0x56]
 
-# GFX12: v_fmac_f32_e32 v5, vcc_hi, v2           ; encoding: [0x6b,0x04,0x0a,0x56]
 0x6b,0x04,0x0a,0x56
+# GFX12: v_fmac_f32_e32 v5, vcc_hi, v2           ; encoding: [0x6b,0x04,0x0a,0x56]
 
-# GFX12: v_fmac_f32_e32 v5, ttmp15, v2           ; encoding: [0x7b,0x04,0x0a,0x56]
 0x7b,0x04,0x0a,0x56
+# GFX12: v_fmac_f32_e32 v5, ttmp15, v2           ; encoding: [0x7b,0x04,0x0a,0x56]
 
-# GFX12: v_fmac_f32_e32 v5, m0, v2               ; encoding: [0x7d,0x04,0x0a,0x56]
 0x7d,0x04,0x0a,0x56
+# GFX12: v_fmac_f32_e32 v5, m0, v2               ; encoding: [0x7d,0x04,0x0a,0x56]
 
-# GFX12: v_fmac_f32_e32 v5, exec_lo, v2          ; encoding: [0x7e,0x04,0x0a,0x56]
 0x7e,0x04,0x0a,0x56
+# GFX12: v_fmac_f32_e32 v5, exec_lo, v2          ; encoding: [0x7e,0x04,0x0a,0x56]
 
-# GFX12: v_fmac_f32_e32 v5, exec_hi, v2          ; encoding: [0x7f,0x04,0x0a,0x56]
 0x7f,0x04,0x0a,0x56
+# GFX12: v_fmac_f32_e32 v5, exec_hi, v2          ; encoding: [0x7f,0x04,0x0a,0x56]
 
-# GFX12: v_fmac_f32_e32 v5, null, v2             ; encoding: [0x7c,0x04,0x0a,0x56]
 0x7c,0x04,0x0a,0x56
+# GFX12: v_fmac_f32_e32 v5, null, v2             ; encoding: [0x7c,0x04,0x0a,0x56]
 
-# GFX12: v_fmac_f32_e32 v5, -1, v2               ; encoding: [0xc1,0x04,0x0a,0x56]
 0xc1,0x04,0x0a,0x56
+# GFX12: v_fmac_f32_e32 v5, -1, v2               ; encoding: [0xc1,0x04,0x0a,0x56]
 
-# GFX12: v_fmac_f32_e32 v5, 0.5, v2              ; encoding: [0xf0,0x04,0x0a,0x56]
 0xf0,0x04,0x0a,0x56
+# GFX12: v_fmac_f32_e32 v5, 0.5, v2              ; encoding: [0xf0,0x04,0x0a,0x56]
 
-# GFX12: v_fmac_f32_e32 v5, src_scc, v2          ; encoding: [0xfd,0x04,0x0a,0x56]
 0xfd,0x04,0x0a,0x56
+# GFX12: v_fmac_f32_e32 v5, src_scc, v2          ; encoding: [0xfd,0x04,0x0a,0x56]
 
-# GFX12: v_fmac_f32_e32 v255, 0xaf123456, v255   ; encoding: [0xff,0xfe,0xff,0x57,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x57,0x56,0x34,0x12,0xaf
+# GFX12: v_fmac_f32_e32 v255, 0xaf123456, v255   ; encoding: [0xff,0xfe,0xff,0x57,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmamk_f16 v5, v1, 0xfe0b, v3          ; encoding: [0x01,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x01,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmamk_f16 v5, v1, 0xfe0b, v3          ; encoding: [0x01,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmamk_f16 v5, v127, 0xfe0b, v3        ; encoding: [0x7f,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x7f,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmamk_f16 v5, v127, 0xfe0b, v3        ; encoding: [0x7f,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmamk_f16 v5, s1, 0xfe0b, v3          ; encoding: [0x01,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x01,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmamk_f16 v5, s1, 0xfe0b, v3          ; encoding: [0x01,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmamk_f16 v5, s105, 0xfe0b, v3        ; encoding: [0x69,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x69,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmamk_f16 v5, s105, 0xfe0b, v3        ; encoding: [0x69,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmamk_f16 v5, vcc_lo, 0xfe0b, v3      ; encoding: [0x6a,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x6a,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmamk_f16 v5, vcc_lo, 0xfe0b, v3      ; encoding: [0x6a,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmamk_f16 v5, vcc_hi, 0xfe0b, v3      ; encoding: [0x6b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x6b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmamk_f16 v5, vcc_hi, 0xfe0b, v3      ; encoding: [0x6b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmamk_f16 v5, ttmp15, 0xfe0b, v3      ; encoding: [0x7b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x7b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmamk_f16 v5, ttmp15, 0xfe0b, v3      ; encoding: [0x7b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmamk_f16 v5, m0, 0xfe0b, v3          ; encoding: [0x7d,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x7d,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmamk_f16 v5, m0, 0xfe0b, v3          ; encoding: [0x7d,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmamk_f16 v5, exec_lo, 0xfe0b, v3     ; encoding: [0x7e,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x7e,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmamk_f16 v5, exec_lo, 0xfe0b, v3     ; encoding: [0x7e,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmamk_f16 v5, exec_hi, 0xfe0b, v3     ; encoding: [0x7f,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x7f,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmamk_f16 v5, exec_hi, 0xfe0b, v3     ; encoding: [0x7f,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmamk_f16 v5, null, 0xfe0b, v3        ; encoding: [0x7c,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0x7c,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmamk_f16 v5, null, 0xfe0b, v3        ; encoding: [0x7c,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmamk_f16 v5, -1, 0xfe0b, v3          ; encoding: [0xc1,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0xc1,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmamk_f16 v5, -1, 0xfe0b, v3          ; encoding: [0xc1,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmamk_f16 v5, 0.5, 0xfe0b, v3         ; encoding: [0xf0,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0xf0,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmamk_f16 v5, 0.5, 0xfe0b, v3         ; encoding: [0xf0,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmamk_f16 v5, src_scc, 0xfe0b, v3     ; encoding: [0xfd,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 0xfd,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmamk_f16 v5, src_scc, 0xfe0b, v3     ; encoding: [0xfd,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmamk_f16 v127, 0xfe0b, 0xfe0b, v127  ; encoding: [0xff,0xfe,0xfe,0x6e,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xfe,0x6e,0x0b,0xfe,0x00,0x00
+# GFX12: v_fmamk_f16 v127, 0xfe0b, 0xfe0b, v127  ; encoding: [0xff,0xfe,0xfe,0x6e,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fmamk_f32 v5, v1, 0xaf123456, v3      ; encoding: [0x01,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x01,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX12: v_fmamk_f32 v5, v1, 0xaf123456, v3      ; encoding: [0x01,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmamk_f32 v5, v255, 0xaf123456, v3    ; encoding: [0xff,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0xff,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX12: v_fmamk_f32 v5, v255, 0xaf123456, v3    ; encoding: [0xff,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmamk_f32 v5, s1, 0xaf123456, v3      ; encoding: [0x01,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x01,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX12: v_fmamk_f32 v5, s1, 0xaf123456, v3      ; encoding: [0x01,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmamk_f32 v5, s105, 0xaf123456, v3    ; encoding: [0x69,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x69,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX12: v_fmamk_f32 v5, s105, 0xaf123456, v3    ; encoding: [0x69,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmamk_f32 v5, vcc_lo, 0xaf123456, v3  ; encoding: [0x6a,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x6a,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX12: v_fmamk_f32 v5, vcc_lo, 0xaf123456, v3  ; encoding: [0x6a,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmamk_f32 v5, vcc_hi, 0xaf123456, v3  ; encoding: [0x6b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x6b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX12: v_fmamk_f32 v5, vcc_hi, 0xaf123456, v3  ; encoding: [0x6b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmamk_f32 v5, ttmp15, 0xaf123456, v3  ; encoding: [0x7b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x7b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX12: v_fmamk_f32 v5, ttmp15, 0xaf123456, v3  ; encoding: [0x7b,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmamk_f32 v5, m0, 0xaf123456, v3      ; encoding: [0x7d,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x7d,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX12: v_fmamk_f32 v5, m0, 0xaf123456, v3      ; encoding: [0x7d,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmamk_f32 v5, exec_lo, 0xaf123456, v3 ; encoding: [0x7e,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x7e,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX12: v_fmamk_f32 v5, exec_lo, 0xaf123456, v3 ; encoding: [0x7e,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmamk_f32 v5, exec_hi, 0xaf123456, v3 ; encoding: [0x7f,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x7f,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX12: v_fmamk_f32 v5, exec_hi, 0xaf123456, v3 ; encoding: [0x7f,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmamk_f32 v5, null, 0xaf123456, v3    ; encoding: [0x7c,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0x7c,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX12: v_fmamk_f32 v5, null, 0xaf123456, v3    ; encoding: [0x7c,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmamk_f32 v5, -1, 0xaf123456, v3      ; encoding: [0xc1,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0xc1,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX12: v_fmamk_f32 v5, -1, 0xaf123456, v3      ; encoding: [0xc1,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmamk_f32 v5, 0.5, 0xaf123456, v3     ; encoding: [0xf0,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0xf0,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX12: v_fmamk_f32 v5, 0.5, 0xaf123456, v3     ; encoding: [0xf0,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmamk_f32 v5, src_scc, 0xaf123456, v3 ; encoding: [0xfd,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 0xfd,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf
+# GFX12: v_fmamk_f32 v5, src_scc, 0xaf123456, v3 ; encoding: [0xfd,0x06,0x0a,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fmamk_f32 v255, 0xaf123456, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x59,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x59,0x56,0x34,0x12,0xaf
+# GFX12: v_fmamk_f32 v255, 0xaf123456, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x59,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_ldexp_f16_e32 v5, v1, v2              ; encoding: [0x01,0x05,0x0a,0x76]
 0x01,0x05,0x0a,0x76
+# GFX12-REAL16: v_ldexp_f16_e32 v5.l, v1.l, v2.l        ; encoding: [0x01,0x05,0x0a,0x76]
+# GFX12-FAKE16: v_ldexp_f16_e32 v5, v1, v2              ; encoding: [0x01,0x05,0x0a,0x76]
 
-# GFX12: v_ldexp_f16_e32 v5, v127, v2            ; encoding: [0x7f,0x05,0x0a,0x76]
 0x7f,0x05,0x0a,0x76
+# GFX12-REAL16: v_ldexp_f16_e32 v5.l, v127.l, v2.l      ; encoding: [0x7f,0x05,0x0a,0x76]
+# GFX12-FAKE16: v_ldexp_f16_e32 v5, v127, v2            ; encoding: [0x7f,0x05,0x0a,0x76]
 
-# GFX12: v_ldexp_f16_e32 v5, s1, v2              ; encoding: [0x01,0x04,0x0a,0x76]
 0x01,0x04,0x0a,0x76
+# GFX12-REAL16: v_ldexp_f16_e32 v5.l, s1, v2.l          ; encoding: [0x01,0x04,0x0a,0x76]
+# GFX12-FAKE16: v_ldexp_f16_e32 v5, s1, v2              ; encoding: [0x01,0x04,0x0a,0x76]
 
-# GFX12: v_ldexp_f16_e32 v5, s105, v2            ; encoding: [0x69,0x04,0x0a,0x76]
 0x69,0x04,0x0a,0x76
+# GFX12-REAL16: v_ldexp_f16_e32 v5.l, s105, v2.l        ; encoding: [0x69,0x04,0x0a,0x76]
+# GFX12-FAKE16: v_ldexp_f16_e32 v5, s105, v2            ; encoding: [0x69,0x04,0x0a,0x76]
 
-# GFX12: v_ldexp_f16_e32 v5, vcc_lo, v2          ; encoding: [0x6a,0x04,0x0a,0x76]
 0x6a,0x04,0x0a,0x76
+# GFX12-REAL16: v_ldexp_f16_e32 v5.l, vcc_lo, v2.l      ; encoding: [0x6a,0x04,0x0a,0x76]
+# GFX12-FAKE16: v_ldexp_f16_e32 v5, vcc_lo, v2          ; encoding: [0x6a,0x04,0x0a,0x76]
 
-# GFX12: v_ldexp_f16_e32 v5, vcc_hi, v2          ; encoding: [0x6b,0x04,0x0a,0x76]
 0x6b,0x04,0x0a,0x76
+# GFX12-REAL16: v_ldexp_f16_e32 v5.l, vcc_hi, v2.l      ; encoding: [0x6b,0x04,0x0a,0x76]
+# GFX12-FAKE16: v_ldexp_f16_e32 v5, vcc_hi, v2          ; encoding: [0x6b,0x04,0x0a,0x76]
 
-# GFX12: v_ldexp_f16_e32 v5, ttmp15, v2          ; encoding: [0x7b,0x04,0x0a,0x76]
 0x7b,0x04,0x0a,0x76
+# GFX12-REAL16: v_ldexp_f16_e32 v5.l, ttmp15, v2.l      ; encoding: [0x7b,0x04,0x0a,0x76]
+# GFX12-FAKE16: v_ldexp_f16_e32 v5, ttmp15, v2          ; encoding: [0x7b,0x04,0x0a,0x76]
 
-# GFX12: v_ldexp_f16_e32 v5, m0, v2              ; encoding: [0x7d,0x04,0x0a,0x76]
 0x7d,0x04,0x0a,0x76
+# GFX12-REAL16: v_ldexp_f16_e32 v5.l, m0, v2.l          ; encoding: [0x7d,0x04,0x0a,0x76]
+# GFX12-FAKE16: v_ldexp_f16_e32 v5, m0, v2              ; encoding: [0x7d,0x04,0x0a,0x76]
 
-# GFX12: v_ldexp_f16_e32 v5, exec_lo, v2         ; encoding: [0x7e,0x04,0x0a,0x76]
 0x7e,0x04,0x0a,0x76
+# GFX12-REAL16: v_ldexp_f16_e32 v5.l, exec_lo, v2.l     ; encoding: [0x7e,0x04,0x0a,0x76]
+# GFX12-FAKE16: v_ldexp_f16_e32 v5, exec_lo, v2         ; encoding: [0x7e,0x04,0x0a,0x76]
 
-# GFX12: v_ldexp_f16_e32 v5, exec_hi, v2         ; encoding: [0x7f,0x04,0x0a,0x76]
 0x7f,0x04,0x0a,0x76
+# GFX12-REAL16: v_ldexp_f16_e32 v5.l, exec_hi, v2.l     ; encoding: [0x7f,0x04,0x0a,0x76]
+# GFX12-FAKE16: v_ldexp_f16_e32 v5, exec_hi, v2         ; encoding: [0x7f,0x04,0x0a,0x76]
 
-# GFX12: v_ldexp_f16_e32 v5, null, v2            ; encoding: [0x7c,0x04,0x0a,0x76]
 0x7c,0x04,0x0a,0x76
+# GFX12-REAL16: v_ldexp_f16_e32 v5.l, null, v2.l        ; encoding: [0x7c,0x04,0x0a,0x76]
+# GFX12-FAKE16: v_ldexp_f16_e32 v5, null, v2            ; encoding: [0x7c,0x04,0x0a,0x76]
 
-# GFX12: v_ldexp_f16_e32 v5, -1, v2              ; encoding: [0xc1,0x04,0x0a,0x76]
 0xc1,0x04,0x0a,0x76
+# GFX12-REAL16: v_ldexp_f16_e32 v5.l, -1, v2.l          ; encoding: [0xc1,0x04,0x0a,0x76]
+# GFX12-FAKE16: v_ldexp_f16_e32 v5, -1, v2              ; encoding: [0xc1,0x04,0x0a,0x76]
 
-# GFX12: v_ldexp_f16_e32 v5, 0.5, v2             ; encoding: [0xf0,0x04,0x0a,0x76]
 0xf0,0x04,0x0a,0x76
+# GFX12-REAL16: v_ldexp_f16_e32 v5.l, 0.5, v2.l         ; encoding: [0xf0,0x04,0x0a,0x76]
+# GFX12-FAKE16: v_ldexp_f16_e32 v5, 0.5, v2             ; encoding: [0xf0,0x04,0x0a,0x76]
 
-# GFX12: v_ldexp_f16_e32 v5, src_scc, v2         ; encoding: [0xfd,0x04,0x0a,0x76]
 0xfd,0x04,0x0a,0x76
+# GFX12-REAL16: v_ldexp_f16_e32 v5.l, src_scc, v2.l     ; encoding: [0xfd,0x04,0x0a,0x76]
+# GFX12-FAKE16: v_ldexp_f16_e32 v5, src_scc, v2         ; encoding: [0xfd,0x04,0x0a,0x76]
 
-# GFX12: v_ldexp_f16_e32 v127, 0xfe0b, v127      ; encoding: [0xff,0xfe,0xfe,0x76,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xfe,0x76,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_ldexp_f16_e32 v127.l, 0xfe0b, v127.l  ; encoding: [0xff,0xfe,0xfe,0x76,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_ldexp_f16_e32 v127, 0xfe0b, v127      ; encoding: [0xff,0xfe,0xfe,0x76,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_lshlrev_b32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x30]
 0x01,0x05,0x0a,0x30
+# GFX12: v_lshlrev_b32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x30]
 
-# GFX12: v_lshlrev_b32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x30]
 0xff,0x05,0x0a,0x30
+# GFX12: v_lshlrev_b32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x30]
 
-# GFX12: v_lshlrev_b32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x30]
 0x01,0x04,0x0a,0x30
+# GFX12: v_lshlrev_b32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x30]
 
-# GFX12: v_lshlrev_b32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x30]
 0x69,0x04,0x0a,0x30
+# GFX12: v_lshlrev_b32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x30]
 
-# GFX12: v_lshlrev_b32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x30]
 0x6a,0x04,0x0a,0x30
+# GFX12: v_lshlrev_b32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x30]
 
-# GFX12: v_lshlrev_b32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x30]
 0x6b,0x04,0x0a,0x30
+# GFX12: v_lshlrev_b32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x30]
 
-# GFX12: v_lshlrev_b32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x30]
 0x7b,0x04,0x0a,0x30
+# GFX12: v_lshlrev_b32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x30]
 
-# GFX12: v_lshlrev_b32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x30]
 0x7d,0x04,0x0a,0x30
+# GFX12: v_lshlrev_b32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x30]
 
-# GFX12: v_lshlrev_b32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x30]
 0x7e,0x04,0x0a,0x30
+# GFX12: v_lshlrev_b32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x30]
 
-# GFX12: v_lshlrev_b32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x30]
 0x7f,0x04,0x0a,0x30
+# GFX12: v_lshlrev_b32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x30]
 
-# GFX12: v_lshlrev_b32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x30]
 0x7c,0x04,0x0a,0x30
+# GFX12: v_lshlrev_b32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x30]
 
-# GFX12: v_lshlrev_b32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x30]
 0xc1,0x04,0x0a,0x30
+# GFX12: v_lshlrev_b32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x30]
 
-# GFX12: v_lshlrev_b32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x30]
 0xf0,0x04,0x0a,0x30
+# GFX12: v_lshlrev_b32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x30]
 
-# GFX12: v_lshlrev_b32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x30]
 0xfd,0x04,0x0a,0x30
+# GFX12: v_lshlrev_b32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x30]
 
-# GFX12: v_lshlrev_b32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x31,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x31,0x56,0x34,0x12,0xaf
+# GFX12: v_lshlrev_b32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x31,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_lshlrev_b64_e32 v[5:6], v1, v[3:4]    ; encoding: [0x01,0x07,0x0a,0x3e]
 0x01,0x07,0x0a,0x3e
+# GFX12: v_lshlrev_b64_e32 v[5:6], v1, v[3:4]    ; encoding: [0x01,0x07,0x0a,0x3e]
 
-# GFX12: v_lshlrev_b64_e32 v[5:6], v255, v[2:3]  ; encoding: [0xff,0x05,0x0a,0x3e]
 0xff,0x05,0x0a,0x3e
+# GFX12: v_lshlrev_b64_e32 v[5:6], v255, v[2:3]  ; encoding: [0xff,0x05,0x0a,0x3e]
 
-# GFX12: v_lshlrev_b64_e32 v[5:6], s1, v[2:3]    ; encoding: [0x01,0x04,0x0a,0x3e]
 0x01,0x04,0x0a,0x3e
+# GFX12: v_lshlrev_b64_e32 v[5:6], s1, v[2:3]    ; encoding: [0x01,0x04,0x0a,0x3e]
 
-# GFX12: v_lshlrev_b64_e32 v[5:6], s105, v[2:3]  ; encoding: [0x69,0x04,0x0a,0x3e]
 0x69,0x04,0x0a,0x3e
+# GFX12: v_lshlrev_b64_e32 v[5:6], s105, v[2:3]  ; encoding: [0x69,0x04,0x0a,0x3e]
 
-# GFX12: v_lshlrev_b64_e32 v[5:6], vcc_lo, v[2:3] ; encoding: [0x6a,0x04,0x0a,0x3e]
 0x6a,0x04,0x0a,0x3e
+# GFX12: v_lshlrev_b64_e32 v[5:6], vcc_lo, v[2:3] ; encoding: [0x6a,0x04,0x0a,0x3e]
 
-# GFX12: v_lshlrev_b64_e32 v[5:6], vcc_hi, v[2:3] ; encoding: [0x6b,0x04,0x0a,0x3e]
 0x6b,0x04,0x0a,0x3e
+# GFX12: v_lshlrev_b64_e32 v[5:6], vcc_hi, v[2:3] ; encoding: [0x6b,0x04,0x0a,0x3e]
 
-# GFX12: v_lshlrev_b64_e32 v[5:6], ttmp15, v[2:3] ; encoding: [0x7b,0x04,0x0a,0x3e]
 0x7b,0x04,0x0a,0x3e
+# GFX12: v_lshlrev_b64_e32 v[5:6], ttmp15, v[2:3] ; encoding: [0x7b,0x04,0x0a,0x3e]
 
-# GFX12: v_lshlrev_b64_e32 v[5:6], exec_lo, v[2:3] ; encoding: [0x7e,0x04,0x0a,0x3e]
 0x7e,0x04,0x0a,0x3e
+# GFX12: v_lshlrev_b64_e32 v[5:6], exec_lo, v[2:3] ; encoding: [0x7e,0x04,0x0a,0x3e]
 
-# GFX12: v_lshlrev_b64_e32 v[5:6], exec_hi, v[2:3] ; encoding: [0x7f,0x04,0x0a,0x3e]
 0x7f,0x04,0x0a,0x3e
+# GFX12: v_lshlrev_b64_e32 v[5:6], exec_hi, v[2:3] ; encoding: [0x7f,0x04,0x0a,0x3e]
 
-# GFX12: v_lshlrev_b64_e32 v[5:6], null, v[2:3]  ; encoding: [0x7c,0x04,0x0a,0x3e]
 0x7c,0x04,0x0a,0x3e
+# GFX12: v_lshlrev_b64_e32 v[5:6], null, v[2:3]  ; encoding: [0x7c,0x04,0x0a,0x3e]
 
-# GFX12: v_lshlrev_b64_e32 v[5:6], -1, v[2:3]    ; encoding: [0xc1,0x04,0x0a,0x3e]
 0xc1,0x04,0x0a,0x3e
+# GFX12: v_lshlrev_b64_e32 v[5:6], -1, v[2:3]    ; encoding: [0xc1,0x04,0x0a,0x3e]
 
-# GFX12: v_lshlrev_b64_e32 v[5:6], 0.5, v[2:3]   ; encoding: [0xf0,0x04,0x0a,0x3e]
 0xf0,0x04,0x0a,0x3e
+# GFX12: v_lshlrev_b64_e32 v[5:6], 0.5, v[2:3]   ; encoding: [0xf0,0x04,0x0a,0x3e]
 
-# GFX12: v_lshlrev_b64_e32 v[5:6], src_scc, v[2:3] ; encoding: [0xfd,0x04,0x0a,0x3e]
 0xfd,0x04,0x0a,0x3e
+# GFX12: v_lshlrev_b64_e32 v[5:6], src_scc, v[2:3] ; encoding: [0xfd,0x04,0x0a,0x3e]
 
-# GFX12: v_lshlrev_b64_e32 v[254:255], 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xfd,0x3f,0x56,0x34,0x12,0xaf]
 0xff,0xfc,0xfd,0x3f,0x56,0x34,0x12,0xaf
+# GFX12: v_lshlrev_b64_e32 v[254:255], 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xfd,0x3f,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_lshrrev_b32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x32]
 0x01,0x05,0x0a,0x32
+# GFX12: v_lshrrev_b32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x32]
 
-# GFX12: v_lshrrev_b32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x32]
 0xff,0x05,0x0a,0x32
+# GFX12: v_lshrrev_b32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x32]
 
-# GFX12: v_lshrrev_b32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x32]
 0x01,0x04,0x0a,0x32
+# GFX12: v_lshrrev_b32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x32]
 
-# GFX12: v_lshrrev_b32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x32]
 0x69,0x04,0x0a,0x32
+# GFX12: v_lshrrev_b32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x32]
 
-# GFX12: v_lshrrev_b32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x32]
 0x6a,0x04,0x0a,0x32
+# GFX12: v_lshrrev_b32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x32]
 
-# GFX12: v_lshrrev_b32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x32]
 0x6b,0x04,0x0a,0x32
+# GFX12: v_lshrrev_b32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x32]
 
-# GFX12: v_lshrrev_b32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x32]
 0x7b,0x04,0x0a,0x32
+# GFX12: v_lshrrev_b32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x32]
 
-# GFX12: v_lshrrev_b32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x32]
 0x7d,0x04,0x0a,0x32
+# GFX12: v_lshrrev_b32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x32]
 
-# GFX12: v_lshrrev_b32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x32]
 0x7e,0x04,0x0a,0x32
+# GFX12: v_lshrrev_b32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x32]
 
-# GFX12: v_lshrrev_b32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x32]
 0x7f,0x04,0x0a,0x32
+# GFX12: v_lshrrev_b32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x32]
 
-# GFX12: v_lshrrev_b32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x32]
 0x7c,0x04,0x0a,0x32
+# GFX12: v_lshrrev_b32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x32]
 
-# GFX12: v_lshrrev_b32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x32]
 0xc1,0x04,0x0a,0x32
+# GFX12: v_lshrrev_b32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x32]
 
-# GFX12: v_lshrrev_b32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x32]
 0xf0,0x04,0x0a,0x32
+# GFX12: v_lshrrev_b32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x32]
 
-# GFX12: v_lshrrev_b32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x32]
 0xfd,0x04,0x0a,0x32
+# GFX12: v_lshrrev_b32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x32]
 
-# GFX12: v_lshrrev_b32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x33,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x33,0x56,0x34,0x12,0xaf
+# GFX12: v_lshrrev_b32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x33,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_max_num_f16_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x62]
 0x01,0x05,0x0a,0x62
+# GFX12-REAL16: v_max_num_f16_e32 v5.l, v1.l, v2.l      ; encoding: [0x01,0x05,0x0a,0x62]
+# GFX12-FAKE16: v_max_num_f16_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x62]
 
-# GFX12: v_max_num_f16_e32 v5, v127, v2          ; encoding: [0x7f,0x05,0x0a,0x62]
 0x7f,0x05,0x0a,0x62
+# GFX12-REAL16: v_max_num_f16_e32 v5.l, v127.l, v2.l    ; encoding: [0x7f,0x05,0x0a,0x62]
+# GFX12-FAKE16: v_max_num_f16_e32 v5, v127, v2          ; encoding: [0x7f,0x05,0x0a,0x62]
 
-# GFX12: v_max_num_f16_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x62]
 0x01,0x04,0x0a,0x62
+# GFX12-REAL16: v_max_num_f16_e32 v5.l, s1, v2.l        ; encoding: [0x01,0x04,0x0a,0x62]
+# GFX12-FAKE16: v_max_num_f16_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x62]
 
-# GFX12: v_max_num_f16_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x62]
 0x69,0x04,0x0a,0x62
+# GFX12-REAL16: v_max_num_f16_e32 v5.l, s105, v2.l      ; encoding: [0x69,0x04,0x0a,0x62]
+# GFX12-FAKE16: v_max_num_f16_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x62]
 
-# GFX12: v_max_num_f16_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x62]
 0x6a,0x04,0x0a,0x62
+# GFX12-REAL16: v_max_num_f16_e32 v5.l, vcc_lo, v2.l    ; encoding: [0x6a,0x04,0x0a,0x62]
+# GFX12-FAKE16: v_max_num_f16_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x62]
 
-# GFX12: v_max_num_f16_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x62]
 0x6b,0x04,0x0a,0x62
+# GFX12-REAL16: v_max_num_f16_e32 v5.l, vcc_hi, v2.l    ; encoding: [0x6b,0x04,0x0a,0x62]
+# GFX12-FAKE16: v_max_num_f16_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x62]
 
-# GFX12: v_max_num_f16_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x62]
 0x7b,0x04,0x0a,0x62
+# GFX12-REAL16: v_max_num_f16_e32 v5.l, ttmp15, v2.l    ; encoding: [0x7b,0x04,0x0a,0x62]
+# GFX12-FAKE16: v_max_num_f16_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x62]
 
-# GFX12: v_max_num_f16_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x62]
 0x7d,0x04,0x0a,0x62
+# GFX12-REAL16: v_max_num_f16_e32 v5.l, m0, v2.l        ; encoding: [0x7d,0x04,0x0a,0x62]
+# GFX12-FAKE16: v_max_num_f16_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x62]
 
-# GFX12: v_max_num_f16_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x62]
 0x7e,0x04,0x0a,0x62
+# GFX12-REAL16: v_max_num_f16_e32 v5.l, exec_lo, v2.l   ; encoding: [0x7e,0x04,0x0a,0x62]
+# GFX12-FAKE16: v_max_num_f16_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x62]
 
-# GFX12: v_max_num_f16_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x62]
 0x7f,0x04,0x0a,0x62
+# GFX12-REAL16: v_max_num_f16_e32 v5.l, exec_hi, v2.l   ; encoding: [0x7f,0x04,0x0a,0x62]
+# GFX12-FAKE16: v_max_num_f16_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x62]
 
-# GFX12: v_max_num_f16_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x62]
 0x7c,0x04,0x0a,0x62
+# GFX12-REAL16: v_max_num_f16_e32 v5.l, null, v2.l      ; encoding: [0x7c,0x04,0x0a,0x62]
+# GFX12-FAKE16: v_max_num_f16_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x62]
 
-# GFX12: v_max_num_f16_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x62]
 0xc1,0x04,0x0a,0x62
+# GFX12-REAL16: v_max_num_f16_e32 v5.l, -1, v2.l        ; encoding: [0xc1,0x04,0x0a,0x62]
+# GFX12-FAKE16: v_max_num_f16_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x62]
 
-# GFX12: v_max_num_f16_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x62]
 0xf0,0x04,0x0a,0x62
+# GFX12-REAL16: v_max_num_f16_e32 v5.l, 0.5, v2.l       ; encoding: [0xf0,0x04,0x0a,0x62]
+# GFX12-FAKE16: v_max_num_f16_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x62]
 
-# GFX12: v_max_num_f16_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x62]
 0xfd,0x04,0x0a,0x62
+# GFX12-REAL16: v_max_num_f16_e32 v5.l, src_scc, v2.l   ; encoding: [0xfd,0x04,0x0a,0x62]
+# GFX12-FAKE16: v_max_num_f16_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x62]
 
-# GFX12: v_max_num_f16_e32 v127, 0xfe0b, v127    ; encoding: [0xff,0xfe,0xfe,0x62,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xfe,0x62,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_max_num_f16_e32 v127.l, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x62,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_max_num_f16_e32 v127, 0xfe0b, v127    ; encoding: [0xff,0xfe,0xfe,0x62,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_max_num_f32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x2c]
 0x01,0x05,0x0a,0x2c
+# GFX12: v_max_num_f32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x2c]
 
-# GFX12: v_max_num_f32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x2c]
 0xff,0x05,0x0a,0x2c
+# GFX12: v_max_num_f32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x2c]
 
-# GFX12: v_max_num_f32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x2c]
 0x01,0x04,0x0a,0x2c
+# GFX12: v_max_num_f32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x2c]
 
-# GFX12: v_max_num_f32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x2c]
 0x69,0x04,0x0a,0x2c
+# GFX12: v_max_num_f32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x2c]
 
-# GFX12: v_max_num_f32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x2c]
 0x6a,0x04,0x0a,0x2c
+# GFX12: v_max_num_f32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x2c]
 
-# GFX12: v_max_num_f32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x2c]
 0x6b,0x04,0x0a,0x2c
+# GFX12: v_max_num_f32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x2c]
 
-# GFX12: v_max_num_f32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x2c]
 0x7b,0x04,0x0a,0x2c
+# GFX12: v_max_num_f32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x2c]
 
-# GFX12: v_max_num_f32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x2c]
 0x7d,0x04,0x0a,0x2c
+# GFX12: v_max_num_f32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x2c]
 
-# GFX12: v_max_num_f32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x2c]
 0x7e,0x04,0x0a,0x2c
+# GFX12: v_max_num_f32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x2c]
 
-# GFX12: v_max_num_f32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x2c]
 0x7f,0x04,0x0a,0x2c
+# GFX12: v_max_num_f32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x2c]
 
-# GFX12: v_max_num_f32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x2c]
 0x7c,0x04,0x0a,0x2c
+# GFX12: v_max_num_f32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x2c]
 
-# GFX12: v_max_num_f32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x2c]
 0xc1,0x04,0x0a,0x2c
+# GFX12: v_max_num_f32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x2c]
 
-# GFX12: v_max_num_f32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x2c]
 0xf0,0x04,0x0a,0x2c
+# GFX12: v_max_num_f32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x2c]
 
-# GFX12: v_max_num_f32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x2c]
 0xfd,0x04,0x0a,0x2c
+# GFX12: v_max_num_f32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x2c]
 
-# GFX12: v_max_num_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x2d,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x2d,0x56,0x34,0x12,0xaf
+# GFX12: v_max_num_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x2d,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_max_num_f64_e32 v[5:6], v[1:2], v[3:4] ; encoding: [0x01,0x07,0x0a,0x1c]
 0x01,0x07,0x0a,0x1c
+# GFX12: v_max_num_f64_e32 v[5:6], v[1:2], v[3:4] ; encoding: [0x01,0x07,0x0a,0x1c]
 
-# GFX12: v_max_num_f64_e32 v[5:6], v[254:255], v[2:3] ; encoding: [0xfe,0x05,0x0a,0x1c]
 0xfe,0x05,0x0a,0x1c
+# GFX12: v_max_num_f64_e32 v[5:6], v[254:255], v[2:3] ; encoding: [0xfe,0x05,0x0a,0x1c]
 
-# GFX12: v_max_num_f64_e32 v[5:6], s[0:1], v[2:3] ; encoding: [0x00,0x04,0x0a,0x1c]
 0x00,0x04,0x0a,0x1c
+# GFX12: v_max_num_f64_e32 v[5:6], s[0:1], v[2:3] ; encoding: [0x00,0x04,0x0a,0x1c]
 
-# GFX12: v_max_num_f64_e32 v[5:6], s[104:105], v[2:3] ; encoding: [0x68,0x04,0x0a,0x1c]
 0x68,0x04,0x0a,0x1c
+# GFX12: v_max_num_f64_e32 v[5:6], s[104:105], v[2:3] ; encoding: [0x68,0x04,0x0a,0x1c]
 
-# GFX12: v_max_num_f64_e32 v[5:6], vcc, v[2:3]   ; encoding: [0x6a,0x04,0x0a,0x1c]
 0x6a,0x04,0x0a,0x1c
+# GFX12: v_max_num_f64_e32 v[5:6], vcc, v[2:3]   ; encoding: [0x6a,0x04,0x0a,0x1c]
 
-# GFX12: v_max_num_f64_e32 v[5:6], ttmp[14:15], v[2:3] ; encoding: [0x7a,0x04,0x0a,0x1c]
 0x7a,0x04,0x0a,0x1c
+# GFX12: v_max_num_f64_e32 v[5:6], ttmp[14:15], v[2:3] ; encoding: [0x7a,0x04,0x0a,0x1c]
 
-# GFX12: v_max_num_f64_e32 v[5:6], exec, v[2:3]  ; encoding: [0x7e,0x04,0x0a,0x1c]
 0x7e,0x04,0x0a,0x1c
+# GFX12: v_max_num_f64_e32 v[5:6], exec, v[2:3]  ; encoding: [0x7e,0x04,0x0a,0x1c]
 
-# GFX12: v_max_num_f64_e32 v[5:6], null, v[2:3]  ; encoding: [0x7c,0x04,0x0a,0x1c]
 0x7c,0x04,0x0a,0x1c
+# GFX12: v_max_num_f64_e32 v[5:6], null, v[2:3]  ; encoding: [0x7c,0x04,0x0a,0x1c]
 
-# GFX12: v_max_num_f64_e32 v[5:6], -1, v[2:3]    ; encoding: [0xc1,0x04,0x0a,0x1c]
 0xc1,0x04,0x0a,0x1c
+# GFX12: v_max_num_f64_e32 v[5:6], -1, v[2:3]    ; encoding: [0xc1,0x04,0x0a,0x1c]
 
-# GFX12: v_max_num_f64_e32 v[5:6], 0.5, v[2:3]   ; encoding: [0xf0,0x04,0x0a,0x1c]
 0xf0,0x04,0x0a,0x1c
+# GFX12: v_max_num_f64_e32 v[5:6], 0.5, v[2:3]   ; encoding: [0xf0,0x04,0x0a,0x1c]
 
-# GFX12: v_max_num_f64_e32 v[5:6], src_scc, v[2:3] ; encoding: [0xfd,0x04,0x0a,0x1c]
 0xfd,0x04,0x0a,0x1c
+# GFX12: v_max_num_f64_e32 v[5:6], src_scc, v[2:3] ; encoding: [0xfd,0x04,0x0a,0x1c]
 
-# GFX12: v_max_num_f64_e32 v[254:255], 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xfd,0x1d,0x56,0x34,0x12,0xaf]
 0xff,0xfc,0xfd,0x1d,0x56,0x34,0x12,0xaf
+# GFX12: v_max_num_f64_e32 v[254:255], 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xfd,0x1d,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_max_i32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x24]
 0x01,0x05,0x0a,0x24
+# GFX12: v_max_i32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x24]
 
-# GFX12: v_max_i32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x24]
 0xff,0x05,0x0a,0x24
+# GFX12: v_max_i32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x24]
 
-# GFX12: v_max_i32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x24]
 0x01,0x04,0x0a,0x24
+# GFX12: v_max_i32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x24]
 
-# GFX12: v_max_i32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x24]
 0x69,0x04,0x0a,0x24
+# GFX12: v_max_i32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x24]
 
-# GFX12: v_max_i32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x24]
 0x6a,0x04,0x0a,0x24
+# GFX12: v_max_i32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x24]
 
-# GFX12: v_max_i32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x24]
 0x6b,0x04,0x0a,0x24
+# GFX12: v_max_i32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x24]
 
-# GFX12: v_max_i32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x24]
 0x7b,0x04,0x0a,0x24
+# GFX12: v_max_i32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x24]
 
-# GFX12: v_max_i32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x24]
 0x7d,0x04,0x0a,0x24
+# GFX12: v_max_i32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x24]
 
-# GFX12: v_max_i32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x24]
 0x7e,0x04,0x0a,0x24
+# GFX12: v_max_i32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x24]
 
-# GFX12: v_max_i32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x24]
 0x7f,0x04,0x0a,0x24
+# GFX12: v_max_i32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x24]
 
-# GFX12: v_max_i32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x24]
 0x7c,0x04,0x0a,0x24
+# GFX12: v_max_i32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x24]
 
-# GFX12: v_max_i32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x24]
 0xc1,0x04,0x0a,0x24
+# GFX12: v_max_i32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x24]
 
-# GFX12: v_max_i32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x24]
 0xf0,0x04,0x0a,0x24
+# GFX12: v_max_i32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x24]
 
-# GFX12: v_max_i32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x24]
 0xfd,0x04,0x0a,0x24
+# GFX12: v_max_i32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x24]
 
-# GFX12: v_max_i32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x25,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x25,0x56,0x34,0x12,0xaf
+# GFX12: v_max_i32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x25,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_max_u32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x28]
 0x01,0x05,0x0a,0x28
+# GFX12: v_max_u32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x28]
 
-# GFX12: v_max_u32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x28]
 0xff,0x05,0x0a,0x28
+# GFX12: v_max_u32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x28]
 
-# GFX12: v_max_u32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x28]
 0x01,0x04,0x0a,0x28
+# GFX12: v_max_u32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x28]
 
-# GFX12: v_max_u32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x28]
 0x69,0x04,0x0a,0x28
+# GFX12: v_max_u32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x28]
 
-# GFX12: v_max_u32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x28]
 0x6a,0x04,0x0a,0x28
+# GFX12: v_max_u32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x28]
 
-# GFX12: v_max_u32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x28]
 0x6b,0x04,0x0a,0x28
+# GFX12: v_max_u32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x28]
 
-# GFX12: v_max_u32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x28]
 0x7b,0x04,0x0a,0x28
+# GFX12: v_max_u32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x28]
 
-# GFX12: v_max_u32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x28]
 0x7d,0x04,0x0a,0x28
+# GFX12: v_max_u32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x28]
 
-# GFX12: v_max_u32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x28]
 0x7e,0x04,0x0a,0x28
+# GFX12: v_max_u32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x28]
 
-# GFX12: v_max_u32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x28]
 0x7f,0x04,0x0a,0x28
+# GFX12: v_max_u32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x28]
 
-# GFX12: v_max_u32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x28]
 0x7c,0x04,0x0a,0x28
+# GFX12: v_max_u32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x28]
 
-# GFX12: v_max_u32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x28]
 0xc1,0x04,0x0a,0x28
+# GFX12: v_max_u32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x28]
 
-# GFX12: v_max_u32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x28]
 0xf0,0x04,0x0a,0x28
+# GFX12: v_max_u32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x28]
 
-# GFX12: v_max_u32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x28]
 0xfd,0x04,0x0a,0x28
+# GFX12: v_max_u32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x28]
 
-# GFX12: v_max_u32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x29,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x29,0x56,0x34,0x12,0xaf
+# GFX12: v_max_u32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x29,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_min_num_f16_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x60]
 0x01,0x05,0x0a,0x60
+# GFX12-REAL16: v_min_num_f16_e32 v5.l, v1.l, v2.l      ; encoding: [0x01,0x05,0x0a,0x60]
+# GFX12-FAKE16: v_min_num_f16_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x60]
 
-# GFX12: v_min_num_f16_e32 v5, v127, v2          ; encoding: [0x7f,0x05,0x0a,0x60]
 0x7f,0x05,0x0a,0x60
+# GFX12-REAL16: v_min_num_f16_e32 v5.l, v127.l, v2.l    ; encoding: [0x7f,0x05,0x0a,0x60]
+# GFX12-FAKE16: v_min_num_f16_e32 v5, v127, v2          ; encoding: [0x7f,0x05,0x0a,0x60]
 
-# GFX12: v_min_num_f16_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x60]
 0x01,0x04,0x0a,0x60
+# GFX12-REAL16: v_min_num_f16_e32 v5.l, s1, v2.l        ; encoding: [0x01,0x04,0x0a,0x60]
+# GFX12-FAKE16: v_min_num_f16_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x60]
 
-# GFX12: v_min_num_f16_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x60]
 0x69,0x04,0x0a,0x60
+# GFX12-REAL16: v_min_num_f16_e32 v5.l, s105, v2.l      ; encoding: [0x69,0x04,0x0a,0x60]
+# GFX12-FAKE16: v_min_num_f16_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x60]
 
-# GFX12: v_min_num_f16_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x60]
 0x6a,0x04,0x0a,0x60
+# GFX12-REAL16: v_min_num_f16_e32 v5.l, vcc_lo, v2.l    ; encoding: [0x6a,0x04,0x0a,0x60]
+# GFX12-FAKE16: v_min_num_f16_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x60]
 
-# GFX12: v_min_num_f16_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x60]
 0x6b,0x04,0x0a,0x60
+# GFX12-REAL16: v_min_num_f16_e32 v5.l, vcc_hi, v2.l    ; encoding: [0x6b,0x04,0x0a,0x60]
+# GFX12-FAKE16: v_min_num_f16_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x60]
 
-# GFX12: v_min_num_f16_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x60]
 0x7b,0x04,0x0a,0x60
+# GFX12-REAL16: v_min_num_f16_e32 v5.l, ttmp15, v2.l    ; encoding: [0x7b,0x04,0x0a,0x60]
+# GFX12-FAKE16: v_min_num_f16_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x60]
 
-# GFX12: v_min_num_f16_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x60]
 0x7d,0x04,0x0a,0x60
+# GFX12-REAL16: v_min_num_f16_e32 v5.l, m0, v2.l        ; encoding: [0x7d,0x04,0x0a,0x60]
+# GFX12-FAKE16: v_min_num_f16_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x60]
 
-# GFX12: v_min_num_f16_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x60]
 0x7e,0x04,0x0a,0x60
+# GFX12-REAL16: v_min_num_f16_e32 v5.l, exec_lo, v2.l   ; encoding: [0x7e,0x04,0x0a,0x60]
+# GFX12-FAKE16: v_min_num_f16_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x60]
 
-# GFX12: v_min_num_f16_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x60]
 0x7f,0x04,0x0a,0x60
+# GFX12-REAL16: v_min_num_f16_e32 v5.l, exec_hi, v2.l   ; encoding: [0x7f,0x04,0x0a,0x60]
+# GFX12-FAKE16: v_min_num_f16_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x60]
 
-# GFX12: v_min_num_f16_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x60]
 0x7c,0x04,0x0a,0x60
+# GFX12-REAL16: v_min_num_f16_e32 v5.l, null, v2.l      ; encoding: [0x7c,0x04,0x0a,0x60]
+# GFX12-FAKE16: v_min_num_f16_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x60]
 
-# GFX12: v_min_num_f16_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x60]
 0xc1,0x04,0x0a,0x60
+# GFX12-REAL16: v_min_num_f16_e32 v5.l, -1, v2.l        ; encoding: [0xc1,0x04,0x0a,0x60]
+# GFX12-FAKE16: v_min_num_f16_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x60]
 
-# GFX12: v_min_num_f16_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x60]
 0xf0,0x04,0x0a,0x60
+# GFX12-REAL16: v_min_num_f16_e32 v5.l, 0.5, v2.l       ; encoding: [0xf0,0x04,0x0a,0x60]
+# GFX12-FAKE16: v_min_num_f16_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x60]
 
-# GFX12: v_min_num_f16_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x60]
 0xfd,0x04,0x0a,0x60
+# GFX12-REAL16: v_min_num_f16_e32 v5.l, src_scc, v2.l   ; encoding: [0xfd,0x04,0x0a,0x60]
+# GFX12-FAKE16: v_min_num_f16_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x60]
 
-# GFX12: v_min_num_f16_e32 v127, 0xfe0b, v127    ; encoding: [0xff,0xfe,0xfe,0x60,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xfe,0x60,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_min_num_f16_e32 v127.l, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x60,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_min_num_f16_e32 v127, 0xfe0b, v127    ; encoding: [0xff,0xfe,0xfe,0x60,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_min_num_f32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x2a]
 0x01,0x05,0x0a,0x2a
+# GFX12: v_min_num_f32_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x2a]
 
-# GFX12: v_min_num_f32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x2a]
 0xff,0x05,0x0a,0x2a
+# GFX12: v_min_num_f32_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x2a]
 
-# GFX12: v_min_num_f32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x2a]
 0x01,0x04,0x0a,0x2a
+# GFX12: v_min_num_f32_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x2a]
 
-# GFX12: v_min_num_f32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x2a]
 0x69,0x04,0x0a,0x2a
+# GFX12: v_min_num_f32_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x2a]
 
-# GFX12: v_min_num_f32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x2a]
 0x6a,0x04,0x0a,0x2a
+# GFX12: v_min_num_f32_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x2a]
 
-# GFX12: v_min_num_f32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x2a]
 0x6b,0x04,0x0a,0x2a
+# GFX12: v_min_num_f32_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x2a]
 
-# GFX12: v_min_num_f32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x2a]
 0x7b,0x04,0x0a,0x2a
+# GFX12: v_min_num_f32_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x2a]
 
-# GFX12: v_min_num_f32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x2a]
 0x7d,0x04,0x0a,0x2a
+# GFX12: v_min_num_f32_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x2a]
 
-# GFX12: v_min_num_f32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x2a]
 0x7e,0x04,0x0a,0x2a
+# GFX12: v_min_num_f32_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x2a]
 
-# GFX12: v_min_num_f32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x2a]
 0x7f,0x04,0x0a,0x2a
+# GFX12: v_min_num_f32_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x2a]
 
-# GFX12: v_min_num_f32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x2a]
 0x7c,0x04,0x0a,0x2a
+# GFX12: v_min_num_f32_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x2a]
 
-# GFX12: v_min_num_f32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x2a]
 0xc1,0x04,0x0a,0x2a
+# GFX12: v_min_num_f32_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x2a]
 
-# GFX12: v_min_num_f32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x2a]
 0xf0,0x04,0x0a,0x2a
+# GFX12: v_min_num_f32_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x2a]
 
-# GFX12: v_min_num_f32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x2a]
 0xfd,0x04,0x0a,0x2a
+# GFX12: v_min_num_f32_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x2a]
 
-# GFX12: v_min_num_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x2b,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x2b,0x56,0x34,0x12,0xaf
+# GFX12: v_min_num_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x2b,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_min_num_f64_e32 v[5:6], v[1:2], v[3:4] ; encoding: [0x01,0x07,0x0a,0x1a]
 0x01,0x07,0x0a,0x1a
+# GFX12: v_min_num_f64_e32 v[5:6], v[1:2], v[3:4] ; encoding: [0x01,0x07,0x0a,0x1a]
 
-# GFX12: v_min_num_f64_e32 v[5:6], v[254:255], v[2:3] ; encoding: [0xfe,0x05,0x0a,0x1a]
 0xfe,0x05,0x0a,0x1a
+# GFX12: v_min_num_f64_e32 v[5:6], v[254:255], v[2:3] ; encoding: [0xfe,0x05,0x0a,0x1a]
 
-# GFX12: v_min_num_f64_e32 v[5:6], s[0:1], v[2:3] ; encoding: [0x00,0x04,0x0a,0x1a]
 0x00,0x04,0x0a,0x1a
+# GFX12: v_min_num_f64_e32 v[5:6], s[0:1], v[2:3] ; encoding: [0x00,0x04,0x0a,0x1a]
 
-# GFX12: v_min_num_f64_e32 v[5:6], s[104:105], v[2:3] ; encoding: [0x68,0x04,0x0a,0x1a]
 0x68,0x04,0x0a,0x1a
+# GFX12: v_min_num_f64_e32 v[5:6], s[104:105], v[2:3] ; encoding: [0x68,0x04,0x0a,0x1a]
 
-# GFX12: v_min_num_f64_e32 v[5:6], vcc, v[2:3]   ; encoding: [0x6a,0x04,0x0a,0x1a]
 0x6a,0x04,0x0a,0x1a
+# GFX12: v_min_num_f64_e32 v[5:6], vcc, v[2:3]   ; encoding: [0x6a,0x04,0x0a,0x1a]
 
-# GFX12: v_min_num_f64_e32 v[5:6], ttmp[14:15], v[2:3] ; encoding: [0x7a,0x04,0x0a,0x1a]
 0x7a,0x04,0x0a,0x1a
+# GFX12: v_min_num_f64_e32 v[5:6], ttmp[14:15], v[2:3] ; encoding: [0x7a,0x04,0x0a,0x1a]
 
-# GFX12: v_min_num_f64_e32 v[5:6], exec, v[2:3]  ; encoding: [0x7e,0x04,0x0a,0x1a]
 0x7e,0x04,0x0a,0x1a
+# GFX12: v_min_num_f64_e32 v[5:6], exec, v[2:3]  ; encoding: [0x7e,0x04,0x0a,0x1a]
 
-# GFX12: v_min_num_f64_e32 v[5:6], null, v[2:3]  ; encoding: [0x7c,0x04,0x0a,0x1a]
 0x7c,0x04,0x0a,0x1a
+# GFX12: v_min_num_f64_e32 v[5:6], null, v[2:3]  ; encoding: [0x7c,0x04,0x0a,0x1a]
 
-# GFX12: v_min_num_f64_e32 v[5:6], -1, v[2:3]    ; encoding: [0xc1,0x04,0x0a,0x1a]
 0xc1,0x04,0x0a,0x1a
+# GFX12: v_min_num_f64_e32 v[5:6], -1, v[2:3]    ; encoding: [0xc1,0x04,0x0a,0x1a]
 
-# GFX12: v_min_num_f64_e32 v[5:6], 0.5, v[2:3]   ; encoding: [0xf0,0x04,0x0a,0x1a]
 0xf0,0x04,0x0a,0x1a
+# GFX12: v_min_num_f64_e32 v[5:6], 0.5, v[2:3]   ; encoding: [0xf0,0x04,0x0a,0x1a]
 
-# GFX12: v_min_num_f64_e32 v[5:6], src_scc, v[2:3] ; encoding: [0xfd,0x04,0x0a,0x1a]
 0xfd,0x04,0x0a,0x1a
+# GFX12: v_min_num_f64_e32 v[5:6], src_scc, v[2:3] ; encoding: [0xfd,0x04,0x0a,0x1a]
 
-# GFX12: v_min_num_f64_e32 v[254:255], 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xfd,0x1b,0x56,0x34,0x12,0xaf]
 0xff,0xfc,0xfd,0x1b,0x56,0x34,0x12,0xaf
+# GFX12: v_min_num_f64_e32 v[254:255], 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xfd,0x1b,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_min_i32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x22]
 0x01,0x05,0x0a,0x22
+# GFX12: v_min_i32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x22]
 
-# GFX12: v_min_i32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x22]
 0xff,0x05,0x0a,0x22
+# GFX12: v_min_i32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x22]
 
-# GFX12: v_min_i32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x22]
 0x01,0x04,0x0a,0x22
+# GFX12: v_min_i32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x22]
 
-# GFX12: v_min_i32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x22]
 0x69,0x04,0x0a,0x22
+# GFX12: v_min_i32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x22]
 
-# GFX12: v_min_i32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x22]
 0x6a,0x04,0x0a,0x22
+# GFX12: v_min_i32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x22]
 
-# GFX12: v_min_i32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x22]
 0x6b,0x04,0x0a,0x22
+# GFX12: v_min_i32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x22]
 
-# GFX12: v_min_i32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x22]
 0x7b,0x04,0x0a,0x22
+# GFX12: v_min_i32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x22]
 
-# GFX12: v_min_i32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x22]
 0x7d,0x04,0x0a,0x22
+# GFX12: v_min_i32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x22]
 
-# GFX12: v_min_i32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x22]
 0x7e,0x04,0x0a,0x22
+# GFX12: v_min_i32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x22]
 
-# GFX12: v_min_i32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x22]
 0x7f,0x04,0x0a,0x22
+# GFX12: v_min_i32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x22]
 
-# GFX12: v_min_i32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x22]
 0x7c,0x04,0x0a,0x22
+# GFX12: v_min_i32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x22]
 
-# GFX12: v_min_i32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x22]
 0xc1,0x04,0x0a,0x22
+# GFX12: v_min_i32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x22]
 
-# GFX12: v_min_i32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x22]
 0xf0,0x04,0x0a,0x22
+# GFX12: v_min_i32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x22]
 
-# GFX12: v_min_i32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x22]
 0xfd,0x04,0x0a,0x22
+# GFX12: v_min_i32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x22]
 
-# GFX12: v_min_i32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x23,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x23,0x56,0x34,0x12,0xaf
+# GFX12: v_min_i32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x23,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_min_u32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x26]
 0x01,0x05,0x0a,0x26
+# GFX12: v_min_u32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x26]
 
-# GFX12: v_min_u32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x26]
 0xff,0x05,0x0a,0x26
+# GFX12: v_min_u32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x26]
 
-# GFX12: v_min_u32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x26]
 0x01,0x04,0x0a,0x26
+# GFX12: v_min_u32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x26]
 
-# GFX12: v_min_u32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x26]
 0x69,0x04,0x0a,0x26
+# GFX12: v_min_u32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x26]
 
-# GFX12: v_min_u32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x26]
 0x6a,0x04,0x0a,0x26
+# GFX12: v_min_u32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x26]
 
-# GFX12: v_min_u32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x26]
 0x6b,0x04,0x0a,0x26
+# GFX12: v_min_u32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x26]
 
-# GFX12: v_min_u32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x26]
 0x7b,0x04,0x0a,0x26
+# GFX12: v_min_u32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x26]
 
-# GFX12: v_min_u32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x26]
 0x7d,0x04,0x0a,0x26
+# GFX12: v_min_u32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x26]
 
-# GFX12: v_min_u32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x26]
 0x7e,0x04,0x0a,0x26
+# GFX12: v_min_u32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x26]
 
-# GFX12: v_min_u32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x26]
 0x7f,0x04,0x0a,0x26
+# GFX12: v_min_u32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x26]
 
-# GFX12: v_min_u32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x26]
 0x7c,0x04,0x0a,0x26
+# GFX12: v_min_u32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x26]
 
-# GFX12: v_min_u32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x26]
 0xc1,0x04,0x0a,0x26
+# GFX12: v_min_u32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x26]
 
-# GFX12: v_min_u32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x26]
 0xf0,0x04,0x0a,0x26
+# GFX12: v_min_u32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x26]
 
-# GFX12: v_min_u32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x26]
 0xfd,0x04,0x0a,0x26
+# GFX12: v_min_u32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x26]
 
-# GFX12: v_min_u32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x27,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x27,0x56,0x34,0x12,0xaf
+# GFX12: v_min_u32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x27,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mul_dx9_zero_f32_e32 v5, v1, v2       ; encoding: [0x01,0x05,0x0a,0x0e]
 0x01,0x05,0x0a,0x0e
+# GFX12: v_mul_dx9_zero_f32_e32 v5, v1, v2       ; encoding: [0x01,0x05,0x0a,0x0e]
 
-# GFX12: v_mul_dx9_zero_f32_e32 v5, v255, v2     ; encoding: [0xff,0x05,0x0a,0x0e]
 0xff,0x05,0x0a,0x0e
+# GFX12: v_mul_dx9_zero_f32_e32 v5, v255, v2     ; encoding: [0xff,0x05,0x0a,0x0e]
 
-# GFX12: v_mul_dx9_zero_f32_e32 v5, s1, v2       ; encoding: [0x01,0x04,0x0a,0x0e]
 0x01,0x04,0x0a,0x0e
+# GFX12: v_mul_dx9_zero_f32_e32 v5, s1, v2       ; encoding: [0x01,0x04,0x0a,0x0e]
 
-# GFX12: v_mul_dx9_zero_f32_e32 v5, s105, v2     ; encoding: [0x69,0x04,0x0a,0x0e]
 0x69,0x04,0x0a,0x0e
+# GFX12: v_mul_dx9_zero_f32_e32 v5, s105, v2     ; encoding: [0x69,0x04,0x0a,0x0e]
 
-# GFX12: v_mul_dx9_zero_f32_e32 v5, vcc_lo, v2   ; encoding: [0x6a,0x04,0x0a,0x0e]
 0x6a,0x04,0x0a,0x0e
+# GFX12: v_mul_dx9_zero_f32_e32 v5, vcc_lo, v2   ; encoding: [0x6a,0x04,0x0a,0x0e]
 
-# GFX12: v_mul_dx9_zero_f32_e32 v5, vcc_hi, v2   ; encoding: [0x6b,0x04,0x0a,0x0e]
 0x6b,0x04,0x0a,0x0e
+# GFX12: v_mul_dx9_zero_f32_e32 v5, vcc_hi, v2   ; encoding: [0x6b,0x04,0x0a,0x0e]
 
-# GFX12: v_mul_dx9_zero_f32_e32 v5, ttmp15, v2   ; encoding: [0x7b,0x04,0x0a,0x0e]
 0x7b,0x04,0x0a,0x0e
+# GFX12: v_mul_dx9_zero_f32_e32 v5, ttmp15, v2   ; encoding: [0x7b,0x04,0x0a,0x0e]
 
-# GFX12: v_mul_dx9_zero_f32_e32 v5, m0, v2       ; encoding: [0x7d,0x04,0x0a,0x0e]
 0x7d,0x04,0x0a,0x0e
+# GFX12: v_mul_dx9_zero_f32_e32 v5, m0, v2       ; encoding: [0x7d,0x04,0x0a,0x0e]
 
-# GFX12: v_mul_dx9_zero_f32_e32 v5, exec_lo, v2  ; encoding: [0x7e,0x04,0x0a,0x0e]
 0x7e,0x04,0x0a,0x0e
+# GFX12: v_mul_dx9_zero_f32_e32 v5, exec_lo, v2  ; encoding: [0x7e,0x04,0x0a,0x0e]
 
-# GFX12: v_mul_dx9_zero_f32_e32 v5, exec_hi, v2  ; encoding: [0x7f,0x04,0x0a,0x0e]
 0x7f,0x04,0x0a,0x0e
+# GFX12: v_mul_dx9_zero_f32_e32 v5, exec_hi, v2  ; encoding: [0x7f,0x04,0x0a,0x0e]
 
-# GFX12: v_mul_dx9_zero_f32_e32 v5, null, v2     ; encoding: [0x7c,0x04,0x0a,0x0e]
 0x7c,0x04,0x0a,0x0e
+# GFX12: v_mul_dx9_zero_f32_e32 v5, null, v2     ; encoding: [0x7c,0x04,0x0a,0x0e]
 
-# GFX12: v_mul_dx9_zero_f32_e32 v5, -1, v2       ; encoding: [0xc1,0x04,0x0a,0x0e]
 0xc1,0x04,0x0a,0x0e
+# GFX12: v_mul_dx9_zero_f32_e32 v5, -1, v2       ; encoding: [0xc1,0x04,0x0a,0x0e]
 
-# GFX12: v_mul_dx9_zero_f32_e32 v5, 0.5, v2      ; encoding: [0xf0,0x04,0x0a,0x0e]
 0xf0,0x04,0x0a,0x0e
+# GFX12: v_mul_dx9_zero_f32_e32 v5, 0.5, v2      ; encoding: [0xf0,0x04,0x0a,0x0e]
 
-# GFX12: v_mul_dx9_zero_f32_e32 v5, src_scc, v2  ; encoding: [0xfd,0x04,0x0a,0x0e]
 0xfd,0x04,0x0a,0x0e
+# GFX12: v_mul_dx9_zero_f32_e32 v5, src_scc, v2  ; encoding: [0xfd,0x04,0x0a,0x0e]
 
-# GFX12: v_mul_dx9_zero_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x0f,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x0f,0x56,0x34,0x12,0xaf
+# GFX12: v_mul_dx9_zero_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x0f,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mul_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x6a]
 0x01,0x05,0x0a,0x6a
+# GFX12-REAL16: v_mul_f16_e32 v5.l, v1.l, v2.l          ; encoding: [0x01,0x05,0x0a,0x6a]
+# GFX12-FAKE16: v_mul_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x6a]
 
-# GFX12: v_mul_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x6a]
 0x7f,0x05,0x0a,0x6a
+# GFX12-REAL16: v_mul_f16_e32 v5.l, v127.l, v2.l        ; encoding: [0x7f,0x05,0x0a,0x6a]
+# GFX12-FAKE16: v_mul_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x6a]
 
-# GFX12: v_mul_f16_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x6a]
 0x01,0x04,0x0a,0x6a
+# GFX12-REAL16: v_mul_f16_e32 v5.l, s1, v2.l            ; encoding: [0x01,0x04,0x0a,0x6a]
+# GFX12-FAKE16: v_mul_f16_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x6a]
 
-# GFX12: v_mul_f16_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x6a]
 0x69,0x04,0x0a,0x6a
+# GFX12-REAL16: v_mul_f16_e32 v5.l, s105, v2.l          ; encoding: [0x69,0x04,0x0a,0x6a]
+# GFX12-FAKE16: v_mul_f16_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x6a]
 
-# GFX12: v_mul_f16_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x6a]
 0x6a,0x04,0x0a,0x6a
+# GFX12-REAL16: v_mul_f16_e32 v5.l, vcc_lo, v2.l        ; encoding: [0x6a,0x04,0x0a,0x6a]
+# GFX12-FAKE16: v_mul_f16_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x6a]
 
-# GFX12: v_mul_f16_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x6a]
 0x6b,0x04,0x0a,0x6a
+# GFX12-REAL16: v_mul_f16_e32 v5.l, vcc_hi, v2.l        ; encoding: [0x6b,0x04,0x0a,0x6a]
+# GFX12-FAKE16: v_mul_f16_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x6a]
 
-# GFX12: v_mul_f16_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x6a]
 0x7b,0x04,0x0a,0x6a
+# GFX12-REAL16: v_mul_f16_e32 v5.l, ttmp15, v2.l        ; encoding: [0x7b,0x04,0x0a,0x6a]
+# GFX12-FAKE16: v_mul_f16_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x6a]
 
-# GFX12: v_mul_f16_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x6a]
 0x7d,0x04,0x0a,0x6a
+# GFX12-REAL16: v_mul_f16_e32 v5.l, m0, v2.l            ; encoding: [0x7d,0x04,0x0a,0x6a]
+# GFX12-FAKE16: v_mul_f16_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x6a]
 
-# GFX12: v_mul_f16_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x6a]
 0x7e,0x04,0x0a,0x6a
+# GFX12-REAL16: v_mul_f16_e32 v5.l, exec_lo, v2.l       ; encoding: [0x7e,0x04,0x0a,0x6a]
+# GFX12-FAKE16: v_mul_f16_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x6a]
 
-# GFX12: v_mul_f16_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x6a]
 0x7f,0x04,0x0a,0x6a
+# GFX12-REAL16: v_mul_f16_e32 v5.l, exec_hi, v2.l       ; encoding: [0x7f,0x04,0x0a,0x6a]
+# GFX12-FAKE16: v_mul_f16_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x6a]
 
-# GFX12: v_mul_f16_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x6a]
 0x7c,0x04,0x0a,0x6a
+# GFX12-REAL16: v_mul_f16_e32 v5.l, null, v2.l          ; encoding: [0x7c,0x04,0x0a,0x6a]
+# GFX12-FAKE16: v_mul_f16_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x6a]
 
-# GFX12: v_mul_f16_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x6a]
 0xc1,0x04,0x0a,0x6a
+# GFX12-REAL16: v_mul_f16_e32 v5.l, -1, v2.l            ; encoding: [0xc1,0x04,0x0a,0x6a]
+# GFX12-FAKE16: v_mul_f16_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x6a]
 
-# GFX12: v_mul_f16_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x6a]
 0xf0,0x04,0x0a,0x6a
+# GFX12-REAL16: v_mul_f16_e32 v5.l, 0.5, v2.l           ; encoding: [0xf0,0x04,0x0a,0x6a]
+# GFX12-FAKE16: v_mul_f16_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x6a]
 
-# GFX12: v_mul_f16_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x6a]
 0xfd,0x04,0x0a,0x6a
+# GFX12-REAL16: v_mul_f16_e32 v5.l, src_scc, v2.l       ; encoding: [0xfd,0x04,0x0a,0x6a]
+# GFX12-FAKE16: v_mul_f16_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x6a]
 
-# GFX12: v_mul_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x6a,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xfe,0x6a,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_mul_f16_e32 v127.l, 0xfe0b, v127.l    ; encoding: [0xff,0xfe,0xfe,0x6a,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_mul_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x6a,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_mul_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x10]
 0x01,0x05,0x0a,0x10
+# GFX12: v_mul_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x10]
 
-# GFX12: v_mul_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x10]
 0xff,0x05,0x0a,0x10
+# GFX12: v_mul_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x10]
 
-# GFX12: v_mul_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x10]
 0x01,0x04,0x0a,0x10
+# GFX12: v_mul_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x10]
 
-# GFX12: v_mul_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x10]
 0x69,0x04,0x0a,0x10
+# GFX12: v_mul_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x10]
 
-# GFX12: v_mul_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x10]
 0x6a,0x04,0x0a,0x10
+# GFX12: v_mul_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x10]
 
-# GFX12: v_mul_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x10]
 0x6b,0x04,0x0a,0x10
+# GFX12: v_mul_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x10]
 
-# GFX12: v_mul_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x10]
 0x7b,0x04,0x0a,0x10
+# GFX12: v_mul_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x10]
 
-# GFX12: v_mul_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x10]
 0x7d,0x04,0x0a,0x10
+# GFX12: v_mul_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x10]
 
-# GFX12: v_mul_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x10]
 0x7e,0x04,0x0a,0x10
+# GFX12: v_mul_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x10]
 
-# GFX12: v_mul_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x10]
 0x7f,0x04,0x0a,0x10
+# GFX12: v_mul_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x10]
 
-# GFX12: v_mul_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x10]
 0x7c,0x04,0x0a,0x10
+# GFX12: v_mul_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x10]
 
-# GFX12: v_mul_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x10]
 0xc1,0x04,0x0a,0x10
+# GFX12: v_mul_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x10]
 
-# GFX12: v_mul_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x10]
 0xf0,0x04,0x0a,0x10
+# GFX12: v_mul_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x10]
 
-# GFX12: v_mul_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x10]
 0xfd,0x04,0x0a,0x10
+# GFX12: v_mul_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x10]
 
-# GFX12: v_mul_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x11,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x11,0x56,0x34,0x12,0xaf
+# GFX12: v_mul_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x11,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mul_f64_e32 v[5:6], v[1:2], v[3:4]    ; encoding: [0x01,0x07,0x0a,0x0c]
 0x01,0x07,0x0a,0x0c
+# GFX12: v_mul_f64_e32 v[5:6], v[1:2], v[3:4]    ; encoding: [0x01,0x07,0x0a,0x0c]
 
-# GFX12: v_mul_f64_e32 v[5:6], v[254:255], v[2:3] ; encoding: [0xfe,0x05,0x0a,0x0c]
 0xfe,0x05,0x0a,0x0c
+# GFX12: v_mul_f64_e32 v[5:6], v[254:255], v[2:3] ; encoding: [0xfe,0x05,0x0a,0x0c]
 
-# GFX12: v_mul_f64_e32 v[5:6], s[0:1], v[2:3]    ; encoding: [0x00,0x04,0x0a,0x0c]
 0x00,0x04,0x0a,0x0c
+# GFX12: v_mul_f64_e32 v[5:6], s[0:1], v[2:3]    ; encoding: [0x00,0x04,0x0a,0x0c]
 
-# GFX12: v_mul_f64_e32 v[5:6], s[104:105], v[2:3] ; encoding: [0x68,0x04,0x0a,0x0c]
 0x68,0x04,0x0a,0x0c
+# GFX12: v_mul_f64_e32 v[5:6], s[104:105], v[2:3] ; encoding: [0x68,0x04,0x0a,0x0c]
 
-# GFX12: v_mul_f64_e32 v[5:6], vcc, v[2:3]       ; encoding: [0x6a,0x04,0x0a,0x0c]
 0x6a,0x04,0x0a,0x0c
+# GFX12: v_mul_f64_e32 v[5:6], vcc, v[2:3]       ; encoding: [0x6a,0x04,0x0a,0x0c]
 
-# GFX12: v_mul_f64_e32 v[5:6], ttmp[14:15], v[2:3] ; encoding: [0x7a,0x04,0x0a,0x0c]
 0x7a,0x04,0x0a,0x0c
+# GFX12: v_mul_f64_e32 v[5:6], ttmp[14:15], v[2:3] ; encoding: [0x7a,0x04,0x0a,0x0c]
 
-# GFX12: v_mul_f64_e32 v[5:6], exec, v[2:3]      ; encoding: [0x7e,0x04,0x0a,0x0c]
 0x7e,0x04,0x0a,0x0c
+# GFX12: v_mul_f64_e32 v[5:6], exec, v[2:3]      ; encoding: [0x7e,0x04,0x0a,0x0c]
 
-# GFX12: v_mul_f64_e32 v[5:6], null, v[2:3]      ; encoding: [0x7c,0x04,0x0a,0x0c]
 0x7c,0x04,0x0a,0x0c
+# GFX12: v_mul_f64_e32 v[5:6], null, v[2:3]      ; encoding: [0x7c,0x04,0x0a,0x0c]
 
-# GFX12: v_mul_f64_e32 v[5:6], -1, v[2:3]        ; encoding: [0xc1,0x04,0x0a,0x0c]
 0xc1,0x04,0x0a,0x0c
+# GFX12: v_mul_f64_e32 v[5:6], -1, v[2:3]        ; encoding: [0xc1,0x04,0x0a,0x0c]
 
-# GFX12: v_mul_f64_e32 v[5:6], 0.5, v[2:3]       ; encoding: [0xf0,0x04,0x0a,0x0c]
 0xf0,0x04,0x0a,0x0c
+# GFX12: v_mul_f64_e32 v[5:6], 0.5, v[2:3]       ; encoding: [0xf0,0x04,0x0a,0x0c]
 
-# GFX12: v_mul_f64_e32 v[5:6], src_scc, v[2:3]   ; encoding: [0xfd,0x04,0x0a,0x0c]
 0xfd,0x04,0x0a,0x0c
+# GFX12: v_mul_f64_e32 v[5:6], src_scc, v[2:3]   ; encoding: [0xfd,0x04,0x0a,0x0c]
 
-# GFX12: v_mul_f64_e32 v[254:255], 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xfd,0x0d,0x56,0x34,0x12,0xaf]
 0xff,0xfc,0xfd,0x0d,0x56,0x34,0x12,0xaf
+# GFX12: v_mul_f64_e32 v[254:255], 0xaf123456, v[254:255] ; encoding: [0xff,0xfc,0xfd,0x0d,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mul_hi_i32_i24_e32 v5, v1, v2         ; encoding: [0x01,0x05,0x0a,0x14]
 0x01,0x05,0x0a,0x14
+# GFX12: v_mul_hi_i32_i24_e32 v5, v1, v2         ; encoding: [0x01,0x05,0x0a,0x14]
 
-# GFX12: v_mul_hi_i32_i24_e32 v5, v255, v2       ; encoding: [0xff,0x05,0x0a,0x14]
 0xff,0x05,0x0a,0x14
+# GFX12: v_mul_hi_i32_i24_e32 v5, v255, v2       ; encoding: [0xff,0x05,0x0a,0x14]
 
-# GFX12: v_mul_hi_i32_i24_e32 v5, s1, v2         ; encoding: [0x01,0x04,0x0a,0x14]
 0x01,0x04,0x0a,0x14
+# GFX12: v_mul_hi_i32_i24_e32 v5, s1, v2         ; encoding: [0x01,0x04,0x0a,0x14]
 
-# GFX12: v_mul_hi_i32_i24_e32 v5, s105, v2       ; encoding: [0x69,0x04,0x0a,0x14]
 0x69,0x04,0x0a,0x14
+# GFX12: v_mul_hi_i32_i24_e32 v5, s105, v2       ; encoding: [0x69,0x04,0x0a,0x14]
 
-# GFX12: v_mul_hi_i32_i24_e32 v5, vcc_lo, v2     ; encoding: [0x6a,0x04,0x0a,0x14]
 0x6a,0x04,0x0a,0x14
+# GFX12: v_mul_hi_i32_i24_e32 v5, vcc_lo, v2     ; encoding: [0x6a,0x04,0x0a,0x14]
 
-# GFX12: v_mul_hi_i32_i24_e32 v5, vcc_hi, v2     ; encoding: [0x6b,0x04,0x0a,0x14]
 0x6b,0x04,0x0a,0x14
+# GFX12: v_mul_hi_i32_i24_e32 v5, vcc_hi, v2     ; encoding: [0x6b,0x04,0x0a,0x14]
 
-# GFX12: v_mul_hi_i32_i24_e32 v5, ttmp15, v2     ; encoding: [0x7b,0x04,0x0a,0x14]
 0x7b,0x04,0x0a,0x14
+# GFX12: v_mul_hi_i32_i24_e32 v5, ttmp15, v2     ; encoding: [0x7b,0x04,0x0a,0x14]
 
-# GFX12: v_mul_hi_i32_i24_e32 v5, m0, v2         ; encoding: [0x7d,0x04,0x0a,0x14]
 0x7d,0x04,0x0a,0x14
+# GFX12: v_mul_hi_i32_i24_e32 v5, m0, v2         ; encoding: [0x7d,0x04,0x0a,0x14]
 
-# GFX12: v_mul_hi_i32_i24_e32 v5, exec_lo, v2    ; encoding: [0x7e,0x04,0x0a,0x14]
 0x7e,0x04,0x0a,0x14
+# GFX12: v_mul_hi_i32_i24_e32 v5, exec_lo, v2    ; encoding: [0x7e,0x04,0x0a,0x14]
 
-# GFX12: v_mul_hi_i32_i24_e32 v5, exec_hi, v2    ; encoding: [0x7f,0x04,0x0a,0x14]
 0x7f,0x04,0x0a,0x14
+# GFX12: v_mul_hi_i32_i24_e32 v5, exec_hi, v2    ; encoding: [0x7f,0x04,0x0a,0x14]
 
-# GFX12: v_mul_hi_i32_i24_e32 v5, null, v2       ; encoding: [0x7c,0x04,0x0a,0x14]
 0x7c,0x04,0x0a,0x14
+# GFX12: v_mul_hi_i32_i24_e32 v5, null, v2       ; encoding: [0x7c,0x04,0x0a,0x14]
 
-# GFX12: v_mul_hi_i32_i24_e32 v5, -1, v2         ; encoding: [0xc1,0x04,0x0a,0x14]
 0xc1,0x04,0x0a,0x14
+# GFX12: v_mul_hi_i32_i24_e32 v5, -1, v2         ; encoding: [0xc1,0x04,0x0a,0x14]
 
-# GFX12: v_mul_hi_i32_i24_e32 v5, 0.5, v2        ; encoding: [0xf0,0x04,0x0a,0x14]
 0xf0,0x04,0x0a,0x14
+# GFX12: v_mul_hi_i32_i24_e32 v5, 0.5, v2        ; encoding: [0xf0,0x04,0x0a,0x14]
 
-# GFX12: v_mul_hi_i32_i24_e32 v5, src_scc, v2    ; encoding: [0xfd,0x04,0x0a,0x14]
 0xfd,0x04,0x0a,0x14
+# GFX12: v_mul_hi_i32_i24_e32 v5, src_scc, v2    ; encoding: [0xfd,0x04,0x0a,0x14]
 
-# GFX12: v_mul_hi_i32_i24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x15,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x15,0x56,0x34,0x12,0xaf
+# GFX12: v_mul_hi_i32_i24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x15,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mul_hi_u32_u24_e32 v5, v1, v2         ; encoding: [0x01,0x05,0x0a,0x18]
 0x01,0x05,0x0a,0x18
+# GFX12: v_mul_hi_u32_u24_e32 v5, v1, v2         ; encoding: [0x01,0x05,0x0a,0x18]
 
-# GFX12: v_mul_hi_u32_u24_e32 v5, v255, v2       ; encoding: [0xff,0x05,0x0a,0x18]
 0xff,0x05,0x0a,0x18
+# GFX12: v_mul_hi_u32_u24_e32 v5, v255, v2       ; encoding: [0xff,0x05,0x0a,0x18]
 
-# GFX12: v_mul_hi_u32_u24_e32 v5, s1, v2         ; encoding: [0x01,0x04,0x0a,0x18]
 0x01,0x04,0x0a,0x18
+# GFX12: v_mul_hi_u32_u24_e32 v5, s1, v2         ; encoding: [0x01,0x04,0x0a,0x18]
 
-# GFX12: v_mul_hi_u32_u24_e32 v5, s105, v2       ; encoding: [0x69,0x04,0x0a,0x18]
 0x69,0x04,0x0a,0x18
+# GFX12: v_mul_hi_u32_u24_e32 v5, s105, v2       ; encoding: [0x69,0x04,0x0a,0x18]
 
-# GFX12: v_mul_hi_u32_u24_e32 v5, vcc_lo, v2     ; encoding: [0x6a,0x04,0x0a,0x18]
 0x6a,0x04,0x0a,0x18
+# GFX12: v_mul_hi_u32_u24_e32 v5, vcc_lo, v2     ; encoding: [0x6a,0x04,0x0a,0x18]
 
-# GFX12: v_mul_hi_u32_u24_e32 v5, vcc_hi, v2     ; encoding: [0x6b,0x04,0x0a,0x18]
 0x6b,0x04,0x0a,0x18
+# GFX12: v_mul_hi_u32_u24_e32 v5, vcc_hi, v2     ; encoding: [0x6b,0x04,0x0a,0x18]
 
-# GFX12: v_mul_hi_u32_u24_e32 v5, ttmp15, v2     ; encoding: [0x7b,0x04,0x0a,0x18]
 0x7b,0x04,0x0a,0x18
+# GFX12: v_mul_hi_u32_u24_e32 v5, ttmp15, v2     ; encoding: [0x7b,0x04,0x0a,0x18]
 
-# GFX12: v_mul_hi_u32_u24_e32 v5, m0, v2         ; encoding: [0x7d,0x04,0x0a,0x18]
 0x7d,0x04,0x0a,0x18
+# GFX12: v_mul_hi_u32_u24_e32 v5, m0, v2         ; encoding: [0x7d,0x04,0x0a,0x18]
 
-# GFX12: v_mul_hi_u32_u24_e32 v5, exec_lo, v2    ; encoding: [0x7e,0x04,0x0a,0x18]
 0x7e,0x04,0x0a,0x18
+# GFX12: v_mul_hi_u32_u24_e32 v5, exec_lo, v2    ; encoding: [0x7e,0x04,0x0a,0x18]
 
-# GFX12: v_mul_hi_u32_u24_e32 v5, exec_hi, v2    ; encoding: [0x7f,0x04,0x0a,0x18]
 0x7f,0x04,0x0a,0x18
+# GFX12: v_mul_hi_u32_u24_e32 v5, exec_hi, v2    ; encoding: [0x7f,0x04,0x0a,0x18]
 
-# GFX12: v_mul_hi_u32_u24_e32 v5, null, v2       ; encoding: [0x7c,0x04,0x0a,0x18]
 0x7c,0x04,0x0a,0x18
+# GFX12: v_mul_hi_u32_u24_e32 v5, null, v2       ; encoding: [0x7c,0x04,0x0a,0x18]
 
-# GFX12: v_mul_hi_u32_u24_e32 v5, -1, v2         ; encoding: [0xc1,0x04,0x0a,0x18]
 0xc1,0x04,0x0a,0x18
+# GFX12: v_mul_hi_u32_u24_e32 v5, -1, v2         ; encoding: [0xc1,0x04,0x0a,0x18]
 
-# GFX12: v_mul_hi_u32_u24_e32 v5, 0.5, v2        ; encoding: [0xf0,0x04,0x0a,0x18]
 0xf0,0x04,0x0a,0x18
+# GFX12: v_mul_hi_u32_u24_e32 v5, 0.5, v2        ; encoding: [0xf0,0x04,0x0a,0x18]
 
-# GFX12: v_mul_hi_u32_u24_e32 v5, src_scc, v2    ; encoding: [0xfd,0x04,0x0a,0x18]
 0xfd,0x04,0x0a,0x18
+# GFX12: v_mul_hi_u32_u24_e32 v5, src_scc, v2    ; encoding: [0xfd,0x04,0x0a,0x18]
 
-# GFX12: v_mul_hi_u32_u24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x19,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x19,0x56,0x34,0x12,0xaf
+# GFX12: v_mul_hi_u32_u24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x19,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mul_i32_i24_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x12]
 0x01,0x05,0x0a,0x12
+# GFX12: v_mul_i32_i24_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x12]
 
-# GFX12: v_mul_i32_i24_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x12]
 0xff,0x05,0x0a,0x12
+# GFX12: v_mul_i32_i24_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x12]
 
-# GFX12: v_mul_i32_i24_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x12]
 0x01,0x04,0x0a,0x12
+# GFX12: v_mul_i32_i24_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x12]
 
-# GFX12: v_mul_i32_i24_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x12]
 0x69,0x04,0x0a,0x12
+# GFX12: v_mul_i32_i24_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x12]
 
-# GFX12: v_mul_i32_i24_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x12]
 0x6a,0x04,0x0a,0x12
+# GFX12: v_mul_i32_i24_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x12]
 
-# GFX12: v_mul_i32_i24_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x12]
 0x6b,0x04,0x0a,0x12
+# GFX12: v_mul_i32_i24_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x12]
 
-# GFX12: v_mul_i32_i24_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x12]
 0x7b,0x04,0x0a,0x12
+# GFX12: v_mul_i32_i24_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x12]
 
-# GFX12: v_mul_i32_i24_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x12]
 0x7d,0x04,0x0a,0x12
+# GFX12: v_mul_i32_i24_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x12]
 
-# GFX12: v_mul_i32_i24_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x12]
 0x7e,0x04,0x0a,0x12
+# GFX12: v_mul_i32_i24_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x12]
 
-# GFX12: v_mul_i32_i24_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x12]
 0x7f,0x04,0x0a,0x12
+# GFX12: v_mul_i32_i24_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x12]
 
-# GFX12: v_mul_i32_i24_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x12]
 0x7c,0x04,0x0a,0x12
+# GFX12: v_mul_i32_i24_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x12]
 
-# GFX12: v_mul_i32_i24_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x12]
 0xc1,0x04,0x0a,0x12
+# GFX12: v_mul_i32_i24_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x12]
 
-# GFX12: v_mul_i32_i24_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x12]
 0xf0,0x04,0x0a,0x12
+# GFX12: v_mul_i32_i24_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x12]
 
-# GFX12: v_mul_i32_i24_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x12]
 0xfd,0x04,0x0a,0x12
+# GFX12: v_mul_i32_i24_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x12]
 
-# GFX12: v_mul_i32_i24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x13,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x13,0x56,0x34,0x12,0xaf
+# GFX12: v_mul_i32_i24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x13,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mul_u32_u24_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x16]
 0x01,0x05,0x0a,0x16
+# GFX12: v_mul_u32_u24_e32 v5, v1, v2            ; encoding: [0x01,0x05,0x0a,0x16]
 
-# GFX12: v_mul_u32_u24_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x16]
 0xff,0x05,0x0a,0x16
+# GFX12: v_mul_u32_u24_e32 v5, v255, v2          ; encoding: [0xff,0x05,0x0a,0x16]
 
-# GFX12: v_mul_u32_u24_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x16]
 0x01,0x04,0x0a,0x16
+# GFX12: v_mul_u32_u24_e32 v5, s1, v2            ; encoding: [0x01,0x04,0x0a,0x16]
 
-# GFX12: v_mul_u32_u24_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x16]
 0x69,0x04,0x0a,0x16
+# GFX12: v_mul_u32_u24_e32 v5, s105, v2          ; encoding: [0x69,0x04,0x0a,0x16]
 
-# GFX12: v_mul_u32_u24_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x16]
 0x6a,0x04,0x0a,0x16
+# GFX12: v_mul_u32_u24_e32 v5, vcc_lo, v2        ; encoding: [0x6a,0x04,0x0a,0x16]
 
-# GFX12: v_mul_u32_u24_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x16]
 0x6b,0x04,0x0a,0x16
+# GFX12: v_mul_u32_u24_e32 v5, vcc_hi, v2        ; encoding: [0x6b,0x04,0x0a,0x16]
 
-# GFX12: v_mul_u32_u24_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x16]
 0x7b,0x04,0x0a,0x16
+# GFX12: v_mul_u32_u24_e32 v5, ttmp15, v2        ; encoding: [0x7b,0x04,0x0a,0x16]
 
-# GFX12: v_mul_u32_u24_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x16]
 0x7d,0x04,0x0a,0x16
+# GFX12: v_mul_u32_u24_e32 v5, m0, v2            ; encoding: [0x7d,0x04,0x0a,0x16]
 
-# GFX12: v_mul_u32_u24_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x16]
 0x7e,0x04,0x0a,0x16
+# GFX12: v_mul_u32_u24_e32 v5, exec_lo, v2       ; encoding: [0x7e,0x04,0x0a,0x16]
 
-# GFX12: v_mul_u32_u24_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x16]
 0x7f,0x04,0x0a,0x16
+# GFX12: v_mul_u32_u24_e32 v5, exec_hi, v2       ; encoding: [0x7f,0x04,0x0a,0x16]
 
-# GFX12: v_mul_u32_u24_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x16]
 0x7c,0x04,0x0a,0x16
+# GFX12: v_mul_u32_u24_e32 v5, null, v2          ; encoding: [0x7c,0x04,0x0a,0x16]
 
-# GFX12: v_mul_u32_u24_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x16]
 0xc1,0x04,0x0a,0x16
+# GFX12: v_mul_u32_u24_e32 v5, -1, v2            ; encoding: [0xc1,0x04,0x0a,0x16]
 
-# GFX12: v_mul_u32_u24_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x16]
 0xf0,0x04,0x0a,0x16
+# GFX12: v_mul_u32_u24_e32 v5, 0.5, v2           ; encoding: [0xf0,0x04,0x0a,0x16]
 
-# GFX12: v_mul_u32_u24_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x16]
 0xfd,0x04,0x0a,0x16
+# GFX12: v_mul_u32_u24_e32 v5, src_scc, v2       ; encoding: [0xfd,0x04,0x0a,0x16]
 
-# GFX12: v_mul_u32_u24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x17,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x17,0x56,0x34,0x12,0xaf
+# GFX12: v_mul_u32_u24_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x17,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_or_b32_e32 v5, v1, v2                 ; encoding: [0x01,0x05,0x0a,0x38]
 0x01,0x05,0x0a,0x38
+# GFX12: v_or_b32_e32 v5, v1, v2                 ; encoding: [0x01,0x05,0x0a,0x38]
 
-# GFX12: v_or_b32_e32 v5, v255, v2               ; encoding: [0xff,0x05,0x0a,0x38]
 0xff,0x05,0x0a,0x38
+# GFX12: v_or_b32_e32 v5, v255, v2               ; encoding: [0xff,0x05,0x0a,0x38]
 
-# GFX12: v_or_b32_e32 v5, s1, v2                 ; encoding: [0x01,0x04,0x0a,0x38]
 0x01,0x04,0x0a,0x38
+# GFX12: v_or_b32_e32 v5, s1, v2                 ; encoding: [0x01,0x04,0x0a,0x38]
 
-# GFX12: v_or_b32_e32 v5, s105, v2               ; encoding: [0x69,0x04,0x0a,0x38]
 0x69,0x04,0x0a,0x38
+# GFX12: v_or_b32_e32 v5, s105, v2               ; encoding: [0x69,0x04,0x0a,0x38]
 
-# GFX12: v_or_b32_e32 v5, vcc_lo, v2             ; encoding: [0x6a,0x04,0x0a,0x38]
 0x6a,0x04,0x0a,0x38
+# GFX12: v_or_b32_e32 v5, vcc_lo, v2             ; encoding: [0x6a,0x04,0x0a,0x38]
 
-# GFX12: v_or_b32_e32 v5, vcc_hi, v2             ; encoding: [0x6b,0x04,0x0a,0x38]
 0x6b,0x04,0x0a,0x38
+# GFX12: v_or_b32_e32 v5, vcc_hi, v2             ; encoding: [0x6b,0x04,0x0a,0x38]
 
-# GFX12: v_or_b32_e32 v5, ttmp15, v2             ; encoding: [0x7b,0x04,0x0a,0x38]
 0x7b,0x04,0x0a,0x38
+# GFX12: v_or_b32_e32 v5, ttmp15, v2             ; encoding: [0x7b,0x04,0x0a,0x38]
 
-# GFX12: v_or_b32_e32 v5, m0, v2                 ; encoding: [0x7d,0x04,0x0a,0x38]
 0x7d,0x04,0x0a,0x38
+# GFX12: v_or_b32_e32 v5, m0, v2                 ; encoding: [0x7d,0x04,0x0a,0x38]
 
-# GFX12: v_or_b32_e32 v5, exec_lo, v2            ; encoding: [0x7e,0x04,0x0a,0x38]
 0x7e,0x04,0x0a,0x38
+# GFX12: v_or_b32_e32 v5, exec_lo, v2            ; encoding: [0x7e,0x04,0x0a,0x38]
 
-# GFX12: v_or_b32_e32 v5, exec_hi, v2            ; encoding: [0x7f,0x04,0x0a,0x38]
 0x7f,0x04,0x0a,0x38
+# GFX12: v_or_b32_e32 v5, exec_hi, v2            ; encoding: [0x7f,0x04,0x0a,0x38]
 
-# GFX12: v_or_b32_e32 v5, null, v2               ; encoding: [0x7c,0x04,0x0a,0x38]
 0x7c,0x04,0x0a,0x38
+# GFX12: v_or_b32_e32 v5, null, v2               ; encoding: [0x7c,0x04,0x0a,0x38]
 
-# GFX12: v_or_b32_e32 v5, -1, v2                 ; encoding: [0xc1,0x04,0x0a,0x38]
 0xc1,0x04,0x0a,0x38
+# GFX12: v_or_b32_e32 v5, -1, v2                 ; encoding: [0xc1,0x04,0x0a,0x38]
 
-# GFX12: v_or_b32_e32 v5, 0.5, v2                ; encoding: [0xf0,0x04,0x0a,0x38]
 0xf0,0x04,0x0a,0x38
+# GFX12: v_or_b32_e32 v5, 0.5, v2                ; encoding: [0xf0,0x04,0x0a,0x38]
 
-# GFX12: v_or_b32_e32 v5, src_scc, v2            ; encoding: [0xfd,0x04,0x0a,0x38]
 0xfd,0x04,0x0a,0x38
+# GFX12: v_or_b32_e32 v5, src_scc, v2            ; encoding: [0xfd,0x04,0x0a,0x38]
 
-# GFX12: v_or_b32_e32 v255, 0xaf123456, v255     ; encoding: [0xff,0xfe,0xff,0x39,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x39,0x56,0x34,0x12,0xaf
+# GFX12: v_or_b32_e32 v255, 0xaf123456, v255     ; encoding: [0xff,0xfe,0xff,0x39,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_pk_fmac_f16 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x78]
 0x01,0x05,0x0a,0x78
+# GFX12: v_pk_fmac_f16 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x78]
 
-# GFX12: v_pk_fmac_f16 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x78]
 0xff,0x05,0x0a,0x78
+# GFX12: v_pk_fmac_f16 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x78]
 
-# GFX12: v_pk_fmac_f16 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x78]
 0x01,0x04,0x0a,0x78
+# GFX12: v_pk_fmac_f16 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x78]
 
-# GFX12: v_pk_fmac_f16 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x78]
 0x69,0x04,0x0a,0x78
+# GFX12: v_pk_fmac_f16 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x78]
 
-# GFX12: v_pk_fmac_f16 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x78]
 0x6a,0x04,0x0a,0x78
+# GFX12: v_pk_fmac_f16 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x78]
 
-# GFX12: v_pk_fmac_f16 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x78]
 0x6b,0x04,0x0a,0x78
+# GFX12: v_pk_fmac_f16 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x78]
 
-# GFX12: v_pk_fmac_f16 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x78]
 0x7b,0x04,0x0a,0x78
+# GFX12: v_pk_fmac_f16 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x78]
 
-# GFX12: v_pk_fmac_f16 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x78]
 0x7d,0x04,0x0a,0x78
+# GFX12: v_pk_fmac_f16 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x78]
 
-# GFX12: v_pk_fmac_f16 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x78]
 0x7e,0x04,0x0a,0x78
+# GFX12: v_pk_fmac_f16 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x78]
 
-# GFX12: v_pk_fmac_f16 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x78]
 0x7f,0x04,0x0a,0x78
+# GFX12: v_pk_fmac_f16 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x78]
 
-# GFX12: v_pk_fmac_f16 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x78]
 0x7c,0x04,0x0a,0x78
+# GFX12: v_pk_fmac_f16 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x78]
 
-# GFX12: v_pk_fmac_f16 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x78]
 0xc1,0x04,0x0a,0x78
+# GFX12: v_pk_fmac_f16 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x78]
 
-# GFX12: v_pk_fmac_f16 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x78]
 0xf0,0x04,0x0a,0x78
+# GFX12: v_pk_fmac_f16 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x78]
 
-# GFX12: v_pk_fmac_f16 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x78]
 0xfd,0x04,0x0a,0x78
+# GFX12: v_pk_fmac_f16 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x78]
 
-# GFX12: v_pk_fmac_f16 v255, 0xfe0b, v255        ; encoding: [0xff,0xfe,0xff,0x79,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xff,0x79,0x0b,0xfe,0x00,0x00
+# GFX12: v_pk_fmac_f16 v255, 0xfe0b, v255        ; encoding: [0xff,0xfe,0xff,0x79,0x0b,0xfe,0x00,0x00]
 
+0x01,0x05,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, v1, v2, vcc_lo ; encoding: [0x01,0x05,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, v1, v2, vcc ; encoding: [0x01,0x05,0x0a,0x42]
-0x01,0x05,0x0a,0x42
 
+0xff,0x05,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, v255, v2, vcc_lo ; encoding: [0xff,0x05,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, v255, v2, vcc ; encoding: [0xff,0x05,0x0a,0x42]
-0xff,0x05,0x0a,0x42
 
+0x01,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, s1, v2, vcc_lo ; encoding: [0x01,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, s1, v2, vcc ; encoding: [0x01,0x04,0x0a,0x42]
-0x01,0x04,0x0a,0x42
 
+0x69,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, s105, v2, vcc_lo ; encoding: [0x69,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, s105, v2, vcc ; encoding: [0x69,0x04,0x0a,0x42]
-0x69,0x04,0x0a,0x42
 
+0x6a,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, vcc_lo, v2, vcc_lo ; encoding: [0x6a,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, vcc_lo, v2, vcc ; encoding: [0x6a,0x04,0x0a,0x42]
-0x6a,0x04,0x0a,0x42
 
+0x6b,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, vcc_hi, v2, vcc_lo ; encoding: [0x6b,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, vcc_hi, v2, vcc ; encoding: [0x6b,0x04,0x0a,0x42]
-0x6b,0x04,0x0a,0x42
 
+0x7b,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, ttmp15, v2, vcc_lo ; encoding: [0x7b,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, ttmp15, v2, vcc ; encoding: [0x7b,0x04,0x0a,0x42]
-0x7b,0x04,0x0a,0x42
 
+0x7d,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, m0, v2, vcc_lo ; encoding: [0x7d,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, m0, v2, vcc ; encoding: [0x7d,0x04,0x0a,0x42]
-0x7d,0x04,0x0a,0x42
 
+0x7e,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, exec_lo, v2, vcc_lo ; encoding: [0x7e,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, exec_lo, v2, vcc ; encoding: [0x7e,0x04,0x0a,0x42]
-0x7e,0x04,0x0a,0x42
 
+0x7f,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, exec_hi, v2, vcc_lo ; encoding: [0x7f,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, exec_hi, v2, vcc ; encoding: [0x7f,0x04,0x0a,0x42]
-0x7f,0x04,0x0a,0x42
 
+0x7c,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, null, v2, vcc_lo ; encoding: [0x7c,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, null, v2, vcc ; encoding: [0x7c,0x04,0x0a,0x42]
-0x7c,0x04,0x0a,0x42
 
+0xc1,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, -1, v2, vcc_lo ; encoding: [0xc1,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, -1, v2, vcc ; encoding: [0xc1,0x04,0x0a,0x42]
-0xc1,0x04,0x0a,0x42
 
+0xf0,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, 0.5, v2, vcc_lo ; encoding: [0xf0,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, 0.5, v2, vcc ; encoding: [0xf0,0x04,0x0a,0x42]
-0xf0,0x04,0x0a,0x42
 
+0xfd,0x04,0x0a,0x42
 # W32: v_sub_co_ci_u32_e32 v5, vcc_lo, src_scc, v2, vcc_lo ; encoding: [0xfd,0x04,0x0a,0x42]
 # W64: v_sub_co_ci_u32_e32 v5, vcc, src_scc, v2, vcc ; encoding: [0xfd,0x04,0x0a,0x42]
-0xfd,0x04,0x0a,0x42
 
+0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf
 # W32: v_sub_co_ci_u32_e32 v255, vcc_lo, 0xaf123456, v255, vcc_lo ; encoding: [0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf]
 # W64: v_sub_co_ci_u32_e32 v255, vcc, 0xaf123456, v255, vcc ; encoding: [0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf]
-0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf
 
-# GFX12: v_sub_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x66]
 0x01,0x05,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, v1.l, v2.l          ; encoding: [0x01,0x05,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x66]
 
-# GFX12: v_sub_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x66]
 0x7f,0x05,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, v127.l, v2.l        ; encoding: [0x7f,0x05,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x66]
 
-# GFX12: v_sub_f16_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x66]
 0x01,0x04,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, s1, v2.l            ; encoding: [0x01,0x04,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x66]
 
-# GFX12: v_sub_f16_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x66]
 0x69,0x04,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, s105, v2.l          ; encoding: [0x69,0x04,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x66]
 
-# GFX12: v_sub_f16_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x66]
 0x6a,0x04,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, vcc_lo, v2.l        ; encoding: [0x6a,0x04,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x66]
 
-# GFX12: v_sub_f16_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x66]
 0x6b,0x04,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, vcc_hi, v2.l        ; encoding: [0x6b,0x04,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x66]
 
-# GFX12: v_sub_f16_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x66]
 0x7b,0x04,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, ttmp15, v2.l        ; encoding: [0x7b,0x04,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x66]
 
-# GFX12: v_sub_f16_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x66]
 0x7d,0x04,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, m0, v2.l            ; encoding: [0x7d,0x04,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x66]
 
-# GFX12: v_sub_f16_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x66]
 0x7e,0x04,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, exec_lo, v2.l       ; encoding: [0x7e,0x04,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x66]
 
-# GFX12: v_sub_f16_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x66]
 0x7f,0x04,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, exec_hi, v2.l       ; encoding: [0x7f,0x04,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x66]
 
-# GFX12: v_sub_f16_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x66]
 0x7c,0x04,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, null, v2.l          ; encoding: [0x7c,0x04,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x66]
 
-# GFX12: v_sub_f16_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x66]
 0xc1,0x04,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, -1, v2.l            ; encoding: [0xc1,0x04,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x66]
 
-# GFX12: v_sub_f16_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x66]
 0xf0,0x04,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, 0.5, v2.l           ; encoding: [0xf0,0x04,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x66]
 
-# GFX12: v_sub_f16_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x66]
 0xfd,0x04,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, src_scc, v2.l       ; encoding: [0xfd,0x04,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x66]
 
-# GFX12: v_sub_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_sub_f16_e32 v127.l, 0xfe0b, v127.l    ; encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_sub_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_sub_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x08]
 0x01,0x05,0x0a,0x08
+# GFX12: v_sub_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x08]
 
-# GFX12: v_sub_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x08]
 0xff,0x05,0x0a,0x08
+# GFX12: v_sub_f32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x08]
 
-# GFX12: v_sub_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x08]
 0x01,0x04,0x0a,0x08
+# GFX12: v_sub_f32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x08]
 
-# GFX12: v_sub_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x08]
 0x69,0x04,0x0a,0x08
+# GFX12: v_sub_f32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x08]
 
-# GFX12: v_sub_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x08]
 0x6a,0x04,0x0a,0x08
+# GFX12: v_sub_f32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x08]
 
-# GFX12: v_sub_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x08]
 0x6b,0x04,0x0a,0x08
+# GFX12: v_sub_f32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x08]
 
-# GFX12: v_sub_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x08]
 0x7b,0x04,0x0a,0x08
+# GFX12: v_sub_f32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x08]
 
-# GFX12: v_sub_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x08]
 0x7d,0x04,0x0a,0x08
+# GFX12: v_sub_f32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x08]
 
-# GFX12: v_sub_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x08]
 0x7e,0x04,0x0a,0x08
+# GFX12: v_sub_f32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x08]
 
-# GFX12: v_sub_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x08]
 0x7f,0x04,0x0a,0x08
+# GFX12: v_sub_f32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x08]
 
-# GFX12: v_sub_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x08]
 0x7c,0x04,0x0a,0x08
+# GFX12: v_sub_f32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x08]
 
-# GFX12: v_sub_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x08]
 0xc1,0x04,0x0a,0x08
+# GFX12: v_sub_f32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x08]
 
-# GFX12: v_sub_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x08]
 0xf0,0x04,0x0a,0x08
+# GFX12: v_sub_f32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x08]
 
-# GFX12: v_sub_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x08]
 0xfd,0x04,0x0a,0x08
+# GFX12: v_sub_f32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x08]
 
-# GFX12: v_sub_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x09,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x09,0x56,0x34,0x12,0xaf
+# GFX12: v_sub_f32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x09,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_sub_nc_u32_e32 v5, v1, v2             ; encoding: [0x01,0x05,0x0a,0x4c]
 0x01,0x05,0x0a,0x4c
+# GFX12: v_sub_nc_u32_e32 v5, v1, v2             ; encoding: [0x01,0x05,0x0a,0x4c]
 
-# GFX12: v_sub_nc_u32_e32 v5, v255, v2           ; encoding: [0xff,0x05,0x0a,0x4c]
 0xff,0x05,0x0a,0x4c
+# GFX12: v_sub_nc_u32_e32 v5, v255, v2           ; encoding: [0xff,0x05,0x0a,0x4c]
 
-# GFX12: v_sub_nc_u32_e32 v5, s1, v2             ; encoding: [0x01,0x04,0x0a,0x4c]
 0x01,0x04,0x0a,0x4c
+# GFX12: v_sub_nc_u32_e32 v5, s1, v2             ; encoding: [0x01,0x04,0x0a,0x4c]
 
-# GFX12: v_sub_nc_u32_e32 v5, s105, v2           ; encoding: [0x69,0x04,0x0a,0x4c]
 0x69,0x04,0x0a,0x4c
+# GFX12: v_sub_nc_u32_e32 v5, s105, v2           ; encoding: [0x69,0x04,0x0a,0x4c]
 
-# GFX12: v_sub_nc_u32_e32 v5, vcc_lo, v2         ; encoding: [0x6a,0x04,0x0a,0x4c]
 0x6a,0x04,0x0a,0x4c
+# GFX12: v_sub_nc_u32_e32 v5, vcc_lo, v2         ; encoding: [0x6a,0x04,0x0a,0x4c]
 
-# GFX12: v_sub_nc_u32_e32 v5, vcc_hi, v2         ; encoding: [0x6b,0x04,0x0a,0x4c]
 0x6b,0x04,0x0a,0x4c
+# GFX12: v_sub_nc_u32_e32 v5, vcc_hi, v2         ; encoding: [0x6b,0x04,0x0a,0x4c]
 
-# GFX12: v_sub_nc_u32_e32 v5, ttmp15, v2         ; encoding: [0x7b,0x04,0x0a,0x4c]
 0x7b,0x04,0x0a,0x4c
+# GFX12: v_sub_nc_u32_e32 v5, ttmp15, v2         ; encoding: [0x7b,0x04,0x0a,0x4c]
 
-# GFX12: v_sub_nc_u32_e32 v5, m0, v2             ; encoding: [0x7d,0x04,0x0a,0x4c]
 0x7d,0x04,0x0a,0x4c
+# GFX12: v_sub_nc_u32_e32 v5, m0, v2             ; encoding: [0x7d,0x04,0x0a,0x4c]
 
-# GFX12: v_sub_nc_u32_e32 v5, exec_lo, v2        ; encoding: [0x7e,0x04,0x0a,0x4c]
 0x7e,0x04,0x0a,0x4c
+# GFX12: v_sub_nc_u32_e32 v5, exec_lo, v2        ; encoding: [0x7e,0x04,0x0a,0x4c]
 
-# GFX12: v_sub_nc_u32_e32 v5, exec_hi, v2        ; encoding: [0x7f,0x04,0x0a,0x4c]
 0x7f,0x04,0x0a,0x4c
+# GFX12: v_sub_nc_u32_e32 v5, exec_hi, v2        ; encoding: [0x7f,0x04,0x0a,0x4c]
 
-# GFX12: v_sub_nc_u32_e32 v5, null, v2           ; encoding: [0x7c,0x04,0x0a,0x4c]
 0x7c,0x04,0x0a,0x4c
+# GFX12: v_sub_nc_u32_e32 v5, null, v2           ; encoding: [0x7c,0x04,0x0a,0x4c]
 
-# GFX12: v_sub_nc_u32_e32 v5, -1, v2             ; encoding: [0xc1,0x04,0x0a,0x4c]
 0xc1,0x04,0x0a,0x4c
+# GFX12: v_sub_nc_u32_e32 v5, -1, v2             ; encoding: [0xc1,0x04,0x0a,0x4c]
 
-# GFX12: v_sub_nc_u32_e32 v5, 0.5, v2            ; encoding: [0xf0,0x04,0x0a,0x4c]
 0xf0,0x04,0x0a,0x4c
+# GFX12: v_sub_nc_u32_e32 v5, 0.5, v2            ; encoding: [0xf0,0x04,0x0a,0x4c]
 
-# GFX12: v_sub_nc_u32_e32 v5, src_scc, v2        ; encoding: [0xfd,0x04,0x0a,0x4c]
 0xfd,0x04,0x0a,0x4c
+# GFX12: v_sub_nc_u32_e32 v5, src_scc, v2        ; encoding: [0xfd,0x04,0x0a,0x4c]
 
-# GFX12: v_sub_nc_u32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x4d,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x4d,0x56,0x34,0x12,0xaf
+# GFX12: v_sub_nc_u32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x4d,0x56,0x34,0x12,0xaf]
 
+0x01,0x05,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, v1, v2, vcc_lo ; encoding: [0x01,0x05,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, v1, v2, vcc ; encoding: [0x01,0x05,0x0a,0x44]
-0x01,0x05,0x0a,0x44
 
+0xff,0x05,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, v255, v2, vcc_lo ; encoding: [0xff,0x05,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, v255, v2, vcc ; encoding: [0xff,0x05,0x0a,0x44]
-0xff,0x05,0x0a,0x44
 
+0x01,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, s1, v2, vcc_lo ; encoding: [0x01,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, s1, v2, vcc ; encoding: [0x01,0x04,0x0a,0x44]
-0x01,0x04,0x0a,0x44
 
+0x69,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, s105, v2, vcc_lo ; encoding: [0x69,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, s105, v2, vcc ; encoding: [0x69,0x04,0x0a,0x44]
-0x69,0x04,0x0a,0x44
 
+0x6a,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, vcc_lo, v2, vcc_lo ; encoding: [0x6a,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, vcc_lo, v2, vcc ; encoding: [0x6a,0x04,0x0a,0x44]
-0x6a,0x04,0x0a,0x44
 
+0x6b,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, vcc_hi, v2, vcc_lo ; encoding: [0x6b,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, vcc_hi, v2, vcc ; encoding: [0x6b,0x04,0x0a,0x44]
-0x6b,0x04,0x0a,0x44
 
+0x7b,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, ttmp15, v2, vcc_lo ; encoding: [0x7b,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, ttmp15, v2, vcc ; encoding: [0x7b,0x04,0x0a,0x44]
-0x7b,0x04,0x0a,0x44
 
+0x7d,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, m0, v2, vcc_lo ; encoding: [0x7d,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, m0, v2, vcc ; encoding: [0x7d,0x04,0x0a,0x44]
-0x7d,0x04,0x0a,0x44
 
+0x7e,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, exec_lo, v2, vcc_lo ; encoding: [0x7e,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, exec_lo, v2, vcc ; encoding: [0x7e,0x04,0x0a,0x44]
-0x7e,0x04,0x0a,0x44
 
+0x7f,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, exec_hi, v2, vcc_lo ; encoding: [0x7f,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, exec_hi, v2, vcc ; encoding: [0x7f,0x04,0x0a,0x44]
-0x7f,0x04,0x0a,0x44
 
+0x7c,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, null, v2, vcc_lo ; encoding: [0x7c,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, null, v2, vcc ; encoding: [0x7c,0x04,0x0a,0x44]
-0x7c,0x04,0x0a,0x44
 
+0xc1,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, -1, v2, vcc_lo ; encoding: [0xc1,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, -1, v2, vcc ; encoding: [0xc1,0x04,0x0a,0x44]
-0xc1,0x04,0x0a,0x44
 
+0xf0,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, 0.5, v2, vcc_lo ; encoding: [0xf0,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, 0.5, v2, vcc ; encoding: [0xf0,0x04,0x0a,0x44]
-0xf0,0x04,0x0a,0x44
 
+0xfd,0x04,0x0a,0x44
 # W32: v_subrev_co_ci_u32_e32 v5, vcc_lo, src_scc, v2, vcc_lo ; encoding: [0xfd,0x04,0x0a,0x44]
 # W64: v_subrev_co_ci_u32_e32 v5, vcc, src_scc, v2, vcc ; encoding: [0xfd,0x04,0x0a,0x44]
-0xfd,0x04,0x0a,0x44
 
+0xff,0xfe,0xff,0x45,0x56,0x34,0x12,0xaf
 # W32: v_subrev_co_ci_u32_e32 v255, vcc_lo, 0xaf123456, v255, vcc_lo ; encoding: [0xff,0xfe,0xff,0x45,0x56,0x34,0x12,0xaf]
 # W64: v_subrev_co_ci_u32_e32 v255, vcc, 0xaf123456, v255, vcc ; encoding: [0xff,0xfe,0xff,0x45,0x56,0x34,0x12,0xaf]
-0xff,0xfe,0xff,0x45,0x56,0x34,0x12,0xaf
 
-# GFX12: v_subrev_f16_e32 v5, v1, v2             ; encoding: [0x01,0x05,0x0a,0x68]
 0x01,0x05,0x0a,0x68
+# GFX12-REAL16: v_subrev_f16_e32 v5.l, v1.l, v2.l       ; encoding: [0x01,0x05,0x0a,0x68]
+# GFX12-FAKE16: v_subrev_f16_e32 v5, v1, v2             ; encoding: [0x01,0x05,0x0a,0x68]
 
-# GFX12: v_subrev_f16_e32 v5, v127, v2           ; encoding: [0x7f,0x05,0x0a,0x68]
 0x7f,0x05,0x0a,0x68
+# GFX12-REAL16: v_subrev_f16_e32 v5.l, v127.l, v2.l     ; encoding: [0x7f,0x05,0x0a,0x68]
+# GFX12-FAKE16: v_subrev_f16_e32 v5, v127, v2           ; encoding: [0x7f,0x05,0x0a,0x68]
 
-# GFX12: v_subrev_f16_e32 v5, s1, v2             ; encoding: [0x01,0x04,0x0a,0x68]
 0x01,0x04,0x0a,0x68
+# GFX12-REAL16: v_subrev_f16_e32 v5.l, s1, v2.l         ; encoding: [0x01,0x04,0x0a,0x68]
+# GFX12-FAKE16: v_subrev_f16_e32 v5, s1, v2             ; encoding: [0x01,0x04,0x0a,0x68]
 
-# GFX12: v_subrev_f16_e32 v5, s105, v2           ; encoding: [0x69,0x04,0x0a,0x68]
 0x69,0x04,0x0a,0x68
+# GFX12-REAL16: v_subrev_f16_e32 v5.l, s105, v2.l       ; encoding: [0x69,0x04,0x0a,0x68]
+# GFX12-FAKE16: v_subrev_f16_e32 v5, s105, v2           ; encoding: [0x69,0x04,0x0a,0x68]
 
-# GFX12: v_subrev_f16_e32 v5, vcc_lo, v2         ; encoding: [0x6a,0x04,0x0a,0x68]
 0x6a,0x04,0x0a,0x68
+# GFX12-REAL16: v_subrev_f16_e32 v5.l, vcc_lo, v2.l     ; encoding: [0x6a,0x04,0x0a,0x68]
+# GFX12-FAKE16: v_subrev_f16_e32 v5, vcc_lo, v2         ; encoding: [0x6a,0x04,0x0a,0x68]
 
-# GFX12: v_subrev_f16_e32 v5, vcc_hi, v2         ; encoding: [0x6b,0x04,0x0a,0x68]
 0x6b,0x04,0x0a,0x68
+# GFX12-REAL16: v_subrev_f16_e32 v5.l, vcc_hi, v2.l     ; encoding: [0x6b,0x04,0x0a,0x68]
+# GFX12-FAKE16: v_subrev_f16_e32 v5, vcc_hi, v2         ; encoding: [0x6b,0x04,0x0a,0x68]
 
-# GFX12: v_subrev_f16_e32 v5, ttmp15, v2         ; encoding: [0x7b,0x04,0x0a,0x68]
 0x7b,0x04,0x0a,0x68
+# GFX12-REAL16: v_subrev_f16_e32 v5.l, ttmp15, v2.l     ; encoding: [0x7b,0x04,0x0a,0x68]
+# GFX12-FAKE16: v_subrev_f16_e32 v5, ttmp15, v2         ; encoding: [0x7b,0x04,0x0a,0x68]
 
-# GFX12: v_subrev_f16_e32 v5, m0, v2             ; encoding: [0x7d,0x04,0x0a,0x68]
 0x7d,0x04,0x0a,0x68
+# GFX12-REAL16: v_subrev_f16_e32 v5.l, m0, v2.l         ; encoding: [0x7d,0x04,0x0a,0x68]
+# GFX12-FAKE16: v_subrev_f16_e32 v5, m0, v2             ; encoding: [0x7d,0x04,0x0a,0x68]
 
-# GFX12: v_subrev_f16_e32 v5, exec_lo, v2        ; encoding: [0x7e,0x04,0x0a,0x68]
 0x7e,0x04,0x0a,0x68
+# GFX12-REAL16: v_subrev_f16_e32 v5.l, exec_lo, v2.l    ; encoding: [0x7e,0x04,0x0a,0x68]
+# GFX12-FAKE16: v_subrev_f16_e32 v5, exec_lo, v2        ; encoding: [0x7e,0x04,0x0a,0x68]
 
-# GFX12: v_subrev_f16_e32 v5, exec_hi, v2        ; encoding: [0x7f,0x04,0x0a,0x68]
 0x7f,0x04,0x0a,0x68
+# GFX12-REAL16: v_subrev_f16_e32 v5.l, exec_hi, v2.l    ; encoding: [0x7f,0x04,0x0a,0x68]
+# GFX12-FAKE16: v_subrev_f16_e32 v5, exec_hi, v2        ; encoding: [0x7f,0x04,0x0a,0x68]
 
-# GFX12: v_subrev_f16_e32 v5, null, v2           ; encoding: [0x7c,0x04,0x0a,0x68]
 0x7c,0x04,0x0a,0x68
+# GFX12-REAL16: v_subrev_f16_e32 v5.l, null, v2.l       ; encoding: [0x7c,0x04,0x0a,0x68]
+# GFX12-FAKE16: v_subrev_f16_e32 v5, null, v2           ; encoding: [0x7c,0x04,0x0a,0x68]
 
-# GFX12: v_subrev_f16_e32 v5, -1, v2             ; encoding: [0xc1,0x04,0x0a,0x68]
 0xc1,0x04,0x0a,0x68
+# GFX12-REAL16: v_subrev_f16_e32 v5.l, -1, v2.l         ; encoding: [0xc1,0x04,0x0a,0x68]
+# GFX12-FAKE16: v_subrev_f16_e32 v5, -1, v2             ; encoding: [0xc1,0x04,0x0a,0x68]
 
-# GFX12: v_subrev_f16_e32 v5, 0.5, v2            ; encoding: [0xf0,0x04,0x0a,0x68]
 0xf0,0x04,0x0a,0x68
+# GFX12-REAL16: v_subrev_f16_e32 v5.l, 0.5, v2.l        ; encoding: [0xf0,0x04,0x0a,0x68]
+# GFX12-FAKE16: v_subrev_f16_e32 v5, 0.5, v2            ; encoding: [0xf0,0x04,0x0a,0x68]
 
-# GFX12: v_subrev_f16_e32 v5, src_scc, v2        ; encoding: [0xfd,0x04,0x0a,0x68]
 0xfd,0x04,0x0a,0x68
+# GFX12-REAL16: v_subrev_f16_e32 v5.l, src_scc, v2.l    ; encoding: [0xfd,0x04,0x0a,0x68]
+# GFX12-FAKE16: v_subrev_f16_e32 v5, src_scc, v2        ; encoding: [0xfd,0x04,0x0a,0x68]
 
-# GFX12: v_subrev_f16_e32 v127, 0xfe0b, v127     ; encoding: [0xff,0xfe,0xfe,0x68,0x0b,0xfe,0x00,0x00]
 0xff,0xfe,0xfe,0x68,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_subrev_f16_e32 v127.l, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x68,0x0b,0xfe,0x00,0x00]
+# GFX12-FAKE16: v_subrev_f16_e32 v127, 0xfe0b, v127     ; encoding: [0xff,0xfe,0xfe,0x68,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_subrev_f32_e32 v5, v1, v2             ; encoding: [0x01,0x05,0x0a,0x0a]
 0x01,0x05,0x0a,0x0a
+# GFX12: v_subrev_f32_e32 v5, v1, v2             ; encoding: [0x01,0x05,0x0a,0x0a]
 
-# GFX12: v_subrev_f32_e32 v5, v255, v2           ; encoding: [0xff,0x05,0x0a,0x0a]
 0xff,0x05,0x0a,0x0a
+# GFX12: v_subrev_f32_e32 v5, v255, v2           ; encoding: [0xff,0x05,0x0a,0x0a]
 
-# GFX12: v_subrev_f32_e32 v5, s1, v2             ; encoding: [0x01,0x04,0x0a,0x0a]
 0x01,0x04,0x0a,0x0a
+# GFX12: v_subrev_f32_e32 v5, s1, v2             ; encoding: [0x01,0x04,0x0a,0x0a]
 
-# GFX12: v_subrev_f32_e32 v5, s105, v2           ; encoding: [0x69,0x04,0x0a,0x0a]
 0x69,0x04,0x0a,0x0a
+# GFX12: v_subrev_f32_e32 v5, s105, v2           ; encoding: [0x69,0x04,0x0a,0x0a]
 
-# GFX12: v_subrev_f32_e32 v5, vcc_lo, v2         ; encoding: [0x6a,0x04,0x0a,0x0a]
 0x6a,0x04,0x0a,0x0a
+# GFX12: v_subrev_f32_e32 v5, vcc_lo, v2         ; encoding: [0x6a,0x04,0x0a,0x0a]
 
-# GFX12: v_subrev_f32_e32 v5, vcc_hi, v2         ; encoding: [0x6b,0x04,0x0a,0x0a]
 0x6b,0x04,0x0a,0x0a
+# GFX12: v_subrev_f32_e32 v5, vcc_hi, v2         ; encoding: [0x6b,0x04,0x0a,0x0a]
 
-# GFX12: v_subrev_f32_e32 v5, ttmp15, v2         ; encoding: [0x7b,0x04,0x0a,0x0a]
 0x7b,0x04,0x0a,0x0a
+# GFX12: v_subrev_f32_e32 v5, ttmp15, v2         ; encoding: [0x7b,0x04,0x0a,0x0a]
 
-# GFX12: v_subrev_f32_e32 v5, m0, v2             ; encoding: [0x7d,0x04,0x0a,0x0a]
 0x7d,0x04,0x0a,0x0a
+# GFX12: v_subrev_f32_e32 v5, m0, v2             ; encoding: [0x7d,0x04,0x0a,0x0a]
 
-# GFX12: v_subrev_f32_e32 v5, exec_lo, v2        ; encoding: [0x7e,0x04,0x0a,0x0a]
 0x7e,0x04,0x0a,0x0a
+# GFX12: v_subrev_f32_e32 v5, exec_lo, v2        ; encoding: [0x7e,0x04,0x0a,0x0a]
 
-# GFX12: v_subrev_f32_e32 v5, exec_hi, v2        ; encoding: [0x7f,0x04,0x0a,0x0a]
 0x7f,0x04,0x0a,0x0a
+# GFX12: v_subrev_f32_e32 v5, exec_hi, v2        ; encoding: [0x7f,0x04,0x0a,0x0a]
 
-# GFX12: v_subrev_f32_e32 v5, null, v2           ; encoding: [0x7c,0x04,0x0a,0x0a]
 0x7c,0x04,0x0a,0x0a
+# GFX12: v_subrev_f32_e32 v5, null, v2           ; encoding: [0x7c,0x04,0x0a,0x0a]
 
-# GFX12: v_subrev_f32_e32 v5, -1, v2             ; encoding: [0xc1,0x04,0x0a,0x0a]
 0xc1,0x04,0x0a,0x0a
+# GFX12: v_subrev_f32_e32 v5, -1, v2             ; encoding: [0xc1,0x04,0x0a,0x0a]
 
-# GFX12: v_subrev_f32_e32 v5, 0.5, v2            ; encoding: [0xf0,0x04,0x0a,0x0a]
 0xf0,0x04,0x0a,0x0a
+# GFX12: v_subrev_f32_e32 v5, 0.5, v2            ; encoding: [0xf0,0x04,0x0a,0x0a]
 
-# GFX12: v_subrev_f32_e32 v5, src_scc, v2        ; encoding: [0xfd,0x04,0x0a,0x0a]
 0xfd,0x04,0x0a,0x0a
+# GFX12: v_subrev_f32_e32 v5, src_scc, v2        ; encoding: [0xfd,0x04,0x0a,0x0a]
 
-# GFX12: v_subrev_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x0b,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x0b,0x56,0x34,0x12,0xaf
+# GFX12: v_subrev_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x0b,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_subrev_nc_u32_e32 v5, v1, v2          ; encoding: [0x01,0x05,0x0a,0x4e]
 0x01,0x05,0x0a,0x4e
+# GFX12: v_subrev_nc_u32_e32 v5, v1, v2          ; encoding: [0x01,0x05,0x0a,0x4e]
 
-# GFX12: v_subrev_nc_u32_e32 v5, v255, v2        ; encoding: [0xff,0x05,0x0a,0x4e]
 0xff,0x05,0x0a,0x4e
+# GFX12: v_subrev_nc_u32_e32 v5, v255, v2        ; encoding: [0xff,0x05,0x0a,0x4e]
 
-# GFX12: v_subrev_nc_u32_e32 v5, s1, v2          ; encoding: [0x01,0x04,0x0a,0x4e]
 0x01,0x04,0x0a,0x4e
+# GFX12: v_subrev_nc_u32_e32 v5, s1, v2          ; encoding: [0x01,0x04,0x0a,0x4e]
 
-# GFX12: v_subrev_nc_u32_e32 v5, s105, v2        ; encoding: [0x69,0x04,0x0a,0x4e]
 0x69,0x04,0x0a,0x4e
+# GFX12: v_subrev_nc_u32_e32 v5, s105, v2        ; encoding: [0x69,0x04,0x0a,0x4e]
 
-# GFX12: v_subrev_nc_u32_e32 v5, vcc_lo, v2      ; encoding: [0x6a,0x04,0x0a,0x4e]
 0x6a,0x04,0x0a,0x4e
+# GFX12: v_subrev_nc_u32_e32 v5, vcc_lo, v2      ; encoding: [0x6a,0x04,0x0a,0x4e]
 
-# GFX12: v_subrev_nc_u32_e32 v5, vcc_hi, v2      ; encoding: [0x6b,0x04,0x0a,0x4e]
 0x6b,0x04,0x0a,0x4e
+# GFX12: v_subrev_nc_u32_e32 v5, vcc_hi, v2      ; encoding: [0x6b,0x04,0x0a,0x4e]
 
-# GFX12: v_subrev_nc_u32_e32 v5, ttmp15, v2      ; encoding: [0x7b,0x04,0x0a,0x4e]
 0x7b,0x04,0x0a,0x4e
+# GFX12: v_subrev_nc_u32_e32 v5, ttmp15, v2      ; encoding: [0x7b,0x04,0x0a,0x4e]
 
-# GFX12: v_subrev_nc_u32_e32 v5, m0, v2          ; encoding: [0x7d,0x04,0x0a,0x4e]
 0x7d,0x04,0x0a,0x4e
+# GFX12: v_subrev_nc_u32_e32 v5, m0, v2          ; encoding: [0x7d,0x04,0x0a,0x4e]
 
-# GFX12: v_subrev_nc_u32_e32 v5, exec_lo, v2     ; encoding: [0x7e,0x04,0x0a,0x4e]
 0x7e,0x04,0x0a,0x4e
+# GFX12: v_subrev_nc_u32_e32 v5, exec_lo, v2     ; encoding: [0x7e,0x04,0x0a,0x4e]
 
-# GFX12: v_subrev_nc_u32_e32 v5, exec_hi, v2     ; encoding: [0x7f,0x04,0x0a,0x4e]
 0x7f,0x04,0x0a,0x4e
+# GFX12: v_subrev_nc_u32_e32 v5, exec_hi, v2     ; encoding: [0x7f,0x04,0x0a,0x4e]
 
-# GFX12: v_subrev_nc_u32_e32 v5, null, v2        ; encoding: [0x7c,0x04,0x0a,0x4e]
 0x7c,0x04,0x0a,0x4e
+# GFX12: v_subrev_nc_u32_e32 v5, null, v2        ; encoding: [0x7c,0x04,0x0a,0x4e]
 
-# GFX12: v_subrev_nc_u32_e32 v5, -1, v2          ; encoding: [0xc1,0x04,0x0a,0x4e]
 0xc1,0x04,0x0a,0x4e
+# GFX12: v_subrev_nc_u32_e32 v5, -1, v2          ; encoding: [0xc1,0x04,0x0a,0x4e]
 
-# GFX12: v_subrev_nc_u32_e32 v5, 0.5, v2         ; encoding: [0xf0,0x04,0x0a,0x4e]
 0xf0,0x04,0x0a,0x4e
+# GFX12: v_subrev_nc_u32_e32 v5, 0.5, v2         ; encoding: [0xf0,0x04,0x0a,0x4e]
 
-# GFX12: v_subrev_nc_u32_e32 v5, src_scc, v2     ; encoding: [0xfd,0x04,0x0a,0x4e]
 0xfd,0x04,0x0a,0x4e
+# GFX12: v_subrev_nc_u32_e32 v5, src_scc, v2     ; encoding: [0xfd,0x04,0x0a,0x4e]
 
-# GFX12: v_subrev_nc_u32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x4f,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x4f,0x56,0x34,0x12,0xaf
+# GFX12: v_subrev_nc_u32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x4f,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_xnor_b32_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x3c]
 0x01,0x05,0x0a,0x3c
+# GFX12: v_xnor_b32_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x3c]
 
-# GFX12: v_xnor_b32_e32 v5, v255, v2             ; encoding: [0xff,0x05,0x0a,0x3c]
 0xff,0x05,0x0a,0x3c
+# GFX12: v_xnor_b32_e32 v5, v255, v2             ; encoding: [0xff,0x05,0x0a,0x3c]
 
-# GFX12: v_xnor_b32_e32 v5, s1, v2               ; encoding: [0x01,0x04,0x0a,0x3c]
 0x01,0x04,0x0a,0x3c
+# GFX12: v_xnor_b32_e32 v5, s1, v2               ; encoding: [0x01,0x04,0x0a,0x3c]
 
-# GFX12: v_xnor_b32_e32 v5, s105, v2             ; encoding: [0x69,0x04,0x0a,0x3c]
 0x69,0x04,0x0a,0x3c
+# GFX12: v_xnor_b32_e32 v5, s105, v2             ; encoding: [0x69,0x04,0x0a,0x3c]
 
-# GFX12: v_xnor_b32_e32 v5, vcc_lo, v2           ; encoding: [0x6a,0x04,0x0a,0x3c]
 0x6a,0x04,0x0a,0x3c
+# GFX12: v_xnor_b32_e32 v5, vcc_lo, v2           ; encoding: [0x6a,0x04,0x0a,0x3c]
 
-# GFX12: v_xnor_b32_e32 v5, vcc_hi, v2           ; encoding: [0x6b,0x04,0x0a,0x3c]
 0x6b,0x04,0x0a,0x3c
+# GFX12: v_xnor_b32_e32 v5, vcc_hi, v2           ; encoding: [0x6b,0x04,0x0a,0x3c]
 
-# GFX12: v_xnor_b32_e32 v5, ttmp15, v2           ; encoding: [0x7b,0x04,0x0a,0x3c]
 0x7b,0x04,0x0a,0x3c
+# GFX12: v_xnor_b32_e32 v5, ttmp15, v2           ; encoding: [0x7b,0x04,0x0a,0x3c]
 
-# GFX12: v_xnor_b32_e32 v5, m0, v2               ; encoding: [0x7d,0x04,0x0a,0x3c]
 0x7d,0x04,0x0a,0x3c
+# GFX12: v_xnor_b32_e32 v5, m0, v2               ; encoding: [0x7d,0x04,0x0a,0x3c]
 
-# GFX12: v_xnor_b32_e32 v5, exec_lo, v2          ; encoding: [0x7e,0x04,0x0a,0x3c]
 0x7e,0x04,0x0a,0x3c
+# GFX12: v_xnor_b32_e32 v5, exec_lo, v2          ; encoding: [0x7e,0x04,0x0a,0x3c]
 
-# GFX12: v_xnor_b32_e32 v5, exec_hi, v2          ; encoding: [0x7f,0x04,0x0a,0x3c]
 0x7f,0x04,0x0a,0x3c
+# GFX12: v_xnor_b32_e32 v5, exec_hi, v2          ; encoding: [0x7f,0x04,0x0a,0x3c]
 
-# GFX12: v_xnor_b32_e32 v5, null, v2             ; encoding: [0x7c,0x04,0x0a,0x3c]
 0x7c,0x04,0x0a,0x3c
+# GFX12: v_xnor_b32_e32 v5, null, v2             ; encoding: [0x7c,0x04,0x0a,0x3c]
 
-# GFX12: v_xnor_b32_e32 v5, -1, v2               ; encoding: [0xc1,0x04,0x0a,0x3c]
 0xc1,0x04,0x0a,0x3c
+# GFX12: v_xnor_b32_e32 v5, -1, v2               ; encoding: [0xc1,0x04,0x0a,0x3c]
 
-# GFX12: v_xnor_b32_e32 v5, 0.5, v2              ; encoding: [0xf0,0x04,0x0a,0x3c]
 0xf0,0x04,0x0a,0x3c
+# GFX12: v_xnor_b32_e32 v5, 0.5, v2              ; encoding: [0xf0,0x04,0x0a,0x3c]
 
-# GFX12: v_xnor_b32_e32 v5, src_scc, v2          ; encoding: [0xfd,0x04,0x0a,0x3c]
 0xfd,0x04,0x0a,0x3c
+# GFX12: v_xnor_b32_e32 v5, src_scc, v2          ; encoding: [0xfd,0x04,0x0a,0x3c]
 
-# GFX12: v_xnor_b32_e32 v255, 0xaf123456, v255   ; encoding: [0xff,0xfe,0xff,0x3d,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x3d,0x56,0x34,0x12,0xaf
+# GFX12: v_xnor_b32_e32 v255, 0xaf123456, v255   ; encoding: [0xff,0xfe,0xff,0x3d,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_xor_b32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x3a]
 0x01,0x05,0x0a,0x3a
+# GFX12: v_xor_b32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x3a]
 
-# GFX12: v_xor_b32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x3a]
 0xff,0x05,0x0a,0x3a
+# GFX12: v_xor_b32_e32 v5, v255, v2              ; encoding: [0xff,0x05,0x0a,0x3a]
 
-# GFX12: v_xor_b32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x3a]
 0x01,0x04,0x0a,0x3a
+# GFX12: v_xor_b32_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x3a]
 
-# GFX12: v_xor_b32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x3a]
 0x69,0x04,0x0a,0x3a
+# GFX12: v_xor_b32_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x3a]
 
-# GFX12: v_xor_b32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x3a]
 0x6a,0x04,0x0a,0x3a
+# GFX12: v_xor_b32_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x3a]
 
-# GFX12: v_xor_b32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x3a]
 0x6b,0x04,0x0a,0x3a
+# GFX12: v_xor_b32_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x3a]
 
-# GFX12: v_xor_b32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x3a]
 0x7b,0x04,0x0a,0x3a
+# GFX12: v_xor_b32_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x3a]
 
-# GFX12: v_xor_b32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x3a]
 0x7d,0x04,0x0a,0x3a
+# GFX12: v_xor_b32_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x3a]
 
-# GFX12: v_xor_b32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x3a]
 0x7e,0x04,0x0a,0x3a
+# GFX12: v_xor_b32_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x3a]
 
-# GFX12: v_xor_b32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x3a]
 0x7f,0x04,0x0a,0x3a
+# GFX12: v_xor_b32_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x3a]
 
-# GFX12: v_xor_b32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x3a]
 0x7c,0x04,0x0a,0x3a
+# GFX12: v_xor_b32_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x3a]
 
-# GFX12: v_xor_b32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x3a]
 0xc1,0x04,0x0a,0x3a
+# GFX12: v_xor_b32_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x3a]
 
-# GFX12: v_xor_b32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x3a]
 0xf0,0x04,0x0a,0x3a
+# GFX12: v_xor_b32_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x3a]
 
-# GFX12: v_xor_b32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x3a]
 0xfd,0x04,0x0a,0x3a
+# GFX12: v_xor_b32_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x3a]
 
-# GFX12: v_xor_b32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x3b,0x56,0x34,0x12,0xaf]
 0xff,0xfe,0xff,0x3b,0x56,0x34,0x12,0xaf
+# GFX12: v_xor_b32_e32 v255, 0xaf123456, v255    ; encoding: [0xff,0xfe,0xff,0x3b,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp16.txt
index 05c8dff02a40..551fb0d31118 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp16.txt
@@ -1,1696 +1,1797 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,GFX12-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,GFX12-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,GFX12-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,GFX12-FAKE16 %s
 
+0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0xe4,0x00,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0xe4,0x00,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0xe4,0x00,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0xe4,0x00,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x40,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x40,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x40,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x40,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x41,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x41,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x41,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x41,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x01,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x01,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x01,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x01,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x0f,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x0f,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x0f,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x0f,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x11,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x11,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x11,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x11,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x1f,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1f,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1f,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x1f,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x21,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x21,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x21,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x21,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x2f,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x2f,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x2f,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x2f,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x50,0x01,0xff
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x50,0x01,0xff]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x50,0x01,0xff]
-0xfa,0x04,0x0a,0x40,0x01,0x50,0x01,0xff
 
+0xfa,0x04,0x0a,0x40,0x01,0x5f,0x01,0x01
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x5f,0x01,0x01]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x5f,0x01,0x01]
-0xfa,0x04,0x0a,0x40,0x01,0x5f,0x01,0x01
 
+0xfa,0x04,0x0a,0x40,0x01,0x60,0x01,0x13
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x60,0x01,0x13]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x60,0x01,0x13]
-0xfa,0x04,0x0a,0x40,0x01,0x60,0x01,0x13
 
+0xfa,0xfe,0xff,0x41,0xff,0x6f,0x0d,0x30
 # W32: v_add_co_ci_u32_dpp v255, vcc_lo, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x41,0xff,0x6f,0x0d,0x30]
 # W64: v_add_co_ci_u32_dpp v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x41,0xff,0x6f,0x0d,0x30]
-0xfa,0xfe,0xff,0x41,0xff,0x6f,0x0d,0x30
 
-# GFX12: v_add_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff
+# GFX12-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_add_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_add_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0xe4,0x00,0xff
+# GFX12-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_add_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_add_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x40,0x01,0xff
+# GFX12-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_add_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x40,0x01,0xff]
 
-# GFX12: v_add_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x41,0x01,0xff
+# GFX12-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_add_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x41,0x01,0xff]
 
-# GFX12: v_add_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x01,0x01,0xff
+# GFX12-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_add_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x01,0x01,0xff]
 
-# GFX12: v_add_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x0f,0x01,0xff
+# GFX12-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_add_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_add_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x11,0x01,0xff
+# GFX12-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_add_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x11,0x01,0xff]
 
-# GFX12: v_add_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x1f,0x01,0xff
+# GFX12-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_add_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_add_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x21,0x01,0xff
+# GFX12-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_add_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x21,0x01,0xff]
 
-# GFX12: v_add_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x2f,0x01,0xff
+# GFX12-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_add_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_add_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff
+# GFX12-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_add_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff]
 
-# GFX12: v_add_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_add_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_add_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x64,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_add_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x60,0x01,0x13]
 
-# GFX12: v_add_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xfd,0x30]
 0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xfd,0x30
+# GFX12-REAL16: v_add_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_add_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xfd,0x30]
 
-# GFX12: v_add_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x1b,0x00,0xff
+# GFX12: v_add_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_add_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0xe4,0x00,0xff
+# GFX12: v_add_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_add_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x40,0x01,0xff
+# GFX12: v_add_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x40,0x01,0xff]
 
-# GFX12: v_add_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x41,0x01,0xff
+# GFX12: v_add_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x41,0x01,0xff]
 
-# GFX12: v_add_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x01,0x01,0xff
+# GFX12: v_add_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x01,0x01,0xff]
 
-# GFX12: v_add_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x0f,0x01,0xff
+# GFX12: v_add_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_add_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x11,0x01,0xff
+# GFX12: v_add_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x11,0x01,0xff]
 
-# GFX12: v_add_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x1f,0x01,0xff
+# GFX12: v_add_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_add_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x21,0x01,0xff
+# GFX12: v_add_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x21,0x01,0xff]
 
-# GFX12: v_add_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x2f,0x01,0xff
+# GFX12: v_add_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_add_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x06,0x01,0x50,0x01,0xff
+# GFX12: v_add_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x50,0x01,0xff]
 
-# GFX12: v_add_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x06,0x01,0x5f,0x01,0x01
+# GFX12: v_add_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_add_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x06,0x01,0x60,0x01,0x13
+# GFX12: v_add_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x60,0x01,0x13]
 
-# GFX12: v_add_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x07,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x07,0xff,0x6f,0xfd,0x30
+# GFX12: v_add_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x07,0xff,0x6f,0xfd,0x30]
 
-# GFX12: v_add_nc_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x1b,0x00,0xff
+# GFX12: v_add_nc_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_add_nc_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0xe4,0x00,0xff
+# GFX12: v_add_nc_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x40,0x01,0xff
+# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x40,0x01,0xff]
 
-# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x41,0x01,0xff
+# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x41,0x01,0xff]
 
-# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x01,0x01,0xff
+# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x01,0x01,0xff]
 
-# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x0f,0x01,0xff
+# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x11,0x01,0xff
+# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x11,0x01,0xff]
 
-# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x1f,0x01,0xff
+# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x21,0x01,0xff
+# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x21,0x01,0xff]
 
-# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x2f,0x01,0xff
+# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x4a,0x01,0x50,0x01,0xff
+# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x50,0x01,0xff]
 
-# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x4a,0x01,0x5f,0x01,0x01
+# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x4a,0x01,0x60,0x01,0x13
+# GFX12: v_add_nc_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x4a,0x01,0x60,0x01,0x13]
 
-# GFX12: v_add_nc_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x4b,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x4b,0xff,0x6f,0x0d,0x30
+# GFX12: v_add_nc_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x4b,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_and_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x1b,0x00,0xff
+# GFX12: v_and_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_and_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0xe4,0x00,0xff
+# GFX12: v_and_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_and_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x40,0x01,0xff
+# GFX12: v_and_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x40,0x01,0xff]
 
-# GFX12: v_and_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x41,0x01,0xff
+# GFX12: v_and_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x41,0x01,0xff]
 
-# GFX12: v_and_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x01,0x01,0xff
+# GFX12: v_and_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x01,0x01,0xff]
 
-# GFX12: v_and_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x0f,0x01,0xff
+# GFX12: v_and_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_and_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x11,0x01,0xff
+# GFX12: v_and_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x11,0x01,0xff]
 
-# GFX12: v_and_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x1f,0x01,0xff
+# GFX12: v_and_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_and_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x21,0x01,0xff
+# GFX12: v_and_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x21,0x01,0xff]
 
-# GFX12: v_and_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x2f,0x01,0xff
+# GFX12: v_and_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_and_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x36,0x01,0x50,0x01,0xff
+# GFX12: v_and_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x50,0x01,0xff]
 
-# GFX12: v_and_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x36,0x01,0x5f,0x01,0x01
+# GFX12: v_and_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_and_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x36,0x01,0x60,0x01,0x13
+# GFX12: v_and_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x36,0x01,0x60,0x01,0x13]
 
-# GFX12: v_and_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x37,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x37,0xff,0x6f,0x0d,0x30
+# GFX12: v_and_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x37,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_ashrrev_i32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x1b,0x00,0xff
+# GFX12: v_ashrrev_i32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_ashrrev_i32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0xe4,0x00,0xff
+# GFX12: v_ashrrev_i32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x40,0x01,0xff
+# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x40,0x01,0xff]
 
-# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x41,0x01,0xff
+# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x41,0x01,0xff]
 
-# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x01,0x01,0xff
+# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x01,0x01,0xff]
 
-# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x0f,0x01,0xff
+# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x11,0x01,0xff
+# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x11,0x01,0xff]
 
-# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x1f,0x01,0xff
+# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x21,0x01,0xff
+# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x21,0x01,0xff]
 
-# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x2f,0x01,0xff
+# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x34,0x01,0x50,0x01,0xff
+# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x50,0x01,0xff]
 
-# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x34,0x01,0x5f,0x01,0x01
+# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x34,0x01,0x60,0x01,0x13
+# GFX12: v_ashrrev_i32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x34,0x01,0x60,0x01,0x13]
 
-# GFX12: v_ashrrev_i32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x35,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x35,0xff,0x6f,0x0d,0x30
+# GFX12: v_ashrrev_i32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x35,0xff,0x6f,0x0d,0x30]
 
+0xfa,0x04,0x0a,0x02,0x01,0x1b,0x00,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1b,0x00,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1b,0x00,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x1b,0x00,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0xe4,0x00,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x00,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0x00,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0xe4,0x00,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x40,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x40,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x40,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x40,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x41,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x41,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x41,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x41,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x01,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x01,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x01,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x01,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x0f,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x0f,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x0f,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x0f,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x11,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x11,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x11,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x11,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x1f,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1f,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x1f,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x1f,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x21,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x21,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x21,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x21,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x2f,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x2f,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x2f,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x2f,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x50,0x01,0xff
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x50,0x01,0xff]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x50,0x01,0xff]
-0xfa,0x04,0x0a,0x02,0x01,0x50,0x01,0xff
 
+0xfa,0x04,0x0a,0x02,0x01,0x5f,0x01,0x01
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x5f,0x01,0x01]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x5f,0x01,0x01]
-0xfa,0x04,0x0a,0x02,0x01,0x5f,0x01,0x01
 
+0xfa,0x04,0x0a,0x02,0x01,0x60,0x01,0x13
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x60,0x01,0x13]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0x60,0x01,0x13]
-0xfa,0x04,0x0a,0x02,0x01,0x60,0x01,0x13
 
+0xfa,0xfe,0xff,0x03,0xff,0x6f,0x0d,0x30
 # W32: v_cndmask_b32_dpp v255, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x03,0xff,0x6f,0x0d,0x30]
 # W64: v_cndmask_b32_dpp v255, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x03,0xff,0x6f,0x0d,0x30]
-0xfa,0xfe,0xff,0x03,0xff,0x6f,0x0d,0x30
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x1b,0x00,0xff
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0xe4,0x00,0xff
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x40,0x01,0xff
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x40,0x01,0xff]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x41,0x01,0xff
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x41,0x01,0xff]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x01,0x01,0xff
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x01,0x01,0xff]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x0f,0x01,0xff
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x11,0x01,0xff
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x11,0x01,0xff]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x1f,0x01,0xff
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x21,0x01,0xff
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x21,0x01,0xff]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x2f,0x01,0xff
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x5e,0x01,0x50,0x01,0xff
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x50,0x01,0xff]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x5e,0x01,0x5f,0x01,0x01
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x5e,0x01,0x60,0x01,0x13
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x5e,0x01,0x60,0x01,0x13]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x5f,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x5f,0xff,0x6f,0xfd,0x30
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x5f,0xff,0x6f,0xfd,0x30]
 
-# GFX12: v_fmac_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x1b,0x00,0xff
+# GFX12: v_fmac_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_fmac_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0xe4,0x00,0xff
+# GFX12: v_fmac_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_fmac_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x40,0x01,0xff
+# GFX12: v_fmac_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x40,0x01,0xff]
 
-# GFX12: v_fmac_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x41,0x01,0xff
+# GFX12: v_fmac_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x41,0x01,0xff]
 
-# GFX12: v_fmac_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x01,0x01,0xff
+# GFX12: v_fmac_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x01,0x01,0xff]
 
-# GFX12: v_fmac_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x0f,0x01,0xff
+# GFX12: v_fmac_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_fmac_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x11,0x01,0xff
+# GFX12: v_fmac_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x11,0x01,0xff]
 
-# GFX12: v_fmac_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x1f,0x01,0xff
+# GFX12: v_fmac_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_fmac_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x21,0x01,0xff
+# GFX12: v_fmac_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x21,0x01,0xff]
 
-# GFX12: v_fmac_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x2f,0x01,0xff
+# GFX12: v_fmac_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_fmac_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x6c,0x01,0x50,0x01,0xff
+# GFX12: v_fmac_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x50,0x01,0xff]
 
-# GFX12: v_fmac_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x6c,0x01,0x5f,0x01,0x01
+# GFX12: v_fmac_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_fmac_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x6c,0x01,0x60,0x01,0x13
+# GFX12: v_fmac_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x60,0x01,0x13]
 
-# GFX12: v_fmac_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x6c,0x7f,0x6f,0xfd,0x30]
 0xfa,0xfe,0xfe,0x6c,0x7f,0x6f,0xfd,0x30
+# GFX12: v_fmac_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x6c,0x7f,0x6f,0xfd,0x30]
 
-# GFX12: v_fmac_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x1b,0x00,0xff
+# GFX12: v_fmac_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_fmac_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0xe4,0x00,0xff
+# GFX12: v_fmac_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_fmac_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x40,0x01,0xff
+# GFX12: v_fmac_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x40,0x01,0xff]
 
-# GFX12: v_fmac_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x41,0x01,0xff
+# GFX12: v_fmac_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x41,0x01,0xff]
 
-# GFX12: v_fmac_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x01,0x01,0xff
+# GFX12: v_fmac_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x01,0x01,0xff]
 
-# GFX12: v_fmac_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x0f,0x01,0xff
+# GFX12: v_fmac_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_fmac_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x11,0x01,0xff
+# GFX12: v_fmac_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x11,0x01,0xff]
 
-# GFX12: v_fmac_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x1f,0x01,0xff
+# GFX12: v_fmac_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_fmac_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x21,0x01,0xff
+# GFX12: v_fmac_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x21,0x01,0xff]
 
-# GFX12: v_fmac_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x2f,0x01,0xff
+# GFX12: v_fmac_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_fmac_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x56,0x01,0x50,0x01,0xff
+# GFX12: v_fmac_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x50,0x01,0xff]
 
-# GFX12: v_fmac_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x56,0x01,0x5f,0x01,0x01
+# GFX12: v_fmac_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_fmac_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x56,0x01,0x60,0x01,0x13
+# GFX12: v_fmac_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x60,0x01,0x13]
 
-# GFX12: v_fmac_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x57,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x57,0xff,0x6f,0xfd,0x30
+# GFX12: v_fmac_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x57,0xff,0x6f,0xfd,0x30]
 
-# GFX12: v_ldexp_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x1b,0x00,0xff
+# GFX12-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_ldexp_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_ldexp_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0xe4,0x00,0xff
+# GFX12-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_ldexp_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_ldexp_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x40,0x01,0xff
+# GFX12-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x40,0x01,0xff]
 
-# GFX12: v_ldexp_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x41,0x01,0xff
+# GFX12-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x41,0x01,0xff]
 
-# GFX12: v_ldexp_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x01,0x01,0xff
+# GFX12-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x01,0x01,0xff]
 
-# GFX12: v_ldexp_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x0f,0x01,0xff
+# GFX12-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_ldexp_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x11,0x01,0xff
+# GFX12-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x11,0x01,0xff]
 
-# GFX12: v_ldexp_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x1f,0x01,0xff
+# GFX12-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_ldexp_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x21,0x01,0xff
+# GFX12-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x21,0x01,0xff]
 
-# GFX12: v_ldexp_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x2f,0x01,0xff
+# GFX12-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_ldexp_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x76,0x01,0x50,0x01,0xff
+# GFX12-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x50,0x01,0xff]
 
-# GFX12: v_ldexp_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x76,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_ldexp_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x76,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x60,0x01,0x13]
 
-# GFX12: v_ldexp_f16_dpp v127, -|v127|, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x76,0x7f,0x6f,0x3d,0x30]
 0xfa,0xfe,0xfe,0x76,0x7f,0x6f,0x3d,0x30
+# GFX12-REAL16: v_ldexp_f16_dpp v127.l, -|v127.l|, v127.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x76,0x7f,0x6f,0x3d,0x30]
+# GFX12-FAKE16: v_ldexp_f16_dpp v127, -|v127|, v127 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x76,0x7f,0x6f,0x3d,0x30]
 
-# GFX12: v_lshlrev_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x1b,0x00,0xff
+# GFX12: v_lshlrev_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_lshlrev_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0xe4,0x00,0xff
+# GFX12: v_lshlrev_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x40,0x01,0xff
+# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x40,0x01,0xff]
 
-# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x41,0x01,0xff
+# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x41,0x01,0xff]
 
-# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x01,0x01,0xff
+# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x01,0x01,0xff]
 
-# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x0f,0x01,0xff
+# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x11,0x01,0xff
+# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x11,0x01,0xff]
 
-# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x1f,0x01,0xff
+# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x21,0x01,0xff
+# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x21,0x01,0xff]
 
-# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x2f,0x01,0xff
+# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x30,0x01,0x50,0x01,0xff
+# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x50,0x01,0xff]
 
-# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x30,0x01,0x5f,0x01,0x01
+# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x30,0x01,0x60,0x01,0x13
+# GFX12: v_lshlrev_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x30,0x01,0x60,0x01,0x13]
 
-# GFX12: v_lshlrev_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x31,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x31,0xff,0x6f,0x0d,0x30
+# GFX12: v_lshlrev_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x31,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_lshrrev_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x1b,0x00,0xff
+# GFX12: v_lshrrev_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_lshrrev_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0xe4,0x00,0xff
+# GFX12: v_lshrrev_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x40,0x01,0xff
+# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x40,0x01,0xff]
 
-# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x41,0x01,0xff
+# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x41,0x01,0xff]
 
-# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x01,0x01,0xff
+# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x01,0x01,0xff]
 
-# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x0f,0x01,0xff
+# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x11,0x01,0xff
+# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x11,0x01,0xff]
 
-# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x1f,0x01,0xff
+# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x21,0x01,0xff
+# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x21,0x01,0xff]
 
-# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x2f,0x01,0xff
+# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x32,0x01,0x50,0x01,0xff
+# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x50,0x01,0xff]
 
-# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x32,0x01,0x5f,0x01,0x01
+# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x32,0x01,0x60,0x01,0x13
+# GFX12: v_lshrrev_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x32,0x01,0x60,0x01,0x13]
 
-# GFX12: v_lshrrev_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x33,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x33,0xff,0x6f,0x0d,0x30
+# GFX12: v_lshrrev_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x33,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_max_num_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x62,0x01,0x1b,0x00,0xff
+# GFX12-REAL16: v_max_num_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_max_num_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max_num_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x62,0x01,0xe4,0x00,0xff
+# GFX12-REAL16: v_max_num_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_max_num_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_max_num_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x62,0x01,0x40,0x01,0xff
+# GFX12-REAL16: v_max_num_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_max_num_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x40,0x01,0xff]
 
-# GFX12: v_max_num_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x62,0x01,0x41,0x01,0xff
+# GFX12-REAL16: v_max_num_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_max_num_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x41,0x01,0xff]
 
-# GFX12: v_max_num_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x62,0x01,0x01,0x01,0xff
+# GFX12-REAL16: v_max_num_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_max_num_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x01,0x01,0xff]
 
-# GFX12: v_max_num_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x62,0x01,0x0f,0x01,0xff
+# GFX12-REAL16: v_max_num_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_max_num_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_max_num_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x62,0x01,0x11,0x01,0xff
+# GFX12-REAL16: v_max_num_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_max_num_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x11,0x01,0xff]
 
-# GFX12: v_max_num_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x62,0x01,0x1f,0x01,0xff
+# GFX12-REAL16: v_max_num_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_max_num_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_max_num_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x62,0x01,0x21,0x01,0xff
+# GFX12-REAL16: v_max_num_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_max_num_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x21,0x01,0xff]
 
-# GFX12: v_max_num_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x62,0x01,0x2f,0x01,0xff
+# GFX12-REAL16: v_max_num_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_max_num_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_max_num_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x62,0x01,0x50,0x01,0xff
+# GFX12-REAL16: v_max_num_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_max_num_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x50,0x01,0xff]
 
-# GFX12: v_max_num_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x62,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_max_num_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_max_num_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_max_num_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x62,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_max_num_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_max_num_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x62,0x01,0x60,0x01,0x13]
 
-# GFX12: v_max_num_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x62,0x7f,0x6f,0xfd,0x30]
 0xfa,0xfe,0xfe,0x62,0x7f,0x6f,0xfd,0x30
+# GFX12-REAL16: v_max_num_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x62,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_max_num_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x62,0x7f,0x6f,0xfd,0x30]
 
-# GFX12: v_max_num_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x2c,0x01,0x1b,0x00,0xff
+# GFX12: v_max_num_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max_num_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x2c,0x01,0xe4,0x00,0xff
+# GFX12: v_max_num_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_max_num_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x2c,0x01,0x40,0x01,0xff
+# GFX12: v_max_num_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x40,0x01,0xff]
 
-# GFX12: v_max_num_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x2c,0x01,0x41,0x01,0xff
+# GFX12: v_max_num_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x41,0x01,0xff]
 
-# GFX12: v_max_num_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x2c,0x01,0x01,0x01,0xff
+# GFX12: v_max_num_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x01,0x01,0xff]
 
-# GFX12: v_max_num_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x2c,0x01,0x0f,0x01,0xff
+# GFX12: v_max_num_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_max_num_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x2c,0x01,0x11,0x01,0xff
+# GFX12: v_max_num_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x11,0x01,0xff]
 
-# GFX12: v_max_num_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x2c,0x01,0x1f,0x01,0xff
+# GFX12: v_max_num_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_max_num_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x2c,0x01,0x21,0x01,0xff
+# GFX12: v_max_num_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x21,0x01,0xff]
 
-# GFX12: v_max_num_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x2c,0x01,0x2f,0x01,0xff
+# GFX12: v_max_num_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_max_num_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x2c,0x01,0x50,0x01,0xff
+# GFX12: v_max_num_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x50,0x01,0xff]
 
-# GFX12: v_max_num_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x2c,0x01,0x5f,0x01,0x01
+# GFX12: v_max_num_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_max_num_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x2c,0x01,0x60,0x01,0x13
+# GFX12: v_max_num_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x2c,0x01,0x60,0x01,0x13]
 
-# GFX12: v_max_num_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x2d,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x2d,0xff,0x6f,0xfd,0x30
+# GFX12: v_max_num_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x2d,0xff,0x6f,0xfd,0x30]
 
-# GFX12: v_max_i32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x1b,0x00,0xff
+# GFX12: v_max_i32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max_i32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0xe4,0x00,0xff
+# GFX12: v_max_i32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_max_i32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x40,0x01,0xff
+# GFX12: v_max_i32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x40,0x01,0xff]
 
-# GFX12: v_max_i32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x41,0x01,0xff
+# GFX12: v_max_i32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x41,0x01,0xff]
 
-# GFX12: v_max_i32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x01,0x01,0xff
+# GFX12: v_max_i32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x01,0x01,0xff]
 
-# GFX12: v_max_i32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x0f,0x01,0xff
+# GFX12: v_max_i32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_max_i32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x11,0x01,0xff
+# GFX12: v_max_i32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x11,0x01,0xff]
 
-# GFX12: v_max_i32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x1f,0x01,0xff
+# GFX12: v_max_i32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_max_i32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x21,0x01,0xff
+# GFX12: v_max_i32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x21,0x01,0xff]
 
-# GFX12: v_max_i32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x2f,0x01,0xff
+# GFX12: v_max_i32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_max_i32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x24,0x01,0x50,0x01,0xff
+# GFX12: v_max_i32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x50,0x01,0xff]
 
-# GFX12: v_max_i32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x24,0x01,0x5f,0x01,0x01
+# GFX12: v_max_i32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_max_i32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x24,0x01,0x60,0x01,0x13
+# GFX12: v_max_i32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x24,0x01,0x60,0x01,0x13]
 
-# GFX12: v_max_i32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x25,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x25,0xff,0x6f,0x0d,0x30
+# GFX12: v_max_i32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x25,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_max_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x1b,0x00,0xff
+# GFX12: v_max_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0xe4,0x00,0xff
+# GFX12: v_max_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_max_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x40,0x01,0xff
+# GFX12: v_max_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x40,0x01,0xff]
 
-# GFX12: v_max_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x41,0x01,0xff
+# GFX12: v_max_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x41,0x01,0xff]
 
-# GFX12: v_max_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x01,0x01,0xff
+# GFX12: v_max_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x01,0x01,0xff]
 
-# GFX12: v_max_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x0f,0x01,0xff
+# GFX12: v_max_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_max_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x11,0x01,0xff
+# GFX12: v_max_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x11,0x01,0xff]
 
-# GFX12: v_max_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x1f,0x01,0xff
+# GFX12: v_max_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_max_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x21,0x01,0xff
+# GFX12: v_max_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x21,0x01,0xff]
 
-# GFX12: v_max_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x2f,0x01,0xff
+# GFX12: v_max_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_max_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x28,0x01,0x50,0x01,0xff
+# GFX12: v_max_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x50,0x01,0xff]
 
-# GFX12: v_max_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x28,0x01,0x5f,0x01,0x01
+# GFX12: v_max_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_max_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x28,0x01,0x60,0x01,0x13
+# GFX12: v_max_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x28,0x01,0x60,0x01,0x13]
 
-# GFX12: v_max_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x29,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x29,0xff,0x6f,0x0d,0x30
+# GFX12: v_max_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x29,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_min_num_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x60,0x01,0x1b,0x00,0xff
+# GFX12-REAL16: v_min_num_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_min_num_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min_num_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x60,0x01,0xe4,0x00,0xff
+# GFX12-REAL16: v_min_num_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_min_num_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_min_num_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x60,0x01,0x40,0x01,0xff
+# GFX12-REAL16: v_min_num_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_min_num_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x40,0x01,0xff]
 
-# GFX12: v_min_num_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x60,0x01,0x41,0x01,0xff
+# GFX12-REAL16: v_min_num_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_min_num_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x41,0x01,0xff]
 
-# GFX12: v_min_num_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x60,0x01,0x01,0x01,0xff
+# GFX12-REAL16: v_min_num_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_min_num_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x01,0x01,0xff]
 
-# GFX12: v_min_num_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x60,0x01,0x0f,0x01,0xff
+# GFX12-REAL16: v_min_num_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_min_num_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_min_num_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x60,0x01,0x11,0x01,0xff
+# GFX12-REAL16: v_min_num_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_min_num_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x11,0x01,0xff]
 
-# GFX12: v_min_num_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x60,0x01,0x1f,0x01,0xff
+# GFX12-REAL16: v_min_num_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_min_num_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_min_num_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x60,0x01,0x21,0x01,0xff
+# GFX12-REAL16: v_min_num_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_min_num_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x21,0x01,0xff]
 
-# GFX12: v_min_num_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x60,0x01,0x2f,0x01,0xff
+# GFX12-REAL16: v_min_num_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_min_num_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_min_num_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x60,0x01,0x50,0x01,0xff
+# GFX12-REAL16: v_min_num_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_min_num_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x50,0x01,0xff]
 
-# GFX12: v_min_num_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x60,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_min_num_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_min_num_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_min_num_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x60,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_min_num_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_min_num_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x60,0x01,0x60,0x01,0x13]
 
-# GFX12: v_min_num_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x60,0x7f,0x6f,0xfd,0x30]
 0xfa,0xfe,0xfe,0x60,0x7f,0x6f,0xfd,0x30
+# GFX12-REAL16: v_min_num_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x60,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_min_num_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x60,0x7f,0x6f,0xfd,0x30]
 
-# GFX12: v_min_num_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x2a,0x01,0x1b,0x00,0xff
+# GFX12: v_min_num_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min_num_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x2a,0x01,0xe4,0x00,0xff
+# GFX12: v_min_num_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_min_num_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x2a,0x01,0x40,0x01,0xff
+# GFX12: v_min_num_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x40,0x01,0xff]
 
-# GFX12: v_min_num_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x2a,0x01,0x41,0x01,0xff
+# GFX12: v_min_num_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x41,0x01,0xff]
 
-# GFX12: v_min_num_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x2a,0x01,0x01,0x01,0xff
+# GFX12: v_min_num_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x01,0x01,0xff]
 
-# GFX12: v_min_num_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x2a,0x01,0x0f,0x01,0xff
+# GFX12: v_min_num_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_min_num_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x2a,0x01,0x11,0x01,0xff
+# GFX12: v_min_num_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x11,0x01,0xff]
 
-# GFX12: v_min_num_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x2a,0x01,0x1f,0x01,0xff
+# GFX12: v_min_num_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_min_num_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x2a,0x01,0x21,0x01,0xff
+# GFX12: v_min_num_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x21,0x01,0xff]
 
-# GFX12: v_min_num_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x2a,0x01,0x2f,0x01,0xff
+# GFX12: v_min_num_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_min_num_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x2a,0x01,0x50,0x01,0xff
+# GFX12: v_min_num_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x50,0x01,0xff]
 
-# GFX12: v_min_num_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x2a,0x01,0x5f,0x01,0x01
+# GFX12: v_min_num_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_min_num_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x2a,0x01,0x60,0x01,0x13
+# GFX12: v_min_num_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x2a,0x01,0x60,0x01,0x13]
 
-# GFX12: v_min_num_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x2b,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x2b,0xff,0x6f,0xfd,0x30
+# GFX12: v_min_num_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x2b,0xff,0x6f,0xfd,0x30]
 
-# GFX12: v_min_i32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x1b,0x00,0xff
+# GFX12: v_min_i32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min_i32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0xe4,0x00,0xff
+# GFX12: v_min_i32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_min_i32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x40,0x01,0xff
+# GFX12: v_min_i32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x40,0x01,0xff]
 
-# GFX12: v_min_i32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x41,0x01,0xff
+# GFX12: v_min_i32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x41,0x01,0xff]
 
-# GFX12: v_min_i32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x01,0x01,0xff
+# GFX12: v_min_i32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x01,0x01,0xff]
 
-# GFX12: v_min_i32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x0f,0x01,0xff
+# GFX12: v_min_i32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_min_i32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x11,0x01,0xff
+# GFX12: v_min_i32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x11,0x01,0xff]
 
-# GFX12: v_min_i32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x1f,0x01,0xff
+# GFX12: v_min_i32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_min_i32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x21,0x01,0xff
+# GFX12: v_min_i32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x21,0x01,0xff]
 
-# GFX12: v_min_i32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x2f,0x01,0xff
+# GFX12: v_min_i32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_min_i32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x22,0x01,0x50,0x01,0xff
+# GFX12: v_min_i32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x50,0x01,0xff]
 
-# GFX12: v_min_i32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x22,0x01,0x5f,0x01,0x01
+# GFX12: v_min_i32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_min_i32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x22,0x01,0x60,0x01,0x13
+# GFX12: v_min_i32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x22,0x01,0x60,0x01,0x13]
 
-# GFX12: v_min_i32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x23,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x23,0xff,0x6f,0x0d,0x30
+# GFX12: v_min_i32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x23,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_min_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x1b,0x00,0xff
+# GFX12: v_min_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0xe4,0x00,0xff
+# GFX12: v_min_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_min_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x40,0x01,0xff
+# GFX12: v_min_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x40,0x01,0xff]
 
-# GFX12: v_min_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x41,0x01,0xff
+# GFX12: v_min_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x41,0x01,0xff]
 
-# GFX12: v_min_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x01,0x01,0xff
+# GFX12: v_min_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x01,0x01,0xff]
 
-# GFX12: v_min_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x0f,0x01,0xff
+# GFX12: v_min_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_min_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x11,0x01,0xff
+# GFX12: v_min_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x11,0x01,0xff]
 
-# GFX12: v_min_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x1f,0x01,0xff
+# GFX12: v_min_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_min_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x21,0x01,0xff
+# GFX12: v_min_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x21,0x01,0xff]
 
-# GFX12: v_min_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x2f,0x01,0xff
+# GFX12: v_min_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_min_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x26,0x01,0x50,0x01,0xff
+# GFX12: v_min_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x50,0x01,0xff]
 
-# GFX12: v_min_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x26,0x01,0x5f,0x01,0x01
+# GFX12: v_min_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_min_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x26,0x01,0x60,0x01,0x13
+# GFX12: v_min_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x26,0x01,0x60,0x01,0x13]
 
-# GFX12: v_min_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x27,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x27,0xff,0x6f,0x0d,0x30
+# GFX12: v_min_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x27,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x1b,0x00,0xff
+# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0xe4,0x00,0xff
+# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x40,0x01,0xff
+# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x41,0x01,0xff
+# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x01,0x01,0xff
+# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x0f,0x01,0xff
+# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x11,0x01,0xff
+# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x1f,0x01,0xff
+# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x21,0x01,0xff
+# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x2f,0x01,0xff
+# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x0e,0x01,0x50,0x01,0xff
+# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x0e,0x01,0x5f,0x01,0x01
+# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x0e,0x01,0x60,0x01,0x13
+# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x0e,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x0f,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x0f,0xff,0x6f,0xfd,0x30
+# GFX12: v_mul_dx9_zero_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x0f,0xff,0x6f,0xfd,0x30]
 
-# GFX12: v_mul_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x1b,0x00,0xff
+# GFX12-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_mul_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mul_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0xe4,0x00,0xff
+# GFX12-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_mul_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mul_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x40,0x01,0xff
+# GFX12-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_mul_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mul_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x41,0x01,0xff
+# GFX12-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_mul_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mul_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x01,0x01,0xff
+# GFX12-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_mul_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mul_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x0f,0x01,0xff
+# GFX12-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_mul_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mul_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x11,0x01,0xff
+# GFX12-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_mul_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mul_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x1f,0x01,0xff
+# GFX12-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_mul_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mul_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x21,0x01,0xff
+# GFX12-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_mul_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mul_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x2f,0x01,0xff
+# GFX12-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_mul_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mul_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x6a,0x01,0x50,0x01,0xff
+# GFX12-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_mul_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mul_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x6a,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_mul_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mul_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x6a,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_mul_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mul_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x6a,0x7f,0x6f,0xfd,0x30]
 0xfa,0xfe,0xfe,0x6a,0x7f,0x6f,0xfd,0x30
+# GFX12-REAL16: v_mul_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x6a,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_mul_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x6a,0x7f,0x6f,0xfd,0x30]
 
-# GFX12: v_mul_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x1b,0x00,0xff
+# GFX12: v_mul_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mul_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0xe4,0x00,0xff
+# GFX12: v_mul_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mul_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x40,0x01,0xff
+# GFX12: v_mul_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mul_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x41,0x01,0xff
+# GFX12: v_mul_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mul_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x01,0x01,0xff
+# GFX12: v_mul_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mul_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x0f,0x01,0xff
+# GFX12: v_mul_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mul_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x11,0x01,0xff
+# GFX12: v_mul_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mul_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x1f,0x01,0xff
+# GFX12: v_mul_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mul_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x21,0x01,0xff
+# GFX12: v_mul_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mul_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x2f,0x01,0xff
+# GFX12: v_mul_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mul_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x10,0x01,0x50,0x01,0xff
+# GFX12: v_mul_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mul_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x10,0x01,0x5f,0x01,0x01
+# GFX12: v_mul_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mul_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x10,0x01,0x60,0x01,0x13
+# GFX12: v_mul_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x10,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mul_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x11,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x11,0xff,0x6f,0xfd,0x30
+# GFX12: v_mul_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x11,0xff,0x6f,0xfd,0x30]
 
-# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x1b,0x00,0xff
+# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0xe4,0x00,0xff
+# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x40,0x01,0xff
+# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x41,0x01,0xff
+# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x01,0x01,0xff
+# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x0f,0x01,0xff
+# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x11,0x01,0xff
+# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x1f,0x01,0xff
+# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x21,0x01,0xff
+# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x2f,0x01,0xff
+# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x14,0x01,0x50,0x01,0xff
+# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x14,0x01,0x5f,0x01,0x01
+# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x14,0x01,0x60,0x01,0x13
+# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x14,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mul_hi_i32_i24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x15,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x15,0xff,0x6f,0x0d,0x30
+# GFX12: v_mul_hi_i32_i24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x15,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x1b,0x00,0xff
+# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0xe4,0x00,0xff
+# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x40,0x01,0xff
+# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x41,0x01,0xff
+# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x01,0x01,0xff
+# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x0f,0x01,0xff
+# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x11,0x01,0xff
+# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x1f,0x01,0xff
+# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x21,0x01,0xff
+# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x2f,0x01,0xff
+# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x18,0x01,0x50,0x01,0xff
+# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x18,0x01,0x5f,0x01,0x01
+# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x18,0x01,0x60,0x01,0x13
+# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x18,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mul_hi_u32_u24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x19,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x19,0xff,0x6f,0x0d,0x30
+# GFX12: v_mul_hi_u32_u24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x19,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_mul_i32_i24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x1b,0x00,0xff
+# GFX12: v_mul_i32_i24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mul_i32_i24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0xe4,0x00,0xff
+# GFX12: v_mul_i32_i24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x40,0x01,0xff
+# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x41,0x01,0xff
+# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x01,0x01,0xff
+# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x0f,0x01,0xff
+# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x11,0x01,0xff
+# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x1f,0x01,0xff
+# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x21,0x01,0xff
+# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x2f,0x01,0xff
+# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x12,0x01,0x50,0x01,0xff
+# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x12,0x01,0x5f,0x01,0x01
+# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x12,0x01,0x60,0x01,0x13
+# GFX12: v_mul_i32_i24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x12,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mul_i32_i24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x13,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x13,0xff,0x6f,0x0d,0x30
+# GFX12: v_mul_i32_i24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x13,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_mul_u32_u24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x1b,0x00,0xff
+# GFX12: v_mul_u32_u24_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mul_u32_u24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0xe4,0x00,0xff
+# GFX12: v_mul_u32_u24_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x40,0x01,0xff
+# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x41,0x01,0xff
+# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x01,0x01,0xff
+# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x0f,0x01,0xff
+# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x11,0x01,0xff
+# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x1f,0x01,0xff
+# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x21,0x01,0xff
+# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x2f,0x01,0xff
+# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x16,0x01,0x50,0x01,0xff
+# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x16,0x01,0x5f,0x01,0x01
+# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x16,0x01,0x60,0x01,0x13
+# GFX12: v_mul_u32_u24_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x16,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mul_u32_u24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x17,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x17,0xff,0x6f,0x0d,0x30
+# GFX12: v_mul_u32_u24_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x17,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_or_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x1b,0x00,0xff
+# GFX12: v_or_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_or_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0xe4,0x00,0xff
+# GFX12: v_or_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_or_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x40,0x01,0xff
+# GFX12: v_or_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x40,0x01,0xff]
 
-# GFX12: v_or_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x41,0x01,0xff
+# GFX12: v_or_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x41,0x01,0xff]
 
-# GFX12: v_or_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x01,0x01,0xff
+# GFX12: v_or_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x01,0x01,0xff]
 
-# GFX12: v_or_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x0f,0x01,0xff
+# GFX12: v_or_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_or_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x11,0x01,0xff
+# GFX12: v_or_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x11,0x01,0xff]
 
-# GFX12: v_or_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x1f,0x01,0xff
+# GFX12: v_or_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_or_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x21,0x01,0xff
+# GFX12: v_or_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x21,0x01,0xff]
 
-# GFX12: v_or_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x2f,0x01,0xff
+# GFX12: v_or_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_or_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x38,0x01,0x50,0x01,0xff
+# GFX12: v_or_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x50,0x01,0xff]
 
-# GFX12: v_or_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x38,0x01,0x5f,0x01,0x01
+# GFX12: v_or_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_or_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x38,0x01,0x60,0x01,0x13
+# GFX12: v_or_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x38,0x01,0x60,0x01,0x13]
 
-# GFX12: v_or_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x39,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x39,0xff,0x6f,0x0d,0x30
+# GFX12: v_or_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x39,0xff,0x6f,0x0d,0x30]
 
+0xfa,0x04,0x0a,0x42,0x01,0x1b,0x00,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1b,0x00,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1b,0x00,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x1b,0x00,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0xe4,0x00,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0xe4,0x00,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0xe4,0x00,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0xe4,0x00,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x40,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x40,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x40,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x40,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x41,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x41,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x41,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x41,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x01,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x01,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x01,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x01,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x0f,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x0f,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x0f,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x0f,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x11,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x11,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x11,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x11,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x1f,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1f,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x1f,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x1f,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x21,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x21,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x21,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x21,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x2f,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x2f,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x2f,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x2f,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x50,0x01,0xff
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x50,0x01,0xff]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x50,0x01,0xff]
-0xfa,0x04,0x0a,0x42,0x01,0x50,0x01,0xff
 
+0xfa,0x04,0x0a,0x42,0x01,0x5f,0x01,0x01
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x5f,0x01,0x01]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x5f,0x01,0x01]
-0xfa,0x04,0x0a,0x42,0x01,0x5f,0x01,0x01
 
+0xfa,0x04,0x0a,0x42,0x01,0x60,0x01,0x13
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x60,0x01,0x13]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x42,0x01,0x60,0x01,0x13]
-0xfa,0x04,0x0a,0x42,0x01,0x60,0x01,0x13
 
+0xfa,0xfe,0xff,0x43,0xff,0x6f,0x0d,0x30
 # W32: v_sub_co_ci_u32_dpp v255, vcc_lo, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x43,0xff,0x6f,0x0d,0x30]
 # W64: v_sub_co_ci_u32_dpp v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x43,0xff,0x6f,0x0d,0x30]
-0xfa,0xfe,0xff,0x43,0xff,0x6f,0x0d,0x30
 
-# GFX12: v_sub_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x1b,0x00,0xff
+# GFX12-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_sub_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_sub_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0xe4,0x00,0xff
+# GFX12-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_sub_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_sub_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x40,0x01,0xff
+# GFX12-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_sub_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x40,0x01,0xff]
 
-# GFX12: v_sub_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x41,0x01,0xff
+# GFX12-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_sub_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x41,0x01,0xff]
 
-# GFX12: v_sub_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x01,0x01,0xff
+# GFX12-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_sub_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x01,0x01,0xff]
 
-# GFX12: v_sub_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x0f,0x01,0xff
+# GFX12-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_sub_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_sub_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x11,0x01,0xff
+# GFX12-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_sub_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x11,0x01,0xff]
 
-# GFX12: v_sub_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x1f,0x01,0xff
+# GFX12-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_sub_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_sub_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x21,0x01,0xff
+# GFX12-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_sub_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x21,0x01,0xff]
 
-# GFX12: v_sub_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x2f,0x01,0xff
+# GFX12-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_sub_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_sub_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff
+# GFX12-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_sub_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff]
 
-# GFX12: v_sub_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_sub_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_sub_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x66,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_sub_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x60,0x01,0x13]
 
-# GFX12: v_sub_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xfd,0x30]
 0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xfd,0x30
+# GFX12-REAL16: v_sub_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_sub_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xfd,0x30]
 
-# GFX12: v_sub_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x1b,0x00,0xff
+# GFX12: v_sub_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_sub_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0xe4,0x00,0xff
+# GFX12: v_sub_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_sub_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x40,0x01,0xff
+# GFX12: v_sub_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x40,0x01,0xff]
 
-# GFX12: v_sub_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x41,0x01,0xff
+# GFX12: v_sub_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x41,0x01,0xff]
 
-# GFX12: v_sub_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x01,0x01,0xff
+# GFX12: v_sub_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x01,0x01,0xff]
 
-# GFX12: v_sub_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x0f,0x01,0xff
+# GFX12: v_sub_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_sub_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x11,0x01,0xff
+# GFX12: v_sub_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x11,0x01,0xff]
 
-# GFX12: v_sub_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x1f,0x01,0xff
+# GFX12: v_sub_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_sub_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x21,0x01,0xff
+# GFX12: v_sub_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x21,0x01,0xff]
 
-# GFX12: v_sub_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x2f,0x01,0xff
+# GFX12: v_sub_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_sub_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x08,0x01,0x50,0x01,0xff
+# GFX12: v_sub_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x50,0x01,0xff]
 
-# GFX12: v_sub_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x08,0x01,0x5f,0x01,0x01
+# GFX12: v_sub_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_sub_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x08,0x01,0x60,0x01,0x13
+# GFX12: v_sub_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x60,0x01,0x13]
 
-# GFX12: v_sub_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x09,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x09,0xff,0x6f,0xfd,0x30
+# GFX12: v_sub_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x09,0xff,0x6f,0xfd,0x30]
 
-# GFX12: v_sub_nc_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x1b,0x00,0xff
+# GFX12: v_sub_nc_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_sub_nc_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0xe4,0x00,0xff
+# GFX12: v_sub_nc_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x40,0x01,0xff
+# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x40,0x01,0xff]
 
-# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x41,0x01,0xff
+# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x41,0x01,0xff]
 
-# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x01,0x01,0xff
+# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x01,0x01,0xff]
 
-# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x0f,0x01,0xff
+# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x11,0x01,0xff
+# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x11,0x01,0xff]
 
-# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x1f,0x01,0xff
+# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x21,0x01,0xff
+# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x21,0x01,0xff]
 
-# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x2f,0x01,0xff
+# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x4c,0x01,0x50,0x01,0xff
+# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x50,0x01,0xff]
 
-# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x4c,0x01,0x5f,0x01,0x01
+# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x4c,0x01,0x60,0x01,0x13
+# GFX12: v_sub_nc_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x4c,0x01,0x60,0x01,0x13]
 
-# GFX12: v_sub_nc_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x4d,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x4d,0xff,0x6f,0x0d,0x30
+# GFX12: v_sub_nc_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x4d,0xff,0x6f,0x0d,0x30]
 
+0xfa,0x04,0x0a,0x44,0x01,0x1b,0x00,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1b,0x00,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1b,0x00,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x1b,0x00,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0xe4,0x00,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0xe4,0x00,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0xe4,0x00,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0xe4,0x00,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x40,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x40,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x40,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x40,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x41,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x41,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x41,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x41,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x01,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x01,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x01,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x01,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x0f,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x0f,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x0f,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x0f,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x11,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x11,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x11,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x11,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x1f,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1f,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x1f,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x1f,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x21,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x21,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x21,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x21,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x2f,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x2f,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x2f,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x2f,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x50,0x01,0xff
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x50,0x01,0xff]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x50,0x01,0xff]
-0xfa,0x04,0x0a,0x44,0x01,0x50,0x01,0xff
 
+0xfa,0x04,0x0a,0x44,0x01,0x5f,0x01,0x01
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x5f,0x01,0x01]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x5f,0x01,0x01]
-0xfa,0x04,0x0a,0x44,0x01,0x5f,0x01,0x01
 
+0xfa,0x04,0x0a,0x44,0x01,0x60,0x01,0x13
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x60,0x01,0x13]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x44,0x01,0x60,0x01,0x13]
-0xfa,0x04,0x0a,0x44,0x01,0x60,0x01,0x13
 
+0xfa,0xfe,0xff,0x45,0xff,0x6f,0x0d,0x30
 # W32: v_subrev_co_ci_u32_dpp v255, vcc_lo, v255, v255, vcc_lo row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x45,0xff,0x6f,0x0d,0x30]
 # W64: v_subrev_co_ci_u32_dpp v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x45,0xff,0x6f,0x0d,0x30]
-0xfa,0xfe,0xff,0x45,0xff,0x6f,0x0d,0x30
 
-# GFX12: v_subrev_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x1b,0x00,0xff
+# GFX12-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1b,0x00,0xff]
+# GFX12-FAKE16: v_subrev_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_subrev_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0xe4,0x00,0xff
+# GFX12-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0xe4,0x00,0xff]
+# GFX12-FAKE16: v_subrev_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_subrev_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x40,0x01,0xff
+# GFX12-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x40,0x01,0xff]
+# GFX12-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x40,0x01,0xff]
 
-# GFX12: v_subrev_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x41,0x01,0xff
+# GFX12-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x41,0x01,0xff]
+# GFX12-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x41,0x01,0xff]
 
-# GFX12: v_subrev_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x01,0x01,0xff
+# GFX12-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x01,0x01,0xff]
+# GFX12-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x01,0x01,0xff]
 
-# GFX12: v_subrev_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x0f,0x01,0xff
+# GFX12-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x0f,0x01,0xff]
+# GFX12-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_subrev_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x11,0x01,0xff
+# GFX12-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x11,0x01,0xff]
+# GFX12-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x11,0x01,0xff]
 
-# GFX12: v_subrev_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x1f,0x01,0xff
+# GFX12-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1f,0x01,0xff]
+# GFX12-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_subrev_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x21,0x01,0xff
+# GFX12-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x21,0x01,0xff]
+# GFX12-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x21,0x01,0xff]
 
-# GFX12: v_subrev_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x2f,0x01,0xff
+# GFX12-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x2f,0x01,0xff]
+# GFX12-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_subrev_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x68,0x01,0x50,0x01,0xff
+# GFX12-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x50,0x01,0xff]
+# GFX12-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x50,0x01,0xff]
 
-# GFX12: v_subrev_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x68,0x01,0x5f,0x01,0x01
+# GFX12-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x5f,0x01,0x01]
+# GFX12-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_subrev_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x68,0x01,0x60,0x01,0x13
+# GFX12-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x60,0x01,0x13]
+# GFX12-FAKE16: v_subrev_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x68,0x01,0x60,0x01,0x13]
 
-# GFX12: v_subrev_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x68,0x7f,0x6f,0xfd,0x30]
 0xfa,0xfe,0xfe,0x68,0x7f,0x6f,0xfd,0x30
+# GFX12-REAL16: v_subrev_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x68,0x7f,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_subrev_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x68,0x7f,0x6f,0xfd,0x30]
 
-# GFX12: v_subrev_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x1b,0x00,0xff
+# GFX12: v_subrev_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_subrev_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0xe4,0x00,0xff
+# GFX12: v_subrev_f32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_subrev_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x40,0x01,0xff
+# GFX12: v_subrev_f32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x40,0x01,0xff]
 
-# GFX12: v_subrev_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x41,0x01,0xff
+# GFX12: v_subrev_f32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x41,0x01,0xff]
 
-# GFX12: v_subrev_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x01,0x01,0xff
+# GFX12: v_subrev_f32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x01,0x01,0xff]
 
-# GFX12: v_subrev_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x0f,0x01,0xff
+# GFX12: v_subrev_f32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_subrev_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x11,0x01,0xff
+# GFX12: v_subrev_f32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x11,0x01,0xff]
 
-# GFX12: v_subrev_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x1f,0x01,0xff
+# GFX12: v_subrev_f32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_subrev_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x21,0x01,0xff
+# GFX12: v_subrev_f32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x21,0x01,0xff]
 
-# GFX12: v_subrev_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x2f,0x01,0xff
+# GFX12: v_subrev_f32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_subrev_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x0a,0x01,0x50,0x01,0xff
+# GFX12: v_subrev_f32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x50,0x01,0xff]
 
-# GFX12: v_subrev_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x0a,0x01,0x5f,0x01,0x01
+# GFX12: v_subrev_f32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_subrev_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x0a,0x01,0x60,0x01,0x13
+# GFX12: v_subrev_f32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x0a,0x01,0x60,0x01,0x13]
 
-# GFX12: v_subrev_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x0b,0xff,0x6f,0xfd,0x30]
 0xfa,0xfe,0xff,0x0b,0xff,0x6f,0xfd,0x30
+# GFX12: v_subrev_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x0b,0xff,0x6f,0xfd,0x30]
 
-# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x1b,0x00,0xff
+# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0xe4,0x00,0xff
+# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x40,0x01,0xff
+# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x40,0x01,0xff]
 
-# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x41,0x01,0xff
+# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x41,0x01,0xff]
 
-# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x01,0x01,0xff
+# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x01,0x01,0xff]
 
-# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x0f,0x01,0xff
+# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x11,0x01,0xff
+# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x11,0x01,0xff]
 
-# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x1f,0x01,0xff
+# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x21,0x01,0xff
+# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x21,0x01,0xff]
 
-# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x2f,0x01,0xff
+# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x4e,0x01,0x50,0x01,0xff
+# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x50,0x01,0xff]
 
-# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x4e,0x01,0x5f,0x01,0x01
+# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x4e,0x01,0x60,0x01,0x13
+# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x4e,0x01,0x60,0x01,0x13]
 
-# GFX12: v_subrev_nc_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x4f,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x4f,0xff,0x6f,0x0d,0x30
+# GFX12: v_subrev_nc_u32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x4f,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_xnor_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x1b,0x00,0xff
+# GFX12: v_xnor_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_xnor_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0xe4,0x00,0xff
+# GFX12: v_xnor_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_xnor_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x40,0x01,0xff
+# GFX12: v_xnor_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x40,0x01,0xff]
 
-# GFX12: v_xnor_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x41,0x01,0xff
+# GFX12: v_xnor_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x41,0x01,0xff]
 
-# GFX12: v_xnor_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x01,0x01,0xff
+# GFX12: v_xnor_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x01,0x01,0xff]
 
-# GFX12: v_xnor_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x0f,0x01,0xff
+# GFX12: v_xnor_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_xnor_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x11,0x01,0xff
+# GFX12: v_xnor_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x11,0x01,0xff]
 
-# GFX12: v_xnor_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x1f,0x01,0xff
+# GFX12: v_xnor_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_xnor_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x21,0x01,0xff
+# GFX12: v_xnor_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x21,0x01,0xff]
 
-# GFX12: v_xnor_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x2f,0x01,0xff
+# GFX12: v_xnor_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_xnor_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x3c,0x01,0x50,0x01,0xff
+# GFX12: v_xnor_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x50,0x01,0xff]
 
-# GFX12: v_xnor_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x3c,0x01,0x5f,0x01,0x01
+# GFX12: v_xnor_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_xnor_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x3c,0x01,0x60,0x01,0x13
+# GFX12: v_xnor_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x3c,0x01,0x60,0x01,0x13]
 
-# GFX12: v_xnor_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x3d,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x3d,0xff,0x6f,0x0d,0x30
+# GFX12: v_xnor_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x3d,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_xor_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x1b,0x00,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x1b,0x00,0xff
+# GFX12: v_xor_b32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_xor_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0xe4,0x00,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0xe4,0x00,0xff
+# GFX12: v_xor_b32_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_xor_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x40,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x40,0x01,0xff
+# GFX12: v_xor_b32_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x40,0x01,0xff]
 
-# GFX12: v_xor_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x41,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x41,0x01,0xff
+# GFX12: v_xor_b32_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x41,0x01,0xff]
 
-# GFX12: v_xor_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x01,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x01,0x01,0xff
+# GFX12: v_xor_b32_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x01,0x01,0xff]
 
-# GFX12: v_xor_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x0f,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x0f,0x01,0xff
+# GFX12: v_xor_b32_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_xor_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x11,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x11,0x01,0xff
+# GFX12: v_xor_b32_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x11,0x01,0xff]
 
-# GFX12: v_xor_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x1f,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x1f,0x01,0xff
+# GFX12: v_xor_b32_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_xor_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x21,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x21,0x01,0xff
+# GFX12: v_xor_b32_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x21,0x01,0xff]
 
-# GFX12: v_xor_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x2f,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x2f,0x01,0xff
+# GFX12: v_xor_b32_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_xor_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x50,0x01,0xff]
 0xfa,0x04,0x0a,0x3a,0x01,0x50,0x01,0xff
+# GFX12: v_xor_b32_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x50,0x01,0xff]
 
-# GFX12: v_xor_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x5f,0x01,0x01]
 0xfa,0x04,0x0a,0x3a,0x01,0x5f,0x01,0x01
+# GFX12: v_xor_b32_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_xor_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x60,0x01,0x13]
 0xfa,0x04,0x0a,0x3a,0x01,0x60,0x01,0x13
+# GFX12: v_xor_b32_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x3a,0x01,0x60,0x01,0x13]
 
-# GFX12: v_xor_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x3b,0xff,0x6f,0x0d,0x30]
 0xfa,0xfe,0xff,0x3b,0xff,0x6f,0x0d,0x30
+# GFX12: v_xor_b32_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x3b,0xff,0x6f,0x0d,0x30]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp8.txt
index 2e33df35af1f..bbf494c153fd 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp8.txt
@@ -1,244 +1,261 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,GFX12-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,GFX12-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,GFX12-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,GFX12-FAKE16 %s
 
+0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05
 # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05]
 # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05]
-0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05
 
+0xea,0xfe,0xff,0x41,0xff,0x00,0x00,0x00
 # W32: v_add_co_ci_u32_dpp v255, vcc_lo, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x41,0xff,0x00,0x00,0x00]
 # W64: v_add_co_ci_u32_dpp v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x41,0xff,0x00,0x00,0x00]
-0xea,0xfe,0xff,0x41,0xff,0x00,0x00,0x00
 
-# GFX12: v_add_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_add_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00
+# GFX12-REAL16: v_add_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_add_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_add_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x06,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x06,0x01,0x77,0x39,0x05
+# GFX12: v_add_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x06,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x07,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x07,0xff,0x00,0x00,0x00
+# GFX12: v_add_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x07,0xff,0x00,0x00,0x00]
 
-# GFX12: v_add_nc_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x4a,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x4a,0x01,0x77,0x39,0x05
+# GFX12: v_add_nc_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x4a,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_nc_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x4b,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x4b,0xff,0x00,0x00,0x00
+# GFX12: v_add_nc_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x4b,0xff,0x00,0x00,0x00]
 
-# GFX12: v_and_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x36,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x36,0x01,0x77,0x39,0x05
+# GFX12: v_and_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x36,0x01,0x77,0x39,0x05]
 
-# GFX12: v_and_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x37,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x37,0xff,0x00,0x00,0x00
+# GFX12: v_and_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x37,0xff,0x00,0x00,0x00]
 
-# GFX12: v_ashrrev_i32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x34,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x34,0x01,0x77,0x39,0x05
+# GFX12: v_ashrrev_i32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x34,0x01,0x77,0x39,0x05]
 
-# GFX12: v_ashrrev_i32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x35,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x35,0xff,0x00,0x00,0x00
+# GFX12: v_ashrrev_i32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x35,0xff,0x00,0x00,0x00]
 
+0xe9,0x04,0x0a,0x02,0x01,0x77,0x39,0x05
 # W32: v_cndmask_b32_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x02,0x01,0x77,0x39,0x05]
 # W64: v_cndmask_b32_dpp v5, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x02,0x01,0x77,0x39,0x05]
-0xe9,0x04,0x0a,0x02,0x01,0x77,0x39,0x05
 
+0xea,0xfe,0xff,0x03,0xff,0x00,0x00,0x00
 # W32: v_cndmask_b32_dpp v255, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x03,0xff,0x00,0x00,0x00]
 # W64: v_cndmask_b32_dpp v255, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x03,0xff,0x00,0x00,0x00]
-0xea,0xfe,0xff,0x03,0xff,0x00,0x00,0x00
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x5e,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x5e,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x5e,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_rtz_f16_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x5f,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x5f,0xff,0x00,0x00,0x00
+# GFX12: v_cvt_pk_rtz_f16_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x5f,0xff,0x00,0x00,0x00]
 
-# GFX12: v_fmac_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05
+# GFX12: v_fmac_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fmac_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x6c,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x6c,0x7f,0x00,0x00,0x00
+# GFX12: v_fmac_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x6c,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_fmac_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x56,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x56,0x01,0x77,0x39,0x05
+# GFX12: v_fmac_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x56,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fmac_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x57,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x57,0xff,0x00,0x00,0x00
+# GFX12: v_fmac_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x57,0xff,0x00,0x00,0x00]
 
-# GFX12: v_ldexp_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x76,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x76,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x76,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_ldexp_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x76,0x01,0x77,0x39,0x05]
 
-# GFX12: v_ldexp_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x76,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x76,0x7f,0x00,0x00,0x00
+# GFX12-REAL16: v_ldexp_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x76,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_ldexp_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x76,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_lshlrev_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x30,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x30,0x01,0x77,0x39,0x05
+# GFX12: v_lshlrev_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x30,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshlrev_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x31,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x31,0xff,0x00,0x00,0x00
+# GFX12: v_lshlrev_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x31,0xff,0x00,0x00,0x00]
 
-# GFX12: v_lshrrev_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x32,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x32,0x01,0x77,0x39,0x05
+# GFX12: v_lshrrev_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x32,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshrrev_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x33,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x33,0xff,0x00,0x00,0x00
+# GFX12: v_lshrrev_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x33,0xff,0x00,0x00,0x00]
 
-# GFX12: v_max_num_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x62,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x62,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_max_num_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x62,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_max_num_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x62,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max_num_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x62,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x62,0x7f,0x00,0x00,0x00
+# GFX12-REAL16: v_max_num_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x62,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_max_num_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x62,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_max_num_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x2c,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x2c,0x01,0x77,0x39,0x05
+# GFX12: v_max_num_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x2c,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max_num_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x2d,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x2d,0xff,0x00,0x00,0x00
+# GFX12: v_max_num_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x2d,0xff,0x00,0x00,0x00]
 
-# GFX12: v_max_i32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x24,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x24,0x01,0x77,0x39,0x05
+# GFX12: v_max_i32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x24,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max_i32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x25,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x25,0xff,0x00,0x00,0x00
+# GFX12: v_max_i32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x25,0xff,0x00,0x00,0x00]
 
-# GFX12: v_max_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x28,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x28,0x01,0x77,0x39,0x05
+# GFX12: v_max_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x28,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x29,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x29,0xff,0x00,0x00,0x00
+# GFX12: v_max_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x29,0xff,0x00,0x00,0x00]
 
-# GFX12: v_min_num_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x60,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x60,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_min_num_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x60,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_min_num_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x60,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min_num_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x60,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x60,0x7f,0x00,0x00,0x00
+# GFX12-REAL16: v_min_num_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x60,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_min_num_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x60,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_min_num_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x2a,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x2a,0x01,0x77,0x39,0x05
+# GFX12: v_min_num_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x2a,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min_num_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x2b,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x2b,0xff,0x00,0x00,0x00
+# GFX12: v_min_num_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x2b,0xff,0x00,0x00,0x00]
 
-# GFX12: v_min_i32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x22,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x22,0x01,0x77,0x39,0x05
+# GFX12: v_min_i32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x22,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min_i32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x23,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x23,0xff,0x00,0x00,0x00
+# GFX12: v_min_i32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x23,0xff,0x00,0x00,0x00]
 
-# GFX12: v_min_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x26,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x26,0x01,0x77,0x39,0x05
+# GFX12: v_min_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x26,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x27,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x27,0xff,0x00,0x00,0x00
+# GFX12: v_min_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x27,0xff,0x00,0x00,0x00]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x0e,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x0e,0x01,0x77,0x39,0x05
+# GFX12: v_mul_dx9_zero_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x0e,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mul_dx9_zero_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x0f,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x0f,0xff,0x00,0x00,0x00
+# GFX12: v_mul_dx9_zero_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x0f,0xff,0x00,0x00,0x00]
 
-# GFX12: v_mul_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x6a,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x6a,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x6a,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_mul_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x6a,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mul_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x6a,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x6a,0x7f,0x00,0x00,0x00
+# GFX12-REAL16: v_mul_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x6a,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_mul_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x6a,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_mul_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x10,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x10,0x01,0x77,0x39,0x05
+# GFX12: v_mul_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x10,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mul_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x11,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x11,0xff,0x00,0x00,0x00
+# GFX12: v_mul_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x11,0xff,0x00,0x00,0x00]
 
-# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x14,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x14,0x01,0x77,0x39,0x05
+# GFX12: v_mul_hi_i32_i24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x14,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mul_hi_i32_i24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x15,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x15,0xff,0x00,0x00,0x00
+# GFX12: v_mul_hi_i32_i24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x15,0xff,0x00,0x00,0x00]
 
-# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x18,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x18,0x01,0x77,0x39,0x05
+# GFX12: v_mul_hi_u32_u24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x18,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mul_hi_u32_u24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x19,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x19,0xff,0x00,0x00,0x00
+# GFX12: v_mul_hi_u32_u24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x19,0xff,0x00,0x00,0x00]
 
-# GFX12: v_mul_i32_i24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x12,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x12,0x01,0x77,0x39,0x05
+# GFX12: v_mul_i32_i24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x12,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mul_i32_i24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x13,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x13,0xff,0x00,0x00,0x00
+# GFX12: v_mul_i32_i24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x13,0xff,0x00,0x00,0x00]
 
-# GFX12: v_mul_u32_u24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x16,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x16,0x01,0x77,0x39,0x05
+# GFX12: v_mul_u32_u24_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x16,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mul_u32_u24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x17,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x17,0xff,0x00,0x00,0x00
+# GFX12: v_mul_u32_u24_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x17,0xff,0x00,0x00,0x00]
 
-# GFX12: v_or_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x38,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x38,0x01,0x77,0x39,0x05
+# GFX12: v_or_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x38,0x01,0x77,0x39,0x05]
 
-# GFX12: v_or_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x39,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x39,0xff,0x00,0x00,0x00
+# GFX12: v_or_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x39,0xff,0x00,0x00,0x00]
 
+0xe9,0x04,0x0a,0x42,0x01,0x77,0x39,0x05
 # W32: v_sub_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x42,0x01,0x77,0x39,0x05]
 # W64: v_sub_co_ci_u32_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x42,0x01,0x77,0x39,0x05]
-0xe9,0x04,0x0a,0x42,0x01,0x77,0x39,0x05
 
+0xea,0xfe,0xff,0x43,0xff,0x00,0x00,0x00
 # W32: v_sub_co_ci_u32_dpp v255, vcc_lo, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x43,0xff,0x00,0x00,0x00]
 # W64: v_sub_co_ci_u32_dpp v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x43,0xff,0x00,0x00,0x00]
-0xea,0xfe,0xff,0x43,0xff,0x00,0x00,0x00
 
-# GFX12: v_sub_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_sub_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sub_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00
+# GFX12-REAL16: v_sub_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_sub_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_sub_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x08,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x08,0x01,0x77,0x39,0x05
+# GFX12: v_sub_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x08,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sub_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x09,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x09,0xff,0x00,0x00,0x00
+# GFX12: v_sub_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x09,0xff,0x00,0x00,0x00]
 
-# GFX12: v_sub_nc_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x4c,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x4c,0x01,0x77,0x39,0x05
+# GFX12: v_sub_nc_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x4c,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sub_nc_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x4d,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x4d,0xff,0x00,0x00,0x00
+# GFX12: v_sub_nc_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x4d,0xff,0x00,0x00,0x00]
 
+0xe9,0x04,0x0a,0x44,0x01,0x77,0x39,0x05
 # W32: v_subrev_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x44,0x01,0x77,0x39,0x05]
 # W64: v_subrev_co_ci_u32_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x44,0x01,0x77,0x39,0x05]
-0xe9,0x04,0x0a,0x44,0x01,0x77,0x39,0x05
 
+0xea,0xfe,0xff,0x45,0xff,0x00,0x00,0x00
 # W32: v_subrev_co_ci_u32_dpp v255, vcc_lo, v255, v255, vcc_lo dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x45,0xff,0x00,0x00,0x00]
 # W64: v_subrev_co_ci_u32_dpp v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x45,0xff,0x00,0x00,0x00]
-0xea,0xfe,0xff,0x45,0xff,0x00,0x00,0x00
 
-# GFX12: v_subrev_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x68,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x68,0x01,0x77,0x39,0x05
+# GFX12-REAL16: v_subrev_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x68,0x01,0x77,0x39,0x05]
+# GFX12-FAKE16: v_subrev_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x68,0x01,0x77,0x39,0x05]
 
-# GFX12: v_subrev_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x68,0x7f,0x00,0x00,0x00]
 0xea,0xfe,0xfe,0x68,0x7f,0x00,0x00,0x00
+# GFX12-REAL16: v_subrev_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x68,0x7f,0x00,0x00,0x00]
+# GFX12-FAKE16: v_subrev_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x68,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_subrev_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x0a,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x0a,0x01,0x77,0x39,0x05
+# GFX12: v_subrev_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x0a,0x01,0x77,0x39,0x05]
 
-# GFX12: v_subrev_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x0b,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x0b,0xff,0x00,0x00,0x00
+# GFX12: v_subrev_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x0b,0xff,0x00,0x00,0x00]
 
-# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x4e,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x4e,0x01,0x77,0x39,0x05
+# GFX12: v_subrev_nc_u32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x4e,0x01,0x77,0x39,0x05]
 
-# GFX12: v_subrev_nc_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x4f,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x4f,0xff,0x00,0x00,0x00
+# GFX12: v_subrev_nc_u32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x4f,0xff,0x00,0x00,0x00]
 
-# GFX12: v_xnor_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x3c,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x3c,0x01,0x77,0x39,0x05
+# GFX12: v_xnor_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x3c,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xnor_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x3d,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x3d,0xff,0x00,0x00,0x00
+# GFX12: v_xnor_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x3d,0xff,0x00,0x00,0x00]
 
-# GFX12: v_xor_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x3a,0x01,0x77,0x39,0x05]
 0xe9,0x04,0x0a,0x3a,0x01,0x77,0x39,0x05
+# GFX12: v_xor_b32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x3a,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xor_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x3b,0xff,0x00,0x00,0x00]
 0xea,0xfe,0xff,0x3b,0xff,0x00,0x00,0x00
+# GFX12: v_xor_b32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x3b,0xff,0x00,0x00,0x00]
-- 
GitLab


From 311c0772f9e67a694f3038ab63ea4ec981ce6a9a Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Wed, 30 Oct 2024 16:50:54 +0000
Subject: [PATCH 609/693] [AMDGPU] Fix test failures after #114232 and #114200

---
 .../AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll     | 12 ++++++++----
 .../AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll     | 12 ++++++++----
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll
index c202476d85ba..ae309f3a614d 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll
@@ -671,7 +671,9 @@ define amdgpu_cs_chain void @nonuniform_callee(ptr %callee, i32 inreg %sgpr, i32
   ; GISEL-GFX11-NEXT:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
   ; GISEL-GFX11-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr0
   ; GISEL-GFX11-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr10
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY3]]
+  ; GISEL-GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY4]]
   ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY2]], 0, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $vgpr8
   ;
@@ -685,10 +687,12 @@ define amdgpu_cs_chain void @nonuniform_callee(ptr %callee, i32 inreg %sgpr, i32
   ; GISEL-GFX10-NEXT:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
   ; GISEL-GFX10-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr0
   ; GISEL-GFX10-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr10
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY3]]
+  ; GISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY4]]
-  ; GISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY5]]
+  ; GISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY6]]
   ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 [[COPY2]], 0, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $vgpr8, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: nonuniform_callee
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll
index a456f549174c..90707e823c14 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll
@@ -671,7 +671,9 @@ define amdgpu_cs_chain void @nonuniform_callee(ptr %callee, i32 inreg %sgpr, i32
   ; GISEL-GFX11-NEXT:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
   ; GISEL-GFX11-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr0
   ; GISEL-GFX11-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr10
-  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY3]]
+  ; GISEL-GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; GISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
   ; GISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY4]]
   ; GISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY2]], 0, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $vgpr8
   ;
@@ -685,10 +687,12 @@ define amdgpu_cs_chain void @nonuniform_callee(ptr %callee, i32 inreg %sgpr, i32
   ; GISEL-GFX10-NEXT:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[REG_SEQUENCE]]
   ; GISEL-GFX10-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr0
   ; GISEL-GFX10-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr10
-  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY3]]
+  ; GISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; GISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
   ; GISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY4]]
-  ; GISEL-GFX10-NEXT:   [[COPY5:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY5]]
+  ; GISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; GISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY6]]
   ; GISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 [[COPY2]], 0, 0, -1, amdgpu_allvgprs, implicit $sgpr0, implicit $vgpr8, implicit $sgpr48_sgpr49_sgpr50_sgpr51
   ;
   ; DAGISEL-GFX11-LABEL: name: nonuniform_callee
-- 
GitLab


From 463a4c16ea9c1a3c1210d0ac39e56a75b43b5a8d Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Wed, 30 Oct 2024 16:53:08 +0000
Subject: [PATCH 610/693] [clang] Remove some uses of
 llvm::StructType::setBody. NFC. (#113691)

It is simple to create the struct body up front, now that we have
transitioned to opaque pointers.
---
 clang/lib/CodeGen/CGBlocks.cpp        |  8 ++--
 clang/lib/CodeGen/CGObjCGNU.cpp       |  4 +-
 clang/lib/CodeGen/CGObjCMac.cpp       | 57 +++++++++++++--------------
 clang/lib/CodeGen/MicrosoftCXXABI.cpp | 12 +++---
 4 files changed, 37 insertions(+), 44 deletions(-)

diff --git a/clang/lib/CodeGen/CGBlocks.cpp b/clang/lib/CodeGen/CGBlocks.cpp
index 41bb8d19d161..bfa9b0a2f836 100644
--- a/clang/lib/CodeGen/CGBlocks.cpp
+++ b/clang/lib/CodeGen/CGBlocks.cpp
@@ -2590,10 +2590,6 @@ const BlockByrefInfo &CodeGenFunction::getBlockByrefInfo(const VarDecl *D) {
   if (it != BlockByrefInfos.end())
     return it->second;
 
-  llvm::StructType *byrefType =
-    llvm::StructType::create(getLLVMContext(),
-                             "struct.__block_byref_" + D->getNameAsString());
-
   QualType Ty = D->getType();
 
   CharUnits size;
@@ -2658,7 +2654,9 @@ const BlockByrefInfo &CodeGenFunction::getBlockByrefInfo(const VarDecl *D) {
   }
   types.push_back(varTy);
 
-  byrefType->setBody(types, packed);
+  llvm::StructType *byrefType = llvm::StructType::create(
+      getLLVMContext(), types, "struct.__block_byref_" + D->getNameAsString(),
+      packed);
 
   BlockByrefInfo info;
   info.Type = byrefType;
diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp
index 7a07284f8a8a..d6f5f2a43cf5 100644
--- a/clang/lib/CodeGen/CGObjCGNU.cpp
+++ b/clang/lib/CodeGen/CGObjCGNU.cpp
@@ -1509,8 +1509,8 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
   GetSectionBounds(StringRef Section) {
     if (CGM.getTriple().isOSBinFormatCOFF()) {
       if (emptyStruct == nullptr) {
-        emptyStruct = llvm::StructType::create(VMContext, ".objc_section_sentinel");
-        emptyStruct->setBody({}, /*isPacked*/true);
+        emptyStruct = llvm::StructType::create(
+            VMContext, {}, ".objc_section_sentinel", /*isPacked=*/true);
       }
       auto ZeroInit = llvm::Constant::getNullValue(emptyStruct);
       auto Sym = [&](StringRef Prefix, StringRef SecSuffix) {
diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp
index 1c16d273a553..47ea636c7564 100644
--- a/clang/lib/CodeGen/CGObjCMac.cpp
+++ b/clang/lib/CodeGen/CGObjCMac.cpp
@@ -5835,15 +5835,7 @@ ObjCTypesHelper::ObjCTypesHelper(CodeGen::CodeGenModule &cgm)
   // struct _objc_protocol_extension *
   ProtocolExtensionPtrTy = llvm::PointerType::getUnqual(ProtocolExtensionTy);
 
-  // Handle recursive construction of Protocol and ProtocolList types
-
-  ProtocolTy =
-    llvm::StructType::create(VMContext, "struct._objc_protocol");
-
-  ProtocolListTy =
-    llvm::StructType::create(VMContext, "struct._objc_protocol_list");
-  ProtocolListTy->setBody(llvm::PointerType::getUnqual(ProtocolListTy), LongTy,
-                          llvm::ArrayType::get(ProtocolTy, 0));
+  // Handle construction of Protocol and ProtocolList types
 
   // struct _objc_protocol {
   //   struct _objc_protocol_extension *isa;
@@ -5852,9 +5844,16 @@ ObjCTypesHelper::ObjCTypesHelper(CodeGen::CodeGenModule &cgm)
   //   struct _objc_method_description_list *instance_methods;
   //   struct _objc_method_description_list *class_methods;
   // }
-  ProtocolTy->setBody(ProtocolExtensionPtrTy, Int8PtrTy,
-                      llvm::PointerType::getUnqual(ProtocolListTy),
-                      MethodDescriptionListPtrTy, MethodDescriptionListPtrTy);
+  ProtocolTy = llvm::StructType::create(
+      {ProtocolExtensionPtrTy, Int8PtrTy,
+       llvm::PointerType::getUnqual(VMContext), MethodDescriptionListPtrTy,
+       MethodDescriptionListPtrTy},
+      "struct._objc_protocol");
+
+  ProtocolListTy =
+      llvm::StructType::create({llvm::PointerType::getUnqual(VMContext), LongTy,
+                                llvm::ArrayType::get(ProtocolTy, 0)},
+                               "struct._objc_protocol_list");
 
   // struct _objc_protocol_list *
   ProtocolListPtrTy = llvm::PointerType::getUnqual(ProtocolListTy);
@@ -5886,8 +5885,6 @@ ObjCTypesHelper::ObjCTypesHelper(CodeGen::CodeGenModule &cgm)
       "struct._objc_class_extension", IntTy, Int8PtrTy, PropertyListPtrTy);
   ClassExtensionPtrTy = llvm::PointerType::getUnqual(ClassExtensionTy);
 
-  ClassTy = llvm::StructType::create(VMContext, "struct._objc_class");
-
   // struct _objc_class {
   //   Class isa;
   //   Class super_class;
@@ -5902,10 +5899,12 @@ ObjCTypesHelper::ObjCTypesHelper(CodeGen::CodeGenModule &cgm)
   //   char *ivar_layout;
   //   struct _objc_class_ext *ext;
   // };
-  ClassTy->setBody(llvm::PointerType::getUnqual(ClassTy),
-                   llvm::PointerType::getUnqual(ClassTy), Int8PtrTy, LongTy,
-                   LongTy, LongTy, IvarListPtrTy, MethodListPtrTy, CachePtrTy,
-                   ProtocolListPtrTy, Int8PtrTy, ClassExtensionPtrTy);
+  ClassTy = llvm::StructType::create(
+      {llvm::PointerType::getUnqual(VMContext),
+       llvm::PointerType::getUnqual(VMContext), Int8PtrTy, LongTy, LongTy,
+       LongTy, IvarListPtrTy, MethodListPtrTy, CachePtrTy, ProtocolListPtrTy,
+       Int8PtrTy, ClassExtensionPtrTy},
+      "struct._objc_class");
 
   ClassPtrTy = llvm::PointerType::getUnqual(ClassTy);
 
@@ -5988,13 +5987,9 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul
   //   const struct _prop_list_t * class_properties;
   // }
 
-  // Holder for struct _protocol_list_t *
-  ProtocolListnfABITy =
-    llvm::StructType::create(VMContext, "struct._objc_protocol_list");
-
   ProtocolnfABITy = llvm::StructType::create(
       "struct._protocol_t", ObjectPtrTy, Int8PtrTy,
-      llvm::PointerType::getUnqual(ProtocolListnfABITy), MethodListnfABIPtrTy,
+      llvm::PointerType::getUnqual(VMContext), MethodListnfABIPtrTy,
       MethodListnfABIPtrTy, MethodListnfABIPtrTy, MethodListnfABIPtrTy,
       PropertyListPtrTy, IntTy, IntTy, Int8PtrPtrTy, Int8PtrTy,
       PropertyListPtrTy);
@@ -6006,8 +6001,9 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul
   //   long protocol_count;   // Note, this is 32/64 bit
   //   struct _protocol_t *[protocol_count];
   // }
-  ProtocolListnfABITy->setBody(LongTy,
-                               llvm::ArrayType::get(ProtocolnfABIPtrTy, 0));
+  ProtocolListnfABITy = llvm::StructType::create(
+      {LongTy, llvm::ArrayType::get(ProtocolnfABIPtrTy, 0)},
+      "struct._objc_protocol_list");
 
   // struct _objc_protocol_list*
   ProtocolListnfABIPtrTy = llvm::PointerType::getUnqual(ProtocolListnfABITy);
@@ -6067,11 +6063,12 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul
   //   struct class_ro_t *ro;
   // }
 
-  ClassnfABITy = llvm::StructType::create(VMContext, "struct._class_t");
-  ClassnfABITy->setBody(llvm::PointerType::getUnqual(ClassnfABITy),
-                        llvm::PointerType::getUnqual(ClassnfABITy), CachePtrTy,
-                        llvm::PointerType::getUnqual(ImpnfABITy),
-                        llvm::PointerType::getUnqual(ClassRonfABITy));
+  ClassnfABITy = llvm::StructType::create(
+      {llvm::PointerType::getUnqual(VMContext),
+       llvm::PointerType::getUnqual(VMContext), CachePtrTy,
+       llvm::PointerType::getUnqual(ImpnfABITy),
+       llvm::PointerType::getUnqual(ClassRonfABITy)},
+      "struct._class_t");
 
   // LLVM for struct _class_t *
   ClassnfABIPtrTy = llvm::PointerType::getUnqual(ClassnfABITy);
diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index 0b0b45ffead9..3802dc8bcafc 100644
--- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -529,31 +529,29 @@ public:
     if (ClassHierarchyDescriptorType)
       return ClassHierarchyDescriptorType;
     // Forward-declare RTTIClassHierarchyDescriptor to break a cycle.
-    ClassHierarchyDescriptorType = llvm::StructType::create(
-        CGM.getLLVMContext(), "rtti.ClassHierarchyDescriptor");
     llvm::Type *FieldTypes[] = {CGM.IntTy, CGM.IntTy, CGM.IntTy,
                                 getImageRelativeType(CGM.UnqualPtrTy)};
-    ClassHierarchyDescriptorType->setBody(FieldTypes);
+    ClassHierarchyDescriptorType =
+        llvm::StructType::create(FieldTypes, "rtti.ClassHierarchyDescriptor");
     return ClassHierarchyDescriptorType;
   }
 
   llvm::StructType *getCompleteObjectLocatorType() {
     if (CompleteObjectLocatorType)
       return CompleteObjectLocatorType;
-    CompleteObjectLocatorType = llvm::StructType::create(
-        CGM.getLLVMContext(), "rtti.CompleteObjectLocator");
     llvm::Type *FieldTypes[] = {
         CGM.IntTy,
         CGM.IntTy,
         CGM.IntTy,
         getImageRelativeType(CGM.Int8PtrTy),
         getImageRelativeType(CGM.UnqualPtrTy),
-        getImageRelativeType(CompleteObjectLocatorType),
+        getImageRelativeType(CGM.VoidTy),
     };
     llvm::ArrayRef<llvm::Type *> FieldTypesRef(FieldTypes);
     if (!isImageRelative())
       FieldTypesRef = FieldTypesRef.drop_back();
-    CompleteObjectLocatorType->setBody(FieldTypesRef);
+    CompleteObjectLocatorType =
+        llvm::StructType::create(FieldTypesRef, "rtti.CompleteObjectLocator");
     return CompleteObjectLocatorType;
   }
 
-- 
GitLab


From cc2d8e7616762710b284aa9af44a297b633b270a Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Wed, 30 Oct 2024 16:54:09 +0000
Subject: [PATCH 611/693] [AArch64] Add assembly/disassembly of FMOP4{A,S}
 (non-widening) single-precision instructions (#113344)

The new instructions are described in
https://developer.arm.com/documentation/ddi0602/2024-09/SME-Instructions
---
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |   3 +
 llvm/lib/Target/AArch64/SMEInstrFormats.td    |  37 +++
 .../fmop4as-fp32-non-widening-diagnostics.s   | 245 ++++++++++++++++++
 .../SME2p2/fmop4as-fp32-non-widening.s        | 179 +++++++++++++
 4 files changed, 464 insertions(+)
 create mode 100644 llvm/test/MC/AArch64/SME2p2/fmop4as-fp32-non-widening-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SME2p2/fmop4as-fp32-non-widening.s

diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index e78cd7146df2..e7389b533354 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -1017,6 +1017,9 @@ let Predicates = [HasSME2p2] in {
   defm FMUL_2Z2Z : sme2_multi2_fmul_mm< "fmul">;
   defm FMUL_4ZZ  : sme2_multi4_fmul_sm<"fmul">;
   defm FMUL_4Z4Z : sme2_multi4_fmul_mm< "fmul">;
+
+  defm FMOP4A : sme2_fmop4as_fp32_non_widening<0, "fmop4a">;
+  defm FMOP4S : sme2_fmop4as_fp32_non_widening<1, "fmop4s">;
 } // [HasSME2p2]
 
 let Predicates = [HasSME2p2, HasSMEB16B16] in {
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index b31bea712a76..2740ac814f9c 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -5454,3 +5454,40 @@ multiclass sme2_bfmop4as_non_widening<bit S, string mnemonic> {
   // Multiple vectors
   def _M2Z2Z_H : sme2_bf16_fp16_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi>;
 }
+
+class sme2_fp32_quarter_tile_outer_product<bit M, bit N, bit S, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>
+    : I<(outs TileOp32:$ZAda),
+        (ins TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),
+        mnemonic, "\t$ZAda, $Zn, $Zm",
+        "", []>, Sched<[]> {
+  bits<2> ZAda;
+  bits<3> Zn;
+  bits<3> Zm;
+
+  let Inst{31-21} = 0b10000000000;
+  let Inst{20} = M;
+  let Inst{19-17} = Zm;
+  let Inst{16-10} = 0b0000000;
+  let Inst{9} = N;
+  let Inst{8-6} = Zn;
+  let Inst{5} = 0;
+  let Inst{4} = S;
+  let Inst{3-2} = 0b00;
+  let Inst{1-0} = ZAda;
+
+  let Constraints = "$ZAda = $_ZAda";
+}
+
+multiclass sme2_fmop4as_fp32_non_widening<bit S, string mnemonic> {
+  // Single vectors
+  def _MZZ_S : sme2_fp32_quarter_tile_outer_product<0, 0, S, mnemonic, ZPR32Mul2_Lo, ZPR32Mul2_Hi>;
+
+  // Multiple and single vectors
+  def _M2ZZ_S : sme2_fp32_quarter_tile_outer_product<0, 1, S, mnemonic, ZZ_s_mul_r_Lo, ZPR32Mul2_Hi>;
+
+  // Single and multiple vectors
+  def _MZ2Z_S : sme2_fp32_quarter_tile_outer_product<1, 0, S, mnemonic, ZPR32Mul2_Lo, ZZ_s_mul_r_Hi>;
+
+  // Multiple vectors
+  def _M2Z2Z_S : sme2_fp32_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_s_mul_r_Lo, ZZ_s_mul_r_Hi>;
+}
diff --git a/llvm/test/MC/AArch64/SME2p2/fmop4as-fp32-non-widening-diagnostics.s b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp32-non-widening-diagnostics.s
new file mode 100644
index 000000000000..c9c59128f420
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp32-non-widening-diagnostics.s
@@ -0,0 +1,245 @@
+// RUN: not llvm-mc -triple=aarch64 -mattr=+sme2p2 < %s 2>&1 | FileCheck %s
+
+// FMOP4A
+
+// Single vectors
+
+fmop4a za0.d, z0.s, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za4.s, z0.s, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, z0.d, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4a za0.s, z15.s, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4a za0.s, z16.s, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4a za0.s, z0.s, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.s..z30.s
+
+fmop4a za0.s, z12.s, z17.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.s..z30.s
+
+fmop4a za0.s, z12.s, z14.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.s..z30.s
+
+fmop4a za0.s, z12.s, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.s..z30.s
+
+// Single and multiple vectors
+
+fmop4a za0.d, z0.s, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za4.s, z0.s, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, z0.d, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4a za0.s, z1.s, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4a za0.s, z16.s, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4a za0.s, z0.s, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, z0.s, {z17.s-z18.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, z0.s, {z16.s-z18.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, z0.s, {z12.s-z13.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+// Multiple and single vectors
+
+fmop4a za0.d, {z0.s-z1.s}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za4.s, {z0.s-z1.s}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z0.d-z1.s}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: mismatched register size suffix
+
+fmop4a za0.s, {z1.s-z2.s}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, {z2.s-z4.s}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z16.s-z17.s}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, {z0.s-z1.s}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.s..z30.s
+
+fmop4a za0.s, {z0.s-z1.s}, z17.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.s..z30.s
+
+fmop4a za0.s, {z0.s-z1.s}, z12.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.s..z30.s
+
+// Multiple vectors
+
+fmop4a za0.d, {z0.s-z1.s}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za4.s, {z0.s-z1.s}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z0.d-z1.d}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z1.s-z2.s}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, {z2.s-z4.s}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z18.s-z19.s}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, {z0.s-z1.s}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z0.s-z1.s}, {z19.s-z20.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, {z0.s-z1.s}, {z16.s-z18.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z0.s-z1.s}, {z10.s-z11.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+
+// FMOP4S
+
+// Single vectors
+
+fmop4s za0.d, z0.s, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4s za4.s, z0.s, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, z0.d, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4s za0.s, z15.s, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4s za0.s, z16.s, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4s za0.s, z0.s, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.s..z30.s
+
+fmop4s za0.s, z12.s, z17.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.s..z30.s
+
+fmop4s za0.s, z12.s, z14.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.s..z30.s
+
+fmop4s za0.s, z12.s, z31.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.s..z30.s
+
+// Single and multiple vectors
+
+fmop4s za0.d, z0.s, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4s za4.s, z0.s, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, z0.d, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4s za0.s, z1.s, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4s za0.s, z16.s, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4s za0.s, z0.s, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, z0.s, {z17.s-z18.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.s, z0.s, {z16.s-z18.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, z0.s, {z12.s-z13.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+// Multiple and single vectors
+
+fmop4s za0.d, {z0.s-z1.s}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4s za4.s, {z0.s-z1.s}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, {z0.d-z1.s}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: mismatched register size suffix
+
+fmop4s za0.s, {z1.s-z2.s}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.s, {z2.s-z4.s}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, {z16.s-z17.s}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.s, {z0.s-z1.s}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.s..z30.s
+
+fmop4s za0.s, {z0.s-z1.s}, z17.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.s..z30.s
+
+fmop4s za0.s, {z0.s-z1.s}, z12.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.s..z30.s
+
+// Multiple vectors
+
+fmop4s za0.d, {z0.s-z1.s}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4s za4.s, {z0.s-z1.s}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, {z0.d-z1.d}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, {z1.s-z2.s}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.s, {z2.s-z4.s}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, {z18.s-z19.s}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.s, {z0.s-z1.s}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, {z0.s-z1.s}, {z19.s-z20.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.s, {z0.s-z1.s}, {z16.s-z18.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, {z0.s-z1.s}, {z10.s-z11.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
diff --git a/llvm/test/MC/AArch64/SME2p2/fmop4as-fp32-non-widening.s b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp32-non-widening.s
new file mode 100644
index 000000000000..e65def17cd1b
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp32-non-widening.s
@@ -0,0 +1,179 @@
+
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sme2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2p2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+// FMOP4A
+
+// Single vectors
+
+fmop4a  za0.s, z0.s, z16.s  // 10000000-00000000-00000000-00000000
+// CHECK-INST: fmop4a  za0.s, z0.s, z16.s
+// CHECK-ENCODING: [0x00,0x00,0x00,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80000000 <unknown>
+
+fmop4a  za3.s, z12.s, z24.s  // 10000000-00001000-00000001-10000011
+// CHECK-INST: fmop4a  za3.s, z12.s, z24.s
+// CHECK-ENCODING: [0x83,0x01,0x08,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80080183 <unknown>
+
+fmop4a  za3.s, z14.s, z30.s  // 10000000-00001110-00000001-11000011
+// CHECK-INST: fmop4a  za3.s, z14.s, z30.s
+// CHECK-ENCODING: [0xc3,0x01,0x0e,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 800e01c3 <unknown>
+
+// Single and multiple vectors
+
+fmop4a  za0.s, z0.s, {z16.s-z17.s}  // 10000000-00010000-00000000-00000000
+// CHECK-INST: fmop4a  za0.s, z0.s, { z16.s, z17.s }
+// CHECK-ENCODING: [0x00,0x00,0x10,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80100000 <unknown>
+
+fmop4a  za1.s, z10.s, {z20.s-z21.s}  // 10000000-00010100-00000001-01000001
+// CHECK-INST: fmop4a  za1.s, z10.s, { z20.s, z21.s }
+// CHECK-ENCODING: [0x41,0x01,0x14,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80140141 <unknown>
+
+fmop4a  za3.s, z14.s, {z30.s-z31.s}  // 10000000-00011110-00000001-11000011
+// CHECK-INST: fmop4a  za3.s, z14.s, { z30.s, z31.s }
+// CHECK-ENCODING: [0xc3,0x01,0x1e,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 801e01c3 <unknown>
+
+// Multiple and single vectors
+
+fmop4a  za0.s, {z0.s-z1.s}, z16.s  // 10000000-00000000-00000010-00000000
+// CHECK-INST: fmop4a  za0.s, { z0.s, z1.s }, z16.s
+// CHECK-ENCODING: [0x00,0x02,0x00,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80000200 <unknown>
+
+fmop4a  za1.s, {z10.s-z11.s}, z20.s  // 10000000-00000100-00000011-01000001
+// CHECK-INST: fmop4a  za1.s, { z10.s, z11.s }, z20.s
+// CHECK-ENCODING: [0x41,0x03,0x04,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80040341 <unknown>
+
+fmop4a  za3.s, {z14.s-z15.s}, z30.s  // 10000000-00001110-00000011-11000011
+// CHECK-INST: fmop4a  za3.s, { z14.s, z15.s }, z30.s
+// CHECK-ENCODING: [0xc3,0x03,0x0e,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 800e03c3 <unknown>
+
+// Multiple vectors
+
+fmop4a  za0.s, {z0.s-z1.s}, {z16.s-z17.s}  // 10000000-00010000-00000010-00000000
+// CHECK-INST: fmop4a  za0.s, { z0.s, z1.s }, { z16.s, z17.s }
+// CHECK-ENCODING: [0x00,0x02,0x10,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80100200 <unknown>
+
+fmop4a  za1.s, {z10.s-z11.s}, {z20.s-z21.s}  // 10000000-00010100-00000011-01000001
+// CHECK-INST: fmop4a  za1.s, { z10.s, z11.s }, { z20.s, z21.s }
+// CHECK-ENCODING: [0x41,0x03,0x14,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80140341 <unknown>
+
+fmop4a  za3.s, {z14.s-z15.s}, {z30.s-z31.s}  // 10000000-00011110-00000011-11000011
+// CHECK-INST: fmop4a  za3.s, { z14.s, z15.s }, { z30.s, z31.s }
+// CHECK-ENCODING: [0xc3,0x03,0x1e,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 801e03c3 <unknown>
+
+// FMOP4S
+
+// Single vectors
+
+fmop4s  za0.s, z0.s, z16.s  // 10000000-00000000-00000000-00010000
+// CHECK-INST: fmop4s  za0.s, z0.s, z16.s
+// CHECK-ENCODING: [0x10,0x00,0x00,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80000010 <unknown>
+
+fmop4s  za3.s, z12.s, z24.s  // 10000000-00001000-00000001-10010011
+// CHECK-INST: fmop4s  za3.s, z12.s, z24.s
+// CHECK-ENCODING: [0x93,0x01,0x08,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80080193 <unknown>
+
+fmop4s  za3.s, z14.s, z30.s  // 10000000-00001110-00000001-11010011
+// CHECK-INST: fmop4s  za3.s, z14.s, z30.s
+// CHECK-ENCODING: [0xd3,0x01,0x0e,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 800e01d3 <unknown>
+
+// Single and multiple vectors
+
+fmop4s  za0.s, z0.s, {z16.s-z17.s}  // 10000000-00010000-00000000-00010000
+// CHECK-INST: fmop4s  za0.s, z0.s, { z16.s, z17.s }
+// CHECK-ENCODING: [0x10,0x00,0x10,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80100010 <unknown>
+
+fmop4s  za1.s, z10.s, {z20.s-z21.s}  // 10000000-00010100-00000001-01010001
+// CHECK-INST: fmop4s  za1.s, z10.s, { z20.s, z21.s }
+// CHECK-ENCODING: [0x51,0x01,0x14,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80140151 <unknown>
+
+fmop4s  za3.s, z14.s, {z30.s-z31.s}  // 10000000-00011110-00000001-11010011
+// CHECK-INST: fmop4s  za3.s, z14.s, { z30.s, z31.s }
+// CHECK-ENCODING: [0xd3,0x01,0x1e,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 801e01d3 <unknown>
+
+// Multiple and single vectors
+
+fmop4s  za0.s, {z0.s-z1.s}, z16.s  // 10000000-00000000-00000010-00010000
+// CHECK-INST: fmop4s  za0.s, { z0.s, z1.s }, z16.s
+// CHECK-ENCODING: [0x10,0x02,0x00,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80000210 <unknown>
+
+fmop4s  za1.s, {z10.s-z11.s}, z20.s  // 10000000-00000100-00000011-01010001
+// CHECK-INST: fmop4s  za1.s, { z10.s, z11.s }, z20.s
+// CHECK-ENCODING: [0x51,0x03,0x04,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80040351 <unknown>
+
+fmop4s  za3.s, {z14.s-z15.s}, z30.s  // 10000000-00001110-00000011-11010011
+// CHECK-INST: fmop4s  za3.s, { z14.s, z15.s }, z30.s
+// CHECK-ENCODING: [0xd3,0x03,0x0e,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 800e03d3 <unknown>
+
+// Multiple vectors
+
+fmop4s  za0.s, {z0.s-z1.s}, {z16.s-z17.s}  // 10000000-00010000-00000010-00010000
+// CHECK-INST: fmop4s  za0.s, { z0.s, z1.s }, { z16.s, z17.s }
+// CHECK-ENCODING: [0x10,0x02,0x10,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80100210 <unknown>
+
+fmop4s  za1.s, {z10.s-z11.s}, {z20.s-z21.s}  // 10000000-00010100-00000011-01010001
+// CHECK-INST: fmop4s  za1.s, { z10.s, z11.s }, { z20.s, z21.s }
+// CHECK-ENCODING: [0x51,0x03,0x14,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 80140351 <unknown>
+
+fmop4s  za3.s, {z14.s-z15.s}, {z30.s-z31.s}  // 10000000-00011110-00000011-11010011
+// CHECK-INST: fmop4s  za3.s, { z14.s, z15.s }, { z30.s, z31.s }
+// CHECK-ENCODING: [0xd3,0x03,0x1e,0x80]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 801e03d3 <unknown>
-- 
GitLab


From d7e6cba030f34162ea45aef1dc18f708b5d2ec70 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 30 Oct 2024 09:55:14 -0700
Subject: [PATCH 612/693] [RISCV] Use bit or bits for some tablegen class
 arguments. NFC

These eventually end up in TSFlags so we should use the same types.
---
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    | 86 +++++++++----------
 1 file changed, 43 insertions(+), 43 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 6ffdae1d7df2..5554fda760eb 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -877,7 +877,7 @@ class VPseudoILoadNoMask<VReg RetClass,
                          bits<3> LMUL,
                          bit Ordered,
                          bit EarlyClobber,
-                         int TargetConstraintType = 1> :
+                         bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$dest, GPRMem:$rs1, IdxClass:$rs2, AVL:$vl,
                   sew:$sew, vec_policy:$policy), []>,
@@ -899,7 +899,7 @@ class VPseudoILoadMask<VReg RetClass,
                        bits<3> LMUL,
                        bit Ordered,
                        bit EarlyClobber,
-                       int TargetConstraintType = 1> :
+                       bits<2> TargetConstraintType = 1> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   GPRMem:$rs1, IdxClass:$rs2,
@@ -1021,7 +1021,7 @@ class VPseudoNullaryPseudoM<string BaseInst> :
 class VPseudoUnaryNoMask<DAGOperand RetClass,
                          DAGOperand OpClass,
                          string Constraint = "",
-                         int TargetConstraintType = 1> :
+                         bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru, OpClass:$rs2,
                   AVL:$vl, sew:$sew, vec_policy:$policy), []>,
@@ -1039,7 +1039,7 @@ class VPseudoUnaryNoMask<DAGOperand RetClass,
 class VPseudoUnaryNoMaskNoPolicy<DAGOperand RetClass,
                                  DAGOperand OpClass,
                                  string Constraint = "",
-                                 int TargetConstraintType = 1> :
+                                 bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins OpClass:$rs2, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
@@ -1055,7 +1055,7 @@ class VPseudoUnaryNoMaskNoPolicy<DAGOperand RetClass,
 class VPseudoUnaryNoMaskRoundingMode<DAGOperand RetClass,
                                      DAGOperand OpClass,
                                      string Constraint = "",
-                                     int TargetConstraintType = 1> :
+                                     bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru, OpClass:$rs2, ixlenimm:$rm,
                   AVL:$vl, sew:$sew, vec_policy:$policy), []>,
@@ -1075,7 +1075,7 @@ class VPseudoUnaryNoMaskRoundingMode<DAGOperand RetClass,
 class VPseudoUnaryMask<VReg RetClass,
                        VReg OpClass,
                        string Constraint = "",
-                       int TargetConstraintType = 1> :
+                       bits<2> TargetConstraintType = 1> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru, OpClass:$rs2,
                   VMaskOp:$vm, AVL:$vl, sew:$sew, vec_policy:$policy), []>,
@@ -1094,7 +1094,7 @@ class VPseudoUnaryMask<VReg RetClass,
 class VPseudoUnaryMaskRoundingMode<VReg RetClass,
                                    VReg OpClass,
                                    string Constraint = "",
-                                   int TargetConstraintType = 1> :
+                                   bits<2> TargetConstraintType = 1> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru, OpClass:$rs2,
                   VMaskOp:$vm, ixlenimm:$rm,
@@ -1133,7 +1133,7 @@ class VPseudoUnaryMask_NoExcept<VReg RetClass,
 class VPseudoUnaryNoMask_FRM<VReg RetClass,
                              VReg OpClass,
                              string Constraint = "",
-                             int TargetConstraintType = 1> :
+                             bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru, OpClass:$rs2, ixlenimm:$frm,
                   AVL:$vl, sew:$sew, vec_policy:$policy), []>,
@@ -1152,7 +1152,7 @@ class VPseudoUnaryNoMask_FRM<VReg RetClass,
 class VPseudoUnaryMask_FRM<VReg RetClass,
                            VReg OpClass,
                            string Constraint = "",
-                           int TargetConstraintType = 1> :
+                           bits<2> TargetConstraintType = 1> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru, OpClass:$rs2,
                   VMaskOp:$vm, ixlenimm:$frm,
@@ -1211,7 +1211,7 @@ class VPseudoBinaryNoMask<VReg RetClass,
                           VReg Op1Class,
                           DAGOperand Op2Class,
                           string Constraint,
-                          int TargetConstraintType = 1> :
+                          bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins Op1Class:$rs2, Op2Class:$rs1, AVL:$vl, sew:$sew), []>,
       RISCVVPseudo {
@@ -1228,7 +1228,7 @@ class VPseudoBinaryNoMaskPolicy<VReg RetClass,
                                 VReg Op1Class,
                                 DAGOperand Op2Class,
                                 string Constraint,
-                                int TargetConstraintType = 1> :
+                                bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru, Op1Class:$rs2, Op2Class:$rs1, AVL:$vl,
                   sew:$sew, vec_policy:$policy), []>,
@@ -1247,8 +1247,8 @@ class VPseudoBinaryNoMaskRoundingMode<VReg RetClass,
                                       VReg Op1Class,
                                       DAGOperand Op2Class,
                                       string Constraint,
-                                      int UsesVXRM_ = 1,
-                                      int TargetConstraintType = 1> :
+                                      bit UsesVXRM_ = 1,
+                                      bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru, Op1Class:$rs2, Op2Class:$rs1, ixlenimm:$rm,
                   AVL:$vl, sew:$sew, vec_policy:$policy), []>,
@@ -1268,8 +1268,8 @@ class VPseudoBinaryMaskPolicyRoundingMode<VReg RetClass,
                                           RegisterClass Op1Class,
                                           DAGOperand Op2Class,
                                           string Constraint,
-                                          int UsesVXRM_,
-                                          int TargetConstraintType = 1> :
+                                          bit UsesVXRM_,
+                                          bits<2> TargetConstraintType = 1> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   Op1Class:$rs2, Op2Class:$rs1,
@@ -1294,7 +1294,7 @@ class VPseudoBinaryMaskPolicyRoundingMode<VReg RetClass,
 class VPseudoTiedBinaryNoMask<VReg RetClass,
                               DAGOperand Op2Class,
                               string Constraint,
-                              int TargetConstraintType = 1> :
+                              bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$rs2, Op2Class:$rs1, AVL:$vl, sew:$sew,
                   vec_policy:$policy), []>,
@@ -1314,7 +1314,7 @@ class VPseudoTiedBinaryNoMask<VReg RetClass,
 class VPseudoTiedBinaryNoMaskRoundingMode<VReg RetClass,
                                           DAGOperand Op2Class,
                                           string Constraint,
-                                          int TargetConstraintType = 1> :
+                                          bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$rs2, Op2Class:$rs1,
                   ixlenimm:$rm,
@@ -1367,7 +1367,7 @@ class VPseudoBinaryMaskPolicy<VReg RetClass,
                               RegisterClass Op1Class,
                               DAGOperand Op2Class,
                               string Constraint,
-                              int TargetConstraintType = 1> :
+                              bits<2> TargetConstraintType = 1> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   Op1Class:$rs2, Op2Class:$rs1,
@@ -1427,7 +1427,7 @@ class VPseudoBinaryMOutMask<VReg RetClass,
                             RegisterClass Op1Class,
                             DAGOperand Op2Class,
                             string Constraint,
-                            int TargetConstraintType = 1> :
+                            bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru,
                   Op1Class:$rs2, Op2Class:$rs1,
@@ -1449,7 +1449,7 @@ class VPseudoBinaryMOutMask<VReg RetClass,
 class VPseudoTiedBinaryMask<VReg RetClass,
                             DAGOperand Op2Class,
                             string Constraint,
-                            int TargetConstraintType = 1> :
+                            bits<2> TargetConstraintType = 1> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   Op2Class:$rs1,
@@ -1470,7 +1470,7 @@ class VPseudoTiedBinaryMask<VReg RetClass,
 class VPseudoTiedBinaryMaskRoundingMode<VReg RetClass,
                                         DAGOperand Op2Class,
                                         string Constraint,
-                                        int TargetConstraintType = 1> :
+                                        bits<2> TargetConstraintType = 1> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   Op2Class:$rs1,
@@ -1498,7 +1498,7 @@ class VPseudoBinaryCarry<VReg RetClass,
                          LMULInfo MInfo,
                          bit CarryIn,
                          string Constraint,
-                         int TargetConstraintType = 1> :
+                         bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              !if(CarryIn,
                 (ins Op1Class:$rs2, Op2Class:$rs1,
@@ -1520,7 +1520,7 @@ class VPseudoTiedBinaryCarryIn<VReg RetClass,
                                VReg Op1Class,
                                DAGOperand Op2Class,
                                LMULInfo MInfo,
-                               int TargetConstraintType = 1> :
+                               bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$passthru, Op1Class:$rs2, Op2Class:$rs1,
                   VMV0:$carry, AVL:$vl, sew:$sew), []>,
@@ -1556,7 +1556,7 @@ class VPseudoTernaryNoMaskWithPolicy<VReg RetClass,
                                      RegisterClass Op1Class,
                                      DAGOperand Op2Class,
                                      string Constraint = "",
-                                     int TargetConstraintType = 1> :
+                                     bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$rs3, Op1Class:$rs1, Op2Class:$rs2,
                   AVL:$vl, sew:$sew, vec_policy:$policy), []>,
@@ -1575,7 +1575,7 @@ class VPseudoTernaryNoMaskWithPolicyRoundingMode<VReg RetClass,
                                                  RegisterClass Op1Class,
                                                  DAGOperand Op2Class,
                                                  string Constraint = "",
-                                                 int TargetConstraintType = 1> :
+                                                 bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$rs3, Op1Class:$rs1, Op2Class:$rs2,
                   ixlenimm:$rm, AVL:$vl, sew:$sew, vec_policy:$policy), []>,
@@ -2105,7 +2105,7 @@ multiclass VPseudoBinary<VReg RetClass,
                          LMULInfo MInfo,
                          string Constraint = "",
                          int sew = 0,
-                         int TargetConstraintType = 1,
+                         bits<2> TargetConstraintType = 1,
                          bit Commutable = 0> {
   let VLMul = MInfo.value, SEW=sew, isCommutable = Commutable in {
     defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
@@ -2123,8 +2123,8 @@ multiclass VPseudoBinaryRoundingMode<VReg RetClass,
                                      LMULInfo MInfo,
                                      string Constraint = "",
                                      int sew = 0,
-                                     int UsesVXRM = 1,
-                                     int TargetConstraintType = 1,
+                                     bit UsesVXRM = 1,
+                                     bits<2> TargetConstraintType = 1,
                                      bit Commutable = 0> {
   let VLMul = MInfo.value, SEW=sew, isCommutable = Commutable in {
     defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
@@ -2147,7 +2147,7 @@ multiclass VPseudoBinaryM<VReg RetClass,
                           DAGOperand Op2Class,
                           LMULInfo MInfo,
                           string Constraint = "",
-                          int TargetConstraintType = 1,
+                          bits<2> TargetConstraintType = 1,
                           bit Commutable = 0> {
   let VLMul = MInfo.value, isCommutable = Commutable in {
     def "_" # MInfo.MX : VPseudoBinaryNoMask<RetClass, Op1Class, Op2Class,
@@ -2180,7 +2180,7 @@ multiclass VPseudoTiedBinary<VReg RetClass,
                              DAGOperand Op2Class,
                              LMULInfo MInfo,
                              string Constraint = "",
-                             int TargetConstraintType = 1> {
+                             bits<2> TargetConstraintType = 1> {
   let VLMul = MInfo.value in {
     def "_" # MInfo.MX # "_TIED": VPseudoTiedBinaryNoMask<RetClass, Op2Class,
                                                           Constraint, TargetConstraintType>;
@@ -2195,7 +2195,7 @@ multiclass VPseudoTiedBinaryRoundingMode<VReg RetClass,
                                          LMULInfo MInfo,
                                          string Constraint = "",
                                          int sew = 0,
-                                         int TargetConstraintType = 1> {
+                                         bits<2> TargetConstraintType = 1> {
     defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
     let VLMul = MInfo.value in {
     def suffix # "_TIED":
@@ -2417,7 +2417,7 @@ multiclass VPseudoBinaryV_WI_RM<LMULInfo m> {
 multiclass VPseudoBinaryV_VM<LMULInfo m, bit CarryOut = 0, bit CarryIn = 1,
                              string Constraint = "",
                              bit Commutable = 0,
-                             int TargetConstraintType = 1> {
+                             bits<2> TargetConstraintType = 1> {
   let isCommutable = Commutable in
   def "_VV" # !if(CarryIn, "M", "") # "_" # m.MX :
     VPseudoBinaryCarry<!if(CarryOut, VR,
@@ -2434,7 +2434,7 @@ multiclass VPseudoTiedBinaryV_VM<LMULInfo m, bit Commutable = 0> {
 }
 
 multiclass VPseudoBinaryV_XM<LMULInfo m, bit CarryOut = 0, bit CarryIn = 1,
-                             string Constraint = "", int TargetConstraintType = 1> {
+                             string Constraint = "", bits<2> TargetConstraintType = 1> {
   def "_VX" # !if(CarryIn, "M", "") # "_" # m.MX :
     VPseudoBinaryCarry<!if(CarryOut, VR,
                        !if(!and(CarryIn, !not(CarryOut)),
@@ -2462,7 +2462,7 @@ multiclass VPseudoVMRG_FM {
 }
 
 multiclass VPseudoBinaryV_IM<LMULInfo m, bit CarryOut = 0, bit CarryIn = 1,
-                             string Constraint = "", int TargetConstraintType = 1> {
+                             string Constraint = "", bits<2> TargetConstraintType = 1> {
   def "_VI" # !if(CarryIn, "M", "") # "_" # m.MX :
     VPseudoBinaryCarry<!if(CarryOut, VR,
                        !if(!and(CarryIn, !not(CarryOut)),
@@ -2640,26 +2640,26 @@ multiclass PseudoVEXT_VF8 {
 //  lowest-numbered part of the source register group".
 // With LMUL<=1 the source and dest occupy a single register so any overlap
 // is in the lowest-numbered part.
-multiclass VPseudoBinaryM_VV<LMULInfo m, int TargetConstraintType = 1,
+multiclass VPseudoBinaryM_VV<LMULInfo m, bits<2> TargetConstraintType = 1,
                              bit Commutable = 0> {
   defm _VV : VPseudoBinaryM<VR, m.vrclass, m.vrclass, m,
                             !if(!ge(m.octuple, 16), "@earlyclobber $rd", ""),
                             TargetConstraintType, Commutable=Commutable>;
 }
 
-multiclass VPseudoBinaryM_VX<LMULInfo m, int TargetConstraintType = 1> {
+multiclass VPseudoBinaryM_VX<LMULInfo m, bits<2> TargetConstraintType = 1> {
   defm "_VX" :
     VPseudoBinaryM<VR, m.vrclass, GPR, m,
                    !if(!ge(m.octuple, 16), "@earlyclobber $rd", ""), TargetConstraintType>;
 }
 
-multiclass VPseudoBinaryM_VF<LMULInfo m, FPR_Info f, int TargetConstraintType = 1> {
+multiclass VPseudoBinaryM_VF<LMULInfo m, FPR_Info f, bits<2> TargetConstraintType = 1> {
   defm "_V" # f.FX :
     VPseudoBinaryM<VR, m.vrclass, f.fprclass, m,
                    !if(!ge(m.octuple, 16), "@earlyclobber $rd", ""), TargetConstraintType>;
 }
 
-multiclass VPseudoBinaryM_VI<LMULInfo m, int TargetConstraintType = 1> {
+multiclass VPseudoBinaryM_VI<LMULInfo m, bits<2> TargetConstraintType = 1> {
   defm _VI : VPseudoBinaryM<VR, m.vrclass, simm5, m,
                             !if(!ge(m.octuple, 16), "@earlyclobber $rd", ""), TargetConstraintType>;
 }
@@ -3202,7 +3202,7 @@ multiclass VPseudoTernaryWithPolicy<VReg RetClass,
                                     LMULInfo MInfo,
                                     string Constraint = "",
                                     bit Commutable = 0,
-                                    int TargetConstraintType = 1> {
+                                    bits<2> TargetConstraintType = 1> {
   let VLMul = MInfo.value in {
     let isCommutable = Commutable in
     def "_" # MInfo.MX : VPseudoTernaryNoMaskWithPolicy<RetClass, Op1Class, Op2Class, Constraint, TargetConstraintType>;
@@ -3218,7 +3218,7 @@ multiclass VPseudoTernaryWithPolicyRoundingMode<VReg RetClass,
                                                 string Constraint = "",
                                                 int sew = 0,
                                                 bit Commutable = 0,
-                                                int TargetConstraintType = 1> {
+                                                bits<2> TargetConstraintType = 1> {
   let VLMul = MInfo.value in {
     defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
     let isCommutable = Commutable in
@@ -3548,7 +3548,7 @@ multiclass VPseudoConversion<VReg RetClass,
                              LMULInfo MInfo,
                              string Constraint = "",
                              int sew = 0,
-                             int TargetConstraintType = 1> {
+                             bits<2> TargetConstraintType = 1> {
   defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
   let VLMul = MInfo.value, SEW=sew in {
     def suffix : VPseudoUnaryNoMask<RetClass, Op1Class, Constraint, TargetConstraintType>;
@@ -3563,7 +3563,7 @@ multiclass VPseudoConversionRoundingMode<VReg RetClass,
                              LMULInfo MInfo,
                              string Constraint = "",
                              int sew = 0,
-                             int TargetConstraintType = 1> {
+                             bits<2> TargetConstraintType = 1> {
   let VLMul = MInfo.value, SEW=sew in {
     defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
     def suffix : VPseudoUnaryNoMaskRoundingMode<RetClass, Op1Class, Constraint, TargetConstraintType>;
@@ -3580,7 +3580,7 @@ multiclass VPseudoConversionRM<VReg RetClass,
                                LMULInfo MInfo,
                                string Constraint = "",
                                int sew = 0,
-                               int TargetConstraintType = 1> {
+                               bits<2> TargetConstraintType = 1> {
   let VLMul = MInfo.value, SEW=sew in {
     defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
     def suffix : VPseudoUnaryNoMask_FRM<RetClass, Op1Class,
-- 
GitLab


From cf6ca98481a90728f1d3adc748aa0f271afc59aa Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Wed, 30 Oct 2024 17:03:35 +0000
Subject: [PATCH 613/693] [Clang] Add and use mangleVendorType helper. NFC.
 (#108970)

---
 clang/lib/AST/ItaniumMangle.cpp | 52 +++++++++++++++------------------
 1 file changed, 24 insertions(+), 28 deletions(-)

diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index b3e46508cf59..14bc260d0245 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -468,6 +468,7 @@ public:
   void mangleLambdaSig(const CXXRecordDecl *Lambda);
   void mangleModuleNamePrefix(StringRef Name, bool IsPartition = false);
   void mangleVendorQualifier(StringRef Name);
+  void mangleVendorType(StringRef Name);
 
 private:
 
@@ -2891,6 +2892,10 @@ void CXXNameMangler::mangleVendorQualifier(StringRef name) {
   Out << 'U' << name.size() << name;
 }
 
+void CXXNameMangler::mangleVendorType(StringRef name) {
+  Out << 'u' << name.size() << name;
+}
+
 void CXXNameMangler::mangleRefQualifier(RefQualifierKind RefQualifier) {
   // <ref-qualifier> ::= R                # lvalue reference
   //                 ::= O                # rvalue-reference
@@ -3413,8 +3418,7 @@ void CXXNameMangler::mangleType(const BuiltinType *T) {
     if (T->getKind() == BuiltinType::SveBFloat16 &&                            \
         isCompatibleWith(LangOptions::ClangABI::Ver17)) {                      \
       /* Prior to Clang 18.0 we used this incorrect mangled name */            \
-      type_name = "__SVBFloat16_t";                                            \
-      Out << "u" << type_name.size() << type_name;                             \
+      mangleVendorType("__SVBFloat16_t");                                      \
     } else {                                                                   \
       type_name = MangledName;                                                 \
       Out << (type_name == Name ? "u" : "") << type_name.size() << type_name;  \
@@ -3436,35 +3440,30 @@ void CXXNameMangler::mangleType(const BuiltinType *T) {
     Out << (type_name == Name ? "u" : "") << type_name.size() << type_name;    \
     break;
 #include "clang/Basic/AArch64SVEACLETypes.def"
-#define PPC_VECTOR_TYPE(Name, Id, Size) \
-  case BuiltinType::Id: \
-    type_name = #Name; \
-    Out << 'u' << type_name.size() << type_name; \
+#define PPC_VECTOR_TYPE(Name, Id, Size)                                        \
+  case BuiltinType::Id:                                                        \
+    mangleVendorType(#Name);                                                   \
     break;
 #include "clang/Basic/PPCTypes.def"
     // TODO: Check the mangling scheme for RISC-V V.
 #define RVV_TYPE(Name, Id, SingletonId)                                        \
   case BuiltinType::Id:                                                        \
-    type_name = Name;                                                          \
-    Out << 'u' << type_name.size() << type_name;                               \
+    mangleVendorType(Name);                                                    \
     break;
 #include "clang/Basic/RISCVVTypes.def"
 #define WASM_REF_TYPE(InternalName, MangledName, Id, SingletonId, AS)          \
   case BuiltinType::Id:                                                        \
-    type_name = MangledName;                                                   \
-    Out << 'u' << type_name.size() << type_name;                               \
+    mangleVendorType(MangledName);                                             \
     break;
 #include "clang/Basic/WebAssemblyReferenceTypes.def"
 #define AMDGPU_TYPE(Name, Id, SingletonId, Width, Align)                       \
   case BuiltinType::Id:                                                        \
-    type_name = Name;                                                          \
-    Out << 'u' << type_name.size() << type_name;                               \
+    mangleVendorType(Name);                                                    \
     break;
 #include "clang/Basic/AMDGPUTypes.def"
 #define HLSL_INTANGIBLE_TYPE(Name, Id, SingletonId)                            \
   case BuiltinType::Id:                                                        \
-    type_name = #Name;                                                         \
-    Out << 'u' << type_name.size() << type_name;                               \
+    mangleVendorType(#Name);                                                   \
     break;
 #include "clang/Basic/HLSLIntangibleTypes.def"
   }
@@ -4035,8 +4034,9 @@ void CXXNameMangler::mangleAArch64FixedSveVectorType(const VectorType *T) {
   if (T->getVectorKind() == VectorKind::SveFixedLengthPredicate)
     VecSizeInBits *= 8;
 
-  Out << "9__SVE_VLSI" << 'u' << TypeName.size() << TypeName << "Lj"
-      << VecSizeInBits << "EE";
+  Out << "9__SVE_VLSI";
+  mangleVendorType(TypeName);
+  Out << "Lj" << VecSizeInBits << "EE";
 }
 
 void CXXNameMangler::mangleAArch64FixedSveVectorType(
@@ -4136,8 +4136,9 @@ void CXXNameMangler::mangleRISCVFixedRVVVectorType(const VectorType *T) {
   }
   TypeNameOS << "_t";
 
-  Out << "9__RVV_VLSI" << 'u' << TypeNameStr.size() << TypeNameStr << "Lj"
-      << VecSizeInBits << "EE";
+  Out << "9__RVV_VLSI";
+  mangleVendorType(TypeNameStr);
+  Out << "Lj" << VecSizeInBits << "EE";
 }
 
 void CXXNameMangler::mangleRISCVFixedRVVVectorType(
@@ -4236,8 +4237,7 @@ void CXXNameMangler::mangleType(const ConstantMatrixType *T) {
   // Mangle matrix types as a vendor extended type:
   // u<Len>matrix_typeI<Rows><Columns><element type>E
 
-  StringRef VendorQualifier = "matrix_type";
-  Out << "u" << VendorQualifier.size() << VendorQualifier;
+  mangleVendorType("matrix_type");
 
   Out << "I";
   auto &ASTCtx = getASTContext();
@@ -4255,8 +4255,7 @@ void CXXNameMangler::mangleType(const ConstantMatrixType *T) {
 void CXXNameMangler::mangleType(const DependentSizedMatrixType *T) {
   // Mangle matrix types as a vendor extended type:
   // u<Len>matrix_typeI<row expr><column expr><element type>E
-  StringRef VendorQualifier = "matrix_type";
-  Out << "u" << VendorQualifier.size() << VendorQualifier;
+  mangleVendorType("matrix_type");
 
   Out << "I";
   mangleTemplateArgExpr(T->getRowExpr());
@@ -4302,7 +4301,7 @@ void CXXNameMangler::mangleType(const ObjCObjectType *T) {
       StringRef name = I->getName();
       QualOS << name.size() << name;
     }
-    Out << 'U' << QualStr.size() << QualStr;
+    mangleVendorQualifier(QualStr);
   }
 
   mangleType(T->getBaseType());
@@ -4436,8 +4435,6 @@ void CXXNameMangler::mangleType(const UnaryTransformType *T) {
   // If this is dependent, we need to record that. If not, we simply
   // mangle it as the underlying type since they are equivalent.
   if (T->isDependentType()) {
-    Out << "u";
-
     StringRef BuiltinName;
     switch (T->getUTTKind()) {
 #define TRANSFORM_TYPE_TRAIT_DEF(Enum, Trait)                                  \
@@ -4446,7 +4443,7 @@ void CXXNameMangler::mangleType(const UnaryTransformType *T) {
     break;
 #include "clang/Basic/TransformTypeTraits.def"
     }
-    Out << BuiltinName.size() << BuiltinName;
+    mangleVendorType(BuiltinName);
   }
 
   Out << "I";
@@ -5311,9 +5308,8 @@ recurse:
     //  <expression> ::= u <source-name> <template-arg>* E # vendor extension
     const TypeTraitExpr *TTE = cast<TypeTraitExpr>(E);
     NotPrimaryExpr();
-    Out << 'u';
     llvm::StringRef Spelling = getTraitSpelling(TTE->getTrait());
-    Out << Spelling.size() << Spelling;
+    mangleVendorType(Spelling);
     for (TypeSourceInfo *TSI : TTE->getArgs()) {
       mangleType(TSI->getType());
     }
-- 
GitLab


From 0c8e12fc64073a889956e790881cdf0d58018372 Mon Sep 17 00:00:00 2001
From: Roland McGrath <mcgrathr@google.com>
Date: Wed, 30 Oct 2024 10:09:34 -0700
Subject: [PATCH 614/693] [libc] Fix fexcept_t type to match canonical ABI and
 API (#113666)

In glibc and musl, fexcept_t is unsigned short int on x86 and
unsigned int on other machines that llvm-libc supports.  Match
that ABI (only different from before on x86) and API (different
everywhere as it was previously signed).
---
 libc/include/llvm-libc-types/fexcept_t.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/libc/include/llvm-libc-types/fexcept_t.h b/libc/include/llvm-libc-types/fexcept_t.h
index 60687bd1318a..5aa09fbbaffc 100644
--- a/libc/include/llvm-libc-types/fexcept_t.h
+++ b/libc/include/llvm-libc-types/fexcept_t.h
@@ -9,6 +9,10 @@
 #ifndef LLVM_LIBC_TYPES_FEXCEPT_T_H
 #define LLVM_LIBC_TYPES_FEXCEPT_T_H
 
-typedef int fexcept_t;
+#if defined(__x86_64__) || defined(__i386__)
+typedef unsigned short int fexcept_t;
+#else
+typedef unsigned int fexcept_t;
+#endif
 
 #endif // LLVM_LIBC_TYPES_FEXCEPT_T_H
-- 
GitLab


From 5192cb772ad58af4b557539791ff8de60ab450a3 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Wed, 30 Oct 2024 16:55:40 +0000
Subject: [PATCH 615/693] [AArch64] Add hidden option to enable subreg liveness
 tracking.

Subreg liveness tracking is disabled by default for now until all issues
are ironed out. This option allows the feature to be used in tests.
---
 llvm/lib/Target/AArch64/AArch64Subtarget.cpp | 9 +++++++++
 llvm/lib/Target/AArch64/AArch64Subtarget.h   | 3 +++
 2 files changed, 12 insertions(+)

diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
index 7fb2a961e031..736d57e6ae2f 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -86,6 +86,13 @@ static cl::alias AArch64StreamingStackHazardSize(
     cl::desc("alias for -aarch64-streaming-hazard-size"),
     cl::aliasopt(AArch64StreamingHazardSize));
 
+// Subreg liveness tracking is disabled by default for now until all issues
+// are ironed out. This option allows the feature to be used in tests.
+static cl::opt<bool>
+    EnableSubregLivenessTracking("aarch64-enable-subreg-liveness-tracking",
+                                 cl::init(false), cl::Hidden,
+                                 cl::desc("Enable subreg liveness tracking"));
+
 unsigned AArch64Subtarget::getVectorInsertExtractBaseCost() const {
   if (OverrideVectorInsertExtractBaseCost.getNumOccurrences() > 0)
     return OverrideVectorInsertExtractBaseCost;
@@ -380,6 +387,8 @@ AArch64Subtarget::AArch64Subtarget(const Triple &TT, StringRef CPU,
     ReserveXRegisterForRA.set(29);
 
   AddressCheckPSV.reset(new AddressCheckPseudoSourceValue(TM));
+
+  EnableSubregLiveness = EnableSubregLivenessTracking.getValue();
 }
 
 const CallLowering *AArch64Subtarget::getCallLowering() const {
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index 50adb7cbf69a..f3dcce3f3994 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -90,6 +90,8 @@ protected:
   unsigned VScaleForTuning = 2;
   TailFoldingOpts DefaultSVETFOpts = TailFoldingOpts::Disabled;
 
+  bool EnableSubregLiveness;
+
   /// TargetTriple - What processor and OS we're targeting.
   Triple TargetTriple;
 
@@ -153,6 +155,7 @@ public:
   const Triple &getTargetTriple() const { return TargetTriple; }
   bool enableMachineScheduler() const override { return true; }
   bool enablePostRAScheduler() const override { return usePostRAScheduler(); }
+  bool enableSubRegLiveness() const override { return EnableSubregLiveness; }
 
   bool enableMachinePipeliner() const override;
   bool useDFAforSMS() const override { return false; }
-- 
GitLab


From 70d35fbdb6c01e2ccd76ce5c5fe7610ab77d0ea1 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Wed, 30 Oct 2024 10:25:26 +1100
Subject: [PATCH 616/693] [ORC] Fix include guard names. NFC.

---
 .../ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
index ef42cc5f798f..8a4740c1dd9c 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
@@ -10,8 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_EXECUTIONENGINE_ORC_JITLINKREDIRECABLEMANAGER_H
-#define LLVM_EXECUTIONENGINE_ORC_JITLINKREDIRECABLEMANAGER_H
+#ifndef LLVM_EXECUTIONENGINE_ORC_JITLINKREDIRECABLESYMBOLMANAGER_H
+#define LLVM_EXECUTIONENGINE_ORC_JITLINKREDIRECABLESYMBOLMANAGER_H
 
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/RedirectionManager.h"
@@ -103,4 +103,4 @@ private:
 } // namespace orc
 } // namespace llvm
 
-#endif
+#endif // LLVM_EXECUTIONENGINE_ORC_JITLINKREDIRECABLESYMBOLMANAGER_H
-- 
GitLab


From b94762d5a7fbf883707c4018dbf43d7525a06e12 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Wed, 30 Oct 2024 10:26:26 +1100
Subject: [PATCH 617/693] [ORC] Add comment on include guard #endif

---
 llvm/include/llvm/ExecutionEngine/Orc/ReOptimizeLayer.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ReOptimizeLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/ReOptimizeLayer.h
index 4adc3efad557..cd185d54b2e7 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ReOptimizeLayer.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ReOptimizeLayer.h
@@ -178,4 +178,4 @@ private:
 } // namespace orc
 } // namespace llvm
 
-#endif
+#endif // LLVM_EXECUTIONENGINE_ORC_REOPTIMIZELAYER_H
-- 
GitLab


From feb2d867fac3b6339c169fff97ddf0716fce6f0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kenji=20Mouri=20/=20=E6=AF=9B=E5=88=A9=20=E7=A0=94?=
 =?UTF-8?q?=E4=BA=8C?= <Mouri_Naruto@Outlook.com>
Date: Thu, 31 Oct 2024 01:34:32 +0800
Subject: [PATCH 618/693] [TLI] Add support for hypot libcall. (#113724)

This patch adds basic support for `hypot`. Constant folding support will
be submitted in a subsequent patch.

Related issue: https://github.com/llvm/llvm-project/issues/113711

Note: It's my first time contributing to the LLVM with encouragement
from one of my friends, @fawdlstty. I learned a lot from
https://github.com/llvm/llvm-project/pull/99611, and thanks for that.

Kenji Mouri
---
 llvm/include/llvm/Analysis/TargetLibraryInfo.def  | 15 +++++++++++++++
 llvm/lib/Analysis/TargetLibraryInfo.cpp           |  2 ++
 llvm/lib/Transforms/Utils/BuildLibCalls.cpp       |  3 +++
 .../Transforms/InferFunctionAttrs/annotate.ll     |  9 +++++++++
 .../tools/llvm-tli-checker/ps4-tli-check.yaml     | 12 ++++++++++++
 llvm/unittests/Analysis/TargetLibraryInfoTest.cpp |  3 +++
 6 files changed, 44 insertions(+)

diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.def b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
index 3e23e398f6a7..fd53a26ef8fc 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.def
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
@@ -1671,6 +1671,21 @@ TLI_DEFINE_ENUM_INTERNAL(htons)
 TLI_DEFINE_STRING_INTERNAL("htons")
 TLI_DEFINE_SIG_INTERNAL(Int16, Int16)
 
+/// double hypot(double x, double y);
+TLI_DEFINE_ENUM_INTERNAL(hypot)
+TLI_DEFINE_STRING_INTERNAL("hypot")
+TLI_DEFINE_SIG_INTERNAL(Dbl, Dbl, Dbl)
+
+/// float hypotf(float x, float y);
+TLI_DEFINE_ENUM_INTERNAL(hypotf)
+TLI_DEFINE_STRING_INTERNAL("hypotf")
+TLI_DEFINE_SIG_INTERNAL(Flt, Flt, Flt)
+
+/// long double hypotl(long double x, long double y);
+TLI_DEFINE_ENUM_INTERNAL(hypotl)
+TLI_DEFINE_STRING_INTERNAL("hypotl")
+TLI_DEFINE_SIG_INTERNAL(LDbl, LDbl, LDbl)
+
 /// int iprintf(const char *format, ...);
 TLI_DEFINE_ENUM_INTERNAL(iprintf)
 TLI_DEFINE_STRING_INTERNAL("iprintf")
diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index 0ee83d217a50..7f0b98ab3c15 100644
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -300,6 +300,7 @@ static void initializeLibCalls(TargetLibraryInfoImpl &TLI, const Triple &T,
       TLI.setUnavailable(LibFunc_expf);
       TLI.setUnavailable(LibFunc_floorf);
       TLI.setUnavailable(LibFunc_fmodf);
+      TLI.setUnavailable(LibFunc_hypotf);
       TLI.setUnavailable(LibFunc_log10f);
       TLI.setUnavailable(LibFunc_logf);
       TLI.setUnavailable(LibFunc_modff);
@@ -331,6 +332,7 @@ static void initializeLibCalls(TargetLibraryInfoImpl &TLI, const Triple &T,
     TLI.setUnavailable(LibFunc_floorl);
     TLI.setUnavailable(LibFunc_fmodl);
     TLI.setUnavailable(LibFunc_frexpl);
+    TLI.setUnavailable(LibFunc_hypotl);
     TLI.setUnavailable(LibFunc_ldexpl);
     TLI.setUnavailable(LibFunc_log10l);
     TLI.setUnavailable(LibFunc_logl);
diff --git a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
index 5fd4fd78c28a..e039457f313b 100644
--- a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -1215,6 +1215,9 @@ bool llvm::inferNonMandatoryLibFuncAttrs(Function &F,
   case LibFunc_fmod:
   case LibFunc_fmodf:
   case LibFunc_fmodl:
+  case LibFunc_hypot:
+  case LibFunc_hypotf:
+  case LibFunc_hypotl:
   case LibFunc_isascii:
   case LibFunc_isdigit:
   case LibFunc_labs:
diff --git a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
index d8266f4c6703..452d90aa98d8 100644
--- a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
+++ b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
@@ -589,6 +589,15 @@ declare ptr @gets(ptr)
 ; CHECK: declare noundef i32 @gettimeofday(ptr nocapture noundef, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
 declare i32 @gettimeofday(ptr, ptr)
 
+; CHECK: declare double @hypot(double, double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+declare double @hypot(double, double)
+
+; CHECK: declare float @hypotf(float, float) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+declare float @hypotf(float, float)
+
+; CHECK: declare x86_fp80 @hypotl(x86_fp80, x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
+declare x86_fp80 @hypotl(x86_fp80, x86_fp80)
+
 ; CHECK: declare i32 @isascii(i32) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare i32 @isascii(i32)
 
diff --git a/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml b/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
index 408b9c399342..d52f3c751b06 100644
--- a/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
+++ b/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
@@ -602,6 +602,18 @@ DynamicSymbols:
     Type:            STT_FUNC
     Section:         .text
     Binding:         STB_GLOBAL
+  - Name:            hypot
+    Type:            STT_FUNC
+    Section:         .text
+    Binding:         STB_GLOBAL
+  - Name:            hypotf
+    Type:            STT_FUNC
+    Section:         .text
+    Binding:         STB_GLOBAL
+  - Name:            hypotl
+    Type:            STT_FUNC
+    Section:         .text
+    Binding:         STB_GLOBAL
   - Name:            isdigit
     Type:            STT_FUNC
     Section:         .text
diff --git a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
index 98f8989d4e6e..982d00c5d335 100644
--- a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
+++ b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
@@ -249,6 +249,9 @@ TEST_F(TargetLibraryInfoTest, ValidProto) {
       "declare %struct* @getpwnam(i8*)\n"
       "declare i8* @gets(i8*)\n"
       "declare i32 @gettimeofday(%struct*, i8*)\n"
+      "declare double @hypot(double, double)\n"
+      "declare float @hypotf(float, float)\n"
+      "declare x86_fp80 @hypotl(x86_fp80, x86_fp80)\n"
       "declare i32 @_Z7isasciii(i32)\n"
       "declare i32 @_Z7isdigiti(i32)\n"
       "declare i64 @labs(i64)\n"
-- 
GitLab


From 04549500562783b01db262de62fe324c7ee471c4 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Wed, 30 Oct 2024 17:42:13 +0000
Subject: [PATCH 619/693] [AArch64] Add assembly/disassembly for FMOP4{A,S}
 (non-widening) double-precision instructions (#113345)

The new instructions are described in
https://developer.arm.com/documentation/ddi0602/2024-09/SME-Instructions
---
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |   5 +
 llvm/lib/Target/AArch64/SMEInstrFormats.td    |  37 +++
 .../fmop4as-fp64-non-widening-diagnostics.s   | 243 ++++++++++++++++++
 .../SME2p2/fmop4as-fp64-non-widening.s        | 180 +++++++++++++
 4 files changed, 465 insertions(+)
 create mode 100644 llvm/test/MC/AArch64/SME2p2/fmop4as-fp64-non-widening-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SME2p2/fmop4as-fp64-non-widening.s

diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index e7389b533354..d77219fa7a30 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -1055,3 +1055,8 @@ let Predicates = [HasSME2p2, HasSMEB16B16] in {
   defm BFMOP4A : sme2_bfmop4as_non_widening<0, "bfmop4a">;
   defm BFMOP4S : sme2_bfmop4as_non_widening<1, "bfmop4s">;
 }
+
+let Predicates = [HasSME2p2, HasSMEF64F64] in {
+  defm FMOP4A : sme2_fmop4as_fp64_non_widening<0, "fmop4a">;
+  defm FMOP4S : sme2_fmop4as_fp64_non_widening<1, "fmop4s">;
+}
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index 2740ac814f9c..1c5ec0969245 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -5491,3 +5491,40 @@ multiclass sme2_fmop4as_fp32_non_widening<bit S, string mnemonic> {
   // Multiple vectors
   def _M2Z2Z_S : sme2_fp32_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_s_mul_r_Lo, ZZ_s_mul_r_Hi>;
 }
+
+class sme2_fp64_quarter_tile_outer_product<bit M, bit N, bit S, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>
+    : I<(outs TileOp64:$ZAda),
+        (ins TileOp64:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),
+        mnemonic, "\t$ZAda, $Zn, $Zm",
+        "", []>, Sched<[]> {
+  bits<3> ZAda;
+  bits<3> Zn;
+  bits<3> Zm;
+
+  let Inst{31-21} = 0b10000000110;
+  let Inst{20} = M;
+  let Inst{19-17} = Zm;
+  let Inst{16-10} = 0b0000000;
+  let Inst{9} = N;
+  let Inst{8-6} = Zn;
+  let Inst{5} = 0;
+  let Inst{4} = S;
+  let Inst{3} = 0b1;
+  let Inst{2-0} = ZAda;
+
+  let Constraints = "$ZAda = $_ZAda";
+}
+
+multiclass sme2_fmop4as_fp64_non_widening<bit S, string mnemonic> {
+  // Single vectors
+  def _MZZ_D : sme2_fp64_quarter_tile_outer_product<0, 0, S, mnemonic, ZPR64Mul2_Lo, ZPR64Mul2_Hi>;
+
+  // Multiple and single vectors
+  def _M2ZZ_D : sme2_fp64_quarter_tile_outer_product<0, 1, S, mnemonic, ZZ_d_mul_r_Lo, ZPR64Mul2_Hi>;
+
+  // Single and multiple vectors
+  def _MZ2Z_D : sme2_fp64_quarter_tile_outer_product<1, 0, S, mnemonic, ZPR64Mul2_Lo, ZZ_d_mul_r_Hi>;
+
+  // Multiple vectors
+  def _M2Z2Z_D : sme2_fp64_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_d_mul_r_Lo, ZZ_d_mul_r_Hi>;
+}
diff --git a/llvm/test/MC/AArch64/SME2p2/fmop4as-fp64-non-widening-diagnostics.s b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp64-non-widening-diagnostics.s
new file mode 100644
index 000000000000..ff9602bc12af
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp64-non-widening-diagnostics.s
@@ -0,0 +1,243 @@
+// RUN: not llvm-mc -triple=aarch64 -mattr=+sme2p2,+sme-f64f64 < %s 2>&1 | FileCheck %s
+
+// FMOP4A
+
+// Single vectors
+
+fmop4a za0.s, z0.d, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4a za8.d, z0.d, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.d, z0.s, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.d..z14.d
+
+fmop4a za0.d, z15.d, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.d..z14.d
+
+fmop4a za0.d, z16.d, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.d..z14.d
+
+fmop4a za0.d, z0.d, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.d..z30.d
+
+fmop4a za0.d, z12.d, z17.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.d..z30.d
+
+fmop4a za0.d, z12.d, z14.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.d..z30.d
+
+fmop4a za0.d, z12.d, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.d..z30.d
+
+// Single and multiple vectors
+
+fmop4a za0.s, z0.d, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4a za8.d, z0.d, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.d, z0.s, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.d..z14.d
+
+fmop4a za0.d, z1.d, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.d..z14.d
+
+fmop4a za0.d, z16.d, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.d..z14.d
+
+fmop4a za0.d, z0.d, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.d, z0.d, {z17.d-z18.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.d, z0.d, {z16.d-z18.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.d, z0.d, {z12.d-z13.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+// Multiple and single vectors
+
+fmop4a za0.s, {z0.d-z1.d}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za8.d, {z0.d-z1.d}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.d, {z0.s-z1.s}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.d, {z1.d-z2.d}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.d, {z0.d-z2.d}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.d, {z16.d-z17.d}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.d, {z0.d-z1.d}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.d..z30.d
+
+fmop4a za0.d, {z0.d-z1.d}, z17.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.d..z30.d
+
+fmop4a za0.d, {z0.d-z1.d}, z12.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.d..z30.d
+
+// Multiple vectors
+
+fmop4a za0.s, {z0.d-z1.d}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za8.d, {z0.d-z1.d}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.d, {z0.s-z1.s}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.d, {z1.d-z2.d}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.d, {z0.d-z2.d}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.d, {z18.d-z19.d}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.d, {z0.d-z1.d}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.d, {z0.d-z1.d}, {z19.d-z20.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.d, {z0.d-z1.d}, {z16.d-z18.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.d, {z0.d-z1.d}, {z10.d-z11.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+// FMOP4S
+
+// Single vectors
+
+fmop4s za0.s, z0.d, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4s za8.d, z0.d, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.d, z0.s, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.d..z14.d
+
+fmop4s za0.d, z15.d, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.d..z14.d
+
+fmop4s za0.d, z16.d, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.d..z14.d
+
+fmop4s za0.d, z0.d, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.d..z30.d
+
+fmop4s za0.d, z12.d, z17.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.d..z30.d
+
+fmop4s za0.d, z12.d, z14.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.d..z30.d
+
+fmop4s za0.d, z12.d, z31.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.d..z30.d
+
+// Single and multiple vectors
+
+fmop4s za0.s, z0.d, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.s..z14.s
+
+fmop4s za8.d, z0.d, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.d, z0.s, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.d..z14.d
+
+fmop4s za0.d, z1.d, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.d..z14.d
+
+fmop4s za0.d, z16.d, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z0.d..z14.d
+
+fmop4s za0.d, z0.d, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.d, z0.d, {z17.d-z18.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.d, z0.d, {z16.d-z18.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.d, z0.d, {z12.d-z13.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+// Multiple and single vectors
+
+fmop4s za0.s, {z0.d-z1.d}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za8.d, {z0.d-z1.d}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.d, {z0.s-z1.s}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.d, {z1.d-z2.d}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.d, {z0.d-z2.d}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.d, {z16.d-z17.d}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.d, {z0.d-z1.d}, z16.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.d..z30.d
+
+fmop4s za0.d, {z0.d-z1.d}, z17.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.d..z30.d
+
+fmop4s za0.d, {z0.d-z1.d}, z12.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.d..z30.d
+
+// Multiple vectors
+
+fmop4s za0.s, {z0.d-z1.d}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za8.d, {z0.d-z1.d}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.d, {z0.s-z1.s}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.d, {z1.d-z2.d}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.d, {z0.d-z2.d}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.d, {z18.d-z19.d}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.d, {z0.d-z1.d}, {z16.s-z17.s}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.d, {z0.d-z1.d}, {z19.d-z20.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.d, {z0.d-z1.d}, {z16.d-z18.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.d, {z0.d-z1.d}, {z10.d-z11.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
diff --git a/llvm/test/MC/AArch64/SME2p2/fmop4as-fp64-non-widening.s b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp64-non-widening.s
new file mode 100644
index 000000000000..b0ad2984ad5a
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp64-non-widening.s
@@ -0,0 +1,180 @@
+
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2,+sme-f64f64 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2,+sme-f64f64 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sme2p2,+sme-f64f64 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2,+sme-f64f64 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2p2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2,+sme-f64f64 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2p2,+sme-f64f64 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+// FMOP4A
+
+// Single vectors
+
+fmop4a  za0.d, z0.d, z16.d  // 10000000-11000000-00000000-00001000
+// CHECK-INST: fmop4a  za0.d, z0.d, z16.d
+// CHECK-ENCODING: [0x08,0x00,0xc0,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80c00008 <unknown>
+
+fmop4a  za5.d, z10.d, z20.d  // 10000000-11000100-00000001-01001101
+// CHECK-INST: fmop4a  za5.d, z10.d, z20.d
+// CHECK-ENCODING: [0x4d,0x01,0xc4,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80c4014d <unknown>
+
+fmop4a  za7.d, z14.d, z30.d  // 10000000-11001110-00000001-11001111
+// CHECK-INST: fmop4a  za7.d, z14.d, z30.d
+// CHECK-ENCODING: [0xcf,0x01,0xce,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80ce01cf <unknown>
+
+// Single and multiple vectors
+
+fmop4a  za0.d, z0.d, {z16.d-z17.d}  // 10000000-11010000-00000000-00001000
+// CHECK-INST: fmop4a  za0.d, z0.d, { z16.d, z17.d }
+// CHECK-ENCODING: [0x08,0x00,0xd0,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80d00008 <unknown>
+
+fmop4a  za5.d, z10.d, {z20.d-z21.d}  // 10000000-11010100-00000001-01001101
+// CHECK-INST: fmop4a  za5.d, z10.d, { z20.d, z21.d }
+// CHECK-ENCODING: [0x4d,0x01,0xd4,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80d4014d <unknown>
+
+fmop4a  za7.d, z14.d, {z30.d-z31.d}  // 10000000-11011110-00000001-11001111
+// CHECK-INST: fmop4a  za7.d, z14.d, { z30.d, z31.d }
+// CHECK-ENCODING: [0xcf,0x01,0xde,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80de01cf <unknown>
+
+// Multiple and single vectors
+
+fmop4a  za0.d, {z0.d-z1.d}, z16.d  // 10000000-11000000-00000010-00001000
+// CHECK-INST: fmop4a  za0.d, { z0.d, z1.d }, z16.d
+// CHECK-ENCODING: [0x08,0x02,0xc0,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80c00208 <unknown>
+
+fmop4a  za5.d, {z10.d-z11.d}, z20.d  // 10000000-11000100-00000011-01001101
+// CHECK-INST: fmop4a  za5.d, { z10.d, z11.d }, z20.d
+// CHECK-ENCODING: [0x4d,0x03,0xc4,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80c4034d <unknown>
+
+fmop4a  za7.d, {z14.d-z15.d}, z30.d  // 10000000-11001110-00000011-11001111
+// CHECK-INST: fmop4a  za7.d, { z14.d, z15.d }, z30.d
+// CHECK-ENCODING: [0xcf,0x03,0xce,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80ce03cf <unknown>
+
+// Multiple vectors
+
+fmop4a  za0.d, {z0.d-z1.d}, {z16.d-z17.d}  // 10000000-11010000-00000010-00001000
+// CHECK-INST: fmop4a  za0.d, { z0.d, z1.d }, { z16.d, z17.d }
+// CHECK-ENCODING: [0x08,0x02,0xd0,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80d00208 <unknown>
+
+fmop4a  za5.d, {z10.d-z11.d}, {z20.d-z21.d}  // 10000000-11010100-00000011-01001101
+// CHECK-INST: fmop4a  za5.d, { z10.d, z11.d }, { z20.d, z21.d }
+// CHECK-ENCODING: [0x4d,0x03,0xd4,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80d4034d <unknown>
+
+fmop4a  za7.d, {z14.d-z15.d}, {z30.d-z31.d}  // 10000000-11011110-00000011-11001111
+// CHECK-INST: fmop4a  za7.d, { z14.d, z15.d }, { z30.d, z31.d }
+// CHECK-ENCODING: [0xcf,0x03,0xde,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80de03cf <unknown>
+
+
+// FMOP4S
+
+// Single vectors
+
+fmop4s  za0.d, z0.d, z16.d  // 10000000-11000000-00000000-00011000
+// CHECK-INST: fmop4s  za0.d, z0.d, z16.d
+// CHECK-ENCODING: [0x18,0x00,0xc0,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80c00018 <unknown>
+
+fmop4s  za5.d, z10.d, z20.d  // 10000000-11000100-00000001-01011101
+// CHECK-INST: fmop4s  za5.d, z10.d, z20.d
+// CHECK-ENCODING: [0x5d,0x01,0xc4,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80c4015d <unknown>
+
+fmop4s  za7.d, z14.d, z30.d  // 10000000-11001110-00000001-11011111
+// CHECK-INST: fmop4s  za7.d, z14.d, z30.d
+// CHECK-ENCODING: [0xdf,0x01,0xce,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80ce01df <unknown>
+
+// Single and multiple vectors
+
+fmop4s  za0.d, z0.d, {z16.d-z17.d}  // 10000000-11010000-00000000-00011000
+// CHECK-INST: fmop4s  za0.d, z0.d, { z16.d, z17.d }
+// CHECK-ENCODING: [0x18,0x00,0xd0,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80d00018 <unknown>
+
+fmop4s  za5.d, z10.d, {z20.d-z21.d}  // 10000000-11010100-00000001-01011101
+// CHECK-INST: fmop4s  za5.d, z10.d, { z20.d, z21.d }
+// CHECK-ENCODING: [0x5d,0x01,0xd4,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80d4015d <unknown>
+
+fmop4s  za7.d, z14.d, {z30.d-z31.d}  // 10000000-11011110-00000001-11011111
+// CHECK-INST: fmop4s  za7.d, z14.d, { z30.d, z31.d }
+// CHECK-ENCODING: [0xdf,0x01,0xde,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80de01df <unknown>
+
+// Multiple and single vectors
+
+fmop4s  za0.d, {z0.d-z1.d}, z16.d  // 10000000-11000000-00000010-00011000
+// CHECK-INST: fmop4s  za0.d, { z0.d, z1.d }, z16.d
+// CHECK-ENCODING: [0x18,0x02,0xc0,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80c00218 <unknown>
+
+fmop4s  za5.d, {z10.d-z11.d}, z20.d  // 10000000-11000100-00000011-01011101
+// CHECK-INST: fmop4s  za5.d, { z10.d, z11.d }, z20.d
+// CHECK-ENCODING: [0x5d,0x03,0xc4,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80c4035d <unknown>
+
+fmop4s  za7.d, {z14.d-z15.d}, z30.d  // 10000000-11001110-00000011-11011111
+// CHECK-INST: fmop4s  za7.d, { z14.d, z15.d }, z30.d
+// CHECK-ENCODING: [0xdf,0x03,0xce,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80ce03df <unknown>
+
+// Multiple vectors
+
+fmop4s  za0.d, {z0.d-z1.d}, {z16.d-z17.d}  // 10000000-11010000-00000010-00011000
+// CHECK-INST: fmop4s  za0.d, { z0.d, z1.d }, { z16.d, z17.d }
+// CHECK-ENCODING: [0x18,0x02,0xd0,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80d00218 <unknown>
+
+fmop4s  za5.d, {z10.d-z11.d}, {z20.d-z21.d}  // 10000000-11010100-00000011-01011101
+// CHECK-INST: fmop4s  za5.d, { z10.d, z11.d }, { z20.d, z21.d }
+// CHECK-ENCODING: [0x5d,0x03,0xd4,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80d4035d <unknown>
+
+fmop4s  za7.d, {z14.d-z15.d}, {z30.d-z31.d}  // 10000000-11011110-00000011-11011111
+// CHECK-INST: fmop4s  za7.d, { z14.d, z15.d }, { z30.d, z31.d }
+// CHECK-ENCODING: [0xdf,0x03,0xde,0x80]
+// CHECK-ERROR: instruction requires: sme2p2 sme-f64f64
+// CHECK-UNKNOWN: 80de03df <unknown>
-- 
GitLab


From 705f3ebf1458c154fe63552ca984be6a16711661 Mon Sep 17 00:00:00 2001
From: Chris Apple <cja-private@pm.me>
Date: Wed, 30 Oct 2024 10:42:40 -0700
Subject: [PATCH 620/693] [rtsan][NFC] Add documentation link to Function
 Effects (#113979)

---
 clang/docs/RealtimeSanitizer.rst | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/clang/docs/RealtimeSanitizer.rst b/clang/docs/RealtimeSanitizer.rst
index 41b8bbb33baf..e5f5abfcd9b4 100644
--- a/clang/docs/RealtimeSanitizer.rst
+++ b/clang/docs/RealtimeSanitizer.rst
@@ -11,11 +11,16 @@ RealtimeSanitizer (a.k.a. RTSan) is a real-time safety testing tool for C and C+
 projects. RTSan can be used to detect real-time violations, i.e. calls to methods
 that are not safe for use in functions with deterministic run time requirements.
 RTSan considers any function marked with the ``[[clang::nonblocking]]`` attribute
-to be a real-time function. If RTSan detects a call to ``malloc``, ``free``,
-``pthread_mutex_lock``, or anything else that could have a non-deterministic
-execution time in a function marked ``[[clang::nonblocking]]``
+to be a real-time function. At run-time, if RTSan detects a call to ``malloc``, 
+``free``, ``pthread_mutex_lock``, or anything else that could have a 
+non-deterministic execution time in a function marked ``[[clang::nonblocking]]``
 RTSan raises an error.
 
+RTSan performs its analysis at run-time but shares the ``[[clang::nonblocking]]`` 
+attribute with the :doc:`FunctionEffectAnalysis` system, which operates at 
+compile-time to detect potential real-time safety violations. For comprehensive 
+detection of real-time safety issues, it is recommended to use both systems together.
+
 The runtime slowdown introduced by RealtimeSanitizer is negligible.
 
 How to build
-- 
GitLab


From 5545f76dc94e76ef6800823bdd1e107ad2264717 Mon Sep 17 00:00:00 2001
From: Sean Perry <perry@ca.ibm.com>
Date: Wed, 30 Oct 2024 13:48:00 -0400
Subject: [PATCH 621/693] Pass the executable name as arg[0] when calling
 ExecuteAndWait() (#114067)

PR https://github.com/llvm/llvm-project/pull/111976 was enabling the
tests updated in the PR to run on all systems. We found a few didn't run
on z/OS. I tracked the problem down to:
1. the ExecuteToolChainProgram() function wasn't passing the executable
name as the first arg. That was causing exec on z/OS to fail.
2. the temp file needs to be a text file so codepage conversion happens.
---
 clang/lib/Driver/ToolChain.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 34de0043ca01..bdf3da0c96ad 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -109,7 +109,8 @@ ToolChain::ToolChain(const Driver &D, const llvm::Triple &T,
 llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
 ToolChain::executeToolChainProgram(StringRef Executable) const {
   llvm::SmallString<64> OutputFile;
-  llvm::sys::fs::createTemporaryFile("toolchain-program", "txt", OutputFile);
+  llvm::sys::fs::createTemporaryFile("toolchain-program", "txt", OutputFile,
+                                     llvm::sys::fs::OF_Text);
   llvm::FileRemover OutputRemover(OutputFile.c_str());
   std::optional<llvm::StringRef> Redirects[] = {
       {""},
@@ -128,7 +129,8 @@ ToolChain::executeToolChainProgram(StringRef Executable) const {
                                          *Str + "'");
     SecondsToWait = std::max(SecondsToWait, 0); // infinite
   }
-  if (llvm::sys::ExecuteAndWait(Executable, {}, {}, Redirects, SecondsToWait,
+  if (llvm::sys::ExecuteAndWait(Executable, {Executable}, {}, Redirects,
+                                SecondsToWait,
                                 /*MemoryLimit=*/0, &ErrorMessage))
     return llvm::createStringError(std::error_code(),
                                    Executable + ": " + ErrorMessage);
-- 
GitLab


From ca1154d1d41c75db6594428a8cdf263cf7041896 Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang@amd.com>
Date: Wed, 30 Oct 2024 11:07:15 -0700
Subject: [PATCH 622/693] AMDGPU: Disable pattern matching "x<<32-y>>32-y" to
 "bfe x, 0, y" (#114279)

It is not correct to lower "x<<32-y>>32-y" to "bfe x, 0, y". When y
equals to 32, the left-hand side is still x (unchanged), however, the
right-hand side will be evaluated to 0. So it is not always correct to
do such transformation.

We may be able to keep the pattern for immediate y while y is within [0,
31]. However, the immediate operands of the sub (32 - y) are easily
folded, and "(x << imm) >> imm" will be lowered to "and x,
(2^(32-imm))-1" anyway. So no bfe matching is needed.
---
 llvm/lib/Target/AMDGPU/SIInstructions.td    | 13 -------------
 llvm/test/CodeGen/AMDGPU/bfe-patterns.ll    | 16 ++++++++++++----
 llvm/test/CodeGen/AMDGPU/extract-lowbits.ll | 20 +++++++++++++++-----
 3 files changed, 27 insertions(+), 22 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index faa0b6d6c3f5..c8a46217190a 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3553,19 +3553,6 @@ def : AMDGPUPat <
   (V_BFE_U32_e64 $src, (i32 0), $width)
 >;
 
-// x << (bitwidth - y) >> (bitwidth - y)
-def : AMDGPUPat <
-  (DivergentBinFrag<srl> (shl_oneuse i32:$src, (sub 32, i32:$width)),
-                         (sub 32, i32:$width)),
-  (V_BFE_U32_e64 $src, (i32 0), $width)
->;
-
-def : AMDGPUPat <
-  (DivergentBinFrag<sra> (shl_oneuse i32:$src, (sub 32, i32:$width)),
-                         (sub 32, i32:$width)),
-  (V_BFE_I32_e64 $src, (i32 0), $width)
->;
-
 // SHA-256 Ma patterns
 
 // ((x & z) | (y & (x | z))) -> BFI (XOR x, y), z, y
diff --git a/llvm/test/CodeGen/AMDGPU/bfe-patterns.ll b/llvm/test/CodeGen/AMDGPU/bfe-patterns.ll
index f54ea615ca66..c57a35aa1880 100644
--- a/llvm/test/CodeGen/AMDGPU/bfe-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/bfe-patterns.ll
@@ -17,7 +17,9 @@ define amdgpu_kernel void @v_ubfe_sub_i32(ptr addrspace(1) %out, ptr addrspace(1
 ; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT:    v_bfe_u32 v2, v2, 0, v3
+; SI-NEXT:    v_sub_i32_e32 v3, vcc, 32, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v2, v3, v2
+; SI-NEXT:    v_lshrrev_b32_e32 v2, v3, v2
 ; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; SI-NEXT:    s_endpgm
 ;
@@ -36,7 +38,9 @@ define amdgpu_kernel void @v_ubfe_sub_i32(ptr addrspace(1) %out, ptr addrspace(1
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    v_bfe_u32 v2, v3, 0, v4
+; VI-NEXT:    v_sub_u32_e32 v2, vcc, 32, v4
+; VI-NEXT:    v_lshlrev_b32_e32 v3, v2, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v2, v2, v3
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
   %id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -215,7 +219,9 @@ define amdgpu_kernel void @v_sbfe_sub_i32(ptr addrspace(1) %out, ptr addrspace(1
 ; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT:    v_bfe_i32 v2, v2, 0, v3
+; SI-NEXT:    v_sub_i32_e32 v3, vcc, 32, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v2, v3, v2
+; SI-NEXT:    v_ashrrev_i32_e32 v2, v3, v2
 ; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; SI-NEXT:    s_endpgm
 ;
@@ -234,7 +240,9 @@ define amdgpu_kernel void @v_sbfe_sub_i32(ptr addrspace(1) %out, ptr addrspace(1
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    v_bfe_i32 v2, v3, 0, v4
+; VI-NEXT:    v_sub_u32_e32 v2, vcc, 32, v4
+; VI-NEXT:    v_lshlrev_b32_e32 v3, v2, v3
+; VI-NEXT:    v_ashrrev_i32_e32 v2, v2, v3
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
   %id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/extract-lowbits.ll b/llvm/test/CodeGen/AMDGPU/extract-lowbits.ll
index 9677ec41ce26..3d9616f02d52 100644
--- a/llvm/test/CodeGen/AMDGPU/extract-lowbits.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract-lowbits.ll
@@ -150,11 +150,21 @@ define i32 @bzhi32_c4_commutative(i32 %val, i32 %numlowbits) nounwind {
 ; ---------------------------------------------------------------------------- ;
 
 define i32 @bzhi32_d0(i32 %val, i32 %numlowbits) nounwind {
-; GCN-LABEL: bzhi32_d0:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_bfe_u32 v0, v0, 0, v1
-; GCN-NEXT:    s_setpc_b64 s[30:31]
+; SI-LABEL: bzhi32_d0:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_sub_i32_e32 v1, vcc, 32, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v0, v1, v0
+; SI-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bzhi32_d0:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_sub_u32_e32 v1, vcc, 32, v1
+; VI-NEXT:    v_lshlrev_b32_e32 v0, v1, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
   %numhighbits = sub i32 32, %numlowbits
   %highbitscleared = shl i32 %val, %numhighbits
   %masked = lshr i32 %highbitscleared, %numhighbits
-- 
GitLab


From a518ed2d815c16010a6262edd0414a5f60a63a39 Mon Sep 17 00:00:00 2001
From: Dana Jansens <danakj@chromium.org>
Date: Wed, 30 Oct 2024 14:30:53 -0400
Subject: [PATCH 623/693] Respect the [[clang::unsafe_buffer_usage]] attribute
 for field and constructor initializers (#91991)

CXXCtorInitializers are not statements , but they point to an
initializer expression which is. When visiting a FunctionDecl, also
walk through any constructor initializers and run the warning
checks/matchers against their initializer expressions. This catches
warnings for initializing fields and calling other constructors, such
as:

struct C {
  C(P* Ptr) : AnUnsafeCtor(Ptr) {}
}

Field initializers can be found by traversing CXXDefaultInitExprs. This
catches warnings in places such as:

struct C {
  P* Ptr;
  AnUnsafeCtor U{Ptr};
};

We add tests for explicit construction, for field initialization, base
class constructor calls, delegated constructor calls, and aggregate
initialization.

Note that aggregate initialization is not fully covered where a field
specifies an initializer and it's not overridden in the aggregate initialization,
such as in:

struct AggregateViaValueInit {
    UnsafeMembers f1;
    // FIXME: A construction of this class does initialize the field
    // through this initializer, so it should warn. Ideally it should
    // also point to where the site of the construction is in
    // testAggregateViaValueInit().
    UnsafeMembers f2{3};
};

void testAggregateViaValueInit() {
    auto A = AggregateViaValueInit();
};

There are 3 tests for different types of aggregate initialization with
FIXMEs documenting this future work.

One attempt to fix this involved returning true from
MatchDescendantVisitor::shouldVisitImplicitCode(), however, it breaks expectations
for field in-class initializers by moving the SourceLocation, possibly
to inside the implicit ctor instead of on the line where the field
initialization happens.

struct C {
  P* Ptr;
  AnUnsafeCtor U{Ptr};  // expected-warning{{this is never seen then}}
};

Tests are also added for std::span(ptr, size) constructor being called
from a field initializer and a constructor initializer.

Issue #80482
---
 clang/lib/Analysis/UnsafeBufferUsage.cpp      | 127 +++++++++++-------
 ...warn-unsafe-buffer-usage-function-attr.cpp | 122 +++++++++++++++++
 ...ffer-usage-in-container-span-construct.cpp |  20 +++
 3 files changed, 224 insertions(+), 45 deletions(-)

diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index fad2f52e89ef..2c68409b846b 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -171,6 +171,12 @@ public:
     return VisitorBase::TraverseCXXTypeidExpr(Node);
   }
 
+  bool TraverseCXXDefaultInitExpr(CXXDefaultInitExpr *Node) {
+    if (!TraverseStmt(Node->getExpr()))
+      return false;
+    return VisitorBase::TraverseCXXDefaultInitExpr(Node);
+  }
+
   bool TraverseStmt(Stmt *Node, DataRecursionQueue *Queue = nullptr) {
     if (!Node)
       return true;
@@ -1972,14 +1978,18 @@ public:
 };
 
 /// Scan the function and return a list of gadgets found with provided kits.
-static std::tuple<FixableGadgetList, WarningGadgetList, DeclUseTracker>
-findGadgets(const Decl *D, const UnsafeBufferUsageHandler &Handler,
-            bool EmitSuggestions) {
+static void findGadgets(const Stmt *S, ASTContext &Ctx,
+                        const UnsafeBufferUsageHandler &Handler,
+                        bool EmitSuggestions, FixableGadgetList &FixableGadgets,
+                        WarningGadgetList &WarningGadgets,
+                        DeclUseTracker &Tracker) {
 
   struct GadgetFinderCallback : MatchFinder::MatchCallback {
-    FixableGadgetList FixableGadgets;
-    WarningGadgetList WarningGadgets;
-    DeclUseTracker Tracker;
+    GadgetFinderCallback(FixableGadgetList &FixableGadgets,
+                         WarningGadgetList &WarningGadgets,
+                         DeclUseTracker &Tracker)
+        : FixableGadgets(FixableGadgets), WarningGadgets(WarningGadgets),
+          Tracker(Tracker) {}
 
     void run(const MatchFinder::MatchResult &Result) override {
       // In debug mode, assert that we've found exactly one gadget.
@@ -2020,10 +2030,14 @@ findGadgets(const Decl *D, const UnsafeBufferUsageHandler &Handler,
       assert(numFound >= 1 && "Gadgets not found in match result!");
       assert(numFound <= 1 && "Conflicting bind tags in gadgets!");
     }
+
+    FixableGadgetList &FixableGadgets;
+    WarningGadgetList &WarningGadgets;
+    DeclUseTracker &Tracker;
   };
 
   MatchFinder M;
-  GadgetFinderCallback CB;
+  GadgetFinderCallback CB{FixableGadgets, WarningGadgets, Tracker};
 
   // clang-format off
   M.addMatcher(
@@ -2068,9 +2082,7 @@ findGadgets(const Decl *D, const UnsafeBufferUsageHandler &Handler,
     // clang-format on
   }
 
-  M.match(*D->getBody(), D->getASTContext());
-  return {std::move(CB.FixableGadgets), std::move(CB.WarningGadgets),
-          std::move(CB.Tracker)};
+  M.match(*S, Ctx);
 }
 
 // Compares AST nodes by source locations.
@@ -3614,39 +3626,9 @@ public:
   }
 };
 
-void clang::checkUnsafeBufferUsage(const Decl *D,
-                                   UnsafeBufferUsageHandler &Handler,
-                                   bool EmitSuggestions) {
-#ifndef NDEBUG
-  Handler.clearDebugNotes();
-#endif
-
-  assert(D && D->getBody());
-  // We do not want to visit a Lambda expression defined inside a method
-  // independently. Instead, it should be visited along with the outer method.
-  // FIXME: do we want to do the same thing for `BlockDecl`s?
-  if (const auto *fd = dyn_cast<CXXMethodDecl>(D)) {
-    if (fd->getParent()->isLambda() && fd->getParent()->isLocalClass())
-      return;
-  }
-
-  // Do not emit fixit suggestions for functions declared in an
-  // extern "C" block.
-  if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
-    for (FunctionDecl *FReDecl : FD->redecls()) {
-      if (FReDecl->isExternC()) {
-        EmitSuggestions = false;
-        break;
-      }
-    }
-  }
-
-  WarningGadgetSets UnsafeOps;
-  FixableGadgetSets FixablesForAllVars;
-
-  auto [FixableGadgets, WarningGadgets, Tracker] =
-      findGadgets(D, Handler, EmitSuggestions);
-
+void applyGadgets(const Decl *D, FixableGadgetList FixableGadgets,
+                  WarningGadgetList WarningGadgets, DeclUseTracker Tracker,
+                  UnsafeBufferUsageHandler &Handler, bool EmitSuggestions) {
   if (!EmitSuggestions) {
     // Our job is very easy without suggestions. Just warn about
     // every problematic operation and consider it done. No need to deal
@@ -3690,8 +3672,10 @@ void clang::checkUnsafeBufferUsage(const Decl *D,
   if (WarningGadgets.empty())
     return;
 
-  UnsafeOps = groupWarningGadgetsByVar(std::move(WarningGadgets));
-  FixablesForAllVars = groupFixablesByVar(std::move(FixableGadgets));
+  WarningGadgetSets UnsafeOps =
+      groupWarningGadgetsByVar(std::move(WarningGadgets));
+  FixableGadgetSets FixablesForAllVars =
+      groupFixablesByVar(std::move(FixableGadgets));
 
   std::map<const VarDecl *, FixItList> FixItsForVariableGroup;
 
@@ -3912,3 +3896,56 @@ void clang::checkUnsafeBufferUsage(const Decl *D,
     }
   }
 }
+
+void clang::checkUnsafeBufferUsage(const Decl *D,
+                                   UnsafeBufferUsageHandler &Handler,
+                                   bool EmitSuggestions) {
+#ifndef NDEBUG
+  Handler.clearDebugNotes();
+#endif
+
+  assert(D);
+
+  SmallVector<Stmt *> Stmts;
+
+  if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
+    // We do not want to visit a Lambda expression defined inside a method
+    // independently. Instead, it should be visited along with the outer method.
+    // FIXME: do we want to do the same thing for `BlockDecl`s?
+    if (const auto *MD = dyn_cast<CXXMethodDecl>(D)) {
+      if (MD->getParent()->isLambda() && MD->getParent()->isLocalClass())
+        return;
+    }
+
+    for (FunctionDecl *FReDecl : FD->redecls()) {
+      if (FReDecl->isExternC()) {
+        // Do not emit fixit suggestions for functions declared in an
+        // extern "C" block.
+        EmitSuggestions = false;
+        break;
+      }
+    }
+
+    Stmts.push_back(FD->getBody());
+
+    if (const auto *ID = dyn_cast<CXXConstructorDecl>(D)) {
+      for (const CXXCtorInitializer *CI : ID->inits()) {
+        Stmts.push_back(CI->getInit());
+      }
+    }
+  } else if (isa<BlockDecl>(D) || isa<ObjCMethodDecl>(D)) {
+    Stmts.push_back(D->getBody());
+  }
+
+  assert(!Stmts.empty());
+
+  FixableGadgetList FixableGadgets;
+  WarningGadgetList WarningGadgets;
+  DeclUseTracker Tracker;
+  for (Stmt *S : Stmts) {
+    findGadgets(S, D->getASTContext(), Handler, EmitSuggestions, FixableGadgets,
+                WarningGadgets, Tracker);
+  }
+  applyGadgets(D, std::move(FixableGadgets), std::move(WarningGadgets),
+               std::move(Tracker), Handler, EmitSuggestions);
+}
diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-function-attr.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-function-attr.cpp
index bfc34b55c1f6..724d444638b5 100644
--- a/clang/test/SemaCXX/warn-unsafe-buffer-usage-function-attr.cpp
+++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-function-attr.cpp
@@ -111,6 +111,37 @@ int testFoldExpression(Vs&&... v) {
     return (... + v);  // expected-warning{{function introduces unsafe buffer manipulation}}
 }
 
+struct HoldsUnsafeMembers {
+    HoldsUnsafeMembers()
+        : FromCtor(3),  // expected-warning{{function introduces unsafe buffer manipulation}}
+          FromCtor2{3}  // expected-warning{{function introduces unsafe buffer manipulation}}
+    {}
+
+    [[clang::unsafe_buffer_usage]]
+    HoldsUnsafeMembers(int i)
+        : FromCtor(i),  // expected-warning{{function introduces unsafe buffer manipulation}}
+          FromCtor2{i}  // expected-warning{{function introduces unsafe buffer manipulation}}
+    {}
+
+    HoldsUnsafeMembers(float f)
+        : HoldsUnsafeMembers(0) {}  // expected-warning{{function introduces unsafe buffer manipulation}}
+
+    UnsafeMembers FromCtor;
+    UnsafeMembers FromCtor2;
+    UnsafeMembers FromField{3};  // expected-warning 2{{function introduces unsafe buffer manipulation}}
+};
+
+struct SubclassUnsafeMembers : public UnsafeMembers {
+    SubclassUnsafeMembers()
+        : UnsafeMembers(3)  // expected-warning{{function introduces unsafe buffer manipulation}}
+    {}
+
+    [[clang::unsafe_buffer_usage]]
+    SubclassUnsafeMembers(int i)
+        : UnsafeMembers(i)  // expected-warning{{function introduces unsafe buffer manipulation}}
+    {}
+};
+
 // https://github.com/llvm/llvm-project/issues/80482
 void testClassMembers() {
     UnsafeMembers(3);  // expected-warning{{function introduces unsafe buffer manipulation}}
@@ -122,4 +153,95 @@ void testClassMembers() {
     UnsafeMembers()();  // expected-warning{{function introduces unsafe buffer manipulation}}
 
     testFoldExpression(UnsafeMembers(), UnsafeMembers());
+
+    HoldsUnsafeMembers();
+    HoldsUnsafeMembers(3);  // expected-warning{{function introduces unsafe buffer manipulation}}
+
+    SubclassUnsafeMembers();
+    SubclassUnsafeMembers(3);  // expected-warning{{function introduces unsafe buffer manipulation}}
+}
+
+// Not an aggregate, so its constructor is not implicit code and will be
+// visited/checked for warnings.
+struct NotCalledHoldsUnsafeMembers {
+    NotCalledHoldsUnsafeMembers()
+        : FromCtor(3),  // expected-warning{{function introduces unsafe buffer manipulation}}
+          FromCtor2{3}  // expected-warning{{function introduces unsafe buffer manipulation}}
+    {}
+
+    UnsafeMembers FromCtor;
+    UnsafeMembers FromCtor2;
+    UnsafeMembers FromField{3};  // expected-warning{{function introduces unsafe buffer manipulation}}
+};
+
+// An aggregate, so its constructor is implicit code. Since it's not called, it
+// is never generated.
+struct AggregateUnused {
+    UnsafeMembers f1;
+    // While this field would trigger the warning during initialization, since
+    // it's unused, there's no code generated that does the initialization, so
+    // no warning.
+    UnsafeMembers f2{3};
+};
+
+struct AggregateExplicitlyInitializedSafe {
+    UnsafeMembers f1;
+    // The warning is not fired as the field is always explicltly initialized
+    // elsewhere. This initializer is never used.
+    UnsafeMembers f2{3};
+};
+
+void testAggregateExplicitlyInitializedSafe() {
+    AggregateExplicitlyInitializedSafe A{
+        .f2 = UnsafeMembers(),  // A safe constructor.
+    };
 }
+
+struct AggregateExplicitlyInitializedUnsafe {
+    UnsafeMembers f1;
+    // The warning is not fired as the field is always explicltly initialized
+    // elsewhere. This initializer is never used.
+    UnsafeMembers f2{3};
+};
+
+void testAggregateExplicitlyInitializedUnsafe() {
+    AggregateExplicitlyInitializedUnsafe A{
+        .f2 = UnsafeMembers(3),  // expected-warning{{function introduces unsafe buffer manipulation}}
+    };
+}
+
+struct AggregateViaAggregateInit {
+    UnsafeMembers f1;
+    // FIXME: A construction of this class does initialize the field through
+    // this initializer, so it should warn. Ideally it should also point to
+    // where the site of the construction is in testAggregateViaAggregateInit().
+    UnsafeMembers f2{3};
+};
+
+void testAggregateViaAggregateInit() {
+    AggregateViaAggregateInit A{};
+};
+
+struct AggregateViaValueInit {
+    UnsafeMembers f1;
+    // FIXME: A construction of this class does initialize the field through
+    // this initializer, so it should warn. Ideally it should also point to
+    // where the site of the construction is in testAggregateViaValueInit().
+    UnsafeMembers f2{3};
+};
+
+void testAggregateViaValueInit() {
+    auto A = AggregateViaValueInit();
+};
+
+struct AggregateViaDefaultInit {
+    UnsafeMembers f1;
+    // FIXME: A construction of this class does initialize the field through
+    // this initializer, so it should warn. Ideally it should also point to
+    // where the site of the construction is in testAggregateViaValueInit().
+    UnsafeMembers f2{3};
+};
+
+void testAggregateViaDefaultInit() {
+    AggregateViaDefaultInit A;
+};
diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
index e97511593bbd..c138fe088b3b 100644
--- a/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
+++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
@@ -157,3 +157,23 @@ namespace test_flag {
 
   }
 } //namespace test_flag
+
+struct HoldsStdSpanAndInitializedInCtor {
+  char* Ptr;
+  unsigned Size;
+  std::span<char> Span{Ptr, Size};  // no-warning (this code is unreachable)
+
+  HoldsStdSpanAndInitializedInCtor(char* P, unsigned S)
+      : Span(P, S)  // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+  {}
+};
+
+struct HoldsStdSpanAndNotInitializedInCtor {
+  char* Ptr;
+  unsigned Size;
+  std::span<char> Span{Ptr, Size}; // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+
+  HoldsStdSpanAndNotInitializedInCtor(char* P, unsigned S)
+      : Ptr(P), Size(S)
+  {}
+};
-- 
GitLab


From 2bc5302706e710d125752c215392043fd5bf80fa Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Wed, 30 Oct 2024 11:36:10 -0700
Subject: [PATCH 624/693] Revert "[lldb] Use Py_InitializeFromConfig with
 Python >= 3.8 (NFC)" (#114290)

Reverts llvm/llvm-project#114112 because this triggers a compile error:

```
no known conversion from 'str_type' (aka 'wchar_t *') to 'const char *' for 3rd argument
  221 | PyAPI_FUNC(PyStatus) PyConfig_SetBytesString(
      |                      ^
  222 |     PyConfig *config,
  223 |     wchar_t **config_str,
  224 |     const char *str);
      |     ~~~~~~~~~~~~~~~
1 error generated.

```
---
 .../Python/ScriptInterpreterPython.cpp        | 68 ++++++++-----------
 1 file changed, 28 insertions(+), 40 deletions(-)

diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index 6158083a9828..7cc38da6a6a9 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -92,38 +92,7 @@ namespace {
 struct InitializePythonRAII {
 public:
   InitializePythonRAII() {
-#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 8) || (PY_MAJOR_VERSION > 3)
-    PyConfig config;
-    PyConfig_InitPythonConfig(&config);
-#endif
-
-#if LLDB_EMBED_PYTHON_HOME
-    typedef wchar_t *str_type;
-    static str_type g_python_home = []() -> str_type {
-      const char *lldb_python_home = LLDB_PYTHON_HOME;
-      const char *absolute_python_home = nullptr;
-      llvm::SmallString<64> path;
-      if (llvm::sys::path::is_absolute(lldb_python_home)) {
-        absolute_python_home = lldb_python_home;
-      } else {
-        FileSpec spec = HostInfo::GetShlibDir();
-        if (!spec)
-          return nullptr;
-        spec.GetPath(path);
-        llvm::sys::path::append(path, lldb_python_home);
-        absolute_python_home = path.c_str();
-      }
-      size_t size = 0;
-      return Py_DecodeLocale(absolute_python_home, &size);
-    }();
-    if (g_python_home != nullptr) {
-#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 8) || (PY_MAJOR_VERSION > 3)
-      PyConfig_SetBytesString(&config, &config.home, g_python_home);
-#else
-      Py_SetPythonHome(g_python_home);
-#endif
-    }
-#endif
+    InitializePythonHome();
 
     // The table of built-in modules can only be extended before Python is
     // initialized.
@@ -148,22 +117,15 @@ public:
       PyImport_AppendInittab("_lldb", LLDBSwigPyInit);
     }
 
-#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 8) || (PY_MAJOR_VERSION > 3)
-    config.install_signal_handlers = 0;
-    Py_InitializeFromConfig(&config);
-    PyConfig_Clear(&config);
-    InitializeThreadsPrivate();
-#else
 // Python < 3.2 and Python >= 3.2 reversed the ordering requirements for
 // calling `Py_Initialize` and `PyEval_InitThreads`.  < 3.2 requires that you
 // call `PyEval_InitThreads` first, and >= 3.2 requires that you call it last.
-#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 2)
+#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 2) || (PY_MAJOR_VERSION > 3)
     Py_InitializeEx(0);
     InitializeThreadsPrivate();
 #else
     InitializeThreadsPrivate();
     Py_InitializeEx(0);
-#endif
 #endif
   }
 
@@ -180,6 +142,32 @@ public:
   }
 
 private:
+  void InitializePythonHome() {
+#if LLDB_EMBED_PYTHON_HOME
+    typedef wchar_t *str_type;
+    static str_type g_python_home = []() -> str_type {
+      const char *lldb_python_home = LLDB_PYTHON_HOME;
+      const char *absolute_python_home = nullptr;
+      llvm::SmallString<64> path;
+      if (llvm::sys::path::is_absolute(lldb_python_home)) {
+        absolute_python_home = lldb_python_home;
+      } else {
+        FileSpec spec = HostInfo::GetShlibDir();
+        if (!spec)
+          return nullptr;
+        spec.GetPath(path);
+        llvm::sys::path::append(path, lldb_python_home);
+        absolute_python_home = path.c_str();
+      }
+      size_t size = 0;
+      return Py_DecodeLocale(absolute_python_home, &size);
+    }();
+    if (g_python_home != nullptr) {
+      Py_SetPythonHome(g_python_home);
+    }
+#endif
+  }
+
   void InitializeThreadsPrivate() {
 // Since Python 3.7 `Py_Initialize` calls `PyEval_InitThreads` inside itself,
 // so there is no way to determine whether the embedded interpreter
-- 
GitLab


From 5bd1af5abcb7b9f92741dd7209e84b5607f7e88a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 30 Oct 2024 18:39:49 +0000
Subject: [PATCH 625/693] [LV] Directly store VPlan in InnerLoopVectorizer
 (NFC).

The current VPlan is already passed to multiple functions and more in
the future. Store it once directly in InnerLoopVectorizer.
---
 .../Transforms/Vectorize/LoopVectorize.cpp    | 59 ++++++++++---------
 1 file changed, 30 insertions(+), 29 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 150fc4a42b48..3d638e52328b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -467,11 +467,12 @@ public:
                       ElementCount MinProfitableTripCount,
                       unsigned UnrollFactor, LoopVectorizationLegality *LVL,
                       LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
-                      ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks)
+                      ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks,
+                      VPlan &Plan)
       : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TLI(TLI), TTI(TTI),
         AC(AC), ORE(ORE), VF(VecWidth), UF(UnrollFactor),
         Builder(PSE.getSE()->getContext()), Legal(LVL), Cost(CM), BFI(BFI),
-        PSI(PSI), RTChecks(RTChecks) {
+        PSI(PSI), RTChecks(RTChecks), Plan(Plan) {
     // Query this against the original loop and save it here because the profile
     // of the original loop header may change as the transformation happens.
     OptForSizeBasedOnProfile = llvm::shouldOptimizeForSize(
@@ -498,7 +499,7 @@ public:
   createVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs);
 
   /// Fix the vectorized code, taking care of header phi's, live-outs, and more.
-  void fixVectorizedLoop(VPTransformState &State, VPlan &Plan);
+  void fixVectorizedLoop(VPTransformState &State);
 
   // Return true if any runtime check is added.
   bool areSafetyChecksAdded() { return AddedSafetyChecks; }
@@ -513,7 +514,7 @@ public:
                             VPTransformState &State);
 
   /// Fix the non-induction PHIs in \p Plan.
-  void fixNonInductionPHIs(VPlan &Plan, VPTransformState &State);
+  void fixNonInductionPHIs(VPTransformState &State);
 
   /// Create a new phi node for the induction variable \p OrigPhi to resume
   /// iteration count in the scalar epilogue, from where the vectorized loop
@@ -541,8 +542,7 @@ protected:
   /// Set up the values of the IVs correctly when exiting the vector loop.
   virtual void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II,
                             Value *VectorTripCount, Value *EndValue,
-                            BasicBlock *MiddleBlock, VPlan &Plan,
-                            VPTransformState &State);
+                            BasicBlock *MiddleBlock, VPTransformState &State);
 
   /// Iteratively sink the scalarized operands of a predicated instruction into
   /// the block that was created for it.
@@ -674,6 +674,8 @@ protected:
   /// Structure to hold information about generated runtime checks, responsible
   /// for cleaning the checks, if vectorization turns out unprofitable.
   GeneratedRTChecks &RTChecks;
+
+  VPlan &Plan;
 };
 
 /// Encapsulate information regarding vectorization of a loop and its epilogue.
@@ -715,10 +717,10 @@ public:
       OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
       LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
       BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
-      GeneratedRTChecks &Checks)
+      GeneratedRTChecks &Checks, VPlan &Plan)
       : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
                             EPI.MainLoopVF, EPI.MainLoopVF, EPI.MainLoopUF, LVL,
-                            CM, BFI, PSI, Checks),
+                            CM, BFI, PSI, Checks, Plan),
         EPI(EPI) {}
 
   // Override this function to handle the more complex control flow around the
@@ -755,9 +757,9 @@ public:
       OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
       LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
       BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
-      GeneratedRTChecks &Check)
+      GeneratedRTChecks &Check, VPlan &Plan)
       : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
-                                       EPI, LVL, CM, BFI, PSI, Check) {}
+                                       EPI, LVL, CM, BFI, PSI, Check, Plan) {}
   /// Implements the interface for creating a vectorized skeleton using the
   /// *main loop* strategy (ie the first pass of vplan execution).
   std::pair<BasicBlock *, Value *>
@@ -773,7 +775,7 @@ protected:
 
   void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II,
                     Value *VectorTripCount, Value *EndValue,
-                    BasicBlock *MiddleBlock, VPlan &Plan,
+                    BasicBlock *MiddleBlock,
                     VPTransformState &State) override {};
 };
 
@@ -789,9 +791,9 @@ public:
       OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
       LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
       BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
-      GeneratedRTChecks &Checks)
+      GeneratedRTChecks &Checks, VPlan &Plan)
       : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
-                                       EPI, LVL, CM, BFI, PSI, Checks) {
+                                       EPI, LVL, CM, BFI, PSI, Checks, Plan) {
     TripCount = EPI.TripCount;
   }
   /// Implements the interface for creating a vectorized skeleton using the
@@ -2751,7 +2753,7 @@ InnerLoopVectorizer::createVectorizedLoopSkeleton(
 void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
                                        const InductionDescriptor &II,
                                        Value *VectorTripCount, Value *EndValue,
-                                       BasicBlock *MiddleBlock, VPlan &Plan,
+                                       BasicBlock *MiddleBlock,
                                        VPTransformState &State) {
   // There are two kinds of external IV usages - those that use the value
   // computed in the last iteration (the PHI) and those that use the penultimate
@@ -2931,11 +2933,10 @@ LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
                                    TargetTransformInfo::TCK_RecipThroughput);
 }
 
-void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State,
-                                            VPlan &Plan) {
+void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
   // Fix widened non-induction PHIs by setting up the PHI operands.
   if (EnableVPlanNativePath)
-    fixNonInductionPHIs(Plan, State);
+    fixNonInductionPHIs(State);
 
   // Forget the original basic block.
   PSE.getSE()->forgetLoop(OrigLoop);
@@ -2966,7 +2967,7 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State,
     for (const auto &Entry : Legal->getInductionVars())
       fixupIVUsers(Entry.first, Entry.second,
                    getOrCreateVectorTripCount(nullptr),
-                   IVEndValues[Entry.first], LoopMiddleBlock, Plan, State);
+                   IVEndValues[Entry.first], LoopMiddleBlock, State);
   }
 
   // Fix live-out phis not already fixed earlier.
@@ -3077,8 +3078,7 @@ void InnerLoopVectorizer::sinkScalarOperands(Instruction *PredInst) {
   } while (Changed);
 }
 
-void InnerLoopVectorizer::fixNonInductionPHIs(VPlan &Plan,
-                                              VPTransformState &State) {
+void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) {
   auto Iter = vp_depth_first_deep(Plan.getEntry());
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
     for (VPRecipeBase &P : VPBB->phis()) {
@@ -7744,7 +7744,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
 
   // 3. Fix the vectorized code: take care of header phi's, live-outs,
   //    predication, updating analyses.
-  ILV.fixVectorizedLoop(State, BestVPlan);
+  ILV.fixVectorizedLoop(State);
 
   ILV.printDebugTracesAtEnd();
 
@@ -9727,7 +9727,7 @@ static bool processLoopInVPlanNativePath(
     GeneratedRTChecks Checks(PSE, DT, LI, TTI, F->getDataLayout(),
                              AddBranchWeights);
     InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width,
-                           VF.Width, 1, LVL, &CM, BFI, PSI, Checks);
+                           VF.Width, 1, LVL, &CM, BFI, PSI, Checks, BestPlan);
     LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \""
                       << L->getHeader()->getParent()->getName() << "\"\n");
     LVP.executePlan(VF.Width, 1, BestPlan, LB, DT, false);
@@ -10215,11 +10215,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
       assert(IC > 1 && "interleave count should not be 1 or 0");
       // If we decided that it is not legal to vectorize the loop, then
       // interleave it.
+      VPlan &BestPlan = LVP.getPlanFor(VF.Width);
       InnerLoopVectorizer Unroller(
           L, PSE, LI, DT, TLI, TTI, AC, ORE, ElementCount::getFixed(1),
-          ElementCount::getFixed(1), IC, &LVL, &CM, BFI, PSI, Checks);
+          ElementCount::getFixed(1), IC, &LVL, &CM, BFI, PSI, Checks, BestPlan);
 
-      VPlan &BestPlan = LVP.getPlanFor(VF.Width);
       LVP.executePlan(VF.Width, IC, BestPlan, Unroller, DT, false);
 
       ORE->emit([&]() {
@@ -10236,15 +10236,16 @@ bool LoopVectorizePass::processLoop(Loop *L) {
       VectorizationFactor EpilogueVF =
           LVP.selectEpilogueVectorizationFactor(VF.Width, IC);
       if (EpilogueVF.Width.isVector()) {
+        std::unique_ptr<VPlan> BestMainPlan(BestPlan.duplicate());
 
         // The first pass vectorizes the main loop and creates a scalar epilogue
         // to be vectorized by executing the plan (potentially with a different
         // factor) again shortly afterwards.
         EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF.Width, 1);
         EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TLI, TTI, AC, ORE,
-                                           EPI, &LVL, &CM, BFI, PSI, Checks);
+                                           EPI, &LVL, &CM, BFI, PSI, Checks,
+                                           *BestMainPlan);
 
-        std::unique_ptr<VPlan> BestMainPlan(BestPlan.duplicate());
         auto ExpandedSCEVs = LVP.executePlan(EPI.MainLoopVF, EPI.MainLoopUF,
                                              *BestMainPlan, MainILV, DT, false);
         ++LoopsVectorized;
@@ -10253,11 +10254,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
         // edges from the first pass.
         EPI.MainLoopVF = EPI.EpilogueVF;
         EPI.MainLoopUF = EPI.EpilogueUF;
+        VPlan &BestEpiPlan = LVP.getPlanFor(EPI.EpilogueVF);
         EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TLI, TTI, AC,
                                                  ORE, EPI, &LVL, &CM, BFI, PSI,
-                                                 Checks);
+                                                 Checks, BestEpiPlan);
 
-        VPlan &BestEpiPlan = LVP.getPlanFor(EPI.EpilogueVF);
         VPRegionBlock *VectorLoop = BestEpiPlan.getVectorLoopRegion();
         VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
         Header->setName("vec.epilog.vector.body");
@@ -10340,7 +10341,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
       } else {
         InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width,
                                VF.MinProfitableTripCount, IC, &LVL, &CM, BFI,
-                               PSI, Checks);
+                               PSI, Checks, BestPlan);
         LVP.executePlan(VF.Width, IC, BestPlan, LB, DT, false);
         ++LoopsVectorized;
 
-- 
GitLab


From c3724ba8667c695f29d5af93f2b0d1b23c1b41e7 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 30 Oct 2024 11:46:15 -0700
Subject: [PATCH 626/693] [RISCV] Add OperandType for vector rounding mode
 operands. (#114179)

Use TSFlags to distinquish which type of rounding mode it is. We use the same tablegen base classes for vxrm and frm sometimes so its hard to have different types for different instructions.
---
 .../Target/RISCV/MCTargetDesc/RISCVBaseInfo.h |  4 +++-
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp      |  7 ++++++
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    | 24 +++++++++++--------
 llvm/test/CodeGen/RISCV/rvv/frm-insert.ll     |  6 ++---
 4 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index b3a6cd40ea03..19103e219cb8 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -341,7 +341,9 @@ enum OperandType : unsigned {
   OPERAND_VEC_POLICY,
   // Vector SEW operand.
   OPERAND_SEW,
-  OPERAND_LAST_RISCV_IMM = OPERAND_SEW,
+  // Vector rounding mode for VXRM or FRM.
+  OPERAND_VEC_RM,
+  OPERAND_LAST_RISCV_IMM = OPERAND_VEC_RM,
   // Operand is either a register or uimm5, this is used by V extension pseudo
   // instructions to represent a value that be passed as AVL to either vsetvli
   // or vsetivli.
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index d5b086861d71..688da1ee1b33 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -2551,6 +2551,13 @@ bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
         case RISCVOp::OPERAND_SEW:
           Ok = Imm == 0 || (Imm >= 3 && Imm <= 6);
           break;
+        case RISCVOp::OPERAND_VEC_RM:
+          assert(RISCVII::hasRoundModeOp(Desc.TSFlags));
+          if (RISCVII::usesVXRM(Desc.TSFlags))
+            Ok = isUInt<2>(Imm);
+          else
+            Ok = RISCVFPRndMode::isValidRoundingMode(Imm);
+          break;
         }
         if (!Ok) {
           ErrInfo = "Invalid immediate";
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 5554fda760eb..399a2386d493 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -92,6 +92,10 @@ def sew : RISCVOp {
   let OperandType = "OPERAND_SEW";
 }
 
+def vec_rm : RISCVOp {
+  let OperandType = "OPERAND_VEC_RM";
+}
+
 // X0 has special meaning for vsetvl/vsetvli.
 //  rd | rs1 |   AVL value | Effect on vl
 //--------------------------------------------------------------
@@ -1057,7 +1061,7 @@ class VPseudoUnaryNoMaskRoundingMode<DAGOperand RetClass,
                                      string Constraint = "",
                                      bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
-             (ins RetClass:$passthru, OpClass:$rs2, ixlenimm:$rm,
+             (ins RetClass:$passthru, OpClass:$rs2, vec_rm:$rm,
                   AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
@@ -1097,7 +1101,7 @@ class VPseudoUnaryMaskRoundingMode<VReg RetClass,
                                    bits<2> TargetConstraintType = 1> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru, OpClass:$rs2,
-                  VMaskOp:$vm, ixlenimm:$rm,
+                  VMaskOp:$vm, vec_rm:$rm,
                   AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
@@ -1135,7 +1139,7 @@ class VPseudoUnaryNoMask_FRM<VReg RetClass,
                              string Constraint = "",
                              bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
-             (ins RetClass:$passthru, OpClass:$rs2, ixlenimm:$frm,
+             (ins RetClass:$passthru, OpClass:$rs2, vec_rm:$frm,
                   AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
@@ -1155,7 +1159,7 @@ class VPseudoUnaryMask_FRM<VReg RetClass,
                            bits<2> TargetConstraintType = 1> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru, OpClass:$rs2,
-                  VMaskOp:$vm, ixlenimm:$frm,
+                  VMaskOp:$vm, vec_rm:$frm,
                   AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
@@ -1250,7 +1254,7 @@ class VPseudoBinaryNoMaskRoundingMode<VReg RetClass,
                                       bit UsesVXRM_ = 1,
                                       bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
-             (ins RetClass:$passthru, Op1Class:$rs2, Op2Class:$rs1, ixlenimm:$rm,
+             (ins RetClass:$passthru, Op1Class:$rs2, Op2Class:$rs1, vec_rm:$rm,
                   AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
@@ -1273,7 +1277,7 @@ class VPseudoBinaryMaskPolicyRoundingMode<VReg RetClass,
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   Op1Class:$rs2, Op2Class:$rs1,
-                  VMaskOp:$vm, ixlenimm:$rm, AVL:$vl,
+                  VMaskOp:$vm, vec_rm:$rm, AVL:$vl,
                   sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
@@ -1317,7 +1321,7 @@ class VPseudoTiedBinaryNoMaskRoundingMode<VReg RetClass,
                                           bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$rs2, Op2Class:$rs1,
-                  ixlenimm:$rm,
+                  vec_rm:$rm,
                   AVL:$vl, sew:$sew,
                   vec_policy:$policy), []>,
       RISCVVPseudo {
@@ -1408,7 +1412,7 @@ class VPseudoTernaryMaskPolicyRoundingMode<VReg RetClass,
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   Op1Class:$rs2, Op2Class:$rs1,
                   VMaskOp:$vm,
-                  ixlenimm:$rm,
+                  vec_rm:$rm,
                   AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
@@ -1475,7 +1479,7 @@ class VPseudoTiedBinaryMaskRoundingMode<VReg RetClass,
              (ins GetVRegNoV0<RetClass>.R:$passthru,
                   Op2Class:$rs1,
                   VMaskOp:$vm,
-                  ixlenimm:$rm,
+                  vec_rm:$rm,
                   AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
@@ -1578,7 +1582,7 @@ class VPseudoTernaryNoMaskWithPolicyRoundingMode<VReg RetClass,
                                                  bits<2> TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$rs3, Op1Class:$rs1, Op2Class:$rs2,
-                  ixlenimm:$rm, AVL:$vl, sew:$sew, vec_policy:$policy), []>,
+                  vec_rm:$rm, AVL:$vl, sew:$sew, vec_policy:$policy), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
diff --git a/llvm/test/CodeGen/RISCV/rvv/frm-insert.ll b/llvm/test/CodeGen/RISCV/rvv/frm-insert.ll
index ccfe94ecad28..54f56eadf003 100644
--- a/llvm/test/CodeGen/RISCV/rvv/frm-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/frm-insert.ll
@@ -559,7 +559,7 @@ define <vscale x 1 x float> @after_fsrm3(<vscale x 1 x float> %0, <vscale x 1 x
 ; CHECK-LABEL: after_fsrm3:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    fsrmi 4
-; CHECK-NEXT:    fsrmi a1, 5
+; CHECK-NEXT:    fsrmi a1, 3
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
 ; CHECK-NEXT:    fsrm a1
@@ -568,7 +568,7 @@ define <vscale x 1 x float> @after_fsrm3(<vscale x 1 x float> %0, <vscale x 1 x
 ; UNOPT-LABEL: after_fsrm3:
 ; UNOPT:       # %bb.0: # %entry
 ; UNOPT-NEXT:    fsrmi 4
-; UNOPT-NEXT:    fsrmi a1, 5
+; UNOPT-NEXT:    fsrmi a1, 3
 ; UNOPT-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; UNOPT-NEXT:    vfadd.vv v8, v8, v9
 ; UNOPT-NEXT:    fsrm a1
@@ -579,7 +579,7 @@ entry:
     <vscale x 1 x float> undef,
     <vscale x 1 x float> %0,
     <vscale x 1 x float> %1,
-    i64 5, i64 %2)
+    i64 3, i64 %2)
   ret <vscale x 1 x float> %a
 }
 
-- 
GitLab


From 71b6f6b8a1cd9a63b9d382fe15f40bbb427939b9 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 30 Oct 2024 11:47:40 -0700
Subject: [PATCH 627/693] [RISCV] Add missing hasPostISelHook = 1 to vector
 pseudos that might read FRM. (#114186)

We need an implicit FRM read operand anytime the rounding mode is
dynamic. The post isel hook is responsible for this when isel creates an
instruction with dynamic rounding mode.

Add a MachineVerifier check to verify the operand is present.
---
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp        |  7 +++++++
 llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td | 17 ++++++++---------
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 688da1ee1b33..04bb964bfc48 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -2630,6 +2630,13 @@ bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
     }
   }
 
+  if (int Idx = RISCVII::getFRMOpNum(Desc);
+      Idx >= 0 && MI.getOperand(Idx).getImm() == RISCVFPRndMode::DYN &&
+      !MI.readsRegister(RISCV::FRM, /*TRI=*/nullptr)) {
+    ErrInfo = "dynamic rounding mode should read FRM";
+    return false;
+  }
+
   return true;
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 399a2386d493..d5b0fa340684 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -6483,7 +6483,7 @@ defm PseudoVFRDIV : VPseudoVFRDIV_VF_RM;
 //===----------------------------------------------------------------------===//
 // 13.5. Vector Widening Floating-Point Multiply
 //===----------------------------------------------------------------------===//
-let mayRaiseFPException = true, hasSideEffects = 0 in {
+let mayRaiseFPException = true, hasSideEffects = 0, hasPostISelHook = 1 in {
 defm PseudoVFWMUL : VPseudoVWMUL_VV_VF_RM;
 }
 
@@ -6516,7 +6516,7 @@ defm PseudoVFWMACCBF16  : VPseudoVWMAC_VV_VF_BF_RM;
 //===----------------------------------------------------------------------===//
 // 13.8. Vector Floating-Point Square-Root Instruction
 //===----------------------------------------------------------------------===//
-let mayRaiseFPException = true, hasSideEffects = 0 in
+let mayRaiseFPException = true, hasSideEffects = 0, hasPostISelHook = 1 in
 defm PseudoVFSQRT : VPseudoVSQR_V_RM;
 
 //===----------------------------------------------------------------------===//
@@ -6528,7 +6528,7 @@ defm PseudoVFRSQRT7 : VPseudoVRCP_V;
 //===----------------------------------------------------------------------===//
 // 13.10. Vector Floating-Point Reciprocal Estimate Instruction
 //===----------------------------------------------------------------------===//
-let mayRaiseFPException = true, hasSideEffects = 0 in
+let mayRaiseFPException = true, hasSideEffects = 0, hasPostISelHook = 1 in
 defm PseudoVFREC7 : VPseudoVRCP_V_RM;
 
 //===----------------------------------------------------------------------===//
@@ -6640,9 +6640,10 @@ defm PseudoVFNCVT_F_X      : VPseudoVNCVTF_W_RM;
 defm PseudoVFNCVT_RM_F_XU  : VPseudoVNCVTF_RM_W;
 defm PseudoVFNCVT_RM_F_X   : VPseudoVNCVTF_RM_W;
 
-let hasSideEffects = 0, hasPostISelHook = 1 in
+let hasSideEffects = 0, hasPostISelHook = 1 in {
 defm PseudoVFNCVT_F_F      : VPseudoVNCVTD_W_RM;
 defm PseudoVFNCVTBF16_F_F :  VPseudoVNCVTD_W_RM;
+}
 
 defm PseudoVFNCVT_ROD_F_F  : VPseudoVNCVTD_W;
 } // mayRaiseFPException = true
@@ -6678,8 +6679,7 @@ let Predicates = [HasVInstructionsAnyF] in {
 //===----------------------------------------------------------------------===//
 // 14.3. Vector Single-Width Floating-Point Reduction Instructions
 //===----------------------------------------------------------------------===//
-let mayRaiseFPException = true,
-    hasSideEffects = 0 in {
+let mayRaiseFPException = true, hasSideEffects = 0, hasPostISelHook = 1 in {
 defm PseudoVFREDOSUM : VPseudoVFREDO_VS_RM;
 defm PseudoVFREDUSUM : VPseudoVFRED_VS_RM;
 }
@@ -6691,9 +6691,8 @@ defm PseudoVFREDMAX  : VPseudoVFREDMINMAX_VS;
 //===----------------------------------------------------------------------===//
 // 14.4. Vector Widening Floating-Point Reduction Instructions
 //===----------------------------------------------------------------------===//
-let IsRVVWideningReduction = 1,
-    hasSideEffects = 0,
-    mayRaiseFPException = true in {
+let IsRVVWideningReduction = 1, hasSideEffects = 0, mayRaiseFPException = true,
+    hasPostISelHook = 1 in {
 defm PseudoVFWREDUSUM  : VPseudoVFWRED_VS_RM;
 defm PseudoVFWREDOSUM  : VPseudoVFWREDO_VS_RM;
 }
-- 
GitLab


From 90786adade22784a52856a0e8b545ec6710b47f6 Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Wed, 30 Oct 2024 12:50:40 -0600
Subject: [PATCH 628/693] [Clang][Sema] Always use latest redeclaration of
 primary template (#114258)

This patch fixes a couple of regressions introduced in #111852.

Consider:

```
template<typename T>
struct A
{
    template<bool U>
    static constexpr bool f() requires U
    {
        return true;
    }
};

template<>
template<bool U>
constexpr bool A<short>::f() requires U
{
    return A<long>::f<U>();
}

template<>
template<bool U>
constexpr bool A<long>::f() requires U
{
    return true;
}

static_assert(A<short>::f<true>()); // crash here
```

This crashes because when collecting template arguments from the _first_
declaration of `A<long>::f<true>` for constraint checking, we don't add
the template arguments from the enclosing class template specialization
because there exists another redeclaration that is a member
specialization.

This also fixes the following example, which happens for a similar
reason:
```
// input.cppm

export module input;

export template<int N>
constexpr int f();

template<int N>
struct A {
  template<int J>
  friend constexpr int f();
};

template struct A<0>;

template<int N>
constexpr int f() {
  return N;
}
```

```
// input.cpp

import input;

static_assert(f<1>() == 1); // error: static assertion failed
```
---
 clang/include/clang/AST/DeclTemplate.h        | 52 ++---------
 clang/lib/AST/Decl.cpp                        | 10 +--
 clang/lib/AST/DeclCXX.cpp                     |  4 +-
 clang/lib/AST/DeclTemplate.cpp                | 56 +++++++++++-
 clang/lib/Sema/SemaDecl.cpp                   |  4 +-
 clang/lib/Sema/SemaInit.cpp                   |  2 +-
 clang/lib/Sema/SemaTemplateInstantiate.cpp    | 14 +--
 clang/test/AST/ast-dump-decl.cpp              |  2 +-
 .../CXX/temp/temp.spec/temp.expl.spec/p7.cpp  | 87 +++++++++++++++++++
 9 files changed, 165 insertions(+), 66 deletions(-)

diff --git a/clang/include/clang/AST/DeclTemplate.h b/clang/include/clang/AST/DeclTemplate.h
index a572e3380f16..0ca3fd48e81c 100644
--- a/clang/include/clang/AST/DeclTemplate.h
+++ b/clang/include/clang/AST/DeclTemplate.h
@@ -857,16 +857,6 @@ public:
   /// \endcode
   bool isMemberSpecialization() const { return Common.getInt(); }
 
-  /// Determines whether any redeclaration of this template was
-  /// a specialization of a member template.
-  bool hasMemberSpecialization() const {
-    for (const auto *D : redecls()) {
-      if (D->isMemberSpecialization())
-        return true;
-    }
-    return false;
-  }
-
   /// Note that this member template is a specialization.
   void setMemberSpecialization() {
     assert(!isMemberSpecialization() && "already a member specialization");
@@ -1965,13 +1955,7 @@ public:
   /// specialization which was specialized by this.
   llvm::PointerUnion<ClassTemplateDecl *,
                      ClassTemplatePartialSpecializationDecl *>
-  getSpecializedTemplateOrPartial() const {
-    if (const auto *PartialSpec =
-            SpecializedTemplate.dyn_cast<SpecializedPartialSpecialization *>())
-      return PartialSpec->PartialSpecialization;
-
-    return SpecializedTemplate.get<ClassTemplateDecl*>();
-  }
+  getSpecializedTemplateOrPartial() const;
 
   /// Retrieve the set of template arguments that should be used
   /// to instantiate members of the class template or class template partial
@@ -2208,17 +2192,6 @@ public:
     return InstantiatedFromMember.getInt();
   }
 
-  /// Determines whether any redeclaration of this this class template partial
-  /// specialization was a specialization of a member partial specialization.
-  bool hasMemberSpecialization() const {
-    for (const auto *D : redecls()) {
-      if (cast<ClassTemplatePartialSpecializationDecl>(D)
-              ->isMemberSpecialization())
-        return true;
-    }
-    return false;
-  }
-
   /// Note that this member template is a specialization.
   void setMemberSpecialization() { return InstantiatedFromMember.setInt(true); }
 
@@ -2740,13 +2713,7 @@ public:
   /// Retrieve the variable template or variable template partial
   /// specialization which was specialized by this.
   llvm::PointerUnion<VarTemplateDecl *, VarTemplatePartialSpecializationDecl *>
-  getSpecializedTemplateOrPartial() const {
-    if (const auto *PartialSpec =
-            SpecializedTemplate.dyn_cast<SpecializedPartialSpecialization *>())
-      return PartialSpec->PartialSpecialization;
-
-    return SpecializedTemplate.get<VarTemplateDecl *>();
-  }
+  getSpecializedTemplateOrPartial() const;
 
   /// Retrieve the set of template arguments that should be used
   /// to instantiate the initializer of the variable template or variable
@@ -2980,18 +2947,6 @@ public:
     return InstantiatedFromMember.getInt();
   }
 
-  /// Determines whether any redeclaration of this this variable template
-  /// partial specialization was a specialization of a member partial
-  /// specialization.
-  bool hasMemberSpecialization() const {
-    for (const auto *D : redecls()) {
-      if (cast<VarTemplatePartialSpecializationDecl>(D)
-              ->isMemberSpecialization())
-        return true;
-    }
-    return false;
-  }
-
   /// Note that this member template is a specialization.
   void setMemberSpecialization() { return InstantiatedFromMember.setInt(true); }
 
@@ -3164,6 +3119,9 @@ public:
     return makeSpecIterator(getSpecializations(), true);
   }
 
+  /// Merge \p Prev with our RedeclarableTemplateDecl::Common.
+  void mergePrevDecl(VarTemplateDecl *Prev);
+
   // Implement isa/cast/dyncast support
   static bool classof(const Decl *D) { return classofKind(D->getKind()); }
   static bool classofKind(Kind K) { return K == VarTemplate; }
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 86913763ef9f..cd173d172637 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -2708,7 +2708,7 @@ VarDecl *VarDecl::getTemplateInstantiationPattern() const {
     if (isTemplateInstantiation(VDTemplSpec->getTemplateSpecializationKind())) {
       auto From = VDTemplSpec->getInstantiatedFrom();
       if (auto *VTD = From.dyn_cast<VarTemplateDecl *>()) {
-        while (!VTD->hasMemberSpecialization()) {
+        while (!VTD->isMemberSpecialization()) {
           if (auto *NewVTD = VTD->getInstantiatedFromMemberTemplate())
             VTD = NewVTD;
           else
@@ -2718,7 +2718,7 @@ VarDecl *VarDecl::getTemplateInstantiationPattern() const {
       }
       if (auto *VTPSD =
               From.dyn_cast<VarTemplatePartialSpecializationDecl *>()) {
-        while (!VTPSD->hasMemberSpecialization()) {
+        while (!VTPSD->isMemberSpecialization()) {
           if (auto *NewVTPSD = VTPSD->getInstantiatedFromMember())
             VTPSD = NewVTPSD;
           else
@@ -2732,7 +2732,7 @@ VarDecl *VarDecl::getTemplateInstantiationPattern() const {
   // If this is the pattern of a variable template, find where it was
   // instantiated from. FIXME: Is this necessary?
   if (VarTemplateDecl *VTD = VD->getDescribedVarTemplate()) {
-    while (!VTD->hasMemberSpecialization()) {
+    while (!VTD->isMemberSpecialization()) {
       if (auto *NewVTD = VTD->getInstantiatedFromMemberTemplate())
         VTD = NewVTD;
       else
@@ -4153,7 +4153,7 @@ FunctionDecl::getTemplateInstantiationPattern(bool ForDefinition) const {
   if (FunctionTemplateDecl *Primary = getPrimaryTemplate()) {
     // If we hit a point where the user provided a specialization of this
     // template, we're done looking.
-    while (!ForDefinition || !Primary->hasMemberSpecialization()) {
+    while (!ForDefinition || !Primary->isMemberSpecialization()) {
       if (auto *NewPrimary = Primary->getInstantiatedFromMemberTemplate())
         Primary = NewPrimary;
       else
@@ -4170,7 +4170,7 @@ FunctionTemplateDecl *FunctionDecl::getPrimaryTemplate() const {
   if (FunctionTemplateSpecializationInfo *Info
         = TemplateOrSpecialization
             .dyn_cast<FunctionTemplateSpecializationInfo*>()) {
-    return Info->getTemplate();
+    return Info->getTemplate()->getMostRecentDecl();
   }
   return nullptr;
 }
diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp
index db0ea62a2323..1c92fd9e3ff0 100644
--- a/clang/lib/AST/DeclCXX.cpp
+++ b/clang/lib/AST/DeclCXX.cpp
@@ -2030,7 +2030,7 @@ const CXXRecordDecl *CXXRecordDecl::getTemplateInstantiationPattern() const {
   if (auto *TD = dyn_cast<ClassTemplateSpecializationDecl>(this)) {
     auto From = TD->getInstantiatedFrom();
     if (auto *CTD = From.dyn_cast<ClassTemplateDecl *>()) {
-      while (!CTD->hasMemberSpecialization()) {
+      while (!CTD->isMemberSpecialization()) {
         if (auto *NewCTD = CTD->getInstantiatedFromMemberTemplate())
           CTD = NewCTD;
         else
@@ -2040,7 +2040,7 @@ const CXXRecordDecl *CXXRecordDecl::getTemplateInstantiationPattern() const {
     }
     if (auto *CTPSD =
             From.dyn_cast<ClassTemplatePartialSpecializationDecl *>()) {
-      while (!CTPSD->hasMemberSpecialization()) {
+      while (!CTPSD->isMemberSpecialization()) {
         if (auto *NewCTPSD = CTPSD->getInstantiatedFromMemberTemplate())
           CTPSD = NewCTPSD;
         else
diff --git a/clang/lib/AST/DeclTemplate.cpp b/clang/lib/AST/DeclTemplate.cpp
index 755ec72f00bf..1db02d0d0444 100644
--- a/clang/lib/AST/DeclTemplate.cpp
+++ b/clang/lib/AST/DeclTemplate.cpp
@@ -993,7 +993,17 @@ ClassTemplateSpecializationDecl::getSpecializedTemplate() const {
   if (const auto *PartialSpec =
           SpecializedTemplate.dyn_cast<SpecializedPartialSpecialization*>())
     return PartialSpec->PartialSpecialization->getSpecializedTemplate();
-  return SpecializedTemplate.get<ClassTemplateDecl*>();
+  return SpecializedTemplate.get<ClassTemplateDecl *>()->getMostRecentDecl();
+}
+
+llvm::PointerUnion<ClassTemplateDecl *,
+                   ClassTemplatePartialSpecializationDecl *>
+ClassTemplateSpecializationDecl::getSpecializedTemplateOrPartial() const {
+  if (const auto *PartialSpec =
+          SpecializedTemplate.dyn_cast<SpecializedPartialSpecialization *>())
+    return PartialSpec->PartialSpecialization->getMostRecentDecl();
+
+  return SpecializedTemplate.get<ClassTemplateDecl *>()->getMostRecentDecl();
 }
 
 SourceRange
@@ -1283,6 +1293,39 @@ VarTemplateDecl::newCommon(ASTContext &C) const {
   return CommonPtr;
 }
 
+void VarTemplateDecl::mergePrevDecl(VarTemplateDecl *Prev) {
+  // If we haven't created a common pointer yet, then it can just be created
+  // with the usual method.
+  if (!getCommonPtrInternal())
+    return;
+
+  Common *ThisCommon = static_cast<Common *>(getCommonPtrInternal());
+  Common *PrevCommon = nullptr;
+  SmallVector<VarTemplateDecl *, 8> PreviousDecls;
+  for (; Prev; Prev = Prev->getPreviousDecl()) {
+    if (CommonBase *C = Prev->getCommonPtrInternal()) {
+      PrevCommon = static_cast<Common *>(C);
+      break;
+    }
+    PreviousDecls.push_back(Prev);
+  }
+
+  // If the previous redecl chain hasn't created a common pointer yet, then just
+  // use this common pointer.
+  if (!PrevCommon) {
+    for (auto *D : PreviousDecls)
+      D->setCommonPtr(ThisCommon);
+    return;
+  }
+
+  // Ensure we don't leak any important state.
+  assert(ThisCommon->Specializations.empty() &&
+         ThisCommon->PartialSpecializations.empty() &&
+         "Can't merge incompatible declarations!");
+
+  setCommonPtr(PrevCommon);
+}
+
 VarTemplateSpecializationDecl *
 VarTemplateDecl::findSpecialization(ArrayRef<TemplateArgument> Args,
                                     void *&InsertPos) {
@@ -1405,7 +1448,16 @@ VarTemplateDecl *VarTemplateSpecializationDecl::getSpecializedTemplate() const {
   if (const auto *PartialSpec =
           SpecializedTemplate.dyn_cast<SpecializedPartialSpecialization *>())
     return PartialSpec->PartialSpecialization->getSpecializedTemplate();
-  return SpecializedTemplate.get<VarTemplateDecl *>();
+  return SpecializedTemplate.get<VarTemplateDecl *>()->getMostRecentDecl();
+}
+
+llvm::PointerUnion<VarTemplateDecl *, VarTemplatePartialSpecializationDecl *>
+VarTemplateSpecializationDecl::getSpecializedTemplateOrPartial() const {
+  if (const auto *PartialSpec =
+          SpecializedTemplate.dyn_cast<SpecializedPartialSpecialization *>())
+    return PartialSpec->PartialSpecialization->getMostRecentDecl();
+
+  return SpecializedTemplate.get<VarTemplateDecl *>()->getMostRecentDecl();
 }
 
 SourceRange VarTemplateSpecializationDecl::getSourceRange() const {
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index f8e5f3c6d309..3e8b76e8dfd6 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -4696,8 +4696,10 @@ void Sema::MergeVarDecl(VarDecl *New, LookupResult &Previous) {
 
   // Keep a chain of previous declarations.
   New->setPreviousDecl(Old);
-  if (NewTemplate)
+  if (NewTemplate) {
+    NewTemplate->mergePrevDecl(OldTemplate);
     NewTemplate->setPreviousDecl(OldTemplate);
+  }
 
   // Inherit access appropriately.
   New->setAccess(Old->getAccess());
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 573e90aced3e..e2a59f63ccf5 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -9954,7 +9954,7 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
     auto SynthesizeAggrGuide = [&](InitListExpr *ListInit) {
       auto *Pattern = Template;
       while (Pattern->getInstantiatedFromMemberTemplate()) {
-        if (Pattern->hasMemberSpecialization())
+        if (Pattern->isMemberSpecialization())
           break;
         Pattern = Pattern->getInstantiatedFromMemberTemplate();
       }
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index b63063813f1b..de0ec0128905 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -343,7 +343,7 @@ struct TemplateInstantiationArgumentCollecter
       // If this function was instantiated from a specialized member that is
       // a function template, we're done.
       assert(FD->getPrimaryTemplate() && "No function template?");
-      if (FD->getPrimaryTemplate()->hasMemberSpecialization())
+      if (FD->getPrimaryTemplate()->isMemberSpecialization())
         return Done();
 
       // If this function is a generic lambda specialization, we are done.
@@ -442,11 +442,11 @@ struct TemplateInstantiationArgumentCollecter
         Specialized = CTSD->getSpecializedTemplateOrPartial();
     if (auto *CTPSD =
             Specialized.dyn_cast<ClassTemplatePartialSpecializationDecl *>()) {
-      if (CTPSD->hasMemberSpecialization())
+      if (CTPSD->isMemberSpecialization())
         return Done();
     } else {
       auto *CTD = Specialized.get<ClassTemplateDecl *>();
-      if (CTD->hasMemberSpecialization())
+      if (CTD->isMemberSpecialization())
         return Done();
     }
     return UseNextDecl(CTSD);
@@ -478,11 +478,11 @@ struct TemplateInstantiationArgumentCollecter
         Specialized = VTSD->getSpecializedTemplateOrPartial();
     if (auto *VTPSD =
             Specialized.dyn_cast<VarTemplatePartialSpecializationDecl *>()) {
-      if (VTPSD->hasMemberSpecialization())
+      if (VTPSD->isMemberSpecialization())
         return Done();
     } else {
       auto *VTD = Specialized.get<VarTemplateDecl *>();
-      if (VTD->hasMemberSpecialization())
+      if (VTD->isMemberSpecialization())
         return Done();
     }
     return UseNextDecl(VTSD);
@@ -4141,7 +4141,7 @@ getPatternForClassTemplateSpecialization(
   CXXRecordDecl *Pattern = nullptr;
   Specialized = ClassTemplateSpec->getSpecializedTemplateOrPartial();
   if (auto *CTD = Specialized.dyn_cast<ClassTemplateDecl *>()) {
-    while (!CTD->hasMemberSpecialization()) {
+    while (!CTD->isMemberSpecialization()) {
       if (auto *NewCTD = CTD->getInstantiatedFromMemberTemplate())
         CTD = NewCTD;
       else
@@ -4151,7 +4151,7 @@ getPatternForClassTemplateSpecialization(
   } else if (auto *CTPSD =
                  Specialized
                      .dyn_cast<ClassTemplatePartialSpecializationDecl *>()) {
-    while (!CTPSD->hasMemberSpecialization()) {
+    while (!CTPSD->isMemberSpecialization()) {
       if (auto *NewCTPSD = CTPSD->getInstantiatedFromMemberTemplate())
         CTPSD = NewCTPSD;
       else
diff --git a/clang/test/AST/ast-dump-decl.cpp b/clang/test/AST/ast-dump-decl.cpp
index e84241cee922..7b998f20944f 100644
--- a/clang/test/AST/ast-dump-decl.cpp
+++ b/clang/test/AST/ast-dump-decl.cpp
@@ -530,7 +530,7 @@ namespace testCanonicalTemplate {
   // CHECK-NEXT: |   `-ClassTemplateDecl 0x{{.+}} parent 0x{{.+}} <col:5, col:40> col:40 friend_undeclared TestClassTemplate{{$}}
   // CHECK-NEXT: |     |-TemplateTypeParmDecl 0x{{.+}} <col:14, col:23> col:23 typename depth 1 index 0 T2{{$}}
   // CHECK-NEXT: |     `-CXXRecordDecl 0x{{.+}} parent 0x{{.+}} <col:34, col:40> col:40 class TestClassTemplate{{$}}
-  // CHECK-NEXT: `-ClassTemplateSpecializationDecl 0x{{.+}} <line:[[@LINE-19]]:3, line:[[@LINE-17]]:3> line:[[@LINE-19]]:31 class TestClassTemplate definition implicit_instantiation{{$}}
+  // CHECK-NEXT: `-ClassTemplateSpecializationDecl 0x{{.+}} <col:5, col:40> line:[[@LINE-19]]:31 class TestClassTemplate definition implicit_instantiation{{$}}
   // CHECK-NEXT:   |-DefinitionData pass_in_registers empty aggregate standard_layout trivially_copyable pod trivial literal has_constexpr_non_copy_move_ctor can_const_default_init{{$}}
   // CHECK-NEXT:   | |-DefaultConstructor exists trivial constexpr defaulted_is_constexpr{{$}}
   // CHECK-NEXT:   | |-CopyConstructor simple trivial has_const_param implicit_has_const_param{{$}}
diff --git a/clang/test/CXX/temp/temp.spec/temp.expl.spec/p7.cpp b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p7.cpp
index 87127366eb58..e7e4738032f6 100644
--- a/clang/test/CXX/temp/temp.spec/temp.expl.spec/p7.cpp
+++ b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p7.cpp
@@ -177,6 +177,93 @@ namespace Defined {
   static_assert(A<short>::B<int*>::y == 2);
 } // namespace Defined
 
+namespace Constrained {
+  template<typename T>
+  struct A {
+    template<typename U, bool V> requires V
+    static constexpr int f(); // expected-note {{declared here}}
+
+    template<typename U, bool V> requires V
+    static const int x; // expected-note {{declared here}}
+
+    template<typename U, bool V> requires V
+    static const int x<U*, V>; // expected-note {{declared here}}
+
+    template<typename U, bool V> requires V
+    struct B; // expected-note {{template is declared here}}
+
+    template<typename U, bool V> requires V
+    struct B<U*, V>; // expected-note {{template is declared here}}
+  };
+
+  template<>
+  template<typename U, bool V> requires V
+  constexpr int A<short>::f() {
+    return A<long>::f<U, V>();
+  }
+
+  template<>
+  template<typename U, bool V> requires V
+  constexpr int A<short>::x = A<long>::x<U, V>;
+
+  template<>
+  template<typename U, bool V> requires V
+  constexpr int A<short>::x<U*, V> = A<long>::x<U*, V>;
+
+  template<>
+  template<typename U, bool V> requires V
+  struct A<short>::B<U*, V> {
+    static constexpr int y = A<long>::B<U*, V>::y;
+  };
+
+  template<>
+  template<typename U, bool V> requires V
+  struct A<short>::B {
+    static constexpr int y = A<long>::B<U, V>::y;
+  };
+
+  template<>
+  template<typename U, bool V> requires V
+  constexpr int A<long>::f() {
+    return 1;
+  }
+
+  template<>
+  template<typename U, bool V> requires V
+  constexpr int A<long>::x = 1;
+
+  template<>
+  template<typename U, bool V> requires V
+  constexpr int A<long>::x<U*, V> = 2;
+
+  template<>
+  template<typename U, bool V> requires V
+  struct A<long>::B {
+    static constexpr int y = 1;
+  };
+
+  template<>
+  template<typename U, bool V> requires V
+  struct A<long>::B<U*, V> {
+    static constexpr int y = 2;
+  };
+
+  static_assert(A<int>::f<int, true>() == 0); // expected-error {{static assertion expression is not an integral constant expression}}
+                                              // expected-note@-1 {{undefined function 'f<int, true>' cannot be used in a constant expression}}
+  static_assert(A<int>::x<int, true> == 0); // expected-error {{static assertion expression is not an integral constant expression}}
+                                            // expected-note@-1 {{initializer of 'x<int, true>' is unknown}}
+  static_assert(A<int>::x<int*, true> == 0); // expected-error {{static assertion expression is not an integral constant expression}}
+                                             // expected-note@-1 {{initializer of 'x<int *, true>' is unknown}}
+  static_assert(A<int>::B<int, true>::y == 0); // expected-error {{implicit instantiation of undefined template 'Constrained::A<int>::B<int, true>'}}
+  static_assert(A<int>::B<int*, true>::y == 0); // expected-error {{implicit instantiation of undefined template 'Constrained::A<int>::B<int *, true>'}}
+
+  static_assert(A<short>::f<int, true>() == 1);
+  static_assert(A<short>::x<int, true> == 1);
+  static_assert(A<short>::x<int*, true> == 2);
+  static_assert(A<short>::B<int, true>::y == 1);
+  static_assert(A<short>::B<int*, true>::y == 2);
+} // namespace Constrained
+
 namespace Dependent {
   template<int I>
   struct A {
-- 
GitLab


From 47d9db762484afadeca1acb60534b6b88784464a Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Wed, 30 Oct 2024 18:51:16 +0000
Subject: [PATCH 629/693] [AArch64] Add asssembly/disassembly for FMOP4{A,S}
 (widening, 2-way, FP16 to FP32) instructions (#113346)

The new instructions are described in
https://developer.arm.com/documentation/ddi0602/2024-09/SME-Instructions
---
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |   3 +
 llvm/lib/Target/AArch64/SMEInstrFormats.td    |  37 +++
 .../fmop4as-fp16-fp32-widening-diagnostics.s  | 243 ++++++++++++++++++
 .../SME2p2/fmop4as-fp16-fp32-widening.s       | 177 +++++++++++++
 4 files changed, 460 insertions(+)
 create mode 100644 llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-fp32-widening-diagnostics.s
 create mode 100644 llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-fp32-widening.s

diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index d77219fa7a30..7357aa3c1f0d 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -1020,6 +1020,9 @@ let Predicates = [HasSME2p2] in {
 
   defm FMOP4A : sme2_fmop4as_fp32_non_widening<0, "fmop4a">;
   defm FMOP4S : sme2_fmop4as_fp32_non_widening<1, "fmop4s">;
+
+  defm FMOP4A : sme2_fmop4as_fp16_fp32_widening<0, "fmop4a">;
+  defm FMOP4S : sme2_fmop4as_fp16_fp32_widening<1, "fmop4s">;
 } // [HasSME2p2]
 
 let Predicates = [HasSME2p2, HasSMEB16B16] in {
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index 1c5ec0969245..867901ac5d90 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -5528,3 +5528,40 @@ multiclass sme2_fmop4as_fp64_non_widening<bit S, string mnemonic> {
   // Multiple vectors
   def _M2Z2Z_D : sme2_fp64_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_d_mul_r_Lo, ZZ_d_mul_r_Hi>;
 }
+
+class sme2_fp16_fp32_quarter_tile_outer_product<bit M, bit N, bit S, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>
+    : I<(outs TileOp32:$ZAda),
+        (ins TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),
+        mnemonic, "\t$ZAda, $Zn, $Zm",
+        "", []>, Sched<[]> {
+  bits<2> ZAda;
+  bits<3> Zn;
+  bits<3> Zm;
+
+  let Inst{31-21} = 0b10000001001;
+  let Inst{20} = M;
+  let Inst{19-17} = Zm;
+  let Inst{16-10} = 0b0000000;
+  let Inst{9} = N;
+  let Inst{8-6} = Zn;
+  let Inst{5} = 0;
+  let Inst{4} = S;
+  let Inst{3-2} = 0b00;
+  let Inst{1-0} = ZAda;
+
+  let Constraints = "$ZAda = $_ZAda";
+}
+
+multiclass sme2_fmop4as_fp16_fp32_widening<bit S, string mnemonic> {
+  // Single vectors
+  def _MZZ_HtoS : sme2_fp16_fp32_quarter_tile_outer_product<0, 0, S, mnemonic, ZPR16Mul2_Lo, ZPR16Mul2_Hi>;
+
+  // Multiple and single vectors
+  def _M2ZZ_HtoS : sme2_fp16_fp32_quarter_tile_outer_product<0, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZPR16Mul2_Hi>;
+
+  // Single and multiple vectors
+  def _MZ2Z_HtoS : sme2_fp16_fp32_quarter_tile_outer_product<1, 0, S, mnemonic, ZPR16Mul2_Lo, ZZ_h_mul_r_Hi>;
+
+  // Multiple vectors
+  def _M2Z2Z_HtoS : sme2_fp16_fp32_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi>;
+}
diff --git a/llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-fp32-widening-diagnostics.s b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-fp32-widening-diagnostics.s
new file mode 100644
index 000000000000..457add20355e
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-fp32-widening-diagnostics.s
@@ -0,0 +1,243 @@
+// RUN: not llvm-mc -triple=aarch64 -mattr=+sme2p2 < %s 2>&1 | FileCheck %s
+
+// FMOP4A
+
+// Single vectors
+
+fmop4a za0.d, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za4.s, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, z0.d, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register
+
+fmop4a za0.s, z15.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register
+
+fmop4a za0.s, z16.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register
+
+fmop4a za0.s, z0.h, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4a za0.s, z12.h, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4a za0.s, z12.h, z14.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4a za0.s, z12.h, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Single and multiple vectors
+
+fmop4a za0.d, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za4.s, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, z0.d, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register
+
+fmop4a za0.s, z1.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register
+
+fmop4a za0.s, z16.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register
+
+fmop4a za0.s, z0.h, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, z0.h, {z17.h-z18.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, z0.h, {z16.h-z18.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, z0.h, {z12.h-z13.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+// Multiple and single vectors
+
+fmop4a za0.d, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za4.s, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z0.d-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: mismatched register size suffix
+
+fmop4a za0.s, {z1.h-z2.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za4.s, {z0.h-z2.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z16.h-z17.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, {z0.h-z1.h}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4a za0.s, {z0.h-z1.h}, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4a za0.s, {z0.h-z1.h}, z12.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Multiple vectors
+
+fmop4a za0.d, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4a za4.s, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z0.d-z1.d}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z1.h-z2.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, {z0.h-z2.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z18.h-z19.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, {z0.h-z1.h}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z0.h-z1.h}, {z19.h-z20.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4a za0.s, {z0.h-z1.h}, {z18.h-z20.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4a za0.s, {z0.h-z1.h}, {z10.h-z11.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+// FMOP4S
+
+// Single vectors
+
+fmop4a za0.d, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4s za4.s, z0.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, z0.d, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register
+
+fmop4s za0.s, z15.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register
+
+fmop4s za0.s, z16.h, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register
+
+fmop4s za0.s, z0.h, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4s za0.s, z12.h, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4s za0.s, z12.h, z14.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4s za0.s, z12.h, z31.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Single and multiple vectors
+
+fmop4s za0.d, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4s za4.s, z0.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, z0.d, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register
+
+fmop4s za0.s, z1.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register
+
+fmop4s za0.s, z16.h, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register
+
+fmop4s za0.s, z0.h, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, z0.h, {z17.h-z18.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.s, z0.h, {z16.h-z18.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, z0.h, {z12.h-z13.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+// Multiple and single vectors
+
+fmop4s za0.d, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4s za4.s, {z0.h-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, {z0.d-z1.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: mismatched register size suffix
+
+fmop4s za0.s, {z1.h-z2.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.s, {z0.h-z2.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, {z16.h-z17.h}, z16.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.s, {z0.h-z1.h}, z16.d
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4s za0.s, {z0.h-z1.h}, z17.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+fmop4s za0.s, {z0.h-z1.h}, z12.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected even register in z16.h..z30.h
+
+// Multiple vectors
+
+fmop4s za0.d, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid matrix operand
+
+fmop4s za4.s, {z0.h-z1.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, {z0.d-z1.d}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, {z1.h-z2.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.s, {z0.h-z2.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, {z18.h-z19.h}, {z16.h-z17.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z0-z14, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.s, {z0.h-z1.h}, {z16.d-z17.d}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, {z0.h-z1.h}, {z19.h-z20.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
+
+fmop4s za0.s, {z0.h-z1.h}, {z18.h-z20.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+fmop4s za0.s, {z0.h-z1.h}, {z10.h-z11.h}
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid vector list, expected list with 2 consecutive SVE vectors in the range z16-z30, where the first vector is a multiple of 2 and with matching element types
diff --git a/llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-fp32-widening.s b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-fp32-widening.s
new file mode 100644
index 000000000000..d615fb85b4fd
--- /dev/null
+++ b/llvm/test/MC/AArch64/SME2p2/fmop4as-fp16-fp32-widening.s
@@ -0,0 +1,177 @@
+
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sme2p2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p2 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2p2 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2p2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+// FMOP4A
+
+// Single vectors
+fmop4a  za0.s, z0.h, z16.h  // 10000001-00100000-00000000-00000000
+// CHECK-INST: fmop4a  za0.s, z0.h, z16.h
+// CHECK-ENCODING: [0x00,0x00,0x20,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81200000 <unknown>
+
+fmop4a  za1.s, z10.h, z20.h  // 10000001-00100100-00000001-01000001
+// CHECK-INST: fmop4a  za1.s, z10.h, z20.h
+// CHECK-ENCODING: [0x41,0x01,0x24,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81240141 <unknown>
+
+fmop4a  za3.s, z14.h, z30.h  // 10000001-00101110-00000001-11000011
+// CHECK-INST: fmop4a  za3.s, z14.h, z30.h
+// CHECK-ENCODING: [0xc3,0x01,0x2e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 812e01c3 <unknown>
+
+// Single and multiple vectors
+
+fmop4a  za0.s, z0.h, {z16.h-z17.h}  // 10000001-00110000-00000000-00000000
+// CHECK-INST: fmop4a  za0.s, z0.h, { z16.h, z17.h }
+// CHECK-ENCODING: [0x00,0x00,0x30,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81300000 <unknown>
+
+fmop4a  za1.s, z10.h, {z20.h-z21.h}  // 10000001-00110100-00000001-01000001
+// CHECK-INST: fmop4a  za1.s, z10.h, { z20.h, z21.h }
+// CHECK-ENCODING: [0x41,0x01,0x34,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81340141 <unknown>
+
+fmop4a  za3.s, z14.h, {z30.h-z31.h}  // 10000001-00111110-00000001-11000011
+// CHECK-INST: fmop4a  za3.s, z14.h, { z30.h, z31.h }
+// CHECK-ENCODING: [0xc3,0x01,0x3e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 813e01c3 <unknown>
+
+// Multiple and single vectors
+
+fmop4a  za0.s, {z0.h-z1.h}, z16.h  // 10000001-00100000-00000010-00000000
+// CHECK-INST: fmop4a  za0.s, { z0.h, z1.h }, z16.h
+// CHECK-ENCODING: [0x00,0x02,0x20,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81200200 <unknown>
+
+fmop4a  za1.s, {z10.h-z11.h}, z20.h  // 10000001-00100100-00000011-01000001
+// CHECK-INST: fmop4a  za1.s, { z10.h, z11.h }, z20.h
+// CHECK-ENCODING: [0x41,0x03,0x24,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81240341 <unknown>
+
+fmop4a  za3.s, {z14.h-z15.h}, z30.h  // 10000001-00101110-00000011-11000011
+// CHECK-INST: fmop4a  za3.s, { z14.h, z15.h }, z30.h
+// CHECK-ENCODING: [0xc3,0x03,0x2e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 812e03c3 <unknown>
+
+// Multiple vectors
+
+fmop4a  za0.s, {z0.h-z1.h}, {z16.h-z17.h}  // 10000001-00110000-00000010-00000000
+// CHECK-INST: fmop4a  za0.s, { z0.h, z1.h }, { z16.h, z17.h }
+// CHECK-ENCODING: [0x00,0x02,0x30,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81300200 <unknown>
+
+fmop4a  za1.s, {z10.h-z11.h}, {z20.h-z21.h}  // 10000001-00110100-00000011-01000001
+// CHECK-INST: fmop4a  za1.s, { z10.h, z11.h }, { z20.h, z21.h }
+// CHECK-ENCODING: [0x41,0x03,0x34,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81340341 <unknown>
+
+fmop4a  za3.s, {z14.h-z15.h}, {z30.h-z31.h}  // 10000001-00111110-00000011-11000011
+// CHECK-INST: fmop4a  za3.s, { z14.h, z15.h }, { z30.h, z31.h }
+// CHECK-ENCODING: [0xc3,0x03,0x3e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 813e03c3 <unknown>
+
+// FMOP4S
+
+// Single vectors
+fmop4s  za0.s, z0.h, z16.h  // 10000001-00100000-00000000-00010000
+// CHECK-INST: fmop4s  za0.s, z0.h, z16.h
+// CHECK-ENCODING: [0x10,0x00,0x20,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81200010 <unknown>
+
+fmop4s  za1.s, z10.h, z20.h  // 10000001-00100100-00000001-01010001
+// CHECK-INST: fmop4s  za1.s, z10.h, z20.h
+// CHECK-ENCODING: [0x51,0x01,0x24,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81240151 <unknown>
+
+fmop4s  za3.s, z14.h, z30.h  // 10000001-00101110-00000001-11010011
+// CHECK-INST: fmop4s  za3.s, z14.h, z30.h
+// CHECK-ENCODING: [0xd3,0x01,0x2e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 812e01d3 <unknown>
+
+// Single and multiple vectors
+
+fmop4s  za0.s, z0.h, {z16.h-z17.h}  // 10000001-00110000-00000000-00010000
+// CHECK-INST: fmop4s  za0.s, z0.h, { z16.h, z17.h }
+// CHECK-ENCODING: [0x10,0x00,0x30,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81300010 <unknown>
+
+fmop4s  za1.s, z10.h, {z20.h-z21.h}  // 10000001-00110100-00000001-01010001
+// CHECK-INST: fmop4s  za1.s, z10.h, { z20.h, z21.h }
+// CHECK-ENCODING: [0x51,0x01,0x34,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81340151 <unknown>
+
+fmop4s  za3.s, z14.h, {z30.h-z31.h}  // 10000001-00111110-00000001-11010011
+// CHECK-INST: fmop4s  za3.s, z14.h, { z30.h, z31.h }
+// CHECK-ENCODING: [0xd3,0x01,0x3e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 813e01d3 <unknown>
+
+// Multiple and single vectors
+
+fmop4s  za0.s, {z0.h-z1.h}, z16.h  // 10000001-00100000-00000010-00010000
+// CHECK-INST: fmop4s  za0.s, { z0.h, z1.h }, z16.h
+// CHECK-ENCODING: [0x10,0x02,0x20,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81200210 <unknown>
+
+fmop4s  za1.s, {z10.h-z11.h}, z20.h  // 10000001-00100100-00000011-01010001
+// CHECK-INST: fmop4s  za1.s, { z10.h, z11.h }, z20.h
+// CHECK-ENCODING: [0x51,0x03,0x24,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81240351 <unknown>
+
+fmop4s  za3.s, {z14.h-z15.h}, z30.h  // 10000001-00101110-00000011-11010011
+// CHECK-INST: fmop4s  za3.s, { z14.h, z15.h }, z30.h
+// CHECK-ENCODING: [0xd3,0x03,0x2e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 812e03d3 <unknown>
+
+// Multiple vectors
+
+fmop4s  za0.s, {z0.h-z1.h}, {z16.h-z17.h}  // 10000001-00110000-00000010-00010000
+// CHECK-INST: fmop4s  za0.s, { z0.h, z1.h }, { z16.h, z17.h }
+// CHECK-ENCODING: [0x10,0x02,0x30,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81300210 <unknown>
+
+fmop4s  za1.s, {z10.h-z11.h}, {z20.h-z21.h}  // 10000001-00110100-00000011-01010001
+// CHECK-INST: fmop4s  za1.s, { z10.h, z11.h }, { z20.h, z21.h }
+// CHECK-ENCODING: [0x51,0x03,0x34,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 81340351 <unknown>
+
+fmop4s  za3.s, {z14.h-z15.h}, {z30.h-z31.h}  // 10000001-00111110-00000011-11010011
+// CHECK-INST: fmop4s  za3.s, { z14.h, z15.h }, { z30.h, z31.h }
+// CHECK-ENCODING: [0xd3,0x03,0x3e,0x81]
+// CHECK-ERROR: instruction requires: sme2p2
+// CHECK-UNKNOWN: 813e03d3 <unknown>
-- 
GitLab


From 408c84f35b8b0338b630a6ee313c14238e62b5e6 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 30 Oct 2024 11:52:49 -0700
Subject: [PATCH 630/693] [RISCV] Add hasPostISelHook to sf.vfnrclip pseudo
 instructions. (#114274)

Add Uses = [FRM] to the underlying MC instructions.

Tweak a couple test cases so the MachineVerifier would have caught this.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td         | 5 +++--
 llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_x_f_qf.ll  | 4 +---
 llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_xu_f_qf.ll | 4 +---
 3 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
index 81467ada0044..1ad3e1b68146 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
@@ -222,7 +222,8 @@ let Predicates = [HasVendorXSfvfwmaccqqq], DecoderNamespace = "XSfvfwmaccqqq",
   def VFWMACC_4x4x4 : CustomSiFiveVMACC<0b111100, OPFVV, "sf.vfwmacc.4x4x4">;
 }
 
-let Predicates = [HasVendorXSfvfnrclipxfqf], DecoderNamespace = "XSfvfnrclipxfqf" in {
+let Predicates = [HasVendorXSfvfnrclipxfqf], DecoderNamespace = "XSfvfnrclipxfqf",
+    Uses = [FRM] in {
   def VFNRCLIP_XU_F_QF : CustomSiFiveVFNRCLIP<0b100010, OPFVF, "sf.vfnrclip.xu.f.qf">;
   def VFNRCLIP_X_F_QF : CustomSiFiveVFNRCLIP<0b100011, OPFVF, "sf.vfnrclip.x.f.qf">;
 }
@@ -405,7 +406,7 @@ multiclass VPseudoSiFiveVFWMACC<string Constraint = ""> {
 
 multiclass VPseudoSiFiveVFNRCLIP<string Constraint = "@earlyclobber $rd"> {
   foreach i = 0-4 in
-    let hasSideEffects = 0 in
+    let hasSideEffects = 0, hasPostISelHook = 1 in
       defm "Pseudo" # NAME : VPseudoBinaryRoundingMode<MxListW[i].vrclass,
                                                        MxListVF4[i].vrclass,
                                                        FPR32, MxListW[i],
diff --git a/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_x_f_qf.ll b/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_x_f_qf.ll
index 3c19616576f5..fbe1a97c201c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_x_f_qf.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_x_f_qf.ll
@@ -13,10 +13,8 @@ declare <vscale x 1 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.nxv1i8.nxv1f32.iXLen(
 define <vscale x 1 x i8> @intrinsic_sf_vfnrclip_x_f_qf_nxv1i8_nxv1f32(<vscale x 1 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_x_f_qf_nxv1i8_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    sf.vfnrclip.x.f.qf v9, v8, fa0
-; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -24,7 +22,7 @@ entry:
     <vscale x 1 x i8> undef,
     <vscale x 1 x float> %0,
     float %1,
-    iXLen 0, iXLen %2)
+    iXLen 7, iXLen %2)
 
   ret <vscale x 1 x i8> %a
 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_xu_f_qf.ll b/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_xu_f_qf.ll
index dbcee311c6e3..dfb0ccd982e8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_xu_f_qf.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_xu_f_qf.ll
@@ -13,10 +13,8 @@ declare <vscale x 1 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.nxv1i8.nxv1f32.iXLen(
 define <vscale x 1 x i8> @intrinsic_sf_vfnrclip_xu_f_qf_nxv1i8_nxv1f32(<vscale x 1 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_xu_f_qf_nxv1i8_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    sf.vfnrclip.xu.f.qf v9, v8, fa0
-; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -24,7 +22,7 @@ entry:
     <vscale x 1 x i8> undef,
     <vscale x 1 x float> %0,
     float %1,
-    iXLen 0, iXLen %2)
+    iXLen 7, iXLen %2)
 
   ret <vscale x 1 x i8> %a
 }
-- 
GitLab


From b3bb6f18bb5b2b8756b585b80d46d13ab3636a18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Wed, 30 Oct 2024 19:56:25 +0100
Subject: [PATCH 631/693] [GlobalISel] Import samesign flag (#114267)

Credits: https://github.com/llvm/llvm-project/pull/111419

Fixes icmp-flags.mir

First attempt: https://github.com/llvm/llvm-project/pull/113090

Revert: https://github.com/llvm/llvm-project/pull/114256
---
 .../CodeGen/GlobalISel/GenericMachineInstrs.h |  2 +-
 .../CodeGen/GlobalISel/MachineIRBuilder.h     |  3 +-
 llvm/include/llvm/CodeGen/MachineInstr.h      |  1 +
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |  9 +--
 .../CodeGen/GlobalISel/MachineIRBuilder.cpp   |  5 +-
 llvm/lib/CodeGen/MIRParser/MILexer.cpp        |  1 +
 llvm/lib/CodeGen/MIRParser/MILexer.h          |  1 +
 llvm/lib/CodeGen/MIRParser/MIParser.cpp       |  5 +-
 llvm/lib/CodeGen/MIRPrinter.cpp               |  2 +
 llvm/lib/CodeGen/MachineInstr.cpp             |  7 ++
 .../CodeGen/AArch64/GlobalISel/icmp-flags.mir | 45 ++++++++++++
 .../GlobalISel/irtranslater-samesign.ll       | 69 +++++++++++++++++++
 12 files changed, 139 insertions(+), 11 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/icmp-flags.mir
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/irtranslater-samesign.ll

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index b6309a9ea0ec..cd7ebcf54c9e 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -28,7 +28,7 @@ namespace llvm {
 class GenericMachineInstr : public MachineInstr {
   constexpr static unsigned PoisonFlags = NoUWrap | NoSWrap | NoUSWrap |
                                           IsExact | Disjoint | NonNeg |
-                                          FmNoNans | FmNoInfs;
+                                          FmNoNans | FmNoInfs | SameSign;
 
 public:
   GenericMachineInstr() = delete;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index c41e74ec7ebd..14a641512a67 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1266,7 +1266,8 @@ public:
   ///
   /// \return a MachineInstrBuilder for the newly created instruction.
   MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res,
-                                const SrcOp &Op0, const SrcOp &Op1);
+                                const SrcOp &Op0, const SrcOp &Op1,
+                                std::optional<unsigned> Flgs = std::nullopt);
 
   /// Build and insert a \p Res = G_FCMP \p Pred\p Op0, \p Op1
   ///
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index 360517324746..ead6bbe1d5f6 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -119,6 +119,7 @@ public:
     Disjoint = 1 << 19,      // Each bit is zero in at least one of the inputs.
     NoUSWrap = 1 << 20,      // Instruction supports geps
                              // no unsigned signed wrap.
+    SameSign = 1 << 21       // Both operands have the same sign.
   };
 
 private:
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 5381dce58f9e..a87754389cc8 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -340,20 +340,17 @@ bool IRTranslator::translateCompare(const User &U,
   Register Op1 = getOrCreateVReg(*U.getOperand(1));
   Register Res = getOrCreateVReg(U);
   CmpInst::Predicate Pred = CI->getPredicate();
+  uint32_t Flags = MachineInstr::copyFlagsFromInstruction(*CI);
   if (CmpInst::isIntPredicate(Pred))
-    MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
+    MIRBuilder.buildICmp(Pred, Res, Op0, Op1, Flags);
   else if (Pred == CmpInst::FCMP_FALSE)
     MIRBuilder.buildCopy(
         Res, getOrCreateVReg(*Constant::getNullValue(U.getType())));
   else if (Pred == CmpInst::FCMP_TRUE)
     MIRBuilder.buildCopy(
         Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
-  else {
-    uint32_t Flags = 0;
-    if (CI)
-      Flags = MachineInstr::copyFlagsFromInstruction(*CI);
+  else
     MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
-  }
 
   return true;
 }
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 59f2fc633f5d..15b916424784 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -898,8 +898,9 @@ MachineIRBuilder::buildFPTrunc(const DstOp &Res, const SrcOp &Op,
 MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred,
                                                 const DstOp &Res,
                                                 const SrcOp &Op0,
-                                                const SrcOp &Op1) {
-  return buildInstr(TargetOpcode::G_ICMP, Res, {Pred, Op0, Op1});
+                                                const SrcOp &Op1,
+                                                std::optional<unsigned> Flags) {
+  return buildInstr(TargetOpcode::G_ICMP, Res, {Pred, Op0, Op1}, Flags);
 }
 
 MachineInstrBuilder MachineIRBuilder::buildFCmp(CmpInst::Predicate Pred,
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.cpp b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
index 5a3806ce5733..1c450b05f49e 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
@@ -216,6 +216,7 @@ static MIToken::TokenKind getIdentifierKind(StringRef Identifier) {
       .Case("exact", MIToken::kw_exact)
       .Case("nneg", MIToken::kw_nneg)
       .Case("disjoint", MIToken::kw_disjoint)
+      .Case("samesign", MIToken::kw_samesign)
       .Case("nofpexcept", MIToken::kw_nofpexcept)
       .Case("unpredictable", MIToken::kw_unpredictable)
       .Case("debug-location", MIToken::kw_debug_location)
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.h b/llvm/lib/CodeGen/MIRParser/MILexer.h
index 3931da3eaae1..d7cd06759cfb 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.h
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.h
@@ -77,6 +77,7 @@ struct MIToken {
     kw_unpredictable,
     kw_nneg,
     kw_disjoint,
+    kw_samesign,
     kw_debug_location,
     kw_debug_instr_number,
     kw_dbg_instr_ref,
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index 45847b5830da..059814c70f82 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -1476,7 +1476,8 @@ bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) {
          Token.is(MIToken::kw_noconvergent) ||
          Token.is(MIToken::kw_unpredictable) ||
          Token.is(MIToken::kw_nneg) ||
-         Token.is(MIToken::kw_disjoint)) {
+         Token.is(MIToken::kw_disjoint) ||
+         Token.is(MIToken::kw_samesign)) {
     // clang-format on
     // Mine frame and fast math flags
     if (Token.is(MIToken::kw_frame_setup))
@@ -1513,6 +1514,8 @@ bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) {
       Flags |= MachineInstr::NonNeg;
     if (Token.is(MIToken::kw_disjoint))
       Flags |= MachineInstr::Disjoint;
+    if (Token.is(MIToken::kw_samesign))
+      Flags |= MachineInstr::SameSign;
 
     lex();
   }
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index a015cd3c2a55..658bbe0e577e 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -837,6 +837,8 @@ void MIPrinter::print(const MachineInstr &MI) {
     OS << "disjoint ";
   if (MI.getFlag(MachineInstr::NoUSWrap))
     OS << "nusw ";
+  if (MI.getFlag(MachineInstr::SameSign))
+    OS << "samesign ";
 
   OS << TII->getName(MI.getOpcode());
   if (I < E)
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index c1bd0bb5b716..941861da5c56 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -596,6 +596,11 @@ uint32_t MachineInstr::copyFlagsFromInstruction(const Instruction &I) {
       MIFlags |= MachineInstr::MIFlag::Disjoint;
   }
 
+  // Copy the samesign flag.
+  if (const ICmpInst *ICmp = dyn_cast<ICmpInst>(&I))
+    if (ICmp->hasSameSign())
+      MIFlags |= MachineInstr::MIFlag::SameSign;
+
   // Copy the exact flag.
   if (const PossiblyExactOperator *PE = dyn_cast<PossiblyExactOperator>(&I))
     if (PE->isExact())
@@ -1770,6 +1775,8 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST,
     OS << "nneg ";
   if (getFlag(MachineInstr::Disjoint))
     OS << "disjoint ";
+  if (getFlag(MachineInstr::SameSign))
+    OS << "samesign ";
 
   // Print the opcode name.
   if (TII)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/icmp-flags.mir b/llvm/test/CodeGen/AArch64/GlobalISel/icmp-flags.mir
new file mode 100644
index 000000000000..59e4de944041
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/icmp-flags.mir
@@ -0,0 +1,45 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -run-pass=none -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            icmp_samesign
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: icmp_samesign
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %x:_(s32) = COPY $w0
+    ; CHECK-NEXT: %y:_(s32) = COPY $w1
+    ; CHECK-NEXT: %cmp:_(s1) = samesign G_ICMP intpred(eq), %y(s32), %y
+    ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %cmp(s1)
+    ; CHECK-NEXT: $w0 = COPY %zext(s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %x:_(s32) = COPY $w0
+    %y:_(s32) = COPY $w1
+    %cmp:_(s1) = samesign G_ICMP intpred(eq), %y:_(s32), %y:_
+    %zext:_(s32) = G_ZEXT %cmp:_(s1)
+    $w0 = COPY %zext
+    RET_ReallyLR implicit $w0
+...
+---
+name:            icmp_differentsign
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: icmp_differentsign
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %x:_(s32) = COPY $w0
+    ; CHECK-NEXT: %y:_(s32) = COPY $w1
+    ; CHECK-NEXT: %cmp:_(s1) = G_ICMP intpred(eq), %y(s32), %y
+    ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %cmp(s1)
+    ; CHECK-NEXT: $w0 = COPY %zext(s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %x:_(s32) = COPY $w0
+    %y:_(s32) = COPY $w1
+    %cmp:_(s1) = G_ICMP intpred(eq), %y:_(s32), %y:_
+    %zext:_(s32) = G_ZEXT %cmp:_(s1)
+    $w0 = COPY %zext
+    RET_ReallyLR implicit $w0
+---
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslater-samesign.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslater-samesign.ll
new file mode 100644
index 000000000000..0173f92c9822
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslater-samesign.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -global-isel -mtriple=aarch64-linux-gnu -O0 -stop-after=irtranslator < %s | FileCheck %s
+
+
+define <2 x i1> @call_icmp_samesign_vector(<2 x i32> %a, <2 x i32> %b) {
+  ; CHECK-LABEL: name: call_icmp_samesign_vector
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $d0, $d1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+  ; CHECK-NEXT:   %2:_(<2 x s1>) = samesign G_ICMP intpred(ult), [[COPY]](<2 x s32>), [[COPY1]]
+  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(<2 x s32>) = G_ANYEXT %2(<2 x s1>)
+  ; CHECK-NEXT:   $d0 = COPY [[ANYEXT]](<2 x s32>)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $d0
+entry:
+  %result = icmp samesign ult <2 x i32> %a, %b
+  ret <2 x i1> %result
+}
+
+define <2 x i1> @call_icmp_vector(<2 x i32> %a, <2 x i32> %b) {
+  ; CHECK-LABEL: name: call_icmp_vector
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $d0, $d1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:_(<2 x s1>) = G_ICMP intpred(ult), [[COPY]](<2 x s32>), [[COPY1]]
+  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(<2 x s32>) = G_ANYEXT [[ICMP]](<2 x s1>)
+  ; CHECK-NEXT:   $d0 = COPY [[ANYEXT]](<2 x s32>)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $d0
+entry:
+  %result = icmp ult <2 x i32> %a, %b
+  ret <2 x i1> %result
+}
+
+define i1 @call_icmp(i32 %a) {
+  ; CHECK-LABEL: name: call_icmp
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $w0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY]](s32), [[C]]
+  ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:_(s8) = G_ZEXT [[ICMP]](s1)
+  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ZEXT]](s8)
+  ; CHECK-NEXT:   $w0 = COPY [[ANYEXT]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
+entry:
+  %result = icmp ult i32 %a, 3
+  ret i1 %result
+}
+
+define i1 @call_icmp_samesign(i32 %a) {
+  ; CHECK-LABEL: name: call_icmp_samesign
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $w0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; CHECK-NEXT:   %2:_(s1) = samesign G_ICMP intpred(ult), [[COPY]](s32), [[C]]
+  ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:_(s8) = G_ZEXT %2(s1)
+  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ZEXT]](s8)
+  ; CHECK-NEXT:   $w0 = COPY [[ANYEXT]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
+entry:
+  %result = icmp samesign ult i32 %a, 3
+  ret i1 %result
+}
-- 
GitLab


From dc1ff883caf687f00bd916ea997321ac411c73fd Mon Sep 17 00:00:00 2001
From: Nick Desaulniers <nickdesaulniers@users.noreply.github.com>
Date: Wed, 30 Oct 2024 11:56:41 -0700
Subject: [PATCH 632/693] [libc][i386] define MINSIGSTKSZ & SIGSTKSZ (#114249)

Link: #93709
---
 libc/include/llvm-libc-macros/linux/signal-macros.h | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/libc/include/llvm-libc-macros/linux/signal-macros.h b/libc/include/llvm-libc-macros/linux/signal-macros.h
index e379fc41efd0..0b7317ebc9b8 100644
--- a/libc/include/llvm-libc-macros/linux/signal-macros.h
+++ b/libc/include/llvm-libc-macros/linux/signal-macros.h
@@ -76,15 +76,12 @@
 #define SS_ONSTACK 0x1
 #define SS_DISABLE 0x2
 
-#ifdef __x86_64__
+#if defined(__x86_64__) || defined(__i386__) || defined(__riscv)
 #define MINSIGSTKSZ 2048
 #define SIGSTKSZ 8192
 #elif defined(__aarch64__)
 #define MINSIGSTKSZ 5120
 #define SIGSTKSZ 16384
-#elif defined(__riscv)
-#define MINSIGSTKSZ 2048
-#define SIGSTKSZ 8192
 #else
 #error "Signal stack sizes not defined for your platform."
 #endif
-- 
GitLab


From bc79ec0c5bc3fce31448419846c343017ae1c5ad Mon Sep 17 00:00:00 2001
From: Fred Tingaud
 <95592999+frederic-tingaud-sonarsource@users.noreply.github.com>
Date: Wed, 30 Oct 2024 19:57:09 +0100
Subject: [PATCH 633/693] [clang][ASTMatcher] Handle variable templates in
 `isInstantiated` and `isInTemplateInstantiation` matchers (#110666)

Fix `isInstantiated` and `isInTemplateInstantiation` matchers, so they
return true for instantiations of variable templates, and any
declaration in statements contained in such instantiations.
---
 clang/docs/ReleaseNotes.rst                   |  2 +
 clang/include/clang/ASTMatchers/ASTMatchers.h |  9 +++--
 .../ASTMatchers/ASTMatchersNarrowingTest.cpp  | 39 +++++++++++++++++++
 3 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 1a179e63f902..402203f89e23 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -745,6 +745,8 @@ AST Matchers
 
 - Fixed a crash when traverse lambda expr with invalid captures. (#GH106444)
 
+- Fixed ``isInstantiated`` and ``isInTemplateInstantiation`` to also match for variable templates. (#GH110666)
+
 - Ensure ``hasName`` matches template specializations across inline namespaces,
   making `matchesNodeFullSlow` and `matchesNodeFullFast` consistent.
 
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index 54e484d41fb1..c77140842d7a 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -6750,7 +6750,8 @@ AST_POLYMORPHIC_MATCHER(isTemplateInstantiation,
 ///   matches 'A(int) {...};' and 'A(unsigned) {...}'.
 AST_MATCHER_FUNCTION(internal::Matcher<Decl>, isInstantiated) {
   auto IsInstantiation = decl(anyOf(cxxRecordDecl(isTemplateInstantiation()),
-                                    functionDecl(isTemplateInstantiation())));
+                                    functionDecl(isTemplateInstantiation()),
+                                    varDecl(isTemplateInstantiation())));
   return decl(anyOf(IsInstantiation, hasAncestor(IsInstantiation)));
 }
 
@@ -6769,9 +6770,9 @@ AST_MATCHER_FUNCTION(internal::Matcher<Decl>, isInstantiated) {
 ///   will NOT match j += 42; as it's shared between the template definition and
 ///   instantiation.
 AST_MATCHER_FUNCTION(internal::Matcher<Stmt>, isInTemplateInstantiation) {
-  return stmt(
-      hasAncestor(decl(anyOf(cxxRecordDecl(isTemplateInstantiation()),
-                             functionDecl(isTemplateInstantiation())))));
+  return stmt(hasAncestor(decl(anyOf(cxxRecordDecl(isTemplateInstantiation()),
+                                     functionDecl(isTemplateInstantiation()),
+                                     varDecl(isTemplateInstantiation())))));
 }
 
 /// Matches explicit template specializations of function, class, or
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
index d696375547ac..056b7c7b571e 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -3342,6 +3342,45 @@ TEST_P(ASTMatchersTest,
                          declStmt(isInTemplateInstantiation())));
 }
 
+TEST_P(ASTMatchersTest, IsInstantiated_MatchesVariableInstantiation) {
+  if (!GetParam().isCXX14OrLater()) {
+    return;
+  }
+
+  EXPECT_TRUE(matches("template<typename T> int V = 10; void x() { V<int>; }",
+                      varDecl(isInstantiated())));
+}
+
+TEST_P(ASTMatchersTest, IsInstantiated_NotMatchesVariableDefinition) {
+  if (!GetParam().isCXX14OrLater()) {
+    return;
+  }
+
+  EXPECT_TRUE(notMatches("template<typename T> int V = 10;",
+                         varDecl(isInstantiated())));
+}
+
+TEST_P(ASTMatchersTest,
+       IsInTemplateInstantiation_MatchesVariableInstantiationStmt) {
+  if (!GetParam().isCXX14OrLater()) {
+    return;
+  }
+
+  EXPECT_TRUE(matches(
+      "template<typename T> auto V = []() { T i; }; void x() { V<int>(); }",
+      declStmt(isInTemplateInstantiation())));
+}
+
+TEST_P(ASTMatchersTest,
+       IsInTemplateInstantiation_NotMatchesVariableDefinitionStmt) {
+  if (!GetParam().isCXX14OrLater()) {
+    return;
+  }
+
+  EXPECT_TRUE(notMatches("template<typename T> auto V = []() { T i; };",
+                         declStmt(isInTemplateInstantiation())));
+}
+
 TEST_P(ASTMatchersTest, IsInTemplateInstantiation_Sharing) {
   if (!GetParam().isCXX()) {
     return;
-- 
GitLab


From d8295e2eeceef37bfd9e0f84918735eff6cfc659 Mon Sep 17 00:00:00 2001
From: Steven Perron <stevenperron@google.com>
Date: Wed, 30 Oct 2024 15:01:02 -0400
Subject: [PATCH 634/693] [SPIRV][HLSL] Handle arrays of resources (#111564)

This commit adds the ability to get a particular resource from an array
of resources using the handle_fromBinding intrinsic.

The main changes are:

1. Create an array when generating the type.
2. Add capabilities from

[SPV_EXT_descriptor_indexing](https://htmlpreview.github.io/?https://github.com/KhronosGroup/SPIRV-Registry/blob/main/extensions/EXT/SPV_EXT_descriptor_indexing.html).

We are still missing the ability to declare a runtime array. That will
be done in a follow up PR.
---
 llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp |  43 ++++-
 .../Target/SPIRV/SPIRVInstructionSelector.cpp |  43 ++++-
 llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 164 +++++++++++++++++-
 .../lib/Target/SPIRV/SPIRVSymbolicOperands.td |   4 +-
 .../CombinedSamplerImageDynIdx.ll             |  41 +++++
 .../CombinedSamplerImageNonUniformIdx.ll      |  48 +++++
 .../{ => hlsl-resources}/HlslBufferLoad.ll    |   0
 .../InputAttachmentImageDynIdx.ll             |  40 +++++
 .../InputAttachmentImageNonUniformIdx.ll      |  47 +++++
 .../hlsl-resources/SampledImageDynIdx.ll      |  66 +++++++
 .../SampledImageNonUniformIdx.ll              |  47 +++++
 .../hlsl-resources/SamplerArrayDynIdx.ll      |  39 +++++
 .../SamplerArrayNonUniformIdx.ll              |  46 +++++
 .../hlsl-resources/StorageImageDynIdx.ll      |  40 +++++
 .../StorageImageNonUniformIdx.ll              |  47 +++++
 .../StorageTexelBufferDynIdx.ll               |  40 +++++
 .../StorageTexelBufferNonUniformIdx.ll        |  47 +++++
 .../UniformTexelBufferDynIdx.ll               |  40 +++++
 .../UniformTexelBufferNonUniformIdx.ll        |  47 +++++
 19 files changed, 871 insertions(+), 18 deletions(-)
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/CombinedSamplerImageDynIdx.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/CombinedSamplerImageNonUniformIdx.ll
 rename llvm/test/CodeGen/SPIRV/{ => hlsl-resources}/HlslBufferLoad.ll (100%)
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/InputAttachmentImageDynIdx.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/InputAttachmentImageNonUniformIdx.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/SampledImageDynIdx.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/SampledImageNonUniformIdx.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/SamplerArrayDynIdx.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/SamplerArrayNonUniformIdx.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageDynIdx.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageNonUniformIdx.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/StorageTexelBufferDynIdx.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/StorageTexelBufferNonUniformIdx.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/UniformTexelBufferDynIdx.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/UniformTexelBufferNonUniformIdx.ll

diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index 64fde8bf67ab..62bd8d1f9d24 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -713,21 +713,36 @@ Register SPIRVGlobalRegistry::buildGlobalVariable(
   return Reg;
 }
 
+static std::string GetSpirvImageTypeName(const SPIRVType *Type,
+                                         MachineIRBuilder &MIRBuilder,
+                                         const std::string &Prefix);
+
 static std::string buildSpirvTypeName(const SPIRVType *Type,
                                       MachineIRBuilder &MIRBuilder) {
   switch (Type->getOpcode()) {
+  case SPIRV::OpTypeSampledImage: {
+    return GetSpirvImageTypeName(Type, MIRBuilder, "sampled_image_");
+  }
   case SPIRV::OpTypeImage: {
-    Register SampledTypeReg = Type->getOperand(1).getReg();
-    auto *SampledType = MIRBuilder.getMRI()->getUniqueVRegDef(SampledTypeReg);
-    std::string TypeName =
-        "image_" + buildSpirvTypeName(SampledType, MIRBuilder);
-    for (uint32_t I = 2; I < Type->getNumOperands(); ++I) {
-      TypeName = (TypeName + '_' + Twine(Type->getOperand(I).getImm())).str();
-    }
-    return TypeName;
+    return GetSpirvImageTypeName(Type, MIRBuilder, "image_");
+  }
+  case SPIRV::OpTypeArray: {
+    MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+    Register ElementTypeReg = Type->getOperand(1).getReg();
+    auto *ElementType = MRI->getUniqueVRegDef(ElementTypeReg);
+    const SPIRVType *TypeInst = MRI->getVRegDef(Type->getOperand(2).getReg());
+    assert(TypeInst->getOpcode() != SPIRV::OpConstantI);
+    MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg());
+    assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT);
+    uint32_t ArraySize = ImmInst->getOperand(1).getCImm()->getZExtValue();
+    return (buildSpirvTypeName(ElementType, MIRBuilder) + Twine("[") +
+            Twine(ArraySize) + Twine("]"))
+        .str();
   }
   case SPIRV::OpTypeFloat:
     return ("f" + Twine(Type->getOperand(1).getImm())).str();
+  case SPIRV::OpTypeSampler:
+    return ("sampler");
   case SPIRV::OpTypeInt:
     if (Type->getOperand(2).getImm())
       return ("i" + Twine(Type->getOperand(1).getImm())).str();
@@ -737,6 +752,18 @@ static std::string buildSpirvTypeName(const SPIRVType *Type,
   }
 }
 
+static std::string GetSpirvImageTypeName(const SPIRVType *Type,
+                                         MachineIRBuilder &MIRBuilder,
+                                         const std::string &Prefix) {
+  Register SampledTypeReg = Type->getOperand(1).getReg();
+  auto *SampledType = MIRBuilder.getMRI()->getUniqueVRegDef(SampledTypeReg);
+  std::string TypeName = Prefix + buildSpirvTypeName(SampledType, MIRBuilder);
+  for (uint32_t I = 2; I < Type->getNumOperands(); ++I) {
+    TypeName = (TypeName + '_' + Twine(Type->getOperand(I).getImm())).str();
+  }
+  return TypeName;
+}
+
 Register SPIRVGlobalRegistry::getOrCreateGlobalVariableWithBinding(
     const SPIRVType *VarType, uint32_t Set, uint32_t Binding,
     MachineIRBuilder &MIRBuilder) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 11ed7d660be0..526305d7ed28 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -260,6 +260,7 @@ private:
                                            SPIRVType *SrcPtrTy) const;
   Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set,
                                   uint32_t Binding, uint32_t ArraySize,
+                                  Register IndexReg, bool IsNonUniform,
                                   MachineIRBuilder MIRBuilder) const;
 };
 
@@ -2616,10 +2617,15 @@ void SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg,
   uint32_t Set = foldImm(I.getOperand(2), MRI);
   uint32_t Binding = foldImm(I.getOperand(3), MRI);
   uint32_t ArraySize = foldImm(I.getOperand(4), MRI);
+  Register IndexReg = I.getOperand(5).getReg();
+  bool IsNonUniform = ArraySize > 1 && foldImm(I.getOperand(6), MRI);
 
   MachineIRBuilder MIRBuilder(I);
-  Register VarReg =
-      buildPointerToResource(ResType, Set, Binding, ArraySize, MIRBuilder);
+  Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize,
+                                           IndexReg, IsNonUniform, MIRBuilder);
+
+  if (IsNonUniform)
+    buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {});
 
   // TODO: For now we assume the resource is an image, which needs to be
   // loaded to get the handle. That will not be true for storage buffers.
@@ -2631,10 +2637,35 @@ void SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg,
 
 Register SPIRVInstructionSelector::buildPointerToResource(
     const SPIRVType *ResType, uint32_t Set, uint32_t Binding,
-    uint32_t ArraySize, MachineIRBuilder MIRBuilder) const {
-  assert(ArraySize == 1 && "Resource arrays are not implemented yet.");
-  return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding,
-                                                 MIRBuilder);
+    uint32_t ArraySize, Register IndexReg, bool IsNonUniform,
+    MachineIRBuilder MIRBuilder) const {
+  if (ArraySize == 1)
+    return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding,
+                                                   MIRBuilder);
+
+  const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType(
+      ResType, ArraySize, *MIRBuilder.getInsertPt(), TII);
+  Register VarReg = GR.getOrCreateGlobalVariableWithBinding(
+      VarType, Set, Binding, MIRBuilder);
+
+  SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType(
+      ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant);
+
+  Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
+  if (IsNonUniform) {
+    // It is unclear which value needs to be marked an non-uniform, so both
+    // the index and the access changed are decorated as non-uniform.
+    buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {});
+    buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {});
+  }
+
+  MIRBuilder.buildInstr(SPIRV::OpAccessChain)
+      .addDef(AcReg)
+      .addUse(GR.getSPIRVTypeID(ResPointerType))
+      .addUse(VarReg)
+      .addUse(IndexReg);
+
+  return AcReg;
 }
 
 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg,
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index db5463f5c7ab..29ce60d9983e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -689,11 +689,31 @@ void RequirementHandler::initAvailableCapabilitiesForVulkan(
     const SPIRVSubtarget &ST) {
   addAvailableCaps({Capability::Shader, Capability::Linkage});
 
-  // Provided by all supported Vulkan versions.
+  // Core in Vulkan 1.1 and earlier.
   addAvailableCaps({Capability::Int16, Capability::Int64, Capability::Float16,
                     Capability::Float64, Capability::GroupNonUniform,
                     Capability::Image1D, Capability::SampledBuffer,
-                    Capability::ImageBuffer});
+                    Capability::ImageBuffer,
+                    Capability::UniformBufferArrayDynamicIndexing,
+                    Capability::SampledImageArrayDynamicIndexing,
+                    Capability::StorageBufferArrayDynamicIndexing,
+                    Capability::StorageImageArrayDynamicIndexing});
+
+  // Became core in Vulkan 1.2
+  if (ST.isAtLeastSPIRVVer(VersionTuple(1, 5))) {
+    addAvailableCaps(
+        {Capability::ShaderNonUniformEXT, Capability::RuntimeDescriptorArrayEXT,
+         Capability::InputAttachmentArrayDynamicIndexingEXT,
+         Capability::UniformTexelBufferArrayDynamicIndexingEXT,
+         Capability::StorageTexelBufferArrayDynamicIndexingEXT,
+         Capability::UniformBufferArrayNonUniformIndexingEXT,
+         Capability::SampledImageArrayNonUniformIndexingEXT,
+         Capability::StorageBufferArrayNonUniformIndexingEXT,
+         Capability::StorageImageArrayNonUniformIndexingEXT,
+         Capability::InputAttachmentArrayNonUniformIndexingEXT,
+         Capability::UniformTexelBufferArrayNonUniformIndexingEXT,
+         Capability::StorageTexelBufferArrayNonUniformIndexingEXT});
+  }
 }
 
 } // namespace SPIRV
@@ -729,6 +749,8 @@ static void addOpDecorateReqs(const MachineInstr &MI, unsigned DecIndex,
              Dec == SPIRV::Decoration::ImplementInRegisterMapINTEL) {
     Reqs.addExtension(
         SPIRV::Extension::SPV_INTEL_global_variable_fpga_decorations);
+  } else if (Dec == SPIRV::Decoration::NonUniformEXT) {
+    Reqs.addRequirements(SPIRV::Capability::ShaderNonUniformEXT);
   }
 }
 
@@ -848,6 +870,136 @@ static void AddAtomicFloatRequirements(const MachineInstr &MI,
   }
 }
 
+bool isUniformTexelBuffer(MachineInstr *ImageInst) {
+  if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
+    return false;
+  uint32_t Dim = ImageInst->getOperand(2).getImm();
+  uint32_t Sampled = ImageInst->getOperand(6).getImm();
+  return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 1;
+}
+
+bool isStorageTexelBuffer(MachineInstr *ImageInst) {
+  if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
+    return false;
+  uint32_t Dim = ImageInst->getOperand(2).getImm();
+  uint32_t Sampled = ImageInst->getOperand(6).getImm();
+  return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 2;
+}
+
+bool isSampledImage(MachineInstr *ImageInst) {
+  if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
+    return false;
+  uint32_t Dim = ImageInst->getOperand(2).getImm();
+  uint32_t Sampled = ImageInst->getOperand(6).getImm();
+  return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 1;
+}
+
+bool isInputAttachment(MachineInstr *ImageInst) {
+  if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
+    return false;
+  uint32_t Dim = ImageInst->getOperand(2).getImm();
+  uint32_t Sampled = ImageInst->getOperand(6).getImm();
+  return Dim == SPIRV::Dim::DIM_SubpassData && Sampled == 2;
+}
+
+bool isStorageImage(MachineInstr *ImageInst) {
+  if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
+    return false;
+  uint32_t Dim = ImageInst->getOperand(2).getImm();
+  uint32_t Sampled = ImageInst->getOperand(6).getImm();
+  return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 2;
+}
+
+bool isCombinedImageSampler(MachineInstr *SampledImageInst) {
+  if (SampledImageInst->getOpcode() != SPIRV::OpTypeSampledImage)
+    return false;
+
+  const MachineRegisterInfo &MRI = SampledImageInst->getMF()->getRegInfo();
+  Register ImageReg = SampledImageInst->getOperand(1).getReg();
+  auto *ImageInst = MRI.getUniqueVRegDef(ImageReg);
+  return isSampledImage(ImageInst);
+}
+
+bool hasNonUniformDecoration(Register Reg, const MachineRegisterInfo &MRI) {
+  for (const auto &MI : MRI.reg_instructions(Reg)) {
+    if (MI.getOpcode() != SPIRV::OpDecorate)
+      continue;
+
+    uint32_t Dec = MI.getOperand(1).getImm();
+    if (Dec == SPIRV::Decoration::NonUniformEXT)
+      return true;
+  }
+  return false;
+}
+
+void addOpAccessChainReqs(const MachineInstr &Instr,
+                          SPIRV::RequirementHandler &Handler,
+                          const SPIRVSubtarget &Subtarget) {
+  const MachineRegisterInfo &MRI = Instr.getMF()->getRegInfo();
+  // Get the result type. If it is an image type, then the shader uses
+  // descriptor indexing. The appropriate capabilities will be added based
+  // on the specifics of the image.
+  Register ResTypeReg = Instr.getOperand(1).getReg();
+  MachineInstr *ResTypeInst = MRI.getUniqueVRegDef(ResTypeReg);
+
+  assert(ResTypeInst->getOpcode() == SPIRV::OpTypePointer);
+  uint32_t StorageClass = ResTypeInst->getOperand(1).getImm();
+  if (StorageClass != SPIRV::StorageClass::StorageClass::UniformConstant &&
+      StorageClass != SPIRV::StorageClass::StorageClass::Uniform &&
+      StorageClass != SPIRV::StorageClass::StorageClass::StorageBuffer) {
+    return;
+  }
+
+  Register PointeeTypeReg = ResTypeInst->getOperand(2).getReg();
+  MachineInstr *PointeeType = MRI.getUniqueVRegDef(PointeeTypeReg);
+  if (PointeeType->getOpcode() != SPIRV::OpTypeImage &&
+      PointeeType->getOpcode() != SPIRV::OpTypeSampledImage &&
+      PointeeType->getOpcode() != SPIRV::OpTypeSampler) {
+    return;
+  }
+
+  bool IsNonUniform =
+      hasNonUniformDecoration(Instr.getOperand(0).getReg(), MRI);
+  if (isUniformTexelBuffer(PointeeType)) {
+    if (IsNonUniform)
+      Handler.addRequirements(
+          SPIRV::Capability::UniformTexelBufferArrayNonUniformIndexingEXT);
+    else
+      Handler.addRequirements(
+          SPIRV::Capability::UniformTexelBufferArrayDynamicIndexingEXT);
+  } else if (isInputAttachment(PointeeType)) {
+    if (IsNonUniform)
+      Handler.addRequirements(
+          SPIRV::Capability::InputAttachmentArrayNonUniformIndexingEXT);
+    else
+      Handler.addRequirements(
+          SPIRV::Capability::InputAttachmentArrayDynamicIndexingEXT);
+  } else if (isStorageTexelBuffer(PointeeType)) {
+    if (IsNonUniform)
+      Handler.addRequirements(
+          SPIRV::Capability::StorageTexelBufferArrayNonUniformIndexingEXT);
+    else
+      Handler.addRequirements(
+          SPIRV::Capability::StorageTexelBufferArrayDynamicIndexingEXT);
+  } else if (isSampledImage(PointeeType) ||
+             isCombinedImageSampler(PointeeType) ||
+             PointeeType->getOpcode() == SPIRV::OpTypeSampler) {
+    if (IsNonUniform)
+      Handler.addRequirements(
+          SPIRV::Capability::SampledImageArrayNonUniformIndexingEXT);
+    else
+      Handler.addRequirements(
+          SPIRV::Capability::SampledImageArrayDynamicIndexing);
+  } else if (isStorageImage(PointeeType)) {
+    if (IsNonUniform)
+      Handler.addRequirements(
+          SPIRV::Capability::StorageImageArrayNonUniformIndexingEXT);
+    else
+      Handler.addRequirements(
+          SPIRV::Capability::StorageImageArrayDynamicIndexing);
+  }
+}
+
 void addInstrRequirements(const MachineInstr &MI,
                           SPIRV::RequirementHandler &Reqs,
                           const SPIRVSubtarget &ST) {
@@ -967,11 +1119,17 @@ void addInstrRequirements(const MachineInstr &MI,
   case SPIRV::OpConstantSampler:
     Reqs.addCapability(SPIRV::Capability::LiteralSampler);
     break;
+  case SPIRV::OpInBoundsAccessChain:
+  case SPIRV::OpAccessChain:
+    addOpAccessChainReqs(MI, Reqs, ST);
+    break;
   case SPIRV::OpTypeImage:
     addOpTypeImageReqs(MI, Reqs, ST);
     break;
   case SPIRV::OpTypeSampler:
-    Reqs.addCapability(SPIRV::Capability::ImageBasic);
+    if (!ST.isVulkanEnv()) {
+      Reqs.addCapability(SPIRV::Capability::ImageBasic);
+    }
     break;
   case SPIRV::OpTypeForwardPointer:
     // TODO: check if it's OpenCL's kernel.
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index 13ad1eb8e8b3..d63438baca7e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -355,7 +355,9 @@ defm GeometryPointSize : CapabilityOperand<24, 0, 0, [], [Geometry]>;
 defm ImageGatherExtended : CapabilityOperand<25, 0, 0, [], [Shader]>;
 defm StorageImageMultisample : CapabilityOperand<27, 0, 0, [], [Shader]>;
 defm UniformBufferArrayDynamicIndexing : CapabilityOperand<28, 0, 0, [], [Shader]>;
-defm SampledImageArrayDymnamicIndexing : CapabilityOperand<29, 0, 0, [], [Shader]>;
+defm SampledImageArrayDynamicIndexing : CapabilityOperand<29, 0, 0, [], [Shader]>;
+defm StorageBufferArrayDynamicIndexing : CapabilityOperand<30, 0, 0, [], [Shader]>;
+defm StorageImageArrayDynamicIndexing : CapabilityOperand<31, 0, 0, [], [Shader]>;
 defm ClipDistance : CapabilityOperand<32, 0, 0, [], [Shader]>;
 defm CullDistance : CapabilityOperand<33, 0, 0, [], [Shader]>;
 defm SampleRateShading : CapabilityOperand<35, 0, 0, [], [Shader]>;
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/CombinedSamplerImageDynIdx.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/CombinedSamplerImageDynIdx.ll
new file mode 100644
index 000000000000..d5e95c782414
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/CombinedSamplerImageDynIdx.ll
@@ -0,0 +1,41 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpCapability Shader
+; CHECK-NEXT: OpCapability SampledImageArrayDynamicIndexing
+; CHECK-NEXT: OpCapability Sampled1D
+; CHECK-NOT: OpCapability
+
+; CHECK-DAG: OpDecorate [[Var:%[0-9]+]] DescriptorSet 3
+; CHECK-DAG: OpDecorate [[Var]] Binding 4
+
+; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[BufferType:%[0-9]+]] = OpTypeImage [[int]] 1D 2 0 0 1 R32i {{$}}
+; CHECK-DAG: [[CombindedType:%[0-9]+]] = OpTypeSampledImage [[BufferType]]
+; CHECK-DAG: [[BufferPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[CombindedType]]
+; CHECK-DAG: [[ArraySize:%[0-9]+]] = OpConstant [[int]] 3
+; CHECK-DAG: [[One:%[0-9]+]] = OpConstant [[int]] 1
+; CHECK-DAG: [[Zero:%[0-9]+]] = OpConstant [[int]] 0
+; CHECK-DAG: [[BufferArrayType:%[0-9]+]] = OpTypeArray [[CombindedType]] [[ArraySize]]
+; CHECK-DAG: [[ArrayPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferArrayType]]
+; CHECK-DAG: [[Var]] = OpVariable [[ArrayPtrType]] UniformConstant
+
+; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
+; CHECK-NEXT: OpLabel
+define void @main() #0 {
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[Zero]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[CombindedType]] [[ac]]
+  %buffer0 = call target("spirv.SampledImage", i32, 0, 2, 0, 0, 1, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_0_2_0_0_1_24(
+          i32 3, i32 4, i32 3, i32 0, i1 false)
+
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[One]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[CombindedType]] [[ac]]
+  %buffer1 = call target("spirv.SampledImage", i32, 0, 2, 0, 0, 1, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_0_2_0_0_1_24(
+          i32 3, i32 4, i32 3, i32 1, i1 false)
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/CombinedSamplerImageNonUniformIdx.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/CombinedSamplerImageNonUniformIdx.ll
new file mode 100644
index 000000000000..68bf3478fa9a
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/CombinedSamplerImageNonUniformIdx.ll
@@ -0,0 +1,48 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpCapability Shader
+; CHECK: OpCapability ShaderNonUniform
+; CHECK-NEXT: OpCapability SampledImageArrayNonUniformIndexing
+; CHECK-NEXT: OpCapability Sampled1D
+; CHECK-NOT: OpCapability
+
+; CHECK-DAG: OpDecorate [[Var:%[0-9]+]] DescriptorSet 3
+; CHECK-DAG: OpDecorate [[Var]] Binding 4
+; CHECK: OpDecorate [[Zero:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ac0:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ld0:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[One:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ac1:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ld1:%[0-9]+]] NonUniform
+
+; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[BufferType:%[0-9]+]] = OpTypeImage [[int]] 1D 2 0 0 1 R32i {{$}}
+; CHECK-DAG: [[CombindedType:%[0-9]+]] = OpTypeSampledImage [[BufferType]]
+; CHECK-DAG: [[BufferPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[CombindedType]]
+; CHECK-DAG: [[ArraySize:%[0-9]+]] = OpConstant [[int]] 3
+; CHECK-DAG: [[One]] = OpConstant [[int]] 1
+; CHECK-DAG: [[Zero]] = OpConstant [[int]] 0
+; CHECK-DAG: [[BufferArrayType:%[0-9]+]] = OpTypeArray [[CombindedType]] [[ArraySize]]
+; CHECK-DAG: [[ArrayPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferArrayType]]
+; CHECK-DAG: [[Var]] = OpVariable [[ArrayPtrType]] UniformConstant
+
+; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
+; CHECK-NEXT: OpLabel
+define void @main() #0 {
+; CHECK: [[ac0]] = OpAccessChain [[BufferPtrType]] [[Var]] [[Zero]]
+; CHECK: [[ld0:%[0-9]+]] = OpLoad [[CombindedType]] [[ac0]]
+  %buffer0 = call target("spirv.SampledImage", i32, 0, 2, 0, 0, 1, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_0_2_0_0_1_24(
+          i32 3, i32 4, i32 3, i32 0, i1 true)
+
+; CHECK: [[ac1]] = OpAccessChain [[BufferPtrType]] [[Var]] [[One]]
+; CHECK: [[ld1]] = OpLoad [[CombindedType]] [[ac1]]
+  %buffer1 = call target("spirv.SampledImage", i32, 0, 2, 0, 0, 1, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_0_2_0_0_1_24(
+          i32 3, i32 4, i32 3, i32 1, i1 true)
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/SPIRV/HlslBufferLoad.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/HlslBufferLoad.ll
similarity index 100%
rename from llvm/test/CodeGen/SPIRV/HlslBufferLoad.ll
rename to llvm/test/CodeGen/SPIRV/hlsl-resources/HlslBufferLoad.ll
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/InputAttachmentImageDynIdx.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/InputAttachmentImageDynIdx.ll
new file mode 100644
index 000000000000..39fdc866af7f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/InputAttachmentImageDynIdx.ll
@@ -0,0 +1,40 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpCapability Shader
+; CHECK-NEXT: OpCapability InputAttachmentArrayDynamicIndexing
+; SCHECK-NEXT: OpCapability InputAttachment
+; CHECK-NOT: OpCapability
+
+; CHECK-DAG: OpDecorate [[Var:%[0-9]+]] DescriptorSet 3
+; CHECK-DAG: OpDecorate [[Var]] Binding 4
+
+; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[BufferType:%[0-9]+]] = OpTypeImage [[int]] SubpassData 2 0 0 2 Unknown {{$}}
+; CHECK-DAG: [[BufferPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferType]]
+; CHECK-DAG: [[ArraySize:%[0-9]+]] = OpConstant [[int]] 3
+; CHECK-DAG: [[One:%[0-9]+]] = OpConstant [[int]] 1
+; CHECK-DAG: [[Zero:%[0-9]+]] = OpConstant [[int]] 0
+; CHECK-DAG: [[BufferArrayType:%[0-9]+]] = OpTypeArray [[BufferType]] [[ArraySize]]
+; CHECK-DAG: [[ArrayPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferArrayType]]
+; CHECK-DAG: [[Var]] = OpVariable [[ArrayPtrType]] UniformConstant
+
+; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
+; CHECK-NEXT: OpLabel
+define void @main() #0 {
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[Zero]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[BufferType]] [[ac]]
+  %buffer0 = call target("spirv.Image", i32, 6, 2, 0, 0, 2, 0)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_6_2_0_0_2_0(
+          i32 3, i32 4, i32 3, i32 0, i1 false)
+
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[One]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[BufferType]] [[ac]]
+  %buffer1 = call target("spirv.Image", i32, 6, 2, 0, 0, 2, 0)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_6_2_0_0_2_0(
+          i32 3, i32 4, i32 3, i32 1, i1 false)
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/InputAttachmentImageNonUniformIdx.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/InputAttachmentImageNonUniformIdx.ll
new file mode 100644
index 000000000000..b05b7eb885b4
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/InputAttachmentImageNonUniformIdx.ll
@@ -0,0 +1,47 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpCapability Shader
+; CHECK-NEXT: OpCapability ShaderNonUniformEXT
+; CHECK-NEXT: OpCapability InputAttachmentArrayNonUniformIndexing
+; SCHECK-NEXT: OpCapability InputAttachment
+; CHECK-NOT: OpCapability
+
+; CHECK-DAG: OpDecorate [[Var:%[0-9]+]] DescriptorSet 3
+; CHECK-DAG: OpDecorate [[Var]] Binding 4
+; CHECK: OpDecorate [[Zero:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ac0:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ld0:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[One:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ac1:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ld1:%[0-9]+]] NonUniform
+
+; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[BufferType:%[0-9]+]] = OpTypeImage [[int]] SubpassData 2 0 0 2 Unknown {{$}}
+; CHECK-DAG: [[BufferPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferType]]
+; CHECK-DAG: [[ArraySize:%[0-9]+]] = OpConstant [[int]] 3
+; CHECK-DAG: [[One]] = OpConstant [[int]] 1
+; CHECK-DAG: [[Zero]] = OpConstant [[int]] 0
+; CHECK-DAG: [[BufferArrayType:%[0-9]+]] = OpTypeArray [[BufferType]] [[ArraySize]]
+; CHECK-DAG: [[ArrayPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferArrayType]]
+; CHECK-DAG: [[Var]] = OpVariable [[ArrayPtrType]] UniformConstant
+
+; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
+; CHECK-NEXT: OpLabel
+define void @main() #0 {
+; CHECK: [[ac0]] = OpAccessChain [[BufferPtrType]] [[Var]] [[Zero]]
+; CHECK: [[ld0]] = OpLoad [[BufferType]] [[ac0]]
+  %buffer0 = call target("spirv.Image", i32, 6, 2, 0, 0, 2, 0)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_6_2_0_0_2_0(
+          i32 3, i32 4, i32 3, i32 0, i1 true)
+
+; CHECK: [[ac1:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[One]]
+; CHECK: [[ld1]] = OpLoad [[BufferType]] [[ac1]]
+  %buffer1 = call target("spirv.Image", i32, 6, 2, 0, 0, 2, 0)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_6_2_0_0_2_0(
+          i32 3, i32 4, i32 3, i32 1, i1 true)
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/SampledImageDynIdx.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/SampledImageDynIdx.ll
new file mode 100644
index 000000000000..0c47eeb606e8
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/SampledImageDynIdx.ll
@@ -0,0 +1,66 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpCapability Shader
+; CHECK-NEXT: OpCapability SampledImageArrayDynamicIndexing
+; CHECK-NEXT: OpCapability Sampled1D
+; CHECK-NOT: OpCapability
+
+; CHECK-DAG: OpDecorate [[Var:%[0-9]+]] DescriptorSet 3
+; CHECK-DAG: OpDecorate [[Var]] Binding 4
+; CHECK-DAG: OpDecorate [[OtherVar:%[0-9]+]] DescriptorSet 3
+; CHECK-DAG: OpDecorate [[OtherVar]] Binding 4
+
+; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[BufferType:%[0-9]+]] = OpTypeImage [[int]] 1D 2 0 0 1 R32i {{$}}
+; CHECK-DAG: [[BufferPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferType]]
+; CHECK-DAG: [[ArraySize:%[0-9]+]] = OpConstant [[int]] 3
+; CHECK-DAG: [[One:%[0-9]+]] = OpConstant [[int]] 1
+; CHECK-DAG: [[Zero:%[0-9]+]] = OpConstant [[int]] 0
+; CHECK-DAG: [[BufferArrayType:%[0-9]+]] = OpTypeArray [[BufferType]] [[ArraySize]]
+; CHECK-DAG: [[ArrayPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferArrayType]]
+; CHECK-DAG: [[Var]] = OpVariable [[ArrayPtrType]] UniformConstant
+
+; CHECK-DAG: [[OtherArraySize:%[0-9]+]] = OpConstant [[int]] 5
+; CHECK-DAG: [[OtherBufferArrayType:%[0-9]+]] = OpTypeArray [[BufferType]] [[OtherArraySize]]
+; CHECK-DAG: [[OtherArrayPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[OtherBufferArrayType]]
+; CHECK-DAG: [[OtherVar]] = OpVariable [[OtherArrayPtrType]] UniformConstant
+
+; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
+; CHECK-NEXT: OpLabel
+define void @main() #0 {
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[Zero]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[BufferType]] [[ac]]
+  %buffer0 = call target("spirv.Image", i32, 0, 2, 0, 0, 1, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_0_2_0_0_1_24(
+          i32 3, i32 4, i32 3, i32 0, i1 false)
+
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[One]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[BufferType]] [[ac]]
+  %buffer1 = call target("spirv.Image", i32, 0, 2, 0, 0, 1, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_0_2_0_0_1_24(
+          i32 3, i32 4, i32 3, i32 1, i1 false)
+  ret void
+}
+
+; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
+; CHECK-NEXT: OpLabel
+define void @DifferentArraySizesAreDifferentVariables() #0 {
+; Make sure we use different variables when the array sizes are different
+; same in case one function calls the other.
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[Zero]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[BufferType]] [[ac]]
+  %buffer0 = call target("spirv.Image", i32, 0, 2, 0, 0, 1, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_0_2_0_0_1_24(
+          i32 3, i32 4, i32 3, i32 0, i1 false)
+
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[OtherVar]] [[One]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[BufferType]] [[ac]]
+  %buffer1 = call target("spirv.Image", i32, 0, 2, 0, 0, 1, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_0_2_0_0_1_24(
+          i32 3, i32 4, i32 5, i32 1, i1 false)
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/SampledImageNonUniformIdx.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/SampledImageNonUniformIdx.ll
new file mode 100644
index 000000000000..ec94a8eeac2e
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/SampledImageNonUniformIdx.ll
@@ -0,0 +1,47 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpCapability Shader
+; CHECK-NEXT: OpCapability ShaderNonUniformEXT
+; CHECK-NEXT: OpCapability SampledImageArrayNonUniformIndexing
+; CHECK-NEXT: OpCapability Sampled1D
+; CHECK-NOT: OpCapability
+
+; CHECK-DAG: OpDecorate [[Var:%[0-9]+]] DescriptorSet 3
+; CHECK-DAG: OpDecorate [[Var]] Binding 4
+; CHECK: OpDecorate [[Zero:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ac0:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ld0:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[One:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ac1:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ld1:%[0-9]+]] NonUniform
+
+; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[BufferType:%[0-9]+]] = OpTypeImage [[int]] 1D 2 0 0 1 R32i {{$}}
+; CHECK-DAG: [[BufferPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferType]]
+; CHECK-DAG: [[ArraySize:%[0-9]+]] = OpConstant [[int]] 3
+; CHECK-DAG: [[One]] = OpConstant [[int]] 1
+; CHECK-DAG: [[Zero]] = OpConstant [[int]] 0
+; CHECK-DAG: [[BufferArrayType:%[0-9]+]] = OpTypeArray [[BufferType]] [[ArraySize]]
+; CHECK-DAG: [[ArrayPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferArrayType]]
+; CHECK-DAG: [[Var]] = OpVariable [[ArrayPtrType]] UniformConstant
+
+; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
+; CHECK-NEXT: OpLabel
+define void @main() #0 {
+; CHECK: [[ac0]] = OpAccessChain [[BufferPtrType]] [[Var]] [[Zero]]
+; CHECK: [[ld0]] = OpLoad [[BufferType]] [[ac0]]
+  %buffer0 = call target("spirv.Image", i32, 0, 2, 0, 0, 1, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_0_2_0_0_1_24(
+          i32 3, i32 4, i32 3, i32 0, i1 true)
+
+; CHECK: [[ac1:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[One]]
+; CHECK: [[ld1]] = OpLoad [[BufferType]] [[ac1]]
+  %buffer1 = call target("spirv.Image", i32, 0, 2, 0, 0, 1, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_0_2_0_0_1_24(
+          i32 3, i32 4, i32 3, i32 1, i1 true)
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/SamplerArrayDynIdx.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/SamplerArrayDynIdx.ll
new file mode 100644
index 000000000000..9371a792f84b
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/SamplerArrayDynIdx.ll
@@ -0,0 +1,39 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpCapability Shader
+; CHECK-NEXT: OpCapability SampledImageArrayDynamicIndexing
+; CHECK-NOT: OpCapability
+
+; CHECK-DAG: OpDecorate [[Var:%[0-9]+]] DescriptorSet 3
+; CHECK-DAG: OpDecorate [[Var]] Binding 4
+
+; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[SamplerType:%[0-9]+]] = OpTypeSampler
+; CHECK-DAG: [[SamplerPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[SamplerType]]
+; CHECK-DAG: [[ArraySize:%[0-9]+]] = OpConstant [[int]] 3
+; CHECK-DAG: [[One:%[0-9]+]] = OpConstant [[int]] 1
+; CHECK-DAG: [[Zero:%[0-9]+]] = OpConstant [[int]] 0
+; CHECK-DAG: [[SamplerArrayType:%[0-9]+]] = OpTypeArray [[SamplerType]] [[ArraySize]]
+; CHECK-DAG: [[ArrayPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[SamplerArrayType]]
+; CHECK-DAG: [[Var]] = OpVariable [[ArrayPtrType]] UniformConstant
+
+; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
+; CHECK-NEXT: OpLabel
+define void @main() #0 {
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[SamplerPtrType]] [[Var]] [[Zero]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[SamplerType]] [[ac]]
+  %buffer0 = call target("spirv.Sampler")
+      @llvm.spv.handle.fromBinding.tspirv.Image(
+          i32 3, i32 4, i32 3, i32 0, i1 false)
+
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[SamplerPtrType]] [[Var]] [[One]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[SamplerType]] [[ac]]
+  %buffer1 = call target("spirv.Sampler")
+      @llvm.spv.handle.fromBinding.tspirv.Image(
+          i32 3, i32 4, i32 3, i32 1, i1 false)
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/SamplerArrayNonUniformIdx.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/SamplerArrayNonUniformIdx.ll
new file mode 100644
index 000000000000..151c4aa6d436
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/SamplerArrayNonUniformIdx.ll
@@ -0,0 +1,46 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpCapability Shader
+; CHECK-NEXT: ShaderNonUniform
+; CHECK-NEXT: OpCapability SampledImageArrayNonUniformIndexing
+; CHECK-NOT: OpCapability
+
+; CHECK-DAG: OpDecorate [[Var:%[0-9]+]] DescriptorSet 3
+; CHECK-DAG: OpDecorate [[Var]] Binding 4
+; CHECK: OpDecorate [[Zero:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ac0:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ld0:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[One:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ac1:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ld1:%[0-9]+]] NonUniform
+
+; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[SamplerType:%[0-9]+]] = OpTypeSampler
+; CHECK-DAG: [[SamplerPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[SamplerType]]
+; CHECK-DAG: [[ArraySize:%[0-9]+]] = OpConstant [[int]] 3
+; CHECK-DAG: [[One]] = OpConstant [[int]] 1
+; CHECK-DAG: [[Zero]] = OpConstant [[int]] 0
+; CHECK-DAG: [[SamplerArrayType:%[0-9]+]] = OpTypeArray [[SamplerType]] [[ArraySize]]
+; CHECK-DAG: [[ArrayPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[SamplerArrayType]]
+; CHECK-DAG: [[Var]] = OpVariable [[ArrayPtrType]] UniformConstant
+
+; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
+; CHECK-NEXT: OpLabel
+define void @main() #0 {
+; CHECK: [[ac0]] = OpAccessChain [[SamplerPtrType]] [[Var]] [[Zero]]
+; CHECK: [[ld0]] = OpLoad [[SamplerType]] [[ac0]]
+  %buffer0 = call target("spirv.Sampler")
+      @llvm.spv.handle.fromBinding.tspirv.Image(
+          i32 3, i32 4, i32 3, i32 0, i1 true)
+
+; CHECK: [[ac1:%[0-9]+]] = OpAccessChain [[SamplerPtrType]] [[Var]] [[One]]
+; CHECK: [[ld1]] = OpLoad [[SamplerType]] [[ac1]]
+  %buffer1 = call target("spirv.Sampler")
+      @llvm.spv.handle.fromBinding.tspirv.Image(
+          i32 3, i32 4, i32 3, i32 1, i1 true)
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageDynIdx.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageDynIdx.ll
new file mode 100644
index 000000000000..908a81777a04
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageDynIdx.ll
@@ -0,0 +1,40 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpCapability Shader
+; CHECK-NEXT: OpCapability StorageImageArrayDynamicIndexing
+; CHECK-NEXT: OpCapability Image1D
+; CHECK-NOT: OpCapability
+
+; CHECK-DAG: OpDecorate [[Var:%[0-9]+]] DescriptorSet 3
+; CHECK-DAG: OpDecorate [[Var]] Binding 4
+
+; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[BufferType:%[0-9]+]] = OpTypeImage [[int]] 1D 2 0 0 2 R32i {{$}}
+; CHECK-DAG: [[BufferPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferType]]
+; CHECK-DAG: [[ArraySize:%[0-9]+]] = OpConstant [[int]] 3
+; CHECK-DAG: [[One:%[0-9]+]] = OpConstant [[int]] 1
+; CHECK-DAG: [[Zero:%[0-9]+]] = OpConstant [[int]] 0
+; CHECK-DAG: [[BufferArrayType:%[0-9]+]] = OpTypeArray [[BufferType]] [[ArraySize]]
+; CHECK-DAG: [[ArrayPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferArrayType]]
+; CHECK-DAG: [[Var]] = OpVariable [[ArrayPtrType]] UniformConstant
+
+; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
+; CHECK-NEXT: OpLabel
+define void @main() #0 {
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[Zero]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[BufferType]] [[ac]]
+  %buffer0 = call target("spirv.Image", i32, 0, 2, 0, 0, 2, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_0_2_0_0_2_24(
+          i32 3, i32 4, i32 3, i32 0, i1 false)
+
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[One]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[BufferType]] [[ac]]
+  %buffer1 = call target("spirv.Image", i32, 0, 2, 0, 0, 2, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_0_2_0_0_2_24(
+          i32 3, i32 4, i32 3, i32 1, i1 false)
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageNonUniformIdx.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageNonUniformIdx.ll
new file mode 100644
index 000000000000..4a582b31d60f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageNonUniformIdx.ll
@@ -0,0 +1,47 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpCapability Shader
+; CHECK: OpCapability ShaderNonUniformEXT
+; CHECK-NEXT: OpCapability StorageImageArrayNonUniformIndexing
+; CHECK-NEXT: OpCapability Image1D
+; CHECK-NOT: OpCapability
+
+; CHECK-DAG: OpDecorate [[Var:%[0-9]+]] DescriptorSet 3
+; CHECK-DAG: OpDecorate [[Var]] Binding 4
+; CHECK: OpDecorate [[Zero:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ac0:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ld0:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[One:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ac1:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ld1:%[0-9]+]] NonUniform
+
+; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[BufferType:%[0-9]+]] = OpTypeImage [[int]] 1D 2 0 0 2 R32i {{$}}
+; CHECK-DAG: [[BufferPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferType]]
+; CHECK-DAG: [[ArraySize:%[0-9]+]] = OpConstant [[int]] 3
+; CHECK-DAG: [[One]] = OpConstant [[int]] 1
+; CHECK-DAG: [[Zero]] = OpConstant [[int]] 0
+; CHECK-DAG: [[BufferArrayType:%[0-9]+]] = OpTypeArray [[BufferType]] [[ArraySize]]
+; CHECK-DAG: [[ArrayPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferArrayType]]
+; CHECK-DAG: [[Var]] = OpVariable [[ArrayPtrType]] UniformConstant
+
+; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
+; CHECK-NEXT: OpLabel
+define void @main() #0 {
+; CHECK: [[ac0]] = OpAccessChain [[BufferPtrType]] [[Var]] [[Zero]]
+; CHECK: [[ld0]] = OpLoad [[BufferType]] [[ac0]]
+  %buffer0 = call target("spirv.Image", i32, 0, 2, 0, 0, 2, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_0_2_0_0_2_24(
+          i32 3, i32 4, i32 3, i32 0, i1 true)
+
+; CHECK: [[ac1:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[One]]
+; CHECK: [[ld1]] = OpLoad [[BufferType]] [[ac1]]
+  %buffer1 = call target("spirv.Image", i32, 0, 2, 0, 0, 2, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_0_2_0_0_2_24(
+          i32 3, i32 4, i32 3, i32 1, i1 true)
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageTexelBufferDynIdx.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageTexelBufferDynIdx.ll
new file mode 100644
index 000000000000..d144dcf505fa
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageTexelBufferDynIdx.ll
@@ -0,0 +1,40 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpCapability Shader
+; SCHECK-NEXT: OpCapability ImageBuffer
+; CHECK-NEXT: OpCapability StorageTexelBufferArrayDynamicIndexing
+; CHECK-NOT: OpCapability
+
+; CHECK-DAG: OpDecorate [[Var:%[0-9]+]] DescriptorSet 3
+; CHECK-DAG: OpDecorate [[Var]] Binding 4
+
+; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[BufferType:%[0-9]+]] = OpTypeImage [[int]] Buffer 2 0 0 2 R32i {{$}}
+; CHECK-DAG: [[BufferPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferType]]
+; CHECK-DAG: [[ArraySize:%[0-9]+]] = OpConstant [[int]] 3
+; CHECK-DAG: [[One:%[0-9]+]] = OpConstant [[int]] 1
+; CHECK-DAG: [[Zero:%[0-9]+]] = OpConstant [[int]] 0
+; CHECK-DAG: [[BufferArrayType:%[0-9]+]] = OpTypeArray [[BufferType]] [[ArraySize]]
+; CHECK-DAG: [[ArrayPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferArrayType]]
+; CHECK-DAG: [[Var]] = OpVariable [[ArrayPtrType]] UniformConstant
+
+; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
+; CHECK-NEXT: OpLabel
+define void @void() #0 {
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[Zero]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[BufferType]] [[ac]]
+  %buffer0 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_5_2_0_0_2_24(
+          i32 3, i32 4, i32 3, i32 0, i1 false)
+
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[One]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[BufferType]] [[ac]]
+  %buffer1 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_5_2_0_0_2_24(
+          i32 3, i32 4, i32 3, i32 1, i1 false)
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageTexelBufferNonUniformIdx.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageTexelBufferNonUniformIdx.ll
new file mode 100644
index 000000000000..2f96eda4518f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageTexelBufferNonUniformIdx.ll
@@ -0,0 +1,47 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpCapability Shader
+; SCHECK-NEXT: OpCapability ImageBuffer
+; CHECK-NEXT: OpCapability ShaderNonUniformEXT
+; CHECK-NEXT: OpCapability StorageTexelBufferArrayNonUniformIndexingEXT
+; CHECK-NOT: OpCapability
+
+; CHECK-DAG: OpDecorate [[Var:%[0-9]+]] DescriptorSet 3
+; CHECK-DAG: OpDecorate [[Var]] Binding 4
+; CHECK: OpDecorate [[Zero:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ac0:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ld0:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[One:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ac1:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ld1:%[0-9]+]] NonUniform
+
+; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[BufferType:%[0-9]+]] = OpTypeImage [[int]] Buffer 2 0 0 2 R32i {{$}}
+; CHECK-DAG: [[BufferPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferType]]
+; CHECK-DAG: [[ArraySize:%[0-9]+]] = OpConstant [[int]] 3
+; CHECK-DAG: [[One]] = OpConstant [[int]] 1
+; CHECK-DAG: [[Zero]] = OpConstant [[int]] 0
+; CHECK-DAG: [[BufferArrayType:%[0-9]+]] = OpTypeArray [[BufferType]] [[ArraySize]]
+; CHECK-DAG: [[ArrayPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferArrayType]]
+; CHECK-DAG: [[Var]] = OpVariable [[ArrayPtrType]] UniformConstant
+
+; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
+; CHECK-NEXT: OpLabel
+define void @main() #0 {
+; CHECK: [[ac0]] = OpAccessChain [[BufferPtrType]] [[Var]] [[Zero]]
+; CHECK: [[ld0]] = OpLoad [[BufferType]] [[ac0]]
+  %buffer0 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_5_2_0_0_2_24(
+          i32 3, i32 4, i32 3, i32 0, i1 true)
+
+; CHECK: [[ac1:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[One]]
+; CHECK: [[ld1]] = OpLoad [[BufferType]] [[ac1]]
+  %buffer1 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_5_2_0_0_2_24(
+          i32 3, i32 4, i32 3, i32 1, i1 true)
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/UniformTexelBufferDynIdx.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/UniformTexelBufferDynIdx.ll
new file mode 100644
index 000000000000..117363241bd9
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/UniformTexelBufferDynIdx.ll
@@ -0,0 +1,40 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpCapability Shader
+; SCHECK-NEXT: OpCapability SampledBuffer
+; CHECK-NEXT: OpCapability UniformTexelBufferArrayDynamicIndexing
+; CHECK-NOT: OpCapability
+
+; CHECK-DAG: OpDecorate [[Var:%[0-9]+]] DescriptorSet 3
+; CHECK-DAG: OpDecorate [[Var]] Binding 4
+
+; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[BufferType:%[0-9]+]] = OpTypeImage [[int]] Buffer 2 0 0 1 R32i {{$}}
+; CHECK-DAG: [[BufferPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferType]]
+; CHECK-DAG: [[ArraySize:%[0-9]+]] = OpConstant [[int]] 3
+; CHECK-DAG: [[One:%[0-9]+]] = OpConstant [[int]] 1
+; CHECK-DAG: [[Zero:%[0-9]+]] = OpConstant [[int]] 0
+; CHECK-DAG: [[BufferArrayType:%[0-9]+]] = OpTypeArray [[BufferType]] [[ArraySize]]
+; CHECK-DAG: [[ArrayPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferArrayType]]
+; CHECK-DAG: [[Var]] = OpVariable [[ArrayPtrType]] UniformConstant
+
+; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
+; CHECK-NEXT: OpLabel
+define void @main() #0 {
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[Zero]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[BufferType]] [[ac]]
+  %buffer0 = call target("spirv.Image", i32, 5, 2, 0, 0, 1, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_5_2_0_0_1_24(
+          i32 3, i32 4, i32 3, i32 0, i1 false)
+
+; CHECK: [[ac:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[One]]
+; CHECK: [[buffer:%[0-9]+]] = OpLoad [[BufferType]] [[ac]]
+  %buffer1 = call target("spirv.Image", i32, 5, 2, 0, 0, 1, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_5_2_0_0_1_24(
+          i32 3, i32 4, i32 3, i32 1, i1 false)
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/UniformTexelBufferNonUniformIdx.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/UniformTexelBufferNonUniformIdx.ll
new file mode 100644
index 000000000000..cec16a8e7c8b
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/UniformTexelBufferNonUniformIdx.ll
@@ -0,0 +1,47 @@
+; TODO(pull/110270): verifier, fix G_BITCAST error "bitcast must change type"
+; RUN: llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpCapability Shader
+; SCHECK-NEXT: OpCapability SampledBuffer
+; CHECK-NEXT: OpCapability ShaderNonUniformEXT
+; CHECK-NEXT: OpCapability UniformTexelBufferArrayNonUniformIndexing
+; CHECK-NOT: OpCapability
+
+; CHECK-DAG: OpDecorate [[Var:%[0-9]+]] DescriptorSet 3
+; CHECK-DAG: OpDecorate [[Var]] Binding 4
+; CHECK: OpDecorate [[Zero:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ac0:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ld0:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[One:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ac1:%[0-9]+]] NonUniform
+; CHECK: OpDecorate [[ld1:%[0-9]+]] NonUniform
+
+; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[BufferType:%[0-9]+]] = OpTypeImage [[int]] Buffer 2 0 0 1 R32i {{$}}
+; CHECK-DAG: [[BufferPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferType]]
+; CHECK-DAG: [[ArraySize:%[0-9]+]] = OpConstant [[int]] 3
+; CHECK-DAG: [[One]] = OpConstant [[int]] 1
+; CHECK-DAG: [[Zero]] = OpConstant [[int]] 0
+; CHECK-DAG: [[BufferArrayType:%[0-9]+]] = OpTypeArray [[BufferType]] [[ArraySize]]
+; CHECK-DAG: [[ArrayPtrType:%[0-9]+]] = OpTypePointer UniformConstant [[BufferArrayType]]
+; CHECK-DAG: [[Var]] = OpVariable [[ArrayPtrType]] UniformConstant
+
+; CHECK: {{%[0-9]+}} = OpFunction {{%[0-9]+}} DontInline {{%[0-9]+}}
+; CHECK-NEXT: OpLabel
+define void @main() #0 {
+; CHECK: [[ac0]] = OpAccessChain [[BufferPtrType]] [[Var]] [[Zero]]
+; CHECK: [[ld0]] = OpLoad [[BufferType]] [[ac0]]
+  %buffer0 = call target("spirv.Image", i32, 5, 2, 0, 0, 1, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_5_2_0_0_1_24(
+          i32 3, i32 4, i32 3, i32 0, i1 true)
+
+; CHECK: [[ac1:%[0-9]+]] = OpAccessChain [[BufferPtrType]] [[Var]] [[One]]
+; CHECK: [[ld1]] = OpLoad [[BufferType]] [[ac1]]
+  %buffer1 = call target("spirv.Image", i32, 5, 2, 0, 0, 1, 24)
+      @llvm.spv.handle.fromBinding.tspirv.Image_f32_5_2_0_0_1_24(
+          i32 3, i32 4, i32 3, i32 1, i1 true)
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-- 
GitLab


From b03c8c4fdda6e58cb1afe3aa90bf9f2df08a7970 Mon Sep 17 00:00:00 2001
From: George Burgess IV <george.burgess.iv@gmail.com>
Date: Wed, 30 Oct 2024 13:28:32 -0600
Subject: [PATCH 635/693] libc: strlcpy/strlcat shouldn't bzero the rest of
 `buf` (#114259)

When running Bionic's testsuite over llvm-libc, tests broke because
e.g.,

```
const char *str = "abc";
char buf[7]{"111111"};
strlcpy(buf, str, 7);
ASSERT_EQ(buf, {'1', '1', '1', '\0', '\0', '\0', '\0'});
```

On my machine (Debian w/ glibc and clang-16), a `printf` loop over `buf`
gets unrolled into a series of const `printf` at compile-time:
```
printf("%d\n", '1');
printf("%d\n", '1');
printf("%d\n", '1');
printf("%d\n", 0);
printf("%d\n", '1');
printf("%d\n", '1');
printf("%d\n", 0);
```

Seems best to match existing precedent here.
---
 libc/src/string/string_utils.h        | 2 +-
 libc/test/src/string/strlcat_test.cpp | 9 +++++++++
 libc/test/src/string/strlcpy_test.cpp | 3 +--
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/libc/src/string/string_utils.h b/libc/src/string/string_utils.h
index 78381e46e480..240b28f15718 100644
--- a/libc/src/string/string_utils.h
+++ b/libc/src/string/string_utils.h
@@ -221,7 +221,7 @@ LIBC_INLINE size_t strlcpy(char *__restrict dst, const char *__restrict src,
     return len;
   size_t n = len < size - 1 ? len : size - 1;
   inline_memcpy(dst, src, n);
-  inline_bzero(dst + n, size - n);
+  dst[n] = '\0';
   return len;
 }
 
diff --git a/libc/test/src/string/strlcat_test.cpp b/libc/test/src/string/strlcat_test.cpp
index 1ffa4b0e921e..5757fc92b39d 100644
--- a/libc/test/src/string/strlcat_test.cpp
+++ b/libc/test/src/string/strlcat_test.cpp
@@ -27,6 +27,15 @@ TEST(LlvmLibcStrlcatTest, Smaller) {
   EXPECT_STREQ(buf, "abcd");
 }
 
+TEST(LlvmLibcStrlcatTest, SmallerNoOverwriteAfter0) {
+  const char *str = "cd";
+  char buf[8]{"ab\0\0efg"};
+
+  EXPECT_EQ(LIBC_NAMESPACE::strlcat(buf, str, 8), size_t(4));
+  EXPECT_STREQ(buf, "abcd");
+  EXPECT_STREQ(buf + 5, "fg");
+}
+
 TEST(LlvmLibcStrlcatTest, No0) {
   const char *str = "cd";
   char buf[7]{"ab"};
diff --git a/libc/test/src/string/strlcpy_test.cpp b/libc/test/src/string/strlcpy_test.cpp
index 5a1e30c12963..ecf0e925a265 100644
--- a/libc/test/src/string/strlcpy_test.cpp
+++ b/libc/test/src/string/strlcpy_test.cpp
@@ -25,6 +25,5 @@ TEST(LlvmLibcStrlcpyTest, Smaller) {
 
   EXPECT_EQ(LIBC_NAMESPACE::strlcpy(buf, str, 7), size_t(3));
   EXPECT_STREQ(buf, "abc");
-  for (const char *p = buf + 3; p < buf + 7; p++)
-    EXPECT_EQ(*p, '\0');
+  EXPECT_STREQ(buf + 4, "11");
 }
-- 
GitLab


From e4dfb51da4cd16cbb3ab18944a43ff5518d9f548 Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Wed, 30 Oct 2024 15:30:34 -0400
Subject: [PATCH 636/693] Fix documentation build

This fixes the build after the removal of the clang-format status page.
---
 clang/docs/index.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/clang/docs/index.rst b/clang/docs/index.rst
index 1096432813fa..66a4540a0bca 100644
--- a/clang/docs/index.rst
+++ b/clang/docs/index.rst
@@ -93,7 +93,6 @@ Using Clang Tools
    ClangCheck
    ClangFormat
    ClangFormatStyleOptions
-   ClangFormattedStatus
    ClangLinkerWrapper
    ClangNVLinkWrapper
    ClangOffloadBundler
-- 
GitLab


From d2109640a3e352b49a698edc232eeaac648fe590 Mon Sep 17 00:00:00 2001
From: Ilya Enkovich <ilya.enkovich@intel.com>
Date: Wed, 30 Oct 2024 14:41:28 -0500
Subject: [PATCH 637/693] [MLIR] [AMX] Fix strides used by AMX lowering for
 tile loads and stores. (#113476)

---
 .../AMX/Transforms/LegalizeForLLVMExport.cpp  | 62 +++++++++----------
 mlir/test/Dialect/AMX/legalize-for-llvm.mlir  | 28 +++++++++
 2 files changed, 58 insertions(+), 32 deletions(-)

diff --git a/mlir/lib/Dialect/AMX/Transforms/LegalizeForLLVMExport.cpp b/mlir/lib/Dialect/AMX/Transforms/LegalizeForLLVMExport.cpp
index c8cfcc3d945b..46c7bfbf3ffc 100644
--- a/mlir/lib/Dialect/AMX/Transforms/LegalizeForLLVMExport.cpp
+++ b/mlir/lib/Dialect/AMX/Transforms/LegalizeForLLVMExport.cpp
@@ -37,40 +37,38 @@ std::pair<Value, Value> getTileSizes(ConversionPatternRewriter &rewriter,
       rewriter.create<LLVM::ConstantOp>(loc, llvmInt16Type, nattr));
 }
 
-/// Verifies if the stride matches proper tile access.
-LogicalResult verifyStride(MemRefType mType) {
-  if (mType.getRank() < 2)
-    return failure();
-  int64_t last = mType.getRank() - 1;
-  int64_t offset;
-  SmallVector<int64_t, 4> strides;
-  if (failed(getStridesAndOffset(mType, strides, offset)) || strides[last] != 1)
-    return failure();
-  return success();
-}
-
 /// Maps the 2-dim memref shape to the 64-bit stride. Note that the buffer
 /// shape may "envelop" the actual tile shape, and may be dynamically sized.
-Value getStride(ConversionPatternRewriter &rewriter,
-                const LLVMTypeConverter &typeConverter, MemRefType mType,
-                Value base, Location loc) {
-  assert(mType.getRank() >= 2);
-  int64_t last = mType.getRank() - 1;
+/// Returns failure if proper stride couldn't be found.
+FailureOr<Value> getStride(ConversionPatternRewriter &rewriter,
+                           const LLVMTypeConverter &typeConverter,
+                           MemRefType mType, Value base, Location loc) {
+  if (mType.getRank() < 2)
+    return failure();
+  int64_t preLast = mType.getRank() - 2;
   Type llvmInt64Type = IntegerType::get(&typeConverter.getContext(), 64);
   unsigned width = mType.getElementType().getIntOrFloatBitWidth();
   assert(llvm::isPowerOf2_64(width) && width >= 8);
   unsigned bytes = width >> 3;
-  if (mType.isDynamicDim(last)) {
-    // Dynamic size needs code to compute the stride at runtime.
+  int64_t offset;
+  SmallVector<int64_t, 4> strides;
+  if (failed(getStridesAndOffset(mType, strides, offset)) ||
+      strides.back() != 1)
+    return failure();
+  if (strides[preLast] == ShapedType::kDynamic) {
+    // Dynamic stride needs code to compute the stride at runtime.
     MemRefDescriptor memrefDescriptor(base);
     auto attr = rewriter.getI64IntegerAttr(bytes);
     Value scale = rewriter.create<LLVM::ConstantOp>(loc, llvmInt64Type, attr);
-    return rewriter.create<LLVM::MulOp>(
-        loc, llvmInt64Type, scale, memrefDescriptor.size(rewriter, loc, last));
+    return rewriter
+        .create<LLVM::MulOp>(loc, llvmInt64Type, scale,
+                             memrefDescriptor.stride(rewriter, loc, preLast))
+        .getResult();
   }
-  // Use direct constant for static size.
-  auto attr = rewriter.getI64IntegerAttr(mType.getDimSize(last) * bytes);
-  return rewriter.create<LLVM::ConstantOp>(loc, llvmInt64Type, attr);
+  // Use direct constant for static stride.
+  auto attr = rewriter.getI64IntegerAttr(strides[preLast] * bytes);
+  return rewriter.create<LLVM::ConstantOp>(loc, llvmInt64Type, attr)
+      .getResult();
 }
 
 struct TileZeroConversion : public ConvertOpToLLVMPattern<TileZeroOp> {
@@ -102,16 +100,16 @@ struct TileLoadConversion : public ConvertOpToLLVMPattern<TileLoadOp> {
     std::pair<Value, Value> tsz =
         getTileSizes(rewriter, *getTypeConverter(), vType, op.getLoc());
     // Determine stride.
-    if (failed(verifyStride(mType)))
+    auto stride = getStride(rewriter, *getTypeConverter(), mType,
+                            adaptor.getBase(), op.getLoc());
+    if (failed(stride))
       return failure();
-    Value stride = getStride(rewriter, *getTypeConverter(), mType,
-                             adaptor.getBase(), op.getLoc());
     // Replace operation with intrinsic.
     Value ptr = getStridedElementPtr(op.getLoc(), mType, adaptor.getBase(),
                                      adaptor.getIndices(), rewriter);
     Type resType = typeConverter->convertType(vType);
     rewriter.replaceOpWithNewOp<amx::x86_amx_tileloadd64>(
-        op, resType, tsz.first, tsz.second, ptr, stride);
+        op, resType, tsz.first, tsz.second, ptr, stride.value());
     return success();
   }
 };
@@ -128,15 +126,15 @@ struct TileStoreConversion : public ConvertOpToLLVMPattern<TileStoreOp> {
     std::pair<Value, Value> tsz =
         getTileSizes(rewriter, *getTypeConverter(), vType, op.getLoc());
     // Determine stride.
-    if (failed(verifyStride(mType)))
+    auto stride = getStride(rewriter, *getTypeConverter(), mType,
+                            adaptor.getBase(), op.getLoc());
+    if (failed(stride))
       return failure();
-    Value stride = getStride(rewriter, *getTypeConverter(), mType,
-                             adaptor.getBase(), op.getLoc());
     // Replace operation with intrinsic.
     Value ptr = getStridedElementPtr(op.getLoc(), mType, adaptor.getBase(),
                                      adaptor.getIndices(), rewriter);
     rewriter.replaceOpWithNewOp<amx::x86_amx_tilestored64>(
-        op, tsz.first, tsz.second, ptr, stride, adaptor.getVal());
+        op, tsz.first, tsz.second, ptr, stride.value(), adaptor.getVal());
     return success();
   }
 };
diff --git a/mlir/test/Dialect/AMX/legalize-for-llvm.mlir b/mlir/test/Dialect/AMX/legalize-for-llvm.mlir
index 992203153939..3cacbd0044f8 100644
--- a/mlir/test/Dialect/AMX/legalize-for-llvm.mlir
+++ b/mlir/test/Dialect/AMX/legalize-for-llvm.mlir
@@ -43,3 +43,31 @@ func.func @mulf(%arg0: memref<?x?xbf16>, %arg1: memref<?x?xf32>) {
   amx.tile_store %arg1[%0, %0], %4 : memref<?x?xf32>, vector<16x16xf32>
   return
 }
+
+// CHECK-LABEL: strides(
+// CHECK: %[[CST_64_1:.+]] = llvm.mlir.constant(64 : i64) : i64
+// CHECK: "amx.tileloadd64"(%{{.+}}, %{{.+}}, %{{.+}}, %[[CST_64_1]]
+// CHECK: %[[CST_128_1:.+]] = llvm.mlir.constant(128 : i64) : i64
+// CHECK: "amx.tileloadd64"(%{{.+}}, %{{.+}}, %{{.+}}, %[[CST_128_1]]
+// CHECK: llvm.mlir.constant(2 : i64) : i64
+// CHECK: llvm.extractvalue %{{.+}}[4, 0]
+// CHECK: %[[STRIDE_1:.+]] = llvm.mul
+// CHECK: "amx.tileloadd64"(%{{.+}}, %{{.+}}, %{{.+}}, %[[STRIDE_1]]
+// CHECK: %[[CST_64_2:.+]] = llvm.mlir.constant(64 : i64) : i64
+// CHECK: "amx.tilestored64"(%{{.+}}, %{{.+}}, %{{.+}}, %[[CST_64_2]]
+// CHECK: %[[CST_128_2:.+]] = llvm.mlir.constant(128 : i64) : i64
+// CHECK: "amx.tilestored64"(%{{.+}}, %{{.+}}, %{{.+}}, %[[CST_128_2]]
+// CHECK: llvm.mlir.constant(2 : i64) : i64
+// CHECK: llvm.extractvalue %{{.+}}[4, 0]
+// CHECK: %[[STRIDE_2:.+]] = llvm.mul
+// CHECK: "amx.tilestored64"(%{{.+}}, %{{.+}}, %{{.+}}, %[[STRIDE_2]]
+func.func @strides(%arg0: memref<16x32xbf16>, %arg1: memref<16x32xbf16, strided<[64, 1]>>, %arg2: memref<16x32xbf16, strided<[?, 1]>>) {
+  %0 = arith.constant 0 : index
+  %1 = amx.tile_load %arg0[%0, %0] : memref<16x32xbf16> into vector<16x32xbf16>
+  %2 = amx.tile_load %arg1[%0, %0] : memref<16x32xbf16, strided<[64, 1]>> into vector<16x32xbf16>
+  %3 = amx.tile_load %arg2[%0, %0] : memref<16x32xbf16, strided<[?, 1]>> into vector<16x32xbf16>
+  amx.tile_store %arg0[%0, %0], %3 : memref<16x32xbf16>, vector<16x32xbf16>
+  amx.tile_store %arg1[%0, %0], %1 : memref<16x32xbf16, strided<[64, 1]>>, vector<16x32xbf16>
+  amx.tile_store %arg2[%0, %0], %2 : memref<16x32xbf16, strided<[?, 1]>>, vector<16x32xbf16>
+  return
+}
-- 
GitLab


From c616f24bcb00150fedc999d47933603e099dd659 Mon Sep 17 00:00:00 2001
From: Vyacheslav Levytskyy <vyacheslav.levytskyy@intel.com>
Date: Wed, 30 Oct 2024 20:49:21 +0100
Subject: [PATCH 638/693] [SPIR-V] Do instruction selection for G_BITCAST on an
 earlier stage (#114216)

This PR implements instruction selection for G_BITCAST on an earlier
stage to avoid MachineVerifier complains on subtle semantics difference
between G_BITCAST and OpBitcast.

We do instruction selections for OpBitcast after IR Translation instead
of calling MIB.buildBitcast() generating the general op code G_BITCAST,
because when MachineVerifier validates G_BITCAST we see a check of a
kind: 'if Source Type is equal to Destination Type then report error
"bitcast must change the type"'. This doesn't take into account the
notion of a typed pointer that is important for SPIR-V where a user may
and should use bitcast between pointers with different pointee types
(https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpBitcast).

It's important for correct lowering in SPIR-V, because interpretation of
the data type is not left to instructions that utilize the pointer, but
encoded by the pointer declaration, and the SPIRV target can and must
handle the declaration and use of pointers that specify the type of data
they point to.

It's not feasible to improve validation of G_BITCAST using just
information provided by low level types of source and destination.
Therefore we don't produce G_BITCAST as the general op code with
semantics different from OpBitcast, but rather lower to OpBitcast
immediately.

See discussion in https://github.com/llvm/llvm-project/pull/110270 for
even more context.
---
 llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp   | 61 ++++++++++++++++---
 .../pointers/phi-valid-operand-types-rev.ll   |  5 +-
 .../SPIRV/pointers/phi-valid-operand-types.ll |  5 +-
 3 files changed, 54 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index 3c2af34dd552..cc34cf877dea 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -165,6 +165,57 @@ static MachineInstr *findAssignTypeInstr(Register Reg,
   return nullptr;
 }
 
+static void buildOpBitcast(SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
+                           Register ResVReg, Register OpReg) {
+  SPIRVType *ResType = GR->getSPIRVTypeForVReg(ResVReg);
+  SPIRVType *OpType = GR->getSPIRVTypeForVReg(OpReg);
+  assert(ResType && OpType && "Operand types are expected");
+  if (!GR->isBitcastCompatible(ResType, OpType))
+    report_fatal_error("incompatible result and operand types in a bitcast");
+  MachineRegisterInfo *MRI = MIB.getMRI();
+  if (!MRI->getRegClassOrNull(ResVReg))
+    MRI->setRegClass(ResVReg, GR->getRegClass(ResType));
+  MIB.buildInstr(SPIRV::OpBitcast)
+      .addDef(ResVReg)
+      .addUse(GR->getSPIRVTypeID(ResType))
+      .addUse(OpReg);
+}
+
+// We do instruction selections early instead of calling MIB.buildBitcast()
+// generating the general op code G_BITCAST. When MachineVerifier validates
+// G_BITCAST we see a check of a kind: if Source Type is equal to Destination
+// Type then report error "bitcast must change the type". This doesn't take into
+// account the notion of a typed pointer that is important for SPIR-V where a
+// user may and should use bitcast between pointers with different pointee types
+// (https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpBitcast).
+// It's important for correct lowering in SPIR-V, because interpretation of the
+// data type is not left to instructions that utilize the pointer, but encoded
+// by the pointer declaration, and the SPIRV target can and must handle the
+// declaration and use of pointers that specify the type of data they point to.
+// It's not feasible to improve validation of G_BITCAST using just information
+// provided by low level types of source and destination. Therefore we don't
+// produce G_BITCAST as the general op code with semantics different from
+// OpBitcast, but rather lower to OpBitcast immediately. As for now, the only
+// difference would be that CombinerHelper couldn't transform known patterns
+// around G_BUILD_VECTOR. See discussion
+// in https://github.com/llvm/llvm-project/pull/110270 for even more context.
+static void selectOpBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
+                             MachineIRBuilder MIB) {
+  SmallVector<MachineInstr *, 16> ToErase;
+  for (MachineBasicBlock &MBB : MF) {
+    for (MachineInstr &MI : MBB) {
+      if (MI.getOpcode() != TargetOpcode::G_BITCAST)
+        continue;
+      MIB.setInsertPt(*MI.getParent(), MI);
+      buildOpBitcast(GR, MIB, MI.getOperand(0).getReg(),
+                     MI.getOperand(1).getReg());
+      ToErase.push_back(&MI);
+    }
+  }
+  for (MachineInstr *MI : ToErase)
+    MI->eraseFromParent();
+}
+
 static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
                            MachineIRBuilder MIB) {
   // Get access to information about available extensions
@@ -202,15 +253,6 @@ static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
       } else {
         GR->assignSPIRVTypeToVReg(AssignedPtrType, Def, MF);
         MIB.buildBitcast(Def, Source);
-        // MachineVerifier requires that bitcast must change the type.
-        // Change AddressSpace if needed to hint that Def and Source points to
-        // different types: this doesn't change actual code generation.
-        LLT DefType = MRI->getType(Def);
-        if (DefType == MRI->getType(Source))
-          MRI->setType(Def,
-                       LLT::pointer((DefType.getAddressSpace() + 1) %
-                                        SPIRVSubtarget::MaxLegalAddressSpace,
-                                    GR->getPointerSize()));
       }
     }
   }
@@ -1007,6 +1049,7 @@ bool SPIRVPreLegalizer::runOnMachineFunction(MachineFunction &MF) {
   removeImplicitFallthroughs(MF, MIB);
   insertSpirvDecorations(MF, MIB);
   insertInlineAsm(MF, GR, ST, MIB);
+  selectOpBitcasts(MF, GR, MIB);
 
   return true;
 }
diff --git a/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-rev.ll b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-rev.ll
index 6fa3f4e53cc5..8d14c3a35996 100644
--- a/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-rev.ll
+++ b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-rev.ll
@@ -1,7 +1,4 @@
-; The goal of the test case is to ensure that OpPhi is consistent with respect to operand types.
-; -verify-machineinstrs is not available due to mutually exclusive requirements for G_BITCAST and G_PHI.
-
-; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: %[[#Char:]] = OpTypeInt 8 0
diff --git a/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types.ll b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types.ll
index 4fbaae255673..07824d4ed6cd 100644
--- a/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types.ll
+++ b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types.ll
@@ -1,7 +1,4 @@
-; The goal of the test case is to ensure that OpPhi is consistent with respect to operand types.
-; -verify-machineinstrs is not available due to mutually exclusive requirements for G_BITCAST and G_PHI.
-
-; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: %[[#Char:]] = OpTypeInt 8 0
-- 
GitLab


From b1320d36339e38b073088fd45013a3c692adb301 Mon Sep 17 00:00:00 2001
From: Nick Desaulniers <nickdesaulniers@users.noreply.github.com>
Date: Wed, 30 Oct 2024 12:59:59 -0700
Subject: [PATCH 639/693] [libc][i386] setjmp/longjmp (#112437)

Link: #93709
---
 libc/include/llvm-libc-types/jmp_buf.h |  7 +++++++
 libc/src/setjmp/x86_64/longjmp.cpp     | 25 ++++++++++++++++++++++-
 libc/src/setjmp/x86_64/setjmp.cpp      | 28 +++++++++++++++++++++++++-
 3 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/libc/include/llvm-libc-types/jmp_buf.h b/libc/include/llvm-libc-types/jmp_buf.h
index 60e033c6c65a..f246e6491cf5 100644
--- a/libc/include/llvm-libc-types/jmp_buf.h
+++ b/libc/include/llvm-libc-types/jmp_buf.h
@@ -19,6 +19,13 @@ typedef struct {
   __UINT64_TYPE__ r15;
   __UINTPTR_TYPE__ rsp;
   __UINTPTR_TYPE__ rip;
+#elif defined(__i386__)
+  long ebx;
+  long esi;
+  long edi;
+  long ebp;
+  long esp;
+  long eip;
 #elif defined(__riscv)
   /* Program counter.  */
   long int __pc;
diff --git a/libc/src/setjmp/x86_64/longjmp.cpp b/libc/src/setjmp/x86_64/longjmp.cpp
index c293c55a6f9f..143c9deb11e9 100644
--- a/libc/src/setjmp/x86_64/longjmp.cpp
+++ b/libc/src/setjmp/x86_64/longjmp.cpp
@@ -11,12 +11,34 @@
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 
-#if !defined(LIBC_TARGET_ARCH_IS_X86_64)
+#if !defined(LIBC_TARGET_ARCH_IS_X86)
 #error "Invalid file include"
 #endif
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifdef __i386__
+[[gnu::naked]]
+LLVM_LIBC_FUNCTION(void, longjmp, (jmp_buf, int)) {
+  asm(R"(
+      mov 0x4(%%esp), %%ecx
+      mov 0x8(%%esp), %%eax
+      cmpl $0x1, %%eax
+      adcl $0x0, %%eax
+
+      mov %c[ebx](%%ecx), %%ebx
+      mov %c[esi](%%ecx), %%esi
+      mov %c[edi](%%ecx), %%edi
+      mov %c[ebp](%%ecx), %%ebp
+      mov %c[esp](%%ecx), %%esp
+
+      jmp *%c[eip](%%ecx)
+      )" ::[ebx] "i"(offsetof(__jmp_buf, ebx)),
+      [esi] "i"(offsetof(__jmp_buf, esi)), [edi] "i"(offsetof(__jmp_buf, edi)),
+      [ebp] "i"(offsetof(__jmp_buf, ebp)), [esp] "i"(offsetof(__jmp_buf, esp)),
+      [eip] "i"(offsetof(__jmp_buf, eip)));
+}
+#else
 [[gnu::naked]]
 LLVM_LIBC_FUNCTION(void, longjmp, (jmp_buf, int)) {
   asm(R"(
@@ -38,5 +60,6 @@ LLVM_LIBC_FUNCTION(void, longjmp, (jmp_buf, int)) {
       [r15] "i"(offsetof(__jmp_buf, r15)), [rsp] "i"(offsetof(__jmp_buf, rsp)),
       [rip] "i"(offsetof(__jmp_buf, rip)));
 }
+#endif
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/setjmp/x86_64/setjmp.cpp b/libc/src/setjmp/x86_64/setjmp.cpp
index f6e82642edd7..5ac10fa87b39 100644
--- a/libc/src/setjmp/x86_64/setjmp.cpp
+++ b/libc/src/setjmp/x86_64/setjmp.cpp
@@ -11,12 +11,37 @@
 #include "src/__support/macros/config.h"
 #include "src/setjmp/setjmp_impl.h"
 
-#if !defined(LIBC_TARGET_ARCH_IS_X86_64)
+#if !defined(LIBC_TARGET_ARCH_IS_X86)
 #error "Invalid file include"
 #endif
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifdef __i386__
+[[gnu::naked]]
+LLVM_LIBC_FUNCTION(int, setjmp, (jmp_buf buf)) {
+  asm(R"(
+      mov 4(%%esp), %%eax
+
+      mov %%ebx, %c[ebx](%%eax)
+      mov %%esi, %c[esi](%%eax)
+      mov %%edi, %c[edi](%%eax)
+      mov %%ebp, %c[ebp](%%eax)
+
+      lea 4(%%esp), %%ecx
+      mov %%ecx, %c[esp](%%eax)
+
+      mov (%%esp), %%ecx
+      mov %%ecx, %c[eip](%%eax)
+
+      xorl %%eax, %%eax
+      retl)" ::[ebx] "i"(offsetof(__jmp_buf, ebx)),
+      [esi] "i"(offsetof(__jmp_buf, esi)), [edi] "i"(offsetof(__jmp_buf, edi)),
+      [ebp] "i"(offsetof(__jmp_buf, ebp)), [esp] "i"(offsetof(__jmp_buf, esp)),
+      [eip] "i"(offsetof(__jmp_buf, eip))
+      : "eax", "ecx");
+}
+#else
 [[gnu::naked]]
 LLVM_LIBC_FUNCTION(int, setjmp, (jmp_buf buf)) {
   asm(R"(
@@ -41,5 +66,6 @@ LLVM_LIBC_FUNCTION(int, setjmp, (jmp_buf buf)) {
       [rip] "i"(offsetof(__jmp_buf, rip))
       : "rax");
 }
+#endif
 
 } // namespace LIBC_NAMESPACE_DECL
-- 
GitLab


From e89f8212333ea8e9b534fb32382bb5cacae71b35 Mon Sep 17 00:00:00 2001
From: Justin Fargnoli <jfargnoli@nvidia.com>
Date: Wed, 30 Oct 2024 13:05:40 -0700
Subject: [PATCH 640/693] [NFC][NVPTX] Cleanup getPreferredVectorAction()
 (#114115)

`v2*16` is a legal type in NVPTX. Thus, this is dead code.
---
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index a95cba586b8f..01abf9591e34 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -1335,8 +1335,6 @@ NVPTXTargetLowering::getPreferredVectorAction(MVT VT) const {
   if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
       VT.getScalarType() == MVT::i1)
     return TypeSplitVector;
-  if (Isv2x16VT(VT))
-    return TypeLegal;
   return TargetLoweringBase::getPreferredVectorAction(VT);
 }
 
-- 
GitLab


From 0167a92e28d5c8eac00595300a1366bdce28678d Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 30 Oct 2024 13:06:58 -0700
Subject: [PATCH 641/693] [RISCV] Use unsigned instead of int64_t for two small
 positive shift amounts. NFC

---
 llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index dc3f8254cb4e..6291842e071a 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -693,7 +693,7 @@ bool RISCVDAGToDAGISel::tryIndexedLoad(SDNode *Node) {
 
   // The constants that can be encoded in the THeadMemIdx instructions
   // are of the form (sign_extend(imm5) << imm2).
-  int64_t Shift;
+  unsigned Shift;
   for (Shift = 0; Shift < 4; Shift++)
     if (isInt<5>(Offset >> Shift) && ((Offset % (1LL << Shift)) == 0))
       break;
@@ -3366,7 +3366,7 @@ bool RISCVDAGToDAGISel::selectSimm5Shl2(SDValue N, SDValue &Simm5,
                                         SDValue &Shl2) {
   if (auto *C = dyn_cast<ConstantSDNode>(N)) {
     int64_t Offset = C->getSExtValue();
-    int64_t Shift;
+    unsigned Shift;
     for (Shift = 0; Shift < 4; Shift++)
       if (isInt<5>(Offset >> Shift) && ((Offset % (1LL << Shift)) == 0))
         break;
-- 
GitLab


From 1c2824e3a44f6c7cfd3e236597c4af671ce7c95e Mon Sep 17 00:00:00 2001
From: Artem Pianykh <arr@fb.com>
Date: Wed, 30 Oct 2024 20:23:20 +0000
Subject: [PATCH 642/693] [NFC][Coro] Add helpers for coro cloning with a
 TimeTraceScope (#112948)

A helper (2 overloads) that consolidates corocloner creation and the
actual cloning. The helpers create a TimeTraceScope to make it easier to
see how long the cloning takes.

Extracted from #109032 (commit 1)
---
 llvm/lib/Transforms/Coroutines/CoroSplit.cpp | 58 +++++++++++++-------
 1 file changed, 38 insertions(+), 20 deletions(-)

diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 0395ee62ae98..070df429bfc2 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -60,6 +60,7 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/PrettyStackTrace.h"
+#include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Coroutines/ABI.h"
 #include "llvm/Transforms/Coroutines/CoroInstr.h"
@@ -118,7 +119,6 @@ private:
 
   TargetTransformInfo &TTI;
 
-public:
   /// Create a cloner for a switch lowering.
   CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
              Kind FKind, TargetTransformInfo &TTI)
@@ -140,6 +140,30 @@ public:
     assert(ActiveSuspend && "need active suspend point for continuation");
   }
 
+public:
+  /// Create a clone for a switch lowering.
+  static Function *createClone(Function &OrigF, const Twine &Suffix,
+                               coro::Shape &Shape, Kind FKind,
+                               TargetTransformInfo &TTI) {
+    TimeTraceScope FunctionScope("CoroCloner");
+
+    CoroCloner Cloner(OrigF, Suffix, Shape, FKind, TTI);
+    Cloner.create();
+    return Cloner.getFunction();
+  }
+
+  /// Create a clone for a continuation lowering.
+  static Function *createClone(Function &OrigF, const Twine &Suffix,
+                               coro::Shape &Shape, Function *NewF,
+                               AnyCoroSuspendInst *ActiveSuspend,
+                               TargetTransformInfo &TTI) {
+    TimeTraceScope FunctionScope("CoroCloner");
+
+    CoroCloner Cloner(OrigF, Suffix, Shape, NewF, ActiveSuspend, TTI);
+    Cloner.create();
+    return Cloner.getFunction();
+  }
+
   Function *getFunction() const {
     assert(NewF != nullptr && "declaration not yet set");
     return NewF;
@@ -1466,13 +1490,16 @@ struct SwitchCoroutineSplitter {
                     TargetTransformInfo &TTI) {
     assert(Shape.ABI == coro::ABI::Switch);
 
+    // Create a resume clone by cloning the body of the original function,
+    // setting new entry block and replacing coro.suspend an appropriate value
+    // to force resume or cleanup pass for every suspend point.
     createResumeEntryBlock(F, Shape);
-    auto *ResumeClone =
-        createClone(F, ".resume", Shape, CoroCloner::Kind::SwitchResume, TTI);
-    auto *DestroyClone =
-        createClone(F, ".destroy", Shape, CoroCloner::Kind::SwitchUnwind, TTI);
-    auto *CleanupClone =
-        createClone(F, ".cleanup", Shape, CoroCloner::Kind::SwitchCleanup, TTI);
+    auto *ResumeClone = CoroCloner::createClone(
+        F, ".resume", Shape, CoroCloner::Kind::SwitchResume, TTI);
+    auto *DestroyClone = CoroCloner::createClone(
+        F, ".destroy", Shape, CoroCloner::Kind::SwitchUnwind, TTI);
+    auto *CleanupClone = CoroCloner::createClone(
+        F, ".cleanup", Shape, CoroCloner::Kind::SwitchCleanup, TTI);
 
     postSplitCleanup(*ResumeClone);
     postSplitCleanup(*DestroyClone);
@@ -1562,17 +1589,6 @@ struct SwitchCoroutineSplitter {
   }
 
 private:
-  // Create a resume clone by cloning the body of the original function, setting
-  // new entry block and replacing coro.suspend an appropriate value to force
-  // resume or cleanup pass for every suspend point.
-  static Function *createClone(Function &F, const Twine &Suffix,
-                               coro::Shape &Shape, CoroCloner::Kind FKind,
-                               TargetTransformInfo &TTI) {
-    CoroCloner Cloner(F, Suffix, Shape, FKind, TTI);
-    Cloner.create();
-    return Cloner.getFunction();
-  }
-
   // Create an entry block for a resume function with a switch that will jump to
   // suspend points.
   static void createResumeEntryBlock(Function &F, coro::Shape &Shape) {
@@ -1872,7 +1888,8 @@ void coro::AsyncABI::splitCoroutine(Function &F, coro::Shape &Shape,
     auto *Suspend = Shape.CoroSuspends[Idx];
     auto *Clone = Clones[Idx];
 
-    CoroCloner(F, "resume." + Twine(Idx), Shape, Clone, Suspend, TTI).create();
+    CoroCloner::createClone(F, "resume." + Twine(Idx), Shape, Clone, Suspend,
+                            TTI);
   }
 }
 
@@ -2001,7 +2018,8 @@ void coro::AnyRetconABI::splitCoroutine(Function &F, coro::Shape &Shape,
     auto Suspend = Shape.CoroSuspends[i];
     auto Clone = Clones[i];
 
-    CoroCloner(F, "resume." + Twine(i), Shape, Clone, Suspend, TTI).create();
+    CoroCloner::createClone(F, "resume." + Twine(i), Shape, Clone, Suspend,
+                            TTI);
   }
 }
 
-- 
GitLab


From 84a78abdf5999e58e4120e20594ac2ad37472295 Mon Sep 17 00:00:00 2001
From: Artem Pianykh <arr@fb.com>
Date: Wed, 30 Oct 2024 20:23:43 +0000
Subject: [PATCH 643/693] [NFC][Utils] Extract CloneFunctionAttributesInto from
 CloneFunctionInto (#112976)

This patch is a part of step-by-step refactoring of CloneFunctionInto.
The goal is to extract reusable pieces out of it that will be later used
to optimize function cloning e.g. in coroutine processing.

Extracted from #109032 (commit 2)
---
 llvm/include/llvm/Transforms/Utils/Cloning.h |  8 +++
 llvm/lib/Transforms/Utils/CloneFunction.cpp  | 54 ++++++++++++--------
 2 files changed, 40 insertions(+), 22 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Utils/Cloning.h b/llvm/include/llvm/Transforms/Utils/Cloning.h
index a4be24e32c52..1e8ef0102450 100644
--- a/llvm/include/llvm/Transforms/Utils/Cloning.h
+++ b/llvm/include/llvm/Transforms/Utils/Cloning.h
@@ -175,6 +175,14 @@ void CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
                        ValueMapTypeRemapper *TypeMapper = nullptr,
                        ValueMaterializer *Materializer = nullptr);
 
+/// Clone OldFunc's attributes into NewFunc, transforming values based on the
+/// mappings in VMap.
+void CloneFunctionAttributesInto(Function *NewFunc, const Function *OldFunc,
+                                 ValueToValueMapTy &VMap,
+                                 bool ModuleLevelChanges,
+                                 ValueMapTypeRemapper *TypeMapper = nullptr,
+                                 ValueMaterializer *Materializer = nullptr);
+
 void CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
                                const Instruction *StartingInst,
                                ValueToValueMapTy &VMap, bool ModuleLevelChanges,
diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp
index 5dc82a8dfb2d..a2d38717f38d 100644
--- a/llvm/lib/Transforms/Utils/CloneFunction.cpp
+++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp
@@ -87,28 +87,14 @@ BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap,
   return NewBB;
 }
 
-// Clone OldFunc into NewFunc, transforming the old arguments into references to
-// VMap values.
-//
-void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
-                             ValueToValueMapTy &VMap,
-                             CloneFunctionChangeType Changes,
-                             SmallVectorImpl<ReturnInst *> &Returns,
-                             const char *NameSuffix, ClonedCodeInfo *CodeInfo,
-                             ValueMapTypeRemapper *TypeMapper,
-                             ValueMaterializer *Materializer) {
-  NewFunc->setIsNewDbgInfoFormat(OldFunc->IsNewDbgInfoFormat);
-  assert(NameSuffix && "NameSuffix cannot be null!");
-
-#ifndef NDEBUG
-  for (const Argument &I : OldFunc->args())
-    assert(VMap.count(&I) && "No mapping from source argument specified!");
-#endif
-
-  bool ModuleLevelChanges = Changes > CloneFunctionChangeType::LocalChangesOnly;
-
-  // Copy all attributes other than those stored in the AttributeList.  We need
-  // to remap the parameter indices of the AttributeList.
+void llvm::CloneFunctionAttributesInto(Function *NewFunc,
+                                       const Function *OldFunc,
+                                       ValueToValueMapTy &VMap,
+                                       bool ModuleLevelChanges,
+                                       ValueMapTypeRemapper *TypeMapper,
+                                       ValueMaterializer *Materializer) {
+  // Copy all attributes other than those stored in Function's AttributeList
+  // which holds e.g. parameters and return value attributes.
   AttributeList NewAttrs = NewFunc->getAttributes();
   NewFunc->copyAttributesFrom(OldFunc);
   NewFunc->setAttributes(NewAttrs);
@@ -140,6 +126,7 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
   // Clone any argument attributes that are present in the VMap.
   for (const Argument &OldArg : OldFunc->args()) {
     if (Argument *NewArg = dyn_cast<Argument>(VMap[&OldArg])) {
+      // Remap the parameter indices.
       NewArgAttrs[NewArg->getArgNo()] =
           OldAttrs.getParamAttrs(OldArg.getArgNo());
     }
@@ -148,6 +135,29 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
   NewFunc->setAttributes(
       AttributeList::get(NewFunc->getContext(), OldAttrs.getFnAttrs(),
                          OldAttrs.getRetAttrs(), NewArgAttrs));
+}
+
+// Clone OldFunc into NewFunc, transforming the old arguments into references to
+// VMap values.
+void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
+                             ValueToValueMapTy &VMap,
+                             CloneFunctionChangeType Changes,
+                             SmallVectorImpl<ReturnInst *> &Returns,
+                             const char *NameSuffix, ClonedCodeInfo *CodeInfo,
+                             ValueMapTypeRemapper *TypeMapper,
+                             ValueMaterializer *Materializer) {
+  NewFunc->setIsNewDbgInfoFormat(OldFunc->IsNewDbgInfoFormat);
+  assert(NameSuffix && "NameSuffix cannot be null!");
+
+#ifndef NDEBUG
+  for (const Argument &I : OldFunc->args())
+    assert(VMap.count(&I) && "No mapping from source argument specified!");
+#endif
+
+  bool ModuleLevelChanges = Changes > CloneFunctionChangeType::LocalChangesOnly;
+
+  CloneFunctionAttributesInto(NewFunc, OldFunc, VMap, ModuleLevelChanges,
+                              TypeMapper, Materializer);
 
   // Everything else beyond this point deals with function instructions,
   // so if we are dealing with a function declaration, we're done.
-- 
GitLab


From bfe486fe764667d514124faf2b39afb7e7322640 Mon Sep 17 00:00:00 2001
From: Renaud Kauffmann <rkauffmann@nvidia.com>
Date: Wed, 30 Oct 2024 13:24:47 -0700
Subject: [PATCH 644/693] Passing descriptors by reference to CUDA runtime
 calls (#114288)

Passing a descriptor as a `const Descriptor &` or a `const Descriptor *`
generates a FIR signature where the box is passed by value.
This is an issue, as it requires a load of the box to be passed. But
since, ultimately, all boxes are passed by reference a temporary is
generated in LLVM and the reference to the temporary is passed.

The boxes addresses are registered with the CUDA runtime but the
temporaries are not, thus preventing the runtime to properly map a host
side address to its device side counterpart.

To address this issue, this PR changes the signatures to the transfer
functions to pass a descriptor as a `Descriptor *`, which will in turn
generate a FIR signature with that takes a box reference as an argument.
---
 flang/include/flang/Runtime/CUDA/memory.h     |  9 +++---
 .../Optimizer/Transforms/CUFOpConversion.cpp  | 11 +++-----
 flang/runtime/CUDA/memory.cpp                 |  9 +++---
 flang/test/Fir/CUDA/cuda-data-transfer.fir    | 28 ++++++++-----------
 4 files changed, 23 insertions(+), 34 deletions(-)

diff --git a/flang/include/flang/Runtime/CUDA/memory.h b/flang/include/flang/Runtime/CUDA/memory.h
index 3c3ae73d4ad7..fb48152d7071 100644
--- a/flang/include/flang/Runtime/CUDA/memory.h
+++ b/flang/include/flang/Runtime/CUDA/memory.h
@@ -36,19 +36,18 @@ void RTDECL(CUFDataTransferPtrPtr)(void *dst, void *src, std::size_t bytes,
     unsigned mode, const char *sourceFile = nullptr, int sourceLine = 0);
 
 /// Data transfer from a pointer to a descriptor.
-void RTDECL(CUFDataTransferDescPtr)(const Descriptor &dst, void *src,
+void RTDECL(CUFDataTransferDescPtr)(Descriptor *dst, void *src,
     std::size_t bytes, unsigned mode, const char *sourceFile = nullptr,
     int sourceLine = 0);
 
 /// Data transfer from a descriptor to a pointer.
-void RTDECL(CUFDataTransferPtrDesc)(void *dst, const Descriptor &src,
+void RTDECL(CUFDataTransferPtrDesc)(void *dst, Descriptor *src,
     std::size_t bytes, unsigned mode, const char *sourceFile = nullptr,
     int sourceLine = 0);
 
 /// Data transfer from a descriptor to a descriptor.
-void RTDECL(CUFDataTransferDescDesc)(const Descriptor &dst,
-    const Descriptor &src, unsigned mode, const char *sourceFile = nullptr,
-    int sourceLine = 0);
+void RTDECL(CUFDataTransferDescDesc)(Descriptor *dst, Descriptor *src,
+    unsigned mode, const char *sourceFile = nullptr, int sourceLine = 0);
 
 } // extern "C"
 } // namespace Fortran::runtime::cuda
diff --git a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
index f1f3a95b220d..e3e441360e94 100644
--- a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
@@ -529,8 +529,8 @@ struct CUFDataTransferOpConversion
       mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
       mlir::Value sourceLine =
           fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
-      mlir::Value dst = builder.loadIfRef(loc, op.getDst());
-      mlir::Value src = builder.loadIfRef(loc, op.getSrc());
+      mlir::Value dst = op.getDst();
+      mlir::Value src = op.getSrc();
       llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
           builder, loc, fTy, dst, src, modeValue, sourceFile, sourceLine)};
       builder.create<fir::CallOp>(loc, func, args);
@@ -603,11 +603,8 @@ struct CUFDataTransferOpConversion
       mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
       mlir::Value sourceLine =
           fir::factory::locationToLineNo(builder, loc, fTy.getInput(5));
-      mlir::Value dst =
-          dstIsDesc ? builder.loadIfRef(loc, op.getDst()) : op.getDst();
-      mlir::Value src = mlir::isa<fir::BaseBoxType>(srcTy)
-                            ? builder.loadIfRef(loc, op.getSrc())
-                            : op.getSrc();
+      mlir::Value dst = op.getDst();
+      mlir::Value src = op.getSrc();
       llvm::SmallVector<mlir::Value> args{
           fir::runtime::createArguments(builder, loc, fTy, dst, src, bytes,
                                         modeValue, sourceFile, sourceLine)};
diff --git a/flang/runtime/CUDA/memory.cpp b/flang/runtime/CUDA/memory.cpp
index fc48b4343eea..4778a4ae7768 100644
--- a/flang/runtime/CUDA/memory.cpp
+++ b/flang/runtime/CUDA/memory.cpp
@@ -73,23 +73,22 @@ void RTDEF(CUFDataTransferPtrPtr)(void *dst, void *src, std::size_t bytes,
   CUDA_REPORT_IF_ERROR(cudaMemcpy(dst, src, bytes, kind));
 }
 
-void RTDEF(CUFDataTransferDescPtr)(const Descriptor &desc, void *addr,
+void RTDEF(CUFDataTransferDescPtr)(Descriptor *desc, void *addr,
     std::size_t bytes, unsigned mode, const char *sourceFile, int sourceLine) {
   Terminator terminator{sourceFile, sourceLine};
   terminator.Crash(
       "not yet implemented: CUDA data transfer from a pointer to a descriptor");
 }
 
-void RTDEF(CUFDataTransferPtrDesc)(void *addr, const Descriptor &desc,
+void RTDEF(CUFDataTransferPtrDesc)(void *addr, Descriptor *desc,
     std::size_t bytes, unsigned mode, const char *sourceFile, int sourceLine) {
   Terminator terminator{sourceFile, sourceLine};
   terminator.Crash(
       "not yet implemented: CUDA data transfer from a descriptor to a pointer");
 }
 
-void RTDECL(CUFDataTransferDescDesc)(const Descriptor &dstDesc,
-    const Descriptor &srcDesc, unsigned mode, const char *sourceFile,
-    int sourceLine) {
+void RTDECL(CUFDataTransferDescDesc)(Descriptor *dstDesc, Descriptor *srcDesc,
+    unsigned mode, const char *sourceFile, int sourceLine) {
   Terminator terminator{sourceFile, sourceLine};
   terminator.Crash(
       "not yet implemented: CUDA data transfer between two descriptors");
diff --git a/flang/test/Fir/CUDA/cuda-data-transfer.fir b/flang/test/Fir/CUDA/cuda-data-transfer.fir
index c33c50115b9f..b99e09fb7646 100644
--- a/flang/test/Fir/CUDA/cuda-data-transfer.fir
+++ b/flang/test/Fir/CUDA/cuda-data-transfer.fir
@@ -15,11 +15,9 @@ func.func @_QPsub1() {
 // CHECK-LABEL: func.func @_QPsub1()
 // CHECK: %[[ADEV:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Eadev"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 // CHECK: %[[AHOST:.*]]:2 = hlfir.declare %{{.*}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Eahost"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
-// CHECK: %[[AHOST_LOAD:.*]] = fir.load %[[AHOST]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-// CHECK: %[[ADEV_LOAD:.*]] = fir.load %[[ADEV]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-// CHECK: %[[AHOST_BOX:.*]] = fir.convert %[[AHOST_LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.box<none>
-// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV_LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.box<none>
-// CHECK: fir.call @_FortranACUFDataTransferDescDesc(%[[AHOST_BOX]], %[[ADEV_BOX]], %c1{{.*}}, %{{.*}}, %{{.*}}) : (!fir.box<none>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> none
+// CHECK: %[[AHOST_BOX:.*]] = fir.convert %[[AHOST]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK: fir.call @_FortranACUFDataTransferDescDesc(%[[AHOST_BOX]], %[[ADEV_BOX]], %c1{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.ref<!fir.box<none>>, i32, !fir.ref<i8>, i32) -> none
 
 func.func @_QPsub2() {
   %0 = cuf.alloc !fir.box<!fir.heap<!fir.array<?xi32>>> {bindc_name = "adev", data_attr = #cuf.cuda<device>, uniq_name = "_QFsub2Eadev"} -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
@@ -76,19 +74,17 @@ func.func @_QPsub4() {
 // CHECK: %[[NBELEM:.*]] = arith.constant 10 : index
 // CHECK: %[[WIDTH:.*]] = arith.constant 4 : index
 // CHECK: %[[BYTES:.*]] = arith.muli %[[NBELEM]], %[[WIDTH]] : index
-// CHECK: %[[ADEV_LOAD:.*]] = fir.load %[[ADEV]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV_LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.box<none>
+// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<none>>
 // CHECK: %[[AHOST_PTR:.*]] = fir.convert %[[AHOST]]#0 : (!fir.ref<!fir.array<10xi32>>) -> !fir.llvm_ptr<i8>
 // CHECK: %[[BYTES_CONV:.*]] = fir.convert %[[BYTES]] : (index) -> i64
-// CHECK: fir.call @_FortranACUFDataTransferDescPtr(%[[ADEV_BOX]], %[[AHOST_PTR]], %[[BYTES_CONV]], %c0{{.*}}, %{{.*}}, %{{.*}}) : (!fir.box<none>, !fir.llvm_ptr<i8>, i64, i32, !fir.ref<i8>, i32) -> none
+// CHECK: fir.call @_FortranACUFDataTransferDescPtr(%[[ADEV_BOX]], %[[AHOST_PTR]], %[[BYTES_CONV]], %c0{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.llvm_ptr<i8>, i64, i32, !fir.ref<i8>, i32) -> none
 // CHECK: %[[NBELEM:.*]] = arith.constant 10 : index
 // CHECK: %[[WIDTH:.*]] = arith.constant 4 : index
 // CHECK: %[[BYTES:.*]] = arith.muli %[[NBELEM]], %[[WIDTH]] : index
-// CHECK: %[[ADEV_LOAD:.*]] = fir.load %[[ADEV]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 // CHECK: %[[AHOST_PTR:.*]] = fir.convert %[[AHOST]]#0 : (!fir.ref<!fir.array<10xi32>>) -> !fir.llvm_ptr<i8>
-// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV_LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.box<none>
+// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<none>>
 // CHECK: %[[BYTES_CONV:.*]] = fir.convert %[[BYTES]] : (index) -> i64
-// CHECK: fir.call @_FortranACUFDataTransferPtrDesc(%[[AHOST_PTR]], %[[ADEV_BOX]], %[[BYTES_CONV]], %c1{{.*}}, %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.box<none>, i64, i32, !fir.ref<i8>, i32) -> none
+// CHECK: fir.call @_FortranACUFDataTransferPtrDesc(%[[AHOST_PTR]], %[[ADEV_BOX]], %[[BYTES_CONV]], %c1{{.*}}, %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i64, i32, !fir.ref<i8>, i32) -> none
 
 func.func @_QPsub5(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}) {
   %0 = fir.dummy_scope : !fir.dscope
@@ -122,19 +118,17 @@ func.func @_QPsub5(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}) {
 // CHECK: %[[NBELEM:.*]] = arith.muli %[[I1]], %[[I2]] : index
 // CHECK: %[[WIDTH:.*]] = arith.constant 4 : index
 // CHECK: %[[BYTES:.*]] = arith.muli %[[NBELEM]], %[[WIDTH]] : index
-// CHECK: %[[ADEV_LOAD:.*]] = fir.load %[[ADEV]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
-// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV_LOAD]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>) -> !fir.box<none>
+// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> !fir.ref<!fir.box<none>>
 // CHECK: %[[AHOST_PTR:.*]] = fir.convert %[[AHOST]]#1 : (!fir.ref<!fir.array<?x?xi32>>) -> !fir.llvm_ptr<i8>
 // CHECK: %[[BYTES_CONV:.*]] = fir.convert %[[BYTES]] : (index) -> i64
-// CHECK: fir.call @_FortranACUFDataTransferDescPtr(%[[ADEV_BOX]], %[[AHOST_PTR]], %[[BYTES_CONV]], %c0{{.*}}, %{{.*}}, %{{.*}}) : (!fir.box<none>, !fir.llvm_ptr<i8>, i64, i32, !fir.ref<i8>, i32) -> none
+// CHECK: fir.call @_FortranACUFDataTransferDescPtr(%[[ADEV_BOX]], %[[AHOST_PTR]], %[[BYTES_CONV]], %c0{{.*}}, %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.llvm_ptr<i8>, i64, i32, !fir.ref<i8>, i32) -> none
 // CHECK: %[[NBELEM:.*]] = arith.muli %[[I1]], %[[I2]] : index
 // CHECK: %[[WIDTH:.*]] = arith.constant 4 : index
 // CHECK: %[[BYTES:.*]] = arith.muli %[[NBELEM]], %[[WIDTH]] : index
-// CHECK: %[[ADEV_LOAD:.*]] = fir.load %[[ADEV]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
 // CHECK: %[[AHOST_PTR:.*]] = fir.convert %[[AHOST]]#1 : (!fir.ref<!fir.array<?x?xi32>>) -> !fir.llvm_ptr<i8>
-// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV_LOAD]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>) -> !fir.box<none>
+// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> !fir.ref<!fir.box<none>>
 // CHECK: %[[BYTES_CONV:.*]] = fir.convert %[[BYTES]] : (index) -> i64
-// CHECK: fir.call @_FortranACUFDataTransferPtrDesc(%[[AHOST_PTR]], %[[ADEV_BOX]], %[[BYTES_CONV]], %c1{{.*}}, %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.box<none>, i64, i32, !fir.ref<i8>, i32) -> none
+// CHECK: fir.call @_FortranACUFDataTransferPtrDesc(%[[AHOST_PTR]], %[[ADEV_BOX]], %[[BYTES_CONV]], %c1{{.*}}, %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i64, i32, !fir.ref<i8>, i32) -> none
 
 func.func @_QPsub6() {
   %0 = cuf.alloc i32 {bindc_name = "idev", data_attr = #cuf.cuda<device>, uniq_name = "_QFsub6Eidev"} -> !fir.ref<i32>
-- 
GitLab


From f7c36d2f88e05a1747fa7916ad2fefdd9d459a55 Mon Sep 17 00:00:00 2001
From: Wanyi <kusmour@gmail.com>
Date: Wed, 30 Oct 2024 17:00:40 -0400
Subject: [PATCH 645/693] [lldb] Fix API test for file redirection to existing
 files (#114119)

API test failed for remote platform in
[#112657](https://github.com/llvm/llvm-project/pull/112657)

Previously when putting files onto remote platform, I used `platform
file write -d <data>` which actually required a `platform file open
<path>` first in order to obtain a file descriptor.
eg. in file
[TestGDBRemotePlatformFile.py](https://github.com/llvm/llvm-project/blob/94e7d9c0bfe517507ea08b00fb00c32fb2837a82/lldb/test/API/functionalities/gdb_remote_client/TestGDBRemotePlatformFile.py#L24-L32)
To fix this, use the `platform put-file` method, which is used in the
`redirect_stdin` from this test already.
---
 .../python_api/process/io/TestProcessIO.py    | 39 +++++++++++--------
 1 file changed, 23 insertions(+), 16 deletions(-)

diff --git a/lldb/test/API/python_api/process/io/TestProcessIO.py b/lldb/test/API/python_api/process/io/TestProcessIO.py
index 3b5c7c48c51f..5d9727add399 100644
--- a/lldb/test/API/python_api/process/io/TestProcessIO.py
+++ b/lldb/test/API/python_api/process/io/TestProcessIO.py
@@ -99,31 +99,38 @@ class ProcessIOTestCase(TestBase):
     @expectedFlakeyLinux(bugnumber="llvm.org/pr26437")
     @skipIfDarwinEmbedded  # debugserver can't create/write files on the device
     def test_stdout_stderr_redirection_to_existing_files(self):
-        """Exercise SBLaunchInfo::AddOpenFileAction() for STDOUT and STDERR without redirecting STDIN to output files already exist."""
+        """Exercise SBLaunchInfo::AddOpenFileAction() for STDOUT and STDERR redirect to output files already exist."""
         self.setup_test()
         self.build()
         self.create_target()
-        self.write_file_with_placeholder(self.output_file)
-        self.write_file_with_placeholder(self.error_file)
-        self.redirect_stdout()
-        self.redirect_stderr()
-        self.run_process(True)
-        output = self.read_output_file_and_delete()
-        error = self.read_error_file_and_delete()
-        self.check_process_output(output, error)
 
-    def write_file_with_placeholder(self, target_file):
+        # Create the output and error files with placeholder
         placeholder = "This content should be overwritten."
+        # Local file directory and working directory are the same for local debugging
+        f = open(self.local_output_file, "w")
+        f.write(placeholder)
+        f.close()
+        f = open(self.local_error_file, "w")
+        f.write(placeholder)
+        f.close()
         if lldb.remote_platform:
             self.runCmd(
-                'platform file write "{target}" -d "{data}"'.format(
-                    target=target_file, data=placeholder
+                'platform put-file "{local}" "{remote}"'.format(
+                    local=self.local_output_file, remote=self.output_file
+                )
+            )
+            self.runCmd(
+                'platform put-file "{local}" "{remote}"'.format(
+                    local=self.local_error_file, remote=self.error_file
                 )
             )
-        else:
-            f = open(target_file, "w")
-            f.write(placeholder)
-            f.close()
+
+        self.redirect_stdout()
+        self.redirect_stderr()
+        self.run_process(True)
+        output = self.read_output_file_and_delete()
+        error = self.read_error_file_and_delete()
+        self.check_process_output(output, error)
 
     # target_file - path on local file system or remote file system if running remote
     # local_file - path on local system
-- 
GitLab


From 4afa9787560d00474c6ab600be70d59fa7eae87f Mon Sep 17 00:00:00 2001
From: Felipe de Azevedo Piovezan <fpiovezan@apple.com>
Date: Wed, 30 Oct 2024 18:04:26 -0300
Subject: [PATCH 646/693] Revert "[Clang][Sema] Always use latest redeclaration
 of primary template" (#114304)

Clang importer doesn't seem to work well with this change, see
discussion in the original PR.

Reverts llvm/llvm-project#114258
---
 clang/include/clang/AST/DeclTemplate.h        | 52 +++++++++--
 clang/lib/AST/Decl.cpp                        | 10 +--
 clang/lib/AST/DeclCXX.cpp                     |  4 +-
 clang/lib/AST/DeclTemplate.cpp                | 56 +-----------
 clang/lib/Sema/SemaDecl.cpp                   |  4 +-
 clang/lib/Sema/SemaInit.cpp                   |  2 +-
 clang/lib/Sema/SemaTemplateInstantiate.cpp    | 14 +--
 clang/test/AST/ast-dump-decl.cpp              |  2 +-
 .../CXX/temp/temp.spec/temp.expl.spec/p7.cpp  | 87 -------------------
 9 files changed, 66 insertions(+), 165 deletions(-)

diff --git a/clang/include/clang/AST/DeclTemplate.h b/clang/include/clang/AST/DeclTemplate.h
index 0ca3fd48e81c..a572e3380f16 100644
--- a/clang/include/clang/AST/DeclTemplate.h
+++ b/clang/include/clang/AST/DeclTemplate.h
@@ -857,6 +857,16 @@ public:
   /// \endcode
   bool isMemberSpecialization() const { return Common.getInt(); }
 
+  /// Determines whether any redeclaration of this template was
+  /// a specialization of a member template.
+  bool hasMemberSpecialization() const {
+    for (const auto *D : redecls()) {
+      if (D->isMemberSpecialization())
+        return true;
+    }
+    return false;
+  }
+
   /// Note that this member template is a specialization.
   void setMemberSpecialization() {
     assert(!isMemberSpecialization() && "already a member specialization");
@@ -1955,7 +1965,13 @@ public:
   /// specialization which was specialized by this.
   llvm::PointerUnion<ClassTemplateDecl *,
                      ClassTemplatePartialSpecializationDecl *>
-  getSpecializedTemplateOrPartial() const;
+  getSpecializedTemplateOrPartial() const {
+    if (const auto *PartialSpec =
+            SpecializedTemplate.dyn_cast<SpecializedPartialSpecialization *>())
+      return PartialSpec->PartialSpecialization;
+
+    return SpecializedTemplate.get<ClassTemplateDecl*>();
+  }
 
   /// Retrieve the set of template arguments that should be used
   /// to instantiate members of the class template or class template partial
@@ -2192,6 +2208,17 @@ public:
     return InstantiatedFromMember.getInt();
   }
 
+  /// Determines whether any redeclaration of this this class template partial
+  /// specialization was a specialization of a member partial specialization.
+  bool hasMemberSpecialization() const {
+    for (const auto *D : redecls()) {
+      if (cast<ClassTemplatePartialSpecializationDecl>(D)
+              ->isMemberSpecialization())
+        return true;
+    }
+    return false;
+  }
+
   /// Note that this member template is a specialization.
   void setMemberSpecialization() { return InstantiatedFromMember.setInt(true); }
 
@@ -2713,7 +2740,13 @@ public:
   /// Retrieve the variable template or variable template partial
   /// specialization which was specialized by this.
   llvm::PointerUnion<VarTemplateDecl *, VarTemplatePartialSpecializationDecl *>
-  getSpecializedTemplateOrPartial() const;
+  getSpecializedTemplateOrPartial() const {
+    if (const auto *PartialSpec =
+            SpecializedTemplate.dyn_cast<SpecializedPartialSpecialization *>())
+      return PartialSpec->PartialSpecialization;
+
+    return SpecializedTemplate.get<VarTemplateDecl *>();
+  }
 
   /// Retrieve the set of template arguments that should be used
   /// to instantiate the initializer of the variable template or variable
@@ -2947,6 +2980,18 @@ public:
     return InstantiatedFromMember.getInt();
   }
 
+  /// Determines whether any redeclaration of this this variable template
+  /// partial specialization was a specialization of a member partial
+  /// specialization.
+  bool hasMemberSpecialization() const {
+    for (const auto *D : redecls()) {
+      if (cast<VarTemplatePartialSpecializationDecl>(D)
+              ->isMemberSpecialization())
+        return true;
+    }
+    return false;
+  }
+
   /// Note that this member template is a specialization.
   void setMemberSpecialization() { return InstantiatedFromMember.setInt(true); }
 
@@ -3119,9 +3164,6 @@ public:
     return makeSpecIterator(getSpecializations(), true);
   }
 
-  /// Merge \p Prev with our RedeclarableTemplateDecl::Common.
-  void mergePrevDecl(VarTemplateDecl *Prev);
-
   // Implement isa/cast/dyncast support
   static bool classof(const Decl *D) { return classofKind(D->getKind()); }
   static bool classofKind(Kind K) { return K == VarTemplate; }
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index cd173d172637..86913763ef9f 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -2708,7 +2708,7 @@ VarDecl *VarDecl::getTemplateInstantiationPattern() const {
     if (isTemplateInstantiation(VDTemplSpec->getTemplateSpecializationKind())) {
       auto From = VDTemplSpec->getInstantiatedFrom();
       if (auto *VTD = From.dyn_cast<VarTemplateDecl *>()) {
-        while (!VTD->isMemberSpecialization()) {
+        while (!VTD->hasMemberSpecialization()) {
           if (auto *NewVTD = VTD->getInstantiatedFromMemberTemplate())
             VTD = NewVTD;
           else
@@ -2718,7 +2718,7 @@ VarDecl *VarDecl::getTemplateInstantiationPattern() const {
       }
       if (auto *VTPSD =
               From.dyn_cast<VarTemplatePartialSpecializationDecl *>()) {
-        while (!VTPSD->isMemberSpecialization()) {
+        while (!VTPSD->hasMemberSpecialization()) {
           if (auto *NewVTPSD = VTPSD->getInstantiatedFromMember())
             VTPSD = NewVTPSD;
           else
@@ -2732,7 +2732,7 @@ VarDecl *VarDecl::getTemplateInstantiationPattern() const {
   // If this is the pattern of a variable template, find where it was
   // instantiated from. FIXME: Is this necessary?
   if (VarTemplateDecl *VTD = VD->getDescribedVarTemplate()) {
-    while (!VTD->isMemberSpecialization()) {
+    while (!VTD->hasMemberSpecialization()) {
       if (auto *NewVTD = VTD->getInstantiatedFromMemberTemplate())
         VTD = NewVTD;
       else
@@ -4153,7 +4153,7 @@ FunctionDecl::getTemplateInstantiationPattern(bool ForDefinition) const {
   if (FunctionTemplateDecl *Primary = getPrimaryTemplate()) {
     // If we hit a point where the user provided a specialization of this
     // template, we're done looking.
-    while (!ForDefinition || !Primary->isMemberSpecialization()) {
+    while (!ForDefinition || !Primary->hasMemberSpecialization()) {
       if (auto *NewPrimary = Primary->getInstantiatedFromMemberTemplate())
         Primary = NewPrimary;
       else
@@ -4170,7 +4170,7 @@ FunctionTemplateDecl *FunctionDecl::getPrimaryTemplate() const {
   if (FunctionTemplateSpecializationInfo *Info
         = TemplateOrSpecialization
             .dyn_cast<FunctionTemplateSpecializationInfo*>()) {
-    return Info->getTemplate()->getMostRecentDecl();
+    return Info->getTemplate();
   }
   return nullptr;
 }
diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp
index 1c92fd9e3ff0..db0ea62a2323 100644
--- a/clang/lib/AST/DeclCXX.cpp
+++ b/clang/lib/AST/DeclCXX.cpp
@@ -2030,7 +2030,7 @@ const CXXRecordDecl *CXXRecordDecl::getTemplateInstantiationPattern() const {
   if (auto *TD = dyn_cast<ClassTemplateSpecializationDecl>(this)) {
     auto From = TD->getInstantiatedFrom();
     if (auto *CTD = From.dyn_cast<ClassTemplateDecl *>()) {
-      while (!CTD->isMemberSpecialization()) {
+      while (!CTD->hasMemberSpecialization()) {
         if (auto *NewCTD = CTD->getInstantiatedFromMemberTemplate())
           CTD = NewCTD;
         else
@@ -2040,7 +2040,7 @@ const CXXRecordDecl *CXXRecordDecl::getTemplateInstantiationPattern() const {
     }
     if (auto *CTPSD =
             From.dyn_cast<ClassTemplatePartialSpecializationDecl *>()) {
-      while (!CTPSD->isMemberSpecialization()) {
+      while (!CTPSD->hasMemberSpecialization()) {
         if (auto *NewCTPSD = CTPSD->getInstantiatedFromMemberTemplate())
           CTPSD = NewCTPSD;
         else
diff --git a/clang/lib/AST/DeclTemplate.cpp b/clang/lib/AST/DeclTemplate.cpp
index 1db02d0d0444..755ec72f00bf 100644
--- a/clang/lib/AST/DeclTemplate.cpp
+++ b/clang/lib/AST/DeclTemplate.cpp
@@ -993,17 +993,7 @@ ClassTemplateSpecializationDecl::getSpecializedTemplate() const {
   if (const auto *PartialSpec =
           SpecializedTemplate.dyn_cast<SpecializedPartialSpecialization*>())
     return PartialSpec->PartialSpecialization->getSpecializedTemplate();
-  return SpecializedTemplate.get<ClassTemplateDecl *>()->getMostRecentDecl();
-}
-
-llvm::PointerUnion<ClassTemplateDecl *,
-                   ClassTemplatePartialSpecializationDecl *>
-ClassTemplateSpecializationDecl::getSpecializedTemplateOrPartial() const {
-  if (const auto *PartialSpec =
-          SpecializedTemplate.dyn_cast<SpecializedPartialSpecialization *>())
-    return PartialSpec->PartialSpecialization->getMostRecentDecl();
-
-  return SpecializedTemplate.get<ClassTemplateDecl *>()->getMostRecentDecl();
+  return SpecializedTemplate.get<ClassTemplateDecl*>();
 }
 
 SourceRange
@@ -1293,39 +1283,6 @@ VarTemplateDecl::newCommon(ASTContext &C) const {
   return CommonPtr;
 }
 
-void VarTemplateDecl::mergePrevDecl(VarTemplateDecl *Prev) {
-  // If we haven't created a common pointer yet, then it can just be created
-  // with the usual method.
-  if (!getCommonPtrInternal())
-    return;
-
-  Common *ThisCommon = static_cast<Common *>(getCommonPtrInternal());
-  Common *PrevCommon = nullptr;
-  SmallVector<VarTemplateDecl *, 8> PreviousDecls;
-  for (; Prev; Prev = Prev->getPreviousDecl()) {
-    if (CommonBase *C = Prev->getCommonPtrInternal()) {
-      PrevCommon = static_cast<Common *>(C);
-      break;
-    }
-    PreviousDecls.push_back(Prev);
-  }
-
-  // If the previous redecl chain hasn't created a common pointer yet, then just
-  // use this common pointer.
-  if (!PrevCommon) {
-    for (auto *D : PreviousDecls)
-      D->setCommonPtr(ThisCommon);
-    return;
-  }
-
-  // Ensure we don't leak any important state.
-  assert(ThisCommon->Specializations.empty() &&
-         ThisCommon->PartialSpecializations.empty() &&
-         "Can't merge incompatible declarations!");
-
-  setCommonPtr(PrevCommon);
-}
-
 VarTemplateSpecializationDecl *
 VarTemplateDecl::findSpecialization(ArrayRef<TemplateArgument> Args,
                                     void *&InsertPos) {
@@ -1448,16 +1405,7 @@ VarTemplateDecl *VarTemplateSpecializationDecl::getSpecializedTemplate() const {
   if (const auto *PartialSpec =
           SpecializedTemplate.dyn_cast<SpecializedPartialSpecialization *>())
     return PartialSpec->PartialSpecialization->getSpecializedTemplate();
-  return SpecializedTemplate.get<VarTemplateDecl *>()->getMostRecentDecl();
-}
-
-llvm::PointerUnion<VarTemplateDecl *, VarTemplatePartialSpecializationDecl *>
-VarTemplateSpecializationDecl::getSpecializedTemplateOrPartial() const {
-  if (const auto *PartialSpec =
-          SpecializedTemplate.dyn_cast<SpecializedPartialSpecialization *>())
-    return PartialSpec->PartialSpecialization->getMostRecentDecl();
-
-  return SpecializedTemplate.get<VarTemplateDecl *>()->getMostRecentDecl();
+  return SpecializedTemplate.get<VarTemplateDecl *>();
 }
 
 SourceRange VarTemplateSpecializationDecl::getSourceRange() const {
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 3e8b76e8dfd6..f8e5f3c6d309 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -4696,10 +4696,8 @@ void Sema::MergeVarDecl(VarDecl *New, LookupResult &Previous) {
 
   // Keep a chain of previous declarations.
   New->setPreviousDecl(Old);
-  if (NewTemplate) {
-    NewTemplate->mergePrevDecl(OldTemplate);
+  if (NewTemplate)
     NewTemplate->setPreviousDecl(OldTemplate);
-  }
 
   // Inherit access appropriately.
   New->setAccess(Old->getAccess());
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index e2a59f63ccf5..573e90aced3e 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -9954,7 +9954,7 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
     auto SynthesizeAggrGuide = [&](InitListExpr *ListInit) {
       auto *Pattern = Template;
       while (Pattern->getInstantiatedFromMemberTemplate()) {
-        if (Pattern->isMemberSpecialization())
+        if (Pattern->hasMemberSpecialization())
           break;
         Pattern = Pattern->getInstantiatedFromMemberTemplate();
       }
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index de0ec0128905..b63063813f1b 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -343,7 +343,7 @@ struct TemplateInstantiationArgumentCollecter
       // If this function was instantiated from a specialized member that is
       // a function template, we're done.
       assert(FD->getPrimaryTemplate() && "No function template?");
-      if (FD->getPrimaryTemplate()->isMemberSpecialization())
+      if (FD->getPrimaryTemplate()->hasMemberSpecialization())
         return Done();
 
       // If this function is a generic lambda specialization, we are done.
@@ -442,11 +442,11 @@ struct TemplateInstantiationArgumentCollecter
         Specialized = CTSD->getSpecializedTemplateOrPartial();
     if (auto *CTPSD =
             Specialized.dyn_cast<ClassTemplatePartialSpecializationDecl *>()) {
-      if (CTPSD->isMemberSpecialization())
+      if (CTPSD->hasMemberSpecialization())
         return Done();
     } else {
       auto *CTD = Specialized.get<ClassTemplateDecl *>();
-      if (CTD->isMemberSpecialization())
+      if (CTD->hasMemberSpecialization())
         return Done();
     }
     return UseNextDecl(CTSD);
@@ -478,11 +478,11 @@ struct TemplateInstantiationArgumentCollecter
         Specialized = VTSD->getSpecializedTemplateOrPartial();
     if (auto *VTPSD =
             Specialized.dyn_cast<VarTemplatePartialSpecializationDecl *>()) {
-      if (VTPSD->isMemberSpecialization())
+      if (VTPSD->hasMemberSpecialization())
         return Done();
     } else {
       auto *VTD = Specialized.get<VarTemplateDecl *>();
-      if (VTD->isMemberSpecialization())
+      if (VTD->hasMemberSpecialization())
         return Done();
     }
     return UseNextDecl(VTSD);
@@ -4141,7 +4141,7 @@ getPatternForClassTemplateSpecialization(
   CXXRecordDecl *Pattern = nullptr;
   Specialized = ClassTemplateSpec->getSpecializedTemplateOrPartial();
   if (auto *CTD = Specialized.dyn_cast<ClassTemplateDecl *>()) {
-    while (!CTD->isMemberSpecialization()) {
+    while (!CTD->hasMemberSpecialization()) {
       if (auto *NewCTD = CTD->getInstantiatedFromMemberTemplate())
         CTD = NewCTD;
       else
@@ -4151,7 +4151,7 @@ getPatternForClassTemplateSpecialization(
   } else if (auto *CTPSD =
                  Specialized
                      .dyn_cast<ClassTemplatePartialSpecializationDecl *>()) {
-    while (!CTPSD->isMemberSpecialization()) {
+    while (!CTPSD->hasMemberSpecialization()) {
       if (auto *NewCTPSD = CTPSD->getInstantiatedFromMemberTemplate())
         CTPSD = NewCTPSD;
       else
diff --git a/clang/test/AST/ast-dump-decl.cpp b/clang/test/AST/ast-dump-decl.cpp
index 7b998f20944f..e84241cee922 100644
--- a/clang/test/AST/ast-dump-decl.cpp
+++ b/clang/test/AST/ast-dump-decl.cpp
@@ -530,7 +530,7 @@ namespace testCanonicalTemplate {
   // CHECK-NEXT: |   `-ClassTemplateDecl 0x{{.+}} parent 0x{{.+}} <col:5, col:40> col:40 friend_undeclared TestClassTemplate{{$}}
   // CHECK-NEXT: |     |-TemplateTypeParmDecl 0x{{.+}} <col:14, col:23> col:23 typename depth 1 index 0 T2{{$}}
   // CHECK-NEXT: |     `-CXXRecordDecl 0x{{.+}} parent 0x{{.+}} <col:34, col:40> col:40 class TestClassTemplate{{$}}
-  // CHECK-NEXT: `-ClassTemplateSpecializationDecl 0x{{.+}} <col:5, col:40> line:[[@LINE-19]]:31 class TestClassTemplate definition implicit_instantiation{{$}}
+  // CHECK-NEXT: `-ClassTemplateSpecializationDecl 0x{{.+}} <line:[[@LINE-19]]:3, line:[[@LINE-17]]:3> line:[[@LINE-19]]:31 class TestClassTemplate definition implicit_instantiation{{$}}
   // CHECK-NEXT:   |-DefinitionData pass_in_registers empty aggregate standard_layout trivially_copyable pod trivial literal has_constexpr_non_copy_move_ctor can_const_default_init{{$}}
   // CHECK-NEXT:   | |-DefaultConstructor exists trivial constexpr defaulted_is_constexpr{{$}}
   // CHECK-NEXT:   | |-CopyConstructor simple trivial has_const_param implicit_has_const_param{{$}}
diff --git a/clang/test/CXX/temp/temp.spec/temp.expl.spec/p7.cpp b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p7.cpp
index e7e4738032f6..87127366eb58 100644
--- a/clang/test/CXX/temp/temp.spec/temp.expl.spec/p7.cpp
+++ b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p7.cpp
@@ -177,93 +177,6 @@ namespace Defined {
   static_assert(A<short>::B<int*>::y == 2);
 } // namespace Defined
 
-namespace Constrained {
-  template<typename T>
-  struct A {
-    template<typename U, bool V> requires V
-    static constexpr int f(); // expected-note {{declared here}}
-
-    template<typename U, bool V> requires V
-    static const int x; // expected-note {{declared here}}
-
-    template<typename U, bool V> requires V
-    static const int x<U*, V>; // expected-note {{declared here}}
-
-    template<typename U, bool V> requires V
-    struct B; // expected-note {{template is declared here}}
-
-    template<typename U, bool V> requires V
-    struct B<U*, V>; // expected-note {{template is declared here}}
-  };
-
-  template<>
-  template<typename U, bool V> requires V
-  constexpr int A<short>::f() {
-    return A<long>::f<U, V>();
-  }
-
-  template<>
-  template<typename U, bool V> requires V
-  constexpr int A<short>::x = A<long>::x<U, V>;
-
-  template<>
-  template<typename U, bool V> requires V
-  constexpr int A<short>::x<U*, V> = A<long>::x<U*, V>;
-
-  template<>
-  template<typename U, bool V> requires V
-  struct A<short>::B<U*, V> {
-    static constexpr int y = A<long>::B<U*, V>::y;
-  };
-
-  template<>
-  template<typename U, bool V> requires V
-  struct A<short>::B {
-    static constexpr int y = A<long>::B<U, V>::y;
-  };
-
-  template<>
-  template<typename U, bool V> requires V
-  constexpr int A<long>::f() {
-    return 1;
-  }
-
-  template<>
-  template<typename U, bool V> requires V
-  constexpr int A<long>::x = 1;
-
-  template<>
-  template<typename U, bool V> requires V
-  constexpr int A<long>::x<U*, V> = 2;
-
-  template<>
-  template<typename U, bool V> requires V
-  struct A<long>::B {
-    static constexpr int y = 1;
-  };
-
-  template<>
-  template<typename U, bool V> requires V
-  struct A<long>::B<U*, V> {
-    static constexpr int y = 2;
-  };
-
-  static_assert(A<int>::f<int, true>() == 0); // expected-error {{static assertion expression is not an integral constant expression}}
-                                              // expected-note@-1 {{undefined function 'f<int, true>' cannot be used in a constant expression}}
-  static_assert(A<int>::x<int, true> == 0); // expected-error {{static assertion expression is not an integral constant expression}}
-                                            // expected-note@-1 {{initializer of 'x<int, true>' is unknown}}
-  static_assert(A<int>::x<int*, true> == 0); // expected-error {{static assertion expression is not an integral constant expression}}
-                                             // expected-note@-1 {{initializer of 'x<int *, true>' is unknown}}
-  static_assert(A<int>::B<int, true>::y == 0); // expected-error {{implicit instantiation of undefined template 'Constrained::A<int>::B<int, true>'}}
-  static_assert(A<int>::B<int*, true>::y == 0); // expected-error {{implicit instantiation of undefined template 'Constrained::A<int>::B<int *, true>'}}
-
-  static_assert(A<short>::f<int, true>() == 1);
-  static_assert(A<short>::x<int, true> == 1);
-  static_assert(A<short>::x<int*, true> == 2);
-  static_assert(A<short>::B<int, true>::y == 1);
-  static_assert(A<short>::B<int*, true>::y == 2);
-} // namespace Constrained
-
 namespace Dependent {
   template<int I>
   struct A {
-- 
GitLab


From 74d8f3952c4acf6d57948983d7c5b0d0a7763c28 Mon Sep 17 00:00:00 2001
From: Helena Kotas <hekotas@microsoft.com>
Date: Wed, 30 Oct 2024 14:06:42 -0700
Subject: [PATCH 647/693] [HLSL] Remove old resource annotations for UAVs and
 SRVs (#114139)

UAVs and SRVs have already been converted to use LLVM target types and
we can disable generating of the !hlsl.uavs and !hlsl.srvs! annotations.
This will enable adding tests for structured buffers with user defined
types that this old resource annotations code does not handle (it
crashes).

Part 1 of #114126
---
 clang/lib/CodeGen/CGHLSLRuntime.cpp           | 10 ++++++++
 .../builtins/RWBuffer-annotations.hlsl        | 24 -------------------
 .../builtins/RWBuffer-elementtype.hlsl        | 14 -----------
 .../RWStructuredBuffer-elementtype.hlsl       | 14 -----------
 .../RasterizerOrderedBuffer-annotations.hlsl  | 20 ----------------
 .../StructuredBuffer-annotations.hlsl         | 22 -----------------
 .../StructuredBuffer-elementtype.hlsl         | 14 -----------
 clang/test/CodeGenHLSL/cbuf.hlsl              |  2 --
 8 files changed, 10 insertions(+), 110 deletions(-)
 delete mode 100644 clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl
 delete mode 100644 clang/test/CodeGenHLSL/builtins/RasterizerOrderedBuffer-annotations.hlsl
 delete mode 100644 clang/test/CodeGenHLSL/builtins/StructuredBuffer-annotations.hlsl

diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp
index 06558ce796f2..7ba0d6150181 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.cpp
+++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -306,6 +306,16 @@ void CGHLSLRuntime::annotateHLSLResource(const VarDecl *D, GlobalVariable *GV) {
       continue;
 
     llvm::hlsl::ResourceClass RC = AttrResType->getAttrs().ResourceClass;
+    if (RC == llvm::hlsl::ResourceClass::UAV ||
+        RC == llvm::hlsl::ResourceClass::SRV)
+      // UAVs and SRVs have already been converted to use LLVM target types,
+      // we can disable generating of these resource annotations. This will
+      // enable progress on structured buffers with user defined types this
+      // resource annotations code does not handle and it crashes.
+      // This whole function is going to be removed as soon as cbuffers are
+      // converted to target types (llvm/llvm-project #114126).
+      return;
+
     bool IsROV = AttrResType->getAttrs().IsROV;
     llvm::hlsl::ResourceKind RK = HLSLResAttr->getResourceKind();
     llvm::hlsl::ElementType ET = calculateElementType(CGM.getContext(), Ty);
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl
deleted file mode 100644
index e1e047485e4d..000000000000
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl
+++ /dev/null
@@ -1,24 +0,0 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
-
-RWBuffer<float> Buffer1;
-RWBuffer<vector<float, 4> > BufferArray[4];
-
-RWBuffer<float> Buffer2 : register(u3);
-RWBuffer<vector<float, 4> > BufferArray2[4] : register(u4);
-
-RWBuffer<float> Buffer3 : register(u3, space1);
-RWBuffer<vector<float, 4> > BufferArray3[4] : register(u4, space1);
-
-
-
-[numthreads(1,1,1)]
-void main() {
-}
-
-// CHECK: !hlsl.uavs = !{![[Single:[0-9]+]], ![[Array:[0-9]+]], ![[SingleAllocated:[0-9]+]], ![[ArrayAllocated:[0-9]+]], ![[SingleSpace:[0-9]+]], ![[ArraySpace:[0-9]+]]}
-// CHECK-DAG: ![[Single]] = !{ptr @Buffer1, i32 10, i32 9, i1 false, i32 -1, i32 0}
-// CHECK-DAG: ![[Array]] = !{ptr @BufferArray, i32 10, i32 9, i1 false, i32 -1, i32 0}
-// CHECK-DAG: ![[SingleAllocated]] = !{ptr @Buffer2, i32 10, i32 9, i1 false, i32 3, i32 0}
-// CHECK-DAG: ![[ArrayAllocated]] = !{ptr @BufferArray2, i32 10, i32 9, i1 false, i32 4, i32 0}
-// CHECK-DAG: ![[SingleSpace]] = !{ptr @Buffer3, i32 10, i32 9, i1 false, i32 3, i32 1}
-// CHECK-DAG: ![[ArraySpace]] = !{ptr @BufferArray3, i32 10, i32 9, i1 false, i32 4, i32 1}
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl
index fa81b53fd9bd..16120a44a9e4 100644
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl
@@ -54,17 +54,3 @@ void main(int GI : SV_GroupIndex) {
   BufF16x2[GI] = 0;
   BufF32x3[GI] = 0;
 }
-
-// CHECK: !{{[0-9]+}} = !{ptr @BufI16, i32 10, i32 2,
-// CHECK: !{{[0-9]+}} = !{ptr @BufU16, i32 10, i32 3,
-// CHECK: !{{[0-9]+}} = !{ptr @BufI32, i32 10, i32 4,
-// CHECK: !{{[0-9]+}} = !{ptr @BufU32, i32 10, i32 5,
-// CHECK: !{{[0-9]+}} = !{ptr @BufI64, i32 10, i32 6,
-// CHECK: !{{[0-9]+}} = !{ptr @BufU64, i32 10, i32 7,
-// CHECK: !{{[0-9]+}} = !{ptr @BufF16, i32 10, i32 8,
-// CHECK: !{{[0-9]+}} = !{ptr @BufF32, i32 10, i32 9,
-// CHECK: !{{[0-9]+}} = !{ptr @BufF64, i32 10, i32 10,
-// CHECK: !{{[0-9]+}} = !{ptr @BufI16x4, i32 10, i32 2,
-// CHECK: !{{[0-9]+}} = !{ptr @BufU32x3, i32 10, i32 5,
-// CHECK: !{{[0-9]+}} = !{ptr @BufF16x2, i32 10, i32 8,
-// CHECK: !{{[0-9]+}} = !{ptr @BufF32x3, i32 10, i32 9,
diff --git a/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
index 727f416cde57..71b5b7a75fa4 100644
--- a/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
@@ -54,17 +54,3 @@ void main(int GI : SV_GroupIndex) {
   BufF16x2[GI] = 0;
   BufF32x3[GI] = 0;
 }
-
-// CHECK: !{{[0-9]+}} = !{ptr @BufI16, i32 10, i32 2,
-// CHECK: !{{[0-9]+}} = !{ptr @BufU16, i32 10, i32 3,
-// CHECK: !{{[0-9]+}} = !{ptr @BufI32, i32 10, i32 4,
-// CHECK: !{{[0-9]+}} = !{ptr @BufU32, i32 10, i32 5,
-// CHECK: !{{[0-9]+}} = !{ptr @BufI64, i32 10, i32 6,
-// CHECK: !{{[0-9]+}} = !{ptr @BufU64, i32 10, i32 7,
-// CHECK: !{{[0-9]+}} = !{ptr @BufF16, i32 10, i32 8,
-// CHECK: !{{[0-9]+}} = !{ptr @BufF32, i32 10, i32 9,
-// CHECK: !{{[0-9]+}} = !{ptr @BufF64, i32 10, i32 10,
-// CHECK: !{{[0-9]+}} = !{ptr @BufI16x4, i32 10, i32 2,
-// CHECK: !{{[0-9]+}} = !{ptr @BufU32x3, i32 10, i32 5,
-// CHECK: !{{[0-9]+}} = !{ptr @BufF16x2, i32 10, i32 8,
-// CHECK: !{{[0-9]+}} = !{ptr @BufF32x3, i32 10, i32 9,
diff --git a/clang/test/CodeGenHLSL/builtins/RasterizerOrderedBuffer-annotations.hlsl b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedBuffer-annotations.hlsl
deleted file mode 100644
index 5155f1290259..000000000000
--- a/clang/test/CodeGenHLSL/builtins/RasterizerOrderedBuffer-annotations.hlsl
+++ /dev/null
@@ -1,20 +0,0 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-pixel -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
-
-RasterizerOrderedBuffer<float> Buffer1;
-RasterizerOrderedBuffer<vector<float, 4> > BufferArray[4];
-
-RasterizerOrderedBuffer<float> Buffer2 : register(u3);
-RasterizerOrderedBuffer<vector<float, 4> > BufferArray2[4] : register(u4);
-
-RasterizerOrderedBuffer<float> Buffer3 : register(u3, space1);
-RasterizerOrderedBuffer<vector<float, 4> > BufferArray3[4] : register(u4, space1);
-
-void main() {}
-
-// CHECK: !hlsl.uavs = !{![[Single:[0-9]+]], ![[Array:[0-9]+]], ![[SingleAllocated:[0-9]+]], ![[ArrayAllocated:[0-9]+]], ![[SingleSpace:[0-9]+]], ![[ArraySpace:[0-9]+]]}
-// CHECK-DAG: ![[Single]] = !{ptr @Buffer1, i32 10, i32 9, i1 true, i32 -1, i32 0}
-// CHECK-DAG: ![[Array]] = !{ptr @BufferArray, i32 10, i32 9, i1 true, i32 -1, i32 0}
-// CHECK-DAG: ![[SingleAllocated]] = !{ptr @Buffer2, i32 10, i32 9, i1 true, i32 3, i32 0}
-// CHECK-DAG: ![[ArrayAllocated]] = !{ptr @BufferArray2, i32 10, i32 9, i1 true, i32 4, i32 0}
-// CHECK-DAG: ![[SingleSpace]] = !{ptr @Buffer3, i32 10, i32 9, i1 true, i32 3, i32 1}
-// CHECK-DAG: ![[ArraySpace]] = !{ptr @BufferArray3, i32 10, i32 9, i1 true, i32 4, i32 1}
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-annotations.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-annotations.hlsl
deleted file mode 100644
index a88ea774f332..000000000000
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-annotations.hlsl
+++ /dev/null
@@ -1,22 +0,0 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
-
-StructuredBuffer<float> Buffer1;
-StructuredBuffer<vector<float, 4> > BufferArray[4];
-
-StructuredBuffer<float> Buffer2 : register(t3);
-StructuredBuffer<vector<float, 4> > BufferArray2[4] : register(t4);
-
-StructuredBuffer<float> Buffer3 : register(t3, space1);
-StructuredBuffer<vector<float, 4> > BufferArray3[4] : register(t4, space1);
-
-[numthreads(1,1,1)]
-void main() {
-}
-
-// CHECK: !hlsl.srvs = !{![[Single:[0-9]+]], ![[Array:[0-9]+]], ![[SingleAllocated:[0-9]+]], ![[ArrayAllocated:[0-9]+]], ![[SingleSpace:[0-9]+]], ![[ArraySpace:[0-9]+]]}
-// CHECK-DAG: ![[Single]] = !{ptr @Buffer1, i32 10, i32 9, i1 false, i32 -1, i32 0}
-// CHECK-DAG: ![[Array]] = !{ptr @BufferArray, i32 10, i32 9, i1 false, i32 -1, i32 0}
-// CHECK-DAG: ![[SingleAllocated]] = !{ptr @Buffer2, i32 10, i32 9, i1 false, i32 3, i32 0}
-// CHECK-DAG: ![[ArrayAllocated]] = !{ptr @BufferArray2, i32 10, i32 9, i1 false, i32 4, i32 0}
-// CHECK-DAG: ![[SingleSpace]] = !{ptr @Buffer3, i32 10, i32 9, i1 false, i32 3, i32 1}
-// CHECK-DAG: ![[ArraySpace]] = !{ptr @BufferArray3, i32 10, i32 9, i1 false, i32 4, i32 1}
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
index 4c30119498ff..205e13b4de39 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
@@ -54,17 +54,3 @@ void main(int GI : SV_GroupIndex) {
   half2 v12 = BufF16x2[GI];
   float3 v13 = BufF32x3[GI];
 }
-
-// CHECK: !{{[0-9]+}} = !{ptr @BufI16, i32 10, i32 2,
-// CHECK: !{{[0-9]+}} = !{ptr @BufU16, i32 10, i32 3,
-// CHECK: !{{[0-9]+}} = !{ptr @BufI32, i32 10, i32 4,
-// CHECK: !{{[0-9]+}} = !{ptr @BufU32, i32 10, i32 5,
-// CHECK: !{{[0-9]+}} = !{ptr @BufI64, i32 10, i32 6,
-// CHECK: !{{[0-9]+}} = !{ptr @BufU64, i32 10, i32 7,
-// CHECK: !{{[0-9]+}} = !{ptr @BufF16, i32 10, i32 8,
-// CHECK: !{{[0-9]+}} = !{ptr @BufF32, i32 10, i32 9,
-// CHECK: !{{[0-9]+}} = !{ptr @BufF64, i32 10, i32 10,
-// CHECK: !{{[0-9]+}} = !{ptr @BufI16x4, i32 10, i32 2,
-// CHECK: !{{[0-9]+}} = !{ptr @BufU32x3, i32 10, i32 5,
-// CHECK: !{{[0-9]+}} = !{ptr @BufF16x2, i32 10, i32 8,
-// CHECK: !{{[0-9]+}} = !{ptr @BufF32x3, i32 10, i32 9,
diff --git a/clang/test/CodeGenHLSL/cbuf.hlsl b/clang/test/CodeGenHLSL/cbuf.hlsl
index 78d9768b22fc..3f9d4514967d 100644
--- a/clang/test/CodeGenHLSL/cbuf.hlsl
+++ b/clang/test/CodeGenHLSL/cbuf.hlsl
@@ -23,6 +23,4 @@ float foo() {
 }
 
 // CHECK: !hlsl.cbufs = !{![[CBMD:[0-9]+]]}
-// CHECK: !hlsl.srvs = !{![[TBMD:[0-9]+]]}
 // CHECK: ![[CBMD]] = !{ptr @[[CB]], i32 13, i32 0, i1 false, i32 0, i32 2}
-// CHECK: ![[TBMD]] = !{ptr @[[TB]], i32 15, i32 0, i1 false, i32 2, i32 1}
-- 
GitLab


From 14045de250ea126029d43ff8f2f68e9614c394bc Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Wed, 30 Oct 2024 23:36:46 +0200
Subject: [PATCH 648/693] [RISCV] Account for factor in interleave memory op
 costs (#111511)

Currently we cost an interleaved memory op as if it were a load/store of
the widened vector type, but this was undercosting in all cases when
compared to the measured performance of todays hardware.

On the x280 at NF=2 and spacemit-x60 at NF=2,3 and 4, a segmented load
is carried out as a wide load and NF LMUL shuffle ops:
https://github.com/preames/bp3-microarch#vlseg_lmul_x_sew_throughput

All other NFs go through a slow path. On the spacemit-x60 this is
proportional to VLMAX * NF, and on the x280 proportional to the number
of segments.

This patch increases the cost by implementing a wide load + NF LMUL
shuffle op cost for the lowest common denominator NF=2, and then a
slower cost proportional to VL for the other NFs.

In a follow up patch we can add a tuning flag to use the faster cost
model for NF=3 and 4 on the spacemit-x60.

Note that the FIXME about illegal vectors seems to have been fixed in
#100436
---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp |  31 ++--
 .../LoopVectorize/RISCV/dead-ops-cost.ll      |  74 ++++-----
 .../LoopVectorize/RISCV/interleaved-cost.ll   | 144 +++++++++---------
 3 files changed, 130 insertions(+), 119 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 988cb194cd60..f050fb569946 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -723,8 +723,7 @@ InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
 
   // The interleaved memory access pass will lower interleaved memory ops (i.e
   // a load and store followed by a specific shuffle) to vlseg/vsseg
-  // intrinsics. In those cases then we can treat it as if it's just one (legal)
-  // memory op
+  // intrinsics.
   if (!UseMaskForCond && !UseMaskForGaps &&
       Factor <= TLI->getMaxSupportedInterleaveFactor()) {
     auto *VTy = cast<VectorType>(VecTy);
@@ -734,19 +733,27 @@ InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
       auto *SubVecTy =
           VectorType::get(VTy->getElementType(),
                           VTy->getElementCount().divideCoefficientBy(Factor));
-
       if (VTy->getElementCount().isKnownMultipleOf(Factor) &&
           TLI->isLegalInterleavedAccessType(SubVecTy, Factor, Alignment,
                                             AddressSpace, DL)) {
-        // FIXME: We use the memory op cost of the *legalized* type here,
-        // because it's getMemoryOpCost returns a really expensive cost for
-        // types like <6 x i8>, which show up when doing interleaves of
-        // Factor=3 etc. Should the memory op cost of these be cheaper?
-        auto *LegalVTy = VectorType::get(VTy->getElementType(),
-                                         LT.second.getVectorElementCount());
-        InstructionCost LegalMemCost = getMemoryOpCost(
-            Opcode, LegalVTy, Alignment, AddressSpace, CostKind);
-        return LT.first + LegalMemCost;
+
+        // Most available hardware today optimizes NF=2 as as one wide memory op
+        // + Factor * LMUL shuffle ops.
+        if (Factor == 2) {
+          InstructionCost Cost =
+              getMemoryOpCost(Opcode, VTy, Alignment, AddressSpace, CostKind);
+          MVT SubVecVT = getTLI()->getValueType(DL, SubVecTy).getSimpleVT();
+          Cost += Factor * TLI->getLMULCost(SubVecVT);
+          return LT.first * Cost;
+        }
+
+        // Otherwise, the cost is proportional to the number of elements (VL *
+        // Factor ops).
+        InstructionCost MemOpCost =
+            getMemoryOpCost(Opcode, VTy->getElementType(), Alignment, 0,
+                            CostKind, {TTI::OK_AnyValue, TTI::OP_None});
+        unsigned NumLoads = getEstimatedVLFor(VTy);
+        return NumLoads * MemOpCost;
       }
     }
   }
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 0e55ad65cdb2..6724afd6ca10 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -410,45 +410,49 @@ define void @gather_interleave_group_with_dead_insert_pos(i64 %N, ptr noalias %s
 ; CHECK-SAME: i64 [[N:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
-; CHECK-NEXT:    [[TMP0:%.*]] = add nuw i64 [[SMAX]], 3
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = add nuw i64 [[SMAX]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 16
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i64 16, i64 [[N_MOD_VF]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; CHECK-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = mul <vscale x 4 x i64> [[TMP10]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 2, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 2, [[TMP8]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP12]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 4, i64 8, i64 12, i64 16, i64 20, i64 24, i64 28>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <8 x i64> [[VEC_IND]], <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
-; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 32
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP6]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <32 x i8>, ptr [[TMP7]], align 1
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT:    [[WIDE_VEC2:%.*]] = load <32 x i8>, ptr [[TMP8]], align 1
-; CHECK-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <32 x i8> [[WIDE_VEC2]], <32 x i8> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <32 x i8> [[WIDE_VEC2]], <32 x i8> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT:    [[TMP11:%.*]] = zext <8 x i8> [[STRIDED_VEC4]] to <8 x i32>
-; CHECK-NEXT:    [[TMP12:%.*]] = zext <8 x i8> [[STRIDED_VEC5]] to <8 x i32>
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[DST]], <8 x i64> [[VEC_IND]]
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i32, ptr [[DST]], <8 x i64> [[STEP_ADD]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> [[TMP11]], <8 x ptr> [[TMP13]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
-; CHECK-NEXT:    call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> [[TMP12]], <8 x ptr> [[TMP14]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <8 x i64> [[STEP_ADD]], <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP13]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i8>, ptr [[TMP14]], align 1
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i8>, <vscale x 4 x i8> } @llvm.vector.deinterleave2.nxv8i8(<vscale x 8 x i8> [[WIDE_VEC]])
+; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = zext <vscale x 4 x i8> [[TMP16]] to <vscale x 4 x i32>
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr i32, ptr [[DST]], <vscale x 4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP17]], <vscale x 4 x ptr> [[TMP18]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
@@ -462,9 +466,9 @@ define void @gather_interleave_group_with_dead_insert_pos(i64 %N, ptr noalias %s
 ; CHECK-NEXT:    [[EXT:%.*]] = zext i8 [[L_1]] to i32
 ; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
 ; CHECK-NEXT:    store i32 [[EXT]], ptr [[GEP_DST]], align 4
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
 ; CHECK-NEXT:    [[EC:%.*]] = icmp slt i64 [[IV]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -481,7 +485,7 @@ loop:
   %ext  = zext i8 %l.1 to i32
   %gep.dst = getelementptr i32, ptr %dst, i64 %iv
   store i32 %ext, ptr %gep.dst, align 4
-  %iv.next = add nsw i64 %iv, 4
+  %iv.next = add nsw i64 %iv, 2
   %ec = icmp slt i64 %iv, %N
   br i1 %ec, label %loop, label %exit
 
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll
index fa346b4eac02..6477f14e3c69 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll
@@ -6,26 +6,26 @@ define void @i8_factor_2(ptr %data, i64 %n) {
 entry:
   br label %for.body
 ; CHECK-LABEL: Checking a loop in 'i8_factor_2'
-; CHECK: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; CHECK: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; CHECK: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; CHECK: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; CHECK: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; CHECK: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; CHECK: Cost of 3 for VF 16: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; CHECK: Cost of 3 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; CHECK: Cost of 5 for VF 32: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; CHECK: Cost of 5 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; CHECK: Cost of 2 for VF vscale x 1: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; CHECK: Cost of 2 for VF vscale x 1: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; CHECK: Cost of 2 for VF vscale x 2: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; CHECK: Cost of 2 for VF vscale x 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; CHECK: Cost of 2 for VF vscale x 4: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; CHECK: Cost of 2 for VF vscale x 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; CHECK: Cost of 3 for VF vscale x 8: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; CHECK: Cost of 3 for VF vscale x 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; CHECK: Cost of 5 for VF vscale x 16: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; CHECK: Cost of 5 for VF vscale x 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; CHECK: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; CHECK: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; CHECK: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; CHECK: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; CHECK: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; CHECK: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; CHECK: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; CHECK: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; CHECK: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; CHECK: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; CHECK: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; CHECK: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; CHECK: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; CHECK: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; CHECK: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; CHECK: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; CHECK: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; CHECK: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; CHECK: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; CHECK: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %p0 = getelementptr inbounds %i8.2, ptr %data, i64 %i, i32 0
@@ -49,16 +49,16 @@ define void @i8_factor_3(ptr %data, i64 %n) {
 entry:
   br label %for.body
 ; CHECK-LABEL: Checking a loop in 'i8_factor_3'
-; CHECK: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
-; CHECK: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
-; CHECK: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
-; CHECK: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
-; CHECK: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
-; CHECK: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
-; CHECK: Cost of 5 for VF 16: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
-; CHECK: Cost of 5 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
-; CHECK: Cost of 9 for VF 32: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
-; CHECK: Cost of 9 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; CHECK: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; CHECK: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; CHECK: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; CHECK: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; CHECK: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; CHECK: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; CHECK: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; CHECK: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; CHECK: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; CHECK: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %p0 = getelementptr inbounds %i8.3, ptr %data, i64 %i, i32 0
@@ -86,16 +86,16 @@ define void @i8_factor_4(ptr %data, i64 %n) {
 entry:
   br label %for.body
 ; CHECK-LABEL: Checking a loop in 'i8_factor_4'
-; CHECK: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
-; CHECK: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
-; CHECK: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
-; CHECK: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
-; CHECK: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
-; CHECK: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
-; CHECK: Cost of 5 for VF 16: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
-; CHECK: Cost of 5 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
-; CHECK: Cost of 9 for VF 32: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
-; CHECK: Cost of 9 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; CHECK: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; CHECK: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; CHECK: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; CHECK: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; CHECK: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; CHECK: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; CHECK: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; CHECK: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; CHECK: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; CHECK: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %p0 = getelementptr inbounds %i8.4, ptr %data, i64 %i, i32 0
@@ -127,14 +127,14 @@ define void @i8_factor_5(ptr %data, i64 %n) {
 entry:
   br label %for.body
 ; CHECK-LABEL: Checking a loop in 'i8_factor_5'
-; CHECK: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
-; CHECK: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
-; CHECK: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
-; CHECK: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
-; CHECK: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
-; CHECK: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
-; CHECK: Cost of 9 for VF 16: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
-; CHECK: Cost of 9 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; CHECK: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; CHECK: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; CHECK: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; CHECK: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; CHECK: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; CHECK: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; CHECK: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; CHECK: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %p0 = getelementptr inbounds %i8.5, ptr %data, i64 %i, i32 0
@@ -170,14 +170,14 @@ define void @i8_factor_6(ptr %data, i64 %n) {
 entry:
   br label %for.body
 ; CHECK-LABEL: Checking a loop in 'i8_factor_6'
-; CHECK: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
-; CHECK: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
-; CHECK: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
-; CHECK: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
-; CHECK: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
-; CHECK: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
-; CHECK: Cost of 9 for VF 16: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
-; CHECK: Cost of 9 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; CHECK: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; CHECK: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; CHECK: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; CHECK: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; CHECK: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; CHECK: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; CHECK: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; CHECK: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %p0 = getelementptr inbounds %i8.6, ptr %data, i64 %i, i32 0
@@ -217,14 +217,14 @@ define void @i8_factor_7(ptr %data, i64 %n) {
 entry:
   br label %for.body
 ; CHECK-LABEL: Checking a loop in 'i8_factor_7'
-; CHECK: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
-; CHECK: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
-; CHECK: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
-; CHECK: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
-; CHECK: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
-; CHECK: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
-; CHECK: Cost of 9 for VF 16: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
-; CHECK: Cost of 9 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; CHECK: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; CHECK: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; CHECK: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; CHECK: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; CHECK: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; CHECK: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; CHECK: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; CHECK: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %p0 = getelementptr inbounds %i8.7, ptr %data, i64 %i, i32 0
@@ -268,14 +268,14 @@ define void @i8_factor_8(ptr %data, i64 %n) {
 entry:
   br label %for.body
 ; CHECK-LABEL: Checking a loop in 'i8_factor_8'
-; CHECK: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
-; CHECK: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
-; CHECK: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
-; CHECK: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
-; CHECK: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
-; CHECK: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
-; CHECK: Cost of 9 for VF 16: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
-; CHECK: Cost of 9 for VF 16: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; CHECK: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; CHECK: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; CHECK: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; CHECK: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; CHECK: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; CHECK: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; CHECK: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; CHECK: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %p0 = getelementptr inbounds %i8.8, ptr %data, i64 %i, i32 0
-- 
GitLab


From 56dcfbef453d6cc390fc7a734db417e047616526 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 30 Oct 2024 14:47:29 -0700
Subject: [PATCH 649/693] [RISCV] Remove duplicate vector conversion pseudos.
 (#114287)

These pseudos used to be handled by CustomInserter to insert the
rounding
mode change for vector ceil, floor, etc. At some point they were changed
to use the InsertReadWriteCSR pass instead of the custom inserter. I
believe
that makes them redundant with the pseudos used by the RVV intrinsics
with rounding mode operand.
---
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    | 114 ------------------
 .../Target/RISCV/RISCVInstrInfoVVLPatterns.td |  20 +--
 2 files changed, 10 insertions(+), 124 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index d5b0fa340684..19557d424d1b 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -1134,46 +1134,6 @@ class VPseudoUnaryMask_NoExcept<VReg RetClass,
   let usesCustomInserter = 1;
 }
 
-class VPseudoUnaryNoMask_FRM<VReg RetClass,
-                             VReg OpClass,
-                             string Constraint = "",
-                             bits<2> TargetConstraintType = 1> :
-      Pseudo<(outs RetClass:$rd),
-             (ins RetClass:$passthru, OpClass:$rs2, vec_rm:$frm,
-                  AVL:$vl, sew:$sew, vec_policy:$policy), []>,
-      RISCVVPseudo {
-  let mayLoad = 0;
-  let mayStore = 0;
-  let hasSideEffects = 0;
-  let Constraints = !interleave([Constraint, "$rd = $passthru"], ",");
-  let TargetOverlapConstraintType = TargetConstraintType;
-  let HasVLOp = 1;
-  let HasSEWOp = 1;
-  let HasVecPolicyOp = 1;
-  let HasRoundModeOp = 1;
-}
-
-class VPseudoUnaryMask_FRM<VReg RetClass,
-                           VReg OpClass,
-                           string Constraint = "",
-                           bits<2> TargetConstraintType = 1> :
-      Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
-             (ins GetVRegNoV0<RetClass>.R:$passthru, OpClass:$rs2,
-                  VMaskOp:$vm, vec_rm:$frm,
-                  AVL:$vl, sew:$sew, vec_policy:$policy), []>,
-      RISCVVPseudo {
-  let mayLoad = 0;
-  let mayStore = 0;
-  let hasSideEffects = 0;
-  let Constraints = !interleave([Constraint, "$rd = $passthru"], ",");
-  let TargetOverlapConstraintType = TargetConstraintType;
-  let HasVLOp = 1;
-  let HasSEWOp = 1;
-  let HasVecPolicyOp = 1;
-  let UsesMaskPolicy = 1;
-  let HasRoundModeOp = 1;
-}
-
 class VPseudoUnaryNoMaskGPROut :
       Pseudo<(outs GPR:$rd),
              (ins VR:$rs2, AVL:$vl, sew:$sew), []>,
@@ -3578,23 +3538,6 @@ multiclass VPseudoConversionRoundingMode<VReg RetClass,
   }
 }
 
-
-multiclass VPseudoConversionRM<VReg RetClass,
-                               VReg Op1Class,
-                               LMULInfo MInfo,
-                               string Constraint = "",
-                               int sew = 0,
-                               bits<2> TargetConstraintType = 1> {
-  let VLMul = MInfo.value, SEW=sew in {
-    defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
-    def suffix : VPseudoUnaryNoMask_FRM<RetClass, Op1Class,
-                                        Constraint, TargetConstraintType>;
-    def suffix # "_MASK" : VPseudoUnaryMask_FRM<RetClass, Op1Class,
-                                                Constraint, TargetConstraintType>,
-                           RISCVMaskedPseudo<MaskIdx=2>;
-  }
-}
-
 multiclass VPseudoConversionNoExcept<VReg RetClass,
                                      VReg Op1Class,
                                      LMULInfo MInfo,
@@ -3620,14 +3563,6 @@ multiclass VPseudoVCVTI_V_RM {
   }
 }
 
-multiclass VPseudoVCVTI_RM_V {
-  foreach m = MxListF in {
-    defm _V : VPseudoConversionRM<m.vrclass, m.vrclass, m>,
-              SchedUnary<"WriteVFCvtFToIV", "ReadVFCvtFToIV", m.MX,
-                         forcePassthruRead=true>;
-  }
-}
-
 multiclass VPseudoVFROUND_NOEXCEPT_V {
   foreach m = MxListF in {
     defm _V : VPseudoConversionNoExcept<m.vrclass, m.vrclass, m>,
@@ -3645,15 +3580,6 @@ multiclass VPseudoVCVTF_V_RM {
   }
 }
 
-multiclass VPseudoVCVTF_RM_V {
-  foreach m = MxListF in {
-    foreach e = SchedSEWSet<m.MX, isF=1>.val in
-      defm _V : VPseudoConversionRM<m.vrclass, m.vrclass, m, sew=e>,
-                SchedUnary<"WriteVFCvtIToFV", "ReadVFCvtIToFV", m.MX, e,
-                           forcePassthruRead=true>;
-  }
-}
-
 multiclass VPseudoVWCVTI_V {
   defvar constraint = "@earlyclobber $rd";
   foreach m = MxListFW in {
@@ -3672,15 +3598,6 @@ multiclass VPseudoVWCVTI_V_RM {
   }
 }
 
-multiclass VPseudoVWCVTI_RM_V {
-  defvar constraint = "@earlyclobber $rd";
-  foreach m = MxListFW in {
-    defm _V : VPseudoConversionRM<m.wvrclass, m.vrclass, m, constraint>,
-              SchedUnary<"WriteVFWCvtFToIV", "ReadVFWCvtFToIV", m.MX,
-                         forcePassthruRead=true>;
-  }
-}
-
 multiclass VPseudoVWCVTF_V {
   defvar constraint = "@earlyclobber $rd";
   foreach m = MxListW in {
@@ -3721,15 +3638,6 @@ multiclass VPseudoVNCVTI_W_RM {
   }
 }
 
-multiclass VPseudoVNCVTI_RM_W {
-  defvar constraint = "@earlyclobber $rd";
-  foreach m = MxListW in {
-    defm _W : VPseudoConversionRM<m.vrclass, m.wvrclass, m, constraint, TargetConstraintType=2>,
-              SchedUnary<"WriteVFNCvtFToIV", "ReadVFNCvtFToIV", m.MX,
-                         forcePassthruRead=true>;
-  }
-}
-
 multiclass VPseudoVNCVTF_W_RM {
   defvar constraint = "@earlyclobber $rd";
   foreach m = MxListFW in {
@@ -3742,17 +3650,6 @@ multiclass VPseudoVNCVTF_W_RM {
   }
 }
 
-multiclass VPseudoVNCVTF_RM_W {
-  defvar constraint = "@earlyclobber $rd";
-  foreach m = MxListFW in {
-    foreach e = SchedSEWSet<m.MX, isF=1, isWidening=1>.val in
-      defm _W : VPseudoConversionRM<m.vrclass, m.wvrclass, m, constraint, sew=e,
-                                    TargetConstraintType=2>,
-                SchedUnary<"WriteVFNCvtIToFV", "ReadVFNCvtIToFV", m.MX, e,
-                           forcePassthruRead=true>;
-  }
-}
-
 multiclass VPseudoVNCVTD_W {
   defvar constraint = "@earlyclobber $rd";
   foreach m = MxListFW in {
@@ -6583,9 +6480,6 @@ defm PseudoVFCVT_XU_F : VPseudoVCVTI_V_RM;
 defm PseudoVFCVT_X_F : VPseudoVCVTI_V_RM;
 }
 
-defm PseudoVFCVT_RM_XU_F : VPseudoVCVTI_RM_V;
-defm PseudoVFCVT_RM_X_F : VPseudoVCVTI_RM_V;
-
 defm PseudoVFCVT_RTZ_XU_F : VPseudoVCVTI_V;
 defm PseudoVFCVT_RTZ_X_F : VPseudoVCVTI_V;
 
@@ -6594,8 +6488,6 @@ let hasSideEffects = 0, hasPostISelHook = 1 in {
 defm PseudoVFCVT_F_XU : VPseudoVCVTF_V_RM;
 defm PseudoVFCVT_F_X : VPseudoVCVTF_V_RM;
 }
-defm PseudoVFCVT_RM_F_XU : VPseudoVCVTF_RM_V;
-defm PseudoVFCVT_RM_F_X  : VPseudoVCVTF_RM_V;
 } // mayRaiseFPException = true
 
 //===----------------------------------------------------------------------===//
@@ -6606,8 +6498,6 @@ let hasSideEffects = 0, hasPostISelHook = 1 in {
 defm PseudoVFWCVT_XU_F     : VPseudoVWCVTI_V_RM;
 defm PseudoVFWCVT_X_F      : VPseudoVWCVTI_V_RM;
 }
-defm PseudoVFWCVT_RM_XU_F  : VPseudoVWCVTI_RM_V;
-defm PseudoVFWCVT_RM_X_F   : VPseudoVWCVTI_RM_V;
 
 defm PseudoVFWCVT_RTZ_XU_F : VPseudoVWCVTI_V;
 defm PseudoVFWCVT_RTZ_X_F  : VPseudoVWCVTI_V;
@@ -6627,8 +6517,6 @@ let hasSideEffects = 0, hasPostISelHook = 1 in {
 defm PseudoVFNCVT_XU_F     : VPseudoVNCVTI_W_RM;
 defm PseudoVFNCVT_X_F      : VPseudoVNCVTI_W_RM;
 }
-defm PseudoVFNCVT_RM_XU_F  : VPseudoVNCVTI_RM_W;
-defm PseudoVFNCVT_RM_X_F   : VPseudoVNCVTI_RM_W;
 
 defm PseudoVFNCVT_RTZ_XU_F : VPseudoVNCVTI_W;
 defm PseudoVFNCVT_RTZ_X_F  : VPseudoVNCVTI_W;
@@ -6637,8 +6525,6 @@ let hasSideEffects = 0, hasPostISelHook = 1 in {
 defm PseudoVFNCVT_F_XU     : VPseudoVNCVTF_W_RM;
 defm PseudoVFNCVT_F_X      : VPseudoVNCVTF_W_RM;
 }
-defm PseudoVFNCVT_RM_F_XU  : VPseudoVNCVTF_RM_W;
-defm PseudoVFNCVT_RM_F_X   : VPseudoVNCVTF_RM_W;
 
 let hasSideEffects = 0, hasPostISelHook = 1 in {
 defm PseudoVFNCVT_F_F      : VPseudoVNCVTD_W_RM;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index 18749f00a10a..33e1ed120cd0 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -2639,8 +2639,8 @@ foreach fvti = AllFloatVectors in {
 // 13.17. Vector Single-Width Floating-Point/Integer Type-Convert Instructions
 defm : VPatConvertFP2IVL_V_RM<riscv_vfcvt_xu_f_vl, "PseudoVFCVT_XU_F_V">;
 defm : VPatConvertFP2IVL_V_RM<riscv_vfcvt_x_f_vl, "PseudoVFCVT_X_F_V">;
-defm : VPatConvertFP2I_RM_VL_V<riscv_vfcvt_rm_xu_f_vl, "PseudoVFCVT_RM_XU_F_V">;
-defm : VPatConvertFP2I_RM_VL_V<any_riscv_vfcvt_rm_x_f_vl, "PseudoVFCVT_RM_X_F_V">;
+defm : VPatConvertFP2I_RM_VL_V<riscv_vfcvt_rm_xu_f_vl, "PseudoVFCVT_XU_F_V">;
+defm : VPatConvertFP2I_RM_VL_V<any_riscv_vfcvt_rm_x_f_vl, "PseudoVFCVT_X_F_V">;
 
 defm : VPatConvertFP2IVL_V<any_riscv_vfcvt_rtz_xu_f_vl, "PseudoVFCVT_RTZ_XU_F_V">;
 defm : VPatConvertFP2IVL_V<any_riscv_vfcvt_rtz_x_f_vl, "PseudoVFCVT_RTZ_X_F_V">;
@@ -2648,14 +2648,14 @@ defm : VPatConvertFP2IVL_V<any_riscv_vfcvt_rtz_x_f_vl, "PseudoVFCVT_RTZ_X_F_V">;
 defm : VPatConvertI2FPVL_V_RM<any_riscv_uint_to_fp_vl, "PseudoVFCVT_F_XU_V">;
 defm : VPatConvertI2FPVL_V_RM<any_riscv_sint_to_fp_vl, "PseudoVFCVT_F_X_V">;
 
-defm : VPatConvertI2FP_RM_VL_V<riscv_vfcvt_rm_f_xu_vl, "PseudoVFCVT_RM_F_XU_V">;
-defm : VPatConvertI2FP_RM_VL_V<riscv_vfcvt_rm_f_x_vl, "PseudoVFCVT_RM_F_X_V">;
+defm : VPatConvertI2FP_RM_VL_V<riscv_vfcvt_rm_f_xu_vl, "PseudoVFCVT_F_XU_V">;
+defm : VPatConvertI2FP_RM_VL_V<riscv_vfcvt_rm_f_x_vl, "PseudoVFCVT_F_X_V">;
 
 // 13.18. Widening Floating-Point/Integer Type-Convert Instructions
 defm : VPatWConvertFP2IVL_V_RM<riscv_vfcvt_xu_f_vl, "PseudoVFWCVT_XU_F_V">;
 defm : VPatWConvertFP2IVL_V_RM<riscv_vfcvt_x_f_vl, "PseudoVFWCVT_X_F_V">;
-defm : VPatWConvertFP2I_RM_VL_V<riscv_vfcvt_rm_xu_f_vl, "PseudoVFWCVT_RM_XU_F_V">;
-defm : VPatWConvertFP2I_RM_VL_V<riscv_vfcvt_rm_x_f_vl, "PseudoVFWCVT_RM_X_F_V">;
+defm : VPatWConvertFP2I_RM_VL_V<riscv_vfcvt_rm_xu_f_vl, "PseudoVFWCVT_XU_F_V">;
+defm : VPatWConvertFP2I_RM_VL_V<riscv_vfcvt_rm_x_f_vl, "PseudoVFWCVT_X_F_V">;
 
 defm : VPatWConvertFP2IVL_V<any_riscv_vfcvt_rtz_xu_f_vl, "PseudoVFWCVT_RTZ_XU_F_V">;
 defm : VPatWConvertFP2IVL_V<any_riscv_vfcvt_rtz_x_f_vl, "PseudoVFWCVT_RTZ_X_F_V">;
@@ -2696,8 +2696,8 @@ foreach fvtiToFWti = AllWidenableBFloatToFloatVectors in {
 // 13.19 Narrowing Floating-Point/Integer Type-Convert Instructions
 defm : VPatNConvertFP2IVL_W_RM<riscv_vfcvt_xu_f_vl, "PseudoVFNCVT_XU_F_W">;
 defm : VPatNConvertFP2IVL_W_RM<riscv_vfcvt_x_f_vl, "PseudoVFNCVT_X_F_W">;
-defm : VPatNConvertFP2I_RM_VL_W<riscv_vfcvt_rm_xu_f_vl, "PseudoVFNCVT_RM_XU_F_W">;
-defm : VPatNConvertFP2I_RM_VL_W<riscv_vfcvt_rm_x_f_vl, "PseudoVFNCVT_RM_X_F_W">;
+defm : VPatNConvertFP2I_RM_VL_W<riscv_vfcvt_rm_xu_f_vl, "PseudoVFNCVT_XU_F_W">;
+defm : VPatNConvertFP2I_RM_VL_W<riscv_vfcvt_rm_x_f_vl, "PseudoVFNCVT_X_F_W">;
 
 defm : VPatNConvertFP2IVL_W<any_riscv_vfcvt_rtz_xu_f_vl, "PseudoVFNCVT_RTZ_XU_F_W">;
 defm : VPatNConvertFP2IVL_W<any_riscv_vfcvt_rtz_x_f_vl, "PseudoVFNCVT_RTZ_X_F_W">;
@@ -2705,8 +2705,8 @@ defm : VPatNConvertFP2IVL_W<any_riscv_vfcvt_rtz_x_f_vl, "PseudoVFNCVT_RTZ_X_F_W"
 defm : VPatNConvertI2FPVL_W_RM<any_riscv_uint_to_fp_vl, "PseudoVFNCVT_F_XU_W">;
 defm : VPatNConvertI2FPVL_W_RM<any_riscv_sint_to_fp_vl, "PseudoVFNCVT_F_X_W">;
 
-defm : VPatNConvertI2FP_RM_VL_W<riscv_vfcvt_rm_f_xu_vl, "PseudoVFNCVT_RM_F_XU_W">;
-defm : VPatNConvertI2FP_RM_VL_W<riscv_vfcvt_rm_f_x_vl, "PseudoVFNCVT_RM_F_X_W">;
+defm : VPatNConvertI2FP_RM_VL_W<riscv_vfcvt_rm_f_xu_vl, "PseudoVFNCVT_F_XU_W">;
+defm : VPatNConvertI2FP_RM_VL_W<riscv_vfcvt_rm_f_x_vl, "PseudoVFNCVT_F_X_W">;
 
 foreach fvtiToFWti = AllWidenableFloatVectors in {
   defvar fvti = fvtiToFWti.Vti;
-- 
GitLab


From 5d35747f6de9295400327744b389f303e3e2b13d Mon Sep 17 00:00:00 2001
From: Michael Jones <michaelrj@google.com>
Date: Wed, 30 Oct 2024 15:05:13 -0700
Subject: [PATCH 650/693] [libc] Refactor statvfs tests (#114147)

The previous statvfs tests had several issues, this patch updates them
to meet current standards.
---
 .../test/src/sys/statvfs/linux/CMakeLists.txt |  6 +-
 .../src/sys/statvfs/linux/fstatvfs_test.cpp   | 81 ++++++++++---------
 .../src/sys/statvfs/linux/statvfs_test.cpp    | 75 ++++++++---------
 3 files changed, 80 insertions(+), 82 deletions(-)

diff --git a/libc/test/src/sys/statvfs/linux/CMakeLists.txt b/libc/test/src/sys/statvfs/linux/CMakeLists.txt
index 1f8688868e04..fa1e9052d1ca 100644
--- a/libc/test/src/sys/statvfs/linux/CMakeLists.txt
+++ b/libc/test/src/sys/statvfs/linux/CMakeLists.txt
@@ -8,8 +8,9 @@ add_libc_unittest(
     statvfs_test.cpp
   DEPENDS
     libc.src.errno.errno
-    libc.src.sys.statvfs.linux.statfs_utils
     libc.src.sys.statvfs.statvfs
+    libc.src.sys.stat.mkdirat
+    libc.src.sys.stat.rmdir
     libc.test.UnitTest.ErrnoSetterMatcher
 )
 
@@ -21,8 +22,9 @@ add_libc_unittest(
     fstatvfs_test.cpp
   DEPENDS
     libc.src.errno.errno
-    libc.src.sys.statvfs.linux.statfs_utils
     libc.src.sys.statvfs.fstatvfs
+    libc.src.sys.stat.mkdirat
+    libc.src.sys.stat.rmdir
     libc.src.fcntl.open
     libc.src.unistd.close
     libc.test.UnitTest.ErrnoSetterMatcher
diff --git a/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp b/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp
index 2f3e0b96ff09..efd1e688280b 100644
--- a/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp
+++ b/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp
@@ -1,49 +1,56 @@
+//===-- Unittests for fstatvfs --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
 #include "hdr/fcntl_macros.h"
 #include "src/__support/macros/config.h"
 #include "src/fcntl/open.h"
+#include "src/sys/stat/mkdirat.h"
 #include "src/sys/statvfs/fstatvfs.h"
-#include "src/sys/statvfs/linux/statfs_utils.h"
 #include "src/unistd/close.h"
+#include "src/unistd/rmdir.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
-#include "test/UnitTest/LibcTest.h"
-#include <linux/magic.h>
+#include "test/UnitTest/Test.h"
+
 using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
 
-#ifdef SYS_statfs64
-using StatFs = statfs64;
-#else
-using StatFs = statfs;
-#endif
-
-namespace LIBC_NAMESPACE_DECL {
-static int fstatfs(int fd, StatFs *buf) {
-  using namespace statfs_utils;
-  if (cpp::optional<StatFs> result = linux_fstatfs(fd)) {
-    *buf = *result;
-    return 0;
-  }
-  return -1;
-}
-} // namespace LIBC_NAMESPACE_DECL
-
-struct PathFD {
-  int fd;
-  explicit PathFD(const char *path)
-      : fd(LIBC_NAMESPACE::open(path, O_CLOEXEC | O_PATH)) {}
-  ~PathFD() { LIBC_NAMESPACE::close(fd); }
-  operator int() const { return fd; }
-};
-
-TEST(LlvmLibcSysStatvfsTest, FstatfsBasic) {
-  StatFs buf;
-  ASSERT_THAT(LIBC_NAMESPACE::fstatfs(PathFD("/"), &buf), Succeeds());
-  ASSERT_THAT(LIBC_NAMESPACE::fstatfs(PathFD("/proc"), &buf), Succeeds());
-  ASSERT_EQ(buf.f_type, static_cast<decltype(buf.f_type)>(PROC_SUPER_MAGIC));
-  ASSERT_THAT(LIBC_NAMESPACE::fstatfs(PathFD("/sys"), &buf), Succeeds());
-  ASSERT_EQ(buf.f_type, static_cast<decltype(buf.f_type)>(SYSFS_MAGIC));
+TEST(LlvmLibcSysFStatvfsTest, FStatvfsBasic) {
+  struct statvfs buf;
+
+  int fd = LIBC_NAMESPACE::open("/", O_PATH);
+  ASSERT_ERRNO_SUCCESS();
+  ASSERT_GT(fd, 0);
+
+  // The root of the file directory must always exist
+  ASSERT_THAT(LIBC_NAMESPACE::fstatvfs(fd, &buf), Succeeds());
+  ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds(0));
 }
 
-TEST(LlvmLibcSysStatvfsTest, FstatvfsInvalidFD) {
+TEST(LlvmLibcSysFStatvfsTest, FStatvfsInvalidPath) {
   struct statvfs buf;
-  ASSERT_THAT(LIBC_NAMESPACE::fstatvfs(-1, &buf), Fails(EBADF));
+
+  constexpr const char *FILENAME = "testdata/statvfs.testdir";
+  auto TEST_DIR = libc_make_test_file_path(FILENAME);
+
+  ASSERT_THAT(LIBC_NAMESPACE::mkdirat(AT_FDCWD, TEST_DIR, S_IRWXU),
+              Succeeds(0));
+
+  int fd = LIBC_NAMESPACE::open(TEST_DIR, O_PATH);
+  ASSERT_ERRNO_SUCCESS();
+  ASSERT_GT(fd, 0);
+
+  // create the file, assert it exists, then delete it and assert it doesn't
+  // exist anymore.
+
+  ASSERT_THAT(LIBC_NAMESPACE::fstatvfs(fd, &buf), Succeeds());
+
+  ASSERT_THAT(LIBC_NAMESPACE::rmdir(TEST_DIR), Succeeds(0));
+
+  ASSERT_THAT(LIBC_NAMESPACE::fstatvfs(fd, &buf), Fails(ENOENT));
+  ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds(0));
+  ASSERT_THAT(LIBC_NAMESPACE::fstatvfs(fd, &buf), Fails(ENOENT));
 }
diff --git a/libc/test/src/sys/statvfs/linux/statvfs_test.cpp b/libc/test/src/sys/statvfs/linux/statvfs_test.cpp
index 5329adb54d64..0b154e7aa3fb 100644
--- a/libc/test/src/sys/statvfs/linux/statvfs_test.cpp
+++ b/libc/test/src/sys/statvfs/linux/statvfs_test.cpp
@@ -1,54 +1,43 @@
+//===-- Unittests for statvfs ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/fcntl_macros.h"
 #include "src/__support/macros/config.h"
-#include "src/sys/statvfs/linux/statfs_utils.h"
+#include "src/sys/stat/mkdirat.h"
 #include "src/sys/statvfs/statvfs.h"
+#include "src/unistd/rmdir.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
-#include "test/UnitTest/LibcTest.h"
-#include <linux/magic.h>
+#include "test/UnitTest/Test.h"
+
 using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
 
-#ifdef SYS_statfs64
-using StatFs = statfs64;
-#else
-using StatFs = statfs;
-#endif
-
-namespace LIBC_NAMESPACE_DECL {
-static int statfs(const char *path, StatFs *buf) {
-  using namespace statfs_utils;
-  if (cpp::optional<LinuxStatFs> result = linux_statfs(path)) {
-    *buf = *result;
-    return 0;
-  }
-  return -1;
-}
-} // namespace LIBC_NAMESPACE_DECL
-
-TEST(LlvmLibcSysStatfsTest, StatfsBasic) {
-  StatFs buf;
-  ASSERT_THAT(LIBC_NAMESPACE::statfs("/", &buf), Succeeds());
-  ASSERT_THAT(LIBC_NAMESPACE::statfs("/proc", &buf), Succeeds());
-  ASSERT_EQ(buf.f_type, static_cast<decltype(buf.f_type)>(PROC_SUPER_MAGIC));
-  ASSERT_THAT(LIBC_NAMESPACE::statfs("/sys", &buf), Succeeds());
-  ASSERT_EQ(buf.f_type, static_cast<decltype(buf.f_type)>(SYSFS_MAGIC));
+TEST(LlvmLibcSysStatvfsTest, StatvfsBasic) {
+  struct statvfs buf;
+  // The root of the file directory must always exist
+  ASSERT_THAT(LIBC_NAMESPACE::statvfs("/", &buf), Succeeds());
 }
 
-TEST(LlvmLibcSysStatfsTest, StatvfsInvalidPath) {
+TEST(LlvmLibcSysStatvfsTest, StatvfsInvalidPath) {
   struct statvfs buf;
+
   ASSERT_THAT(LIBC_NAMESPACE::statvfs("", &buf), Fails(ENOENT));
-  ASSERT_THAT(LIBC_NAMESPACE::statvfs("/nonexistent", &buf), Fails(ENOENT));
-  ASSERT_THAT(LIBC_NAMESPACE::statvfs("/dev/null/whatever", &buf),
-              Fails(ENOTDIR));
-  ASSERT_THAT(LIBC_NAMESPACE::statvfs(nullptr, &buf), Fails(EFAULT));
-}
 
-TEST(LlvmLibcSysStatfsTest, StatvfsNameTooLong) {
-  struct statvfs buf;
-  ASSERT_THAT(LIBC_NAMESPACE::statvfs("/", &buf), Succeeds());
-  char *name = static_cast<char *>(__builtin_alloca(buf.f_namemax + 3));
-  name[0] = '/';
-  name[buf.f_namemax + 2] = '\0';
-  for (unsigned i = 1; i < buf.f_namemax + 2; ++i) {
-    name[i] = 'a';
-  }
-  ASSERT_THAT(LIBC_NAMESPACE::statvfs(name, &buf), Fails(ENAMETOOLONG));
+  // create the file, assert it exists, then delete it and assert it doesn't
+  // exist anymore.
+  constexpr const char *FILENAME = "testdata/statvfs.testdir";
+  auto TEST_DIR = libc_make_test_file_path(FILENAME);
+
+  ASSERT_THAT(LIBC_NAMESPACE::mkdirat(AT_FDCWD, TEST_DIR, S_IRWXU),
+              Succeeds(0));
+
+  ASSERT_THAT(LIBC_NAMESPACE::statvfs(TEST_DIR, &buf), Succeeds());
+
+  ASSERT_THAT(LIBC_NAMESPACE::rmdir(TEST_DIR), Succeeds(0));
+
+  ASSERT_THAT(LIBC_NAMESPACE::statvfs(TEST_DIR, &buf), Fails(ENOENT));
 }
-- 
GitLab


From 50c44478fe3f680374edf1363d2a3617b8ff2a0b Mon Sep 17 00:00:00 2001
From: George Burgess IV <george.burgess.iv@gmail.com>
Date: Wed, 30 Oct 2024 16:08:03 -0600
Subject: [PATCH 651/693] [libc] fix behavior of strrchr(x, '\0') (#112620)

`strrchr("foo", '\0')` is defined to point to the end of `foo`, rather
than returning NULL. This wasn't caught by tests, since llvm-libc's
`ASSERT_STREQ(nullptr, "");` is not an assertion error.

While I'm here, refactor the test slightly to check for NULL more
specifically. I considered adding fancier `ASSERT`s (and changing the
semantics of `ASSERT_STREQ`), but opted for a more local fix by fair
dice roll.
---
 libc/src/string/string_utils.h    |  6 ++++--
 libc/test/UnitTest/LibcTest.h     |  8 +++++++
 libc/test/src/string/StrchrTest.h | 36 +++++++++++++++++++------------
 3 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/libc/src/string/string_utils.h b/libc/src/string/string_utils.h
index 240b28f15718..22a1876da536 100644
--- a/libc/src/string/string_utils.h
+++ b/libc/src/string/string_utils.h
@@ -239,11 +239,13 @@ LIBC_INLINE constexpr static char *strrchr_implementation(const char *src,
                                                           int c) {
   char ch = static_cast<char>(c);
   char *last_occurrence = nullptr;
-  for (; *src; ++src) {
+  while (true) {
     if (*src == ch)
       last_occurrence = const_cast<char *>(src);
+    if (!*src)
+      return last_occurrence;
+    ++src;
   }
-  return last_occurrence;
 }
 
 } // namespace internal
diff --git a/libc/test/UnitTest/LibcTest.h b/libc/test/UnitTest/LibcTest.h
index 2b972004e9ee..1707c3c0fdcf 100644
--- a/libc/test/UnitTest/LibcTest.h
+++ b/libc/test/UnitTest/LibcTest.h
@@ -162,6 +162,14 @@ protected:
                           (unsigned long long)RHS, LHSStr, RHSStr, Loc);
   }
 
+  // Helper to allow macro invocations like `ASSERT_EQ(foo, nullptr)`.
+  template <typename ValType,
+            cpp::enable_if_t<cpp::is_pointer_v<ValType>, ValType> = nullptr>
+  bool test(TestCond Cond, ValType LHS, std::nullptr_t, const char *LHSStr,
+            const char *RHSStr, internal::Location Loc) {
+    return test(Cond, LHS, static_cast<ValType>(nullptr), LHSStr, RHSStr, Loc);
+  }
+
   template <
       typename ValType,
       cpp::enable_if_t<
diff --git a/libc/test/src/string/StrchrTest.h b/libc/test/src/string/StrchrTest.h
index 74e172de9595..8c3fe5293008 100644
--- a/libc/test/src/string/StrchrTest.h
+++ b/libc/test/src/string/StrchrTest.h
@@ -40,14 +40,16 @@ template <auto Func> struct StrchrTest : public LIBC_NAMESPACE::testing::Test {
     const char *src = "abcde";
 
     // Should return null terminator.
-    ASSERT_STREQ(Func(src, '\0'), "");
+    const char *nul_terminator = Func(src, '\0');
+    ASSERT_NE(nul_terminator, nullptr);
+    ASSERT_STREQ(nul_terminator, "");
     // Source string should not change.
     ASSERT_STREQ(src, "abcde");
   }
 
   void characterNotWithinStringShouldReturnNullptr() {
     // Since 'z' is not within the string, should return nullptr.
-    ASSERT_STREQ(Func("123?", 'z'), nullptr);
+    ASSERT_EQ(Func("123?", 'z'), nullptr);
   }
 
   void theSourceShouldNotChange() {
@@ -74,11 +76,13 @@ template <auto Func> struct StrchrTest : public LIBC_NAMESPACE::testing::Test {
 
   void emptyStringShouldOnlyMatchNullTerminator() {
     // Null terminator should match.
-    ASSERT_STREQ(Func("", '\0'), "");
+    const char empty_string[] = "";
+    ASSERT_EQ(static_cast<const char *>(Func(empty_string, '\0')),
+              empty_string);
     // All other characters should not match.
-    ASSERT_STREQ(Func("", 'Z'), nullptr);
-    ASSERT_STREQ(Func("", '3'), nullptr);
-    ASSERT_STREQ(Func("", '*'), nullptr);
+    ASSERT_EQ(Func("", 'Z'), nullptr);
+    ASSERT_EQ(Func("", '3'), nullptr);
+    ASSERT_EQ(Func("", '*'), nullptr);
   }
 };
 
@@ -114,7 +118,9 @@ template <auto Func> struct StrrchrTest : public LIBC_NAMESPACE::testing::Test {
     const char *src = "abcde";
 
     // Should return null terminator.
-    ASSERT_STREQ(Func(src, '\0'), "");
+    const char *nul_terminator = Func(src, '\0');
+    ASSERT_NE(nul_terminator, nullptr);
+    ASSERT_STREQ(nul_terminator, "");
     // Source string should not change.
     ASSERT_STREQ(src, "abcde");
   }
@@ -122,9 +128,9 @@ template <auto Func> struct StrrchrTest : public LIBC_NAMESPACE::testing::Test {
   void findsLastBehindFirstNullTerminator() {
     static const char src[6] = {'a', 'a', '\0', 'b', '\0', 'c'};
     // 'b' is behind a null terminator, so should not be found.
-    ASSERT_STREQ(Func(src, 'b'), nullptr);
+    ASSERT_EQ(Func(src, 'b'), nullptr);
     // Same goes for 'c'.
-    ASSERT_STREQ(Func(src, 'c'), nullptr);
+    ASSERT_EQ(Func(src, 'c'), nullptr);
 
     // Should find the second of the two a's.
     ASSERT_STREQ(Func(src, 'a'), "a");
@@ -132,7 +138,7 @@ template <auto Func> struct StrrchrTest : public LIBC_NAMESPACE::testing::Test {
 
   void characterNotWithinStringShouldReturnNullptr() {
     // Since 'z' is not within the string, should return nullptr.
-    ASSERT_STREQ(Func("123?", 'z'), nullptr);
+    ASSERT_EQ(Func("123?", 'z'), nullptr);
   }
 
   void shouldFindLastOfDuplicates() {
@@ -146,11 +152,13 @@ template <auto Func> struct StrrchrTest : public LIBC_NAMESPACE::testing::Test {
 
   void emptyStringShouldOnlyMatchNullTerminator() {
     // Null terminator should match.
-    ASSERT_STREQ(Func("", '\0'), "");
+    const char empty_string[] = "";
+    ASSERT_EQ(static_cast<const char *>(Func(empty_string, '\0')),
+              empty_string);
     // All other characters should not match.
-    ASSERT_STREQ(Func("", 'A'), nullptr);
-    ASSERT_STREQ(Func("", '2'), nullptr);
-    ASSERT_STREQ(Func("", '*'), nullptr);
+    ASSERT_EQ(Func("", 'A'), nullptr);
+    ASSERT_EQ(Func("", '2'), nullptr);
+    ASSERT_EQ(Func("", '*'), nullptr);
   }
 };
 
-- 
GitLab


From 36d56925706a32a065ec50d5a6b418e1f29a27b3 Mon Sep 17 00:00:00 2001
From: gulfemsavrun <gulfem@google.com>
Date: Wed, 30 Oct 2024 15:10:29 -0700
Subject: [PATCH 652/693] Revert "[TLI] Add support for hypot libcall."
 (#114312)

Reverts llvm/llvm-project#113724
---
 llvm/include/llvm/Analysis/TargetLibraryInfo.def  | 15 ---------------
 llvm/lib/Analysis/TargetLibraryInfo.cpp           |  2 --
 llvm/lib/Transforms/Utils/BuildLibCalls.cpp       |  3 ---
 .../Transforms/InferFunctionAttrs/annotate.ll     |  9 ---------
 .../tools/llvm-tli-checker/ps4-tli-check.yaml     | 12 ------------
 llvm/unittests/Analysis/TargetLibraryInfoTest.cpp |  3 ---
 6 files changed, 44 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.def b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
index fd53a26ef8fc..3e23e398f6a7 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.def
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
@@ -1671,21 +1671,6 @@ TLI_DEFINE_ENUM_INTERNAL(htons)
 TLI_DEFINE_STRING_INTERNAL("htons")
 TLI_DEFINE_SIG_INTERNAL(Int16, Int16)
 
-/// double hypot(double x, double y);
-TLI_DEFINE_ENUM_INTERNAL(hypot)
-TLI_DEFINE_STRING_INTERNAL("hypot")
-TLI_DEFINE_SIG_INTERNAL(Dbl, Dbl, Dbl)
-
-/// float hypotf(float x, float y);
-TLI_DEFINE_ENUM_INTERNAL(hypotf)
-TLI_DEFINE_STRING_INTERNAL("hypotf")
-TLI_DEFINE_SIG_INTERNAL(Flt, Flt, Flt)
-
-/// long double hypotl(long double x, long double y);
-TLI_DEFINE_ENUM_INTERNAL(hypotl)
-TLI_DEFINE_STRING_INTERNAL("hypotl")
-TLI_DEFINE_SIG_INTERNAL(LDbl, LDbl, LDbl)
-
 /// int iprintf(const char *format, ...);
 TLI_DEFINE_ENUM_INTERNAL(iprintf)
 TLI_DEFINE_STRING_INTERNAL("iprintf")
diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index 7f0b98ab3c15..0ee83d217a50 100644
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -300,7 +300,6 @@ static void initializeLibCalls(TargetLibraryInfoImpl &TLI, const Triple &T,
       TLI.setUnavailable(LibFunc_expf);
       TLI.setUnavailable(LibFunc_floorf);
       TLI.setUnavailable(LibFunc_fmodf);
-      TLI.setUnavailable(LibFunc_hypotf);
       TLI.setUnavailable(LibFunc_log10f);
       TLI.setUnavailable(LibFunc_logf);
       TLI.setUnavailable(LibFunc_modff);
@@ -332,7 +331,6 @@ static void initializeLibCalls(TargetLibraryInfoImpl &TLI, const Triple &T,
     TLI.setUnavailable(LibFunc_floorl);
     TLI.setUnavailable(LibFunc_fmodl);
     TLI.setUnavailable(LibFunc_frexpl);
-    TLI.setUnavailable(LibFunc_hypotl);
     TLI.setUnavailable(LibFunc_ldexpl);
     TLI.setUnavailable(LibFunc_log10l);
     TLI.setUnavailable(LibFunc_logl);
diff --git a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
index e039457f313b..5fd4fd78c28a 100644
--- a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -1215,9 +1215,6 @@ bool llvm::inferNonMandatoryLibFuncAttrs(Function &F,
   case LibFunc_fmod:
   case LibFunc_fmodf:
   case LibFunc_fmodl:
-  case LibFunc_hypot:
-  case LibFunc_hypotf:
-  case LibFunc_hypotl:
   case LibFunc_isascii:
   case LibFunc_isdigit:
   case LibFunc_labs:
diff --git a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
index 452d90aa98d8..d8266f4c6703 100644
--- a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
+++ b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
@@ -589,15 +589,6 @@ declare ptr @gets(ptr)
 ; CHECK: declare noundef i32 @gettimeofday(ptr nocapture noundef, ptr nocapture noundef) [[NOFREE_NOUNWIND]]
 declare i32 @gettimeofday(ptr, ptr)
 
-; CHECK: declare double @hypot(double, double) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
-declare double @hypot(double, double)
-
-; CHECK: declare float @hypotf(float, float) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
-declare float @hypotf(float, float)
-
-; CHECK: declare x86_fp80 @hypotl(x86_fp80, x86_fp80) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
-declare x86_fp80 @hypotl(x86_fp80, x86_fp80)
-
 ; CHECK: declare i32 @isascii(i32) [[NOFREE_NOUNWIND_WILLRETURN_WRITEONLY]]
 declare i32 @isascii(i32)
 
diff --git a/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml b/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
index d52f3c751b06..408b9c399342 100644
--- a/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
+++ b/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
@@ -602,18 +602,6 @@ DynamicSymbols:
     Type:            STT_FUNC
     Section:         .text
     Binding:         STB_GLOBAL
-  - Name:            hypot
-    Type:            STT_FUNC
-    Section:         .text
-    Binding:         STB_GLOBAL
-  - Name:            hypotf
-    Type:            STT_FUNC
-    Section:         .text
-    Binding:         STB_GLOBAL
-  - Name:            hypotl
-    Type:            STT_FUNC
-    Section:         .text
-    Binding:         STB_GLOBAL
   - Name:            isdigit
     Type:            STT_FUNC
     Section:         .text
diff --git a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
index 982d00c5d335..98f8989d4e6e 100644
--- a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
+++ b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
@@ -249,9 +249,6 @@ TEST_F(TargetLibraryInfoTest, ValidProto) {
       "declare %struct* @getpwnam(i8*)\n"
       "declare i8* @gets(i8*)\n"
       "declare i32 @gettimeofday(%struct*, i8*)\n"
-      "declare double @hypot(double, double)\n"
-      "declare float @hypotf(float, float)\n"
-      "declare x86_fp80 @hypotl(x86_fp80, x86_fp80)\n"
       "declare i32 @_Z7isasciii(i32)\n"
       "declare i32 @_Z7isdigiti(i32)\n"
       "declare i64 @labs(i64)\n"
-- 
GitLab


From 1cecc58c3f15e3d0fe97b7f764d498e4005557e0 Mon Sep 17 00:00:00 2001
From: Artem Belevich <tra@google.com>
Date: Wed, 30 Oct 2024 15:13:06 -0700
Subject: [PATCH 653/693] [NVPTX] instcombine known pointer AS checks.
 (#112964)

The change improves the code in general and, as a side effect, avoids crashing
on an impossible address space casts guarded by `__isGlobal/__isShared`, which
partially fixes https://github.com/llvm/llvm-project/issues/112760
It's still possible to trigger the issue by using explicit AS casts w/o
AS checks, but LLVM should no longer crash on valid code.
---
 llvm/include/llvm/Support/NVPTXAddrSpace.h    |  33 +++
 .../Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h |  12 +-
 .../Target/NVPTX/NVPTXTargetTransformInfo.cpp |  63 +++-
 .../Transforms/InstCombine/NVPTX/isspacep.ll  | 277 ++++++++++++++++++
 4 files changed, 372 insertions(+), 13 deletions(-)
 create mode 100644 llvm/include/llvm/Support/NVPTXAddrSpace.h
 create mode 100644 llvm/test/Transforms/InstCombine/NVPTX/isspacep.ll

diff --git a/llvm/include/llvm/Support/NVPTXAddrSpace.h b/llvm/include/llvm/Support/NVPTXAddrSpace.h
new file mode 100644
index 000000000000..93eae39e3d23
--- /dev/null
+++ b/llvm/include/llvm/Support/NVPTXAddrSpace.h
@@ -0,0 +1,33 @@
+//===---------------- NVPTXAddrSpace.h -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// NVPTX address space definition
+///
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_NVPTXADDRSPACE_H
+#define LLVM_SUPPORT_NVPTXADDRSPACE_H
+
+namespace llvm {
+namespace NVPTXAS {
+enum AddressSpace : unsigned {
+  ADDRESS_SPACE_GENERIC = 0,
+  ADDRESS_SPACE_GLOBAL = 1,
+  ADDRESS_SPACE_SHARED = 3,
+  ADDRESS_SPACE_CONST = 4,
+  ADDRESS_SPACE_LOCAL = 5,
+
+  ADDRESS_SPACE_PARAM = 101,
+};
+} // end namespace NVPTXAS
+
+} // end namespace llvm
+
+#endif // LLVM_SUPPORT_NVPTXADDRSPACE_H
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h
index 815b600fe93a..d06e2c00ec3f 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h
@@ -16,18 +16,10 @@
 #ifndef LLVM_LIB_TARGET_NVPTX_MCTARGETDESC_NVPTXBASEINFO_H
 #define LLVM_LIB_TARGET_NVPTX_MCTARGETDESC_NVPTXBASEINFO_H
 
+#include "llvm/Support/NVPTXAddrSpace.h"
 namespace llvm {
 
-enum AddressSpace {
-  ADDRESS_SPACE_GENERIC = 0,
-  ADDRESS_SPACE_GLOBAL = 1,
-  ADDRESS_SPACE_SHARED = 3,
-  ADDRESS_SPACE_CONST = 4,
-  ADDRESS_SPACE_LOCAL = 5,
-
-  // NVVM Internal
-  ADDRESS_SPACE_PARAM = 101
-};
+using namespace NVPTXAS;
 
 namespace NVPTXII {
 enum {
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index e35ba25b4788..31087a0054e9 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -15,10 +15,12 @@
 #include "llvm/CodeGen/CostTable.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/IR/Constants.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsNVPTX.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include <optional>
 using namespace llvm;
@@ -117,7 +119,8 @@ bool NVPTXTTIImpl::isSourceOfDivergence(const Value *V) {
 }
 
 // Convert NVVM intrinsics to target-generic LLVM code where possible.
-static Instruction *simplifyNvvmIntrinsic(IntrinsicInst *II, InstCombiner &IC) {
+static Instruction *convertNvvmIntrinsicToLlvm(InstCombiner &IC,
+                                               IntrinsicInst *II) {
   // Each NVVM intrinsic we can simplify can be replaced with one of:
   //
   //  * an LLVM intrinsic,
@@ -413,11 +416,65 @@ static Instruction *simplifyNvvmIntrinsic(IntrinsicInst *II, InstCombiner &IC) {
   llvm_unreachable("All SpecialCase enumerators should be handled in switch.");
 }
 
+// Returns an instruction pointer (may be nullptr if we do not know the answer).
+// Returns nullopt if `II` is not one of the `isspacep` intrinsics.
+static std::optional<Instruction *>
+handleSpaceCheckIntrinsics(InstCombiner &IC, IntrinsicInst &II) {
+  Value *Op0 = II.getArgOperand(0);
+  // Returns true/false when we know the answer, nullopt otherwise.
+  auto CheckASMatch = [](unsigned IID, unsigned AS) -> std::optional<bool> {
+    if (AS == NVPTXAS::ADDRESS_SPACE_GENERIC ||
+        AS == NVPTXAS::ADDRESS_SPACE_PARAM)
+      return std::nullopt; // Got to check at run-time.
+    switch (IID) {
+    case Intrinsic::nvvm_isspacep_global:
+      return AS == NVPTXAS::ADDRESS_SPACE_GLOBAL;
+    case Intrinsic::nvvm_isspacep_local:
+      return AS == NVPTXAS::ADDRESS_SPACE_LOCAL;
+    case Intrinsic::nvvm_isspacep_shared:
+      return AS == NVPTXAS::ADDRESS_SPACE_SHARED;
+    case Intrinsic::nvvm_isspacep_shared_cluster:
+      // We can't tell shared from shared_cluster at compile time from AS alone,
+      // but it can't be either is AS is not shared.
+      return AS == NVPTXAS::ADDRESS_SPACE_SHARED ? std::nullopt
+                                                 : std::optional{false};
+    case Intrinsic::nvvm_isspacep_const:
+      return AS == NVPTXAS::ADDRESS_SPACE_CONST;
+    default:
+      llvm_unreachable("Unexpected intrinsic");
+    }
+  };
+
+  switch (auto IID = II.getIntrinsicID()) {
+  case Intrinsic::nvvm_isspacep_global:
+  case Intrinsic::nvvm_isspacep_local:
+  case Intrinsic::nvvm_isspacep_shared:
+  case Intrinsic::nvvm_isspacep_shared_cluster:
+  case Intrinsic::nvvm_isspacep_const: {
+    auto *Ty = II.getType();
+    unsigned AS = Op0->getType()->getPointerAddressSpace();
+    // Peek through ASC to generic AS.
+    // TODO: we could dig deeper through both ASCs and GEPs.
+    if (AS == NVPTXAS::ADDRESS_SPACE_GENERIC)
+      if (auto *ASCO = dyn_cast<AddrSpaceCastOperator>(Op0))
+        AS = ASCO->getOperand(0)->getType()->getPointerAddressSpace();
+
+    if (std::optional<bool> Answer = CheckASMatch(IID, AS))
+      return IC.replaceInstUsesWith(II, ConstantInt::get(Ty, *Answer));
+    return nullptr; // Don't know the answer, got to check at run time.
+  }
+  default:
+    return std::nullopt;
+  }
+}
+
 std::optional<Instruction *>
 NVPTXTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
-  if (Instruction *I = simplifyNvvmIntrinsic(&II, IC)) {
+  if (std::optional<Instruction *> I = handleSpaceCheckIntrinsics(IC, II))
+    return *I;
+  if (Instruction *I = convertNvvmIntrinsicToLlvm(IC, &II))
     return I;
-  }
+
   return std::nullopt;
 }
 
diff --git a/llvm/test/Transforms/InstCombine/NVPTX/isspacep.ll b/llvm/test/Transforms/InstCombine/NVPTX/isspacep.ll
new file mode 100644
index 000000000000..dedd85e1a8cd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/NVPTX/isspacep.ll
@@ -0,0 +1,277 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instcombine -mtriple=nvptx64-nvidia-cuda -S | FileCheck %s
+target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+; Source data in different AS.
+@shared_data = dso_local addrspace(3) global i32 undef, align 4
+@global_data = dso_local addrspace(1) externally_initialized global i32 0, align 4
+@const_data = dso_local addrspace(4) externally_initialized constant i32 3, align 4
+
+; Results get stored here.
+@gen = dso_local addrspace(1) externally_initialized global i8 0, align 1
+@g1 = dso_local addrspace(1) externally_initialized global i8 0, align 1
+@g2 = dso_local addrspace(1) externally_initialized global i8 0, align 1
+@s1 = dso_local addrspace(1) externally_initialized global i8 0, align 1
+@s2 = dso_local addrspace(1) externally_initialized global i8 0, align 1
+@c1 = dso_local addrspace(1) externally_initialized global i8 0, align 1
+@c2 = dso_local addrspace(1) externally_initialized global i8 0, align 1
+@l = dso_local addrspace(1) externally_initialized global i8 0, align 1
+
+declare i1 @llvm.nvvm.isspacep.global(ptr nocapture)
+declare i1 @llvm.nvvm.isspacep.shared(ptr nocapture)
+declare i1 @llvm.nvvm.isspacep.const(ptr nocapture)
+declare i1 @llvm.nvvm.isspacep.local(ptr nocapture)
+
+define dso_local void @check_global(ptr nocapture noundef readnone %out, ptr nocapture noundef readnone %genp,
+; CHECK-LABEL: define dso_local void @check_global(
+; CHECK-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr nocapture noundef readnone [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[GEN0:%.*]] = tail call i1 @llvm.nvvm.isspacep.global(ptr [[GENP]])
+; CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[GEN0]] to i8
+; CHECK-NEXT:    store i8 [[STOREDV]], ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
+; CHECK-NEXT:    store i8 1, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
+; CHECK-NEXT:    store i8 1, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
+; CHECK-NEXT:    ret void
+;
+  ptr addrspace(1) %gp,
+  ptr addrspace(3) %sp,
+  ptr addrspace(4) %cp,
+  ptr addrspace(5) %lp) local_unnamed_addr {
+entry:
+  ; No constant folding for generic pointers of unknown origin.
+  %gen0 = tail call i1 @llvm.nvvm.isspacep.global(ptr %genp)
+  %storedv = zext i1 %gen0 to i8
+  store i8 %storedv, ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
+
+  %isg1 = tail call i1 @llvm.nvvm.isspacep.global(ptr addrspacecast (ptr addrspace(1) @global_data to ptr))
+  %isg18 = zext i1 %isg1 to i8
+  store i8 %isg18, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
+
+  %gp_asc = addrspacecast ptr addrspace(1) %gp to ptr
+  %isg2 = tail call i1 @llvm.nvvm.isspacep.global(ptr %gp_asc)
+  %isg28 = zext i1 %isg2 to i8
+  store i8 %isg28, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
+
+  %iss1 = tail call i1 @llvm.nvvm.isspacep.global(ptr addrspacecast (ptr addrspace(3) @shared_data to ptr))
+  %iss18 = zext i1 %iss1 to i8
+  store i8 %iss18, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
+
+  %sp_asc = addrspacecast ptr addrspace(3) %sp to ptr
+  %iss2 = tail call i1 @llvm.nvvm.isspacep.global(ptr %sp_asc)
+  %iss28 = zext i1 %iss2 to i8
+  store i8 %iss28, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
+
+  %isc1 = tail call i1 @llvm.nvvm.isspacep.global(ptr addrspacecast (ptr addrspace(4) @const_data to ptr))
+  %isc18 = zext i1 %isc1 to i8
+  store i8 %isc18, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
+
+  %cp_asc = addrspacecast ptr addrspace(4) %cp to ptr
+  %isc2 = tail call i1 @llvm.nvvm.isspacep.global(ptr %cp_asc)
+  %isc28 = zext i1 %isc2 to i8
+  store i8 %isc28, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
+
+  ; Local data can't ihave a constant address, so we can't have a constant ASC expression
+  ; We can only use an ASC instruction.
+  %lp_asc = addrspacecast ptr addrspace(5) %lp to ptr
+  %isl = call i1 @llvm.nvvm.isspacep.global(ptr nonnull %lp_asc)
+  %isl8 = zext i1 %isl to i8
+  store i8 %isl8, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
+
+  ret void
+}
+
+define dso_local void @check_shared(ptr nocapture noundef readnone %out, ptr nocapture noundef readnone %genp,
+; CHECK-LABEL: define dso_local void @check_shared(
+; CHECK-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr nocapture noundef readnone [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[GEN0:%.*]] = tail call i1 @llvm.nvvm.isspacep.shared(ptr [[GENP]])
+; CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[GEN0]] to i8
+; CHECK-NEXT:    store i8 [[STOREDV]], ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
+; CHECK-NEXT:    store i8 1, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
+; CHECK-NEXT:    store i8 1, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
+; CHECK-NEXT:    ret void
+;
+  ptr addrspace(1) %gp,
+  ptr addrspace(3) %sp,
+  ptr addrspace(4) %cp,
+  ptr addrspace(5) %lp) local_unnamed_addr {
+entry:
+  ; No constant folding for generic pointers of unknown origin.
+  %gen0 = tail call i1 @llvm.nvvm.isspacep.shared(ptr %genp)
+  %storedv = zext i1 %gen0 to i8
+  store i8 %storedv, ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
+
+  %isg1 = tail call i1 @llvm.nvvm.isspacep.shared(ptr addrspacecast (ptr addrspace(1) @global_data to ptr))
+  %isg18 = zext i1 %isg1 to i8
+  store i8 %isg18, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
+
+  %gp_asc = addrspacecast ptr addrspace(1) %gp to ptr
+  %isg2 = tail call i1 @llvm.nvvm.isspacep.shared(ptr %gp_asc)
+  %isg28 = zext i1 %isg2 to i8
+  store i8 %isg28, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
+
+  %iss1 = tail call i1 @llvm.nvvm.isspacep.shared(ptr addrspacecast (ptr addrspace(3) @shared_data to ptr))
+  %iss18 = zext i1 %iss1 to i8
+  store i8 %iss18, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
+
+  %sp_asc = addrspacecast ptr addrspace(3) %sp to ptr
+  %iss2 = tail call i1 @llvm.nvvm.isspacep.shared(ptr %sp_asc)
+  %iss28 = zext i1 %iss2 to i8
+  store i8 %iss28, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
+
+  %isc1 = tail call i1 @llvm.nvvm.isspacep.shared(ptr addrspacecast (ptr addrspace(4) @const_data to ptr))
+  %isc18 = zext i1 %isc1 to i8
+  store i8 %isc18, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
+
+  %cp_asc = addrspacecast ptr addrspace(4) %cp to ptr
+  %isc2 = tail call i1 @llvm.nvvm.isspacep.shared(ptr %cp_asc)
+  %isc28 = zext i1 %isc2 to i8
+  store i8 %isc28, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
+
+  ; Local data can't have a constant address, so we can't have a constant ASC expression
+  ; We can only use an ASC instruction.
+  %lp_asc = addrspacecast ptr addrspace(5) %lp to ptr
+  %isl = call i1 @llvm.nvvm.isspacep.shared(ptr nonnull %lp_asc)
+  %isl8 = zext i1 %isl to i8
+  store i8 %isl8, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
+
+  ret void
+}
+
+define dso_local void @check_const(ptr nocapture noundef readnone %out, ptr nocapture noundef readnone %genp,
+; CHECK-LABEL: define dso_local void @check_const(
+; CHECK-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr nocapture noundef readnone [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[GEN0:%.*]] = tail call i1 @llvm.nvvm.isspacep.const(ptr [[GENP]])
+; CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[GEN0]] to i8
+; CHECK-NEXT:    store i8 [[STOREDV]], ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
+; CHECK-NEXT:    store i8 1, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
+; CHECK-NEXT:    store i8 1, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
+; CHECK-NEXT:    ret void
+;
+  ptr addrspace(1) %gp,
+  ptr addrspace(3) %sp,
+  ptr addrspace(4) %cp,
+  ptr addrspace(5) %lp) local_unnamed_addr {
+entry:
+  ; No constant folding for generic pointers of unknown origin.
+  %gen0 = tail call i1 @llvm.nvvm.isspacep.const(ptr %genp)
+  %storedv = zext i1 %gen0 to i8
+  store i8 %storedv, ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
+
+  %isg1 = tail call i1 @llvm.nvvm.isspacep.const(ptr addrspacecast (ptr addrspace(1) @global_data to ptr))
+  %isg18 = zext i1 %isg1 to i8
+  store i8 %isg18, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
+
+  %gp_asc = addrspacecast ptr addrspace(1) %gp to ptr
+  %isg2 = tail call i1 @llvm.nvvm.isspacep.const(ptr %gp_asc)
+  %isg28 = zext i1 %isg2 to i8
+  store i8 %isg28, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
+
+  %iss1 = tail call i1 @llvm.nvvm.isspacep.const(ptr addrspacecast (ptr addrspace(3) @shared_data to ptr))
+  %iss18 = zext i1 %iss1 to i8
+  store i8 %iss18, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
+
+  %sp_asc = addrspacecast ptr addrspace(3) %sp to ptr
+  %iss2 = tail call i1 @llvm.nvvm.isspacep.const(ptr %sp_asc)
+  %iss28 = zext i1 %iss2 to i8
+  store i8 %iss28, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
+
+  %isc1 = tail call i1 @llvm.nvvm.isspacep.const(ptr addrspacecast (ptr addrspace(4) @const_data to ptr))
+  %isc18 = zext i1 %isc1 to i8
+  store i8 %isc18, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
+
+  %cp_asc = addrspacecast ptr addrspace(4) %cp to ptr
+  %isc2 = tail call i1 @llvm.nvvm.isspacep.const(ptr %cp_asc)
+  %isc28 = zext i1 %isc2 to i8
+  store i8 %isc28, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
+
+  ; Local data can't have a constant address, so we can't have a constant ASC expression
+  ; We can only use an ASC instruction.
+  %lp_asc = addrspacecast ptr addrspace(5) %lp to ptr
+  %isl = call i1 @llvm.nvvm.isspacep.const(ptr nonnull %lp_asc)
+  %isl8 = zext i1 %isl to i8
+  store i8 %isl8, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
+
+  ret void
+}
+
+define dso_local void @check_local(ptr nocapture noundef readnone %out, ptr nocapture noundef readnone %genp,
+; CHECK-LABEL: define dso_local void @check_local(
+; CHECK-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr nocapture noundef readnone [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[GEN0:%.*]] = tail call i1 @llvm.nvvm.isspacep.local(ptr [[GENP]])
+; CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[GEN0]] to i8
+; CHECK-NEXT:    store i8 [[STOREDV]], ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
+; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
+; CHECK-NEXT:    store i8 1, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
+; CHECK-NEXT:    ret void
+;
+  ptr addrspace(1) %gp,
+  ptr addrspace(3) %sp,
+  ptr addrspace(4) %cp,
+  ptr addrspace(5) %lp) local_unnamed_addr {
+entry:
+  ; No constant folding for generic pointers of unknown origin.
+  %gen0 = tail call i1 @llvm.nvvm.isspacep.local(ptr %genp)
+  %storedv = zext i1 %gen0 to i8
+  store i8 %storedv, ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
+
+  %isg1 = tail call i1 @llvm.nvvm.isspacep.local(ptr addrspacecast (ptr addrspace(1) @global_data to ptr))
+  %isg18 = zext i1 %isg1 to i8
+  store i8 %isg18, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
+
+  %gp_asc = addrspacecast ptr addrspace(1) %gp to ptr
+  %isg2 = tail call i1 @llvm.nvvm.isspacep.local(ptr %gp_asc)
+  %isg28 = zext i1 %isg2 to i8
+  store i8 %isg28, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
+
+  %iss1 = tail call i1 @llvm.nvvm.isspacep.local(ptr addrspacecast (ptr addrspace(3) @shared_data to ptr))
+  %iss18 = zext i1 %iss1 to i8
+  store i8 %iss18, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
+
+  %sp_asc = addrspacecast ptr addrspace(3) %sp to ptr
+  %iss2 = tail call i1 @llvm.nvvm.isspacep.local(ptr %sp_asc)
+  %iss28 = zext i1 %iss2 to i8
+  store i8 %iss28, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
+
+  %isc1 = tail call i1 @llvm.nvvm.isspacep.local(ptr addrspacecast (ptr addrspace(4) @const_data to ptr))
+  %isc18 = zext i1 %isc1 to i8
+  store i8 %isc18, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
+
+  %cp_asc = addrspacecast ptr addrspace(4) %cp to ptr
+  %isc2 = tail call i1 @llvm.nvvm.isspacep.local(ptr %cp_asc)
+  %isc28 = zext i1 %isc2 to i8
+  store i8 %isc28, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
+
+  ; Local data can't have a constant address, so we can't have a constant ASC expression
+  ; We can only use an ASC instruction.
+  %lp_asc = addrspacecast ptr addrspace(5) %lp to ptr
+  %isl = call i1 @llvm.nvvm.isspacep.local(ptr nonnull %lp_asc)
+  %isl8 = zext i1 %isl to i8
+  store i8 %isl8, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
+
+  ret void
+}
+
-- 
GitLab


From d043670d66ce7958aec7837ee572f3dc8948f11a Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Thu, 31 Oct 2024 07:26:12 +0900
Subject: [PATCH 654/693] [mlir][func] Replace `ValueDecomposer` with target
 materialization (#114192)

The `ValueDecomposer` in `DecomposeCallGraphTypes` was a workaround
around missing 1:N support in the dialect conversion. Since #113032, the
dialect conversion infrastructure supports 1:N type conversions and 1:N
target materializations. The `ValueDecomposer` class is no longer
needed. (However, target materializations must still be inserted
manually, until we fully merge the 1:1 and 1:N drivers.)

Note for LLVM integration: Register 1:N target materializations on the
type converter instead of "decompose value conversions" on the
`ValueDecomposer`.
---
 .../Func/Transforms/DecomposeCallGraphTypes.h |  62 +---------
 .../Transforms/DecomposeCallGraphTypes.cpp    | 111 +++++++++---------
 .../Func/TestDecomposeCallGraphTypes.cpp      |  60 ++++++----
 3 files changed, 93 insertions(+), 140 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Func/Transforms/DecomposeCallGraphTypes.h b/mlir/include/mlir/Dialect/Func/Transforms/DecomposeCallGraphTypes.h
index 1d311b37b37a..1be406bf3adf 100644
--- a/mlir/include/mlir/Dialect/Func/Transforms/DecomposeCallGraphTypes.h
+++ b/mlir/include/mlir/Dialect/Func/Transforms/DecomposeCallGraphTypes.h
@@ -23,70 +23,10 @@
 
 namespace mlir {
 
-/// This class provides a hook that expands one Value into multiple Value's,
-/// with a TypeConverter-inspired callback registration mechanism.
-///
-/// For folks that are familiar with the dialect conversion framework /
-/// TypeConverter, this is effectively the inverse of a source/argument
-/// materialization. A target materialization is not what we want here because
-/// it always produces a single Value, but in this case the whole point is to
-/// decompose a Value into multiple Value's.
-///
-/// The reason we need this inverse is easily understood by looking at what we
-/// need to do for decomposing types for a return op. When converting a return
-/// op, the dialect conversion framework will give the list of converted
-/// operands, and will ensure that each converted operand, even if it expanded
-/// into multiple types, is materialized as a single result. We then need to
-/// undo that materialization to a single result, which we do with the
-/// decomposeValue hooks registered on this object.
-///
-/// TODO: Eventually, the type conversion infra should have this hook built-in.
-/// See
-/// https://llvm.discourse.group/t/extending-type-conversion-infrastructure/779/2
-class ValueDecomposer {
-public:
-  /// This method tries to decompose a value of a certain type using provided
-  /// decompose callback functions. If it is unable to do so, the original value
-  /// is returned.
-  void decomposeValue(OpBuilder &, Location, Type, Value,
-                      SmallVectorImpl<Value> &);
-
-  /// This method registers a callback function that will be called to decompose
-  /// a value of a certain type into 0, 1, or multiple values.
-  template <typename FnT, typename T = typename llvm::function_traits<
-                              std::decay_t<FnT>>::template arg_t<2>>
-  void addDecomposeValueConversion(FnT &&callback) {
-    decomposeValueConversions.emplace_back(
-        wrapDecomposeValueConversionCallback<T>(std::forward<FnT>(callback)));
-  }
-
-private:
-  using DecomposeValueConversionCallFn =
-      std::function<std::optional<LogicalResult>(
-          OpBuilder &, Location, Type, Value, SmallVectorImpl<Value> &)>;
-
-  /// Generate a wrapper for the given decompose value conversion callback.
-  template <typename T, typename FnT>
-  DecomposeValueConversionCallFn
-  wrapDecomposeValueConversionCallback(FnT &&callback) {
-    return
-        [callback = std::forward<FnT>(callback)](
-            OpBuilder &builder, Location loc, Type type, Value value,
-            SmallVectorImpl<Value> &newValues) -> std::optional<LogicalResult> {
-          if (T derivedType = dyn_cast<T>(type))
-            return callback(builder, loc, derivedType, value, newValues);
-          return std::nullopt;
-        };
-  }
-
-  SmallVector<DecomposeValueConversionCallFn, 2> decomposeValueConversions;
-};
-
 /// Populates the patterns needed to drive the conversion process for
-/// decomposing call graph types with the given `ValueDecomposer`.
+/// decomposing call graph types with the given `TypeConverter`.
 void populateDecomposeCallGraphTypesPatterns(MLIRContext *context,
                                              const TypeConverter &typeConverter,
-                                             ValueDecomposer &decomposer,
                                              RewritePatternSet &patterns);
 
 } // namespace mlir
diff --git a/mlir/lib/Dialect/Func/Transforms/DecomposeCallGraphTypes.cpp b/mlir/lib/Dialect/Func/Transforms/DecomposeCallGraphTypes.cpp
index 357f993710a2..de4aba2ed327 100644
--- a/mlir/lib/Dialect/Func/Transforms/DecomposeCallGraphTypes.cpp
+++ b/mlir/lib/Dialect/Func/Transforms/DecomposeCallGraphTypes.cpp
@@ -14,52 +14,48 @@ using namespace mlir;
 using namespace mlir::func;
 
 //===----------------------------------------------------------------------===//
-// ValueDecomposer
+// Helper functions
 //===----------------------------------------------------------------------===//
 
-void ValueDecomposer::decomposeValue(OpBuilder &builder, Location loc,
-                                     Type type, Value value,
-                                     SmallVectorImpl<Value> &results) {
-  for (auto &conversion : decomposeValueConversions)
-    if (conversion(builder, loc, type, value, results))
-      return;
-  results.push_back(value);
+/// If the given value can be decomposed with the type converter, decompose it.
+/// Otherwise, return the given value.
+// TODO: Value decomposition should happen automatically through a 1:N adaptor.
+// This function will disappear when the 1:1 and 1:N drivers are merged.
+static SmallVector<Value> decomposeValue(OpBuilder &builder, Location loc,
+                                         Value value,
+                                         const TypeConverter *converter) {
+  // Try to convert the given value's type. If that fails, just return the
+  // given value.
+  SmallVector<Type> convertedTypes;
+  if (failed(converter->convertType(value.getType(), convertedTypes)))
+    return {value};
+  if (convertedTypes.empty())
+    return {};
+
+  // If the given value's type is already legal, just return the given value.
+  TypeRange convertedTypeRange(convertedTypes);
+  if (convertedTypeRange == TypeRange(value.getType()))
+    return {value};
+
+  // Try to materialize a target conversion. If the materialization did not
+  // produce values of the requested type, the materialization failed. Just
+  // return the given value in that case.
+  SmallVector<Value> result = converter->materializeTargetConversion(
+      builder, loc, convertedTypeRange, value);
+  if (result.empty())
+    return {value};
+  return result;
 }
 
-//===----------------------------------------------------------------------===//
-// DecomposeCallGraphTypesOpConversionPattern
-//===----------------------------------------------------------------------===//
-
-namespace {
-/// Base OpConversionPattern class to make a ValueDecomposer available to
-/// inherited patterns.
-template <typename SourceOp>
-class DecomposeCallGraphTypesOpConversionPattern
-    : public OpConversionPattern<SourceOp> {
-public:
-  DecomposeCallGraphTypesOpConversionPattern(const TypeConverter &typeConverter,
-                                             MLIRContext *context,
-                                             ValueDecomposer &decomposer,
-                                             PatternBenefit benefit = 1)
-      : OpConversionPattern<SourceOp>(typeConverter, context, benefit),
-        decomposer(decomposer) {}
-
-protected:
-  ValueDecomposer &decomposer;
-};
-} // namespace
-
 //===----------------------------------------------------------------------===//
 // DecomposeCallGraphTypesForFuncArgs
 //===----------------------------------------------------------------------===//
 
 namespace {
-/// Expand function arguments according to the provided TypeConverter and
-/// ValueDecomposer.
+/// Expand function arguments according to the provided TypeConverter.
 struct DecomposeCallGraphTypesForFuncArgs
-    : public DecomposeCallGraphTypesOpConversionPattern<func::FuncOp> {
-  using DecomposeCallGraphTypesOpConversionPattern::
-      DecomposeCallGraphTypesOpConversionPattern;
+    : public OpConversionPattern<func::FuncOp> {
+  using OpConversionPattern::OpConversionPattern;
 
   LogicalResult
   matchAndRewrite(func::FuncOp op, OpAdaptor adaptor,
@@ -100,19 +96,22 @@ struct DecomposeCallGraphTypesForFuncArgs
 //===----------------------------------------------------------------------===//
 
 namespace {
-/// Expand return operands according to the provided TypeConverter and
-/// ValueDecomposer.
+/// Expand return operands according to the provided TypeConverter.
 struct DecomposeCallGraphTypesForReturnOp
-    : public DecomposeCallGraphTypesOpConversionPattern<ReturnOp> {
-  using DecomposeCallGraphTypesOpConversionPattern::
-      DecomposeCallGraphTypesOpConversionPattern;
+    : public OpConversionPattern<ReturnOp> {
+  using OpConversionPattern::OpConversionPattern;
+
   LogicalResult
   matchAndRewrite(ReturnOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const final {
     SmallVector<Value, 2> newOperands;
-    for (Value operand : adaptor.getOperands())
-      decomposer.decomposeValue(rewriter, op.getLoc(), operand.getType(),
-                                operand, newOperands);
+    for (Value operand : adaptor.getOperands()) {
+      // TODO: We can directly take the values from the adaptor once this is a
+      // 1:N conversion pattern.
+      llvm::append_range(newOperands,
+                         decomposeValue(rewriter, operand.getLoc(), operand,
+                                        getTypeConverter()));
+    }
     rewriter.replaceOpWithNewOp<ReturnOp>(op, newOperands);
     return success();
   }
@@ -124,12 +123,9 @@ struct DecomposeCallGraphTypesForReturnOp
 //===----------------------------------------------------------------------===//
 
 namespace {
-/// Expand call op operands and results according to the provided TypeConverter
-/// and ValueDecomposer.
-struct DecomposeCallGraphTypesForCallOp
-    : public DecomposeCallGraphTypesOpConversionPattern<CallOp> {
-  using DecomposeCallGraphTypesOpConversionPattern::
-      DecomposeCallGraphTypesOpConversionPattern;
+/// Expand call op operands and results according to the provided TypeConverter.
+struct DecomposeCallGraphTypesForCallOp : public OpConversionPattern<CallOp> {
+  using OpConversionPattern::OpConversionPattern;
 
   LogicalResult
   matchAndRewrite(CallOp op, OpAdaptor adaptor,
@@ -137,9 +133,13 @@ struct DecomposeCallGraphTypesForCallOp
 
     // Create the operands list of the new `CallOp`.
     SmallVector<Value, 2> newOperands;
-    for (Value operand : adaptor.getOperands())
-      decomposer.decomposeValue(rewriter, op.getLoc(), operand.getType(),
-                                operand, newOperands);
+    for (Value operand : adaptor.getOperands()) {
+      // TODO: We can directly take the values from the adaptor once this is a
+      // 1:N conversion pattern.
+      llvm::append_range(newOperands,
+                         decomposeValue(rewriter, operand.getLoc(), operand,
+                                        getTypeConverter()));
+    }
 
     // Create the new result types for the new `CallOp` and track the indices in
     // the new call op's results that correspond to the old call op's results.
@@ -189,9 +189,8 @@ struct DecomposeCallGraphTypesForCallOp
 
 void mlir::populateDecomposeCallGraphTypesPatterns(
     MLIRContext *context, const TypeConverter &typeConverter,
-    ValueDecomposer &decomposer, RewritePatternSet &patterns) {
+    RewritePatternSet &patterns) {
   patterns
       .add<DecomposeCallGraphTypesForCallOp, DecomposeCallGraphTypesForFuncArgs,
-           DecomposeCallGraphTypesForReturnOp>(typeConverter, context,
-                                               decomposer);
+           DecomposeCallGraphTypesForReturnOp>(typeConverter, context);
 }
diff --git a/mlir/test/lib/Dialect/Func/TestDecomposeCallGraphTypes.cpp b/mlir/test/lib/Dialect/Func/TestDecomposeCallGraphTypes.cpp
index 92216da9f201..de511c58ae6e 100644
--- a/mlir/test/lib/Dialect/Func/TestDecomposeCallGraphTypes.cpp
+++ b/mlir/test/lib/Dialect/Func/TestDecomposeCallGraphTypes.cpp
@@ -21,23 +21,40 @@ namespace {
 /// given tuple value. If some tuple elements are, in turn, tuples, the elements
 /// of those are extracted recursively such that the returned values have the
 /// same types as `resultTypes.getFlattenedTypes()`.
-static LogicalResult buildDecomposeTuple(OpBuilder &builder, Location loc,
-                                         TupleType resultType, Value value,
-                                         SmallVectorImpl<Value> &values) {
-  for (unsigned i = 0, e = resultType.size(); i < e; ++i) {
-    Type elementType = resultType.getType(i);
-    Value element = builder.create<test::GetTupleElementOp>(
-        loc, elementType, value, builder.getI32IntegerAttr(i));
-    if (auto nestedTupleType = dyn_cast<TupleType>(elementType)) {
-      // Recurse if the current element is also a tuple.
-      if (failed(buildDecomposeTuple(builder, loc, nestedTupleType, element,
-                                     values)))
-        return failure();
-    } else {
-      values.push_back(element);
+static SmallVector<Value> buildDecomposeTuple(OpBuilder &builder,
+                                              TypeRange resultTypes,
+                                              ValueRange inputs, Location loc) {
+  // Skip materialization if the single input value is not a tuple.
+  if (inputs.size() != 1)
+    return {};
+  Value tuple = inputs.front();
+  auto tupleType = dyn_cast<TupleType>(tuple.getType());
+  if (!tupleType)
+    return {};
+  // Skip materialization if the flattened types do not match the requested
+  // result types.
+  SmallVector<Type> flattenedTypes;
+  tupleType.getFlattenedTypes(flattenedTypes);
+  if (TypeRange(resultTypes) != TypeRange(flattenedTypes))
+    return {};
+  // Recursively decompose the tuple.
+  SmallVector<Value> result;
+  std::function<void(Value)> decompose = [&](Value tuple) {
+    auto tupleType = dyn_cast<TupleType>(tuple.getType());
+    if (!tupleType) {
+      // This is not a tuple.
+      result.push_back(tuple);
+      return;
     }
-  }
-  return success();
+    for (unsigned i = 0, e = tupleType.size(); i < e; ++i) {
+      Type elementType = tupleType.getType(i);
+      Value element = builder.create<test::GetTupleElementOp>(
+          loc, elementType, tuple, builder.getI32IntegerAttr(i));
+      decompose(element);
+    }
+  };
+  decompose(tuple);
+  return result;
 }
 
 /// Creates a `test.make_tuple` op out of the given inputs building a tuple of
@@ -82,8 +99,8 @@ static Value buildMakeTupleOp(OpBuilder &builder, TupleType resultType,
 
 /// A pass for testing call graph type decomposition.
 ///
-/// This instantiates the patterns with a TypeConverter and ValueDecomposer
-/// that splits tuple types into their respective element types.
+/// This instantiates the patterns with a TypeConverter that splits tuple types
+/// into their respective element types.
 /// For example, `tuple<T1, T2, T3> --> T1, T2, T3`.
 struct TestDecomposeCallGraphTypes
     : public PassWrapper<TestDecomposeCallGraphTypes, OperationPass<ModuleOp>> {
@@ -123,12 +140,9 @@ struct TestDecomposeCallGraphTypes
           return success();
         });
     typeConverter.addArgumentMaterialization(buildMakeTupleOp);
+    typeConverter.addTargetMaterialization(buildDecomposeTuple);
 
-    ValueDecomposer decomposer;
-    decomposer.addDecomposeValueConversion(buildDecomposeTuple);
-
-    populateDecomposeCallGraphTypesPatterns(context, typeConverter, decomposer,
-                                            patterns);
+    populateDecomposeCallGraphTypesPatterns(context, typeConverter, patterns);
 
     if (failed(applyPartialConversion(module, target, std::move(patterns))))
       return signalPassFailure();
-- 
GitLab


From 04e876e6c6eee5332f5fff30c8778abe82ebf52f Mon Sep 17 00:00:00 2001
From: Artem Belevich <tra@google.com>
Date: Wed, 30 Oct 2024 15:34:08 -0700
Subject: [PATCH 655/693] Revert "[NVPTX] instcombine known pointer AS checks."
 (#114319)

Reverts llvm/llvm-project#112964

Crashes MLIR: https://lab.llvm.org/buildbot/#/builders/138/builds/5665
---
 llvm/include/llvm/Support/NVPTXAddrSpace.h    |  33 ---
 .../Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h |  12 +-
 .../Target/NVPTX/NVPTXTargetTransformInfo.cpp |  63 +---
 .../Transforms/InstCombine/NVPTX/isspacep.ll  | 277 ------------------
 4 files changed, 13 insertions(+), 372 deletions(-)
 delete mode 100644 llvm/include/llvm/Support/NVPTXAddrSpace.h
 delete mode 100644 llvm/test/Transforms/InstCombine/NVPTX/isspacep.ll

diff --git a/llvm/include/llvm/Support/NVPTXAddrSpace.h b/llvm/include/llvm/Support/NVPTXAddrSpace.h
deleted file mode 100644
index 93eae39e3d23..000000000000
--- a/llvm/include/llvm/Support/NVPTXAddrSpace.h
+++ /dev/null
@@ -1,33 +0,0 @@
-//===---------------- NVPTXAddrSpace.h -------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-/// \file
-/// NVPTX address space definition
-///
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_SUPPORT_NVPTXADDRSPACE_H
-#define LLVM_SUPPORT_NVPTXADDRSPACE_H
-
-namespace llvm {
-namespace NVPTXAS {
-enum AddressSpace : unsigned {
-  ADDRESS_SPACE_GENERIC = 0,
-  ADDRESS_SPACE_GLOBAL = 1,
-  ADDRESS_SPACE_SHARED = 3,
-  ADDRESS_SPACE_CONST = 4,
-  ADDRESS_SPACE_LOCAL = 5,
-
-  ADDRESS_SPACE_PARAM = 101,
-};
-} // end namespace NVPTXAS
-
-} // end namespace llvm
-
-#endif // LLVM_SUPPORT_NVPTXADDRSPACE_H
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h
index d06e2c00ec3f..815b600fe93a 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h
@@ -16,10 +16,18 @@
 #ifndef LLVM_LIB_TARGET_NVPTX_MCTARGETDESC_NVPTXBASEINFO_H
 #define LLVM_LIB_TARGET_NVPTX_MCTARGETDESC_NVPTXBASEINFO_H
 
-#include "llvm/Support/NVPTXAddrSpace.h"
 namespace llvm {
 
-using namespace NVPTXAS;
+enum AddressSpace {
+  ADDRESS_SPACE_GENERIC = 0,
+  ADDRESS_SPACE_GLOBAL = 1,
+  ADDRESS_SPACE_SHARED = 3,
+  ADDRESS_SPACE_CONST = 4,
+  ADDRESS_SPACE_LOCAL = 5,
+
+  // NVVM Internal
+  ADDRESS_SPACE_PARAM = 101
+};
 
 namespace NVPTXII {
 enum {
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index 31087a0054e9..e35ba25b4788 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -15,12 +15,10 @@
 #include "llvm/CodeGen/CostTable.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/IR/Constants.h"
-#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsNVPTX.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include <optional>
 using namespace llvm;
@@ -119,8 +117,7 @@ bool NVPTXTTIImpl::isSourceOfDivergence(const Value *V) {
 }
 
 // Convert NVVM intrinsics to target-generic LLVM code where possible.
-static Instruction *convertNvvmIntrinsicToLlvm(InstCombiner &IC,
-                                               IntrinsicInst *II) {
+static Instruction *simplifyNvvmIntrinsic(IntrinsicInst *II, InstCombiner &IC) {
   // Each NVVM intrinsic we can simplify can be replaced with one of:
   //
   //  * an LLVM intrinsic,
@@ -416,65 +413,11 @@ static Instruction *convertNvvmIntrinsicToLlvm(InstCombiner &IC,
   llvm_unreachable("All SpecialCase enumerators should be handled in switch.");
 }
 
-// Returns an instruction pointer (may be nullptr if we do not know the answer).
-// Returns nullopt if `II` is not one of the `isspacep` intrinsics.
-static std::optional<Instruction *>
-handleSpaceCheckIntrinsics(InstCombiner &IC, IntrinsicInst &II) {
-  Value *Op0 = II.getArgOperand(0);
-  // Returns true/false when we know the answer, nullopt otherwise.
-  auto CheckASMatch = [](unsigned IID, unsigned AS) -> std::optional<bool> {
-    if (AS == NVPTXAS::ADDRESS_SPACE_GENERIC ||
-        AS == NVPTXAS::ADDRESS_SPACE_PARAM)
-      return std::nullopt; // Got to check at run-time.
-    switch (IID) {
-    case Intrinsic::nvvm_isspacep_global:
-      return AS == NVPTXAS::ADDRESS_SPACE_GLOBAL;
-    case Intrinsic::nvvm_isspacep_local:
-      return AS == NVPTXAS::ADDRESS_SPACE_LOCAL;
-    case Intrinsic::nvvm_isspacep_shared:
-      return AS == NVPTXAS::ADDRESS_SPACE_SHARED;
-    case Intrinsic::nvvm_isspacep_shared_cluster:
-      // We can't tell shared from shared_cluster at compile time from AS alone,
-      // but it can't be either is AS is not shared.
-      return AS == NVPTXAS::ADDRESS_SPACE_SHARED ? std::nullopt
-                                                 : std::optional{false};
-    case Intrinsic::nvvm_isspacep_const:
-      return AS == NVPTXAS::ADDRESS_SPACE_CONST;
-    default:
-      llvm_unreachable("Unexpected intrinsic");
-    }
-  };
-
-  switch (auto IID = II.getIntrinsicID()) {
-  case Intrinsic::nvvm_isspacep_global:
-  case Intrinsic::nvvm_isspacep_local:
-  case Intrinsic::nvvm_isspacep_shared:
-  case Intrinsic::nvvm_isspacep_shared_cluster:
-  case Intrinsic::nvvm_isspacep_const: {
-    auto *Ty = II.getType();
-    unsigned AS = Op0->getType()->getPointerAddressSpace();
-    // Peek through ASC to generic AS.
-    // TODO: we could dig deeper through both ASCs and GEPs.
-    if (AS == NVPTXAS::ADDRESS_SPACE_GENERIC)
-      if (auto *ASCO = dyn_cast<AddrSpaceCastOperator>(Op0))
-        AS = ASCO->getOperand(0)->getType()->getPointerAddressSpace();
-
-    if (std::optional<bool> Answer = CheckASMatch(IID, AS))
-      return IC.replaceInstUsesWith(II, ConstantInt::get(Ty, *Answer));
-    return nullptr; // Don't know the answer, got to check at run time.
-  }
-  default:
-    return std::nullopt;
-  }
-}
-
 std::optional<Instruction *>
 NVPTXTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
-  if (std::optional<Instruction *> I = handleSpaceCheckIntrinsics(IC, II))
-    return *I;
-  if (Instruction *I = convertNvvmIntrinsicToLlvm(IC, &II))
+  if (Instruction *I = simplifyNvvmIntrinsic(&II, IC)) {
     return I;
-
+  }
   return std::nullopt;
 }
 
diff --git a/llvm/test/Transforms/InstCombine/NVPTX/isspacep.ll b/llvm/test/Transforms/InstCombine/NVPTX/isspacep.ll
deleted file mode 100644
index dedd85e1a8cd..000000000000
--- a/llvm/test/Transforms/InstCombine/NVPTX/isspacep.ll
+++ /dev/null
@@ -1,277 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -passes=instcombine -mtriple=nvptx64-nvidia-cuda -S | FileCheck %s
-target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
-target triple = "nvptx64-nvidia-cuda"
-
-; Source data in different AS.
-@shared_data = dso_local addrspace(3) global i32 undef, align 4
-@global_data = dso_local addrspace(1) externally_initialized global i32 0, align 4
-@const_data = dso_local addrspace(4) externally_initialized constant i32 3, align 4
-
-; Results get stored here.
-@gen = dso_local addrspace(1) externally_initialized global i8 0, align 1
-@g1 = dso_local addrspace(1) externally_initialized global i8 0, align 1
-@g2 = dso_local addrspace(1) externally_initialized global i8 0, align 1
-@s1 = dso_local addrspace(1) externally_initialized global i8 0, align 1
-@s2 = dso_local addrspace(1) externally_initialized global i8 0, align 1
-@c1 = dso_local addrspace(1) externally_initialized global i8 0, align 1
-@c2 = dso_local addrspace(1) externally_initialized global i8 0, align 1
-@l = dso_local addrspace(1) externally_initialized global i8 0, align 1
-
-declare i1 @llvm.nvvm.isspacep.global(ptr nocapture)
-declare i1 @llvm.nvvm.isspacep.shared(ptr nocapture)
-declare i1 @llvm.nvvm.isspacep.const(ptr nocapture)
-declare i1 @llvm.nvvm.isspacep.local(ptr nocapture)
-
-define dso_local void @check_global(ptr nocapture noundef readnone %out, ptr nocapture noundef readnone %genp,
-; CHECK-LABEL: define dso_local void @check_global(
-; CHECK-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr nocapture noundef readnone [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[GEN0:%.*]] = tail call i1 @llvm.nvvm.isspacep.global(ptr [[GENP]])
-; CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[GEN0]] to i8
-; CHECK-NEXT:    store i8 [[STOREDV]], ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
-; CHECK-NEXT:    store i8 1, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
-; CHECK-NEXT:    store i8 1, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
-; CHECK-NEXT:    ret void
-;
-  ptr addrspace(1) %gp,
-  ptr addrspace(3) %sp,
-  ptr addrspace(4) %cp,
-  ptr addrspace(5) %lp) local_unnamed_addr {
-entry:
-  ; No constant folding for generic pointers of unknown origin.
-  %gen0 = tail call i1 @llvm.nvvm.isspacep.global(ptr %genp)
-  %storedv = zext i1 %gen0 to i8
-  store i8 %storedv, ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
-
-  %isg1 = tail call i1 @llvm.nvvm.isspacep.global(ptr addrspacecast (ptr addrspace(1) @global_data to ptr))
-  %isg18 = zext i1 %isg1 to i8
-  store i8 %isg18, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
-
-  %gp_asc = addrspacecast ptr addrspace(1) %gp to ptr
-  %isg2 = tail call i1 @llvm.nvvm.isspacep.global(ptr %gp_asc)
-  %isg28 = zext i1 %isg2 to i8
-  store i8 %isg28, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
-
-  %iss1 = tail call i1 @llvm.nvvm.isspacep.global(ptr addrspacecast (ptr addrspace(3) @shared_data to ptr))
-  %iss18 = zext i1 %iss1 to i8
-  store i8 %iss18, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
-
-  %sp_asc = addrspacecast ptr addrspace(3) %sp to ptr
-  %iss2 = tail call i1 @llvm.nvvm.isspacep.global(ptr %sp_asc)
-  %iss28 = zext i1 %iss2 to i8
-  store i8 %iss28, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
-
-  %isc1 = tail call i1 @llvm.nvvm.isspacep.global(ptr addrspacecast (ptr addrspace(4) @const_data to ptr))
-  %isc18 = zext i1 %isc1 to i8
-  store i8 %isc18, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
-
-  %cp_asc = addrspacecast ptr addrspace(4) %cp to ptr
-  %isc2 = tail call i1 @llvm.nvvm.isspacep.global(ptr %cp_asc)
-  %isc28 = zext i1 %isc2 to i8
-  store i8 %isc28, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
-
-  ; Local data can't ihave a constant address, so we can't have a constant ASC expression
-  ; We can only use an ASC instruction.
-  %lp_asc = addrspacecast ptr addrspace(5) %lp to ptr
-  %isl = call i1 @llvm.nvvm.isspacep.global(ptr nonnull %lp_asc)
-  %isl8 = zext i1 %isl to i8
-  store i8 %isl8, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
-
-  ret void
-}
-
-define dso_local void @check_shared(ptr nocapture noundef readnone %out, ptr nocapture noundef readnone %genp,
-; CHECK-LABEL: define dso_local void @check_shared(
-; CHECK-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr nocapture noundef readnone [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[GEN0:%.*]] = tail call i1 @llvm.nvvm.isspacep.shared(ptr [[GENP]])
-; CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[GEN0]] to i8
-; CHECK-NEXT:    store i8 [[STOREDV]], ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
-; CHECK-NEXT:    store i8 1, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
-; CHECK-NEXT:    store i8 1, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
-; CHECK-NEXT:    ret void
-;
-  ptr addrspace(1) %gp,
-  ptr addrspace(3) %sp,
-  ptr addrspace(4) %cp,
-  ptr addrspace(5) %lp) local_unnamed_addr {
-entry:
-  ; No constant folding for generic pointers of unknown origin.
-  %gen0 = tail call i1 @llvm.nvvm.isspacep.shared(ptr %genp)
-  %storedv = zext i1 %gen0 to i8
-  store i8 %storedv, ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
-
-  %isg1 = tail call i1 @llvm.nvvm.isspacep.shared(ptr addrspacecast (ptr addrspace(1) @global_data to ptr))
-  %isg18 = zext i1 %isg1 to i8
-  store i8 %isg18, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
-
-  %gp_asc = addrspacecast ptr addrspace(1) %gp to ptr
-  %isg2 = tail call i1 @llvm.nvvm.isspacep.shared(ptr %gp_asc)
-  %isg28 = zext i1 %isg2 to i8
-  store i8 %isg28, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
-
-  %iss1 = tail call i1 @llvm.nvvm.isspacep.shared(ptr addrspacecast (ptr addrspace(3) @shared_data to ptr))
-  %iss18 = zext i1 %iss1 to i8
-  store i8 %iss18, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
-
-  %sp_asc = addrspacecast ptr addrspace(3) %sp to ptr
-  %iss2 = tail call i1 @llvm.nvvm.isspacep.shared(ptr %sp_asc)
-  %iss28 = zext i1 %iss2 to i8
-  store i8 %iss28, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
-
-  %isc1 = tail call i1 @llvm.nvvm.isspacep.shared(ptr addrspacecast (ptr addrspace(4) @const_data to ptr))
-  %isc18 = zext i1 %isc1 to i8
-  store i8 %isc18, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
-
-  %cp_asc = addrspacecast ptr addrspace(4) %cp to ptr
-  %isc2 = tail call i1 @llvm.nvvm.isspacep.shared(ptr %cp_asc)
-  %isc28 = zext i1 %isc2 to i8
-  store i8 %isc28, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
-
-  ; Local data can't have a constant address, so we can't have a constant ASC expression
-  ; We can only use an ASC instruction.
-  %lp_asc = addrspacecast ptr addrspace(5) %lp to ptr
-  %isl = call i1 @llvm.nvvm.isspacep.shared(ptr nonnull %lp_asc)
-  %isl8 = zext i1 %isl to i8
-  store i8 %isl8, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
-
-  ret void
-}
-
-define dso_local void @check_const(ptr nocapture noundef readnone %out, ptr nocapture noundef readnone %genp,
-; CHECK-LABEL: define dso_local void @check_const(
-; CHECK-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr nocapture noundef readnone [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[GEN0:%.*]] = tail call i1 @llvm.nvvm.isspacep.const(ptr [[GENP]])
-; CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[GEN0]] to i8
-; CHECK-NEXT:    store i8 [[STOREDV]], ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
-; CHECK-NEXT:    store i8 1, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
-; CHECK-NEXT:    store i8 1, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
-; CHECK-NEXT:    ret void
-;
-  ptr addrspace(1) %gp,
-  ptr addrspace(3) %sp,
-  ptr addrspace(4) %cp,
-  ptr addrspace(5) %lp) local_unnamed_addr {
-entry:
-  ; No constant folding for generic pointers of unknown origin.
-  %gen0 = tail call i1 @llvm.nvvm.isspacep.const(ptr %genp)
-  %storedv = zext i1 %gen0 to i8
-  store i8 %storedv, ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
-
-  %isg1 = tail call i1 @llvm.nvvm.isspacep.const(ptr addrspacecast (ptr addrspace(1) @global_data to ptr))
-  %isg18 = zext i1 %isg1 to i8
-  store i8 %isg18, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
-
-  %gp_asc = addrspacecast ptr addrspace(1) %gp to ptr
-  %isg2 = tail call i1 @llvm.nvvm.isspacep.const(ptr %gp_asc)
-  %isg28 = zext i1 %isg2 to i8
-  store i8 %isg28, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
-
-  %iss1 = tail call i1 @llvm.nvvm.isspacep.const(ptr addrspacecast (ptr addrspace(3) @shared_data to ptr))
-  %iss18 = zext i1 %iss1 to i8
-  store i8 %iss18, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
-
-  %sp_asc = addrspacecast ptr addrspace(3) %sp to ptr
-  %iss2 = tail call i1 @llvm.nvvm.isspacep.const(ptr %sp_asc)
-  %iss28 = zext i1 %iss2 to i8
-  store i8 %iss28, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
-
-  %isc1 = tail call i1 @llvm.nvvm.isspacep.const(ptr addrspacecast (ptr addrspace(4) @const_data to ptr))
-  %isc18 = zext i1 %isc1 to i8
-  store i8 %isc18, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
-
-  %cp_asc = addrspacecast ptr addrspace(4) %cp to ptr
-  %isc2 = tail call i1 @llvm.nvvm.isspacep.const(ptr %cp_asc)
-  %isc28 = zext i1 %isc2 to i8
-  store i8 %isc28, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
-
-  ; Local data can't have a constant address, so we can't have a constant ASC expression
-  ; We can only use an ASC instruction.
-  %lp_asc = addrspacecast ptr addrspace(5) %lp to ptr
-  %isl = call i1 @llvm.nvvm.isspacep.const(ptr nonnull %lp_asc)
-  %isl8 = zext i1 %isl to i8
-  store i8 %isl8, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
-
-  ret void
-}
-
-define dso_local void @check_local(ptr nocapture noundef readnone %out, ptr nocapture noundef readnone %genp,
-; CHECK-LABEL: define dso_local void @check_local(
-; CHECK-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr nocapture noundef readnone [[GENP:%.*]], ptr addrspace(1) [[GP:%.*]], ptr addrspace(3) [[SP:%.*]], ptr addrspace(4) [[CP:%.*]], ptr addrspace(5) [[LP:%.*]]) local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[GEN0:%.*]] = tail call i1 @llvm.nvvm.isspacep.local(ptr [[GENP]])
-; CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[GEN0]] to i8
-; CHECK-NEXT:    store i8 [[STOREDV]], ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
-; CHECK-NEXT:    store i8 0, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
-; CHECK-NEXT:    store i8 1, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
-; CHECK-NEXT:    ret void
-;
-  ptr addrspace(1) %gp,
-  ptr addrspace(3) %sp,
-  ptr addrspace(4) %cp,
-  ptr addrspace(5) %lp) local_unnamed_addr {
-entry:
-  ; No constant folding for generic pointers of unknown origin.
-  %gen0 = tail call i1 @llvm.nvvm.isspacep.local(ptr %genp)
-  %storedv = zext i1 %gen0 to i8
-  store i8 %storedv, ptr addrspacecast (ptr addrspace(1) @gen to ptr), align 1
-
-  %isg1 = tail call i1 @llvm.nvvm.isspacep.local(ptr addrspacecast (ptr addrspace(1) @global_data to ptr))
-  %isg18 = zext i1 %isg1 to i8
-  store i8 %isg18, ptr addrspacecast (ptr addrspace(1) @g1 to ptr), align 1
-
-  %gp_asc = addrspacecast ptr addrspace(1) %gp to ptr
-  %isg2 = tail call i1 @llvm.nvvm.isspacep.local(ptr %gp_asc)
-  %isg28 = zext i1 %isg2 to i8
-  store i8 %isg28, ptr addrspacecast (ptr addrspace(1) @g2 to ptr), align 1
-
-  %iss1 = tail call i1 @llvm.nvvm.isspacep.local(ptr addrspacecast (ptr addrspace(3) @shared_data to ptr))
-  %iss18 = zext i1 %iss1 to i8
-  store i8 %iss18, ptr addrspacecast (ptr addrspace(1) @s1 to ptr), align 1
-
-  %sp_asc = addrspacecast ptr addrspace(3) %sp to ptr
-  %iss2 = tail call i1 @llvm.nvvm.isspacep.local(ptr %sp_asc)
-  %iss28 = zext i1 %iss2 to i8
-  store i8 %iss28, ptr addrspacecast (ptr addrspace(1) @s2 to ptr), align 1
-
-  %isc1 = tail call i1 @llvm.nvvm.isspacep.local(ptr addrspacecast (ptr addrspace(4) @const_data to ptr))
-  %isc18 = zext i1 %isc1 to i8
-  store i8 %isc18, ptr addrspacecast (ptr addrspace(1) @c1 to ptr), align 1
-
-  %cp_asc = addrspacecast ptr addrspace(4) %cp to ptr
-  %isc2 = tail call i1 @llvm.nvvm.isspacep.local(ptr %cp_asc)
-  %isc28 = zext i1 %isc2 to i8
-  store i8 %isc28, ptr addrspacecast (ptr addrspace(1) @c2 to ptr), align 1
-
-  ; Local data can't have a constant address, so we can't have a constant ASC expression
-  ; We can only use an ASC instruction.
-  %lp_asc = addrspacecast ptr addrspace(5) %lp to ptr
-  %isl = call i1 @llvm.nvvm.isspacep.local(ptr nonnull %lp_asc)
-  %isl8 = zext i1 %isl to i8
-  store i8 %isl8, ptr addrspacecast (ptr addrspace(1) @l to ptr), align 1
-
-  ret void
-}
-
-- 
GitLab


From 67c485798a16c4c656ff7a8a38cc98fe46d25154 Mon Sep 17 00:00:00 2001
From: Caio Oliveira <caio.oliveira@intel.com>
Date: Wed, 30 Oct 2024 15:39:32 -0700
Subject: [PATCH 656/693] [mlir][spirv] Ignore extra comma for category_args in
 gen_spirv_dialect.py (#111776)

In the code being parsed, the comma separates following traits from the
category args. If there's no category args, it is still present.
---
 mlir/utils/spirv/gen_spirv_dialect.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mlir/utils/spirv/gen_spirv_dialect.py b/mlir/utils/spirv/gen_spirv_dialect.py
index 2fb540ef1032..70c3d9db1688 100755
--- a/mlir/utils/spirv/gen_spirv_dialect.py
+++ b/mlir/utils/spirv/gen_spirv_dialect.py
@@ -989,6 +989,7 @@ def extract_td_op_info(op_def):
     op_tmpl_params, _ = get_string_between_nested(op_def, "<", ">")
     opstringname, rest = get_string_between(op_tmpl_params, '"', '"')
     category_args = rest.split("[", 1)[0]
+    category_args = category_args.rsplit(",", 1)[0]
 
     # Get traits
     traits, _ = get_string_between_nested(rest, "[", "]")
-- 
GitLab


From 6e75eec866133620dcba956bc7d6dbc554642249 Mon Sep 17 00:00:00 2001
From: Caio Oliveira <caio.oliveira@intel.com>
Date: Wed, 30 Oct 2024 15:40:08 -0700
Subject: [PATCH 657/693] [mlir][spirv] Remove code for de-duplicating symbols
 in SPIR-V grammar (#111778)

SPIR-V grammar was updated in upstream to have an "aliases" field
instead of duplicating symbols with same values. See
https://github.com/KhronosGroup/SPIRV-Headers/pull/447 for details.
---
 mlir/utils/spirv/gen_spirv_dialect.py | 101 +++-----------------------
 1 file changed, 10 insertions(+), 91 deletions(-)

diff --git a/mlir/utils/spirv/gen_spirv_dialect.py b/mlir/utils/spirv/gen_spirv_dialect.py
index 70c3d9db1688..99ed3489b4cb 100755
--- a/mlir/utils/spirv/gen_spirv_dialect.py
+++ b/mlir/utils/spirv/gen_spirv_dialect.py
@@ -127,44 +127,6 @@ def split_list_into_sublists(items):
     return chuncks
 
 
-def uniquify_enum_cases(lst):
-    """Prunes duplicate enum cases from the list.
-
-    Arguments:
-     - lst: List whose elements are to be uniqued. Assumes each element is a
-       (symbol, value) pair and elements already sorted according to value.
-
-    Returns:
-     - A list with all duplicates removed. The elements are sorted according to
-       value and, for each value, uniqued according to symbol.
-       original list,
-     - A map from deduplicated cases to the uniqued case.
-    """
-    cases = lst
-    uniqued_cases = []
-    duplicated_cases = {}
-
-    # First sort according to the value
-    cases.sort(key=lambda x: x[1])
-
-    # Then group them according to the value
-    for _, groups in itertools.groupby(cases, key=lambda x: x[1]):
-        # For each value, sort according to the enumerant symbol.
-        sorted_group = sorted(groups, key=lambda x: x[0])
-        # Keep the "smallest" case, which is typically the symbol without extension
-        # suffix. But we have special cases that we want to fix.
-        case = sorted_group[0]
-        for i in range(1, len(sorted_group)):
-            duplicated_cases[sorted_group[i][0]] = case[0]
-        if case[0] == "HlslSemanticGOOGLE":
-            assert len(sorted_group) == 2, "unexpected new variant for HlslSemantic"
-            case = sorted_group[1]
-            duplicated_cases[sorted_group[0][0]] = case[0]
-        uniqued_cases.append(case)
-
-    return uniqued_cases, duplicated_cases
-
-
 def toposort(dag, sort_fn):
     """Topologically sorts the given dag.
 
@@ -197,14 +159,12 @@ def toposort(dag, sort_fn):
     return sorted_nodes
 
 
-def toposort_capabilities(all_cases, capability_mapping):
+def toposort_capabilities(all_cases):
     """Returns topologically sorted capability (symbol, value) pairs.
 
     Arguments:
       - all_cases: all capability cases (containing symbol, value, and implied
         capabilities).
-      - capability_mapping: mapping from duplicated capability symbols to the
-        canonicalized symbol chosen for SPIRVBase.td.
 
     Returns:
       A list containing topologically sorted capability (symbol, value) pairs.
@@ -215,13 +175,10 @@ def toposort_capabilities(all_cases, capability_mapping):
         # Get the current capability.
         cur = case["enumerant"]
         name_to_value[cur] = case["value"]
-        # Ignore duplicated symbols.
-        if cur in capability_mapping:
-            continue
 
         # Get capabilities implied by the current capability.
         prev = case.get("capabilities", [])
-        uniqued_prev = set([capability_mapping.get(c, c) for c in prev])
+        uniqued_prev = set(prev)
         dag[cur] = uniqued_prev
 
     sorted_caps = toposort(dag, lambda x: name_to_value[x])
@@ -229,36 +186,12 @@ def toposort_capabilities(all_cases, capability_mapping):
     return [(c, name_to_value[c]) for c in sorted_caps]
 
 
-def get_capability_mapping(operand_kinds):
-    """Returns the capability mapping from duplicated cases to canonicalized ones.
-
-    Arguments:
-      - operand_kinds: all operand kinds' grammar spec
-
-    Returns:
-      - A map mapping from duplicated capability symbols to the canonicalized
-        symbol chosen for SPIRVBase.td.
-    """
-    # Find the operand kind for capability
-    cap_kind = {}
-    for kind in operand_kinds:
-        if kind["kind"] == "Capability":
-            cap_kind = kind
-
-    kind_cases = [(case["enumerant"], case["value"]) for case in cap_kind["enumerants"]]
-    _, capability_mapping = uniquify_enum_cases(kind_cases)
-
-    return capability_mapping
-
-
-def get_availability_spec(enum_case, capability_mapping, for_op, for_cap):
+def get_availability_spec(enum_case, for_op, for_cap):
     """Returns the availability specification string for the given enum case.
 
     Arguments:
       - enum_case: the enum case to generate availability spec for. It may contain
         'version', 'lastVersion', 'extensions', or 'capabilities'.
-      - capability_mapping: mapping from duplicated capability symbols to the
-        canonicalized symbol chosen for SPIRVBase.td.
       - for_op: bool value indicating whether this is the availability spec for an
         op itself.
       - for_cap: bool value indicating whether this is the availability spec for
@@ -313,10 +246,7 @@ def get_availability_spec(enum_case, capability_mapping, for_op, for_cap):
     if caps:
         canonicalized_caps = []
         for c in caps:
-            if c in capability_mapping:
-                canonicalized_caps.append(capability_mapping[c])
-            else:
-                canonicalized_caps.append(c)
+            canonicalized_caps.append(c)
         prefixed_caps = [
             "SPIRV_C_{}".format(c) for c in sorted(set(canonicalized_caps))
         ]
@@ -357,7 +287,7 @@ def get_availability_spec(enum_case, capability_mapping, for_op, for_cap):
     return "{}{}{}".format(implies, "\n  " if implies and avail else "", avail)
 
 
-def gen_operand_kind_enum_attr(operand_kind, capability_mapping):
+def gen_operand_kind_enum_attr(operand_kind):
     """Generates the TableGen EnumAttr definition for the given operand kind.
 
     Returns:
@@ -388,13 +318,12 @@ def gen_operand_kind_enum_attr(operand_kind, capability_mapping):
         # Special treatment for capability cases: we need to sort them topologically
         # because a capability can refer to another via the 'implies' field.
         kind_cases = toposort_capabilities(
-            operand_kind["enumerants"], capability_mapping
+            operand_kind["enumerants"]
         )
     else:
         kind_cases = [
             (case["enumerant"], case["value"]) for case in operand_kind["enumerants"]
         ]
-        kind_cases, _ = uniquify_enum_cases(kind_cases)
     max_len = max([len(symbol) for (symbol, _) in kind_cases])
 
     # Generate the definition for each enum case
@@ -412,7 +341,6 @@ def gen_operand_kind_enum_attr(operand_kind, capability_mapping):
             value = int(case_pair[1])
         avail = get_availability_spec(
             name_to_case_dict[name],
-            capability_mapping,
             False,
             kind_name == "Capability",
         )
@@ -648,11 +576,9 @@ def update_td_enum_attrs(path, operand_kinds, filter_list):
     ]
     filter_list.extend(existing_kinds)
 
-    capability_mapping = get_capability_mapping(operand_kinds)
-
     # Generate definitions for all enums in filter list
     defs = [
-        gen_operand_kind_enum_attr(kind, capability_mapping)
+        gen_operand_kind_enum_attr(kind)
         for kind in operand_kinds
         if kind["kind"] in filter_list
     ]
@@ -762,7 +688,7 @@ def get_description(text, appendix):
 
 
 def get_op_definition(
-    instruction, opname, doc, existing_info, capability_mapping, settings
+    instruction, opname, doc, existing_info, settings
 ):
     """Generates the TableGen op definition for the given SPIR-V instruction.
 
@@ -771,8 +697,6 @@ def get_op_definition(
       - doc: the instruction's SPIR-V HTML doc
       - existing_info: a dict containing potential manually specified sections for
         this instruction
-      - capability_mapping: mapping from duplicated capability symbols to the
-                     canonicalized symbol chosen for SPIRVBase.td
 
     Returns:
       - A string containing the TableGen op definition
@@ -840,7 +764,7 @@ def get_op_definition(
     operands = instruction.get("operands", [])
 
     # Op availability
-    avail = get_availability_spec(instruction, capability_mapping, True, False)
+    avail = get_availability_spec(instruction, True, False)
     if avail:
         avail = "\n\n  {0}".format(avail)
 
@@ -1021,7 +945,7 @@ def extract_td_op_info(op_def):
 
 
 def update_td_op_definitions(
-    path, instructions, docs, filter_list, inst_category, capability_mapping, settings
+    path, instructions, docs, filter_list, inst_category, settings
 ):
     """Updates SPIRVOps.td with newly generated op definition.
 
@@ -1030,8 +954,6 @@ def update_td_op_definitions(
       - instructions: SPIR-V JSON grammar for all instructions
       - docs: SPIR-V HTML doc for all instructions
       - filter_list: a list containing new opnames to include
-      - capability_mapping: mapping from duplicated capability symbols to the
-                     canonicalized symbol chosen for SPIRVBase.td.
 
     Returns:
       - A string containing all the TableGen op definitions
@@ -1079,7 +1001,6 @@ def update_td_op_definitions(
                     opname,
                     docs[fixed_opname],
                     op_info_dict.get(opname, {"inst_category": inst_category}),
-                    capability_mapping,
                     settings,
                 )
             )
@@ -1186,14 +1107,12 @@ if __name__ == "__main__":
     if args.new_inst is not None:
         assert args.op_td_path is not None
         docs = get_spirv_doc_from_html_spec(ext_html_url, args)
-        capability_mapping = get_capability_mapping(operand_kinds)
         update_td_op_definitions(
             args.op_td_path,
             instructions,
             docs,
             args.new_inst,
             args.inst_category,
-            capability_mapping,
             args,
         )
         print("Done. Note that this script just generates a template; ", end="")
-- 
GitLab


From 3a782ef97de771af9fd565e0043d49bdd0f2c850 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Wed, 30 Oct 2024 15:47:48 -0700
Subject: [PATCH 658/693] [lldb] Add a link to LLDB's Discord channel on the
 website (#114289)

Looking at #114276, I realized we have a link to Discourse on the
website, but not Discord. I think it would be helpful to have that link
there for real-time community discussion.
---
 lldb/docs/index.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lldb/docs/index.rst b/lldb/docs/index.rst
index e2c15d872b4b..fb22bdecad37 100644
--- a/lldb/docs/index.rst
+++ b/lldb/docs/index.rst
@@ -181,6 +181,7 @@ interesting areas to contribute to lldb.
 
    Source Code <https://github.com/llvm/llvm-project>
    Releases <https://github.com/llvm/llvm-project/releases>
+   Discord <https://discord.com/channels/636084430946959380/636732809708306432>
    Discussion Forums <https://discourse.llvm.org/c/subprojects/lldb/8>
    Developer Policy <https://llvm.org/docs/DeveloperPolicy.html>
    Bug Reports <https://github.com/llvm/llvm-project/issues?q=is%3Aissue+label%3Alldb+is%3Aopen>
-- 
GitLab


From 1cb599835ccf7ee8b2d1d5a7f3107e19a26fc6f5 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Thu, 31 Oct 2024 06:39:55 +0800
Subject: [PATCH 659/693] [RISCV] Remove redundant +zfh from +zvfh[min] tests.
 NFC

In the vast majority of f16 tests we don't end up emitting any scalar
code that needs +zfh, so remove it.
---
 .../RISCV/rvv/65704-illegal-instruction.ll    |  2 +-
 llvm/test/CodeGen/RISCV/rvv/ceil-vp.ll        |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/cmp-folds.ll      |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll        |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll        |  4 ++--
 .../CodeGen/RISCV/rvv/extload-truncstore.ll   |  4 ++--
 .../CodeGen/RISCV/rvv/extract-subvector.ll    |  4 ++--
 .../RISCV/rvv/fceil-constrained-sdnode.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/fceil-sdnode.ll   |  4 ++--
 .../RISCV/rvv/ffloor-constrained-sdnode.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/ffloor-sdnode.ll  |  4 ++--
 .../RISCV/rvv/fixed-vectors-binop-splats.ll   |  4 ++--
 .../RISCV/rvv/fixed-vectors-bitcast.ll        |  8 +++----
 .../RISCV/rvv/fixed-vectors-ceil-vp.ll        |  8 +++----
 .../rvv/fixed-vectors-compressstore-fp.ll     |  4 ++--
 .../rvv/fixed-vectors-deinterleave-load.ll    |  4 ++--
 .../RISCV/rvv/fixed-vectors-expandload-fp.ll  |  4 ++--
 .../RISCV/rvv/fixed-vectors-extract.ll        |  8 +++----
 .../fixed-vectors-fceil-constrained-sdnode.ll |  4 ++--
 ...fixed-vectors-ffloor-constrained-sdnode.ll |  4 ++--
 .../RISCV/rvv/fixed-vectors-floor-vp.ll       |  8 +++----
 .../RISCV/rvv/fixed-vectors-fmaximum-vp.ll    |  8 +++----
 .../RISCV/rvv/fixed-vectors-fmaximum.ll       |  8 +++----
 .../RISCV/rvv/fixed-vectors-fminimum-vp.ll    |  8 +++----
 .../RISCV/rvv/fixed-vectors-fminimum.ll       |  8 +++----
 ...d-vectors-fnearbyint-constrained-sdnode.ll |  4 ++--
 .../RISCV/rvv/fixed-vectors-fp-bitcast.ll     |  4 ++--
 .../RISCV/rvv/fixed-vectors-fp-buildvec.ll    |  8 +++----
 .../RISCV/rvv/fixed-vectors-fp-conv.ll        |  4 ++--
 .../RISCV/rvv/fixed-vectors-fp-interleave.ll  |  8 +++----
 .../RISCV/rvv/fixed-vectors-fp-splat.ll       |  4 ++--
 .../RISCV/rvv/fixed-vectors-fp2i-sat.ll       |  4 ++--
 .../CodeGen/RISCV/rvv/fixed-vectors-fp2i.ll   |  8 +++----
 .../RISCV/rvv/fixed-vectors-fpext-vp.ll       |  8 +++----
 .../RISCV/rvv/fixed-vectors-fptosi-vp-mask.ll |  8 +++----
 .../RISCV/rvv/fixed-vectors-fptosi-vp.ll      |  8 +++----
 .../RISCV/rvv/fixed-vectors-fptoui-vp-mask.ll |  8 +++----
 .../RISCV/rvv/fixed-vectors-fptoui-vp.ll      |  8 +++----
 .../RISCV/rvv/fixed-vectors-fptrunc-vp.ll     |  8 +++----
 ...fixed-vectors-fround-constrained-sdnode.ll |  4 ++--
 .../CodeGen/RISCV/rvv/fixed-vectors-fround.ll |  8 +++----
 ...d-vectors-froundeven-constrained-sdnode.ll |  4 ++--
 .../RISCV/rvv/fixed-vectors-froundeven.ll     |  8 +++----
 ...fixed-vectors-ftrunc-constrained-sdnode.ll |  4 ++--
 .../CodeGen/RISCV/rvv/fixed-vectors-i2fp.ll   |  8 +++----
 .../CodeGen/RISCV/rvv/fixed-vectors-insert.ll |  4 ++--
 .../rvv/fixed-vectors-interleave-store.ll     |  4 ++--
 .../RISCV/rvv/fixed-vectors-load-store.ll     |  4 ++--
 .../CodeGen/RISCV/rvv/fixed-vectors-load.ll   |  4 ++--
 .../RISCV/rvv/fixed-vectors-masked-gather.ll  |  8 +++----
 .../RISCV/rvv/fixed-vectors-masked-load-fp.ll |  4 ++--
 .../RISCV/rvv/fixed-vectors-masked-scatter.ll |  8 +++----
 .../rvv/fixed-vectors-masked-store-fp.ll      |  4 ++--
 .../RISCV/rvv/fixed-vectors-nearbyint-vp.ll   |  4 ++--
 .../rvv/fixed-vectors-reduction-fp-vp.ll      |  4 ++--
 .../RISCV/rvv/fixed-vectors-rint-vp.ll        |  4 ++--
 .../RISCV/rvv/fixed-vectors-round-vp.ll       |  8 +++----
 .../RISCV/rvv/fixed-vectors-roundeven-vp.ll   |  8 +++----
 .../RISCV/rvv/fixed-vectors-roundtozero-vp.ll |  8 +++----
 .../rvv/fixed-vectors-shuffle-reverse.ll      |  8 +++----
 .../rvv/fixed-vectors-shuffle-transpose.ll    |  4 ++--
 .../rvv/fixed-vectors-shuffle-vslide1down.ll  |  4 ++--
 .../rvv/fixed-vectors-shuffle-vslide1up.ll    |  4 ++--
 .../rvv/fixed-vectors-shufflevector-vnsrl.ll  |  4 ++--
 .../RISCV/rvv/fixed-vectors-sitofp-vp-mask.ll |  4 ++--
 .../RISCV/rvv/fixed-vectors-sitofp-vp.ll      |  8 +++----
 .../CodeGen/RISCV/rvv/fixed-vectors-store.ll  |  4 ++--
 .../rvv/fixed-vectors-strided-load-combine.ll |  6 ++---
 .../RISCV/rvv/fixed-vectors-uitofp-vp-mask.ll |  4 ++--
 .../RISCV/rvv/fixed-vectors-uitofp-vp.ll      |  8 +++----
 .../RISCV/rvv/fixed-vectors-vcopysign-vp.ll   |  4 ++--
 .../RISCV/rvv/fixed-vectors-vfabs-vp.ll       |  8 +++----
 .../fixed-vectors-vfadd-constrained-sdnode.ll |  4 ++--
 .../RISCV/rvv/fixed-vectors-vfclass-vp.ll     |  4 ++--
 .../RISCV/rvv/fixed-vectors-vfclass.ll        |  4 ++--
 .../fixed-vectors-vfcmp-constrained-sdnode.ll |  4 ++--
 ...fixed-vectors-vfcmps-constrained-sdnode.ll |  4 ++--
 .../fixed-vectors-vfdiv-constrained-sdnode.ll |  4 ++--
 .../RISCV/rvv/fixed-vectors-vfmacc-vp.ll      |  4 ++--
 ...fixed-vectors-vfmadd-constrained-sdnode.ll |  4 ++--
 .../RISCV/rvv/fixed-vectors-vfmax-vp.ll       |  8 +++----
 .../RISCV/rvv/fixed-vectors-vfmin-vp.ll       |  8 +++----
 .../RISCV/rvv/fixed-vectors-vfmsac-vp.ll      |  4 ++--
 ...fixed-vectors-vfmsub-constrained-sdnode.ll |  4 ++--
 .../fixed-vectors-vfmul-constrained-sdnode.ll |  4 ++--
 .../RISCV/rvv/fixed-vectors-vfmuladd-vp.ll    |  4 ++--
 .../RISCV/rvv/fixed-vectors-vfneg-vp.ll       |  8 +++----
 .../RISCV/rvv/fixed-vectors-vfnmacc-vp.ll     |  4 ++--
 ...ixed-vectors-vfnmadd-constrained-sdnode.ll |  4 ++--
 .../RISCV/rvv/fixed-vectors-vfnmsac-vp.ll     |  4 ++--
 ...ixed-vectors-vfnmsub-constrained-sdnode.ll |  4 ++--
 ...fixed-vectors-vfpext-constrained-sdnode.ll |  4 ++--
 ...xed-vectors-vfptrunc-constrained-sdnode.ll |  8 +++----
 .../RISCV/rvv/fixed-vectors-vfrdiv-vp.ll      |  4 ++--
 .../RISCV/rvv/fixed-vectors-vfrsub-vp.ll      |  4 ++--
 ...fixed-vectors-vfsqrt-constrained-sdnode.ll |  4 ++--
 .../RISCV/rvv/fixed-vectors-vfsqrt-vp.ll      |  8 +++----
 .../fixed-vectors-vfsub-constrained-sdnode.ll |  4 ++--
 .../fixed-vectors-vfw-web-simplification.ll   | 10 ++++----
 .../CodeGen/RISCV/rvv/fixed-vectors-vfwadd.ll |  4 ++--
 .../RISCV/rvv/fixed-vectors-vfwmacc.ll        |  4 ++--
 .../CodeGen/RISCV/rvv/fixed-vectors-vfwmul.ll |  4 ++--
 .../CodeGen/RISCV/rvv/fixed-vectors-vfwsub.ll |  4 ++--
 .../RISCV/rvv/fixed-vectors-vp-splat.ll       |  4 ++--
 .../RISCV/rvv/fixed-vectors-vpgather.ll       |  4 ++--
 .../CodeGen/RISCV/rvv/fixed-vectors-vpload.ll |  4 ++--
 .../RISCV/rvv/fixed-vectors-vpscatter.ll      |  4 ++--
 .../RISCV/rvv/fixed-vectors-vpstore.ll        |  4 ++--
 .../RISCV/rvv/fixed-vectors-vselect-vp.ll     |  8 +++----
 .../RISCV/rvv/fixed-vectors-vselect.ll        |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/floor-vp.ll       |  8 +++----
 .../test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll | 16 ++++++-------
 llvm/test/CodeGen/RISCV/rvv/fmaximum-vp.ll    |  8 +++----
 .../test/CodeGen/RISCV/rvv/fminimum-sdnode.ll | 16 ++++++-------
 llvm/test/CodeGen/RISCV/rvv/fminimum-vp.ll    |  8 +++----
 .../rvv/fnearbyint-constrained-sdnode.ll      |  4 ++--
 .../CodeGen/RISCV/rvv/fnearbyint-sdnode.ll    |  4 ++--
 .../CodeGen/RISCV/rvv/fold-binary-reduce.ll   |  2 +-
 llvm/test/CodeGen/RISCV/rvv/fptosi-sat.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/fptoui-sat.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/frint-sdnode.ll   |  4 ++--
 .../RISCV/rvv/fround-constrained-sdnode.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/fround-sdnode.ll  |  4 ++--
 .../rvv/froundeven-constrained-sdnode.ll      |  4 ++--
 .../CodeGen/RISCV/rvv/froundeven-sdnode.ll    |  4 ++--
 .../RISCV/rvv/ftrunc-constrained-sdnode.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/ftrunc-sdnode.ll  |  4 ++--
 .../test/CodeGen/RISCV/rvv/half-round-conv.ll |  4 ++--
 .../CodeGen/RISCV/rvv/insert-subvector.ll     |  4 ++--
 .../CodeGen/RISCV/rvv/legalize-load-sdnode.ll |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/masked-load-fp.ll |  8 +++----
 .../test/CodeGen/RISCV/rvv/masked-store-fp.ll |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/masked-tama.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/masked-tamu.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/masked-tuma.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/masked-tumu.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll |  8 +++----
 .../test/CodeGen/RISCV/rvv/mscatter-sdnode.ll |  8 +++----
 .../RISCV/rvv/named-vector-shuffle-reverse.ll | 24 +++++++++----------
 llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll   |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/rint-vp.ll        |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/round-vp.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/roundeven-vp.ll   |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/roundtozero-vp.ll |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/splat-vectors.ll  |  4 ++--
 .../RISCV/rvv/unaligned-loads-stores.ll       |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/undef-vp-ops.ll   |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/unmasked-ta.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vcompress.ll      |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vcopysign-vp.ll   |  4 ++--
 .../RISCV/rvv/vector-deinterleave-fixed.ll    |  4 ++--
 .../RISCV/rvv/vector-deinterleave-load.ll     |  4 ++--
 .../CodeGen/RISCV/rvv/vector-deinterleave.ll  |  8 +++----
 .../RISCV/rvv/vector-interleave-fixed.ll      |  8 +++----
 .../RISCV/rvv/vector-interleave-store.ll      |  4 ++--
 .../CodeGen/RISCV/rvv/vector-interleave.ll    | 12 +++++-----
 llvm/test/CodeGen/RISCV/rvv/vfabs-sdnode.ll   |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfabs-vp.ll       |  4 ++--
 .../RISCV/rvv/vfadd-constrained-sdnode.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfadd-sdnode.ll   |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfadd-vp.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfadd.ll          |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/vfclass-sdnode.ll |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfclass-vp.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfclass.ll        |  4 ++--
 .../RISCV/rvv/vfcmp-constrained-sdnode.ll     |  4 ++--
 .../RISCV/rvv/vfcmps-constrained-sdnode.ll    |  4 ++--
 .../CodeGen/RISCV/rvv/vfcopysign-sdnode.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfcvt-f-x.ll      |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfcvt-f-xu.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfcvt-rtz-x-f.ll  |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfcvt-rtz-xu-f.ll |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfcvt-x-f.ll      |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfcvt-xu-f.ll     |  4 ++--
 .../RISCV/rvv/vfdiv-constrained-sdnode.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfdiv-sdnode.ll   |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfdiv-vp.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfdiv.ll          |  4 ++--
 .../test/CodeGen/RISCV/rvv/vfma-vp-combine.ll |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfma-vp.ll        |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmacc-vp.ll      |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmacc.ll         |  4 ++--
 .../RISCV/rvv/vfmadd-constrained-sdnode.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmadd-sdnode.ll  |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmadd.ll         |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmax-sdnode.ll   |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmax-vp.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmax.ll          |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmerge.ll        |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmin-sdnode.ll   |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmin-vp.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmin.ll          |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmsac-vp.ll      |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmsac.ll         |  4 ++--
 .../RISCV/rvv/vfmsub-constrained-sdnode.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmsub-sdnode.ll  |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/vfmsub.ll         |  4 ++--
 .../RISCV/rvv/vfmul-constrained-sdnode.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmul-sdnode.ll   |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmul-vp.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmul.ll          |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmuladd-vp.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmv.f.s.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmv.s.f.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfmv.v.f.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfncvt-f-f.ll     |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/vfncvt-f-x.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfncvt-f-xu.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfncvt-rod-f-f.ll |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfncvt-rtz-x-f.ll |  4 ++--
 .../test/CodeGen/RISCV/rvv/vfncvt-rtz-xu-f.ll |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfncvt-x-f.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfncvt-xu-f.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfneg-sdnode.ll   |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfneg-vp.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfnmacc-vp.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfnmacc.ll        |  4 ++--
 .../RISCV/rvv/vfnmadd-constrained-sdnode.ll   |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfnmadd-sdnode.ll |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfnmadd.ll        |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfnmsac-vp.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfnmsac.ll        |  4 ++--
 .../RISCV/rvv/vfnmsub-constrained-sdnode.ll   |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfnmsub-sdnode.ll |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfnmsub.ll        |  4 ++--
 .../RISCV/rvv/vfpext-constrained-sdnode.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfpext-sdnode.ll  |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/vfpext-vp.ll      |  8 +++----
 .../RISCV/rvv/vfptoi-constrained-sdnode.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfptoi-sdnode.ll  |  8 +++----
 .../test/CodeGen/RISCV/rvv/vfptosi-vp-mask.ll |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/vfptosi-vp.ll     |  8 +++----
 .../test/CodeGen/RISCV/rvv/vfptoui-vp-mask.ll |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/vfptoui-vp.ll     |  8 +++----
 .../RISCV/rvv/vfptrunc-constrained-sdnode.ll  |  8 +++----
 .../test/CodeGen/RISCV/rvv/vfptrunc-sdnode.ll |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/vfptrunc-vp.ll    |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/vfrdiv-vp.ll      |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfrdiv.ll         |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfrec7.ll         |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfredmax.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfredmin.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfredosum.ll      |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfredusum.ll      |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfrsqrt7.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfrsub-vp.ll      |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfrsub.ll         |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfsgnj.ll         |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfsgnjn.ll        |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfsgnjx.ll        |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfslide1down.ll   |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfslide1up.ll     |  4 ++--
 .../RISCV/rvv/vfsqrt-constrained-sdnode.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfsqrt-sdnode.ll  |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfsqrt-vp.ll      |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfsqrt.ll         |  4 ++--
 .../RISCV/rvv/vfsub-constrained-sdnode.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfsub-sdnode.ll   |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfsub-vp.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfsub.ll          |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwadd-sdnode.ll  |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwadd.ll         |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwadd.w.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwcvt-f-f.ll     |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/vfwcvt-f-x.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwcvt-f-xu.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwcvt-rtz-x-f.ll |  4 ++--
 .../test/CodeGen/RISCV/rvv/vfwcvt-rtz-xu-f.ll |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwcvt-x-f.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwcvt-xu-f.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwmacc.ll        |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwmsac.ll        |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwmul-sdnode.ll  |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwmul.ll         |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwnmacc.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwnmsac.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwredosum.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwredusum.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwsub-sdnode.ll  |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwsub.ll         |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwsub.w.ll       |  4 ++--
 .../RISCV/rvv/vitofp-constrained-sdnode.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vitofp-sdnode.ll  |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/vloxseg-rv32.ll   |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vloxseg-rv64.ll   |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vlseg-rv32.ll     |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vlseg-rv64.ll     |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32.ll   |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64.ll   |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vlsseg-rv32.ll    |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vlsseg-rv64.ll    |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vluxseg-rv32.ll   |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vluxseg-rv64.ll   |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vmfeq.ll          |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vmfge.ll          |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vmfgt.ll          |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vmfle.ll          |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vmflt.ll          |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vmfne.ll          |  4 ++--
 .../test/CodeGen/RISCV/rvv/vpgather-sdnode.ll |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/vpload.ll         |  8 +++----
 .../CodeGen/RISCV/rvv/vpscatter-sdnode.ll     |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/vpstore.ll        |  8 +++----
 .../CodeGen/RISCV/rvv/vreductions-fp-vp.ll    |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll   |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll     |  8 +++----
 .../test/CodeGen/RISCV/rvv/vsitofp-vp-mask.ll |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vsitofp-vp.ll     |  8 +++----
 llvm/test/CodeGen/RISCV/rvv/vslidedown.ll     |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vslideup.ll       |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vsoxseg-rv32.ll   |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vsoxseg-rv64.ll   |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vsseg-rv32.ll     |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vsseg-rv64.ll     |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vssseg-rv32.ll    |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vssseg-rv64.ll    |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vsuxseg-rv32.ll   |  2 +-
 llvm/test/CodeGen/RISCV/rvv/vsuxseg-rv64.ll   |  2 +-
 .../test/CodeGen/RISCV/rvv/vuitofp-vp-mask.ll |  4 ++--
 llvm/test/CodeGen/RISCV/rvv/vuitofp-vp.ll     |  8 +++----
 322 files changed, 790 insertions(+), 790 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/rvv/65704-illegal-instruction.ll b/llvm/test/CodeGen/RISCV/rvv/65704-illegal-instruction.ll
index 5ced89c17c42..da477aa2043c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/65704-illegal-instruction.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/65704-illegal-instruction.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -mtriple=riscv64 -mattr=+v,+f,+m,+zfh,+zvfh \
+; RUN: llc -mtriple=riscv64 -mattr=+v,+f,+m,+zvfh \
 ; RUN:  < %s | FileCheck %s
 
 declare <16 x i8> @llvm.vector.extract.v16i8.nxv8i8(<vscale x 8 x i8>, i64 immarg)
diff --git a/llvm/test/CodeGen/RISCV/rvv/ceil-vp.ll b/llvm/test/CodeGen/RISCV/rvv/ceil-vp.ll
index 15cff650765e..f9588ffb5da5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/ceil-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/ceil-vp.ll
@@ -1,14 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFHMIN
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/cmp-folds.ll b/llvm/test/CodeGen/RISCV/rvv/cmp-folds.ll
index 4c40b7c74451..7a995a8d29f9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/cmp-folds.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/cmp-folds.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 8 x i1> @not_icmp_sle_nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
 ; CHECK-LABEL: not_icmp_sle_nxv8i16:
diff --git a/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll b/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll
index 9ea1394a1dd2..2c9f633b8901 100644
--- a/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
 ; RUN: llc -mtriple=riscv32 -mattr=+v,+zvbb,+m -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK-ZVBB
diff --git a/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll b/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll
index d36240e493e4..ed86755d5f48 100644
--- a/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
 ; RUN: llc -mtriple=riscv32 -mattr=+v,+zvbb,+m -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK-ZVBB
diff --git a/llvm/test/CodeGen/RISCV/rvv/extload-truncstore.ll b/llvm/test/CodeGen/RISCV/rvv/extload-truncstore.ll
index dfc70299d015..63fd13d98c7a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/extload-truncstore.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/extload-truncstore.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x i8> @sextload_nxv1i1_nxv1i8(ptr %x) {
 ; CHECK-LABEL: sextload_nxv1i1_nxv1i8:
diff --git a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll
index 63fa87852b64..8e9751502460 100644
--- a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple riscv32 -mattr=+m,+d,+zfh,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple riscv64 -mattr=+m,+d,+zfh,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple riscv32 -mattr=+m,+d,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple riscv64 -mattr=+m,+d,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 4 x i32> @extract_nxv8i32_nxv4i32_0(<vscale x 8 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv8i32_nxv4i32_0:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fceil-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fceil-constrained-sdnode.ll
index d8781495abd7..4e549a5aa7c3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fceil-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fceil-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x half> @ceil_nxv1f16(<vscale x 1 x half> %x) strictfp {
diff --git a/llvm/test/CodeGen/RISCV/rvv/fceil-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fceil-sdnode.ll
index ee16b476dc84..35936574e8fe 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fceil-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fceil-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/ffloor-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/ffloor-constrained-sdnode.ll
index 1df452d8641c..f6b47743d115 100644
--- a/llvm/test/CodeGen/RISCV/rvv/ffloor-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/ffloor-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x half> @floor_nxv1f16(<vscale x 1 x half> %x) strictfp {
diff --git a/llvm/test/CodeGen/RISCV/rvv/ffloor-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/ffloor-sdnode.ll
index 00e21ce8992b..d26b74c7c139 100644
--- a/llvm/test/CodeGen/RISCV/rvv/ffloor-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/ffloor-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-binop-splats.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-binop-splats.ll
index 29489be4dcb5..327b168ffe6b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-binop-splats.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-binop-splats.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zfh,+zvfh -verify-machineinstrs | FileCheck %s -check-prefixes=CHECK,RV32
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh -verify-machineinstrs | FileCheck %s -check-prefixes=CHECK,RV64
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh -verify-machineinstrs | FileCheck %s -check-prefixes=CHECK,RV32
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh -verify-machineinstrs | FileCheck %s -check-prefixes=CHECK,RV64
 
 define <1 x i1> @v1i1(i1 %x, i1 %y) {
 ; CHECK-LABEL: v1i1:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll
index f124d550df16..9ad1d7167c6a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll
@@ -1,12 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+v,+d,+zfh,+zvfh -verify-machineinstrs \
+; RUN: llc -mtriple=riscv32 -mattr=+v,+d,+zvfh -verify-machineinstrs \
 ; RUN:     -target-abi=ilp32d < %s | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+v,+d,+zfh,+zvfh -verify-machineinstrs \
+; RUN: llc -mtriple=riscv64 -mattr=+v,+d,+zvfh -verify-machineinstrs \
 ; RUN:     -target-abi=lp64d < %s | FileCheck %s --check-prefixes=CHECK,RV64
-; RUN: llc -mtriple=riscv32 -mattr=+zve32f,+zvl128b,+d,+zfh,+zvfh \
+; RUN: llc -mtriple=riscv32 -mattr=+zve32f,+zvl128b,+d,+zvfh \
 ; RUN:     -verify-machineinstrs -target-abi=ilp32d < %s | FileCheck %s \
 ; RUN:     --check-prefixes=ELEN32,RV32ELEN32
-; RUN: llc -mtriple=riscv64 -mattr=+zve32f,+zvl128b,+d,+zfh,+zvfh \
+; RUN: llc -mtriple=riscv64 -mattr=+zve32f,+zvl128b,+d,+zvfh \
 ; RUN:     -verify-machineinstrs -target-abi=lp64d < %s | FileCheck %s \
 ; RUN:     --check-prefixes=ELEN32,RV64ELEN32
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ceil-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ceil-vp.ll
index befbfb88550b..f7840be8f0c6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ceil-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ceil-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <2 x half> @llvm.vp.ceil.v2f16(<2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compressstore-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compressstore-fp.ll
index 36fbdd8e0664..9d42f2b6adee 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compressstore-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compressstore-fp.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64
 
 declare void @llvm.masked.compressstore.v1f16(<1 x half>, ptr, <1 x i1>)
 define void @compressstore_v1f16(ptr %base, <1 x half> %v, <1 x i1> %mask) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
index 9f8de22b25c2..1d7496397670 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zfh,+zvfh | FileCheck %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh | FileCheck %s
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh | FileCheck %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh | FileCheck %s
 
 ; Integers
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-expandload-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-expandload-fp.ll
index 8b31166e313d..707e0797a114 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-expandload-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-expandload-fp.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64
 
 declare <1 x half> @llvm.masked.expandload.v1f16(ptr, <1 x i1>, <1 x half>)
 define <1 x half> @expandload_v1f16(ptr %base, <1 x half> %src0, <1 x i1> %mask) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll
index d309da6df7dc..f2052ccc4627 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32NOM
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+zvfh,+f,+d,+m -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32M
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64NOM
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+zvfh,+f,+d,+m -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64M
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32NOM
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zvfh,+f,+d,+m -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32M
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64NOM
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zvfh,+f,+d,+m -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64M
 
 define i8 @extractelt_v16i8(ptr %x) nounwind {
 ; CHECK-LABEL: extractelt_v16i8:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fceil-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fceil-constrained-sdnode.ll
index 404fb72b8abe..84895715e814 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fceil-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fceil-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <1 x half> @ceil_v1f16(<1 x half> %x) strictfp {
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ffloor-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ffloor-constrained-sdnode.ll
index 2319aab370d2..3c99870dba95 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ffloor-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ffloor-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <1 x half> @floor_v1f16(<1 x half> %x) strictfp {
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-floor-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-floor-vp.ll
index c1b4c5fda6c6..87061581af73 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-floor-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-floor-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <2 x half> @llvm.vp.floor.v2f16(<2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum-vp.ll
index 51eb63f5f922..731f57a3a6d2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v,+m -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v,+m -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <2 x half> @llvm.vp.maximum.v2f16(<2 x half>, <2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum.ll
index 02c2fafc8978..ae592119cf88 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <2 x half> @llvm.maximum.v2f16(<2 x half>, <2 x half>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum-vp.ll
index 03e0ac42c442..dae1399d6690 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v,+m -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v,+m -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <2 x half> @llvm.vp.minimum.v2f16(<2 x half>, <2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum.ll
index b15d697f0754..8e042fc0785e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <2 x half> @llvm.minimum.v2f16(<2 x half>, <2 x half>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fnearbyint-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fnearbyint-constrained-sdnode.ll
index 719dd5249428..0b9fabb832e2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fnearbyint-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fnearbyint-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x half> @llvm.experimental.constrained.nearbyint.v2f16(<2 x half>, metadata, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-bitcast.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-bitcast.ll
index 5f5015c9ad16..5b35c0083ca0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-bitcast.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-bitcast.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:   --check-prefixes=CHECK,RV32-FP
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:   --check-prefixes=CHECK,RV64-FP
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
index 96b9b2bac299..924732e554f0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVFH
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+zvfh,+zba,+zbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVFH
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64ZVFH,RV64V
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+zvfh,+rva22u64 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64ZVFH,RVA22U64
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVFH
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zvfh,+zba,+zbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVFH
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64ZVFH,RV64V
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zvfh,+rva22u64 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64ZVFH,RVA22U64
 ; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zvfhmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVFHMIN,RV32-NO-ZFHMIN
 ; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zvfhmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64ZVFHMIN,RV64-NO-ZFHMIN
 ; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfhmin,+zvfhmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVFHMIN,RV32-ZFHMIN
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-conv.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-conv.ll
index d0dc70fd8115..ff52f5d2039e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-conv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-conv.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s
 
 define void @fpext_v2f16_v2f32(ptr %x, ptr %y) {
 ; CHECK-LABEL: fpext_v2f16_v2f32:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-interleave.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-interleave.ll
index f3b124aa34dc..a138b02b6139 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-interleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-interleave.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh,+m,+zvl128b -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,V128,RV32-V128
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+m,+zvl128b -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,V128,RV64-V128
-; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh,+m,+zvl512b -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,V512,RV32-V512
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+m,+zvl512b -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,V512,RV64-V512
+; RUN: llc -mtriple=riscv32 -mattr=+v,+zvfh,+m,+zvl128b -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,V128,RV32-V128
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfh,+m,+zvl128b -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,V128,RV64-V128
+; RUN: llc -mtriple=riscv32 -mattr=+v,+zvfh,+m,+zvl512b -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,V512,RV32-V512
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfh,+m,+zvl512b -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,V512,RV64-V512
 
 ; Test optimizing interleaves to widening arithmetic.
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-splat.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-splat.ll
index 250b3e90cbbb..0d0ef9c87946 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-splat.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-splat.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-RV32
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,RV64-ZVFH
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-RV32
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,RV64-ZVFH
 ; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zvfhmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,RV64-ZVFHMIN,RV64-ZVFHMIN-NOZFHMIN
 ; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfhmin,+zvfhmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,RV64-ZVFHMIN,RV64_ZVFHMIN-ZFHMIN
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i-sat.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i-sat.ll
index d92dc3edecb0..f0e6df629847 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i-sat.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i-sat.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
 
 define void @fp2si_v2f32_v2i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: fp2si_v2f32_v2i32:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i.ll
index 9d92018db2e8..da0bc5983a12 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+zvfh,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,ZVFH32
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+zvfh,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,ZVFH64
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+zvfhmin,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,ZVFHMIN32
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+zvfhmin,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,ZVFHMIN64
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zvfh,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,ZVFH32
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zvfh,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,ZVFH64
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zvfhmin,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,ZVFHMIN32
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zvfhmin,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,ZVFHMIN64
 
 define void @fp2si_v2f32_v2i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: fp2si_v2f32_v2i32:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fpext-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fpext-vp.ll
index 31d7844251a7..a1466d46f1ba 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fpext-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fpext-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x float> @llvm.vp.fpext.v2f32.v2f16(<2 x half>, <2 x i1>, i32)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptosi-vp-mask.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptosi-vp-mask.ll
index 602662b18429..bc86be6f62fd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptosi-vp-mask.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptosi-vp-mask.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <4 x i1> @llvm.vp.fptosi.v4i1.v4f16(<4 x half>, <4 x i1>, i32)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptosi-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptosi-vp.ll
index a4050b716e78..f6c992280c6e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptosi-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptosi-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <4 x i7> @llvm.vp.fptosi.v4i7.v4f16(<4 x half>, <4 x i1>, i32)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptoui-vp-mask.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptoui-vp-mask.ll
index c5bfd41ec951..c41f14076db3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptoui-vp-mask.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptoui-vp-mask.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <4 x i1> @llvm.vp.fptoui.v4i1.v4f16(<4 x half>, <4 x i1>, i32)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptoui-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptoui-vp.ll
index b652cdd88c7c..af225f4d95aa 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptoui-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptoui-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfhmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <4 x i7> @llvm.vp.fptoui.v4i7.v4f16(<4 x half>, <4 x i1>, i32)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptrunc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptrunc-vp.ll
index cd123cdf33a8..e64c7c87132e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptrunc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptrunc-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
 
 
 declare <2 x half> @llvm.vp.fptrunc.v2f16.v2f32(<2 x float>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fround-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fround-constrained-sdnode.ll
index e855d9504ff4..131fa53b3599 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fround-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fround-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 ; This file tests the code generation for `llvm.experimental.constrained.round.*` on scalable vector type.
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fround.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fround.ll
index 3f1bc0343584..b21be367f8ef 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fround.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fround.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 ; This file tests the code generation for `llvm.round.*` on fixed vector type.
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-froundeven-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-froundeven-constrained-sdnode.ll
index 9976cd2a8ab2..37f2e59ad751 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-froundeven-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-froundeven-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 ; This file tests the code generation for `llvm.experimental.constrained.roundeven.*` on scalable vector type.
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-froundeven.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-froundeven.ll
index 9607aa09d89d..13d62bb24441 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-froundeven.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-froundeven.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 ; This file tests the code generation for `llvm.roundeven.*` on fixed vector type.
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ftrunc-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ftrunc-constrained-sdnode.ll
index eac26451d5a8..b911722368ce 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ftrunc-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ftrunc-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <1 x half> @trunc_v1f16(<1 x half> %x) strictfp {
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-i2fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-i2fp.ll
index 9cdc9b81c953..29f8730021ce 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-i2fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-i2fp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+zvfh,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,ZVFH32
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+zvfh,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,ZVFH64
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+zvfhmin,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,ZVFHMIN32
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+zvfhmin,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,ZVFHMIN64
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zvfh,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,ZVFH32
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zvfh,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,ZVFH64
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zvfhmin,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,ZVFHMIN32
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zvfhmin,+zvfbfmin,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,ZVFHMIN64
 
 define void @si2fp_v2i32_v2f32(ptr %x, ptr %y) {
 ; CHECK-LABEL: si2fp_v2i32_v2f32:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll
index 7853e91ca53a..87f9bfbd1aaf 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
 
 define <4 x i32> @insertelt_v4i32_0(<4 x i32> %a, i32 %y) {
 ; CHECK-LABEL: insertelt_v4i32_0:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleave-store.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleave-store.ll
index 8de9cc25ae09..7de9b59c6853 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleave-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleave-store.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zfh,+zvfh | FileCheck %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh | FileCheck %s
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh | FileCheck %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh | FileCheck %s
 
 ; Integers
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load-store.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load-store.ll
index 3bf8d10654ac..2fab2b76ee27 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load-store.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s
+; RUN: llc -mtriple=riscv32 -mattr=+v,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s
 
 define void @v2i8(ptr %p, ptr %q) {
 ; CHECK-LABEL: v2i8:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll
index 22cde3c36ef6..8f1e026d09c0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s
+; RUN: llc -mtriple=riscv32 -mattr=+v,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s
 
 define <5 x i8> @load_v5i8(ptr %p) {
 ; CHECK-LABEL: load_v5i8:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
index 9c6ec6aef603..5802f45d311b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32V
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64V
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+zve32f,+zvl128b -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zvfh,+zve32f,+zvl128b -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVE32F
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+zve32f,+zvl128b -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zvfh,+zve32f,+zvl128b -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64ZVE32F
 
 declare <1 x i8> @llvm.masked.gather.v1i8.v1p0(<1 x ptr>, i32, <1 x i1>, <1 x i8>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-fp.ll
index 79b05334cb1f..f1d300b300a6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-fp.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
 
 define void @masked_load_v1f16(ptr %a, ptr %m_ptr, ptr %res_ptr) nounwind {
 ; CHECK-LABEL: masked_load_v1f16:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
index 323f08acac28..a445c8fe0817 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32V
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+zve32f,+zvl128b -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zvfh,+zve32f,+zvl128b -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVE32F
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+zve32f,+zvl128b -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zvfh,+zve32f,+zvl128b -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64ZVE32F
 
 declare void @llvm.masked.scatter.v1i8.v1p0(<1 x i8>, <1 x ptr>, i32, <1 x i1>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-store-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-store-fp.ll
index a1e81ea41c24..80110b3eef4d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-store-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-store-fp.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
 
 define void @masked_store_v1f16(<1 x half>* %val_ptr, <1 x half>* %a, <1 x half>* %m_ptr) nounwind {
 ; CHECK-LABEL: masked_store_v1f16:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll
index 93b4f7d2a9c9..3fab9ce63678 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x half> @llvm.vp.nearbyint.v2f16(<2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-fp-vp.ll
index 6d5be7f14bf7..6684e6d223ea 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-fp-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 declare half @llvm.vp.reduce.fadd.v2f16(half, <2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-rint-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-rint-vp.ll
index 1f856d04ca89..79dc2db8b169 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-rint-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-rint-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x half> @llvm.vp.rint.v2f16(<2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-round-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-round-vp.ll
index 0f587232680d..2228147176de 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-round-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-round-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <2 x half> @llvm.vp.round.v2f16(<2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundeven-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundeven-vp.ll
index 0fb7e6a7de56..336ffc8603fa 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundeven-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundeven-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <2 x half> @llvm.vp.roundeven.v2f16(<2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundtozero-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundtozero-vp.ll
index 927f96b64422..9f7124f1e4d9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundtozero-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundtozero-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <2 x half> @llvm.vp.roundtozero.v2f16(<2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
index cbf9829826fb..f5c45ba9ea58 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV64
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVBB,RV32-ZVBB
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVBB,RV64-ZVBB
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV64
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh,+zvbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVBB,RV32-ZVBB
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh,+zvbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVBB,RV64-ZVBB
 
 define <2 x i1> @reverse_v2i1(<2 x i1> %a) {
 ; NO-ZVBB-LABEL: reverse_v2i1:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-transpose.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-transpose.ll
index dffe0e0646ec..814e35f201dc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-transpose.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-transpose.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1down.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1down.ll
index ff5f6960ed4e..f531ff3a835e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1down.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1down.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll
index e0b2dd1af918..b3390b6eeecc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
index 4ef65032469e..a3e50685889d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+zvl256b \
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh,+zvl256b \
 ; RUN:   -lower-interleaved-accesses=false -verify-machineinstrs \
 ; RUN:   | FileCheck %s --check-prefixes=CHECK,V
-; RUN: llc < %s -mtriple=riscv64 -mattr=+f,+zve32f,+zfh,+zvfh,+zvl256b \
+; RUN: llc < %s -mtriple=riscv64 -mattr=+f,+zve32f,+zvfh,+zvl256b \
 ; RUN:   -lower-interleaved-accesses=false -verify-machineinstrs \
 ; RUN:   | FileCheck %s --check-prefixes=CHECK,ZVE32F
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sitofp-vp-mask.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sitofp-vp-mask.ll
index 67c045cc2b18..a1390a8b1c0d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sitofp-vp-mask.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sitofp-vp-mask.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh < %s | FileCheck %s
 
 declare <4 x half> @llvm.vp.sitofp.v4f16.v4i1(<4 x i1>, <4 x i1>, i32)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sitofp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sitofp-vp.ll
index bf0eab77d0ac..a2d41de5d185 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sitofp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sitofp-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh < %s | FileCheck %s \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh < %s | FileCheck %s \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfhmin < %s | FileCheck %s \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfhmin < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfhmin < %s | FileCheck %s \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfhmin < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFHMIN
 
 declare <4 x half> @llvm.vp.sitofp.v4f16.v4i7(<4 x i7>, <4 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll
index 169d99abb13c..5232d0d69fad 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s
+; RUN: llc -mtriple=riscv32 -mattr=+v,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s
 
 define void @store_v5i8(ptr %p, <5 x i8> %v) {
 ; CHECK-LABEL: store_v5i8:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll
index b49e323478e8..ed72883e9d05 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,CHECK-NO-MISALIGN,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,CHECK-NO-MISALIGN,RV64
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+unaligned-vector-mem -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,RV64,RV64-MISALIGN
+; RUN: llc -mtriple=riscv32 -mattr=+v,+zvfh -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,CHECK-NO-MISALIGN,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfh -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,CHECK-NO-MISALIGN,RV64
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfh,+unaligned-vector-mem -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,RV64,RV64-MISALIGN
 
 ; RUN: llc -mtriple=riscv64 -mattr=+f,+zfh,+zve64f,+zvl128b,+zvfh -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,CHECK-NO-MISALIGN,ZVE64F
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-uitofp-vp-mask.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-uitofp-vp-mask.ll
index adfb26cd3106..e625c46a5714 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-uitofp-vp-mask.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-uitofp-vp-mask.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh < %s | FileCheck %s
 
 declare <4 x half> @llvm.vp.uitofp.v4f16.v4i1(<4 x i1>, <4 x i1>, i32)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-uitofp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-uitofp-vp.ll
index e28d55f46abc..a0d5d2ccc848 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-uitofp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-uitofp-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh < %s \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh < %s \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh < %s \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh < %s \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfhmin < %s \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfhmin < %s \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfhmin < %s \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfhmin < %s \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <4 x half> @llvm.vp.uitofp.v4f16.v4i7(<4 x i7>, <4 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vcopysign-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vcopysign-vp.ll
index 77a095303675..f1dc476e5a43 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vcopysign-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vcopysign-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x half> @llvm.vp.copysign.v2f16(<2 x half>, <2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfabs-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfabs-vp.ll
index 90a856605c70..08f486b60132 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfabs-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfabs-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <2 x half> @llvm.vp.fabs.v2f16(<2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfadd-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfadd-constrained-sdnode.ll
index 441cbebf5675..599f505808ab 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfadd-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfadd-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x half> @llvm.experimental.constrained.fadd.v2f16(<2 x half>, <2 x half>, metadata, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfclass-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfclass-vp.ll
index 09b9e7ce4c53..690c8af7fc8e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfclass-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfclass-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <2 x i1> @isnan_v2f16(<2 x half> %x, <2 x i1> %m, i32 zeroext %evl) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfclass.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfclass.ll
index a1c36db5cfc3..85e8638301de 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfclass.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfclass.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <2 x i1> @isnan_v2f16(<2 x half> %x) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfcmp-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfcmp-constrained-sdnode.ll
index 1a75c50f2b64..4e9862b05f40 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfcmp-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfcmp-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <1 x i1> @llvm.experimental.constrained.fcmp.v1f16(<1 x half>, <1 x half>, metadata, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfcmps-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfcmps-constrained-sdnode.ll
index 83037baf3fab..97641ff6d92d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfcmps-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfcmps-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <1 x i1> @llvm.experimental.constrained.fcmps.v1f16(<1 x half>, <1 x half>, metadata, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfdiv-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfdiv-constrained-sdnode.ll
index fb9612d09504..1bc880d93af1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfdiv-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfdiv-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x half> @llvm.experimental.constrained.fdiv.v2f16(<2 x half>, <2 x half>, metadata, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmacc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmacc-vp.ll
index 2d6e1fd02dee..bc13e1d217a9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmacc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmacc-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x half> @llvm.vp.fma.v2f16(<2 x half>, <2 x half>, <2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmadd-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmadd-constrained-sdnode.ll
index 52d96fc63fad..b8f3f0fef041 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmadd-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmadd-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 ; This tests a mix of vfmacc and vfmadd by using different operand orders to
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmax-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmax-vp.ll
index c83a298cb501..1f3c7a915d84 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmax-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmax-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <2 x half> @llvm.vp.maxnum.v2f16(<2 x half>, <2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmin-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmin-vp.ll
index 60dbededb90a..a3dbd3345137 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmin-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmin-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <2 x half> @llvm.vp.minnum.v2f16(<2 x half>, <2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmsac-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmsac-vp.ll
index fc6578225aa6..99fc03523567 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmsac-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmsac-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x half> @llvm.vp.fma.v2f16(<2 x half>, <2 x half>, <2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmsub-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmsub-constrained-sdnode.ll
index 652198b0d446..268494bf337e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmsub-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmsub-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 ; This tests a mix of vfmsac and vfmsub by using different operand orders to
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmul-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmul-constrained-sdnode.ll
index d5e96c88f938..c8148a5e8d49 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmul-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmul-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <1 x half> @llvm.experimental.constrained.fmul.v1f16(<1 x half>, <1 x half>, metadata, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmuladd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmuladd-vp.ll
index 6c695b43d271..3912a37e6beb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmuladd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfmuladd-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x half> @llvm.vp.fmuladd.v2f16(<2 x half>, <2 x half>, <2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfneg-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfneg-vp.ll
index 019923ffdfde..968fd9f9bab8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfneg-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfneg-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <2 x half> @llvm.vp.fneg.v2f16(<2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmacc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmacc-vp.ll
index 6d65ab4083f7..4ab94444b1b8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmacc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmacc-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x half> @llvm.vp.fma.v2f16(<2 x half>, <2 x half>, <2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmadd-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmadd-constrained-sdnode.ll
index b7f5dd49b350..afc89717596b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmadd-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmadd-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 ; This tests a mix of vfnmacc and vfnmadd by using different operand orders to
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmsac-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmsac-vp.ll
index df705270664b..4d9b002cc785 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmsac-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmsac-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x half> @llvm.vp.fma.v2f16(<2 x half>, <2 x half>, <2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmsub-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmsub-constrained-sdnode.ll
index ace96c1a571d..d9863bb36c73 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmsub-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfnmsub-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 ; This tests a mix of vfnmsac and vfnmsub by using different operand orders to
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfpext-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfpext-constrained-sdnode.ll
index 5321f731441e..59fd8bbd1795 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfpext-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfpext-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x float> @llvm.experimental.constrained.fpext.v2f32.v2f16(<2 x half>, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfptrunc-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfptrunc-constrained-sdnode.ll
index 4ac72bf0a3b0..1f74691437ad 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfptrunc-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfptrunc-constrained-sdnode.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v,+zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v,+zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x float> @llvm.experimental.constrained.fptrunc.v2f32.v2f64(<2 x double>, metadata, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfrdiv-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfrdiv-vp.ll
index bd354b7dae80..fb813d4381a7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfrdiv-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfrdiv-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x half> @llvm.vp.fdiv.v2f16(<2 x half>, <2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfrsub-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfrsub-vp.ll
index 0903ef8c8ec3..63c2d1f2e7db 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfrsub-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfrsub-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x half> @llvm.vp.fsub.v2f16(<2 x half>, <2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfsqrt-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfsqrt-constrained-sdnode.ll
index 9f29d14050de..62d03e1ab588 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfsqrt-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfsqrt-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x half> @llvm.experimental.constrained.sqrt.v2f16(<2 x half>, metadata, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfsqrt-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfsqrt-vp.ll
index 988b200ae536..c1e63cbf0b13 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfsqrt-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfsqrt-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <2 x half> @llvm.vp.sqrt.v2f16(<2 x half>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfsub-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfsub-constrained-sdnode.ll
index f9d40d7a117b..e6001352a237 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfsub-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfsub-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x half> @llvm.experimental.constrained.fsub.v2f16(<2 x half>, <2 x half>, metadata, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfw-web-simplification.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfw-web-simplification.ll
index cb50ca4a7212..1144f776e7fb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfw-web-simplification.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfw-web-simplification.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs %s -o - --riscv-lower-ext-max-web-size=1 | FileCheck %s --check-prefixes=NO_FOLDING
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs %s -o - --riscv-lower-ext-max-web-size=2 | FileCheck %s --check-prefixes=NO_FOLDING
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs %s -o - --riscv-lower-ext-max-web-size=3 | FileCheck %s --check-prefixes=FOLDING,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfhmin,+f,+d -verify-machineinstrs %s -o - --riscv-lower-ext-max-web-size=3 | FileCheck %s --check-prefixes=FOLDING,ZVFHMIN
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfh,+f,+d -verify-machineinstrs %s -o - --riscv-lower-ext-max-web-size=1 | FileCheck %s --check-prefixes=NO_FOLDING
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfh,+f,+d -verify-machineinstrs %s -o - --riscv-lower-ext-max-web-size=2 | FileCheck %s --check-prefixes=NO_FOLDING
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfh,+f,+d -verify-machineinstrs %s -o - --riscv-lower-ext-max-web-size=3 | FileCheck %s --check-prefixes=FOLDING,ZVFH
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfhmin,+f,+d -verify-machineinstrs %s -o - --riscv-lower-ext-max-web-size=3 | FileCheck %s --check-prefixes=FOLDING,ZVFHMIN
 ; Check that the default value enables the web folding and
 ; that it is bigger than 3.
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=FOLDING
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfh,+f,+d -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=FOLDING
 
 define void @vfwmul_v2f116_multiple_users(ptr %x, ptr %y, ptr %z, <2 x half> %a, <2 x half> %b, <2 x half> %b2) {
 ; NO_FOLDING-LABEL: vfwmul_v2f116_multiple_users:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwadd.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwadd.ll
index afea1dc6d3c2..4bd521725f48 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwadd.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh,+f,+d -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+v,+zvfh,+f,+d -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+f,+d -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfh,+f,+d -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 define <2 x float> @vfwadd_v2f16(ptr %x, ptr %y) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwmacc.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwmacc.ll
index 5140d89b7830..a48be7687106 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwmacc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwmacc.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <1 x float> @llvm.fma.v1f32(<1 x float>, <1 x float>, <1 x float>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwmul.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwmul.ll
index 319994d26556..84c126217789 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwmul.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwmul.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh,+f,+d -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+v,+zvfh,+f,+d -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+f,+d -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfh,+f,+d -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 define <2 x float> @vfwmul_v2f16(ptr %x, ptr %y) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwsub.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwsub.ll
index 2c706cad9742..b8b26a4d5adf 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwsub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwsub.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh,+f,+d -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+v,+zvfh,+f,+d -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+f,+d -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfh,+f,+d -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 define <2 x float> @vfwsub_v2f16(ptr %x, ptr %y) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splat.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splat.ll
index a31405f75e8a..2e3e36e45d57 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splat.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splat.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+v,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+v,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: llc -mtriple=riscv32 -mattr=+v,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+v,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
 
 define <1 x i8> @vp_splat_v1i8(i8 %val, <1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_splat_v1i8:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpgather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpgather.ll
index 028fb9a626f0..f66974e51140 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpgather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpgather.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64
 
 declare <2 x i8> @llvm.vp.gather.v2i8.v2p0(<2 x ptr>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll
index f204d812c14f..351d7d4cd9b0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x i8> @llvm.vp.load.v2i8.p0(ptr, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpscatter.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpscatter.ll
index c05503987619..0a61bc42326f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpscatter.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpscatter.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64
 
 declare void @llvm.vp.scatter.v2i8.v2p0(<2 x i8>, <2 x ptr>, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpstore.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpstore.ll
index f396790f4f17..d34292abdce0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpstore.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpstore.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 declare void @llvm.vp.store.v2i8.p0(<2 x i8>, ptr, <2 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll
index 0a2ed3eb1ffb..99aafdbcde12 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d -riscv-v-vector-bits-min=128 \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v,+m -target-abi=ilp32d -riscv-v-vector-bits-min=128 \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d -riscv-v-vector-bits-min=128 \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v,+m -target-abi=lp64d -riscv-v-vector-bits-min=128 \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 declare <1 x i1> @llvm.vp.select.v1i1(<1 x i1>, <1 x i1>, <1 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll
index 2194651a95e5..a3bba2dd8265 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,RV64
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,RV64
 
 define void @vselect_vv_v6i32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 ; RV32-LABEL: vselect_vv_v6i32:
diff --git a/llvm/test/CodeGen/RISCV/rvv/floor-vp.ll b/llvm/test/CodeGen/RISCV/rvv/floor-vp.ll
index 03d1fb6c8d29..40c855b5d045 100644
--- a/llvm/test/CodeGen/RISCV/rvv/floor-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/floor-vp.ll
@@ -1,14 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFHMIN
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll
index d8c3ab27cfad..ec5b0136c383 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll
@@ -1,19 +1,19 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <vscale x 1 x bfloat> @llvm.maximum.nxv1bf16(<vscale x 1 x bfloat>, <vscale x 1 x bfloat>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fmaximum-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fmaximum-vp.ll
index dd01e1c1ee66..7b70a0daf11c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fmaximum-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fmaximum-vp.ll
@@ -1,16 +1,16 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v,+m \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v,+m \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v,+m \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v,+m \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 \
-; RUN:     -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m \
+; RUN:     -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFHMIN
 ; RUN: llc -mtriple=riscv64 \
-; RUN:     -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m \
+; RUN:     -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFHMIN
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll
index 2371840002f4..4061cbca0c48 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll
@@ -1,19 +1,19 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <vscale x 1 x bfloat> @llvm.minimum.nxv1bf16(<vscale x 1 x bfloat>, <vscale x 1 x bfloat>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fminimum-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fminimum-vp.ll
index 85cac8d18705..2526b8765177 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fminimum-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fminimum-vp.ll
@@ -1,16 +1,16 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v,+m \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v,+m \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v,+m \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v,+m \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 \
-; RUN:     -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m \
+; RUN:     -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFHMIN
 ; RUN: llc -mtriple=riscv64 \
-; RUN:     -mattr=+d,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m \
+; RUN:     -mattr=+d,+zvfhmin,+zfbfmin,+zvfbfmin,+v,+m \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFHMIN
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fnearbyint-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fnearbyint-constrained-sdnode.ll
index 372937bb5ca5..f22cd77db7a4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fnearbyint-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fnearbyint-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.experimental.constrained.nearbyint.nxv1f16(<vscale x 1 x half>, metadata, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fnearbyint-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fnearbyint-sdnode.ll
index 9498c65ba9a1..89769615365c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fnearbyint-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fnearbyint-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll b/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll
index adfae5ede7bb..5bc1ab9820d6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+zbb -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+zbb -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s
 
 define i64 @reduce_add(i64 %x, <4 x i64> %v) {
 ; CHECK-LABEL: reduce_add:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fptosi-sat.ll b/llvm/test/CodeGen/RISCV/rvv/fptosi-sat.ll
index 393cd5c7f52e..52e5ecf9cb8a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fptosi-sat.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fptosi-sat.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 ; Float
diff --git a/llvm/test/CodeGen/RISCV/rvv/fptoui-sat.ll b/llvm/test/CodeGen/RISCV/rvv/fptoui-sat.ll
index a7efa4b3de94..02b43c2d9529 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fptoui-sat.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fptoui-sat.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK32
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK64
 
 ; Float
diff --git a/llvm/test/CodeGen/RISCV/rvv/frint-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/frint-sdnode.ll
index 7fac8949c551..3fd37384ada9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/frint-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/frint-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/fround-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fround-constrained-sdnode.ll
index aaa7a538e70f..095f44cfb63e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fround-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fround-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 ; This file tests the code generation for `llvm.experimental.constrained.round.*` on scalable vector type.
diff --git a/llvm/test/CodeGen/RISCV/rvv/fround-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fround-sdnode.ll
index 193773b0c89c..fd834e9eb527 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fround-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fround-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/froundeven-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/froundeven-constrained-sdnode.ll
index cdc01d658778..051939d988f8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/froundeven-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/froundeven-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 ; This file tests the code generation for `llvm.experimental.constrained.roundeven.*` on scalable vector type.
diff --git a/llvm/test/CodeGen/RISCV/rvv/froundeven-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/froundeven-sdnode.ll
index 052ee2d3a43c..851465882467 100644
--- a/llvm/test/CodeGen/RISCV/rvv/froundeven-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/froundeven-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/ftrunc-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/ftrunc-constrained-sdnode.ll
index 21615b516da8..d07bc2c6bf74 100644
--- a/llvm/test/CodeGen/RISCV/rvv/ftrunc-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/ftrunc-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x half> @trunc_nxv1f16(<vscale x 1 x half> %x) strictfp {
diff --git a/llvm/test/CodeGen/RISCV/rvv/ftrunc-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/ftrunc-sdnode.ll
index b29b24a9ce7b..2b3c952679ea 100644
--- a/llvm/test/CodeGen/RISCV/rvv/ftrunc-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/ftrunc-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/half-round-conv.ll b/llvm/test/CodeGen/RISCV/rvv/half-round-conv.ll
index 5cd9996c7ba3..4d47c265a974 100644
--- a/llvm/test/CodeGen/RISCV/rvv/half-round-conv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/half-round-conv.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+zfh,+zvfh,+v -verify-machineinstrs < %s | \
+; RUN: llc -mtriple=riscv32 -mattr=+zvfh,+v -verify-machineinstrs < %s | \
 ; RUN:   FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+zfh,+zvfh,+v -verify-machineinstrs < %s | \
+; RUN: llc -mtriple=riscv64 -mattr=+zvfh,+v -verify-machineinstrs < %s | \
 ; RUN:   FileCheck %s
 
 ; ================================================================================
diff --git a/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll
index 71f90153b212..e2298774a9b8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple riscv32 -mattr=+m,+d,+zfh,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple riscv64 -mattr=+m,+d,+zfh,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple riscv32 -mattr=+m,+d,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple riscv64 -mattr=+m,+d,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 8 x i32> @insert_nxv8i32_nxv4i32_0(<vscale x 8 x i32> %vec, <vscale x 4 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv8i32_nxv4i32_0:
diff --git a/llvm/test/CodeGen/RISCV/rvv/legalize-load-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/legalize-load-sdnode.ll
index eada90e055df..e9e1303d1076 100644
--- a/llvm/test/CodeGen/RISCV/rvv/legalize-load-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/legalize-load-sdnode.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh,+f,+d -verify-machineinstrs < %s | FileCheck %s
 
 ; Check that we are able to legalize scalable-vector loads that require widening.
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-load-fp.ll b/llvm/test/CodeGen/RISCV/rvv/masked-load-fp.ll
index 9c7ad239bcad..c301d4975e71 100644
--- a/llvm/test/CodeGen/RISCV/rvv/masked-load-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/masked-load-fp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+zvfbfmin,+v -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+zvfbfmin,+v -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zvfbfmin,+v -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zvfbfmin,+v -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+zvfbfmin,+v -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+zvfbfmin,+v -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x bfloat> @masked_load_nxv1bf16(ptr %a, <vscale x 1 x i1> %mask) nounwind {
 ; CHECK-LABEL: masked_load_nxv1bf16:
diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-store-fp.ll b/llvm/test/CodeGen/RISCV/rvv/masked-store-fp.ll
index ddb56e0d979a..586af50266f9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/masked-store-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/masked-store-fp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+zvfbfmin,+v -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+zvfbfmin,+v -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zvfbfmin,+v -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zvfbfmin,+v -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+zvfbfmin,+v -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+zvfbfmin,+v -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s
 
 define void @masked_store_nxv1bf16(<vscale x 1 x bfloat> %val, ptr %a, <vscale x 1 x i1> %mask) nounwind {
 ; CHECK-LABEL: masked_store_nxv1bf16:
diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll b/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll
index 5c14ed1e813c..420597b009f3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s --check-prefixes=CHECK,RV64
 
 declare <vscale x 1 x i64> @llvm.riscv.vle.mask.nxv1i64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-tamu.ll b/llvm/test/CodeGen/RISCV/rvv/masked-tamu.ll
index 4098270d365a..0e771eb7c431 100644
--- a/llvm/test/CodeGen/RISCV/rvv/masked-tamu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/masked-tamu.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s --check-prefixes=CHECK,RV64
 
 declare <vscale x 1 x i64> @llvm.riscv.vle.mask.nxv1i64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-tuma.ll b/llvm/test/CodeGen/RISCV/rvv/masked-tuma.ll
index 4cd7e143be66..c4c2fc88913b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/masked-tuma.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/masked-tuma.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s --check-prefixes=CHECK,RV64
 
 declare <vscale x 1 x i64> @llvm.riscv.vle.mask.nxv1i64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-tumu.ll b/llvm/test/CodeGen/RISCV/rvv/masked-tumu.ll
index c8719e6a2e7c..ec0ebb10e8f7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/masked-tumu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/masked-tumu.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh\
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh\
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s --check-prefixes=CHECK,RV64
 
 declare <vscale x 1 x i64> @llvm.riscv.vle.mask.nxv1i64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
index 189ba08dddc7..9f6fc3b5d7d1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
@@ -1,14 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,RV64
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfhmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zvfhmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfhmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zvfhmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,RV64
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll
index 29db67b4b0a4..f75f8dfedc54 100644
--- a/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll
@@ -1,14 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,RV64
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfhmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zvfhmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfhmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zvfhmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,RV64
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
index a6c6db345032..20296c09998b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
@@ -1,16 +1,16 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-UNKNOWN
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-256
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-512
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-UNKNOWN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-256
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-512
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-UNKNOWN
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-256
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-512
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-UNKNOWN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-256
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-512
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-UNKNOWN
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-256
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-512
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-UNKNOWN
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-256
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-512
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfhmin,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-UNKNOWN
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-256
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-512
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfhmin,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-UNKNOWN
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-256
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-512
 
 ;
 ; VECTOR_REVERSE - masks
diff --git a/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll b/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll
index 5aa773b01e69..12c7b9cf13e1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/rint-vp.ll b/llvm/test/CodeGen/RISCV/rvv/rint-vp.ll
index a454f9dd97ce..7b6027991c32 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rint-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rint-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/round-vp.ll b/llvm/test/CodeGen/RISCV/rvv/round-vp.ll
index a4936483e8a1..6f5c1eab7f07 100644
--- a/llvm/test/CodeGen/RISCV/rvv/round-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/round-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/roundeven-vp.ll b/llvm/test/CodeGen/RISCV/rvv/roundeven-vp.ll
index 9857009002eb..447962a7542f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/roundeven-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/roundeven-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/roundtozero-vp.ll b/llvm/test/CodeGen/RISCV/rvv/roundtozero-vp.ll
index 11830c924867..7249069294c4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/roundtozero-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/roundtozero-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
index 5ba4efa8458c..3e6f8953a515 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/splat-vectors.ll b/llvm/test/CodeGen/RISCV/rvv/splat-vectors.ll
index 8e422fa6f76b..2e6df1184017 100644
--- a/llvm/test/CodeGen/RISCV/rvv/splat-vectors.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/splat-vectors.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+v,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+v,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+v,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+v,+d,+zvfh -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 4 x i32> @splat_c3_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: splat_c3_nxv4i32:
diff --git a/llvm/test/CodeGen/RISCV/rvv/unaligned-loads-stores.ll b/llvm/test/CodeGen/RISCV/rvv/unaligned-loads-stores.ll
index 1491bb6c337a..b5613a4a6358 100644
--- a/llvm/test/CodeGen/RISCV/rvv/unaligned-loads-stores.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/unaligned-loads-stores.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple riscv32 -mattr=+d,+zfh,+zvfh,+v < %s \
+; RUN: llc -mtriple riscv32 -mattr=+d,+zvfh,+v < %s \
 ; RUN:    -verify-machineinstrs | FileCheck %s
-; RUN: llc -mtriple riscv64 -mattr=+d,+zfh,+zvfh,+v < %s \
+; RUN: llc -mtriple riscv64 -mattr=+d,+zvfh,+v < %s \
 ; RUN:    -verify-machineinstrs | FileCheck %s
-; RUN: llc -mtriple riscv32 -mattr=+d,+zfh,+zvfh,+v,+unaligned-vector-mem < %s \
+; RUN: llc -mtriple riscv32 -mattr=+d,+zvfh,+v,+unaligned-vector-mem < %s \
 ; RUN:    -verify-machineinstrs | FileCheck --check-prefix=FAST %s
-; RUN: llc -mtriple riscv64 -mattr=+d,+zfh,+zvfh,+v,+unaligned-vector-mem < %s \
+; RUN: llc -mtriple riscv64 -mattr=+d,+zvfh,+v,+unaligned-vector-mem < %s \
 ; RUN:    -verify-machineinstrs | FileCheck --check-prefix=FAST %s
 
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/undef-vp-ops.ll b/llvm/test/CodeGen/RISCV/rvv/undef-vp-ops.ll
index 1bfc0f432eb5..6b0abeef657e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/undef-vp-ops.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/undef-vp-ops.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 ; Test that we can remove trivially-undef VP operations of various kinds.
diff --git a/llvm/test/CodeGen/RISCV/rvv/unmasked-ta.ll b/llvm/test/CodeGen/RISCV/rvv/unmasked-ta.ll
index bcc7bb9f072f..8640b61e6462 100644
--- a/llvm/test/CodeGen/RISCV/rvv/unmasked-ta.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/unmasked-ta.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s --check-prefixes=CHECK,RV64
 
 declare <vscale x 1 x float> @llvm.riscv.vfmacc.nxv1f32.nxv1f32(
diff --git a/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll b/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll
index aeb3f6c17485..3052108a12e3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s --check-prefixes=CHECK,RV64
 
 declare <vscale x 1 x i8> @llvm.riscv.vle.nxv1i8(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vcompress.ll b/llvm/test/CodeGen/RISCV/rvv/vcompress.ll
index b763e116a9f6..5ee82e6d95d4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vcompress.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vcompress.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfhmin,+zvfbfmin \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfhmin,+zvfbfmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfhmin,+zvfbfmin \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vcompress.nxv1i8(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vcopysign-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vcopysign-vp.ll
index e8a7d7907585..ccf82b93d6b7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vcopysign-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vcopysign-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
index eb02fd895f18..4fed94401f0a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zfh,+zvfh | FileCheck %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh | FileCheck %s
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh | FileCheck %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh | FileCheck %s
 
 ; Integers
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
index 54373d94f8f5..2521e4d707b1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zfh,+zvfh,+m | FileCheck --check-prefixes=CHECK,RV32 %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+m | FileCheck --check-prefixes=CHECK,RV64 %s
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh,+m | FileCheck --check-prefixes=CHECK,RV32 %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh,+m | FileCheck --check-prefixes=CHECK,RV64 %s
 
 ; Integers
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
index 28f7eb4329e3..14fe477f537c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin | FileCheck %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin | FileCheck %s
-; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfhmin,+zvfbfmin | FileCheck %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfhmin,+zvfbfmin | FileCheck %s
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zvfh,+zvfbfmin | FileCheck %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvfh,+zvfbfmin | FileCheck %s
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s
 
 ; Integers
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll
index 99872c199a1e..e730ae230d5a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zfh,+zvfh | FileCheck -check-prefixes=CHECK,RV32 %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh | FileCheck -check-prefixes=CHECK,RV64 %s
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvbb,+zfh,+zvfh | FileCheck %s --check-prefix=ZVBB
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvbb,+zfh,+zvfh | FileCheck %s --check-prefix=ZVBB
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh | FileCheck -check-prefixes=CHECK,RV32 %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh | FileCheck -check-prefixes=CHECK,RV64 %s
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvbb,+zvfh | FileCheck %s --check-prefix=ZVBB
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvbb,+zvfh | FileCheck %s --check-prefix=ZVBB
 
 ; Integers
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
index a06aa2d02b11..8fc6bb6e2b7a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zfh,+zvfh | FileCheck --check-prefixes=CHECK,RV32 %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh | FileCheck --check-prefixes=CHECK,RV64 %s
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh | FileCheck --check-prefixes=CHECK,RV32 %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh | FileCheck --check-prefixes=CHECK,RV64 %s
 
 ; Integers
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
index 83c235d8e87a..362d8a8f372d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zfh,+zvfh,+zvfbfmin | FileCheck %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+zvfbfmin | FileCheck %s
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zfh,+zvfhmin,+zvfbfmin | FileCheck %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfhmin,+zvfbfmin | FileCheck %s
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvbb,+zfh,+zvfh,+zvfbfmin | FileCheck %s --check-prefix=ZVBB
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvbb,+zfh,+zvfh,+zvfbfmin | FileCheck %s --check-prefix=ZVBB
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh,+zvfbfmin | FileCheck %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh,+zvfbfmin | FileCheck %s
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfhmin,+zvfbfmin | FileCheck %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin | FileCheck %s
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvbb,+zvfh,+zvfbfmin | FileCheck %s --check-prefix=ZVBB
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvbb,+zvfh,+zvfbfmin | FileCheck %s --check-prefix=ZVBB
 
 ; Integers
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfabs-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfabs-sdnode.ll
index c8313c902697..1d8638844af7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfabs-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfabs-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfabs-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfabs-vp.ll
index b9be6eb1fa37..42b71d412fde 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfabs-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfabs-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfadd-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfadd-constrained-sdnode.ll
index 53be153f8ff2..c6c92db62bf6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfadd-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfadd-constrained-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfadd-sdnode.ll
index c3c0958f7096..53a13b511a79 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfadd-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfadd-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfadd-vp.ll
index 4c298ab2b5e6..00ff3456a8e2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfadd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfadd-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfadd.ll b/llvm/test/CodeGen/RISCV/rvv/vfadd.ll
index ae7d7d5d1962..e5f7545eea6f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfadd.ll
@@ -1,15 +1,15 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfhmin,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfhmin,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
-; RUN: sed 's/iXLen/i32/g' %s | not --crash llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfhmin \
+; RUN: sed 's/iXLen/i32/g' %s | not --crash llc -mtriple=riscv32 -mattr=+v,+zvfhmin \
 ; RUN:   -target-abi=ilp32d 2>&1 | FileCheck %s --check-prefixes=ZVFMIN
-; RUN: sed 's/iXLen/i64/g' %s | not --crash llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfhmin \
+; RUN: sed 's/iXLen/i64/g' %s | not --crash llc -mtriple=riscv64 -mattr=+v,+zvfhmin \
 ; RUN:   -target-abi=lp64d 2>&1 | FileCheck %s --check-prefixes=ZVFMIN
 
 ; ZVFMIN: LLVM ERROR: Cannot select: intrinsic %llvm.riscv.vfadd
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfclass-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfclass-sdnode.ll
index c97278480f1a..862a8355d432 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfclass-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfclass-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfclass-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfclass-vp.ll
index be2d576597da..36e1bea1f999 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfclass-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfclass-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 2 x i1> @isnan_nxv2f16(<vscale x 2 x half> %x, <vscale x 2 x i1> %m, i32 zeroext %evl) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfclass.ll b/llvm/test/CodeGen/RISCV/rvv/vfclass.ll
index 7c7cdab19aae..293300bd8dd6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfclass.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfclass.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i16> @llvm.riscv.vfclass.nxv1i16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfcmp-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfcmp-constrained-sdnode.ll
index 1cc9ea029d45..21c5f757e455 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfcmp-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfcmp-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x i1> @llvm.experimental.constrained.fcmp.nxv1f16(<vscale x 1 x half>, <vscale x 1 x half>, metadata, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfcmps-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfcmps-constrained-sdnode.ll
index 9a10359228e5..56284d90a146 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfcmps-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfcmps-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x i1> @llvm.experimental.constrained.fcmps.nxv1f16(<vscale x 1 x half>, <vscale x 1 x half>, metadata, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfcopysign-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfcopysign-sdnode.ll
index c1e1450d0b0a..b28981ff196a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfcopysign-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfcopysign-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-x.ll b/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-x.ll
index bc8440920cd8..87d7885d4410 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-x.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-x.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfcvt.f.x.v.nxv1f16.nxv1i16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-xu.ll b/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-xu.ll
index 9cf47f993ee4..1557e33dd773 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-xu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-xu.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfcvt.f.xu.v.nxv1f16.nxv1i16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfcvt-rtz-x-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfcvt-rtz-x-f.ll
index 1caddaf3feec..3b641ea5bdf4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfcvt-rtz-x-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfcvt-rtz-x-f.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i16> @llvm.riscv.vfcvt.rtz.x.f.v.nxv1i16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfcvt-rtz-xu-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfcvt-rtz-xu-f.ll
index 42e55a5f170e..2fdb4b13acc9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfcvt-rtz-xu-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfcvt-rtz-xu-f.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i16> @llvm.riscv.vfcvt.rtz.xu.f.v.nxv1i16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfcvt-x-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfcvt-x-f.ll
index 582c302dd2a1..2ea0f668dc21 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfcvt-x-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfcvt-x-f.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i16> @llvm.riscv.vfcvt.x.f.v.nxv1i16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfcvt-xu-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfcvt-xu-f.ll
index 708b38b8ed11..a7a742d12dc6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfcvt-xu-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfcvt-xu-f.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i16> @llvm.riscv.vfcvt.xu.f.v.nxv1i16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfdiv-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfdiv-constrained-sdnode.ll
index aa59732e1e1e..ab517de846b0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfdiv-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfdiv-constrained-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfdiv-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfdiv-sdnode.ll
index f7db2be35d72..8d0c3bcf1675 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfdiv-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfdiv-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfdiv-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfdiv-vp.ll
index 0fe6c5dec426..cdc0dc0dbca3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfdiv-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfdiv-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfdiv.ll b/llvm/test/CodeGen/RISCV/rvv/vfdiv.ll
index 3f67c433bcbf..03094db58059 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfdiv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfdiv.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfdiv.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfma-vp-combine.ll b/llvm/test/CodeGen/RISCV/rvv/vfma-vp-combine.ll
index ab8a595dde5d..35bed86d6117 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfma-vp-combine.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfma-vp-combine.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x double> @llvm.vp.fma.nxv1f64(<vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfma-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfma-vp.ll
index f0c74d064016..a65c2fba81ac 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfma-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfma-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmacc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfmacc-vp.ll
index 54855e6152b9..ef583b748b9c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmacc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmacc-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.vp.fma.nxv1f16(<vscale x 1 x half>, <vscale x 1 x half>, <vscale x 1 x half>, <vscale x 1 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmacc.ll b/llvm/test/CodeGen/RISCV/rvv/vfmacc.ll
index 5586b52b64ec..1f0db104df7a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmacc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmacc.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfmacc.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmadd-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfmadd-constrained-sdnode.ll
index dea411348ce5..50bf6eb78044 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmadd-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmadd-constrained-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfmadd-sdnode.ll
index 2df2212c43db..1de8ce51bfe3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmadd-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmadd-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmadd.ll b/llvm/test/CodeGen/RISCV/rvv/vfmadd.ll
index c44690d23f08..fb04888a84de 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmadd.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfmadd.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmax-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfmax-sdnode.ll
index b5604add6d25..8ff7453a0e9a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmax-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmax-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmax-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfmax-vp.ll
index 6e38881b4d60..345a05d98f77 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmax-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmax-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmax.ll b/llvm/test/CodeGen/RISCV/rvv/vfmax.ll
index 458815c98b25..52067e00a54b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmax.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmax.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfmax.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmerge.ll b/llvm/test/CodeGen/RISCV/rvv/vfmerge.ll
index e47c2a47d6c6..e227cff7054f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmerge.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmerge.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vmerge.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmin-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfmin-sdnode.ll
index 9212ddab5b1e..de49aed6e52b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmin-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmin-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmin-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfmin-vp.ll
index f1d6b2100ae9..6f153acda01e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmin-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmin-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmin.ll b/llvm/test/CodeGen/RISCV/rvv/vfmin.ll
index 842c78dce02f..a69bb9e3d6c0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmin.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmin.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfmin.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmsac-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfmsac-vp.ll
index f1d5562131b8..31369b69bee1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmsac-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmsac-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.vp.fma.nxv1f16(<vscale x 1 x half>, <vscale x 1 x half>, <vscale x 1 x half>, <vscale x 1 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmsac.ll b/llvm/test/CodeGen/RISCV/rvv/vfmsac.ll
index 4eac7b63fd88..319c94543540 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmsac.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmsac.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfmsac.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmsub-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfmsub-constrained-sdnode.ll
index 7ec241bf7424..8bd82336de56 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmsub-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmsub-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfhmin,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfmsub-sdnode.ll
index 433b0d1cbdd8..1f99d0e3a5b4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmsub-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmsub-sdnode.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs -early-live-intervals < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs -early-live-intervals < %s | FileCheck %s
 
 ; This tests a mix of vfmsac and vfmsub by using different operand orders to
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmsub.ll b/llvm/test/CodeGen/RISCV/rvv/vfmsub.ll
index 626b40e132c7..23b4479fa8c9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmsub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmsub.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfmsub.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmul-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfmul-constrained-sdnode.ll
index 999b06ba5a57..9a68da58096d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmul-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmul-constrained-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmul-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfmul-sdnode.ll
index 2ab04a45c818..bbacbaa8e5e4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmul-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmul-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmul-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfmul-vp.ll
index 3114fb5d3bfa..7112cf3b7683 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmul-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmul-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmul.ll b/llvm/test/CodeGen/RISCV/rvv/vfmul.ll
index b73d03fe36c7..03084ebc3ae3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmul.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmul.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfmul.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmuladd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfmuladd-vp.ll
index abda6750e5a8..ceefe709fe2a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmuladd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmuladd-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.vp.fmuladd.nxv1f16(<vscale x 1 x half>, <vscale x 1 x half>, <vscale x 1 x half>, <vscale x 1 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmv.f.s.ll b/llvm/test/CodeGen/RISCV/rvv/vfmv.f.s.ll
index af1c378c5681..3779b0ab18d8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmv.f.s.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmv.f.s.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+d,+v,+zfh,+zvfh -target-abi lp64d -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+v,+zfh,+zvfh -target-abi ilp32d -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+v,+zvfh -target-abi lp64d -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+v,+zvfh -target-abi ilp32d -verify-machineinstrs < %s | FileCheck %s
 
 declare half @llvm.riscv.vfmv.f.s.nxv1f16(<vscale x 1 x half>)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmv.s.f.ll b/llvm/test/CodeGen/RISCV/rvv/vfmv.s.f.ll
index 1e863a4adbc2..912dfe499016 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmv.s.f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmv.s.f.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s --check-prefixes=CHECK,RV64
 
 declare <vscale x 1 x half> @llvm.riscv.vfmv.s.f.nxv1f16(<vscale x 1 x half>, half, iXLen)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmv.v.f.ll b/llvm/test/CodeGen/RISCV/rvv/vfmv.v.f.ll
index 237ef11d154b..a3d3443e48c6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmv.v.f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmv.v.f.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfmv.v.f.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-f.ll
index 183ffa8a668a..83250a0f90ea 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-f.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfhmin \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfhmin \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfhmin \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfhmin \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 declare <vscale x 1 x half> @llvm.riscv.vfncvt.f.f.w.nxv1f16.nxv1f32(
   <vscale x 1 x half>,
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-x.ll b/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-x.ll
index aef119faf5f7..81b684978baf 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-x.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-x.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfncvt.f.x.w.nxv1f16.nxv1i32(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-xu.ll b/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-xu.ll
index bc287e4bdef1..697c062c7a71 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-xu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-xu.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfncvt.f.xu.w.nxv1f16.nxv1i32(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfncvt-rod-f-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfncvt-rod-f-f.ll
index f5a019d3152d..c0e5c6991aec 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfncvt-rod-f-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfncvt-rod-f-f.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfncvt.rod.f.f.w.nxv1f16.nxv1f32(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfncvt-rtz-x-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfncvt-rtz-x-f.ll
index 65373bfbdb44..4079e1c055c5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfncvt-rtz-x-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfncvt-rtz-x-f.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vfncvt.rtz.x.f.w.nxv1i8.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfncvt-rtz-xu-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfncvt-rtz-xu-f.ll
index cafffa0d51f5..9ce9fbfa8f19 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfncvt-rtz-xu-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfncvt-rtz-xu-f.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vfncvt.rtz.xu.f.w.nxv1i8.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfncvt-x-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfncvt-x-f.ll
index 334d5eba0300..5831bb33ff90 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfncvt-x-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfncvt-x-f.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vfncvt.x.f.w.nxv1i8.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfncvt-xu-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfncvt-xu-f.ll
index bea99a0e81a3..3ef0a9519719 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfncvt-xu-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfncvt-xu-f.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vfncvt.xu.f.w.nxv1i8.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfneg-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfneg-sdnode.ll
index b4ec691796a7..9f456e97be11 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfneg-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfneg-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfneg-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfneg-vp.ll
index af2668a9b0c5..6fa6c26890c3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfneg-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfneg-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmacc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmacc-vp.ll
index ee3ed603ff6d..3b5cbb685a42 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmacc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmacc-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.vp.fma.nxv1f16(<vscale x 1 x half>, <vscale x 1 x half>, <vscale x 1 x half>, <vscale x 1 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmacc.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmacc.ll
index 01f4715274b6..31df27853cb3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmacc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmacc.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfnmacc.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmadd-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmadd-constrained-sdnode.ll
index 5ec089a2dcac..332ab89b2585 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmadd-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmadd-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfhmin,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmadd-sdnode.ll
index 61acb88b17bd..07c85bc67339 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmadd-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmadd-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 ; This tests a mix of vfnmacc and vfnmadd by using different operand orders to
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmadd.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmadd.ll
index ae4cfef35e61..6f41ed177bea 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmadd.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfnmadd.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmsac-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmsac-vp.ll
index 14dba24daf5f..edeb554bc6d3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmsac-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmsac-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.vp.fma.nxv1f16(<vscale x 1 x half>, <vscale x 1 x half>, <vscale x 1 x half>, <vscale x 1 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmsac.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmsac.ll
index 071f546b4f60..50497d92764a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmsac.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmsac.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfnmsac.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmsub-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmsub-constrained-sdnode.ll
index 286492bce296..8b968017841f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmsub-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmsub-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfhmin,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmsub-sdnode.ll
index 72f64b23f758..a356da80e163 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmsub-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmsub-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 ; This tests a mix of vfnmsac and vfnmsub by using different operand orders to
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmsub.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmsub.ll
index 4922cf40e503..c5d5bb1fe0b3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmsub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmsub.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfnmsub.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfpext-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfpext-constrained-sdnode.ll
index 2375f5def3da..3999b97d6e25 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfpext-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfpext-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x float> @llvm.experimental.constrained.fpext.nxv1f32.nxv1f16(<vscale x 1 x half>, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfpext-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfpext-sdnode.ll
index 2f8b1d501f00..3b2de0185f90 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfpext-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfpext-sdnode.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v,+zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v,+zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x float> @vfpext_nxv1f16_nxv1f32(<vscale x 1 x half> %va) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfpext-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfpext-vp.ll
index 9d10b0209cbe..5962d38b1baa 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfpext-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfpext-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 2 x float> @llvm.vp.fpext.nxv2f32.nxv2f16(<vscale x 2 x half>, <vscale x 2 x i1>, i32)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptoi-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfptoi-constrained-sdnode.ll
index 47f68837cd57..6ebdcec4a403 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptoi-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptoi-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x i1> @llvm.experimental.constrained.fptosi.nxv1i1.nxv1f16(<vscale x 1 x half>, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptoi-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfptoi-sdnode.ll
index 4edaa3825e58..37e14783d187 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptoi-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptoi-sdnode.ll
@@ -1,14 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+zvfbfmin \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v,+zvfbfmin \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v,+zvfbfmin \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFHMIN
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptosi-vp-mask.ll b/llvm/test/CodeGen/RISCV/rvv/vfptosi-vp-mask.ll
index 2e9ceadb9659..b7f2133144e7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptosi-vp-mask.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptosi-vp-mask.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 define <vscale x 2 x i1> @vfptosi_nxv2i1_nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfptosi_nxv2i1_nxv2bf16:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptosi-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfptosi-vp.ll
index f42b603509c2..a8ea06221133 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptosi-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptosi-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 define <vscale x 2 x i7> @vfptosi_v4i7_v4bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfptosi_v4i7_v4bf16:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptoui-vp-mask.ll b/llvm/test/CodeGen/RISCV/rvv/vfptoui-vp-mask.ll
index 2cf158ddbd50..8ac5992bd5eb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptoui-vp-mask.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptoui-vp-mask.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 define <vscale x 2 x i1> @vfptoui_nxv2i1_nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfptoui_nxv2i1_nxv2bf16:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptoui-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfptoui-vp.ll
index 403bc595b9bb..9062d8a6f202 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptoui-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptoui-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 define <vscale x 2 x i7> @vfptoui_v4i7_v4bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfptoui_v4i7_v4bf16:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-constrained-sdnode.ll
index 65e753445ace..566920d577ce 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-constrained-sdnode.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v,+zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v,+zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x float> @llvm.experimental.constrained.fptrunc.nxv1f32.nxv1f64(<vscale x 1 x double>, metadata, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-sdnode.ll
index e930e1fe42f0..dcec2200b130 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-sdnode.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v,+zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v,+zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x half> @vfptrunc_nxv1f32_nxv1f16(<vscale x 1 x float> %va) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-vp.ll
index da16feeddecd..16c8fa728500 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+m,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+m,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v,+m,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v,+m,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 2 x half> @llvm.vp.fptrunc.nxv2f16.nxv2f32(<vscale x 2 x float>, <vscale x 2 x i1>, i32)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfrdiv-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfrdiv-vp.ll
index 876f8d945638..e8688abc63a5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfrdiv-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfrdiv-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.vp.fdiv.nxv1f16(<vscale x 1 x half>, <vscale x 1 x half>, <vscale x 1 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfrdiv.ll b/llvm/test/CodeGen/RISCV/rvv/vfrdiv.ll
index f73e7dce9212..9d29db4b1868 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfrdiv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfrdiv.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfrdiv.nxv1f16.f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfrec7.ll b/llvm/test/CodeGen/RISCV/rvv/vfrec7.ll
index 914b3b33fbe5..98d82144a333 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfrec7.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfrec7.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfrec7.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfredmax.ll b/llvm/test/CodeGen/RISCV/rvv/vfredmax.ll
index 4219abbbaa1d..f1ed95512741 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfredmax.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfredmax.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 4 x half> @llvm.riscv.vfredmax.nxv4f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfredmin.ll b/llvm/test/CodeGen/RISCV/rvv/vfredmin.ll
index 9fcd233fdc14..5dfa5a1f2b20 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfredmin.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfredmin.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 4 x half> @llvm.riscv.vfredmin.nxv4f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfredosum.ll b/llvm/test/CodeGen/RISCV/rvv/vfredosum.ll
index bb489e0f380c..a85850b0c450 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfredosum.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfredosum.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 4 x half> @llvm.riscv.vfredosum.nxv4f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfredusum.ll b/llvm/test/CodeGen/RISCV/rvv/vfredusum.ll
index c1463102c8e6..b3101450493e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfredusum.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfredusum.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 4 x half> @llvm.riscv.vfredusum.nxv4f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfrsqrt7.ll b/llvm/test/CodeGen/RISCV/rvv/vfrsqrt7.ll
index f13fae2614eb..97d6e2924178 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfrsqrt7.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfrsqrt7.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfrsqrt7.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfrsub-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfrsub-vp.ll
index bd941dc1a777..e2864ea30ec7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfrsub-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfrsub-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.vp.fsub.nxv1f16(<vscale x 1 x half>, <vscale x 1 x half>, <vscale x 1 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfrsub.ll b/llvm/test/CodeGen/RISCV/rvv/vfrsub.ll
index 110475341988..c3406c273051 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfrsub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfrsub.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfrsub.nxv1f16.f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfsgnj.ll b/llvm/test/CodeGen/RISCV/rvv/vfsgnj.ll
index e7f0b7ab8a89..73aaf32471db 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfsgnj.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfsgnj.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfsgnj.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfsgnjn.ll b/llvm/test/CodeGen/RISCV/rvv/vfsgnjn.ll
index 2ac48e0b9f9d..06dc5656bb6c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfsgnjn.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfsgnjn.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfsgnjn.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfsgnjx.ll b/llvm/test/CodeGen/RISCV/rvv/vfsgnjx.ll
index b9bbd8982d74..891cda277a44 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfsgnjx.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfsgnjx.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfsgnjx.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfslide1down.ll b/llvm/test/CodeGen/RISCV/rvv/vfslide1down.ll
index 9317a8a21f49..dd036d1e1724 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfslide1down.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfslide1down.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfslide1down.nxv1f16.f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfslide1up.ll b/llvm/test/CodeGen/RISCV/rvv/vfslide1up.ll
index c71cc13566f6..6eead91c1787 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfslide1up.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfslide1up.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfslide1up.nxv1f16.f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfsqrt-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfsqrt-constrained-sdnode.ll
index 9da1e0a576d5..d92db0b5a3a7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfsqrt-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfsqrt-constrained-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfsqrt-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfsqrt-sdnode.ll
index de31a02cd154..a51b0e4efecf 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfsqrt-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfsqrt-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfsqrt-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfsqrt-vp.ll
index 574c2e052630..00542284ebae 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfsqrt-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfsqrt-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfsqrt.ll b/llvm/test/CodeGen/RISCV/rvv/vfsqrt.ll
index 3e3eea9f353c..500a07ad87ed 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfsqrt.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfsqrt.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfsqrt.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfsub-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfsub-constrained-sdnode.ll
index e40427a305f6..9b24b1df0f06 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfsub-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfsub-constrained-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfsub-sdnode.ll
index e56cfd9ee4eb..a2137eaa7a95 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfsub-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfsub-sdnode.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfsub-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfsub-vp.ll
index dd57b65b50f4..02647c1927c2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfsub-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfsub-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zfbfmin,+zvfbfmin,+v \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfsub.ll b/llvm/test/CodeGen/RISCV/rvv/vfsub.ll
index 04590a522366..96c915c6dbf1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfsub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfsub.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfsub.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfwadd-sdnode.ll
index d7f5b109aa7c..68014ff4206f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwadd-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwadd-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x double> @vfwadd_vv_nxv1f64(<vscale x 1 x float> %va, <vscale x 1 x float> %vb) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwadd.ll b/llvm/test/CodeGen/RISCV/rvv/vfwadd.ll
index b42a1fe46e67..d980803cb389 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwadd.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x float> @llvm.riscv.vfwadd.nxv1f32.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwadd.w.ll b/llvm/test/CodeGen/RISCV/rvv/vfwadd.w.ll
index 76246eba9480..8eb2a2c0391b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwadd.w.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwadd.w.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x float> @llvm.riscv.vfwadd.w.nxv1f32.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-f-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-f-f.ll
index 89c5d5a9a68f..4f03188cf380 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-f-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-f-f.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfhmin \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfhmin \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfhmin \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfhmin \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 declare <vscale x 1 x float> @llvm.riscv.vfwcvt.f.f.v.nxv1f32.nxv1f16(
   <vscale x 1 x float>,
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-f-x.ll b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-f-x.ll
index cc8eeaaba256..3c1e07b4a5ef 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-f-x.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-f-x.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfwcvt.f.x.v.nxv1f16.nxv1i8(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-f-xu.ll b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-f-xu.ll
index 841278924d0f..17ea8f50a694 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-f-xu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-f-xu.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.riscv.vfwcvt.f.xu.v.nxv1f16.nxv1i8(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-rtz-x-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-rtz-x-f.ll
index f3a73e4fa363..f9f426cd3c9b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-rtz-x-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-rtz-x-f.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i32> @llvm.riscv.vfwcvt.rtz.x.f.v.nxv1i32.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-rtz-xu-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-rtz-xu-f.ll
index d1d70aeee45e..61d2ad5bf892 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-rtz-xu-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-rtz-xu-f.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i32> @llvm.riscv.vfwcvt.rtz.xu.f.v.nxv1i32.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-x-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-x-f.ll
index 9a80e02bbbbb..8b545585c56d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-x-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-x-f.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i32> @llvm.riscv.vfwcvt.x.f.v.nxv1i32.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-xu-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-xu-f.ll
index 98caaf91ab3c..476e2398c479 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-xu-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-xu-f.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i32> @llvm.riscv.vfwcvt.xu.f.v.nxv1i32.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwmacc.ll b/llvm/test/CodeGen/RISCV/rvv/vfwmacc.ll
index 225ba1c14031..354f16956173 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwmacc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwmacc.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x float> @llvm.riscv.vfwmacc.nxv1f32.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwmsac.ll b/llvm/test/CodeGen/RISCV/rvv/vfwmsac.ll
index 5e3f63b95b2f..bd0d616fa617 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwmsac.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwmsac.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x float> @llvm.riscv.vfwmsac.nxv1f32.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwmul-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfwmul-sdnode.ll
index 6b16171721f1..f00ff4b6d2ce 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwmul-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwmul-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x double> @vfwmul_vv_nxv1f64(<vscale x 1 x float> %va, <vscale x 1 x float> %vb) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwmul.ll b/llvm/test/CodeGen/RISCV/rvv/vfwmul.ll
index bc5759f469ad..dae29efc75bf 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwmul.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwmul.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x float> @llvm.riscv.vfwmul.nxv1f32.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwnmacc.ll b/llvm/test/CodeGen/RISCV/rvv/vfwnmacc.ll
index fc8e15273f08..e1db8cb72276 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwnmacc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwnmacc.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x float> @llvm.riscv.vfwnmacc.nxv1f32.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwnmsac.ll b/llvm/test/CodeGen/RISCV/rvv/vfwnmsac.ll
index b51faf9082c8..5c62112aa9e3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwnmsac.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwnmsac.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x float> @llvm.riscv.vfwnmsac.nxv1f32.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwredosum.ll b/llvm/test/CodeGen/RISCV/rvv/vfwredosum.ll
index dbf7e27d318e..bbb019f2f589 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwredosum.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwredosum.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 2 x float> @llvm.riscv.vfwredosum.nxv2f32.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwredusum.ll b/llvm/test/CodeGen/RISCV/rvv/vfwredusum.ll
index 9710051186c8..05044ef689a9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwredusum.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwredusum.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 2 x float> @llvm.riscv.vfwredusum.nxv2f32.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfwsub-sdnode.ll
index 787de48be7f0..b9f66d5d3082 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwsub-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwsub-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x double> @vfwsub_vv_nxv1f64(<vscale x 1 x float> %va, <vscale x 1 x float> %vb) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwsub.ll b/llvm/test/CodeGen/RISCV/rvv/vfwsub.ll
index 0e3e5f8aabfd..4f263c63e545 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwsub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwsub.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x float> @llvm.riscv.vfwsub.nxv1f32.nxv1f16.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwsub.w.ll b/llvm/test/CodeGen/RISCV/rvv/vfwsub.w.ll
index 90f92226dcdd..fdb48a6f10d3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwsub.w.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwsub.w.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x float> @llvm.riscv.vfwsub.w.nxv1f32.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vitofp-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vitofp-constrained-sdnode.ll
index 90e5f58a603a..f25a27ca2b90 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vitofp-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vitofp-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x half> @llvm.experimental.constrained.sitofp.nxv1f16.nxv1i1(<vscale x 1 x i1>, metadata, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vitofp-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vitofp-sdnode.ll
index f5f8ee91c31c..0f76968485fb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vitofp-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vitofp-sdnode.ll
@@ -1,14 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v,+zvfbfmin \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+zvfbfmin \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v,+zvfbfmin \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+v,+zvfbfmin \
 ; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+zvfbfmin \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+v,+zvfbfmin \
 ; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
 ; RUN:     --check-prefixes=CHECK,ZVFHMIN
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vloxseg-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vloxseg-rv32.ll
index 481505a2095c..4f7286aeeda1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vloxseg-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vloxseg-rv32.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zvfh,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare target("riscv.vector.tuple", <vscale x 1 x i8>, 2) @llvm.riscv.vloxseg2.triscv.vector.tuple_nxv1i8_2t.nxv1i8(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, <vscale x 1 x i8>, i32, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vloxseg-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vloxseg-rv64.ll
index c308512753f2..7b1d545ff9e9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vloxseg-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vloxseg-rv64.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zvfh,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare target("riscv.vector.tuple", <vscale x 1 x i8>, 2) @llvm.riscv.vloxseg2.triscv.vector.tuple_nxv1i8_2t.nxv1i8(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, <vscale x 1 x i8>, i64, i64)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vlseg-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vlseg-rv32.ll
index b96874fe9098..e6a98c90037d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vlseg-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vlseg-rv32.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zvfh,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare target("riscv.vector.tuple", <vscale x 1 x i8>, 2) @llvm.riscv.vlseg2.triscv.vector.tuple_nxv1i8_2t(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, i32, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vlseg-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vlseg-rv64.ll
index d6cbf362e7ec..809ae2d2bebf 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vlseg-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vlseg-rv64.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zvfh,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare target("riscv.vector.tuple", <vscale x 1 x i8>, 2) @llvm.riscv.vlseg2.triscv.vector.tuple_nxv1i8_2t(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, i64, i64)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32.ll
index 0e4915895ef3..b89097b8ff97 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zvfh,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare {target("riscv.vector.tuple", <vscale x 1 x i8>, 2), i32} @llvm.riscv.vlseg2ff.triscv.vector.tuple_nxv1i8_2t(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, i32, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64.ll
index 632fbc1e4431..68acb3beb068 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zvfh,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare {target("riscv.vector.tuple", <vscale x 1 x i8>, 2), i64} @llvm.riscv.vlseg2ff.triscv.vector.tuple_nxv1i8_2t(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, i64, i64)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vlsseg-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vlsseg-rv32.ll
index 4b475dd96e00..a87d51692227 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vlsseg-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vlsseg-rv32.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zvfh,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare target("riscv.vector.tuple", <vscale x 1 x i8>, 2) @llvm.riscv.vlsseg2.triscv.vector.tuple_nxv1i8_2t(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, i32, i32, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vlsseg-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vlsseg-rv64.ll
index 6cc95979eb13..7b5421fba3dc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vlsseg-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vlsseg-rv64.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zvfh,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare target("riscv.vector.tuple", <vscale x 1 x i8>, 2) @llvm.riscv.vlsseg2.triscv.vector.tuple_nxv1i8_2t(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, i64, i64, i64)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vluxseg-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vluxseg-rv32.ll
index 10bfdec0e2c9..d1ca40bcc0db 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vluxseg-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vluxseg-rv32.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zvfh,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare target("riscv.vector.tuple", <vscale x 1 x i8>, 2) @llvm.riscv.vluxseg2.triscv.vector.tuple_nxv1i8_2t.nxv1i8(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, <vscale x 1 x i8>, i32, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vluxseg-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vluxseg-rv64.ll
index 28f70ce08bfe..3b9db2655e03 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vluxseg-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vluxseg-rv64.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zvfh,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare target("riscv.vector.tuple", <vscale x 1 x i8>, 2) @llvm.riscv.vluxseg2.triscv.vector.tuple_nxv1i8_2t.nxv1i8(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, <vscale x 1 x i8>, i64, i64)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmfeq.ll b/llvm/test/CodeGen/RISCV/rvv/vmfeq.ll
index 2e5b67c93fce..9ca78c872bef 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmfeq.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmfeq.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i1> @llvm.riscv.vmfeq.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmfge.ll b/llvm/test/CodeGen/RISCV/rvv/vmfge.ll
index b5ca47707c8a..7cf18a701581 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmfge.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmfge.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i1> @llvm.riscv.vmfge.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmfgt.ll b/llvm/test/CodeGen/RISCV/rvv/vmfgt.ll
index 971249d38d1b..b78f2da4ae25 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmfgt.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmfgt.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i1> @llvm.riscv.vmfgt.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmfle.ll b/llvm/test/CodeGen/RISCV/rvv/vmfle.ll
index f19a181a365a..940e4d043f63 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmfle.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmfle.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i1> @llvm.riscv.vmfle.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmflt.ll b/llvm/test/CodeGen/RISCV/rvv/vmflt.ll
index 0a0464221933..10ddfb8f014e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmflt.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmflt.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i1> @llvm.riscv.vmflt.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmfne.ll b/llvm/test/CodeGen/RISCV/rvv/vmfne.ll
index 520099247e0f..4d8a95de1d3d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmfne.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmfne.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32d | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvfh \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d | FileCheck %s
 
 declare <vscale x 1 x i1> @llvm.riscv.vmfne.nxv1f16(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vpgather-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vpgather-sdnode.ll
index 84c8321b5b93..34f8f35ee98c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vpgather-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vpgather-sdnode.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+zvfbfmin,+v \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+zvfbfmin,+v \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64
 
 declare <vscale x 1 x i8> @llvm.vp.gather.nxv1i8.nxv1p0(<vscale x 1 x ptr>, <vscale x 1 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vpload.ll b/llvm/test/CodeGen/RISCV/rvv/vpload.ll
index 0a98b672fb19..5683a7b75885 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vpload.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vpload.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+zvfbfmin,+v \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+zvfbfmin,+v \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.vp.load.nxv1i8.p0(ptr, <vscale x 1 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vpscatter-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vpscatter-sdnode.ll
index 0028f3035c27..329f97da64ea 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vpscatter-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vpscatter-sdnode.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v,+m \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zvfbfmin,+v,+m \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v,+m \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zvfbfmin,+v,+m \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+zvfbfmin,+v,+m \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+zvfbfmin,+v,+m \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+zvfbfmin,+v,+m \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+zvfbfmin,+v,+m \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64
 
 declare void @llvm.vp.scatter.nxv1i8.nxv1p0(<vscale x 1 x i8>, <vscale x 1 x ptr>, <vscale x 1 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vpstore.ll b/llvm/test/CodeGen/RISCV/rvv/vpstore.ll
index d935e52149d2..7168b07e8197 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vpstore.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vpstore.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+zvfbfmin,+v \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfhmin,+zvfbfmin,+v \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+zvfbfmin,+v \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfhmin,+zvfbfmin,+v \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare void @llvm.vp.store.nxv1i8.p0(<vscale x 1 x i8>, ptr, <vscale x 1 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll
index f3ccf74019bb..13d1ac508847 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare half @llvm.vp.reduce.fadd.nxv1f16(half, <vscale x 1 x half>, <vscale x 1 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll b/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll
index d1e947e2f336..f386fd9cd3ae 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfh \
 ; RUN:   -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfh \
 ; RUN:   -verify-machineinstrs | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vrgatherei16.vv.nxv1i8(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
index ee0617c93148..b7e2c92350a2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x i1> @llvm.vp.select.nxv1i1(<vscale x 1 x i1>, <vscale x 1 x i1>, <vscale x 1 x i1>, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsitofp-vp-mask.ll b/llvm/test/CodeGen/RISCV/rvv/vsitofp-vp-mask.ll
index d1c2cf325bec..ec16e58f6e57 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsitofp-vp-mask.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsitofp-vp-mask.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s
 
 define <vscale x 2 x bfloat> @vsitofp_nxv2bf16_nxv2i1(<vscale x 2 x i1> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vsitofp_nxv2bf16_nxv2i1:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsitofp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vsitofp-vp.ll
index d163988b3d41..8b6e437fbc0a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsitofp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsitofp-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 define <vscale x 2 x bfloat> @vsitofp_nxv2bf16_nxv2i7(<vscale x 2 x i7> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vsitofp_nxv2bf16_nxv2i7:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll b/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll
index fc26ac25fe08..2be187c50af2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfh \
 ; RUN:   -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfh \
 ; RUN:   -verify-machineinstrs | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vslidedown.nxv1i8(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vslideup.ll b/llvm/test/CodeGen/RISCV/rvv/vslideup.ll
index 4880bf2bc66d..1e3ede7fee9c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vslideup.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vslideup.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfh \
 ; RUN:   -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zfh,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfh \
 ; RUN:   -verify-machineinstrs | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vslideup.nxv1i8(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsoxseg-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vsoxseg-rv32.ll
index c24895a0e638..6b54ce4974f3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsoxseg-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsoxseg-rv32.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zvfh,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare void @llvm.riscv.vsoxseg2.triscv.vector.tuple_nxv1i8_2t.nxv1i8(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, <vscale x 1 x i8>, i32, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsoxseg-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vsoxseg-rv64.ll
index c8e7c4375405..70fb9c2b348d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsoxseg-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsoxseg-rv64.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zvfh,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare void @llvm.riscv.vsoxseg2.triscv.vector.tuple_nxv1i8_2t.nxv1i8(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, <vscale x 1 x i8>, i64, i64)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsseg-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vsseg-rv32.ll
index 330ec59d3459..7b80d45a924d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsseg-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsseg-rv32.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zvfh,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare void @llvm.riscv.vsseg2.triscv.vector.tuple_nxv1i8_2t(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, i32, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsseg-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vsseg-rv64.ll
index 877eeeaf1003..6ce326be23ee 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsseg-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsseg-rv64.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zvfh,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare void @llvm.riscv.vsseg2.triscv.vector.tuple_nxv1i8_2t(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, i64, i64)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssseg-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vssseg-rv32.ll
index df443d6f4d93..a0a583c046c4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssseg-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssseg-rv32.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zfh,+zvfh \
+; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zvfh \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare void @llvm.riscv.vssseg2.triscv.vector.tuple_nxv1i8_2t(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, i32, i32, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssseg-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vssseg-rv64.ll
index dd6faad09f49..bdd809841d2d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssseg-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssseg-rv64.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zfh,+zvfh \
+; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zvfh \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare void @llvm.riscv.vssseg2.triscv.vector.tuple_nxv1i8_2t(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, i64, i64, i64)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsuxseg-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vsuxseg-rv32.ll
index 9119d42ba0ae..316c7ccb7e41 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsuxseg-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsuxseg-rv32.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv32 -mattr=+zve64d,+f,+d,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare void @llvm.riscv.vsuxseg2.triscv.vector.tuple_nxv1i8_2t.nxv1i8(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, <vscale x 1 x i8>, i32, i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsuxseg-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vsuxseg-rv64.ll
index 82698e6da2ab..22be2ebca8fd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsuxseg-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsuxseg-rv64.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zfh,+zvfh,+zvfbfmin \
+; RUN: llc -mtriple=riscv64 -mattr=+zve64d,+f,+d,+zvfh,+zvfbfmin \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare void @llvm.riscv.vsuxseg2.triscv.vector.tuple_nxv1i8_2t.nxv1i8(target("riscv.vector.tuple", <vscale x 1 x i8>, 2), ptr, <vscale x 1 x i8>, i64, i64)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vuitofp-vp-mask.ll b/llvm/test/CodeGen/RISCV/rvv/vuitofp-vp-mask.ll
index 5426102efc73..616dc697b284 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vuitofp-vp-mask.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vuitofp-vp-mask.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s
 
 define <vscale x 2 x bfloat> @vuitofp_nxv2bf16_nxv2i1(<vscale x 2 x i1> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vuitofp_nxv2bf16_nxv2i1:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vuitofp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vuitofp-vp.ll
index 7c96a9e9e10f..499bd4aa667c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vuitofp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vuitofp-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zfh,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zfh,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfh,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+zvfhmin,+zvfbfmin < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 define <vscale x 2 x bfloat> @vuitofp_nxv2bf16_nxv2i7(<vscale x 2 x i7> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vuitofp_nxv2bf16_nxv2i7:
-- 
GitLab


From 6ffefbbc2593d82117924c5c18d2a4ed7689ea3f Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Wed, 30 Oct 2024 16:02:34 -0700
Subject: [PATCH 660/693] [LinkerWrapper] Remove handling of special bitcode
 flags (#114298)

Summary:
These flags were used in the very early days while we were trying to
port stuff. Now that we just pass bitcode to the device link job it
can be easily replaced by `-Xoffload-linker foo.bc`.
---
 clang/docs/ClangLinkerWrapper.rst              |  3 ---
 .../ClangLinkerWrapper.cpp                     | 18 ------------------
 .../clang-linker-wrapper/LinkerWrapperOpts.td  | 10 ----------
 3 files changed, 31 deletions(-)

diff --git a/clang/docs/ClangLinkerWrapper.rst b/clang/docs/ClangLinkerWrapper.rst
index 99352863b477..e69cdba434c9 100644
--- a/clang/docs/ClangLinkerWrapper.rst
+++ b/clang/docs/ClangLinkerWrapper.rst
@@ -30,14 +30,11 @@ only for the linker wrapper will be forwarded to the wrapped linker job.
   USAGE: clang-linker-wrapper [options] -- <options to passed to the linker>
 
   OPTIONS:
-    --bitcode-library=<kind>-<triple>-<arch>=<path>
-                           Extra bitcode library to link
     --cuda-path=<dir>      Set the system CUDA path
     --device-debug         Use debugging
     --device-linker=<value> or <triple>=<value>
                            Arguments to pass to the device linker invocation
     --dry-run              Print program arguments without running
-    --embed-bitcode        Embed linked bitcode in the module
     --help-hidden          Display all available options
     --help                 Display available options (--help-hidden for more)
     --host-triple=<triple> Triple to use for the host compilation
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 561b73c73ad7..fc985bfe1d6c 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -600,17 +600,6 @@ Expected<StringRef> clang(ArrayRef<StringRef> InputFiles, const ArgList &Args) {
   for (StringRef Arg : Args.getAllArgValues(OPT_compiler_arg_EQ))
     CmdArgs.push_back(Args.MakeArgString(Arg));
 
-  for (StringRef Arg : Args.getAllArgValues(OPT_builtin_bitcode_EQ)) {
-    if (llvm::Triple(Arg.split('=').first) == Triple)
-      CmdArgs.append({"-Xclang", "-mlink-builtin-bitcode", "-Xclang",
-                      Args.MakeArgString(Arg.split('=').second)});
-  }
-
-  // The OpenMPOpt pass can introduce new calls and is expensive, we do
-  // not want this when running CodeGen through clang.
-  if (Args.hasArg(OPT_clang_backend) || Args.hasArg(OPT_builtin_bitcode_EQ))
-    CmdArgs.append({"-mllvm", "-openmp-opt-disable"});
-
   if (Error Err = executeCommands(*ClangPath, CmdArgs))
     return std::move(Err);
 
@@ -1362,13 +1351,6 @@ getDeviceInput(const ArgList &Args) {
     }
   }
 
-  for (StringRef Library : Args.getAllArgValues(OPT_bitcode_library_EQ)) {
-    auto FileOrErr = getInputBitcodeLibrary(Library);
-    if (!FileOrErr)
-      return FileOrErr.takeError();
-    InputFiles[*FileOrErr].push_back(std::move(*FileOrErr));
-  }
-
   SmallVector<SmallVector<OffloadFile>> InputsForTarget;
   for (auto &[ID, Input] : InputFiles)
     InputsForTarget.emplace_back(std::move(Input));
diff --git a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
index a3e819938004..57d918db0a73 100644
--- a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
+++ b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
@@ -22,22 +22,12 @@ def host_triple_EQ : Joined<["--"], "host-triple=">,
 def opt_level : Joined<["--"], "opt-level=">,
   Flags<[WrapperOnlyOption]>, MetaVarName<"<O0, O1, O2, or O3>">,
   HelpText<"Optimization level for LTO">;
-def bitcode_library_EQ : Joined<["--"], "bitcode-library=">,
-  Flags<[WrapperOnlyOption]>, MetaVarName<"<kind>-<triple>-<arch>=<path>">,
-  HelpText<"Extra bitcode library to link">;
-def builtin_bitcode_EQ : Joined<["--"], "builtin-bitcode=">,
-  Flags<[WrapperOnlyOption]>, MetaVarName<"<triple>=<path>">,
-  HelpText<"Perform a special internalizing link on the bitcode file. "
-           "This is necessary for some vendor libraries to be linked correctly">;
 def device_linker_args_EQ : Joined<["--"], "device-linker=">,
   Flags<[WrapperOnlyOption]>, MetaVarName<"<value> or <triple>=<value>">,
   HelpText<"Arguments to pass to the device linker invocation">;
 def device_compiler_args_EQ : Joined<["--"], "device-compiler=">,
   Flags<[WrapperOnlyOption]>, MetaVarName<"<value> or <triple>=<value>">,
   HelpText<"Arguments to pass to the device compiler invocation">;
-def clang_backend : Flag<["--"], "clang-backend">,
-  Flags<[WrapperOnlyOption]>,
-  HelpText<"Run the backend using clang rather than the LTO backend">;
 def dry_run : Flag<["--"], "dry-run">,
   Flags<[WrapperOnlyOption]>,
   HelpText<"Print program arguments without running">;
-- 
GitLab


From a39fb30a4928eef0619bcfaa709645309dff32f9 Mon Sep 17 00:00:00 2001
From: lntue <lntue@google.com>
Date: Wed, 30 Oct 2024 19:08:47 -0400
Subject: [PATCH 661/693] [libc] Fix usage of std::nullptr_t in LibcTest.h.
 (#114321)

---
 libc/test/UnitTest/LibcTest.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/test/UnitTest/LibcTest.h b/libc/test/UnitTest/LibcTest.h
index 1707c3c0fdcf..b4e3819ea958 100644
--- a/libc/test/UnitTest/LibcTest.h
+++ b/libc/test/UnitTest/LibcTest.h
@@ -165,7 +165,7 @@ protected:
   // Helper to allow macro invocations like `ASSERT_EQ(foo, nullptr)`.
   template <typename ValType,
             cpp::enable_if_t<cpp::is_pointer_v<ValType>, ValType> = nullptr>
-  bool test(TestCond Cond, ValType LHS, std::nullptr_t, const char *LHSStr,
+  bool test(TestCond Cond, ValType LHS, cpp::nullptr_t, const char *LHSStr,
             const char *RHSStr, internal::Location Loc) {
     return test(Cond, LHS, static_cast<ValType>(nullptr), LHSStr, RHSStr, Loc);
   }
-- 
GitLab


From 847f4ef21b4a953bb6dd6477791e8d95b6db2509 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 30 Oct 2024 16:22:23 -0700
Subject: [PATCH 662/693] [X86] Use getAllOnesConstant instead of
 getConstant(-1). NFC (#114299)

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 34bc5d76c15c..22cba69af41f 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -49321,7 +49321,7 @@ static SDValue combineVectorShiftImm(SDNode *N, SelectionDAG &DAG,
   if (!LogicalShift && ISD::isBuildVectorAllOnes(N0.getNode()))
     // N0 is all ones or undef. We guarantee that the bits shifted into the
     // result are all ones, not undef.
-    return DAG.getConstant(-1, SDLoc(N), VT);
+    return DAG.getAllOnesConstant(SDLoc(N), VT);
 
   auto MergeShifts = [&](SDValue X, uint64_t Amt0, uint64_t Amt1) {
     unsigned NewShiftVal = Amt0 + Amt1;
-- 
GitLab


From c1858cdd1dafd29f56cf6274cc03c1567d249daa Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 30 Oct 2024 16:32:38 -0700
Subject: [PATCH 663/693] [clang-link-wrapper] Fix a warning

This patch fixes:

  clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp:221:23:
  error: unused function 'getInputBitcodeLibrary'
  [-Werror,-Wunused-function]
---
 .../ClangLinkerWrapper.cpp                    | 27 -------------------
 1 file changed, 27 deletions(-)

diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index fc985bfe1d6c..ebafd7eb7774 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -216,33 +216,6 @@ void printCommands(ArrayRef<StringRef> CmdArgs) {
   exit(EXIT_FAILURE);
 }
 
-/// Create an extra user-specified \p OffloadFile.
-/// TODO: We should find a way to wrap these as libraries instead.
-Expected<OffloadFile> getInputBitcodeLibrary(StringRef Input) {
-  auto [Device, Path] = StringRef(Input).split('=');
-  auto [String, Arch] = Device.rsplit('-');
-  auto [Kind, Triple] = String.split('-');
-
-  llvm::ErrorOr<std::unique_ptr<MemoryBuffer>> ImageOrError =
-      llvm::MemoryBuffer::getFileOrSTDIN(Path);
-  if (std::error_code EC = ImageOrError.getError())
-    return createFileError(Path, EC);
-
-  OffloadingImage Image{};
-  Image.TheImageKind = IMG_Bitcode;
-  Image.TheOffloadKind = getOffloadKind(Kind);
-  Image.StringData["triple"] = Triple;
-  Image.StringData["arch"] = Arch;
-  Image.Image = std::move(*ImageOrError);
-
-  std::unique_ptr<MemoryBuffer> Binary =
-      MemoryBuffer::getMemBufferCopy(OffloadBinary::write(Image));
-  auto NewBinaryOrErr = OffloadBinary::create(*Binary);
-  if (!NewBinaryOrErr)
-    return NewBinaryOrErr.takeError();
-  return OffloadFile(std::move(*NewBinaryOrErr), std::move(Binary));
-}
-
 std::string getMainExecutable(const char *Name) {
   void *Ptr = (void *)(intptr_t)&getMainExecutable;
   auto COWPath = sys::fs::getMainExecutable(Name, Ptr);
-- 
GitLab


From 51628faa0122d61b7725ec869fa5c0be1d739edd Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 30 Oct 2024 16:33:18 -0700
Subject: [PATCH 664/693] [RISCV] Sink hasPostISelHook = 1 for vector pseudos
 into the subclasses that set HasRoundModeOp. NFC (#114294)

---
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    | 41 +++++++++++--------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 19557d424d1b..57460425e338 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -1074,6 +1074,7 @@ class VPseudoUnaryNoMaskRoundingMode<DAGOperand RetClass,
   let HasVecPolicyOp = 1;
   let HasRoundModeOp = 1;
   let UsesVXRM = 0;
+  let hasPostISelHook = 1;
 }
 
 class VPseudoUnaryMask<VReg RetClass,
@@ -1115,6 +1116,7 @@ class VPseudoUnaryMaskRoundingMode<VReg RetClass,
   let UsesMaskPolicy = 1;
   let HasRoundModeOp = 1;
   let UsesVXRM = 0;
+  let hasPostISelHook = 1;
 }
 
 class VPseudoUnaryMask_NoExcept<VReg RetClass,
@@ -1226,6 +1228,7 @@ class VPseudoBinaryNoMaskRoundingMode<VReg RetClass,
   let HasVecPolicyOp = 1;
   let HasRoundModeOp = 1;
   let UsesVXRM = UsesVXRM_;
+  let hasPostISelHook = !not(UsesVXRM_);
 }
 
 class VPseudoBinaryMaskPolicyRoundingMode<VReg RetClass,
@@ -1250,6 +1253,7 @@ class VPseudoBinaryMaskPolicyRoundingMode<VReg RetClass,
   let UsesMaskPolicy = 1;
   let HasRoundModeOp = 1;
   let UsesVXRM = UsesVXRM_;
+  let hasPostISelHook = !not(UsesVXRM_);
 }
 
 // Special version of VPseudoBinaryNoMask where we pretend the first source is
@@ -1297,6 +1301,7 @@ class VPseudoTiedBinaryNoMaskRoundingMode<VReg RetClass,
   let IsTiedPseudo = 1;
   let HasRoundModeOp = 1;
   let UsesVXRM = 0;
+  let hasPostISelHook = 1;
 }
 
 class VPseudoIStoreNoMask<VReg StClass, VReg IdxClass, int EEW, bits<3> LMUL,
@@ -1384,6 +1389,7 @@ class VPseudoTernaryMaskPolicyRoundingMode<VReg RetClass,
   let HasVecPolicyOp = 1;
   let HasRoundModeOp = 1;
   let UsesVXRM = 0;
+  let hasPostISelHook = 1;
 }
 
 // Like VPseudoBinaryMaskPolicy, but output can be V0 and there is no policy.
@@ -1454,6 +1460,7 @@ class VPseudoTiedBinaryMaskRoundingMode<VReg RetClass,
   let IsTiedPseudo = 1;
   let HasRoundModeOp = 1;
   let UsesVXRM = 0;
+  let hasPostISelHook = 1;
 }
 
 class VPseudoBinaryCarry<VReg RetClass,
@@ -1554,6 +1561,7 @@ class VPseudoTernaryNoMaskWithPolicyRoundingMode<VReg RetClass,
   let HasSEWOp = 1;
   let HasRoundModeOp = 1;
   let UsesVXRM = 0;
+  let hasPostISelHook = 1;
 }
 
 class VPseudoUSSegLoadNoMask<VReg RetClass,
@@ -6352,7 +6360,7 @@ let Predicates = [HasVInstructionsAnyF] in {
 //===----------------------------------------------------------------------===//
 // 13.2. Vector Single-Width Floating-Point Add/Subtract Instructions
 //===----------------------------------------------------------------------===//
-let mayRaiseFPException = true, hasPostISelHook = 1 in {
+let mayRaiseFPException = true in {
 defm PseudoVFADD  : VPseudoVALU_VV_VF_RM;
 defm PseudoVFSUB  : VPseudoVALU_VV_VF_RM;
 defm PseudoVFRSUB : VPseudoVALU_VF_RM;
@@ -6361,7 +6369,7 @@ defm PseudoVFRSUB : VPseudoVALU_VF_RM;
 //===----------------------------------------------------------------------===//
 // 13.3. Vector Widening Floating-Point Add/Subtract Instructions
 //===----------------------------------------------------------------------===//
-let mayRaiseFPException = true, hasSideEffects = 0, hasPostISelHook = 1 in {
+let mayRaiseFPException = true, hasSideEffects = 0 in {
 defm PseudoVFWADD : VPseudoVFWALU_VV_VF_RM;
 defm PseudoVFWSUB : VPseudoVFWALU_VV_VF_RM;
 defm PseudoVFWADD : VPseudoVFWALU_WV_WF_RM;
@@ -6371,7 +6379,7 @@ defm PseudoVFWSUB : VPseudoVFWALU_WV_WF_RM;
 //===----------------------------------------------------------------------===//
 // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
 //===----------------------------------------------------------------------===//
-let mayRaiseFPException = true, hasSideEffects = 0, hasPostISelHook = 1 in {
+let mayRaiseFPException = true, hasSideEffects = 0 in {
 defm PseudoVFMUL  : VPseudoVFMUL_VV_VF_RM;
 defm PseudoVFDIV  : VPseudoVFDIV_VV_VF_RM;
 defm PseudoVFRDIV : VPseudoVFRDIV_VF_RM;
@@ -6380,14 +6388,14 @@ defm PseudoVFRDIV : VPseudoVFRDIV_VF_RM;
 //===----------------------------------------------------------------------===//
 // 13.5. Vector Widening Floating-Point Multiply
 //===----------------------------------------------------------------------===//
-let mayRaiseFPException = true, hasSideEffects = 0, hasPostISelHook = 1 in {
+let mayRaiseFPException = true, hasSideEffects = 0 in {
 defm PseudoVFWMUL : VPseudoVWMUL_VV_VF_RM;
 }
 
 //===----------------------------------------------------------------------===//
 // 13.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions
 //===----------------------------------------------------------------------===//
-let mayRaiseFPException = true, hasSideEffects = 0, hasPostISelHook = 1 in {
+let mayRaiseFPException = true, hasSideEffects = 0 in {
 defm PseudoVFMACC  : VPseudoVMAC_VV_VF_AAXA_RM;
 defm PseudoVFNMACC : VPseudoVMAC_VV_VF_AAXA_RM;
 defm PseudoVFMSAC  : VPseudoVMAC_VV_VF_AAXA_RM;
@@ -6401,7 +6409,7 @@ defm PseudoVFNMSUB : VPseudoVMAC_VV_VF_AAXA_RM;
 //===----------------------------------------------------------------------===//
 // 13.7. Vector Widening Floating-Point Fused Multiply-Add Instructions
 //===----------------------------------------------------------------------===//
-let mayRaiseFPException = true, hasSideEffects = 0, hasPostISelHook = 1 in {
+let mayRaiseFPException = true, hasSideEffects = 0 in {
 defm PseudoVFWMACC  : VPseudoVWMAC_VV_VF_RM;
 defm PseudoVFWNMACC : VPseudoVWMAC_VV_VF_RM;
 defm PseudoVFWMSAC  : VPseudoVWMAC_VV_VF_RM;
@@ -6413,7 +6421,7 @@ defm PseudoVFWMACCBF16  : VPseudoVWMAC_VV_VF_BF_RM;
 //===----------------------------------------------------------------------===//
 // 13.8. Vector Floating-Point Square-Root Instruction
 //===----------------------------------------------------------------------===//
-let mayRaiseFPException = true, hasSideEffects = 0, hasPostISelHook = 1 in
+let mayRaiseFPException = true, hasSideEffects = 0 in
 defm PseudoVFSQRT : VPseudoVSQR_V_RM;
 
 //===----------------------------------------------------------------------===//
@@ -6425,7 +6433,7 @@ defm PseudoVFRSQRT7 : VPseudoVRCP_V;
 //===----------------------------------------------------------------------===//
 // 13.10. Vector Floating-Point Reciprocal Estimate Instruction
 //===----------------------------------------------------------------------===//
-let mayRaiseFPException = true, hasSideEffects = 0, hasPostISelHook = 1 in
+let mayRaiseFPException = true, hasSideEffects = 0 in
 defm PseudoVFREC7 : VPseudoVRCP_V_RM;
 
 //===----------------------------------------------------------------------===//
@@ -6475,7 +6483,7 @@ defm PseudoVFMV_V : VPseudoVMV_F;
 // 13.17. Single-Width Floating-Point/Integer Type-Convert Instructions
 //===----------------------------------------------------------------------===//
 let mayRaiseFPException = true in {
-let hasSideEffects = 0, hasPostISelHook = 1 in {
+let hasSideEffects = 0 in {
 defm PseudoVFCVT_XU_F : VPseudoVCVTI_V_RM;
 defm PseudoVFCVT_X_F : VPseudoVCVTI_V_RM;
 }
@@ -6484,7 +6492,7 @@ defm PseudoVFCVT_RTZ_XU_F : VPseudoVCVTI_V;
 defm PseudoVFCVT_RTZ_X_F : VPseudoVCVTI_V;
 
 defm PseudoVFROUND_NOEXCEPT : VPseudoVFROUND_NOEXCEPT_V;
-let hasSideEffects = 0, hasPostISelHook = 1 in {
+let hasSideEffects = 0 in {
 defm PseudoVFCVT_F_XU : VPseudoVCVTF_V_RM;
 defm PseudoVFCVT_F_X : VPseudoVCVTF_V_RM;
 }
@@ -6494,7 +6502,7 @@ defm PseudoVFCVT_F_X : VPseudoVCVTF_V_RM;
 // 13.18. Widening Floating-Point/Integer Type-Convert Instructions
 //===----------------------------------------------------------------------===//
 let mayRaiseFPException = true in {
-let hasSideEffects = 0, hasPostISelHook = 1 in {
+let hasSideEffects = 0 in {
 defm PseudoVFWCVT_XU_F     : VPseudoVWCVTI_V_RM;
 defm PseudoVFWCVT_X_F      : VPseudoVWCVTI_V_RM;
 }
@@ -6513,7 +6521,7 @@ defm PseudoVFWCVTBF16_F_F :  VPseudoVWCVTD_V;
 // 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions
 //===----------------------------------------------------------------------===//
 let mayRaiseFPException = true in {
-let hasSideEffects = 0, hasPostISelHook = 1 in {
+let hasSideEffects = 0 in {
 defm PseudoVFNCVT_XU_F     : VPseudoVNCVTI_W_RM;
 defm PseudoVFNCVT_X_F      : VPseudoVNCVTI_W_RM;
 }
@@ -6521,12 +6529,12 @@ defm PseudoVFNCVT_X_F      : VPseudoVNCVTI_W_RM;
 defm PseudoVFNCVT_RTZ_XU_F : VPseudoVNCVTI_W;
 defm PseudoVFNCVT_RTZ_X_F  : VPseudoVNCVTI_W;
 
-let hasSideEffects = 0, hasPostISelHook = 1 in {
+let hasSideEffects = 0 in {
 defm PseudoVFNCVT_F_XU     : VPseudoVNCVTF_W_RM;
 defm PseudoVFNCVT_F_X      : VPseudoVNCVTF_W_RM;
 }
 
-let hasSideEffects = 0, hasPostISelHook = 1 in {
+let hasSideEffects = 0 in {
 defm PseudoVFNCVT_F_F      : VPseudoVNCVTD_W_RM;
 defm PseudoVFNCVTBF16_F_F :  VPseudoVNCVTD_W_RM;
 }
@@ -6565,7 +6573,7 @@ let Predicates = [HasVInstructionsAnyF] in {
 //===----------------------------------------------------------------------===//
 // 14.3. Vector Single-Width Floating-Point Reduction Instructions
 //===----------------------------------------------------------------------===//
-let mayRaiseFPException = true, hasSideEffects = 0, hasPostISelHook = 1 in {
+let mayRaiseFPException = true, hasSideEffects = 0 in {
 defm PseudoVFREDOSUM : VPseudoVFREDO_VS_RM;
 defm PseudoVFREDUSUM : VPseudoVFRED_VS_RM;
 }
@@ -6577,8 +6585,7 @@ defm PseudoVFREDMAX  : VPseudoVFREDMINMAX_VS;
 //===----------------------------------------------------------------------===//
 // 14.4. Vector Widening Floating-Point Reduction Instructions
 //===----------------------------------------------------------------------===//
-let IsRVVWideningReduction = 1, hasSideEffects = 0, mayRaiseFPException = true,
-    hasPostISelHook = 1 in {
+let IsRVVWideningReduction = 1, hasSideEffects = 0, mayRaiseFPException = true in {
 defm PseudoVFWREDUSUM  : VPseudoVFWRED_VS_RM;
 defm PseudoVFWREDOSUM  : VPseudoVFWREDO_VS_RM;
 }
-- 
GitLab


From b01e2a8b5620466c3b80cc6f049efbc90b9d103a Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth@google.com>
Date: Wed, 30 Oct 2024 16:56:30 -0700
Subject: [PATCH 665/693] [llvm] Allow always dropping all llvm.type.test
 sequences

Currently, the `DropTypeTests` parameter only fully works with phi nodes
and llvm.assume instructions. However, we'd like CFI to work in
conjunction with FatLTO, in so far as the bitcode section should be able
to contain the CFI instrumentation, while any incompatible bits are
dropped when compiling the object code.

To do that, we need to drop the llvm.type.test instructions everywhere,
and not just their uses in phi nodes. This patch updates the
LowerTypeTest pass so that uses are removed, and replaced with `true` in
all cases, and not just in phi nodes.

Addressing this will allow us to fix #112053 by modifying the FatLTO
pipeline.

Reviewers: pcc, nikic

Reviewed By: pcc

Pull Request: https://github.com/llvm/llvm-project/pull/112787
---
 clang/lib/CodeGen/BackendUtil.cpp             |  7 ++--
 .../llvm/Transforms/IPO/LowerTypeTests.h      | 13 +++++-
 llvm/lib/Passes/PassBuilderPipelines.cpp      | 15 ++++---
 llvm/lib/Transforms/IPO/LowerTypeTests.cpp    | 41 ++++++++++++-------
 .../LowerTypeTests/drop_type_test.ll          | 22 ++++++++++
 .../LowerTypeTests/drop_type_test_phi.ll      |  2 +-
 6 files changed, 75 insertions(+), 25 deletions(-)
 create mode 100644 llvm/test/Transforms/LowerTypeTests/drop_type_test.ll

diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index f01813080751..ae33554a66b6 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -1013,9 +1013,10 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
     if (IsThinLTOPostLink)
       PB.registerPipelineStartEPCallback(
           [](ModulePassManager &MPM, OptimizationLevel Level) {
-            MPM.addPass(LowerTypeTestsPass(/*ExportSummary=*/nullptr,
-                                           /*ImportSummary=*/nullptr,
-                                           /*DropTypeTests=*/true));
+            MPM.addPass(LowerTypeTestsPass(
+                /*ExportSummary=*/nullptr,
+                /*ImportSummary=*/nullptr,
+                /*DropTypeTests=*/lowertypetests::DropTestKind::Assume));
           });
 
     // Register callbacks to schedule sanitizer passes at the appropriate part
diff --git a/llvm/include/llvm/Transforms/IPO/LowerTypeTests.h b/llvm/include/llvm/Transforms/IPO/LowerTypeTests.h
index eb682c437b94..02adcd8bfd45 100644
--- a/llvm/include/llvm/Transforms/IPO/LowerTypeTests.h
+++ b/llvm/include/llvm/Transforms/IPO/LowerTypeTests.h
@@ -195,6 +195,13 @@ struct ByteArrayBuilder {
 
 bool isJumpTableCanonical(Function *F);
 
+/// Specifies how to drop type tests.
+enum class DropTestKind {
+  None,   /// Do not drop type tests (default).
+  Assume, /// Drop only llvm.assumes using type test value.
+  All,    /// Drop the type test and all uses.
+};
+
 } // end namespace lowertypetests
 
 class LowerTypeTestsPass : public PassInfoMixin<LowerTypeTestsPass> {
@@ -202,13 +209,15 @@ class LowerTypeTestsPass : public PassInfoMixin<LowerTypeTestsPass> {
 
   ModuleSummaryIndex *ExportSummary = nullptr;
   const ModuleSummaryIndex *ImportSummary = nullptr;
-  bool DropTypeTests = true;
+  lowertypetests::DropTestKind DropTypeTests =
+      lowertypetests::DropTestKind::None;
 
 public:
   LowerTypeTestsPass() : UseCommandLine(true) {}
   LowerTypeTestsPass(ModuleSummaryIndex *ExportSummary,
                      const ModuleSummaryIndex *ImportSummary,
-                     bool DropTypeTests = false)
+                     lowertypetests::DropTestKind DropTypeTests =
+                         lowertypetests::DropTestKind::None)
       : ExportSummary(ExportSummary), ImportSummary(ImportSummary),
         DropTypeTests(DropTypeTests) {}
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 488554c84c1c..c391853c8d0c 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -1135,7 +1135,8 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
   // post link pipeline after ICP. This is to enable usage of the type
   // tests in ICP sequences.
   if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink)
-    MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
+    MPM.addPass(LowerTypeTestsPass(nullptr, nullptr,
+                                   lowertypetests::DropTestKind::Assume));
 
   invokePipelineEarlySimplificationEPCallbacks(MPM, Level);
 
@@ -1750,7 +1751,8 @@ ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
   if (Level == OptimizationLevel::O0) {
     // Run a second time to clean up any type tests left behind by WPD for use
     // in ICP.
-    MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
+    MPM.addPass(LowerTypeTestsPass(nullptr, nullptr,
+                                   lowertypetests::DropTestKind::Assume));
     // Drop available_externally and unreferenced globals. This is necessary
     // with ThinLTO in order to avoid leaving undefined references to dead
     // globals in the object file.
@@ -1801,7 +1803,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
     // Run a second time to clean up any type tests left behind by WPD for use
     // in ICP.
-    MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
+    MPM.addPass(LowerTypeTestsPass(nullptr, nullptr,
+                                   lowertypetests::DropTestKind::Assume));
 
     invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
 
@@ -1879,7 +1882,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
     // Run a second time to clean up any type tests left behind by WPD for use
     // in ICP (which is performed earlier than this in the regular LTO
     // pipeline).
-    MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
+    MPM.addPass(LowerTypeTestsPass(nullptr, nullptr,
+                                   lowertypetests::DropTestKind::Assume));
 
     invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
 
@@ -2060,7 +2064,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
   MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
   // Run a second time to clean up any type tests left behind by WPD for use
   // in ICP (which is performed earlier than this in the regular LTO pipeline).
-  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
+  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr,
+                                 lowertypetests::DropTestKind::Assume));
 
   // Enable splitting late in the FullLTO post-link pipeline.
   if (EnableHotColdSplit)
diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index 3fcfc6a87677..9369b91d9c7f 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -118,10 +118,16 @@ static cl::opt<std::string> ClWriteSummary(
     cl::desc("Write summary to given YAML file after running pass"),
     cl::Hidden);
 
-static cl::opt<bool>
+static cl::opt<DropTestKind>
     ClDropTypeTests("lowertypetests-drop-type-tests",
-                    cl::desc("Simply drop type test assume sequences"),
-                    cl::Hidden, cl::init(false));
+                    cl::desc("Simply drop type test sequences"),
+                    cl::values(clEnumValN(DropTestKind::None, "none",
+                                          "Do not drop any type tests"),
+                               clEnumValN(DropTestKind::Assume, "assume",
+                                          "Drop type test assume sequences"),
+                               clEnumValN(DropTestKind::All, "all",
+                                          "Drop all type test sequences")),
+                    cl::Hidden, cl::init(DropTestKind::None));
 
 bool BitSetInfo::containsGlobalOffset(uint64_t Offset) const {
   if (Offset < ByteOffset)
@@ -399,7 +405,7 @@ class LowerTypeTestsModule {
   const ModuleSummaryIndex *ImportSummary;
   // Set when the client has invoked this to simply drop all type test assume
   // sequences.
-  bool DropTypeTests;
+  DropTestKind DropTypeTests;
 
   Triple::ArchType Arch;
   Triple::OSType OS;
@@ -542,7 +548,7 @@ public:
   LowerTypeTestsModule(Module &M, ModuleAnalysisManager &AM,
                        ModuleSummaryIndex *ExportSummary,
                        const ModuleSummaryIndex *ImportSummary,
-                       bool DropTypeTests);
+                       DropTestKind DropTypeTests);
 
   bool lower();
 
@@ -1828,9 +1834,10 @@ void LowerTypeTestsModule::buildBitSetsFromDisjointSet(
 /// Lower all type tests in this module.
 LowerTypeTestsModule::LowerTypeTestsModule(
     Module &M, ModuleAnalysisManager &AM, ModuleSummaryIndex *ExportSummary,
-    const ModuleSummaryIndex *ImportSummary, bool DropTypeTests)
+    const ModuleSummaryIndex *ImportSummary, DropTestKind DropTypeTests)
     : M(M), ExportSummary(ExportSummary), ImportSummary(ImportSummary),
-      DropTypeTests(DropTypeTests || ClDropTypeTests) {
+      DropTypeTests(ClDropTypeTests > DropTypeTests ? ClDropTypeTests
+                                                    : DropTypeTests) {
   assert(!(ExportSummary && ImportSummary));
   Triple TargetTriple(M.getTargetTriple());
   Arch = TargetTriple.getArch();
@@ -1882,7 +1889,7 @@ bool LowerTypeTestsModule::runForTesting(Module &M, ModuleAnalysisManager &AM) {
           M, AM,
           ClSummaryAction == PassSummaryAction::Export ? &Summary : nullptr,
           ClSummaryAction == PassSummaryAction::Import ? &Summary : nullptr,
-          /*DropTypeTests*/ false)
+          /*DropTypeTests=*/DropTestKind::None)
           .lower();
 
   if (!ClWriteSummary.empty()) {
@@ -1949,7 +1956,8 @@ void LowerTypeTestsModule::replaceDirectCalls(Value *Old, Value *New) {
   Old->replaceUsesWithIf(New, isDirectCall);
 }
 
-static void dropTypeTests(Module &M, Function &TypeTestFunc) {
+static void dropTypeTests(Module &M, Function &TypeTestFunc,
+                          bool ShouldDropAll) {
   for (Use &U : llvm::make_early_inc_range(TypeTestFunc.uses())) {
     auto *CI = cast<CallInst>(U.getUser());
     // Find and erase llvm.assume intrinsics for this llvm.type.test call.
@@ -1959,9 +1967,13 @@ static void dropTypeTests(Module &M, Function &TypeTestFunc) {
     // If the assume was merged with another assume, we might have a use on a
     // phi (which will feed the assume). Simply replace the use on the phi
     // with "true" and leave the merged assume.
+    //
+    // If ShouldDropAll is set, then we  we need to update any remaining uses,
+    // regardless of the instruction type.
     if (!CI->use_empty()) {
-      assert(
-          all_of(CI->users(), [](User *U) -> bool { return isa<PHINode>(U); }));
+      assert(ShouldDropAll || all_of(CI->users(), [](User *U) -> bool {
+               return isa<PHINode>(U);
+             }));
       CI->replaceAllUsesWith(ConstantInt::getTrue(M.getContext()));
     }
     CI->eraseFromParent();
@@ -1972,16 +1984,17 @@ bool LowerTypeTestsModule::lower() {
   Function *TypeTestFunc =
       Intrinsic::getDeclarationIfExists(&M, Intrinsic::type_test);
 
-  if (DropTypeTests) {
+  if (DropTypeTests != DropTestKind::None) {
+    bool ShouldDropAll = DropTypeTests == DropTestKind::All;
     if (TypeTestFunc)
-      dropTypeTests(M, *TypeTestFunc);
+      dropTypeTests(M, *TypeTestFunc, ShouldDropAll);
     // Normally we'd have already removed all @llvm.public.type.test calls,
     // except for in the case where we originally were performing ThinLTO but
     // decided not to in the backend.
     Function *PublicTypeTestFunc =
         Intrinsic::getDeclarationIfExists(&M, Intrinsic::public_type_test);
     if (PublicTypeTestFunc)
-      dropTypeTests(M, *PublicTypeTestFunc);
+      dropTypeTests(M, *PublicTypeTestFunc, ShouldDropAll);
     if (TypeTestFunc || PublicTypeTestFunc) {
       // We have deleted the type intrinsics, so we no longer have enough
       // information to reason about the liveness of virtual function pointers
diff --git a/llvm/test/Transforms/LowerTypeTests/drop_type_test.ll b/llvm/test/Transforms/LowerTypeTests/drop_type_test.ll
new file mode 100644
index 000000000000..e1d0573924a4
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/drop_type_test.ll
@@ -0,0 +1,22 @@
+; RUN: opt -S -passes=lowertypetests -lowertypetests-drop-type-tests=all < %s | FileCheck %s
+
+define void @func() {
+entry:
+  %0 = tail call i1 @llvm.type.test(ptr null, metadata !"foo")
+  br i1 %0, label %exit, label %trap
+
+trap:
+  unreachable
+
+exit:
+  ret void
+  ; CHECK-LABEL: entry:
+  ;  CHECK-NEXT: br i1 true, label %exit, label %trap
+  ; CHECK-LABEL: trap:
+  ;  CHECK-NEXT: unreachable
+  ; CHECK-LABEL: exit:
+  ;  CHECK-NEXT: ret void
+}
+
+declare i1 @llvm.type.test(ptr, metadata) #0
+attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
diff --git a/llvm/test/Transforms/LowerTypeTests/drop_type_test_phi.ll b/llvm/test/Transforms/LowerTypeTests/drop_type_test_phi.ll
index 3cf4d447605d..820865826dc7 100644
--- a/llvm/test/Transforms/LowerTypeTests/drop_type_test_phi.ll
+++ b/llvm/test/Transforms/LowerTypeTests/drop_type_test_phi.ll
@@ -1,5 +1,5 @@
 ; Test to ensure dropping of type tests can handle a phi feeding the assume.
-; RUN: opt -S -passes=lowertypetests -lowertypetests-drop-type-tests -mtriple=x86_64-unknown-linux-gnu %s | FileCheck %s
+; RUN: opt -S -passes=lowertypetests -lowertypetests-drop-type-tests=assume -mtriple=x86_64-unknown-linux-gnu %s | FileCheck %s
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
-- 
GitLab


From 36b79156db117bddd98927d869bb51cf70a047d1 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Wed, 30 Oct 2024 19:59:48 -0400
Subject: [PATCH 666/693] [gn] port 508263824f4ef (BuiltinsX86.td)

---
 llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn | 4 ++++
 llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn           | 1 +
 2 files changed, 5 insertions(+)

diff --git a/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn b/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn
index e9d9a3be27a6..c945c8ac42e4 100644
--- a/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn
@@ -95,6 +95,10 @@ clang_tablegen("BuiltinsRISCV") {
   args = [ "-gen-clang-builtins" ]
 }
 
+clang_tablegen("BuiltinsX86") {
+  args = [ "-gen-clang-builtins" ]
+}
+
 # ARM CDE, MVE, and NEON.
 
 clang_tablegen("arm_neon") {
diff --git a/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn
index 1b193af6c30a..31b4ba6304a2 100644
--- a/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn
@@ -26,6 +26,7 @@ static_library("Basic") {
     "//clang/include/clang/Basic:Builtins",
     "//clang/include/clang/Basic:BuiltinsBPF",
     "//clang/include/clang/Basic:BuiltinsRISCV",
+    "//clang/include/clang/Basic:BuiltinsX86",
     "//clang/include/clang/Basic:DiagnosticGroups",
     "//clang/include/clang/Basic:RegularKeywordAttrInfo",
     "//clang/include/clang/Basic:arm_cde_builtins",
-- 
GitLab


From cf9d1c1486ef53213b434700a4117d71a2cb67e3 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Thu, 31 Oct 2024 08:10:07 +0800
Subject: [PATCH 667/693] [SDAG] Simplify `SDNodeFlags` with bitwise logic
 (#114061)

This patch allows using enumeration values directly and simplifies the
implementation with bitwise logic. It addresses the comment in
https://github.com/llvm/llvm-project/pull/113808#discussion_r1819923625.
---
 llvm/include/llvm/CodeGen/SDPatternMatch.h    | 16 +---
 llvm/include/llvm/CodeGen/SelectionDAG.h      |  8 +-
 llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 20 +++--
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 40 ++++------
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp |  7 +-
 .../SelectionDAG/LegalizeIntegerTypes.cpp     |  6 +-
 .../SelectionDAG/LegalizeVectorOps.cpp        | 12 +--
 .../SelectionDAG/LegalizeVectorTypes.cpp      |  4 +-
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |  2 +-
 .../SelectionDAG/SelectionDAGBuilder.cpp      | 17 ++--
 .../CodeGen/SelectionDAG/SelectionDAGISel.cpp |  7 +-
 .../CodeGen/SelectionDAG/TargetLowering.cpp   | 77 +++++--------------
 .../Target/AArch64/AArch64ISelLowering.cpp    | 15 +---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 17 ++--
 .../Target/SystemZ/SystemZISelLowering.cpp    |  5 +-
 .../CodeGen/SelectionDAGPatternMatchTest.cpp  |  9 +--
 16 files changed, 90 insertions(+), 172 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h
index b3e249b7ebd5..96667952a16e 100644
--- a/llvm/include/llvm/CodeGen/SDPatternMatch.h
+++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h
@@ -533,9 +533,7 @@ struct BinaryOpc_match {
       if (!Flags.has_value())
         return true;
 
-      SDNodeFlags TmpFlags = *Flags;
-      TmpFlags.intersectWith(N->getFlags());
-      return TmpFlags == *Flags;
+      return (*Flags & N->getFlags()) == *Flags;
     }
 
     return false;
@@ -668,9 +666,7 @@ inline BinaryOpc_match<LHS, RHS, true> m_Or(const LHS &L, const RHS &R) {
 template <typename LHS, typename RHS>
 inline BinaryOpc_match<LHS, RHS, true> m_DisjointOr(const LHS &L,
                                                     const RHS &R) {
-  SDNodeFlags Flags;
-  Flags.setDisjoint(true);
-  return BinaryOpc_match<LHS, RHS, true>(ISD::OR, L, R, Flags);
+  return BinaryOpc_match<LHS, RHS, true>(ISD::OR, L, R, SDNodeFlags::Disjoint);
 }
 
 template <typename LHS, typename RHS>
@@ -813,9 +809,7 @@ template <typename Opnd_P, bool ExcludeChain = false> struct UnaryOpc_match {
       if (!Flags.has_value())
         return true;
 
-      SDNodeFlags TmpFlags = *Flags;
-      TmpFlags.intersectWith(N->getFlags());
-      return TmpFlags == *Flags;
+      return (*Flags & N->getFlags()) == *Flags;
     }
 
     return false;
@@ -848,9 +842,7 @@ template <typename Opnd> inline UnaryOpc_match<Opnd> m_ZExt(const Opnd &Op) {
 
 template <typename Opnd>
 inline UnaryOpc_match<Opnd> m_NNegZExt(const Opnd &Op) {
-  SDNodeFlags Flags;
-  Flags.setNonNeg(true);
-  return UnaryOpc_match<Opnd>(ISD::ZERO_EXTEND, Op, Flags);
+  return UnaryOpc_match<Opnd>(ISD::ZERO_EXTEND, Op, SDNodeFlags::NonNeg);
 }
 
 template <typename Opnd> inline auto m_SExt(const Opnd &Op) {
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index e82bdb690616..db111b0875a6 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1064,17 +1064,13 @@ public:
   /// addressing some offset of an object. i.e. if a load is split into multiple
   /// components, create an add nuw from the base pointer to the offset.
   SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset) {
-    SDNodeFlags Flags;
-    Flags.setNoUnsignedWrap(true);
-    return getMemBasePlusOffset(Ptr, Offset, SL, Flags);
+    return getMemBasePlusOffset(Ptr, Offset, SL, SDNodeFlags::NoUnsignedWrap);
   }
 
   SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, SDValue Offset) {
     // The object itself can't wrap around the address space, so it shouldn't be
     // possible for the adds of the offsets to the split parts to overflow.
-    SDNodeFlags Flags;
-    Flags.setNoUnsignedWrap(true);
-    return getMemBasePlusOffset(Ptr, Offset, SL, Flags);
+    return getMemBasePlusOffset(Ptr, Offset, SL, SDNodeFlags::NoUnsignedWrap);
   }
 
   /// Return a new CALLSEQ_START node, that starts new call frame, in which
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 26488413fe58..ae07420479e1 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -391,6 +391,7 @@ public:
     None = 0,
     NoUnsignedWrap = 1 << 0,
     NoSignedWrap = 1 << 1,
+    NoWrap = NoUnsignedWrap | NoSignedWrap,
     Exact = 1 << 2,
     Disjoint = 1 << 3,
     NonNeg = 1 << 4,
@@ -419,7 +420,7 @@ public:
   };
 
   /// Default constructor turns off all optimization flags.
-  SDNodeFlags() : Flags(0) {}
+  SDNodeFlags(unsigned Flags = SDNodeFlags::None) : Flags(Flags) {}
 
   /// Propagate the fast-math-flags from an IR FPMathOperator.
   void copyFMF(const FPMathOperator &FPMO) {
@@ -467,15 +468,23 @@ public:
   bool operator==(const SDNodeFlags &Other) const {
     return Flags == Other.Flags;
   }
-
-  /// Clear any flags in this flag set that aren't also set in Flags. All
-  /// flags will be cleared if Flags are undefined.
-  void intersectWith(const SDNodeFlags Flags) { this->Flags &= Flags.Flags; }
+  void operator&=(const SDNodeFlags &OtherFlags) { Flags &= OtherFlags.Flags; }
+  void operator|=(const SDNodeFlags &OtherFlags) { Flags |= OtherFlags.Flags; }
 };
 
 LLVM_DECLARE_ENUM_AS_BITMASK(decltype(SDNodeFlags::None),
                              SDNodeFlags::Unpredictable);
 
+inline SDNodeFlags operator|(SDNodeFlags LHS, SDNodeFlags RHS) {
+  LHS |= RHS;
+  return LHS;
+}
+
+inline SDNodeFlags operator&(SDNodeFlags LHS, SDNodeFlags RHS) {
+  LHS &= RHS;
+  return LHS;
+}
+
 /// Represents one node in the SelectionDAG.
 ///
 class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
@@ -1013,6 +1022,7 @@ public:
 
   SDNodeFlags getFlags() const { return Flags; }
   void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
+  void dropFlags(unsigned Mask) { Flags &= ~Mask; }
 
   /// Clear any flags in this node that aren't also set in Flags.
   /// If Flags is not in a defined state then this has no effect.
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index ceaf5d664131..fe0fe348ac60 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -1210,7 +1210,7 @@ SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL,
     SDNodeFlags NewFlags;
     if (N0.getOpcode() == ISD::ADD && N0->getFlags().hasNoUnsignedWrap() &&
         Flags.hasNoUnsignedWrap())
-      NewFlags.setNoUnsignedWrap(true);
+      NewFlags |= SDNodeFlags::NoUnsignedWrap;
 
     if (DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
       // Reassociate: (op (op x, c1), c2) -> (op x, (op c1, c2))
@@ -2892,11 +2892,11 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) {
         if (N->getFlags().hasNoUnsignedWrap() &&
             N0->getFlags().hasNoUnsignedWrap() &&
             N0.getOperand(0)->getFlags().hasNoUnsignedWrap()) {
-          Flags.setNoUnsignedWrap(true);
+          Flags |= SDNodeFlags::NoUnsignedWrap;
           if (N->getFlags().hasNoSignedWrap() &&
               N0->getFlags().hasNoSignedWrap() &&
               N0.getOperand(0)->getFlags().hasNoSignedWrap())
-            Flags.setNoSignedWrap(true);
+            Flags |= SDNodeFlags::NoSignedWrap;
         }
         SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N1), VT, A,
                                   DAG.getConstant(CM, DL, VT), Flags);
@@ -2920,12 +2920,12 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) {
             N0->getFlags().hasNoUnsignedWrap() &&
             OMul->getFlags().hasNoUnsignedWrap() &&
             OMul.getOperand(0)->getFlags().hasNoUnsignedWrap()) {
-          Flags.setNoUnsignedWrap(true);
+          Flags |= SDNodeFlags::NoUnsignedWrap;
           if (N->getFlags().hasNoSignedWrap() &&
               N0->getFlags().hasNoSignedWrap() &&
               OMul->getFlags().hasNoSignedWrap() &&
               OMul.getOperand(0)->getFlags().hasNoSignedWrap())
-            Flags.setNoSignedWrap(true);
+            Flags |= SDNodeFlags::NoSignedWrap;
         }
         SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N1), VT, A,
                                   DAG.getConstant(CM, DL, VT), Flags);
@@ -2987,11 +2987,8 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
 
   // fold (a+b) -> (a|b) iff a and b share no bits.
   if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
-      DAG.haveNoCommonBitsSet(N0, N1)) {
-    SDNodeFlags Flags;
-    Flags.setDisjoint(true);
-    return DAG.getNode(ISD::OR, DL, VT, N0, N1, Flags);
-  }
+      DAG.haveNoCommonBitsSet(N0, N1))
+    return DAG.getNode(ISD::OR, DL, VT, N0, N1, SDNodeFlags::Disjoint);
 
   // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
   if (N0.getOpcode() == ISD::VSCALE && N1.getOpcode() == ISD::VSCALE) {
@@ -9556,11 +9553,8 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
 
   // fold (a^b) -> (a|b) iff a and b share no bits.
   if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
-      DAG.haveNoCommonBitsSet(N0, N1)) {
-    SDNodeFlags Flags;
-    Flags.setDisjoint(true);
-    return DAG.getNode(ISD::OR, DL, VT, N0, N1, Flags);
-  }
+      DAG.haveNoCommonBitsSet(N0, N1))
+    return DAG.getNode(ISD::OR, DL, VT, N0, N1, SDNodeFlags::Disjoint);
 
   // look for 'add-like' folds:
   // XOR(N0,MIN_SIGNED_VALUE) == ADD(N0,MIN_SIGNED_VALUE)
@@ -10210,7 +10204,7 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
       SDNodeFlags Flags;
       // Preserve the disjoint flag for Or.
       if (N0.getOpcode() == ISD::OR && N0->getFlags().hasDisjoint())
-        Flags.setDisjoint(true);
+        Flags |= SDNodeFlags::Disjoint;
       return DAG.getNode(N0.getOpcode(), DL, VT, Shl0, Shl1, Flags);
     }
   }
@@ -13922,11 +13916,8 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
   // fold (sext x) -> (zext x) if the sign bit is known zero.
   if (!TLI.isSExtCheaperThanZExt(N0.getValueType(), VT) &&
       (!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) &&
-      DAG.SignBitIsZero(N0)) {
-    SDNodeFlags Flags;
-    Flags.setNonNeg(true);
-    return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0, Flags);
-  }
+      DAG.SignBitIsZero(N0))
+    return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0, SDNodeFlags::NonNeg);
 
   if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
     return NewVSel;
@@ -14807,10 +14798,9 @@ SDValue DAGCombiner::reduceLoadWidth(SDNode *N) {
   uint64_t PtrOff = PtrAdjustmentInBits / 8;
   SDLoc DL(LN0);
   // The original load itself didn't wrap, so an offset within it doesn't.
-  SDNodeFlags Flags;
-  Flags.setNoUnsignedWrap(true);
-  SDValue NewPtr = DAG.getMemBasePlusOffset(
-      LN0->getBasePtr(), TypeSize::getFixed(PtrOff), DL, Flags);
+  SDValue NewPtr =
+      DAG.getMemBasePlusOffset(LN0->getBasePtr(), TypeSize::getFixed(PtrOff),
+                               DL, SDNodeFlags::NoUnsignedWrap);
   AddToWorklist(NewPtr.getNode());
 
   SDValue Load;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 6ba12cfb8c51..61ed94ce38c4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1697,12 +1697,9 @@ SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
     SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
   }
 
-  SDNodeFlags Flags;
-  Flags.setDisjoint(true);
-
   // Store the part with the modified sign and convert back to float.
-  SDValue CopiedSign =
-      DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit, Flags);
+  SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit,
+                                   SDNodeFlags::Disjoint);
 
   return modifySignAsInt(MagAsInt, DL, CopiedSign);
 }
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index ee9c95c85937..45487c887b74 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -4674,9 +4674,9 @@ void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(SDNode *N, SDValue &Lo,
       DAG.getNode(ISD::SHL, dl, ShAmtVT, SrlTmp,
                   DAG.getConstant(Log2_32(ShiftUnitInBits), dl, ShAmtVT));
 
-  Flags.setExact(true);
-  SDValue ByteOffset = DAG.getNode(ISD::SRL, dl, ShAmtVT, BitOffset,
-                                   DAG.getConstant(3, dl, ShAmtVT), Flags);
+  SDValue ByteOffset =
+      DAG.getNode(ISD::SRL, dl, ShAmtVT, BitOffset,
+                  DAG.getConstant(3, dl, ShAmtVT), SDNodeFlags::Exact);
   // And clamp it, because OOB load is an immediate UB,
   // while shift overflow would have *just* been poison.
   ByteOffset = DAG.getNode(ISD::AND, dl, ShAmtVT, ByteOffset,
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index c80da28b3dc3..a8a171d932ff 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -1700,11 +1700,8 @@ SDValue VectorLegalizer::ExpandVP_FCOPYSIGN(SDNode *Node) {
   SDValue ClearedSign =
       DAG.getNode(ISD::VP_AND, DL, IntVT, Mag, ClearSignMask, Mask, EVL);
 
-  SDNodeFlags Flags;
-  Flags.setDisjoint(true);
-
   SDValue CopiedSign = DAG.getNode(ISD::VP_OR, DL, IntVT, ClearedSign, SignBit,
-                                   Mask, EVL, Flags);
+                                   Mask, EVL, SDNodeFlags::Disjoint);
 
   return DAG.getNode(ISD::BITCAST, DL, VT, CopiedSign);
 }
@@ -1886,11 +1883,8 @@ SDValue VectorLegalizer::ExpandFCOPYSIGN(SDNode *Node) {
       APInt::getSignedMaxValue(IntVT.getScalarSizeInBits()), DL, IntVT);
   SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, Mag, ClearSignMask);
 
-  SDNodeFlags Flags;
-  Flags.setDisjoint(true);
-
-  SDValue CopiedSign =
-      DAG.getNode(ISD::OR, DL, IntVT, ClearedSign, SignBit, Flags);
+  SDValue CopiedSign = DAG.getNode(ISD::OR, DL, IntVT, ClearedSign, SignBit,
+                                   SDNodeFlags::Disjoint);
 
   return DAG.getNode(ISD::BITCAST, DL, VT, CopiedSign);
 }
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 5409ae7d9671..eccda73548e8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -1381,16 +1381,14 @@ void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
   unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
 
   if (MemVT.isScalableVector()) {
-    SDNodeFlags Flags;
     SDValue BytesIncrement = DAG.getVScale(
         DL, Ptr.getValueType(),
         APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
     MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
-    Flags.setNoUnsignedWrap(true);
     if (ScaledOffset)
       *ScaledOffset += IncrementSize;
     Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
-                      Flags);
+                      SDNodeFlags::NoUnsignedWrap);
   } else {
     MPI = N->getPointerInfo().getWithOffset(IncrementSize);
     // Increment the pointer to the other half.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 5403d787861d..40ca3235ca0c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -12377,7 +12377,7 @@ bool SDNode::hasPredecessor(const SDNode *N) const {
 }
 
 void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
-  this->Flags.intersectWith(Flags);
+  this->Flags &= Flags;
 }
 
 SDValue
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 203e80e36b46..95125928cdc6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4318,7 +4318,7 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
         SDNodeFlags Flags;
         if (NW.hasNoUnsignedWrap() ||
             (int64_t(Offset) >= 0 && NW.hasNoUnsignedSignedWrap()))
-          Flags.setNoUnsignedWrap(true);
+          Flags |= SDNodeFlags::NoUnsignedWrap;
 
         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
                         DAG.getConstant(Offset, dl, N.getValueType()), Flags);
@@ -4484,10 +4484,9 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
   // Round the size of the allocation up to the stack alignment size
   // by add SA-1 to the size. This doesn't overflow because we're computing
   // an address inside an alloca.
-  SDNodeFlags Flags;
-  Flags.setNoUnsignedWrap(true);
   AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
-                          DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);
+                          DAG.getConstant(StackAlignMask, dl, IntPtr),
+                          SDNodeFlags::NoUnsignedWrap);
 
   // Mask out the low bits for alignment purposes.
   AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
@@ -11224,15 +11223,13 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
 
     // An aggregate return value cannot wrap around the address space, so
     // offsets to its parts don't wrap either.
-    SDNodeFlags Flags;
-    Flags.setNoUnsignedWrap(true);
-
     MachineFunction &MF = CLI.DAG.getMachineFunction();
     Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
     for (unsigned i = 0; i < NumValues; ++i) {
-      SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
-                                    CLI.DAG.getConstant(Offsets[i], CLI.DL,
-                                                        PtrVT), Flags);
+      SDValue Add =
+          CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
+                          CLI.DAG.getConstant(Offsets[i], CLI.DL, PtrVT),
+                          SDNodeFlags::NoUnsignedWrap);
       SDValue L = CLI.DAG.getLoad(
           RetTys[i], CLI.DL, CLI.Chain, Add,
           MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(),
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 981ab18b59c1..0d99ae9cdebd 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -4224,11 +4224,8 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
 
       // Set the NoFPExcept flag when no original matched node could
       // raise an FP exception, but the new node potentially might.
-      if (!MayRaiseFPException && mayRaiseFPException(Res)) {
-        SDNodeFlags Flags = Res->getFlags();
-        Flags.setNoFPExcept(true);
-        Res->setFlags(Flags);
-      }
+      if (!MayRaiseFPException && mayRaiseFPException(Res))
+        Res->setFlags(Res->getFlags() | SDNodeFlags::NoFPExcept);
 
       // If the node had chain/glue results, update our notion of the current
       // chain and glue.
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 758b3a5fc526..8ab7935347d5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -1489,19 +1489,13 @@ bool TargetLowering::SimplifyDemandedBits(
     SDNodeFlags Flags = Op.getNode()->getFlags();
     if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
                              Depth + 1)) {
-      if (Flags.hasDisjoint()) {
-        Flags.setDisjoint(false);
-        Op->setFlags(Flags);
-      }
+      Op->dropFlags(SDNodeFlags::Disjoint);
       return true;
     }
 
     if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
                              Known2, TLO, Depth + 1)) {
-      if (Flags.hasDisjoint()) {
-        Flags.setDisjoint(false);
-        Op->setFlags(Flags);
-      }
+      Op->dropFlags(SDNodeFlags::Disjoint);
       return true;
     }
 
@@ -1806,14 +1800,9 @@ bool TargetLowering::SimplifyDemandedBits(
       APInt InDemandedMask = DemandedBits.lshr(ShAmt);
       if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
                                Depth + 1)) {
-        SDNodeFlags Flags = Op.getNode()->getFlags();
-        if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
-          // Disable the nsw and nuw flags. We can no longer guarantee that we
-          // won't wrap after simplification.
-          Flags.setNoSignedWrap(false);
-          Flags.setNoUnsignedWrap(false);
-          Op->setFlags(Flags);
-        }
+        // Disable the nsw and nuw flags. We can no longer guarantee that we
+        // won't wrap after simplification.
+        Op->dropFlags(SDNodeFlags::NoWrap);
         return true;
       }
       Known.Zero <<= ShAmt;
@@ -1897,14 +1886,9 @@ bool TargetLowering::SimplifyDemandedBits(
         APInt DemandedFromOp(APInt::getLowBitsSet(BitWidth, BitWidth - CTLZ));
         if (SimplifyDemandedBits(Op0, DemandedFromOp, DemandedElts, Known, TLO,
                                  Depth + 1)) {
-          SDNodeFlags Flags = Op.getNode()->getFlags();
-          if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
-            // Disable the nsw and nuw flags. We can no longer guarantee that we
-            // won't wrap after simplification.
-            Flags.setNoSignedWrap(false);
-            Flags.setNoUnsignedWrap(false);
-            Op->setFlags(Flags);
-          }
+          // Disable the nsw and nuw flags. We can no longer guarantee that we
+          // won't wrap after simplification.
+          Op->dropFlags(SDNodeFlags::NoWrap);
           return true;
         }
         Known.resetAll();
@@ -2456,15 +2440,11 @@ bool TargetLowering::SimplifyDemandedBits(
         return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
     }
 
-    SDNodeFlags Flags = Op->getFlags();
     APInt InDemandedBits = DemandedBits.trunc(InBits);
     APInt InDemandedElts = DemandedElts.zext(InElts);
     if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
                              Depth + 1)) {
-      if (Flags.hasNonNeg()) {
-        Flags.setNonNeg(false);
-        Op->setFlags(Flags);
-      }
+      Op->dropFlags(SDNodeFlags::NonNeg);
       return true;
     }
     assert(Known.getBitWidth() == InBits && "Src width has changed?");
@@ -2528,7 +2508,7 @@ bool TargetLowering::SimplifyDemandedBits(
       if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) {
         SDNodeFlags Flags;
         if (!IsVecInReg)
-          Flags.setNonNeg(true);
+          Flags |= SDNodeFlags::NonNeg;
         return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src, Flags));
       }
     }
@@ -2836,13 +2816,9 @@ bool TargetLowering::SimplifyDemandedBits(
                              DemandedElts, KnownOp0, TLO, Depth + 1) ||
         // See if the operation should be performed at a smaller bit width.
         ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
-      if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
-        // Disable the nsw and nuw flags. We can no longer guarantee that we
-        // won't wrap after simplification.
-        Flags.setNoSignedWrap(false);
-        Flags.setNoUnsignedWrap(false);
-        Op->setFlags(Flags);
-      }
+      // Disable the nsw and nuw flags. We can no longer guarantee that we
+      // won't wrap after simplification.
+      Op->dropFlags(SDNodeFlags::NoWrap);
       return true;
     }
 
@@ -2858,12 +2834,10 @@ bool TargetLowering::SimplifyDemandedBits(
       SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
           Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
       if (DemandedOp0 || DemandedOp1) {
-        Flags.setNoSignedWrap(false);
-        Flags.setNoUnsignedWrap(false);
         Op0 = DemandedOp0 ? DemandedOp0 : Op0;
         Op1 = DemandedOp1 ? DemandedOp1 : Op1;
-        SDValue NewOp =
-            TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
+        SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1,
+                                        Flags & ~SDNodeFlags::NoWrap);
         return TLO.CombineTo(Op, NewOp);
       }
     }
@@ -2880,9 +2854,8 @@ bool TargetLowering::SimplifyDemandedBits(
       SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
       // Disable the nsw and nuw flags. We can no longer guarantee that we
       // won't wrap after simplification.
-      Flags.setNoSignedWrap(false);
-      Flags.setNoUnsignedWrap(false);
-      SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
+      SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1,
+                                      Flags & ~SDNodeFlags::NoWrap);
       return TLO.CombineTo(Op, NewOp);
     }
 
@@ -6157,9 +6130,7 @@ static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
 
   SDValue Res = Op0;
   if (UseSRA) {
-    SDNodeFlags Flags;
-    Flags.setExact(true);
-    Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
+    Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, SDNodeFlags::Exact);
     Created.push_back(Res.getNode());
   }
 
@@ -6220,9 +6191,7 @@ static SDValue BuildExactUDIV(const TargetLowering &TLI, SDNode *N,
 
   SDValue Res = N->getOperand(0);
   if (UseSRL) {
-    SDNodeFlags Flags;
-    Flags.setExact(true);
-    Res = DAG.getNode(ISD::SRL, dl, VT, Res, Shift, Flags);
+    Res = DAG.getNode(ISD::SRL, dl, VT, Res, Shift, SDNodeFlags::Exact);
     Created.push_back(Res.getNode());
   }
 
@@ -8447,9 +8416,7 @@ TargetLowering::createSelectForFMINNUM_FMAXNUM(SDNode *Node,
     SDValue SelCC = DAG.getSelectCC(SDLoc(Node), Op1, Op2, Op1, Op2, Pred);
     // Copy FMF flags, but always set the no-signed-zeros flag
     // as this is implied by the FMINNUM/FMAXNUM semantics.
-    SDNodeFlags Flags = Node->getFlags();
-    Flags.setNoSignedZeros(true);
-    SelCC->setFlags(Flags);
+    SelCC->setFlags(Node->getFlags() | SDNodeFlags::NoSignedZeros);
     return SelCC;
   }
 
@@ -11805,10 +11772,8 @@ SDValue TargetLowering::expandVECTOR_COMPRESS(SDNode *Node,
 
       // Re-write the last ValI if all lanes were selected. Otherwise,
       // overwrite the last write it with the passthru value.
-      SDNodeFlags Flags{};
-      Flags.setUnpredictable(true);
       LastWriteVal = DAG.getSelect(DL, ScalarVT, AllLanesSelected, ValI,
-                                   LastWriteVal, Flags);
+                                   LastWriteVal, SDNodeFlags::Unpredictable);
       Chain = DAG.getStore(
           Chain, DL, LastWriteVal, OutPtr,
           MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 31a720ed7b5c..e8c02c098797 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -7927,10 +7927,8 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
                 APInt(Ptr.getValueSizeInBits().getFixedValue(), PartSize), DL,
                 Ptr.getValueType());
           }
-          SDNodeFlags Flags;
-          Flags.setNoUnsignedWrap(true);
           Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
-                            BytesIncrement, Flags);
+                            BytesIncrement, SDNodeFlags::NoUnsignedWrap);
           ExtraArgLocs++;
           i++;
         }
@@ -8986,12 +8984,9 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
                 APInt(Ptr.getValueSizeInBits().getFixedValue(), PartSize), DL,
                 Ptr.getValueType());
           }
-          SDNodeFlags Flags;
-          Flags.setNoUnsignedWrap(true);
-
           MPI = MachinePointerInfo(MPI.getAddrSpace());
           Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
-                            BytesIncrement, Flags);
+                            BytesIncrement, SDNodeFlags::NoUnsignedWrap);
           ExtraArgLocs++;
           i++;
         }
@@ -11777,8 +11772,7 @@ SDValue AArch64TargetLowering::getSqrtEstimate(SDValue Operand,
       SDLoc DL(Operand);
       EVT VT = Operand.getValueType();
 
-      SDNodeFlags Flags;
-      Flags.setAllowReassociation(true);
+      SDNodeFlags Flags = SDNodeFlags::AllowReassociation;
 
       // Newton reciprocal square root iteration: E * 0.5 * (3 - X * E^2)
       // AArch64 reciprocal square root iteration instruction: 0.5 * (3 - M * N)
@@ -11807,8 +11801,7 @@ SDValue AArch64TargetLowering::getRecipEstimate(SDValue Operand,
       SDLoc DL(Operand);
       EVT VT = Operand.getValueType();
 
-      SDNodeFlags Flags;
-      Flags.setAllowReassociation(true);
+      SDNodeFlags Flags = SDNodeFlags::AllowReassociation;
 
       // Newton reciprocal iteration: E * (2 - X * E)
       // AArch64 reciprocal iteration instruction: (2 - M * N)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index af7a39b2580a..e7898747fcce 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -3990,10 +3990,9 @@ static SDValue lowerBuildVectorViaPacking(SDValue Op, SelectionDAG &DAG,
     A = DAG.getNode(ISD::AND, SDLoc(A), XLenVT, A, Mask);
     B = DAG.getNode(ISD::AND, SDLoc(B), XLenVT, B, Mask);
     SDValue ShtAmt = DAG.getConstant(ElemSizeInBits, ElemDL, XLenVT);
-    SDNodeFlags Flags;
-    Flags.setDisjoint(true);
     return DAG.getNode(ISD::OR, ElemDL, XLenVT, A,
-                       DAG.getNode(ISD::SHL, ElemDL, XLenVT, B, ShtAmt), Flags);
+                       DAG.getNode(ISD::SHL, ElemDL, XLenVT, B, ShtAmt),
+                       SDNodeFlags::Disjoint);
   };
 
   SmallVector<SDValue> NewOperands;
@@ -6022,11 +6021,8 @@ static SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG,
   SDValue ClearedSign =
       DAG.getNode(ISD::AND, DL, XLenVT, MagAsInt, ClearSignMask);
 
-  SDNodeFlags Flags;
-  Flags.setDisjoint(true);
-
-  SDValue CopiedSign =
-      DAG.getNode(ISD::OR, DL, XLenVT, ClearedSign, SignBit, Flags);
+  SDValue CopiedSign = DAG.getNode(ISD::OR, DL, XLenVT, ClearedSign, SignBit,
+                                   SDNodeFlags::Disjoint);
 
   return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, CopiedSign);
 }
@@ -13291,9 +13287,8 @@ combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG,
     EVT ReduceVT = EVT::getVectorVT(*DAG.getContext(), VT, RHSIdx + 1);
     SDValue Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ReduceVT, SrcVec,
                               DAG.getVectorIdxConstant(0, DL));
-    auto Flags = ReduceVec->getFlags();
-    Flags.intersectWith(N->getFlags());
-    return DAG.getNode(ReduceOpc, DL, VT, Vec, Flags);
+    return DAG.getNode(ReduceOpc, DL, VT, Vec,
+                       ReduceVec->getFlags() & N->getFlags());
   }
 
   return SDValue();
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 1fa2dbfb26fc..3999b54de81b 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -2676,10 +2676,7 @@ static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL,
            (N->getOperand(0) == C.Op1 && N->getOperand(1) == C.Op0))) {
         // Disable the nsw and nuw flags: the backend needs to handle
         // overflow as well during comparison elimination.
-        SDNodeFlags Flags = N->getFlags();
-        Flags.setNoSignedWrap(false);
-        Flags.setNoUnsignedWrap(false);
-        N->setFlags(Flags);
+        N->dropFlags(SDNodeFlags::NoWrap);
         C.Op0 = SDValue(N, 0);
         C.Op1 = DAG.getConstant(0, DL, N->getValueType(0));
         return;
diff --git a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
index dc40e5893b65..1402c1d5b139 100644
--- a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
@@ -193,9 +193,8 @@ TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) {
   SDValue And = DAG->getNode(ISD::AND, DL, Int32VT, Op0, Op1);
   SDValue Xor = DAG->getNode(ISD::XOR, DL, Int32VT, Op1, Op0);
   SDValue Or  = DAG->getNode(ISD::OR, DL, Int32VT, Op0, Op1);
-  SDNodeFlags DisFlags;
-  DisFlags.setDisjoint(true);
-  SDValue DisOr = DAG->getNode(ISD::OR, DL, Int32VT, Op0, Op3, DisFlags);
+  SDValue DisOr =
+      DAG->getNode(ISD::OR, DL, Int32VT, Op0, Op3, SDNodeFlags::Disjoint);
   SDValue SMax = DAG->getNode(ISD::SMAX, DL, Int32VT, Op0, Op1);
   SDValue SMin = DAG->getNode(ISD::SMIN, DL, Int32VT, Op1, Op0);
   SDValue UMax = DAG->getNode(ISD::UMAX, DL, Int32VT, Op0, Op1);
@@ -293,10 +292,8 @@ TEST_F(SelectionDAGPatternMatchTest, matchUnaryOp) {
   SDValue Op3 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 3, Int32VT);
 
   SDValue ZExt = DAG->getNode(ISD::ZERO_EXTEND, DL, Int64VT, Op0);
-  SDNodeFlags NNegFlags;
-  NNegFlags.setNonNeg(true);
   SDValue ZExtNNeg =
-      DAG->getNode(ISD::ZERO_EXTEND, DL, Int64VT, Op3, NNegFlags);
+      DAG->getNode(ISD::ZERO_EXTEND, DL, Int64VT, Op3, SDNodeFlags::NonNeg);
   SDValue SExt = DAG->getNode(ISD::SIGN_EXTEND, DL, Int64VT, Op0);
   SDValue Trunc = DAG->getNode(ISD::TRUNCATE, DL, Int32VT, Op1);
 
-- 
GitLab


From d67f2bd45de4808597189ca67f809e037211576e Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Thu, 31 Oct 2024 00:17:29 +0000
Subject: [PATCH 668/693] [gn build] Port 84b7bcfcac02

---
 llvm/utils/gn/secondary/llvm/unittests/Target/AMDGPU/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/llvm/unittests/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Target/AMDGPU/BUILD.gn
index 75c693e446a3..502aa13e1de8 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Target/AMDGPU/BUILD.gn
@@ -17,6 +17,7 @@ unittest("AMDGPUTests") {
   include_dirs = [ "//llvm/lib/Target/AMDGPU" ]
   sources = [
     "AMDGPUUnitTests.cpp",
+    "CSETest.cpp",
     "DwarfRegMappings.cpp",
     "ExecMayBeModifiedBeforeAnyUse.cpp",
     "PALMetadata.cpp",
-- 
GitLab


From dbece8edb4072a7e77feb948619b46ba7cc48923 Mon Sep 17 00:00:00 2001
From: Chris Apple <cja-private@pm.me>
Date: Wed, 30 Oct 2024 17:39:28 -0700
Subject: [PATCH 669/693] [rtsan][NFC] Put in comment describing why freeing a
 nullptr is safe (#113720)

Just documenting this for future devs.

Also moved to `nullptr` and deleted unnecessary braces as per the coding
standard.
---
 compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp b/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
index 890d6c11c407..a65871b17da5 100644
--- a/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
@@ -431,9 +431,12 @@ INTERCEPTOR(void, free, void *ptr) {
   if (DlsymAlloc::PointerIsMine(ptr))
     return DlsymAlloc::Free(ptr);
 
-  if (ptr != NULL) {
+  // According to the C and C++ standard, freeing a nullptr is guaranteed to be
+  // a no-op (and thus real-time safe). This can be confirmed for looking at
+  // __libc_free in the glibc source.
+  if (ptr != nullptr)
     __rtsan_notify_intercepted_call("free");
-  }
+
   return REAL(free)(ptr);
 }
 
-- 
GitLab


From f582cd3dc70fa8c9519f74f16ab0a33ad663038e Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 30 Oct 2024 17:49:51 -0700
Subject: [PATCH 670/693] [SelectionDAG] Fix a warning

This patch fixes:

  llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp:1489:17: error:
  unused variable 'Flags' [-Werror,-Wunused-variable]
---
 llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 8ab7935347d5..fabcbc5f0e85 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -1486,7 +1486,6 @@ bool TargetLowering::SimplifyDemandedBits(
   case ISD::OR: {
     SDValue Op0 = Op.getOperand(0);
     SDValue Op1 = Op.getOperand(1);
-    SDNodeFlags Flags = Op.getNode()->getFlags();
     if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
                              Depth + 1)) {
       Op->dropFlags(SDNodeFlags::Disjoint);
-- 
GitLab


From dafb90dedcda1ad7b94b0bcdbbe7478f7d0f31f6 Mon Sep 17 00:00:00 2001
From: Akira Hatanaka <ahatanak@gmail.com>
Date: Wed, 30 Oct 2024 18:00:59 -0700
Subject: [PATCH 671/693] [NFC] Call base class method in
 DarwinAArch64TargetInfo::getOSDefines (#114241)

This is needed for a private patch we'll be upstreaming in the future.
---
 clang/lib/Basic/Targets/AArch64.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 3d8de0294d4b..e35ee2b7b9c3 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -1714,7 +1714,7 @@ void DarwinAArch64TargetInfo::getOSDefines(const LangOptions &Opts,
   if (Triple.isArm64e())
     Builder.defineMacro("__arm64e__", "1");
 
-  getDarwinDefines(Builder, Opts, Triple, PlatformName, PlatformMinVersion);
+  DarwinTargetInfo<AArch64leTargetInfo>::getOSDefines(Opts, Triple, Builder);
 }
 
 TargetInfo::BuiltinVaListKind
-- 
GitLab


From e99c4906e44ae3f921fa05356909d006cda8d954 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Thu, 31 Oct 2024 02:20:10 +0100
Subject: [PATCH 672/693] [libc++] Granularize <cstddef> includes (#108696)

---
 libcxx/include/__algorithm/copy_move_common.h |  2 +-
 libcxx/include/__algorithm/inplace_merge.h    |  1 +
 libcxx/include/__algorithm/mismatch.h         |  2 +-
 libcxx/include/__algorithm/shuffle.h          |  2 +-
 libcxx/include/__algorithm/simd_utils.h       |  2 +-
 libcxx/include/__algorithm/stable_partition.h |  1 +
 libcxx/include/__algorithm/stable_sort.h      |  1 +
 libcxx/include/__atomic/aliases.h             |  3 +-
 libcxx/include/__atomic/atomic.h              |  2 +-
 libcxx/include/__atomic/atomic_ref.h          |  3 +-
 libcxx/include/__atomic/cxx_atomic_impl.h     |  3 +-
 .../__charconv/from_chars_floating_point.h    |  2 +-
 libcxx/include/__charconv/to_chars_integral.h |  2 +-
 .../__compare/common_comparison_category.h    |  2 +-
 libcxx/include/__concepts/swappable.h         |  2 +-
 libcxx/include/__coroutine/coroutine_handle.h |  3 +-
 libcxx/include/__exception/exception_ptr.h    |  1 -
 libcxx/include/__exception/nested_exception.h |  1 -
 libcxx/include/__exception/operations.h       |  1 -
 .../include/__filesystem/directory_iterator.h |  1 -
 libcxx/include/__filesystem/path.h            |  1 -
 libcxx/include/__filesystem/path_iterator.h   |  3 -
 .../recursive_directory_iterator.h            |  1 -
 libcxx/include/__flat_map/flat_map.h          |  1 +
 libcxx/include/__format/buffer.h              |  1 -
 .../include/__format/escaped_output_table.h   |  2 +-
 .../extended_grapheme_cluster_table.h         |  2 +-
 libcxx/include/__format/format_arg.h          |  1 +
 libcxx/include/__format/format_args.h         |  2 +-
 libcxx/include/__format/format_context.h      |  1 -
 libcxx/include/__format/format_string.h       |  2 +-
 .../__format/formatter_floating_point.h       |  1 -
 libcxx/include/__format/formatter_output.h    |  3 +-
 libcxx/include/__format/formatter_pointer.h   |  2 +-
 .../__format/indic_conjunct_break_table.h     |  2 +-
 .../include/__format/width_estimation_table.h |  2 +-
 libcxx/include/__functional/bind.h            |  1 -
 libcxx/include/__functional/hash.h            |  1 -
 libcxx/include/__hash_table                   |  1 +
 libcxx/include/__iterator/access.h            |  2 +-
 libcxx/include/__iterator/aliasing_iterator.h |  2 +-
 .../include/__iterator/back_insert_iterator.h |  2 +-
 libcxx/include/__iterator/data.h              |  1 -
 libcxx/include/__iterator/empty.h             |  1 -
 .../__iterator/front_insert_iterator.h        |  2 +-
 .../include/__iterator/incrementable_traits.h |  2 +-
 libcxx/include/__iterator/insert_iterator.h   |  2 +-
 libcxx/include/__iterator/istream_iterator.h  |  2 +-
 .../include/__iterator/istreambuf_iterator.h  |  2 +
 libcxx/include/__iterator/iterator.h          |  2 +-
 libcxx/include/__iterator/iterator_traits.h   |  2 +-
 libcxx/include/__iterator/ostream_iterator.h  |  2 +-
 .../include/__iterator/ostreambuf_iterator.h  |  2 +-
 libcxx/include/__iterator/reverse_access.h    |  1 -
 .../include/__iterator/segmented_iterator.h   |  2 +-
 libcxx/include/__iterator/size.h              |  3 +-
 libcxx/include/__iterator/wrap_iter.h         |  2 +-
 libcxx/include/__mdspan/default_accessor.h    |  3 +-
 libcxx/include/__mdspan/extents.h             |  3 +-
 libcxx/include/__mdspan/layout_left.h         |  3 -
 libcxx/include/__mdspan/layout_right.h        |  4 +-
 libcxx/include/__mdspan/layout_stride.h       |  2 -
 libcxx/include/__mdspan/mdspan.h              |  3 -
 libcxx/include/__memory/align.h               |  2 +-
 libcxx/include/__memory/aligned_alloc.h       |  1 -
 libcxx/include/__memory/allocate_at_least.h   |  2 +-
 libcxx/include/__memory/allocation_guard.h    |  1 -
 libcxx/include/__memory/allocator.h           |  2 +-
 libcxx/include/__memory/allocator_traits.h    |  2 +-
 libcxx/include/__memory/array_cookie.h        |  2 +-
 libcxx/include/__memory/assume_aligned.h      |  2 +-
 .../include/__memory/builtin_new_allocator.h  |  1 -
 libcxx/include/__memory/compressed_pair.h     |  1 +
 libcxx/include/__memory/destruct_n.h          |  2 +-
 libcxx/include/__memory/pointer_traits.h      |  2 +-
 .../include/__memory/raw_storage_iterator.h   |  2 +-
 libcxx/include/__memory/shared_ptr.h          |  2 +-
 libcxx/include/__memory/temporary_buffer.h    |  3 +-
 libcxx/include/__memory/unique_ptr.h          |  3 +-
 .../__memory/unique_temporary_buffer.h        |  2 +-
 libcxx/include/__memory/uses_allocator.h      |  1 -
 .../__memory_resource/memory_resource.h       |  3 +-
 .../monotonic_buffer_resource.h               |  2 +-
 .../__memory_resource/polymorphic_allocator.h |  3 +-
 .../include/__memory_resource/pool_options.h  |  2 +-
 .../synchronized_pool_resource.h              |  1 -
 .../unsynchronized_pool_resource.h            |  2 +-
 libcxx/include/__numeric/midpoint.h           |  2 +-
 libcxx/include/__ostream/basic_ostream.h      |  1 -
 libcxx/include/__pstl/backends/libdispatch.h  |  2 +-
 libcxx/include/__pstl/backends/std_thread.h   |  1 -
 libcxx/include/__pstl/cpu_algos/cpu_traits.h  |  1 -
 libcxx/include/__pstl/cpu_algos/find_if.h     |  1 -
 .../__pstl/cpu_algos/transform_reduce.h       |  1 -
 .../include/__random/discard_block_engine.h   |  2 +-
 .../include/__random/discrete_distribution.h  |  1 -
 .../__random/independent_bits_engine.h        |  2 +-
 libcxx/include/__random/log2.h                |  2 +-
 .../__random/mersenne_twister_engine.h        |  2 +-
 .../piecewise_constant_distribution.h         |  1 +
 .../__random/piecewise_linear_distribution.h  |  1 +
 .../include/__random/shuffle_order_engine.h   |  2 +-
 .../__random/subtract_with_carry_engine.h     |  2 +-
 .../__random/uniform_int_distribution.h       |  2 +-
 libcxx/include/__ranges/access.h              |  2 +-
 libcxx/include/__ranges/counted.h             |  2 +-
 libcxx/include/__ranges/drop_view.h           |  2 +-
 libcxx/include/__ranges/elements_view.h       |  1 -
 libcxx/include/__ranges/empty_view.h          |  2 +-
 libcxx/include/__ranges/istream_view.h        |  2 +-
 libcxx/include/__ranges/repeat_view.h         |  1 +
 libcxx/include/__ranges/single_view.h         |  3 +-
 libcxx/include/__ranges/size.h                |  3 +-
 libcxx/include/__ranges/subrange.h            |  2 +-
 libcxx/include/__ranges/take_view.h           |  1 -
 libcxx/include/__ranges/to.h                  |  2 +-
 libcxx/include/__split_buffer                 |  2 +-
 .../__stop_token/intrusive_shared_ptr.h       |  2 +-
 libcxx/include/__string/char_traits.h         |  1 -
 .../include/__string/constexpr_c_functions.h  |  2 +-
 libcxx/include/__system_error/error_code.h    |  1 -
 .../include/__system_error/error_condition.h  |  1 -
 libcxx/include/__utility/in_place.h           |  2 +-
 libcxx/include/__utility/integer_sequence.h   |  2 +-
 libcxx/include/__utility/pair.h               |  2 +-
 libcxx/include/__utility/priority_tag.h       |  2 +-
 libcxx/include/__utility/small_buffer.h       |  2 +-
 libcxx/include/__utility/swap.h               |  2 +-
 libcxx/include/__variant/monostate.h          |  2 +-
 libcxx/include/array                          |  1 +
 libcxx/include/atomic                         |  1 +
 libcxx/include/barrier                        |  2 +-
 libcxx/include/bitset                         |  1 -
 libcxx/include/charconv                       |  1 +
 libcxx/include/compare                        |  1 +
 libcxx/include/concepts                       |  5 +-
 libcxx/include/coroutine                      |  1 +
 libcxx/include/exception                      |  1 +
 .../include/experimental/__simd/aligned_tag.h |  2 +-
 .../include/experimental/__simd/declaration.h |  2 +-
 .../include/experimental/__simd/reference.h   |  2 +-
 libcxx/include/experimental/__simd/scalar.h   |  2 +-
 libcxx/include/experimental/__simd/simd.h     |  2 +-
 .../include/experimental/__simd/simd_mask.h   |  2 +-
 libcxx/include/experimental/__simd/traits.h   |  2 +-
 libcxx/include/experimental/__simd/utility.h  |  2 +-
 libcxx/include/experimental/__simd/vec_ext.h  |  2 +-
 libcxx/include/experimental/iterator          |  1 +
 libcxx/include/experimental/memory            |  5 +-
 libcxx/include/experimental/propagate_const   |  5 +-
 libcxx/include/experimental/simd              |  4 ++
 libcxx/include/experimental/type_traits       |  4 ++
 libcxx/include/experimental/utility           |  4 ++
 libcxx/include/initializer_list               |  7 ++-
 libcxx/include/iterator                       |  1 +
 libcxx/include/latch                          |  3 +-
 libcxx/include/module.modulemap               | 10 +++-
 libcxx/include/mutex                          |  1 -
 libcxx/include/new                            |  3 +-
 libcxx/include/numbers                        |  1 +
 libcxx/include/semaphore                      |  3 +-
 libcxx/include/span                           |  3 +-
 libcxx/include/stdexcept                      |  1 +
 libcxx/include/stop_token                     |  1 +
 libcxx/include/string_view                    |  3 +-
 libcxx/include/tuple                          |  3 +-
 libcxx/include/typeindex                      |  1 +
 libcxx/include/typeinfo                       |  4 +-
 libcxx/include/utility                        |  1 +
 libcxx/include/valarray                       |  2 +-
 libcxx/include/variant                        |  1 +
 libcxx/src/memory_resource.cpp                |  1 +
 .../random_shuffle.cxx1z.pass.cpp             |  1 +
 .../copy_move_unwrap_reverse.pass.cpp         |  2 +-
 ...using_non_transparent_comparators.pass.cpp |  1 +
 .../sequences/deque/asan_turning_off.pass.cpp |  1 +
 libcxx/test/libcxx/transitive_includes.gen.py |  2 +-
 .../test/libcxx/transitive_includes/cxx03.csv | 19 ++++++
 .../test/libcxx/transitive_includes/cxx11.csv | 19 ++++++
 .../test/libcxx/transitive_includes/cxx14.csv | 19 ++++++
 .../test/libcxx/transitive_includes/cxx17.csv | 19 ++++++
 .../test/libcxx/transitive_includes/cxx20.csv | 27 +++++++++
 .../test/libcxx/transitive_includes/cxx23.csv | 59 ++-----------------
 .../test/libcxx/transitive_includes/cxx26.csv | 59 ++-----------------
 .../template.bitset/includes.pass.cpp         |  6 +-
 .../alg.fill/fill.pass.cpp                    |  1 +
 .../alg.nonmodifying/alg.count/count.pass.cpp |  1 +
 .../alg.count/ranges.count.pass.cpp           |  1 +
 .../mismatch/mismatch.pass.cpp                |  1 +
 .../atomics.types.generic/address.pass.cpp    |  3 +-
 .../invocable.compile.pass.cpp                |  2 +-
 .../regular_invocable.compile.pass.cpp        |  3 +-
 .../equality_comparable.compile.pass.cpp      |  2 +-
 .../equality_comparable_with.compile.pass.cpp |  5 +-
 .../totally_ordered.compile.pass.cpp          |  3 +-
 .../totally_ordered_with.compile.pass.cpp     |  5 +-
 .../constructible_from.compile.pass.cpp       |  1 +
 .../destroy_elements.pass.cpp                 |  1 +
 .../views/mdspan/MinimalElementType.h         |  1 +
 .../mdspan/extents/CtorTestCombinations.h     |  5 +-
 .../views/mdspan/extents/comparison.pass.cpp  |  5 +-
 .../views/mdspan/extents/conversion.pass.cpp  |  6 +-
 .../views/mdspan/extents/ctad.pass.cpp        |  3 +-
 .../views/mdspan/extents/obs_static.pass.cpp  |  3 +-
 .../views/mdspan/extents/types.pass.cpp       |  3 +-
 .../mdspan/layout_left/comparison.pass.cpp    |  5 +-
 .../mdspan/layout_left/ctor.default.pass.cpp  |  3 +-
 .../mdspan/layout_left/ctor.extents.pass.cpp  |  3 +-
 .../layout_left/ctor.layout_right.pass.cpp    |  3 +-
 .../layout_left/ctor.layout_stride.pass.cpp   |  4 +-
 .../mdspan/layout_left/ctor.mapping.pass.cpp  |  3 +-
 .../layout_left/index_operator.pass.cpp       |  5 +-
 .../mdspan/layout_left/properties.pass.cpp    |  5 +-
 .../layout_left/required_span_size.pass.cpp   |  3 +-
 .../layout_left/static_requirements.pass.cpp  |  4 +-
 .../mdspan/layout_right/comparison.pass.cpp   |  5 +-
 .../mdspan/layout_right/ctor.default.pass.cpp |  3 +-
 .../mdspan/layout_right/ctor.extents.pass.cpp |  3 +-
 .../layout_right/ctor.layout_left.pass.cpp    |  3 +-
 .../layout_right/ctor.layout_stride.pass.cpp  |  4 +-
 .../mdspan/layout_right/ctor.mapping.pass.cpp |  3 +-
 .../layout_right/index_operator.pass.cpp      |  3 +-
 .../mdspan/layout_right/properties.pass.cpp   |  5 +-
 .../layout_right/required_span_size.pass.cpp  |  4 +-
 .../layout_right/static_requirements.pass.cpp |  4 +-
 .../layout_stride/ctor.default.pass.cpp       |  3 +-
 .../layout_stride/ctor.extents_array.pass.cpp |  3 +-
 .../layout_stride/ctor.extents_span.pass.cpp  |  3 +-
 .../mdspan/layout_stride/deduction.pass.cpp   |  5 +-
 .../is_exhaustive_corner_case.pass.cpp        |  7 +--
 .../mdspan/layout_stride/properties.pass.cpp  |  3 +-
 .../layout_stride/required_span_size.pass.cpp |  3 +-
 .../static_requirements.pass.cpp              |  4 +-
 .../views/mdspan/mdspan/CustomTestAccessors.h |  3 +-
 .../span.cons/iterator_len.pass.cpp           |  4 +-
 .../span.cons/iterator_sentinel.verify.cpp    |  3 +-
 .../span.objectrep/as_bytes.pass.cpp          |  4 +-
 .../span.objectrep/as_writable_bytes.pass.cpp |  4 +-
 .../syserr/is_error_code_enum.pass.cpp        |  4 +-
 .../syserr/is_error_condition_enum.pass.cpp   |  2 +
 .../simd.class/simd_ctor_broadcast.pass.cpp   |  3 +
 .../test/std/experimental/simd/test_utils.h   |  5 +-
 .../incrementable_traits.compile.pass.cpp     |  5 +-
 .../iter_difference_t.compile.pass.cpp        |  1 +
 ...ndirectly_readable_traits.compile.pass.cpp |  2 +-
 .../back.insert.iterator/types.pass.cpp       |  1 +
 .../front.insert.iterator/types.pass.cpp      |  1 +
 .../insert.iterator/types.pass.cpp            |  2 +
 .../test/std/numerics/bit/byteswap.pass.cpp   |  1 +
 .../exclusive.scan/exclusive_scan.pass.cpp    |  5 +-
 .../inclusive.scan/inclusive_scan.pass.cpp    |  5 +-
 .../inclusive.scan/inclusive_scan_op.pass.cpp |  4 +-
 .../inclusive_scan_op_init.pass.cpp           |  4 +-
 ...sform_exclusive_scan_init_bop_uop.pass.cpp |  5 +-
 .../transform_inclusive_scan_bop_uop.pass.cpp |  5 +-
 ...sform_inclusive_scan_bop_uop_init.pass.cpp |  5 +-
 .../rand.dist.samp.plinear/eval.pass.cpp      |  5 +-
 .../eval_param.pass.cpp                       |  5 +-
 .../range.chunk.by/ctor.default.pass.cpp      |  1 +
 .../range.chunk.by.iter/deref.pass.cpp        |  3 +-
 .../iterator/member_typedefs.compile.pass.cpp |  1 +
 .../range.repeat.view/iterator/minus.pass.cpp |  4 +-
 .../range.subrange/ctad.compile.pass.cpp      |  3 +-
 .../remove_prefix.pass.cpp                    |  3 +-
 .../remove_suffix.pass.cpp                    |  3 +-
 .../string.view.modifiers/swap.pass.cpp       |  3 +-
 .../string.view/string.view.ops/copy.pass.cpp |  3 +-
 .../string.view.ops/substr.pass.cpp           |  5 +-
 .../func.search/func.search.bm/hash.pass.cpp  |  2 +-
 .../func.search.bm/hash.pred.pass.cpp         |  2 +-
 .../func.search/func.search.bmh/hash.pass.cpp |  4 +-
 .../func.search.bmh/hash.pred.pass.cpp        |  4 +-
 .../func.wrap.func.con/deduct_F.pass.cpp      |  3 +-
 .../unord.hash/pointer.pass.cpp               |  4 +-
 .../construct_at.pass.cpp                     |  3 +-
 .../temporary.buffer/overaligned.pass.cpp     |  3 +-
 .../temporary_buffer.pass.cpp                 |  5 +-
 .../meta/meta.rel/is_invocable.pass.cpp       |  5 +-
 .../meta.rel/is_nothrow_invocable.pass.cpp    |  4 +-
 .../allocate_overaligned_request.pass.cpp     |  3 +-
 ...sync_allocate_overaligned_request.pass.cpp |  3 +-
 .../sync_deallocate_matches_allocate.pass.cpp |  3 +-
 ...sync_allocate_overaligned_request.pass.cpp |  3 +-
 ...nsync_deallocate_matches_allocate.pass.cpp |  3 +-
 libcxx/utils/generate_escaped_output_table.py |  2 +-
 ...enerate_extended_grapheme_cluster_table.py |  2 +-
 .../generate_indic_conjunct_break_table.py    |  2 +-
 .../utils/generate_width_estimation_table.py  |  2 +-
 libcxxabi/src/private_typeinfo.cpp            |  4 +-
 libcxxabi/test/test_aux_runtime.pass.cpp      | 10 +++-
 290 files changed, 493 insertions(+), 437 deletions(-)

diff --git a/libcxx/include/__algorithm/copy_move_common.h b/libcxx/include/__algorithm/copy_move_common.h
index 8a98451a8f96..d76bf4903aaa 100644
--- a/libcxx/include/__algorithm/copy_move_common.h
+++ b/libcxx/include/__algorithm/copy_move_common.h
@@ -13,6 +13,7 @@
 #include <__algorithm/unwrap_iter.h>
 #include <__algorithm/unwrap_range.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__iterator/iterator_traits.h>
 #include <__memory/pointer_traits.h>
 #include <__string/constexpr_c_functions.h>
@@ -24,7 +25,6 @@
 #include <__type_traits/is_volatile.h>
 #include <__utility/move.h>
 #include <__utility/pair.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__algorithm/inplace_merge.h b/libcxx/include/__algorithm/inplace_merge.h
index 62a8bc53e23f..ad3fe6a7a505 100644
--- a/libcxx/include/__algorithm/inplace_merge.h
+++ b/libcxx/include/__algorithm/inplace_merge.h
@@ -18,6 +18,7 @@
 #include <__algorithm/rotate.h>
 #include <__algorithm/upper_bound.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__functional/identity.h>
 #include <__iterator/advance.h>
 #include <__iterator/distance.h>
diff --git a/libcxx/include/__algorithm/mismatch.h b/libcxx/include/__algorithm/mismatch.h
index cb83347584b1..556bd4216307 100644
--- a/libcxx/include/__algorithm/mismatch.h
+++ b/libcxx/include/__algorithm/mismatch.h
@@ -15,6 +15,7 @@
 #include <__algorithm/simd_utils.h>
 #include <__algorithm/unwrap_iter.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__functional/identity.h>
 #include <__iterator/aliasing_iterator.h>
 #include <__iterator/iterator_traits.h>
@@ -27,7 +28,6 @@
 #include <__utility/move.h>
 #include <__utility/pair.h>
 #include <__utility/unreachable.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__algorithm/shuffle.h b/libcxx/include/__algorithm/shuffle.h
index c9c56ce8c2c0..7177fbb469ba 100644
--- a/libcxx/include/__algorithm/shuffle.h
+++ b/libcxx/include/__algorithm/shuffle.h
@@ -11,12 +11,12 @@
 
 #include <__algorithm/iterator_operations.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__iterator/iterator_traits.h>
 #include <__random/uniform_int_distribution.h>
 #include <__utility/forward.h>
 #include <__utility/move.h>
 #include <__utility/swap.h>
-#include <cstddef>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__algorithm/simd_utils.h b/libcxx/include/__algorithm/simd_utils.h
index 56518dafa319..4e3e4f2b9404 100644
--- a/libcxx/include/__algorithm/simd_utils.h
+++ b/libcxx/include/__algorithm/simd_utils.h
@@ -14,10 +14,10 @@
 #include <__bit/countl.h>
 #include <__bit/countr.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/is_arithmetic.h>
 #include <__type_traits/is_same.h>
 #include <__utility/integer_sequence.h>
-#include <cstddef>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__algorithm/stable_partition.h b/libcxx/include/__algorithm/stable_partition.h
index 5df5e8eaf689..0438f589a39d 100644
--- a/libcxx/include/__algorithm/stable_partition.h
+++ b/libcxx/include/__algorithm/stable_partition.h
@@ -12,6 +12,7 @@
 #include <__algorithm/iterator_operations.h>
 #include <__algorithm/rotate.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__iterator/advance.h>
 #include <__iterator/distance.h>
 #include <__iterator/iterator_traits.h>
diff --git a/libcxx/include/__algorithm/stable_sort.h b/libcxx/include/__algorithm/stable_sort.h
index ec556aad82e8..43f591ac02b0 100644
--- a/libcxx/include/__algorithm/stable_sort.h
+++ b/libcxx/include/__algorithm/stable_sort.h
@@ -15,6 +15,7 @@
 #include <__algorithm/iterator_operations.h>
 #include <__algorithm/sort.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__debug_utils/strict_weak_ordering_check.h>
 #include <__iterator/iterator_traits.h>
 #include <__memory/destruct_n.h>
diff --git a/libcxx/include/__atomic/aliases.h b/libcxx/include/__atomic/aliases.h
index afc64eaaa69e..3c84747bebb8 100644
--- a/libcxx/include/__atomic/aliases.h
+++ b/libcxx/include/__atomic/aliases.h
@@ -14,9 +14,10 @@
 #include <__atomic/contention_t.h>
 #include <__atomic/is_always_lock_free.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
+#include <__cstddef/size_t.h>
 #include <__type_traits/conditional.h>
 #include <__type_traits/make_unsigned.h>
-#include <cstddef>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__atomic/atomic.h b/libcxx/include/__atomic/atomic.h
index af6d12b5e4ce..113475cb1f00 100644
--- a/libcxx/include/__atomic/atomic.h
+++ b/libcxx/include/__atomic/atomic.h
@@ -14,6 +14,7 @@
 #include <__atomic/cxx_atomic_impl.h>
 #include <__atomic/memory_order.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__functional/operations.h>
 #include <__memory/addressof.h>
 #include <__type_traits/enable_if.h>
@@ -25,7 +26,6 @@
 #include <__type_traits/remove_pointer.h>
 #include <__type_traits/remove_volatile.h>
 #include <__utility/forward.h>
-#include <cstddef>
 #include <cstring>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index 465cd9a77ea7..eef15983b983 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -25,10 +25,11 @@
 #include <__concepts/arithmetic.h>
 #include <__concepts/same_as.h>
 #include <__config>
+#include <__cstddef/byte.h>
+#include <__cstddef/ptrdiff_t.h>
 #include <__memory/addressof.h>
 #include <__type_traits/has_unique_object_representation.h>
 #include <__type_traits/is_trivially_copyable.h>
-#include <cstddef>
 #include <cstdint>
 #include <cstring>
 
diff --git a/libcxx/include/__atomic/cxx_atomic_impl.h b/libcxx/include/__atomic/cxx_atomic_impl.h
index 18e88aa97bec..86a57d1d5d8f 100644
--- a/libcxx/include/__atomic/cxx_atomic_impl.h
+++ b/libcxx/include/__atomic/cxx_atomic_impl.h
@@ -12,11 +12,12 @@
 #include <__atomic/memory_order.h>
 #include <__atomic/to_gcc_order.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__memory/addressof.h>
+#include <__type_traits/enable_if.h>
 #include <__type_traits/is_assignable.h>
 #include <__type_traits/is_trivially_copyable.h>
 #include <__type_traits/remove_const.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__charconv/from_chars_floating_point.h b/libcxx/include/__charconv/from_chars_floating_point.h
index 5cd3fc4a41ea..811e518a81db 100644
--- a/libcxx/include/__charconv/from_chars_floating_point.h
+++ b/libcxx/include/__charconv/from_chars_floating_point.h
@@ -14,8 +14,8 @@
 #include <__charconv/chars_format.h>
 #include <__charconv/from_chars_result.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__system_error/errc.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__charconv/to_chars_integral.h b/libcxx/include/__charconv/to_chars_integral.h
index fd92be4b4ce9..710299df9b4d 100644
--- a/libcxx/include/__charconv/to_chars_integral.h
+++ b/libcxx/include/__charconv/to_chars_integral.h
@@ -18,6 +18,7 @@
 #include <__charconv/to_chars_result.h>
 #include <__charconv/traits.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__system_error/errc.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/integral_constant.h>
@@ -26,7 +27,6 @@
 #include <__type_traits/make_32_64_or_128_bit.h>
 #include <__type_traits/make_unsigned.h>
 #include <__utility/unreachable.h>
-#include <cstddef>
 #include <cstdint>
 #include <limits>
 
diff --git a/libcxx/include/__compare/common_comparison_category.h b/libcxx/include/__compare/common_comparison_category.h
index 7aeb3da03a4f..215922abad6b 100644
--- a/libcxx/include/__compare/common_comparison_category.h
+++ b/libcxx/include/__compare/common_comparison_category.h
@@ -11,8 +11,8 @@
 
 #include <__compare/ordering.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/is_same.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__concepts/swappable.h b/libcxx/include/__concepts/swappable.h
index d339488a087a..985c733021a0 100644
--- a/libcxx/include/__concepts/swappable.h
+++ b/libcxx/include/__concepts/swappable.h
@@ -14,6 +14,7 @@
 #include <__concepts/common_reference_with.h>
 #include <__concepts/constructible.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/extent.h>
 #include <__type_traits/is_nothrow_assignable.h>
 #include <__type_traits/is_nothrow_constructible.h>
@@ -22,7 +23,6 @@
 #include <__utility/forward.h>
 #include <__utility/move.h>
 #include <__utility/swap.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__coroutine/coroutine_handle.h b/libcxx/include/__coroutine/coroutine_handle.h
index 4557a6643c23..e2cde20498d8 100644
--- a/libcxx/include/__coroutine/coroutine_handle.h
+++ b/libcxx/include/__coroutine/coroutine_handle.h
@@ -11,11 +11,12 @@
 
 #include <__assert>
 #include <__config>
+#include <__cstddef/nullptr_t.h>
+#include <__cstddef/size_t.h>
 #include <__functional/hash.h>
 #include <__memory/addressof.h>
 #include <__type_traits/remove_cv.h>
 #include <compare>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__exception/exception_ptr.h b/libcxx/include/__exception/exception_ptr.h
index 2cb250dfd42d..32c56fc5cc12 100644
--- a/libcxx/include/__exception/exception_ptr.h
+++ b/libcxx/include/__exception/exception_ptr.h
@@ -14,7 +14,6 @@
 #include <__memory/addressof.h>
 #include <__memory/construct_at.h>
 #include <__type_traits/decay.h>
-#include <cstddef>
 #include <cstdlib>
 #include <new>
 #include <typeinfo>
diff --git a/libcxx/include/__exception/nested_exception.h b/libcxx/include/__exception/nested_exception.h
index cc20b038c871..d560b6bbc35a 100644
--- a/libcxx/include/__exception/nested_exception.h
+++ b/libcxx/include/__exception/nested_exception.h
@@ -22,7 +22,6 @@
 #include <__type_traits/is_final.h>
 #include <__type_traits/is_polymorphic.h>
 #include <__utility/forward.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__exception/operations.h b/libcxx/include/__exception/operations.h
index c8744eb297a4..15520c558a0b 100644
--- a/libcxx/include/__exception/operations.h
+++ b/libcxx/include/__exception/operations.h
@@ -10,7 +10,6 @@
 #define _LIBCPP___EXCEPTION_OPERATIONS_H
 
 #include <__config>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__filesystem/directory_iterator.h b/libcxx/include/__filesystem/directory_iterator.h
index e0246d8001e1..621e9bf9258e 100644
--- a/libcxx/include/__filesystem/directory_iterator.h
+++ b/libcxx/include/__filesystem/directory_iterator.h
@@ -22,7 +22,6 @@
 #include <__ranges/enable_view.h>
 #include <__system_error/error_code.h>
 #include <__utility/move.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__filesystem/path.h b/libcxx/include/__filesystem/path.h
index 2eb60810af02..88c800fdf86d 100644
--- a/libcxx/include/__filesystem/path.h
+++ b/libcxx/include/__filesystem/path.h
@@ -22,7 +22,6 @@
 #include <__type_traits/remove_const.h>
 #include <__type_traits/remove_pointer.h>
 #include <__utility/move.h>
-#include <cstddef>
 #include <string>
 #include <string_view>
 
diff --git a/libcxx/include/__filesystem/path_iterator.h b/libcxx/include/__filesystem/path_iterator.h
index f4d486d86cf3..e0f601662d46 100644
--- a/libcxx/include/__filesystem/path_iterator.h
+++ b/libcxx/include/__filesystem/path_iterator.h
@@ -14,9 +14,6 @@
 #include <__config>
 #include <__filesystem/path.h>
 #include <__iterator/iterator_traits.h>
-#include <cstddef>
-#include <string>
-#include <string_view>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__filesystem/recursive_directory_iterator.h b/libcxx/include/__filesystem/recursive_directory_iterator.h
index caa1396eb301..1be92a8bd5f4 100644
--- a/libcxx/include/__filesystem/recursive_directory_iterator.h
+++ b/libcxx/include/__filesystem/recursive_directory_iterator.h
@@ -21,7 +21,6 @@
 #include <__ranges/enable_view.h>
 #include <__system_error/error_code.h>
 #include <__utility/move.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__flat_map/flat_map.h b/libcxx/include/__flat_map/flat_map.h
index 9ca32d5295bd..5c14c0ac693b 100644
--- a/libcxx/include/__flat_map/flat_map.h
+++ b/libcxx/include/__flat_map/flat_map.h
@@ -23,6 +23,7 @@
 #include <__concepts/convertible_to.h>
 #include <__concepts/swappable.h>
 #include <__config>
+#include <__cstddef/byte.h>
 #include <__flat_map/sorted_unique.h>
 #include <__functional/invoke.h>
 #include <__functional/is_transparent.h>
diff --git a/libcxx/include/__format/buffer.h b/libcxx/include/__format/buffer.h
index ce9ac0c81e31..618b8ef02564 100644
--- a/libcxx/include/__format/buffer.h
+++ b/libcxx/include/__format/buffer.h
@@ -37,7 +37,6 @@
 #include <__type_traits/conditional.h>
 #include <__utility/exception_guard.h>
 #include <__utility/move.h>
-#include <cstddef>
 #include <string_view>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__format/escaped_output_table.h b/libcxx/include/__format/escaped_output_table.h
index bdf86cb6f99c..7a0b35239861 100644
--- a/libcxx/include/__format/escaped_output_table.h
+++ b/libcxx/include/__format/escaped_output_table.h
@@ -63,7 +63,7 @@
 
 #include <__algorithm/ranges_upper_bound.h>
 #include <__config>
-#include <cstddef>
+#include <__cstddef/ptrdiff_t.h>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__format/extended_grapheme_cluster_table.h b/libcxx/include/__format/extended_grapheme_cluster_table.h
index 7dbc239f5f5c..7653a9e03b81 100644
--- a/libcxx/include/__format/extended_grapheme_cluster_table.h
+++ b/libcxx/include/__format/extended_grapheme_cluster_table.h
@@ -63,8 +63,8 @@
 
 #include <__algorithm/ranges_upper_bound.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__iterator/access.h>
-#include <cstddef>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__format/format_arg.h b/libcxx/include/__format/format_arg.h
index 3f39d2aec81f..a973ccd43c42 100644
--- a/libcxx/include/__format/format_arg.h
+++ b/libcxx/include/__format/format_arg.h
@@ -13,6 +13,7 @@
 #include <__assert>
 #include <__concepts/arithmetic.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__format/concepts.h>
 #include <__format/format_parse_context.h>
 #include <__functional/invoke.h>
diff --git a/libcxx/include/__format/format_args.h b/libcxx/include/__format/format_args.h
index e19b4458e41a..b98663c06ea4 100644
--- a/libcxx/include/__format/format_args.h
+++ b/libcxx/include/__format/format_args.h
@@ -11,10 +11,10 @@
 #define _LIBCPP___FORMAT_FORMAT_ARGS_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__format/format_arg.h>
 #include <__format/format_arg_store.h>
 #include <__fwd/format.h>
-#include <cstddef>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__format/format_context.h b/libcxx/include/__format/format_context.h
index 019a71011a71..ecbf62770cde 100644
--- a/libcxx/include/__format/format_context.h
+++ b/libcxx/include/__format/format_context.h
@@ -23,7 +23,6 @@
 #include <__memory/addressof.h>
 #include <__utility/move.h>
 #include <__variant/monostate.h>
-#include <cstddef>
 
 #ifndef _LIBCPP_HAS_NO_LOCALIZATION
 #  include <__locale>
diff --git a/libcxx/include/__format/format_string.h b/libcxx/include/__format/format_string.h
index a499afee8874..5db5973dd588 100644
--- a/libcxx/include/__format/format_string.h
+++ b/libcxx/include/__format/format_string.h
@@ -12,10 +12,10 @@
 
 #include <__assert>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__format/format_error.h>
 #include <__iterator/concepts.h>
 #include <__iterator/iterator_traits.h> // iter_value_t
-#include <cstddef>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__format/formatter_floating_point.h b/libcxx/include/__format/formatter_floating_point.h
index fc95dd3f22bb..9ffe20a1a72a 100644
--- a/libcxx/include/__format/formatter_floating_point.h
+++ b/libcxx/include/__format/formatter_floating_point.h
@@ -36,7 +36,6 @@
 #include <__utility/move.h>
 #include <__utility/unreachable.h>
 #include <cmath>
-#include <cstddef>
 
 #ifndef _LIBCPP_HAS_NO_LOCALIZATION
 #  include <__locale>
diff --git a/libcxx/include/__format/formatter_output.h b/libcxx/include/__format/formatter_output.h
index 34c4c87313a4..457f5f53b2dc 100644
--- a/libcxx/include/__format/formatter_output.h
+++ b/libcxx/include/__format/formatter_output.h
@@ -16,6 +16,8 @@
 #include <__bit/countl.h>
 #include <__concepts/same_as.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
+#include <__cstddef/size_t.h>
 #include <__format/buffer.h>
 #include <__format/concepts.h>
 #include <__format/formatter.h>
@@ -28,7 +30,6 @@
 #include <__memory/pointer_traits.h>
 #include <__utility/move.h>
 #include <__utility/unreachable.h>
-#include <cstddef>
 #include <string_view>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__format/formatter_pointer.h b/libcxx/include/__format/formatter_pointer.h
index 6e0fa9a1b4f1..4ef48c168d0d 100644
--- a/libcxx/include/__format/formatter_pointer.h
+++ b/libcxx/include/__format/formatter_pointer.h
@@ -11,13 +11,13 @@
 #define _LIBCPP___FORMAT_FORMATTER_POINTER_H
 
 #include <__config>
+#include <__cstddef/nullptr_t.h>
 #include <__format/concepts.h>
 #include <__format/format_parse_context.h>
 #include <__format/formatter.h>
 #include <__format/formatter_integral.h>
 #include <__format/formatter_output.h>
 #include <__format/parser_std_format_spec.h>
-#include <cstddef>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__format/indic_conjunct_break_table.h b/libcxx/include/__format/indic_conjunct_break_table.h
index 39dd45da771f..df6cfe6a02f3 100644
--- a/libcxx/include/__format/indic_conjunct_break_table.h
+++ b/libcxx/include/__format/indic_conjunct_break_table.h
@@ -63,8 +63,8 @@
 
 #include <__algorithm/ranges_upper_bound.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__iterator/access.h>
-#include <cstddef>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__format/width_estimation_table.h b/libcxx/include/__format/width_estimation_table.h
index 23a08746b910..5b4b3950c6a1 100644
--- a/libcxx/include/__format/width_estimation_table.h
+++ b/libcxx/include/__format/width_estimation_table.h
@@ -63,7 +63,7 @@
 
 #include <__algorithm/ranges_upper_bound.h>
 #include <__config>
-#include <cstddef>
+#include <__cstddef/ptrdiff_t.h>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__functional/bind.h b/libcxx/include/__functional/bind.h
index 4251ef74ab7b..f82c1517249b 100644
--- a/libcxx/include/__functional/bind.h
+++ b/libcxx/include/__functional/bind.h
@@ -17,7 +17,6 @@
 #include <__type_traits/invoke.h>
 #include <__type_traits/is_reference_wrapper.h>
 #include <__type_traits/is_void.h>
-#include <cstddef>
 #include <tuple>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__functional/hash.h b/libcxx/include/__functional/hash.h
index 8abec99f7290..f7b89f759b5f 100644
--- a/libcxx/include/__functional/hash.h
+++ b/libcxx/include/__functional/hash.h
@@ -20,7 +20,6 @@
 #include <__type_traits/underlying_type.h>
 #include <__utility/pair.h>
 #include <__utility/swap.h>
-#include <cstddef>
 #include <cstdint>
 #include <cstring>
 
diff --git a/libcxx/include/__hash_table b/libcxx/include/__hash_table
index 560e873adc38..8e4cb3c914dc 100644
--- a/libcxx/include/__hash_table
+++ b/libcxx/include/__hash_table
@@ -15,6 +15,7 @@
 #include <__assert>
 #include <__bit/countl.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__functional/hash.h>
 #include <__iterator/iterator_traits.h>
 #include <__math/rounding_functions.h>
diff --git a/libcxx/include/__iterator/access.h b/libcxx/include/__iterator/access.h
index acc4f60bf697..d42855f92548 100644
--- a/libcxx/include/__iterator/access.h
+++ b/libcxx/include/__iterator/access.h
@@ -11,7 +11,7 @@
 #define _LIBCPP___ITERATOR_ACCESS_H
 
 #include <__config>
-#include <cstddef>
+#include <__cstddef/size_t.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__iterator/aliasing_iterator.h b/libcxx/include/__iterator/aliasing_iterator.h
index 94ba577078b5..aeb5b4a88ec3 100644
--- a/libcxx/include/__iterator/aliasing_iterator.h
+++ b/libcxx/include/__iterator/aliasing_iterator.h
@@ -10,10 +10,10 @@
 #define _LIBCPP___ITERATOR_ALIASING_ITERATOR_H
 
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__iterator/iterator_traits.h>
 #include <__memory/pointer_traits.h>
 #include <__type_traits/is_trivial.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__iterator/back_insert_iterator.h b/libcxx/include/__iterator/back_insert_iterator.h
index 6d3dd4b12966..9a5948753388 100644
--- a/libcxx/include/__iterator/back_insert_iterator.h
+++ b/libcxx/include/__iterator/back_insert_iterator.h
@@ -11,11 +11,11 @@
 #define _LIBCPP___ITERATOR_BACK_INSERT_ITERATOR_H
 
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__iterator/iterator.h>
 #include <__iterator/iterator_traits.h>
 #include <__memory/addressof.h>
 #include <__utility/move.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__iterator/data.h b/libcxx/include/__iterator/data.h
index b7c1603652b0..5f2624c2b819 100644
--- a/libcxx/include/__iterator/data.h
+++ b/libcxx/include/__iterator/data.h
@@ -11,7 +11,6 @@
 #define _LIBCPP___ITERATOR_DATA_H
 
 #include <__config>
-#include <cstddef>
 #include <initializer_list>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__iterator/empty.h b/libcxx/include/__iterator/empty.h
index 773f2776955b..f2c653bcb329 100644
--- a/libcxx/include/__iterator/empty.h
+++ b/libcxx/include/__iterator/empty.h
@@ -11,7 +11,6 @@
 #define _LIBCPP___ITERATOR_EMPTY_H
 
 #include <__config>
-#include <cstddef>
 #include <initializer_list>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__iterator/front_insert_iterator.h b/libcxx/include/__iterator/front_insert_iterator.h
index 7f2c54ec8744..80819cd22ae6 100644
--- a/libcxx/include/__iterator/front_insert_iterator.h
+++ b/libcxx/include/__iterator/front_insert_iterator.h
@@ -11,11 +11,11 @@
 #define _LIBCPP___ITERATOR_FRONT_INSERT_ITERATOR_H
 
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__iterator/iterator.h>
 #include <__iterator/iterator_traits.h>
 #include <__memory/addressof.h>
 #include <__utility/move.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__iterator/incrementable_traits.h b/libcxx/include/__iterator/incrementable_traits.h
index a228b228f6e5..37c8daddf8a8 100644
--- a/libcxx/include/__iterator/incrementable_traits.h
+++ b/libcxx/include/__iterator/incrementable_traits.h
@@ -12,13 +12,13 @@
 
 #include <__concepts/arithmetic.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__type_traits/conditional.h>
 #include <__type_traits/is_object.h>
 #include <__type_traits/is_primary_template.h>
 #include <__type_traits/make_signed.h>
 #include <__type_traits/remove_cvref.h>
 #include <__utility/declval.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__iterator/insert_iterator.h b/libcxx/include/__iterator/insert_iterator.h
index 8b7574dc9ec0..b3311042014f 100644
--- a/libcxx/include/__iterator/insert_iterator.h
+++ b/libcxx/include/__iterator/insert_iterator.h
@@ -11,12 +11,12 @@
 #define _LIBCPP___ITERATOR_INSERT_ITERATOR_H
 
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__iterator/iterator.h>
 #include <__iterator/iterator_traits.h>
 #include <__memory/addressof.h>
 #include <__ranges/access.h>
 #include <__utility/move.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__iterator/istream_iterator.h b/libcxx/include/__iterator/istream_iterator.h
index 58c9ac6d4ccc..a6c74d00178d 100644
--- a/libcxx/include/__iterator/istream_iterator.h
+++ b/libcxx/include/__iterator/istream_iterator.h
@@ -11,13 +11,13 @@
 #define _LIBCPP___ITERATOR_ISTREAM_ITERATOR_H
 
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__fwd/istream.h>
 #include <__fwd/string.h>
 #include <__iterator/default_sentinel.h>
 #include <__iterator/iterator.h>
 #include <__iterator/iterator_traits.h>
 #include <__memory/addressof.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__iterator/istreambuf_iterator.h b/libcxx/include/__iterator/istreambuf_iterator.h
index 51c4ecff351f..162873b9559e 100644
--- a/libcxx/include/__iterator/istreambuf_iterator.h
+++ b/libcxx/include/__iterator/istreambuf_iterator.h
@@ -16,6 +16,8 @@
 #include <__iterator/default_sentinel.h>
 #include <__iterator/iterator.h>
 #include <__iterator/iterator_traits.h>
+#include <__string/char_traits.h>
+#include <iosfwd>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__iterator/iterator.h b/libcxx/include/__iterator/iterator.h
index ba9308f3c224..1591655313dd 100644
--- a/libcxx/include/__iterator/iterator.h
+++ b/libcxx/include/__iterator/iterator.h
@@ -11,7 +11,7 @@
 #define _LIBCPP___ITERATOR_ITERATOR_H
 
 #include <__config>
-#include <cstddef>
+#include <__cstddef/ptrdiff_t.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__iterator/iterator_traits.h b/libcxx/include/__iterator/iterator_traits.h
index 4d9ad480cc4a..eb6ba8b62fb3 100644
--- a/libcxx/include/__iterator/iterator_traits.h
+++ b/libcxx/include/__iterator/iterator_traits.h
@@ -18,6 +18,7 @@
 #include <__concepts/same_as.h>
 #include <__concepts/totally_ordered.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__fwd/pair.h>
 #include <__iterator/incrementable_traits.h>
 #include <__iterator/readable_traits.h>
@@ -36,7 +37,6 @@
 #include <__type_traits/remove_cvref.h>
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__iterator/ostream_iterator.h b/libcxx/include/__iterator/ostream_iterator.h
index 05697e62d9dc..93ecc03010d0 100644
--- a/libcxx/include/__iterator/ostream_iterator.h
+++ b/libcxx/include/__iterator/ostream_iterator.h
@@ -11,12 +11,12 @@
 #define _LIBCPP___ITERATOR_OSTREAM_ITERATOR_H
 
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__fwd/ostream.h>
 #include <__fwd/string.h>
 #include <__iterator/iterator.h>
 #include <__iterator/iterator_traits.h>
 #include <__memory/addressof.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__iterator/ostreambuf_iterator.h b/libcxx/include/__iterator/ostreambuf_iterator.h
index 401b6f3f2360..621ffd4f988c 100644
--- a/libcxx/include/__iterator/ostreambuf_iterator.h
+++ b/libcxx/include/__iterator/ostreambuf_iterator.h
@@ -11,12 +11,12 @@
 #define _LIBCPP___ITERATOR_OSTREAMBUF_ITERATOR_H
 
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__fwd/ios.h>
 #include <__fwd/ostream.h>
 #include <__fwd/streambuf.h>
 #include <__iterator/iterator.h>
 #include <__iterator/iterator_traits.h>
-#include <cstddef>
 #include <iosfwd> // for forward declaration of ostreambuf_iterator
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__iterator/reverse_access.h b/libcxx/include/__iterator/reverse_access.h
index 54d7270b04a5..f6e60c3fb75b 100644
--- a/libcxx/include/__iterator/reverse_access.h
+++ b/libcxx/include/__iterator/reverse_access.h
@@ -12,7 +12,6 @@
 
 #include <__config>
 #include <__iterator/reverse_iterator.h>
-#include <cstddef>
 #include <initializer_list>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__iterator/segmented_iterator.h b/libcxx/include/__iterator/segmented_iterator.h
index f3cd1e5fa1f5..8cb54a35a7f5 100644
--- a/libcxx/include/__iterator/segmented_iterator.h
+++ b/libcxx/include/__iterator/segmented_iterator.h
@@ -41,8 +41,8 @@
 //   Returns the iterator composed of the segment iterator and local iterator.
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/integral_constant.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__iterator/size.h b/libcxx/include/__iterator/size.h
index 876e6963f77d..84e2e3b21f1d 100644
--- a/libcxx/include/__iterator/size.h
+++ b/libcxx/include/__iterator/size.h
@@ -11,9 +11,10 @@
 #define _LIBCPP___ITERATOR_SIZE_H
 
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
+#include <__cstddef/size_t.h>
 #include <__type_traits/common_type.h>
 #include <__type_traits/make_signed.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__iterator/wrap_iter.h b/libcxx/include/__iterator/wrap_iter.h
index 549d8ff2dbd7..2856833e6007 100644
--- a/libcxx/include/__iterator/wrap_iter.h
+++ b/libcxx/include/__iterator/wrap_iter.h
@@ -13,13 +13,13 @@
 #include <__compare/ordering.h>
 #include <__compare/three_way_comparable.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__iterator/iterator_traits.h>
 #include <__memory/addressof.h>
 #include <__memory/pointer_traits.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_convertible.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__mdspan/default_accessor.h b/libcxx/include/__mdspan/default_accessor.h
index 1cc5f15545fc..d6f3ddb998e9 100644
--- a/libcxx/include/__mdspan/default_accessor.h
+++ b/libcxx/include/__mdspan/default_accessor.h
@@ -18,12 +18,11 @@
 #define _LIBCPP___MDSPAN_DEFAULT_ACCESSOR_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/is_abstract.h>
 #include <__type_traits/is_array.h>
 #include <__type_traits/is_convertible.h>
 #include <__type_traits/remove_const.h>
-#include <cinttypes>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__mdspan/extents.h b/libcxx/include/__mdspan/extents.h
index 3d2c2771a834..edbc30a7a40e 100644
--- a/libcxx/include/__mdspan/extents.h
+++ b/libcxx/include/__mdspan/extents.h
@@ -21,6 +21,7 @@
 #include <__config>
 
 #include <__concepts/arithmetic.h>
+#include <__cstddef/byte.h>
 #include <__type_traits/common_type.h>
 #include <__type_traits/is_convertible.h>
 #include <__type_traits/is_nothrow_constructible.h>
@@ -29,9 +30,7 @@
 #include <__utility/integer_sequence.h>
 #include <__utility/unreachable.h>
 #include <array>
-#include <cinttypes>
 #include <concepts>
-#include <cstddef>
 #include <limits>
 #include <span>
 
diff --git a/libcxx/include/__mdspan/layout_left.h b/libcxx/include/__mdspan/layout_left.h
index 59574e83b0d7..288b3dd8038e 100644
--- a/libcxx/include/__mdspan/layout_left.h
+++ b/libcxx/include/__mdspan/layout_left.h
@@ -27,9 +27,6 @@
 #include <__type_traits/is_nothrow_constructible.h>
 #include <__utility/integer_sequence.h>
 #include <array>
-#include <cinttypes>
-#include <cstddef>
-#include <limits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__mdspan/layout_right.h b/libcxx/include/__mdspan/layout_right.h
index d1acdb41238f..72922d1049c7 100644
--- a/libcxx/include/__mdspan/layout_right.h
+++ b/libcxx/include/__mdspan/layout_right.h
@@ -19,6 +19,7 @@
 
 #include <__assert>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__fwd/mdspan.h>
 #include <__mdspan/extents.h>
 #include <__type_traits/common_type.h>
@@ -26,9 +27,6 @@
 #include <__type_traits/is_convertible.h>
 #include <__type_traits/is_nothrow_constructible.h>
 #include <__utility/integer_sequence.h>
-#include <cinttypes>
-#include <cstddef>
-#include <limits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__mdspan/layout_stride.h b/libcxx/include/__mdspan/layout_stride.h
index c57f596431c7..bb93de977514 100644
--- a/libcxx/include/__mdspan/layout_stride.h
+++ b/libcxx/include/__mdspan/layout_stride.h
@@ -32,8 +32,6 @@
 #include <__utility/integer_sequence.h>
 #include <__utility/swap.h>
 #include <array>
-#include <cinttypes>
-#include <cstddef>
 #include <limits>
 #include <span>
 
diff --git a/libcxx/include/__mdspan/mdspan.h b/libcxx/include/__mdspan/mdspan.h
index 1ff4fd4ba4a8..3f9b35b185b1 100644
--- a/libcxx/include/__mdspan/mdspan.h
+++ b/libcxx/include/__mdspan/mdspan.h
@@ -37,9 +37,6 @@
 #include <__type_traits/remove_reference.h>
 #include <__utility/integer_sequence.h>
 #include <array>
-#include <cinttypes>
-#include <cstddef>
-#include <limits>
 #include <span>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__memory/align.h b/libcxx/include/__memory/align.h
index bbb995f4a8c8..402eac338092 100644
--- a/libcxx/include/__memory/align.h
+++ b/libcxx/include/__memory/align.h
@@ -10,7 +10,7 @@
 #define _LIBCPP___MEMORY_ALIGN_H
 
 #include <__config>
-#include <cstddef>
+#include <__cstddef/size_t.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__memory/aligned_alloc.h b/libcxx/include/__memory/aligned_alloc.h
index 33fe8af77df7..fb36983d9c3d 100644
--- a/libcxx/include/__memory/aligned_alloc.h
+++ b/libcxx/include/__memory/aligned_alloc.h
@@ -10,7 +10,6 @@
 #define _LIBCPP___MEMORY_ALIGNED_ALLOC_H
 
 #include <__config>
-#include <cstddef>
 #include <cstdlib>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__memory/allocate_at_least.h b/libcxx/include/__memory/allocate_at_least.h
index a10e4fbaead3..9b5a8bcbd459 100644
--- a/libcxx/include/__memory/allocate_at_least.h
+++ b/libcxx/include/__memory/allocate_at_least.h
@@ -10,8 +10,8 @@
 #define _LIBCPP___MEMORY_ALLOCATE_AT_LEAST_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__memory/allocator_traits.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__memory/allocation_guard.h b/libcxx/include/__memory/allocation_guard.h
index cb870af7be67..66d6a5002c29 100644
--- a/libcxx/include/__memory/allocation_guard.h
+++ b/libcxx/include/__memory/allocation_guard.h
@@ -14,7 +14,6 @@
 #include <__memory/addressof.h>
 #include <__memory/allocator_traits.h>
 #include <__utility/move.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__memory/allocator.h b/libcxx/include/__memory/allocator.h
index cd146da8e7eb..ddb4179940b8 100644
--- a/libcxx/include/__memory/allocator.h
+++ b/libcxx/include/__memory/allocator.h
@@ -11,6 +11,7 @@
 #define _LIBCPP___MEMORY_ALLOCATOR_H
 
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__memory/addressof.h>
 #include <__memory/allocate_at_least.h>
 #include <__memory/allocator_traits.h>
@@ -20,7 +21,6 @@
 #include <__type_traits/is_void.h>
 #include <__type_traits/is_volatile.h>
 #include <__utility/forward.h>
-#include <cstddef>
 #include <new>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__memory/allocator_traits.h b/libcxx/include/__memory/allocator_traits.h
index f4d9679807ae..499b30b85b6c 100644
--- a/libcxx/include/__memory/allocator_traits.h
+++ b/libcxx/include/__memory/allocator_traits.h
@@ -11,6 +11,7 @@
 #define _LIBCPP___MEMORY_ALLOCATOR_TRAITS_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__fwd/memory.h>
 #include <__memory/construct_at.h>
 #include <__memory/pointer_traits.h>
@@ -23,7 +24,6 @@
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
 #include <__utility/forward.h>
-#include <cstddef>
 #include <limits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__memory/array_cookie.h b/libcxx/include/__memory/array_cookie.h
index 10b29c9dcc78..806a9e99ecaf 100644
--- a/libcxx/include/__memory/array_cookie.h
+++ b/libcxx/include/__memory/array_cookie.h
@@ -12,10 +12,10 @@
 
 #include <__config>
 #include <__configuration/abi.h>
+#include <__cstddef/size_t.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_trivially_destructible.h>
 #include <__type_traits/negation.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__memory/assume_aligned.h b/libcxx/include/__memory/assume_aligned.h
index c7ba2a99c7e2..08f1772cd6df 100644
--- a/libcxx/include/__memory/assume_aligned.h
+++ b/libcxx/include/__memory/assume_aligned.h
@@ -12,8 +12,8 @@
 
 #include <__assert>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/is_constant_evaluated.h>
-#include <cstddef>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__memory/builtin_new_allocator.h b/libcxx/include/__memory/builtin_new_allocator.h
index c6f7f3c5ff52..128288efb05b 100644
--- a/libcxx/include/__memory/builtin_new_allocator.h
+++ b/libcxx/include/__memory/builtin_new_allocator.h
@@ -11,7 +11,6 @@
 
 #include <__config>
 #include <__memory/unique_ptr.h>
-#include <cstddef>
 #include <new>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__memory/compressed_pair.h b/libcxx/include/__memory/compressed_pair.h
index 9d44775bdb48..a7acaaff9da0 100644
--- a/libcxx/include/__memory/compressed_pair.h
+++ b/libcxx/include/__memory/compressed_pair.h
@@ -11,6 +11,7 @@
 #define _LIBCPP___MEMORY_COMPRESSED_PAIR_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/datasizeof.h>
 #include <__type_traits/is_empty.h>
 #include <__type_traits/is_final.h>
diff --git a/libcxx/include/__memory/destruct_n.h b/libcxx/include/__memory/destruct_n.h
index 78635ad0af04..66adefb0f51f 100644
--- a/libcxx/include/__memory/destruct_n.h
+++ b/libcxx/include/__memory/destruct_n.h
@@ -10,9 +10,9 @@
 #define _LIBCPP___MEMORY_DESTRUCT_N_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_trivially_destructible.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__memory/pointer_traits.h b/libcxx/include/__memory/pointer_traits.h
index 98961ddf9709..4acf3d18401a 100644
--- a/libcxx/include/__memory/pointer_traits.h
+++ b/libcxx/include/__memory/pointer_traits.h
@@ -11,6 +11,7 @@
 #define _LIBCPP___MEMORY_POINTER_TRAITS_H
 
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__memory/addressof.h>
 #include <__type_traits/conditional.h>
 #include <__type_traits/conjunction.h>
@@ -23,7 +24,6 @@
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
 #include <__utility/forward.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__memory/raw_storage_iterator.h b/libcxx/include/__memory/raw_storage_iterator.h
index 774878aa1c5e..2ee4c074d8d3 100644
--- a/libcxx/include/__memory/raw_storage_iterator.h
+++ b/libcxx/include/__memory/raw_storage_iterator.h
@@ -11,11 +11,11 @@
 #define _LIBCPP___MEMORY_RAW_STORAGE_ITERATOR_H
 
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__iterator/iterator.h>
 #include <__iterator/iterator_traits.h>
 #include <__memory/addressof.h>
 #include <__utility/move.h>
-#include <cstddef>
 #include <new>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h
index 65870ba574c2..e5adbedce1a2 100644
--- a/libcxx/include/__memory/shared_ptr.h
+++ b/libcxx/include/__memory/shared_ptr.h
@@ -13,6 +13,7 @@
 #include <__compare/compare_three_way.h>
 #include <__compare/ordering.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__exception/exception.h>
 #include <__functional/binary_function.h>
 #include <__functional/operations.h>
@@ -51,7 +52,6 @@
 #include <__utility/move.h>
 #include <__utility/swap.h>
 #include <__verbose_abort>
-#include <cstddef>
 #include <new>
 #include <typeinfo>
 #if _LIBCPP_HAS_ATOMIC_HEADER
diff --git a/libcxx/include/__memory/temporary_buffer.h b/libcxx/include/__memory/temporary_buffer.h
index 219e03f99bc0..d18717f52d1c 100644
--- a/libcxx/include/__memory/temporary_buffer.h
+++ b/libcxx/include/__memory/temporary_buffer.h
@@ -11,10 +11,9 @@
 #define _LIBCPP___MEMORY_TEMPORARY_BUFFER_H
 
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__memory/unique_temporary_buffer.h>
 #include <__utility/pair.h>
-#include <cstddef>
-#include <new>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__memory/unique_ptr.h b/libcxx/include/__memory/unique_ptr.h
index 6e42ef1eaa1a..4ed6393b1209 100644
--- a/libcxx/include/__memory/unique_ptr.h
+++ b/libcxx/include/__memory/unique_ptr.h
@@ -15,6 +15,8 @@
 #include <__compare/compare_three_way_result.h>
 #include <__compare/three_way_comparable.h>
 #include <__config>
+#include <__cstddef/nullptr_t.h>
+#include <__cstddef/size_t.h>
 #include <__functional/hash.h>
 #include <__functional/operations.h>
 #include <__memory/allocator_traits.h> // __pointer
@@ -46,7 +48,6 @@
 #include <__utility/forward.h>
 #include <__utility/move.h>
 #include <__utility/private_constructor_tag.h>
-#include <cstddef>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__memory/unique_temporary_buffer.h b/libcxx/include/__memory/unique_temporary_buffer.h
index 8a8c327be69f..4f47c84e2f8d 100644
--- a/libcxx/include/__memory/unique_temporary_buffer.h
+++ b/libcxx/include/__memory/unique_temporary_buffer.h
@@ -13,10 +13,10 @@
 #include <__assert>
 #include <__config>
 
+#include <__cstddef/ptrdiff_t.h>
 #include <__memory/allocator.h>
 #include <__memory/unique_ptr.h>
 #include <__type_traits/is_constant_evaluated.h>
-#include <cstddef>
 #include <new>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__memory/uses_allocator.h b/libcxx/include/__memory/uses_allocator.h
index 16504e8b2a99..20aa0e6b7f27 100644
--- a/libcxx/include/__memory/uses_allocator.h
+++ b/libcxx/include/__memory/uses_allocator.h
@@ -13,7 +13,6 @@
 #include <__config>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_convertible.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__memory_resource/memory_resource.h b/libcxx/include/__memory_resource/memory_resource.h
index ea85e50cd568..f93f10fe21a2 100644
--- a/libcxx/include/__memory_resource/memory_resource.h
+++ b/libcxx/include/__memory_resource/memory_resource.h
@@ -10,8 +10,9 @@
 #define _LIBCPP___MEMORY_RESOURCE_MEMORY_RESOURCE_H
 
 #include <__config>
+#include <__cstddef/max_align_t.h>
+#include <__cstddef/size_t.h>
 #include <__fwd/memory_resource.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__memory_resource/monotonic_buffer_resource.h b/libcxx/include/__memory_resource/monotonic_buffer_resource.h
index f45b30fdb386..c5a2b556707f 100644
--- a/libcxx/include/__memory_resource/monotonic_buffer_resource.h
+++ b/libcxx/include/__memory_resource/monotonic_buffer_resource.h
@@ -10,9 +10,9 @@
 #define _LIBCPP___MEMORY_RESOURCE_MONOTONIC_BUFFER_RESOURCE_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__memory/addressof.h>
 #include <__memory_resource/memory_resource.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__memory_resource/polymorphic_allocator.h b/libcxx/include/__memory_resource/polymorphic_allocator.h
index fb36d5cad78e..30fa5c2170d5 100644
--- a/libcxx/include/__memory_resource/polymorphic_allocator.h
+++ b/libcxx/include/__memory_resource/polymorphic_allocator.h
@@ -11,10 +11,11 @@
 
 #include <__assert>
 #include <__config>
+#include <__cstddef/byte.h>
+#include <__cstddef/max_align_t.h>
 #include <__fwd/pair.h>
 #include <__memory_resource/memory_resource.h>
 #include <__utility/exception_guard.h>
-#include <cstddef>
 #include <limits>
 #include <new>
 #include <tuple>
diff --git a/libcxx/include/__memory_resource/pool_options.h b/libcxx/include/__memory_resource/pool_options.h
index 442959836c7e..324b8aaa8502 100644
--- a/libcxx/include/__memory_resource/pool_options.h
+++ b/libcxx/include/__memory_resource/pool_options.h
@@ -10,7 +10,7 @@
 #define _LIBCPP___MEMORY_RESOURCE_POOL_OPTIONS_H
 
 #include <__config>
-#include <cstddef>
+#include <__cstddef/size_t.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__memory_resource/synchronized_pool_resource.h b/libcxx/include/__memory_resource/synchronized_pool_resource.h
index 50a673c2861d..2679afc16617 100644
--- a/libcxx/include/__memory_resource/synchronized_pool_resource.h
+++ b/libcxx/include/__memory_resource/synchronized_pool_resource.h
@@ -13,7 +13,6 @@
 #include <__memory_resource/memory_resource.h>
 #include <__memory_resource/pool_options.h>
 #include <__memory_resource/unsynchronized_pool_resource.h>
-#include <cstddef>
 #include <mutex>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__memory_resource/unsynchronized_pool_resource.h b/libcxx/include/__memory_resource/unsynchronized_pool_resource.h
index 783db84262af..92da16c559fe 100644
--- a/libcxx/include/__memory_resource/unsynchronized_pool_resource.h
+++ b/libcxx/include/__memory_resource/unsynchronized_pool_resource.h
@@ -10,9 +10,9 @@
 #define _LIBCPP___MEMORY_RESOURCE_UNSYNCHRONIZED_POOL_RESOURCE_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__memory_resource/memory_resource.h>
 #include <__memory_resource/pool_options.h>
-#include <cstddef>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__numeric/midpoint.h b/libcxx/include/__numeric/midpoint.h
index 5ef30d4ec50f..2ba80e5cca07 100644
--- a/libcxx/include/__numeric/midpoint.h
+++ b/libcxx/include/__numeric/midpoint.h
@@ -11,6 +11,7 @@
 #define _LIBCPP___NUMERIC_MIDPOINT_H
 
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_floating_point.h>
 #include <__type_traits/is_integral.h>
@@ -21,7 +22,6 @@
 #include <__type_traits/is_void.h>
 #include <__type_traits/make_unsigned.h>
 #include <__type_traits/remove_pointer.h>
-#include <cstddef>
 #include <limits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__ostream/basic_ostream.h b/libcxx/include/__ostream/basic_ostream.h
index 1b1c026706cd..fc72247725b4 100644
--- a/libcxx/include/__ostream/basic_ostream.h
+++ b/libcxx/include/__ostream/basic_ostream.h
@@ -23,7 +23,6 @@
 #  include <__type_traits/void_t.h>
 #  include <__utility/declval.h>
 #  include <bitset>
-#  include <cstddef>
 #  include <ios>
 #  include <locale>
 #  include <new> // for __throw_bad_alloc
diff --git a/libcxx/include/__pstl/backends/libdispatch.h b/libcxx/include/__pstl/backends/libdispatch.h
index a92d0978e5c6..701367b505c8 100644
--- a/libcxx/include/__pstl/backends/libdispatch.h
+++ b/libcxx/include/__pstl/backends/libdispatch.h
@@ -16,6 +16,7 @@
 #include <__algorithm/upper_bound.h>
 #include <__atomic/atomic.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__exception/terminate.h>
 #include <__iterator/iterator_traits.h>
 #include <__iterator/move_iterator.h>
@@ -37,7 +38,6 @@
 #include <__utility/exception_guard.h>
 #include <__utility/move.h>
 #include <__utility/pair.h>
-#include <cstddef>
 #include <new>
 #include <optional>
 
diff --git a/libcxx/include/__pstl/backends/std_thread.h b/libcxx/include/__pstl/backends/std_thread.h
index 19b985f860a1..dd2c3f15403e 100644
--- a/libcxx/include/__pstl/backends/std_thread.h
+++ b/libcxx/include/__pstl/backends/std_thread.h
@@ -22,7 +22,6 @@
 #include <__pstl/cpu_algos/transform_reduce.h>
 #include <__utility/empty.h>
 #include <__utility/move.h>
-#include <cstddef>
 #include <optional>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__pstl/cpu_algos/cpu_traits.h b/libcxx/include/__pstl/cpu_algos/cpu_traits.h
index 5e59752fa572..ec1622419d04 100644
--- a/libcxx/include/__pstl/cpu_algos/cpu_traits.h
+++ b/libcxx/include/__pstl/cpu_algos/cpu_traits.h
@@ -10,7 +10,6 @@
 #define _LIBCPP___PSTL_CPU_ALGOS_CPU_TRAITS_H
 
 #include <__config>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__pstl/cpu_algos/find_if.h b/libcxx/include/__pstl/cpu_algos/find_if.h
index cd92e5a99f12..ebb4ecb4a0ed 100644
--- a/libcxx/include/__pstl/cpu_algos/find_if.h
+++ b/libcxx/include/__pstl/cpu_algos/find_if.h
@@ -21,7 +21,6 @@
 #include <__type_traits/is_execution_policy.h>
 #include <__utility/move.h>
 #include <__utility/pair.h>
-#include <cstddef>
 #include <optional>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__pstl/cpu_algos/transform_reduce.h b/libcxx/include/__pstl/cpu_algos/transform_reduce.h
index aafbf1ca96b4..e9f622d832cd 100644
--- a/libcxx/include/__pstl/cpu_algos/transform_reduce.h
+++ b/libcxx/include/__pstl/cpu_algos/transform_reduce.h
@@ -20,7 +20,6 @@
 #include <__type_traits/is_arithmetic.h>
 #include <__type_traits/is_execution_policy.h>
 #include <__utility/move.h>
-#include <cstddef>
 #include <new>
 #include <optional>
 
diff --git a/libcxx/include/__random/discard_block_engine.h b/libcxx/include/__random/discard_block_engine.h
index 07f599067279..f319557a5736 100644
--- a/libcxx/include/__random/discard_block_engine.h
+++ b/libcxx/include/__random/discard_block_engine.h
@@ -10,11 +10,11 @@
 #define _LIBCPP___RANDOM_DISCARD_BLOCK_ENGINE_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__random/is_seed_sequence.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_convertible.h>
 #include <__utility/move.h>
-#include <cstddef>
 #include <iosfwd>
 #include <limits>
 
diff --git a/libcxx/include/__random/discrete_distribution.h b/libcxx/include/__random/discrete_distribution.h
index 931f7704ff97..3ce4a495fb0c 100644
--- a/libcxx/include/__random/discrete_distribution.h
+++ b/libcxx/include/__random/discrete_distribution.h
@@ -14,7 +14,6 @@
 #include <__random/is_valid.h>
 #include <__random/uniform_real_distribution.h>
 #include <__vector/vector.h>
-#include <cstddef>
 #include <initializer_list>
 #include <iosfwd>
 #include <numeric>
diff --git a/libcxx/include/__random/independent_bits_engine.h b/libcxx/include/__random/independent_bits_engine.h
index 0f4a7b82b98f..20f56e9b5756 100644
--- a/libcxx/include/__random/independent_bits_engine.h
+++ b/libcxx/include/__random/independent_bits_engine.h
@@ -10,6 +10,7 @@
 #define _LIBCPP___RANDOM_INDEPENDENT_BITS_ENGINE_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__fwd/istream.h>
 #include <__fwd/ostream.h>
 #include <__random/is_seed_sequence.h>
@@ -18,7 +19,6 @@
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_convertible.h>
 #include <__utility/move.h>
-#include <cstddef>
 #include <limits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__random/log2.h b/libcxx/include/__random/log2.h
index c96a5247ff6d..fbf35bab9172 100644
--- a/libcxx/include/__random/log2.h
+++ b/libcxx/include/__random/log2.h
@@ -10,8 +10,8 @@
 #define _LIBCPP___RANDOM_LOG2_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/conditional.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__random/mersenne_twister_engine.h b/libcxx/include/__random/mersenne_twister_engine.h
index 1f50e608ce8d..9dd87f9ce71a 100644
--- a/libcxx/include/__random/mersenne_twister_engine.h
+++ b/libcxx/include/__random/mersenne_twister_engine.h
@@ -12,9 +12,9 @@
 #include <__algorithm/equal.h>
 #include <__algorithm/min.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__random/is_seed_sequence.h>
 #include <__type_traits/enable_if.h>
-#include <cstddef>
 #include <cstdint>
 #include <iosfwd>
 #include <limits>
diff --git a/libcxx/include/__random/piecewise_constant_distribution.h b/libcxx/include/__random/piecewise_constant_distribution.h
index a864f848143b..5afe2ebeda3b 100644
--- a/libcxx/include/__random/piecewise_constant_distribution.h
+++ b/libcxx/include/__random/piecewise_constant_distribution.h
@@ -11,6 +11,7 @@
 
 #include <__algorithm/upper_bound.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__random/is_valid.h>
 #include <__random/uniform_real_distribution.h>
 #include <__vector/vector.h>
diff --git a/libcxx/include/__random/piecewise_linear_distribution.h b/libcxx/include/__random/piecewise_linear_distribution.h
index 24aa6cce91cf..0d14f882cbbb 100644
--- a/libcxx/include/__random/piecewise_linear_distribution.h
+++ b/libcxx/include/__random/piecewise_linear_distribution.h
@@ -11,6 +11,7 @@
 
 #include <__algorithm/upper_bound.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__random/is_valid.h>
 #include <__random/uniform_real_distribution.h>
 #include <__vector/comparison.h>
diff --git a/libcxx/include/__random/shuffle_order_engine.h b/libcxx/include/__random/shuffle_order_engine.h
index f54ed17e3838..53f6c0897110 100644
--- a/libcxx/include/__random/shuffle_order_engine.h
+++ b/libcxx/include/__random/shuffle_order_engine.h
@@ -11,12 +11,12 @@
 
 #include <__algorithm/equal.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__random/is_seed_sequence.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_convertible.h>
 #include <__utility/move.h>
-#include <cstddef>
 #include <cstdint>
 #include <iosfwd>
 
diff --git a/libcxx/include/__random/subtract_with_carry_engine.h b/libcxx/include/__random/subtract_with_carry_engine.h
index 926333cdda45..e087ab4a3c2c 100644
--- a/libcxx/include/__random/subtract_with_carry_engine.h
+++ b/libcxx/include/__random/subtract_with_carry_engine.h
@@ -12,10 +12,10 @@
 #include <__algorithm/equal.h>
 #include <__algorithm/min.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__random/is_seed_sequence.h>
 #include <__random/linear_congruential_engine.h>
 #include <__type_traits/enable_if.h>
-#include <cstddef>
 #include <cstdint>
 #include <iosfwd>
 #include <limits>
diff --git a/libcxx/include/__random/uniform_int_distribution.h b/libcxx/include/__random/uniform_int_distribution.h
index 4e3ca3efe568..fa2c33755b73 100644
--- a/libcxx/include/__random/uniform_int_distribution.h
+++ b/libcxx/include/__random/uniform_int_distribution.h
@@ -11,11 +11,11 @@
 
 #include <__bit/countl.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__random/is_valid.h>
 #include <__random/log2.h>
 #include <__type_traits/conditional.h>
 #include <__type_traits/make_unsigned.h>
-#include <cstddef>
 #include <cstdint>
 #include <iosfwd>
 #include <limits>
diff --git a/libcxx/include/__ranges/access.h b/libcxx/include/__ranges/access.h
index c0a40c5e1017..bbacef3eae6b 100644
--- a/libcxx/include/__ranges/access.h
+++ b/libcxx/include/__ranges/access.h
@@ -12,6 +12,7 @@
 
 #include <__concepts/class_or_enum.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__iterator/concepts.h>
 #include <__iterator/readable_traits.h>
 #include <__ranges/enable_borrowed_range.h>
@@ -21,7 +22,6 @@
 #include <__type_traits/remove_reference.h>
 #include <__utility/auto_cast.h>
 #include <__utility/declval.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__ranges/counted.h b/libcxx/include/__ranges/counted.h
index e365deca4e63..65bf1a371ec7 100644
--- a/libcxx/include/__ranges/counted.h
+++ b/libcxx/include/__ranges/counted.h
@@ -12,6 +12,7 @@
 
 #include <__concepts/convertible_to.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__iterator/concepts.h>
 #include <__iterator/counted_iterator.h>
 #include <__iterator/default_sentinel.h>
@@ -22,7 +23,6 @@
 #include <__type_traits/decay.h>
 #include <__utility/forward.h>
 #include <__utility/move.h>
-#include <cstddef>
 #include <span>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__ranges/drop_view.h b/libcxx/include/__ranges/drop_view.h
index bd66371f4ed2..87f66f17a2ab 100644
--- a/libcxx/include/__ranges/drop_view.h
+++ b/libcxx/include/__ranges/drop_view.h
@@ -15,6 +15,7 @@
 #include <__concepts/constructible.h>
 #include <__concepts/convertible_to.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__functional/bind_back.h>
 #include <__fwd/span.h>
 #include <__fwd/string_view.h>
@@ -42,7 +43,6 @@
 #include <__utility/auto_cast.h>
 #include <__utility/forward.h>
 #include <__utility/move.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__ranges/elements_view.h b/libcxx/include/__ranges/elements_view.h
index ac0d8dbbd52b..c99282f37960 100644
--- a/libcxx/include/__ranges/elements_view.h
+++ b/libcxx/include/__ranges/elements_view.h
@@ -37,7 +37,6 @@
 #include <__utility/declval.h>
 #include <__utility/forward.h>
 #include <__utility/move.h>
-#include <cstddef>
 #include <tuple> // std::get
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__ranges/empty_view.h b/libcxx/include/__ranges/empty_view.h
index 6c04b0200c35..fc08492110f5 100644
--- a/libcxx/include/__ranges/empty_view.h
+++ b/libcxx/include/__ranges/empty_view.h
@@ -11,10 +11,10 @@
 #define _LIBCPP___RANGES_EMPTY_VIEW_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__ranges/enable_borrowed_range.h>
 #include <__ranges/view_interface.h>
 #include <__type_traits/is_object.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__ranges/istream_view.h b/libcxx/include/__ranges/istream_view.h
index cd7096d35c2c..1820ef2a4c1f 100644
--- a/libcxx/include/__ranges/istream_view.h
+++ b/libcxx/include/__ranges/istream_view.h
@@ -14,6 +14,7 @@
 #include <__concepts/derived_from.h>
 #include <__concepts/movable.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__fwd/istream.h>
 #include <__fwd/string.h>
 #include <__iterator/default_sentinel.h>
@@ -22,7 +23,6 @@
 #include <__ranges/view_interface.h>
 #include <__type_traits/remove_cvref.h>
 #include <__utility/forward.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__ranges/repeat_view.h b/libcxx/include/__ranges/repeat_view.h
index 53e4beb270ad..93ceaf1711d3 100644
--- a/libcxx/include/__ranges/repeat_view.h
+++ b/libcxx/include/__ranges/repeat_view.h
@@ -15,6 +15,7 @@
 #include <__concepts/same_as.h>
 #include <__concepts/semiregular.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__iterator/concepts.h>
 #include <__iterator/iterator_traits.h>
 #include <__iterator/unreachable_sentinel.h>
diff --git a/libcxx/include/__ranges/single_view.h b/libcxx/include/__ranges/single_view.h
index 45244f34994d..955578b99cf5 100644
--- a/libcxx/include/__ranges/single_view.h
+++ b/libcxx/include/__ranges/single_view.h
@@ -12,6 +12,8 @@
 
 #include <__concepts/constructible.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
+#include <__cstddef/size_t.h>
 #include <__ranges/movable_box.h>
 #include <__ranges/range_adaptor.h>
 #include <__ranges/view_interface.h>
@@ -20,7 +22,6 @@
 #include <__utility/forward.h>
 #include <__utility/in_place.h>
 #include <__utility/move.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__ranges/size.h b/libcxx/include/__ranges/size.h
index 40b0c6b6aad7..5da3a6ff268e 100644
--- a/libcxx/include/__ranges/size.h
+++ b/libcxx/include/__ranges/size.h
@@ -13,6 +13,8 @@
 #include <__concepts/arithmetic.h>
 #include <__concepts/class_or_enum.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
+#include <__cstddef/size_t.h>
 #include <__iterator/concepts.h>
 #include <__iterator/iterator_traits.h>
 #include <__ranges/access.h>
@@ -22,7 +24,6 @@
 #include <__type_traits/remove_cvref.h>
 #include <__utility/auto_cast.h>
 #include <__utility/declval.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__ranges/subrange.h b/libcxx/include/__ranges/subrange.h
index 144746babb32..a40eab3c5a25 100644
--- a/libcxx/include/__ranges/subrange.h
+++ b/libcxx/include/__ranges/subrange.h
@@ -17,6 +17,7 @@
 #include <__concepts/derived_from.h>
 #include <__concepts/different_from.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__fwd/subrange.h>
 #include <__iterator/advance.h>
 #include <__iterator/concepts.h>
@@ -40,7 +41,6 @@
 #include <__type_traits/remove_const.h>
 #include <__type_traits/remove_pointer.h>
 #include <__utility/move.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__ranges/take_view.h b/libcxx/include/__ranges/take_view.h
index 8e2d354b58a5..39f99cee6b4d 100644
--- a/libcxx/include/__ranges/take_view.h
+++ b/libcxx/include/__ranges/take_view.h
@@ -42,7 +42,6 @@
 #include <__utility/auto_cast.h>
 #include <__utility/forward.h>
 #include <__utility/move.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__ranges/to.h b/libcxx/include/__ranges/to.h
index 52666075da3e..76249bdd9891 100644
--- a/libcxx/include/__ranges/to.h
+++ b/libcxx/include/__ranges/to.h
@@ -15,6 +15,7 @@
 #include <__concepts/derived_from.h>
 #include <__concepts/same_as.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__functional/bind_back.h>
 #include <__iterator/iterator_traits.h>
 #include <__ranges/access.h>
@@ -30,7 +31,6 @@
 #include <__type_traits/type_identity.h>
 #include <__utility/declval.h>
 #include <__utility/forward.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__split_buffer b/libcxx/include/__split_buffer
index c4817601039f..6fc3d9255946 100644
--- a/libcxx/include/__split_buffer
+++ b/libcxx/include/__split_buffer
@@ -14,6 +14,7 @@
 #include <__algorithm/move.h>
 #include <__algorithm/move_backward.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__iterator/distance.h>
 #include <__iterator/iterator_traits.h>
 #include <__iterator/move_iterator.h>
@@ -35,7 +36,6 @@
 #include <__type_traits/remove_reference.h>
 #include <__utility/forward.h>
 #include <__utility/move.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__stop_token/intrusive_shared_ptr.h b/libcxx/include/__stop_token/intrusive_shared_ptr.h
index f00cea5bc2b6..d20c5227ec72 100644
--- a/libcxx/include/__stop_token/intrusive_shared_ptr.h
+++ b/libcxx/include/__stop_token/intrusive_shared_ptr.h
@@ -13,10 +13,10 @@
 #include <__atomic/atomic.h>
 #include <__atomic/memory_order.h>
 #include <__config>
+#include <__cstddef/nullptr_t.h>
 #include <__type_traits/is_reference.h>
 #include <__utility/move.h>
 #include <__utility/swap.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__string/char_traits.h b/libcxx/include/__string/char_traits.h
index 107f0a96a022..fff045be0180 100644
--- a/libcxx/include/__string/char_traits.h
+++ b/libcxx/include/__string/char_traits.h
@@ -24,7 +24,6 @@
 #include <__string/constexpr_c_functions.h>
 #include <__type_traits/is_constant_evaluated.h>
 #include <__utility/is_pointer_in_range.h>
-#include <cstddef>
 #include <cstdint>
 #include <cstdio>
 #include <iosfwd>
diff --git a/libcxx/include/__string/constexpr_c_functions.h b/libcxx/include/__string/constexpr_c_functions.h
index e62a7b0cd1b3..f50eac34a1c0 100644
--- a/libcxx/include/__string/constexpr_c_functions.h
+++ b/libcxx/include/__string/constexpr_c_functions.h
@@ -10,6 +10,7 @@
 #define _LIBCPP___STRING_CONSTEXPR_C_FUNCTIONS_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__memory/addressof.h>
 #include <__memory/construct_at.h>
 #include <__type_traits/datasizeof.h>
@@ -25,7 +26,6 @@
 #include <__type_traits/is_trivially_lexicographically_comparable.h>
 #include <__type_traits/remove_cv.h>
 #include <__utility/is_pointer_in_range.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__system_error/error_code.h b/libcxx/include/__system_error/error_code.h
index 475f2bb96a56..6fa673b4ff71 100644
--- a/libcxx/include/__system_error/error_code.h
+++ b/libcxx/include/__system_error/error_code.h
@@ -17,7 +17,6 @@
 #include <__system_error/errc.h>
 #include <__system_error/error_category.h>
 #include <__system_error/error_condition.h>
-#include <cstddef>
 #include <string>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__system_error/error_condition.h b/libcxx/include/__system_error/error_condition.h
index 42898c1f0e90..bfee6528c3f1 100644
--- a/libcxx/include/__system_error/error_condition.h
+++ b/libcxx/include/__system_error/error_condition.h
@@ -16,7 +16,6 @@
 #include <__functional/unary_function.h>
 #include <__system_error/errc.h>
 #include <__system_error/error_category.h>
-#include <cstddef>
 #include <string>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__utility/in_place.h b/libcxx/include/__utility/in_place.h
index 459b27167526..edaa4e02c55f 100644
--- a/libcxx/include/__utility/in_place.h
+++ b/libcxx/include/__utility/in_place.h
@@ -10,9 +10,9 @@
 #define _LIBCPP___UTILITY_IN_PLACE_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/remove_cvref.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__utility/integer_sequence.h b/libcxx/include/__utility/integer_sequence.h
index ccce9433e7a8..35eb606ee37f 100644
--- a/libcxx/include/__utility/integer_sequence.h
+++ b/libcxx/include/__utility/integer_sequence.h
@@ -10,8 +10,8 @@
 #define _LIBCPP___UTILITY_INTEGER_SEQUENCE_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/is_integral.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__utility/pair.h b/libcxx/include/__utility/pair.h
index 78534a3f399f..cca6490476db 100644
--- a/libcxx/include/__utility/pair.h
+++ b/libcxx/include/__utility/pair.h
@@ -13,6 +13,7 @@
 #include <__compare/synth_three_way.h>
 #include <__concepts/different_from.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__fwd/array.h>
 #include <__fwd/pair.h>
 #include <__fwd/tuple.h>
@@ -43,7 +44,6 @@
 #include <__utility/forward.h>
 #include <__utility/move.h>
 #include <__utility/piecewise_construct.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__utility/priority_tag.h b/libcxx/include/__utility/priority_tag.h
index a159ce7f1afb..ef7cf162b9b4 100644
--- a/libcxx/include/__utility/priority_tag.h
+++ b/libcxx/include/__utility/priority_tag.h
@@ -10,7 +10,7 @@
 #define _LIBCPP___UTILITY_PRIORITY_TAG_H
 
 #include <__config>
-#include <cstddef>
+#include <__cstddef/size_t.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__utility/small_buffer.h b/libcxx/include/__utility/small_buffer.h
index 9e13797573d2..70e068f89f62 100644
--- a/libcxx/include/__utility/small_buffer.h
+++ b/libcxx/include/__utility/small_buffer.h
@@ -10,13 +10,13 @@
 #define _LIBCPP___UTILITY_SMALL_BUFFER_H
 
 #include <__config>
+#include <__cstddef/byte.h>
 #include <__memory/construct_at.h>
 #include <__type_traits/decay.h>
 #include <__type_traits/is_trivially_constructible.h>
 #include <__type_traits/is_trivially_destructible.h>
 #include <__utility/exception_guard.h>
 #include <__utility/forward.h>
-#include <cstddef>
 #include <new>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__utility/swap.h b/libcxx/include/__utility/swap.h
index ecfbdec75a2a..666d6d50f0d9 100644
--- a/libcxx/include/__utility/swap.h
+++ b/libcxx/include/__utility/swap.h
@@ -10,6 +10,7 @@
 #define _LIBCPP___UTILITY_SWAP_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_assignable.h>
 #include <__type_traits/is_constructible.h>
@@ -18,7 +19,6 @@
 #include <__type_traits/is_swappable.h>
 #include <__utility/declval.h>
 #include <__utility/move.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__variant/monostate.h b/libcxx/include/__variant/monostate.h
index 16f156609eb7..c5d2dacaf420 100644
--- a/libcxx/include/__variant/monostate.h
+++ b/libcxx/include/__variant/monostate.h
@@ -12,8 +12,8 @@
 
 #include <__compare/ordering.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__functional/hash.h>
-#include <cstddef>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/array b/libcxx/include/array
index 0e9af4198632..b1a9f0d29e68 100644
--- a/libcxx/include/array
+++ b/libcxx/include/array
@@ -118,6 +118,7 @@ template <size_t I, class T, size_t N> const T&& get(const array<T, N>&&) noexce
 #include <__algorithm/swap_ranges.h>
 #include <__assert>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__fwd/array.h>
 #include <__iterator/reverse_iterator.h>
 #include <__iterator/wrap_iter.h>
diff --git a/libcxx/include/atomic b/libcxx/include/atomic
index ebd46238eec9..716d198bc236 100644
--- a/libcxx/include/atomic
+++ b/libcxx/include/atomic
@@ -620,6 +620,7 @@ template <class T>
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #  include <cmath>
 #  include <compare>
+#  include <cstddef>
 #  include <cstdlib>
 #  include <cstring>
 #  include <type_traits>
diff --git a/libcxx/include/barrier b/libcxx/include/barrier
index abc014e8aaf5..36c30c7fe2e7 100644
--- a/libcxx/include/barrier
+++ b/libcxx/include/barrier
@@ -52,11 +52,11 @@ namespace std
 #  include <__assert>
 #  include <__atomic/atomic_base.h>
 #  include <__atomic/memory_order.h>
+#  include <__cstddef/ptrdiff_t.h>
 #  include <__memory/unique_ptr.h>
 #  include <__thread/poll_with_backoff.h>
 #  include <__thread/timed_backoff_policy.h>
 #  include <__utility/move.h>
-#  include <cstddef>
 #  include <cstdint>
 #  include <limits>
 #  include <version>
diff --git a/libcxx/include/bitset b/libcxx/include/bitset
index 645c172f3be4..9f14b69e7a9b 100644
--- a/libcxx/include/bitset
+++ b/libcxx/include/bitset
@@ -136,7 +136,6 @@ template <size_t N> struct hash<std::bitset<N>>;
 #include <__functional/unary_function.h>
 #include <__type_traits/is_char_like_type.h>
 #include <climits>
-#include <cstddef>
 #include <stdexcept>
 #include <string_view>
 #include <version>
diff --git a/libcxx/include/charconv b/libcxx/include/charconv
index 29c6875008ab..8f5e697eec43 100644
--- a/libcxx/include/charconv
+++ b/libcxx/include/charconv
@@ -111,6 +111,7 @@ _LIBCPP_END_NAMESPACE_STD
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #  include <cmath>
 #  include <concepts>
+#  include <cstddef>
 #  include <cstdint>
 #  include <cstdlib>
 #  include <cstring>
diff --git a/libcxx/include/compare b/libcxx/include/compare
index 8a41835b1489..de0e4c7ec228 100644
--- a/libcxx/include/compare
+++ b/libcxx/include/compare
@@ -172,6 +172,7 @@ namespace std {
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #  include <cmath>
+#  include <cstddef>
 #  include <type_traits>
 #endif
 
diff --git a/libcxx/include/concepts b/libcxx/include/concepts
index e89d216a5937..6db2caebaf5b 100644
--- a/libcxx/include/concepts
+++ b/libcxx/include/concepts
@@ -158,11 +158,8 @@ namespace std {
 
 #include <version>
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #  include <cstddef>
-#endif
-
-#if _LIBCPP_STD_VER <= 20 && !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES)
 #  include <type_traits>
 #endif
 
diff --git a/libcxx/include/coroutine b/libcxx/include/coroutine
index ee54388ad5aa..18601717768c 100644
--- a/libcxx/include/coroutine
+++ b/libcxx/include/coroutine
@@ -59,6 +59,7 @@ struct suspend_always;
 #endif
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
 #  include <iosfwd>
 #  include <limits>
 #  include <type_traits>
diff --git a/libcxx/include/exception b/libcxx/include/exception
index 64463e02cb16..88eaaf06bf4a 100644
--- a/libcxx/include/exception
+++ b/libcxx/include/exception
@@ -89,6 +89,7 @@ template <class E> void rethrow_if_nested(const E& e);
 #endif
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
 #  include <cstdlib>
 #  include <type_traits>
 #endif
diff --git a/libcxx/include/experimental/__simd/aligned_tag.h b/libcxx/include/experimental/__simd/aligned_tag.h
index e364e146a601..d208bf5c4fbb 100644
--- a/libcxx/include/experimental/__simd/aligned_tag.h
+++ b/libcxx/include/experimental/__simd/aligned_tag.h
@@ -11,9 +11,9 @@
 #define _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__memory/assume_aligned.h>
 #include <__type_traits/remove_const.h>
-#include <cstddef>
 #include <experimental/__simd/traits.h>
 
 #if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
diff --git a/libcxx/include/experimental/__simd/declaration.h b/libcxx/include/experimental/__simd/declaration.h
index 2ac7224159cf..1b4fcf958516 100644
--- a/libcxx/include/experimental/__simd/declaration.h
+++ b/libcxx/include/experimental/__simd/declaration.h
@@ -11,7 +11,7 @@
 #define _LIBCPP_EXPERIMENTAL___SIMD_DECLARATION_H
 
 #include <__config>
-#include <cstddef>
+#include <__cstddef/size_t.h>
 
 #if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
 
diff --git a/libcxx/include/experimental/__simd/reference.h b/libcxx/include/experimental/__simd/reference.h
index cba460baaa95..b9fe962348ad 100644
--- a/libcxx/include/experimental/__simd/reference.h
+++ b/libcxx/include/experimental/__simd/reference.h
@@ -11,13 +11,13 @@
 #define _LIBCPP_EXPERIMENTAL___SIMD_REFERENCE_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_assignable.h>
 #include <__type_traits/is_same.h>
 #include <__utility/declval.h>
 #include <__utility/forward.h>
 #include <__utility/move.h>
-#include <cstddef>
 #include <experimental/__simd/utility.h>
 
 _LIBCPP_PUSH_MACROS
diff --git a/libcxx/include/experimental/__simd/scalar.h b/libcxx/include/experimental/__simd/scalar.h
index d7ac1225fd78..da318d2f4650 100644
--- a/libcxx/include/experimental/__simd/scalar.h
+++ b/libcxx/include/experimental/__simd/scalar.h
@@ -12,8 +12,8 @@
 
 #include <__assert>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/integral_constant.h>
-#include <cstddef>
 #include <experimental/__simd/declaration.h>
 #include <experimental/__simd/traits.h>
 
diff --git a/libcxx/include/experimental/__simd/simd.h b/libcxx/include/experimental/__simd/simd.h
index 8d8d96518d97..fd919e75e32f 100644
--- a/libcxx/include/experimental/__simd/simd.h
+++ b/libcxx/include/experimental/__simd/simd.h
@@ -11,12 +11,12 @@
 #define _LIBCPP_EXPERIMENTAL___SIMD_SIMD_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_integral.h>
 #include <__type_traits/is_same.h>
 #include <__type_traits/remove_cvref.h>
 #include <__utility/forward.h>
-#include <cstddef>
 #include <experimental/__simd/declaration.h>
 #include <experimental/__simd/reference.h>
 #include <experimental/__simd/traits.h>
diff --git a/libcxx/include/experimental/__simd/simd_mask.h b/libcxx/include/experimental/__simd/simd_mask.h
index 03e9da8519bf..6b6f671bf3e6 100644
--- a/libcxx/include/experimental/__simd/simd_mask.h
+++ b/libcxx/include/experimental/__simd/simd_mask.h
@@ -11,9 +11,9 @@
 #define _LIBCPP_EXPERIMENTAL___SIMD_SIMD_MASK_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_same.h>
-#include <cstddef>
 #include <experimental/__simd/declaration.h>
 #include <experimental/__simd/reference.h>
 #include <experimental/__simd/traits.h>
diff --git a/libcxx/include/experimental/__simd/traits.h b/libcxx/include/experimental/__simd/traits.h
index b817df604ef7..0bcc2eeba5ce 100644
--- a/libcxx/include/experimental/__simd/traits.h
+++ b/libcxx/include/experimental/__simd/traits.h
@@ -12,9 +12,9 @@
 
 #include <__bit/bit_ceil.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_same.h>
-#include <cstddef>
 #include <experimental/__simd/declaration.h>
 #include <experimental/__simd/utility.h>
 
diff --git a/libcxx/include/experimental/__simd/utility.h b/libcxx/include/experimental/__simd/utility.h
index 01736925d155..fd9fcecc7986 100644
--- a/libcxx/include/experimental/__simd/utility.h
+++ b/libcxx/include/experimental/__simd/utility.h
@@ -11,6 +11,7 @@
 #define _LIBCPP_EXPERIMENTAL___SIMD_UTILITY_H
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/is_arithmetic.h>
 #include <__type_traits/is_const.h>
 #include <__type_traits/is_constant_evaluated.h>
@@ -21,7 +22,6 @@
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
 #include <__utility/integer_sequence.h>
-#include <cstddef>
 #include <cstdint>
 #include <limits>
 
diff --git a/libcxx/include/experimental/__simd/vec_ext.h b/libcxx/include/experimental/__simd/vec_ext.h
index 6e8400948d46..abc7e9595be9 100644
--- a/libcxx/include/experimental/__simd/vec_ext.h
+++ b/libcxx/include/experimental/__simd/vec_ext.h
@@ -13,10 +13,10 @@
 #include <__assert>
 #include <__bit/bit_ceil.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__type_traits/integral_constant.h>
 #include <__utility/forward.h>
 #include <__utility/integer_sequence.h>
-#include <cstddef>
 #include <experimental/__simd/declaration.h>
 #include <experimental/__simd/traits.h>
 #include <experimental/__simd/utility.h>
diff --git a/libcxx/include/experimental/iterator b/libcxx/include/experimental/iterator
index e3a9c771fe61..2488bcfc155a 100644
--- a/libcxx/include/experimental/iterator
+++ b/libcxx/include/experimental/iterator
@@ -122,6 +122,7 @@ _LIBCPP_END_NAMESPACE_LFTS
 _LIBCPP_POP_MACROS
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
 #  include <iosfwd>
 #  include <type_traits>
 #endif
diff --git a/libcxx/include/experimental/memory b/libcxx/include/experimental/memory
index bf8a154690af..48e42a0a88a6 100644
--- a/libcxx/include/experimental/memory
+++ b/libcxx/include/experimental/memory
@@ -50,6 +50,8 @@ public:
 */
 
 #include <__config>
+#include <__cstddef/nullptr_t.h>
+#include <__cstddef/size_t.h>
 #include <__functional/hash.h>
 #include <__functional/operations.h>
 #include <__type_traits/add_lvalue_reference.h>
@@ -57,7 +59,7 @@ public:
 #include <__type_traits/common_type.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_convertible.h>
-#include <cstddef>
+#include <version>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -192,6 +194,7 @@ _LIBCPP_END_NAMESPACE_STD
 #endif // _LIBCPP_ENABLE_EXPERIMENTAL
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
 #  include <limits>
 #endif
 
diff --git a/libcxx/include/experimental/propagate_const b/libcxx/include/experimental/propagate_const
index 510d374bb4bf..8466d4e9c7ef 100644
--- a/libcxx/include/experimental/propagate_const
+++ b/libcxx/include/experimental/propagate_const
@@ -108,6 +108,8 @@
 */
 
 #include <__config>
+#include <__cstddef/nullptr_t.h>
+#include <__cstddef/size_t.h>
 #include <__functional/operations.h>
 #include <__fwd/functional.h>
 #include <__type_traits/conditional.h>
@@ -128,7 +130,7 @@
 #include <__utility/forward.h>
 #include <__utility/move.h>
 #include <__utility/swap.h>
-#include <cstddef>
+#include <version>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -484,6 +486,7 @@ _LIBCPP_END_NAMESPACE_STD
 _LIBCPP_POP_MACROS
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
 #  include <type_traits>
 #endif
 
diff --git a/libcxx/include/experimental/simd b/libcxx/include/experimental/simd
index 35120b4b4aab..1a868513d160 100644
--- a/libcxx/include/experimental/simd
+++ b/libcxx/include/experimental/simd
@@ -85,4 +85,8 @@ inline namespace parallelism_v2 {
 #include <experimental/__simd/traits.h>
 #include <experimental/__simd/vec_ext.h>
 
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
+#endif
+
 #endif /* _LIBCPP_EXPERIMENTAL_SIMD */
diff --git a/libcxx/include/experimental/type_traits b/libcxx/include/experimental/type_traits
index a4bb59afaf4a..6980fc3c51e4 100644
--- a/libcxx/include/experimental/type_traits
+++ b/libcxx/include/experimental/type_traits
@@ -148,6 +148,10 @@ constexpr bool is_detected_convertible_v = is_detected_convertible<_To, _Op, _Ar
 
 _LIBCPP_END_NAMESPACE_LFTS
 
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#  endif
+
 #endif /* _LIBCPP_STD_VER >= 14 */
 
 #endif /* _LIBCPP_EXPERIMENTAL_TYPE_TRAITS */
diff --git a/libcxx/include/experimental/utility b/libcxx/include/experimental/utility
index cbc7ad140e40..00151b967e49 100644
--- a/libcxx/include/experimental/utility
+++ b/libcxx/include/experimental/utility
@@ -43,4 +43,8 @@ struct _LIBCPP_TEMPLATE_VIS erased_type {};
 
 _LIBCPP_END_NAMESPACE_LFTS
 
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
+#endif
+
 #endif /* _LIBCPP_EXPERIMENTAL_UTILITY */
diff --git a/libcxx/include/initializer_list b/libcxx/include/initializer_list
index 680ca1cd20d5..8b9325069c12 100644
--- a/libcxx/include/initializer_list
+++ b/libcxx/include/initializer_list
@@ -43,7 +43,8 @@ template<class E> const E* end(initializer_list<E> il) noexcept; // constexpr in
 */
 
 #include <__config>
-#include <cstddef>
+#include <__cstddef/size_t.h>
+#include <version>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -95,4 +96,8 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Ep* end(initia
 
 } // namespace std
 
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
+#endif
+
 #endif // _LIBCPP_INITIALIZER_LIST
diff --git a/libcxx/include/iterator b/libcxx/include/iterator
index fca75f0a19ed..63fbae220b60 100644
--- a/libcxx/include/iterator
+++ b/libcxx/include/iterator
@@ -743,6 +743,7 @@ template <class E> constexpr const E* data(initializer_list<E> il) noexcept;
 #endif
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
 #  include <cstdlib>
 #  include <exception>
 #  include <new>
diff --git a/libcxx/include/latch b/libcxx/include/latch
index b56e49bc768b..92dadf68bcaa 100644
--- a/libcxx/include/latch
+++ b/libcxx/include/latch
@@ -48,7 +48,7 @@ namespace std
 #  include <__atomic/atomic_base.h>
 #  include <__atomic/atomic_sync.h>
 #  include <__atomic/memory_order.h>
-#  include <cstddef>
+#  include <__cstddef/ptrdiff_t.h>
 #  include <limits>
 #  include <version>
 
@@ -124,6 +124,7 @@ _LIBCPP_POP_MACROS
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #  include <atomic>
+#  include <cstddef>
 #endif
 
 #endif // _LIBCPP_LATCH
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index af8c3c15eb27..70f91249a58e 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -13,7 +13,10 @@ module std_config [system] {
 module std_core [system] {
   module cstddef {
     module byte         { header "__cstddef/byte.h" }
-    module max_align_t  { header "__cstddef/max_align_t.h" }
+    module max_align_t  {
+      header "__cstddef/max_align_t.h"
+      export *
+    }
     module nullptr_t    { header "__cstddef/nullptr_t.h" }
     module ptrdiff_t    { header "__cstddef/ptrdiff_t.h" }
     module size_t       { header "__cstddef/size_t.h" }
@@ -1408,7 +1411,10 @@ module std [system] {
     module indirectly_comparable      { header "__iterator/indirectly_comparable.h" }
     module insert_iterator            { header "__iterator/insert_iterator.h" }
     module istream_iterator           { header "__iterator/istream_iterator.h" }
-    module istreambuf_iterator        { header "__iterator/istreambuf_iterator.h" }
+    module istreambuf_iterator        {
+      header "__iterator/istreambuf_iterator.h"
+      export std.string.char_traits
+    }
     module iter_move                  { header "__iterator/iter_move.h" }
     module iter_swap                  { header "__iterator/iter_swap.h" }
     module iterator_traits {
diff --git a/libcxx/include/mutex b/libcxx/include/mutex
index 02c52dd72f02..427fce5f3ec4 100644
--- a/libcxx/include/mutex
+++ b/libcxx/include/mutex
@@ -199,7 +199,6 @@ template<class Callable, class ...Args>
 #include <__thread/id.h>
 #include <__thread/support.h>
 #include <__utility/forward.h>
-#include <cstddef>
 #include <limits>
 #ifndef _LIBCPP_CXX03_LANG
 #  include <tuple>
diff --git a/libcxx/include/new b/libcxx/include/new
index 75e2b8742df6..290ad9e97f8d 100644
--- a/libcxx/include/new
+++ b/libcxx/include/new
@@ -87,12 +87,12 @@ void  operator delete[](void* ptr, void*) noexcept;
 */
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__exception/exception.h>
 #include <__type_traits/is_function.h>
 #include <__type_traits/is_same.h>
 #include <__type_traits/remove_cv.h>
 #include <__verbose_abort>
-#include <cstddef>
 #include <version>
 
 #if defined(_LIBCPP_ABI_VCRUNTIME)
@@ -367,6 +367,7 @@ inline constexpr size_t hardware_constructive_interference_size = __GCC_CONSTRUC
 _LIBCPP_END_NAMESPACE_STD
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
 #  include <cstdlib>
 #  include <type_traits>
 #endif
diff --git a/libcxx/include/numbers b/libcxx/include/numbers
index f48ba4baf38f..191563c3d8a5 100644
--- a/libcxx/include/numbers
+++ b/libcxx/include/numbers
@@ -158,6 +158,7 @@ _LIBCPP_END_NAMESPACE_STD
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #  include <concepts>
+#  include <cstddef>
 #  include <type_traits>
 #endif
 
diff --git a/libcxx/include/semaphore b/libcxx/include/semaphore
index bf6317c587e2..98122c96459a 100644
--- a/libcxx/include/semaphore
+++ b/libcxx/include/semaphore
@@ -54,10 +54,10 @@ using binary_semaphore = counting_semaphore<1>; // since C++20
 #  include <__atomic/atomic_sync.h>
 #  include <__atomic/memory_order.h>
 #  include <__chrono/time_point.h>
+#  include <__cstddef/ptrdiff_t.h>
 #  include <__thread/poll_with_backoff.h>
 #  include <__thread/support.h>
 #  include <__thread/timed_backoff_policy.h>
-#  include <cstddef>
 #  include <limits>
 #  include <version>
 
@@ -181,6 +181,7 @@ _LIBCPP_POP_MACROS
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #  include <atomic>
+#  include <cstddef>
 #endif
 
 #endif // _LIBCPP_SEMAPHORE
diff --git a/libcxx/include/span b/libcxx/include/span
index a32f7a372e2a..896a3cd89018 100644
--- a/libcxx/include/span
+++ b/libcxx/include/span
@@ -148,6 +148,8 @@ template<class R>
 #include <__concepts/convertible_to.h>
 #include <__concepts/equality_comparable.h>
 #include <__config>
+#include <__cstddef/byte.h>
+#include <__cstddef/ptrdiff_t.h>
 #include <__fwd/array.h>
 #include <__fwd/span.h>
 #include <__iterator/bounded_iter.h>
@@ -173,7 +175,6 @@ template<class R>
 #include <__type_traits/remove_reference.h>
 #include <__type_traits/type_identity.h>
 #include <__utility/forward.h>
-#include <cstddef> // for byte
 #include <initializer_list>
 #include <stdexcept>
 #include <version>
diff --git a/libcxx/include/stdexcept b/libcxx/include/stdexcept
index daa7b501a869..8415d3339f7e 100644
--- a/libcxx/include/stdexcept
+++ b/libcxx/include/stdexcept
@@ -278,6 +278,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 _LIBCPP_END_NAMESPACE_STD
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
 #  include <cstdlib>
 #  include <exception>
 #  include <iosfwd>
diff --git a/libcxx/include/stop_token b/libcxx/include/stop_token
index d4e651d9541f..cf8d0cf9b919 100644
--- a/libcxx/include/stop_token
+++ b/libcxx/include/stop_token
@@ -50,6 +50,7 @@ namespace std {
 #endif // !defined(_LIBCPP_HAS_NO_THREADS)
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
 #  include <iosfwd>
 #endif
 
diff --git a/libcxx/include/string_view b/libcxx/include/string_view
index 5beac404fb5b..0edda7aeb1a7 100644
--- a/libcxx/include/string_view
+++ b/libcxx/include/string_view
@@ -208,6 +208,8 @@ namespace std {
 #include <__algorithm/min.h>
 #include <__assert>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
+#include <__cstddef/size_t.h>
 #include <__functional/hash.h>
 #include <__functional/unary_function.h>
 #include <__fwd/ostream.h>
@@ -233,7 +235,6 @@ namespace std {
 #include <__type_traits/remove_cvref.h>
 #include <__type_traits/remove_reference.h>
 #include <__type_traits/type_identity.h>
-#include <cstddef>
 #include <iosfwd>
 #include <limits>
 #include <stdexcept>
diff --git a/libcxx/include/tuple b/libcxx/include/tuple
index e7e14b8d12d4..c3f7b8041686 100644
--- a/libcxx/include/tuple
+++ b/libcxx/include/tuple
@@ -214,6 +214,7 @@ template <class... Types>
 #include <__compare/ordering.h>
 #include <__compare/synth_three_way.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__fwd/array.h>
 #include <__fwd/pair.h>
 #include <__fwd/tuple.h>
@@ -262,7 +263,6 @@ template <class... Types>
 #include <__utility/move.h>
 #include <__utility/piecewise_construct.h>
 #include <__utility/swap.h>
-#include <cstddef>
 #include <version>
 
 // standard-mandated includes
@@ -1411,6 +1411,7 @@ _LIBCPP_POP_MACROS
 // clang-format on
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
 #  include <exception>
 #  include <iosfwd>
 #  include <new>
diff --git a/libcxx/include/typeindex b/libcxx/include/typeindex
index 6398aa40d616..9f8e65befcba 100644
--- a/libcxx/include/typeindex
+++ b/libcxx/include/typeindex
@@ -98,6 +98,7 @@ struct _LIBCPP_TEMPLATE_VIS hash<type_index> : public __unary_function<type_inde
 _LIBCPP_END_NAMESPACE_STD
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
 #  include <iosfwd>
 #  include <new>
 #  include <utility>
diff --git a/libcxx/include/typeinfo b/libcxx/include/typeinfo
index 252afe59a0aa..28713077c688 100644
--- a/libcxx/include/typeinfo
+++ b/libcxx/include/typeinfo
@@ -57,12 +57,13 @@ public:
 */
 
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__exception/exception.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_constant_evaluated.h>
 #include <__verbose_abort>
-#include <cstddef>
 #include <cstdint>
+#include <version>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -383,6 +384,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 _LIBCPP_END_NAMESPACE_STD
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
 #  include <cstdlib>
 #  include <type_traits>
 #endif
diff --git a/libcxx/include/utility b/libcxx/include/utility
index f97907fbf72e..138cc3ba3189 100644
--- a/libcxx/include/utility
+++ b/libcxx/include/utility
@@ -301,6 +301,7 @@ template <class T>
 #endif
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
 #  include <cstdlib>
 #  include <iosfwd>
 #  include <type_traits>
diff --git a/libcxx/include/valarray b/libcxx/include/valarray
index b3b48958f92b..2f7a1a7c5b49 100644
--- a/libcxx/include/valarray
+++ b/libcxx/include/valarray
@@ -352,6 +352,7 @@ template <class T> unspecified2 end(const valarray<T>& v);
 #include <__algorithm/unwrap_iter.h>
 #include <__assert>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__functional/operations.h>
 #include <__memory/addressof.h>
 #include <__memory/allocator.h>
@@ -361,7 +362,6 @@ template <class T> unspecified2 end(const valarray<T>& v);
 #include <__utility/move.h>
 #include <__utility/swap.h>
 #include <cmath>
-#include <cstddef>
 #include <new>
 #include <version>
 
diff --git a/libcxx/include/variant b/libcxx/include/variant
index 2e158a4eea31..ee80fb0b5ab5 100644
--- a/libcxx/include/variant
+++ b/libcxx/include/variant
@@ -1631,6 +1631,7 @@ _LIBCPP_END_NAMESPACE_STD
 _LIBCPP_POP_MACROS
 
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#  include <cstddef>
 #  include <exception>
 #  include <tuple>
 #  include <type_traits>
diff --git a/libcxx/src/memory_resource.cpp b/libcxx/src/memory_resource.cpp
index 299f810948fd..3d0d4ead1be7 100644
--- a/libcxx/src/memory_resource.cpp
+++ b/libcxx/src/memory_resource.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <cstddef>
 #include <memory>
 #include <memory_resource>
 
diff --git a/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.random.shuffle/random_shuffle.cxx1z.pass.cpp b/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.random.shuffle/random_shuffle.cxx1z.pass.cpp
index c1acc100a660..4e51014f20b1 100644
--- a/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.random.shuffle/random_shuffle.cxx1z.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.random.shuffle/random_shuffle.cxx1z.pass.cpp
@@ -26,6 +26,7 @@
 // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 
 #include <algorithm>
+#include <cstddef>
 #include <vector>
 
 #include "test_macros.h"
diff --git a/libcxx/test/libcxx/algorithms/alg.modifying.operations/copy_move_unwrap_reverse.pass.cpp b/libcxx/test/libcxx/algorithms/alg.modifying.operations/copy_move_unwrap_reverse.pass.cpp
index f295b807864e..2a85e7b5ddcc 100644
--- a/libcxx/test/libcxx/algorithms/alg.modifying.operations/copy_move_unwrap_reverse.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/alg.modifying.operations/copy_move_unwrap_reverse.pass.cpp
@@ -15,9 +15,9 @@
 
 #include <algorithm>
 #include <cassert>
+#include <cstddef>
 #include <cstdint>
 #include <iterator>
-#include <ranges>
 #include <type_traits>
 
 #include "test_iterators.h"
diff --git a/libcxx/test/libcxx/algorithms/robust_against_using_non_transparent_comparators.pass.cpp b/libcxx/test/libcxx/algorithms/robust_against_using_non_transparent_comparators.pass.cpp
index eaa5d44385be..39870ebe7ff0 100644
--- a/libcxx/test/libcxx/algorithms/robust_against_using_non_transparent_comparators.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/robust_against_using_non_transparent_comparators.pass.cpp
@@ -8,6 +8,7 @@
 
 #include <algorithm>
 #include <cassert>
+#include <cstddef>
 #include <iterator>
 
 #include "test_macros.h"
diff --git a/libcxx/test/libcxx/containers/sequences/deque/asan_turning_off.pass.cpp b/libcxx/test/libcxx/containers/sequences/deque/asan_turning_off.pass.cpp
index e9b9cde64ee9..b31775a87348 100644
--- a/libcxx/test/libcxx/containers/sequences/deque/asan_turning_off.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/deque/asan_turning_off.pass.cpp
@@ -17,6 +17,7 @@
 // This test confirms that those allocators work after turning off annotations.
 
 #include <cassert>
+#include <cstddef>
 #include <deque>
 #include <new>
 
diff --git a/libcxx/test/libcxx/transitive_includes.gen.py b/libcxx/test/libcxx/transitive_includes.gen.py
index 2693617bcb0e..f01dbac26a8e 100644
--- a/libcxx/test/libcxx/transitive_includes.gen.py
+++ b/libcxx/test/libcxx/transitive_includes.gen.py
@@ -73,7 +73,7 @@ else:
 {lit_header_restrictions.get(header, '')}
 
 // TODO: Fix this test to make it work with localization or wide characters disabled
-// UNSUPPORTED: no-localization, no-wide-characters, no-threads, no-filesystem, libcpp-has-no-experimental-tzdb, no-tzdb
+// UNSUPPORTED: no-localization, no-wide-characters, no-threads, no-filesystem, libcpp-has-no-experimental-tzdb
 
 // When built with modules, this test doesn't work because --trace-includes doesn't
 // report the stack of includes correctly.
diff --git a/libcxx/test/libcxx/transitive_includes/cxx03.csv b/libcxx/test/libcxx/transitive_includes/cxx03.csv
index 48c501863cb7..ae4254183fc8 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx03.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx03.csv
@@ -7,6 +7,7 @@ algorithm compare
 algorithm concepts
 algorithm cstddef
 algorithm cstdint
+algorithm cstdio
 algorithm cstdlib
 algorithm cstring
 algorithm ctime
@@ -86,6 +87,7 @@ array compare
 array concepts
 array cstddef
 array cstdint
+array cstdio
 array cstdlib
 array cstring
 array ctime
@@ -1181,13 +1183,17 @@ istream utility
 istream variant
 istream vector
 istream version
+iterator cctype
 iterator cmath
 iterator compare
 iterator concepts
 iterator cstddef
 iterator cstdint
+iterator cstdio
 iterator cstdlib
 iterator cstring
+iterator cwchar
+iterator cwctype
 iterator exception
 iterator initializer_list
 iterator iosfwd
@@ -1397,15 +1403,19 @@ mdspan variant
 mdspan vector
 mdspan version
 memory atomic
+memory cctype
 memory climits
 memory cmath
 memory compare
 memory concepts
 memory cstddef
 memory cstdint
+memory cstdio
 memory cstdlib
 memory cstring
 memory ctime
+memory cwchar
+memory cwctype
 memory exception
 memory initializer_list
 memory iosfwd
@@ -1551,15 +1561,19 @@ numeric variant
 numeric vector
 numeric version
 optional atomic
+optional cctype
 optional climits
 optional cmath
 optional compare
 optional concepts
 optional cstddef
 optional cstdint
+optional cstdio
 optional cstdlib
 optional cstring
 optional ctime
+optional cwchar
+optional cwctype
 optional exception
 optional initializer_list
 optional iosfwd
@@ -1871,15 +1885,19 @@ regex variant
 regex vector
 regex version
 scoped_allocator atomic
+scoped_allocator cctype
 scoped_allocator climits
 scoped_allocator cmath
 scoped_allocator compare
 scoped_allocator concepts
 scoped_allocator cstddef
 scoped_allocator cstdint
+scoped_allocator cstdio
 scoped_allocator cstdlib
 scoped_allocator cstring
 scoped_allocator ctime
+scoped_allocator cwchar
+scoped_allocator cwctype
 scoped_allocator exception
 scoped_allocator initializer_list
 scoped_allocator iosfwd
@@ -2499,6 +2517,7 @@ unordered_map compare
 unordered_map concepts
 unordered_map cstddef
 unordered_map cstdint
+unordered_map cstdio
 unordered_map cstdlib
 unordered_map cstring
 unordered_map ctime
diff --git a/libcxx/test/libcxx/transitive_includes/cxx11.csv b/libcxx/test/libcxx/transitive_includes/cxx11.csv
index 48c501863cb7..ae4254183fc8 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx11.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx11.csv
@@ -7,6 +7,7 @@ algorithm compare
 algorithm concepts
 algorithm cstddef
 algorithm cstdint
+algorithm cstdio
 algorithm cstdlib
 algorithm cstring
 algorithm ctime
@@ -86,6 +87,7 @@ array compare
 array concepts
 array cstddef
 array cstdint
+array cstdio
 array cstdlib
 array cstring
 array ctime
@@ -1181,13 +1183,17 @@ istream utility
 istream variant
 istream vector
 istream version
+iterator cctype
 iterator cmath
 iterator compare
 iterator concepts
 iterator cstddef
 iterator cstdint
+iterator cstdio
 iterator cstdlib
 iterator cstring
+iterator cwchar
+iterator cwctype
 iterator exception
 iterator initializer_list
 iterator iosfwd
@@ -1397,15 +1403,19 @@ mdspan variant
 mdspan vector
 mdspan version
 memory atomic
+memory cctype
 memory climits
 memory cmath
 memory compare
 memory concepts
 memory cstddef
 memory cstdint
+memory cstdio
 memory cstdlib
 memory cstring
 memory ctime
+memory cwchar
+memory cwctype
 memory exception
 memory initializer_list
 memory iosfwd
@@ -1551,15 +1561,19 @@ numeric variant
 numeric vector
 numeric version
 optional atomic
+optional cctype
 optional climits
 optional cmath
 optional compare
 optional concepts
 optional cstddef
 optional cstdint
+optional cstdio
 optional cstdlib
 optional cstring
 optional ctime
+optional cwchar
+optional cwctype
 optional exception
 optional initializer_list
 optional iosfwd
@@ -1871,15 +1885,19 @@ regex variant
 regex vector
 regex version
 scoped_allocator atomic
+scoped_allocator cctype
 scoped_allocator climits
 scoped_allocator cmath
 scoped_allocator compare
 scoped_allocator concepts
 scoped_allocator cstddef
 scoped_allocator cstdint
+scoped_allocator cstdio
 scoped_allocator cstdlib
 scoped_allocator cstring
 scoped_allocator ctime
+scoped_allocator cwchar
+scoped_allocator cwctype
 scoped_allocator exception
 scoped_allocator initializer_list
 scoped_allocator iosfwd
@@ -2499,6 +2517,7 @@ unordered_map compare
 unordered_map concepts
 unordered_map cstddef
 unordered_map cstdint
+unordered_map cstdio
 unordered_map cstdlib
 unordered_map cstring
 unordered_map ctime
diff --git a/libcxx/test/libcxx/transitive_includes/cxx14.csv b/libcxx/test/libcxx/transitive_includes/cxx14.csv
index 6191c9012c63..f14b31700037 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx14.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx14.csv
@@ -7,6 +7,7 @@ algorithm compare
 algorithm concepts
 algorithm cstddef
 algorithm cstdint
+algorithm cstdio
 algorithm cstdlib
 algorithm cstring
 algorithm ctime
@@ -88,6 +89,7 @@ array compare
 array concepts
 array cstddef
 array cstdint
+array cstdio
 array cstdlib
 array cstring
 array ctime
@@ -1209,13 +1211,17 @@ istream utility
 istream variant
 istream vector
 istream version
+iterator cctype
 iterator cmath
 iterator compare
 iterator concepts
 iterator cstddef
 iterator cstdint
+iterator cstdio
 iterator cstdlib
 iterator cstring
+iterator cwchar
+iterator cwctype
 iterator exception
 iterator initializer_list
 iterator iosfwd
@@ -1429,15 +1435,19 @@ mdspan variant
 mdspan vector
 mdspan version
 memory atomic
+memory cctype
 memory climits
 memory cmath
 memory compare
 memory concepts
 memory cstddef
 memory cstdint
+memory cstdio
 memory cstdlib
 memory cstring
 memory ctime
+memory cwchar
+memory cwctype
 memory exception
 memory initializer_list
 memory iosfwd
@@ -1585,15 +1595,19 @@ numeric variant
 numeric vector
 numeric version
 optional atomic
+optional cctype
 optional climits
 optional cmath
 optional compare
 optional concepts
 optional cstddef
 optional cstdint
+optional cstdio
 optional cstdlib
 optional cstring
 optional ctime
+optional cwchar
+optional cwctype
 optional exception
 optional initializer_list
 optional iosfwd
@@ -1910,15 +1924,19 @@ regex variant
 regex vector
 regex version
 scoped_allocator atomic
+scoped_allocator cctype
 scoped_allocator climits
 scoped_allocator cmath
 scoped_allocator compare
 scoped_allocator concepts
 scoped_allocator cstddef
 scoped_allocator cstdint
+scoped_allocator cstdio
 scoped_allocator cstdlib
 scoped_allocator cstring
 scoped_allocator ctime
+scoped_allocator cwchar
+scoped_allocator cwctype
 scoped_allocator exception
 scoped_allocator initializer_list
 scoped_allocator iosfwd
@@ -2550,6 +2568,7 @@ unordered_map compare
 unordered_map concepts
 unordered_map cstddef
 unordered_map cstdint
+unordered_map cstdio
 unordered_map cstdlib
 unordered_map cstring
 unordered_map ctime
diff --git a/libcxx/test/libcxx/transitive_includes/cxx17.csv b/libcxx/test/libcxx/transitive_includes/cxx17.csv
index 5d46162e3f89..d4bc0a38c164 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx17.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx17.csv
@@ -7,6 +7,7 @@ algorithm compare
 algorithm concepts
 algorithm cstddef
 algorithm cstdint
+algorithm cstdio
 algorithm cstdlib
 algorithm cstring
 algorithm ctime
@@ -86,6 +87,7 @@ array compare
 array concepts
 array cstddef
 array cstdint
+array cstdio
 array cstdlib
 array cstring
 array ctime
@@ -1205,13 +1207,17 @@ istream utility
 istream variant
 istream vector
 istream version
+iterator cctype
 iterator cmath
 iterator compare
 iterator concepts
 iterator cstddef
 iterator cstdint
+iterator cstdio
 iterator cstdlib
 iterator cstring
+iterator cwchar
+iterator cwctype
 iterator exception
 iterator initializer_list
 iterator iosfwd
@@ -1421,15 +1427,19 @@ mdspan variant
 mdspan vector
 mdspan version
 memory atomic
+memory cctype
 memory climits
 memory cmath
 memory compare
 memory concepts
 memory cstddef
 memory cstdint
+memory cstdio
 memory cstdlib
 memory cstring
 memory ctime
+memory cwchar
+memory cwctype
 memory exception
 memory initializer_list
 memory iosfwd
@@ -1575,15 +1585,19 @@ numeric variant
 numeric vector
 numeric version
 optional atomic
+optional cctype
 optional climits
 optional cmath
 optional compare
 optional concepts
 optional cstddef
 optional cstdint
+optional cstdio
 optional cstdlib
 optional cstring
 optional ctime
+optional cwchar
+optional cwctype
 optional exception
 optional initializer_list
 optional iosfwd
@@ -1895,15 +1909,19 @@ regex variant
 regex vector
 regex version
 scoped_allocator atomic
+scoped_allocator cctype
 scoped_allocator climits
 scoped_allocator cmath
 scoped_allocator compare
 scoped_allocator concepts
 scoped_allocator cstddef
 scoped_allocator cstdint
+scoped_allocator cstdio
 scoped_allocator cstdlib
 scoped_allocator cstring
 scoped_allocator ctime
+scoped_allocator cwchar
+scoped_allocator cwctype
 scoped_allocator exception
 scoped_allocator initializer_list
 scoped_allocator iosfwd
@@ -2523,6 +2541,7 @@ unordered_map compare
 unordered_map concepts
 unordered_map cstddef
 unordered_map cstdint
+unordered_map cstdio
 unordered_map cstdlib
 unordered_map cstring
 unordered_map ctime
diff --git a/libcxx/test/libcxx/transitive_includes/cxx20.csv b/libcxx/test/libcxx/transitive_includes/cxx20.csv
index 20fe9878ce3e..304166547abf 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx20.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx20.csv
@@ -7,6 +7,7 @@ algorithm compare
 algorithm concepts
 algorithm cstddef
 algorithm cstdint
+algorithm cstdio
 algorithm cstdlib
 algorithm cstring
 algorithm ctime
@@ -29,15 +30,19 @@ algorithm utility
 algorithm variant
 algorithm version
 any atomic
+any cctype
 any climits
 any cmath
 any compare
 any concepts
 any cstddef
 any cstdint
+any cstdio
 any cstdlib
 any cstring
 any ctime
+any cwchar
+any cwctype
 any exception
 any initializer_list
 any iosfwd
@@ -63,6 +68,7 @@ array compare
 array concepts
 array cstddef
 array cstdint
+array cstdio
 array cstdlib
 array cstring
 array ctime
@@ -97,15 +103,19 @@ atomic ratio
 atomic type_traits
 atomic version
 barrier atomic
+barrier cctype
 barrier climits
 barrier cmath
 barrier compare
 barrier concepts
 barrier cstddef
 barrier cstdint
+barrier cstdio
 barrier cstdlib
 barrier cstring
 barrier ctime
+barrier cwchar
+barrier cwctype
 barrier exception
 barrier initializer_list
 barrier iosfwd
@@ -1191,13 +1201,17 @@ istream utility
 istream variant
 istream vector
 istream version
+iterator cctype
 iterator cmath
 iterator compare
 iterator concepts
 iterator cstddef
 iterator cstdint
+iterator cstdio
 iterator cstdlib
 iterator cstring
+iterator cwchar
+iterator cwctype
 iterator exception
 iterator initializer_list
 iterator iosfwd
@@ -1407,15 +1421,19 @@ mdspan variant
 mdspan vector
 mdspan version
 memory atomic
+memory cctype
 memory climits
 memory cmath
 memory compare
 memory concepts
 memory cstddef
 memory cstdint
+memory cstdio
 memory cstdlib
 memory cstring
 memory ctime
+memory cwchar
+memory cwctype
 memory exception
 memory initializer_list
 memory iosfwd
@@ -1561,15 +1579,19 @@ numeric variant
 numeric vector
 numeric version
 optional atomic
+optional cctype
 optional climits
 optional cmath
 optional compare
 optional concepts
 optional cstddef
 optional cstdint
+optional cstdio
 optional cstdlib
 optional cstring
 optional ctime
+optional cwchar
+optional cwctype
 optional exception
 optional initializer_list
 optional iosfwd
@@ -1881,15 +1903,19 @@ regex variant
 regex vector
 regex version
 scoped_allocator atomic
+scoped_allocator cctype
 scoped_allocator climits
 scoped_allocator cmath
 scoped_allocator compare
 scoped_allocator concepts
 scoped_allocator cstddef
 scoped_allocator cstdint
+scoped_allocator cstdio
 scoped_allocator cstdlib
 scoped_allocator cstring
 scoped_allocator ctime
+scoped_allocator cwchar
+scoped_allocator cwctype
 scoped_allocator exception
 scoped_allocator initializer_list
 scoped_allocator iosfwd
@@ -2519,6 +2545,7 @@ unordered_map compare
 unordered_map concepts
 unordered_map cstddef
 unordered_map cstdint
+unordered_map cstdio
 unordered_map cstdlib
 unordered_map cstring
 unordered_map ctime
diff --git a/libcxx/test/libcxx/transitive_includes/cxx23.csv b/libcxx/test/libcxx/transitive_includes/cxx23.csv
index 5ee89ec307cc..48d4425c0333 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx23.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx23.csv
@@ -1,7 +1,6 @@
 algorithm cctype
 algorithm climits
 algorithm compare
-algorithm cstddef
 algorithm cstdint
 algorithm cstring
 algorithm ctime
@@ -15,7 +14,6 @@ algorithm optional
 algorithm ratio
 algorithm tuple
 algorithm version
-any cstddef
 any cstdint
 any cstring
 any initializer_list
@@ -25,7 +23,6 @@ any typeinfo
 any version
 array cctype
 array compare
-array cstddef
 array cstdint
 array cwchar
 array cwctype
@@ -35,7 +32,6 @@ array new
 array stdexcept
 array version
 atomic climits
-atomic cstddef
 atomic cstdint
 atomic cstring
 atomic ctime
@@ -43,7 +39,6 @@ atomic limits
 atomic ratio
 atomic version
 barrier climits
-barrier cstddef
 barrier cstdint
 barrier cstring
 barrier ctime
@@ -57,7 +52,6 @@ bit version
 bitset cctype
 bitset climits
 bitset compare
-bitset cstddef
 bitset cstdint
 bitset cstdio
 bitset cstring
@@ -105,7 +99,6 @@ ccomplex tuple
 ccomplex typeinfo
 ccomplex version
 charconv cerrno
-charconv cstddef
 charconv cstdint
 charconv initializer_list
 charconv limits
@@ -170,7 +163,6 @@ codecvt string_view
 codecvt tuple
 codecvt typeinfo
 codecvt version
-compare cstddef
 compare cstdint
 compare limits
 compare version
@@ -205,14 +197,12 @@ complex string_view
 complex tuple
 complex typeinfo
 complex version
-concepts cstddef
 concepts version
 condition_variable atomic
 condition_variable cctype
 condition_variable cerrno
 condition_variable climits
 condition_variable compare
-condition_variable cstddef
 condition_variable cstdint
 condition_variable cstdio
 condition_variable cstring
@@ -231,7 +221,6 @@ condition_variable tuple
 condition_variable typeinfo
 condition_variable version
 coroutine compare
-coroutine cstddef
 coroutine cstdint
 coroutine cstring
 coroutine limits
@@ -274,7 +263,6 @@ cwchar cwctype
 cwctype cctype
 deque cctype
 deque compare
-deque cstddef
 deque cstdint
 deque cstring
 deque cwchar
@@ -285,14 +273,12 @@ deque new
 deque stdexcept
 deque tuple
 deque version
-exception cstddef
 exception cstdint
 exception cstdlib
 exception new
 exception typeinfo
 exception version
 execution version
-expected cstddef
 expected cstdint
 expected initializer_list
 expected new
@@ -328,23 +314,18 @@ experimental/iterator tuple
 experimental/iterator typeinfo
 experimental/iterator variant
 experimental/iterator version
-experimental/memory cstddef
 experimental/memory cstdint
 experimental/memory cstring
 experimental/memory version
-experimental/propagate_const cstddef
 experimental/propagate_const version
-experimental/simd cstddef
 experimental/simd cstdint
 experimental/simd limits
 experimental/simd version
-experimental/type_traits cstddef
 experimental/type_traits cstdint
 experimental/type_traits initializer_list
 experimental/type_traits type_traits
 experimental/type_traits version
 experimental/utility compare
-experimental/utility cstddef
 experimental/utility cstdint
 experimental/utility initializer_list
 experimental/utility limits
@@ -432,7 +413,6 @@ format typeinfo
 format version
 forward_list cctype
 forward_list compare
-forward_list cstddef
 forward_list cstdint
 forward_list cwchar
 forward_list cwctype
@@ -481,7 +461,6 @@ fstream version
 functional array
 functional cctype
 functional compare
-functional cstddef
 functional cstdint
 functional cstring
 functional cwchar
@@ -529,7 +508,6 @@ future thread
 future tuple
 future typeinfo
 future version
-initializer_list cstddef
 initializer_list version
 iomanip bitset
 iomanip cctype
@@ -648,11 +626,14 @@ istream string_view
 istream tuple
 istream typeinfo
 istream version
+iterator cctype
 iterator compare
 iterator concepts
-iterator cstddef
 iterator cstdint
+iterator cstdio
 iterator cstring
+iterator cwchar
+iterator cwctype
 iterator initializer_list
 iterator iosfwd
 iterator limits
@@ -660,7 +641,6 @@ iterator new
 iterator variant
 iterator version
 latch climits
-latch cstddef
 latch cstdint
 latch cstring
 latch ctime
@@ -670,7 +650,6 @@ latch version
 limits version
 list cctype
 list compare
-list cstddef
 list cstdint
 list cstring
 list cwchar
@@ -708,7 +687,6 @@ locale typeinfo
 locale version
 map cctype
 map compare
-map cstddef
 map cstdint
 map cstring
 map cwchar
@@ -722,10 +700,8 @@ map tuple
 map version
 mdspan array
 mdspan cctype
-mdspan cinttypes
 mdspan compare
 mdspan concepts
-mdspan cstddef
 mdspan cstdint
 mdspan cwchar
 mdspan cwctype
@@ -736,7 +712,6 @@ mdspan span
 mdspan stdexcept
 mdspan version
 memory compare
-memory cstddef
 memory cstdint
 memory cstring
 memory initializer_list
@@ -749,7 +724,6 @@ memory_resource cctype
 memory_resource cerrno
 memory_resource climits
 memory_resource compare
-memory_resource cstddef
 memory_resource cstdint
 memory_resource cstdio
 memory_resource cstring
@@ -772,7 +746,6 @@ mutex cctype
 mutex cerrno
 mutex climits
 mutex compare
-mutex cstddef
 mutex cstdint
 mutex cstdio
 mutex cstring
@@ -790,12 +763,10 @@ mutex string_view
 mutex tuple
 mutex typeinfo
 mutex version
-new cstddef
 new version
 numbers version
 numeric climits
 numeric compare
-numeric cstddef
 numeric cstdint
 numeric cstring
 numeric ctime
@@ -807,7 +778,6 @@ numeric ratio
 numeric tuple
 numeric version
 optional compare
-optional cstddef
 optional cstdint
 optional cstring
 optional initializer_list
@@ -902,7 +872,6 @@ random cctype
 random climits
 random cmath
 random compare
-random cstddef
 random cstdint
 random cstdio
 random cstring
@@ -924,8 +893,8 @@ random version
 ranges cctype
 ranges compare
 ranges concepts
-ranges cstddef
 ranges cstdint
+ranges cstdio
 ranges cstring
 ranges cwchar
 ranges cwctype
@@ -969,14 +938,12 @@ regex typeinfo
 regex vector
 regex version
 scoped_allocator compare
-scoped_allocator cstddef
 scoped_allocator cstdint
 scoped_allocator limits
 scoped_allocator new
 scoped_allocator tuple
 scoped_allocator version
 semaphore climits
-semaphore cstddef
 semaphore cstdint
 semaphore cstring
 semaphore ctime
@@ -985,7 +952,6 @@ semaphore ratio
 semaphore version
 set cctype
 set compare
-set cstddef
 set cstdint
 set cstring
 set cwchar
@@ -1000,7 +966,6 @@ shared_mutex cctype
 shared_mutex cerrno
 shared_mutex climits
 shared_mutex compare
-shared_mutex cstddef
 shared_mutex cstdint
 shared_mutex cstdio
 shared_mutex cstring
@@ -1019,7 +984,6 @@ shared_mutex tuple
 shared_mutex version
 source_location cstdint
 source_location version
-span cstddef
 span initializer_list
 span limits
 span stdexcept
@@ -1055,7 +1019,6 @@ sstream typeinfo
 sstream version
 stack cctype
 stack compare
-stack cstddef
 stack cstdint
 stack cstring
 stack cwchar
@@ -1069,7 +1032,6 @@ stack tuple
 stack version
 stop_token atomic
 stop_token climits
-stop_token cstddef
 stop_token cstdint
 stop_token cstring
 stop_token ctime
@@ -1104,7 +1066,6 @@ streambuf version
 string cctype
 string climits
 string compare
-string cstddef
 string cstdint
 string cstdio
 string cstring
@@ -1120,7 +1081,6 @@ string tuple
 string version
 string_view cctype
 string_view compare
-string_view cstddef
 string_view cstdint
 string_view cstdio
 string_view cstring
@@ -1208,7 +1168,6 @@ system_error cctype
 system_error cerrno
 system_error climits
 system_error compare
-system_error cstddef
 system_error cstdint
 system_error cstdio
 system_error cstring
@@ -1256,23 +1215,19 @@ thread tuple
 thread typeinfo
 thread version
 tuple compare
-tuple cstddef
 tuple cstdint
 tuple limits
 tuple version
 type_traits cstdint
 type_traits version
 typeindex compare
-typeindex cstddef
 typeindex cstdint
 typeindex limits
 typeindex typeinfo
 typeindex version
-typeinfo cstddef
 typeinfo cstdint
 typeinfo version
 unordered_map compare
-unordered_map cstddef
 unordered_map cstdint
 unordered_map cstring
 unordered_map initializer_list
@@ -1283,7 +1238,6 @@ unordered_map stdexcept
 unordered_map tuple
 unordered_map version
 unordered_set compare
-unordered_set cstddef
 unordered_set cstdint
 unordered_set cstring
 unordered_set initializer_list
@@ -1293,20 +1247,17 @@ unordered_set optional
 unordered_set tuple
 unordered_set version
 utility compare
-utility cstddef
 utility cstdint
 utility initializer_list
 utility limits
 utility version
 valarray cmath
-valarray cstddef
 valarray cstdint
 valarray initializer_list
 valarray limits
 valarray new
 valarray version
 variant compare
-variant cstddef
 variant cstdint
 variant cstring
 variant initializer_list
diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv
index ee17223e66be..944002f4974d 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx26.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv
@@ -1,7 +1,6 @@
 algorithm cctype
 algorithm climits
 algorithm compare
-algorithm cstddef
 algorithm cstdint
 algorithm cstring
 algorithm ctime
@@ -15,7 +14,6 @@ algorithm optional
 algorithm ratio
 algorithm tuple
 algorithm version
-any cstddef
 any cstdint
 any cstring
 any initializer_list
@@ -25,7 +23,6 @@ any typeinfo
 any version
 array cctype
 array compare
-array cstddef
 array cstdint
 array cwchar
 array cwctype
@@ -35,7 +32,6 @@ array new
 array stdexcept
 array version
 atomic climits
-atomic cstddef
 atomic cstdint
 atomic cstring
 atomic ctime
@@ -43,7 +39,6 @@ atomic limits
 atomic ratio
 atomic version
 barrier climits
-barrier cstddef
 barrier cstdint
 barrier cstring
 barrier ctime
@@ -57,7 +52,6 @@ bit version
 bitset cctype
 bitset climits
 bitset compare
-bitset cstddef
 bitset cstdint
 bitset cstdio
 bitset cstring
@@ -105,7 +99,6 @@ ccomplex tuple
 ccomplex typeinfo
 ccomplex version
 charconv cerrno
-charconv cstddef
 charconv cstdint
 charconv initializer_list
 charconv limits
@@ -170,7 +163,6 @@ codecvt string_view
 codecvt tuple
 codecvt typeinfo
 codecvt version
-compare cstddef
 compare cstdint
 compare limits
 compare version
@@ -205,14 +197,12 @@ complex string_view
 complex tuple
 complex typeinfo
 complex version
-concepts cstddef
 concepts version
 condition_variable atomic
 condition_variable cctype
 condition_variable cerrno
 condition_variable climits
 condition_variable compare
-condition_variable cstddef
 condition_variable cstdint
 condition_variable cstdio
 condition_variable cstring
@@ -231,7 +221,6 @@ condition_variable tuple
 condition_variable typeinfo
 condition_variable version
 coroutine compare
-coroutine cstddef
 coroutine cstdint
 coroutine cstring
 coroutine limits
@@ -274,7 +263,6 @@ cwchar cwctype
 cwctype cctype
 deque cctype
 deque compare
-deque cstddef
 deque cstdint
 deque cstring
 deque cwchar
@@ -285,14 +273,12 @@ deque new
 deque stdexcept
 deque tuple
 deque version
-exception cstddef
 exception cstdint
 exception cstdlib
 exception new
 exception typeinfo
 exception version
 execution version
-expected cstddef
 expected cstdint
 expected initializer_list
 expected new
@@ -328,23 +314,18 @@ experimental/iterator tuple
 experimental/iterator typeinfo
 experimental/iterator variant
 experimental/iterator version
-experimental/memory cstddef
 experimental/memory cstdint
 experimental/memory cstring
 experimental/memory version
-experimental/propagate_const cstddef
 experimental/propagate_const version
-experimental/simd cstddef
 experimental/simd cstdint
 experimental/simd limits
 experimental/simd version
-experimental/type_traits cstddef
 experimental/type_traits cstdint
 experimental/type_traits initializer_list
 experimental/type_traits type_traits
 experimental/type_traits version
 experimental/utility compare
-experimental/utility cstddef
 experimental/utility cstdint
 experimental/utility initializer_list
 experimental/utility limits
@@ -432,7 +413,6 @@ format typeinfo
 format version
 forward_list cctype
 forward_list compare
-forward_list cstddef
 forward_list cstdint
 forward_list cwchar
 forward_list cwctype
@@ -480,7 +460,6 @@ fstream version
 functional array
 functional cctype
 functional compare
-functional cstddef
 functional cstdint
 functional cstring
 functional cwchar
@@ -528,7 +507,6 @@ future thread
 future tuple
 future typeinfo
 future version
-initializer_list cstddef
 initializer_list version
 iomanip bitset
 iomanip cctype
@@ -647,11 +625,14 @@ istream string_view
 istream tuple
 istream typeinfo
 istream version
+iterator cctype
 iterator compare
 iterator concepts
-iterator cstddef
 iterator cstdint
+iterator cstdio
 iterator cstring
+iterator cwchar
+iterator cwctype
 iterator initializer_list
 iterator iosfwd
 iterator limits
@@ -659,7 +640,6 @@ iterator new
 iterator variant
 iterator version
 latch climits
-latch cstddef
 latch cstdint
 latch cstring
 latch ctime
@@ -669,7 +649,6 @@ latch version
 limits version
 list cctype
 list compare
-list cstddef
 list cstdint
 list cstring
 list cwchar
@@ -707,7 +686,6 @@ locale typeinfo
 locale version
 map cctype
 map compare
-map cstddef
 map cstdint
 map cstring
 map cwchar
@@ -721,10 +699,8 @@ map tuple
 map version
 mdspan array
 mdspan cctype
-mdspan cinttypes
 mdspan compare
 mdspan concepts
-mdspan cstddef
 mdspan cstdint
 mdspan cwchar
 mdspan cwctype
@@ -735,7 +711,6 @@ mdspan span
 mdspan stdexcept
 mdspan version
 memory compare
-memory cstddef
 memory cstdint
 memory cstring
 memory initializer_list
@@ -748,7 +723,6 @@ memory_resource cctype
 memory_resource cerrno
 memory_resource climits
 memory_resource compare
-memory_resource cstddef
 memory_resource cstdint
 memory_resource cstdio
 memory_resource cstring
@@ -771,7 +745,6 @@ mutex cctype
 mutex cerrno
 mutex climits
 mutex compare
-mutex cstddef
 mutex cstdint
 mutex cstdio
 mutex cstring
@@ -789,12 +762,10 @@ mutex string_view
 mutex tuple
 mutex typeinfo
 mutex version
-new cstddef
 new version
 numbers version
 numeric climits
 numeric compare
-numeric cstddef
 numeric cstdint
 numeric cstring
 numeric ctime
@@ -806,7 +777,6 @@ numeric ratio
 numeric tuple
 numeric version
 optional compare
-optional cstddef
 optional cstdint
 optional cstring
 optional initializer_list
@@ -901,7 +871,6 @@ random cctype
 random climits
 random cmath
 random compare
-random cstddef
 random cstdint
 random cstdio
 random cstring
@@ -923,8 +892,8 @@ random version
 ranges cctype
 ranges compare
 ranges concepts
-ranges cstddef
 ranges cstdint
+ranges cstdio
 ranges cstring
 ranges cwchar
 ranges cwctype
@@ -968,14 +937,12 @@ regex typeinfo
 regex vector
 regex version
 scoped_allocator compare
-scoped_allocator cstddef
 scoped_allocator cstdint
 scoped_allocator limits
 scoped_allocator new
 scoped_allocator tuple
 scoped_allocator version
 semaphore climits
-semaphore cstddef
 semaphore cstdint
 semaphore cstring
 semaphore ctime
@@ -984,7 +951,6 @@ semaphore ratio
 semaphore version
 set cctype
 set compare
-set cstddef
 set cstdint
 set cstring
 set cwchar
@@ -999,7 +965,6 @@ shared_mutex cctype
 shared_mutex cerrno
 shared_mutex climits
 shared_mutex compare
-shared_mutex cstddef
 shared_mutex cstdint
 shared_mutex cstdio
 shared_mutex cstring
@@ -1018,7 +983,6 @@ shared_mutex tuple
 shared_mutex version
 source_location cstdint
 source_location version
-span cstddef
 span initializer_list
 span limits
 span stdexcept
@@ -1054,7 +1018,6 @@ sstream typeinfo
 sstream version
 stack cctype
 stack compare
-stack cstddef
 stack cstdint
 stack cstring
 stack cwchar
@@ -1068,7 +1031,6 @@ stack tuple
 stack version
 stop_token atomic
 stop_token climits
-stop_token cstddef
 stop_token cstdint
 stop_token cstring
 stop_token ctime
@@ -1103,7 +1065,6 @@ streambuf version
 string cctype
 string climits
 string compare
-string cstddef
 string cstdint
 string cstdio
 string cstring
@@ -1119,7 +1080,6 @@ string tuple
 string version
 string_view cctype
 string_view compare
-string_view cstddef
 string_view cstdint
 string_view cstdio
 string_view cstring
@@ -1207,7 +1167,6 @@ system_error cctype
 system_error cerrno
 system_error climits
 system_error compare
-system_error cstddef
 system_error cstdint
 system_error cstdio
 system_error cstring
@@ -1255,23 +1214,19 @@ thread tuple
 thread typeinfo
 thread version
 tuple compare
-tuple cstddef
 tuple cstdint
 tuple limits
 tuple version
 type_traits cstdint
 type_traits version
 typeindex compare
-typeindex cstddef
 typeindex cstdint
 typeindex limits
 typeindex typeinfo
 typeindex version
-typeinfo cstddef
 typeinfo cstdint
 typeinfo version
 unordered_map compare
-unordered_map cstddef
 unordered_map cstdint
 unordered_map cstring
 unordered_map initializer_list
@@ -1282,7 +1237,6 @@ unordered_map stdexcept
 unordered_map tuple
 unordered_map version
 unordered_set compare
-unordered_set cstddef
 unordered_set cstdint
 unordered_set cstring
 unordered_set initializer_list
@@ -1292,20 +1246,17 @@ unordered_set optional
 unordered_set tuple
 unordered_set version
 utility compare
-utility cstddef
 utility cstdint
 utility initializer_list
 utility limits
 utility version
 valarray cmath
-valarray cstddef
 valarray cstdint
 valarray initializer_list
 valarray limits
 valarray new
 valarray version
 variant compare
-variant cstddef
 variant cstdint
 variant cstring
 variant initializer_list
diff --git a/libcxx/test/libcxx/utilities/template.bitset/includes.pass.cpp b/libcxx/test/libcxx/utilities/template.bitset/includes.pass.cpp
index 42deaaa4b2d3..214b0eff8d9d 100644
--- a/libcxx/test/libcxx/utilities/template.bitset/includes.pass.cpp
+++ b/libcxx/test/libcxx/utilities/template.bitset/includes.pass.cpp
@@ -6,16 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-// test that <bitset> includes <cstddef>, <string>, <stdexcept> and <iosfwd>
+// test that <bitset> includes <string>, <stdexcept> and <iosfwd>
 
 #include <bitset>
 
 #include "test_macros.h"
 
-#ifndef _LIBCPP_CSTDDEF
-#error <cstddef> has not been included
-#endif
-
 #ifndef _LIBCPP_STRING
 #error <string> has not been included
 #endif
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/fill.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/fill.pass.cpp
index 481d565961b2..619dc7242a36 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/fill.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/fill.pass.cpp
@@ -16,6 +16,7 @@
 #include <algorithm>
 #include <array>
 #include <cassert>
+#include <cstddef>
 #include <vector>
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.count/count.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.count/count.pass.cpp
index 7654a4b0c7f0..7250c49a7ff9 100644
--- a/libcxx/test/std/algorithms/alg.nonmodifying/alg.count/count.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.count/count.pass.cpp
@@ -18,6 +18,7 @@
 
 #include <algorithm>
 #include <cassert>
+#include <cstddef>
 #include <vector>
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.count/ranges.count.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.count/ranges.count.pass.cpp
index b6631add7e48..6030bed47ec6 100644
--- a/libcxx/test/std/algorithms/alg.nonmodifying/alg.count/ranges.count.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.count/ranges.count.pass.cpp
@@ -25,6 +25,7 @@
 #include <algorithm>
 #include <array>
 #include <cassert>
+#include <cstddef>
 #include <ranges>
 #include <vector>
 
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp
index 2a51127a4591..56af9f234d07 100644
--- a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp
@@ -33,6 +33,7 @@
 #include <algorithm>
 #include <array>
 #include <cassert>
+#include <cstddef>
 #include <functional>
 #include <utility>
 #include <vector>
diff --git a/libcxx/test/std/atomics/atomics.types.generic/address.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/address.pass.cpp
index 0926628a2e9a..dbe1841762f2 100644
--- a/libcxx/test/std/atomics/atomics.types.generic/address.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.generic/address.pass.cpp
@@ -65,9 +65,10 @@
 // };
 
 #include <atomic>
+#include <cassert>
+#include <cstddef>
 #include <new>
 #include <type_traits>
-#include <cassert>
 
 #include <cmpxchg_loop.h>
 
diff --git a/libcxx/test/std/concepts/concepts.callable/concept.invocable/invocable.compile.pass.cpp b/libcxx/test/std/concepts/concepts.callable/concept.invocable/invocable.compile.pass.cpp
index 9b9bac27174e..f9c8f645b284 100644
--- a/libcxx/test/std/concepts/concepts.callable/concept.invocable/invocable.compile.pass.cpp
+++ b/libcxx/test/std/concepts/concepts.callable/concept.invocable/invocable.compile.pass.cpp
@@ -11,8 +11,8 @@
 // template<class T, class U>
 // concept invocable;
 
-#include <chrono>
 #include <concepts>
+#include <cstddef>
 #include <functional>
 #include <memory>
 #include <random>
diff --git a/libcxx/test/std/concepts/concepts.callable/concept.regularinvocable/regular_invocable.compile.pass.cpp b/libcxx/test/std/concepts/concepts.callable/concept.regularinvocable/regular_invocable.compile.pass.cpp
index bfd20751861d..f3547a3ad97c 100644
--- a/libcxx/test/std/concepts/concepts.callable/concept.regularinvocable/regular_invocable.compile.pass.cpp
+++ b/libcxx/test/std/concepts/concepts.callable/concept.regularinvocable/regular_invocable.compile.pass.cpp
@@ -11,11 +11,10 @@
 // template<class T, class U>
 // concept regular_invocable;
 
-#include <chrono>
 #include <concepts>
+#include <cstddef>
 #include <functional>
 #include <memory>
-#include <random>
 #include <type_traits>
 
 template <class R, class... Args>
diff --git a/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable.compile.pass.cpp b/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable.compile.pass.cpp
index 126606ef7ab3..ca0f40eb77d4 100644
--- a/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable.compile.pass.cpp
+++ b/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable.compile.pass.cpp
@@ -14,11 +14,11 @@
 #include <concepts>
 
 #include <array>
+#include <cstddef>
 #include <deque>
 #include <forward_list>
 #include <list>
 #include <map>
-#include <memory>
 #include <optional>
 #include <set>
 #include <unordered_map>
diff --git a/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable_with.compile.pass.cpp b/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable_with.compile.pass.cpp
index e0edd1f332f8..0afbe582ba89 100644
--- a/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable_with.compile.pass.cpp
+++ b/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable_with.compile.pass.cpp
@@ -14,15 +14,12 @@
 #include <concepts>
 
 #include <array>
+#include <cstddef>
 #include <deque>
 #include <forward_list>
 #include <list>
 #include <map>
-#include <memory>
 #include <optional>
-#include <set>
-#include <unordered_map>
-#include <unordered_set>
 #include <vector>
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered.compile.pass.cpp b/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered.compile.pass.cpp
index d95de10f35cd..6f8324eaf764 100644
--- a/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered.compile.pass.cpp
+++ b/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered.compile.pass.cpp
@@ -14,11 +14,10 @@
 #include <concepts>
 
 #include <array>
+#include <cstddef>
 #include <deque>
 #include <forward_list>
 #include <list>
-#include <map>
-#include <memory>
 #include <optional>
 #include <set>
 #include <unordered_map>
diff --git a/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered_with.compile.pass.cpp b/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered_with.compile.pass.cpp
index 0d7bd288c0a1..dffc33265aeb 100644
--- a/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered_with.compile.pass.cpp
+++ b/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered_with.compile.pass.cpp
@@ -14,15 +14,12 @@
 #include <concepts>
 
 #include <array>
+#include <cstddef>
 #include <deque>
 #include <forward_list>
 #include <list>
 #include <map>
-#include <memory>
 #include <optional>
-#include <set>
-#include <unordered_map>
-#include <unordered_set>
 #include <vector>
 
 #include "compare_types.h"
diff --git a/libcxx/test/std/concepts/concepts.lang/concept.constructible/constructible_from.compile.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concept.constructible/constructible_from.compile.pass.cpp
index fe0ecece3382..e3b58d622e45 100644
--- a/libcxx/test/std/concepts/concepts.lang/concept.constructible/constructible_from.compile.pass.cpp
+++ b/libcxx/test/std/concepts/concepts.lang/concept.constructible/constructible_from.compile.pass.cpp
@@ -14,6 +14,7 @@
 
 #include <array>
 #include <concepts>
+#include <cstddef>
 #include <memory>
 #include <string>
 #include <type_traits>
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/destroy_elements.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/destroy_elements.pass.cpp
index 6cbf1441b464..a245131c7869 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/destroy_elements.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/destroy_elements.pass.cpp
@@ -13,6 +13,7 @@
 #include <algorithm>
 #include <array>
 #include <cassert>
+#include <cstddef>
 #include <vector>
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/containers/views/mdspan/MinimalElementType.h b/libcxx/test/std/containers/views/mdspan/MinimalElementType.h
index fe7f0e1f2383..1d1a2c375210 100644
--- a/libcxx/test/std/containers/views/mdspan/MinimalElementType.h
+++ b/libcxx/test/std/containers/views/mdspan/MinimalElementType.h
@@ -9,6 +9,7 @@
 #ifndef TEST_STD_CONTAINERS_VIEWS_MDSPAN_MINIMAL_ELEMENT_TYPE_H
 #define TEST_STD_CONTAINERS_VIEWS_MDSPAN_MINIMAL_ELEMENT_TYPE_H
 
+#include <cstddef>
 #include <memory>
 #include <type_traits>
 
diff --git a/libcxx/test/std/containers/views/mdspan/extents/CtorTestCombinations.h b/libcxx/test/std/containers/views/mdspan/extents/CtorTestCombinations.h
index 18d4f4b61fb2..36f95704631f 100644
--- a/libcxx/test/std/containers/views/mdspan/extents/CtorTestCombinations.h
+++ b/libcxx/test/std/containers/views/mdspan/extents/CtorTestCombinations.h
@@ -9,9 +9,10 @@
 
 // <mdspan>
 
-#include <mdspan>
-#include <cassert>
 #include <array>
+#include <cassert>
+#include <cstddef>
+#include <mdspan>
 #include <span>
 
 #include "../ConvertibleToIntegral.h"
diff --git a/libcxx/test/std/containers/views/mdspan/extents/comparison.pass.cpp b/libcxx/test/std/containers/views/mdspan/extents/comparison.pass.cpp
index 574290ebec85..1d713044e60f 100644
--- a/libcxx/test/std/containers/views/mdspan/extents/comparison.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/extents/comparison.pass.cpp
@@ -17,11 +17,10 @@
 // if lhs.extent(r) equals rhs.extent(r) for every rank index r of rhs, otherwise false.
 //
 
-#include <mdspan>
 #include <cassert>
-#include <concepts>
+#include <cstddef>
+#include <mdspan>
 #include <span> // dynamic_extent
-#include <type_traits>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/containers/views/mdspan/extents/conversion.pass.cpp b/libcxx/test/std/containers/views/mdspan/extents/conversion.pass.cpp
index f6834b0b4133..7baaa7ec9898 100644
--- a/libcxx/test/std/containers/views/mdspan/extents/conversion.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/extents/conversion.pass.cpp
@@ -28,15 +28,13 @@
 //          (((Extents != dynamic_extent) && (OtherExtents == dynamic_extent)) || ... ) ||
 //          (numeric_limits<index_type>::max() < numeric_limits<OtherIndexType>::max())
 
-#include <mdspan>
 #include <cassert>
-#include <concepts>
+#include <cstddef>
 #include <limits>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 
-#include "test_macros.h"
-
 template <class To, class From>
 constexpr void test_implicit_conversion(To dest, From src) {
   assert(dest == src);
diff --git a/libcxx/test/std/containers/views/mdspan/extents/ctad.pass.cpp b/libcxx/test/std/containers/views/mdspan/extents/ctad.pass.cpp
index 1a6501b39139..82f111153ce6 100644
--- a/libcxx/test/std/containers/views/mdspan/extents/ctad.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/extents/ctad.pass.cpp
@@ -16,8 +16,9 @@
 // Remarks: The deduced type is dextents<size_t, sizeof...(Integrals)>.           // until C++26
 // Remarks: The deduced type is extents<size_t, maybe-static-ext<Integrals>...>.  // since C++26
 
-#include <mdspan>
 #include <cassert>
+#include <cstddef>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 
diff --git a/libcxx/test/std/containers/views/mdspan/extents/obs_static.pass.cpp b/libcxx/test/std/containers/views/mdspan/extents/obs_static.pass.cpp
index 29dd9e2d2707..1d5f61d02e4f 100644
--- a/libcxx/test/std/containers/views/mdspan/extents/obs_static.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/extents/obs_static.pass.cpp
@@ -26,8 +26,9 @@
 //   Returns: Di.
 //
 
-#include <mdspan>
 #include <cassert>
+#include <cstddef>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <utility>
 
diff --git a/libcxx/test/std/containers/views/mdspan/extents/types.pass.cpp b/libcxx/test/std/containers/views/mdspan/extents/types.pass.cpp
index 2924da91f77e..10bc76947395 100644
--- a/libcxx/test/std/containers/views/mdspan/extents/types.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/extents/types.pass.cpp
@@ -22,9 +22,10 @@
 //  ...
 //  }
 
-#include <mdspan>
 #include <cassert>
 #include <concepts>
+#include <cstddef>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 
diff --git a/libcxx/test/std/containers/views/mdspan/layout_left/comparison.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_left/comparison.pass.cpp
index c8b4083291a6..151da5ba6174 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_left/comparison.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_left/comparison.pass.cpp
@@ -15,11 +15,10 @@
 //                                      `
 // Constraints: extents_type::rank() == OtherExtents::rank() is true.
 
-#include <mdspan>
 #include <cassert>
-#include <concepts>
+#include <cstddef>
+#include <mdspan>
 #include <span> // dynamic_extent
-#include <type_traits>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/containers/views/mdspan/layout_left/ctor.default.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_left/ctor.default.pass.cpp
index 5a4040317d24..12c59a4caf10 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_left/ctor.default.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_left/ctor.default.pass.cpp
@@ -14,9 +14,10 @@
 //
 // constexpr mapping() noexcept = default;
 
-#include <mdspan>
 #include <cassert>
+#include <cstddef>
 #include <cstdint>
+#include <mdspan>
 #include <span> // dynamic_extent
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/containers/views/mdspan/layout_left/ctor.extents.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_left/ctor.extents.pass.cpp
index 46505cb961bb..299012dc5af8 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_left/ctor.extents.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_left/ctor.extents.pass.cpp
@@ -17,9 +17,10 @@
 //
 // Effects: Direct-non-list-initializes extents_ with e.
 
-#include <mdspan>
 #include <cassert>
+#include <cstddef>
 #include <cstdint>
+#include <mdspan>
 #include <span> // dynamic_extent
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_right.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_right.pass.cpp
index 5f9bd4344d0e..1e8f8fb54d5e 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_right.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_right.pass.cpp
@@ -20,9 +20,10 @@
 //
 // Preconditions: other.required_span_size() is representable as a value of type index_type
 
-#include <mdspan>
 #include <cassert>
+#include <cstddef>
 #include <limits>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 
diff --git a/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_stride.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_stride.pass.cpp
index 34489b7c52d7..1668c26a697d 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_stride.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_stride.pass.cpp
@@ -23,10 +23,10 @@
 //
 // Effects: Direct-non-list-initializes extents_ with other.extents().
 
-#include <mdspan>
 #include <array>
 #include <cassert>
-#include <limits>
+#include <cstddef>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 
diff --git a/libcxx/test/std/containers/views/mdspan/layout_left/ctor.mapping.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_left/ctor.mapping.pass.cpp
index 63b3c50c7317..737e5f3d2572 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_left/ctor.mapping.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_left/ctor.mapping.pass.cpp
@@ -18,9 +18,10 @@
 //
 // Preconditions: other.required_span_size() is representable as a value of type index_type
 
-#include <mdspan>
 #include <cassert>
+#include <cstddef>
 #include <limits>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 
diff --git a/libcxx/test/std/containers/views/mdspan/layout_left/index_operator.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_left/index_operator.pass.cpp
index 40cd6bc2812e..84c3ef45c69b 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_left/index_operator.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_left/index_operator.pass.cpp
@@ -23,11 +23,12 @@
 // Preconditions:
 //   * extents_type::index-cast(i) is a multidimensional index in extents_.
 
-#include <mdspan>
-#include <type_traits>
 #include <cassert>
+#include <cstddef>
 #include <cstdint>
+#include <mdspan>
 #include <span> // dynamic_extent
+#include <type_traits>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/containers/views/mdspan/layout_left/properties.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_left/properties.pass.cpp
index 19f523824cfc..32442ecd5a0e 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_left/properties.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_left/properties.pass.cpp
@@ -26,11 +26,10 @@
 //   };
 // }
 
-#include <mdspan>
 #include <cassert>
-#include <concepts>
+#include <cstddef>
+#include <mdspan>
 #include <span> // dynamic_extent
-#include <type_traits>
 #include <utility>
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/containers/views/mdspan/layout_left/required_span_size.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_left/required_span_size.pass.cpp
index 4cb111d29827..9ad61b0799c1 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_left/required_span_size.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_left/required_span_size.pass.cpp
@@ -14,9 +14,10 @@
 //
 // Returns: extents().fwd-prod-of-extents(extents_type::rank()).
 
-#include <mdspan>
 #include <cassert>
+#include <cstddef>
 #include <cstdint>
+#include <mdspan>
 #include <span> // dynamic_extent
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/containers/views/mdspan/layout_left/static_requirements.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_left/static_requirements.pass.cpp
index 7a6add60efcd..6410fecdab59 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_left/static_requirements.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_left/static_requirements.pass.cpp
@@ -76,9 +76,9 @@
 //    Result: A constant expression ([expr.const]) of type bool.
 //    Returns: true only if m.is_strided() is true for all possible objects m of type M.
 
-#include <mdspan>
 #include <cassert>
-#include <concepts>
+#include <cstddef>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 #include <utility>
diff --git a/libcxx/test/std/containers/views/mdspan/layout_right/comparison.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_right/comparison.pass.cpp
index 03c78ca5e91d..a65d7d39db8e 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_right/comparison.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_right/comparison.pass.cpp
@@ -15,11 +15,10 @@
 //                                      `
 // Constraints: extents_type::rank() == OtherExtents::rank() is true.
 
-#include <mdspan>
 #include <cassert>
-#include <concepts>
+#include <cstddef>
+#include <mdspan>
 #include <span> // dynamic_extent
-#include <type_traits>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/containers/views/mdspan/layout_right/ctor.default.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_right/ctor.default.pass.cpp
index f02174416f33..d644b0ff18d8 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_right/ctor.default.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_right/ctor.default.pass.cpp
@@ -14,9 +14,10 @@
 //
 // constexpr mapping() noexcept = default;
 
-#include <mdspan>
 #include <cassert>
+#include <cstddef>
 #include <cstdint>
+#include <mdspan>
 #include <span> // dynamic_extent
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/containers/views/mdspan/layout_right/ctor.extents.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_right/ctor.extents.pass.cpp
index 9c2c39bc3cb3..cd0cff838fac 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_right/ctor.extents.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_right/ctor.extents.pass.cpp
@@ -17,9 +17,10 @@
 //
 // Effects: Direct-non-list-initializes extents_ with e.
 
-#include <mdspan>
 #include <cassert>
+#include <cstddef>
 #include <cstdint>
+#include <mdspan>
 #include <span> // dynamic_extent
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_left.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_left.pass.cpp
index 61aba5dae682..994d98a80321 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_left.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_left.pass.cpp
@@ -20,9 +20,10 @@
 //
 // Preconditions: other.required_span_size() is representable as a value of type index_type
 
-#include <mdspan>
 #include <cassert>
+#include <cstddef>
 #include <limits>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 
diff --git a/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_stride.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_stride.pass.cpp
index 3bc7d82f8ed8..89321f860dc3 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_stride.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_stride.pass.cpp
@@ -23,10 +23,10 @@
 //
 // Effects: Direct-non-list-initializes extents_ with other.extents().
 
-#include <mdspan>
 #include <array>
 #include <cassert>
-#include <limits>
+#include <cstddef>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 
diff --git a/libcxx/test/std/containers/views/mdspan/layout_right/ctor.mapping.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_right/ctor.mapping.pass.cpp
index eeea5ab021e9..ee00c688301e 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_right/ctor.mapping.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_right/ctor.mapping.pass.cpp
@@ -18,9 +18,10 @@
 //
 // Preconditions: other.required_span_size() is representable as a value of type index_type
 
-#include <mdspan>
 #include <cassert>
+#include <cstddef>
 #include <limits>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 
diff --git a/libcxx/test/std/containers/views/mdspan/layout_right/index_operator.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_right/index_operator.pass.cpp
index 989078f17d30..42d4e9a2d24b 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_right/index_operator.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_right/index_operator.pass.cpp
@@ -23,9 +23,10 @@
 // Preconditions:
 //   * extents_type::index-cast(i) is a multidimensional index in extents_.
 
-#include <mdspan>
 #include <cassert>
+#include <cstddef>
 #include <cstdint>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 
diff --git a/libcxx/test/std/containers/views/mdspan/layout_right/properties.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_right/properties.pass.cpp
index 674a7ac98cbf..857ec3cb0583 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_right/properties.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_right/properties.pass.cpp
@@ -26,11 +26,10 @@
 //   };
 // }
 
-#include <mdspan>
 #include <cassert>
-#include <concepts>
+#include <cstddef>
+#include <mdspan>
 #include <span> // dynamic_extent
-#include <type_traits>
 #include <utility>
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/containers/views/mdspan/layout_right/required_span_size.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_right/required_span_size.pass.cpp
index 0128d8c26a83..2ffd1f41f963 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_right/required_span_size.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_right/required_span_size.pass.cpp
@@ -14,10 +14,10 @@
 //
 // Returns: extents().fwd-prod-of-extents(extents_type::rank()).
 
-
-#include <mdspan>
 #include <cassert>
+#include <cstddef>
 #include <cstdint>
+#include <mdspan>
 #include <span> // dynamic_extent
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/containers/views/mdspan/layout_right/static_requirements.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_right/static_requirements.pass.cpp
index 2b11d17c6717..b7e01d14532d 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_right/static_requirements.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_right/static_requirements.pass.cpp
@@ -76,9 +76,9 @@
 //    Result: A constant expression ([expr.const]) of type bool.
 //    Returns: true only if m.is_strided() is true for all possible objects m of type M.
 
-#include <mdspan>
 #include <cassert>
-#include <concepts>
+#include <cstddef>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 #include <utility>
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.default.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.default.pass.cpp
index 108c4c6fca98..055986d80c72 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.default.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.default.pass.cpp
@@ -20,9 +20,10 @@
 // Effects: Direct-non-list-initializes extents_ with extents_type(), and for all d in the range [0, rank_),
 //          direct-non-list-initializes strides_[d] with layout_right::mapping<extents_type>().stride(d).
 
-#include <mdspan>
 #include <cassert>
+#include <cstddef>
 #include <cstdint>
+#include <mdspan>
 #include <span> // dynamic_extent
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_array.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_array.pass.cpp
index cecfb79ea686..bbda6d4a7363 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_array.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_array.pass.cpp
@@ -27,10 +27,11 @@
 // Effects: Direct-non-list-initializes extents_ with e, and for all d in the range [0, rank_),
 //         direct-non-list-initializes strides_[d] with as_const(s[d]).
 
-#include <mdspan>
 #include <array>
 #include <cassert>
+#include <cstddef>
 #include <cstdint>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_span.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_span.pass.cpp
index d0f26ad23df9..f5db6768f2c0 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_span.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_span.pass.cpp
@@ -27,10 +27,11 @@
 // Effects: Direct-non-list-initializes extents_ with e, and for all d in the range [0, rank_),
 //         direct-non-list-initializes strides_[d] with as_const(s[d]).
 
-#include <mdspan>
 #include <array>
 #include <cassert>
+#include <cstddef>
 #include <cstdint>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/deduction.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/deduction.pass.cpp
index ca88a9f8e044..233eebff0ccd 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_stride/deduction.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/deduction.pass.cpp
@@ -10,13 +10,12 @@
 
 // <mdspan>
 
-#include <mdspan>
 #include <array>
 #include <cassert>
-#include <concepts>
+#include <cstddef>
 #include <cstdint>
+#include <mdspan>
 #include <span> // dynamic_extent
-#include <type_traits>
 #include <utility>
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/is_exhaustive_corner_case.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/is_exhaustive_corner_case.pass.cpp
index 589e32f86e39..9f51cc01cf9d 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_stride/is_exhaustive_corner_case.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/is_exhaustive_corner_case.pass.cpp
@@ -19,14 +19,11 @@
 //     range [1, rank_), where pi is the ith element of P.
 //   - Otherwise, false.
 
-#include <mdspan>
 #include <array>
 #include <cassert>
-#include <concepts>
+#include <cstddef>
+#include <mdspan>
 #include <span> // dynamic_extent
-#include <type_traits>
-
-#include "test_macros.h"
 
 template <class E>
 constexpr void
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/properties.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/properties.pass.cpp
index b1eb84b375b6..eac1029882e3 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_stride/properties.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/properties.pass.cpp
@@ -38,10 +38,11 @@
 //     range [1, rank_), where pi is the ith element of P.
 //   - Otherwise, false.
 
-#include <mdspan>
 #include <array>
 #include <cassert>
 #include <concepts>
+#include <cstddef>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/required_span_size.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/required_span_size.pass.cpp
index 870518994a93..629849646bb2 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_stride/required_span_size.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/required_span_size.pass.cpp
@@ -19,10 +19,11 @@
 //
 //   Returns: REQUIRED-SPAN-SIZE(extents(), strides_).
 
-#include <mdspan>
 #include <array>
 #include <cassert>
+#include <cstddef>
 #include <cstdint>
+#include <mdspan>
 #include <span> // dynamic_extent
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/static_requirements.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/static_requirements.pass.cpp
index a69fb4f287c3..8131ecde5d76 100644
--- a/libcxx/test/std/containers/views/mdspan/layout_stride/static_requirements.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/static_requirements.pass.cpp
@@ -76,9 +76,9 @@
 //    Result: A constant expression ([expr.const]) of type bool.
 //    Returns: true only if m.is_strided() is true for all possible objects m of type M.
 
-#include <mdspan>
 #include <cassert>
-#include <concepts>
+#include <cstddef>
+#include <mdspan>
 #include <span> // dynamic_extent
 #include <type_traits>
 #include <utility>
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/CustomTestAccessors.h b/libcxx/test/std/containers/views/mdspan/mdspan/CustomTestAccessors.h
index b68268d172a1..0795926cb43f 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/CustomTestAccessors.h
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/CustomTestAccessors.h
@@ -17,9 +17,10 @@
 #ifndef TEST_STD_CONTAINERS_VIEWS_MDSPAN_MDSPAN_CUSTOM_TEST_ACCESSORS_H
 #define TEST_STD_CONTAINERS_VIEWS_MDSPAN_MDSPAN_CUSTOM_TEST_ACCESSORS_H
 
+#include <cassert>
+#include <cstddef>
 #include <mdspan>
 #include <type_traits>
-#include <cassert>
 
 // This contains a bunch of accessors and handles which have different properties
 // regarding constructibility and convertibility in order to test mdspan constraints
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/iterator_len.pass.cpp b/libcxx/test/std/containers/views/views.span/span.cons/iterator_len.pass.cpp
index fbbd3d6ff404..fcc2625ee659 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/iterator_len.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/iterator_len.pass.cpp
@@ -14,10 +14,10 @@
 //  If Extent is not equal to dynamic_extent, then count shall be equal to Extent.
 //
 
-
-#include <span>
 #include <cassert>
+#include <cstddef>
 #include <iterator>
+#include <span>
 #include <type_traits>
 
 template <std::size_t Extent>
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/iterator_sentinel.verify.cpp b/libcxx/test/std/containers/views/views.span/span.cons/iterator_sentinel.verify.cpp
index a31aa2af7b9d..937d8d921ea0 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/iterator_sentinel.verify.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/iterator_sentinel.verify.cpp
@@ -15,8 +15,9 @@
 //   If Extent is not equal to dynamic_extent, then last - first shall be equal to Extent.
 //
 
-#include <span>
+#include <cstddef>
 #include <iterator>
+#include <span>
 
 template<class T, std::size_t Extent>
 std::span<T, Extent> createImplicitSpan(T* first, T* last) {
diff --git a/libcxx/test/std/containers/views/views.span/span.objectrep/as_bytes.pass.cpp b/libcxx/test/std/containers/views/views.span/span.objectrep/as_bytes.pass.cpp
index 1f58d0f969f7..44b658fb8375 100644
--- a/libcxx/test/std/containers/views/views.span/span.objectrep/as_bytes.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.objectrep/as_bytes.pass.cpp
@@ -16,9 +16,9 @@
 //              : sizeof(ElementType) * Extent>
 //     as_bytes(span<ElementType, Extent> s) noexcept;
 
-
-#include <span>
 #include <cassert>
+#include <cstddef>
+#include <span>
 #include <string>
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/containers/views/views.span/span.objectrep/as_writable_bytes.pass.cpp b/libcxx/test/std/containers/views/views.span/span.objectrep/as_writable_bytes.pass.cpp
index 6b7bd5dcf0c1..d38d69d9fee1 100644
--- a/libcxx/test/std/containers/views/views.span/span.objectrep/as_writable_bytes.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.objectrep/as_writable_bytes.pass.cpp
@@ -16,9 +16,9 @@
 //              : sizeof(ElementType) * Extent>
 //     as_writable_bytes(span<ElementType, Extent> s) noexcept;
 
-
-#include <span>
 #include <cassert>
+#include <cstddef>
+#include <span>
 #include <string>
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/diagnostics/syserr/is_error_code_enum.pass.cpp b/libcxx/test/std/diagnostics/syserr/is_error_code_enum.pass.cpp
index 3f614efee203..437d0f0a9117 100644
--- a/libcxx/test/std/diagnostics/syserr/is_error_code_enum.pass.cpp
+++ b/libcxx/test/std/diagnostics/syserr/is_error_code_enum.pass.cpp
@@ -12,8 +12,10 @@
 
 // template <> struct is_error_code_enum<> : public false_type {};
 
-#include <system_error>
+#include <cstddef>
 #include <string>
+#include <system_error>
+
 #include "test_macros.h"
 
 template <bool Expected, class T>
diff --git a/libcxx/test/std/diagnostics/syserr/is_error_condition_enum.pass.cpp b/libcxx/test/std/diagnostics/syserr/is_error_condition_enum.pass.cpp
index e9916f2427a5..f4d5057948ac 100644
--- a/libcxx/test/std/diagnostics/syserr/is_error_condition_enum.pass.cpp
+++ b/libcxx/test/std/diagnostics/syserr/is_error_condition_enum.pass.cpp
@@ -12,9 +12,11 @@
 
 // template <class T> constexpr bool is_error_condition_enum_v;
 
+#include <cstddef>
 #include <string>
 #include <system_error>
 #include <type_traits>
+
 #include "test_macros.h"
 
 template <bool Expected, class T>
diff --git a/libcxx/test/std/experimental/simd/simd.class/simd_ctor_broadcast.pass.cpp b/libcxx/test/std/experimental/simd/simd.class/simd_ctor_broadcast.pass.cpp
index 8a291632a8ab..fbdaa438f788 100644
--- a/libcxx/test/std/experimental/simd/simd.class/simd_ctor_broadcast.pass.cpp
+++ b/libcxx/test/std/experimental/simd/simd.class/simd_ctor_broadcast.pass.cpp
@@ -14,6 +14,9 @@
 // [simd.class]
 // template<class U> simd(U&& value) noexcept;
 
+#include <algorithm>
+#include <experimental/simd>
+
 #include "../test_utils.h"
 
 namespace ex = std::experimental::parallelism_v2;
diff --git a/libcxx/test/std/experimental/simd/test_utils.h b/libcxx/test/std/experimental/simd/test_utils.h
index 3c227a43c2f4..4c7b459cc8eb 100644
--- a/libcxx/test/std/experimental/simd/test_utils.h
+++ b/libcxx/test/std/experimental/simd/test_utils.h
@@ -9,12 +9,13 @@
 #ifndef LIBCXX_TEST_STD_EXPERIMENTAL_SIMD_TEST_UTILS_H
 #define LIBCXX_TEST_STD_EXPERIMENTAL_SIMD_TEST_UTILS_H
 
-#include <algorithm>
 #include <array>
 #include <cassert>
+#include <cstddef>
+#include <experimental/simd>
 #include <type_traits>
 #include <utility>
-#include <experimental/simd>
+
 #include "type_algorithms.h"
 
 namespace ex = std::experimental::parallelism_v2;
diff --git a/libcxx/test/std/iterators/iterator.requirements/iterator.assoc.types/incrementable.traits/incrementable_traits.compile.pass.cpp b/libcxx/test/std/iterators/iterator.requirements/iterator.assoc.types/incrementable.traits/incrementable_traits.compile.pass.cpp
index 8413f912e576..6d07e973ec30 100644
--- a/libcxx/test/std/iterators/iterator.requirements/iterator.assoc.types/incrementable.traits/incrementable_traits.compile.pass.cpp
+++ b/libcxx/test/std/iterators/iterator.requirements/iterator.assoc.types/incrementable.traits/incrementable_traits.compile.pass.cpp
@@ -11,12 +11,9 @@
 // template<class T>
 // struct incrementable_traits;
 
-#include <iterator>
-
 #include <concepts>
 #include <cstddef>
-
-#include "test_macros.h"
+#include <iterator>
 
 template <class T>
 concept check_has_difference_type = requires {
diff --git a/libcxx/test/std/iterators/iterator.requirements/iterator.assoc.types/incrementable.traits/iter_difference_t.compile.pass.cpp b/libcxx/test/std/iterators/iterator.requirements/iterator.assoc.types/incrementable.traits/iter_difference_t.compile.pass.cpp
index bd6664fe957a..99512f7c3ba1 100644
--- a/libcxx/test/std/iterators/iterator.requirements/iterator.assoc.types/incrementable.traits/iter_difference_t.compile.pass.cpp
+++ b/libcxx/test/std/iterators/iterator.requirements/iterator.assoc.types/incrementable.traits/iter_difference_t.compile.pass.cpp
@@ -14,6 +14,7 @@
 #include <iterator>
 
 #include <concepts>
+#include <cstddef>
 #include <vector>
 
 template <class T>
diff --git a/libcxx/test/std/iterators/iterator.requirements/iterator.assoc.types/readable.traits/indirectly_readable_traits.compile.pass.cpp b/libcxx/test/std/iterators/iterator.requirements/iterator.assoc.types/readable.traits/indirectly_readable_traits.compile.pass.cpp
index 835aa9c7be27..4fd935e460ca 100644
--- a/libcxx/test/std/iterators/iterator.requirements/iterator.assoc.types/readable.traits/indirectly_readable_traits.compile.pass.cpp
+++ b/libcxx/test/std/iterators/iterator.requirements/iterator.assoc.types/readable.traits/indirectly_readable_traits.compile.pass.cpp
@@ -14,9 +14,9 @@
 #include <iterator>
 
 #include <concepts>
+#include <cstddef>
 #include <memory>
 #include <optional>
-#include <string>
 #include <vector>
 
 template <class T>
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iterator/types.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iterator/types.pass.cpp
index f8d3e2b4fdc7..68edbb1cb91d 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iterator/types.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iterator/types.pass.cpp
@@ -27,6 +27,7 @@
 //   typedef void                        pointer;
 // };
 
+#include <cstddef>
 #include <iterator>
 #include <type_traits>
 #include <vector>
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iterator/types.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iterator/types.pass.cpp
index f71ba368ab86..c74f9704d0c3 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iterator/types.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iterator/types.pass.cpp
@@ -28,6 +28,7 @@
 //   typedef output_iterator_tag         iterator_category;
 // };
 
+#include <cstddef>
 #include <iterator>
 #include <type_traits>
 #include <vector>
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iterator/types.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iterator/types.pass.cpp
index 08864868342a..faf5ca5d6183 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iterator/types.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iterator/types.pass.cpp
@@ -28,9 +28,11 @@
 //   typedef void                   pointer;
 // };
 
+#include <cstddef>
 #include <iterator>
 #include <type_traits>
 #include <vector>
+
 #include "test_macros.h"
 
 template <class C>
diff --git a/libcxx/test/std/numerics/bit/byteswap.pass.cpp b/libcxx/test/std/numerics/bit/byteswap.pass.cpp
index b87faf150177..9d4e328ed9d0 100644
--- a/libcxx/test/std/numerics/bit/byteswap.pass.cpp
+++ b/libcxx/test/std/numerics/bit/byteswap.pass.cpp
@@ -10,6 +10,7 @@
 
 #include <bit>
 #include <cassert>
+#include <cstddef>
 #include <cstdint>
 #include <utility>
 
diff --git a/libcxx/test/std/numerics/numeric.ops/exclusive.scan/exclusive_scan.pass.cpp b/libcxx/test/std/numerics/numeric.ops/exclusive.scan/exclusive_scan.pass.cpp
index bef2dd786696..f6cac33e9443 100644
--- a/libcxx/test/std/numerics/numeric.ops/exclusive.scan/exclusive_scan.pass.cpp
+++ b/libcxx/test/std/numerics/numeric.ops/exclusive.scan/exclusive_scan.pass.cpp
@@ -16,12 +16,11 @@
 //                                   OutputIterator result, T init);
 //
 
-#include <numeric>
 #include <algorithm>
 #include <array>
 #include <cassert>
-#include <functional>
-#include <iterator>
+#include <cstddef>
+#include <numeric>
 
 #include "test_macros.h"
 #include "test_iterators.h"
diff --git a/libcxx/test/std/numerics/numeric.ops/inclusive.scan/inclusive_scan.pass.cpp b/libcxx/test/std/numerics/numeric.ops/inclusive.scan/inclusive_scan.pass.cpp
index 299d085d01a0..1c79eebb8ee2 100644
--- a/libcxx/test/std/numerics/numeric.ops/inclusive.scan/inclusive_scan.pass.cpp
+++ b/libcxx/test/std/numerics/numeric.ops/inclusive.scan/inclusive_scan.pass.cpp
@@ -16,12 +16,11 @@
 //                                   OutputIterator result, T init);
 //
 
-#include <numeric>
 #include <algorithm>
 #include <array>
 #include <cassert>
-#include <functional>
-#include <iterator>
+#include <cstddef>
+#include <numeric>
 
 #include "test_macros.h"
 #include "test_iterators.h"
diff --git a/libcxx/test/std/numerics/numeric.ops/inclusive.scan/inclusive_scan_op.pass.cpp b/libcxx/test/std/numerics/numeric.ops/inclusive.scan/inclusive_scan_op.pass.cpp
index 87bea923eed3..fb39b4f06ece 100644
--- a/libcxx/test/std/numerics/numeric.ops/inclusive.scan/inclusive_scan_op.pass.cpp
+++ b/libcxx/test/std/numerics/numeric.ops/inclusive.scan/inclusive_scan_op.pass.cpp
@@ -17,12 +17,12 @@
 //                    OutputIterator result,
 //                    BinaryOperation binary_op); // C++17
 
-#include <numeric>
 #include <algorithm>
 #include <array>
 #include <cassert>
+#include <cstddef>
 #include <functional>
-#include <iterator>
+#include <numeric>
 
 #include "test_macros.h"
 #include "test_iterators.h"
diff --git a/libcxx/test/std/numerics/numeric.ops/inclusive.scan/inclusive_scan_op_init.pass.cpp b/libcxx/test/std/numerics/numeric.ops/inclusive.scan/inclusive_scan_op_init.pass.cpp
index 2e21d38f4734..4e07306c2931 100644
--- a/libcxx/test/std/numerics/numeric.ops/inclusive.scan/inclusive_scan_op_init.pass.cpp
+++ b/libcxx/test/std/numerics/numeric.ops/inclusive.scan/inclusive_scan_op_init.pass.cpp
@@ -17,12 +17,12 @@
 //                    OutputIterator result,
 //                    BinaryOperation binary_op, T init); // C++17
 
-#include <numeric>
 #include <algorithm>
 #include <array>
 #include <cassert>
+#include <cstddef>
 #include <functional>
-#include <iterator>
+#include <numeric>
 
 #include "test_macros.h"
 #include "test_iterators.h"
diff --git a/libcxx/test/std/numerics/numeric.ops/transform.exclusive.scan/transform_exclusive_scan_init_bop_uop.pass.cpp b/libcxx/test/std/numerics/numeric.ops/transform.exclusive.scan/transform_exclusive_scan_init_bop_uop.pass.cpp
index 52272205307a..7e1017d4e304 100644
--- a/libcxx/test/std/numerics/numeric.ops/transform.exclusive.scan/transform_exclusive_scan_init_bop_uop.pass.cpp
+++ b/libcxx/test/std/numerics/numeric.ops/transform.exclusive.scan/transform_exclusive_scan_init_bop_uop.pass.cpp
@@ -18,13 +18,12 @@
 //                                           BinaryOperation binary_op,
 //                                           UnaryOperation unary_op);
 
-
-#include <numeric>
 #include <algorithm>
 #include <array>
 #include <cassert>
+#include <cstddef>
 #include <functional>
-#include <iterator>
+#include <numeric>
 
 #include "test_macros.h"
 #include "test_iterators.h"
diff --git a/libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/transform_inclusive_scan_bop_uop.pass.cpp b/libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/transform_inclusive_scan_bop_uop.pass.cpp
index 80ead01e9a79..1dd7661bb42e 100644
--- a/libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/transform_inclusive_scan_bop_uop.pass.cpp
+++ b/libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/transform_inclusive_scan_bop_uop.pass.cpp
@@ -18,13 +18,12 @@
 //                                           BinaryOperation binary_op,
 //                                           UnaryOperation unary_op);
 
-
-#include <numeric>
 #include <algorithm>
 #include <array>
 #include <cassert>
+#include <cstddef>
 #include <functional>
-#include <iterator>
+#include <numeric>
 
 #include "test_macros.h"
 #include "test_iterators.h"
diff --git a/libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/transform_inclusive_scan_bop_uop_init.pass.cpp b/libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/transform_inclusive_scan_bop_uop_init.pass.cpp
index 18be676c7a54..1269c3f68236 100644
--- a/libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/transform_inclusive_scan_bop_uop_init.pass.cpp
+++ b/libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/transform_inclusive_scan_bop_uop_init.pass.cpp
@@ -19,13 +19,12 @@
 //                                           UnaryOperation unary_op,
 //                                           T init);
 
-
-#include <numeric>
 #include <algorithm>
 #include <array>
 #include <cassert>
+#include <cstddef>
 #include <functional>
-#include <iterator>
+#include <numeric>
 
 #include "test_macros.h"
 #include "test_iterators.h"
diff --git a/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.plinear/eval.pass.cpp b/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.plinear/eval.pass.cpp
index ec7f72edf9ee..c2d280406852 100644
--- a/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.plinear/eval.pass.cpp
+++ b/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.plinear/eval.pass.cpp
@@ -15,13 +15,12 @@
 
 // template<class _URNG> result_type operator()(_URNG& g);
 
-#include <random>
 #include <algorithm>
 #include <cassert>
 #include <cmath>
-#include <iterator>
+#include <cstddef>
 #include <limits>
-#include <numeric>
+#include <random>
 #include <vector>
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.plinear/eval_param.pass.cpp b/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.plinear/eval_param.pass.cpp
index 9bcb2ed3afac..e31b4c5837fe 100644
--- a/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.plinear/eval_param.pass.cpp
+++ b/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.plinear/eval_param.pass.cpp
@@ -15,13 +15,12 @@
 
 // template<class _URNG> result_type operator()(_URNG& g, const param_type& parm);
 
-#include <random>
 #include <algorithm>   // for sort
 #include <cassert>
 #include <cmath>
-#include <iterator>
+#include <cstddef>
 #include <limits>
-#include <numeric>
+#include <random>
 #include <vector>
 
 #include "test_macros.h"
diff --git a/libcxx/test/std/ranges/range.adaptors/range.chunk.by/ctor.default.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.chunk.by/ctor.default.pass.cpp
index 98c6cb7af5f5..96d96053f74b 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.chunk.by/ctor.default.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.chunk.by/ctor.default.pass.cpp
@@ -16,6 +16,7 @@
 #include <ranges>
 
 #include <cassert>
+#include <cstddef>
 #include <type_traits>
 
 constexpr int buff[] = {-2, 1, -1, 2};
diff --git a/libcxx/test/std/ranges/range.adaptors/range.chunk.by/range.chunk.by.iter/deref.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.chunk.by/range.chunk.by.iter/deref.pass.cpp
index 8cc9bdac0b1d..6a5ce1335ecc 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.chunk.by/range.chunk.by.iter/deref.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.chunk.by/range.chunk.by.iter/deref.pass.cpp
@@ -17,13 +17,12 @@
 #include <algorithm>
 #include <array>
 #include <cassert>
+#include <cstddef>
 #include <concepts>
 #include <functional>
-#include <utility>
 
 #include "../types.h"
 #include "test_iterators.h"
-#include "test_macros.h"
 
 template <class Iter, class Sent = sentinel_wrapper<Iter>>
 constexpr void test() {
diff --git a/libcxx/test/std/ranges/range.factories/range.repeat.view/iterator/member_typedefs.compile.pass.cpp b/libcxx/test/std/ranges/range.factories/range.repeat.view/iterator/member_typedefs.compile.pass.cpp
index 2ef09a6d9216..d7202e35e66d 100644
--- a/libcxx/test/std/ranges/range.factories/range.repeat.view/iterator/member_typedefs.compile.pass.cpp
+++ b/libcxx/test/std/ranges/range.factories/range.repeat.view/iterator/member_typedefs.compile.pass.cpp
@@ -20,6 +20,7 @@
 
 #include <cassert>
 #include <concepts>
+#include <cstddef>
 #include <cstdint>
 #include <ranges>
 #include <type_traits>
diff --git a/libcxx/test/std/ranges/range.factories/range.repeat.view/iterator/minus.pass.cpp b/libcxx/test/std/ranges/range.factories/range.repeat.view/iterator/minus.pass.cpp
index 3a373741f4c0..523eefa69b4c 100644
--- a/libcxx/test/std/ranges/range.factories/range.repeat.view/iterator/minus.pass.cpp
+++ b/libcxx/test/std/ranges/range.factories/range.repeat.view/iterator/minus.pass.cpp
@@ -11,10 +11,10 @@
 // friend constexpr iterator operator-(iterator i, difference_type n);
 // friend constexpr difference_type operator-(const iterator& x, const iterator& y);
 
-#include <ranges>
 #include <cassert>
-#include <cstdint>
 #include <concepts>
+#include <cstddef>
+#include <ranges>
 
 constexpr bool test() {
   // <iterator> - difference_type
diff --git a/libcxx/test/std/ranges/range.utility/range.subrange/ctad.compile.pass.cpp b/libcxx/test/std/ranges/range.utility/range.subrange/ctad.compile.pass.cpp
index 670f9808b902..2bf6ca6fb0a0 100644
--- a/libcxx/test/std/ranges/range.utility/range.subrange/ctad.compile.pass.cpp
+++ b/libcxx/test/std/ranges/range.utility/range.subrange/ctad.compile.pass.cpp
@@ -13,7 +13,8 @@
 #include <ranges>
 
 #include <cassert>
-#include "test_macros.h"
+#include <cstddef>
+
 #include "test_iterators.h"
 
 using FI = forward_iterator<int*>;
diff --git a/libcxx/test/std/strings/string.view/string.view.modifiers/remove_prefix.pass.cpp b/libcxx/test/std/strings/string.view/string.view.modifiers/remove_prefix.pass.cpp
index 4bfa0f38829d..26db908428c3 100644
--- a/libcxx/test/std/strings/string.view/string.view.modifiers/remove_prefix.pass.cpp
+++ b/libcxx/test/std/strings/string.view/string.view.modifiers/remove_prefix.pass.cpp
@@ -12,8 +12,9 @@
 
 // void remove_prefix(size_type _n)
 
-#include <string_view>
 #include <cassert>
+#include <cstddef>
+#include <string_view>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/strings/string.view/string.view.modifiers/remove_suffix.pass.cpp b/libcxx/test/std/strings/string.view/string.view.modifiers/remove_suffix.pass.cpp
index 6d57f6c24a48..b6edccc68c9e 100644
--- a/libcxx/test/std/strings/string.view/string.view.modifiers/remove_suffix.pass.cpp
+++ b/libcxx/test/std/strings/string.view/string.view.modifiers/remove_suffix.pass.cpp
@@ -12,8 +12,9 @@
 
 // void remove_suffix(size_type _n)
 
-#include <string_view>
 #include <cassert>
+#include <cstddef>
+#include <string_view>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/strings/string.view/string.view.modifiers/swap.pass.cpp b/libcxx/test/std/strings/string.view/string.view.modifiers/swap.pass.cpp
index de0c5f423c4e..d18a7b28688d 100644
--- a/libcxx/test/std/strings/string.view/string.view.modifiers/swap.pass.cpp
+++ b/libcxx/test/std/strings/string.view/string.view.modifiers/swap.pass.cpp
@@ -12,8 +12,9 @@
 
 // void swap(basic_string_view& _other) noexcept
 
-#include <string_view>
 #include <cassert>
+#include <cstddef>
+#include <string_view>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/strings/string.view/string.view.ops/copy.pass.cpp b/libcxx/test/std/strings/string.view/string.view.ops/copy.pass.cpp
index 1d97723b90ed..0b27a05056c6 100644
--- a/libcxx/test/std/strings/string.view/string.view.ops/copy.pass.cpp
+++ b/libcxx/test/std/strings/string.view/string.view.ops/copy.pass.cpp
@@ -18,10 +18,11 @@
 // Effects: Equivalent to std::copy_n(begin() + pos, rlen, s).
 // Returns: rlen.
 
-#include <string_view>
 #include <algorithm>
 #include <cassert>
+#include <cstddef>
 #include <stdexcept>
+#include <string_view>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/strings/string.view/string.view.ops/substr.pass.cpp b/libcxx/test/std/strings/string.view/string.view.ops/substr.pass.cpp
index 1c8e0aebabd4..62b0259c175f 100644
--- a/libcxx/test/std/strings/string.view/string.view.ops/substr.pass.cpp
+++ b/libcxx/test/std/strings/string.view/string.view.ops/substr.pass.cpp
@@ -16,10 +16,11 @@
 // Effects: Determines the effective length rlen of the string to reference as the smaller of n and size() - pos.
 // Returns: basic_string_view(data()+pos, rlen).
 
-#include <string_view>
 #include <algorithm>
-#include <stdexcept>
 #include <cassert>
+#include <cstddef>
+#include <stdexcept>
+#include <string_view>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pass.cpp b/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pass.cpp
index 1704eb67562e..b9209ac2c89b 100644
--- a/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pass.cpp
@@ -33,9 +33,9 @@
 
 #include <algorithm>
 #include <cassert>
+#include <cstddef>
 #include <functional>
 
-#include "test_macros.h"
 #include "test_iterators.h"
 
 template <typename T> struct MyHash {
diff --git a/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pred.pass.cpp b/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pred.pass.cpp
index 3648be9bb06a..d616301767ed 100644
--- a/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pred.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pred.pass.cpp
@@ -33,9 +33,9 @@
 
 #include <algorithm>
 #include <cassert>
+#include <cstddef>
 #include <functional>
 
-#include "test_macros.h"
 #include "test_iterators.h"
 
 template <typename T> struct MyHash {
diff --git a/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pass.cpp b/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pass.cpp
index d0e6d5f8bfb6..19f5710819c4 100644
--- a/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pass.cpp
@@ -31,10 +31,10 @@
 // };
 
 #include <algorithm>
-#include <functional>
 #include <cassert>
+#include <cstddef>
+#include <functional>
 
-#include "test_macros.h"
 #include "test_iterators.h"
 
 template <typename T> struct MyHash {
diff --git a/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pred.pass.cpp b/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pred.pass.cpp
index 59b5b30d7f1b..3d29a05efd40 100644
--- a/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pred.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pred.pass.cpp
@@ -31,10 +31,10 @@
 // };
 
 #include <algorithm>
-#include <functional>
 #include <cassert>
+#include <cstddef>
+#include <functional>
 
-#include "test_macros.h"
 #include "test_iterators.h"
 
 template <typename T> struct MyHash {
diff --git a/libcxx/test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/deduct_F.pass.cpp b/libcxx/test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/deduct_F.pass.cpp
index 381bcda76170..064f7acdf71a 100644
--- a/libcxx/test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/deduct_F.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/deduct_F.pass.cpp
@@ -13,8 +13,8 @@
 
 // UNSUPPORTED: c++03, c++11, c++14
 
+#include <cstddef>
 #include <functional>
-#include <type_traits>
 #include <utility>
 
 #include "test_macros.h"
@@ -153,4 +153,3 @@ struct invalid_c_vararg { R operator()(int, ...) { return {}; } };
 static_assert(!can_deduce<invalid_rvalue_ref>);
 static_assert(!can_deduce<invalid_c_vararg>);
 static_assert(!can_deduce<std::nullptr_t>);
-
diff --git a/libcxx/test/std/utilities/function.objects/unord.hash/pointer.pass.cpp b/libcxx/test/std/utilities/function.objects/unord.hash/pointer.pass.cpp
index 577a9bf0e15b..448c5ba143c1 100644
--- a/libcxx/test/std/utilities/function.objects/unord.hash/pointer.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/unord.hash/pointer.pass.cpp
@@ -19,10 +19,10 @@
 
 // Not very portable
 
-#include <functional>
 #include <cassert>
+#include <cstddef>
+#include <functional>
 #include <type_traits>
-#include <limits>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.construct/construct_at.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.construct/construct_at.pass.cpp
index 272441ebedc2..92e8879d05bf 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.construct/construct_at.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.construct/construct_at.pass.cpp
@@ -13,8 +13,9 @@
 // template <class T, class ...Args>
 // constexpr T* construct_at(T* location, Args&& ...args);
 
-#include <memory>
 #include <cassert>
+#include <cstddef>
+#include <memory>
 #include <utility>
 
 #include "test_iterators.h"
diff --git a/libcxx/test/std/utilities/memory/temporary.buffer/overaligned.pass.cpp b/libcxx/test/std/utilities/memory/temporary.buffer/overaligned.pass.cpp
index 4c66370fac92..3fa935527086 100644
--- a/libcxx/test/std/utilities/memory/temporary.buffer/overaligned.pass.cpp
+++ b/libcxx/test/std/utilities/memory/temporary.buffer/overaligned.pass.cpp
@@ -22,12 +22,11 @@
 //   return_temporary_buffer(T* p);
 
 #include <cassert>
+#include <cstddef>
 #include <cstdint>
 #include <memory>
 #include <utility>
 
-#include "test_macros.h"
-
 struct alignas(32) A {
     int field;
 };
diff --git a/libcxx/test/std/utilities/memory/temporary.buffer/temporary_buffer.pass.cpp b/libcxx/test/std/utilities/memory/temporary.buffer/temporary_buffer.pass.cpp
index 5f7fc4571906..39a4767d874e 100644
--- a/libcxx/test/std/utilities/memory/temporary.buffer/temporary_buffer.pass.cpp
+++ b/libcxx/test/std/utilities/memory/temporary.buffer/temporary_buffer.pass.cpp
@@ -19,12 +19,11 @@
 //   void
 //   return_temporary_buffer(T* p);
 
-#include <memory>
 #include <cassert>
+#include <cstddef>
+#include <memory>
 #include <utility>
 
-#include "test_macros.h"
-
 int main(int, char**)
 {
     std::pair<int*, std::ptrdiff_t> ip = std::get_temporary_buffer<int>(5);
diff --git a/libcxx/test/std/utilities/meta/meta.rel/is_invocable.pass.cpp b/libcxx/test/std/utilities/meta/meta.rel/is_invocable.pass.cpp
index 32db9d12099f..2eecf7925d73 100644
--- a/libcxx/test/std/utilities/meta/meta.rel/is_invocable.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.rel/is_invocable.pass.cpp
@@ -18,13 +18,12 @@
 // Fn and all types in the template parameter pack ArgTypes shall be
 //  complete types, cv void, or arrays of unknown bound.
 
-#include <type_traits>
+#include <cstddef>
 #include <functional>
 #include <memory>
+#include <type_traits>
 #include <vector>
 
-#include "test_macros.h"
-
 struct Tag {};
 struct DerFromTag : Tag {};
 
diff --git a/libcxx/test/std/utilities/meta/meta.rel/is_nothrow_invocable.pass.cpp b/libcxx/test/std/utilities/meta/meta.rel/is_nothrow_invocable.pass.cpp
index fa6048e869e1..47e81f38e54b 100644
--- a/libcxx/test/std/utilities/meta/meta.rel/is_nothrow_invocable.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.rel/is_nothrow_invocable.pass.cpp
@@ -12,12 +12,10 @@
 
 // is_nothrow_invocable
 
+#include <cstddef>
 #include <type_traits>
-#include <functional>
 #include <vector>
 
-#include "test_macros.h"
-
 struct Tag {};
 
 struct Implicit {
diff --git a/libcxx/test/std/utilities/utility/mem.res/mem.res.monotonic.buffer/mem.res.monotonic.buffer.mem/allocate_overaligned_request.pass.cpp b/libcxx/test/std/utilities/utility/mem.res/mem.res.monotonic.buffer/mem.res.monotonic.buffer.mem/allocate_overaligned_request.pass.cpp
index cf3dfe21d967..ad967798747a 100644
--- a/libcxx/test/std/utilities/utility/mem.res/mem.res.monotonic.buffer/mem.res.monotonic.buffer.mem/allocate_overaligned_request.pass.cpp
+++ b/libcxx/test/std/utilities/utility/mem.res/mem.res.monotonic.buffer/mem.res.monotonic.buffer.mem/allocate_overaligned_request.pass.cpp
@@ -14,8 +14,9 @@
 
 // class monotonic_buffer_resource
 
-#include <memory_resource>
 #include <cassert>
+#include <cstddef>
+#include <memory_resource>
 
 #include "test_macros.h"
 #include "count_new.h"
diff --git a/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/sync_allocate_overaligned_request.pass.cpp b/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/sync_allocate_overaligned_request.pass.cpp
index 35a6a8f135a4..f3e1e4c026ef 100644
--- a/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/sync_allocate_overaligned_request.pass.cpp
+++ b/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/sync_allocate_overaligned_request.pass.cpp
@@ -14,9 +14,10 @@
 
 // class synchronized_pool_resource
 
-#include <memory_resource>
 #include <cassert>
+#include <cstddef>
 #include <memory> // std::align
+#include <memory_resource>
 
 #include "count_new.h"
 #include "test_macros.h"
diff --git a/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/sync_deallocate_matches_allocate.pass.cpp b/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/sync_deallocate_matches_allocate.pass.cpp
index d5b3b6e08a42..f55ab1d1aa5c 100644
--- a/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/sync_deallocate_matches_allocate.pass.cpp
+++ b/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/sync_deallocate_matches_allocate.pass.cpp
@@ -15,9 +15,10 @@
 
 // class synchronized_pool_resource
 
-#include <memory_resource>
 #include <algorithm>
 #include <cassert>
+#include <cstddef>
+#include <memory_resource>
 #include <new>
 #include <vector>
 
diff --git a/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/unsync_allocate_overaligned_request.pass.cpp b/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/unsync_allocate_overaligned_request.pass.cpp
index cd8d8a774592..5153a54ef3f1 100644
--- a/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/unsync_allocate_overaligned_request.pass.cpp
+++ b/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/unsync_allocate_overaligned_request.pass.cpp
@@ -14,9 +14,10 @@
 
 // class unsynchronized_pool_resource
 
-#include <memory_resource>
 #include <cassert>
+#include <cstddef>
 #include <memory> // std::align
+#include <memory_resource>
 
 #include "count_new.h"
 #include "test_macros.h"
diff --git a/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/unsync_deallocate_matches_allocate.pass.cpp b/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/unsync_deallocate_matches_allocate.pass.cpp
index fe5f4736a7b1..a0e86f82e9b2 100644
--- a/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/unsync_deallocate_matches_allocate.pass.cpp
+++ b/libcxx/test/std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.mem/unsync_deallocate_matches_allocate.pass.cpp
@@ -15,9 +15,10 @@
 
 // class unsynchronized_pool_resource
 
-#include <memory_resource>
 #include <algorithm>
 #include <cassert>
+#include <cstddef>
+#include <memory_resource>
 #include <new>
 #include <vector>
 
diff --git a/libcxx/utils/generate_escaped_output_table.py b/libcxx/utils/generate_escaped_output_table.py
index 41524e8fe718..59dd707ae612 100755
--- a/libcxx/utils/generate_escaped_output_table.py
+++ b/libcxx/utils/generate_escaped_output_table.py
@@ -218,7 +218,7 @@ TABLES_HPP_TEMPLATE = """
 
 #include <__algorithm/ranges_upper_bound.h>
 #include <__config>
-#include <cstddef>
+#include <__cstddef/ptrdiff_t.h>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/utils/generate_extended_grapheme_cluster_table.py b/libcxx/utils/generate_extended_grapheme_cluster_table.py
index 558b60618613..eba88a4f4877 100755
--- a/libcxx/utils/generate_extended_grapheme_cluster_table.py
+++ b/libcxx/utils/generate_extended_grapheme_cluster_table.py
@@ -214,8 +214,8 @@ MSVC_FORMAT_UCD_TABLES_HPP_TEMPLATE = """
 
 #include <__algorithm/ranges_upper_bound.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__iterator/access.h>
-#include <cstddef>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/utils/generate_indic_conjunct_break_table.py b/libcxx/utils/generate_indic_conjunct_break_table.py
index e41f6e9be233..580d8157ffeb 100755
--- a/libcxx/utils/generate_indic_conjunct_break_table.py
+++ b/libcxx/utils/generate_indic_conjunct_break_table.py
@@ -207,8 +207,8 @@ MSVC_FORMAT_UCD_TABLES_HPP_TEMPLATE = """
 
 #include <__algorithm/ranges_upper_bound.h>
 #include <__config>
+#include <__cstddef/ptrdiff_t.h>
 #include <__iterator/access.h>
-#include <cstddef>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/utils/generate_width_estimation_table.py b/libcxx/utils/generate_width_estimation_table.py
index d8c036f34e83..f81f0ba77489 100644
--- a/libcxx/utils/generate_width_estimation_table.py
+++ b/libcxx/utils/generate_width_estimation_table.py
@@ -246,7 +246,7 @@ TABLES_HPP_TEMPLATE = """
 
 #include <__algorithm/ranges_upper_bound.h>
 #include <__config>
-#include <cstddef>
+#include <__cstddef/ptrdiff_t.h>
 #include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxxabi/src/private_typeinfo.cpp b/libcxxabi/src/private_typeinfo.cpp
index 9dba91e1985e..2f631041f74c 100644
--- a/libcxxabi/src/private_typeinfo.cpp
+++ b/libcxxabi/src/private_typeinfo.cpp
@@ -41,9 +41,11 @@
 // Defining _LIBCXXABI_FORGIVING_DYNAMIC_CAST does not help since can_catch() calls
 // is_equal() with use_strcmp=false so the string names are not compared.
 
-#include <cstdint>
 #include <cassert>
+#include <cstddef>
+#include <cstdint>
 #include <string.h>
+
 #include "abort_message.h"
 
 #ifdef _LIBCXXABI_FORGIVING_DYNAMIC_CAST
diff --git a/libcxxabi/test/test_aux_runtime.pass.cpp b/libcxxabi/test/test_aux_runtime.pass.cpp
index dde553864e38..499382c782ac 100644
--- a/libcxxabi/test/test_aux_runtime.pass.cpp
+++ b/libcxxabi/test/test_aux_runtime.pass.cpp
@@ -25,9 +25,13 @@ bool bad_typeid_test () {
     class A { virtual void f() {}};
     class B { virtual void g() {}};
 
-    B *bp = NULL;
-    try {bool b = typeid(*bp) == typeid (A); ((void)b); }
-    catch ( const std::bad_typeid &) { return true; }
+    B* bp = nullptr;
+    try {
+      bool b = typeid(*bp) == typeid(A);
+      ((void)b);
+    } catch (const std::bad_typeid&) {
+      return true;
+    }
     return false;
 }
 
-- 
GitLab


From 3243e3d8872585091d65ea7ff0639155b4c1dd7a Mon Sep 17 00:00:00 2001
From: jimingham <jingham@apple.com>
Date: Wed, 30 Oct 2024 18:26:38 -0700
Subject: [PATCH 673/693] Fix stepping away from the bottom-most frame of a
 virtual inlined call stack. (#114337)

The computation of 'Thread::IsVirtualStep" was wrong - it called being
at the bottom of a virtual call stack a "virtual step" but that is
actually when you've gotten to concrete code and need to step for real.

I also added a test for this.
---
 lldb/source/Target/ThreadPlanStepInRange.cpp   |  3 ++-
 .../inline-stepping/TestInlineStepping.py      | 18 +++++++++++++++++-
 .../inline-stepping/calling.cpp                |  2 +-
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/lldb/source/Target/ThreadPlanStepInRange.cpp b/lldb/source/Target/ThreadPlanStepInRange.cpp
index 325a70619908..224a17d896cc 100644
--- a/lldb/source/Target/ThreadPlanStepInRange.cpp
+++ b/lldb/source/Target/ThreadPlanStepInRange.cpp
@@ -489,7 +489,8 @@ bool ThreadPlanStepInRange::DoWillResume(lldb::StateType resume_state,
 bool ThreadPlanStepInRange::IsVirtualStep() {
   if (m_virtual_step == eLazyBoolCalculate) {
     Thread &thread = GetThread();
-    if (thread.GetCurrentInlinedDepth() == UINT32_MAX)
+    uint32_t cur_inline_depth = thread.GetCurrentInlinedDepth();
+    if (cur_inline_depth == UINT32_MAX || cur_inline_depth == 0)
       m_virtual_step = eLazyBoolNo;
     else
       m_virtual_step = eLazyBoolYes;
diff --git a/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py b/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
index f52e0f0fd5bc..3283918f8527 100644
--- a/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
+++ b/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
@@ -364,7 +364,9 @@ class TestInlineStepping(TestBase):
         step_sequence = [["// In max_value specialized", "into"]]
         self.run_step_sequence(step_sequence)
 
-    def run_to_call_site_and_step(self, source_regex, func_name, start_pos):
+    def run_to_call_site_and_step(
+        self, source_regex, func_name, start_pos, one_more_step_loc=None
+    ):
         main_spec = lldb.SBFileSpec("calling.cpp")
         # Set the breakpoint by file and line, not sourced regex because
         # we want to make sure we can set breakpoints on call sites:
@@ -408,6 +410,14 @@ class TestInlineStepping(TestBase):
                 # stepping for this function...
                 break
 
+        if one_more_step_loc:
+            thread.StepInto()
+            frame_0 = thread.frame[0]
+            self.assertEqual(
+                frame_0.line_entry.line,
+                line_number(self.main_source, one_more_step_loc),
+                "Was able to step one more time",
+            )
         process.Kill()
         target.Clear()
 
@@ -420,3 +430,9 @@ class TestInlineStepping(TestBase):
         self.run_to_call_site_and_step(
             "In caller_trivial_inline_2", "caller_trivial_inline_2", 3
         )
+        self.run_to_call_site_and_step(
+            "In caller_trivial_inline_3",
+            "caller_trivial_inline_3",
+            4,
+            "After caller_trivial_inline_3",
+        )
diff --git a/lldb/test/API/functionalities/inline-stepping/calling.cpp b/lldb/test/API/functionalities/inline-stepping/calling.cpp
index d7ee56b3c079..ba71c25a3c64 100644
--- a/lldb/test/API/functionalities/inline-stepping/calling.cpp
+++ b/lldb/test/API/functionalities/inline-stepping/calling.cpp
@@ -95,7 +95,7 @@ void caller_trivial_inline_1() {
 
 void caller_trivial_inline_2() {
   caller_trivial_inline_3(); // In caller_trivial_inline_2.
-  inline_value += 1;
+  inline_value += 1;         // After caller_trivial_inline_3
 }
 
 void caller_trivial_inline_3() {
-- 
GitLab


From e9b7fe8e5a5819cb632d02529712535ca1b83f02 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Thu, 31 Oct 2024 09:32:01 +0800
Subject: [PATCH 674/693] [clangd] [Modules] Use ASTReader directly in
 IsModuleFileUpToDate (#113879)

@kadircet mentioned in
https://github.com/llvm/llvm-project/commit/448d8fa880be5cae0f63c3b248f07f647013a5a4#diff-fb3ba8a781117ff04736f951a274812cb7ad1678f9d71d4d91870b711ab45da0L285
that:

> this is definitely a functional change, clangd is used in environments
that solely relies on VFS, and doesn't depend on ASTUnit at all.

> right now this is both introducing a dependency on ASTUnit, and making
all the logical IO physical instead. can you instead use the regular
compiler utilities in clangd, and get the astreader from
CompilerInstance directly, which is VFS-aware, and doesn't depend on
ASTUnit ?

This tries to resolve the problem by creating ASTReader directly and use
VFS to create the FileManager.
---
 clang-tools-extra/clangd/ModulesBuilder.cpp | 63 ++++++++++++++-------
 1 file changed, 41 insertions(+), 22 deletions(-)

diff --git a/clang-tools-extra/clangd/ModulesBuilder.cpp b/clang-tools-extra/clangd/ModulesBuilder.cpp
index 1eeff468ef12..97f67ddf5495 100644
--- a/clang-tools-extra/clangd/ModulesBuilder.cpp
+++ b/clang-tools-extra/clangd/ModulesBuilder.cpp
@@ -12,6 +12,7 @@
 #include "clang/Frontend/FrontendAction.h"
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Serialization/ASTReader.h"
+#include "clang/Serialization/InMemoryModuleCache.h"
 
 namespace clang {
 namespace clangd {
@@ -127,50 +128,68 @@ struct ModuleFile {
   std::string ModuleFilePath;
 };
 
-bool IsModuleFileUpToDate(
-    PathRef ModuleFilePath,
-    const PrerequisiteModules &RequisiteModules) {
-IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
-      CompilerInstance::createDiagnostics(new DiagnosticOptions());
-
+bool IsModuleFileUpToDate(PathRef ModuleFilePath,
+                          const PrerequisiteModules &RequisiteModules,
+                          llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
   auto HSOpts = std::make_shared<HeaderSearchOptions>();
   RequisiteModules.adjustHeaderSearchOptions(*HSOpts);
   HSOpts->ForceCheckCXX20ModulesInputFiles = true;
   HSOpts->ValidateASTInputFilesContent = true;
 
+  clang::clangd::IgnoreDiagnostics IgnoreDiags;
+  IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
+      CompilerInstance::createDiagnostics(new DiagnosticOptions, &IgnoreDiags,
+                                          /*ShouldOwnClient=*/false);
+
+  LangOptions LangOpts;
+  LangOpts.SkipODRCheckInGMF = true;
+
+  FileManager FileMgr(FileSystemOptions(), VFS);
+
+  SourceManager SourceMgr(*Diags, FileMgr);
+
+  HeaderSearch HeaderInfo(HSOpts, SourceMgr, *Diags, LangOpts,
+                          /*Target=*/nullptr);
+
+  TrivialModuleLoader ModuleLoader;
+  Preprocessor PP(std::make_shared<PreprocessorOptions>(), *Diags, LangOpts,
+                  SourceMgr, HeaderInfo, ModuleLoader);
+
+  IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache = new InMemoryModuleCache;
   PCHContainerOperations PCHOperations;
-  std::unique_ptr<ASTUnit> Unit = ASTUnit::LoadFromASTFile(
-      ModuleFilePath.str(), PCHOperations.getRawReader(), ASTUnit::LoadASTOnly,
-      Diags, FileSystemOptions(), std::move(HSOpts));
+  ASTReader Reader(PP, *ModuleCache, /*ASTContext=*/nullptr,
+                   PCHOperations.getRawReader(), {});
 
-  if (!Unit)
-    return false;
+  // We don't need any listener here. By default it will use a validator
+  // listener.
+  Reader.setListener(nullptr);
 
-  auto Reader = Unit->getASTReader();
-  if (!Reader)
+  if (Reader.ReadAST(ModuleFilePath, serialization::MK_MainFile,
+                     SourceLocation(),
+                     ASTReader::ARR_None) != ASTReader::Success)
     return false;
 
   bool UpToDate = true;
-  Reader->getModuleManager().visit([&](serialization::ModuleFile &MF) -> bool {
-    Reader->visitInputFiles(
+  Reader.getModuleManager().visit([&](serialization::ModuleFile &MF) -> bool {
+    Reader.visitInputFiles(
         MF, /*IncludeSystem=*/false, /*Complain=*/false,
         [&](const serialization::InputFile &IF, bool isSystem) {
           if (!IF.getFile() || IF.isOutOfDate())
             UpToDate = false;
         });
-
     return !UpToDate;
   });
-
   return UpToDate;
 }
 
 bool IsModuleFilesUpToDate(
     llvm::SmallVector<PathRef> ModuleFilePaths,
-    const PrerequisiteModules &RequisiteModules) {
-  return llvm::all_of(ModuleFilePaths, [&RequisiteModules](auto ModuleFilePath) {
-    return IsModuleFileUpToDate(ModuleFilePath, RequisiteModules);
-  });
+    const PrerequisiteModules &RequisiteModules,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
+  return llvm::all_of(
+      ModuleFilePaths, [&RequisiteModules, VFS](auto ModuleFilePath) {
+        return IsModuleFileUpToDate(ModuleFilePath, RequisiteModules, VFS);
+      });
 }
 
 // StandalonePrerequisiteModules - stands for PrerequisiteModules for which all
@@ -347,7 +366,7 @@ bool StandalonePrerequisiteModules::canReuse(
   SmallVector<StringRef> BMIPaths;
   for (auto &MF : RequiredModules)
     BMIPaths.push_back(MF.ModuleFilePath);
-  return IsModuleFilesUpToDate(BMIPaths, *this);
+  return IsModuleFilesUpToDate(BMIPaths, *this, VFS);
 }
 
 } // namespace clangd
-- 
GitLab


From de7ad6b6820a265160507f8c6cf8ce5e07c4d5d8 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Wed, 30 Oct 2024 08:41:30 -0700
Subject: [PATCH 675/693] [lldb] Use Py_InitializeFromConfig with Python >= 3.8
 (NFC) (#114112)

This fixes the deprecation warning for Py_SetPythonHome, which was
deprecated in Python 3.11. With this patch, when building against Python
3.8 or later, we now use Py_InitializeFromConfig instead.

Fixes #113475
---
 .../Python/ScriptInterpreterPython.cpp        | 63 ++++++++++---------
 1 file changed, 35 insertions(+), 28 deletions(-)

diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index 7cc38da6a6a9..44fd05150ebc 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -92,7 +92,33 @@ namespace {
 struct InitializePythonRAII {
 public:
   InitializePythonRAII() {
-    InitializePythonHome();
+#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 8) || (PY_MAJOR_VERSION > 3)
+    PyConfig config;
+    PyConfig_InitPythonConfig(&config);
+#endif
+
+#if LLDB_EMBED_PYTHON_HOME
+    static std::string g_python_home = []() -> std::string {
+      if (llvm::sys::path::is_absolute(LLDB_PYTHON_HOME))
+        return LLDB_PYTHON_HOME;
+
+      FileSpec spec = HostInfo::GetShlibDir();
+      if (!spec)
+        return {};
+      spec.AppendPathComponent(LLDB_PYTHON_HOME);
+      return spec.GetPath();
+    }();
+    if (!g_python_home.empty()) {
+#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 8) || (PY_MAJOR_VERSION > 3)
+      PyConfig_SetBytesString(&config, &config.home, g_python_home.c_str());
+#else
+      size_t size = 0;
+      wchar_t *python_home_w = Py_DecodeLocale(g_python_home.c_str(), &size);
+      Py_SetPythonHome(python_home_w);
+      PyMem_RawFree(python_home_w);
+#endif
+    }
+#endif
 
     // The table of built-in modules can only be extended before Python is
     // initialized.
@@ -117,15 +143,22 @@ public:
       PyImport_AppendInittab("_lldb", LLDBSwigPyInit);
     }
 
+#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 8) || (PY_MAJOR_VERSION > 3)
+    config.install_signal_handlers = 0;
+    Py_InitializeFromConfig(&config);
+    PyConfig_Clear(&config);
+    InitializeThreadsPrivate();
+#else
 // Python < 3.2 and Python >= 3.2 reversed the ordering requirements for
 // calling `Py_Initialize` and `PyEval_InitThreads`.  < 3.2 requires that you
 // call `PyEval_InitThreads` first, and >= 3.2 requires that you call it last.
-#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 2) || (PY_MAJOR_VERSION > 3)
+#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 2)
     Py_InitializeEx(0);
     InitializeThreadsPrivate();
 #else
     InitializeThreadsPrivate();
     Py_InitializeEx(0);
+#endif
 #endif
   }
 
@@ -142,32 +175,6 @@ public:
   }
 
 private:
-  void InitializePythonHome() {
-#if LLDB_EMBED_PYTHON_HOME
-    typedef wchar_t *str_type;
-    static str_type g_python_home = []() -> str_type {
-      const char *lldb_python_home = LLDB_PYTHON_HOME;
-      const char *absolute_python_home = nullptr;
-      llvm::SmallString<64> path;
-      if (llvm::sys::path::is_absolute(lldb_python_home)) {
-        absolute_python_home = lldb_python_home;
-      } else {
-        FileSpec spec = HostInfo::GetShlibDir();
-        if (!spec)
-          return nullptr;
-        spec.GetPath(path);
-        llvm::sys::path::append(path, lldb_python_home);
-        absolute_python_home = path.c_str();
-      }
-      size_t size = 0;
-      return Py_DecodeLocale(absolute_python_home, &size);
-    }();
-    if (g_python_home != nullptr) {
-      Py_SetPythonHome(g_python_home);
-    }
-#endif
-  }
-
   void InitializeThreadsPrivate() {
 // Since Python 3.7 `Py_Initialize` calls `PyEval_InitThreads` inside itself,
 // so there is no way to determine whether the embedded interpreter
-- 
GitLab


From 14f3cdc8e2ffc6ce88f9010bf6317f214dd8bcaf Mon Sep 17 00:00:00 2001
From: Alexey Samsonov <vonosmas@gmail.com>
Date: Wed, 30 Oct 2024 19:05:24 -0700
Subject: [PATCH 676/693] [libc][bazel] Add BUILD rules for float16 math
 functions. (#114187)

Adds libc_math_function rules for various f16* and *f16 functions.
Closes #114140
---
 .../llvm-project-overlay/libc/BUILD.bazel     | 331 ++++++++++++++++++
 1 file changed, 331 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index d4aeaea6fac8..820163415f98 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -1702,6 +1702,17 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "sincosf16_utils",
+    hdrs = ["src/math/generic/sincosf16_utils.h"],
+    deps = [
+        ":__support_common",
+        ":__support_fputil_fp_bits",
+        ":__support_fputil_nearest_integer",
+        ":__support_fputil_polyeval",
+    ],
+)
+
 libc_support_library(
     name = "explogxf",
     srcs = ["src/math/generic/explogxf.cpp"],
@@ -1773,6 +1784,18 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "expxf16",
+    hdrs = ["src/math/generic/expxf16.h"],
+    deps = [
+        ":__support_cpp_array",
+        ":__support_fputil_cast",
+        ":__support_fputil_fp_bits",
+        ":__support_fputil_nearest_integer",
+        ":__support_fputil_polyeval",
+    ],
+)
+
 ################################ math targets ##################################
 
 libc_math_function(
@@ -1882,6 +1905,8 @@ libc_math_function(name = "canonicalizel")
 
 libc_math_function(name = "canonicalizef128")
 
+libc_math_function(name = "canonicalizef16")
+
 libc_math_function(
     name = "cbrt",
     additional_deps = [
@@ -1906,6 +1931,8 @@ libc_math_function(name = "ceill")
 
 libc_math_function(name = "ceilf128")
 
+libc_math_function(name = "ceilf16")
+
 libc_math_function(name = "copysign")
 
 libc_math_function(name = "copysignf")
@@ -1914,6 +1941,8 @@ libc_math_function(name = "copysignl")
 
 libc_math_function(name = "copysignf128")
 
+libc_math_function(name = "copysignf16")
+
 libc_math_function(
     name = "cos",
     additional_deps = [
@@ -1950,6 +1979,13 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "coshf16",
+    additional_deps = [
+        ":expxf16",
+    ],
+)
+
 libc_math_function(
     name = "cospif",
     additional_deps = [
@@ -1965,6 +2001,15 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "cospif16",
+    additional_deps = [
+        ":__support_fputil_multiply_add",
+        ":__support_macros_optimization",
+        ":sincosf16_utils",
+    ],
+)
+
 libc_math_function(name = "daddl")
 
 libc_math_function(name = "daddf128")
@@ -2048,6 +2093,13 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "expf16",
+    additional_deps = [
+        ":expxf16",
+    ],
+)
+
 libc_math_function(
     name = "exp10",
     additional_deps = [
@@ -2072,6 +2124,20 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "exp10f16",
+    additional_deps = [
+        ":expxf16",
+    ],
+)
+
+libc_math_function(
+    name = "exp10m1f16",
+    additional_deps = [
+        ":expxf16",
+    ],
+)
+
 libc_math_function(
     name = "exp2",
     additional_deps = [
@@ -2096,6 +2162,13 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "exp2f16",
+    additional_deps = [
+        ":expxf16",
+    ],
+)
+
 libc_math_function(
     name = "exp2m1f",
     additional_deps = [
@@ -2104,6 +2177,13 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "exp2m1f16",
+    additional_deps = [
+        ":expxf16",
+    ],
+)
+
 libc_math_function(
     name = "expm1",
     additional_deps = [
@@ -2135,6 +2215,101 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "expm1f16",
+    additional_deps = [
+        ":expxf16",
+    ],
+)
+
+libc_math_function(name = "f16add")
+
+libc_math_function(name = "f16addf")
+
+libc_math_function(name = "f16addf128")
+
+libc_math_function(name = "f16addl")
+
+libc_math_function(name = "f16div")
+
+libc_math_function(name = "f16divf")
+
+libc_math_function(name = "f16divf128")
+
+libc_math_function(name = "f16divl")
+
+libc_math_function(
+    name = "f16fma",
+    additional_deps = [
+        ":__support_fputil_fma",
+    ],
+)
+
+libc_math_function(
+    name = "f16fmaf",
+    additional_deps = [
+        ":__support_fputil_fma",
+    ],
+)
+
+libc_math_function(
+    name = "f16fmaf128",
+    additional_deps = [
+        ":__support_fputil_fma",
+    ],
+)
+
+libc_math_function(
+    name = "f16fmal",
+    additional_deps = [
+        ":__support_fputil_fma",
+    ],
+)
+
+libc_math_function(name = "f16mul")
+
+libc_math_function(name = "f16mulf")
+
+libc_math_function(name = "f16mulf128")
+
+libc_math_function(name = "f16mull")
+
+libc_math_function(
+    name = "f16sqrt",
+    additional_deps = [
+        ":__support_fputil_sqrt",
+    ],
+)
+
+libc_math_function(
+    name = "f16sqrtf",
+    additional_deps = [
+        ":__support_fputil_sqrt",
+    ],
+)
+
+libc_math_function(
+    name = "f16sqrtf128",
+    additional_deps = [
+        ":__support_fputil_sqrt",
+    ],
+)
+
+libc_math_function(
+    name = "f16sqrtl",
+    additional_deps = [
+        ":__support_fputil_sqrt",
+    ],
+)
+
+libc_math_function(name = "f16sub")
+
+libc_math_function(name = "f16subf")
+
+libc_math_function(name = "f16subf128")
+
+libc_math_function(name = "f16subl")
+
 libc_math_function(name = "fabs")
 
 libc_math_function(name = "fabsf")
@@ -2143,6 +2318,8 @@ libc_math_function(name = "fabsl")
 
 libc_math_function(name = "fabsf128")
 
+libc_math_function(name = "fabsf16")
+
 libc_math_function(name = "fadd")
 
 libc_math_function(name = "faddl")
@@ -2157,6 +2334,8 @@ libc_math_function(name = "fdiml")
 
 libc_math_function(name = "fdimf128")
 
+libc_math_function(name = "fdimf16")
+
 libc_math_function(name = "fdiv")
 
 libc_math_function(name = "fdivl")
@@ -2192,6 +2371,8 @@ libc_math_function(name = "floorl")
 
 libc_math_function(name = "floorf128")
 
+libc_math_function(name = "floorf16")
+
 # TODO: Add fma, fmaf, fmal, fmaf128 functions.
 
 libc_math_function(name = "fmax")
@@ -2202,6 +2383,8 @@ libc_math_function(name = "fmaxl")
 
 libc_math_function(name = "fmaxf128")
 
+libc_math_function(name = "fmaxf16")
+
 libc_math_function(name = "fmaximum")
 
 libc_math_function(name = "fmaximumf")
@@ -2210,6 +2393,8 @@ libc_math_function(name = "fmaximuml")
 
 libc_math_function(name = "fmaximumf128")
 
+libc_math_function(name = "fmaximumf16")
+
 libc_math_function(name = "fmaximum_mag")
 
 libc_math_function(name = "fmaximum_magf")
@@ -2218,6 +2403,8 @@ libc_math_function(name = "fmaximum_magl")
 
 libc_math_function(name = "fmaximum_magf128")
 
+libc_math_function(name = "fmaximum_magf16")
+
 libc_math_function(name = "fmaximum_mag_num")
 
 libc_math_function(name = "fmaximum_mag_numf")
@@ -2226,6 +2413,8 @@ libc_math_function(name = "fmaximum_mag_numl")
 
 libc_math_function(name = "fmaximum_mag_numf128")
 
+libc_math_function(name = "fmaximum_mag_numf16")
+
 libc_math_function(name = "fmaximum_num")
 
 libc_math_function(name = "fmaximum_numf")
@@ -2234,6 +2423,8 @@ libc_math_function(name = "fmaximum_numl")
 
 libc_math_function(name = "fmaximum_numf128")
 
+libc_math_function(name = "fmaximum_numf16")
+
 libc_math_function(name = "fmin")
 
 libc_math_function(name = "fminf")
@@ -2242,6 +2433,8 @@ libc_math_function(name = "fminl")
 
 libc_math_function(name = "fminf128")
 
+libc_math_function(name = "fminf16")
+
 libc_math_function(name = "fminimum")
 
 libc_math_function(name = "fminimumf")
@@ -2250,6 +2443,8 @@ libc_math_function(name = "fminimuml")
 
 libc_math_function(name = "fminimumf128")
 
+libc_math_function(name = "fminimumf16")
+
 libc_math_function(name = "fminimum_mag")
 
 libc_math_function(name = "fminimum_magf")
@@ -2258,6 +2453,8 @@ libc_math_function(name = "fminimum_magl")
 
 libc_math_function(name = "fminimum_magf128")
 
+libc_math_function(name = "fminimum_magf16")
+
 libc_math_function(name = "fminimum_mag_num")
 
 libc_math_function(name = "fminimum_mag_numf")
@@ -2266,6 +2463,8 @@ libc_math_function(name = "fminimum_mag_numl")
 
 libc_math_function(name = "fminimum_mag_numf128")
 
+libc_math_function(name = "fminimum_mag_numf16")
+
 libc_math_function(name = "fminimum_num")
 
 libc_math_function(name = "fminimum_numf")
@@ -2274,6 +2473,8 @@ libc_math_function(name = "fminimum_numl")
 
 libc_math_function(name = "fminimum_numf128")
 
+libc_math_function(name = "fminimum_numf16")
+
 libc_math_function(
     name = "fmod",
     additional_deps = [
@@ -2302,6 +2503,13 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "fmodf16",
+    additional_deps = [
+        ":__support_fputil_generic_fmod",
+    ],
+)
+
 libc_math_function(
     name = "fmul",
     additional_deps = [
@@ -2321,6 +2529,8 @@ libc_math_function(name = "frexpl")
 
 libc_math_function(name = "frexpf128")
 
+libc_math_function(name = "frexpf16")
+
 libc_math_function(name = "fromfp")
 
 libc_math_function(name = "fromfpf")
@@ -2329,6 +2539,8 @@ libc_math_function(name = "fromfpl")
 
 libc_math_function(name = "fromfpf128")
 
+libc_math_function(name = "fromfpf16")
+
 libc_math_function(name = "fromfpx")
 
 libc_math_function(name = "fromfpxf")
@@ -2337,6 +2549,8 @@ libc_math_function(name = "fromfpxl")
 
 libc_math_function(name = "fromfpxf128")
 
+libc_math_function(name = "fromfpxf16")
+
 libc_math_function(
     name = "fsqrt",
     additional_deps = [
@@ -2372,6 +2586,8 @@ libc_math_function(name = "getpayloadl")
 
 libc_math_function(name = "getpayloadf128")
 
+libc_math_function(name = "getpayloadf16")
+
 libc_math_function(name = "hypot")
 
 libc_math_function(
@@ -2390,6 +2606,8 @@ libc_math_function(name = "ilogbl")
 
 libc_math_function(name = "ilogbf128")
 
+libc_math_function(name = "ilogbf16")
+
 libc_math_function(name = "ldexp")
 
 libc_math_function(name = "ldexpf")
@@ -2398,6 +2616,8 @@ libc_math_function(name = "ldexpl")
 
 libc_math_function(name = "ldexpf128")
 
+libc_math_function(name = "ldexpf16")
+
 libc_math_function(name = "llogb")
 
 libc_math_function(name = "llogbf")
@@ -2406,6 +2626,8 @@ libc_math_function(name = "llogbl")
 
 libc_math_function(name = "llogbf128")
 
+libc_math_function(name = "llogbf16")
+
 libc_math_function(name = "llrint")
 
 libc_math_function(name = "llrintf")
@@ -2414,6 +2636,8 @@ libc_math_function(name = "llrintl")
 
 libc_math_function(name = "llrintf128")
 
+libc_math_function(name = "llrintf16")
+
 libc_math_function(name = "llround")
 
 libc_math_function(name = "llroundf")
@@ -2422,6 +2646,8 @@ libc_math_function(name = "llroundl")
 
 libc_math_function(name = "llroundf128")
 
+libc_math_function(name = "llroundf16")
+
 libc_math_function(
     name = "log",
     additional_deps = [
@@ -2450,6 +2676,13 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "logf16",
+    additional_deps = [
+        ":expxf16",
+    ],
+)
+
 libc_math_function(
     name = "log10",
     additional_deps = [
@@ -2478,6 +2711,13 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "log10f16",
+    additional_deps = [
+        ":expxf16",
+    ],
+)
+
 libc_math_function(
     name = "log1p",
     additional_deps = [
@@ -2532,6 +2772,13 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "log2f16",
+    additional_deps = [
+        ":expxf16",
+    ],
+)
+
 libc_math_function(name = "logb")
 
 libc_math_function(name = "logbf")
@@ -2540,6 +2787,8 @@ libc_math_function(name = "logbl")
 
 libc_math_function(name = "logbf128")
 
+libc_math_function(name = "logbf16")
+
 libc_math_function(name = "lrint")
 
 libc_math_function(name = "lrintf")
@@ -2548,6 +2797,8 @@ libc_math_function(name = "lrintl")
 
 libc_math_function(name = "lrintf128")
 
+libc_math_function(name = "lrintf16")
+
 libc_math_function(name = "lround")
 
 libc_math_function(name = "lroundf")
@@ -2556,6 +2807,8 @@ libc_math_function(name = "lroundl")
 
 libc_math_function(name = "lroundf128")
 
+libc_math_function(name = "lroundf16")
+
 libc_math_function(name = "modf")
 
 libc_math_function(name = "modff")
@@ -2564,6 +2817,8 @@ libc_math_function(name = "modfl")
 
 libc_math_function(name = "modff128")
 
+libc_math_function(name = "modff16")
+
 libc_math_function(
     name = "nan",
     additional_deps = [
@@ -2596,6 +2851,14 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "nanf16",
+    additional_deps = [
+        ":__support_str_to_float",
+        ":errno",
+    ],
+)
+
 libc_math_function(name = "nearbyint")
 
 libc_math_function(name = "nearbyintf")
@@ -2604,6 +2867,8 @@ libc_math_function(name = "nearbyintl")
 
 libc_math_function(name = "nearbyintf128")
 
+libc_math_function(name = "nearbyintf16")
+
 libc_math_function(name = "nextafter")
 
 libc_math_function(name = "nextafterf")
@@ -2612,6 +2877,8 @@ libc_math_function(name = "nextafterl")
 
 libc_math_function(name = "nextafterf128")
 
+libc_math_function(name = "nextafterf16")
+
 libc_math_function(name = "nextdown")
 
 libc_math_function(name = "nextdownf")
@@ -2620,10 +2887,14 @@ libc_math_function(name = "nextdownl")
 
 libc_math_function(name = "nextdownf128")
 
+libc_math_function(name = "nextdownf16")
+
 libc_math_function(name = "nexttoward")
 
 libc_math_function(name = "nexttowardf")
 
+libc_math_function(name = "nexttowardf16")
+
 libc_math_function(name = "nexttowardl")
 
 libc_math_function(name = "nextup")
@@ -2634,6 +2905,8 @@ libc_math_function(name = "nextupl")
 
 libc_math_function(name = "nextupf128")
 
+libc_math_function(name = "nextupf16")
+
 libc_math_function(
     name = "pow",
     additional_deps = [
@@ -2671,6 +2944,8 @@ libc_math_function(name = "remainderl")
 
 libc_math_function(name = "remainderf128")
 
+libc_math_function(name = "remainderf16")
+
 libc_math_function(name = "remquo")
 
 libc_math_function(name = "remquof")
@@ -2679,6 +2954,8 @@ libc_math_function(name = "remquol")
 
 libc_math_function(name = "remquof128")
 
+libc_math_function(name = "remquof16")
+
 libc_math_function(name = "rint")
 
 libc_math_function(name = "rintf")
@@ -2687,6 +2964,8 @@ libc_math_function(name = "rintl")
 
 libc_math_function(name = "rintf128")
 
+libc_math_function(name = "rintf16")
+
 libc_math_function(name = "round")
 
 libc_math_function(name = "roundf")
@@ -2695,6 +2974,8 @@ libc_math_function(name = "roundl")
 
 libc_math_function(name = "roundf128")
 
+libc_math_function(name = "roundf16")
+
 libc_math_function(name = "roundeven")
 
 libc_math_function(name = "roundevenf")
@@ -2703,6 +2984,8 @@ libc_math_function(name = "roundevenl")
 
 libc_math_function(name = "roundevenf128")
 
+libc_math_function(name = "roundevenf16")
+
 libc_math_function(name = "scalbln")
 
 libc_math_function(name = "scalblnf")
@@ -2711,6 +2994,8 @@ libc_math_function(name = "scalblnl")
 
 libc_math_function(name = "scalblnf128")
 
+libc_math_function(name = "scalblnf16")
+
 libc_math_function(name = "scalbn")
 
 libc_math_function(name = "scalbnf")
@@ -2719,6 +3004,8 @@ libc_math_function(name = "scalbnl")
 
 libc_math_function(name = "scalbnf128")
 
+libc_math_function(name = "scalbnf16")
+
 libc_math_function(name = "setpayload")
 
 libc_math_function(name = "setpayloadf")
@@ -2727,6 +3014,8 @@ libc_math_function(name = "setpayloadl")
 
 libc_math_function(name = "setpayloadf128")
 
+libc_math_function(name = "setpayloadf16")
+
 libc_math_function(name = "setpayloadsig")
 
 libc_math_function(name = "setpayloadsigf")
@@ -2735,6 +3024,8 @@ libc_math_function(name = "setpayloadsigl")
 
 libc_math_function(name = "setpayloadsigf128")
 
+libc_math_function(name = "setpayloadsigf16")
+
 libc_math_function(
     name = "sin",
     additional_deps = [
@@ -2797,6 +3088,13 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "sinhf16",
+    additional_deps = [
+        ":expxf16",
+    ],
+)
+
 libc_math_function(
     name = "sinpif",
     additional_deps = [
@@ -2804,6 +3102,15 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "sinpif16",
+    additional_deps = [
+        ":__support_fputil_nearest_integer",
+        ":__support_fputil_polyeval",
+        ":sincosf16_utils",
+    ],
+)
+
 libc_math_function(
     name = "sqrt",
     additional_deps = [
@@ -2832,6 +3139,13 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "sqrtf16",
+    additional_deps = [
+        ":__support_fputil_sqrt",
+    ],
+)
+
 libc_math_function(
     name = "tan",
     additional_deps = [
@@ -2872,6 +3186,13 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "tanhf16",
+    additional_deps = [
+        ":expxf16",
+    ],
+)
+
 libc_math_function(name = "totalorder")
 
 libc_math_function(name = "totalorderf")
@@ -2880,6 +3201,8 @@ libc_math_function(name = "totalorderl")
 
 libc_math_function(name = "totalorderf128")
 
+libc_math_function(name = "totalorderf16")
+
 libc_math_function(name = "totalordermag")
 
 libc_math_function(name = "totalordermagf")
@@ -2888,6 +3211,8 @@ libc_math_function(name = "totalordermagl")
 
 libc_math_function(name = "totalordermagf128")
 
+libc_math_function(name = "totalordermagf16")
+
 libc_math_function(name = "trunc")
 
 libc_math_function(name = "truncf")
@@ -2896,6 +3221,8 @@ libc_math_function(name = "truncl")
 
 libc_math_function(name = "truncf128")
 
+libc_math_function(name = "truncf16")
+
 libc_math_function(name = "ufromfp")
 
 libc_math_function(name = "ufromfpf")
@@ -2904,6 +3231,8 @@ libc_math_function(name = "ufromfpl")
 
 libc_math_function(name = "ufromfpf128")
 
+libc_math_function(name = "ufromfpf16")
+
 libc_math_function(name = "ufromfpx")
 
 libc_math_function(name = "ufromfpxf")
@@ -2912,6 +3241,8 @@ libc_math_function(name = "ufromfpxl")
 
 libc_math_function(name = "ufromfpxf128")
 
+libc_math_function(name = "ufromfpxf16")
+
 ############################## inttypes targets ##############################
 
 libc_function(
-- 
GitLab


From 8127162427c5f8c28d6292e1d4b4ce8a00b2d5a2 Mon Sep 17 00:00:00 2001
From: Feng Zou <feng.zou@intel.com>
Date: Thu, 31 Oct 2024 10:14:25 +0800
Subject: [PATCH 677/693] [X86][AMX] Support AMX-FP8 (#113850)

Ref.: https://cdrdv2.intel.com/v1/dl/getContent/671368
---
 clang/docs/ReleaseNotes.rst                   |  1 +
 clang/include/clang/Basic/BuiltinsX86_64.def  |  6 ++
 clang/include/clang/Driver/Options.td         |  2 +
 clang/lib/Basic/Targets/X86.cpp               |  6 ++
 clang/lib/Basic/Targets/X86.h                 |  1 +
 clang/lib/Headers/CMakeLists.txt              |  1 +
 clang/lib/Headers/amxfp8intrin.h              | 95 +++++++++++++++++++
 clang/lib/Headers/immintrin.h                 |  4 +
 clang/lib/Sema/SemaX86.cpp                    |  4 +
 clang/test/CodeGen/X86/amx_fp8.c              | 27 ++++++
 clang/test/CodeGen/X86/amx_fp8_errors.c       | 10 ++
 clang/test/CodeGen/X86/amx_fp8_inline_asm.c   | 32 +++++++
 llvm/include/llvm/IR/IntrinsicsX86.td         | 17 ++++
 .../llvm/TargetParser/X86TargetParser.def     |  1 +
 llvm/lib/Target/X86/X86.td                    |  3 +
 llvm/lib/Target/X86/X86ISelLowering.cpp       | 10 +-
 llvm/lib/Target/X86/X86InstrAMX.td            | 39 ++++++++
 llvm/lib/Target/X86/X86InstrPredicates.td     |  1 +
 llvm/lib/TargetParser/Host.cpp                |  4 +
 llvm/lib/TargetParser/X86TargetParser.cpp     |  1 +
 llvm/test/CodeGen/X86/amx_fp8_intrinsics.ll   | 20 ++++
 llvm/test/MC/Disassembler/X86/AMX/amx-fp8.txt | 34 +++++++
 llvm/test/MC/X86/AMX/amx-fp8-att.s            | 33 +++++++
 llvm/test/MC/X86/AMX/amx-fp8-intel.s          | 33 +++++++
 24 files changed, 384 insertions(+), 1 deletion(-)
 create mode 100644 clang/lib/Headers/amxfp8intrin.h
 create mode 100644 clang/test/CodeGen/X86/amx_fp8.c
 create mode 100644 clang/test/CodeGen/X86/amx_fp8_errors.c
 create mode 100644 clang/test/CodeGen/X86/amx_fp8_inline_asm.c
 create mode 100644 llvm/test/CodeGen/X86/amx_fp8_intrinsics.ll
 create mode 100644 llvm/test/MC/Disassembler/X86/AMX/amx-fp8.txt
 create mode 100644 llvm/test/MC/X86/AMX/amx-fp8-att.s
 create mode 100644 llvm/test/MC/X86/AMX/amx-fp8-intel.s

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 402203f89e23..145786bcc59b 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -661,6 +661,7 @@ X86 Support
 
 - Supported intrinsics for ``MOVRS AND AVX10.2``.
   * Supported intrinsics of ``_mm(256|512)_(mask(z))_loadrs_epi(8|16|32|64)``.
+- Support ISA of ``AMX-FP8``.
 
 Arm and AArch64 Support
 ^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/Basic/BuiltinsX86_64.def b/clang/include/clang/Basic/BuiltinsX86_64.def
index e1e613560167..68904ae8abcd 100644
--- a/clang/include/clang/Basic/BuiltinsX86_64.def
+++ b/clang/include/clang/Basic/BuiltinsX86_64.def
@@ -155,6 +155,12 @@ TARGET_BUILTIN(__builtin_ia32_cmpccxadd64, "SLLiv*SLLiSLLiIi", "n", "cmpccxadd")
 // AMX_FP16 FP16
 TARGET_BUILTIN(__builtin_ia32_tdpfp16ps, "vIUcIUcIUc", "n", "amx-fp16")
 
+// AMX FP8
+TARGET_BUILTIN(__builtin_ia32_tdpbf8ps, "vIUcUIcUIc", "n", "amx-fp8")
+TARGET_BUILTIN(__builtin_ia32_tdpbhf8ps, "vIUcUIcUIc", "n", "amx-fp8")
+TARGET_BUILTIN(__builtin_ia32_tdphbf8ps, "vIUcUIcUIc", "n", "amx-fp8")
+TARGET_BUILTIN(__builtin_ia32_tdphf8ps, "vIUcUIcUIc", "n", "amx-fp8")
+
 // RAO-INT
 TARGET_BUILTIN(__builtin_ia32_aadd64, "vv*SOi", "n", "raoint")
 TARGET_BUILTIN(__builtin_ia32_aand64, "vv*SOi", "n", "raoint")
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9d595984b63c..2b9ee1a0e669 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -6300,6 +6300,8 @@ def mamx_fp16 : Flag<["-"], "mamx-fp16">, Group<m_x86_Features_Group>;
 def mno_amx_fp16 : Flag<["-"], "mno-amx-fp16">, Group<m_x86_Features_Group>;
 def mamx_int8 : Flag<["-"], "mamx-int8">, Group<m_x86_Features_Group>;
 def mno_amx_int8 : Flag<["-"], "mno-amx-int8">, Group<m_x86_Features_Group>;
+def mamx_fp8 : Flag<["-"], "mamx-fp8">, Group<m_x86_Features_Group>;
+def mno_amx_fp8 : Flag<["-"], "mno-amx-fp8">, Group<m_x86_Features_Group>;
 def mamx_tile : Flag<["-"], "mamx-tile">, Group<m_x86_Features_Group>;
 def mno_amx_tile : Flag<["-"], "mno-amx-tile">, Group<m_x86_Features_Group>;
 def mcmpccxadd : Flag<["-"], "mcmpccxadd">, Group<m_x86_Features_Group>;
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 82d29ea9fea5..4988682a22f0 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -428,6 +428,8 @@ bool X86TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasAMXTILE = true;
     } else if (Feature == "+amx-complex") {
       HasAMXCOMPLEX = true;
+    } else if (Feature == "+amx-fp8") {
+      HasAMXFP8 = true;
     } else if (Feature == "+cmpccxadd") {
       HasCMPCCXADD = true;
     } else if (Feature == "+raoint") {
@@ -947,6 +949,8 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__AMX_FP16__");
   if (HasAMXCOMPLEX)
     Builder.defineMacro("__AMX_COMPLEX__");
+  if (HasAMXFP8)
+    Builder.defineMacro("__AMX_FP8__");
   if (HasCMPCCXADD)
     Builder.defineMacro("__CMPCCXADD__");
   if (HasRAOINT)
@@ -1077,6 +1081,7 @@ bool X86TargetInfo::isValidFeatureName(StringRef Name) const {
       .Case("amx-fp16", true)
       .Case("amx-int8", true)
       .Case("amx-tile", true)
+      .Case("amx-fp8", true)
       .Case("avx", true)
       .Case("avx10.1-256", true)
       .Case("avx10.1-512", true)
@@ -1195,6 +1200,7 @@ bool X86TargetInfo::hasFeature(StringRef Feature) const {
       .Case("amx-fp16", HasAMXFP16)
       .Case("amx-int8", HasAMXINT8)
       .Case("amx-tile", HasAMXTILE)
+      .Case("amx-fp8", HasAMXFP8)
       .Case("avx", SSELevel >= AVX)
       .Case("avx10.1-256", HasAVX10_1)
       .Case("avx10.1-512", HasAVX10_1_512)
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index e8aad3ec5a74..a1b2a0cec209 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -157,6 +157,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
   bool HasAMXINT8 = false;
   bool HasAMXBF16 = false;
   bool HasAMXCOMPLEX = false;
+  bool HasAMXFP8 = false;
   bool HasSERIALIZE = false;
   bool HasTSXLDTRK = false;
   bool HasUSERMSR = false;
diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index 0211d1870b30..818de5a8e1d2 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -149,6 +149,7 @@ set(x86_files
   amxcomplexintrin.h
   amxfp16intrin.h
   amxintrin.h
+  amxfp8intrin.h
   avx10_2_512bf16intrin.h
   avx10_2_512convertintrin.h
   avx10_2_512minmaxintrin.h
diff --git a/clang/lib/Headers/amxfp8intrin.h b/clang/lib/Headers/amxfp8intrin.h
new file mode 100644
index 000000000000..0f5ddc87e5a7
--- /dev/null
+++ b/clang/lib/Headers/amxfp8intrin.h
@@ -0,0 +1,95 @@
+/*===------------- amxfp8intrin.h - AMX intrinsics -*- C++ -*----------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===------------------------------------------------------------------------===
+ */
+
+#ifndef __IMMINTRIN_H
+#error "Never use <amxfp8intrin.h> directly; include <immintrin.h> instead."
+#endif /* __IMMINTRIN_H */
+
+#ifndef __AMXFP8INTRIN_H
+#define __AMXFP8INTRIN_H
+#ifdef __x86_64__
+
+/// Peform the dot product of a BF8 value \a a by a BF8 value \a b accumulating
+/// into a Single Precision (FP32) source/dest \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// void _tile_dpbf8ps (__tile dst, __tile a, __tile b)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c TDPBF8PS instruction.
+///
+/// \param dst
+///    The destination tile. Max size is 1024 Bytes.
+/// \param a
+///    The 1st source tile. Max size is 1024 Bytes.
+/// \param b
+///    The 2nd source tile. Max size is 1024 Bytes.
+#define _tile_dpbf8ps(dst, a, b) __builtin_ia32_tdpbf8ps((dst), (a), (b))
+
+/// Perform the dot product of a BF8 value \a a by an HF8 value \a b
+/// accumulating into a Single Precision (FP32) source/dest \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// void _tile_dpbhf8ps (__tile dst, __tile a, __tile b)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c TDPBHF8PS instruction.
+///
+/// \param dst
+///    The destination tile. Max size is 1024 Bytes.
+/// \param a
+///    The 1st source tile. Max size is 1024 Bytes.
+/// \param b
+///    The 2nd source tile. Max size is 1024 Bytes.
+#define _tile_dpbhf8ps(dst, a, b) __builtin_ia32_tdpbhf8ps((dst), (a), (b))
+
+/// Perform the dot product of an HF8 value \a a by a BF8 value \a b
+/// accumulating into a Single Precision (FP32) source/dest \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// void _tile_dphbf8ps (__tile dst, __tile a, __tile b)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c TDPHBF8PS instruction.
+///
+/// \param dst
+///    The destination tile. Max size is 1024 Bytes.
+/// \param a
+///    The 1st source tile. Max size is 1024 Bytes.
+/// \param b
+///    The 2nd source tile. Max size is 1024 Bytes.
+#define _tile_dphbf8ps(dst, a, b) __builtin_ia32_tdphbf8ps((dst), (a), (b))
+
+/// Perform the dot product of an HF8 value \a a by an HF8 value \a b
+/// accumulating into a Single Precision (FP32) source/dest \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// void _tile_dphf8ps (__tile dst, __tile a, __tile b)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c TDPHF8PS instruction.
+///
+/// \param dst
+///    The destination tile. Max size is 1024 Bytes.
+/// \param a
+///    The 1st source tile. Max size is 1024 Bytes.
+/// \param b
+///    The 2nd source tile. Max size is 1024 Bytes.
+#define _tile_dphf8ps(dst, a, b) __builtin_ia32_tdphf8ps((dst), (a), (b))
+
+#endif /* __x86_64__ */
+#endif /* __AMXFP8INTRIN_H */
diff --git a/clang/lib/Headers/immintrin.h b/clang/lib/Headers/immintrin.h
index 65ad72bc479f..6184e9c84796 100644
--- a/clang/lib/Headers/immintrin.h
+++ b/clang/lib/Headers/immintrin.h
@@ -648,6 +648,10 @@ _storebe_i64(void * __P, long long __D) {
 #include <amxcomplexintrin.h>
 #endif
 
+#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_FP8__)
+#include <amxfp8intrin.h>
+#endif
+
 #if !defined(__SCE__) || __has_feature(modules) ||                             \
     defined(__AVX512VP2INTERSECT__)
 #include <avx512vp2intersectintrin.h>
diff --git a/clang/lib/Sema/SemaX86.cpp b/clang/lib/Sema/SemaX86.cpp
index 6a4d78f0ca90..0e43b030e70d 100644
--- a/clang/lib/Sema/SemaX86.cpp
+++ b/clang/lib/Sema/SemaX86.cpp
@@ -640,6 +640,10 @@ bool SemaX86::CheckBuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall) {
   case X86::BI__builtin_ia32_tdpfp16ps:
   case X86::BI__builtin_ia32_tcmmimfp16ps:
   case X86::BI__builtin_ia32_tcmmrlfp16ps:
+  case X86::BI__builtin_ia32_tdpbf8ps:
+  case X86::BI__builtin_ia32_tdpbhf8ps:
+  case X86::BI__builtin_ia32_tdphbf8ps:
+  case X86::BI__builtin_ia32_tdphf8ps:
     return CheckBuiltinTileRangeAndDuplicate(TheCall, {0, 1, 2});
   }
 }
diff --git a/clang/test/CodeGen/X86/amx_fp8.c b/clang/test/CodeGen/X86/amx_fp8.c
new file mode 100644
index 000000000000..9c79514f8912
--- /dev/null
+++ b/clang/test/CodeGen/X86/amx_fp8.c
@@ -0,0 +1,27 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown  -target-feature +amx-fp8  \
+// RUN: -emit-llvm -o - -Werror -pedantic | FileCheck %s
+#include <immintrin.h>
+
+void test_amx(void *data) {
+  //CHECK-LABEL: @test_amx
+  //CHECK: call void @llvm.x86.tdpbf8ps(i8 1, i8 2, i8 3)
+  _tile_dpbf8ps(1, 2, 3);
+}
+
+void test_amx2(void *data) {
+  //CHECK-LABEL: @test_amx2
+  //CHECK: call void @llvm.x86.tdpbhf8ps(i8 1, i8 2, i8 3)
+  _tile_dpbhf8ps(1, 2, 3);
+}
+
+void test_amx3(void *data) {
+  //CHECK-LABEL: @test_amx3
+  //CHECK: call void @llvm.x86.tdphbf8ps(i8 1, i8 2, i8 3)
+  _tile_dphbf8ps(1, 2, 3);
+}
+
+void test_amx4(void *data) {
+  //CHECK-LABEL: @test_amx4
+  //CHECK: call void @llvm.x86.tdphf8ps(i8 1, i8 2, i8 3)
+  _tile_dphf8ps(1, 2, 3);
+}
diff --git a/clang/test/CodeGen/X86/amx_fp8_errors.c b/clang/test/CodeGen/X86/amx_fp8_errors.c
new file mode 100644
index 000000000000..77cbd34905b8
--- /dev/null
+++ b/clang/test/CodeGen/X86/amx_fp8_errors.c
@@ -0,0 +1,10 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +amx-tile -target-feature +amx-fp8 -verify
+
+#include <immintrin.h>
+
+void test_amx(void *data) {
+  _tile_dpbf8ps(4, 3, 3); // expected-error {{tile arguments must refer to different tiles}}
+  _tile_dpbhf8ps(4, 3, 3); // expected-error {{tile arguments must refer to different tiles}}
+  _tile_dphbf8ps(4, 3, 3); // expected-error {{tile arguments must refer to different tiles}}
+  _tile_dphf8ps(4, 3, 3); // expected-error {{tile arguments must refer to different tiles}}
+}
diff --git a/clang/test/CodeGen/X86/amx_fp8_inline_asm.c b/clang/test/CodeGen/X86/amx_fp8_inline_asm.c
new file mode 100644
index 000000000000..49331bd9d368
--- /dev/null
+++ b/clang/test/CodeGen/X86/amx_fp8_inline_asm.c
@@ -0,0 +1,32 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown  -target-feature +amx-fp8 -emit-llvm -o - -Wall -Werror -pedantic | FileCheck %s
+
+void f_tilemul(short a)
+{
+  //CHECK:  call void asm sideeffect "tileloadd 0(%rsi,%r13,4), %tmm0   \0A\09tileloadd 0(%rdx,%r14,4), %tmm6   \0A\09tdpbf8ps %tmm6, %tmm0, %tmm7    \0A\09tilestored %tmm7, 0(%r12,%r15,4) \0A\09", "~{memory},~{tmm0},~{tmm6},~{tmm7},~{dirflag},~{fpsr},~{flags}"()
+  __asm__ volatile ("tileloadd 0(%%rsi,%%r13,4), %%tmm0   \n\t"
+                    "tileloadd 0(%%rdx,%%r14,4), %%tmm6   \n\t"
+                    "tdpbf8ps %%tmm6, %%tmm0, %%tmm7    \n\t"
+                    "tilestored %%tmm7, 0(%%r12,%%r15,4) \n\t"
+          ::: "memory", "tmm0", "tmm6", "tmm7");
+
+  //CHECK:  call void asm sideeffect "tileloadd 0(%rsi,%r13,4), %tmm0   \0A\09tileloadd 0(%rdx,%r14,4), %tmm6   \0A\09tdpbhf8ps %tmm6, %tmm0, %tmm7    \0A\09tilestored %tmm7, 0(%r12,%r15,4) \0A\09", "~{memory},~{tmm0},~{tmm6},~{tmm7},~{dirflag},~{fpsr},~{flags}"()
+  __asm__ volatile ("tileloadd 0(%%rsi,%%r13,4), %%tmm0   \n\t"
+                    "tileloadd 0(%%rdx,%%r14,4), %%tmm6   \n\t"
+                    "tdpbhf8ps %%tmm6, %%tmm0, %%tmm7    \n\t"
+                    "tilestored %%tmm7, 0(%%r12,%%r15,4) \n\t"
+          ::: "memory", "tmm0", "tmm6", "tmm7");
+
+  //CHECK:  call void asm sideeffect "tileloadd 0(%rsi,%r13,4), %tmm0   \0A\09tileloadd 0(%rdx,%r14,4), %tmm6   \0A\09tdphbf8ps %tmm6, %tmm0, %tmm7    \0A\09tilestored %tmm7, 0(%r12,%r15,4) \0A\09", "~{memory},~{tmm0},~{tmm6},~{tmm7},~{dirflag},~{fpsr},~{flags}"()
+  __asm__ volatile ("tileloadd 0(%%rsi,%%r13,4), %%tmm0   \n\t"
+                    "tileloadd 0(%%rdx,%%r14,4), %%tmm6   \n\t"
+                    "tdphbf8ps %%tmm6, %%tmm0, %%tmm7    \n\t"
+                    "tilestored %%tmm7, 0(%%r12,%%r15,4) \n\t"
+          ::: "memory", "tmm0", "tmm6", "tmm7");
+
+  //CHECK:  call void asm sideeffect "tileloadd 0(%rsi,%r13,4), %tmm0   \0A\09tileloadd 0(%rdx,%r14,4), %tmm6   \0A\09tdphf8ps %tmm6, %tmm0, %tmm7    \0A\09tilestored %tmm7, 0(%r12,%r15,4) \0A\09", "~{memory},~{tmm0},~{tmm6},~{tmm7},~{dirflag},~{fpsr},~{flags}"()
+  __asm__ volatile ("tileloadd 0(%%rsi,%%r13,4), %%tmm0   \n\t"
+                    "tileloadd 0(%%rdx,%%r14,4), %%tmm6   \n\t"
+                    "tdphf8ps %%tmm6, %%tmm0, %%tmm7    \n\t"
+                    "tilestored %%tmm7, 0(%%r12,%%r15,4) \n\t"
+          ::: "memory", "tmm0", "tmm6", "tmm7");
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index 0ecca157077f..d1807d26a874 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -5994,6 +5994,23 @@ let TargetPrefix = "x86" in {
                         [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
                          llvm_x86amx_ty, llvm_x86amx_ty,
                          llvm_x86amx_ty], []>;
+
+  def int_x86_tdpbf8ps : ClangBuiltin<"__builtin_ia32_tdpbf8ps">,
+              Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+                        [ImmArg<ArgIndex<0>>,
+                         ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
+  def int_x86_tdpbhf8ps : ClangBuiltin<"__builtin_ia32_tdpbhf8ps">,
+              Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+                        [ImmArg<ArgIndex<0>>,
+                         ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
+  def int_x86_tdphbf8ps : ClangBuiltin<"__builtin_ia32_tdphbf8ps">,
+              Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+                        [ImmArg<ArgIndex<0>>,
+                         ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
+  def int_x86_tdphf8ps : ClangBuiltin<"__builtin_ia32_tdphf8ps">,
+              Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+                        [ImmArg<ArgIndex<0>>,
+                        ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.def b/llvm/include/llvm/TargetParser/X86TargetParser.def
index 073e19f8187c..19e8e0013ef6 100644
--- a/llvm/include/llvm/TargetParser/X86TargetParser.def
+++ b/llvm/include/llvm/TargetParser/X86TargetParser.def
@@ -264,6 +264,7 @@ X86_FEATURE_COMPAT(AVX10_2_512,     "avx10.2-512",            0)
 //FIXME: make MOVRS _COMPAT defined when gcc landed relate patch.
 X86_FEATURE       (MOVRS,           "movrs")
 X86_FEATURE       (ZU,              "zu")
+X86_FEATURE       (AMX_FP8,         "amx-fp8")
 // These features aren't really CPU features, but the frontend can set them.
 X86_FEATURE       (RETPOLINE_EXTERNAL_THUNK,    "retpoline-external-thunk")
 X86_FEATURE       (RETPOLINE_INDIRECT_BRANCHES, "retpoline-indirect-branches")
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 6bedf9e1d13a..c7882acc044e 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -270,6 +270,9 @@ def FeatureAMXFP16     : SubtargetFeature<"amx-fp16", "HasAMXFP16", "true",
 def FeatureAMXCOMPLEX : SubtargetFeature<"amx-complex", "HasAMXCOMPLEX", "true",
                                          "Support AMX-COMPLEX instructions",
                                          [FeatureAMXTILE]>;
+def FeatureAMXFP8 : SubtargetFeature<"amx-fp8", "HasAMXFP8", "true",
+                                     "Support AMX-FP8 instructions",
+                                     [FeatureAMXTILE]>;
 def FeatureCMPCCXADD : SubtargetFeature<"cmpccxadd", "HasCMPCCXADD", "true",
                                         "Support CMPCCXADD instructions">;
 def FeatureRAOINT : SubtargetFeature<"raoint", "HasRAOINT", "true",
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 22cba69af41f..58598fefe0e7 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -37420,7 +37420,11 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   case X86::PTDPBUSD:
   case X86::PTDPBUUD:
   case X86::PTDPBF16PS:
-  case X86::PTDPFP16PS: {
+  case X86::PTDPFP16PS:
+  case X86::PTDPBF8PS:
+  case X86::PTDPBHF8PS:
+  case X86::PTDPHBF8PS:
+  case X86::PTDPHF8PS: {
     unsigned Opc;
     switch (MI.getOpcode()) {
     // clang-format off
@@ -37431,6 +37435,10 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     case X86::PTDPBUUD: Opc = X86::TDPBUUD; break;
     case X86::PTDPBF16PS: Opc = X86::TDPBF16PS; break;
     case X86::PTDPFP16PS: Opc = X86::TDPFP16PS; break;
+    case X86::PTDPBF8PS: Opc = X86::TDPBF8PS; break;
+    case X86::PTDPBHF8PS: Opc = X86::TDPBHF8PS; break;
+    case X86::PTDPHBF8PS: Opc = X86::TDPHBF8PS; break;
+    case X86::PTDPHF8PS: Opc = X86::TDPHF8PS; break;
     // clang-format on
     }
 
diff --git a/llvm/lib/Target/X86/X86InstrAMX.td b/llvm/lib/Target/X86/X86InstrAMX.td
index 99deacc811a1..202232ccb8bc 100644
--- a/llvm/lib/Target/X86/X86InstrAMX.td
+++ b/llvm/lib/Target/X86/X86InstrAMX.td
@@ -267,3 +267,42 @@ let Predicates = [HasAMXCOMPLEX, In64BitMode] in {
     }
   } // SchedRW = [WriteSystem]
 }
+
+// AMX-FP8
+let Predicates = [HasAMXFP8, In64BitMode] in {
+  let SchedRW = [WriteSystem] in {
+    let Constraints = "$src1 = $dst" in {
+      class AMX_FP8_BASE<bits<8> Opcode, string Opstr> :
+        I<Opcode, MRMSrcReg4VOp3, (outs TILE:$dst),
+          (ins TILE:$src1, TILE:$src2, TILE:$src3),
+          !strconcat(Opstr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
+          []>, VEX, VVVV;
+    }
+
+    def TDPBF8PS : AMX_FP8_BASE<0xfd, "tdpbf8ps">, T_MAP5, PS;
+    def TDPBHF8PS : AMX_FP8_BASE<0xfd, "tdpbhf8ps">, T_MAP5, XD;
+    def TDPHBF8PS : AMX_FP8_BASE<0xfd, "tdphbf8ps">, T_MAP5, XS;
+    def TDPHF8PS : AMX_FP8_BASE<0xfd, "tdphf8ps">, T_MAP5, PD;
+
+    let usesCustomInserter = 1 in {
+      // Pseudo instructions, using immediates instead of tile registers.
+      // To be translated to the actual instructions in X86ISelLowering.cpp
+      def PTDPBF8PS : PseudoI<(outs),
+                              (ins u8imm:$src1, u8imm:$src2, u8imm:$src3),
+                              [(int_x86_tdpbf8ps timm:$src1, timm:$src2,
+                                timm:$src3)]>;
+      def PTDPBHF8PS : PseudoI<(outs),
+                               (ins u8imm:$src1, u8imm:$src2, u8imm:$src3),
+                               [(int_x86_tdpbhf8ps timm:$src1, timm:$src2,
+                                 timm:$src3)]>;
+      def PTDPHBF8PS : PseudoI<(outs),
+                               (ins u8imm:$src1, u8imm:$src2, u8imm:$src3),
+                               [(int_x86_tdphbf8ps timm:$src1, timm:$src2,
+                                 timm:$src3)]>;
+      def PTDPHF8PS : PseudoI<(outs),
+                              (ins u8imm:$src1, u8imm:$src2, u8imm:$src3),
+                              [(int_x86_tdphf8ps timm:$src1, timm:$src2,
+                                timm:$src3)]>;
+    }
+  }
+}
diff --git a/llvm/lib/Target/X86/X86InstrPredicates.td b/llvm/lib/Target/X86/X86InstrPredicates.td
index 7fb566fba518..5b659d3b072d 100644
--- a/llvm/lib/Target/X86/X86InstrPredicates.td
+++ b/llvm/lib/Target/X86/X86InstrPredicates.td
@@ -183,6 +183,7 @@ def HasAMXTILE   : Predicate<"Subtarget->hasAMXTILE()">;
 def HasAMXBF16   : Predicate<"Subtarget->hasAMXBF16()">;
 def HasAMXINT8   : Predicate<"Subtarget->hasAMXINT8()">;
 def HasAMXCOMPLEX : Predicate<"Subtarget->hasAMXCOMPLEX()">;
+def HasAMXFP8    : Predicate<"Subtarget->hasAMXFP8()">;
 def HasUINTR     : Predicate<"Subtarget->hasUINTR()">;
 def HasUSERMSR   : Predicate<"Subtarget->hasUSERMSR()">;
 def HasCRC32     : Predicate<"Subtarget->hasCRC32()">;
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index 5c4e3a9dc52b..fd34a276cf3c 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -1876,6 +1876,10 @@ const StringMap<bool> sys::getHostCPUFeatures() {
       MaxLevel >= 0x19 && !getX86CpuIDAndInfo(0x19, &EAX, &EBX, &ECX, &EDX);
   Features["widekl"] = HasLeaf7 && HasLeaf19 && ((EBX >> 2) & 1);
 
+  bool HasLeaf1E = MaxLevel >= 0x1e &&
+                   !getX86CpuIDAndInfoEx(0x1e, 0x1, &EAX, &EBX, &ECX, &EDX);
+  Features["amx-fp8"] = HasLeaf1E && ((EAX >> 4) & 1) && HasAMXSave;
+
   bool HasLeaf24 =
       MaxLevel >= 0x24 && !getX86CpuIDAndInfo(0x24, &EAX, &EBX, &ECX, &EDX);
 
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index 586df5748aa8..7d60b81d4bb1 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -598,6 +598,7 @@ constexpr FeatureBitset ImpliedFeaturesAMX_BF16 = FeatureAMX_TILE;
 constexpr FeatureBitset ImpliedFeaturesAMX_FP16 = FeatureAMX_TILE;
 constexpr FeatureBitset ImpliedFeaturesAMX_INT8 = FeatureAMX_TILE;
 constexpr FeatureBitset ImpliedFeaturesAMX_COMPLEX = FeatureAMX_TILE;
+constexpr FeatureBitset ImpliedFeaturesAMX_FP8 = FeatureAMX_TILE;
 constexpr FeatureBitset ImpliedFeaturesHRESET = {};
 
 constexpr FeatureBitset ImpliedFeaturesPREFETCHI = {};
diff --git a/llvm/test/CodeGen/X86/amx_fp8_intrinsics.ll b/llvm/test/CodeGen/X86/amx_fp8_intrinsics.ll
new file mode 100644
index 000000000000..f5d3f6ec9ec2
--- /dev/null
+++ b/llvm/test/CodeGen/X86/amx_fp8_intrinsics.ll
@@ -0,0 +1,20 @@
+; RUN: llc < %s -O0 -mtriple=x86_64-unknown-unknown -mattr=+amx-tile,+amx-fp8 | FileCheck %s
+
+; CHECK-LABEL: test_amx:
+; CHECK:       # %bb.0:
+; CHECK:    tdpbf8ps        %tmm3, %tmm2, %tmm1
+; CHECK:    tdpbhf8ps        %tmm3, %tmm2, %tmm1
+; CHECK:    tdphbf8ps        %tmm3, %tmm2, %tmm1
+; CHECK:    tdphf8ps        %tmm3, %tmm2, %tmm1
+
+define void @test_amx(){
+call void @llvm.x86.tdpbf8ps(i8 1, i8 2, i8 3)
+call void @llvm.x86.tdpbhf8ps(i8 1, i8 2, i8 3)
+call void @llvm.x86.tdphbf8ps(i8 1, i8 2, i8 3)
+call void @llvm.x86.tdphf8ps(i8 1, i8 2, i8 3)
+ret void
+}
+declare void @llvm.x86.tdpbf8ps(i8 %tile0, i8 %tile1, i8 %tile2)
+declare void @llvm.x86.tdpbhf8ps(i8 %tile0, i8 %tile1, i8 %tile2)
+declare void @llvm.x86.tdphbf8ps(i8 %tile0, i8 %tile1, i8 %tile2)
+declare void @llvm.x86.tdphf8ps(i8 %tile0, i8 %tile1, i8 %tile2)
diff --git a/llvm/test/MC/Disassembler/X86/AMX/amx-fp8.txt b/llvm/test/MC/Disassembler/X86/AMX/amx-fp8.txt
new file mode 100644
index 000000000000..e714a52d2c31
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/AMX/amx-fp8.txt
@@ -0,0 +1,34 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64 | FileCheck %s --check-prefixes=ATT
+# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL
+
+# ATT:   tdpbf8ps %tmm4, %tmm5, %tmm6
+# INTEL: tdpbf8ps tmm6, tmm5, tmm4
+0xc4,0xe5,0x58,0xfd,0xf5
+
+# ATT:   tdpbf8ps %tmm1, %tmm2, %tmm3
+# INTEL: tdpbf8ps tmm3, tmm2, tmm1
+0xc4,0xe5,0x70,0xfd,0xda
+
+# ATT:   tdpbhf8ps %tmm4, %tmm5, %tmm6
+# INTEL: tdpbhf8ps tmm6, tmm5, tmm4
+0xc4,0xe5,0x5b,0xfd,0xf5
+
+# ATT:   tdpbhf8ps %tmm1, %tmm2, %tmm3
+# INTEL: tdpbhf8ps tmm3, tmm2, tmm1
+0xc4,0xe5,0x73,0xfd,0xda
+
+# ATT:   tdphbf8ps %tmm4, %tmm5, %tmm6
+# INTEL: tdphbf8ps tmm6, tmm5, tmm4
+0xc4,0xe5,0x5a,0xfd,0xf5
+
+# ATT:   tdphbf8ps %tmm1, %tmm2, %tmm3
+# INTEL: tdphbf8ps tmm3, tmm2, tmm1
+0xc4,0xe5,0x72,0xfd,0xda
+
+# ATT:   tdphf8ps %tmm4, %tmm5, %tmm6
+# INTEL: tdphf8ps tmm6, tmm5, tmm4
+0xc4,0xe5,0x59,0xfd,0xf5
+
+# ATT:   tdphf8ps %tmm1, %tmm2, %tmm3
+# INTEL: tdphf8ps tmm3, tmm2, tmm1
+0xc4,0xe5,0x71,0xfd,0xda
diff --git a/llvm/test/MC/X86/AMX/amx-fp8-att.s b/llvm/test/MC/X86/AMX/amx-fp8-att.s
new file mode 100644
index 000000000000..904539ec4917
--- /dev/null
+++ b/llvm/test/MC/X86/AMX/amx-fp8-att.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple x86_64 --show-encoding %s | FileCheck %s
+
+// CHECK: tdpbf8ps %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe5,0x58,0xfd,0xf5]
+          tdpbf8ps %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbf8ps %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe5,0x70,0xfd,0xda]
+          tdpbf8ps %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbhf8ps %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe5,0x5b,0xfd,0xf5]
+          tdpbhf8ps %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbhf8ps %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe5,0x73,0xfd,0xda]
+          tdpbhf8ps %tmm1, %tmm2, %tmm3
+
+// CHECK: tdphbf8ps %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe5,0x5a,0xfd,0xf5]
+          tdphbf8ps %tmm4, %tmm5, %tmm6
+
+// CHECK: tdphbf8ps %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe5,0x72,0xfd,0xda]
+          tdphbf8ps %tmm1, %tmm2, %tmm3
+
+// CHECK: tdphf8ps %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe5,0x59,0xfd,0xf5]
+          tdphf8ps %tmm4, %tmm5, %tmm6
+
+// CHECK: tdphf8ps %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe5,0x71,0xfd,0xda]
+          tdphf8ps %tmm1, %tmm2, %tmm3
diff --git a/llvm/test/MC/X86/AMX/amx-fp8-intel.s b/llvm/test/MC/X86/AMX/amx-fp8-intel.s
new file mode 100644
index 000000000000..4191ae6f5cd1
--- /dev/null
+++ b/llvm/test/MC/X86/AMX/amx-fp8-intel.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple x86_64 -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
+
+// CHECK: tdpbf8ps tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe5,0x58,0xfd,0xf5]
+          tdpbf8ps tmm6, tmm5, tmm4
+
+// CHECK: tdpbf8ps tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe5,0x70,0xfd,0xda]
+          tdpbf8ps tmm3, tmm2, tmm1
+
+// CHECK: tdpbhf8ps tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe5,0x5b,0xfd,0xf5]
+          tdpbhf8ps tmm6, tmm5, tmm4
+
+// CHECK: tdpbhf8ps tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe5,0x73,0xfd,0xda]
+          tdpbhf8ps tmm3, tmm2, tmm1
+
+// CHECK: tdphbf8ps tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe5,0x5a,0xfd,0xf5]
+          tdphbf8ps tmm6, tmm5, tmm4
+
+// CHECK: tdphbf8ps tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe5,0x72,0xfd,0xda]
+          tdphbf8ps tmm3, tmm2, tmm1
+
+// CHECK: tdphf8ps tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe5,0x59,0xfd,0xf5]
+          tdphf8ps tmm6, tmm5, tmm4
+
+// CHECK: tdphf8ps tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe5,0x71,0xfd,0xda]
+          tdphf8ps tmm3, tmm2, tmm1
-- 
GitLab


From f0bae562dc3b30e55e5f92e698adf291e024d9a5 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 30 Oct 2024 19:15:51 -0700
Subject: [PATCH 678/693] [GISel] Return const APInt & from
 getIConstantFromReg. NFC (#114320)

This matches what the call to ConstantInt::getValue() returns. Let the
caller make a copy if needed.
---
 llvm/include/llvm/CodeGen/GlobalISel/Utils.h | 2 +-
 llvm/lib/CodeGen/GlobalISel/Utils.cpp        | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index 4016247376c4..37653631cc23 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -180,7 +180,7 @@ std::optional<int64_t> getIConstantVRegSExtVal(Register VReg,
                                                const MachineRegisterInfo &MRI);
 
 /// \p VReg is defined by a G_CONSTANT, return the corresponding value.
-APInt getIConstantFromReg(Register VReg, const MachineRegisterInfo &MRI);
+const APInt &getIConstantFromReg(Register VReg, const MachineRegisterInfo &MRI);
 
 /// Simple struct used to hold a constant integer value and a virtual
 /// register.
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index dcbbb0871a84..5cee07461d7e 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -287,7 +287,8 @@ std::optional<APInt> llvm::getIConstantVRegVal(Register VReg,
   return ValAndVReg->Value;
 }
 
-APInt llvm::getIConstantFromReg(Register Reg, const MachineRegisterInfo &MRI) {
+const APInt &llvm::getIConstantFromReg(Register Reg,
+                                       const MachineRegisterInfo &MRI) {
   MachineInstr *Const = MRI.getVRegDef(Reg);
   assert((Const && Const->getOpcode() == TargetOpcode::G_CONSTANT) &&
          "expected a G_CONSTANT on Reg");
-- 
GitLab


From 55dbacbf0746afa243224b58d4b5f86d3b54774e Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 30 Oct 2024 19:16:23 -0700
Subject: [PATCH 679/693] [RISCV] Remove RISCVISD::VFCVT_X(U)_F_VL by using
 VFCVT_RM_X(U)_F_VL with DYN rounding mode. NFC (#114306)

---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 30 ++++-----
 llvm/lib/Target/RISCV/RISCVISelLowering.h     |  2 -
 .../Target/RISCV/RISCVInstrInfoVVLPatterns.td | 67 -------------------
 3 files changed, 15 insertions(+), 84 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e7898747fcce..0b5c46f2c370 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -3030,6 +3030,7 @@ static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc) {
   case ISD::VP_FROUND:
     return RISCVFPRndMode::RMM;
   case ISD::FRINT:
+  case ISD::VP_FRINT:
     return RISCVFPRndMode::DYN;
   }
 
@@ -3101,6 +3102,8 @@ lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
   switch (Op.getOpcode()) {
   default:
     llvm_unreachable("Unexpected opcode");
+  case ISD::FRINT:
+  case ISD::VP_FRINT:
   case ISD::FCEIL:
   case ISD::VP_FCEIL:
   case ISD::FFLOOR:
@@ -3120,10 +3123,6 @@ lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
     Truncated = DAG.getNode(RISCVISD::VFCVT_RTZ_X_F_VL, DL, IntVT, Src,
                             Mask, VL);
     break;
-  case ISD::FRINT:
-  case ISD::VP_FRINT:
-    Truncated = DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, IntVT, Src, Mask, VL);
-    break;
   case ISD::FNEARBYINT:
   case ISD::VP_FNEARBYINT:
     Truncated = DAG.getNode(RISCVISD::VFROUND_NOEXCEPT_VL, DL, ContainerVT, Src,
@@ -3294,8 +3293,10 @@ static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG,
   }
 
   auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
-  SDValue Truncated =
-      DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, ContainerVT, Src, Mask, VL);
+  SDValue Truncated = DAG.getNode(
+      RISCVISD::VFCVT_RM_X_F_VL, DL, ContainerVT, Src, Mask,
+      DAG.getTargetConstant(RISCVFPRndMode::DYN, DL, Subtarget.getXLenVT()),
+      VL);
 
   if (!VT.isFixedLengthVector())
     return Truncated;
@@ -6166,7 +6167,7 @@ static unsigned getRISCVVLOp(SDValue Op) {
   case ISD::VP_LRINT:
   case ISD::LLRINT:
   case ISD::VP_LLRINT:
-    return RISCVISD::VFCVT_X_F_VL;
+    return RISCVISD::VFCVT_RM_X_F_VL;
   }
   // clang-format on
 #undef OP_CASE
@@ -6179,7 +6180,7 @@ static bool hasPassthruOp(unsigned Opcode) {
          Opcode <= RISCVISD::LAST_RISCV_STRICTFP_OPCODE &&
          "not a RISC-V target specific op");
   static_assert(RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP ==
-                    130 &&
+                    128 &&
                 RISCVISD::LAST_RISCV_STRICTFP_OPCODE -
                         ISD::FIRST_TARGET_STRICTFP_OPCODE ==
                     21 &&
@@ -6205,7 +6206,7 @@ static bool hasMaskOp(unsigned Opcode) {
          Opcode <= RISCVISD::LAST_RISCV_STRICTFP_OPCODE &&
          "not a RISC-V target specific op");
   static_assert(RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP ==
-                    130 &&
+                    128 &&
                 RISCVISD::LAST_RISCV_STRICTFP_OPCODE -
                         ISD::FIRST_TARGET_STRICTFP_OPCODE ==
                     21 &&
@@ -11545,6 +11546,11 @@ SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG) const {
         }
       }
     }
+    // VFCVT_RM_X_F_VL requires a rounding mode to be injected before the VL.
+    if (RISCVISDOpc == RISCVISD::VFCVT_RM_X_F_VL &&
+        ISD::getVPExplicitVectorLengthIdx(Op.getOpcode()) == OpIdx.index())
+      Ops.push_back(DAG.getTargetConstant(RISCVFPRndMode::DYN, DL,
+                                          Subtarget.getXLenVT()));
     // Pass through operands which aren't fixed-length vectors.
     if (!V.getValueType().isFixedLengthVector()) {
       Ops.push_back(V);
@@ -15705,10 +15711,6 @@ static SDValue performFP_TO_INTCombine(SDNode *N,
       unsigned Opc =
           IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
       FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask, VL);
-    } else if (FRM == RISCVFPRndMode::DYN) {
-      unsigned Opc =
-          IsSigned ? RISCVISD::VFCVT_X_F_VL : RISCVISD::VFCVT_XU_F_VL;
-      FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask, VL);
     } else {
       unsigned Opc =
           IsSigned ? RISCVISD::VFCVT_RM_X_F_VL : RISCVISD::VFCVT_RM_XU_F_VL;
@@ -20272,8 +20274,6 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(VFCVT_RTZ_XU_F_VL)
   NODE_NAME_CASE(VFCVT_RM_X_F_VL)
   NODE_NAME_CASE(VFCVT_RM_XU_F_VL)
-  NODE_NAME_CASE(VFCVT_X_F_VL)
-  NODE_NAME_CASE(VFCVT_XU_F_VL)
   NODE_NAME_CASE(VFROUND_NOEXCEPT_VL)
   NODE_NAME_CASE(SINT_TO_FP_VL)
   NODE_NAME_CASE(UINT_TO_FP_VL)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 0b07ad7d7a42..9ae70d257fa4 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -307,8 +307,6 @@ enum NodeType : unsigned {
   FCOPYSIGN_VL, // Has a passthru operand
   VFCVT_RTZ_X_F_VL,
   VFCVT_RTZ_XU_F_VL,
-  VFCVT_X_F_VL,
-  VFCVT_XU_F_VL,
   VFROUND_NOEXCEPT_VL,
   VFCVT_RM_X_F_VL,  // Has a rounding mode operand.
   VFCVT_RM_XU_F_VL, // Has a rounding mode operand.
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index 33e1ed120cd0..9d434cef5a96 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -270,8 +270,6 @@ def SDT_RISCVSETCCOP_VL : SDTypeProfile<1, 6, [
   SDTCisSameAs<0, 5>, SDTCisVT<6, XLenVT>]>;
 
 // Float -> Int
-def riscv_vfcvt_xu_f_vl : SDNode<"RISCVISD::VFCVT_XU_F_VL", SDT_RISCVFP2IOp_VL>;
-def riscv_vfcvt_x_f_vl : SDNode<"RISCVISD::VFCVT_X_F_VL", SDT_RISCVFP2IOp_VL>;
 def riscv_vfcvt_rm_xu_f_vl : SDNode<"RISCVISD::VFCVT_RM_XU_F_VL", SDT_RISCVFP2IOp_RM_VL>;
 def riscv_vfcvt_rm_x_f_vl : SDNode<"RISCVISD::VFCVT_RM_X_F_VL", SDT_RISCVFP2IOp_RM_VL>;
 
@@ -1206,24 +1204,6 @@ multiclass VPatConvertFP2IVL_V<SDPatternOperator vop, string instruction_name> {
   }
 }
 
-multiclass VPatConvertFP2IVL_V_RM<SDPatternOperator vop, string instruction_name> {
-  foreach fvti = AllFloatVectors in {
-    defvar ivti = GetIntVTypeInfo<fvti>.Vti;
-    let Predicates = !listconcat(GetVTypePredicates<fvti>.Predicates,
-                                 GetVTypePredicates<ivti>.Predicates) in
-    def : Pat<(ivti.Vector (vop (fvti.Vector fvti.RegClass:$rs1),
-                                (fvti.Mask V0),
-                                VLOpFrag)),
-              (!cast<Instruction>(instruction_name#"_"#ivti.LMul.MX#"_MASK")
-                  (ivti.Vector (IMPLICIT_DEF)), fvti.RegClass:$rs1,
-                  (fvti.Mask V0),
-                  // Value to indicate no rounding mode change in
-                  // RISCVInsertReadWriteCSR
-                  FRM_DYN,
-                  GPR:$vl, ivti.Log2SEW, TA_MA)>;
-  }
-}
-
 
 multiclass VPatConvertFP2I_RM_VL_V<SDPatternOperator vop, string instruction_name> {
   foreach fvti = AllFloatVectors in {
@@ -1289,25 +1269,6 @@ multiclass VPatWConvertFP2IVL_V<SDPatternOperator vop, string instruction_name>
   }
 }
 
-multiclass VPatWConvertFP2IVL_V_RM<SDPatternOperator vop, string instruction_name> {
-  foreach fvtiToFWti = AllWidenableFloatVectors in {
-    defvar fvti = fvtiToFWti.Vti;
-    defvar iwti = GetIntVTypeInfo<fvtiToFWti.Wti>.Vti;
-    let Predicates = !listconcat(GetVTypePredicates<fvti>.Predicates,
-                                 GetVTypePredicates<iwti>.Predicates) in
-    def : Pat<(iwti.Vector (vop (fvti.Vector fvti.RegClass:$rs1),
-                                (fvti.Mask V0),
-                                VLOpFrag)),
-              (!cast<Instruction>(instruction_name#"_"#fvti.LMul.MX#"_MASK")
-                  (iwti.Vector (IMPLICIT_DEF)), fvti.RegClass:$rs1,
-                  (fvti.Mask V0),
-                  // Value to indicate no rounding mode change in
-                  // RISCVInsertReadWriteCSR
-                  FRM_DYN,
-                  GPR:$vl, fvti.Log2SEW, TA_MA)>;
-  }
-}
-
 
 multiclass VPatWConvertFP2I_RM_VL_V<SDNode vop, string instruction_name> {
   foreach fvtiToFWti = AllWidenableFloatVectors in {
@@ -1361,28 +1322,6 @@ multiclass VPatNConvertFP2IVL_W<SDPatternOperator vop,
   }
 }
 
-multiclass VPatNConvertFP2IVL_W_RM<SDPatternOperator vop,
-                                string instruction_name> {
-  // Reuse the same list of types used in the widening nodes, but just swap the
-  // direction of types around so we're converting from Wti -> Vti
-  foreach vtiToWti = AllWidenableIntToFloatVectors in {
-    defvar vti = vtiToWti.Vti;
-    defvar fwti = vtiToWti.Wti;
-    let Predicates = !listconcat(GetVTypePredicates<vti>.Predicates,
-                                 GetVTypePredicates<fwti>.Predicates) in
-    def : Pat<(vti.Vector (vop (fwti.Vector fwti.RegClass:$rs1),
-                               (fwti.Mask V0),
-                               VLOpFrag)),
-              (!cast<Instruction>(instruction_name#"_"#vti.LMul.MX#"_MASK")
-                  (vti.Vector (IMPLICIT_DEF)), fwti.RegClass:$rs1,
-                  (fwti.Mask V0),
-                  // Value to indicate no rounding mode change in
-                  // RISCVInsertReadWriteCSR
-                  FRM_DYN,
-                  GPR:$vl, vti.Log2SEW, TA_MA)>;
-  }
-}
-
 multiclass VPatNConvertFP2I_RM_VL_W<SDNode vop, string instruction_name> {
   foreach vtiToWti = AllWidenableIntToFloatVectors in {
     defvar vti = vtiToWti.Vti;
@@ -2637,8 +2576,6 @@ foreach fvti = AllFloatVectors in {
 }
 
 // 13.17. Vector Single-Width Floating-Point/Integer Type-Convert Instructions
-defm : VPatConvertFP2IVL_V_RM<riscv_vfcvt_xu_f_vl, "PseudoVFCVT_XU_F_V">;
-defm : VPatConvertFP2IVL_V_RM<riscv_vfcvt_x_f_vl, "PseudoVFCVT_X_F_V">;
 defm : VPatConvertFP2I_RM_VL_V<riscv_vfcvt_rm_xu_f_vl, "PseudoVFCVT_XU_F_V">;
 defm : VPatConvertFP2I_RM_VL_V<any_riscv_vfcvt_rm_x_f_vl, "PseudoVFCVT_X_F_V">;
 
@@ -2652,8 +2589,6 @@ defm : VPatConvertI2FP_RM_VL_V<riscv_vfcvt_rm_f_xu_vl, "PseudoVFCVT_F_XU_V">;
 defm : VPatConvertI2FP_RM_VL_V<riscv_vfcvt_rm_f_x_vl, "PseudoVFCVT_F_X_V">;
 
 // 13.18. Widening Floating-Point/Integer Type-Convert Instructions
-defm : VPatWConvertFP2IVL_V_RM<riscv_vfcvt_xu_f_vl, "PseudoVFWCVT_XU_F_V">;
-defm : VPatWConvertFP2IVL_V_RM<riscv_vfcvt_x_f_vl, "PseudoVFWCVT_X_F_V">;
 defm : VPatWConvertFP2I_RM_VL_V<riscv_vfcvt_rm_xu_f_vl, "PseudoVFWCVT_XU_F_V">;
 defm : VPatWConvertFP2I_RM_VL_V<riscv_vfcvt_rm_x_f_vl, "PseudoVFWCVT_X_F_V">;
 
@@ -2694,8 +2629,6 @@ foreach fvtiToFWti = AllWidenableBFloatToFloatVectors in {
 }
 
 // 13.19 Narrowing Floating-Point/Integer Type-Convert Instructions
-defm : VPatNConvertFP2IVL_W_RM<riscv_vfcvt_xu_f_vl, "PseudoVFNCVT_XU_F_W">;
-defm : VPatNConvertFP2IVL_W_RM<riscv_vfcvt_x_f_vl, "PseudoVFNCVT_X_F_W">;
 defm : VPatNConvertFP2I_RM_VL_W<riscv_vfcvt_rm_xu_f_vl, "PseudoVFNCVT_XU_F_W">;
 defm : VPatNConvertFP2I_RM_VL_W<riscv_vfcvt_rm_x_f_vl, "PseudoVFNCVT_X_F_W">;
 
-- 
GitLab


From ccdfd1a182fc718997f21583aea95d321f03c967 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Thu, 31 Oct 2024 02:27:09 +0000
Subject: [PATCH 680/693] [gn build] Port 8127162427c5

---
 llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
index b47189accd13..d6e7d5490ce0 100644
--- a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
@@ -128,6 +128,7 @@ copy("Headers") {
     "ammintrin.h",
     "amxcomplexintrin.h",
     "amxfp16intrin.h",
+    "amxfp8intrin.h",
     "amxintrin.h",
     "arm64intr.h",
     "arm_acle.h",
-- 
GitLab


From 75aaa312ffa9aa044b84bd1b32491937795c110a Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Wed, 30 Oct 2024 20:18:29 -0700
Subject: [PATCH 681/693] [lldb] Fix formatting and whitespace in
 ScriptInterpreterPython (NFC)

---
 .../Python/ScriptInterpreterPython.cpp        | 75 +++++++++----------
 1 file changed, 35 insertions(+), 40 deletions(-)

diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index 44fd05150ebc..7c2b6517468f 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -453,8 +453,9 @@ ScriptInterpreterPythonImpl::ScriptInterpreterPythonImpl(Debugger &debugger)
   // Reloading modules requires a different syntax in Python 2 and Python 3.
   // This provides a consistent syntax no matter what version of Python.
   run_string.Clear();
-  run_string.Printf("run_one_line (%s, 'from importlib import reload as reload_module')",
-                    m_dictionary_name.c_str());
+  run_string.Printf(
+      "run_one_line (%s, 'from importlib import reload as reload_module')",
+      m_dictionary_name.c_str());
   PyRun_SimpleString(run_string.GetData());
 
   // WARNING: temporary code that loads Cocoa formatters - this should be done
@@ -770,21 +771,19 @@ llvm::Expected<unsigned>
 ScriptInterpreterPythonImpl::GetMaxPositionalArgumentsForCallable(
     const llvm::StringRef &callable_name) {
   if (callable_name.empty()) {
-    return llvm::createStringError(
-        llvm::inconvertibleErrorCode(),
-        "called with empty callable name.");
-  }
-  Locker py_lock(this, Locker::AcquireLock |
-                 Locker::InitSession |
-                 Locker::NoSTDIN);
-  auto dict = PythonModule::MainModule()
-      .ResolveName<PythonDictionary>(m_dictionary_name);
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "called with empty callable name.");
+  }
+  Locker py_lock(this,
+                 Locker::AcquireLock | Locker::InitSession | Locker::NoSTDIN);
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      m_dictionary_name);
   auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
       callable_name, dict);
   if (!pfunc.IsAllocated()) {
-    return llvm::createStringError(
-        llvm::inconvertibleErrorCode(),
-        "can't find callable: %s", callable_name.str().c_str());
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "can't find callable: %s",
+                                   callable_name.str().c_str());
   }
   llvm::Expected<PythonCallable::ArgInfo> arg_info = pfunc.GetArgInfo();
   if (!arg_info)
@@ -1266,8 +1265,7 @@ Status ScriptInterpreterPythonImpl::SetBreakpointCommandCallback(
 
 // Set a Python one-liner as the callback for the watchpoint.
 void ScriptInterpreterPythonImpl::SetWatchpointCommandCallback(
-    WatchpointOptions *wp_options, const char *user_input,
-    bool is_callback) {
+    WatchpointOptions *wp_options, const char *user_input, bool is_callback) {
   auto data_up = std::make_unique<WatchpointOptions::CommandData>();
 
   // It's necessary to set both user_source and script_source to the oneliner.
@@ -1293,8 +1291,7 @@ Status ScriptInterpreterPythonImpl::ExportFunctionDefinitionToInterpreter(
   std::string function_def_string(function_def.CopyList());
 
   Status error = ExecuteMultipleLines(
-      function_def_string.c_str(),
-      ExecuteScriptOptions().SetEnableIO(false));
+      function_def_string.c_str(), ExecuteScriptOptions().SetEnableIO(false));
   return error;
 }
 
@@ -2075,7 +2072,8 @@ int ScriptInterpreterPythonImpl::GetIndexOfChildWithName(
   {
     Locker py_lock(this,
                    Locker::AcquireLock | Locker::InitSession | Locker::NoSTDIN);
-    ret_val = SWIGBridge::LLDBSwigPython_GetIndexOfChildWithName(implementor, child_name);
+    ret_val = SWIGBridge::LLDBSwigPython_GetIndexOfChildWithName(implementor,
+                                                                 child_name);
   }
 
   return ret_val;
@@ -2467,7 +2465,8 @@ bool ScriptInterpreterPythonImpl::LoadScriptingModule(
   // the lifetime of the process in which this LLDB framework is living.
   const bool does_contain_executed = ExecuteOneLineWithReturn(
       command_stream.GetData(),
-      ScriptInterpreterPythonImpl::eScriptReturnTypeBool, &does_contain, exc_options);
+      ScriptInterpreterPythonImpl::eScriptReturnTypeBool, &does_contain,
+      exc_options);
 
   const bool was_imported_globally = does_contain_executed && does_contain;
   const bool was_imported_locally =
@@ -2684,7 +2683,7 @@ bool ScriptInterpreterPythonImpl::RunScriptBasedParsedCommand(
       args_arr_sp->AddStringItem(entry.ref());
     }
     StructuredDataImpl args_impl(args_arr_sp);
-    
+
     ret_val = SWIGBridge::LLDBSwigPythonCallParsedCommandObject(
         static_cast<PyObject *>(impl_obj_sp->GetValue()), debugger_sp,
         args_impl, cmd_retobj, exe_ctx_ref_sp);
@@ -2786,8 +2785,7 @@ bool ScriptInterpreterPythonImpl::GetDocumentationForItem(const char *item,
 
   if (ExecuteOneLineWithReturn(
           command, ScriptInterpreter::eScriptReturnTypeCharStrOrNone,
-          &result_ptr,
-          ExecuteScriptOptions().SetEnableIO(false))) {
+          &result_ptr, ExecuteScriptOptions().SetEnableIO(false))) {
     if (result_ptr)
       dest.assign(result_ptr);
     return true;
@@ -2885,7 +2883,7 @@ uint32_t ScriptInterpreterPythonImpl::GetFlagsForCommandObject(
   return result;
 }
 
-StructuredData::ObjectSP 
+StructuredData::ObjectSP
 ScriptInterpreterPythonImpl::GetOptionsForCommandObject(
     StructuredData::GenericSP cmd_obj_sp) {
   StructuredData::ObjectSP result = {};
@@ -2930,10 +2928,10 @@ ScriptInterpreterPythonImpl::GetOptionsForCommandObject(
     PyErr_Clear();
     return {};
   }
-    return py_return.CreateStructuredObject();
+  return py_return.CreateStructuredObject();
 }
 
-StructuredData::ObjectSP 
+StructuredData::ObjectSP
 ScriptInterpreterPythonImpl::GetArgumentsForCommandObject(
     StructuredData::GenericSP cmd_obj_sp) {
   StructuredData::ObjectSP result = {};
@@ -2978,11 +2976,10 @@ ScriptInterpreterPythonImpl::GetArgumentsForCommandObject(
     PyErr_Clear();
     return {};
   }
-    return py_return.CreateStructuredObject();
+  return py_return.CreateStructuredObject();
 }
 
-void 
-ScriptInterpreterPythonImpl::OptionParsingStartedForCommandObject(
+void ScriptInterpreterPythonImpl::OptionParsingStartedForCommandObject(
     StructuredData::GenericSP cmd_obj_sp) {
 
   Locker py_lock(this, Locker::AcquireLock | Locker::NoSTDIN, Locker::FreeLock);
@@ -2990,7 +2987,7 @@ ScriptInterpreterPythonImpl::OptionParsingStartedForCommandObject(
   static char callee_name[] = "option_parsing_started";
 
   if (!cmd_obj_sp)
-    return ;
+    return;
 
   PythonObject implementor(PyRefType::Borrowed,
                            (PyObject *)cmd_obj_sp->GetValue());
@@ -3016,10 +3013,9 @@ ScriptInterpreterPythonImpl::OptionParsingStartedForCommandObject(
   if (PyErr_Occurred())
     PyErr_Clear();
 
-  // option_parsing_starting doesn't return anything, ignore anything but 
+  // option_parsing_starting doesn't return anything, ignore anything but
   // python errors.
-  unwrapOrSetPythonException(
-      As<bool>(implementor.CallMethod(callee_name)));
+  unwrapOrSetPythonException(As<bool>(implementor.CallMethod(callee_name)));
 
   // if it fails, print the error but otherwise go on
   if (PyErr_Occurred()) {
@@ -3029,8 +3025,7 @@ ScriptInterpreterPythonImpl::OptionParsingStartedForCommandObject(
   }
 }
 
-bool
-ScriptInterpreterPythonImpl::SetOptionValueForCommandObject(
+bool ScriptInterpreterPythonImpl::SetOptionValueForCommandObject(
     StructuredData::GenericSP cmd_obj_sp, ExecutionContext *exe_ctx,
     llvm::StringRef long_option, llvm::StringRef value) {
   StructuredData::ObjectSP result = {};
@@ -3065,15 +3060,15 @@ ScriptInterpreterPythonImpl::SetOptionValueForCommandObject(
 
   if (PyErr_Occurred())
     PyErr_Clear();
-    
+
   lldb::ExecutionContextRefSP exe_ctx_ref_sp;
   if (exe_ctx)
     exe_ctx_ref_sp.reset(new ExecutionContextRef(exe_ctx));
   PythonObject ctx_ref_obj = SWIGBridge::ToSWIGWrapper(exe_ctx_ref_sp);
-    
-  bool py_return = unwrapOrSetPythonException(
-      As<bool>(implementor.CallMethod(callee_name, ctx_ref_obj, long_option.str().c_str(), 
-                                      value.str().c_str())));
+
+  bool py_return = unwrapOrSetPythonException(As<bool>(
+      implementor.CallMethod(callee_name, ctx_ref_obj,
+                             long_option.str().c_str(), value.str().c_str())));
 
   // if it fails, print the error but otherwise go on
   if (PyErr_Occurred()) {
-- 
GitLab


From 97788089988a2ace63d717cadbcfe3443f380f9c Mon Sep 17 00:00:00 2001
From: apple-fcloutier <75502309+apple-fcloutier@users.noreply.github.com>
Date: Wed, 30 Oct 2024 20:34:38 -0700
Subject: [PATCH 682/693] [ObjC] Insert method parameters in scope as they are
 parsed (#113745)

Before this change, ParseObjc would call the closing
`MaybeParseAttributes` before it had created Objective-C `ParmVarDecl`
objects (and associated name lookup entries), meaning that you could not
reference Objective-C method parameters in
`__attribute__((diagnose_if))`. This change moves the creation of the
`ParmVarDecl` objects ahead of calling `Sema::ActOnMethodDeclaration` so
that `MaybeParseAttributes` can find them. This is already how it works
for C parameters hanging off of the selector.

This change alone is insufficient to enable `diagnose_if` for
Objective-C methods and effectively is NFC. There will be a follow-up PR
for diagnose_if. This change is still useful for any other work that may
need attributes to reference Objective-C parameters.

rdar://138596211
---
 clang/include/clang/Sema/SemaObjC.h |   6 +-
 clang/lib/Parse/ParseObjc.cpp       |  10 ++-
 clang/lib/Sema/SemaDeclObjC.cpp     | 112 ++++++++++++++--------------
 3 files changed, 69 insertions(+), 59 deletions(-)

diff --git a/clang/include/clang/Sema/SemaObjC.h b/clang/include/clang/Sema/SemaObjC.h
index 1332eb4f4d42..791a7f45b832 100644
--- a/clang/include/clang/Sema/SemaObjC.h
+++ b/clang/include/clang/Sema/SemaObjC.h
@@ -351,6 +351,10 @@ public:
     ParsedAttributesView ArgAttrs;
   };
 
+  ParmVarDecl *ActOnMethodParmDeclaration(Scope *S, ObjCArgInfo &ArgInfo,
+                                          int ParamIndex,
+                                          bool MethodDefinition);
+
   Decl *ActOnMethodDeclaration(
       Scope *S,
       SourceLocation BeginLoc, // location of the + or -.
@@ -359,7 +363,7 @@ public:
       ArrayRef<SourceLocation> SelectorLocs, Selector Sel,
       // optional arguments. The number of types/arguments is obtained
       // from the Sel.getNumArgs().
-      ObjCArgInfo *ArgInfo, DeclaratorChunk::ParamInfo *CParamInfo,
+      ParmVarDecl **ArgInfo, DeclaratorChunk::ParamInfo *CParamInfo,
       unsigned CNumArgs, // c-style args
       const ParsedAttributesView &AttrList, tok::ObjCKeywordKind MethodImplKind,
       bool isVariadic, bool MethodDefinition);
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index 28ccd3061f84..e69fa1524819 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -1454,7 +1454,7 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 
   SmallVector<const IdentifierInfo *, 12> KeyIdents;
   SmallVector<SourceLocation, 12> KeyLocs;
-  SmallVector<SemaObjC::ObjCArgInfo, 12> ArgInfos;
+  SmallVector<ParmVarDecl *, 12> ObjCParamInfo;
   ParseScope PrototypeScope(this, Scope::FunctionPrototypeScope |
                             Scope::FunctionDeclarationScope | Scope::DeclScope);
 
@@ -1495,7 +1495,9 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
     ArgInfo.NameLoc = Tok.getLocation();
     ConsumeToken(); // Eat the identifier.
 
-    ArgInfos.push_back(ArgInfo);
+    ParmVarDecl *Param = Actions.ObjC().ActOnMethodParmDeclaration(
+        getCurScope(), ArgInfo, ObjCParamInfo.size(), MethodDefinition);
+    ObjCParamInfo.push_back(Param);
     KeyIdents.push_back(SelIdent);
     KeyLocs.push_back(selLoc);
 
@@ -1567,8 +1569,8 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
                                                    &KeyIdents[0]);
   Decl *Result = Actions.ObjC().ActOnMethodDeclaration(
       getCurScope(), mLoc, Tok.getLocation(), mType, DSRet, ReturnType, KeyLocs,
-      Sel, &ArgInfos[0], CParamInfo.data(), CParamInfo.size(), methodAttrs,
-      MethodImplKind, isVariadic, MethodDefinition);
+      Sel, ObjCParamInfo.data(), CParamInfo.data(), CParamInfo.size(),
+      methodAttrs, MethodImplKind, isVariadic, MethodDefinition);
 
   PD.complete(Result);
   return Result;
diff --git a/clang/lib/Sema/SemaDeclObjC.cpp b/clang/lib/Sema/SemaDeclObjC.cpp
index 78acfeddb786..3b19c9b21c25 100644
--- a/clang/lib/Sema/SemaDeclObjC.cpp
+++ b/clang/lib/Sema/SemaDeclObjC.cpp
@@ -4720,13 +4720,67 @@ static void checkObjCDirectMethodClashes(Sema &S, ObjCInterfaceDecl *IDecl,
           diagClash(IMD);
 }
 
+ParmVarDecl *SemaObjC::ActOnMethodParmDeclaration(Scope *S,
+                                                  ObjCArgInfo &ArgInfo,
+                                                  int ParamIndex,
+                                                  bool MethodDefinition) {
+  ASTContext &Context = getASTContext();
+  QualType ArgType;
+  TypeSourceInfo *DI;
+
+  if (!ArgInfo.Type) {
+    ArgType = Context.getObjCIdType();
+    DI = nullptr;
+  } else {
+    ArgType = SemaRef.GetTypeFromParser(ArgInfo.Type, &DI);
+  }
+  LookupResult R(SemaRef, ArgInfo.Name, ArgInfo.NameLoc,
+                 Sema::LookupOrdinaryName,
+                 SemaRef.forRedeclarationInCurContext());
+  SemaRef.LookupName(R, S);
+  if (R.isSingleResult()) {
+    NamedDecl *PrevDecl = R.getFoundDecl();
+    if (S->isDeclScope(PrevDecl)) {
+      Diag(ArgInfo.NameLoc,
+           (MethodDefinition ? diag::warn_method_param_redefinition
+                             : diag::warn_method_param_declaration))
+          << ArgInfo.Name;
+      Diag(PrevDecl->getLocation(), diag::note_previous_declaration);
+    }
+  }
+  SourceLocation StartLoc =
+      DI ? DI->getTypeLoc().getBeginLoc() : ArgInfo.NameLoc;
+
+  // Temporarily put parameter variables in the translation unit. This is what
+  // ActOnParamDeclarator does in the case of C arguments to the Objective-C
+  // method too.
+  ParmVarDecl *Param = SemaRef.CheckParameter(
+      Context.getTranslationUnitDecl(), StartLoc, ArgInfo.NameLoc, ArgInfo.Name,
+      ArgType, DI, SC_None);
+  Param->setObjCMethodScopeInfo(ParamIndex);
+  Param->setObjCDeclQualifier(
+      CvtQTToAstBitMask(ArgInfo.DeclSpec.getObjCDeclQualifier()));
+
+  // Apply the attributes to the parameter.
+  SemaRef.ProcessDeclAttributeList(SemaRef.TUScope, Param, ArgInfo.ArgAttrs);
+  SemaRef.AddPragmaAttributes(SemaRef.TUScope, Param);
+  if (Param->hasAttr<BlocksAttr>()) {
+    Diag(Param->getLocation(), diag::err_block_on_nonlocal);
+    Param->setInvalidDecl();
+  }
+
+  S->AddDecl(Param);
+  SemaRef.IdResolver.AddDecl(Param);
+  return Param;
+}
+
 Decl *SemaObjC::ActOnMethodDeclaration(
     Scope *S, SourceLocation MethodLoc, SourceLocation EndLoc,
     tok::TokenKind MethodType, ObjCDeclSpec &ReturnQT, ParsedType ReturnType,
     ArrayRef<SourceLocation> SelectorLocs, Selector Sel,
     // optional arguments. The number of types/arguments is obtained
     // from the Sel.getNumArgs().
-    ObjCArgInfo *ArgInfo, DeclaratorChunk::ParamInfo *CParamInfo,
+    ParmVarDecl **ArgInfo, DeclaratorChunk::ParamInfo *CParamInfo,
     unsigned CNumArgs, // c-style args
     const ParsedAttributesView &AttrList, tok::ObjCKeywordKind MethodDeclKind,
     bool isVariadic, bool MethodDefinition) {
@@ -4768,60 +4822,10 @@ Decl *SemaObjC::ActOnMethodDeclaration(
       HasRelatedResultType);
 
   SmallVector<ParmVarDecl*, 16> Params;
-
-  for (unsigned i = 0, e = Sel.getNumArgs(); i != e; ++i) {
-    QualType ArgType;
-    TypeSourceInfo *DI;
-
-    if (!ArgInfo[i].Type) {
-      ArgType = Context.getObjCIdType();
-      DI = nullptr;
-    } else {
-      ArgType = SemaRef.GetTypeFromParser(ArgInfo[i].Type, &DI);
-    }
-
-    LookupResult R(SemaRef, ArgInfo[i].Name, ArgInfo[i].NameLoc,
-                   Sema::LookupOrdinaryName,
-                   SemaRef.forRedeclarationInCurContext());
-    SemaRef.LookupName(R, S);
-    if (R.isSingleResult()) {
-      NamedDecl *PrevDecl = R.getFoundDecl();
-      if (S->isDeclScope(PrevDecl)) {
-        Diag(ArgInfo[i].NameLoc,
-             (MethodDefinition ? diag::warn_method_param_redefinition
-                               : diag::warn_method_param_declaration))
-          << ArgInfo[i].Name;
-        Diag(PrevDecl->getLocation(),
-             diag::note_previous_declaration);
-      }
-    }
-
-    SourceLocation StartLoc = DI
-      ? DI->getTypeLoc().getBeginLoc()
-      : ArgInfo[i].NameLoc;
-
-    ParmVarDecl *Param =
-        SemaRef.CheckParameter(ObjCMethod, StartLoc, ArgInfo[i].NameLoc,
-                               ArgInfo[i].Name, ArgType, DI, SC_None);
-
-    Param->setObjCMethodScopeInfo(i);
-
-    Param->setObjCDeclQualifier(
-      CvtQTToAstBitMask(ArgInfo[i].DeclSpec.getObjCDeclQualifier()));
-
-    // Apply the attributes to the parameter.
-    SemaRef.ProcessDeclAttributeList(SemaRef.TUScope, Param,
-                                     ArgInfo[i].ArgAttrs);
-    SemaRef.AddPragmaAttributes(SemaRef.TUScope, Param);
+  for (unsigned I = 0; I < Sel.getNumArgs(); ++I) {
+    ParmVarDecl *Param = ArgInfo[I];
+    Param->setDeclContext(ObjCMethod);
     SemaRef.ProcessAPINotes(Param);
-
-    if (Param->hasAttr<BlocksAttr>()) {
-      Diag(Param->getLocation(), diag::err_block_on_nonlocal);
-      Param->setInvalidDecl();
-    }
-    S->AddDecl(Param);
-    SemaRef.IdResolver.AddDecl(Param);
-
     Params.push_back(Param);
   }
 
-- 
GitLab


From 948249d80483a85c2f20b709d977e124473dd10b Mon Sep 17 00:00:00 2001
From: Adam Yang <hanbyang@microsoft.com>
Date: Wed, 30 Oct 2024 20:44:54 -0700
Subject: [PATCH 683/693] Revert "[DXIL] Add GroupMemoryBarrierWithGroupSync
 intrinsic" (#114322)

Reverts llvm/llvm-project#111884
---
 llvm/include/llvm/IR/IntrinsicsDirectX.td     |   2 -
 llvm/lib/Target/DirectX/DXIL.td               |  54 --------
 llvm/lib/Target/DirectX/DXILOpLowering.cpp    |  45 ++-----
 .../group_memory_barrier_with_group_sync.ll   |   8 --
 llvm/utils/TableGen/DXILEmitter.cpp           | 122 ++----------------
 5 files changed, 22 insertions(+), 209 deletions(-)
 delete mode 100644 llvm/test/CodeGen/DirectX/group_memory_barrier_with_group_sync.ll

diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index dada42636899..e30d37f69f78 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -92,6 +92,4 @@ def int_dx_step : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty, L
 def int_dx_splitdouble : DefaultAttrsIntrinsic<[llvm_anyint_ty, LLVMMatchType<0>], 
     [LLVMScalarOrSameVectorWidth<0, llvm_double_ty>], [IntrNoMem]>;
 def int_dx_radians : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
-
-def int_dx_group_memory_barrier_with_group_sync : DefaultAttrsIntrinsic<[], [], []>;
 }
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 263ca50011aa..1e8dc63ffa25 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -294,43 +294,6 @@ class Attributes<Version ver = DXIL1_0, list<DXILAttribute> attrs> {
   list<DXILAttribute> op_attrs = attrs;
 }
 
-class DXILConstant<int value_> {
-  int value = value_;
-}
-
-defset list<DXILConstant> BarrierModes = {
-  def BarrierMode_DeviceMemoryBarrier              : DXILConstant<2>;
-  def BarrierMode_DeviceMemoryBarrierWithGroupSync : DXILConstant<3>;
-  def BarrierMode_GroupMemoryBarrier               : DXILConstant<8>;
-  def BarrierMode_GroupMemoryBarrierWithGroupSync  : DXILConstant<9>;
-  def BarrierMode_AllMemoryBarrier                 : DXILConstant<10>;
-  def BarrierMode_AllMemoryBarrierWithGroupSync    : DXILConstant<11>;
-}
-
-// Intrinsic arg selection
-class Arg {
-  int index = -1;
-  DXILConstant value;
-  bit is_i8 = 0;
-  bit is_i32 = 0;
-}
-class ArgSelect<int index_> : Arg {
-  let index = index_;
-}
-class ArgI32<DXILConstant value_> : Arg {
-  let value = value_;
-  let is_i32 = 1;
-}
-class ArgI8<DXILConstant value_> : Arg {
-  let value = value_;
-  let is_i8 = 1;
-}
-
-class IntrinsicSelect<Intrinsic intrinsic_, list<Arg> args_> {
-  Intrinsic intrinsic = intrinsic_;
-  list<Arg> args = args_;
-}
-
 // Abstraction DXIL Operation
 class DXILOp<int opcode, DXILOpClass opclass> {
   // A short description of the operation
@@ -345,9 +308,6 @@ class DXILOp<int opcode, DXILOpClass opclass> {
   // LLVM Intrinsic DXIL Operation maps to
   Intrinsic LLVMIntrinsic = ?;
 
-  // Non-trivial LLVM Intrinsics DXIL Operation maps to
-  list<IntrinsicSelect> intrinsic_selects = [];
-
   // Result type of the op
   DXILOpParamType result;
 
@@ -869,17 +829,3 @@ def WaveGetLaneIndex : DXILOp<111, waveGetLaneIndex> {
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
-
-def Barrier : DXILOp<80, barrier> {
-  let Doc = "inserts a memory barrier in the shader";
-  let intrinsic_selects = [
-    IntrinsicSelect<
-        int_dx_group_memory_barrier_with_group_sync,
-        [ ArgI32<BarrierMode_GroupMemoryBarrierWithGroupSync> ]>,
-  ];
-
-  let arguments = [Int32Ty];
-  let result = VoidTy;
-  let stages = [Stages<DXIL1_0, [compute, library]>];
-  let attributes = [Attributes<DXIL1_0, []>];
-}
diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index b5cf1654181c..8acc9c1efa08 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -106,43 +106,17 @@ public:
     return false;
   }
 
-  struct ArgSelect {
-    enum class Type {
-      Index,
-      I8,
-      I32,
-    };
-    Type Type = Type::Index;
-    int Value = -1;
-  };
-
-  [[nodiscard]] bool replaceFunctionWithOp(Function &F, dxil::OpCode DXILOp,
-                                           ArrayRef<ArgSelect> ArgSelects) {
+  [[nodiscard]]
+  bool replaceFunctionWithOp(Function &F, dxil::OpCode DXILOp) {
     bool IsVectorArgExpansion = isVectorArgExpansion(F);
     return replaceFunction(F, [&](CallInst *CI) -> Error {
-      OpBuilder.getIRB().SetInsertPoint(CI);
       SmallVector<Value *> Args;
-      if (ArgSelects.size()) {
-        for (const ArgSelect &A : ArgSelects) {
-          switch (A.Type) {
-          case ArgSelect::Type::Index:
-            Args.push_back(CI->getArgOperand(A.Value));
-            break;
-          case ArgSelect::Type::I8:
-            Args.push_back(OpBuilder.getIRB().getInt8((uint8_t)A.Value));
-            break;
-          case ArgSelect::Type::I32:
-            Args.push_back(OpBuilder.getIRB().getInt32(A.Value));
-            break;
-          default:
-            llvm_unreachable("Invalid type of intrinsic arg select.");
-          }
-        }
-      } else if (IsVectorArgExpansion) {
-        Args = argVectorFlatten(CI, OpBuilder.getIRB());
-      } else {
+      OpBuilder.getIRB().SetInsertPoint(CI);
+      if (IsVectorArgExpansion) {
+        SmallVector<Value *> NewArgs = argVectorFlatten(CI, OpBuilder.getIRB());
+        Args.append(NewArgs.begin(), NewArgs.end());
+      } else
         Args.append(CI->arg_begin(), CI->arg_end());
-      }
 
       Expected<CallInst *> OpCall =
           OpBuilder.tryCreateOp(DXILOp, Args, CI->getName(), F.getReturnType());
@@ -609,10 +583,9 @@ public:
       switch (ID) {
       default:
         continue;
-#define DXIL_OP_INTRINSIC(OpCode, Intrin, ...)                                 \
+#define DXIL_OP_INTRINSIC(OpCode, Intrin)                                      \
   case Intrin:                                                                 \
-    HasErrors |=                                                               \
-        replaceFunctionWithOp(F, OpCode, ArrayRef<ArgSelect>{__VA_ARGS__});    \
+    HasErrors |= replaceFunctionWithOp(F, OpCode);                             \
     break;
 #include "DXILOperation.inc"
       case Intrinsic::dx_handle_fromBinding:
diff --git a/llvm/test/CodeGen/DirectX/group_memory_barrier_with_group_sync.ll b/llvm/test/CodeGen/DirectX/group_memory_barrier_with_group_sync.ll
deleted file mode 100644
index baf93d4e177f..000000000000
--- a/llvm/test/CodeGen/DirectX/group_memory_barrier_with_group_sync.ll
+++ /dev/null
@@ -1,8 +0,0 @@
-; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s
-
-define void @test_group_memory_barrier_with_group_sync() {
-entry:
-  ; CHECK: call void @dx.op.barrier(i32 80, i32 9)
-  call void @llvm.dx.group.memory.barrier.with.group.sync()
-  ret void
-}
\ No newline at end of file
diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp
index 8bebe608eece..e74fc00015b4 100644
--- a/llvm/utils/TableGen/DXILEmitter.cpp
+++ b/llvm/utils/TableGen/DXILEmitter.cpp
@@ -32,20 +32,6 @@ using namespace llvm::dxil;
 
 namespace {
 
-struct DXILArgSelect {
-  enum class Type {
-    Index,
-    I32,
-    I8,
-  };
-  Type Type = Type::Index;
-  int Value = -1;
-};
-struct DXILIntrinsicSelect {
-  StringRef Intrinsic;
-  SmallVector<DXILArgSelect, 4> Args;
-};
-
 struct DXILOperationDesc {
   std::string OpName; // name of DXIL operation
   int OpCode;         // ID of DXIL operation
@@ -56,7 +42,8 @@ struct DXILOperationDesc {
   SmallVector<const Record *> OverloadRecs;
   SmallVector<const Record *> StageRecs;
   SmallVector<const Record *> AttrRecs;
-  SmallVector<DXILIntrinsicSelect> IntrinsicSelects;
+  StringRef Intrinsic; // The llvm intrinsic map to OpName. Default is "" which
+                       // means no map exists
   SmallVector<StringRef, 4>
       ShaderStages; // shader stages to which this applies, empty for all.
   int OverloadParamIndex;             // Index of parameter with overload type.
@@ -84,21 +71,6 @@ static void ascendingSortByVersion(std::vector<const Record *> &Recs) {
   });
 }
 
-/// Take a `int_{intrinsic_name}` and return just the intrinsic_name part if
-/// available. Otherwise return the empty string.
-static StringRef GetIntrinsicName(const RecordVal *RV) {
-  if (RV && RV->getValue()) {
-    if (const DefInit *DI = dyn_cast<DefInit>(RV->getValue())) {
-      auto *IntrinsicDef = DI->getDef();
-      auto DefName = IntrinsicDef->getName();
-      assert(DefName.starts_with("int_") && "invalid intrinsic name");
-      // Remove the int_ from intrinsic name.
-      return DefName.substr(4);
-    }
-  }
-  return "";
-}
-
 /// Construct an object using the DXIL Operation records specified
 /// in DXIL.td. This serves as the single source of reference of
 /// the information extracted from the specified Record R, for
@@ -185,63 +157,14 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
                            OpName);
   }
 
-  {
-    DXILIntrinsicSelect IntrSelect;
-    IntrSelect.Intrinsic = GetIntrinsicName(R->getValue("LLVMIntrinsic"));
-    if (IntrSelect.Intrinsic.size())
-      IntrinsicSelects.emplace_back(std::move(IntrSelect));
-  }
-
-  auto IntrinsicSelectRecords = R->getValueAsListOfDefs("intrinsic_selects");
-  if (IntrinsicSelectRecords.size()) {
-    if (IntrinsicSelects.size()) {
-      PrintFatalError(
-          R, Twine("LLVMIntrinsic and intrinsic_selects cannot be both "
-                   "defined for DXIL operation - ") +
-                 OpName);
-    } else {
-      for (const Record *R : IntrinsicSelectRecords) {
-        DXILIntrinsicSelect IntrSelect;
-        IntrSelect.Intrinsic = GetIntrinsicName(R->getValue("intrinsic"));
-        auto Args = R->getValueAsListOfDefs("args");
-        for (const Record *Arg : Args) {
-          bool IsI8 = Arg->getValueAsBit("is_i8");
-          bool IsI32 = Arg->getValueAsBit("is_i32");
-          int Index = Arg->getValueAsInt("index");
-          const Record *ValueRec = Arg->getValueAsOptionalDef("value");
-
-          DXILArgSelect ArgSelect;
-          if (IsI8) {
-            if (!ValueRec) {
-              PrintFatalError(R, Twine("'value' must be defined for i8 "
-                                       "ArgSelect for DXIL operation - ") +
-                                     OpName);
-            }
-            ArgSelect.Type = DXILArgSelect::Type::I8;
-            ArgSelect.Value = ValueRec->getValueAsInt("value");
-          } else if (IsI32) {
-            if (!ValueRec) {
-              PrintFatalError(R, Twine("'value' must be defined for i32 "
-                                       "ArgSelect for DXIL operation - ") +
-                                     OpName);
-            }
-            ArgSelect.Type = DXILArgSelect::Type::I32;
-            ArgSelect.Value = ValueRec->getValueAsInt("value");
-          } else {
-            if (Index < 0) {
-              PrintFatalError(
-                  R, Twine("Index in ArgSelect<index> must be equal to or "
-                           "greater than 0 for DXIL operation - ") +
-                         OpName);
-            }
-            ArgSelect.Type = DXILArgSelect::Type::Index;
-            ArgSelect.Value = Index;
-          }
-
-          IntrSelect.Args.emplace_back(std::move(ArgSelect));
-        }
-        IntrinsicSelects.emplace_back(std::move(IntrSelect));
-      }
+  const RecordVal *RV = R->getValue("LLVMIntrinsic");
+  if (RV && RV->getValue()) {
+    if (const DefInit *DI = dyn_cast<DefInit>(RV->getValue())) {
+      auto *IntrinsicDef = DI->getDef();
+      auto DefName = IntrinsicDef->getName();
+      assert(DefName.starts_with("int_") && "invalid intrinsic name");
+      // Remove the int_ from intrinsic name.
+      Intrinsic = DefName.substr(4);
     }
   }
 }
@@ -454,29 +377,10 @@ static void emitDXILIntrinsicMap(ArrayRef<DXILOperationDesc> Ops,
   OS << "#ifdef DXIL_OP_INTRINSIC\n";
   OS << "\n";
   for (const auto &Op : Ops) {
-    if (Op.IntrinsicSelects.empty()) {
+    if (Op.Intrinsic.empty())
       continue;
-    }
-    for (const DXILIntrinsicSelect &MappedIntr : Op.IntrinsicSelects) {
-      OS << "DXIL_OP_INTRINSIC(dxil::OpCode::" << Op.OpName
-         << ", Intrinsic::" << MappedIntr.Intrinsic;
-      for (const DXILArgSelect &ArgSelect : MappedIntr.Args) {
-        OS << ", (ArgSelect { ";
-        switch (ArgSelect.Type) {
-        case DXILArgSelect::Type::Index:
-          OS << "ArgSelect::Type::Index, ";
-          break;
-        case DXILArgSelect::Type::I8:
-          OS << "ArgSelect::Type::I8, ";
-          break;
-        case DXILArgSelect::Type::I32:
-          OS << "ArgSelect::Type::I32, ";
-          break;
-        }
-        OS << ArgSelect.Value << "})";
-      }
-      OS << ")\n";
-    }
+    OS << "DXIL_OP_INTRINSIC(dxil::OpCode::" << Op.OpName
+       << ", Intrinsic::" << Op.Intrinsic << ")\n";
   }
   OS << "\n";
   OS << "#undef DXIL_OP_INTRINSIC\n";
-- 
GitLab


From 6bf214b7c6d74ec581bc52a9142756a1d1df6df0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Thu, 31 Oct 2024 04:56:41 +0100
Subject: [PATCH 684/693] [GlobalISel][AArch64] Legalize G_INSERT_VECTOR_ELT
 for SVE (#114310)

There are patterns for:
* {nxv2s32, s32, s64},
* {nxv4s16, s16, s64},
* {nxv2s16, s16, s64}
---
 .../llvm/CodeGen/GlobalISel/LegalizerInfo.h   |  20 +
 .../CodeGen/GlobalISel/LegalityPredicates.cpp |  11 +
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |   4 +
 .../GISel/AArch64PostLegalizerLowering.cpp    |  51 ++-
 .../GlobalISel/legalize-vector-insert-elt.mir | 423 ++++++++++++++++++
 5 files changed, 501 insertions(+), 8 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-vector-insert-elt.mir

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
index 6d71c150c8da..6811b37767cb 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
@@ -273,6 +273,11 @@ inline LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type) {
 LegalityPredicate
 typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1,
               std::initializer_list<std::pair<LLT, LLT>> TypesInit);
+/// True iff the given types for the given tuple of type indexes is one of the
+/// specified type tuple.
+LegalityPredicate
+typeTupleInSet(unsigned TypeIdx0, unsigned TypeIdx1, unsigned TypeIdx2,
+               std::initializer_list<std::tuple<LLT, LLT, LLT>> TypesInit);
 /// True iff the given types for the given pair of type indexes is one of the
 /// specified type pairs.
 LegalityPredicate typePairAndMemDescInSet(
@@ -504,6 +509,15 @@ class LegalizeRuleSet {
     using namespace LegalityPredicates;
     return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types));
   }
+
+  LegalizeRuleSet &
+  actionFor(LegalizeAction Action,
+            std::initializer_list<std::tuple<LLT, LLT, LLT>> Types) {
+    using namespace LegalityPredicates;
+    return actionIf(Action,
+                    typeTupleInSet(typeIdx(0), typeIdx(1), typeIdx(2), Types));
+  }
+
   /// Use the given action when type indexes 0 and 1 is any type pair in the
   /// given list.
   /// Action should be an action that requires mutation.
@@ -615,6 +629,12 @@ public:
       return *this;
     return actionFor(LegalizeAction::Legal, Types);
   }
+  LegalizeRuleSet &
+  legalFor(bool Pred, std::initializer_list<std::tuple<LLT, LLT, LLT>> Types) {
+    if (!Pred)
+      return *this;
+    return actionFor(LegalizeAction::Legal, Types);
+  }
   /// The instruction is legal when type index 0 is any type in the given list
   /// and imm index 0 is anything.
   LegalizeRuleSet &legalForTypeWithAnyImm(std::initializer_list<LLT> Types) {
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp
index 8fe48195c610..dc7ed6cbe8b7 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp
@@ -49,6 +49,17 @@ LegalityPredicate LegalityPredicates::typePairInSet(
   };
 }
 
+LegalityPredicate LegalityPredicates::typeTupleInSet(
+    unsigned TypeIdx0, unsigned TypeIdx1, unsigned TypeIdx2,
+    std::initializer_list<std::tuple<LLT, LLT, LLT>> TypesInit) {
+  SmallVector<std::tuple<LLT, LLT, LLT>, 4> Types = TypesInit;
+  return [=](const LegalityQuery &Query) {
+    std::tuple<LLT, LLT, LLT> Match = {
+        Query.Types[TypeIdx0], Query.Types[TypeIdx1], Query.Types[TypeIdx2]};
+    return llvm::is_contained(Types, Match);
+  };
+}
+
 LegalityPredicate LegalityPredicates::typePairAndMemDescInSet(
     unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx,
     std::initializer_list<TypePairAndMemDesc> TypesAndMemDescInit) {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 6024027afaf6..7beda0e92a75 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -978,6 +978,10 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
   getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT)
       .legalIf(
           typeInSet(0, {v16s8, v8s8, v8s16, v4s16, v4s32, v2s32, v2s64, v2p0}))
+      .legalFor(HasSVE, {{nxv16s8, s32, s64},
+                         {nxv8s16, s32, s64},
+                         {nxv4s32, s32, s64},
+                         {nxv2s64, s64, s64}})
       .moreElementsToNextPow2(0)
       .widenVectorEltsToVectorMinSize(0, 64)
       .clampNumElements(0, v8s8, v16s8)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
index b40fe55fdfaf..0bf0a4bf27c4 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
@@ -161,6 +161,8 @@ bool matchREV(MachineInstr &MI, MachineRegisterInfo &MRI,
   Register Dst = MI.getOperand(0).getReg();
   Register Src = MI.getOperand(1).getReg();
   LLT Ty = MRI.getType(Dst);
+  if (Ty.isScalableVector())
+    return false;
   unsigned EltSize = Ty.getScalarSizeInBits();
 
   // Element size for a rev cannot be 64.
@@ -196,7 +198,10 @@ bool matchTRN(MachineInstr &MI, MachineRegisterInfo &MRI,
   unsigned WhichResult;
   ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
   Register Dst = MI.getOperand(0).getReg();
-  unsigned NumElts = MRI.getType(Dst).getNumElements();
+  LLT DstTy = MRI.getType(Dst);
+  if (DstTy.isScalableVector())
+    return false;
+  unsigned NumElts = DstTy.getNumElements();
   if (!isTRNMask(ShuffleMask, NumElts, WhichResult))
     return false;
   unsigned Opc = (WhichResult == 0) ? AArch64::G_TRN1 : AArch64::G_TRN2;
@@ -217,7 +222,10 @@ bool matchUZP(MachineInstr &MI, MachineRegisterInfo &MRI,
   unsigned WhichResult;
   ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
   Register Dst = MI.getOperand(0).getReg();
-  unsigned NumElts = MRI.getType(Dst).getNumElements();
+  LLT DstTy = MRI.getType(Dst);
+  if (DstTy.isScalableVector())
+    return false;
+  unsigned NumElts = DstTy.getNumElements();
   if (!isUZPMask(ShuffleMask, NumElts, WhichResult))
     return false;
   unsigned Opc = (WhichResult == 0) ? AArch64::G_UZP1 : AArch64::G_UZP2;
@@ -233,7 +241,10 @@ bool matchZip(MachineInstr &MI, MachineRegisterInfo &MRI,
   unsigned WhichResult;
   ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
   Register Dst = MI.getOperand(0).getReg();
-  unsigned NumElts = MRI.getType(Dst).getNumElements();
+  LLT DstTy = MRI.getType(Dst);
+  if (DstTy.isScalableVector())
+    return false;
+  unsigned NumElts = DstTy.getNumElements();
   if (!isZIPMask(ShuffleMask, NumElts, WhichResult))
     return false;
   unsigned Opc = (WhichResult == 0) ? AArch64::G_ZIP1 : AArch64::G_ZIP2;
@@ -288,7 +299,10 @@ bool matchDupFromBuildVector(int Lane, MachineInstr &MI,
                              MachineRegisterInfo &MRI,
                              ShuffleVectorPseudo &MatchInfo) {
   assert(Lane >= 0 && "Expected positive lane?");
-  int NumElements = MRI.getType(MI.getOperand(1).getReg()).getNumElements();
+  LLT Op1Ty = MRI.getType(MI.getOperand(1).getReg());
+  if (Op1Ty.isScalableVector())
+    return false;
+  int NumElements = Op1Ty.getNumElements();
   // Test if the LHS is a BUILD_VECTOR. If it is, then we can just reference the
   // lane's definition directly.
   auto *BuildVecMI =
@@ -326,6 +340,8 @@ bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI,
 // Check if an EXT instruction can handle the shuffle mask when the vector
 // sources of the shuffle are the same.
 bool isSingletonExtMask(ArrayRef<int> M, LLT Ty) {
+  if (Ty.isScalableVector())
+    return false;
   unsigned NumElts = Ty.getNumElements();
 
   // Assume that the first shuffle index is not UNDEF.  Fail if it is.
@@ -357,12 +373,17 @@ bool matchEXT(MachineInstr &MI, MachineRegisterInfo &MRI,
   assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
   Register Dst = MI.getOperand(0).getReg();
   LLT DstTy = MRI.getType(Dst);
+  if (DstTy.isScalableVector())
+    return false;
   Register V1 = MI.getOperand(1).getReg();
   Register V2 = MI.getOperand(2).getReg();
   auto Mask = MI.getOperand(3).getShuffleMask();
   uint64_t Imm;
   auto ExtInfo = getExtMask(Mask, DstTy.getNumElements());
-  uint64_t ExtFactor = MRI.getType(V1).getScalarSizeInBits() / 8;
+  LLT V1Ty = MRI.getType(V1);
+  if (V1Ty.isScalableVector())
+    return false;
+  uint64_t ExtFactor = V1Ty.getScalarSizeInBits() / 8;
 
   if (!ExtInfo) {
     if (!getOpcodeDef<GImplicitDef>(V2, MRI) ||
@@ -423,6 +444,8 @@ void applyNonConstInsert(MachineInstr &MI, MachineRegisterInfo &MRI,
 
   Register Offset = Insert.getIndexReg();
   LLT VecTy = MRI.getType(Insert.getReg(0));
+  if (VecTy.isScalableVector())
+    return;
   LLT EltTy = MRI.getType(Insert.getElementReg());
   LLT IdxTy = MRI.getType(Insert.getIndexReg());
 
@@ -473,7 +496,10 @@ bool matchINS(MachineInstr &MI, MachineRegisterInfo &MRI,
   assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
   ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
   Register Dst = MI.getOperand(0).getReg();
-  int NumElts = MRI.getType(Dst).getNumElements();
+  LLT DstTy = MRI.getType(Dst);
+  if (DstTy.isScalableVector())
+    return false;
+  int NumElts = DstTy.getNumElements();
   auto DstIsLeftAndDstLane = isINSMask(ShuffleMask, NumElts);
   if (!DstIsLeftAndDstLane)
     return false;
@@ -522,6 +548,8 @@ bool isVShiftRImm(Register Reg, MachineRegisterInfo &MRI, LLT Ty,
   if (!Cst)
     return false;
   Cnt = *Cst;
+  if (Ty.isScalableVector())
+    return false;
   int64_t ElementBits = Ty.getScalarSizeInBits();
   return Cnt >= 1 && Cnt <= ElementBits;
 }
@@ -698,6 +726,8 @@ bool matchDupLane(MachineInstr &MI, MachineRegisterInfo &MRI,
   Register Src1Reg = MI.getOperand(1).getReg();
   const LLT SrcTy = MRI.getType(Src1Reg);
   const LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
+  if (SrcTy.isScalableVector())
+    return false;
 
   auto LaneIdx = getSplatIndex(MI);
   if (!LaneIdx)
@@ -774,6 +804,8 @@ bool matchScalarizeVectorUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI) {
   auto &Unmerge = cast<GUnmerge>(MI);
   Register Src1Reg = Unmerge.getReg(Unmerge.getNumOperands() - 1);
   const LLT SrcTy = MRI.getType(Src1Reg);
+  if (SrcTy.isScalableVector())
+    return false;
   if (SrcTy.getSizeInBits() != 128 && SrcTy.getSizeInBits() != 64)
     return false;
   return SrcTy.isVector() && !SrcTy.isScalable() &&
@@ -987,7 +1019,10 @@ bool matchLowerVectorFCMP(MachineInstr &MI, MachineRegisterInfo &MRI,
   if (!DstTy.isVector() || !ST.hasNEON())
     return false;
   Register LHS = MI.getOperand(2).getReg();
-  unsigned EltSize = MRI.getType(LHS).getScalarSizeInBits();
+  LLT LHSTy = MRI.getType(LHS);
+  if (LHSTy.isScalableVector())
+    return false;
+  unsigned EltSize = LHSTy.getScalarSizeInBits();
   if (EltSize == 16 && !ST.hasFullFP16())
     return false;
   if (EltSize != 16 && EltSize != 32 && EltSize != 64)
@@ -1183,7 +1218,7 @@ bool matchExtMulToMULL(MachineInstr &MI, MachineRegisterInfo &MRI) {
   MachineInstr *I1 = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI);
   MachineInstr *I2 = getDefIgnoringCopies(MI.getOperand(2).getReg(), MRI);
 
-  if (DstTy.isVector()) {
+  if (DstTy.isFixedVector()) {
     // If the source operands were EXTENDED before, then {U/S}MULL can be used
     unsigned I1Opc = I1->getOpcode();
     unsigned I2Opc = I2->getOpcode();
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vector-insert-elt.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vector-insert-elt.mir
new file mode 100644
index 000000000000..6d24478cbfb3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vector-insert-elt.mir
@@ -0,0 +1,423 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=aarch64-apple-ios -mattr=+sve -aarch64-enable-gisel-sve=1 -global-isel -start-before=legalizer -stop-after=instruction-select %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SELECT
+# RUN: llc -O0 -mtriple=aarch64-apple-ios -mattr=+sve -aarch64-enable-gisel-sve=1 -global-isel -start-before=legalizer -stop-after=regbankselect %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-REGBANK
+# RUN: llc -O0 -mtriple=aarch64-apple-ios -mattr=+sve -aarch64-enable-gisel-sve=1 -global-isel -run-pass=legalizer  %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-LEGAL
+
+---
+name:            test_insert_vector_elt_nxv_16_s8_idx_0
+body:             |
+  bb.1:
+    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_16_s8_idx_0
+    ; CHECK-SELECT: %vec:zpr = COPY $z0
+    ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0
+    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $xzr
+    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
+    ; CHECK-SELECT-NEXT: [[DUP_ZR_B:%[0-9]+]]:zpr = DUP_ZR_B [[COPY]]
+    ; CHECK-SELECT-NEXT: [[INDEX_II_B:%[0-9]+]]:zpr = INDEX_II_B 0, 1, implicit $vg
+    ; CHECK-SELECT-NEXT: [[PTRUE_B:%[0-9]+]]:ppr_3b = PTRUE_B 31, implicit $vg
+    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_B:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_B [[PTRUE_B]], [[INDEX_II_B]], [[DUP_ZR_B]], implicit-def dead $nzcv
+    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_B %vec, [[CMPEQ_PPzZZ_B]], %elt
+    ; CHECK-SELECT-NEXT: $z0 = COPY %result
+    ;
+    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_16_s8_idx_0
+    ; CHECK-REGBANK: %vec:fpr(<vscale x 16 x s8>) = COPY $z0
+    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0
+    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = G_CONSTANT i64 0
+    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 16 x s8>)
+    ;
+    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_16_s8_idx_0
+    ; CHECK-LEGAL: %vec:_(<vscale x 16 x s8>) = COPY $z0
+    ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0
+    ; CHECK-LEGAL-NEXT: %idx:_(s64) = G_CONSTANT i64 0
+    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 16 x s8>)
+    %vec:_(<vscale x 16 x s8>) = COPY $z0
+    %elt:_(s32) = COPY $w0
+    %idx:_(s64) = G_CONSTANT i64 0
+    %result:_(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec(<vscale x 16 x s8>), %elt(s32), %idx(s64)
+    $z0 = COPY %result(<vscale x 16 x s8>)
+...
+---
+name:            test_insert_vector_elt_nxv_16_s8_constant
+body:             |
+  bb.1:
+    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_16_s8_constant
+    ; CHECK-SELECT: %vec:zpr = COPY $z0
+    ; CHECK-SELECT-NEXT: %elt:gpr32common = MOVi32imm 5
+    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0
+    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
+    ; CHECK-SELECT-NEXT: [[DUP_ZR_B:%[0-9]+]]:zpr = DUP_ZR_B [[COPY]]
+    ; CHECK-SELECT-NEXT: [[INDEX_II_B:%[0-9]+]]:zpr = INDEX_II_B 0, 1, implicit $vg
+    ; CHECK-SELECT-NEXT: [[PTRUE_B:%[0-9]+]]:ppr_3b = PTRUE_B 31, implicit $vg
+    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_B:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_B [[PTRUE_B]], [[INDEX_II_B]], [[DUP_ZR_B]], implicit-def dead $nzcv
+    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_B %vec, [[CMPEQ_PPzZZ_B]], %elt
+    ; CHECK-SELECT-NEXT: $z0 = COPY %result
+    ;
+    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_16_s8_constant
+    ; CHECK-REGBANK: %vec:fpr(<vscale x 16 x s8>) = COPY $z0
+    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = G_CONSTANT i32 5
+    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
+    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 16 x s8>)
+    ;
+    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_16_s8_constant
+    ; CHECK-LEGAL: %vec:_(<vscale x 16 x s8>) = COPY $z0
+    ; CHECK-LEGAL-NEXT: %elt:_(s32) = G_CONSTANT i32 5
+    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
+    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 16 x s8>)
+    %vec:_(<vscale x 16 x s8>) = COPY $z0
+    %elt:_(s32) = G_CONSTANT i32 5
+    %idx:_(s64) = COPY $x0
+    %result:_(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec(<vscale x 16 x s8>), %elt(s32), %idx(s64)
+    $z0 = COPY %result(<vscale x 16 x s8>)
+...
+---
+name:            test_insert_vector_elt_nxv_16_s8
+body:             |
+  bb.1:
+    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_16_s8
+    ; CHECK-SELECT: %vec:zpr = COPY $z0
+    ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0
+    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0
+    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
+    ; CHECK-SELECT-NEXT: [[DUP_ZR_B:%[0-9]+]]:zpr = DUP_ZR_B [[COPY]]
+    ; CHECK-SELECT-NEXT: [[INDEX_II_B:%[0-9]+]]:zpr = INDEX_II_B 0, 1, implicit $vg
+    ; CHECK-SELECT-NEXT: [[PTRUE_B:%[0-9]+]]:ppr_3b = PTRUE_B 31, implicit $vg
+    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_B:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_B [[PTRUE_B]], [[INDEX_II_B]], [[DUP_ZR_B]], implicit-def dead $nzcv
+    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_B %vec, [[CMPEQ_PPzZZ_B]], %elt
+    ; CHECK-SELECT-NEXT: $z0 = COPY %result
+    ;
+    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_16_s8
+    ; CHECK-REGBANK: %vec:fpr(<vscale x 16 x s8>) = COPY $z0
+    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0
+    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
+    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 16 x s8>)
+    ;
+    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_16_s8
+    ; CHECK-LEGAL: %vec:_(<vscale x 16 x s8>) = COPY $z0
+    ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0
+    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
+    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 16 x s8>)
+    %vec:_(<vscale x 16 x s8>) = COPY $z0
+    %elt:_(s32) = COPY $w0
+    %idx:_(s64) = COPY $x0
+    %result:_(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec(<vscale x 16 x s8>), %elt(s32), %idx(s64)
+    $z0 = COPY %result(<vscale x 16 x s8>)
+...
+---
+name:            test_insert_vector_elt_nxv_8_s16_idx_0
+body:             |
+  bb.1:
+    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_8_s16_idx_0
+    ; CHECK-SELECT: %vec:zpr = COPY $z0
+    ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0
+    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $xzr
+    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
+    ; CHECK-SELECT-NEXT: [[DUP_ZR_H:%[0-9]+]]:zpr = DUP_ZR_H [[COPY]]
+    ; CHECK-SELECT-NEXT: [[INDEX_II_H:%[0-9]+]]:zpr = INDEX_II_H 0, 1, implicit $vg
+    ; CHECK-SELECT-NEXT: [[PTRUE_H:%[0-9]+]]:ppr_3b = PTRUE_H 31, implicit $vg
+    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_H:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_H [[PTRUE_H]], [[INDEX_II_H]], [[DUP_ZR_H]], implicit-def dead $nzcv
+    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_H %vec, [[CMPEQ_PPzZZ_H]], %elt
+    ; CHECK-SELECT-NEXT: $z0 = COPY %result
+    ;
+    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_8_s16_idx_0
+    ; CHECK-REGBANK: %vec:fpr(<vscale x 8 x s16>) = COPY $z0
+    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0
+    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = G_CONSTANT i64 0
+    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 8 x s16>)
+    ;
+    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_8_s16_idx_0
+    ; CHECK-LEGAL: %vec:_(<vscale x 8 x s16>) = COPY $z0
+    ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0
+    ; CHECK-LEGAL-NEXT: %idx:_(s64) = G_CONSTANT i64 0
+    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 8 x s16>)
+    %vec:_(<vscale x 8 x s16>) = COPY $z0
+    %elt:_(s32) = COPY $w0
+    %idx:_(s64) = G_CONSTANT i64 0
+    %result:_(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec(<vscale x 8 x s16>), %elt(s32), %idx(s64)
+    $z0 = COPY %result(<vscale x 8 x s16>)
+...
+---
+name:            test_insert_vector_elt_nxv_8_s16_constant
+body:             |
+  bb.1:
+    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_8_s16_constant
+    ; CHECK-SELECT: %vec:zpr = COPY $z0
+    ; CHECK-SELECT-NEXT: %elt:gpr32common = MOVi32imm 5
+    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0
+    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
+    ; CHECK-SELECT-NEXT: [[DUP_ZR_H:%[0-9]+]]:zpr = DUP_ZR_H [[COPY]]
+    ; CHECK-SELECT-NEXT: [[INDEX_II_H:%[0-9]+]]:zpr = INDEX_II_H 0, 1, implicit $vg
+    ; CHECK-SELECT-NEXT: [[PTRUE_H:%[0-9]+]]:ppr_3b = PTRUE_H 31, implicit $vg
+    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_H:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_H [[PTRUE_H]], [[INDEX_II_H]], [[DUP_ZR_H]], implicit-def dead $nzcv
+    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_H %vec, [[CMPEQ_PPzZZ_H]], %elt
+    ; CHECK-SELECT-NEXT: $z0 = COPY %result
+    ;
+    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_8_s16_constant
+    ; CHECK-REGBANK: %vec:fpr(<vscale x 8 x s16>) = COPY $z0
+    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = G_CONSTANT i32 5
+    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
+    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 8 x s16>)
+    ;
+    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_8_s16_constant
+    ; CHECK-LEGAL: %vec:_(<vscale x 8 x s16>) = COPY $z0
+    ; CHECK-LEGAL-NEXT: %elt:_(s32) = G_CONSTANT i32 5
+    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
+    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 8 x s16>)
+    %vec:_(<vscale x 8 x s16>) = COPY $z0
+    %elt:_(s32) = G_CONSTANT i32 5
+    %idx:_(s64) = COPY $x0
+    %result:_(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec(<vscale x 8 x s16>), %elt(s32), %idx(s64)
+    $z0 = COPY %result(<vscale x 8 x s16>)
+...
+---
+name:            test_insert_vector_elt_nxv_8_s16
+body:             |
+  bb.1:
+    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_8_s16
+    ; CHECK-SELECT: %vec:zpr = COPY $z0
+    ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0
+    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0
+    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
+    ; CHECK-SELECT-NEXT: [[DUP_ZR_H:%[0-9]+]]:zpr = DUP_ZR_H [[COPY]]
+    ; CHECK-SELECT-NEXT: [[INDEX_II_H:%[0-9]+]]:zpr = INDEX_II_H 0, 1, implicit $vg
+    ; CHECK-SELECT-NEXT: [[PTRUE_H:%[0-9]+]]:ppr_3b = PTRUE_H 31, implicit $vg
+    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_H:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_H [[PTRUE_H]], [[INDEX_II_H]], [[DUP_ZR_H]], implicit-def dead $nzcv
+    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_H %vec, [[CMPEQ_PPzZZ_H]], %elt
+    ; CHECK-SELECT-NEXT: $z0 = COPY %result
+    ;
+    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_8_s16
+    ; CHECK-REGBANK: %vec:fpr(<vscale x 8 x s16>) = COPY $z0
+    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0
+    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
+    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 8 x s16>)
+    ;
+    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_8_s16
+    ; CHECK-LEGAL: %vec:_(<vscale x 8 x s16>) = COPY $z0
+    ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0
+    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
+    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 8 x s16>)
+    %vec:_(<vscale x 8 x s16>) = COPY $z0
+    %elt:_(s32) = COPY $w0
+    %idx:_(s64) = COPY $x0
+    %result:_(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec(<vscale x 8 x s16>), %elt(s32), %idx(s64)
+    $z0 = COPY %result(<vscale x 8 x s16>)
+...
+---
+name:            test_insert_vector_elt_nxv_4_s32_idx_0
+body:             |
+  bb.1:
+    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_4_s32_idx_0
+    ; CHECK-SELECT: %vec:zpr = COPY $z0
+    ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0
+    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $xzr
+    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
+    ; CHECK-SELECT-NEXT: [[DUP_ZR_S:%[0-9]+]]:zpr = DUP_ZR_S [[COPY]]
+    ; CHECK-SELECT-NEXT: [[INDEX_II_S:%[0-9]+]]:zpr = INDEX_II_S 0, 1, implicit $vg
+    ; CHECK-SELECT-NEXT: [[PTRUE_S:%[0-9]+]]:ppr_3b = PTRUE_S 31, implicit $vg
+    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_S:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_S [[PTRUE_S]], [[INDEX_II_S]], [[DUP_ZR_S]], implicit-def dead $nzcv
+    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_S %vec, [[CMPEQ_PPzZZ_S]], %elt
+    ; CHECK-SELECT-NEXT: $z0 = COPY %result
+    ;
+    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_4_s32_idx_0
+    ; CHECK-REGBANK: %vec:fpr(<vscale x 4 x s32>) = COPY $z0
+    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0
+    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = G_CONSTANT i64 0
+    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 4 x s32>)
+    ;
+    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_4_s32_idx_0
+    ; CHECK-LEGAL: %vec:_(<vscale x 4 x s32>) = COPY $z0
+    ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0
+    ; CHECK-LEGAL-NEXT: %idx:_(s64) = G_CONSTANT i64 0
+    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 4 x s32>)
+    %vec:_(<vscale x 4 x s32>) = COPY $z0
+    %elt:_(s32) = COPY $w0
+    %idx:_(s64) = G_CONSTANT i64 0
+    %result:_(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec(<vscale x 4 x s32>), %elt(s32), %idx(s64)
+    $z0 = COPY %result(<vscale x 4 x s32>)
+...
+---
+name:            test_insert_vector_elt_nxv_4_s32_constant
+body:             |
+  bb.1:
+    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_4_s32_constant
+    ; CHECK-SELECT: %vec:zpr = COPY $z0
+    ; CHECK-SELECT-NEXT: %elt:gpr32common = MOVi32imm 5
+    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0
+    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
+    ; CHECK-SELECT-NEXT: [[DUP_ZR_S:%[0-9]+]]:zpr = DUP_ZR_S [[COPY]]
+    ; CHECK-SELECT-NEXT: [[INDEX_II_S:%[0-9]+]]:zpr = INDEX_II_S 0, 1, implicit $vg
+    ; CHECK-SELECT-NEXT: [[PTRUE_S:%[0-9]+]]:ppr_3b = PTRUE_S 31, implicit $vg
+    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_S:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_S [[PTRUE_S]], [[INDEX_II_S]], [[DUP_ZR_S]], implicit-def dead $nzcv
+    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_S %vec, [[CMPEQ_PPzZZ_S]], %elt
+    ; CHECK-SELECT-NEXT: $z0 = COPY %result
+    ;
+    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_4_s32_constant
+    ; CHECK-REGBANK: %vec:fpr(<vscale x 4 x s32>) = COPY $z0
+    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = G_CONSTANT i32 5
+    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
+    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 4 x s32>)
+    ;
+    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_4_s32_constant
+    ; CHECK-LEGAL: %vec:_(<vscale x 4 x s32>) = COPY $z0
+    ; CHECK-LEGAL-NEXT: %elt:_(s32) = G_CONSTANT i32 5
+    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
+    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 4 x s32>)
+    %vec:_(<vscale x 4 x s32>) = COPY $z0
+    %elt:_(s32) = G_CONSTANT i32 5
+    %idx:_(s64) = COPY $x0
+    %result:_(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec(<vscale x 4 x s32>), %elt(s32), %idx(s64)
+    $z0 = COPY %result(<vscale x 4 x s32>)
+...
+---
+name:            test_insert_vector_elt_nxv_4_s32
+body:             |
+  bb.1:
+    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_4_s32
+    ; CHECK-SELECT: %vec:zpr = COPY $z0
+    ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0
+    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0
+    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
+    ; CHECK-SELECT-NEXT: [[DUP_ZR_S:%[0-9]+]]:zpr = DUP_ZR_S [[COPY]]
+    ; CHECK-SELECT-NEXT: [[INDEX_II_S:%[0-9]+]]:zpr = INDEX_II_S 0, 1, implicit $vg
+    ; CHECK-SELECT-NEXT: [[PTRUE_S:%[0-9]+]]:ppr_3b = PTRUE_S 31, implicit $vg
+    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_S:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_S [[PTRUE_S]], [[INDEX_II_S]], [[DUP_ZR_S]], implicit-def dead $nzcv
+    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_S %vec, [[CMPEQ_PPzZZ_S]], %elt
+    ; CHECK-SELECT-NEXT: $z0 = COPY %result
+    ;
+    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_4_s32
+    ; CHECK-REGBANK: %vec:fpr(<vscale x 4 x s32>) = COPY $z0
+    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0
+    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
+    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 4 x s32>)
+    ;
+    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_4_s32
+    ; CHECK-LEGAL: %vec:_(<vscale x 4 x s32>) = COPY $z0
+    ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0
+    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
+    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
+    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 4 x s32>)
+    %vec:_(<vscale x 4 x s32>) = COPY $z0
+    %elt:_(s32) = COPY $w0
+    %idx:_(s64) = COPY $x0
+    %result:_(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec(<vscale x 4 x s32>), %elt(s32), %idx(s64)
+    $z0 = COPY %result(<vscale x 4 x s32>)
+...
+---
+name:            test_insert_vector_elt_nxv_2s64_idx_0
+body:             |
+  bb.1:
+    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_2s64_idx_0
+    ; CHECK-SELECT: %vec:zpr = COPY $z0
+    ; CHECK-SELECT-NEXT: %elt:gpr64sp = COPY $x0
+    ; CHECK-SELECT-NEXT: %idx:gpr64common = COPY $xzr
+    ; CHECK-SELECT-NEXT: [[DUP_ZR_D:%[0-9]+]]:zpr = DUP_ZR_D %idx
+    ; CHECK-SELECT-NEXT: [[INDEX_II_D:%[0-9]+]]:zpr = INDEX_II_D 0, 1, implicit $vg
+    ; CHECK-SELECT-NEXT: [[PTRUE_D:%[0-9]+]]:ppr_3b = PTRUE_D 31, implicit $vg
+    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_D:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_D [[PTRUE_D]], [[INDEX_II_D]], [[DUP_ZR_D]], implicit-def dead $nzcv
+    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_D %vec, [[CMPEQ_PPzZZ_D]], %elt
+    ; CHECK-SELECT-NEXT: $z0 = COPY %result
+    ;
+    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_2s64_idx_0
+    ; CHECK-REGBANK: %vec:fpr(<vscale x 2 x s64>) = COPY $z0
+    ; CHECK-REGBANK-NEXT: %elt:gpr(s64) = COPY $x0
+    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = G_CONSTANT i64 0
+    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64)
+    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 2 x s64>)
+    ;
+    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_2s64_idx_0
+    ; CHECK-LEGAL: %vec:_(<vscale x 2 x s64>) = COPY $z0
+    ; CHECK-LEGAL-NEXT: %elt:_(s64) = COPY $x0
+    ; CHECK-LEGAL-NEXT: %idx:_(s64) = G_CONSTANT i64 0
+    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64)
+    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 2 x s64>)
+    %vec:_(<vscale x 2 x s64>) = COPY $z0
+    %elt:_(s64) = COPY $x0
+    %idx:_(s64) = G_CONSTANT i64 0
+    %result:_(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec(<vscale x 2 x s64>), %elt(s64), %idx(s64)
+    $z0 = COPY %result(<vscale x 2 x s64>)
+...
+---
+name:            test_insert_vector_elt_nxv_2_s64_constant
+body:             |
+  bb.1:
+    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_2_s64_constant
+    ; CHECK-SELECT: %vec:zpr = COPY $z0
+    ; CHECK-SELECT-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 5
+    ; CHECK-SELECT-NEXT: %elt:gpr64sp = SUBREG_TO_REG 0, [[MOVi32imm]], %subreg.sub_32
+    ; CHECK-SELECT-NEXT: %idx:gpr64sp = COPY $x0
+    ; CHECK-SELECT-NEXT: [[DUP_ZR_D:%[0-9]+]]:zpr = DUP_ZR_D %idx
+    ; CHECK-SELECT-NEXT: [[INDEX_II_D:%[0-9]+]]:zpr = INDEX_II_D 0, 1, implicit $vg
+    ; CHECK-SELECT-NEXT: [[PTRUE_D:%[0-9]+]]:ppr_3b = PTRUE_D 31, implicit $vg
+    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_D:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_D [[PTRUE_D]], [[INDEX_II_D]], [[DUP_ZR_D]], implicit-def dead $nzcv
+    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_D %vec, [[CMPEQ_PPzZZ_D]], %elt
+    ; CHECK-SELECT-NEXT: $z0 = COPY %result
+    ;
+    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_2_s64_constant
+    ; CHECK-REGBANK: %vec:fpr(<vscale x 2 x s64>) = COPY $z0
+    ; CHECK-REGBANK-NEXT: %elt:gpr(s64) = G_CONSTANT i64 5
+    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
+    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64)
+    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 2 x s64>)
+    ;
+    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_2_s64_constant
+    ; CHECK-LEGAL: %vec:_(<vscale x 2 x s64>) = COPY $z0
+    ; CHECK-LEGAL-NEXT: %elt:_(s64) = G_CONSTANT i64 5
+    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
+    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64)
+    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 2 x s64>)
+    %vec:_(<vscale x 2 x s64>) = COPY $z0
+    %elt:_(s64) = G_CONSTANT i64 5
+    %idx:_(s64) = COPY $x0
+    %result:_(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec(<vscale x 2 x s64>), %elt(s64), %idx(s64)
+    $z0 = COPY %result(<vscale x 2 x s64>)
+...
+---
+name:            test_insert_vector_elt_nxv_2_s64
+body:             |
+  bb.1:
+    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_2_s64
+    ; CHECK-SELECT: %vec:zpr = COPY $z0
+    ; CHECK-SELECT-NEXT: %elt:gpr64sp = COPY $x0
+    ; CHECK-SELECT-NEXT: %idx:gpr64sp = COPY $x0
+    ; CHECK-SELECT-NEXT: [[DUP_ZR_D:%[0-9]+]]:zpr = DUP_ZR_D %idx
+    ; CHECK-SELECT-NEXT: [[INDEX_II_D:%[0-9]+]]:zpr = INDEX_II_D 0, 1, implicit $vg
+    ; CHECK-SELECT-NEXT: [[PTRUE_D:%[0-9]+]]:ppr_3b = PTRUE_D 31, implicit $vg
+    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_D:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_D [[PTRUE_D]], [[INDEX_II_D]], [[DUP_ZR_D]], implicit-def dead $nzcv
+    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_D %vec, [[CMPEQ_PPzZZ_D]], %elt
+    ; CHECK-SELECT-NEXT: $z0 = COPY %result
+    ;
+    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_2_s64
+    ; CHECK-REGBANK: %vec:fpr(<vscale x 2 x s64>) = COPY $z0
+    ; CHECK-REGBANK-NEXT: %elt:gpr(s64) = COPY $x0
+    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
+    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64)
+    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 2 x s64>)
+    ;
+    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_2_s64
+    ; CHECK-LEGAL: %vec:_(<vscale x 2 x s64>) = COPY $z0
+    ; CHECK-LEGAL-NEXT: %elt:_(s64) = COPY $x0
+    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
+    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64)
+    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 2 x s64>)
+    %vec:_(<vscale x 2 x s64>) = COPY $z0
+    %elt:_(s64) = COPY $x0
+    %idx:_(s64) = COPY $x0
+    %result:_(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec(<vscale x 2 x s64>), %elt(s64), %idx(s64)
+    $z0 = COPY %result(<vscale x 2 x s64>)
+...
-- 
GitLab


From 6effab990c5c1b4fe55fcd43004a1fd88145bb8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Thu, 31 Oct 2024 05:41:16 +0100
Subject: [PATCH 685/693] Revert "[GlobalISel][AArch64] Legalize
 G_INSERT_VECTOR_ELT for SVE" (#114353)

Reverts llvm/llvm-project#114310
---
 .../llvm/CodeGen/GlobalISel/LegalizerInfo.h   |  20 -
 .../CodeGen/GlobalISel/LegalityPredicates.cpp |  11 -
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |   4 -
 .../GISel/AArch64PostLegalizerLowering.cpp    |  51 +--
 .../GlobalISel/legalize-vector-insert-elt.mir | 423 ------------------
 5 files changed, 8 insertions(+), 501 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-vector-insert-elt.mir

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
index 6811b37767cb..6d71c150c8da 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
@@ -273,11 +273,6 @@ inline LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type) {
 LegalityPredicate
 typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1,
               std::initializer_list<std::pair<LLT, LLT>> TypesInit);
-/// True iff the given types for the given tuple of type indexes is one of the
-/// specified type tuple.
-LegalityPredicate
-typeTupleInSet(unsigned TypeIdx0, unsigned TypeIdx1, unsigned TypeIdx2,
-               std::initializer_list<std::tuple<LLT, LLT, LLT>> TypesInit);
 /// True iff the given types for the given pair of type indexes is one of the
 /// specified type pairs.
 LegalityPredicate typePairAndMemDescInSet(
@@ -509,15 +504,6 @@ class LegalizeRuleSet {
     using namespace LegalityPredicates;
     return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types));
   }
-
-  LegalizeRuleSet &
-  actionFor(LegalizeAction Action,
-            std::initializer_list<std::tuple<LLT, LLT, LLT>> Types) {
-    using namespace LegalityPredicates;
-    return actionIf(Action,
-                    typeTupleInSet(typeIdx(0), typeIdx(1), typeIdx(2), Types));
-  }
-
   /// Use the given action when type indexes 0 and 1 is any type pair in the
   /// given list.
   /// Action should be an action that requires mutation.
@@ -629,12 +615,6 @@ public:
       return *this;
     return actionFor(LegalizeAction::Legal, Types);
   }
-  LegalizeRuleSet &
-  legalFor(bool Pred, std::initializer_list<std::tuple<LLT, LLT, LLT>> Types) {
-    if (!Pred)
-      return *this;
-    return actionFor(LegalizeAction::Legal, Types);
-  }
   /// The instruction is legal when type index 0 is any type in the given list
   /// and imm index 0 is anything.
   LegalizeRuleSet &legalForTypeWithAnyImm(std::initializer_list<LLT> Types) {
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp
index dc7ed6cbe8b7..8fe48195c610 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp
@@ -49,17 +49,6 @@ LegalityPredicate LegalityPredicates::typePairInSet(
   };
 }
 
-LegalityPredicate LegalityPredicates::typeTupleInSet(
-    unsigned TypeIdx0, unsigned TypeIdx1, unsigned TypeIdx2,
-    std::initializer_list<std::tuple<LLT, LLT, LLT>> TypesInit) {
-  SmallVector<std::tuple<LLT, LLT, LLT>, 4> Types = TypesInit;
-  return [=](const LegalityQuery &Query) {
-    std::tuple<LLT, LLT, LLT> Match = {
-        Query.Types[TypeIdx0], Query.Types[TypeIdx1], Query.Types[TypeIdx2]};
-    return llvm::is_contained(Types, Match);
-  };
-}
-
 LegalityPredicate LegalityPredicates::typePairAndMemDescInSet(
     unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx,
     std::initializer_list<TypePairAndMemDesc> TypesAndMemDescInit) {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 7beda0e92a75..6024027afaf6 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -978,10 +978,6 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
   getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT)
       .legalIf(
           typeInSet(0, {v16s8, v8s8, v8s16, v4s16, v4s32, v2s32, v2s64, v2p0}))
-      .legalFor(HasSVE, {{nxv16s8, s32, s64},
-                         {nxv8s16, s32, s64},
-                         {nxv4s32, s32, s64},
-                         {nxv2s64, s64, s64}})
       .moreElementsToNextPow2(0)
       .widenVectorEltsToVectorMinSize(0, 64)
       .clampNumElements(0, v8s8, v16s8)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
index 0bf0a4bf27c4..b40fe55fdfaf 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
@@ -161,8 +161,6 @@ bool matchREV(MachineInstr &MI, MachineRegisterInfo &MRI,
   Register Dst = MI.getOperand(0).getReg();
   Register Src = MI.getOperand(1).getReg();
   LLT Ty = MRI.getType(Dst);
-  if (Ty.isScalableVector())
-    return false;
   unsigned EltSize = Ty.getScalarSizeInBits();
 
   // Element size for a rev cannot be 64.
@@ -198,10 +196,7 @@ bool matchTRN(MachineInstr &MI, MachineRegisterInfo &MRI,
   unsigned WhichResult;
   ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
   Register Dst = MI.getOperand(0).getReg();
-  LLT DstTy = MRI.getType(Dst);
-  if (DstTy.isScalableVector())
-    return false;
-  unsigned NumElts = DstTy.getNumElements();
+  unsigned NumElts = MRI.getType(Dst).getNumElements();
   if (!isTRNMask(ShuffleMask, NumElts, WhichResult))
     return false;
   unsigned Opc = (WhichResult == 0) ? AArch64::G_TRN1 : AArch64::G_TRN2;
@@ -222,10 +217,7 @@ bool matchUZP(MachineInstr &MI, MachineRegisterInfo &MRI,
   unsigned WhichResult;
   ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
   Register Dst = MI.getOperand(0).getReg();
-  LLT DstTy = MRI.getType(Dst);
-  if (DstTy.isScalableVector())
-    return false;
-  unsigned NumElts = DstTy.getNumElements();
+  unsigned NumElts = MRI.getType(Dst).getNumElements();
   if (!isUZPMask(ShuffleMask, NumElts, WhichResult))
     return false;
   unsigned Opc = (WhichResult == 0) ? AArch64::G_UZP1 : AArch64::G_UZP2;
@@ -241,10 +233,7 @@ bool matchZip(MachineInstr &MI, MachineRegisterInfo &MRI,
   unsigned WhichResult;
   ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
   Register Dst = MI.getOperand(0).getReg();
-  LLT DstTy = MRI.getType(Dst);
-  if (DstTy.isScalableVector())
-    return false;
-  unsigned NumElts = DstTy.getNumElements();
+  unsigned NumElts = MRI.getType(Dst).getNumElements();
   if (!isZIPMask(ShuffleMask, NumElts, WhichResult))
     return false;
   unsigned Opc = (WhichResult == 0) ? AArch64::G_ZIP1 : AArch64::G_ZIP2;
@@ -299,10 +288,7 @@ bool matchDupFromBuildVector(int Lane, MachineInstr &MI,
                              MachineRegisterInfo &MRI,
                              ShuffleVectorPseudo &MatchInfo) {
   assert(Lane >= 0 && "Expected positive lane?");
-  LLT Op1Ty = MRI.getType(MI.getOperand(1).getReg());
-  if (Op1Ty.isScalableVector())
-    return false;
-  int NumElements = Op1Ty.getNumElements();
+  int NumElements = MRI.getType(MI.getOperand(1).getReg()).getNumElements();
   // Test if the LHS is a BUILD_VECTOR. If it is, then we can just reference the
   // lane's definition directly.
   auto *BuildVecMI =
@@ -340,8 +326,6 @@ bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI,
 // Check if an EXT instruction can handle the shuffle mask when the vector
 // sources of the shuffle are the same.
 bool isSingletonExtMask(ArrayRef<int> M, LLT Ty) {
-  if (Ty.isScalableVector())
-    return false;
   unsigned NumElts = Ty.getNumElements();
 
   // Assume that the first shuffle index is not UNDEF.  Fail if it is.
@@ -373,17 +357,12 @@ bool matchEXT(MachineInstr &MI, MachineRegisterInfo &MRI,
   assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
   Register Dst = MI.getOperand(0).getReg();
   LLT DstTy = MRI.getType(Dst);
-  if (DstTy.isScalableVector())
-    return false;
   Register V1 = MI.getOperand(1).getReg();
   Register V2 = MI.getOperand(2).getReg();
   auto Mask = MI.getOperand(3).getShuffleMask();
   uint64_t Imm;
   auto ExtInfo = getExtMask(Mask, DstTy.getNumElements());
-  LLT V1Ty = MRI.getType(V1);
-  if (V1Ty.isScalableVector())
-    return false;
-  uint64_t ExtFactor = V1Ty.getScalarSizeInBits() / 8;
+  uint64_t ExtFactor = MRI.getType(V1).getScalarSizeInBits() / 8;
 
   if (!ExtInfo) {
     if (!getOpcodeDef<GImplicitDef>(V2, MRI) ||
@@ -444,8 +423,6 @@ void applyNonConstInsert(MachineInstr &MI, MachineRegisterInfo &MRI,
 
   Register Offset = Insert.getIndexReg();
   LLT VecTy = MRI.getType(Insert.getReg(0));
-  if (VecTy.isScalableVector())
-    return;
   LLT EltTy = MRI.getType(Insert.getElementReg());
   LLT IdxTy = MRI.getType(Insert.getIndexReg());
 
@@ -496,10 +473,7 @@ bool matchINS(MachineInstr &MI, MachineRegisterInfo &MRI,
   assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
   ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
   Register Dst = MI.getOperand(0).getReg();
-  LLT DstTy = MRI.getType(Dst);
-  if (DstTy.isScalableVector())
-    return false;
-  int NumElts = DstTy.getNumElements();
+  int NumElts = MRI.getType(Dst).getNumElements();
   auto DstIsLeftAndDstLane = isINSMask(ShuffleMask, NumElts);
   if (!DstIsLeftAndDstLane)
     return false;
@@ -548,8 +522,6 @@ bool isVShiftRImm(Register Reg, MachineRegisterInfo &MRI, LLT Ty,
   if (!Cst)
     return false;
   Cnt = *Cst;
-  if (Ty.isScalableVector())
-    return false;
   int64_t ElementBits = Ty.getScalarSizeInBits();
   return Cnt >= 1 && Cnt <= ElementBits;
 }
@@ -726,8 +698,6 @@ bool matchDupLane(MachineInstr &MI, MachineRegisterInfo &MRI,
   Register Src1Reg = MI.getOperand(1).getReg();
   const LLT SrcTy = MRI.getType(Src1Reg);
   const LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
-  if (SrcTy.isScalableVector())
-    return false;
 
   auto LaneIdx = getSplatIndex(MI);
   if (!LaneIdx)
@@ -804,8 +774,6 @@ bool matchScalarizeVectorUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI) {
   auto &Unmerge = cast<GUnmerge>(MI);
   Register Src1Reg = Unmerge.getReg(Unmerge.getNumOperands() - 1);
   const LLT SrcTy = MRI.getType(Src1Reg);
-  if (SrcTy.isScalableVector())
-    return false;
   if (SrcTy.getSizeInBits() != 128 && SrcTy.getSizeInBits() != 64)
     return false;
   return SrcTy.isVector() && !SrcTy.isScalable() &&
@@ -1019,10 +987,7 @@ bool matchLowerVectorFCMP(MachineInstr &MI, MachineRegisterInfo &MRI,
   if (!DstTy.isVector() || !ST.hasNEON())
     return false;
   Register LHS = MI.getOperand(2).getReg();
-  LLT LHSTy = MRI.getType(LHS);
-  if (LHSTy.isScalableVector())
-    return false;
-  unsigned EltSize = LHSTy.getScalarSizeInBits();
+  unsigned EltSize = MRI.getType(LHS).getScalarSizeInBits();
   if (EltSize == 16 && !ST.hasFullFP16())
     return false;
   if (EltSize != 16 && EltSize != 32 && EltSize != 64)
@@ -1218,7 +1183,7 @@ bool matchExtMulToMULL(MachineInstr &MI, MachineRegisterInfo &MRI) {
   MachineInstr *I1 = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI);
   MachineInstr *I2 = getDefIgnoringCopies(MI.getOperand(2).getReg(), MRI);
 
-  if (DstTy.isFixedVector()) {
+  if (DstTy.isVector()) {
     // If the source operands were EXTENDED before, then {U/S}MULL can be used
     unsigned I1Opc = I1->getOpcode();
     unsigned I2Opc = I2->getOpcode();
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vector-insert-elt.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vector-insert-elt.mir
deleted file mode 100644
index 6d24478cbfb3..000000000000
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vector-insert-elt.mir
+++ /dev/null
@@ -1,423 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -O0 -mtriple=aarch64-apple-ios -mattr=+sve -aarch64-enable-gisel-sve=1 -global-isel -start-before=legalizer -stop-after=instruction-select %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SELECT
-# RUN: llc -O0 -mtriple=aarch64-apple-ios -mattr=+sve -aarch64-enable-gisel-sve=1 -global-isel -start-before=legalizer -stop-after=regbankselect %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-REGBANK
-# RUN: llc -O0 -mtriple=aarch64-apple-ios -mattr=+sve -aarch64-enable-gisel-sve=1 -global-isel -run-pass=legalizer  %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-LEGAL
-
----
-name:            test_insert_vector_elt_nxv_16_s8_idx_0
-body:             |
-  bb.1:
-    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_16_s8_idx_0
-    ; CHECK-SELECT: %vec:zpr = COPY $z0
-    ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0
-    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $xzr
-    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
-    ; CHECK-SELECT-NEXT: [[DUP_ZR_B:%[0-9]+]]:zpr = DUP_ZR_B [[COPY]]
-    ; CHECK-SELECT-NEXT: [[INDEX_II_B:%[0-9]+]]:zpr = INDEX_II_B 0, 1, implicit $vg
-    ; CHECK-SELECT-NEXT: [[PTRUE_B:%[0-9]+]]:ppr_3b = PTRUE_B 31, implicit $vg
-    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_B:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_B [[PTRUE_B]], [[INDEX_II_B]], [[DUP_ZR_B]], implicit-def dead $nzcv
-    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_B %vec, [[CMPEQ_PPzZZ_B]], %elt
-    ; CHECK-SELECT-NEXT: $z0 = COPY %result
-    ;
-    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_16_s8_idx_0
-    ; CHECK-REGBANK: %vec:fpr(<vscale x 16 x s8>) = COPY $z0
-    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0
-    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = G_CONSTANT i64 0
-    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 16 x s8>)
-    ;
-    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_16_s8_idx_0
-    ; CHECK-LEGAL: %vec:_(<vscale x 16 x s8>) = COPY $z0
-    ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0
-    ; CHECK-LEGAL-NEXT: %idx:_(s64) = G_CONSTANT i64 0
-    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 16 x s8>)
-    %vec:_(<vscale x 16 x s8>) = COPY $z0
-    %elt:_(s32) = COPY $w0
-    %idx:_(s64) = G_CONSTANT i64 0
-    %result:_(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec(<vscale x 16 x s8>), %elt(s32), %idx(s64)
-    $z0 = COPY %result(<vscale x 16 x s8>)
-...
----
-name:            test_insert_vector_elt_nxv_16_s8_constant
-body:             |
-  bb.1:
-    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_16_s8_constant
-    ; CHECK-SELECT: %vec:zpr = COPY $z0
-    ; CHECK-SELECT-NEXT: %elt:gpr32common = MOVi32imm 5
-    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0
-    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
-    ; CHECK-SELECT-NEXT: [[DUP_ZR_B:%[0-9]+]]:zpr = DUP_ZR_B [[COPY]]
-    ; CHECK-SELECT-NEXT: [[INDEX_II_B:%[0-9]+]]:zpr = INDEX_II_B 0, 1, implicit $vg
-    ; CHECK-SELECT-NEXT: [[PTRUE_B:%[0-9]+]]:ppr_3b = PTRUE_B 31, implicit $vg
-    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_B:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_B [[PTRUE_B]], [[INDEX_II_B]], [[DUP_ZR_B]], implicit-def dead $nzcv
-    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_B %vec, [[CMPEQ_PPzZZ_B]], %elt
-    ; CHECK-SELECT-NEXT: $z0 = COPY %result
-    ;
-    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_16_s8_constant
-    ; CHECK-REGBANK: %vec:fpr(<vscale x 16 x s8>) = COPY $z0
-    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = G_CONSTANT i32 5
-    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
-    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 16 x s8>)
-    ;
-    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_16_s8_constant
-    ; CHECK-LEGAL: %vec:_(<vscale x 16 x s8>) = COPY $z0
-    ; CHECK-LEGAL-NEXT: %elt:_(s32) = G_CONSTANT i32 5
-    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
-    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 16 x s8>)
-    %vec:_(<vscale x 16 x s8>) = COPY $z0
-    %elt:_(s32) = G_CONSTANT i32 5
-    %idx:_(s64) = COPY $x0
-    %result:_(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec(<vscale x 16 x s8>), %elt(s32), %idx(s64)
-    $z0 = COPY %result(<vscale x 16 x s8>)
-...
----
-name:            test_insert_vector_elt_nxv_16_s8
-body:             |
-  bb.1:
-    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_16_s8
-    ; CHECK-SELECT: %vec:zpr = COPY $z0
-    ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0
-    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0
-    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
-    ; CHECK-SELECT-NEXT: [[DUP_ZR_B:%[0-9]+]]:zpr = DUP_ZR_B [[COPY]]
-    ; CHECK-SELECT-NEXT: [[INDEX_II_B:%[0-9]+]]:zpr = INDEX_II_B 0, 1, implicit $vg
-    ; CHECK-SELECT-NEXT: [[PTRUE_B:%[0-9]+]]:ppr_3b = PTRUE_B 31, implicit $vg
-    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_B:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_B [[PTRUE_B]], [[INDEX_II_B]], [[DUP_ZR_B]], implicit-def dead $nzcv
-    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_B %vec, [[CMPEQ_PPzZZ_B]], %elt
-    ; CHECK-SELECT-NEXT: $z0 = COPY %result
-    ;
-    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_16_s8
-    ; CHECK-REGBANK: %vec:fpr(<vscale x 16 x s8>) = COPY $z0
-    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0
-    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
-    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 16 x s8>)
-    ;
-    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_16_s8
-    ; CHECK-LEGAL: %vec:_(<vscale x 16 x s8>) = COPY $z0
-    ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0
-    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
-    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 16 x s8>)
-    %vec:_(<vscale x 16 x s8>) = COPY $z0
-    %elt:_(s32) = COPY $w0
-    %idx:_(s64) = COPY $x0
-    %result:_(<vscale x 16 x s8>) = G_INSERT_VECTOR_ELT %vec(<vscale x 16 x s8>), %elt(s32), %idx(s64)
-    $z0 = COPY %result(<vscale x 16 x s8>)
-...
----
-name:            test_insert_vector_elt_nxv_8_s16_idx_0
-body:             |
-  bb.1:
-    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_8_s16_idx_0
-    ; CHECK-SELECT: %vec:zpr = COPY $z0
-    ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0
-    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $xzr
-    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
-    ; CHECK-SELECT-NEXT: [[DUP_ZR_H:%[0-9]+]]:zpr = DUP_ZR_H [[COPY]]
-    ; CHECK-SELECT-NEXT: [[INDEX_II_H:%[0-9]+]]:zpr = INDEX_II_H 0, 1, implicit $vg
-    ; CHECK-SELECT-NEXT: [[PTRUE_H:%[0-9]+]]:ppr_3b = PTRUE_H 31, implicit $vg
-    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_H:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_H [[PTRUE_H]], [[INDEX_II_H]], [[DUP_ZR_H]], implicit-def dead $nzcv
-    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_H %vec, [[CMPEQ_PPzZZ_H]], %elt
-    ; CHECK-SELECT-NEXT: $z0 = COPY %result
-    ;
-    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_8_s16_idx_0
-    ; CHECK-REGBANK: %vec:fpr(<vscale x 8 x s16>) = COPY $z0
-    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0
-    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = G_CONSTANT i64 0
-    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 8 x s16>)
-    ;
-    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_8_s16_idx_0
-    ; CHECK-LEGAL: %vec:_(<vscale x 8 x s16>) = COPY $z0
-    ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0
-    ; CHECK-LEGAL-NEXT: %idx:_(s64) = G_CONSTANT i64 0
-    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 8 x s16>)
-    %vec:_(<vscale x 8 x s16>) = COPY $z0
-    %elt:_(s32) = COPY $w0
-    %idx:_(s64) = G_CONSTANT i64 0
-    %result:_(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec(<vscale x 8 x s16>), %elt(s32), %idx(s64)
-    $z0 = COPY %result(<vscale x 8 x s16>)
-...
----
-name:            test_insert_vector_elt_nxv_8_s16_constant
-body:             |
-  bb.1:
-    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_8_s16_constant
-    ; CHECK-SELECT: %vec:zpr = COPY $z0
-    ; CHECK-SELECT-NEXT: %elt:gpr32common = MOVi32imm 5
-    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0
-    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
-    ; CHECK-SELECT-NEXT: [[DUP_ZR_H:%[0-9]+]]:zpr = DUP_ZR_H [[COPY]]
-    ; CHECK-SELECT-NEXT: [[INDEX_II_H:%[0-9]+]]:zpr = INDEX_II_H 0, 1, implicit $vg
-    ; CHECK-SELECT-NEXT: [[PTRUE_H:%[0-9]+]]:ppr_3b = PTRUE_H 31, implicit $vg
-    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_H:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_H [[PTRUE_H]], [[INDEX_II_H]], [[DUP_ZR_H]], implicit-def dead $nzcv
-    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_H %vec, [[CMPEQ_PPzZZ_H]], %elt
-    ; CHECK-SELECT-NEXT: $z0 = COPY %result
-    ;
-    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_8_s16_constant
-    ; CHECK-REGBANK: %vec:fpr(<vscale x 8 x s16>) = COPY $z0
-    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = G_CONSTANT i32 5
-    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
-    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 8 x s16>)
-    ;
-    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_8_s16_constant
-    ; CHECK-LEGAL: %vec:_(<vscale x 8 x s16>) = COPY $z0
-    ; CHECK-LEGAL-NEXT: %elt:_(s32) = G_CONSTANT i32 5
-    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
-    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 8 x s16>)
-    %vec:_(<vscale x 8 x s16>) = COPY $z0
-    %elt:_(s32) = G_CONSTANT i32 5
-    %idx:_(s64) = COPY $x0
-    %result:_(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec(<vscale x 8 x s16>), %elt(s32), %idx(s64)
-    $z0 = COPY %result(<vscale x 8 x s16>)
-...
----
-name:            test_insert_vector_elt_nxv_8_s16
-body:             |
-  bb.1:
-    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_8_s16
-    ; CHECK-SELECT: %vec:zpr = COPY $z0
-    ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0
-    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0
-    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
-    ; CHECK-SELECT-NEXT: [[DUP_ZR_H:%[0-9]+]]:zpr = DUP_ZR_H [[COPY]]
-    ; CHECK-SELECT-NEXT: [[INDEX_II_H:%[0-9]+]]:zpr = INDEX_II_H 0, 1, implicit $vg
-    ; CHECK-SELECT-NEXT: [[PTRUE_H:%[0-9]+]]:ppr_3b = PTRUE_H 31, implicit $vg
-    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_H:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_H [[PTRUE_H]], [[INDEX_II_H]], [[DUP_ZR_H]], implicit-def dead $nzcv
-    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_H %vec, [[CMPEQ_PPzZZ_H]], %elt
-    ; CHECK-SELECT-NEXT: $z0 = COPY %result
-    ;
-    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_8_s16
-    ; CHECK-REGBANK: %vec:fpr(<vscale x 8 x s16>) = COPY $z0
-    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0
-    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
-    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 8 x s16>)
-    ;
-    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_8_s16
-    ; CHECK-LEGAL: %vec:_(<vscale x 8 x s16>) = COPY $z0
-    ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0
-    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
-    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 8 x s16>)
-    %vec:_(<vscale x 8 x s16>) = COPY $z0
-    %elt:_(s32) = COPY $w0
-    %idx:_(s64) = COPY $x0
-    %result:_(<vscale x 8 x s16>) = G_INSERT_VECTOR_ELT %vec(<vscale x 8 x s16>), %elt(s32), %idx(s64)
-    $z0 = COPY %result(<vscale x 8 x s16>)
-...
----
-name:            test_insert_vector_elt_nxv_4_s32_idx_0
-body:             |
-  bb.1:
-    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_4_s32_idx_0
-    ; CHECK-SELECT: %vec:zpr = COPY $z0
-    ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0
-    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $xzr
-    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
-    ; CHECK-SELECT-NEXT: [[DUP_ZR_S:%[0-9]+]]:zpr = DUP_ZR_S [[COPY]]
-    ; CHECK-SELECT-NEXT: [[INDEX_II_S:%[0-9]+]]:zpr = INDEX_II_S 0, 1, implicit $vg
-    ; CHECK-SELECT-NEXT: [[PTRUE_S:%[0-9]+]]:ppr_3b = PTRUE_S 31, implicit $vg
-    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_S:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_S [[PTRUE_S]], [[INDEX_II_S]], [[DUP_ZR_S]], implicit-def dead $nzcv
-    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_S %vec, [[CMPEQ_PPzZZ_S]], %elt
-    ; CHECK-SELECT-NEXT: $z0 = COPY %result
-    ;
-    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_4_s32_idx_0
-    ; CHECK-REGBANK: %vec:fpr(<vscale x 4 x s32>) = COPY $z0
-    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0
-    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = G_CONSTANT i64 0
-    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 4 x s32>)
-    ;
-    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_4_s32_idx_0
-    ; CHECK-LEGAL: %vec:_(<vscale x 4 x s32>) = COPY $z0
-    ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0
-    ; CHECK-LEGAL-NEXT: %idx:_(s64) = G_CONSTANT i64 0
-    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 4 x s32>)
-    %vec:_(<vscale x 4 x s32>) = COPY $z0
-    %elt:_(s32) = COPY $w0
-    %idx:_(s64) = G_CONSTANT i64 0
-    %result:_(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec(<vscale x 4 x s32>), %elt(s32), %idx(s64)
-    $z0 = COPY %result(<vscale x 4 x s32>)
-...
----
-name:            test_insert_vector_elt_nxv_4_s32_constant
-body:             |
-  bb.1:
-    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_4_s32_constant
-    ; CHECK-SELECT: %vec:zpr = COPY $z0
-    ; CHECK-SELECT-NEXT: %elt:gpr32common = MOVi32imm 5
-    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0
-    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
-    ; CHECK-SELECT-NEXT: [[DUP_ZR_S:%[0-9]+]]:zpr = DUP_ZR_S [[COPY]]
-    ; CHECK-SELECT-NEXT: [[INDEX_II_S:%[0-9]+]]:zpr = INDEX_II_S 0, 1, implicit $vg
-    ; CHECK-SELECT-NEXT: [[PTRUE_S:%[0-9]+]]:ppr_3b = PTRUE_S 31, implicit $vg
-    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_S:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_S [[PTRUE_S]], [[INDEX_II_S]], [[DUP_ZR_S]], implicit-def dead $nzcv
-    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_S %vec, [[CMPEQ_PPzZZ_S]], %elt
-    ; CHECK-SELECT-NEXT: $z0 = COPY %result
-    ;
-    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_4_s32_constant
-    ; CHECK-REGBANK: %vec:fpr(<vscale x 4 x s32>) = COPY $z0
-    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = G_CONSTANT i32 5
-    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
-    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 4 x s32>)
-    ;
-    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_4_s32_constant
-    ; CHECK-LEGAL: %vec:_(<vscale x 4 x s32>) = COPY $z0
-    ; CHECK-LEGAL-NEXT: %elt:_(s32) = G_CONSTANT i32 5
-    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
-    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 4 x s32>)
-    %vec:_(<vscale x 4 x s32>) = COPY $z0
-    %elt:_(s32) = G_CONSTANT i32 5
-    %idx:_(s64) = COPY $x0
-    %result:_(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec(<vscale x 4 x s32>), %elt(s32), %idx(s64)
-    $z0 = COPY %result(<vscale x 4 x s32>)
-...
----
-name:            test_insert_vector_elt_nxv_4_s32
-body:             |
-  bb.1:
-    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_4_s32
-    ; CHECK-SELECT: %vec:zpr = COPY $z0
-    ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0
-    ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0
-    ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32
-    ; CHECK-SELECT-NEXT: [[DUP_ZR_S:%[0-9]+]]:zpr = DUP_ZR_S [[COPY]]
-    ; CHECK-SELECT-NEXT: [[INDEX_II_S:%[0-9]+]]:zpr = INDEX_II_S 0, 1, implicit $vg
-    ; CHECK-SELECT-NEXT: [[PTRUE_S:%[0-9]+]]:ppr_3b = PTRUE_S 31, implicit $vg
-    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_S:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_S [[PTRUE_S]], [[INDEX_II_S]], [[DUP_ZR_S]], implicit-def dead $nzcv
-    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_S %vec, [[CMPEQ_PPzZZ_S]], %elt
-    ; CHECK-SELECT-NEXT: $z0 = COPY %result
-    ;
-    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_4_s32
-    ; CHECK-REGBANK: %vec:fpr(<vscale x 4 x s32>) = COPY $z0
-    ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0
-    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
-    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 4 x s32>)
-    ;
-    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_4_s32
-    ; CHECK-LEGAL: %vec:_(<vscale x 4 x s32>) = COPY $z0
-    ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0
-    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
-    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64)
-    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 4 x s32>)
-    %vec:_(<vscale x 4 x s32>) = COPY $z0
-    %elt:_(s32) = COPY $w0
-    %idx:_(s64) = COPY $x0
-    %result:_(<vscale x 4 x s32>) = G_INSERT_VECTOR_ELT %vec(<vscale x 4 x s32>), %elt(s32), %idx(s64)
-    $z0 = COPY %result(<vscale x 4 x s32>)
-...
----
-name:            test_insert_vector_elt_nxv_2s64_idx_0
-body:             |
-  bb.1:
-    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_2s64_idx_0
-    ; CHECK-SELECT: %vec:zpr = COPY $z0
-    ; CHECK-SELECT-NEXT: %elt:gpr64sp = COPY $x0
-    ; CHECK-SELECT-NEXT: %idx:gpr64common = COPY $xzr
-    ; CHECK-SELECT-NEXT: [[DUP_ZR_D:%[0-9]+]]:zpr = DUP_ZR_D %idx
-    ; CHECK-SELECT-NEXT: [[INDEX_II_D:%[0-9]+]]:zpr = INDEX_II_D 0, 1, implicit $vg
-    ; CHECK-SELECT-NEXT: [[PTRUE_D:%[0-9]+]]:ppr_3b = PTRUE_D 31, implicit $vg
-    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_D:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_D [[PTRUE_D]], [[INDEX_II_D]], [[DUP_ZR_D]], implicit-def dead $nzcv
-    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_D %vec, [[CMPEQ_PPzZZ_D]], %elt
-    ; CHECK-SELECT-NEXT: $z0 = COPY %result
-    ;
-    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_2s64_idx_0
-    ; CHECK-REGBANK: %vec:fpr(<vscale x 2 x s64>) = COPY $z0
-    ; CHECK-REGBANK-NEXT: %elt:gpr(s64) = COPY $x0
-    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = G_CONSTANT i64 0
-    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64)
-    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 2 x s64>)
-    ;
-    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_2s64_idx_0
-    ; CHECK-LEGAL: %vec:_(<vscale x 2 x s64>) = COPY $z0
-    ; CHECK-LEGAL-NEXT: %elt:_(s64) = COPY $x0
-    ; CHECK-LEGAL-NEXT: %idx:_(s64) = G_CONSTANT i64 0
-    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64)
-    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 2 x s64>)
-    %vec:_(<vscale x 2 x s64>) = COPY $z0
-    %elt:_(s64) = COPY $x0
-    %idx:_(s64) = G_CONSTANT i64 0
-    %result:_(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec(<vscale x 2 x s64>), %elt(s64), %idx(s64)
-    $z0 = COPY %result(<vscale x 2 x s64>)
-...
----
-name:            test_insert_vector_elt_nxv_2_s64_constant
-body:             |
-  bb.1:
-    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_2_s64_constant
-    ; CHECK-SELECT: %vec:zpr = COPY $z0
-    ; CHECK-SELECT-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 5
-    ; CHECK-SELECT-NEXT: %elt:gpr64sp = SUBREG_TO_REG 0, [[MOVi32imm]], %subreg.sub_32
-    ; CHECK-SELECT-NEXT: %idx:gpr64sp = COPY $x0
-    ; CHECK-SELECT-NEXT: [[DUP_ZR_D:%[0-9]+]]:zpr = DUP_ZR_D %idx
-    ; CHECK-SELECT-NEXT: [[INDEX_II_D:%[0-9]+]]:zpr = INDEX_II_D 0, 1, implicit $vg
-    ; CHECK-SELECT-NEXT: [[PTRUE_D:%[0-9]+]]:ppr_3b = PTRUE_D 31, implicit $vg
-    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_D:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_D [[PTRUE_D]], [[INDEX_II_D]], [[DUP_ZR_D]], implicit-def dead $nzcv
-    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_D %vec, [[CMPEQ_PPzZZ_D]], %elt
-    ; CHECK-SELECT-NEXT: $z0 = COPY %result
-    ;
-    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_2_s64_constant
-    ; CHECK-REGBANK: %vec:fpr(<vscale x 2 x s64>) = COPY $z0
-    ; CHECK-REGBANK-NEXT: %elt:gpr(s64) = G_CONSTANT i64 5
-    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
-    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64)
-    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 2 x s64>)
-    ;
-    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_2_s64_constant
-    ; CHECK-LEGAL: %vec:_(<vscale x 2 x s64>) = COPY $z0
-    ; CHECK-LEGAL-NEXT: %elt:_(s64) = G_CONSTANT i64 5
-    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
-    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64)
-    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 2 x s64>)
-    %vec:_(<vscale x 2 x s64>) = COPY $z0
-    %elt:_(s64) = G_CONSTANT i64 5
-    %idx:_(s64) = COPY $x0
-    %result:_(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec(<vscale x 2 x s64>), %elt(s64), %idx(s64)
-    $z0 = COPY %result(<vscale x 2 x s64>)
-...
----
-name:            test_insert_vector_elt_nxv_2_s64
-body:             |
-  bb.1:
-    ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_2_s64
-    ; CHECK-SELECT: %vec:zpr = COPY $z0
-    ; CHECK-SELECT-NEXT: %elt:gpr64sp = COPY $x0
-    ; CHECK-SELECT-NEXT: %idx:gpr64sp = COPY $x0
-    ; CHECK-SELECT-NEXT: [[DUP_ZR_D:%[0-9]+]]:zpr = DUP_ZR_D %idx
-    ; CHECK-SELECT-NEXT: [[INDEX_II_D:%[0-9]+]]:zpr = INDEX_II_D 0, 1, implicit $vg
-    ; CHECK-SELECT-NEXT: [[PTRUE_D:%[0-9]+]]:ppr_3b = PTRUE_D 31, implicit $vg
-    ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_D:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_D [[PTRUE_D]], [[INDEX_II_D]], [[DUP_ZR_D]], implicit-def dead $nzcv
-    ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_D %vec, [[CMPEQ_PPzZZ_D]], %elt
-    ; CHECK-SELECT-NEXT: $z0 = COPY %result
-    ;
-    ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_2_s64
-    ; CHECK-REGBANK: %vec:fpr(<vscale x 2 x s64>) = COPY $z0
-    ; CHECK-REGBANK-NEXT: %elt:gpr(s64) = COPY $x0
-    ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0
-    ; CHECK-REGBANK-NEXT: %result:fpr(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64)
-    ; CHECK-REGBANK-NEXT: $z0 = COPY %result(<vscale x 2 x s64>)
-    ;
-    ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_2_s64
-    ; CHECK-LEGAL: %vec:_(<vscale x 2 x s64>) = COPY $z0
-    ; CHECK-LEGAL-NEXT: %elt:_(s64) = COPY $x0
-    ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0
-    ; CHECK-LEGAL-NEXT: %result:_(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64)
-    ; CHECK-LEGAL-NEXT: $z0 = COPY %result(<vscale x 2 x s64>)
-    %vec:_(<vscale x 2 x s64>) = COPY $z0
-    %elt:_(s64) = COPY $x0
-    %idx:_(s64) = COPY $x0
-    %result:_(<vscale x 2 x s64>) = G_INSERT_VECTOR_ELT %vec(<vscale x 2 x s64>), %elt(s64), %idx(s64)
-    $z0 = COPY %result(<vscale x 2 x s64>)
-...
-- 
GitLab


From 50896e7ef5794a74ff91066f845d24ad664bc0dc Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 30 Oct 2024 19:27:15 -0700
Subject: [PATCH 686/693] [ARM] Use getSignedConstant. NFC

---
 llvm/lib/Target/ARM/ARMISelLowering.cpp | 2 +-
 llvm/lib/Target/ARM/ARMInstrThumb2.td   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index a98b7a842092..e0022190d87c 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -6051,7 +6051,7 @@ static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
                             DAG.getConstant((1 << BW) - 1, DL, VT));
   if (IsSigned)
     Max = DAG.getNode(ISD::SMAX, DL, VT, Max,
-                      DAG.getConstant(-(1 << BW), DL, VT));
+                      DAG.getSignedConstant(-(1 << BW), DL, VT));
   return Max;
 }
 
diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td
index 61635bd1629e..cb20aacb539a 100644
--- a/llvm/lib/Target/ARM/ARMInstrThumb2.td
+++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td
@@ -77,8 +77,8 @@ def t2_so_imm_not_XFORM : SDNodeXForm<imm, [{
 
 // t2_so_imm_neg_XFORM - Return the negation of a t2_so_imm value
 def t2_so_imm_neg_XFORM : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(-((int)N->getZExtValue()), SDLoc(N),
-                                   MVT::i32);
+  return CurDAG->getSignedConstant(-((int)N->getZExtValue()), SDLoc(N),
+                                   MVT::i32, /*isTarget=*/true);
 }]>;
 
 // so_imm_notSext_XFORM - Return a so_imm value packed into the format
-- 
GitLab


From 00cbb68fb7591997af52584eaa7d7f8ef81e5288 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 30 Oct 2024 21:21:22 -0700
Subject: [PATCH 687/693] [LegalizeDAG] Use getSignedConstant. NFC

---
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 61ed94ce38c4..4ea2054360a4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1770,7 +1770,7 @@ void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
   Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size);       // Value
   if (Alignment > StackAlign)
     Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
-                       DAG.getConstant(-Alignment.value(), dl, VT));
+                       DAG.getSignedConstant(-Alignment.value(), dl, VT));
   Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);     // Output chain
 
   Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
-- 
GitLab


From a33fd61862efc03cfde2cc84c2a2d6f7f1c55983 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 30 Oct 2024 21:43:10 -0700
Subject: [PATCH 688/693] [RISCV] Remove dead code from IntrinsicsRISCVXsf.td.
 NFC

---
 llvm/include/llvm/IR/IntrinsicsRISCVXsf.td | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td b/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td
index 4279661473d8..bf20080229aa 100644
--- a/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td
+++ b/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td
@@ -10,13 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-class VCIXSuffix<string range> {
-  list<string> suffix = !cond(!eq(range, "c"): ["e8mf8", "e8mf4", "e8mf2", "e8m1", "e8m2", "e8m4", "e8m8"],
-                              !eq(range, "s"): ["e16mf4", "e16mf2", "e16m1", "e16m2", "e16m4", "e16m8"],
-                              !eq(range, "i"): ["e32mf2", "e32m1", "e32m2", "e32m4", "e32m8"],
-                              !eq(range, "l"): ["e64m1", "e64m2", "e64m4", "e64m8"]);
-}
-
 let TargetPrefix = "riscv" in {
   // Output: (vector_out)
   // Input: (bit<27-26>, bit<24-20>, scalar_in, vl) or
-- 
GitLab


From 6da5968f5ecc2a2e8b0697e335f4dec1b3bbfd01 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Thu, 31 Oct 2024 07:15:17 +0200
Subject: [PATCH 689/693] [RISCV] Lower scalar_to_vector for supported FP types
 (#114340)

In https://reviews.llvm.org/D147608 we added custom lowering for
integers, but inadvertently also marked it as custom for scalable FP
vectors despite not handling it.

This adds handling for floats and marks it as custom lowered for
fixed-length FP vectors too.

Note that this doesn't handle bf16 or f16 vectors that would need
promotion, but these scalar_to_vector nodes seem to be emitted when
expanding them.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 16 +++-
 .../RISCV/rvv/fixed-vectors-scalarized.ll     | 92 +++++++++++++++++++
 ...fixed-vectors-vitofp-constrained-sdnode.ll | 27 ++----
 llvm/test/CodeGen/RISCV/rvv/pr63596.ll        | 54 +++++++----
 4 files changed, 146 insertions(+), 43 deletions(-)
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/fixed-vectors-scalarized.ll

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 0b5c46f2c370..3b3f8772a089 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1403,7 +1403,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         }
 
         setOperationAction({ISD::BUILD_VECTOR, ISD::VECTOR_SHUFFLE,
-                            ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
+                            ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
+                            ISD::SCALAR_TO_VECTOR},
                            VT, Custom);
 
         setOperationAction(
@@ -6511,9 +6512,16 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
     if (VT.isFixedLengthVector())
       ContainerVT = getContainerForFixedLengthVector(VT);
     SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
-    Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Scalar);
-    SDValue V = DAG.getNode(RISCVISD::VMV_S_X_VL, DL, ContainerVT,
-                            DAG.getUNDEF(ContainerVT), Scalar, VL);
+
+    SDValue V;
+    if (VT.isFloatingPoint()) {
+      V = DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, ContainerVT,
+                      DAG.getUNDEF(ContainerVT), Scalar, VL);
+    } else {
+      Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Scalar);
+      V = DAG.getNode(RISCVISD::VMV_S_X_VL, DL, ContainerVT,
+                      DAG.getUNDEF(ContainerVT), Scalar, VL);
+    }
     if (VT.isFixedLengthVector())
       V = convertFromScalableVector(VT, V, DAG, Subtarget);
     return V;
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-scalarized.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-scalarized.ll
new file mode 100644
index 000000000000..4621f339ca88
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-scalarized.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v -verify-machineinstrs | FileCheck %s
+
+define <8 x float> @fpext_v8bf16(<8 x bfloat> %x) {
+; CHECK-LABEL: fpext_v8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmv.x.w a0, fa0
+; CHECK-NEXT:    fmv.x.w a1, fa1
+; CHECK-NEXT:    fmv.x.w a2, fa2
+; CHECK-NEXT:    fmv.x.w a3, fa3
+; CHECK-NEXT:    fmv.x.w a4, fa4
+; CHECK-NEXT:    fmv.x.w a5, fa5
+; CHECK-NEXT:    fmv.x.w a6, fa6
+; CHECK-NEXT:    fmv.x.w a7, fa7
+; CHECK-NEXT:    slli a7, a7, 16
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vmv.s.x v8, a7
+; CHECK-NEXT:    slli a6, a6, 16
+; CHECK-NEXT:    vmv.s.x v9, a6
+; CHECK-NEXT:    vslideup.vi v9, v8, 1
+; CHECK-NEXT:    slli a5, a5, 16
+; CHECK-NEXT:    vmv.s.x v8, a5
+; CHECK-NEXT:    slli a4, a4, 16
+; CHECK-NEXT:    vmv.s.x v10, a4
+; CHECK-NEXT:    vslideup.vi v10, v8, 1
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vslideup.vi v10, v9, 2
+; CHECK-NEXT:    slli a3, a3, 16
+; CHECK-NEXT:    vmv.s.x v8, a3
+; CHECK-NEXT:    slli a2, a2, 16
+; CHECK-NEXT:    vmv.s.x v9, a2
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vi v9, v8, 1
+; CHECK-NEXT:    slli a1, a1, 16
+; CHECK-NEXT:    vmv.s.x v11, a1
+; CHECK-NEXT:    slli a0, a0, 16
+; CHECK-NEXT:    vmv.s.x v8, a0
+; CHECK-NEXT:    vslideup.vi v8, v11, 1
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v9, 2
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v10, 4
+; CHECK-NEXT:    ret
+  %y = fpext <8 x bfloat> %x to <8 x float>
+  ret <8 x float> %y
+}
+
+define <8 x float> @fpext_v8f16(<8 x bfloat> %x) {
+; CHECK-LABEL: fpext_v8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmv.x.w a0, fa0
+; CHECK-NEXT:    fmv.x.w a1, fa1
+; CHECK-NEXT:    fmv.x.w a2, fa2
+; CHECK-NEXT:    fmv.x.w a3, fa3
+; CHECK-NEXT:    fmv.x.w a4, fa4
+; CHECK-NEXT:    fmv.x.w a5, fa5
+; CHECK-NEXT:    fmv.x.w a6, fa6
+; CHECK-NEXT:    fmv.x.w a7, fa7
+; CHECK-NEXT:    slli a7, a7, 16
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vmv.s.x v8, a7
+; CHECK-NEXT:    slli a6, a6, 16
+; CHECK-NEXT:    vmv.s.x v9, a6
+; CHECK-NEXT:    vslideup.vi v9, v8, 1
+; CHECK-NEXT:    slli a5, a5, 16
+; CHECK-NEXT:    vmv.s.x v8, a5
+; CHECK-NEXT:    slli a4, a4, 16
+; CHECK-NEXT:    vmv.s.x v10, a4
+; CHECK-NEXT:    vslideup.vi v10, v8, 1
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vslideup.vi v10, v9, 2
+; CHECK-NEXT:    slli a3, a3, 16
+; CHECK-NEXT:    vmv.s.x v8, a3
+; CHECK-NEXT:    slli a2, a2, 16
+; CHECK-NEXT:    vmv.s.x v9, a2
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vi v9, v8, 1
+; CHECK-NEXT:    slli a1, a1, 16
+; CHECK-NEXT:    vmv.s.x v11, a1
+; CHECK-NEXT:    slli a0, a0, 16
+; CHECK-NEXT:    vmv.s.x v8, a0
+; CHECK-NEXT:    vslideup.vi v8, v11, 1
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v9, 2
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v10, 4
+; CHECK-NEXT:    ret
+  %y = fpext <8 x bfloat> %x to <8 x float>
+  ret <8 x float> %y
+}
+
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vitofp-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vitofp-constrained-sdnode.ll
index 3dec7daf66ac..5eb54fc7e299 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vitofp-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vitofp-constrained-sdnode.ll
@@ -412,30 +412,20 @@ declare <1 x half> @llvm.experimental.constrained.sitofp.v1f16.v1i7(<1 x i7>, me
 define <1 x half> @vsitofp_v1i7_v1f16(<1 x i7> %va) strictfp {
 ; RV32-LABEL: vsitofp_v1i7_v1f16:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    slli a0, a0, 25
 ; RV32-NEXT:    srai a0, a0, 25
 ; RV32-NEXT:    fcvt.h.w fa5, a0
-; RV32-NEXT:    fsh fa5, 14(sp)
-; RV32-NEXT:    addi a0, sp, 14
-; RV32-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
-; RV32-NEXT:    vle16.v v8, (a0)
-; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vfmv.s.f v8, fa5
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vsitofp_v1i7_v1f16:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -16
-; RV64-NEXT:    .cfi_def_cfa_offset 16
 ; RV64-NEXT:    slli a0, a0, 57
 ; RV64-NEXT:    srai a0, a0, 57
 ; RV64-NEXT:    fcvt.h.w fa5, a0
-; RV64-NEXT:    fsh fa5, 14(sp)
-; RV64-NEXT:    addi a0, sp, 14
-; RV64-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
-; RV64-NEXT:    vle16.v v8, (a0)
-; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV64-NEXT:    vfmv.s.f v8, fa5
 ; RV64-NEXT:    ret
   %evec = call <1 x half> @llvm.experimental.constrained.sitofp.v1f16.v1i7(<1 x i7> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <1 x half> %evec
@@ -445,15 +435,10 @@ declare <1 x half> @llvm.experimental.constrained.uitofp.v1f16.v1i7(<1 x i7>, me
 define <1 x half> @vuitofp_v1i7_v1f16(<1 x i7> %va) strictfp {
 ; CHECK-LABEL: vuitofp_v1i7_v1f16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    andi a0, a0, 127
 ; CHECK-NEXT:    fcvt.h.wu fa5, a0
-; CHECK-NEXT:    fsh fa5, 14(sp)
-; CHECK-NEXT:    addi a0, sp, 14
-; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
-; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v8, fa5
 ; CHECK-NEXT:    ret
   %evec = call <1 x half> @llvm.experimental.constrained.uitofp.v1f16.v1i7(<1 x i7> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <1 x half> %evec
diff --git a/llvm/test/CodeGen/RISCV/rvv/pr63596.ll b/llvm/test/CodeGen/RISCV/rvv/pr63596.ll
index 8bb62eaa8e9e..dbd4224c7ef0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/pr63596.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/pr63596.ll
@@ -9,36 +9,54 @@ define <4 x float> @foo(ptr %0) nounwind {
 ; CHECK-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 1
+; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    lhu s0, 0(a0)
 ; CHECK-NEXT:    lhu s1, 2(a0)
 ; CHECK-NEXT:    lhu s2, 4(a0)
 ; CHECK-NEXT:    lhu a0, 6(a0)
 ; CHECK-NEXT:    fmv.w.x fa0, a0
 ; CHECK-NEXT:    call __extendhfsf2
-; CHECK-NEXT:    fsw fa0, 4(sp)
-; CHECK-NEXT:    fmv.w.x fa0, s2
+; CHECK-NEXT:    fmv.w.x fa5, s2
+; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    fmv.s fa0, fa5
 ; CHECK-NEXT:    call __extendhfsf2
-; CHECK-NEXT:    fsw fa0, 12(sp)
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vslideup.vi v8, v9, 1
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
 ; CHECK-NEXT:    fmv.w.x fa0, s1
 ; CHECK-NEXT:    call __extendhfsf2
-; CHECK-NEXT:    fsw fa0, 8(sp)
-; CHECK-NEXT:    fmv.w.x fa0, s0
+; CHECK-NEXT:    fmv.w.x fa5, s0
+; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    fmv.s fa0, fa5
 ; CHECK-NEXT:    call __extendhfsf2
-; CHECK-NEXT:    fsw fa0, 0(sp)
-; CHECK-NEXT:    addi a0, sp, 4
-; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; CHECK-NEXT:    vle32.v v9, (a0)
-; CHECK-NEXT:    addi a0, sp, 12
-; CHECK-NEXT:    vle32.v v10, (a0)
-; CHECK-NEXT:    addi a0, sp, 8
-; CHECK-NEXT:    vle32.v v11, (a0)
-; CHECK-NEXT:    mv a0, sp
-; CHECK-NEXT:    vle32.v v8, (a0)
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vi v10, v9, 1
-; CHECK-NEXT:    vslideup.vi v8, v11, 1
+; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vslideup.vi v8, v9, 1
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 2
+; CHECK-NEXT:    vslideup.vi v8, v9, 2
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    add sp, sp, a0
 ; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
-- 
GitLab


From 9bb5af8a42ab5971baaff45c0c40ebd766196528 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Wed, 30 Oct 2024 22:27:12 -0700
Subject: [PATCH 690/693] [TableGen] Replace StringRef::slice with substr. NFC

---
 llvm/utils/TableGen/AsmMatcherEmitter.cpp         | 12 ++++++------
 llvm/utils/TableGen/Common/CodeGenInstruction.cpp |  3 ++-
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
index e3d9d010f9ae..ade393c11b7a 100644
--- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
@@ -1001,7 +1001,7 @@ void MatchableInfo::tokenizeAsmString(const AsmMatcherInfo &Info,
     char Char = String[i];
     if (Variant.BreakCharacters.contains(Char)) {
       if (InTok) {
-        addAsmOperand(String.slice(Prev, i), false);
+        addAsmOperand(String.substr(Prev, i - Prev), false);
         Prev = i;
         IsIsolatedToken = false;
       }
@@ -1010,7 +1010,7 @@ void MatchableInfo::tokenizeAsmString(const AsmMatcherInfo &Info,
     }
     if (Variant.TokenizingCharacters.contains(Char)) {
       if (InTok) {
-        addAsmOperand(String.slice(Prev, i), IsIsolatedToken);
+        addAsmOperand(String.substr(Prev, i - Prev), IsIsolatedToken);
         InTok = false;
         IsIsolatedToken = false;
       }
@@ -1021,7 +1021,7 @@ void MatchableInfo::tokenizeAsmString(const AsmMatcherInfo &Info,
     }
     if (Variant.SeparatorCharacters.contains(Char)) {
       if (InTok) {
-        addAsmOperand(String.slice(Prev, i), IsIsolatedToken);
+        addAsmOperand(String.substr(Prev, i - Prev), IsIsolatedToken);
         InTok = false;
       }
       Prev = i + 1;
@@ -1032,7 +1032,7 @@ void MatchableInfo::tokenizeAsmString(const AsmMatcherInfo &Info,
     switch (Char) {
     case '\\':
       if (InTok) {
-        addAsmOperand(String.slice(Prev, i), false);
+        addAsmOperand(String.substr(Prev, i - Prev), false);
         InTok = false;
         IsIsolatedToken = false;
       }
@@ -1045,7 +1045,7 @@ void MatchableInfo::tokenizeAsmString(const AsmMatcherInfo &Info,
 
     case '$': {
       if (InTok) {
-        addAsmOperand(String.slice(Prev, i), IsIsolatedToken);
+        addAsmOperand(String.substr(Prev, i - Prev), IsIsolatedToken);
         InTok = false;
         IsIsolatedToken = false;
       }
@@ -1059,7 +1059,7 @@ void MatchableInfo::tokenizeAsmString(const AsmMatcherInfo &Info,
       size_t EndPos = String.find('}', i);
       assert(EndPos != StringRef::npos &&
              "Missing brace in operand reference!");
-      addAsmOperand(String.slice(i, EndPos + 1), IsIsolatedToken);
+      addAsmOperand(String.substr(i, EndPos + 1 - i), IsIsolatedToken);
       Prev = EndPos + 1;
       i = EndPos;
       IsIsolatedToken = false;
diff --git a/llvm/utils/TableGen/Common/CodeGenInstruction.cpp b/llvm/utils/TableGen/Common/CodeGenInstruction.cpp
index 1c0ab594d931..f72fe4c6fd56 100644
--- a/llvm/utils/TableGen/Common/CodeGenInstruction.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenInstruction.cpp
@@ -566,7 +566,8 @@ std::string CodeGenInstruction::FlattenAsmStringVariants(StringRef Cur,
     }
 
     // Select the Nth variant (or empty).
-    StringRef Selection = Cur.slice(VariantsStart, VariantsEnd);
+    StringRef Selection =
+        Cur.substr(VariantsStart, VariantsEnd - VariantsStart);
     for (unsigned i = 0; i != Variant; ++i)
       Selection = Selection.split('|').second;
     Res += Selection.split('|').first;
-- 
GitLab


From f8d1ffd0acdc3495bd0c3afa6f632a7d065d46b2 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Wed, 30 Oct 2024 22:30:33 -0700
Subject: [PATCH 691/693] [NFC] Remove references to deleted
 `ClangFormattedStatus` file (#114331)

---
 clang/docs/ClangFormat.rst | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/clang/docs/ClangFormat.rst b/clang/docs/ClangFormat.rst
index dbd9c91ae508..7afad5b15b2d 100644
--- a/clang/docs/ClangFormat.rst
+++ b/clang/docs/ClangFormat.rst
@@ -363,8 +363,3 @@ those as well).
 
 These commands use the file paths shown in the diff output
 so they will only work from the root of the repository.
-
-Current State of Clang Format for LLVM
-======================================
-
-The following table :doc:`ClangFormattedStatus` shows the current status of clang-formatting for the entire LLVM source tree.
-- 
GitLab


From a8575c14596a52d2edfb34b307009c58099f3973 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang@sifive.com>
Date: Thu, 31 Oct 2024 13:35:37 +0800
Subject: [PATCH 692/693] [RISCV] Sink ordered reduction check into FAdd. NFC
 (#114180)

---
 llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index f050fb569946..807026845683 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1564,13 +1564,6 @@ RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
   }
 
   // IR Reduction is composed by two vmv and one rvv reduction instruction.
-  if (TTI::requiresOrderedReduction(FMF)) {
-    Opcodes.push_back(RISCV::VFMV_S_F);
-    for (unsigned i = 0; i < LT.first.getValue(); i++)
-      Opcodes.push_back(RISCV::VFREDOSUM_VS);
-    Opcodes.push_back(RISCV::VFMV_F_S);
-    return getRISCVInstructionCost(Opcodes, LT.second, CostKind);
-  }
   unsigned SplitOp;
   switch (ISD) {
   case ISD::ADD:
@@ -1590,6 +1583,13 @@ RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
     Opcodes = {RISCV::VMV_S_X, RISCV::VREDAND_VS, RISCV::VMV_X_S};
     break;
   case ISD::FADD:
+    if (TTI::requiresOrderedReduction(FMF)) {
+      Opcodes.push_back(RISCV::VFMV_S_F);
+      for (unsigned i = 0; i < LT.first.getValue(); i++)
+        Opcodes.push_back(RISCV::VFREDOSUM_VS);
+      Opcodes.push_back(RISCV::VFMV_F_S);
+      return getRISCVInstructionCost(Opcodes, LT.second, CostKind);
+    }
     // We can't promote f16/bf16 fadd reductions.
     if ((LT.second.getVectorElementType() == MVT::f16 &&
          !ST->hasVInstructionsF16()) ||
-- 
GitLab


From fdc78120bde1426b223196bec645b6f27fc60f0e Mon Sep 17 00:00:00 2001
From: Longsheng Mou <longshengmou@gmail.com>
Date: Thu, 31 Oct 2024 14:08:54 +0800
Subject: [PATCH 693/693] [mlir][docs] Fix typo in bufferization
 documentation(NFC) (#114342)

---
 .../include/mlir/Dialect/Bufferization/IR/BufferizationOps.td | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
index 1c70a4b8df92..7bcc3b9e7998 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
@@ -1,4 +1,4 @@
-//===- BufferizationOps.td - Bufferization op definitions ----------*- tablegen -*-===//
+//===- BufferizationOps.td - Bufferization op definitions --*- tablegen -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -602,7 +602,7 @@ def Bufferization_DeallocOp : Bufferization_Op<"dealloc", [
     ```
     Deallocation will be called on `%a0` if `%cond0` is 'true' and neither
     `%r0`, `%r1`, or `%r2` are aliases of `%a0`. `%a1` will be deallocated when
-    `%cond1` is set to 'true' and none of `%r0`, %r1`, `%r2`, and `%a0` are
+    `%cond1` is set to 'true' and none of `%r0`, `%r1`, `%r2`, and `%a0` are
     aliases.
 
     Note that this can be an expensive operation if there are many operands that
-- 
GitLab